From 21a9efc4cddbce661073544db31a63639686310a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 28 Nov 2017 05:28:49 -0800
Subject: [PATCH 0001/1554] Add complex dtypes support for
 `tf.squared_difference`

This fix tries to address the issue raised in 14932 where
complex dtypes are not supported for `tf.squared_difference`,
which is different from the doc string in `math_ops.cc`
(see `BINARY_FEWER`).

This fix adds the complex64 and complex128 support in kernel,
and adds additional test cases.

This fix fixes 14932.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/cwise_op_squared_difference.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_op_squared_difference.cc b/tensorflow/core/kernels/cwise_op_squared_difference.cc
index 78fefc69c7..d0ff271df6 100644
--- a/tensorflow/core/kernels/cwise_op_squared_difference.cc
+++ b/tensorflow/core/kernels/cwise_op_squared_difference.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER5(BinaryOp, CPU, "SquaredDifference", functor::squared_difference,
-          float, Eigen::half, double, int32, int64);
+REGISTER7(BinaryOp, CPU, "SquaredDifference", functor::squared_difference,
+          float, Eigen::half, double, int32, int64, complex64, complex128);
 #if GOOGLE_CUDA
 REGISTER4(BinaryOp, GPU, "SquaredDifference", functor::squared_difference,
           float, Eigen::half, double, int64);
-- 
GitLab


From 4f5e66aca388ee13e925d173a82644eed9d5a760 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 28 Nov 2017 05:32:08 -0800
Subject: [PATCH 0002/1554] Add test cases for complex dtypes support with
 `tf.squared_difference`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/math_ops_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index d314124ccd..7078ac99c8 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -203,7 +203,9 @@ class SquaredDifferenceTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes()
   def testSquaredDifference(self):
-    for dtype in [np.int32, np.float16]:
+    for dtype in [np.float16, np.float32, np.float64,
+                  np.int32, np.int64,
+                  np.complex64, np.complex128]:
       x = np.array([[1, 2, 3], [4, 5, 6]], dtype=dtype)
       y = np.array([-3, -2, -1], dtype=dtype)
       z = (x - y) * (x - y)
-- 
GitLab


From f7bb3741549e791f687fa8289fb281717eae7426 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 24 Feb 2018 19:05:52 +0000
Subject: [PATCH 0003/1554] Add additional test to cover squared difference for
 complex

where imag parts are not 0.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/math_ops_test.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index 7078ac99c8..3224e40db2 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -213,6 +213,16 @@ class SquaredDifferenceTest(test_util.TensorFlowTestCase):
         z_tf = self.evaluate(math_ops.squared_difference(x, y))
         self.assertAllClose(z, z_tf)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testComplexSquaredDifference(self):
+    for dtype in [np.complex64, np.complex128]:
+      x = np.array([[1+3j, 2+2j, 3+1j], [4-1j, 5-2j, 6-3j]], dtype=dtype)
+      y = np.array([-3+1j, -2+2j, -1+3j], dtype=dtype)
+      z = (x - y) * (x - y)
+      with test_util.device(use_gpu=True):
+        z_tf = self.evaluate(math_ops.squared_difference(x, y))
+        self.assertAllClose(z, z_tf)
+
 
 @test_util.with_c_api
 class ApproximateEqualTest(test_util.TensorFlowTestCase):
-- 
GitLab


From ab5cdb187d96e3a865724c3d41671dd253288456 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 24 Feb 2018 21:22:26 +0000
Subject: [PATCH 0004/1554] Enable squared_difference complex on CPU only

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/math_ops_test.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index 3224e40db2..533a00e737 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -204,8 +204,7 @@ class SquaredDifferenceTest(test_util.TensorFlowTestCase):
   @test_util.run_in_graph_and_eager_modes()
   def testSquaredDifference(self):
     for dtype in [np.float16, np.float32, np.float64,
-                  np.int32, np.int64,
-                  np.complex64, np.complex128]:
+                  np.int32, np.int64]:
       x = np.array([[1, 2, 3], [4, 5, 6]], dtype=dtype)
       y = np.array([-3, -2, -1], dtype=dtype)
       z = (x - y) * (x - y)
@@ -219,7 +218,7 @@ class SquaredDifferenceTest(test_util.TensorFlowTestCase):
       x = np.array([[1+3j, 2+2j, 3+1j], [4-1j, 5-2j, 6-3j]], dtype=dtype)
       y = np.array([-3+1j, -2+2j, -1+3j], dtype=dtype)
       z = (x - y) * (x - y)
-      with test_util.device(use_gpu=True):
+      with test_util.device(use_gpu=False):
         z_tf = self.evaluate(math_ops.squared_difference(x, y))
         self.assertAllClose(z, z_tf)
 
-- 
GitLab


From 7950d197767ef24a1525b809a310b82020f665ba Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Wed, 1 Aug 2018 13:43:35 -0700
Subject: [PATCH 0005/1554] MKL DNN: Adding support of fusing Pad and Conv2D in
 MKL DNN optimized code

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 298 +++++++++++++++++-
 tensorflow/core/graph/mkl_layout_pass_test.cc |  70 ++++
 tensorflow/core/kernels/BUILD                 |  28 ++
 tensorflow/core/kernels/mkl_conv_ops.cc       |  96 +++++-
 tensorflow/core/kernels/mkl_conv_ops.h        |  28 +-
 tensorflow/core/ops/nn_ops.cc                 |  49 +++
 6 files changed, 554 insertions(+), 15 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index c22e0a3872..d0abe5da35 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2432,6 +2432,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     csinfo_.mkl_conv2d_with_bias = "_MklConv2DWithBias";
     csinfo_.mkl_conv2d_grad_filter_with_bias =
         "_MklConv2DBackpropFilterWithBias";
+    csinfo_.mkl_pad_with_conv2d = "_MklPadWithConv2D";
+    csinfo_.pad = "Pad";
+    csinfo_.pad_with_conv2d = "__MklDummyPadWithConv2D";
     csinfo_.relu = "Relu";
     csinfo_.relu_grad = "ReluGrad";
     csinfo_.tanh = "Tanh";
@@ -2508,6 +2511,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     rinfo_.push_back({csinfo_.mul,
                       mkl_op_registry::GetMklOpName(csinfo_.mul),
                       CopyAttrsDataType, AlwaysRewrite});
+    rinfo_.push_back({csinfo_.pad_with_conv2d, csinfo_.mkl_pad_with_conv2d,
+                      CopyAttrsPadWithConv2D, AlwaysRewrite});
     rinfo_.push_back({csinfo_.relu, mkl_op_registry::GetMklOpName(csinfo_.relu),
                       CopyAttrsDataType, AlwaysRewrite});
     rinfo_.push_back({csinfo_.relu_grad,
@@ -2546,6 +2551,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     minfo_.push_back({csinfo_.conv2d_grad_filter, csinfo_.bias_add_grad,
                       csinfo_.conv2d_grad_filter_with_bias,
                       GetConv2DBackpropFilterOrBiasAddGrad});
+    minfo_.push_back({csinfo_.pad, csinfo_.conv2d,
+                      csinfo_.pad_with_conv2d, GetPadOrConv2D});
+    //TODO : Need to check if pad is with zero or not
+    // if is zero then replace, if not then do not replace
   }
 
   // Standard interface to run pass
@@ -2628,7 +2637,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string mkl_conv2d_grad_filter;
     string mkl_conv2d_grad_filter_with_bias;
     string mkl_conv2d_with_bias;
+    string mkl_pad_with_conv2d;
     string mul;
+    string pad;
+    string pad_with_conv2d;
     string relu;
     string relu_grad;
     string tanh;
@@ -2734,6 +2746,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
   // Helper function to merge different nodes
   Status MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g, Node* m, Node* n);
+  Status MergePadWithConv2D(std::unique_ptr<Graph>* g, Node* m, Node* n);
   Status MergeConv2DBackpropFilterWithBiasAddGrad(std::unique_ptr<Graph>* g,
                                                   Node* m, Node* n);
 
@@ -2771,6 +2784,59 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return n;
   }
 
+  // Find Pad or Conv2D node that can be merged with input node 'm'.
+  // If input 'm' is Pad, then check if there exists Conv2D node that can be
+  // merged with 'm'. If input 'm' is Conv2D, then check if there exists BiasAdd
+  // node that can be merged with 'm'.
+  static Node* GetPadOrConv2D(const Node* m) {
+    CHECK_NOTNULL(m);
+    Node* n = nullptr;
+
+    if (m->type_string() == csinfo_.pad) {
+      // If m is Pad, then Conv2D is the output of Pad.
+      for (const Edge* e : m->out_edges()) {
+        if (!e->IsControlEdge() &&
+            e->dst()->type_string() == csinfo_.conv2d) {
+          n = e->dst();
+          break;
+        }
+      }
+    } else {
+      CHECK_EQ(m->type_string(), csinfo_.conv2d);
+      // If m is conv2D, Go over all input edges
+      // and search for Pad  Node.
+      for (const Edge* e : m->in_edges()) {
+        if (!e->IsControlEdge() &&
+            e->src()->type_string() == csinfo_.pad) {
+          n = e->src();
+          break;
+        }
+      }
+    }
+    // Check if only VALID type of padding is used
+    // or not.
+    if (n != nullptr) {
+      const Node* conv_node;
+      if (m->type_string() == csinfo_.conv2d)
+        conv_node = m;
+      else
+        conv_node = n;
+      string padding;
+      TF_CHECK_OK(GetNodeAttr(conv_node->def(), "padding", &padding));
+      if (padding != "VALID")
+        // Then do not merge.
+        // Only VALID type of padding in conv op can be
+        // merged with Pad op.
+        n = nullptr;
+    }
+    if (n == nullptr) {
+      VLOG(1) << "MklLayoutRewritePass: Could not find matching "
+              << "Pad and Conv2D node for merging. Input node: "
+              << m->DebugString();
+    }
+
+    return n;
+  }
   // Find Conv2DBackpropFilter or BiasAddGrad node that can be merged with input
   // node 'm'. If input 'm' is Conv2DBackpropFilter, then check if there exists
   // BiasAddGrad node that can be merged with 'm'. If input 'm' is BiasAddGrad,
@@ -3090,6 +3156,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   static void CopyAttrsDataType(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb);
+  static void CopyAttrsPadWithConv2D(const Node* orig_node, NodeBuilder* nb);
+  static void CopyAttrsFromPadAndConv2D(const Node* orig_node1, const Node* orig_node2, 
+                                        NodeBuilder* nb);
   static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsReshape(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsSplit(const Node* orig_node, NodeBuilder* nb);
@@ -3289,6 +3358,8 @@ int MklLayoutRewritePass::SetUpContiguousInputs(
     // 2nd input (slot 1) of _MklConv2D and _MklConv2DWithBias.
     for (const Edge* e : filter_node->out_edges()) {
       if ((e->dst()->type_string() == csinfo_.mkl_conv2d ||
+           // add check for mkl_pad_with_conv2d  
+           e->dst()->type_string() == csinfo_.mkl_pad_with_conv2d ||
            e->dst()->type_string() == csinfo_.mkl_conv2d_with_bias) &&
           e->dst_input() == kConv2DFilterInputSlotIdx
           /* filter is 2nd input of Conv2D and _MklConv2D. */) {
@@ -3598,6 +3669,65 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
 }
 
+//used in rinfo when replacing __MklDummyPadWithConv2D by _MklPadWithConv2D
+void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
+          NodeBuilder* nb) {
+  DataType Tpaddings;
+  DataType T;
+  string data_format;
+  string padding;
+  std::vector<int32> strides;
+  bool use_cudnn_on_gpu;
+
+  // Get all attributes from old node 1.
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
+  TF_CHECK_OK(
+      GetNodeAttr(orig_node->def(), "use_cudnn_on_gpu", &use_cudnn_on_gpu));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tpaddings", &Tpaddings));
+
+  // Add attributes to new node.
+  nb->Attr("T", T);
+  nb->Attr("strides", strides);
+  nb->Attr("padding", padding);
+  nb->Attr("data_format", data_format);
+  nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
+  nb->Attr("Tpaddings", Tpaddings);
+}
+
+//used with MergePadWithConv2D
+void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
+          const Node* orig_node2, NodeBuilder* nb) {
+  DataType Tpaddings;
+  DataType T;
+  string data_format;
+  string padding;
+  std::vector<int32> strides;
+  bool use_cudnn_on_gpu;
+
+  // Get all attributes from old node 1.
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "T", &T));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "padding", &padding));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "data_format", &data_format));
+  TF_CHECK_OK(
+      GetNodeAttr(orig_node1->def(), "use_cudnn_on_gpu", &use_cudnn_on_gpu));
+  // Get all attributes from old node 2.
+  TF_CHECK_OK(GetNodeAttr(orig_node2->def(), "Tpaddings", &Tpaddings));
+
+  // Add attributes to new node.
+  nb->Attr("T", T);
+  nb->Attr("strides", strides);
+  nb->Attr("padding", padding);
+  nb->Attr("data_format", data_format);
+  nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
+
+
+  // Add attributes to new node.
+  nb->Attr("Tpaddings", Tpaddings);
+}
 void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node,
                                          NodeBuilder* nb) {
   DataType T;
@@ -3824,7 +3954,7 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   // If 'm' is BiasAdd, then 'n' is Conv2D. Since Conv2D feeds BiasAdd,
   // BiasAdd is successor node, and Conv2D predecessor node.
   Node* pred = m->type_string() == csinfo_.bias_add ? n : m;
-  Node* succ = m->type_string() == csinfo_.bias_add ? m : n;
+  Node* succ = m->type_string() == csinfo_.bias_add ? m : n; 
 
   // 1. Get all attributes from input nodes.
   DataType T_pred, T_succ;
@@ -3963,6 +4093,161 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   return Status::OK();
 }
 
+Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
+                                                    Node* m, Node* n) {
+  CHECK_EQ(((m->type_string() == csinfo_.pad &&
+             n->type_string() == csinfo_.conv2d)) ||
+               ((n->type_string() == csinfo_.pad &&
+                 m->type_string() == csinfo_.conv2d)),
+           true);
+      
+  // Conv2D is successor node, and Pad predecessor node.
+  Node* pred = m->type_string() == csinfo_.pad ? m : n;
+  Node* succ = m->type_string() == csinfo_.pad ? n : m;
+
+  // 1. Get all attributes from input nodes.
+  DataType T_pred, T_succ;
+  string padding;
+  std::vector<int32> strides;
+  std::vector<int32> dilations;
+  string data_format_pred, data_format_succ;
+  bool use_cudnn_on_gnu;
+  TF_CHECK_OK(GetNodeAttr(pred->def(), "T", &T_pred));
+  TF_CHECK_OK(GetNodeAttr(succ->def(), "T", &T_succ));
+  TF_CHECK_OK(GetNodeAttr(succ->def(), "padding", &padding));
+  TF_CHECK_OK(GetNodeAttr(succ->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(succ->def(), "dilations", &dilations));
+  // data format for pad is not available and not necessary, thus
+  // we dont need to match data format
+  // TF_CHECK_OK(GetNodeAttr(pred->def(), "data_format", &data_format_pred));
+  TF_CHECK_OK(GetNodeAttr(succ->def(), "data_format", &data_format_succ));
+  TF_CHECK_OK(GetNodeAttr(succ->def(), "use_cudnn_on_gpu", &use_cudnn_on_gnu));
+  // We check to ensure that data formats of both succ and pred are same.
+  // We expect them to be same, so we can enforce this as assert.
+  // But assert can be too strict, so we enforce this as a check.
+  // If the check fails, then we do not merge two nodes.
+  // We also do same check for devices.
+  // if (data_format_pred != data_format_succ || T_pred != T_succ ||
+  if (T_pred != T_succ ||
+      pred->assigned_device_name() != succ->assigned_device_name() ||
+      pred->def().device() != succ->def().device()) {
+    return Status(error::Code::INVALID_ARGUMENT,
+                  "data_format or T attribute or devices of Conv2D and "
+                  "Pad do not match. Will skip node merge optimization");
+  }
+
+  const int succ_num = succ->num_inputs();
+  gtl::InlinedVector<Node*, 4> succ_control_edges;
+  gtl::InlinedVector<std::pair<Node*, int>, 4> succ_in(succ_num);
+  FillInputs(succ, &succ_control_edges, &succ_in);
+
+  const int pred_num = pred->num_inputs();
+  gtl::InlinedVector<Node*, 4> pred_control_edges;
+  gtl::InlinedVector<std::pair<Node*, int>, 4> pred_in(pred_num);
+  FillInputs(pred, &pred_control_edges, &pred_in);
+
+  // We need to ensure that Pad only feeds to Conv2D (some other operator is
+  // not expecting output of Pad). If this is not the case, then we cannot
+  // merge Conv2D with Pad.
+  const int kFirstOutputSlot = 0;
+  for (const Edge* e : pred->out_edges()) {
+    if (e->src_output() == kFirstOutputSlot && e->dst() != succ) {
+      return Status(error::Code::INVALID_ARGUMENT,
+                    "Pad does not feed to Conv2D, or "
+                    "it feeds Conv2D but has multiple outputs. "
+                    "Will skip node merge optimization");
+    }
+  }
+
+  // 2. Get inputs from both the nodes. ( ? ? Explanation of the following)
+  // Find the 2 inputs from the Pad and the Filter input from the Conv2D.
+  // Get operand 0, 1 of conv2D.
+  CHECK_EQ(pred->in_edges().size(), 2);  // Pad must have 2 inputs.
+  // Get operand 1 of add_bias???
+  // Conv2D must have 2 inputs: pad output and Filter
+  CHECK_EQ(succ->in_edges().size(), 2);
+
+  // We will use the node name of Conv2D as the name of new node
+  // Build new node. We use same name as original node, but change the op
+  // name.
+  NodeBuilder nb(succ->name(), csinfo_.pad_with_conv2d);
+  nb.Input(pred_in[0].first, pred_in[0].second);  // In1 (input data)  of Pad
+  // pred_in[1] will be 2nd Tensorflow tensor for Conv2D.
+  nb.Input(succ_in[1].first, succ_in[1].second);  // In2 (filter) of conv2d
+  // In1 of Conv2D is same as output of Pad.
+  // Thus, only need to add In2 of Conv2D
+  nb.Input(pred_in[1].first, pred_in[1].second);  // In2 (paddings) of Pad
+
+  // Copy attributes from Pad and conv2D to PadWithConv2D.
+  CopyAttrsFromPadAndConv2D(const_cast<const Node*>(succ), const_cast<const Node*>(pred),
+                            &nb);
+
+  // Copy the device assigned to old node to new node.
+  nb.Device(succ->def().device());
+
+  // Create node.
+  Node* new_node;
+  TF_CHECK_OK(nb.Finalize(&**g, &new_node));
+  CHECK_NOTNULL(new_node);
+
+  // Incoming data edges from 'pred' node and 'succ' node to new 'new_node'
+  // node are already copied in BuildNode. 
+  // We handle control edges now.
+  for (const Edge* e : pred->in_edges()) {
+    if (e->IsControlEdge()) {
+      // Allow duplicate while adding control edge as it would fail (return
+      // NULL) if we try to add duplicate edge.
+      CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node, true));
+    }
+  }
+  for (const Edge* e : succ->in_edges()) {
+    if (e->IsControlEdge()) {
+      // Allow duplicate while adding control edge as it would fail (return
+      // NULL) if we try to add duplicate edge.
+      CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node, true));
+    }
+  }
+
+  // Incoming edges are fixed, we will fix the outgoing edges now.
+  // First, we will fix outgoing control edges from 'pred' node.
+  for (const Edge* e : pred->out_edges()) {
+    if (e->IsControlEdge()) {
+      // Allow duplicate while adding control edge as it would fail (return
+      // NULL) if we try to add duplicate edge.
+      CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst(), true));
+    }
+  }
+
+  // Second, we will fix outgoing control and data edges from 'succ' node.
+  for (const Edge* e : succ->out_edges()) {
+    if (e->IsControlEdge()) {
+      // Allow duplicate while adding control edge as it would fail (return
+      // NULL) if we try to add duplicate edge.
+      CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst(), true));
+    } else {
+      // Conv2D has only 1 output (at slot 0) and merged node also has only 1
+      // output (at slot 0).
+      const int kPadWithConv2DOutputSlot = 0;
+      CHECK_NOTNULL((*g)->AddEdge(new_node, kPadWithConv2DOutputSlot, e->dst(),
+                                  e->dst_input()));
+    }
+  }
+
+  // Copy device assigned to old node to new node.
+  // It's ok to use pred or succ as we have enforced a check that
+  // both have same device assigned.
+  new_node->set_assigned_device_name(pred->assigned_device_name());
+
+  VLOG(1) << "MklLayoutRewritePass: Merged old node:" << pred->DebugString()
+          << ", and node: " << succ->DebugString()
+          << ", into node:" << new_node->DebugString();
+
+  (*g)->RemoveNode(succ);
+  (*g)->RemoveNode(pred);
+
+  return Status::OK();
+}
+
 Status MklLayoutRewritePass::MergeConv2DBackpropFilterWithBiasAddGrad(
     std::unique_ptr<Graph>* g, Node* m, Node* n) {
   CHECK_EQ(((m->type_string() == csinfo_.bias_add_grad &&
@@ -4096,6 +4381,12 @@ Status MklLayoutRewritePass::MergeNode(std::unique_ptr<Graph>* g, Node* m,
         m->type_string() == csinfo_.conv2d))) {
     return this->MergeConv2DWithBiasAdd(g, m, n);
   }
+  if (((m->type_string() == csinfo_.pad &&
+        n->type_string() == csinfo_.conv2d)) ||
+      ((n->type_string() == csinfo_.pad &&
+        m->type_string() == csinfo_.conv2d))) {
+    return this->MergePadWithConv2D(g, m, n);
+  }
 
   if (((m->type_string() == csinfo_.bias_add_grad &&
         n->type_string() == csinfo_.conv2d_grad_filter)) ||
@@ -4207,9 +4498,10 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const {
   }
 
   // We make an exception for __MklDummyConv2DWithBias and
-  // __MklConv2DBackpropFilterWithBias since their names do not match Mkl node
-  // names.
+  // __MklConv2DBackpropFilterWithBias, __MklDummyPadWithConv2D since their names 
+  // do not match Mkl node names.
   if (n->type_string() != csinfo_.conv2d_with_bias &&
+      n->type_string() != csinfo_.pad_with_conv2d && 
       n->type_string() != csinfo_.conv2d_grad_filter_with_bias &&
       !mkl_op_registry::IsMklOp(mkl_op_registry::GetMklOpName(n->type_string()),
                                 T)) {
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index a41f5861af..020e3c9168 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -2012,6 +2012,76 @@ TEST_F(MklLayoutPassTest, Basic) {
             "A->C;A->D;B->C:1;B->D:1");
 }
 
+// Test set 0: Pad + Conv2D; padding is VALID
+// A = input(image), B = input(paddings), C= Pad = input of conv2D, 
+// D=input(filter), E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
+// After layout pass
+// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2) 
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
+            "A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
+            "DMT/_2->E:5;E->Z;Y->Z:1");
+}
+
+// Test set 0: Pad + Conv2D; padding is SAME
+// A = input(image), B = input(paddings), C= Pad = input of conv2D, 
+// D=input(filter), E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
+// After layout pass - No merging 
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Negative) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+           "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/_1(Const);"
+           "E(_MklConv2D);Y(Input);Z(Zeta)|A->C;B->C:1;C->E;"
+           "C:control->DMT/_0:control;C:control->DMT/_1:control;"
+           "D->E:1;DMT/_0->E:2;DMT/_1->E:3;E->Z;Y->Z:1");
+}
+
 // Test set 1: Conv2D + AddBias
 
 // C=Conv2D(A,B); E=BiasAdd(C,D); Z=Zeta(E,Y)
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 6126e8b7ba..f14542068d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -39,6 +39,7 @@ load(
     "cc_header_only_library",
     "if_not_windows",
     "if_override_eigen_strong_inline",
+    "tf_cc_test_mkl",
 )
 load("@local_config_sycl//sycl:build_defs.bzl", "if_sycl")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
@@ -1129,6 +1130,7 @@ tf_cuda_cc_test(
     ],
 )
 
+
 tf_cc_test(
     name = "decode_wav_op_test",
     size = "small",
@@ -6124,6 +6126,7 @@ tf_mkl_kernel_library(
     ] + if_mkl(["@mkl_dnn"]),
 )
 
+
 tf_mkl_kernel_library(
     name = "mkl_tfconv_op",
     prefix = "mkl_tfconv",
@@ -6269,6 +6272,31 @@ tf_mkl_kernel_library(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_fused_ops_test",
+    size = "small",
+    srcs = ["mkl_fused_ops_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":mkl_conv_op",
+        ":mkl_tfconv_op",
+        ":conv_ops",
+        ":image",
+        ":ops_testutil",
+        ":ops_util",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ]
+)
+
 # NOTE(lespeholt): This rule is deprecated, please use:
 # tensorflow/core/util/batch_util.h
 cc_library(
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 62396eeb8b..d4ec831cf2 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -753,9 +753,31 @@ class MklConv2DOp : public OpKernel {
   TensorFormat data_format_;
 };
 
+
+#define REGISTER_MKL_CPU(T)                                         \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv2D")                        \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .Label(mkl_op_registry::kMklOpLabel), \
+                          MklConv2DOp<CPUDevice, T, false>);        \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias")                \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .Label(mkl_op_registry::kMklOpLabel), \
+                          MklConv2DOp<CPUDevice, T, true>);         \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")          \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .Label(mkl_op_registry::kMklOpLabel), \
+                          MklDummyOp<CPUDevice, T>);
+
+TF_CALL_float(REGISTER_MKL_CPU);
+#undef REGISTER_MKL_CPU
+
 #else
 
-template <typename Device, typename T, bool biasEnabled>
+template <typename Device, typename T, typename Tpadding, bool biasEnabled,
+          bool padEnabled>
 class MklConv2DOp : public OpKernel {
  public:
   ~MklConv2DOp() {}
@@ -814,6 +836,11 @@ class MklConv2DOp : public OpKernel {
                    dilations, strides;
       memory::dims dst_dims_tf_order, dst_dims_mkl_order;
 
+      // If pad with conv2d fusion is enabled
+      if (padEnabled) {
+        PadWithConvFusion(context, padding_left, padding_right);
+      }
+
       // Get shapes of input tensors in MKL-DNN order
       MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
                              dilations_);
@@ -822,7 +849,7 @@ class MklConv2DOp : public OpKernel {
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &src_dims, &filter_dims,
           &strides, &dilations, &dst_dims_tf_order, &dst_dims_mkl_order,
-          &padding_left, &padding_right);
+          &padding_left, &padding_right, padEnabled);
       if (!context->status().ok()) return;
 
       // Check for corner case - if there is nothing to compute, return.
@@ -869,7 +896,6 @@ class MklConv2DOp : public OpKernel {
       // MKLDNN dilation starts from 0.
       dilations[kDilationH] -= 1;
       dilations[kDilationW] -= 1;
-
       // get a conv2d fwd from primitive pool
       MklConv2DFwdPrimitive<T>* conv2d_fwd = nullptr;
       if (biasEnabled) {
@@ -937,13 +963,53 @@ class MklConv2DOp : public OpKernel {
         errors::Aborted("Operation received an exception:", error_msg));
     }
   }
+  
+  void PadWithConvFusion(OpKernelContext* context, memory::dims &padding_left,
+                         memory::dims &padding_right){
+    const Tensor& paddings_tf = MklGetInput(context, 2);
+    OP_REQUIRES(context, paddings_tf.dims() == 2,
+                errors::InvalidArgument("paddings must be 2-dimensional: ",
+                                        paddings_tf.shape().DebugString()));
+    Tpadding* paddings = nullptr;
+    // To get individual pad, need to flatten the tensor
+    paddings = static_cast<Tpadding*>(const_cast<Tpadding*>
+                                     (paddings_tf.flat<Tpadding>().data())); 
+    // For NHWC format:
+    // paddings[0], paddings[1], paddings[6], paddings[7] should be zero 
+    // if the paddings_tf is [ [0, 0] [1,2] [3,4] [0,0] ]
+    // paddings = {0, 0, 1, 2, 3, 4, 0, 0} ; flat method is row major
+    // then, values are: top = 1, bottom =2, left=3, right=4
+    // For NCHW format, 
+    // paddings[0], paddings[1], paddings[2], paddings[3] should be zero 
+    // similar explanation as NHWC format will apply.
+    string data_format = ToString(data_format_);
+    if(data_format == "NHWC"){
+      pad_top = paddings[2]; 
+      pad_bottom = paddings[3]; 
+      pad_left = paddings[4]; 
+      pad_right = paddings[5]; 
+    }
+    else if (data_format == "NCHW"){
+      pad_top = paddings[4]; 
+      pad_bottom = paddings[5]; 
+      pad_left = paddings[6]; 
+      pad_right = paddings[7]; 
+    }
+    // Create padding arrays for MKL DNN convolutions.
+    // MKL-DNN uses asymetric padding.
+    padding_left = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
+    padding_right = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
+  }
 
  private:
   std::vector<int32> strides_;
   std::vector<int32> dilations_;
+  int64 pad_top, pad_left; 
+  int64 pad_bottom, pad_right; 
   Padding padding_;
   TensorFormat data_format_;
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
+  const int kInputIndex_Pad = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
   const int kDilationH = 0, kDilationW = 1;
   engine cpu_engine = engine(engine::cpu, 0);
@@ -1036,26 +1102,44 @@ class MklConv2DOp : public OpKernel {
   }
 };
 
-#endif
 
 #define REGISTER_MKL_CPU(T)                                         \
   REGISTER_KERNEL_BUILDER(Name("_MklConv2D")                        \
                               .Device(DEVICE_CPU)                   \
                               .TypeConstraint<T>("T")               \
                               .Label(mkl_op_registry::kMklOpLabel), \
-                          MklConv2DOp<CPUDevice, T, false>);        \
+                          MklConv2DOp<CPUDevice, T, int32, false, false>); \
   REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias")                \
                               .Device(DEVICE_CPU)                   \
                               .TypeConstraint<T>("T")               \
                               .Label(mkl_op_registry::kMklOpLabel), \
-                          MklConv2DOp<CPUDevice, T, true>);         \
+                          MklConv2DOp<CPUDevice, T, int32, true, false>);  \
   REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")          \
                               .Device(DEVICE_CPU)                   \
                               .TypeConstraint<T>("T")               \
                               .Label(mkl_op_registry::kMklOpLabel), \
+                          MklDummyOp<CPUDevice, T>);                \
+  REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                 \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .TypeConstraint<int32>("Tpaddings")   \
+                              .Label(mkl_op_registry::kMklOpLabel), \
+                   MklConv2DOp<CPUDevice, T, int32, false, true>);  \
+  REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                 \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .TypeConstraint<int64>("Tpaddings")   \
+                              .Label(mkl_op_registry::kMklOpLabel), \
+                   MklConv2DOp<CPUDevice, T, int64, false, true>);  \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyPadWithConv2D")           \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .TypeConstraint<int32>("Tpaddings")   \
+                              .Label(mkl_op_registry::kMklOpLabel), \
                           MklDummyOp<CPUDevice, T>);
 
 TF_CALL_float(REGISTER_MKL_CPU);
+#endif
 
 }  // namespace tensorflow
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 3f154ff33b..c6487a4512 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -232,7 +232,7 @@ class MklDnnConvUtil {
       const memory::dims& strides, const memory::dims& dilations,
       memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
-      memory::dims* pad_r) {
+      memory::dims* pad_r, bool padEnabled=false) {
     CHECK_NOTNULL(output_dims_tf_order);
     CHECK_NOTNULL(output_dims_mkl_order);
     CHECK_NOTNULL(pad_l);
@@ -268,7 +268,19 @@ class MklDnnConvUtil {
             GetWindowedOutputSizeVerboseV2(input_cols, filter_cols,
                                  dilation_cols, stride_cols, padding_,
                                  &out_cols, &pad_left, &pad_right));
-
+    // If padEnabled, i.e., pad and conv op are fused, then
+    // all pads are already passed from pad op through 
+    // *pad_l and *pad_r 
+    if(padEnabled) { 
+       pad_top = static_cast<int64>((*pad_l)[0]);
+       pad_left = static_cast<int64>((*pad_l)[1]);
+       pad_bottom = static_cast<int64>((*pad_r)[0]);
+       pad_right = static_cast<int64>((*pad_r)[1]);
+       // update the out_rows and out_cols based on all 
+       // sides of the pads coming from pad op. 
+       out_rows = out_rows + (pad_top + pad_bottom ) / stride_rows;  
+       out_cols = out_cols + (pad_left + pad_right ) / stride_cols; 
+    }
     // Tensorflow output is in data_format order. (NHWC or NCHW)
     TensorShape out_shape =
         ShapeFromFormat(data_format_, out_batch, out_rows, out_cols, out_depth);
@@ -283,8 +295,12 @@ class MklDnnConvUtil {
     *output_dims_mkl_order = mkldnn_sizes;
 
     // Now handle padding. MKL-DNN uses asymetric padding.
-    *pad_l = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
-    *pad_r = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
+    // But, if padEnabled, i.e., pad and conv op are fused,
+    // then, *pad_l and *pad_r are already set from pad op
+    if(!padEnabled) {
+      *pad_l = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
+      *pad_r = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
+    }
   }
 
   // Calculate output and pad size of forward Convolution operator.
@@ -325,7 +341,7 @@ class MklDnnConvUtil {
       memory::dims* strides, memory::dims *dilations,
       memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
-      memory::dims* pad_r) {
+      memory::dims* pad_r, bool padEnabled=false) {
     CHECK_NOTNULL(input_dims);
     CHECK_NOTNULL(filter_dims);
     CHECK_NOTNULL(strides);
@@ -344,7 +360,7 @@ class MklDnnConvUtil {
     GetOutputAndPadSizeInMklOrder(input_shape, filter_shape,
                                   *strides, *dilations,
                                   output_dims_tf_order, output_dims_mkl_order,
-                                  pad_l, pad_r);
+                                  pad_l, pad_r, padEnabled);
     if (!context_->status().ok()) return;
   }
 };
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index f947d4c30d..8bb22a8372 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1573,6 +1573,55 @@ NOTE Do not invoke this operator directly in Python. Graph rewrite pass is
 expected to invoke these operators.
 )doc");
 
+REGISTER_OP("__MklDummyPadWithConv2D")
+    .Input("input: T")
+    .Input("filter: T")
+    .Input("paddings: Tpaddings")
+    .Output("output: T")
+    .Attr("T: {half, float, double}")
+    .Attr("strides: list(int)")
+    .Attr("use_cudnn_on_gpu: bool = true")
+    .Attr(GetPaddingAttrString())
+    .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
+    .Attr("Tpaddings: {int32, int64} = DT_INT32")
+    .SetShapeFn(shape_inference::Conv2DShape)
+    .Doc(R"doc(
+Dummy node that enables fusing Pad and Conv2D operator for MKL. This node
+does not perform anything. It is just created as an intermediate output of
+merging Pad and Conv2D.
+
+NOTE Do not invoke this operator directly in Python. Graph rewrite pass is
+expected to invoke these operators.
+)doc");
+
+REGISTER_OP("_MklPadWithConv2D")
+    .Input("input: T")
+    .Input("filter: T")
+    .Input("paddings: Tpaddings")
+    .Input("mkl_input: uint8")
+    .Input("mkl_filter: uint8")
+    .Input("mkl_paddings: uint8")
+    .Output("output: T")
+    .Output("filter_output: T")
+    .Output("mkl_output: uint8")
+    .Output("mkl_filter_output: uint8")
+    .Attr("T: {half, float, double}")
+    .Attr("strides: list(int)")
+    .Attr("use_cudnn_on_gpu: bool = true")
+    .Attr(GetPaddingAttrString())
+    .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
+    .Attr("Tpaddings: {int32, int64} = DT_INT32")
+    .SetShapeFn(shape_inference::Conv2DShape) 
+    .Doc(R"doc(
+MKL version of Pad and Conv2D operator. Uses MKL DNN APIs to perform
+Pad and 2D convolution to the output of convolution.
+
+NOTE Do not invoke this operator directly in Python. Graph rewrite pass is
+expected to invoke these operators.
+)doc");
+
 REGISTER_OP("_MklConv2DBackpropFilter")
     .Input("input: T")
     .Input("filter_sizes: int32")
-- 
GitLab


From dd63093a599081accfe2a2d2ca8c029d413a15d7 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Thu, 2 Aug 2018 08:43:06 -0700
Subject: [PATCH 0006/1554] adding unit test for pad+conv2d fusion op

---
 tensorflow/core/kernels/mkl_fused_ops_test.cc | 164 ++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 tensorflow/core/kernels/mkl_fused_ops_test.cc

diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
new file mode 100644
index 0000000000..216e8d0206
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -0,0 +1,164 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifdef INTEL_MKL
+#ifndef INTEL_MKL_ML  // We don't support fusion in MKL ML
+#include "tensorflow/cc/ops/const_op.h"
+#include "tensorflow/cc/ops/image_ops.h"
+#include "tensorflow/cc/ops/nn_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/conv_ops_gpu.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/public/session.h"
+
+namespace tensorflow {
+
+// Helper class for converting MKL tesnors to TF tensor and comparing to
+// expected values
+
+const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
+const TensorShape dummy_shape({8});
+
+class ConvMklToTF : public OpsTestBase {
+ public:
+  template <typename T>
+  void ConvertAndCompare(DataType dtype, const Tensor& first,
+                         const Tensor& second, const Tensor& expected) {
+    // Create an MKL to TF conversion node and execute it
+    TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf")
+                     .Input(FakeInput(dtype))     // Input
+                     .Input(FakeInput(DT_UINT8))  // Mkl second tensor
+                     .Attr("T", dtype)
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+    AddInputFromArray<T>(first.shape(), first.flat<T>());
+    AddInputFromArray<uint8>(second.shape(), second.flat<uint8>());
+    TF_ASSERT_OK(RunOpKernel());
+
+    const Tensor& output = *GetOutput(0);
+    test::ExpectTensorNear<T>(expected, output, 1e-5);
+  }
+  void TestBody(){};
+};
+
+// Testing fusion of pad and convolution
+
+class FusedPadConvOpTest : public OpsTestBase {
+ public:
+  template <typename T>
+  void Run(DataType dtype, Tensor& image, Tensor& filter, Tensor& padding,
+           Tensor& expected, const string data_format) {
+    const int stride = 1;
+
+    // Create a fused pad+conv2d node
+    TF_EXPECT_OK(NodeDefBuilder("fused_pad_conv_op", "_MklPadWithConv2D")
+                     .Input(FakeInput(dtype))     // Input
+                     .Input(FakeInput(dtype))     // Filter
+                     .Input(FakeInput(DT_INT32))  // Padding
+                     .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                     .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                     .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                     .Attr("padding", "VALID")
+                     .Attr("data_format", data_format)
+                     .Attr("T", dtype)
+                     .Attr("strides", {1, stride, stride, 1})
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+
+    // Setting up inputs and execute
+    AddInputFromArray<T>(image.shape(), image.flat<T>());
+    AddInputFromArray<T>(filter.shape(), filter.flat<T>());
+    AddInputFromArray<int32>(padding.shape(), padding.flat<int32>());
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Compare output to expected results
+    const Tensor& first = *GetOutput(0);
+    const Tensor& second = *GetOutput(2);
+    ConvMklToTF conv_comp;
+    conv_comp.ConvertAndCompare<T>(dtype, first, second, expected);
+  }
+};
+
+TEST_F(FusedPadConvOpTest, PaddingConvTest) {
+  const int depth = 1;
+  const int image_width = 4;
+  const int image_height = 3;
+  const int image_batch_count = 1;
+  Tensor image(DT_FLOAT, {image_batch_count, image_height, image_width, depth});
+  test::FillValues<float>(&image, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+
+  const int filter_size = 3;
+  const int filter_count = 1;
+  Tensor filter(DT_FLOAT, {filter_size, filter_size, depth, filter_count});
+  test::FillValues<float>(&filter, {1, 4, 7, 2, 5, 8, 3, 6, 9});
+
+  const int padding_height = 4;
+  const int padding_width = 2;
+  Tensor padding(DT_INT32, {padding_height, padding_width});
+  test::FillValues<int32>(&padding, {0, 0, 3, 4, 1, 2, 0, 0});
+
+  Tensor expected(DT_FLOAT, TensorShape({1, 8, 5, 1}));
+  test::FillValues<float>(
+      &expected,
+      {0,  0,   0,   0,   0,   24, 42,  60,  33,  12,  105, 150, 183, 95,
+       32, 235, 312, 357, 178, 56, 187, 234, 261, 121, 32,  106, 126, 138,
+       59, 12,  0,   0,   0,   0,  0,   0,   0,   0,   0,   0});
+
+  Run<float>(DT_FLOAT, image, filter, padding, expected, "NHWC");
+}
+
+TEST_F(FusedPadConvOpTest, PaddingConvTestNchw) {
+  const int depth = 1;
+  const int image_width = 4;
+  const int image_height = 3;
+  const int image_batch_count = 1;
+  Tensor image(DT_FLOAT, {image_batch_count, depth, image_height, image_width});
+  test::FillValues<float>(&image, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+
+  const int filter_size = 3;
+  const int filter_count = 1;
+  Tensor filter(DT_FLOAT, {filter_size, filter_size, depth, filter_count});
+  test::FillValues<float>(&filter, {1, 4, 7, 2, 5, 8, 3, 6, 9});
+
+  const int padding_height = 4;
+  const int padding_width = 2;
+  Tensor padding(DT_INT32, {padding_height, padding_width});
+  test::FillValues<int32>(&padding, {0, 0, 0, 0, 3, 4, 1, 2});
+
+  Tensor expected(DT_FLOAT, TensorShape({1, 1, 8, 5}));
+  test::FillValues<float>(
+      &expected,
+      {0,  0,   0,   0,   0,   24, 42,  60,  33,  12,  105, 150, 183, 95,
+       32, 235, 312, 357, 178, 56, 187, 234, 261, 121, 32,  106, 126, 138,
+       59, 12,  0,   0,   0,   0,  0,   0,   0,   0,   0,   0});
+
+  Run<float>(DT_FLOAT, image, filter, padding, expected, "NCHW");
+}
+}  // namespace tensorflow
+#endif  // INTEL_MKL_ML
+#endif  // INTEL_MKL
-- 
GitLab


From fe3bd66099d6e27bf45dabbe7f4cc34276194fc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Wed, 1 Aug 2018 13:37:26 +0800
Subject: [PATCH 0007/1554] ENH: register float64 GPU kernel for Conv3d

---
 tensorflow/core/kernels/conv_ops_3d.cc             | 14 +++++++++++++-
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc       |  5 +++++
 tensorflow/python/kernel_tests/conv_ops_3d_test.py |  4 ++--
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc
index a1eed4e68c..009fd3733a 100644
--- a/tensorflow/core/kernels/conv_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_ops_3d.cc
@@ -525,10 +525,19 @@ namespace functor {
       const GPUDevice& d, typename TTypes<T, 5, int>::ConstTensor in, \
       const std::array<int, 3>& padding_left,                         \
       const std::array<int, 3>& padding_right,                        \
-      typename TTypes<T, 5, int>::Tensor out, TensorFormat format);
+      typename TTypes<T, 5, int>::Tensor out, TensorFormat format);   \
+  template <>                                                         \
+  void NHWCToNCHW<GPUDevice, T, 5>::operator()(                       \
+      const GPUDevice& d, typename TTypes<T, 5>::ConstTensor in,      \
+      typename TTypes<T, 5>::Tensor out);                             \
+  template <>                                                         \
+  void NCHWToNHWC<GPUDevice, T, 5>::operator()(                       \
+      const GPUDevice& d, typename TTypes<T, 5>::ConstTensor in,      \
+      typename TTypes<T, 5>::Tensor out);
 
 DECLARE_GPU_SPEC(Eigen::half);
 DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(double);
 #undef DECLARE_GPU_SPEC
 
 }  // namespace functor
@@ -540,6 +549,9 @@ REGISTER_KERNEL_BUILDER(
 REGISTER_KERNEL_BUILDER(
     Name("Conv3D").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     Conv3DOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("Conv3D").Device(DEVICE_GPU).TypeConstraint<double>("T"),
+    Conv3DOp<GPUDevice, double>);
 #endif  // GOOGLE_CUDA
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index a5fa48f85e..f8520cc307 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -1068,18 +1068,23 @@ template struct functor::PadInput<GPUDevice, float, int, 4>;
 template struct functor::PadInput<GPUDevice, Eigen::half, int, 4>;
 
 // For 3d ops.
+template struct functor::TransformFilter<GPUDevice, double, int, 5>;
 template struct functor::TransformFilter<GPUDevice, float, int, 5>;
 template struct functor::TransformFilter<GPUDevice, Eigen::half, int, 5>;
 
+template struct functor::ReverseTransformFilter<GPUDevice, double, 5>;
 template struct functor::ReverseTransformFilter<GPUDevice, float, 5>;
 template struct functor::ReverseTransformFilter<GPUDevice, Eigen::half, 5>;
 
+template struct functor::NHWCToNCHW<GPUDevice, double, 5>;
 template struct functor::NHWCToNCHW<GPUDevice, float, 5>;
 template struct functor::NHWCToNCHW<GPUDevice, Eigen::half, 5>;
 
+template struct functor::NCHWToNHWC<GPUDevice, double, 5>;
 template struct functor::NCHWToNHWC<GPUDevice, float, 5>;
 template struct functor::NCHWToNHWC<GPUDevice, Eigen::half, 5>;
 
+template struct functor::PadInput<GPUDevice, double, int, 5>;
 template struct functor::PadInput<GPUDevice, float, int, 5>;
 template struct functor::PadInput<GPUDevice, Eigen::half, int, 5>;
 
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index 0b531125f3..7a0f51dfef 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -52,11 +52,11 @@ class Conv3DTest(test.TestCase):
   def _DtypesToTest(self, use_gpu):
     if use_gpu:
       if not test_util.CudaSupportsHalfMatMulAndConv():
-        return [dtypes.float32]
+        return [dtypes.float64, dtypes.float32]
       else:
         # It is important that float32 comes before float16 here,
         # as we will be using its gradients as reference for fp16 gradients.
-        return [dtypes.float32, dtypes.float16]
+        return [dtypes.float64, dtypes.float32, dtypes.float16]
     else:
       return [dtypes.float64, dtypes.float32, dtypes.float16]
 
-- 
GitLab


From 6a788c7084d1fb90c1884365114ae9fd979196fe Mon Sep 17 00:00:00 2001
From: Yuxin Wu <ppwwyyxxc@gmail.com>
Date: Sat, 4 Aug 2018 11:36:45 -0700
Subject: [PATCH 0008/1554] Support empty inputs in some maxpool kernels.
 (#21338)

---
 tensorflow/core/kernels/maxpooling_op_gpu.cu.cc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
index 0c7a236b2f..89ffe6494e 100644
--- a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
@@ -384,6 +384,8 @@ bool MaxPoolForwardNoMask_NCHW_VECT_C::operator()(
     int32* top_data, const Eigen::GpuDevice& d) {
   const int kThreadsPerBlock = 1024;
   const int output_size = batch * channels * pooled_height * pooled_width;
+  if (output_size == 0)
+    return true;
   MaxPoolForwardNoMaskKernel_NCHW_VECT_C<<<
       (output_size + kThreadsPerBlock - 1) / kThreadsPerBlock, kThreadsPerBlock,
       0, d.stream()>>>(output_size, bottom_data, height, width, channels,
@@ -402,6 +404,8 @@ bool MaxPoolForwardWithOptionalArgmax<T>::operator()(
     int64* mask, const Eigen::GpuDevice& d, bool propagate_nans) {
   const int kThreadsPerBlock = 1024;
   const int output_size = batch * channels * pooled_height * pooled_width;
+  if (output_size == 0)
+    return true;
   if (propagate_nans) {
     MaxPoolForwardNHWC<true>
         <<<(output_size + kThreadsPerBlock - 1) / kThreadsPerBlock,
@@ -430,6 +434,8 @@ bool MaxPoolBackwardNoMask<T>::operator()(
   const int kThreadsPerBlock = 1024;
 
   const int bottom_size = batch * channels * height * width;
+  if (bottom_size == 0)
+    return true;
   SetZero<<<(bottom_size + kThreadsPerBlock - 1) / kThreadsPerBlock,
             kThreadsPerBlock, 0, d.stream()>>>(bottom_size, bottom_diff);
 
@@ -449,6 +455,8 @@ bool MaxPoolBackwardWithArgmax<T>::operator()(
     const int64* mask, const int top_offset, const int bottom_offset,
     T* bottom_diff, const Eigen::GpuDevice& d) {
   const int kThreadsPerBlock = 1024;
+  if (input_size == 0)
+    return true;
   SetZero<<<(input_size + kThreadsPerBlock - 1) / kThreadsPerBlock,
             kThreadsPerBlock, 0, d.stream()>>>(input_size, bottom_diff);
   MaxPoolBackward<<<(output_size + kThreadsPerBlock - 1) / kThreadsPerBlock,
@@ -466,6 +474,8 @@ bool MaxPoolGradBackwardNoMask<T>::operator()(
     const int pad_l, const T* top_diff, T* bottom_diff,
     const Eigen::GpuDevice& d) {
   const int num_kernels = batch * channels * pooled_height * pooled_width;
+  if (num_kernels == 0)
+    return true;
   CudaLaunchConfig config = GetCudaLaunchConfig(num_kernels, d);
 
   if (data_format == FORMAT_NHWC) {
@@ -489,6 +499,8 @@ bool MaxPoolGradBackwardWithArgmax<T>::operator()(
     const int output_size, const int input_size, const T* top_diff,
     const int64* mask, const int top_offset, const int bottom_offset,
     T* bottom_diff, const Eigen::GpuDevice& d) {
+  if (input_size == 0)
+    return true;
   CudaLaunchConfig config = GetCudaLaunchConfig(output_size, d);
   MaxPoolGradBackward<<<config.block_count, config.thread_per_block, 0,
                         d.stream()>>>(output_size, top_diff, mask, top_offset,
-- 
GitLab


From 7f94025fe72369117bf32d69156f0bd947402c96 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Fri, 10 Aug 2018 20:00:27 -0700
Subject: [PATCH 0009/1554] Addressing the reviews for fused PAD and Conv2d PR

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 75 +++++++++----------
 tensorflow/core/kernels/BUILD                 | 22 +++++-
 tensorflow/core/kernels/mkl_conv_ops.cc       |  8 +-
 tensorflow/core/kernels/mkl_conv_ops.h        | 12 ++-
 tensorflow/core/kernels/mkl_fused_ops_test.cc |  8 +-
 5 files changed, 69 insertions(+), 56 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index d0abe5da35..1e85b50d99 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2786,50 +2786,48 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
   // Find Pad or Conv2D node that can be merged with input node 'm'.
   // If input 'm' is Pad, then check if there exists Conv2D node that can be
-  // merged with 'm'. If input 'm' is Conv2D, then check if there exists BiasAdd
+  // merged with 'm'. If input 'm' is Conv2D, then check if there exists Pad
   // node that can be merged with 'm'.
   static Node* GetPadOrConv2D(const Node* m) {
     CHECK_NOTNULL(m);
     Node* n = nullptr;
 
+    const Node* conv_node; 
     if (m->type_string() == csinfo_.pad) {
       // If m is Pad, then Conv2D is the output of Pad.
       for (const Edge* e : m->out_edges()) {
         if (!e->IsControlEdge() &&
             e->dst()->type_string() == csinfo_.conv2d) {
           n = e->dst();
+          conv_node = n; 
           break;
         }
       }
     } else {
       CHECK_EQ(m->type_string(), csinfo_.conv2d);
-      // If m is conv2D, Go over all input edges
+      // If m is conv2D, Go over all input edges 
       // and search for Pad  Node.
       for (const Edge* e : m->in_edges()) {
         if (!e->IsControlEdge() &&
             e->src()->type_string() == csinfo_.pad) {
           n = e->src();
+          conv_node = m; 
           break;
         }
       }
     }
-    // Check if only VALID type of padding is used
-    // or not.
+    // Check if only VALID type of padding is used 
+    // or not. 
     if (n != nullptr) {
-      const Node* conv_node;
-      if (m->type_string() == csinfo_.conv2d)
-        conv_node = m;
-      else
-        conv_node = n;
       string padding;
       TF_CHECK_OK(GetNodeAttr(conv_node->def(), "padding", &padding));
-      if (padding != "VALID")
-        // Then do not merge.
-        // Only VALID type of padding in conv op can be
+      if (padding != "VALID") 
+        // Then do not merge. 
+        // Only VALID type of padding in conv op can be 
         // merged with Pad op.
         n = nullptr;
     }
-    if (n == nullptr) {
+    else {
       VLOG(1) << "MklLayoutRewritePass: Could not find matching "
               << "Pad and Conv2D node for merging. Input node: "
               << m->DebugString();
@@ -3669,7 +3667,7 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
 }
 
-//used in rinfo when replacing __MklDummyPadWithConv2D by _MklPadWithConv2D
+// Used in rinfo when replacing __MklDummyPadWithConv2D by _MklPadWithConv2D
 void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
           NodeBuilder* nb) {
   DataType Tpaddings;
@@ -3677,11 +3675,13 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
   string data_format;
   string padding;
   std::vector<int32> strides;
+  std::vector<int32> dilations;
   bool use_cudnn_on_gpu;
 
-  // Get all attributes from old node 1.
+  // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
   TF_CHECK_OK(
@@ -3691,13 +3691,14 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
+  nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
   nb->Attr("data_format", data_format);
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
   nb->Attr("Tpaddings", Tpaddings);
 }
 
-//used with MergePadWithConv2D
+// Used with MergePadWithConv2D
 void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
           const Node* orig_node2, NodeBuilder* nb) {
   DataType Tpaddings;
@@ -3705,11 +3706,13 @@ void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
   string data_format;
   string padding;
   std::vector<int32> strides;
+  std::vector<int32> dilations;
   bool use_cudnn_on_gpu;
 
   // Get all attributes from old node 1.
   TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node1->def(), "data_format", &data_format));
   TF_CHECK_OK(
@@ -3720,12 +3723,10 @@ void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
+  nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
   nb->Attr("data_format", data_format);
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
-
-
-  // Add attributes to new node.
   nb->Attr("Tpaddings", Tpaddings);
 }
 void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node,
@@ -3954,7 +3955,7 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   // If 'm' is BiasAdd, then 'n' is Conv2D. Since Conv2D feeds BiasAdd,
   // BiasAdd is successor node, and Conv2D predecessor node.
   Node* pred = m->type_string() == csinfo_.bias_add ? n : m;
-  Node* succ = m->type_string() == csinfo_.bias_add ? m : n; 
+  Node* succ = m->type_string() == csinfo_.bias_add ? m : n;
 
   // 1. Get all attributes from input nodes.
   DataType T_pred, T_succ;
@@ -4095,11 +4096,10 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 
 Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
                                                     Node* m, Node* n) {
-  CHECK_EQ(((m->type_string() == csinfo_.pad &&
+  CHECK(((m->type_string() == csinfo_.pad &&
              n->type_string() == csinfo_.conv2d)) ||
                ((n->type_string() == csinfo_.pad &&
-                 m->type_string() == csinfo_.conv2d)),
-           true);
+                 m->type_string() == csinfo_.conv2d)));
       
   // Conv2D is successor node, and Pad predecessor node.
   Node* pred = m->type_string() == csinfo_.pad ? m : n;
@@ -4117,22 +4117,18 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   TF_CHECK_OK(GetNodeAttr(succ->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "strides", &strides));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "dilations", &dilations));
-  // data format for pad is not available and not necessary, thus
-  // we dont need to match data format
-  // TF_CHECK_OK(GetNodeAttr(pred->def(), "data_format", &data_format_pred));
+  // Data format for pad is not available and not necessary, thus
+  // dont need to match data format for Pad
   TF_CHECK_OK(GetNodeAttr(succ->def(), "data_format", &data_format_succ));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "use_cudnn_on_gpu", &use_cudnn_on_gnu));
-  // We check to ensure that data formats of both succ and pred are same.
-  // We expect them to be same, so we can enforce this as assert.
-  // But assert can be too strict, so we enforce this as a check.
-  // If the check fails, then we do not merge two nodes.
-  // We also do same check for devices.
-  // if (data_format_pred != data_format_succ || T_pred != T_succ ||
+  // Check if the data types and devices of both succ and pred are the same.
+  // Assert is not used,  because it can be too strict.
+  // Don't need to check for data formats because it is not available in Pad.
   if (T_pred != T_succ ||
       pred->assigned_device_name() != succ->assigned_device_name() ||
       pred->def().device() != succ->def().device()) {
     return Status(error::Code::INVALID_ARGUMENT,
-                  "data_format or T attribute or devices of Conv2D and "
+                  "T attribute or devices of Conv2D and "
                   "Pad do not match. Will skip node merge optimization");
   }
 
@@ -4159,11 +4155,10 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
     }
   }
 
-  // 2. Get inputs from both the nodes. ( ? ? Explanation of the following)
-  // Find the 2 inputs from the Pad and the Filter input from the Conv2D.
-  // Get operand 0, 1 of conv2D.
-  CHECK_EQ(pred->in_edges().size(), 2);  // Pad must have 2 inputs.
-  // Get operand 1 of add_bias???
+  // 2. Get inputs from both the nodes.
+
+  // Pad must have 2 inputs: "input" and paddings.
+  CHECK_EQ(pred->in_edges().size(), 2); 
   // Conv2D must have 2 inputs: pad output and Filter
   CHECK_EQ(succ->in_edges().size(), 2);
 
@@ -4497,8 +4492,8 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const {
     return nullptr;
   }
 
-  // We make an exception for __MklDummyConv2DWithBias and
-  // __MklConv2DBackpropFilterWithBias, __MklDummyPadWithConv2D since their names 
+  // We make an exception for __MklDummyConv2DWithBias,
+  // __MklConv2DBackpropFilterWithBias, and __MklDummyPadWithConv2D since their names 
   // do not match Mkl node names.
   if (n->type_string() != csinfo_.conv2d_with_bias &&
       n->type_string() != csinfo_.pad_with_conv2d && 
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index f14542068d..b057b78ace 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -22,6 +22,7 @@ package_group(
         "//learning/brain/research/sparse_matrix/...",
         "//learning/faster_training/...",
         "//tensorflow/...",
+        "//third_party/car/...",
     ],
 )
 
@@ -783,7 +784,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "quantize_and_dequantize_op",
     prefix = "quantize_and_dequantize_op",
-    deps = ARRAY_DEPS,
+    deps = ARRAY_DEPS + [":cwise_op"],
 )
 
 tf_kernel_library(
@@ -1130,7 +1131,6 @@ tf_cuda_cc_test(
     ],
 )
 
-
 tf_cc_test(
     name = "decode_wav_op_test",
     size = "small",
@@ -2855,6 +2855,8 @@ tf_kernel_library(
     srcs = [] + if_mkl([
         "mkl_batch_matmul_op.cc",
     ]),
+    # <prefix>*impl.h are excluded by default from the CPU build, add explicitly.
+    hdrs = ["batch_matmul_op_impl.h"],
     # Override EIGEN_STRONG_INLINE to inline when --define=override_eigen_strong_inline=true,
     # to avoid long compiling time. See https://github.com/tensorflow/tensorflow/issues/10521
     copts = if_override_eigen_strong_inline(["/DEIGEN_STRONG_INLINE=inline"]),
@@ -3791,7 +3793,7 @@ tf_kernel_library(
         "spacetodepth_op.h",
         "spacetodepth_op_gpu.cu.cc",
     ],
-    visibility = ["//visibility:private"],
+    visibility = [":friends"],
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -4888,6 +4890,7 @@ filegroup(
         "fill_functor.cc",
         "fill_functor.h",
         "function_ops.cc",
+        "function_ops.h",
         "gather_functor.h",
         "gather_nd_op.cc",
         "gather_nd_op.h",
@@ -5379,6 +5382,18 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "android_whole_file_read_ops",
+    srcs = if_android(["whole_file_read_ops.cc"]),
+    copts = tf_copts(),
+    linkopts = ["-ldl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:android_tensorflow_lib_lite",
+    ],
+    alwayslink = 1,
+)
+
 #   Quantization-specific OpKernels
 
 tf_kernel_library(
@@ -6126,7 +6141,6 @@ tf_mkl_kernel_library(
     ] + if_mkl(["@mkl_dnn"]),
 )
 
-
 tf_mkl_kernel_library(
     name = "mkl_tfconv_op",
     prefix = "mkl_tfconv",
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index d4ec831cf2..b5ae312fa5 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -979,9 +979,11 @@ class MklConv2DOp : public OpKernel {
     // if the paddings_tf is [ [0, 0] [1,2] [3,4] [0,0] ]
     // paddings = {0, 0, 1, 2, 3, 4, 0, 0} ; flat method is row major
     // then, values are: top = 1, bottom =2, left=3, right=4
-    // For NCHW format, 
+    // For NCHW format: 
     // paddings[0], paddings[1], paddings[2], paddings[3] should be zero 
     // similar explanation as NHWC format will apply.
+    int64 pad_top, pad_left; 
+    int64 pad_bottom, pad_right; 
     string data_format = ToString(data_format_);
     if(data_format == "NHWC"){
       pad_top = paddings[2]; 
@@ -1004,8 +1006,6 @@ class MklConv2DOp : public OpKernel {
  private:
   std::vector<int32> strides_;
   std::vector<int32> dilations_;
-  int64 pad_top, pad_left; 
-  int64 pad_bottom, pad_right; 
   Padding padding_;
   TensorFormat data_format_;
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
@@ -1139,7 +1139,7 @@ class MklConv2DOp : public OpKernel {
                           MklDummyOp<CPUDevice, T>);
 
 TF_CALL_float(REGISTER_MKL_CPU);
-#endif
+#endif // INTEL_MKL_ML
 
 }  // namespace tensorflow
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index c6487a4512..aae4d767a2 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -281,6 +281,14 @@ class MklDnnConvUtil {
        out_rows = out_rows + (pad_top + pad_bottom ) / stride_rows;  
        out_cols = out_cols + (pad_left + pad_right ) / stride_cols; 
     }
+    // Handle padding. MKL-DNN uses asymetric padding.
+    // But, if padEnabled, i.e., pad and conv op are fused,
+    // then, *pad_l and *pad_r are already set from pad op.
+    // In that case they need not set here.
+    else {
+      *pad_l = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
+      *pad_r = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
+    }
     // Tensorflow output is in data_format order. (NHWC or NCHW)
     TensorShape out_shape =
         ShapeFromFormat(data_format_, out_batch, out_rows, out_cols, out_depth);
@@ -297,10 +305,6 @@ class MklDnnConvUtil {
     // Now handle padding. MKL-DNN uses asymetric padding.
     // But, if padEnabled, i.e., pad and conv op are fused,
     // then, *pad_l and *pad_r are already set from pad op
-    if(!padEnabled) {
-      *pad_l = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
-      *pad_r = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
-    }
   }
 
   // Calculate output and pad size of forward Convolution operator.
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 216e8d0206..e408886861 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -1,4 +1,4 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,11 +33,11 @@ limitations under the License.
 
 namespace tensorflow {
 
-// Helper class for converting MKL tesnors to TF tensor and comparing to
+// Helper class for converting MKL tesnors to TF tensors and comparing to
 // expected values
 
-const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
-const TensorShape dummy_shape({8});
+static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
+static const TensorShape dummy_shape({8});
 
 class ConvMklToTF : public OpsTestBase {
  public:
-- 
GitLab


From 819afabbeda709a94894c894515b62c85d236d50 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 13 Aug 2018 10:44:36 -0700
Subject: [PATCH 0010/1554] modifying the ifdef INTEL_MKL_ML to
 INTEL_MKL_ML_ONLY

---
 tensorflow/core/kernels/mkl_fused_ops_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index e408886861..900325ac91 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #ifdef INTEL_MKL
-#ifndef INTEL_MKL_ML  // We don't support fusion in MKL ML
+#ifndef INTEL_MKL_ML_ONLY  // We don't support fusion in MKL ML
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/image_ops.h"
 #include "tensorflow/cc/ops/nn_ops.h"
@@ -160,5 +160,5 @@ TEST_F(FusedPadConvOpTest, PaddingConvTestNchw) {
   Run<float>(DT_FLOAT, image, filter, padding, expected, "NCHW");
 }
 }  // namespace tensorflow
-#endif  // INTEL_MKL_ML
+#endif  // INTEL_MKL_ML_ONLY
 #endif  // INTEL_MKL
-- 
GitLab


From 6b292c27c7ad09a89c8b75c2505e6472b533a4e1 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 13 Aug 2018 15:03:18 -0700
Subject: [PATCH 0011/1554]  formatted as Clang format for Google code
 compliance, replaced directive INTEL_MKL_ML by INTEL_MKL_ML_ONLY, and merged
 with master

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 69 +++++++++----------
 tensorflow/core/graph/mkl_layout_pass_test.cc | 19 ++---
 tensorflow/core/kernels/mkl_conv_ops.cc       | 47 +++++++------
 tensorflow/core/kernels/mkl_conv_ops.h        | 42 +++++------
 4 files changed, 86 insertions(+), 91 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 65b999b193..84e8ea8f70 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2551,10 +2551,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     minfo_.push_back({csinfo_.conv2d_grad_filter, csinfo_.bias_add_grad,
                       csinfo_.conv2d_grad_filter_with_bias,
                       GetConv2DBackpropFilterOrBiasAddGrad});
-    minfo_.push_back({csinfo_.pad, csinfo_.conv2d,
-                      csinfo_.pad_with_conv2d, GetPadOrConv2D});
-    //TODO : Need to check if pad is with zero or not
-    // if is zero then replace, if not then do not replace
+    minfo_.push_back(
+        {csinfo_.pad, csinfo_.conv2d, csinfo_.pad_with_conv2d, GetPadOrConv2D});
+    // Merge Pad and Conv2d, only if the pad op is "Pad"
+    // Doesn't merge if pad op is "PadV2" or "MirrorPad"
   }
 
   // Standard interface to run pass
@@ -2792,42 +2792,39 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     CHECK_NOTNULL(m);
     Node* n = nullptr;
 
-    const Node* conv_node; 
+    const Node* conv_node;
     if (m->type_string() == csinfo_.pad) {
       // If m is Pad, then Conv2D is the output of Pad.
       for (const Edge* e : m->out_edges()) {
-        if (!e->IsControlEdge() &&
-            e->dst()->type_string() == csinfo_.conv2d) {
+        if (!e->IsControlEdge() && e->dst()->type_string() == csinfo_.conv2d) {
           n = e->dst();
-          conv_node = n; 
+          conv_node = n;
           break;
         }
       }
     } else {
       CHECK_EQ(m->type_string(), csinfo_.conv2d);
-      // If m is conv2D, Go over all input edges 
+      // If m is conv2D, Go over all input edges
       // and search for Pad  Node.
       for (const Edge* e : m->in_edges()) {
-        if (!e->IsControlEdge() &&
-            e->src()->type_string() == csinfo_.pad) {
+        if (!e->IsControlEdge() && e->src()->type_string() == csinfo_.pad) {
           n = e->src();
-          conv_node = m; 
+          conv_node = m;
           break;
         }
       }
     }
-    // Check if only VALID type of padding is used 
-    // or not. 
+    // Check if only VALID type of padding is used
+    // or not.
     if (n != nullptr) {
       string padding;
       TF_CHECK_OK(GetNodeAttr(conv_node->def(), "padding", &padding));
-      if (padding != "VALID") 
-        // Then do not merge. 
-        // Only VALID type of padding in conv op can be 
+      if (padding != "VALID")
+        // Then do not merge.
+        // Only VALID type of padding in conv op can be
         // merged with Pad op.
         n = nullptr;
-    }
-    else {
+    } else {
       VLOG(1) << "MklLayoutRewritePass: Could not find matching "
               << "Pad and Conv2D node for merging. Input node: "
               << m->DebugString();
@@ -3155,7 +3152,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsPadWithConv2D(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsFromPadAndConv2D(const Node* orig_node1, const Node* orig_node2, 
+  static void CopyAttrsFromPadAndConv2D(const Node* orig_node1,
+                                        const Node* orig_node2,
                                         NodeBuilder* nb);
   static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb);
   static void CopyAttrsReshape(const Node* orig_node, NodeBuilder* nb);
@@ -3356,7 +3354,7 @@ int MklLayoutRewritePass::SetUpContiguousInputs(
     // 2nd input (slot 1) of _MklConv2D and _MklConv2DWithBias.
     for (const Edge* e : filter_node->out_edges()) {
       if ((e->dst()->type_string() == csinfo_.mkl_conv2d ||
-           // add check for mkl_pad_with_conv2d  
+           // add check for mkl_pad_with_conv2d
            e->dst()->type_string() == csinfo_.mkl_pad_with_conv2d ||
            e->dst()->type_string() == csinfo_.mkl_conv2d_with_bias) &&
           e->dst_input() == kConv2DFilterInputSlotIdx
@@ -3669,7 +3667,7 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
 
 // Used in rinfo when replacing __MklDummyPadWithConv2D by _MklPadWithConv2D
 void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
-          NodeBuilder* nb) {
+                                                  NodeBuilder* nb) {
   DataType Tpaddings;
   DataType T;
   string data_format;
@@ -3700,7 +3698,8 @@ void MklLayoutRewritePass::CopyAttrsPadWithConv2D(const Node* orig_node,
 
 // Used with MergePadWithConv2D
 void MklLayoutRewritePass::CopyAttrsFromPadAndConv2D(const Node* orig_node1,
-          const Node* orig_node2, NodeBuilder* nb) {
+                                                     const Node* orig_node2,
+                                                     NodeBuilder* nb) {
   DataType Tpaddings;
   DataType T;
   string data_format;
@@ -4095,12 +4094,12 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 }
 
 Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
-                                                    Node* m, Node* n) {
+                                                Node* m, Node* n) {
   CHECK(((m->type_string() == csinfo_.pad &&
-             n->type_string() == csinfo_.conv2d)) ||
-               ((n->type_string() == csinfo_.pad &&
-                 m->type_string() == csinfo_.conv2d)));
-      
+          n->type_string() == csinfo_.conv2d)) ||
+        ((n->type_string() == csinfo_.pad &&
+          m->type_string() == csinfo_.conv2d)));
+
   // Conv2D is successor node, and Pad predecessor node.
   Node* pred = m->type_string() == csinfo_.pad ? m : n;
   Node* succ = m->type_string() == csinfo_.pad ? n : m;
@@ -4158,7 +4157,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // 2. Get inputs from both the nodes.
 
   // Pad must have 2 inputs: "input" and paddings.
-  CHECK_EQ(pred->in_edges().size(), 2); 
+  CHECK_EQ(pred->in_edges().size(), 2);
   // Conv2D must have 2 inputs: pad output and Filter
   CHECK_EQ(succ->in_edges().size(), 2);
 
@@ -4174,8 +4173,8 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   nb.Input(pred_in[1].first, pred_in[1].second);  // In2 (paddings) of Pad
 
   // Copy attributes from Pad and conv2D to PadWithConv2D.
-  CopyAttrsFromPadAndConv2D(const_cast<const Node*>(succ), const_cast<const Node*>(pred),
-                            &nb);
+  CopyAttrsFromPadAndConv2D(const_cast<const Node*>(succ),
+                            const_cast<const Node*>(pred), &nb);
 
   // Copy the device assigned to old node to new node.
   nb.Device(succ->def().device());
@@ -4186,7 +4185,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   CHECK_NOTNULL(new_node);
 
   // Incoming data edges from 'pred' node and 'succ' node to new 'new_node'
-  // node are already copied in BuildNode. 
+  // node are already copied in BuildNode.
   // We handle control edges now.
   for (const Edge* e : pred->in_edges()) {
     if (e->IsControlEdge()) {
@@ -4493,10 +4492,10 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const {
   }
 
   // We make an exception for __MklDummyConv2DWithBias,
-  // __MklConv2DBackpropFilterWithBias, and __MklDummyPadWithConv2D since their names 
-  // do not match Mkl node names.
+  // __MklConv2DBackpropFilterWithBias, and __MklDummyPadWithConv2D since their
+  // names do not match Mkl node names.
   if (n->type_string() != csinfo_.conv2d_with_bias &&
-      n->type_string() != csinfo_.pad_with_conv2d && 
+      n->type_string() != csinfo_.pad_with_conv2d &&
       n->type_string() != csinfo_.conv2d_grad_filter_with_bias &&
       !mkl_op_registry::IsMklOp(mkl_op_registry::GetMklOpName(n->type_string()),
                                 T)) {
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 2925b1bde0..d1b39ceeca 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -2013,11 +2013,11 @@ TEST_F(MklLayoutPassTest, Basic) {
 }
 
 // Test set 0: Pad + Conv2D; padding is VALID
-// A = input(image), B = input(paddings), C= Pad = input of conv2D, 
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
 // D=input(filter), E = Conv2D, Z = Zeta
 // C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
 // After layout pass
-// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2) 
+// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
   CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
@@ -2049,10 +2049,10 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
 }
 
 // Test set 0: Pad + Conv2D; padding is SAME
-// A = input(image), B = input(paddings), C= Pad = input of conv2D, 
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
 // D=input(filter), E = Conv2D, Z = Zeta
 // C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
-// After layout pass - No merging 
+// After layout pass - No merging
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Negative) {
   CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
@@ -2075,11 +2075,12 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Negative) {
       "node { name: 'Z' op: 'Zeta'"
       " attr {key: 'T'                 value { type: DT_FLOAT } }"
       " input: ['E', 'Y']}");
-  EXPECT_EQ(DoMklLayoutOptimizationPass(),
-           "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/_1(Const);"
-           "E(_MklConv2D);Y(Input);Z(Zeta)|A->C;B->C:1;C->E;"
-           "C:control->DMT/_0:control;C:control->DMT/_1:control;"
-           "D->E:1;DMT/_0->E:2;DMT/_1->E:3;E->Z;Y->Z:1");
+  EXPECT_EQ(
+      DoMklLayoutOptimizationPass(),
+      "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/_1(Const);"
+      "E(_MklConv2D);Y(Input);Z(Zeta)|A->C;B->C:1;C->E;"
+      "C:control->DMT/_0:control;C:control->DMT/_1:control;"
+      "D->E:1;DMT/_0->E:2;DMT/_1->E:3;E->Z;Y->Z:1");
 }
 
 // Test set 1: Conv2D + AddBias
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 8c8be197f9..7ee9f66810 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -963,39 +963,38 @@ class MklConv2DOp : public OpKernel {
         errors::Aborted("Operation received an exception:", error_msg));
     }
   }
-  
-  void PadWithConvFusion(OpKernelContext* context, memory::dims &padding_left,
-                         memory::dims &padding_right){
+
+  void PadWithConvFusion(OpKernelContext* context, memory::dims& padding_left,
+                         memory::dims& padding_right) {
     const Tensor& paddings_tf = MklGetInput(context, 2);
     OP_REQUIRES(context, paddings_tf.dims() == 2,
                 errors::InvalidArgument("paddings must be 2-dimensional: ",
                                         paddings_tf.shape().DebugString()));
     Tpadding* paddings = nullptr;
     // To get individual pad, need to flatten the tensor
-    paddings = static_cast<Tpadding*>(const_cast<Tpadding*>
-                                     (paddings_tf.flat<Tpadding>().data())); 
+    paddings = static_cast<Tpadding*>(
+        const_cast<Tpadding*>(paddings_tf.flat<Tpadding>().data()));
     // For NHWC format:
-    // paddings[0], paddings[1], paddings[6], paddings[7] should be zero 
+    // paddings[0], paddings[1], paddings[6], paddings[7] should be zero
     // if the paddings_tf is [ [0, 0] [1,2] [3,4] [0,0] ]
     // paddings = {0, 0, 1, 2, 3, 4, 0, 0} ; flat method is row major
     // then, values are: top = 1, bottom =2, left=3, right=4
-    // For NCHW format: 
-    // paddings[0], paddings[1], paddings[2], paddings[3] should be zero 
+    // For NCHW format:
+    // paddings[0], paddings[1], paddings[2], paddings[3] should be zero
     // similar explanation as NHWC format will apply.
-    int64 pad_top, pad_left; 
-    int64 pad_bottom, pad_right; 
+    int64 pad_top, pad_left;
+    int64 pad_bottom, pad_right;
     string data_format = ToString(data_format_);
-    if(data_format == "NHWC"){
-      pad_top = paddings[2]; 
-      pad_bottom = paddings[3]; 
-      pad_left = paddings[4]; 
-      pad_right = paddings[5]; 
-    }
-    else if (data_format == "NCHW"){
-      pad_top = paddings[4]; 
-      pad_bottom = paddings[5]; 
-      pad_left = paddings[6]; 
-      pad_right = paddings[7]; 
+    if (data_format == "NHWC") {
+      pad_top = paddings[2];
+      pad_bottom = paddings[3];
+      pad_left = paddings[4];
+      pad_right = paddings[5];
+    } else if (data_format == "NCHW") {
+      pad_top = paddings[4];
+      pad_bottom = paddings[5];
+      pad_left = paddings[6];
+      pad_right = paddings[7];
     }
     // Create padding arrays for MKL DNN convolutions.
     // MKL-DNN uses asymetric padding.
@@ -1124,13 +1123,13 @@ class MklConv2DOp : public OpKernel {
                               .TypeConstraint<T>("T")               \
                               .TypeConstraint<int32>("Tpaddings")   \
                               .Label(mkl_op_registry::kMklOpLabel), \
-                   MklConv2DOp<CPUDevice, T, int32, false, true>);  \
+                          MklConv2DOp<CPUDevice, T, int32, false, true>);  \
   REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                 \
                               .Device(DEVICE_CPU)                   \
                               .TypeConstraint<T>("T")               \
                               .TypeConstraint<int64>("Tpaddings")   \
                               .Label(mkl_op_registry::kMklOpLabel), \
-                   MklConv2DOp<CPUDevice, T, int64, false, true>);  \
+                          MklConv2DOp<CPUDevice, T, int64, false, true>);  \
   REGISTER_KERNEL_BUILDER(Name("__MklDummyPadWithConv2D")           \
                               .Device(DEVICE_CPU)                   \
                               .TypeConstraint<T>("T")               \
@@ -1139,7 +1138,7 @@ class MklConv2DOp : public OpKernel {
                           MklDummyOp<CPUDevice, T>);
 
 TF_CALL_float(REGISTER_MKL_CPU);
-#endif // INTEL_MKL_ML
+#endif // INTEL_MKL_ML_ONLY
 
 }  // namespace tensorflow
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 3955bd919d..cd24ae02c4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -230,9 +230,8 @@ class MklDnnConvUtil {
   virtual inline void GetOutputAndPadSizeInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
       const memory::dims& strides, const memory::dims& dilations,
-      memory::dims* output_dims_tf_order,
-      memory::dims* output_dims_mkl_order, memory::dims* pad_l,
-      memory::dims* pad_r, bool padEnabled=false) {
+      memory::dims* output_dims_tf_order, memory::dims* output_dims_mkl_order,
+      memory::dims* pad_l, memory::dims* pad_r, bool padEnabled = false) {
     CHECK_NOTNULL(output_dims_tf_order);
     CHECK_NOTNULL(output_dims_mkl_order);
     CHECK_NOTNULL(pad_l);
@@ -269,17 +268,17 @@ class MklDnnConvUtil {
                                  dilation_cols, stride_cols, padding_,
                                  &out_cols, &pad_left, &pad_right));
     // If padEnabled, i.e., pad and conv op are fused, then
-    // all pads are already passed from pad op through 
-    // *pad_l and *pad_r 
-    if(padEnabled) { 
-       pad_top = static_cast<int64>((*pad_l)[0]);
-       pad_left = static_cast<int64>((*pad_l)[1]);
-       pad_bottom = static_cast<int64>((*pad_r)[0]);
-       pad_right = static_cast<int64>((*pad_r)[1]);
-       // update the out_rows and out_cols based on all 
-       // sides of the pads coming from pad op. 
-       out_rows = out_rows + (pad_top + pad_bottom ) / stride_rows;  
-       out_cols = out_cols + (pad_left + pad_right ) / stride_cols; 
+    // all pads are already passed from pad op through
+    // *pad_l and *pad_r
+    if (padEnabled) {
+      pad_top = static_cast<int64>((*pad_l)[0]);
+      pad_left = static_cast<int64>((*pad_l)[1]);
+      pad_bottom = static_cast<int64>((*pad_r)[0]);
+      pad_right = static_cast<int64>((*pad_r)[1]);
+      // update the out_rows and out_cols based on all
+      // sides of the pads coming from pad op.
+      out_rows = out_rows + (pad_top + pad_bottom) / stride_rows;
+      out_cols = out_cols + (pad_left + pad_right) / stride_cols;
     }
     // Handle padding. MKL-DNN uses asymetric padding.
     // But, if padEnabled, i.e., pad and conv op are fused,
@@ -342,10 +341,9 @@ class MklDnnConvUtil {
   inline void GetConvFwdSizesInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
       memory::dims* input_dims, memory::dims* filter_dims,
-      memory::dims* strides, memory::dims *dilations,
-      memory::dims* output_dims_tf_order,
-      memory::dims* output_dims_mkl_order, memory::dims* pad_l,
-      memory::dims* pad_r, bool padEnabled=false) {
+      memory::dims* strides, memory::dims* dilations,
+      memory::dims* output_dims_tf_order, memory::dims* output_dims_mkl_order,
+      memory::dims* pad_l, memory::dims* pad_r, bool padEnabled = false) {
     CHECK_NOTNULL(input_dims);
     CHECK_NOTNULL(filter_dims);
     CHECK_NOTNULL(strides);
@@ -361,15 +359,13 @@ class MklDnnConvUtil {
     if (!context_->status().ok()) return;
     GetStridesInMklOrder(strides);
     GetDilationsInMklOrder(dilations);
-    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape,
-                                  *strides, *dilations,
-                                  output_dims_tf_order, output_dims_mkl_order,
-                                  pad_l, pad_r, padEnabled);
+    GetOutputAndPadSizeInMklOrder(
+        input_shape, filter_shape, *strides, *dilations, output_dims_tf_order,
+        output_dims_mkl_order, pad_l, pad_r, padEnabled);
     if (!context_->status().ok()) return;
   }
 };
 
-
 /////////////////////////////////////////////////////////////////////
 ///  Common class that implements Conv2DBackpropFilter and Input
 /////////////////////////////////////////////////////////////////////
-- 
GitLab


From 53f2aefe86f8b50addd4b67eb20eb91135b1fac7 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Tue, 14 Aug 2018 13:23:16 -0700
Subject: [PATCH 0012/1554]  fixed, so that now not allowing duplicate control
 edges, alos cleaning up the comments

---
 tensorflow/core/graph/mkl_layout_pass.cc | 15 ++++++---------
 tensorflow/core/kernels/mkl_conv_ops.h   |  4 ----
 2 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 84e8ea8f70..9157080330 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -4189,16 +4189,14 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // We handle control edges now.
   for (const Edge* e : pred->in_edges()) {
     if (e->IsControlEdge()) {
-      // Allow duplicate while adding control edge as it would fail (return
-      // NULL) if we try to add duplicate edge.
-      CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node, true));
+      //Don't allow duplicate edge
+      (*g)->AddControlEdge(e->src(), new_node, false);
     }
   }
   for (const Edge* e : succ->in_edges()) {
     if (e->IsControlEdge()) {
-      // Allow duplicate while adding control edge as it would fail (return
-      // NULL) if we try to add duplicate edge.
-      CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node, true));
+      //Don't allow duplicate edge
+      (*g)->AddControlEdge(e->src(), new_node, false);
     }
   }
 
@@ -4206,9 +4204,8 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // First, we will fix outgoing control edges from 'pred' node.
   for (const Edge* e : pred->out_edges()) {
     if (e->IsControlEdge()) {
-      // Allow duplicate while adding control edge as it would fail (return
-      // NULL) if we try to add duplicate edge.
-      CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst(), true));
+      //Don't allow duplicate edge
+      (*g)->AddControlEdge(new_node, e->dst(), false);
     }
   }
 
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index cd24ae02c4..ebaf1a9947 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -300,10 +300,6 @@ class MklDnnConvUtil {
     mkldnn_sizes[MklDnnDims::Dim_H] = static_cast<int>(out_rows);
     mkldnn_sizes[MklDnnDims::Dim_W] = static_cast<int>(out_cols);
     *output_dims_mkl_order = mkldnn_sizes;
-
-    // Now handle padding. MKL-DNN uses asymetric padding.
-    // But, if padEnabled, i.e., pad and conv op are fused,
-    // then, *pad_l and *pad_r are already set from pad op
   }
 
   // Calculate output and pad size of forward Convolution operator.
-- 
GitLab


From f6c9e054a042bf0f518a740380f3f96a28e8c5be Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Fri, 17 Aug 2018 12:45:06 -0700
Subject: [PATCH 0013/1554]  not allowing duplicate edges, and, add two unit
 tests in mkl_layout_pass_test to test if common input and common output of
 pad an conv2D work correctly for pad+conv2D fusion

---
 tensorflow/core/graph/mkl_layout_pass.cc      |   2 +-
 tensorflow/core/graph/mkl_layout_pass_test.cc | 212 ++++++++++++------
 2 files changed, 143 insertions(+), 71 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 9157080330..6d99e57417 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -4214,7 +4214,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
     if (e->IsControlEdge()) {
       // Allow duplicate while adding control edge as it would fail (return
       // NULL) if we try to add duplicate edge.
-      CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst(), true));
+      (*g)->AddControlEdge(new_node, e->dst(), false);
     } else {
       // Conv2D has only 1 output (at slot 0) and merged node also has only 1
       // output (at slot 0).
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index d1b39ceeca..248520a7f4 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1994,6 +1994,10 @@ REGISTER_OP("_MklInput2")
     .Output("o: uint8")
     .Output("o1: uint8")
     .SetIsStateful();
+REGISTER_OP("Output2")
+    .Input("i: float")
+    .Input("i1: float")
+    .SetIsStateful();
 
 /////////////////////////////////////////////////////////////////////
 //  Unit tests related to node merge optiimization
@@ -2012,76 +2016,6 @@ TEST_F(MklLayoutPassTest, Basic) {
             "A->C;A->D;B->C:1;B->D:1");
 }
 
-// Test set 0: Pad + Conv2D; padding is VALID
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = Conv2D, Z = Zeta
-// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
-// After layout pass
-// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
-TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
-  InitGraph(
-      "node { name: 'A' op: 'Input'}"
-      "node { name: 'B' op: 'Int32Input'}"
-      "node { name: 'C' op: 'Pad'"
-      " attr { key: 'T'                value { type: DT_FLOAT } }"
-      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
-      " input: ['A', 'B']}"
-      "node { name: 'D' op: 'Input'}"
-      "node { name: 'E' op: 'Conv2D'"
-      " attr { key: 'T'                value { type: DT_FLOAT } }"
-      " attr { key: 'data_format'      value { s: 'NHWC' } }"
-      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
-      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
-      " attr { key: 'padding'          value { s: 'VALID' } }"
-      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
-      " input: ['C', 'D'] }"
-      "node { name: 'Y' op: 'Input'}"
-      "node { name: 'Z' op: 'Zeta'"
-      " attr {key: 'T'                 value { type: DT_FLOAT } }"
-      " input: ['E', 'Y']}");
-  EXPECT_EQ(DoMklLayoutOptimizationPass(),
-            "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
-            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
-            "A:control->DMT/_0:control;A:control->DMT/_1:control;"
-            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
-            "DMT/_2->E:5;E->Z;Y->Z:1");
-}
-
-// Test set 0: Pad + Conv2D; padding is SAME
-// A = input(image), B = input(paddings), C= Pad = input of conv2D,
-// D=input(filter), E = Conv2D, Z = Zeta
-// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
-// After layout pass - No merging
-TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Negative) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
-  InitGraph(
-      "node { name: 'A' op: 'Input'}"
-      "node { name: 'B' op: 'Int32Input'}"
-      "node { name: 'C' op: 'Pad'"
-      " attr { key: 'T'                value { type: DT_FLOAT } }"
-      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
-      " input: ['A', 'B']}"
-      "node { name: 'D' op: 'Input'}"
-      "node { name: 'E' op: 'Conv2D'"
-      " attr { key: 'T'                value { type: DT_FLOAT } }"
-      " attr { key: 'data_format'      value { s: 'NHWC' } }"
-      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
-      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
-      " attr { key: 'padding'          value { s: 'SAME' } }"
-      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
-      " input: ['C', 'D'] }"
-      "node { name: 'Y' op: 'Input'}"
-      "node { name: 'Z' op: 'Zeta'"
-      " attr {key: 'T'                 value { type: DT_FLOAT } }"
-      " input: ['E', 'Y']}");
-  EXPECT_EQ(
-      DoMklLayoutOptimizationPass(),
-      "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/_1(Const);"
-      "E(_MklConv2D);Y(Input);Z(Zeta)|A->C;B->C:1;C->E;"
-      "C:control->DMT/_0:control;C:control->DMT/_1:control;"
-      "D->E:1;DMT/_0->E:2;DMT/_1->E:3;E->Z;Y->Z:1");
-}
 
 // Test set 1: Conv2D + AddBias
 
@@ -2389,6 +2323,144 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_ConvBpropInput_FilterFwd) {
             "E:3->G:4;F->G;F:control->DMT/_3:control;G->Z;X->Y:1;X->Z:1");
 }
 
+// Test set 3: Pad + Conv2D fusion 
+// padding is VALID type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
+// After layout pass
+// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
+            "A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
+            "DMT/_2->E:5;E->Z;Y->Z:1");
+}
+// Pad + Conv2D fusion with padding is VALID,
+// Input node pointing to both Pad and Conv2D
+// A = input(image), B = input(paddings), C= Pad 
+// E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,A); Z=Zeta(E,Y)
+// After layout pass
+// _MklPadWithConv2D(A, A, B, DMT/_0, DMT/_1, DMT/_2)
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_Input) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'A'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);DMT/_0(Const);DMT/_1(Const);"
+            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;A->E:1;"
+            "A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;B->E:2;DMT/_0->E:3;DMT/_1->E:4;"
+            "DMT/_2->E:5;E->Z;Y->Z:1");
+}
+// Pad + Conv2D with padding is VALID,
+// Input node pointing to both Pad and Conv2D
+// Output of both Pad and Conv2D feeds one node (Z as Output2)
+// A = input(as image), B = input(as paddings), C= Pad 
+// E = Conv2D, Z = Output2
+// C=Pad(A,B); E=Conv2D(C,A); Z=Output(C,E)
+// After layout pass - No merging, since Pad and Conv2D both
+// feed to the same node (Z)
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_InOutput) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'A'] }"
+      "node { name: 'Z' op: 'Output2'"
+      " input: ['C', 'E']}");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);B(Int32Input);C(Pad);DMT/_0(Const);DMT/_1(Const);"
+            "E(_MklConv2D);Z(Output2)|A->C;A->E:1;B->C:1;C->E;C->Z;"
+            "C:control->DMT/_0:control;C:control->DMT/_1:control;"
+            "DMT/_0->E:2;DMT/_1->E:3;E->Z:1");
+}
+// Pad + Conv2D; padding is SAME
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
+// After layout pass - No merging
+TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Negative) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'SAME' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  EXPECT_EQ(
+      DoMklLayoutOptimizationPass(),
+      "A(Input);B(Int32Input);C(Pad);D(Input);DMT/_0(Const);DMT/_1(Const);"
+      "E(_MklConv2D);Y(Input);Z(Zeta)|A->C;B->C:1;C->E;"
+      "C:control->DMT/_0:control;C:control->DMT/_1:control;"
+      "D->E:1;DMT/_0->E:2;DMT/_1->E:3;E->Z;Y->Z:1");
+}
 /////////////////////////////////////////////////////////////////////
 //  Unit tests related to rewriting node to Mkl node
 /////////////////////////////////////////////////////////////////////
-- 
GitLab


From ccccbe7259f9862e115e267fcce3d8bfb119b3cf Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Mon, 27 Aug 2018 09:28:30 +0800
Subject: [PATCH 0014/1554] Support addition of gradient operations in a graph
 for golang

---
 tensorflow/go/graph.go      |  67 ++++++++++
 tensorflow/go/graph_test.go | 253 ++++++++++++++++++++++++++++++++++++
 2 files changed, 320 insertions(+)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index 32a77550ee..27dc2d84c7 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -147,6 +147,73 @@ func (g *Graph) Operations() []Operation {
 	return ops
 }
 
+// AddGradients adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
+// i.e., d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...
+// This is a simplified version of AddGradientsWithPrefix() without prefix
+func (g *Graph) AddGradients(y []Output, x []Output, dx []Output) ([]Output, error) {
+	return g.AddGradientsWithPrefix("", y, x, dx)
+}
+
+// AddGradientsWithPrefix adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
+// i.e., d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...
+// This is a variant of AddGradients that allows to caller to pass a custom
+// name prefix to the operations added to a graph to compute the gradients.
+func (g *Graph) AddGradientsWithPrefix(prefix string, y []Output, x []Output, dx []Output) ([]Output, error) {
+	var (
+		cprefix = C.CString(prefix)
+
+		cy  = make([]C.TF_Output, len(y))
+		cx  = make([]C.TF_Output, len(x))
+		cdx = make([]C.TF_Output, len(dx))
+		cdy = make([]C.TF_Output, len(x))
+
+		pcy  *C.TF_Output
+		pcx  *C.TF_Output
+		pcdx *C.TF_Output
+		pcdy *C.TF_Output
+
+		status = newStatus()
+	)
+
+	if len(y) > 0 {
+		pcy = &cy[0]
+		for i, o := range y {
+			cy[i] = o.c()
+		}
+	}
+	if len(x) > 0 {
+		pcx = &cx[0]
+		for i, o := range x {
+			cx[i] = o.c()
+		}
+		pcdy = &cdy[0]
+	}
+	if len(dx) > 0 {
+		pcdx = &cdx[0]
+		for i, o := range dx {
+			cdx[i] = o.c()
+		}
+	}
+
+	// If prefix is "", the C.TF_AddGradientsWithPrefix need cprefix to be nil but not ""
+	if len(prefix) == 0 {
+		C.TF_AddGradientsWithPrefix(g.c, nil, pcy, C.int(len(y)), pcx, C.int(len(x)), pcdx, status.c, pcdy)
+	} else {
+		C.TF_AddGradientsWithPrefix(g.c, cprefix, pcy, C.int(len(y)), pcx, C.int(len(x)), pcdx, status.c, pcdy)
+	}
+
+	if err := status.Err(); err != nil {
+		return nil, err
+	}
+	dy := make([]Output, len(x))
+	for i, co := range cdy {
+		op := &Operation{co.oper, g}
+		dy[i] = Output{op, int(co.index)}
+	}
+
+	return dy, nil
+}
+
 // OpSpec is the specification of an Operation to be added to a Graph
 // (using Graph.AddOperation).
 type OpSpec struct {
diff --git a/tensorflow/go/graph_test.go b/tensorflow/go/graph_test.go
index b8d65c54f6..d9126f36ac 100644
--- a/tensorflow/go/graph_test.go
+++ b/tensorflow/go/graph_test.go
@@ -19,6 +19,7 @@ package tensorflow
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"testing"
 )
 
@@ -80,3 +81,255 @@ func TestGraphWriteToAndImport(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestGraphAddGradients(t *testing.T) {
+	g := NewGraph()
+	x1, err := Placeholder(g, "x1", Float)
+	if err != nil {
+		t.Fatal(err)
+	}
+	x2, err := Placeholder(g, "x2", Float)
+	if err != nil {
+		t.Fatal(err)
+	}
+	op0, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y0",
+		Input: []Input{x1},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y0 := op0.Output(0)
+	op1, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y1",
+		Input: []Input{y0},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y1 := op1.Output(0)
+	op2, err := g.AddOperation(OpSpec{
+		Type:  "AddN",
+		Input: []Input{OutputList([]Output{y0, x2})},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y2 := op2.Output(0)
+
+	grads0, err := g.AddGradients([]Output{y1}, []Output{x1}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grads0) != 1 {
+		t.Fatal(len(grads0))
+	}
+	if grads0[0].DataType() != Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), Float)
+	}
+
+	grads1, err := g.AddGradients([]Output{y2}, []Output{x1, x2}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grads1) != 2 {
+		t.Fatal(len(grads1))
+	}
+	if grads1[0].DataType() != Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), Float)
+	}
+	if grads1[1].DataType() != Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads1[1].DataType(), Float)
+	}
+
+	sess, err := NewSession(g, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c1, _ := NewTensor(float32(3.0))
+	c2, _ := NewTensor(float32(2.0))
+	outputs, err := sess.Run(
+		map[Output]*Tensor{x1: c1, x2: c2},
+		[]Output{grads0[0], grads1[0], grads1[1]},
+		nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(outputs) != 3 {
+		t.Fatal(len(outputs))
+	}
+	if outputs[0].Value().(float32) != 108.0 {
+		t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value())
+	}
+	if outputs[1].Value().(float32) != 6.0 {
+		t.Fatalf("Got %v, wanted float 6.0", outputs[1].Value())
+	}
+	if outputs[2].Value().(float32) != 1.0 {
+		t.Fatalf("Got %v, wanted float 1.0", outputs[2].Value())
+	}
+}
+
+func TestGraphAddGradientsSums(t *testing.T) {
+	g := NewGraph()
+	x, err := Placeholder(g, "x", Float)
+	if err != nil {
+		t.Fatal(err)
+	}
+	op0, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y0",
+		Input: []Input{x},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y0 := op0.Output(0)
+	op1, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y1",
+		Input: []Input{y0},
+	})
+	y1 := op1.Output(0)
+
+	grad, err := g.AddGradients([]Output{y0, y1}, []Output{x}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grad) != 1 {
+		t.Fatal(len(grad))
+	}
+	if grad[0].DataType() != Float {
+		t.Fatalf("Got DataType %v, wanted %v", grad[0].DataType(), Float)
+	}
+
+	sess, err := NewSession(g, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c, _ := NewTensor(float32(3.0))
+	outputs, err := sess.Run(
+		map[Output]*Tensor{x: c},
+		[]Output{grad[0]},
+		nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if outputs[0].Value().(float32) != 114.0 {
+		t.Fatalf("Got %v, wanted float 114.0", outputs[0].Value())
+	}
+}
+
+func TestGraphAddGradientsWithInitialValuesToGraph(t *testing.T) {
+	g := NewGraph()
+	x, err := Placeholder(g, "x", Float)
+	op0, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y0",
+		Input: []Input{x},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y0 := op0.Output(0)
+	op1, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y1",
+		Input: []Input{y0},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y1 := op1.Output(0)
+
+	grads0, err := g.AddGradients([]Output{y1}, []Output{y0}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grads0) != 1 {
+		t.Fatal(len(grads0))
+	}
+	if grads0[0].DataType() != Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), Float)
+	}
+
+	grads1, err := g.AddGradients([]Output{y0}, []Output{x}, []Output{grads0[0]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grads1) != 1 {
+		t.Fatal(len(grads1))
+	}
+	if grads1[0].DataType() != Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), Float)
+	}
+
+	sess, err := NewSession(g, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c, _ := NewTensor(float32(3.0))
+	outputs, err := sess.Run(
+		map[Output]*Tensor{x: c},
+		[]Output{grads1[0]},
+		nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if outputs[0].Value().(float32) != 108.0 {
+		t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value())
+	}
+}
+
+func TestGraphValidateGradientsNames(t *testing.T) {
+	g := NewGraph()
+	x, err := Placeholder(g, "x", Float)
+	if err != nil {
+		t.Fatal(err)
+	}
+	op0, err := g.AddOperation(OpSpec{
+		Type:  "Square",
+		Name:  "y0",
+		Input: []Input{x},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	y0 := op0.Output(0)
+
+	grads0, err := g.AddGradients([]Output{y0}, []Output{x}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads0[0].Op.Name(), "gradients/") {
+		t.Fatalf("Got name %v, wanted started with gradients/", grads0[0].Op.Name())
+	}
+
+	grads1, err := g.AddGradients([]Output{y0}, []Output{x}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads1[0].Op.Name(), "gradients_1/") {
+		t.Fatalf("Got name %v, wanted started with gradients_1/", grads1[0].Op.Name())
+	}
+
+	grads2, err := g.AddGradientsWithPrefix("more_gradients", []Output{y0}, []Output{x}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads2[0].Op.Name(), "more_gradients/") {
+		t.Fatalf("Got name %v, wanted started with more_gradients/", grads2[0].Op.Name())
+	}
+
+	grads3, err := g.AddGradientsWithPrefix("even_more_gradients", []Output{y0}, []Output{x}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads3[0].Op.Name(), "even_more_gradients/") {
+		t.Fatalf("Got name %v, wanted started with even_more_gradients/", grads3[0].Op.Name())
+	}
+}
-- 
GitLab


From 5d2226e93d48c1f4a0730edc573b886cc6b87e68 Mon Sep 17 00:00:00 2001
From: jackonan <jackon.zhao@gmail.com>
Date: Fri, 7 Sep 2018 10:42:55 +0800
Subject: [PATCH 0015/1554] Fix ps0 OOM when workers too many.

When training with MonitoredTrainingSession in distributed mode,
all ops related with report_uninitialized_xxx will be placed on ps0,
which may lead to OOM when workers up to thousands.

We use a environment variable to solve it. When OOM happens, users can
set the local device to fix the problem, such as
os.environ['TF_LOCAL_DEVICE'] = '/job:worker/task:index'.
---
 tensorflow/python/ops/resources.py | 4 +++-
 tensorflow/python/ops/variables.py | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/resources.py b/tensorflow/python/ops/resources.py
index db6740643c..98ed07999f 100644
--- a/tensorflow/python/ops/resources.py
+++ b/tensorflow/python/ops/resources.py
@@ -21,6 +21,7 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import os
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -86,7 +87,8 @@ def report_uninitialized_resources(resource_list=None,
     resource_list = shared_resources() + local_resources()
   with ops.name_scope(name):
     # Run all operations on CPU
-    with ops.device("/cpu:0"):
+    local_device = os.environ.get("TF_LOCAL_DEVICE", "/cpu:0")
+    with ops.device(local_device):
       if not resource_list:
         # Return an empty tensor so we only need to check for returned tensor
         # size being 0 as an indication of model ready.
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index d03d93beeb..ac0b36efc5 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import enum  # pylint: disable=g-bad-import-order
 
+import os
 import six
 
 from tensorflow.core.framework import attr_value_pb2
@@ -2283,7 +2284,8 @@ def report_uninitialized_variables(var_list=None,
     # Run all operations on CPU
     if var_list:
       init_vars = [state_ops.is_variable_initialized(v) for v in var_list]
-    with ops.device("/cpu:0"):
+    local_device = os.environ.get("TF_LOCAL_DEVICE", "/cpu:0")
+    with ops.device(local_device):
       if not var_list:
         # Return an empty tensor so we only need to check for returned tensor
         # size being 0 as an indication of model ready.
-- 
GitLab


From a9a6c8efec20b67b0f9d49dec6c890ec4b1ac18c Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Sat, 8 Sep 2018 20:40:04 +0800
Subject: [PATCH 0016/1554] Combine AddGradients and AddGradientsWithPrefix
 Methods in golang

---
 tensorflow/go/graph.go      | 13 +++----------
 tensorflow/go/graph_test.go | 18 +++++++++---------
 2 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index 27dc2d84c7..6fe2b6b86d 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -147,18 +147,11 @@ func (g *Graph) Operations() []Operation {
 	return ops
 }
 
-// AddGradients adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
-// i.e., d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...
-// This is a simplified version of AddGradientsWithPrefix() without prefix
-func (g *Graph) AddGradients(y []Output, x []Output, dx []Output) ([]Output, error) {
-	return g.AddGradientsWithPrefix("", y, x, dx)
-}
-
 // AddGradientsWithPrefix adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
 // i.e., d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...
-// This is a variant of AddGradients that allows to caller to pass a custom
-// name prefix to the operations added to a graph to compute the gradients.
-func (g *Graph) AddGradientsWithPrefix(prefix string, y []Output, x []Output, dx []Output) ([]Output, error) {
+// This methods allows to caller to pass a custom name prefix to the operations
+// added to a graph to compute the gradients.
+func (g *Graph) AddGradients(prefix string, y []Output, x []Output, dx []Output) ([]Output, error) {
 	var (
 		cprefix = C.CString(prefix)
 
diff --git a/tensorflow/go/graph_test.go b/tensorflow/go/graph_test.go
index d9126f36ac..d8f32dbaa9 100644
--- a/tensorflow/go/graph_test.go
+++ b/tensorflow/go/graph_test.go
@@ -119,7 +119,7 @@ func TestGraphAddGradients(t *testing.T) {
 	}
 	y2 := op2.Output(0)
 
-	grads0, err := g.AddGradients([]Output{y1}, []Output{x1}, nil)
+	grads0, err := g.AddGradients("", []Output{y1}, []Output{x1}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -130,7 +130,7 @@ func TestGraphAddGradients(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), Float)
 	}
 
-	grads1, err := g.AddGradients([]Output{y2}, []Output{x1, x2}, nil)
+	grads1, err := g.AddGradients("", []Output{y2}, []Output{x1, x2}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -194,7 +194,7 @@ func TestGraphAddGradientsSums(t *testing.T) {
 	})
 	y1 := op1.Output(0)
 
-	grad, err := g.AddGradients([]Output{y0, y1}, []Output{x}, nil)
+	grad, err := g.AddGradients("", []Output{y0, y1}, []Output{x}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -245,7 +245,7 @@ func TestGraphAddGradientsWithInitialValuesToGraph(t *testing.T) {
 	}
 	y1 := op1.Output(0)
 
-	grads0, err := g.AddGradients([]Output{y1}, []Output{y0}, nil)
+	grads0, err := g.AddGradients("", []Output{y1}, []Output{y0}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -256,7 +256,7 @@ func TestGraphAddGradientsWithInitialValuesToGraph(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), Float)
 	}
 
-	grads1, err := g.AddGradients([]Output{y0}, []Output{x}, []Output{grads0[0]})
+	grads1, err := g.AddGradients("", []Output{y0}, []Output{x}, []Output{grads0[0]})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -301,7 +301,7 @@ func TestGraphValidateGradientsNames(t *testing.T) {
 	}
 	y0 := op0.Output(0)
 
-	grads0, err := g.AddGradients([]Output{y0}, []Output{x}, nil)
+	grads0, err := g.AddGradients("", []Output{y0}, []Output{x}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -309,7 +309,7 @@ func TestGraphValidateGradientsNames(t *testing.T) {
 		t.Fatalf("Got name %v, wanted started with gradients/", grads0[0].Op.Name())
 	}
 
-	grads1, err := g.AddGradients([]Output{y0}, []Output{x}, nil)
+	grads1, err := g.AddGradients("", []Output{y0}, []Output{x}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -317,7 +317,7 @@ func TestGraphValidateGradientsNames(t *testing.T) {
 		t.Fatalf("Got name %v, wanted started with gradients_1/", grads1[0].Op.Name())
 	}
 
-	grads2, err := g.AddGradientsWithPrefix("more_gradients", []Output{y0}, []Output{x}, nil)
+	grads2, err := g.AddGradients("more_gradients", []Output{y0}, []Output{x}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -325,7 +325,7 @@ func TestGraphValidateGradientsNames(t *testing.T) {
 		t.Fatalf("Got name %v, wanted started with more_gradients/", grads2[0].Op.Name())
 	}
 
-	grads3, err := g.AddGradientsWithPrefix("even_more_gradients", []Output{y0}, []Output{x}, nil)
+	grads3, err := g.AddGradients("even_more_gradients", []Output{y0}, []Output{x}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-- 
GitLab


From 9f64bbb0091d0e6ee917ab40e65fc19fa1425da2 Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Wed, 12 Sep 2018 11:26:15 +0800
Subject: [PATCH 0017/1554] Fix gradients test in golang

1. fix some test name.
2. add test for existing prefix.
---
 tensorflow/go/graph_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/graph_test.go b/tensorflow/go/graph_test.go
index d8f32dbaa9..067c7db5c3 100644
--- a/tensorflow/go/graph_test.go
+++ b/tensorflow/go/graph_test.go
@@ -223,7 +223,7 @@ func TestGraphAddGradientsSums(t *testing.T) {
 	}
 }
 
-func TestGraphAddGradientsWithInitialValuesToGraph(t *testing.T) {
+func TestGraphAddGradientsWithInitialValues(t *testing.T) {
 	g := NewGraph()
 	x, err := Placeholder(g, "x", Float)
 	op0, err := g.AddOperation(OpSpec{
@@ -332,4 +332,9 @@ func TestGraphValidateGradientsNames(t *testing.T) {
 	if !strings.HasPrefix(grads3[0].Op.Name(), "even_more_gradients/") {
 		t.Fatalf("Got name %v, wanted started with even_more_gradients/", grads3[0].Op.Name())
 	}
+
+	_, err = g.AddGradients("even_more_gradients", []Output{y0}, []Output{x}, nil)
+	if err == nil {
+		t.Error("AddGradients should have failed if gradients name is already existing")
+	}
 }
-- 
GitLab


From e40c032d62075621053262758c065f9bfb9faed9 Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Wed, 12 Sep 2018 15:00:23 +0800
Subject: [PATCH 0018/1554] Support addition of gradient operations in op
 package for golang

1. Add Gradients method in op package
  Enforce uniqueness of custom prefixes for gradients.
2. Add unit test for Gradients
---
 tensorflow/go/op/gradients.go      |  41 +++++
 tensorflow/go/op/gradients_test.go | 233 +++++++++++++++++++++++++++++
 2 files changed, 274 insertions(+)
 create mode 100644 tensorflow/go/op/gradients.go
 create mode 100644 tensorflow/go/op/gradients_test.go

diff --git a/tensorflow/go/op/gradients.go b/tensorflow/go/op/gradients.go
new file mode 100644
index 0000000000..2dce134d5f
--- /dev/null
+++ b/tensorflow/go/op/gradients.go
@@ -0,0 +1,41 @@
+/*
+Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package op
+
+import tf "github.com/tensorflow/tensorflow/tensorflow/go"
+
+// Gradients adds gradients computation ops to the graph according to scope.
+//
+// Arguments:
+//  prefix: unique string prefix applied before the names of nodes added to the graph to
+//    compute gradients. If null, will use "Gradients".
+//  y: output of the function to derive
+//  x: inputs of the function for which partial derivatives are computed
+//  dx: if not null, the partial derivatives of some loss function L w.r.t. y
+//
+//  return the partial derivatives
+func Gradients(scope *Scope, prefix string, y []tf.Output, x []tf.Output, dx ...tf.Output) (output []tf.Output) {
+	var err error
+	if prefix == "" {
+		prefix = "Gradients"
+	}
+	if output, err = scope.graph.AddGradients(scope.opName(scope.uniqueName(prefix)), y, x, dx); err != nil {
+		scope.UpdateErr("Gradients", err)
+		return
+	}
+	return output
+}
diff --git a/tensorflow/go/op/gradients_test.go b/tensorflow/go/op/gradients_test.go
new file mode 100644
index 0000000000..7d03fada74
--- /dev/null
+++ b/tensorflow/go/op/gradients_test.go
@@ -0,0 +1,233 @@
+/*
+Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package op
+
+import (
+	"strings"
+	"testing"
+
+	tf "github.com/tensorflow/tensorflow/tensorflow/go"
+)
+
+func TestAddGradients(t *testing.T) {
+	var (
+		s  = NewScope()
+		x1 = Placeholder(s.SubScope("x1"), tf.Float)
+		x2 = Placeholder(s.SubScope("x2"), tf.Float)
+		y0 = Square(s.SubScope("y0"), x1)
+		y1 = Square(s.SubScope("y1"), y0)
+		y2 = AddN(s.SubScope("y2"), []tf.Output{y0, x2})
+	)
+
+	grads0 := Gradients(s, "", []tf.Output{y1}, []tf.Output{x1})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if len(grads0) != 1 {
+		t.Fatal(len(grads0))
+	}
+	if grads0[0].DataType() != tf.Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float)
+	}
+
+	grads1 := Gradients(s, "", []tf.Output{y2}, []tf.Output{x1, x2})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if len(grads1) != 2 {
+		t.Fatal(len(grads1))
+	}
+	if grads1[0].DataType() != tf.Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), tf.Float)
+	}
+	if grads1[1].DataType() != tf.Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads1[1].DataType(), tf.Float)
+	}
+
+	graph, err := s.Finalize()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sess, err := tf.NewSession(graph, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c1, _ := tf.NewTensor(float32(3.0))
+	c2, _ := tf.NewTensor(float32(3.0))
+	outputs, err := sess.Run(
+		map[tf.Output]*tf.Tensor{x1: c1, x2: c2},
+		[]tf.Output{grads0[0], grads1[0], grads1[1]},
+		nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(outputs) != 3 {
+		t.Fatal(len(outputs))
+	}
+	if outputs[0].Value().(float32) != 108.0 {
+		t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value())
+	}
+	if outputs[1].Value().(float32) != 6.0 {
+		t.Fatalf("Got %v, wanted float 6.0", outputs[1].Value())
+	}
+	if outputs[2].Value().(float32) != 1.0 {
+		t.Fatalf("Got %v, wanted float 1.0", outputs[2].Value())
+	}
+}
+
+func TestAddGradientsSums(t *testing.T) {
+	var (
+		s  = NewScope()
+		x  = Placeholder(s.SubScope("x"), tf.Float)
+		y0 = Square(s.SubScope("y0"), x)
+		y1 = Square(s.SubScope("y1"), y0)
+	)
+
+	grad := Gradients(s, "", []tf.Output{y0, y1}, []tf.Output{x})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if len(grad) != 1 {
+		t.Fatal(len(grad))
+	}
+	if grad[0].DataType() != tf.Float {
+		t.Fatalf("Got DataType %v, wanted %v", grad[0].DataType(), tf.Float)
+	}
+
+	graph, err := s.Finalize()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sess, err := tf.NewSession(graph, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c, _ := tf.NewTensor(float32(3.0))
+	outputs, err := sess.Run(
+		map[tf.Output]*tf.Tensor{x: c},
+		[]tf.Output{grad[0]},
+		nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if outputs[0].Value().(float32) != 114.0 {
+		t.Fatalf("Got %v, wanted float 114.0", outputs[0].Value())
+	}
+}
+
+func TestAddGradientsWithInitialValues(t *testing.T) {
+	var (
+		s  = NewScope()
+		x  = Placeholder(s.SubScope("x1"), tf.Float)
+		y0 = Square(s.SubScope("y0"), x)
+		y1 = Square(s.SubScope("y1"), y0)
+	)
+
+	grads0 := Gradients(s, "", []tf.Output{y1}, []tf.Output{y0})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if len(grads0) != 1 {
+		t.Fatal(len(grads0))
+	}
+	if grads0[0].DataType() != tf.Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float)
+	}
+
+	grads1 := Gradients(s, "", []tf.Output{y0}, []tf.Output{x}, grads0[0])
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if len(grads1) != 1 {
+		t.Fatal(len(grads1))
+	}
+	if grads1[0].DataType() != tf.Float {
+		t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), tf.Float)
+	}
+
+	graph, err := s.Finalize()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sess, err := tf.NewSession(graph, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c, _ := tf.NewTensor(float32(3.0))
+	outputs, err := sess.Run(
+		map[tf.Output]*tf.Tensor{x: c},
+		[]tf.Output{grads1[0]},
+		nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if outputs[0].Value().(float32) != 108.0 {
+		t.Fatalf("Got %v, wanted float 108.0", outputs[0].Value())
+	}
+}
+
+func TestValidateGradientsNames(t *testing.T) {
+	var (
+		s  = NewScope()
+		x  = Placeholder(s.SubScope("x"), tf.Float)
+		y0 = Square(s.SubScope("y0"), x)
+	)
+
+	grads0 := Gradients(s, "", []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads0[0].Op.Name(), "Gradients/") {
+		t.Fatalf("Got name %v, wanted started with Gradients/", grads0[0].Op.Name())
+	}
+
+	grads1 := Gradients(s, "", []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads1[0].Op.Name(), "Gradients_1/") {
+		t.Fatalf("Got name %v, wanted started with Gradients_1/", grads1[0].Op.Name())
+	}
+
+	grads2 := Gradients(s, "more_gradients", []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads2[0].Op.Name(), "more_gradients/") {
+		t.Fatalf("Got name %v, wanted started with more_gradients/", grads2[0].Op.Name())
+	}
+
+	sub := s.SubScope("sub")
+	grads3 := Gradients(sub, "even_more_gradients", []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads3[0].Op.Name(), "sub/even_more_gradients/") {
+		t.Fatalf("Got name %v, wanted started with sub/even_more_gradients/", grads3[0].Op.Name())
+	}
+
+	grads4 := Gradients(sub, "even_more_gradients", []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(grads4[0].Op.Name(), "sub/even_more_gradients_1/") {
+		t.Fatalf("Got name %v, wanted started with sub/even_more_gradients_1/", grads4[0].Op.Name())
+	}
+}
-- 
GitLab


From f8ec0f101bac066faa2e917ac714ca9eea310eac Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Thu, 13 Sep 2018 22:40:49 -0700
Subject: [PATCH 0019/1554]  adding checks that pad fusion works only Conv2D

---
 tensorflow/core/kernels/mkl_conv_ops.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 54670c8521..4b54ce1d52 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -900,7 +900,10 @@ class MklConvOp : public OpKernel {
 
       bool isConv2D = (strides_.size() == 4);
       // TODO(Intel-tf) Add check to make sure padEnabled is true only for 2D
-
+      if(!isConv2D){
+        OP_REQUIRES(context, padEnabled,
+                errors::InvalidArgument("Pad+Conv fusion only works for 2D"));
+      }
       // Create memory for user data.
       // Describe how the inputs and outputs of Convolution look like. Also
       // specify buffers containing actual input and output data.
-- 
GitLab


From 0e87ed82815053b4f1c038975382d72282fdf97f Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Sun, 16 Sep 2018 11:20:26 -0700
Subject: [PATCH 0020/1554]  Adding two unit tests for pad+conv2d fusion. They
 test if the two merging ops get control edge from a common op, then the
 merged node will have only one control edge.

---
 tensorflow/core/graph/mkl_layout_pass.cc      |  21 +++-
 tensorflow/core/graph/mkl_layout_pass_test.cc | 111 ++++++++++++++++++
 2 files changed, 128 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index ef8a2b0838..d3a4112ee9 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -4192,10 +4192,23 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
 
   // 2. Get inputs from both the nodes.
 
-  // Pad must have 2 inputs: "input" and paddings.
-  CHECK_EQ(pred->in_edges().size(), 2);
-  // Conv2D must have 2 inputs: pad output and Filter
-  CHECK_EQ(succ->in_edges().size(), 2);
+  // Pad must have 2 data inputs: "input" and paddings.
+  int PadDataInputEdges = 0;
+  for (const Edge* e : pred->in_edges()) {
+    if (!e->IsControlEdge()) {
+      PadDataInputEdges++;
+    }
+  }
+  CHECK_EQ(PadDataInputEdges, 2);
+
+  // Conv2D must have 2 data inputs: pad output and Filter
+  int ConvDataInputEdges = 0;
+  for (const Edge* e : succ->in_edges()) {
+    if (!e->IsControlEdge()) {
+      ConvDataInputEdges++;
+    }
+  }
+  CHECK_EQ(ConvDataInputEdges, 2);
 
   // We will use the node name of Conv2D as the name of new node
   // Build new node. We use same name as original node, but change the op
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 248520a7f4..e9e234010c 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1928,6 +1928,13 @@ static void InitGraph(const string& s, Graph* graph,
 class MklLayoutPassTest : public ::testing::Test {
  public:
   MklLayoutPassTest() : graph_(OpRegistry::Global()) {}
+  // Return Node* from the Node Name
+  Node* FindNode(const string& name) {
+    for (Node* node : graph_.nodes()) {
+      if (node->name() == name) return node;
+    }
+    LOG(FATAL) << name;
+  }
 
   void InitGraph(const string& s, const string& device = kCPUDevice) {
     ::tensorflow::InitGraph(s, &graph_, device);
@@ -1998,6 +2005,9 @@ REGISTER_OP("Output2")
     .Input("i: float")
     .Input("i1: float")
     .SetIsStateful();
+REGISTER_OP("Output")
+    .Input("i: float")
+    .SetIsStateful();
 
 /////////////////////////////////////////////////////////////////////
 //  Unit tests related to node merge optiimization
@@ -2359,6 +2369,107 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
             "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
             "DMT/_2->E:5;E->Z;Y->Z:1");
 }
+// Test if input control edges do not duplicate after merge.
+// If both the merging ops have input control edge from a common op
+// then, the merged op will have only one control edge from that 
+// common op.
+// padding is VALID type
+// A = input(image), A1 = input, B = input(paddings), 
+// C= Pad = input of conv2D,
+// D=input(filter), E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
+// C:control->A1:control
+// E:control->A1:control
+// After layout pass
+// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
+// A1:control->E:control
+TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithConv2D_Positive) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A1' op: 'Input'}"
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  Node* a1 = FindNode("A1");
+  Node* c = FindNode("C");
+  Node* e = FindNode("E");
+  const Edge* edge = graph_.AddControlEdge(a1, c);
+  const Edge* edge_1 = graph_.AddControlEdge(a1, e);
+  ASSERT_TRUE(edge != nullptr);
+  ASSERT_TRUE(edge_1 != nullptr);
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
+            "A1:control->E:control;A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
+            "DMT/_2->E:5;E->Z;Y->Z:1");
+}
+// Test if output control edges does not duplicate after merge.
+// If both the merging ops have output control edge to a common op,
+// then after merge, the merged op will have only one control edge
+// to that commom op.
+// padding is VALID type
+// A = input(image), B = input(paddings), C= Pad = input of conv2D,
+// D=input(filter), E = Conv2D, Z = Zeta
+// C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
+// C:control->A1:control
+// E:control->A1:control
+// After layout pass
+// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
+// E:control->A1:control (only one control edge)
+TEST_F(MklLayoutPassTest, ControlEdge_PadWithConv2D_Positive) {
+  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  InitGraph(
+      "node { name: 'A1' op: 'Input'}"
+      "node { name: 'A' op: 'Input'}"
+      "node { name: 'B' op: 'Int32Input'}"
+      "node { name: 'C' op: 'Pad'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'Tpaddings'        value { type: DT_INT32 } }"
+      " input: ['A', 'B']}"
+      "node { name: 'D' op: 'Input'}"
+      "node { name: 'E' op: 'Conv2D'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " attr { key: 'data_format'      value { s: 'NHWC' } }"
+      " attr { key: 'use_cudnn_on_gpu' value { b: false } }"
+      " attr { key: 'strides'          value { list: {i: 1, i:1, i:1, i:1} } }"
+      " attr { key: 'padding'          value { s: 'VALID' } }"
+      " attr { key: 'dilations'        value { list: {i: 1, i:1, i:1, i:1} } }"
+      " input: ['C', 'D'] }"
+      "node { name: 'Y' op: 'Input'}"
+      "node { name: 'Z' op: 'Zeta'"
+      " attr {key: 'T'                 value { type: DT_FLOAT } }"
+      " input: ['E', 'Y']}");
+  Node* a1 = FindNode("A1");
+  Node* c = FindNode("C");
+  Node* e = FindNode("E");
+  const Edge* edge = graph_.AddControlEdge(c, a1);
+  const Edge* edge_1 = graph_.AddControlEdge(e, a1);
+  ASSERT_TRUE(edge != nullptr);
+  ASSERT_TRUE(edge_1 != nullptr);
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
+            "A:control->DMT/_0:control;A:control->DMT/_1:control;"
+            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
+            "DMT/_2->E:5;E->Z;E:control->A1:control;Y->Z:1");
+}
 // Pad + Conv2D fusion with padding is VALID,
 // Input node pointing to both Pad and Conv2D
 // A = input(image), B = input(paddings), C= Pad 
-- 
GitLab


From 4e140eed6b4f6722b94cf85432d4519b8c5ce0bf Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 17 Sep 2018 10:05:40 -0700
Subject: [PATCH 0021/1554] changing the name of the unit tests

---
 tensorflow/core/graph/mkl_layout_pass_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index e9e234010c..9ad45a2cfd 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -1928,7 +1928,7 @@ static void InitGraph(const string& s, Graph* graph,
 class MklLayoutPassTest : public ::testing::Test {
  public:
   MklLayoutPassTest() : graph_(OpRegistry::Global()) {}
-  // Return Node* from the Node Name
+  // Ashraf added
   Node* FindNode(const string& name) {
     for (Node* node : graph_.nodes()) {
       if (node->name() == name) return node;
@@ -2383,7 +2383,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
 // After layout pass
 // _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
 // A1:control->E:control
-TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithConv2D_Positive) {
+TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
   CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A1' op: 'Input'}"
-- 
GitLab


From 2a8f7bcc59bc4e36ea88f4187028b4461f5f1072 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Tue, 18 Sep 2018 11:28:21 -0700
Subject: [PATCH 0022/1554] minor change in the two unit tests

---
 tensorflow/core/graph/mkl_layout_pass_test.cc | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 9ad45a2cfd..60a7f138c8 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -2378,11 +2378,11 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
 // C= Pad = input of conv2D,
 // D=input(filter), E = Conv2D, Z = Zeta
 // C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
-// C:control->A1:control
-// E:control->A1:control
-// After layout pass
-// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
+// A1:control->C:control
 // A1:control->E:control
+// After layout pass:
+// _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
+// A1:control->E:control (only one control edge)
 TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
   CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
@@ -2411,8 +2411,8 @@ TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
   Node* e = FindNode("E");
   const Edge* edge = graph_.AddControlEdge(a1, c);
   const Edge* edge_1 = graph_.AddControlEdge(a1, e);
-  ASSERT_TRUE(edge != nullptr);
-  ASSERT_TRUE(edge_1 != nullptr);
+  ASSERT_NE(edge, nullptr);
+  ASSERT_NE(edge_1, nullptr);
   EXPECT_EQ(DoMklLayoutOptimizationPass(),
             "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
             "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
@@ -2430,10 +2430,10 @@ TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
 // C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
 // C:control->A1:control
 // E:control->A1:control
-// After layout pass
+// After layout pass:
 // _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
 // E:control->A1:control (only one control edge)
-TEST_F(MklLayoutPassTest, ControlEdge_PadWithConv2D_Positive) {
+TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithConv2D_Positive) {
   CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A1' op: 'Input'}"
@@ -2461,8 +2461,8 @@ TEST_F(MklLayoutPassTest, ControlEdge_PadWithConv2D_Positive) {
   Node* e = FindNode("E");
   const Edge* edge = graph_.AddControlEdge(c, a1);
   const Edge* edge_1 = graph_.AddControlEdge(e, a1);
-  ASSERT_TRUE(edge != nullptr);
-  ASSERT_TRUE(edge_1 != nullptr);
+  ASSERT_NE(edge, nullptr);
+  ASSERT_NE(edge_1, nullptr);
   EXPECT_EQ(DoMklLayoutOptimizationPass(),
             "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
             "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
-- 
GitLab


From c475edf9513d6bceae992775869fb9dd0b2c848a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 24 Sep 2018 05:43:36 +0000
Subject: [PATCH 0023/1554] Fix for tf.keras.regularizers.{l1,l2}(0.) with
 tf.get_variable

This fix tries to address the issue in 22470 where
tf.keras.regularizers.{l1,l2}(l=0.) with tf.get_variable returns
```
AttributeError: 'float' object has no attribute 'name'
```

The issue only happens when `l=0.` as in that case, `regularization = 0.`
was returned directly (and `0.` does not have a `name` attribute as a float number)

This fix convert regularization = 0. to tensor.

This fix fixes 22470.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/regularizers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/regularizers.py b/tensorflow/python/keras/regularizers.py
index 28b6ad4c65..0d139d748c 100644
--- a/tensorflow/python/keras/regularizers.py
+++ b/tensorflow/python/keras/regularizers.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import six
 
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
@@ -54,7 +55,7 @@ class L1L2(Regularizer):
     self.l2 = K.cast_to_floatx(l2)
 
   def __call__(self, x):
-    regularization = 0.
+    regularization = ops.convert_to_tensor(0.)
     if self.l1:
       regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
     if self.l2:
-- 
GitLab


From 7f51123b0e8ab0532af04a5ccad3ab9605128db4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 24 Sep 2018 05:51:09 +0000
Subject: [PATCH 0024/1554] Add dtype to convert_to_tensor

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/regularizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/regularizers.py b/tensorflow/python/keras/regularizers.py
index 0d139d748c..fd0748f3ae 100644
--- a/tensorflow/python/keras/regularizers.py
+++ b/tensorflow/python/keras/regularizers.py
@@ -55,7 +55,7 @@ class L1L2(Regularizer):
     self.l2 = K.cast_to_floatx(l2)
 
   def __call__(self, x):
-    regularization = ops.convert_to_tensor(0.)
+    regularization = ops.convert_to_tensor(0., dtype=K.floatx())
     if self.l1:
       regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
     if self.l2:
-- 
GitLab


From b9e5738fb4b5f53e8d15aadd75506f662b04643d Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Fri, 28 Sep 2018 07:36:41 +0800
Subject: [PATCH 0025/1554] Fix comment in graph.go.

Method name is AddGradients but not AddGradientsWithPrefix.
---
 tensorflow/go/graph.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index 6fe2b6b86d..4204ad1cc4 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -147,7 +147,7 @@ func (g *Graph) Operations() []Operation {
 	return ops
 }
 
-// AddGradientsWithPrefix adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
+// AddGradients adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
 // i.e., d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...
 // This methods allows to caller to pass a custom name prefix to the operations
 // added to a graph to compute the gradients.
-- 
GitLab


From 72ba6b227128c29847c9d3e4ceaded1032b9b8cc Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Fri, 28 Sep 2018 08:20:28 +0800
Subject: [PATCH 0026/1554] Fix calls to C.CString() in graph.go to avoid
 memory leaks.

---
 tensorflow/go/graph.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index 4204ad1cc4..096a182840 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -153,7 +153,7 @@ func (g *Graph) Operations() []Operation {
 // added to a graph to compute the gradients.
 func (g *Graph) AddGradients(prefix string, y []Output, x []Output, dx []Output) ([]Output, error) {
 	var (
-		cprefix = C.CString(prefix)
+		cprefix *C.char
 
 		cy  = make([]C.TF_Output, len(y))
 		cx  = make([]C.TF_Output, len(x))
@@ -189,12 +189,13 @@ func (g *Graph) AddGradients(prefix string, y []Output, x []Output, dx []Output)
 	}
 
 	// If prefix is "", the C.TF_AddGradientsWithPrefix need cprefix to be nil but not ""
-	if len(prefix) == 0 {
-		C.TF_AddGradientsWithPrefix(g.c, nil, pcy, C.int(len(y)), pcx, C.int(len(x)), pcdx, status.c, pcdy)
-	} else {
-		C.TF_AddGradientsWithPrefix(g.c, cprefix, pcy, C.int(len(y)), pcx, C.int(len(x)), pcdx, status.c, pcdy)
+	if len(prefix) != 0 {
+		cprefix = C.CString(prefix)
+		defer C.free(unsafe.Pointer(cprefix))
 	}
 
+	C.TF_AddGradientsWithPrefix(g.c, cprefix, pcy, C.int(len(y)), pcx, C.int(len(x)), pcdx, status.c, pcdy)
+
 	if err := status.Err(); err != nil {
 		return nil, err
 	}
-- 
GitLab


From 777e6a4e194e4cc141feb6b250702c0e4946ca2d Mon Sep 17 00:00:00 2001
From: wangsiyu <siyu.wsy@gmail.com>
Date: Mon, 1 Oct 2018 13:51:36 +0800
Subject: [PATCH 0027/1554] Make colocations be compatible with
 DistributionStrategy in SyncReplicasOptimizer

---
 tensorflow/python/training/sync_replicas_optimizer.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index 7afaa92699..99d2563fc6 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer
 from tensorflow.python.training import queue_runner
 from tensorflow.python.training import session_manager
@@ -245,7 +246,9 @@ class SyncReplicasOptimizer(optimizer.Optimizer):
     # local_anchor op will be placed on this worker task by default.
     local_anchor = control_flow_ops.no_op()
     # Colocating local_step variable prevents it being placed on the PS.
-    with ops.colocate_with(local_anchor):
+    distribution_strategy = (
+        distribution_strategy_context.get_distribution_strategy())
+    with distribution_strategy.colocate_vars_with(local_anchor):
       self._local_step = variable_scope.variable(
           initial_value=0,
           trainable=False,
-- 
GitLab


From 8c9d5eb52b0d8e551b0e751186edfcdecbef62fb Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Mon, 1 Oct 2018 18:02:32 +0800
Subject: [PATCH 0028/1554] Remove  from func Gradients() in gradients.go

---
 tensorflow/go/op/gradients.go      |  9 ++------
 tensorflow/go/op/gradients_test.go | 34 ++++++++++++------------------
 2 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/tensorflow/go/op/gradients.go b/tensorflow/go/op/gradients.go
index 2dce134d5f..2397e40bf4 100644
--- a/tensorflow/go/op/gradients.go
+++ b/tensorflow/go/op/gradients.go
@@ -21,19 +21,14 @@ import tf "github.com/tensorflow/tensorflow/tensorflow/go"
 // Gradients adds gradients computation ops to the graph according to scope.
 //
 // Arguments:
-//  prefix: unique string prefix applied before the names of nodes added to the graph to
-//    compute gradients. If null, will use "Gradients".
 //  y: output of the function to derive
 //  x: inputs of the function for which partial derivatives are computed
 //  dx: if not null, the partial derivatives of some loss function L w.r.t. y
 //
 //  return the partial derivatives
-func Gradients(scope *Scope, prefix string, y []tf.Output, x []tf.Output, dx ...tf.Output) (output []tf.Output) {
+func Gradients(scope *Scope, y []tf.Output, x []tf.Output, dx ...tf.Output) (output []tf.Output) {
 	var err error
-	if prefix == "" {
-		prefix = "Gradients"
-	}
-	if output, err = scope.graph.AddGradients(scope.opName(scope.uniqueName(prefix)), y, x, dx); err != nil {
+	if output, err = scope.graph.AddGradients(scope.opName(scope.uniqueName("Gradients")), y, x, dx); err != nil {
 		scope.UpdateErr("Gradients", err)
 		return
 	}
diff --git a/tensorflow/go/op/gradients_test.go b/tensorflow/go/op/gradients_test.go
index 7d03fada74..2bcb3e88eb 100644
--- a/tensorflow/go/op/gradients_test.go
+++ b/tensorflow/go/op/gradients_test.go
@@ -33,7 +33,7 @@ func TestAddGradients(t *testing.T) {
 		y2 = AddN(s.SubScope("y2"), []tf.Output{y0, x2})
 	)
 
-	grads0 := Gradients(s, "", []tf.Output{y1}, []tf.Output{x1})
+	grads0 := Gradients(s, []tf.Output{y1}, []tf.Output{x1})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -44,7 +44,7 @@ func TestAddGradients(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float)
 	}
 
-	grads1 := Gradients(s, "", []tf.Output{y2}, []tf.Output{x1, x2})
+	grads1 := Gradients(s, []tf.Output{y2}, []tf.Output{x1, x2})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -98,7 +98,7 @@ func TestAddGradientsSums(t *testing.T) {
 		y1 = Square(s.SubScope("y1"), y0)
 	)
 
-	grad := Gradients(s, "", []tf.Output{y0, y1}, []tf.Output{x})
+	grad := Gradients(s, []tf.Output{y0, y1}, []tf.Output{x})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -139,7 +139,7 @@ func TestAddGradientsWithInitialValues(t *testing.T) {
 		y1 = Square(s.SubScope("y1"), y0)
 	)
 
-	grads0 := Gradients(s, "", []tf.Output{y1}, []tf.Output{y0})
+	grads0 := Gradients(s, []tf.Output{y1}, []tf.Output{y0})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -150,7 +150,7 @@ func TestAddGradientsWithInitialValues(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float)
 	}
 
-	grads1 := Gradients(s, "", []tf.Output{y0}, []tf.Output{x}, grads0[0])
+	grads1 := Gradients(s, []tf.Output{y0}, []tf.Output{x}, grads0[0])
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -190,7 +190,7 @@ func TestValidateGradientsNames(t *testing.T) {
 		y0 = Square(s.SubScope("y0"), x)
 	)
 
-	grads0 := Gradients(s, "", []tf.Output{y0}, []tf.Output{x})
+	grads0 := Gradients(s, []tf.Output{y0}, []tf.Output{x})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -198,7 +198,7 @@ func TestValidateGradientsNames(t *testing.T) {
 		t.Fatalf("Got name %v, wanted started with Gradients/", grads0[0].Op.Name())
 	}
 
-	grads1 := Gradients(s, "", []tf.Output{y0}, []tf.Output{x})
+	grads1 := Gradients(s, []tf.Output{y0}, []tf.Output{x})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
@@ -206,28 +206,20 @@ func TestValidateGradientsNames(t *testing.T) {
 		t.Fatalf("Got name %v, wanted started with Gradients_1/", grads1[0].Op.Name())
 	}
 
-	grads2 := Gradients(s, "more_gradients", []tf.Output{y0}, []tf.Output{x})
-	if err := s.Err(); err != nil {
-		t.Fatal(err)
-	}
-	if !strings.HasPrefix(grads2[0].Op.Name(), "more_gradients/") {
-		t.Fatalf("Got name %v, wanted started with more_gradients/", grads2[0].Op.Name())
-	}
-
 	sub := s.SubScope("sub")
-	grads3 := Gradients(sub, "even_more_gradients", []tf.Output{y0}, []tf.Output{x})
+	grads3 := Gradients(sub, []tf.Output{y0}, []tf.Output{x})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(grads3[0].Op.Name(), "sub/even_more_gradients/") {
-		t.Fatalf("Got name %v, wanted started with sub/even_more_gradients/", grads3[0].Op.Name())
+	if !strings.HasPrefix(grads3[0].Op.Name(), "sub/Gradients/") {
+		t.Fatalf("Got name %v, wanted started with sub/Gradients/", grads3[0].Op.Name())
 	}
 
-	grads4 := Gradients(sub, "even_more_gradients", []tf.Output{y0}, []tf.Output{x})
+	grads4 := Gradients(sub, []tf.Output{y0}, []tf.Output{x})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(grads4[0].Op.Name(), "sub/even_more_gradients_1/") {
-		t.Fatalf("Got name %v, wanted started with sub/even_more_gradients_1/", grads4[0].Op.Name())
+	if !strings.HasPrefix(grads4[0].Op.Name(), "sub/Gradients_1/") {
+		t.Fatalf("Got name %v, wanted started with sub/Gradients_1/", grads4[0].Op.Name())
 	}
 }
-- 
GitLab


From 84209c3bd430f82ef247f668f8bf8b1f77420b6e Mon Sep 17 00:00:00 2001
From: jackonan <jackon.zhao@gmail.com>
Date: Sat, 6 Oct 2018 16:02:09 +0800
Subject: [PATCH 0029/1554] Change the key to
 'TF_DEVICE_FOR_UNINITIALIZED_VARIABLE_REPORTING'.

---
 tensorflow/python/ops/resources.py | 2 +-
 tensorflow/python/ops/variables.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/resources.py b/tensorflow/python/ops/resources.py
index 98ed07999f..6cd12b7bbb 100644
--- a/tensorflow/python/ops/resources.py
+++ b/tensorflow/python/ops/resources.py
@@ -87,7 +87,7 @@ def report_uninitialized_resources(resource_list=None,
     resource_list = shared_resources() + local_resources()
   with ops.name_scope(name):
     # Run all operations on CPU
-    local_device = os.environ.get("TF_LOCAL_DEVICE", "/cpu:0")
+    local_device = os.environ.get("TF_DEVICE_FOR_UNINITIALIZED_VARIABLE_REPORTING", "/cpu:0")
     with ops.device(local_device):
       if not resource_list:
         # Return an empty tensor so we only need to check for returned tensor
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index ac0b36efc5..784761e17e 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -2284,7 +2284,7 @@ def report_uninitialized_variables(var_list=None,
     # Run all operations on CPU
     if var_list:
       init_vars = [state_ops.is_variable_initialized(v) for v in var_list]
-    local_device = os.environ.get("TF_LOCAL_DEVICE", "/cpu:0")
+    local_device = os.environ.get("TF_DEVICE_FOR_UNINITIALIZED_VARIABLE_REPORTING", "/cpu:0")
     with ops.device(local_device):
       if not var_list:
         # Return an empty tensor so we only need to check for returned tensor
-- 
GitLab


From 7efb78e55c9993068fdc82df3d2df9d989d111e4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 10 Oct 2018 14:22:32 +0000
Subject: [PATCH 0030/1554] Add test case for tf.keras.regularizers.{l1,l2}(0.)
 with tf.get_variable

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/integration_test.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index 3c0f73b1c3..0e6fc79dd3 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.layers import core as tf_core_layers
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import rnn_cell
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
@@ -312,6 +313,15 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=0)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  def test_regularizers_with_get_variable(self):
+    # Test case for GitHub issue 22470.
+    with self.cached_session():
+      v = variable_scope.get_variable(
+          "v",
+          shape = [4, 4],
+          initializer=keras.initializers.glorot_uniform(),
+          regularizer=keras.regularizers.l2(0.))
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 406c4ad08f00aafec76e95478f4b1e7d87a5427a Mon Sep 17 00:00:00 2001
From: Alexis Louis <wimacod@gmail.com>
Date: Thu, 11 Oct 2018 14:38:47 +0200
Subject: [PATCH 0031/1554] README Updates

Clarification of code examples for fast copy-paste replication. (I did run the code myself and encountered those little missing elements)
- Estimator declaration placed after config declaration
- loss_fn() incoherent with previous loss variable declaration
---
 tensorflow/contrib/distribute/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
index 2e025765e4..2389b89eb6 100644
--- a/tensorflow/contrib/distribute/README.md
+++ b/tensorflow/contrib/distribute/README.md
@@ -131,7 +131,7 @@ def model_fn(features, labels, mode):
     return tf.estimator.EstimatorSpec(mode, loss=loss)
 
   if mode == tf.estimator.ModeKeys.TRAIN:
-    train_op = tf.train.GradientDescentOptimizer(0.2).minimize(loss_fn())
+    train_op = tf.train.GradientDescentOptimizer(0.2).minimize(loss)
     return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
 ```
 
@@ -248,10 +248,10 @@ start multi-worker training using `tf.estimator.train_and_evaluate`:
 
 ```python
 def model_main():
-  estimator = ...
   distribution = tf.contrib.distribute.CollectiveAllReduceStrategy(
       num_gpus_per_worker=2)
   config = tf.estimator.RunConfig(train_distribute=distribution)
+  estimator = tf.estimator.Estimator(model_fn=model_fn, config=config)
   train_spec = tf.estimator.TrainSpec(input_fn=input_fn)
   eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn)
   tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
@@ -324,13 +324,13 @@ start training.
 On your laptop, you can run
 
 ```python
-estimator = ...
 distribution = tf.contrib.distribute.CollectiveAllReduceStrategy(
     num_gpus_per_worker=2)
 config = tf.estimator.RunConfig(
     experimental_distribute=tf.contrib.distribute.DistributeConfig(
         train_distribute=distribution,
         remote_cluster={"worker": ["host1:port", "host2:port", "host3:port"]}))
+estimator = tf.estimator.Estimator(model_fn=model_fn, config=config)
 train_spec = tf.estimator.TrainSpec(input_fn=input_fn)
 eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn)
 tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
-- 
GitLab


From f0cd69e047eb8f29538a27a41c17da20a1c59f2b Mon Sep 17 00:00:00 2001
From: himkt <himkt@klis.tsukuba.ac.jp>
Date: Fri, 12 Oct 2018 13:17:58 +0900
Subject: [PATCH 0032/1554] Fix indentation in CRF1d

---
 tensorflow/contrib/crf/python/ops/crf.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/crf/python/ops/crf.py b/tensorflow/contrib/crf/python/ops/crf.py
index 43bb43129b..7653b48e9c 100644
--- a/tensorflow/contrib/crf/python/ops/crf.py
+++ b/tensorflow/contrib/crf/python/ops/crf.py
@@ -38,12 +38,12 @@ tf_unary_scores, tf_sequence_lengths, tf_transition_params, _ = session.run(
     [unary_scores, sequence_lengths, transition_params, train_op])
 for tf_unary_scores_, tf_sequence_length_ in zip(tf_unary_scores,
                                                  tf_sequence_lengths):
-# Remove padding.
-tf_unary_scores_ = tf_unary_scores_[:tf_sequence_length_]
+    # Remove padding.
+    tf_unary_scores_ = tf_unary_scores_[:tf_sequence_length_]
 
-# Compute the highest score and its tag sequence.
-tf_viterbi_sequence, tf_viterbi_score = tf.contrib.crf.viterbi_decode(
-    tf_unary_scores_, tf_transition_params)
+    # Compute the highest score and its tag sequence.
+    tf_viterbi_sequence, tf_viterbi_score = tf.contrib.crf.viterbi_decode(
+        tf_unary_scores_, tf_transition_params)
 """
 
 from __future__ import absolute_import
-- 
GitLab


From 650172a574504223ec2bdb328ed7c985389313d7 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 13 Oct 2018 20:30:26 +0000
Subject: [PATCH 0033/1554] Update test case for complex support of squared
 difference

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/math_ops_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index 533a00e737..62645230ee 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -217,7 +217,7 @@ class SquaredDifferenceTest(test_util.TensorFlowTestCase):
     for dtype in [np.complex64, np.complex128]:
       x = np.array([[1+3j, 2+2j, 3+1j], [4-1j, 5-2j, 6-3j]], dtype=dtype)
       y = np.array([-3+1j, -2+2j, -1+3j], dtype=dtype)
-      z = (x - y) * (x - y)
+      z = np.conj(x - y) * (x - y)
       with test_util.device(use_gpu=False):
         z_tf = self.evaluate(math_ops.squared_difference(x, y))
         self.assertAllClose(z, z_tf)
-- 
GitLab


From 3a06e557619ebaa5437d1506af058b858806e9c7 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 13 Oct 2018 20:30:47 +0000
Subject: [PATCH 0034/1554] Update squared difference implementation for
 complex types.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/cwise_ops.h | 32 +++++++++++++++--------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index 06918075a4..5afb97dc52 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -296,27 +296,31 @@ struct less_equal : std::binary_function<T, T, bool> {
   }
 };
 
-// Functor that enables composition of multiple Eigen functors.
-template <typename Scalar, typename UnaryFunctor, typename BinaryFunctor>
-struct scalar_compose_op {
+// Functor that enables squared difference functor.
+template <typename Scalar>
+struct scalar_squared_difference_op {
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar
   operator()(const Scalar& a, const Scalar& b) const {
-    return UnaryFunctor()(BinaryFunctor()(a, b));
+    const Scalar v = scalar_difference_op<Scalar>()(a, b);
+    return scalar_product_op<Scalar>()(v, scalar_conjugate_op<Scalar>()(v));
   }
   template <typename Packet>
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
   packetOp(const Packet& a, const Packet& b) const {
-    return UnaryFunctor().packetOp(BinaryFunctor().packetOp(a, b));
+    const Packet v = scalar_difference_op<Scalar>().packetOp(a, b);
+    return scalar_product_op<Scalar>().packetOp(v, scalar_conjugate_op<Scalar>().packetOp(v));
   }
 };
 
-template <typename Scalar, typename UnaryFunctor, typename BinaryFunctor>
-struct functor_traits<scalar_compose_op<Scalar, UnaryFunctor, BinaryFunctor>> {
+template <typename Scalar>
+struct functor_traits<scalar_squared_difference_op<Scalar>> {
   enum {
-    Cost = functor_traits<UnaryFunctor>::Cost +
-           functor_traits<BinaryFunctor>::Cost,
-    PacketAccess = functor_traits<UnaryFunctor>::PacketAccess &&
-                   functor_traits<BinaryFunctor>::PacketAccess
+    Cost = functor_traits<scalar_difference_op<Scalar>>::Cost +
+           functor_traits<scalar_conjugate_op<Scalar>>::Cost +
+           functor_traits<scalar_product_op<Scalar>>::Cost,
+    PacketAccess = functor_traits<scalar_difference_op<Scalar>>::PacketAccess &&
+                   functor_traits<scalar_conjugate_op<Scalar>>::PacketAccess &&
+                   functor_traits<scalar_product_op<Scalar>>::PacketAccess
   };
 };
 
@@ -709,7 +713,7 @@ struct rint : base<T, scalar_rint_op<T>> {};
 // pow(x, y) = x ^ y
 // maximum(x, y) = x > y ? x : y
 // minimum(x, y) = x < y ? x : y
-// squared_difference(x, y) = (x - y) * (x - y)
+// squared_difference(x, y) = conj(x - y) * (x - y)
 
 template <typename T>
 struct add : base<T, Eigen::internal::scalar_sum_op<T>> {
@@ -812,9 +816,7 @@ struct atan2 : base<T, scalar_atan2_op<T>> {};
 
 template <typename T>
 struct squared_difference
-    : base<T, Eigen::internal::scalar_compose_op<
-                  T, Eigen::internal::scalar_square_op<T>,
-                  Eigen::internal::scalar_difference_op<T>>> {};
+    : base<T, Eigen::internal::scalar_squared_difference_op<T>> {};
 
 template <typename T>
 struct less : base<T, Eigen::internal::less<T>, bool> {};
-- 
GitLab


From 82642d91dbe6fbba87e6a582e396ca91df1f6440 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 13 Oct 2018 20:43:59 +0000
Subject: [PATCH 0035/1554] Fix `Experimental clang-format Check` error

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/cwise_ops.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index 5afb97dc52..2682a25868 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -308,7 +308,8 @@ struct scalar_squared_difference_op {
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
   packetOp(const Packet& a, const Packet& b) const {
     const Packet v = scalar_difference_op<Scalar>().packetOp(a, b);
-    return scalar_product_op<Scalar>().packetOp(v, scalar_conjugate_op<Scalar>().packetOp(v));
+    return scalar_product_op<Scalar>().packetOp(
+        v, scalar_conjugate_op<Scalar>().packetOp(v));
   }
 };
 
-- 
GitLab


From fbd637423bc82662f010cc10a320c573b2eda172 Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Mon, 15 Oct 2018 08:18:16 +0800
Subject: [PATCH 0036/1554] Fix comment for AddGradients in graph.go

---
 tensorflow/go/graph.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index 096a182840..fb191fb51e 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -147,10 +147,11 @@ func (g *Graph) Operations() []Operation {
 	return ops
 }
 
-// AddGradients adds operations to compute the partial derivatives of sum of `y`s w.r.t `x`s,
-// i.e., d(y_1 + y_2 + ...)/dx_1, d(y_1 + y_2 + ...)/dx_2...
-// This methods allows to caller to pass a custom name prefix to the operations
-// added to a graph to compute the gradients.
+// AddGradients add operations to compute the partial derivatives of the sum of tensors in y
+// with respect to tensors in x, i.e., d(y[0] + y[1] + ...) / d x[0], d(y[0] + y[1] + ... ) / d x[1] etc.
+//
+// prefix, if non-empty, is the name prefix used for all operations added to the graph to compute
+// these gradients.
 func (g *Graph) AddGradients(prefix string, y []Output, x []Output, dx []Output) ([]Output, error) {
 	var (
 		cprefix *C.char
-- 
GitLab


From 1bae040045254e1ca51366391d52bb79b8c1b067 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 15 Oct 2018 22:21:21 +0000
Subject: [PATCH 0037/1554] Update the documentation for tf.fft

This fix is related to 17332. While complex128 has been
added in tf, the documentation still says:
```
tf.spectral.fft
input: A Tensor. Must be one of the following types: complex64, complex128. A complex64 tensor.
```

This fix updates the documentation so that it matches the actual behavior.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/api_def/base_api/api_def_FFT.pbtxt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_FFT.pbtxt b/tensorflow/core/api_def/base_api/api_def_FFT.pbtxt
index 4e48d6c169..0ba2327371 100644
--- a/tensorflow/core/api_def/base_api/api_def_FFT.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FFT.pbtxt
@@ -3,13 +3,13 @@ op {
   in_arg {
     name: "input"
     description: <<END
-A complex64 tensor.
+A complex tensor.
 END
   }
   out_arg {
     name: "output"
     description: <<END
-A complex64 tensor of the same shape as `input`. The inner-most
+A complex tensor of the same shape as `input`. The inner-most
   dimension of `input` is replaced with its 1D Fourier transform.
 
 @compatibility(numpy)
-- 
GitLab


From 8e820ae830bf24f83119db0dbb3f21937a3cc33c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 15 Oct 2018 22:23:22 +0000
Subject: [PATCH 0038/1554] Update documentation for tf.ifft/ifft2d/fft2d

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/api_def/base_api/api_def_FFT2D.pbtxt  | 4 ++--
 tensorflow/core/api_def/base_api/api_def_IFFT.pbtxt   | 4 ++--
 tensorflow/core/api_def/base_api/api_def_IFFT2D.pbtxt | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_FFT2D.pbtxt b/tensorflow/core/api_def/base_api/api_def_FFT2D.pbtxt
index 555f8e6067..c7b780a56f 100644
--- a/tensorflow/core/api_def/base_api/api_def_FFT2D.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FFT2D.pbtxt
@@ -3,13 +3,13 @@ op {
   in_arg {
     name: "input"
     description: <<END
-A complex64 tensor.
+A complex tensor.
 END
   }
   out_arg {
     name: "output"
     description: <<END
-A complex64 tensor of the same shape as `input`. The inner-most 2
+A complex tensor of the same shape as `input`. The inner-most 2
   dimensions of `input` are replaced with their 2D Fourier transform.
 
 @compatibility(numpy)
diff --git a/tensorflow/core/api_def/base_api/api_def_IFFT.pbtxt b/tensorflow/core/api_def/base_api/api_def_IFFT.pbtxt
index b793c99cf7..c17a840005 100644
--- a/tensorflow/core/api_def/base_api/api_def_IFFT.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_IFFT.pbtxt
@@ -3,13 +3,13 @@ op {
   in_arg {
     name: "input"
     description: <<END
-A complex64 tensor.
+A complex tensor.
 END
   }
   out_arg {
     name: "output"
     description: <<END
-A complex64 tensor of the same shape as `input`. The inner-most
+A complex tensor of the same shape as `input`. The inner-most
   dimension of `input` is replaced with its inverse 1D Fourier transform.
 
 @compatibility(numpy)
diff --git a/tensorflow/core/api_def/base_api/api_def_IFFT2D.pbtxt b/tensorflow/core/api_def/base_api/api_def_IFFT2D.pbtxt
index 7f38f14308..7458d233ec 100644
--- a/tensorflow/core/api_def/base_api/api_def_IFFT2D.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_IFFT2D.pbtxt
@@ -3,13 +3,13 @@ op {
   in_arg {
     name: "input"
     description: <<END
-A complex64 tensor.
+A complex tensor.
 END
   }
   out_arg {
     name: "output"
     description: <<END
-A complex64 tensor of the same shape as `input`. The inner-most 2
+A complex tensor of the same shape as `input`. The inner-most 2
   dimensions of `input` are replaced with their inverse 2D Fourier transform.
 
 @compatibility(numpy)
-- 
GitLab


From 310f454e928c3e78d9126cb68190ed1cfc6b28d5 Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Tue, 16 Oct 2018 16:53:52 +0800
Subject: [PATCH 0039/1554] Stop using uniqueName for Gradients in gradient.go

To keep the code style consistent with other functions.
---
 tensorflow/go/op/gradients.go      |  2 +-
 tensorflow/go/op/gradients_test.go | 37 +++++++++++-------------------
 2 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/tensorflow/go/op/gradients.go b/tensorflow/go/op/gradients.go
index 2397e40bf4..2eaa7e70ab 100644
--- a/tensorflow/go/op/gradients.go
+++ b/tensorflow/go/op/gradients.go
@@ -28,7 +28,7 @@ import tf "github.com/tensorflow/tensorflow/tensorflow/go"
 //  return the partial derivatives
 func Gradients(scope *Scope, y []tf.Output, x []tf.Output, dx ...tf.Output) (output []tf.Output) {
 	var err error
-	if output, err = scope.graph.AddGradients(scope.opName(scope.uniqueName("Gradients")), y, x, dx); err != nil {
+	if output, err = scope.graph.AddGradients(scope.opName("Gradients"), y, x, dx); err != nil {
 		scope.UpdateErr("Gradients", err)
 		return
 	}
diff --git a/tensorflow/go/op/gradients_test.go b/tensorflow/go/op/gradients_test.go
index 2bcb3e88eb..1febd08366 100644
--- a/tensorflow/go/op/gradients_test.go
+++ b/tensorflow/go/op/gradients_test.go
@@ -44,8 +44,9 @@ func TestAddGradients(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float)
 	}
 
-	grads1 := Gradients(s, []tf.Output{y2}, []tf.Output{x1, x2})
-	if err := s.Err(); err != nil {
+	sub := s.SubScope("sub")
+	grads1 := Gradients(sub, []tf.Output{y2}, []tf.Output{x1, x2})
+	if err := sub.Err(); err != nil {
 		t.Fatal(err)
 	}
 	if len(grads1) != 2 {
@@ -58,7 +59,7 @@ func TestAddGradients(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads1[1].DataType(), tf.Float)
 	}
 
-	graph, err := s.Finalize()
+	graph, err := sub.Finalize()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -150,8 +151,9 @@ func TestAddGradientsWithInitialValues(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads0[0].DataType(), tf.Float)
 	}
 
-	grads1 := Gradients(s, []tf.Output{y0}, []tf.Output{x}, grads0[0])
-	if err := s.Err(); err != nil {
+	sub := s.SubScope("sub")
+	grads1 := Gradients(sub, []tf.Output{y0}, []tf.Output{x}, grads0[0])
+	if err := sub.Err(); err != nil {
 		t.Fatal(err)
 	}
 	if len(grads1) != 1 {
@@ -161,7 +163,7 @@ func TestAddGradientsWithInitialValues(t *testing.T) {
 		t.Fatalf("Got DataType %v, wanted %v", grads1[0].DataType(), tf.Float)
 	}
 
-	graph, err := s.Finalize()
+	graph, err := sub.Finalize()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -198,28 +200,17 @@ func TestValidateGradientsNames(t *testing.T) {
 		t.Fatalf("Got name %v, wanted started with Gradients/", grads0[0].Op.Name())
 	}
 
-	grads1 := Gradients(s, []tf.Output{y0}, []tf.Output{x})
-	if err := s.Err(); err != nil {
-		t.Fatal(err)
-	}
-	if !strings.HasPrefix(grads1[0].Op.Name(), "Gradients_1/") {
-		t.Fatalf("Got name %v, wanted started with Gradients_1/", grads1[0].Op.Name())
-	}
-
 	sub := s.SubScope("sub")
-	grads3 := Gradients(sub, []tf.Output{y0}, []tf.Output{x})
+	grads1 := Gradients(sub, []tf.Output{y0}, []tf.Output{x})
 	if err := s.Err(); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(grads3[0].Op.Name(), "sub/Gradients/") {
-		t.Fatalf("Got name %v, wanted started with sub/Gradients/", grads3[0].Op.Name())
+	if !strings.HasPrefix(grads1[0].Op.Name(), "sub/Gradients/") {
+		t.Fatalf("Got name %v, wanted started with sub/Gradients/", grads1[0].Op.Name())
 	}
 
-	grads4 := Gradients(sub, []tf.Output{y0}, []tf.Output{x})
-	if err := s.Err(); err != nil {
-		t.Fatal(err)
-	}
-	if !strings.HasPrefix(grads4[0].Op.Name(), "sub/Gradients_1/") {
-		t.Fatalf("Got name %v, wanted started with sub/Gradients_1/", grads4[0].Op.Name())
+	Gradients(sub, []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err == nil {
+		t.Error("Gradients should have failed if executed more than once for scope of the same namespace")
 	}
 }
-- 
GitLab


From 02987bd1e3459ba7ecfe689bb182ce8e692b70c4 Mon Sep 17 00:00:00 2001
From: "Yuan (Terry) Tang" <terrytangyuan@gmail.com>
Date: Tue, 16 Oct 2018 09:26:55 -0400
Subject: [PATCH 0040/1554] Add bullet points so Reduction values are clearer

---
 tensorflow/python/ops/losses/losses_impl.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 8a8a81ab5c..03cdc69ae5 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -39,13 +39,14 @@ class Reduction(object):
   """Types of loss reduction.
 
   Contains the following values:
-  `NONE`: Un-reduced weighted losses with the same shape as input.
-  `SUM`: Scalar sum of weighted losses.
-  `MEAN`: Scalar `SUM` divided by sum of weights.
-  `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
-  `SUM_OVER_NONZERO_WEIGHTS`: Scalar `SUM` divided by number of non-zero
+
+  * `NONE`: Un-reduced weighted losses with the same shape as input.
+  * `SUM`: Scalar sum of weighted losses.
+  * `MEAN`: Scalar `SUM` divided by sum of weights.
+  * `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+  * `SUM_OVER_NONZERO_WEIGHTS`: Scalar `SUM` divided by number of non-zero
      weights.
-  `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`.
+  * `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`.
   """
 
   NONE = "none"
-- 
GitLab


From 9cc0aeb553d168b29abede0358f3fb3903cd8fd6 Mon Sep 17 00:00:00 2001
From: Gitea <jpds@protonmail.com>
Date: Thu, 18 Oct 2018 09:44:35 +0100
Subject: [PATCH 0041/1554] [Docker] Upgraded images to Ubuntu 18.04 LTS.

---
 tensorflow/tools/docker/Dockerfile                   | 4 ++--
 tensorflow/tools/docker/Dockerfile.devel             | 4 ++--
 tensorflow/tools/docker/Dockerfile.devel-mkl         | 4 ++--
 tensorflow/tools/docker/Dockerfile.devel-mkl-horovod | 4 ++--
 tensorflow/tools/docker/Dockerfile.mkl               | 4 ++--
 tensorflow/tools/docker/Dockerfile.mkl-horovod       | 4 ++--
 6 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile
index 205128ad58..6676de02a4 100644
--- a/tensorflow/tools/docker/Dockerfile
+++ b/tensorflow/tools/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         curl \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
+        libpng-dev \
         libzmq3-dev \
         pkg-config \
         python \
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index a3893a2713..c256dd364e 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
+        libpng-dev \
         libzmq3-dev \
         pkg-config \
         python-dev \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-mkl b/tensorflow/tools/docker/Dockerfile.devel-mkl
index bd2883ddba..2341c0e8cc 100755
--- a/tensorflow/tools/docker/Dockerfile.devel-mkl
+++ b/tensorflow/tools/docker/Dockerfile.devel-mkl
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
 
@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
+        libpng-dev \
         libzmq3-dev \
         libssl-dev \
         pkg-config \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod b/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod
index df084e029c..5e24617b21 100755
--- a/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod
+++ b/tensorflow/tools/docker/Dockerfile.devel-mkl-horovod
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="Cong Xu <cong.xu@intel.com>"
 
@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
+        libpng-dev \
         libzmq3-dev \
         pkg-config \
         python-dev \
diff --git a/tensorflow/tools/docker/Dockerfile.mkl b/tensorflow/tools/docker/Dockerfile.mkl
index ac41cffe4b..dad27697fa 100755
--- a/tensorflow/tools/docker/Dockerfile.mkl
+++ b/tensorflow/tools/docker/Dockerfile.mkl
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
 
@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         curl \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
+        libpng-dev \
         libzmq3-dev \
         pkg-config \
         ${PYTHON} \
diff --git a/tensorflow/tools/docker/Dockerfile.mkl-horovod b/tensorflow/tools/docker/Dockerfile.mkl-horovod
index 0432cd5e80..19dc45c62c 100755
--- a/tensorflow/tools/docker/Dockerfile.mkl-horovod
+++ b/tensorflow/tools/docker/Dockerfile.mkl-horovod
@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="Cong Xu <cong.xu@intel.com>"
 
@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         curl \
         libfreetype6-dev \
         libhdf5-serial-dev \
-        libpng12-dev \
+        libpng-dev \
         libzmq3-dev \
         pkg-config \
         python \
-- 
GitLab


From 4f9d57337b71fe0ab3f25696db456e6a446ef54a Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Mon, 22 Oct 2018 10:53:25 +0800
Subject: [PATCH 0042/1554] Redundant transpose removal: transpose + conv2d +
 transpose -> conv2d.

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 514 ++++++++++++++++--
 tensorflow/core/graph/mkl_layout_pass_test.cc | 295 ++++++++++
 2 files changed, 771 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 69735aac02..233c5ab39b 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -25,6 +25,8 @@ limitations under the License.
 #include <unordered_set>
 #include <utility>
 #include <vector>
+#include <tuple>
+
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -310,6 +312,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     csinfo_.slice = "Slice";
     csinfo_.softmax = "Softmax";
     csinfo_.split = "Split";
+    csinfo_.transpose = "Transpose";
     // Element-wise ops. Ensure you also add any new ops to IsOpElementWise
     // in the MklUtil.h (IsMklElementWiseOp method) to ensure that the
     // MklInputConversion op is added before it.
@@ -508,6 +511,33 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     minfo_.push_back({csinfo_.conv2d_grad_filter, csinfo_.bias_add_grad,
                       csinfo_.conv2d_grad_filter_with_bias,
                       GetConv2DBackpropFilterOrBiasAddGrad});
+
+    //
+    // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D (NHWC) + Transpose (NHWC->
+    // NCHW) " => "Conv2D (NCHW). Such patterns occur frequently in Keras.
+    // Note: we use the term "merge" is to combine (exactly) 2 nodes into one, while "fusion" is
+    // for 3+ nodes situation.
+    //
+
+    // Transpose + Conv2d + Transpose:
+    std::vector<int> transpose_to_nhwc = { NCHW::dim::N, NCHW::dim::H, NCHW::dim::W, NCHW::dim::C };
+    std::vector<int> transpose_to_nchw = { NHWC::dim::N, NHWC::dim::C, NHWC::dim::H, NHWC::dim::W };
+    auto CheckForTransposeToNHWC =
+        std::bind(CheckForTranspose, std::placeholders::_1, transpose_to_nhwc);
+    auto CheckForConv2dOp =
+        std::bind(CheckForMklOp, std::placeholders::_1, csinfo_.conv2d);
+    auto CheckForTransposeToNCHW =
+        std::bind(CheckForTranspose, std::placeholders::_1, transpose_to_nchw);
+    auto FuseConv2D =
+        std::bind(FuseTransposeMklOpTranspose, std::placeholders::_1,
+                  std::placeholders::_2, std::placeholders::_3, "NCHW");
+    finfo_.push_back({
+      "transpose-elimination for Conv2D", {
+        CheckForTransposeToNHWC, CheckForConv2dOp, CheckForTransposeToNCHW
+      },
+      // CheckForMklOp
+      FuseConv2D, CopyAttrsConv
+    });
   }
 
   // Standard interface to run pass
@@ -530,7 +560,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string name;      // Original name of op of the node in the graph
     string new_name;  // New name of the op of the node in the graph
     // A function handler to copy attributes from an old node to a new node.
-    std::function<void(const Node*, NodeBuilder*)> copy_attrs;
+    std::function<void(const Node*, NodeBuilder*, bool)> copy_attrs;
     // A rule under which to rewrite this node
     std::function<bool(const Node*)> rewrite_rule;
   } RewriteInfo;
@@ -560,6 +590,42 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     std::function<Node*(const Node*)> get_node_to_be_merged;
   } MergeInfo;
 
+  // structure to specify information used in node fusion of 2+ operators
+  typedef struct {
+    std::string pattern_name; // name to describe this pattern, such as
+                              // "Transpose_Mklop_Transpose".
+    std::vector<std::function<bool(const Node *)> >
+        node_checkers; // extra restriction checker for these ops
+    std::function<
+        Status(std::unique_ptr<Graph> *, std::vector<Node *> &,
+               std::function<void(const Node *, NodeBuilder *nb, bool)>)>
+        fuse_func;
+    std::function<void(const Node *, NodeBuilder *nb, bool)> copy_attrs;
+  } FusionInfo;
+
+  //
+  // dimension indices for 2D tensor.
+  // 
+  struct NCHW {
+    enum dim { N = 0, C = 1, H = 2, W = 3 };
+  };
+
+  struct NHWC {
+    enum dim { N = 0, H = 1, W = 2, C = 3 };
+  };
+
+
+  //
+  // dimension indices for 3D tensor.
+  //
+  struct NCDHW {
+    enum dim { N = 0, C = 1, D = 2, H = 3, W = 4 };
+  };
+
+  struct NDHWC {
+    enum dim { N = 0, D = 1, H = 2, W = 3, C = 4 };
+  };
+
   /// Structure to store all constant strings
   /// NOTE: names are alphabetically sorted.
   typedef struct {
@@ -619,6 +685,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string requantize;
     string tanh;
     string tanh_grad;
+    string transpose;
     string reshape;
     string slice;
     string softmax;
@@ -637,6 +704,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   /// Maintain info about nodes to be merged
   std::vector<MergeInfo> minfo_;
 
+  /// Maintain info about nodes to be fused
+  std::vector<FusionInfo> finfo_;
+
   /// Maintain structure of constant strings
   static ConstStringsInfo csinfo_;
 
@@ -815,6 +885,121 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return n;
   }
 
+  // Return a node that can be fused with input node 'n'
+  //
+  // @return tuple. If we can find such nodes, the first
+  // element of the tuple is a true. Otherwise, it's false.
+  std::tuple<bool, std::vector<Node *>, const MklLayoutRewritePass::FusionInfo>
+      CheckForNodeFusion(Node *n) const;
+
+  // Fuse nodes in the vector "nodes"
+  Status FuseNode(std::unique_ptr<Graph> *g, std::vector<Node *> &nodes,
+                  const MklLayoutRewritePass::FusionInfo fi);
+
+  static Status FuseTransposeMklOpTranspose(
+      std::unique_ptr<Graph> *g, std::vector<Node *> &nodes,
+      std::function<void(const Node *, NodeBuilder *nb, bool)> copy_attrs,
+      string data_format);
+
+  static bool CheckForTranspose(const Node *node, std::vector<int> perm) {
+    //
+    // Check node node, to see if it's "Transpose"
+    //
+    if (node->type_string() != "Transpose")
+      return false;
+
+    //
+    // Check if has out control edge. If true, this is a training graph.
+    // Currently we focus on inference and do no fusion in training.
+    //
+    for (const Edge *e : node->out_edges()) {
+      if (e->IsControlEdge()) {
+        return false;
+      }
+    }
+
+    //
+    // If "Transpose" has input control edges, don't fuse on it.
+    //
+    for (const Edge *e : node->in_edges()) {
+      if (e->IsControlEdge()) {
+        return false;
+      }
+    }
+
+    //
+    // If "Transpose" has multiple output data edges, also don't fuse it.
+    //
+    if (node->num_outputs() > 1 || node->out_edges().size() > 1)
+      return false;
+
+    // Check "perm" attribute, make sure it's what we want.
+    //
+    for (const Edge *e : node->in_edges()) {
+      if (!e->IsControlEdge()) {
+        const Node *perm_node = e->src();
+
+        const int kPermTensorIndex = 1;
+        if (perm_node->type_string() == "Const" && e->dst_input() == kPermTensorIndex) {
+          // we find the "perm" node, now try to retrieve its value.
+          const TensorProto *proto = nullptr;
+          CHECK_EQ(GetNodeAttr(perm_node->def(), "value", &proto).ok(), true);
+
+          DataType type;
+          GetNodeAttr(perm_node->def(), "dtype", &type);
+
+          //
+          // Here we directly access to the "tensor_context", rather than
+          // "int_val". This is because we find "int_val" is
+          // not set properly under some circumstances.
+          //
+          if (type == DT_INT32) {
+            const int type_size = 4;
+            const int *tensor_content = reinterpret_cast<const int *>(proto->tensor_content().c_str());
+            const int tensor_content_size = proto->tensor_content().size() / type_size;
+  
+            std::vector<int> perm_value(tensor_content, tensor_content + tensor_content_size);
+
+            return perm_value == perm;
+
+          } else if (type == DT_INT64) {
+            const int type_size = 8;
+            const long *tensor_content = reinterpret_cast<const long *>(proto->tensor_content().c_str());
+            const int tensor_content_size = proto->tensor_content().size() / type_size;
+  
+            std::vector<long> perm_value(tensor_content, tensor_content + tensor_content_size);
+            std::vector<long> long_perm(perm.cbegin(), perm.cend());
+
+            return perm_value == long_perm;
+
+          }
+
+          return false;
+        }
+      }
+    }
+
+    return false;
+  }
+
+  static bool CheckForMklOp(const Node *node, string name = "") {
+    if (!name.empty() && node->type_string() != name) {
+      return false;
+    }
+
+    // if mklop has multiple outputs, don't fuse it.
+    if (node->num_outputs() > 1)
+      return false;
+
+    if (node->out_edges().size() > 1)
+      return false;
+
+    DataType T;
+    TF_CHECK_OK(GetNodeAttr(node->def(), "T", &T));
+    return mkl_op_registry::IsMklOp(
+        mkl_op_registry::GetMklOpName(node->type_string()), T);
+  }
+
   // Check if the node 'n' has any applicable rewrite rule
   // We check for 2 scenarios for rewrite.
   //
@@ -1070,22 +1255,39 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   // We need operator-specific function to copy attributes because the framework
   // does not provide any generic function for it.
   // NOTE: names are alphabetically sorted.
-  static void CopyAttrsAddN(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsBiasAddGrad(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsConcat(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsConcatV2(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsConv(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsDataType(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsQuantizedPooling(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsQuantizedConv2D(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsQuantizedConcat(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsReshape(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsRequantize(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsSlice(const Node* orig_node, NodeBuilder* nb);
-  static void CopyAttrsSplit(const Node* orig_node, NodeBuilder* nb);
+  static void CopyAttrsAddN(const Node *orig_node, NodeBuilder *nb,
+                            bool change_format = false);
+  static void CopyAttrsBiasAddGrad(const Node *orig_node, NodeBuilder *nb,
+                                   bool change_format = false);
+  static void CopyAttrsConcat(const Node *orig_node, NodeBuilder *nb,
+                              bool change_format = false);
+  static void CopyAttrsConcatV2(const Node *orig_node, NodeBuilder *nb,
+                                bool change_format = false);
+  static void CopyAttrsConv(const Node *orig_node, NodeBuilder *nb,
+                            bool change_format = false);
+  static void CopyAttrsDataType(const Node *orig_node, NodeBuilder *nb,
+                                bool change_format = false);
+  static void CopyAttrsFusedBatchNorm(const Node *orig_node, NodeBuilder *nb,
+                                      bool change_format = false);
+  static void CopyAttrsLRN(const Node *orig_node, NodeBuilder *nb,
+                           bool change_format = false);
+  static void CopyAttrsPooling(const Node *orig_node, NodeBuilder *nb,
+                               bool change_format = false);
+  static void CopyAttrsQuantizedPooling(const Node* orig_node, NodeBuilder* nb,
+                                        bool change_format = false);
+  static void CopyAttrsQuantizedConv2D(const Node* orig_node, NodeBuilder* nb,
+                                       bool change_format = false);
+  static void CopyAttrsQuantizedConcat(const Node* orig_node, NodeBuilder* nb,
+                                       bool change_format = false);
+  static void CopyAttrsReshape(const Node *orig_node, NodeBuilder *nb,
+                               bool change_format = false);
+  static void CopyAttrsRequantize(const Node* orig_node, NodeBuilder* nb,
+                                  bool change_format = false);
+  static void CopyAttrsSlice(const Node* orig_node, NodeBuilder* nb,
+                             bool change_format = false);
+  static void CopyAttrsSplit(const Node *orig_node, NodeBuilder *nb,
+                             bool change_format = false);
+
 
   // Generate a graph node in graph 'g' representing a dummy Mkl tensor node,
   // using node for original node 'orig_node' and return it in '*out'.
@@ -1586,8 +1788,8 @@ void MklLayoutRewritePass::AddWorkSpaceEdgeIfNeeded(
 // Op-specific functions to copy attributes from old node to new node
 //////////////////////////////////////////////////////////////////////////
 
-void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node,
-                                         NodeBuilder* nb) {
+void MklLayoutRewritePass::CopyAttrsConv(const Node *orig_node, NodeBuilder *nb,
+                                         bool change_format) {
   DataType T;
   string data_format;
   string padding;
@@ -1599,18 +1801,72 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node,
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
-  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
 
   // Add attributes to new node.
   nb->Attr("T", T);
-  nb->Attr("strides", strides);
-  nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
-  nb->Attr("data_format", data_format);
+
+  if (!change_format) {
+    nb->Attr("strides", strides);
+    nb->Attr("dilations", dilations);
+
+    TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
+    nb->Attr("data_format", data_format);
+  } else {
+    std::vector<int32> new_strides;
+    std::vector<int32> new_dilations;
+    if (strides.size() == 5) {
+      //
+      // "strides" and "dilations" also need to be changed according to "data_format",
+      // in this case, is "NDHWC" to "NCDHW".
+      //
+
+      new_strides = { 
+                      strides[NDHWC::dim::N], 
+                      strides[NDHWC::dim::C],
+                      strides[NDHWC::dim::D],
+                      strides[NDHWC::dim::H],
+                      strides[NDHWC::dim::W]
+                    };
+      nb->Attr("strides", new_strides);
+
+      new_dilations = { 
+                      dilations[NDHWC::dim::N], 
+                      dilations[NDHWC::dim::C],
+                      dilations[NDHWC::dim::D],
+                      dilations[NDHWC::dim::H],
+                      dilations[NDHWC::dim::W]
+                    };
+      nb->Attr("dilations", new_dilations);
+
+    } else {
+      //
+      // "strides" and "dilations" also need to be changed according to "data_format",
+      // in this case, is "NHWC" to "NCHW".
+      //
+      
+      new_strides = {
+                      strides[NHWC::dim::N],
+                      strides[NHWC::dim::C],
+                      strides[NHWC::dim::H],
+                      strides[NHWC::dim::W]
+                    };
+      nb->Attr("strides", new_strides);
+
+      new_dilations = {
+                      dilations[NHWC::dim::N],
+                      dilations[NHWC::dim::C],
+                      dilations[NHWC::dim::H],
+                      dilations[NHWC::dim::W]
+                    };
+      nb->Attr("dilations", new_dilations);
+    }
+  }
 }
 
 void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node,
-                                         NodeBuilder* nb) {
+                                         NodeBuilder* nb,
+                                         bool change_format) {
   DataType T;
   int N;
 
@@ -1624,7 +1880,8 @@ void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsBiasAddGrad(const Node* orig_node,
-                                                NodeBuilder* nb) {
+                                                NodeBuilder* nb,
+                                                bool change_format) {
   DataType T;
   string data_format;
   std::vector<int32> strides;
@@ -1641,7 +1898,8 @@ void MklLayoutRewritePass::CopyAttrsBiasAddGrad(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsLRN(const Node* orig_node,
-                                        NodeBuilder* nb) {
+                                        NodeBuilder* nb,
+                                        bool change_format) {
   DataType T;
   int depth_radius;
   float bias;
@@ -1664,7 +1922,8 @@ void MklLayoutRewritePass::CopyAttrsLRN(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsPooling(const Node* orig_node,
-                                            NodeBuilder* nb) {
+                                            NodeBuilder* nb,
+                                            bool change_format) {
   DataType T;
   string data_format;
   string padding;
@@ -1686,7 +1945,8 @@ void MklLayoutRewritePass::CopyAttrsPooling(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsDataType(const Node* orig_node,
-                                             NodeBuilder* nb) {
+                                             NodeBuilder* nb,
+                                             bool change_format) {
   DataType T;
 
   // Get all attributes from old node.
@@ -1697,7 +1957,8 @@ void MklLayoutRewritePass::CopyAttrsDataType(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsQuantizedPooling(const Node* orig_node,
-                                                     NodeBuilder* nb) {
+                                                     NodeBuilder* nb,
+                                                     bool change_format) {
   DataType T;
   string data_format;
   string padding;
@@ -1717,7 +1978,8 @@ void MklLayoutRewritePass::CopyAttrsQuantizedPooling(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
-                                                    NodeBuilder* nb) {
+                                                    NodeBuilder* nb,
+                                                    bool change_format) {
   DataType Tinput, Tfilter, out_type;
   string padding;
   string data_format("NHWC");
@@ -1747,7 +2009,8 @@ void MklLayoutRewritePass::CopyAttrsQuantizedConv2D(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsRequantize(const Node* orig_node,
-                                               NodeBuilder* nb) {
+                                               NodeBuilder* nb,
+                                               bool change_format) {
   DataType Tinput, out_type;
 
   // Get all attributes from old node.
@@ -1760,7 +2023,8 @@ void MklLayoutRewritePass::CopyAttrsRequantize(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node,
-                                            NodeBuilder* nb) {
+                                            NodeBuilder* nb,
+                                            bool change_format) {
   DataType T;
   DataType Tshape;
 
@@ -1773,7 +2037,8 @@ void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsSlice(const Node* orig_node,
-                                          NodeBuilder* nb) {
+                                          NodeBuilder* nb,
+                                          bool change_format) {
   DataType T;
   DataType Index;
 
@@ -1786,7 +2051,8 @@ void MklLayoutRewritePass::CopyAttrsSlice(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsSplit(const Node* orig_node,
-                                          NodeBuilder* nb) {
+                                          NodeBuilder* nb,
+                                          bool change_format) {
   DataType T;
   string data_format;
   int num_split;
@@ -1803,7 +2069,8 @@ void MklLayoutRewritePass::CopyAttrsSplit(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsConcat(const Node* orig_node,
-                                           NodeBuilder* nb) {
+                                           NodeBuilder* nb,
+                                           bool change_format) {
   DataType T;
   int N;
 
@@ -1817,7 +2084,8 @@ void MklLayoutRewritePass::CopyAttrsConcat(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsConcatV2(const Node* orig_node,
-                                             NodeBuilder* nb) {
+                                             NodeBuilder* nb,
+                                             bool change_format) {
   DataType T;
   int N;
   DataType tidx;
@@ -1834,7 +2102,8 @@ void MklLayoutRewritePass::CopyAttrsConcatV2(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsFusedBatchNorm(const Node* orig_node,
-                                                   NodeBuilder* nb) {
+                                                   NodeBuilder* nb,
+                                                   bool change_format) {
   DataType T;
   float epsilon;
   string data_format;
@@ -2231,7 +2500,8 @@ Status MklLayoutRewritePass::RewriteNode(std::unique_ptr<Graph>* g,
     return s;
   }
 
-  ri->copy_attrs(const_cast<const Node*>(orig_node), &nb);
+  const bool kPartialCopyAttrs = false;
+  ri->copy_attrs(const_cast<const Node*>(orig_node), &nb, kPartialCopyAttrs);
 
   // Set the Mkl layer label for this op.
   if (DataTypeIsQuantized(orig_node->input_type(0)) ||
@@ -2391,6 +2661,151 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const {
   return nullptr;
 }
 
+//////////////////////////////////////////////////////////////////////////
+//           Helper functions for node fusion
+//////////////////////////////////////////////////////////////////////////
+Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
+    std::unique_ptr<Graph> *g, std::vector<Node *> &nodes,
+    std::function<void(const Node *, NodeBuilder *nb, bool)> copy_attrs,
+    string data_format) {
+  Node *transpose_to_nhwc = nodes[0];
+  Node *mklop = nodes[1];
+  Node *transpose_to_nchw = nodes[2];
+
+  const int transpose_nhwc_num_inputs = transpose_to_nhwc->num_inputs();
+  gtl::InlinedVector<Node *, 4> transpose_nhwc_control_edges;
+  gtl::InlinedVector<std::pair<Node *, int>, 4> transpose_nhwc_in(
+      transpose_nhwc_num_inputs);
+  FillInputs(transpose_to_nhwc, &transpose_nhwc_control_edges,
+             &transpose_nhwc_in);
+
+  const int mklop_num_inputs = mklop->num_inputs();
+  gtl::InlinedVector<Node *, 4> mklop_control_edges;
+  gtl::InlinedVector<std::pair<Node *, int>, 4> mklop_in(mklop_num_inputs);
+  FillInputs(mklop, &mklop_control_edges, &mklop_in);
+
+  const int transpose_nchw_num_inputs = transpose_to_nchw->num_inputs();
+  gtl::InlinedVector<Node *, 4> transpose_nchw_control_edges;
+  gtl::InlinedVector<std::pair<Node *, int>, 4> transpose_nchw_in(
+      transpose_nchw_num_inputs);
+  FillInputs(transpose_to_nhwc, &transpose_nchw_control_edges,
+             &transpose_nchw_in);
+
+  // We will use the node name of Conv2d as the name of new node
+  // Build new node. We use same name as original node, but change the op
+  // name.
+  NodeBuilder nb(mklop->name(), mklop->type_string());
+
+  for (int i = 0; i < mklop_num_inputs; i++) {
+    if (mklop_in[i].first == transpose_to_nhwc) {
+      // Fill "x":
+      nb.Input(transpose_nhwc_in[0].first, transpose_nhwc_in[0].second);
+    } else {
+      // Fill inputs other than "x":
+      nb.Input(mklop_in[i].first, mklop_in[i].second);
+    }
+  }
+
+  copy_attrs(const_cast<const Node *>(mklop), &nb, true);
+  nb.Attr("data_format", data_format);
+
+  // Copy the device assigned to old node to new node.
+  nb.Device(mklop->def().device());
+
+  // Create node.
+  Node *new_node;
+  TF_CHECK_OK(nb.Finalize(&**g, &new_node));
+  CHECK_NOTNULL(new_node);
+
+  // Fill outputs.
+  for (const Edge *e : transpose_to_nchw->out_edges()) {
+    if (!e->IsControlEdge()) {
+      const int kConv2DWithBiasOutputSlot = 0;
+      CHECK_NOTNULL((*g)->AddEdge(new_node, kConv2DWithBiasOutputSlot, e->dst(),
+                                  e->dst_input()));
+    }
+  }
+
+  // Copy device assigned to old node to new node.
+  new_node->set_assigned_device_name(mklop->assigned_device_name());
+
+  (*g)->RemoveNode(transpose_to_nhwc);
+  (*g)->RemoveNode(mklop);
+  (*g)->RemoveNode(transpose_to_nchw);
+
+  return Status::OK();
+}
+
+Status
+MklLayoutRewritePass::FuseNode(std::unique_ptr<Graph> *g,
+                               std::vector<Node *> &nodes,
+                               const MklLayoutRewritePass::FusionInfo fi) {
+  return fi.fuse_func(g, nodes, fi.copy_attrs);
+}
+
+std::tuple<bool, std::vector<Node *>, const MklLayoutRewritePass::FusionInfo>
+MklLayoutRewritePass::CheckForNodeFusion(Node *a) const {
+  bool found_pattern = false;
+  std::vector<Node *> nodes;
+  const FusionInfo *fi_ptr = nullptr;
+
+  for (auto fi = finfo_.begin(); fi != finfo_.end(); ++fi) {
+    assert(fi->ops.size() == fi->node_checkers.size());
+    nodes.clear();
+    fi_ptr = &*fi;
+    //
+    // Make sure node "a" and its succeding nodes (b, c ...), match the pattern
+    // defined in fusion info (ops[0], ops[1], ...),
+    // aka. "a->b->c" matches "op1->op2->op3"
+    //
+
+    // Initialize "current_node" as node "a".
+    Node *current_node = a;
+    for (auto node_index = 0; node_index < fi->node_checkers.size();
+         ++node_index) {
+      // Make sure current node meet the requirement of corresponding node
+      // checker.
+      auto check_node = fi->node_checkers[node_index];
+      if (current_node == nullptr ||
+          (check_node && check_node(current_node) == false)) {
+        found_pattern = false;
+        nodes.clear();
+        break;
+      }
+
+      // Add current_node to "fusion_nodes":
+      nodes.push_back(current_node);
+
+      // If current node is not the last node we want to check, check next node.
+      if (node_index != fi->node_checkers.size() - 1) {
+        // Find current node's direct descendant, which will be used in next
+        // iteration.
+        auto check_next_node = fi->node_checkers[node_index + 1];
+        for (const Edge *e : current_node->out_edges()) {
+          if (!e->IsControlEdge()) {
+            Node *candidate_node = e->dst();
+
+            if (check_next_node(candidate_node) == false) {
+              current_node = nullptr;
+            } else {
+              current_node = candidate_node;
+              break;
+            }
+          }
+        }
+      } else {
+        found_pattern = true;
+      }
+    }
+
+    if (found_pattern == true) {
+      break;
+    }
+  }
+
+  return make_tuple(found_pattern, nodes, *fi_ptr);
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 //              Post-rewrite Mkl metadata fixup pass
 ///////////////////////////////////////////////////////////////////////////////
@@ -2516,6 +2931,29 @@ bool MklLayoutRewritePass::RunPass(std::unique_ptr<Graph>* g) {
 
   DumpGraph("After running MklLayoutRewritePass(NodeMerge)", &**g);
 
+  order.clear();
+  GetReversePostOrder(**g, &order);  // This will give us topological sort.
+  for (Node *n : order) {
+    // If node is not an op or it cannot run on CPU device, then skip.
+    if (!n->IsOp() || !CanOpRunOnCPUDevice(n)) {
+      continue;
+    }
+
+    auto check_result = CheckForNodeFusion(n);
+    bool found_pattern = std::get<0>(check_result);
+    std::vector<Node *> nodes = std::get<1>(check_result);
+    const FusionInfo fi = std::get<2>(check_result);
+
+    // if "found_pattern" is true, we can do the fusion.
+    if (found_pattern) {
+      if (FuseNode(g, nodes, fi) == Status::OK()) {
+        result = true;
+      }
+    }
+  }
+
+  DumpGraph("After running MklLayoutRewritePass(NodeFusion)", &**g);
+
   order.clear();
   GetReversePostOrder(**g, &order);  // This will give us topological sort.
   for (Node* n : order) {
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 7e2d1f7878..b09ef3b970 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -455,6 +455,301 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_ConvBpropInput_FilterFwd) {
             "E:3->G:4;F->G;F:control->DMT/_3:control;G->Z;X->Y:1;X->Z:1");
 }
 
+TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Positive) {
+  InitGraph(
+    "node { name: 'Input0' op: 'Input'}"
+    "node { name: 'Input1' op: 'Input'}"
+    "node { name: 'Const0' op: 'Const'"
+    "  attr {"
+    "   key: 'dtype'"
+    "   value {"
+    "     type: DT_INT32"
+    "   }"
+    "  }"
+    " attr {"
+    "   key: 'value'"
+    "   value {"
+    "     tensor {"
+    "       dtype: DT_INT32"
+    "       tensor_shape {"
+    "         dim {"
+    "           size: 4"
+    "         }"
+    "       }"
+    "       tensor_content: '\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000'"
+    "     }"
+    "   }"
+    " }"
+    "}"
+    "node { name: 'Const1' op: 'Const'"
+    "  attr {"
+    "   key: 'dtype'"
+    "   value {"
+    "     type: DT_INT32"
+    "   }"
+    "  }"
+    " attr {"
+    "   key: 'value'"
+    "   value {"
+    "     tensor {"
+    "       dtype: DT_INT32"
+    "       tensor_shape {"
+    "         dim {"
+    "           size: 4"
+    "         }"
+    "       }"
+    "       tensor_content: '\\000\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000\\002\\000\\000\\000'"
+    "     }"
+    "   }"
+    " }"
+    "}"
+    "node {              \
+      name: 'Transpose0' \
+      op: 'Transpose'    \
+      input: 'Input0'    \
+      input: 'Const0'    \
+      attr {             \
+        key: 'T'         \
+        value {          \
+          type: DT_FLOAT \
+        }                \
+      }                  \
+      attr {             \
+        key: 'Tperm'     \
+        value {          \
+          type: DT_INT32 \
+        }                \
+      }                  \
+    }"                   
+    "node {                 \
+      name: 'Conv2D'        \
+      op: 'Conv2D'          \
+      input: 'Transpose0'   \
+      input: 'Input1'       \
+      attr {                \
+        key: 'T'            \
+        value {             \
+          type: DT_FLOAT    \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'data_format'  \
+        value {             \
+          s: 'NHWC'         \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'dilations'    \
+        value {             \
+          list {            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+          }                 \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'padding'      \
+        value {             \
+          s: 'SAME'         \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'strides'      \
+        value {             \
+          list {            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+          }                 \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'use_cudnn_on_gpu' \
+        value {                 \
+          b: true               \
+        }                       \
+      }                         \
+    }"
+    "node {              \
+      name: 'Transpose1' \
+      op: 'Transpose'    \
+      input: 'Conv2D'    \
+      input: 'Const1'    \
+      attr {             \
+        key: 'T'         \
+        value {          \
+          type: DT_FLOAT \
+        }                \
+      }                  \
+      attr {             \
+        key: 'Tperm'     \
+        value {          \
+          type: DT_INT32 \
+        }                \
+      }                  \
+    }"
+    "node { name: 'Relu' op: 'Relu'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " input: ['Transpose1'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "Const0(Const);Const1(Const);"
+            "Conv2D(_MklConv2D);DMT/_0(Const);DMT/_1(Const);Input0(Input);"
+            "Input1(Input);Relu(_MklRelu)|Conv2D->Relu;Conv2D:2->Relu:1;DMT/_0->Conv2D:2;DMT/_1->Conv2D:3;Input0->Conv2D;"
+            "Input0:control->DMT/_0:control;Input0:control->DMT/_1:control;Input1->Conv2D:1");
+}
+
+TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Negative) {
+  InitGraph(
+    "node { name: 'Input0' op: 'Input'}"
+    "node { name: 'Input1' op: 'Input'}"
+    "node { name: 'Const0' op: 'Const'"
+    "  attr {"
+    "   key: 'dtype'"
+    "   value {"
+    "     type: DT_INT32"
+    "   }"
+    "  }"
+    " attr {"
+    "   key: 'value'"
+    "   value {"
+    "     tensor {"
+    "       dtype: DT_INT32"
+    "       tensor_shape {"
+    "         dim {"
+    "           size: 4"
+    "         }"
+    "       }"
+    "       tensor_content: '\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000'"
+    "     }"
+    "   }"
+    " }"
+    "}"
+    "node { name: 'Const1' op: 'Const'"
+    "  attr {"
+    "   key: 'dtype'"
+    "   value {"
+    "     type: DT_INT32"
+    "   }"
+    "  }"
+    " attr {"
+    "   key: 'value'"
+    "   value {"
+    "     tensor {"
+    "       dtype: DT_INT32"
+    "       tensor_shape {"
+    "         dim {"
+    "           size: 4"
+    "         }"
+    "       }"
+    "       tensor_content: '\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000'"
+    "     }"
+    "   }"
+    " }"
+    "}"
+    "node {              \
+      name: 'Transpose0' \
+      op: 'Transpose'    \
+      input: 'Input0'    \
+      input: 'Const0'    \
+      attr {             \
+        key: 'T'         \
+        value {          \
+          type: DT_FLOAT \
+        }                \
+      }                  \
+      attr {             \
+        key: 'Tperm'     \
+        value {          \
+          type: DT_INT32 \
+        }                \
+      }                  \
+    }"                   
+    "node {                 \
+      name: 'Conv2D'        \
+      op: 'Conv2D'          \
+      input: 'Transpose0'   \
+      input: 'Input1'       \
+      attr {                \
+        key: 'T'            \
+        value {             \
+          type: DT_FLOAT    \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'data_format'  \
+        value {             \
+          s: 'NHWC'         \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'dilations'    \
+        value {             \
+          list {            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+          }                 \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'padding'      \
+        value {             \
+          s: 'SAME'         \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'strides'      \
+        value {             \
+          list {            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+            i: 1            \
+          }                 \
+        }                   \
+      }                     \
+      attr {                \
+        key: 'use_cudnn_on_gpu' \
+        value {                 \
+          b: true               \
+        }                       \
+      }                         \
+    }"
+    "node {              \
+      name: 'Transpose1' \
+      op: 'Transpose'    \
+      input: 'Conv2D'    \
+      input: 'Const1'    \
+      attr {             \
+        key: 'T'         \
+        value {          \
+          type: DT_FLOAT \
+        }                \
+      }                  \
+      attr {             \
+        key: 'Tperm'     \
+        value {          \
+          type: DT_INT32 \
+        }                \
+      }                  \
+    }"
+    "node { name: 'Relu' op: 'Relu'"
+      " attr { key: 'T'                value { type: DT_FLOAT } }"
+      " input: ['Transpose1'] }");
+  EXPECT_EQ(DoMklLayoutOptimizationPass(),
+            "Const0(Const);Const1(Const);"
+            "Conv2D(_MklConv2D);DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);"
+            "Input0(Input);Input1(Input);Relu(_MklRelu);"
+            "Transpose0(Transpose);Transpose1(Transpose)|Const0->Transpose0:1;Const1->Transpose1:1;"
+            "Conv2D->Transpose1;DMT/_0->Conv2D:2;DMT/_1->Conv2D:3;DMT/_2->Relu:1;Input0->Transpose0;"
+            "Input1->Conv2D:1;Transpose0->Conv2D;Transpose0:control->DMT/_0:control;"
+            "Transpose0:control->DMT/_1:control;Transpose1->Relu;Transpose1:control->DMT/_2:control");
+}
+
 /////////////////////////////////////////////////////////////////////
 //  Unit tests related to rewriting node to Mkl node
 /////////////////////////////////////////////////////////////////////
-- 
GitLab


From 8d4805d8a3d2b2b48c69d9cd7f358996cb9a87a1 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 22 Oct 2018 11:09:58 -0700
Subject: [PATCH 0043/1554]  cleaning up MKL ML code in LRN kernel

---
 tensorflow/core/kernels/mkl_lrn_op.cc | 680 +-------------------------
 tensorflow/core/ops/nn_ops.cc         |   8 -
 2 files changed, 2 insertions(+), 686 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc
index 22ff4cd80f..4d46abb0a4 100644
--- a/tensorflow/core/kernels/mkl_lrn_op.cc
+++ b/tensorflow/core/kernels/mkl_lrn_op.cc
@@ -22,6 +22,7 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 #include <vector>
+#include "mkldnn.hpp"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -29,25 +30,18 @@ limitations under the License.
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/mkl_util.h"
 #include "tensorflow/core/util/tensor_format.h"
 
 #if !defined(IS_MOBILE_PLATFORM)
 #include "tensorflow/core/util/work_sharder.h"
 #endif
 
-#ifndef INTEL_MKL_ML_ONLY
-#include "mkldnn.hpp"
 using mkldnn::lrn_across_channels;
 using mkldnn::lrn_backward;
 using mkldnn::lrn_forward;
 using mkldnn::prop_kind;
 using mkldnn::stream;
-#else
-#include "mkl_dnn.h"
-#include "mkl_dnn_types.h"
-#endif
-
-#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 
@@ -69,672 +63,6 @@ void GetBandMatrix(int depth, int depth_radius,
 
 }  // namespace
 
-#ifdef INTEL_MKL_ML_ONLY
-
-template <typename T>
-class MklLRNOp : public OpKernel {
- public:
-  ~MklLRNOp() {}
-
-  explicit MklLRNOp(OpKernelConstruction* context) : OpKernel(context) {
-    int64 depth_radius64;
-    OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64));
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(depth_radius64, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("depth_radius = ", depth_radius64,
-                                " larger than int max"));
-    depth_radius_ = static_cast<size_t>(depth_radius64);
-
-    OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_));
-    OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_));
-    OP_REQUIRES_OK(context, context->GetAttr("beta", &beta_));
-    workspace_enabled_ = false;
-    OP_REQUIRES_OK(context,
-                   context->GetAttr("workspace_enabled", &workspace_enabled_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    MklLRNOpContext mkl_context;
-
-    const Tensor& input = MklGetInput(context, 0);
-    GetMklShape(context, 0, &mkl_context.input_shape);
-    bool input_in_mkl_format = mkl_context.input_shape.IsMklTensor();
-
-    // Sanity checks
-    mkl_context.in_dims = input_in_mkl_format
-                              ? mkl_context.input_shape.GetDimension()
-                              : input.dims();
-    OP_REQUIRES(context, mkl_context.in_dims == 4,
-                errors::InvalidArgument("input must be 4-dimensional"));
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input.NumElements(), std::numeric_limits<int>::max()),
-        errors::InvalidArgument("argument to LRN too large"));
-
-    if (!input_in_mkl_format) {
-      mkl_context.MklDefaultToEigen(context, depth_radius_, bias_, alpha_,
-                                    beta_, input);
-      return;
-    }
-
-    if (input_in_mkl_format) {
-      // MKL supports normalization over channel dimension only
-      if (mkl_context.input_shape.tf_dim_idx(mkl_context.in_dims - 1) ==
-          MklDims::C) {
-        mkl_context.lt_input =
-            static_cast<dnnLayout_t>(mkl_context.input_shape.GetCurLayout());
-        workspace_enabled_ = true;
-      } else {
-        Tensor converted_tensor =
-            ConvertMklToTF<T>(context, input, mkl_context.input_shape);
-        mkl_context.MklDefaultToEigen(context, depth_radius_, bias_, alpha_,
-                                      beta_, converted_tensor);
-        return;
-      }
-    }
-
-    int kernel_size = 2 * depth_radius_ + 1;
-
-    CHECK_EQ(dnnLRNCreateForward_F32(
-                 &mkl_context.lrn_fwd, NULL, mkl_context.lt_input, kernel_size,
-                 static_cast<float>(alpha_ * kernel_size), beta_, bias_),
-             E_SUCCESS);
-
-    // Allocate output tensor and shape
-    Tensor* output = nullptr;
-    Tensor* workspace = nullptr;
-
-    // Convert Inputs if needed
-    Tensor mkl_tmp_input_buf_tensor;
-    mkl_context.MklPrepareLRNInputs(context, &mkl_tmp_input_buf_tensor);
-
-    // Allocate Layer Outputs
-    mkl_context.MklAllocateOutputs(context, &output, &workspace,
-                                   workspace_enabled_);
-
-    Tensor mkl_tmp_workspace_buf_tensor;
-    mkl_context.MklPrepareLRNOutputs(context, output, workspace,
-                                     &mkl_tmp_workspace_buf_tensor,
-                                     workspace_enabled_);
-
-    // Execute LRN.
-    CHECK_EQ(dnnExecute_F32(mkl_context.lrn_fwd, mkl_context.lrn_res),
-             E_SUCCESS);
-
-    // Release MKL resources.
-    mkl_context.MklCleanup();
-  }
-
- private:
-  typedef struct {
-    size_t in_dims;
-    size_t in_sizes[4];
-    size_t in_strides[4];
-    size_t out_sizes[4];
-    size_t out_strides[4];
-    MklShape input_shape;
-    dnnPrimitive_t lrn_fwd = nullptr;
-    dnnPrimitive_t convert_input = nullptr;
-    dnnLayout_t lt_input = nullptr;
-    dnnLayout_t lt_internal_input = nullptr;
-    dnnLayout_t lt_internal_workspace = nullptr;
-    dnnLayout_t lt_internal_output = nullptr;
-    void* lrn_res[dnnResourceNumber];
-
-    // Convert Inputs if needed
-    void MklPrepareLRNInputs(OpKernelContext* context,
-                             Tensor* mkl_tmp_input_buf_tensor) {
-      const Tensor& input = MklGetInput(context, 0);
-      void* mkl_buf_input =
-          const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
-
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&lt_internal_input, lrn_fwd,
-                                                dnnResourceSrc),
-               E_SUCCESS);
-
-      void* mkl_buf_convert_input = nullptr;
-      bool mkl_convert_input = false;
-      mkl_convert_input = !dnnLayoutCompare_F32(lt_internal_input, lt_input);
-
-      if (mkl_convert_input) {
-        CHECK_EQ(dnnConversionCreate_F32(&convert_input, lt_input,
-                                         lt_internal_input),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_internal_input,
-                       &mkl_buf_convert_input);
-        CHECK_EQ(dnnConversionExecute_F32(convert_input, mkl_buf_input,
-                                          mkl_buf_convert_input),
-                 E_SUCCESS);
-        dnnDelete_F32(convert_input);
-      }
-
-      lrn_res[dnnResourceSrc] =
-          (mkl_convert_input) ? mkl_buf_convert_input : mkl_buf_input;
-    }
-
-    // Allocate Layer Outputs
-    void MklAllocateOutputs(OpKernelContext* context, Tensor** output,
-                            Tensor** workspace, bool workspace_enabled_) {
-      TensorShape mkl_output_tf_shape; /* First tensor */
-      MklShape mkl_output_mkl_shape;   /* Second tensor */
-
-      mkl_output_mkl_shape.SetMklTensor(true);
-      mkl_output_mkl_shape.SetMklLayout(lrn_fwd, dnnResourceDst);
-      mkl_output_mkl_shape.SetTfLayout(in_dims, input_shape.GetSizes(),
-                                       input_shape.GetStrides());
-      mkl_output_mkl_shape.SetTfDimOrder(in_dims,
-                                         input_shape.GetTfToMklDimMap());
-      mkl_output_tf_shape.AddDim(
-          dnnLayoutGetMemorySize_F32(
-              static_cast<dnnLayout_t>(mkl_output_mkl_shape.GetMklLayout())) /
-          sizeof(T));
-      AllocateOutputSetMklShape(context, 0, output,
-                                mkl_output_tf_shape /* First tensor */,
-                                mkl_output_mkl_shape /* Second Tensor */);
-
-      if (workspace_enabled_) {
-        TensorShape mkl_workspace_tf_shape; /* First tensor */
-        MklShape mkl_workspace_mkl_shape;   /* Second tensor */
-        mkl_workspace_mkl_shape.SetMklTensor(false);
-        mkl_workspace_mkl_shape.SetMklLayout(lrn_fwd, dnnResourceWorkspace);
-        // Assumes workspace has same TF layout and TF dim order as input
-        mkl_workspace_mkl_shape.SetTfLayout(in_dims, input_shape.GetSizes(),
-                                            input_shape.GetStrides());
-        mkl_workspace_mkl_shape.SetTfDimOrder(in_dims,
-                                              input_shape.GetTfToMklDimMap());
-        mkl_workspace_tf_shape.AddDim(
-            dnnLayoutGetMemorySize_F32(static_cast<dnnLayout_t>(
-                mkl_workspace_mkl_shape.GetMklLayout())) /
-            sizeof(T));
-        AllocateOutputSetMklShape(context, 1, workspace,
-                                  mkl_workspace_tf_shape /* First tensor */,
-                                  mkl_workspace_mkl_shape /* Second Tensor */);
-      }
-    }
-
-    void MklPrepareLRNOutputs(OpKernelContext* context, Tensor* output,
-                              Tensor* workspace,
-                              Tensor* mkl_tmp_workspace_buf_tensor,
-                              bool workspace_enabled_) {
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&lt_internal_workspace, lrn_fwd,
-                                                dnnResourceWorkspace),
-               E_SUCCESS);
-
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&lt_internal_output, lrn_fwd,
-                                                dnnResourceDst),
-               E_SUCCESS);
-
-      void* mkl_buf_output =
-          const_cast<void*>(static_cast<const void*>(output->flat<T>().data()));
-      lrn_res[dnnResourceDst] = mkl_buf_output;
-
-      void* mkl_buf_workspace = nullptr;
-      if (workspace_enabled_) {
-        mkl_buf_workspace = const_cast<void*>(
-            static_cast<const void*>(workspace->flat<T>().data()));
-      } else {
-        AllocTmpBuffer(context, mkl_tmp_workspace_buf_tensor,
-                       lt_internal_workspace, &mkl_buf_workspace);
-      }
-      lrn_res[dnnResourceWorkspace] = mkl_buf_workspace;
-    }
-
-    // Fallback implementation - Taken from lrn_op.cc
-    // TODO(inteltf) Check if we can use EigenLRNOp directly instead of making a
-    // copy.
-    void MklDefaultToEigen(OpKernelContext* context, int depth_radius_,
-                           float bias_, float alpha_, float beta_,
-                           const Tensor& input) {
-      const int batch = static_cast<int>(input.dim_size(0));
-      const int rows = static_cast<int>(input.dim_size(1));
-      const int cols = static_cast<int>(input.dim_size(2));
-      const int depth = static_cast<int>(input.dim_size(3));
-      const int nodes = cols * rows;
-
-      auto in_shaped = input.shaped<T, 2>({nodes * batch, depth});
-      // Multiplying the input with the band matrix has the effect of reducing
-      // the
-      // correct patch along the depth.
-      Eigen::Tensor<T, 2, Eigen::RowMajor> multiplier(depth, depth);
-      GetBandMatrix<T>(depth, depth_radius_, &multiplier);
-
-      Tensor *output, *workspace;
-      MklShape mkl_output_mkl_shape, mkl_workspace_mkl_shape;
-      mkl_output_mkl_shape.SetMklTensor(false);
-      mkl_output_mkl_shape.SetDimensions(4);
-      AllocateOutputSetMklShape(context, 0, &output, input.shape(),
-                                mkl_output_mkl_shape);
-
-      mkl_workspace_mkl_shape.SetMklTensor(false);
-      mkl_workspace_mkl_shape.SetDimensions(4);
-      AllocateOutputSetMklShape(context, 1, &workspace, input.shape(),
-                                mkl_workspace_mkl_shape);
-
-      auto out_shaped = output->shaped<T, 2>({nodes * batch, depth});
-      Eigen::array<DimPair, 1> dims = {{DimPair(1, 0)}};
-      auto tmp = in_shaped.square().contract(multiplier, dims) * alpha_ + bias_;
-      if (beta_ == T(1)) {
-        out_shaped.device(context->eigen_cpu_device()) =
-            in_shaped * tmp.inverse();
-      } else if (beta_ == T(0.5)) {
-        out_shaped.device(context->eigen_cpu_device()) =
-            in_shaped * tmp.rsqrt();
-      } else {
-        out_shaped.device(context->eigen_cpu_device()) =
-            in_shaped * (tmp.log() * -beta_).exp();
-      }
-    }
-
-    // Release MKL resources.
-    void MklCleanup() {
-      dnnDelete_F32(lrn_fwd);
-      dnnLayoutDelete_F32(lt_internal_input);
-      dnnLayoutDelete_F32(lt_internal_workspace);
-      dnnLayoutDelete_F32(lt_internal_output);
-    }
-  } MklLRNOpContext;
-
-  typedef typename Eigen::Tensor<T, 1, Eigen::RowMajor>::DimensionPair DimPair;
-
-  bool workspace_enabled_;
-  int depth_radius_;
-  float bias_;
-  float alpha_;
-  float beta_;
-};
-
-template <typename T>
-class MklLRNGradOp : public OpKernel {
- public:
-  explicit MklLRNGradOp(OpKernelConstruction* context) : OpKernel(context) {
-    int64 depth_radius64;
-    OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64));
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(depth_radius64, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("depth_radius = ", depth_radius64,
-                                " larger than int max"));
-    depth_radius_ = static_cast<int>(depth_radius64);
-    OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_));
-    OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_));
-    OP_REQUIRES_OK(context, context->GetAttr("beta", &beta_));
-    workspace_enabled_ = false;
-    OP_REQUIRES_OK(context,
-                   context->GetAttr("workspace_enabled", &workspace_enabled_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    MklLRNGradOpContext mkl_context;
-    mkl_context.depth_radius_ = depth_radius_;
-    mkl_context.bias_ = bias_;
-    mkl_context.alpha_ = alpha_;
-    mkl_context.beta_ = beta_;
-
-    const Tensor& in_grads = MklGetInput(context, 0);
-    const Tensor& in_image = MklGetInput(context, 1);
-    const Tensor& out_image = MklGetInput(context, 2);
-
-    GetMklShape(context, 0, &mkl_context.ingrad_shape);
-    GetMklShape(context, 1, &mkl_context.inimage_shape);
-    GetMklShape(context, 2, &mkl_context.outimage_shape);
-
-    bool ingrad_in_mkl_format = mkl_context.ingrad_shape.IsMklTensor();
-    bool inimage_in_mkl_format = mkl_context.inimage_shape.IsMklTensor();
-    bool outimage_in_mkl_format = mkl_context.outimage_shape.IsMklTensor();
-
-    mkl_context.in_dims = inimage_in_mkl_format
-                              ? mkl_context.inimage_shape.GetDimension()
-                              : in_image.dims();
-    OP_REQUIRES(context, mkl_context.in_dims == 4,
-                errors::InvalidArgument("input images must be 4-dimensional"));
-
-    if (!workspace_enabled_) {
-      mkl_context.MklDefaultToEigen(context);
-      return;
-    }
-
-    if (ingrad_in_mkl_format || inimage_in_mkl_format) {
-      const MklShape* tmp_mkl_shape = (ingrad_in_mkl_format)
-                                          ? &mkl_context.ingrad_shape
-                                          : &mkl_context.inimage_shape;
-      if (tmp_mkl_shape->tf_dim_idx(mkl_context.in_dims - 1) != MklDims::C) {
-        // Fallback to eigen
-        mkl_context.MklDefaultToEigen(context);
-        return;
-      } else {  // MKL supports normalization over channel dimension only
-        for (int i = 0; i < mkl_context.in_dims; i++) {
-          mkl_context.in_sizes[i] = mkl_context.out_sizes[i] =
-              tmp_mkl_shape->GetSizes()[i];
-          mkl_context.in_strides[i] = mkl_context.out_strides[i] =
-              tmp_mkl_shape->GetStrides()[i];
-        }
-      }
-    } else {
-      // Fallback to eigen
-      mkl_context.MklDefaultToEigen(context);
-      return;
-    }
-
-    // Dimensions check for sanity purpose
-    if (ingrad_in_mkl_format) {
-      OP_REQUIRES(
-          context, mkl_context.ingrad_shape.GetDimension() == 4,
-          errors::InvalidArgument("input gradient must be 4-dimensional"));
-    } else {
-      OP_REQUIRES(
-          context, in_grads.dims() == 4,
-          errors::InvalidArgument("input gradient must be 4-dimensional"));
-    }
-
-    if (outimage_in_mkl_format) {
-      OP_REQUIRES(
-          context, mkl_context.outimage_shape.GetDimension() == 4,
-          errors::InvalidArgument("Output image must be 4-dimensional"));
-    } else {
-      OP_REQUIRES(
-          context, out_image.dims() == 4,
-          errors::InvalidArgument("Output image must be 4-dimensional"));
-    }
-
-    // Prepare mkl input layout
-    mkl_context.MklPrepareLRNInputsLayouts(context);
-    int ksize = 2 * depth_radius_ + 1;
-
-    CHECK_EQ(dnnLRNCreateBackward_F32(
-                 &mkl_context.lrn_bwd, NULL, mkl_context.lt_input,
-                 mkl_context.lt_output, ksize,
-                 static_cast<float>(alpha_ * ksize), beta_, bias_),
-             E_SUCCESS);
-
-    // Allocate output tensor and shape.
-    TensorShape mkl_output_tf_shape; /* First tensor */
-    MklShape mkl_output_mkl_shape;   /* Second tensor */
-    mkl_output_mkl_shape.SetMklTensor(true);
-    CHECK_NE(mkl_context.lrn_bwd, nullptr);
-    mkl_output_mkl_shape.SetMklLayout(mkl_context.lrn_bwd, dnnResourceDiffSrc);
-    mkl_output_mkl_shape.SetTfLayout(mkl_context.in_dims, mkl_context.out_sizes,
-                                     mkl_context.out_strides);
-    if (ingrad_in_mkl_format) {
-      mkl_output_mkl_shape.SetTfDimOrder(
-          mkl_context.in_dims, mkl_context.ingrad_shape.GetTfToMklDimMap());
-    } else {
-      mkl_output_mkl_shape.SetTfDimOrder(
-          mkl_context.in_dims, mkl_context.inimage_shape.GetTfToMklDimMap());
-    }
-    mkl_output_tf_shape.AddDim(
-        dnnLayoutGetMemorySize_F32(
-            static_cast<dnnLayout_t>(mkl_output_mkl_shape.GetMklLayout())) /
-        sizeof(T));
-    Tensor* output = nullptr;
-    AllocateOutputSetMklShape(context, 0, &output, mkl_output_tf_shape,
-                              mkl_output_mkl_shape);
-
-    // Get pointers to output data.
-    void* user_output =
-        const_cast<void*>(static_cast<const void*>(output->flat<T>().data()));
-
-    Tensor mkl_tmp_input_buf_tensor, mkl_tmp_image_buf_tensor,
-        mkl_tmp_outimage_buf_tensor;
-    // Convert Inputs if needed
-    mkl_context.MklPrepareLRNGradInput(context, &mkl_tmp_input_buf_tensor,
-                                       &mkl_tmp_image_buf_tensor,
-                                       &mkl_tmp_outimage_buf_tensor);
-
-    // We do not do any conversion for output. But we simply emit it
-    // in MKL format.
-    mkl_context.res_lrn_bwd[dnnResourceDiffSrc] = user_output;
-    // Execute LRN backward using dnnExecute
-    CHECK_EQ(dnnExecute_F32(mkl_context.lrn_bwd, mkl_context.res_lrn_bwd),
-             E_SUCCESS);
-    // Release MKL resources.
-    mkl_context.Mklcleanup();
-  }
-
- private:
-  typedef struct {
-    int depth_radius_;
-    float bias_;
-    float alpha_;
-    float beta_;
-    size_t in_dims;
-    size_t in_sizes[4];
-    size_t in_strides[4];
-    size_t out_sizes[4];
-    size_t out_strides[4];
-    MklShape ingrad_shape, inimage_shape, outimage_shape;
-    dnnPrimitive_t lrn_bwd = nullptr;
-    dnnPrimitive_t convert_input = nullptr;
-    dnnLayout_t lt_input = nullptr;
-    dnnLayout_t lt_output = nullptr;
-    dnnLayout_t lt_bdw_input = nullptr;
-    dnnLayout_t lt_workspace = nullptr;
-    dnnLayout_t lt_internal_input = nullptr;
-    void* res_lrn_bwd[dnnResourceNumber];
-
-    // prepare mkl input
-    void MklPrepareLRNInputsLayouts(OpKernelContext* context) {
-      bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor();
-      bool inimage_in_mkl_format = inimage_shape.IsMklTensor();
-      if (!ingrad_in_mkl_format) {
-        CHECK_EQ(dnnLayoutCreate_F32(&lt_input, in_dims, in_sizes, in_strides),
-                 E_SUCCESS);
-      } else {
-        lt_input = static_cast<dnnLayout_t>(ingrad_shape.GetCurLayout());
-      }
-
-      if (!inimage_in_mkl_format) {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&lt_output, in_dims, out_sizes, out_strides),
-            E_SUCCESS);
-      } else {
-        lt_output = static_cast<dnnLayout_t>(inimage_shape.GetCurLayout());
-      }
-    }
-
-    // convert input if needed
-    void MklPrepareLRNGradInput(OpKernelContext* context,
-                                Tensor* mkl_tmp_input_buf_tensor,
-                                Tensor* mkl_tmp_image_buf_tensor,
-                                Tensor* mkl_tmp_outimage_buf_tensor) {
-      const Tensor& in_grads = MklGetInput(context, 0);
-      const Tensor& in_image = MklGetInput(context, 1);
-      const Tensor& workspace = MklGetInput(
-          context,
-          3); /*Worskpsace is enabled, get the buffer to the workspace */
-
-      void* user_input = const_cast<void*>(
-          static_cast<const void*>(in_grads.flat<T>().data()));
-      void* user_fwd_input = const_cast<void*>(
-          static_cast<const void*>(in_image.flat<T>().data()));
-      void* workspace_buffer = const_cast<void*>(
-          static_cast<const void*>(workspace.flat<T>().data()));
-
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&lt_workspace, lrn_bwd,
-                                                dnnResourceWorkspace),
-               E_SUCCESS);
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&lt_bdw_input, lrn_bwd,
-                                                dnnResourceDiffDst),
-               E_SUCCESS);
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&lt_internal_input, lrn_bwd,
-                                                dnnResourceSrc),
-               E_SUCCESS);
-
-      bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor();
-      if (ingrad_in_mkl_format) {
-        if (!dnnLayoutCompare_F32(lt_bdw_input, lt_input)) {
-          AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_bdw_input,
-                         &res_lrn_bwd[dnnResourceDiffDst]);
-          ingrad_shape.GetConvertedFlatData(lt_bdw_input, user_input,
-                                            res_lrn_bwd[dnnResourceDiffDst]);
-        } else {
-          res_lrn_bwd[dnnResourceDiffDst] = user_input;
-        }
-      } else {
-        if (!dnnLayoutCompare_F32(lt_bdw_input, lt_input)) {
-          CHECK_EQ(
-              dnnConversionCreate_F32(&convert_input, lt_input, lt_bdw_input),
-              E_SUCCESS);
-
-          AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, lt_bdw_input,
-                         &res_lrn_bwd[dnnResourceDiffDst]);
-          CHECK_EQ(dnnConversionExecute_F32(convert_input, user_input,
-                                            res_lrn_bwd[dnnResourceDiffDst]),
-                   E_SUCCESS);
-          dnnDelete_F32(convert_input);
-        } else {
-          res_lrn_bwd[dnnResourceDiffDst] = user_input;
-        }
-      }
-
-      bool inimage_in_mkl_format = inimage_shape.IsMklTensor();
-      if (inimage_in_mkl_format) {
-        if (!dnnLayoutCompare_F32(
-                lt_internal_input,
-                static_cast<dnnLayout_t>(inimage_shape.GetCurLayout()))) {
-          AllocTmpBuffer(context, mkl_tmp_image_buf_tensor, lt_internal_input,
-                         &res_lrn_bwd[dnnResourceSrc]);
-          ingrad_shape.GetConvertedFlatData(lt_internal_input, user_fwd_input,
-                                            res_lrn_bwd[dnnResourceSrc]);
-        } else {
-          res_lrn_bwd[dnnResourceSrc] = user_fwd_input;
-        }
-      } else {
-        if (!dnnLayoutCompare_F32(
-                lt_internal_input,
-                static_cast<dnnLayout_t>(inimage_shape.GetCurLayout()))) {
-          CHECK_EQ(dnnConversionCreate_F32(
-                       &convert_input,
-                       static_cast<dnnLayout_t>(inimage_shape.GetCurLayout()),
-                       lt_internal_input),
-                   E_SUCCESS);
-
-          AllocTmpBuffer(context, mkl_tmp_image_buf_tensor, lt_internal_input,
-                         &res_lrn_bwd[dnnResourceSrc]);
-          CHECK_EQ(dnnConversionExecute_F32(convert_input, user_fwd_input,
-                                            res_lrn_bwd[dnnResourceSrc]),
-                   E_SUCCESS);
-          dnnDelete_F32(convert_input);
-        } else {
-          res_lrn_bwd[dnnResourceSrc] = user_fwd_input;
-        }
-      }
-
-      res_lrn_bwd[dnnResourceWorkspace] = workspace_buffer;
-    }
-
-    // Fallback implementation - Taken from lrn_op.cc
-    // TODO(intelft) Check if we can use EigenLRNOp directly instead of making a
-    // copy.
-    void MklDefaultToEigen(OpKernelContext* context) {
-      Tensor in_grads;
-      Tensor in_image;
-      Tensor out_image;
-
-      GetMklShape(context, 0, &ingrad_shape);
-      GetMklShape(context, 1, &inimage_shape);
-      GetMklShape(context, 2, &outimage_shape);
-
-      if (ingrad_shape.IsMklTensor()) {
-        in_grads =
-            ConvertMklToTF<T>(context, MklGetInput(context, 0), ingrad_shape);
-      } else {
-        in_grads = MklGetInput(context, 0);
-      }
-
-      if (inimage_shape.IsMklTensor()) {
-        in_image =
-            ConvertMklToTF<T>(context, MklGetInput(context, 1), inimage_shape);
-      } else {
-        in_image = MklGetInput(context, 1);
-      }
-
-      if (outimage_shape.IsMklTensor()) {
-        out_image =
-            ConvertMklToTF<T>(context, MklGetInput(context, 2), outimage_shape);
-      } else {
-        out_image = MklGetInput(context, 2);
-      }
-
-      const int64 batch = static_cast<int64>(in_grads.dim_size(0));
-      const int64 rows = static_cast<int64>(in_grads.dim_size(1));
-      const int64 cols = static_cast<int64>(in_grads.dim_size(2));
-      const int64 depth = static_cast<int64>(in_grads.dim_size(3));
-      const auto nodes = cols * rows;
-
-      auto grads_shaped = in_grads.shaped<T, 2>({nodes * batch, depth});
-
-      auto in_shaped = in_image.shaped<T, 2>({nodes * batch, depth});
-      auto activations = out_image.shaped<T, 2>({nodes * batch, depth});
-
-      Tensor* output;
-      MklShape mkl_output_mkl_shape;
-      mkl_output_mkl_shape.SetMklTensor(false);
-      mkl_output_mkl_shape.SetDimensions(4);
-      AllocateOutputSetMklShape(context, 0, &output, in_grads.shape(),
-                                mkl_output_mkl_shape);
-
-      auto out_shaped = output->shaped<T, 2>({nodes * batch, depth});
-      out_shaped.setZero();
-      auto shard = [this, activations, in_shaped, grads_shaped, out_shaped,
-                    depth](int64 begin, int64 end) {
-        for (int64 i = begin; i < end; ++i) {
-          for (int64 j = 0; j < depth; ++j) {
-            int64 depth_begin = std::max<int64>(0, j - depth_radius_);
-            int64 depth_end = std::min<int64>(depth, j + depth_radius_ + 1);
-
-            T norm(0);
-            for (int64 k = depth_begin; k < depth_end; ++k) {
-              norm += in_shaped(i, k) * in_shaped(i, k);
-            }
-            norm = alpha_ * norm + bias_;
-            DCHECK_GT(norm, T(1e-6));
-            for (int64 k = depth_begin; k < depth_end; ++k) {
-              T dyi = T(-2) * alpha_ * beta_ * in_shaped(i, k) *
-                      activations(i, j) / norm;
-              if (k == j) {
-                dyi += Eigen::numext::pow(norm, -beta_);
-              }
-              dyi *= grads_shaped(i, j);
-              const_cast<typename TTypes<T, 2>::Tensor&>(out_shaped)(i, k) +=
-                  dyi;
-            }
-          }
-        }
-      };
-      auto worker_threads =
-          *(context->device()->tensorflow_cpu_worker_threads());
-      Shard(worker_threads.num_threads, worker_threads.workers, nodes * batch,
-            depth * depth, shard);
-    }
-
-    // release mkl resources
-    void Mklcleanup() {
-      bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor();
-      bool inimage_in_mkl_format = inimage_shape.IsMklTensor();
-      if (!ingrad_in_mkl_format) {
-        CHECK_EQ(dnnLayoutDelete_F32(lt_input), E_SUCCESS);
-      }
-
-      if (!inimage_in_mkl_format) {
-        CHECK_EQ(dnnLayoutDelete_F32(lt_output), E_SUCCESS);
-      }
-      dnnDelete_F32(lrn_bwd);
-      dnnLayoutDelete_F32(lt_bdw_input);
-      dnnLayoutDelete_F32(lt_workspace);
-    }
-  } MklLRNGradOpContext;
-
-  typedef typename Eigen::Tensor<T, 1, Eigen::RowMajor>::DimensionPair DimPair;
-  bool workspace_enabled_;
-  int depth_radius_;
-  float bias_;
-  float alpha_;
-  float beta_;
-};
-
-#else
-
 template <typename T>
 class MklLRNOp : public OpKernel {
  public:
@@ -847,7 +175,6 @@ class MklLRNOp : public OpKernel {
                             MklDnnData<T>* src_dnn_data,
                             MklDnnData<T>* dst_dnn_data,
                             MklDnnData<uint8>* wksp_dnn_data = nullptr) {
-
     // Check for input reorder
     src_dnn_data->CheckReorderToOpMem(lrn_fwd_desc.src_primitive_desc());
 
@@ -1160,7 +487,6 @@ class MklLRNGradOp : public OpKernel {
       MklDnnData<T>* output_diff_src,
       const memory::primitive_desc& target_diff_dst_pd,
       const MklDnnData<uint8>* workspace_dnn_data = nullptr) {
-
     // Check for input reordering on the diff dst input
     input_gradient_diff_dst->CheckReorderToOpMem(
         lrn_bkwd_desc.diff_dst_primitive_desc());
@@ -1345,8 +671,6 @@ class MklLRNGradOp : public OpKernel {
   float beta_;
 };
 
-#endif  // INTEL_MKL_ML_ONLY
-
 #define REGISTER_MKL_LRN_CPU(T)                                     \
   REGISTER_KERNEL_BUILDER(Name("_MklLRN")                           \
                               .Device(DEVICE_CPU)                   \
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 38fe45936a..bee0add89c 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -2172,11 +2172,7 @@ REGISTER_OP("_MklLRN")
     .Input("input: T")
     .Input("mkl_input: uint8")
     .Output("output: T")
-#ifdef INTEL_MKL_ML_ONLY
-    .Output("workspace: T")
-#else
     .Output("workspace: uint8")
-#endif
     .Output("mkl_output: uint8")
     .Output("mkl_workspace: uint8")
     .Attr("depth_radius: int = 5")
@@ -2200,11 +2196,7 @@ REGISTER_OP("_MklLRNGrad")
     .Input("input_grads: T")
     .Input("input_image: T")
     .Input("output_image: T")
-#ifdef INTEL_MKL_ML_ONLY
-    .Input("workspace: T")
-#else
     .Input("workspace: uint8")
-#endif
     .Input("mkl_input_grads: uint8")
     .Input("mkl_input_image: uint8")
     .Input("mkl_output_image: uint8")
-- 
GitLab


From c0814e3861c0b88caebc24d4ef1ce5e61a213f2e Mon Sep 17 00:00:00 2001
From: Rholais Lii <rholais@gmail.com>
Date: Tue, 23 Oct 2018 13:44:40 +0800
Subject: [PATCH 0044/1554] Fix comments to match usage

Fix comments of `sparse_softmax_cross_entropy_with_logits` to match usage.
---
 tensorflow/python/ops/nn_ops.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index e31d162285..2477271a22 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -1980,8 +1980,9 @@ def sparse_softmax_cross_entropy_with_logits(
   on `logits` internally for efficiency.  Do not call this op with the
   output of `softmax`, as it will produce incorrect results.
 
-  A common use case is to have logits and labels of shape
-  `[batch_size, num_classes]`, but higher dimensions are supported, in which
+  A common use case is to have logits of shape
+  `[batch_size, num_classes]` and have labels of shape
+  `[batch_size]`, but higher dimensions are supported, in which
   case the `dim`-th dimension is assumed to be of size `num_classes`.
   `logits` must have the dtype of `float16`, `float32`, or `float64`, and
   `labels` must have the dtype of `int32` or `int64`.
-- 
GitLab


From f1acc3f9e4552cb6ffd8c7c4a281192cc58f734f Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Wed, 24 Oct 2018 01:48:08 +0000
Subject: [PATCH 0045/1554] Resolved merge conflicts and squashed into one
 commit

---
 tensorflow/python/ops/cond_v2.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 998c3e08f6..175b53b1a2 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -136,11 +136,11 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
     # correct output structure
     tensors = tuple(array_ops.identity(t) for t in tensors)
 
-    result = tuple(tensors[:num_cond_outputs])
-    if len(result) == 1:
-      return result[0]
-    else:
-      return result
+    # Packing output tensors in the same nested structure as the true and false
+    # functions return
+    result = nest.pack_sequence_as(structure=true_graph.structured_outputs, 
+                                   flat_sequence=tensors[:num_cond_outputs])
+    return result
 
 
 @ops.RegisterGradient("If")
-- 
GitLab


From ddd89530bf5022996c2f3564a5af0eda08eb81d2 Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Wed, 24 Oct 2018 01:55:01 +0000
Subject: [PATCH 0046/1554] Forgot to update tests

---
 .../kernel_tests/control_flow_ops_py_test.py  | 258 ++++++++----------
 1 file changed, 113 insertions(+), 145 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 6487503ff9..7a746ca362 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 
 import collections
 import math
-import sys
 import time
 
 import numpy as np
@@ -456,7 +455,7 @@ class ControlFlowTest(test.TestCase):
     self.assertTrue(ind.dtype == np.int64)
 
   def testCondColocation(self):
-    with self.session(use_gpu=True):
+    with self.test_session(use_gpu=True):
       with ops.device("/cpu:0"):
         v = variables.Variable(7.0)
 
@@ -471,7 +470,7 @@ class ControlFlowTest(test.TestCase):
           self.assertDeviceEqual(op.device, "/cpu:0")
 
   def _testCond_1(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       x = constant_op.constant(10)
       pred = math_ops.less(1, 2)
       fn1 = lambda: math_ops.add(x, 1)
@@ -510,40 +509,27 @@ class ControlFlowTest(test.TestCase):
       result = r.eval()
     self.assertAllEqual(12, result)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testCondPruning(self):
-    v1 = variables.Variable(7)
-    v2 = variables.Variable(7)
-    v3 = variables.Variable(7)
+  @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
+  def testCond_4(self):
+    with self.cached_session():
+      v1 = variables.Variable(7)
+      v2 = variables.Variable(7)
+      v3 = variables.Variable(7)
 
-    def f():
       age = constant_op.constant(3)
       max_age = constant_op.constant(2)
       pred = math_ops.greater(age, max_age)
       fn1 = lambda: [state_ops.assign(v1, 1).op, state_ops.assign(v2, 2).op]
       fn2 = lambda: [state_ops.assign(v3, 3).op, constant_op.constant(10).op]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      self.assertEqual(len(r), 2)
-      return r[1]
 
-    f_defun = eager_function.defun(f)
-
-    if not context.executing_eagerly():
-      with self.cached_session():
-        variables.global_variables_initializer().run()
-        result = f().eval()
-        self.assertEqual(True, result)
-        # Only second cond result was fetched, so v1 assign shouldn't run.
-        self.assertEqual(7, v1.eval())
-        self.assertEqual(2, v2.eval())
-        self.assertEqual(7, v3.eval())
-
-    result = f_defun()
-    self.assertEqual(True, self.evaluate(result))
-    # Both v1 and v2 branch assignments should be run in defun.
-    self.assertEqual(1, self.evaluate(v1))
-    self.assertEqual(2, self.evaluate(v2))
-    self.assertEqual(7, self.evaluate(v3))
+      variables.global_variables_initializer().run()
+      self.assertEqual(len(r), 2)
+      result = r[1].eval()
+      self.assertAllEqual(True, result)
+      self.assertAllEqual(7, v1.eval())
+      self.assertAllEqual(2, v2.eval())
+      self.assertAllEqual(7, v3.eval())
 
   def testCond_5(self):
     with self.cached_session():
@@ -584,6 +570,85 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
       self.assertAllEqual([11, 12], sess.run(r))
 
+  def testCondListOutput(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: [math_ops.add(x, y), math_ops.add(x, y)]
+      fn2 = lambda: [y, y]
+      r = control_flow_ops.cond(pred, fn1, fn2)
+      test_result = sess.run(r)
+      self.assertListEqual([210, 210], test_result)
+
+  def testTupleOutput(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: (math_ops.add(x, y), math_ops.add(x, y))
+      fn2 = lambda: (y, y)
+      r = control_flow_ops.cond(pred, fn1, fn2)
+      test_result = sess.run(r)
+      self.assertTupleEqual((210, 210), test_result)
+
+  def testDictOutput(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
+      fn2 = lambda: {"a": y, "b": y}
+      r = control_flow_ops.cond(pred, fn1, fn2)
+      test_result = sess.run(r)
+      self.assertDictEqual({"a": 210, "b": 210}, test_result)
+
+  def testNestedListOutput(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: [[math_ops.add(x, y), math_ops.add(x, y)]]
+      fn2 = lambda: [[y, y]]
+      # Pass strict=True flag as cond_v2 allows for tensors to be
+      # in nested output structures as singletons
+      r = control_flow_ops.cond(pred, fn1, fn2, strict=True)
+      test_result = sess.run(r)
+      self.assertListEqual([[210, 210]], test_result)
+
+  def testNestedTupleOutput(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: ((math_ops.add(x, y), math_ops.add(x, y)))
+      fn2 = lambda: ((y, y))
+      r = control_flow_ops.cond(pred, fn1, fn2)
+      test_result = sess.run(r)
+      self.assertTupleEqual(((210, 210)), test_result)
+
+  def testNestedDictOutput(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: {"a": {"c": math_ops.add(x, y)}, "b": {"d": math_ops.add(x, y)}}
+      fn2 = lambda: {"a": {"c": y}, "b": {"d": y}}
+      r = control_flow_ops.cond(pred, fn1, fn2)
+      test_result = sess.run(r)
+      self.assertDictEqual({"a": {"c": 210}, "b": {"d": 210}}, test_result)
+
+  def testCheckNestedOutputStruct(self):
+    with self.cached_session() as sess:
+      x = constant_op.constant(10)
+      y = constant_op.constant(200)
+      pred = math_ops.less(1, 2)
+      fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
+      fn2 = lambda: {"c": y, "d": y}
+      with self.assertRaisesRegexp(ValueError, "The two structures don't have the same nested structure"):
+        r = control_flow_ops.cond(pred, fn1, fn2)
+        test_result = sess.run(r)
+
   def testCondRef(self):
 
     with self.cached_session():
@@ -638,6 +703,8 @@ class ControlFlowTest(test.TestCase):
       merged_op = control_flow_ops.merge([assign_v, orig_v])
       self.assertAllEqual([1.0], sess.run(merged_op.output))
 
+  @test_util.disable_control_flow_v2(
+      "b/112477618 (Operation returned from cond)")
   def testCondSwitchIdentity(self):
     # Make sure the recv identity is not removed by optimization.
     with session.Session(config=opt_cfg()) as sess:
@@ -652,6 +719,8 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
       sess.run(r)
 
+  @test_util.disable_control_flow_v2(
+      "b/112477618 (Operation returned from cond)")
   def testCondRecvIdentity(self):
     # Make sure the switch identity is not removed by optimization.
     with session.Session(config=opt_cfg()) as sess:
@@ -750,111 +819,6 @@ class ControlFlowTest(test.TestCase):
       ]
       self.assertAllEqual(dense_gv, [0.0, 2.0])
 
-  # TODO(b/117945658): reenable
-  @test_util.run_in_graph_and_eager_modes
-  def DISABLED_testCondAutoControlDeps(self):
-
-    def branch_fn():
-      logging_ops.print_v2("A")
-      logging_ops.print_v2("B")
-      with ops.control_dependencies([logging_ops.print_v2("C")]):
-        return constant_op.constant(10)
-
-    def build_cond():
-      return control_flow_ops.cond(
-          constant_op.constant(True), branch_fn, lambda: 0)
-
-    def build_nested_cond():
-      return control_flow_ops.cond(
-          constant_op.constant(True), build_cond, lambda: 0)
-
-    # In v1 graph mode, pruning should make only "C" print.
-    if not context.executing_eagerly():
-      with self.cached_session():
-        with self.captureWritesToStream(sys.stderr) as printed:
-          self.assertEqual(build_cond().eval(), 10)
-        self.assertEqual(printed.contents(), "C\n")
-
-        with self.captureWritesToStream(sys.stderr) as printed:
-          self.assertEqual(build_nested_cond().eval(), 10)
-        self.assertEqual(printed.contents(), "C\n")
-
-    # In defuns, all prints should execute in program order.
-    # This doesn't work with legacy control flow.
-    if control_flow_ops.ENABLE_COND_V2:
-
-      @eager_function.defun
-      def cond():
-        return build_cond()
-
-      with self.captureWritesToStream(sys.stderr) as printed:
-        self.assertEqual(self.evaluate(cond()), 10)
-      self.assertEqual(printed.contents(), "A\nB\nC\n")
-
-      @eager_function.defun
-      def nested_cond():
-        return build_nested_cond()
-
-      with self.captureWritesToStream(sys.stderr) as printed:
-        self.assertEqual(self.evaluate(nested_cond()), 10)
-      self.assertEqual(printed.contents(), "A\nB\nC\n")
-
-  # TODO(b/117945658): reenable
-  @test_util.run_in_graph_and_eager_modes
-  def DISABLED_testWhileAutoControlDeps(self):
-
-    def cond(i, unused_x):
-      logging_ops.print_v2("A")
-      return i < 2
-
-    def body(i, x):
-      logging_ops.print_v2("B")
-      with ops.control_dependencies([logging_ops.print_v2("C")]):
-        x = array_ops.identity(x)
-      with ops.control_dependencies([logging_ops.print_v2("D")]):
-        return i + 1, x
-
-    def build_while():
-      return control_flow_ops.while_loop(
-          cond, body, [constant_op.constant(0), constant_op.constant(0)])
-
-    def build_nested_while():
-      return control_flow_ops.cond(
-          constant_op.constant(True), build_while, lambda: (0, 0))
-
-    # In v1 graph mode, pruning should make only "D" print.
-    if not context.executing_eagerly():
-      with self.cached_session():
-        with self.captureWritesToStream(sys.stderr) as printed:
-          self.assertEqual(build_while()[0].eval(), 2)
-        self.assertEqual(printed.contents(), "D\nD\n")
-
-        with self.captureWritesToStream(sys.stderr) as printed:
-          self.assertEqual(build_nested_while()[0].eval(), 2)
-        self.assertEqual(printed.contents(), "D\nD\n")
-
-    # In defuns, all prints should execute in program order.
-    # This doesn't work with legacy control flow.
-    if control_flow_ops.ENABLE_WHILE_V2:
-
-      @eager_function.defun
-      def while_loop():
-        return build_while()[0]
-
-      with self.captureWritesToStream(sys.stderr) as printed:
-        self.assertEqual(self.evaluate(while_loop()), 2)
-      self.assertEqual(printed.contents(), "A\nB\nC\nD\nA\nB\nC\nD\nA\n")
-
-      @eager_function.defun
-      def nested_while_loop():
-        return build_nested_while()[0]
-
-      # TODO(b/117840611): calling nested_while_loop fails in eager
-      if not context.executing_eagerly():
-        with self.captureWritesToStream(sys.stderr) as printed:
-          self.assertEqual(self.evaluate(nested_while_loop()), 2)
-        self.assertEqual(printed.contents(), "A\nB\nC\nD\nA\nB\nC\nD\nA\n")
-
   # Microbenchmark: 256,000 iterations/s.
   @test_util.disable_control_flow_v2("b/116630618 (Times out)")
   def testWhile_1(self):
@@ -1089,7 +1053,7 @@ class ControlFlowTest(test.TestCase):
 
     final_without_xla_context = create_while_loop()
 
-    with self.session(use_gpu=False) as sess:
+    with self.test_session(use_gpu=False) as sess:
       opts = config_pb2.RunOptions(trace_level=config_pb2.RunOptions.FULL_TRACE)
       run_metadata = config_pb2.RunMetadata()
 
@@ -1205,7 +1169,7 @@ class ControlFlowTest(test.TestCase):
       self.assertLess(len(unique_allocs), 756)
 
   def _testWhile_Gpu_1(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       n = constant_op.constant(1.0)
       c = lambda x: math_ops.less(x, 10.0)
       b = lambda x: math_ops.add(x, 1.0)
@@ -1217,7 +1181,7 @@ class ControlFlowTest(test.TestCase):
     self._testWhile_Gpu_1(use_gpu=True)
 
   def _testWhile_Gpu_2(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       n = constant_op.constant(1.0)
       c = lambda x: math_ops.less(x, 10.0)
 
@@ -1359,7 +1323,7 @@ class ControlFlowTest(test.TestCase):
             [i.get_shape(), tensor_shape.TensorShape([None, 5])])
 
   def _testNestedWhile_1(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       n = constant_op.constant(0)
 
       def cpu_sum(s):
@@ -1386,7 +1350,7 @@ class ControlFlowTest(test.TestCase):
 
   def _testNestedWhile_2(self, use_gpu):
     # Test the cases that A -> Enter and Exit -> A are partitioned.
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       s0 = constant_op.constant(2.0)
 
       def inner_loop(s):
@@ -1565,7 +1529,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(10, r.eval())
 
   def _testCondWhile_3(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu) as sess:
+    with self.test_session(use_gpu=use_gpu) as sess:
       p = array_ops.placeholder(dtypes.bool)
       n = constant_op.constant(0.0)
 
@@ -1947,7 +1911,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(2048.0, r.eval())
 
   def _testWhileGrad_Mul(self, use_gpu, p_iters):
-    with self.cached_session(use_gpu=use_gpu) as sess:
+    with self.test_session(use_gpu=use_gpu) as sess:
       a = constant_op.constant(3.0, name="a")
       v = constant_op.constant(2.0, name="v")
       c = lambda v: math_ops.less(v, 100.0)
@@ -1967,7 +1931,7 @@ class ControlFlowTest(test.TestCase):
 
   def _testNestedWhileCondWhileGrad(self, use_gpu):
 
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       v = constant_op.constant(1.0)
 
       def inner_loop(s):
@@ -2273,7 +2237,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(1.0, g.eval())  # y_f_d = x + 1.0, dy_f_d/dx = 1.0
 
   def _testNestedWhileGrad_Simple(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       v = constant_op.constant(1.0)
 
       def inner_loop(s):
@@ -2366,7 +2330,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(2.999, var.eval())
 
   def _testWhileCondGrad_Simple(self, use_gpu):
-    with self.cached_session(use_gpu=use_gpu):
+    with self.test_session(use_gpu=use_gpu):
       v = ops.convert_to_tensor(2.0, name="v")
       n = ops.convert_to_tensor(100.0, name="n")
       one = ops.convert_to_tensor(1.0, name="one")
@@ -2777,6 +2741,8 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(4.0, i.eval(feed_dict={d: 1}))
       self.assertAllClose(2.0 * math.sqrt(2), i.eval(feed_dict={d: 2}))
 
+  @test_util.disable_control_flow_v2(
+      "b/112477618 (Operation returned from cond)")
   def testCase(self):
     with self.cached_session():
       x = constant_op.constant(1)
@@ -2829,6 +2795,8 @@ class ControlFlowTest(test.TestCase):
 
       self.assertAllEqual(r6.eval(), 0)
 
+  @test_util.disable_control_flow_v2(
+      "b/112477618 (Operation returned from cond)")
   def testCaseSideEffects(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(-1)
@@ -3361,7 +3329,7 @@ class TupleTest(test.TestCase):
 class AssertTest(test.TestCase):
 
   def testGuardedAssertDoesNotCopyWhenTrue(self):
-    with self.session(use_gpu=True) as sess:
+    with self.test_session(use_gpu=True) as sess:
       with ops.device(test.gpu_device_name()):
         value = constant_op.constant(1.0)
       with ops.device("/cpu:0"):
-- 
GitLab


From 5e8a5e488b80f544cf810404cfd380305638df10 Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Wed, 24 Oct 2018 01:59:54 +0000
Subject: [PATCH 0047/1554] Commited wrong test file

---
 .../kernel_tests/control_flow_ops_py_test.py  | 186 ++++++++++++++----
 1 file changed, 148 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 7a746ca362..98f8fd4e96 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 
 import collections
 import math
+import sys
 import time
 
 import numpy as np
@@ -455,7 +456,7 @@ class ControlFlowTest(test.TestCase):
     self.assertTrue(ind.dtype == np.int64)
 
   def testCondColocation(self):
-    with self.test_session(use_gpu=True):
+    with self.session(use_gpu=True):
       with ops.device("/cpu:0"):
         v = variables.Variable(7.0)
 
@@ -470,7 +471,7 @@ class ControlFlowTest(test.TestCase):
           self.assertDeviceEqual(op.device, "/cpu:0")
 
   def _testCond_1(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       x = constant_op.constant(10)
       pred = math_ops.less(1, 2)
       fn1 = lambda: math_ops.add(x, 1)
@@ -509,27 +510,40 @@ class ControlFlowTest(test.TestCase):
       result = r.eval()
     self.assertAllEqual(12, result)
 
-  @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
-  def testCond_4(self):
-    with self.cached_session():
-      v1 = variables.Variable(7)
-      v2 = variables.Variable(7)
-      v3 = variables.Variable(7)
+  @test_util.run_in_graph_and_eager_modes
+  def testCondPruning(self):
+    v1 = variables.Variable(7)
+    v2 = variables.Variable(7)
+    v3 = variables.Variable(7)
 
+    def f():
       age = constant_op.constant(3)
       max_age = constant_op.constant(2)
       pred = math_ops.greater(age, max_age)
       fn1 = lambda: [state_ops.assign(v1, 1).op, state_ops.assign(v2, 2).op]
       fn2 = lambda: [state_ops.assign(v3, 3).op, constant_op.constant(10).op]
       r = control_flow_ops.cond(pred, fn1, fn2)
-
-      variables.global_variables_initializer().run()
       self.assertEqual(len(r), 2)
-      result = r[1].eval()
-      self.assertAllEqual(True, result)
-      self.assertAllEqual(7, v1.eval())
-      self.assertAllEqual(2, v2.eval())
-      self.assertAllEqual(7, v3.eval())
+      return r[1]
+
+    f_defun = eager_function.defun(f)
+
+    if not context.executing_eagerly():
+      with self.cached_session():
+        variables.global_variables_initializer().run()
+        result = f().eval()
+        self.assertEqual(True, result)
+        # Only second cond result was fetched, so v1 assign shouldn't run.
+        self.assertEqual(7, v1.eval())
+        self.assertEqual(2, v2.eval())
+        self.assertEqual(7, v3.eval())
+
+    result = f_defun()
+    self.assertEqual(True, self.evaluate(result))
+    # Both v1 and v2 branch assignments should be run in defun.
+    self.assertEqual(1, self.evaluate(v1))
+    self.assertEqual(2, self.evaluate(v2))
+    self.assertEqual(7, self.evaluate(v3))
 
   def testCond_5(self):
     with self.cached_session():
@@ -546,7 +560,6 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(4, count.eval())
 
   def testCond_6(self):
-
     with self.cached_session():
       v1 = variables.Variable([7])
 
@@ -603,7 +616,7 @@ class ControlFlowTest(test.TestCase):
       test_result = sess.run(r)
       self.assertDictEqual({"a": 210, "b": 210}, test_result)
 
-  def testNestedListOutput(self):
+  def testEmbeddedListOutput(self):
     with self.cached_session() as sess:
       x = constant_op.constant(10)
       y = constant_op.constant(200)
@@ -616,7 +629,7 @@ class ControlFlowTest(test.TestCase):
       test_result = sess.run(r)
       self.assertListEqual([[210, 210]], test_result)
 
-  def testNestedTupleOutput(self):
+  def testEmbeddedTupleOutput(self):
     with self.cached_session() as sess:
       x = constant_op.constant(10)
       y = constant_op.constant(200)
@@ -627,7 +640,7 @@ class ControlFlowTest(test.TestCase):
       test_result = sess.run(r)
       self.assertTupleEqual(((210, 210)), test_result)
 
-  def testNestedDictOutput(self):
+  def testEmbeddedDictOutput(self):
     with self.cached_session() as sess:
       x = constant_op.constant(10)
       y = constant_op.constant(200)
@@ -703,8 +716,6 @@ class ControlFlowTest(test.TestCase):
       merged_op = control_flow_ops.merge([assign_v, orig_v])
       self.assertAllEqual([1.0], sess.run(merged_op.output))
 
-  @test_util.disable_control_flow_v2(
-      "b/112477618 (Operation returned from cond)")
   def testCondSwitchIdentity(self):
     # Make sure the recv identity is not removed by optimization.
     with session.Session(config=opt_cfg()) as sess:
@@ -719,8 +730,6 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
       sess.run(r)
 
-  @test_util.disable_control_flow_v2(
-      "b/112477618 (Operation returned from cond)")
   def testCondRecvIdentity(self):
     # Make sure the switch identity is not removed by optimization.
     with session.Session(config=opt_cfg()) as sess:
@@ -819,6 +828,111 @@ class ControlFlowTest(test.TestCase):
       ]
       self.assertAllEqual(dense_gv, [0.0, 2.0])
 
+  # TODO(b/117945658): reenable
+  @test_util.run_in_graph_and_eager_modes
+  def DISABLED_testCondAutoControlDeps(self):
+
+    def branch_fn():
+      logging_ops.print_v2("A")
+      logging_ops.print_v2("B")
+      with ops.control_dependencies([logging_ops.print_v2("C")]):
+        return constant_op.constant(10)
+
+    def build_cond():
+      return control_flow_ops.cond(
+          constant_op.constant(True), branch_fn, lambda: 0)
+
+    def build_nested_cond():
+      return control_flow_ops.cond(
+          constant_op.constant(True), build_cond, lambda: 0)
+
+    # In v1 graph mode, pruning should make only "C" print.
+    if not context.executing_eagerly():
+      with self.cached_session():
+        with self.captureWritesToStream(sys.stderr) as printed:
+          self.assertEqual(build_cond().eval(), 10)
+        self.assertEqual(printed.contents(), "C\n")
+
+        with self.captureWritesToStream(sys.stderr) as printed:
+          self.assertEqual(build_nested_cond().eval(), 10)
+        self.assertEqual(printed.contents(), "C\n")
+
+    # In defuns, all prints should execute in program order.
+    # This doesn't work with legacy control flow.
+    if control_flow_ops.ENABLE_COND_V2:
+
+      @eager_function.defun
+      def cond():
+        return build_cond()
+
+      with self.captureWritesToStream(sys.stderr) as printed:
+        self.assertEqual(self.evaluate(cond()), 10)
+      self.assertEqual(printed.contents(), "A\nB\nC\n")
+
+      @eager_function.defun
+      def nested_cond():
+        return build_nested_cond()
+
+      with self.captureWritesToStream(sys.stderr) as printed:
+        self.assertEqual(self.evaluate(nested_cond()), 10)
+      self.assertEqual(printed.contents(), "A\nB\nC\n")
+
+  # TODO(b/117945658): reenable
+  @test_util.run_in_graph_and_eager_modes
+  def DISABLED_testWhileAutoControlDeps(self):
+
+    def cond(i, unused_x):
+      logging_ops.print_v2("A")
+      return i < 2
+
+    def body(i, x):
+      logging_ops.print_v2("B")
+      with ops.control_dependencies([logging_ops.print_v2("C")]):
+        x = array_ops.identity(x)
+      with ops.control_dependencies([logging_ops.print_v2("D")]):
+        return i + 1, x
+
+    def build_while():
+      return control_flow_ops.while_loop(
+          cond, body, [constant_op.constant(0), constant_op.constant(0)])
+
+    def build_nested_while():
+      return control_flow_ops.cond(
+          constant_op.constant(True), build_while, lambda: (0, 0))
+
+    # In v1 graph mode, pruning should make only "D" print.
+    if not context.executing_eagerly():
+      with self.cached_session():
+        with self.captureWritesToStream(sys.stderr) as printed:
+          self.assertEqual(build_while()[0].eval(), 2)
+        self.assertEqual(printed.contents(), "D\nD\n")
+
+        with self.captureWritesToStream(sys.stderr) as printed:
+          self.assertEqual(build_nested_while()[0].eval(), 2)
+        self.assertEqual(printed.contents(), "D\nD\n")
+
+    # In defuns, all prints should execute in program order.
+    # This doesn't work with legacy control flow.
+    if control_flow_ops.ENABLE_WHILE_V2:
+
+      @eager_function.defun
+      def while_loop():
+        return build_while()[0]
+
+      with self.captureWritesToStream(sys.stderr) as printed:
+        self.assertEqual(self.evaluate(while_loop()), 2)
+      self.assertEqual(printed.contents(), "A\nB\nC\nD\nA\nB\nC\nD\nA\n")
+
+      @eager_function.defun
+      def nested_while_loop():
+        return build_nested_while()[0]
+
+      # TODO(b/117840611): calling nested_while_loop fails in eager
+      if not context.executing_eagerly():
+        with self.captureWritesToStream(sys.stderr) as printed:
+          self.assertEqual(self.evaluate(nested_while_loop()), 2)
+        self.assertEqual(printed.contents(), "A\nB\nC\nD\nA\nB\nC\nD\nA\n")
+
   # Microbenchmark: 256,000 iterations/s.
   @test_util.disable_control_flow_v2("b/116630618 (Times out)")
   def testWhile_1(self):
@@ -1053,7 +1167,7 @@ class ControlFlowTest(test.TestCase):
 
     final_without_xla_context = create_while_loop()
 
-    with self.test_session(use_gpu=False) as sess:
+    with self.session(use_gpu=False) as sess:
       opts = config_pb2.RunOptions(trace_level=config_pb2.RunOptions.FULL_TRACE)
       run_metadata = config_pb2.RunMetadata()
 
@@ -1169,7 +1283,7 @@ class ControlFlowTest(test.TestCase):
       self.assertLess(len(unique_allocs), 756)
 
   def _testWhile_Gpu_1(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       n = constant_op.constant(1.0)
       c = lambda x: math_ops.less(x, 10.0)
       b = lambda x: math_ops.add(x, 1.0)
@@ -1181,7 +1295,7 @@ class ControlFlowTest(test.TestCase):
     self._testWhile_Gpu_1(use_gpu=True)
 
   def _testWhile_Gpu_2(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       n = constant_op.constant(1.0)
       c = lambda x: math_ops.less(x, 10.0)
 
@@ -1323,7 +1437,7 @@ class ControlFlowTest(test.TestCase):
             [i.get_shape(), tensor_shape.TensorShape([None, 5])])
 
   def _testNestedWhile_1(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       n = constant_op.constant(0)
 
       def cpu_sum(s):
@@ -1350,7 +1464,7 @@ class ControlFlowTest(test.TestCase):
 
   def _testNestedWhile_2(self, use_gpu):
     # Test the cases that A -> Enter and Exit -> A are partitioned.
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       s0 = constant_op.constant(2.0)
 
       def inner_loop(s):
@@ -1529,7 +1643,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(10, r.eval())
 
   def _testCondWhile_3(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu) as sess:
+    with self.cached_session(use_gpu=use_gpu) as sess:
       p = array_ops.placeholder(dtypes.bool)
       n = constant_op.constant(0.0)
 
@@ -1911,7 +2025,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(2048.0, r.eval())
 
   def _testWhileGrad_Mul(self, use_gpu, p_iters):
-    with self.test_session(use_gpu=use_gpu) as sess:
+    with self.cached_session(use_gpu=use_gpu) as sess:
       a = constant_op.constant(3.0, name="a")
       v = constant_op.constant(2.0, name="v")
       c = lambda v: math_ops.less(v, 100.0)
@@ -1931,7 +2045,7 @@ class ControlFlowTest(test.TestCase):
 
   def _testNestedWhileCondWhileGrad(self, use_gpu):
 
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       v = constant_op.constant(1.0)
 
       def inner_loop(s):
@@ -2237,7 +2351,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(1.0, g.eval())  # y_f_d = x + 1.0, dy_f_d/dx = 1.0
 
   def _testNestedWhileGrad_Simple(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       v = constant_op.constant(1.0)
 
       def inner_loop(s):
@@ -2330,7 +2444,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(2.999, var.eval())
 
   def _testWhileCondGrad_Simple(self, use_gpu):
-    with self.test_session(use_gpu=use_gpu):
+    with self.cached_session(use_gpu=use_gpu):
       v = ops.convert_to_tensor(2.0, name="v")
       n = ops.convert_to_tensor(100.0, name="n")
       one = ops.convert_to_tensor(1.0, name="one")
@@ -2741,8 +2855,6 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(4.0, i.eval(feed_dict={d: 1}))
       self.assertAllClose(2.0 * math.sqrt(2), i.eval(feed_dict={d: 2}))
 
-  @test_util.disable_control_flow_v2(
-      "b/112477618 (Operation returned from cond)")
   def testCase(self):
     with self.cached_session():
       x = constant_op.constant(1)
@@ -2795,8 +2907,6 @@ class ControlFlowTest(test.TestCase):
 
       self.assertAllEqual(r6.eval(), 0)
 
-  @test_util.disable_control_flow_v2(
-      "b/112477618 (Operation returned from cond)")
   def testCaseSideEffects(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(-1)
@@ -3329,7 +3439,7 @@ class TupleTest(test.TestCase):
 class AssertTest(test.TestCase):
 
   def testGuardedAssertDoesNotCopyWhenTrue(self):
-    with self.test_session(use_gpu=True) as sess:
+    with self.session(use_gpu=True) as sess:
       with ops.device(test.gpu_device_name()):
         value = constant_op.constant(1.0)
       with ops.device("/cpu:0"):
-- 
GitLab


From 67e2c47e2ff1222e141c51ac2794aa9b9207a573 Mon Sep 17 00:00:00 2001
From: Matt Conley <mconley@nvidia.com>
Date: Wed, 24 Oct 2018 13:30:35 -0700
Subject: [PATCH 0048/1554] Disable denormal test on ARM until the architecture
 is supported.

---
 tensorflow/python/kernel_tests/denormal_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/denormal_test.py b/tensorflow/python/kernel_tests/denormal_test.py
index 71a528c4aa..9f65a18c14 100644
--- a/tensorflow/python/kernel_tests/denormal_test.py
+++ b/tensorflow/python/kernel_tests/denormal_test.py
@@ -35,8 +35,8 @@ class DenormalTest(test.TestCase):
       self.assertEqual(tiny, tiny / 16 * 16)
 
   def _flushDenormalsTest(self, use_gpu, dtypes):
-    if platform.machine() == "ppc64le" or platform.machine() == "s390x":
-      # Disabled denormal_test on power/s390x platform
+    if platform.machine() == "ppc64le" or platform.machine() == "s390x" or platform.machine() == "aarch64":
+      # Disabled denormal_test on power/s390x/aarch64 platform
       # Check relevant discussion - https://github.com/tensorflow/tensorflow/issues/11902
       return
     with self.cached_session(use_gpu=use_gpu):
-- 
GitLab


From 183324ff17b4669707731fea957a1648e754960d Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Thu, 25 Oct 2018 00:34:46 +0000
Subject: [PATCH 0049/1554] Fixed versioning error

---
 tensorflow/python/ops/cond_v2.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 175b53b1a2..714cfbf4db 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -468,6 +468,10 @@ def _check_same_outputs(true_graph, false_graph):
         "  true_fn: %s\n"
         "  false_fn: %s" % (true_output_types, false_output_types))
 
+  # Make sure both structured outputs for both graphs have the same structure
+  nest.assert_same_structure(true_graph.structured_outputs, 
+                             false_graph.structured_outputs)
+
 
 def _get_output_shapes(true_graph_outputs, false_graph_outputs):
   output_shapes = [
-- 
GitLab


From 76fd3b394bb170437baee31516967f4d583869be Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Thu, 25 Oct 2018 13:32:54 +0800
Subject: [PATCH 0050/1554] Fix clang-format issues.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 242 +++++++++++------------
 1 file changed, 113 insertions(+), 129 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 233c5ab39b..e041ab14ca 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -22,10 +22,10 @@ limitations under the License.
 #include <memory>
 #include <queue>
 #include <set>
+#include <tuple>
 #include <unordered_set>
 #include <utility>
 #include <vector>
-#include <tuple>
 
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
@@ -513,15 +513,19 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       GetConv2DBackpropFilterOrBiasAddGrad});
 
     //
-    // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D (NHWC) + Transpose (NHWC->
+    // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D
+    // (NHWC) + Transpose (NHWC->
     // NCHW) " => "Conv2D (NCHW). Such patterns occur frequently in Keras.
-    // Note: we use the term "merge" is to combine (exactly) 2 nodes into one, while "fusion" is
+    // Note: we use the term "merge" is to combine (exactly) 2 nodes into one,
+    // while "fusion" is
     // for 3+ nodes situation.
     //
 
     // Transpose + Conv2d + Transpose:
-    std::vector<int> transpose_to_nhwc = { NCHW::dim::N, NCHW::dim::H, NCHW::dim::W, NCHW::dim::C };
-    std::vector<int> transpose_to_nchw = { NHWC::dim::N, NHWC::dim::C, NHWC::dim::H, NHWC::dim::W };
+    std::vector<int> transpose_to_nhwc = {NCHW::dim::N, NCHW::dim::H,
+                                          NCHW::dim::W, NCHW::dim::C};
+    std::vector<int> transpose_to_nchw = {NHWC::dim::N, NHWC::dim::C,
+                                          NHWC::dim::H, NHWC::dim::W};
     auto CheckForTransposeToNHWC =
         std::bind(CheckForTranspose, std::placeholders::_1, transpose_to_nhwc);
     auto CheckForConv2dOp =
@@ -531,13 +535,12 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     auto FuseConv2D =
         std::bind(FuseTransposeMklOpTranspose, std::placeholders::_1,
                   std::placeholders::_2, std::placeholders::_3, "NCHW");
-    finfo_.push_back({
-      "transpose-elimination for Conv2D", {
-        CheckForTransposeToNHWC, CheckForConv2dOp, CheckForTransposeToNCHW
-      },
-      // CheckForMklOp
-      FuseConv2D, CopyAttrsConv
-    });
+    finfo_.push_back(
+        {"transpose-elimination for Conv2D",
+         {CheckForTransposeToNHWC, CheckForConv2dOp, CheckForTransposeToNCHW},
+         // CheckForMklOp
+         FuseConv2D,
+         CopyAttrsConv});
   }
 
   // Standard interface to run pass
@@ -592,20 +595,20 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
   // structure to specify information used in node fusion of 2+ operators
   typedef struct {
-    std::string pattern_name; // name to describe this pattern, such as
-                              // "Transpose_Mklop_Transpose".
-    std::vector<std::function<bool(const Node *)> >
-        node_checkers; // extra restriction checker for these ops
-    std::function<
-        Status(std::unique_ptr<Graph> *, std::vector<Node *> &,
-               std::function<void(const Node *, NodeBuilder *nb, bool)>)>
+    std::string pattern_name;  // name to describe this pattern, such as
+                               // "Transpose_Mklop_Transpose".
+    std::vector<std::function<bool(const Node*)> >
+        node_checkers;  // extra restriction checker for these ops
+    std::function<Status(
+        std::unique_ptr<Graph>*, std::vector<Node*>&,
+        std::function<void(const Node*, NodeBuilder* nb, bool)>)>
         fuse_func;
-    std::function<void(const Node *, NodeBuilder *nb, bool)> copy_attrs;
+    std::function<void(const Node*, NodeBuilder* nb, bool)> copy_attrs;
   } FusionInfo;
 
   //
   // dimension indices for 2D tensor.
-  // 
+  //
   struct NCHW {
     enum dim { N = 0, C = 1, H = 2, W = 3 };
   };
@@ -614,7 +617,6 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     enum dim { N = 0, H = 1, W = 2, C = 3 };
   };
 
-
   //
   // dimension indices for 3D tensor.
   //
@@ -889,30 +891,29 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   //
   // @return tuple. If we can find such nodes, the first
   // element of the tuple is a true. Otherwise, it's false.
-  std::tuple<bool, std::vector<Node *>, const MklLayoutRewritePass::FusionInfo>
-      CheckForNodeFusion(Node *n) const;
+  std::tuple<bool, std::vector<Node*>, const MklLayoutRewritePass::FusionInfo>
+  CheckForNodeFusion(Node* n) const;
 
   // Fuse nodes in the vector "nodes"
-  Status FuseNode(std::unique_ptr<Graph> *g, std::vector<Node *> &nodes,
+  Status FuseNode(std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
                   const MklLayoutRewritePass::FusionInfo fi);
 
   static Status FuseTransposeMklOpTranspose(
-      std::unique_ptr<Graph> *g, std::vector<Node *> &nodes,
-      std::function<void(const Node *, NodeBuilder *nb, bool)> copy_attrs,
+      std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
+      std::function<void(const Node*, NodeBuilder* nb, bool)> copy_attrs,
       string data_format);
 
-  static bool CheckForTranspose(const Node *node, std::vector<int> perm) {
+  static bool CheckForTranspose(const Node* node, std::vector<int> perm) {
     //
     // Check node node, to see if it's "Transpose"
     //
-    if (node->type_string() != "Transpose")
-      return false;
+    if (node->type_string() != "Transpose") return false;
 
     //
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
     //
-    for (const Edge *e : node->out_edges()) {
+    for (const Edge* e : node->out_edges()) {
       if (e->IsControlEdge()) {
         return false;
       }
@@ -921,7 +922,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     //
     // If "Transpose" has input control edges, don't fuse on it.
     //
-    for (const Edge *e : node->in_edges()) {
+    for (const Edge* e : node->in_edges()) {
       if (e->IsControlEdge()) {
         return false;
       }
@@ -930,19 +931,19 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     //
     // If "Transpose" has multiple output data edges, also don't fuse it.
     //
-    if (node->num_outputs() > 1 || node->out_edges().size() > 1)
-      return false;
+    if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
 
     // Check "perm" attribute, make sure it's what we want.
     //
-    for (const Edge *e : node->in_edges()) {
+    for (const Edge* e : node->in_edges()) {
       if (!e->IsControlEdge()) {
-        const Node *perm_node = e->src();
+        const Node* perm_node = e->src();
 
         const int kPermTensorIndex = 1;
-        if (perm_node->type_string() == "Const" && e->dst_input() == kPermTensorIndex) {
+        if (perm_node->type_string() == "Const" &&
+            e->dst_input() == kPermTensorIndex) {
           // we find the "perm" node, now try to retrieve its value.
-          const TensorProto *proto = nullptr;
+          const TensorProto* proto = nullptr;
           CHECK_EQ(GetNodeAttr(perm_node->def(), "value", &proto).ok(), true);
 
           DataType type;
@@ -955,23 +956,28 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
           //
           if (type == DT_INT32) {
             const int type_size = 4;
-            const int *tensor_content = reinterpret_cast<const int *>(proto->tensor_content().c_str());
-            const int tensor_content_size = proto->tensor_content().size() / type_size;
-  
-            std::vector<int> perm_value(tensor_content, tensor_content + tensor_content_size);
+            const int* tensor_content =
+                reinterpret_cast<const int*>(proto->tensor_content().c_str());
+            const int tensor_content_size =
+                proto->tensor_content().size() / type_size;
+
+            std::vector<int> perm_value(tensor_content,
+                                        tensor_content + tensor_content_size);
 
             return perm_value == perm;
 
           } else if (type == DT_INT64) {
             const int type_size = 8;
-            const long *tensor_content = reinterpret_cast<const long *>(proto->tensor_content().c_str());
-            const int tensor_content_size = proto->tensor_content().size() / type_size;
-  
-            std::vector<long> perm_value(tensor_content, tensor_content + tensor_content_size);
+            const long* tensor_content =
+                reinterpret_cast<const long*>(proto->tensor_content().c_str());
+            const int tensor_content_size =
+                proto->tensor_content().size() / type_size;
+
+            std::vector<long> perm_value(tensor_content,
+                                         tensor_content + tensor_content_size);
             std::vector<long> long_perm(perm.cbegin(), perm.cend());
 
             return perm_value == long_perm;
-
           }
 
           return false;
@@ -982,17 +988,15 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return false;
   }
 
-  static bool CheckForMklOp(const Node *node, string name = "") {
+  static bool CheckForMklOp(const Node* node, string name = "") {
     if (!name.empty() && node->type_string() != name) {
       return false;
     }
 
     // if mklop has multiple outputs, don't fuse it.
-    if (node->num_outputs() > 1)
-      return false;
+    if (node->num_outputs() > 1) return false;
 
-    if (node->out_edges().size() > 1)
-      return false;
+    if (node->out_edges().size() > 1) return false;
 
     DataType T;
     TF_CHECK_OK(GetNodeAttr(node->def(), "T", &T));
@@ -1255,23 +1259,23 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   // We need operator-specific function to copy attributes because the framework
   // does not provide any generic function for it.
   // NOTE: names are alphabetically sorted.
-  static void CopyAttrsAddN(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsAddN(const Node* orig_node, NodeBuilder* nb,
                             bool change_format = false);
-  static void CopyAttrsBiasAddGrad(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsBiasAddGrad(const Node* orig_node, NodeBuilder* nb,
                                    bool change_format = false);
-  static void CopyAttrsConcat(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsConcat(const Node* orig_node, NodeBuilder* nb,
                               bool change_format = false);
-  static void CopyAttrsConcatV2(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsConcatV2(const Node* orig_node, NodeBuilder* nb,
                                 bool change_format = false);
-  static void CopyAttrsConv(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
                             bool change_format = false);
-  static void CopyAttrsDataType(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsDataType(const Node* orig_node, NodeBuilder* nb,
                                 bool change_format = false);
-  static void CopyAttrsFusedBatchNorm(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb,
                                       bool change_format = false);
-  static void CopyAttrsLRN(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb,
                            bool change_format = false);
-  static void CopyAttrsPooling(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb,
                                bool change_format = false);
   static void CopyAttrsQuantizedPooling(const Node* orig_node, NodeBuilder* nb,
                                         bool change_format = false);
@@ -1279,16 +1283,15 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                                        bool change_format = false);
   static void CopyAttrsQuantizedConcat(const Node* orig_node, NodeBuilder* nb,
                                        bool change_format = false);
-  static void CopyAttrsReshape(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsReshape(const Node* orig_node, NodeBuilder* nb,
                                bool change_format = false);
   static void CopyAttrsRequantize(const Node* orig_node, NodeBuilder* nb,
                                   bool change_format = false);
   static void CopyAttrsSlice(const Node* orig_node, NodeBuilder* nb,
                              bool change_format = false);
-  static void CopyAttrsSplit(const Node *orig_node, NodeBuilder *nb,
+  static void CopyAttrsSplit(const Node* orig_node, NodeBuilder* nb,
                              bool change_format = false);
 
-
   // Generate a graph node in graph 'g' representing a dummy Mkl tensor node,
   // using node for original node 'orig_node' and return it in '*out'.
   // TODO(nhasabni) We should move this to mkl_util.h
@@ -1788,7 +1791,7 @@ void MklLayoutRewritePass::AddWorkSpaceEdgeIfNeeded(
 // Op-specific functions to copy attributes from old node to new node
 //////////////////////////////////////////////////////////////////////////
 
-void MklLayoutRewritePass::CopyAttrsConv(const Node *orig_node, NodeBuilder *nb,
+void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
                                          bool change_format) {
   DataType T;
   string data_format;
@@ -1817,55 +1820,40 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node *orig_node, NodeBuilder *nb,
     std::vector<int32> new_dilations;
     if (strides.size() == 5) {
       //
-      // "strides" and "dilations" also need to be changed according to "data_format",
+      // "strides" and "dilations" also need to be changed according to
+      // "data_format",
       // in this case, is "NDHWC" to "NCDHW".
       //
 
-      new_strides = { 
-                      strides[NDHWC::dim::N], 
-                      strides[NDHWC::dim::C],
-                      strides[NDHWC::dim::D],
-                      strides[NDHWC::dim::H],
-                      strides[NDHWC::dim::W]
-                    };
+      new_strides = {strides[NDHWC::dim::N], strides[NDHWC::dim::C],
+                     strides[NDHWC::dim::D], strides[NDHWC::dim::H],
+                     strides[NDHWC::dim::W]};
       nb->Attr("strides", new_strides);
 
-      new_dilations = { 
-                      dilations[NDHWC::dim::N], 
-                      dilations[NDHWC::dim::C],
-                      dilations[NDHWC::dim::D],
-                      dilations[NDHWC::dim::H],
-                      dilations[NDHWC::dim::W]
-                    };
+      new_dilations = {dilations[NDHWC::dim::N], dilations[NDHWC::dim::C],
+                       dilations[NDHWC::dim::D], dilations[NDHWC::dim::H],
+                       dilations[NDHWC::dim::W]};
       nb->Attr("dilations", new_dilations);
 
     } else {
       //
-      // "strides" and "dilations" also need to be changed according to "data_format",
+      // "strides" and "dilations" also need to be changed according to
+      // "data_format",
       // in this case, is "NHWC" to "NCHW".
       //
-      
-      new_strides = {
-                      strides[NHWC::dim::N],
-                      strides[NHWC::dim::C],
-                      strides[NHWC::dim::H],
-                      strides[NHWC::dim::W]
-                    };
+
+      new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
+                     strides[NHWC::dim::H], strides[NHWC::dim::W]};
       nb->Attr("strides", new_strides);
 
-      new_dilations = {
-                      dilations[NHWC::dim::N],
-                      dilations[NHWC::dim::C],
-                      dilations[NHWC::dim::H],
-                      dilations[NHWC::dim::W]
-                    };
+      new_dilations = {dilations[NHWC::dim::N], dilations[NHWC::dim::C],
+                       dilations[NHWC::dim::H], dilations[NHWC::dim::W]};
       nb->Attr("dilations", new_dilations);
     }
   }
 }
 
-void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node,
-                                         NodeBuilder* nb,
+void MklLayoutRewritePass::CopyAttrsAddN(const Node* orig_node, NodeBuilder* nb,
                                          bool change_format) {
   DataType T;
   int N;
@@ -1897,8 +1885,7 @@ void MklLayoutRewritePass::CopyAttrsBiasAddGrad(const Node* orig_node,
   nb->Attr("data_format", data_format);
 }
 
-void MklLayoutRewritePass::CopyAttrsLRN(const Node* orig_node,
-                                        NodeBuilder* nb,
+void MklLayoutRewritePass::CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb,
                                         bool change_format) {
   DataType T;
   int depth_radius;
@@ -2037,8 +2024,7 @@ void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsSlice(const Node* orig_node,
-                                          NodeBuilder* nb,
-                                          bool change_format) {
+                                          NodeBuilder* nb, bool change_format) {
   DataType T;
   DataType Index;
 
@@ -2051,8 +2037,7 @@ void MklLayoutRewritePass::CopyAttrsSlice(const Node* orig_node,
 }
 
 void MklLayoutRewritePass::CopyAttrsSplit(const Node* orig_node,
-                                          NodeBuilder* nb,
-                                          bool change_format) {
+                                          NodeBuilder* nb, bool change_format) {
   DataType T;
   string data_format;
   int num_split;
@@ -2665,28 +2650,28 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const {
 //           Helper functions for node fusion
 //////////////////////////////////////////////////////////////////////////
 Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
-    std::unique_ptr<Graph> *g, std::vector<Node *> &nodes,
-    std::function<void(const Node *, NodeBuilder *nb, bool)> copy_attrs,
+    std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
+    std::function<void(const Node*, NodeBuilder* nb, bool)> copy_attrs,
     string data_format) {
-  Node *transpose_to_nhwc = nodes[0];
-  Node *mklop = nodes[1];
-  Node *transpose_to_nchw = nodes[2];
+  Node* transpose_to_nhwc = nodes[0];
+  Node* mklop = nodes[1];
+  Node* transpose_to_nchw = nodes[2];
 
   const int transpose_nhwc_num_inputs = transpose_to_nhwc->num_inputs();
-  gtl::InlinedVector<Node *, 4> transpose_nhwc_control_edges;
-  gtl::InlinedVector<std::pair<Node *, int>, 4> transpose_nhwc_in(
+  gtl::InlinedVector<Node*, 4> transpose_nhwc_control_edges;
+  gtl::InlinedVector<std::pair<Node*, int>, 4> transpose_nhwc_in(
       transpose_nhwc_num_inputs);
   FillInputs(transpose_to_nhwc, &transpose_nhwc_control_edges,
              &transpose_nhwc_in);
 
   const int mklop_num_inputs = mklop->num_inputs();
-  gtl::InlinedVector<Node *, 4> mklop_control_edges;
-  gtl::InlinedVector<std::pair<Node *, int>, 4> mklop_in(mklop_num_inputs);
+  gtl::InlinedVector<Node*, 4> mklop_control_edges;
+  gtl::InlinedVector<std::pair<Node*, int>, 4> mklop_in(mklop_num_inputs);
   FillInputs(mklop, &mklop_control_edges, &mklop_in);
 
   const int transpose_nchw_num_inputs = transpose_to_nchw->num_inputs();
-  gtl::InlinedVector<Node *, 4> transpose_nchw_control_edges;
-  gtl::InlinedVector<std::pair<Node *, int>, 4> transpose_nchw_in(
+  gtl::InlinedVector<Node*, 4> transpose_nchw_control_edges;
+  gtl::InlinedVector<std::pair<Node*, int>, 4> transpose_nchw_in(
       transpose_nchw_num_inputs);
   FillInputs(transpose_to_nhwc, &transpose_nchw_control_edges,
              &transpose_nchw_in);
@@ -2706,19 +2691,19 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
     }
   }
 
-  copy_attrs(const_cast<const Node *>(mklop), &nb, true);
+  copy_attrs(const_cast<const Node*>(mklop), &nb, true);
   nb.Attr("data_format", data_format);
 
   // Copy the device assigned to old node to new node.
   nb.Device(mklop->def().device());
 
   // Create node.
-  Node *new_node;
+  Node* new_node;
   TF_CHECK_OK(nb.Finalize(&**g, &new_node));
   CHECK_NOTNULL(new_node);
 
   // Fill outputs.
-  for (const Edge *e : transpose_to_nchw->out_edges()) {
+  for (const Edge* e : transpose_to_nchw->out_edges()) {
     if (!e->IsControlEdge()) {
       const int kConv2DWithBiasOutputSlot = 0;
       CHECK_NOTNULL((*g)->AddEdge(new_node, kConv2DWithBiasOutputSlot, e->dst(),
@@ -2736,18 +2721,17 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   return Status::OK();
 }
 
-Status
-MklLayoutRewritePass::FuseNode(std::unique_ptr<Graph> *g,
-                               std::vector<Node *> &nodes,
-                               const MklLayoutRewritePass::FusionInfo fi) {
+Status MklLayoutRewritePass::FuseNode(
+    std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
+    const MklLayoutRewritePass::FusionInfo fi) {
   return fi.fuse_func(g, nodes, fi.copy_attrs);
 }
 
-std::tuple<bool, std::vector<Node *>, const MklLayoutRewritePass::FusionInfo>
-MklLayoutRewritePass::CheckForNodeFusion(Node *a) const {
+std::tuple<bool, std::vector<Node*>, const MklLayoutRewritePass::FusionInfo>
+MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
   bool found_pattern = false;
-  std::vector<Node *> nodes;
-  const FusionInfo *fi_ptr = nullptr;
+  std::vector<Node*> nodes;
+  const FusionInfo* fi_ptr = nullptr;
 
   for (auto fi = finfo_.begin(); fi != finfo_.end(); ++fi) {
     assert(fi->ops.size() == fi->node_checkers.size());
@@ -2760,7 +2744,7 @@ MklLayoutRewritePass::CheckForNodeFusion(Node *a) const {
     //
 
     // Initialize "current_node" as node "a".
-    Node *current_node = a;
+    Node* current_node = a;
     for (auto node_index = 0; node_index < fi->node_checkers.size();
          ++node_index) {
       // Make sure current node meet the requirement of corresponding node
@@ -2781,9 +2765,9 @@ MklLayoutRewritePass::CheckForNodeFusion(Node *a) const {
         // Find current node's direct descendant, which will be used in next
         // iteration.
         auto check_next_node = fi->node_checkers[node_index + 1];
-        for (const Edge *e : current_node->out_edges()) {
+        for (const Edge* e : current_node->out_edges()) {
           if (!e->IsControlEdge()) {
-            Node *candidate_node = e->dst();
+            Node* candidate_node = e->dst();
 
             if (check_next_node(candidate_node) == false) {
               current_node = nullptr;
@@ -2933,7 +2917,7 @@ bool MklLayoutRewritePass::RunPass(std::unique_ptr<Graph>* g) {
 
   order.clear();
   GetReversePostOrder(**g, &order);  // This will give us topological sort.
-  for (Node *n : order) {
+  for (Node* n : order) {
     // If node is not an op or it cannot run on CPU device, then skip.
     if (!n->IsOp() || !CanOpRunOnCPUDevice(n)) {
       continue;
@@ -2941,7 +2925,7 @@ bool MklLayoutRewritePass::RunPass(std::unique_ptr<Graph>* g) {
 
     auto check_result = CheckForNodeFusion(n);
     bool found_pattern = std::get<0>(check_result);
-    std::vector<Node *> nodes = std::get<1>(check_result);
+    std::vector<Node*> nodes = std::get<1>(check_result);
     const FusionInfo fi = std::get<2>(check_result);
 
     // if "found_pattern" is true, we can do the fusion.
-- 
GitLab


From 8d068a857dfbeb5cd417eb342a6ba7c330e0a270 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Thu, 25 Oct 2018 13:46:07 +0800
Subject: [PATCH 0051/1554] Fxi clang-format issues.

---
 tensorflow/core/graph/mkl_layout_pass_test.cc | 238 +++++++++---------
 1 file changed, 126 insertions(+), 112 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index b09ef3b970..f4f2ab2a97 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -457,53 +457,57 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_ConvBpropInput_FilterFwd) {
 
 TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Positive) {
   InitGraph(
-    "node { name: 'Input0' op: 'Input'}"
-    "node { name: 'Input1' op: 'Input'}"
-    "node { name: 'Const0' op: 'Const'"
-    "  attr {"
-    "   key: 'dtype'"
-    "   value {"
-    "     type: DT_INT32"
-    "   }"
-    "  }"
-    " attr {"
-    "   key: 'value'"
-    "   value {"
-    "     tensor {"
-    "       dtype: DT_INT32"
-    "       tensor_shape {"
-    "         dim {"
-    "           size: 4"
-    "         }"
-    "       }"
-    "       tensor_content: '\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000'"
-    "     }"
-    "   }"
-    " }"
-    "}"
-    "node { name: 'Const1' op: 'Const'"
-    "  attr {"
-    "   key: 'dtype'"
-    "   value {"
-    "     type: DT_INT32"
-    "   }"
-    "  }"
-    " attr {"
-    "   key: 'value'"
-    "   value {"
-    "     tensor {"
-    "       dtype: DT_INT32"
-    "       tensor_shape {"
-    "         dim {"
-    "           size: 4"
-    "         }"
-    "       }"
-    "       tensor_content: '\\000\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000\\002\\000\\000\\000'"
-    "     }"
-    "   }"
-    " }"
-    "}"
-    "node {              \
+      "node { name: 'Input0' op: 'Input'}"
+      "node { name: 'Input1' op: 'Input'}"
+      "node { name: 'Const0' op: 'Const'"
+      "  attr {"
+      "   key: 'dtype'"
+      "   value {"
+      "     type: DT_INT32"
+      "   }"
+      "  }"
+      " attr {"
+      "   key: 'value'"
+      "   value {"
+      "     tensor {"
+      "       dtype: DT_INT32"
+      "       tensor_shape {"
+      "         dim {"
+      "           size: 4"
+      "         }"
+      "       }"
+      "       tensor_content: "
+      "'\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000"
+      "\\000\\000'"
+      "     }"
+      "   }"
+      " }"
+      "}"
+      "node { name: 'Const1' op: 'Const'"
+      "  attr {"
+      "   key: 'dtype'"
+      "   value {"
+      "     type: DT_INT32"
+      "   }"
+      "  }"
+      " attr {"
+      "   key: 'value'"
+      "   value {"
+      "     tensor {"
+      "       dtype: DT_INT32"
+      "       tensor_shape {"
+      "         dim {"
+      "           size: 4"
+      "         }"
+      "       }"
+      "       tensor_content: "
+      "'\\000\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000\\002\\000"
+      "\\000\\000'"
+      "     }"
+      "   }"
+      " }"
+      "}"
+      "node {              \
       name: 'Transpose0' \
       op: 'Transpose'    \
       input: 'Input0'    \
@@ -520,8 +524,8 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Positive) {
           type: DT_INT32 \
         }                \
       }                  \
-    }"                   
-    "node {                 \
+    }"
+      "node {                 \
       name: 'Conv2D'        \
       op: 'Conv2D'          \
       input: 'Transpose0'   \
@@ -573,7 +577,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Positive) {
         }                       \
       }                         \
     }"
-    "node {              \
+      "node {              \
       name: 'Transpose1' \
       op: 'Transpose'    \
       input: 'Conv2D'    \
@@ -591,65 +595,71 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Positive) {
         }                \
       }                  \
     }"
-    "node { name: 'Relu' op: 'Relu'"
+      "node { name: 'Relu' op: 'Relu'"
       " attr { key: 'T'                value { type: DT_FLOAT } }"
       " input: ['Transpose1'] }");
   EXPECT_EQ(DoMklLayoutOptimizationPass(),
             "Const0(Const);Const1(Const);"
             "Conv2D(_MklConv2D);DMT/_0(Const);DMT/_1(Const);Input0(Input);"
-            "Input1(Input);Relu(_MklRelu)|Conv2D->Relu;Conv2D:2->Relu:1;DMT/_0->Conv2D:2;DMT/_1->Conv2D:3;Input0->Conv2D;"
-            "Input0:control->DMT/_0:control;Input0:control->DMT/_1:control;Input1->Conv2D:1");
+            "Input1(Input);Relu(_MklRelu)|Conv2D->Relu;Conv2D:2->Relu:1;DMT/"
+            "_0->Conv2D:2;DMT/_1->Conv2D:3;Input0->Conv2D;"
+            "Input0:control->DMT/_0:control;Input0:control->DMT/"
+            "_1:control;Input1->Conv2D:1");
 }
 
 TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Negative) {
   InitGraph(
-    "node { name: 'Input0' op: 'Input'}"
-    "node { name: 'Input1' op: 'Input'}"
-    "node { name: 'Const0' op: 'Const'"
-    "  attr {"
-    "   key: 'dtype'"
-    "   value {"
-    "     type: DT_INT32"
-    "   }"
-    "  }"
-    " attr {"
-    "   key: 'value'"
-    "   value {"
-    "     tensor {"
-    "       dtype: DT_INT32"
-    "       tensor_shape {"
-    "         dim {"
-    "           size: 4"
-    "         }"
-    "       }"
-    "       tensor_content: '\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000'"
-    "     }"
-    "   }"
-    " }"
-    "}"
-    "node { name: 'Const1' op: 'Const'"
-    "  attr {"
-    "   key: 'dtype'"
-    "   value {"
-    "     type: DT_INT32"
-    "   }"
-    "  }"
-    " attr {"
-    "   key: 'value'"
-    "   value {"
-    "     tensor {"
-    "       dtype: DT_INT32"
-    "       tensor_shape {"
-    "         dim {"
-    "           size: 4"
-    "         }"
-    "       }"
-    "       tensor_content: '\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000\\000\\000'"
-    "     }"
-    "   }"
-    " }"
-    "}"
-    "node {              \
+      "node { name: 'Input0' op: 'Input'}"
+      "node { name: 'Input1' op: 'Input'}"
+      "node { name: 'Const0' op: 'Const'"
+      "  attr {"
+      "   key: 'dtype'"
+      "   value {"
+      "     type: DT_INT32"
+      "   }"
+      "  }"
+      " attr {"
+      "   key: 'value'"
+      "   value {"
+      "     tensor {"
+      "       dtype: DT_INT32"
+      "       tensor_shape {"
+      "         dim {"
+      "           size: 4"
+      "         }"
+      "       }"
+      "       tensor_content: "
+      "'\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000"
+      "\\000\\000'"
+      "     }"
+      "   }"
+      " }"
+      "}"
+      "node { name: 'Const1' op: 'Const'"
+      "  attr {"
+      "   key: 'dtype'"
+      "   value {"
+      "     type: DT_INT32"
+      "   }"
+      "  }"
+      " attr {"
+      "   key: 'value'"
+      "   value {"
+      "     tensor {"
+      "       dtype: DT_INT32"
+      "       tensor_shape {"
+      "         dim {"
+      "           size: 4"
+      "         }"
+      "       }"
+      "       tensor_content: "
+      "'\\000\\000\\000\\000\\002\\000\\000\\000\\003\\000\\000\\000\\001\\000"
+      "\\000\\000'"
+      "     }"
+      "   }"
+      " }"
+      "}"
+      "node {              \
       name: 'Transpose0' \
       op: 'Transpose'    \
       input: 'Input0'    \
@@ -666,8 +676,8 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Negative) {
           type: DT_INT32 \
         }                \
       }                  \
-    }"                   
-    "node {                 \
+    }"
+      "node {                 \
       name: 'Conv2D'        \
       op: 'Conv2D'          \
       input: 'Transpose0'   \
@@ -719,7 +729,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Negative) {
         }                       \
       }                         \
     }"
-    "node {              \
+      "node {              \
       name: 'Transpose1' \
       op: 'Transpose'    \
       input: 'Conv2D'    \
@@ -737,17 +747,21 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Negative) {
         }                \
       }                  \
     }"
-    "node { name: 'Relu' op: 'Relu'"
+      "node { name: 'Relu' op: 'Relu'"
       " attr { key: 'T'                value { type: DT_FLOAT } }"
       " input: ['Transpose1'] }");
-  EXPECT_EQ(DoMklLayoutOptimizationPass(),
-            "Const0(Const);Const1(Const);"
-            "Conv2D(_MklConv2D);DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);"
-            "Input0(Input);Input1(Input);Relu(_MklRelu);"
-            "Transpose0(Transpose);Transpose1(Transpose)|Const0->Transpose0:1;Const1->Transpose1:1;"
-            "Conv2D->Transpose1;DMT/_0->Conv2D:2;DMT/_1->Conv2D:3;DMT/_2->Relu:1;Input0->Transpose0;"
-            "Input1->Conv2D:1;Transpose0->Conv2D;Transpose0:control->DMT/_0:control;"
-            "Transpose0:control->DMT/_1:control;Transpose1->Relu;Transpose1:control->DMT/_2:control");
+  EXPECT_EQ(
+      DoMklLayoutOptimizationPass(),
+      "Const0(Const);Const1(Const);"
+      "Conv2D(_MklConv2D);DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);"
+      "Input0(Input);Input1(Input);Relu(_MklRelu);"
+      "Transpose0(Transpose);Transpose1(Transpose)|Const0->Transpose0:1;Const1-"
+      ">Transpose1:1;"
+      "Conv2D->Transpose1;DMT/_0->Conv2D:2;DMT/_1->Conv2D:3;DMT/"
+      "_2->Relu:1;Input0->Transpose0;"
+      "Input1->Conv2D:1;Transpose0->Conv2D;Transpose0:control->DMT/_0:control;"
+      "Transpose0:control->DMT/"
+      "_1:control;Transpose1->Relu;Transpose1:control->DMT/_2:control");
 }
 
 /////////////////////////////////////////////////////////////////////
-- 
GitLab


From 013d74dd7e96b96decccc059f9bdf2f2b01348c5 Mon Sep 17 00:00:00 2001
From: Keno Fischer <keno@juliacomputing.com>
Date: Thu, 25 Oct 2018 16:23:33 -0400
Subject: [PATCH 0052/1554] [XLA:GPU/CuDNN] Add support for 1x1 window reversal

CuDNN supports both convolution (CUDNN_CONVOLUTION) and cross correlation
(CUDNN_CROSS_CORRELATION). However, only the latter was hooked up, causing

    Tensorflow error: Status: Hit a case for convolution that is not implemented on GPU.

for convolutions of the first kind (corresponding to convolutions with both
window dimensions reversed at the HLO level). Reversing the dimensions (i.e.
doing convolutions rather than cross correlations) is the default behavior
for the Flux.jl ML framework, so it's easy to hit this error trying to run
pre-existing Flux models through the Julia:XLA->XLA:GPU compilation path.

Plumb through the reversal option to CuDNN to make this pattern work. The
same HLO already works fine against the CPU and TPU backends.
---
 .../compiler/xla/service/gpu/cudnn_conv_rewriter.cc  |  6 +++++-
 .../compiler/xla/service/gpu/cudnn_conv_runner.cc    |  7 +++++++
 tensorflow/compiler/xla/tests/convolution_test.cc    | 12 ++++++++++++
 tensorflow/compiler/xla/window_util.cc               | 11 +++++++++++
 tensorflow/compiler/xla/window_util.h                |  1 +
 tensorflow/stream_executor/cuda/cuda_dnn.cc          |  8 ++++----
 tensorflow/stream_executor/dnn.cc                    |  3 ++-
 tensorflow/stream_executor/dnn.h                     | 11 +++++++++++
 8 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc
index c46672c598..73e2a1f719 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_rewriter.cc
@@ -77,7 +77,11 @@ bool CanImplementAsCudnnForwardConv(HloInstruction* conv) {
     return false;
   }
 
-  if (window_util::HasWindowReversal(conv->window())) {
+  // CuDNN can perform either cross correlation (no reversal),
+  // or convolution (all dimensions reversed).
+  if (dnums.input_spatial_dimensions_size() == 2
+          ? !window_util::AllOrNoneReversed(conv->window())
+          : window_util::HasWindowReversal(conv->window())) {
     return false;
   }
   return true;
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
index 0006e85e16..5c93bcc2d2 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
@@ -138,6 +138,7 @@ Status RunCudnnConvImpl(CudnnConvParams params,
 
   const int num_dimensions = window.dimensions_size();
   CHECK_LE(num_dimensions, 3);
+  CHECK_GE(num_dimensions, 1);
   // cuDNN does not support 1D convolutions. We therefore express 1D
   // convolutions as 2D convolutions where the first spatial dimension is 1.
   // This matches the behavior of TF (see definition of conv1d in
@@ -148,10 +149,15 @@ Status RunCudnnConvImpl(CudnnConvParams params,
            output_shape.element_type())
       << ShapeUtil::HumanString(output_shape);
 
+  // If one dimension is reversed, we need to have all dimensions reversed (so
+  // we're doing convolution not cross correlation).
+  const bool dims_reversed = window.dimensions()[0].window_reversal();
+
   CHECK_EQ(num_dimensions, dnums.input_spatial_dimensions_size());
   CHECK_EQ(num_dimensions, dnums.kernel_spatial_dimensions_size());
   CHECK_EQ(num_dimensions, dnums.output_spatial_dimensions_size());
   for (const WindowDimension& dim : window.dimensions()) {
+    CHECK_EQ(dims_reversed, dim.window_reversal());
     CHECK_EQ(dim.padding_low(), dim.padding_high());
     CHECK_EQ(dim.base_dilation(), 1)
         << "cudnn does not support base dilation; it "
@@ -198,6 +204,7 @@ Status RunCudnnConvImpl(CudnnConvParams params,
 
   ConvolutionDescriptor convolution_descriptor(effective_num_dimensions);
   convolution_descriptor.set_group_count(feature_group_count);
+  convolution_descriptor.set_convolution_not_crosscorr(dims_reversed);
   for (int dim = 0; dim < num_dimensions; ++dim) {
     convolution_descriptor
         .set_zero_padding(
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 3aebf78466..6c905f50b6 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -951,6 +951,18 @@ ENTRY Test {
   EXPECT_TRUE(RunAndCompare(kHlo, ErrorSpec{0.001}));
 }
 
+XLA_TEST_F(ConvolutionHloTest, DISABLED_ON_CPU(ConvolveF64ForwardReversed)) {
+  constexpr char kHlo[] = R"(
+HloModule TestModule
+
+ENTRY Test {
+  %arg0 = f64[3,56,56,16] parameter(0)
+  %arg1 = f64[3,3,3,64] parameter(1)
+  ROOT %conv = f64[54,54,16,64] convolution(%arg0, %arg1), window={size=3x3 rhs_reversal=1x1}, dim_labels=f01b_i01o->01bf
+})";
+  EXPECT_TRUE(RunAndCompare(kHlo, ErrorSpec{0.001}));
+}
+
 XLA_TEST_F(ConvolutionHloTest, DISABLED_ON_CPU(ConvolveF64BackwardFilter)) {
   constexpr char kHlo[] = R"(
 HloModule TestModule
diff --git a/tensorflow/compiler/xla/window_util.cc b/tensorflow/compiler/xla/window_util.cc
index 8ea8dbab25..f113a705b4 100644
--- a/tensorflow/compiler/xla/window_util.cc
+++ b/tensorflow/compiler/xla/window_util.cc
@@ -185,6 +185,17 @@ bool HasWindowReversal(const Window& window) {
   return false;
 }
 
+bool AllOrNoneReversed(const Window& window) {
+  if (window.dimensions().size() == 0) {
+    return true;
+  }
+  bool reversed = window.dimensions()[0].window_reversal();
+  return std::all_of(window.dimensions().begin(), window.dimensions().end(),
+                     [&](const WindowDimension& dim) {
+                       return dim.window_reversal() == reversed;
+                     });
+}
+
 bool HasDilation(const Window& window) {
   return HasBaseDilation(window) || HasWindowDilation(window);
 }
diff --git a/tensorflow/compiler/xla/window_util.h b/tensorflow/compiler/xla/window_util.h
index 1fb9e855fc..099d7ecdd5 100644
--- a/tensorflow/compiler/xla/window_util.h
+++ b/tensorflow/compiler/xla/window_util.h
@@ -56,6 +56,7 @@ bool HasWindowDilation(const Window& window);
 bool HasDilation(const Window& window);
 
 bool HasWindowReversal(const Window& window);
+bool AllOrNoneReversed(const Window& window);
 
 // Returns true if the given logical dimension is inactive in the sense that it
 // has window bound 1, no striding and no padding.
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 5de9a2185c..6f60dd5c29 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -686,10 +686,10 @@ class CudnnConvolutionDescriptor {
     CHECK_CUDNN_OK(cudnnSetConvolutionNdDescriptor(
         handle_.get(), convolution_descriptor.ndims(), padding.data(),
         strides.data(), dilations.data(),
-        // NOTE(keveman): cuDNN supports convolution and cross correlation.
-        // However, almost all the use cases do cross correlation, so just
-        // hard coding it here.
-        CUDNN_CROSS_CORRELATION, data_type));
+        convolution_descriptor.convolution_not_crosscorr()
+            ? CUDNN_CONVOLUTION
+            : CUDNN_CROSS_CORRELATION,
+        data_type));
 
     // NOTE(benbarsdell): This only applies if tensor op math is enabled
     //                      and algo selection is set to Default.
diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc
index 8a5bcf4280..8c304db3cd 100644
--- a/tensorflow/stream_executor/dnn.cc
+++ b/tensorflow/stream_executor/dnn.cc
@@ -443,7 +443,8 @@ ConvolutionDescriptor::ConvolutionDescriptor(int ndims)
       dilation_rates_(ndims, 1),
       pad_alignment_(PadAlignment::kDefault),
       group_count_(1),
-      ndims_(ndims) {}
+      ndims_(ndims),
+      convolution_not_crosscorr_(false) {}
 
 ConvolutionDescriptor::ConvolutionDescriptor()
     : ConvolutionDescriptor(/*ndims=*/2) {}
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 621b155240..a352727127 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -496,6 +496,11 @@ std::ostream& operator<<(std::ostream& str, dnn::PadAlignment alignment);
 //   cells between each filter element in the "y dimension".
 // - horizontal_dilation_rate: there will be (horizontal_dilation_rate - 1)
 //   skipped cells between each filter element in the "x dimension".
+// - convolution_not_crosscor: By default (convolution_not_crosscor == false),
+//   we perform cross correlation rather than convolution. With the flag set,
+//   we perform convolution. Convolution and cross correlation are related by
+//   rotating the filter by 180 degrees (or equivalently flipping all spatial
+//   dimensions).
 class ConvolutionDescriptor {
  public:
   // By default construction, there is no zero-padding and the filter stride is
@@ -552,6 +557,10 @@ class ConvolutionDescriptor {
     group_count_ = group_count;
     return *this;
   }
+  ConvolutionDescriptor& set_convolution_not_crosscorr(bool conv) {
+    convolution_not_crosscorr_ = conv;
+    return *this;
+  }
   int64 zero_padding_height() const {
     return GetDim(zero_padding_, DimIndex::Y);
   }
@@ -577,6 +586,7 @@ class ConvolutionDescriptor {
   PadAlignment pad_alignment() const { return pad_alignment_; }
   int group_count() const { return group_count_; }
   int ndims() const { return ndims_; }
+  bool convolution_not_crosscorr() const { return convolution_not_crosscorr_; }
 
   std::vector<int64> strides() const { return filter_strides_; }
   std::vector<int64> dilations() const { return dilation_rates_; }
@@ -590,6 +600,7 @@ class ConvolutionDescriptor {
   PadAlignment pad_alignment_;
   int group_count_;
   int ndims_;
+  bool convolution_not_crosscorr_;
   // TODO(leary) cudnn provides these fields, but need to characterize what
   // their effect is -- they may be boolean rather than integral.
   // int64 upscale_input_x;
-- 
GitLab


From 684ce69f568e89e34883d91b65c3983c333e53d9 Mon Sep 17 00:00:00 2001
From: Karl Lessard <karl@kubx.ca>
Date: Thu, 25 Oct 2018 22:18:22 -0400
Subject: [PATCH 0053/1554] Prevent memory leak by storing strings instead of
 StringPiece in vector

---
 tensorflow/core/common_runtime/eager/attr_builder.cc |  2 +-
 tensorflow/core/common_runtime/eager/attr_builder.h  | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/attr_builder.cc b/tensorflow/core/common_runtime/eager/attr_builder.cc
index 5c8369de87..e9f2188656 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder.cc
@@ -99,7 +99,7 @@ Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out) {
 #define DEFINE_SET_ATTR(value_type, value_field)                             \
   template <>                                                                \
   AttrBuilder& AttrBuilder::Set(StringPiece attr_name, value_type&& value) { \
-    value_field.push_back(std::make_pair(attr_name, value));                 \
+    value_field.push_back(std::make_pair(string(attr_name), value));         \
     return *this;                                                            \
   }
 
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.h b/tensorflow/core/common_runtime/eager/attr_builder.h
index c114ea4ba0..a48ba686d2 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.h
+++ b/tensorflow/core/common_runtime/eager/attr_builder.h
@@ -96,7 +96,7 @@ class AttrBuilder {
   template <class T>
   AttrBuilder& Set(StringPiece attr_name, T&& value) {
     MayBeInitializeNodeDef();
-    SetInAttrValueMap(node_def_->mutable_attr(), attr_name, value);
+    SetInAttrValueMap(node_def_->mutable_attr(), string(attr_name), value);
     return *this;
   }
 
@@ -107,7 +107,7 @@ class AttrBuilder {
 
  private:
   template <class T>
-  using AttrVec = tensorflow::gtl::InlinedVector<std::pair<StringPiece, T>, 2>;
+  using AttrVec = tensorflow::gtl::InlinedVector<std::pair<string, T>, 2>;
 
   void MayBeInitializeNodeDef();
   // Fill `m` with the attr-value pairs set via AttrBuilder::Set() so far, as
@@ -119,7 +119,7 @@ class AttrBuilder {
   void FillAttrValueMap(AttrValueMap* m, bool include_those_in_node_def) const;
 
   template <class T>
-  void SetInAttrValueMap(AttrValueMap* m, StringPiece attr_name,
+  void SetInAttrValueMap(AttrValueMap* m, const string& attr_name,
                          T&& value) const {
     DCHECK(!node_def_finalized_)
         << "Calling SetInAttrValueMap after BuildNodeDef.";
@@ -128,12 +128,12 @@ class AttrBuilder {
     AttrValue attr_value;
     if (found == nullptr) {
       SetAttrValue(value, &attr_value);
-      m->insert(AttrValueMap::value_type(string(attr_name), attr_value));
+      m->insert(AttrValueMap::value_type(attr_name, attr_value));
     } else {
       // TODO(ashankar): Do what is done in
       // NodeDefBuilder::CheckInconsistency(attr_name, *found, attr_value);
       SetAttrValue(std::forward<T>(value), &attr_value);
-      (*m)[string(attr_name)] = attr_value;
+      (*m)[attr_name] = attr_value;
     }
   }
 
-- 
GitLab


From 901912bdaa9668a4e4fa0e90b873b0f9d2d717f6 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 26 Oct 2018 12:45:57 +0800
Subject: [PATCH 0054/1554] Replace "=>" with "into" for readability.

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index e041ab14ca..374c74c903 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -515,7 +515,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     //
     // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D
     // (NHWC) + Transpose (NHWC->
-    // NCHW) " => "Conv2D (NCHW). Such patterns occur frequently in Keras.
+    // NCHW)" into "Conv2D (NCHW)". Such patterns occur frequently in Keras.
     // Note: we use the term "merge" is to combine (exactly) 2 nodes into one,
     // while "fusion" is
     // for 3+ nodes situation.
-- 
GitLab


From 906a3527b9f4be3446a1f9e49332555bb3dd93cc Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 26 Oct 2018 12:46:26 +0800
Subject: [PATCH 0055/1554] Fix a grammatically incorrect in comments.

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 374c74c903..be91763a99 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -516,7 +516,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D
     // (NHWC) + Transpose (NHWC->
     // NCHW)" into "Conv2D (NCHW)". Such patterns occur frequently in Keras.
-    // Note: we use the term "merge" is to combine (exactly) 2 nodes into one,
+    // Note: we use the term "merge" to combine (exactly) 2 nodes into one,
     // while "fusion" is
     // for 3+ nodes situation.
     //
-- 
GitLab


From b7ecda56d59248845103399a2eae73348995d63b Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 12:59:01 +0800
Subject: [PATCH 0056/1554] Merge the line with its previous line.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index be91763a99..b0ae480d3d 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -517,8 +517,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     // (NHWC) + Transpose (NHWC->
     // NCHW)" into "Conv2D (NCHW)". Such patterns occur frequently in Keras.
     // Note: we use the term "merge" to combine (exactly) 2 nodes into one,
-    // while "fusion" is
-    // for 3+ nodes situation.
+    // while "fusion" is for 3+ nodes situation.
     //
 
     // Transpose + Conv2d + Transpose:
-- 
GitLab


From 932a04eed44a7c02181245210c72decb8d12e963 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 13:00:30 +0800
Subject: [PATCH 0057/1554] 2+ should be 3+.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index b0ae480d3d..2891979be2 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -592,7 +592,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     std::function<Node*(const Node*)> get_node_to_be_merged;
   } MergeInfo;
 
-  // structure to specify information used in node fusion of 2+ operators
+  // structure to specify information used in node fusion of 3+ operators
   typedef struct {
     std::string pattern_name;  // name to describe this pattern, such as
                                // "Transpose_Mklop_Transpose".
-- 
GitLab


From f60f477407fb3e203146abdb6ec7fce6b9735d50 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 13:03:11 +0800
Subject: [PATCH 0058/1554] Capitalized the beginning of the sentences.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 2891979be2..77715b1515 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -592,12 +592,12 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     std::function<Node*(const Node*)> get_node_to_be_merged;
   } MergeInfo;
 
-  // structure to specify information used in node fusion of 3+ operators
+  // Structure to specify information used in node fusion of 3+ operators
   typedef struct {
-    std::string pattern_name;  // name to describe this pattern, such as
+    std::string pattern_name;  // Name to describe this pattern, such as
                                // "Transpose_Mklop_Transpose".
     std::vector<std::function<bool(const Node*)> >
-        node_checkers;  // extra restriction checker for these ops
+        node_checkers;  // Extra restriction checker for these ops
     std::function<Status(
         std::unique_ptr<Graph>*, std::vector<Node*>&,
         std::function<void(const Node*, NodeBuilder* nb, bool)>)>
@@ -606,7 +606,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   } FusionInfo;
 
   //
-  // dimension indices for 2D tensor.
+  // Dimension indices for 2D tensor.
   //
   struct NCHW {
     enum dim { N = 0, C = 1, H = 2, W = 3 };
-- 
GitLab


From a3e878dfb91d83d788d1901c863660f1fa51c951 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 13:05:07 +0800
Subject: [PATCH 0059/1554] Remove empty comment lines.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 77715b1515..c04887c36d 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -903,37 +903,28 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
       string data_format);
 
   static bool CheckForTranspose(const Node* node, std::vector<int> perm) {
-    //
     // Check node node, to see if it's "Transpose"
-    //
     if (node->type_string() != "Transpose") return false;
 
-    //
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
-    //
     for (const Edge* e : node->out_edges()) {
       if (e->IsControlEdge()) {
         return false;
       }
     }
 
-    //
     // If "Transpose" has input control edges, don't fuse on it.
-    //
     for (const Edge* e : node->in_edges()) {
       if (e->IsControlEdge()) {
         return false;
       }
     }
 
-    //
     // If "Transpose" has multiple output data edges, also don't fuse it.
-    //
     if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
 
     // Check "perm" attribute, make sure it's what we want.
-    //
     for (const Edge* e : node->in_edges()) {
       if (!e->IsControlEdge()) {
         const Node* perm_node = e->src();
@@ -948,11 +939,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
           DataType type;
           GetNodeAttr(perm_node->def(), "dtype", &type);
 
-          //
           // Here we directly access to the "tensor_context", rather than
           // "int_val". This is because we find "int_val" is
           // not set properly under some circumstances.
-          //
           if (type == DT_INT32) {
             const int type_size = 4;
             const int* tensor_content =
-- 
GitLab


From 299c1649d1d48bc8d1e6a7fad505c20f2baaae8b Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 26 Oct 2018 13:12:54 +0800
Subject: [PATCH 0060/1554] Fix a grammatically incorrect comment line.

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index c04887c36d..ce364c062d 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -903,7 +903,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
       string data_format);
 
   static bool CheckForTranspose(const Node* node, std::vector<int> perm) {
-    // Check node node, to see if it's "Transpose"
+    // Check if node's type is "Transpose"
     if (node->type_string() != "Transpose") return false;
 
     // Check if has out control edge. If true, this is a training graph.
-- 
GitLab


From 43ccbbf8b2d2ac4d46f76ee72d000120512358d3 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 26 Oct 2018 13:21:03 +0800
Subject: [PATCH 0061/1554] Use "CHECK_EQ" instead of "CHECK".

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index ce364c062d..78b1353997 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -934,7 +934,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
             e->dst_input() == kPermTensorIndex) {
           // we find the "perm" node, now try to retrieve its value.
           const TensorProto* proto = nullptr;
-          CHECK_EQ(GetNodeAttr(perm_node->def(), "value", &proto).ok(), true);
+          CHECK(GetNodeAttr(perm_node->def(), "value", &proto).ok());
 
           DataType type;
           GetNodeAttr(perm_node->def(), "dtype", &type);
-- 
GitLab


From cd31f0bcd6b52b8d9bd5d8430bddd77388a4fff0 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 13:27:47 +0800
Subject: [PATCH 0062/1554] Explain the high level idea of comparing 'perm' and
 'perm_node'.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 78b1353997..21ceb66fca 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -924,7 +924,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     // If "Transpose" has multiple output data edges, also don't fuse it.
     if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
 
-    // Check "perm" attribute, make sure it's what we want.
+    // We compared the tensor containing the permutation order ("perm_node")
+    // with our desired order ("perm"). If they're exactly match, this check
+    // succeed and returns true.
     for (const Edge* e : node->in_edges()) {
       if (!e->IsControlEdge()) {
         const Node* perm_node = e->src();
@@ -934,7 +936,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
             e->dst_input() == kPermTensorIndex) {
           // we find the "perm" node, now try to retrieve its value.
           const TensorProto* proto = nullptr;
-          CHECK(GetNodeAttr(perm_node->def(), "value", &proto).ok());
+          CHECK_EQ(GetNodeAttr(perm_node->def(), "value", &proto).ok(), true);
 
           DataType type;
           GetNodeAttr(perm_node->def(), "dtype", &type);
-- 
GitLab


From d667d30ff7132db51537c38984c97a589d9e7612 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 26 Oct 2018 13:46:25 +0800
Subject: [PATCH 0063/1554] Fix a typo.

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 21ceb66fca..bce9995371 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -941,7 +941,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
           DataType type;
           GetNodeAttr(perm_node->def(), "dtype", &type);
 
-          // Here we directly access to the "tensor_context", rather than
+          // Here we directly access to the "tensor_content", rather than
           // "int_val". This is because we find "int_val" is
           // not set properly under some circumstances.
           if (type == DT_INT32) {
-- 
GitLab


From a11786c809cd013fd4d4f97c94ce94a524b5bc17 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 13:52:05 +0800
Subject: [PATCH 0064/1554] Remove some empty lines.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index bce9995371..76ae4faa45 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -955,7 +955,6 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                                         tensor_content + tensor_content_size);
 
             return perm_value == perm;
-
           } else if (type == DT_INT64) {
             const int type_size = 8;
             const long* tensor_content =
@@ -969,12 +968,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
             return perm_value == long_perm;
           }
-
           return false;
         }
       }
     }
-
     return false;
   }
 
-- 
GitLab


From 4f0c4445c9e09b4360fbce8ce8f1b50722dced56 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 13:56:30 +0800
Subject: [PATCH 0065/1554] Check if node is nullptr.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 76ae4faa45..909f84123c 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -976,6 +976,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   }
 
   static bool CheckForMklOp(const Node* node, string name = "") {
+    if (node == nullptr) return false;
+    
     if (!name.empty() && node->type_string() != name) {
       return false;
     }
-- 
GitLab


From ffc909edb6b1c4376ed2870d6d65fd4b5f17460f Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 14:01:03 +0800
Subject: [PATCH 0066/1554] Add description for FuseTransposeMklOpTranspose().

---
 tensorflow/core/graph/mkl_layout_pass.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 909f84123c..78d6962cc3 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -897,6 +897,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   Status FuseNode(std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
                   const MklLayoutRewritePass::FusionInfo fi);
 
+  // Fuse tranpose(to "NHWC") + mklop("NHWC") + transpose(to "NCHW") into mklop("NCHW").
+  // Here "mklop" can be any MKL-DNN supported op, such as Conv2D.
   static Status FuseTransposeMklOpTranspose(
       std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
       std::function<void(const Node*, NodeBuilder* nb, bool)> copy_attrs,
@@ -977,7 +979,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
   static bool CheckForMklOp(const Node* node, string name = "") {
     if (node == nullptr) return false;
-    
+
     if (!name.empty() && node->type_string() != name) {
       return false;
     }
-- 
GitLab


From 738039381b33e1e39bf4089fa2c1341aeddb6308 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 14:03:00 +0800
Subject: [PATCH 0067/1554] Remove empty comment lines.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 78d6962cc3..4cfe6aff1a 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -1810,12 +1810,9 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
     std::vector<int32> new_strides;
     std::vector<int32> new_dilations;
     if (strides.size() == 5) {
-      //
       // "strides" and "dilations" also need to be changed according to
       // "data_format",
       // in this case, is "NDHWC" to "NCDHW".
-      //
-
       new_strides = {strides[NDHWC::dim::N], strides[NDHWC::dim::C],
                      strides[NDHWC::dim::D], strides[NDHWC::dim::H],
                      strides[NDHWC::dim::W]};
@@ -1827,11 +1824,9 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
       nb->Attr("dilations", new_dilations);
 
     } else {
-      //
       // "strides" and "dilations" also need to be changed according to
       // "data_format",
       // in this case, is "NHWC" to "NCHW".
-      //
 
       new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
                      strides[NHWC::dim::H], strides[NHWC::dim::W]};
-- 
GitLab


From c4ab34f66163bc8face7f588ebba94cadd306d6f Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 14:06:57 +0800
Subject: [PATCH 0068/1554] Move nb->Attr() out of if statement.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 4cfe6aff1a..850a6968a3 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -1821,8 +1821,6 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
       new_dilations = {dilations[NDHWC::dim::N], dilations[NDHWC::dim::C],
                        dilations[NDHWC::dim::D], dilations[NDHWC::dim::H],
                        dilations[NDHWC::dim::W]};
-      nb->Attr("dilations", new_dilations);
-
     } else {
       // "strides" and "dilations" also need to be changed according to
       // "data_format",
@@ -1834,8 +1832,8 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
 
       new_dilations = {dilations[NHWC::dim::N], dilations[NHWC::dim::C],
                        dilations[NHWC::dim::H], dilations[NHWC::dim::W]};
-      nb->Attr("dilations", new_dilations);
     }
+    nb->Attr("dilations", new_dilations);
   }
 }
 
-- 
GitLab


From aa14839dfd414adda0fb0260db10f24de18e1e1c Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Fri, 26 Oct 2018 14:17:04 +0800
Subject: [PATCH 0069/1554] Fix a variable name mis-spelling.

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 850a6968a3..8b29f29746 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2657,7 +2657,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   gtl::InlinedVector<Node*, 4> transpose_nchw_control_edges;
   gtl::InlinedVector<std::pair<Node*, int>, 4> transpose_nchw_in(
       transpose_nchw_num_inputs);
-  FillInputs(transpose_to_nhwc, &transpose_nchw_control_edges,
+  FillInputs(transpose_to_nchw, &transpose_nchw_control_edges,
              &transpose_nchw_in);
 
   // We will use the node name of Conv2d as the name of new node
-- 
GitLab


From 8afe630a977c94c331c31daaca1d52da6df11303 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 14:42:35 +0800
Subject: [PATCH 0070/1554] Fix a typo.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 8b29f29746..c6cb8552b7 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2657,7 +2657,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   gtl::InlinedVector<Node*, 4> transpose_nchw_control_edges;
   gtl::InlinedVector<std::pair<Node*, int>, 4> transpose_nchw_in(
       transpose_nchw_num_inputs);
-  FillInputs(transpose_to_nchw, &transpose_nchw_control_edges,
+  FillInputs(transpose_to_nhwc, &transpose_nchw_control_edges,
              &transpose_nchw_in);
 
   // We will use the node name of Conv2d as the name of new node
@@ -2689,8 +2689,8 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   // Fill outputs.
   for (const Edge* e : transpose_to_nchw->out_edges()) {
     if (!e->IsControlEdge()) {
-      const int kConv2DWithBiasOutputSlot = 0;
-      CHECK_NOTNULL((*g)->AddEdge(new_node, kConv2DWithBiasOutputSlot, e->dst(),
+      const int kTransposeWithMklOpOutputSlot = 0;
+      CHECK_NOTNULL((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot, e->dst(),
                                   e->dst_input()));
     }
   }
-- 
GitLab


From f05a5ac3a0a2418a7ab9e6b8a907b01b2335491a Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 14:58:53 +0800
Subject: [PATCH 0071/1554] Copy requested_device and
 assigned_device_name_index.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index c6cb8552b7..125f7a15fb 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2698,6 +2698,10 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   // Copy device assigned to old node to new node.
   new_node->set_assigned_device_name(mklop->assigned_device_name());
 
+  // Copy requested_device and assigned_device_name_index
+  new_node->set_requested_device(mklop->requested_device());
+  new_node->set_assigned_device_name_index(mklop->assigned_device_name_index());
+
   (*g)->RemoveNode(transpose_to_nhwc);
   (*g)->RemoveNode(mklop);
   (*g)->RemoveNode(transpose_to_nchw);
-- 
GitLab


From 976a706932cfee6bcaa37970b07c064308524a2f Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 15:02:27 +0800
Subject: [PATCH 0072/1554] Remove an assert line.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 125f7a15fb..cacf563fff 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2722,7 +2722,6 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
   const FusionInfo* fi_ptr = nullptr;
 
   for (auto fi = finfo_.begin(); fi != finfo_.end(); ++fi) {
-    assert(fi->ops.size() == fi->node_checkers.size());
     nodes.clear();
     fi_ptr = &*fi;
     //
-- 
GitLab


From 0309943937acb3f9b0b7df011f94c3d55cd5b66c Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 26 Oct 2018 15:15:08 +0800
Subject: [PATCH 0073/1554] Fix a typo.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index cacf563fff..a7f86e56da 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2657,7 +2657,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   gtl::InlinedVector<Node*, 4> transpose_nchw_control_edges;
   gtl::InlinedVector<std::pair<Node*, int>, 4> transpose_nchw_in(
       transpose_nchw_num_inputs);
-  FillInputs(transpose_to_nhwc, &transpose_nchw_control_edges,
+  FillInputs(transpose_to_nchw, &transpose_nchw_control_edges,
              &transpose_nchw_in);
 
   // We will use the node name of Conv2d as the name of new node
-- 
GitLab


From 1d94242feccdb2afd3583c0332de090eab2f3811 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Sat, 27 Oct 2018 00:04:52 +0800
Subject: [PATCH 0074/1554] Replace the greedy search algorithm in
 "CheckForNodeFusion()" with a stack-based one, to avoid missing some
 patterns.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 78 +++++++++++++-----------
 1 file changed, 43 insertions(+), 35 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index a7f86e56da..0ace4a1fd1 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -26,6 +26,8 @@ limitations under the License.
 #include <unordered_set>
 #include <utility>
 #include <vector>
+#include <stack>
+#include <set>
 
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
@@ -2717,12 +2719,9 @@ Status MklLayoutRewritePass::FuseNode(
 
 std::tuple<bool, std::vector<Node*>, const MklLayoutRewritePass::FusionInfo>
 MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
-  bool found_pattern = false;
-  std::vector<Node*> nodes;
   const FusionInfo* fi_ptr = nullptr;
 
   for (auto fi = finfo_.begin(); fi != finfo_.end(); ++fi) {
-    nodes.clear();
     fi_ptr = &*fi;
     //
     // Make sure node "a" and its succeding nodes (b, c ...), match the pattern
@@ -2730,51 +2729,60 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
     // aka. "a->b->c" matches "op1->op2->op3"
     //
 
-    // Initialize "current_node" as node "a".
-    Node* current_node = a;
-    for (auto node_index = 0; node_index < fi->node_checkers.size();
-         ++node_index) {
-      // Make sure current node meet the requirement of corresponding node
-      // checker.
-      auto check_node = fi->node_checkers[node_index];
-      if (current_node == nullptr ||
-          (check_node && check_node(current_node) == false)) {
-        found_pattern = false;
-        nodes.clear();
-        break;
-      }
+    std::stack<Node *, std::vector<Node *>> work_stack;
+    std::set<Node *> visited_nodes;
+    auto node_checker = fi->node_checkers.begin();
 
-      // Add current_node to "fusion_nodes":
-      nodes.push_back(current_node);
+    Node *current_node = nullptr;
+    if (a != nullptr) {
+      work_stack.push(a);
+    }
 
-      // If current node is not the last node we want to check, check next node.
-      if (node_index != fi->node_checkers.size() - 1) {
-        // Find current node's direct descendant, which will be used in next
-        // iteration.
-        auto check_next_node = fi->node_checkers[node_index + 1];
-        for (const Edge* e : current_node->out_edges()) {
+    while (!work_stack.empty()) {
+      current_node = work_stack.top();
+
+      if ((*node_checker)(current_node)){
+        if (node_checker == (fi->node_checkers.end() - 1)) {
+          // We find a match, break and return.
+          std::vector<Node *> nodes;
+          while (!work_stack.empty()) {
+            nodes.insert(nodes.begin(), work_stack.top());
+            work_stack.pop();
+          }
+          
+          return make_tuple(true, nodes, *fi_ptr);
+        }
+
+        bool all_succ_has_been_visited = true;
+        for (const Edge *e : current_node->out_edges()) {
           if (!e->IsControlEdge()) {
-            Node* candidate_node = e->dst();
+            Node *candidate_node = e->dst();
 
-            if (check_next_node(candidate_node) == false) {
-              current_node = nullptr;
-            } else {
-              current_node = candidate_node;
+            // If the candidate node has not been visited, push it to stack.
+            if (visited_nodes.find(candidate_node) == visited_nodes.end()) {
+              work_stack.push(candidate_node);
+              ++ node_checker;
+              all_succ_has_been_visited = false;
               break;
             }
+
+            // All successor nodes of current node has been visited (no match found),
+            // pop the stack and mark current node as "visited".
+            if (all_succ_has_been_visited) {
+              visited_nodes.insert(current_node);
+              work_stack.pop();
+              -- node_checker;
+            }
           }
         }
       } else {
-        found_pattern = true;
+        // current node doesn't match, just break and stack will help us roll back.
+        break;
       }
     }
-
-    if (found_pattern == true) {
-      break;
-    }
   }
 
-  return make_tuple(found_pattern, nodes, *fi_ptr);
+  return make_tuple(false, std::vector<Node *>(), *fi_ptr);
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-- 
GitLab


From 095ea5e8cf1b6df75c63dc38f9a8b0338392805f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Huan=20LI=20=28=E6=9D=8E=E5=8D=93=E6=A1=93=29?=
 <zixia@zixia.net>
Date: Tue, 30 Oct 2018 01:52:41 +0800
Subject: [PATCH 0075/1554] fix missing `#` in code example

---
 tensorflow/python/keras/layers/embeddings.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/layers/embeddings.py b/tensorflow/python/keras/layers/embeddings.py
index 824a0b069e..d2ba8933ba 100644
--- a/tensorflow/python/keras/layers/embeddings.py
+++ b/tensorflow/python/keras/layers/embeddings.py
@@ -45,11 +45,11 @@ class Embedding(Layer):
     model = Sequential()
     model.add(Embedding(1000, 64, input_length=10))
     # the model will take as input an integer matrix of size (batch,
-    input_length).
+    # input_length).
     # the largest integer (i.e. word index) in the input should be no larger
-    than 999 (vocabulary size).
+    # than 999 (vocabulary size).
     # now model.output_shape == (None, 10, 64), where None is the batch
-    dimension.
+    # dimension.
 
     input_array = np.random.randint(1000, size=(32, 10))
 
-- 
GitLab


From 1dd6c365ef424a4757da4bf562294fb2bc51dbca Mon Sep 17 00:00:00 2001
From: BY Shen <byshen@gmail.com>
Date: Tue, 30 Oct 2018 12:40:36 +0800
Subject: [PATCH 0076/1554] Update .gitignore for tflite downloads directory.

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 1ef4c297ee..8d7808ba01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,7 +24,7 @@ Pods
 Podfile.lock
 *.pbxproj
 *.xcworkspacedata
-/tensorflow/contrib/lite/downloads/**
+/tensorflow/contrib/lite/tools/make/downloads/**
 /tensorflow/contrib/lite/gen/**
 /tensorflow/contrib/lite/examples/ios/simple/data/*.txt
 /tensorflow/contrib/lite/examples/ios/simple/data/*.tflite
-- 
GitLab


From 59617ccaca8c5980f5418a0b612b040ac8d1afba Mon Sep 17 00:00:00 2001
From: Ouwen Huang <ouwen.oh@gmail.com>
Date: Tue, 30 Oct 2018 05:37:22 +0000
Subject: [PATCH 0077/1554] Added note on weight decay for tf.contrib.opt
 optimizers.

---
 .../python/training/weight_decay_optimizers.py  | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py b/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py
index 200b0d2008..1e8351b70f 100644
--- a/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py
+++ b/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py
@@ -59,6 +59,23 @@ class DecoupledWeightDecayExtension(object):
   Note that this extension decays weights BEFORE applying the update based
   on the gradient, i.e. this extension only has the desired behaviour for
   optimizers which do not depend on the value of'var' in the update step!
+  
+  Note: when applying a decay to the learning rate, be sure to manually apply
+  the decay to the `weight_decay` as well. For example:
+
+  ```python
+    decay = tf.train.piecewise_constant(tf.train.get_global_step(), 
+                                        [10000, 15000], [1e-1, 1e-2, 1e-3])
+    lr = 1*decay
+    wd = 1e-4*decay
+
+    # ...
+
+    optimizer = tf.contrib.opt.MomentumWOptimizer(learning_rate=lr,
+                                                  weight_decay=wd,
+                                                  momentum=0.9,
+                                                  use_nesterov=True)
+  ```
   """
 
   def __init__(self, weight_decay, **kwargs):
-- 
GitLab


From 6df3eddd533c464dcb300e1896bee2bdf4146dcf Mon Sep 17 00:00:00 2001
From: Pedro Monreal <pmgdeb@gmail.com>
Date: Tue, 30 Oct 2018 18:32:02 +0100
Subject: [PATCH 0078/1554] Fixed some spellings in core.

---
 .../common_runtime/collective_param_resolver_local.cc     | 2 +-
 .../core/grappler/costs/op_level_cost_estimator_test.cc   | 2 +-
 tensorflow/core/grappler/grappler_item_builder.cc         | 2 +-
 tensorflow/core/kernels/adjust_contrast_op.cc             | 4 ++--
 tensorflow/core/kernels/deep_conv2d.cc                    | 2 +-
 tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc       | 8 ++++----
 tensorflow/core/kernels/dynamic_partition_op.cc           | 2 +-
 tensorflow/core/kernels/fractional_avg_pool_op.cc         | 2 +-
 tensorflow/core/kernels/fused_batch_norm_op.cc            | 4 ++--
 tensorflow/core/kernels/mkl_softmax_op.cc                 | 4 ++--
 .../core/kernels/quantized_resize_bilinear_op_test.cc     | 2 +-
 tensorflow/core/kernels/stage_op.cc                       | 2 +-
 tensorflow/core/platform/cpu_feature_guard.cc             | 2 +-
 tensorflow/core/platform/numa_test.cc                     | 2 +-
 tensorflow/core/profiler/internal/tfprof_code.cc          | 2 +-
 tensorflow/core/profiler/internal/tfprof_node.cc          | 2 +-
 tensorflow/core/protobuf/master.proto                     | 2 +-
 tensorflow/core/util/tensor_bundle/tensor_bundle.cc       | 4 ++--
 18 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index 1bc873d0c5..44bf99764b 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -660,7 +660,7 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
-                                        ", recevied second claim from ",
+                                        ", received second claim from ",
                                         cp->default_rank);
         } else {
           ir->source_rank = cp->default_rank;
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 998bd59dce..c9ce63a8ef 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -832,7 +832,7 @@ TEST_F(OpLevelCostEstimatorTest, GetTensorShapeProtoFromTensorProto) {
   EXPECT_FALSE(
       GetTensorShapeProtoFromTensorProto(tensor_proto, &tensor_shape_proto));
 
-  // Check GetTensorShapeProtoFromTensorProto() resturns correct values.
+  // Check GetTensorShapeProtoFromTensorProto() returns correct values.
   {
     std::vector<int64> shape_expected = {10, 20, 30, 40};
     GetTensorProto(DT_INT32, {4}, shape_expected,
diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index cf99f4908b..984760ab99 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -519,7 +519,7 @@ std::unique_ptr<GrapplerItem> GrapplerItemFromMetaGraphDef(
         }
         if (!iter->second.has_tensor() ||
             iter->second.tensor().string_val_size() != 1) {
-          LOG(INFO) << "Unexected AttrValue proto: "
+          LOG(INFO) << "Unexpected AttrValue proto: "
                     << iter->second.DebugString();
           return nullptr;
         }
diff --git a/tensorflow/core/kernels/adjust_contrast_op.cc b/tensorflow/core/kernels/adjust_contrast_op.cc
index 72155fd037..5e1ca166db 100644
--- a/tensorflow/core/kernels/adjust_contrast_op.cc
+++ b/tensorflow/core/kernels/adjust_contrast_op.cc
@@ -320,13 +320,13 @@ class AdjustContrastOpv2<CPUDevice> : public AdjustContrastOpV2Base {
     int64 batch = outputs.dimension(0);
     int64 image_size = outputs.dimension(1);
     int64 channels = outputs.dimension(2);
-    // Similar to the reduction case, a straighforward implementation of this
+    // Similar to the reduction case, a straightforward implementation of this
     // does not utilize vectorization well because of the small channel size.
     // This algorithm repeatedly increases the area to be copied, and leads to
     // much better vectorinizations in the copy.
     for (int64 i = 0; i < batch; i++) {
       // Copy over the inputs into outputs in this batch. Effectively:
-      // outputs(i, :, k) = inputs(i, k). An example of how this algorith works:
+      // outputs(i, :, k) = inputs(i, k). An example of how this algorithm works:
       //
       //    x = float[1, 3], y = float[2048, 3]
       //    round 0
diff --git a/tensorflow/core/kernels/deep_conv2d.cc b/tensorflow/core/kernels/deep_conv2d.cc
index 1aa8c72d66..e3dabd687c 100644
--- a/tensorflow/core/kernels/deep_conv2d.cc
+++ b/tensorflow/core/kernels/deep_conv2d.cc
@@ -787,7 +787,7 @@ struct TransformOutputTile {
             const int64 shard_base = sr * filter_shards_col + sc;
             const int64 out_buf_base = tile_base + out_depth_base + shard_base;
 
-            // Calcuate output indices and outputs to drop (if needed).
+            // Calculate output indices and outputs to drop (if needed).
             const int64 out_r_start =
                 in_r + args.pad_rows - sr * tile_stride_rows;
             // NOTE: The index 't' for 'num_tiles is used in index calculation
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
index 76afd6f18c..f429b368ab 100644
--- a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
@@ -175,7 +175,7 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNHWCSmall(
   assert(CanLaunchDepthwiseConv2dGPUSmall(args));
   // Holds block plus halo and filter data for blockDim.x depths.
   extern __shared__ __align__(8) unsigned char shared_memory[];
-  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  static_assert(sizeof(S) <= 8, "Insufficient alignment detected");
   S* const shared_data = reinterpret_cast<S*>(shared_memory);
 
   const int num_batches = args.batch;
@@ -459,7 +459,7 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNCHWSmall(
   assert(CanLaunchDepthwiseConv2dGPUSmall(args));
   // Holds block plus halo and filter data for blockDim.z depths.
   extern __shared__ __align__(8) unsigned char shared_memory[];
-  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  static_assert(sizeof(S) <= 8, "Insufficient alignment detected");
   S* const shared_data = reinterpret_cast<S*>(shared_memory);
 
   const int num_batches = args.batch;
@@ -1176,7 +1176,7 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall(
   assert(CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(args, blockDim.z));
   // Holds block plus halo and filter data for blockDim.x depths.
   extern __shared__ __align__(8) unsigned char shared_memory[];
-  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  static_assert(sizeof(S) <= 8, "Insufficient alignment detected");
   S* const shared_data = reinterpret_cast<S*>(shared_memory);
 
   const int num_batches = args.batch;
@@ -1448,7 +1448,7 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
   assert(CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(args, blockDim.x));
   // Holds block plus halo and filter data for blockDim.z depths.
   extern __shared__ __align__(8) unsigned char shared_memory[];
-  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  static_assert(sizeof(S) <= 8, "Insufficient alignment detected");
   S* const shared_data = reinterpret_cast<S*>(shared_memory);
 
   const int num_batches = args.batch;
diff --git a/tensorflow/core/kernels/dynamic_partition_op.cc b/tensorflow/core/kernels/dynamic_partition_op.cc
index 3c988db5e6..572d04ae2c 100644
--- a/tensorflow/core/kernels/dynamic_partition_op.cc
+++ b/tensorflow/core/kernels/dynamic_partition_op.cc
@@ -142,7 +142,7 @@ class DynamicPartitionOp : public DynamicPartitionOp_Shared {
         OP_REQUIRES(
             c, FastBoundsCheck(p, num_partitions_),
             errors::InvalidArgument("indices[", i,
-                                    "] has been asynchronously overwitten and "
+                                    "] has been asynchronously overwritten and "
                                     "is no longer in range!"));
         auto oi = output_index[p];
         OP_REQUIRES(c, FastBoundsCheck(oi, out_flat[p].dimension(0)),
diff --git a/tensorflow/core/kernels/fractional_avg_pool_op.cc b/tensorflow/core/kernels/fractional_avg_pool_op.cc
index 135d002345..61234479ea 100644
--- a/tensorflow/core/kernels/fractional_avg_pool_op.cc
+++ b/tensorflow/core/kernels/fractional_avg_pool_op.cc
@@ -223,7 +223,7 @@ class FractionalAvgPoolGradOp : public OpKernel {
     // Once we figure out the original contributors, we just need to evenly
     // divide the value of this element among these contributors.
     //
-    // Internally, we divide the out_backprop tensor and store it in a temparary
+    // Internally, we divide the out_backprop tensor and store it in a temporary
     // tensor of double type. And cast it to the corresponding type.
     typedef Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>>
         ConstEigenMatrixMap;
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index d89f1592bd..dbd3bb05db 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -248,7 +248,7 @@ struct FusedBatchNorm<GPUDevice, T, U> {
                   Tensor* saved_inv_var, TensorFormat tensor_format,
                   bool is_training) {
     auto* stream = context->op_device_context()->stream();
-    OP_REQUIRES(context, stream, errors::Internal("No GPU stream avalible"));
+    OP_REQUIRES(context, stream, errors::Internal("No GPU stream available"));
 
     const int64 batch_size = GetTensorDim(x, tensor_format, 'N');
     const int64 channels = GetTensorDim(x, tensor_format, 'C');
@@ -389,7 +389,7 @@ struct FusedBatchNormGrad<GPUDevice, T, U> {
                   Tensor* scale_backprop, Tensor* offset_backprop,
                   TensorFormat tensor_format) {
     auto* stream = context->op_device_context()->stream();
-    OP_REQUIRES(context, stream, errors::Internal("No GPU stream avalible"));
+    OP_REQUIRES(context, stream, errors::Internal("No GPU stream available"));
 
     const int64 batch_size = GetTensorDim(x, tensor_format, 'N');
     const int64 channels = GetTensorDim(x, tensor_format, 'C');
diff --git a/tensorflow/core/kernels/mkl_softmax_op.cc b/tensorflow/core/kernels/mkl_softmax_op.cc
index cfab529662..3bf17bc449 100644
--- a/tensorflow/core/kernels/mkl_softmax_op.cc
+++ b/tensorflow/core/kernels/mkl_softmax_op.cc
@@ -56,7 +56,7 @@ class MklSoftmaxOp : public OpKernel {
       MklDnnShape src_mkl_shape;
       GetMklShape(context, src_idx, &src_mkl_shape);
 
-      // src_dims is the dimenstion of src_tensor
+      // src_dims is the dimension of src_tensor
       // dim of the dst will also be same as src_dims
       auto src_tf_shape = src_mkl_shape.IsMklTensor()
                               ? src_mkl_shape.GetTfShape()
@@ -68,7 +68,7 @@ class MklSoftmaxOp : public OpKernel {
       // Here "x" data format in MKL is used for 1 dim tensor, "nc" for 2 dim tensor, 
       // "tnc" for 3 dim tensor, "nchw" for 4 dim tensor, and "ncdhw" for 5 dim tensor.
       // Each of the simbols has the following meaning:
-      // n = batch, c = channels, t = sequence lenght, h = height,
+      // n = batch, c = channels, t = sequence length, h = height,
       // w = width, d = depth 
       switch (input_dims) {
         case 1:
diff --git a/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc b/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc
index e6133415d0..6fc4894592 100644
--- a/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc
+++ b/tensorflow/core/kernels/quantized_resize_bilinear_op_test.cc
@@ -273,7 +273,7 @@ void TestResizeBilinearOneDim() {
         << expected_val << ", " << resized_image_val;
   }
 
-  // Value testing with reference implemenatation
+  // Value testing with reference implementation
   CheckTensorValue<qint32>(image_quantized_tensor.flat<qint32>().data(),
                            outputs.at(0).flat<qint32>().data(),
                            /*batch_size=*/1,
diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc
index 73a02a34cf..c91bdc43cf 100644
--- a/tensorflow/core/kernels/stage_op.cc
+++ b/tensorflow/core/kernels/stage_op.cc
@@ -151,7 +151,7 @@ class Buffer : public ResourceBase {
   }
 
   // Are there a limit number of elements or a memory limit
-  // configued on this buffer?
+  // configured on this buffer?
   bool IsBounded() const { return capacity_ > 0 || memory_limit_ > 0; }
 
   bool IsCapacityFull() const { return buf_.size() >= capacity_; }
diff --git a/tensorflow/core/platform/cpu_feature_guard.cc b/tensorflow/core/platform/cpu_feature_guard.cc
index 9d00aa7b7f..2efe0c0876 100644
--- a/tensorflow/core/platform/cpu_feature_guard.cc
+++ b/tensorflow/core/platform/cpu_feature_guard.cc
@@ -41,7 +41,7 @@ void CheckFeatureOrDie(CPUFeature feature, const string& feature_name) {
   }
 }
 
-// Check if CPU feature is inclued in the TensorFlow binary.
+// Check if CPU feature is included in the TensorFlow binary.
 void CheckIfFeatureUnused(CPUFeature feature, const string& feature_name,
                           string& missing_instructions) {
   if (TestCPUFeature(feature)) {
diff --git a/tensorflow/core/platform/numa_test.cc b/tensorflow/core/platform/numa_test.cc
index 8b39ecd59c..91789efd1e 100644
--- a/tensorflow/core/platform/numa_test.cc
+++ b/tensorflow/core/platform/numa_test.cc
@@ -44,7 +44,7 @@ TEST(Numa, Malloc) {
 
 TEST(Numa, SetNodeAffinity) {
   // NOTE(tucker): This test is not reliable when executed under tap because
-  // the virtual machine may not have access to all of the availble NUMA
+  // the virtual machine may not have access to all of the available NUMA
   // nodes.  Not sure what to do about that.
   EXPECT_EQ(-1, port::NUMAGetThreadNodeAffinity());
   if (port::NUMAEnabled()) {
diff --git a/tensorflow/core/profiler/internal/tfprof_code.cc b/tensorflow/core/profiler/internal/tfprof_code.cc
index 744e1e95de..0c26855a43 100644
--- a/tensorflow/core/profiler/internal/tfprof_code.cc
+++ b/tensorflow/core/profiler/internal/tfprof_code.cc
@@ -183,7 +183,7 @@ class Samples {
   // This method adds the statistics of graph nodes created by the python
   // call.
   void Add(const CodeNode* node, const std::vector<uint64>& location_ids) {
-    // displayed leaf might not be true leaf. Retrive the true leaves for
+    // displayed leaf might not be true leaf. Retrieve the true leaves for
     // stats.
     std::vector<const CodeNode*> all_leaf = FetchAllLeaf(node);
     CHECK(!all_leaf.empty()) << node->name();
diff --git a/tensorflow/core/profiler/internal/tfprof_node.cc b/tensorflow/core/profiler/internal/tfprof_node.cc
index 86cb20de7b..8796234be0 100644
--- a/tensorflow/core/profiler/internal/tfprof_node.cc
+++ b/tensorflow/core/profiler/internal/tfprof_node.cc
@@ -151,7 +151,7 @@ void ExecStep::AddMemoryStats(const string& dev,
   }
 
   // TODO(xpan): Make this more accurate:
-  // High level: Memory tracking is suspicous and requires large scale
+  // High level: Memory tracking is suspicious and requires large scale
   // clean up.
   // Investigte the memory usage difference between CPU/GPU with OpViewTest.
   //
diff --git a/tensorflow/core/protobuf/master.proto b/tensorflow/core/protobuf/master.proto
index 03022875e6..c104463c51 100644
--- a/tensorflow/core/protobuf/master.proto
+++ b/tensorflow/core/protobuf/master.proto
@@ -224,7 +224,7 @@ message CloseSessionResponse {
 message ResetRequest {
   // A list of container names, which may be empty.
   //
-  // If 'container' is not empty, releases resoures in the given
+  // If 'container' is not empty, releases resources in the given
   // containers in all devices.
   //
   // If 'container' is empty, releases resources in the default
diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc
index 2dcb57a1f9..3709ee5ae3 100644
--- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc
+++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc
@@ -785,7 +785,7 @@ Status BundleReader::GetBundleEntryProto(StringPiece key,
   TF_RETURN_IF_ERROR(
       ParseEntryProto(iter_->key(), iter_->value(), &entry_copy));
   if (!TensorShape::IsValid(entry_copy.shape())) {
-    return errors::DataLoss("Invaid tensor shape: ", key, " ",
+    return errors::DataLoss("Invalid tensor shape: ", key, " ",
                             ProtoShortDebugString(entry_copy.shape()));
   }
 
@@ -895,7 +895,7 @@ Status BundleReader::ReadCurrent(Tensor* val) {
   BundleEntryProto entry;
   TF_RETURN_IF_ERROR(ParseEntryProto(iter_->key(), iter_->value(), &entry));
   if (!TensorShape::IsValid(entry.shape())) {
-    return errors::DataLoss("Invaid tensor shape: ", iter_->key(), " ",
+    return errors::DataLoss("Invalid tensor shape: ", iter_->key(), " ",
                             ProtoShortDebugString(entry.shape()));
   }
 
-- 
GitLab


From da4235299bd7e0089108634074b659d353102969 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 10:54:36 +0800
Subject: [PATCH 0079/1554] Add a comment to the constraint of "inference-only"
 to note it will eventually be removed, if we enabled this fusion for training
 in the future.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 0ace4a1fd1..30874dcf9e 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -912,6 +912,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
+    // Note: this constraint will eventually be removed, if we enabled this fusion for training
+    // in the future. 
     for (const Edge* e : node->out_edges()) {
       if (e->IsControlEdge()) {
         return false;
-- 
GitLab


From aab29e70ec2de097a04fb36aa2a60e2d286be1de Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 11:26:48 +0800
Subject: [PATCH 0080/1554] Move a cheaper early return to the top of the
 function.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 30874dcf9e..31d11d4aaf 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -910,6 +910,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     // Check if node's type is "Transpose"
     if (node->type_string() != "Transpose") return false;
 
+    // If "Transpose" has multiple output data edges, also don't fuse it.
+    if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
+
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
     // Note: this constraint will eventually be removed, if we enabled this fusion for training
@@ -927,9 +930,6 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
       }
     }
 
-    // If "Transpose" has multiple output data edges, also don't fuse it.
-    if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
-
     // We compared the tensor containing the permutation order ("perm_node")
     // with our desired order ("perm"). If they're exactly match, this check
     // succeed and returns true.
-- 
GitLab


From 6e3d7a22a7fa5622f32cee16a9162feab8f5a376 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 11:31:43 +0800
Subject: [PATCH 0081/1554] Set strides outside of if-else.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 31d11d4aaf..0df78ddc6f 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -1820,7 +1820,6 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
       new_strides = {strides[NDHWC::dim::N], strides[NDHWC::dim::C],
                      strides[NDHWC::dim::D], strides[NDHWC::dim::H],
                      strides[NDHWC::dim::W]};
-      nb->Attr("strides", new_strides);
 
       new_dilations = {dilations[NDHWC::dim::N], dilations[NDHWC::dim::C],
                        dilations[NDHWC::dim::D], dilations[NDHWC::dim::H],
@@ -1832,11 +1831,12 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
 
       new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
                      strides[NHWC::dim::H], strides[NHWC::dim::W]};
-      nb->Attr("strides", new_strides);
+      
 
       new_dilations = {dilations[NHWC::dim::N], dilations[NHWC::dim::C],
                        dilations[NHWC::dim::H], dilations[NHWC::dim::W]};
     }
+    nb->Attr("strides", new_strides);
     nb->Attr("dilations", new_dilations);
   }
 }
-- 
GitLab


From 753c474ba8fbbb11d213fe650240b5f30d074058 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 11:38:29 +0800
Subject: [PATCH 0082/1554] Fix a comment line.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 0df78ddc6f..82e714fdf9 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2664,8 +2664,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   FillInputs(transpose_to_nchw, &transpose_nchw_control_edges,
              &transpose_nchw_in);
 
-  // We will use the node name of Conv2d as the name of new node
-  // Build new node. We use same name as original node, but change the op
+  // We use same name as original node, but change the op
   // name.
   NodeBuilder nb(mklop->name(), mklop->type_string());
 
-- 
GitLab


From 2711e155b9d4ce280efe52d00d65a2b52c5be473 Mon Sep 17 00:00:00 2001
From: zldrobit <zldrobit@126.com>
Date: Wed, 31 Oct 2018 12:20:35 +0800
Subject: [PATCH 0083/1554] add dynamic shape support to dense_image_warp

---
 .../image/python/ops/dense_image_warp.py      | 74 +++++++++++--------
 1 file changed, 43 insertions(+), 31 deletions(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index 9c7ada7afb..2ac7ff2a6b 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -25,7 +25,7 @@ from tensorflow.python.framework import ops
 
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-
+from tensorflow.python.ops import check_ops
 
 def _interpolate_bilinear(grid,
                           query_points,
@@ -60,28 +60,34 @@ def _interpolate_bilinear(grid,
       msg = 'Grid must be 4 dimensional. Received size: '
       raise ValueError(msg + str(grid.get_shape()))
 
-    batch_size, height, width, channels = shape
+    batch_size, height, width, channels = \
+      array_ops.shape(grid)[0], \
+      array_ops.shape(grid)[1], \
+      array_ops.shape(grid)[2], \
+      array_ops.shape(grid)[3]
+    shape = [batch_size, height, width, channels]
     query_type = query_points.dtype
     grid_type = grid.dtype
 
-    if (query_points.shape.rank != 3 or
-        query_points.shape.dims[2].value != 2):
-      msg = ('Query points must be 3 dimensional and size 2 in dim 2. Received '
-             'size: ')
-      raise ValueError(msg + str(query_points.get_shape()))
-
-    _, num_queries, _ = query_points.get_shape().as_list()
-
-    if height < 2 or width < 2:
-      msg = 'Grid must be at least batch_size x 2 x 2 in size. Received size: '
-      raise ValueError(msg + str(grid.get_shape()))
-
-    alphas = []
-    floors = []
-    ceils = []
-
-    index_order = [0, 1] if indexing == 'ij' else [1, 0]
-    unstacked_query_points = array_ops.unstack(query_points, axis=2)
+    with ops.control_dependencies([
+        check_ops.assert_equal(len(query_points.get_shape()), 3, message=
+                               'Query points must be 3 dimensional'),
+        check_ops.assert_equal(array_ops.shape(query_points)[2], 2, message=
+                               'Query points must be size 2 in dim 2.')]):
+      num_queries = array_ops.shape(query_points)[1]
+
+    with ops.control_dependencies([
+        check_ops.assert_greater_equal(height, 2, message=
+                                       'Grid must be at least batch_size'
+                                       'x 2 x 2 in size.'),
+        check_ops.assert_greater_equal(width, 2, message=
+                                       'Grid must be at least batch_size'
+                                       'x 2 x 2 in size.')]):
+      alphas = []
+      floors = []
+      ceils = []
+      index_order = [0, 1] if indexing == 'ij' else [1, 0]
+      unstacked_query_points = array_ops.unstack(query_points, axis=2)
 
     for dim in index_order:
       with ops.name_scope('dim-' + str(dim)):
@@ -112,16 +118,17 @@ def _interpolate_bilinear(grid,
         alpha = array_ops.expand_dims(alpha, 2)
         alphas.append(alpha)
 
-    if batch_size * height * width > np.iinfo(np.int32).max / 8:
-      error_msg = """The image size or batch size is sufficiently large
-                     that the linearized addresses used by array_ops.gather
-                     may exceed the int32 limit."""
-      raise ValueError(error_msg)
-
-    flattened_grid = array_ops.reshape(grid,
-                                       [batch_size * height * width, channels])
-    batch_offsets = array_ops.reshape(
-        math_ops.range(batch_size) * height * width, [batch_size, 1])
+    with ops.control_dependencies([
+        check_ops.assert_less_equal(
+            math_ops.cast(batch_size * height * width, dtype=dtypes.float32),
+            np.iinfo(np.int32).max / 8,
+            message="""The image size or batch size is sufficiently large
+                       that the linearized addresses used by array_ops.gather
+                       may exceed the int32 limit.""")]):
+      flattened_grid = array_ops.reshape(
+          grid, [batch_size * height * width, channels])
+      batch_offsets = array_ops.reshape(
+          math_ops.range(batch_size) * height * width, [batch_size, 1])
 
     # This wraps array_ops.gather. We reshape the image data such that the
     # batch, y, and x coordinates are pulled into the first dimension.
@@ -182,7 +189,12 @@ def dense_image_warp(image, flow, name='dense_image_warp'):
                 of dimensions.
   """
   with ops.name_scope(name):
-    batch_size, height, width, channels = image.get_shape().as_list()
+    batch_size, height, width, channels = \
+      array_ops.shape(image)[0], \
+      array_ops.shape(image)[1], \
+      array_ops.shape(image)[2], \
+      array_ops.shape(image)[3]
+
     # The flow is defined on the image grid. Turn the flow into a list of query
     # points in the grid space.
     grid_x, grid_y = array_ops.meshgrid(
-- 
GitLab


From fa54ac8e616127862bdb2f9f0c3e9324274e360d Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 13:03:30 +0800
Subject: [PATCH 0084/1554] Add a comment to clarify that patterns in finfo_
 shows up first will get applied first.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 82e714fdf9..f726f01ce6 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -514,6 +514,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       csinfo_.conv2d_grad_filter_with_bias,
                       GetConv2DBackpropFilterOrBiasAddGrad});
 
+    // The fusion patterns in "finfo_" that show up first will get applied first,
+    // for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC, A->B->C->D to ABCD},
+    // since the first gets applied first, the final graph will be ABC->D.
+
     //
     // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D
     // (NHWC) + Transpose (NHWC->
-- 
GitLab


From a554e9d7f8dfcc562b568c678f93a88185d7dd05 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 13:12:04 +0800
Subject: [PATCH 0085/1554] Use std::unordered_set instead of std::set, for
 better performance.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index f726f01ce6..305b80df34 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2735,7 +2735,7 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
     //
 
     std::stack<Node *, std::vector<Node *>> work_stack;
-    std::set<Node *> visited_nodes;
+    std::unordered_set<Node *> visited_nodes;
     auto node_checker = fi->node_checkers.begin();
 
     Node *current_node = nullptr;
-- 
GitLab


From 13ed0286c40c81f23a73a9cd773da4d1dd27197a Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 31 Oct 2018 13:35:28 +0800
Subject: [PATCH 0086/1554] Fix 2 bugs in CheckNodeForFusion(): 1. The else
 case for node_checker fails are not handled properly. Should pop the stack,
 rather then break the while loop entirely. 2. The nested level of stack op
 when node_check succeed is wrong.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 305b80df34..65bd568f6c 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2770,19 +2770,22 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
               all_succ_has_been_visited = false;
               break;
             }
-
-            // All successor nodes of current node has been visited (no match found),
-            // pop the stack and mark current node as "visited".
-            if (all_succ_has_been_visited) {
-              visited_nodes.insert(current_node);
-              work_stack.pop();
-              -- node_checker;
-            }
           }
         }
+
+        // All successor nodes of current node has been visited (no match found),
+        // pop the stack and mark current node as "visited".
+        if (all_succ_has_been_visited) {
+          visited_nodes.insert(current_node);
+          work_stack.pop();
+          -- node_checker;
+        }
+
       } else {
-        // current node doesn't match, just break and stack will help us roll back.
-        break;
+        // current node doesn't match, pop stack to roll back.
+        visited_nodes.insert(current_node);
+        work_stack.pop();
+        -- node_checker;
       }
     }
   }
-- 
GitLab


From 1df5f7a302864741f0e814d030bebc7094553627 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 1 Nov 2018 16:34:43 -0700
Subject: [PATCH 0087/1554] Fix the data type issue in compute_output_shape

---
 tensorflow/python/keras/layers/wrappers.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index c795b2aa7e..82c96e386a 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -441,10 +441,15 @@ class Bidirectional(Wrapper):
 
   @tf_utils.shape_type_conversion
   def compute_output_shape(self, input_shape):
-    output_shape = tuple(self.forward_layer.compute_output_shape(
-        input_shape).as_list())
+    forward_layer_output_shape \
+      = self.forward_layer.compute_output_shape(input_shape)
+    if getattr(forward_layer_output_shape, 'as_list', None) is None:
+      output_shape = tuple(forward_layer_output_shape)
+    else:
+      output_shape = tuple(forward_layer_output_shape.as_list())
+
     if self.return_state:
-      state_shape = output_shape[1:]
+      state_shape = list(output_shape[1:])
       output_shape = output_shape[0]
 
     if self.merge_mode == 'concat':
-- 
GitLab


From 772566443a8e2f4dc962efd622967f8d2a80319d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Fri, 2 Nov 2018 11:14:03 +0800
Subject: [PATCH 0088/1554] TST: use session instead

---
 tensorflow/python/kernel_tests/conv_ops_3d_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index b3e73fa841..3ff23852e0 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -413,7 +413,7 @@ class Conv3DTest(test.TestCase):
       elif data_type == dtypes.float16:
         tolerance = 1e-3
 
-      with self.cached_session(use_gpu=use_gpu):
+      with self.session(use_gpu=use_gpu):
         orig_input_tensor = constant_op.constant(
             input_data, shape=input_shape, dtype=data_type, name="input")
         filter_tensor = constant_op.constant(
-- 
GitLab


From 7e14f51c49c35fe577f0700ccec8b72c17d9f1fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Fri, 2 Nov 2018 11:45:00 +0800
Subject: [PATCH 0089/1554] ENH: register double for grad

---
 tensorflow/core/kernels/conv_grad_ops_3d.cc        | 8 ++++++++
 tensorflow/python/kernel_tests/conv_ops_3d_test.py | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc
index bab91f5e86..9271bfdf04 100644
--- a/tensorflow/core/kernels/conv_grad_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc
@@ -1859,6 +1859,14 @@ class Conv3DBackpropFilterOp<GPUDevice, T> : public OpKernel {
                           Conv3DBackpropFilterOp<GPUDevice, T>);
 TF_CALL_half(REGISTER_GPU_KERNEL);
 TF_CALL_float(REGISTER_GPU_KERNEL);
+REGISTER_KERNEL_BUILDER(
+    Name("Conv3DBackpropInput").Device(DEVICE_GPU).TypeConstraint<double>("T"),
+    Conv3DBackpropInputOp<GPUDevice, double>);
+REGISTER_KERNEL_BUILDER(Name("Conv3DBackpropInputV2")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<double>("T")
+                            .HostMemory("input_sizes"),
+                        Conv3DBackpropInputOp<GPUDevice, double>);
 #undef REGISTER_GPU_KERNEL
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index 3ff23852e0..dd90676f09 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -413,7 +413,7 @@ class Conv3DTest(test.TestCase):
       elif data_type == dtypes.float16:
         tolerance = 1e-3
 
-      with self.session(use_gpu=use_gpu):
+      with self.test_session(use_gpu=use_gpu):
         orig_input_tensor = constant_op.constant(
             input_data, shape=input_shape, dtype=data_type, name="input")
         filter_tensor = constant_op.constant(
-- 
GitLab


From 92878d5ae1596858c7682a2e93e5fe8969066854 Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Fri, 2 Nov 2018 20:50:36 +0000
Subject: [PATCH 0090/1554] Fixed import nest issue

---
 tensorflow/python/ops/cond_v2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 714cfbf4db..7e6c95ac88 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -34,6 +34,8 @@ from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.util import nest
+
 
 # NOTE(skyewm): TensorFlow uses protected class methods and fields to signify
 # that they aren't part of the official public API. These protected members
-- 
GitLab


From 58f69a5899c8a902956b876c3fa8baab6e60e7c7 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 6 Sep 2018 15:59:07 -0700
Subject: [PATCH 0091/1554] Allows user to specify int8 scales via TF nodes.

Basic working example - all tensors have range of -5 to 5 [skip ci]

Range of -0.1 to 0.1 seems to work decently [skip ci
]

Load quantization numbers from txt file. Switching between prequant and calib now controlled by two bools (in convert_node.h and convert_graph.h - working on merging to a single control arg [skip ci]

stuff in scripts for testing prequantized int8 [skip ci]

Add python interface using dict and support dynamic op

debug prints and inference scripts

propagate

First mode where we simply tell the user which ranges are missing if not all are supplied. Still need to infer some ranges backward through shuffle etc. Fix ConvertIdentity bug and add error checking to input binding [skip ci]

cleanup and fix more [skip ci]

Set softmax range. Fix some cases of inferring ranges. Remove theoretical Matmul infer. Don't print missing ranges for internal unnamed tensors

Remove unneeded conversion of dict -> unordered_map (it had bugs with python2 strings causing python2 build to fail) [skip ci]

Fix syntax error and infer backwards in another shuffle situation

Pass weights through quant nodes unchanged. Fix broadcast issue with Relu6. Remove some problematic backwards infers

Cleanup

Cleanup

Change arg to use_calibrate. Add unction to copy ranges. Add comments for features which are incomplete or not used. Fix identity conversion.

Add missing code for Relu6

Add missing range copies and fix issue with const ITensor*

Properly compute symmetric range. Fix const cast bug

Fix bug with use_calibration arg. Change quantization range interface to use Converter class. Change range inferring algorithm to allow for multiple hops. Formatting fixes.

Quant ops are format-agnostic to layout optimizer now. Exclude quant ops from FP16 and FP32 graphs. Rename scales to ranges. Formatting

Add tests for quantization

Fix bug with symmetric range calculation by switching to std::max. Improve tests (they will be broken until Transpose op is added). Add check for when mode is not int8 and calib=true

Fix tests. Add comments and checks, small fixes.

Fix candidate ops check. Log missing tensors unconditionally. Improve some comments. Revert ConvertIdentity to original implementation - the name check in convert_node is sufficient and adding the no-op shuffles introduced problems

Don't use shuffle no-op as conversion for quantize, instead just pass tensor through. Shuffle no-op is not optimized away like we thought so it can interfere with TRT's node fusion, causing reduced accuracy

Const bool
---
 tensorflow/contrib/tensorrt/BUILD             |   1 +
 .../contrib/tensorrt/convert/convert_graph.cc |  59 ++--
 .../contrib/tensorrt/convert/convert_graph.h  |   7 +-
 .../tensorrt/convert/convert_graph_test.cc    |   8 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 252 +++++++++++++++++-
 .../contrib/tensorrt/convert/convert_nodes.h  |  39 ++-
 .../tensorrt/convert/trt_optimization_pass.cc |  10 +
 .../tensorrt/convert/trt_optimization_pass.h  |   4 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc |  14 +-
 .../contrib/tensorrt/kernels/trt_engine_op.h  |   4 +
 .../contrib/tensorrt/ops/trt_engine_op.cc     |   3 +-
 .../contrib/tensorrt/python/trt_convert.py    |  19 +-
 .../contrib/tensorrt/segment/segment.cc       |   4 +-
 tensorflow/contrib/tensorrt/segment/segment.h |   4 +-
 tensorflow/contrib/tensorrt/test/base_test.py |   4 +-
 .../tensorrt/test/quantization_test.py        | 164 ++++++++++++
 .../test/tf_trt_integration_test_base.py      |  45 +++-
 .../grappler/optimizers/layout_optimizer.cc   |   5 +
 18 files changed, 590 insertions(+), 56 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 20bcd2447e..64a2fd50c3 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,6 +491,7 @@ cuda_py_tests(
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
+        "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
         "test/vgg_block_nchw_test.py",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 1f5591fe2a..40d03ace6f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -85,7 +85,7 @@ TrtCandidateSelector::TrtCandidateSelector(
     const grappler::GraphProperties& graph_properties)
     : graph_properties_(graph_properties) {}
 
-Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
+Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node, int precision_mode) {
   // TODO(laigd): move this set to TrtNodeValidator where it should belong.
   // LINT.IfChange
   static const std::set<string> candidate_ops = {
@@ -128,11 +128,24 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
     "Prod",
     "Max",
     "Min",
+    "Relu6",
   };
+  bool is_supported_op_type = (candidate_ops.count(node->type_string()) ||
+      PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string()));
+#if NV_TENSORRT_MAJOR >= 5
+  static const std::set<string> quantize_ops = {
+    "QuantizeV2",
+    "Dequantize",
+    "QuantizeAndDequantizeV2",
+    "QuantizeAndDequantizeV3",
+    "FakeQuantWithMinMaxVars",
+  };
+  if (precision_mode == INT8MODE &&
+      quantize_ops.count(node->type_string())) {
+    is_supported_op_type = true;
+  }
+#endif
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc)
-  const bool is_supported_op_type =
-      (candidate_ops.count(node->type_string()) ||
-       PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string()));
   if (!is_supported_op_type) {
     return errors::Unimplemented("Op type ", node->type_string(),
                                  " is not supported.");
@@ -219,7 +232,8 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode, int minimum_segment_size, bool is_dyn_op,
-    int max_cached_engines, std::vector<int> cached_engine_batches) {
+    int max_cached_engines, std::vector<int> cached_engine_batches,
+    bool use_calibration) {
   // Create GrapplerItem.
   tensorflow::grappler::GrapplerItem item;
   item.fetch = output_names;
@@ -284,6 +298,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       list->add_i(batch);
     }
   }
+  parameters["use_calibration"].set_b(use_calibration);
 
   // Run optimizer.
   tensorflow::grappler::MetaOptimizer meta_opt(nullptr, rw_cfg);
@@ -563,27 +578,31 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
       }
     }
   }
+
+  const bool calibrate_int8 = (info.precision_mode == INT8MODE && info.use_calibration);
+  // Build the engine and get its serialized representation.
   string segment_string;
-  if (info.engine_type == EngineInfo::EngineType::TRTStatic ||
-      info.precision_mode == INT8MODE) {
+  if (info.engine_type == EngineInfo::EngineType::TRTStatic || 
+      calibrate_int8) {
     // Create static engine for fp32/fp16 mode, and test validity of the engine
-    // for int8 mode. We don't want engine to fail at the calibration time.
-    // So we are constructing a FP32 engine here to check its validity, and if
-    // it is a valid engine then we put the serialized graphdef to the op.
-    // Otherwise we skip node creation for this engine.
+    // for int8 calibration mode. We don't want engine to fail at the
+    // calibration time. So we are constructing a FP32 engine here to check its
+    // validity, and if it is a valid engine then we put the serialized graphdef
+    // to the op. Otherwise we skip node creation for this engine.
     Logger trt_logger;
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     // TODO(sami): What happens if 1st dim is not batch?
     TF_RETURN_IF_ERROR(ConvertGraphDefToEngine(
         info.segment_graph_def,
-        info.precision_mode == INT8MODE ? FP32MODE : info.precision_mode,
+        calibrate_int8 ? FP32MODE : info.precision_mode,
         max_batch_size, info.max_workspace_size_bytes, input_shapes,
         &trt_logger, alloc, /*calibrator=*/nullptr, &engine,
+        info.use_calibration,
         /*convert_successfully=*/nullptr));
     TrtUniquePtrType<nvinfer1::IHostMemory> engine_data(engine->serialize());
     segment_string =
         string((const char*)engine_data->data(), engine_data->size());
-    if (info.precision_mode == INT8MODE) {
+    if (calibrate_int8) {
       // See above comment about why not putting this inside the 'else' branch.
       segment_string = info.segment_graph_def.SerializeAsString();
     }
@@ -595,7 +614,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   // conversion.
   string prec_string;
   TF_RETURN_IF_ERROR(GetPrecisionModeName(info.precision_mode, &prec_string));
-  if (info.precision_mode == INT8MODE &&
+  if (info.precision_mode == INT8MODE && calibrate_int8 &&
       !TRTResourceManager::instance()->getManager("TRTCalibration")) {
     LOG(ERROR) << "Failed to construct calibration storage";
   }
@@ -631,6 +650,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
           .Attr("cached_engine_batches", {max_batch_size})
           .Attr("workspace_size_bytes", info.max_workspace_size_bytes)
           .Attr("precision_mode", prec_string)
+          .Attr("use_calibration", info.use_calibration)
           .Attr("OutT", out_types)
           .Finalize(&trt_node);
   if (!status.ok()) {
@@ -862,12 +882,13 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     segment_options.exclude_node_list.insert(node);
   }
   segment_options.minimum_segment_size = params.minimum_segment_size;
+  segment_options.precision_mode = params.precision_mode;
   tensorflow::tensorrt::segment::SegmentNodesVector initial_segments;
   TrtCandidateSelector candidate_selector(*params.graph_properties);
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
       &graph,
       std::bind(&TrtCandidateSelector::IsTensorRTCandidate, &candidate_selector,
-                std::placeholders::_1),
+                std::placeholders::_1, std::placeholders::_2),
       // Input validation is already done by TrtCandidateSelector, so we don't
       // need to check the input edges.
       [](const Edge* edge) { return true; }, OutputEdgeValidator(),
@@ -901,10 +922,14 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       continue;
     }
     curr_engine.precision_mode = params.precision_mode;
-    curr_engine.engine_type =
-        (params.is_dyn_op || params.precision_mode == INT8MODE
+    if (params.use_calibration && params.precision_mode != INT8MODE) {
+      return tensorflow::errors::Unimplemented(
+           "Calibration with FP32 or FP16 is not implemented. ");
+    }
+    curr_engine.engine_type = ((params.is_dyn_op || params.use_calibration)
              ? EngineInfo::EngineType::TRTDynamic
              : EngineInfo::EngineType::TRTStatic);
+    curr_engine.use_calibration = params.use_calibration;
     curr_engine.cached_engine_batches = params.cached_engine_batches;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
     StrAppend(&curr_engine.engine_name, "my_trt_op_", t);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 1c9d82105a..eaa1edbfe4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -39,7 +39,7 @@ class TrtCandidateSelector {
 
   // Returns OK iff 'node' is a TF-TRT conversion candidate, which will be added
   // to TRT subgraph and later converted into TRT engine.
-  Status IsTensorRTCandidate(const tensorflow::Node* node);
+  Status IsTensorRTCandidate(const tensorflow::Node* node, int precision_mode);
 
  private:
   // The TF-TRT node converter used to verify whether individual node is
@@ -63,6 +63,7 @@ struct ConversionParams {
         cluster(nullptr),
         is_dyn_op(false),
         fixed_input_size(true),
+        use_calibration(true),
         max_cached_engines(1) {}
   const tensorflow::GraphDef* input_graph_def;
   const std::vector<string>* output_names;
@@ -76,6 +77,7 @@ struct ConversionParams {
   bool is_dyn_op;  //  Whether to create engine on conversion or execution time
   bool fixed_input_size;   // Assume non-batch ranks of input tensors are fixed
   int max_cached_engines;  // maximum number of cached engines
+  bool use_calibration;
   std::vector<int> cached_engine_batches;  // list of cached engines
 };
 
@@ -95,7 +97,8 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode = 1, int minimum_segment_size = 3,
     bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batches = {});
+    std::vector<int> cached_engine_batches = {},
+    bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
index f10729987f..925de7885f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
@@ -94,16 +94,16 @@ TEST(TrtCandidateSelector, Basics) {
   TF_EXPECT_OK(graph_properties.InferStatically(true));
 
   TrtCandidateSelector selector(graph_properties);
-  TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node()));
+  TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node(), FP32MODE));
   ExpectStatus(
-      selector.IsTensorRTCandidate(incompatible_matmul.operation.node()),
+      selector.IsTensorRTCandidate(incompatible_matmul.operation.node(), FP32MODE),
       error::INVALID_ARGUMENT,
       "transpose_a is not supported for TensorRT FullyConnected "
       "(op: MatMul), at: incompatible_matmul");
-  ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node()),
+  ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node(), FP32MODE),
                error::UNIMPLEMENTED, "Op type Sin is not supported");
   ExpectStatus(selector.IsTensorRTCandidate(
-                   matmul_with_incompatible_input.operation.node()),
+                   matmul_with_incompatible_input.operation.node(), FP32MODE),
                error::INTERNAL,
                "Failed to convert input with index 0 to a TRT_TensorOrWeights");
 }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index a6f954391d..48e8a6fc51 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -95,6 +95,9 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
     case tensorflow::DataType::DT_INT8:
       *trt_dtype = nvinfer1::DataType::kINT8;
       break;
+    case tensorflow::DataType::DT_QINT8:
+      *trt_dtype = nvinfer1::DataType::kINT8;
+      break;
     case tensorflow::DataType::DT_HALF:
       *trt_dtype = nvinfer1::DataType::kHALF;
       break;
@@ -634,6 +637,16 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
           ostrides);
       break;
     }
+    case tensorflow::DataType::DT_INT8:
+    case tensorflow::DataType::DT_UINT8:
+    case tensorflow::DataType::DT_QINT8:
+    case tensorflow::DataType::DT_QUINT8: {
+      Reorder2({k, c}, static_cast<int8_t const*>(iweights.GetValues()),
+               istrides,
+               static_cast<int8_t*>(const_cast<void*>(oweights->GetValues())),
+               ostrides);
+      break;
+    }
     default:
       LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
                  << DataTypeString(iweights.type_);
@@ -812,13 +825,18 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
     TRT_TensorOrWeights& output = outputs[i];
     string output_name = node_def.name();
     if (i != 0) output_name = StrCat(output_name, ":", i);
-    // We need to check the name before setting it. For Identity op where the
-    // output is the input, if its input is one of the engine input, setting
-    // the name here will overwrite engine input bindings which will cause
-    // runtime error.
+    // We need to check the name before setting it. If the input is one of the
+    // engine input, setting the name here will overwrite engine input
+    // bindings which will cause runtime error. 
     if (output.is_tensor()) {
       const char* tensor_name = output.tensor()->getName();
-      if (tensor_name == nullptr || std::strlen(tensor_name) == 0) {
+      if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) {
+        // TRT initializes tensor names as "(Unnamed ITensor* N)". We rename
+        // them to match their corresponding TensorFlow name.
+        // Note: ITensors that we create internally within TF-TRT which are
+        // not inputs or outputs of a node will not be renamed. This is a
+        // potential cause of confusion if an error message or warning
+        // mentions the unnamed tensor.
         output.tensor()->setName(output_name.c_str());
       }
     }
@@ -930,6 +948,7 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor,
 
   nvinfer1::IShuffleLayer* layer = this->network()->addShuffle(*input_tensor);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Transpose");
+  MarkQuantizationRangesAsInferrable(input_tensor, layer->getOutput(0));
 
   nvinfer1::Permutation permutation;
   for (int32_t i = 0; i < dims.nbDims; ++i) {
@@ -976,6 +995,8 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
           *const_cast<nvinfer1::ITensor*>(input.tensor()));
       TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
       layer->setReshapeDimensions(dims);
+      MarkQuantizationRangesAsInferrable(
+          const_cast<nvinfer1::ITensor*>(input.tensor()), layer->getOutput(0));
       *tensor = layer->getOutput(0);
     }
   } else {
@@ -987,6 +1008,95 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
   return tensorflow::Status::OK();
 }
 
+void Converter::MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input,
+                                                   nvinfer1::ITensor* output) {
+  quantization_infer_.push_back({input, output});
+  quantization_infer_.push_back({output, input});
+}
+
+void Converter::ProvideQuantizationRange(nvinfer1::ITensor* tensor,
+                                         float min_range, float max_range) {
+  float symmetric_range = std::max(std::abs(min_range), std::abs(max_range));
+  quantization_ranges_[tensor] = symmetric_range;
+}
+
+void Converter::ApplyQuantizationRanges(bool warn_missing_ranges) {
+  // Infer ranges across marked ops
+  PropagateQuantizationRanges();
+  // Get all tensors from network
+  std::set<nvinfer1::ITensor*> all_tensors;
+  std::set<nvinfer1::ITensor*> tensors_missing_ranges;
+  for (int i = 0; i < this->network()->getNbLayers(); i++) {
+    nvinfer1::ILayer* layer = this->network()->getLayer(i);
+    for (int j = 0; j < layer->getNbInputs(); j++)
+      all_tensors.insert(layer->getInput(j));
+    for (int j = 0; j < layer->getNbOutputs(); j++)
+      all_tensors.insert(layer->getOutput(j));
+  }
+  // Apply ranges
+  for (auto tensor : all_tensors) {
+    auto it = quantization_ranges_.find(tensor);
+    if (it != quantization_ranges_.end()) {
+      float range = it->second;
+      VLOG(1) << "Setting range for: " << tensor->getName() << ": " << range;
+#if NV_TENSORRT_MAJOR >= 5
+      tensor->setDynamicRange(-range, range);
+#endif
+    } else {
+      tensors_missing_ranges.insert(tensor);
+    }
+  }
+  // Warn user about tensors that are missing ranges. If TRT fuses some layers
+  // then these tensors may not actually be required, which is why this is
+  // just a warning. If we are still missing ranges even after fusion,
+  // Builder::buildCudaEngine() will return nullptr and we will catch the
+  // error at that point.
+  if (warn_missing_ranges) {
+    for (auto tensor : tensors_missing_ranges) {
+      // Note: there may be some warnings for "(Unnamed ITensor* N)". These
+      // are tensors which are created internally by TF-TRT. The ranges for
+      // these unnamed ITensors are always inferred from user provided ranges,
+      // thus there will also be a warning for the range(s) the user missed.
+      LOG(WARNING) << "Quantization range was not found for "
+                    << tensor->getName() << ". "
+                    << "This might be okay if TensorRT does not need the range"
+                    << "(e.g. due to node fusion).";
+    }
+  }
+}
+
+void Converter::PropagateQuantizationRanges() {
+  // Propagate ranges across edges in quantization_infer_ until no new
+  // information is added.
+  // Note: this function modifies quantization_infer_, it might be better to
+  // modify a copy instead if we for some reason need quantization_infer_
+  // later.
+  bool information_added = true;
+  while (information_added) {
+    information_added = false;
+    for (auto it = quantization_infer_.begin();
+        it != quantization_infer_.end();) {
+      auto input_tensor_range = quantization_ranges_.find(it->first);
+      auto output_tensor_range = quantization_ranges_.find(it->second);
+      if (input_tensor_range != quantization_ranges_.end() &&
+          output_tensor_range == quantization_ranges_.end()) {
+        // Input has range but output doesn't: copy range
+        quantization_ranges_[it->second] = input_tensor_range->second;
+        information_added = true;
+        VLOG(1) << "Copy quantization range: "
+                << it->first->getName() << " -> " << it->second->getName();
+      }
+      // We can remove edges when the output range is known
+      if (quantization_ranges_.find(it->second) !=
+          quantization_ranges_.end()) {
+        it = quantization_infer_.erase(it);
+      } else {
+        ++it;
+      }
+    }
+  }
+}
+
 Status Converter::GetInputs(const tensorflow::NodeDef& node_def,
                             std::vector<TRT_TensorOrWeights>* inputs) const {
   for (auto const& input_name : node_def.input()) {
@@ -1788,6 +1898,8 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
         nvinfer1::DimsHW(padding[0].second, padding[1].second));
     TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name());
+    params->converter->MarkQuantizationRangesAsInferrable(
+        const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
   }
@@ -1795,6 +1907,11 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
   nvinfer1::IPoolingLayer* layer = params->converter->network()->addPooling(
       *const_cast<nvinfer1::ITensor*>(tensor), type, ksize);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+  // TODO(tmorris): Average pooling may not be entirely safe to infer
+  // quantization range through (at least forwards - backwards should be fine).
+  // Max pooling is okay.
+  params->converter->MarkQuantizationRangesAsInferrable(
+      const_cast<nvinfer1::ITensor*>(tensor), layer->getOutput(0));
 
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
@@ -1823,6 +1940,101 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertQuantize(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.at(0).is_weights()) {
+    params->outputs->push_back(inputs.at(0));
+    return tensorflow::Status::OK();
+  }
+  nvinfer1::ITensor* tensor = const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor());
+  // Min
+  TRT_ShapedWeights weights_min = inputs.at(1).weights();
+  auto weights_min_ptr = static_cast<float*>(const_cast<void*>(weights_min.GetValues()));
+  float min_range = weights_min_ptr[0];
+  // Max
+  TRT_ShapedWeights weights_max = inputs.at(2).weights();
+  auto weights_max_ptr = static_cast<float*>(const_cast<void*>(weights_max.GetValues()));
+  float max_range = weights_max_ptr[0];
+  // Store ranges for tensor
+  params->converter->ProvideQuantizationRange(tensor, min_range, max_range);
+  // Sometimes, TRT may not quantize a tensor, either because it chooses to
+  // execute a higher precision kernel or because of op fusion. In these cases,
+  // accuracy will suffer if the model was trained to expect quantization at
+  // that tensor. We should consider adding a clip(tensor, min_range, max_range)
+  // operation here to ensure that any arbitrarily placed quantize node will
+  // execute as expected. However, this will negatively affect performance. If
+  // users train their models in a way which models inference as close as
+  // possible (i.e. not quantizing in place where fusion will occur), then there
+  // is no problem with the current implementation.
+  params->outputs->push_back(inputs.at(0));
+  return tensorflow::Status::OK();
+}
+
+// TODO(pdavoodi): we should update relu6 implementation once TensorRT supports
+// Relu6 natively.
+tensorflow::Status ConvertRelu6(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  // ***************************************************************************
+  // TensorRT does not implement Relu6 natively. This function converts Relu6 op
+  // to available TensorRT ops: Relu6(x) = min(Relu(x), 6)
+  // ***************************************************************************
+
+  // Input Tensor 
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  
+  // Relu operation i.e. Relu(x) = max(0, x)
+  nvinfer1::IActivationLayer* relu_layer = 
+      params->converter->network()->addActivation(
+          *const_cast<nvinfer1::ITensor*>(tensor),
+          nvinfer1::ActivationType::kRELU);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(relu_layer, node_def.name());
+  
+  // Large range of relu is problematic during quantization in INT8 precision mode.
+  // Setting dynamic range of relu = [0.f, 6.0f] helps with quantization.
+  // TRT only uses dynamic ranges in INT8 precision mode,
+  // and this does not affect the FP32 path.
+  params->converter->ProvideQuantizationRange(
+      relu_layer->getOutput(0), 0.0f, 6.0f);
+  
+  // Create a constant layer to store the floating point weight i.e. 6.0f This
+  // tensor will be broadcasted uniformly during elementwise `min` operation.
+  // The constant has to have the same rank as the input in order for TRT to
+  // broadcast
+  nvinfer1::Dims dims;
+  dims.nbDims = relu_layer->getOutput(0)->getDimensions().nbDims;
+  for (int i = 0; i < dims.nbDims; i++)
+    dims.d[i] = 1;
+  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
+      tensorflow::DataType::DT_FLOAT, dims);
+  auto weights_ptr = static_cast<float*>(const_cast<void*>(
+      weights.GetValues()));
+  weights_ptr[0] = 6.f;
+  nvinfer1::IConstantLayer* const6_layer = 
+      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_layer, node_def.name());
+  params->converter->ProvideQuantizationRange(
+      const6_layer->getOutput(0), 0.0f, 6.0f);
+  
+  // ElementWise Min Operation
+  // Min op is a nop for INT8 execution path, as the input tensor
+  // to this layer will only have values in range [0.f, 6.0f].
+  const nvinfer1::ITensor* tensor_l  = relu_layer->getOutput(0);
+  const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0);
+  nvinfer1::IElementWiseLayer* relu6_layer = 
+      params->converter->network()->addElementWise(
+          *const_cast<nvinfer1::ITensor*>(tensor_l),
+          *const_cast<nvinfer1::ITensor*>(tensor_r),
+          nvinfer1::ElementWiseOperation::kMIN);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(relu6_layer, node_def.name());
+  nvinfer1::ITensor* output_tensor = relu6_layer->getOutput(0);
+  params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 6.0f);
+
+  params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
 tensorflow::Status ConvertScale(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
@@ -1872,6 +2084,9 @@ tensorflow::Status ConvertScale(OpConverterParams* params) {
         params->converter->network()->addShuffle(
             *const_cast<nvinfer1::ITensor*>(tensor));
     TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name());
+    params->converter->MarkQuantizationRangesAsInferrable(
+        const_cast<nvinfer1::ITensor*>(tensor), shuffle_layer->getOutput(0));
+
     nvinfer1::Dims reshape_dims;
     reshape_dims.nbDims = 3;
     reshape_dims.d[0] = 0;                          // 0 copy from the input
@@ -1911,6 +2126,8 @@ tensorflow::Status ConvertScale(OpConverterParams* params) {
     if (channel_index != 0) {
       shuffle_layer->setSecondTranspose(permutation);
     }
+    params->converter->MarkQuantizationRangesAsInferrable(
+        const_cast<nvinfer1::ITensor*>(output_tensor), shuffle_layer->getOutput(0));
     output_tensor = shuffle_layer->getOutput(0);
   }
 
@@ -2069,6 +2286,9 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
 }
 
 tensorflow::Status ConvertIdentity(OpConverterParams* params) {
+  // TODO(tmorris): TRT's Identity layer does not get optimized away as of TRT
+  // 5.0, however once we know that it does it would be nice to use that
+  // instead.
   params->outputs->push_back(params->inputs.at(0));
   return tensorflow::Status::OK();
 }
@@ -2691,6 +2911,8 @@ tensorflow::Status ConvertSoftmax(OpConverterParams* params) {
   layer->setAxes(1 << (nbDims - 1));
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  // Quantization range for SoftMax is always (0, 1)
+  params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f);
   params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
@@ -2787,6 +3009,14 @@ void Converter::RegisterOpConverters() {
   op_registry_["MatMul"] = ConvertMatMul;
   op_registry_["BatchMatMul"] = ConvertBatchMatMul;
   op_registry_["TopKV2"] = ConvertTopK;
+  op_registry_["Relu6"] = ConvertRelu6;
+# if NV_TENSORRT_MAJOR >= 5
+  op_registry_["QuantizeV2"] = ConvertQuantize;
+  op_registry_["Dequantize"] = ConvertQuantize;
+  op_registry_["QuantizeAndDequantizeV2"] = ConvertQuantize;
+  op_registry_["QuantizeAndDequantizeV3"] = ConvertQuantize;
+  op_registry_["FakeQuantWithMinMaxVars"] = ConvertQuantize;
+#endif
 
   plugin_converter_ = ConvertPlugin;
 }
@@ -2798,6 +3028,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     Logger* logger, nvinfer1::IGpuAllocator* allocator,
     TRTInt8Calibrator* calibrator,
     TrtUniquePtrType<nvinfer1::ICudaEngine>* engine,
+    bool use_calibration,
     bool* convert_successfully) {
   engine->reset();
   if (convert_successfully) *convert_successfully = false;
@@ -2812,7 +3043,11 @@ tensorflow::Status ConvertGraphDefToEngine(
     builder->setHalf2Mode(true);
   } else if (precision_mode == INT8MODE) {
     builder->setInt8Mode(true);
-    builder->setInt8Calibrator(calibrator);
+    if (use_calibration) {
+      builder->setInt8Calibrator(calibrator);
+    } else {
+      builder->setInt8Calibrator(nullptr);
+    }
   }
 
   // Create the network.
@@ -2881,6 +3116,11 @@ tensorflow::Status ConvertGraphDefToEngine(
   TF_RETURN_IF_ERROR(converter.RenameAndMarkOutputTensors(output_tensors));
   if (convert_successfully) *convert_successfully = true;
 
+  // Apply user provided quantization ranges to tensors
+  const bool warn_missing_ranges = (precision_mode == INT8MODE &&
+                                    !use_calibration);
+  converter.ApplyQuantizationRanges(warn_missing_ranges);
+
   // Build the engine.
   VLOG(1) << "Starting engine creation";
   engine->reset(builder->buildCudaEngine(*converter.network()));
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 5cc28b33e7..2daca99ed9 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -92,7 +92,8 @@ struct EngineInfo {
   EngineInfo()
       : engine_type(EngineType::TRTStatic),
         max_workspace_size_bytes(0),
-        precision_mode(FP32MODE) {}
+        precision_mode(FP32MODE),
+        use_calibration(true) {}
 
   string engine_name;
   string device;
@@ -109,6 +110,7 @@ struct EngineInfo {
   int maximum_cached_engines;
   std::vector<int> cached_engine_batches;
   int precision_mode;
+  bool use_calibration;
 };
 
 // Constructs a graphdef from the segment in the given graph. Adds placeholder
@@ -146,6 +148,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     Logger* logger, nvinfer1::IGpuAllocator* allocator,
     TRTInt8Calibrator* calibrator,
     TrtUniquePtrType<nvinfer1::ICudaEngine>* engine,
+    bool use_calibration,
     bool* convert_successfully);
 
 // Helper class for the segmenter to determine whether an output edge from the
@@ -425,6 +428,22 @@ class Converter {
   // Is the converter operating in fp16 mode?
   bool is_fp16() const { return is_fp16_; }
 
+  // This should be called on the inputs and outputs of any layer we create
+  // where we know that the quantization range does not change during that
+  // operation. (e.g. Reshape, Transpose, Identity, MaxPool).
+  void MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input,
+                                          nvinfer1::ITensor* output);
+
+  // This function should be called when we know the quantization range of a
+  // tensor, either from a quantize/dequantize node or when the output is a
+  // fixed range (e.g. SoftMax, Relu6, Sigmoid).
+  void ProvideQuantizationRange(nvinfer1::ITensor* tensor,
+                                float min_range, float max_range);
+
+  // Should be called when full TRT network has been constructed and before
+  // building the engine.
+  void ApplyQuantizationRanges(bool warn_missing_ranges);
+
   // Below are helper methods for op converters to add different layers to the
   // TRT network.
 
@@ -457,6 +476,8 @@ class Converter {
 
   void RegisterOpConverters();
 
+  void PropagateQuantizationRanges();
+  
   // Registered op converters by op type.
   std::unordered_map<string, OpConverter> op_registry_;
 
@@ -472,6 +493,22 @@ class Converter {
   // Store the weights added during construction of trt_network_.
   TrtWeightStore weight_store_;
 
+  // During conversion, this table is populated with quantization ranges per
+  // tensor. ApplyQuantizationRanges() will use this table to set the TensorRT
+  // quantization ranges. Since TRT only supports symmetric ranges, we will
+  // store the range as a single float = max(abs(min_range), abs(max_range)).
+  // Range refers to the floating point values, e.g. min_range = 0.0f, max_range
+  // = 6.0f for Relu6.
+  std::unordered_map<nvinfer1::ITensor*, float> quantization_ranges_;
+
+  // Edges where quantization ranges can be inferred (copied) across ops - from
+  // first tensor to second tensor. PropagateQuantizationRanges() will propagate
+  // known ranges from quantization_ranges_ across these edges, adding the new
+  // ranges to quantization_ranges_ so that they can be applied in
+  // ApplyQuantizationRanges().
+  std::vector<std::pair<nvinfer1::ITensor*,nvinfer1::ITensor*>>
+      quantization_infer_;
+
   const bool is_fp16_;
 
   // Batch size of inputs to trt_network_ added by AddInputTensor(). During
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
index b30d94b028..4ac7e21d34 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
@@ -67,6 +67,9 @@ tensorflow::Status TRTOptimizationPass::Init(
     TF_RETURN_IF_ERROR(GetPrecisionMode(
         Uppercase(params.at("precision_mode").s()), &precision_mode_));
   }
+  if (params.count("use_calibration")) {
+    use_calibration_ = params.at("use_calibration").b();
+  }
   return tensorflow::Status::OK();
 }
 
@@ -222,6 +225,12 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(true));
   tensorflow::tensorrt::convert::ConversionParams cp;
 
+  if (use_calibration_ && precision_mode_ != INT8MODE) {
+    LOG(ERROR) << "Calibration with FP32 or FP16 is not implemented. "
+               << "Falling back to use_calibration = False.";
+    use_calibration_ = false;
+  }
+
   std::vector<string> nodes_to_preserve;
   for (const auto& n : item.NodesToPreserve()) {
     auto tokens = str_util::Split(n, ":");
@@ -250,6 +259,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   cp.is_dyn_op = is_dynamic_op_;
   cp.cached_engine_batches = batches_;
   cp.max_cached_engines = max_cached_batches_;
+  cp.use_calibration = use_calibration_;
   auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp);
   VLOG(1) << "Returning from " << name_;
   return status;
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
index 71b51d1368..3e8dc0978e 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
@@ -38,7 +38,8 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
         maximum_batch_size_(-1),
         is_dynamic_op_(false),
         max_cached_batches_(1),
-        max_workspace_size_bytes_(256LL << 20) {
+        max_workspace_size_bytes_(256LL << 20),
+        use_calibration_(true) {
     VLOG(1) << "Constructing " << name_;
   }
 
@@ -67,6 +68,7 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
   std::vector<int> batches_;
   int max_cached_batches_;
   int64_t max_workspace_size_bytes_;
+  bool use_calibration_;
 };
 
 }  // namespace convert
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 019446813a..780343d662 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -124,8 +124,10 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
   OP_REQUIRES_OK(context,
                  context->GetAttr("segment_funcdef_name", &funcdef_name_));
   OP_REQUIRES_OK(context, GetPrecisionMode(precision_string, &precision_mode_));
-  calibration_mode_ =
-      (precision_mode_ == INT8MODE && calibration_data.size() == 0);
+  OP_REQUIRES_OK(context,
+                 context->GetAttr("use_calibration", &use_calibration_));
+  calibration_mode_ = (use_calibration_ &&
+      (precision_mode_ == INT8MODE && calibration_data.size() == 0));
   if (calibration_data.size()) {
     calibrator_.reset(new TRTInt8Calibrator(calibration_data));
     calibration_data.resize(0);
@@ -308,7 +310,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
   std::vector<void*> buffers(num_binding);
   for (int i = 0; i < ctx->num_inputs(); i++) {
     const string input_name = StrCat(kInputPHName, i);
-    const size_t binding_index =
+    const int binding_index =
         trt_engine_ptr->getBindingIndex(input_name.c_str());
     if (binding_index == -1) {
       LOG(ERROR) << "Input node not found, at " << input_name;
@@ -345,7 +347,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
   for (int i = 0; i < ctx->num_outputs(); i++) {
     // Create an output tensor
     const string output_name = StrCat(kOutputPHName, i);
-    const size_t binding_index =
+    const int binding_index =
         trt_engine_ptr->getBindingIndex(output_name.c_str());
     Tensor* output_tensor = nullptr;
 
@@ -497,7 +499,8 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
         segment_graph_, precision_mode_, batch_size, workspace_size_, shapes,
-        &logger, allocator, calibrator_.get(), &engine, &convert_successfully);
+        &logger, allocator, calibrator_.get(), &engine,
+        use_calibration_, &convert_successfully); 
     if (!status.ok()) {
       if (convert_successfully) {
         // This means it fail to build the engine even when the network is built
@@ -586,6 +589,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
         *segment_graph, INT8MODE, cres->calibrator_->getBatchSize(),
         workspace_size_bytes, shapes, &cres->logger_, cres->allocator_.get(),
         cres->calibrator_.get(), &cres->engine_,
+        /*use_calibration=*/true,
         /*convert_successfully=*/nullptr);
     if (!s.ok()) {
       LOG(ERROR) << "Calibration failed: " << s;
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 8fe0675891..b545f497f3 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -130,6 +130,10 @@ class TRTEngineOp : public AsyncOpKernel {
 
   // The finalized calibrator for inference.
   std::unique_ptr<TRTInt8Calibrator> calibrator_;
+
+  // If true, create calibration graph for INT8 mode. Otherwise, we are using
+  // user-provided quantization ranges.
+  bool use_calibration_;
 };
 
 }  // namespace tensorrt
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index e0c7b62723..ce04e5806e 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -39,8 +39,9 @@ REGISTER_OP("TRTEngineOp")
     .Attr("cached_engine_batches: list(int) = []")
     .Attr("max_cached_engines_count: int = 1")
     .Attr("workspace_size_bytes: int")
-    .Attr("precision_mode: {'FP32', 'FP16', 'INT8', 'INT8CALIB'}")
+    .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}")
     .Attr("calibration_data: string = ''")
+    .Attr("use_calibration: bool = true")
     .Input("in_tensor: InT")
     .Output("out_tensor: OutT");
 // TODO(jie): TF requires concrete output shape for concrete input shapes.
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index bb81fbf93f..98a647dc57 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -70,7 +70,8 @@ def tensorrt_rewriter_config(rewriter_config=None,
                              minimum_segment_size=3,
                              is_dynamic_op=False,
                              maximum_cached_engines=1,
-                             cached_engine_batch_sizes=None):
+                             cached_engine_batch_sizes=None,
+                             use_calibration=True):
   """Returns a RewriterConfig proto for TRT transformation.
 
   Args:
@@ -95,6 +96,12 @@ def tensorrt_rewriter_config(rewriter_config=None,
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
+    use_calibration: this argument is ignored if precision_mode is not INT8.
+      if set to True, a calibration graph will be created to calibrate the
+      missing ranges. The calibration graph must be converted to an inference
+      graph using calib_graph_to_infer_graph() after running calibration.
+      if set to False, quantization ranges will be expected for every tensor in
+      the graph. If a range is missing, an error will occur.
 
   Returns:
     A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
@@ -138,6 +145,7 @@ def tensorrt_rewriter_config(rewriter_config=None,
                        "maximum_cached_engines items.")
     optimizer.parameter_map["cached_engine_batches"].list.i.extend(
         cached_engine_batch_sizes)
+  optimizer.parameter_map["use_calibration"].b = use_calibration
   return rewriter_config
 
 
@@ -151,6 +159,7 @@ def create_inference_graph(input_graph_def,
                            maximum_cached_engines=1,
                            cached_engine_batch_sizes=None,
                            rewriter_config=None,
+                           use_calibration=True,
                            input_saved_model_dir=None,
                            input_saved_model_tags=None,
                            output_saved_model_dir=None,
@@ -184,6 +193,12 @@ def create_inference_graph(input_graph_def,
       common batch size(s) the application is going to generate.
     rewriter_config: a RewriterConfig proto to append the TensorRTOptimizer to.
       If None, it will create one with default settings.
+    use_calibration: this argument is ignored if precision_mode is not INT8.
+      if set to True, a calibration graph will be created to calibrate the
+      missing ranges. The calibration graph must be converted to an inference
+      graph using calib_graph_to_infer_graph() after running calibration.
+      if set to False, quantization ranges will be expected for every tensor in
+      the graph. If a range is missing, an error will occur.
     input_saved_model_dir: the directory to load the SavedModel which contains
       the input graph to transforms. Used only when input_graph_def is None.
     input_saved_model_tags: list of tags to load the SavedModel.
@@ -326,7 +341,7 @@ def create_inference_graph(input_graph_def,
   rewriter_config = tensorrt_rewriter_config(
       rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
       minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batch_sizes)
+      cached_engine_batch_sizes, use_calibration)
 
   # Run Grappler.
   transformed_graph_def = tf_optimizer.OptimizeGraph(
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 4f64b7a952..80acab9ea3 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -389,7 +389,7 @@ void ContractEdge(SimpleEdge* edge, SimpleGraph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::Graph* tf_graph,
-    const std::function<Status(const tensorflow::Node*)>& candidate_fn,
+    const std::function<Status(const tensorflow::Node*, int)>& candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& input_candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& output_candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
@@ -414,7 +414,7 @@ tensorflow::Status SegmentGraph(
               << " (excluded by segmenter option).";
       node = nullptr;
     } else {
-      const Status status = candidate_fn(node->tf_node());
+      const Status status = candidate_fn(node->tf_node(), options.precision_mode);
       if (!status.ok()) {
         VLOG(1) << "Not a TF-TRT candidate: " << node->name() << ": " << status;
         node = nullptr;
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index b9693aad1b..802daec6a6 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -38,6 +38,8 @@ struct SegmentOptions {
   // Segment must contain at least this many nodes.
   int minimum_segment_size = 2;
   std::set<string> exclude_node_list;
+  // Quantization nodes are only included for quantized precisions
+  int precision_mode;
 };
 
 // Get the subgraphs of a graph that can be handled by TensorRT.
@@ -51,7 +53,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::Graph* tf_graph,
-    const std::function<Status(const tensorflow::Node*)>& candidate_fn,
+    const std::function<Status(const tensorflow::Node*, int)>& candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& input_candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& output_candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 18096e0ff1..cbff661f99 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -197,7 +197,9 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     """Whether to run the test."""
     # Disable the test in fp16 mode since multiple matmul and add ops together
     # can cause overflow.
-    return run_params.precision_mode != "FP16"
+    return ((run_params.precision_mode != "FP16") and
+            not (trt_test.IsQuantizationMode(run_params.precision_mode) and
+            not run_params.use_calibration))
 
 
 class PartiallyConvertedTestB(PartiallyConvertedTestA):
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
new file mode 100644
index 0000000000..a6934bf490
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -0,0 +1,164 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import nn_impl
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.platform import test
+
+
+def build_graph(input_name, input_dims, output_name,
+                add_quantization_nodes=False, dtype=dtypes.float32):
+  def quantize(x, r):
+    if add_quantization_nodes:
+      x = gen_array_ops.fake_quant_with_min_max_vars(x, -r, r)
+    return x
+  g = ops.Graph()
+  with g.as_default():
+    x = array_ops.placeholder(
+        dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+    x = quantize(x, 100)
+    filt1 = constant_op.constant(
+          0.3, shape=(3, 3, 1, 32), dtype=dtype, name='filt1')
+    x = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='VALID')
+    bias1 = constant_op.constant(0.3, shape=(32,), name="bias1", dtype=dtype)
+    x = nn.bias_add(x, bias1)
+
+    x = quantize(x, 6)
+    x = nn.relu6(x)
+    filt2 = constant_op.constant(
+          0.3, shape=(3, 3, 32, 64), dtype=dtype, name='filt2')
+    x = nn.conv2d(x, filt2, strides=[1, 1, 1, 1], padding='VALID')
+    bias2 = constant_op.constant(0.3, shape=(64,), name="bias2", dtype=dtype)
+    x = nn.bias_add(x, bias2)
+
+    x = quantize(x, 6)
+    x = nn.relu6(x)
+    x = math_ops.reduce_mean(x, [1, 2])
+    x = quantize(x, 6)
+    # FC1
+    fc_w1 = constant_op.constant(
+          0.3, shape=(64, 512), dtype=dtype, name='fc_w1')
+    x = math_ops.matmul(x, fc_w1)
+    x = quantize(x, 6)
+    fc_b1 = constant_op.constant(
+          0.3, shape=(512,), dtype=dtype, name='fc_b1')
+    x = nn.bias_add(x, fc_b1)
+    x = quantize(x, 6)
+    x = nn.relu6(x)
+    # FC2
+    fc_w2 = constant_op.constant(
+          0.3, shape=(512, 10), dtype=dtype, name='fc_w2')
+    x = math_ops.matmul(x, fc_w2)
+    x = quantize(x, 25)
+    fc_b2 = constant_op.constant(
+          0.3, shape=(10,), dtype=dtype, name='fc_b2')
+    x = nn.bias_add(x, fc_b2)
+    x = quantize(x, 25)
+    x = array_ops.identity(x, name=output_name)
+  return g
+
+class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Create a graph containing single segment with no quantization ranges."""
+    input_name = "input"
+    input_dims = [100, 28, 28, 1]
+    output_name = "output"
+    g = build_graph(input_name, input_dims, output_name,
+                    add_quantization_nodes=False)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(100, 10)])
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.precision_mode == "INT8" and
+            not run_params.use_optimizer and
+            not run_params.dynamic_engine)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    if run_params.use_calibration:
+      return ["my_trt_op_0"]
+    return []
+
+class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Create a graph containing single segment with no quantization ranges."""
+    input_name = "input"
+    input_dims = [100, 28, 28, 1]
+    output_name = "output"
+    g = build_graph(input_name, input_dims, output_name,
+                    add_quantization_nodes=True)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(100, 10)])
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.precision_mode == "INT8" and
+            not run_params.use_optimizer)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Create a graph containing single segment with no quantization ranges."""
+    input_name = "input"
+    input_dims = [100, 28, 28, 1]
+    output_name = "output"
+    g = build_graph(input_name, input_dims, output_name,
+                    add_quantization_nodes=True)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(100, 10)])
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.precision_mode == "FP32" or
+            run_params.precision_mode == "FP16")
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3",
+            "my_trt_op_4", "my_trt_op_5", "my_trt_op_6"]
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index a725d0651c..a6f51640b7 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -42,14 +42,15 @@ TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
     "gdef", "input_names", "input_dims", "output_names", "expected_output_dims"
 ])
 
-RunParams = namedtuple(
-    "RunParams",
-    ["use_optimizer", "precision_mode", "dynamic_engine", "test_name"])
+RunParams = namedtuple("RunParams", [
+    "use_optimizer", "precision_mode", "dynamic_engine", "test_name",
+    "use_calibration"
+])
 
 ConversionParams = namedtuple("ConversionParams", [
     "max_batch_size", "max_workspace_size_bytes", "precision_mode",
     "minimum_segment_size", "is_dynamic_op", "maximum_cached_engines",
-    "cached_engine_batch_sizes", "rewriter_config"
+    "cached_engine_batch_sizes", "rewriter_config", "use_calibration"
 ])
 
 PRECISION_MODES = ["FP32", "FP16", "INT8"]
@@ -139,11 +140,15 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_op=run_params.dynamic_engine,
         maximum_cached_engines=1,
         cached_engine_batch_sizes=None,
-        rewriter_config=None)
+        rewriter_config=None,
+        use_calibration=run_params.use_calibration)
 
   def ShouldRunTest(self, run_params):
     """Whether to run the test."""
-    return True
+    # This setting combination requires quantization nodes to be present in
+    # order to build the engine.
+    return not (IsQuantizationMode(run_params.precision_mode) and
+                not run_params.use_calibration)
 
   def VerifyRunForEngine(self, engine_name, graph_state, expect_run=True):
     """Verify the state of a particular engine after sess.run()."""
@@ -209,7 +214,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
           conversion_params.minimum_segment_size,
           conversion_params.is_dynamic_op,
           conversion_params.maximum_cached_engines,
-          conversion_params.cached_engine_batch_sizes)
+          conversion_params.cached_engine_batch_sizes,
+          conversion_params.use_calibration)
 
       graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg)
     else:
@@ -301,7 +307,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_op=conversion_params.is_dynamic_op,
         maximum_cached_engines=conversion_params.maximum_cached_engines,
         cached_engine_batch_sizes=conversion_params.cached_engine_batch_sizes,
-        rewriter_config=conversion_params.rewriter_config)
+        rewriter_config=conversion_params.rewriter_config,
+        use_calibration=conversion_params.use_calibration)
 
   def _WriteGraph(self, run_params, gdef, graph_state):
     if graph_state == GraphState.ORIGINAL:
@@ -400,9 +407,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_engine = not node.attr["static_engine"].b
         self.assertEqual(run_params.dynamic_engine, is_dynamic_engine,
                          node.name)
+        self.assertEqual(node.attr["use_calibration"].b,
+                         run_params.use_calibration,
+                         node.name)
 
         has_calibration_data = len(node.attr["calibration_data"].s)
         if (IsQuantizationMode(run_params.precision_mode) and
+            run_params.use_calibration and 
             graph_state == GraphState.INFERENCE):
           self.assertTrue(has_calibration_data, node.name)
         else:
@@ -449,7 +460,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                                 config_no_trt, GraphState.ORIGINAL)
 
     # Run calibration if necessary.
-    if IsQuantizationMode(run_params.precision_mode):
+    if (IsQuantizationMode(run_params.precision_mode) and
+        run_params.use_calibration):
 
       calib_config = self._GetConfigProto(run_params, GraphState.CALIBRATE)
       logging.info("Running calibration graph, config:\n%s", str(calib_config))
@@ -519,18 +531,24 @@ def _AddTests(test_class):
 
   use_optimizer_options = [False, True]
   dynamic_engine_options = [False, True]
-  for (use_optimizer, precision_mode, dynamic_engine) in itertools.product(
-      use_optimizer_options, PRECISION_MODES, dynamic_engine_options):
+  use_calibration_options = [False, True]
+  opts = itertools.product(use_optimizer_options, PRECISION_MODES,
+                           dynamic_engine_options, use_calibration_options)
+  for (use_optimizer, precision_mode, dynamic_engine, use_calibration) in opts:
     if IsQuantizationMode(precision_mode):
       if use_optimizer:
         # TODO(aaroey): if use_optimizer is True we need to get the inference
         # graphdef using custom python wrapper class, which is not currently
         # supported yet.
         continue
-      if not dynamic_engine:
+      if not dynamic_engine and use_calibration:
         # TODO(aaroey): construction of static calibration engine is not
         # supported yet.
         continue
+    else:
+      if use_calibration:
+        # Don't calibrate in FP32 or FP16 mode
+        continue
 
     conversion = "OptimizerConversion" if use_optimizer else "ToolConversion"
     engine_type = ("DynamicEngine" if dynamic_engine else "StaticEngine")
@@ -539,7 +557,8 @@ def _AddTests(test_class):
         use_optimizer=use_optimizer,
         precision_mode=precision_mode,
         dynamic_engine=dynamic_engine,
-        test_name=test_name)
+        test_name=test_name,
+        use_calibration=use_calibration)
     setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params))
 
 
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 7dc62e24df..50e03467d4 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -108,6 +108,7 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "Conj",
                                           "Cos",
                                           "Cosh",
+                                          "Dequantize",
                                           "Digamma",
                                           "Div",
                                           "Elu",
@@ -119,6 +120,7 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "Exit",
                                           "Exp",
                                           "Expm1",
+                                          "FakeQuantWithMinMaxVars",
                                           "Fill",
                                           "Floor",
                                           "FloorDiv",
@@ -161,6 +163,9 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "PreventGradient",
                                           "Prod",
                                           "Polygamma",
+                                          "QuantizeAndDequantizeV2",
+                                          "QuantizeAndDequantizeV3",
+                                          "QuantizeV2",
                                           "Pow",
                                           "Real",
                                           "RealDiv",
-- 
GitLab


From abda4224e8d6b8e227a579a481e9f0ebe5fdb346 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 18 Oct 2018 15:14:38 -0700
Subject: [PATCH 0092/1554] Add FakeQuantWithMinMaxArgs

---
 .../contrib/tensorrt/convert/convert_graph.cc |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 33 ++++++++++++++-----
 .../grappler/optimizers/layout_optimizer.cc   |  1 +
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 40d03ace6f..b9c6dc7fde 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -139,6 +139,7 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node, i
     "QuantizeAndDequantizeV2",
     "QuantizeAndDequantizeV3",
     "FakeQuantWithMinMaxVars",
+    "FakeQuantWithMinMaxArgs",
   };
   if (precision_mode == INT8MODE &&
       quantize_ops.count(node->type_string())) {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 48e8a6fc51..f3358ba11d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1947,15 +1947,29 @@ tensorflow::Status ConvertQuantize(OpConverterParams* params) {
     params->outputs->push_back(inputs.at(0));
     return tensorflow::Status::OK();
   }
-  nvinfer1::ITensor* tensor = const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor());
-  // Min
-  TRT_ShapedWeights weights_min = inputs.at(1).weights();
-  auto weights_min_ptr = static_cast<float*>(const_cast<void*>(weights_min.GetValues()));
-  float min_range = weights_min_ptr[0];
-  // Max
-  TRT_ShapedWeights weights_max = inputs.at(2).weights();
-  auto weights_max_ptr = static_cast<float*>(const_cast<void*>(weights_max.GetValues()));
-  float max_range = weights_max_ptr[0];
+  float min_range = 0.0f;
+  float max_range = 0.0f;
+  if (inputs.size() == 1) {
+    // Get ranges from attributes
+    TFAttrs attrs(node_def);
+    min_range = attrs.get<float>("min");
+    max_range = attrs.get<float>("max");
+  } else if (inputs.size() == 3) {
+    // Get ranges from inputs
+    // Min
+    TRT_ShapedWeights weights_min = inputs.at(1).weights();
+    auto weights_min_ptr = static_cast<float*>(const_cast<void*>(
+        weights_min.GetValues()));
+    min_range = weights_min_ptr[0];
+    // Max
+    TRT_ShapedWeights weights_max = inputs.at(2).weights();
+    auto weights_max_ptr = static_cast<float*>(const_cast<void*>(
+        weights_max.GetValues()));
+    max_range = weights_max_ptr[0];
+  } else {
+    return tensorflow::errors::InvalidArgument(
+        "Expected 1 or 3 inputs for quantize node, at ", node_def.name());
+  }
   // Store ranges for tensor
   params->converter->ProvideQuantizationRange(tensor, min_range, max_range);
   // Sometimes, TRT may not quantize a tensor, either because it chooses to
@@ -3016,6 +3030,7 @@ void Converter::RegisterOpConverters() {
   op_registry_["QuantizeAndDequantizeV2"] = ConvertQuantize;
   op_registry_["QuantizeAndDequantizeV3"] = ConvertQuantize;
   op_registry_["FakeQuantWithMinMaxVars"] = ConvertQuantize;
+  op_registry_["FakeQuantWithMinMaxArgs"] = ConvertQuantize;
 #endif
 
   plugin_converter_ = ConvertPlugin;
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 50e03467d4..b40438e98f 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -121,6 +121,7 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "Exp",
                                           "Expm1",
                                           "FakeQuantWithMinMaxVars",
+                                          "FakeQuantWithMinMaxArgs",
                                           "Fill",
                                           "Floor",
                                           "FloorDiv",
-- 
GitLab


From 6e07831307ebad7f154052cc387da22b6169a4d1 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 23 Oct 2018 17:48:18 -0700
Subject: [PATCH 0093/1554] Add more checks on quantize node inputs.

Add ConvertQuantize tests

Fix line length formatting

Set quantization range for IConstantLayer created when weights are reshaped via PrepareTensorForShape. Since the weights are constant we can use the value of the weights to determine the range.

Add missing inference of quantization range. Allow ConvertReshape to work on weights

Fix reshape for weights

Fix reshape for weights again
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 56 +++++++++++++-
 .../contrib/tensorrt/convert/convert_nodes.h  |  4 +
 .../tensorrt/convert/convert_nodes_test.cc    | 67 ++++++++++++++++
 .../tensorrt/test/quantization_test.py        | 76 ++++++++-----------
 4 files changed, 156 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index f3358ba11d..f3403fbdc6 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -969,6 +969,32 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor,
   return tensorflow::Status::OK();
 }
 
+Status Converter::GetWeightRange(const TRT_ShapedWeights& weights,
+                                 float* out_min,
+                                 float* out_max) {
+  switch (weights.type_) {
+    case tensorflow::DataType::DT_FLOAT: {
+      auto inp = static_cast<float const*>(weights.GetValues());
+      auto result = std::minmax_element(inp, inp + weights.count());
+      *out_min = *result.first;
+      *out_max = *result.second;
+      break;
+    }
+    case tensorflow::DataType::DT_HALF: {
+      auto inp = static_cast<Eigen::half const*>(weights.GetValues());
+      auto result = std::minmax_element(inp, inp + weights.count());
+      *out_min = Eigen::half_impl::half_to_float(*result.first);
+      *out_max = Eigen::half_impl::half_to_float(*result.second);
+      break;
+    }
+    default:
+      return tensorflow::errors::Unimplemented(
+          "Data type not supported: " +
+          tensorflow::DataTypeString(weights.type_));
+  }
+  return tensorflow::Status::OK();
+}
+
 Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
                                         const nvinfer1::Dims& dims,
                                         const nvinfer1::ITensor** tensor) {
@@ -1004,6 +1030,21 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
         this->network()->addConstant(dims, input.weights().GetTrtWeights());
     TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
     *tensor = layer->getOutput(0);
+    // We need to set a quantization range for the output tensor of the
+    // IConstantLayer. Here we set the range to [min(weights), max(weights)].
+    float min_range = 0.0f;
+    float max_range = 0.0f;
+    TF_RETURN_IF_ERROR(
+        GetWeightRange(input.weights(), &min_range, &max_range));
+    // Avoid setting range to 0 because TRT will throw an error. If the weights
+    // are zero then the range doesn't matter: using 127.0f should ensure the
+    // quantized weight will be exactly zero.
+    if (min_range == 0.0f && max_range == 0.0f) {
+      min_range = -127.0f;
+      max_range = 127.0f;
+    }
+    ProvideQuantizationRange(const_cast<nvinfer1::ITensor*>(*tensor),
+        min_range, max_range);
   }
   return tensorflow::Status::OK();
 }
@@ -1555,6 +1596,8 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
         nvinfer1::DimsHW(padding[0].second, padding[1].second));
     TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name());
+    ctx.MarkQuantizationRangesAsInferrable(
+        const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
     VLOG(2) << "TENSOR after: " << DebugString(tensor->getDimensions());
@@ -1943,7 +1986,9 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
 tensorflow::Status ConvertQuantize(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.at(0).is_weights()) {
+  if (inputs.size() > 0 && inputs.at(0).is_weights()) {
+    // TensorRT will automatically quantize weights, so we will ignore ranges
+    // for weights.
     params->outputs->push_back(inputs.at(0));
     return tensorflow::Status::OK();
   }
@@ -1952,10 +1997,19 @@ tensorflow::Status ConvertQuantize(OpConverterParams* params) {
   if (inputs.size() == 1) {
     // Get ranges from attributes
     TFAttrs attrs(node_def);
+    if (attrs.count("min") == 0 || attrs.count("max") == 0) {
+      return tensorflow::errors::InvalidArgument(
+          "Min or max attribute not found for quantize, at ", node_def.name());
+    }
     min_range = attrs.get<float>("min");
     max_range = attrs.get<float>("max");
   } else if (inputs.size() == 3) {
     // Get ranges from inputs
+    if (!inputs.at(1).is_weights() || !inputs.at(2).is_weights()) {
+      return tensorflow::errors::InvalidArgument(
+          "Min and max for quantize must be weights not tensors, at ",
+          node_def.name());
+    }
     // Min
     TRT_ShapedWeights weights_min = inputs.at(1).weights();
     auto weights_min_ptr = static_cast<float*>(const_cast<void*>(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 2daca99ed9..ec9cea38d5 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -478,6 +478,10 @@ class Converter {
 
   void PropagateQuantizationRanges();
   
+  // Gets the min and max value in a TRT_ShapedWeights
+  Status GetWeightRange(const TRT_ShapedWeights& weights,
+                        float* out_min, float* out_max);
+
   // Registered op converters by op type.
   std::unordered_map<string, OpConverter> op_registry_;
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index c3a39395f3..38df6995b6 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -434,6 +434,10 @@ class ConverterTest : public ::testing::Test {
 
   int batch_size() const { return converter_->batch_size_; }
 
+  std::unordered_map<nvinfer1::ITensor*, float>* GetQuantizationRanges() {
+    return &quantization_ranges_;
+  }
+
  private:
   Logger logger_;
   // These members are ordered in a way such that the destruction order is:
@@ -1130,6 +1134,69 @@ TEST_F(OpConverterTest, ConvertMatMul) {
         EXPECT_THAT(output_data, ElementsAre(2, 3));
       }
     }
+}
+
+TEST_F(ConverterTest, ConvertQuantize) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def =
+        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {});
+    ExpectStatus(converter_.ConvertNode(node_def), error::INVALID_ARGUMENT,
+                 "Expected 1 or 3 inputs for quantize node, at my_quantize");
+  }
+  {
+    // Missing attributes, should fail
+    converter_.Reset();
+    NodeDef node_def =
+        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {"input"});
+    AddTestTensor("input", {1, 2, 3});
+    ExpectStatus(converter_.ConvertNode(node_def), error::INVALID_ARGUMENT,
+        "Min or max attribute not found for quantize, at my_quantize");
+  }
+  {
+    // All inputs are tensors, should fail
+    converter_.Reset();
+    NodeDef node_def = MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2",
+                                   {"input", "weights_min", "weights_max"});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestTensor("weights_min", {1});
+    AddTestTensor("weights_max", {1});
+    ExpectStatus(converter_.ConvertNode(node_def), error::INVALID_ARGUMENT,
+        "Min and max for quantize must be weights not tensors, at my_quantize");
+  }
+  {
+    // Ranges set via attributes, ok.
+    converter_.Reset();
+    NodeDef node_def =
+        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {"input"});
+    AddTestTensor("input", {1, 2, 3});
+    AttrValue attr_min;
+    attr_min.set_f(-6.0f);
+    AttrValue attr_max;
+    attr_max.set_f(6.0f);
+    node_def.mutable_attr()->insert({"min", attr_min});
+    node_def.mutable_attr()->insert({"max", attr_max});
+    TF_EXPECT_OK(converter_.ConvertNode(node_def));
+    TRT_TensorOrWeights output = converter_.GetTensorOrWeights("my_quantize");
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = converter_.GetQuantizationRanges();
+    EXPECT_EQ((*ranges).count(output.tensor()), 1);
+    EXPECT_EQ((*ranges)[output.tensor()], 6.0f);
+  }
+  {
+    // Ranges set via inputs, ok.
+    converter_.Reset();
+    NodeDef node_def = MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2",
+                                   {"input", "weights_min", "weights_max"});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights_min", DT_FLOAT, {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", DT_FLOAT, {1}, {6.0f}); 
+    TF_EXPECT_OK(converter_.ConvertNode(node_def));
+    TRT_TensorOrWeights output = converter_.GetTensorOrWeights("my_quantize");
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = converter_.GetQuantizationRanges();
+    EXPECT_EQ((*ranges).count(output.tensor()), 1);
+    EXPECT_EQ((*ranges)[output.tensor()], 6.0f);
   }
 }
 
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
index a6934bf490..f32b15b038 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -43,44 +43,13 @@ def build_graph(input_name, input_dims, output_name,
   with g.as_default():
     x = array_ops.placeholder(
         dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
-    x = quantize(x, 100)
-    filt1 = constant_op.constant(
-          0.3, shape=(3, 3, 1, 32), dtype=dtype, name='filt1')
-    x = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='VALID')
-    bias1 = constant_op.constant(0.3, shape=(32,), name="bias1", dtype=dtype)
-    x = nn.bias_add(x, bias1)
-
-    x = quantize(x, 6)
-    x = nn.relu6(x)
-    filt2 = constant_op.constant(
-          0.3, shape=(3, 3, 32, 64), dtype=dtype, name='filt2')
-    x = nn.conv2d(x, filt2, strides=[1, 1, 1, 1], padding='VALID')
-    bias2 = constant_op.constant(0.3, shape=(64,), name="bias2", dtype=dtype)
-    x = nn.bias_add(x, bias2)
-
-    x = quantize(x, 6)
-    x = nn.relu6(x)
-    x = math_ops.reduce_mean(x, [1, 2])
-    x = quantize(x, 6)
-    # FC1
-    fc_w1 = constant_op.constant(
-          0.3, shape=(64, 512), dtype=dtype, name='fc_w1')
-    x = math_ops.matmul(x, fc_w1)
-    x = quantize(x, 6)
-    fc_b1 = constant_op.constant(
-          0.3, shape=(512,), dtype=dtype, name='fc_b1')
-    x = nn.bias_add(x, fc_b1)
-    x = quantize(x, 6)
-    x = nn.relu6(x)
-    # FC2
-    fc_w2 = constant_op.constant(
-          0.3, shape=(512, 10), dtype=dtype, name='fc_w2')
-    x = math_ops.matmul(x, fc_w2)
-    x = quantize(x, 25)
-    fc_b2 = constant_op.constant(
-          0.3, shape=(10,), dtype=dtype, name='fc_b2')
-    x = nn.bias_add(x, fc_b2)
-    x = quantize(x, 25)
+    x = quantize(x, 10.0)
+    x = x + 5
+    x = quantize(x, 15.0)
+    x = x - 5
+    x = quantize(x, 10.0)
+    x = x * 0.1
+    x = quantize(x, 1.0)
     x = array_ops.identity(x, name=output_name)
   return g
 
@@ -89,7 +58,7 @@ class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
   def GetParams(self):
     """Create a graph containing single segment with no quantization ranges."""
     input_name = "input"
-    input_dims = [100, 28, 28, 1]
+    input_dims = [100, 100]
     output_name = "output"
     g = build_graph(input_name, input_dims, output_name,
                     add_quantization_nodes=False)
@@ -98,7 +67,7 @@ class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[(100, 10)])
+        expected_output_dims=[(100, 100)])
 
   def ShouldRunTest(self, run_params):
     return (run_params.precision_mode == "INT8" and
@@ -116,7 +85,7 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
   def GetParams(self):
     """Create a graph containing single segment with no quantization ranges."""
     input_name = "input"
-    input_dims = [100, 28, 28, 1]
+    input_dims = [100, 100]
     output_name = "output"
     g = build_graph(input_name, input_dims, output_name,
                     add_quantization_nodes=True)
@@ -125,7 +94,7 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[(100, 10)])
+        expected_output_dims=[(100, 100)])
 
   def ShouldRunTest(self, run_params):
     return (run_params.precision_mode == "INT8" and
@@ -135,12 +104,20 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ["my_trt_op_0"]
 
+  def ExpectedAbsoluteTolerance(self, run_params):
+    """The absolute tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
+  def ExpectedRelativeTolerance(self, run_params):
+    """The relative tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
 class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
     """Create a graph containing single segment with no quantization ranges."""
     input_name = "input"
-    input_dims = [100, 28, 28, 1]
+    input_dims = [100, 100]
     output_name = "output"
     g = build_graph(input_name, input_dims, output_name,
                     add_quantization_nodes=True)
@@ -149,7 +126,7 @@ class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[(100, 10)])
+        expected_output_dims=[(100, 100)])
 
   def ShouldRunTest(self, run_params):
     return (run_params.precision_mode == "FP32" or
@@ -157,8 +134,15 @@ class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3",
-            "my_trt_op_4", "my_trt_op_5", "my_trt_op_6"]
+    return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2"]
+  
+  def ExpectedAbsoluteTolerance(self, run_params):
+    """The absolute tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
+  def ExpectedRelativeTolerance(self, run_params):
+    """The relative tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 9dda56272f87cc1ebd66eeaa6e8f3cf0421fe4fe Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 2 Nov 2018 14:25:37 -0700
Subject: [PATCH 0094/1554] Fix merge conflicts

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index f3403fbdc6..456840e537 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1596,7 +1596,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
         nvinfer1::DimsHW(padding[0].second, padding[1].second));
     TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name());
-    ctx.MarkQuantizationRangesAsInferrable(
+    params->converter->MarkQuantizationRangesAsInferrable(
         const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
@@ -2025,7 +2025,10 @@ tensorflow::Status ConvertQuantize(OpConverterParams* params) {
         "Expected 1 or 3 inputs for quantize node, at ", node_def.name());
   }
   // Store ranges for tensor
-  params->converter->ProvideQuantizationRange(tensor, min_range, max_range);
+  params->converter->ProvideQuantizationRange(
+      const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor()),
+      min_range,
+      max_range);
   // Sometimes, TRT may not quantize a tensor, either because it chooses to
   // execute a higher precision kernel or because of op fusion. In these cases,
   // accuracy will suffer if the model was trained to expect quantization at
-- 
GitLab


From 08159b275a5e66e5713ab25befcd7b2034d631bb Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 2 Nov 2018 15:39:14 -0700
Subject: [PATCH 0095/1554] Improve use_calibration arg documentation

---
 .../contrib/tensorrt/python/trt_convert.py       | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 98a647dc57..95f47c9014 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -100,8 +100,11 @@ def tensorrt_rewriter_config(rewriter_config=None,
       if set to True, a calibration graph will be created to calibrate the
       missing ranges. The calibration graph must be converted to an inference
       graph using calib_graph_to_infer_graph() after running calibration.
-      if set to False, quantization ranges will be expected for every tensor in
-      the graph. If a range is missing, an error will occur.
+      if set to False, quantization nodes will be expected for every tensor in
+      the graph (exlcuding those which will be fused). If a range is missing,
+      an error will occur. Please note that accuracy may be negatively affected
+      if there is a mismatch between which tensors TRT quantizes and which
+      tensors were trained with fake quantization.
 
   Returns:
     A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
@@ -176,7 +179,7 @@ def create_inference_graph(input_graph_def,
     max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
       engine can use at execution time. This corresponds to the 'workspaceSize'
       parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
-    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
+    precision_mode: one of TrtPrecisionMode.supported_precision_modes(). 
     minimum_segment_size: the minimum number of nodes required for a subgraph to
       be replaced by TRTEngineOp.
     is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
@@ -197,8 +200,11 @@ def create_inference_graph(input_graph_def,
       if set to True, a calibration graph will be created to calibrate the
       missing ranges. The calibration graph must be converted to an inference
       graph using calib_graph_to_infer_graph() after running calibration.
-      if set to False, quantization ranges will be expected for every tensor in
-      the graph. If a range is missing, an error will occur.
+      if set to False, quantization nodes will be expected for every tensor in
+      the graph (exlcuding those which will be fused). If a range is missing,
+      an error will occur. Please note that accuracy may be negatively affected
+      if there is a mismatch between which tensors TRT quantizes and which
+      tensors were trained with fake quantization.
     input_saved_model_dir: the directory to load the SavedModel which contains
       the input graph to transforms. Used only when input_graph_def is None.
     input_saved_model_tags: list of tags to load the SavedModel.
-- 
GitLab


From b11c33a5d1792ba7e4ff0a687d43c0322b3458d0 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Mon, 5 Nov 2018 11:37:17 -0800
Subject: [PATCH 0096/1554] Update the code style for line break

---
 tensorflow/python/keras/layers/wrappers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index 82c96e386a..c4feab651c 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -441,8 +441,8 @@ class Bidirectional(Wrapper):
 
   @tf_utils.shape_type_conversion
   def compute_output_shape(self, input_shape):
-    forward_layer_output_shape \
-      = self.forward_layer.compute_output_shape(input_shape)
+    forward_layer_output_shape = self.forward_layer.compute_output_shape(
+        input_shape)
     if getattr(forward_layer_output_shape, 'as_list', None) is None:
       output_shape = tuple(forward_layer_output_shape)
     else:
-- 
GitLab


From b0d15134f110ce380a0e769c4f415d41fbea2677 Mon Sep 17 00:00:00 2001
From: joaak <29533036+joaak@users.noreply.github.com>
Date: Mon, 5 Nov 2018 15:51:37 -0500
Subject: [PATCH 0097/1554] update tokenizer code to remove bug

---
 .../image_captioning_with_attention.ipynb     | 2301 +++++++++--------
 1 file changed, 1155 insertions(+), 1146 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
index 3acecd283c..09ea021c44 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
@@ -1,1184 +1,1193 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "K2s1A9eLRPEj"
+   },
+   "source": [
+    "##### Copyright 2018 The TensorFlow Authors.\n",
+    "\n",
+    "Licensed under the Apache License, Version 2.0 (the \"License\").\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "Cffg2i257iMS"
+   },
+   "source": [
+    "# Image Captioning with Attention\n",
+    "\n",
+    "<table class=\"tfo-notebook-buttons\" align=\"left\"><td>\n",
+    "<a target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb\">\n",
+    "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>  \n",
+    "</td><td>\n",
+    "<a target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "QASbY_HGo4Lq"
+   },
+   "source": [
+    "Image captioning is the task of generating a caption for an image. Given an image like this:\n",
+    "\n",
+    "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n",
+    "\n",
+    "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n",
+    "\n",
+    "Our goal is to generate a caption, such as \"a surfer riding on a wave\". Here, we'll use an attention-based model. This enables us to see which parts of the image the model focuses on as it generates a caption.\n",
+    "\n",
+    "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n",
+    "\n",
+    "This model architecture below is similar to [Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044). \n",
+    "\n",
+    "The code uses [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager), which you can learn more about in the linked guides.\n",
+    "\n",
+    "This notebook is an end-to-end example. If you run it, it will download the  [MS-COCO](http://cocodataset.org/#home) dataset, preprocess and cache a subset of the images using Inception V3, train an encoder-decoder model, and use it to generate captions on new images.\n",
+    "\n",
+    "The code requires TensorFlow version >=1.9. If you're running this in [Colab]()\n",
+    "\n",
+    "In this example, we're training on a relatively small amount of data as an example. On a single P100 GPU, this example will take about ~2 hours to train. We train on the first 30,000 captions (corresponding to about ~20,000 images depending on shuffling, as there are multiple captions per image in the dataset)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
     "colab": {
-      "name": "image_captioning_with_attention.ipynb",
-      "version": "0.3.2",
-      "views": {},
-      "default_view": {},
-      "provenance": [
-        {
-          "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg",
-          "timestamp": 1530222436922
-        }
-      ],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    },
-    "accelerator": "GPU"
+    "colab_type": "code",
+    "id": "U8l4RJ0XRPEm"
+   },
+   "outputs": [],
+   "source": [
+    "# Import TensorFlow and enable eager execution\n",
+    "# This code requires TensorFlow version >=1.9\n",
+    "import tensorflow as tf\n",
+    "tf.enable_eager_execution()\n",
+    "\n",
+    "# We'll generate plots of attention in order to see which parts of an image\n",
+    "# our model focuses on during captioning\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Scikit-learn includes many helpful utilities\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.utils import shuffle\n",
+    "\n",
+    "import re\n",
+    "import numpy as np\n",
+    "import os\n",
+    "import time\n",
+    "import json\n",
+    "from glob import glob\n",
+    "from PIL import Image\n",
+    "import pickle"
+   ]
   },
-  "cells": [
-    {
-      "metadata": {
-        "id": "K2s1A9eLRPEj",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "##### Copyright 2018 The TensorFlow Authors.\n",
-        "\n",
-        "Licensed under the Apache License, Version 2.0 (the \"License\").\n"
-      ]
-    },
-    {
-      "metadata": {
-        "id": "Cffg2i257iMS",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "# Image Captioning with Attention\n",
-        "\n",
-        "<table class=\"tfo-notebook-buttons\" align=\"left\"><td>\n",
-        "<a target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb\">\n",
-        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>  \n",
-        "</td><td>\n",
-        "<a target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
-      ]
-    },
-    {
-      "metadata": {
-        "id": "QASbY_HGo4Lq",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "Image captioning is the task of generating a caption for an image. Given an image like this:\n",
-        "\n",
-        "![Man Surfing](https://tensorflow.org/images/surf.jpg) \n",
-        "\n",
-        "[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg), License: Public Domain\n",
-        "\n",
-        "Our goal is to generate a caption, such as \"a surfer riding on a wave\". Here, we'll use an attention-based model. This enables us to see which parts of the image the model focuses on as it generates a caption.\n",
-        "\n",
-        "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n",
-        "\n",
-        "This model architecture below is similar to [Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044). \n",
-        "\n",
-        "The code uses [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager), which you can learn more about in the linked guides.\n",
-        "\n",
-        "This notebook is an end-to-end example. If you run it, it will download the  [MS-COCO](http://cocodataset.org/#home) dataset, preprocess and cache a subset of the images using Inception V3, train an encoder-decoder model, and use it to generate captions on new images.\n",
-        "\n",
-        "The code requires TensorFlow version >=1.9. If you're running this in [Colab]()\n",
-        "\n",
-        "In this example, we're training on a relatively small amount of data as an example. On a single P100 GPU, this example will take about ~2 hours to train. We train on the first 30,000 captions (corresponding to about ~20,000 images depending on shuffling, as there are multiple captions per image in the dataset)\n"
-      ]
-    },
-    {
-      "metadata": {
-        "id": "U8l4RJ0XRPEm",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# Import TensorFlow and enable eager execution\n",
-        "# This code requires TensorFlow version >=1.9\n",
-        "import tensorflow as tf\n",
-        "tf.enable_eager_execution()\n",
-        "\n",
-        "# We'll generate plots of attention in order to see which parts of an image\n",
-        "# our model focuses on during captioning\n",
-        "import matplotlib.pyplot as plt\n",
-        "\n",
-        "# Scikit-learn includes many helpful utilities\n",
-        "from sklearn.model_selection import train_test_split\n",
-        "from sklearn.utils import shuffle\n",
-        "\n",
-        "import re\n",
-        "import numpy as np\n",
-        "import os\n",
-        "import time\n",
-        "import json\n",
-        "from glob import glob\n",
-        "from PIL import Image\n",
-        "import pickle"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "b6qbGw8MRPE5",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Download and prepare the MS-COCO dataset\n",
-        "\n",
-        "We will use the [MS-COCO dataset](http://cocodataset.org/#home) to train our model. This dataset contains >82,000 images, each of which has been annotated with at least 5 different captions. The code below will download and extract the dataset automatically.  \n",
-        "\n",
-        "**Caution: large download ahead**. We'll use the training set, it's a 13GB file."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "krQuPYTtRPE7",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "annotation_zip = tf.keras.utils.get_file('captions.zip', \n",
-        "                                          cache_subdir=os.path.abspath('.'),\n",
-        "                                          origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n",
-        "                                          extract = True)\n",
-        "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n",
-        "\n",
-        "name_of_zip = 'train2014.zip'\n",
-        "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n",
-        "  image_zip = tf.keras.utils.get_file(name_of_zip, \n",
-        "                                      cache_subdir=os.path.abspath('.'),\n",
-        "                                      origin = 'http://images.cocodataset.org/zips/train2014.zip',\n",
-        "                                      extract = True)\n",
-        "  PATH = os.path.dirname(image_zip)+'/train2014/'\n",
-        "else:\n",
-        "  PATH = os.path.abspath('.')+'/train2014/'"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "aANEzb5WwSzg",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Optionally, limit the size of the training set for faster training\n",
-        "For this example, we'll select a subset of 30,000 captions and use these and the corresponding images to train our model. As always, captioning quality will improve if you choose to use more data."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "4G3b8x8_RPFD",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# read the json file\n",
-        "with open(annotation_file, 'r') as f:\n",
-        "    annotations = json.load(f)\n",
-        "\n",
-        "# storing the captions and the image name in vectors\n",
-        "all_captions = []\n",
-        "all_img_name_vector = []\n",
-        "\n",
-        "for annot in annotations['annotations']:\n",
-        "    caption = '<start> ' + annot['caption'] + ' <end>'\n",
-        "    image_id = annot['image_id']\n",
-        "    full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n",
-        "    \n",
-        "    all_img_name_vector.append(full_coco_image_path)\n",
-        "    all_captions.append(caption)\n",
-        "\n",
-        "# shuffling the captions and image_names together\n",
-        "# setting a random state\n",
-        "train_captions, img_name_vector = shuffle(all_captions,\n",
-        "                                          all_img_name_vector,\n",
-        "                                          random_state=1)\n",
-        "\n",
-        "# selecting the first 30000 captions from the shuffled set\n",
-        "num_examples = 30000\n",
-        "train_captions = train_captions[:num_examples]\n",
-        "img_name_vector = img_name_vector[:num_examples]"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "mPBMgK34RPFL",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "len(train_captions), len(all_captions)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "8cSW4u-ORPFQ",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Preprocess the images using InceptionV3\n",
-        "Next, we will use InceptionV3 (pretrained on Imagenet) to classify each image. We will extract features from the last convolutional layer. \n",
-        "\n",
-        "First, we will need to convert the images into the format inceptionV3 expects by:\n",
-        "* Resizing the image to (299, 299)\n",
-        "* Using the [preprocess_input](https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3/preprocess_input) method to place the pixels in the range of -1 to 1 (to match the format of the images used to train InceptionV3)."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "zXR0217aRPFR",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "def load_image(image_path):\n",
-        "    img = tf.read_file(image_path)\n",
-        "    img = tf.image.decode_jpeg(img, channels=3)\n",
-        "    img = tf.image.resize_images(img, (299, 299))\n",
-        "    img = tf.keras.applications.inception_v3.preprocess_input(img)\n",
-        "    return img, image_path"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "MDvIu4sXRPFV",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Initialize InceptionV3 and load the pretrained Imagenet weights\n",
-        "\n",
-        "To do so, we'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture. \n",
-        "* Each image is forwarded through the network and the vector that we get at the end is stored in a dictionary (image_name --> feature_vector). \n",
-        "* We use the last convolutional layer because we are using attention in this example. The shape of the output of this layer is ```8x8x2048```. \n",
-        "* We avoid doing this during training so it does not become a bottleneck. \n",
-        "* After all the images are passed through the network, we pickle the dictionary and save it to disk."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "RD3vW4SsRPFW",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "image_model = tf.keras.applications.InceptionV3(include_top=False, \n",
-        "                                                weights='imagenet')\n",
-        "new_input = image_model.input\n",
-        "hidden_layer = image_model.layers[-1].output\n",
-        "\n",
-        "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "rERqlR3WRPGO",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Caching the features extracted from InceptionV3\n",
-        "\n",
-        "We will pre-process each image with InceptionV3 and cache the output to disk. Caching the output in RAM would be faster but memory intensive, requiring 8 \\* 8 \\* 2048 floats per image. At the time of writing, this would exceed the memory limitations of Colab (although these may change, an instance appears to have about 12GB of memory currently). \n",
-        "\n",
-        "Performance could be improved with a more sophisticated caching strategy (e.g., by sharding the images to reduce random access disk I/O) at the cost of more code.\n",
-        "\n",
-        "This will take about 10 minutes to run in Colab with a GPU. If you'd like to see a progress bar, you could: install [tqdm](https://github.com/tqdm/tqdm) (```!pip install tqdm```), then change this line: \n",
-        "\n",
-        "```for img, path in image_dataset:``` \n",
-        "\n",
-        "to:\n",
-        "\n",
-        "```for img, path in tqdm(image_dataset):```."
-      ]
-    },
-    {
-      "metadata": {
-        "id": "Dx_fvbVgRPGQ",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# getting the unique images\n",
-        "encode_train = sorted(set(img_name_vector))\n",
-        "\n",
-        "# feel free to change the batch_size according to your system configuration\n",
-        "image_dataset = tf.data.Dataset.from_tensor_slices(\n",
-        "                                encode_train).map(load_image).batch(16)\n",
-        "\n",
-        "for img, path in image_dataset:\n",
-        "  batch_features = image_features_extract_model(img)\n",
-        "  batch_features = tf.reshape(batch_features, \n",
-        "                              (batch_features.shape[0], -1, batch_features.shape[3]))\n",
-        "\n",
-        "  for bf, p in zip(batch_features, path):\n",
-        "    path_of_feature = p.numpy().decode(\"utf-8\")\n",
-        "    np.save(path_of_feature, bf.numpy())"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "metadata": {
-        "id": "nyqH3zFwRPFi",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Preprocess and tokenize the captions\n",
-        "\n",
-        "* First, we'll tokenize the captions (e.g., by splitting on spaces). This will give us a  vocabulary of all the unique words in the data (e.g., \"surfing\", \"football\", etc).\n",
-        "* Next, we'll limit the vocabulary size to the top 5,000 words to save memory. We'll replace all other words with the token \"UNK\" (for unknown).\n",
-        "* Finally, we create a word --> index mapping and vice-versa.\n",
-        "* We will then pad all sequences to the be same length as the longest one. "
-      ]
-    },
-    {
-      "metadata": {
-        "id": "HZfK8RhQRPFj",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# This will find the maximum length of any caption in our dataset\n",
-        "def calc_max_length(tensor):\n",
-        "    return max(len(t) for t in tensor)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "b6qbGw8MRPE5"
+   },
+   "source": [
+    "## Download and prepare the MS-COCO dataset\n",
+    "\n",
+    "We will use the [MS-COCO dataset](http://cocodataset.org/#home) to train our model. This dataset contains >82,000 images, each of which has been annotated with at least 5 different captions. The code below will download and extract the dataset automatically.  \n",
+    "\n",
+    "**Caution: large download ahead**. We'll use the training set, it's a 13GB file."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "oJGE34aiRPFo",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# The steps above is a general process of dealing with text processing\n",
-        "\n",
-        "# choosing the top 5000 words from the vocabulary\n",
-        "top_k = 5000\n",
-        "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k, \n",
-        "                                                  oov_token=\"<unk>\", \n",
-        "                                                  filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')\n",
-        "tokenizer.fit_on_texts(train_captions)\n",
-        "train_seqs = tokenizer.texts_to_sequences(train_captions)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "krQuPYTtRPE7"
+   },
+   "outputs": [],
+   "source": [
+    "annotation_zip = tf.keras.utils.get_file('captions.zip', \n",
+    "                                          cache_subdir=os.path.abspath('.'),\n",
+    "                                          origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n",
+    "                                          extract = True)\n",
+    "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n",
+    "\n",
+    "name_of_zip = 'train2014.zip'\n",
+    "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n",
+    "  image_zip = tf.keras.utils.get_file(name_of_zip, \n",
+    "                                      cache_subdir=os.path.abspath('.'),\n",
+    "                                      origin = 'http://images.cocodataset.org/zips/train2014.zip',\n",
+    "                                      extract = True)\n",
+    "  PATH = os.path.dirname(image_zip)+'/train2014/'\n",
+    "else:\n",
+    "  PATH = os.path.abspath('.')+'/train2014/'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "aANEzb5WwSzg"
+   },
+   "source": [
+    "## Optionally, limit the size of the training set for faster training\n",
+    "For this example, we'll select a subset of 30,000 captions and use these and the corresponding images to train our model. As always, captioning quality will improve if you choose to use more data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "8Q44tNQVRPFt",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "tokenizer.word_index = {key:value for key, value in tokenizer.word_index.items() if value <= top_k}\n",
-        "# putting <unk> token in the word2idx dictionary\n",
-        "tokenizer.word_index[tokenizer.oov_token] = top_k + 1\n",
-        "tokenizer.word_index['<pad>'] = 0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "4G3b8x8_RPFD"
+   },
+   "outputs": [],
+   "source": [
+    "# read the json file\n",
+    "with open(annotation_file, 'r') as f:\n",
+    "    annotations = json.load(f)\n",
+    "\n",
+    "# storing the captions and the image name in vectors\n",
+    "all_captions = []\n",
+    "all_img_name_vector = []\n",
+    "\n",
+    "for annot in annotations['annotations']:\n",
+    "    caption = '<start> ' + annot['caption'] + ' <end>'\n",
+    "    image_id = annot['image_id']\n",
+    "    full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n",
+    "    \n",
+    "    all_img_name_vector.append(full_coco_image_path)\n",
+    "    all_captions.append(caption)\n",
+    "\n",
+    "# shuffling the captions and image_names together\n",
+    "# setting a random state\n",
+    "train_captions, img_name_vector = shuffle(all_captions,\n",
+    "                                          all_img_name_vector,\n",
+    "                                          random_state=1)\n",
+    "\n",
+    "# selecting the first 30000 captions from the shuffled set\n",
+    "num_examples = 30000\n",
+    "train_captions = train_captions[:num_examples]\n",
+    "img_name_vector = img_name_vector[:num_examples]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "0fpJb5ojRPFv",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# creating the tokenized vectors\n",
-        "train_seqs = tokenizer.texts_to_sequences(train_captions)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "mPBMgK34RPFL"
+   },
+   "outputs": [],
+   "source": [
+    "len(train_captions), len(all_captions)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "8cSW4u-ORPFQ"
+   },
+   "source": [
+    "## Preprocess the images using InceptionV3\n",
+    "Next, we will use InceptionV3 (pretrained on Imagenet) to classify each image. We will extract features from the last convolutional layer. \n",
+    "\n",
+    "First, we will need to convert the images into the format inceptionV3 expects by:\n",
+    "* Resizing the image to (299, 299)\n",
+    "* Using the [preprocess_input](https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3/preprocess_input) method to place the pixels in the range of -1 to 1 (to match the format of the images used to train InceptionV3)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "olQArbgbRPF1",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# creating a reverse mapping (index -> word)\n",
-        "index_word = {value:key for key, value in tokenizer.word_index.items()}"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "zXR0217aRPFR"
+   },
+   "outputs": [],
+   "source": [
+    "def load_image(image_path):\n",
+    "    img = tf.read_file(image_path)\n",
+    "    img = tf.image.decode_jpeg(img, channels=3)\n",
+    "    img = tf.image.resize_images(img, (299, 299))\n",
+    "    img = tf.keras.applications.inception_v3.preprocess_input(img)\n",
+    "    return img, image_path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "MDvIu4sXRPFV"
+   },
+   "source": [
+    "## Initialize InceptionV3 and load the pretrained Imagenet weights\n",
+    "\n",
+    "To do so, we'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture. \n",
+    "* Each image is forwarded through the network and the vector that we get at the end is stored in a dictionary (image_name --> feature_vector). \n",
+    "* We use the last convolutional layer because we are using attention in this example. The shape of the output of this layer is ```8x8x2048```. \n",
+    "* We avoid doing this during training so it does not become a bottleneck. \n",
+    "* After all the images are passed through the network, we pickle the dictionary and save it to disk."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "AidglIZVRPF4",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# padding each vector to the max_length of the captions\n",
-        "# if the max_length parameter is not provided, pad_sequences calculates that automatically\n",
-        "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "RD3vW4SsRPFW"
+   },
+   "outputs": [],
+   "source": [
+    "image_model = tf.keras.applications.InceptionV3(include_top=False, \n",
+    "                                                weights='imagenet')\n",
+    "new_input = image_model.input\n",
+    "hidden_layer = image_model.layers[-1].output\n",
+    "\n",
+    "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "rERqlR3WRPGO"
+   },
+   "source": [
+    "## Caching the features extracted from InceptionV3\n",
+    "\n",
+    "We will pre-process each image with InceptionV3 and cache the output to disk. Caching the output in RAM would be faster but memory intensive, requiring 8 \\* 8 \\* 2048 floats per image. At the time of writing, this would exceed the memory limitations of Colab (although these may change, an instance appears to have about 12GB of memory currently). \n",
+    "\n",
+    "Performance could be improved with a more sophisticated caching strategy (e.g., by sharding the images to reduce random access disk I/O) at the cost of more code.\n",
+    "\n",
+    "This will take about 10 minutes to run in Colab with a GPU. If you'd like to see a progress bar, you could: install [tqdm](https://github.com/tqdm/tqdm) (```!pip install tqdm```), then change this line: \n",
+    "\n",
+    "```for img, path in image_dataset:``` \n",
+    "\n",
+    "to:\n",
+    "\n",
+    "```for img, path in tqdm(image_dataset):```."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "gL0wkttkRPGA",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# calculating the max_length \n",
-        "# used to store the attention weights\n",
-        "max_length = calc_max_length(train_seqs)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "Dx_fvbVgRPGQ"
+   },
+   "outputs": [],
+   "source": [
+    "# getting the unique images\n",
+    "encode_train = sorted(set(img_name_vector))\n",
+    "\n",
+    "# feel free to change the batch_size according to your system configuration\n",
+    "image_dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "                                encode_train).map(load_image).batch(16)\n",
+    "\n",
+    "for img, path in image_dataset:\n",
+    "  batch_features = image_features_extract_model(img)\n",
+    "  batch_features = tf.reshape(batch_features, \n",
+    "                              (batch_features.shape[0], -1, batch_features.shape[3]))\n",
+    "\n",
+    "  for bf, p in zip(batch_features, path):\n",
+    "    path_of_feature = p.numpy().decode(\"utf-8\")\n",
+    "    np.save(path_of_feature, bf.numpy())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "nyqH3zFwRPFi"
+   },
+   "source": [
+    "## Preprocess and tokenize the captions\n",
+    "\n",
+    "* First, we'll tokenize the captions (e.g., by splitting on spaces). This will give us a  vocabulary of all the unique words in the data (e.g., \"surfing\", \"football\", etc).\n",
+    "* Next, we'll limit the vocabulary size to the top 5,000 words to save memory. We'll replace all other words with the token \"UNK\" (for unknown).\n",
+    "* Finally, we create a word --> index mapping and vice-versa.\n",
+    "* We will then pad all sequences to the be same length as the longest one. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "M3CD75nDpvTI",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Split the data into training and testing"
-      ]
+    "colab_type": "code",
+    "id": "HZfK8RhQRPFj"
+   },
+   "outputs": [],
+   "source": [
+    "# This will find the maximum length of any caption in our dataset\n",
+    "def calc_max_length(tensor):\n",
+    "    return max(len(t) for t in tensor)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "iS7DDMszRPGF",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# Create training and validation sets using 80-20 split\n",
-        "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector, \n",
-        "                                                                    cap_vector, \n",
-        "                                                                    test_size=0.2, \n",
-        "                                                                    random_state=0)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "oJGE34aiRPFo"
+   },
+   "outputs": [],
+   "source": [
+    "# The steps above is a general process of dealing with text processing\n",
+    "\n",
+    "# choosing the top 5000 words from the vocabulary\n",
+    "top_k = 5000\n",
+    "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k, \n",
+    "                                                  oov_token=\"<unk>\", \n",
+    "                                                  filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')\n",
+    "tokenizer.fit_on_texts(train_captions)\n",
+    "train_seqs = tokenizer.texts_to_sequences(train_captions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "XmViPkRFRPGH",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "8Q44tNQVRPFt"
+   },
+   "outputs": [],
+   "source": [
+    "tokenizer.word_index['<pad>'] = 0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "uEWM9xrYcg45",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Our images and captions are ready! Next, let's create a tf.data dataset to use for training our model.\n",
-        "\n"
-      ]
+    "colab_type": "code",
+    "id": "0fpJb5ojRPFv"
+   },
+   "outputs": [],
+   "source": [
+    "# creating the tokenized vectors\n",
+    "train_seqs = tokenizer.texts_to_sequences(train_captions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "Q3TnZ1ToRPGV",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# feel free to change these parameters according to your system's configuration\n",
-        "\n",
-        "BATCH_SIZE = 64\n",
-        "BUFFER_SIZE = 1000\n",
-        "embedding_dim = 256\n",
-        "units = 512\n",
-        "vocab_size = len(tokenizer.word_index)\n",
-        "# shape of the vector extracted from InceptionV3 is (64, 2048)\n",
-        "# these two variables represent that\n",
-        "features_shape = 2048\n",
-        "attention_features_shape = 64"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "olQArbgbRPF1"
+   },
+   "outputs": [],
+   "source": [
+    "# creating a reverse mapping (index -> word)\n",
+    "index_word = {value:key for key, value in tokenizer.word_index.items()}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "SmZS2N0bXG3T",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# loading the numpy files \n",
-        "def map_func(img_name, cap):\n",
-        "    img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n",
-        "    return img_tensor, cap"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "AidglIZVRPF4"
+   },
+   "outputs": [],
+   "source": [
+    "# padding each vector to the max_length of the captions\n",
+    "# if the max_length parameter is not provided, pad_sequences calculates that automatically\n",
+    "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "FDF_Nm3tRPGZ",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n",
-        "\n",
-        "# using map to load the numpy files in parallel\n",
-        "# NOTE: Be sure to set num_parallel_calls to the number of CPU cores you have\n",
-        "# https://www.tensorflow.org/api_docs/python/tf/py_func\n",
-        "dataset = dataset.map(lambda item1, item2: tf.py_func(\n",
-        "          map_func, [item1, item2], [tf.float32, tf.int32]), num_parallel_calls=8)\n",
-        "\n",
-        "# shuffling and batching\n",
-        "dataset = dataset.shuffle(BUFFER_SIZE)\n",
-        "# https://www.tensorflow.org/api_docs/python/tf/contrib/data/batch_and_drop_remainder\n",
-        "dataset = dataset.batch(BATCH_SIZE)\n",
-        "dataset = dataset.prefetch(1)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "gL0wkttkRPGA"
+   },
+   "outputs": [],
+   "source": [
+    "# calculating the max_length \n",
+    "# used to store the attention weights\n",
+    "max_length = calc_max_length(train_seqs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "M3CD75nDpvTI"
+   },
+   "source": [
+    "## Split the data into training and testing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "nrvoDphgRPGd",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Model\n",
-        "\n",
-        "Fun fact, the decoder below is identical to the one in the example for [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n",
-        "\n",
-        "The model architecture is inspired by the [Show, Attend and Tell](https://arxiv.org/pdf/1502.03044.pdf) paper.\n",
-        "\n",
-        "* In this example, we extract the features from the lower convolutional layer of InceptionV3 giving us a vector of shape (8, 8, 2048). \n",
-        "* We squash that to a shape of (64, 2048).\n",
-        "* This vector is then passed through the CNN Encoder(which consists of a single Fully connected layer).\n",
-        "* The RNN(here GRU) attends over the image to predict the next word."
-      ]
+    "colab_type": "code",
+    "id": "iS7DDMszRPGF"
+   },
+   "outputs": [],
+   "source": [
+    "# Create training and validation sets using 80-20 split\n",
+    "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector, \n",
+    "                                                                    cap_vector, \n",
+    "                                                                    test_size=0.2, \n",
+    "                                                                    random_state=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "AAppCGLKRPGd",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "def gru(units):\n",
-        "  # If you have a GPU, we recommend using the CuDNNGRU layer (it provides a \n",
-        "  # significant speedup).\n",
-        "  if tf.test.is_gpu_available():\n",
-        "    return tf.keras.layers.CuDNNGRU(units, \n",
-        "                                    return_sequences=True, \n",
-        "                                    return_state=True, \n",
-        "                                    recurrent_initializer='glorot_uniform')\n",
-        "  else:\n",
-        "    return tf.keras.layers.GRU(units, \n",
-        "                               return_sequences=True, \n",
-        "                               return_state=True, \n",
-        "                               recurrent_activation='sigmoid', \n",
-        "                               recurrent_initializer='glorot_uniform')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "XmViPkRFRPGH"
+   },
+   "outputs": [],
+   "source": [
+    "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "uEWM9xrYcg45"
+   },
+   "source": [
+    "## Our images and captions are ready! Next, let's create a tf.data dataset to use for training our model.\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "ja2LFTMSdeV3",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "class BahdanauAttention(tf.keras.Model):\n",
-        "  def __init__(self, units):\n",
-        "    super(BahdanauAttention, self).__init__()\n",
-        "    self.W1 = tf.keras.layers.Dense(units)\n",
-        "    self.W2 = tf.keras.layers.Dense(units)\n",
-        "    self.V = tf.keras.layers.Dense(1)\n",
-        "  \n",
-        "  def call(self, features, hidden):\n",
-        "    # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n",
-        "    \n",
-        "    # hidden shape == (batch_size, hidden_size)\n",
-        "    # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n",
-        "    hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
-        "    \n",
-        "    # score shape == (batch_size, 64, hidden_size)\n",
-        "    score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n",
-        "    \n",
-        "    # attention_weights shape == (batch_size, 64, 1)\n",
-        "    # we get 1 at the last axis because we are applying score to self.V\n",
-        "    attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
-        "    \n",
-        "    # context_vector shape after sum == (batch_size, hidden_size)\n",
-        "    context_vector = attention_weights * features\n",
-        "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
-        "    \n",
-        "    return context_vector, attention_weights"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "Q3TnZ1ToRPGV"
+   },
+   "outputs": [],
+   "source": [
+    "# feel free to change these parameters according to your system's configuration\n",
+    "\n",
+    "BATCH_SIZE = 64\n",
+    "BUFFER_SIZE = 1000\n",
+    "embedding_dim = 256\n",
+    "units = 512\n",
+    "vocab_size = len(tokenizer.word_index)\n",
+    "# shape of the vector extracted from InceptionV3 is (64, 2048)\n",
+    "# these two variables represent that\n",
+    "features_shape = 2048\n",
+    "attention_features_shape = 64"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "AZ7R1RxHRPGf",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "class CNN_Encoder(tf.keras.Model):\n",
-        "    # Since we have already extracted the features and dumped it using pickle\n",
-        "    # This encoder passes those features through a Fully connected layer\n",
-        "    def __init__(self, embedding_dim):\n",
-        "        super(CNN_Encoder, self).__init__()\n",
-        "        # shape after fc == (batch_size, 64, embedding_dim)\n",
-        "        self.fc = tf.keras.layers.Dense(embedding_dim)\n",
-        "        \n",
-        "    def call(self, x):\n",
-        "        x = self.fc(x)\n",
-        "        x = tf.nn.relu(x)\n",
-        "        return x"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "SmZS2N0bXG3T"
+   },
+   "outputs": [],
+   "source": [
+    "# loading the numpy files \n",
+    "def map_func(img_name, cap):\n",
+    "    img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n",
+    "    return img_tensor, cap"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "V9UbGQmERPGi",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "class RNN_Decoder(tf.keras.Model):\n",
-        "  def __init__(self, embedding_dim, units, vocab_size):\n",
-        "    super(RNN_Decoder, self).__init__()\n",
-        "    self.units = units\n",
-        "\n",
-        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
-        "    self.gru = gru(self.units)\n",
-        "    self.fc1 = tf.keras.layers.Dense(self.units)\n",
-        "    self.fc2 = tf.keras.layers.Dense(vocab_size)\n",
-        "    \n",
-        "    self.attention = BahdanauAttention(self.units)\n",
-        "        \n",
-        "  def call(self, x, features, hidden):\n",
-        "    # defining attention as a separate model\n",
-        "    context_vector, attention_weights = self.attention(features, hidden)\n",
-        "    \n",
-        "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
-        "    x = self.embedding(x)\n",
-        "    \n",
-        "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
-        "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
-        "    \n",
-        "    # passing the concatenated vector to the GRU\n",
-        "    output, state = self.gru(x)\n",
-        "    \n",
-        "    # shape == (batch_size, max_length, hidden_size)\n",
-        "    x = self.fc1(output)\n",
-        "    \n",
-        "    # x shape == (batch_size * max_length, hidden_size)\n",
-        "    x = tf.reshape(x, (-1, x.shape[2]))\n",
-        "    \n",
-        "    # output shape == (batch_size * max_length, vocab)\n",
-        "    x = self.fc2(x)\n",
-        "\n",
-        "    return x, state, attention_weights\n",
-        "\n",
-        "  def reset_state(self, batch_size):\n",
-        "    return tf.zeros((batch_size, self.units))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "FDF_Nm3tRPGZ"
+   },
+   "outputs": [],
+   "source": [
+    "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n",
+    "\n",
+    "# using map to load the numpy files in parallel\n",
+    "# NOTE: Be sure to set num_parallel_calls to the number of CPU cores you have\n",
+    "# https://www.tensorflow.org/api_docs/python/tf/py_func\n",
+    "dataset = dataset.map(lambda item1, item2: tf.py_func(\n",
+    "          map_func, [item1, item2], [tf.float32, tf.int32]), num_parallel_calls=8)\n",
+    "\n",
+    "# shuffling and batching\n",
+    "dataset = dataset.shuffle(BUFFER_SIZE)\n",
+    "# https://www.tensorflow.org/api_docs/python/tf/contrib/data/batch_and_drop_remainder\n",
+    "dataset = dataset.batch(BATCH_SIZE)\n",
+    "dataset = dataset.prefetch(1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "nrvoDphgRPGd"
+   },
+   "source": [
+    "## Model\n",
+    "\n",
+    "Fun fact, the decoder below is identical to the one in the example for [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb).\n",
+    "\n",
+    "The model architecture is inspired by the [Show, Attend and Tell](https://arxiv.org/pdf/1502.03044.pdf) paper.\n",
+    "\n",
+    "* In this example, we extract the features from the lower convolutional layer of InceptionV3 giving us a vector of shape (8, 8, 2048). \n",
+    "* We squash that to a shape of (64, 2048).\n",
+    "* This vector is then passed through the CNN Encoder(which consists of a single Fully connected layer).\n",
+    "* The RNN(here GRU) attends over the image to predict the next word."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "Qs_Sr03wRPGk",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "encoder = CNN_Encoder(embedding_dim)\n",
-        "decoder = RNN_Decoder(embedding_dim, units, vocab_size)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "AAppCGLKRPGd"
+   },
+   "outputs": [],
+   "source": [
+    "def gru(units):\n",
+    "  # If you have a GPU, we recommend using the CuDNNGRU layer (it provides a \n",
+    "  # significant speedup).\n",
+    "  if tf.test.is_gpu_available():\n",
+    "    return tf.keras.layers.CuDNNGRU(units, \n",
+    "                                    return_sequences=True, \n",
+    "                                    return_state=True, \n",
+    "                                    recurrent_initializer='glorot_uniform')\n",
+    "  else:\n",
+    "    return tf.keras.layers.GRU(units, \n",
+    "                               return_sequences=True, \n",
+    "                               return_state=True, \n",
+    "                               recurrent_activation='sigmoid', \n",
+    "                               recurrent_initializer='glorot_uniform')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "-bYN7xA0RPGl",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "optimizer = tf.train.AdamOptimizer()\n",
-        "\n",
-        "# We are masking the loss calculated for padding\n",
-        "def loss_function(real, pred):\n",
-        "    mask = 1 - np.equal(real, 0)\n",
-        "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
-        "    return tf.reduce_mean(loss_)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "ja2LFTMSdeV3"
+   },
+   "outputs": [],
+   "source": [
+    "class BahdanauAttention(tf.keras.Model):\n",
+    "  def __init__(self, units):\n",
+    "    super(BahdanauAttention, self).__init__()\n",
+    "    self.W1 = tf.keras.layers.Dense(units)\n",
+    "    self.W2 = tf.keras.layers.Dense(units)\n",
+    "    self.V = tf.keras.layers.Dense(1)\n",
+    "  \n",
+    "  def call(self, features, hidden):\n",
+    "    # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n",
+    "    \n",
+    "    # hidden shape == (batch_size, hidden_size)\n",
+    "    # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n",
+    "    hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
+    "    \n",
+    "    # score shape == (batch_size, 64, hidden_size)\n",
+    "    score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n",
+    "    \n",
+    "    # attention_weights shape == (batch_size, 64, 1)\n",
+    "    # we get 1 at the last axis because we are applying score to self.V\n",
+    "    attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
+    "    \n",
+    "    # context_vector shape after sum == (batch_size, hidden_size)\n",
+    "    context_vector = attention_weights * features\n",
+    "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
+    "    \n",
+    "    return context_vector, attention_weights"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "PHod7t72RPGn",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Training\n",
-        "\n",
-        "* We extract the features stored in the respective `.npy` files and then pass those features through the encoder.\n",
-        "* The encoder output, hidden state(initialized to 0) and the decoder input (which is the start token) is passed to the decoder.\n",
-        "* The decoder returns the predictions and the decoder hidden state.\n",
-        "* The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
-        "* Use teacher forcing to decide the next input to the decoder.\n",
-        "* Teacher forcing is the technique where the target word is passed as the next input to the decoder.\n",
-        "* The final step is to calculate the gradients and apply it to the optimizer and backpropagate.\n"
-      ]
+    "colab_type": "code",
+    "id": "AZ7R1RxHRPGf"
+   },
+   "outputs": [],
+   "source": [
+    "class CNN_Encoder(tf.keras.Model):\n",
+    "    # Since we have already extracted the features and dumped it using pickle\n",
+    "    # This encoder passes those features through a Fully connected layer\n",
+    "    def __init__(self, embedding_dim):\n",
+    "        super(CNN_Encoder, self).__init__()\n",
+    "        # shape after fc == (batch_size, 64, embedding_dim)\n",
+    "        self.fc = tf.keras.layers.Dense(embedding_dim)\n",
+    "        \n",
+    "    def call(self, x):\n",
+    "        x = self.fc(x)\n",
+    "        x = tf.nn.relu(x)\n",
+    "        return x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "Vt4WZ5mhJE-E",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# adding this in a separate cell because if you run the training cell \n",
-        "# many times, the loss_plot array will be reset\n",
-        "loss_plot = []"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "V9UbGQmERPGi"
+   },
+   "outputs": [],
+   "source": [
+    "class RNN_Decoder(tf.keras.Model):\n",
+    "  def __init__(self, embedding_dim, units, vocab_size):\n",
+    "    super(RNN_Decoder, self).__init__()\n",
+    "    self.units = units\n",
+    "\n",
+    "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
+    "    self.gru = gru(self.units)\n",
+    "    self.fc1 = tf.keras.layers.Dense(self.units)\n",
+    "    self.fc2 = tf.keras.layers.Dense(vocab_size)\n",
+    "    \n",
+    "    self.attention = BahdanauAttention(self.units)\n",
+    "        \n",
+    "  def call(self, x, features, hidden):\n",
+    "    # defining attention as a separate model\n",
+    "    context_vector, attention_weights = self.attention(features, hidden)\n",
+    "    \n",
+    "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
+    "    x = self.embedding(x)\n",
+    "    \n",
+    "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
+    "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
+    "    \n",
+    "    # passing the concatenated vector to the GRU\n",
+    "    output, state = self.gru(x)\n",
+    "    \n",
+    "    # shape == (batch_size, max_length, hidden_size)\n",
+    "    x = self.fc1(output)\n",
+    "    \n",
+    "    # x shape == (batch_size * max_length, hidden_size)\n",
+    "    x = tf.reshape(x, (-1, x.shape[2]))\n",
+    "    \n",
+    "    # output shape == (batch_size * max_length, vocab)\n",
+    "    x = self.fc2(x)\n",
+    "\n",
+    "    return x, state, attention_weights\n",
+    "\n",
+    "  def reset_state(self, batch_size):\n",
+    "    return tf.zeros((batch_size, self.units))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "UlA4VIQpRPGo",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "EPOCHS = 20\n",
-        "\n",
-        "for epoch in range(EPOCHS):\n",
-        "    start = time.time()\n",
-        "    total_loss = 0\n",
-        "    \n",
-        "    for (batch, (img_tensor, target)) in enumerate(dataset):\n",
-        "        loss = 0\n",
-        "        \n",
-        "        # initializing the hidden state for each batch\n",
-        "        # because the captions are not related from image to image\n",
-        "        hidden = decoder.reset_state(batch_size=target.shape[0])\n",
-        "\n",
-        "        dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * BATCH_SIZE, 1)\n",
-        "        \n",
-        "        with tf.GradientTape() as tape:\n",
-        "            features = encoder(img_tensor)\n",
-        "            \n",
-        "            for i in range(1, target.shape[1]):\n",
-        "                # passing the features through the decoder\n",
-        "                predictions, hidden, _ = decoder(dec_input, features, hidden)\n",
-        "\n",
-        "                loss += loss_function(target[:, i], predictions)\n",
-        "                \n",
-        "                # using teacher forcing\n",
-        "                dec_input = tf.expand_dims(target[:, i], 1)\n",
-        "        \n",
-        "        total_loss += (loss / int(target.shape[1]))\n",
-        "        \n",
-        "        variables = encoder.variables + decoder.variables\n",
-        "        \n",
-        "        gradients = tape.gradient(loss, variables) \n",
-        "        \n",
-        "        optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n",
-        "        \n",
-        "        if batch % 100 == 0:\n",
-        "            print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, \n",
-        "                                                          batch, \n",
-        "                                                          loss.numpy() / int(target.shape[1])))\n",
-        "    # storing the epoch end loss value to plot later\n",
-        "    loss_plot.append(total_loss / len(cap_vector))\n",
-        "    \n",
-        "    print ('Epoch {} Loss {:.6f}'.format(epoch + 1, \n",
-        "                                         total_loss/len(cap_vector)))\n",
-        "    print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "Qs_Sr03wRPGk"
+   },
+   "outputs": [],
+   "source": [
+    "encoder = CNN_Encoder(embedding_dim)\n",
+    "decoder = RNN_Decoder(embedding_dim, units, vocab_size)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "1Wm83G-ZBPcC",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "plt.plot(loss_plot)\n",
-        "plt.xlabel('Epochs')\n",
-        "plt.ylabel('Loss')\n",
-        "plt.title('Loss Plot')\n",
-        "plt.show()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "-bYN7xA0RPGl"
+   },
+   "outputs": [],
+   "source": [
+    "optimizer = tf.train.AdamOptimizer()\n",
+    "\n",
+    "# We are masking the loss calculated for padding\n",
+    "def loss_function(real, pred):\n",
+    "    mask = 1 - np.equal(real, 0)\n",
+    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
+    "    return tf.reduce_mean(loss_)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "PHod7t72RPGn"
+   },
+   "source": [
+    "## Training\n",
+    "\n",
+    "* We extract the features stored in the respective `.npy` files and then pass those features through the encoder.\n",
+    "* The encoder output, hidden state(initialized to 0) and the decoder input (which is the start token) is passed to the decoder.\n",
+    "* The decoder returns the predictions and the decoder hidden state.\n",
+    "* The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
+    "* Use teacher forcing to decide the next input to the decoder.\n",
+    "* Teacher forcing is the technique where the target word is passed as the next input to the decoder.\n",
+    "* The final step is to calculate the gradients and apply it to the optimizer and backpropagate.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "xGvOcLQKghXN",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Caption!\n",
-        "\n",
-        "* The evaluate function is similar to the training loop, except we don't use teacher forcing here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
-        "* Stop predicting when the model predicts the end token.\n",
-        "* And store the attention weights for every time step."
-      ]
+    "colab_type": "code",
+    "id": "Vt4WZ5mhJE-E"
+   },
+   "outputs": [],
+   "source": [
+    "# adding this in a separate cell because if you run the training cell \n",
+    "# many times, the loss_plot array will be reset\n",
+    "loss_plot = []"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "RCWpDtyNRPGs",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "def evaluate(image):\n",
-        "    attention_plot = np.zeros((max_length, attention_features_shape))\n",
-        "\n",
-        "    hidden = decoder.reset_state(batch_size=1)\n",
-        "\n",
-        "    temp_input = tf.expand_dims(load_image(image)[0], 0)\n",
-        "    img_tensor_val = image_features_extract_model(temp_input)\n",
-        "    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n",
-        "\n",
-        "    features = encoder(img_tensor_val)\n",
-        "\n",
-        "    dec_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)\n",
-        "    result = []\n",
-        "\n",
-        "    for i in range(max_length):\n",
-        "        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n",
-        "\n",
-        "        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n",
-        "\n",
-        "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
-        "        result.append(index_word[predicted_id])\n",
-        "\n",
-        "        if index_word[predicted_id] == '<end>':\n",
-        "            return result, attention_plot\n",
-        "\n",
-        "        dec_input = tf.expand_dims([predicted_id], 0)\n",
-        "\n",
-        "    attention_plot = attention_plot[:len(result), :]\n",
-        "    return result, attention_plot"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "UlA4VIQpRPGo"
+   },
+   "outputs": [],
+   "source": [
+    "EPOCHS = 20\n",
+    "\n",
+    "for epoch in range(EPOCHS):\n",
+    "    start = time.time()\n",
+    "    total_loss = 0\n",
+    "    \n",
+    "    for (batch, (img_tensor, target)) in enumerate(dataset):\n",
+    "        loss = 0\n",
+    "        \n",
+    "        # initializing the hidden state for each batch\n",
+    "        # because the captions are not related from image to image\n",
+    "        hidden = decoder.reset_state(batch_size=target.shape[0])\n",
+    "\n",
+    "        dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * BATCH_SIZE, 1)\n",
+    "        \n",
+    "        with tf.GradientTape() as tape:\n",
+    "            features = encoder(img_tensor)\n",
+    "            \n",
+    "            for i in range(1, target.shape[1]):\n",
+    "                # passing the features through the decoder\n",
+    "                predictions, hidden, _ = decoder(dec_input, features, hidden)\n",
+    "\n",
+    "                loss += loss_function(target[:, i], predictions)\n",
+    "                \n",
+    "                # using teacher forcing\n",
+    "                dec_input = tf.expand_dims(target[:, i], 1)\n",
+    "        \n",
+    "        total_loss += (loss / int(target.shape[1]))\n",
+    "        \n",
+    "        variables = encoder.variables + decoder.variables\n",
+    "        \n",
+    "        gradients = tape.gradient(loss, variables) \n",
+    "        \n",
+    "        optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n",
+    "        \n",
+    "        if batch % 100 == 0:\n",
+    "            print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, \n",
+    "                                                          batch, \n",
+    "                                                          loss.numpy() / int(target.shape[1])))\n",
+    "    # storing the epoch end loss value to plot later\n",
+    "    loss_plot.append(total_loss / len(cap_vector))\n",
+    "    \n",
+    "    print ('Epoch {} Loss {:.6f}'.format(epoch + 1, \n",
+    "                                         total_loss/len(cap_vector)))\n",
+    "    print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "fD_y7PD6RPGt",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "def plot_attention(image, result, attention_plot):\n",
-        "    temp_image = np.array(Image.open(image))\n",
-        "\n",
-        "    fig = plt.figure(figsize=(10, 10))\n",
-        "    \n",
-        "    len_result = len(result)\n",
-        "    for l in range(len_result):\n",
-        "        temp_att = np.resize(attention_plot[l], (8, 8))\n",
-        "        ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n",
-        "        ax.set_title(result[l])\n",
-        "        img = ax.imshow(temp_image)\n",
-        "        ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n",
-        "\n",
-        "    plt.tight_layout()\n",
-        "    plt.show()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "1Wm83G-ZBPcC"
+   },
+   "outputs": [],
+   "source": [
+    "plt.plot(loss_plot)\n",
+    "plt.xlabel('Epochs')\n",
+    "plt.ylabel('Loss')\n",
+    "plt.title('Loss Plot')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "xGvOcLQKghXN"
+   },
+   "source": [
+    "## Caption!\n",
+    "\n",
+    "* The evaluate function is similar to the training loop, except we don't use teacher forcing here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
+    "* Stop predicting when the model predicts the end token.\n",
+    "* And store the attention weights for every time step."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "io7ws3ReRPGv",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "# captions on the validation set\n",
-        "rid = np.random.randint(0, len(img_name_val))\n",
-        "image = img_name_val[rid]\n",
-        "real_caption = ' '.join([index_word[i] for i in cap_val[rid] if i not in [0]])\n",
-        "result, attention_plot = evaluate(image)\n",
-        "\n",
-        "print ('Real Caption:', real_caption)\n",
-        "print ('Prediction Caption:', ' '.join(result))\n",
-        "plot_attention(image, result, attention_plot)\n",
-        "# opening the image\n",
-        "Image.open(img_name_val[rid])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "RCWpDtyNRPGs"
+   },
+   "outputs": [],
+   "source": [
+    "def evaluate(image):\n",
+    "    attention_plot = np.zeros((max_length, attention_features_shape))\n",
+    "\n",
+    "    hidden = decoder.reset_state(batch_size=1)\n",
+    "\n",
+    "    temp_input = tf.expand_dims(load_image(image)[0], 0)\n",
+    "    img_tensor_val = image_features_extract_model(temp_input)\n",
+    "    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n",
+    "\n",
+    "    features = encoder(img_tensor_val)\n",
+    "\n",
+    "    dec_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)\n",
+    "    result = []\n",
+    "\n",
+    "    for i in range(max_length):\n",
+    "        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n",
+    "\n",
+    "        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n",
+    "\n",
+    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
+    "        result.append(index_word[predicted_id])\n",
+    "\n",
+    "        if index_word[predicted_id] == '<end>':\n",
+    "            return result, attention_plot\n",
+    "\n",
+    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
+    "\n",
+    "    attention_plot = attention_plot[:len(result), :]\n",
+    "    return result, attention_plot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "Rprk3HEvZuxb",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "## Try it on your own images\n",
-        "For fun, below we've provided a method you can use to caption your own images with the model we've just trained. Keep in mind, it was trained on a relatively small amount of data, and your images may be different from the training data (so be prepared for weird results!)\n"
-      ]
+    "colab_type": "code",
+    "id": "fD_y7PD6RPGt"
+   },
+   "outputs": [],
+   "source": [
+    "def plot_attention(image, result, attention_plot):\n",
+    "    temp_image = np.array(Image.open(image))\n",
+    "\n",
+    "    fig = plt.figure(figsize=(10, 10))\n",
+    "    \n",
+    "    len_result = len(result)\n",
+    "    for l in range(len_result):\n",
+    "        temp_att = np.resize(attention_plot[l], (8, 8))\n",
+    "        ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n",
+    "        ax.set_title(result[l])\n",
+    "        img = ax.imshow(temp_image)\n",
+    "        ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n",
+    "\n",
+    "    plt.tight_layout()\n",
+    "    plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
-    {
-      "metadata": {
-        "id": "9Psd1quzaAWg",
-        "colab_type": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        }
-      },
-      "cell_type": "code",
-      "source": [
-        "image_url = 'https://tensorflow.org/images/surf.jpg'\n",
-        "image_extension = image_url[-4:]\n",
-        "image_path = tf.keras.utils.get_file('image'+image_extension, \n",
-        "                                     origin=image_url)\n",
-        "\n",
-        "result, attention_plot = evaluate(image_path)\n",
-        "print ('Prediction Caption:', ' '.join(result))\n",
-        "plot_attention(image_path, result, attention_plot)\n",
-        "# opening the image\n",
-        "Image.open(image_path)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+    "colab_type": "code",
+    "id": "io7ws3ReRPGv"
+   },
+   "outputs": [],
+   "source": [
+    "# captions on the validation set\n",
+    "rid = np.random.randint(0, len(img_name_val))\n",
+    "image = img_name_val[rid]\n",
+    "real_caption = ' '.join([index_word[i] for i in cap_val[rid] if i not in [0]])\n",
+    "result, attention_plot = evaluate(image)\n",
+    "\n",
+    "print ('Real Caption:', real_caption)\n",
+    "print ('Prediction Caption:', ' '.join(result))\n",
+    "plot_attention(image, result, attention_plot)\n",
+    "# opening the image\n",
+    "Image.open(img_name_val[rid])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "Rprk3HEvZuxb"
+   },
+   "source": [
+    "## Try it on your own images\n",
+    "For fun, below we've provided a method you can use to caption your own images with the model we've just trained. Keep in mind, it was trained on a relatively small amount of data, and your images may be different from the training data (so be prepared for weird results!)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "autoexec": {
+      "startup": false,
+      "wait_interval": 0
+     }
     },
+    "colab_type": "code",
+    "id": "9Psd1quzaAWg"
+   },
+   "outputs": [],
+   "source": [
+    "image_url = 'https://tensorflow.org/images/surf.jpg'\n",
+    "image_extension = image_url[-4:]\n",
+    "image_path = tf.keras.utils.get_file('image'+image_extension, \n",
+    "                                     origin=image_url)\n",
+    "\n",
+    "result, attention_plot = evaluate(image_path)\n",
+    "print ('Prediction Caption:', ' '.join(result))\n",
+    "plot_attention(image_path, result, attention_plot)\n",
+    "# opening the image\n",
+    "Image.open(image_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "colab_type": "text",
+    "id": "VJZXyJco6uLO"
+   },
+   "source": [
+    "# Next steps\n",
+    "\n",
+    "Congrats! You've just trained an image captioning model with attention. Next, we recommend taking a look at this example [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb). It uses a similar architecture to translate between Spanish and English sentences. You can also experiment with training the code in this notebook on a different dataset."
+   ]
+  }
+ ],
+ "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "collapsed_sections": [],
+   "default_view": {},
+   "name": "image_captioning_with_attention.ipynb",
+   "private_outputs": true,
+   "provenance": [
     {
-      "metadata": {
-        "id": "VJZXyJco6uLO",
-        "colab_type": "text"
-      },
-      "cell_type": "markdown",
-      "source": [
-        "# Next steps\n",
-        "\n",
-        "Congrats! You've just trained an image captioning model with attention. Next, we recommend taking a look at this example [Neural Machine Translation with Attention]( https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb). It uses a similar architecture to translate between Spanish and English sentences. You can also experiment with training the code in this notebook on a different dataset."
-      ]
+     "file_id": "1HI8OK2sMjcx9CTWVn0122QAHOuXaOaMg",
+     "timestamp": 1530222436922
     }
-  ]
+   ],
+   "toc_visible": true,
+   "version": "0.3.2",
+   "views": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
 }
-- 
GitLab


From c5b1ad0ea01634d0fb6bf6ba59ce014f1e66b6d9 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 25 Oct 2018 22:20:40 +0000
Subject: [PATCH 0098/1554] Improve shape function of tf.sparse_reduce_sum

This fix tries to address the issue raised in 23114 where
the shape function of tf.sparse_reduce_sum did not
infer the shape even if the input shape were known.

This fix improves the shape function.

This fix fixes 23114.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/sparse_ops.cc | 41 ++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc
index bc0cb2095d..c54c8e87c7 100644
--- a/tensorflow/core/ops/sparse_ops.cc
+++ b/tensorflow/core/ops/sparse_ops.cc
@@ -423,7 +423,46 @@ REGISTER_OP("SparseReduceSum")
     .Attr("keep_dims: bool = False")
     .Output("output: T")
     .Attr("T: numbertype")
-    .SetShapeFn(shape_inference::UnknownShape);
+    .SetShapeFn([](InferenceContext* c) {
+      bool keep_dims = false;
+      TF_RETURN_IF_ERROR(c->GetAttr("keep_dims", &keep_dims));
+
+      const Tensor* shape_tensor = c->input_tensor(2);
+      const Tensor* axes_tensor = c->input_tensor(3);
+      if (shape_tensor != nullptr && axes_tensor != nullptr) {
+        auto shape_vec = shape_tensor->flat<int64>();
+        auto axes_vec = axes_tensor->flat<int32>();
+
+        int64 ndims = shape_vec.size();
+        std::unordered_set<int64> axes;
+        for (int i = 0; i < axes_vec.size(); i++) {
+          axes.insert((axes_vec(i) + ndims) % ndims);
+        }
+
+        std::vector<DimensionHandle> dims;
+        if (keep_dims) {
+          dims.reserve(ndims);
+          for (int d = 0; d < ndims; ++d) {
+            if (axes.find(d) == axes.end()) {
+              dims.push_back(c->MakeDim(shape_vec(d)));
+            } else {
+              dims.push_back(c->MakeDim(1));
+            }
+          }
+        } else {
+          for (int d = 0; d < ndims; ++d) {
+            if (axes.find(d) == axes.end()) {
+              dims.push_back(c->MakeDim(shape_vec(d)));
+            }
+          }
+
+        }
+
+        c->set_output(0, c->MakeShape(dims));
+        return Status::OK();
+      }
+      return shape_inference::UnknownShape(c);
+    });
 
 REGISTER_OP("SparseReduceSumSparse")
     .Input("input_indices: int64")
-- 
GitLab


From e008c4dc79bbf9cea6a6137a9f465718835a9c40 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 25 Oct 2018 22:23:06 +0000
Subject: [PATCH 0099/1554] Add test case for improved shape function of
 tf.sparse_reduce_sum

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/sparse_ops_test.py    | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index a45ce2e13b..7594fa8889 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -657,6 +657,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
     self._compare(sp_t, reduction_axes, ndims, True, False)
     self._compare(sp_t, reduction_axes, ndims, True, True)
 
+
   def testSimpleAndRandomInputs(self):
     if np.__version__ == "1.13.0":
       self.skipTest("numpy 1.13.0 bug")
@@ -722,6 +723,45 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
                                                       reduced.eval().shape)
         self.assertLess(err, 1e-3)
 
+  def _testSparseReduceSumShape(self, sp_t, reduction_axes, ndims, keep_dims):
+    densified = sparse_ops.sparse_tensor_to_dense(sp_t).eval()
+
+    np_ans = densified
+    if reduction_axes is None:
+      np_ans = np.sum(np_ans, keepdims=keep_dims)
+    else:
+      if not isinstance(reduction_axes, list):  # Single scalar.
+        reduction_axes = [reduction_axes]
+      reduction_axes = np.array(reduction_axes).astype(np.int32)
+      # Handles negative axes.
+      reduction_axes = (reduction_axes + ndims) % ndims
+      # Loop below depends on sorted.
+      reduction_axes.sort()
+      for ra in reduction_axes.ravel()[::-1]:
+        np_ans = np.sum(np_ans, axis=ra, keepdims=keep_dims)
+
+    tf_ans = sparse_ops.sparse_reduce_sum(sp_t, reduction_axes, keep_dims)
+    self.assertAllEqual(np_ans.shape, tf_ans.get_shape().as_list())
+
+  def testSparseReduceSumShape(self):
+    sp_t = sparse_tensor.SparseTensor(self.ind, self.vals, self.dense_shape)
+
+    with self.session(use_gpu=False):
+      self._testSparseReduceSumShape(sp_t, None, ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, None, ndims=2, keep_dims=True)
+      self._testSparseReduceSumShape(sp_t, 0, ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, 0, ndims=2, keep_dims=True)
+      self._testSparseReduceSumShape(sp_t, [1], ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, [1], ndims=2, keep_dims=True)
+      self._testSparseReduceSumShape(sp_t, [0, 1], ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, [0, 1], ndims=2, keep_dims=True)
+      self._testSparseReduceSumShape(sp_t, [1, 0], ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, [1, 0], ndims=2, keep_dims=True)
+      self._testSparseReduceSumShape(sp_t, [-1], ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, [-1], ndims=2, keep_dims=True)
+      self._testSparseReduceSumShape(sp_t, [1, -2], ndims=2, keep_dims=False)
+      self._testSparseReduceSumShape(sp_t, [1, -2], ndims=2, keep_dims=True)
+
 
 class SparseMathOpsTest(test_util.TensorFlowTestCase):
 
-- 
GitLab


From b51975e5339edba945b6c750f7a140dff7b4bcf9 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 25 Oct 2018 22:29:11 +0000
Subject: [PATCH 0100/1554] Improvement in tests

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/sparse_ops_test.py    | 22 +++++++------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index 7594fa8889..2ebc10c4a9 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -747,20 +747,14 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
     sp_t = sparse_tensor.SparseTensor(self.ind, self.vals, self.dense_shape)
 
     with self.session(use_gpu=False):
-      self._testSparseReduceSumShape(sp_t, None, ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, None, ndims=2, keep_dims=True)
-      self._testSparseReduceSumShape(sp_t, 0, ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, 0, ndims=2, keep_dims=True)
-      self._testSparseReduceSumShape(sp_t, [1], ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, [1], ndims=2, keep_dims=True)
-      self._testSparseReduceSumShape(sp_t, [0, 1], ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, [0, 1], ndims=2, keep_dims=True)
-      self._testSparseReduceSumShape(sp_t, [1, 0], ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, [1, 0], ndims=2, keep_dims=True)
-      self._testSparseReduceSumShape(sp_t, [-1], ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, [-1], ndims=2, keep_dims=True)
-      self._testSparseReduceSumShape(sp_t, [1, -2], ndims=2, keep_dims=False)
-      self._testSparseReduceSumShape(sp_t, [1, -2], ndims=2, keep_dims=True)
+      for keep_dims in [True, False]:
+        self._testSparseReduceSumShape(sp_t, None, ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, 0, ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, [1], ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, [0, 1], ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, [1, 0], ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, [-1], ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, [1, -2], ndims=2, keep_dims=keep_dims)
 
 
 class SparseMathOpsTest(test_util.TensorFlowTestCase):
-- 
GitLab


From 71787f94307d6f8f18255883f53d1544e09f005b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 25 Oct 2018 22:33:07 +0000
Subject: [PATCH 0101/1554] Pylint fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/sparse_ops_test.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index 2ebc10c4a9..f81de76716 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -748,13 +748,13 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
 
     with self.session(use_gpu=False):
       for keep_dims in [True, False]:
-        self._testSparseReduceSumShape(sp_t, None, ndims=2, keep_dims=keep_dims)
-        self._testSparseReduceSumShape(sp_t, 0, ndims=2, keep_dims=keep_dims)
-        self._testSparseReduceSumShape(sp_t, [1], ndims=2, keep_dims=keep_dims)
-        self._testSparseReduceSumShape(sp_t, [0, 1], ndims=2, keep_dims=keep_dims)
-        self._testSparseReduceSumShape(sp_t, [1, 0], ndims=2, keep_dims=keep_dims)
-        self._testSparseReduceSumShape(sp_t, [-1], ndims=2, keep_dims=keep_dims)
-        self._testSparseReduceSumShape(sp_t, [1, -2], ndims=2, keep_dims=keep_dims)
+        self._testSparseReduceSumShape(sp_t, None, 2, keep_dims)
+        self._testSparseReduceSumShape(sp_t, 0, 2, keep_dims)
+        self._testSparseReduceSumShape(sp_t, [1], 2, keep_dims)
+        self._testSparseReduceSumShape(sp_t, [0, 1], 2, keep_dims)
+        self._testSparseReduceSumShape(sp_t, [1, 0], 2, keep_dims)
+        self._testSparseReduceSumShape(sp_t, [-1], 2, keep_dims)
+        self._testSparseReduceSumShape(sp_t, [1, -2], 2, keep_dims)
 
 
 class SparseMathOpsTest(test_util.TensorFlowTestCase):
-- 
GitLab


From 74784ac463b55adcae0534b117d4ccd5eac82c0c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 31 Oct 2018 20:12:55 +0000
Subject: [PATCH 0102/1554] Fix `Experimental clang-format Check`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/sparse_ops.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc
index c54c8e87c7..95d009c9b2 100644
--- a/tensorflow/core/ops/sparse_ops.cc
+++ b/tensorflow/core/ops/sparse_ops.cc
@@ -455,7 +455,6 @@ REGISTER_OP("SparseReduceSum")
               dims.push_back(c->MakeDim(shape_vec(d)));
             }
           }
-
         }
 
         c->set_output(0, c->MakeShape(dims));
-- 
GitLab


From e64bfb83dc7338720a9a7bc5c4414a1f6379dfe4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 4 Nov 2018 18:43:45 +0000
Subject: [PATCH 0103/1554] Fix //tensorflow/core:ops_sparse_ops_test

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/sparse_ops_test.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/core/ops/sparse_ops_test.cc b/tensorflow/core/ops/sparse_ops_test.cc
index 6a9b5ce4d3..00283c5993 100644
--- a/tensorflow/core/ops/sparse_ops_test.cc
+++ b/tensorflow/core/ops/sparse_ops_test.cc
@@ -133,6 +133,13 @@ TEST(SparseOpsTest, SparseToDense_ShapeFn) {
 
 TEST(SparseOpsTest, SparseReduceSum_ShapeFn) {
   ShapeInferenceTestOp op("SparseReduceSum");
+  TF_ASSERT_OK(NodeDefBuilder("test", "SparseReduceSum")
+                   .Input({"input_indices", 0, DT_INT64})
+                   .Input({"input_values", 1, DT_INT64})
+                   .Input({"input_shape", 2, DT_INT64})
+                   .Input({"reduction_axes", 3, DT_INT32})
+                   .Attr("keep_dims", false)
+                   .Finalize(&op.node_def));
 
   // Shape fn always yields unknown.
   INFER_OK(op, "?;?;?;?", "?");
-- 
GitLab


From 70768d6b0614eeb12d8cd07c2a3b1e4795d73dc7 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 4 Nov 2018 18:46:02 +0000
Subject: [PATCH 0104/1554] Add shape function to SparseReduceMax

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/sparse_ops.cc | 87 +++++++++++++++++--------------
 1 file changed, 47 insertions(+), 40 deletions(-)

diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc
index 95d009c9b2..3529d47449 100644
--- a/tensorflow/core/ops/sparse_ops.cc
+++ b/tensorflow/core/ops/sparse_ops.cc
@@ -39,6 +39,51 @@ Status SparseSparseMinOrMaxShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
+Status SparseReduceShapeFn(InferenceContext* c) {
+  // Input 0: input_indices
+  // Input 1: input_values
+  // Input 2: input_shape
+  // Input 3: reduction_axes
+  // Attr: keep_dims
+  bool keep_dims = false;
+  TF_RETURN_IF_ERROR(c->GetAttr("keep_dims", &keep_dims));
+
+  const Tensor* shape_tensor = c->input_tensor(2);
+  const Tensor* axes_tensor = c->input_tensor(3);
+  if (shape_tensor != nullptr && axes_tensor != nullptr) {
+    auto shape_vec = shape_tensor->flat<int64>();
+    auto axes_vec = axes_tensor->flat<int32>();
+
+    int64 ndims = shape_vec.size();
+    std::unordered_set<int64> axes;
+    for (int i = 0; i < axes_vec.size(); i++) {
+      axes.insert((axes_vec(i) + ndims) % ndims);
+    }
+
+    std::vector<DimensionHandle> dims;
+    if (keep_dims) {
+      dims.reserve(ndims);
+      for (int d = 0; d < ndims; ++d) {
+        if (axes.find(d) == axes.end()) {
+          dims.push_back(c->MakeDim(shape_vec(d)));
+        } else {
+          dims.push_back(c->MakeDim(1));
+        }
+      }
+    } else {
+      for (int d = 0; d < ndims; ++d) {
+        if (axes.find(d) == axes.end()) {
+          dims.push_back(c->MakeDim(shape_vec(d)));
+        }
+      }
+    }
+
+    c->set_output(0, c->MakeShape(dims));
+    return Status::OK();
+  }
+  return shape_inference::UnknownShape(c);
+}
+
 }  // namespace
 
 REGISTER_OP("SparseAddGrad")
@@ -401,7 +446,7 @@ REGISTER_OP("SparseReduceMax")
     .Attr("keep_dims: bool = False")
     .Output("output: T")
     .Attr("T: realnumbertype")
-    .SetShapeFn(shape_inference::UnknownShape);
+    .SetShapeFn(SparseReduceShapeFn);
 
 REGISTER_OP("SparseReduceMaxSparse")
     .Input("input_indices: int64")
@@ -423,45 +468,7 @@ REGISTER_OP("SparseReduceSum")
     .Attr("keep_dims: bool = False")
     .Output("output: T")
     .Attr("T: numbertype")
-    .SetShapeFn([](InferenceContext* c) {
-      bool keep_dims = false;
-      TF_RETURN_IF_ERROR(c->GetAttr("keep_dims", &keep_dims));
-
-      const Tensor* shape_tensor = c->input_tensor(2);
-      const Tensor* axes_tensor = c->input_tensor(3);
-      if (shape_tensor != nullptr && axes_tensor != nullptr) {
-        auto shape_vec = shape_tensor->flat<int64>();
-        auto axes_vec = axes_tensor->flat<int32>();
-
-        int64 ndims = shape_vec.size();
-        std::unordered_set<int64> axes;
-        for (int i = 0; i < axes_vec.size(); i++) {
-          axes.insert((axes_vec(i) + ndims) % ndims);
-        }
-
-        std::vector<DimensionHandle> dims;
-        if (keep_dims) {
-          dims.reserve(ndims);
-          for (int d = 0; d < ndims; ++d) {
-            if (axes.find(d) == axes.end()) {
-              dims.push_back(c->MakeDim(shape_vec(d)));
-            } else {
-              dims.push_back(c->MakeDim(1));
-            }
-          }
-        } else {
-          for (int d = 0; d < ndims; ++d) {
-            if (axes.find(d) == axes.end()) {
-              dims.push_back(c->MakeDim(shape_vec(d)));
-            }
-          }
-        }
-
-        c->set_output(0, c->MakeShape(dims));
-        return Status::OK();
-      }
-      return shape_inference::UnknownShape(c);
-    });
+    .SetShapeFn(SparseReduceShapeFn);
 
 REGISTER_OP("SparseReduceSumSparse")
     .Input("input_indices: int64")
-- 
GitLab


From 1b5467a65dab5007754c3aa1bd9f3fd5b3797cc6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 4 Nov 2018 18:59:38 +0000
Subject: [PATCH 0105/1554] Add additional test cases for sparse_reduce_max
 shape function

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/sparse_ops_test.py    | 35 +++++++++++--------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index f81de76716..372c27bf09 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -657,7 +657,6 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
     self._compare(sp_t, reduction_axes, ndims, True, False)
     self._compare(sp_t, reduction_axes, ndims, True, True)
 
-
   def testSimpleAndRandomInputs(self):
     if np.__version__ == "1.13.0":
       self.skipTest("numpy 1.13.0 bug")
@@ -723,12 +722,19 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
                                                       reduced.eval().shape)
         self.assertLess(err, 1e-3)
 
-  def _testSparseReduceSumShape(self, sp_t, reduction_axes, ndims, keep_dims):
+  def _testSparseReduceShape(self, sp_t, reduction_axes, ndims, keep_dims,
+                             do_sum):
     densified = sparse_ops.sparse_tensor_to_dense(sp_t).eval()
 
+    np_op = np.sum
+    tf_op = sparse_ops.sparse_reduce_sum
+    if not do_sum:
+      np_op = np.max
+      tf_op = sparse_ops.sparse_reduce_max
+
     np_ans = densified
     if reduction_axes is None:
-      np_ans = np.sum(np_ans, keepdims=keep_dims)
+      np_ans = np_op(np_ans, keepdims=keep_dims)
     else:
       if not isinstance(reduction_axes, list):  # Single scalar.
         reduction_axes = [reduction_axes]
@@ -738,23 +744,24 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
       # Loop below depends on sorted.
       reduction_axes.sort()
       for ra in reduction_axes.ravel()[::-1]:
-        np_ans = np.sum(np_ans, axis=ra, keepdims=keep_dims)
+        np_ans = np_op(np_ans, axis=ra, keepdims=keep_dims)
 
-    tf_ans = sparse_ops.sparse_reduce_sum(sp_t, reduction_axes, keep_dims)
+    tf_ans = tf_op(sp_t, reduction_axes, keep_dims)
     self.assertAllEqual(np_ans.shape, tf_ans.get_shape().as_list())
 
-  def testSparseReduceSumShape(self):
+  def testSparseReduceSumOrMaxShape(self):
     sp_t = sparse_tensor.SparseTensor(self.ind, self.vals, self.dense_shape)
 
     with self.session(use_gpu=False):
-      for keep_dims in [True, False]:
-        self._testSparseReduceSumShape(sp_t, None, 2, keep_dims)
-        self._testSparseReduceSumShape(sp_t, 0, 2, keep_dims)
-        self._testSparseReduceSumShape(sp_t, [1], 2, keep_dims)
-        self._testSparseReduceSumShape(sp_t, [0, 1], 2, keep_dims)
-        self._testSparseReduceSumShape(sp_t, [1, 0], 2, keep_dims)
-        self._testSparseReduceSumShape(sp_t, [-1], 2, keep_dims)
-        self._testSparseReduceSumShape(sp_t, [1, -2], 2, keep_dims)
+      for do_sum in [True, False]:
+        for keep_dims in [True, False]:
+          self._testSparseReduceShape(sp_t, None, 2, keep_dims, do_sum)
+          self._testSparseReduceShape(sp_t, 0, 2, keep_dims, do_sum)
+          self._testSparseReduceShape(sp_t, [1], 2, keep_dims, do_sum)
+          self._testSparseReduceShape(sp_t, [0, 1], 2, keep_dims, do_sum)
+          self._testSparseReduceShape(sp_t, [1, 0], 2, keep_dims, do_sum)
+          self._testSparseReduceShape(sp_t, [-1], 2, keep_dims, do_sum)
+          self._testSparseReduceShape(sp_t, [1, -2], 2, keep_dims, do_sum)
 
 
 class SparseMathOpsTest(test_util.TensorFlowTestCase):
-- 
GitLab


From b399b3281d494bf8f9b04a6f4df01ad6073026d5 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 5 Nov 2018 19:31:25 +0000
Subject: [PATCH 0106/1554] Move shape function to common_shape_fns.cc

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/framework/common_shape_fns.cc | 45 +++++++++++++++++
 tensorflow/core/framework/common_shape_fns.h  |  3 ++
 tensorflow/core/ops/sparse_ops.cc             | 49 +------------------
 3 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index e934cbfb54..b6000f1dd0 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1549,6 +1549,51 @@ Status ExplicitShapes(InferenceContext* c) {
   return Status::OK();
 }
 
+Status SparseReduceShapeFn(InferenceContext* c) {
+  // Input 0: input_indices
+  // Input 1: input_values
+  // Input 2: input_shape
+  // Input 3: reduction_axes
+  // Attr: keep_dims
+  bool keep_dims = false;
+  TF_RETURN_IF_ERROR(c->GetAttr("keep_dims", &keep_dims));
+
+  const Tensor* shape_tensor = c->input_tensor(2);
+  const Tensor* axes_tensor = c->input_tensor(3);
+  if (shape_tensor != nullptr && axes_tensor != nullptr) {
+    auto shape_vec = shape_tensor->flat<int64>();
+    auto axes_vec = axes_tensor->flat<int32>();
+
+    int64 ndims = shape_vec.size();
+    std::unordered_set<int64> axes;
+    for (int i = 0; i < axes_vec.size(); i++) {
+      axes.insert((axes_vec(i) + ndims) % ndims);
+    }
+
+    std::vector<DimensionHandle> dims;
+    if (keep_dims) {
+      dims.reserve(ndims);
+      for (int d = 0; d < ndims; ++d) {
+        if (axes.find(d) == axes.end()) {
+          dims.push_back(c->MakeDim(shape_vec(d)));
+        } else {
+          dims.push_back(c->MakeDim(1));
+        }
+      }
+    } else {
+      for (int d = 0; d < ndims; ++d) {
+        if (axes.find(d) == axes.end()) {
+          dims.push_back(c->MakeDim(shape_vec(d)));
+        }
+      }
+    }
+
+    c->set_output(0, c->MakeShape(dims));
+    return Status::OK();
+  }
+  return UnknownShape(c);
+}
+
 }  // namespace shape_inference
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 3a496e06ae..362899b947 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -310,6 +310,9 @@ Status ExplicitShape(InferenceContext* c);
 // Shape function for multiple-output ops with an explicit "shapes" attribute.
 Status ExplicitShapes(InferenceContext* c);
 
+// Shape function for SparseReduceMax and SparseReduceSum.
+Status SparseReduceShapeFn(InferenceContext* c);
+
 }  // namespace shape_inference
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc
index 3529d47449..de08a10784 100644
--- a/tensorflow/core/ops/sparse_ops.cc
+++ b/tensorflow/core/ops/sparse_ops.cc
@@ -39,51 +39,6 @@ Status SparseSparseMinOrMaxShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
-Status SparseReduceShapeFn(InferenceContext* c) {
-  // Input 0: input_indices
-  // Input 1: input_values
-  // Input 2: input_shape
-  // Input 3: reduction_axes
-  // Attr: keep_dims
-  bool keep_dims = false;
-  TF_RETURN_IF_ERROR(c->GetAttr("keep_dims", &keep_dims));
-
-  const Tensor* shape_tensor = c->input_tensor(2);
-  const Tensor* axes_tensor = c->input_tensor(3);
-  if (shape_tensor != nullptr && axes_tensor != nullptr) {
-    auto shape_vec = shape_tensor->flat<int64>();
-    auto axes_vec = axes_tensor->flat<int32>();
-
-    int64 ndims = shape_vec.size();
-    std::unordered_set<int64> axes;
-    for (int i = 0; i < axes_vec.size(); i++) {
-      axes.insert((axes_vec(i) + ndims) % ndims);
-    }
-
-    std::vector<DimensionHandle> dims;
-    if (keep_dims) {
-      dims.reserve(ndims);
-      for (int d = 0; d < ndims; ++d) {
-        if (axes.find(d) == axes.end()) {
-          dims.push_back(c->MakeDim(shape_vec(d)));
-        } else {
-          dims.push_back(c->MakeDim(1));
-        }
-      }
-    } else {
-      for (int d = 0; d < ndims; ++d) {
-        if (axes.find(d) == axes.end()) {
-          dims.push_back(c->MakeDim(shape_vec(d)));
-        }
-      }
-    }
-
-    c->set_output(0, c->MakeShape(dims));
-    return Status::OK();
-  }
-  return shape_inference::UnknownShape(c);
-}
-
 }  // namespace
 
 REGISTER_OP("SparseAddGrad")
@@ -446,7 +401,7 @@ REGISTER_OP("SparseReduceMax")
     .Attr("keep_dims: bool = False")
     .Output("output: T")
     .Attr("T: realnumbertype")
-    .SetShapeFn(SparseReduceShapeFn);
+    .SetShapeFn(shape_inference::SparseReduceShapeFn);
 
 REGISTER_OP("SparseReduceMaxSparse")
     .Input("input_indices: int64")
@@ -468,7 +423,7 @@ REGISTER_OP("SparseReduceSum")
     .Attr("keep_dims: bool = False")
     .Output("output: T")
     .Attr("T: numbertype")
-    .SetShapeFn(SparseReduceShapeFn);
+    .SetShapeFn(shape_inference::SparseReduceShapeFn);
 
 REGISTER_OP("SparseReduceSumSparse")
     .Input("input_indices: int64")
-- 
GitLab


From 1994b259d998c0a5c73107beb9750ef43b3797c5 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 10 Oct 2018 14:24:13 +0000
Subject: [PATCH 0107/1554] Update to avoid useless node in the graph based on
 review

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/integration_test.py |  2 +-
 tensorflow/python/keras/regularizers.py     | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index 0e6fc79dd3..dfe62276fe 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -318,7 +318,7 @@ class KerasIntegrationTest(test.TestCase):
     with self.cached_session():
       v = variable_scope.get_variable(
           "v",
-          shape = [4, 4],
+          shape=[4, 4],
           initializer=keras.initializers.glorot_uniform(),
           regularizer=keras.regularizers.l2(0.))
 
diff --git a/tensorflow/python/keras/regularizers.py b/tensorflow/python/keras/regularizers.py
index fd0748f3ae..cbcdae214f 100644
--- a/tensorflow/python/keras/regularizers.py
+++ b/tensorflow/python/keras/regularizers.py
@@ -55,12 +55,14 @@ class L1L2(Regularizer):
     self.l2 = K.cast_to_floatx(l2)
 
   def __call__(self, x):
-    regularization = ops.convert_to_tensor(0., dtype=K.floatx())
-    if self.l1:
-      regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
-    if self.l2:
-      regularization += math_ops.reduce_sum(self.l2 * math_ops.square(x))
-    return regularization
+    if self.l1 or self.l2:
+      regularization = ops.convert_to_tensor(0., dtype=K.floatx())
+      if self.l1:
+        regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
+      if self.l2:
+        regularization += math_ops.reduce_sum(self.l2 * math_ops.square(x))
+      return regularization
+    return None
 
   def get_config(self):
     return {'l1': float(self.l1), 'l2': float(self.l2)}
-- 
GitLab


From 3043dc1546a807f4a5a770853c11e4de9a3e205b Mon Sep 17 00:00:00 2001
From: mrTsjolder <mrtsjolder@gmail.com>
Date: Tue, 6 Nov 2018 10:02:28 +0100
Subject: [PATCH 0108/1554] update references and unify citation style

Refer to published work if possible.
Use (author-year) style in text and have links in `References` field of docs.
---
 tensorflow/python/ops/init_ops.py | 72 +++++++++++++++++++++++--------
 1 file changed, 55 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py
index 4fe6d05620..a7978a2985 100644
--- a/tensorflow/python/ops/init_ops.py
+++ b/tensorflow/python/ops/init_ops.py
@@ -360,8 +360,7 @@ class UniformUnitScaling(Initializer):
   A similar calculation for convolutional networks gives an analogous result
   with `dim` equal to the product of the first 3 dimensions.  When
   nonlinearities are present, we need to multiply this by a constant `factor`.
-  See [Sussillo et al., 2014](https://arxiv.org/abs/1412.6558)
-  ([pdf](http://arxiv.org/pdf/1412.6558.pdf)) for deeper motivation, experiments
+  See (Sussillo et al., 2014) for deeper motivation, experiments
   and the calculation of constants. In section 2.3 there, the constants were
   numerically computed: for a linear layer it's 1.0, relu: ~1.43, tanh: ~1.15.
 
@@ -371,6 +370,10 @@ class UniformUnitScaling(Initializer):
       `tf.set_random_seed`
       for behavior.
     dtype: The data type. Only floating point types are supported.
+    
+  References:
+      [Sussillo et al., 2014](https://arxiv.org/abs/1412.6558)
+      ([pdf](http://arxiv.org/pdf/1412.6558.pdf))
   """
 
   @deprecated(None,
@@ -532,6 +535,10 @@ class Orthogonal(Initializer):
       `tf.set_random_seed`
       for behavior.
     dtype: The data type.
+  
+  References:
+      [Saxe et al., 2014](https://openreview.net/forum?id=_wzZwKpTDF_9C)
+      ([pdf](https://arxiv.org/pdf/1312.6120.pdf))
   """
 
   def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32):
@@ -576,7 +583,7 @@ class ConvolutionDeltaOrthogonal(Initializer):
   The shape of the tensor must have length 3, 4 or 5. The number of input
   filters must not exceed the number of output filters. The center pixels of the
   tensor form an orthogonal matrix. Other pixels are set to be zero. See
-  algorithm 2 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393
+  algorithm 2 in (Xiao et al., 2018).
 
 
   Args:
@@ -586,6 +593,10 @@ class ConvolutionDeltaOrthogonal(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
+    
+  References:
+      [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
+      ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
   """
 
   def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32):
@@ -642,6 +653,10 @@ class ConvolutionOrthogonal(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
+    
+  References:
+      [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
+      ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
   """
 
   def __init__(self, gain=1.0, seed=None, dtype=dtypes.float32):
@@ -698,7 +713,7 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal):
   filters must not exceed the number of output filters.
   The orthogonality(==isometry) is exact when the inputs are circular padded.
   There are finite-width effects with non-circular padding (e.g. zero padding).
-  See algorithm 1 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393
+  See algorithm 1 in (Xiao et al., 2018).
 
   Args:
     gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
@@ -707,6 +722,10 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
+    
+  References:
+      [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
+      ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
   """
 
   def __call__(self, shape, dtype=None, partition_info=None):
@@ -834,7 +853,7 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal):
   filters must not exceed the number of output filters.
   The orthogonality(==isometry) is exact when the inputs are circular padded.
   There are finite-width effects with non-circular padding (e.g. zero padding).
-  See algorithm 1 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393
+  See algorithm 1 in (Xiao et al., 2018).
 
   Args:
     gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
@@ -844,6 +863,10 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal):
       `tf.set_random_seed`
       for behavior.
     dtype: The data type.
+    
+  References:
+      [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
+      ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
   """
 
   def __call__(self, shape, dtype=None, partition_info=None):
@@ -951,7 +974,7 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal):
   filters must not exceed the number of output filters.
   The orthogonality(==isometry) is exact when the inputs are circular padded.
   There are finite-width effects with non-circular padding (e.g. zero padding).
-  See algorithm 1 [Xiao et al., 2018] in: https://arxiv.org/abs/1806.05393
+  See algorithm 1 (Xiao et al., 2018).
 
   Args:
     gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
@@ -960,6 +983,10 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
+    
+  References:
+      [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
+      ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
   """
 
   def __call__(self, shape, dtype=None, partition_info=None):
@@ -1139,13 +1166,15 @@ class GlorotUniform(VarianceScaling):
   where `fan_in` is the number of input units in the weight tensor
   and `fan_out` is the number of output units in the weight tensor.
 
-  Reference: http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
-
   Args:
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
     dtype: The data type. Only floating point types are supported.
+
+  References: 
+      [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html)
+      ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf))
   """
 
   def __init__(self,
@@ -1181,13 +1210,15 @@ class GlorotNormal(VarianceScaling):
   where `fan_in` is the number of input units in the weight tensor
   and `fan_out` is the number of output units in the weight tensor.
 
-  Reference: http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
-
   Args:
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
     dtype: The data type. Only floating point types are supported.
+
+  References: 
+      [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html)
+      ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf))
   """
 
   def __init__(self,
@@ -1244,9 +1275,11 @@ def lecun_normal(seed=None):
       An initializer.
 
   References:
-      - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
-      - [Efficient
-      Backprop](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf)
+      - Self-Normalizing Neural Networks, 
+      [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks)
+      ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf))
+      - Efficient Backprop,
+      [Lecun et al., 1998](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf)
   """
   return VarianceScaling(
       scale=1., mode="fan_in", distribution="truncated_normal", seed=seed)
@@ -1267,8 +1300,11 @@ def lecun_uniform(seed=None):
       An initializer.
 
   References:
-      LeCun 98, Efficient Backprop,
-      http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
+      - Self-Normalizing Neural Networks, 
+      [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks)
+      ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf))
+      - Efficient Backprop,
+      [Lecun et al., 1998](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf)
   """
   return VarianceScaling(
       scale=1., mode="fan_in", distribution="uniform", seed=seed)
@@ -1289,7 +1325,8 @@ def he_normal(seed=None):
       An initializer.
 
   References:
-      He et al., http://arxiv.org/abs/1502.01852
+      [He et al., 2015](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html)
+      ([pdf](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf))
   """
   return VarianceScaling(
       scale=2., mode="fan_in", distribution="truncated_normal", seed=seed)
@@ -1310,7 +1347,8 @@ def he_uniform(seed=None):
       An initializer.
 
   References:
-      He et al., http://arxiv.org/abs/1502.01852
+      [He et al., 2015](https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html)
+      ([pdf](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf))
   """
   return VarianceScaling(
       scale=2., mode="fan_in", distribution="uniform", seed=seed)
-- 
GitLab


From ea684a74937fdf18ade43e42b9b320118f70d3c1 Mon Sep 17 00:00:00 2001
From: joaak <29533036+joaak@users.noreply.github.com>
Date: Tue, 6 Nov 2018 16:49:18 -0500
Subject: [PATCH 0109/1554] replace index_word with tokenizer.index_word

---
 .../image_captioning_with_attention.ipynb     | 25 +++----------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
index 09ea021c44..12c5eff2b4 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb
@@ -441,25 +441,6 @@
     "train_seqs = tokenizer.texts_to_sequences(train_captions)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "autoexec": {
-      "startup": false,
-      "wait_interval": 0
-     }
-    },
-    "colab_type": "code",
-    "id": "olQArbgbRPF1"
-   },
-   "outputs": [],
-   "source": [
-    "# creating a reverse mapping (index -> word)\n",
-    "index_word = {value:key for key, value in tokenizer.word_index.items()}"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1031,9 +1012,9 @@
     "        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n",
     "\n",
     "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
-    "        result.append(index_word[predicted_id])\n",
+    "        result.append(tokenizer.index_word[predicted_id])\n",
     "\n",
-    "        if index_word[predicted_id] == '<end>':\n",
+    "        if tokenizer.index_word[predicted_id] == '<end>':\n",
     "            return result, attention_plot\n",
     "\n",
     "        dec_input = tf.expand_dims([predicted_id], 0)\n",
@@ -1092,7 +1073,7 @@
     "# captions on the validation set\n",
     "rid = np.random.randint(0, len(img_name_val))\n",
     "image = img_name_val[rid]\n",
-    "real_caption = ' '.join([index_word[i] for i in cap_val[rid] if i not in [0]])\n",
+    "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n",
     "result, attention_plot = evaluate(image)\n",
     "\n",
     "print ('Real Caption:', real_caption)\n",
-- 
GitLab


From 3914ff714ba4561ecff329012d02c3de027ee4a5 Mon Sep 17 00:00:00 2001
From: Matt Conley <mconley@nvidia.com>
Date: Tue, 6 Nov 2018 14:52:18 -0800
Subject: [PATCH 0110/1554] Fix AWS, GCP, HDFS, Kafka and Ignite disable config
 options

---
 tensorflow/BUILD | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 859dc3b8d7..a81f132fe2 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -213,31 +213,31 @@ config_setting(
 #
 config_setting(
     name = "no_aws_support",
-    define_values = {"no_aws_support": "false"},
+    define_values = {"no_aws_support": "true"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "no_gcp_support",
-    define_values = {"no_gcp_support": "false"},
+    define_values = {"no_gcp_support": "true"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "no_hdfs_support",
-    define_values = {"no_hdfs_support": "false"},
+    define_values = {"no_hdfs_support": "true"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "no_ignite_support",
-    define_values = {"no_ignite_support": "false"},
+    define_values = {"no_ignite_support": "true"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "no_kafka_support",
-    define_values = {"no_kafka_support": "false"},
+    define_values = {"no_kafka_support": "true"},
     visibility = ["//visibility:public"],
 )
 
-- 
GitLab


From f60e430827c759755922cdaea80ae6af954cb934 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 6 Nov 2018 15:33:35 -0800
Subject: [PATCH 0111/1554] Quantization-aware training full test on MNIST

---
 .../tensorrt/test/quantization_mnist_test.py  | 197 ++++++++++++++++++
 .../quantization_mnist_test_data/checkpoint   |   2 +
 .../model.ckpt-14070.data-00000-of-00001      | Bin 0 -> 686728 bytes
 .../model.ckpt-14070.index                    | Bin 0 -> 961 bytes
 4 files changed, 199 insertions(+)
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
new file mode 100644
index 0000000000..e648c3388f
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -0,0 +1,197 @@
+import tensorflow as tf
+import tensorflow.contrib.tensorrt as trt
+import numpy as np
+import argparse
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+INPUT_NODE_NAME = 'input'
+OUTPUT_NODE_NAME = 'output'
+
+def build_graph(x):
+  def quantize(x, r):
+    x = tf.fake_quant_with_min_max_args(x, -r, r)
+    return x
+
+  def dense_layer(x, num_inputs, num_outputs, quantization_range, name='dense'):
+    """Equivalent to tf.layers.dense but with a quantization range between
+    the MatMul and BiasAdd."""
+    with tf.variable_scope(name) as scope:
+      kernel = tf.get_variable('kernel', shape=[num_inputs, num_outputs],
+          dtype=tf.float32, initializer=tf.keras.initializers.glorot_uniform())
+      bias = tf.get_variable('bias', shape=[num_outputs,],
+          dtype=tf.float32, initializer=tf.keras.initializers.zeros())
+      x = tf.matmul(x, kernel)
+      x = quantize(x, quantization_range)
+      x = tf.nn.bias_add(x, bias)
+    return x
+
+  x = quantize(x, 1)
+  # Conv + Bias + Relu6
+  x = tf.layers.conv2d(x, filters=32, kernel_size=3, use_bias=True)
+  x = tf.nn.relu6(x)
+  # Conv + Bias + Relu6
+  x = tf.layers.conv2d(x, filters=64, kernel_size=3, use_bias=True)
+  x = tf.nn.relu6(x)
+  x = tf.reduce_mean(x, [1, 2])
+  x = quantize(x, 6)
+  # FC1
+  x = dense_layer(x, 64, 512, 6, name='dense')
+  x = quantize(x, 6)
+  x = tf.nn.relu6(x)
+  # FC2
+  x = dense_layer(x, 512, 10, 25, name='dense_1')
+  x = quantize(x, 25)
+  x = tf.identity(x, name=OUTPUT_NODE_NAME)
+  return x
+
+def preprocess_fn(x, y):
+  x = tf.cast(x, tf.float32)
+  x = tf.expand_dims(x, axis=2)
+  x = 2.0 * (x / 255.0) - 1.0
+  y = tf.cast(y, tf.int32)
+  return x, y
+
+def run(is_training, use_trt, batch_size, num_epochs, model_dir):
+  """Train or evaluate the model.
+
+  Args:
+    is_training: Whether to train or evaluate the model. In training mode,
+      quantization will be simulated where the fake_quant_with_min_max_args
+      are placed.
+    use_trt: If true, use TRT INT8 mode for evaluation, which will perform real
+      quantization. Otherwise use native TensorFlow which will perform
+      simulated quantization. Ignored if is_training is True.
+    batch_size: Batch size.
+    num_epochs: How many epochs to train. Ignored if is_training is False.
+    model_dir: Where to save or load checkpoint.
+  """
+  # Get dataset
+  train, test = tf.keras.datasets.mnist.load_data()
+  
+  def eval_input_fn():
+    mnist_x, mnist_y = test
+    dataset = tf.data.Dataset.from_tensor_slices((mnist_x, mnist_y))
+    dataset = dataset.apply(tf.data.experimental.map_and_batch(
+        map_func=preprocess_fn,
+        batch_size=batch_size,
+        num_parallel_calls=8))
+    dataset = dataset.prefetch(buffer_size=tf.contrib.data.AUTOTUNE)
+    dataset = dataset.repeat(count=1)
+    iterator = dataset.make_one_shot_iterator()
+    features, labels = iterator.get_next()
+    return features, labels
+
+  def train_input_fn():
+    mnist_x, mnist_y = train
+    dataset = tf.data.Dataset.from_tensor_slices((mnist_x, mnist_y))
+    dataset = dataset.shuffle(2*len(mnist_x))
+    dataset = dataset.apply(tf.data.experimental.map_and_batch(
+        map_func=preprocess_fn,
+        batch_size=batch_size,
+        num_parallel_calls=8))
+    dataset = dataset.prefetch(buffer_size=tf.contrib.data.AUTOTUNE)
+    dataset = dataset.repeat(count=num_epochs)
+    iterator = dataset.make_one_shot_iterator()
+    features, labels = iterator.get_next()
+    return features, labels
+
+  def model_fn(features, labels, mode):
+    if is_training:
+      logits_out = build_graph(features)
+    else:
+      graph_def = get_graph_def(use_trt, batch_size, model_dir)
+      logits_out = tf.import_graph_def(graph_def,
+          input_map={INPUT_NODE_NAME: features},
+          return_elements=[OUTPUT_NODE_NAME+':0'],
+          name='')[0]
+    loss = tf.losses.sparse_softmax_cross_entropy(
+        labels=labels,
+        logits=logits_out)
+    tf.summary.scalar('loss', loss)
+    classes_out = tf.argmax(logits_out, axis=1, name='classes_out')
+    accuracy = tf.metrics.accuracy(
+        labels=labels,
+        predictions=classes_out,
+        name='acc_op')
+    tf.summary.scalar('accuracy', accuracy[1])
+    if mode == tf.estimator.ModeKeys.EVAL:
+      return tf.estimator.EstimatorSpec(
+          mode,
+          loss=loss,
+          eval_metric_ops={'accuracy': accuracy})
+    elif mode == tf.estimator.ModeKeys.TRAIN:
+      optimizer = tf.train.AdamOptimizer(learning_rate=1e-2)
+      train_op = optimizer.minimize(
+          loss,
+          global_step=tf.train.get_global_step())
+      return tf.estimator.EstimatorSpec(
+          mode,
+          loss=loss,
+          train_op=train_op)
+
+  tf_config = tf.ConfigProto()
+  tf_config.gpu_options.allow_growth = True
+  estimator = tf.estimator.Estimator(
+      model_fn=model_fn,
+      model_dir=model_dir,
+      config=tf.estimator.RunConfig(session_config=tf_config))
+  if is_training:
+    estimator.train(train_input_fn)
+  results = estimator.evaluate(eval_input_fn)
+  print('accuracy:', results['accuracy'])
+  return results
+
+def get_graph_def(use_trt, batch_size, model_dir):
+  # Load graph and freeze
+  with tf.Graph().as_default() as graph:
+    with tf.Session() as sess:
+      x = tf.placeholder(shape=(None, 28, 28, 1),
+                         dtype=tf.float32,
+                         name=INPUT_NODE_NAME)
+      logits_out = build_graph(x)
+      # Load weights
+      saver = tf.train.Saver()
+      checkpoint_file = tf.train.latest_checkpoint(model_dir)
+      saver.restore(sess, checkpoint_file)
+      # Freeze
+      graph_def = tf.graph_util.convert_variables_to_constants(
+          sess,
+          sess.graph_def,
+          output_node_names=[OUTPUT_NODE_NAME]
+      )
+  # Convert with TF-TRT
+  if use_trt:
+    print('nodes before:', len(graph_def.node))
+    graph_def = trt.create_inference_graph(graph_def,
+        outputs=[OUTPUT_NODE_NAME],
+        max_batch_size=batch_size,
+        precision_mode='int8',
+        max_workspace_size_bytes=4096 << 19,
+        minimum_segment_size=2,
+        use_calibration=False,
+    )
+    print('tftrt total nodes:', len(graph_def.node))
+    print('trt only nodes',
+        len([1 for n in graph_def.node if str(n.op)=='TRTEngineOp']))
+  return graph_def
+
+
+class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
+
+  def testEval(self):
+    acc_tf = run(is_training=False,
+        use_trt=False,
+        batch_size=128,
+        num_epochs=None,
+        model_dir='./quantization_mnist_test_data')['accuracy']
+    acc_tftrt = run(is_training=False,
+        use_trt=True,
+        batch_size=128,
+        num_epochs=None,
+        model_dir='./quantization_mnist_test_data')['accuracy']
+    self.assertAllClose(acc_tf, 0.9717)
+    self.assertAllClose(acc_tftrt, 0.9744)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint
new file mode 100644
index 0000000000..4e69206e97
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint
@@ -0,0 +1,2 @@
+model_checkpoint_path: "model.ckpt-14070"
+all_model_checkpoint_paths: "model.ckpt-14070"
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001 b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
new file mode 100644
index 0000000000000000000000000000000000000000..c1876b7a4c53a13696bd5a5aabce09a7455e36d7
GIT binary patch
literal 686728
zcmZQzU|`^_^D{jf6TV;U<%Iofp02W$`LSew<Mz1y_ZY72uX3KVpCib2KTph~edpe`
z?EmUAbH8=E>i#`_-TPUe&)okd-DKYq=B4{}W#;VvE#bZYl+pbC5;OhwC-nR87c2_g
z>-*GW|Ev_o1BJ5o`wxU2*ng<1b-zk&!TyqSbvDlJllMz~Zr^`%=g$4p9V@JF-oCuk
z_ebq6nbUH+G8VkCIyN=K`uNP%yB~Gl*md#YbF0(^+E!9Gy?3Q>S6DthSF|(a$)fE&
z=eO_NYWdruIV005>srsQ{6ivE%rDb+oo4Rc_4jd+wYNrqb(2Z_Zk5WTR+^#LcO@<m
zv<fou+{ty8!TO|bi-pW9)?FX;V|V{CS!20sB8R1`h`E)_v~?B+jwV*Z8#Y*VUlz8y
zo_XCeakhq4#Zhj{6W1PE$XK*l2L5|z@hG6z;&ZT@mA<^FWspLzMXRZu)gJZz7E^BW
zS+Y%fW+`KvVwF5i((2b`4NLEX@fOv>qLwDhR$BfF6SMjnC1I5_LDJG|_IC@>iQg?^
z*T1uUE|9$cke;#qg-i47xilv1Z^~!i-z@mT?$N)g`>v#C+OJ%}y<hd+&wbl1LhU0f
z%<NyC`e$EtirIeGBmI4>ma2A2*}M*0=9lba*m%s|ZBCJWU%^y+B_q52+fTi;_mZ};
z{rm68{+?@b`;B&+*>9Cg+^eLZvHylxn|<k<TANQ3=GgNnY46QnSY=z0Xl3_5=;FSb
z0I&V~KgrrBKY6#u@%<w^oqH1YyB`VcyM2-0zRcs@UIWqi{hLf3_kGzgY5&f`2K&o)
zzwKJ?@Y=t&5Vf6D@ND1NJ;wIQkCyDyF50}W<B-0Ma^jVJGgi#r*BCa#-o5Lkt=h&1
zHs@Bw+6nh8w7L6K;DEv7WA@F_Vh%prt@dxZq_OYjg0%f9(lPdiGC%B-Y902Sda~G_
zck=H2vv)S^PZK(1zcX&tz5@<Mb}3gF_jlb8w|BfOv`_cPDLbdLZFU=#UH31rXt7%p
zV!YoV?AiV`ZyNTQUuxKY<D#+cZ076xqf12h>r`E~JKWv6=l_bS`y*C(+g&!hweOnc
zyM3D#`t6yga_^4_RIs-wI<hbQ!67@AgyVMebJpxjzO-v!>B1NL_T`<ozoh!y?hYsC
zzB$HR4$k-b_swB5v+G=YX|Gl{k6nqnhh45@-Tq6{mhMaH^|gQV;=tb9?OFDbKOXF>
z*s^E8YkA+^2~&^l^IOWepO4AdUVF;|`<Q@7c7^Sy_s_q>YtInFXE(o3&(1~QrJbAG
zg?%$0uid+?hIPNV&ICJ)BPIK^rZw)paja*5gg*O$FFl*=yXMTbX*k-rFSpsu&gZ)2
ze&$Zb{lR{X`)kGYZSMwj?G@adyZ_oB9s6xRW9&7a+_#^xICj6<4H^5SXD@8eZSCLt
zCM(?9!ZgtSx~jvz!wlc{wd6dolWBOcZ*AXGyKK)UyLttc{T}-wZ67TXu@^B9*}K-o
z*lyzrW!rsc6z$~sn(a;U|L#jKUa)`qxs|r(JLUFnKX0=?{!{G!sU;fwzP?hio7Mh%
zpJ7;`eQ?$TJAJFyHo^{P_j~)VvAe${c;B?2PJ7=nDDE$*IAL#fWb(dKo)`8D$2#mc
zYx-fkV3Nar+l9GyX7w}e9c#Jm?4NksYfa|b@3*#QpRpT@{U0ZN2UCu_`}=n#+rPPU
zeQ)T(xBK#9?e=R6Ub6SHxMjnTGu19_L#nO!?2opmL^JLAxbE9u>vi7WG4G$<d#!`}
z9;?OgUvkOZeuKb;eWzVb>@=S-?o<EZZnxJ)+fMWN(!G(BboS5xSg^mwp<v%#=A?b;
zHBW4uga!6*Yb@R`JL9~abzg=ZU%`a^#z8aewVwRg=h(S)AODKneZpR|_bI)QwGV$(
zVK+Bb&EBPtd;c3j-u+9rU)sB}vU$(1ojvx=hx+zS)o9vR)F*GZGStX!_4YaY4g0I@
z9iCscKYZ-D{R_sG`=_4B*uR8v%6|SL=l$tlp6u?Ey=$lbaHoC#*M<A+#iZ@?SiabP
zc9!3#So(NhKiBI0*>5%XZ_rq58@74l-qT|7_R)-A?X2aW+RyVmU~iIq&~D1cIs30A
z{<70-yk~!;eVyGRt1bJc6*=2C2QgY(F)>?dv|H>x;M1|If3f=R3eNJKJ(beCd3Zdn
zR?azZm3jG$MVN%}&X#1xU7^=icc~rl-W6!svr|IJVuySDDa%`L!gqvdOYQu0qkY%R
z%TKI~7HjVIvA5c-`ts1OFR#|_I<;h;)zz?XR(oxltxU@`c1s>{vU*)`d)I~)dv|S{
z_GqV^ikIc}-rKv(f2`PbTa#_~;sq?b1?GD1Zei2g;n?DDJ>eC%RiD6VOMM>IUE2z0
z?&7`CwQG~{ja^r+*z9CHXKvN5xM3I5Z{3~jmQ#0KeCTPtj>&81EFp{C+n!XIcZZ(d
zWzOJXwO*mj+I8yYU7b%NcE2jCvzk#Yw5#D<yjAXjm%Es!6z%+QV)3r!4o<t??$p^;
zbJbz@MH9!}AJ(Yt`hG6aI{npj%XNK8R>~1NmQN-A?wZM*v5V&@pJm+sojX4ZUfj9o
z_`Mx7S>ty-VHevq*;dV(?}WZ}TByfv{k;#Z{z#m$klf>Gr8cM8x=KuG*LFv>T@fP6
zJ2}?gw%n=`V14@d9;@I21*>lTQ@d_R<nQ8``f8V0`te;q({5XcP2;luspVq5`(U4C
zS51iJx=_Pirym@)Vhpw4_4*m(E)^e{U6W5|TP=R1VEu&W!Hy8#f}J%#Pww2{*09^G
zWd80g)o*r1_b#=3yNqjB@mnD)zqO)PcZ#)lhn8;M=^d`Kt1C=>cd`6cD;5zctALud
zyXxhwcWvc=v@;;kW%ulxi+BAm6ttSnkZkR;DZ-L9Z@FdVt(oQl$39tJHQv3Wi%nve
zHoK`+);$NSxDClxwt`Z-Sf^~ZxNv`|)sbB)Ryz~>cZaR$-j#Eybl2=>6+36LJ6q@X
z&)<1}&(&QkSsg9Kf6m*v{k+buPxm9Oj&t3yJS(kgb=xy>mu8sD&eNB9b{jO!+O^rC
z*J@?4l(ln}f<=Vm2TPyExmM{*b$12rx8JE)I?HOJ?HsFH6ZY>&KKjOtiD~_=I}`FO
zO=nA5AIK=T7EQ0%eIaI-<+kVvJGt|}ncpf<wSK5_$m;pZ4OR@%>^qxt-L2-l<g;#)
zZQ12MQPc8|*z8@mrgd1>OxE6&SFE&KTQ=O<ZoaW~|4VD@?Ukh#zovb$ER^Tn<+xVD
zx_OJ!t}Q%<yS%n>?No}lwEDfh$J&v9%}%zZT2?2@j_*1>XVPw`-*b0YUbEWmD=KDn
zRqV+w-L7T3>@=S3=+RKvU9xfcu5FzBR=E!ZcQ3dmWK~dhc1Nai(yrUPEv)+9Yg>M<
zoUr@*MHXwTGYwWVxIL`TNwIE^<ImpZrEF$-NPD?ep3g1|yR8np=G9HN+Oj6nGViRE
z^~GN-yRK&zSta-Fws^B;rq%lP-ragD40r23ZQmu&|7h3Ga~+l)-i4NrxDM^oPW`ac
zBD26k%lPuHgOk}U*c)zJ?udS6#r1WLrKT98<pG|E-FmOMtzXHhTTjilwytm$vOM<K
zXxEu`3#+fc^{pyXwp;$1n6YzT#38Hoo=rQKE_rU*?<loPy`s<RRsK1vFSCoSmEP6v
zRyftP>%M37uJy}r>{Mv--6iPUWu;}Y){40x$<p_OsnwcKCoP{>a$D9W_gEP9U9)7Y
zwX$&Vd2V@ZzM@r7!&M8{D~B!IFK@T(58<?woEu}YlH-s?lj1&$wJg#Wymu~I7JWWs
zp%g4^aX09(WwzEgbEeHVEK-tLEN3(XS^l<rV#(vR(c<1x5sQsWk}YmBhFY53*0L0n
zoM`!E)e4Ka4|^?U{TH|RFLS}-o}Qz{b^hs=e-~L>WY?Urv_BJJ)zltqX??rfV(E<8
zmRp{mwfJ}<-6FzjwnfnSc@}3cJ~4m$_Kao8+c_3|9&Q#(7v8oEzRzW`L}Z>tji{96
z;z$|G$4vJu|E4xs%yHMX_}8jzaXZ=2vdX*Da%yv@rH|A)bN7$eE&8{xS!87|x0o*y
zWHE77f@S6bYm3ge$1S~*Cs+yUH(UNIzhW`TJ=5~Ssr?qkPjW48YVWmBI?!TKU}|o$
zv1p3r@21@rz5+rPvl)I^vbb1Tn6KJy5!pP&@<O$?<(!^XmW@*nTD;jf)55F8!eV})
zkmZV5n=E)r=UGZ^++vaYmfMnVcA3Riqk|Ufyjv_xf6cTkziepHYk%3&_tG!Rt`$2i
zB!o^{?4A*4xmM+sh3s`k3z53Z7QOl}ER0_KH)mdO&r<vHT8ov>MJ-;i9kMh|b+&M4
zXtbEyvd=<mdY5JX#ch_ZqK7PYewVSRT^(c*Qln?N`CX-j$L<=-ccn`$8ucezykWPt
zSlVW4nbUa4BHmWT@=a2S#nf%<EmIhxt@gf6v?!lF%i?jHfTeccWsCYH=ghTjwp(1T
znQYNk&TA2zb;xpcQ<+7Uou9>ryrY)OAO1BLeLcfMVKIy44INj@zLJ@ikABXy*sW!1
z@kpe`B7S{<Wng!;WovSfrAhe~bIDzsEY4hcZysdx&_bxH-9qVNi)H-*Pm8l+$1Jrc
z%(P;=Jln#yB*j8h+|1JC`9%v&!O!O2r*>PYPwTL#I-y{pxqhMLucsR=CX4e}OloJd
z`rES0{9oXDizh~nmZBWGmSuMHEYB_LwD|Kc*kaA!P78~@rj{-tnwEO=f-PU|JZ&*&
z_jwDmy(i7H_aC+}n<;47`0=#mqhrPvN!vbH9w>fb+55iKGDn}u^5{ByOGBUA7Dx86
zSTOqXT1Ia@Z{g!+YSEGN$}(8G(L!BD(jss3X-k{jITk1C4_drF&TCo2Jj1fAYrf^h
z?Bf=%7sy#y<ZQLLZBSup7t3JzA-UIbXLp&!ZjW;oC3|LCJkHo?AtU<EVi#AQW!^PE
zi<YXBmdoS4tW2lJTISX&SvJS<Sst|fW0AW_)j~FMf<>+33X6{Yb{2Jk2Q39|XIs?m
z^tC9+erP$ldXYuylV=tiN(C$%eU&U<q^`4E({<gV$641xdEpj|q!%HU=5^eb`E!yj
zmlW_=gqVD?a0!`jQGV`^h2q01i?Tu{s}6+%3y*16EX}4gSQS~NS$<omWI3@n&N40V
zkHuq=3X6H=JeJnH(=0MuJS@Ji-eH+!u*kybkd#HsEPksJ<_e30wwEjdS9w_0`*m2_
z@jtM<CU@N8%?fUd1osIR+l;&{KZ@B~?voC%3_BTR(KYqG#rD#0i<3Lo?c0{_y7xoe
z)P0jm-S&EhM(<^rTfaBuN9o>0H~4J+CYSA9=vT6rF?-LxNlI__ZB>}NPx{^8ef*c^
z@A;+vYR~zA_PthyllIE*E!~@TsCn<d*QI-p#s%-yI(Kv*Pkqha-CJGuv83hgP57{J
z-;-~T?C$8t*(Da1?R}Kqy020`XD^4F_1>$q*6$PiEwz7s%bI-&MXUCDzvJ5<^H|8v
zQbu8)`r}=DZPz#L6&2#%|K4ui-W?BC?JE^Lv3IUnz}`0R_4|@uDeui%5wice=fr)p
zxa{|7t4`h<9&5Ar-`mo?VoQ?un#9}fE&Nxy*F02p-<*H-dmoCI@3nYYx$nTU(!FKd
zJ@?Akvsi(~0`ji2?*h@t*vXG+CydX*S7GTjW6KU+^*5H+X9Vp4iEFTaMA4JixWN)6
z4#I2`WWnZs5N-#n*&cXcmj}C-rM-w0n9ssG6RsAd{^~ukUFIo=!E$__J9a(3aS}yd
zfblZeEfZF~MG*(-z4!SA*gYUNE<CTA59}Uv`3tlDnuXK`YKCu)Hh*_)ndVn3b<GuP
zXK2>D9?-C=S!O1YyH_o5XNTIKgFY4?wl6WSbXBr&6cIGrvwy43F*5;6&gciqXTSbd
z`=&3bmJ$_a9$vmu?ch;Ot&bl*n)Cc$ref(JXVJwbtNHTibMrrihUUg1f)*E6H>rxB
znPeUpn5kuE;H%kiE>1`2M7_Ctnwmu{`vvu{3%{A|3U4+S%+@nlt}s{QhP|D}*{mn#
zUv%0u?=7EbW_Tt=<@6dRtvfb5)HcjZS5cLjV!m&gw`$wQ6rCBo!5UGkY_y6_%BWSB
zODT5*?^C-uom+F)=40j;XNzfs$nt0yx!dV1IlD}=$1_LEx;o3AZ_%v1B?q$hpX*t(
zx0Q3%?vq9~dl^G#?(IC~WWQnSuf2z>kMGTKs@cECFJyo9{T{mojlT9KYR>zvBr)0Y
zz3Q{E7LnMO!_&F9_wx$-iMyEhDQ@_>JN1yO{oRfOdlQ{=_AkHpeeWT4mIH5t)a^rE
ztq+)~XIQWOy}>>)<mBGKbvk>v1ZV8?a^T<Z)lz6D`&VbL!B3C<U9KVan`X?~SD+-n
zmu>60y$9yWAMl^JcF()EtM)hMUfo-!-(W4rV!O}Y@7^BIhGTn$w=di)d^Kd>j%f*d
zUhu8hvx~`Y@3R%Wd*-b<xM!Ve*xoZM`t5J8;kSCbRB89t<D&a^l-%EYZN8LkLf3Zt
zwEN3!?6-C7>vJ{Od&W4`=4I{My&^>>dp|L}x7+b+-JZ**f9!eoIqv|^!NYcor|sV>
zq%vubx9+~ZWz(1KKjx5c<J&rQ@0lOR_Abn*v~!;@$HqypY|oM{Q*Ac-|JWPu@_pYT
zlNEdOCYIT6yTH4D5ASw6es+d^|Cna)6WbiQcVB|UUI|U#eOsP$9}um3v0pmDXs@}b
z?SA<cm3A)uhxf77KiIo!&yBs+?|<3~D<|wp-TQQZ*ugn_w?0<0nR&!~&x>8=Hh<=M
zAK>i^+w1hCYhO`G=idIGEc;Fr$?pDa+q+lg$i=;DO0Vx->}YTA;xTV;_{?J4@DH*3
z^lk0-UR%_=SM_`K{#`~Ud*uX*4phcm-|h3O%Ia>?r@a^dsO~)_8@vC+T&{f)%irwX
z-{Ee1d7<fkk=D}vhXot<&sApM$MJTjz53(Yz320-_8#A;y!TN%`~KbM=j=6C-fFXV
z*<ZWv>&y1q$Zps_q5J3FmrASlPg2RUfBngA-`UV5)&>nHZI!Pt+LLg2&fZ1&2ll<+
z`)~hhvG@CbtF!Liwxa!jjcB0#@5sb`CwN2l?)ueg6YFHKKkN3}z3H<w_Xn~G@AH|N
zzjynIx%(ap%(In0*=4`v=(&BHmAv;QxBT97>*&k9&lc<Lm3PnE>+pojHoN)4-igaw
z_A%;n?QQQpvPVg-Z=cwguX~T!TJF=B-(@dzuiR#DI?q0{1Dw|DOKNOBSKrucdS%^S
zt?52?{OKR}mYFo|z36>=zkR6H{<%jO4jlcWYCprPao^MV`S#OP*!FF?ps`zTne?8F
zuuFEgb?5IbOp@Gt!D+d@#?sck7ZTSW@RQWtbEVYvz?2!{`*nCv?0+Hqch9x6EcTpR
zqxSYIJlgwd(&T;CCDI2R-)ih@TC;rb(x*9gTt^q!sVqFU?@0R5JxWi{?cI2nb^oFe
zwS6&F@9Yw;Jm1?WaC7hP10VOEHrBA+X7|KSFLB{s!)?C%T2Bh>&Hk`{k5A{*y~fMm
z>^UvAWA6u**1d-NtoNOg7T)K^Gkb6RrU!ea4qe@Ane=7PN-?ed#a+z%RHV1q$<Jc9
zIWVnmFI%0=ZeNj~d$-1k9=Pym*WRD*jC<D|Wwv|FYG$|3?ET(ShVA=)ZN0HK=`Y)X
zYx}qCU6WqE*X0DmK5^IF{c{d3*ju)k-!@1)eSe!W!@jW4`F8i#%-y?U{n~wMGkNw;
zJfmV?FD<Z7YP!t<mE!(AR`0Iuea3TW-@h*t?caVpxNpbziF@aBAK0(7Yt#NWvU~UC
zi^uN`-g|WKhpU(OGRLa!J#>7oom9*7J*PO=+RS;hW}nNqO?x+F_wT<PGhrV?Z_GaX
ztjT+J@Z8;dQn+((WYy>0-nt_D-d=UJir@2k|K{qCd%IuB?)k#oxbFhX`Ms92f7)oL
zo!fu;p!?o}ESr7mnfGnnZtk|upT@JVA(MaKi%5<G&8z<Hbqsy8he1=`ZUwjZfm=qx
zdp)kr-!HZA{hn1yO8a$YKiw-bdHU|WzaOp3JiQJ?IC$+%ocv?=dhS>Io-s__v+>J(
zySs5~ZNA^uwN3FUw~xNMabK6!lfB!XZrrbt`D>5t9lgB*_pJ8yIUd`$-+STSyB~!1
z<(JOcXO#8U*6q6azJRxJdw4!I?tOXU?f&-Z68pFVpY5wnc)8d2ZQI_z#;f;jyFYVp
zeS76T>$*vMedkWzyYcdly}|jNHjYViY*OldZLV<q+B;o+-`<9hAiHDhH|@Q%eBs`O
zEp2-}6WMJfCSBNjSnAmRj+fng-w8JDS4z0O_u#Tidu=|f-zz6+zjw;=ko~{E|Jb{F
zmG*A+Qndrimi5|QR=BZ0{-1|^PqxZlhyA(sB|G$X-~ak_@1q{(eY4d1_ZMXC+WRFr
z*4p*c3%iR3ulMTMTOHVVD|YV*-aY#@zxCR)gdW<f{PxgZU(fG$k~>-Ug(Y3u>(kY_
z@9?YK0}hQ#_P>ZfvsZ|__5e$4^*)Uom-hX1Z{H(t_k8c)pxgUjbnn<}kkfBlZpOLK
z!(DOj@<mtos(JD4eag1aPMhcU-r3hm_bJ&e+#9~|sI^V|%e~GUuG_q-sN22G_R`*|
z4m<aYoj<?#?X~W`5|y_5d>{SUEBO1z-X}db?ey18uo2Kaz5n`ZKHJrL8~56nZQs4P
zt;4!-y7GRV#w?qSc}Mm>YT-MuX2G33-@9(@6Lq-0Z>zoAzTkQP_cDCHx94}Hm2I`O
z<=%Db9CqvX+3#^MHQp=L6ku=mEoHA+QrrI3Cu8=?DWBfAvee$*MeMpA^F@A};|@pc
zf{g@iwocC2+qvoVzQZ#+4+!m?y#Jf@fju4E$p`x8CE4fqFWBeLp|MXj#A5GvA-nw(
z)@NGvt8B0f-p0C@ui^Y&rxc%k-6m7^J~<p>pXPOT@2eN#`!4ZZ+q05MbRUB?<G!8y
znfDb(-`pd%Eo9F+>1q3?@$TJwkZs={m%N$#gsryj-T&<O-t5Ljc0Ziw?I~;L-_O%3
zv3Hti$DV1+7w;9xGu`_<;-Vc3@3P%2m23Cjn)75oV~*VZ$*W%4-_Kz`Ffs1--fu@2
z+g?|W+rwRRaaT*&roI19FxX33AK%L!eR%KYipBeT7Tw#+=Ko{=t<|sgGBiEkFEc@A
z|C5Bs{VB{x_V&as-rrg@Y44wBhjuM2n7{8pV$uFLQ<U~=nTzjf*nQ6~_8qJJi$fjz
zjD<A!I_^@~JN=HTo#|t~eP@(3_D|CKxQFW(mu*`b$KIVsr|&h*`@f%YiHgmpcfI>^
zoC^1zdCIxBXWIL{ZjzmQ3zp05b=W&?uWQB)JBzQ)HbKE$`=;ry-B)<7dGCbwn|mb_
z7w<b_BD7cBFYtiEV~%}}pKk08>6*V=L}KfnLlqPE-`W3t*NfMS_twVEuzSAb@&12$
zLi;DIo3!sCZ_3_h6Ry}@Uy-=CjbZ2B8m0Wb@mAdX*G;~^H~dQHUgr5;_HGH>`(Ad<
z+CS&X>b(=2rtDvNIo1AQ^KQGk|7LqFe!AK!zq`JdgY(ngmC_sbZ3yr_U~pCNz{`Y%
zdyDz@+sQ=R*ol{4-FM{k#J$ggMD`v~jokmkboZXrBck@J&k63`lu&J(dFJdstqn)_
z>`|<;w^^RF@5;=eeG|^rSv*eX-dpk3ZtsKJEc@2<s#!Y=f7s2x;F<L{>6X2VcQWr2
z+0eeP=i5e`#^$tr_a}4O*(NU7d*JqUJE_*Pz58lr>}_qe-5ccfY_Hb_wf)jl1owUN
zjoCZz(jGfs%|iQ2p^W=y*|XYxSuV71Qp246cYNmWE!uo)@7auJdr!2u?GHS-XV3KN
zoqN8d&$3(fdfMKMqW}BCc1hS=h_<rZDWQ8nmv`|#DY0ob3;}HR%Ze2C@Ez6Mljt*Z
z-<dp~{g;@7?DOOI?wy-;Vt1)>)V`-bm+ia3kg_|zfpZ_9^XdJ{YG3!}Wrf-?-f!6Z
zg>TZH)uyid4(d$ZE0DC`j_X3)-jI#E_FjJbeedQFseK<5zVDsBfPL>}Io-W{rgD2b
z6qfAg44AezYg5nO{~LPt71lBCOMBF1D{yIoz2B3@z0W55?f*Zy(MENB-`<3dm3x-3
z2JAI`E^q%?>890w{!e?g?3(uLKU%qW23zU=O=ffK`Cl6Bo~qbrFEEdFU##uUy$6KW
z*vQWG-CMk>eD8-H^Y=K%Ub0*Mb@rZlCG+-2vpVcuD0ggM-y*L4E&QwYbMv+A{dthX
zVaG)Necqe@?TrtZwr`f+G5cdBGIocjs_iZ1dB4vld)dC$+#~y3lP2wLJJhvDapuW=
z*=H_WF-_{;r^TYQ*UC_4@3qTJ``C9b+WXIG(*BhPSMHrsuCnj#H19q7&-d*;VQ_Zu
z!-y|?&n|nhSNz+py$8|+_WzDOvG>`Z^SgDc^7b7_ySI0pfvT-$s*An*1ZLad-xF*f
z1@79r*7o_{!k?S=zCFaV@6}08TfdJwd!nYy*c)Q;#a8L~BKuz=k_XhjYVX%Or?QXZ
zI@9i?zD;}oszmRd?>S>{cT)D=D&K^?oAqDY91pUy`*MMKuT}iD1Jf;3_PQ<Kc0lGQ
z_ugAOckDMh%)9RqyQ1Bt-_?7y_paV6ec|xFiX$Qp9uuzY>pQV<ue;mU{f=U7_Qo9x
z_nqvt*jryyws+&BqqfFuYxioo2_KmAMr@zl#eLR+j+^%0G_2a2b)eaPgZ-Jk1;uUq
zw!T}phflq0k7?<Ry?@<~?Y-UgdQT!Z>%MCCi~Egg*X>PM)wK6mV(&gJS=PM~(;BRE
zGXwWcp8I4k`xK@FNrgXcj(iE*+ik<TFDZSs&5DZ;?e&Y7?wK;9*T%PI|Nb4TBoEk3
zJiGtIvt4#_U;gebc$8qzdw;pD!_nz`HQrV1je0I`FU&k`uU6#Zy)pq0_NiZ%-uu`@
zZokILu-!TCa(1^5|F=DUNAiGZ_~gC+o=n`=apZ<I#{t*9aeUMFWr(ca|MTPZeQ&K7
z?@j2rX}?57-Tv(Ii~F=Q;;mKvcI-{dKeM;*)5ATXXV?xfg-+PpmO9l&ll%1^tzS#_
zG=AkgAXk*IH)OiUKAxO{y(f|iZL%FDcN_BFwc@gUwRhnb!9B8HrR_a`^6fcTTD@n<
z`F(rlHBQ-E-u}zx;#tN6#syJ(BaTYj>p51~7|fiocT>sTy^DNb@AcSTXdjUyyw|{d
z(_ZIWFYIJ5sM_y7U3Va<T*|)T<s}=LThr~zS?BDLa#^u=+3j6>l^nD7oB#T|<G?q@
zJ#%ws*<YB)xv&4M`vITVEc>FZWe>bg?6z;(60?8T>6*P?wr#TKf7-bBt<H_TF;ZRo
z9!RX--;>H_Uv6!+H@hdsp3R2Gp_Y5=z6cqgy|=fD>=XIBaG%eqKYKHT<?Ijh^6ae%
z>)l%sv|{hdU1EFoEi$v;_H)5rZMnjIi@)jYePyAu_uq$Wd)}!j?{(!jvH9|wZQml-
zW?Mna$$JDhb?#n$ifv!^+Q)kyipJP#eAu>s=@a?gQjeb6y?N5Jx96eaZa0~D8})xD
z_n!68wZGJyW%WDu{a$V@6MLm)x%PPn0}rTH``dr?mEJeu&T5;x?9=w1|FL230`^&Z
z6Yd4r@om4k_gtIFUKcTI`*~7|dzU+v9oW|OZEuMY<AHt0cG_ECp1kkoPQksd@mK9_
zEj0J|%@p2y!*9~QCC@SsTr0V5=X9`pkJ%Yd`?!*CcJcW)_T9VsWiOvZ@7^5~&g{Re
z@_6qef13ked!O#z#vZq)Tr_3Rnm&WQ`z<pZKD!^?+uNzX@8xQ_eG)FZd;OQN?7Jd)
zf3Jo7*FC%ox9>G>I=+7;`;NUUa<}be{^qrB-S(q<cc0GO`((*4ySL4WdtYw<YNxy`
z(dK6L{5@yC>F)jhmv`UGOTG36(*&&r8b9sf&fL2H;-9;ATX-JsFTT*T|Jv2td-Yx9
z?YCWR-g}gVYhUyezde=BjdrhoGVCq5(6Wckz|j7wEZ;tx+@J$K`}W%~o!V({bMunz
zmlI8P7x&8AOuR3%_vEbydnFr8_8xx~Vt4NH`Td`F$sJg#e|T@A1@Hc{xvh5d|2*Ay
zYmVt&HI??gg`WDhZ&Z!<eh-^`K;8M&?q_eE_Xvv#?u%7o-*^6x)PbwMQ}#OQsqTCD
z<LzF}ZOr?g-)yky-@0?}!{)U%d*WsH7(7~F_x(r1-l=x2d!8QIw%5f(Z%?G4%RcwC
zg|@P+<$I3&Vm;s-UcYz6X(n5{4J-G~W@g<hW^%!<nX`TOc7rE-S7x#wj9hobp83MF
zeKTgg-8ZZ6>)vVFjQe}0IPB$(Ke5+`=fz$b?`*plYgqRE(pbJHf!}9;r$y}^r!I#B
zS(?qZ#hX_iFx{B7w_b_=z{;E~>y-y5+r>2f-y4^6ey?QLvVGUOckCDZwrsy$nBd-X
zuY&hE@V?tSt@Gf%`Io!*{z+2ZcT+;qt|VjX-iIcU2UvAo_Wm}Rx3{4>+*ZA{VUK%;
z^1*qNHrVXpmD%^8IA*Uof5YAzJHPCiI_b||shOsG1AKk=%5HPncSg=+@3jSLdk?v8
z*tg$k%3hV9()%vHT(X~ScC5|U>r-tmO8(fJE^&455)q+&cj6`YCbI|cSFD`4CnRs~
z9<`17d)q4o_6Keg*mv&ZMZ3fuLVLy7wDz5A+PGK5bMM~AAC~TYSh#Tibhd>%tKWUv
zJ72%Vez7IvKIUt>`?v#i_TGLqZJ+AlSo`pUUu{pW{I&Nf+b+AA$9C>DJrJ<h>fGUd
z2O6I3dppa~cFRGzeU}$7@9X(IdAIO4#(j(a`0kzUd3bN$iv|1j=3n2t`Goj>%UiGZ
zs#euoKaAb8uVTX5z4={z`%Ye;vp0>2dtdmfC3~#Y4ffvg{<GI^XTzT8;@G_tI9m1w
z_pIM1_i^i9bDyhw*UA;{D?A*uXHo*ozP!78_FiJUw&%?;vje|MZ|*(4K5*}XBm4H8
zJ!x&j#HhUQj?2qEIk^>kH@sYRp!n)f`|n4d@AoJ$vTu5zzwe@6vYjx8g3YA6|Mu3*
zIKTJn-Mani2lV#{pP0ON%NuE%!z;q}b{?C&e}AR@-YB(S2gE!aZBMrw9QbjLXP=V%
z6Z?R*oO}05p4iJ}H+|pl_FV^>KY!RSu;7EumdT2CdULw%eyy9bPf?6<pLKlK-uxHG
z?7Zqi_jb80I<WES%w47%PTBlnw6wXkvf9Sie(r&@`M+$ud&>7!7@gY7ym#9k^Er!l
z9}$<@oBLbO_O|t-z3YFU+y8j+n!S^5*6&^CrLnJ_Wy_xV_m=Igjb6J?>Hj2~_bG4o
zW=Fo>%k+4vjfeL=tF~9C_wM?XZU5MG&R#7W=6x3{#P)kXTen~DUhsbXhu8OpPI<p4
z=o^#$gmatrvULjYVeDSKr>wARU-h1SyEki8?)|S7f57Cz5*y=v`3Iuw()L!&?K$8a
z_RB7qv+clfjYjMF&mH$$8FuUqFkP_sw#tot`);q<|B_c`pKGd;?OAJG`}>D__bDpv
z+&6b~(_W8%i8hDaoA%ip;ofV((YL?acf(#;gU-D&o#}fgw6yKLINN`JX=%{jpfjQS
zu9@)fWBDeqkHzr*-sit|@BJt&YWu7%aIef-Q~TZt+IzR#vRG9lsO*jU!Lu({hRb$J
z){=csVjk{Q5f$5Sx!lZ#>%hgmZ@MP#HDXw@H{T<`-s4KbUaq4Nd)PjD9k7+1Xzwgp
zxc{B{rG1XauiCb??6f!9w8-Y*s&ji^-?iDhHJ4%kGsPu)_4QZTL~Js!ul;*@PxGd{
z{i;`=?@9O@yML?s6#Eyo-TQxc_3f?EEVF;8v(HAs_37R$z0>=4zuL0z?|Ly?CYM>e
zjm5oe9Xj^dRo3L}dm?VN_n}Gl-nE8X>;%?r+Y>E#Z{N~;=l3psUS}&cJ$A3Z=lQ)4
zu72O|EaAA%-_CAd#EZVYyKZ*w@$5|9dn-)FCTQmxn<e{<_c~o}+_&nG@ZNiq8~1LU
zT)&SsAY|_^J}E1f?M3@_Z?o-H&|<M)9?`Rxd)}|T9!u8mN-WrI{Yd}d{-VBxdlx$D
z?9DfyXjd~e)c(0!!`|iR-S@p_;@?;A(Y?QWCgZ;2k68Ad%sIH1Q(63gQD*htqS7UM
z!vrSpfAp-!=IfD}c1mq>`#v9F-6!j>Vav8?f$foJNB1t-{%*gY*pxj&J=gYzp53rd
zYDI@#;a&&(<?}uF<UKLjf0k2uzrNM1y|xM8_FiNu-8)5L*?zgi-+OP~_-v;>t9!4c
z%_N&#<GuSj#dG%5ZPT{f=d{s^LqB3)vla8c^+ylyuE?FeCsLYwuVa4Q-Y4E%`;7Z1
z?YB*g-OCt!-bOObeBX;rMtcud@!ACDhS@J$I?+0rdD;GT=FhCZ{A}Et6}r=Up~(Kd
zF8?Rk-!l-}CpFb}uj14Vb_*VI960P?cfi?p;Xa2(nZ1)IY_|8%WZ4&4{eG{c&5^y%
zFCW;O1RmL2k$uA2c+MjG&Vw8GKG_m;AaUy!t6itH50qRzyYIh*!~uJu>w5yY{cP7|
zFW=q!VAb9Qnm6~kJXx{-(P1I`$$NbEdRZ$u2+j?&|NV8*-s2P4_pOiIwRgSu)cq?M
z{r0$Ad%s`xuhiaIU7z>tf55Z5li%I;rCq!I=Gr}b^BX<(g*us8Z)|+E=l`jtdzEKj
zwW%zaw)ejYr?vM>J^O9H>h=^`_U{#Vo3t;<M_}K&Sx@&cFn!zYvBPuk36;b996uTF
zamn!5YsTZUcZyWA4NKhreN(3FvH5a2Yj5n7ZFZJ%8|?nQ@874|*<@$o_;YW2Q|G>r
z**bfVdY<1aW}CM6Q^R6A^-zmFPu8xr>6wtUuetu+-u1`U@8j%SxaXoS@BW#>ru&UI
z$n9UUp2y}&o}4{ro9I499h<%X7fs){WrC*ND_0fU?XR@<Zc%=|m*aKl{=SJb_Q_9f
z-#bhA-rfMS7dAWn@9jNSG|5ghWr|I|$2%Kejl6xcp555%@~nHGhS8(F&$tu!2}qjn
zWqb2&&mI$xy?p)e_FmWL+k2y+eow9By#1*XeETc{&hAy#Y}+UQ+<lMg-FddbdSdp$
z=Fj(TbL_W!eMw>Oa=V?@;v&oT?oKY*bB^QM{`-ZxdtF*Xt$(CD+Arv+wJ)}Puz%VW
zNBcL=EcS{h3LOwWFncfioppPQ|18}5-(|}F(xiWT9C~)|75~g_@1Zt%@7x`G_FekM
zv~Pig?f&flS^F29UAq6%LC$?_vyR$lZ~wFR;qDDKH~+NmYjAzAf6DuZ_QE0D`)<DT
zwQqRx(T;27+I><o7xwzB5#D#dfqQ@KmOp#fan<a%{<O~e<n-@*uO(04^JdBKy-&_1
z+CS)G*{#CbvhT_7OM7LeweFo)yk_rf^IJAzAMfw&U$A2D)|=k@*WbFZ_i&lP-aq%{
z_WfEOZ?n;?ZEuRO{=SP7756@HPqueH)V9~#pknVk>vZd%xd(SYt9!aXY_Y`NPb)Kb
zNA$MXFOfWHKXY#1{^Kgr_DOw~dvEdZ9$0NBx95WFfxUSSy!#kj4(&ftG~Z@%bo<`O
z9gFM*mq*&BTJEstWaHbH^kA{=_DM(gF=q1b|9r1~ud4lid*7`*`=(5qwCCC>-hH{!
z{(EQtNw7C%*t@qYntR`mRY`ly-`v`_z*Kb~cWL+DtsOJ>J-GN~kMeO#`yQh!d()@H
z?bS@G-CH2J-{x5AM*D61c=o*)VBN=47P|YU=F+{XGJ$*lJHFo=d;5&dp{q>$608&V
zPv+gU_thkoJq7wR_c^clwO1`b-B#I2-mXUF&7PL?t_N~ly7zudWZiSES8(5}imQ9m
z7mC>ne{I|QcpbyOTeEHUM<=r!2#`LyFQiG%KKbawy@6kj+B-xa+jBYZ>P`-weS1DB
z-m!0x-m{nM%GNz!OaI!5)r9Zuy0>?~z@j~SCo@Ic)n1xsZ`-h9zfP3kzOpl__S?kf
z?|pOg++Lm60sFdikJxI8-m%M<ZrOW<-En_L{Vw}y@(cHsb8OyY(<8laPWIIO+&?S#
zdVi63*xARsZ~IjT+s^BT`|i&_Wg~uW#r{f@7MrEQ)Ap_8y}UQ#+x)$nCob;Tp4+{f
zk7NB_jvqJonjAS`|AXb#UWX0)_IB*^*mrM2(;mJv_pOX$ME0A!XWu*H0*}4-H0wQr
z5wUwc95(N*|9x@q$A#whyXHOF+h(+FZ<#>LzAYg??dB9Y?LV`8x1HhCavPr)Rr^+N
z-n{ooL4g(Xr<HqIQ~CE_4ZgegWw*~>t=&5Ix%0$#KR9Qy|4YP#z3y9!_s@5^ZfneE
zuzy0w-Mw;qwd{NfAMZ81m9_U(=gNI=n4R~fPTFM0dFkq2yZN^E?dp9trLWfP6V`6u
z`;F=H-j~mk_piU7y*J8=(f-<=lD&mp@AtHZite4%_+sz#Ijd~_Z*SW3;z!uNtcxG_
zCg|+m8^gMP?>V#Ly|It|_lk%t**kBC`QFpdukPh}v2O3$?aBMJj+*TK^>CHVjwzx0
z*&843JvNznzxnGud#mc)Y`*?@z1v*&`JUqT^ZTE4^X%IjbA0do)vIjREnaefS7?&$
z@3L+Ck7+O6yEUtL|0?SPdw+f6-?#7H`Ms&%9`AE(h}dg2m2+R~jf?xFRap0mW*)TL
zI6+}=&|D_F!+m%57Wwnpi!Iz{^B_a_Ko!T;y`SzZ*~^>OurFqVifw-;%YJnixxM0R
zdiPD-UcL8$$Ig8@?^^a)Pu;mU#cPFK-3pOCQ|J8No5|n1XNOkn-ke)Y_Wd}#fA6H-
zkM{-X-QD|pz5BkcJGJ-l+HmacYTvQvW>@vz$ECk)-iRpLoXk(%U&qn4cPGcYz3qYA
z`?8;(-)kcfxwqkt)BeeCKJA^FlX;*^hGn0WcY}>j@twUjZ^id|`F7YA%$dE%u)JYU
zoBX%^f7vhEPdR40KjQ5)`&jjhdjyZIw_7e`wdehp6;_-9|MxyvTDJd|@`1g-ch1@?
zp!v!+=)2P%$s@n_FH2azH*LPsfy*1T?af85?$7$lw69tEhW*@q+<WJm{I=1UJ#k-|
z@|*)}g>?6y7MQws>(O=l?O*@gr|#9VZ|&PTdrwZ)*>}g7b-&$A6WdrJodX%{oA-)u
zYq7l`GIL+U3whh1x&!+!>MQKM9O%67`7hJG6(;RA%lS6#<=pULZ@<RIy$^qU+3h$%
zeE+K^2J4`cEc+O~FWa}=+so$Hk0rZ3v+vrK#VFXkd)Bx&?ewSJnFn|7;hSx0b6)w{
z-eqezta;Y#+G~?(YQ3Su-u_#Hpnb>%?gR653iee7%k4X-pkROKdAseFos3q&)y;b~
zo2J+*KX=^Y!#&R?IeUrS<mNrQYuBva|8wKIy+0k?4s?i^*=6kHJ8;FMYwzUHmc8!1
z&3nw5w(d4NrnztN=7sxD`$pMso?x<<=R|{j)w!p3-_;lF`&J*n_gD(6O=LFTftvCO
zdrz;G*&lb5WnZsBtnL0|t8G=+tlqo+`wP2DCm|cb6qkL6>~#0qZrEs(cF1;*tsUDw
z*W~iOnHyeO9$Zjkcc^sDp1A27_K5tk+n3j3xc4nzjZMQWrM(8<-tKB=DBJH;_<D~^
z{GPo+lb7s`o!hrp`_mKq1!+(Bx*hPbd0#eRzk!pi{nZH!`yah(wexW|+9x%=*SdSh
z;obe87VSB#6~6a-))V{s@`W}ZODp#--Z0rd^&I;?n^~UwuXkMBd*n{xeufrShbQI@
z`%dZW?~Y+zW3S!YZ<Ay`f6v+<{rj#9Xz%;;iPzzc&EY+!trP7(sIRbZh)LVWTyoAj
zcTL;gc}(&L^jFQWKJ??Jy=jQvz9;FOdoQj!x_5r8!JZDQ9s6}no%Z#A>D?E{nZI|g
z8pqxjs#|R4uIja6tNy-MS<imA)vT}fQQ{MAmS(K9HW4n{x4B~R-m>3Pd*}Fs+oT&G
z-MdRb>A)%0h`kxtTK7auWZL7=aAfZ~mV^5=EUou4DJ|OTn6htQZ`+-H?f2*GpU^M3
zKPz<go==AFZEczM>`ByGwm1Fyy*+a!H|&e6nYlOi$IZPzuFc%bs1UMu%eD!2)f#Pk
z*CrU)fBJK0zsRI`yFVfa_Zo0Z+1+{ke{Yxgj=lQE6ZiZLy0O18Gs8|`&K;ZMd7JiW
zCTbmEom{x@MyBoFuz$aH2TV}4Uz&4!ZwT9z{r>+H_f>ju?(2`azwhnI$9r{ymmJ_U
zpS|yPt-`+DJiGS>Kd9Z?(iOZ{JFUXDa;f58{-#g6yHgtWZ$5BouZ0%>zQ9PseVyXZ
z_j226>^oO>b^lVX-Fxq5#~%0`bYgE;slJWE=0AIP@pbN%(=6Syai6QrWaqlQZP)S+
z<habW3w5vBzp~zJ|LOg6_xuUjx6j{;&u06T<9j8aUD)%z?#aF#)7I|UlohwP;jhMi
zwW_8)FOK!@-!ysq-Zc`<`)hgT*>4q0-p9fJc8}#D13L*Zraf&}w(W_OyR=Vf2IGO*
zrq%X`Bpdg7+HTr^&(y(wA6xW3-PKR_UNf}b6ZwD2zWj?!`)m?g_fL5JYp;FaoV|-~
zW$rum{Htwk&~p2!X)1eqf34f=^+aypY?fntohRSk+iBOocYBA;o|9SI_ln;zv%ASC
zyf6RE(!D-E!}jgecHJvl(7x~0Cx82WNjvxOY@J|VZNs$h)Z{rf*_Gn^lH^bB?K0}N
zXDrUzyJFj>-AZp+4qmyu!7k{9&HkD54(~nnUSwZeP59n9FO>EQx=8F<P|m-5h3f0Q
zz9+r*&Q_ka_qmPBzI2KBz562H>@Qp*zpvMfd%uK>qy5F$O?x%hcI?$&r)W2a(QB{!
z-w)PPj3(?8>07t|{6@9|{I)->^GuiTi*Ts33sgV4FMisoy$&LK_T(I$xW7qw_TKz%
z0ozm0+V)r%GVH7Wb7l8dqwc+G{rBuP+zZ-kdM<h2LB2zK+m)pDg;Y%5>v22UrfhY=
zUa6^VdxRG+vD=j(y)USA>z=-@N&6n26Wmvv%)Rfexvu^3E7`l(xf<<f-MQB0mbArQ
zg|K_p=@lIN+Wq+U&D9jxTaub))4rtFn*aW_eE|*E?c5oI_U~Ewc5jy7$^A$Ex7#ds
z&)fSVnsaYIqx$~sO^^3Vi>|e{Q%cwuym{lE<LW2v&MoWOJKL$#?un57e(rZVcJ^XF
z_Le{O*#BnxTdSPDKYPUH_w37zS!}nlWSiYGm;HP99Z%cORqMPz`pd3;ek*V7b^gG(
zx9>;O-U;(L_6hFcwB7sc<X%&&zP(K{tM{clEwcGKon`+ViKxB8d%N~s<Uh98f+2Y?
z*Sbf0PtE(b=fd{by|E7*_Acg<I^g<7d2i6!#e1H9FW#5cbY}0$|7-U$d@<k8exAo>
zR)x*p`B#efcHI=*TcX6WkB8fD?}WbY{f2s0Hf(H)dl}E}xBYv{eg9gg_4_hR-tD{l
zYmapSbHV;qGhgh@IeKsJox1yb9d}*WzvJOyt5wn4_b5Ko+ZX88VQpoqV4JA5WsjB)
z%l@50jt6qpV)ieZ&9;wKw0Zw={=B{Vhg<eO`QEqhoqX{=rmkMw4oQu@k2XKp`_gOY
zzUoJwd$-&@wwFcf!rsqEV-7t2-L|(c(`<kDn%=#SRrzc}z4q)|)y}%_;PyTHYX8>m
zosq!5@7*t<eeJuh?+q5axYuXWC!5|6&U-7BzVEgB9%$<^_n-A&jTL*77uW9d%wx6r
znJH^i%gVS<`4-RKCEu&}chBV6dsSY1Z>WU)o~gS`_LhCQynjc+@x4E<FW%euKIuSs
z?XkUq%jWO*<mR&D`O~;palu`?TjyJB>fKK7wXa>hH-%As&zA%_n-ulNy>jzD+qoZb
zun{}cy?>|s;XT&oGxj|Ri?tPcQnjz(q~Knm-M4MpRG#e(_n5HvE^Fz&(~b=X_SJ>#
z@9a?A`(Ur^fkpG5?VJDN#=g$nd3)Yp;NEBTdfonziuS$s%}#dCM;6)yJbk~XRchD1
zuelk!Lk&;aKd4*2_aUR<K3OHveSdOfZJ$`$*_^S}*~_z4W$!cL1$(ymKG{F{!aVEP
zyO*q_*3a0NUo(GC-ix4p?DInHE-!nq*IwcN{yY57_Zr>M+`EfMe^=&Ly}g>R7w_+R
z-@aG7c&5#*_?QEx`)}^;SaW+nr?T>XHV^ZC%=xnRNB)cM3tSVw_wS!ad)KG@w%d8+
z`CdM@w7mwCIqgI2Pg;e)58lsawPjDZ--f-3D~k5%Oq#s!V8@)js$Dzn9FlhJ6%Ur*
zo6od)-{buX2d))Q-M8uf+P!sGS`K8+J8l0yXv4meO$YbZ{t4JK+dIeh)+VNXt7^^m
z?aJG>yGH86-VXv__BC3)*t`9~zI}d+YW8+`{@MF<p~&9GyxF^&4t=rua%KJ=>2s%S
z;y*6ln{2^xV9ji%y%U(b_L%l8-?#d{>E5LhGxxdit+3m)<Ie6shnn|o|8{2YpYu(7
z`&yazi89sfT~^g?|CyO*-_x2)d$-y5?AH&@-2dvg&i<2kPwzi|<l5ec(^Bjf{N&qf
zH??H%+#9ufm%p>L|Hsd@?}6#lJ<s1iv0ccwX>aTHru~YFyZ4qPOt-i7>e$ct^OxP(
zyn}n|7M|Fj`c}mz%Z1naYSE>A78}F&KW$%aS2jyyk42cp{!Pq!`%j(Nx$oU<u6>>>
zF7DmbH{VVnm2sb~<(mD%y5@Vo@4agin#{LPDDvH2_sre<&0ZAjE1&AQ?=Qc`-b^L2
zy;hs+_6FQ}v?u6w_TKu_$M#-Z(Po=tb8K&J(xJW1O6L38Z~fad|KHBNx1+Z0O>$@0
zcjFl6fzqQNtdAwU+uaj)X75bRIX2bv_U(IpYQo-Ux}v)$s4d%jYlrIoZ7JXPai;y-
z|E@S<Pm>&nz1PW(-6i+x_I9qmyyyO9>3uEd`u1LNT(Nhv_+<N6LQnU`f9&4xCvbl6
zMxogKnNK#_CH^(vf6~x*@2o{Eb~<Z*?R_M0ZSQy9_I)Xx3<sD(yX=nLSid(@e5qaN
z<qP&VnlA0TzWeRo>fFbBc6{Ewr{&nJUAC*L_K9BQ-IsCl>23$z+I?d8Uf9$<F|hv=
z$F;AAS7hIjTSxb9vKO@xF45Yn<o|HbjFZf}`_!K8H9S3SzopRQz2%4A@A=HFw(o=J
z78|n+^?iSS*x9y}Y42U3roM09r_(kzQ{`<EkG$Bkt<+_2q)707Rojle%&psZ8ATe~
zB?_qQdzSOSPX6xIy)z#2?X$R5Xm{<-y1hA~1$)`OH`we{;<vwKGjngzy9qXS`U-o^
zv`hBpm>#yv&^)x4ZMl#Ak)6x;Z&N#BH!t(o?(U<N`*V)g?o~OvXOD?WnC*#4j{E*e
zHSG_zo3%IP!N0xRYMu8RJfFL7ox+b@t9mwA9n6W^-?#1fp1U61cHcgk?0s14WxG2~
zX5Z4O=l8zdIA^b?>k%6_|2g~0OF!=Qo~*gIb63*dJmJr~o5KC~RLb1iTP-ZH|AP4c
zz3P`Y*t~0q+gIZ{*XE>4ux;1UA9ngNclJI%Gtu5oqkOMJSK=Odt9N^^&T82EL2IVn
z>fUF&Yj<<+tNCraKh>INf8KL%yWMNv+L^C*wRN1sY5#hK*xpHZSMUAxK7OxxgRkB8
zV6J`5A7u7)F5|Ud`18wN+x~Ah8|FvuHJ@c~XA{e@-`q*We)7bpdx8!xwOjrC_ujed
zEB4mxUbrvZC)K`0X7B#Vg7<g5a?;<QIn8MgGh@R(({t<hDz^W%&VR$R-{Y*!-g%PO
z_Eab?*xTseviGQ2`#y;$OZHyhW4nK;*6+QqAE)fw`Cf5fU2FH=2b@dyoK3aZ`)lXg
z-7+EWd;RLS@2{QBx_2x0x4kxHHTxcLMebb|6}^XvbBkSb!K6LbY8eOSi3jb;(!RG>
z`uW_wy33dDRr~bL?w&NyzA%P@y=xmL9XPvfmR(ZNdmGKQAML#Eb?xQnZnu@F__<d+
zgk|4_+&_EUyx!ThI3L_yxx#q2r1#u?FHGlHC%C@Zue<!iUj63<`;}QV>@Qgu+neUE
zw3(#1VBbpT*gb0`y7xK;&)O$jJY&CciNJvuK2!JFi>*AcUa-~fLidh+EVYt*^*!hB
zowHnh@5D9p_AF~>wmB^szgMtj$6oHAXZLD}8SnkloOs|M_m<s!ZT9<?zB#>@KVaA1
z)9Y;ZhTOSn<CCMYJ9F)}-9=Yt?)^KFdEa)4>b)hCrtW+Hz{2MJU7>w!rt9q#p9$|_
z_#eLikmZcMO!p`5DRH^7*I#>^%}(*Xc3r(I_O9K3aj$Rm`uzc?*YB?|T(j?lFX#Ti
z?OJ;Tm)7p%iPztI&xU`m^TQ?^3$@GkIzJ!mk<4V>$FBNiZ{VAUdp0$t>{r#8Xp=W_
z-Tob(FYWk^kMDn7zH;vl(~SL3UNi08nUuXp=~w5zZjsCT+2l3$-+%jlZ@_oX{Y+aG
z_wsC7u<vMrsLd5d;eDN7rT6c<@NBPRRQf*q(w@BwqbBd26>YxzVinKcjBjoG<+}Ly
zEZX3@PlQu+Uyk!R8(G6|dwXU&?KSy$aBo@=<36b~(YBNJ^w@mSp0L*~Kz{GmsqK5Y
z>st2avmdi-pWeH7^0{Mnf(6|+mG(h<4!SSgyI?l+J`O`gd!wVr_H3WqxmR!d;(g&}
z+wHbgf8VEiXYRhYYI(MGJ5KI%^Ap-@k+EbC<DXr7zw8pVdo(3^FX#3b)>fhJ_DzB3
z_O5glwx2Khe)qTT$$MVcrrJNSzG-*2V#?lMi;C^ye*NE*&%I}F2y>v_#C8L_f)`dc
zazP$@BcIN(YYo)3pEL2~K7Rq`eJ3@e_V`s8?o~R|xOb1Wu>HGbA9gnvl-Xq6OW!NT
zer0cnoq*k;vrK!pEn?l5b@ImEyy+MAPE_Nv=?J*9$LdM(UWtk&dz0^8-m9~UXP?vb
zAG;kMmF>$Z;M~WavS@E-#*6*Rm0xV;+`Y4}_H*;z*UZ!Q-p-KOdtG+nUWxb1?KwId
z_RPEYcklAav-XE3_Sh{Av)X4L9%a|yv|;a5wrRF;ts*uNtq1qaVpHAw`-Q#Tg?T1>
zF9a{$+nU|H@2Nb)zN0=T_Q!3N*vs7~y{~nBxBWA(w*BXg*6rQSwSM2^;`e*Q?ycW_
z-{b$@CZ&1yuI-xk)kPonYR+ulw<Tot9^ckE`(_=vxTkeZ@80DWOZIzj2;WospT~~n
z{PDf-PHFAEyXW8DUA}Dl>SK89Yhqs6++aJsSD@j}p1+pT`y{zpZR}$C_a*lI*n6-w
z%KG%FAN!c6AF=ur`^fh2rJ{Wc#oG7!y<WX{Q)t8fXM!{L#(r$te<JhO-q|LKd(F1r
z-zysP-};Wza=ZKM>-P$(UD&M?&UQfXi1>b$TbcWPExGoqWK{26&8TR<;_b7&4^H*$
z{d|mRuMG#we({*!ds*eY_7?j!?e~g&wZ}0xc>mft*Y<RLIcpt~=de$0>Tx@Up51$X
zpIBkbv18-jCugkpein4Moo;1tKr86re)iXU_Re^-dH?jNqx-ej-r84FHG8i>eyere
zWD)zNi$wQ1TunaU`h;g+&hd&}&J*_ST{5?6Z%_QP18%oP_I-V*u<xwcg1wu#S@#-5
z80=NH{kYd#<-zWG^XBii-dt;Uc7^HQ|9b`Y-B>bt-#hD*d(S3m?n_9VV5jl;$KLLk
zvIG4V$M#k(Hr*T1%)Mvg--~;uWvAL5b6&UCdI9S`=GkBN+bvPCYfN?6U+o%Wx9P#L
zy$_aOxBFkqzwe}f&fZxkx7*CHFteX}f8m}wAx2wg@yUBd-g?<239R0CuxR7nx@1F#
z_LLR-ZM$>qZ)xq^lm5ni|B?7pd+%1}?A_qLc3+kLxBVNLW%pOfFW%c`cWeK@=Xv|~
zZoakepw8mGF8=fO8mgYLi-?!scXr8~{UMLP?yYMQ-Lq47j&0C91IxvA6A#!%o!D!p
zqq=VihtR&OO&xn}3mf*%T>WeB6|siB!jY@?a4Z(yue5N}o~q}ndrxzQ?0aOBx%XQ5
z;=Pmaz1w$4h;d)l-OIL8Q(x?z`lN2}A1=myD^B<9eXAF-KmJSG-p@~C_f}1Oy4PH2
z*}mZN&V7xKEcfeWPq*gmS+Q^0ks~&~1?)DjECX$(1U=f<d;8*EzNM%4u3o%sU&Vv3
zdnP<NWh-VPzBgRx>)vM^P5Ua^_U{X+xw==$`K#Tgrnfe6M=STL3FhqE|8@Vq-?24z
zDc5H1-MB+y-<`A<w%gV}-n(T!+rBTH+xJfSH*deQ_fhLB9<%qID3jg$Lj1wr$(L^K
zb74HRTX=i)KF898y-^H$`?ADe?dI@GvdOVOviHBnqdnf$T>EB6e%<RA!m+QmYsVfF
z32B?}n-llVNngD8ZhYY04}B}_+?h`7J$^*ufaLnnJxz1k_jnj3?5VYF-n(CA@xGk6
zjJ?rZoA#<6ZruNO^+LNfYrFTCbZ)WtvD>_Ni%yj7*|S&oh-ftL{onL^`|;a<_g-24
zbnl|jg?n!t>e^@Za;J5zcFO+EwY>YvYyR&G)O%;AF1Fp?_pY_gK3@g9f<LXhzaLv-
zvwTnYzFlV~?7#Ak@4%rKU3*=hN*u8NlD?ng#?^gd9aVd$CLiDXXiuXZV>_=+4%6oS
z*VixJ^Qe$(Uy9Aky(dGe_dNLFvd_)v>fZU6<@Q<V-rl?0<MiGKYd`OmKQFmg&ib&8
z*-??bKPz7D?>xJ9&&6M7Y*r~R-<LR9eJ{(iUwfT$-R&ap?%69gbIsoD)c1S0O262f
z62N3LS^dG@t-^Qg_NLw4yMABK-r}#N`?a2y?|<SaxBn|g)BcIic=l@8Pqpp+&a$un
z{P{i8_8IN1-q>t^!+y2RoP&(}5--@;q<>nux9OkPzEd?(dyn_M-xn}Jb^p6XP5ZWQ
z;o2vjle&+~plz?U{m;D|Ggj|=@@3_|oFaC6{rvX5uZkS^1=u$4Z*5+?PlDfUpYZcf
zJJ%{H?sqRR+iUE~w_lip(e{YuoV{(I*6*#HEwFd@3F-ZkI!E{HU0$%yTsdYh^XxBs
zI2&K=J#dhF@8P@C_ondf->bWN;@-c3lXuVfbIHd1oZ`Oc)1>wkY(KkKX|{rW{47@6
z=_bDR1+3pKn}6QgdmzAO?}nu%d-H4y_MNGEvFA_Z*S-Doy7vE&YTftwPr}|uLW}oz
zGA7tozj3j5zglU<<;1k_(UkXl-)kk?l~2vvD;4}}@6PvAY~>Wq_ExDp+P8FL^4`yD
z`t3}ckJ=gQ&9pCLow`?W#XsAt+vn}QbL{WlLv82wd1NoxAHBYPe`_b>Udii{``py-
z+ijbYw6C~+&)!?pIQQLkd1PmF{?=X(UmH8S{E$8A6)X1Mdp&QT+S_iM%U6^4olTS2
z8)mU{@2l=Rd$L+N_kHwn-P>lYxTn0}%^v2T=l1sbckMrUb@tvr(QEc5rY+vrpH#4S
zf2`s@#yFY%56<k_qvX+U*S%ZFR`cWby?#|@wvQid+w(&@e*YS^wR@M`J-t^!JA0eG
z|H^$Ep0nCt>)yT3xxsE9r_SX4iGN-9GM{m^771+FQ?~u!{s*a>_I_^8-do5#XMe()
zs=fD@KCr8NoU?bnd!e1L!R7snRUGZMZJV%HW*_r``(dhk`I|rPp2D(d-(9~tJN>Tz
zd+i=9+`F>>!@j4CJ^ON65A6Ntqq5gw?RM+KbH45G;k~tIZPG3~se?=Q*0#9q-Oj&Y
z-%{nNd-*2J-N$-s!CpPv+<klWpY3gaw`Q-M-;KT2&+qQ#tDU|#g6Z4dT@O0#YIgDL
zn|)ts-=9#EeRHC@_sT2t+dfa8y*K`n$(~Z7bo>82KW!GNZ``YL`^(-1FQ@IDHPgrb
zCCjG0y{R+z`nWFHAE3^@pSe|Ff4xNazHeS!du0<R*aRA!+8e?3ZST)j8GHYJxMHVv
z|Ips{O^5g15`Ss;Yxn-W!UwbVAC=|ZdvrU~0f&v6_P=K-94Is1wKwqV%>9=#GWXt6
zW84?l@3_||qH;g~f=~N?7A&yYa);YKr{lNX(+-w>nQC(T^p~3NJrOQ^;FSdX-X-fq
z_b<D|YNMB4wmUX%*Y3R=U+oT%6SJ3Eq;1=pYrgN9&*Ht=Cg1n`?fkyy)0f!2R<~~2
zl>cA0_sFSb`*O~;><TYc-ShZW)jr03|MsZ!F57eF@8A95KW6QX|EjUyrzULg2dzVU
zCpONvG3aUCdvk-<{(TPjc0Zr{bFay~EqfOpT4R&0E^GJsc7yE)p6z>gY0uoZ`fJ%<
zQQ6xzzZX^S_4G`%4>13`xAQcMt)fBQekY9ydoy%)*qvYVaPOvVy!HoHHSL#w)M2;d
zc<<gvH>~z8yu`LojqlIi(9Ki#EpsxpJHFts-FhXqeK%Y5?S9<7vF}FC;(gL8a{D;z
z_wK!6F0%je&nJ6o=Ca#n?_{u6`?u1j_mSDYZS{&a_o}Yi9%f<O$ElIEZ>ufez5~Tg
zyBA8V+-qgax%Wt?s_oqJMSCp{&$KhUSg`ldq}zKhE||Wrv|ehjo%M&^QTO-nJGk@h
zUO$#9`}_Ah_ZnOm*eiEu(q6B~i+e18*zDh>@nG*YXU;vRO5E%_(?j=7I@M(-Ti9(E
z_^xTM%T~ty>x>$#u74}CnlkJDUR%>2dpHcc_Ojg1-8(CtalfG8T&t;!GW(<IIc+x_
z?66PWcw)b3@nn0!DW~?7?cHbB%)4nXpWubP+ZQadjY)O1>wl?iS6#Vb?^B`qcJ4E~
z?6++?w~xDpW$&+0(S3aHU+wq#dS&nS-4}P?&h^?m=RLno!P<bmr;AVTwLR)@ANQAk
zpHSVmy&eMl_bl96wU>Wu!=4F$IrcrN_+uUMc)E>Rt;K<2-yM4;cCzlftI@nKvC?#p
zT6DTic+SneI~SVno%Gbn&g}=2ZO7x9y&slq?a9u(yVp*uYk$p^w|m)xeD`MFpK2$_
zZ@16Cl-VwQqNUxLnK65x$Xf3IyW{8Hs`#UO&nlnYD_{C~pa1>ty|cdt+iaCn-v1|{
z%vPqM$G%u*wT+gft-b8Td;4c?i?K1-#=3_=eB-{YuQ%>ZkUVYkeAfXx#x!kvpS2$L
zZ=VV8+p}K9eqW)({_2Wt`}B?D_a;rP-76E=YR~X_!QQMWZ##Y_i#<FKGxs*m=h-Xz
z_3d8ogl;>&zmxa*O26IvZ_ACn-1|lMPV4H}>ydtO?|Rw0HrZ(*dzQc1xZf>HaNn`I
zNxMZ)*6usEwsUXz&qKBsm6zLRN9?q5@O9tcKcjZ<)jJ&f)(5WNJL%H0z4MO0vP=HM
zx-X@1;@&m6dJdl_@z|B^%-=7}q`rUJ)46*E7Zln*t6XRE+4G*Yh|0>nzkN>GM;h1c
z4NBd-mus7j{e#}wdl!j3+kd>g!*<&Ct@~qx1Rb_@<l9DU{JYo4Z>9YlriFW6EN0)U
zcdT)r=z2%{6}%Jo-I7|j_rZz^`}-n4>`!V;+_&euuJ!#8w!Qxnx9=~|owoP){Nwgh
zzZ~9s@^Pundr{$iR-657Pj%@W(49JOpU3XfeRkJR?k!m6XZt7k!JeQ+)Aus{PPA#h
z)4BJ%+D&_dlP~seD-qo1``UHiiWh!+g(Q6U&VAQoXY#$>CQ;bNu0Hk7o_V`BSerac
z-&^zg$=;fKOZI+!!m!u3bJ^a`C$am&gktuEZTGVilnvacHR1hUHLDG~ci1krk$T~?
z_t=ed)}r4|*mB&rvEh2V!DiMOAv=W-IqRn)KkOdME#3P@`{bU^v(@_pHl481xc_(W
zggnpv0{cz&zF2i~@54#?`xb3}XS?#Q<o=3Ew!J0N&UU<d%J!ZvJN7Nh=GoWqRB&$y
zm+gKN=Z3wr@1@$+O{v&>OulOGOm^3O{^|Gj-e0P?@2{uI-mNuu`*yA2-d8Lrz0a=A
zaJMU;z0J2NntPZ2c(V7f#%ep$Q(ab?N7(kJ87uE=wqL*7M}%#kk;yc>oQgwxe6K3)
zKe<zS&nLI+y>o&m@715RfA3d~;Qb$uvhCfL^=)rj%WT_|&L{WRXV%+QE)hQvzcF`j
z<$10B&ArLC25Q^(-p^dQ_rvF>`=jsf-23n6T5GOnz4nPuuI_C(x5Vztmj1npj&b{x
zp8ndqOx<dq-rK;vE@^A_ixrCPTP@MOm)A95pKAL3ef4S%c0w{`dmdP~?t8#7ckcs<
zC;Ob<tlRUSLuH>c^WuFy4L9~m$(HV0G&OSX%Nu=rnN}RxXP@(K?*g9%`|K9`?p5+s
z-nV2H|32@1*Y_?zRBFRz`g`vgXY;+?{O|U@{C;y^tUl{L$-l4mPX9K0pFrQwy<q}s
zd&_mJ?a~#d@9~`(xMwZHslB(i%I}?*Ex32?`m()S!*=Z3o4IxGR2^}fR4-@ysB^M*
z?{;hN4>xPG=T_CSm04+Iw?F;yp3fqu_kK_}-?iq&<9!cW8~484cX981c~-l-ir4qr
z27KP1|6rAEAk*IcwadM1r&<WwJGedEdwfEhou-}G-WaREJ-(|r_uc1yyZ<Hgj{T=H
z*Y25bbZGyAGwu6!obTG_!lP=NvUuIzirhLIraKSzHidBRpY}R?Z{FYFy`T1P+V@Dp
z+?H?EXFHJ|k-d(U!uu9VvhI`kRJvE=YS5ldF-P_u`FX*nks*3-9ADFZ_v96O`>hx4
z5fj<H&)3&tkGBHTKIun+_CJ^D+OU23V)d_Rr_J+=f_vX+7VS2@5VnU|<=DPEx1FqK
zJbb-($Ba{Ue{Y=Ldxk@1e^~vKeVpbj`yRgiZ5JUdynBj+=)P%BrS?93f79;Q*#}m6
z8!lUOF6Fa-!+djZ`<CB!JB;7&J#u@2-M#b~_Pwk2?+;03-FHWv?ZBHqoA<hDZ?-us
zB4O8E_jYf<LgQVJ4o=$p=;3cW?c;s>z9#P3r+8R!-<dhPtiLSl-2b4;$hL)V!oDCO
z*?nu-oc3zlo!fJz`rqCuJ;k>3n3?xEt*Y5)e`mp7o6Eg>y*#@2YW!QVcWTbpJtp4E
z_deDV++Q!lzi;kU{(TB>z4yh2GucFJKVYk1!0*ug@AY1bcc1r9-)*&b%fd5zE_NmF
zjhk1xH(c1(p3CCv-Zvk1TC1hq+`lJymTjW!j(sAj4*PfCKE8J`@813O|3htZR7>}I
zo>{+_lkuv(B2)C<(gwRdk1qDvi}h~UbM&juftwcR_Q+3D*`IiDp*_QLZSysgXYG~Z
zRIzuR^=xn8-eY^W7Hr=arf}YN!Gfm!``i2W7PbiN+pw<G-j#pKK7qDLdwc$_+k0Z(
z+5K^!BKHQ^f3d&n!?RaKVdLH}kCyLi*|Kl%s<lZ6K3Zq(d%C=OU)T&an^d+<Ht+U!
z>{YqCVlT(+puH}_%zHxjcG)n;&Dwj}_50q)-8=U^{a3lSI_jIXsYvAkZU23{Tf7qv
z+-c|AyNYw>-q7%Ud;Qh^?Ol=}?vT5-b?=hpJ8S~PB@Sdg{JlHZ{?q<VwTta9_8i`;
z_Q}*fihK56yPLfG&cCkRYdv$-{*7DM_8shhzGt=Wj03+vbnN{e&21;jl(Dx+oW<c$
zM&thN4NL5ACq3PJTxGs(4JY5e-0hyWamwrVtu>u@K&MRF{_)Jty|cGO?6=AAKOmSo
ze;=2j>OP}A()(6Edv15=S<fEMK(>9ql4kBTdm*}~@kHmo^9{~>A8cHD!0)En-s#>7
z`&zA3_En|{?(KW?Y;Q{1$36GYBwEcbdbu~fuW-NcuQ_{7&&{y@SERCc(xWYViyryy
zYrijiaO%;&d$;Uea==5-ZLe`F%RaBo(KZEamV51TR@<KT<lD!5zkILL^`-lx$`;yR
zivGDjx`2DX^R1tIJK`?eU;ApcuSJ1x-%YVzo1oCc`wlg#?)ky`a_`%FuMT{+p0w8}
zT=_tt;E}!WYP=7ekeh4&Hjm|i<cqy~qK-(}FIvX3@4g4)zE4|@?^|Nfb70?$68khR
z9$V?p=KEuPw%T)h7w>C`7umP}>HfWcJ}lX*UOjv7HhK30D__pvdusBLy&vZ;-}htr
z#XU2YvD(+%IJT!(MRi|A+|<4L-GO@#Ge_EFJKeMieX(or?3Ii6a{pX%pl#x}y#@aE
z)`xa<?kj(>eD7hFP5Tmsr`Y@QblXH6<+Yz(9=-Pt@9{n6w<qttk(s}Dg1|{T<v`wj
zJ#+5vP1^eNK+^vecJt4i-0O36i{10B`)!=vpV+3YIkC539_PM$hk5sUPw&~^c7DZP
zjrNp1OM0f-eUjtayVw8R{(2ssefoRN52VkTXkR$3a(~3i&3glmPq$_F@7OE1e*fOp
z92xugy43f_#8&PXiJff|_cL+-^yiB9hkjq)=W=|--gSz{_Dq<rwO{i9*S=?xxA*vc
z`)*S@`{7;>(L4JTn%?Z~S|h%nS#YMUEt~5;AKOcNHSb^BTU@+$Pf0iDzV0&z_r@g&
z?t5EpwtrI4-o0nfaocoqr|t83b$9P`2dRAu)zkNL$gtYBxA`29oL6k~?$iCf9aFyV
z^(otC^Cp+e&T{F@y}NjJ?Nu$x+b=BPvwy{E7JDV*y?e~H*V_g~tk~~)@Acl6PL6%M
zeoxyIu<M$=^i;-u{#R}Hu5b+BKV4_OjrSJ^d-r#H_fBVFuwRk+b$=Icpxyp;6?->+
zp1Z%ZwZq1sddA*UGmh?am!7u2<>Mv04}aF~^?V<=|I3^v+m&y__SttW+Ph(I@7~R2
zh4x0#DSHp?*R$R0d}6QmoaK9WAFkcEw#mTuinQc`@{~V&9vgb@(+INPYq5U%9=^k}
zd%a3nY_GUI+5049-(K%83lF?Z{jf(P=kMOSRJMJ)mig}ez2W!XuU^b{JRy(w-r;Al
zYrM_Bcj|}7dwqW=@8#R~f3MXhuLI}K&f9zB`<cA~Ie+a|B(U$7kZs-fXm82^WyK47
zjeQ&U&zpSEdd6nOy^0wR_I}mp-tTa}d#~NB#=Vd4N!zJC?%rFJzkC16yokNqc1_){
z<)O0wLCf_0`!Ai`D{%A4{<q9od+&VXvze6Iu<vtkkL{}m)9eyVdiO5rTws?_a>6b{
zFlrxT!N<LeZs_a_OqqMYJeP6bOTokYi{~uZ>us}hZ|pC(y@Cnc`woe-?moF{`o7hF
zz4q-s7`eCi{ENNm=PaxP?ws8_W#%@U7Z+yl)-esT)7Uk|dgFA)eUmPv><h_Xzjs>W
zmc0RqdIuc)boaDfIle#b%@-SmHEw&)`-SY;9bvt<y>8L|UEj~`U9<bz-p?g@2kefO
z*-m&Ccwo}DU^~BLSsT0L4*TPW-S@`)JGgh_R;|4=9?##`|Lf%5AD&D0@>G7YS@}zH
zkNbq~z1v=9?CF}3Z*%Bv)c$}rcDv6KPxeN-a_;>jxoxj?=$E~ZFPZF{rSNb6#1BXJ
z|I~T6_bpS{0a=E*`!`+Nzpwf?lWn9c>%QD&Z|vA4%547UG3>iLpM7u5iqCsgbXfND
zrA@OfSt;zWwCdEJU$t-cYHh6BbMVu-Jsvhw_O9`%+grY#eV<6v7MmYdatE%=mfFkS
z`EIZHg9ZC~t}<F5;h49Nf8v4zb2VA^-QY|+;P~_U-pI$#cHb4)wfDgJ8GBFVp4`7a
zf@AN``yMtk#AOex+WB;!ZPU5~J5pQhBi;$^>)?=aNcMWXS9D$d-tCL_*qAXZ?)S-<
zV`H^YcCXpZcY7_?`R|>0kJ)~DZOWdtPg(8aSUC1~d^510*!6dB@UN|Q$>)~rU9<bc
z-mS;??>ji*+5vmZhx_HO&e>Z#b=d*+fSvnej+gICyM1Ku%hh}Le0uTR-bQWy-iq0-
z`{!sr+M_Jlxo66sZ~GWix9$x*)@^^|+$3whptpNX8in@BtZCX?E6l&|T2`XX(-X0K
zBb%1)Rdu&H;4L+Kx9zsAdrLOE@AGDUu$RTLYM-&#UAswhg7)xS{ku<7e(s(?mW`ID
zqipxGpK;j}uXuEy>*c+BJM@0<4KX~h|9|h({p;=f_aAMTzrQ)~)!y6ktM_p@Ufpw}
zZPi}yE6jTqCAZt3$jRJme=u}!PbQC@^i#FHoh#e-?{JLUyQgcG-JRI$`=)Z++3Uvc
z-FqstXFtyd>Ae#~+V|+(pSSPLFRlHz8dmMkdm+AW)|-v{ckcbV_pJJnea$Va_HO5R
zX!Aqry=@<(>|U=f!vlKUDz<8wEqlM%z1?Fy&2I1U13mj!yz<+3F28x-au)7=|JTUx
zbz8u)@5+e{d;j`p?JbI$y}LB(pq*2A@7~=hOZUFM>bUoOT)-X=7tOs@QpN|ezV+>0
zak=q;|GdJzE55Sq>*(Rwml88)@7v-(w)rpE_HFxcd~elC<Nf|7U+gw&$Q|g|%DO*t
z=M$TY`RDDg&)c!bW73<w?dMqcPG;iU=g<<cH)r#ez2Xll4n**C?+ZNnf4|}*16!Mp
z)B6opCfobFPv6fWv~};k*I(?6*8bSbvWDHpcc<sR+aBBp@)lg%z4=+v-j=UN_Pt(E
zW3SK@yl>K~7@G|Y2X~uBNZJRx_U&CUZ}GmRUflas!mjR_<88K2=v(LBO`;0+C!RW6
zXTDS2JInjz-j{FLY!bx(?QPh++vaiS)jc+Ia`)cMy1)NtLfl@Kvr&6XXNm61`!2S3
zcfH@fb5a}kUuDYNQ<8ggU-I5l*3(uT*}ExZkIk9@j(u$BJ@)?!ez~_^&CW(<&K|q(
z{0jRG%zEtmo^9M0u>a-WVBR@<bq-7Ky}DL-9}k<~UY+Tl_SW}q?U4&UVk11S%g%D&
z*1Z!VU)wYM?%lIN{hU3UK+JyC%Zv9-%}Lu^nAo&`+Ut(JMU1cZaP=SCx3_e`J}VFP
z13Rqk_AV6qx344Zs-4EtJ^S8I{kKP9-RZrYS$g)j#hUj1ie$4d_;O_T|JMS0cW2Mr
z*Y#-8-k6;`_8sZr-*+-7d|!m%$DQ_i%=>u#nfKjs-?3NUafww)=U$sYHq-rEYuon<
zes0@4*U^8Uz}d-rv&-gM7e8EPUoCfhZ`|a*{bAvU_U7h)+B@63cdv2xu|2<e!tG<q
zy7w&0E4I-M;6EUF*<#PVm5uf~Q|8;Bn|^VRDce!ILf<gkpF6kh`Ly)F-p&~-ZP+rk
z_IBQRws&*LQTvuW`8~VO?X!!hk=t7mddPl*K+C@EOBdU>=}p`#dheFqQ>SL@b761y
z7~VXx&uibO{h~bk_giJ2*i)`)Wqa^yfxVTV<31}vxqVq8oA<t2HQgp%(|E750h|3a
ztM_|V=kK-I#r0<2q1nIousZSDZ8-jWZ*)%DJ~ibVdu<=}?vYDp+H0_J-=3!9m-am2
zl-qkS$ZEgz<$ZgVKHS}FEfTh`>?XsW^WV1Y{h6S<|C&tQp7-a?4&0b1xi970?!9UY
zmhMfwKHZu_u4ez2l;3+<Os?<k=v{i?XTXzvUZ<7!>u53CKeg`L`}?woy;vF7-v0`r
zdxN)f?~4!Kw(pO9r%l}1&wD!b8|{vG?%eyVevf@q!p%KPx{updcKzSG*lnHNiT;Yc
z=YNOqyL@>0o`tV}?)87$vrp1k|G<yZ=KUeh-|YHkTXx`m?bH36o~_>Zb|&}UPdYF5
zZn2qY@1^<CdhunG{coRrx6U{DzjsE0{l5DfI`(q%oZRnolw%*$|L=Q`2>sprYFdoV
z)%R955*jWxe;>WvyHziHZ>OdH{>Vvx_Ds9_Z?A^)q<zU^toszsx$k=~FJ{O3KyOd?
zy88VgPb&6KeB8CSjInQz?2BHTsDxJgph*YzxTzo5d*!>|fi1tM+x?bau<w@Kw0&DN
zPwdgzsJH)Vis|0Qs8xG+e@xn2zo~O?%cQM)Q`VO3m3_3yuB^yquUW%}{qg_L?A>R5
zWB-}!qV}0byZ7zh-DLIKncsfpt<=3ISr_bm#HhJXXTz`k9(K?7zjO`So9N_VulzoD
ze+1*|eY1p{_D(#w-X=Zs$o`H8EB3@3+_A4eeCHmv1onN^{rmSiWj@}k)zY-T<oir(
zp|Z*Qj?Fl;_Z)-#-Y*{6HX3IR?6o<ua_^Lg_j}wf@YuT=z27@G!E`Unmo@vUrmx?7
z`Q|s9dDkTDUU1Fb<GX9Q-RvsXeMO2B_D-x~+dV<`>R!)@ar<^AaPDKZIkorSr`>xQ
z`8xM0#yZ=*^PID9VxRC{cFXpC<{Q55eYoh>-u)Wu_C#sa*-zNSZ~dnrX78%x&wC9x
zq-{PbU9mf08ekK(uX(qbpM?EanHBpC_&@LI&NQ^Ek}=tvwDi;7RmO|=72Mu$_qRgX
zUhsO^o}yrWyS#TycJZbc_ho0U+IxKEw7u_`W$mx5I=1K6UopFFA$@yKY__+(@MqWF
zF1NPb9T5j?ZneGJyU^Nx-_ad+_U35I?)&3@$*TBy{N7g_0k&eRT=tx{VX>DIt>3E~
zx6}HXT>U=n{La0mh82686PN9Op%ZUo*Sc<heU9?pZC%Uu8j2j-yJp2K8@nPCTZK)_
z_PSM-?k)atZvPavX?D}2+U&2_$k}gJ6W+^KzrybJsk3{P?&s|_uGwiLa71xG*PCCv
z)*tNM>t(Ur?kU^qy-M$l_8m%{v*+r`Y5TZu^w^(|+_lfa&}lF4yA?Jow>Q~b5a-!z
zVy3k3gX`&i{RhSD_w6{lx4`Q9K1tK6J#%ZP?>m2>X>V@V0-FNux%(YYZ{2&?bHDwq
z1@HFqiplShS?{s$(WS+E*^RdB-*;yDzUPO|@14NPy6^767kk~;9@&*8AG!C`SxH-w
zYmfH6>yfs9*1%+$W?a5!V@=CGK7qzPcavrI9WOe(U-s|bJ+8<4_NTcq?2Ej-YtQ9R
zhxguY?%O;6c$R%_dgtEi%}4eI7+&4K;enq0vDE2%FWboM`*QQ2joq=$`yO^P?z=7Y
z%v$%w-@O9-1^YJ_ZnLTTV7d3|MXm!cW=-8Ivr}$gqJ*w>(C)sy?|wVjUG31eFS0wl
zSBaU|e)ateyV)Gh?UlW{doQ<vz=6*mcJ^J8OnWb%S+Rdv$-({C_s!lHH{X7*;F9Ni
zN=upTw#r)X75VpM|AK7|d#f@<_r#sKw2#@7Y2O2f`3L40?AvXkCTFXhe|qn4ztekG
zrE~6^w<p9xrFZIHR<lZ*@UIj0U0<4K^JTq_&E6~1_ifXexhLUj$-e0u-rAn%dcW7`
zaomBoT#Ys^a};f|pX%&gyq3edo1uH(jeR{fm)0`veYVo(z&w_R`-}Ca?Y}0mZ=WLr
z$KJ?)i|iuU7Vb51eY2O@>cU<b`A7EY^M3D%dv<qk_zQl!#<-PxX0QHhcS7;9P0~Lt
zhYU{L14mjL?foxZ+1s$Wd;cz#t2R5ro%ZgIy|J%#>ihkLpLGwM{QY-t?l0Q|-_@n}
zzxc3dpT_$2d*clkS_y5ewGURGuvdTDrTzAIig!O``Mj&{<ElNnzc1`vv9aVpNMzN%
zNbMQ>E?ix)XOidZy$4%3_9;c)+jCzuYHwT4mc7eGT5XTt>)xBD61=y;p>*GZj|qF3
z51H-NH0|7%cJtWYAL|a<3M5KfZ|u6eck%lDd$Znt+QZ@IyuW0F^4?z?8TQ(ppKRy(
zGkt%UF3-OIX;%BoC!6m2-oAfdcISn?PY&JQv)yuy%`984{pP3B_ZoI+@7*RDXXn5r
zWnCuiX7?&j$EJYw>mKWb)BEL2XY5n?_H%FI2A_R1PtLbdi}BcdJFR@*$=LXPXWkv!
z_pDWW_cWi^dy8)I9@ysFwXY|%bFb|G^Lt*4IqiS6a^K#Si52^0mXz*2wIF48k6PQl
zRcROY_MCseXK`ub-UD0j?cI21_THmWPxjdVZnAl>ZsJ}eL38U+>7{$4D!6Ta|6|_0
zc(2UfNlS9}&7JaaZ@%d=Yw6MkJDrYQdzlZ;*w3@Cc~8=j6MO14`1c)lxVcxH?}goh
zrwsdiS^n%k(zty8m+LcZE31y#ud%nYT{~gU-oxKF*p*yTwBFKxYmdZ#|Gi>XR`yF6
zmhN>+$=w^6$!R~iLT%Sn&1H5ocD~))a@Eiwn_F=Inf6lqkJXFzo?Cl-UqQj*y>@PQ
z_qO$3*f;Bn@PWy*t@qF4oU})2qUwR%Sf%}3S~>efS-JOCJDlHn`ST%L(_4r4PSyIi
zPnhe(-gjaxHZHOo_MSHswyhGau>Z8bZErTm-@Ww$jQh^4XWbVfBf2-yE@jWhogI6p
z{Cu(Ze(>!5!d$+4xzeWY+EOrM-@B|cdxfuV*&7ogXdnByYVY?cuC`Y-)>v<!plefZ
zV`Q_}dh_0G8cX*@-o3l`z#Ng?sVkT5=lmM5FJ|uTeY1V}4scIr+FLk#?taxd^Y;7~
zPun}~_SU_bZHMe7N~Z2T^NVe-{+Fe8FQ<IpyVGXv{>F3b_Z~FazF)>g!v6Ymmi?-t
zxAt!D)7Y=}W3Tl~zstL~h?nnEZ%f;ERmtDJuE2WlM!}1AjIDq6-SU{X&*<sqy~}PI
z><#dow*S_wDSJcLwe0`2u+#RXaNM5X0$h9a_pjLN8jxqlcfw)cTh5k!%YQNL3uD>3
zCu%O+-cH`;y%yC;do#-IY`B6u?Hn8w_DoFvXB$~NdEZKwh`qa>h3_?Ap=0}X&k38Q
zEGqlMuKn5j+D&+GR|DT3Cw10+duA-%D?O`mPlUv38>?Ag_Q&Z-@6TR$$$nDg>iyp(
zENwkc@3G6Sys<a$mBha5k5=tE=pkZ%Ew*Vd>w~p>x7I$f3$)y@S1nk4pRxA)y@wmR
z?7x}k**d&mZQq&9U>z~d$j<oTwY}AfOKiMv?%v0E=EVNHmoDr-X4`M0xzE7v3wwk8
zeeXkipO*aH%fE2m-W%sW+wD5_YR?6|Ec;2hYJ0@CMecbSd3K+ZsjKb&n=9<C$}d~r
zpQycW-T8}q8zy`1F49-qE5dtx&q-Bl8`1AP``E909O$`udQb4Ow>IyV&Di&5^|HN7
ze{A3DeMWWPpPFlXKhBtHcRlLa-Wfk`?yb9PzUMX5Dx1!wllB*OezdOMx^S=3h3I{I
zr}6DqGwiWn;uOC>cT&{81k>gFosRbI%}t!V$L-<2y}rzM_t`h}@2yeu+<PMTo$bSn
zC3~&D&$i#bTYaB-ueH5ZlE{JM`x^G%(KWCx-M)PPzaQOuwY?N=(+Vf-`#C?#PH=Yc
z{+b&dd-gaM?K=_7Y4=QR&0b-hS$kjHT(mdu?~(m$<-+$`Z~S4$;J9lqgV}Q%zQe2c
zO@Ft@rgRnW{@qVn_qHFLwvSs%dawO-?tSKd_xBh*xoY!nMx*U#t=W73pO|X5k;B90
zS%>1@Ct)-9x#_>(TmRt6-W->6`zOa~?e*e0Wbe7--`*XyeEW(;x%Zv9wQTQJjc)tB
zR_uFE-#EJ0*?q!(?%ei0n~XQxZ8H?KuUfQyugrF1yT0godm{yv_k1nm-BrYLZ2vin
zJ9}e0j#++tywUd8*GYSMF81%gw(r8;ccrlhT9di=+VOtdpX|1MkG7rd{<|(`_Z~m~
zemD2r6?<8aPu+j5p8G)IvXeF%j_USGn|SwEn{V7VtL6G$1*hqI{4brg&D(W*Pt5)q
z`?mC#*v#eDwfWvRf8X<hX?v%1zuFg~yvjECz`ni5yE*r)p8s_3W4_kCycfRgc_emW
zuj$U?dz)-e?|(jlac{58bsP7GJN7y4+qyU6tH@s8B&Yqi#jC9KtRfC1T58#rPCK+m
zNRW5me9!fJBl~mh&du}N8~LrthQVUX{+pT;_kUa1Vjp26XMgo}=3e>3-1`$bcJG~W
zt!M8PzD0YzPO;g~-jrtZ|Jr8jO?I2?jCP*b+c&e_{&!cP&E5u0d&}6v`*}p(+Ql8+
zw6{H3VBgfe-21Zl-r7uVY1_A2+kgLK)hTxOIpg*oYk0gb_aTS9QC$7L7oj)zu5#_P
z{#88JcBRa_z0oqa?DFnk-YYy$(6&c>+rAt2Pxt)!zjObmmkahzSU7c`;K86hCtGUw
ze42Q5@AvWrd-;zZ-?Omt@@|J|dIz>{(B0$xzQLArdHz1j)u;AWaB=KAbFp#1Q{2?O
zs;&?0)N0uF9&(?%=To}8%|pwsJ-iXy_TBwIZLjm@@V%#ETI~NE>f68g_Z<7=u4{XC
zUVOLb#oQhH7JgZ`SN6TdUe9NRHcO>C>?bZ0+}D@dzo$j_*}j;WDtq^6EVs|UplGYP
zoX>%k>*#*R+j2I-%7J@@#hUlMTX|^D2C-RtFMPhTSJJ0!|I?#g`_1@v?`1WavX8q<
za=+<?Rr_Aq$nR}vGuW%mkhlMksL;M0{}}9-Pwd|NHSEq_)}$r-Y7cAfO?5tR_y6|{
zTm2^feT)+X_DxFMz5B?##d{0pB=0rLIK0Q@%hSE4lY|bq^0Vz@GP$$2s(0Z&k<?dv
zlNK!8Ya)KuZqCBLd(|pi?XvlQ?cL?pws*eUs=YmCfqSPf*VwOoHg#`K((=6?^6Tw{
zpJ(l}YUi?_vSyOq5(j=8mBli9_t(wXo29>J@1KSgo0bhrY)>+=?CU?qx%Y$6L_3W;
z^K3r4OWH*!aP2u<z20tb>b?DzR+sElPaNL!VegB5g&KW(ZM3fKITiNWn%B0*PQkZv
zzvL$AeXAt<_ImiL?QgohXWxQOhP{5N{CoLLn)f|&e6_daV)Ndqi+|YMz9VUSKjz)u
zd2JK-R=cs-&yV-rw^O=sUq<QEz2^+(*c`jJYp>LXNxPei%x!Ocy}vhV%6jX-o2545
zhV1)#zAWAM=l<tC{@hD!m$OW;H-0r?&#n_U_Um!)*z-9i-6qsDbFa*Q-n~sNo9y0X
z7w(lY?%pG*e{VmR%qF|TUuN&W_x#De29<w%{InG9cz&qvow=yT=8?$dy*@6R?UK^e
z_r&bGVk68r*Z$S!=)K)%WDh*Jt!`_lRkdHXAlZIFbIiVr6-|5fy1ng--wE&a$$PlB
zZN1yROD*O00iHMa#S3We-5)XAzNToO{VKL8`<zcs*ki@Gd#|Ly7yF#&5qnQ`&9yfW
zXWRF)S8?x^U%&UIeEz+gQLFDjZ%Xst{~}7ZUmhLZvoQCG&D$&Q_gY2z?XA3ba_`eO
z%=`5A$?ZSLAGlY7-FWYTYYX?;t-ilk?&J5paXY>D7tB1lmwRo~zOzp2_8f74Y9o?=
z%w|j3?7bC>IQK{UH1D<4e7pDb^2Yt4Dogeq(>rQE;ZuXn9$Bru`4^b?UR>I>>v;b2
zy=T`>*c-aM((cx*mc5S8-tVbhsA6v&_-e1~n>F_Bb7oj~?Yn1ZH*@yB1MKYkOIHZ*
zsf_m9=kkte-^VjNHWuvr_c^=i+2_?T9dIw`vRTHwYu}0otoDW6+xKQU&Dz^^D{(J_
zarS=O&vkpPEII6d{SV*G-L!r0Mg^&TbMNomJ$dnYyN54S_ii`o*e6iFcdy{qX?wPG
zaqjDAW!PuYE@yMSLU3QQ+=hLo_wMi2yne-YR@I_?nR<~ng<2tdS+;2H`(N~9uVBym
zy@sE^TgR2Y+`Fdd<zCtCSND1Z7~8H{8nairZ0lYtrn&pn{=V2Js>88Y)_~!FQpoi^
z5=R>MxzxAp=JHy;ccFaN-t_qv`<c^f_X-3t?UU8Gx9{<l>3h<|KknhZ>AUA?(UyI#
zH)rjSKl*r|<{{&~*K_9X->@Qc@47qewkAxr`-)~A*k}8Ct=(_dOMB-WowoP6>z=)K
z!87*ho||iPaKqQV@w(jm?z8AwAN<I_&!_y--i*2)o5$5t_8mEMZ*R8cnthdeGxuJY
z5VUV$)Q;WeeB8FP{7v@SoX*@^adp3qgb=?~Qh1)-DgmK=Zh^wP10U(`Yw+Z?d0@0>
zuV;>w{p2@+doAlK_K6)}**pK9v&|GKmVL^aZ2O*m4YN0Ju(iqBd~fgb_bcr<7c=Z<
zGFIB3BOkektDw(z?s8tc6wgn4nLYOGjjy%ZdrDE>UTvzl%?_`Iy<+oh?Spye?N!^=
zvF`)tJ=<AzN_!7C_3YpJgwH;GYu{dR?YezMWpnoOv0vP)d^>ty%EV>+59mDEf26N<
zZ#1vzeh!=d{TVO!?CTGDzjueN;oeVztL#J`Z{Pdp_niG(tNLtwye8NlPifzqb}Q93
zIN-nC?88_0O!*?WuggPW?+oS}HdnMtY_u-T+cURr^4_*x5qtOSsn{QG*s|y3T9$nd
z&x-e5U8KDC;Vl1sdR3Ec{XZ?<J8R#mees5Rw*FJ>cYpbQdr$b^^?SJ*z4re3&ay}2
z_0GLHIiKvD*k0`WQ1H|)jc57ZcYY@OKCiuKclsgc-ZxW^?Cm|gU@uGk%)MT+OZHka
z?%&&xywdLF=hVIDasusU-^$qgsM^PFLaXLpsiSNMdYBIHU0u6(zb@b7J$t4}?n#*_
zzOQS-{yp#F9_>#Q?A$v;vv{A~P1}81KJ)e!>=xU1`Mc2GbB=#(&jq#Y?Gf|a$9?Jb
z-i*!-dsG`g?&bQYvbVCk&F<|JBO9R@`+f6wi0-}ZJJF^#;f3`!pH+KLPT{q6tl4L?
zf9;ulqWsN!w@qHM*KuR?z9#oTt3Sd!ZMMA<v`_Zqw|#z4-?l`N)3zpl+ulCmS9=Zj
z?cKXYFwj1>mSeBu47PndhL`NZdL{Nd>{w~%DWYbdwM=;5%)?!FH>U9JD_DAFuiu=Q
z-B&A&><b%C?zN0xxp!e%?OrXHU3+u?wb*OAt=r3Eeq^5j+wQ$D3#1)(8m`-G(yimb
z<HWu9g!uivA`a{Kao+iE_h{;#z1<rQ@6B{MyKm}c1_!fmTlO(~Z`rf>*2O*6T~F;4
zf649@d8g+PG{bYx8cQMD^IkKoR#j@)l+25=|8a?XuZ%|EzWVcydy_ln?L9bUrOjro
zNqd@Dlx^LgEVJQb;I^AN;mTeYjjFxU2e|fetUte3Z|<bM>mGI4&ws{hvr^#mKF)Z7
zy%|Ld_ex|*?p<|@cb{d!?fsz@x_dnzDDG`Nyut4MwUm8Jsv_-JRv+1S`z`0beQLV<
z_0$FTe4b~r_t@Ksd;Rwt+b-C+bZ_Kp*1ffI|Mti2`L|d2>Y06Cjd=GO+HTpmqfE<w
z?rr0JvrUrs$~>66FJ<<+y?ZY+?h9dGzjyZ32ll$vul6m_Xtt^NDQcs#X{~+ZjFtOl
znTYO_wA{1T_SZx^hDoJ+k3`P0pJrsa_uJmCJ$1!adly>u?5(hQyq|qR%|4C!f&1=!
zI<oikc7;8)ee%0A4m{d3Wx@MB>&`apoqA};-s68J?zPrlytn%D)P16V8us|ze7SdS
zNBF+<Gg^CV9HjT|?BBMh>TKN}(VED;i<~+3$v@SyOP%y??|IL38|{bw_Nyfx+lq^A
z-(|h_`rZ>fU-xW!YGo&J=i8nWrZs!FSFPR?VYXo3r?T^V_k9Sp(fjwpPF?Nh-XkvV
z_BEgQ_wl{3v|B#0YTxyzEc^Z^{NFqA%t^cYpjI0O*IRp|y0fkKzH7CA8sV`w@V0@)
zmM44ne->F{7hC1N@Am)8dsnjy@4K;8V&Cd+m3^$8iTh`1P24--^z6MGN|*1edhozT
zU{SQa*`$U0qGmSj6PNhEw_~#6KJO<idrx<-v=KYWw0G8un|mcz81B>iblX}rPQ><1
z@sfSB4*lQT=k{ywqy78slA8VZRKGN|vr1y!mv~8L@2w>Qd-$5w_6C~x*{hXW?s-&K
zxz~T|dz+JU?Do&8%iBMRp?_a|ljuI@?acO18)NrQIxoI&>u&CSRfczMN@E!I8sFcx
zSLBtkeM<2CJ=5<@KA`ty`QF_3b_aCo9S>x)h3^0IYuesE7iGIUYbV?2^}gTx`uECx
z@_hyea$`E}w@BUH>!cfJ-(IlE&S=NceR0_~dzU%t?Y+wLWS{HFS9||$dB0D2wcx&Q
zcQ5ZfQmwm}{oAoU?R?AaEjLH)-7~Rq-@DBM`#$+^+?zh*{@yRUboZ{fU9~s-A@{!X
zYwPXbItA`!<@>y6s<_R*ZsUu4?Ut_FyZq9|{dLQ8_x@j4Wxwm<#yuOJ7TILUoU!g%
zUAm`@neD*Eb-a7a|E%51@YZO5>Zh*#?>?sQ6%Q}mf0b|cUcQ;z_XXwl?~Qm_uzTiS
z`@M~_p8HiA*6*$KW8TXnKiRJS=as!q+aFs`6kyyZb5dnL^E{h<7gtu;)tX-0`}cRi
z{#gMEHcwyl?2UQ0dS8uPfqm(MVB2Hw4(xr+@n!$6^P+YxmmBwO+w^Yl`8$922E@JD
z$2$Au-t@qm`@SoFvtC$xY|ol=;`^%0_U+Ye>D%WRZ?R8wSNYyo$}xN87Uk?&<ZyED
zTPenU=l1#8w5agxQ;2Qae=%5V@7Z(Sd-JTl_O(=0@7<bNv`1TJ`~D!M*1Zpc`u15j
zaO|zRlD_9s_b2PUCMWltUBb2hbsW>a!szXLR(-m&cj6hD{deDb?c3O|ao}Fc%Dpz-
zPwZq>t@mXq%I+4pX}njV^Zov%SI+GDCbZK=Y*zCAl`?zxMy!5qs}kH|v*MJj-My}N
z`>yBAvi0A!aL<RA;d_5?<KEj}#JzXxbK8Aa{n_`s%qiX1+_`;k_bTT7oBOWq+rjO-
zH%6$&M*Q57J)N%q_HC3tzBg^R{{H@5-*=zWG`7BaXv;p!32b}0-%0M*&RD)TwsFZ`
zvr6H;a<>oe&AxPPuR;TljoPEQy)V^{?LIPf%f8ueXZQY`I(2W1zvjNv`6u?gU*Nmv
z5v#_5hdfbx_vm)n2Q>e;+Gkd}_h01Ky$&;W?tS}xuidnbm-aG6^6y)ruw(ybwPSmI
zwNC66`?SICU~$8qqX~L;;-;-OpT!z%A{O!PyQ6NtpWVA_uimH6dpTBbv9o$J*`|K>
z#l4@J&e)uLaM*7BzE^h5n;+WsCvM;SxXH$D&L*S1Y{$0lovbRqkIRw!06)t#`&%EE
z?=d;*X#Y@>X@7s()qRaj;?|G&FYjG^Z}R?Asdl?E4ky@{+S=^7{A|@;b6dN82a1;L
z@ittwxBW5KJ~h^P`&xHS-}@xU#&+VF!oBK=Ec-a$?by?;DYVDY<;ebf_7ir$j(KX`
z9`bGP?-~2|a`Kz++d9GC{wzm{wQfb@zO}4(_Xa=n-|PB#>E35{e0z2N_3zV}HhWK^
z@AbVKE^OZa#H7GB>yMG$s*gGQJT`0Z+b*)(=E?CpdwbjMY~^M6_gxp2+o#|2X>Y_s
z*}ZdCi`sVxs_s3=Uu~zlG=1+!IiGz84F1_&zVh9sCEUzfefzK7@8-_hYdf=R@9x~d
zJ$I(G?e~e8WH-C~?p~L6Cwu2!6T3;F*Y_=E;oPU8rLvc4Der!nqPKhbKODCUkzlkH
zvg+6ySlPBO#*}4`bN$c#0*zgJzb;y}??gJ&K67E|ef)ik_HqZY?Q{6OXs;U!?>@C}
z+xD$DsMvewTZ^s6+tz(gME~rW&Mvl3q2R0ClAAC0{(W3yKV98#?>nwtHnsbw>=ktW
zxO<O6-2T3snKrx<SN6`Vw6ec`dz<Z5`>F%Gt2ho!ahSCC!HLHG^8^|9o#xTsTc60b
z@86yCcF*g1_hs1J+IylTaR1c)4|^Fsf)A{&V%s;l`t&|t8>@Y<{$1W%U%<U<GJD&8
zsgL~oTATa!uC3X>k7wm7`+23e>{ur6-rKFKXuqgQ$ewZazI_$jm+q<mrM@>kCDATV
zdgb0Ez8CwgbLZ~8xUh1s@lv&Y(IHcJw{hv(Ph1td+r)CxUV)%FHv5C`?`?Nw+I#kw
zxmCOGyS-oYIQI2$N!lNa`LlP0(3-tDGxhd)_|4jTZy(=2)$B?8cRcmkE3CO`|DAT$
zJx!CZ?m3~oc5iE}{_dqKRQE}9WZS4H&f6ojc#HkJr=9!7oEPnVy|vj+Fu-_kSK3Lt
zi6Z8E8!h+k)%IF%{qU>D{tff*?OAm<c<&r3d%NkA*6f)-ZGv5OfR61IL&1HjUk}*r
zogZqi$h2ec^2>|%ykUE{cmI|6y$NeB>`Rt<XeXl0wm;&s^uF>--21=YR@!%G#?5_?
zIG$TE7mM$!^}AwwXtk?N%N{m+pGo|CU;kjUO<UTwPm%NJUVfhm`@^5K?`yD~wohym
z%f9K?-|g0|p0L+N=FRTl#Kn8dcKq0@8XRF`e3@^LXlsGZK11Do4$eFG9@uZQZ?lim
zffX*ndlR-Y+FKR~?R|Z0l8v_ZsXh0<^X;2a#bs;!@xWgGV*-1R9yHiLf59%hg$L~R
z#~ADGXIt95SK)^Ez6hm#dw*Ck?(?cyus3ag^8W7gTXyRTUf(-&j>&<LQ@8ibHC5h!
zeE+OH+m>A3Cs`G2e@OJGEkC=?Ua`Xp2j*S>xR?2V!`_RIm-iigS+if^VV3>N_aggR
zdtK~5va#)-m414k`%0#LZ#fwD9o@Oot|rxG@3nbP?4B=Qvv(E0nvJ*CwY@?`fA`9q
z=CtOn<lK8z`GK{^HSWDRw^!|bT6|^CL^;NNOV&5<RW@+n`$1W0zxnS?do8?P?>Y5<
z#=hIts(UStHtux@zF^PrX2IS~f0gZzzNpz{^v`-1U)9V#xlZbPS#-?z+r=gBUA1w)
zjd7pfzIRXO*<OfSVaN4#kDd0<%Ds$2dHYY#J+QY%T77TEQr>;V8`SsBVK}h2N^^&e
zKrOd@|0ltHPL(cp7xwJiTUmR_E;BY`-vskdb}Xz8yXVOE*d?&6-fMG~b+4?#!F|G4
zlk6w*rrUe-?b&;(fq8#ORE#~p&CGq*+s^C>pD}rlKi8)HJK6PYxaW1))ts2R_g(%*
zo5q%hdu?py?ahi*vrlnj+xPR7^uATCr}sovDDK<;amHTlD-C;2U3j`zDEHJJ<41n>
zMez)KKWwbo>rz;_k2h}J9_>YU_d5FQv)g_)ZSNPZW_#wx7xu`tFWP&-qtB)xKxXfa
zDlfbBPK)=3uaw*yr6YA9jy1;q|BKuMPdt402hHW$$0D+8-@NHZZPwrTxHod=1e-=%
zGrOd9ulF8!W^ZlB&1jP}HFR&~<pcZe|F5*=yeDzsslwv@?9Eg5%T+AiJN5Hs+up2-
zy$L<GdnUKt*!O$w^<6=GC)!W{{&x5B0#*ALYrO1hejeU;q;TQhoXt^tX72g8ulv>7
zy(=bZACS3nXm9<KCwmz?^7hVq7-?g~&c9bV{M_F2-zV;yy6D(m6aD7Bav?AF3W#`H
zS-2+eO{gif`DK1?-{Ve$y?1Wj-20U=XWzaqfqh?&f8NV5vu^*p{`$S&mD%^7nAE;^
zU2(QeiT3wBB5`c{EIp+6w0z6oW5~d|H)rh{`((!Uecxvtw%zC3VO#szX7Ax5F8da~
zx3&&X59}45H+Appa~t=Str6Mv)@{+=GZI<{cE4fV`}N<N{fzfGZ0}Xa+i@wKv%6xr
zVL#I`&V7Z;KkV1vzhbY?nv%VeCYSaF#5eD+JE3aFF=g^z!{@E`ygh67DqY{T?}~oc
z-jhF+_kDT5ZojAB$9itL@`2AAB=)rie6VUr`LXwOlfhmK`Q7^-%s#$*p5x?whjw)B
zO=5bvSM<lky)BEG_N_TLb?-4dp1r%5wAoEft=SuvZ?nfH+kNjEqg{KpnFd<3|71L%
zAo_F9bKkIiR`YxJNHuKRYkvRj-i3d4_p%F~v{$y9w^zaQ<KD`9W&2Y$ZrSbiUF!fR
z`wIKe%%Hum{xsXm?eVgi?q9y=pMs~2*QO123l>h<o8s|&@8|Zm1F!X?_O|}-J`m(}
zVUI}ZtOGnxglsc&`}bc~e6*LTMC3qc<C8s=)~ohrPTsd~<>Z<Brq8Ojzy0N)&FQxx
z_H$0y>}T89voH9U(LU|}>-JuE<+6WsLwsM6?OOY%>Sy;Fd}`cl6nSO$gw2ch+9;p0
zn>kB)Z;nl??To(2z3XRe-y5|5^j`mM0eg)De%LI|khkG@x!kVrd7lmM^(q^_XyLu;
z4}R{o5_n?s{~5>rDa}lKE1$01zqRlFUM9Bld+)Oz+^b+KwKwDHQ+vzm`n{)f9`D)z
zch$Zx@6znsS5_R5*P3Nt>U?)kY4Ke9hkTFsZcN*^cdb^wjWqMjeG85)-#g3P!{)Zg
zUHg8<88!j@xA!kPHgWH?Z*}{<4=mb~5h{7$g4>(DHD_z=|MB$i;SpomJLA!ueP5Z)
z_D_k9->1s2v)5*l(SEyYXY4nAez32_YSvzdmo0lwyuH1zu<rb>McXIYPKf)ox1(*<
z-nOFNJ#*gA->s0OXm4MCc+W(s)_u26UEN)Ebn+hEHTU-DZ(`ZUn^tSfI$`48os$&z
zH$Q&2_e`XL)#MuCeJ^KM@42VLwr@wih<*LNb9<LYtg}}ZUA=cB&+R?d_x1OF?aJCK
z&01==;i9_Dl)cRR7#Ljbd%WH47cN_~pSfOd|F4fBdnX=#xX*rHgUxH@w!N_$4K|($
zZg!488TNY2HQ&2YX7m2DDf{;JyL{R|*>Lin&dpl;S(Ewp-CA0-Keg%op6rJ*`={od
z+cU?l(dLZCecPG`EB2ZtHSQOh{d8}Mi-di5N!Q-gooDwczKY)KTPC{ibg1<H{So5(
z?nK4ypS|zW-fI#!_w3{L*<Dh<W3S9p5BqIiJNHJO<lXlxn|JTV$BXxrPFk_I@=V`e
zw(5O*9UiZ@k#B!scTDN)p8EL(d-Ve*?Y(d;%*IRKXs_AI0=peImhbiFinIR|$Y;w^
zptILLF?7$bWoEn6q%-%ItejypdF9_dzY~|+F<g_j7jj;(f69vF{lA+n_MWSa*?-^m
z^<I7F`n{2>p6{_^NU^&t!?;f{K4VW|#mT+L^@a9TJUzGH?&t2km9te2<a=$f;h4^|
ze-7)8y>q2v?T+l-vPYO<skKI0^S-4~YwU%V#M$TRE9`S!=56<P8OQ#`XSVH=mN>rm
zxZ9$=%ip@}t1LI$yEy9L{{6jed(+Q1?G-5avDZj`pG~^~yWP<oW$XXF{QC~+ec!v|
zVDny6;rW(p8#e9Me7I{bzw?IOaVjNt6FxoID_Xj7@1X^g_8G)K+WTj(;y!Oq#sfcV
z3io=cbJ(>n*|WPhjCqfZtDQ}(F5^Cl!gF>X6j}FmG7Im!?UcW-NMq4H`(^WN8@!g;
zI9z+TN4!yb&jS{@eS6M6+<SLp``#w+jr(`BuG|}NO~~dE7r%XACd)qUg}HY9onCwO
zU#Qt0$Y8PevdXgEwVQdLd%`^111B!+ReOHg`p4b&eHrZT`%g_gWmoZndEdqInKrlm
zAMCB;mf6Qx*1Wgx;nBU5HgO(!DiFChOZvvXM$2h?e=a|`*QqIZU&e{1y$$!9_6yHm
zx3}%O-@f%NrPk_~?(K>Hae7bK_S<`BuaC50e(-(I9M`1%L30)NCja!?`^-jk-*<bV
zeSX*G?2!s#w?FN?V9!lI8T+8l4>q2PYj?k$nYGt6$9b>ljqH6}EKlw2l`OIeou<71
zM6;NERao!-H`Nw)n`Pha_1kH_zy3V;o(=aO?_EEIcb_dg>wcXtSNAUdT)cOe@MPO(
z>D_y_yKo<fV1B-rZ-d!^hVK{lX{%Tq@Jl_jXLW+G{T+q`D`(fr-KrX^_kB~Xw)&Rb
zY`^BQrFEZxhus;W9k$&p2lus3m)&RO#kg;d)ABvXmPPNC-QTo-N=)-!?|GZ|Ufj!U
zd*4aX_H9(Z{jVKo_FijSyw5FPeb3{GzxKZA@7&w@ZHA5bdO7P?FKYJeda`)`)MoL$
zZI7nzU98u)Z=!?x-jvF!Jy&fe?z><4ZSPMH1^a{?23x@eH}~>B_+&lf`RTpdLS1$e
zSC{Xa^7DX=Zf2j|TD$rCv#y`C6`x#bw?B|$@0(vqcCKri_8yl$zvsH)dYg5t|Jm$W
z_Fzx6EUS&`rLY5!(kAR>+%$9F*%HltT+K%IkE%Eh>|NAqU+l4J@4?p9`%~|R?Y+E_
z-8Mz&^ge|(E%xebFYR0D!Llb>ZM9v4<B$D|@=okaHSO5j`CM|ZzPYLWJsp<4`{zdO
zH|jRo8}pBK-=dw``#z_w+`CO$z`n|a-`4hZ;=ccZ{d-eRJlcEoP4}KQE$O{~*4^7X
z(^1vt<@DEfuNEKJYkt*muc4*p-m|}%_idh|yN6kSjs0te?R%A1tL!fiZ`*tP`r|!c
zzOwA&y?bu2*uVRB-18^dXx!x8cmK-!eH&QP?YFw@+2<H8Y~Q-*v+cS(>-~Z2toMGM
z{9y0qYVN%+%iHaumr3p|xxH#{?QaSD=ZjPK_TP=$r`<Ab@BK*OeTjO!_V+If@BhKE
z!RBaJsa+mp+Flo}DSMQ*w(YZClDS{SI>qj%<L148r>5_#T-0TMo;!VCk4MKIQ#sCk
za%U{}t1SDlS8*cu{*vjl_Zt4cy?6VS#C_cVnfJ}TXm3CL@SHsf6{7oE+!gowt`^>>
zzUs-|Sx?XGdCvZHFLUOmy}#;h>^*y*?Y8;z%4Qma^4`a%C+_t>WV82gRK|hLPcGQR
zWYybkxV^`QwdKOz{@Q7K1aB?dyZy1yew(MNdoDfZ+Q;->XTR(}#{G)+Z}+O#Pul-`
z1Jm9~Tj$#A?GxB{By#Cqg_+KK{vA*_a5$Z3pYL2g>#3FQ`>Sjk_C8eDZW|vivu}yk
z#C^AmI_!7-+h%+9(8-;jmR{bwT!U#}9P{bDwj3w+xi8e(XW_46Um@DHH~HSGed^(#
zZ5IAz+IOf!d0+Fq+j}pi&E3Z<-n-{g?mFA%)zbSMoDS}_-m_}otEt_4@4V>Tx8Xs~
zUSW|ETcLv0dliqg>`4!pu_r-#*4~~|TlVOkSiINRY1h8DmzVBQcqzJX!u$ArGpzjf
zI^Wr2Ej_=%&S?F<y@9I^?)x=MaBuC4JvM=hm+ftHtKH+yFJ~VU?QWyJxn|E*gZp+7
zS)%qIV-M_~r_H#hsK#mUZN4V^z2A52&F5IP$3MPe@5J@1_WGDL?G<Rfvs*us!~Xg6
zX?tBi3mn+vuw!q5?y-F>`=#s`U%s>7r}mrmmR;-jF8nC5H~43njm*X=`~04`AJEHr
zWH-<7&7O5xf9$8;T(nQ`{Ox_!yZZL>wn*%&St_+}PF2C)_rGTE7m4ZFYdUTI-j^%F
z_nuH%V#6X<vbT70nf1Nr4f{A>^z6;!d$IR~-@`q}n!oN1-Z*LRvYcgm((|_3bDw&*
z=NbQ#y;WIVd-F32ckAxlwRej9BD*W+IBn~5nD^&S`nk94gYn+aKcDTLrnG8rdy4jc
zp>p<pXXZET{gFCx-<=s<_GLwRHWfm__Es}H_Syyi+w1zN$cEV|Vs~pjr_JRLhioMl
zo!Ygw>ENztZTbBMPipsWoONQ~`j476cb}}?yQ%fC9bd`?+fZZuz1179*e=!Lwc%l5
z-K$r7dLP^IO}2}2gzdXR^!IX2y|AxL!_%(PzH;9_JB@u|3s&#>`fcmJc~h_LwF{cH
zZ=TK0y%u5&do-OY_I)WX-SacU*6vEP(_Zf<etXWS3GNenB)xY|L*d@KKIPrlc4qFq
zwfx=Qr0mmn!HKK({1#bYvqiXj-&J>U>m%FaYz@+K_bbaBw&@g_YHxHVYmd^#Hk*l_
z_Ir2x3GFMH6>V3(YwsTJ`QP?VJJGa1YC)pCTJo#?PV*<)CS_XLHn*<a@3Y{NO?%Mx
zy(!!_Hs(3{``12T+;@oM(H@@7V)ow_9^8Golk>pEYoUARXrHoEeyD3dRqw#w`Vf!3
zTlQ_=cj3p9y+_>ociU^E@0+W}e_&h0BRiKcj=eQ`_I9_rME0*fbYUNV_x!zk5)SV9
zx~X~p%sVG{Pka2+&ftvvzN+7ndsE`(+ur=Ob<g^D7IrDyZ|@D~__Wt<!K1yZ#_fAQ
zO)cJA64SA_w9L--*!-1yn<oF;zva-9y|PRxd!_oj_l5{Z*f^i}*t6jAF8fbbv3p~>
zt@p9jG3-@8m1*;=apB%9w#$207%A>IUlhJ~t;p-W-fg${Pc1!VFXwY^U(Kbd`zvdn
z?3NQ{-QQSaZ^O~sxVOE#-zM~&{(j%i6V}a5347c5Vh*gF?7a8dTcrcjJ~-}Ws_;7S
zJ-WmGQJUi3Q`Ox2-u&UUS2cRIcN+VSy(=4d_QhLWw&TwJX@5ehXYZ^gN&7PeFZTb|
zS+#HCJKeqS&lc}pnQyzlU2ex-Wr6edv)3~0i>_?6Ej-<2bLeQ+9z%OgdsElOeHkIo
z_x1_b>^0rOWvjuKyEpP#$KJEATdY&MzwJ$&&$jRL^8I^VwEpZ}e`(^rqEid@Ub(HY
zN0*<+zRdFZULjX6yNL|D_Z|}Q-;>wFyw9b6pUvJGYW5}fIQO_MX54qRXP$k@H}(U(
zQ3vdnIyu=$YEInyJ66-qxsPF=?(}tg%^chJ%yK$sUnR0&@7@D{_BLOhYCpG}f1lZ&
z9rm7@jeApPr`jJekKA`8?x+3Z3vBzUpRcl;#niSptNr5M56N@(MX*e^pD3zh&+uDd
z-)TpyeH!+^?KiM2-WQuceXm=Nz}`oP_uIeN*<mANZ0pb+G;Oc$jPkwJDtq>L6)xVp
z@t2bQ_bp+2W&FDKy>;T-Tao9m_vY+}yLnew?iI<OxA$iU>%Isj7kjBs)AmZLH0`x0
zdu5ZkNpJ7VV6MG=t+o57U*NM%VNl$A`01RzY8G}jW_)~m-#g#jd(k$~UioR)-q`Pt
z_s&c3+3zqVX`k!GIs2RTH`xodt+Tni^o*UM&Hp{_k52A6bYtV*5HbG!w=LRjo-F;j
zTdpJe08^~*-VCO>`~PWt+Z*<D+x`U$itMYHh4zX}V%sMy6TENoG}FC5R!`l1{7&0G
z(V7YSuYAk23w~CycXE=2y_@T5yGbSU_i5(*+q*78XRmIR<z8D^!+q`_*V!dVYVGsl
zY}sr3Zt5QORZDI1QUa}hRZO%|=DlH~7F@F@L23HlnHLhRW@_cz)N*F;UHfL~9vS_`
zw$DG`-<$h7Wba<?Uwb#C81H=)wRTUI%G>?=oICesEKAy_9(QD~*wu%7Zy#i^)?Bt@
z?^FTt10F%LdzEjm*qc|wVP~{K>Og#Twe2M@|NV=eU)al*am9}1TK?VwABDZYuLtf4
z>5AKb?OpTUe~Z8ES@}iFcAxUwy&SWC*dBh;x933k727`_JNBJ4e{QQgf88FQ1ke2p
z>~r@%7y7#Q^3hfMw*6JNllvUJZ;ABdz3Z8m?Em@Dci)k9EA|!r-@5zO7U8|s*Qe~=
z&ndd^)5CrH%T6}!y_y`g_x$I>`y6ko@12#&vj2YP%e^<6Z1x#cn(wJzcxG?As>)th
zKHt5c4)*NcY4&?B!wTDdM-RT<%l?da@3WgW`?lX)Wz%!^=ich~ZF`Fsy4a|1tgu@#
zJ$aYCEz92RZ~OO3PrSaDdy41Y$$pwP%-*?sr`_3SH|N@PyPezK?zPlzv-`3wbg%2h
zYx_8Feb_5&xOs1)qx9a-lNt9%CY|5=;?wTED;xstDyq2lElCjC-_^0$)*<=pKIe|7
zb{C#s-CxSL(^|aC#lFMi*Pa!vm3x0&n6l6Go8$qOkIU_Ec(m=gvOvJ@(yAx>*8Q8h
zFJ-#+-W_#H`wq>H*dOn@&Zcr_f!*!vdi&H@YVMul7q!nxwb1syx~9E@+@ZZ^l&9@u
zE@0c6ROG*Ba@XU%pML+{8(UYk_vrIAHe&Y4`<rjB-lO|FY_CXs!@kqpaW?BS&hJ&*
zdT#&TTPyb(o&9CE*`IT7eDTD+|C%oC-Sv`V-#Zo`JNvntZJ1wm?LB?7V&4-fG5cff
zyY}a5n(Xh|E3x<4*EIXh&9=4^j<4P8#r$XQ`e}CiB={5eL|hcv+cTm0z}2?hd#z{3
z9SB)+W{;2g%zaOn*V;Q}T->|tj=s&*<%S1X>-hJb<MrIr_GZ$)GNp(6|9^?L4-HY=
z_jq=m{e#Ds_G_KKvX4QKb>H%%OZRRN@UyoLxVyL1_Vzx7Z8>{6o=&q(QB}10dpd4!
zuJ6{p`qL-u<p|xhci#Vqz5d?2_B7o6xA(T;m%R%m^!BoyzOmP9ht|G#EHCyhe7j_?
zaFNhn|Jc}FY`+t1SyyN8Uw`+W%|~gy{hg_s_saf$XY*cekxfA=$G(M!4%x|w=j}DK
z$k?mWsj;`R^!&aj_UZe6ex15k^CtJc81<ue$9{S3t+08ySIJvo-}fK<4hpV!_O@^_
z?&F%8Y^SeQv}ZE&e)~J=mexkaHTG<Ce%kN9C2IdM!Ex`eC-dx2&1~6Qw1Z*aTf0;H
z6ck<UwtCArgk7%QTYsz3R%UmPebUaw`(}8h?v0Z&+H+awn2ku_9c#_<J@!Jc=Il|B
zy}IX*N&jA->-%@7>|)xV{&TmjKuP1i<CDtwR5m`}W2(-z_tJI7ebvsp_jtbAW22X=
zX4g=aV#D>Ieecs30sCxuoA+Mm-m$m-#+ALtD<1DX({*OQraspmo?YDgoRix2s=Yn6
zcgOX+dkbth_Zgm8WgUGg+5YW8<NbfXG23qs-nz#<%x<^5Dwkd8jT3w2blCUtT3+7U
zaQ^7NjkzcGuDTs=<932?|CX*-dtcr9U^kn~-}*v<>b?$Jd%LVX1$J|M%WcB$H179b
z%eD94nTa+vz3cW`+_1LKs+QkBVcX$7p_YRCO^zMd7d3g&KE34vdsC9N_B@d_*l*Np
zYBT$np`G_vu6+zk+xBYn2Jib{{&KI!x~KbNWoGUx{OGf<KdoU;LU7`qmHc1#*7DWb
z@Jr6KDKRwO)BUr1pLaX+KA{A*eLJ?_*lWRlZ11tE=DkjZ6ZZd3>D#kNx71GBka17Q
zTBd#bZMW=J4BEff>;<d69qR(?OFXxCe>k{x-!}p7eKK`x?DlA}?OWjKwzqt3$Nr5!
z-S%ykXWbXtH+4@%;D>##h5CE7&EMO+D!;X_=jxVSCE-T)JHD^md+bf5{Q{obdp8P*
z?9)rxZgb98VxLUgp1oTy#Oys9FJ~KXwqbAjB|ZC3&sg?($6wng95Z=;?6JxF6esN4
z+rF+~kMbH-`(lp^yLlg++!txoWD}-oxOYcO?Y`!(i*4_h_3wS<vwE+TMa{mLoXvZu
z{=C0;*72sjA)mJI^<8s+&%C=I_I~W0y8nU!)4nA2O?#g%(cKsHpl$E0O^5b2Z4cbP
zJHvZ#Tf-gulOC>njRd~#t=yWl*G}r`-o2k3><s_;?~V4~vbX=~u6?f=dF_&yneSV9
zN_qd`^ON`Tf4FM*y{*^g{j=h|D$9lTdff}yKR^AL)y9px_Hy$J*-cG5y|*fMrcIf_
zpWQZW$M<dgs$plh@4Nl)m1}L@efna1b0W{agL(RUB_Ge-*L=Euf5(eRdl&cld-NE1
z?bw<h+XeY9-#4d=f1mq@OM7=)8|>dI!@l=ZV9`E-rpCP@vGq2Rn>hEawc^_=6tA#<
zmDj61AGb`~xA}j|-bqo>d;D46?%|mAcyFLx&)!pQp!*3$_U~=ky!W1Wq0Q4p6ZUm&
zIbaj}BW$nB>KprO?%cI;Iyu$8FztZN_DfrB=Idwe{d|dU-<1r9{k2c0?Q!S7xR*o7
zcVGR2ZTlXTFxfwEZP`D`Uvlqdk@@?#a!=T6D06Y|(a+cSip-h0&saX!M)=^fy(ad1
z_nH+O+5|X$v0J<9%-#bJPT1K`lee4yp>$tr#In8a8=3Y$Kij*vf2!@C-`ef_#Nwsx
zQ@iKcZ9BoRZ^q~R{nM`;-a9S&*go;*6MGh~ytwzO+bp{cmV5TTT=I9{;wc~Y>^K#$
zS8B$}eI6IT?LE!fWOE_(#NLpbFZVwEqP6#e!jZiltq1lVITEpV?ZWGO7pS!E)m1m!
zr_8{xxA54yy-)2e_I>4U-D?=kw)g7BD|?q}H|+gj<FJp<t8=fc$>}}!&)e^<T)b&d
z$RlYRC5HYz`ZnzQqTg%oJ91?0K6|~R`woaL-rtzSx7S#q%XZhTRePH^pV|92%Vh5X
z^>h2*%d}dbWtO&nACa=3DKKxZoSV%4ZQV__OHA497DaL&&=owi?^EXin^Tv!*nMO)
z-Fs8+<lg0-UHc*{J@=>e#q49qaNP56UDdw8r(5^`NfX$&Re@`t!4;W3z1=eV1x&8*
z{a3Sj|96{4D`Vfjy%L^x_L*M2xmR|^!hIi3+U&b{&R}1z(dNC&_sp@W44z{1XQJHR
z)Yor!fAid8<ND>(z8LS%ds}Q-_sv-}cV8n5@4ihfclL%J?65oIa?oa5+H8A?EjM=@
z=P9+h;CXYeMVIbg-*0F3_In!cO*+3~ueHPN{XJJE?L8fAXD_)i#m->M`MtaN!}l`I
z;j-Jda^2pN?aTJEx<u_?A!J~a+qZM?+ePziIo4_Ky?=_=-nMbA&He>T?5f`0v5`7^
z%<kt@>%9v!ckN@|vwwHm?2w)3zXj~Q+;w`tebiUmz989sl2_K*+Lle(&!(|y-}078
z)(Qdt_Wp6+X1A+$_ujvkui5x-Ub*-CL~9#9j;4J*-i5Y|>KpgFDBs>IzcFNAYzON;
z<pqAWcQRG?^1s_-Q>s?6_x#BZ)-k`A+U=C9*vmM#W=~L(<h~k1xxMzAChhBCJYhHK
z_5>UD*}L|0epqbN8^f^gLEOVVZ<BoYKI^UAzv}Dsy_uR1cir66dSHv=mVGk3tL&Vs
zg!W0kk={2sBgs}>T4;CHR{gy~PAvNlzE9cr^&9g(8MRG&J@xn5&YL2*&q8{m-JBUa
zciq06vhStt+I;~(c<f`+{_HNldSKs`I<Gw$OW*GGeQ?l5PcmbF#Md{rnbqg_W}kez
zFI9K(e(Q*5`<ma1>}zdJ-P5J<V(+`fyZ7AObHHxa1=T%@F@}31D);Ow|JAT3z~uP;
z3D>&!iKQFwi`&|}X9}O?o(J27ZETk9whnl}xi9U=-95}rTkXFlitfv4pR%`DeC|Gz
zcT4w(FIL(&lkwOdZk1ho14CWy?=0%v6EA#V?|z>5drl|1Tdx$$+E=A9(S}Q>XYZVe
ztM}cqa@)(f`PV)-AM^eDcDmcthM3z<T$;0|&3naOque=rqE5@$r)(?T>+OGY?{$Wr
zefC8hds~E??JgO!?R%zdY@c*$*8UgsoAw!>xV+ofSn+_QmdW1e%t?C>x%KXo;<mBB
zJ^!P9z@!s<6&eNXulO9@e?j!vzONyjdv`x+-@Coxz1_Loi+lcUWjJu_z|*~Q>#S{M
z-%ITA7JI*E%BBcA+wBwgG3Lteiz#8)CEF9ZS9#Z+J=PN#_PvcgyIaA0=3b@k(`|+I
z@9zl@n!2~{`oet;?i~Bn7EIskA^UfKH206at_x=DpXu7Ow`IYv-SUs8>@7ZFyVpk9
zU?0b-w%reTdG@W;FxmU2h-v?hvU_%?*iY~473<wAoPTI<YiHu#`%`!CZT_`#ul1tf
z{fT?L_ihR@uy%alWB31h%wClbZ1x|O|L(oqGQ+N7=8S#G3x3(Ldz{>RN@LPK6ZtoL
zOLR8xJ^bqAz8|*b`#!8cVpqP?Yme#iCHoUsbnh<>KeO*uP}*LLjiGzFCg0d+v}O8U
zo8OLh0ymHB<+wk=R*kQ7-}Tt|Jr=%v2VNc2+}HhX`o2}q@9piHlwzy9!+)>I4x_zB
zOV;oGn6PxO#DCZQFQupKQCh{n&u*>7zP0zw_xcv{?Gr9F*srwe$S#gmM{ElOZtR_`
zxqNT$zK*^6f4}XK&iCD~%3HbDE9cYhjLqlv30<48uY+}#ZT)_S{r(Bd_rBQi)h@E(
z%igqY())a#9^Bh|p<(|<qkX%cuDZVW>K2{@Z<{0b7Wq`!ZS(oFx8|agU0~&r{q-kK
z*uBb>x6HPXKhU^ec~7u7tL-m-%Y8LUFZbCSUf6qD;`v_FHf3A0^i8%49!l<GRnpj(
z<;lC}OxZCzaV?g;>ASbtRi59m_w%(Wd)@{-+57Va)4syQ&HLUb^X#*(=-Hd7@nLU}
zUh5vq^St{mD{tMK=(}RC%qQ-BJER}k{W;gXhuN>!wn)8apVIT4HuYM9d#}$E-G4?>
zXzzV<VfzC6!o6)%#cdl_HSM*q4cxnGi>O@<W5wQke-G_tirKR7$XX@4btYHs+?;0a
zi~YxEThhA4j^)TIoBu{!`x5*Z_FcUB!0t7_$=*=K*EZdClk9xg*4c0h7umn9oVmwQ
zsoDN^qm=#c>2CHS9Ua!?Ti@-=^5fa7wXJ6FzL~-M1aG_9pP%Dpe_+k5y<7BW?+Xt7
zZa3|0<Gx!9r|!92cWv)U^J)8=&;Q%IT-?oGdd~UXD{V|{3b$(SeNh*&*KvydzM}{K
z?{$}z-nV0h;J!xfk9!@tZ|~i-<=bA@_?3Ixr%l~^L92bARP%$qE7mjbd-h=4K5K3D
zz2@$V_ZB^RvDf$dSDUGt9roHzPxm(NsI&1}#<VX}|H0mj$uWDU+~?U>s?4_UU)P6y
zh6Q5#!e$uQ`52hk)=xjPce{4M-t!V4_NsAo?Cq1^w)g()pSCWEZ>_%tF4^;&p}<!1
zY1f_wyZ7viVhY*YaIIwzL&yKUuU4?wudK7(+f%m7cKK!Ty|%mLt-Mz3-+N29#<n8+
z&^`-O?!D4C5882R#_Zqz=GneyZp-#;dvJ5_Y2|Z!Qr@iCTRhij|NG}__CA+6viH#^
zseOhn>3bA!-mooih_@;E8nTb;0HgIaotC|Imgn{|o2T!MxP57_o}|dG8A}e><ymy@
z{u|t2qh_tWPi+;i&9ZpG-LsUm?N2!G-kaLvZGV~XmQ6yV_TIZ;Z2ML-%j{jfdbXWu
zckP}<Zv^+vmov8)|KD$)_4k~u=SklEEt{J6hEyi*onW?Sk8-Q{zKEM=_r8B@yD!q~
z<X!>ozP&pQIqcWGTd}v|Zqt6Pg%kEn-@D4bbkmakPj-B<^WM_3H_73=y<PXcJu7Y~
z?rn^y-p8xfZqvJDgYA3u+PzBl@9mmiZr{J``h$JODIfRje$v0UEp6JKX~q2e;^n99
zTVdI5W3}68uXX5+J)GA2_bz`MxKEUUbszh@?tP26Ht%hFD{UJYw0W<OYS7-vyUX|b
z{Ic9@arpP%AGf9Uo;TUHXQk_>-E23T_I8<^vz8ZTu)8I%yZ3{}I=iA-du=`k@!9fL
zHQ3C_J7N86^%}e6TeEkc+}v%`v`u$^Lp9^RA1_1pHchOzXZdWgH{@y0-X{fL_dYZ`
zz4zm(qc+RFzOl7a6t&rAaK+~Kv-|sU;sW<BirThc>A{!XF;O@7S<0Wao7fs?Z#;)-
z-=4(Q{Xc9?_HI@^w6{!l!afbQnEjqdjBNi3+_aJHn_`!@Bg1aVlBN4rv@q{mmu9#(
zblS>&HCZBi!)|!)HBS4o$8Ng2?NXl3-9_$_drOY%*h~=2+-I6UW3S9hzJ2Pp5BGY%
zVcOd!ZMC;}3&Xx=m)Z82?f7eJ)|$KLsotGE#o3m7ovz*5>+hhvr+%I9ev|hbZ5DGk
z?3Z5ma&NK7oV``&@9%l9dSH+4>Q;N^#w&Zzq`%xd$#v?!(=STwlKOw|JIV6HZoO&h
z-gR=G_P?+5?0e<GXB)m}_Fk#CQ|$Pj%-MU?v1RY(4Nv!X%XjXbxItk5!iKoLQQJfJ
zuiyD%-;P-u?5+iM@40*++&+(c!rq{(2lsApXWiTM{g%D<opt;7tvkJQcE?0J)73}p
z=T<-2XPj_%?_Kr&J!0X%_A|)-wK<ml-0tMTvc3L#d-fiY72YSVeSL2#Gm~A*@w<CZ
z=@#xg>?mlJ&?c~#hwbEEuiD#tjh?scRY}~s=Z#zQey!t5Hug`O_kO(Cu&-nJlfA)y
zse4~&PTjxWFMRLfXG`tnA6(h{E8B7}v&*KvXMQs76A;^CyFc;YUfKCJyXQTAZa1@L
z+Wslr_4``qRPU2<T)cNi#<V?)IlA`FT6<x4<EqDd+n$*0>o`AouhqMYdwEZm?AKT+
zv`=ljgndwWn^o){^F4e0<o3tgOy0-oGi5LH_qY3;Tz}cz=~vqA?X=&<ZuU7lwV5mT
zvnOuayEjvEKilqByNdS{_PO+|-Fxa<=bn{281`$tJ++r5x?o?C<n6s>_rmu2{C&8W
z&9%igoypk#+hN&#&khCci@c??SAFThy?4%?+<UbCko6fWo_*fg0{iYu=<i#Sc4=?o
zEzW&`b5!@$AKA5cj+f}(`;%n$E!R4<H}7xz-pw{ayX!7C+5DRtwfEMlJ$p|Tp4fk}
zWb<DBGYWfm{a9sR>cP2xZ*hxl&6A_HE03?<IjK9|)@<qz>zSsP_uksHXK&@_4fZyM
zi*5cW`|rJ{Dz|s%V$HqUpIYrJpUvItk-=hjvP6HstYp}}$Ti%1&6E@O`*eM_=2{%L
zr=;}EKKDOO_Oa9a?W>&j?cG%K**5a_jQ#CB%lCOc)3Nz6{m$O`Iu~r4yF2!7&*a$u
z(nV^Y=T(`#)6(SkE&P6N?|a+%`zm%7>}!#lzi-Pw)_q@%g={xB-r9Rrxy$C#+4w!#
z(LDPmcr)*d*<!l4G3dzN(qPAZ&eIt8Z4hDH_w{4vKCjh3chCPDZCh+=vNt@&c`r-D
zi`~yfIQAaD{AUl_p^tm~S^RC{y;tomH@jeK6H~D7|LT|fiYFho`8+{k?<);~y(`n-
zS<5%%?wLN1eZRy9&V7A(5B9!N*Rr*HP`dZHL-}6+%%gib9cuU4-<H~^wn%R;U!2a~
z61Us7=c3d0w5(^_leu&6KADS(`(rol+<VbRW}n|PnSBlO-|zpC|7y?w&E0!{?O3_@
z%Z8Ty(tlX?tu;~J&&qaV??T()y?5VV+;>0BefOhuru|37IBl+R{oMQba>3r^pHJ?6
z?jyQa{_)Peg$wWQ6}_`{kI%;w`~GT7-Fx8kOq+~3UHjN{>-ReRda$=UaoxVe-Ft0v
z6VC4EyCP=ueflDstIHYo*e*}rTYqE4zRh{rdsx<e+q;e7$bN>Xj(xLK^6a<WtFXJF
zT(P%IH)v04`QyEY-N}2Wo{HTo^uW@dOL4)TQ+wEKY=5%bF1*0Eug<K=u3t`YZ?4XY
zefNS)>`$KAYbU`tYwt{>lzrX{y7ul4n_yEOd}tr@l&kiQ61(k+UA6aw&s@6Cf@AmI
z=F%zq;#c(V{kXGjkMol9eJU=z`!x4(+Nm=<-;>|gyXP|dp}mLLZreP6%e#NunJDXt
zKQs2Fi3IJP5@5T>Ft2g1_92eFZpQjHj7l^2&XP9T?_MvtPi@ORTagD#_SIBu-!r3?
zd!OBot$RABE!+F~jh=m0$)3GiBTiZ0_+4ooxu0pDOt65R%YW0oSMy}{mVEoO@5!a*
zb~?*$*&SPyxnFy`#NLvB%=Uf1KJU>{@7T*bVa47iffT#Ui)nk`+dSPXWoop~<HzGY
z*KAn#-@A5k?^PZ)dx<{h1GQh(_CFF3+w=aw61x|ykN3Wv-?aC;+Jt?N9JTFt^Q^Yd
z>-@2Iqrf^lKdamW)+NpR+$V0>YwN_buWm}|o~Mrl_Pxx1wtr3eguR{&XDzdKGVbea
zYS=CI;HK?^)id|SiaG2PKgO^xg=N>?saN0b{XIEj&uo_Ud)0-v+88h0v9GNqagWoY
z6??f9SMK#`yt~Ki=#9Nox?k)~ICpJ#!kumV+bTWwHXOaM=gbi=8@=7T_D-8=XPdpL
z+h)2)z+O|=6?PuQ$#w#BQ|$_0FSPyVw$s-2mF_;3z;v5DzNdTAQXKbgZt}KYEhw<B
z_erzO<_vMWb@S%!HNOyMduqMXzHd`2?cJ<Z?T$PX*{7lQb?=XN?E6+dlH8lx#Jumh
z{FZ$h%a!-<)Tpw1eudNaY{HiPw`=d(-FvOO&xLQx9{-~9y}y2H@6#yXvDbN~v;EfC
zV|%5)thG6$J9%G(S-7>&>PL23Y+r4ze1Eamw{_FrtYVJ6Vbj~K-xzJ&d%?AK?>b?w
zeOLM)*g3p3-z#*r-&RZI)81=2tv2b8{@P4Zn6Ur7-Tb{*Sx?%f6szrZUGQ=5W&ay{
z6n%f}x&8XvzO*fTd$0f1*?T%`(>`?$(fta#3+<Mvm+m*5&c3gNyVag$yM)ajeW6`x
ztKaN>xb4IKN4JjcRafEPGtK_hey;_|dl{xJvGZ1awYPoaSKF8R&+XzhlkMi`Gwe05
zU$u{m^YY$Knd-e4*t+(e;W)L|rCVfQ)ZdqTy|k3~9edzy=dakmxB0o!zC|az_x}Bp
zu=lvzrM(~4o!q<aw$#3(JH%{V`KIo(o3vnWJ<G{`acv82GE<-Kz5Oh3uaoQ2z46t|
z`zkCgZ1Q{N+pPS4ea{}v=XMz<7w)aAoNddoW9q)0g}?SrR_fX-czBx~<5Si>uIbu)
zKW)*pac(_tvn_1j-V4@?tY5tSW7p`_XtUg+&1UNV^R_cHS?tw$_Uzl}<GlY<w%OiY
z`AhA#-(cGJwxw_HgOqc7@6_e)|6|v*_in7mULTFec9(xQ?~RWw*njLztxa0_A=^8?
z*KE%gCD;od@!q>HFTyUP@#9|kW1M@ln2+uC6AakD@7@nv)92^+lm<+-tFfB8f5x5d
z`#6(d>~Znc+`IhVemn1D4ExrmPTXhf^LdY=jMg5};6K*7o6hX{bwGRnqa9UyIzPtj
zJ8{B%?~%nS`$`Tz+MCVTwCA@ikL^^Ut9wGT^X%?Ns@WVYceLF(Q+*!`+nGIY_y5@Q
zcki10H=cUi9Qh_~SAKGy&0U8ldk$4D*{gh*cVCF>wS8L-blU7(AYr3<e8D~~1u1(b
z+q-rO`z!1(KJMC+v`b|FVGHJc9upVa%nZ@nTYN-)e{$#1y=%J;?){z@xj$+1fj#f{
z%G#H{?z7Q;6K$XDYi(EAcGgZHntxwp$+>;C@}KvZY@f0B`L6zb2j3~#yT-n^E36l^
z{jiT`f665OeJ9c;?5nhh+S3yI*lO!s1v?!zrhT#XX?wm)=3CD#y=k*|&6a&3&!_EK
znk#2F%lWX)EpfJeM=f^mJzuW0Z=TSKy-t4)?X6hVW)nVl&EDOaHV(^_IQJPx7uzJK
zt=JdHEw?wg|H9tVN3Zv5%O2jFn%B0^r}gRX*Rt>T{<v^u_s_&5dqbEv?F+oWZ7+x7
z*1an8U+nw+U(lAZ$HaEy(=E39|1{eQ-;}cZCZ1;N@zQkf{0#^8E;*CAzbs|n?shH9
zy`RNR4=7CT+Vk9#eP6TBjlBYK`g^83PqR5Nk9WUsM)zLFJpud6j(^^hu#<V8L!+ae
z#jcI}OQn<Se5&s4jWjZ_+xwV#-$m)8`!;5<?iJ6Nzc;w<j-452z+R{4_wD8`7ut96
zD3eX7YRz5_%lCT^f79P*+84C9?z-f@m&Z=-QS!9fYqn8!?}16T_cs6G-urU}<37z@
z6YZrzL%=-y_AHvozjsl0r?uyVmc2e<XZA&j9ori_`|jRbXO8Wi;_YcIThqK(apnKL
zQR1chFQ3@Dx4kEO?>3hi`!A@p?)SUwX@9EwkX`qcIeRxgceGdM+`czOgx6-<f!ljI
zt%~+r9hkBA@6K0y4o>mezeIcb-V+il?Wct%+N9iy+b^~~+dkm3;_mH5oAz#xG&>M>
zWZhnohI@O}v+VXIt>3?|yWh|Lm%Y*6<>zGV>(bQieCJQy_xY^$-mju7taAm<?(^5S
z+*@UlWgF$7x-Ztjz$UlryUo-04Ey{&R#@(g&fAyJeP-|SK+oNCSI^iRHg)#iYKBdF
z&1Nj$`*_yzJ%={t?tAEaZSTcTe|KM4Rku&;h>Z<j%9*`KtzGS37TmD;+7WE$Jezr6
zbkIwyEB9CLeIeCgonWDBH|xcQy^f8)_L>==u*(o<w7+*taes^BoqgddKXx;{j<9bI
zda>t=&d0rSi|+2(yKbHRiN8;Fe_vF;SNA}H9dD5Jo<&pa?OIu;*a~p$vi}|P$X>!L
z-(JfiVz2VUo%Zu89`4OOcz$nSeCxi1LuPg{0Wa-8Yx?bd|77pJj)Q#mv59B*71iI^
z+mgC=PwZSy`;Aij_r|erv(GeoY~6g=a__{b(|c?W%iHcZ57}4MAhuWNQOdrij^@2b
zCb8|E*!Xyl!>`kO3)el`>&M7pb8M%)J+CkO-if>VZ6eIy?&UsIvBy1r>fU+hB<!W`
z^jfcf_{c7wOMdT}$_2a2{NrpK0{>dws9j;}v3Z3}oBy=Eih)V{E*?|Ze=1$T&V@1B
z{^_wzdowfo_r2Y;b+3`|w!I>Jy!&Rd9^SXXqH^z2k&`y>*5uiJzN)-e`a-td)R{c{
z`dQc7O>qv~^XWdD{W9lsdsZ)+urDY#%x2D&U3+(2KDcj|Puo6slbQR|`K@-x?-JQ}
z=!&e}pId$Vj(I=bn`^gvZ{5W*yUiO)_lj-Vwx`c<*51|sPVBAT#J4ZhgLB`psHuDV
zR<`XuzGms(rU;e2JjL?+c3AD*`}M+;-Bl@v_v-q!?yd4&V<+^r-R3VJ_udXSu6;%)
zTK4V>WZ5_K*{^*UeBSO2pFVHzw}hj6{i>MvO?Ozg_kQReo0@O^`xW=z+bb!Nu&e%U
zhn>$a^ZhGla@bkrxE|PXRD18t6`^~j{wdp9+nn0l;I?k>k85l97qs8q>r<k-N2XcW
zUc0Md&$5ijeU-{<Y|eRKv|bec%T7*a*WP-@mwPu(U27L2KV@%1f8*YzS$g|YL|*SJ
z7W-|pn(^=6UG>xU?b_nMhr6_YU%gV%-gdLBy)rF)`*M6Y?2*a$*{?L4ZO@gLZF|`~
z?(SQ)HDGT<*QtG5T#i_&NtEu}wOe$b1pnW?jXli!cB}o|Yt{R8uZPB)y;VxD_D!tc
zxtA#@&*s$ru6<ixbMDK3wsr3#E9U(Si+lENsa?K*-=YgPuOx5T7`^Y_`(|p@UjDZa
zZ0qgI_BNGYx4!$+-tOHE>3y<3q4xXw4%uB<!mxMBq)k>itVw&H>K@wr^xn=ruPV3L
zeQsa6*M{l9o`t{N>>dfT?OXg=+dgOUX<M@omUb(7x7t;nx?wkUU&7uuUCw*&-MY6o
zKPP_gf{)VsGPNey=gniYpZRpk-l<OUb_uML_Hmv)vv29r2R1zw`g@ZWhuXcLJbf?g
z)7^GIg4Oq4c<^q|Vd1^|?p<!OozxJ$@6eByz2@}`_6a)s?$uu@VcU|ucJEo!{5>h(
z1NUC6;@^8uF=9X8tdo0FH_x-V%R6u1-swO0GAGX4%k%TJUB~SEd!-eY+GTF;+8cM&
zZttCG%Wal3ckWe*aNqa6S8MO%L+7lWbgu8)KI^uv!RlvwF7z7gpC!1}Mo>%7KFNB8
z<+`V5b_)m`w>EznwO_Av@7@!^zxJ*?a$q0VTgH8l`(5^34v*N|8|P<V$$WKxNrJh3
z?PDgJE%#^a@3&6f<HmGuFaIX1eIG4Z_8Y!?VYiUIWp8QgMZ3qWi}z1VYu%^x``F&B
zxexbR$HduxeerpB(Nms%>*wCuE8SVV*SMu)&*N)ft^Y<`-Y3hxe6L~M#JvVer}s|n
znYi~XgYsUMdsp^6Rz6^J{LA&dIvL{oyz(Dg@0qi2uaB&%b?aTRy`CFO_wFg!Y_;hA
zBpYcqKKliJ-}VNbnz6@R=FZ;jYqsrGkE*u6>iTMLZ9CUK!FY`Wu}>D;#UF6n?{VhL
z{``IS_WBk^?w>RF_a4K(bJqV4Iqgkxsj`>*QLtCO`p(|>ieK%j7VX^o;#P|Nte99U
zs~UH^XJ2(3?74F6IR8!A`%xlt&z_A|dvC6<+naW$Vc(fNNxMKdrv0&#F7N$jmv8r5
zwAKFJvn~79d4An%5~;Xn&uW$Zh0!zj$|Se!XFGapZ=2F`+p3Vs`<kyt?7gVpV$Us=
zyU(xJbD!QZ*S&{ls_ZQ>IJEcH!XE4R8QFV3?E>9{H^uf;`r|#k9i02#?c2C_oBF@K
zhqyRxdaL60$KUDL8{cYqplxZI&Gz+5dwa6;_nx(2+j}VYh&?0I3hR^8U+wOHCAa@`
zfT#U~*8Teo9cS$`ZVK9Ka!Pyuok@IqA6$5|cll$secA^S_jb;DyZ6fuqrE}Wxpqyb
zdTi#rFtES6>gt}onuhkL{_F4G*wtl!>+}D;nG8SnIo}T48{=kayIzHJ&mQaI1G-Fn
z_KnXN_N;JM+Q+N3(Wdz6nSBzTm-b3-eYyAG{2rSW6`p;!t_axiHP`J;&Re(l<!_aJ
zt$o3JOV(~Zu+J}N&!k07`?A-~-n(QU`@VUfPwcH>71*sQ^l<Or^sjqOUT`0<b^E-x
zH6eX(bwAU-J8r-Bo}9(HuUV|%K&Vm3-t(X24m5dB+?yr&cki5bwtby4k$ZjCD(=(0
zx?%6kpW=ISa`x?8#aOiW$jz*ML6MU7#f{wizN+}zDtIs0JNeqLy%riA)@^M+_RD9y
z*b}~$bI<=J%m?I3`}W41FR=5I|GandiO78oCob8|J>+npn|Iru^=~BY5B~7ktIy4_
zPwdc{eU}Wf4mg&dv|~R#bI-d+-)-mY65Ri0srSCKn^*5G%hTO=-7C$$$%tv6iQRL%
zUmJz?Wghy!S8Qw5zS4}3HjQ`I958J)wE16Jxo^uVroA?rQu`iWv)@~?$Ia&ZvnJaY
ztqfLMau^RX{NUKTZgJV3SKlA(z4O0&udv6~y&u~Y?QG8+-uoub*goFx{ho%M$M)J)
zT(S^x6Shh8ooaWB|K8q^t>rdvm-FneUy*7nrqjA_cE*RjsY@PM>n;DYZ?gW5z3+22
z@10=ZxOd;riTjk8Htbn=|Ip4dg9CQE1y=3d)UwVl^Vf#GDXAOx%SWHKn|n`o{}qjb
zy{z9C+S&F#+50%dY_I+rQLF7GEcPm@G5as3tljOlwtcVfbZ@)%+Qa)UssG-~vgGkz
zBjzRhZiH;!yY=pRyML?qTd!d`y*Ktp+dipp$~JeU-|hRoBG=~BF~NO#V)OSZslDIx
z>(=Z&D<?$n_V8b0W5FlBhw)$UzSg)c){73i*r>l<zR%-!sg2&W&%1wnF0r@!eRMZh
zv*7`bDc!q26?5<Fb6>hw>977?Uwyv)3nV7&)e@+-@%#RHU)*Kgz3zMp_Ia(Y`#;KC
z?Q`w3-e1yrbMMA!*Y@5{W7zlUjOBhcw<$KVb9(l6Gq1K~ZnoL;p1IAAllku6YW7Gw
zQR&$ICVo5jUoASgcf#gbd+lxu?c4b=bWd6Jp?!NU>f1|)y6v5NnaOsA&b56R>znsp
z{NAzeg51$PJmLrUUY1{D_q|$k?}1_~JH>FZy<4Un+%sS0&^{OQq`f{O`}SEH_UujK
zsopoG;K1Is<yw1<55C!Zl96ejyk!60IM2|%M<z8LFnYgX@7k5i_MYnX+ZPnxvsd8A
z$-UBMZ*BemF53J3cC}sC?#{g#Qw8^SZcyLLS;M+d;$FgmWf|^!&-jY&_Df^7`}0Tg
zK(1q(?TOFI2e{^)-}`J<uH6Q~#kO)%pZ4m>zS!dzc5ZLe<vn}lLUZ>zTtB>b;_<q@
zlNLPMFJIiTr&VM2{vVO7`_*GN?-P15Z*Q`oz=4JHxcB{>`E1Xk#n$^|EkpLL2zS|c
z<0s#qvz&~3`QO&;U;6U#-amp;d#fbo@12sj*=B+A&)r)JSPygus_YF?+PP=T+BJKE
z&z;}H-l1*BqTjRcp1$F}?Dy~Xh}-_%Q~m7Hp2&EHy%S2W+2owo+!<l<VsArGn$5gD
zQZ~~zSMTGu2-*Al<9?e|Y0dqb{N_8^syFZF|FD101mEX-`YRvqS<}8__viAY{n<C(
z>}8*IeXkm?^MN~><@-|K?Y0;Bs$$=K^4^}Li%j;0TB3V*v2yPz;M}|SgptkulXt&b
zH!)nYIr)FV{tm%|d(E~VvR~LLxObLpj=ktc%l)ZO?Ckv+`SvC)p10qiLvhdE`@uHS
z)2{40*I|5M#q}+AFPEq8>E@5$KQBJuz-IBw`|`NBZQTmQ_wD1Uv8xqm*jsY*yxr1b
z-hGnuwpq^2n7(i6bHlynS6cUK9+%pCB4gUVYq1CRHcf4^@y_Mk7r&Zi?<u8!dtdG5
z-j~yTZC`p&%U&r~+a0oHqWcz~_SrkTi+7(nuik;YSIcZJ&*nXl9ldR@!84Y9M-P48
ztL%Jo@6!B`{rbo5?mg-|e=qyl!w2{;gxc5~Uu3r`Gs|8|<mld}Zw~f4CyV#m9AC2M
z!WPDT+H-XGuS)o{m;2A=y)z?R4pd2Z?R|T6xxLeu&Ryr{NZZRlQrOQCYG*&iV*lO<
zu7drhcZK#YTw=R7tL)4^L5J)Ee>P`Z%WY!ZyV!*3z|p9``*!_n+vm*Qw%4m>v5oVe
zZTs&si0!d*_p`T(P}^6!RCVvA^w52C+E4FIxBqYVWwyP|>yFZW59}XXiVCgX%W>uW
zF42hZdzMDO-TOBwY4^jV2m38jxc15=)Z1MD=(X>cj?g~MS7-MMe;2gN-?eyekI)Xg
z%WF^Xb&lC#<8<=i-n7eldlzvZ-+yMi&)&khr}zF`dUL<izn1;Wrmf!BzJArdU%VUl
zmK65votq-Cv*GZPz4u;C*em#TvE8D)nR_gL9Nin}BezdcjC<d+pJMxtPHfmKR{p}S
zXlca$x>a@d%L1p`L~J~`_uQ58J?%_oHtr@Cd*$j5*fGtRw!bF)*PeRLdHc;Xlx)u^
zZr)cEF1<H2_x9dB-P(2`(KGf&hUV?}G@ffS?=!>RqJ6CU<XVjP#@u$cZ8%WAZw80s
zzQ~Yedsj5y*c*27``(OGtozn`tlAsrJ!!9d&`&$teNuZ=**EQ7uR390*3OxGZ6h!4
z_3lmF&%3Q^PsofN`(N?=+<UqA#ojHcXZ92en(u9>mfOGZ(dNCTQyQ!}KCj(>O>U9h
z*U$z#p5<J2Eyez}EJqaVcfXV0`#{uuZ{_2adq2(g*>7-Tt(DWI*L#-D7eDap@Y_AF
z$}R0#y<YF-G;Xmk+|;!%E7fPeg*)rMXa6Se<0<2@w(i=u$5G(;KBiZa2R^N>w9DM1
zx^K@X$pe2DD(>GndF4K(XH2%K^XBed6|1y=_4mTP?UthU8$_M=UO2XJZ%TyfKCf`6
zJ+oQ)4@{Jwv?u<M#XgIS*1hMBRa!>~?bs8bS8B6VjA`Ff>GgYWGUy%9cHg#FQjcjL
zTXg5%S<jc-9Cw<r*OYJOe(T+D_b$nvvR}F5!XEXxU-nvSowqr9Rba2$nmPOZm#y5J
zZx+3`WGcu0u!*<#MBiX`*s7klKTYoU?qk;<+c;}I-1G5U*PgZW{#t9A%h}Jr#=Y-B
z+n&8=UsvqkIDOe(Sxr^@Np-UOzS~T&k1SF<VDJBA|7|v&eg1M!_O0OK*_%E2=H7}s
z7x$U2h_-L2UcNtr_xPUsN4oZ3HJZFHG34dmy(cH`oqtSxuf2oq{)6rt_Bt;Ruq$Gp
zx_95_+`Y{&&hAw>xO7kIyW6&=>>PWYR|o977JPE=u^poO{+C7X4Q~Crx5i}2o_!P5
z_GV4|xX;*~b)Vbh`FqkOyZ60GxV`t#NzQ#U{)+ErW74(t2;$y9qw$i>nuo`2+S{3J
z*FQJdd&~XBURK5Hd%dd<?Y-5XyPy4Cp8fviE&Er<sqOc)JiI$RwPD|Ct)#uHwM}+k
z2tT^l@AyRf)86~{#OW~Y3*=vA@9<4<pU}rW`~DS0>^(E{f?e+KmA0C#ZF>&+@$Yjm
zer<bDk98kA_uRdvk5=qc)|hIyU|zcI!*7#q<|uvLx2jIlPR?oh-akLr?TyI!w^xl#
zaQ`0Rz1FK)H`?<pe7tvayZYX(eoOXgzn{PNL_^(vOUu%I4Nb-S?(EpHH*d|`y)Ues
z_RcU(+naFm<evL~_4Y_gOxQcC>&M=Bo2I>2zkk~6+2dm^aA@CNj<9()OH#ku<bArb
zf5xBfd-eM|_DHU0+gG_MWABRF(zZ8NG41_&uye1>nz{StiwN%jvj3EQL7Sre-X-Gu
zEDrp!tDoJpTj#dmzT5j4_a)o%?9X&qx%b>k!#!7~?%aRR&)(MMcAI^PpwV8P5O({t
z2eYj<xtiJYU3S~Mn`4{ZYIU`}<~wBfKH{|BD^hjbTD8}IfAila8()Q2`^?QB?w`M6
z<39C(qkC`7X51J2;*I?kOS8SnyT98;_BHR-_PDq=cJ-QlTz#+inyq2nFT_7@&l$&n
zeU{Zsdz0Eu?44`Ny!ZIxqP^lPL-(c~jM&o^xXF%XlF?qir;GP~(W%}SHUFSZNdbfH
zjc2d+F~m>YYxV5>zOZFo*5V9P_BhtN+hi%7-}~sN^?vWaUN-g`b8Su(O|)C<5VJ4Y
zXpe1>%>I4O9A9l(gx&TZ6=T}_piXG-pR)}6R<7;0)!WRuFL-L9P5i&`eUfSkd*^Pi
zv%68eZ0}~5oc+Q5*Y?f(thxW8eXUKp>g|2IVmS8|e97Lcv;E=Tw>q{qu5&Kfv1f?v
zOWg5%ztWYQeat(S?PJsx-h1VI%ig8i<n4^lmh5r;y<z|IFFku3nrGW=Yn;8;?epTj
z0U^ou$ISfpz4_j<Pi(^Rz4ETF_e^-TZSSo_j(rB7m+ZZ|bnV`++t%)V!^5-BTV%=J
zhnHvUOYUE|yJp{!J!{*}+s*Kk+soPLXa6Q~gN?Us)n3l<4K{DW7VoWjXtOUpKxywJ
z<@eTU$4l*0H4FD2PF1u2(B@&!Q!2Ul=jB`b#B6r%bzh^tXSu?$J)X1O?FE0`-J3Gc
z(q@H(v>lJZp}o~=diIM`HrWI`Z?d<K`n9j`8Q&h`r5yXjjjr2e+<3fqL*|CP!V%~8
z9ZBWhx3*&6{x7opJFfM%?$e)s$ZqTIW&0j*DDQnOHs9t;O4~l>ixPXEeec|-EWN?{
z<+sl^fvwAJ4xMA!cfP7>um0N|`*y$T*gM7e&7Qz3Q}=3X6zy?H+iT-{Ud^`hR_vZd
zTV(AewJmlz{g}Mx(v3BHufLpcqiXqgug9++d+wZU*xULoW54tBK%2_Mo4fr=nQW3W
zn{2u+FSUQd>bp1kY^F^<U+{i$fi?R+J1(`CePwKaQl4S&7CAmUn-AyqdcJ4dn^XOI
z?}r@>_MR3i_kQDJvwd<{=m3}atG)TU>h|ulP4@_0eZ9|lr{8{;o96p77yQ_JzrEYO
zUvuMLkz(<^-(xcOy_k8#Uin_!{;gSldry=F*q8k2-G5^0u6=fG0{e{o3ih55F0o!0
zUby%B<px`)SpU8MpA_w#>BPCuGkeC~DbWu0_X|$iFvtAas~R(RuMGR$z3m$~_qi08
z@5xPAv*)4Sr9CPtTKmnGSncgt^?aAa|4I8c9O&4aJ*{@{q$3{t9z1xwHz975U0}HO
zo_LN|dtV%n-fO8h!{(4>kNs4()7BeO=k47YdT8%IJ=c9xzcAT{R21)LYgE}SRJCBg
z-p|E*?-rI?X;tsr`}uRgei>7aeS*g<_qvos+X=sUxzl4ts2$@rD_f2>=KUTDR{Pyf
z+_2lUE!1YIF!TNzZV8*iEc^E6_;K%(3NhVZThL_xaFXJ_t8A=$o^7<;zkT)VeMh@0
z_U<`;X>TXf+P%J0vi4@jp5Hs|e9PV^9*ujm-yGP-GiCc;{%P&|cfPv2_uH>OHWu>x
z_r@g)*~(9h-s`^Lw#_Ybg}spn{P$-5x7~l_!nwUl1p@nGCWP<v|9*Ind!WJIEx&H>
zl?+YXyI)>&|NM>Zdqh0+_OAJJeQyV!;J#_|m=9>o-EC8zkh3>)^~QZCZi^l`bS~O1
zxPikiAj*I5!Baczj?HD=+jdxSpU~}nd;7$-?0?_-wntLw%<i+L0{d=ETDJGj-0Ak#
z<()PPlI`}#_srhE%i6)7L6v8(z|#f$C1VcnO?G&_*Q|Q&zAKg-_UqRe?^obBv$wPM
z&c0Jlns)EkKHO)Zt+D6gx1)R0+Y;@5pT4~J_nbTX7EegG<vPH!H{<x+eaRPj_h~d`
z?Vta>ai8}6CHq{q80^jNn7Aj+`SG47frs}lo-bg#w7p<&webnNLn6X^ounA{+6PVA
zds41qFT*1Lz5Fto`<`B4-1mCX#6A3ld-k@+oZl<A(Qj}1G=_cZnI8KWWG~zMPM&>V
z2V14xG={1BkJ&2jKWietPySbd?d#{-b`zqt_pV>{X77Owb$jdg$nJ}ududPCsl|Kb
zyx#4TJJ7XPD@E0Q>9!Ml*9AoE`=7REzq#q@eacsV?M;f2J0Lv2XRpV{4|{oR_V4@h
z^XfiRG0T0;z9;u&rBv<vJY)HOrI15=OZq?V{bO{0Z<N;heNOYst@)C8_VsT%y!VRv
zzP;XNQ}#WQ;@;DK?w9QoTh_f(cjfG}mXz9)I+b^C%FBs+Pw>9p8}ic7CT`-Lz3x|+
z?Y%jXW#3DC;k_Ok68E_-H?%o;V*B1J+;jKx|Jb(sT>01i2W4CLN_WZbO?!1?@1&P)
zdlzm^-Oty~w9n^k(%z4^s`lsWCD@s5Hn+c~zQeAsICsx6)r)q`dMEaZIq>W|*}r?w
zhl8{1RCaM$TYBx^n`61$PInsLUIkG(`@q<zd*eMf+I_gadf&{lOM4@(uCcl2dVJqs
zyZXJI!P<NO9-p(X(6-(F!p?_wHEX8rovU!(?*5fi``8ae?2|n{XK%jlroC5gJhqeC
z_jRvIs+`@1v`2gUgf8y=r80S+SjzIfW^zIY#2R<(%f2{q-^q_hEw9fL-j~wCy3ed=
z;@&9tUwh5g^6%}qwa<p7P-XADt`oLTyK47|Ci2*DFuCrv2t2aKZHB{MrM+SH%qI=^
z{OfMnv$f7}Z_$$jduO=_9r&_}b#HF8i`DTB+52vtd9d$u(L}p7yO|GgX6x@&NtnJ*
z@y3@umi$`#e82D8lYM!ry^{8-z1&3)_nM!Yzt5y7V2|}o^ZgpK|MyP)Xu1E<yQ=*U
zR`BkxIr-VDp!~Sq&%Y1$)|@!9_jc0xy+($D_RL59_GKwN**oi&?EbW=Yxh0yII=Gz
z>C@ijYiI1;_Nir`r4Gx!`2i^hVwN1RFx>pl`pt8$eG=v6dv$f=?PD(|>|6acd7ro0
zC+jWoT6@JfYOM967Vg=0;+VC+{=dCnjZ^IQSmfErtvI%qnMG?~&NqR*pXLeJsFt$s
z&(=@h`%>WE{($Eed!HN#+M|&6VQ<pA&OI0R)$Ets%CPUm(iMAI{@=2juex|&(Dk1E
z-A!-zG~Lv)k>1a{FEXTa@0m=cy~|`8_wL;yWq*5JsEuH^=iZt?TYEQ4u6=&VU3)LB
zkJ}qw?`MBLXV$**h2^$7mx67g5;*r6&Azj@j!}DW#``1tY@)mD<fp#e_f%#5-izW<
z`~GN~+B;?b+9RI1Z0`~7S9^SNcG`6bX6@}yJ7m8}<ksHJf*<#0zFxLZP>$I)g-Leb
zzK+IypUvOyHLg6r_xt_Qy@G|@`*ws+-}^WH)ZU7J+x8}G*>9tJ#$@-+bIWW}x6j>I
z_;1tR3*7wsG&*GWzSUFSE8wMW7y0eY-s!AvdwCu<?|nSy#ojM>ZTGvVGw<1W!Ew*B
zI!jv}=LuGPPiyU^rd{0cc{6&S?mSuhog!2AzWL9$Z{93{eUDn|_s2<}-`%2<vG+|o
z$9^$2{(W0df3_?8+qZY_T3vh5Ji!CfCgS^6#oXI#Q2EQonfvnI+KA`Y2esPvZTHZ%
zo7kkjf482<-f5!Zd+#;+@BjAa*Ise=YkSWZ^6e9|pRjMQ%7eX%694S(y`Hj{#c9Ug
zTP}V3`d0DoEm^L&kKMIuuZgnjKHIZb_K0e}-P`uz+}@k}@9bq<>%6B)?A_j_>n_=@
zeV?$m<yQFK2<3VEa!#r2U9n)_UcaNa_UD_m?LD!%Z2$S1D>nD?KJ7Jn`F+o)qhI$n
zFFCUBc<GYemHfZ=s&?z_d*8KjuY#wxz2nU&`z#jzeHn91_ub5J-CHs-V|S_U`(4w|
znC$Odux0O7nW@%uw`lEcf6}$r`b~q~dP9YM)@KwBq{%$qdptE|f7G4XHUV1<_BXwK
zy|*ujao^#Gjr(})pV`H_{oAj1?$BNnfjj&EY1Hrc+O=_?uJiOgCO&<8H|1a6JN>1{
zp4IQ??BDy?elL4S-rncVv-hq}li53AN~V1kx2o+C-Vb}-b$IrzSd(GZ{%F$Pi#J=W
zr{~_^Q(1b-dSTw2{hL28-#O!*tL?e9wfo*hFW&nsn0uc_%&EOS887$zb&NQ`TX0~{
zHqkG8=V-p(+r3|KU(^ykdyBce`z9`bvFF{Ib$e63o!$G!@1|Y5e5U<YW{Z6XF23Kl
z>s|leiF@YlVOuwGufSTh{U#@)_hx@%+Q)OhYrnY4n!VTfGVCNwe(y2SDYN%qdU)Tm
zkTd(wc`)zu&i}se?XPKjj|*JhGs%3*KI>P<_SMC#vg`cEzR%D3#J-cS829Juzuo(!
zWW(NLc8~X7QCnblYWB{(raLy+os+QM`%3fI-g9mD_Zj`_*xM-NV(sq5v-iWtX?s`2
zPTk8m+jx(R%$dD2xZC$UUvP6b!}t4pbUjrL)bIbiCn@gpp6<Id_i28+xaZj2O?&&_
z+a2%-*4o7zxMhDtlId=nUp{*b%=>KCn`P|X7~ipf)0=5~x95J_6Kx*5?_x`hZA)RI
zy%(qK{`ss{)-M+O>^natdhfX&k$vktpX@nO-)UFybpGC@PrLSR*(Y>h@11FT*%n;2
z3;nZYucWYved5Gd`)8f+-M{AK%Dn<XGxsrPaP3=?{LuRHjJ|!%``!0hUzN6>bMO4#
z{NP=7PvSD|*d5R8OZYw4Ix4nf&!HC`d!=Vz+$(lz!>+9RNA~7Q*Vss2&E9wKtl+-c
z67jbD@zJ&lwT}BXKi;#Kw;_HHhmPT%{L<fh*W8V=JuiG<&zTH0d&Sl9dkeN*+<V+C
zYoEiNu01+3i}t>H+huRl|81{PubTbVL#BHZrrO#*b;#elD?xZ~!kTsan3aY0MK{jg
z`{T*qeLpUB+iD)&Y`6Pc^nOR<N4wJQ@3;H=pMM|!+zr;E&w2JRRZ8ru;#<DAvg!EV
z;*f)Ov2DtG!X{j_OZ#_h_i?$mb}Y3VcBlQ8*#~ld+uLt$zh5=fbnl5J+xIM*J$c_|
z<yJfP=zaUW#h&g?X=bu}zDe1xC3?obTw|4e3*R>G-On-2_SM5HyNqX@vK8F*de56R
zhPJyq=kM+02;Xb>YWKc%=a~1+uV>#U{(1M__e>LP7WQr4`#(y3@4RQvt=8{fVZHQ_
zuKoMB+It`0(%9>pvSFW)4D-H}4=?xHK2YB0R@t%lUZ|+Uw~PC&uiTh!lawa7S2rVX
zueaShyTTji_wH~owNVz)JCHcr&2DQ@;Qo7WS@ugYitn2*(6cZ4a-+?hO{ex+F+AM6
zVOO}lTguG60a=Uprk8Tq3r%>kcaKVy-Jy_`d-u(*v19po-2TdSFMFwoC)Pqs7VVu`
zw0DnZ-R8Y3m&n*`v6*DQeQlzBY=y?YR|`t^XZlarx8u#WeP3nR_vT&W+1IgWt-a8d
zn!UT+3hdNe_U|e6YO`JNf9Jl*QaXDRJw@#I&6vJNM^0zo8ovd*m9FUQ5xBJ9N}Er3
zug}(4n=@<9@9i<L+b_F0WcT!>hkLUtqxQW`W!opP;@{pil^pvk@1M7s<hF62@b!wl
zl|0^iOrELjo&WFP-h^n6{lYvecGq+|*+__OvEzAm$nNi#rS>X`-uBJ)iMHS8NAJt|
zpugvV(5AgtQq1;l;#0Mq73#f5j8%5;GOZ2!%1*4_d+GQ*yCui9_8oebYwz0|w_j@V
z-2F2vTK7hl%-+Y>FJsf@Tx%_OeA+&-)N6L?&od6Z4VBwloc(rxO`q2OHTlW=RNC|Q
z{x5m8XUBA>{nLJh?iJE{zi;P@mwRh=<?ppN<gq>eWy0Q>YkKXnK1uG4=-}F?Dz##7
z<J_&*YNw{}&AV=9^MGgd-qVhHd#jJg?+bRCu($X`psi`+)O|nR##*PX>eySgHgeCB
zj4OMyXPWLS>B-(Jx8%a!O_i+sB+l^cdsc92zqFXb-o|%-cHhcr-uLOX=Kk1^zwMq(
zl;7_!&S<-E`bj$>p6fOb6;JP)aC!0GmWgNfu_*rAt99kT-Zhdt?Iw!$SUsBbVXx5n
z6MG%^?Aezzxq1J=yXO1nIY;m1=~T9hI9#xIP5<=0o5Usdv7allE4FIi8<sU?@8gS~
z_6hxA*~iRWu#Yi$(cXtQHtcQ5?zG!*p?k0VqZ#`fuld{bN-fyaymQGu(Mf!JkFZGX
z<J`!!?`KW?zH@uBc5Ccgy|;PR`@N|T)c5{me6@Fe9Pi#43;ylx*;cmK@{*a2i%rQs
zm%{aXcbq!6H-EW?UFD^(dtaD2?eFpZxjTP$%iacUgS{;irtLLtR@|@IcWiH)cirBv
z=OgXsPI<nM>&ZX63452@JvBdJ{mntZ&XfP<?s@0$?)@EjXz!Qpl6LyGbNBjk&fd#g
zIN$Ea263z4%iHajJma#}mX^1FJHLJ3bx~EjvboIr-u|oE@3MZw-Y+bHdmjW=?>qM1
z*M7;pE&I5aoY_0a?7wY}?yr68m!IADL~GsNt2qXHP8|!f^X}Yb!&SA_u895Co|LlG
zJ!cD#?bH7^Z?DR+i2dj18SIna7rbxlkN11nt@i8<U$=Mnk=z)YpqstcIi8>P)JO&H
zuFqU+)7ib#rc_*cpYpejd;bQ%+S_vN?w<Rq{CmAWPu{=g=b^ogw|?)5ZPDN3e184j
zM>f;<KV<r`cYd^q4g1;aw)L4#`x-+e9M+wa+i$z}>|RbS4|~>4hxW2;d%8F1SH<4P
z=lbp37q#0kpVzW+o+7co>Tki`H5u&omJuKJa4u=I{dVuc{xb^U`@jD9vFA9;j{R%6
z=Ik{-%DQji@$h|B9Q*BRvRM!4)^Y4RaqRMb<*3#B9=x5mFFD}No?O!hdu>15-Me0O
z?q1p0`2Cx5<o9`Z)$jclIdkv53Z{KOJMY`QEuCxg?ZBPA{Na52?l!jDPW(UFI<}x=
z?{8&Q+Y9U1ZLYH4v(ffx*_-f`XP-=<wWVp^hrRC~?b++?5wkD#qx9ao3q||FU;W&Z
z^0Q^{i>3>Ew?{9sS#q(-F3fGFmDlN|*7G@++0I%kvhVFlZHKGty6mhY%=g-EKX1=b
zW4X6@+qyk#o6qb`nKXUBhOOV;<uS&4Rp++vpJ^(v@41lw{taQm`<}n$wg0NV(k>!E
z$$skBb9+rfgY7Szlh`}y&!xRDSCs5q!}#28X4D(I3lp~QJ*Dtr-wH)B2kSHU_SN;V
z?Auq-y=VH_t#*+oH2126ayz`wo4IGQ;@dq}C+*(D*70cXfj`~$mjhSq>Gjgt$9wg{
zo~UPzdn;p9_7r6M?6Fz@c<(CisJ$O%cJAAwdvNdZ_fz+p=NIpbket0cn=fFG;py}C
zS1Z}~F|yvV+g<!>uk6PkHc_*T_ew61+T$bIxPSY!pVpVY=<h!8y~9o|O=SO{PYV02
zR;wMzo)f$`ek+?j*S95m?6q0<?R<LD=Egm4dsmzJyR_4f?`1#E=x|1C)9&EIfA@>}
z9^PBw*0lfXkIQx?@21&)eayFSvsjJo5~*i<UxjAbD0WEg)fAm&uXEOGf7J6Gd(S_$
z*q@jF*xtnN>b_}nIrcSGKG<8E`DFjk{m=G>RLR&cIeu!-nV>6sdp+jwnR%++=A*8U
z{oNaR`|4kt?-R1Qw|DAZRhx$9p1p2Y-rKM(eqwd_{cLONy|VTi-wO8bNi5l0;$g7w
zq8#HsDdqmX3m*sVKRBb?hUas#eSsAFo?0i)eFqp;?hW3`zi;pHwRYvXg?p!H%-YL5
zeW~3E;oyB@`@YzvSU20LyxYGwOjXD(YsUP&(#IP1e4oO&H(8xy|KWdY_xyTNw)eTu
zV>`hT2J5L8zS=F@lE2sM1DE~!^5^?sIe*{VU(~qw*R7>?VTA>|+xso|F8`#rH)%$&
z{mSmS`(*#k-P0*Aus>ffWq<8~h5ObR_U?UqaLL}T$?^Mz`w#4mUy`@q>~y&8q-K@9
zGlJUoSvcL=%iDKl@BSHjdo?r7_igT4x#t9n?7q1>s`je)Ki^w;cg5aK8L#%p`<L5A
z?moWve2I!}|DCpdK`XcK4XDtv{o8hT|2n<*dyg-U*&jCV{+>qDJ$qwzI_z2H8)_4q
zG0pDZQ-*zO->urcC;XJ%(-5`&M)?u;D;O@@ey!KvHzR1Gy|3lpy>pGk_dbh0ZY?@v
z!M-DG4SSb7UA%X}`RV%;JSXinlSw>a^6<*u6HJHptJ)gxe`(cmV9&BQdzD|DxBoo(
z`(BO&mVG=6=I+Z_HqCzedsc^A33K<FEX%cHleMzz`!{{xEWt~ACm&wAC!ku;UTvem
zUNMuc`wt&ov**e_5nHdUg1zzG&-Si7D(!H*ZPwn3OC9^N1P<=m_W1tZ<3?F~JyxFE
zd-lV@y;8S+>{(QMW8aKhy!&1qG1_ao>Gj_B{h#*!ss6QlL7Vk{j)>)ZP0q{PmnvHA
zv1S#qoscwb@39rT_DZ-M+V68HZ%-8;>%PN@>umb8>-L|XHf`VT4+;ky?l<lI-0pKg
zv})$wjjW7&r!_Cw`#yHXe(S=Q)&W%~tvq}b?Cz(<?=82m*nj(i;NI<rR1WYpiR_!4
zR%|C&9<rA=c!qt-GVZ;6^Y_?9YV6r}Pcm))AvK-@JZoR>jXbww-`<n$c3TYQ?vp<`
zYwyD-m%SCCZ2SJt?b!S9p49#$#twS}e{}3wf1!8Zp~gg;D9v5_jypZyYsD;JE6t|9
z_w1u@yJc78SeI)2u<2=K-*<8CjXl$>rrIw2vS9DYSqt|nAD_Ojt?uI9v@O|t(<V6X
zzk5DxZ-Z^~zUG$SdslaL?d3ejxQ{cHci$?0x&8A_N!r|>%e^~mp6dQe8<G7BKMUDb
z?@`%%Cdg~goeLdy=Z{D3Tk>GlUdN4W`wG@A-sfm?Vo!4OuDxE5759DLJaMl%FO&Vo
zT}FFbvsi7n`10&4n6u2zm~Hc(r4@qvPh7dZr?8-5@7cHA`#$jY>}Qa>y-&=YVc+(o
zL;DtcUD$7ZtY@#d{LMW<)AFqE9dFuiWpj9Mi?!GOPibm<lzubr+g?z$?_9*py?$vP
z2TG>K@8uEyv)6J*>|P$8|2CEf_wIeUQgiSB$LsgT{k^=$xAL3qJFW}XiOFqy9Vflo
zyM}Y?UWqd|_BeW{9dKHA*yiLG!ToCt*6zJoaegmn&EGxjyeWI~9U|;B`2_ZwR9xFL
z`Ki&~hmSjKeBVypccin~K52`#ZCO#Y-KwG*n-AVg_p)uhz1KT4Y`>L#?_QS0;x=zy
zYV3D9T5loXGs$kp&!;wdT`~5H1mgDVE)hA<SNV7Ep$888@4LG1JvRHY^`eQB>{NWe
z?C0{oZ&#AIW>3!|3w!6ATlN>l2JTy?u+nPlxfy$$v_%gbnRjLHyeYr;9sN_V*LLf&
zy#{AnZKio|+IvM+z^*y@=^pLowfi=lJF>U6pw9N=E{45gUH|sp<M7$LXhPLqmVZC@
z&pGpYceX#vzCE=c_d1-OY4cd5(faU<D*H!KW_#?;dGCMoI)3lkf2n)Vg+1K+ayIkc
zJH>nU)izw(tIrv{SG41tt&-O)yWQJMZ5<MOZG!Kq?R$5|aUWX`*X~(0VS57=C+-dQ
zmEYGZux+>9q0@WgGRyZ1rO57WIrC%pspYfxE=%aMo%!&gts}eZz93KDeLYt=_wNo|
zz1Ql{%)QdBYJ2AXU%Y?*ocnucd|0%XGk31t+fNtw?{r?eFLp}co;~kE_a51CcYocL
zvwQg-MeeKFQndH5=;l4EzFxDQ*?M~KH8(E%XGa?M8BdYjH%X^qug1Bgz4B%&_Ra}C
zu{Um~zO~?k#d~*uW8Js8QG0LCw)uOKpU&Ku=pwjpSvBk4wZ?Jwe?QponSGMee$Jeu
zd;B8%_F9&2+Z&ghXMIUJZ~u~3^S$<3@Aj@yVzIMWlV{(ZVP${7iR-}4*Sq)bjLX`8
zZO*a1Wf9Nz{+8IkcUz8({cHKVdn5D~?>#H1x8HrX+1}q~M-D8!BEIi``?LeMX1na)
zzC6fYmT~`Hv&-M@(rmi-7Oz^q_r!yJ`xYM)vbR&s-nYAtZ|~$Y4E79(d-iu#Rqj*R
z<FGe%D&M}F%1iAOr}6CD@k?XhlX+r$Z$&=Z+x|RcZ^WFQz2^kk_a$p+?wdJPc5kVY
z#9nWOX?vF(oVWM0%=5i#|8Czs<D<de1KZEqFRoi)y?y?iz1mAB?3?gl!rrfvr}w<P
z^I-q=`>Sl`^*r8tM#yx}&BF$Jjb7Z@%OkdT&)j*;`*t6#-Sg(lmc5gFPTOx4p0vNV
zv&Jq}<;k9Xe;RGAXC1RGefWB>tfsq7;fZg1OuD@HAMKoBqki_n-YrX0_D?ojwnxaX
z!H#L$yFI3Fuh~t{`MmE=JCA*d!Qs7cCJFCnWBt4LqQ$not($uGC0TgdyY1uK&#IKL
zw=Xkdzxup+`*~|$?6W`Gv!`mN&_2IEb^Euh_u6a1#Js;`lA!ImX{@%<LgxEAboT7=
z4A$6Vqw&yYznbg5dHEmqa`7|lO=i5a_tdn>d!Jtj+{@9BwU<?s$L`d`?!9vlu<X0|
zZSLNw2NU)*noqYa=~A&}mfN=HT1x(Y)oZW!7U?MOwO_5fmqCMZUv0gieT^pTKC8v+
z_ePp2?(g<UwUc_aeZSobvwbPDcWk<stlY0xZD#Z3#O1vocP-jGktb@u(4u#H-m7ij
zQ*Lo{{}JhRd*3ny9GEhtZtv@<r}w)zY1wa7Zm?rKer4~jCI9VCtzq5UIl*mbhxv+q
z4&CAVzt-#78yyzfE3)~E-R;e1?R+(a_FcNkym!0juDv2pAKP{B5!)A1q+)l;X~o`_
z$xJp`6aMXSJ2Pi5-}g$pJpq6ADi+AtTHl*u{rnHtzDdH0dtXMc-`m{tYxhgxuDy><
z7VrCCVz_t8($YN>?>p^#cW3+F$dvbch2?wfJP+;K^XCxDz8~`&_Gr1N?L92iviJLs
zmwWlP71{lnRj~K7$ilt-5<PZJ8I|^mi+9@<cJ=J%Tqv|x#8_&-!{Zfu|4f~`H$3v!
zZdvoy`~Ro}>=kjDwRh1DDf|BijrLUeu^wPr*}nH0uigPcKFxiaDHHeeG&AlMTNiBi
zMd<0?3xVhMUMp+gXLoGH-rjq=?04BMxAFbyyKmc5CHsQw8}}Wr=G@nL@5*k0buVl_
z1zz84udT8#PNsWr^mFUIyN?y_-CQnj>+-wTPEOuz@6VM=`wnjr*mrSnz}_ofr8dd)
zMfScvcWke|Jjd=g>=*a3XRp|!y;OGZhgOSyf@16T${Ka<vE8xUe&-R!y?c(g*zYTA
z+<UoGeDAps?tMFj^7eNB{=3g_`NX|5yG!?2vAXa7^ZAm!qCoM1&6+;8NpGI*eLAgm
zUzzNly{4b__9`bY+2hF(wJ)=NolX4uzkAF7rS0cy`)K{@d*Fd-Tfgs($o#VZnbM5?
zX?&&!ypLJ#trfX#uQADPPvYYvd-GLv_Z^tuw}0bV1AAjt#(ffxSJ_A0Wk290?zGR$
z=-=KQFAwjD7L`44Zr}60+aD(G-*xlR-uB~9_q=JJym#g1p1s#*X4~!iHDm9^?JfKE
zGXB`x=rdujKGT+6)dxQAwPjqn_u4LweLt>k+rK<y>)x`3)Ap(_&foV>zHQHJ>qRzp
zNB-;&F4((QsYcNL?{`64b!%H|PP4yzr)=ilXEsIBZk5}Qy#e*2ds|vP?LAV(?W^zi
z?9UZkyT5JkslA<^TlQc2!(;a|x_j@3{-V9h>Wu6;Z>-<jrg&lRzFVCK?wPFLEBK3b
zpKaLQz2=*E>=h2Yw693e-*4h8xwoL~&i>;ayZ7Gb`DEj~_w2sibGO*n{XDh5Lb`MB
z@#MCByB}&h_^GYfC%A9xUS)|#dxMid+wF3>y0^{u(_Z#2{(Yu(+xEr>N$u0vW4EXG
zNu=HGcf0oficZ*9=J0XvmWa)J?CS*gWt?u`D|h+Lp8I#q_kRDRXTPg=<DTZN%6nh0
zW88gp{p`KkVh{I5+*!WQp!VY495Jy2+n0ClbvnCX@0yDGy$_0B?kWGt<G|~?$;Ny|
z?jDy2Ui&Y(vuzmfA3v~>DfPgq?9#mviE;-!U7Pk=ZL!>QFmavDm)d*w$EI`bQ&Etx
zy)o_1ewJxIHU@U`2Ny)|*c&0jb<p^5tzFc*1^er!dhLC7(t7_S&98f{85;H~a!lJN
z>1SlG>c@Yeg{^Qe(-W=zFXVpO6#CBEmtZ=7uTvu5-mA5e4m}GL_FYxkWao6!U~j^;
zGrKu8zVEfW(6yUIyURX3`i>3Dmg}~so=)G}DfM=5a`T@(YI1dZg?6mAxuNxW?`h4w
z_WLid-Mg!9+n&yNp1m`l$?UbU<FN5aE#9wjl+U&;Y}x*0A1BzXQ|sFs{^k4L(!UFA
z_)hiNAGdkAd&vX7eG~2+wcD%iV}EJ2&4EZw8HdwlAFbP6RqRgt|J!pW@XFq@E^Qm-
z-Oc;I&U?CNxj&=r??p=cpQioYtMXd>z@3u^_b8rg-d7p(#J=w1%>zd+u<Z5uCgfmR
z&9LWE6~jK`;+%co{&O5~d~n20{QAeedvZ40r7=%F@KSE+zRPzU_DtW<viB|XuKkJk
zBlhfHq`3Ed%(uPs^{j24=*d`Lu@BqhwoS&~+VRC+wG+Pk`nN9I)5iU5Z$qi|-u*k4
z?d|H`ve(arZ|}9p-UF$(*!ER@oww)E8ks%YX7KH6URb-QvrX4N+vJPQ$-hnZw=d7x
zJ3Eni-{<Qr`?$p(?v2#mv4{U&=-!{%EB1(OskIOM8D}$Tui<``O(F+E`2_bGFKgd#
z$K|p2{r3lZligYNncicySIX$!d;8~YYqpw=_WyW=Y&=&59H`mny4Uoe;sMX^1@=yF
z<?QYpUAFh3)9n2|yWZ`-V8?B{;a<$XcM{L-)<=pTh|FWz_mJIo|CXbXwmWBR-nU*q
zWAEf8ZF|Kml=sO!^xa(}8opOdX3uU;)dp*G&ux1}Q=aa9y?(Oo%H&o1(x05!yZemR
zUKhSuyHxko@7c~-wpWw)!`|0ln{6B#ob64-#r8IJEwIVc*kK*BL3eM~Ch@&h+j#Be
zX6&(9knLk{A0}oiq%nEVj3r|GnofV+8~*q5{_x<_d-p4z+T*-y?tX{u?e<rNBM;nb
zRo%~MsAv=5^J3r9?{D^+x1QN^VPU<^=^$U5)@2QQuNOVH;XfO_x4ocs&tA<N`;`yB
z+WSbO_JFUW{(hO0yKM^dzgT~~@X~(4fhwyswHJHWXgTjIYL?%>k)g`&qkra}M0;m@
zb+t+R&oS)Yw^BlD&y=g@_wHK}vR~xx@4c<ud+dB21omw&uiaa4P<r3asuO!(wH@AP
zux{BNhYQX7PH(%jm)Y2HuhWa=)=Py=_ZFts?fvkEYv1gxi|x|R@$XxaJ!7w!cgVgC
zLW}l3kZRbg8Tx+z8GWgp>U|aaI=0N)d;Zqty=zxB?ztt%x#vG$^*-a{;d`e)-Mg3R
zPVqj@cS`$jc^<P}&9TWYTz>PO@As7K;-1ald+C<2wO`7dy?kQ+`>)1qwk~Q}vbXc9
z=>9!=2lga$Fxs6JS!lh(R?u$F<@a{EmdkA%Dq{C~d{)?h<JCGFy`t8=duopCV_j0R
z-{oYCU8K&{-9-!E?x|+3+#hpk{oXf=;`Z9i7u{zkV|w6rf#^QZy8`<;PT$%)`RmTT
z&tI<Fm+$>*?}alW2aa&b>@j<swvYGt)II&NOZEypIk#u4E7LyH=~wp}thusR@5;CR
z(b-S;R@PYW6%%~EmupkA%_W_{J(Gmh?9Ugc@8-XrYG3r{n2mSl)7>()R(tz@sqFo@
zhjIUnEvNQu?R~blvwOz=jLj|fMajAcW_4Y*byv;b`{cdmK09W=y~h(YZKWLC_pWNR
z*?-cFb?=j9Gxq*he`gmMxogkjn!tV8d%E`;v#B1a5SKV`V7BvtquGi!`_`<m7ipTm
zcWU4jo7`<1_a$!LZ}%|6z`n8N-ComOkL}oA``9a*_3v|$<k`3V@{?W3?CbYW)VsU)
zkayg^igpLv9Z^*_QQuzfd6<81Z%61kyUi?>d)7Qs-M1i{Yu~S@BKxif#qB-AyKwJ{
zEE{W%F4x_BRY3>xLpJPTI+VNj%;KzlvwzIr>-pl?o--Y@_SvP+-WxA#XMex#>t6F-
z-MufoKkuFJ;p*OPbA;_)O-bGx-8<KM+5S8GqW;g{pI^P$ex2O813n!}dy@+D4(!^z
zcJKDP68oflPFUY!oMK=9@b2DIQ#b9Mon?PuN&@S?<ntHzuMrm5x3MJtK;~wf15>LH
z+poNSd9PtL%YkhQg8NpUX4|X0`M|!es%`dXTvqL`m@#p0l3v}O%jeqm|2#5bpXXi0
zeQ7p-_p<HZWq)Iu(4JeK>+RL9i|p%X?bti#Z~xwmF9Lf7*wpqZPJLsW*#3KO)aI7G
zh5}6c+E29XQC;_G_vtRieT=t+_nyCX!~X4BmVLh$Y~4F+PSjqu@O~TneK+?mVvx6A
zV)9_``LKQa8{bIm+Ys-(=h}ozd#=p>v?qxz&|d4*gT3;Wp?jwNJ8bXzug$(`amInx
zzqWRr-}-Hz&fwo)z^!egyo+VuEv6fL?amnOpZMHr?{a0^z2ZDs_P-t*?KxJs^uUb>
zi9JU*v>f=`=wTl`bJ_lLRm=A-x+7v=q_xqe#@Ki7;r3(uE*yEXk5@L+o=t~gpGoZ6
zeJPA9?SuX&?Gs-5VQ-yL$DX8xANSspytOxLx7YrDY1VynclYf5X285};?C;58@Fw@
zw>kc1uVBwA+k5ZV?Ol91cW>b5$h~b#+-zUw<nDEN%C+zIJH5TTCjQ?Wx<q>Kx<<==
zg4&z*-bgsL=fw<;{j*rr_8#o<*<WnTu<!QzM4RW^_wU_n`DX8qE1UQ1%IDn|pSf{w
z_DbJ<9(UdMyOxCPU%U6}UY`Q{y}K8x+cUpCyr=8T!reQ5|J+-DFnxcH&!oL+OP}q1
zw35Ri`kunx{zcXM+xxxt981gCZ}M!4{U2TaeZ1v*dmd~Ev-c98zPI09bg#3|u6?s?
z+4i5BWw!tA+_t?ZjfD>UO3<?x)pxe3Drwt0_r`f^3yBkZ->Mt$<^Ozk-@PYmcej;A
z@Ag*Rw=br#ea9>=#(n$`rrI#c-?ojDU9|Vgr4xIm9-OrIpk|d#=c{vjw>k3d%d_p@
zGnaX(O+%;lzPol?_KCVZ-+SrLkG*P+zxJ!EKHZ!4YR>)z3^{w}I7RI7;s3bT`ONXX
zNo(%f-ByvYIrgY)Z}`j|_OiU@_L<J}_Djukvj4N=+1_J+<@YeIW8C+2<@3Etl?;1>
zzWdn+rLWujRak2;`|@Q6*7>vV^N@JIcj7FWeG!7K_O0(P*uQEwIB>)8ueDI<pZ%U5
z-h1WGt=b!!b7h~z-DmdeCcoJ?w|~yw6K4+Xvrar~uX-VUpYZjzy%ih0_Q-JVw43W5
zuy@afQ~PhS$?TiE=f>VdCbzxOA)R}@`1adR3~SvZC?dDVq4)J()lFOW?0Rjww{A1b
zJ~NM=z3!K1+XU~lwNI3qvG?-#w!KZu-t8^(+_KknVej6(7A^<QGUnL$7CY^8QtsYc
zf4O|m<d4Gp4tjmvd;fXUzQU$Qd+)q`y7zCv65GHP9riooO!r?A65D@a$DO^qW<S~g
zaOJ1HM{g<HewpvPcbin~evu74`&NHBw%aj2&pzQb`#zClclUqcR@vwIWB>jKS_|!q
zw|ufb{Xcf^@p&xvPS*Q&?>e=7udG(jzFEK3_dC5$+rK>N;9e#p*#lqaD%f{fF5NeE
zjrZQukACl!f7QIt{G{$)wZHoNPb;eLd%GcHuj0|Pedp)C*jshlV&74vdwY-7ZnaIm
z!?5q*L-Bo#kCUuxp0n)p5V&joKjr-1mbqefuhy-$?(^Qi=e5J+ecUVb_N<!UyhqJp
zy?t!W^1az+NB2h-AK8=Z&$##D<p!I;+)sPAv@fySap2Y7pzF)69oHS&&%M)oKjVWR
z`vv~0*ne5FaIcAFxBZrXPi$36e0E<InzZ-3=92w&n|JKJ<h^#!>Q7Vb-(*Pa-6z_8
zp!m_6y_fZM4z$Hs*!xY`w(n!r^1TKfy7m*2rtjISF=Ov-qZ#`SznHy0(vsKynR5GH
z?)0bos<r;?3(H@z&nEcn-axtadsy0D?6v=?XcZG~Z@=oR@ZKY(3-&Cq_-+-aw_{Jd
zM3w!I%PIR(w!PkaO<ZQ5af-xV&*I6}>38Gy_D}q<_q6Aqy&KY%5Ads4@8x)Oaj(Yp
z<$F(w=I_;8p0n5NqLahXi2gmQ#bNueT@>1v;`_u#IZ$Sw^q=c{#T%>am#ZDy`{DT>
zoBa00`=d?I*nQu*YyUyR&I4VO7Te1IUAIpvfMcJ!>9@VhxDM~0)8c7gT->zh#*|E(
zdk-ocqWkXeea=7m!2KCh_Gqp=dVuRgw!O;quQnetH1?_#2G}ie?cAfAl(E;fpmpE-
z#C>*4KU}eYqpiMI^~tZj=T025XFR-cU!2pWJs!Nqd)Yh_?Ry^V+?&BQ&;H+u;=SiA
zC+ykFzjg1$2`_AB?)0;tS{ksAOC^5q*2}xC^qkh&B%ICNtMugE-lzAO_AL#*yl2L%
z&GyF_R_xW>)VJ6DS;M}nm8<u*9G|f#^URX{Z3eUV#`3P-^W=)(-pO1RdmF#6+I{|%
ztMw+uSo@zdzU=J^HQvi|c9ZR+^>_BaTldypFTdT+DpYpgw=^Yt?VDwLmO9_x>++*(
zkD!Ly-pi9#+XSud-aFyYce^#R0ybTHPuoA4AGUY)>*@Bp_VXU-dn;yt;hxGKLycGa
zS5DevlOkNTH|Jl<KKDgS?BfKd*}v{$*|%wS=l<ETruHZ9?%elKY5v|@sxEuHT*Vyf
zH_zBB$Fah0uASOG`@2i`zA5k8J89aDy??^}>;-hr@0njaV_$edz}~|dhpcC`EVfa|
zI<i-+qhasq^9%QuA3tVy^EI1|&{h6@=T=ACuGJRYJKJ{S-fOeX_G>ZCv2pn8eIQH7
za_>iW=e@pZ6*lSG0{c3qZL+_+{_CC(Zm0GxzH`TJ+5WVB*)=l`+`Z?q|Cxxm?XxES
zgW?qmHsZo__V)K|-Ya}%p51qa+TA|`PVW6#@n%1Z9{av`8%`haZ<w?vzq|c_$fd{j
z`{p0re}?tN-UWHP?e6bcV#9Oc<ert6r|vuLdBEmbO6!3t{a_ozPwV!-e5+~iF?;zw
zxzcZY6qM3;@8GVn-@UnW?}GB$eV);|d-!%Q-#htU*1qb*4>p&5wCoQ&-LkjaOV#fG
z5#D{qTO0Q}9i6;4hvVbk;IPKMM)uG4URt+r?|H@gy)GMHS+5ZD+jmP>aG&P(&o;9&
zZtT~-`PKT0NX|Z&ji+rot5f!#b>!W5wXS9F#HoMwD~Vg~ec#Kl?*VJ0U57--egn<d
zcBTf4_E|`s+8d*_aj%@R-oCAANB10GQNH)Nh5r83HS_nF-EXwXRhHiGfB)%TCB-9l
zr{2%r8@O!WUcYy@_O<6`+g!a=Y;)?}qJ5D{YxjP!+O;=aaK*mw`*zxuYc1Y4^``w^
zS+$7$Us(6<`@g?rA1~+9y~$;Q`<RZ_@3*nn-~CAB?tZsxEc@J;FYo>IYtuf~IrI0*
zE}FZa|D*7}b&-1e+<gx0Vq$u1{iyfeZXVI}y>GU%>}xyZX|wsTubs;KyL%L$Y}y<A
zh;`qi&NX{ywjSTx7SX)Vbsp>9(q!Yk$2RruO^VsEH)(3?-fuOp_MRy!+NU{f)86m9
zO!uDG->~OZ*=xJG1#J71w;#9jo#167*tpd;Rn5;f_Uoj*rRv}J&VCTS|KY-)du8fF
z_MSJLZ>L_(yzl9+KD+mf4SQ=#G;LWnC+?G*%X}bbf$MGw+X#C;J)V6_8Bf_XKJMRF
zF>T&{g^v^LHJt1AI<Q}~`*Oc?-}cML_GuYU*>myzs=a%5CmfjXv~=&&$hWqS)ZO;F
zoR`>JdcJR;9M9Ult9CH&-(qrNZ)EV~eeR3QY$jfMVRLVJ?A{%G4Exr!9^3m<mT}(+
z@3;f+`7`!zFM7LIA=7D}$fr$vkJKbv={FwS-@U(oZ|;ij{nJ;Z?tLniZBu9YVz0uM
zoZa%gh4xxOyZ7EySi4u3fq(y;f`#_QduHrwb-HfH%eQrRfRN7q_rE6C&Yu2jumAiT
zdrlTA*mW#Bx_4>ij=dK?a2@#6%e=2~v(SDnu1$MxeYUk%Wc{(PVqMVw^yoEvBczq~
z^98u<6;8L=d*y4`J`R>=cAXzn><g_v?)`ePaqpkHzWvYk`|R7e=-{54i3|6ZOI+BS
zn|XilZQtDe-*`p$EtM^?-QH1aGjoFH-g=%#cGu4r*ecjQ*_*QT&)$b|U3)d<zwK2!
z!fq3tw0^H_pXT0QlC^fLf)DJSlks!!jm47tHhcHjcpRU*_v}28{ok3HY_}b=uwQ7|
zwr9)2BQ`6Z&aydSvtaLOD;9eZ6ZU-{gwNSXtogX_)7>rm-Z|*lXMCHncfriFdsB@J
z?f0+m*=sM?zt`^GqrGMuC)-E7ys}r&?9U!eeTV(4eq7qiJ>{|e&2y%E%dM5|H-EG|
zz{d1!-~Fju_J%%i-7jdhaZkDbd7Jed@Ale6SlMg7d}MzseEZ%g0TH_-CT;uW{m1vk
zSh(%J@aM|jxyNVQmnMkq$^6l|?`m_(-h}}(_DVUJ?oCa0-P5*fuif<Ph5IHPDcbky
zr@+3&R-C)}N_XwOsLryF-E+%c)7ynMjNcyH);;gvE4y~P&5n{u`?}J3_Eybr-TSCY
z&LQe*$KE&lGWN}4V6{<Uy|H(Ptdh<DZ<Tursy5htddt6WiCy;IPlkK<=EU!}i+Nza
zr|5&^fqO2jd!4v;+ALS%+jHH9VV@7<uHEq}UHd+*yuP;}?S@UzoP+jzxZ7?1%Np+Q
z6}Yfx!?8pAHYx45OBL8?<373AditLkc1w32*n9uR={<ttjy8<vzu4I{EZ@&3T(|eq
zL*YFV{|)Rf%-Ol`k0|@Tf?Fr7zXs;oUuC+vH#YE;-H{!q_Jnvg?Wvn)xNm)6`rdUj
zx9oSXx7#aG^?&dBBSQP`d@A4D|Mu`+iOUgtS>8<A%R2SpUhbc)`@d~s+jqC=%HEA9
z-S(-dz1(ZIglF$7W`+YsE4S=re&uZ6Yf-<qJ%7EGh3xFTuh#tC>w7NF?n<lPZkC+>
zy)S3H*}IFCW&f(?J9d&LPxqNQN$(Yn(b%7M&17E~ul-(@b7?lb9$)vJQ()dV|D>oj
z>+T-A-vwOzm^7RA=3kq*_vu!aef6$B``20V+4<BR*xQ!kZI@dlyzlq=8k;Q-+V=f%
z6}CIlAhFNsci*1Eyjiw}yQc4F?$z4orqr-EcE^sr6QcX~8!eo<H}bj4{wmw3z3OiF
z_M|eU?K4}MzRRZ3ao^G%T>EA&PS|I_p|mG{=A^xgCa~_?G<E%+2b^E`Y+w=G=XsF9
zK4O!^zSLXq_v)-u+UFO!aPLH^`n^%zXZCA6d2M}4uxp=G{?5Hq4nEpj!Nj`n#TlEu
zn;w|j@jOf3YtZ;~Z@2CxJH6?S`%<g^?(N$?cmJLb%=_dgx$oyW*S>cjOV8eZw~u?S
zcMI%yXk_0Pyp?5NgoyusnXB!4o;_M>KUFVd?~$jjc42mx_uu;UZC`n4((X4}!uw~+
z7VHuFy=|{yc#o~=X+!(6mc90$tNz(6t9@=)xLoBxQ^@RnSs$bKs@5~@YwtK}A6fWm
z?~{iN2flw<x%Z`3-QLgEa{FYBuI!a^Rk2rpb9L|Eu8@6^W=#7;vzqs=K7M|$N76o<
zx{{N7Hv}x-yC|w*zm~$Ky+ucM?480qao;LFu6-tj-nN-Wk_UeJZrdw=>#lv##b<k+
zTK?`n;(TTAwVGLbSxevT+o%6>PnGzIy{wrB?5>@jW!GKMx^F*o!U3MM5_>N%Uv=P8
z0sFr0+Shxh9ay{fU&-nHO5b_*ZJE`umxEKo&L&{~-bv4l_6z*u*f&e0djEn~V*9<7
zg!WIqHh*{c5l*}B{rmUc?zwC;sdvS`xoN5URWl~-7il`Vcc0*#eY+IP_KB1w@B7E6
zyie<+l}*+esr{)Ny{y~&tPd<;Qr~-AjLUYZ=7fD0H>&Nu@<)8Xar?KucF%eD?s|D-
zPnDw4UYiqV_I~IN-n(z^$Gt|idu(DJ?zguQow1iOZ2R85ea-v)&WY_dGC#97%BpVP
zWAlZ3lYNu+-N{SXYjpkJ-oF+i`_>$=*(=#Sf4}1ou6-}GWA;9OB)GraFmIo5=+Zs5
zb9nX%vQFKrWOu`k{|Cq3oHGyi9+<b=+Nzjsf555!y@m3J_iW#0Xy<ew$NFVexZOmi
z6MHtKOta&?yl(&38lnBZo?g3qz9`wU>pa=JPWH~;b4R52O_MsaUr9f1-xt37dmWEI
z-e>;B<-i(Am3?+40{e2d8t>lV6u)04X7Ap6^=<nazFX`)P#d~;V)xm7rS1FoZk>H+
z|IU!keQ~_4`{o|;+2fw}bnlD<X?wK~YV8#~du6Y&=E}WAP1pAY1&8m=%6zlec&)*{
zwV!A0Ee|i)bGS#&PPg;o-um0S_D$khve&1Y!`5hW-`=<op?#~(+xOk^ox8U)>%(5J
z*)R8YHQ(8HdqRW#)fM~fzC@<)oyR1!&pR`3@7Zm9dw-lbv}bC|uf1X1e0v(?YW7-c
zG8}krc5+Yr-Z1+oXFGOt6e-v{|MA<~_-Ea|MNfP8{yn*6zi-Gx8`-<Md)Mzcxhwg1
z^!{*DkA2(L$nLpxqRH;urYrle<W1jqocY||^k4h;1}oj!-&g!@udt5Wo))t!dvmKl
z?77!xwJ%ry;9k*+zWpV<8}~k$R<zHkdFtNc86|rc-CemiMXhot?>_#$KV~i5J6Bl2
zPUp{xz4z{Xw-#5C-giHWXYa2)Df@WiUfA!96y2L8Yrens(=QwS`3(EG{1@A7I>oT>
z^~|<?eK+3j-NC+kPY?sc{x9=??6WBQV7EGge{WsShrJf=zxN5(a_{pi-fnaC!e*PZ
zzn$&gyKdYSa_`{Y+X)l)m*wu+J5l$h9phe}ecv@W_wm|&v0E*-%&to<Zm&c8`Tb#T
zs(ZB)boSP%<?L<QUSPNDii920Cw}X~n(4NyW-06!-gjzW$JyO`rrgWhYvnj$|DTDz
zdy~=^+f91awf98(jJ<c`i}n@%TW@py=Y;*WwM~1^>-O%uYRbEhFLu@5zRGWVGYyXJ
zy}ij|?~*GI_c*(<+i!o!w@>)@t-bo$i}!i_FSXt?W$K>r{#kYsMrUlU@k!V#{NJ|c
zNzl~20m|3+Zs@zRcgs?neY<_!Y(gdW?44~TZ?|>%x_xsN#M(I+7VdMI^?#28uk=2r
zxvhIQGVj?tLyvhMQ^h+wzVgS`huUNJ%KW@&xB2ATz4QO|?{AUcvgd%+ll_b1&FmlL
z<=A!0nC_LD(|Dl!lakFp=bw88Sg-BlH%hQG)PKHDajme8iQ-w?O?FoMxl*U@^JIOz
z*LOk6o~7S;_Ekrh*eK7*+57y>)V-Syuibmz`sSXVBk_A1w<qs^xvpVfKX1xD7Tptj
z`d4)AT|004-V=E<_wrQw?^T?tw0B!$tKItKtGid1H}5T*y<(s0&KWkh@6X+9aGdWz
zdwj*-O@A-ii3HELHrO=7rtwbw-nH!sdpf0;*kybD-aB79cJHFjWcz=`68kUwm$X;B
z;JH8E#@e<eLt;Por_Q~vbmr|n`u)t_b5f@Jf1ZB0mod6{kJfqD{afn~@0q&(x!ts_
z>-R3Xy}&MQw}{<|!`JP8{<PkE{?z4tTl32Hx_Nl*tvP;dU%Z>PeOP6go$(9nJr{pW
zw4LhXy<b(Pd!P0D=Dn9SrtMj9DQEvaqZxZkyI=1OW}dg_)9!-370TlKWTxNR%WO4g
zUz^0~y)}{(_no)t*~4xbU|aPsZf~Yc``*i`a<=DUb9ZMk1n%c5y0Ry?gJo|{rrkc%
zoauYiroGtfxzBe0YMCW_cO_rj|H7-wMnFAzuk?mTdyPD&+wk$9w=?S&+`Gi)>)zJ>
z7Q0i*m-a;$e%>Eax8C;aj|VoVPh8u-d}sUaP}_TZPkf%SSLWhYTS1+VdkzWv*v#Df
zdw-bulD&J=3=b^NIK8(Z`P}~6G7G!>-|uXhD%<w15PQ3?@JQ{RhybHKIaB8D<DMh5
zzv^Do{*;e9_HK)JJ#b)J@4jTe%zfJ1I`&GpJ=~i-|KL6oslR&?O&Jdu_x#;+pX=G)
zxTT5v?6a=yo!UIbrthNr-cQY1wh_8Z_Ud)b-aG%%hdtjTj_$d=zTbw|;o@GFJInWl
zUHY)+vVh0l1#>I*1*v53<=@Y5%Xo9ozCA4+ySdjc+c)oV)}BB6oA(?vT(Xx{B+Ta2
z+<<-We?QoJMRD8SqG(I|RC{Cl(4VRMmq#7izf7pzR^qPezUoO_`?e%+*n8o@^S#ec
z743W1)VcS&LBL)^Zhrfj&C9KSuyXG=44c09fOyFM$yr?XaVH<yJ%8J4W3cwbK9%DI
zyS>yF+ISc*+*h$Sald-3tNr|)bNBwyPqP1RT4(Rewsv2LIrqN0xVXI>Q%~9DyB^z{
zdHD2x@z(0S!s>pu^S>zAZc0tDat@ufU$1YuP3a-mee3hO_D(Ug-TUMIUK`u%`PSF1
zpYAPYNZdOwcHRC12RQa!3V*Wav3}dWhxaz_-Lhus-cpMt`(E|OTUBXa-@hn#{a!bf
zlY9PU-P)^suy=3Er%JmQX)=3fZ@jSg;k;B^ee(mhX<iBY*0#R1v+&<%bGrG%K9!g2
z_H3``*r#*m=3ceP8r#{HdV7kZefNf*>)F3iwAGfMWA(n}5AW`|X=P*|v39*(^HHJw
zeCers=AOE>@55*3J)RX!d$jWT?Q*B>-_M<Gx&LwN9jiyn>+OAuqz}kWx7zz7@WtLA
zB{%m@Ulepe@$vaR9g_?8wO#wWcl{ogz3o{ld!HT^+}qyUw~sqtbD#Wl|9xEB8TK7b
zl-kR3_3Yk9(Hr;jFthEgS83kUAf#kp`=4dsZ=wBr0xp{EOAold_g}@Q-FG77?SkXH
z_ReP8W4GAn>s~E~J$uUI|L(p0{O}&Ty|4C7bqw9hA1S!6J8sc_pQVfKoX=I*@39ow
zUsv*DulPID{Wo%w_I_K$wQqsmVjHKQdbTG!S@s%#>)(4$^3J{!i<|e#9e;24ll9PE
zy=;N~el?5iAI-kLmuc~$y*_Wh?f32G*vBHdVQ*~NvV9d_%IqU%+S(mFrn+~-{l@(|
zIoItw`x5tQ2$}Bv{6%EnvxkxPidU@nOm@7#zdPXm-jXVYeUqoC?0p=gW;>Phw%yc|
zulDYE&bqHr`sCh;&tmr;ea^P;cle9F%y*9NH7)1eC#h#_7bMBH@9xPO8)*TZJ@u_0
z_i|Qz*?XY-;r130yS;&i?fW0T=G<5ILuN0}grj?H1Saq8H<PpX+_!dbPW)1v2Q%yK
z_Z2?aTUpC_;K^Qt17DqM_O>1Td7#3tVsGjio_+4lJ$v^}tFo{Crnz^%|4EykV_)|x
zod3U<`QhsQ1(QVfMQz-0;Pxqh`zxQz_pLfFyLYob`~IqKMjL|z+iZMyU)i@`Q)!>`
zoPq<sQtEs6EQ>mjBVV&mXp`r@o9exLw<)pjWfppD$2L1@@BBAa`%*L;_Sm1>ytmH4
za^J+0p?h^(xa@Uq=I(PyD6$obW8BxAZ@5=}*V4UbwzOC;$z!mwnwzpWaL&(t6$;n(
z#If`2o8B0*=fT>jy-qug_hc{qw?FN~w7uqbFZMr;t=X%g61vwkf3J<<(wVz>*16j=
zX-(Qw)mLS6V6&8cp>xB&a$V;Ak^a4Qh10L@5jxMdZ;^c2-V6;M+fDKh_I^xyWSg5l
zWpDKB_C1dx6%R~2qrSKG%>MmTr6l)WTHR~!`Ok7+dqeL&nd?*c>Y1?b+joE0o^0n2
zd$r^b?49G`Yrl7)&i-$n`}fLik+P4tnY#aa{Gxr0jW_l_IKFj{$Yq8DlAgkQTScAs
z&F|9L^J~W4oqy-q@0IUAZeu)&W#8+QE&HNw`0eYeH`=>WPHEq#?3}$#6A$k7dh>4Y
z-Hz>hqGcQHmnwbPoAcRfZ(NnazVf7<d%r1i?oI7Ww_B!>wYOo0k$vO&n|q6O7TL7@
zUb9#6n8n`t9}n5NcJx~RXi3`pSYfH%QMtDL{aXv{vhz3Wuj$X*%jIaY|BPSQo`oEY
zdzi0k?Y*Zw%f@y!^S-(4t$VlLeqeWFUF+U6J_`E;JP+6?^Do~&)s5M1_IAmA>N)54
zYQI}-_kT)>&90((d+PKy?G1S$WG{OB)_yVWn!T|;+50ofnd}%&?%v16xOQ(r+os)-
zD{bxW-jdko`Mt(Y#9+&w=sI^B&rNst8ps*%Wt{H0@AH8q3;rp-d%M?X?Uk9TxA*=O
zrM+_7*Y0gUDYy48$Ctf@250RQ?i}26g-K}NLhr|WXRTXglN9)R&j&ddTLJq&d+ol8
z*+*JF*_->W(WWWy>E8Lre(zl}|INO8F`xD(x5(O@_T0X|X6fy{&(7}Kx9YQs{a&-~
zy%XNp*>-K2yVrY3&0d?0m-iYd2-))qOy5()G|Q%J*^hm_+=BZAo8Roqo_=XhW!KYv
zABETLQ@WdLCt$(7PhR!q{#gy-d(3y<u*ng)y3cl*-2S7rT>IB<oxR8Ub)VfNb*=s8
zPP6xEyMEYvM{DNZN82m+FMBd??=@C-dy^^5`;LXF?{(3CzW2}Vr+Yg(`u0zEirDA=
zXZAj=?~?oee-^YY<ea<b^#9j;f956bJ+NWj-keHdyEy-sdu#3-+3WR>W1o>|@!q_$
z(|g05V)rfye!jb=V$!}_+ve<*{XK2(y|nFn<$o*h{o)X@|D*ZJy#a>Y`*wD|+jqj-
za$lEPq^)+nmfgja)7G7{ByBg;SnTsX?X~Be^R2z{H--1lnDlb*L($!PCtO^;f7ug(
zeOFgT*lFq4?%h*pvG?6e5&KR5GxmP1?BB~&6SjY*+@-y%t557rXDQz&=aaUdC){@L
zum4Z>PE=#qw<Z7geq$l$y|aIw+?&wNz0Zte@%}F<$M#yLJhN4|+O&7oD&alfJ-6)(
z|9xby&?fE!%qNTY-rXU;Z%^F3z1k7mY)k`|?A5p+xM$)9DVy_!7p*qdui5u-tMcAG
zHo1FkGvxMVWZu}V!?j?qoWL@>N#%)qAJ3B7zjY?lzAO4O_bT_a?=4j2+Q+)Ob016N
z?LAw6JMX<GKhs7~U(-(8d#>GsXIl2Uck1`Ze@w7@b7I2Yr@J(5=ei%=yDx%upR7CU
zzTT-Vdmeb}?=O%t-m~)>$9{Fu%X=hL)$J6jHt*Y{o@V#qzsTO;W_|l@{U7(v$~d#P
ztnT<;Ay<ukXS*!+O5Odr*W^Htou$0OJ{#60``lt??-e`jvFE$fgZ($RFWG%edWzlI
zqLq8Uuexu|B-y*SRn&ej_iDrahipFVJ#sy5U*e<7dzs`T_ZoOD+`FtlXz$I!OEz5_
z{r3uNXxzsx-LRA4&Q@Eo?>YO-YWVi8nw!6OkLp|N4eCqxvV3K<fAVeXUdu}dt><hG
z*;Ar=ac`JT;{JE%ChXmGuYYfE-<5q^E*!JH7-VIC-!tvN`K9)|OC6`|-?e)8UX4#|
z`{X`U?EUbZ&Hh?^{hm#aqW7*2ZQfn?jd$M{i{_m|>t5{Dn6qWSv6%e<=dZl`jk%qz
zQYOzi;Ly2euZPFOz0!QW`#$~kw7YiWq^+0slfAFK?e-geT(f_^R^vXG$@lh7u}$2&
z+w}Tg7Sk_#c|ES~lWX6yN9_ii?Sxwr`}W_yyEjwFV85lz{(UT4;`V))?%un)Xy4wc
z^3(V7@pITd6Ybnv`0VT+8v)&YRz}&oKj|~=D?4tnXRpY=y_}6!dv7qY?0fO%#a@05
z#RI7UH>@tXJ-2DknQL=>!>qmU)->B{nOpA_J2ZLk7JX*>cZ*Ez4t^0j;2mnUuU{v9
zPxvP`JJ+*8dqdQx?VT#eV=XlC@IHkq?Y(!z7w@&vKW!(%yJc_2BH06x#)tM=*);9H
zBjK~}miuhGE%6O|xwgpJhnzIr%i}uT=GX2W`#K!Y+eu&Kv(5J1zlT95&5reRpMB9f
z-hF*{%=h)}f4(RB;9t8I0l&TS24ed!+|b>(x8~0ty>C7HxTYEGt*LuqtJc}MH#K|x
zULnpod#?V-+q;x;^4=Q^OZL{dbz5r{|K0oUU&6k>7UsK?ey-osvNLqwtg{>THd(Xp
zm6mSY&*a;;x2K$If7FX)o4~cl_lp1ivG)w?tv$t)ylnHbdiPqr?%5k<@Yn9RpX>hS
z<5Ty$?y$DmrXIZKiJ$ep7k{SPsGfVecge)tdzu6@_p9-$?AiIqX>Zx|d^_Qrt9QrE
zY1!Y?#<s8Kg694YB|GhQ%N^J^lV8GSPZXow`9%x%rmT^)E&5%z@Ae|D{mmOI_8(?Q
z*xSS}zQ6qC!hI=>J=WJuI`_^yBD(LL=9_&R&+gspW5ajg@?W-na?9TCy?bNDJ}I`v
zdz$On_B~s$aQB7g!?uw-MEB|`a_*bzU%EHzxznEW#|`#KC-U#RGw0!+-FBsW@0MrS
zTH1x~d%1ts-qWESdyhFE+oQYx%3fKq#QlE`x9zoTZrY<fFMe<Q|JQr-O>WxpSuyW>
zviQ)R#H*j}PUITxFWa4Bf8p!RePLdSdlx?N-mi8vY;Rk@{k^((KknV{-D;;`*JY)!
z=KJ2)??d+aHp%TR-n!CWEs$~V+ym41)gS%6Pkf8$zLY85d-I-K@9(hKVxzOOV9&%Y
z=k^vHV{$N(>D%9W@U``+gR=Ymw8HH2pEm6?jE>uT!cKT!ukv1d{k$`KxNl9{A9qJ%
z-{~i;dv7-~+Me^-vsbtA<Gu??-}YwBd$l*2Eoo1lbL<`wh1NY@(&zV_&bzwzZP~WH
z3eQdUrCnXUhquepHZOMZ-U%A#_HgZ5zV|6(&ViThH}+OMEwhR55#4)!&fdM9vt@0}
zm;c@CHb2O^BZ+_Cx7pM8?D^Yfr|R>@&b@l!e%`Zg`xhrN?G0JPd!V#!j^!k_i#Cp*
zp6@+&?yXJdq0M_|^|M%4OgU=r@vX!rrTEu=AvMN**NV67-}-ukolR`S{zm0~n|D*6
z>|-`6*_-;kXYY?WtM@%F-L=mp{NVo2JO1pwd#1)deceR+kUvfP(jLg|-Q4qPui}+k
zb_sgN_I@=A-tRD3YoBi0`aQfCcki22X}tIUMDu+)^L2JBRG+oZa$?vW_v_8xwKtOX
zFt@+o^Xq@bp5OCc?v9S+bNHlgxwp*QVeipb^Y%S@y<qPqgZ<VH&${<tQVZM5`Xg|^
z-Y&Pj6|%y6!~VMN6}bFtPn6(J+t1J6?_GC8a_`Tt(R&L#zVFk1HGRMH5;yzx?>FvA
z+QYj4|Gp)AxjJ6%)p=67cT?|5yE9oE_bM$|yw|hE>%h$uv3oNjXYYTerf0jkw|9SI
zQ<t6H4deYP$&2=W6`ExKA>D6pXS@EM3HN#TrQDde|I`a<+c1+4yTwy#_ZOQ@vO9k>
zYajDT=6%kQ;`=sKsqSaL%CV2BZN<Lg8J2tH(<kj|oxaG%l<Uu4rz`*Ms=e>-ow()r
z-V;%Z`?mkBut~}Jxp!{m<Gl*2PVX)CTE4eL%6xy%OSXO2t^M|FPPwvItgm^mSt9=)
z$*;fc8yQ;mMlapEztKc;Ur-R!o(*5M_ohTn+ndI_!|s&m`n{hPz1@2v)5Si>mDwRW
zWY+$>a%=Z3EvvV^CDLb?t#xyE!N#n;Z}%4LKDF_i?cRlbHZvXH?d6-R=CDSle9t{O
zkpunz^!8>&O*nA#WR?9QJ!X5(|I_#CnKj#gvf8mXrev1QcY&&XQi;a<6>lpZ5Zf-Z
zH#*h)K(C~o{Yu}meXWO%@7=JDVc(X|f&11!@7inrTE~8kq2XRunZmtqe=gqJ`8aZ~
zo=VpKB*Phd#h<L+l_-C3?`)k7dk<a--WwBhXwSp16ZX!Fe6rVcmdO6S2l@6%{eQRj
zL{HnkDY*>$zON74lNj5vcV@zM8)fb}_GgNB@12_cYwz1Nd3)pbueMpo#IP^uyROZQ
z^<jIC-8Qm)zG=0+!(1WzziSQl3)R2c%eMBZ?UDW$dsa0q+asItVDGYDefG*bO84Hl
zw0y7khSmG#az5U>{LVgmU#G^s;<pm*?Z0szu<enyJ9koI_a(Vq`_f-O*z0W2xi>Ja
z%C6Sl*xpvU*?z|YzJ2jmr|!><__VKO?fQM`AFFJ8!>8<3lje7Ll=yqkgiX`-shFMF
z+in_c&9rONZq8DUeFhT*?3higcS{tz?mKijeeag`+`V5Ej_kEaUcGmD%)-5_JD%?S
zv&ngn)(+Oarxf1oEzxh>*Y>P&ue?~7?VS}5_Jx?L>{WXuctE-G*)Ge)je8e&KG}Qw
z`Lo?q-u$)W56Im6<RQmipS#NTsU_X}&)#G_u({{z{u<4Hd*&@Ua$q_0v%R<b&+NT2
z&ByvzsFD4*nHTr==%3m9;-~-q#civ1OS!rnc)2BEZ}*M^2aM)!xAW`WwST`;fz4vx
z{rePb?DjhD`?~k~&b|9G#aZ`Go3rS^w#F5Eo?enX(Eq2xZtAbfeNnA)`y@9t?A>wm
zv|WGqf!&FI_x7vWYwb-f$lrUsZq;6XH}-ws{+zShyySq5rSd<U?SI$rDbN4Bx5<=y
zUv2!$y@t|)`=qu`-fMNWeSeL?>AjcvCR&Ra80<SegL8M(ragO<E068#Kl8-eD*O5F
zj|W@#F7s{MdtEJM&zD=9_v%WQ?B8H0u&?iv=$>O1t$X+VD&Mo=(0w~bnZx_MKjrQ{
z_r-PJ(OJv)D!*abXL0D#-YwcX`xO;t@9j8oYwz^kcKaq;t=T)fwA0pai|C%mb=<aR
z#3tD7<GH_&tATf)c7owvi_)gOw=-;QKFn*~7ueZnYczkEP4IK&eRaww_w77=X5TLR
zDf^;B9_`&DymaqRvqk&YPq@El{`cSe)lWUyBR;)juYCCYecygw*xPh(-F_LV-FrUY
z4caF#Z|dGjA^-Q@jg_<E=jPnU_~WhBCXu>5`dJEls+lGC7$&msjk{;MuQQ2b@8r|B
z_vS<d*?qG$->vv|`u>33OZIX_p4_YQaPD50NtgC|J8s)&vzBw;l!hQ{g;`2=N?#W4
z5u6cc&vZ@HzP)_C?au9fc0u1a?w#|yd5^!8%$^4q{_Oefe{!$w)JuEY6@2#Jy1=q;
z&ueje2g&_5f-$q~8fLHBS0S)!w`k3$-L8V_`^}Fn*du-O?cS-5Li>2)kJ|-JF|w2V
zE4z2+GERH(3EFnga`x{#=v}gB>xQX&(qBH^ZymG7`gfp&y*2kKn@Mb1dl}7+?bEkr
z+I#1c?f$pPJNMKro4wE7@zdU>FX?;#*;wvPKU=!D*^g`Qt@wYt*ygkB&-s|N_tD8K
zd-=rK_Fb!0-&^$kicN>5z3sZf>w7hHd-n@%oV7R9yKS%Meq(DjA-_E}5<l%OeCFH7
z7HPLP=2YljonF^{FW;8iS?HXwD|_CwH~qoi{f7_jwNicf%bM{>&t4J6RQr0p-Fx@?
z-n5w_Zo0p3{)W9wSLE%K<EHQN<<UG)_y7I=jm-l4B^f{NnQPd$-^%{b-XiNCdqWx|
z_C0xW%_d-m%z*&wyL;CuM;_Syh;#p}z|?)MMsM~uEjqh*Pmz|L{I8n5F9k2}-?Z@Z
zo?Jt--RG(!_GC4D+B<7+n4QvHrhRWVEwV9jnYDNQ*+g3{Htju^Wj5?Fxc*~rW&2`l
zn~rDuBwoGU)9}G)?+)KNd)&N*_KAK=w$&?lwo5TNwU;|%t9_Nq^u3)o18n4PR9HV=
z@n!Ga?nV3m=1tryFKo2eYwiZSQ+p2Dt$EF2FKO7Yf7{c}y}?W7*{wLVc<(;bOMCyx
zyxQ~MyTk6xv?*2x`OfW~HUHrLql;eeReSGe-`!NRSMvHv+n@RJ4i!BH_G@~d?7lg(
zb>DXP?mY(_vv%*>61^|3xP9NEcc%LvuhQA4c=7DMFM@yU9u@A|cfN~ZpKwdq-fisB
z_BU3@?G@zG-v99Am%aOLF0fg&{M+8;&f<I9Tbk^R-)-6(vxj-#3}>c&XT7G{I!x`{
z`ypwvwaohkdtL4<vS|%+wU_NWw0Ewbug$IpYxe1^nXp$panWA8W~BqKmgm`={oS<R
z;o<AOk=G9H4ZQGWulP5fJvC>U>=u4C*gNOZj=d&zTKi0;&+OsgQn%k6qP^d1gVny&
ze?0pWl|SxX^M`HU{%QaB&iSUczc}7#uT1IYy^($o_J6rhwWmCPnXTG`V|(-h8}`lO
zdA0A~s-nHi-9PQMYL~OuoYuE@_8eoIFA0+S{-(t5|I@f~Uq#hP>*!bA_U%?8_TtqS
z_Dyfs+c)Lv+`W^F&e^BdcI|EY*KhwXM|tn7qKkVK^xXH|nf_q!26LkW-#k|AbzgOO
zulzEJede0K_U^bKxL3vP#9sA!y}e6<m+jS;G&s-_y>@TZ+$noDe%0Ccud-wB$%3bQ
z&04|^T$}!2?@#BJ1B%T~dlfYE_wwESw0H4a=Di|vZ|$xo@$5@EJb!QfjHLa+G2iTb
zI#=4UrkL;J5<k3m#%6*2r5$qnI*fAno|-dlZ^wGEea5%N_G<bY?OhSnXfKibXK$8l
z;-2|kEA~pbB<}ZM>Ds$$a_E5q?z7gvcR1PKGZEY8ap}=s;p4OSdBv~U@7yO~r@e3c
z-Y4>$`;YV|@9$>v+2?F+vQL+%b&s9prM*(z?EB_#JiNc~*rC1ZeZP0-wHNI(zPWF&
zQzP4fkXJ>!o%rJRai_oCJLBn#y~gG}d#8LV+2bfXdymDmU3=fCmmXO4;P~Df^)h=6
zt&Z*8)f>F$(EM$CP1i>6(~;i3mp69X{tYec`{Jg~*xTf%Z}aJXhqZ4I^S*<2U-zsq
zVcffbrN{1LbiaLEyYYVJOq~NWK0n<X(Dq`#;S&2jN7~Qt)zfC!S90l|?b1hE_ey8K
z-Mj4Di+v|w+3tQQQDYywY|~zbPWOG&MLX;>drs{42=>`?@8|3Nm)5V@)3f;1-dWl^
z_nt~+-Yt>GWbgP_eD5MAuKmZDEe~Y#oZc5SPifz3pSycqj2ZSvm*3o5RCsqkQ^%UU
zshgrLXI+Tex0mJq-sN9g>_5Nr-s{*Kv9Chn@814xHa1T~U+noWcXw~3|NFfw*KzHW
zWjnS1@7qb%Yv)bhd;M<HK2NLH)_Zn2?7dfddjDE!Z<|eX75DS3db+n~$ECgBRXuD>
z9!=XDbj)CX{Pmf8RKD%l6Zpo>o|loyUP`EK|HS)m52RN9+nY4$>w)XZx_b}m&E3lu
z&c3(&tM!4z^Zk48&obP5y;XT1Pi3Xer|T>B?|Ob{?<U{v`;}vV+c_@jJD_7fd+$O6
zPy7B02lsw`)VB9^+0=crxlY;j9=W^!y5x@CoINY;86D#fd|KVV@0C0AzMSq)d-v}8
zxX-dYbMK>?rUSNni}$Wd{J2*m=fFPY|NMJ)bF}U2T*tH9Fu-fCgAwmOk(5(=uB@7B
zllmleFWcG+d-a+WY#NFM4`$u}vzP7jzP(Ofn)_ZnKfkw^?eX4v<!N@Z%5(P?u3obD
zaoxqe7u8?xabG-r@Af#Gy$PrG9B9%E**l$Y{oW<X*891cn-BQ6`0bD0)^Df2e2G<I
ztDODmBRsatl#KRrPd>Wa?%QKK2PN-4Ctq0a-JsxXpPW)`Bj<H|-;bYewtv|4_TN9n
zdSJ<P2D>j}clNT~ID6m>^V!|vX*2ik>P*^av+C%6Gar@x%gT4{UEr{O-(G+A{reby
z?S0L?ZqL3qu6q{#(%Cmll4GCNI{pLc-W&Ixohxs1#C_X7!SD8am!Cblf2-eyefF(6
z`>wR7?_KnL)}FY{ocpv_KHt0I)uz3kpR)Hxl`!qink2LL`J#E&Rk3~hra5WtwZH0U
z)%VcgfJ%Dp-b&kv`@N6M-plQEa!-Kj`@K`wT;E&&B*WHuJ=b23RU7w~xarv+_{+VY
z`6i>oqC-aZks9ZA^Uc+_eYfSVO`*EU-u3D2HZ@Am>>O=Z?o~~>w)cf!*ntTbCt5rG
z-ESWtroQ)A&@}sv&$9N5*i78NRE}YvcH+tX`{I}FUB~oeZ-Df<eY}gx><!X_?dA1f
z?Qtlzwb$N%XMfS0S^IQC7Vj-7m)$E}$#&q+wyid&MV0Ms8=Uq`X?C_%SLxYTcIon7
zzGdh4e|fcbujcKJeW4o;@7;3ch0V3rDSMxEz1VF&+ufGEYQx?Y;hOs$n~U~txhS<)
zrCVd)%jF06NEj&Zdy+4B;B3oJYv+P*`|s?SzIR8^%e_AoKkk{l;H$MM?-tt-S^2%b
zpN#e%5o@>K)Aipj^z$wI)s3n4pLcoe<9+|w_SBXmdqV^Q_Q>73X=C?2#{T%Ejy>~2
z9@}U&x*TAv*k&Ww)oFi}_l|Y?oTqlv-hJM;X@%te#0$Ij@+xs2m~J}JS}pw5t`d(8
z`*;%g_g%j;-+t0bxxIH<EbNqoJnZxL&)MfW``})eFpj<3*tPdRo4~#AkVUP1d{*0@
zm!2>8F8rLluPF4=o|lUR4lMeiyEi~Id!Nxw=6#{x7wp}5_{Z+{KHa_Z)PC)$|95n+
z<P+8dmv_|JOiM_%<$u0vpJmY2y)hSV>=n1&VpkMvxp!Nfo&BDvGxn@DSKoDKzVF_=
zW!ihUX3Fh1uHJ36an;woy3$?yU;bLZKlrV}ewX8{`%nMdzIS@NuHEzcAA9HWHt+rP
zFJ!Ofw2AvNpGxj+_LthbQ1iI;-7PHp7R{Qsf7+Lxy{hxo_wT#dv2Wd}8TPi<8*H3A
zAKPnhdbn5P7SG-grtACm{hDE;A7*jD{+Gz!(pwYvuaCO0_i<FgzE5vU_nvw0v}gV-
zMO(GGQ}*u3YPVxLx5D=0yV-k#UcTL%^mftShuMDnCointS77hE@4H*QwP5DSy=!L_
z?b-j!Z|}5kt8J!T)Y>aJ)4_hqy^g*44_EA+Et0U$leKA&-G<|PONH$BU+sCnx7qpV
z{_+{}wzm#E-y5mkyJt>>-rkpGyX~@9Iqbc1D#XUAXwrV)+adNh%+A{9vM}1unVn?o
zw@+nXjr%(r2gA%gW)oiQRj4txuMrO0dtlX<-A>gr_V2rMe0Nc_w!QuR*1Zfna_q9U
z_3W!#x?}&TG%lO&I}ZDgPSM<J_`TgG<m0k^|7vsW)ouLk)8zi_y}M!CzE7n`>{h>6
zvrl%-*S)KP7VX{gkY)dA_J%!8(sS%`nr80hN&dV0xz?+_oB`Tahh+}zyZZI)Uc0$c
z`}R9+-F4))#a=CeWGkMIn|r%Y+1k#N<KMeD``&)*eNXmGI#9M(GfZ>uRpWk}Fh!|-
z`}4l;pE>2vUe?6d`{fdU@4XSTeQ&aD)9%n0oA!u5KesPUR(P+t5TDI^iv#v5ri<)q
z8yW1+WgOb~X6m&)o8~nfXua}j?|NC~y&NZ|?$yb$-@8%Z)ZVb2RW>dnJ^RCtKid0R
zRLg#jxZ7@%C1>q@>-G1mpLo6hY{{=Z%nLs6Tl)HhjZ`fAUW319_NfPT*g0RHx1ZH!
z=iW^^O8Y}SJNL`#<?gd6o4?m1hHIbowDSGonOF8MdFOh-qfU6Ad#f0zJ-)m9hNW$d
z)G536t#7Qt_8s3lrCMw6uHFZG8>N@-;l8(I@1zHrd#4`n-M#4NpZ%ZKZ?_5jvdFe4
zq-meTJMMia5+Cf%u9dZ)FmcVE(pO#wPQ~`@J;fMhc|NMjCSm1do3js`_g@nX+;fIS
z$Hu(!h5gGtU+uf^Ox@p=C3K)k?xRgV|E>MfX-D>YhVbmW!O^w1VGY~<bm4`2-<@>W
zd)v-){{n+adk=36-1k^YVBhm8+;)4#`5oeU^!MAwciLPFdu4wj?DU@PD$n-L_`78v
ztM(*Y?$hS`Zj>wS4RO%lC;pN5fc=%j`>qGs?QKvK+1LD|XrDpk!M*#cm+kM`wtY|5
z&EI>BKgsWVG;`wK<JXk!d#@GjTWlw_?_|b=y_HQ>dyLYQZG22sY<{wf+pgE+*t_Xb
z_5L^7SN6DZ_wHrT6xb&taM0>i{d1cTvrYTX@ZPX_7XNvF#hRUaHyKRW+r8zbjpJMW
zy^qAt*)3UlZ_h++-MwkXUiSPymG*P+{kE&_+Gl5YR%`ELy}msarYrVZGyUIFRI+sM
z(e9acJ5)>eWSu;)=c%KMeUG_|txoZ5`{>{6ZKT%g+RqC#*th76#D42HEB1ObZreY9
zyNKnniHG-i8}jV?CNX{A@^4G_-<-s>Pc>w*-M*JB`@ihjy-z&y`JTnLeEVDlf9wxE
zqp^4UxjMUj;zGMFb~IV{eCXJ>?8JN<tAd98g6p{KjvsK?cdndy--J!|do%WM?+fmG
zzkAhmIorI=)q53ZhVPqe!n}{?u;kun|0nKKo4IO_j%L+fx5TgecF(ojd+u+?p5_DR
z_v)=|v##GMvhOYLx4l1RJlbdB%(8D*#?w8kj~VQj`^LPlt2@~~<UP;+v&9qkwk#Dn
zpk=kgX4Vn@z49VV`<Cd$?bZ3gwr{oQ*FBdSi}v5{+Oy|Kp0ItB%l17B1n=*=Wcgvg
zmfG8Wim`|Gw$Bx}lNBo2o20pDudDa{y$bhC>|VL0*{^!ew6E*S&;48P9^1D-t#sew
zQoFsf>woSxcp__muYUHPZmpyH=bU4<Ij+&NcbjGXUW4U}_8R&7+CQx~*cYr}w9imN
zcJJ*YaeH0$TK96t^zS{Gd42CQ$t`<Blo#0*Ki#mmS8w6omI>bbmJ9CQ`{mH{y>D1<
z+D834ySK$wZU5T6Y<q7l`LxH|)@@Is?&rO3m0J717IE&ID)V`-lJgV0a(UhZS7H|1
z_o;Cmh+gkz+q1j)fJsT!-fh<=?{W5(-&=iok$vHbJA2>9UbBA2SG)hAAm2W*WO=&>
z91HgHeVBe=y;R?R=?QK7|CccA$>h7e@8#69d+nE1?ydejZ=cN+pS^$HdmdOc>C|q|
z+v^W(GJ0rxEphq2jN0J6E3^7`8$_D!GxSl}^FwjNesP&4wnq+R?VWLy#YXbw%e_{u
zH|(Za{oi|DfM>6Y!~H$$#RB&ppChyP_?t$Xh?3I1#}$w6ZFEcAzx7b(-p0A@HkZOH
z_FmdqyLW%XguOPK&h3|oKD;-}o%KMB;e)-u9QW89zA0y$-P~&P?p>k%(mf~luz7dc
zTxl_~PjEBazkqd~{U%qfeY1J^_qtSGv~^g$ZLjX;{d?R}H}3uR^sN0((FxW8j~?t5
zTlZoA>{CnjCfshY*L(17Z_iUUdltju{ht~x?6bPpZBygJexRg?d7r95*<O)ZVf)tX
zinX6-b#&ix&4|4UCvxlykG$O9=)G^B=7ckQ(u6iyPt?4+-}0aMUQ3^peap7Q@2xsI
zbMN}nReM)BZ`gBoZkz3Up8mbbENuJs%W&J2u8gsMr!r~xJ=Qh5CHIu>k^Owgdg1Qb
z`(u+<?fqu6%2uhaYu_UlqrK1mf8EQe@@QYu36Z_t25$C?wwLX_)^ug>i+3OQHVVAl
zbHeo1-X*UD_wBhav-`nCC%ZcpQ}z{VEw)`?`e1*$sg!MLl+uCMZOr>T-)-GHSL*9t
ztNx(<8E2FBzIb+Zuj-TacB~(*_Ohkb?EhCeVXyJ;^!@ymQTv(B-LyNX_kQmm-!j{u
z1vmCo9kAW|vD9eq*XOLZZ{B3vf4lg8uknX?yY81dcB!Is_ciQw+<QWeecwjW0=vMb
z>3h!J`@7HW(bm0+vyWQO=321t>waDv3ti^@SGkS$e!FwScDwkmy|-jP?d9!UVUs?=
zcF#QR-o5;Ci}vm@ns0s0W656At!8^!<mLD6{#s%4*k|*e1^2vc^ORTa^;*zqudM#r
zM)AR#z414@Y^sH>?wuX-VNd=O+r71}?ECgDUBCa5nb(0+9_@DL7#Zxd1?2WN)E~6F
z-gIGaQpI)au<qr1Q}}1uE&aJ}uQq3lO=4{JzJeOoy*ZJL`?)TO?=71xWM|;rvaeF0
z&Hj7!R2%;Op#6ew>-W}s#qE7M=jc8*{*rxi@9yodxt+HsVpqammpDbcjypm76r>mI
z{oy-(Z^~lteVr!T_rA}0YbU0ryqE9cn!N(DYxb>`;MvEuapQjLUETXEW)$u#Qx(|z
zvf%gL4YL^bz3yDM_qt#AUYno2d*>***zXaS**h<becvm|+I_doIQQ9W74FqJ+qEy#
zXr>KIe!#w;OKbO-m0#SuPf=iR`^o)#j_>!{pL^o$-oGaV_Pu5g+s_oobf9Y4Nt+0H
zPy5+FrtPgQ<Fx;NRA8To3%kvOM5SG?COxuazrS$r9S#E<1&7G}0eUz0K4j*%ueN-!
zH*r<}-nkdo@2j4-#OB?k>b-v_pE$t$^T3|78G(B{xK8cUJ!`oC@jqd^|22(!n{N5;
zi~MeC*Ck!M*Y$(pp2YC!doyJc_6vTPyEl0ki@oJf>AgHtQ*9FK^7g4-lG$5lI(z@)
zwkLaM8cXgww4rI|apo_30+&ne{jM-|&ziJ(duvv{*mFspbq}9U``!i2oO>^c>F-;Y
zz_|CdwYYT>gZ92t-=Ev?u+HEARIGIG&hDnYwOohxCbS6dJ^AjG?Xfp6_pbO>u~+Cf
z%l@U=9JYJbGTWQfDcBd?U9{IKA={40L1&N7<i&eK9^Kq4WuR&=)6>1zBzNgvvGu9@
zRav&}c_-g)r|!tQZ?<rtoia<&-dU%$_BQ|P-g|M+hy8nf6s%QV|KAgxuyXG*Z!Y`a
zuNUvz&=RmG;>L5kQt3>4%ViAvTDDKwyKxueJ{x`J{ip5SZNgM1?49j6cW=)ry}eyC
ziue8eH`_XVyXpR&3p4j!UeUWxcY532X_J}v6@_x`d#xG2n`!=Gn;ki8_U?;Tv3ops
z>K?9+CR;C$uzgqmp4eNzq<F7!{T#c3TSa>v!gB2QCePiQ)nR25*!yJf>GU&u3!8)v
zum`{06I;r%&;Drfev`lR_q{n_Wgj3{wZCg$)*fZ&rTdrsUA|Y{@8+I`Gd1@L$lBZ8
z;J&<<PwdoQmPOwCgj_!F)tFMcf6uY&dpM8D98h@eyuT!Sn%$l{=6zQMX4>byy1DnM
z?#(?X#ZT{Zzwvb6)`he8F#J2b_gvt|{W0+mcdI!a+xPm*wLQKT-Fw4csqb4HCbUnp
zfa$=b8%Dcd%ZThP>*Lt>*lgL}Q=32USw5$2-<yOddw)#S*}HnrvpojR%=<WA@$W60
z^mos?yGw0$-#TlnDfeJ+$;8gRw_oho%PjR|5A)3#HWQZr*q5;E>t3TpiU)R<hws^;
zZobzng<~II)rY-4nQeRJ?I-T_NVD3zXWC2q;wL-o^kb*(KiT89zkJ0M+nE)*`wx4p
zw@Ef#wU>=OXOH~a&HG}O?AW{W=(;_(om=*woa?gJ=l|+`5e~2RT7=KF%SiRJKl3Be
zuEs`w@2>}k?N85{yLX4yyS@CI7VXQl$+hPY%h-SLXwKfY?~ePbvL@``*O<AFFI-};
z!Q1w|n^&&3Q(nrr&*ee){zjhkz2V&tc4o&N+*iGO!`@}5cG+K;uyuFuevy6N#|8JD
ze4wzmy`5*@M^Wy5Q9mB;?$-UdS83y}y$%5ld%sT=wNcJt+2?hAn$5#cvNqY-`TJ*u
z@7jChmD&E(J9qbL9J#gELrBzSR?xLQKTa|4cTzsHCpDITpFmQtJ(uH;eet=8`^}uM
z?pJTH+Gl<>ZGXMtioIsfKJWb~b#*Vp64U+n91C~ZKUB1xRLi~JV4?ZmDQ`FI=Ph&E
zYx-$|-IX1m_o`Vo?q_29w?}}faj)FFE}O%>3-@?x%-px2%H7WQ>ZbkPwj6t=S#sOw
z)Ri6RGCH{@+%|o$me|a_e(M$YE3ioKOIEnQfAP^~8<j)P_T)6`><cuVxwlzso1NYh
zM!U4mnY*VO&fGI|)z#gl0Z;ZmWlY(d_~X#t*0o7{{ECk3|9tYS^<S1iTke3sedhlj
z?=`tTXZMTh#rx07vD?0V^J@Pc*=2j*oH@34?>~2&s68k5F0L2bFU)moFXM+QE7kwq
z_OU;i?3*WkvNI8VyMMAl)80n$mIJ{zmsyGJnZL*C-t4`~0_J-U{a9#Yusq6o$${7V
zC0w`f{qQ8f{`cqPT{62I_U{P(wzpm9>VB<n0{iYer0!o_thqP)@`1e~t!wr*9^P%w
zw#4GVs&<xrn<E$xM72KHzqZ+KUwCZW-kz_?dyo8(wU;<*wb#!4$Nn!l>-RnqK4!yh
zxO8{Z1Q#2=f3xg&O?b2Cy!PikZ+>jvYc})w-pc<6_nxZ!xwr6c*xntrxAyLx!E@k)
zRH{w$|5n>M-zMyvdi%!Slh5|<@zUftz~*>m@7ohH2dt#O+kEi7xOalG|DK$s@Agj3
zK4kxE%b&fi&LKA5Z{zK+Px@+SrJud;aMIrc*T0<ETOG@H@Z!s9yBnjp_i^dH+<T*B
z_5SLZqkB(iFxp!G5jfDaf^TnouigRG$(?(9n|Ss+G;FoM_h0z{Yr}Ty6CcX#C#l`o
zE5r7BuU!1{eMbdU?Z3ahy}$HG_+CjaS9=eQKl|&HC+~ZFJ9KZ<vtN5wI<DNOyF_5$
zV*l>_`#3)DiT7N!_j5wqKH)fFTgCQoc17oQ@6B5zySGunaL;wUuX`^2zq@BoBImx+
zmHm5L{;=)K-@f3$5vkt2ZN*1+U%oSa-@ELid*j-C_FVFoIbit0V6UzB^8N4PChfZS
zr(<u>>9oBJTio_4y;j^G=&)t)&9XUrPi<AOj}nR7@A!MV-3r-Kd%sl4y&GAr_UE5|
zxcAPTrF-v09p5V(6lv$d6SMd1!8LmV{ATTcIcv&Z(;MyfTc35<Tyz$4V6SM}r+Y<l
zzlsyX-pO69wkIN4_qBchy*DOUZr?UJRr@9zTYKSJ>%Hb#Z1z5<rtIIpeEB}vA8LCy
z9N)0#{s+%}it~2w-Qu@nzjpGOJ)vj%_BOG+**o{;hrR1%^X#L0`)p^t=Giyr@Ta|V
zDtq@<T{qqPJ6Lz`?gG=jr>9-sd)0i~{yAG#?ft34u<z35Y5N%FP2DYfWBK0Cza#83
z&+^$Q-(%b_-YvebC3o`PSq@D5TxvG%VbC<%Z|5{)uXpp}J;Jt!_FPb(X#eHJv3(DA
zneNqFIDe1L=LP$Xp3d1LaeT?%EglT}3R@=bSD(MYI>2D7&4c+|_Gi^s?QOogZNI*e
z$DYU5yZ6smth0X{B5!{rC&0$;eu6#Ez0ke4nBLl4F0b4tza@JA)e~p-E6Y6CbE9w8
zUfK3XcB{4K?t3b=dhea)b$i9EFWAk_IBi{Np<=HuXRw!f$MU@!PU!3lEwtPFl)rNy
z&k6m#D+(U&iECTGw^3`=-j8Y9_sSX0*eg?Ub+7uZ*?W%OUcYbVa(9~rT&y-fxK{4_
z6S>hwIYY(Pi1WYQ(XU7M`XwK*H?pzWbMNV;y{;K7`?8-#*xcl`+pj$>XOCp%vAwm|
zSJ>;^jJE$SmcB>0UHE|Od9!_sCeGO(U$$s3<JFaWR@_qDd+A-Oox-lzy&-i>`<(YW
z@Be37xaSbRxqXwS*PatG()Rqj&e?Z7bhJBh>+as`Mx1tc!nf~{nEiV1N}Ier7yLB$
zu3YD6U&^7n_xqbU`!@vN+$YHTc;6|mg1z5fZ{IVsnEOD#p6A{Oqucv_{=Tud_@|8R
z-qOf@H%xl>1}WaQpCt5pPua85eL5%8ZB&Ji>^=DG=iZ2QaeKwMX4qV)Nw=Am8*i7v
zGGp(D4-@yEHnZJVTE@6<;cQb|wa6d)r?xHITV#EA|CZ8cdsG(9-Fy5++}`z1Htn5o
zaq-?Svy}Ii&-uQ0|M7?Rk3II-waoN(Si}=_;NZvkws{-)4u*Ps-t%=m|DL+uFMIF(
zx^3t7<oVvTeT@6EW3&!@SiO3$@BC2v#j`s1F8i}-|IyX0`&Hk4+b=y&VBfWTuDx+r
zm+U<gHEGW^lcW3o71`N`v2Q-GJCtc}XMfIt{|653Z{S_OPw6J>UdGV6y$?F<_UmsF
z+Gn{f`M}$K3-(5N^z6O3&c@bF-hA(UzZLd1ckb-1?R>n4mt)dijt`1EZ|+^WBdPoE
z?&cjc_BKmZ?Bxw$IPm4p`n^uU9ea6hPTeOwan@e{)I)nBJD1pOtMA{Fv6{nfdDpGI
zYCc={)ZFpe!x{a`dgby@`)^8K+cVK`p^fJJ1N#d3*!LD^iQC<_4BS7rdfi@GIcd8|
z?;q~fvEtjS=Ec9)Z-dhQ{@(7r^Jd6cPk%SbZb1y^ZZ?-2d-ZE8ZC5@`-<Lh-*FMXg
zANG|@>)QJ-F=8Lb@88x+$|W{M3<vf(t<$wv(JkJ$`@oF7-o2Oi={cny$kqJ1_t@Ua
zHo;Gp?m0WrY5$YrcY8VC@$9b+ZrJ--uxGExrqlbTf79B#<o>38+m4>vTfQf7-*wF;
zdrt(vv1v*=y*HBIcJB<m6?-dx9NoJ$`{rIArQ|)fooDt2)~W7WT>ED4oKS;3F5E13
z_j1_w+C`S{=l1E_``)_ODw|>U-sbzgdneoQ@B3`OZBO&Dvb_;&U+-hJowT3pM2P*@
zK6!h?xb=G*S47%{eo)!7cKcN8hiOxH?_Idk?#}t(z1%y_+1zPr+piiiY43vWF8j%R
z3-<E(uCi-l&E7XzB-Xb2%9lNQbE6I@rY+yA_{Ph|nQ`I10|DmtIUBV0FHwE7_w>hm
zcAx%N*>JyU+BbjNa~lrHjJ-ciQ|%7CU$}Rg>1Vt3lc(-&`_gP%*cGtv&z7xwTe7Y8
zw{eT_n>5p7pQpFbzHPFydul}Q?hRSIdT+rF&AkOqx2=!Xo7+9NyS?|8h|}IPBCdNg
z<+%58zgWE2EU<Q8Q6ish;O^}GT;?2mroQ;F=lh&{dymNVTgy83?GrBw+q*}=VQ;mA
zqx}|_b^9zip4hK{_|4YqxsGk>PnH8$U;W>!dHBxm)oW+%&D+OeuY3LYUJqUIz3WZB
z?N7UPc-NCfGWNedzubE+)b&7SvHpRmu08vGKK$GJimQFU;h!6O*^bNYed5@-@5asV
zb~*Nn2Q21j?W<bibU<&d+5VEuW&0#FH|<@ecW1BKlKcDg7!U7xWz)C++zZ~lAD_>&
z{lHkcx1sR#p4(zk_SqBW?q%Dt&UTwc=H5RpyZ1`z+3xkfcx*4X$z+?}4w=3GB(?Te
zv|ifV&GOI6RoioKwq5<+<!(WHC*OT$$-uzCpmfV~Cx}MI<z=hPVEn2xn|B;e`fZ;1
zx!r;#Xyta0_^NN_yFfGyC+S-6fU)l=-?9L)TaJ2y>AyQOEhJYPSStB7f%RtGYX#d2
zQUk)OEr)hiUz7uzSNUIN*Uj&7<{)tx?&4?%^WQBz1U3&Q4x;ynHo@%$@v&jcr<o}B
z!PF%0)&+}Cp1Tsm+@3wZ;QoJ|>}6i1+Oo5n+uMRiXwNo~{<QZuQNk?5=`h$VkQ{3g
z7udZ2+;U*IMVqYJA*Rp0^D(0qSWoBnRp4*{sRiMMNee6(GEKm8I?l^>ENuS>)(?^c
z;ZW&~V7c^*bHL#M5`*DuCvq^vU~({8#F!KAHkcTQ{x%~GEZ&w~h2kEN+~)LIU^_u<
z)`Z#Ht(%@(<lIiR43gOf_S0X^^>BHRnm3>3fz1N3-5;&D0@2Ke)xmLkVI{+^C2As8
zJPa;i{=el8V0n-l5Vrl_xby435U_cN!q@ItWhnw?gXBPX+NbAW{-m!`a63TaAl%lw
zbt{O*hv!Us4R$lSn%@&bz~(idngTW-S^iXs4qR-Zx9-jp&sLeKO)oN6efk6(|LY9r
z!R0~v^l#jQ#~E|Sb}-vRNeN7!uDfFq$uZm9*>D;-oH>jd!1jXlfv~LE&mA0IwP5qC
z`={*`;_LvkL2@9x@&F{vRntUK^nv6K+cTl41&N{K?=P$|^nui{Z`%cS-<2a77-sxT
zErF-2i^~`9)YJFdIcsB$CC6k5uv&?Tg<!oPb3oW@V;Wc-#NPa2CD=Y@iMinTKEJMQ
z*P(^lmO14zV7~v!Qm{NoEeP+oFxmC%Y$sTM%I5x^!E5iJ$k*)M0OrqKtBjIvKzhSU
z&w}-V*tl@cogS?5|6kV;Y;ODJD6qSb^;g-hh39|Yb5d5-dc79bPbXR45HbXZ3+t~M
z6g?JBE%5L-@;DQ$M%1+#tj6QO{#~l>%Pc1R$O5}#Ug-re4YRKxNOIThk6XZM{np>!
zwR_4!u)ARL3}0%%>Yq=<jQ`xFZ{haB)uS<LtYt954W#6I8l;^0uQVB)E<s{2{G-br
zY$u2xAt7h7bji6LpJta@o-FJIi&x4^f!zy|2jNvbo8WG)*L(+N#|s<*tKD^#ea8+5
z7OP0Xd*Hb0dUO>m57Gz1&G!#%pKNyntoF#&d0WK4cB1&{e9BC)KJUlNP{cucH%~^y
z6NrxuzivDa&a>!p8M}JG=3U<vj^Y2wvbk_Q%{PU%SDX{wX;>F(xrk{o*qs8mQ0u?E
zloq(3rc5*jhxfVDEnu~&I~MMg5n!{rbX^)8W;`A@;c7u{nq?fbeb<68uphUYF54!c
za2V_^kQ@km8?l4+CH#W87bFJ5jem?#)WXD(={=7E;eJ6DuQqK4yR+ky8`ykgxmGc2
zuzNsaej77)tap^!xiGqTyFxi5IIPz0tw7PkeYh9y4ly+uF#D&NiYZ8c)W!Vm>putY
zDBQFj93FApk|_SK&G6pozj7MbyyxK@yWby|Mv+f*z6*AXj&UP6UxCboVZ*?!Sp5c*
z2hpal7{UGov61m+r(&=>HBSYD-H9xxqjw5!?*4}&+k|!8&65{wvC#ip3id<hO9K?Q
zY_!?|=U<ZP2fKgmWLvPilc#*xKKFN+W#&~kF#qNr7qC3czO**Q9j?V?;BeK<Jh|;&
zsx6oelh=%fl>gScx?uOh#6k2w_9%Ec1>$4FfxHvIZbp}zwqY&UybyWJ^8eI>Lva6`
zUo5(FZ`QJ%*Y;{#XxRFI&8mKT5yjrGPU>(!h1typo4aPO7g+6PX|^4Fz00=k$lVI&
zuZq0^mWR1VCdYc`sZvP(SD3hM=Z;o=xLF`I!3S4^-LiQdYP|uHdpzS7*h~-`7e2?y
zhGHH}UG6jquz33G*%)TrK71c8_QrG74*i6SJNlnIvG{vn4p_XIj}x3;VP;wWlLp5v
zh<{ZGl7BfjegLa2n!0DlyUA}X%2grZyoEg+Y%fSH2y-ym>^#W|@xN&1ydAri^P<Ra
zoE!x<=j};b6mgJVM>{dNy&yg|Y+Ldb9De9>qFPhH=Ba6{1e=d6Z&L#)k3eEszmISG
zFEnlEjO_vz+4FXR&AJvDgW~_ter9+cdl;AjHg|UNBomN++w$^l+LD5nrJsAiZrxN4
zkq60vF#kMu3;PsEdviUf;La<Z87S%BD&+)N-wL6fV7ozjVEFUa#Ta5RIT&5gVGec=
zj1Qw@8<WA}9NSUbav*s{M+<nEEN9)j<4}6&_K=51wuTzSg5Bo+2USn96rvAtY)L9u
zjo(5Ja2b*wwbq2qV$%-6-B-c<vwIp)+}!wzcgHQJiC{l2-qf(m^aL+hKgb;*>|4_a
zcFP+D%=+)4?-H;aNF6S`SJfHpPjq=V=8a(Sd6An?;u558CxZdlJs|d@O-Ht^I?A*&
zdqUFo*!|Yvy!&q%q`wA|2jOSaPJzt=v3V9TgWXhra6Z_dra$Gke^}hH^R@glu=uK-
z6ToJG)Pk_K_VpcRdo00js8Cnk{i*FHioD9<HDI@}&+dZT0n!J;|3s&P%>=P=;i|<U
z@U)4n#<@liY~C^@d$9S)a-Of2z}-;E{?zPI<-;ASnzfeRp8vsqa$6$|b}vlN4aRz~
zSs;GJ+=*az_pz7Yc(gw4YW|4D-SXo~Ua&nf)`-3mNd0@2<{kb2UV`Q7^q%jSqzq}7
zg5*H>f|nN9k0}dK%T18novD>@cY^rX@NM@JtZFu8Lh}E56>F5T9ArlG3<a=zK<pEB
zE@olntUH)HbazasT?F=jio!#<JV;IUsp()pgV-+@7K8nL{aZ9Ry$5#hHLbXwwsYYm
zNLl@9f+pBMAhjSI^w8GaV8$Y_d8h6+?zDXYDPur#AUt933b35vEETZbATb!eEAks`
z2aJ!6o}CZr=cCKDuR04(7YhH_P~sA#?$n{xU^_wVY%j?j;vbxMR;lz@6m5tBhowll
z%WjYy49oB##x&-|d4bKXjg$bpH|9+HHhaC7mU(M;fX#fnObILx)BEe5-p)t2SAyLT
zC9k*Jra%QAmLN6ZM-9MohfHpR%>#+S@Q#gU;IM}AvC*j?_oA2wQ{&pV5ghMJ&c|Z(
zp*c^#1j~Wc@+nQ)n&kFoyG^B+g_Ev4IPQ!d%7Og?k_X|i)Ggp}0kLOW=z#4jbK?iA
zb#AUO=UOXjnKWG$tjG4zD!5vZ`s<&jn=aXD4UWeik;Mka$GTDcIaOi{*qj;c%_!m^
zy*JeM8iQzjxVQg3M*9w=J}}}u*qt}eVD^8EzeCz?Aazp~du%u7nX*G{<3@|3$xp%I
z@@YpKI8H$FAbhRYA8Zzg&8T?~%w9G1Avjz)OV69nI@WAie_asl{}%#h!SW!rAiU<s
zmu<0o&Vv28zhv^xj$QF6aVmJI9jvdsj0Yv$Kzg6fIS39n5E~a>5V8;>+(2qC9MA!$
z!=Kkyq4)(PC&9ZP>>d!?MJjg3AK~RY5+Yr;o!a^vY?fiTGK!x512SN7kov$cx54hu
z`JM_ED@^Cv{=7JB$EJPkU~#z`KNSC84yf39YI{A{yfQ`8-CvnpQRICEc)<RjnS2z(
z|7pjkf#U;YFD|@y9Y5Hg=<=_;o5A)-{hWzm?&+&3U^_v29yaXX{<Cr3cH!n6bM_}n
zU_S^{rh(lHk_TbM$%sCSwfjM^x%qNhV6_Q4p_W<`XIgl(27t|s{h9!l2k8UhCwC|8
zn0DS09NvFl`0dnvdIZHi&TTuv`fL{eL=gw+<t%>+)(2wa#Ln2;e^aM1fX#C}kqtH<
zS-;l$4zSxnV)Jetw2-ea-zm+$&tiUM1K2*t#0C^Sspdjpvq0+8qtAfj{RIaHIE=pS
zm)hl(*|@E-pa!gmcP?VA1f+i3uGu@Sc_3}P^eOLlCh6Zs37;c}A$>fbrKsZuAiZ~<
zH-gOsv2o#VT%9Q9!PK?;K+384di-E_!o)#z*gQ_S|Gzq=?VS5a&Rk{ciXAR1AA|E-
zm{T2!y%QE8#%v~TiUWtm-n3cZ{J)TIllkoh3wD^N@qxw9N}dGMAbUV~){#d$nroJV
z&6}X;vMb5@1UQUAav-d?=>ym;3tyt<8<3p;1ju+Ch>Z)Yh;G5^|0Dd(U~^TR=AqaJ
zGAGL7I9LwEw%xpL#|GBa9i<bbETZ<rfz`6+p9Iq&c@S<eLbRcRs*Z#GH$QeZ*iW~a
z+jj2PN;dnN25B>UW<bgXkUkI&ySjgeXb&G)?fY{FcPuhS9UomVO&e?%`|2JHe{k7D
z+EgHOaNz*%_$}Dv_+CKTe;z)laS2kp<-k0!A3*GugeyCCF2A=ke0PRL=)+*JTNW%z
z0@EOQ5Kd#;4R@PSmLZt^+~zDet=PuYm|u9baXTa9G_altA*lU-Rg=j(kNk)9e@-bL
z-nn4f9I$?nxgczQ_z^fRRQRsK-2@T`;rjoOF+~s?7hdb*g<>8|UBicSVDtWDCxXKs
zCJv%A{;UDZf!MPvUsz}?-?8(Lz1jA?uOVYvD}-H8^yF<egWI8Wrx~nwl@6q@lz60S
z`v(=d9Z&C<gUjl<XA8mQ1ITO;KH}Z9^P@^7*gVH49J|6#A;uCw@>%?lcEhgy$te0j
zatt+%;4lWWaS;4Yh_)HBlHD(7g5%?+T`QPImb=k_=o7zR^?FBW*->+D$56|iew|>o
z4jWPDe>Svug2h2*c+cwwhsB$tYG5(V!>`Rlr%$vLiO>i0ZSwlS@*uS!JYnOxostR5
z!2XZE9I{i)Q3cEf$${{B7A3I1dmizi=mW`3yMa2E3=%`fTC+ES(+|4%J0}sac|9*N
z$N!X5?||J3(&G?$Y3CHal{>#22;ITT1*uCGi>RQKnYMx1V6#B#mug%AyZ4J1WW3Mm
z<^%Kl<|}qw+Xk7(`r45MP9Gq(AiQePft}B9-Uj>m<-2veqU4QG>g%fK>%eY#QecZB
z4$}K}*)6a>5E~b6@R7&R2U6QV5i(}?!^jt0PJ_f?So_d7c$mF@#koCo&vWxtp5H9?
zy`2J1Z%)mTD1M0Hht%I7{VG|Id{uaJGT3drTN8H{*xt9;xmN=mE+_3SgK3aj5MC|B
zwA1p$3$VF5kN<Alk}m}IFGvoA6<eo*?aFgzg1ZSM4#J*|ZD4&MHZJ@#R29WMm^x!t
z$h;_zFB>>t!o)%JQ^|a|p3BR7&DZa)*?w&1T#KhSA>+UAjEhj>zs(IYmj=?CAqANW
z-InYN_GjDSds{p<lw0n5tO*W}#>QHBS^=p)UbbqR^#@2EJi1DGN9*0sDDKI+Z3ou(
z;j|Vw96)+t*lOc1jJSiz!Dz|tY*^KV99jxCZ~05iIizFrmxA36GD9um*N*7#A9l=f
z<=np4^%7Wo=8R47@Byir@D0&s4bjL1n|plL9<V<)v9XyyuGzmWXlW+c9nZddfaO8@
zK)CPIiyi$L;b1?m*m`KE^yvZ=dABae9PhOLHz?vDy`NtlCK&&LB9L|tNIf!k%WwzB
z!};B_Q2YW?Gi_xf+%E1N4Q3kRVLPAMEwl`hY6AP|N-rx){M!aR1cwVqugQ8#u(`6V
zJYaLzn;7qS)97Rwo&y=H6y5n7EDur(!d&t4JFC(Uf&K51GIPhH+iGAoNDhQ2i}Zuj
zeTyh$T>(f8hBrvXgYAIvvC)xO=1oEBn6lS`-ModP96U}55`*FPC@#3Uf+-iwXYp!U
zXgKb*FsOr!Vb3u}o&R3K`2p^yHSheudZptb;}(nWpV(QZ!M^j!%0FQ7T$e6**n;f)
z=P$macqXK*xZD%5Q(Jg1*uNk-5EgR34Gwd$cu2bfBnHDTZ{Hyp|8laJ?M9IP!w+_X
z-SPJ1I~4msax&3V;c;+G{JUAcrjw;d*>cO%Yxjcvu=(jlxI9RGO*ACWgV>2tap3ZH
zU$8DXjcnmh+_kvoou#g(J(wT4^cvV+kXjJl`1$mXIUbL}a;4G-c1#FE9n-8cgp3W=
z*PVffF-RW>FAS2w5Ce%L<IFG~aK1wp1F^f348VG2EZkB20+Ks?ung=^5ZilFpV^0e
z<sFRs=Uc8bVg!q4&f1NlC+QPnZNh>h&%ypny88sIw%>4<MTPiw3;qi$!0FApoedm5
zAhSWZ<-PCDb(21W)v}(F*y*benKuQ=f$$s?NMHQ5f-Km*ATbzb)ph~vgYmJ^v%C>)
zIb`_>_iuvZ{jLLM`_Fapbg&;l>hDa=-5K*GVTVNPBFiorNSkTKRdE#mpWh0p+d=A$
z<F&#5Tl#%9I8IA6UhR^RoM_?tKMBk?Jo_GQHb}kiwWyt&kG%uS{cW7H^Wdivuzrvn
z2+NgE0{gK(I|4-<B&QO!0mU64F?8&FW&$|e(Zx-F9s`Hx*~775yOHGs?s0?d1c_yq
zGg}=@Y~RN7c8ewV9wBgk+wt=tik{Yc5pX~81Z@MeIm;paU;WDHU5(SWS=`P#01j^n
zgL81RLFUUma@<w*Iv;G_Y^`UzG+#7<-35{ZVFT8QVD~qkJpracVlX^q^K}d{m>i7u
z{Gx{uZXh`!o(8ZxpG7Xl7()~C5rw-kcGY#`!1FJ5h#%#%jF(vm7T^8?b^QBXFrxpf
z$9WO#{xw}~V6~sFgzjADdfBq@1{2tiCtZ(%<w0%$;rUP1x2a}1gX1w`*DTXhZG~WW
zf#g6~BO8*}1Q;RV3KE0idCE>GYGLBY^xHratYI{>NE)nn1H)r*`HHM3imeFjc959O
zritbuj~RFVdM<8x_XDK=7cc4z_6tZJgg5sh#`MleLi(wz7D3vrhVG|!O1<7-88IKS
zW<t;v(N+TK{pNeyT<D@W*u20Uo@P(oA?wFLav<EC&H$G4{Epd=W806tZU^au;mh6^
z!2X2sVe}zh25`D~ZW#m)dzd(gR(^sQ|9`{yW_#)ZnVl~)ckkFAnE^ISWq%deO(3-(
zELW5TkJE*_6u|5`%0}Sy{;Jt?+xqh_wm&w5)R|RJW8q;7(%bcU?#^3Q8DR4`9xLyf
z+4&YFzUIhp1-s8IS{y|jq_;o*0!G|{#F6pkyLZ9vK^6nCjsBK^^{SW51m|gx7z}gI
zSps%Dh|lqL&sGb~4vS?@vn=upA!AWz+s>f)VY8<n+-;I|kouOr^fFjYpIz_HpTR7a
zk<j*!9)~$N3_)gt@Nbc$+jjVG0sGOvGkM$13I1SxAUP18UUnDEKdgh8lLCo@@V8%W
zC}x4g(6L?>Vk{q7PWo0ZSg%KLKiGU^IVas7xc{qL3U}_dox0QZGJ{2KfF0PMyGv)F
z=y{=vXtO?;23e<ay1WsrcKN}L+u1^9Zp#yZj1gTI+yu54W?xp6`3~N%c3|_eyJqfu
zAeM^~U%Q_|#v)B$PeTz0>5tk9=~IB%xUkeo>|u2HJY=1M`;xC<yOH%bM!$l)QE_$E
zHk;bU?Uz1wSW4#H1BZ+9qfOxWg6TOM_#Z3|;@f5}0K0R3Rts3oYL^{5`(0QpXNq!x
z`Iqip0n3Bbg0S;{=N&c8kTtk3pG@D8@)uHmg5*HBG)D=nFYIO`iawB>I5TQIfW*-8
z&usy4zaYygX)A)w(>%5koDY!YE|@Wc?F5OL-q~a#!4SXm-|2Z4-*&fy!>Z5T51d~?
z@*uq8hzPhW0kJPmfs9k1gRZ;0_P}LF*VRIcnX6BM&GaqnKyi<1f2;Z1*N|~sUN4m$
zC)ZV>$bVZ6S^s3537LBZ*#X1dw-1BU7>tjNUhv@zig_?KzO0aWwvS9Z!G3~?gJ{`3
zd*Nlm+$ysjOL|i+LKYs{@#6S5uzjXhsAI+@?zM0`++uUU=FO^n2o6{8i%ZReuhi~v
zF1!WS`;+k>m<HJc!n^nD?tH!%QpRQE-`}ZlcPczBf#mO9gN)B^`ep;S10)W@VGl#W
zc7fQqaAT!5IG)kv|K+U(i?jaVLy1d}x({C=Yn4H4>A5XCnX>|RS~+Q1EYxHL>p2^Y
zx@OW{4YB^iJM<QKzD+6ZDcHR$zI?a1wV`Kc<553w{BJ&ux@JS;slu)rCqu9s4hZkx
z6>=GMzH-m4Sg^h%<<(%jL2iQK_@7>2IT#-seaaGbJPD>Ii*py)eaak&v2K{$jtInD
zlclVTMW_AtofCdtu{<LIDfe5<EWqxCso%RFGUf;3-(S8H>`s4qHgMdOKQ-Q|zcbD9
znipj5R7v+TSRSMngu9uAcXFLO0QSedCDJ<-|3#z7-+p%(Y>tP)W3YQcdSLkDjQ#L*
z3KK)8V@x6CFuM4x#gKaH(md4l7$9|rZC%0c0kJz|7H&C}GtWX_b(dLjh85Ti#>SAj
ze2_c{8(drmce7?|0oc7JJh#F0!;pD9s`q*C^m+zq52YQ6MoHhFPcm2(H)Mj<eve<U
zBeV1&iu~?CNSm`qA_gVjfb138b_txvKx|w%>@cJ~f-djEG2IFz#!-x@>p*-EUK$H&
zbAs3(Htse*u0Lg`b4<;4S1V3%+SV7>LYe=M7D9}F>bv-Z&0Y0f4D1i1OP9_5>M2@T
z8%zSHnLQKB;b8?b-_L#0&d}ov!E!B!N_JU(>__qEr7iJbIp>2HFydO%kDs9bdHo>s
zks!N}ab;f{*q)OHT^Rji`PN5pyV4u^wtmSn+<uGUvw5k3CB#p&UxDKbW?nZBWQ-ET
z4>o|TQ<~D82X>plijzC)RG(T@XMF_g$#-uBry-D95Z>{TZR_iekao4LK<k$7$GgG$
zKyn~_&H*wWuzFJ~*u5Yz7`Au-nQMgcvC)jRh_W16US!Wqa6E7^$b$WZELRC#a|sfA
zsD9D(VO;ADF;z1Q-z`tU;ZoBg296VuJP5B+&;XkSVm~Nd1ZF#&(g6D--qhbhv3aRw
zy!Zleo|3bU2g`%hg7EZ|knN$;ieNd<%o95n?tO~lPxA&y-}+RlGD^6C^nM6S1=|5)
z<H9B1AZxbJ<<ELU*2TWP{0nS1viw?w3UFM3#H5*j?l`yb!j7`tt~<K<9KmjwyekaF
z|1vC)wiHPHg~O0?zcT_d?$EG)s>SCi*UX)-LFTHzu0)-0Z#^lt>)+o>uz3qUOx>kE
zrw}E4QbezV<45cAR<PY5J79PX*9N#dVPfcX-)bIkenc0KFouj%KAQ<iugLO>0+9VU
zATjT!$Ib5F3$-wIu(8NUo&k2t;{pk=dqMIb{Pe0P+|6-)&~Uxt1P)hM#|;+g5qXxD
zL2JSMhPr&XT9Dqf^A0<L3ktw-zD%EIr@o6Sio9I%Yp}i~yA%|0kY4c(*9iK*c|Hq9
zxPkQM*oc6|wREO~%?F9WaQL2Tus=b3Eqx)21euJTX_Ho4Jha;fPXCv3AoUeU9)w?S
zm4d6^lc5hbch;x>U~@xEq;^FFnwxK&*9_(tK05}M2k8UhXg9Z=?~g$8Ms=~ouDHw6
zDDlOw>JIi}e(fg={|n?S2b&2p2N(XNU5ud*r1sBiNZ<L7HlnQq5(nYL2?}sG+KV6E
zF)w8I_CL4TEOz;sfW^CO_M+I^>;_p&2hw}le-c=Ykv3%i*PJVY7U`#ScFs(PtkpXr
zybr}aKSgVIE?oo}|G0eP(ayd!MBf`^-UBwsSj;sE$ap<S42Dyld;^CKjE{}B$wp0Y
zFg5vMlfiMBI`tPg?ZL!BG-EDgog0Wfy;OEb-ruk0z5Hh_DztULdJe`jqWJ&rja_he
zoH_?t-+F7_9k7}oTg7)Vm^4}x^*Dn0uRd-<vG1Kr&yIqPT;Tj<l*qU9{8Isx_}7(C
z2b;~sz7-|hKyFKVgc?>LF?4)07&6a?E*|3onNP5o1(}yXmOt5!=rb(YV!X36W6Ms7
zx3L!6{5FH#68z=_JiUYT3GTOtyKU!ONWY@L>oHi(R>j60=N|-_wmyWc|6lHl+P*tb
zlDu=m@<4Fd{Css~=ch>2^`A$y=Yqp)OCjpI2#~!iRb;^WKx|ytd?I2j7g>$s9szJV
ztxhV%Fr#P|qK<YvsJ0_WPj07%Hq#D)38G-NLhAZp8fIRyY8%}D0*k!C{ubBH28Y!r
zs|!0!Iqq*aoyP{&!ygz7_76;NO`QEs9w|w%oQdh=og6z++iT3*8Ng;oyDNifm_87_
zDTQ$Px5yrmH$i$}IN~p4Tx?ehVy!<+j_)&KJ@?MrleWJR<KFq<-F3?}e#lzfPVRKD
zdtvGy>_Ute`*jP0({J2*$e4b&;rkuh;d3oN29|;K{4_xAkNk~d+_`%LWQ@OBlYNJb
zpBOmYU}lH3IfMDjj$-a(@H?admV>FqMyKlSLkVw~nm;a(`J|&p>%i`UiG%3lclyEk
z9K_z-`)TJdjh3Ca=RMf2-T<k)7sc9w^D9Umglmcs?J%|X%fa!#$o4+iTp6VaJFTB~
zn(vgj1m;H^MvN7M^j>^=Y3JLBUa*`-S@^EYkMhCc2a*Hf6S<JIamy|PMI0nI;VWcM
zA&89&o2#Jiw}Po#^-~$_PIb}kU>YV4qKg(ifVZpM@|(Bqne1<|$&<@+hvY@DT9(hv
zDB-TflMd&H=RoqpcX@xXpIWXQ+WAlXh}o$FEMR``3CQ|*kUb!L_4~CQ|E(b7f7NZr
zw(aYIw0A&qAiPLc0c`dr)(~)h1&P7%_1xF+c!G(c)2d|>aK9kSP5V~>&QFUxg)z+d
z)1C>o6QthtfYCNDmi!%`=Q3Mtd<QA_N?K6cKRo`3dAR)Vko}*cMt<P1N-z{LXRYeC
zydeUa5B)3{2(}kwHVA*yny}@pi7?o{9(9KuS8P^*^?~F-SY<n8-w12gAB=Py^iCM-
z+Afej7@mFPB8EPY92@IKaCr7wegfMK5`*E{a}n!1I8Vy#>^u5=$JU@J+a30-1DloJ
zzyOX*xcc7jV6#B{pYD)-T4vKB>$Q0=Z#6yoWY6{$v-W}g^)K=YTrEg_;<ev9Hi$k3
zyMgmz>dxLX(O^43av*$*p&cAQBK-m=;RcfH^haGQ2ogibTKSmk#z1QL&qDIQF|P`W
zdq8sH@ABYchY!p*XLM@bnX~JbWyl0ku(-w@)HdVQx%=RKNLf$F_(q~7q<)dlI=9nu
zL88S!F~~U1F+J4r-MeQ*cV1v*2b+6vhPcJYCwbto0J#H%#d<lw{$`0mtrtOZ8#OJ#
zVFP00!hO|{b?E5w2Ulx??ewTzieeu~9mj`#a52_PD|bkWI-2jCFxlcO7o^_%{6GgK
zPaPJB1G^EVx3Ya2*jz1c9&j1Cm*>dNrbEYfe4hv@|E3kxpv3Rjm2Nv8bPIvip1=5b
zr+@q@uzrwPAk3WwS^vU#<0XnXNKWuMVx9uT$A)W9W6m3c)crdh0d~h{nF5SF<nD)<
z6IvU$%Y1U|eao2VVmmi{g!I$X_o0@%jx!Kr^UVh!{gg@iv%!A4Y0bB5yS<{7s?IrZ
zp1HRWakc`;9fvGxwh8>;1COVKol@WVY}EyLSc2r=g+S(p_FEwG7)Tt1Pb?KiQ4117
z$Nf$HC~*aoS2)`Q4$ljVEx`E_CJv%aUP1PTg4oh7g=Tm5p4;KwbjEV2I4d|^Omch-
z_6tZJgv;h4*8i^(tpvMi(sRhX!r{pex2s&6y`xkbGFCt7BkKJB{rQqRn4=->s&5}>
zY;U%>g5sXyFi8L8`~gTl0NDq_udSbB^&3ncM87VFjKhG~$XN3h<U9`j$}q5>kmU*;
zc7wwPBo@{tyHjkF-Hyqgvn&n1a)Hy#xlT(I|2uae^2_Dpknw-x9xia4zF8!-TkoE>
z<*iMSc81@yKDgN+^KZY}Xu*>u16J#Kf8Ne{QK<F*(SFDr|IBcEjQam<17tlW$Q)dF
z%@#JWo6+SB-$T{~9^K;!PN&H7b?=PeVZy)ZspbDbm7N~@3@uBl=7Y!op1eJSqG$EO
zNVpv$^Dl$__u-8Xn4W(ucPD53MT@0xA?=uX<v!r_0kd!W#*$s8Htt~a!t`(Kn(c~O
zKC+xT3pTr+>nNCp=>ySI0*X-L0VIZwb$Z+|!VRRR%M&u6+LAR1>?e>I3^z4@fv3R;
zw=ZT{zpw0gvi6_(+il_Ca7nrJ0>xhQ2UFpG+Gh<Z3zpkJ&Zd|br?u01&1%b-WQZQ7
zc1XPgvrB01T$8s4A@d(4XJ>D#U+#_K&%9p9ypdTE<~Ys9#?#<1hM9$pKC@vrig_?K
z;-6H(;yRUi7<EX-c1f^5L28+vMVZf;^KVDivhUlL?pFr;`)oVv`qy9m+u;7+!3x<6
zcWpLgzk{`_*3M_Fo|fCiAY&#E`%%|y^QskXuel8wgA1CWvcn|+)x8%~-+}Fn$k>A6
z&JUb_2>O3>C8942@(&D4T^0nV3;oy_xL;v%c^;6lD-eIp%o#gW)R*jty)A9AN5&Uy
zpTQDHUIxj7aMCZtnr3Io)!?wGiG}QmetK%*&i8*mTL|uk?D5d36aveG^nvjApbb0g
zdLjF*PJJ-iF|QjkP6(0%VQy(hu=|#GqPB}bavB9yU^x&Q7tWoIef@XT3IEL?J<&V6
zFzUY>0aozzA5?E{?w9w_Vh!7j9p*>f!R77OgJ;2dLHa?MEBzl>9K;U0_6D4v6k{QK
zXqyX;?$Dj6Y-w8v*~hztdk%`5e;d5rbSMR~pUOWjW&1+0qj0l8=FNM-503v7ML}@b
zgT!Dsybv<R3gcs=-TzCXm<LnCT?bkBQ&)Tt#XTT-?Zhy+|CjL?@0=r7z4K=LAu|y#
z$ok(0TT#!{nY0?AzL-k~?9Q|&tHEZhoe;ls#)muGiozg!1`DqmgXKZ?fUs8DWDEcM
zYT&#$X|=`92s6Z3DoFlH!e_8KQ}|KG%|LSgijeUb5E~c1ev=o)KA5`9^=x2qzXUIg
zeARUcan{tPnOC+kZr`}0B5j6cNZB5+TcUMP&nSBH9<r7KWcJHE$Xvj4{_o&;o46o&
zmwtYr<!i;8;Be*=MT`}J)carI*uK^PvNwdiZQ1r0wg<uP0?C1Jco5|L6Shdyabb|$
z_7F{QxPjQX@Xjy%Sk=7Bhs@<Ye&+>myTi<QKTQDab`YPRrNON6blZ-?q-7RIHbc_1
zkAecYz5>aEFvkW+T?%5)KL^>T{ib+6*bIw1+B;XQaI$Eh0jb9&m?8V_L25y`=U}<{
zGyfT2wYtw9?YO`GA&UP6Tp;uR=ZcPi!ycpuhHF@>u=))q528PQ(nBe4V1VZyq|bVr
z<0OU|J4;T%?PAXf+E)KD&|>ngUi17ZmEgRz_?R!)y)d&9S0U=ADUy&qGTF`7!SVfe
z+tlq*MY=oK<JW-s2lP<S@nOE#wZn(!Em-bh-Lf5<Tp{g!nAs&8kAl<Zk6zSu!yx(Z
zU91>x2Z<x&(r83lMi!Sj2$`o|sR!A6jV!lAr3&u9L&r|;oV-h5XLo1hHq{-FG3*x+
z0x0n>unv)?cW;KQ$x4i@0GnYtcjAu8vYb1vErRU-T6p~sN_=p~NbmZy9y0!!<si6g
z{Q+hadEUQ}F_+($F~|RJZdnPo1KI89>=^$$DCWWBC+fw3<GAwPUW_m-p8?s!0aDw+
z>uM2w=*iA|aZfCBHKV}xJr}h`@&Ch6#F&PwFAKO{ZCM9dn`^GMaHp^C1WSWbPOx69
zr<YLNQ|%;SUVRpF2AQOu-gd*JFDU8Xs}!>5?w}&-xj7)W9ntLq`vJtpg-`xe!0;bP
z?YTI}{-4!1E5K<QBnHE}Ga=`Zf%r4{q|BAB)LAT;y}_c-o*8VP1K%7JH_kLcoas@0
zq#x|Rl?NbWT4F2Ob{4%Z-?4X|6gZ7fOX~pBAhSW(Xa4>jvnnBD!wVPj@4PAxsbfHL
zAbf@QH(0%)IOaSS)8QJ7vJ0dK8Sg|}Zx54;d1M8SryfSgJP=GAM6dQotO-9fug^?B
zlXa)`g{B?FXCQg-i2Nm#_;>hr3+!i*-g^F6u(`blSi$jK5U#z;GWCV!zhuZB|J`#T
z^P?cOAiU3d?T&D6$bJgV&A+xEUWPgzq1?g^wyVyh1H-Qs>5w%nAaiix^3yqRw;`*U
zU{wJwS7f#NG0c@&xe6=?QvZ=@ubJsNr%fxK1#S~zO9RJ$xh-P82c+g{-~o7gc*(a3
z%$|M;(uQW`s@ri-s@-CH+G%h+GHga&$Nbwx)NJ~5$o{EUr#Vd5|D1{99?u}i`ls-7
zM=;WH-JS%nT_88$!uO}%1GmS}<)0je?El%kvTq~09LJIMV1I(d=jOE-Pd>`9vzjA&
zyUzBxU~wO=2jI8`$%Ak~h8;YNKeM=j*%?BRb9<T#`*(a=Ji*fH_7$)mk<*ZKmq7YJ
zIC1Mkb5<ir8fTbuXX}e~ko`j-IS^LA4{0}FV1>*PgT!EX*@jOTVFHtb(FYRpv8p-P
z{1xoZgB9Ljcfr(y=nD^3!2SfWog*A}ZvTCCheW50#s4Xg^TsB1pFz>1aR8BLrPe~`
zccr-@>wi^q#deCUWwxBRf*Y*o?L^f5tyMqScP+8)2g{k<SKKK)X(J?^fGlTVV3=X+
z0Cwj$JqQgI22&mvBEkB=92`W9O*>dWy7cj_kh~nmidjBhvw`GYke>gMHD-^5W>`cT
zA2EOOX(~7^30gt=av*sS=6Zven`tbb54MkM=@GEomZ#3%X>f6_W$hQpe&{?#)bm)x
zO(S+(P<aRTbMCg6JASFnKuOb9%NW7=KOO0e36Py2ymtFJcszml*zijM?D5Z`57~!f
zz6NzIE69wDbDD5-=kiRn2rlf{X~80AX_He3?khb|V*tAeq!xr*rbG6&g4h#YLfZds
zADF>v*p)(dnOk?6Uz!eCQ#v~awLOw5W4rT2xdS+?RQK!e@|XAy4nL4tAY3I?3=SLN
zq6aAAAh{FUAm>Jc*tqcD1vMD@Kx&giA^uN%aslpFkT?kIu^^p6_<!PdnJ$T)jpnVE
zuhv1%(s<bkiA#`L5T0`zvHox36v%w#mrRJCgmO7{JnM4Uc_H%w*dJ|E-k`Y0WJ2$D
zq33#Fxz@MBJ6j?UXHS64F8FQ(mUGUjLeU43<BW2}nzuo6F#J&VAXYW=Jp#eyY+jEG
zysU@m$(R%bPlHlT96O=}inq_=4!1mLJ`o&NJ?l~D%qz|!&QP134mqcV`BDHl{-4EF
z?Q*}yWGO6m4XoE(9kQPvW|!WDj_s@wkh#+57dGsWnrDUL&&H)z;Iy&$oiB<wNdIop
zOJFlWY+TqlYC1;y2Bh}boKA4K7#4AW%?F9W@Y36}z;Yn|!j6QUr=lk8{FX4?Vvg7z
za9nJB0y(b*BoD&7s-ocG)nxJn%<j>FtZBc0f98&f)kzjc(uQDr`dGbD{1KRFyJPlM
z$XIc=?)IHuBM@Z?$m|WJCgAvSv3ic84<r}E3h6h1*tqcAe#n|-ba~$6kTYEG7)%1Y
z3t3*obq+jSt9m+jT0c+P+4*#z`Cf)AVE<c4L+U}89@}f?aP>kHmxKMk;|yfnLi^X3
z9cRLVw;NA|oMSW9b0ye6Fulrb>vs0>LfSkHOS*P;d_i3wu|cF9Y^RkzWWFAz4@5`Z
zL>-3)iJ{}Yt4uJ$4WwpTFk~)k|B99HxCV)X@P0o=updBdg=*GqpR(rfkehkJQj28`
z*zMifsCm?(BoXfa1OdqSe$t2UV6hvuCwKm5ZL&x?*aQyen;TL4zsGFEcPi*X_8_0I
z4Kn|D6EfZpvKxf=UxDN`#{|sv{}VpAU`^W~eK4%G*ct2&7#~K<v_Q^<yZs&W{Fl8C
zAZIRu)b3U9-x*T1VyDHA>6ZT=#Dl|SRs;(;Z-eAPn8T1497iBF?}i6pHP7}!&c7A7
z+`CJ2?gxwDF36dJ#x9V3cObPO%-_9vXJi^=4C=eb-JMErf1$*e-Fe9Vhx?f|;C3}g
z4-8k|{Rj?I7#|y5yvh{CJeV3QP6_ZBp4f^N7-9IN9C1cT=AM3&#_d)Xk}jXko%eqO
z$A6I%;(T0?d4K94XB&aolBKQSHbTLsYOvdmX*TS9K3l@VViu&0dn8l@9#$are0){g
zx4z5;hv%aL{p~!@AA!RUBnQH)CPLQuSFc4K7XrzZcFzUZ#~?N?yuElD*gfd-azD+$
z_GEeAz%YZ$M*{A~h?xsb9xUqIF(+t&8K1&$a9Caa23gApGjnAVWXuc1|Mnl!-s*lZ
z37qcOivI6VskE@v6o>2?7J76NY%fSH2<zKTH@{FX2)6Iv3^kMY?WpUuI1M3dL)wnn
zW8_m|amXAl$Q)dFpWSM3oT1DAyARoiGH<Cg*luKbi>runmH#j7cdR<_W=Ht#2^Jh5
z<H2rt--){KJjWl=A9mG$0^U!VI2$rv)l~6pyWfwA+qZT@_8A;=;X+A|S|u~K=XF5V
z%;^2zzGHh5V(kUUO#<(qf!*(M1k$eniNUaxGbByH_}FOQtBACWEdOabWIf@A`H*#*
z$Z}EF5oc!0E#ulbF<5G+baLMgS5?Rwl(Uk8;PeX9W4IBr=L^I?9KsBaU+2}A!1Rx-
z5Q{Hnyyh#u{R4+H_Xfy$CLpySd?w|{&X!Y<@g#OLhh1|NcA$ih@#3Fg|K~L!<|IM-
zK-lF!<g8&38yCLv1u`CqF28vLq#Q0?2H9VREbsksBG^ulSTvKBDgRIZ9Zz<MTFwzm
z1^c0YIqDcjry1ldO_2KR9n-;Tp00wd&3n!-x0CU}*ByHdAZ^tdst>^V0;Cp%6}Fw+
zQE?qo|B0{m-`V4r08V2dIS|fs*bP>HL=kdc8AuF<A3r0!{^Q42$U0}39vCg41DU7$
z_%j|{PQk=MbVVBCTq&^xlkMyU?mHz+WUQV}HwUY|SK){f|7%qcZSz{A$>4IDxikW-
z=HV5iT`$9rS~g@u`U8OyrC@oGJs>Q)oNHGnALI<iU&X>Zn{*-h5+n!0##xZ{i9Eij
z^QIuVS!W2(|5}?&0GC-H^~hNI4dgtHMs~>DBeGoG6ht4i<WS;{Ws}zKNG-Xzqr1WZ
zoGvW)L)J3E^b|EC#u!T<LiVq|QMLicqqlFXMTOX<9bsvZwFA%JqxSEP>^`vLgbd{T
z{QH0R>@=T(=u3dicen=`i#quS;$Dy#3{Sqh1DrNsd~CGab?oaOU+xV8>s|gIvZe)D
zugez1p17x9jLi3j9^URWck1>n!H~I0ZcEfNYFvXhgVP1bj8-Mc`R}En^T6@HMB(2~
zL7|lviY|I!{_At7{rDw^S$1Xg{RQV4kMO5Eu8LTqqz|9fkaN2H=g6aogY5d<0$Ec6
zV&lSl-$BNf(dC^rlECH6;zxF1cOlE$JVl&QcX;_K%ZUPCEvh@0TJAmzIdAID`r{~i
zWV;aU;m<Xb!R9`xg`9V=_SL~%oAk0Q@1{cbI38RCX+y*8yMJ}h&So#j{vW@#6T6B`
zQO|$&?S|Y3F}vh0xIBgF1JPVl(=fz9;>h@x0~1ELf#kcK)`9h&_j3S;JxC0OpK3tP
zss!;(qwblqUJBjmbF0UaW9mb&TKj`&``;%gg8d9qe_{fpUFmiK(*8f~(y+7Y`76s^
zJ0Nq$Ki)&;9YAV9c(tL?j_w^I;PAY6*nj8FLdgCBkQ@k0HI;zV=}+0SVE2N=VA#VL
zwXFaXN2aHo;l$7fl0V-l2Nvf^x&t;JBnHEgW{C4&TzA%*b2e|@*)J($wPWcVa5`W9
zC;{wVnEDtS#GbB6cOYZIBH?XdwMHyzyXw3DTIN_l&Oczx;zfy%4SZ*IOp}G~!ITu(
znZMBv>@JwuxrZQg@c}n6_i;`wgsjJcsl`SIbhE?b23dZqB;+2JzlZ8k`~p%l|25>E
z0}y-aRK*=KtX(@_&0*T!dk1p=N!M2;6g|&d5o?p&-a^`Gt77(S0_nH#7TWr$RA?t7
z%LH(EIJB38%SDhJ2)D%_-+BJU6tH=HbJ=$%Pv4B<|8;Mlfb~g=w4mq%={4)_Mo|kA
zL&vw2^ijeaCa>Mg3pQ`B8Vfj0!^A<f?=#5RMIbgSFPnJ+Q|xw0)k&5NZy@LYd^3jh
z*Fo|i{7QE#*enoRMg`K}%vXh+k*U3|Yv**u0!#ZVkT7LE2WcOJ)Piu%SH7LGA<w{R
zaJ8T7)-|h?;bwv4Bl#fv^d5ScV)(V$68qV1Abl{b^a9ehhVfzaI>QLCog&Pr>kUEj
zXMGTT6w`%!EcUvK?C`tKVbRR|5?udTc|q1Pfz*QVo@0l>W`Wq;J-*=j_JJ8>Ty^sL
zDht8;Oj~bpL(bIMFRz2r9}!*KzkTKg$Qe%>>&131sYM!lV_;xl-3u9uP5zBKZVa;P
z>1oKA0*H+Z%g8|XEThY3e1XjI1*p#jr!Qpr4B3-#|J-H_-flS6c8BD}-0dmXet`WS
zE)6NmVR{}$BhEkNoC#Uq62b+^w+|YFcg%W!%VOnM$eltf15w96c^QxH$ou{uoCa^l
z{ok>`262`;$o$_nufc8+nTEOklXpF2P7S0E7fzWRh>>nUYI{N;`+r;CLe^=6#9&zD
ziWS&BApW{pCFWD#J=po}*-i7A*^o6|)A%8GJAvdu__?wKJdd^NLDoWwKez`@2aWrc
zw?F@OYKO!N$o{Y6Hj*gm`+<emj^Gf;*-&baZ|#ixkC;mbnVs?ia&E*4Q_OW7a#e)a
zzv{l}z*_#T(}A>$KYjj+5&tFYg5hF8=OuPHJ8#&|^<|IQhUc>2_)q^*0+$E5amm&F
z@Gwrg3mH2}Sa8M=B==3vZ|g17c{{EvLi)p#Z^VPm0Lg*yt>vjZWU3+NO&0dsH8;i;
z9N!>05SC4X)FW$qN>RcMBxhg**&797<HE;oqMj!NQ>X0L3l5j2I@GhQLGmHqH{kyB
zW9P8i=P7J4L3;A`rIXdb{>&&o4VMS07ifc=c>`ikSq0fUbKN-|>`$%?<L%QPtl8O}
zw*s7|nra$Q{P*&I*3N?>kolxs7lz%TPa*jPWEKc#_11#xhSk3?&v99C1+q5)qz)H8
z7*-8Wo5*VXk3#yapEg3y-b0p~Ze<1cp9$y19n-_Acm8a<z3mOd7jPUjbwbYTh3R?N
zhq%)t>cJJT8rL0=^ELjhUc8<0F2BX%MUZ{Wsm`eDTasrU+EJtpspFphFx-{eyaF6A
zFtcSZL(cu?I&>Wz_Aqe}9oT|;mO4lb9ou9Ifb$``xS`f&usvDZe8KU8ET6y!Idcmn
zwrswZMT#rO4i_su^Oe6K<Dv_L5%M54Z=h$MfY_00SHb>Q%2@)Af6FJ$W_LoncCM7<
z1KSfZ6LOvvNG%ALADF!3dJyDp6oc<ec0SX%1=a_W1K}I%Am^yKMRtJA1&P7%H@RsT
zVlX)vZ9N}z+!Q1iG;tR=U1WW42d6KP7z{5qMU3wo7O3ys^;Too{otb(CgqSeBiA}f
z@OovKdU0OJoF9lk_s%PDxIVuJId_3^3B!(uQmZU0<o|%v)chxqbC*DBL0Id-p`8vb
zLJ;#RJ$JXvf}G0%k^|v#8%P~zwh~gVfW%;Uav=7+4U-4ayNn_IF%TOW^EvDThjX<(
z=J?PqPsF_%|B4T6eY|G!j)^BFEz57ofa6dq8Pb-6nOA024o{ovxsdj&v0DjPZO6CU
zJJgQITUz-;=1xR@pr-%UNBrA=GKzx3=8XsE4jI|?DD8uj`V3%qddkFN#5MQkHmqqI
z<}MKZ(G>NrB9IseNBn@C+4VIUan}_{9EA5=fZWvwVtaQ_*>RsCY3F>sW(!3Z$oZc?
zq#<`QfaF1V<$J_Fgd8DA`|qRhORzs`TNj$=SDRa&iiYenJgSR&{`0bG#%-CWB*AHL
zdFhRviT#Y=^aC;%gjr`p=5M;6Tf)N_Bo4xtqbe}OK;p=_k`r~kB}|T2lmndp%M%uW
z?S_eiXgxW|Tm^_dQ#{ggr|p6r1>f_{^{(51{p9uovR@k{55n9(A?;!i`(ZDnzxA;4
zt`UfzCiih$=#rWpJ(?H6>4E1v>U!kK|2FOPbB3HJRNtqvn^6gJZxl!`2#3Ce+>J85
zem;g@PuLOO{~vw82kcLfS;$z|5i%~wG6QnI0J2=45Twlk64Uu5Z8_uRs~t<<Y_j+p
z0y!(C+6%S)BYR;DJkB1qSA*F(FCb%6X-*F2Mps(Rm7F1E%8&nuvqC`T@93JnV^=F=
z{qyC6Dm%k}rlX|uMGqkRhGras+}8)v1H*@#&w}Fv#>YlSYX8OvH;_7q_mI0>f4g9&
zp*yD$eTF^jc&%c-Wm*Jnm}&9OX)ZXw1<!}}S3w3bFfdeUBF=PMTLn1_B-n`&oUZOZ
z<lQyt&^(J9?3v&^K2MnwoS&gu8H`vT?ofKP3M@C>igV}5ZIJb2F!}wSkTrKE&6w+d
zLj@rBCc)HVqnBnv#(B`?rhe`Rhs(8>khAxY<u_)<fZYQU+wQm4Tx<F49lDCuJKWvo
zf!i)y>QL8zaPIpEj&qRuGD*n(1+I0Fa|bQD>b6HM6WDpx64H<Bn<t3k|A(40b~K-X
ztPR{gUuu{7OHH_0AoDIgfvlT!WtswZFGviA-Gm@_qrmvs=(FwTP|Sm=d9>CY>=&J5
zkh{lV;vo95737>F5Iasj+LGhqe)F^U`7M5jS%TB1=z~?@ybO{DVNFRy{IB^5x!;22
z(-LqPDhO8X402+$2>-4Q=1bW^)?I=0f$*ESUpp4gfZTD%?k>6GhdkuIPmml4%N~H7
zHClQXbDiFUEp9080I5gE*ETqT-Ha|C^5HEw@63qD979NXh&WftWih*XrRts?JpSLe
zJQRi8;c%G+walndgPbJ{wxjL<IDQNDAoC!Gm7W%xTJyF?ibR9ms{U#ZihDHY=k2)m
zdp=lCH2=JvsZof#WI^U>W<$=Am*Io7*Fj=1Ja?53I80%DY&3rx;w(F4`N$uTb!f-<
zAp7)@<raFRz}@I|V3Wl$|D8LYSWGo9oCulQC^?5Z|MC6<q+A4<;d1#SI9)wegxmw;
z@3?AvPPoVRIz~OPTdSi`<BTJgb7x}{<c!1Vu6uWiOhH|*@^;mBaJbF-nu?K*XFIP2
z%Yp30g+EoU$Iu5-8*qdj?4HBYnEijJ#fQLt0I6eq9K3Tq|Dqi}K1<B^6}E!Qzby|T
z?RAhm2uE!^4_D899kNb;Wu_!JzZTXfSX{Ogw^&pI=~H}obqpT1Aia{Y%XV%Pg|tEB
zH8Xdf5J0`JZmEne*zC)UnEM}JdPB|z1DS&hFWimVo`9(fS_0W~d&CEG3_a~4WIaAe
zEss%?)vKoK+xaf0Z@>NGF4*ng7QP3E9Y`L8n~x*zY~)=R1Xkm^4Km*)`r*%x4W1`=
zBz=Rl@#iV528Rns9|#B7ckkRIZ45Tgi{Eb-??lM{agZDc$Js#k=VkG!VASLMMv%G@
zqz)IhNx{4~7^IfJ7;-Mc^_g$Mc@HE8!*5TS!_$?p-P9f1pPV<XQDND>$7Vh_E({+b
z&PNC7*|HC^<{!jfD*OTLKgkNnz75s8zB`*k=30CcfV5RiC*A;u4@fNtugf*w$>+HV
zoUU%3Tf6n)!|&jASs*zOUds)cN9}Zhw5vg4Fq}07GNu6IW228pLeADlmvcD)*|T>|
z6|)YBR%!$L6Qrm1z?bdI*ZkSRdUA$ENvR*$Z~f9%;JO_o55iyK5qC^?x<bZ(eu{~J
z{dv!&W_!O)xaIFVkabo)+K9aZAieYK9n3HPd<KsHsRhwH7sR5j50LY+0-IyL!T@YH
zNDmBeOhnw%0uw{0!}=lPyy)U$uOR0iDYIhs|9fsC?v+nsX|+gRthB@Bs`PdR?H+JE
z)z*K9rw@?19;+_G-6sC*4!B;i)QSVEy-<H(tApsK?fWdFz+qa`W`L63ezxmutJQ;y
z4J)0Ux%23sC18CZb3r&nL<k%<3Wlip1|+xrFKRgq5<|y}>kxMnAj_%G4hENtX;(1U
z{~8}f^tFuhn0Nl3{d&iw5JU6JD<NYWp`Rgpdtm0-O|Jm^4aCn}1BvJ05Xkxm9W%b2
z1)i>!Th~MOIGj<0qz{l<5bl}!c<1&Qh<RBb8F!jp>P89MC>3ikU;K&%*u5Y<F#O}W
zEm#i5$42X|n1!Jaq)ti@(w<`f1lgAj5`*DmYY}zgwtvk#K7M?%ZF=oG3)AnWVEZf{
zokod6Epx;g)5(2H!C}Fu51AwOd~LCFuT<BLd|SwV(6z;o^>Hw}0@I)FI41$w|4`)9
zzH^lm>i%fs?WW-PPu++ZV}R*xn*}+K493Ss{}RJI-vOk~up4qNM9UdS`wAom!*P!x
zbFLu%j#ru%<?W6;v)+1GRit!*-I9?3Ig=1155hVd_~2#p*3HYn>={c;z-2_I?%G`p
z`4hLVV~5OtiN&MN?^ec@@6`Cl1s?zC<B;CD^uvCX_=<9P1GaOT(0UYckiCj8QP+}!
z#L%(pS;#r(=;GJ;AmeFY&O`2AK$brz_YIz(UzADie0@Z4XVYZ<oo7Bm?mu(b3EA5K
z(_^v^F~*!a2{Nys=m%LtVRn1<4n<*!9X^_n_0L?PsO`8<RjYPhxdS=>FL<Zd&I6WH
zz-bI-_Oylb!Tj5bh&u*gawZLg&;L5sj@X+7(+8q2)j;-{iX=eB^g&`U?6nJWcQJ^6
z{Pt^$+P?D^x`)^;W3NEQzqWFCfa3%t55nvvkh6_I?00J*@jE4FEja#V4_w~0q`cR{
z<E$o_pHv7r9~q<;girrovGe=yV_-R>d6GMAuNHvQ4@eG#C%lF9e?DaY$B2LbGRWC@
zAa%HK0pAXc`#nHv7Yd#RoA)cx0c<x&42Bmqy$9DhAb#}Mo0i<aw_CV<Y_$yZ@dlU8
z%Ix1!^yo<<&P&^33h|%#H^_eX3w@EhE`69|vEV`;ILzGd-UG)c$ZQb)WBGcg2s31@
zK<$J3yC%yb_L6|)g|i^*1&TcF;b{^i4#KlU5&LdHd~CSnry)vs!qjYNxDWOV$46~&
zc?A;((UXoq>KqU|b5@+`-yDyfY8w=+wq_@T{UCl5lD9$fApEz@2^^Ln_U~hmv5;IX
z$h_VD2VZyY`1Zx}+H*)dySDxWTrEhw-FeF$(@P*@qsx`tc9z`tM)9ZJM95g=%OjZM
z7>lNF0oS!4b8z9~@!BZn!PJ$gK-T}Rzc3FT*D$%*zmT;HApV~(3TA>=qjqMVl(X`a
zg^YWu8=0cS|B8M@oAq`#<oufxT9A9oN@EOmHP$__tUGZP9Cq&w5pyLV^M9Hg+tIWY
za-UL6c-zhuA0cP#gXBOsN)|Hy>mW4&CEP%A-@m(p;}FEgh0Sj<Vdw*?Wo3r!vv|D@
z(oP47!SI2ON_ZM|OY1P7dQ5w#Ozwg0m2vODVYT2X<ZcU?dRb%0Tmp!%C<19CX5~WG
zkn4ngHH%X|v_s|w<oqk|^HN|lKx#pF$%|t<7g<B@cyU~9y=(p?#M&E>{67ZBJbH^5
zVow4{9E7X-7GaHBkQ@weZ<oQE{_SJ5!D%O-b0<dnkNEBk4_8KqwOeoVNSgnOSZFzI
z=2dXG7_Eck1DGAV-@SnQDO3g$7Q37wd#HbDAKq!6KGmXm9ptP+_7#Y{0@8m<;^2<f
zZb(1o$wWr8rQOfqW`X4A&xVY}M7%zM;ny|Or=s)~K<d%)vF15&zaYz<Eh_-~U(8V+
z!`#LnKfwM3sW<XDX7O_M)$Oa;`L<QYGJxGC)`Pn4{%8qgTnwbXz!Y-NnQbfN{AbUQ
zSKDM2-<a`<LH7S;v)h8r0I3CG{yCyM&gDV&mDUL;?{qCj-TxZ82y$OaX9ncFUXUIb
zo;(XOrwrp`qa|iS_9LRpy<Z4PXR98duH^u!)2&j6he>S33X3)_4vU=&b1aUW`3p`f
z-=OD!gVcg>7`rFjPZh0@yW@_iL)I<J^gY}eo#JXK^$5~d=U<C@9`ojw$vZD3Le4SY
z5Zkmp>P;z1e&z9mxKH(OHr!1hJ3+Xh4svc7h>Z)MS3tc_1g0*{2XddF_~s6jwk$~g
z{sKr|1+hysn#|`h`R@!|q-l9S9da+=1yj`fU%uW$+_C$J0dh{6#7xLJ9{>M5+3`JW
z$qw$Nkg{_nFY1|vsxM8q=Wl|n{}SH9x$FNk$XRe8yFu7r7}Dn4W5JK%|G41o;P?Qk
z!-d12V(<T){sLJ~zx8(nxLiiopLKu<p8kWK<9B$k+`i*hq5X~p+aTghel{q2D&rt)
zK0s!at%LMe!fFb^;hMg<Yr9u^_l|8FA!{d(8ltY9Omj@xIr{}<j(_=y{W~|6LfWMu
zvp`tj0Hkk_`$-m_#z5jAe5}+ELkuL2j6Xupc|;Ziu_Y&N2dA^G;nz|80+MrFgjf?i
z-AK=BrqPxi+OBODea6AyIFP!13`NgbM#P+Tb1mfTpjhKK;Bx2Fk+s_oF50wnQtM-I
zS+#J93)l>hJs>Q^Heu)eKajnWH-kHN1)Yt6hdoGsEf?e-v59|C`voAm2cD2J6~xAc
z|1^1GE&pGqK=xt&`~o@C2U$PIdqn*&w|DXmmA4CbehJ-U(fC>l?Dl1RkiHB|Pt9}0
z9J+U8AUIAo$U)ZmPMO2MbMaP1%d<U@I2QAY1)BlWyQM#8XXn&i;PBkG_W#bmji~+O
zOee@V!mk#{dKH*H5N-K=FN#?pF?4)Qt^uo>&y|pKfja~sYx<D&ebGUjtMc~7UlW0!
zQ!G}03bgpT^#j;%$;&pN<hwmKh%x(lM<Dwgnqwemr0nUuxBbGAww>%d7l7R%?uI%Z
zY1_MJN9Q*aaGbk_@$R(9L7mgz^_3GWXTyLvlNRPyFTRQ3GzQ~iqtj>RgY89^%V_lj
zrwemg)UzBx>gMW0*13b&|BrL-IF~K8YgWK*3q4WDcthkgZg77UBoD%e<gSDB5{S*a
zWe=ErYmYEEPS<{K-uds@d&|cUqQGrNDRxM`15yjZ)42tAS$y~emJ`jNv#YTs2gSV{
z>yCrXVenT5hXY6t4EwDq0ow)RW25ULM8W<<m%CuB3eJ!JD^cg$LFztFMC8Gu{q0*L
zezotY463%Y-3mFwU&R#CmjS5-;W7_I+(<8mtW$0jf$Smwu+)0zKP?5ze|?c)_n*;)
z%sYVefpBZJ_x3lLkTD}e^~c+{Z{b3T(<A&x!D;PCH0JpaGxEQI{Qxot7Z%>|0Nl<&
zm$&POoTuS*9<rtnS)S?R4!9fkiZGh~SW~_8;D)yrY}y`RKg?i;tXG5SnNy0`oAY(1
z4LDAI9)jE<_VLBB9da9^EZvf_!0{Nt0NEc5(`&)Ic*mFjko#m_$}8^FNkCoae`r4B
zY~b$``oMV)rVm6fnj(ghRzPCtIN$O)hCYy*^M@hpA6^Kf?&Sl?NneHB)d6DH^z~RW
zJo~eQJ<o54&YBtEI_=VANP86|55iSj6yfghPlBu+UAqW!Ue#9h>86fJ%eI#*K=wHQ
z*uM}QpCEl8yt(H4&QB|}!Rh}DN8_%VZyw<A1IdA~_Y%nZ{gf$?yGKD{Fg&?D4y)f_
z@*sL+u{GB8FTF|(Y>#jCEfn{F^ay4k_CKik*;^`!>h4U<{b-RIb{DMnPTnVQT!Pes
z@U7K|HeDF|Vz9Zo9*}#SxxF%W7BH<c7c7F@nX>f<>iExbF4vs`b0A|yrwZeCG)?^q
z4@;2Qe5T95c76U@f#KIiNl^?jkRD{5z5unnfyupCq6@ZX_J^O~Gz}96(WdSY_kh^j
zzp_~cuUcwRC9=WdBtPVA_vJjO=|7$kQHQ8C-2j`b7VrV=?niElyZ*4OvN*XOlE>p!
zEWq+0vq4yWuj@{84oF!kc=*mvA-_&=_<`g=xc@3-|AVY0W?9FVikPPYsl|pDZ}LMi
z52nVkB@yfn?W(z8cf!O$^nWYF{LgOPww;qMf8P=Iz-x!9E@Un4^FN6G5lFpo%`Uhd
z8h0S~sqTBW5ga$~!r692zGvHcvfvCj?O4x8oyT4oF=OZFvva|5kSM!&=QN2e;5-I0
z7lifKK+erDy^30Ig5)^T2+#j=9K}BV9bPF1F5^t}5qCF$%mm^2XCQSqh^=JmW_fkT
zGYfh99~N%4)4}EK%I}ad8IU{(AM&~h_A7{;VGlX?*WmUvaJUvN(B9Q~g2(bykOkO}
z0rGdi;SN#@!W()mcfS7zx&K!`?%j6F2-Nn^<uj1A)?)WmQS^cIZkYtxQwU<?!Zm&j
z;BZHm?>nytcJq}=%>BV<ej~<UKSb`>am0mjry}=ai&gK=f%RP1hLq(nGw)wQ?4hx_
ze-W%^Mu-mBzdIM7v#42JvHfxyB>i{YVny-C#nh`iI_(dF-MHXD<IXKhA>|UxZ1=*K
zU^&-k2jFQ8Cbx1QqMZ&CL#G8KFxL%%)F>vL1IND@-!iZ}L1Hl6l936v6U6_$Rn3w^
z>Hc=+nt&Y#g$=-Jk3Cia(;#^e=CDQFP1C0aIg`NN0&+*tikj|icTNgfY?}bNtG@gk
z>iqYe-1j>~FV=(Y%bc#UGeh((O85x1LC&15H-M}w0oez`)t;#5u*1ZW>3Ke=^NldM
zxq0is_5?&j=H6lAAUfc25Zr%O&dxiWIVbH1es;rRZxQ4yuJ<t<VE2O5g78^S$QT8P
zU3w2vzrV<bv>}`4a_qdmX|?5{Gv(m?pqhhv9(|tSvz?-Lkn?kwOTXSO@))s}24r4*
z+cvOW>scWEb&wbg%YFO;cPC5?o!-w0>F1)0{}X`Z)2h>my?Zb<cfRidyA{M|y`yin
zDrd6!F@ZFTf8mfZQ#L-xnmmv^2<L`F*1Un(E88Gy?%#lv%_%`IcJ`D>StP84o@<bd
zSStWhA6W8fr$Pc`Y~=Xit2>pRZ9z%rlbIl64R&gfab=Jm7~Zyo6Wnfq@v+f&W}}8T
zOwE?<+Te25j0G~M0}}_)exDbE!yLpG+wpq)e}(Gpr7gECE~`S$zn@_TDF;CEApCyb
zN_ad3M$QJipIt2-9EJ|9dw1HD>uxo5hn(Xu^Na(yd;sYK;lS*k9rJ|tfz|$PGuXcK
zkP<jug5*GW^IpjPR0+AO!1)y<2E%7^I^phwiJ{Y87F)q(HoEvHCdj@pLsrZ-l>bRY
zzM84<!ECv{$<C*j6f8C6A!qX!LGSj4nW?-Cv8P?(*<7&uub;{VhlyYn^G?RNM$6N6
zkanDC5M+D^rk5?Ed&fh@qhSB<h+47Zt`TJYC`^9Y;ay-quD=6WuK^PW(HVCkcld(X
zxbTL%sOv3Y>Xyu2362NnEXcZCm^g^8<y8aQ31XL+&)#vw{P+%sw<+dRxsAYmiIWPa
zz<vYCgD~I0#b7^!*dGmDz~<(*LDp-0+A-bS!CrOSTP_W7{7FCR1=Aq4Alx-`#f}pT
zAbmRt0fC*;7f{=4hK`Uu{_^>#^8z5fJESnjE<obQ_)oVuI6WeZf!Nm{K;{!1K0wYX
z1c|}$pBBhHpCEqs52<ZxE7f*pTw7_W^cAuP$0HVUmJLWAgg<_T+-(SAU;6-=3;)Cn
zsqb`nKku9;B5&Df0@+i#XwD22_jH}vxl>mWvKIAM(rWYW&8XvFKSLn%GfjWpQS^cA
zGS7#sn*_0OVeWk^;BkYjX2YV{;QSw*;slRtn7p<Wq6~ie`|6IZp<z2xH!xf3#>9cm
zDm90!?SZLh5k&0iI~xTl|7+ABXOmoR@!0w7?=1@!YsfzH<2k6~yQ2TLTL^eU#^A)J
zEZS+(`w+#SpG@k&d1K%7t0>|iyAL0J3{H<AHZB}+HyP|^bosS4hrsDD(AW**{Es77
z5PL1|?$)u`9{1itn`ee4+xc9uTb!1n&i_fQLafcr41$b>yl{c+kv^2LW!KH`1(uqr
zkTsrNv$mu7<L>lLW~sl#z~d>ZPANMdKS4dO`Sc3NTwB3=#2u3`zbrZdS&staW22Mi
zTY>$FE~nMG1RO5=6HxD40jXQF5HeQ*Vz0RVY1^||tQMc1cUao~gWM${sRy~+93&6I
z{}~`>K7rU@!monu6OMuG=~{nz`cAh=>v#ODg^VfQTghq#QU}A0bs9T=ctY9+$EHi|
zoV&OWrEEDG4moS&{z24z^C10sF$^ebL1O6mtd17gpXlPZ=Bk71h2!fGcNf9by!3$N
zT@YWaak_<7{29yp@$6P&`R~AbtgKPbo!AzJ=r4=wLe}XQ$2<nt&Hrc0?f#&)#qysW
z<Q`96?|I;S2QnLk?M`ppnX&F6SnYu&pLaYB5JCx`=?@`w{3Aisas#BdC<}5PF^G)|
zhsrz!#|OH+(<aEB#j(2HV7rm!|IUZZ;ef>ci?CR&kPh2%FXG<zzy!$rzvN^#aC!yF
zgYXVD$i0XlwoxslouWS%vL<v}v(HZ6E)I)$R!Dnj`b*UNzt5f3-*w6kvY$lE?8Yv+
z^{Dw{&V^oZ`LrqvaVHDN9uVHe^p9ZvZ{3G^Zvsf~KKov9J$1819V2`$-G;1<1*vng
zHQM?pMs=r(JCoI)8<6`3;<-`xe>b~A&Qb=cH*A^*&flLVT7b)nWch<TEmuCVY?%z%
zQz<(Gaz6-2EePw(^V+y05OM~CPgVZTxlNEYrXV>GZc>4)wQfp7oG}X$2Vt$pv$3x2
z0Lj7dOT7(Xf5P}Mdi7JtIiv^s7K6hVCJv(2cOd3>ZY56L@-n}9=Z?usE!VGw%+tM9
zh1^#MQVYV?dWf|-XT>1vlpjgF1^d7K<J+Ci#=KU$_1}Q=s)8cwIzGe7DcesqLiS+t
z#5V6#pMtv2$IBiv*Sz2X=KXJl(v<|`|E#ey*8HEo5pqwZy$I$WimvI1`~8-DjNkF9
zjd|z6lhPKi?ICv>u3V0KNBxqutH5y%3X63IAm%=dO9lHsb)WLiqEl%WPs1U7RnC6Y
z^DVym^6Z*r3)%lAmuR(fnGov!heyniwdGf5%*Kevt)~$CG(dJ^!}@8BaJM1L&rn$b
zKL6_}&lZgI|98>@co}7WzkKVzxR!0KzH==!86oW=!>77nzroC_>^FzUgFzApIJ{$-
zA^R4EU0Qc$%n-M{_No=^{-)*I!SXP@(a#TVvx<SNabBsLzOBk^7K;4E2uNIbyhIH*
zkp5L4Ofl+ikT^1a&tC_38?t!bJ;=SdS^u75r2pGBkoEW=^=W-8EhaKf+&*Q;+HLuo
z3&G{TN;B%Zv%~@!xET*z^}+14Sjf3O|Bi_6%)7JMJiiUnXI`0!y1#Rx%G@2d`61_x
zHvMJTxwINFmkP2!mVGri--L9=V#MQxCy??8qz)H8ZUH%u6kUF)2ju+6PY)4m7hr0V
z|0B*fPGI)6nD%R)Md{gZmKzfx?eyFisP#^$AmVOrwRMnj?{8-z=jv=Yq_Zo+`h!Jl
z9%PUI9!bdj5X>$C<5xRU+931l;;ZhPr?lMzrxTd`;(L&>xTgY;`($9^AX?|AElOGe
ziJ{|1CBj(Ml(&k5#SI%V@4w!70CB$QkMmErNqXKjKQ=4JGA9#~?yZ(U&US~{Rka9m
z<_U=3Q>X~ePsby6g7XOLC%>I5{Y5NguR->I{Ok?`+Y3?)!fp>%?>KQ7vIk-5T9I8`
zxsZJmAUP1e#R!>mx)zRlZZ=5H{|4gTF%TacUbP9eyoIUZkb?Ao^Y?y$=XIFeq%6oj
zR}eo{ZNiT39dZ_x(`>f?n79?3m-hZ&gc3(QsfaTjR;fbv$dpCT1gG1bt?zeSm>|3(
zO?nfUZ<G(|AA`&W<Nc01S4Uog$i?{W*k}qlHw-Mtz`!8?60$yriEkT<ogn!Mj}hlo
zgZS96e-ff^jx6uu2e}uFP4*Tzo{;5Q*mB@v6K-v?{NvVSaj>athv|CA86=$=ko|fv
zJ@rcweU_K^AbTU;<`sa=sChVNM@nDg&NXe2`(JtaAnk6L-eQNNJ1c4-cO@K=+PO3C
z4C;7Q-@*W}Io4Al=e5D~f#`c-gvY--uR!i&1gS^HoRN@xI^!T@Up}&&kS1j99!PBM
zqZMYo=2|<ymG)VF$bpO{i*rHNc7o(V*gp!f4q@AT$XRZgG0EV3P;_g;&X9X+E%%5)
z=7ZR+A>)o9eIR^m{q!BquOaJcmqm2%tl^YKDd%MR3c=;6r}SPFagg5KX^=CVKx|xi
z$~)Bh1g7pY8|0kHonEN-(SYQuUtR~F-2-Cpsp{JCxXOJ;?RS&yrvxDPT&=1S0H;@w
zJP2Q#H4W~zhB^1Z?4ws9dxY<stL%Jqw_~STGNc^)^BZ-4N2%_$9qXUI0Gl^INpk1c
zvxvLbKxQ950hx<(x`bK(8SX{g(*;tC4NEFdMk#M#YVN*u0rxY+65KHI)g)EK`lnX$
zIXf1F3h#Q@Io<Na5y&~^mpmY6`NPagaf6)s3gRETqzg7T$qjPfNfVRJu2oa5tSqi&
zg8LaW!co^FIYvnBdg2-fmebw6bf*|Q>iz)v63F?d^9!`V=?P>93|}mQ)Eh89Hrgu|
z5+CStJTbq(W!dT7xftf24EzaBb09r6%WSu1tbDnBgPgIYrz@mPlM)F9=Sh$}2zRCQ
zz{Ai5I{$Y%ax2)3)D?_7cYk2Fe0U-O9LIj2nc-?ddfC=KGfh*4tS4Yxld;1|;~Lmq
zAUP0T8wxoO({MiGj7^X@2><SxfKhgX#F26It9cmy1Ie%6aUAT9j|(CFAdnag*VaJp
z90&3DP2)6sxLR%J4zCK!L$Z*39y(hV9A6-L5MI%TxMRGn{WCbd<#9m971P+J?|d)S
zXgT@mVsMz9da4f2A0T}oydwAQwrxg`^*>q#D|XmaO$ECPBnQIF7$JSJJI0W)Cy*En
zpL@#&4r3S}8!es)xsMNB?wIOHusi;R%mTX$S$<NhCpfM^VnSB3miF(R&D>5qZ0CsH
z1aALj{tiLW)2ahG^97{-t2?AGXtk*t95);Dmhbq!!`4FR4rCwN%IYKF@Byg>;k0_b
zU5Bb6?WtKWKJ3W%V+6YkBnQHUD<JEa8I2KZGC|@XytCOI#T_6qbUa-Xb-o=YU+xV#
zXRb%a5F-q?twP+1s3T=<d40!;?ZpesEcg3C=J4x(K18wiAn#$go6Rml<}cp7hty3S
zc6_@Q@4UHPPXMyVf0sUFeE`fIS+Og3@N_}OioRt9ZohC4vG)L^X4xOeJ|fOF(^1j|
zNG|#<WIYOqjSD|6HAD$-m^%3%kn>b;uYt@{z{ElH19wPW1!7BUov_SfZLpa9+t<>n
zj}h#Ky&7&P>EGxAq%H-imn*pmwr|C{<=`;-#3#DTXVZF%K0nC5t{!L9{Y?|t*X-P|
z&<L!SMO0>&);q}gfgrO$*#9WxEC!8hnB&-2(lPt4AU()f_i8#kZji;-oQI6l#{8TG
zPwO!GL(35J`)%6HJ0}%9+;Q!PiAAA2WbNwRxu|Qj_h&%v`~>MY_zW56ydVi#BmM5Y
z=FZ*e0z2h*-vj4|4dRe|2T}{dCz(2S{$B;@<EVX;-udcX95^gMav)qQ1{t4s|7?Ka
z54%I!C~86K(edubka`|n{II|sa9(z?NJS}cLF#mRAY~4So%%A&qI8zFW&Gt+mdn;`
z0GFklYao6E$%F9xc*L0%0nugPG{Wr&nYY(-)!TJc{inqv7RX%%s~$k=0gyfrZaS*9
zGjO#xI1XI;nzsM-LT%F;%R&1MMqS|W1L=X`C)>^u9RE@Ge1f%)Y&aP*uYM*P^9->d
zImlWikUd{)7<an$GVNrHGu*1u%?S4Yr8khiJxCse*L6YWyg_VPm4{$8Po*Wnaib)=
z)S~#}E%Tz^kbaD^2V_4hNG%Aja}2gfuzCpg+kM8OotxIpwFHU7u#UoHaM%=`y@DbR
zl21*Bv`0W}T)1)%WS=*>e8&#RTIQgaka1~bd5Lw1HsMDmCrg9<Uv}(hbKh}!;cal5
z(dkFDk3ssjl?KAgP<Cs`y&98ldV>8QmN#d|dJ8_wyCJS%{-v{!JH$ZxK)7@MtQ~KI
zTfq8H?-bt|+%FDpKY`>xc=KaOyYb*z%<<o<-H3f^Ahp==>o<`47+vnjGsrr-{!Ylb
zWXST}j)=LoO2#si{>}Gy%*uXcDI5;jEAq?^(hh*>v0qvSF7rVABzH((SZ;$q*uQ&t
zwRRl|&af!6bp!Jm>JV#%K<Z;mdUpQwJr9<P4A-*gC`BFr6#D`hpV2Pk#)yBZqo`*o
zfXqY3&TpLHZbO#)(+p{cSFXh@|4a5A1iJ^MKDWBha!M?RB@@F^OWPty+r*G-H(Va1
zhIt)gOt-@l(y#u|0y)R+ua)mEW9=M^*Orho!~UBrL-B`GnC`AayCL&!s!I7gUDq7|
zrv;F?AiQY*Nw8gsD^Tyv0Leu!&_bz)Kw{{aMXCtJKA3!w#uft*`|DmUjPj3-`3<-X
z1IZg5mo)Qewz9Zzc;@!mYu1AEskuMu7=@f46F6Uh)C=m!gY{OfgzR@vyLV!zH(RI0
zB2FoAK7M72dS^*;{nzb0t6aeTDBQbq+uwTB`*8(JAnOzSO%ZEsK=y#Jh&yBs2gJsO
z59l(3$2rmEZ&yRo*_On)VD}=+`&>qhi3RUpvV}dsYv-CZ*_M~CKL`7vB@y-hN8!(i
z_;>cv1&8;MdyqBcPxvS7jErlx+;|(ZA7<+#)crl48Xeo2lOcO>8=jl(xN;r!KE@kA
zG{Ny7@d9%k??QwEhTD<-0b)Oy-i9^(S6_sjoxJFkDA-RRwJ^-J*#j=N=MMAEUwmtJ
zl-}(!f8Bc@9M8MuA!9r+^&U-Ka658}AomrfKY^TEzfSRmxza%a^9=?^!Fn?qtKn)v
z`hVR$zhg=c<X)YH_fGF{<>*F9|2vH!`zbEI#_a$6nF2Xe1Y`~_ylxW0ZOCdC2lIpD
zX+9fd?h#q8^ey5{2(IO4Ex+b@?c6)vX2;y=U%=^K^)2L1W|$tgnTWRe^l6vDakB9F
zZ7@Ca&?57X1v9qCt+N69>+TKIaZHZ*{+*KYBH;9IRcx}$x;_N#E|}Q|-$Kd}gKah_
z;Rcd_Q3Tn;4`SoO2d|*cr@_?qYf6CasmjGXA8qe62XLALsjXiuWKn3ac1L8n_4ZZ%
z8Q{8|<K2A}w?uwMoUyJP1}XPfXhZsczgqftNaqFY%yEaTGmFu@4h|QP*&zINTI<fD
zd)?r0H4G2kS+&gr#cb7QkoAIR1Q2`7LHa=WY#n4z35bmgcZ)lL-Ha~3>L+B)!t8h+
zaM&Zu_eMhcBOtL_-e)@*H}2la9sABgz!Wl1)5HfU+d%Rl?D!!79F`z9*9*w~?Zx&%
zU~}hmNbXv;$<K1<BvCM5=qNi%{eIb7ZkK(BJXr2@r`WDj4Ulu$KxTn(js|4k{MuEh
z;vl)%uL%1;$pLl0B}g9#zww8xv%KJU8J^cc;vjry(HVFgy>nb{8TqYl$B|u4J5J8`
z0LRl?6~ufuNWE$;;%o{j&5hvrE4=j)Y{u_9KHC|x-t2hREdl1Q=X(a02k8Uhd-I%k
zh9zZy)7viQ_?>?kW8h&4mbZb7qpdrSdDiD*XULhFAa%HK?~NBI=E2nMvxMAbd+Hoy
z?-fiOME__%1aC8M=IOV*b0OGV?P|J3X!Uz=S_%||+-(R_3&L_ii1K1h5aj;Ly6t`7
zI86zx-6?W@j)mKLDKOvdIO;jhOYd9n(%lSM^I&^y>Nb^m?I`JEb3A0-@a@-AG5o<U
z1es$1xd9iJJPNr#2VMSWHe@{Y_yZxZyO8B;IuLuNR^~HV{%2Tc(b#+5(q%8?Uf?gA
zQ0M<={zc3&^jUrYyYtr@$ew=gqKmu2ZI4=hUjx}Q!M$cXI32+3d#rtS$M3?$U^jG$
zZrgQtZU9R9Sh*O|$DOqqGOhyC2cl<4qplSMiJ{}3@cUr*po?2nLe9T^BL&&hj4aO^
z3n_C!Vr$EUc3wXCbjP&s+sxbUK-P@z+xr8YUP1C8?0pxp|7Gb_$hn6m>mloXOC}iY
zRG<0UqQC1AIG??$ftUf(2f~^Lr*^dd+y*v}ahmq7j$Q32{y)y_0IoM4*+IsEKzd-<
z?Gox(F-#npZrTNDk0OhK*!f*s!S*PzL&`Le7z}&pBF?{bDY&|0|7qWy(#u^f<*Fd}
zAJp=o&VPB&?1IPX)*Z=UHlzMJu%Cqgi0!&`XpW^EGo+1Qbrtm-gKg8F?5JwF2UdIQ
znc%L==BVrcH@89B8r}|R81d*P8VU9T%-z`NJ|;N~eIRwJ@oM0F_Ih?VIPHPNU|4gv
z8#t~&{P)i`?eHiu*eUc?%tBq211#>NfO=;fp9>;StDB^Q<EuBX5!|lsdKO|Xvd(O$
z$0W%42T~5G>&^GyTeb6vHsm}htv77D>s<U%{A8j5$>Y6!nD@Wsynhe23*=5*__(4M
z*q`X~sS%KMA+vHJ^EJrwn`eFk`x7LVe(>OqHJlYYDzr1VmOO==#kb-S>KXlf8zAF?
zAoVLUuYlce7tsoi(-Kp^op*MdSgulqtbdM$&O3tCf^fOC>CUg_kn&Ha_Suej^Q$O+
zlC6fEIlXV;8nC$_Juv*wjPUvYQ+>~Y>nxZa7=3WBEx0|ZdL40hD@^Y6;p1@s`{pjQ
zyts6w1^>si?V*Pu>jL>(QSUH19rh5M=0W;<tS*Ah4Xp?On~_~=w7r?rWIN+e$T*hJ
z1=RCK^Pd~<lu+3PcBAWr=IuKgQP<WT_zk%mCvGk3x)6|E7vG|;6$Oc*<H?^O=cl2I
z-|c{$WzOHF0gfkR`5E?zG9!NP8H+$4{+)c@p4(mS?*qq;Krv)&52ojn2IR~i5T8BV
z7M$OhHt>PXox-$rdtG+GwzoSVdk5IRc!K3YYC(9-(d3;b>mg~SW(nucZJQNP)2lq>
z{?F^4kh}-d1H)#ukhvik9~&*aN&&?_n3|Pwko%P698vr7Ao;nukn{**ClxF0Jo`+4
zXTqADW?b7Kd$;Y-?%m^27K6K4<Zv@Mo#v!K*1yFGecP5N*|0NC6mq7_mb1LzZ~>VO
z!V0Gj?NDogoTuva*LauJ0@U*{Wfnu$5#={-0GkWa1H*qzoxo`g#>Yl`_8-UUf0J&=
zc#7kde<<z&nQ>($VobPWx9g6tnNN0@_f9b7@`s$2aF!i%4>m|G2=mny!rkVT1j&;M
z6ZfwJ$yF=g-QpuYZ--Dn<lOs{g+Ec;oSQAZ)1M!*$Lr}KlU>{6Q1@%j`3>oxeOQFq
zPj@?sIHwL|7dE_553(NtUGCsz$e#Te^+1gB-%bWH9|h855p%>+AtliwYrg*u^{P^E
z`!9Uw8gM=U$%Am#7DV}<;{)lZf4UJ4R{QXwrg@wA)SVtzlEC5ot`yQf2I&Lg|Hfjw
zZe>H}f2XY9w=2K~(e?nzOMXxVyKlyR$hk@&F&MUyfb3z1@v+h6LSgW<i7YRr3%O^+
zJpr-S2_|oH7*YSPyY<(yblR~UOgAT+TV8{-UD(?o_qD^+S3KMTkAMAGE^yd{FhbV<
zTQ~~uh+Z{s$D(hZVDYUw^WkMBNPpSf1Lh`Q`oLjx?w<IrDb9<*;Rcce;qr%&_V3B7
zkaNXBVlaHo7jf=3Obngg_Y1S$0IA{lz66{v`rjV~m(w6I82+n=I9K(=!aZhc6PE5^
zaQtg7*u?=Zr?;MfjGe&Lo6JR=>BO4?Ip;E>$sFv@>uej$A|`k5{Q4OZ9+wtogT-Nb
zZDloga`r>!@t5*W+u8pUah3u|O?Pkx*v{jwnC-vVq-Pjz2kAk^4aYF|je_JG#323D
zHC{)+=?f$V!%AV0wOSzl#m`rE*d-|Jvdy?<Uc?S57cQ@coL2^t2Vv3UE#R;Ou@k;S
z>cZ+~$UO&KXIJgG_G6m)qf$u!XU+22;B){|3&L#QzwgY^=L4@NcI>L&EpP|2ei$SN
z!t?bY^C->}P}j|X<YE=E?`;Li!Eok0NIMV4htaPkpM%rci+03VA56}C5@fA5i0?N0
z>h`Cv`F2bx`(|GDk{|5$tvQJG5g;|&T2F!F2*lp513BO1c|T;WoG@GGj#~X`7QVHR
z`=4KTq3*Xg-t=!fLjYvGz5hh*ou2m~{bZ0?AS^9D1?<j6suA!!1`-Eh7WPvpX$d5T
zj=y_A&Z9*aXWtBIPdxvLdHz+e1Y`^sq$g~y<BpHki*|lKAhW%7tpYf`nM9zTe|0S%
zQkR0%H(!9<*I^|42<*1$f&x1(m|ilUc@DCsVAeFqI#7^W5MC;CeuvrxJ#ZTQ8mhHx
z_6*3l97qm?qxm3n4sW+j1&0Gj42HS(-T~VI<71<Jx-Vmd8%SN(rEB0cd;TZn?k$iQ
z41cKj4fZF9uOV$_;Z|p|Bh_QS+13I9aQxr?G7T;dQls|U8Y~WCueO5B<r%XxfybRs
zT{AbIld8O9!a2zJ*Y8K0P~7|||NhQtM<IJ<I#+P*me2hIHw$DQza(TG_0=_)`&fP(
z8-V=)Qiltx@z`K3|GiWp?J&Vm%<~UV)<D*yfy}H;pS?ruY28lF6UVl1Y=O)Ro?M7}
z{>`3sOX2QFiiViGh|?MD&+8I$JD4@EZvXRm2RO|7Gf~HPwgflq>^ArdF8}=&Zrkbi
z1oiy;O-+zJ=V9uQvv)vlg5ihDQO6-+;>h#_=0dpJkj0k^LGJ&}`1uJXyg_Q7col=)
z3Sz(M;I=Ay(q~at!eW`|Ee%%tvcnjhUP1C89ANMop2ql(L)Nb)owx(`Q~pVTU3IN;
zmVXr>{h!rOAm{FY^nq~5{|!5LvP1S$eKMT5_1pF^6!-S+t_7RZXa_MDqz8uQ-a?E!
z!^F_(N6o1FO=0qrcS7z3`?d#j4Y_ItqI{dUrpfY0P}dHzuL0X%tn3E2-BeT|>i}V9
zWn_lJ-JCFG5~SZ-v}GemeoN@HZT3|%JExz4%qeMXQv|0AkQ@l}h}!N{Ds=>h_p+3!
zJ169!uK#){4LKX6SFQ*i#vpwl94PS_tPjM-h1LH^V@?03*mc3~^z*olVa7rs$l73#
znV$;xSgcI;-7zQYp2fn&*5GkLjZf22%A#W-h_jjRU4xv{7I*-1|CvYnf*r3t4sBNw
zd=0ke>=o2|4AmQV?(A;%1Ix`Y6xgzN2WopgRveNRZ&lZT(;mo8Fs!@+wI2!-N2b@O
zWuk;POirqU1zcyPIOm}F1thN~iCF(3vV4_gqHLGNABNfH2i+j$ZPD8EVE2O5g7Abl
zh%+YV_AUe4H}Q!v*u8msjkf2lS+_%M0_04ci_Y3$d5}I3Zi|)LvF9e_j&>cr&>c^<
zLheEZ$$_wR3AAq%g?aw%)q8};zXRnFchi8(gW;ku$UX$|hEL#dfQf_XW6F?r4v0PN
z{2GfBUP?PWzP#M3XgLAwc8v&kFb$Fi;dLF5yaZxTFoWz3h?It$l^(@bXy#JEy5o@-
z7dRiQ%>IB9-nAk3b_nxB&bSxucHYVPO%Ww*4VWPHMq(7g4v;+{?8yx|8yUpLg$2GC
z!}BJx8qEh<V0#K1A!lzR%W0ih0S}YdLRZT*`agDT6I-?;o(-~A@US-Oy<-zncfs?f
z;O~WC_Q8P1;4oU4{Lidn?$Pa&T71Fceg=rS5|Eu29@p&5kcW)%u42vJ>5a6P5+u*N
z2eRgJW(zl1KS&IQyTwu062ruiX@ynT=f7MyAnPBh10ZYfkoBxP4moodB(}*r!s?0B
zar3XLi5AMs_`z{kdt?>3>;TDw@bqBB{jXg`kalXdIpn^Z&T9qRZz;O(m@q#O9H!+)
zsQZ{Svod!*Vz~~MbNjx0XWg6WDDky#9;B{u6IVhJ2ibc&5p}INNDLjfEw#d0{#iZZ
z2FFkAa?J6+H@=8-wezCR?3ixpxm~k)%l3OW*MQyG--oy#9%P?`*8y-^0<q({A^o&v
zw;^-di+^PA(A%}j{81m|oG~vq$hZ?oEeKCI_<e`D?OJfSZei`(w&f#a`~)Ni!r$Z|
z=blQv`U-Y0NDPM8)&;`tg^8il61@w+<q*2K`C~{sQ>QBt!;H1h5aSAebhnt*A5Px6
z`~|alxjv-Koc;p!{`X8j$ekA;JKo7a`neIUKf&&8J-%xDZ+=mWvaqe-u-jVzxkDbL
z7KA_N1npSjC<$&){C^;{>(D9Gb$&aKL)O2)Y=zvX1JVP-#~(rFFkpOa^uicOo<)~?
z^G5-kpC05EV)*~8;Uc)0W|E*~vDNc!a|AY;%N@7_j;93@sbCsrX0rog{m(Q<ac~{E
z$R=Y0Nd8`<zv<M;Av+H7-vaAd_!PD6aHaC&4#wxn;IIj9;M}RY19fj5k1b?uKu`^F
z#umt25Y`CZgEekJaxh%1g?;~9T_0p!DE~dk-gB6G5N)giS*r<RFTUGh{!Z)Mj&r9>
zw_E5z_Ww;@3K^3J$%F6>b;OwtW%nU%8G%`lJ5djHChVBHV76KGLdcrooHdo;bOF)_
z!un5^?=<L!td%P~t-LdBB5M2Ss{4Mh`k&`9&;K#zM%>d2G7B5_T#q?#0#f(u9^~v`
z`$WjSbRaPpHaUz~|5~-W*);ml-0d^Z?BBl1dj&WS-^_3Z*JUvEMRJI>ISzr!;Bv}w
z%|);p)AH_};(OO`yLunej=kZ6T8=$_VYV|y%^sZIw%L2{T&#jR&Q-7oa>nhdNto;Z
zT$&*JMquv1Mt8XfgToJ9ZudIKywKcO$htgadHp#%z<C2CCVuSQj&&d3?$9-`wt86(
zsoNjzTM70XNFIdO)mMQ13}P$T3xLbpMSmgViXIDB@7ld`uEl3v$e7vMW03n1L25x*
zjBCQqGuk}h@ekoB*<DRxf8l0<<Zp}igTqGI{yd5}NN#@_;q!0oWeebb0qKF^>)DVw
z-q*L+qxc0R|K>a-uY%ZnBBXbow#wZhC$N4;#xg5#e%rqvGL{XJ2Vt#$h&#q!&nf_`
z@z;Qy3E-fpy5q=JrkxTq<-qZ9CKGiYfBg!zo#uv+aJ_62wA17;<lZEZxgcD`0NLYV
z9FKYb;{!X$nsSgjTv*{E_Ws{Rj+@{z=UfY<?}e=2b5|BT%uJoOT3(cRW>GhBmgQOg
zli>8<za|At!}KhfG#joyyxa^N{|BNoz-9VWp1NKCL~dG4w1lJ?$uDT<xU#G5x>vgd
ztaectxA}bIV6c9e+51mI)+ZcOfvnetiGyfnX^5F1HZE+`Sq)A<=<=N{Kf&p#`?nm}
zZe)4iDUf-7kXY}h6_ziwF6`JGFvZg1lomJ+-+tu<r&o|X2p{Z3+|5*Bxf^WmoOKdl
zbC=Jm-X*c!!SaF8c5ph2(TDT{K>9#9!dP+V%ZN&_TGo44c6nu@o>w}-5;7KjVh`py
zUd}%e_wInq!iJ-&lu*JGrsli?q&_*mI1eLynu8$w+(Bv|*l_P)ELpL`ljo<!@=H^}
zY5T3G3)n9pc@XC8wSvdLhS4N&7=1{B>{~egpLv(JS+phVf5<xiJ0Iu4)q?bTm95==
z;S*%Psn!Y3U8PIjg7t&sK-hdY<i7Yz9hmDlSG6P3DM&3g9OQ*rZ-CS(hUbFaQF=og
zB`t&G{(kj^`!C0-#yn?-_V!=VPb@{JL+0@IRYC5n0jUMygZ_}b4q|g?LiSa-8@~aY
zk#VbK*S+I&Ex8sHgZZc4A?^wWsXyE>*Zf1K4><lMGnjVu2cour1W!QbgWU{vfYTL7
z4-7YiEkFq?m^d;$^Vv4AKas^i?EA~#fXx$~iaM7MlB<$G2u~B8|Fta+#_ihi>ZrmF
z;U38Szu)da=Bq$zL3rD`yI`|G>=kX0vjICRA@}MeR^K+CwTy44VKC&}kBnC@P{Lbz
z=Y^e_H>QKbrvAFaE`c48`BRWtAe?R@4KAmSq@&)80g{t-!E6(P#F6pT>2g@pf6omq
zu(@29QTHQ&^yUO0@@Vmir(5Iq_ibPQCf~BLWZn*t{0U3Q8K)pN2v2zmx!V-PHv9mI
z-{{Cfu%F&nJ=&QtLElon19BfmQ6cI)#vIS+9U<0``_w*5MepdcMjihVlY9iubJm+T
zW5lEF9K^nRklonu<%`(Q2vuIw1onU6bV&OeS?{yYki7~Zv3ZI!Ep1o5HD?Xt+L8Vk
z65eN6P}{H*lp%YRLFyI%Oa_}<5ReJ(Bi24C-*Lc3cl*od2f^{4G!t=FBuKsPi-w&V
zuWo|(aqMI)+A3Cu+CGe6gY2U+yA0V^0n!7*pJNE0e?3bN`}zM$6ZOG)hg}r0*BNGR
z=|Myp^`(E}jz9SYJ6P7(S?C3Xfa5}}A9c*IE*q&%Z-SiveLfp<hFWr2&d!>|Jr<o&
z*T88fB<(*+JcgEs@9fiq%z3uNs_qn(GXuK|<`%Co$lj=Z7a(J=FmVw5emUf92oM_=
zw*QN{ZWg50&aTc9M4yeoJpa~T8FE)ANIuHx$__ahy&Vr_c3UX^g6tU(`vEzVA0!XL
zm8lZ&JZ9?%nZNr}1=%C<B#nIsci_gIT5lltIDWW&1e^{)`asw?XP&w84oKTLFT!!x
zIwfy#`T@y-aMLcxzWLviQOC_ea-X$h!SMlN<HD0(pq3{vbzDc7!T$H(1UY*aCJv%k
zv{=L4@cv@*j;f@N9ov@e-hOf)C)mC<#*j5xAhjTDT($=ur^h)V^Z$y;kh0m#-@-I+
zOVf^S$%$Y!@p`XO{J+%6bf@cH$XStf4;SrR`gR#u9mre|HlGRE8|JVBa;_ps42Iv$
zK%7$q6GNxNb2fnei7x&i88T;KItlX(xib}zGj~CH?*4yg=@&6=d(^!7+a5gr1RnoB
zlJyTvgXBT@%6-UMN)X$2k~lb>D@;-X)8A~SnhUeF?KJa;-1~jQ3bO73q!xsqWxDVB
zy}S#oHv6Rfu9-Y9z+nuM17WWm$lMU4F6y`tNRIK=C9Hk}$-!{T)f#x(go%OZ!g|Pg
z^xNiO-k~vV<wkJX4N`Y`XQpNBW2v1-&YrLk5`>J+Y`SX;uCGDzAbc)a1@4Zwry%=0
z7S}@d?L1I=xI<bcYDbMHWR1U{Eb2W@_b%?)A-Ers{!fVU?rP(M>~{v43&K<GL&Ed#
z(Z49^0wg!pnhP9mAT}=CoCWE#qsx2NW`f;(p*#hi)?sRXE<>EFCt$6#gX!oebH+7F
z+xaB-fy?xtZ&Ok1?MOtNO_s6)GOm2)EaXnG8#D7aFTNtYbMuKE;BxKlIfy*WE@O+%
zonhgSIpcdT_w4w#05O&dQX_X7GRC#h4l%|E5(nYFWXPF<AT}<%T^hB%fvGE=0BIL{
z{DQ2@g^7b`_F0g<+93A(cXQ2)cE8-I?NDj)dIDr$NJIK=l=#p40Wk}t{z)n1yg_pn
z$h^w^gDyL7dIwp`Hp+taIDSE#Edx@&G+%3%`kd2Xxixp*?Apz~791}iIS^jA8?ruk
zRm&t4agf~A8q~X_Kw{|luaPa*^k4rAvWD@(xpI{J2GT#L6X8ZPA8)hCd%CvGU;Eu6
zi%AR|@7FfDgX?RMS`b!AgS3l5Z2zq#;PS2B0MaJ<bwGcoWZZWP^EZ%w*fl;TlyuBD
z`JQRW$9k~2DW^~E;C#{yPA4F<KzLXAHSl=azkbNLI!FwLZM7l$kzssn^qM7nDCWV`
zl!QRe8M>~FxrXj*MGn|KAhnZtmh5n5mfmT!qtQY*(gJK&?k~uhv><s9?pcjkW5<&V
z+2<=*&kD|0yLNot;s4svVooaLJnVz}P{*4mHkj<x%7?5$cT<+wWz&y1O9o{2{S}b0
zv3I{v&rJu(MK#o;ge6D}9ot_+oK1!-_eE(QI6bYjLZo$={FQ@VaQ~|wkF?wnx@<@D
z)7jfj3MPZoO4+xYaCwj(X*<YR5{T_}0#aY;zn2874WHX--o5k8)(uu!V7@Hl32+$#
zQVYV_O+vfwv_ST;URCkh^<p*ZdCe<C+`(nBhyv=l03f|*jzZenAT}<1c)mZ@^uOW+
z<Q$&mf~aFTAag3E9>D!)oF}%EpC@7`vrd+U=srjq=5a;6|9MRlV*gKTvlcj9Ym|P2
z{d-YTWas+QR`b<Ms=)sEU4VLzi=M`@ohwhD0L$fFSg`Zq#Tb<Ei4%jI2ejnbOpLhx
zeH63r4st&-E|Q0&b7V0P`}dbqVDr9LBj&n6;vl@x3bCJ~X(FrDE&)S}%PvfoQ|?{^
zuj%-ustzu%Kx#o)+6yrcb#EGEE#$Kt$hgujhYvfO-UwLAZib{8K}W=136Ner-HSWZ
zJ0WMh1Rh+vbLojQV0VG!KzKpoLU7n@-*6h7_dsGWY{idSmcYc3X|v;s;P^lm1F=_s
zgWTDE#29kF0!R#oqhb-`KVD%IOrOtX-oA3`j~&N@Ex~#E+NDNt83$9p^95r3^T-0o
z+M&0zAmQG#^TD>gtAutQS1$+q%QYb!#UF7$cJHWp0LdFC)-Twh8HUuCU|?WKTm{+x
zcsUy~4+OIlMBBGP)`o%DxUlWPLs<PU`*{l39p~Ip_Zx!D@w<iC6F)<^*h0so$-<eN
z-*WyRA8@;mbwWCd{}*;V0h<NVt3CyCc9xJF<h;WZS<XAB+sIqWyobzj+Zz1=hXY70
z2&>+mxHI!RB#zlQO}2|}K|NdRM+{_-<M&KiaM*+Nz_7&y$apM_kBydlc?j%gbUAC~
zt6=l&BrD;09i}EK12O)^mvzL_B4&ZbZH^n;_xP}Z-QJgk*dGAW^TxRm?*EckkiC5B
zsvp7m-><l3$6>Lz+gIk*g6;Vff_i6}VxQ}dd!L_z-6Hh3Y=_$q)HR*5Qx<{K|FZ_f
z8bgqsAe?jp^&D!D7&@;1gnj>~NV7lKeTfW+xl5Sd1*^=!c7phq!q!>7TDWyv-EyfN
z+B>g-!&1Hr_5SPkfkp82(6SUVf8RF|l5S=0v}~`oX0co>(GOPh)C_VyEXZsSKK>+c
zM-dyOZ@+Hk^&JP!@}Z>H$|sO^<eUu1d?iQ^3{NwKw2fhWZ1kzb^WZo`my3^q=v|j|
zAMRI}nq$Q)!SMm&Zwd9?dDl33XVwGZooVrqHSKCC>EO5o$%C-TFHX3dJ1Sbi?72T6
z^SvvS7<PJ_*;;7HLe77kJNq_@d-ibL-??X*9oW20f`4{gW<@=ZA)AK{EN8L^b>0+Y
zZ|Gij47Y>Ck#XC{eJJ+9<kW9`1E<;f5|HyXVB#Qp-WJF?e<1eVSamDjk2}oQI`7{R
zb(9<27Mpq6AFLN755k9a5$mw!R^I~4<xj8!r!RK)AEq9m4d!fDAoqIRQAC}On0YLB
z*Ym|eV70s0ig%n2LtW=_&5{pX*Zk?k-0NlOj+$3MZb8SBzCq@r(8aIx^@7u~*9*w_
z0<!#Vy`}K@@B3=I{lIgM9dSQ3ElO1<g7stvp9lL9rl;+yDmacn{7>f4^-qf+XH$Bl
zPTG0#@`LR~pCJ1lx||bG;^WE|#vRxHZUV>SbyH#UEj(V}^aC;rgtvcz)O&}&V6OjE
z<!T0p8%P~4%wDL0RZVmY<Sx6w)ob8xhnev^2ywpivel7hd7Lskedcs*dzk|{=Us0#
z7dTE}>hGLG?9Iw6fUFJgP(BQ{*URJ24(W0p%gs+9<Ds#um{II|uW@}xo$xoX|K|xG
z-FfjIVlD|}{*|d~!RnVQVeaGq&|HNQmLT=$_~~rK*bK7VoJUIFIOLj)80&_~f2?@|
zb~}jQepS@6XzCO58?m)JvX~sfW!U!>sORoJS_C;u45a?UBFK5u0S%CQjHm9}x2<~T
zk?oB))`RuVo&mYf45Su>d3N33@nY^FV-VZ0jA8rk`{5{l(p>{ti@M<OcW~MRse@sO
zDK2n#!o<+&g8kUrKQmuK+N>{4AnUS`_0>Fv+(!Zuvs$-y$H_X?ou)HoEd4Tiz+raw
zC*({DkUR*lO+l<7DgFl8SFmgHA+Xz)d_TN1>!`lPk8H@E^5-qyDCyf%;Ka@;{Ka6k
zb8c_h@o5iaUpB~G5H^_)xld4fPAS-KkQfYWs6pD8Fg`Zg?Fsh#&!yidfz7*dxd)uS
zkoCrEL&iBlVmBV7nIF8WwL@9?@OFb@9<VzD#WteoiQbMFo3FV8xxc7;lK?o6{JcJC
zXKdWv9Y_9N1E=v9#)!QVAoH((DA>Vh4Vi<#$>FuLmpKEazp`UKWZz^#Ip#TL3y3)X
z?Ro^{&R&pR$k<coGB`bQF2)>J?uhpWyA`C~IQG?!iO)4HS1B7=*)>7xo6eaUV84On
zLHL9vVvJQp0@A)~D!&L;!*~1ME~keREoW|rv@^P$QTsola$(z>E((C>Q`Pf-?esMc
z2D=MnE(pI`16ltlZflGZE?~Jh%sX*G;>h?@yC#ZxFuCK*koDxPTOoZqm^g@LTMxOr
z8pM7fXtDEBcG%93F9j^#GeO#Jccwzls{zS_@T|`f@HDOd@*ueXR{IXJSD+~P<WBRs
zH9MyYLiRXJQuu<Bzi-Gy?L4Y22M+H|9L+mjoFV5OfXoHqb@L(hhU*#BdyznLRokP$
z`ao=4IA41zihVG3zK$v2aOo7{MM=vb`Bg?HaC0mE-LX)c^vW{p$OFso-fCbyQ&>ab
z^&?2VhvEUSIEcNs5wd=v;7SKL{u@6$++}L9-Li=fa>kR>@m+AWAoV*v#ddu;bqK6B
zGiT5C#!sl@-~SjP^Zy6`L(ZNB>49MxMbvp{m^d<hD&!f8TVZmC-f@G|(+gI}-GeZ3
z5S?3uSpQ~_B4c%Az0VHUfD;xQ@2vux<sT{vPOl)fAbd#+QKm(6y$7cSvptag?i<>t
z?rgvH#$ulSb8whWnk@+~cR>0;cz@~IopCoI=YLQ8abXwFderl3k4$_Cj-SQMsOQ3i
z^j`47KDG~%gW-3j*w;UX>p||5e87*nhdOQ+<gOx+8K*hknTM{Lxg+=YG)tQakh5rn
z{ZY%^hx(AQ1d#gwl1AY4_EruuruFB8>n_1l$(9+#6Ttp3IcW|KACOuQZZf%IeqzpT
zaM)b2yts2hz6grEedYvknQQVHa;_ps4-8K$uSE$<m^d=M`!3`@a%3?OTjYQhI4(~l
zJqFVtF&O?Vk68bvo6%;zjL~)H%<0oDf4+d^m8DJ=;PeVpA1#I$|9embxmPslYztV8
z&g{Nj^UbzeI&6aM<I~%^6~%uxXRn%9^hSWydc5Dab4?)X`EN`%(%`T$Pv!)tSD2k3
z`lfvsIBY;{T-f&t>V7JiI&F^IV0TE&+X6NpCJv%w>=E&A&Zf4*@z|9eaqMzC+WtY-
zOfHis0mmgsEeN|wBi5lT6jcYibBzIH{fAz;ghlPP#}=2YAnQNawNdNipvh{x5>p^+
z{O3Q2+xgvA7_1*;E(m9A76rRg>2D;M28qG&>FRI{x5MOM^nPW?JQs`)qbpZ2g4<KO
z&+Y^FX<*_YddgYI*;pX<E2pk4=ecf~9bwS5{B`aQIK1UIpq`o5YOM+{)4n~0oPTYn
z2w5X}V#AZ2?iYSqh}azi>ot3ZI{&}XcJ{UvPatJb16Skr6IWPJ{AtWm1`cbrP}F`g
z$ZeTdTrkouNE{g-cg+ExbA~JiVz=e8f&H&Cc_Y|eATb#3<>&z03E~H|rC0`QeKTJW
zC~U!_s0+?-$JIB2^@8L<_}v}{aGnRTU+;sA$^3qA4HmobqHjlTG@scoZ7Z;zS!*Ek
zKp?dsyw7^=PQU6E;5Z0Ot2KME>^X{`_AQ6Z&ra%woL2zS1H<b)A$bGF$41|+oC0<S
zy4<B(koDA8-ec~oKYqIr9wq@s&vw{Ko!x2}bHU=tQpnkqIf0O~jbUbPFGifhvab-b
zMkFjt25j#<!?Qa#Y^}3cX=Vsct5IjoP~!JPHrLLV`4G7cmuz-ctP?@;|Dh{;z;W@-
z3iaLukln>cOW^JV@v-4Q`Pj#QwuC_L)40oux&AqIHRP-|ka?cWt9ICPNSJ58QP^=Q
z3bNj6qxe#=UqJF8{OmpAZpuS!ko(FK1;oH*+M@}YJ0}-0?2Ij51$Jwh8|pf?d-J#~
zxGzB3JW}@+b_K47+#d}x7la#o4uk#grh-^w1QG|~j?0j{20(0Fm@Q=jINhPkCj~*y
zAv^OFa}P=RbwvF3FdOU?cp9-SeTB)6Blj!7euz;--T!o?9kFI>p5%UT`)|!($i3zE
z3Kl!~Ei8Au@<;^d9j{ZU`@hdfCG3=io<aKNIO`6xL#TbLh)BrXsAO3bI6WczW7#&y
z-O$Kl=<F%Q*!#aFp%1|3P4;X+iA#|Fb%lsI@ip~(EuSw{vsmS`eTR=WWGwa3e$+MU
zd#w?3mUVuR^Lul`>cQrA8ar-dXjR&|(FJl=RZ<&bt{7yO6JO-6XK9c#_^-_F-O(im
znLh!^fw0wM$lUneAE@^NgXHcoL-v$`*tl@nEXa9G=<*9(A#2%A7NMSH3sQIU$OW+5
zLG1hG&n!d@BzG*Cu+}2Ou?$?!bI$w&mj|gSQbg=A44V#_-;28l+3ys^*|1YY-^^l?
z5+olt?Lp0t+N;=idd-HMLBN#ve&^g@ka<&(xgh-9G6S6cUtC44H$ZaLuh)UoBZ!R)
z-`^F7Rn06G$k{2sf-vX5xMUG?5C(Iu?r7kX-{CBfW8wT>7Mz#P%YH{OGqf4ern@~A
za!=hNUC4cRP2aX|pCxCy^MpNQj%&v}L|+Ny=2efxckc3p+~d3V#m60o@1Wk-S!V}1
z|20t<amN5i9|(Vmh3EsZap7QP$hsSJ`A1B8;C6=Xe$=}xK<akwWB`|4AogOP58F3y
znZI++!cCUF?vOZi_+t#02dQb&M&y4U|F7WkU1d5WSk0#EA9wD5YHxLNHDv!khYw<H
z8Az{Sl<zJfcgT3!fraL~{v753y9*=-!kcp-=V?|1VvheCEkvBt15%3(uZw`($Am6d
z;~@#IXC$p_z%;V_=InFu_?>s}rODo7vfF)m^ey>sc7VlCL_+SXhUvNe9I?k{huIHs
zIk$WrWNn6PU;EAt_MH|2;*fK#be^KF$2B{C+Pv!AJ#ZSAOq{mE$qzCv4KusNd^*^D
zYq}77jA3$4{Ayr17#|z0c>;3pCc0d(6lDL$#z4$4v_l&IxKgWSrmvg0<H-J>7S;h~
zV0YF8A;!l+=FZdM0LM9qJ;C}bSdIE;$T=9#Eb?}SXB1eTa)<QsbCe<T4j{E4?AUl^
zNAOL^-k5{u5AHPF135nhBnQIMO0&RzY<h}XE`sD5&SPKO2a<!~uk5Jv4KVSS=OF7z
z&i-19QkH?_7w(=6_5+ChNPdxJj!vgpv!&RMOKOlllh^&{;B)|z2jOLp(!q5Jh+X<>
zAvi3Aogn95b|m_3|M29VMYS-b?ZDyyX&-~sg7B*;M!Sp?A@>dV?|8QTza;8D*99+M
zfX(ha<qD=jdSKXU8f1<U#>Yk*a6tOw=yFHgA?v@R)*#NhfT{7Zgv>dC_#Y-j?65P}
z*xAp+VL5F&<V@QO+cQzZENe<O*v}yK3SN*oLSu1A9w}M2WGCbN7K`>9HDEm-5)gMq
zfz+Sq@7#Hz@(Nh)f#{l@Yi8xa!yY6*Wj5p-sW0J>ekDi@h8O=wj8nkG&}rx6*w257
z`SJ}Mp1DGZyY66mZ+A?DyJ6mk2bL|t_1n!uEG?Sv`~cT~y!l4pvJIwwiU{J4mE!e~
z^QPtHA$8Mq{j!~J_RhADNQ9i#5;7OEKNO~S#k$u!=d(lBzjkl>zq3RSa^D+FzEKr&
zK9{$?6xeQ<IEc>qih2$wNDLi!zKVv&4YFLrF-SaFEP=E=k>xJlfwV_JV!vY+?7S|j
zveQFGWXJE#)!_9H3tdsyD0b^Z=KMhF8I_yB;rdJ(a=##ZXUR^d2*aHkx7on%Kk);x
zRsy7c_3ECTkE$W-IAc?m?~qG{^pinyApAxQvj0o^I_CN(iT(%RbOTa{3$KaO!|MO*
znN48#gnvVn-7qsW|3J=e1Mx+@*>^PTW3<Tq>~0};-T_?yPdmLGTqc3!L3p#-5^$Lh
zVlS|SjPuOh3E2lQx3GBU>n&*(9>$P069V3lbwMDtAS|g@y_5ed<g6H@7c4uRJ7nQz
zf#h{0lfY@w$NVgcI7qJ5Y!z4z#Kwgs`yuD2pvxb8?+*6E*KMa!$}*6;NyR?!a9#c7
zkmbjn^DIIZRajiCd<k|#OAKne`wG(ipEpiJ_P0w*%><jvyMND)!b9d}-!=+^`G1(Z
z!Ql?F2ZWu^iS6{dQU`X+_LQLQxsiO}^aGLu;qFV2_PND-$h;az42GAxLfY6cJ~ny=
zBkFnkFg5doAoa{TSHxa_nB02laCo@>X542fn!0;OMVZphnxJ=Jcdm1}0+$ErIV5lZ
z94;XCW+4}Fdh_;ztg8z?IAe!b_($_~&md=Jx@4m6|Gxh<aA&VjB_wRNcI;R#ih5oV
z&+agATpV(|g%U0xyOv+WKDG^#gW(17nD>T(<R0II%rl#vhpg8HiNWy9IfygLIP)*<
zF!?XJGyOoX<qg+9usdbXU4#1<q^GwPvBvhdEu@~F5(t@Bec7(JE9}e~bDI>%eQ0$L
zQP(jCFI>KJ^&Dw%d0H@$VP}c%TCg1;b3u521!Vmz|5gbM|KE*=+^GpthYOc2+>fCT
zq;|bj3|QRs17z<NNDPKw-+Tv;-^AL>mO_18JL}!rcgn1j0{3lu)>fdTVKs3`oe$FQ
za|<#T=H12uHotsv+s+g3r|s}@hU|xkbb{QE3Q`NgeHm@L9xQ{L8LsaxxuaPEah4iL
zzC2?;IR3v)5CfYB5`*E;-)i7+gYmJ^*Y~1^H%v{HfDYI%UvFfB(=<#RM7Iky!Q<dj
z|4j3&8H+3~NqjNqop}eW_9Yu1xEutj1>xzp7l6e<Z1JBEe{YF{+<)^{Ok|gh?Mw^#
z3y^t;YiBIsW`oq*-Ra+9eNZ3lkN4^F<`bSl*3W_DK={92FE}pbOd$K9L1Hj0=c$Kc
z7EBzO7S@}oi_C|yAH0ItV{{*~_ZTJ)qM0_ygToxee!R%SZ0?EmJB((iSVpBo#szQX
zL(Z!P$%F8w1Bf%!KS)V}-Mdc*(%wz+=-+A9(qt(b*bO$b&wZ8=NDmCF9A@9K!ZaEz
zw}+*D=Yx-sG8QI(a9<4Ays5>AJ(e)J7HesYGy#)?(cP)o??0T!2sz7imMCWV=l;C_
z9<Fg*dKO0}<!tv^<G!QH2U3T>xCZGv!R#pUK&(U1YKHWGx=%y;IUz^2c06Zw+b+t;
z1}-yfDpkP#f$2Sb&tvDL-Xd^%I+M$=P4I;ril3M-LfU^ZXHKK!IgtJ*&oTEdfy9xq
zo8|!&^I&qXa%Y)ETS=&BF*2EQs>iF?c0W|^Yb{qf)Xkx4EaPrEb>=)J;Vc2gnwn$A
z>AU((S$7DV_D}p~oDpcP`f-Vz`G&If@_8K%%1=ZNDanREF+E}QOVO#UP~}U-8ndJY
z;)=Yp3QY_zGAn4Cahcqc{%i8fXNt+%zSRm0m+qR1D1@rYcvUE$6q~HR<ZqU#O6ej~
z2R>z$f9bxalMMPy?QH`z=id%ft_!=V+;E%SblcGiWt(sd69ZXe#oiTPRV>~LD~7C`
zBWK5d(!_&RPWG88tD2VRYL$sXs;atWJxW*J>&j1Ey+?8JhG5lI&L>TmzxGr~p6sil
z=&7Snef+-a(L_nrouxNT<&zv$FX{LiH>kO&u2251B7L@3^^$GBl4`$^X@q`+@|)Su
z6kkp$Hd;0z(2VEue3Q~RYtwnlt28+DnanSkwJYXxEmNHCoTBjd6Pww7Sv}>({kEzn
z7MwTh`Z7bl&$h*6#fSTfh7XNR_pH(|_3ExNXPvW5DXpc%)ZhA_>Vm?j%1*0em8NGt
zH#x8{-E?VJn~M4icN6RK8WY2LEE@KOuF5*1qN;y8{mf4??NnY}+h=@m!cm2mYgwxE
zouZYNGPNq&3vV^*kyKMKQaq|U|MzXBE59>ULtKt4y#MSXcioFm<#*3SW%tR+X2v><
z%5&9s%U&}-r}9FCOJ&n{ZZ%$0MYCL+pUS!^1|~&b_mz1VE-HWC?58qo`gcV;?f^4>
z%~OgEhKh=HbG?k0J9e6{^3^xLwSmXXcr~Y*mh*e_D{MFARSd5xC|)g<U$a5qJmBYh
zMb2s+m7PmNEHZ9tDfDGmn`eFBqA0lA!2Dh6PV*KvZi_>YmMW^*9x#8gomth<P)Rv#
z`vjFg7YfbhTZx)(ce|+kd1t4|9@9{BmZgieEUukanl4qZeB@Swd7;mJrC&T7P0bY)
z73SRcQTb!JOgUcVr+m&~J`)4aST&pV7uA+Vi79=3H%Ue4;7Yk=%a_XAx`-)d+n6c#
zxHg)lzj9IX{A?pv%6&kEaR-~Sn)_ze9|>zrjmkq*=kCil6?w8#wX#-7)pxtIs+(%E
zlHUPNv;GiW<pmrJibX0?CbOB|n`$OnnchCS%!J{Qn%YE*5c8Gn@(TM@_A2rJ7FRNk
z<Tm?c!lh*6E24Je_<u9+KNX6RVuy{NC@xU?`ozchx8Y6G+d_e+C%7gk-m&d7b15!T
z+o`Cos&z|Befvc(lZviiCNu7fs$|SHH{Dg;VwTXiNp<P`DJr+7@TfeT-ewZ|-A-l6
z%vmPpyOb4I$V^c6anVxbTi76PSe;>N)f1)Q_jRw@f00*87vcleb}r9U-2V58{0=2?
zrQ?sZmBSV;F#X-prF2o}nu7L@n+nfARVuH4wn_E2*-=wfTS>J&k9Ev?tyI-Y61r8t
z>`+x}ex|Q9)w<U7h;gOL%*TI~m92Zt*cktr{fJ1ka58*u8oH=gtw?vLS@IKmrK!s*
zlxD4#Qe0HB!t6@!Kjk|&@2RT1Z8R_b{9VCcH`pTPaFX(td3ol+pBc?tnA9vfd2E$$
z|7J70bnm$uYw!t`p8lI!KSIx$Zxrn`i*$`r)hxen%JoUs{L?C1E&b&$l<)DCs&xj-
zn9D9MQ~B^E)O_~SB}$7DMbr|PWvW!aTBLL|RNX9PmyjAm^9OaE?5~Ouaeb-=!5b9c
zFfW%c3~E+s(B7i_ntPU+f*p&>f{AbB_C(vLe^r%J*}t|#y*7Z?R9;R|J>>co^Mh;?
z)l1ba)UTLmsQ<mWMlCBc+jQQ~LZua(Cn&d<9kpONYh-$;<%0RtW4nz{>s{A6`P$Ip
zpm&4Z1IN`W@BU9w_R{JwUBAmqxjNQY<IQts(`Qfl6wh;gGgrO*OYIH+N^}3>g{C<h
zewwcMxKv^7+-@`OTV@&;pR}mIpH-*x-OJ8==k3WB)t%Q>TEw24C9Iok7Vf;qz%^{L
zn!DyJWw#~A%<AmC)kR;6nyxIiSG<34r~0(-X{z@hPgm%?eaqZ&>s<NM?ef|gYb@0=
zpF3zIq@7Uiv*wZCZg*R`q3xNPyV`Mc+1n>oXP61AS#MgVwNI~4{UFmhb&H1^O`Tj0
zt7cjMFyvqoQgaJBuUfXENHyt8vDzx}1XHG23`+7RCMnIg*{Of_kCpkhyfo9gB3`qD
ziy2fMHm^08Jg{9>g0)uV+jA@BBRQ+gbe?l7vr61nJAUN3Ip110#RKBPrYcig6qy#h
zH#I!nWmcKK*{t%MzM@CbJF|7BkJP5tv#GIc5K`?wpkQ>+=#6nUSDOm+><#+=k9V6c
zIbp8l#&=oe_{}#;n!EVTX5aQ#3Gps6Y2$TJ)Mm6)bF81F;1P6E){XO(p(~HBqDYO1
z+5^egs;r$X>IHno%6(_^<R=`RrZgpclA2%NZ{tflTU6g4IjYKj@Pp?5+pg-l=5y8N
z8XYq|H#Jl>^S6`HpU)+#YOD6CJkzRDT{xLd>1CdR>Cbr!l=G7&C{_jd8YcddGdJEp
z+w{z2X0tU0{pzJ0JI!wuGbpT_bx2{F^Z~`+j*HBWioH<U<;bn-H-*V!U&<QAJBKft
z&e_nfSmW~2?D(%;X8mVc%pW$lE1hT*H2;)luGXD?T-o~IS{0G)%*F*;J4~K+98=lo
zve+PAV2LS*@O!OvgKx^Q50<E^h+j8bF7#DdVapwpyXTK6p1T*TD#2~0yum0$G1kJ*
zIQ)^Xe9}y7wa;N$O5YpWRa+fA6?OVU<c~j7R-SIOSE=%Nr^%iDs>%{Vg7On?_NnCE
zKCGnpwO`G+bcK0Utc+@hjhy-Jk}TEOTt!un?KP^4U*1>AV&XLG{mZGcKHFBgWs{)k
zokT(N>#YpNQ}u#P-f&D;yOS$w@lr=q_Pp&v<t>3C%0{O`%q<fhC|%g>uNu3}*!=2?
z8H!8zl=NcLG89h+Wf&Dc<28LHC1&=sc!Q$9hk=>By}Mf9<TMqJ^YNN@!d1-n^)4}U
zxG+KG$aw>Eb(7tu6%}kc2}&_4r^}gCnHC;3zxk>}WqWwC5r+_;;;REMR2TEwDqBje
zmOnZ{&a`({pyH7xJ+0Qq8&qyEPg0#0TB&sW#9sOD4Lr(wqFhx&cFUW`S@NhXv~W?0
zT{BC`ylA0n`4w@sCz;*G=KI7|q9&~|4BzohIj!lg@^--_l?`n2s=JDWOr}dTC@z~~
ztJF~BVEpfox9P8}R#Vfb^Gy~SXs92|J7>Bs=#gB+ReR-*_eF}~>{6y#!39dW4ZSLn
z|2oa;i@(YLcl=^{rb|cR$LmnDvmV@LPyIKTP2vnu6l_#6dpuWORbq9R^2|6FHM@n*
zCf|QfH)e5ZP)?h~XfVOf*EHyspH}^66J`JN@yZjPwwgw6`>FJ(H{Im3;vf0L509xh
zf6Z5U>RBp3Ct;(}f%9FeZblqx7xJd59GR!5+TQt6(c}xOyu+_3CA~BHsvVUxOpg38
zP`UdxQKhX>Of7mUliK1af3=DxRueN_L)CMdH%$+Tg{f|PzFuYNTnp7>F=dKkOw&!I
zIf9kNT$xqwe+w{^Q(bPh#5UA4@y-F0@9jU-8dVpXd|G9zn0|{z$-KBtUh%xS=?|rJ
z<;rswsxd5AP293B$mx8_F=_3(rzCZHrpXuaZAMxlji!QGp-N|xPMCy*SgJjI>aX%C
zKU2wfLc6K0@B-5x!HZS4I&3oi`?}I(#?w7&JA<Q?6~E6?T|dRu_~z}U%Cc|Po7Qnj
zD~Xrbs`kJ9uCy~_vSO^x9aF!%xe9x3_G>JB{$9b+W3$SF*#!!UGuRY%n@>=V6l+!T
z>9IGF?hR0uJaAUb^F@TR(HAG>y_}KiO!=Co51&-2x+&~6iBbtx<?#wvWnq}9Dp$Tn
zsqDL|X-|!dO5V}{<#kWkOy2L?V!G*kyIJ4*55{3rJJmu?_?qWTD^!$g>{MvF;-YYK
zHJj;7P99~w)GF2G>ut<}s$&()^R&z!O=DE%Gz~O$7My6hKfu{+Q+~LzlZud8gGZv8
z`PWXB-xI6Ucjn$RwKtt>GD9d-Ro;osMBFsXEOo7~7RO~@<u^GSRqOMXn4CTGMR})f
zpJ}XHfl_%(pX%3b&dRCNt}0k=pJ!5}5T?lVpHX8&uDO!w2UAsF@3RWGPF+&)+cHJ@
z)2n3V!@(y^HgzvjzGY!9|M`u+^2396DvRQ-sj~Xjo6YuBR}pzyVtlA3U*+HPjmjly
z$tp^RUMf9N>@fXXYN<3uFIYbPS*O8+wZ7)@-)0!^u-<7}-?LujowB0ErbSET<uu)t
z&U^n;JRkPNtmxfbMNhRb6(Pl&=E2pk6xJnv*WVkzM*iO6#YTZ=WX+C#d2Fg3Ev0b8
zX|7qweLq!D*`oZ~QN1&t!+2RTlWDEtZRO4X=9rw0o@pAGETJiNbFQ-Z{ZGnUN|u_v
zcxt8GSHWm>Gi;;6rgu#$2|so#yh#5nKk2!UF++H|)WWsKYOEbB%716GtFn7qDPES{
zEmyoHTe1CEp7O-WY^E}7r<9(2-L2?&YrleQ>;+|wELYVD%MDC<E7aB9=N>fuSU5+$
zR$Ny-XOW(|^5fNNDNTW<OU$*EgYSM-da^0ZMC;2-lX<4mMnck4jn8eIsp+|Bm)V<*
zzH&mYhg1@C-ze)H{AS8CPf^*{)K$YOP|hT4s*qCmUp~WN{`YE+HSQRA?^$KMc8{U)
zi8=cfYJaXVVRo_5Sgl;G?%8=-y`%TJ@nPk!Cc#aoRZ5E4&F(JoF<pDMLvvTp3N=Ol
zt;+hbGfh=2gVmpQE-{WvUZPmZc|cwH(NjgG_8ao+*f*J+^7B_{C=k%FnQEt&wZ~QC
z*41yy4?1qkzfGT~tR(SS?SNN{scPK;)m|$>wGCXm6goS5)ML_4s3*6qGrj!ZU8U|w
zmBl)f0+k&y+$zmmid7`eaw&bA)M>V_qD^`A$*D>jtKH09cNdwjo9koxg00c?l8BGm
z>2pWSetn%JpLSxZ;-Q-h75CSkH@oZqQ7K!uMx`^h*kqbIi{dp&730i!6GaV=K9ijP
zBBqk<mrPw)7?g}JxtS)K=cs!8nW=nW)fcsSyn$x?ZkFa;s~#vHIkwq+o{Nc@)SMKp
zJ4HUqZ#L+wWce|f7x5`5&vkP#n>6K`{GmN7RsJeQD!p|Jmwz=+)_lK!jN&gRPK}9|
zB9$(`OH@gpYou7BeN^tn#zduA2No(dUNbZ22((a2GFq<CYNDcG;BTp9>=3N>*TB^D
zca)l{*Q`F%=yGS(k9LPuw)I=7`dUv?%DY%+T6ju8#pcr!rKSFlOqy5CG+V3v!(=K)
zgDIcZ0#(nox#sqc-U{qLf)&49bX79F>tUwOqpGZVKuMKvdbWA{J}xC&p$o>-bA*&W
z?EhxmEGl58m~`DF{@^vG*Ax28w9-t~eyq?^(cBcID($k-m}Mca+1{@#sy}|Dm_GAK
zHO+aqPh<504i$@?uBxAoikg(Xdatao`lIpIJ(m=Hd3{tJckwBE#8@c2c@k!7wlh`!
zgWGadR`FFz=cSWXbAq-gHg678$ol<O>3p4pa!IbINt9~0a<uG3>B~J=6%-52mAe<L
zRn_5IX{z3uulBI$nNgR(1+~9xD%A{2Jk>m&1*$gl9yVz|ze#D6{8FVY-JcBFr43A(
zIQfnLhQ2jERb8$oRNP{=dV9Wn&-|&%H$tW<otms<dY;u)sm7{HefReQ)7lJ21>eZ^
z2Haf?s%3wij5ingnjBi6Z<e*_roy(l^(M#fwWwP+Y*6cCvzFWL*ke-mv&rPn&a=w@
zPiL8Iy4q-xc=?y=WQHZGpJPra$MDQCn{#=Gs^{W(!v*076ke2`Q;TjAQaY*rOE%$f
zgHd9Ps^a2TCK`b+eyQ~7x2Vq$n5E?ZO<11in}+hL)_zq-z1yb0KGmztYS^#xK2Bbq
zf3cBT;nO8*yE&aKHm%54E0Aw6OaJ{;?dJV*H7Vl)HP@|Os>+OAW{cD(DCM*iD)tz7
zn_Tn`u(($sX!4}7+tgwITeZc)3oYONKO@8Tf1k2kpoFqAFSo_JV~k26DLv})?DZCZ
zu16_$i)9-1a_?0wTkdCk)_;bXla-2DlhGvkbIga#E34bp7YnUdOD;O5so}WKRA=Hm
zGrq<@%AXf>n!5(On&sNA)k)L+p{n$3wNebHsfAU4jM|L0W+ON23WZQLCiO#V1&Z89
zb>$wvXEc$To+01p)1}D}qpoW6xnKRG>jx#<j21b$ce|B%)32)9Ca9XfIk!or?9D9|
z{f%doW=ch=eM)<yHfQ2u<JrIXG@h7w8=Ww!(kOT;rg2kVL&HP&nEEk~yGGwKb(QT~
zI8+|V@|ajviJ0<Ue`~zO!oq~j$wp_Y-g48-m<jURx(}&VJYA@w^k|F8_WMdI-KJWa
zJZWD|5B`6k^ww{`@k0w~jjMA-O+P2hGC3WmXLiLmTuFM@SCeizS<RK3gEW-;C+JAH
z`5UKZs~UI59#FMAdEU^p&E9zKYF@P&JEy8=-(ghQ93f$1F(Fgq(YGB&JTmi@PVHW=
z;h5g6W)ZEWs8qygtR17Jcq>pyM@`U2y?COD<}Km7Dk=XrDCBUjP;vJAtzMzJ$4F)C
zO*QFv3hEYzO?94rU!<|@;1><!tYEXXrU|MiIQWdF=LD%*A6HPdSka?ee~VG6f?eNK
zWqr4@s+g(Lb+dbhi4Gd(Z#%Xb`;|{HY51P4wsEz;1;-^Bg=(gDh3(#k3X|HR%$nm1
zl`0i2Rc!=x%`eU`Q_R_L&ydU3NJ;7Y7sF15bEYS^y)Y^KHB;&4&wXakU-_ze-g8p1
zU`@~{nmNnx^ut2qDfQ~A95GU+(&o*kyw7%N`+c=l-f8q$bx)a>*=^=~$~Pw#85JK%
zR#LrQt~z^(lk%?qaK)PZtA-1LPbp|$jZ@=0aY2#UT|sq4oTLKRid701B#o4r-PM#-
zEhn1%ar&X8?%^o6_I;a1X}*_oe>9JpN!nAhAJ5}e-k;{MSW~!LW$Pmym5xW7Rg6>r
zDL!sdH8cF8t!y%tS;;D**6b2To0*)Pp7D`TK9i*zMKxZ}4KRO}`CgVOQB=um-c-f%
zoqx@mtF9_0KF(J;aOj}%_HG_U5!P=8C-|Nz&YRw4<o)rhvCgqaMo(*2DgIPVGF1v}
zQ=RoTS^0^IyT;;YN@mFuE}MN(IIp~U`9$-bWyefiVi#$Dy>d@kPnc2VY@ek0oIE}i
z(V1N)IX#jJwH3^&=VGdrL?iad{!{fd;|z$ByR?-<qiB-3(z^6VDwT>m6|D2j<R?1_
zDLJorrSz;o(VUC*zv5}Tg$fN38{|zjjg`W0>8P?VJZH*xJyK<r#5t1$wJMdB9D*uI
z*LqZ{l7GqXc{SJcSjlXqpQ;O$w2n+SmhcfUFLz`$Ti?=Xl2&j?ZS6^Ri)gkx@~8Vm
z6_@_lE$d#xX}0Bzk<yX*Whx%2zs;WIw8;I~^wgyM&m+ZOF?&qv7Acq=b(&_T|6z)v
zU&?&5?*}7Q=j1gj8!h^$9yiO=^!n-!lO};s<-Z@)Ory11%u4QS>!ugBDTQ@SQrRh!
zZkD-<RXO-Yn`u<mMg_;r-6|Dox=Mx};tJ;_EldwGD5|J6uTx*^-KcP;BU>f!#ZNi6
z@450R|IaBkJ=mZa@k`58B6y49ANjX(``ZpFnCnef^1kVzYW0NQM6uUa^;OnKvuAbH
zs`Ii|s7yHQq-x{;S>fXfZ_@`X{>oB)_mvC39W+-IxNc?`v($XcLj_Z1_LFMC4;GlE
zKekX5{q$bZD|xN_>10FGx#xV812bb)V?2(S^lTB9{p;swKK;~7rTU=vW|50LO*P-%
zG23y2U1_PXmFbMF4r-g3@>SklSJx5w_ruKU6NC8%XI+)<Te@aXKen4O>M9s0&8|_N
zz#FRSaALLT(;xSge`+i+d(G3OSS(PWs;|6X<)!mY#XDlA=2dzEitla)X>`wHQCzn~
zS#{d?HU+Jz?DCtXZz=~gZc}nTlwqdHtEgOc!CTh*0h4l2sju?&OaIikB|n(PZrr7!
z8lY%a^6je1sRv~$2d=zTapd<?>^2rMlQ%f46rM0q=}_GU)8N#_raPN0Od?YEn`BPS
zQ+3GKG~bggt5B?XU2(6outNUrD6{ehu}Wvx9Z~u2;9&NS_qT$g-9n>nNw*aL$GREG
z^)56C<!dp?Ouw(_f8n_4@rKLb@!wU8-zo3xTWex+qSVw{d$x-BuOFs!rCiKz1S%_f
zEn-u?VWF(*6*<S`*Qu$>7Rg^tSna+jmPxRxDlRNmN)O$yu+^E}tXF@XY|JfJwV8Z+
zisnT*DtAH-$h)0hqQJLtrqZ6}8cGpacT97_3YBWaROJsV?38zOKd9u&xInE#=a`Yi
zw*=L^dwWgp1|_Q=-tbFh-IF3!Db7kov5!$E=chI*o2=<pE;S7?5oPQ)-K9R)%;-9s
zNm;G5y12p@lee9B<jr*UDcY}@ARkk}VzTrwm$HR~o@)6`F4L4RM-?{CcQV_lxn60#
zONQw)aaL2qGXZ8t{<J7{3bUKsU+JoLWr2;#=i_b4LjDg-^NxKn*<Z6yW!tej6X$81
zX1i4kH4Y^5E6XhSsA{5aWFlC0NV%i`l4($po6`BpB-MS2Uz81{Rw(kPeKUC`u}R_a
z_9l%dOD`+rESaOCv5P^$y>+sJ?Hy0$=?Oa(8B*emb0@r3TH#+S_pZl8`Q4s#O8L8^
z)i>wdFqxM&T`l=2x0%~4R(09#xoYzLJ!+4g=czWUYM7oBZc?(CutZsF(>ycw5^3|e
z>x)d*30au-g>6t%VGS}}dGLq)x}0;$@0ZL~+WlDBY?+d|GIRe1^@T=~X3;)N6<%DF
zGMSh*S5+-)y~*^-oyIQ5ubaLqzoIbn!z0t=#7*iRde7A!MSCiBsIE1=JITpR#%Gzz
z-w8ZsMrJ2Xt~gv!KYj45s?F&MD$)AVrqhq=sdX#FnVvQXQQT@KsQzB9Q|ZvNc?v8G
zWz8BHq7>4OpHjQnVXS(jd78TX+mlLh#pmUD>z6Cn30zZU{mE<EefFJ-WD=8Vx$bKP
z-ji)=^J0|LC51apSEz4P`6wxGzI8H_YQxlY70zvMRc<FMs`M<VF)37$QVxBzK)I|+
z!EBzKyV?KZ9^>$}i%n)<{iANUBHy&@w1k3`9gDKlNdcv$LG7li|JW*fO;A#`^9VBg
zXR}PUt%cFp&PPzO&_=-I-OR1VUQ=hAB|QyQuo2}qRSDu#J2*2yWs-Y~YKj22S>d)i
zvxM8OD#2IS%@*%AH*ro()newDrY!IARpm=cv)Q%olgi6%-A!12U02xk-A#2*!g;0t
z84e0eM<1Jp_p~YgxxGO>=*S;sw;kN7O+8x`8Q1KRy}4XjnTsh%#cxTN+2g&FlqV`r
zS5E3)sqWD5T&0KUs#-(N9n<|rovMuXhfKKD?y1_wHma^L7FMnBwo>adoNe0WTC2n@
zQKF<|yW04&gsSOUCN8rJymcmRrrc_uR9>5P&tR6<vhq`T=j^Vm;NE8X?0Bw{5^Jy8
zqTM3qws+JNpE7SZz1aIe;owm@(|zZUn=-%sXF5ltL1F$o3$ubPQ`L^VT&_At)=vFK
z$}iKbliH@n^VC#U2DO?ny-_d=S{AGkpV_LiK;1%FKl#3i^GrUK$GdNtyvWZ{T+zX#
z_F<ubqUy~9vYTFCF*%Z9t9Xq)OZ5!b5mont+tgInrYc7-W|W^&WTsSKn60*Uo`hMB
zSETCp_ls03<bvd9S4ydGOxI9lk=bV&G2yMs{h|rxi<N(<lqu{}Ibb8M%D|$axM@zW
zS)K%ga+FV%^0uq8<~Cawncq#$HaA;+!PGA)MlFqv*?e37blI6XD;2YHl;oS|cAKsL
z?xmFVOkDN+<6hIyr+eiKb_AI{e|bak<9t@LWr;<mlb5NO|C@hZk@2aMX=SaD+U+!N
zm7{w_)m;8@n!98yHQRS^s!E{3e>0tHQs&DG3bd9?7FGVH8>?Ek@|dZ*=VIlZKKkZ@
z$G<D4U-43HJFled@Nc=oQSSir)@>Gws}5;tq;B9+jD8=X(%fkzud+5$!J?E~xoU@x
z!ZY4_Gp0ITB}G35g+GT+Dy~k-R<c=jNiF!4nepqV=hfe`FE&akH`7S}`dEFd;~Vw&
zojvNmU4@NjEBPz$I;5?V%${qUE^KThvpv_;HK5LT-bO?1d(w=is~Nt^OWx^F-EQKo
z!e(e?B9Wn|@~8Bl#@;t6rd9HWN`HzUm~PiTp&s~5+tg;?0Ta)kGfWr6>M6BNd12hq
z|5W4PHbsq;I&s~@o*zuMtyyi9d2p&~QT;cA!&9|PHr@zRV@h*X7paa{IlcI_$w3=Q
zjbN)a#^>jYDrIriY4oMOQl0gCregJyM5CxTlNIjny`i;&Sy25>`d^Lz8>XoQ{XHoE
z)NP^4oXb1Z58hs2B6D@Fn#!k_YT>HiwIU2NHMU)urEzG_QPcFqsj6~|z8Yj~{;m4?
zLYpf82L;td2Yl3W_iQ%N=ulO<zSc&`z(K&|M8^%YvgQyIqvn++kF&3-xCjVXyg%G1
zuiBiUqCCf6`AHw6S=fJO<<yLsYFk1?%xz~MQM@QJ!<f-<g@R_7jR~X49MiBRx~2>j
z&5EfP8_l$&mZ*ij+oziHxJ&nN1DBD<q8ekn?jn`PJRgj1A9`;xdEGf}v6_CBz%BgB
zNfS<)PP?tEVsPn{QE_LRqEH;4+J`g?HJ8t4<&Ll(F*>y(Qeoq4AGPVaXI1|Nol>)G
zZc(<f=2T#7ov6gSqg_oc^PX{NQ=Y2vf~BfVVGoqJob}X0@|0B#S9h5FYH3h8e|WX2
z+=e|WR=;Xhqy^@vJZp1N-7fv!<oAv?rA@7zDsD&4nSQrtGwYkJY@EO@Wb*m#FEyd}
zxu!QA1Y|i^bSuXkj#l#dlW01NMN;|PCVf@Lh>a$;PcO=UwCXXEc&4VPS@F|oqXvVK
z$I)L#<ts`R%68_PY%yU}(@3pVx#F^4HEvIhiMI%!Y4F@|l}WqpO{W_jHMuG?L2dCF
zdF5?4uc&kja~i9;E>m7}X`M;-Id{d262YprsVkI1KQ}3q@9r~YobW>7(x*TTwLkY&
zJOTw(cdRW|5?alzuqXP2Qb=OC>VicZP2%@=skqCpP~j`=RG9TdMs3o0YjrX0-KPHM
z>{SAf^q8zDk5=Jqf2N%NW3tM@mAtBJH^rMCIdNO*{kDG!PoJMNdYpI3^yK3yrnYaq
zOe#uERe!v>Y<8~KUNPA^TKQnwW+lVBTTK5gIj&T>x>-f{pS}6?sRtFDqy0_SZOf8V
z<@YkxS-rs2D<#QHU$H_l{3nlD(?V}mLya0`&N2lxJ)3w_Cxv?^Hw=|kT%x`k&&oYv
z`pj5RBjdA*vTTBw$`1ic(_;du%4~vBCccGM3im(!P;qVVSA3-CARm?DVsh>BCRMh>
z6I5AimZ(UrP*e3*U8%&s^}6ijoqbCA+agrA?s{XYs2H!J*yy6-^>4XS(*1j?24|&J
zl{<P&o5J-}uiV{WG%M6cwMuukN?mK0>c)+Kl{Ti#FuCt^M5%uAGsQqAWfMPT8ME0N
zcbG2YsWts@p<Uyl4YT?0hqn|AKWHkYDBhR*dCbtP+bv$1WvPPd+*vGUVz-VcWH4!&
z?O1ND^lrvW)6+{eO;d`U&5xFvE5&@)HeGh>lWOjUH_DPzMU-aUm|+^@z0*`DR#T;7
z;X)Htr<=xC?b$V^o~cj{zqV3k|4K{q;-y!V9cLSwswbY6Kk^_&wff9S#T&9siZj-!
z8eLs~LuK`n2dcB)H7WN<&Q;Alt)jrcs7Ow=>yh$_xpS2#+-5aXQoX3;67x>}c#WQ#
zYO=A4cC@Y9jwd@zmaE=WX4vrDL|uE6^7##&%726dR5+$7D!mTPGD~CIq<F`(UGdh5
zIHQD~3+B(ZrkP9jZ#UKZJ6Too;|cSpvzui_g!d^_?p-6lM0~MX<AVkz)p#itZ}~>^
zXP&YO)2wfqyOwq+o@JYEF1eN8g45=#c|{bT;<hp&i@mKsRGRFhl}(mvs2ZPSHf4Wx
z*!%<UO66Yfev{;!81oIzwmMH9@G0jr{#N?)?v44D7w*cFov)a#?QE8_>Tp*nSuU%#
zO2|;YVs^es;Y4wT-7=Eu#(O3!y|mt`()_<h{_EcWx$8$N6j?*<6kpD2H)(O>S9<WN
zL-vrsF*T8C>dN*^b!u@ZYfY-Br>mYUyKiDyvt4zYth{R3jA&K%9&M#vAqi$1=2s|h
zSa4cl|4T8$9=la$5zk(kZoJ86%GBVcy4cIt{EDZj;{6&=CEeF@O0#_h%>GB}E4lHC
zt1(UcVfLxnRdE~ZO4D!C{1rppGEMThv&>qaSel+R*HZd=|B_kg-eR@vawS!5HCEM#
zm~NBbv)fIw&)TSD+w3rDmaj46T3ex-QIez5cA;M7!JT9i4s$7$w~;SR3{?dbxh#5B
zg)TZN{q8N3|Dv?TBxs_g{Hog1Y7HlDD5bskRP((St0;atM1CjZUL}TYp2`nqs+iWq
zEmzv}(@?tAvRSFh(?QuT@vy3_?NsB1i$&BTWB;0#dRVE|%x+P=x?EB1QN$S~{w<1T
zQL*wWCu^Q63w4E<B^LfO<LKF8(Ue(WdfH)vT7G1f+1oG0ijr^Ylx}9;k@w7-U@GwI
zn{pr9Jym9>H)i)gE>*lAw9!H=e!g;>Cy#~E>_+oz0TLEb>q3-$4m>xjSn^P<{@{KU
ztt+3jHf*snFTV2DynA1$s_6Pfrt&w1EUFIe)8W2&TzQj&ml}tfuem{YxXPuyz2-~a
z&QVfR6;capo2mAy^s$m`+C9@+!E5qI_1QI)O(!a*Ub&*Gba}3V@!ER|>Axnbh}At-
zE_R$~{Dz%Pr6AW;GBatV`jSN*DoU3U)mt`AF*{%{t<J>p-dt(^MD;Dd&DEFAR8+sU
zX02NI(^ym1qj^f(+Uk@K|2b}<npbA}#81yWE_Awa(8{M;b1zM|cvHhHJL~@%l{F?i
zm9IO?nfZqZD(5NrYP8*EGjpluQndXdV{Yrkte(H}yV>-N&&H+)?wW4-rlqh!Th@&8
zi<!nk>sEE$<Yt}kPKD;ea^)80t@l*UrSq7l?pa~F>EZ?h7M;avn~z;r{&PIcOin0B
zJ)-4?@djNV#Y3sP)RV$}l)E0BmCvhxWv-|eCztw7THBw)Qf*bMy+&*GWo3rjALOL9
zCMi!k_(UyaIh#e>t&^&kEyUD5ADF3?tK6mDsC-(zZ~qq)MgBdiPak^epIaxSCi~&E
z>XA?7s$Ww|)OOZ5nFyqQSCrl}P3cnF7X5%^Uh|_45~d$7=$ai*`l2!|zt%kEq@>)*
z#s-z}=6YqpvKBK1(Va@H8*Zt6T-0r@y3Is!-;~9s&ziO=`mEh$=JbB1*|fjf<~o|!
z<n3ZEn{5wysCMZwv)Z04AC<`y#0~i$PB-Z)2vPa)MpEy>m3Jny8%4Bp_&%yU<?>Z>
zUEgH(=Dn}V6Z=LJ{<q1B2X|Pjg?TcloD}Smdtd#<sLWegas5<oHPv}PRp+L&s^1Q-
zSMF!XlG`k=rBr%zqFNh=fH7N6i|Up?r&RyA@Mz7*wN*dQuB$$&MblJ}BS$r(<&YuI
z>pIorF6ULmr?;wZ<6Wot^4(nH>TA1{J)CYS?nx*!%;djscClU6H2>vaQ{l;->THLm
znoE61RmfUWpjceCSK(_!gW2X2*2)#avZ|MUI-B#m$|^oijWhevlBOi^Qo!74^;)y<
zw~Nh%uT4|pzRYNzQ)r@Af9$d{vtWzrr>;rH)l6k(*_q2#<~~|xs9$=<<V(aUZHJUO
z%JN>DRplppm>=GMR5{-`!ep;ekK#(>IMr{bN>mj1_A73hY-i%v^Gspp1u?akvs;uB
z*B7cz@BFAx>aa;JUZ+p_q3{8vIQ7#e+q%4zjl-|XvhBA~JMh6x*)I0Hn$zP6<~Jg$
zRIT^FHC0%ery5`quKHuqTGa;0ttvOKc9{M(cUFma?^AyB)Z1i@y1H2jN4M#p?DZyG
z)4A33kKHtv*5X(4yPc?fZQ6FFwF~>rysF+PeQ_vKU3l)Hx!-?Q#l>6hn)Ju<EB)V)
zW7?>E(8T=3L-Q?Kdz8+!7npvt^;Yxvm9A3pV~SeU!(C>3PFtCkiEUT;+1zWoDyYNM
z<Jx0QO`jy?A1!lLU%dThKEo+MC1tXradppGMZT=Js+s?<Dfj&kR#>uC#dPQ2-zs^`
z>ojk&Oi<yRV4&)rTBWr0O`iOrt(TRnZhNaLmUf!QxSFYOb~335h`Y)!y|hwQQA$H?
z!Y>n}w9Py!Q*UiCN=#-^nQ?!f@+y&Nl~W1ws=u#i81pzgDXK7VDEWP`GUf<sF}-&x
z%XI&wLgO88HPu<IHkr=4FiAE`!$-L)?UrI-ps`6Sr<zjwgiaOyCv~RhlN=RFxYf+A
zIIdLK^FPAOUOCL{ua=3~dd@lWvVBEn(P46`G9R*)>)vv!PpT0&UM6j57O9}1eBl6x
z;iFl+rZaCY)!z9tNO{eV|4Mhg&CJC8M3hhGXqlAN+9~+IJgQO`)vVUP{*wITYuk;}
zS3Xo&Z@FBxEAoKK<h$moUH>?gtewr}_M075ObXXiP5-ZIe8xjd<?51hl_o()b<Tr~
zYF)yc)OfgKjL)#Bss1euGd(grTJ@siER~*x2C5NvekvAC{9>Y+ZKkYW@=$rRj)>W>
z;83$deQh(Ampe>ORjpQQO>r<?{LxrmZu@%0C2O|JZPIZxop~o#`BRmV>h%qmOvFU5
zD$L{7Hr>7XuF|dlGN%1;EylUmHkrP;ovoyCVS>r;4Mu8<--fBowhB-T`@PV#W4Ekn
ziQqDo1yYkuk8ZCs{Sua;k@hA~x%@%5>OZX$Mh~~`Q%>OeYg)b5Ua8yLN|iC6S4I8!
zK}D{V^``YT|K<4jYBlz@>M8zwDyX`{%v*u0E>fZLXMu9P*J35LvVLQhJ|*R&B2Q)J
z9=W3Oq40z<N2j}bf}?=x#>b_qulSiv4(W!duIYADjl42lRj0sAng7px)AHCO%B$0j
zl{KCmGfLQQY3A0k#Z2JMS>u;YwQ3i)%rU!iX^(<oP=~^|`8(zF@}`?AuZd9JvcFdK
zdu^YY!t^Y~7|l|%TTaWAjPEd-Wdu$&-Sk1rZ0F-hWyUwgW{;Ue)wH~uR6cuNQoVL0
z)bzF8caz;qgH@IMx{MEWKQ`OhP@%oSCs)~`t6J4<2aDN%g~!UXB2rA-Z4{L_DtlF%
z@~V_q^tmanvHNa(poUX!UiMS<%#}JyH3prkJaT3V@e?}~7F^z~+`o5$@*P$llhX_v
zl%uoDWaqs%REg1DsA8FSNp+_1SyN{nZxxe0IffH`3RGUkZ&Kc*Td(qz<%!axN?y~r
zPg|50)=!r2vyU{`RQ1;E^=4TU=CiV9;_sHLh$X%;e_yyrfqlYX#oSM46(7n@Gh1_G
zg3{xsnJSv=W|&{J?oepSFg9uBNtX-JP%(Y^pVjQr`|oDv{j!SgdmYWL33;kslXOwO
zGXIfs(7Bz)pKckM>fe8&e9~FVc!S$jlRJtmnicO_l*{JHsA$PDo3GobqikAy%xIS1
zGx<Z}O)5(mgBAHiycIUDjWmi1VV1wU+e_{EM=s?Ab0*bQPb(CQ-=30r#$lp#R@z@V
zMWWp_qWzANY<`ns&cp}G@2{;_-uf^|Rd*h%8GDJk`ssWgll!rA)P*wi)!%H<QI|Wu
zNbSRY3)9;VCMsXNBB3lTG}-9tqr)b)1|25rUQRXkyV|VDbSd1-ws5n&V8U^g2uVSe
zY%g!qMb3RniaM?uXZ$ajW*dK2eCm6^*pKCr8uPz0V?VD9lg-83On&8;D%z+XHBk<;
z)Oh=@QayLU9MvxKOC~d|&ziWOeWP+jN!)bKnl4k*b!XL9eO;mU(8*I}>-B2WXDSZr
zHVT|3sd|SLEz}OG8#S>i|9GP)zv=FDlY1v8DNLQ(pl*83TJ7UPI}NV?UzMG1PL#KP
zv{d=P+1F|vveQi!1NN!P@8ec0_EuB)W?!fNT<oMegUwwt>vL@?qM`@P>yj!|SP~pm
za?R(e+$t<reAC%!_ITeU<$`NJl(K#Pm`$7HZ!V;|%<OkVp6SbdZ&VAj_{^CjOcbYu
zrYS}^>nNn2+hjJG<)G4o%dIL?`97Ootd>{c-6Ur!m6)Z(#a?E*bj>T1iTn4MX)Y~S
z^4q9rdc&<iwVh#watQAMMcF(_^U}mqW~cc7E1&<#Z0^Ch+U)<DH|k54pHUJ&J71;!
z(FwC#Od`trm|D$FEqtU9(6UWsPj$OeQ=Y4Qp17s?@!t~_xj!9K|2c1sVqwB_<@n%L
z3f`J8<Y!#GqICN7Eu{({NAp=#vy~bb$;n5}oFM;MnMt{DRhe4fr}ZZHmpZBb32HG(
z^K?;Nl6pf$FDpRR?MSjx-R=Zah54E)5zgO~{#+F_c{pXWStwhIsm}F7ri)L^Q*AH{
zGM}-gS+UUel!C~ld5YXRerAzVXDROs(^2hkh&0bU7NU5r;F`$+o9RlX>6)gc0(xey
znNFq$+69%_(vr=@a*foo?`o>tH~y;>>gr_D;UsI;JL$j5o})ia>^%>eY;s(xF2uM^
znW-;FHDzj#$wl@@%KzJXOr+-CRqU(uRXwnWPkE)u0)^J3^(Lpi_bKE^EKvP+(qBop
zBvAG3q-ly5sv6}}(leAT|2ZhXX!>Pr{dl|b`E?iNmjAe^yfo>#a>j<!s*=LBrvGPU
zs?A=cXtd1og4&bj8nvJu{%Y-y!c}7nkC+tMO;GZGd|s)CbC<qykf7=QtA9)aw&<Iv
zFDq7ay<Tb7F7ZWS*XPN~%Wh3m5)LUfoe?Rhl(Df(efgRVrvJM66%;gYnpp4pr?TZh
zqe(>ZdXu?#7MaaEo}zee>n@WC-&@od>|Lv7-PtGiXTH4Yk)%9h_4e1w_s)GWwtZM(
z^6UFlRc*B;s@l5JDjaER%nTN6P_>yRV03oJOZhn`&Z+&9w^R~Ov6hdyDr#gtXN{uw
zr^o7i|DLLt-Ktm5JhV`WFDy(x<%gc~f3Z2L+r6aBJ_Sxv(NkNa(z)}sa#Wd#+O>Tr
z)OP)dvJl)_t~QxX&Sb}tr)tsc)oR}7iq)>~s#aZH_t4a+!a{kkOQRy^oHfQ0dJoKH
z8J$hPPP<{68of&`^_8IIA6<C`w&K0Y(}Q%C9cN!KzqZ;}X_IG<`h^3GmdZ_LitU$I
zoA9QtRCN-cVK!AH)66hF#3Iw@uflgnNAs+!9qQ9ItyimQP|{Ek?>AG}JkjiXoT5ta
zd>ON~g$GPer%GxwyFXP8)t{+6=XaTfxPi5r9=p16_vdN_)hZ@+#jl%{Y!d&=+>~N4
zsh?dS*Y}!N(|3ZhYLsP{`n=<Blw>aO$`u=ODeq`Ls4Ak>W1dpFTIG4!6_q?SdnJ>4
zdo}Nr?`j<C=ZsE1;m{Cs4KSMDQLQ19B&MO3FRwBC#zFPvMRJCjf=iV79rh@HDdsWB
zQao(3F*nTEoK4y!cZItS-^LQt>9&0edvp)0a`V4Znd{kYvSUY&vdu0{%_#jE(=GQ8
zDXrG~Y;2Mzr}0KC&-4KAZj-gVCT7ktvhsG*ubZ$G%W9?v1Z#LKnWi)QM1t|Zpcvz>
zoDHgW^|nUMu`7(c`pwnSW~@~I&2v%NN<Y(NQAvfys+K|{&$h!#Wv@4A%-8Kvy|s3}
z+@*u&#zEWD6^~36&`GNZRNozJqIuWqwTg$Sk9_N7MU`!J-_`G5<TSSZbVY5;DHZjL
z-nu%<)iX3&XMfO`nmXN#=W4ntvq+l(hi#PV=OSfQ{q|nfzIjSYV!t1n7<*<abDvRB
za#@*RXmg6koQ)&WxaQnlldWHM)sFA)G<V>;AXnG*Pkupgrb6twS!VpVCn=qBa96#*
zpvk=Oy|Q9iN0ZUz`tyn^D_Kk*-CS-exzy0C%xjL)T$vm*$zyJ6IYqWA6FR-sZ+y!&
zzVlDPWbFf8)lE#ICcI$}Ogy!VwKiEQDa&M>P`z%p#BA!dOUfFIiAI}0*ek^tm#F4m
zd#%Ly{D9(%mvx5XPJHs_kGj<!t^T8U#Lq!h_t#DNEgJ(Bq9)lY`xx{qCA`~V5|%Ho
z>^0+@tOI+A#wF>E%5$d+sIkZ;nBU*OP^DE#)cjK7ZWT}NK$X(ocPi&tf)$s!-#0bW
zu2KGaTS#f@lK-ZEjTFq9?oBp5{^gf(ydsB2Z_i(|8Hr~ULWAxraIIifWaUgS`~F8k
zDfvf@%A$_1X4(!J3g@2WnLJUNqFBE2ipi;v24f%F$7a>l%1YIa$4!)LdsGV#CMpZ4
z>Zv;MKQm*~Xf@+{#-Q@VJJxLd;diFO+axuMRG5^%zFeZBCoN@e$GAc{cHTjgV<)&3
zGwj$@k8NO4R_4%?bA6Lx<|GxPu;@#<x{3M`#cZ)Il@F7)DAZkAA(!dEt=v-mLP<-{
z&wOf#n-W{Ath@#L3)#aH?kg$0(N|ScWH39G7of7GUftx$&ny+a_be(awoOuTlr~oE
z__4>d%1uNmd)s!Usk~o}6ze6;({{(0H7++X;kmtEE%<eU`Tq%X<QE-hR$LSKLRyly
z#O%*vZ>63Er7E)?=$jYbog;sVZI<a2X-UPFz7M87w@u8FtUs9bXY5rxGJmDn)pzcy
z#!u>$&0>_*`~EhWili}^Or17M*>UqdlLPy{n016X=~{VxR=WJyTIG!28MDQuY|7rd
zTTSzmg%sB;Uaul%Ua!0}-Cp7OVI9+JEoO3i9h21SQ_~f0K3uNSK4-D~5v?8as<G#l
zxT{5!a##<VaJHOQ?49E%yY9BU(s_G6WveVz)dYd1#-E<4s`~POG5cs)s2cyFL&f)=
zv#K!XYehZnd8W~8o-4U6xu$Hney91<^>fW`E!t_mNrll=#%sOW(rq&4atEg<TuE4`
zIIE~#)~znb^vl0^<>{4ys*fdRm@Jzpr*OYm+x+kS4@x;DAIysOEH{}pf4$lAUD8Su
zl%z~Ivze(mH<hSNm~W)Rn9pSHWj(<xP$^JF=$WON)n`ug!yi-(Rv(?N{O6&J>Zgw#
zriyBuDq1ZI%&rUFQ%tMQQDy%1Tcv8*e8qfqOY^0|4DvB^wKN*iFDM)>N>ZKjGDyK~
zL#6zyZEKa+G2Bv8OQ|utc#1>$!--1Sk}0_=Jej+cCp>ts_AAoSH0S?rmE=1eri`i=
zRd(zzQdv;*LM8USw&K5C0cNw~#g$K<ja1tE@{md1EK{?)KhBt%3j8&xS)Q-D`mdw;
zg|07h4)fR*C%8;i5LupPmcG(oDM4Ysin{L$v$~W=3V%Pln0Btcr?B7{pNV=Er`a1W
zZBw7>Pl_VTi_P|FpHoeHbWb@U-&@fvPsx<mxyz(6Ypsf8>~7QU=?lz?g<dJ>7e7!E
zDp61kuKsV_8a-S2wxxsV#xQLq9X%%1gO{?EG7nu*INNa6bbqvoJjaZ5wJAwnikG<N
zs@$vKQkbsDrqK6puhO;N|B5?2mYZs5yDKTjewDra|GVOoMnUCmzsl97=Or6TyiQeJ
z`u4i9@#acZsf&!NN7fgq>i<bqJgj!tB>7i^veu<~W!I)z#v*TXO}QGLm~CD%&zP_5
zky_fV113$3Ewa+$QHs71`tm6rbB#GSGAO^BWUP8&W~lL;qu&%%q$|z#EZ?dmT`6W(
zvr)#ByG_Py*_S0s*GvzY{9kUbcKN%R%HEZ?lrNpHFwNh;-SlM1F_lH{qfGY1-8D0w
zpsW4+^E)N;_7AG7m)$dUP1&RD#j)O0A*@;HerJ^GzQ;>dqp}T@7Oy&Ol6P~8{DIdM
z8k794$Ol(nR#8q|CjZ(cL*Y|!s4}CVoKgX+ozdeO2IZ3}3#F?QR8^a2HYtDZ(NbUH
zr)MHFd$O8n<RsI!Fjn=utLCc31$C+|PnfT2z-(f=I<Z}8#r#FeYo0ALGp&1J_HO=j
z(?}&R(-5-_YS!GxOwTp($ZLGLr2ORnA*EIUE;9zP`%1Q+8`LkKnPnOz%dMEcN6z#{
z&OFtce1Frwxtu0_8+pw_e>y5|_giEtW3oy8o68fm#V0;0)UX{kEld<PYumY1WnQ+R
zS;&-kCIx3#tBKm(RJ|iMMdhctr%7ysg<9q=R<o-U*DCsD3#u19YE-%-wN}B$;gjhu
zHdguntdrEro|>sHE1RZ1f8!&iQ%CX@LQb_S+nQceRWdznyzs^YmH&lIsy3Ipl}y@N
z)LJ*GsXN9kGEJy{sS@41)2!?}kLtB0vsA=h38@~NF069gk-^0At*Y{ZGxwCAZDTW=
z@><eNB*D|v{K`8M-D3~cqyN1ztt=`~%&_>YR95D#=-2wvH2h$Ia#p66>XqiTrnmg&
z%7sM#F|j$St+eCXWs`o+Q$`PN?=<sVxkyoiKiy>BN&z*Mf+!WiI3>jy;ZbI{<rU0M
z)qANppX@iC-(qeev%W>6Q<6z}_ZcHqhPPE_M?;q@uhlg*ebUUNczbH3syF{(rTVo^
z3J>k?o38CQRJ(t}O+D!JRb}7PM^q**$x!r?nIUhvJzx2jeX5G;NguP>@>`VC^cs~F
znC7Y9cji*P%J)Gn+LPb(*~wPbES4S<r(O3{mz1@rhJII6U6*E~cJb9NQ&(<ZrK`r%
zlqU9c8Fy)Fn>DK&o4M6KHsO4-S=C|o5wmTDAqp=Zc&Y5QDOTRL@4ad5Q6;6bj(uu>
z8Dq@np7^0qm*j26!E!>O&t|u&$}wNFDFTsZ_q7fv@SmG!_ELMQn&JILs^=8g)b7>H
zG39-?)0nkIPQ}k5!T7IetJ%3jl^P-@y(%56B9*(O4NO8-epC*AFK+s1p}pds!z^mc
zUYaUa7k-kvcvsHEY}srz(fY?KyN(@F)sEh(7WpPe`LW9+xyxn~l+<hs)sEghZ2Ie2
zx@x)cT-Cr&|CD|-3aPI(vs9hTG0oKI1%v8b&xdC2kv~<Yw;xq0msM81mguSYefL%~
z#n;c3_E#1vCv87qcEP;Ie5u4w^DC>4m}XmBs!g|2G;evhOa4+*vcm2M_vCJ?^qA$}
z4^_I~C#~vmuEosla)SJn<09sBJeMh!o!x0RQ{cI2aEFKa1#Mv^4JJ`Dne)7A*|U9B
z7V14!$j|XI58lIL=2dc3CCh8QS+&M1^Z(x#X+M7rnt!ZQ{T$0{wrKer<-I5E&2<lC
zD8>IZRpm*`P~QLdi$e2sU-O*p@8r$nHmEOn&8qMyx?E+y%q#hmj5Fk)UsX^pJe8!#
zC@5>@uH&P$Icutny$XwR`bjb6s`<;+WJ59yuR0u4-?%*9c-CQajj2_S)YoR-Qx~=A
zQFkgmY&1(oN%_-HRux&+R%7#L9mWBBSC~57x@p8+;-vklZlZ};eye;=T(|0{eLX7K
ztIrv!ZQQOrf6rfymJVUlE|oB)Hz(E2j>}$BPx&;@bjGQC(<0A4(?Zra3a4|fnpk!{
z)yT0@(g@k1qWf>UsL9{NYSUWr$*Lje(hP6TEik#i`nV>id4hUWmypWM1xrl)B-J!t
zCW@I%IUA`IxTQ`bZbhbAYr=Yk9WVWj?mH(b7)IXLG71q^7oGT9<H+}gDoRJr$RFEa
zs1n+=U7hi#p3$NWbJUuCd{B#S`mUwRk)$EeKTBhZKd+hGlOEM8J?usg;(w@ak#1FG
z>rhZVZ{Vlq`_RC|*gai|`LTx5`KJOV(a-jniLf6vPT9$9YANTc(!rB#{^0UP84Kws
zl_d;a%F2ra%x1lssKgdNQ|+y<qxpj?OBHX<DKO4orJ`^nWs>R4mVQ&i!wzOT_r(>=
zwws&vu`X8olets1yro;WreD=)5$_G7Z@fM#ON}#)TQd?(CYfH<3dmoflCxl{(uZ1M
zvxG1ymFw|mjLiGP6n9wisdaeIRf!TgCii6HVWYYvVR?b#NHy)FS5$q~&#Q?DPf$K6
zF-LCqd>JKuop!Z6*N4X3Y?-QoYnQ9$om-(&ldG&Q=bEhgXTOFCpVk7E#qnIG7r!4@
zap-DSnYCcG%JsFDs=DkVrbSl#$}%rGRNlGvn`XSaVJf@eg^7lhgNfV!gKApw+fA3w
zu#;XOlBj%PMVV4bL9l7i;qOYd!iK6YUR9<C-!77G%;z*odh4dx{qKr#j{~;}!xU~4
z&(Qx0yW7s0Y~W&43%FgS(h@OC@mbt96Y<y>lmGSIDvl1yrt7+#OwG7?R1Yy<QsPZx
zQ#HDgV5GcmwX#+ZlWD?*`-)-Ld{w=ib|?w(?obfsY%#6fzFgsGv$h6P<^z=t%;KuE
z`8t(CES}3>dGtleczKd)z;<5~nUE$GwX+LU+?IwZPCYN7)-_X4JzhA=wEcje%C>cK
zCY*X{DlFyVD#v){sR%FTSLNE~Y%1J&L8&BTs$#QrjS=5L6|+Q-7p5O;C!1XDv{#*Q
zZH`%wu$!Xa+!W<)7YdY~#O*abc-K=|CuEX}$Wb5jg%>Z$|86TVO|R9GyI=RlH2kKr
znNxeJS+mLl#aAIoW^-n{sfzt-R&LN&Rq>lO&va(-Ig{)IKa{uc=`s#ZJ!yI>)K=q_
z=ToH#lkO=eoW5+ja-X)c<~ttKTuFU}iz)wA7Myla7T}DL&zr?<-0Z-iD)oMe${N;{
zDiLS&RmJUVmA>EUm!Bv2O^L%QL^VKcvS|i)vP$M6H<bgoR;UQ%-%*ue=v3W%I>1<H
znV9O&mHUjBHk+xocnGLu)fK3EE#pycT&iNaPq9^L_op(&pY=yf)_Ta8B^_8{7NFQ{
z`kudE<5=M%Gp+S)a{I0uDEZ`PDs=ALVp?0urQFKMtLkngXTo1_P?3Ma8#BR)eoFqg
z{mjxL_L^E)>zMnVTd4T|VT9@b%r~lq-5e^)ulST*yjB~ZwsJEQ6Dv{<=wdcqtKw|>
z<L4?(hOA@CW;Sjr`Mwcm31xLE=lrD162h1jdN(Jj3Z55HdDrKxSm6A@$W7?7;t5qA
z^_R<zD2M;pqFPpHq9E{mx!l!1zm-<KT&b)w<&eo2iOEV#(i;>{EIzMx+b>t8+w8xp
z=e7=`xl2zfmxNRsWKUbE+#R8>99kEo!g@zv>8*;EnP_^RBG>uViZ^d5n7p2L+5Boq
zy!qTS6HIv$c-8WEy)>7fJVVYcZ<~Vu!~%uA6@g|eWhW?wmrJS~I{U@cLivmQx<o+>
z9%*qUlX_<JD(9u<RZ`C8efHZG=Gn+ws4x4hBJhY$`G~iVx{JyLL!F;27GIxfDL=S3
z)5NGt(cDWSUU!~Xq_SuBDy6y@0SmQBxynZB_e^Bnc;!AXb5?nH&PDC{cW$|}Yp$8p
zUhh&oks_$}AV@}8%;KWT`HLb7QIp?FYsa=LFl9+8?wx+m@a`XDrHv<D6$C%!s&oF9
zQEphRt#;$9oN<I}gQ~-s??(DBcB#r5X{bK6s!_H6)U34hg|gYpNh!(&H#imf?*28{
z*w1Os=ILR!THnVs?vIt~>Wxa~eGjH9M)%xOoY|G2C{jApbbh$6vLKVJnyc_$v&&4g
z6*fh(n%Qc0D44OtnKTtlGqbsS!9-|LqmpR%A2X)kd1{+O)Kvpde^m6e_A;&$<1{T)
z@>4NtjxljK|KHT>NT+7w4nGwm>&Yqya+*yq+y7SH-tA_(AV*12K)6?xqqkK#aZ|K{
zrwXTW=^IW3=awk7j|qIrJuki0ly(Ox*45|Ay?)lLJZop1@|$WeqveZUDjmIfOwLWF
zTlIv~KILa-)@n;x9~)FEFsdE<Y-(I$tD&a-TT7K|vY=WIw~2C%Q?1$0j}Mja{W_w2
z_Wdrihf_Y7ZSnePv1&J`DP!nXwF~bB%w>Fk%KlT3RQfDXBHuZc+q7Wob!ErWL#l25
zH%!#RW+>VJW4080S)*LdS!j`S^_y8#c96OHLtW*LtPt~k$`{oRcU@Hx36t0EwhT7=
z%I$3-9c-n#Ah_1_#bQ&7S$scryB9F1aJC4k<*^<z7eBLCWv_*r`E>W`O7emNYL35E
z)n66eQ8LurWg2!cLjL3e1`XYLO-iD6Vrqq3)D-QHsw?PqiKrw$|EawBWT&B`<u2u=
zu|^82i<vai_|L1X*sP-d^@pZ;;1e13o2<+h@&6~PUtDjgKI?|O`sQUT)vB_5O&b&L
zE2&ycSMIf$WASYrquF6GcQcWSdB*?ZA876UG27yKvyoi6`xX`14VRVe>YB~U7u{9z
z-07pSlf}S1_~{dcgw|NoTx}-xZQd_UnGz40>^i#5tfxa?;TJQbnH!shM(oEH^)QP{
z9drE^<}1r%EN(|Ks%lh=n_qVMYWnWPT!W&trD_ppHB>gOsxgcG@2h@s`FG<BH47ED
zZ`!Rcz;sqgORZ5}@7D+OLvz&B*e6PBN7&k^1?O67xa_~IY+t~pkm)^F`DgH5wQa3N
z7MHmXsXja)q_#A?TdQ5RL0xm^DfN4OlZ>Cs?^Eq*>C`u=6;+eTIjhR|I#*R+JX6g)
z?55H0mwT0F=G7>r%-}RkUwOoA@&#!Vrpsz(soT|6<HJnNKXd8Iaq`!w$p4t8++R>^
zCboZ`(naN4YFB1nG&4^VP@EF*&9wV9v(l~N&t_6vgUw<dnwv-VZ<GJsI@jz&=R>up
z%URU~uK1~0+xO{zJJxRcw$xK4^lY^La?5ANsutWjvwi-m)ZJ)MN*3C0mV7)zrTg0~
z<Eat>idQx{s2wU6S5>h!k-Nb6-#GNuQblW~Kvh*!Cbchy%<8AQtCWo|3d=RjzomH0
zrd#b|o2K!}*^R1!=gz1WXiIAKhw7?N&0MW^cF_;x87_IM=AT{|F22*M`l{!+iusmy
z)z6BOO8d8}8Q)i_QQjKNuC!au#aOYS%k1=hQIpObX|o%bSE?_wU1nb9lB<w@X||%1
zdba#e(I&IJcUsDfb_%Ncd)v&Wx#lW9TA65OIQh5I2BS%4Yu_F>m7Qy1p3go<sp{fX
zv)6U{YE|BQm31Sz)Yo&i8@X-LFl*n?s$zEivC)iTD`UZo2|C5+I+a%z&QpEcBx!EE
zZjJJsEPj(y=2?nYw#TU2Y0p$K^^;Xvc)H1?g=vn0@uw)&isSp0w!Uak)m<*8py<vm
zA2nl#vTyW1B{zvSlX(*&mG614ku`VYR9~^6MOm2NNsZrH*?iKkX{x+S=a>pa^r*^z
znWif8<%+69`B@dM#57a6?J6o+?p?~9;fy9ZGd7rBIltWOv0{R$WF?Q9KHqM0?dE2s
z>0Zvt*H>vOt@n~PPgW~XmQbCh>h)LC;?Ks#^1ZFg%{aW4DHa(um>pYZYgWReZm~OG
zL^)@#zFDBXr&<krxr*o;U6tIGn&zK47)|%@xS(>0!^b2{dbXME2?@;;MF*6&sI61I
zd-{&~i#gwwuiAGS-%wkxl&<zvwQ0{E<v9y76z|T{G`%$^Nlhf_q()U_go+}sv+7C}
z3niu0EeefNkt&aQ^Hp2^%r|Eh$y9L_J)^wouBD32&9|zicTLoM>vkBJJZ4boiQ+MQ
zs>Y#W@wQ7@*UVVu#Y28o?br3jokwRW8Z@;l9!_gE&a9YkT9I+gB=gx5qg(P`>ecba
zW}?y@vWuQsE8o&KQVM05YTT)JN3nT%yNb<f10!E4HHGM8UbD`v7ZglZh?&mL>om;~
z5;Zk5Z<l}aG2AT5Nlx`|UXyaSn7+p51%3whLX~D-<*$@-roAxi_)%tZie;(J6`=~{
zNlNFHX1tX#i#lwo+`&+8^eNInVJFuym8T}fY6-g!$(J?$G1gxZrZ`#hy^5jS4i#Z}
zOVx8s+)8iLW#vAqF)Hf6Q&Tm(mSLP@q@hx_w?l=Yd#U;h!!N4S8t16S-AXi?#44d`
z!kJ{kY2vF|e7#V`!&^-?X>EYgz6M#-FGrY_1ty+WcBxop`pSdJ?231<nX1Pg6N7_S
z)C3MKGrigNNoMvpZN*|)cX@s_7t=@6?Uh$f(pQz?dt{P)X1?N{H5W|3?}=2}esPOw
z^)wa}iL<LsmoWw?`5o#pSz)NE_Rl9(W!h0om3e!2nJz4EH;LW6K&3M{-t<YfirK`o
zqFQGs_bD4r_Eh~~({8*^|Do~+8(uT<b^|5p6Go~x;ssQy4R<I?&yO`-aCx_!=wSnm
z>q#6+cD>=MXO#FApExX#=UuI)yg+|}()p!zM%782$}1Ta6$)?VtFD&$rCb=rp)Oiq
zY*KMMSyd~f!$4NfQT4jtPZckV397$@`<3ThwlY;+QK}r0S*Sd%SI+c#p}d){<2$p6
z{D;Q>8|u{7RYjYzFqp|KmseAKQnXjT?&m#|D;hJE+avN+KlQhmS?}joVqCh;Oj4mi
z>5{`Hv(9rTO#7uKn6iD{rnIcRz^rPnkD6Iwk4mFcm&$I>{l@m`*GvjcB~_jO>@{M1
zYi}05-&SXq?RMoY!a}O+=GvKE3=&eYyUlKTej=OV)TA!esrzm#U$g$KFiGgR@zE$Q
z#c=t}>dy~^DKE<3qk81WQH3p+{N>VS=_(tQtWcJ2bu~`$Xjb0s^GJTmhWn~MiHB64
z%b2KD@yVJh@};V*73MOSdZt1}`r|p}1-E)sat?h~@>y<W;yQJO((B)I6n-%W7<~QM
zZpMFgt%=aFDzhoPOI3<=PnrLSiC44~{;e1^pG#4#V!xR~x|H%h#a0#Hc3txwzuM(X
zXC5%s-!(;E>4UY|v$zkYA*E&JTz*Rw(=-{(_FZ&Ry`|=*ob|Xx(fZLE<M14RQ;)9I
z%J;b4jhAfXGP%IBN;4%mNco2ahYI6EE%VePoXW?Nn~XcRyqDkixIraU)J0MD#1w@O
zhOS1<4i+lY7YnE^StF;szwC!fuc?ya`HBZ}Hb&c(Ix3@-H_n@4`uP{9vi{<UiciW{
zsm5qrQ`Va=q89bZ#&ln_wt8jz0b_yfbJXWH8>_FFVWR$`evz6(p`|Ia<qze$W(#Gf
zfLVqw-l>}kT%T!rziOfJ4X-3kMPpMlCzW-I>^HZnY`M>-a@AAObVnDb^3RE`8e)rQ
zn5nogR^+`MV%og$n%cLhH73z1fu=_1mCbf9(o<3iv^8Z~Wu=kTQl-w#5ujw#D{K0H
z!%Sl{eL2+`EdnMB3>KMc@~u_vInuAz@#~+;jUz3lcN&e=t%4OzQZ~vf)=fC1e#Uc?
z^4n9|^4>;jCc=&Gitk_SR{LOYqQ*VNOk;lWALT7qRps6tkWl%U|4OYebgSvbGe=d|
zmNTjqe@{_<TUM-Iv;U;}e#WI{sb-xj3zmwQ2Z+?G2xg_J@K>x?IhequbX}m#Y|WuC
z<&KbJN{cI}nf0|hn&(!Rna^5QXu9g#JJn_L49$yvd{u}$y;I>+vZegAQ!Qp1h1-<Q
z9_UcfU65toBK}K$U+;P|kCPLX)*enayLn`b>CBn&=6Z4qlrpy)o2_TbR-HF{in1YV
znUZgJr+N3IFQ%DAqAJTD=a_Y`&oF<O?XAJ{cB#_tYa3Onc_x|tp7CF4mh)V*DF&|<
zzca5_$=Dj9bWFih;nyQubJyn?a$4fW>elPx6vQtIs;p&aQ(!zWLBZERU3rI{gYva>
zS#wYRQ%YO>Z%B!+5LdbobXYmN-byX+;xXf@OfOYU3wGlZKipL>`ae`D`4OWUV|QGM
z-Qc!K)VgQN53a~7TlgF?-uirl8PAc~rq{OfnjL7Guj=LPXujckzM{l}FvT^8d=<F=
zGMZWLJF2|i%~EySY94deliG^CliE%5HZm&znyX>fe#Xq~vz43Kr7{8KOg{~?W(Qq0
zX+w3D_ET?_oZ6*KrpF7Lh45ciso^+dq9DL!k}hqi;XGlk^7em$s;5n*O~tg=Dyv<Q
zH9ha4qj+Mux9XvssY)^@e<_5AT{PyNeOq;}g^XIvrV^zd|6)~b_azEub0*54nYdB;
z+pA{fwf$R-?ZeJ0Yh)ak-pF@XHJ-&-C1t9l+SlMoCR0CTtDUcVW_(ESlA2L&gIZ8?
zpxXMmajNYTxlJXl<|+L<`9P^?(>|lUtEQTka_=%#Uyx>UVQI12+sDCX`*$stKXhz@
za@*m4rH6BlnckP0q@>W$rCz)Ltm%w-UGmihf@aw#U#mRau4S55zQ@$SJlO25rKw_G
zS*fYV#8&n1A?wxBlse?EP5ohVImp9gsg;3B-$ZT`8>diHTjL384AHYyQ!mS?oLKtD
z^qBloRgG9t6X!;Ig%XAHYCrh&RHnKV%1{0?-RSVPOu6@Q%$lK+uT|>L)~oj#PgYX1
z+AS}y7o+?wd%EgdhZxgtjc%2D8Y@&3TY1$*{}`xs-jPz*<+x*>U)!Ws$1%^umi3w1
zgCmV<K~oyljyRU7UgzaA{aI6}tiM}HiT8c1anCU+^NdNEX2GIfW^+F8P!s;Q!{Un2
zEydv6^~%c|A1d{$Up6nFeM0H_s~&ZQ7FA0{*JTQ(zuL|2u3f0Ara#C0H|s33WoOwf
zUoCsCSme)WZu6~E{qTkLYNj`@tKH+3GJm^yv8luH0F@<ACmHXUS7R2>_C+ge^=8$B
zF-<Cm<>p$XS?j2|&K5F$ICqZ1CQ)YfWL+g?3DzdrS(=MXp7WnkyRW0J5#TSTdUi*R
zx{~EhrMbIQ<Q~N>QEs_-LRE*E*}_w8zsmEb{VIPvT~#v}9Mzc42&lh~e{RU|ltW`z
zUA)n`^jeKeN5wUEl*(xA;M=LbvgeS&^*k+=u8EtKWwkCF@9k<cVNTs=ln};jGLP3q
zXU<+<)7wX0%WE7uq`J+JOZ9wEoJr@oKIL62H8tNgeKzsm-J$e|^Ox~`VHu4L0&1o!
z%`X~Ddr6v2JT+I*U~0X|#wl`|wF#jbw<IU(yzMSE)?U<Nyl2Hx)yt9jMwcyA3|FUp
zP&2u;Ontr1Di!a5UK3aM91US#2E$p+_mt#!Y|^;ObzXV?F)kV2mzBo54dfL68S?4q
zeDG2Kc1u@NI^vaz)VE!-3Wriuo*jRyF3aU<>|}gRZS7Vi^%vUmI$5u}H5dXvXjq3`
zFg*~Mq8g`TsBf1Sqq=k(r|R5;ovKU4yOb7rTrk#h7F2#~+pIJpmBaA0SiTwm^bX^Y
zf83`0OZ3&2aB7;TcAb+o4(U=jb90Zp#5!HGsS_2H8$LR!GE}vjU)E4iWM8<_c(Kp{
z#mG6lX7^@KHqD;wWN!UDTFID?-^^r^jaue;2bDJy*wxyP+%tae^UT;hKuR?|x85L3
zC(I<(I7sU$Ym4$xv#F|y%+JlzSk5Vb+GAp5{c(n(9#f$zi@2atCKroRfci_L`tCjQ
zt_@sjH?6iQt^d}ndNeIXq55r>oS?gs@&~03r6Suz)BBTolts?Yk`t_H)@an)t-Q`a
zO-<)GySZV>NtJi$O=ia*98+OhK0#&Sd<oSCgLR78vMWt1WNnmx$4Mzo3+XW3_rTWd
zn9EKx_cJ0UCNaO&?_GOgcK=PjqEAtVLdtDs`O`5`W{gaEN}?Y?t?zf{8#H_6KV9=P
z>-w@nF^NIUO!4jkQ@)~U=B`4{O7lMmn?)vetETL)Rqj|NrS#x{x4GEt2-8iXoGRC?
z98Bl0-(=P!w?f@C%2g>;^1aHWUS9LFZBEMPQkhNNqPi5fTXU$|pMI{in=wOPzB|dx
zPee<tYtAIKm#cm&xGI*YtmNTRNa59x&z?MA`O7hD<sGjZ%;ntYDd{||l)2$HO{tin
zOZn{!BUP3aR;H6~S*y5LpEZ&?9i}2wc~F^EZGwuJ={m)4)90Gr>fEPjc6PCnQ-!Zd
zP~lm#$#L_{IxcFMbbdIdCXvr;F8V}Pwl3$RLQ>od*)0dv%+kMQDs_F&RuOEFH{Zqm
zT)uRokl92wCPmrXJ!bi(MrOUHZ04PYOp0;wTg-a+>{Q==ZCBp!epzkNZa32lEgB}i
zjT@9JenyxW9yx4w&%ImM;<=Ub%C85MU4^8~P26miYjdMb|6W+Cpy4$~<>9xP$~_mX
z<?W_8o5r7Rlzn*7Tz%s{XGI~m8!D$AEfr4o?3PnWX;S*CEU09(Ce9@9Q>o&WCMns1
zXQrwe()!B!SJ$d0#>twd#Br)>{@!a=X%?({`l6Z2d>L!i$Bz?~PWPTR{b)QvspHIQ
z<&F3Do8NKFHQVdG!#rZqCzIyZS875#?akRfzL(9r<fd4x&L!`0a<}O-R(<8KYdlqZ
z?<<=w(rQyIdM#&e`R%)s^8qFED@-OP$E0k{@}ll3p3)34ZD!S0+ap=8vihN=&Nm-+
zbJiDPW|Q>8RbFa^n|W2JnY%S98sy)<sl2N9g33{YX{HYULsbs#YcvZA{-_xFCPMY`
z^J^+^GBp(4XIYqU;W{t-<t~@TT(hf+8nV+>y+1Q5HVT}O7kZzed^z%=lJv<5X8pap
zlst5P%3m(AR$Z})UFB2JdNsX^tR`M5r&J>2OHE==o={mmFIwf-k3T9eezhs;#Ppi|
zSoumx$vjsnkh#u8^5splT?-n`9(fp<_RBS^e%z;N{(>!2fwl6ig4=gBg@|pLW^ya{
zDjk`5N=0jtgt@lgWQ9;sA+v;sF$#PaJWXZ=HkiFk;Ws@v_nG3mwHwU%-X2yB*!E6Y
zrzb)2{QLVRAH~H?#8<9TnS46i^o~udnU3)XHMuW5%2BI0RllBFZ1Sh6Q`x(@*R*Ew
z1V#4s|5Ty^9hB0TZYY#LXEIIF43z&ice&aYsjrF`9$!+KS+huCpU68|ADz`ot9p%<
z;(yhetVo`u)D{pZH@EG(${jBuWiBQMwM!Sb8N5Chr|R%X)3CIuMs;`oGZo##sjBr;
zIFz`Go|x1{$SG^jYgE<_X);y0scU*`lB9XX>aE7%>b&X?gk?>ebPvg@247MTd^B6`
z^S#@~#=qp0eJ>fSD&);E=7>mBWSnr>Ed4;R(ro|bX6!$-OdcpLHkosOw$g@fbJJ5s
zMrtk$E-DM!Yt{SM)|#Z2Z8ynny{%%xz-}`Ac&=H>B~zUlY`V&t-P=?j)U=x_D;`v4
zTw!F|^&?2hsV7i1{Ok->htfo)8#SLzE`4N@yIXFgv3<=m1$7M_)kAmE6&$($%d$<d
zQI_4Wro{Wa(eQZE7p2Us3o_gjcc?ymK0(=a@eQ?GtN4tKmP}ULy=1Kk%M}*&gKy@k
zJu2={OHf##dQdmhbh#m?a=7#q<y6l$Gp5gNW*+9V%}j(mO~YqyRI58-W_DJ)Ro;8)
zHf51-H<YSY95bCNy<KUY#|HJ*q;%76`OOOd&T*J&7|&OIymOD~tKepn{z4103}rdR
z(%mtpJ0dr!Z!UVQ7VmAUSTgCjY5Sftrg=@zRpL*4H#NV}YHII&S?!PYZPk$Gbt>j*
z_l-K5Ow=4^W}0~?-B(<eAgInUWu~$UgSvv%&xxk%U--#if1aU!@Ti3<`>g5e5tkX2
z<#s)l_Yj+)e1!F)>edr}Mo%K|sC+E@tD;z>ugdkgTCLntPW=a)vFW9|PgNMr-OTv%
zg;j&aH>i|4N~&sEDyalHOfd1$WmMi5_)&TK{&S{brD|rZR=H*cr@x!5Eq<;(sYt_2
zdCF9U%*2~YdEDHJ$8wjLK4~^q)-u#m%?*5S%CjU^cHahGvy)vaO3%Lbn7+Sp)8vrr
zI<w`xI!b3>Fq^&*;ZgI=^-|gPR!x3qRj}Da(N@#T{pl*%>!VDUvh|yuoUmH`%e&u7
zY`$8mRR>i~m)%&QeBSG`X^f?`(wkWks@C()E8P_~R5TJlYr4qtr{Y`FaP<W@Hz;4p
ze4`@T9H97RjjEz?M4R$o|5g<VgAUVVi#^Kk|A#7Dvrkl4{cEneZRHBJS9zu;3Qeu5
z>({cFq~Ca=x;Umob?rG7)kBLc)p|oFn&$1lqO?PLfs*UZ0K-#)lg++3B$#QcFEWX@
zUak5mZkAd884Y>q40n}+4{6HApWRK3BX%lH+uN&l{~?e0(i0sDN0w%oWj)YQT*_Q+
z=5yzs*_-3+W^DgIC_HTwGykSHRqbuea@8|Q94a4q&YHBbyP92Js-^NeM#fmx$llCa
zNkQwEQ>RMFOc~`OIZv|%+rBElzR_ly^wml6eFmpmL%*9!w)YhI#{!B*1s^ZST>a6e
zx^Bt|)%~VB)i$NXDf{>N%3n3zsU-U;RZZyNRb$O1F{(3IW~)9}wpZ2cIlFq!RRgtS
zHLFe7wU|}ag?Y>hgP2t>+3r^HnWC&J?)pRVqs|+%ZCiqs_P=#e4r{n<Rx+=}{K6x9
z3$qiqO#8KGswLYfncF9Z$@~9lRY)=ElDpJ+(5zc{y;9Q?B~{h{L37T+h4MU8=9>45
z98&0?Txss)>tWVwf7(pXm`zDfbG6xLO?I`qIbka2m>JY0w*{Erdu(R<=JGR@l@_&T
z%O~<%1f<N=nYI0iQu91l)xs7*v&?s0%K46F=81w#N;1n-RCQmoDhqCxP-I<eWNv<3
zQcgDJtoqtd&I)Hvyi*Zf|3jXY)l07Vsf4mdd4r;fZnWv&B0r^V!p_phE3;J&-}h3!
z^R`Xx=N>nMHmzgoN?pQ6njsb%Z$Ca(Uu1P%{fS$<`iF<I#%lgsl~4U(RWaM#V(ja0
zV6s4Tm+5S7MkD80&e|?jKTY;*+9KaFu}77sc#TT+EF;6o#<9xzwtqEVx3L?p=G9PY
zo0@5s$aO)zt9^-Sz}Al@54NbAmR^ljIFzhpYPI*72E!#KjrQwmx~3jajkdnZHLKaX
zP*r8Y2SXEv`6d$2s<gc~rK@}2uT$wPjx>G4q@uAP-_MwxuT4p&s6pd$pOqTt)>ipr
z22n=SD!(XfX1T9r^F&yEebyI^J6=mwx|8O}-)PWMS>3Tw{pm>`qq*;<su|RMP!s2S
ztHs41q;au-md2e^$)@QmJ5`Tu;4*%{{fFxFsjaGS@2RTZkoQ!JIizHqF!iHSC#Q`P
zm)sBI!;5Xre3x%A3jF=cq-24MO8P4Xb2%$#ITeRE72%`Hl_Mqynmu7zuGFhBQ%&Ti
zjk#=ors9i?7UP<Y^A%Kh)XYwv&oiym{%rR3XQ9Hf;5VkyrxvMQxwT!DWo@%=C&y=_
z1HH<|*N+>j$gSfr`dxg~c&_zVtz+-Esz}|<S6aXpZzi@_NX2$(iP6Hlg^C{21l8R1
zk`!kv@0WdbAkVn(Nr2qz%~w>H-oC9W#CAdL_0);VDzEuu7gVt+*>g6k`S|6U*x$=h
zHT=3nb(5*1THXW^^<Pf<YQ4MJjU8vMQCT}j#l$J^sLBMnxhlyjdsPIiOjKDnJDY~i
zWmDewNJM3+jjgFv`!&<jvx`g<87ocH7d=<An0d)m_jQRv+_NO*AE8Z3#``-=CmlYm
zwEBXv>VMxerk9Tk%Lje`YN8*;pm^c1kBOe*W0SD0nkLQ1H!5y-P%+KW{-zpPoTt+E
zF+)-Rp}1-Jn-#|9O+6}}&7VzDWUEd8>=IW`)q9{6eu+`luiew=<oBh@{-2Fai@A;{
z&d>E#%`oax@~9VAJe?zKs#bYd-bUnyy3>YhDrcGHRpTB`QJShRukh;ZXQhOaT-9{F
z4&w<AC#sx_?oye0SVCF$gtXfCipOfk(lKT#EE+1cD=p0Q=4Pv82=7zgRn(x;YtN->
z<)vVH{^kv(4K526WyQQqt*54$$}HYy;&@@P@f$}a)#CXkW((#olE2+qr~H<Eg_7oy
z^`?5ObCs5SYfy<>5^mPgxlZBZRaTR)tr~KRE0j!X!$ggF42w)x6rE96xcQ~Y8J8T@
zhK5$<H)-i=E8d(oWhyvhdcn;~Wj1@Z>A}ce(?B6}jY)q6lvn?;RarBCwQ1ynVrBWg
zvL>M~?<>qX$f~-rZ?<9>>vXv=k$zM2ipL7C@5HJ7Y+Is|Ey<udsdJ9fziW%-e5NNT
zRquCFm5Y}(v#+*MQT!OD67`TrZr0J8s&DkFRjpNana*PLRn>nLZz9*3ta{`gx9YS{
zJ*qd1qm>>o?=xv>EKzQ})2h@PcEd#3R>&-8%|g>hCml^yH)m*k(Aa2JrnOe_&zakb
zUc7VVpD0P0Z8*!TJZ*u!YHOmjsoS+v@=^&hrpfk2N*-(YOfP6oGi`WVZoX{KF{R6Q
zx0$qD<yD(=L|DaP$9csR<yWSQ9%YycChS%T?6)#iUfpUUd#X=Gkm0bBVqmzc(a9%f
zk^hz`cW2KrHDq0?C|X&fYAPPBbno3<#mF)t6L)QWwG)NwRdf2w6_@mys#d>>SBP0J
zD_?U+PUS_yEM?b~<z~g(d6Yx{x=6g3<FBIAD5mn@n3S6A)~lw|LWEQ%e%@gk9P&>&
zrsKZygUT3{3v9a-{pydHS+D=AxXCU;sYWf`G_P@rd3MnZbB(=~X5D80ReroMvRJj3
zUH;wPSqdx8uaN)o$k9Bo<eTD!o4P7zzSddfHc2RK{CCJ)W!X#x2Crvk1unD9ogVd?
z^F7?IsK>Cwyhv7FwXVTKx!jLcAuoTAnKHkbS@9b^mGim6rr-W{n!k=bqj^bXrqa5r
zUzD5o?K5kuy`tnL_1QGbR$lS(svwo?T+Yhj&nL?}e{L~dy8Mjt);Tq5cWhrOIO}Pt
z2o+f<>{NLm|LBsak^`TiVvTaV8P9<@r3dasa!+p_Q=W3|hf><L_p0Cb&os@Lo205b
z$=&SvjXkOYYeZCSw|c6cm2_8n#`)Vc?!BY3|2zq$txE4qw|!GFJ7l@i#OJWHN$0Nx
zs^_wM%%1!bljpInQqnZ=RQfPW!R*B0R3#o>2{oBdwPusfgeuAk_82qWEmv%t&TDew
zZiq=h-v$%y37v|63YpB_2sEkP&(Kg^Ai<=jT@-3^E;7{Alc_=_&^64|Ww({t+@JH*
zTjxzyx$`SVC4+67X~`2~6=xfDlTS`v3N@3btM;6osJQRmdihP#A543`rzmXd`lrU!
z!l3--oUK~v-KmNd#SQXpv3g2dwDpvysJ=0+S6isGXWJ(ExgWME<j=mP)W*-D7B|z`
zbZw=8n(!Mhv+mv6YCA)wtFF>kSF^i4RWb78MpOAar7F|j`lvL0*kN{4)5PrJv8fgv
zZ`(~y>Ks%H5o0t<$>mex{Vb}KJ}E@LMeUzy^9~u6n{v-o{lD~?pJ=(Fz_;1YLc0B#
zGP{MKc?~DO`Giy27WN9E${neb&3xPLsl}vSQAvt<sdX#hta(S6f!VPo?J5@^FEBBS
zx3V}TFQn`6fJwzOxk+t4d$M_rSg?xJ25oa)k4Z}JI91io&Ei#=z;{|{#w-Ezrk%G`
znsT@`84~mrr(T<^`ZVj9La2DUf<<MQ%38*2%0F0~%(C)sDwnIb%U$)=S7*DSqw-jI
zmm1HFV$%pQHTB&^ede=P^sBo~HCE5KqN5)Bc!OGP?;VpFzpa$6`(ITmvgxyM&NeaI
z7t?KCxI4>um&6q<2D`%+Gncl={jlAtviMZL@+-aNrk{!#mCLqyX_STunHfzDRGfbB
zrumnhEb2YAx6L^$*v)>n6q*Tso3HS8`!BQdIs=WBLUrnM`CD{+Z$CAwdER5;#kF0f
z{Mu==Y>p_?x0|;Z9N#ljEp^&ECHDHWW(T<a)R$klX!2o?vEqNW?drx~qg40YxFvr|
z!Owi!rBsDjZZ>Vbqqb^}g&rEa@(wB|wF$_(R5dF9Tl!FK&aK_%=2H);K07a_#;|0i
zR{xfX>Vk!*)!#)*o9?c?tr|aPvB?)+Ni{x|YpT0Ai&Z^2D%B3evYMS{Wm8&bHc#>W
zAq(Sk>Q~L;K3zBUbu~3x{P?5Fxie-K1@R$rH*a>TJhLuQE|TdqtG;(uX(!)3wb;mZ
z^GSy-74z%vnQjV5Rb2mkmg$vzJF^pcs%BDA{))EiSDC%+dZFg|mP<`iFj4+_o3P0_
zw=lD~%^y|foS19+KgHUVU%yY|%g2Q(Q5}<&+p-UuWUY!<G5^G6D#o{2p^MEz%_25e
zv2N}Sx!OCkOeS9Jm9JlCsLIv(LG{Hl2KAVvI^|zeC&)L>Xi*9+oT%pP(q?krrbkuu
z`(9Nw<HwqN`Q6l4bp2H2R4_8Vl2xFp<``nqx}`}~m*cF;x{xZ>m^aOe6-wt#CC~3x
z?wZW49J7wi`0Q&A^9z+RX3utYn1!56R%f$Nun<=|qtNWlq_l-kS8@HMnP%7SiYgav
z6I4A~+HBsjLS6AT|7NqPOPZCY`YkZKyM30~gYAaq49h!}rW>p?>ptnIw(1v)${f!S
zCDob#O}6>8nYk-3R9V|zX6jNWXqv;$uep9tvhw5D=c>PXgw4)Od#h~zc&=$*y`IvR
zBPpsS{)NhIx3?*(n4B?A==YS*Ypqu0yx*+or+iOEC;O|y133=)^9@bPi!~N2sl+>(
zzQ3WYyzbv(Sr`A^$_|_6Dsi>ds@>chU_M3hh02QQlg)p<PE%#okX9A-i&Hf;y05~X
zwa#?S1Syq6TnCjBd1jgJzCYKjx@M>0mi6CF)M}a3_jz=f+bIgkr?VF;bH6cHz8)fP
z{<Y<)QtzVx)rBFC%uaL8SM0cwq4y-TS0V1(WuxP92TeG#jLjeVl`57=wwS&wb5ZkO
zTB;K2(yfvH<CvL@^HVeHG6R)=x^m{1FTOUJbm@b(%;7$j7v)P-Qct}zU$iq<Woh#q
zqbyNH#kXr-s5-vSRc_S(EPo_xs%eYSMzJqV30m_HtW{xZxu`0#F<D8rDMrrTHbA*Z
z)<boTe~J0>GyE!DnJy}P2e=j0c&4bn=2lk|+QwnLakYU;W`T{d<{y5Q8S@2I9+&y6
zOna@O>N)q0iRQ{2#qUcOD|V;%8!!4f#q{)dVY5Uf4O5<V0_r7(9cDUr_Q-z-@Kx^M
z`=EGOKFXBk^lGIH|G6r+8J3&HNt~6R?B8P6lKfiA@|}p;o=6chJ5d3%a@)Cz;tT(o
zGM}+mEkC(b`C_?-QerT#sldKslLEc~711eg4b8jlO_h!oYp|5xP;%%#q`Y=Pvx(qF
zF6GOf`%E~apUdCeaa<*XdxOg4-S_0BuPigFdwWs&RGgaXybV1n(<Uja#!i+{VoLfc
zH?Mq>(rq0R)#OPVOc<j)RW`c&syw<hOSO@kS#6cZ3Ds3+EsU6#NU6FRl$ox)@2)D?
zzePp*m%i%huO}5|+wz+7Raz?tec@K&5M6G1;mI}A?*Eo%&z>-utj%1jzMw$Y^mBH)
zg2yRi#TrXz*~Oj(#%ba6l+UlXP+cxE%dk?zOrd$FtI3`_49Z&?^i48k!%d0==9z{#
zbt|Pani&0V*HHU;Fiz$9(HxcM*6ya$^>&$fDNR$py@lU2)BK@n#zJxJuWL>z%a=5&
z{`)Y?BxTM*<)j}QO=s;}qxeE9LA9bmM)~TDrHWQF#msKA?^g(T-=sO`XpX{WLv_{a
za7%@+y#@+%CUhuo@?N84waLUJz)Mtl|M@7{^>dn)x)yXO-^iC%Tl%-jbc<<~>Q2F4
zQzy+B)oj^RRgR>Esu_DZl#)JJo9>+Fp)%vRq>A2|7}L$&PfeBlW6dH>B#iysOVt*)
zhM85nC@B`L*HkQJpDyR4)M>ik*-TkMvO@KpO^aF110h9~hmTAT`a3Cyf8jQLwaCQO
za;}6~V1U1}>MS0!8jftWlcIA~l1o(7B1BqEAKwu-^)?bvZ50SL;R^e07ICjxJ4rcH
zd7=Lk)$jg{W}a>jm6N=0nG_#yQrZzYT~%*-r}FOnG{vN|yG)H@gq41kmT2S!-BFag
z->&i`=as_wC9I0Q1xuA@ho>q()nqmG&udY>KQB$5Z=<B*|8^^thc!P`*Y0{``eBy6
z%2oy)(`#QERPIk$qAb;zrlM5-M5(}ZyJ_RxP9^Wf{EB66SB%#l`DAwIwwv*dgde8b
zi<YaHC-GYNYiY_)Gd5Q;|NB?*vVFf<WkkACOMbdaNxp^o-RFxGmKfeOI@6mj;cl?S
z=s}N*>CuN1&G>Z}D=fU2X_me)UG>221m(pQ&8j=L)ti`ctT4M6rJ-`-Be&U!uj;0~
zwwpCX7RD>*MWm=)I_7AW>~E#KT#U(NlFMa<E!+E4dar~icFZ?bxH@N($(yf><+oVb
zsCl$pP&#e8N#)3i8H$q?S>$x1#Fg$Jj8d-tS!9~6ep^YvbBf|ppOXqL`m2;v-mF$#
z@Ws%yg<D5mI6K;GY3yuuw_O_QlS+)#^X{xub5yrCZQU)UTzW!UiGPij>6GoxrfjL3
z4fmgWXe4VqS#$Z@Bva`XSL7s2PO1b-GO2LR<}oXMGgIlpUnh;{7Zi;TzvNMBoff70
zbl+RGFD{LS|0Ly1&afRal6!GcUiX8N$*0-I8eit+sW-~1sdu=Po4B#xH(B(4yUNtV
zH%uotYnYxDJD~Z)c!OHXNk-)_0WoH`EyC4RF4h@&Y`U#rwtJttW<ZW2x3;C6vgZ|(
z`%euN)_tqixckFd&E45s<6`|Y<(aB)<<@V~QvO~2UhP8p8dKK#-Kz1rf@&XMXe#u&
zcc_2+bX47F*>qE(CSjEvw_D~j9%ZToWr?ZSNJOc8E_|ct%)8UfKDSpnyZ4OJugEfU
zD+gV(wNC72CvEqG7Tu}W9hqw;z0O!cLQqMOVST3J_smwaRV$g5=SkJ6OxRvy{MxWo
zf!}kT;RKzhipv7KjNg5@Vrp`n)$DxpWyQ9=M@&AvO;G(mYq>J(%RTB(9!xgd$lYVM
zXm7f*^lCNplV4|=u_$fP*;SOO?D>ybr7?7?d3d3g@`<wJrjh+$<g8|{P?@#FM#)5q
zSAGY-vAJE#OF5^<uQaOFwUj;tZC1Hhm8&2uvrF!J{4}NJV|$b)ZfP)odfi$n^gXZQ
zj?MB4hdD1OmQL<bEqFJ@WTk<w>VtR_v-?*(RTq8QuhKQaPPOd2u2OyWN>dX)eHEGf
zFUoQ|Jxv=^?wakhJ7MgkUT<paI9GMz8E*3zsa*<A$p%V09vxMjHgk(9Q(B+$BLNN7
z{Y@ffmx5gt=X3j+a2#E#q&SPyg!4zA@tw_vCKGy2Dix`IG`0V0rFOU5O69@QSQSm@
zB9oM@rDhA(FHp(cW@+~K#UfLcYpoimI}a#Jd}vZlP^vLDoA6k<TA;$TuKkhXr>k+Q
z2NK<t#ms6H6oihLYVB`O<Q8{Q+gP8XWbA!eWy!QE#rd(43gWfC%6&z5mF(7wnl$Jp
zDhEuISD3COsldi1t-N%Gq?(ND6w}#}1!^f5>P!?So>N=TU8z=b*iS9jJ6!d?!&Vdb
z@=Z!2hgp?A*!(r#ys_F;Dap+^{O)q&v)dcg%sMZdE)+MDf5taSIY4ZW((T9=(@R%$
zm5yX~swWjhnJj;JP+{3KCqveQ?5cAr42;h#tufj6e2$sP;^PX|Grt+vYBi~Ei(0Lg
zJAb#VxwxNcqo}dT)w+|)bJf~SHrxs^iMTXRb;;H3s{6mlEAzL`F;o1sLv?D-5u*Yb
zdBx1rC)KR_7bw1ZwpA`*&n@FEdi&*ruTIc#s1{YN`q802TWhfrV@#f0$Kj(&aSJA@
znu|-D6&&hSnJ>Fng-OOzzW$qm+EWfm^|J@eE!?f@)!dr4n9me_rnW{pUyb!ciJG5X
zkE%+dqZyyp3Z(*_1B#W7vrHw-nJi*GtWCHt<(vIKa8J!SRnt<&O-oKM>bUZ%cVfya
z@*);-k~<V-Y`WFGAFi{II4iCA%Hh6I_3e$SL4gO2pFc}CQ~d2=URe4>?uf3Vxi?>{
z`u~Z`)t(uB(Kybq$t=%jqxnycugZU395auoxMceBs<U?Erx&V{zK@jrrFbn49Ewug
z>f~d5Cw`us+Eqq%nY}%V@oyK)?0$d8^b3one89~mn%f;TRnr1`)laRzrDPn<A*UO$
zL}``(X;lNi6!Z4|^Hh2zAFAAIRZ(2w8l<+lM^L?4?x)e)U6LB}E2bJT=2mGesu9zO
z*U{DZS$bOiC&yc34<SC~?Up~4rq-S?_SU~(QZl8&M3%SL#Jk^6C*LW?Y|@Vgd5P+?
zs)>7#s#x!yWMVdtO-1{brsfyVHKv!JolyD~G~dKQPC~=(%vuw}c?(V6ZD2C{z~`t`
z=sd$@QH!)@0h7Om*WXDx+(n_rwQ7wfcee?sB`&No*7-TX=v}^y+LEe1^|<*;D)pxO
zj8rD3Yb@_-GIq3`rr001UZZGHr&^5ZQTZ&3`9{0bb|}t{`lPL$9-_X`(M0oE;%k*&
z-COd$i~UuuxcyXjNPchR@!_GG$UbTH?;_?pcV92mP+9&{!}4{#nO}Xfs(+E6spzB_
zRn<8vs*xM}RZmBUC@!}NFfCB*S5~+bsLX7^VDh`{xmkzjHsi~&Zl<cU4ApvPzc9aW
zVS{}AEK5bbI(`N1>KSGeT&5~*GPO``*6K2kpQNB@oBPi2hjftA-6$m^i>&FUJV$n!
z=txgds+pH<HbW>}E!QYTB}{ywI{(c-M(!6m%}hUhRSBE)#B{<oKhus+##&eSRFret
z1l1a%T1-njt||Xqde-RK!WN~(Q+2ApbNZA|Cf!k765MIBsai#GyL5zF#(Qf;r}M%p
ze%C`3`m(1fG_P_{-n~p)De3Vtla*T!D_!`%QrdBMy+*wqi}J_CHfoO>wwayWXruD2
z^Q$=*`)-vxkxD9xOiNTYq@Gk<wVK1M);>^qQGT7$%F8RvzP53gNxhzAw0uFl@xDYJ
zjjnADX2O~K<T#AKC>E`op*Y?9w3$$#xRN1ffl4-Coyqk}GZjp^J@t2OuT}hfh1F>0
zZ)=muveu@Xp4BSuTGwWh^SVuS{q169!>K3LPh9Rd+Yvd}?B7W{<<_Dg^BsmNrZT@f
zbpCt1P@d-cR9WxPG;^O0CKZ9X4aWE4dgb>oVORYVF<<G@+<CIfz5!-8?Y7Bm$PUpc
z3{_V8X8TZO;<<E%)T69&A^D9;%&pgzSU07aXIY+9d}(@Ip<B{kKIYXc#oO!jR3q0o
zn}wGtt3+S9VbYZqp|Z(9MrEIMt4iw27<pBJ@1_cK8kDB~OjEk?Xs2nL;Tf~IbE{3a
zc<GtuH*>0I{$6bUd=9TdU2lqlo?@eX#pE6{vH$u?67S1YPWLsLhpx3#xNYifB5A!q
z@v4W4iSKL`v(KBk%n~{0DSF!UnZ580P}M&@U3vNf2lZulW|$n_By1Y@#YH)RZK}!Q
z!c%6qGs<;$>clEPa|looQC?x@bx>HjzNF9Ow=k39h1HW(3U~P_J-1L-xKkixcDQ4^
z{O9sejSZ}3ib}U;sx0f3RygxYPQJzRg;KqUoYHYEM^oqi?~2R6td!OJX0C9+VU|)H
zhqUSi-!>z+*(|D^6W*Hg7sjeu{Me{+q{>wF+*)%5O`Ykc9&9Gc+c(Ts<~zU1eAW4#
zrkWme&BcH6o5Vkupw7BJ-h6McqM}WUnZkjSQ{}j~lpEjB$W=Z$HCmNx(|(iHB3I;-
zrwN&_?zK_&4moZ%fAwoq<2C2a93B}fecE!t^slI{+Tq4>mGC`gI`hBHH(RgRX_jlL
zt9&+u#q`-L1#_qSmIhbqWL2IlW>%d!af+Gnfj`Pl5$a~E`R^zQRL7~_T02{1=KP6@
z+AbmH=aRCOJ{<SgtWRFA&}h6>RVlJrf$I;0{P&};m3@D3Do?uWZ}y;2P<g?wBsrOt
zqDme7eaeC6tZFMmW|_7ZtW!CxwZ`n-oO3FU+bdOStDmT>&%U4#(!*``)bfJT$t8Ts
zW}l19-bomkc?emWH01v_p1Z0`Rp5rF8S}Bdassa|DCXTLS17vaWVUSC6eU%+gDU6N
z+&8KFwNGK;4OXMh8LEnNG^ZFJyxDGYR@cDfOw}<((ZC#&59}9IZ@m7kTrAI~Z0B>$
zB*^8niBB`X%96fyrt2ld%((mJYq&=gD8GCor+T~E%4~w=4COkNHj@Q+cPLsh3aEbc
zJFK+9iB)mZYXdX=UAYRsb@!@!Hzz78&5~8woUEZ>P$s6})4WpYsJ)KTQ&kJI`E9&P
zS59@xC3SWwocP+Nbhq)C>hw}(quNw2)prY%O<d1JsXnP^R4wDoQvH1}Pl0jm8I$u`
z4a#SpA6E`tHOq9{ifbmv!&}W-JRTYS-+e+|a)p73zon)8vC}b%Hx;eqPc6wdzTRl1
zoU3H0YVGmfWM}gp1^x>xW>#W%m2@1wnQT=yG%Y`-VCGObPwDwbMbq#v7HZdWgH*Q0
z8LPUrXPKUqUSOhK&!IBMX|7RC%tN!O$HaAt)F&!W|M67St6#?K=iwd7AOA9%`bp|5
zNd}gv?z^>4IckN5(rUxIrs@i!iaw`GHIHa-R9G{2j>`SlC*(_8r4@Akx+}k${7!L`
zNwe{_#_dXuY-Vz2U+F5v)b3Q;oT;m3{%x}HCbgMr>+@O69Ir8}Cv?nEJFnHJR<Uco
z>Ize3)2r4BN<6RYlp7Y_GTm|fky)0Hq}l$M>n4`(539vEDw`(kGm^dMcT@Se%S0tx
zv0{^#S1puPJlCmPU+XuTcJG+N^Q^hXZSo6M7ayBsvgqMb6C2O3rfyq;6xg~a8Z%_A
zR`1$$TP=n+QTgt^Leq;iC8nu*Ix0I=ADO;aQ#TFOs?zvS_e8a#l11g7UZTk<XDc=C
zdTq0W;<xfvcLda3?4^`u?e$W)TVrk(*dnVq<w~?h*lknQOW&5MyByxFbhwybA)&QS
z`OJ+AsufdanC#hcL1ohhc2(xWDGH&IooX!;eyCl!{@>)7N3P0I-)OU24&PJ+ONvy!
z?!KZD&!D05qdC~*%>qtkhi#e4vn88N*KFBpx_)-3>DBsqCY);=HRi0DYueFSrjW4V
zf>P`1H;T+G0;YFM-IQCGsi-RKe{ED&&LHO_C2xG@$0bGS7f*~||I0T~ZP{h^Vb2_e
zbEhX7YyM<Xn|n4<g<0*0T9yWbS-Ts9nbU<_WxpREP4zU|jGZeFY1hiVRyKIBNaa(d
zwV6@lE@i#vC8mE)S1TIa^jBrG%u(9@KUhItaii()*=?$g=Q}kr_^&F5?QBpjpJ=W)
ziAz%M{!4e|1&a$*IHH!BeQxYmK9svrIZJ1g`W^ZIDs%7uP*vY1Y<9(Hx~lbWRkJ5N
zk5#27wW}uH5LG?l?V#2!-(_m=)1-9#w3%W@gokNV$v>0Cil3&+cezX$&dyO~y4_~B
zY2iINkAeu5*@g|uVGp;M9$(w9lw8-Zb|><wnePEZ#k;=Eru#X1<eso_n(~UxGYQ~$
zY`R7)M&X6n8dC|+S!%oXtySG~dxxsr4M8)3`vIm~rK?ohc7HW_*r#hYtz)J-Yj=f8
z!ND%&zdR>R6Rdev64|>=GM?{KQ0HS%JCpW6L9geJT;I7hrZMl06^a*cR?XdeM)ls@
zt!k``s+AA?XO{ETlT}(3o}>2PBf`w}cAly(?`+j=_kPJek&;lio#CX~?x|&}vSY7`
zaXqv7d(KBH3-k}De0#>OT6a2H!S(oAvtNe)lxj)>l~<hIZq6_>)J!oU##~vN)zsQ}
zjygy8Br`(^2Dz+htct8c0t$gkPMX$TouR}aFQNL>%fKY+%VPQa2alPuTw188dotDZ
zMWmQnz~`@K&+7LoO4;<9PJ6?t7JV>SWm)7;wT_>2%o<+qGd(p=L`8DpX;Z0d2hCT-
zPSD++xlh?CSXK4b!rA7UyepK0xT?)2{S{P9f1{`RQ8!T8G`dDXqVcf#n>l|K=FITc
zbPBLj<X}CeBE67JLApOeUe58mQZ>T|g;@b}&1NQlSFANXATN9Nj6&>5Zl&Pcn^d>V
zeQ&h6;JW%OH&bKnPbM1o+8(HXcKWDZ-Z4QvSYV$~?Qb#Vy(`R>kG|e*oUJ2ZBpSET
z)HmXvQP=}L?Q7E;OqdTok>_@rpvt~CR^?Zom`Tx<ugbdw|7cVj7MorXZB%ld!C_`L
z`JDQ<Uy7!@J*_4mEoPerIlC%tbrvyEns{F$<*BsBqg%4NceD&lb9Wpw(&wL}TJY_+
z0bg>9$;v}M8eE&*)jvedQ@%cPn#osADUCmxtBl{u-BJ`NuhQ73a#3~dyUPk~`KOGG
zSo)N%DDTy>I3%iGeU3r%_0#Dp7yG0XYM#ZZ9Pr+*&Ziq^vh4OewKrd1s&U%=(7JOd
zSHrqvrp8PYF*CkB3sp;h&Nb0=`>#5wzg_iuzl!R8??APJR~)9!S(-|F1j-d(%Q=~J
zHO(+%yVhp1M8nk7Ox9DS>i=zXhKgocUF`xD@5$N9yzV|`p-z2DU8`rQ?XHS2=liuo
zarV?-#`{!0$yX#QnY8&=nx36-)<h)zmBLvE6|*a5%hhJ@+@rcKr$d)#<76Yw#Pz0D
z;`3GZtD2b}PCjX}d9$uo_moDJ^+zWv$1C-k@U7KXdA%XiIPdN!g^9O#)S7>KsonSE
zmG4pTGkGOgub{u-p(@XdOR7)04ydIU_bVU&y+poXzp0Y)(N?t$YY!XW5-wN$pFdyK
zHb+*mVy%w)ibs=Gx1ZQ+{5D5PMLB1asfWQbm9OjbR7~bHs+^E^Rh^&5YC4sxN9iH&
za^=%e+f4KB&oV78oMfVOq{8_1)9LCfLKsa?pJkHYvL;Wtwk}_3iCdk?^@!g}r%Vl0
z8|#D(*T~x`C_QO5DD81nkUJx7yy#tnvEZA(M&06H<uBU3H}JjmO7$X3gUT{yf3>1}
zb|%g9OigxK^eMl!xoq<P_HUCF|CVXR-v6RJ$0J5%&o2g3)>-qF?F5&aY&@K-=yN$=
z^+%nb((x&+3Jf#WnD$8DP-t}b)Z~o5tithdg6hRHZAzl|i{uxEKT-mXNB)$#X~MWA
zU!}Whok}6=e}&LU6}4`Kx2jgFx0)&!CaMU`NiqHVHCLtZ?^osA{nJ#q?(nH@(|%>*
zbm4;1-&yQRW>HH_rW+-h<{W7^wcPmFIE_P7HKRD$>|);(MZMxoW#-GNO0$}IOfPqy
zRhrH?StXA1lG!P(+w$zZOHI4g{>pF5tTeqP{mvv;E#GX9RHtI!%tNM8<teI1PE1ff
zwWd^g>wXclL$eJ`O^f|i;+kigWKUjhdYt=(>c`&(N=@^NRYLZyHhoi=seCWU+{Dhh
zOyR~1PSr<Ok196n-Id#Q)82H#>^}KLN<6BZ_S00RKK`td<XfZUC-qFWm;aH{;x(SC
zYi3HDWi>{soY4qS;d{xhSheGhDksAR)tR?BO%<*htBM|FHBnq2sVd^WTxCUDqw4KC
z7G*{u1=HB5*-D1nqm`0*4w}sGS#IiHxyMXNSlU!?LZQZqy+_T$+O-r8E)h^_H`pu}
z;>2S1tkyu8WuCIC`qk&AQFrYXc{<9?od0Vn{Wn`{YN@Vic1+mI{LtlArNypSP5ir=
z)p*;4RnAM_SB&vqWqNwLtm&WA^Hp@icuaSld}zApp^KXS31MYs%_}PWF*;_8v>z$A
z7$unfIKZjsw6;LC<M2_XyQ`Kfme?2?&ztg8)hp(p8Y`=((o*j})hWzt<lT3D1>a3n
zy>PX1MQ?}c)5-%%U(QO&|IHCsTX7>m<<YwdYEG56jrTwOuPn4q&vawb4P^_v<;sd#
z!72qoY)WVE9WhJSJF6JK)?G=yPTy3->AU%(<9E%AnR?C6Suar$d&XwrFWD-)bn9k?
zc4ICDA=Rs93D=sG_8Y0F<gI#XzBqZRy#H23i(~x73Rzuy%&iT-nj6eaH&?9;QM|I5
z%i`DzQB|uPYvrdrzvQPMsxu8U`fR>SQ%hyZPkz(qv1R6w9gI2+g_D(3D|?k!+?Z@W
zkAqS9%z9a~&@cbwv-bI@^xRicySm6l?$O3uChPgi<SWiiQJdH6p;*RRpz^2YuKe9!
z`{gP=?o|A3D5TiaWNo5vW}(#Z)J^(P^9;2jZ&l?i)&@0OwFZ-ypIcS89b0Y69==ai
z^q7L`F~20$;?HjsW#tsiZdJ`y{-+nIbZBY2>96wTX4AM%n#wMjWzw<CUUj!ir&)!1
zh~l2>I!byu`xU!e{+S8}Fe^V0l2$X$UT-GibV?yqcekmFON?TAnX&0Rk#(lLBJ!pW
z?#C+K{k6pGvw4$RzzRLpi<z4gY&}9v-!-<IUNxAjGS|z^^rNo6*;CsY>Z}*+RYd0A
zQQ7o7!SwS1NtFi;6HKJ<u_(^?I9;`4eUj3PzwhMpn>Lv4z4=sud-^1`+4bv{e!bUL
zo4kFc;u<3n`3L?A%E~EP%3@R8Oc!iAt0a43k*v+D&k8}s%F0#=s%l<ynT>9Wv#J#=
zziN7y$3QLkcD3p=DOokEk`5&uMmMt$b2wDe9X(aHp5!$Td(C6c{ARaB=hh3xS3|d{
zP1cYzwK}Y#V7J3pF}E;IHdErUN&io2m6qezRln8mHe)|!t8{t7WQ&!>UCKU3?wLz(
z$T2?^vCf=hRj2Z<B?ab}XJ1s4uf3!4sEk`X(4^3O`^mZHEFC*kmYGa6S@^Ko!Z?Ff
zubb0YrG2H7no`AOb564=mEW0{%+{^ArKBw&qZV-Wh3Y9cKIKJ|bj-i6X;;V!;?&g1
za93RE`a$)|Cq+e(e|m}`k~u0z<3B3jE%r0v|G7!I=G<DjeQ$cz!}e}baXZ|i*8O&~
z8H<{ndb845^FK=_s85bFR?qfVQQz};y;^qeRTGvMTuOpF_bZw9Hd<73NSTE!SY#H`
z`r62e?}65}s#g{df@EZ)pKMdfQ8}QTZJ%Mr;oPm%R^p}MV!YCfyWU%|-tv*z`%VsZ
zh6SAF_Qq399zU6FwtUT7`41u~X0}ZR8V?TFs;lR;>4bOuGh>%5wb<^vK}CHPgZae#
za+4JmD-85MuT?v{V4qUao6Tl_D#F#H4{SI7Q4^?m`TTZu4pnRAG<_?1R;y(5Lj{6L
ztdY#x>ps}11qHclcuE~p)|(+FKdnYu*}CbWn%+593+D$%R8?&y)n>k&qjkEdQ{A`q
zn7X*NvdKw}tE!d1;*1m5OQ@9wTvlC`lCR43rc&*#&>_=D+@BS#b}UfTa$alLTz=0?
zRK(LP_*a|R`IR44es9;aNDy(9<&<tw+1EBv`IUZ+S)?$hGTVx~YQ62h&8jabD#}^5
zn%ykVSKNNP#q70UyV=j?DQ0C`8WfgYPcb_n^-N7Mom)-zDyJOJ#UsZ5K1Z4Tcz#^v
z(wq*{ugkSe9>wm_c(v|;%2%yG<r8~2Oyi7#R5nkTZTctAPQjViMs4EK4wZXmKjcm<
zcQdvr;8%D(k4rUF;H&CCVP^H*c`eG10$OEl)oqkMv`$ouf5>lgf}u-w)#@Xv^A>Yy
zeGRlyFJxv>Pn$BsWM5&uYQZNq<8{#!RCm38s&b#FP4$YSrs4}1EmO((!^$sGn3Opz
z=NT5Zy)>KUveoQ+-~_XcZ==;4`xGp!ub)@2zxzt@>}Mtg@kjY)ucm7#|45Wpov6Fi
zY+*j9;_g-r^H-|`l#dm~ne#6`W;WeP+x(>e3?-kL4Q3oq?bYn-*j09{o3Hf$j)O_<
zw@+qm_B|@sd+JO#EaEa*cw(>Sk3Yi7zb5@tJ+{usEINC)@)A!;v%_viN_TcAsXl#h
zN`>29LrI&l!szdeURB*M4XT@8zE<$P`Cf%(n}k9Lv#$K6$@`V7raV##c6nzq|BSox
zVnrKi`&nXY0{eWF8w`2XSEwbK&k0LWm3y$+tRpC2HO(?xb)Lj>Rb!u}Dx6b=P3JoJ
zsR-O)Q=U{VZ+cvJxv7LwqiMgei|NS;nrerxPd5*3wo)qLHB;WnJVj|{!dkOQGTzEH
zhbvVJYJ<#g$Zt}R75HTG^98@s@hSYKv%CvTs^vH=taDZ=y=load7JO6HZiY7MN50C
z;_eH==J#2G%sz9rs+>*OXL@(?Zj-+WF<Q)Vhm;Q%cBv{$GF#l1(NO73xL{ngqEm6&
zqtB`Vd)_L0cGN4(+OymA0!N}+qw`_SKl>-BFdv9jUE?03v^+yV{+o=5%C?X|RXeG}
z=JVs6R2=3As7$WmSCn5oUzOP<LT%li9ftqyl~k;sav9eobEz1`omSSm?5XnKQBzfS
zONNQxGAqR&PG=M^J`ymN;5ln5q-<j5+$ChH-6f>1voF%D_7<n?#;QPN9W_~{#HJ*Z
z%K}G~&V)`^xp`d4Z1NKxg;JjFW@Z=TWEF!~nN9T#GK<#wXxb4GqG04|Y_=-SQk8AZ
zY-Kk)LzSxN1;*u4zGf>nvZ`!zoM*U>>9I+_@jNZnKcdR#(=(Loy^Brlrb#I4>}E3k
z-lHMEC+euma)t+LH*EjN%P(&?%nspD4x4#T<#@(Im6`8MRrk!5Q+m&MO7_}TDJ8Xk
zrmCHLxQtK#vsSrfo}yC!!BRc*36q-R&QEH0|8f~EzQe7Wv4z3(8n3l#kHRDs<2)VJ
zN%PhzR*Q?6&T-&Yc1#gek<8y=nm+M}sbhVU*@1;GjhjzQQP<$tF|9iKNj6ovNMYW^
zWwO0T-x%jy?pJ1aHBuGko@#WrEnCs}t)S_WLorI<556+l`TM3ZZ(5^iS6!|W#~~Tx
z8(a!%4T*Uw`o`ZCm^LgjjZ$82%p&tmd3p6|6O-yDGn3pdZO<dDDj#?~Rev=xn_fyk
ztlaZsiRn>$2Bj&jUaIE2-pbFXolsmf-OMcO<9+!>t70@cT3i)#jFMIN7V{|nkM5P%
zy%nT9Cvt_-Z=OTO6P(T|O>Op(>7RI4*;etNa`v3<s_hdenH&o)SA8wJ)Fjh9S=GZP
zNOf7meATq=-<9$sQ%&bD*r&`O#Hw=i{zlV9wL)fXH51H&s@sg89xhVzdBS3*ncbnl
zXfQ$HR(!c!{z4Yhv_x~|ActDjOPZZ#e?H7p46$)AlWyOnl$FMAcISMH=`OYe(-S9(
zmH$genu-5QR9jv+Q>F1)u2N;m5>uxMOr|1So~r#FON}K(Tg>+N%j+DuUab6l@@mzl
zG$yl@zqgh5#tEA&-+xvqf^mv!`vf=TV&QbfsQOK&QghbG<!1j<Kh!R$`2LEx>iS4t
z#Vs=@DA;}7qCDgLRAuo4{U$5dv@09bFvyh(Mk{f5EmILOVOA6Vn{HaMvs}gK9=8c+
zUAxLH(LKudRhv}iC4E$4)Shc9(7a2D^Waj&D?&ZSylb|baU7I0eR@mFY;XP&6&`7E
zi?j{aiu!R9N=qi!DEi)AY_=fiv(i!TMis_g24>wew#jQWaGDB-h${Hyd@?!N)L`0L
z%xkWb=AgLd)_2nxipi=9Rt3s2`>)E|ZC_`y$gj$*Lt8`T;BFzaRqy0X6kWA7y8W&z
zeJE^EnXAxf<`-h5{P8)9Y0Wt&1;(r%6=Aj>#bq;E6*f&cWPGVlP`+DhxoV#24kfn4
zc`6Noj*2QD4#_ru+p5&<Xs3KW`l4x+0H^Zr^)nTpuP{)S)L5&$_s1*MeC_L|m&Ekd
zCntNERyEI2Z#2|WZ;>}p-x9V&tt6b?lv8b$@&;~BrC|T_CPuS4O|4f}nC$6pF^VrQ
z(^RsxFim7yF8_Sr5fy)aJ{66upC%#Io0YD;b<(J<GBf=);hv(syOi;smk-p$K1iF$
z?OScE(4TI4^k}rAiamqLg_|ZC=M@UoR|`&8I5VTy)bi4K6A9Z(D(a%sO{OkqFg+8l
zr=dD~g__}24wXfZT+Ei)JE~WQ>@)g#k3;dp@dN4~mar)QPnMPDN`7i`-|~jSN9AMc
zvaR-NXN;XSj=H~7zTh!I_ECMKvUSZPweX7frd->0sJ4~!sYRCd%6IjYtFPO9Og(@v
z-%R#onhL|THgl`p`6}Tx@hY~ClT~C=mMcWH&o^8Ac)s%5Zy%K0ji#H|2mLS&dL3pK
zG2?*AWxq3O+b4CKwZA%{Q1<blg4F7@@;!EzX2}xF%GD|@Dr**hHQ7^tR{n1KZ(~N0
zeM+A)FPLns$TDpe++eo(LXJ{S^C=Vg%~`5*I+iJK@Vlup;m|p=bIx*RyshQRuF2A7
z%S+FhmHx`o@mMFReE(FF3QzPV^J*D3W$}f5rfgTV6~0W{pz?W9zf!u@8hHn+Uh~t7
zCa5||UeU<<wOVn8o2yEZ*=za7&8!OdCBG}3iD6T|WLR$A_;aGtcWxzx)6>iqWP+KM
z&n{6_m3(1ga!JQa^|P>_saB)E>VZ$+RXQ7zR0A~sDpsgpFs)!WQCV~CpK{aQPLn>x
zxn@tN8kp{Tzs_{YvU#co9a`p___ry#)NNPHyX>yu!gR*8X8LsH|LrEKrK!QDi}ub`
zQ2J$Pn!*;KoF1uYy3_o;$$1qS(~CCj%1tM4o64(Os-4gARCy~Os&quz(^RXD&&>SK
zDiu=^J+sQ?Hl})6-s;KItCin7Em8e=#KgGK<*f3&sy(JfMs`YG1#zlrdg{t2*1lF~
z-kf8qRJKB4eVCZqi-{?UhvN^cT*^MC(C)#Ykf=3NS#z?pvQr3)3HP^!%I0=AWh+Da
zl!|sdSGGBBuXcK2hpCToj#}UR+s2B`=hc+g)TkA24^oqR7OZ;D=a7kgf1gsoO?#zN
zUA`v!Lt;%$CI^^q|Gm;U>w1ye^0OyRWyRzbP1>d?hpMbly0B@T={+81rDqpA)lcyL
zHoYs>tf2mg)%2eFUzH^~k|ykucTF5ETFqEnrz%RESzxjwu1S5e$!fJ*M{mh>F?E?9
zf1_mF{Nu5*WfYr<QKqozvHh{CTU6Gm<}ZGw{7fay?Dg(Vs@$I1CPj0L6b{IpR!e($
zNzpE5rR?vH-NxIuPf|@>xJ+ZyJ0{g_8BOY{sq2))SJ%shoDfh}el%aT?B-|FAm%wL
zK1WulR9>x8?EI~(ruyDYT`oh?!kVK+%`d6VY(v6xHKnp5HP^N>H5<=HRhNta(<m`t
z<;Vl)6#MFLn~0crnkT0%Gi7BvX?87Qr<%1Ni={*OY=tSj+m%)8nUp6kSZ#iH+XN*J
zrf&7H?86orPWu#|GIf~tcq~(mnJ;Db*umROEwInx@%?6niPvVDu{yV@{|jHOw&te4
zYSCMB^9q4Z^8{Ocl`M&=W>YqBn=-BS)C~G|Qq@3ojq;iOkrwT{tkoV1w3zHHla@c*
zz@%<;aFLS3rx+REja{Y}@4Zu+c9~Bz$686XLZn0eaQQwZtI1|^*8aN6f3!ENMs&Y2
z|Fm_XiWko#l@b#PWp7&>HHB&F>ULYzjH;IkX}mY`H_BX5t)aC=R3qM7MI$2RxO)8m
zc;h6ORm!Ez$CViFFERcuc;4jD^EoDIddEy2-?z|-+UjSfG2w!|#r#XEHuu?7x6g|(
zS(5Wk`Cg-@=B`{7)6!pSlsHy0n@&lT)R-d^W4d1Fy2*1T0khST9~82Df=ry>NNd^{
z`D;|Zo~HAM-O2cm_jD8P6Sq`voy|0UpgrC2_+>@4dAAm-w|^E<iLq8S=Dt^?G0pj%
zvCE?miqo!a&|uwkMpcOSla#EtlhM4#vPulM-)Y}i5UE~cZmOBL=&g#5jJWKr!|#<f
z8o#P{T;wqJVYs9AJzr7%&jJIThpVS)+`Ru?qpF<GtT(z))viy}BxPH%D&uDjRrR1L
zs-_d?C@zxtWl}b^S=r#Pg|b_)nDI8%CNrB;g(gdXFEo7|tE=`=Wxe^cI#>DC6~`3j
zp6`~=bu%~Pmp!4R9_FOFaaN1D>d*HI6YMlh=IW^^t!wHtne=^@Y1FM^vu=egN-L9o
zo1WziR{NinreZqjwaN;Oy(YO?GG>!m#Z;fXerFPUAj!l!W}=3LIJ5G%c>-#CKg*d_
zsT@`gyxV8&r@LQi;iYob=CiKK^@rq@e&xI}Zu_aLaJpSuZJ)lA!amb}73q#+3exLa
z<!A0lRIZy}uJk6{!1UbDYf6u{&6az6wNPWP#XRMu3PEc1d1B^4f9I$)@AWZX=(Ag;
zK`c*Yd;ANP28s6y@rx&!P7rQTUMJL~bhu=`S!u>!ljWZjOpDeB7>jhUYH+!`nUyEG
zC|0j;RcLr<so*u?k7>JquTq0jv&xgz>1N66R?ADjoo;-Jk3)$~t;3|(w9Uv#N!C2T
zj8`e=mxyutmtIv_<9_APPyZCBon|x__+nu8cj7MPNq=6L>MPGTF<H1kyZi&Ia<T6+
z71!c^^Pj(_DOVqpG3Ky6s-SGbp=v22qFiDWEmv*5!0gZcGL`MqXQ<o%U8tx%wMa!X
z#!I0p<*saKkel+u$&Z!v`=**7Z1Pp=^q3>B*(4&zSGz}Pu7|Per=6{)f{el{_I6w*
zDgl8iMte9_4jK2V1aJSV5bwOt^#46}rKgtNN-_R&W|kpa%v@amo2KsRHf^imQlHeK
zZGJLLSfR~ng2J?&%<??9?9En-ZdH_*u27LGY%xEoqO8z<&eQbS{9wgRv9nG8glU`E
z{<JfTsGhF)Yt|gIpjt20?IshID}u9Cg%gWScL`~lx>vU-&#P!O7Uu6TTPHX{cS&re
z@}E?0m5iY4W}?pomAzx%8BclnMIq5_l1k=^eM+Y}0_E*f^v!;TE|L}gc}?AJ6|>^6
zoySyGo?55i*~uk8H}`;&thcYy`CDwJ1|?idg^jB6Z*>zDN{@b30$m@ieD{>`$`3bG
zwoXhpYispT-C|m$^5d|PD*KCViYwSwnBFU%ro`#FMtS3byXJG&|1vG*t21ZU`fSYl
zXrg+!N4vRm>VJ8e?jD6{Jfd>)S367&*;*(c$?{jtvzut5cF10FLd{V#sVa5lfTyd>
z&Uu=c-u`ZAHrL-&>GGU&rduV|)wqLeRkV&c>-42=Hd|Bw-qbS0S2;Da$F$p6$^1Hx
zoxwTdU=^+VeJV{FQ_Z4xDyST0TxUATR7WxCair?oa50sN6b;2m8yn5}*6Pb|e`BIq
zJ}XL5XTu%U?WGSDUjN=GZ~c0rvP%W4GG}Ct+4BD%m0BLHky9vjP-e*eu6)?<psMn>
zGE>>iT`InJna!G1FQ`0DDp1L-`>Nv0cv|81Zzi)nAu`H`p0O%RKUOu{e67z+-Rr68
zF6JA?tLmnxZjsV3%lNxo-l+4Kg1JhvLU6!4Q#Z|BN-=&1Rh-Mr%yvvjSJ;)PY`Q(`
zlmg3jZBtK=k0#4E|1jC8bzSkU!2;9#6K7R_2{Eat+Vd%VpR>xeXvQLwYyXo}=1oa9
z<<xL8<DKQGzMaWg`M_&K)x3ZFX7wT6%5hUSn1JTI?}o6c=JTIaO5$c#{GjM=_CKpq
zR%KP8y6rnj#dWpLD%O`aDTL&nmNR|(Q0f1-lZx(BUzjSc`LD<#6)j_Ed_ew9#v>({
z&x=(vosS#nIQppSon<r*oSUgSWe=08$Bh!z$=W>%my=$YJUAbxY_jWuvcL;{v)C9#
z)0j;&%)WiUX5@Q$hx&{&$Bh?y{gj)uy<frP<_x)g&&`cz@>?qBRvD{qYMo%*Z(*qT
z-(1*i{ikV4jF;}2o+}VD6%5WWiFMqeRQqd^>8&DTweII(D!nt9)IQ4=n#wkCng04`
zsZug~vXNy`zM1`m?K-k%3zXGYzEtHhbu?qSw^6y!`>pZv+%~1tt0Gk2y}hZzal2e;
z{sLh$-D^kWi#&ofXMERCaK3R-B~){r!lmOs<PPYCD1Ty6QhFPD#>jK+cBMr+hvlMs
z(o`OpRw^Gn`9pPnLWxmB^>nqX=EbJFSy<E?TxO_QXLqV~vdvR9t%x&iQ0i8yIo+i!
ze5B4)$aS^Zsi`N;mj2ppa!zZ%nrTm$Y2Nm!vM+kiC?{I3Q#x?A*F=KlmC|RPb?W)5
zY{oLVlN5xvS(}EkE>JzUm&x?~?`0;Qs#0e6m&}(JUY}$9E^D>=+tfR18?O5(+<SiC
zw8MkNboWbV73tnJrV1C_%;w0=(TM!_L)D{^TjkvgCX=7lR%+feE6sMa?@*X2B&dGI
zRzfK)Dn$NtiG|tvP*%nK1r8dlPfS$nKQB`MH}8{Dl2D=ix|S^E@Sanu&SeLUMc?dK
znc2pn`uD(m1qRI)waI30)T-BgG1>TSsfzaNP_wG=-zw*CEm66w%&59`2d7Hulg%aq
z@{-D>-*T1jc}+CkxbLY+K|z>V{m%lEBYKt^EYn%dL_W+>I5_c=lG+{*#ljjh)3xg|
zlrPHZsIF~JH$CnASMI?JP16)PKBaGNg(m*Xc}yP$t~OJ;xLa|V^+uyA8D=%<7r`nQ
zoyAm~HTcZ@Pf3`1?OUtN{8z{HH0xATzIj2~``ykc`~3K%!v1ltDf6dA$~lo&Os(XN
z6dx=NRDHf@fzryyHx%^SM9e<1>8o`e@6ZTLJgdBi$w#%s!c<XJxKW`pv|d@*zD=cP
zrnwo<)OE_Y?I$WfKDkgm@0Osdn#~l|xWA01`M0L4ewXMl^=N#oy60uLYWGVy)tO-q
zYA1#JO}D-XRB~Q7S<zAOyvezlUrkM(d70h5rEB88phs09v&u~P+hK*+_z0EL54x4r
zb7q>#?>?%O_iKV$z-bBdf`ESt{jSr@`omAlF-kL=na*M{Q(F1WG&8$VapS@1W<4P@
z)p!rAR{haGL6Q51huK!i2PQjsOI40`TA9?el$bfhUQ|23xLU<!-bUs8WgARP&;C%J
zx^9^XzsW|0j>{}+3wM1`;8>_9|2#g@^pr!J{9T3VDp~uEsTSYeq_&i=LOJYkwru*&
zbxLhaMQRmsA5AM>l&Ze0n67%CM_S=<C%^jtH;$^_?ct{HG;gW!^ei`f(fvZD@6<6B
zg;h$bbF4BH=CiFfTPt@?iR)3PvWhr^MdrihW;(s|%-7W1HE~<sq<;F2lbP2M4TXS!
zJ_Yrl0J*bm2TkWNWGH=MmQ|Ik?=}@mcaT5vz0RyS@~GlE^<1-`M>d#Fx$SQzVJELN
zZ?d%6$%Wi%x2$7T%2I?>RhKE5_v+p<@pR@@dBPTCI?-!``Ksh~x~uG(l^1`CQ?0z#
zXU?D1rR);%+4NIro}#Fuv8uyTTV-3m9)+2k*(^AkBNbD&b7|IDmMSP8-k=iPm!Z(7
ze@$+|Z87EG<Q7H6Y=1MGZMI5wQ>5e-4_sBS+3`hb%acW_KXRN5)<>LCU$}3Yao#;s
zjl1{nsoVa4tnR}wLES5n)A)PwbEQKDBFfhl78y6X8JQ$$T{Zo3-o{9e!CL!P+FN7q
zE7kIIttY7-37M_JU3koB?b~E!+o<0fj_FLMX}wKKDf>;#cv`NiZxEVe`tyyl>72v8
zrn)6Z6&!ZGGVz>oUt`^TX$|pTnz~nSRhvAqY%@)7YgPS}9B3$V?YhbFt5>xyolaM`
z4`)$+$Wd<+@>fG+w$KG*6B$XR^0(C*&SqL_MoZe|8DqnYPTmezbZp<J^}R?^UHc=0
z<`;(fDgtG5<-N;KE3dHGuKrL#+laA$j#|*x4{GmPerk2hOVrpsW2Q!X*i_RyaWhr-
z-CSnOR?DDfm)x#eI9ExvvM4~UW9J@|cEuc}Tdcnn)Z%^^e^}vW=9qolghgY%sk4}a
zO7w}9=H4?5<Te!Ns_cHWSh*=x-Yj&}45bSjXQ}nYc$jaBYF5m$KWEbKbVL4Euz=~*
z-_uMVOyxB5iQBH=nSIoBzR@zZ73+4Z#_4wGE;zo<I4)(E=||PsDiUEcOzO^@H~u8q
zqVeifr%KJUAZ4lj#-?g)Vk(XruZ^YI<|wq6^Qc*r^Qb--S|BH@({Fs*%~h6f??lz4
z=Bujd7fz_%wV$9Y!{#MxyMsq5=0b~_$Kn_h#f8PHp1g}x53akY%==efUDb1&YJ@GT
z$+vaRDy5sWOrP&rpki^cQ{{EcJe3umrm6v~k4<VAA1Lj5dsO*b=S0)>ftyX$a`H?s
zaa$QrPM)By+{R>TmnI}@?vt!MJ93ti+_5AR2FW){7mbWmXLd=N%>R8(eqYiiW0upO
z<by@DO;%ZLHhMbsjImKog96hz1EUGcUa96Es#bYio1|0~&uAL4GSOJe^@g%b;8c?p
z)sCib!lq~``TbK4aBNfQnS0vAhIP8KlEyBRiLFx<{oCVJ3&htb33IY3F1&o!bm@vp
zg=M!iHMhUHu41sXQFY>z9;MX<kLCV4KT|4eNm12snQKzJr$NQKX_3lafqM$YEfQ+2
z_li`tSlmt3qa#(kjLS^@FSV(B;nr5+Fq)~dT}n{(Y(b8xaom5UfJ#QC(`%L(YaS6Y
z+x-5YDf7oJ6Q_kbs%zP8%^v7}Rfv9&qr73Cno_2TxmmwDqw>^>DJsJ1Oy>K`v}C_6
z=Q4Y*W~A61`N{NI-g8rfBnvY~tuRIQGHtWxC*o8m>2xR`i8!g47p7|VRs4j>;X-p2
zjgz@1zt~=x=3G%$x7yRC_{hai<<_n+({xikW$%u6Cg-O!Di%obsIGdIs?<{aUH-|v
zdnU;;jH+J*Y*d&2?^VeyVp8>Ikx^RSvs_`{2T|o6xBOL`iZ+;Dz8|5YR1%`1)NH8|
zqI6Hy{n%4g9+_67o(tltUT^h{b*~t!P7csesXvmhx_`Qw@~y+4O;3pJP+I=iLMgFk
zqDjQ{_of1$ip<vtG@9I;SfUY8)N9sXeL=>R^P%F&!%_<62enQ8&%IDOe2q`_Mp}c(
z5$)58ZKVq4H#hE4JmR5b78&AZR`IpdyuK?$ajES=)8>}%s$EU0D))<aDL%H~Ghrx?
zGRs#tRZ+P1)@1%OJM+6qhqRuFZ&2pD9H#O<;ken}h-{U_^Q~s{c$X{849rnI8N5ry
zJW*6}^@IkazrSS_-<{p4t|XwRoT#-%mFewAh5Eo6Ila$Mlq$p4DgVD%VSN2-vr>>>
zsiO6U+iH)qT2(eD&sNKSBw<+X{!@A1&q~AW#8b-pqAAMPzIm$5_@bedb5Pgp`J@g-
z;XNHn3k3F>IxqWUUR-nA{O_z*v(hcfs?Cc8EYgy$$(0@Xrr@Qoq;Ngo#>`?%gOc}2
zWt9t3x6Ssa7|SaiY_L%CTBs-ycFw#vaiuxywl!w#ms%9=-%+)AXUL~&y~A4hzxqnG
zlPRZ-dHa+s{;Ynk{6s<7bW7JtGwYXpx{hAElte!zDaU`BYAzrkudJ?c%`|k=1G$!5
zUzO`c+tn<po#lR)XqikgUZrqY{JBcm!F!4-alI;TF)QV_><E&rx2aQHvQJUbN^rkH
z-SQ-*bR}(tuy+~i=l+T)3&wv?6HetajtHKp+MavR<ko_tsyEJ>t8NIdRXw$`N=agq
zyxGjz@ydz9c}nY4|Cr=;aG1N#v@<Kpdv8*F+d(yLO@rCj*uRRiw3L-5vusv)oz84l
zyFXbu_okwn_mmE^i@loiB6d|~+mcr+P6??tJ(Q$swm_%K<N)I&rNSLM%qF)rs4-15
zROL=wp!g%Z*7TU$6jQUqOH?AA-A%9VOfpsY8l~a#_mOhjy_YIK#GaX)m-?#wI9$QB
zAZoVaojFrg-CV4d<E7^*d@zwQ%|6pEzxP(D>how9C5c*PHN}tD6p{o#$iFH&qP*2D
zN4aX}Ok<V}lFI)BB4x!S?y7ispHbE;&Q<##$!?fae^)h^Rl-<XLPE`1MN>7iPf#tz
zPgVKtU4OIq@28cYc<ZTr|MuBj{iU{f`-dGCi+JuC>vnOfPqsg8x>e=844>L01!LPT
zdA`;~CYF4RDi1duRK4q~Z2B_&qEg3&1r~cHH!A&(v$nYTgxUPz<`~n|06Asbc^Az&
zj~rBUe)U#G{<@3y`p4|%-+i{4hqbY(#=Tf?(!TzIx!I~tJ;6t9Dr>6VsaAa5VLsP<
zx5_+qRkH<$b}32k<yTvC*Go<C>j9;o6?*2%sn_L-qYrDS^rb7=>T0VkOcPSFkhvvy
z(EO*eQRsW++#_;E(T75nopq<n&+s{=?q>B)Wl~9nn)QW1^A*MN>fR6jn5P_>sLr*|
zL_JeaQGMh5Rcg0HcA03DzEpbcwNxou)7;{w><QBro9Ct;53U;7u-(-9o%6wBZ^}`*
zw|_UQ<e6Vn{<3eqnNLrbk`jxTMk>n`^UIUv6+g|}ZnE2*Mg7vwpQe*p0!<RlEzI99
z+$F!Ubc$(DlcC0&TlMNOiWNE&`Ay7c=SEscRh?D&IEmeS%i7(hhn`O}n8mwVEkdwe
zxp3M)Ge5f!b<S@`jfEoH6*bFts(T&iP>S{{mr?O)HMcphr!ujbRr^qzwOYTHlZHS0
z3FXQ^>*Sm!1uKVbyrI^0)yzU<@h(*r8F96DkNUKJZ>d(lyX=^{)~zO^$6GF`*3bQF
z@c*-z+T5^<st2DHs$Q5?tmevWWy&{ui_+WfZHkP$?-?-t*kx8SW2%{1RD)T8{}+`D
zD>W=G-Kv+lzqv)lGjf6Q$t*du6?Os23L1CSEY53~?Va>mA?h8g`SXuyic`-DnQvS%
z*=(NGJ~Kbfg$kFau$!eXc&aA!jayAbXQI-1qy0uLxm(OW8qHKW^{2`7M3A*H=ca5e
zhPHbuhi^+M^W4=pUH&dhCB^%r>Fci=3T@Nv)oz3<sC~NeUhcHUJwtBx<%<8~GF8Ow
z|EdbVXI9sXs8!~=a#}W8S4OE-w_8p6e7f<st~OPPywj@7-^puD-=n3z`CPerOYaOL
z^S@Q9fm>b}a<xoXm1{krqT|`5dTio8#VNU;OwR0SP!7&wP*$07$1wEkDzn1*Z_Q?k
zd7HgHU9HacOvxg!I7|N7bz8+X9LjRX-*TEMcS$QdcFU-S3EG<$<z7{^ZJuZ@>A<Hv
zr83cc!rErD$aRY5e3i$P_Snmq<?J(6bCqUMQU5(bb)EE8qpY8L=B&}}DhxBKOn3a=
zYmzNKS-bzarE+EZ4OO|2L#CO$N0kq6QZQp#YN*8bF+r8{%01Pm21QC|=EWK@{Ev}8
zVR=<`-ZdLVRS65#K0Y0VsX_(v*~fP%ub#@QtlGq9lE2bg+31Il^tMC}_3taEDbMOr
zRCg5NG4E*XP_4V$Y__wiPxV>NBGnX*i>ew^H>q4%R%5Crpr-OQ%R-sgPQx^MTd?V_
zi=WK|KTI~&W*1RAU)yJ1@IPB=fwR7H<&WcvUA4;QZ%-yEZ;_v%dLWU};;?w5+}4JJ
zX7BjYl{S{kn{7E}Zr0T%YVqcdvT_!yw3*P#K()Qy4Jvyb$`#w*IGBIFZ)tk=Qj1Ej
z6_4p@!G~t6_XcWqFu5o<r)*MPpF7q3NctV+cHeB%=z>K`Dl6Wp-s*a$>{8LHSetv@
zbk)`gYLgOjG*_MRQF%JeQT2tHv{HMKmg1hv6I5!{%T#;1`pmVCRH;PnxT{?LJxS$Q
z?MKz!!qsX*8!`-+IE$<NxKwEnvVc)VL~?=hFMdZA=V|h)+dR^Y12b1EYDlhDRQmVW
z=%CSk(}10<W|1qWndm9#s{ht#H48ZGF6*l3t<3izTZzjh!`P#-TghHys!Dl5i%Hi4
zS%tjb7iM$KU&wO@&M?b1{A_w~qoC=sJ`MSECaz}R?wG3P1WZ<5YgDan`TeWDg6(m$
zHh~gl+soWW6D!<J*7Jzy$guh<^IsNKUUPSWso*9z<?PHLld5hl`O;-aRov}#)R&$U
zRtUWR*6<Xkgp%6r4XQr2TUFK^vsCTf&#%<q-X`nrtEIT<y@6_tw2$%3FIFl?E;p;N
zd+k$S^Wu~0Y^gtLj~B@qeiMGLVqv3bvi7rqYVPYol~sP~s?Tyflnz@6nM!Ouq_jVb
zU1dkKkXe!BHdD9usb((~9~xgivPQjBK*`ioXsX=zxETtq`$H7`uDF<_95Yk?=Ao~8
z^!+~LDSg`%*G06NR+XD5M!#EV8mrQ7yfU!CWZ(Y}iig&JH!ijkSIc)#QE_}QOLf7l
zE2g=59!4c?S}I!q7MLjIG@IR0`>1W}rK!UFk3}_t=eo(FiwBjrr<9o9xXY;2w#HcX
zlGR0}yK7q&@7YJ2ZT<00W>u)DX5zwMidJb8RlV&GDjxf^Q!YDPU0Lee93{RIdE@ZV
z{Yo{RymHf@O;Twr<yDy{HDA@?2b;;Uj#O2%o>b!ra^b39-w3GM2~APu+1sStP^@a|
z#bd9W9d|;xx^|*jKAVx5%k=`YADWgXH?u3%X6|@px}tuU?80^#MPuQo@>jZpO}DL_
zq&#m?zA8g&nwjcNRi%;*)@B+@ZYYYXx|kVzrkKX0s+-!)_@cD;dzBeOVz}BQmIW%;
z?_5>c;n`(+RsOQc;{G7jnHmd?@6~TG$z2$r^PsFuc`B2?szKO7)AgaODne^iOhS)+
zP~3W{S9Mibr1GZE@`|&(<4i?b9;)Q$=c~868!Kh-cc_Z^S|~1K`zouQ9-<t>I#v1h
z{@2DI_ZcduXzY=H!17DwXV(ptIU8iv=I)<va<8L7Wm)cAqd3JGD&OCKSFU>AsdB>g
zi_+(a8Kx81o+vGHe5v^7-#cT@g>hyzU)|08b-c{(pIol;(^cF;{i3Jhx(`}P*)u09
zBrRz(+Y<9v=~Qs1N++wd>C$C95}P?K%{B$EP+Y<G+thK|3)2F1P4he!cSY8lEv84V
z#Hud!i&ids7pABzRB57hfX6H?h*L$Ni_=W<hl0u7EmPFzZ*5jGdDNh?%Zk_RlRBI7
zw2hBVZ)L7h$dB$*G3jzwcAoc4;j_dwqo*^r%H85Rq0+OvRjIChs>*q5SB0{rVG4l@
z*p#2=RVq)ql4$C3LPgmyV4<ST-g;H}H}{ksB6QTYw-%WG++(E96jp0;dcs_F%_e<y
z#%X5iT{jl0o#E^^>E7^PdE=6)N*DHZm^3x<o4)wC!nB1k$SB}Yux9dkX;Z!~UL}Q_
z`&4oi9xCr>+-$<GZ?D{+?yR9-6mFLCGgxstYmw=m3)j>v47p6xB5F-8#qpRe-IA`<
z;&9ufIM_@h_GpQER;sW3b1QbUZ`aQnXTM-m4U-Kup0!ibbfKVv`p#>eYExtGsqpxj
znq_@9QICB(%ShdxO>tWCA$5+2OO!uOIWN1`>aoc+@m(r`it+0HU508W9-C_DyWCVh
z-NhuAJx@{PWX~ft2KOhXzE+1-jgPUaZ4EO~oTr?xzQFUiy7`9rX6&nrRBqpwGvD>T
zKt)nJOJ&yVl`5xrEfigUc$r=Q<)_TMUQM~NdA<3>A}KSC)new8Zq71U8*@%AG&b1m
z@S7}!$BXk6^509!_?8HoajaRZq@CBQl2IgT{@_xlLZ`Q}*`CG?N|S$mG!0qFY5FMi
zlbNXTTBXmQ=bBd7W~jQ|Sg5>eQ;>>eb-cNUm5gck-b&^Ca7k06Lo(*~=5^`xJeE-Y
zWi(&KM|QILU3XUHHFKp*4~54m&X(VzA`#o5^!c8gf^X~*^Do!utKMGIuF>N$K_RKj
zN~KrUT|pzwMM1D#ReADSZ{_a$2IgXw+m-rR9?Rb1l2XW1IiY+m`;E%V_tT6Sk9w)P
zzEm^O{}87-IqI`Ymv5S?!N!S7PQ}kmCuQAI&X2#S92Cr9e4|a>?A)}ErqcT_n!dQU
zP__DtuQ@x<bA?~qsuVBn7m-)1eq?%M=_2Laqc*AzllPfy+t92SbNi-gmw~PF$-e!j
zryi!6_D#q!J@s5q`RlPJ(<u%{YO}w4sZ7(^tyJsQV5)Yv-BiD8w~En1PO~!syG*9X
zF{)2pSEsyRe4^^geskmh(kqo`hBKLE^g1Xt@kFb-uI5)3R>@M#D_Lr?wBV>5=a+42
z|DsPRq#Tn_WtiuzP<(fn!mrO;m9NxvDl2*Nnatj^OxerSS}NtVp{kcypvsnY!D^dx
z*iE?SWvaPrhnYO;yr`zOvtBLvNU$2mmUz`e1^%Yh$L1-W`^u+uZ{-G)bd_VKD}E`P
zP1c-XoN~TcZSM6((>1Q93QHdKDZlZUq@*&r*_2&mjnednPIc!!k4zUA=*jO|s9<*F
z;v1Do_pY1t3LZ6OTHR$Pa(lm`_||QvlcJl|b1c@X?J&C|SJAiA<WJNNlfG0rl{kGa
zlbwIW&D0t7H7cFvsD>*tsTdbNHw{!>uIeEzZ_2+mRpEWXX*JngEtQ+C(eefX(~Xa{
zotN{DXw$Ta_^zVO*{tq<W2Vvvk8XL#^eSb6A2U=Z{k~$t@@0z3{58u}4$F3`{R=cu
zv)!(%KBIM!xsz{?+JaLuX5OAJ)RGU^s!g8OsAk_*s@ie(o5_kT4ax^SRFvZ9{xNnG
z-eq=T(*v`E9){*|YqzUyjQM4;UUQ$KROfc(|J~D+{x7*|uG4)<>Gk<;bzA)-7JK#P
z%Ig&fneWt{ubRXiV{R|<!t7w}Ckweh8x-B=x0~^5wyVc(T%(r6t*4ymRbbwq`_+tX
zSE`D@&o`#}YAeic3i+v<zn-mn;#a@QF##_NM^i1e^DhsY@@IZh2<&23S7>cgF3{O7
zyDiPx)Y>>&{ZHIJjl@)b)gQTq>e~6Ul=A2H%RdoVrF?VULDfzV9t*w0TU9oBpHsPi
zyFoQm)n3hMy1x3{8}AL@o)*?PIXm1)=v<9P^9wPJKLQFG^}qJ3x1N|}bo8{niU#Lh
zrMe?Q#;)79naG@)Wisu1r%Ava6P=BI7G@#I{EFJ=&Zs{6%&pox@4T_n0!Nk7C{0bX
z1|yRT!V{H_cLbRpF%#2p`LNGad;1!br6+mKf)0o%y{I^9T=rc?Gkj8j#?LE#I`^53
zjhTHIP2J+2t8#s8H2yX3u0icJeYJBh=BT&a*sOB)iHGr>Q@I)yuh~u9w=yWnwrtYS
zdX}JKUbt7PX-l}#YPMRXz=N-~rFVs?`<>I%OzF6#Vjm|h)0~&8^8Ed4^$zPm<5?ZI
z)r?hC)UWPS*17qnQ{(8_&l>#2g{G?`@>M@5G8>n!s8xN`E~<LEcY<orq%<Xl5^j?^
zx3fx@-4vA1?<_I&U!`Vdkn+TY>kpq<PpqCA!=|O?h5<?PY|&2?4t=;SKjovM*^>V<
z%Bw%Ss~Xk`nJ2G`RkTwKG@Ux{n4<l)Bc{hpmz(ChshYnnT&{FdyxY|9mxo&X)o2w{
z!z`tUzsF6q`yZRe@+qn=pEJ=UWbSJtQHg!(vwnV6`X_rz_1E+^GwBnDltbSHnWU%~
zE2Z}osv1AiR~AokQj*=<ZPd$=ud2M8Np*eXV};$5lvL-)=O|<({*pO*&qLWmsZ?o_
z%68LauNz7s?_bIYOE+r>9p0_%acY9vF)uE2g}7rXtx`|S7EL;>^4EEx%3MD&)#rQd
z75#;UOp{(~D|?scD}BqKV)k{bj;Z;J1!m%}a*aRV|E#{E%G1nT+FkLv$wUS3r59yV
zpO~9<+-XqaO<$=pEpffMpZGF)s|Gc*qlcB0_zSj}7DNb}ezGewSDuliRMpjFI$yS5
zRY|f_`R~;=ioKrc=2s+GO>ZyQq`cO%+GK7`r&+*%E$yv)>y#GCF{{d*jyIoap{QJX
z$G~`R-WtVgK3u8{W}lSyonI)Q?tj2+OO%J&^lM(~%baH`nEtm_ney_Y{99>dg|N`m
z$_~CB$}{*n&2uMASGx6nu1v<pE`@z_rYTn(Hdg((YN-iF&THlOTh1FDT<Wdza>@ne
zbJKcM&biK4)JQZpU7J2zF(Ic`N!&EaY-8wrGglTyvml0Zrl(v4)kPWqm>u}4E${Px
zo5DmfNd?g*dS<5%b1L01DpYwRt!*xPW~2Pxg?~)@<(QRDpKLMX&{HyNd%4!^$Z;n{
zXRRe>%yOQp#~*ho3%wLjKU~LdwqRzesi8}ya_!1m<2Bm(ru={A=&r8sQuaPSQF*ET
zU9)3*RFpNA-ZL)DsZcm!(5G_wi=48}0w=kX6E)04bjwuNZ7xt({gt73)reKq^yh4a
z&KyqpB~BVj?)$x!6b;syoOEDRVzUpBZ!7mzVLfc1oZ;=SnsdnAbkdnsD&JT-%$Q$T
zs20VWs4TJ3RsGW(rsVQszG<w*G$onUJCxI}3R=|nYnY{4shDSl95e3NvQyp2yTv@^
z+5$Na*$f5w2kHveAB#+V-oIB8+wY;e!KJ`dW7cZL{9I?V@1cdt$zN8PB{FB4_=bg>
z_MAyn+`QtVX=js?TE@<Ll|w(>bTlUYHY@txZ2D_bo$`YH7fk=?E;E}j-PFJ=rC%l3
zR#GK!TA&&03Ktd5R87+by5Wizk9}1IuE;BM$yzJ0s82CxxP4u&P>4lyx`D3JMaxgB
z?U$!1&VBkyZY_(D^5*0pN^RSwm?b-`QQCCfS5Da9ROROm6_rk>&8pIeo)~YseMrTL
zBha)(^`eUFiV~G97IxJ|Y+j1hA$(>TVbhfkG96Xw+OyA8*Xya-;aSVgrf-)u*;@8f
z^}X_A(>0MIiYE0eiiTY}3Q51s%(SMQRMKd=uHt5V#LPuYUSY}Y6f@t*T?#p#9j1kj
ztY$CzH=3AE=Te%;5@qJF^_c1k-#^OJg#Ib4nc-q4st{!|O}$a&^&EB6tY6Az9*wLT
z6Z0gMeRfK#cBsrTogkK~tlYw5dgH`Q#rF>wRcohpE1h%}Qmho#GwW~VQZV}yuD<W^
zWrh1I=c~+}IYS}qXqNnLky}b4=d_fxHEx;gkWW-<`(rA%Jt08(&rUw&-T&vP&Xn*p
z<h*2~dSAJ~AbnxFs!PW+m92A2R2|)KDKhTqH3|H7SE=juZsoa2pUqhFGfnT92bsBf
zw;5G0dZaFPc!}|h$sgs6mcEmBnG!3XcA&x7@`${0OS*+>`{68OVZIHDCwxDcz7t)g
z_<5zMS=RzTlafQL48MF|q2&MYqp8R_Ewzw)Au6$+z8X<nOHD(!?KajqQLAFsecY&C
zdWK1c=L?-X%eN`tc(zj2ZeNb+yoDQ;-~Qcb6e8%Zv}L!as!+I$%GFy{N;`^0&Dx&-
zmNV|>)x4{BM&Y}PgK8YRv0|0Cgxtw^O=VY2JtY;RWrn;r)+in4ik6?~#Grb4$s}bP
zji;*oXRa9L1WZ<2aIeE;zB!9}&Agdva|*lF828UnJ?_$Gx{GVIlF*LH%3@_4rdz(`
znSHSnFn`yx#pGi3NwuQreWvC&+vVeKUQm{iex>9Tdc#EY!%C&u((BZFms~b(u75A@
z`;^73cf%6ZeM=UZN^c1<SvH5wOe!~ALHjM2N!zE@>YGaMsVz#Fs}vZgVpcJy%VguK
z7M0eFR#O#gA#)?G4y{&+ud4N*JXDOz@0b`C8K~)$=bNo6`lgV`E1<qScAiqPl$5;V
z4s$b+tLJ3f1&TCYZP!tqcxs-y=H!1$s+V@l{k__vJWuMlsy4f#@%EV0DmQHYsPr%;
zE1$M4R|{M4LQVMQ3X@sWmaA;hFfu!L?Tg9;o+T=!Rh+8cjXWy1I73aHN>3^UHP2Oc
z`@X}JbtaSP-+m@@tDQ_HB~uMGPO9xQz4^0GeofDErTP!w6z*+jFs*VEQBHcGp(=kT
z$n<Z{0Xeo)tY*3m#}$=2B~5?dYBa69%V%zDETrV{|G0^nDuY^uOt?z_y-LMkm%XN2
zbvKwW9yqNWba}POi={l~Ue;H&%YMI6X0Cgr5<Y?5tgC0MvWXC%S@oMp#pq-o)n82a
zm3TV8Dg0<RH(RM8uee9EN#mjEPUR#ccU7ybrHa2FbSq?M<|%JG(yNkH-fVhu(OTt*
zB^k>0(<Z393fQPd=wzu1{J3Wv89G^2a=*6e>oreRm!0iVJ*_0Ks-ESb_T|WD)246V
zlw@yjP-OWq!#LV%nc3P?{bmMi6HU}6wyS1zS(=@dw@|R^4*=a6t(=z0WOm_poN~YL
z1U1`uZu7%CA{DAL51N@}x5yjaO)y(n(`@#7%PP~k@24s@Y<y(q8#Ghxp3`bo7a4Pf
zUC$nwa+<oEt<FtT`M6opWTUUR+5i6kn*DpTRAQr>l}k6coBln<tird`(lm!HK|#2Z
zU9HK=R(XBqYWd_>%S{R@&L~7*R93z5`j{%q{_ScR4H3$7xmD%ae10i;ZAn*??+7wU
z)JRu7uws^~ca)#%rY&shdKO8lotOGexFv6>_*zJq9aZ_EGQDiSigLJys`sZYicQyh
z&A46|D22LARDPYNV{!Veh53OGE6t0a%`*A8xk^2&g3;`v^aJ_)bMgun)kozlUn!c^
z9_v<`@<&NE@{o(!mpm?oXHValMeX!alv>GQ{&&YSQ>Nd^rk_nzlw=Inn`Oy!s-2RI
zRIykftuFp|rMbxee3PBMb}G)B{Y@7IYnmH{oY0M#)2KYr!cf)GipShBrcT*+Yq_cS
z?d6Jx*XpQBCJQOE7)dJ%-I26tGB;9C>3gp6ggZiE=4&2RMn-Oha{d?cAy?#;{c7ea
z_WfIKx;D{X$#SKde7RG&((O%Q$|9@gs!GeRF^K$jT0LVQm(jaGa}Ax}_tkxs?x^eL
zb*Y<dVloz5dPUjEf3ed3WzEJplM+pwGg-~JPA@Z<t!b^jq_EV4_2OH(JF@+%i92?y
zT=_8DaCfu4^3q4YG(t{IGP2SSR+{x9&8%e2MfLbMhfU2Ue==EiS-|vUN{a%=R(I3K
zuODd0?2^&=+^ePQ<6CIF!cyE!=Xkg3@<Ue)SH75JVszrRHp9Xk_18=fly7-Rn`Yft
z(+~@6HNN{wRmt3~PQ&r^Rn?VNX43t;*^T-xBq&O5*rV0QA*pWJ{72)s;5-#pRyMhg
zkN1_i7p_<T+2CySQDVAU*w0UDw>&;*o!jN7apc5I4Vw)QO!W0;s6O88YpiqgziJ;_
zhw4@%Wz~0=ebh8qHB2rX{h;(_!z{(;^YV;2r*WE1IKpH4|H3y@$GJ`_Gwx0|U$VzW
z;rG3C6`i=n%AwKUOr<;Zl}mbOsr{Pzz%2ETgW}(ZDW-J?1r#`Delv~pS!){6chIc$
zEvw=yOE=R>*`;bXZT716|E<@Z{ClTy%<46!YR|e<j#|W<+}eN8Xa~Eo2LFa0mEd?=
z<^In{Oed;Hscg3BFp)Q(uF$|LtY+oKsiNvuC|5SQ$7qxOJ7uK}aVl$*uBn=ro>DW+
z=ukd?xJOostw!lkOs!h@v8N`v^NLjk_bgErnkuOl>@B3euTxDeNc6rDD{H0-dw!CM
z%>4N(9oCCgBwwvpanjLIy>k45$=}>tN?{VOl;_^?HBH%a-*myGcczBdbc{JnR;dSc
zpEtQ(->dMVIzf3ZcbC#t#~UWPQ;sV!t}s)zyDx8AxJ6d}-FF_-Q=dM_%PH?PS@TcS
z<WORRvDdCh#bco|CK9ios^&*kswC%iDqVQE!1NV+s}V~PrwYG#l?n62Zc~={%~}V-
z8C2BH&QN*99A=^sG+Eg_<CMwKk0pwXvf-+JcjqYSviwl^HTjL{1-WGT`6}-;mOVeM
za%W<eYIECsr6BJF`4IKnO1b$tsvm#+F`mZJt#T!$PlZvlO(8a1Lha#vFI9c%lcsXj
z+A1?*JIsEs&Q#f#wqN;^OP0!>eSE61(eF(PHD4%c6-`wNV?Aa1SZ}k*oXB<)=k`g)
z@A$h^6?Z>0{Uvorfqiwg^527#lxD`Mm}cJMSAJmFpmJxYu<6AG+zP_nyNr%ryseOU
z$;L!<#wMeMzaE>u^*^ZK_bJerUo2ZSD{z{!l*=s@rGrOJ?<lM>?R$4!IV?im^wh&(
z6AL3L%@vDJDTPG_su(HQn+2(-DNns)Z(QJLub^khrTS^(8AU(SPjcE1=a`x##w&>b
z7f|2&eU6I%hVLq@HH}J|L5gxa^7ks)^&6|oHpZLn`mLhUE|;LP%WA(Y-@Z$#!Yu1m
z-|bsyy7rN)>WrlSCOlFZs%PGCsCxN!sirbYE2#vFnI>|iD?3aUQx-UT)#T`kg{Gf)
z9-77Ki<vy+4b<2^y})edDIdi+zr~6-Y64_aTiMJe9KNg6y3InB^>UzDTjWuNIltpg
zA6%QJq*nahWa4Xivn`yl=5K^<DV;sL(BzCQznX`pyvh!yK*eRH*=8@c#F*TB)u3`c
z=d#KEOP*%cIg09xMqJ8$JMvYd?tM1B(7s&x!Yq5UH4D!vR#eog&a_yl)V%kz;*x`p
zP1fd2Q}fdjQUBB>rTAMyRyDoPPC-CmqC(GdX_Y9~`O0as>Sk}&-c!;wtdUweK~=eC
zpN7iofKIhXkNQnt?~qgZuyLB%S1Umk@zguY|9w+cR4-R49#*|$Ci`Dq$#<!T@&~6#
zv&T;J%&YiUn}4YiG5dA=f{M<gr{;1`^5s%LF)EtzD#?eZbDOu-Dk&NIYO1U{{K-7k
zYqy--vIpiJ#~c->JMA{x@PgG`#n0RPV(lu$nQ@EF*X-6;72DvYd?;a}ywU+-^IKa!
zm?<3EtIWM4(Co#!<>t#(r)bTs$x=Fz6sPiZ*D^DoL+6!NY&0-CmB6I9va3RcW#1+x
zrK_*x8KPRu<Yx;g#u$35SBBOryuH_{-2X^b!S|B7!lzJSrF#bsDMTmxnf+J#qxe)q
zMK<{;gYvd*CzNjSKTvC%?qKTmB1JXpqovu63Hw#2{^V8lwRcce*{Gp3L1~Za_U>|J
zfv9sz9m~4RzA)>W?OJDN5|LJAY~}n<)%I(YX%A<l!pWE;O01JimC^&|npUxhD+|O+
zs43hLHPug8qVTX~ozcr(#}zy)RvBm18=CkwW|~Z!uud^3b-wAINlj`#E$XVxJ=`iG
z3R6t0SNt=X+nl1ZF!zYb{+?FT0~42NzI~Xdvf|=QmAOtWX1jQFR5qMlWvr`as_@TY
zf$Db0=ZcdyZkFGCir1`dmXPAgkhkiF<|mZae$`Wp{oSW%ymYM`r|AJDt3WR0OGR8}
z6Y^b^9;VkTXw8mS(4A$aWE$M3R#bT1M5Own>XDE7X8vwcYWcRaRWFIlsFmv*D_Gv-
zFyoaERPodMsl2kp)!awj)GYH@tc9iaD?`VZ8`U!f4j7vhO;TLH`Ja58K&pJk>&2!s
z+7(oe6+Ka1z|&>kw?R^g{h)wFn#CmL;EN(=2fwqK#~(Ou&bn-p@~3lM=Akc-sd>M?
zqw?|=t9DV!Hgm4@AEse<UMep?)MzNOvd4nm%1Q5R(jgU#TzxgxYhTPem?Kqc8=cK|
z&rwy%J1MRf6DXmwDek4x#!gm?;`m<*ote#=EW90xGv6wx9Wd8bTzB)4!g1RvDyz=$
zs?2DYHj9ycsQi+xN&e^;1N95Fg({~vl&P%@Pc-{-%~U;AUcfwXOTT(eypj45U1RkY
zrfq5)Ws6P3yS^#vT)C+DYK5~!kIODI=8_h3MI$HEKBHq=-<POZ2A7!287l5q`E0yg
z`Fv1=+2qxX%BR12YP`FiVcujkNl|3ZM{}FiEb0&Ac`YV~M46e|eln}N)S^(>^~p>#
zS65?=T7`O(U6W3i#9~uVpDqiD`~Ot_$u2aTlP+Om&%DTB)tiZGj(gaYGv8&HaX$1@
zU*V%*+P7+o!g2ep>PsG{sPfnympk#V#k67S4f$w;yIMc8-PNqsJT#QP99RC$^Iz`y
zH6i6~6CSAj+qc|&!m8t{2HS+yTxYG;TBS2def#gT>NBekn=~EzrdlLaYijXaMlEgO
zMO9w82GzB^wQ5(b&CCLpGb?SrbxBFQc9kjj(r0GR7@wLxurfBAI`_AVR3?i>Us;KC
zMR~7^iE)Lp?a_3zZxNiz`*iQCEhzeB+F+`vC~@Mm>0{n}#mgKzX6kVqrt{}=nf<p=
zQ}k~9WZL!UjhgTgZZ+1Xi}GR`Z%tg{+|8;56jeV2Rhu!o&NVURc&(x=*PwDhE=<Ms
z=0{`Cok=R{`yQG8j!9MQ?zL4*S~XwsTESs?;rK<StX9d2hf`;(9TRw}8oBp@n!)S|
z$^{8O<yBU9DybgsRcngBVj?Wqs=Bafr|O={tD3EWF6#AV!D_5m9-E{x7OI}|F)-Ep
z(W&Zo>$b{&#adP6MMo62y*4r1YkOWfGP^}tIMdBkzxJ7#O8gbG%dJ6Xat%J}t1hux
zoRa8K*t*U^@m=^A`TOBB&1_{{l?B-(R3Ed*nJrUXqM-2am07ZZkn&-(r)JU3=4Lj$
zZ_ET|Oi^MNk}#Wj+eS^fTSmpmCsT=m>zAp*T@SP34;Pfb&CE0F{eHw`G4mnS-dBmr
z{)xX-&mCQ5vO(s8vZ#Qo*}+?mN->}6RYm)1m3g+mS6tBPV7eoMMP=slRch`<cNNM^
zcc>)Hmr$^)Z<W)!F-y7s!hWT-@>5L}*GMYAJIN&9cTGXbeEKe>YBNdox`}_xT<&dA
z>H4<Se5ypEs>%Z)Rn-%&s?G{8RZLH$n@*cAqau5^N~!SEG1Dyv+RSb`@|g5;@tHQ)
zC97BTvzoV<&s4Z&V5U4PTvd6G+cz^aiC;>hC1I*d?)RB;l(i}PpIKrMbn=ekJ(KrF
zpCgu-d~RKB?%V98=stCs$=ywkYH14nD(}V1G}zzYHM`xp)9kP6A!WV$TIS1-ikZm#
z3DIc^n5Q!NwXe!ahDYZ2|2L@UyYDsOXEjuG{P0}Wc3zY6!tk&1b6!m_y^tQM`t*6R
zR)yjQmB0_*Rrk(yQ#!nQqAdU18Ojnnf>i@1tTJbdw^muOEm$Rc!%W4RsL85TeF18&
z_K%Ee|JbXjDCU}|De9=OJm*#UV;!v$cUDF9y{fdS;@xSA9x+ptiY=^7_U-XERg`r%
z%bB~=r1~(UdW&3_8B=ACe9W>?<yPspiXCAtrY}#KD?PDas*?3u!|YnYO8I3PE6mtA
zRAu$eB+NE%pKmI&eX;2R-#*2ks^?7gH%F+Nuiv43vHXVupUHI7>HmaGr*7y|x#ga2
z68<sF^vHpH^`|O76io_^RCu0qnPe;$RZiac$V9_jPw|TI9hC%!K9%Ptf(mnfIGgNx
z{Z-!XoxIwgXALTR9zRs1I;EAq7hRVBwRMrw!BAb*|DQIR9G+sM(kS7jBA<UmC4-+;
ztv_#pT1m)ELp||rDppFi#`?u-s!?(qRQ#Svs`_kgP}toOYkDl$NO^xsgEG^yNoLtk
zt4+F9O-&zIrx`6SsL%*CWHp|DNmJp)RV(?n$~^gdrzaanyLc<RFECTxRp4yYCZDDF
z^X6Hj)|gzSm$Dp2;ZN3?9O=1dVplm!X*<(ZljDEH)oPaKs~kSMT+MW<tXY%f4Wp;s
zD$4oxnue-NHO;)FPwC`cZC4S#&ZsIet<mg>-a6%=>)s}cO-4%puNbS|_|2zmd$LQh
z_rg5058j*Po@O1>{L`AQXjZ&lRbjHWqFK)o`Ea)i<u_Nkl(+buF%dK}Q4V@zCfDff
ztQ@4bUHR~-PSsTxZkilB5TUBewAa*TN2uxt!vfW*K{Hjmn&vAAHOiWvjnh{-?8>E5
zv_{aZqJ6LF_pfYbx8s<MU)`Chruo{_bc!aIVwG*NVzS#!`2_y&Ci=5Fm9uu0s?Pr>
zZYFCJs`$a;fQhrIrt+nllg8%VS4~z=bT&1T_fr<G(=~hNoT?VZI!C2aK~Ck%MLW~2
zxfP}xylPc`X8W03zOcaTg><OS{*7mq*EL^Oee@yKEc)$r<r01c<8@QED_+%`p}HmH
zxU$2;I7I=2ezOh1Zc3~3FK93xQc+U+^IK(yh>&8;r%4JOlh-Iqzpqzr@)9>S_RdxQ
zwZlg~ead}Br{lUR5qTnN*Un8f-680$vX5(>X&+mw%DdYOlr1VFRHUDMRGKRO%~a>Z
z7NxAPO-dh*Y%yKDQ{4RZuLCAAt(<1-{wb?A=4~+dwGC4Ewm4mhL2s7g*X%B{j^4{k
z%A2xP!pj;>9FKa*PfAKP&UZPdz&cyPbji`ZCQsJ9G&_G}k%Hj)>8ATcGgNIwOOz*E
zvr;}&bj9R*nyr~l)pF$&*V$%GwQr3VZ;4dmIQ2y-e^H-`!I7<|jF(K5f3&SKJ^s5@
z;mX6MD%L8C6#p)FREV^$Hg#zbR8XDcr>=K*qtX}I5S5AyL&XmLKsi^|1xh!jNh&+W
zF_`Vvo2q1=_)4+xeXIPmGZU0gaU`jUPvke<_}E;1x=^!Ojrkn)<n?Om+-pqLlXq-X
zGn=buy5hK`^2607iu>(aP5G)qOfOE{sDC%-kO9;GcFj*8MU0*A*2r>6T~?8OA*^z>
zYL@BZkB^kP4V^U3=XM#p8t^Ik-#n?q`RA@$&a<fobEim}yzS&QF^&#Y;N*Q`{7%YH
z<L9YNb?fac>Lt?0OppFpWg_+PiwggpF4N;GH%(4$mejW2u~1FwX_N9q#iM4tMS<!u
zkF<>#AMcg(64|F7w{@4|V$OR~9pPN2D$S+}aW~Fsygulzwk+LSW6rma%H^faGIETJ
z%4vCT)jl+zG`+HBtLhUsJ~jDYI`UimTh-5L9aRr^-)WkjeMC8BvbDK>Q=&?>i;_z2
zH!qdF_ctq=bo7{IF?K6^m|atn+j!AD{%wNU_pN#+>|NG|Y)g98dslcH>%7`0|JdGI
zVc8mG#p4OxW+%K4C`q1eQb}7i(<G*JvO?eLPQyIDcqN<W6vHd$xXem)rOdWxoL9Us
zUCC_uv@q3~Vq2B-t}M}*R?lj2!Qzn#Q%<u|%<7e9pQR?5eNgDpwg0h3S?0n@Wig8Y
zi`|cPmBZ$4H%Y6Vp<w2`N~KgySt-ilggoEuX!DmV7Rcw=iED0H$ENge-!qk+8kO?T
zUy8^xFsxN7{i~+@@S327*M*5nizaz1O1Uy9eApkQ)PMA%%25thlj_s*s&jo`n0|ZV
zs=EBiO_iQ$MyiLVZB!KB^1}3*ldZ~EeL0nxpAF1(ez=-(9`-Z6y>6oEjVqs3oo{KF
zSER5gK8mkXT$pd5*!x!6R6F3Rve<V`)w0(IjAS&YD3m4cG<M2yQm%PD&(tI?%w%0G
zi^&nei%QA1r%nDnGg7;HJzRyuHCcI$&{kvhkB>|@$lp|s$`UtI`*GGdJUT<uIDkPV
z+$TqM;iuguX;027ACV3=EilVh{CF){^=?X@GS`pg3R7I9%tFs@RJfY<Mm<KCU&;Hl
zw5s*dtqLCh&&s`;RH=NH?~l?c5f>BX8&1kx_O}%*`8O%39DS)Y@d=Mwa!I4<oV5*V
z{Kxf8KNp-+b7d)4le!kJwwg0u)!K5JiLz0j(y5K<ikB;Qm`py=X8NRQnhCeuX~QKs
zi`1TlJT_@~zghloO}}!_<ZVhU*4?Ily46bQEuHGRY@((v?due7wUruOKgX(CqOiu0
z-*KDCCev24ps%|Xc>eq``d`+lzVPsJwWn@B<qlR%GhOp@i?M)Lj7sd9NRuGjJtp~S
zGgQ<ZR;%uGDpY1T<YKmP@*&kk&O(#>8?_W<ZBDA)OpaCb;Fgm;_@&K6jzL1zzdc^F
zBtc%aOte#7UV5$4z87a?s_w5>s!y7zD%!wj_BCX#$}hE@Dii!}$yM&xQY(G<K<#Sa
z9&=aOezh~*eil6|pQ-IP&Q`m6x>Rk)tbWy7BIag`U5_cnzfDv+7PQE8#-Y{bS#O>h
z8&z&LYj*puW_n_mg?sXDd7eF2lwAykl%=+9G<W^frIcvXrM^scy?IyxtKx<pF_URO
zm#OaBx5i}OMlDnAX~GtI8MEb|*-tY&_`gY=cmGl~&HuI<UJcL96j+rlTq^lh4&|y?
zsIi%vCWIc<`Z@i$Di=qz@}v59=Brl)s@>(dX8L%lru^c+4C-N*uPPRo7s~dT+L>8Q
zSuAHIepvJMO<mP#8J+4ZVi%OYsn3;}Rytit=kZ}xrP2fDuNb;ix_uw3YzmWAl=cWv
zn`_LjzO8n)k(q_I#{2!|#=nl%X_y=o(KwoEta16@S@q>Y5hh2aq?P~ZhbXU|b;p=v
z=`s_yr87+Amo}Iv1S{%1_4PG-b+%YO>C1i9Y~cqgzhw(eD!GMK_H$@ys@~@|xtl#t
zDPr|e6PDS08mTW>OhcTm7+d6gG&P+Zqx8@DiSd2`8O@_6J{mja`gG=;TWM^0uExaq
zU5DCV&uWu{t6Gif)~c!noy}IaEjCxxi@apmQJkQWU}R)s_4S~l{^#`?Wly`+M88~<
z-+d?6Sm}zZ(u}av+6QX<)c3vB)V%uloeJx;K!p<)Dk{FFpVc2ooi@7Uc~h;XQ$k%g
z(p*O&b-sot?|+Seyr-u9sR61UA2iJlZAnz^NK;p3pWdx{CHt|w&Gb-H-kzz-Lh1p^
zLQO|a^1N4?IXoyd5}8qFdcoUQ?U;Il`NTOh<o8I<P<(W#K|%kBjG6Q!Hf7&dYt=@R
zrDpvVR}{S11dZFu-IZ4I`y0DXVKfPRd*0MuZ>o}M$}iJU4nH*`pG=i+2d1m>^Vgcp
zY&mMiHrGuh`+kJkpMQ@`Qm;E{HqYx<{xF|iO|3x7H2KMCW!}GYOlmmqDNYt@Q8mgu
zqim&gS@CSMu4(x7%Zf&P9qOAKJQbS_CaRPk-J#&tSRj94p|7%;r>W927dg|kW6PAh
zgIZ<P>gzO4uezwDS++tg$GODpLaK&}L;F7SohNswd=(K@`Oz{_MKW-z;!^wNro79G
zm7OI6lpJg?m<8(JH(k1ElfjY4^9}c{_^V!KRcj)&VyQes>obK8RVT%7MLlNr#o<a(
zZAB`JTGpF5TIMQ58_m|g#xhmW!})^2TX`1~{iq-_{^x~?ALqrHu<dA7t@}A$xtM`b
z<Jwkz^S*RVv$NOdDS1nYnXgzLZQ}GvOn2SVC(8Bx+mttH-!;EBg+WC$B+|Is_p$sw
zLvGdPz|BhMp01Rgo?~NXer36Q!RI~=1qFU3rvqH7fnW3#X6ndG&3b!TDYo;qQpC-Z
z<{!LRl^!~(E8hEUFF!9)T*<E1P<1=+PSXqDUMTxopEtd@#a~6po?qokQL~Ekg%tVC
zf9{&jX3SM;m^M{0eIA3^#tEy<`qHPE9FP<?<?2#Yw^}G;Uh!5|LAdLre78W3Jj*`?
zvwwvr6@{5<RMsc_G?P8(tYELO!gxubzY^;zIb-{~g{FFT>rEqFQx)~6Z8Ce39HzQ)
z(lq547tW~(EZlGQ+cnf|PnU`E&U1TA->B4@$zNHctHm-!d0M4`N=wx<GdE>jWr>M#
zCRTyd6_!~}QJM22Ug>X#n!Njy>t;Qi3l&Q5%+#15b5e1$`vH~t7yA@qPVmZ|OF68h
zTxg)Q{mc^6Dzg}+^+uOuKW?0(VEnaBNocWws%%HB3Cs3&l?zc%jKjhlRhQPSRq21B
zuIjGms$j}8!}MLYuJZYyFG@4NJT-Ty{$eV=P|n=hVwSPRXBCasxLM{EWychr_^p<U
ziPV(m|1NGSx<X1>=4F_wqB^VD0d8JJ;o5^{XM;{F&pETw%qrE`Y-WU)*}3y2O6EPg
z&HD6J)ofo@s`zEO>fHP%YyR+nw3#X6W2M<EBaL{sFqwy6aW~LW+NARKTB^$2WA^4M
zQHm<2qW?|at(H?-In7ITNy1{4AG%hG0n?|M=f19#zog!&$;7=uQPi7N?GW=D1-<h<
z@~7wBQMS2jt6a6V&#dsjr}E?ZA@T`n`pQeRk11cvSg*Qhs;+6-mns$UDUoJ=hWk{^
z3iDNhHBPH+;#XC$?&3DfS<a)}kv2)GC;6G#|K#(g&PIQX!e*^BI=Nx98sEJq##Z9l
za?kgADy~-ktq|~bqp8Ka7Nw-h11e%6>c%JcE>~E^;h>+8Ia%T6OdjLge>R!yyU1%|
z-PEb*J!PZu6^3J~=ac^`s~0*cAG>U1_GZFHW1F;A<#Nt^lijO!n|jEUXqRfeQFhaN
zq++QnWnOY%nsP}>m9cT0y~0ujcGdhzpOuRArYab+g_y0cu~KONdO*W7w^Q-|->WJ$
zSGFmvy?#lys&R_a!y+Rku^EYGKM%i9WWVVr^JtTpqD+>zlI^hsRgM1!hUK3HRPQ-{
zH!c@-SB=+TQ(eXyqk166QSQ0IGZP8hCS~(;4$2bWE|^;UGci3dg~==}T}1z;PL;;y
zDk}r;@CEYkSo{@Eyc1WLBfQO6szq2?X@-&NsoF;-OrP8o57|kZ798?bvivx~c=snO
z)7~4*rt^j8DlL~(G&`cHsdn8wS*7TQx_aQrRI?w(ZYEDST9gZn91QoLn{LKqBBd)m
z#YyGjsyV8yI_+kfPP>#lmpnG=sDGsBZXBxG{c)eN)B#4Ngw2P|p8r^{5ahZ+^P{Y)
zqTfM(Rl_g46ci3Fk?T)!P?mh|qco{vit*hCKb7vvvB~e|SfTX${8^=3!F#IZuHwc?
zx)as1S{+Sv-!iKEFwRi3-rl2@nY2iCMbm0i|JC~xZ*#s=TEDo@tfXz4$;EYQW^2ye
zG7em>uU@_`%=GsUOPSgYr<5D?k1MHi>@=1>cTVZD&06&o?v=*70#g(}H+C82ep#cs
z*LSMX!w;dRBL0s}57gSr|DG|y<QmU%^|>6^)V_UNtU7Cwx#@<8YLnywDV3h!Qe)le
z9cF%~e04tUWKxT5kx?#LCvP_8zoQzP?|M_O*ha<Rx7_N<F%63K4?5)CbY_}0&b_F-
zShhk_Zm*kaR_GRW!Ip1IOf25=Cn6b?yJYXG{taj}*{6L*#rh4is_~*l3jc+>)$08p
zsh#5fZo<tcsggU{(9~Y^p31g8RVw{1CsZz-wN&wbvC1T650kRS`e#Z>=WNWvRxUNU
zAFXaSJN%2$B7uI5D51lqhq_7?xI{N7weQ-iC@#QZI-QAAdEOOu)umg@OnZOoE1bO7
zY3y%4M`^dHvoVW~pXrHri_B`TnJRMc;WOdk`>uM{F-PS<#~$_E?<{5$^|qQ6ExMyr
zeqyG{^sOIFmxyfC>DrW}V&JBta*a{dEJX8=vR+1$X{f7*(y<gr)wek-6gTnoD@4y(
zY+C*0hT0aDn;PF%yi&H$SfeV+WTyCSTa`S|;&5fF;}cZaesP&Ujku^>KWnG5w9S5X
z{S(5fPAb`|$6lT=ZI)W9s<?BiS<HeLs!8WtRGkk<tIpDJQL|baY}z$7ROwlTq*82?
zsF|<gRTG{Y`%D8jCm3@oOjX_YVZAB8N`QRP-ei@;+ttc*?CVVBg87vn#7<P3Qz&hA
zD)o=TE>m68qCNLze-+L*Y4nmZ{=9Og>DuJ23aQr^P50`}ReN$`jcWPlCZ#M6CNmpT
zX;b4#dsPf=dQ7x`_nB@FKd5<phpEcirP3->idxOMWrS7s__r9_TXM-ibZ1t3b<I$b
z_d~J#omXE?6I!bk*9+vTIek5<`lx8R8q<U(<(}D7<@UbFR+2MIRZIQfY<B&BmTFPn
z4Atmu^JLd<5>dam=89^YC5P!tzEBnC%Il^^371u_pWLN#YxNJ6ZL62be=$F0)@~u7
z-1Bm$l9Z8vg?vVenY)Fg*-V-129lReHGJ4g44v+B%g?*NS)SeCsKT5>9A^BVl9kSS
zN~(S<n`W{)_J@3@$t>fb?0%&`MUM;*rly)bQaNCH=HWI)y-z>P45l%wO%P2|VQ4AW
z5c_Ur;i7C}tlVC%>~5@Va^Z%*`QZ!sdVVtQDqEcnsytN*w`iZXPWiyYWhMvwVimPA
zlvRI!Fj4-%DWY(1UZh3qoy`jO{~gy%+nJ`g+9OQ$Mvk7s_Xq!EjdB^3qj@JPa;-Kt
zKOvN*WE`%jpru`|=zQ(HqVlR9)uV?t8+9#ytKQ6e%{cS2na1|z_tcHA{ZNlAoveO-
z!)2pTjl)U{qy3e5{_Qh9r@GaMJM)sM!pEzIY*Q7rUBjOlAFel%FZY|Rs`;u><<-Ld
z#(9B<l<ju>*6`LZFf|o#RhllwZ|1f7q`I=fU6X(iL6g)AOH6~_>M2DyCYZRk-PPF4
zAgOVoTUOWpse|cFdtT$uKbh68uX<vzBw5Kcdq<|0{x2hS-*|hK50V`wZV8eaGQTQ}
zuL$pxKk87W@u=~HYH7NG!rOlnjU7rG6?${$X<a!Zp?+fhUyZKRSt^@5A{8`G@~ix)
z+@c;YX>M}6bb(s!zsG7?ec!bruI6ZHOq{7<HKX3t`u%EE>7AQQbrKoXF3xLJ4QkU+
zeQ_^HZKb(|X|T1MQe?lGQnUMIlb|h$W_8BmCe3$KOfya{QTa2g%bd}9mJHwDDwTyg
zZOT&`Z<!_@{irloW47AM=}l&VtLqde?{zfM=({BUm#yDqdH*S6wv0$q`3=7ne%5xF
zCOuf8b~bZ|>WtG3y6-1!G!|P`Zni~nv&wB*UbC06#U>TkK548v6Q;7&NJC{>^fKc&
z%<3wvnrlsZ7#=COa`LHJcL%7Mud0&YdVs^U<7u^`sepyL(blu7al6*4-Q6%l`FQhk
z`SZmxN~@+csU0xnH!;7LsrvTQEY*Ma<|`bmQ&+DEovJqFl&SH(;1=aF-W4VWeyu8-
z3QAOJD}q#9-utMEcsiNtiA-10(0QQr?WwleL^C-PEzX%nt1=E7I=&9jh-x@un0wV(
z=1)SP@_WB^N}X>MP1OC{l-Rc#sotLAsDH8Qv4ZaObRA=H38gbB=7zSdr;VA6a*W>I
zeJyXE<Zi5+a$mKkt6AlJYQFmV(le&UY-L8<&loCyV7_WJNAj*oN4bSg^yCf|kMFL^
z(SKFV!b=w@-)#D0<Z;bYso&RCRp5TJ;!^ic1-9@Hrl%OoWj8!npxMTLUnMs9tZLS~
z8A|4XA#&E+nw6Y?H>u|Dlrw!4)vFS?X|;+);~j;+9cpUnxrVAWPxhEv??_ddCMah1
zx4uM0>?FTR@4x9PDKGd`rE?2R4kWx#inM1`ek?S_RPBMhsq=+}rltos8lPM0sv4~>
zWG205u3}1CiE`5_btTp((x%mCxs?Uer>bafb~H1Y880U)*J?UT=#}C+hiX&y?>CK_
zHwK!0xn-xQnWt;QwK_v}4&zK^@m@&<xkJuoRZ)DVCVp3yGj3O!D9tlB&EIoZ-6=*;
zNx6E4%BwOpGci?1Wv7*X#yhO}6g6u_RFy4`C^G+BE8qTqzUf-cSk-O;Up3*HPL=6C
zhg75&m@8Rc?UTPEEuk!R!&jC4ldai~MrW1jogpfdY8VtYBwbT&OWms)<lAaed{15V
zNa$RXBS!*MgI+FGsW7NgmD2vMWbp2gsh{msrBvy2N|!1gn4Fuk!}Nr!pZVm!62|Ve
zp&E7P51UF_waagCou}BesZUmN*HqKUDn?~FWqH+|i;T<)KQ2_XX6`ksXZBaz&|zj;
zb*#@!?q!wv%<aueZ`JRcUbSXX+x$>hMY&?B;^w`tOpkBcWfD=mOXbT$OOunUTFfVv
zW@<^xmnh%yzMyiKW0F~yGmFZ(vUz63TfQph+$~nU_h^mM>%fVMW!aNWS{9efY}$WG
zUHbbq#eMgiRg>E8C<J_$Raor4K>1AjV&&7z?wK4<D^~hspdvqGN{4EWQNGHB4drSN
z4&@q4Je5-UTert_1v9HkfaoUW?F*7reyY4xjDO8(Zjk(1QL)!g`Juos)6frJ%(<<3
zEOf3YnC*DJMCIk|C*~JpMdYNG99Fo*$|tWUve9g(W4Y3|>uM@?R^H}fEpl?cmn|)(
zPHI#X+AzU9J-Off>w!yVp?VRD!kZtNKdRPLUHZvaIX|ge{_~_Fv&Ysi%s<DzRgUti
zGn?lWZ7zD;U*}GLz7oq}K9zZiugw;GKcjTKzSZni?k9!kKMGZ*ZGWpeC6i6Q<p`sh
zQQuX$O`@FYTWxw3&NjSMUa`Abo-sR4K5K=blFp%b3ZFjoo4mYtU2$J)g4~I2UA3pn
zg_M`_395U1Z!mGSpQL*JzM$FeZwFOZ#VV;Tn;fU=x%iOcypqMHm%278I~zMGKemiD
z`{TOOOsegiY4gc=<Hy^4R2Sd$HvKzmpJHHgp;FxA?+RbO&N9{Xa#t37B(1hbf3K<g
z-u-e<SI#k2ZBAB-{3m8QMKZ_4FOJ)EotT-@o$wc?f%jU});u><Ws&4p{4|x>EOwi_
zX=P)fN~f-{>BT-X(<TOC4X@hQ$`>>RRAnEjnPoHcsFciSGTHOtq{6)~%T;%ON>yU{
zEUxhKB7<3j{369Q@}cTW59BDZ%w|_ByvnO+rCT8{^{Yww;-w!-BGOY#`J9(4&G$Vm
zKeM4n;RhRo@?GJhYQou5464mHtGcaUZOXh}Sk3fwrmAD5m|F8vD<z&3U$d$2z9>gr
z{ioczXqEYnEwbiJ4Y?LvWseL!)%L6ZmYQue^T$c~!xBH_eWl&xdxhCdr)Td_7I!|c
zy6fLjvm1pQmG(B9SY)2rr1WEQr1`7I4(8_r_nC2I<tRV8@ZWsO-2H00i$1H&`KqmL
z@v_srUMbl8z!rNIGhH5Ihutz3+kc<cTYv3|%GQ0~RO7`?nj0`pP?_-5)g;C3tzxa7
zfEwS`xvKwvKTt}^{c7&9_n?CLmLkoc_FPI_*L>8b{1;YQU23XeeA!ZEmjjQA_=hDX
zio2I9FPC~Mzp=kvy<qoEmBzFTwHUE)W{ppD)CDe2F@Ib(K|S|`p?WF1mb%sYEo$qS
zUmE{WxuW>R|BvEKMr(_w`FUm+-pnxzY0)taWIC(0N>$x*NvVL0_k{y0HH&U2EASsM
zvksl1<hjFB<EHxz^A$QxiVKyr&5!(HQBPmnWv)HH$#nTIZF4I|DTRmj%4T<qbv05A
zD%5YXHtD?F-ETU7X064&&{HZ*H(#1v>bYyYX!kUOpp{G1-c5d^q?UNX?CsJ>_4SW+
zOfHB_Q~0`nn|hnJtV)P|zs&I+4Q8E7dQ`(pZ)rVLa8rwDa@TM@dRDnRkyB3l?S7^A
z{SVYwr?OZ~b3CTXQ!1wBb!dT>mPv=YTGT0Zt!dkh=iYgv%JR0;Wa&OxwMSphtDaa~
zuIlusPVK#Uq#4Ug7A3Q-+myV&S()~n-C=fn(OfgZJ`b~&^3N)jZOj%0?uTWKF7>K>
z@}HvIbEnkI!9riz_WwOK<(bu{FLs|+@b{T&*5_KR_|$5b*>0ErrY|hC%-k7&D@;*4
zY&x&tmD+53ZZ-Dga`}th!KSYR6wDqk6jFU<RA4r>H`=6X>MZrIUs_b+&hx6QZLl%S
zH;-14Pe?L5qs*@8a@$^Q>KRFuTeBnOr-X8uMt%-d<hvZE_Okkg>M@(QYP-{_m7mVP
zBKIM!O)2+#w_5WF8<W(YM%9REhg7}i|JD==uvF)J+^UwbT*bubQmN`jQ(uz}N;6f@
z-h8VP+0?EY*BGfF<Q-!+L4LjR`|n}OtB&k4d1kfBOgSsZTu{W=tm=rX`s~-N7D2)q
z3fmgqDTFScBG;r<XSR?hQu+HsIaRB$OQx~<1&UvexS6{*@hkfrbT_|xbgkK!rsrnT
zeCL!nbw8V4duyV$hgD3)=vKK>m*`2;(ArIAmTUhguX|`<ws=Rc=@ip;%?|Dy<t^8k
z)E>*<FutI;Re6E!c{9zu=}J!v3sp~*b*N04n6K2o)X2p3b(FmLO&0Y<u0P~GTbNb<
z?GRG<|2I!QaQ-%B^VclO9R_<$wyf1q{<-p@jGt(x>XS$P%I}_Qsq0D_ntP?0sw&@-
zG{3bsS9Q^cdez$%^HtgF7pU0sS(~aGC8<QJGb+{WIBohTro*(MPRHzM#xE27$~|h2
zE@_#Ey_Qv)Get_-W?Gh##*>?7cYJ%4f0#C?CeGwEKb5mYL1LSM=?;Tpr6~#%P4eDo
znks!~w2)Ptr*vwXj`4&GK5B>fdsK4osjJnp37YTKJ!e`Z=d1Fp^{;7<QN3xw^(gIi
zS)3{#%z{-PPT6FhULm1!!()=^gLiWk+v47++C<+{{`k{U;q@d7v%5Vz)I=wUYw2v8
zq2l#vvTD|3UZwLtWferOS*tMok5M&9y=lIZDPJY%h>nWF%;$=0g%_zBnJ1~4*KIQV
zuwPH*hv8~tt?9ZdPu4tAE?nrSvhanLs`)h=(>qTs70rd~l!WI;8h1`SZ#uiA)r>oI
zjmdT%CUsfGHnTew%~Dd{fy(>8NGZA9IB$|NHBKpX@&c9GRclQz+;or+)_ZAY_F%R=
zhxjD3UH1jde&o5EO3g4)*mM4`srfS>)tyy4mG@_aC^u;Enojv2U>0mtud;Htlu5_-
z&n7&r_p~nW_g0embyNA1t&7Q}mrBaQ+0LfFmv$=jox7$oO?rdc+dcjYuLHD<x4Q@^
zSjNv&)7aIkGNW8Tb;=rMrJDU*@~^}Ll=kcDt6Iqz7|)6{P;v53RFToWt=7@Ys8+J+
zvYLa&YQyF0Cae7GE;gR+qNp1Bp-YA9vb^fub3KYrCODWfGczf_GHy`5Eq%!B!?_72
z(FgQR@9qdUIy|RBL;m?S!}Z7C%E!n{E3m(tte~{<r18Ih*Ok6o8LLXi9yN4O-=wIs
z-O>29hn&({mIC8C^B5D(wR?^1cbh2LJ`*;*A0novnVzk}qVq-dTTP)^icpZj$5cDz
zUmqC^H+C#GRg-+9qrH5h%BAmzRB{gJn5FUVR<6t0U{n(DRx$Uhj%vK0nsUSfU&Z%Y
z=gejWm&x<xZP5Ji?W1DVz9XtGyACP}{3wxq94Df@a3Q1e!e<61%8s(iZxx&51I+Fy
zyMJL(N&M8IdfCp`BtAG^^#+rJ>0P%-)jWA$)!+N(tLm-oRr>dIyGd5t3gvy@O;x1z
z&CP=Uh?>ox|IbwDdAV_ocB7i)?M~C_lYAAHE_kJ2RO>Fc?!q6Fb2}xJCka%jt`gj9
zdX@W!BFnW^rU@dmlvtD;OuJS-Fu9kPY}&iIT6x1ISF>HyQq}BdELN#;NK+}`-D*1R
zcckh5KYc0*dkRcs(wI$oU+>TfPd=>tX!1tY-krNli(0QL|KeU^bi7VQN$1UU)fu}N
zDkolUQ7rScHd|+TSdnk8vBuj2%M|sTwNynl@)Z@s*D6GuTBw}tHcdHN{;^5>yKv>=
zGq2<rTwW-deqXMlG|fmYE=9ofr%kQO`X5hBcrH&+F@Jwv+4)Gj$_%zoN>4T^n_Vq>
zr?e^UuhPftWu_M_51F->q?z?cbC@NUK2S;F+HXF2nxmrFH(4camw5`#8B5HL$o)`C
zzSX9(IpD6z!8222`_$x3zfR;-x;%T6sax7?(<ag1W_5)Z6tpu=nVejhrfQ<qt~~E{
zg#v52fa&f2>888<mMLq!>NUGK@1TkORUV}eRc%Vlb-Pv6gjh_uC(0<#Ot3TypR28C
zymp1k{j@$sR@Rpa=3+9Yr^A1#FqocITYjrTX~JtEm8P`~3VFx9<PVj<Q0hP9sXYJs
zDboPyyGpBGpHvKWYF6y{H&^+DYp2?_uzJ&4cU$!y;mfA0o9C$WX=$kc4m430T(VN_
zIP(INb(ve0f2vMa{AVL$dNg~diB|9n;~fkB7=*6Q(RAIp$9Sp!1BDEcQz|BJcvND7
zo|u&UZ&CjA%u%E2y_kv1v@?oxjiwrH4!W-P>%&8%p!?p&_6EC6w{4%PXr7{NtlDLy
z(bScrE<L+J>Cg*jvty!DO)g(xQ1$WEG<hS*Z4xv8wWh%FDQYnZqAK=rbIl${Ijg5~
ziy4WUgvqni>{p+*lvnwV@?r7dbL^&^{jG|ey#*Q}mt54+t~qOnaNJdXDBUJgaeAV%
z`_<cO{@L1QrJmbVWdsD&Ci2M2P1;|qKJCR(^$CBp%@&DBs$8DRZGP}XoQm4oLY4A^
zO)9PR#qyh9b(v{s%~2M(oT{X|?47x?0*{&cp#qbiFT@PJYbL87+I_=F`0s1^pZc|O
zYqhx)UR1`LWf$=(zx8cZQT}9P>UZ<7g5`Gu<3GOLN@?6?M!Zw6nieLNnSK0IqV)Z?
zs9D#?7}a&hS1F4;-l*>1E^Bcw_=*Y3?_lLxsUXw(U3zAX&l`15OrD~=xJ5~2rkj(+
zcUEC#gGOPK@6SXPen~G=`Q6>9^wzpoUSj71bIIDn$`P^(njgcCDz0C>M<wh(v%*hn
z28E|b7?m9jOqF++UN>K}X{S<O=M)8rIlC0vq;!<s9vn~+-L}BEO595I5`U}dv5mf}
z+f%t!cT7!Ft+W+TTrrWuZ2c4^72XaFl@k**%nnaVHd`(D!!%<5MAMavuBm>1ZEBu&
zr&ZDX&R&HNIeBu@QgNobN;i}R`pi_{EeJABJHfB`@-vg^bK`d9s^`jPo0i3xSZ1-C
z&Yq{M{ITn#ajdnGT8&SPiu^`jrQ;D#OhS5^OtZPJDBs%8Z6+rWViKDqrzxy-TX|>G
zG}W|uIVKO{*D60upJ3{$aYZqGdWx!GLZmW_hp}RFqJdd$*B<5grI*yZ*{l>7vT>^x
zuUf3IQ$|+d<eZhtnVhxCskPHg@=x|EPyZnx_c-IPLPryW3h!DLwW-<pri(-y)fB3?
znwal6uNL;VQf-@enA*`bp{hNX&zeZiJEipD>Ul-U?Ym4;@|;aMlCGIvc-(KeWK)IO
ziraHdisfG`@Wn4sUSl*xDWbT>bm`YjCApWK>RmI{%tE=D6n?oInI1pNpvrz*$mq@q
zE7K>PHD-clClohl?KX*e*r+~5e7Twi*CYApM{`UAzM2|Wuo$Tr3g#Jqcq?GmS)-#G
z&bdJK+7)4y%1<v%*<-h=>PQ=#uKlU25W3-%THD1Liid^lWu~*wGP!xMRPku(d`+g~
zLaL2n_3Gc;=PBLDe;{|?M^xE%&kWVdJ#$U_S1eS?^INBKcbkd)VqI;ugKv(giEY|%
zzVYJ(wNvUP<}RGi)vis*S3AsItG1S<UG*=&f>}48netk`xk^_LbDBC=vzSi^&NtIN
z{MBsV)U#?arB5xiq%{<yS@tSlGPt93QgE?(x-zTs!oQvBi<Re^|9#i2uxvG>+5Ytl
zRL!0`n~BevU@FtO!eWnhkpi1<g{dfii@N8+m1>!s$5kd@_BAg`n`FK?Q(r~sMUr`;
zY=CJg(**UehgPaOncYy1{>NkSQpZ;9c3Fp+lv2EWc^sqqW!|Go=Y7A+rWnea>Df!E
zZ=PPR>9s;twaTnn{bs{_C6Os-WI1CDl%1rvsV;u6XmRP&Y?a`RPgD+V*r`;)Y_0Zx
zCxd$ZH4Y<(mFgP1?`$&?wyM{ddtX>%bFhZSapyDY6_;~OCU4%ZynjxO^7LnI#tQp#
zObQmiH~DMYZ*uUyicZRDZ!_U7v*f!rKUR&L$E>>ZVw}m{IvbUu4Vs#h)E5}5%GW7X
zs*9Qil<{h`s-~EB6vmnyKlKB2R)He(w%NwzQZkynY<?OJf)jMSE`Bh6;S_1o`pQr3
z%)LO9KTG_KTs~N+3BRpZ@0z5bYIiN!&^;kbLsN6LiHohdLZR^{joMaGwF^rR$n=G$
z8-2JZr8N1*ac#Hbf$CqrYH212-cvcX_KsYEfSSs;4{y|CSXqq~K3r1E;8s*`KWC)#
zdgm04#03nRi(C3l_Z&}AeX~W|bZK#kYJ0Gj>gEU2RCoToq%b$n$Mis4f^t*ZROJIc
z;wD$xe9g{tE;V^)cF^=yrLh{D-7ItFYB7bXRZ5B%SJ=ouT_9jquK8KXuGd-h?C;~I
z@0%DEpP4K-QI=~~`nl_giS?QS6Svo=%=TNIR`RrsFg>d4r}j83MP=i{>xvz3Ell0H
z63t>CS*YZtE1R*O{$hN3-bJ-7OZ}C7V)@i!S6?@E3*WB%MA*pmj30}V$>L_!dnZ>Y
zCoC{m5<9udWX91QijOmy)Vs?46^c|gs`N;<C_Lauk=NRlu6*}*pi=DhJ*FA!*D5Vp
zbVK&Smtu{l8+w&r9=@hFqwTs`SxcMB4>?EkojJQy99mmcen>r4QC1FC=qofgt&%QM
z7E{($YMK4s?2pqDlZi4tCYRo88g29Xrhe(Xn<?`*Q$^{UYvnD>kI5^amp7{joTH>_
z)unRgl(pIMulMCIr7keuc2ijCsZN0L{m@y)fvURZCmeZ|cD)oZes0*SDtd9QGP|mX
z+Jod|bM@w>rt__>l+8_hP0xq#HokIdf=*47uJY;yZ7MTME|~wH+Nx|fU)9)LFiT-w
z2dC<UkDSVyDP?lZ1^3M?k9w)?TrQ)bS-)CQfPbyZ`_M)5wRt{rG1`^NmLElx>o#0B
zKPuLz#P!-%p)*lbJ}>;85@V2=s>u~K)8AV6l!K}SO;?EqsZ>m1RtcEXu3~*;r9xQT
z0n@)eI!f&prxfq4oorgxH^(ezhq3A7{}HBVuSu)-hM1YJm|>#O`!h%3X~$Z5nGalM
zXJ5!G?&hvl`MV;{?0CZ|dG&v~rp1cgN_FoJn>4yjH<gl%F|%k}plHjr#cb2>P}L=e
zI+Xd0pR4&L|1+DTsckmpyS4J>j|)r}i_9>ccJHz7q!kmCeHFx2)K6|U+hQ!PEMK_C
zBq2amLB(&XN|MNWrTHzK@<rk&%yyabD46w$YIM(9r8q(DjLOyKr3wKZrt%za)0D*H
zvy~*66HVWkBq$wMJ}bB4;Zns<6&ICutu|AQoATD=w4I;I8~tL_wa1KA!!PElBqu4T
zO2?j5oLV-~bnW*=N=_b!m2OO8v~UyRG`l7}*KCV?t?}F%a}6!#S>}ZrObV9%>*b>+
znk(c#Ej8tOGePM^jK6Aw@qbgFrg@5gyfn<N=)6@v9Uf?Q-}Inq#qJ2xnzy1#SL(Nz
zeZHuqmg3!@@;f3(C)VG<{Lk$zrptQ0l?_%hnW}A+HDf&FVX%GkRh66C8<ib`ILt%i
ztyPrg{x|9^{;Fty%0;z4P*^2-`c#GQ+-uBroJwSOCpKy3q-iSY@JXmiZ=R%h`Gu7{
zKX;7smbnhf53K6UE@`SMdt96&r*Lt#atH&b$}jD8s&C@+Ob^WHRZ*Q_Vz#m1u!^Wc
zk;;F$-zpvtJQdRFSDD5ueNggLysPB0W53xPw^OFO`X`#o-(O<5JY=@otfVQ%sv_4E
zvX<;oIR1Ny!aCko(*T|2N_t<9s7!ZfH1%m=P}nMX$K+4Cui_r{1{2<M5+-NOPMKWP
zxu9r1$IL{a{<P}jb<8R%HU0_<Z%s1uzj@v`TdYjEeByrNntn6Wf9KC>@d+<gHf!Tl
z{TTkztlh3mIrwzC@o7hOh0I56s(MTLmH({Tpy2#p$}DqLi~^@_yN2161jVfGJt`*|
zrYd;F$ja?7I;V7F_g%&OOVVZ>zwDLXh`Y!>>#tC7(%q@_<6o5O&-7A5{xTlb$U7NE
zVe-MMT<QN+YNbn5XXR<jw=7y~@{`j``Hf$O@^2S@GyT(frYd@COc}55(C5EhtHGfb
zYH%wjP~qL@5QQC6*%fYu^%^fWzN>U;im9qwwy23Gcei4Wzq4tFwu+L{#`PvTO(Ldq
zT*{30FWaJYqetH?XSIe}&#N#Mc}@-u|NIWK{B282<mxk&E3~tWG<<ta_C!|b)<4iw
zc~`bvb<e-+rsu=gE4!pGF)(b>Q=HropgQH}Je8z0TcyBz`DT;dk`zVdr)u{13n-=s
z#;KNU(pTiJoFHv--BvkLD^=;^!Ac{w6->(U>z2r~Y|&DwJ6^2(xaF$q)VdVIs?rH+
zeMX;*6`7gTzh0WIwo0H&ZL!>Z)s0`)n(CjbRr;!YQK@|26*K-A2Gfg@GG<kuUmG{x
zS5;qbdcvg9zeaA_k5kHlrQFIBQZE^Mv`<h<{I*6tr~ZkNz5O+X#T#ar$Sm2YIze-*
z@oIBk)AzOArVatp^1n=(O;5a8u6{-Gy4w2le#+qywWbAA<{PKa;#0Y?U&46zrK4th
zBKvgm@;KD^u7)XlOO=@IIc=l1XS;`Ku|c6?|6y+Re4aCkwUZ{uPKw-T*3Kua5LH~F
z`R9>`>NVMo>LHzM%F@4-<qkbvtu*n_1=T-AXN;c-omSzpWmJ9h;*~;<NTb@in~&5S
zHqJ3oVOCbz@jcG;?e*I#$}CG%j{SP6lDn2$rR15S>CNq&%Kh&*D><*;X*N^gvdNYu
zrDl^X`i<uNoS`vUh0*lSwH5N!`=%<T7Plx)FXJ@zZsAilVbfOqvth64zbZ|I<4e>{
zC53vFWbfn}Z=N*I)Kz-DS-Ro|#qEy;P2<0PQ~mKVN~I+Hky?xT7SmN<RE-1A<S2(7
z4mA#7y=-Rk<Dt&_-B~JMd!tpVEH;}>oqI^x;pJXardu3J7IUps7x$ScJ^pW^uz1E?
zGp+MG<R?ww(3Hw}r+krhj_O<P7R6lcXoYCi24ze4=_(gkI?Q&@d#AiyY_9SR$))PT
zK~k#Lwh5{~&x)FcU0S8O?yZfPwZLoDO65+~UsL2&jaIs-g}x9ry_&gJY3-tVrPpU)
znoi&R-jt)O+|1}>i}8-uQdOQCGff4i>{56zJxgViRk5<%wHDLshJ4EAxs%jH^LCnv
z79}YBckDE)zxZ1IX!#w}!avVUOkQ6z6*|?Z_`r_Y%&cdQn)jB~s`Z<9D)>Zqn0d6?
zm_7^Gq_Vsr#YF#tf*GGPx8@RaNfos~N0oppYfU%%GOHY~4L6Rmc`t8h%B9xtsH-?%
zq(?p|h0%;LAWbpxXOAlTslBRizAjMveIQ1;#QUrq(~r|iH!h^9)r4`H89d8Uov>r3
z>Z+dY@-i2A)jtFYsC_=+Yr0~|Bo()$<ECoYZ>s!ybwb5HS44G2m%jY{xOrxuN}njn
zo<69g;`Pe>%n3I$;iuAO*9zYoT+^1*NV@N6SQs%=p+0+;e3s^Q`68W%rW&0dO7AM<
zRkN-Nn$F(OsW9{P4HJ)(OO-@*(~YK;OPKu*urfWmNKfhgTyb*&Q8u+ThT$r{YP=dz
zMiCa>oU4p{moz9ddrdWpXiYIYVY5*0-&!7(Re|!V>33Kx)@b%B|GB1REVJ5DarYio
z)%R)|$|mJi3iI=-ET&9vS4>h_pji^DrubP|OErTpR>753RX!zAMcMQyqf*MPWoCPY
z+LclbpUO@Y5>#4x@2pbPuP)VB#kB_Z#!uC`H=HoGt~1y8d*+V%1l3pSO=gqT^(|wJ
zC5@PrFYFaka@erSxG|#F#P<(_*=j2zgTfga+R|DFjGr9NllM`ZrK*;_QYBIKqtU6%
ze&rJR-x>k>zfDw5Cn|+xo15|1ol@U_c&6#rCtpmIjq6R>x}GQ)EM+&nnQ~VnBT`Z$
zJ6v7&fAVgVa;F;8OCOJ_p3{ytY&+&}`tk2X?V7#5>Yu84R6Gwwm_%@>X$0->G}1lr
zLEciXRwIg^P0g))j%;&+n(<koIf|^W=V~3vlvdZh_D`dqd7;YBP;I&U@_ot~&o`^L
zU2ijD)t;sHsq~GS&52)H_QLTR3WsNE#LJ7APTsRtwLtQ+iQ`*FwU-91s#}!QRHreA
zs@0dcnr?d6q_p0&Rw?qjrpfknL9<uCznXA&Z#LyR<FDdiG1ojpGgDqexj{wo%RJ>9
zS$9nxrb;Pi7|d4N&3WAPnYyE5{TfNr^$fxaT{>Z=#%r5RZb$~2`JWO|T)l$FbZg%V
zwQQB0s?7-vx@`NWnwYPWG%G&4L1p#(Tc$JQ{~0@~3aBl-;;V9jlTn3P_JT=Zp@fRY
zVk1-D|2Gt#xCp3)wk%YQuG5o0Y~5tS=(bm3hsp#sZ??0lEpl7bzOCv}KD76QT)>)G
zrH8BP)cl@(Gnuv_O_k%pBGtFvJCwEv$f=*4bVlt-=Obg=DXq$VFBh7K7q_bX5bIG{
zrr)J9Q^ivCuhB0PPVEg!p4uOjB6H2moXi#*pZzXs;xT)+;hD~8jnxGLhFSNH$e-^G
zSN`F8PRS;a!9?j~y3*n4rmB`Z&l*koRIl)7JF_ughNM!_*$SiYvo9NI?~pNG+uyD5
zt6bf9bLL~!vZe+VvvbaB@8>g{RYx-!YAGft$F2%7e3>F-GWmqK&K<{6m7m{tDK{(K
zHT`&Jit>vs(+rnwv`}LD>85(fg+oc~t(wAm)%B*Ik9ElHjhn0)wcx4BT*uw25oZ=E
zIbJZ8-66AHNyNHJRX2IF$=_Y`RQ#SURADZCufTm+RBgd}J=MSEOr}$X{8fZszczJ$
z-KKKgSzo1g;xv^Lmw8kJ`7fLJOk`Ky^8TrkRcwIiR6Tt&V?9~3TmJWq^=AaDN~Ya1
zHSU&IsJs=XywaIN$-@1f=@}<2<;fP^DosH(W(OGarDt!oGBet3qQv~d#VqK~3sWgp
zQ?uk56BYj+eqfrsC|=cb_eAB`&r%BWFIbpG?R{YK@Xj~o3*3Dsm%{Fw?%ecJ{q+1?
zrRx)$RgNVKnX0b0RsQsUhKbf01|_>k{Hn#bUn(yDW~E@wu+=ocD@rzg@dGtJ^G219
zhqqOP{z)sHJh4E5r&M2gvuud!<pYOIU%$3fdGOs|#dy+I<$C)^s`gXns2Z)GXz;p(
zMK$XCS0kN%L)E#ztyFT1@>O@QRZyNTC1y5RZIROK2NRX{C?7Cgq;G2WT)5TT%um#~
z+cjFlk;~Hb7LT9o3-8H_i~@5Mc3vzqxu0=ZDLR!;HTa8)=@hH^iiImqnr+wZP|Ufq
z&Ghxo9JA{0Jm#<J`4mmp%{Tks^jr1v9|e{6;_V7DmB&o>oVjGW?(;;IvrMukQC+{y
zxsHVBe0Dsiyq(Ei#g2cv`OQl<Di5cdn}w|5R@|qZquTm#k&5};UWNL!dnRubFUmM^
zDQnDGGfhdMa*gWAvNpw1)y=Y-cV1E2J$;Gt+{^+)wb!0XiLBy^%E^n>7;Ab|WFM5N
z%@i^*N;el%slCZ=T(8fevazpC*(fboMQf*m(q-i>X0yr~6klswDhC@DnmwHR-Mnf}
zpv9iGI%YSQaj1GcQ?mH5TTD*n;c*4Ez(9Es&Umv3mRU+awy3EDeoHsAFM1^RNOz%y
z*O3>BTWao_^X#rR-!^rnX{_fr1*6%F7QgwWR4)t1DgWQ9qP`%w-*i<6o5c@@o64OA
zQD#TQe9T_0d#7_$Fi)vB<-PLv?=#J29Ai?>lw4}2%ehNIzq?39Wc_NjPsgO?3saR%
zt-Iyq)GarwWj+3(a3n%j<z!c${QnaY@+;z*l_vcaRLuA5HFT~2qo|bUp&+Xyr{2>i
zt~|d-L0xX;1)~caXQ?h)#c#^OcTRQoI~&zC@^z|ao)SvCbLN`PjG3r>&?Hs)(=RTw
zt0Fq)2CmI!_2zqxxu)5vcJK5t{d{7wqUm}ErL8`j<#>CxnA(^cD~H4=s-01LXDX%Y
zDy?&So|)P0vr49VEM{$d7fl_i?wafuDNx!H!D}YFvPErDf`KX@M~Wi<AvQB*hiuc_
zk`k3^s*a}8ehntFCXDJ)zosj%@iSAk<(4t6{C{6rHEWN_i!K%=_fsoWy$&fTZOau@
zyyW%P^r{P+!ivw*>T*rf6@RXNt*X7_r+mzhdkSi`UzANhv?+H92%Cg5vnW>wFvz4_
zoUi;|>zwjCrt4~}{aB2hExJ^NoSlu29bi{mxm`syvRg#$ovN7fk&XOj_1$}vS6*UK
zNu1tk(K#*8T<N*7MUwSD!|HWk)Vaeh8h?qpDnCDBo<eMCkK(=qk4-AHvXv8~_o@E<
zdeGGA_FW~%P$`QD%UMdShD*#}FXlF1^(@Nh+Q&Ca7oV6~{A}H=W@mXv<!N)VHh*h_
zdG$JX^RLl)D#nWqP2OCyGF!4sL|?d*RdwgaUe&$FHO-wa&QpntTx;}q*)he7DeP*m
zoFA#aztOA|aG}SX^U!6v7B)rABa;p*75uJM>t&g!<p0P>PP1o$vfeQ!m6-B7hV}pZ
zl$U9}lg~VQN!{<>Z<UDbe6@<oRP*<&TIz4kTrj`dI8lAVTO)OkLRIyu|LfE)Bp)z7
z6vL`4xO=lAkMaU@!SoxZ=DZtBm)Gl>`s5wgnzKOO(%ojA>=ea)Dii!4E8B51n3rTR
zDBC-DY8*bqV6i&OSaDnREK`Sj4C))^A2PclaMDEo+C1}PD>o`|Y}jJ5qF+yAzgv~M
z8&i&sL#MNucUgi(fY42qeTv`BBy1*`e%{n+p!a!!+O*sb<&7uJ%$xJV)VZ=J8Q1$9
zQRw)zU46FoPNmSp9#Rvz70mB>o>G0Ma!+e@lCv6Lt+U3L1qYNf+~Z{B9Nm=HZN93e
zDPm!9s(!C3vy7NpL)9cLr56?IKF5!%yKkFelsDs%>iZWu##<R>)w0UYtCq7DsBYO^
zp=Oj5ZMK~6l@hPQMI|v-X4Cj<1!i{Z70gq<>zeIk;!vHN%Vlx)@E*At=XzE8XLT!o
zO0+V284;?yq2-<$hk%gjjmxnLx+_J@^ViN%{BV`W{B?l7*{k(FW?PMA6+ZJun(|G3
zp%&x8tJZ#nT`BlsovC(kzM0tMuPSGJjLga%)l46YywqgUnxL|{^PO`0tw%-^#4}VF
zr)@Rc_Rmg1*uzE5shC|&uKvB8mB>-!8ApQ@x6Ik1_PX_@YB=`?wZ##k$}=u{$}e2I
zS!v4VHnoLoZy8C=tyNXfJ*jFtK|ssBMN^%%hEbidvDYZ+NwR8HRin}Rjnh>hx1Ufs
z+g`6agK42+Y^b|gilCozyta+9<Kt^4%L7`?bn<qZE84T0RrrRgN9xF0?4EO5j%^yR
zLeqCPxlNme%-RlVC_niwqxv~j#x&!;o>I8j9rMz#B&EIICz(&IY&FZA#%o^ed{Jpf
z;#Jel-0ErvIpkF)muINXzo%l#Y+`7>bibtvL-{?^9_?&1ZP6s{*z-%2*Onhv{ph&I
zG(F+2a$n{zv(NlWO1ejqRUO{lRW**Aq$Djp&uH&~4{{=0{_0w3`xOK%qf~1*vnakk
zrzJnvty|f>m0fwwd`qLn>@3QQ4JOO}$X}!u%6&$;Zn}~Bj!O^B`cFrz{^fpYmYLb1
z+R-{kb%Nm@RnCdCRBo>QXS!~UhRVZYVWqnHH%xgwyiHrAqRe-5-87lDx>@Z}GN<|7
z15Xw221+YmxZbVkCGx`TPqnRb+Wk(|ES}Tm)2%Mb|C@i_OfK}F68ns7)0mHSX0;}A
z7T3kql%*A7&35?*s=1^VsjSG4SK(t;G5>hvwn=lTk4nx0XVcx;6U}-4e%9hxw@8_N
z!y?t0libbUJ^ra2c%sNGFEU!mB=w!@489%8tNEH0*J$^faqQJsUh~XC>)Mh?l|>&@
zREyu<Q530gP<&E0L#1O$lPc@xMDr>6T`Jd<m{n%1W>GnD_^YZ?uBDoA%LhX>M+FtT
z%RL4beX=UL6W1%x;<HrY`L3dRbLv5pcl!Gj%eVI`J?PqIlx23?bWimrGgp~*lO<ai
z)mQ&KXI2_sA)6H9qs(dLs&sj_nF%v%niAK(xhnN+TTS*T2+H>hnwd+wg(*y)vfC`%
z{*oy_<5lD921T;%lDuY&|D04Si%%&t^Chb{-(GC&6|mDRTuM%*CiJXvu!ptDj-Ri!
z?_IpEq!l|+xo!DglL@Kj%5(GVOslVnDg0e^Q|0qT2ldC_^%Yhq|1@A@u~eM#K~DYr
z{HZEGzDTLQmgG@tjS7`Bw|Jw-uc@i(HOI*K{2p@^H<d;egSuJjLhbKW_pjwwfBS8r
zfx^B_mDBQjj7qO7sIHPIP&s^6QI%`6ozh!-G1K>r=afX<=PCPsoNHFa`^IFluaIdC
z^E*SX!|58fC(8`~6fRImbdOf}yt7fUZPkBc*)@}tuE!auo`{n+y1AuY(JSkau}St~
zr30lNCjRW~CNHP=8mgC`Q~avwX3C%}sy6X_gi6LE7WI!TN6nlBUg)3FF;HH^HOFw{
zQXP}Wyn?!WEN7@ROZ2I<UfpTBZpTLDTH8nZa-j@L2b(lhZ|Uz;YU2@7Gz|D}R{FX}
zt}e1zvvOj%6335Qs=+BNN~#Ak<#|fiE9D72RuZ2qZk)1NTUlbgmwe!#AXP>^4wd5*
zyHu@v-W#tw?x#APS=_{JXQ1k?bK<JHZqrmtnM;*V&zNqK{n<cSbEA&RRM~@Ozb)&`
zE|u*z{eM>7Bsiu)ZQ0%hrdz*>%YRWVQ&_W9N<mi9(6mLbSb6E+T-8QvP1D*do=Vf-
zZ8E)85U!{ux7zfBV3BD!x0mUHEu6~N<x9<OhsCO0nz=wl@~4$*P4qs~?fmaeSWT{~
zSe%_}!r9ntlq&FDCsOBu^6qPas?Hl2OqXW8Qog0nW+Io&tW-X!Pn99iSGgj`UoqOZ
z*)-2rQTf@W-Rd_2l9U2u`&6s=+7w0X$`t-T@luX+o2q<&y}yY@B(JiQ!!h}*cFn4V
z^$%6-Wy00cISWkIq?fAPUwPix@9jL5<W6RlAG11D=I35g+QAuWc6r?|rAzO=D6N;+
zX4>{N(#-gjo_WU3ho+O~iKs5jTxV|A)1wgA^-)pk|1SCSZwkyTSooCLm|9i7gm##I
zn9(VBY<jR6r*gH@j??mH?6ba^`j_aNyROeuyt(|ViS+bH)vzOt%8vK^72@L>O%0dC
znCf2HtenhXZI<rfYZh>(LgRkM8l}ti8&rb$H<+ya#i7hox4|sx^EbsXm$@neKVB=v
zZ*x^Vz+7tb%>A_Nx|de!Z`NrkNi~|QR2cIrJfG98koA*a`R23^<u^gAP2R<^DNlR2
zM3F%|SVhO@z4HC%`D%OxQ%#Lqoz;aeo0=Xgn5+KPS5KXDx4HVhPm9!QAMY`4NEJ~r
z_~EFuYsDFp6qDs9(l-~HPC9eZ(EhNeroW$p$-N$arPZ@esvIsnqAY&ztw}`IV&$eH
zCk<N-K{Fo)ZN=d2MW)+hPOGJ=GMRjSZD;&i;;U(gRgBU@Hvto^PsSQ27<1J#LbVmc
zkCvG&$n-N_6)mRPd1#iAQucn6#CZ&wSpkJ=>kB@qT(_(-3#>6ySISB^a*xkdyzO~V
zJ(Xpa@|F9~W%AYwm_|gbSJK{Hq!D+^SS>ouP$T%jR^^-<ujHS{xu{It`be!d=Zoot
zU;9+6lQ`8J&*aMg63A6&`gu&f_s2xD_2z~uujH1R-Mts5!V==GvbAE0$}V*U1-`Ei
zW`_Rq%0)IUN<aCgn7?9gG|kxVZ}#_Xk|Bd?zq-Vp65~s3Dhe$}brlwrzLa}YY-*-A
zBUH&-zC*<{@3z?l#au<-qC=)!Ix$KNr!E>zXqjlbX~$f%C+e$}S^~4piu~hM?>cN$
zzQeDozJ8Ui#fLZL#xAc+l=+P38QHDOGqZR<LwEYaX63Lw$tn}4GFh<HiYos#J88@!
z@>#wiWVy=j7ZplBH}90MeC=&fv`tX)<D)woE`i00YmUrO;rsemVV#JTym=R|vi*e|
z<*hRk%=H6yD1~xIOKYXrDBj<CN%>UpZk4;2a*Y|Aj8vI}R+{iV4pTjS@RLf|ku+5&
zVF#sb{nw^xDch8bPphbCX9%0U?T$8kR##|N-_u}fsPS0!<P>@H!{2liR=)l(KkK}p
z?8$^)(~p6Bm2)<ksGfVTZ*t%$x6)hvqo&bjca%J)G@JeV^}~d3zP0K3HWy{n&vT7M
zPHL%zUdm8OH>^@(`E<fmH-yd9i&s`<Uf31WrN{M5gCz4bXN9q=aD~>YuI^zskrG{}
z+`90V>4BLBN}Eo`sK$4hDtn3aD86>GG4tHLS3#GHNu#GkPtoj@zv@&bbH&Xns})2<
zRw<u()2}?uE!N~)e1mfS%8#-yK73LxNoiDZ@p+&cFF(V0_Kg~~cRj03?s8sITe!VO
zZSK1uwLSadRRfHZP0i%bD-{JDP+T5)!{m(8anp0hQ_UQDIt`yst5-X><c#s(>Dmgd
zx$~5-%4I73UHZ<1e`<qL-keVL+VWSXYWe+g*SzG-%>I8=;b?kothV!@sq%@rW?ZXJ
zDsGv;VwQBZN&PMR3bo421_jGm+@=Sw|1p^t>#uSsai;N;uAgRl!m?Vex^1dWldM&K
z-B2<6R<T<38EdZT$7!7k^|wx`oqoZkvVkRE{w#ZoiN?uA3Xh}zX^O`&tLne3P@h^@
zpme==io9NbkMh(dGgOl}D@`ss&Qjr6xmM+eg`ApmyRMpF?mD%*qE+Tz8PnC)S8<v1
zOnRY~Tu`HyGN(yxV_ShLNAPsB;)-tNxvl+5(<3!ZC#=^nvrS%a{_8rkdEVN?YC)NA
zEJ~OX75C`wQhrv_q7-*@uX)9C9p%G|yVTdnYFOOs+9|I$)7IQ;!3<Sy4pnoLtp8@y
z7oWD6J8y;JVF!1!TOV80&#|viyYkUp`7>*$ITyoDGvC5AmE0r0&A#l(GS}TVQ{~aW
zF4b2evsH3JrkgLQ(N^2gX>8uAxnJS<9Y%G@g&r!#yf$*1j2X;KQw!CMxt3@e)C#If
zsAjA0=51A4u_H`@`TcoiznO<snYG`WTTEWBQj~L1W#OYfl^kYAHKwQU)ufkk8967b
zX<RcbF$%d|uffbFs*%24MPrxuG4+*9fhOlzGF0ZB-K*UD`nS=qNv0;9f;Of_2fa+Z
zS1IUNn8uo!|4CPT`ux6X)uAsc$vgB+B-iw))U|4ACOn>I+__jnsp)R5=`34L4H1u}
zrnv?sCKulSGG#sbPBFFrw$Z)gGMd}012ty3wCQLHT`_*9b=ibB(o)T6zl@3MDSl(q
znVxDahf3AgC?%>cwUae^B$utBx%{<>wqmb>Gw&9SJ24@u>@)Vt{oSZ<=v=*8X|3LI
zZLL#&>cx#Jn%c(uR3i5A%imS$R=H~bO#NlGhw=2Ht7;3csHh(eQP=UE(4p~Pn^ALv
zfwif7Y?A7_qC^wn#7b4Ra7opG^vSA@qLxZV@kypWRkxM2PtQ>Hlb>&V)c%BNbK)IS
zskZl~?G{FA$<x-EH%D)g{}v~pa8mcPobp0Fv&$D4l-aD^Rd<R_Fzt?=r<feN)b#uQ
z!-_>8Pne2|xSB3}c-L&E;szxzy=f-tF|KN>j-;tf;1X8)+g)N>TQ}eI`J7&rXDeTt
z9tpZ_yh&1AN#tX;a?+BMs)Y-SO_|iTC>K26WxD*On$oUQwW_rnRw{EHC{yzOe%$zj
z$zc^c;hSn^yCx{?>(x@779Xc@H^f@u_O3wXJDi0|0TL%ni#B&C#fZO^TFYFbv9{%i
zvb@hTHI_v`%%<wDQK?+|#jNf385QaJ<to+3IaR&SH7Z1H`((1gUR7D~ot%<7&q6aB
zr(R>PtVXjd?HtCJ-aJ<4`q*m9ynT+sE;}xTLo<HMgbK%+iSS1%-MTzW<<8MA^NoK~
z6jVgm&B~WCC?!m>HCgyX#Pq=$ck@|xiArx5^qCf3?Nyzl)~eihH(q7goL2L5J2Xx5
zIWv@xW@i|8u+1}bV+_<0?fa&*;r=_7!*e^#=bq9~7P1O8Va=&f)aK+;W!nB(>3;fF
z`HKlI=KC%9RPw4lG}7{z70*uYRLS<cA-_90O5s}KE#;J+XyyFH-R74h7AZZNW+TTQ
z{7k-c-&W;cXH8Uj(ke_6B=0Jhh6tN<)dr|^tKC%gc-Eq#U(%!)`Eb5zukLombhFip
zQf*bH4558yi#;XG?jD<L%Cc5c{hEr8`TVe}^2Ze{749EURLGmcW43Fzy`uSs3YA8e
zNv2_I`{n<2YnYwn+N>0?VZUh`3$vMONUK?eahT%6=-FoX?uM%J%xP2Bf4pA3Z21SX
z>A!E89(|mm+`_fc^t?fiNozfuo}~2xWsTtV%9B1CnWbElR{s6i$238tKw<HPDJspE
zf|SGE1LdS!e9dmpo~QU*cdxny!)?VD0T$Jq`Zk3r(e3iv&sZzHpHZoldw|FE)DjP+
zUqxS~pM*?S+3YE*9Q(>i^|eN)X^@46iuoczGpQ?js&P>kDz|bKRWHOwD=qw+V|wT5
zMkP+}RZ2?|1uVE<{4*^P*>0-)HqH14zpI7_(-d>TP(1~UeRt*eKk`;w{Bw<Ie37Qo
zfp%}z+aX3~u1A{{eI_)SEHzP6NoiVc%DB+o)bH+8Q?rlKih?o|%!-1P)fnGosF*GA
z*AZM4Z~jEF-1MNZvU1Y*BGZn`;U)&Awg&b=Z&X%)2~c*7Ut_lDzqQKVEB=NHj4ml&
zTW_nn^W;XQ+-nErUU4iipUV|29~5Y-xu0i&l8l;)T13Db#l=<=<)$uQt8`|HuJSIg
znP%ekqRM}K_sezb6sS%(BB`>hWt-~fIwliK_PHvrS9F@bk3FMOeV|#TUYlJt^Ysyh
zF7+hSv+LI>J?!RFo_Tk&nM>1jGqsnq%;aBp8T|dXMvY;QgYnHS1w|d_84AnAR?1IN
z*lik`GE>R_;T09<+51d=gdWH{)ij%(T|HIt){A7*HDOw&t`qf5xo&+@{Griqn(*hi
zs#NuV<+@8p6?-0EHQV0FW1{q7nlg(uk4cY+jA`}5P%YkwsmkoN?5gE23(Veq^iZC<
z<g&3s8o%O#N9?N0q_!xn@sv`u53@C6ROXc5u+3dVQu?0)yWlmIOb2cSoh#E77~X$U
z>i-n1q;%BU^u9!fQmqr0obunTihCMGl)bD&RZ}O;F*^O4LG{hP?*=iNL8`%W&sDOd
z%2X$K?^9$burc9y!m9isv|CyG?{l+j&u5xeA9pZa{w2=v-=0zpIU_kE8L7z%f0jAO
zf0C<GsAykq%%rzgiHqA*RXM8OSY^=;#m@(KnWUZls;IOn)1>ak6O+Fu{~GzJtWfHe
zN;F%{sH)a@I#}gd8mHz)|3)({Ggsp&#}+ED__xof!(gRBV$c>{MMp!G9*qXohaX)`
zfA3hK9MrqmaDAkP;-$S_s^yFvDryJfmCnt1X*yl%mcm>CH%(KAHpSe6iK=>8;flv%
zPRrlY7E+Guid4ER%WTB2^+KsK<fnX?Z@#MhxoOJB7%r+hyEYh{{XJ35wIRp2LWx<u
z&}XJvUwW6?snFS~L3UG2&vkYv^@SNIn|ZOB|4|Gw6=Tpf<Biy6JYj>P`ZUHw6YsBw
z6f`R?E1P)nDl5%VG?^RPujKZ2jryl3JtM=SXaz6x`zD9(ZB^w}d1>ObV1<d=mUG7I
zX6;s}ED1B!<Xx%0dHW5uqK^kvoOfuO>19Y6yU$iqX^ZzXzJ6qy8S}nfIyc!F)Ovf~
zDLX7|GRwBoSNp`TXri&cL-E`N9`)b-iAw4LDY6bn_|2z&7M3@h6RUYQ&|dYP$_jOl
zdCbZS=d#Nm%&}B%UU^nk{hYgTkkT!cji0`$v`IB6b(I&Zxkf)$t4>fi+3-+N<>x~?
z(|5HuR3eNgsob&tqH;BbN#)=2TGRE{Un=<omnxObJ8fp7lxez6bfVcAu?<E~O{Qy9
zmcBLqYOgHk@6)EVG`Usr*7vC<EVA87+8ec08C3t8%s+iX-ls;&bn^XqO77C)CSIOP
zO^em@%`{~`DMo+#Wx6W$r|Om!fhxl3YgH{9+f4NfZy8VhH%a+_{6?ccFCEQG$~1MC
zyC<p?>s6?{le9En-f>Y`tK7!4@I<U)HLIoSyO=pj($C}++UD7sZ(gCGXr%R7qqXg^
za!=x9)#qUk6>nJWk=v-~psde5M`h#k-KN)#?<;>_lcBthcY=D5p_%I2l}W1WKdmul
zO<kfo-+i;GR>XT%&h9Q%<t{nZW1C#mb{=drJ>Q_IY{J^9<a{I6G$Y`qS@(pMW&)r0
z8&}m9t9n$&o7z8<R`~9psN#IlLs`^&t?7bmuFB~*CaT3uY%|+vASo|Bm&07`;WowH
z>YL22sM(p#k=$ZB=XtH7!HrpFU1#Q~H4Cj%RcPo_2-I>n+jB77?3?svl{S}SCUbu`
zn6me)Yh8K3s#2YztdiqB&4hK8fJ)hwc#{`j8x=AqaH*Z|V^NxzsIL$=b+75sI%Y-R
z-My-Ze0Qq)@Gn;TH$hi<oqU13gC>`9nMktQhRY94a=yo_=02RII#c<xie><-dM>w)
znw;`hllu{KRi4QlG5uuzR7EuUkV^0dMOB-dxr%1*Q_S*bq$q7$%%J?Z_>1|=qchC{
z%$A#lcq<!vyD)1w7dsni{PR}W{As29)DH*b`a2lRdjCl&P1aFVwFsSS`fQS=Lcg)G
zncbs4r6bw5O@uEhm_6Tf#FX`nnbILsTl3#OY-&Q2!c;8s)iqYe<ytiGHyEF2X;fyd
z<~Lq2Yk}!?_Lq9W_kSyIj~7(^eZ0$jt!urq`Rb=evwjOH?l;y{)d}KMK6{%>QS6<C
zh2oBN3Zl0wHTOmfD85v6Q>~xruTW@QqVRC5j<S2!U&SW3Po}dhQk0r}3+20m_ozr-
z_E4T6*`+FS>zhIK{)g%_^(BnF)GRdO3-78sJb9#k>~Np@qAh&JXF1O+FX4QrxMjJC
z$^Go<CMizxW=l5Z8OYXaXwOu?YP@pmCAr&wXR1y=v{&U)FPG7txVy?J%YSKHsa<RQ
z-OF6*lU%gfv5UvlB~0F!noQqmBENyp^h>6_V)Dl*(`R$;Y51^7Yltzc>sqSvnn-=y
zX<GYrhiXgYYeV<_xyEz&Id!hP1gRS+1*#|o_?Wc1t7>G;+GV7`8mn-$vQC3FXTGYp
zf|W#jLAK%AKqjR`;n`XZWzy=exqfTRh?uVOJ8Y873(?ogj*M&6i;pN8)e6s0^OE|c
zmh|?Embaph#&^$I8Y*^^O@1z3t9tg0vB~QyCbfcB?W&?`YO4DygVZ|q#+lAPCa$dd
zbhi@E+y>)p|3#)%-`AN=sJURO7U!w5yl0vD(_`BeBE%|HmU8wet2KNv4e3r){--@#
zO=l~+Y59Y@3eJ5iO}}LMD)RaNHl6d-*7T7=h1v95-xO4(>P#42m#b}U*`wMMmai*v
z<BrLn@;Ro5Uu;m(ZrNzMSijf!`W{EsU{({AsdJtwFK1e6;x<=Vh40ilQ?aWv71jic
zsmUx!P?fC~kXJUGVr<LFs?2A$POWtHDb@T<Th#1M#3=9am6e}2W0%spBNb`|Hgcv#
zmMN-Na+ayKt_)OttRbi#F#Dcbv)LD;ji0tFPxf~<nHAWj(tB>1ilO;Jm8`uws<YH)
znk?JIt*peVq@2*LWOl4#j!C?dkLjfQ&kX!uhily3*J1Fz_mRRira)zlhs%`aKk_zt
zFmJw+|1ERXtd}>9{g39z_wL(e5)fjl)U<P=v3a3@@v7ca#v4D)Q1mK!Y#eU%LiMy;
zy^4Io3{{qWj%H>h4-DqX1u9>iw$yO$nF+=P{1Q54LQyKMMrtZsKlPcGDNa<rYk$k2
z<?Io~g0F$9iKPNc^CzV%>}kGidf#G-!W@YJO=g{YDt?a_sY+j+rewOiO+HS4yOM!<
zscK`YhKcc<r7BG`C#eWuEmtsb7Ex=Ttfy-DzjW`(S9kVqTA#ePL!)=!oZrcI3=9km
zM_5kpF_mfBmm9fwUr|2e-qI`H`x>uB?bSWJ&SqUu>)y=1`FnLfmF`ttV!f~Sz2e^P
zQ-OP*>-6ls>=Uy0oSu>mgX7Y@&Zk%H-B_z(Cpag0ukqqZwx=h@+E4#ky4QA1^4`Gp
zb#`GqclR>Y3+(&-xpZ&4XYzj6`t*Hf%X{}7tUSI)VpZYZ9gQ#c>bw`QNpw29_t5L)
zy~RB%_sVdo*nP8Hz1P1gWbav>y>`EEIqiR?@@L;_(+>N3=4pF7mrdEbC7^H5Uyqsl
zw2O0XgVuQ4f6Z94r}(PY{yPaudxhms?3HWn-?zt;f1k8x;NIWaNqd>5H{1K!y6)Rs
zuy!wJ*CxA|mcYH&WKQqRUVMD7^V<D;9j;E?*JXEYuau13{s$jQ_BN|e-+Sh3*WT{V
zmc4Pci}q$0s_!#gpS*W&dH=qBKQ8SRdR@A=@5`=zrJ_oE#k)K9K8|_4H+JRLy%Hzc
z_wV-a+pjPqWZ$xF+xIXXtJ=5Vwz~bU%&Gev=BL=-`&_zrcSGRb6?10oRf^uR?^%1n
zz99RF`>t{%?YmqqvVTiO^4=}-E;h3FOZV1HoU+&O`^kM08(Q{ju1Vgz`P#&Npm<}g
z7O}s5vD2>B^nhK+%Qbs1oshQIXNumN{Ht`Ytc0MwF?-S8y_Qe+CNNIlTTwdIPT93>
zUt5sUUiOdk_bSYqw(sTD*!?#i9olz-HGkg~!O45xJq_F&|GjkYw}qMZ$FINKyCZ1s
z{><wyY|k*9vW_^>W}PS8ySKHaW3T`56#ME`rF%<$mF`_E7qVAvQSx54HGO+hzLoBk
z6ZW$^vq)@T@vpmk+jsu9vzoWay7xfo-g)<@*{jw{?iVcV+uJ{@d+(GrrF$>i{I%Wt
z!)X7uV|@DpK9=s~%Fo~T`^j``b{XD%|0|B|^Wko^>nNDD@BSVBef-Dk_bOeTxX*7z
z^4?`r#rIxhSGMnZ6Sensy1DhTpQU^6=RDfe@*!}q_P5f#7t9Uzyp<^0t8_kR-!F%V
zeF9=Zdv`_j@9R3+wl}(KmfaPH<h@@ut=zkFqS0Q(yifZ|g@gC@nI-N$6tKv)X@8_W
zvzg@nay6cPpLwJ9p8tGx&)4&%d#x_5-FIb8^4@7(x_f5_FWg%#p1Jq=vVeX5wx{+!
zDi*X4jBd4`yky(n&?(CH786(Q&1jR~x0QA2zDbHo_Wqws_c}F-@4J|=Vy~I|+`W~d
z;`@r9#ew5+%TxA!xmKZjZH1ogJ6gDHH{%gid+DS-duOt*-FIa79sAY4O7~j0&)@q?
zcf$U^4iomR+3B>WtkZt4-H+0}AAgqa)dA&Y`^|eFpDo=R{p#7C`95iTxBV#HoA7e>
zzOXgPdtb)|?|CHKxljLF>E2i$<9*B?%l4VAOWs?0J$Rprk<R`-U*GL{8!>I4MOfdy
zDf3J1v%OaD)8_uNSCwz#o>0YUd%t*0-n;5_(cVu#O7~tWZ{4@mDtPbJrG|T!d@tS0
zmlCpf^X%}w^K5%;r#|D@CtIkwcg~Cn`_3Qxwr|#E+r1qhOZOVhZ`-Rm<E-7JFU9*d
z&obKQ^R9I7+;+CTN~sd|TmDAvWiPw9PerwUU$8{l9>dlfd#7!izPEy<eXpNbhn4@i
z_<fT;uD4s@e|;a@Tjjm68!Y#4kn7oNz2DB3nRE8Om>#v=29KBQdm7xi_t3nUy~e>S
z_s&{VyLa=dIeQ~iC+%e^ZQOh6n%7=XUNO;_zqf?LcHd4r{k?qE6Zd(#P2bI$R%R2W
zHfLYBUg6#gw|n<$J6G+iky*V*-PYVTdsT_OM1{y+QNO=?ttKh%jriZQ?@z;yz0xy8
z_oi)sY+EoRc<-iLGxoh$oxE4dI$+;t!Jxh6DbaRO$1-h0&5rF=lijyZZJFkNiMvsI
zC)}L6SG;%azFTFV_O4`Jy?5L9(!HziM(+C%K4;IX15*2@eQen4?ayqp%8PSv^sHig
z9^O-Xb8<QMirMepd+E*OegBW;?Nv-?+qeJJ#=ZT{wRStsf849Kn{nTy@8WwuuS?z=
zEdPJc;esf;RhyFcUj0$J_mc+O-h{)QdlM`x_Rjifuz&HI8auT*=DqD%F8j6!ZrYz3
zHE(b8+T^{5iaG4fz8CCM6=2?L`DeF%l%Ui;<)=(GT$TQNd2Da&bq$l;x3y5kcEYvp
zy+Mn#><>f-*}F*!?~m<svrjt2x1Up+d4KHon0>E)&)MrHFvTwXTGxI%&gi{w)+O&<
zuBv3mcB*i1X1dn?>FbjBKDavF&fP%Wj-g0r@A{3UdpBPa1*iYT{*(7qcm?k*=+@iY
zz}UU-|HQVv8D`@4+6sYt`_?7zUAKOl{o@~#_xAm~ws&Sw!Cott%Drd*Y}_k;Vai@_
z-D!K9Ji6_+H}>ysoteBh``mr|i5wSgS^MVi`?PGLb%TBJe(6hsdk=dn?RD|)+S|7v
zd2g|&;NF7^SMK|FYr)=-FQt2<LGibAm0e!>gni15Ec-sB1@FB!(Q5D7za{(sWNg^K
zxJrND%R<(DPfsxJeJxzF|5Ch-z3Q6ey}PG>vH#<ey)Uq*d9SkD&fO}~GJE@#FYLbY
zXw|-qSh>9poMC%ctvk85e|7R+`;36S-&ZE@JrTvZ-)MgQ-bpO;_APl|y0`XA)LzD`
z`TK<@TJN8(_it}?its+c7Zdg!`OmelUaE2L;-95^pSTO})e)B7+qzGB|D~(b?Yh2n
z?e+creNSK6GrNf1hJ90bKijp;TDwm&RDAE*n{W2pKdst(OTBOJueVWq4KJ|meR=iX
z-pqUR!Fd3b50J6O(HL`NenHM@3uHbq>@^!Kx4_h~r*m3?*il!QcgmfU+6`iZaKB3n
zn613+=qeC@XIB3%)#FEZg4p=5q~21iyVI1ngVcQQe6_1XrbQXV{>gY0Y!8SH!p_32
z)*!n2!F8~>*O~P?AU-yH@~9%*Ue;UrV0U3t_sEbBY!)^-V#OpR+`#TyqZ`MItcN4A
z6vN&!51CCcHSXyPEMaUIz4lx5&Ry-DC~i6;E45W|$K9QpS<As;)Hc%|Y#&I^q;Ild
z_MO}qaGXX8J8T1q!|;|yCAfVsF&KSz%N(#h9iJt4-wKq6$1hCY^U`As_rv5)IXi*%
zg7^zg%YoehVoPWpwz6(a+YDmE@M5zZt3x89C}JQvwS||!{<q5f38q0}=vX-G2u?Y4
zwaEMep1UaeiqH70hsjG;=kK}_zhj3>c??Q?f#jcaeqRluk@2)Evf#9&^3`I;yjwHD
z{{Apw3X1vq+6Peh$ojFdL-_mQVf&loHaz_7KJkLZAL-kH?Zl=RS!}`7nPBzEV(9Fb
z$}7QchKaZSY9Y#On0jLA$BUXU!r;S^T5#F(o!JjV4rZ=ayRYR6F@J*ofT<^kX7VZA
z1=Cl4VkVruuxSRI4U-4aW*bgftDf)#yUA(yxg8)eWUQF|7o6XAgkIgfH8TdQ8OZ7j
zWly8T!R~~0mZx}n%#hU}v;B{#fb9l}%~95dr;oG?f^arS4jFf6JcozzioZ&5HkayZ
zi`484F#qe!^H}3vP}B(QKEqq9cJ0p>1G6t1$y+v+w4$i*lvraGUymsB<E%o#;h?^p
z8(cQA&IkkBBQiq@%tv+~I=fCPA0BQ+0em?91=81X!C_7Bvo5f^Sc5)V-Ro(B>(%`5
z4V=b6>hR%?PxIkn3R71<b-iUlQXPuj?-%rI1&6-|+chiqCzyt*Cx%`qIRWgpL~~xa
zUhDsdH#+_eK(Pm#S=N?Zy8J;!TVZlPn)ZXs^EE-8yYxS#gWbQQ{U11tVQSFn^bi{@
z55Z+xeS5^T<StFriBw&03Q|*iuua=3)k<TdN{^+NqpPOQEg@~^a8A`wcO^}(FEX08
z)5SIGm&U5EkyO;aJDpiKtmUK1`w!iUOu4UBrn0t~%`x1i-k>C8sP3p_&TFruW*(?#
znZNC@#n&Icnvaq_HEyOIu#~*mq&cNJ-!S&?5_LabL5uboKP+}_s#SDxou}Eg>a6lI
z;SVZTF4gF8oeb389KTsxZDN3CiS{%L*#>_zJH24dWs@u|k8KV#&b`5@+cvq-GVDo^
zT9)lnm218WH3iisS*AR@ZqylLYT5QcOzUkEg9-bhH9E5rUA6PqUR1IUe5&owr>|-;
z!AvQ7L6Lbu{s}E%KON1FJG?E=iL0o8+%r{s?qwY<lcg@|&))A-_xO;g#oF`2JUr1w
z+jmckdYhJ%7T<O*&0Uw7)y=l|s#!ZO)|@@1P&=_dMoV3Dy4vjDm$c5z-mImu&P~&J
zwYtHJxIDFsj<!ZS5AaxBQ|eOYUcFLLDo{k_Uu=!hg$0dTmsbQ@a(tSv?k2rKZEcsA
z=F@;<28EKrrYZVN+D|q-(VpjDW!90hU&SE6U&sINMb((~teP#+#TK7k-&+2??xeBQ
zk5K~@Zyyq@&6+NMHJhyd)=bqvLgSu?m(|v+OPYdfZ)*PZh_#F~l~!Y!`Op0FMpw1R
zJ9<pZPEOXb6Ufkh@sm^YD(75deUU!XwHvl+@kCG8jcj7qT*Ysq_SZ63qq$|C#az{6
zT9wMvEn^<%n%1su(aS5EtSw<Hpeg=;md5)JRhF6lAu4|TPc_4XwyK)Ntx`LBlT}Me
zTt&mYb*cFqyX_kHK6`4nf8TAf_fv*e`uyW6x4xaVjOj5mWBt^lA%4?X(`-k)ir9}<
zi{n!l8y`ARr!95CTKz~zmg3F7HCo%*tt<pDxaqW2Ra;bMtkrEY@=?8U{hFrCk6r2y
zoy*jtiw|gUXic<~)Mrw&*6Y+XY>n2uT6<m7^z$T@XX`CAo5I61Rc(be=1-cXa^xM4
zhVye)HR(U$2D@HKs84y-Y~g<Ffb#Fk{hHUF3Tm!ewBE3$;I!)Vf&|S;+se$%rOhmE
z-8DAynUSpBcVvdfv8^4NBGUV{{x9k`eetSMqo~YHvuKBiT5d?4<}~x^=CYZ;r5NmQ
znmTWDw1}`z(0msmW%<rnMcd;;qWa}!XAHN>hFR83>QPsI#;F?h<%Uw^Bqepul3U83
zUM)2Q#o_zk;(BkcF3@sRG|_p#MpJwGGHc6&hg>z6cuQ;in19CNc$K);3-#;j4o1f;
zAN^jU!YDAqtS@Myj6=sJjp=S(nj!P&X--&gqsY{ys>-~hTEm=ogXYwl#~SI+n=B7=
z25Cg9|I#_WgHM5%Nm3)DDNDyKOjO5n37=+kR<ZU=$5R$OKQC*?TF0x(M$a+MGZC_s
zj#{jfV$iB+Jy}Kipa!#goAZ9{kmoFl89mIJ%gZX|Up2o|vD#@T?x)zQ!Mf?bYVVaZ
zD$}J^bS;e<HN6h((SGxIjap~z3*8lWuWH<7aZxWnSERa>dz$&NL_WP4N6u*Wi$7Bf
zw>_*Oq+DZm#Qm*$ZFP#-ET-2=6R%lo8gkDtSSx*2dtc6Uwa7_lwW|LMn18rG!Lq=G
zRga}uR`YIgmchxxmYPpj>FY}D>d*;J6xGQ8b6oE@Z-b_zjF?v6qYh1hA6#;GK2KN3
zw@KFQvUXJuoT#J;3g-n!deo;JG1a}R(yy6$Xqw8T@BcNI)uiZnyZKvOUgoGJ)6}B%
zWm2HU-E%i}j2yP;9y!)%`7c{a`--`T)^BEiEoR^Snm>Bv)KAWyZRubkU@8AlUrS?S
zqq*!deT}<4Ou8~@`&93UN9zmqDwz3u%~n&n|HG24Rnz>ou#M`j<DME<mh~7+Z@90e
z>{FxRmC>cKSKrGpKIfRGp#_uqn`9~NdwHw05_fLUcz*erdE3z~##dbSYxP>L(^w-n
zRo#towr2en3$+U$3v^dZ+iV$F{Y86|WTo~}TS3e4y^WgNHfd__KOATA^stuJq0(;}
z6Pd25FIlre<MxM5T8Zaw8lO<_uvlZZRqHgnke=eq`{t7pd$jl-Zr2FPdT+Y#1+&GZ
z-*Q@l;!7>grZ#ANWRW!b=E9(yKQ&P^#_FJEyqmD4^aBBnqV3x)pZjoI#rp6Vy?e=E
z@i^p&#ocZ(!@QLoTBb*Jbs~39vDAIzYpLmBuJP`}63r|>ar3MjPAXUSpETz?X{+hX
zywKulw}IjN?mm+pA@wSPZ`!p$@nX+9MeFD5Hr2`7{;BgU)YVO!`$;`l%E<DYkc!G3
zMsv+<|D%>0t39+^YFDW5E0j@BuU(^2moQtc)}B>+g1f0&X!K))*}r#K#4vHHC0xB_
zabroBh0zT`4VU{7n(P1XGmCbfsPX=htfrdnN2OAmZnf1~U$i*#4{FaoETru;b+fwq
z34V=;xAB&m!P8WoCuOMoc@v<;dUBpdyqkdeX~F51cYIhh3;DgZdJoLgIzCmy_~i*3
zOA}v5jk`jzma6C7wb}mHX{MR{()`viT~jb<u~vbvsy4&5ecImd(o~&fE~u?cm(n<u
zxlnQXuU+bzj!!IH6V26}?L{ot3G%9&t(>Oj#8ztjWy3j3&P%3hYaYp|bp<FZJ2^Dy
zOz&8#bwDXXbG`UP?U}wOHIF+nn#rtJG+pV;trHadQ9ERrqJ^Y=h1NGu9}6Yj4$I)b
zn%Zm~hb_g8q}5-l1!@NT5ms|<<JPoWFRu1^;$m<f2)ej!@9h|geGc0`?OA;7=)N__
z1MC+hv)Y7AJZS5;VckBx;#OPdKpETii!=9$t8464o3e55l|yd(euzHXJGs7R@3pUa
z`wkk-+FP~g>fW9EqW11`Y}|8sKhwVH#?SUf=il8oA(4My&&s{FTy>xJekwAx=QJs{
z)?0tjwpjVezViFC_kCIad2h$gzc!IePVSXi%C^6zVfwyrhnV&rbaL98?{{bK)X#2v
zXWu)z_r74rp2v#E_UemU@Ave&X)78pv`=B_QXBKHNB25bTJF1C^U3a7mGoZSWU0Le
zZYAyAeDJ$nxlQK2$9876dOON(Rot8QozUO4*Xzk5+bsK)`|KNj?UhMWvF%;$Y}*!o
z#P+}q=Y4Ccoc9*gdDyR#>D?O@qqooHP|?2b$N%<jjO*UFD)HCe%e#K;-Fij9hKrli
zChV2jz8w-1_6DYz+s^*WweS4x9eZ28&e&IS(tO{d$+k8E+-LVK{5WH8YvKRBI=5Et
z=~2tEou#pEuW0XW``6W7`|f1l+WYv;w0%Eb$?okcXxnGN_;{~v^_;!IJfi#N%k%GZ
zS+Q~N;or9V``q>R-D~~1cXf`)UXfKAwpN$V?-sCMu=oChlD!5CT<w2^z1a6{2d^#P
zirxEOgp1os^SkaBI&{bGEK|2#E2p5X%%$V|c;y!FRod`!?;Eq{`vSe&_ewbC+5RjI
z-d!eWw%`83YTFYjTKn8Gd-m>@Q?PA3P`ID_CHH=>!&COn>onPWg2~VJc*Cl_nSROk
zzt=9<6Pt2v|LYdReFvHu?JhhD*n51{rM-W;RQA33m1-a7Kh<{r)75*ACn)dL5$UuI
z-~DpW`93Qf&9Du&O$V#@{b<qL+tA}@e|%c%-kI`C_m(+^+rO^+y|48H+dj^=nR}=7
zneTPmu*m+%;$Qn-iz@GXp}u-gwL!-|`{Lkz`y6BKwe^<n%X#tNcF&RN`_^n*ws+#k
z)q58{*}hL@BCD<Q&L-Q9emeW7G)~>Kr>?=)Sft5D>1pHcH4oeNwuTz+Wn!Oh+jpLQ
zUyRn;eSOP2_eB&*?OWQ|y|?kvo87Xql6#N!?%Dez%zoeU2LXGx6h~N<DY5TethLo9
z+0V*;aj((-WfwH}u^zs&_pLeqULGlpy*geVwmk7}`>S_v*;CT}+vct=<KD)*SN4``
z7Vc{b>)6Anz;Az6y<uN*Wwh<QIeq(Y^w#WU{!_WHC!lrTW#M0Y{_lFUr%y6`-|>4=
z`<`TP+PiAE_`ZG0SN191nY%Y?7qgxBjzzYI|2FSq-DP20;L2)$VZO<})Vw|W;<(@1
zJ(&8+Mo5p-w!>nhz5KC*`@Wv?wo~Umy!XSjPu7}x_x4^DT)0PR&D_1U8!Yy+O0w*I
zlGC^M(xp9n7p=A0e`@vKz2S3X_MQ3HvhUC8UwanD`|X=mdV8ObLh9bP35)h+{O+*5
zQ2uuB6Yq$98<?lt+<!4;{|d1wdxgbU?%}s>+`DEri*2x0r)`hZw7n~?p0xQZ+qmyh
z_sYF7Yj*8jS~7oc&Zi}NK75z9ZOW9{r(?Kh?=K4*n?szw`#hN^?pyWz@ZR(CO18g)
znfGzEsO^tS;j^oi?%Eep-o2OiZ@}J;)9w2olw{d9Uv1hyRj+y9vVD^Ks{ZoW&B!$0
zr_v;}PkEO8-u3TZ?sMCAa_`^I&-StBHSE2}XSpxlcGkX_A3nCb4g~G($>!Pn_sp7o
z=6g5T#K`U6`)$iT%ZIzG_PSo4V)dnds?9d<&wCZP%xvc<?A^0UsCMt3|L6B?nAg7N
z+2r;6jxZSPlRLC_kKL8+dy^*T?0f$@dSA<ip1t2rs_vcn$Hg|-PTAfw?YQm5)aJdh
zr&sO$EbwKo^Q--P?Kzg&&l2?BSDNr;&*L`#eP^R=_bL?5+h_JIY;T%G)ZUxzOZUol
zth2oqy?cKmuY~<KRj0krHecQQqmkLJW{==LS?5AqHeW~E&nB|_TCO<k4SzXx|9lT-
z+e?=%Z56zC+jxBtKG3{q`M%;1>3w&<a_xKXo4-%tr|@2vJIVY09eum!;WhXD6TM|^
z881uk6Z+Y2YxDZ_-gc3~J$@dR_R>X+cIs9pdjoW3_GbETw-#UZV()@g`|N)4Y1>6M
zaqr>3>c21Waqa$`A3t|5no+d(yNaxRIJe8*-_|@fpF@QY95-Xw*OmTr-=!ist4@an
zdpY`g_MI0ow|CU?+9MrizrRf5udRGU$G%|pMf+Y9vfIyOUuetqJ#HV<&XRqa@6YX3
zGe2+d_q)wzW81^s&+c~EzInsHFF)e5&B1^zd#X~4_HQ~6zIU4Oq<t+qS8Udo2JO3$
zGJkLI8D=|{y={8~?tI=;@3U*4=r3FAg-0jYZE$0;zH!!JZ*P6*e*LAKdxLiE*t?N2
zZ=bon>)v~D)AwrX)$KEw%x;_4!*3H*wab1*=k~qF1*h5nId*n`-lmp)O)ppPIh9no
z?{Ib7UZ&D-w&(Kd_ht%)?dkDk-?w1VnSI3*j_<8m`C?Ci0h8U;B+Y$c90&HzVfnpx
z(pTTT*_ML)c(s`KtE`@CA9}N6pWoB^eH@-W*8G{b>}CZX-Y+0Ndr#j7Cp%fjP5azG
z8}2*0qR6)Fc#q914~~7?mT%r0XsEkyo~M>=KyJTH)j|2amOPnzbz7zOT7Ofsm#b9X
zYhL<$&x%`XZ7U<?_L*qiwGB9P(T-P7VBeHg5B8qWUAiwPhJByk6>r-D`(?Iv)(318
z`X=p^s(fQ-xi)66_GO2CO>ui{)sl1V?Zxf)-9Bx*pNk=7|8LV9*6+El*w^s2@3~O$
zd{1g4$KK{%j(rlzzxOUt?cT#GpT5^wYtr5W9Oiqqissq$Sn=38v9Gq-zOiiY!G*lG
zuR`bV-Q~b+J2iRTUg5<PY=5<6?EP?GY45{wx4ppzc6-A@I(D(~ud(Gz;j!cRa>HiT
z#L2dLVUzYQ-?7oQZe^Qo59=-4x8kX`R_2~Ir;Z%6Wq$a?=G6Mgy;oh-Y%Vm;*~3>F
zVf*9Q!o6qx`uFaAWoE1Q;=PT_qfdJt#Kl|pNY1eF(%G<g-@=+bPo%EfmYvzTm$QGp
z?Vf|PZIi4v?QOV`Y`Z{Z@$Tuahiyx*bJ;HJWV7us3E%tVpq{P&(OBya2SK~VYc}lZ
z-e9)(#D3=8zuZl1ucjT}op|7nt(oB&+j6J3w)Y!b_f9Zju|0OzX|LIe$9tVq<ZSFe
z$L)C#{%Q9CQ}MkA(`VWUP1v#b)r?kKQGq(^*-cS42CF-E@7cP1k5>}gzJJNVwl8)2
zc2_uZ@A)37xG(r?@!kvr9h*kQm^~Yq#BE&~vuvBB1ovfxKi)H6|HkfHS?Bi7c0FVp
zHlt=Qhf&ksrJN~y3!XUJPR`i9S75>6y)hizcK><~*>d)6wk=B(u~q15+B=!=&Yt@W
zDtiSC=k9jeEVyso>9e*Ojm>+LqYv&q_y4s`ZL6eR@=_JsLl2twmK~UDtGVczO#*MA
z&7Bnud&T$r*h(mN+5F3zy|-gC-@cpvy?d|u%I`h8sKD02`?J;lGaY-Arfu9C!Qi;(
zTHab4sTnu-yfQvzqZe?})`R`rUa?z`ZDMU=_tvsJw=JAEZ|{ZPAlu1?2lgr?ZLv98
zCcf83EXmfi`{dqVpH=p$ZhpG=$MTN7y{t#9ePj;p36XkX!|SoaM$V;R*XiZWw*B)?
z?>!>IX*<tI(Y9Xg%Wht$*L$X2{cCI8B5!-akI7c{xxrr32hn?1vK-$lY1_BwhgsB~
z(8U+`99VsN*PVmHd$&c2?tSpD!)Ep-@qIGfjeAWb_S@P`6}Gi05Zjv*acq~QYxiF9
zva@>~o(t^tEnwL<%Z<@?vFhQyd<N_H8dPNLv67f<v+VqGTlZH^d*{B`zxS<&>0X1C
zMs^OS?{`JVB-rYvt=@Yf-e`}-)46*XzIfO*u=MTuKT&cY^EJ<XmwGw&m8?>;yZQC&
zo`<$a_D28g*{jguv^(E>_uiQ~{d?!82k)Kzwsx<YyNk`?Co*<(SPb^sG@Z7M%sywM
z#x1^&`{t@W#l=teJXqFht*&ovD?59tZOet2JzK2a@3HLC-aEPdn2p2F+cwXq`R-nk
z6=b_7*?jl$pmnyA1v~b7g-Y6eusyX~MD&Pl%WG%bk1~7r_RajUcZ!*i?FNZ^HY?g!
z?45ID+g`)V+iZilkMC7F{MqIQ|MtBGcS~(_UvAqQ-nen^41fRKwX(5$?oDvBIhc~T
zTe9=1ZNsDMHW?Cg_NEuv+ishly7wlN=)NQ;*}bQdZ|_~$wQp~2jQHMo?j-B;YrJhF
z0(R^@a<kdCz@=|b-gLQr7xot0o=;n}H!0`Jt^}@@z2+Ns?q#_nY#ZL2y<>*#VVhGg
z+xK>sud=O}tzujAEy(tQ@ti&3VxMgvFE`mcmvx~{THk5gU5(#%Ta<9xCS2^cRoo=A
zx6`C=?+V4|d!v<4>=j=SzjwFJlD#*!aM-N7_|103Sst51Q3q_U73c3&TUfQX;ZC3J
zGij~8JLM(z3Ks3MZ95rh9T2v9uh_%$dm2CH?ah0<cyD*$GTT4D7Vj;XwQ=u?Qqw&a
z`?GANm?zo(zBSdxL1@V?0gqL-)e^h+GT866o$0t^?~xhdwhK=F*t_Var|qUDhP_M@
zU-mFWPTZ}qqsrD;z`<I1>RapFFFZE=!U}fR`X<>br5v!W-E)8E{Nm=l4NOM6=H5PM
zyJgKJ+oXEAy@9(mY&S73v|YniW&1~c{$BIT*86VsHty*u4%_>#<J<1X_ulOlN%_2&
zHzn6Lfrr8Nw(26=-EH=^?R^fm*N%7XWe#~_`^u+tFaP=_yR*A^ZKH4Z?$MFFzh_nZ
z>AkO|T=p)=d1mv2efFLY+pq7wc6F;wx|8wV5{?zNEmm`EZ~scOb(ti$FF)z)UhOcI
zJt7sdwt8vP_WpP4-gDwz<KFdZt86U{_SjCHbbjyrGn;o^U*TaZot$bj$z{@B$+nGq
zb)L=LJ1Jz}-pcmQy{fY6djk$k-m_cd<K6(qzk9A!blFBqPO#l<@pLbXq{7~ZG4i(U
zDy(}autx7a$iQkB=c!`z{9Tf5#D+CCEHWZCCRV%lhUs$KntX_|S=FRydq8oy?T<4y
zw%KOFb}SsVdw(jn?lsT(v!|zgvXy%2=G|XB#rOS)o?=_EY2F?mp@}vQInK5-<~+Aq
zkf31Wk~(kilZZ)sj&#KCvEtjfms_`g?|dTx+uf?)_O6}iy*I;bvhCl1rF&KCm+fWU
za(=H%&tltaQ$%bmCNtQ!nDp&=mS(b7fq`S+ud0YW5jns2T>ZekcSf$C&GD*^J@1q^
z?OD;kXm6fL$e!kq_Py5`5AWSjUAu=N$=doH^RB&114H)a&EVTtQzUAA#z)CcfqnY!
zC1-E$IgpfPJtbqdjljO=ySL6u+H2Q$a!=}lNw)pP{Co8#+}SJStiH!h)z<dg%ei|*
zoXc&08Jx4_V4k=~c-{BCGjhcDNS*&@EvB`=w!r70?SWaI)+x$Pd%rvq+UFyB*Y>8|
zwY~BekL?u_J8bLWC1uO{C~MEZvNg6^iAVMxdRAuZ_i2-DjPPCCqf@l)Sas&@{j*od
zcCPjIJslzw_s&*3W;@?7$@Wsm9NTmHclJsgW3Y88ow8@XP~4u3$J=(htykI`qH@*t
z3iHanclLPgE%+#~mouni_lM-x-S0%t@0IX#+B0ds*uFA<g?%iC)9nRsov_IjnrD6Q
z!I8bXZ<+R5bV=-cc#>lut9jSnCrRb|gyx^!tM9$aCe^5AZ<{3lKFi+=>@1k%_8xbB
zWLw#pzJD9Dj;$w~pMAyBsrEDH@7^Q9X}|X>+qS(^t+woAer&hzf^X#BuZw5xulUMn
zS1!=KFQK!;_6HxwzWnn7do$P~Y{mW=?%}qvwdIbvx7VkA;r`$V_Wj2fo7oEa8{7FU
z?X}Z6{bgUdMZLY)&*OVd?rhlm`s9_pO5a!5i#>9)+cT5j;n~^Wdyn1jwXM4-XQvlp
zzISK$ihTu3OZWODi0wN(?c-kI(hK|7DQw)AxA5#9&!2tv-(}ft_t}N-?SFo4kL^63
zeYRpPd+iP??E7W0*#7M-fxUw6jeA#zGVHtDxN>jIrwe<Xgv|Dxyue|zPOxX6cE_W=
zX7lFQ%$|8{-?82ed-)EQ*nD5`#g_Z8_`X#!`}Wz+wY2v?*}gAx(u}>v4UPNu{JCcL
z&SBv`xo^q#<;z%X1JhpYm1Q>A*SM`~@6!WJ`(x^6?R%>fuvaJG?fwta8?7~Eui3KC
zKel_H&eT0Cp3S#+|DJxpW^uEPa9i_UhEq&?-A??n`(9#X|JCB+-dpp&?-eecYR~*>
z`(Ekt7kl^pd$gZ1>)_ts3k4natL(5jcg)#7)o#N+L%mhICTFqk^L#74Pb1V}zq=66
z{ufKRY&Wfs+1q<EcCVUC&)x$wmhYcYF=KDQ!Os2FZu9Ii1g_g|n?HSTdXm__Kda>S
z-P#&rzb1%d@0nwwwiY-2_ogWy-F3f>XBRsMmn|p5hkbiF1@{@u`LMU`z0-j>(KmZd
z3M6dra|G`%Tdug*`ce2^KEJJdIXCm~%X7cJUp7)?zq4HLUV)smd-rG+?mNdTy*Jt3
z)qb~T>OQ+NS%>d@8~4?J<lVP*e%Iav`@Oc-^JDf3znQVOdHtOIEvt^~*-*@}SM6}k
zp1A2#_FAP0?3=eDdhe|j2lt-RUc5JWYti1UM-%r(XiM9i%6-1KV+Gqjy)3o;)>gcG
zw;Iarlm5cL*ZJ(MJ&D!-_B1V&-piS>aqs+$jrRF3Irg~+$M4(!bIIPd-P3GhKXmMw
za%Gd95kL37+L$9YWv8_Eb-OLy8@A10|K0-|_WssqvaA03*Zv>#vc2x9ZFZ91z4jX9
zv+TJRGJkLPl>7U;Do@*pe0{Wc<I<CR1N#~G8S`D+Ys?pBlOwy>)-J_!-@ojGdzZQ0
zv$dJUwC~95Tl*qDaofD%IknemO47b>F<15;<q+H3;`Ypb;^ck%3L0kGndpS?wYzhF
z?`-298@=nDd%ZWC?_0rjXK(QzcUzCUd-r5yPTd>fyTn#rqJ7Vy_i}sci^KM=e$2kF
z<GJrXj@Dy)Ul&Z@!yB__Z))+<{gMhfdslzIvG48++5H~^pV;%>U%M~l&AYuJFPH5t
zQ<}ZY@N3uJnQLWjHOhnbF5S3%FS}aP-e=QJ*m})$-COB=V{h#@zCGU@Qf+3hbJ^><
z>h#{vYP<Gj-`KTRc?a{}rI|*1AMTyL*PnUTKCR;)>;z+W?A2JjXs_gC`MtCLtl7K#
z;EKIXujA}x`j_vUa8<>&{V>a3Lzn11Gv1Wg8fvuKdbFO||H<l-^~Hzp_VO6++I!G_
z?Y^mspZ9uX&9gK7b9k@Dn*}z3>tgnO@@lnRR41{IXWqZPauGaxvzm7AopxSy-#v$u
zdsQ91c5$~G?u{(r-8Xfi#J=Y|l6&7?@7>$!{n++c@L%i1ZTt5HZ<X3x7Q?&OjEl)3
zh0nt7^mGwBJ-0b_tSg!J32n07x7>HSP35x{`<hh!_huOg9GE`mi|r*DyWP!d`}fTe
zjooWt`hM@}^RxC|zPQsaEZKE`;i`Rmr%u~!&tI@&-=@Q}ZB0bF_v$@6vG1?QgFRVm
zx9;(1@Yt&qJ!|jg;|ndmJdCjmYk6j)-QK!aRgTqmf7RDLN1xi*%|6JnZ>>Vnz6n!i
z?Dj4AWofocY@fo?=zV!dl<m(3`RqGB_r~6PW`g?_&)(mA_V4PwuRLb$57yjcvzkx9
zcEz?!*7wXa_PXqwzL#U={Czd2MEASO3GdT&PTkwulWH%y^w?gBd9U~SHU-&FTJqM;
zlK1f5_+20OcQ2ILH$!**9vLzFy+Y5n?%7iMZ|@<i5BptGR_$%?7CbQF*A)9(@2BlK
zzv+to!x{4S4QU+ve=Nw_=OK`8`%vWG{_FLT_F}$ndo`{l?`3A*ytk_V(cbLjc>6S+
zH+$1r%<a#c=-W-xoM#uA_<FB^|L467e-G@nydYxx<N6ZoRa3s~P56Fc&yD(J)=Q@*
z+A~$=?LBj7``%-bYxZ$XTxokDQFLG5@ASRDjBRbM&fT*&aB0I{k9pm@_1eVky4LsG
z#4i50S4Yopud*MD{qtvUtQ8X7c87%M?r-B2v<~9lxc9Tk0-MAfy}gS<rtgi6725w@
z)x&zK&Bndkde!$XoMvNh_BC-|I*0e(u<sXa^o^$3E)KCgz*h9#?!|VmeXSMV`x-vY
z-Q!pywU<fi&HhN!S9?{rb?;~YqPVwOpk+_#jsRQRvX6TwuYSCDgLU~n!)e{Nmiv$H
zU6F0K@2Gmy-Y1RDHoxjG?|sK|);>~XyNzL#&3=t52dw$(R@tULHQsME=fwUww?6OH
zuGQXWbb)8DbIIhr-JcTnh+KWQx07+Et?A8*J$5<|_fCCu%C6Qe$#&cQZMN>arS0Xv
zt=#9rD8JwRQN3Lpd-vW)65H&}`Aqkk*T(Lt@jAZubm6o;(i_|MW_?(=_rS{hy<Pfc
zd%ahk*?Z{9fxQhnTX$ccGiy)R?vB0t7I)aX|9obf{zKVT;LDFaYhrHhe%`)t&yDOC
zHcekw@0Ipb*}MGbl)W!1ci9;H4BRWQL10fo!^zz)*E08RUpL#<<U+FTvZ9l=)oYpT
z4z8GIBlcm2?W@ZfdjpwD_A)3m?fvm$>E5&NHtkIgP~P|9?Ww&iS37Jg8;{%aKQP(r
z&@<6iM`@nzt(J*<>%ubk3Lm{~ZBn++w!v)AUd_LZdtMqi?e*zXwS9Mf@!ndi3|s5(
zi}%hwxnWP>LT1|y<%YHwrJ441#7WxiwN=@7KSRQHT6)Rebt+POU!D4GE#n-vw`0m0
zn~Ec^tVQPg+HTsi(&o$2Gkc4x%xpKXP1$=Tl5<bOx&zjeYS!2)YRlWMx!G$wZBhN+
zyWFm}8|KE@&hFo1J3aR3-fe*|_U@GVuy?xMtUZ<mTlR__oxexdQDX0j83{Jm4*2i2
zYwO;d@_e3+cH|`6?6AVUBFD4#&d%@HEB#K(wjt5S=ECAmTZe`THV1Ytvw7a(y0_((
zj_o0?RePNNwAxxkEZh4pT;6WUboSl*)CFv(Y<94<_;=m*<mA#lA;)xWXZp?EoBZ{>
z4ga&*wtR)tZNF`?+}rau*mmO0*uC$cY}hMiz_M5J?BTu7)Z}eNthVks{!!g_;i6(2
z^^_|%4mb8%iLol~y|Ck?&8FgwdwcR*ZD!ruXS+_iz((Htk!{~AKbvg(1$&JSt+y3k
z%VK-PZTH^02c>LM+uQceYu;@8C^2&H`4u<!_O(siyC9)&w^G=ty>i_T_H1Kh-^ZW0
zXYa@9zxP(yweQt=^2H`&X_BqynUj0gMp*AvP~+J9LvgaL7~dP4kR#vsPW^XhPi>=z
zP1w^@wlh8S_szL_Z;zn<zrCBPHEg_m`1ZN1OtEEraAEI-g2%hH79Q9eVza{b%=CGC
zmuz{t_rgS8+Z`@T_jJlF-1|?V%eE)?^Im-mF`I-PTzgZ$9@rbHKY4Gg>IR#$Ev0*>
zO<%Hi-4f+}pM2NuO%qyfyC(Mf9uFstJu`lb?G5qIv~_ruZ!7CH*=B;Ct?dOrQCpKe
z&9*yIKie$$A-B8bJI_Ax10VN3IP}rFf8!S$!KRqKHy(xVk>fjRJ%L%#_WQR4+n}#(
zHhb<_*`C<2&{j=;?Ott<JGM3t&1?e_LTr744%*zZ;M=#>`QzU9X_IZgte3G>Khj})
zd;Z?NDfOJTQ}~%|Kg6%Kc^bm7Z_B><wrabk?NxAfwXR}gvOBx<lFfk)H}=-sN9;;F
z>AQD9%Ddf@^0w`*n4@R=#JqLSk5B!3d!9G#?Rvnl*L;fD-k3e+wsUW1?3H;OYh!eZ
zd7qO0dD|ary!R+94Yz&g&bY7iPMXbeFTs7mOE2!WT6=iUg6z+`ciM>VJN@M6?tqZ2
zJ(CVPT3X~y-8*rnkBwFJT3e<A*Y>&?^zT-F!eqy_OK0x^$K$*91a|M8@3m_$v*+2p
zeYd;!Zp+W#JE7Opmc2n^?~2)HZI}<Y>`f3pZ`*B|y;rVT-0ltkroAzOn`}Q^*=C!4
z^33i;hLbi^yj1sYaXY#9b0Vkhq!-iordIasEfG7iN22G*o&Ya)yMyN!?Y;9^(2k>$
zfA7V`%lE8Fm|-jJpS3sLxoVHzU3r_8WoCOAHqYF>c-{h=k6VoP>F6}tNT~<fzV?r?
zy;vu2BT)E%cd+jko7bi}dsk0lwOjI2be~7YoV^o5zV7+ae%iLJ{fbS-#lF2AI&8Me
zh4p*SSC{Vn$#-n;f+chIel+FUmk|DO_v@dN_DXDw+1qt){@$JWEVjl)yKM8F8}{t9
zG27d_M%<2x`=!ksoAb6S-v8Yz`S9yr)4er&xleW2%G!0?YANm8tK@%XPfW71ZCUYR
z+w%v+_no_|zi;o8#yz?n{(FtoB5muJ3GFjFyJGL3?az1bv6-{?nCeZNL%(!w-b`ZN
z_u|4#+qpG<w%%Xf+T4D)fA7jYGxlT^FWf79v2*X*@5lGfYCf{}AN!QO7SEpTG1K6-
z{dBQ$Z&zcVt&W(Woeqb+?W2hH-G!%@?2W#YYhxsL#YUvI!`j`cW{+#XhwYZ*TDHyB
zO|~nJoZj2gQ@{7#qlLCl8Xnqw*nNGEb@|afkxkBfd-LDzZC$0f=bx&_UgpyIdlvlp
zYjZJZ#oi@TH|^OVc4^O!jeK^;j&HM3?62Ot++M|Yv7dn5CcXK4O-@|1$vN@cn&rUn
zJ$IrxZQK9O*md2}Z||Lwk9*f`UTC{)*Y&+j%ckw^tUtE*r`rzOz1CcN>%A`QnYU+`
zZNlq&w(pJk_MY+Dv`5mvVeiwT@VySJ`E4(1SngeOd7&-OrPVf-SNZl8WLewJe-^(d
zt)|O1bDsL%^oIg_zswfj_u}f2Jsj2x_ndrk&341SwYF2N#P-STDYR|d@zZ9;jgPiJ
zH&xmme7D(VLWj`4JLdCjy||V4-an_ZC&5>G&mzxtdnIS)?Wvixb<d?$H}=T2Jh$T8
zsIX7vc%RLN$1ANrTx{JtPw&Xy*8xp7CKFn0<}<zDEwJCjrmU7@FGF;Y)jUgSyRNmZ
zwv*L__P*6Wv%6)A@*e(2-nO%|*lm1&b?)81{I<=l7o7V{b-wIb7Rb0aPC{j`kO#}Y
ziU1zlBM0u=cpkZEn^su6SG%Qr?}mR@ZEy1j?pa;JXLosFgH2TT(Y*qk*?YHyEZf7o
z_2Vuh-}$yJqSN<WnZ4ZBX2DvU=6S4kvrVP#may;Ho8lp0oA*zD?-}KNdlnW?v|W5(
zY>)CP2HWjUO}6%o+P3Y?3j0pYWZV~WQpkFaT8r)3d@;Kn-dZ*X_ORQ2*)U=6@#V2L
zEbNVYRz&LA&O5!vR$JoT9_EtGw&CqvHjaD~_geg)Z96S-&)#DBzP)dDw(R}G^k?sf
z{mnKH&dA#QE^)KHHF1OO6sL_g2@}rkww&-}&or;uwp|Q7Hvg9{wGM3iX5$t+V|Sa?
zo4paYSMJ?=X!7pz)Af7*7_Htr>9E1BA11T+Tv|DO&vhQgy*hW@ZPi{T?EPc?cFzO0
z!+RcyPT9+7tYIrLNyK)tLit|v3omRPOceL6db-p$M0on%jUjrrPoK@%yWQ-DE&tlZ
zdw0AwusO#ub?>&rlWgy>9^Mo4bFyv6OQpRZ-?{8ryyt=Sud{Z0iyln2ZMa)+!;-vW
z?@Y;?HU&;ocH1WJ*=s2%ye}eZwk_8&g}obIZ?|sKk+WfVw9x9s_QN)oPrleH-jcIh
zDl^%3g5RFqTCwf6S?A^UnNOA6*Dz1qM#YWCmSx$;y+0ls+G^-K+WMER-YfET-JVr-
zcX!Xf)wkDTZ_J*zdl&ABE}gS`rqrpu9KZhW(K^_&w=Sg7cGtYFy}!kJ_NK*K?7dKU
zV~>i5t?i=xdK(^={nq#S1omzAQP?MGsJBPv|CYTeZ~AN#@>Ol7O!>TLU*1vMFZZrl
zOXw}yd%<tXZke5Nwrnm^dpES4+I^x<U{Bz^Vw+5hHf!Na*0v14_wAmcAY@y>)3Qf6
zdy(z4du_H8WHtBRY@fMXqDIt~LA_|NWz-*A@m2x5jGxE%YS@SF`OUIoufR&aJ#~kw
z_sVq~-#dlJ({_dN4;w*&3wwU?{n<M=eDB^FQ7`s}nJnHLTAjD|L;gzJlUJwijWO7>
z=YgTq-tQ_CZ4y1E?$woK*=NOPxYs7C##Z~i|L$2==I;&5mEKzyq-d9`u-SIMTgP50
zk)^gfs;=%~J1V+2$RlL;2eWT`40yI#f2cdS_rA7<ZR+j=HbN&8_dam7-NSN6eOLa|
z$$KA&=j~RD^V)lB)}dV|6#Diu$4s`ppe?j_e!IlJE9xG$)katMzLAZ#HF*=c_u!3}
z)=Q=}?p?B<*Y>yeuDx3%X4<%ad1$M(^|bAaW2JioS6<q?;qe`tOXr&QuI-v*`=ZXs
z_HOsSz0OOg*!*5(Y5R$V#nw#XyDd|I{O)Im_-!qwGT5s7&Dy&r<=EbodEfW!Ugf^`
zVh@*1cKCVQDMqHY@7VY5mG)n4+q`7^o@0t@_XeKWZsm7=gYC2Frh7I1t=${4VWX{!
zN|UY5VJ}<BE<xK}o%{A4@VK_eBU@~5!I9Lx>hXJR4DIgj&DcHN##4UHUY*s=Ho9MQ
zZ8+GQ_tx!Z-`fzcX0_m@xot%2?7hVi)q4}xt=?0pv0(3P$EkaT+w856*~ISM@?e_n
zmogb!4+f3BI<dR=7V!SDnUJwyud(FGy?;)*+Dc8FYdhEIhAmV1eB0PB;`?-7oZNH1
zo^PMc!<xP4rth|W6|QL4|JB6ik2TZYBVVTMJ#^u)ZH*=G-j0gbHXjz8-|fER%-)8%
zFLwEwY}woOsBdq`{!M#!JmKD3Qa-`<bk-r;l#u>CKE9Lp28J)*yGW*eZ@h8io+U?j
z*~l}U*z0mZV)v@2R(lWZY1wnv^Wa{s^X;}#(;96iPJg&(dRvmMfu*wT=FQD}*T$T&
zS+GWD&&%4md+jY$_J&A_@Al}+v&pbd+f$#LxZC(Rr`-oxF1ww71oz!~$Yr}xk$K;T
zgq^kr4}|XJp7MI{>W(dYHZ)||-ut?LZ|0k>y{C^T+ivN7uqR`^sO_S~J+@1fKJI-b
z$+yqo>l&NfznnI_Z$kHeaQ$U-tI>P!lh)0)9k=J()+eypo%!6nXZ8ENwvrD5Y!8GS
z*(<Zj#@2t)#=Uhciu+oR)$HBbd(SqadX?3slT-HC-!s_Dxaz`QkIVCHvcCxJ6WcU>
zZ{|Y1y<Ml`_Z|>*w=J2w+D7bI^<KsP=X?L_I$5XpEVj9|a>Cw6_uXxtrXJe6DCFCo
zTP;&<cidIAx-|de9{(51Y)|Yh*z>O_dT+`9&3l(PS?*o<>g3*A{PXtm>}K9q<leE@
zxJkz@;109x{8{hzJmdJa_jHZ%-Yx$x*vR&>*m7hV?5P#y-Pa_zVsE!kmaW3-4SOfP
zi?g+~X0w~@xWM*m#eUn|ZYP`YgsV0`K2F|SIM3PUa&q$Cb8oKMtka3w>zQn^*P?EQ
zZC%S++p^2;w&CrU_vWnO-8*g4LEE5heS2aq_wH#wG=0xy?OV2s8K>EP*!|CXbLSS@
zHNhwMxY|Y77EXD*cgOK*dtVez-?L}KlD&@kdDbyX3-{JKpWfT}HF3A~p~bdUntiq{
zj}mQ*pZe}i?DMkOQ$1tP1eHTJ3D=%l=iOr2`*a2CzBJ$MduJJG+VGs>wte+^u5H>j
z{yl;By!QUTd3?7<kNMt5SD)-@i0IfmSCD_-C$@mSQe9rVUC)TyJ!_HO`$_k>ZA4D{
z-b|k})(%U~@6ErhzUKl{m+iqFvbI|$>e&iS>)Y$t>}y+fmTT{KebK!O?oHh5uw|yr
zi$z|x4Fch|<(;j2{_VSFbFoRv)<D*N&%1lOY<IYd+X}qDZfnzWW6#A*^Svv#1=^lS
z=CkEF`Df4jCEWXjxjOblcg)^<dt<0g!RKh(X<p5H8E$x3Go*an+mIn)GwJZ5y_|yU
zZFFYN+RM<#Z_9FQ-(IfgPJ0qx@!7QrXxMU?h}r~ASiLvk-K{-l%nWuq?n$<}t?he_
zScL7a2rSuq;qn68SO53f3N*IuJ!}!T_iy3U9cy<i+I!&og57iF^Y$+5S!SC$Eyb2W
zUSwaz-VJ*_E=cV4HS5_M#nZ4iEN$N2cQd{A7W-=LJ*`o>w>VH?H$#blZQx2XTfwxt
zz1jVjtrI$A_AU`>u(f$`*5)Id`rZRgM{MG2|Lk3|aK+xl+tM~cJ2>~reATo~=l^2+
z<GRw`-TQL)s#+f2yTbId&FjS_d#B(2VOwyi$u{UW|6Y0iHMZN5c=xVmd}ia3RI%52
z#`3+^GdJ0u_T092!keSEZ41iw{;Swx<Iu5t*QxuVdnMk^+<V{{+um2+yY?zrh1%X^
zZrl6yZ0_DQTpMgV8@cvH2{P<$Xt3XH;QZaTREl}uOu5f{rhPbVGsi$>ue4A5-e+-q
zcFkh1Y(r*6*m`y`?eXfKzqjz>X50K<llGb#cJAd~_s&Ldlj+`=70YbwBzpE%*L&GU
zua?;N;B~C6%Z?TshP+Oz#a}kq8uZVx-jNz)b8+Qlo5Mn-yFVH%vy};&X#22U+E%A#
z#~u;x_`UN(E%z=>*t^$Vdf(m*hS_`1J?GhX!|ar8ht@2ckWaJsZVHRpJHzhmp43j>
zeQ~Ea>}Cd@+-q=gzV(a)TWr3(x3iU6;JvrsE!B3Vpu#>~iH5xan<nm=d-R9xpG&=a
zTc`Bf%sVZzZ`mQ|y?-B_+v~7RdCvk>3+pwnSMD*Eonp&-<(UnOVfJ3WV;gP7?(eca
z@jQ92$W2w-C)VAzA-y+j88-;nE&1_g?~!-^_Fi1#Vrvs<Wh*u%+h)u4je9=7sNSQ#
zbQ-Adw%2(6?maU?t8K6PxZ3<Vr?c0?P<l@w)6qQ{4;*ZZr%l*fWj|xD1b>I^`p}Jg
z^JeYXJ7wPwn==Upwijnl+0)@<urH{_-{!`~?>2kXX4$L=H{4@Ysk67>^QyhRMg03-
zt!v-QP<Lr({H!y3FN<pJQCz)ruh!k_z1!Q~*jg-PvYWAPgH6i3LR;=L5B40)im*L&
zLuMaC(MsDY-kiO=qo(hDXa3$gGS6Wz$Kkm)7FS*OmfYamS8?2E@7@&*wj0vDY@a;l
z*xgaK%R0s5f{pkl&VAVu-FtZxzwKq|Shcr6b>g1Y;p_I6ujkv#pgL(!&7TW<GVW}%
z(Yd$KW^2n7+c_<t_B^|yWOIXk(cbDyEA|H5mfUO57HK<Y`DEL|mHYM{`T2g2=Cwp?
z%lcT`<Og=PinmR7`yD&CXWd@Uy#nVJ>`8vDvUkm8HE^3OyJgm1Z3$JojVo8~WwPCB
zE7x_xc4x1qjn`+(y|1FS*(x0J-Ls@<=H9o=)Ax4o|GSsR-Ng1ow$k2?dAxgTY_x0x
zKEB<(<g>7?hVvv_*UkKPCp+Kot#3}Uoy8lq_m<qWy#b1Q_ile7w{N~Z>%OY4<h?6)
ztM56JK4p)G*iKu)o5%N_IKXBTDn4g#dsY2jjkY+OHr1K7Qn%;q4QA!tckaxxJ&Y%J
z?=3zWvp2ie(^lb8w(ZZf#JzcwY;1a@+V}9MdhXr9WN8zyp38Rb9M8R0t|5Cxu5GhT
zf5W*~#f*KQg@wZ2xYee6tEMXK6KUSMS3-=zwy}@PR$-Td^%lM)+sciT_C}a)-?PuV
z-}X)Esy#C->TS(Dm)bsW-o96D(Mem+z;m|KRyFL6EZ)9%!Tbi>-;Z|fZFJjb^KDbx
z-r7~i_8x0C+N;C1Z*Tt-<-H}vvbG<KqxT+h`MSI7n!MeXyX>|}bBb*8<M;2qa+1Mr
ziRon9r!!0U+RYHy_nDb*Zw$w6TcwwiY}ToC+unK9VRPAKi*2KUmF=Wsj(aC({n;xO
zd)_u+nu={gP{r;eZ@%tzF>$kPP_EsRD7R~`-_JJNSqD$;QFwI2CO$RB=E@qQJ<Sud
z_T0+s-dk}=b5ExF7275c37ZqIOKlQ;@9ep>w$yfy?wdUssgL*k$a%Z>WsmAU4FiR}
z6VAumEL<V9cR_UTuB1h~Yzpe$Sbycy+1t9G*_yLTVBhOyjCMi-6ZWVr72JL0&`F!9
zJGb^a%(mVmThn2i!0vDJ=DUGSO{mM>FNO^^v;NfC=x>~4TXZDaR!njBUXBe*Z8?%E
zY=d1t?S55au{Z2C+g{xnfqOf6`fXY_E#GUAw`*^X?RDD#eqmdENddcAQ}^#xp7P$-
zB}Q`Z(#p%bb+*5>DcM@PSA4nQ-Wtunwh#V_?7rG`*w#(xw{7j6+k4`+?Ad)meYH*A
ziLyO2j0J4Z-u2l#U7c;;ORrNl*SS^pN;hA$y;VASuVq%sUh`)Wd$!GBvr*`3wDrEO
zYFG1)$xcCgo{i{{6Sk8|rR}mWl-nMNxMDTgMQP8TFlk$p=iByv`5?Eis+qy|T+bBS
zFXhIz*H3fW$uY)SZ!EvFcW>SU8|nTe+kIK<_a1q{wlB+X-`@F`qis9P_S&AhAF|gt
zJag|8H#b{zhvGdlU$@wbxtUrOlrY;KW4^Y#BJ`<k{FAP|%83bk@3AhieXrWKH-$6W
z_M-UnJrB~3?R}@Ve(#RhUAC`R|F(_0w9mTjw9dZcQw;W0FmUbr?DEG(;M@)y{i!qe
zEI8e1EAq9>_Rht5d$cV0_ZIPP*>ik<^4_}1y|!HortCfN_=xqs#|pLwx7qIPy<TO@
zAwFv_$Av<hHq-XKRaWYDLL7~@Cy&VNi(k*PcNX7+J(geMc0W2UzW149qV3M4S$kIm
zvfJ_=oMLPDPH?Zr?c6;+Gbio2X5O@SF5l`s+RGQ&3Z!J(P6_PUdn{IL-{kbedtc?~
z+QxEUu{rc>qHW7vb=y1D)AxL9E!wM4+_Xo~rEu@h^a`5~^S$=Ovk2HRFWzO_aJ<V(
zMx$X*h@#uxG}#q<8Nz&b-#w&kSO4vWZPGvHeIA{UtY>M=+v~Mabl=<p(Y+2WnY$Hc
z_SpVVe7$Ff`+8fIix#$8nM?LYJgl&d5a+VxD_CR8(|mbP&xdI?9+DM%5BPQM{^QbZ
zd*s0a+t<@Q_fGktZ2MuNpbcB$Y1<$9FKnV7tlnF_(P{5ACds{DZW-Ga{9R<dv88&i
z_DYq#es0tEhTL3YyQAinjap;jUZy3d_j366+cbPh+^aQbkFCc;ciU2-b9*;jXx!s=
zaGC9o@U6BC3U1aM_fl<?e%tJQGyU*hKhCDTw<=EU^(dIWchABjdkUts>|OO%VJ}Pi
zj=fG(&hFjthj$NK=v><+s(Q8)?mV>#cv)k^FrmfDXXzxHHy<DEiHVK16_H)Fr=^|2
z_RI7z+xSgu_9~n-w9N^qusxo%W$%p=ZCi`9eYTHk7VIs#CTlBW&u5#Z+h^<heB0g@
z{v~_PC2rWeC%Sg;n@{)mRPmm%&3G_<uS(8ATa{}Q_cmmeSzS<bwfXXRwe45y&3p3>
z@7-Hrw{lO&WAVK!o@Uuh>0`7NlWwr>NIhV)E^UWx#bIIFDPQ~c9PpfIW%YliEsL~@
z?LLF;whDPgdm6Gi_ug1#xz|W$kL@wrm9_@5&vyGqE!``Y=CJo%Z|B}OcUJA)le)q-
z^-$d22XpV)%$O#)mm_`Ko(pWhZQ}G+*&3dT*gGXIWUqzKIvay^bFA%TR@w%rKHMXB
zrGIb2@hP?@d@F2st?9SDvTpyLdokU%vleyiT@$;|=GpPRwhA4~tvlH7*@TI&u-*4?
z=H8H_-}gK_BC|KGT50bB^^<#-D6ikUBJYchZK~7WB}O}KSmy4v72un1-8OgY-ju#-
z+exld_fB}%y(j4WQQIe>?0cvDSzxOWRIvBKLJ=DW^;5Ppa=zO9@KfD;`Tp6xAMAfy
zbE$8$bvS;&CV|(%c7ZAP9+!+4dlE!vS@T@#+#_&Ye^1Mr<+cSIp6xj`({}HInG3Bu
zqTOv0rR(?lg&nb7$*^?q4Lv`b1@$Lw9p-+r`tWp%?a7S>d%u62z1J(UcyCF{0^111
zr8YdaCbk9zv#mA4<!!c<HrVc47PB{?bFcNBhbp#vRQB7x*j%yqK*g~=4eEt9QBEv-
zUEa*I{XFsH-s=*Z_CAeVv6tb)nH>yqJ9cZ#Txla9xnxfh5C7h{(+6yY4oB{FT{3^~
zv^AW27x8bf&E4#|cgn4B+Y?7t@2%kLwQcs>wD*&IgYA_6^Y)Z+_St^8-m&+{XGhx^
zzszkDvZwFaz)-x$AgazLu5ZHL2YVZ=A4Id->`-66NAJWITLFD$+dVIRY}LdU+A;*4
zwQe}N&bFbCXU~-@llRyxoMvl0H+t`f)-N^!Th>||1l8MC7+>16Z}EBCjt$IvH+W9i
z!?2BYPf7a;+n$?JwqlQ#@BOjZ##X|X*Y?bz&wEaY$nTYXHFqzQ&T3nMUtg_diZ||=
z_@rSkPobD?hs9=_wVC_&-p~=SU10BHE8)L?Pr<4Fy#-~5_AXhpcF&Km0k%Dhx7r5i
zy|;1Lwa50?f!@7$K6>u`_TP8!B2VGH7IHCrWVzPw?R(_0x9A?nULDUe+fQ2>_Re|x
zYfngKvu%mQZrc@WKJHQ3`)kiWpR=}Z{tdQor|R#$@_WjjODT(NWen%pDrlzfne+ef
z9)<f3);wV|Y(Hr9+Wxv;xc6^E|K4+(n)kLHblH3D+a6o4$+CNQ$j-F=6}`+hrT6IG
z=i6uRUGVCdZT^MLdwW)X+rx5hpY2SohP`(d?X=O-Ut-H^IA<?M$r9TK{VQ#@<gnS+
zXr9^QaAvmEi$@iEjx0O8$M44`+Xy!|n;8$YZM)7Nus!kJa*vJ7pFI}?*6-y#Hfe7}
zT%qle`P25axC_~~7>4c*`Ek_t2GcH^DNzl36V~O}C>YCHXNauWQ?O5D&x2XZ_TKpB
zy(f<+dGCqUg?kEqm)ai4DcsF5;lQ3h5;e9f6Xxy}P)pyd@anIPVyV&I57W2qT@jMM
zSL{*cUa74c_Wn4tf3HHv5!(aa*0v00DposETx?Gq*0$~Ywt26_jTT!4Q!d*DliylT
zxW=*Pj>UA_YnOud2HdFJyD8(e?Y#bO+aA?c+aJH2ZBMV7y4T?Of<1d?9=4S!Z?ILl
zonX5mc!KS_t0sHz>}9m=(%QFIDP+BE!ZlgzJH5?&4H^Y(J%bMH<xu0>b6e@i-h7eZ
zy*^(ZZI9eNvqxp~cAE${SDSA#n{0ncthZ&TnPN3#(li@J<|%t)j-0UFFhki^Bk$6l
z4GEj~9N>Sj$3$_CZO5|XwrRrrdlRnm?9KYNdoRm-1=}m1x7qHQ?qhpq&4s<W&vW-S
zomy_I(6Dt+*t#Wqd9HQs*^zm4PeZJ;t<^Qgy*ciTw!RZi?mZwWV;j=3(3W+v=iZiw
zVzxYNv9@2_3T$6I2;UpPnqZre+q>t(i#VI4iOcqKG)}a==-Ih9<v5FN!tr~17%o1y
z?kf@8yI_y7jnwxkwhXyHZ5ZBM*qz{i$);dljcrWC0o#fx!h7f4Yv0@Oal;;lfGyUa
zE;#IcvH79Ro<29*h`IZ14lHrBJ+{4Q@6WKpy%DzJwm<qpZ7w{TYWw0t$KF%ZPTO`I
z=&?Ov8e?<6NoTLu>vOgmr5d)!^5ypit!}bSuuZfM)84&z!WpH#Qvx$>XNct5_S`D6
zT@@a>_fd@d-T>dAy%RJmY+URV_uf%|x982*owfzb&+oQ-JKMHlmh+w^lV|SPBHwJI
zaaVayiJ_$J%0nmiDlOP+Td{h-O-5p_O>>3r-U}WdtPe;B+1z?D-Bu^2ZSSrp$M-%^
ze6jn)uT6U*3V8Rp{5ie%=ahuK8(y5+{n+ck-iD(6HVPFMw)eiDvVD@%vo~ha+dXW|
zLi<c*e(Z8zY`)J;O!xrj2D`l_YAg40w@uokE+1ys+H-Ml&)TNF!ao@Id9I$j=l<*E
z`_9Qv+$-_?@ot_t?tN)uuD0f1#r92T=dk5IWV-i#0M~w@Bc^-9-iz*6SZi-z!`!fE
z*3!u~N*yBmrFJOnGdQemGyVD*8+{gsecro@?dHytvpKx=soi>=AlqYWn)h9GnZ0+?
z6@`5~e_!w2q87O?;LSCg#ee77hIYy9)p>Y+UonU1elx-Kd*__nvd{6^p}kLU{Mr{a
zH+7%&^6h&WPqytbVLD(Vo;H7<R9NRe%WD<3SAQ$+b-r=mu9ZW4pZQ6jz26Py?K56H
z#nvJ1sEuLnwY@hki63aRY2TOgR(4-<z%hF^qfIs+gc$cNi&?SP<camZX#eMXOPRLW
zEED0d`!dCBFZaS08?*jt``(_Iw&%=~LwomRneY2nFny2Y)Tn)|6JG6|d1dY12Q3r!
zEmv0EYqyDK@6E-n`&R4h-}_+wq<!~h@7lNfZ`Ho_3iJ0~^v&PLda7aXefD$vcqIAk
zlbTs<&i#I1^WN;pUaJ^B+Xaz<`}^<q?z4_n-e;Pxw(sP&<UJOKjr%s8TeSCK_f0E5
z-bMTTR(0$%{qb?HCEvPz)4y%mTlV>)-DVDc`>&dPb|>;;_D;KUcE5z*mAyAY4p=SM
zf3u(GS?^xAC5P<V8FKda8@t;jYw+w3=lHw#v1IUG&-t_VRdpZQC%8XupYrv6d-o*9
z?_H5qzgM90iEXK@&VIM2s{1-(9qm$>AKJujnzY}sLTm5M6`%GV+a<Jb`=Q=FdYMu-
zp3j_Znf9L9dx;};uT<^gy*qo$_S$c0wda_0aPQ{tE_+r=N7(<Iad}S?=f%C+_fz&w
znBu-aWlhoEXNvuMKV<msVb)JSur6L@@AgA$_St$J+&%Zn;(fn=*w}vfr@cSMPig-v
z=i__7tvtE!nmhZxZqE39?=Cj)Rk&ccck!!()_L#F+p69>y?4)g?tN#I1#DLwUbj~#
z-fQnF)3<w54;<Nh;nRh^Zf1M;6>MzWd&I<M_i?@8eb;x%?@e-S*ndWVYu~EB+I!PK
z@z@#_E!)SqsBdq{{yTfW#~B=Gj6J>YV3M1iFJtLm+1assk3QIHGwp%uz7vV;_PZXh
z*}LuQ?>#H)@9ljjq-49ry54q&>&d-Uhb#6cH^|rth0NM}Y{ClrKWTTYEk*Y4+Y{8h
z?-OU!{$le(wjAQ0_G%qJx>t--c;B@=|9yH48*Db;*WP>0t!q!g$s2p`tvbFhlgW0k
zPlMq8CR4?|E~Yp3&P|%W&%jJ(-wuJLHdb4A?NfLdw2x^X%Yhxog!Ve<3t1~G>+kIn
z-Lcm!zIX4BY*Aaj*Fk%wp5ETOGkei)l_+DovY4-X4!QI1WfwfSH^_Lx-VB~kwy)or
z?sKfYxwq_}*S@Brdi%(-<9quo=IlEyp|F4N`|kax1*Yts*v7SwQ)Rxjz=J<F-WvA%
zgev>?iso;!ak!;s%cv*5$0Po(O`(LUb*j-HYbNPUHtSdg_AO#*+Go`N%f`uQ*}fKr
zZrksZb@z3dH|^c!d&T;wM&rKMn#=ZS$E5FzkICB`s{PSA`jOzCkXEI=d=EqSSFf7C
zuPVHLuVh@K?Zm46d-GpD+iNmabk8znUWX^=GxmD;FzzpBlHBKAKHZK%VAft)>9hNm
zHJ;wLd&`Tx*KK6?KjAR7;dBYIU7NOdZ>FmI-cIN0y}Q(&@BYH^*QRgY#=RLP413>1
zc<$NJWNIh6a{Au(-TC{%d<^z}e5bfi@VBHL$Lf1~t9;$}@SgIpU(}Vk@6M?o`;<9W
z9SB{VXT`s@abMNtWi}7uW%nt5wXjVU&)e_gY-Jx>d}8nPOWX&p*8kk|U24ic#u6{v
z)8?l8&19?WbNp{u3#(Y|*G)9B&yqT|H*ZU@?TnzMwsl|B_MP2vW^Y?&!`=!R=KZak
z1o!Uzba+4S%&5Itg%kI__ix&pWVUDDW7WF7_u^gm`Xs#DtADv~U&lekeeE+2?z`C^
zZ0rAwX@C9l!o6E}rR}>XdB*ld3G=@BTc!6M>M=afeP7>ZRVv@Uorj!llh*a@^-b7d
z%XNFUoz0Bu1H7?+_t;!qU~5u%YhT~98MZ$oKH02T;(6fW?tgag?@!*lz`=jNiTItp
z7vqffTQ|?x`(VG|zUHjZef|^F4xC)Sb8pK(E4whW#{KH+X6y<1nQpJYkZr%f?TEb<
zf9CA_boG&qMbFiJ6Pwldf5~#QmAmJ=x1_~)-;pQttT}f??!PCbXZzs_!-3#;Hv6|U
zl<v>YbFo&?y0%v-eaYUaFW2`z2t2UIeaVr%-n^gpcGcJIwYV;7pSG{VcH*P5y=Ltb
z_ld<Vx7}6hVcS;mf3NbS9eWGSO!xUZOyB#pKx*HCV~_R=`+3{PB_H1-V|;tx2F3aN
zS6ephJF!4$pGR1@{Ub-teJl3NwpIInVXsisjlCg4ul6q9pt`SZqU_#n)28k<^}279
zuk>-h#rduKY_6Q%TPnQW>d$MNy=GC|_7B+p@3md^Wbdyj-}bI))wj7Cuye1^H^%)(
zEoSVkEbrLAZu`Bx3T`a>;<tX^`$o)duXx(Ly&dV>_cho{?c=>TYcKD+vwM~{a_w`_
zQnWpMi)mkF{_cGZMI5#}C2H-M6nShHA3DA_G$>?$4DUQU>*|WVylcPgZCI6Sy<*Pm
zy?N<wwz(HYZ4M-aSr@1*-CHO1XWuhRvwcTp<Zbg=H1`D_^|D*eTfFbAz!Tf;HIMAR
z&z!cm*I?29vdJ^{G6^K^{cQ1n?<^Ioy$cvv_O|3k?-h_=u=jJ}@x9l$yY}+POxU|l
zWZGVtFWI&sUsY@uMAz-kpIW%L<IGi?B~L``Qj*MV1vdHFGFZObwS9fn-iniHw)c8?
z_Z4JavN5XD->c#Bde64M=WTfo9JZaI$7XvbsBv$@Eh!rdgAaR_HO{m>v0|ys(|;>$
z(nMAE`8-}=vt|9Gy}YbDY|A-2_w;R_Y%3(fxYuTDzU`4Wr}r8aTkbu<r)=w?U|{oW
zW8~h8j#F&y_V?Hd-R!iTwX9)p#smqQ?rn8@6>OVrJ0?lmnrs%d(FxkU_ebj<+qJW&
z>`ids-q&7MvsdVotet?Pkqy_xv$i$Qt88bUf3UZx_M`1F-7Z_c06E(k@7~+25udc@
zjdiW<iTfEgOXQ#07%a2e!|*0?uLJWY+e6JW_SU7Xwhb{dus#25)*i!y+;(Mi_u7hG
z>e=fqC}XF#<DM;F`a0W|>bZLtC~8{Q|9ZA}3v;_I$F$db``Fg+&C6W6H(yhE->;Xa
zZ2NXu+a{RowGCj-w=KD-W?Ln`)z;(N%soA&m9_=9mRX<p^=l7<&q<pIz8+fzzX`VM
zO1AG^v~GvZtE(q$7u>(Id(Nf^TZM}Ly+5up+e$<}wBd?j+soq7zjxAH-@Oh7J9h6+
zI<eOw!@}B*W0S3l;PSnPZnW6imd)O4VLor~mRsv>t^PgV<K-Q<cT#P?t;6K4HXZB%
zwmO=7_WVg$y62vX>fX3>{QLNAL#^+G{o7;kj?2ckdHdd&J$-w=2KMfK;i+RIptfso
zQ|D=0=g`8v>&3S1y<GKfub+$X-lF)Odt>(1@9i+Iv8^y`-Xqi2yZ1=3z3l;?e>Qbb
znD!k=_^|gt<<mW1G$+_vuqN+mU=Ojr*v4#Emae*YfzNiEJ3CbN9=qAK_oL&Ey*GX=
zw-H#zzIR7;hi%`9_PrU`KkwOa{ORrq|4!}Ubbhi|p?aE)mBt#|r74s5ez+!N8~5$R
z-WOS^wzvFT_dfr#d~fw~r#(@Yhi%vM{kB!V#<tI=<k+4aPnc|OI6tvj@{o0J&bP&T
zm${zV>%kXiTYhfB-jqf6_nZg{wY_-kwe8xYbN6myZQ5%U7`}IQ39sz|vzInA=4`a>
zou_Lv;Y80KhNN{?+V5F+GvtTvEn0nYcfh8^y-fQr+aBn#w&7VZdr!^E1GZnZg=`|9
z9NjAvwq@_>Hw*V3Iell3M@pw{K*s{xX+Iv>C<M>7ono@rcJ=2swnslR?OE`t!B%WR
z&)#gy(|c`hneX)wb+gThoo~}{`nS!8M=SPz`Q^X&UWxnO3-=G(OqdpLvt>=b?E@Xj
zT^ir&ZEvmEZJTkB$M#u%(cV*qzI&7Q^zIGlZm?Oe(Ybfpkq+BKpSRe?bR^p{xGdSV
zVDB1RpP7udli6eUS`-)B2DE9~wEX3=wYE{+`~J%{+c$|z_9zIZ?{T}a!S=?2qCEzk
z+xDg;H`+FZZ?zRt7v1OWzJG6q_&1v#tJOAZvX||-w#j_&1SOWe%bbmOH*DqGTcG)O
zH&5hQ+rw+k_Wo(<wN>%d*n7oT)0Xv&<K6}5rS`tib+?UK&~2MG$!G8J{d4v@++1b5
zZfDTm8{T{NR;hQ`{#wVo_sY+$Hd2cZ?v?Rp+j}AQ*`A4K*6)pH`fK}VX0GkC@?EwU
zmKyEdCwySf1jiM7Up$^<TM@@#b0Ou^UT+N<J7KX7oBaE#d#5ax+V{w)+4exSwDqP7
zp0-u2r)?b`P1!B<yxBJ5Rp6dBxe0q0I4bX!eY|$>A(3r+b#A@dbC+4luA*zsUIF{(
zyPX*??1{YfWzQ+o23rYlbKArDk$Zm>@$cd5Q{0n(+`{_H#%(qSc7L^c<G{N2SKHFP
z-zGHMrrwyjCpUA-UYEPwHcKsa_P*I2ZTsQof9tk(*S#HDj(cAOZninM?2^^Fq8+wp
zZgAUf`_N}wGHc%+1w9ShCmZB#qq7g&{Qu)^JL^)Z?SW!Wn-8T{d(M5EV|!!A+`a3H
zCG8~T!uDQRr?L0Jvr{&cHt)1ms6J_PVf!7M9)(@DrBmAW@;S`3Epu+MUDZFyRzmFJ
zo>NWTwja2++g?eowdH!*y|=+qagSZmqP+$Glx?3V?YA|`*lGJ!bNgO_IZAsjWHs$&
z_ix%;wS1<nLS&9@L)j*q4HvC!9P209D%^dtTkheEy*Yct_wL*&wda@11=}4xZ8m3=
z(``3&9k;GK*<d?CC(1@)iG=OH8Ha6UpLOi@&bht!@`bv+0hOzE?`Yn-_d$23%>xT&
zyC0`F?KNOqxkqQ;M%%xCwfD-cXtZUR=(+2p+U~tv$Di%JwQbAZim5yH2G0}Uce!4D
zZ^EJnw%al}_U<T5vvKgbWOL}~VcS1FtM+#6`?P0XX8+zvx4ZWWuT-}cYCmfmA=F|k
z_iC~244*GH;ZjO_pYT=MzDsS|%WJ%P@ASs}y&h+3Y){B*?3vftYP%ruy!CcJ)xGc3
zcG|}N@7;4?=L72xx?;9pET-<=F_+Kw(ds#SPfR^+>s}SHS3`1vZPZ5Yy*Iwx+0(P?
z@*ZxNMSGis_U_T^a@pH-eZt-|`DbkzST5|cF4|{nJ2!l<$3ua=Tl^2%_5^k8y%5`D
z^E1lF_Jw+%?Y3FxY?(DL*-m7ew#Vhg#626HZQ6SwV7~Ro8#a3<TtB&6exlXh@Ba$-
zYPkCCwfG@p`=`3wcHy3;y$jMLZ1ta=->cvwVY7^X!rl!{)_V`gO546WK7a2H*8{d~
z(>wN7J?^npJ+N-io!Xr?f_xEsW9zo<^(&gW_r)=1n==s)_mucgvz`8?Yj4R*7uyND
zs%$4%KHIHVBWyS4otSOF+pT*IJ{{a+!yvV9WBsJP-%iipyC$34)+%hKElbzgz26c~
z+gAKLWW!)*zW2ne(|eiSQ})i7!?LHWUv8g`>t5T8yBqhM`1ILEpzVmYguy(US^u`$
z{yVV2mV?dC_JO;%b&5CB-aAuy?H({U**G+;-hHT~cdtyFm|a=ICY#dl2lsk#H0(JT
zC}Niyd1X&x-K4#R3R`Vwu=?$plaXVarMcYJ;&{33jg1X9CpP8n<vby8XV5sqdXv{Q
z+XZ*?Y+V9Y>``M?+k2qzxOKPB;=Qe07j0P$=j>h8wa)g0hti%8d!+2<IA7i4EcVYf
z{>7xdX;*I8MuZ-+J&^Qt?;4-Ido3oi?QuyI-y^jBoNdLQ4BI4K-n~-6C-<s5w%Irs
z9o=JbCT7o!max4lL5;Qn?o(~1SgP6jJASm$x}3b%O+;g_ndI8NH~(|*+jCdgjwxn^
zZH-92ZJy~0+oHw%`~1YG?A7smzK6k^fA6Jz(`{q+tk@g1YN72vDaJk5<{Y*y;c&N|
zlX7_PyO)P-FZ-R|`)K>Vy&X0SZ3VXY?%DH8c%Q<Yy|$m$>+abl&|vHKRcQ}Lg__L~
z;q<+tEJyY}u=r=afn%cWALqus9$`;x9&obl*>QK{-jIWKwgzlrHW$(!*%<DfWUH~r
z#?~xQZSRgPS8O&EXxf-<p1(I<dZ&$9?mk<;Tl;J|n$q`t_}#SUz;6j#fn~Y2=kFcd
z`>gYb%`<<ceGIn`+q_vb+gAVWzP-xp3T+$&l&qD#+4glmYTR>y<E-u7z#V&cznZ_7
zMfHL0DVeRdH=Md{&g^<`bL3FAZ2-@c-9<C5?mZ(MZ2N4Y%wDU=URy2Sp1mIN?6wy2
zkM?x%EV6Z$ny|Mk_}-p3(!qNp>I7{pCM>WqeAm49!JO4LGxewM%{_T^Z&T0oJ*_{=
z_XbaPwyqMMVw*lC#CCyG%HBCqJ8hRvTeX)(WU}o6`_FssBsA{*5O&4J=M%r}Wh2&o
zrW+3JeX+~QMudrDpJdRKy|=X2?Vb0`WN%N#s=ZRdhi&bWw%Ar!Z`e~~x^M3V>pq)?
zYWqD57I$|m9yqb5gV*1dv3SwmDeo5TF%T5Ad-QejUV|gTdtZ0(?tUtHbZ<XLoUQv3
zS=*^T#Wpt{+_4o&(zE@zVyf->nzMVE&HDGoew)0v&mnD(PLi?h27OiAKPLnBKIBQ?
z%dmNY&62yDZS~mC?<u+RX3xI0b8IKxId04AacS?pl=HSHKWXnhao5>q(Y@Ba9J_68
zSFBsJr%_sU@5}=Mdl?fq?A=zd*H-Re$X-(uoxO98uiImtqqisGo9x~P$Nue!X|S@D
zdRw#C;=&J`bdEW;cP8uaO_q3MGcSvIuSzwC?VStr_p)Ve*}Ek@(l*Li!LHmn*@j_<
zscqhTzP(FbO7^Cu%iFBo{=~+{Wa{1-wJx?C%!h2Z=r--0>b-BTm*&j97h*+i?@H|0
zd!_f-9){DOZ8m(Bve_f}ZI7JiZrhyvleXfsgKbwV*}B(b`{BL3QIUIJC>+`wl(Bv9
zxw<oZ53Jo~b5vvDUX!?F+Z8?`dk!AYwb7a4XUoYXXuBX|ign6K9~;h9Vs<~?WZ9ls
zY_yjvexmK&n_u^aYp>Y*;o4!Ft~JVgJ<N6Xd`OsK6HwE+XU|My+kltbZ4W)1Zp-rd
z?4IRG$+lXZa{K(>_U=gw4&Srq%|Tn4NaMXzUrw@Jk=SArwCt5lgQ4{vZg*!}7maPU
z5xd=O%brc$Yj7pTmi6($y%w>HZ31>q+TEKfxYw-y_TEjLvuxR3<XJCQS!Q#wUC6HU
zc+cLH%^UVc=;Yg4J{R0K;e*(|l*RjOg*Z>x8q^%KWysmEw_)OKo6;EWeFa79_U@Wk
zyO$?xt?du@d3z_n+_yLG^YpzMJC5$@77^SV^Rjub>&_px3vNc+T5dGh%kZqyaz~!c
zo-b<cdjnRg*-n%HZhIx}hpk6o{T`K)6?=lE_S=-s*}JzlTW+sF)nS`gVr{!s(jsgV
z-cPm7nzP8Zbm79icGs%*E~wzJZk7MI$3%49UXdS(wmcpMwqc>0ZST}H?cs2UwmHEc
zYtvx!+D3bl!rtc&oOY|+!uN9V*X-R~9ke%X$28k_f?M|XNG!IkeK}+AoqyN%+_<~Z
z_Swgidq2wTwAJI;ZhJ0emaUTw%ij5t+%`Frj_hUlXSLhmXO(qA=hHn4)9Us{Xl2`k
zJ(_R(L$rF&ruL4#+Ur-^c*d;STd{DT?J|Yidynua*+#8DV!P(lfjw{LO||*6r^|*%
zYyF<JmyYcf*_UOjv8#J;wJ7htSq)9LFJoG5rZUdi`-4Bu=6~&3+mbRiTRG*`wj!$i
zdu)x5?@hWo$(F-KVDE<ToqHC{bg=CXzPXpzyk+mc#nboB{U*DYV=2?#^J@<8T{ml=
z?T`DGdzJS#?|rjCWN(GF%$^>;uX|kmFYKK@<NqFqgLO6n#*%xp_Y2r%+>f`tnx?b&
z+Q%if7vA5t>3I9V=5Bq!UW2`FZBBX0?mgv_xA)EVZF@S*_U(D2o^Sh3Y~7w$>3y~@
zZe`fA+-0<_XbG|XJvqgO{eAS_hOYs;Vh+pNHYhIHEAmmo*3Iy)txf-#y&9{Q?afV)
zv91?p*!QLBl<hz7skW*1)q73S#B3Xm^jh`!w(q?#ZTTLaRYrSH^z7No<g$5h0QcOz
zEPnI%G;Gz~CBgW}=DO$hy&`Np`-Jw$?u`?V-Ft1)^1b!WZF?8|lCX6u{;=mkP5JK6
z-)HSj?3`#*XWOuMQNTKz)7w04C)GFECVcR-Uhzl7_Sdg|TOXNydz^0_v;E}0&31wM
zqdn>Ev-aMdExK=dvc|qWU$^fOT`+UsgtNEyZ)j}a8$b8Jo(he^ee&Da+8@x$u-$W=
zXP;7O(w<m}4YnWG+_wooBC_wXzPgQiSIxfO%NqM82<PufQ4+J2o9Mpx-H}84QdVuV
z$xu0AXLm@$zFIHBW=g^Aed$Z~?*FB@Y_9{?v)%g~j@ciM*V-p>;n1Gs+TDBeKk)1i
zu#~oc_#<Xt*Vm1EK1V9-W4pV3Pl(doeU{1%dw<<i+_z1ici%A+?Y)2V;_M&!^6V4S
ze6sI4chA1X%RcQpFEV3ar0w0kr=z6zEv+xK{qLuJV1@PPy;_HN?{^F6-J3P@&OW|x
z5&N``{j(8Vo3Sr=pXOdpk2QM_E#bGjvF?mLm*ASczjcr8-T!I&{{N@0@6G*mWRLE}
zeS6{~Z1zcb&Dv*?%)9UIuI7E)C%m>%`7XTg=82VirTDw|8n*x2Q?k+Aws`%Uy+#L?
z?2~@FcHfMqzk91w)b{<H`(m%|C9b_3x$pO>T{^M%ppNGL-YUiY+&iZ1%g<=ryYS4E
zeQN2a_M4Qg-*@HNlKqExkL-P6eQmen8o7NUN`LpJSPI)8Jnz1bBjVt`6=sWVc<M#=
z@d*3c)rBzH_WoenXUBeVpNZsk+nV(*dnav)-{<gK)JD3zWlzzq$o(_krtUYm@^kOv
zd{6tm5A|$!yo%UcU%AeHit*XK)Arcx4apYY8}59}?u^dGeI;SCw(shC_a41cy<hq1
z&VAb=i}s25uH1Jr`Qe@e+P%BXUbODr(=%cJ>#3zSjjgKt+0@zgw@QB7S9NCj-bn6q
zd-vNl?-h(vurHc?d)NG~342*CAKCjeUuDlhIflLe1yuHNmF}=J=AW`x|FVw_=UwOh
z54uk7HQ30$?-GmlzP%T!_Am4F*t7B?=RQf^2YVMbt=_lzR@(lzmFxD^f2iLZX|B5O
zTtTpHWS*RTQu@MukK}Cj-}G6$@79vIeTh|P_fB0gZQs?!X?qTZ9N69dug`vq^4q=b
zMWL3Qx`*~2)mpdr?`O@uJDpW+AAD%rThcLO@1#XL_c3x-?9+SGv{$!~fB(-_?E6AB
zl<k{BIc-m0cDGsj;rrely;}RDrf%QMpfq7`BIgPFKSm2}TeKqgyKTH;>t?IG&)r*c
zx5FXpeg9s{?LXzY$Hw-8!rqdz9Q&?qxML;4UAFJ`@#A}=CN8sovm(Ntq2bfsN%lSV
zDj#m|U6$9kPn*ZYuB5feUNoL(-`*D=_qx74v{%-uac}C`?!9p;#ddm|n)k{~VcZ|J
z{PA9ae5-xlCYx+}(x2H+SfI2|UF_JtJ&#lE*L>~X`{0iBzV;j2_uZT|bD!gn5BsP8
zpJL-6&#>>IhUh-0hcovc@t(Q&_)^(@DcW^=PA9bOU8&n)^KH>V`>oBrd!MN{?ft-Z
z(w5oi;od`M_V3YcQrY*-#AEL%tH*o1tr#4xCuZ-<bez9WV3&gJvbj_3gl(nvbv=sQ
z%Qa=O&6dxxd-=bAv;A9Fy*I*!Yu^fOJ=<GRclX%vitW8%yl?M?b=UWlxX;`3e3|S%
zd(Y>4_u1Xu^C7o(->$3W`_8P-v@MBm-uvk5*}d-%OWSGat=V^VmH)o7(~`D%n^)|;
zahzlSvL*BPMqX6hpP3ZC@9XR4y?&qc_sbV**s5l8@0-*3bB|)=40}CkuKf{9=IopL
ze&XJH?|Jsl(JQm=6_VWdQ0~~i7h)4_+r-c8+H>>59<ECTdwDnW?pxJ5+g7~o@t%N-
z>iagy)a~t?bbYTq|D?T^Dam_}UVFJ$>)T_iTPK$9GhfKP_p=<|zA39#?@etC+#C98
zqwOxC%za1K-M9VY?`3y&af7{w<@UX^ek|R4#pT-Go!`RuT|N3`56fXQdl^I1{RbwT
z-g~{b(>}YBWnX)z+rH4<wzjFYx%&ikHS8DFd)bI(>D%w!GtK^O-Qm4I=85gIJD|G1
zi80RhlY)}fcJX(6^{+77x7aZ3m0#tyr)>Y<z5R~XdsVqxZ9Ymg>^pNRe*bIf#=Shc
zJNFh$UAs^G=gqy9Gd=d*<yW-}-tyM2oQ>1Y*KzK?4|Rv^dpBR*YqXPVZ<XXs`|Bn>
zd%rc!*!%B)i;elhm3!Jt4_UX(DcWx+w9Wo%-VEF7Ra0y>HM8zN5q{j(_<Q-@!?Rb~
zofek0HUHpY6Pf8?Co-peA1BWwJBD@VY;sRr+1nQ1v@dAEWP6t#SFOZV4D1s=FW7(H
zS8o6EjPLs*Qy18+u8iJW($>Du@cK5}3sX4lrwJ(TUsGJMXLa?Xy$3!S+OImtw{O$K
zS^M+?XYBc~&B;zRw!v<?h?5;Zqujou8~FA;5IeH><)M__kA79}UHwXU-&(gH`}A98
z?^PA&J}_a*!oAYXllSV_vFw-em~G2xR%BbuQ@yu7Y^}`!!`s$&m)UkJr#9HmTd>kz
zREK$gL`L?W)p9BJ*S)RmBNf{BE%cgW`|kGU{f?YW`!1(D+6FA^-o0$qzr8iJGp(D}
zwC$@~y=U+3MSpEX5B2Uj=^$k9KPhD257DH3ch++5dnDRud;WUte$9e;_8#vQ_s#3s
zuy=BW{N8)}WbE^j_wKKH&$MsZ^P~GUj>_!2tR!x`&uEj4`sK5G!)w!R-~HaOucuwo
z_DJH2JyU~k?)l)hbkA9#OMCBCyX?IfK4<^N%q4q;dASeFOv|)#mE5`a`?@x}%}3ex
zecIKsZ`yZ_eXpKO+qZu1C7aIatyUtJ<o7)+pR~7QLeQRj%^&SfdhXbty_apDqR_Uz
zv9E90XC0lqZ$j*gy<Soi?Isj;?R~t|Wgpjl>3yKN{_j_J?sd2Ry7z`Q(_WQ4t$nAw
zChUDOLEHB6r#pM!UNYUYcT()WTm9?@de6_;TOizEdvglQz8MOP`*|<3@8vB}+IK!_
z>)yN8-}Z(a>e}~#VfNmFD<w9!>;3krI|%J%h<CAFbHv<s(}MtOA?}6y1P|-)i;w$g
z*R*_%ec|DKdo`9Vx9^)`XXhod$o5hh$36#N`~3}<8}}*1t*|Zld1B8~jqA3SC43Ie
zk^%eXr9a&JN92!nS1jMYAB&Us*|9ydx!UKocV$cR-diyf_TRjbZr_*HxBE+T#Qr2{
zxxEWi|Lt4+eEWVi^-p`Hj;!1B=X>|wjwz??Z#nDlKlRqpK2t$tU&NE$d*i(P_wgC|
z?_tT)+V`h%!`==?Ra?7shJAeZ_8thB+rLlKLe|zU=<@z0=lS+6&U?ML_pX`ktzS#`
zRxF6$xAx8*E3>e3_8Eeqd$txv*nH$(wa<KZ{@!DI*IVl-PTPCr`OQ7%s`>kt7z^0U
zd9%t!d&}&7Ss^xij!iqV_ukz^JDV>_d-l6Z?tgnz-cCt6cVFZh7uz6{_I)9fSM7h?
zwQ*lj^^*M=ZBO^y7uMbL{^%c@Df?dT%Q|RbzyI*QeVh`tdo>#4?5E1=+va5U>~jb(
zwVt79y!VKdv7O=WXZF!Q81}_kcI=CZnz47<v>W@ZRvp+sfy;9Ld)+kac;C$Z5mUD8
zt;=NH`_=Y>z3D%jz4;f8?-%{wXzRI4-#+J##Qv=sZMJ7(WA=RPkG3=FIAni0^T58_
z2YL3{=&SCXwD`5nij{3PE2n=s@br=W-h*AI?GNYZ+bSz5?YGolY8yL2aBqgm8k^O(
zd-lJQKW953Y2BU~8`$?w-Nm(UrVi8IJz8zHkMuL`S8NI1m;X*`@1Fk2_E*@}?R9y=
zVO!JBY~L66V4rlp>)xd`m-bBPpJ@An>+}I0KOy^X#uN6rIc&H6wUl$;O%BkW^BwzN
zhws>DS9;WLkMp;ESGLFQUEXzJ?;VY{eQ}W+_D*1v-}fup(k^9J?q0d(lY7_fy|d@n
zJ?4F^PrLVcoLIQG@tpr&AJ#oKk7~E?ySbu#pRAXyZPdXGJ87P8R-EzH`&Wjq+k23g
z!!~tls?Bu<hyAQi8u#X=$?aW!@qv9Rqt?D^MnC(?3$A-b&x!7<7kadJO3F-Yk-z@?
z+VY?5wc8MEd+o!s-2t<g*t%?8v)|8rm+eLewF4R7-S-=xTfE1jvuB^*dj5TH+SB&k
zTA*QfgJHUTPFA<A+U%Hp5qi`1>Tf=_*W!-ezAJ9KZ7Y0RY&D%0?=_sAu($p+*M1%;
zt-S@BEC*!X9^T8&+qLiP>7oM;?l!i*w;u0(7wK}q>PX1G?apckWUS}ezOcD&BjMP;
zr}3wnt#e)P-U5**+o#&M_A>A(SVyhywQW;(+<#@((LG0ZKe9Q{cg*I&R0W&obKUmp
z_iOKyD1WlImnm^y?ddi9Po~=Jv%EK9-@4qL_QAm+_EL94_O;$s+t-kvWqaiS+rAgy
zI`%#&K55f2{m@>WhrPBkZ$Ix9cRIb7;aY-?H`_CtVt<W2k&@bbi>?Oklie_Vud2`F
zefM7O*()@?d2hfTNn5=(rG1K5*4Y2e`oA~lPv$;_uiy5}IS{;evgQJN+ah1v$Y0F%
zsk8j{UHT-xS8<z`Z4RsUUPieYdp|Exv^_fOz`j^Xq5bpY&+U6$tFq7BYx}<CDraoE
z67K8^TGg^I_VR{3pVMyaTVeQmZ)E(W-F`os_Z~cXX76TcfqjL>{(B<IjQ9TOFW&1X
zaKzU1)wjKe<Gc5HA6vY4!_k&~D{uYXJL_A{KK`jad;V=J->YA~WseWPtNpUAj{83J
zckjzJF5J&_F4k7(K-0budC9$Ap9J<z3_HA6s)T+2G*9(?A&;c@mj33oS6r}pkI5^Q
zea=n1`%?Jj_DN^)+Ka^hvbN@$V?Vd*<UU?GCENW^3-_+|`EDceS8MNP`~7<vWc;o7
z9`CZdIn!)!kdyX)_e2Yu81cTnqAfFQr^HIxTfE@kU)`a)@2A<qy^%eY`zl|ov*ET#
z-4}V|`rgZD{OnWJChtA?t#jWQ<Hh!z57yiC{MxYRMU~v%?bi?PUBKjIJ;lp+&)eyb
z_Lf}J-E-)&;=alEWF00KI`6%j_IeNFuGe<8ncdb;FUjrGTxEH{Y|WMZ?-_UOUAOJV
zK4nvfeHwudyLp2c_k83G+UMNcx%YSI{QdVIhV8C5`(&SOlVUHSHO2n<iA@J2z0K@G
z9PjS?^`m{CrCyKiO(rILgQo@u7GD+Kr!4Si?^91(d!~~1y{_#y_Qg%;*(?0*hyCq_
z&iyS0;`TLd6Zd-Qa_*CPwQujb-~6_`>o)H_kz}>|6nnwGY5~@L2^TH)e>i*D=EnU{
zn{N|D>`R#i_C8I^-pkt`xjSV0vOVj<W%pGdZ`^gWPj;V5{H*<T+t%)v+)=()+H>Xs
zyS}aaSHDf%mz5W?hb{iLy}Vw?-Y0v6_ca-`+s@d{xVNW9ai9Nbm3`T};%s+5IJS3_
zjl|xoZzJq(TkY7FHZ6MJWxj`d|L@n|TWa;n{%Xd<{fVAEdq0IY?)!Iz(?;*X;(fOE
zG7gtd9NfG7Q1|}Hwo>~hfB3fd?xxzkzc(J(d!DmvkDTr4eThLD`&gq^@12yVxbMg`
z%e{)Ti}!9ab=&)cFJ`~s9j?9K3q$QouSM^D$gq3QdY;$&R2X*Imp@h5wa3zUU;ET`
z`=+xU-s|kWV(*1Y?LBvHgzw#$ykhU}o~iq9rY^9{$#SzbIKsBiWUJnRX4Osm?)xYl
z;93^Ezbb#q-lq8d`<5x5+8Zu>a<Bf9D|@XCkL`)@-LrSUu%C_C?J2emvQKum=mhM$
z=E!CJYS-+2x2`GdZ*WrI6aIsBAIGbbePN1a_CFsA*p{&;?=v@*-N);~u~+C~z}|;%
zA6j^xd$7yZp?Tkl{PTO+dV2TV<dfU$w2*C|Z06j3^7k0_O{jC(rx3W-&g!Dx-nb(l
z?bnuW+i!Sfx;1~>nY~RD-`dsHY}jjfVe>u{brIYDz23G7p406b?o8kJ{6vSX$!z(3
z7p^JpJ0s*`n<m_{cftknJ?Fo2?$i3>yuan@&i(ex%lF*(NwnX1Aj>}Z8_WLgxi9Tv
z_G|2uwK=x8JkM-zxpb+$NxPK&MV2{swbK)<7l-88KM~2@d-qhyUazd$eG@8q_mzH?
zvQ@B-wl&zru(#7rbKkjhOKcJ~F4!p@Z`fOJYqc+CBfD)ww)@_H3?2JC=PujxzI3)d
zm)M^@Wy^K;evkaKS8_?z-VUKjd({~A_bG|;+IoEH+?QXevRBGn#da&#&V7!hzI&55
z$J<7^$nN{1b#3p36`S_DrGK;wX!x<Wmv!mhhYgGN^)Ip7H>-T!zN(FucFKn8whC|k
z_qJ(X*%NSXm0iRBx%+=;_UvnBt=YG8{>=R?M>g&K?=Zu*N2F%&H<OQhO?OG#e=b|K
zZ@&0-8~LKZeF1-8?B@E(wO_{U`aZ6t_xptQ_U*P#nq&Vu{L@}W3Gu!EqY~^^d+)XR
z>tkTAQMG*UJn!E9jeZOF@f>j7`!Zqrp7;H8?N%=cwQE!9*|&Dv&3(&kHt*S>d46xe
z#r-yUE9>q4o{QP{+QoFgbB^5p<*y&@^$!fU?H7pHoBuM|c8>C`{eQ2u?DY%e-K$l!
zZ*R<sroE;K*Y+&?;Jnu({K38y$8URIXlCw{xDmFWS2)G?P3-+W+85vK-?I0Z-QQVh
zdvhP{+&jal!N%n6F<XhBYpkDMP24y6z4_kc`3v?{{57!omF>4b>hyzsvtCW#_d-8%
z-|4uFeW$P8-y?UmWuMfcU3;F{JML^rK4iCZk-cr5pqp*ltG_neYj|yc-S)N)R$|__
zb?cNpod22kE!w|wpD*{Py%rC)?LAeuW-sr_Gka$QgxGIA`DCy3vS)i`lp^-@zmK+i
zDX@A^z0B`jx^o@(+c*gA*A7^?@9XNTdnyC=?e+BQv##;r-q+x{+%EY1w!OhO`1h@o
zZQ9$igK3X}htB@R+Z3!1wXEAWeF}rUK=Ut~bk)E6N`HCoP06a-w`}hU+o|Vg?XTW7
zWpDrK*nJ71JNMq0qq%S20r7oZZ5sPHc2C{=Z@bFA4YRWL`aV9g&t>+Cy<TB6_jgN8
z*mLdpo_&n}cka7#`}*Fyv*zsm*Ii}*a^Zx13l@m)uklK>J@nwKwfet?Jrdn3_C`%&
z-e-R0_g?*qs2xeMJNMf9c<p5jHMj5jow`S^X{v3V;;y}ikBQsPR@<@Ha<$4n*>xQI
zW>0V2H^H;g)+MOkHnaKIKEEvu`y0PI?YC8E-PaT?v5&*^fwhRn@jZU(qWgsO-tOJI
zcFJD63tsy)|8eZKSu?@5EN{u)tZ-`^(T~6Pem%x!_nP73-l~GOy?@f?@6(#SbYIJ^
z*EY=z3->nlNbQp-%-?f%xqw|_A;&%jk=ZsE9inW*cq{iUY3<*u_E%@$-<eDI85m68
z8?$(kZFz3v-ZP?Yd(Y?f+cxvh-v41&*#2W{cJI3^dvotG*QkAuyo2}p%;~Z{pmlU#
zLH+x^LG5k(rayVG+d5_S-k8Xyy%P?u*}G-A%igo61@_JTtY@1p&c1Jk#ai1QA)~#Q
zyIS}4t^KxlUh=$s=>i9A18wixxTPl8+r_WltGoB_-pGfX_A<KD_p&<s?k)Z?bN_pT
zb9;Lp2<(&U;oZ0Xi{8GX^S1kL7dPzxA8fV%t`Ez;IHe`~FGU#IiqEm#*LUFm9?5y!
zb{k$C+PCO@z}{@bN&B@}&e>&dkl6P_<%RXU+{ycX<hI+s`Vwp1Hkp5)z+HiTa_4kx
zr^__k{}MB@F$gTP)!5^;*Gfiwf2g{E{nZ6q_EsIex>w-kqJ6v1EZ=)};=w%?hZFY7
zKiIg>VXE;ymZf6*VwCLc12R=@{f=MWefZh?{Vok$dxO%g_jG*OWfSf>dvA20yshYl
zioIKQ*X%#a>9yDV=%u|ke|PUYxkzE}ynu^)?;2|E|Ln<Uf4ujBjmql^`)1|DeeWF_
z_BqC@?3uNdW#6O)ZTn6JC+=6`Pv1M?;qCpg7O(AHEg$aF>+Ij(`oh@WQ=emhI<MP4
zsSj87DjJCHpMUE3Ue)+jd&>+0_D*te*k9V(xwoe`&rW}dxow`r-o3Y#F4}!FH?WWP
zQQT*7M921pxS4Ilv1psW&3ku;Y+q}`u(@R4>Fl08caNU7ogd<Dv#WjD9{FSj`^=@Q
z_lnjnus(3>vd#Z^;eA%$#P-F%;oPenW3X?DTGsvquUY$j%U<tw{TQ^rO!KfkS4i_d
zr^Qlxr^XrEne$89ZYtupEs>kHx1c@JrX$98@7J$O_ig+6V{dYg-(HVrGW%4D-1aBh
z&)e7BX}eED<jmgqly`e&nv3?WjBc?na%<klE*Y`!rv1geddA^)TQ{w@yWJ_g?}k>`
z-q+`h_bvbZc5h6J+1_8pANGE@F>|lQoEQ7nOVr!wb4Kr-ke;#E;Df3y)8Q@~2c9>3
zZtyqQ_nLF<n;bjK*5k>%y*q?#to>fS*_W&ly3f4!(cU_d^nIn<SMEEx?El_H>u>F~
z5k0@>#og(9FMa&Cr+R{&z4zHO`+ldF+8WLe+AG%OWpDRBcVFC#Pxig*Sogck(zdlx
z6LB#ARk&B@&@-Ew<<k4E&S<b*_eg%<--FuwDxTKv%{cMdraph|o~v{C_N|>WYhPFO
zx&3t=d-j&SiP+1L>a}-6kgDC^wxxTRkFoCS(Qn<Gyr0+BJ!#dxm^+vEPB3Y=NfNf&
zcl61%JugrD+C2EJX&bnTb6?BHIr~0bJhu18qb2)R3D4O#>(R<RmrIZAo0bz`TRX*i
z?<`NIeNJv-`)wSS?(0nbv+qg!qP=T>I`7|CCALpDX3f4#`6JdcFYfOBnG(2fr(4Uu
zFXEni75C5Fy)2?`uK*9<{@Pi6d#z@x?v*s2Yk#ibhuuGeBl|u#b=xIO5ZK+hgn1v+
zgVlQ-IBV=nRp;3Le3Nhcp+$J#sT?Ld`G|e{Y94Rid%MADpR?Zy+Y0l}y|v}9_O6=f
zu>Z+VnSFv@tM=Z0G1=Zic-vln=DT|))p70n;q102hM{eraoN+ov;5U<FR$U=Cl+>h
z@ABU%`*gC*_i1xK-*aD1a-WAt%bu7!s(ZDIjQ84f@!Fm?f4nbydAF^PQ}tfg(m=a(
z{T`d0$<y{Oab0UK*4eR7<$vDZS*tqt?L6IV8xnYM-~8A8_D4jg*+vFx+g91w?Onu>
zZ!=x2z_u=Z`QAGLv-ic=U)pCIe|B%x_Z$0G?76e&wx!wL7aM%++4&yt53_l`xB9~V
zeHY%b?#-xsw~vFvcb~&~2m3kZFZZ6#&D_Ui#=K8tZ@jHR^@Dwh=RfZLynWR^*U9>O
zd0F}PAJ5yqZ^P0L)@OIH@BQlIWOrm!<o+#-<My%#t=g9oKi|gU*5tjf{?4@BtG8vJ
zeX`BIx6gL(dr@F-*ATR9|4+-Ty?zDRw)vHx_dYJ*wNLu7Yi~mHvAyh51@^tGJi9mR
z)2h8ieamb!d+qlzKdjs*BGzT=EMm0pu+EFU;T$ve<|(S$tDG>lQ*?N^*H(Sv-kd}2
zwv*C5cfX6AxbMWC#rrH)U$kKmU$k#}_|?5Ni9YrZ9FOmt{`$hcXUz-uWqc{v_s-$b
zUMZh7`^>b}_Nuw=-uomla39x{w!N)_zxE`=G~2WV@$5^Sz_!olz#Q9Tj~#o@dWr6P
zwr|Hiy?-ft<4hLp`Bbc6+kWlL-rF+1`<ef3+v~P|{ocb1^Y{3zdTY1$*3P{`J6ZPG
zaXTM~@A|MW`C<2-dj}Zz=_+sCTO_H!FJackz15Zr_Fm!S*n8E!)kc3!tL;4g#e4hj
zGwfwptGP${{igj&YmE2n&2iuB@JDgKn`rP}=lAROeSPwJpKUtB-c7v|_C9|mzfVnz
z*XB#H%-$7!Df?cDeBS4={fEt8m9%|v#j9<?c1_*qpT)k<te;`uX2WGRH$+$LJ1ZV=
zpvP{$O~$(`HuvKv+b7=k*>`~7VgJPW&U<x^oZtKG^CH`u{5pHYKJM6aBAvrlL4$wa
z@6&hphU|Q~=a%!H{j#rD@BKCHyiIjR+Fp&#O?!;;_F6AHd}r^5Wj40ae~kA1OBA+U
z7GJlolzZ#G58Eg1o4kAD{_L#`dzI~3_x_0L-n%(^_MSaGtoy!9-ENx|*l#<-D$90p
z&Jml22j$i_it&4A`s~^3yW4WFzo*!~n!B_2w$#S%?r;j+cS^r|-{LEG_ZWQZ*xO{o
zxvz2Vv%M2*ocD=t(%rX2#?N+Fg~{GM5p4SvB#-WOzx&p1QB=@A_pMiL0(y7sxx>8I
z`cDS$zLWa<_r^Zg+b1Q}V9%$qao?%FjD1|T4{Uczx9(#;z_I_@%Gvv}(xmq}dpzH_
z(?oI~ugw>mFQ2>iJ~+JHdde&Ry?-aG@B4NA^<JYd9Q%%?@7`ORXS(l=p@eOjjnKXV
z-p%_?S~>4kj%eJsVec0^snd~r-{jr5ZE^m*@7v->`!u@a_VdkJx&KgDoXz$2sr#x~
z=GvUHS+uWn(-9lH6K!^Vfd}{2dxY&h`f}#}{UZ7{9m$jSDeGI<Z0WJu-}^CcKi|Kk
zJ)U#d*`<H;*{88)tBu_1?!BRNkL{I~S+%!pozY&0voW?ie!kzQXWG8^)<XIHKOP+2
z71PpX#nydp@5G;n_r9FTYJZz)>t3Tm&VAcC_U{u_IJ<Yz>+SnC=l|c+>Ls&(2j`r<
zUzW%1JN)7NJ_lC=TRTb9y;px}?YnUN{NB94nYM}2m-qf_<=N+*8N2sq*mpa((h~<F
z+&=AXJMXZMtKpiR)nhGNg-+(ZKGzry#Lm{+zgzm(Uam*t_9>BB`*!*#?3*lCW!>Ll
zu<vD3^1i0Ym3G^@XW8seU2ga2a+S^bCAs_FTuMB!ic4U>(95WOvpMtjnP&Xod;ir5
zyZ7Hd+vPa+?Pc6^VXrOE7rV7AlKXsb2-q91e73jq&bxgdE_~S+@P7NgJTt|;x2}io
zK9siF_CQCIO&mv{ZQ_p4dy+Q2+NXWs?B1kvSL~l%Ilp)La?yRF=N{YMkCos1T3cw}
zqJ>iX=K5OiaXV$aFTHj0UZDr3d!1fy+kY)V%f5P-i0#yO-}ftXU9jWw$lB+A+GF49
zcXIpIYfJ5$^G<Xh!%PF4x%sd5EMk1HH{8{CU+KoqJts~Z?Geemzt^)NdOz3EN&D^x
zPv3j&)Xlvv!b^4+wuSC@=bdb?a*|_TnKk#mO%`8lHu-ee#(m@6e_%@I-i)Jl``?D^
z@8kSqzSlH2a<2#5qrGouaN6u>e7R4=g=gQ5mPva%uG;MN(TUo7ii2;TxXAXsvYlo2
za_fZl-70Ca-I9EL&k7!GTb82?`?F5)+D~%t-TUh5qrFBNOZRDCjor7OeX7llx>{QY
zrGI;)g`4*76pyl<6<uaGZJwrW3{UvpUvi2E81C)bm%2OI-Y50sKGwgJ_XvH6v0q?(
zY47)s%l4+a3)|j&rm}CFvgW?VQwwb_y>GDHz;SXfhpDx#ec1);JyF88{I@!6ax$#<
zmboq56Pwp=<GU)-Zq*H^z42>1Y;(QdSRE^4vHej$%f>Wr<z5bkG#eXP1KS`&XS?dF
zjQboW&9mK7m$0|#ZI|us?X&hOEZ%2ZlPGSJJC9`_v-wwBJHLyzbJ}LvR{mD8UBCa}
z-h|kg-Sc#`Z9mNYZY}jv!*0uMGuwtn%iWLGP2ZiPFm>-9_A|E3l^uI;y=vV%`J3e4
z_T?w`90*{yGqumy{pIqez0&PpY(+$+?0i$b_l9fr?48oT(dPYXmA(IV^6gX1++^c&
zuwu`{%cu4x?pwNN|E}{k_7mA`&x={@_0U?lr>$$sUTwCvz1nwL_IT);*$S?1wq4Rc
zb$8+BGTYl_d-i^~vv6-siP_#K_9nIwjfeK~X{Xwx`LNm@5M66~;>Y2=SEkIc{diPz
zuU7qGn+2g)ch57g+#A?zy7%nQV|(XbuHKt&AihtI%V1x1)Q-LTcJtYFPMx!t@4T*^
zsxFV6%8s*pdN$77^X_Bl-sG>>_A;2ww%Qwc(6&kGtnCSb$2Ls|)%OZHs@U0@ZM9iG
z<;ULeT7K*I#p3%Mv?TY1<jUJ^cz(*}BhT7B>St!}3h3Ou`=Ywb-gAFC_m)^qvTc^%
zZhPk#^S<H(du;9Qtls-NuG_Z7e#Pz-?q#;7kCbgV0_NFnEL&q^!0>XfzQfeLoGY02
z#4b|b+i_{5P1FWAyEjTpY%T8VfbQ7alUzK{#_Cz_-itwLdldv`?5XfSx%VVPfo*QO
zk<F8im3tGjF7Np#c5?5|=Lc*okNMkdc)Zqj!@Lc9XRZ|7cdjRXZ?xZ|y<IOB+3L=l
zv-ij01$*~=_P0H;c*WimMQitJJwI&A>N#o8-|2F8QjZtx_4F3lbH6!Yuc<fVp18U8
zdrf9U?G2w=zIUC$sl8gBbM~GryS4Y<wH>xU54YJ?Ih*XO+?&65*5?_v44XP_oV*nG
zJ(*pycdJR0t$l36-r7BD_l9xz@3j>7v7Te9xVPq9-=3s;wtdHH`}f|E*}eDQw8p(j
zaU6Sh+WGIPpUu9{aF428Xm8eDl|@HvAF&tPa`#@dF?hOW&#QpbHXE#S_F778+Ow(m
zqV1|%T{g>IKJFFXz_3r*?fG7px!Y};xt{O6uV%FO=A84kJA)%^Ba#p8a<IE-ooK$?
zc8(*v?UUBGdv&d=Z957?ZQpqL+B)qvwO0H8&?e>$yRAXvU+X^(clX4JKi>N?HE6G6
zjr?BG56AcZJGIGn#iN|PPx!d^ooRW$*K?_;ZQU34eLN?m_xer{-M5wF!rm$O7TZ+J
zGT$3h8*Mw!*>KN?mZiHBzCX8qKBssuuYd909R8Ji=RBUeXF{FbUcWPo_BIr^>}oN(
zZF42-^&U}GOS|S1r}loBy3OX1$E)3+j!fJ8@+jZFhB=j1ixgw_9&z5e_lg6z?c=r6
zZC|NZ@4dLCYwurQ>%CSRkM8Z2`MCES!-2i`EDzdtDB9XcY%#H&=j60^iH^1H9|cX@
zV{aPvYUJweIhVt{cl(Uyy??vA_uPBQz4ye?3pVciLiYM_&9^=1w`ni?&bc<x8z=1j
zu`k?KV`A2x>~{jT?JHi|=p5N=GyhrRo<%tmZ5a(7?74REf$bN)?!8qPAMHK!{Gbir
zF=x9H<qLc7_<z}ZcV_<HE!FC_F%KhcH6NYZo7CxL%Vx87Z%2^1t@aYeebHey)>BUO
z?5*(Kuve^O!Cu|R#e1XW&G+oRyJ4r&b@{z9CX@D*SR2@0thU<gk{oB-A-8-_-HMfa
zSNwCc-n_DXFT?$1mLESe?5+9wZ*TXJ4x2kH$85iK@$K{fA+>LfhUMPJcNq5l4t}{e
zI^fRUfID0FCNL`4K6<ZUTXZ&N?~+b-+q~8#dqou`_HB1b-R<};V(-m^={EBxaqP7)
z5#PHd)O)X})XhE1x#sLqUlO_3?$|op|2L=F&bj$$uW8SUy*FRJ+ACgazxQ)~p{?u9
z$+p!ezT5hwI@lg%mD~5H<LsWbNq6>YZR*|oY`d#nu>Re>PndITH7q~c?8#2BU9`Z=
zCR|Ws@5a~0R;!)o>{*knVf%5#p}pG~9c*8`l(ot2KW*##X8oQEmco10C+6A2aK5v3
zEkCo@Hfhz~nVs5u^<<9hy{ELphUxGl+Z8j9?OF6E(sr@XkG%yFBDQ9KZETI6e%>Qs
z-nh5<goBM(YNpM`7CF1#heoz*Z#3AN{$Fez!&GRit2$$k?HBjG8J1V~o+>KZ`~Q=n
z?P|s4d-vShwCC5nEW3lr`}dj`_t}1m`?=@IvKTvo#;Z09zW=e=QP^%P{UyixP|>=*
z29ft|SHv8(xoP%y@5?;dy$^g1Y>fn7TbnKZvAc18rR{y0`FlmV=j?sA>cw7@h9kDK
zSNmGOU%z(mX$8)G&lwx`o@-*;{V?5i?`*@#wsIzG_WEyHylaQX$~{_d5AJ!jOw{(n
z<QKcQFqH4zmAc&4c%9N-mn_A7FZe&~nI6PwE41;Gt-^d;YlX>wt*zHO*(zK5+rF;Q
z+Y`Do*lva7j6I)tX4?J@XWi$Wr)sCS)^e{#e(LVGjNkV5$!@R~UbcI0p~R8B|3#Z^
z>XR1P&hl2b_497C?NL3wH)8wZy}|kpd%u^h*qbtO*&fM{*KHH$1@2{>xO;bd-i5t=
zF*o=2C<|DN`mVF(Z2Pk(#!<sI;r(7~k5yuJ6T77LT~KYbKGZ3)&u>?}P2V?XyM4O?
z_P+lz*H*sc;NF;rO11}#rdaPt+-|Fp$86`aV5{wex&yW|?oGDlnZUd6-feMP`TVDQ
zBIG~s3BP=JZ;yz;-V=6mHvDV9?yZby*u%Tv#=ct1^8Gm>U3*m4f8DFjp}fx^W5xdM
z|9SQ<&`ICd^^$pS_;s#*4zDKcOA-;^7ZAPN*7c&oK6SP%+jaFxd(VfQ-aGx#wY}`#
zoc0=C3O0pDx%NHun!Mj9dih=prp-3|3+n9yOd9rVk80Rsn<jeTkmsy@5+z;xi@T!s
zUN6|Zzy9$nyU!)-_nl6a+jsgThpmi(<-Qn)wteEAFKpyZS$7{?u-$g++Ix2Dzt-EG
zt>@nNcYBJx)~S-c)7-`P2}?ZP@BQxFUa3T*y@wCW?w#?pc7N!h>V1cuW9_qYHtn6b
zVyUf|+1h=MMqjKyNzdQclrqCsNO-rcf`_WD=!*k3mpOv=*)Hnc>yqASU)s3H=Af|E
zz6sg$_Wo?M+PBwp-9CkPR{LB^{OxaRl<xUAW7QtVk5l#)-&nf0(J8_<XiL++b2h@Z
zGXwJXB@}7yW3X<sI=*J+zVf%pdlNd0Y&HwOwVQK+ZJ&YHAKPU+BlaIz*t9SFoRMwQ
z8NYpjs)F|Gyl3y5P}yiN5z4!#(4B4HEK`NO*VeS}PpaC#FCmm~pYp5DeX*Hrc7HOx
zY_A6I+?REUdGEP?M%(G@{q3d-wb`pVZQ6UwdhNagq5-zCvkvZ?D73?Fq33+t3G&+e
zv>x8xZys5=m*Hcm^{YKU_Z^D9xF?YF(EfrAZ*3lkrR~3VcgDWZmc@JjXiDz;AGdaY
zex9;Dn_~YyucW8@xHY8qO}Lx4Cu(ip-lPqjb`rHEdp7=kY4`6Tt9{ch&i#kCW$oSW
z@NVzz57YL#R9M(Q5H#8IYr|pNxi`i4rL-K~TR!3C-Va^7Y?pC}9++;rX7AQp@2xk!
z@ILU^Zkdhn^mBXdCDr!}NbIp$U7x+z_QJBg59;68T)W<H$6lpj<IZ`0AICQJJu4NI
z_62E)?_KpdWZ%)5FZZoElVX4N%cXsmuh;Ii(-z%#O>oIR4>gW`^SeTAj%*b?u)EoP
zZ^<jEeJcc|ZLTd!-)pXW+Q#D4vc2aX_S)nZGVQycvvuF+8SeYet)H?t==NPZ;myT+
zEk83Ic(6}wpWN=oy;A-c_wwI9w@;zFclQZnzP(NDZ2RV?`R!vlC}{7I&bRN)`RRN8
zFG}rubwP1ovj5aQd|Gn0ZA$xX_VR7o>oc`|?-zzY`&^uY_kLRN!tTdPfdgzYr}r8i
ze7|q%gV%fb-gNF$xh1*(kB+&W-~T?_Q%lYE)vCnqWo=2bjpU2lH<@S8{(=V)d+deJ
z?u*)>zfWmv^uDezrad`Dzil45Jlu2n?4*6>J~I1HOsTc~n`*H8XZ=yzpx@i}{gutJ
zTbkIl*CM=XPsq>1dlChY?M>)(*^~eH#NJ<b18h=7?(Ge@m2A6YbL0NU6Ta-poZ7Ni
zpli<VXoH@;5AyqM*&`qC{ZO!N@04Bn`?}uj-4|rpyiZRlY+t$Pn!QFUJM9{qxNN*q
z{r1`P>h2Yo%Ce7>L1>@efz7sdQB8aOtk&#RIkL;zK}d2h$K;+pYyB7OjaWEkuYX?q
zzQ~Ckdv`_f?NwagxzF%o&pxw9A^VO^KV*C0+Zmhulm?rG+tc=an|66`+7zL^Tr4a0
zu1x)7cRl^r-T<$FedV)P?Bjd8W4E-0z`lsj*Z0<Du<e_aqO)(O-K>4aLf`B+ChXoP
zWZ1G#_f)ryXK&lSr0CiE%=i}UtH_qv7n=TM-_dOud)4b@_MSYlb+2bh=H8pzHTE7k
zc6x7ks`%cAi9CA})~MO=y!yX)cH5Ud-!6UM`)K}*eQ$Dw_Gt>c?z6jb+4i1#=Dv5!
z0`{<Go7vygJiX6{d-uK%$@%-8yG8acDqFU1R#choa=sh;=D*^!{lL1=mZNBfee_Gk
zy)K~#_Xpan*?aIz|GsZaS@!+EDzJZPE9-$%*S_wR*FL_drc`17?O*$>WhXh>S}nR}
zXCQQ8@5iY8Jx{LBwQY)3+t(>4y62>v_5N8^RlD=dUH9$~TfKMb%IWr3HZHb3ym|HB
zI>nHETnbV9-dM=(y}x4JzO~H1Y@|BF_j#<juy@L}?tTAq*6hE0XyV>HrrN!y+Dh$Y
zzVPkcd@6bG+UOltqV=JByI(TyD~&y18~8-%K&A7By_|JZ_fFb#$8M7U8td(g{r5KS
z{$i(4tiS(e!^FMIV~^Xp%~IcIzg}+N;_~-<^<`W4F8=PZ@81uTeVH51?NvFaWUu>#
zZ{MshQ|x4>IqqW>RNpVSEPMZnACvcKr)lopTRz!deVf_-rd=YocbZ@9-6+bm|Fhe!
zJqPco??1Hn*<QzOEH?L6Pv8G>SLObao$a<kr8oAfyj0wqVY+qiojuKan`Rd8eX!)|
zzFS<=_deKu$=+&Vn{B+1g6)ot{`;mpX0|Qqoxk_^jMcm6njG3yuBWpv;^~(?vJQIt
zlImCNdsfM4KT+V`-tJr@`xSER_RXDb`x4q%_wK(@XYX{?Vz0|J`#lxiLN@G?tozov
zuHM7?_Um4wz_T_MQVM(dmhkR7yKSP~OO^xsCTUvkyZPki-fSsj+fai~b{6Nv><a%b
z-rHYnvd?Yn*1Zchytn2N+-1+4Ut=TonAcvg&~x7+)=PWMm>=zZI(PD3t4H^DJqum4
zZ?^Bpy_(ev_9mY_wO682$2Qbd%-Z!K>%Ljf3-+t-pJLs~#<ss}f7jk=M@08*v(?_O
zVp(m^T*<L-Wxw3M!xvrmRjm%(_sDGSUbZa@Y?uEs-op~OeD8gW&i&UFy!URNvfTDP
ztKdF!PN@SuCUf`wI&{}|-Lb8ELwV-e>Mj3h+xv6o-gOU}_o_xY+H!W?wQ<~_yO(GG
z=Uo?;H(EE8uiWji(8Jc?T$A-5<BmNKwy)g#!eQB-o(nT=yV_d!GDyW+3;$ZWHzrHo
z_RNxg+Y8^*_xxERU>pBQZ*NP{(>=R7I`;|$?6;YwIMtS+IdJa_;SGCdELgI)`n1|!
zj>upelUZ`MCRbkUS#a#G%?!D-wo>iuYz_Y2v|%{pzFVnlg6)<seKs57f^1eCS-5Ax
ze<@qDC!1_DSIpSk5%bW7q0-uxZAH)CGfqslHg+p*Wn%00o=9!o`y``juSweVJsrm%
z*vR;6+s<^}yLZO50NW{VIrg5rePnOys~LNLt}NQ?a(4Hgn!ZxotoBW|_abN83bfhU
ztWDXsS8(5wy)z=6ZH}CuX`A4#xQ9VHb<Yn`o;^=K2<;7!a<DZFFWLJb+}FlOr`_hj
z6z;tVxvaYzly~jQn0wXc(&lxxEd2BKeDkf|8}a@79;uaydrP=>?NR%Ge6Nf^)7~vB
z1#A<758F)0vD{-|?rxL9dV5b$a{S(el{aiS^uF#9m?^VIWMb>yeU2OVG6|ou-B2^d
zc0<j#JtB7lY|VN&_d1;9v)wcM!5&BM!+V#Qh1lMaJ78OJ`=w3C_qM$b=KWSnJofC3
zu<f+bSgc`tY0)v;5~F3d4A*3K&vRP1SL0vfo|bE`_xxFAZTsc?qCFms?R#tNT5O-C
zcJ6IR_S>`MWaZwe=I8d-xZ2waI8@o}D5$V45m;oKRlafWP1^-~S6$q{w_zfewb?9-
zy$3w>Y#HKq?JPdNXm501>0Y5*oqH2rIoe*>JjdqJvLm)XcDC+4Q`l;&64PPh8X33u
zg=C9O+5O43i&q5f-Lq%;o@pnF_j=^-*~_pX+A6}8*Vfx~)?OErwYCnv%6mLtMeY4@
zBzw=Ad}~{goGE)x$n3J+a_+2cieZm!gVTvU|0<hpr`&4UThAA@moe?o-X}e|wq}c2
z_A0dY@AWz4ZEKLQ$@)aw%snidx@<I-Otr0<WM=!|)VAI8=C#<`@NBl#@QK*-!TyQ0
z%t}7nr`Fr|&P$nMn{{!5?S};Ay?YvW+Pv_ZZNsH~%r@X{nC+=^XKk674%xm@J-$~)
zYmV)L7fiMZK?b{XQqJ2(ZCkvz@Jioa4lj;93I0L0G3n>_YAv3%*UVIGZ$YB$-VB!Y
zwh8}=tP^IQwRI3~-J7)djIHLkhP@06-tP_x*0<H@{$$g!ceYKJ(3-t(F4*tQ;C*M4
zuu0wahUx4*6Z$q;m$huRm9*^MYY@7|mSJbRP0ZbewldCqwo4D5-@Dy6ZSQCHjeA4p
z_St^mwcV@w?AYF^UJiSU-W=PjDxA2N;Wh8Be<gN%3$7irnU#3nHln-B=FsK+wksa~
zvw6YZwCBhof7=TzeKsQ3Vr<{B%-t(;%-;4y8=sAWc8YDsJqDX)DIK;ZhlFfz9G+u)
zn|I0H|1&1+{jr+UhHdA<z1-&$_8wWc)K=!PkL>}?342Z$huMBG60lA9skyg7tk`-_
z|Ajq9x~ulyu&c1We(Lz%f<`BsmVhi<2CopC;B6Uu*BBV>wZD2`ugJXJwl7vr-uq}h
z`(A-{ZF>*s^X}C+G{Ls%%SPLVLlgIGV12S@iDvm;1&!Hz70e23ei(4tKG3+g``MG(
zwh6rzduJ3M*)wHFx6P61>uld7ncBLEmG9+$AFx-0Z_nOKdJ}Cwu*=vy)7x*$CVS5I
z-&@JO+ke;X4G59m6Y_Pp?V{VqZI7&2vv)yIt#wEC6x&Ukn{8(pyx1ena&YgHtL=Mv
zX1&}qwfn$cgWv@=XNnf@{Sq*9?-uTldv;{0>@mr&vo(-z+H-?zyUi-m6Sh<L9Nf#S
zxoq#2+OWOvx^?z4Ok8f2vTFC<WnYfj@+^<pYvSLtw<7=0UWN&3mZB|IdrfBZ>`n99
zZd<lAWN+&8gL}_7T-~!{(l*-_KR9eJxcl0cC>^(T>RY&Xj)>e|32OlxhoDMZhhQsP
z4QAQBGDo-DEMGKnuY%<2z45z^_pa&QX7gf`l`X@MeYOEU-PYIGllL+#d9cT3&LP{1
zh>5l_XFP4EU8~>QqI_ac0aK=}&aGK{1(av)ePds1%W~|3?Uk+^+nkDXdnTkj-W_H>
zWiP||Q@a_i>)Xg&J#M=t>f|2oCz^W?_{_G6S$@WL(!$lY0bGl%1Ab4imO8!3_CvJE
z-fNFI_8!@`$2RAou<fhGhxf|qOy7I*k^SChPuA_d@{M<|!mf?B6-h0&0v`qTOyknq
zTk+-S9+TW{d;Z8}+J5BE-}_?LQJX4`)q8Ekr`v3q{B}=`MXc?DBkX%cer?;^v2CC2
zR{iCB*Sw!*J4b(|Z9wj9+Y3LIS--KU*c;M%e9t-0j=eV;pY3r_i?P;zU%fYNhQ;13
zyIc1Dh?`{dBe%)cxBt-IjviB+1m@t~##$D8)tYD6PWpMs_DfBTt=ZI7wll8j+B)nF
z+WVoH$>vc1E?d5+R@)6HJ8d(jN9`%F=(hc_D}4`x*zP@6AI$e&I=#)7eOCNlhV5%@
zZ^s?po7gyO@0@Efd$U~T*zS;Sux*(2aQ8m1)wXrd7TB(xQLs1Sm9g!L`{8@!<QLoK
zEZuKgQeA61VfvE2Kh&1l&XI|*Et|03w&7)g)pbj~y&}(xY!3)USzptiu$QZ)-&XF}
zSzBx8n7z02toAZ&@3r+yiP}3u_1NAMfiL%nq#v^Nv0Sz1_odLiT1$7^RwORjyFhii
zjl<)9Td92uZLe@x+7?K5?M-o1+dJQG?%oXyPi(Fnbhl+#`_p>Otw>w7PjhWc_7&T%
zyd1ID;^nnH)3@#4yW!SR+m=$%y#W(G+e|yvZaaaG&qhL|)%K3;`8^x@=k5($zI*S2
zCeA&Ve(tc9I=ITV?q1AZ4x#CLB)SgU<V6|p&8g|MefN3oUb(Z=_s)-!w^=sn_+EjD
zihEz`J=|+{Lv-)W|GW08I8U;T*b%a~JSB7Q10|{55!&8+{#Ku|ZBX4{n=*0Y-aETH
zZ0DUhY%9C!oXrL^Kb!a2JFJ<iKkv1j|IVgB=!zA;_?x{qHuKx=5IwzDc>99A9wDW+
zO3s{nlUB&vE#D@yFZ9@vy(b-a?p>I?d@qabj=d|)fA4uEy1>?PkJDaJiORhUW{3Ag
ztXyXk{iJ<w#1a9!1#6b?mf62)FKfN5oo4qG+qw+5y;(~=_Fh^!Z?BH#O552FbM_h}
zUEIBak<VtyenZ<ik^Q#YrQhv*ad^hw8%j&|?&{mQ*KF5<y*ya~wkHz5@2Q^KwmW$K
z`n?8OSNG)eZrkhC_ugisS%mEdHb>hXW!!sb*a+@%css*ZC1#oJ+o=clXuL48`Lm(g
zcE?wNeK~((Y$sb!+PmV!PFn-sLwk1B&#`^-ZLQ6m-j#b5o{R2TvVhTc{Y|aC>DL+d
z-HM&JSMJ4Rn?$cody~3Z?It*6+uod@X?vr%eeZ_mC3_c0RM<u&2<;UK*0(iS>uhu2
zqV*ml=a#*DW-YPV@HNC{Y0-MyKhg7TH;7c(+)UcN_gDIYy&P)`Y!i;Ywz+ppWzPr3
zragLlkMI7#c+UEbjlo`{2l9KjT|Kn-LA=nu=$`aF9HLRS8sdRApL>+|hNL{O6<B{~
zw|AhVZTGECTaTODY&BHQ+9>~3vQx-bw)?=j&_-(N{Jm_!&-TPRM(q|+VA~hY?Y;MQ
z<%Yc?O)7h>n{;dwRAu&tn6cR|h?~CWVef*yFII1`ot3@JmPH~F+{QPUlxri=k+bKO
zw#VMMS-iHZ@*DOl^$YI#^J~T4GY<~!brD{$x8wi8J?mcb?a`S3*5*pRq}^|;x3&jj
z+ik35=G$EBi{6{l8o2kS0N-A%zkGIA7A@KP;^<r3_wIFj+ZNxjUGSLA+Suxq^^1Tu
z+u4iD_wEQ-V|#<2$0p~P=w5CImA#986z>hYsIu4KV?AhZ{LZZbEB3nT^X!Y6a&qsc
zMXh^%oZs4d=>M>(mioNc^ZwnvBClob9&ifUDclOOcFA2}``b8s?~ylxw)Z9G?k(N_
zYtIIGKAVajPTLKwEw%zvC)<AU7T+tgPSaLiXvy9V6&srw;(dFTU16|0^QUsJL-)45
zKk^Irv^{9qd!J+0-sHl)d;eXE+IytBZ_hs24SPR+IkVS?U(8l#>Sk*pwcfoN%gp!s
z9@ny+%s$2X55sgD0rf|F78M8YjobLvb{F&J-3kZS?42-c&fYIi*Vvi_uHEaiYVlsF
z*37*HeLHN!<6rHWdz#J8;!4XNwHwQ9=bQ+#E&032Mxuag@0&GUdn(HNZ95n2+?)06
z++H@1Lwm2>KV{>x?#!Murs8&oSIx0Hpe|?|@$#z80?D5?^Q4#BdhbuQ*%!3JX33`8
zd!|$_vON(u+2;N1oV^E5&9qI3OW%8vU19GM*>0N|?o750Kf`SW?l9PXvwLYXP4T#O
zLhGTuwJMivyIK#~Rwp^{UC+5{?~FzLwmY+Q_hgvP+QXEyaqolwdA3>7?Y0aa%WN*h
zFR@l=J7VkSd&Ks{{9AiAY~tF}#jIq<ll{)NdBuC16t}B;F5LTPE4kZuPs3hzo0y7q
zHvbxa+aA4kdatG;ukDKTBep9d&e--H-oLjwV$a?m^B&v$_-<x%AlKd2ahKfQM*rh`
z-b{0}Ej#<xhAS`KR(+Y1jn}0Odt3B<Z5<Bx?ryQ<v6C>Xu-zbOyC;Qx`QG*&y4ELx
zX6zLgS+)10Dd#@p*XQ;+Yp>jUV&;DvDIx2<3ao2vDjMGJIk0)|-ixB|ZACoaS#!LR
z+}Ev;x%c)z?Y%B$Gi{lZP51uU617(?E!#Hv%hx?uYDIU2+?uoJgbn9j&ly{6)nk12
z-jlDiy)uJ&@2rNSd-)tJ_VP^Fyyr;lWSbSd3vCj_Dt8N9dSb(Tm(|XDo6X)20-QGL
ztuA{79NzE#S5mt7TI2q`EQif)D}qaHeO7SqHM<>Z+jPu#Z|9dSd!-&9v0YR$)%JwR
zoIN~~1#M+cZ`td!AaSoq@PfTQb9nb^eQ33jn^R=_bY8_?1Ct$lmwnx|E2MSdUXiT+
zJ?E5?_qJt-?|q<`VXdIjzIO(jsEwBR`MtmY_1gY3blGcCwsTLw-n+YdCvLEfc%Et-
z*6VL;puOI@N_Xzweabw08CLAs;T|8jm#6T?9tW>Ewh}juZ7=wLuwjff+aqJ4Xxs8y
z-1hV=8M`=!ti4y3_1n&w!e{&8j**Q+@uS^Sv}f;WD6q51I3aGkL@m^I)hXw_SK4d#
z#$7pRyCD6Fb#hGFUa5S^y^AjT+X^%%+VTW`-E(^Vti66QoqI(Jh4*fhnrQnvp>gkL
z9Ra(bw?5W=qC0J$FEHENCa}Qv((*&LF^W32^*Unvj+894onsYbTj6fLXTh4~)?6!?
z_g1jQ@6B7TV5jzdmhHjfiF<WsY_jc{kZ0@aqhNQ3|HGaH#$z@gKAhTX;;+1yA+de$
z)rU%U32Bwq4bzpa&P@=vQ{2dJ_vT5Rt(|x4UY3&Wdmo(V*lQ9JVLOLid+(&M*u5c#
zByD+LzumhnzJBkt$J6$HITvheaA1*508@f(*Ph6|Z<}xKG2z~B^P~LFt{08jwh~=W
z_w-b$?p@R&WLI~4ifz<{9eWq9V&4;Rn#pF`uUR&m?-%V|tCF=hqASN{MxU<DMb^o?
zYhtF@N`LOOJ)wEj_Cal}_0ieW_rBcew|B*ghCQlJj_z$zT4~E9dtmPkyAs>kmdp1_
zXujX$k&v{<G<b#W4c0CjpGH4hE$QpF^HYEBX34$3XU^q~dutw;@11ktjBVJpCfkVE
z{Wcrswb|Ir-C|?5Y`$%fGs9jE{wI4>a#!wE2$0^hf#;)b)Wkb`cjT+@$vgJPW>t>D
zz7wZ*@5;F4x%cQJ5!)kC=l9Og*R?&sxWv|IlhfXnUH*GFSt;7tJYw9}a4~C_#SahL
zHT?Vc-0%vwjVPS3S1V-k-sY@5d*gmh*=srHxoybD?|a^yUu~<Fp=4ttv~-U~5c58s
zB;LJe-X`z9dDwZc_`K<RR`MO*>+t=FwN^LtKHiO~dm9v<?m5ZTzPBc*z;*)1kG)gw
zEw`1Lq_X#pHm7aF0Xf^Aq6k|FhIU&<`9FIvcAeOJhpW)`g($Oa(sD`LXQvO_DlBc-
zn_^pGdw=boz19ov?+JL+y~k$N_dQy_T5UZhNbdQmr)1laH(}2;7cZMLj#u_r{Hd^!
z`5nHu;pE}H9uDVi|1H?I_rcn(y$=i<tQAb`Y%I9u?>(|e%0@1Dm+h{YB{o-LrERy~
zR@lRE?9Cnvu^ToQZZ6%MtHx&=!<(^JX6e$se?FYv6TrG=uR-{;Jr0L{@A}HIY|n|O
znzl+$C)gh04YjQ(pRni7f>v8ko!5KiuB^1xaDKPvL*Gr?Fp0Ug6=&sa*^c+_?F&3@
zdv<&1UU{7xdlw`)>|Sg$d#}mcb+&snf7+V$?%n&&I&|*|-vzci&-&~wTQ9m-|B|ci
zGk#_}2acWA59B&+4g4kcihS>~<v&omr^8HO-;5cZwl3<cZ8wH5ux*-pXs?&xn!Puo
z!}cC<xn<o~G1+!PAJ?9}^S15rQJKE?&f{f!!}7cKR&nmIT{!*O-qZc!wqo<!_6jKO
z+&guX>h3=KzP(1xQg#z#9`0p0VX$NSjRxCGU-sIZDcHT|PyDYvCw8^&nbDALbK6AT
zPNQ7Z_RG>;w)y)R_g?rfviFW(;?4v5YJ0ZbeZS}IoM*NH_UyLHo*V29xx%}5%I)8K
z_UQEQ(fK59yDVnfUh{=Ddmnhq?yX_>wf*hEyKjEH^IoC)leTlSFW5NFG}*g$pZ{J3
z>o0p3WX`u;xV*`xnLB&0{?WU(JKd-6jrlgwriwjiuiWHSdl<M|cJV%*y?4!$Mq7)6
z=l3ivwX(jE*l4qZyJOFf@=hBGnYKL<b_{kW@>BMd32n2TaJJM|G573VgVlDnR<lm-
zT`0DGFT=Al)+w1xwhT<)EaiDi_L|KSv6IM|xrbwp`kq4p8*D^h&D?vSIA)Il_nzJB
znymJ^JGR<Jth3o$w0_^-{WhU{6r!4J4?jP?_tIK6o1|mA_wISwV^gtxzKy)zb6bxd
zVcRx_jJ-$wPV9ZV^sr4Qo14wn!mhm!$|d)1XgXn?VYAHk$kC~L9$By2Tfy#a+oHp3
zm!)K6bwEIH_mh?3wld-~ZFA0Y*uLx%-e;@vX72%4-`zL37ujk$ckIn)5wtCHUb*+g
zku7^>94xYJ(5SX8ID2RJ8>hxSrOG?^+$cW0+tcvj-cu{L+xAAu*d2ZQ%~r?cg6-OG
zANF2&{@liBk*JN2mE~R;QNF!$BE5TCXZqP%8CULAxRh(nVE1zmQ*xE9glUA$4?8!T
zsG|>U_i=03K7KH1Z|JUjdu_Hz@4XZC&errqlWj?7myHeILR$%jp1r5L*!FSMGw&79
z@3Hx5oVwS5Q`pw0eYZ`_j&rt~ZZF$=Hcw{X8s@{cFZ34evFhGqn^rK#R=RYa?VokI
zwikNM_cGW>TdwMqu;ad9VLL<VkIjYlVA~m|ckD42owGMUq06>)LC@X=8q@bMd<n1$
zOjxyd$B}n?Uc6gu`<^RdFVD+!w*Ni(>|~w!?LJIcX`_+-U{A}e^?Mt-HrdV#pS1U2
zRo>n&hj{nJpVqXQv*`8S3GZ@jJXT${F?^A;TjQYg-v6x=Z8Jnu_8hek+b7PmXs-a{
z1?w4&VS5tHX4qI1&e*e1bGMCwL#Az5={8%3MlIWG&!^co{8P5Nsam_2L%G@}?cHkI
z7mmt%)FQsy_?b=F{rivPzM?Codk^rQxAwTfVynC5!QLXiGq#hu_SxQPonUh*{=i;`
zZ;`e)r`@u>wJ+HA?N0}ruHx*yvF4v_-+Aud`)6mGt<}Y)wwtycvVF8Cbng<~S+*?k
z$+mCOW%e?ByJ6#gfO}sUchcTn{jz(X?U%RPF#XUThhKSnIOe$TnXyU0_RA4R+r}2%
zy$UO??0#V+VH;e>W_RNGi9N@y*6y{_USp&Clf$+~b@QIx?yqb&=o#DGXjb0CcuaY3
z!jT!)E|aSFioI&uE236w`(TIS9-WgScEzi>_i9`_WAjj{&Gv#<jE%egfxQ=Q+uORF
z{AT5Q*nRI$y*+yyri<IMSRUE?Na6cl;qQO<a2#T_dG)B<+M9o(ZNl9h*1eC<><waR
z+Pk1Qa?gv3Nw&u&D))Yo*}C@y<K#Ux`}gl%G0)w0`}BsrGxA#Ze9Sc4<8<l3UXy!b
zdz)JNZU0~1y*J^$ke!#%^1TzX?%Mp$y0BNN_w?R3@vCi}A05~u);h_S&s%P9?S?p;
zY0o+Bq=Yi}ZhUuauf#<Mn+;PHY*>DD?kT)mu-CXCYwwHo-}jicq*^c7K5_4j2rk<X
z0;;wS!Fo0UCp2vrrJS?1ZT`6TH}j^wZ{L>e-L8CL@3w;Cy&|g)+m@I7w)K$~vwgC5
zj*aARfqlQcm)lxzSZO2T^VjB))9JlhGpE}6+}vX0-}-g0gJ_LSc5<$*5AU?SAKq`<
z%ka}>*NO8}ZB7ZY?Y(^G{a*9g^Y=|Zx7AL=M}6<H9@Tx<UOMi(Y(8_ZBWLzLR(Ym<
z9e+35Y6wK`yDQ_cZ{yshd&3`v?%maWYVRXM-hDd1&e;A_xw-eo!iT$3@+a+I5Fl(D
zzv99^aglZVLXwZ}U36>qezVV8?WWFHx>xXf-kwXjHg>FGuKWB&oAxdH9<sNx^_|^x
z^-257wTt%Mp1yML?PtDw;@A4^)%bIHkLWrb+rEo`_cEpx?EPfwZa+0*g{|l9O*Ua`
z&h6)){BYkj`_z5Wk#2T2xhrfhG49wC^-*=7c`N_^D?1wYl|K2hPv*j;eHYkI?yH+I
zaW4bE*gntA9s9yA%-L7LDZSU;vSn{gSp7cc15JC9qm%dQY%{Q%z4h^)<WA1L+xAc0
z+qixC-XxYu`<~DKv$yJX<o+GC7xyatP}{dSD0APY7Q1~jIwbZ!c%Z-U!;RLx6AhjB
z+0T~R=hN4>_tK^r`<(wK+g5bm*efI*V|%(XV((As-FweSwC~rl=Ghl^uF_`jx6plO
zzKZW#!h2xf?b*ru4&1-Fr#A56-VWbAdn!fd+3w0ev`^=q$3E7`rTZj8GVSwRw(nKf
zR@|qO_+t<6*S5V2^A7G)77Vc$OyS-q%&~XhkE%~LiDn!2x(B-2&${=(PGt8g+ln4(
z`x;$U+m1P>Z6;rl+V|&q=U!)>lzj}ocdb1<SoSlvIoqcAY3x;z-DR7ZuxIZ)$p~A)
z%eVGD*{-)QTzTT2*1hR_cQxAWb?9KU+b6YjufWWQ`;1K{+9joQ@AX`_$mX$6iOqSv
z^R|1Ae%)teG<9#tB0bxgDZP6+5~l1usM59fgZtIpBH01^co)~~6A5D9cdUTfUboA2
zZ^p0Lw$>FE_GgY8?U(vK&E`f%vCS#*ti9LU{O#K&<m~P2O5T^^nPr>J>9TJ|^1^+|
zo6Gm}8Eo3OW$LMYEPLbktz$CUTXuNi-c*Nq`+mirvgQ7{VsG{`*?sQDhxbZ6Qr>&)
z)O_37y(jjaGqvA4t+~$n-y(^9Go;q<{rt9l-yX4^eW4~gdy=1gu=Vz6-P?RwV*lai
zXZEJwp0eleVa2^?960tJdhpohqlm}8(zQbN2VC^*DwquS?qf)`mG~EGyJ3UZUV(6R
zTbafAdtZDJ*!S@E%Do~R;%!gsZ`?cgU8eoHSAX^{d-dGTc&EdD4skb|1=CvXq-V|F
zr+qeSU*MgW``pYH+e&XgzL#NZ+}`_hU+liWNX%|;LBrlPyH4!e{nTo&?;<I?bpiWr
zZQdQ<yTSQ_&6+(2?d0sv?q&MhwQo|$jJ-1Z#BDu3D(;iqU}zulwQnEa=EZhF+H-6l
zd<?Sw6gO?}j%iJMR|<0Py|S6X*5v#pn_1GgY(m`&_U_Abu{-;~V=w=d>3hX5OxpLT
z;OyRI*Gl#ozTL8~>SpBLV@_H7=4lAnzfnHB_nD5>K2O#Sb`ffdb_q{i_f-Z&?t7Nl
zvG>Bs7Mmw56Ko$W{<Y`U9Fx7eA9D8@)Yj~EyYIV~zv{x?f6Jw8PW%h7-C5ngcVlVh
zJ|>ChHcRE#@4K$QYG1`_uYD4WxNPm>o?3l6-oD>4;;8LykJ5eNhj#CsDAcfbP37Kw
zS3YaoPW4~9zw&t8KA)RB``Vd*?agPoymvu+scp-W&b_y+=kA+&+InxZR?oh~=fV39
zE#%&RP@-#}iE@$c(o6$e>H7xz7S$Zu*Ak<(tMk<5y#Z&#Y+mkty|-sx^xm~6D)vZT
ztk}0<v!reNviQBfr1kdd`tt3~5MHo1aZ}U2Ps!rD^A@}BU6NnCf8Fu0y$|i~*eqP`
zyZ^4b+Fo@V#(g@{I{Wgk`t9|LyR|ps-<-W6lP2xkzU=&79({TH^Z$+PR~>J%EjfGK
z_Q9Rm*5PkU_sNvc+<WoW&pqoa-tC>2veD-0zKq=m?;G!%Bg$o)?lIeX!~A?ZX`h8Q
zZ<H_XHI6vGw=DVSzJiQhd)Kpn-*?L9i}lCcvc1or1nymMpnd;|+K;=hobBHG;M0b^
zuhwtbXSb<+uTh=HKFQ$8`&g}P_HR=>XWRbmtWDD?mwg*51Z|W1ZTBuWZr^9_^T1|}
zo$cPHS!TAEHCF6>Yj<VO{C9h7-b%&W{V%z-_v!4l`?_*U_HSrfx%YYda$C9eo9yb`
z(`?PV*6%TYp1x1bop0X@Uh(}4cbC{qaIx5T-o<4f&(}G2E{nI>3eV2m|0{If{<MwV
zHhpoI_C^2Mx&QK%HFj?~rS~11I(uJ1mxKK|?lSu;S~snd&pqE89+kd-LCqqYCTGU|
z4T|l1r3>x%i%!<HJ+VV`e*@EYE2F#j_N30*y*J}6(>~$F$$KaA1nqX*x^&;My%~FB
zyT9yZh*h?AeZ;g+!}W!o*;2o~7X3l{w){!lx1MSGUXHw@d+)WS@4K<-oXw9LkN0@b
zs<1COxPAAw$m`Y>|6J_~m(JKXIZSe2v+?e||5c=HzDMk}x%=Sb-pm6J_sIRt*vEfu
z!@fK#E887wgl(oIWbbv2blrbZX4}563l#RTou9mShT@++O2?G##ByHk`EAj(?}6Q+
zy|<<;+;ir!@P4@$Z}#!ot=RjGFTpmacILiI)8E-#>b|sBqS0@!oAc7WxrbNmyRqAM
zUrbcv-n6w5dmEgkY~r76*~@Dsuve*!Z$E#}>Af81i}s}~-)$4|?7xjCue5z^gRfo8
znVNk}!6|!x7HrzL-v9F6%I$r7GgkHPotz}Jw}XqxcD|FGoo?=A+mhwq_imiM$!4yK
zi~TXB+I@u=-tBw1Hqrjr-<^9Kjq3La?c}suws?{4lNSQEchCOW>#E<ew=4FNjZSj7
z?KG49w%f{Q*cMDMw#j(9aBqP{jO`P-5L=ODdu+G8PTgDZcHVBexjuUtEMDwcB4=sa
z|9{8c3SQ2=Ee)&pCTQ)m7CO{t`yg%I-Us`-Y!19h-_6I{v3E<_CfjDV(|iAIWZavu
z@X($Kb?tj+#4X&jZpqfY9t<n?h8$SGhpTb#UY-|o_G;`Du-S3I%yz^2a+``JVm1x!
zzSc8zF52iYov^hyf5=9|S<rTN-^#ru(X(xyoZe*HyW{BIf@iDuMinvdJuB6<w}kod
zo)6KHdv{#mv}Mt&*(-2)x^={516%bIiF*qstlVRqc7Crzx#XT3d!qKTX1ncuwl#Qf
z!W0eb31w^cIxTM7d%)<P)dk<EJ(?<~_Xd2Jy~oNgcy9zlr0o%bskRFqT-(iH_R;3X
z+|#ztPI2zlur}ViRn2VgrrM;vPi7vootD?LH^bx5UK5rM+b2Aa_SDrG?ag@o(8g!w
zblV8)PMbeJOKcg=yV)=)&Dr~-ywG-$g6G~3b<=G#n5%8%;u-h4{Fu9^xp(8<M}b>y
zE2>0oH^j`^TT!*x_Jydd?Vr>0ZLf4}*yH!C$99+B4qHvm{d)ygsqDQWI?tBDsKYkm
z$&5WW)>hj-3u@iVA?La0iE*Co1Ll8wSVK4NJ+GLuH(+M3?T1arc1y@E*i+-uYx|<<
zj*Xsp-(Ee&Zd(DRt=3l*ckDSMGJ9{rWntSMa|_#w|1)h2YIAHZYUb}<vBlbU0?)F&
zD%I|`^H=Nay%D$8R$=Zjn;&1F?|BlUZ+mUhWLvSX2W?v#uI#b<dCWGZZ_eJDMKQKB
z*TwGLBYbXe2q&K{!~5-f*k<pw^~ztq_uR=zwqp5Adt;^?ux+@_Y;$GiY1<p>JhnD-
zHt*q|KV|Ql4K=p4=ML|+_{nDLRyld^%R;Ta7LShEp1t3^_rMH>Jr1gCck!wn+Pm|<
z{oeAEGxx@|?6hrorDCJ;#dnX)|Gj&R|4iH)z<X!U1eQ~_XWzT;-O$%zoh!3{Z^R1$
zTMonPHVn4)yQZp6-n*lH-=3_tPTP(tdbT^asO?R-)na|2(ASn}<`Ua4b`G}D=Qiz~
zanHlHM0VBQyt|FIJKi?$WyqgzrM8A^FGEwA<&J}_Hs5R8_HN;g+G`T2Vmrt3`<@p9
z`}f+_t+i#1ZrW?0cHD;hZrNV9O^kalIK|mM2vpgvaM|Cs=H!Mwj~b`ge)O8SH*}uX
zUWHID>xP|^Z2d&%>@Ar-VXyb;O?xveH{0HDIb+k)wqb8a^>Ley-j91kuCBAa#yihe
zV)a{_>%TqrhS?mq-E@5E-aBDy_w4hw-a9ilVsA>+bla!zw(hl9v&ObT_uuXrw-aoO
zvM$)htn0Mx_;b$Yiaww1iZ>T*7y`m~@hqOdx8z&2t<1G$dp-CM+0<Uo+FP)wWUoWr
zEUN_^&uu<@Y}j+WXxZMdH153&Q7twqv;XZ`^lPE*nRybn212p63L<tk990)=Do!Wb
zysBQXmq*OswzcBa-WRV8_fF6dwN(jgw%rlzY5R#|qV0Y!t-VV!yKHSvu-FP+TCrE-
z`J}zeOgGx<9y_>K%x=rxC1M-*l>Fkd4YZxMcT0)4ZPbFyy$qc*_B0$^Y!%_CxTj%N
z;+_zvvwIKt&)viIYR29z=X>@t+-2FTaxP_WY1pZ~D?UxP@oZhRm!Uy?`<mM=Hk;YI
z_ZEmG*|sj6viF_KQCp!S=k``z?$|rwmEE2zIjd~?>{r@WY&WvKu*ln1<7ScV1Foso
z5_Uzlk-tOs{;6Qv`!1w#uYiHR?Ex#5JraLU@2&6`vt6Rux#vL8;yrKQcJFn_Xy5B(
zzs&ZS<-WaovsUa~aV6E}iHzi4$+h$MdL&M@UD3F44_Dzt+c|9RwmZ%j?h%_^x|b*Q
z*q%kD>ur4=N7?@KX5Sm7w8i$ydp6q&(FZn8^NRL{m?Yb-xO{z&$7SEWJ7zQQ;XG5l
z*Q9Tr?UBSgd)6G<XWQ_j*7i^R)jd-RZtZzxJ<Ybpt7$L8!)w+rdyedt>gwH_n|W$)
zPxSV^8aD)OmR#RxW6&&X!;v&;Ps=Tty>c}Ydrx?+vNg*W-5a1e(Y7hC!Pb6%;$DW?
z$yUL9iF*$`y|GK+n~&{s^EG<~@(gSfth)Cu3%1>xwS0rEm|)M|d)^6qU*C<`+wiS@
z@0At<TZN=syMG+3v+;>x-@8L+wynm!xqEwrrrM<E9o%cKxqk13-wd`X3nlm7I1y)i
zPj!-QNL{wAjL~-68BWjlM7M0%YxAacufpFv8;(sUZQtxWW4l6`)wX4h(B4Z6CfQCo
zuw?JFuWM~}m|6Emtc|hN__K1ag~x)uEJxPcUOCijd!Xcn)fCBjHs6w&_vSrbZ@VC_
zZMUWE(!Edp9QGcVv~iDdZOGmW$**k=pO4=wG5L>8XGGWDT_-*FZd10}`(g93Jr7zg
zTKfp?vi-5+q)mwPDcfaz9eaQMZ?aXH&u#m}*2lIavu&@JZ{6Onx908*5h$~*SeIbC
zgu!(0k=GaZc;uGvJ#ePgrb=z4Z495LZ9&!}o93>SdmUt)ZJKPl_7)VD?9rbyfA5Ed
zOLj*YOtclalCtMR(6+q`c9>gFyOX?EL3-jIyQ1B;?8<ZZp73<G?eUJW{cXQ(uYr2A
zb<4#yHU~1->=ubvv^6Pew2jD$*b^Y8x%Ugt#yz|Ea`vtenYeewjwO2?pRU?#^P%77
z&;p6QJvuSAc9%r=-qCrt$Lev*UcCpq_gZl;*t>ymk4;9DzwMk;^0o`C8g@@8X0y!}
zvE6%M`wH7#`CWVSP9*Ksv+T81RGhPSRzTif2L{o-&dYT6Zo3$7vp~YcwnlM{&0X`>
zy-S{V*aRP`-D|LZfo-f9zulAS7Hj40hxeY%xV&eI*vh?g-URK<(6`-FA9`tT=NpcF
zZT~jdb_Y7|owtT#ufde{dk$Pu+GCm0Y1{F6h3ye3-o0uvTz1Y05;lKVi`%8lTfDa-
zRABGvAU@l&_Kv+-!pwX5<&$iVI)v|4m)5jZiaKiR*tBDBT=J^D_x5$}wNNy(ZR-o(
zZTa|wO~NnB-5-u<?&b5}x%Y*);6BeC)_e6b`S(6uTwp7D!Nzt*_C?z!(A^6CoO{$g
z_gEENWwtvox5+9^IAX767Rz2;$uHIskF9N&Np9P7<Ia}7Crqc?9uWI!<?r#(#$bEG
zp83=Jt(`c}?(UvlX`}Jv#GWl%5A7{Ec)?nXnQ`x*|3<b}ZPRuu7^d0OeNnPopipDG
z_``1-?^teIP8WfF584>_F0Wy=X}mSx#_YPwUIPt<eZ6V(_A<OXW#c;etIgXL6ZaO>
z@Y^j*R=3^%M$ES1sKj2ri9&mwS+?zM@LRp7<%-bWn{z5{i)KdcofCF&@80fg8^5=H
zw$J%$_e2=;?30P`+<V1m)84uTw!1wfWB2;Zj<Q|-OJtv&Ub(H-_AFaB*&BO~vG?w6
zdDUyX)LmlV{P#QeYFmZwl_<Tw*CO|)t=4&_ecKq?Y^*dCZN;7}vb8Cky@&1L3EN3K
z58JTHh}!Phns3XI?!9+yOq;F3g>c&_hGl!DF9z&g7w~%TjU~Q&7k!r7_eRylCP4Jy
z-Z?H^dy>!c?YnSV*LI0s%-;X|p4mK)5!@^6ePM6)<_mlO9ecXx)8|=xXFQSLcWNKI
z^>mT8-6_eN_pT_)*)wVO8r!7_`PTPje%YKUS+qw)U(W8Z538+V#OJ*euet5L5n5>L
zFgwrIvPOO1g;ia9BkCJ%E6S(X?z+IfkDuqpUXGAQHV=*(S<m6-vDH{FZfkxf-{zk9
zoV}}~JNNhti0u1Y+O@ZyZPDJ6JLP*rXMeG2Jo0EyA<vq<Q>4@OZZb@?*5LfHC;#cn
zy+`itv;Fq%n5|9VDccD*{_gIdSG-5bL)v!Pp%b<Ox5902<Z{|g*xI(YbSc}uyfUqQ
zEN_f#?|<UhyP1>I?rP$Wy;Gm0+kVL_u)SV5b8o^L%e{dN340IJ%I)^3Xx*E|p<%~3
zb<19fs{b}C{BGF<giNsENDkin>LrUU1H-ZH+?QEx#3uXiO*Wdk*MZ%B&xidDwiV3G
zds{xZ+1@>K(^lZp{N2w&-`d{Fon{lx_iXR8MG||T9O>A5Ks;#AFQMIgqYE$W`B%7h
z?~IVmdrZF6+dAl#+FTKGvK6>-anBpOJA38_<?nTkD%pF4qh*f*)2hA7Z613Mh1}n>
z@Z3!sp>53jBEDa>`H-k?%Q2hD)_wjV>y=AH?asNbv8~xCWIKWXgiYz=$<`r#%C-%a
zY}Wa~Yi$J#UH6`wJk{pT1JS(<*&UYObzE$CxhL*DFn@<NlhHM6pX&`aHHrSV2@&^q
ztJL=FVc<Ps`)$g`z1%fbJ5AS^?e$?eWE&xrv1g6JracMoukG2UJag||&;GqiqD?l2
z`y{Lrt}oqtpw?!0#r(OpZBMt@F1aMWk8{^y+Z*DsHf7>vwgtxz*iMpJy?6JWd3#Gj
z)oeGE{@<<j_n_^$U|HLSO>9<2`|NF(eYCI*+jVNMtkLSd<}!P1@}_HBuYT!l<9$nf
zANP)Td&}eZ?DbaOx!2|P?>(>Fe(wDfHgV4mc~P4sYm)b_XFX`UsbuHg+2<4Xu8k7d
zm&m(nw?%5*p7(!`TFnYLXnVTlt*vBZ+ukWN9QLq!v)hULNZM`{y|q`R|LdM_{q5F2
zW;EJzOc%0su`9HFBQ0e+FN<++Jh!kdgG{!y1K&xjLgfQ{^XjMU`NSk-mvFvw_qw`C
zw!Q)Cw#zIQ@BKGt^<I}O_T6Q%KlU=Jp0SPMZLnFNU%2<fOi|mVrQWt@><;gpD*0@0
z>oUE)6DH2vv+gP5zE!Ut?M;bo+H0{h*mlcb8{6Dj^K8Gw+}*P=_@S-DB-y=^D>vH8
zb$_=x9k$z6KS^kB;krqC9cHuc>(zU+M`Y3*o6wKfYz3y3*)s0_V5@AVZtG{fdQaw*
z)wTufKX-S%k=?gkdHUW53uoFW%~)d7ntI))Px+i}^dTPGT@RaWRW$$X=@T@!otAue
zkCe|W+xyi9wi!BaZ2hhnTD45;v)x-QW9OBy(I#un6kECK4Yn^FJ@+Pc)Y<B+Kf7D-
z)CcP(ci+7qmaf?2U9`{o_s=hT51T3Ni_<!_cmCxEHn(dsY~)TT?_06%ht1W?GxsXY
z{9|2mY>LgIy<2VX%!sp{a8tro<?pjSMui48o0xd_-rFN)E8VYTyZ70ty+`Y=?&Y|>
zWY2+D-g`9~rr9#~Y}q?U<lLUi9Ws00-T7-9{`|M?aYc*03#;z#*-^5{YOROQ-gohw
zdp}$<vDMQOw8`gsZLP61&i3!GMSBIfE?8fCRkv5-y4PM8<4D^-wfk+4@l4$tJ#qS;
zw6u3N;&!rj+dnSa%W&=LZUz&5D;qImyAz8XZKprbwaqro+B>Ocg6%%JNw&Nk8~1jI
z7wz?C$gnNl^=dD3wXRKHE%)9zJC5(Ya{B9D{i*BszWuAT_mf$*?U(r8y%Gx~_eyLJ
zvMsuDV(*3X%ywVq$6D&_+G+D(XS!{cNax;}j_G?JJ#*U2aA(Sn$M-MVUY2m&%eL*E
z&9o!`Y}F30wq0>0)@IK0FSfJxtgwB*M0f8^m)SN2>lEyIzwELp*?f5K4W205gqyQ%
z(p(?eByBxxyVG#>UIy9ud)F|g?sZ+U#pZ;y^<M7Y%X{A|x7y7q+rHOU(9)K}eb?TD
zk+${+?`iMr`eU|lrOdazEBs#UQ7D+PcdqQIy;4WE>^&r5y{~z#wjH~|Bb#j^6Zc+t
zA!hSya=QJSHGKQ_=6$r8(KLU*R_y$}xnHL4J1MDU7i1-|uliNt-rpOI_Ns>X@B5))
zx__x-<=#_<5A0(nckka}t-o*Q^Ame5MJMcS;TPV=k#D=t%=XgWfQdqTBp0*o-J6(h
ze}3_%{SPj@-TN+Qz5V0-6?<6^Fz$;M?zW%%H`Ipv47;^9W1sD{xmEV&?V9%8S%>%Q
zZTh`;`;AQ7b1_x>S}W)84wsy}Z;90Oy)RqY_R0P^x<}Wxdw=OauYGTl-t6&P&1{!a
zd2FwYoaR3M^@4j>vr6r~Be#3shsbAppZqei3jm!7mRq&w8B^1~=dDxrsy&>x_n~y-
zzN^dn_VRFb?)!AEanI90DLa0d3;X_aT;6;9(9^waoY(C{uWIaj()Q1edAqj#at_^n
zP80j~D6XHh??CF@eOH2K?K4|5Z;!-`<$E`s^WMATZTsGBDU0pcmHqcW5!<xSP+NE(
z+dY2!-Jf{&EoyVww`N_Dt!Dfuo1mbR_NjKS?X?AV?R95exKGNib+1W;;(?XBg!bo`
zGT6UQE8ZLa=7s%-)(2LcpgR1w`2NY#kM|1fKWn#qy4XISi!b+wxm)ksd|7L+c&yc4
z|HNkd2VRry<)2R6+cazWz5^<Z`=&%O?z`}Td*9^g-|e-F9QJ+Cdt-lbzn&fU*R%U_
zmTt7&m~nOQqe(CKR&M>hUt-xyn@T3>ee=EV@3m6S+dFmtCL2AQ)q8uRw(b9KYPv6D
z!PmVT9=O;(+I7lSknP*<%Gq^xYU?=c51MS*yS$svcD^9<zF#ZY?F+Z$+9wBj?UOtH
zd2h;^nthv2tlXPsv1va`&zybJo-!Z!z5Cd{z&E1%l)P2<9kE=sFDB^7o=ofZz1!+E
z4qSb&wKqVQe{bw$y}h<hdbSU;ChXDJGI8&kms|E;SoqFH%-eon=vJ0}A=g6o#-E;L
zw{gj58}k#r_Sv&T_7<;Mx%XpT>fYx9y7oC;zxM8!_R1zr<?a3pj4B5*m+I_g5O}aR
zr-Els2D|m1^Jg~iz1FyJpHbM)y(|{ydv)F(-0RIIwXf#LeEX9fhPD^OpYFGw$atVU
zwtKH0m+66ZOfPL5rv&Z$E&64jI;;GC!`N+mLzi;yGkht%Z-Hjb-cz$P_c_L<?oVOv
z*uz$(VRs`dZm;*{nfn%e{khv_=eGS~(F*&Xh&JrIq!e+$vFWyrZSaY`Oz%N^^``Cn
z$|z_%W3krWp8v(RT@kW-d9Fz9n`LXdulmNKy(<0zd%N!_>^u18hmFzt>-!IXTd;S@
z<LR~@d*<&wcY<Tz=QOK5x4m*~e;i%D$L5j3zMH#G?!T8Qw@=e5dEfp8zieAp+U;kT
z+q#$YqWHcTZ|%Lhex7>*rrPc7kaXMgzEIqD?bXD+=e28Xlw#NIZNJB{?`atS-kc1U
zeRXpd?$v5e*e50UZm;H~+53L#yx%)X!e`%Pt?0ckSgiNnjXAL2?lSlO=4;veE`?m!
z6P){PZ-SiK{zI!7_UUQ9vE$+D-)CmNc(2uEMY{`MjP_KyEwI^eN_nrmW&8eut%>`7
zUfj7)wBp$AR|gpOX9cJ1?S0d?FV@^<pX$jIdmV}<?7P{lysyA=*WR+d5qq>+(`>CD
z>Fnd&_Hu8deE1&2wr@5KMYVh0OIGdm6Q8|T`=!o4H~l^P5_Z__)%|y6@9C9a_uE}c
zvYKu#yw9?wb^jOUihVxwX76RI^tBIJX>aS`*u5`k@wC17uB7jK{IY6KC;!iVr}jUv
z?{tvZck*k~zKV=&>-W3u_T71Z(Pom0lHGwz&-X_i?%KP${^))U(Xaa+I33=5DyC!a
z$MP+EH{AKY_tq`By`M6)_V`Fm-52wu+=e;flU?)@aogz0GxsX%9@y)XX=*1g(`x^8
z6U*N3#R+?xSI*maLO*Y>{H`hcBsoNFXERkCII>J@-@8lS_grY2WIH8=b>AGB)q9+d
zhuO3HgzoKM_i?XolKj5)45#<b6IZoWJMq*`e<kz2+1gFJBmbY<+qdSF{pTI$_H1nr
z*=wrs#xCNbtew;UIeULi-)I-;YPWC4AK(3YLPz%;Icv6ev;F10vR;1nxBZm%9Z=*u
z@GzrqUy^#4T~*5x`{vG!y(tWi_HH_T`|j^3+G`o2V*jZ%YM-29{$Bs|RePVU^FGja
za_3&|gkF1Q)v&!2bqn`v1a7r^(k)^C@x9Dm2_><8C$#3+EIZM-m+#}XU2H#k_PXXb
z?K9k^z3*;+$o>x;EB0=GnPS~>NZ0<vqceN^rY+s$SRS)CIAPh|%v-$sbU!xRobsN&
zH|$>be!I<``yX%S+ou)ee4t^`3A>eFZ|!^iS$Utu3r72C5i9rl*WcS)H);0Xn7^O*
z`r7sEeRAs6UJsvtdwy&bv<ZFiac^86uYE~#+dj5ODf?C{8SM3S&fKe77h%8jy}zC2
z9?gAA&DZQ*`r)6o!1`zVjN>NT=Rd998)v<7U&m^vy<T2ht!4ME->d&*lWlKZ-X0;(
z=lg{9WcJOI-(n+YD6((j3XXkKwKZ)oK2_WMW_zT4hW4Vpr4L%{7rt6(J2lF8uWrg!
zyU8lg_bJwX*mL$1-#+DuyZ1>>XxPiXVfx+~WvVvkRl4?0F}Y@QT(HspWRJU@b^83h
zf%m5FP4QJda3to`zUy42_Fs1?IB;LswP)4@PWxxqf9>6@+Pb%=@}<p(m6!M2PPN`w
zEUL86=lKp>v87gf4W1pfIb9sJH}k@}y~UY3_wJa#c<)``6MMUtoZc(Sy~(C?@<!VS
z&m?z8+-2S~a~6kP&qYp~gpe0Dq4Jyebi^mycsx39+u&TYw<7QOo(IwGw*Ovj*t<cg
ze^1PFb~}&cGTRAkRyG>5E%vT747cUq{b3IWpQY_w^KUjw7kTe~=(^4}=yLsDGmdbZ
zsBMRBKa@G!PLc1lU2%SoZ4$>(Td8Bhb|sE{`w}jy?0J&WWy3dViY<%Z65EFQ6x##g
zMK+FgOnX>eIQC6en`)c0GtIWdgT;2k0oA>GR$SU+*12@A?6vMaV%s?FK1|uR_s`ai
zdwmNc_SWoc+`H#ryN$vmM%$Wc3vEB2TeNpcB%AGJ8KHd&y#~7je&z2u#6HP3B=7s4
z&->E%E(<wl>!g&q*Wd25%@L8-y_Z)_-YaH*U~g<@rmcX$+&yyy`fab6GTJhfi`%_;
zmTudjtg`RhkJEdfPm9`nYO2uQCCzJW8`78WU7eM%_rZpZ*2;fR?X{R{XUlNplchqM
z_nv7b$M#N<G_h?GSh?3mP-3sh94VVkYYy3Vb1B++H&yQ~I6ldC-ID3HH#ySwwrpjx
zz3nGys}UAsD<surE2q@H_l<?Ut=@}1+mLC!wr^7S>@H7Rymt?`nymx#MeC`XPVQae
zE^IrWS<}|Tb+xU5$iBUeG95N2BIeups4v<haz4iP!H#t6`zPi0*-P%;`=ou9txQyr
zZJl`c-oJ7O_AoIYwJk4~wmr7C)wb$$t8H8DDqGoN1>2Xtd^QZBr!56G%-s9<i;E5O
z@9sSfnkxGo(z^CendrN>;SS?&k>_GIMYr$meeuTFM#OC2o)tVPwjUTK+NN>W*~*1%
zvB_GtcJJRm6Zguj*4caM>x#W!8zlC<^l09D@9mDgOSZ1tGpS|UUXzKsw$u7MY*m|f
z+o)6@vb|;{ZTCX-<F0ioynEGITK7mDkKFrY%Vyi9CpGuPbS>MP(PCgTciCfG=@nD0
z%`z_RG1%j3dwBiDy%!Bt>|~O%Z0}9mus1;U+8%p{&3l);U9-2MKw^(tK=YnGI%T#l
zQC54qXWQ>pQjppku*2PE!~Ehs^-~|(<_8w-tvtoNuV`BL-jypD?e1jR+cGfDw0eGj
zne7cZ)je~xKJ4+meQ2-XpZdKgKCiZ6u)n?AVftg68+JGMxFnnH`PMnf*32c<Cc>{_
zug9Owdl&;|+OC|Gu~+84xvc?fg6*8WO16%Cm3#lr%HDhD<RaS!zU$V@)||B!;8C#c
z_tM#W-u9KP!#dSH=TjW_*4Vz-&0D~=uRbh#?}Vm>yRR%}+MA*I&UV2Y7n`#Mr)&(@
zeBL|r{>;4$n*?_XU0|_MaPP2f$THad?d`NZ>$Zs6ZFq9Q=7s-Fo8&uMdxid=+<jze
z?cRzOcbf?>_-s_pPT4zmUc}xAg`T}PIQQ7Z@(1qy*>AQ-Ldk4z+%)#RQma_@7ChN*
zbBRHD@5EU%_Xg`Q?-OyKw|58IU7O0)bN9adSiHOK<zCx)FZSEMd~|$Ip5n~CTpp8c
zO~mKgZr>=l@28xpt;(DAHZ0i_ZA+Xs+PpLtwmbQB-`*pux9!m>s<su_!fy9TL1OR3
zi7k7Ve4k{y>)#pMmXZkDJ@5Ez6KY@X;S^=GJz$(|TjJYmyT&fw_VBEUw#nyq@114F
zvhTp>^|l{98SJ{V`0NB8JhG8G)o;7OZrR?)O^^0w`xWn1TYhY>(?X8Df6C|D9$P22
zcR|7aJt22??S1X{bnnMImbNTTQ*CEGoMsyw|6$Ktft|J;5wd%K{F<~k<z14kPTdvT
z!bP)fWt7|Z9H@U`b7t-)TQ2JzdoRS=?b)BU%X-JnR@=p92lp2EmhCYx`oG5{;O(Ay
z!i#L*_{^|9ctz1}aig%EmaoV@Cyir!%dWZaZ8-gHce}gjz87^lHU&9{tey7$+q1)J
zfz6ak9@_$cJDXK2HrX6W+-jS@uFp2XOKZ2q+n0OqMi}g^5twHCz^!G^L8-#MhiaGa
z{nc>6W{SG6?U(g;_Ixn=wkPC(uPxJw4qJhhXZ9p~K4v><-BjDB@0|AXO|jam6U}Hh
zz2WlSD|0{Wd9d!c4TElp_1;Si`{s0Pwq3JD*EYo9jBS&J+};Z!hxhhRQnsyowZKO5
zmDs+MS~K>_oZhqNL~5X|WKrnej?l|{0^GCrRK#iRo%cgzuSBNI9tFeRJ;r-K+w_WX
z?Dgi}wRgz^HCvH?OKheT&))NcPs&DxKi;-3as6Ho+vj_(JlnW;frO0p(|4A8R~ZUg
zS3Z%mlek@HGtGMH-nvDXYy>Wt?OFPT#qL**=U#?wTze<P1ln|{PquZ5Tep|(U(epl
zU9o%FPIK=yu*<S7I{JI>15x(f(~ab97AV)+E-1dT`?!PJ-g$!4tV@2f>|OM&esAeL
z>%A2L9yVfWH}+Pqez9lzvhuy0ET?Shdmq}GxJ%exI=p7@Hp4BpPi`yQ-nr6kyY`^G
z?XH~pwtJ#C>}@+b#rBirBHMX9y|&Lj?zhd_eQS>e|9P8CL7TlQGO~LWCPiC+lzVBb
zW!!EX6U(*N<=Uq`7v?|O^UQUj?K3t-yF0VAZ5=)wv)#FSr>&-Ky=_?X5!;5zlWhg~
z>TJ0WZrS_OE_d(O_|<!lKDOL@t$3epf2s0535#boKK3cL5z9C2l~^RT$Drz{?VGZ@
zdz894ZCym#Y(G`+-g_);(OwxB4ck86$+lKDTy}3ZEw$}?w9ht~@!YOouUl<vo=x2|
z*Y&3j$H_*U2ySCruY(6{o!L(AWm8+TcizI^)=sV6w%(Gf_db}AZ@Z91e{bTW3wzT)
zDDR#2Dr?_44*va<k{0isyLpoB5=*zeVf#28HpI8suJUZ!x6o$kUa6bawk+zJdv8Ya
z?(6bR-D{J(Z?C;9^S;BvT{i1}c<!~_=W6@6%*THIEB1Zm59932mPXq}c%0npYkt7K
zXJ6NWK=sLch1;+0J=-?dK4#m_z0Hn7`{&!1@7cuGXm8Op%YOCwiTnPV-`=}o67z0J
zg-d%_)9={ipPaNOE@1NB*-hDd4_w~9fAym6`={7T?5mKPYCr38qy0jEj{V0b-?FoC
zh}nB&tKhyvF5mYux@qlCl1SgfzUlG)1>3LgVY1q8e_B!A#;5Yq-m+=4_C0lr*|YhF
z$UbI~TYHthwCuMw?%Q|l<Mh3ca;NTJ878{VW#7`hGtcwwt#iJ+_q*BVy}9>v_6B|1
zW^Xiq%U<Sk*}Y0UKWx+;AKE<O_u6}~o_pWR`UiWHFNf{>EU&juWsdSbhw`2Kx<oi^
zT0gYz{j|ShuR^EP-V;Il?U?>1*(X0O+$R^<xp!|@!oF1#*!Ia>S+Q^H-Ie=4*?ru5
zBb{mAHxDrzmCR3eF@1^qba!mum)<9^Z`Qxp`!;oP?X#JdV;i@najyc$l)WzV4eUFT
zr`WS@=(Xvc*tYLz-i5t(!Ta|mIw$P6eRqEULS@E%XH9+fUufXk_fE%h@2mbcyBdk7
zd+%Bwu-liuV9%kqSN2!=&)*lT&a>~wgl&7*2CTBZIYrmr<<9bbmNh5$-LtLT^W`J^
zz6sZ-*&KX$#6CO3eV^zXKl>Zk?(bhv=dthpvbMeVe1-RlC~4ZXey-dbeok%A9$&sa
z8nblw{95yL?>)WT-F&9ktZ#qZxi>??eji(<^}d*|*86_!|6&uc+F-B#nxF%>jF;F4
zYOUH^t<1MiJ5OX^U;^|0TF-R5cYS;JE%AJ|M_+ErKJG>HY^OMF+ONNP(Y`my0tXm&
z&)#?U-le_i7Hs>h3(oGn^VZ&G)*^3PcJF@si~Lvj-uX9a&u8BDy+Nr9Z9Y1R?DOif
z->Y;o$5!Rfq`hU|YxXIe_uYG!<Ne-_93i{J2h95fXDqgVCL^^^xl?}Mx*K=*t}wV~
zcW71j-h<Ki_h$e1-EZTfZ_im7zt^KsV&A3wwKne)a`)PvQP`I(nP7J*;kC^***AL=
zmvHTKIc;O>VeoICiSe7gSrV)FUzGN<*HG}VE@RuY?|v`qKCMaH_R0KMY$xq6c;LyJ
z9;<GlsC}|g7xo5B?B1&}BW~Z8drtclW<}c0_WZQhNi@NxaKqYtKmHfl9{Rq)Zpjtn
zed>&&``#qpwQAQ3vbph9Yu~N6oclNZU9m5JYS(`G9q;$5FLm6zb?4DN98=l%ET7M^
z@A%R1J+iuc_R9VJXcN{ev^!9CqdnX8#e4O5*V!C9%D>NVN6+qQDYN!+2wvQGp6SP4
z#|sJjRw#eA)qQek?->==eV-R+?mr)_bD(0A!#;~P)_vjCW_wLIRrb}+P}*n5Hq9m@
z_un4T&DHz(p55E~L^@|L<J}2+58jF0n_%+HCQL}fR{D|UzV#gY_68rBy>F%7nSD|T
z^L8m5<liYf;rw3fIw9NU_ic9hg|Yh%e4V!ErqZ&#{UuHNbS?_)S@Yh>zSOCI->Gk#
zY?7Z$*}JP<W#5jY;(KS;pWfrSI%MB3n>>5AOE&vzE^gk-8eqOp_EzJbuVUY==daD%
zXKtTq`}ha%KEuG@do5B!_L^-k+$*o4vp0N|*gjiPRvQMd%X{UtHt((4$+7oEjOpGp
z7Ss1$Kf}D&Ky=MM2Ep{*Yd2or<5n-=aP=Vn9?|o<`_?Wuv(J;=YO`%g)n4;sbM~9n
z7w_f%!M-m@K*;vo7WRD$yqc{NuWa16&(L?@()Wz}UjEFttG+6<kIi`XKF>eQ`%;%o
z+*kbb)xMO;bN5CiZQ3spwbwpn!S}rySrhhjs21$~E$h3-w6T59q#g3MhZh~&QySW`
zH{STXJ#&Do?F0TCo7KYC_tttl+Uv1gv5hy#w>`OL>t4S5{rfWGAMWkna(=J6waDHB
zUr+7pek`?b=ON~OGk<Qemu}m(H>`N!9=p8E{RT!N`?R;t*jwY)zW1)tWE<Y~!M2_!
zrS^$0PT0#Ywrg*0r2M|^a+&t+S55YP*fU}8grD2@m)9(`pI78#d+F{Q8&65beOgS%
z_vzm2-OKfrW8dX_=6mIrN9?<|Tx8##rp&!dR(RW`rqt}4@a@e$$>V?Qy>2YomtwqV
zALrBjy%iw~_Xi0o+V_ZTvr%Vb+IMW#!F@-!-`SfX7;Rs0>a>lC<fQ!vI->Xfl%8+5
z_`$`!CtkSi<=VPv@BKtC+mr@@y;qCQ?Y+FCb>FJ0w|g7T^xHk|`my(Zzwo|Qf20po
zF(0r#+j-W8`Mb?t|DO`J_T@kJ)|fEZPJ1(L?~Jyj{e~PI2VQwh+<WE7l>Hp`6Zdmy
z-`*oQPsMiiMX~*Csv=gq7PIVqtfsY3j$^;ITjYd2ew*d?acvOUmoQyvFZ((vTbGMX
z_Lq(>+_&qY{=V)-BKzJy?zHAp@!TgVxzsN8g6zKU8Djew-@V-1q03=kF8F7!x0&xA
z#~Wq#67hZeE@zzCYjDbXPv8%;y~b~!+Q!~%-<MOyVJo(6(cYdT_WQn0mfL5N$hNOs
z{M7E1>$clhe0#U&#I)o4gbLO6YHnxRx8dh*yC}iLeM{#o-Xm1VyYGho;=N&VO?!@H
z9^L!buzzp-qJHbIhOGPkJQlR?t(>)A)93D9&Bx_?Vl!0jx8!WtXMg;SUE>aRdw=_?
zy_2OT+D|cixkt2W*<R%tGxz@9TfBGe|4Do6r-tsm`o4AVo)5EZR?X<y>lV|vmtpR^
z-FNo<*rVZg$0ptOn(YUZAA8M>b!~6B9ko^avuUpc>k6AW_cd+hukYSFVdF;|yZbwA
zW8ztCAOC3C>vD7Qo`RkOdl@D^-}OLSeDA7VYxb&Zwc7OEF|*Bmad>a9<K#U|i@f$0
z?l`koCHl^u_x8JXUlVGx-My>P_Cf4J>rjORn}3G0_BJH@*|?Pb-=lVGt*u#T&))kB
zb#2qlYWAL(A-b2rkjLg_VYRL2oVm8!4s6}qH??uE>fbjumP*=t3%+i*v03S2tID=x
zuS4o?>l-T0wgQV>ZPVoU@7<8T(f06?vfUCgoO@)&Qf$m>&e@(hXlwgo)8;)3x)p3s
ztzEu1OzNYp%%ObS1J~Z|PC7qt?+>?3Tb>U$_Y}y5*hsMKwdqUUXUiABZP$2Pz*b_*
zADeHx6!tE;R%g2*q-*yLKS`TE1t<0dB)zg;Fj3I<Uhgd12f?v>8j2-%?YUIFC+mTT
z-Ti~@dySv(w!OFL@?N&nb8Kt=y6iQ$rfjP*JARMMu}s_WtWP%XHxl<6O_^jn$LZLf
z`P0nyaxCrJ>!sgd`)c2(Jr`@w?+rPfV(TEBX1#<f*H&}Jq}`jJOxz<MAZ$0`Hj9mq
zRHIFe4%^<sZyRj*g3s(tx;M>s>B0WJk*WOF0m<!q7)rvdB{W?3icQ&Ut8BH*c3;mb
z+pCHNwgolTdt`)`?VTIfyLY>c-`<ohzxKXhE4I1vY?E#Mz5{#jz3AQR;nB92XM)P!
z2bxYc3an1nu^P;_s~<G){q=HzZB`}U-U_1$du~7cZ=3T>bZ<!KYMVVfeQjB~7TYjt
z9opN|FKc`1<Q3Zwsu?yD<{R#H<8RwD@yw*Xg%@k~+PJCPu9s}sdm!wImF$_rd$%Z^
zw7tkuu-D+9k?pO9rM86yr}o||Te~+Q%zf{M+JAc_+TPkUuqj&!O_*f6PvFpA7Tw2t
z7?M_4b-g&ecSBBv?c%<9d){Rpx9#CgvF+Qi&30pImaX5<y|$V{m3tW!cJ8`myk@V#
z1j{`KzX<K)(9N^G@<ZQt(TAP3#-e7nb$^c7&a_P2+mYpAd-1sIUV-^sdp9UZ?M-+P
zV=LC@xaYNg!=8JLzpWIsg=|+gH13^g7Q0vH3d`O%|M_i~g`C{meOYa9MEYZ!OLwQ+
z?zy>ZkHES8)+-O3+dFGpudR~xUfY8{{B}C0<MwV`tYN#MZpoeoa~-P{b!~eS{IBep
zG54{J)*0Wua(AZM<^*@`ou{#LZzD&Q?GLeoHcNVU?RlAbZjXeUx=q7mY1@{SXKW%}
z_U%pBRcl@JZJup|h>w-`lB0Wjtak0?(VT2^W5Y8WAL|m^)qatCdrp<x#+6I%-J=_6
ztH5z-x2xxETb((wd+YK|Y(For-CH7Ty!X%VdwVQ8diGxMGTF1?!hzlOY&>?EoWXlN
z=1J^*{bS?a6(u)qf0o>_apOH>Td?qy^$e2-HtBu!d$+i%**ayNvu*h_#r8y(<(|a>
z>-SEXdv(tb;oiN=e>Crnys&hS@`0SaG5Z;852XLuqjPxG9)@=w)_0`0*#1ymyVvWu
zvh9ZtYPMXlr)>9?ueYt{wBE~5nQ3eE#CY$!o|k*r`d96}w)oSY+deOBoSge@C8Akv
zkA(8u3Y&eg-5}9x^S=JEtwZKn>$&fz?fvqCX|MHZe!EnrtG1g;diT!nwzakJJ7VL?
z6twrsQx4mjCL!BPmJ|2RiDR-YnR(LI@60?~mP;#b-9(%B&Uk3F_dsdg?k!b2_iSOm
zZM(zhl8xC0mVGyTME11A9^TtpddhZIDvRyMn9aL)bbPjMdG>6t6SMqYh2RU;DFUIk
zi{JF@ZF(}%_SXC3du91-ZF`oo+Lp*P+LX0!+sm~>e{TxU#ywkRx7uc}xMg+NefOS1
zp`5+z92VNH+QGk9=IL46Z5Qv_82e=Hb>Dbw?_;MUwj8(j?}_SC+oO1ib>G3y(`-Zz
zEVMne`H$_tn0>bHPHSwpEGXH_ki@k&MMG-u^p>T2tr)NFoue*b_kZt?y%FC}?MXY)
zy>|uU!oBTP%6s2k5Z*Uy{{FoY!7ps4&E0AHY--zH@2vrQ_iWf=Ybfw*Pgzx$t-OuK
z-Ud~Xef;w_*y_A#+VjDl)pq&54|`Um#n>!1yu0V(v<rKuH*)M{ZFkyx<)e+Q(^fIN
z$M4c@7pxH3E7*8&&;2Fe_gZ|ovZqyMrEOh7l=b4)=)L`wC-&B5FW9r<y_?Na?PA*n
zCo}eNl&!OkWnHkh`P8w!RXa}Eq{y$a{T8;-Heu2}n-Blq?%j4HVy_g(fxQjom3t<i
zIJZ~BUBvb`^X$DbOWO9VnA2l>aN}p2J(d-=4G(ATYOFiC*Xq3OUXkAtdqYf5?$uFS
zvX@~s?=Jhz7i^QBoUxT-EZjTcg_Ui`XJ^}n2b=c(3*EK1>u;~Ef~<h;<Ef@L|G8%G
zO<r?&Z)?8#UaMV4_R7wQvt8$-z4wgJ0^1$@?za6a&g`8RB)eB*$GW{ej9q(nWaZdw
zahKlb5qo>D$Je8KK9p+j-EO3@H|3aytx0Il-u~l<_UPP=u>HH6$M)N9C)+&@R(nDw
z?Ae<$_0t}{m2Yg8&F<ZMUC(0gjn`89v}{h=?ywKHy|=(=FNac(Et_=Uo}6lX+mI6q
zwk>SuZGGbB?QvD)*~fc4*``1^&DLhm?L85f>}^AyKi{h`YlDqUe%GFV7fbi9Zk@V!
z6Z`$WkDjgE+cB$gFGE_9t;2kYeHod$wg<#~_VUDw*k1Ravqya<tL@F>mbPuvzu5?E
z(6#vxeZ{6HHPyEGQu7{%qn~!Yc;LJD(L3RNr9o5ol%A{E$I{=pU!ZKNb>S_cy%p>;
z_gOYx+yDJk`(B$ph5MeY<=eOEciY}vsY81&1S#(0cp7C}vDbRvnwfKK4txr+t^6yy
zPvN19?dHl|_AFl}?M~kO$j<e_zWx7wp6{7!Rcmi;AameX+A7<rH@WuaRrc5`_4e-D
z|1oC2o(${Wigge7oAN%j3!OM;-|f`pd#`MYu(f{Rw(s)N<2HfQ%JymozTC^Wi)G&`
z+57vSJ{3E_H07W5*K{t2F4j<clQWF_9~36-PcjbO^CwK&c4FAwz0ro>_rI9EW-mwG
z+WlX<uI#m5A7H=4dydVIN~3+=DO2|)IBwdrS(tgBrQU=+>|0OltJY56_u*;j-d<DT
z{l~<2+Uiv6?|a{|Vz2T|xqaL&3-{@ZDp+mTnr`3jG{JVw;Wf6#S5oaPWnS&Q9I$Th
zE%U#76_wp>99!M?g{`dLyC`<wUhYkO`%bP?+;_&Pd(T22p}jAP-r9sOceXDP+p;g`
zdDXrnWwUJsjpy$ZiPyKu?A^L=gPHe!-R|DK72on~T5W{(&3m@dPV6^>1MkP!eG8uE
z?fZ0P^8Sa{+H8LO?6U27)wlQ1*?D^(ol4oyz}{^4J>u3LN%^9Eb8HUoo!z={&$<rX
z{qH*E56o&gYZE+i>3-Wu7xxAfJlieD@@xOIJ#BkU6s_$nj(pqm$DZ#%MDpi-*{+hd
z{S_;0f;89dU$a?dZ@GQLz7W?F`&PA@?p=F)*4_i}-L2&;#q5qublIn^x!C@*_JjSA
zE7bPw^G>(zzQMWYYfghr;7nJ$2Ca14DXLbs;Ub>40rhr!4}|mV)i7MLS4r*pKA|5$
z`*feo-D{M&YyScR<Gph-m+oowEjyqrq3-a`?bO~kk0<VVaB|0<3txNp|8v>7@A`tq
zeV-rn>~($LwQrqt_@3g$clSqe-`=MrZMOel@AZ8GNnLwy&bqMIo%`rMK|YtgQzu;B
z`?{RPfnnR8J+9lf?)6!*#OD2p<9jD1HttmtIkESW`2RiKa}VrQFE!lvSO3x84;_1L
zvcLG+<uI7oHcB10uS(?Fw{4roKD{PoTj5>P?d}{Dw0+>WXm8-oy8RO$l-rvqFR~4Y
z@3blG`?Z%nG<6TN>&?B-_Ilf$kW#eGeSO6y%9Lwgd1k+DlCt)JmD_ap_8TACKj+^n
z`yWQ?`*f%8v6ub4YA<JV*1kqIIeRYc`~v|A->jcq&D&R8#Ix^j$H%=k>pl1F*=)a0
zfbYPb;*gztV}Ek)dwqQSzWAV7do3&0@BJWUx7X3Oc^^~hguUmd)a{*FaBpwB>cst(
z+1vN=IbYwek*#REbpPhP*Yqp)hHf}-W8wdFuffM-Hd2+Ews(Bj+ZbGHw5^l6v0uHE
zXP*OaiS37;cYEY=&fC8JCc8KJ1mpgWM|#$4j+gG+s#Rpy9(;9=7hA%<%s1Eef0Z>l
zpkVcApXQvveW!1Iw^=9Kx>uiL&0dZTv9=3D*6yj1Jhqo(4gcO-9CK_L9j*3W^Y~+<
z_I~}|3jMRT29I_2Is^sno#_+4Pb}s3zIXfHS>O1&#&*^hQL7bgHTxESld${T*1E5u
zrr+lF_APr?c8KqrvoUe++g4_WiJI~IUe4j(r*wGVUf&hhto<7H?)~|7!QO%^GxlEp
z#pEz6u))S@qr^V$+N6D-4?nl9J@{d7&CQMbl#^fW{bYXB*7?HZy`2u`d->F#?0x-}
zd*8xG3-@~F-`Z=P*S$BOa*6eXSqp5}E>hiR)i1wKY7Ns~-D?i}-kiC<cgc&1d*l7h
z9Qykjb~7<5?AyA0$-X?3Z#La`d-vrD-rc`&Vv6;}7i;!?_<3OOo)0(n>NB3%`)~F7
z{k^6C_We*PwOLTF=5SfM-sbGQgL|*M@7=R<p74IA;^zJF?>hH({ZO@g7nEc7(BziQ
zbj!JWFTYUTr+?gUZ$!DKjohKwy}$U^+0Ea5X3wS*0uHLDckOLm%xmkJJi}J^&;fh5
zT!;Ni$CLM76N}s1bzs@P^W}GKwp^aSH!N}UKE+S1`z9tb?Q^X8Wc|n`;y}ZCbsOgs
zYJ0C}+wFfnY380YUJGsN%lY>;_=Vd(W$Cx+IP11Q%72ypkuU3QQ&f2OsqJd8fB*90
z-oL9`_7)%8zF&q>+&=cwrM&|G_U;$?GimRs#fp12O;Ovu%HhJE<BuzB+W+p}|E_fU
z-Z^z_`!2tWw?BO0=U$y~_5+s8fevS9+U|Fku-VsS@p!M+v+lhClV{p%iLc(XOSgIV
zhbRB`y_k64=Fg>D_OmxGwvLQ1*ne;P)qSr7Gxqf?zp|&q^@vT!-rakjm95<CahY%L
zw*4lyQ?gI*tF0EbRnsrDUl6!^ug<L{d#ApwI}l>F)b@uL``%DNu6?}L6*l)G%=Q`1
z%i43Hc<vsBqS*Z^#~$tPv{u;bYTUiQ>FmG#oAoO9J()RqueJF6{h!W0-zD(kwzasV
z^1h1yjW)`ng8S0GF5WBcaLtCt!q)cs39h}Z<q-}SV)FJ`1!V8#5G&jhvd+ME*8`*d
z+j$o5U%bL<-vy1NeGAk0_l7)(u)WJ9xQ}<qZX4l6Y_=(r`SyLgY;4<gZ~b0(M%KMw
zHaPB0NME+^_`$S2yZHC)QLT{NS7E(wufc~|)+zaicE<Jow%42f&1U)&g99lK`)$Nj
zoA#boXWRET_|kr3fr@<`>Rx+yTFC92eOBA1I^)LPw%mrjO|gsi1gG*^SM4s^ce?iS
zew`OBdspq-V3WJMeQ%nD*FM8lclI7W(`|oBLfh7YIndtdX7gUn-(ve_wtd=b7jx3)
zfu{66%a7N0UwEmo_e1=t-Ho1dHeVO}>{Ynlv`<@ei~WIw6MMejo@i}%EP3BV8*RId
zQNQ+<WOnYesF}6bNnzpMQ{|iX9b99y&)jOs-ZG>3ecjC>cKsL9_UfeV-5clXu;+T;
zoc+m8$$KO_c=so|Ufj2(hSk>e|CIf+Lap{O_|)v2dUopGoNH6<HtMh3x4%x%u5I`9
zy@9{p*e4X~@4q9Gyl+|G#JxJk`}aOkf3er_S<${nc^mic$vn9CSAXu_JG>6||NrdW
zSJ1g_x530^`&&2t-M4e)x_$A5ayIYd18hFePqr~md%yR@n*;VeE8O=p&-UH#y=&9n
zMT+A4;`cAvD|1Z9Hs<+`eg3C~c7NEwZX4Wpcke07Z~MGw>+XyE=4Kt0<hK8Pk=*W#
ze4)KM3Q>ElS#|e4IJ$4&Bi97mO8?9IMKhP|VR!tww_YiJ-@lj7_Eu)F?d>qrvz_pY
zaj*L4UHhJOdF**ryJc@g@q>L06D4e4vuW5W?%cn(KK-|ifQ0=1+3t7kUpx)jm$Qg_
z?~dLT`@TqI?O(V3%D%wUz55$v{#zfFxwJQD>!LjuYa?w0f^+vr=jrZCH}AAo{={T2
zvQWUTgYVZ~E{SH_>Cde8T6+oZTQO07uewR!-nEMy_Eq&w-&+~`b)TxJ^gipY;ro|u
zy<($OH+?_z&G&l+JX&r3u9&?)>ix&P*F`Pt56(QWcll-}`+lo)`?egIu)B64`#w<(
zYkR>Kwf&XMYxdoDZ{D{iVa=XTXZiPf%$c~SRK?a_v%YEXtJc-_l9%%Ki6y%3|8~{a
z*8K>hO|9*sy&Wxb2UI&Z?Y+46@ZMKnT=u4E*Y348VcXX+d-vXSnVoj;3#<2CUdgsE
z`*D-KNKfzHmdqV{d*W{ITje`zpYOKTy_^=Jdo@awZJk#n?2FsjV=rYtc^~`V=DiHu
zGxoI<Uf+A%H`G2)?a03O7sc)GrGMS0)cAX^)Xv0xmrHy0dH1>RP4Nk}O53#5Zl9mj
zUa_oNYo@a8dyVDJ+QctBzo%e#)1IxD{PsF1@7${qw0GZrC*^(nwk+Md=j_|PlM40r
zh&+<nCorjgugS)~y%O7E_A>upVAsJ@W%F~^*}azB;RpU7Sh1f~?&aS7#R;}ol#lFP
znIX8(@zAQ>M$xSMFFd}x*DB(<jmXWPd#*0kvOTs~)&8eQ&|ZEMr33Z~BKEw^I{PxS
z*X})$vTLttmC8Qh=xzHpvV7U+pg!AX@1mBy59gTgW4l#to${-2@4}7`wg)OSZA8Cq
zv`uB+zW4IS-TOSB-r2L_q~?KjVc+&nc-OYi#N_RM18Ym$+NT}1Px$KW_pIK#?~A$S
zK0D36y;as7dmZM7+nQZ%+k47<{@$k5A$z|G?XYTM=CJiDG~autaK62*y!PJZX3uOj
zjP}{OY1HhE<FdDz)3j#qk9BH$*{AaCtMuQ!zj$`f9@Wj=`(&1J?_VJrZ@=jELmQ1e
zrF|xwj@r&>duy}#B=4SppS3oB{>0fT>^`zP`^yZQ8}plOW|*?tL~>c})r$VMcdlZ@
zUT>{+dma5&+Ws!sv(IY()x9D%)Axn!K5g42E3+@oeAE6H3vcdSrvG_wcITPBHFmeH
zA3VIfk9XVEz4tx1?9J=e>??lYx_7~f7q+$^HTF#|e73js4!`Y6+n9Zy5?AlH$zj+Z
z_wd}_cPn1)eIswOk5}x#zMVxI_SI(b?)#APeBa5nwR_WxOZK#8Zr(e^PQ&*1(K~x<
z4`}W?k>a!W_rk|}8LGUk)wloN+Zw{W@AHpyd)oI*+qW{5b6<IP>^{Zc|M$vi|Jak-
z#kOxoh~)lSNrQc#gLm)CFAm>7xruFW*wxm3@h_%ZnQgniZ{n$jy<7fD+SX)8*`<eZ
z+nVSs*gHGo;hx@#nfvy?65i*ile14~Dyx;{kBNK!IqtMA-+a~XXyv!PFH%Kq*V>Ei
zy{R&N?-RN3y%(;V*k1m2Xy0yA=6zAd9QFYp6KyM#*V^1E`?q)L1kL?h-iYr1mOgWD
zhRlq8!dE=@dEJPzj^4L=-yZSxww)(aY+XJI?Mt2*ypQQllwGRkc55?Vhkg6^l<Ze|
zd0<ahLeJhuJNEB+;p%0Z!MteC6wR%687o@%*DQQvt<BZGH`RZp{j)oZY|>>V_deQJ
zzrWIRu6?@fv%SuJm-p{@oV71~0rUP7mXG(^U)`|xt9SOkIb}inw`-X1^<K4TkE8Oo
z{pmcC`_ldQ+yAV#urJ+u&+hhqw|#}|{(F<J&D$^Gy3Stg(1bnL_*U-Sn9;nCDN<=K
z<NJzzg>tO>W=-_p&;Q3{f8=4g{Wg=7ZTH*y?R{uJZSV0_SN4{qe%_<ER(h|<m$rR6
zOwoIucl_Ds@c#JT8YMg1I=?x48^xyYt(-Z3&zh8(HXEdb_wp1N?t2|S)4F-(tG#<R
zZ{NQyv~6GJecQd>zmxa--fP?!w6$rU1CQyxl=+hO<*%pNyt^j4@5WkBTT8oc+x5F<
z?bT*<*f&Au&R&gsF1Ej)Zrrn5Y~B9ASGoHHY(@66^)J{x=l{gLe>k4n9iEzEe@Q8A
z--NF9dt;Pj_x4ze?{BDPxBtI{d7nt$f&Hu!5&Mb+1NWA`T)MaA+=9Jz<}UlHHeKDP
z@lRl1)0ryUcXoYy-QsQc&Y#a}o8CEnul(oFcBZj%`|^$19Cic?+bo}CXdAZhk?mcT
zEp{G)_xCa{<=gl7M#jFtOAJ;)4NSHi=Jhs5wMA`@A6dQktp2inS6KAygpWk*-SDY!
z?-#u-`xjnI+NZx^$Nt+Q7x!BDEw#G;>GQt*#?jUu@g;lzeEYlCKVkBo#qP8AZa(&I
z4~Ne8Jtso;?e*HixzE3rVc!?s_4`a!?CtI!oxFF>t$F**dGq!yu(`MYW#yqg52WAk
zWl3MRSN*%`zJ9ia`}$&9_U>CYW3T1qS^Ju8x%a6_HrRHyPTad>O4FXFd~Wt0(id#b
zvYpv?qN#5mr`w#pzh^w!yLL&f{f?*wwjNFk_S`#Cu&@8|ynT1X#O)YX6zwc<`m#5)
zVz!<2t&)8o43F*JD&D>KT%zp0xq+hl1d_Jew)k@FGqsDh{l<LArgj>Sy}`mmw&!ja
z?f3YXzyHX!$i0b|FYoV|t843^b9uMkB$vI*KFQg4eze?QuUu|_`K158#@ws>lQzuW
zdwyBTKF#v%eaGY{?3?wyb#Jqi_@1?LllBF&Xzq)%GPQkV@XXFdh0pdY=Ul6x&0F^^
z%@y9a;n>7|JkIm?SSXn7_sg}n$vF6P&x84E?IzES-g`}R*WN=(llKb6$?f}OF1nAE
z^Ro?0{WjaIzjFIFtbA+}owaK3r=4r}Iq%Wlo9H)t|AeU@Y-`ut-Rr)rX7A@G5&Nc#
zb?@IlHF;m{j6C}-4a@i1$hYjBVNh)=bNlzcmCe0-ciK(g$2pt9E@8f?y}>?T+Xh~l
zy*)eM?Q#2e-{$%})_tEHI_wPv*Y3S;+PZJr-SvCFZ_=^9-M-r{w&}ZFnnM2GO}+j5
zAG~$4?s!wMC;h94UD!{)y#;$C_9tX=+CFi(z3=WrSv$7tbN4u9WZV8VSiEm#-_?Dt
z2e<D#sHDE{mBQA&7xr@5x|w&`W(%I%ub3va_g3F6I~LaSb~}4{_S&&N+2hIDXWMyy
z+1`ka>Go3?mhO@2*uQs~d#SA}(~P~m|4!L*^EU5QzPHnE-;(lueI9CiXRQ2gd*s^w
zy$jY}-5tysv;Xyyg8dqg&g?nPXSz3{P+;$N4sHA7t*&;BHlF+L6ffH=Ts&!C$>N56
zA2m<dZ*9D{Z;sm*d$09Y`#c$v_x(3~wzs!x?>?`mmV1RioU`@)oNOPeyl<~ZVZz=6
z*Z%DJey(=kf|&KYH{TK1`*X+Ty=&LY+ZuLF->3V<YhTaGAlnDqn)Y4!IK%eTi$i;t
zc&ype8+m!JLpk?e)5^#DGFHCa^Lo9^zB8Te_C7p+?1F9T_vQN)?z<;(Z7)mZ`@Qx3
zYI}K}Z?k=}ztcux!=ByVg=_Y{JGFeT>BmWSGiDX;eUPzeU$2F-{i~GYdleU6wBNQ*
zYwuOd=X*cLMeobA=iVn$Fn@2$->Z9@<r;TSoh@d&WtG~#)j~gPLq09ryRXLIZew5a
zzUt=Pdyi!5@0I@AWcUB|{JoF5+V?4~D%gJ}^4Q+o*)Dspu6k?NwPNnR;E5;gnd1`n
z+TC1jdn)3nZAX&y-gP3c_C}>`vOT=acwdH-$zE-q^1U&o8|~z_G43<Dy>#D;<Es0X
zyYAQ<@wj83;QG`19;9pTy~@3OpTxGq_T~K7Y{fJW?pr-Y+-}y2<#zHX*{!RpQ}<5c
zUts(0Yt7!Bys>+Bo{QakCF|FoZSVN^o!A>|YZCW-kHOA@Jtx|Z>?wG3&RWtmbYDlM
z#=a*97w-Mr@3ZfH(usX%5*YWL$n~?0xD~&*?5l<SwEb*$ta)wwGG@r_;p$mx8yTTz
zU$MN_wxD$0{#PF>_Idm~XYG2~YwypZ<$Dv(^zT;Yd}zb9kLN(`G&Orx4$pnRZoS)^
zGr4g89qY(_9l|a9{vW+>vu5Y)y(dIA?YZckV9RYcZLdYm%01;A;d>dxZte;FP`USz
z*4@1>d;aW^U_NPcZBO5xD~z4{ehXc<t-rWxpHq|N-Y+*q_sw1Tdw+$&{JneKo%dB-
z-EFtcBy#V^x6FI(OQzXvX;t61txIs<xv4$2*$aN}Z4p!0`^}};KFUPJzL7y~ufGT9
z{%cFt*?Jyby;noy-JXJD+WXlq%kB%Z(B6AD*?n(t+Q$71{AsppbGPqpeSE^M+3Wb8
z>-ved2Fd$uE@+*#_F&kw*RoY(|2d&uHjA|n?41zbVC}J?b-$N`?S8jqM|byah}hS4
z?eM-AB5k&b{#*CmxGcRl_0wYeb$dH)H$I-axBRB({(_nVwpQ!S_wdKRw3}hFc<-C+
zJ$q(aPutsLxWm>u^W@(9svGwl@yoJ%@|kJxRBzV(+R~c)18Pca4_f}*^X%eZyYHrZ
z_VL$C+WI`)zt1ALWv{xuymjei4cqj4X8W4gZP@2mH{D+S&4j&ua;5e<3=i#im^SQF
zDH1=hLw@~!j&5fA#`ClG`8WvgTj-u)|MBw`yXxynwu=9m>|`HLw-33~zn87sV1F!=
zq;2O%g?-`N4ErZZ_Uwz|KeTtR#D={#Hj`~8U7x?#=iyXamx6739?lQjx1;r<%>~Cj
zy8{hR?+tI-x3_!o5!<?pO#5oO9QWBwvf3A>_G7QcmxY$oc5vEBaZRzE{$}z%9`RQD
z^i{L>a(h?pGvi&e-&kSVJ|Tk{drt6s+CP1}!`A-5zC8?*NB2hDH?kE=`n&hy^W=R#
zvD5cvbXMDNALZD0mU+^?b5A<=30+RyYcea*w&uFEZAZM8{iHd92j+5j?elt}zAtft
z(Oxm}h5McKRSy&h-L-kEx_W=gyX<}5ySD93uHoHlq_D`=E~Cws`NWI8r*jqeHN0f8
z72%q?*K4Dh?ecl6_Ih5Iwq1X$WB-I-ANN?csoE#G)bIWBaKT>Bi&^$5cDL=5KKAX4
zv$fwBy>#-v3EMvH-E}&C--9H9z3+>;_L-_>?M>v#*}u1cwq5)T>3wTep6zvh#=ie?
z+w^^3<(v0Agj(%=sCCUYcju!07QPZTUmUyk9@r$j`?D+8zK+%F_h#&$yC<fHaqm5b
zw>At1<96pwX|iSLSZxz?XUg6=tL1E8c%|>zFlohZg-h4>%rHB*H>1C1@8u7B_LhCQ
zx;OMr?cRcIPi-zsTG}k@PT70qcGun&+b-A^UOBK=zH#>6E%}RVHN?_vI@@;bO}IJJ
zcGB|cd-sU9+w?wLVw=pvY0Ds>w!@n<a<6uupxx)CCDz9#EVZpWa>iy|K#Q$zX7Ao8
zzj=EPOsU@;w0VZD&)Uno8)BPmFIc?Tv%P1>-UE8ZyJeQj*%l;6+qUVl?+vLuW1BRY
z*DmPUo4pJU2dpl7>h4{cc4xPU`R+Z*ag%Jj_f6PqmdLYDdFQdcYpt*C*${nxuYji8
z-lln{_vA0^+AE=WV6O&4;9i|ilD3>SSNC>hSMIgZIAr@GcfL(n@^afB54PHvDXq7)
zis#)UpJig>CEaHm&@g$A%{%6OA|V@XAK#q3SD`B0`j`23+xD})drliK-Rr9KZx>5-
zn=ON?+^+2hIri;1wRDg1z8+hK2i%qntUIk<z4YCyHcMge`i+)*y)u{Brflf9;pqEr
zvmlINU$~yf-lpX@_vSLL+Z)h&-{yeFF55lrtM|_0$=jQ?Q+}_@8n(SlI@j9vIrZC$
zWX-k}X>hhx5SFuzxP0B_h4XQni7y@ZR{vMuD|4pHHfElS?GJBp+q1WF_Iii$**tk<
zY@7EaW3Rzkp1lvYZP`1ksc`RyYfXD6>{)A@&bnytzcV>|CVe*F8}V<!o(JJ(dslgX
zu_?LGvL{AS#O9K_p6#S-2lmcd*|*nNY2jWcYxaFTsz>)M*cZK9KvcnM(SpaeT?Tvh
z+O_BI_4s(m=5FEay#imJ?AfGn*d{6J{9ak%{d*sm@7nvS<;32lt(>+2MVG8g#A0k;
zxlFcwV%xFTws_%QyMH@vdSAY=DU*uaYrWWH?}y8My9IS8>}}<lv*$YJligc7?(Jca
zJhZp@Ud7&w;`cTz^Iz=!a%0Qhd++4-3M)$Ov3s?C@1KkcTjhZIy$!2Y?k$l0u~)}%
z>z<Op^Lz4CvuwlmAF%DT@!gxSW}nRshGN?n`n&foj&<C7^#_C9l=m98C$G-id#3sL
zo)0s`Y?o|Zyhr5E9^2LvO#8}&T=ss5uh^3>{dI57Cw@DniHdeEH%)C9oqVuof%)S-
z3RUN=ZTc;3|9!66dm+cuMq%FRy>08-Z2vFp+q*T=bZ?3-!`>6GqirR;j@Sx)o3!`!
zOkL~hWaGV4M4WAFI8$urpIo^2Lzu{(farXiKN_`r_uc8=%TOh0Ddly1ug|1ed%t%q
z-TUMJp1tMk;`WC9*lp8r-gfVUOw~OH1Rn32F)ee?ja}S(y}mc?-PbI&SNQ2Hn+NNU
z*fa?5u=d%&YJ0$9lFc5$*|rm`MQyA11?_!&J$LV$Y!=%m#oKMK=gqV+vRY`%p~||~
z=zHg0hU|Sd2|2ubEEcufPTM$j&o8bGd)eFNZ4aDTXtgD0wJpQk-_|irO12z#rrSzP
z>D~M8WbfWB+#0qEIQVRqz3kcR(mHF;^jPz~cMgcyPKkYGvp_U`&xIFLtbf@a-D|^P
zY{TufeecAF-)$0nYHW82i0(amQ`*iUzTB2atI2kcY`pE<(igiQboT7cx&3laLc$H}
z@Fh*Q8ioqCcLG&xzXU9^by_5EC&6N~ch9}awso(fc558r-XmJPY)_86;GUg1s<ukW
z)Ay=zFzw@+8EO02L(uL)V*g&22>!hqI=r?5UF-IyG?eevX_;ng&}?jTAwbgh_`fxK
zU+leOTlxHsErW{bu0PK=TQ6goX7h8ou-%P6%C@?9XYAb{o3nRLr|7;r`)}HQ(7j=k
z<T2Uyo1^;Pucdl>xvX~XT|1Y}&OxkiuY*F(o*mW9wgsP7>|ImCW~;##V7qV92HWe>
zANKweU%K~OL9UJW)Rw&sJ;!%1c*?Zr!c{4oG>+waU#=D0$Ckst*UeSJu0x4s&y4_e
z+c0hKT@B1@c60r|Vtemp`ra*fciDs_wb^>)a_u=QzJBik-^ASvXII(?<`(Uhsur{3
zDxA5uZu|7T4AVHR6*z5dYZbQaHQ`#jXX=WLds%|HZJGRU*f^|i-PIGZdhgZq2lwXu
z-?>NT?FHL=zm@lXYdCHjAU|Wzn-U${ly>&LA7WE&TF!^=`SK~qHstYn+XmaTU3wK~
z_IRi#*h<vR-*fQaqP+qwGprqwukHS3)U<cko@sk0I4IaGT03p8TlMB$2YjV=3u(^U
zYovN(&%wus_i|icW$ThOW$(PIeYSf%Pua@6=eKp(DZVG5VwJVf-syY1bj0?l$T#j)
z-0^7dN&Q&sN&jp2y6k1LJHYy2&xE<0wk&6-@4Yi)s*O%w@LqvuAvXKVllS_)USRvA
z`l_uatHj<<mk-;fE55hA5Pa6=&Vm_x)ikHr9#|{7_l2?J9+l4>dpRC$vXxlVVq2J|
zY!?(Fy7$2UT-#{L>$XypnCy<I-QKg;Xxd(esvgUNON(t52wLvBvH6P4jUT?Y3~Y;c
zcWCUjVVD$Uv*G_4Yn`q}Th`92wl5Y{?cKw&cyCq2VOt01TAK@kf_8JBtk}!5jlp)`
z(S>`Pe`M^{I(lKR_nNJH{z!(|YP{&OQ934X_fRL_HiWrmFT;ah=4oo@Y+Xvb_q+((
zxR*t+bMLRt!?r#ey?df2vG02nxn}Q|1g^acX7AjavCePL2cro#3skvm9Tv;m8f=_n
z%W(Us%>y%I+Y9x(Y%{d-_O57~VWXPrxmQH}=pLb|oqG+^rtHo>I&-g&J-eOB(dD)o
z3H^J2{9J0Y%wdsjROJ%eql<oR|L3%LFGI~`o3gHHdo#KV_bLR;*{k;J<z6Ns!+jkA
z$@@3$oVR!P#pQeNehb;R#($3emz&RRQtp=RTbL+s6R`K<-Z^Vi_c6$yvfd?ibnlAd
zvVD`c-?XtYY~H;@)6}*^Q)-`5?&AFsQ|Ili+PueZ!POb|^ZeQONuRv9cVEJ?{f1YI
z_kR8;v#(vS*?zmxrhNws*zG@fr`o0q+^{Rzu-N|5)%p8$d>`0+y0K&L>5>b3TlszV
z{l2?@?{?9$y%(4L*!f-CYCl&|jJ@wOmVMIQ?fY-=P2bmX<^8@r%~$uz-delY>$m(K
z@mbUNCVEuc{}$#wz_feW{sR}U?|mp`Yj??9XP@C8jXhsDD)(*waoF}v%%QzG>Dqhu
z-|sx&8+3E;r^(`b&u&saP&S=wuM<n#UeS`Ndn=cD?Mr-eXrHe$`(6&IgLVZ%NA@PA
zo!RqH%3vSchVZ>QO|5%ZGR)cQt6aDDd9mX@ro1ZKNfB%IMtPjvS0!Ar_x7rdHXo{l
z_f@K#-76CLW#8t<Tn@ip)bER0=(1NdU1{I*ruqBNxli1;HpKKm#CJ)X6)&gky(ZzX
zC&S;z&TrmQ`+qlF_7$|9*mF3x-!|d!kG;)#xAwNIkg!deuzqi}g3bOT3lsK#EWfnp
z)5A^sB3wiEPC4Ja&tm7~{ZGz3-v8~L_pY9~kN4mEIAQNHo(+2w=B>1s+w;d}=8QFa
zZy%Q2S9G?)K0=y#KjZw6y@z}cSbcfNW3RNT*>37==KXDtkMA{5oVNGK$GLkQ>OA*8
z4(B?s`mOiAz6a_KFK@otFAyxZznXv5-d+D#_q9)3zBhZt;XS9twD#_P;jy>XRN1!O
zFn#aG9dq_xSbxykeN(l)jFsj-o&SIL${n0+|H5y(ZNS03dp`y7*iE(zwPQZ_XK!u|
z%f1ag-nP=3R}Xx9HPJ3_&y;-`9^89Z-iq9(`;BYg6USNmEiZrGXSh*izhLRxeavgp
z_kFKc*=N0O>%RX%llLYsTDVu$Fvb3LztUbthLwA>Sey2)6>Ql%E!=nSKOeEZj0Riw
ze&Bw(=atFsefhlQ)>A9H_qK<N**;bf+`B^fnccmqclWAsi0l*VeX@7y3?ciR!*aH7
zRz9*eJ?3V=>&bPyA77K~O%f$+*D34mQ&}HoySG(*-@_;7`?E@q?k)MWbT2>im%YxV
z@AoLaKVciVeCghl$o2yZa(3HA|D3b8I^M}n$v|}9sogI7GJi_j-;j^9|7W_yc6&nc
zKHcaodziB#_VO|K?K^Z)!G6MSBOABqm3z~6?A-fs`{8}zlNj~{+<CIUzj?vlHy0P}
z+axr5|MG%ud*fKk_FgEpJK!7@wQsLU<pEo_V|y1Inzgs}{NcTm)b8z-y?<pde}?;B
z`542!nHLu9y)aqdmd&_rZ;{{Yy>=<_dtN==YP;*Ez+T1dBHK@Y5AW+yIB4^dw`!kD
zV!{3k&*Jx`x9#88pW$TpR;GLZwngT9y6#BqlYV`2@9%fJ_Xc!r+9kd9)L#C-bN3eN
zo!NWUI%99ga}S$4kN4SLySix4jq`{0@^P8%GYIV5n|J)<-jav=_pNgi+UFMfe6P`}
z4SPQ}UEaGUOwxX?{FA+_pT_Nbmvvz;;~vpHo=emBSKHp*EBEHB{gwaA_Wg_gwO97S
z&%MjT#rG!W&a#o(-MII0XNkRs&6|A=O*8h@-8Zz!y83=^d4|_MY4%6^Y!)uu7cIJc
zuWrEaeL5Rm_JwYawUv6Ay7#H=`Mtld-P(I`;orUIH97XZXZXMS%k9LyXF22dI;&0J
zXSnj--aPBheVSI~d)uG#?mK!))Ip~0_g>j1>3ypYYVBu@s^2$rb?Lt3yZQ%A8r*E{
zCePcq+3CXGWI3^Y+u0WG-FH>YUQ@7bztOCTd*+#NIh2Pn>?^tQbnhxBwY_gA)Y(mZ
zFKll$Y2V%pZLAJX-dql~Zu55kjB2*scRPF^^BJyvGPecxseCTB-N*mjuG4a|?WN9t
z`?U3V_sJ~?-YuQ{*LpvT=ziyn#ru=%jciyZtM2tv&)BDGIL{`av1Z@zyLS8RJs;Wc
z)!(zX`#|x&WIhgu&#kNXsva@hbD*YSU)$H&dv(I6?kQkgyQk_;_wFZcY<q9tG_co@
zadlX<G~afbXv<#X(=YAW>-23V9_`yZYl`jx?YqzRJ^Zm@@60Qw_OE!?x>w0IbZ^ZM
z;eAq}%==Q9rtW=I&V1mxV%e@^;h*ilmvZetcqQNd^U){!<;~e07^M>TxqXb?S9|37
zUOCf!`?dEK?OQNs^4^)NrtcL~m~8*ajB78u+cf*{n<wqP&N<&+#XWz2;;z?rykC#(
zeHe9dPsih3Hk^L{_wrRS*+h8w?fr3b^S;;T1FZkGa_x_l*u1x`cH-U--_P$CUwdlr
zN6qbfB3O-W4(cr4tGcjz?_IA6doE~fwYGV9Xg}i<z5QF5kMI5ddgg&T^NIs?pKJC#
z_F1rZo^as)WsGY3IyOJs`^9U)Ua`wfdo#ksZ8A^o+ZR72cW<BAHroT7EBAT|$=E+v
zSh|ma*?3<}mF(VAv1YcVJ1X{nvYcsWS`@W!T?x~^byk=5B%hwKk4LG0zqG@gy<aTz
z54ifY>|J3Vu*XV7ZQqq^X|@yYnA`lWW7x0G<hu9C!X}%h&-eBo^jy8O-08bbO5U}-
zWx1{PGY@at`?N&b{=(9ty>iM1wgPL__8<5mZh!6mrM=VEwC&A2RJ70c^PRn4HlN+g
zQChZV+SM#;hl%BT4=sw{Co_4<{tu>odn5ii>=ox0-oNYG;(eEIs~yOCduX3t=7K%1
zE_Uv@?xwdVbf(tcRXwe{SLsdM%gg(F?-C9vn+pf-+HCofvuDZOEjBCLRQJxiBC<C_
zm~&52w9ww8MoVlJ53=u$tvYJ0wtVj1pXct`JbD+ur{~Ygy@kDh_6Etk-!p653Y%&A
z6K%y_@3UoCre;08%X#k#mkWFUEO4@wl{&RIWZjxQdcyZ@^m{C9TLh-=J`=&dZ`Xb4
zeL4#B_xzN<Z2K%xZ%^N6nSC3U@7$Bo5VBYQwdCHA#|v#Lp0?S{mN~Ka(D|c#H|{!S
zv(dxM)+KhUt@>R3eHDI(Z2wEJ+U_}eVz0@91vYoCgj)Z7*JKqhbJF&R*LK@uc6;~w
z%A4(3$t-Fo9(Bk@JfzX4De?Xuqls&6*wx?I{=B5QM|{qly^mg~*glwZ-ex`5gT3!o
zwAhMWm$$iWf7GUm`M2$NXT!Y(&o1m0?Vh@KTF?dCu5HqL=k2bsS#S8o_6`rTom1dE
z+m(4Y_r9`UVv}}1b?@@@n!THT$n1S5cilRJEoCqJ-Z^{EUOTzBZU33QJ0#=w3bDT0
zyU3hr-yZpoyB|u3?#)PdvzceSc5lMj3)V+iT5UTx-`Pxiy?^fvsnop-VykVlDtq^&
z7q{B3+wgwR!aBXZv*h3IH7S?b>$&ciO+*`ujTrl!z31!x?!9sN+Md%7e%PK|an^=m
zf#e?HpjmtSB3SHPrmnJmTytXY)~KI*%0w7#@A<LqV~u{ZH+<zc+X)9=*l3jev0ZWE
zi0$oi*?q+-zI!Df9NYVH(Sg0D$4~8f|Fy@~<Dbahil$(j1425sU$&InT28yZH-5*V
zy#g04ZP&DU?zQ#3xhM0=+r4_TI&BY>^X>DV?X<V+$m_jN4^7(RGULcz4y~hmrn=7E
zTW3;l8+T81Z>MJe-o09SdpGFi?|r%Z!=5FdOKqpkli4?C@9sS+r`Y#;y^`Ixg72(N
z2KOAB`d|s$X>u3$UZ_~JM_%i)HRCqjz2>WQ_BPC#zqgQS!(QdXf_tU+{@d#yAigJ0
zHFU2^8keo;#BQ4h>t|ZO4*X+Nme8^HrN|wd)wWCaPLN>Rr?X_{-t|sX_f~a1*n8#5
z|2-2XFWYUhv1RXq-2pa%S1#EaSjF24?@+guI`6%g%X+TuQ}NEdAB}@;8<b`C^yq!J
zem!rk4X@Q?n-`UzZFVjHWZNw!vCpHZX>XnFWZQi+!fn4*@3CR<GPm~mb7Ie4-x+)5
zX0Ed}*x7Fz+q-)2qJ28HB>@Na*6QrpV-nuB_fPMBTb)Cn_a3@>Yp<9;ldXrJu1&z@
zGd8IvGB$4}>ev?TIkWc*<H@}j0(aOZ$Z^_k)l%5!$YHfdRabxSBNvvv6PNPa>b>c)
z*?WK2o>{jy+Md3>cJG<DGxsX7bJ$w=6z&ZOv)L=*)VcSkHiPYlvgv!pCvV=nscWk3
zrEpOj{zLM6Kg_M%<6^jOuh6nXwg!ip_Ws^mwD-rRO}0Dy?QLGn*s<qG{{)-Og%|fK
z-2T5?F5=pr`O5t^3oK6B=9SCZ7NqsroSD4aCf)Dao&{6#_C7M@+ZVk2@7_t8`)t2?
z?B1K^Fv+&6YvCS-jZ3ZB%IDa=37=v!!Emb0Dlz}P8x2nF^-nps*Gpr~Uf0Ru`(`{p
zVJptLb?-SA=DiHNFIczQ&fUA@XN9dfla}pOhUI&miWb}YESzTZ(bv<aph(K@g6KNy
z6R-d6eL0QQHn&mN;@;23z3m;1dn1aw_eOcv?9Dcty_bKl&tBP;@Ak&pp53$V5TD&q
z&m-1bx>oOzj;Y_1aR2=7fZjVccM{ob9Wq__zWZ=)?}Kw9HV&mJ)@*(o_w0<*v8`}i
zVRNE@VV_?Z?_Pzl8G8k;MA^L4JH0ogrhCuAoh-I94m9rxE7h^xS24*(p(@<gf=j?A
zE=zi!?`566J1#`q9_<m_6V`v-X3Yf^YfjdwdyDQY-}{11dr!%OsWz$lEjH`oO82(R
zy=`L{qPTb7$^1S1-Msr=haK2E{l|OT&B+t@N(gi9vp!I?_t5<{HZuyQ?zP{&VNb`Q
zW4phKEZ-Y?&2aCFwSs$@l9>0tY*V%^af!FJ=sdW$@4w*QQ_{Y+3YYw>Bg$sj^37ba
zH~5gWt^4xLdv(vPv3YXi-QJBckN0XX65bbcsncfbqK7t)9W!kGq80Z_C>*e@6ky)N
z=YQAs^;Q!bk2hOv-u&R-`zwNFuhJiWo7a}{ds)*K+U__OV{-&_=Ec{Ed$;XbYWu)^
zg3YBY*S#h=skU!7uCZj!>)(~IG01l7xi7ZQ{_Nay_@Cuo1+4{p%im3~jj&g-xj(tg
zw(pVa9`3KdcQ+f(-g`=?Yw!Qb*Y*^>)!p-5jnUTi-Y1)tLGpWjroP%+CiT(AASHgU
z!Oxp}7wFyHozTN-8~)?q-aebDdz14&?8)HMv0*#+!$u)|xou+&w_URH;XU`gp6{*o
zW!d}SO5Wb%cdzX^>fW|D<=n45ao&u33pY0HsnJfcy(;z9wm-pj@3F5}_8w^JvORNt
z>0Y0GhPEX>3VU^yE#JFC^tQFc@2Pt_4B~7g^Hld8{&{xqj-pMrXMbDmjmuEj%OGcE
z%XWSD-kB4%Y^Rv&?2S<WYEy7ec<&zFW}C?u?QI^ao!_hBoVa)L;kC90qGk8|`etg|
z@#L*d_1=CP(S@J(mN0zVV_LP?HeqJh9v;5sHd9RU_f|A<?6ID$WqT*)lWoIJcAE+E
z8*K|&&e$ZLk=ysfp>@x9zW%)me|2|1n%HHVak0^6@wH8Rj!)dNw@}e?&%etBd*3|l
zw|#O-+IDe`qMh5j$$JmD_U`q$>1C_HS7Dnocj{igekR)u^89w~UZ?gfnYnNetJ|@?
zetS9gSx>98Wm~w)wo|QkZ*lSBy>AjbZS>FU?X5W7Z!OY!(DsMEs2zjL3R{u0{d?be
z74D6f{$w+g!Funx-;b?k*xA{1Y*=CQD*BYIq124sK_>h6ZvVe|@43S%d*9?#>{WTc
z%ywE`-QF8+?`&_iFS8AGWw+DHTD51*6p?+_ZLE8u@37cf%;mQElKp?rt=>I*GZb&{
z(M#jnw?W+3_IuHyy{XoMw##Q{?KR*k+3V30y!S)(Uz_v}p?&Vh`0aRVWo&PIeYD-S
zymN2G+wZ%Z)s6Nph|%3Gw`iKJ&GYEJEhiV-P7x8a{d2uzk0KB6Ud}x~Y`5IpZyU67
z!Csf8d-e)??66&2TV&&$ziqEe|8!dg<14#)mh11`@}t(;?(1gTgbB`j?LW@nt5-eG
z*2nXL?e}{%dxH&nZ52u%SvPdC?3%cAmaSclrPXi7Kei38#P@#UjNNP4`fslt&&EB6
zrk>urSv|>SWz*WdKVD9=mGarRS17P#FN4`p>)8h{*(w;;?%sT8<KEeNyY~j%j<h{^
z>d;=L$p`lu6h5{&|JP)1$0V=4FaPY`d*%$E?X2|mwmJ@idvk<&?D#Zp+cauO?c3eK
zxUb;Ut=%E#boV;+-M4Yc*krq7O~qb^<g~pR(T8nKrE>Q0Fzm4{3GA^wFs*KP+{}r#
z5^cNol<=?LvvYgDtx935&9$jrd!MSWwAGOE+#_?n*+wL@eJ`K;GTT$#8hg%}O|tDj
zuypT(1RlFNf7k3e!xwFv<F<a!JWh9;e;vE`UfQm`ci(n_y;6soY!7TyvsRfQZ!5iv
zZJ*jYslBXAR#?j`7ubX)i|l*B_is<t+;z6iw<hmB5};r^P3g_vj(ZGyTsAw{23%{g
zVOYar^XS6dJ!+G~!S`K0m@I1dzUH6p2R+GsH{ST#p6`s_+oQ#?Z^k<<8;L_)HYrDL
z?+Fn9YIEv*<lg6UpKO<SH|#xW7P!}B*5thkeEPOeq9gV$unf00_N>|Cd{cdINBjoc
z%rM`*uRdPb`(RGf-b4Np`xxq{?{1i^Yo+pvd+*i_OZIw1W!k39XR)1W9I$uRyzSOI
z*ez{SQ~vH{JAQiaX<n<n9TMMcR?l2z>sb`Fx8zizZE=;lZNAphJti#8wnf4>ZH1(n
z_nzBwe@|4>aodA&XZ9Sd;NAD8`_Nv6gCA{vNBiv#ezn;4!{`3pb9zl}*>7{(y)m-c
zb0tgO*6N>(tw+t3J<P?Ny9*^5_eLB_wLP<;VQ&Sa-JYJizibwS*X^;J$hS8px77A#
zR=bT%{!|;i8A^MrR-Ck5t22GCMd8gomfPF*>MG^$y=HJ=?+^dkd!Fx0+glUAde0AY
zBij|fCfZKm=h|ym=d{<!eEQxWbI#gcaGtvR@59Y|8BU$CN!aVXTi2r4O8A4yo@*0o
z_HGcExc9(@16F%hez#r3xPPy9!*gr1lKFd21gO|X3oYFnk=nFp!8(5%+5W4xCL(<M
z@_PGhXGAsc@lPt<o3dn)tupV7y*^LlZ6$8`?QNJ|yKBQeC)-0illPtu%HL}herB)y
zt)e{)iyqmS+!3~2VjW=nq5q6c#+UZJbIpDCR+f71ZD?9=ouQPq*MWJ7?X)EaZ4bNd
z+B@^YYTJ<S5w=rj%Gmu#TxKJ3-FWW-2TfbfLm7LoyqRg+5pHJdd3VWPgP6&C<~l9j
zTTxeQ8(-XQo7?h#&knmuHaB}FT0iT&zxM;bpq<TcQ`<PNv$k$FUABJlJNLS9iSISn
ztKIu_@tr;8I+lB%aUZm`+O)*BAmYT{b3#jZU*4d+*Y2|1zTPCCJ(pfQwF#?bv7TvD
zzBhoo(RPt-maP{j+rBT?e($jrcG!D3Xqj~bm;9b<A7AXvJT!MN#}k%4i+;H7eGpK(
z`^1{hHmA;>+GFCHV5{c6)%O2^)Lrc5wR>9vW%tgRFJ{YrRBi7)!6|#=Ud*t~PhPpV
zRC}JSX~E6i24{}%4tg|Uud;BSZI-gq9?yywHs0@#+ITC9+Z|)ww|9zvkB$4vi?#xf
zjcxmPo!Kk2t$A-qMuqL%{$lH%?-jPjDoM8g7M{0NtEsg$KFVQht0`_*X7|eW%WQ^y
z6$TQv&u{GA+ni{%*JuChJrxCMdpGJe*hZS3uvMy>XuCag)ozE)ORNj0ZQ09kS=Z|I
zy}5gLGnv{JW+&Ueh!okQeR`pd0>>|#TYGu-_TA~VeQ-U0FAEp5ZTOZJ+kFYUY}I6M
z?TT1owr9oVPqx!OiSK<7bIj)d&!e^(x(PPlpBL|y5#zDddM3P2ZS_9e7_$X?P18^9
zt$*IIx3Fuy&6Q;qd$@(~+lCk|wN<#_VcYe7v+e4%^LrbnU9+C^BFa`HrP}sLQ->|H
z@YcNoiEg%gdgtvm=n%F!RCmVa-`)w9Lc;8O#W<PwSwu_QwoP-hWsp0+H^f)S)?(%!
zo2Np8`vhk9*gD$Yw_!N<)_P_5d7D|E?e<=;;@kcEPW+y3nW=jpoW8WT@tLgMgXQjf
zLQcurrnv614GI5k!!n(3FVm$3wlNwzY*^A=Y(Kouwpo0Axy@$2H+v@BZrpQXLe!pJ
zsrq}f^WWRXgk|hK7V5EAR6@?y$D(x)Lv{KtKBh-DPvh?G>GG_%xgc?JZ%jt5t()*(
z+lrYR_WW3Qey_w*g*_7UHt&|^ziI2~tFrg)&iuUxg-`Fj%C}{&-32DQkB%mLUTpQT
zS+sDqZKHX~-g%;%Z8NqP?vb^Q-OGQ4-|j&2?p-_VS?peId1^aF<ky}PzE^CdWj@+I
z^vT_;5FxNThkKE&uY{D{9i}z5E@i>CMtXgF^<E3?U0Wkyd!}2^c8e~@-irG3wog~C
zwV5%6$yT5+*fwv`ZaepXHv43rx7Y{>%(W>laNTFpy2`F}N!H$Z=VsVC<Zj;0Fim~e
ze5uC0KNc>sbvwLx?}z(&HV-c5?m79Rac}vGguQB0x7a@FJG{@Lzs=^yrA0QmVGHdF
zw&>dMt(d!SdR2;D>dlFJRRkDq7aa?;voM=wWAk;f-5IOdwl^0}+V{b1iCsmMg{_T%
znXSXT3sy6Z?6log+P9bGK+7J3U@cpny}S4N-^$zP(6Yep#6B<E^?KX(Ex1*-k1N=E
zuinj2y9OgYn-%)A_XwO?wD;T5eRgktIq$PhO4`S8<nUhIy(#+??#-~-Q6aNOB*xHI
z?Bf|*f$omo4<3l@7P>jZ?pfw~TZ74(HebDG?|X3Sz4eXeMB8Js+pRD7?%Bg-QN1r<
z<{s-z;oyB8f6wk&A+yhB)0-1}D{^+&Icz*_`{c{wy#mi(?TKmNw>uTM&#sVr!Cq!v
z1v?9!lY0*AvEDl)@`CNVz|FP}#lQBvSd_6(dhW5k7ruPjt?}5v_HwG<J_o1OHoR^-
z_N}TvX`8V++IGr$7uzca-M0A)7TO)K>99GmHrjS(=gNIQoL1~DIrMDL3G)V<ikyx;
z+Y7Sxbw2CgcVpLIn+vzY_Vs+@vCX(JY2U*oy>@~brTa{lOYJ>wIB8$g?8?12BpLQ{
z3FO%&eCXNh&bZ$$;Lnvk37?nQmH*J+>rgerD#6a)hIh|oyHop`_Z`1*YVWLvJM4~M
z-Lh9gvcr~XTfp8&eN*?%xO8GK&&;NM#uIn#J2KyCpU3@38xBEP+l1x1yKm}r?z?bI
z#`exp{=KZJ5_U2-XYBp+W5J#^yF6^0E?3$ytcl&_)9Am~N+{pX%yY}$tQ{+Djmp>9
zrOztgcSlg#cGp_jy$tV8TK<$P+*jV;v`_ertX)Bc*PaYR$-OtO9N$}zH_>iO?M|Dk
zDXZ-k9DB7#q-cXp+tP`9&zt(~`|y9~-c|Vv_s*F;#qPn&j6HKIvu!ih)Y}{~KDzgH
zboJi%_Uv|SrHOls{zdJzQCn~O;i&gMlY<Lvc?uf$t|=3;&HcW^&cR>O#z1bC?YqqR
zwk(ew_hqmgu<=<n-A3ElXP;B{-hHo5aO~|Y@88?S(7X4jr0U+ztSNi{o+{dBWD>tO
z!~2xY21&WSLb=QL{<7U=x21jiUeoZzeVgnP?5fyK?d5*B*Dk^1zjdSI4!c|SUAB`x
zb=m10eYwYCQp?_bD;Mltp=4(}TX)A^ugFDqIrAplvWM-nyR`I}ZPb^Zz3I|h?dE@X
z+NTt-)Yj&X)4s2>eD~#^nX>mx$T{2A%60ove$CkPCnv$~Ut{lHQ@P;10yDQ-N9f$y
zvm@HY)<H{RkHM!YHeVLZv27~dX*=iodb<xCXRO2f5A8KyV87SnIRD;n-N*J?nEtf6
z^mvM`g^aSTV#>mO45eo**LAk;oyu3bZ|3y5c8R;T?fsy%(00O()w?}(T=%YB*R}V`
zz4^8d-}-DF-kr4hcV*4q3-eak9$7Kf_T-%9dl?!Q?wru(Y7?;4*hWH7cJBkNwB06r
zS+*%T(zfRw<?X$+;)rd7>r}fS$&<ErUuWBHkpFB`;mT#p<x;Xw;l!)mfjef~@i_b0
zPKumvYm~fb@00bL_WpgbeD4)Ed)vl?yX@xp#qSGQywHaG)2zK6*_&<e2dC_vz_H9)
zL3M`p#jR8Ke)}=cRzZHp?n`@4@BJ}TeNR~OE?b7r2W>p;h4vnv+-}z_qQ6&Q*FNjd
z!5jA`9C5K->~v=D1LJ4bDmVRYGj@vF^jjzIlM-vTT~NQvS|D=izBj(!whnb#)^6`y
z_RWymzvsegjy+uadTr%ruiYDABxm!i*xK$&X~4dP=Ogw`)m>#LcUWbwz_J?KJhusZ
zmri%L+xd8(UBOZ=Th})$?ASlW+d0V0*d=yw)!uxQb9?)qo!cw9uXb-x%K_Ww6E%A`
zoNC`&;;?@&kG`qxg3}dy{?E+Z>tMThpZlAY`=-=0*rt@P+9NUTz}^Gz9@r$<_3c^p
z?t<<9>RLO-r)7JK?lJ7`*}lZ?L3_5X%hHv$&u*@^+r7=u_Dor{t&BU5t;T=nJq?=|
z?uxUj-?#Ep(Z0CW$+la*H`%-uUAu2VSdOj5p=2A`CoTJ`Z_lw?@wC|{ucL3@?YYf+
zi))+hq^$JpGPKHUe^m6@&a*sVr{l<G8*enj?w97meRGy=*vtQY=Dr^fJMH{z`s^4q
z6s_Xl>h2BXp1beb;u&^=&S&=qEjeLZcs0>Z!PVC$_iBjUtK#i@wRjiU9q(RlXIePZ
zZj$|yy~SVm@8wunW_#e3&F&1Y-aQkTT6RaWeYUk|Tf1ky_QZYCnkjpo)&%bT_;0FR
zjV_O^ME3?;l{XvqR?VDet7Eg(_DfifZPTkMw$GHB_Ld}1v72%Gl<nC|W&3vgnP)eD
zo$bEpb%}eg`YhY05wmFTf#ScrrdgicyQX2XO@y6<&8!{EY>!OO+k4<EpKYV|w!Ptp
zj_iA|ztLJ@LGRw9Y$kg<uCLs?Xxi3&95svW7`pjvW$jPyUBuVAw?=8!KC@|o`vS5L
z?PW>w-^XH_wwL8~o9#l5d3%d*JMVoI-nRGe_42)O8}{3B2(Ps%*w$dP>>lIZhV@F;
z0_NNI&Qa~O^Z0RP&-sf%b`0H4wocVsZDW(l?CuCB?pr$9dT)jK4%-P=z4o#29<n`f
zUd2}6b;jOLTXyX=a?;*gSTNmAICHk0$b@6IF?E`Do9!F-6;#gHqp^6?zHJ{J_Wdb6
zwdei*<9l~pl(d_uY`#~ar^sg3<ehsNlA5d^{O#RS9B;ALblSzeaxW!p-P{`Ys_fOU
zTX^-tKAu-0d$(~*+QjV-+{^dmsol5zIkw&lp6)H=lHAA6lDUWTuGYTirrLY=D6reP
zul}>QM(DZiA6>?MB}{>PfAZJb*zI6F!2Vur-;K%J?fBQ(?&q7edXLn(wEZjRuG;@!
zr+e=QE!lnXVcPcg`{wR*%wV=Vyk)*EW0<_nw6@g!zvQRy`=nQIvuxkCJxjNX?sGgo
zY2T{Ir+aU&wzQpp=F(oCMRWFFUd*vyXX$SnhGt)TZKd_Q_I={n_iD-J{q6Z%Y_?9;
z-Pg<7Zu{hP*#2FI+wJVve%ilJU(9xct<C<q5zYIA{7m*<PF}IE<Lf<}pls1S>NgZ^
zuf|CpkeEDmZ)^Px>*=h!_J1`L+Z(Q-w#RFu|K2k1j=dS5yKU6$&h1?-dwYLCfuL<y
znbf{{!fdtzs=xQHEz;i?DiN~pV3gFp42IBsAG#me6#Y24w@<urUv16(Jqtf9uyN^>
z+Apx+=HAWeX|`dHui0LG*uT%hvwZK`_$m8#J>RnbK=#aiYaY1o<&AaO;~e&HZ-H&Y
zUiH~m_dQ6xVJEy(Wglbu?Y*zcx9p!%Hh=HlXUuyWc<1lECT_6zN{iZoKU!}4XWod|
z``*TP-<gRjdpCWJvYYxbZ=dJcfc?Rn{_kC<RcQY-YpIRm;~9JVGL824gtzX_Dw?*h
z<LR}%l^yZ+;%8d+GK9XhkvQ{Z&(F;c`xTx|+HVtFW@q(d=DuYjtv2iO8u$Lq(6P-7
zX0$)`%EXqTd*}Y6t*7>JGI#DXVe{A%BDrU8bN$J^K1*2lw>}Wr<5Ik9x9_p9drT)a
z?)_cych75uS$m9T*w}|mH{EL=scJi|;luvC7qe~Ws0HtRe%)*TWqDQGY!&5wvnKHD
z+wj?8&%{eC2jq{Y?|%__X5W&$lsy+8Oxb6tD{gyW*0udJwj|octemo6{MF=roX*nw
z%<df9dv?jGy@wC7>|5<GyRS8!d%x$KlY32?WcK`tSKZgLR%36#g_ONpvK?)Vtq<-!
zHt)+`ne;F_`^%1dmz^};o3|@q|FZ{+_WbnOV&4+@%SNx)$5#HyPwUeuZu?xf9^1>M
z*1q@5Oqcy7hePdK?^x}BapI5d%WnRC_s&?_&bg?yFXWxWzRf3J@8#OsyZ4&b?Y&1d
zI`=N!QndecCzI`i{onSf35gw;ov?ZDgeRx%Hr?sod#<cwpK|pFJ3ha3`-8el`*zLo
z**Ep3uT_Ougmvy!t9>VSueA;N;<WdF(xiPAnUi<#u1eo~>vr7U3=Mz#OnHTUn%i0T
zsa#sO->YZxp0Mhfd*ggd9E^T1+1LDj>)s<xuC@=mg!V0pT)HPo{NwH~-^2C>83pXU
z*RQ<yN6Y6ueJi8)@D=gxWjUd+FVIoXCMa#i-nX-)_X%ft?|RUqZ*$@3pS=ecSnoT3
z$;1BlbCG?~*J}1D<z3kC!0~ebd)dalLOp$Z1797sy=Am~Pm2ZXKJ{k%y(V6qd$(>n
zx3}Y@)V_UwEBCU^*RZL09Jc4dhWUHfvexd?iD=u`p>}ugj=t^tidqEs+s@s(*Z<8G
zn=ZredsP<89(eS~#O`ro=f15aNA`ZaBCyY5$JhOn{LS~ua_R5iQ`odG-&SSMtP8dK
zjc-`(HEF(Mvqrdmuh>Vv11}3V?fbDJ+D0sP%iixtv-ZXPpSAaf%FBJl7W?;XT)*1p
z^qG=9+Rv}<{i^d}ulBZ-Jteo+*)Dv$Z?CZF);$xpSnYlL=cA3xU1QrH4r=@2rDgYR
zVdt@x)_2*rqkWal_B(&}9`UQQ&&$}cCnmbe_EtFef#6m7`;ytO*s1MWy|3(0|DLm1
z^Y*Q8oo>5EI(VOopvPWK^L@6b=il7#aB}Njjm7Wnc19=K9GB|fSCP`NS3lipzq8Nu
zeUfI|_RL$%<M8^XfW60JxqYc8T5LT(dGGhUwRi6m8)Msz^RMjHJ*8}~y|&MG%G?tB
zm=l-wij}GEJ^rR`uiTebdrP~F{noop_c{qn@B6f(c;BMTb$e4cJlUH*)6VX6MWo$)
z*>pRH+w%J!zU|q|StGR1$M?JKT!z#8ZknvN5&W-eyGD&;??n;$eUleR+OnBM?oSb^
z+VA;zg{|y~Pqs5G&+VPe-nKVwUAN7j>xcIxZm`&|?s9*x<|}Qx|JpnEif?ALe`YOZ
zJ9}@(o>ZAXd+JX+*>(Df+FZVPdjIN2N&BzM$n1BLS+j48c)$H`C!u{>&35}rxh?j_
zc(&Uexx(n6C%Is+x9Zxx*B%_&zw!9$y$P>F?7P`S_jTV&w9WmZvR@<nm|fk}rF$b>
z!uK86c5Qc5<m$a{YPa@!&6{m|)aA*(x9rpQ9%f+Oe{&_vp4njscF)Y}*nivSsm-iM
zoi-~BUhKIjH)ZdJzDIlZYo^<VaI)?Tzj4Oa$NQw+4VC?S&$jUI54*^3pTB<5-YwVX
z><(dlvws8MoV_g$wzl7P9o*Zureg2pDdBq+&n>hyl1bWoKJe$>OkTyk%S>hMkKSIt
zFJ<Ssy^OgpY_2VMu(w+C<o=>s1N#F<TJ{N@HrsdW!J<8TS+?7%H6O7TDEqQU?CrMw
zOPV?MNjR$R-Jf!5&xC4MTfPh8d(#fz+IOLP!tTTN=6lnYN$q=j=JDPmRVVgdyeP7-
zA?mPwmfK<*p-jpBr*7}qE&uza%_S?&{r%Hb4@lhc+q>$X`MwfXxxM>TBW#{N=Gb>X
z<l&wLQ+)RbzI5GdmDyy!)NJkkfDnegU%Y4T{h%te-$7u)zPL}mcFYSz>>b3|?0R;D
zJIr6aY41dfMSC0W2=3#1{C%(CRnvVjP4D(-I3Ki?(P`Ry^!Wb0>OX$jJecFP?^3Vq
zzKXzYw#RSj><umE-B)|?z&_>QSN7_jZQl2<gk$fuAnv{FOV;nx$jsV%<SyUd0~eV0
z$ur*B8_O%UM>M5pk88H$-oBNS_V3l)XuU^s*S_-STlV)So!z_rxxxN^J(2z0v(5Gf
zGqms3n)Js`^v%kBmv5BX+U`u<yHiNeeqE1=Eld9DeH{|>_O^N5-g{+&?mh*}X|@sN
zS$iUEB=$*`tL&-VUv59aUUh#t+q=CgH(U4J_HEc}!YH@@PSCG?Z~yPy8!}yEPn)NO
zEyspRyDw)&_MQ1vW<Ra)t(D4iCA+Kx^0pEojeFN#>EHKw?$y1*tDo9jK77-rc#+b+
zX|a;~cD>@>cjCr$JIyW@8=oig`&>7&?A!Rycki}?zWcgXy|D2JU$NhX<Imo!a}D-A
z`5V0NMxElm%A$>XLta$wi;Mog_q}2Nz6q1L_g(gzx%XSetbM;1&)@65XQl07W8-~o
z)0y{O$UL%dW7fX?4C^NCQ~S}d*Gs&4-&Du8eKWUA-S@0%!rrQb7xqS7<gsQHlHF%?
zp2;@&<zKtkU)JrLBYkK0m$|C@S^JjRFzQd*D`LHL@9B#4z3ZnQ-k<(q;{L{L#(jmR
znfo>bcUm{_CfeWL>TWkLEOfu+o5I}(-+AtN|MKCUfLwVS*~L!#`Fuq7CCNM5f1c*B
z=eO-X`%QYU_Bn^8?0pl|w%6cA#y-CHXZCwN>)Q7>yL(^tnTdNpE?sV;GW*XSn@jum
zMKL7rTd+IJc3qUl{&y>8?_q3zznf$4JR9HI-L~#qIQQ*HJhb=4i^H}n_4xL-Ee+WF
zekc3BY4dsa`ulb6bNq09@6E>x_WoLSdq3A=;eGL6Chtx8c4hxfJ?Z`Pb<%fjd$MHj
zmDVy_6}!{6@1DfkX<U!pXVA{Jx9juNeRmUdYz>Xp+5c>6-KQ~;%ONl4+&-V_f%~k#
zckQcsx_EEp-i>?m53kr8D4x0J;={bX{B~M<H72U=-RRh{M<a9V-Z$&!+b}e(+iN@Z
zyUh{Bj(r!yoUGz{>h~tip1*JZdcl1^=auX~wQRZV{LN3TWhctp@GUL2)jVutv&$lQ
z?^>%*`}$wbvHySm@4nx$QhQHE&ECtSdvkB}|JgRhLGJc30^9b!+GuWLkUhiZKxvt+
z(d8!ldW)lbSIu6uPd!*+?|=25)?5<V_94GA_S|>7wC`b+psnniIs3N=w%V>->a=gs
zLDPK?*X`XCHa~qIQ;Vuy!haR(DHHAX)&<Y9x}DOsZ^3;(+r^ur_BrhA-g|!H^1aFq
z%yw~&y?X;1754pFHrw{MV$;6RWk2mMv})T12q*2i!uo2@iD&$~x#~mqZed(%vueiH
zJqOj7@7*z<W$&~lx(<5;cJI}93Ei_OD|O%E3Dfo-4ZXhi>ErGDw0cGN{XLhzFXL{k
zo$=jTTd`#s`vQcI?PoUmZ7<xMypLOtbKmh>%r+A~^6k-VJGVC`UVCrK$}M|WnELEX
z_<wG%O5w3RhxRVmYob_b6IQu>uT|GK>jXB_eRGzt-K+m&+rIDpbN0paGwt(mIcpQ~
z^w*vP4>jz4zc=iA`c8b`2WBJNDcc(N1}i$+f8Mg#s_&VHJx|rNeQK+@_su9?VtZbb
zdtY~u=Dr;g@AsZz>)QX4)o=d^PS*WOoC*8xbWPv)yXnr}C-V%gpO?4n`=u_mhl^2j
zpK!DF-df+YyNi<#+H@Vu*ej@+Z*%PcpLMw5jlGAQ@7NSvf4RrnNM@hqgim{as25uo
z{Mos0p-#l!`FgRo^K{DXjaD@8J@wpZU*gYbd%?L=_nIp2vXYSbvG3N--`4C;_U^0a
zoV#b=ot68lz31%JssF$Kn~>AKxx$?`@2o!9t!#X}cd_UCeQI9E_RhSjVmGz%o89GY
zocoSjvpWcEo4x=1UBBJe9~|F%`-j~=AEQruubhzG7ohuYk4Ue?z6%bo_8L8$Zhvv-
z|GiUmIrsfEnY`EKhtz&mndSS`vi<g6`E_aUskX!W*6bG9=j~RzPyW&AJ$;u&9k}$m
z_f;vH?(_d*vHzrn-rk(AZF}<_bq}-@i0+FiS-98lo1Sgio$S5OGP(9{(2m}J#O9uD
zNZ%}5Hr<zdbLwB%S?^f9*F&ReuQsQ;O+wXOdrnLFedhB*_Aj4jw9irKvkgPMz}{0m
z5B9EO=iX;rdujg>wnuv$E1T@hloV{DeI)jWc{JObUEXXLU^#i;nYRn~vE+ZXIZ(1>
zf7>LB{fVox_pXn$wDIJtuy469vQN9TZ@<yJ6*lGhhwV2#V6od#+PHto?D>0CH!<wJ
zFuTh}_RjUai$y)HCH6J%-Td(5zJ=buY<e3RY#&rQ*sc_n+_z27+Rk^n!amo<JbSGL
z?e-Rb`nq>gAM4(O2hQx;a9wh5+_uB}tlzxezjblz-i0jg_OH|I?U$d3+jn<`$-eF1
zckUC)`@Ea~y!1Y?%71(PgBMu`ORCv&&pWVBNvLnni^ZO{rYrXE{;$Mu|NDCTzV{mL
z`#$mu?+Z@cYcoY@!+w|LUi;&|v+Wbk;@l?@esS+kYuWvtFFN<l`!2GNt75a=y%d$b
zpS;=k)pfqu%X*n<?+4yZwu+t4_UZkXvAuZ0)HYm2d0*73^?Tc8PwkDjXW8ek)w}=M
zIp%$TTGZ@q)M9NnNU`r*Ea<RbPi)6Njpu^<${8y6eTj&$o$PsV4_`6MzFDoi_P(65
zc<-D}@x86@rtaTxdG$WMzp;B43m@F;rM})az;gM%i&xz2G7GHi4+hTJ%l=+^U)SdY
zHhz=YZRNDt_Bzh+-m4LEXYcYS+xDK4Ilb3yQODjFYg}zl9+<n=hw<&6hA@>qPW{vN
zuJP{O6LPc5Hqp7w=GZHny+VJC_xhAD*tRI@*v_r}xbxHJ{kymR_TBq)M)zJNxdXO_
zzdHB!HLtRr!Y6L~&-mot0}b`IX$9uC*S)p(luW$1w=1#BW^UUY+dKI}`*OlqY*|h}
zwSBu!WuJD+G8>x|)_q+rhPDdgocj)h9o=hFH_hh2g*mnp!e{Q?9}r@zCo$8u!T;QD
zwi68dX8m2h_oz|n-jA#c_bzz<(#G{@$=;3M`uFt3SlSAIjNE&|>y)i8pR}E&shF({
z*TlW$zu4^-%QV_v>+Z7tcU^qnsk&!-e{Ip<`~JrEJqcICZ8urD+3XXr+bdwY)Mjgy
z?7j<1zBYH3@7z13e1px)V4i&uONDH|xV+i3yyx{^d4nT1?%gW3F;4_-{URprmI+*7
z`y_w!UY*VSw#61<dv7otvAy;rVQ-eowY>(nFWQ{=@zdst!wH*pzt-%n+jqpK+jRTh
zH_8`mH8mXe`sq6FmAG4Iv%vD6b;iG`do*Sow7KCXvo}Pk+xGCL9X3Ymh4wnDyV!2{
zczAd5TTdGWc9A{imCbwe>~`6fyUXp{eEs1b-zSsyHVAv$F0tIR_e{?IJq)JvcCNWC
zVJ8%tvsdZtcH0IMOWTL)_ifK9thLSEU%t2HCYx=>MOoV`>aT5g<k;CTK0Um5f8hE(
zmSU-U{k;zCo#OZ1CZu=M-Y5D(dl|kP+g4by>`nMqzPC8w@Lr|8ymk!L0X8g|%WYlE
z^z5>}XzVc-Ub$DifZa||IbrYFA8Tx<ta`kc<K{{071bMTW91t6GR#|RJ85OdUZtFq
zwhZS}ch^iS*!wE?udUQxf7>RbgVvHSE%uhm@Y$W;EVeJ|+z#8O&IB8QEgSZ}-)d(g
z(9>xB#Qgs*Y0dJzLGgEOq_>OP*>Gp=ee$H$wn3HEu3?ev?n;{#dl-eb?#(+Ux=&Ml
zg{_)*ylstevF*I@!!|o!p4+Q_<BKiRA%(r6Iv4jW-r#5Z<SV1CLUfGHj0an7E&mnm
z-FZW856k<ldwAq6Y+lY}x7$**&?ceY+Io6(sIA7o4SN_`TX$LS6}Dqu-D&Io#$YdF
z%*DOpEy8w__}1Gx_?+L<rz~kVLCbURmmP6?BNhnSE|yl^Bem|}9-k)hy&)dvwrl=N
z+WH%?>@x^lwb$&zCab8Rb+&sWKksFq%(7Q3-gWN}h7)^~o^QAPu5!)Rpn=(Tj*pEk
z!{nP*es4e88a=JryNNl)*2#)(Z>YE5-aT7Z@71vL+B5lDw5{s&CDs?d_3kxTw`+Hf
zZ<norfTG>r^!GNO<1X0rKH{_8C#twNMs1GmBVln{x4Z?m&oA5Be)ZV3TXXHUJ(^c6
z_6o5#+2$%Q-FrHte6PpBd3%_6nD*Kq`LUPh@@AVqT=Vt>-wxT^T)%nm-Z&Y%D^+c_
zO9E%@b(yl%#=zO$w&Nq;o|HukZTUn5_Fc(m+{e(UW;6NazP&dBl=kf2z_7QW_M6R$
zs36<U%iC?gM0MEmH6GkMNAb+w%!pQ-O>JU!4>pMItFF`AyUgU=o}PvYd*p?)_6j&X
z-+OV|>b-n5jJ9<)+ig!h*|2wuv5#%bq|7}BLR|N#wVm9%M{3JnTh~dp7tDUxgzVnA
z_y3i7yL%4E*>N(Q+S73=&&Ij(sBID7ecLOu7u()3zqD62kjKuTxXqSrU7*dE-F-Hh
zsR?^yJh*Iy8rIreb6;sI*KvAp#l-`*UbP)Iyz#7dVd)3<-rK-ww`p&qEx*yZJu{@P
z?up@5v<*49a_{5V*EZQJxa>3!3EAo!&f4oC&2ASEUT#}I_nb|UpqedHo66q((+}<4
zccO9c^O*B{HJ+*13SVEd_r)ABoBa4zThEt=ZG+PeTVL5)YI}n3$)3yh348PZ-`k_&
zu*jBS-M&3Lc{%rK9FN~?aOKTjcSUKtX$x6w%fe>weXljewjsZG&;55w`y|hOw{dcJ
zw|#W;<sOIZU%PK;mfEt!E#7<mz&hI>dEtBXCy3cD7M!^^Z0}p^g^%CbC~rBr_r}vD
zHY-oQ+^c8EWqUupeebdNH}|A0&$O{B*s<4qCI7ykFOqBuHr3i@D(l#Yyz;PR4STWs
zz^1u-7}l}wZCJo)_by_Gt<8rP8?)ogwnnSi_daSlVB2v1->!HiOWQANd~97dY}+GU
zXSdf+fYJ5{zq;*n_K!9au6nk<-3faQ!q{vzmU!FND!1EyG4$T+n!VW8Wpa^iZRhm8
z89BXs4K82bYp_h%F7_akZN|zIHs^0{+aq{c)$V|^-ku=6<<<^KRlDPIyY{lH&DcAq
ztj5;YK6USbhpo2d-Gckpth>JF7$1w>il03;7pL0nImcpU<MVd@-hDx{_SR&z+U}k-
zYwz5r+xM#Xir5|yGPiwPy3uCRm8W~VW+mHd_-?n&+PBBH^uDC6bh*P`_Pgx+jO8cW
zzMSZ{H{<rYy-Sw3?agZ1x;H>P%ElnGZ%@sF?mc0yMSCT@p4xiu<+I~u+F~2g&#=cb
zrE9O!rpdOQLh5!G;_up+PLtS|?~%4wM0=*KdRW}vNd}*6<CqTEUdz2<D=i&lyK3j<
zy`p_b_a=St*lWn_W^16df6s&i#<uGp9JbAv*0*PkZiQ|7A=^DAZ#URZFSfA>6|c26
zd)8`eV!z2&DxPicG@fbJ7drRsiR)*y)mm${cY=NI-p$PlwvuuOZF@{s?mfn^$o5r*
z*ItR|mbRAn?(MZLX|S<Wm$rL$h;Q$%!2Nq2mY?1GpnUzFMk6u174G7DmQC7g%kyBV
z4RgEUUJXgneQnE3_x}E|Zr^m{3;QJ8jrMFVXtcTddEP!H^YeR~j3(^m$>P|z?S%f`
zI{PzwX6@(Nm)Eg(kKj_Kz4L2J_vwCb-`mIaaPLe1m3xmEEZmzZHDh06)R)~R#xM4{
zU!1x>_tfjXXYFR#EoFPU@0f|wzWbSXZPqIB*suP?xbJ}7PFq7kwtXg{QFhWxpV%!w
z9ky?$N0tra`HtNO&W7z<rMbvv!<Wf>AFpJwy_Ge0uXvW{{sfiY{fx$o_Zm)8v!A<x
z)js(1?S0~c_WNb?MD_`azuEie?S7l(ArtI&Ht(}7=smdi(@l2U`bKu!{ae#*@Ao{|
z`{Z-_zHc&(dnKMZ+T5PJd++YIa{HHidG1^IrDAW&vZr?5OS$(&HTUejl=f>cXHLuB
zbIYCgB}{VO^OgVEKJ^)GyZ0^)-TSO^-kvRgzwa$b@3HpRdA0Z7ABBCFHQD#&tkJNJ
zJ<GYfIdH?i)d2x}eM&#?O`WT|&qH*^Ue?oIHfz4i>~mdLv2R=cI-A0o6ZU!kxV=we
z>*9Uywx-+X>=)kW(RpR>n}^){UQb}Q(`hWT&tQt%_p`QZ-;4>T_UU!s-#gJdaPKwN
zfIU1ZI{Th+Ki;=>TKxW90X+NSHq`7}6QjEK+G7WMrZ?yJP3U0VzkkA#U43~H_S^Sx
z@9Wo{YU5C`+L9-(XV128{dQ$GbM}gJZLkyDAhln8!;4+Z&#LcRY5i<()XmoYClr`%
z)c>#FCnI}d@9PQ0do@h{?`667e((0rM*Gaxez((BIJI|j;QhT#PCNH@{(ZbVZKcNE
zi9GT9SZ{Fe<0^I8`_O93Uj0L^d%iqu-1~1^-R?v7cJ}_+Qu{76p4&T9=hD8tsq6N*
zezM!EbIE^y)tt?C+BVnrF6UmmH}GwN4byfh`+q5~Y>W4A+xOAGd#`VR#=d92Htn^%
zmuoj6nsI+1>n)r4LS_4IFY(#OvrBj1DUCz>?ulI9n?Lo&?)*1)_F0Vo_v-9Bw^w7E
z@7{nkew%N#8~0WjoZOqadY%nO-@&~KT+8<{TrJvnQ)<E<-mV4vYV|bt22^j`XF2DV
z&BkLq`!c@V+*=>dX}5`S+TIJ-?(LP^GIxLJ6)}6y^@sNf{8rqj*vGv0+}Q<tt)^ez
z`%-(e{gqwK)>=0X?VWjCXV1CVioMyl1NRn|+~3P|Wa0j%*Npqu&5+pFecizRMRV%j
zS26ti0(IH!Oc^5f*PjaC>v^tXpZ)ixd-H{FTJv9z*|$z8!j^5p+Pw+(;x@nYl=f|3
zlCw|aQpaAK^SAAUa+UWfRTS@IF`cqscjbb;^&v|8e!p2``zI%RAFo}Uee$tL>)jLU
z_ii{WyU#nDV_#?8p}ofIK3R)ov+VowOKk7qC1!io+M4a7yq#@oJd*ZCWQFe8s`Jz4
z+>w)e8Dcv2)rGq6*>p#3pY6sR`x$!EY=c9$?E8Oo?|%Ek4Ez5VoY;5z)6P9Nt~J`W
zuZ-WzDJHp3qVNA+o{-!<tCg<r{i8W&FOSKpy$0E5ZGsLz+xsCzZSTn)&uy#MS?uFd
z|7`Q$c=5h-6SwY5Pz%{Bplr71p`hQMuDkE;-Yws-@7J^ZeRe|1dp&2n+s5=AvlICw
zy7$av9h*%P9QVyG?%T8Fh{Yb0m#_BzZRg%oWU^uJmy_N5SylenwcNSBuR+9bAH&iI
z``nJo?zOg?vR8r4ZXe&Bd3&cDo!#daSG4C~MYzq^?ML^jBy{ch`%hz^8QcE7svY0<
z_I1baZ8&#f&u6=Tdu4nA_dO3Q+bdc;ao^)V%Xdrt{AT0U-C!43YqXcGMQh*tZ9Vp3
zy^HsL@rd2`;X&cP<E_Rv;(zAv`(s?b_tJ@rd&}(Y_r5*-bN~09j{6S%Pu-jGYqg!d
z`HQ`CXB6(6UiNtJnU2N#?wcL3m7ZX-uR&DEVdby6`&VR2?7NVrxbN<!#C^5@t8Ell
zvhJJzP<Y>}xxD*s^<LaN<Li`tPcJ>)YvEUBTa>lL_Gb0hJ!kSb?R$8VY_l>&_ib3e
zabH*7LMx6><@+`UJMLSv`iuQW$JV{dx3}$W6Y$zU=ZM$dn@1n-S*87I|Hs$qdv`Zn
z-gCR--rk?$+xBW#Htr2u#JcZ8@xuLy3$E=BInTOpR?EbF`*ttcD>LEr-h~>=_f;&I
zWB27b_ukY+>ud~qCHGb4EVi5SE!ZaMg806kpqzb|YL4zpOfTD0!Tft4_m;PI9<>_#
zH~Tj3Psx9~FUNh#KD848`(#Tptd&bW_hqwi?bpA{vv2bv1>55*AKEdk?b`d1!`V)=
zCS<RE<Q)6*D=qsDDX!d?H=Ean=Vr@Zle#~9s%wwiNVT8YyEyUk9!}o-`_doj?Dbt|
zVt+%6f8W}DNA?IC?X>&y($LoJdV#H1N#ve_+^u_s^0fDC_IhnoWX7@g<DbU;#~ml_
zpI#xb&-Bsl{md@Cc8p!y_Fc?fwEMkx(B6OHO?$L^-tA@8v$M(m{Acf#mDBceHQDU*
zvMJl!`{>tR&6me*UYz~7XI<HWeL8Bq`_?8h?=xZfzW4UlnEgV&$+kVkpZ4-paqSD5
z5x(ctET#SG+f?nidE{)j{S(^Hv$=m?TbZxzr}=d@8={`>Dd{=4=l$<Z`~Ee_*$VzW
zu~++D(4OO*^Y_>n&9^yIczkc)epx%06oGyF#XsA%O|9R1v2)(uO}}OL?+*yGy~uNZ
zuY_XAz6U>h_vKiI?UtBRvG<up?w%VDR_&cvxMPpr>{t7=^40dud~|s4>O*a|c4x}>
zFE5|6FKb7sz39f3_Ir9H_sw|l!R`r@``)KNOZL9x=-n$+`+QHpKK*@7$KUT=e9V0B
z)x$scGM{DNH~Yr&y=8MP_ZbR5-QRUsbnjRG%3a@2>+M;0L&W~Sqr={Ri=y_0uRF81
z@%Q(=6AKsZdok&zO?!9e-kgOF`#y(D*?pZTxsPG?guUsSx_g)|{@K?ypJ(490mgkn
z68!ej@04uI?>@G@J(0&gqFZ;5)cWAP>>nQ5uI@O!H)xZ_{(@=mY?znNus4X@ZP%#3
zbl-8t2Yb~X&D`S<Be1XLmBPN98q>X*jVEl^gtl7spYONd9#~_4=&G~ry-!?r^?$?c
zQVpK$`#ZVU{%d&B-u(d+_kLr4yw~&mK70MD#rq$`3)_EIb=*6z_Mff&f!}*i)_3f6
z*e+;$debIbmV;;aoSeCSkNvk@`&HM6?px{Fy!W^7HapJN7kh2_Irmz5aqj)Nd;8w=
zd%E{5ox`>7^MPk}7kIR7AIB-|jdM`3GiBVncgB=;d-byf_t~$s*sECRx{p8k_1@;g
zr)@sP+}wBd(Ue`5mUs6qm#DDI*fHOBSDDd1=g)ubT|DRPOW1MHc3DF4KKYRLePO~^
z_f32BXrIpZAA48kD(vm(TxOfRL&;WJ{m}mIB>jC3at?b>J+iP@+$pqgcaPq_{|$@x
z`q)J5z2SMwo}u4w@3hqXy(x!-_Jyo@yk}W;?EbeWGwt2{MeWb8tGCq>6x*NrQe$u9
zUZ#CH9eMjLzEoQ??>J_c^+IXiX_3GC%56^XTi<eSuYHlw9?|a__SMBpZEfOD>{FLK
zvo|~D|K59^oA&OWv3+l^^Q`@Cr{wqjId#!KS#09ooSnM+i;cT&Q;l}n$p04J^Jk6u
z-Zl9X_D+~SVQ*2;%-w6smhJsw)wn0U{(y}ntL}b{y;A$;W?kMZ_{QG;>IsRxQ+&Ml
z9&|rwSNiOfolc?FUOi!1+kMTa_r`W=91uS`)voSl={|<}5BA=w&)63wsbj0bqHBL~
z@xOhlx}R-1zg*ro>7V7k+9R*_9#|i5^K{>Lo7IPx@BXsy{l21CuXY<*)!EJ|V%&GA
zqj7KP)K4}C-qzcGQsCKVp{BgAslRRChJ|bQMph*4{qK-qH^X@QUX3@6_VZlN?A6Om
zv0lNhWjkw+r~U0ziG9*4C-+4EP1@tYx@Z6Uk1F>6?WXTteq^gnv{0e#Lx$qL0Z;h$
z*S?nD<Lz>PZ(4q>?TUvJY@+Qd?O8-LZC@Ys-XF83!2Ya>$=-V9L;JE1eYKg+zi}V0
zL-#%})xLfIHtw}~z?`+WKc!-C(>wls7i5I@>7+>SSNXJgFKZ!(U7Q1tjgiQMeG<&l
zd)%`555#+1+`GcYV6VO4qP-C{fA>g4-`&f8LGwVO$l878Wq0>0ytA{3s4BDVnbW)X
z#um1Hs~tY>?eJjQoBx0PUTfX$d$YD|+<W9gg`JIK+MWr#Om-_aHQ0tYFSChQ)@S{p
z#%u30UKQJED<$_`kMP@XUwvyY!-W2Qx9>69Hy1s%+4-r$Hfx^yUfcPWdzO9^->0K!
zwNIo-#P)}!^j=v_*L};Hj_hISSz*KbZO7h8$G`49`Iu>6Y4+#6xnJt`aR_JF&fqK9
zXY6FO?=_3!K2dGIz3pdq?$Om+Wn&}Hw7-b`&E7RSITnZZ{@iQB%D7*@{Ni2-={)<h
zGyC_wc`jnpSh&gV7>o42nPsf|8ZS%k{eNG}J}<Ipe`NRGeOXf`@45Ei^j@yGjeE<j
zRrgs`iS7$}c5bhA>g|1Nt%CNN1zp)QZL`+CITt?c`cS&Xx^HirO~YTseZ_w-*hHG<
z?{!#lV(-m=T>Ciw#q18?Zr=CK=+0hE-hzF)m526gJbtq`yt&JIyY2(K1@F7|aq`vd
z`|zP+|GQ~7cF)Y+vG=m%E1P8@6Zd_+_I{65in878KlAP1>3`oV{Q1!S`3uBswT{{B
zQ#vKIuaYa>ra#wzf121#+wDJC4n()^wVN2?zPG~ftnK-_5Zj9{ocHawSiQ&Tm-yZV
zjOX?}UtYEMSCGH`S}j}aS(#k>B15;^T-i8j|E-XBcH3np?Ol3r^3GRki}waEZQ9E_
zJ8ItrzqPhO+>`cCu;ttLB-nSK!(4ZJr3GL1rcJ2aQ`~jW-tJD!-dZV@oqOk5+Ftiu
zYU^|+dhf|HDSMy9zk3oTCHKXYAG2Yv_-?<tyK#?(_{qKDfoJyXf5@>vIp21VxwzRr
z_HSSI+D-M@zuwT_X1^8VK7Tgvee8bm`|Hlk*gLCrg<Vl{)PA$oU+msU+}Y2pEopZt
zY1Ll&0*8Goc5v_Io!_^A$=a5E&Wpb7y;JdTPm|FdyAJz`y^aDM_P3tB-0M~xvyW$S
z#lA%eU3OJ=NA_+l6WsU2v%yBOj&U!Gz}dYTl7V~gHq`Fp>3g*&IH=Ly{hHg}(rjhh
zUra~#ZE6?Z_atoYUXLqXHgc~v?Rmp)vsY=>cN?pF%l8;qo!z&rV%t8YqQbqJuAld7
zexJ6#_gSSi%gvd4LtYfyexCh!Z+qU^y^6v+_G~jcw^#MhEt~n@SM9rZS84Bt<aFB)
zk1BS*Wtp?z=;5q=i9aIu={wA~x>NpfZ?*freahb@?L~4r_o+7X?F)J;xOaB7*S@B8
z=k{mnv+vFNw#MFeKF7Xes}}4P%4gmyyu4?xW@w|8rNfbZieI?*Uj4OvZ-sL3zK>~X
zHhX=h?p=23+TM2M4tp`db9?vh%(lOlxM!~p<GsBPI#=!6`R9WDshuh|nnlw4ZY12c
zae5(bBjGS%Z(rRVTh;JWwqIRl?v>CAvD@`%h24@{6K%g+AKZIO_tE~x+w%9-yFT6j
zy=C{_nOSRXCQn_zkL}9-y{g_5_Ez}t?wj=Nf{jeM#=ZqldTnmVGucSk&9^S)tlI0K
z(X#7@!Hhkp`Bv;(U>0Y4YsPHbg6Yw=)>Dt#F<8jiRtU}BQ_N<$cgeDF+XWSKcew_d
z?yc-_-`g^OhwYSz8e4{sGj@h~ckX3iO5F2jTg$#Bf!ww$W0P%EA5FJoNV45EJ2z$z
zqjUAXjRHM;y3_LaE(p4AcXjU}8v&;Mwp+Iz+{@r!YNx&E#XgPu+_r8St$SDORJFDL
zHEnN}S%U34chP+h{8elOl<(UZZVa+}{eZ<jWDAdN!^%yzqV{L)K7W0@x5@mRoxtH}
z8>an}_x4|vwR^gR*?yYSJ{t}1-hJVWD{MV$*V$RGTDW_;uk_v+v7LJhWTb7r^sKRc
z{%ObFX_gUo8pn6+oe;QjkD1ZUecyCS_hx)Qv4?+u-`-hg5AJ<nZ)UxcYs=mT$v5o|
z9iOw;b8o}m%pV{2Dx}KUP2ihmy<;VtjbP-Nz329e*s-tf+Iw`<p}l`^`Px0Xdwwr(
zB-h>zn-=a_{nNxI=j0mOxAW%Oo!w}=ciM`@d)ZbjvbC~%yjP=P!rlh$Y5UG4NZGCl
znY35#OZ8rXeDgi`-NX0w+-b78t*yGRH`{jKhTbK%Iulmy&AQaRcfr}K*0<K2vg7eP
zxv!v(Z|?%5E!HW2Lv3sS1@C*bbLPIE0v`JsqPE-K66~<sGIh7@R+g;271!JM@&B8@
zH#3^WKH);4^^w;_b`tNl?z#6&XYW$x4>oOEzuARq-QM>=Zt8B9q)yw6No#EzZXVhz
z+wp9#&ojS$c1O1ET^w?KUt->;y$k1w*anpE*nO?ebMJ#Qj=R4*Snex*`g70ogWUU~
zE>7S3<kHqX>{{(MC#&=Jh92E)$0*&ncjvdWdn25`+BL2cv$YpGw0Fn#n|68^?%O@N
zE3o&#lm|8wx1{V1Sk!Dc@#IFk13C9?E;;e+RcczZ_uDdQJD;sf_C1tM+1qE7vrlbX
z_g=o2b8J_$G})AV%-!3?Ah5UT*CU&=Ztb=T&$Mkti|*OBbgtgR#KCKuW81X1i}AqT
z%x&Lo*|ghjt)wjWPD<{#EqSnfuU=8x-YD6Ky<b1-*<F~pd(Rc=$$M-vzU+BXD`LCw
z%j~@y-p$z^eRku%^~WCXdpc$RzE8Jb?MXeww)fI{Uz?qpc6)=p+V^-oKe+cnz1i--
z%~ST?%KET(gQK|p=EdoI>x201&!lAAbvowkonF;#SF9Cg`6EHjR<2jiF6r4!+w_g9
zHmnMB_NJ|Dw!P1#vezzqtL-Aec3XqfS++9Fn`}ew@$EfzbN}8Y^)|NG68G#YxUqP5
zM698$+M)P;<*HNn_Uuu#6}MSpr_eTW?}e3|`=?KuZhLha)4ofK7VgRL(Y6tI$Y<NF
z^~A<bak=e^FpE86rr-81ef7}J-!6A=#9`JwUru`221z#WHIiVo?ci>-U7-7T_lnic
zwxN3a_O59RwzbvL-FrMG$4=txGrJi&s{0)BZ`e#wcw(&~GIP(7V1~W?AMfs+_*=+U
z#x7tl!}^-t3{~Q`%bVuzVbN&a!&0(zPpP1!UAfiKy&O4hwgOA9+sx?HvX#)6u={rO
z#hyPq8Ex0SG}|ZkN^0-hrR(-iiOSr)US{6j10jF*F7R-(W#?+xx2%1k?UoK6JDF=2
zY~>8L+NyLd+w*be2iw^ljdr<T=Iq;cRM^(FL&?tlvG<-Ghkn>_8_3&nSuEVs^y7=I
zP~Nh=FQ%TfT_Njl6XrW-N5I<nJrY%QwnZ1#?@f51Z|m@E(q50kpEe0!llF)fGw+>c
zt7MnMxO}fzTlt<P#TRXhtvA>OY)G?x_h`o6z?+Bm^)6XqJMYM4+mG|-*lJ1!?!Bn>
z$o5>$p}qPK4()aLyW7Uf{nS4G<c57uBzg9(-Q}@=P3zvhK1ba5GW<^3C3t<A^}bVl
z`|cQj+oQa&Xx|huCA*UQXZOD3y<`33X|HW(;fuWw6x(fB53ua*F#T@Buzc?Bn9ezN
z3KjXbc_mV|3X5mi`afyj7gr}|ck9%Iy-xyaZOd0r+k1dr*XFyH%sx%F^S0tEy7n>D
zl<ta|v25=R*C;!V9p*L(=U?n@cKg0h>9W|~%QGigwM((u&H6NR@0GJJ_N<!IyYJs;
z{{7F{5AE3?s$pZA-McR*bMBr4$|<({YnAP^zV5aYTB^0@QP`1vN9<4RZCkV0_CwPr
zn|BA!+umsIu$eIN%D%d}llIQxe7{e7ozuP}8;WhkOda<fJU7GkjX|Vs`qBM+%dUyr
zt^7ZGUt8V%eNWX~_i~vx?|t^PcVEKxUwbBBU1rA@yk>7C&+2{mUftT$xnh#tUEf!G
zqfUzNopg8aUOA2i+Yd~pd#=2U+$-=+*w!F?<KDP4-uo_aZm_v~#CorQxuU&DqWGR2
zpW61c$+PXfrF74B0mpP(h6%g&d=opc_r1$&+s7STb{8fcwBC06g<a#(d^?+L@qN;d
zcH2J74BB_O=gMx&*?+BHte?2oK+eN<PWzKRI>rla7Hw&?6?h<HyGpao_G^5v9m`!i
z+XoU%d(U*I+R6kU-{Y!3d7tBjIy;H{S+*7&^K6AwMECyv{m1Ucg=PEBq}ba2N`1XI
zWa%o~JC8kW|5=3CNGx4o<5xCy@BQg=cDiqc?HHU+?$LhQxwmk4(%xh7D{Z&^oWD1;
zY>J)FDdWA7ixuqceoozcp*hI5jIq~FWBuBFrvj_@&WYS+TcYuNkLFxXTZd`&dkP!X
zY~5Ba-+SfGPFn|ev)$fY8FqE)vu)4q<=Xo;`P)8+bD6eH*B{#%J>G7&qb0#sA@QXh
zgPz$=yJoh13(gnVNStrk(@+v>9mjQW@4WLudo`Cc?<@GYaCbn-t-bE8oA<i0McTfa
z*|9g{sgtegxz9FHmTGqK7t{B?;h(m*DV}woe?YbEzSuu|1AdF{dr%f()53IZ51Z8n
zn+F_fHYRcQw$m=C?KNMQU|VLrYH#i2ZMF?RXWMoraO|^}Z?`#_^>*)y7rXcB?G@R}
z*I{gz(JZvLKjF#VO-<kS#<_~@%@uLld&Kqpp3`+nd#m?v-^=q_dN0EPxjiP!TWlx%
z+P~NPT8Xt+;?cc2dw1@&KeS`7lIPaFGj6Hdx}ION_t&~g+kaPg?-eXNz1Lz>*Peo`
z7@OvL>3zPHb8R0bFzz$zXtq_VuD0dsIl5=_mEOHS_Fmq*g`t0s+PliVSHzC*U2ayp
z*C|MEU)1>m8-b*IHtJU$_eP)GVtuP($KDHD{#mCtNA6|xK4N=*-4xrbP5pb5&8F;K
z-p{)4o%x5oEv8+2_xxb84L$eOW{>#8J-7L;+5A7hd~Zj!*dF^8$M!VoMeH?Yo4beq
z$p@RM=>|6cv?tg$tT}62_HeK5jJLWr4PUfu!{*5DO^D9i`{PQn&HN8b_8wXBZBLwn
zj@`LW8hgUerR`OC=xBRw0>j=X4c2?pYU1}E*wVSzMncZ^i(#QP_uNA^B@<otR*1&!
zS;5I*=XQFDt%BEfo2~o`d(YqIwPXA_Z||<N3VRJc^V=~@yS7W@>m}Qzj0^UDWir^E
z_gi@18=WJz|5lprUEN`@w`h&Q&IOT9yZ_AHXY1&_YA?ezKbvWV{kyNN*V_A=b>Uu%
zvVuLyhn#F|Y;$d%bc@@1JifX&+fiZP!8KcKws=eL)p*HmD_K5k@1!TZ`{po**%%aS
z?KyKi%eLW3kj?Is6ZRhC(6^hs?BibX6WsgWy_#y9a_`NaYbVa_G5R&lmU-$5n*i?>
zd%7<@-Fq!+)86femG<)75!$=kGJY?&>$JTc$`@@4+1YH(B@Aru@I=`%uuk1O!&cSy
z)V5W7C-S!Lu1RLG(<r{Y_k5*;^`k`Tz4Iq-vgOFAu&L|Vu=mH*DZA&`$J&ahZLnRk
zZRVaIs%f?liznFr@|kRVRO-pzyz|0)dA}am>kwdMePsPIn}lmqZAD7w+pN6u!#1``
zcTd!{7q-@CKiYhf6R~yF%h<c3p?&X`LI+zrY1h5+&8PP!<aOB|Sul4`?VpL(+h3;b
z4R}1yCM;>LHT!KIo9Cq>`((G*+nQ}ZV!P?|n!RjgNqZHtRrU#<UANcAeYZ_33(LL<
ztF>%yiC?f4$u6}08|QADY;k<g)RtE^r|z$_{e7!t&xDVaHg?^@dv_d`w@oWZ-rIL*
zfh|w%2HOV)5B3QEd1<2+&%Ku=PRUmLm$GfvuKjy|$?vpPIV`hx$3Z*W*tQdUBbz4e
z{rqHxt(H8GO~$9Wdp7)AV|y#BWlzuqfxQgprtj6hX=+>I{dZ5>KHa@uQN6Z|ZfyJ9
z1!mYPKGd-DTQq&|EzS156=l7)>3U-OYNyTFdqwHto@q1J+gS85*z8N_*qh;Hy{E~{
zYp;KB{a%MzzxO;zy<%IyX=hupY|&mD)Be2^e^=U6@JsGxo4V5`VUhlBMvbR?X6c{W
z+riDc=aS{sy@?-{_s$VnWBc_q@4mMS>TQ=9EwFVCx83{Tzszp+3YEPUHXeKHZB1+y
z<5>50$!*^2prmcfdwaEQo7&esZe0DgMr(KORhf6edg+oMdwF)|?!9_p%HG%SZ|pTl
zTChhTvU!h-ZS&rM*Eek1^q23oI=0<*!tc{IB`ijJxn{hz5sFf=Ezmh<Qx(u)d+ci8
z-U&kMY{Nse_OiY0x6QZRV#~bvhOM>vS=+a!eYVzDR_#7=VY%&syW6dcr*GTC!YjA0
z;6|S<!zTk<U*n%PQk^=s=i(S`85GQSrg%@bkv+w4_jWmho$wOVz1j2R?CjsN?_=NY
zXMKFxguR^4X74SU&1Uz8V}bQP;Ut^6c9M3jZx8I1d%(GS{_cyma&z7GGOGyMt&QGk
zoA#`2Z~Q~HeOjEYd&_^-?R9uFaj%LVgKd%e8k+)#K3lPzy?ZV!S+yr)I=`Lh_Wiqe
zUie^ZS!iY((S6Zo*Y90>vmB=HomnfhPb#x;ub;yN+h@N5_kMS4wq0Yf(stQ)i@g)%
zJZ#+C9rp?^K4t4_6k?NnU)i=b`~03WndkS02hOzJ(6V|@^^x$s{x^d5h|45cc`To_
z_p^?goz6Bb8)aRwz01$Z+a=d5-jlIljcq}dg01$QyEZ9%S@)WT?cAfBE4TNa<aV1E
z3-;}mjyk$`q0w#IIKRfd8*Pv7Szk8O_W#Gbwx8sZ_Z(Y)aqog{&uw-#B<<ah(YVL$
z^PIhpdREw)uG_sg_QCeO6?d20-Y@xVv!gu1=I53>wmPe8ZH?k&_OiRa+v{b%$hLYP
zi=DxpQ#MhV5w-;p?RyO5TlOAbyUn&M_NDD{HRXNYX?OSfY?*4Kd#u*>mP!0xrqo4y
z|1S60d-<%--WhL?@3qO;X1k!{)$Ua~GW!Hxw(ngtZ^d3M`D3;P_ha@hx;E41z>^1i
zdu62TLRf`tZ(OXjVKCF)X}o%^ZPRkqy+>He_O=`e+k30XaqlH(DO-c^y}Q}&RomQB
zI<S|)P05xquxgLqH^sdg!rC?==O^0mSl-*SvHJSnVoBM3d%h^zitKK)T~vE&FMr&V
zy@e0C_dI&Suy5MuNqbWa8umu{x7hA{^W4_6c*<UP%XNDnZ|SvVnCW0WqwnUf?lZA_
z57>XTdN9#_Zwv1Sn+D#--M+Pa`|h#$?Y(0oy;nhvZQta8g1zmJKJJ~}-Lz+lwyf=j
zX<KdQ`_=9}k+Ej)G|h#!tEx8H&e`&Q?}U}pY<El-+T+3Awx_)E(e59mPiza_me^*P
z-`TTd0jn*;(ffNXH}dV<A%1yJq+`lnqln|S4EGIfl2+ZeJ&=^XTUY7e-VH%hZ3DMW
z+56?Y+}`PFt$U`}@$Z#){ckgEz4jjY%QyEPcqC)P_<(Jn=TXVM=CeHa7C#N$yI|UX
z8->NIY(8|?*f7i#wPtuAXfx+=*It9zZMKi5yxPsNi`mXXkZX^>*nHbp)^>Y+|A_Bv
zV6xxEb7q_E1BD&7hgUGzm~k-eb=?)ew?lI2o)!E_wpDqhwxZ_xdnepJX`@>3#U@$%
z&fb*mb-OKs)NOh>m+j36?6Zl9^|W2O^xfVUS{yd+Zy)dZ_V)XpV{032Uj$Uzo=n!?
z%f8Idw&B0|uBx8dws-b*+QxgHx3PGvuq#BWeDA)My|!6?-21Y(G3?#*bI+cc%}q89
zE05ZAG<)u~y0m!j2SvU;*@5Twmil|!-d$vD`@*=*_FpCA-U*j4+3xY=u#MR&x7SYU
zsBP@0iF+F=HrNP0+pxDm=CO5w`3su`23u?kZkO(va*1z`+s_%c4QFTXWvde3`{LeR
z+nBBmHXp<`TF>a0+V}O=FWUo?X4neO<J==I<GEKOXvbay=E*ifO<QbFJQUb-V9Qb4
zLR$vgE%RquhuiPjt8(Gf9uKjWJv^V*+5|*C+*5PX#kO?Oygdre276~{9o>6oL#A!m
zoK?0}$y@g_>^NbW*c`cA=KPgCY%dG<&g#^#o#VE7Z|=cEd)F#7TF+uv*r#&k>z+Bw
z{A`<Z4fc8bT(gHG+ui2FT@9ND;U8@ZswVB-`|R`H6^RZuU%oZ$6}h{1Z_0-N+bai;
z+4|PL-_vo*cuyX`qFuVyYg>l}>U*boRPL3@vap@3Ex1p}tiyIsqLQuL-gaA$znQkj
zrWNe{5!<%+(!+JO4Y55o>u&JwQF3b7yX}em-X9m5Y;2l3cZW?|x9fLdt*we@u`LHf
zs!h#}c{XOhx9>4gceTCYy4<#T%e1|LqUZLm;dr;l+w1h+h?P6`uDbrvrY~xutxoC*
z+f}@r`|eEdwrx4#vbTys+wR%L347o5v+v~%-M?4#&JLSZk^*)G*Sc-_Sz2w3>N@r&
zpQzq@_v(qg|J_dR4VkEByT$ap?Pr;ey#ij*wgxild%5n<+53i>ZJ$uwwLR~Bjcu!h
znd}l6cJJAllCt+}d)(dvM`;_5rk#5&+O=%gFZyDeGF5f2$EohUw%h0I-T$|F&z0Yk
z_f}YP@70Or-8(<>p^XgJraduUEqmj)YwdM7w9?jK`y-nj4obEg_D-{Yu>IC<fv8}c
z{kPrsUYg6gmuc&ty;~)U_nuMOXj7xzVB_z1&Sr*&ne9ZsD|^r0I%X5nB4GDt)|NdF
z%((X+e!tz?scYI^9><M)zw~+8&X()n-Ml1h@5B>__a>+xu(j9UxA*MhMSIm|Y_d(e
zI%jW0N%Ec#i8*^rzMI)DEm^zQ!N|mR30IPB2;1B}LKO%1Y9t=A6_8uKSFSX0FVBsR
zJva9p+Z*y#&(`Rp!`|A2|9c&tueC0)J+xceRdnCCS?$*Em8R{rXyMu$VY$kNXPSkr
zNv^Z4<Z{owF3nbZh0}iSO6qB}{cd`8uj(|meSg(sZPvu_*iGR7ZN0Zzb+5{ihqgDL
zG22Bf=CED7o@ZZG_whYBrHA&Od-%tuwt~y{)5elLe)8RWJUXY_*eNpXol!U6M#F0U
z-kjqJwk(~UHZk`8wtp6K?^bX)zxRO1ESns?@VyBR`)yu^H0-gN#lP1`{($Y*-Jh+3
zBX-$_m73U?ekil46JB8(oBiJA$!zAmo@OnUKW{MXdtrLsW~RBc-6P)Jwy8yD_R4H{
zxi{~0zpYQ)F<bxBs(agbrT1xEyuIhewIzG6c6;nyID^-A))oJ~tG*t!Em${guls&s
z>)nD~d!K0X?yKQ7+1vAe<6f>g_xG-wpk%A*xpmJ9N98>)?gZ~Smpsw-l|k^{b!|)c
zGHm<3Q)buNy>*&wc3y4sY-Kif+D=&i%Z4rDm+hJrORcBvnr^*NBVw=bP1$`rR(aUo
zdn95b71yzMul5dGF(cKz3W}k&%hNa6?z9lIJ5hDqcGsh&dt0n{ZAF?+>|v0Q-S_p&
zqdjf~40c)@PuX65Gs#v?m~HPIX;s_(8Q*P8&$!xro87o~OGkxmoKD+b-*db6JX>|r
zcJAz1wu*@?d-aOv+eV*!ws%X+sy(la7TUgQmEC)(#(hs=OX%JeH`#5sF&gjHDLi3q
zo4a7|y5%3OmMqKLqjBx8O$qnZJq!-4wnsR__MU5Mw2hg&(e_Vv)ZQz{G;9l$?(e;-
z(X@A|Me>fkXI*<wvaHxUfpwPc@^csWF4=W^ugCf<+lGw8whqlIyIbyZ?#pU)*jwxQ
z+9v769@~Vs>unV-dhS({7u@wy%VqDAauGY#xH)@&{I0cWJuJT0aq;Kf9GoBa91J$v
zdqDVsja@kNzIidKdpR!M+|#i0tnG<ajkf2%nc5!OvSM#fmB${NkEXWQ8WZ+zh`MEy
zoO5E&i3^2$_5T*^efso>EyJu|yR$`R?bYS#+FPUDXPXukV7toizV(OJlHEZ)etV~s
z?zC0E_rPXC$W!Yd{<8aGIWO8CcyZ50$1%^g>5Kf{SqJTGCl|f4ji?ddw}a!cjfJ<;
z9uLRZJ<{#`wmBi|Y&yOt*@zvIvU}kF&}PcMx3*tyS?xI>Jl~dM;U}A;X5xDt^k&)`
z7g_FI6TZ!M#bQp|=dJVh{7kX3>2*!p`yp9;U#;_`-5LL@Z5Zlq+4Qt+uwB0U-CkDj
zhP}PB;`esBtL@v+By^x8q2G3EjHK<UZI}1Bw9c~+d~tH`-^+FT`idvn*c(;as=C?i
zW$q8!E4c2=?!z^q`xGDEu<etcxOYz6)jb)D?Dl3qy0c%H<;~v2F4p~(Uv}-USAS)_
zdsY1Y4U4br`;aGQv$<J!kI?d|_KK1#_9c8jywBA8jqUaoVh*2s`}PMeuG<$S^uU_;
z|5}?B`da(;o@3v4=if7%%LVebt{*w}aj;#sf2DhAe*mBO-aR#z`*|c^*!3(tyYD;y
z8++CKr+elY?c8g2Alr7~j70nCZ{O@Su3fjk_%QPxK?WiF%0SV54-fy^qj6Z=mgyPm
z-bG>^d*}3<*y_CrwBKJGv+vznhJAA`FSI{A&(B(PV$j~CRONkPi9h%L*mY{}3aP_;
z=V(goKj(dJub&nFzBO;F_T6-@vYE=&xmR_i*S;hBLiSGaN!wQ|ZfX1C;MBcs;*0m4
z7c8<p=p<@ec6Ik2Z`-3*FTU)ud8s0AZ&mHHZ_du7z3qbj`yRWR?Ke-HV)ylo>VEdW
zrh5c*kJ$82n``|_EYz;w(cAvz8<Ty14deG+C@!!+bnKsv>s9Z4T$L4j7vK4``>wHs
z{i2Ec?O$2m++Ff|?LMBc+P!)o#q9#VWZB>BXR}}A*}ZrFJzj?geG~R7C+ym*x8?AD
z#rZq;`cJE~XAHc)cWKqq{Z;HQ_IW-Qv*liwy?6HiK6{f$n|*7-Pwabh=hL2&ISRW<
zq|WYDh}yR2&h5+l1g-_{?c2rfpnIa%PH1EF{<|Rpwub~*cIO<PxcBna<p+e%%--u>
zP`A5rQIE}^35qt=w}kf`^4hm|`-AQK|2@~-mpAYFj+a_acB=n`t*<!s@3rlBvuj;w
zy<a0ye4i~x(H@5)gFTY}+4fj#&a?NNF>jx5(R$mXx77B1HZb42p!3@PxKfw>UQJu~
zm$97QmlTz>Z}s<idv`yXv6mrk+uja)gMGS_9PN7L!}bP9mhO>!$+J(}IoC#F^Y%TQ
z_kwqeJKeJWWMQ|rRIh2@dcmuEJr~&Q5i|?kYooDh?;FpZ)}La|TbtCs+d1F${$2%n
z7JKGLQv2@fu<R>+Qnugcarb_a<NSMhOilJaebv2Z$pwzRRR%SCLVqu?cPU-D$NC@N
zzP}Da`yTn<u(>eph<!=S?>#-r&-d}G&e(6eP+;FSgTHpMM|*8vI9=E`|M{MMYHd;n
zQh4v}t>2Qf@3e-{KJEpcdvnAe>~-MQ+;>o{a_{m_fp+Ir1o!gh+}?N1mUoYVUeA8p
zwFma>HPznxv*_1;SsB@VkDgw#;f{>j-|n<^-?{b)`)y+QY>(|cw6{N3a9^&Fgw3nT
zTzmZl_wAj`x@PaUH8=N~HH+*Gl;d&Oy!PInFQ(=;zXMbEw)?2;TYGZBo|LXl`}`ca
zZG$FF+~@h@f&HA18vAD5E8CZ)t-U`wzTK92f6H#;8P5B@Z`->^O3%+GLXCZ&*u06h
znQzbT-SeJzubK|?-W`luwx@3j*}A@O+$#`$b1$cx>%Pi;v-Ymop=i6W*?*s|MejaQ
z(Tcss{7U;2`<(U)?>)Bv&yk}2?AzP+Nxop(r@Q6o-ldb3_uI{~-E(Xf*M4QOsC}wV
zH+P?$;$#<fDP!;Jm6P_`eRH$*QnlP4r@(aJfY0Q8DOH;L=I!U-CwF0^P5-HN`z$AI
z+;?sc^PZ`-zxMrN__XJaL4fVPPxEX#cNFZ+krdu1`d)LNn!|xTg}R6LZcv!8JN?H=
z>poVky&KLM+CDZf-B-iNxi9v_w7som2Y2sZ{B&>E>p6QHdwyB{b_m~h;-KTc^QR5=
z2c6x!SA1diK7lo#_j2@{+#6zbaPQ{p)(4b}SJ@wD^|i??v$hLSyu5eE^iBI%qtDy;
zhaB7c{ww$X+S4=l>YQuX7bv`HU&$Rs+t=C^d-r`^v}f@h*L{B{+uE-3o3l4N_U*pf
zPhqwdp*Q!<oV3tp6OX#>`}ZGq&uV5q;Len4f9Bq{J+-DM_cl#mzt3{-YU}5_^Y-l*
z+F?6I>($=Q=5u>5e&^Wt`|Y%S0Ss+><!u=Em7WSYaCrC9-7b5S?Y>^PxOZZ4$ewj#
zj(e}a`(}4YdCmT9e$#ETq>t?Va_{M0{Yq2Y)~FME7ELbLzoqVx{q6OS_Zqk;?u$NL
zyzkozw*9;tLiXlqtL~F3R@j#pu6iKAr^KdaI@1ATzrFj*WOMi3fB$Cx`P{z!VYk@r
zn->=B^FC>~hkHu%KDLrg`?ss6?LClwWp8!V>ixoPO?y`azuvpg$91n$nA!e`O?Gx6
z1%dlBm_F}e<UhOjO2~~p<{z7EKOCL6_phI-?eFl+eJhyw_fF^0w$oU*X7Ad4^Y%_?
zW!`r~PjH_?!x@`{d6V}FL|w2h_#(Ym(ec7wr%xQV78$bpr!W`U=a`jPv3ED_XQ^3k
zpY{Lj-adAdz2{!)*?%r$+4u0)CF>K>i}&VjcHDbi=iQ!kxAJ|j=eAkhS+s1gXeZyk
zCC{(z=S!Ql?`G+ry?2&vu*xit-s``3v%QnyCHr+dGxn8F%G+l${hjrv@7?<}&s^AN
zZntS~o?){6t)E7FH3W6{G4%58b6{9+^Zm5ZK4tlD`~G%E?^*LQ$%akq%-(m+oO`qH
z{Mq|(Ro`A0#`^tQ%T)KgFTQSHYsX?Iqs6`_G{$nTv~_}AsH*VZo0@X_UTh59v-z)x
z&EXc6-MZpm_Da}w><x6}+#9E$Wv@7^a&Oy~j6KI2PVJpP>)gJF`p5TW2(Z~7To&N4
zj;CwyjkgB-OsaVId^r2oMriZFy~i`w?=5@sX79%tqWeNF9pAI??Rwi8Tdnu#f9BZd
z$<we`;@Mpr7n${YUoC!Mqp<3Nt%WM@o~z4r_kNfgXZtC6!Y(E^!+jUGG1>aApKr5n
z+WI{f=X-2ftv>D9Q0!`>5OjC<j<yJ!QxXcc8JF(cG(1V)tK2Va=ldsR@9KknwqgN=
zdn0$U?>p7bW3$CY!#2x5a_`@4)qU}&1NMGvQm~ER6l{BE{{KA|`_I@e;MKL=&+5Ln
zc5bchq20S}L$qe>eYc@+ubS7!y}ZZx_8btd+LQ8hi;e7^b9>%zezSMi@?F-4pK|Wo
z)4F-@iT3Avgq7oL5+6UbN@$(BS0uW7_Z{Uew!ZPfw(sL6?|t!R-5w9&HrqrCtG%zC
z+4m_J_}g9(&fF_|?#<p~K5Tn8#0lEG5c9EZn`3Bmi+7Q&gE`Ocb;p+O5nest*7nwG
z+ZuVVy$vbSmc=RzwiVpjdpQ=(usyvod#|x%&0b@jlD&M}x%TEfoMM~ZAF_ARp{`wv
z-&^cW-mGSOvpII}_PHx;eIMKGm5Ecam0Pf5@2Y)!_BJf3vT0-NvsJEA-TNWpsP%_J
zg}oW4eXT!V6yCc#sBzCs$v#`-W(HgHqm%YN+I?;>%c8BeVavMrwids(QQ}{3Ya8jf
zH{vv-O~U4VyA^x`Y&_SV+RL0+zjs3v(;n^#%l0t8xW6aldYtXFN<mwOr?>aSP55uS
zr*F!hQ`VBUpC;ViyCP@zo{WSydpEA&wEMNWY|n~w3vCOkSohk!JiPbH$$xvIv=sMF
z`N6wy$_-)Lq-#g^&icr+Z%tDE?jPLyZLhp}vG>pg+r8U<>ew#0ZEP#_^4wkp<%&H*
zhm-eSS@w3XSNguaoy~mv0*c?+$oX#GyRF9Ft|EtjpF;NK-PVy|wp%o3*nZ$--Y3Ah
zbGKwX*FMLHwR==2ec8)!X4%dKW*vL~ykE6vQDK;kxafDAHSH$0+{+*D{kgetFBi)p
z+qoS#_O5A4vNd}&eebP9{I<Ju^!C2V;@l_xblKk4>}J~&S@-t@w%hK#_M>o5_vX(w
z9FnEB|K{(t4ZXH&Z{y$7dkt2<wvDh3vpsuI#WvR9wN1r_j=f^<Pw#EFS!jE~sdx{|
z{f~RBZynlu*00xg(b*X`h0C|uGJkustF3N>WkB;5+kYQ~_Ek+jzE`~W@Lq<UD!asa
zg!Y8yO}D+&Vzk$AP3K-7Z42Aq48QgqjoP(WolV5{jL#06ON%XR&uv}2*Q9Ii-ZMEJ
zwgG=DZEcr0+Pnx|xc7|t;ynuW?R%y)oZj<)Grx`5g3mVThOBn=vsw0i__uQJa$cw1
zLM12n_LPV2GP$|SHt53Hy&AF~_oS?PzWYEc|DNwoGxt1st!-<tc=O(e2UqT8=xf{i
zKH<pT^~tAge*EFH)612%oh5(CW>F2(-UC6iY%_QL-ur%Or>)M#KHFbbOtuRgxc8n>
z5#Be&Xv-csx7&Lbm2=p=U4G71>pIuI6Z<*$R4hNa_Y%Wen;$oOZ6^t9+q!04-IJs=
z-S)+qeA{h8tM)EBsA;>!NOG^yD~-L52gL2>XfWBX`0c+(BDct<P9n^PQ}&e2*UKmO
z7F;>9`|(uneaE=B?fvk2t+j}b%U-6KE?c29>~>}H_iUDx|Fw1ac+yHqu-hi|{|;+`
z&tLa=1#|8*ui@V7H;=*A`QwtklTW<2IcyNJSBSI4wm6p4E>e(hub|eAJ#Q`*TRjg>
zwn_c6e{b9;g?$NcCfa@v-L;oN=e|vj=%&4%*QVQ)#?0P3f%)E^N6h`Uq0a*L&S2x-
zr(b+w@7~>;_ptm6+`D1Rdg~V=K6{J8m+$pGtzj!C^wf6$b(_6+PG8ugRd#OoRNeo3
zx3@d)6?A2`)hxSi6LqO)FTc+Rn+<wdwzb!^Z6(ZW_RQFjXsa`S?%uYWulF2&*R%6%
zGt0hp`{nGk-d?sXxLUq<%`DG7#|8H8-6<ZpSLpfqy)4p8Z1O{1*iN_}v`6>ilfA0p
zC3|K5a_`+#_kJ&HZn8~ls?*-}htu{7KCrMocc9bOn9*VHgItT<8>YUsslH~ix7_fz
zt!@8>z2#58+B)8xy?0l=h+Wo6Z(HT#g8PgL_S<e;FR<tQx%D>k%oVnV971-+6*V@=
zhqv1r&+FKon)A~3hvsJ+Q(2q6Hi!T3sd`wsxB8-hEx*QMTZI?AdxPdp-osVUYg_R`
zc<;sLc-!OZg?r~*%&|T8#>&ps^1AJ(xV?Mcss6U<;QDVX%KUcEk3v4%%hz_;HW+E|
zReb$>@2P`-_x9O8-s3L&%SIqy#`bUjGMn@T)_c<x@7Q!~mE3(aD`HQ}mC3ft{|fDU
zpmD-Rne)P4-J2)(w&*RkEqT0ZZ$rkpJ@4i6Yy))H?vAyP-q&dpzxT*ey}hh2IPDHP
z9kTH+`@HwVCrMj_iM}@1y4m*L$z<8PrDwOz#L0a7)^A#9Yxwn|b>c|}TQ^R<eGwe&
zd%bK=?k(KSyJx3E$6kiS?mfv@H|)*w&E3m%RKl)utK`1#>u=lq_&ddBPvK_U4;fB-
zlX6w}-U+?GmxJ}(o(I!e?KXHs*c5EG-qUbZ)M{3As7?6e?A^<P+IP$D-eQ|(-Lyw4
z_Kl5s0@q&d!b5xa{SL6z`h0g!>yN^{7Viu9I_!RLlYUd)#^xvEUZ${Zd*1)rz4wCQ
zed~SkGi_PKWA^-sO0de|ld-#gNW)gC-)irLkB@A;7NqQrQQ5G^L+`uIYV~Z}LOn&>
zZD057&1WpK)%z8_*Ji5Y-WkWFZH?>~?J)>b+%xfVzwI9%M!U3qQoGvJxc6;oJGocT
zzH9H^i%ol<v}@SrbQJCtIJwM*A(qkVrZ~Udi><zU!#$MuB&T@pEAl+PpX1$b+ga?%
zdoNd**r=?N-FNtP=HBkqx%)KUitasc!LV<C_0oMUGiCQ>6#UwoswccJ_8_y3QeX34
zw=~whH)=)qsebO;ul$U2pUr)t{qj8i`{&oP?rT3e&6er5mL1bNX4_@nQv3XMXYW(_
zvv{ADDBJ!MFQ?nQ5@p=K<W1*3-4=&^&X>IQWLs?5YgM>#Pe7FazF!wocOMh*w_O%-
z-X?jjo4s=AUc1kE=l0BJ+-1Kl;qcxsx^?@M-WS_vvrO7M#edJ<QlAXltM8obCBD4a
zS7s(`7i2B7ujKN*eV5ZS_B9-K+cW>_)_wEYf7n)Rez51)9G$&tJ2>|5nbEW_MIn9f
zxmR2F>i6|nf8|!(w<}$9pW$D}ednI5?*02B$oBF+TidD1U-!=4z_f4k{7$=FYKHr`
zXD-{Db;5Dq*EE~mb#@>2iqCA`+dk>~-U;8A?Ax!cVD)oe?B0avHG7wTW!Ss>->rR@
zk16jBedlf?I&an9Cr795bCW)}e{Jl}eaZ8U_xkNH*~@Invv>O4$$Nv^-0i-*J=yL1
zR$`x5Y1zKDWeocRx+-n$pXTi|+1+HzB44z3Gl#^!oKD{T%MPC2J6CDbzO;w2d!-JX
z+0X9pv;QRPB0Jru^?O;S^4ZsYU$JL};io<Sw&>Vr)<4{9_T`$5(!pbUzw5r+XP<O#
z-z9@}HlZ6%>`7P=YG=&ZvER_5)RsAR?Y{C1P1~RY5;ji+m26u~=kIUvv)VgJk<WhB
ziH?0YLLK)%bZfO074x?}^00mHp@(k!?RgsZ_WW48m)mgr-hxl!dwn|R?%BV@)%Ktb
z%YofDc=j<y8SNEV_<7%3^GADU&NSFt_&32WGuGAKaB|n)meY@H0-EghMkRCaXEr>y
zPx|_%eOo<>_U6nK+&Am}FB|#(h4y>?DDLx9s@lEn)#QD%N=^6a?8w-sQ4((>qN2N}
zV*<~<>x=yMNBq;>b9dRkJy+Tv+upg$ZQB-bYwxGrKHI`I9=r46_w21a?y&E!JlDP%
zTNU@FhlcN4AEmPI%T^n^YfAU_wrYOa`|swwy{~Wd*gd(YWt+SI=3ZsiX?8(Z?G7|7
z?%B6?Gt<7s#uxYQ%vRc~e9wG$*<7{*>8h$Whb=_+mHs$kGjZNCYoDBpc30g?_9{Cx
z+f`Ju+ijdDw(p$J;r-lK_-(g_b?uuaC9>b(9;f{`A@+Um=Q!-MaIoCB&*_6rhL+&I
zijCd-8mv5bi3TUyXgt)}ExcBJU-7=?y_d{x?a!RWzsJqu<=*oKoAz^Dtg{XH%d~IV
zo!$HUC9C$aSoYh#;P>0Jberkk%%kCZgJ0>|Iy`XL_hp)(ZCZ%MUMYsiJx?-z?ODX_
zxIet)(B5PwRa+SoY1>Irv-YlTXSNGJ&|<4wq-;BDvG?9TytDTAvBd9VDxSA5fa`$m
z&%lYcJAWVCQyJ5~w_$VtUgcT(dsP)!_Zm)4-+Suge_L<i^?N^UpSf4`{@J}>{ATXm
zF8pM#1hb&+(%-rJUfxLBH!am+ul<UUeQi;n_WcVuy7!mIn!RWDaoY$sy|!zb%)H<8
z^PjyPs()<eaOLm4mlU}F*g~Ftp6^%KFHF+icmBt%y;EKK>^LoCcK>GmZu2Nvc5l**
zWc!X=JO?;VT-?WS&BL~|AaviHnj8CEnD6gfys&AXT_w-H{x6^REtH$KmwEkx-M=1(
z?)}lxxwk5^dC$$>p1qPx6ZV$h;@CTF)dbr}1L=J&ht&7kEV;9{;nq8=paA|o={$?~
zIRtg?e_<@Qk4=exU&o!r`z6GB_c`m`+P89U(tf?_-8NmF6ZbjK*R;(}uiYomd13Es
zFAW=+cFz3^!yoQVmwmCXv9NeAljGEVRdfAq?H(N1H$Uvy{=^%4wmXY#?V1V<_Rr_@
z+<W2Cmc2dZZ2P#Y+xKpp&tbP^=dwLK#;^9-2dM0|EuCP$RC&EkXqMmJxnJMz{a9kS
z|A2$&zHZ|`d#^0twzpkj%0Atnc6;3xmF#Plx?#)nSiw$7UT>eOjNiVNchmPRUVLzG
z*$<_C8;(ocdFW*BzHWAC54(x#UXvf2_Y|w^?A779yzfJT=l*%EpZC}mHtb8hx6I!4
zx$<5o{YO?G9RBPt<LB7tKWFA1pZTozU9+C;{qOeBdRpE5y{a7od!?tD?&F)f+-~RM
zReQfQpR@Co_p|>l`ea}H@uT}#ayRc|=9#k3Z(8lX#lPA1>NQ-m`!n^?{>8nst%IX1
z_tu>2woefHyXPR=KRes|Jp2Bu#oJ$s3$x2VbJF(7!?L|^7p>dt;=6W_R6YA{HirJa
zi4lwUn9RPl@06VV-W%7}@85Ul!EU9ESNF=7#M)2hlG~Th%DGQbgmbUA>aM-%D^Kiw
zdR}hd<S<Pe#b=-QMY*!=@0WYIC(Y#dK1Ty-`&kA}`@;8X?|XJ?)}Div5B8en2<?mf
zyWhq|R@Ig>mT6zl_b+?5E;wWp;V8FfS)%8j1kX?V)?9Acmt?<YFK^JsJ)6GV*`x5!
z!&V?9$3Af(!#?$Xv3&tMEcP;b`0kGi<FZ$N&tSW8g7$v7ZQu7+8uRV_zh=wcJ^B2$
ze?R){J-RD+-{l*d_PYDs*!y$evb`n`O>Iu-HtmgIwb{E#qS9_w_QJiK8Ort#wp_Pf
zSRlTyvfFuoae}lhYqE~5{H2_I#}}U8%bmAzFV|X0+l6Jz_V#v6usI<cZ!0xnn{9z3
zr)_dVk?onsocjdAXCHX;WcR)|o(%Tcr`GR}dDyjw?_`%<O6^*kxb3Zb%;pyEef6Zn
za>>yndmp`K*&Fe_Zcn^j=H5APT5L~D-L~h7^CH^~9uBrQsxtN{{PwrK(4n_?Lcy{<
zHi?#dXR!X=(~zXO`%b`A+da2?Y@_3*@4cio(e^-KrHyyQ!o6?ivDrE}v+ddN`|F;Z
zhA3N~dFi$-@=3N1?3vbkj#lmcrn-8sO5(n~d-7-5thuy!uj1FJy?cy%Y#F9{TTSV7
zwEfYMyZ3<6I@<&8kF4JtPu^>+Ghy%E*6O`$?ya+}V_#!ia@g3m!*$;tmf3x_4dueS
zQnjl0njSyD_le4)y$nf9c557%yl2rFuDvs=O!jDAJGPhM?ycP$0*>uv@RHlpV6|fR
zgvM^07b!lr7YcsctTExR<+?k?R)wW|?-%`zHau)g_arpf*d~7t*&DF6&)RY4q`f;f
zp0=4`TW-V3y=U*UbEj-g?@roVW3X~h!97n~*#}Ga&S{ggJy3GWTBYm59@!)N_j;I2
zv|VOryI1aH;9l*DS$ka~Lv44IJ+iql_4S@8hi$eu)S_(M^0(OrNPV^bxw>!fBj(b*
zj}OG}ec;@_$L{o6TY*z1dm}1J_v~Vuw|9?&%HFm!r)`r0R@!oz@3&1jc)>Q~%vM{4
zWDZ*nCr{gwNy}}`)@-nCVxDNr;QV-x*TY%1ubfueHXMntTCy`^uhN7awiz-ZdnRl+
zZJQt#V$+b^YipsvXOnSH%XS-A<K7jy^Y>gkwbr(S{jg1yS<BuhZd>*^Y`AUnZ6fd9
z4xbNuME;$#ReLyXuZ2(co)i9}dnERrwe4EVy7vK-fsHSV#@@QKqI(nOPq1b9*|ayI
zY_sizI}`S-IPYfLl;g8EsCR<xqh+nO4hd7Ndp4f5{dan@Er-MQy*E}}wb>H9Y449E
zH}@!|oZ7p8Z~Wd{Y%Y60v>mnGxoP&^hAlk11j5eSeyCkx>-%%#-op<)_8xr^w>N6h
zoV|+~Bln)zRkAn4lhf9r*w-fMS<l`Rf1~zlJUwY+r)9L)uKBQSnDF_%6?T92oSEKX
z%d_+2o)4cM*a)18+;iaU9_zsPj=e$Evut;K(c63Dgv6dJ)%|-W$>!}na=pt|LSN5T
zzhdg%Wjg2f9(golZ;+$VUWubu_8f>v-YwA5Z7U#_w&zF7gFPQYVr{G6o!iSM8@6{x
z>{;6@SJ&AxO>4G&<F?ZFhy2bxA$J9By|`O#O&kkt*&{dZwaNXqXWiHNdrPKw+orJ0
z-jlXaac_mFfNfM7|K1s$r8XMxcI<IcIlkw?sixi5LbLa7&zQaUOIft-#FT@3-QP^z
zt03rY?eaFxHsPa;?FN_UHdE|MZ2iqL_exZqw4Jaq#5U!fwC$^6nY|yf>unh<LTq<j
zYu?*oJ=-?n!=k+(me1aE)5Bx$#QENPcj_+P%VB@sW|2hy-h^B&TN~L`wl`WMY@h7$
zu$@p9V{<KKh3$u*ntSj47TkO9K%4C||LwMG4;<Y4Lng}B)@|9|3VnCmwI}!Q<yaA5
z!?nEER$;+TYq!EvwpX52+0NK~X73Ij|2-V0e0v(MTG`%sR%?@_w|B2a{~DWs72&oT
z^Um6aJZRXnL9561((~DSAH22PQ~b$uujwVty#g=IY*#G&Z^QG@eD58xg?n}UF6>Rw
z?zBB{lGFCB&z`*<SNGaHn7DQ?ixsDB<jm;3LH(Y4Zv<|zt<ssgcmIwfdtZF1vYEji
zV`~t)-{!)~uDuNBglwEnpWS=MYO3vbleE1nwtcg?;gn&k66Ijau<PE=2TRiS9^jX^
z-DbMPcE#aPn;#XMY;_;Z*?TtP(B2i-cH698zIX2@{-b+WEDo?`nSR!mMQFNhqWzw|
zCwNZVFkG5!+cGQ9b_Mrhn+f?|dnU|?uw8dhZZF5?LpC#-1Z^2^&$1CX^}u=?U+3P4
znJe~4lpe92!tSxxD)Yo%nao<-tts30ZaOe&@4K~2Y;R2Iv|adn?_RFM<$GO9oAzpd
z-@bRoT36ewJ?*w)8ZEXbE?ex$vEQ)Q#U{{JXMUFLjQdi1f4t(c-6gwm@9ThtdmUCy
z+~u(4myN~^Hrt9*rnWULS$peWuGt$GIA^cM-YQ!U&o?&FhZpXx5ZJRP;m-ow1(!bC
z%-XPbuTQ4;-kjwpZJCTW*;?5DvuQ|MwD-cq9$Pib^|n(_uihK*&SFnQm4WSo7+V_&
z(Rq6_Zspk;y-VCXL)zE2_lW!65Kl{6241s0Gq`WqxQn0Pt8lMvPqgddy$suYZC*GP
z*)Dk8VUxRV*4{tQJ8d83&#-04_`2uE?Uy!7pDy29v7vI$q6Oz{Bkij83Pj!8-Eh-?
zcg+b~+XGk9ch7m(VjHpKsEx$uX?uQMIcY1mv)R_Kwbk~?>#jW<kH6Y%;1}GZp|{mW
z;?hdn8MifUOAf8EnQ<u6*6UCAUV~>vdyhI#+S_>BV6RNVOxpz#Mb-j0l5H1+f8A5U
zZelCKc5Ba(p9lBu*zsx4hJPx1t|ngCs~}pwx28nEcGo=Sy%sMN_vFM~u)X_X``!x)
zA8hI(=Ghv(-MY7K$C|wd0-CLaZ+q>nRo%Ha=lYyI2~}0Q|NPdq^^mBtowK=luf<9)
z+e=1!_f~i&?q$zV+54epkM)bfk2V{2%-n0E?QZMfx_ZxtU-N7}{Oa5Nt9`!h5!>#)
zEtdOjbsRU^ept=3*C9I5dW-eay}o;n?Y-u1ytnen>b;yR&g|WMs&22JUBB%NZ8zID
zDZP7kWGUHBPO02mbNH0)fyIh@WGek_8@i;ecgRn(z0mgD=Fi3xdkgmV*d(Y6+Fo0>
zb?+<tm9`D?bF9{U>)-Ps{;18&Q%QT9p3Jv>Q8Cpf!#Q|Q!=3fED$O;vPi`98N;Men
zU6N3^*NVYm?|Orpy%%hI_ilVW+4f#e$KH8MWNlY|d9ioVWD~o96GD467VWpqUUz@b
z3+<!39~da?%hYVx>+kr^W`;?d&Ei5`+b1%%dp$m7+j^Ay?b+-;eYbhd@x9#>kK3MB
zn``^#u<$<q3%6}ka}{jor?A_4PvP9RrZ8l0$%f5)4doQ~{y({M4|id>?W=t=Z2Nh4
z>^*y{bngk(QX64^kG-Kv9=1j^kL`ZLrLvb%?%Lkp9ISgYCO@!Ah-KO%5;AemDxHnC
zVoB@vFvRuktul|=d%z~jwvmN--?uL(Y@@}a_GVYzvk7Bq+S@MKYg7GUq3y4ivusx{
zn7r5b<o&%SM`qYOWtz5kf`Q52nvm|jk_S0$oa^7)IK5fAw;_3-O$hr&+m2J8Y;MFH
zvZ;7lxcAcA`L+y=2lkYGi`@Gv@73N3jjelcoA0zqJJMmhM!j@zKi^Xu0~?vWGCj}s
z?6%&s*CKJs-m~5B_q>)1w6@-Uc5lLhf<0Z<2W<s9&e@t2`Pxd=?A^QPg^}$t|H8fX
z@4wjYS3kT*WvAZW>c`TyyutTvmtQ?++cKAZZyDEN+qSxQdxdZN*p&P*wtYB5Y2W)>
z=l3`~I=(l%ca=?AC+ofgvO;^VY~E(GWa<Q)GgsH_J+Ud$=F(@ieNRg7+Fnu>-V?EY
z!|pdzXV{)NQn6?MURPU%av9rwUV6LF^rY`y@p0>(1dr=B=eZW|U2?H*kII(mwh!iR
zv^o3t-`@MqW_#T_j_vj4udwZqc)mv=X~FJt-cXwv&AGNL?=x(@pa0tXEt|o1<KvBc
zB^9ph`G2K(&%KO>y$7`0_oi|d?WsJUxR>wHADb4f_Pq+hM%LHwtJsELpJMyrxS-9>
z)m1iAm1C@?cg@)IV4v*XCMP#*N2g<Z%2Jo@ee`(N-e2E%_I8wg*mJjV-QM{-bZoDw
z`|tHu<guG(Y+*Ym>)4(hwFP_Md{DMs$jH7ga4DCqdfk58kOt4ab??0Qa=P>EJ9;G9
zmcce}_aV+b)?a=b@4ex%c+cEpx_f8%i|+HtTx7ef$7ydz$Dut^f4i-!pWNKjuqI)*
zwWh<~Z<nlWYaCtozBqAWuUl!IZL^h*9p5C*y{i;?_RKgkd+!O=GTR>6MK<B`4mN9o
zT=!Zr9<g<q*<|Z-rfSb$gIBgM4qdZ(vT6U`lU3*UY%Th{m+RZiz3x$KZL_AH-dokw
zvDfH+sO`b`!uuZS8Sh;%JInTW$ke^pbms28wXf55xrN)_orw~A%x<r<?K{u1*U?UX
z&lBHOHf%yOZ1obl_D=q^+*T`f#a_L`du(@etM7f2nz46M@>Ls|mDl#psaDuGA@}^=
zxDQIUFT{Lof&{c}J?*FL<+J~|x4op-=IeEfy{#*E?e(4c-FC;%-Fv_LzulV<e%SU&
z$_Cr7j?4BQePp!vN2bU=pD!i0yTw%Y)t_V7bGp>mN?qyY9-CDxw%7CB_oh6u+3Rp{
zvn}Tu7TdUO0eh#W*zaxexxYv4Z=3BNjVXIP7tGkZg@t`DdrX0C-O`}F4Xk{7nNuqF
zu3oLY&-uTN?W=16dpj=K+IBXT*mT7S?PF<KY1<Smwzowz$+mdGJlj<9zP)07FKsUI
zEZw`Mu+_%v$XZ*T;#qrM*Sy<PZFFL<n)U=6wPh>zK0fopreNO5J(YH!_Z(L~V$+eP
zV#{<ucTd#XXM5~^J-7Ywl*jh9?cTkgZPfN{+%$8q!b~0O4-)*g?Th303U4^GH=()0
zdPBnG-FI|9?XmbOYy1C*!Cu8!nOzA+zxHmhkhFOtud!>w>Z3M!*_&;O44if=tX;OZ
z=`@R7r}(zLJN%B?I{0+#IvK~e?@yN5zWsMs>}7lU)FvjfW3P3>n%!BVk8J)+H|<r}
zcicMYW{>TZSXG-W=8JoU#bRyFRxaK9Gs0r;Ux`0^+h@4!oo=$qddDI4y{p5|?Vg%^
za?e43F57x8Z`-qWR(r2}u(my<*Jyj6>C+yq>c4xXdT-k#2lm-mWaRE?@ocl%nssIG
zyn^_>78h3Sxw%5#F2TmzMmI;^)}~9>Rxp*rcGbsEdjlHm_xheH-+O0rpzX?N(>+as
z8oO(!JM2BglWA)#)oi=}dB1IvNdKN$p5}Y+^{un{;oQBaSzmu|y2<Lj+1s|*+*s~t
z6Y^xO4d3aWy-llT+rEg3x4m)r{ho#tt=;l_AKM14QrYV&zGm-kUd`Qa@+a6#yVPJS
z@_F~3b%`Hs4&`j$J9AB+t!dX}n@z&A_eL*2wYSFc+MfQ(gSH71`1hKWYT3S+`pP!-
z+$!71xUYNfCB51^p@iLb_U0>lzZkvUqj<n#?~B@ZHq(Rc_eO7S-0Sef#P)#-!@ddy
zvAq@US+=M4n(UE1dC+<TbJt$KYl3^<hq><F`)=*t^tG;gr`(RSt($ei#$K7j_Q?8M
zwl<C1_L@58?77_e!0JwhvF(<vHG6B89oS=h;n^P10>-@z9v61%@_e*;zkbc$y}aD}
zqIjhCTG;HjN$Xj=w=A${?=D5%U0c*w+2{v7v{i|2vE@0?Xrt2`ZTr>Xw{6=Amc5@>
z9NinUaf|I7TYft)tHXQe&2_gey1v0?wO5dBi4?c(^dtQ?7Tvq{KJm)24b|1$yJF2j
zTPuM!o06-nd!Jf*?me?=${s$au)XKhx9_cbKiii1y0C3Qg0t-hy_+@%Vwr88S$whi
zHEpWxU%nlC|E*ZHXOGtHJ(9nhY|TT(_WpSoZ~G*`eXq9N#61DWw%AP1wccB}>CWEI
z5(jJ<+SK+2ic9S~Tex9wn9QTSf_5zX=5=r0d)Djo9>%Efy<sPx?NzaCw9W6@u;+rH
z=pJrszkN}QJNGVNyI^yxu5&Nnk&Sl$E-tmz2^8LY^Tmd}&&>60B(J*d+4wHO_T#ZL
zwj5otwo(G;Y!$wk+Ps@G#dc1fw(W=XIs2TpY_#q9*JaDHxO1PkyS?qR9dq^_n%BQ?
zliZrUU$^tv`q|3b8C+Xx8*f~<@2bdSTc5`#>}JZ=@9Xeov}JMHY;$6s>fV|wXKmA2
ztM`T<pJ3C#d&OqLykMKtqATshW}n>W5VdGm&6ehUtLyjL^?ZG8`#>+qZpMxqd;G$e
z*iL=uxu@}MnBC&bzIMT<4EM?RbK1=_Ib-*2_J_R>xuxwMOzF3cw-&YA5p-uy)v>30
zX6x139aQ4lC$8MKSLktw-Hf&Fd$!5;?R^lsd9TM^1KTI_m)a)%3)&kmux($|!4_ME
zH~hAXBiQ!7INP(g?_m30hnW3V*C(#F;r%MU?}Y2-y-T@1+a$Y%?zPZm+uQOz%=TB}
zB-^x1+uc>u&)W6~b?s~L6S7*f*4cK8a+vLx4a@9)e9y7H@PE#}snSdKxP0W<yZ)NZ
z9+4e0_Fg|6zjyJmhP_%xwd~e}Z`k)Bo^5x%P4r%#Vkz6RL439xx2M^O9n#nrD|69W
zrhJ0!niUy)#jZ`?H_;@|Zc_UhyNeY^_Hvf)w!3|E>)wR>NB3S%*|2YqslM&g#^rV)
z%4clLzRtAUv_H>ggX@z$W))KV61f)HY2T>d$1qWAPk_$T-5cDJY`rF)ww<OeW4BO$
zwataTZPq#shxSSDUbWBTPUqfJwFmZcegC^xj$ddWC%5Uo43*kF5@r&1=Unsm#wkA8
zyZiMy8*VG%ec#Rm*gg4jYL8d{9$O~gX?Ee;F4)-A2<&~tw9$^y<f@&N!;U@drG<O-
z56-oDURb&}_S|B-fWPy1=Utd*tMt3u_MXVz-R1JL_q~WK+mk7Hb)O7}y)8o$&z_qn
zX6>C3=WfHe!G7<0ZpOXRE5r942yNUGrp#xvD)Hl<6%!QqhBVFJGjaB<Jr`QjY$Z(Q
z?{P^lw$1ULvgd{4Gn+5(&e(2d^4_a&U1#^;?5w>%Qoigdc=*fuy3!Q88#U2;d<3rT
znG<?ycU0fbeRh8p?ZW02+3X8S-F^GjOPePpXKkOdwc8oQE#FhObD1qeuiY-K8_#!_
zFMDK{=euU_1;?YduDZ+Yy6ggNO<pwbn^}8u@4jnGY`^?f*?YvYa35>e9@~HXbN3#&
z%eN=|u(;hB1}$5SM>2a}e0yT!U({j~^7*Xoq{t_G0)mumPb%`;u6Vf4wqQZG&7Ao=
zY)`#Eu=k8f+uqMr$Lvl%oU`{vuI9e??vHIYh4t+Heg2}g#D*od8kvV|nFZNwH=NpN
zTV5Wz_d)g)>lt^Q>^@GuW_xBqh3$eSJ!=^+wtb!K0(OUHPO|0VTx*;0d%5k(5Bv8m
zU~;m~Kisfq%IoHRnn7v%tTZxhSS<JM(KzL9qhOY4^X2{Ny)8|9_T1dly7%C0|Gh0`
zTlOYs1ns#qW8a<=>AP&^)UMg{Gq-(@j^DC94s$#999Xu{=JN5oHf}BKwkD2Wtqmq}
z+OC!>-d7$vZEx49)V=q1G3-lHp0d}YciBEEkuKZMdmh-$X_2vY>iTTsS=PIEzMcDC
ziz+AE-{<DrM&H_G7rS%PzA)c!dwPA|@8ww8WYd~*Vy~u_mhIiW)AzPV9NL?u)nqGm
zZnf>ptMm5;{|(x;^`ZV=p5IEgb5z&u^@!NJcZY_(%^W2Y+qS3;dv82q+dEf)#qQ0m
zTYD-tsoLG)m~P9p^Rr!IcjKN5n%DO1FkZTEm+SpKsoO8^dA9SY9m9r~HjE#5?F^p2
zv*F15ZlnJ1)?Sup3-%S9-?%S<N!ZqD&H}r}fA?(y#7|iJt2OS`TDp4A4Ic{|ot_?B
zt#>hYtf!XSHSA8eoipqD-Yd)&wxYYg@72-eva^_W%ckW+if!S2XY0i+dA6HdKiKBH
z$=tK;kj~yL&slaEZf$nw4nNq{y>*X`lyZk{M#y&?{+^`0p8}Zd7H(K*ck>0W-LW$B
zz5jeC*=-44X?w<`YL7zt5j*||i}yAxWZ8G@xv)*K-HpA>)64cGRBYQ9f4j{tW}%*K
z&RdzieLMU2<xOp|z5Kt&rn05gZt2}#yUL&4d$-<Cw+rA2x79ZJW#h4g-!{Q{_nwTl
zBevYll6D=TLVFMAwC*!$IkNA<<$ZfUhql>Ooj7Nkn7PbOhk4;%wd|d?{ePnO{fgGG
zdm*)CZ-FN79)8)-eJR!kw#L7g+I<vUxo6MYP`e&hg*|NtL-svwyJ=%vvDk*=`34&{
zMi*O`zvpdv7}o98VqxFAp?jarncAgxudXlNtN!ty4a?%ycJ-6D@A<oo-B#kk*1bC}
zOYCc5(%8F?#n5(-+l9T`*k{=q?^tKYygki!_k?+NsY@o<Mc>@Dci(zun|S42``llz
zvpp{?VfSIf@;wP#j_pq5R<-@$XJ+GYah}bp_6K{iB@fwJ96o3Ffg!}^!r2SE|IApk
zx8m#;TN$PBeH>*o>@GO=?bh(D+!L{5-X5Qg{B}M{9d-@t>-NN*tlu}k==9z#;ybJl
zd^%+-(DHAu&PNWr=_cE4pK3MiNjt#5H-vfho`Xz!dn>AWZA6z}+3RrPklnGu$+l}d
zyY05P9kO+}zrf04^NoEz(mVGx2)gcy==``x^3lA#dB^4Us7X(=)wlY&_u&5Yy&C%)
z_cm`^X~%rkVV`9|-QJw`6*kfT`1Z+gY1$TCx^0&qw#sfXi=pjATh+bO-NN?j8%x>=
zsjS#L%jKL+$i>aJ0vD2PS3lC&D`0%m=I(~7y@j(=_G+z~Z~Ne|q3zqeb9?8VT4sCd
zs>t57RXg{xT|RGnsD@=PLo>%-6}LHiUa<%4ojz~nUXfz<y#XSxY#!Lv*h&^I+3Pgx
zgl)h=E1T);2lj3`p=0YJ$!41}|Jfe4&RKgIn3vi7I3i+u>dZ#l9i0q&GG<KN<8aNx
zcE^6Ny#bHb+w5<N-|Mg<)21O<+{U9K)h6UoxUI>OWZRy^1$#5jOx#=ZuibXT*IT<g
zDpuJx+`YGJy?NH&-wjLlPKY^Ub0$c@mf!!tUc)In_8M!r?yb;q+$#~|X!Al+aL<dW
zGxk)3{<NuRaIu+uK5}o^_13*TePOmwWRLH;p&?_-FuiY=#&0oOkJmaj`6(Ora<|p&
zO?hQvJ0+^$w!_EWrq%Ss-g&;ld&7Ax_G+BkX|v(B%ARBYllDHjrDi)vO~%&dVS?=y
z9wXa_=4baV(4K31dgb=L2g)OD@AO{Zv$%iG-U{oPwh1$z?YXdldr!~R`FmVsDs2q{
zi)|A8n5+f1p4vU9bgnJa^{Tx;WVhIGtkScMIg?=fLGQWEndOUY6rNjKPjg>un|5S|
zt@F%@d#^1$XZx_DdaoA$6x$5PQ#KqI_wV_0wbC|TW9wc6k7C;fi49f`9a?*M+M{i=
zz8tV+cwT3#GIfJ(#ournjy0vWSC%ZXiE9(w8*ppe-e)TAdl^bL*=%u{x3^o=WN)eL
z?7apiCv7w~Jh0JNv)!hlK6B54JJz=Atk>=>P}s6Jp=O%xRrRvH9upVteHLH5H)WTb
zZCZNx-tVQgd!;g`?N!Mtunnu;V%vMUes4$CC!3$FDSPkqWY|iqeZJ?6OVi$nnId~7
zO^@vTu(R5>)OzRM4~aKyv?NyA>dkGo^-Z(cdqmUQcF&xUz3Wt0*(S}IZ>wNzV0(eB
z&-zDex^4G^xqDy4f3Y#!-)H+_!YNzfmuvP~6pHQ13X#~`adXn%J2yA)t%z&fn{eQz
zO@hUqz4HQR*t*CZ+gs2zZMTf;0-GGqPkVSgJ#C)|FW$RnpM&kkxh;DGB5qkf={&Pn
zCcnovqKMnZmw)Hp4<QwMe%$?H6LF}|=12K*+nQZ_ZQpKRx7TMvfGy8HiM=lI^KCs6
z7;G{+U)%ioyWjR)u+82-8WZ*i+_c%lvAxPRXxdWS4=cQFBk!o~y^!{0kAg<eUWT8O
zb|rY;v3BA*yLZkf@x3aa8Ewm&X4(FT;@D$gJ8SQU&ueyTEOywV;Oc4H@M(qZi@UwH
z99rM@1i$LoyKKT%Tahap_JZ#E*4Wiz)Adnn?~P&)Td|wFY@@#Q?w#Pa&bBLKmu<vl
z<~=Q@O1420PueE<CG6qI*0Z^>_un3e$G+AEek!(iPOr3Gz`?xd)6`wIL3wL!WfWNU
z8WpzgUGj$A_8ZIOy*yd_Y#C~db{qWbunlz2-Mgf2imk`X*ESte&NgAUCfgba?cUvT
zeUr@(i+#2){J-1G3!b$%q-vYZgT|jW4sF@H1JdpH9)GN|ms4QR-UsWa+rHU9&vsAZ
z0b7Ahv-fT@n_!!8?(J@&#wE71^Ed7Nnr6S(AwSRN{M7S%f3cq4tFU{yZCdUE+f5U?
z_p10FwVfb2b$7t2dYfO{mhYXnVS%lX*V4Ub^E&rdJT9`?s*=9<!=`*&i%wJ96ABad
zwj^=v4Ra6KYc%KZUX~{jwx!jDdk<tz+Z&O)eeWu6j=dWkb!<JlKG^I{YTvu#{HeWp
z?<@BTEN8au*;Ti9NqwSi0PpqP0cACNikv6!eXn|Euf^@WJq6NPwg(T*+-q=Prp=|&
z4%?ovJ$sim_S&up;kA8mg<-E!BGX>C3#)CXFfQAhl(A~B>@N4cd)BVoQ?OORW=Gya
z+b7?r+P3x7>=jwKcJEE5U3)L&w^-**+OpT<!sfjK$Hex!)f()rIe)^|;j8HGD@K?1
zaQxe3t8rz*-l$0-dw1QNVC%72(N;@(jqSA4=l0H+@y}*L=z3d@aFM+yj<(ozeeJZ>
zaanD<W<j%UOTc#95B^MhcNp{S;km)S_dw%>UH3Qj?{z3Mv3?-B%r@%b3R^aP*}Vnf
zUV9_*f7>wJ$=TUp6=p5bxXrd;+S}a>&6BNaQf};#cxr5W^8ewz$5=Pm+U#kusW2DX
z+rpz}`{2njn`hcbZ6{?PvTcany?6RDhrJ<<9eZQsPwzR<o@^7)`M~D3iS=FwWlP%=
zjZ631&0N2C!<VDhjmFdVrq!IX{h;AzYxn%1?TH<qY*^O0+q#&|vi;OC-&Uot#P&r8
zukDX%@iqYylx%F8*6#iMeg0mbH5Y8pOkQGptEtI$!QTSwoEl@>(z^%uD$M-9=ketY
zd)2tj_qrX?+N-0)W}9-<Y_HjIzP%5AZnEt$nzN_dXvf|Ug6?~)wlCQ0^Kzk0f@+UV
z+vVQ96PHif`^E5(Er;*+y*K9A?BTJ?v_17?!rqKpKN}9|ew#h7nQSlHw(Kp4J!Ad4
z+;OkxkC44O-*?&WXpXYYxY=&2VB%=IgUiC^&wPH{_Jam{4fs-R6W-YDxhCJUcS|Xw
zt>C2{d&9o1vTaB>vOC7Mz;?@yW}CwQ?R!`Jm|?r()_$7>rxsW{6r8hhSh{3yMbj0V
z90~Wm1vkC-cG#!eo@k%6*TwPYo(*lLd*++X+ba<G#roX+xqAhU9@?uCWNgc<R=HPe
zck$jckL_(YNQBz%u+!d?@iM{Y@#n0)8x&{k{UY6IdnWY9o~HiVy&5S7HX-g)Y#J7c
z?k)T=d9PT<CfgIveKs80>U(6GYxeG9Yu@|tP1D{J_0w%{f2-WP?8Sk-6PDcHEiu2t
zc8m7rz3m5I?{&ELYcJ~>#eM67e(paoW&NHLRX_He{Uo_BYUNbh*B=e{KL7G>uifN>
zHo;4pEjDCY@3U2_vH5s_W8Z~h%Y6<K&umwvH0?D>b+uJib+Ik#+G0P)d$Em&|0LV}
z?>E?+XDrwoG5w<bk<ZHZ$p=2{ty;fmZ}#=Ob^%F`_tqSd-v7%|U~iuz+X2a{hP_=m
zjr%^^T(r3_v({o)Zp)roT*vlhc8c1p$W*fJzIbl$QosAQ*8MB@IlsNKcU|kReH*za
z?_0{XW?!bjf_>3`ZF|$jPwiYT(r=r+bhEu*q>cT3r?mYBZ~j@!nOxnQ`KfWA)0OnS
zhj{w->16%3xiTkwZ}{fLdr!0%+OsT=+1K#-?cNHvbNku0FSc<|ShCl#uWe7+oZEX(
zn@-#3@|n$MvURwf0oM+jJBjOU_SE$5^NW(SwLLv+uUzm(+wK($ckKwv-e-LN(B8j3
zvv#v>+P_aXX3<{H83*>T?z^*ZM_#e5k3{2sPxU{x8~b0{2LBb=`_fct-{PfS`*NM9
z>|?J{+xM&Y$?l6c@9b3!yS!J&a--d+-_rYy>t^iRP<M3SovP~nJl$=3W!_ipHEEf+
zCr_klPiJV0{R}JPeIkb0d&@;q_OadDYJ2z0{C)EN-}k574Bf~5|Nq|4igWEZte?4;
zW2NX`=3RGfrE0ltk6mQi|Kw=2EtmI<eYW!V_O)c?*d{ZY@BMB%*Y3IJs(m*(uGzTs
zf7n+b)N1pf<F<`6xAMLR#zJ=ba;x|9KJweI+VFJW(<N^EpKUp`_uPh;HYbk$+hdY2
z)xLmf{_g6qS9`g#KJLA6N@{QFs)oH;AJ*?_X6oD@8|=MLJ|f;WP?Tps*Iuo?bwLGt
zjY1aK*U#9#zdHBk-lW%BdnaXd?=_9wy^qa4XMdXcl6~jcX6|i1&9zU=P+*@zUy|L$
zWlsApn_t;i`EA=i_Fp}FnHJsKyY7YaJ~q{fd(UQE+FR&*&_2OBd#`@{q&;^fRQGMW
zr@lA8R(hX>TIZfqtLN;^@>*uSeEZscT4_J_*3^paJGA%9zB$!=dm|oCvwL)L!`|<W
zYxcIR2;O_{<dwa7AJ*?(!#Z!TUvK07s*S(xK1dbr|525(ckx^X8}nZpdp9Ot+w1)&
zWPjw$^u0C@KJ4Y{D&L!uX}$NBo2=ar|3!N`R3rCmy56^Yy~lO$`?+!lL{@k1z5UZ_
z--bn-_cci}+BZJ`y7%6Sb9<wudUtCc65r>4-e})sMPa);)=9Pz4fFQdy$iRV7sa-(
zPe6J1``5GWovNkwrETfn_hj+){nOQo_g+2t$ENRGtNlu?(0w~uKJTAwT)o%fneFcN
z-yZIrEN;Gc!uth#J&tSdNsQ#!7n}EBx7|5~eQVi!_v=QB?Ca<(*{kP#a<4{Cvvo*w
zgY7z7_q{W>oZRbp{?i^8i;x4-+x+&HZCSZ*eSg2*3?WUs`idX6DIYlYz1uIim;dO#
zz3Crk?A2)VwY|0B)!yF2{CgX&@7wdOc=g`8n^*TJ^55LcqtLWxtxV*;pYONs)!<vT
z_tDL{`vR>e?KNBYeeZRTcYCjwO7DGJE4ROU%j|t0KWFT_>?OH3B;nGY(`~zVJ8w_f
zJ1y*w-CG~seFlc+dtX>G@00eb-1~OIq`i}EYWCh=8fW{+!Qeni%A|dd(x&a@Td`nw
zVp{87+t%m%=KS&6mm(at*Yj1wzP!SVyC)nvZ}X&re@|tY{azsr_r0E%`)!uw3hm7@
z*uQtp>@#~eI2-LVULLep>zDAp0Hf9W${1buy6m2|Pj91&{X|BGy?J)5b~fiU_C+O4
z*=MJ<b|1$DFNa-IJ8WnB*6myKe%2l)CdPdZmsIxdaG7AA&7rdY%H=J)H%)J~zr@F|
z&%|o&zAsxJ?A`Kh^}ZOJsy*W8kL^7aEOp=pm)Sl=4j!9mY1Lf{yiEIh#f0`%SbOYU
z8yjW2Fh$PRdUAtpvJ8W*Rn&yN^A_6LUi4zv_b>O*zBe(;_qyn5?oLdS+M~UvVc)+e
z|MqgvTfNWgMugp!Z%q4tuUxtJm&LSwCss+?|5gv!%g=ejMwXk+fopZhUY5CQ_A*V{
zyZ2W|(cay$?6%LhPTz0;;;8+KA{AS<%~Ne%tk~=eoHF;W{L-;I>Qnpv=Kp8+trPCp
z>pP#@{-A^MJ`ui5+y0w|d+%f)*}L&e^gdSCB>TJ<to!ultg&tVz~}J&m(Kpmhu7?P
zG#;|Iws~VKCnvP;QEad6?mnJ<mg_gzUBC2suS40hy-kUU`<A@mvK4fHxZkajWuN7S
zb^8~KM%(9pUtw41J;A0;ym{|}a|L@IZm-+xG23bHp=N2DFu4``xaVKp6WM8V!0w9s
zUKUHsy=PA6*`LrCxBcSCW82noZ|?=2Xd4~Q*L(eq=I!+}pSrhApJBhiTe<y=;=A|0
zKhtIxee!}G@6Ln!`aV9{`@Bzc|Iyi|`!=$k*~`qFyVo!L`QGl}DSPU|v-WX$AKBCJ
zE6%2ThNA7(dM$^}$rk(M0{-pci?Ou58yjM4E}v#^)ws@1@xYV4o=h!!y?XWb7<gsb
zH{C7XU+e#1?^?m+eH)nm?2%ctb8nKzhux~FllMAY_uX6Zb@sl;#iIMxEzGo5ylS}b
zn@H8(w&4GJr=)z|JN0exzDBE+d%X75+b@V~*!#4#Y)_8L@;#nXllRBm-nF;gvTWa0
z{R#WBBn|f(?^(6?SHzCJ-S;hRC;t(!bxiBBW9qTDFXhv)J>bl-&xF&<ZZ?0<J}nRX
zeK87sHrr>;vkjIyZP&EFW3NHX+r25By0*Us)NKVmDeY5RwRUf$OWfXBJvDnbDB0`{
zaBJElkn_amm+1-Hd9`+X8Pp<dIj*hTbK~bu+jWs`wq6oKdqWm%wpqiTV0&oKLEA;W
z2lpCmow3Jn)oj}@e`WU`P~5OL<XD_7!zaJJ2H`(!Tmq-sc(lH?IeWrj?>wtS(7En=
zU6vQx7QEfOCo$e^uWH<xy&Ddn*<)aCWb>zs+xALdy6pyw3AXblZrl5$;Ln~0juqC6
zz5Vy<xJ=o@(|>61k11Ju8D8Aq<FM1-<{x*%-j2xIdyF!AY_I9e+k2&Q!rpn)y7p=m
z1=wCWBE9$8&vUjbm}+fHj_%$&qrA$7{fXn=1^&Bi7hOGT%WztLkA&!aTY-w}yC3L%
zvGFjRyw{@UyzL~GHMV>1ZM9i&Zj#OMC8~R;=*-%C{@R4Sjve#&T0Du^qmw_+mRD!;
z-UBJ0b_ezDvlTFYW}_t}y*KPjgY6XFAX}Mh6Zb5$oN2q_(IcCK|JwGtTwiK4bB_Ps
z3jHKo!4<3aS_E9&Gj;aby*csSHcm@U>@`lDzBi>TdT++WHMaMhH}AdVynXK))|oa8
ztQTz;9c<bgV4l4v$*|A1Lw(iW153|Xv2+}=4U_D*m0{by*TGM6Pt>%Pdn11G>?yf(
z%65e%v#o{lEgPSdWwx$thxS%YY_Xk^I%iM2(&4=iu0OO%S(|0MXP&U_8?GgLzwynq
zRo&#cSK&>N?fz5e_Hv0&-h08Wz_wuy>)s#nhinB_dF<_Am}0y3_^iEK_v-EC5j}1D
zNw#_KikbiRJeac6+Cpa09+k_l_i!v<y*F+1G+WIjYJ1n5X0@FmA#KwySh`p5SBtG<
z{i?kg33WC+Ol*4>oN(FApkK9Tn=bR-0*9Hq?RM_p+aTC!QzMpTdn2>Y=ESUDHaeUu
z_DHZb?LB1BVjHKRxwrdn=-xkz<7}^7EwMG2^w%cjp17^e9f`dmZU=4mm`=78;$O3u
zeW~}}JAW7NdA>Vf?~<*Ywrmm0Y>m&X+<Q(ld~X7Klx@f5342sIwf26veAsqI{Z5<o
z)Wdrn=2qEQd}Xk)ie9#N>z$)}TmQH1ZEBF%TU6U`>!7%7cSmxD?Eyh<>nd%by*<mF
zY-gUou-EPSxxHsj3foq=Ufc6VciSG3v|?L_6MuFkOu4zIXhD<h+3a(BXIvEDYgXE1
zdqHx~9<zvDd-n*=uz7Zne{asenYJ184fn|PRPW_@vccxW@+RA^%;mP34t{%WHq6<Z
z^3l!qa7V-5BE{If1_42~4y(@ZdFQlkZ^G7-dwKS*+1sMiVQZ7kV*9S6b?*`%^}XHQ
zOZO_=E3rNEnQ`wd(GJ_wXN~v1;Fh=DP`+yKDxU^hYwpE+RpOg%59(~&`|r+i+Xn46
zD~E`)HXP;;ZCpG`ZGUW@vB#yb&UXKCo4qR}<81p=SMN<w`)sp-m%(O774zOZs}I_`
z$4uKhvpr((U618^-596a2D}#7YZ111uZPMu+rt?Zdmnr(vAy6{xrd?0cF&JZd^Q|$
zQ*FiV&+JubxnSM<ZT;REPMd6V`1aZ!V9eNC?P0Z-`EBrC1||hthGGYs8?PtsiMX_I
z?~I&Pwjz#mYy;-)wb{wuv$vs5+4iv0+PxnbXWOKlF|^&me0s0SnO@r?K?Qr6U*_!9
z*>!EtFV`O16JO)^_MJIsD-(Xu_TjD*d%fmv-|Jwpch4Elep?2Eo7)XnH0>?;e%*%W
zb&Ktk&)K$1cx7z)b}h4Au&l&(P3(+4E(u%qzK~gCBX@7Z-WYo~+XMqITUW-ky$TjF
zdmLL5_X>6%-FwxnZ?DhvNwz)RJ+=;SBKI^G9o;Jztgv^<#x=H4=Op)jH`%gRsr;nv
z4)-3L2lFCr^n982T3lALwP9?xQ9HHP_Cb2}?wmd~+XYi<Y<k3|*-EU;*}H_H)|TPS
zZ0isI3~hU6^4NZO_}k_Sm;c^H+EZ;CLNa#G*_&YdBH*S?-|mgJGEpD)d}vYL+o!l|
z?}Rmhd*>h9vA4q`*merLwCxJfW}5?(C+~T)X5k(I#<_bM^jBDibSc}`e41^$Ug^|c
z+xKhs+PHPu$gwZj+oc}1cY*8N-7zXhY-dd4vNiJSvAvMLb?=1N&9+?G&3nyfFW%d6
z*Ua`+@n+i_c~Z7kpIP_HtbDm=S!}bdNave94Yu5S7>pm<ygl8$H}}=Fy#j4(_ndmR
z&Q^E-ioKh5dG1a6uDI7JscWx>rjqT7@RfTlcD%C@5PfRXu=JsIglw7Zg=uwrPP{a+
zeIYwx5690A+Zi|gZ2ryJvNz(?Pn&>0KWwgiIlp&}XszvmRS$MOd#AS7q;vV+2YEa8
zO7Fk0S5dZS@4GXLZPnJy-7AoIV^79>E87`WntK!iqj$3@pS5K%n6o!e?f71^vxR#<
zilptG@krD5Lx<L$4=;Q6Oj%=TYvYn>`;4o_Hr#6I-ZO6|+U|Z+zV}boxxJt6hVPYD
zbKNUr6l~k@=&Y3x_eonBcL&=|n~L^I{MWSg+taakK|qtO!bc@*1xr2aRXTmPJ<KV#
z5010!VfeJx>f<N>y$L_|SufaIZaeGYWLqWi4%-bY`)ou`9JBRN=(l}kyvDX5uX2yk
zx|4eq1ci2cm`m<izI)H!6Q`E#o#)uQH{s5?y*Ad<Y$oI^w$<IXaIe9Nw>F<vHSL}B
zXs+#ni8XsHO84)LRa>%m&z+TfcN7HJoTxl%JHzAfUX2O8dk$-~?Cm+CY`ezv@SX=z
zan=l_HMTs0yY|jZ@!7k8ZO-n5*_*9bL>;jW`qZ~KOJeojqUTfgI`l~GS<}{M%dnBj
zc7dAl?gesbdsN=!*)rVb*!|{m)!rpf+xNU^YOtN4>tPe|t7xyW>)O56tP}UnkQdtX
zpuNZ1nf3Ty#@#>nT&ru@Tis-_cR}+J>mUB-_dMcVVjC8_#`Z_R0$Yh$YI_XK=I&Xh
zb;|a{-i5XfJ0|aO(|u^uV6xF_TXd4G`=_40XAW$#W&Gl?*L3wETMnHadxW$aZM`C0
zZ9XV@+6qpYXZM5S$X*9W|Gis$C)#{juz!!W`<cCR94qz~$xYoGJ}JpgCu`lF8`tV=
z7i?m;-Y`#Q&%TgkTf4$u+q>dN_MB>1YV~XGGTZz5_IB!ej(cCMJh^AX-W(elkJ)?f
zT{f^y+s|s}`o3tdS4Hk#>60#Y-&XIjeeh7-cC*pRy$jx$?a>VC+}p!&)XwEav@L_m
z9Gfb`0-Fn66Lv58^n6c``=q@RGv4lbP?T&dHb-)=dvT-fM~A4r42?Z|lU`5UJK^fW
zy(z6Rdw;0(+eR3@w<$?wu$>uqZtq9-346-|!}co7Jz^u3w%FFi=z>kqf_7W&dvEtl
z&X~0KikggV&Pzqxie&vgdo`=<KFs#D70jEt?*}*M-m;HdZ21;2?wz7~VehxY!TT5_
z=Iu4GSYpF}F2VLe@%p{mPZroEEUDXLaB9}x&yxlAO?};Lb7WKZ-W~5!Y$If%Y@^tx
z?Nu(Hx_6gM(Vh&K@I4*Rr`qlC7Pb8_XOeYCw*6j%uGu!-E~5J;{y)6;=I>{=GWLSD
z7c?4d@AT}qO^#i*cau)(-go^wY!_^A-Sh36-d=`~^ES&&8f|B%I_!yxT4}rTy0x_x
z+ag=dO&4u`e4k+}mszm)fK0CK`I}BQXC4aJ@|`$ryYCI_UYA=_Y(+Lb*;7}xbnkbq
zceV>&<l20A?`V5JY?`f$rh@I`;)8n^PLkP|W~;ETs&~rX4Cxm(Q-r7NO`mPOcTT1B
zUWVV1yWgx?V>2hYV(&HOBYSUrF13x9%i8OFi_0!AvETNJ=4sm)p}%`nYBugsb7;10
z$Putf5iQ<h-x0UhV72L9C$q-AZ?2rTwRw1cZ^#`p+XH+1cLnU`*xMq1);6NCXD>(N
zESuRYJNNcjwc8%N#I}#2!^p;FYMiZ%Wr;1*F$G)E71Q@NI2KsPv8(J=-#^Wk%dKvY
zh4&7dh`HNswcMlj&M;!!8~9v$pMdFFYc|(4dl~##cAdzcZ)+d0ZqJ9krF+)Zd+qyl
zJI%Jou6FM_i@mnX*Q)JZFSu>*4^^GLTvn2{&Nc;m8GO=rwgzw6%kbW4PlL3Xt)zE_
z?T+V)wgq=q+j0x&@9mhmYww5CezrC<dF>KTt*}vO;@#8#=E`23j|*%ae6HE#9-nA;
zMY-AbT+L+L+6Dc48+-$7U)Y`88!<O%Psq+>+lK08yUyL{w7n6vY0rvqIa}|aZMKUy
z_wQAiu+4VO-AOiUPHx^C{A|nK4f$WKmxY|M&6+aH_G$I1y=(lZ+H5wRwfE-*0lRum
zHrqEc@_Q{xviCT=^0i*km9*EDXZ_wRwmr5H(_h-`5y-POahtIx<$cuN1GNu!^Q#u`
zHPP;{y|Yno-)y#ndt(-Fv1wS4xaU9$n{7^W>z)mD`>f~mUfz>?)Wgne1@qn&Ivlp|
z?)BPU_|9XKa(t!jl*D6u{njqGJ)!c(rYUXd-pYf!_x`Ek+-qkyWAC?ljkaR!KKowi
zv+VP8K4;7IaJj9ORo31kr3Y<|Tw3gIRGqSYa_^w6n%quXiJhf;JIY-5&JbU2TQOmw
z?G7_xo5S+Q_A;2*+3G~ku#IuqXq)AF()PysO4}8WuG%QbP2RIWBh*IS<c`h!$Sbx1
z{m=GH@|<kD)ud)$ub}Y08O{^-T%WkiZpUhNTb%+y+bs?YZQtGQ+#4d+xW{!OlkF=n
zVY?*(thPbP()*aodu&-$Lu~_G`uDIO`(!&w>Xxn2);3$!n~!Ww@>TXVU8>uAKqGUv
z@JgG#iOXB}3P>pLdBMc8=VWBm-lUJsdzqfc+GgDAvsvIOVci(Jc5gJZh3&2@Jo^;-
z6KrkSFYI;Tle9hODz`U>P0n^o@{7HaSEY7moENwK6|rmYe~-33nwbfEOCwhAy}kaQ
z?X0p{dpEqUwUv1~Z|{{Iakjghm)Y*`>f5VfEN!>7eA8aHu>E$k->L6Yxx;Deu<E!~
z%BPt&FP|6gQ@VM~wv2Dmo`43Yy)&AaZI2vTZM)$59@~O{d3(Nf1=wy7-(xKlqh)=1
z#cx~Zyrs5T@y&M0X8UZfU3y`w7G=73Pv#_>_$XPs<y8rGJbj#dZ7UM@TC*P8vqyBA
zP0cRv-7g+o+w<<*q&?MlOYL@CFxlH-x6)?+mc4r&-U--h>)P&})iB#uW7DfWXF?|0
z{BmmCTPF3vcES|zJui68_rAH^XyfvAu5H8dEqhbswQMt_|Jfuc8rqZuiSI43Pq7tu
z_O$cv%eI>`Y3JUCC_B5m`Cs-HWE$HxWXalIlz*~UspafmsiOyN8RGeO=@we;HTb@K
zH&?-KTaO3HwlB2**eLvF-gDeg%+{mkh)uOa^Iis#c-tIh23w2S8*G=X>faZz?1gp0
z)PK7#?0#h9+U~HgBHwY}v6UP4Dn6TN$93h9t&8>s+Zj%|wi+pBwhmmUca<=0v3a3<
z*v7G@Vy~3W%DoMW=WMmOC)xIV-C<jFVB6j<<Nda3v-I{}xIEdqLSDhvv8>1T6l3k4
zDV0Kd8y3H^*?FdQZ?Z<^z84N&d;99m_BQ<Hu=%quckkwB;roIo>e(@f$L-lV|A5UY
zu4T3vd+yt$ZqeU6J>&D<y!G2{HLREHy;iqr?*XlpJr2<m_C)BG*c|BpZT)8dLE9LW
zgSM7}%C?tgHtbFL-fsJ%&eHbV?Bcx@XN7D(AF<eL^p$7tr}quklSN+GG{ora*>c3g
zR;TE>?c}s`d+sb>Zj)P7yyp*Fk!|PS6ML)cPVP-!wtMfFX}9+DKRCA6A^*m1j`AzE
zZ;H5Vd*XR)EhL`oVXc#}TR*qUhE-?LUjGAPcDyP#_f#MHZ>#rF!Pcd^(`HYtm+h^f
z*|yeZ#d{+T8SFi>e44HP=fpj?_+QxU{L#9X<MVNwQ!n`UR@m*>D|i2tt(oU$TP-W*
zeHv@>Y$TSfv%Q@ayjNmsiLKvy2Ak7N2lmF#YO%Q))NK3gos!*!QpLU2jSYLf*UaA?
z(0_N2T~OBEJ8K&EOfuND_eL$hZT^i-dzq!2_OxxP+Pf&}%-+9eSM1gJ_Q{54g4o^+
z|J61dmn`00eTCEZM9<Q_Jb^uX*nOtjm}v{!F(@pxtO#DU=fnG3Hh=BSZ57s9Sw~wg
z-&<m9y*KURti9FCe^?i2N!ywpYTKh1;JWwY`jvZ4F3q&{Qn23ZbK}RJ8>>_Iep8uk
zo2}}%*HUqYO~cZ>-2#o8Hce}z_DZTV+5R$T+MB)Mv&|9JnR^z9rdq$8khS;5AvxO>
zB5bycGrabOq-EF^oT}bq!B%N&Gr8LK_sh$B3=VAD!_dyKS8PtVt;JLOJqkBWt(O)!
z?qQC<xo6V*3$`Nj9Bf1HJM9$+pS`D|;l9m@TdQrP*xB~7rFQO_aWQ=F{~fFLu9$ac
z&-_)NY*%Op+sb;m>@93$+8bs$)AmoO*WUUW>ulF&^4d+w7Pn>S*V=vU$EUp+!D4p0
z8vc7vWN+AegIC>l->hx6K|G2!3tcYk6$py4Ex7R3`UJP5%@560wzH)q?QXbA?OpMc
z$@W5R%-$Uy+<Sa<*X&U@WwVjVox3;tQT3h-+vB$3H=pb+NR-{XY$BUo3)|+s5fP$$
zW?YlAP1$<Hwxw0gmQi`L&5VGJdo9WhY#-hhvb!cJWSevKfbC2-UOSFoi*~J=%wu=*
zX4hVar)_&LK5n)C@#fr~6CR~|v<eT|@^P-RU3hWJ-Z1vgy%s4e_HNjeVq0)r)0XMd
z#J%sbx9*iW%D#8QRpmX2Ygp{2c!utsenM~Wf5)D^tO<fvOg^t|=A6v1{k=@Vj$y$~
zD}}9d_LMBj->WM$(PrQIMq7_t-}XFp@ZWo<b^o5sf`@wzHq5hK+}dh$Z_@m|8e*qy
z!en^uSk?>K&iGMo8<1(bXaAkfy#jn^Y!sf)-76s0XY=Oy?7e;3=WPF;Sh2UpW}Z#t
zvnRH1{Dk&y=$^C3H(g=xuBwE+M{2L^ap2(HbL7M9y+ucP?X(v?v$^zC+jj9Uw!IQZ
zt85)=9_*QSfn{$OYx3SOF6q7U8<*{EviNFae!+il^2xNl5xfd}Jlv1&y;xAUC!zO<
z&5QVqJ@+`y?w!%{*k*w!ll8yY`!>5KHrlQ-+GhKt_x7HQU+K0tBwp@W#?ZNUMt0#I
z0k^c>70H%+9SrYSO)x!Ry-xC|?RTwZHv3}B_a3NWv^mM$XS;4~$lg0Q`S-F&O|v-@
zp>8en_^FML$CJHJC#~J<R>W?nKEK#z<AFVU4ISt14cPg9&&d*A+Z>HeHVo<?cAlwk
z-J=lBU{^4O-{!%5bz7;YOKsD2X78C`^V&v3KVYxkUs1dF8g;gFthU?spSRx2A%EBA
z`NJgJkaku(zov%0niaQgS;8mSRKDxnqpGB{_rdy9+nIKo_bRMS-JNyt_}(KW5BG5E
zZ`gZP`t+WUxtw-t8TxxwoD}TVnMm5*PUf%^ey_gwz~(8|oz)Zft_-_mdwBgJ+W_tO
zy(OPYZKv2O+ch*ywX%7>XiqVp=iXVrHrk#LirSlSqr<jAqsP{5*POj<TwC@!r>@xh
z`~#b9=hlt3r+u6EGFW8Vy5CjW+wy7u-dS6o*?gX|Z7&1cL0iKsPi?;bbloeXb!*Q`
zgQ&gY-9mOAQ#S9dWq4#;EVzE}<;|aM3s$D?Wzt=`_vQ9OdzY*fvyCy^Z7VCxyf1a>
zZYz^@&bA9e4R<flK4`->U1aaHGON8SH*eYNVXd{thmCpfjaeFdQ(M*d?)av$*J^_G
z-v5=6dmlI*u?<cBzIVY-n>`N4-|lhQ{d-RU?@Jpi?~=W0YZGkm`!v}Gt(vfR!BJ~l
z3v&)Thk{c!8v@Vm)nV@1`y)QVcJ)@(y&iV5wgL$UZM!0-?Pc4nXs4_tyU(jPVegJT
zH*BmsZrCJh9NBwb!_xNmuETp1s(0?rSk$%0;pigUj-x)ddoH%^-QauJwyHVJHtTrb
z-tY_jwlkJV*)bSQv?ygNvz}1gV9T{-yRD$}g1rnocy}3m+_l$ZcZ|)I>TR}b_LtkL
zolvmrl3?D)z{<P3Prk(ViR^US$f)wYH`cteaZr9|TNEC!S7VCD9*#{{ZCI8**ux=P
zz2|8okKGGx7Tc!8t$SYlXWwU3mbEwIeUQzJ7cF}jR^GI737oaZLV3qtiI1&&TNYj4
zQy5*a*GSQP?~QjNdksP->~+*#xM#zJuQn|q2W(ILF0h?w;jnia^WHsmUJ82~{?6R3
z^>OQ7lgMp*4~SY=zh@J-yRk}i@4g=$dy5hk?YP}n+b;UhXS>7C*k;B1hc;`{yliXS
z6!tb0*IGH$w_1G&-D?v!@#XFf@~wLtE{9n+OuB8O*tBG?OaAmd4F}owJh^;%?+q2L
zJvH$m*3*ji>`i&XX!}J;c(2e6VLOT9e>P|S7VOoU*tM7Kvew@43DWx*r*Gez@g#81
z^$k6H>#EP$T0BzN^C8XDw!-oA9*51Jtcui__I~N;-?KmR(VhUE={D|XPVYT)fWcPz
z>?PY3*H75&ziPKPE|7ok{r{(J-?B{Id(xD5Pl#Ul-nmN_?K!IWWlw-zpEYyTmc0wq
z-`e=??b>~2ZmW&X$_aa)d@-_>3I1%WuuFHhR(HXkf4O`1<e7EuZ9G5E_UDWPdp8<B
zv*|n9XEUAE+V-af!@gw}{(D<|oA$;tzuB|U`pKS#<fXfmeK+pi$7s8kVW!9)saLvo
zEW3GaHlFg@ds2+eCZ>49-XA;X?pd*xd+(YD#e2UppRt`-x_R%SOQ&ojG^%V0)bjT{
zUa7g~apfBu;Uo8VAAZ7Ow>o9Q-jgvhdylU#w0%-zW_$X|`Mvq4KHD&CePrDvInnmT
zr>8atpP$-$-lAylgw$`gH$vrX1*Wat{WQDXHgx5>y&Z|0Y`?wXw4EID$7a*v#d~J(
z7TQ{0WZE~uBW&+VAB}zcpSkUQ@yEh;Z~Z%)T?!}m{?6pF{lT5QcZE)}ty71Ft&n1u
zwfL5K)@$NzZ0+9K+V(uq+8cCc-rgrqT5Te3T(>!M<?^2MpC8&@b2i_zZne=~nJ~e9
z&Mp&deTy1w^BkG>Y363`6;Rt?>(q4Lrs><^z3X?K-|KvFq0QHg2W{rQcx82GecRrO
zS8RLz{+zOPiet4~w|V~FzUF1NIdfU}ZmJisYcpKAm*Yj4b@~=V+Z!ya_ckyl?NL13
zx%aQ(7uz)FMK(Vxmha8bjJ0{Th-qI~@JE~X-z@j8n<BWEJMf*2fmW;ab`!_F_9ahk
zX3Fo`n{<55UeTB}dz~Dv**KRc@7<U=(?;8F_uj)FI`=YM>fNiB)w0*@Q^8)RCkyvJ
znBKI<^Y&7ktM}~oYIOPRP5Gl``^fD5?$--6_LjJX?fqbFU^8c?zHRE8JevTE1AA|N
z$lZH!mi^w!#XPouoO*0u&(p98XWhB?G4I*kyQZ$Ny=T<2*YaGeb-7}y?Wa@AZFl_J
zvbRHx(KaDn-u70<*}aOJO7{G8WZU<B!8IGHKGwajH!19G`2NZ|+FyE~pJ0%!!_KTd
z43=+pKZw+@O=?+YYx#u9cJK6adtY}N@3p<P(#D}?ja7hqx$Rtg>3s(r&e`z){J&?;
z)b)Fp{${pQRrs`LN$xvaorKSO=6g2pmB@~?o!g_o_tN`YdpyqU+k0+j^WNr)uQq-g
zPTLmT7qD$GU$x8W<5$}o55)InA9}L);nxLw7yE46dqZaF-h{am_GG@6*?ZK+c`t*0
z>t4gp-)%DFCHB1iC1PjL;Jjz<lVy7sEo<8=Y1zEz2*0V#WZyK~nk)LYo&4MPnlYcX
zowTWIZ_M81wxthN?Y*YeZOgXui_JC0`!<z}pYQ$nRDRz%)lGW^q88h>JeA(tP%zDQ
zZbX>%_lF$&c8095Wt|zgceOLKUBWqQ+nzGseP6St?VWR4#WqXS%+@7v>K?`%@qIED
zulH0)I_{O)#j$tZ3k|zXf(rZm8Mo|BiE7+4+w19Gjz`V5cB?+@4fEP=%~8B^&x7|f
z_s&ht+}mCKXs<#WqwNLFD%+PD6Kx8%<?Pw;?6?ib%bq>O$2RZI(3rB<^401+49*PJ
zlT6-PcS&*DW}iB|H{$j6y|K;Vdp#y~TKC-lx%Z4K|K0_24Qzd+I&HPCO}G7XX0@%&
zlh(Z^Z7H@hS8&_j<=wki^Uuw_IUWmaU(VQWYjJboUXc&-whxvY+je;zvgzd!w-c+p
zyZ6Znd)tGqO?ywo#M@4YtFXNgba+or$NRmRc^CF31kT)hLPczE@|hER8$|l{W~}13
zy>WDc)#>v|drx1t*`sx6&Yq0VoA<m4nYYKc*md`wQ@?F@FJ{=gA;)cRtZ|lYaP`8y
zY!bG6-@3N$o%7McR&#sX9*3U!dvsrK-*f-*j=hE42lleq=Gv}b8)v(3#xC1Wd5ib#
z-&SkuS-s6>%gc3pPG`K?`=UD7R_{od?dHGo`*KxV_8t$=+oRNCV{<{ydGDO%Q}^~R
z>al(r)oClFTf4_*-MhUEAuIMivt!<S<V>#Zhmh+wIU7Cpww5aGz4nE5FPCbkO%0Q#
zEuYnUn@2)Awqk}KZP$xy@13$bWAFEk_Iu0NC)>O@s=9a4hIE@go~5>pR|9P~{Zp~s
zlySlK-g`+~hVZR>z4aMwSN{0BhwV+W?YH2|dpFGgVZEbi>fY2mBiobf_wFs}*ku!T
z_^xfmeT}^z=APTTwXw@)e_f%iMKJf?+L^O#4_a@rJ->gJ?Xs^6Z4=jo*sNLCX8SYX
zz@CRwj_&p6-@4bM!Ov#F7E2q;$-iw3>YwdpooZ-Tlp<xPBvxa4^TNu#TPJMa8_}U=
z^I6*0*5r(k?Wb!Ud+(j|-8-Q}+{P`#aqp5?F53l5*!Hz0wb`~O^6m{hz`XaxE&&^<
z^Uc;jwtTcnSA4N2*W|+9(5KqAMka-Ot?Z<1S^wX&{oN#Q``Sy)hV?h=o)x*Wd)J<b
zwoTYE!zRZt$2La(+V1W4g?n%K?y-%L_TT$I=+544yl#6=>wD}Ku&CZMgJ0Pu?0w)~
zzgIrC5<ed8oy{S+H)Qin+lF<nyE4?B_SDL7?csS^X<KeoxhMSSMyokJ1-2}#NB8C{
zt=}us{daHs?n$;EB_nMV9a-%hPW;-Hk+5iw`nJv1Rt2YRtKt;*g_<hv%~PIg+aYzp
z_OEZ_-ia;}`|KJ_?0lu9_PzOkV$ZA<Q}#aS-)XbQv1d<%m$a?*m1-Mt(;8cbQ_QyU
z7j5=>o-?yyN^04ANocXHwg1H3Qx3e|6UuyiuS0CdUT1!0J06EH>$vwb_c9;+vNvSi
z$KAI$xNUA}o!t9i?kO9NnufhiJ}kBm%JX*LxM^o|c~aKi*Zec~YMyi4d+Wzi+Xm<3
zyH0)BWBXucg>8ejr)_}{k8RwXWwzb(d2H|Z`|O=P?fRZ|yQ}x=%s6FRP&s!m3rmNM
z%GU~8Kh^1b-~5uW{hG1L*5&riJ%PIq?kzLo+3UQ2lkLHIcWs_jUfScKBxaW^&Skr6
zqm12hOM!j;ig))~|8&}WcgMWFe#YK=CtN#eQ&7Ff=AO$d+x8a?HWtg8_q?!gwF$^e
z-?MMy8ruzD`)m(wJZ<}Flk(o3+;jI9=Ska6U#+_LbA#I6)pG)E!+tjI4g7U}FMC3+
zt$*GwTZ^3MHm}`O_nj(Z+>?~-u=nJhGd44uT=#lJZMM~V(Qlg&*uSUZ+}XXG)DPLJ
zJW;THcI&Kd4ukLBDFunPF;d5E#jK9(iFlW|S0b0m)}^-F_EPQdy?f3i*?LURwOMlG
zgsrmr!M(y-%60-8i8ikzf7&b;pR~6sCUI{squ{>g$+mlQtCsAwD_pcU!g<bKZ~HTQ
zxA?BwGa+r0?E#TRyJsx&+j~o!d9VM0#ytyKwp*vnW#7wiLVU;S&r!C^rFQL=Dv+|9
zpgMo=#|zzi7jUcDTBXn5JICSco`%SYwim1z_ez|&wdaGzNt<;cmV0BAIP4f|XYX3K
zka@5Bi&cA53fJzHIC#v~I9qS8@|Ep-S~yc|gHK+x{V|JUPrT5{y#~=QcmMcRZo4bw
zh0PAj_j`XwuHEayq;I=t!g<>X9X!@rE?u@lcNgvn`Xykq?A@%r3|_ab{SWKyy=nW*
zHf+)w+cldH+gi-z-+R$#_a504nSDo2-P}9lV3}=#(Y8G=O1x}+g!k+{%j>xJT!7x*
z19uPYiHu>h?fbIbHfm|e-X$9*?%BfLxwlM3W^V@9hdsLj&+L6|DY~yN@3D159rLb-
zS9oo?-kr8BnD=Ej<LeK56*#`y%&+IPlbE>6`o$D|+vlb?_FgIo-76xHvsZe;tKFiu
zjC(&Y8QQF=JYsw2;*Gs;Q@U)I+_17OzPx6yo!mp)gn*gWTdL34{^C{J%M!VIZ;1O0
z8!z>Rdl!5D-<whY+Qw*!(B6iR>-S!ns%R_W8E^CJ%Zj}}k|x?danQ0o?frA_ap#1+
zlea##o$Y#ZkHGr<yBT<Q?)ezFdGB1^3wzt_7uzmnE8Y9&{)s(e4_5Crm|$rmb1%lW
zy|Bw>j*<1=KzoLLDl&4m*Df~f%?^;ZyBjrmZ-de^>mJ^|y#i8QdncT|Z>_X>pDo9x
zLfaG4m-alE?PY6oYKhI<4O{l^T<B;map}$;4|jE2Umq2_>(_SgeP^-QR$>L`Uhmp-
zd$)<Q*|oE#+y4I1vL~@zb)Ui%*1a~Wm-fs*bz|@14=49-V7|NOmEy9!%jTT6Zg?eb
zyI;y~Z$a0Z-6uW&>~;UwvFA*-$lj=SpS>$i%-%CgXtAwE{07?uD^;s6XRhsuSh&@e
zfAhh;_t;O{I_#ZieWI|=)@c2fy>E7(+4JZoudT|uo;~OM<@QDtEZxJ`*t18apv5*)
z#(b}Nx13$a4_n(J`GmbEEl=;gI_KbCjf{`B_b0E|E2H3SJK^~<+b5lDb_q?FZ5maU
z?s+V>WbdI5KX+gJc6jfO>;{_&OZshY-b}FFyfS33S=m`zqmtCUEYB|3J~*eeXWilh
zd$YAZ+jOW)+Fi|=y;s@UeD9(8|Mr~qeq$ThebSbJgT-di^G~)x9KE)aS54kq^R#o%
ziNB6E(|@Y&wYZzQ_Ynu@-cL#ydpTnF+T2h7y(>7lcFz)n-Fw21ytIw{nrAEa@1(7T
zf2D0&#s*u4lfk=BG&k<<+TLq>tYxNcocZj%>DNVU55yj}-Qmk_+x}5@-vP#sU5nWE
z+a6yVx_8IZp1n+MEB7+IRoKOFmDM^_G-=Pp&u?tsD!kgOcju6;#n&Tyb{Wd-)oG62
z`{?{kTM_-)ds&ii@2POxVAE?+vUdr~jJ-_{)b@J&P1>tr{MP2&AIH7d4+`0||GK_c
z!Y0MW;&#xU9kv&248lL!d}@o>>!cxNXJD&h`^^5}-U4}JTf2iRY_)%s?%l9v-tJ%L
zm+Vc@inW$HzG`oXlZY+HUdeq8ZcMfcX9aC;`)2LEw6EBvZuvUfhMUE<52oy}ZrI(h
zmrb?9_Ik-y+l;$sY+wJ4-7{fvhxJW%cDpBqEqm{AcH6EgHnnv)SiL7EAkOx|n#DFO
zMaH%#>XdB0lyBL!;!Dw9jW~asmie-KZ%v(Vd%{@JmOow9j^V=_tA?GIcgvK_+WW^q
zX7302Rn|Y0FWB&<3foyc&)M@r{($Y;wTiY8F-Cj8{wUkaUJ+_r^JbOpJf4ZR8CC{+
z7=qMopY?3A4XN(5P5N_g@5hN7_8R_nvYqny>)uz_+4t(N{=H|#&(^(r?8<FV%u~0W
z`?Jx;LhFW&%-s;%^UiDc{tBOO8z0|j!)Rn{BY3-NZ>>bvo`e@swo?MKZC`8c-dlCH
z*S2S3(q0CU<+f8kS=t<6T(XzpxsMg|?p1rQF}2vL2qf5^$U40z%8AjAC8F16+oaUJ
z$5h<+D#Tr~ac3~LeN?%6@4i10dmC;@THRndxM$xasl8MBpYOU8HOKZq(=6+QK|Z#M
zhkn_Xtew3lqka3HZ7UjVvkGJO-cV7sU2<rHZB6b$+Xl6Tdz*g#-ur;xaSzMSn!OIL
z`)qIA4c_x-d(WOT5oUXvLNoXF>Fe+PsVBF0!P`39f<<5V{8`bsx1(i^?RM#;y#`@(
zY$tbg?Ukt8yVvoW(w^qm&3i4G>i4!hT4Nhhn_>INe)e7+#-p~=W`^5-S?g`P>eo@*
z1xyY$x#ElWR;V4fUGl<tZ^uh<n}7<by}w+K>`fM|*?VO1>AefiE!ex@xSs8Y150-&
zo;<bps@&1N9;K&j3fQ!4FYKSYN8pP4-WEgoy}M-X_dY1)*?S_r+E$_Nul3AB8~5rw
z@7X)UHgk_hd(hq^HoI&T;u>t_gbwdLz{9(D^Xrv+zrCHdcMY?utxd-vTZe}owmi4h
zZ8H{6*!$$?<vo!h=l5p(-(Wk(t;zOEScPqd6xW^|`%l@*EH1K@xc+!gOVQdrWui;=
zvS@AGd+5cXy&qDnZNv<x>}_~_z`BErbx*)C$-U{k%l6J%yT|ra``*1XPB_{;Nc7ty
zk#o`Jf{E*%lvsb;DO1ka{+RjM=F_X$wx@ng-}|C+pN)7;_TE)oU3<SBZm<o~?XbP`
zih1vz)9QP<)EoBhxV_j`L2~(?a+{{T)6^RGHq7hWyV>1yFGuu*y%ia<HXTp9Y%QKY
z-?Q{@>|P1oDYkXVZF?C+PVD43ZfN`Ai@a?^>sl+fqepB%)>`a!;oE8}aB$M@6(^N#
zzW7<$eq-Hft0dIBH*R&i?Swvi+qzJVy=%Dk?d|zDeb1p4J8U=XS-UqZXtyoHu0@s_
zrDZl;mCSn+)*iF5@&33c<Bi>(2`N)-XH06db(yAXTVs`FyQDnDw&i|>t>2H%y$K<g
zZ0_9MZOc=6!FHPd5?h81KW*X`FSaf4zGZXgN06<HO_wc?mXqzVDf?^-tZwa*7CXAv
z<l08tb2n%0wMewI*&?oEJK^HK-3;OHELj#$ww+;Px<}*pJliEd<o2d!wC<h3xY5?a
z)XA3R`D~k%9Tv8G{rdKLMKsw;7$(_nt2t(S#^BnX6Af9m9sg}@9QuW9qGd|<igeB2
zyTG<<w}();EyFXjy&E4!?B)5m#CG2W`@Jf8)AzjD)xGC|+MYc!9!vJ-ST3?jPddD}
zq(;M5dEffIAD*wb*^w8!XSG`2-UGW5cQ<Ujx9g1YB-@U3mOb1L_v}^Z2(p#1k+7AR
z!f9K7`PAMs+k0(9a?)*oZ0xeFv6J0<$M~G>mPu1>b$(vhlOQCr=R@*;n*h;$HZjIb
zwj$OgwtlzTZO@#ZVPkc1g>Bo@lY7rRQQ7;(cHN$LZF6lKK8EaZVVbhHKuF8Bf^Es(
z3qNygbS|^)ozQr2?}E>qHVkVcY%WZT+{3~?*~VteB3q8e&o)b%7w>ry(O}Cn|M(t`
zGEQ3-*{!xG6e?{`zAf1+^zNMPi3y8s6F&D@KXKe*v-8vGy#YFIHamXn?|Blq#I{06
z-p0s`VXwj+&b=}mQ*11*?%sRmY@F?e#S(jSUToT{^8L)7CGnDbIg(BH#$5KZ-M4dr
zE!Trhd#gMb?JY?@WV`J~m+j54g1r}e4(^?2f82J8a?c($snfQC%Nq6?)SB%%uv*VH
zV$tHg2h3LNaVT48+xlK<Z{rQqy&0=rZJxO;-+O0!neAPsy1gd$Ew(4rciA>9++=lS
z(l%R_>6*3*(Gq)h>nzzDqsd{bBj03e;+$gpg>S9Ri-%WjuFRig(|x9BZ^vaf+uu9V
z_U0Is*v?Q{YiqYQbZ^0i#=YMRrtf{?)Vqi0#Qr_=7Ob;nKUlYyBka44{hjrDZ)o4&
z^J3!Cy%x7P_b@Esu`O6Hw>Ks{ZST^&b9)&k7Vc&UnzD1-+!otw`<nJPaGkN57dvq;
z$GITeB}ba~Mu~Rq4RT(&S7}|x-h%Fvd!F>q-^2Fdgss%x3$_7;2W*%8y|SnEhQ;0u
zKQe5Do*%Gf<GirfM>*8?#9oa(32)7IhZOF&O}fyqcbC?-y&GD)ZO;5?x7~4SiR}l!
z6E;su+xD7SpV(_)f7Uvn@3xJG`)QkkQ~zu{EcVzkO!#KQ5OZda?!6;>?KmgcCUl!x
z&)6Wf_ls|nZNSb2Hp**f@BN!~Vy}py*xox2GHexC*!P@io^JcCEOBq2dYA22-r0NS
z_zBt;d`-5Upv$}Wn%nBVErKie#J+3Y%V*ZRH=?}5X2OrFHVJA@wp&CSY#mOY*mYy+
z7TW_&JNC}_d1CK2)0Vx_XLs)1FqzjTBdKz)PtPga1w4Okjx9Q{w_%mfo=N+H_b!=t
zZjZ#CzcwMAt$XBZrr0*{%Ua(%(Yn`WZ}{G*`PzHmSbf`LAiRDL$GyXQW~|-5mtjwt
z?G4`zd!)>k*{(M1*n2>w)HX=gY_Ek*fK9>5hc*maFDzLuEZ7rL;A{K%N!Z?~T@riu
z=v?39A@y&M&2LxRs+kUZEh1U=mKZA8%7jm_iP>agThcMfR-s~s?SmaEd$!J*wf9`+
zVcQoQD)!VxblX-kOxw$MW}z*ko6lYb<tNq)G7i`-xR|$xA*{&uM~dLyq@NpXGd2s_
zo;j&+8*9?EH=|W^FVEz*Jq`hhwi&XA_FQ3jwda7h+a96tJ$tQ+^7mfZpkN#GXOYc|
zi!HV)Nwu~mkNNj*`53bI&)da&3IrZm&ta9>d!t&*_QpRa+aLb(_BOETT6?f|*#6iy
z+15{%cW=y3Iom7K!}kU)Yujt`c$)1I!*E-Ljq7(i9ND>h#mfTQ$NCrc_8r@0D`4}%
zCSc(mn@0yl_WIn^-WwoRx93CPr#&I&&3g>DF5jDU>ik}fi1&LOlQ!&adDUQR@o>Xl
z6_W$DQ+Oxs={VcJ$LMOa?SdIqwhgA?yEk+f?=e{X(nf>(=$;8po;C)eul988+PSCb
znb_V34=(II^F3~F4|{=aU#R`w+zVlQk36-roueqY*Wow!Zjq4rwgC?&?$+ea+8bcJ
z$ofF_;@uAS-E0L6Kkt4sY3JUohP1sK7TMZLG)%JTklVU<htNdZZAxr=gB%v_^=AIR
z*WmUR8=VundmJu`TXz}GwB0sGXK&|(Wwu4%dG<(tSYoU1ooN#hz0SJHZOh*LY*$-{
z78YxhQ-^JNb3FGLHU{k7<2YgOmxKwnQ`_0>CVXwT*?D}TE$`W$-8?(l_bD_#vgNfo
zy4NGRchCEH2V3VGd3${|H|)*0&$#c%RXf`Rc2nCoUJq;&TDI?95n*dHd-;~V4r}My
z?Ckq*^GNE<?&IIi?EQB^c5jNx2HU*k_j?zG&a<5mHg%8D+Q+u>Q!H#toX**Lc~7+c
zvu)a*C53x-wYna&O$ch;ec;_ME9FyPZ00Yhv*CQQ-S*wYDSHzpChdv0H{AOrXz?E2
zNm8~2@Ap{e8QAZZIVfh^lO}6-Aa?d139cC1bxtpBUNmXh{@g!hZ*;cIo|AWP*e<=(
zXuI7aZSRS4Hd_}tee0)7Hti|ev}^CvHFs=&SMJ|Cg`>wd$(4O?QPYgQPaaLMx%G#`
z?sU44jYChfjob-g+p0QYyLAy~ZEfN^Z5iSZS>4@yVef%A%r?^=Zrf{Yw`K24hNF8I
zE7a|c+t0t({Ljg~ig%OusBPW1`?bx=z54|eY-K*o*n9l)q`iC8B=)5oG~JuVsc+ZC
z{bX<R?%=%+NnJJ@7!+)Ls@eAXByY4$>N;q<_CmUCNZ1Lh42|?X^LR}6Hf))?SK^qQ
zZNt8ewzHl)+MZ2P-)nI)%Jx{s-90{mbv6f;+4ufv6}L^0ow`@pOKM-r1WVgHe`oAT
z)2!WF;E--xp|X5$`P`efcTX?cdqGFqhOv-q-<^7S+qx|UHs;Is?A@}p*XGQeOxsF%
z-@P;C{@N_|_1+s2Zn*b}{E0m)zHZ&CqIG}o+`C14TX-~V7p(oWr;&ZStwZ5G+dFp|
z_WJyuWAioe(cTw<WqYGcj@ugOS=z{XOtD?^iPdIL+iIH^TMyYV$Y<CX?wYr^VBd#5
z(hZe+r|c}<tMb%%FV7y~ea$uVZDO8j+Qz?P-S>OxlD(T`|Jr1fPPAdDcC}WzaC~oA
zTEX5Ii)HqHk&D`^!@hY>U$v%<xL)yIhsM6W{!5MaUh&r7n{juB&B+ogn`OsZ_txZ8
z+DbIA?)&zXZJ+)ABHIch|DBy5m+o!rpT0Nt<D|W-{~p<Mfunctn!Qzf*BGC)<%`~7
z+x1r7PT=33Jt53Wwk{iGY*-DMZA{*M-RmUq)b@*Nv2AO_!MznNb8RN{EVOPqxpJ?T
zzv13Bb^`krOr2@FLZHBQN?+%moKw?mjebAaTNHO<&y2(Fdsz&`Y&j0C-t%~6@ZRaa
zm+k#?B;57@?|!Qd1(WtZUwUNkAKtRP7p5lcIm<kI?}nV#-HNLo>}^h1x%cO$$h{r~
zMYdv>x^_C`u<qSyC9|(~s;&+11vy)V-zRo=)IZ$Qvr2NW$?MB|LerG@I<<G$vU>is
zJ@C80mf@b4RTP_yZJ7TA+odLx_DXc^v+Wa@uy@w01AD(TE7;mB*=<|gue;Zy@}>>n
zt7h99Ga~o4bST*}Y<{$7$IU}FC9V7RhHRN(^EDuKZ_V{Rdt6>j-@A71k-Z&<>TUfp
zSK7vx9^LzKouI8l_?bNhYg%?Q_AJ>eDB`y_;O!NgG~e&GWmoswYHxbB$956BUBFT~
z+Y^G5_e${2-!tXIn!V*WLihIls@=Q4RA^szveDj@_sLfB2QKV+!)IW-R#|S}6HzYP
z^<w?D_s%`FF<2(DZ^4sxn_brXY?JQHwH3RsW81NwYtMrB8P+Vznf7Y13hYa~#J%Uj
z<ps7ZDzo;6IPvUD+Y@7Z@5|r42ex0`HA^nYwy0Ld>R(di-e+v0b`h@MY&k2=+dR6^
zV%wv(Z12Q1qP8*O^0sUellR==2(lHr!n|+MXKq^w(=ywFlmB+do%6TeQ?YjM{SNoN
zhn0Qzve(|)yTMd)-?RPaY@;m~?!CbJ&i1qacAJgIkL|r3_h)aJ)*ai>;?;W#Sr_dM
z6<EHfb*l0{i&HXtZ@oQa`*!P1+t0a&_v}q8+FLQ5#nxiwSDUTJ4Q<s;*X)f=JZ7V0
zx!U&btaG;PPGYv7!uaeCG8Wo~t8U!eu;rN*zuTL=9JP6SiY9aJU2wyBx8JJ_ThGID
z_Ed;YvK0(IZq41Oxi@*a<UW%dhxg1}v16~oOr|{%lPqi-&L6Y>YI13h#GX~Q9eI;%
z{$2Fk`{4Mly>D{rZGTrE*}H7{1nZtp#(RoBY})(#j`H5W_GPv`<qWnA7k}+Na;43-
z{)522BU%!+Dmshy*3@wCwYogncGvrLwjL=rcfN~Aw>h9;vR7`|n>{Jp`u8eanZ0*Q
z`;|S%i{I_N!=PX*Hv5RJ)i1TZ3J+uV%rUmIU2tvH9+49@dpEEL+Fq93zvsonQfrmC
z|9eGM-1Zt&Ufvxt=bNq9<>R(`M^5d%Rc608;|7CG?SJXLy?R&nYzTd9!;vGtZ|0^=
zd#{Qq+OGUGYwrxse|x5D^Vu!YxoxYwc=28b<)ga;!pdw?KJ3_2@xfpZPs&Q$-4EvP
zowT>vHv8)BJ;iQJ`%Wf)u-%g)VEb(GZrce<YHYI-j@THf?b$ow$E3Y$`P26pylb!x
zF}l0=@zx!Cc>V9~HBI5#*TKxNFN5*4Z3Kg=&61OM_dNICw)e@jg1zS>zSz9kB5(Jj
zBzez^o#i$h%A5B7IMTNF-t_%eevTo#D=b-US-fIx8NPMc*uGe8J0tY`p2~MRdxiEs
z-Lqz1^BxW@R=Z5?6ShYj8uuzp-LQN2oa45p@6Goz`^?@eA?{&YlDc5eihJ|->I=v0
zt$kRt_swi)n@P`^_tnU3-}^>wx@|)D37ZqALTtElckhj8%(mSS;k8F+^?utWc?<R|
znJaC}@w~^jJdn?>RiVx1Tu`fR)qBmoY<@y}`QJ(IJF-9CHZP0YR^jX}+Xmkdt1h!8
zw)s(RdskOY+`FY!dGDUxVYcNO40Z|U)ogkuPu%<9&)Gde`qOO_cd_j&nmyI_(Of>e
z`Wyz^s(lM>T57%big~K;Gn1cht8<ih?}ZiHY&VpI+cI=A+T0Lfu-he3yH{cJ2b-^Z
zX6+SY7T()2b*fDid%JB}fY#pU3l#SqfB$Ij4S)8%4>nA)t%=-kD>eCst--WhTUG;m
z8_o%f_w@f4w6oY^x##wCg?-u=U+pQ?sM^~u{ba9$z{}lJ+L!F@-RQD6UQ=st?BT+_
z^X_%o-uUWf`$MOFPm9Fly+ZZPwsWm^?RB2SY8(B;e#e(<%{Cf~-tIXs!)_~l^THnK
z)U$g(q}kevbZxfjU%ScHLnYrfE1%y+!a3j8N!D_&aKfs+5~t;DSR8)td0^vVqww3q
z=E3ib-5Qy^wm%vd?LD=z#a3t1?L98H*V?A73$yukbDiy-IYl-NHchr_5qx_XWW08!
zu92`?A);w}!AotA#Q6z(v_zNf*<p8h?@qIQdwCM&Y<DpA?&Y}CVr!tWes|4sF}u!9
zyY{}3*<{P$muGu|pWk*#Sm<8cH&ga3-@<P<BVK!t@d_T>3%1j2)7DSeD{@}d_VRJp
zy@g4vd*2jr*+y7<*j!lKYJKiw<KCq|x$G{5p0!ovwcc~~pQY^%O*PxwoICfPJS1!P
z(RQb8hDWX~!|Rhf!&_PR#pV3oEge70=Eov2+x2Gdd;f6y+kT&Uz;<Oz^`3W3lWl9d
z3T+=8w%q$uLu&8F=0;n^_mlT7SUu6Yrs?F~ioUb948b>c%@(cN%QAy&p9`OuEyJG4
zmaC>J?z<onZ_DG(W7oYlXD<uSESvXlkJ~nKaoE+}-Lg040o&d^u@`J*9oqMPm0`2n
zan;tQ<N0jcJ8A8^404<H-my7pduIj1-sPwN?@^m7W5;%J{@$9Zn!TFBUV9rfJa@Y>
zEZ$oqG}ETCRDR#9)2H_u2xr;c@Uyp#XJ4}Sz=!p=c6+z%U0f+?8za1aZxNg7-ogip
zwu&JSY%@16w|x<LZ;zJd4ci&YXKiHm9^Av?FJpT{-+#}o?-OhjeqFHnrD?ynM1GUa
z#n{(-b*66FqxoXNUWbLJZFTZnY%@ZB*)|{BYsrxNZqKHz3v6{BRM{SRp|H2@%*4GX
zLe|(Os7}~BYtl;F2R{OBFXZ31`E{7vHe#c#tvlE1y#<>O*j}mLxp&^0cH0X#R@px5
zJ!AXj;9J`WjcVJ%vz+_RZ8O^|5K^~ih2$LDH+MJgy>LIvHsGzvp2d>3d-oMB*wZty
z)V6~EpzXR(3v31bD)vV5Ywta_>8!25+(~<-o;2I8m@jCnwCU7d|8p1j9^g;hd+6);
zJq3@;_O_J$+B1VWZO?~El6%_jPuR=A^<wY4C0}h0_f6azlp?Y>ncH^n`AormPh?i@
z%}+YCm&KRg&S!F^tuKGUUY=v;Z9_fT_Z<`5yhqUW$KJ_`tM+bS^R{)mq-|AqeZgKW
z3FdvPRyWw5Dt%#7u|?R%Bk++8gD}(Hh3aB<F7IaBWJGMRQMk0p+N8+YW?k>%y}_~Z
zdsVDT_I#ZxW3$C~+FrK{NA_I17;O7%0izv{-<>@%XWRFDp0LXH-!ZklQZk8qpR=sl
zJFRT3t)sEpUZFKJZ9Di4Y=pKm?&ZmCw>8LFv?rY7?4ET?*6!`9W4Al3yxCSvzRh;S
z5l<VzUrY8D{N~%cXJ5aqxq;api32P4a-4{<)k#}y8*Oa4H*U>R+nQVF_V}2_?#;N<
zWb2<3vzO;}-(H0tX=|^_wR>YuzT0zpC71O&7wNrTYO#BH_w?@Njy<tg^3|ffuO}Yf
zJN0kZ-j^-Pwtwt@?G0;D-s@o7vX`xlckcqHS$hiRuC`ScP_#44+PatF*807&3QT(z
zoG-9B(4({`KwQt3C#lMI*2K$u3THmwbLGcT+b7xUY)wN}@AWI|-5XGSbZ<k8r%i~O
z^xlH2ExT8Jv)#is@wx2`sl9veD0J@eXufVE^+9^?<rOA-Sw76#>vE^i_V*v}y-Qwb
z*t(>h-OF0iV_V(jzvss6=Dh~C8a6dip|&P^&-NTRFSMtleTuCE&o!HbRW8;pm5#Oy
z+eCM#YHiwk=))!3=wEEMPh>4^^K@746*-`2=TIhNlRdr9w)BC+UXdyv+c^(pY*|j-
z+jF3z$EM9sVecW;^)_ja-)s*o)!+T>1*7%4ta-a5zOd~LTH9iKVBK2l4?n-`d2svp
zo@tg#ZTSOZ_NralWE=5s?;Z(7MH~4N0lTne+xA{KyV3TTm#UpY_)BXSp{IL%yt-_6
znOwDI*`{rKp=<5#n5nXRC;VEv_h*mV-t#lw*l^D(**o3gr|r%oH*Fo{EA~1m_U&!Z
z+i4@DHP6;!vf`eNRtxqlSn0KQ5_7NZlgW&>9IxAL8CD+K)pASPHZCrFXGQPwJ&z8Z
zww=%wX5+CU)7Dx++xB)?%HA6%5AW?*Gk325x0>w%?gh3LPXz1)s(<Wx#ks`x%~=+^
zQ#)7N9<Ua(P2tSld%*0J?H1{78?`;B_BwDs*%KhBu;)PduDyOadVAmXOYhwxpJXdB
zZ^E8+1+#2th}i52cRaYa^!9n1cN1Fn?$Ld-hy8KYUN`p(wh8t=wgLrPYzyA@@2OeE
zviI&(#(g$<Y_=z(ZtYE2=DJs4j?$i8i)ZYW@!GWaL;B-A2BtiFef?Byt#)kKdyj3l
z?Sv<ZHnuyo_nx`rz0c$P-MvTp7wnU*Uub_`J!<bW0|r~s3io}R7=PK7v$5<w{&32^
zIhUgL+Arp^Rg~YiSLLek-rWK!`-H3&_FbIsySHby*WRVpjQf_(-np0U=i5EKp}Y3Z
z+W*7u`)wcl!hmCYORt&Q+cwtiYnZ!auYdQ;y=Q0g>`&WTu`lPhmHpC3d-m?WvUxAV
z$C>u5UJd)^RBg6?GA(oOM0t&UiTMwAOWW}5J8$h^D_`+t&lOSI15<aX@4uPKZrk3@
zWxs6o-M#EbFYYVYJ#k;!+f#cZj_B;$I^)^idDZ&+s;0E>dsfl1uPx&2UX^t5{jyh@
zY_Hd4?EZSSci;O)m%UpKZrR&j*0Oij<%M?Z!|L}5t-iUpgD-Xe(ylvuMa9MUafY{9
ztMkh5`*6N}Uu>EBKF`93b_&-xZRZv6>`PXjvu}ssU2CHgJ=VA4%=byp+PrspPR%}x
z?>+netd!rkX8r4ZyI;xfJ0C4(t808_uhyjpdvAXExX<kEar=}6hka*OZM5-A2-?TY
zcWvK<<NNm=S=zaOiM#LK^$%X`jZih-m$rpz|HnH@c52gG_x<hqu&1)NXCM2OOE!5~
zv#mqgS@tb|I?cvQS81P_;bA+Oz($*<qs#X7xvt!ML3qag(3rdX_idlMf5ETyd!@J|
z_O&cgw7sS!yHA<*to^B1iu;_mOxmY0y=d>zgG%-cd^`4<?K!@8ca8SGnJ2{e^Y4(g
z|6$0yzxGA`zO{Cbtz%Wr?Y{Sa*4~A|O7>F)S!}e@{_N{A7ueq}>1S80P_Xyjy8U}4
z(;94jxo_-UWv{rec$dUJC6j43v&Em=sQ+)@D^>7xugR|c_JNXB`#RqF@5}w#XP?3`
z*>-w`?7qBN7WQK5681Sp(`{sOdG?(=`E9R*{RI2Z^X>P=C|U0_FPGY9b4O&K%2(EX
z+go?+Zz(;nPiW4=y_-KR+ZUT?u+QSH-M(j=TlYOvWZv7m`O;p+ck}jV_)oCe`<-DQ
zGwZ*-&9Bz)Rla<AZ<d^tZMOvXJ~Pgiy)(77_mwWx+jl}zV&6%*7CX+1Li?_8Y1yun
zX|VaSAYyN78`nN3wXgf4-m~o6+|6h!7u2(F#@BuOqCPm=9No5i?|*Y8TYU=~8_PLQ
z_eSRw?XL}(vUftIifv-@*}dJfJogr9+}wXDc=O)Iokshup6=a0ZQYi=J3s%~e}2vM
zJ@X1P_uac~WdHrZW&36W{k_|Io%h*zm+$@Rp|IOECS)Hw+YhT(9h>%UTc^KoiF(K0
z>2qf83)h*tx1l)8)}*?8?<9+D`z&tr+1qJ9*i)VLY_FSipuKR%(tYpp`0ST-rR?pC
z|FL(+oa=jSO&8hRc)4=#<EhFv0gvNsCv=_L-F&ZMud8R}{v~49_vHVm*!%C5=f2$M
zEB82MyxRMdrFWmyZEf4f0qgeN%Q&@v*Op~_7>}&l=lgN~-tJS4`-KFb?yKNe-FN-^
zq`mys;d@1wI@^BFS+{rIrkuSWcJl7aJ^pS_U+dYu>Dur2p3h#l_f6;`n_e-AJ-TgH
z`)2zk?oBA!w(p_NqkTe~<o4yRP~K;`U3^~z(`UP@dtTUHFKyi?dTG<%?H{9d-<Gx9
z|E;ugPeI)8{hKP4_r<?hy?05&pM6iQPVJQu5wU*uoPD=Jj?Vr+OT+Aa_%`iJn8>;B
zjzikMKYyh682hU4>nRi7_b+$iUO|C|eYw-B_r6`tv^S#MYmZ6Q+Py*_#r8e;$-OVq
zCV1}~R;hg&S0?XiaQC+1clY1-UT6F6*BM3oj%}6MSGxC~ZQiGZy?W0V?fu$hw{O4l
zj6Gsr9X28x()J!JZ?w-XVc92C+qG|N%&fhqHCy)`d;fdybHnNOr5o4m&&b}n+hf%f
zyL}0V_wF*(-dDGWWuNokdsgK?>g~^)n(bSlaMb>Lvw?l?cB#GJUme&RGyB%Q-Qqj<
zGM4b}W1FP3Z_QD8`@H9RdroPn9k`&UzW3*M1DpK)wR`P)H`{G_$hkjZ`EHx0xqAEN
z%-yo@=5f}2v3q9jd;Q9KUryb1JB4iay&)G4?`5BsZ+|_1@?Mrx@AsOSZ`{8-Z0g?M
zCbIjgM4R{Cvv|69&$TUkYqGA}y$_YNzv{Dp_Z@w{eS9rl`;(uD?>!jCzmFl+!2ZoO
z{sXD(ANHQ``nTVtF>0T@Cx?AWuh_l~Yzh0ev@F_tXG5KRhz8$2XRD^Yt*W~A*X|#+
z(|)L9pW-ZLnOEJo$38%G-``NxJ$c7G?5dNF*qeJh+veKt+pE^Wvp?S5$0qevl6|%#
z(>}xO?e+^i|Ljw_9Bv=^&tUJykf?oc<#+CV<zcutXKTV<-7BK|BBt!x=W*!y-gEh!
zc3e*9_C7i9yNBIBaNmyO3VW3w3G9oR@pLb1_>8^gS}ObQb06ND)B9>~cM0qMYrk^$
zTe-a5doGT7UsTTIJ#BaA?9=yNYa>1Niruy~qWhLL?by5bsOa7d0k(Z1Tc_+zZ#lH@
z?)Ue*XYUu>H_dF`-fa)8_bxd$eIJK#+&)$lrhVN`EB8FPCu&>rV)dTb6OH?x@@MZ`
zcK_1eA7P>UA}2k!R|^!}yKJSwUcb)Nef2-K*`ykA*@{%x+aBZS+TDMjbN^nWvwJ<0
z8}{}DpW3T=zRITT#L2x|KAYHH+cd*IVGGN?+SqNj>TmAt-LYWJ-v3`j?dGn#WjDuo
z-Cia4+<g~wTJ{~gXldIY#k;TXwa~su4qe-beqP(^xdwJKWt#SNseA3&U;M%*PiCuK
z!xW=^;*6&I8tvZfZ}F1Z`y<W8KEyR`FQ54#>)U2?_j)mI+LQ3WbD#TKhJ7v4fA-Y0
z9^aeAI&p7F$06ItCA;?qJbz=oD0Qyw0k(g;l+GO7dv?31ZR4|})*UJad-E6W*ef+{
zo$ZswjW#(R0(;%EZTC#LtFZUE-G#kNe$TcM3f^YRwxrvZMZ{)r$4-I0hI0?tPHNe>
z_lS<7t;51l+aHpXYzr>#+xy08v#rZZe!H#SANS7XKW1yQJ8AFUA0oDejv;&JbWN~b
zaekAnpQYp8FD;Aq{{9$m!@pL<=Fs{@dq34qwrzSg&9;H5c$dSMBepijwf4GBU$OU1
zq|n~_ehxdIO*{9xtew24;|-sk{+a&0YYcd7H>3*ey(@OxR;#{!?|x~~z1OUC_a@%V
z+`Dec>)kFI`)mu2ov?Lqxw>0GWV(%%^ieCDNj-bFGELjNr}NmJh8b6^CWeOZ<!N4T
zW6g1IkM_RUz0cCxY~8;&?{#(Ey|++B$5v*`X4?P`UE4LSOZRwvxo<Ne{eVrd&w{-c
z_Q!2w_8r;F%g?iK&4vzJuXyHt25)xR{%H%hE#>pw`{T_<+mgc%_p~s@*si!<Z+m54
zw=KKHhP^G$6ZXhWySVp&z(X66RZO;fgx1)KZvD6CMM|uVXk*G=2Zn!EE{Bx%J?UC$
z+c3A;HX-4-Z3APNt<SkE+Xc*wwr{uJ+*4DeVOx{lX;T)w!gfQwpzVf~dRyJt!`55(
z&fGQWQq^98m0h;C#18L`@Lz4~qTOSgz>#OYLZ4&LUddLQeOHd!N-!wcG6-MZ#c}Gc
z?VY-5Hh-4L*fLuv+pYDRzSrm2)V=b30(Pp|;&vbNHtjvywtg?O^uoOlKD@Bi{KvO<
zQIVr{f60ZtS2X!-PjB38E3#8zuSn|dy*zbhdl#4o+eTX6w!NvQYByo$gS{Nhyf*XJ
z@bA5_DAM-Hn=Q6qqzd=mmDkzp!?b5_S$>mk=RqSIfz{7#=0`BwEj^&WSLgY|J?!h-
z_H5ev%4WrN1KZ2T_^o5!@YsH+WVfCEgv-w7-lx5{kGJm?Tf@5dbcKd(;Qw~phK>ch
z8F&JA3$0pjn;{#sr(?R)-U$85J;&1~?QQ=ud+&qF|2F>?DD4xhJij+#s*uet2EV;(
z#<%wxXcz9eaq8!u^+u9*Je)^tqXJ*pNZ(Z5cjBMS-amKBZKEbGvHfPfWcQX}YukI)
z{(BRSF0|FkzO{Fc_8i+EH8*US_ix?H7SX>qj*-#U;$E|DjB47hmMwGl&e-<OrYxnx
z=K24vd&0#s_I5nGZ!@uI)?QETO?w;C<@OXDmat`5{m63f+oOAT$fVjd@IJR$lellU
zz=D%DfklV*?yj}pwbagb?-8>Dwg+Cg?op7BvFW-UZ5t@R(DpXp?L8Nltlk?D`NXEF
za`E1{tIPJT`f$#+Bhh~k_elm@&!>%6Ro55pm3ZH>_sIvDy_zEY`}(DSTiac3w0-3)
zyVqcTs4Y*twCx9$hP}UjE!jIiY3p7~AJ#p)t*`IB;xosF)l_fq0`ZF7JuQW{ds=4N
zto7P!Gi9Q<Z4+zv-rTH7d#fJp+VhBYldaqRO|~s>d~IyvChYxEt!umQrorAuM$Ua(
z+3(oK{k^%j=f&(jLY#N^%oA*|4!GQH({yT{wRw2dUag5-cK2$R+IILYw*AuEZTpYm
z%ARYw3vHM9%-qY*TC#VIMX{}q#p*pXGQZkHvDMg0{FSz=j^4W0QS;kgH|;~V+YDdZ
zT=c22WypWDTU|iWR(`p`UfCzx_E!A&v%O((dhb3*m%Xh%llC4-7v9^%<hHkjYuBEY
z2J-t}$fVga^q#ORnz7rq>t4q0CA}(p7Zi!wO4$1E)#)~{4Qjf&XS;v4EyKzUyVi*B
z-OKGGzBlDvzpb86lda2wX?qzyy4cF~`s}^3=fqyMxGj4pC`sA`sEFFM7;W6MROY<x
zEj2+q52gSclh=K=IWG71_%E8Zw@I~aufW?UyZ`=F+&eFA-`*8vl6xm4AKSA-+RVm#
z^Ca5>k@q%iFO=;>zRs{wRF~O1cgZYk6V@erWMp^R9`W|u`}<PbUae)}d+lGU*zM^$
zX#3^5)!rM^uG{47J!+dedDdRGkL|Wxw*7l8ez@()+f=_ty-C#W#NsULjzyDfxBL&Y
zePTFy?~jeYY!+>4vh}D}u>COq+MW)PYc|#^_U={So?<&S=bg>p5TSkXZ&&Vp#e3Se
z$9J<$gND4#o5X#4!`qJRz40@9&$dZIdjl3v+<THKYVQWg$+mNEWZDGGxN75BH)AjR
zx{5tMDj)aE&9<}sp?}g=B&%)jlj-yJ{OSC<C-`mqUa^3=wyPO`*i2(!vAz8ueDAz1
ze|DAK4zLmUP_t*sQ%;*`ZTGzk>^|=~_F}0m@22B>yPluhd%|wx-iA~)>o3dN_NLjd
zvULe$+&AO&>b<ic@$da`d4g?{A^%>6=PbK_nTy)qh?d{O5!bdiV9F1hoL^eDJFU|9
z9zFQn_V1)Ed$*XI*@|nh*!et7w3WZ*vo}g~kL>~RZ#FkvuGv`CDDUft*tMtV%t>3m
zzR9+G1orQ_bGgf=LV3N-B5z*12;&L14U+7(8f_PC@+$W1{kUM(9)T2ATidePdj-zr
z?OAcD*_P!<-QFmZJlpbbHG6F~ec9u>{*LXlswkV5g(q#50=C<(5c9Ep;xlE>w=ar&
zGpY{TyeXJu>kzhMx0%V}y+3C2*)kN?+pLLNy!Q^fuI(-B4Ynz=^|p<ixA*cJN9~QN
z(6V{LZML_lk9qF{!AHB>DmK_0*qmnTq4LRQ-JipI6f_L>y#9S^_wL2)whmj@?LF{x
zvek`=IkwezFWT^bDcieYC7WHrgpYeZROIbpGAiHep7Ge`%9AsDuO+zey|I#UkJsY3
zy{gi4_L@ZK?|s8IV~<+&+&%w<X782Yowc{ZreRN0HK*NPd8xhETyyujmdn~s*mHcZ
z%%0V@EX?M%5pO&9nmwJk_lnI%+wE5_><zxMz*_#%#=Qr8mRYZn-npk~={DQcrR@8(
zeD>Hnah<beIm2SRW%>6#8&gm1eZQDx--Gfx8-^8u);$$sdtby^*pwX3veli{V6FS@
z=-z^_XKYS%?Y31bYO?i_s<i!`>b*ClVvB9#%}aZhy<cV{lg4EC+`e<~bVlvH=br1>
zvIni+`@yu*MsVW>+saMPZI6FU-z$+g!6sNi$wt~}@7~oS`)vZML-tH|i`hG&x51|3
zk%$fR^=Y;ba?jW#FjQFgxTM=o2s&qbVNtZLn_ru4d_2S6l)2mYDlo6!ec;axE5*;A
zdzSlV*)mKCvgFulvPWj~&poRSvDqzKc7AWw0&}api(Gphw(r}W{Xu!}o+&469Woy8
z<v6x*kLUUcd-ERcvt64wZLf>)ioFtB*YCaKzkg51A&$LUuCCa#pu*Z#HEY9O4{^pl
z%sE_k4Kp^`@SIiJlfZgycSmIO9;T+lwwE^Cwz+a)gRRK5owgEEhi$EnuiVSBBHuRV
z0fVjfP7%AC8xQYY+upc$#vdkIUeD;gIY;?yJEJbxEMA~&`&f0y-i<t8Yy}P)+eR$j
zx0k6<c<*#3qrJ(e<@PP`^x9iqduXqQp@J<_Yo9HLm7<-@cALGI%=YYYl~Ay2aM!fe
z>YQoYUhJ{^SjV0{3??473qA?&@tmt*R~n>XXSLPLCYk-pUXeRn_A+F<@7`T@+cqwd
zd!Nj+J{yLfDSIF6-DNXJlFjZso5o(DllgmRDwXebc`s$VcKhPJcCDRz>m6M7-aK{7
z+GfSfy~`_i?49{{%AOmqAMDXl;<KBapu5LmF}Kaud{f(<4?kKc1YNM1@@t}vgxd_8
z2CHLxC(8)g`TqH1JEvFH)=S#Vc0K!N+aNRLy+3A6-<zPc*ILN5)@H_p1AEFAP1`%=
zPVnAhj{|!rM5o*8+<Lz!I`G_{36^_p*j#n?F6o+KbLP#FJ=R{edz;=ZwSBx%!!|&i
zWe-z$*Pf>qKWtX?zqj2c&S^WdcH!Qudl%czYMQq9+tOvWTop_9HhrADH|BDjZQk8A
zwhs<Zw|Qf|U@vbf_r4uL{(HLIe((LawrB6L2@7r8-f!3|(sRMuefO5V9Y?urFFZYM
zvyZpk_QKR`8-ag&t#^34@42V3Z*NK03|oaQZ8oyL(zahe81D5@aIme)m}T><QOcI(
zUZico&7U@18yWU?oLg+mr+a9xb{z9Q|MPr%dz&xposhC&Z<hSLy?0)p+7rREWpBXj
z(7g{8TWuA*B5Y5nbL`D%)7>L&@M`btPi=euGS}>N_`J_%OPb(bo{j5m`z~#@t*X?u
zUC9!<cY#lVO|O23O_B6O+XHtG?0%AUdhd$W$~Gz7Y<pLQ=<H1sT(|d&;O0F{R*idW
zg@pGlz1qCjVO_;;zZG(JGbiucE3l$zZ$qB<-sMRewj7%#@97ET+UIw`X|I&B_FiVg
z3$`{FTlcsWi0!*`{nVcONxyAHYEy03i5%Fw<Mq6~Z8zBW9=STf)}#A~O+k8#^$#T>
zyLXa@ZCi3S>@l=uvpZE%W1T%|i><%8fbEGNZ*AwDv)gOW-C-Mcu*sIOj%}AkaH8#J
z5s|%8O{;7@3e;>J9<kZ%I2vO6M_tBtUa{ofrq>O7Po*r`TT&=$_l)tJtu_OLox~@F
zy#|LeY#QA5ST$%&*_&3$u(!B~$4*mls%^uCw^n(})c3CZ^V`;8x2tWG<>I|TlN9&$
zZCh)5;A*aoM`DuA0p)6&2e0PXa6Q{*`y}b=9v!ZEdmhHs+Wwlsyq6=g%2rOKcW(eo
zwavxfEW4{Dy7nq2uC&z^Fy6gAj%%OQsa>`qeJl3<`JQKcWU{C2QP)1(Cx`p@sIm&!
zt*M^5_sN2ZwkwzP@0D8FWvg+o$M$2E_nsR)EB2nLUTdSV=#b4D_afWR42$-jn%TWK
z+OA@6R-gRd9bI$wtnX{xdsUptCY^iJUWQJky#|WwtxZf~ZTH>3v)5+R<~=3PbN4zd
zG~8prCt&L!{nX~>7Gc{y-mX25j{MoH^mMuHab6j_993c4FhT3RcdYYlr>(cz8+>d3
z-imVnJ!(>$ZP&OR+@r9#*QPM!(cUx1SMOP&nqkXhrnEQiV59A7>!r4q8^ZT)Fh5}9
zqW^GDQ}Y5_3-_&imfZZlr)?3NEwlJn+dtRV+M1kwv3C!b_#TzU+P%H{s&+amXKed?
zF4!{Mh}t=~cJ^MG)q=Jg_V3;!F-d#Rgh%PNtnse4{7=32zC5~n@9sq_Y>t%9vGv)0
z+%|>5)z)3X$9hK@|DK%d278s%C+wZFvD0>0?}6QEk*2l_)PL+Lt$enJ<(ayz!O5F@
z7^b`1e&F1+H==v-p1T`E_GTVu**l@CcW=hDm3x-IN!weuENbrrgGig6f*H1xCQY)<
z@Nl&;+u5`CfKk})U4@c+`zChUUhoRtlX3Rno(~f1Y|m=!-@9bq!o8Af_S-(t%(Z0*
zn__t)YPZc3)#FwNBA(iOaW$|tcqn39XSZhWn_H9idi3bn79729b3jPnde+v3d-yU}
z?=29EvC+v;v;CvgWSe*Nf~}O`AzS8c^7|wtui7YcNbUQwi__M2diLJ4VjK7VKJjgj
z&Z8T94lMa*^YhJ{JwKMcvDvp&b+4a9$lf+C>%Chv*lph=Z?HB0^4rGatneOL&V74l
zWX11|dnvg0g6IruhSD7C4(=1S407jfcFWf7RePFd_pu>i@3zMFy)Gx3ZCE9C?p2q{
z-OJ;&%eG-nrmew=WqW(x3EQqHnzff<r;*LF^pCbv3W99g>bC4%bkc3_v=do-f5x%z
zb6}~mabjt(S++`hU(Gb(y<R3i_KHgD?F+Epw3p+rwXM>2Cff#4J}W)XZd<?44ST!P
zx9^SnvwrW5zq&R{jvn1RW96?s61z0^i0qoW_r#$T+bb7YY%?t1*&IkMwDmFly4U5<
zLYvriqITwu`g=pP_uD#mZ`*qy^|p0~#7&#0r_%PWPFc2Z@!lr8ESDX&e^+bnJv8&Y
z?eeyMyH}6T?LB1MWXo2ew>Ke~d-oajd3#%w=GY~_v)HS)Z>g=s(KWUeHB)S_@h{%{
zifi89Tf9s6-f>%Cd+AT--i9ZO_nyh$wYNa4c5h1D-#wFj+ifc*_S@J@I<)u4hj%sy
zp7GlXION!DQ2cIV;mx&ojmF8nm$J{;9?zGtmipCd`*_kqyK576>?_gRXxmY$ZL_d*
z^IpY}Z+in8gl#7>v+q5i=DS-VLdeGCsDf>+!Fg-lBP(p#QYP8n4M^H+=i#uoC2)gn
z^|^z4XZ=ato8Vq<Ew5#}@0EA+-mLOoTY(7PJ&X6A*_-6nYB&9h)ZPI3&3iR_PVDtr
z*1K;5%R<`|`h9zzMN98(*j#1(I$@{nmQ(F}tTe;-9!!bd+sk!i?+fX}HVQFu`y8FU
z_az*hx3_M`B-?lHDtmL1cG)-y@b7I9QL=M5`eM&>wSD`{U9)W2FIw9@OEBEKM*O_3
z*B05m$@zJ98IFCnGY;t3CUAbUnKyUt-W*5sy$3WW+dg?|X8YyTmOTd~KU@DZsN3^E
zqkGSnJZIZ!8SnPmeObD9h9lpe#h?85?s#cnd**SE-Tc{Fwgvr(w%V}^_8$6Iv)7ih
zW3RU6`h5>Le_CgU|Jb7wmSg*3<BdHNR9p5w$XmKMAo`B=2J!WKKR2GRJ>tG{&zpBL
zwgy}NTN})oW>fYhX0N|e^WJ}34(xTA#jyALs_A=w9J;XA)Vp)9FjMs22G83zBI+4?
zTcox2E{ROFE!i`9?*#T;dmGdYY#TOBv~}S6V4GrXY<HHgWbbn0^?MDBOl?lZ3+z2}
zbpKu}^BTKFYaRCP@l&-8_<dmSo85i3{1GC1GHyBVu{h6Q=b4zlr^Twz*67ki+eunW
z_x^FtvejGt$fjo1gnb?s{x%y9?AW8AH(~Ev<pp~U_(g2i^=Iy7h-BV%MyJa5tn$RY
zLW!-mm-Kh-6>ZPA3z>S-R!C#v-ns}2yOozW@15X0*;Zc3Z*R^t#l0MP5qk~pi0u(@
zFSFgEG1vC;v&nmRoLp}E=6~<r0D}+KAvY%P)myRJmc2sT?f}PMYlUB&wq^`xY}s<=
z*}gt~Zf}slTHC3|^Y%ua<lp<9*TOE~OP1}9gn4^=oR98amGE-cpTNm`5>Hm|{lI0g
z_rkvYHhZGZ*xKAqw(YarXZs+kz-Gyu{yhwH#4R_RJh}J5$L>8hns@HKGR4tW!P46L
z1;=C?hR?ffY;MlpGb2*X=IGQzdnG1%+wy-)vdgO4X3HQk)yCp)i|s7o<+iK+tL*aP
z=IrG=;<qngqVis!dOO>-th@KV+o-tLVO_`G1--{@-2zzcgd@ZDC>%Ip&Ct8qa@Jjy
zeG<N<whdcT>=-L{?v*@lws%82gH3esslACyH|?EPm9<xBS;yXIPK#`pW%k;ZoSd<z
zp)Y1v!tLd|7wlQG_d(NBYZd?QJrnj!vJJS-X6yH8lbytuPqsVMX4p3PZnizMPtLAy
z;Y!>8jGOE#?%UdZxT;{Q!OgQrV$E%vBOj*hc_C7{$LG8MzP+~odu`$sY;Uzn*-bD@
zv|VPhz?Q>bVXrd7T06eI8!bJSUa;Xm#%9MYad(f`iRF7A{M&D>;jz&6N99TzVNJ`u
z0akBqo_VO*>PRoMEm6z0S@86oO@rd2T?ew(?KV3mw0BQ#u-%NdLwioTUD*3UEOJl5
zjXBmX{44i7s+w<iR%y{58|?+QI@3e<p0v%|Yq9t8o|3BmJ@NfYb|NmH_pohd-+Q5V
zrEP-h`#l}=pV<iP{kpf9mt)_(#k*`@Glkg+En2Z}!^&3M11taA2rBW|#fKT&t>P8k
zo4W1!9+9@ny;@Uw><;YJw28P~Z)0gWYj3UX<J}GSSM0gGDsH#G)akwRCr;a!vGJE}
zNz>H5xm?bBZ}=axox#Ip%M-Z8_Oj8XJ<m*<_Fl<7wkPt%);%k}w%CMzySwLuV9MSz
z7613#dM#?#&#1hQ$L!S}%h$nsPxZIi@&x_e!_&3e#_Pi}+XeaN)`As1d%f*W?%h$<
zzsIE2z;@4;YkQCU+iaU5lw@<Kve$OPq@q0zOYYgsIR9_YqD>2IyN{XLy;%HV@1D+b
z+usLP*d5C3-F?C)+IExOY1{V`S@(71SJ~`os<t&@+qx$v@AjTomsZ=ld_1#v%~AHf
zV%w+f{S|w`)_0k}?gjBvtX<O%@9kOHw)fynhrMQo8~3^^+^|i_54N>=plItk?a1EG
zF3W8E=P27<ykN0+$z5K%89{b?oTl&J8&n&%H_U@`?}Hx?Z7f!B+x|!_vzZZd+-Aem
zVB3a8jW$<gr`j&45wf)vvELi@KW=Y>yWPGAai{lQNH5&u<Ho!1jsp9>KMQ;IO!ykO
z=SkNt8-bZ;Z31R#?MbLRV8d|W-*y4hJ-c@?UEcd}am!wjsFJ<AmiE|MCN8xLm}6=4
zCZx~irROzUgJ1QwJ(mveec>{DZ-nbH8<&Pnwk@s+dr$rk-FxATjm?BaRogl*Wm~17
z>3e<7bnW5HUb@#lXz5;?)Jj_o`>i%>?ya|t`zvP`$Z~e)hsf<VZYx`CnSMRAoy2wA
z=0i5ao@ce~dtD|+*q%Aeve(Xi>fQx1{JV9$9BgAgYuZj?m}x6pv&s4y?=D*gonqU2
zp;PweW`DGOG9lF_V}_57wD?imGYhZnoqtWnPTQn;?-$q3z4yur_fF_2vGqDOVK2w!
z-8K&rT6P<<Fxu^6mDqPm<brL3&+1(cr(A4nGq>z@czVM2)Qir&&)!$=z4>PHUe{UD
zb{S$xwr$EYZOi7ru~iG=-McwY$#(CYUK@v84O=;{W42m%?%6yp7PYx?bD7PPFok^)
z`|NBxL{Hf;+`hDX#@?_!C)O?8tFa>3#=(C6ZVtbRd+U}Qx9u?0v7Pfqz&7-9_g>%h
z8GBc3UT7mA*|^tSMZm@@B70AU{Pw-_{ZsdTd9Y<q>7zq7AJ%NNy?c_?MlX4)?VfTT
z+pbrgHUW>O?0L|zd2im#j=dpWZng=GjeEDxU%PjK#avsSADXt8{_nDt_}sqtw86VQ
zE&p2g7M@tSH^fxP_Nc3>)tO`^+esVO+rGF}XS+#&ZEwMei@Rf51otH@`Mq1`kb~_K
z<4Jq|Ngc3t_<7GNXshJjO)|@E6Ph_}EN(2a>0z<4?btZUhT+nIT@6VB`#9I|*(hiw
zSwG=CY%4jjaIe<4je9LN=-4i(Hrt!xlC*c}Wud)ZT@!4%c6Zy(2|BgsRxpd*nn~;S
zKEHfw?^2Im+eH^u_ckb3+MHkSZu4=L?A}YOqxb$%jIy<<=GbR(zu9)uu@lyQRgrr{
zO6+XiV)yUOC}gygVd>fXZuJJ+)a(29R-{h0O=~%3TV}h+>dD-OJ?lO5_tu<Wym!V%
zrad|RdA5?rME6bd`DatUtY)vxihdh=VV%7)*N)mINyXW|(3aeHq|9*d`}Rk+84gXh
zN&GwZrnpt@37Mv5n<4yT&mQ;xw(7sUtk)b4vz1&}ZYw?c?w*TxkJxJ6+_X0@UfOo&
zGS0nrGUxVg;9=S0pm5n{N6!jdi@asFFUl6}S^coxX2T^P8;>L*+r?=vd*6Ouvd6=6
z^WKSTYxWw=vav1VJ!Na@&15TE_S42zGT+vrt!eLpXP<0-GOF5ryHdNC!{UXF2FE*_
z2E78CZ*#Wo{j)^Y_QcBPHjWB8d#eKv?^Rs5d9T-%Z+kUf814<pUbi>FhHG#4bK!lT
zsw?-pe{J0RWz!U!xIH^;FS#$<YqD~oZDq61-ZrP~y=(57@BOeqYtJ|1dfP8tytZu4
zYJ1;=2-z_`JZ972GQ}$6vC-ZO+ETVDI_b8)Iyrlv$BNndoH=RJ>wRRe;Y|I#HWwN9
zJ-M4?8|yV^@BZC$_O!g%YQxh1e@{oklRXwz?R!6QJhwJ!XS2QWWT}mZ`u@ENcP8w0
z$dB49qBzM`G*)5Xia)#e?v_|=dqP}vpX2L|dsB4-_kIabwKX`qW^cL20h_bJYI`|$
zy6;)yEw=Z^mi>FVA06C#V5P0~RmUTH7n~Hdy;81as}c8Z&pD$*wx{?e>`ExOVg10K
z)%Jk!$-T1oxb4&!Rrd-^`?ANeuX3-9dBxseQ&sI&L~pfaiMhY$!ISGYj=VQ*Gdp(f
zjW}}8#^wLoJ>t5G))haZ_cqx6+MV-|&5n1Eh+TW}^gWkydiUPXGPd2ZZI12rt$DjH
ze7RtI=?k~rmZ+(FR89Ww_37ibeX(xqo*6&(+RT`$Yilquc8`qoCR?lIqqYwP9@=zY
z{ktcItJ(JG-<f+#K6%;B`1!~7_{ybwWsTPEJyChX=DZ@~-pm)-woa34_9o1VwiS@!
zwu|_)&i29O8k-JVEt>-iD)&SwOtN*{Fkx@lgdMgjZqsb<{Qs~=_x9wyx|4bBzAxNk
zd*$U^8->M%dou3S*-U!4!dCX~M4K)D7VNn_O~|fs%2J!;$;|tj%a-r0I=Io+Nbb9B
zWbn4V*KYpVQ>C)eHs4!zUxz=tZG_q(8?~FKY&V#f?FqOUV#9j=?jHU$RXZO0^1UoS
za`xW*RJm9A+y1>jf;ji&i22%vrBB~0VRv=UXYcoWH>ES!aUSipVW>QB)9LQJcfrih
zHg6U`wpsD_!rnUt$8Dv$eE0q-pKUv@z+f+vBA4xx%bRS~>^b)xVcu;UC@*NIFtcD!
z&w+(|4{TdxBXoAxUIE58Hd}O}Y$f_i_DqP|u}Afyie0i}%wF|Fb$bKmF0vMB?YFkL
z`_Lx-_E}rGJ+o~yjEik1JbhsEDRzf#4Da<lHx}3LneHoMXCwZ{_MK7LULAw2HX<|E
z*}iFQ*~?pT(DqK-^u0Rb^KBW#CtJrbU*5C2ddpskN&{Q7Ny}_ssHxaSwVT>jOyS?>
z^M2=^2i(7GH<hLC<#=$>W<{{%9_!26wh6q8_8y4(yyu?|>)z+pxqG*GZL%%>_0o2M
zYr^hh7R|ODdV6;(wA`=>`0lnR{<h2Bd+QDN-u0ipw^7O8de$1=y$q*SZD$vr+}q&)
zYPa35RkjIdmhE|ZYL)eN%^iDFMUD0{?K!shz@z%zlcFv6SX`fDqaLwrui3_9wmvJj
z?^#fI#paU258E5RIrior=-un3rEhbv>8Y*8p$|40qRe~Po*vzM=d!ZxAFhXc92^5|
zH4eJj^2Ijn4VBZeUNAdp_k!t(dsO+2_PP|B?affzus0w?)#gmX?mg_LC42v-Uf8=Y
z?Zn;>|3B`zaA}2g@Qg>hBNp!18*|jbcGAj{J+^ly?cL<fv2UW$<-G?4e%VZ@_-k{5
z>%UF(-zJ+39#h-@wkK^@*sa)Oz#6niq++J+qK!R!OV%sfK9&~Wm!SCDW~I2NZN~hd
zJ(<z`b_Fd0HrGqU?0QuM_xT)~WE;YN)n=_T(>|jq6ZfjKJ=yCk!?w3cp=0l~_#3uT
zX-D^V_f6Y-AZnkDWjl-Yue{l|H?Jkyrh7lxQ<=7CZ~EF4+ur8~_J-Y`vv<)PLE8&m
zNw%!DNB0V3?A#;!Mt84$|G_=g5&5?B7q8jdV6$gW#6p3+7NRkG<d5+0-THXF^}7qp
zY~S;!?7QbFvM-0v#P-J=lRd8AXWHq-s@Yze(XoeP?G{@`^Idxv9Fnqgm#^IuB|XnZ
zrR$~b)+Zu%Cmm(>uIQM!w}hL|`e1#EO@NBbo>~i$Jp~y7wl6lz+rHuxv|EtlU@NyI
zbnnw&Tz2i&uWi^q^zJQCp1IdTRDQ2Qy14b>x6}8usGPMepYdYP8_vypE87n5-RAsv
zZ`?EGeVej*_VV12wza#-zwgkRroBDg9eZB{EVYTgac-}goxW}4_V?BdC*JQ(&OBf{
zk#WAwmZRHj>Uxja_B`Kb`%&caUI(EZ>-XO-*@h{e-OIUZg-u^jxQ(LCX`5}wI&7aT
zezx~i+(nyvIt%u4h>Pt#tuMN7_0t1;YL;;C+m|6@yO4jW<+o&(eaBe$+IpN@X``{>
zqV1u{bM~$b*tRD~a{perxh>YSIGOe?I{9Ib{lqhSHT;h65#7kQH~SR-KJ|b6wySS&
z*a}Fs?me?;{obM<cWrNS-M6i=G_^g_eAxEt2i|?1&76CSQvdCppTl9Z?8<c8)pyrg
z)!GN!Y^q4vtrmRW_KneOo6}!TSsQu9?BY6Izc-d=_1=;TPJ2)1ckVrQE6Vm4ck|vY
zZ%S>$g)H|5e>`r>XI*Fe)x^Ozw0+v%7aTYDT$-=G_wbzGHZ41t_g1f+v4`30yUlW*
z$u<oeYIkSmT(UXC_F|8Ya@gLfY6|;qb_egBVJBm2TY7J=a`EK7;%0_>cf0N0`ytGD
zuZ17uKIRL7Hu*Q?Y`5}nw3+r~)1KFRPV8~WX4^e)0sp=Y&sz7oZPDJlfKg{RbNkvo
z6At>=$oikPU3q@#o{8!Uc3*n>Z0}ovu)Pv05w=PKj&?KTp6xNOx81wgL}wQ_!wK6Z
z3l8mV_%~^{a+dgBlZYPM50+}SseI>bU;DkbQBi5#yUni4_5g$JUe)8ywk7*3ZLhSm
z+47#Z**mRMZjaW<#=UxfZrX19)nNO)GSpUH-)HYKRyo^$!jtxjXB^$@eD~4bhI5Iw
z8mcC?0lW9vRK1(E_or0j-VF~0_swO^uyvTZ!uH3v$9p^8%-j1mQp5Ia?tB~Gj5&Lw
zvli`*3OsGAmGW`#f&vfg^C?sIGPb?gyP<mR-WPA?*tQ%N+sAPCisgpcqPEV_*Y>W;
z*kJo5BGTGq|6|(*5e3_&ar5@-&75PSx#7fKck73Hl`m@U^<|5&RWK3U!zL=TueoFC
z-cmC;n*#BDdu942?Ooxr+V;KKguTmmt=VgG?DC#)`vrSh?0aoKYwX?otxj%lnf=tg
zFW9H=?aa>H+o-tBw!wXwZP>isdw-tF+Z!S9(S~pL?Y)(&_U`ql7O`C?es%Z#DFu5b
z1exryS5w-%b>5M^h1(l!6%~qY!ix{u9yrl!ZGLdEP3i1owokPBZNsBx?+N;Kf3KsQ
z?cPp9**)(Q?e^qtS!46mr_J_kg|Ut5?>V-HC2#laV`AFN6(YBnZEB9S_?<1b6PEQ`
zH&n;(@e=5?z2!1}@9c%~duM;Wy7z=noQ=Ys`!*&)EB4yo=GeE*WToxaZ3TP2)wJ$e
z#By}6$~oSBKN_0%{M9zF{ZRhZ=2r9)TgBiY+p8}`_gd`l-K*B`zxTE`<KD-Uy?3|j
zE#7;3VYZF*X$9MZZ@X;O#BcANr`l)h@o9w(Q}XJ)f>%2BKKRMLSIfQCR$F$J%_;xH
zy=Laz`%Zp7VEe$`aPJ?lq}_hUIc?u1itXW@6k@yN_9W|DC!Xy+<;=Oy)TG?@$Cv9?
z-Axj9oDa_1rWV}VD|BhnUW1$^HeE@(_ioHPVLRP)hApSc3EL1K@x3dg`R!EtChjeJ
z^~F~G_Kdx@i+1d(T`Fp`w*Kkf^V|OIJ$oi%Pt}YKdpXNb*ed<)u|4?5)Q-z5)ppO;
z?|YnO{cJ)Qb!|UAeYE$7YSZ2fo%Oa0!j*03_v`Fk^R#>KffDDvbz3*>{d_QSFGKZ{
zy_vRit=Feaw*6I5x3_x^pWXitTDJbNvUYcR&hA;x)?zzHHp15B=?t428`keV_PKda
zjS$~ngE=$zIx;BR76(M`Jz>ga)0Z{Z_Q(^7z3j>}_m&^7*fU|*g}tdq&1{d#?AZH4
z^2?qzfB)^-d^>z^)!mggl~tSe{y56M?+8=--h2U-y-shl_gdXLve)gO#9oJ)VteMl
zNwvvZq`kLm(waTLa_jas|2k%~Xv2g(?7GwU&b86n%k{2cuV%<T+hCT>HVal7?q%69
zcdzv$c3a*XUu?tNI`?)gl-=vvrD*GQ>EGVv_p|I4rFHJTu4b}VRbFN9*B#UM)E_L^
zt5+p$7r=DD`pu-?z50)0Y$v!2*e$=sZ8s~9&6dMVde7Qw8(WPE>AkJDX6|8)4BVR_
zyxz8x)!SCF(cR|Q=8U}$E~)O(-?L>;*_w%a)3+?$8@EZ>j^*}J8`dIOyFDkL+cxfU
zvz>JPv#rC0+TDHow(gah&}++JdUUt+T=_j`m_u#(@44^gceA#!OMkfc_Y_&X$>xV_
z=hkHJiBrzmdn9&&O@s0Iz1FOcZU1$**)=MQTQ3dUW?Q`D<sPF5>wU{?J#6l}#O`tU
zyL;DWPtko9Z|>}2Km)(R`?etqiF{Apfy^g{9h_Od8KzFWHQf@#PLY{o`BKYx7l;kQ
zzV}vw*|X}3SAzI$8}oPW=rpqcvGL)S(k{!LS`)x-+<Jz27sp38MUeU}#~z^Q^$J^K
z1rk5B`6$?a%N{pv5FZ<65xcV;CO*A=8Q5JgJ~F-XWFy!tWHD0Nd2-HRcQN_@<wn+P
zr__pJ?|&!ljW9Lh4k6|+HjFM(*R*(jzzA#}Oib-=;%4tA|DEc4?|{R|`b9ci{#t(>
zm~FIV4%n=pr>|^*=>yS^<^{v;1M!jZDj!9#-{;kq?z;PG3Os&cYF?%P$8bMPu1a7T
zSTBe_!CD>c1`vA}v%8hvrfHi%Y#7$NJJqs!-9-#{w6047$8T@iGMwpAJY5uSHq1O?
zXvvCsaQA`K-*9`s7DR*aZnJYcw@zuXD7K%-1L7m&EnlCnLgs_mdRkAx?(6!na(l$w
zwP16TE0==BK=L5`tD^vg4-&(NQ?*XO!}e$Zn>9>rf+i$CHSftF$~?1%ZE$m7W}wsJ
zT-(5IhKa`(eFUc?7@u17opbHr_(V3Zk+BJ!t~SV9V93GLwJ&J3cyo9bL4Uy1lS6xE
z_w0n}JFw~$oZWRa6|23PBKNI0=S=|1J#Li3h>OZ#O|X9^ull&F!?O@>9?TAOTF7n!
zN*rV!PBNeUEXWjH9gM&7RT0>35dYetMtB~G3krp^L2}4==~X7UoX*{i@ba(0`;+<X
zgOD;mo%0ihUqEK~Y7{}-*S2V9{rVWNIU<Yv&B`CFMDbgEq>feIWG}eeTD~!W!$EH2
zYp{LS%B;cWTq;Gab3k^W<I;P~aC5BRDS+)nR|Dg_9J#q#P$L0scF$KKE8fBaxVsjr
zT?5+<Q;Un<yz&y<KA4&(eT<fR{H-W<%iq1O@!DYr*segsXJ8tpo*4SWUQw_;ud`-@
z!v>^h-R#L5*+OQZ*aH$r$LDvO>+p%(fS2LtPE7{;EiAuk=Zh7Pw0hOzFx*^p{iFQR
z5Eu=C(GVC7fzc2c4S~@R0ENJcvOsVf0>lR4+f&L&Zqp>ncftE~+V>5>?GGE?DD%li
zhv4n7w4YDG?8>_$;QoJ^?wcJck-}Kpn;>_<@b4@U%Np0jtswq?2bG-xrlLw9_7#3l
z6nCEC69KoMHgEF*i>Kk~|2HhQ2Ac;mN8bDj*j*qtI$qdj1NIBLI4OK~ekriKmd*cz
z(*IvH^$do+AzyPgz|3wocm(dV!}uV2;nVzWJ8ELU{sZwr`2W}5&CiO9c9=cX0QcKc
zE;hjBGp2-r**_NOfz2wu$N_IVg7iLa?S$J0;v?hP?;!oP%*UoXEH-X|$1hAx&1?s7
zI~K-ArnfH=gooRS9YSC?faKNfr&+WrGN72X`OkB6<M0(2{eSz1Ibbt-g8IQUNIyDm
z{_chmhUjXr@r^qdp`^P2-{<iDzpa_gjt!GPn#ccFMTswvIVY`NVD$e_^TmL}HhE?1
zw&V5x!EUWF+mB-Y-n4~aagZ6r;D>t|;b9y4#s!>yK>E(+PXhb5<aH#9`5-xLc(bAy
z+#GE3AhEWm{P6ez@on8?QOX057&UO(?q}d|LN?2(e-F5iyIEp2h8#?t*cmtTqV%6&
zf5XJ6O-l&efQM&YSqq$fLZ}t1y-_Q8tnM^H#tfEdxMGCY;Z;mv|Gaw?w@c4O4{jbV
zcW=;-M2Q1el@Auxmcb^t)c<<s19z9#{+n>UPhtb%Y;?VsbuPp6zTwFk@bp;od*-%u
z^Z8)@<tclxy7jcdG_Zdb-1xiW{OmJeHs4Q0ixaGC!1`czEz)|ly=~cCxZBF>znFpK
z&c`hU)7u<9z<NWc=EBQgkUTcrbM7tNEeq68$EeWNr|3>xbvC>RY)7>6Pm61Z`QUE2
zdUz{F{GjUx@j0%rVC0p&xgr)bh1uZw52Ws!nUbbXX(QO*Es9HUhBrtpHe7OXIoMz3
zJUVc_OKK*s?|XO>&c~(?Bvz6asC{>;7Cat9_ql?@Go6cf=NzLpu%Bg@C=xUWWY#Di
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C
z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVCuA@IpZ1iXd-<|DUT>KN-FU~;%<r{e4Ib!7iG
z8-mv#ycJEdY-$yOo3;J~>uzNIaf0<brOl^;*HIvgqqA@IU$bCewh_EuWaE;ko$4PT
z`~Uhr_`%Hv>9Hv72Cwt*ova032T`M6p#u`fhL7tzg4KfLoYtL0F$W|D!>@Bt*BHXY
zNuhUCO$3{N`T1m&{bc>$(!t`$W-aU7ybi|y?R5(rjxaumR{q$r(^32Yig;4J+7{;X
zi*`ivMuXS?vA^wruUDDzJ_^iUFQ*HZTh4C|4o8q#Abhse9An)bNF0P&T;#!VBW~)n
zYa`Pl@IEb&I1FDDn1<ngnB3aCop3iA{5}qL14wTDhDla?i!4y=EIG8>;>m(Elr@1M
zwbGv^g6$7!LaZSLi6dhsRw=kW$l`?9j}q8W^mPX$!^6G3Q+MauVkHaj_xpL#%@MWY
zUX3nxFtiW6o~Qcc|E)KtUIe@OkS`0|&mi^M-@;J%ATfOS$Erqf8V0E;JkAT(8=DDP
zqf}+wjuMX`b=a`7)DoO=1XHi+$Om>aj4!rqKG;7nKDFp2TN_cz8IZY&xB0+pwu~03
zW5|KzL6~J)m8C-5Zko6WWIhOsd%l31&vtMvJj}eL&Y*-1NG%8l9MrO&#B>j=E__x1
zyet99f$*%{wP1b68@zY*UqD@74U$L4&UgJ#(&OxJ?q;`kHG|hiqN@e*eUE>Fr;h_W
zJmKl%#oHRNy&yH{SXVs;F2~n<7w+G+#s22zHUGhWI5@)*#T_s+eug)J^{xt<Y+-*=
z9;{~GG4`F}J{RHUfYhFyCSuK_xe8p?fcT~2=fQqZRZIk%AOBnvOmCU~0=|X~qz)Ut
zBXb$<$K4MOg6+hn4kmWg=QKDR+`WS=SXx@)?y6IG33fY79WL7Fh9WpVk>$SyJg~TW
zwG?bFO#Gu2yVl<HCt$JSP0xuk8>SaTt4+TH_SXmI&2an9Gk>1QzqlN1CP)n~eEr6C
zE!F1^@UVN<Wev7J{z2akpBHA}`ZZ;)Ey1wDW&Wu6Xb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2DQ41r9phu|?9nD1Jj$rBuhfT@Mi*Zda3$A?XoCxgcy!oxgw
zuAZ6%ALCFqf{aDRJcR83FYoQ$x#G<N@K^}U9Bg#aZ4s+Zz3JdFrHtUnUHMBCl|kxS
z7a{gVgZQWH7l7S;^Eo$|o*2u5bNqi#L<4+$NL~r|*<$)7^<ck%%qJILJoXvvp8FiN
zDC7U}CR5;H12bD<cMyDR{9@o`@YoJa4n*Heaxe?7S%VU0yIg;4E@|c7>66t94x=Q!
zqhR|$>O+%_!R*67SAfS^tv_Ffj~#*Jng40P?E~?V@u`ERV83rH3g30|<|KIh!qj*-
z*@MTMV0>g+u%I2R7bNz;<~^KkeAU=0dhuSAvHnm`Z%eyaRunOizKxIF!1@H=1flpB
zBnQK{O1*K)!PH=*FU|A@n+=m|)}Id_|BrG%X5J>EzteU9E|mBJsXh1w^ZY-af7`(E
zWXV*sJu+Z3nB6$94#j-Q$~h=}kXiU}(xgN1w6Wyz8E_ne)aSi&1&d#0dk7v!1&QIq
zVa0F3>hY<`X6AsK4^r3pAepQ%1?k0yCwWMM!v<MROQ!&M+&^&QEDSlAx=T)l7S7f4
z2>Ju2o*eqZj7xAof2cnSXR8O@!s@O@P8sWr7nNW))yONu%MzH~PnVws`|IjErrj0-
z&*0|4<k9H|>zz>IAbAbDrQ7dIDCIJ;p3}8`V7ozL-Y*xz^=5^4!`UD?WPIZQVhsJ?
z)Ms#i)!sU29<3|^PM@M?ld-y0c=1-S{BH)!UFqujV7qrSMOzwhSHjH!+4cNv#}0vs
zYvE=vpDX}}L;T07;Ihg6lQh_zld0ZdK1dHPEGLdQhcV)|Jy;(uwF<$DSFgOj6Ralv
z=u1oHPq*N12<g5Ib^|WGF!8O^_kjHc<AZ4JL{ZE5)hh7wH$h^P%xtwzmidFjSxt)_
zY$r&ZSp2)c8SJ(u;U>7d4$TnW*vA|UHWQyYH%j|-+T?lRW%xoH4RH7$jhEbY>*02=
z`~Bo137Ug!$9pwHxH~}NGTF=|`$^I5JiPp_-SQLcrbiQ|T1w4^oL2%e_eRofu)jfU
z5N5G<-nlU#608=)$A#A@vs$h#I1irh{GwaG6L$V@O0E@(JCC~l1@ohS3xnxjB};Lh
z|9fxCL}QqJ(%V?U;RfR)(;sKpfz3h|BbEL8lpNTP`!=AS{~L4T8cKM8%$-{;0I!di
z2rR%T<8}Z2Hm_Oq62(m{eD5}gmNxAWwi5)0QS8fiaCwc-ka_4i0hhsLiEhbVc-jHk
zw{(Rf+&&N=87JsCgU#WtP~PQ{k_(Stm>TI7o;btp+}kLypFwKf#96`SfY<_&Pb~d%
zby3XH`}Nww^#l)!7)WiAK_1wQMSFf>gab$(gmt)|!_5ZqiN#lC4x#AtH;{wZ|Jf2p
zcD|{$*b)DKDaQFf;-9-P&i~n2vlU$a=RNv0ZPF@ku({7-&Va>W?(rykO0c|xsRhwN
z^55WT<At6wJU@4dL&}cxQjFmE1gXV`?_V>-8AmXEN<NxkH^cZ@5#cCqhlx{4b4t%f
zDQ7_DH9s^1`{V4RV;FKEc@PfzP-H&Ikn}Z4Aicz5g@DU&KYMCr!^1xCE*D%J*(|4$
z0IO3EAoInAxqTSvQ*f&t*gxx!m+pEnQygv{vKh$i!y6E1c7eo2mD4Ti1uo+}|L5ly
zBY3=OtQCf*kKIZOP~s9~77Q0EBKk5OM&@vT$$Zu{|Fg#qoY#79UV)3l^nqykzmR$^
zA+>3z(7}CRJF*@y?s%2_9YsB-(JIT`+g#!PjXt;w-0%4RHXR&B%b)KD+moXpKrlb_
zI#s~UIr!iM*iMjJaAC`xQD8L|5fPRvcCLZDYl7Z<a9rTh2NS=ZWC0ITm{<$n8H?#j
zcJS~8iF<fNXgrt-Ib&|wFEg;6AaP>xvE38F{*pUZ3=R{JKF+$k8??mk5H#!kwo6(M
zf2zaFaFxpwz-FAe!nO0QXb{-_%3`_TFb0`R4CY)f1&$+_+7I7#NzxC}=aeY|?;mQP
zhnzXLciG<^uA3dfZUU)U$gmE~7F^8$KL5`?Zq3f6g4f{j1X6<ypGtjUd3+{2IK6n?
z(A@R<{C)8Gf5+||M+q-cwQ6uYZ7-|=i#xp(!P)<3+l)AqOL@&4u)DC??fMI{mJypA
zv0^(nLE7Y#*VLf&Cu?@JVAvb_Mi*Wm*Ox%nzaiW6(*Mwo*j<oz4ouwSm+zLF%kJ;k
zy~Y6?MvoW12Kyf*|NbasjKoL}(#D(mSpeKd0Lg){>+(RjeIPzE&e84yn=|j3&aT#w
zj~Mmyj-`Jw+z-=#XI>UqFNp8{SOXj{Aoii#QdY|s&O|XwU}Ci8mz%LDVj#6L`H*(x
z?f*jH`~ng~$H`MG38o)(Js|#3Nf&Uw0kJ{2_H_rm{x9TZ-znW8u~Ys34~+VM`f<$l
ze|`#G;QX`oBFFYGnN?tauhxQ;ry#RI_-<4jSRBMA22WdoNRy`gyW!#Y=?CQef308s
zMEQHB*Abj?1arqE6GXodB(^W@98qS2)RKaiK6S<@SH83Mfy3?u2V{L2HuH}Ek+Qh+
z9MaduCQl8qZ2t-HG<N#WG&o!I6ZY{XMqNSc-T&RdVf$J{0wcVvE*}8bIXhHu@4EAJ
z2i!bdZgtou4v%B7nupDnL7h9mZ6h!rjmVl447M92+{T^<*Lz})E1V6IL&jH5yTH@n
z*%A#nJ8kMB^Bdf+z;QEq+H|aL4Jf+@R{vpn{Z2>E_hA1zy}M|k^gtV|4`$EpOVccq
zjPJwESZMnZ9KW)KE5Kz_jKf-RxU`z5f%!0f*yyjPIpBW$w)HC5er)PMVlhW<uKN3-
z39RPo%gL5+IX=MMFzxUcczp>{iwj>myaH|?OpUXJuw_!uPLweHWW86jcOztsyzk&N
zu$?gV#L&BgA>$)Ik7vO3?wd4u!<8z;Jr20c*{qeNbLwaV+&>qV%m9~TXPB9G)w3)B
zyT6j97j7;tbCAVr{hxs22qu;Q9ruOtiJ?Q+=fUfL#>0@YCDv&Zc81=Q1iJ}j#=`r2
za5jj4apAum;+@;z@dOgXhK(QfSzh`46kKm@nJTl(@E8Yp{Qp*nDN5R_RWAbjyMD!Y
zu(&&(`u}t=WUm0ozRw(x@hlJ<9aq}&gUv!0Cxw6PJ7hd&-v-q2|Fy>jQNjacrqi6C
z@O>2TBlW=k0g1zK{y+Ae$0cu|xM@z~%}oby8E%(2&Iry^w}l~T6QpNC*lDmCcMdj!
z%aT{o^9(`aFf2Li9o#;c7>s_+Ee6(WU(>bohKesdeqr+a`!-;>A0{_tnE+TXh~JzK
zIsX{MwpIRZ$*ZW1QrGwU|1mH9H65cJ!YC#OHe)UWWIqQ;KRQ<2#YHgvpz8tgt@(Vx
zVFhA?u$#_ec>S*{@yk3V%4}!+e^r$B8c1F4vmF@yf7{UY;P{yERAie;>>_aa##1H>
zb~{K92>-u|R=$G-@!&k+KX89J@8W@*b*OtjSbS-&7>fIG8O2@t3alQNJV^Yw*#)rs
zL2MA-zfqr{9*{axu^)#pIBbyhZMJ}%5BrS60YeU^j(JgpMeZF$IRX=>Cf#r14EOVF
z&82Ym#~rBmM8WLTx%An}N`f1l{%3x(1DC-tIS~En2IO4UTOZ?gt*tPFn+KA^hMVhA
z&wjQ1_-;q;X597vf?XP5yFq5@#e~D_r?q#K;b{dVhm7^ol;H6!8=(uX!$I=DM2t+?
z8^ghJ@(m#vegUa@=>ciuoK)Mpv&s(A_E`JC&|;!>FWelEI{6nRmMzwZ^JlLef!s-!
zyyQQ){JFksIoSN@kA%<vRaBb=54UeMkUQl-cHqME<RJT{Y%Tt5cbu*O*DEah3TN6v
zHtXbjX?U2z#It*^STuS=%5RuhE8|{G=WxiKZmvd`Q2Y;5gO9%b%o6Oj({k-#8l*OQ
zrSOJ`8>szRkT^PSF)`7JT&o7J|7BGtfc;i}p?;?hM-$lnE$XYmafz;Xls_5*qaiRF
z0;3@?3PwX<Gz4HFz-{#trT@R>ZXvuq2~$Iee&DqL-apJaaUa|U)rv^lsV{T~+$I6(
zU+@}o7pa5O-W4EzPR!Mv@#47W-a%?WSUG{$>O8{~aDOMqa>}k%AKk$H|0f{=DDLd5
zGywAjLLv9X6fW+;IsUJ^vj^U8yE*wY+>Idnka5#x$eohNVx+SFRc`~k%R#{yWk0zL
z%R3BvXZgy(`@sP!n0HI)l(p^T2vdi<9pn~`Zq_YJ@)>s??D-51qcc2+`(kq<CxPuY
z*#qg1lwFAgkE4L}fw1@V7Px&NJ~D3N{{)W53x?Xe9yi{F$1hCHo#0rU>;IP4goE{h
z)Y@NY0=ogk{<1jIYIkQ7O8-A#hqR^lg<=#jklMXGkUNAsW<cgyL1O55QF$r64UH~`
zkAFNB_0F5?OgG{E|ArF1os0h{>~!64g%V#N^X6R-z}Wx)Ynn2+y?y;_?)G&H_JY|8
zS)af(NDl~K7MTtY0}!7Op3yNAo;G&*CByYr--opEi;R*`(f}^=x>s`Ij3byHKgQkg
zxB`h?l{rUNn1b};!=Eg}z<m{DHH!s(z+xdOJ}BXZtmgFNTNW=bBlb!o%TbFh?f3)k
z=d=eh@UWlIgjl<ZY}eVdwbpU9kTse|7cpbR#m$wM!2Xgmk=xC(n{XMYX_y9&V~{;u
z&GHt!=5i?IGDsYT!&&#i-Dq`|7rs7C^SBs_IWYAgTKVQZco-k_z6v*6ad+4Dl^0XN
zX~4r!0o*?asYk{?gCXlzqdgpUMax0fEkDV4ZE=Hb4~qH{1}>|WH>JVl5J>;Z8UMiL
zjmdn-*iU)?L2y`^yxt5J2dTk@8_s0I{djLu5l(-B^iR>9xXMr59;~MOp5b=gWv}7!
z`TwFQ+&v(*xbPQ^GjRK0YMOp&SULnd!^0OOemQ%#*3Q_+V7r2(e}Mf15+@eVRnh|6
zbKe|shQqqP9UETFUko-ApE=<snL5`mZGrdyWt9Ga(;4?#(Orx_%HX^{T>^D}7@v8g
za-$(I8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*
zAut*O!!rcRIHXYKaNJhOg6Ahl@$Hlai{SIaTNSy$^BAua94vaSvw-I$KxW-CDFw4l
z?&W~jQ0qB0?TYRuyslU0rnl9n5_Yf~x9Dx&brHJ%@3odBc+LuBHV7M=3WDcpOe7t_
z_OD_v!#V%IN_!T3-e#YIKiFL$GtjYhFe}(Bba7JnI}+xD%~<!_24z0E=CT8Zy?wj(
z!sp|Kgdp?p$ad+(7VT`k!2tFrO#Fq^xh=E$H|(_HDguX*L&86>eIWV7hr(dBY&=@v
zxj!YLzbNx=$5>=B=G{U1KzMHv<jnd|J?`CyCo99}oIr9QY&vZLhWkO{Ags*<IfDtr
zHuvNPy8*=3C}OsDTF4DgD<H9{27Fc-lY~&jK;lnJqQUXo-`D`3%L9ocV`T{ixL=XQ
z39+ZVI|4QvrY>-Tz<L-v-@tO$m*aVs@&8p(;tM3--hUKh{y#c)HQ0Tv-v7482U>vL
zQ}Hbr#r$pAh&9?Ea|q$fD!%Zr<(%3855IF&OknZ5HnLzl37K;%rU$GZpSmsG4PZCJ
z)L01F5oI<^J+ZWx$pdg2L{_h3bQU}h%@c<E%#mm<FUvV)khx7{z0_oTZnK1!dByW)
z!`V+2vF|%9R@iMFeMSo$wxz4r!pjnvo967>4fbo6>$2T1_UgdRgUO@Q?><GN#KEha
zI!l{Q3vk$=s|E4zK2!nQ4Pu`=!U0bol|8pm%mK;6@UA24;b~Gj$q?@5KO2M0xfyxE
zd<_N(ta0C@3pq#3-6(sPdNC*1jtqZg%c#jxDC#Ag6s%eIA@;65H0J<^gHP5QaM{FY
znF!`PzuQf4PJYv*I(WF5nX<tB2=fmzy^{am>U8ZDV0W>)r&-l0>B031oNEWqkt3_e
z#*W((4-ZqAd@RdeOMACS6ua|Pl(hcs{sFctm!$$s!_*T)udi7Kc3Z2{X?WP()4#IO
zPR|X+9&BdC7jM(~-d+wb!-LP<0;iF58UI}pvDskvw;A%m;~AS7qhg~WFd71*Aut*O
zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0wXU37J2ocjP1TS8$|N>!K2yt;p505S0U%U
zzDiKo8R>2g9_s*w>ct~>!Q&mpcc*~&|7~^Ix|7Wdb^HjV4jsQeG}Th|;}r0iheD6|
zE~j-B;Pd|?r=s4?d$uwK>~8;wUf}Vb?pPL_=l{;qI}9F=0@>9yO$hEr5FZ(PwjBhI
zS0Rg$%GTb?2Oc95<@$|so?KXxJ%+sk<?QhDcn-;6-i<6__h|?7w;r%RVeW9S-n+?o
zcli#R=aJwsHUIrT;PQgIA?J3xf3yOJN%;DSC~;l!z5_nr05cCp-zwn&yL;-VH9J2o
z+XRnan0(ce1Pu4X<RlOGf$anFoy?Qq?rCtJVOq4M0mUr6TW*#t@2pV7K>8x3A@_DV
z2(W_F4M+?f8+sjvn~g4qkFS5|G<dudrY6c{CVc<j>;lG}7td`mkN>ZV5?>&-9M;|7
zxhoJGhEF>|&ib9pGi}>WlXKwovQ9-C#a-!g+ri^>Fg=9mOZ{izeqJhxI2SZbd<9s1
zhMFXb`*E3f`EoeUID+X>W<%`N1c}`?VFTL%Vp9`WEVlrsPZTr1S%Smm^Rqf|xS_~F
zIWs!C%<U&}67&aDIa!nu&n<X({@k(w&fdNTvX>d%P*cv+R&&-u%8m7#Sm0#|Ox;(O
zKVW}#Xcg|7eA}8}c%I(@IlBg?|JTZ1i}Itm@Bh_*hBzO({I481pMvyGx^oF7-GRhl
z`16`6@G$ld34_Olx3~26iZktCxqf-<aW8Of2iRQ6%?ozEE%*uc&xNdW7E`aGmcdt)
zm@GcOnFY`LygM#~&sjfQ$^#CMSNX5NVU?Ir2QJTHZo@{07bC{UO>PK+?Z>7LBxa~9
zxQahc39LqC`YiKFRfw~LR205}?FOmCht;K4g8hZ8#@6JQ#dL`pc=&?kHv2?qKCg$I
zr991$0YeW+4L<A}0XYZ%{lh)rFbAo9zj*QnmJ|lC`$2qMSf6vAc4Vg^ybQmT$qWw9
zStq9Kyjc!8$8Orw%_O@Cmwlt+qaiRF0;3@?8UmvsFd72GGX(a(hupyh^Ih&YFM{n5
zm|7Tp>AwJc{Z^K_2v~n|sO$D)*?Zw_5$WmH;5K~d#|Uu$|7MxRPWQXt!R;5AIoRmX
z9ZM`H+fD|z7w?^XuoHIve~qOeO1q5bb2*sLd!z$wKfd#Sr(Q?gt&`!-0CpEPyJaUC
zfz84uN30mrO&hSgmRW0{oF{8|8*#oh%q;m@19*RYvgKEdu&J0BX7MUI3*J8g>0@I)
zxcNk6-;Uz08{jZH@+b>zA4qO_1LQ6swzI#$ZQ!E|C&1f|Ao=o%TyXn9d}J)L$^h*5
zv=o_LYMyx*{qw&a_i%<=R1RYO+sZqTvqL~;n@w)BL_hyu<Akf_t;y#x+_5*M7u-I+
zxceWN2I)t~KaI|!gax`hF23v{)VrfLd1u1=|FSc~cg8A4?ZkHfPYTm6jQ)S?PgQV#
zC*yzjHh)vdnZ+gIc_?m95C27yJAZzThllMy-ZprAs%+2!=O@k5>lkSRn_rt&^}@};
zCJz!b$e#j^XAm2NcWNQd#scxFg*i8Hql5{_-0SJw;B@(TdozX{NFIcfBIlVO3Q-~G
z50F}N@dBA8;BW=$f6!nAXHQT+jy2vk-3YS^xDGj6xlEP^L(R)}7jT*go!+vm-e4o#
zJdpX=@Z<B$@Hhs^Uvm+(l=vM4J~J1@N5?-c5oL$OZ^(K)kQ&{~xYsMMn1~o-G5vx#
zpXy=i4D-P49N;uiIuqe{ka;kiDjN!x|5QGAXP)RJu)9KE?XxJI^a{oA{f@FbW*tPF
zpX{)5EjWHJe-QzvSN&s<JH-}TrozV`V0NI>bB_Lpn|-<cDcF8=H86gP?!;A!8IXId
zS_G$C2F#uZ5A(=OHk@e-rXNJ_kU^|l1o1)Ga^7Q$-vW^J+aNv&KQ&&Y$$ha6?4Ffd
z)rc}1qz{JqceH`smc4H&*bI>P_o&znEJ<o8X#gaSj(0cu>rAdUh1a#dTOfC8eoN@z
zd9@C*hV}Z#M{sk|^^fvLLtr!nMnhmU1V%$(Gz3ON02BhjDYL<CG!PqvSM{Uz2S8$k
z@LMT8c$-90Y7)3jRGB}=qWgmwxUB=y@3;oC=Pnl7|CfIgy+bm!1gsV$j|=mK23bBe
z+z;-z{{7gtbJxdX;Q9YhYt%b4id}8N?Wg4#kUo1vj~mYU|D610aC;PF=YniVp9;iA
z#~KpY@Bc<uhmYT=tq69P|HS1e^U0kn;^AQf)Bmjk(f@bYuovF`g~?B`@3t^Ffw-#-
zB>uy-cQcQ$#g3q_kTnvo=QDuq1IgW41Gy7(f#*zc9QXYc28Sa^4up@hn(BjSY<S<N
z9pJc$;Q7AO*TrETx|+jHh&?vwasmyIb8tc8zFQ!3ejs+tp@$Yv*Uvy1O9;KqYWa8F
zd-%EskU9|lptuIyhjJ+11a=2V3>^o*G{h-~t`?cE;f=VT2qd1WylyRs2H~(<kvn`B
z9Ki^Ckle2?nD_sxSKk4<v*AbAw%3x7`=VS0UV+0Kqz8mAOmoACYmhiDeEjEmxcZ6e
z>hSn{<thVq*V2L}c>2X<21wlSLJQm+5FZ;}-I$5eUs&U?pD44jnT1bmWn>pPoRHNR
zp5F)VOD5ex^i7fFLF~1jlgz^13EvY2QcEs&I~E2HrzWogxc@nwEWqK7Ztm6MQ<n3V
zrGWKBcXYwk!_>Vz#0K_jE`!=Gerr*%I+z?bnk^ai-rf}r8!S#dyM*)pUk$@fxS8I!
z`r+{x*ZLFAMz_cD%Tah<$#@J|TLV&)*!j-v?qtXvQy=8Z;pT$mV7P6n2iROM_bEHm
zC)@#>vnK1D#RIlmDE)YwC+BzSG$GF9K2QnS-`RRS0vtyBnIUVmT&>@M$0}jwW1|Ze
ziNeGFixKYqp&-4W{=dNW8(=%uC&gHHEDwhJBQ&EG#f>2K*zo^Nf8lyy@|!ddm`|t?
zg6BVwc(tXH=JM@Jz;1K&oDU8QkT|h;i;^7J9zX8c;4lU0TbCZYp^{}a*h~-~7p{_-
zqrLT_Jb3&Zq$c^{ad0|IY&YAvfB6is{zO4exVa!Tgm8}B2b8jEqA0{~g!CcHA2^~8
zFVi_+Le_-rSbTHGp*F-grpdtu@EG1beogTGzwhTV?edYb0J|UAEOd6#!}XRiOH;w=
zWvj5(uKJJ16hUfEepLsDBZv*c{I`FD{e8jP0!*J-{2XWfKPB}VIIV#6`Q3%M3&cjp
z8N1(-9KYyh7=6?SyJynANhst0=T8@c!vk4;$(BZVJ*e!3xE~UxMkca<=jtR#I|nAV
zc_ZVNFKQEah)z`or*qLN#Cowic^=@jEBSc^IF3)V2!g{Arq^=%G>ma~n4H&w31D~E
z2)pde5&DOrZW6~z4EMwIe#|@r54X8NR$w=P<h(pvE&uG$ho=>=SU#WSJ<ruBVjyw8
z)_q_zcyoo|aRm}b#*;1jz-~kqBZj>{=silhJ7hKuKK`G{=d$yax$ahM_y2bET49X;
z>-j<M0Gr_Vb3@xI$lcS15C5aM+27|4N$zA}Hid_+w0k@}{EkSfg2j7hVZWCi-R}jZ
z)4=M{#gX~T7NXw0d+|F5O1?r?O9{KAg9#j;$Y!{&Hv^~3?d$Y0<Y4OL*YQ~#pNF{b
z9wtsrdX}m`+|MQ2GvRFOcw?;Y^6U|}=GRFFhi%6Hrzmj^au=`0IdJ+?3RtqMyacnI
z4^o4SiyKha|IO(<VG)mK{hz~a7qHzRGZK0;;OWC668Ac}Yv(3|-2~F(;=B@ESAf`2
z1&wCki)+CCw2zzxra|J!SpNc~J<qf_Xy-zMJK*q1=U!{^`lBq0`T~Cmt2teLV0n;!
z?_U$a@vGCb2W<Bm)_Y)kyA(H~ltUo3=(sQsaTjqIyC>X_=;~nnO|QR$^H>r8T#Kns
z4B>8Y3iLv8BTPLyy>v2WJrFIzZuu@(0>y6EpuL(84LiYNUk$f_?L;>VAK%Sj0oZM+
zD_g)}0a6>ia`JldmnmQ~L3~{Jhw26GOFd@rK3v7dhv2klxV>Vh(3wl%G+`Pt1#CVp
zGtkASFA{*e10?7AiS+pc36;(8{=b+Yr2egXy~@%ftpMyMkh#*66Txim*OK5eG~?5v
zoxiLIpJO6kdE4^$;k)3v?Z{o}T^FJCKif;xJE0#;WCEX`z|sw=Z#rXJv_SS^!~cYu
z;eI|Owgl`hZ0gRYkXOGToAq}87I3^~{alRF|L5Jk9>ZP<hbDMEyJ@BfMm!olX0dcy
z5Ct|5W{26**e%zZ>vwk8XoAziI<Gl!`M5vLVD>sz5pW(z(&9(Sce~E4hua4;4@Uni
zehE(7ioLUTS?`((k6)O43@_v?8W<m$wm3BltQRCU;Vu{04Is7&|6NOV!zh%x{^_4-
zmN$+|pooFgR#*K5yV-=x5^feq92v{H#=y-+7AM5M;kF#b%r0Xoc>VwT&xD<IHEXf1
z|34I$fpPxN4MrbuJWag+W_x7<WFEKqG-O^5<_<rr%V0GyK0$gNEBGuAkh+V@ZNTXo
z#C}`h2~IPo^IyW#FGw60&XdL6MuX`wI_QlOhN0QwM41iKPb~e|WDYn!k=1{j{1e>&
z@0kd>Lm62P#;$R3G<UlA2JCN`7`5pgs}SqOb6z9*KPRJRVYPRD<{K*;lX+k{!^_+l
z;idWD2e`hAc`mnmo(k^!t6~0lau0P3#xnbzg(RN-f6mc1u-zd23Ug<{<L|lZ0XQ2Z
zhm1Qy5as2ys0DC02mQHWR>_zM=C_-2Voj&H|M!6Ni{OSwJ0+gfg6#-DIBomRsmD;%
z|DJQz{2|v;xZ9R&fy}+fC~pUsO)ipq!RE}p(g-fkk==*Res{wGZccE&9oSBEH8B1z
zDULOd<MP37`DE*6^?X4nm<^KuBDx#wHxL^iPGWumw-2U{CoSGGCJYh2An}=Z+?LD<
zDF&OL`#u)0yFlvE@#b<vusx~b%5c56KcCuYe;9Iq5xRP0{;I(F+FNeu!OQT16D(lA
zb#avKe7c4g9JX%nA!EJB`bOEKAut*OqaiRF0;3@?8UmvsfE)sob{c})sxUDr6~tax
zm>412et9>%|90iGA-El)^KznP@J>x|I|-z}?hNW#zwuKwc5btR%oTv-abY(TU(2Y2
z>%ncFm7J@0ra}AvJr&bY`V(@U8sPTSsUwi_hqRt!ILH5kiZ+1F1KG*=<T2chAU-lq
z3GoK|1zC(#_N7m0U^6-uG*SBhyNmW?*!w_8AKo8lS;qrzGb7vA_m$1O_PH+DpD^(q
zKh|und)T_;yr3J{Pt6MJ;Nh|%TNtcX`l~v)z5V|W>i+-31)gyGVCKQ-V;dlI>+!~d
zyWXii$57X`sTkajg{eoT_xa9-yCLAN5uCkQ_MYXJLS1-T0qNOmf8E0DJ!1X{BnHCs
zbos#gK4?Jt5FjyhJdb%EIBe0y3Gw&&O+(Qq=9LZa|F^KM+L>oL1#ADmkP&nIf87Mg
zUA(1XtV{LJaf8jR{kaR=E{D0p_0Sd+K1>`JEz60>E9{Nd;5Y=SRTV4&rx{;~Jop$O
zNE{blu-X=99KrOg%=ZJk8OGOfo`I3>U~-huxf{2llrtc+{KJjG{r@xD%)oIC5<|!0
zf)!>a_51|=fv%4f{=H?c;BW=$nXIM{XTQjT%<+T7k?~?Pb*l|eZ-LddJ!Qj)+s;*&
z!Q<W!AEfRoN;C$mL)MSZmeTG*iG!nF`^<H%f^p9O-!b5YhsXW7hv0gHrRKxg=yqH)
z5QfW5bleS(WBC)+=6scU;P795#St!!tiLPdAz1$Dcj2A9BG<s??2D?hIBuwn;<xPy
zot7V$%!P|_y@sqU`?v<O-b!2A46Jvnq7Jw`Lbd~)oiXb^JX|lBe+BD9SA)z?Du(#4
zMLE{;T1O*XuNB91jC6;r55^Wc+6WI*nApTw=PjgmBIZ*;;$F%T8hK`r@xz7h=Mr=m
zNDnezkUj$(KC>S}_Q`?dHc#HMq4^l<I2=eE9hc+<YVY-34IhhDuU`oE+f^fuoqtzB
z&iZI}FUJ~(=w^-bM?+vV1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!n
zMnhmU1V%$(Gz3ONU^E1VZwQ=X{Eadf(--j(Jl8{tkI$Z24Bxl5WBwoToP=oqhV9Lh
zJi&7vAhVWt&jzzAq38b_)c@L^#h**iu8YqUEd}K>!1JAtUU=;cFck&w|F3@8iZb`|
zLHj0{?|<Vz*vuJt&i`LnnF*gGv1?lkb{EKv=s3q%9&8r6I4S(Jr()pwTj!ss=l|Dt
zrDNDz!N>*=$I^)pQQ`$;=eoR8CTDjsf&B^MgK*-lGn<#|sP9NVCJLVaZ!2$wo4YQo
z80>x@cTsShWE|xNha*TI2&ZpF+&2K?BV#8O$X%UB)&*Kr-F}ZT|Nlem9o!ydeIT|E
zH$*RpEf#^;Yyb7fO^Y*OhA3u5$uBa$5Hc6z9D;@{b+FqN{SoJ6gUmz5g3f>8^L)tS
zgxIXSr6^(BWh@2X|9_pOYKIEvOL+P~Hmi~W^ZY+1Q$Mgf(<8XIS-RZ=vv15|LUET0
zA3I8%A)ASfo#ECE58EYt=iz#9L__v{zghAZtNYN+nz<CQ4;oz##{aVN061;H_=l%X
zL|LB!6Q`8!{Mm$J56C=Qc}?*7@bYq>!1LuGF?77ZO3Un6cQ`ye(B-MY7kUC&M-4Ko
z>SX{tY){Twg%Xb-c@WOOaM5z=(Pv<FoV`6LX$d61s#X!~*YH<-yWX}WU^N4z4u(He
z)WPEzB*t;%nngR$3XD7fk^^DJv_`P`Ah!RVZn$2?_C?_I2NFlemLK+l-3t;w!SV>M
z?uq=7ZHrBWz-G8_d5_htdL4z}_{qE>w<Dyg6Ku}Pg>4p-zM<yL!%tmzGQ75g$JLW1
zv%vaHuPTAvd$G<GZ12+>nD>c*+=YxMT6n|F`K5yYOxh{Bf~yjAA!iJ3Kf!9zP^S&o
zJF9vL*uBVRV`B$~Anx3N$v+F3XJ%D`TF>mrx}ot%8#3>{FH9L6ZZP%4(9=~^z+vpY
z+ZL{O$;8R)x9<4>=i@TxhiZWKZl0I$`G1~I=3u|og&6OYG=`KJzh^E7n~%#3bn(|!
z@hIu{d{93)j?mTN<3A2zg1f0J!4|Avzn*_*P9owChpad+@cIh#V_hpjW~pj+?mRlZ
z10GKxF>H8ZVvOYvx7pyjjr~*R&e_82!1ezTQPjIG#Q%qa>)9W(A#1|$o&S>`fH>pC
zJN^kyH$<Gg3HA#%cM&V*kP6wOcG$88W&gkWV`*@BAe&PlC<d<^<NtAC*nQ_vgL$7R
zKT0}z&+NKo<?Q$!Tkphy<FR+U0XQ5$=JtGV2dmw{=m*Y!rqu%Aa0JPL@G<ed@b(6X
zkBndOIDqx0JbJhDeS9%Soe&Ve9K-!E{h^<?z}-HP1+gZnB-_+dTX-GZk0AB8nipBj
zyp4J$07(4XkyT(bzD;!jy8|SKj!XCv`5aviAHQ$T0<hUIHOTA#+#cQAemZc=c5L_m
zeBNn^vHy=fPz{{l_|~^=6Z@ABcB>8#H<*T*|Et{z9R4sqAv$lKFFb77Z~MaIb5`+7
za9XX?)dAZ{$ei+)cDOmX)IQQkKymABY0}mz;xZRqyo~1&Mi?-q#)IpJ|Lh(p@qw&2
zmgl^AvJvF29b|E8visv|z~KrqbBzLG{om|Kkh5Aq;>ftl)Zc1J<rT1=gGq<MWiYaQ
zQ5+<`*9*+s^>)i^f^qdM@d#=hl<c<n^91+#KPzAG!Q*w`hRJZfH@hdo*~snzu~QFU
zgu5e^eFHo$e6sd$oAj>?9RA0qDZtf$^uVz2;mKh6l?VKH9^U>Dtfnc&-Q4h)J(vws
zm;6k|@=DPqaJYcPn)S`V>pKr#nFh8`pmixYto-xSQOY5ZT68?CmKSbzeIDu=Jm_ja
z>k6*YT>b&<Mu~GvEI2(7ZHE*KJ+R&AdeHe5uQ2!jJ2W4(VBU?o_b|^~NmJ!o1UQ^!
zLrC8rt|GP-oaS2|BKmQc`j2S2xU<9aIl9|m{Fb|Z+TMH5!s}X%03C2RN<DJiSv3)|
zM%HiP8L-=6dPeEd5Eu=C(GVC7fzc2c4S~@RKo0?z942tT5hlK?I1SuBf$@o<xwmP;
z+a&#-kg-l%hc!DpPAJ0LsnHBoVD@RTUEuzIhgHJP>PE~n7t!sTab~ThNWgJ$J11t<
zmz^9J?t}aP;=3DA+BrKLAZJ8%euCVO+Zo$}bN^qCF5><lQ31p`6X<peZ(IRxQ=*HL
z!vBA8I=EfeV7~&T|Nq_;G9H3#*52tR@V@cZf|D3_Z#>wz<JsgVDE@Zs_SjOyHg(5~
zj~(E6T=x{R=MQA=<86@pKqR<VfZM<)6R)6*|LZ-`f!haCkBt30dcpSGe6n!ooOSOo
z`v2j}kAveASr3fe849@v2gFbM37P)@u@4;-vSM6~y8o{zLfmp^o-2m=jM0$$Fu&E*
zgZm61{pdLP1@`_Kx;k|JtizAN;Q<q0_o@@V|Bq=^!w$oqo*mfk|9QC!v;W@~FAO$o
zy7{qfSFT%w!)()64wSfK*|7pF4zrICU8hzF4_ijZ25|ZT>3MSW3%GCf>})o?Zv+y@
zh3Cu(gqwp)&4i7R`?6s2pD&yz%50c=V(Gi`m}mU$VK@LTYwA{91&1Lvvu1u5x7hbT
zhM+&N=_OVyFJTclTtVtTt%mHe1hG3}3bBSoh1D&qqT<(JJ*py@{r@ld*<k;uE;+yJ
zVdPt^W`N9x;dcwxqoj{LUp;o*NSFuiJHyl<)4ID`;NiiyQVO0vv@&<2n1ifG-Y^^-
z)*vyj>J#uhHS4JUcBP~<VE4v+e~i_wD^$h6=HAU;xbyWH$o;hwmo2l{x1$S1J!i=(
z%Vgn3aJYc%`Ob0!><7P-65#TC(|5?ZJU?C6f%zacxbTv`IdF5Xy}b>#AD3Ft`2Qh&
z$Q?yIuBw)@9PV(vm!ml0{>G&rB)&_!3?8N+J_t{~A!}Ka0I63%d=Q?@7oqv>?jo>z
zCd!b$|L@&aez4nqto#ml*PWu+4bfFP@OT8-gALC)%BYh#F$7+Q=PsBEj>ma(X6*Fa
zrU7>UoMq8qw_($dO-w!TG29&>d87L8C}kH&j1Z37cM4v9=OnHL>%VyPj#*>)L$F^#
z`m?Kc!r36c<?5iF%v!fm(ho=+9e)yCVJWlPA6&QP^uF79`GXty{NF3W0x15BR+j{a
zU#wgx*#2ld=l}Pg-2e_ZkX@n)cHnRWvC*+ffjo*G=<>wypIwHWOWc1_of}#G)&yY;
zd+%AUfRD%T(m|X<4%652^QgJ3s1}Nw46R}}&)0akW7a-Xa2Rd(L!4LnUiThY?MuBd
zuv}7RogvID5dF$74QKstc^Wc4^>4ECu6IQu@b(=@9|)&9J7KsVBo4wFTe`t<0Ag?B
z)`7FLCLFTdr5X%RD<FBP&lfEwFKfndM@^tPSReN-J(P3<(gVZWEm6-+g^3fQJD>jr
zn+;RfWjqUB{|j7~+EG*2Xo2ng-$F)6+ZCn{L_a#K3Qof*4{vW<ngcnjfZbp<xI6{P
zgRtafMX(wWn;8736|ukjy$E7HC+|4q>`<A)Y;d_s$gCNxnmFSKruQHh<V-dgKWM%Y
zNp6MdBSg>1WI+iNkovuBN5KAIO7umE50D%TZ(VTO{F9F}L4Uy1lSBWk<AbNM=kEgH
zY_?OS@Nxp#+)V+qttS5#0lVq+@*oT~2euc0{d%s;Vb}S^@^JHz%|K?`T`ULZQ<zx$
z#jWN=&v4)WcZ25?+|0gZkaIdgdbdu%eTE#{{AF-Eq>K>fpDqZGGQYWD8Q2dNHmzV9
zWEL`3h};R*`~J|eolk?hz;U(xcDlu?q!5&}V$-i>>AtBD?zYtLw%~fc$pUiTz&jPl
zStuVHwZY{PvK{E`2NHka=7>#(oCS`q2F8y$a&whZ8st0}Q$B0Uws()<?t1!3k)T~5
za|Dxq!0iL^L3mdEV+*^Bka=YgAB0U}BQ(BEod^ztyGAVFZ~=)Ei)YT<0Jg`gQwVGZ
zNMAzT-3@%vdIZhNT{}<v2ZtrR{x7f*1E;fFQ64)_r@erfyVn|SF39XrJQ@O{Aut*O
zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?@<O2f
z?pyF29L(qcpXq?-5ny~`=(N+C>tX7`c0kr`^*mZ){@A?%ZkD{8EO@@mi|H_U9(-@C
z%Fee(zr*KgVEWN%ty!KHsn31D^GFg$4R>;Ugv|d-iX+yUfb_&Y=>^Ywu&+D{HZum#
z{eOJN@5ASWeowy!cO%FgWNc8G12z*`j8yjRH$h<Y(a-;{zIq)jj%?nu+CupJ|B7Xh
zJ-Nv88GIHy>MtPn`-0?tr`BywSeCKl_05CeFtV75n3K{?^asx=>3i^k=ZVs9<e<#|
zbN2XQ%)5ik1mQ0=i@|<xj9t8Q;npIQIeU=&%VNZxsUSWw&ajIB+XoVpo-zmQ1`vC}
z<e8Sb7RylV+<SGR#p-KH80Nb^ybLzOZ22B=e1i0&<EQnId&$wo3Gt^e$fKBfIQTw%
z|No?2oIB>#AF#l-|4-p~H~8*KWb>KU=77y=SirE2rx<c(fwvqh$~@}ax>$G^AnV1&
zmU)TTtL3o-vYr;C_I$Q9IL*vGk&a?MNDdpWdiW2l9-ACYOjZ(g?_G(e3)nv}d1}$?
zI78ui4rb=pC@FB+<FHc_9M>>$Wcr|?q($4NJp}!MtcO%~`cgz1`!8t+x9jUW#5z`F
zGdX-GS=9)#g3GgQEo~TaQDdY6_Rj*xnL7o=R5ARGY!-;U?Zzc=UIVc~c;1C#bD7_`
z=l?a$+Tm_=ocar{_r=fKa5l&sWXvCj*w_8<oh&?#yIfPZm2IC6<}duC2d0trf!Gqq
zbiwJDzd?8BZXPwTe`YYN?x^!j1N#%C2825UK3UFv3ps}$#4jtT0jsStIRd8F9Zdn7
ze?99Vd=49=4jX1Jh=KdDcC#){f5G%l<$|mSchjA{qjlP1ust9-?+>?7+z1lGhL<L@
z!tH~}zZc$N@xN;W*zYj0I-?sJ5lm~rVygd1-z)o2^A6Z;vc>gq`|b+stuU&>ydM*0
zCyf4?GFSVR>mm4BtWvgB;Iwyo`}Q3>Ga>g1N-=CA7*;UzM(NQI7!85Z5Eu=C(GVC7
zfsq*k#vyLtwi(R#+6&de>*!&8V(6%B6?mIuS^?zVc7gMHJ0C|Hf!oL+GhX*s!P`3^
zzFc6)&a=*@C}JRSbZqVL!BS5UvbW7OMrs#q|6jWPNpKqsran6cvJRHh6tZT2#Wcr-
zFgbL(+TRk~1_O!T$`ypW5yVHv^_wB>USu&+*<Ts=fy48xy<{7*-t@QZ820Y{st4~I
zSFG!SyAh^Ob$`0~mQW#-FiU^Sx7qur{EqBkcW^xRajAms1L^U24Y}uT^ZeJ~u>r?-
zK?Wdk7(QYh54R5{2BRnKo&pZv>4#Z&34HN}$1hCY&Jy?8GS5$Y!P}rO9TDrFo6f{r
zZepoL>FeLM+hOr;LKB8N3WY4e_J4M1Lh&!meh{t3&jwZlVv~aT%^6YjIeET;um7*r
zs<xP%eR&7A{=cB<)0yb@6u+Ja?qA3;Jll3ELkJvavQtpcbGz_vD?B{V^<(3=Y($*z
z(qRng*MQVLn0Xi+7U!Nzg5w$_h7Xt2L)NI{Q=_B_*&_l|GxLuD*bW$<TC`IiH@L5Y
zY+g*oGVtCb>)sNa;q!Q%wwax+GDcoOHiuGn%yWdF!z`1*aR@Rq*QO06oq)tZxa#x*
zD~7i_z~Qvr8nKrSBnQIGwU9E^Iy7%r_k13(I*>Rnyw42v>=%VjQH#}<!QinGY<jLQ
zMeJF=c|IJT-)}IZo>haa_g82-+%Ntax4~%^B)>Ucb(=FYWUMb}h7(q|KD-1u!)qq@
zzn%IMAp3qSb9gMy3nI?21DSPPaKX+anv8HWI{OR3aV}%;08X#!Qtn_k_$Ld3`5-;G
z@Er}vS(hOB#e9A6I05m|v7zq7RnPD40o%3TdjEDu*EqNvjyN%b#nJVk^NT0Gf$M?E
zS-!q%u`GH$JbXdo%@4yhYCT%O?wO*D{k%kwI$ZdvyBOG>x^Iwr5~Oa^%E=piGS(1u
zXR0Ej&h3Ay@UfGFCuf1<@p<6Iol=^|!RGc#ZpN9HKz5Dd(GVC7fzc2c4S~@R7!85Z
z5Eu=C(GVC7fzc2c4S`WG8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF
z0;3@?8Un*F1ngxVqpZb9cp`$b-hmh&o6nmFUmxbMD;d1TW9^)loe%mv;Olno3YWv#
zATxFBYj<9Vokwu3!ZgLlmghEK1+QP@<uKaiwC)G^{Qt{~QP2OKn+LfEOM?$`C)1h5
z={g{Lu;JzX|KV#bs>S6{+=Z+@r^5v77i2L~*<#A~z~LFj70iRIx6L{Y!(Nvr2KYKS
z%_o@q|1xKW?({UN1g|}TnYH-s^v%78o^D^V?HV|Y4x}H0yIDc+A(*}2Y7_XL)wU^j
z;cG`g`VT%+hua6@BjX2bkhMAqm)UmQIr$Ak-7~%a819GZ_rA^pwim?D`u!iyo+7=`
zk}oX-#Vohodo3iZyinEzg7n>d3%P&L*^nK)t_389j&CfNLkSCXd0c!CSJZV&Oircn
z^?znjT02)LeF3fiLy0etS$}?|&Ii%RIH_PcI1R7;=D+F0Ye_KsP2*I!pFwK-xFF|+
zgV@Aiqf>}{bHf_4_6n@;o;Wznr`%~q$+KWNEX2jBtzh+7q~JoXL7%~Hh6$u?oQN~L
zVQR>sU6<B@*D4|FHREOji(PZiLU9AKnuaSj7C{TQ(8NtJdtRq_!OcIl9&*MqNN%1O
z>fHz+F%a%OsbC#{RT~_(D+HM_(x6MkRItAarCN5yHP3~c4bqDZM@+B;mk}_z1y{H%
zcR1oc|6hA6<SrGE`t3@P`$a%(4s*yE_#ieqek1A+cjxIE$oaJ(HGQ(}<_F%Ug464?
zdc>YPkQ@xhO@ORH{cy5%XUXNYV1K6{=&_t2h<e|k(Xov?RV=TA%>wD=o!190lN1kJ
z1*g~W_qyP)*sM%={U38ZVjW@G3lp$DkeS#pA7emQMD;nay-&aLT6Xmy(n~p$9?rCd
zY}Sf}ZD4;Pi*+twvCLpQgAzBd4yI}?HlGg`JCL##9;V3p@UcTe*um~mm@f<W)AhN+
z8*a?91)GUaFN0l%j?a5Wco{y^4sy17)t~=6&lkJ{yFcUBXK*;+HD^?EGz3ONU^E0q
zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(
zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!n
zMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmwg+R*M<={KG
zV2+xUTLC^t0LCYVmXN#zKflGzX)gHufnyxpI~o-b=TtoWudo|rmh+t)@Ez*<R>Hg9
z9bO7Q0|6w54QqOaT0Wn74t$P;`!}gwAJ<lZ&;Q|PgWL}X)5BD834DG+2A4FLc7G{?
z^ZXxQ0b%(0KOAo_;Ph{mkrLQ1FgwVhr?LJ5+qHIw4KK3(!@YYk><wAb4?i0xH6C(~
z5wf1dJCAqh3B3dR4<`OHOLmL6^41--pVooT*jRnQ3NG(faTLsEIi&zTbK=GYcCZ?l
z-WOVv;P%19VD$NRWw74f+l{-#?Zx2dJHg~b!fe3t3F9Nv^ZTzM!tDgYz1s69S{>be
z2cA|ydfo&bv79Ti8$}ExF8k;i*v*M<Tfj6(3>~-0%tY9OE{TV4{TKCYC4Yl?YhlXS
z!y|WfpV?=jYySk}{2y)Ez|}ByF#2%xGH@CWdVO@;{CNi8FpEFt2Nr{=sYsbck~?Qy
z_zO=PjZ>_xVdm@&nF$upXy!&SA104Z=Q8u*j3b!ZogchW-1h9@FYwtlFnMaxPtN?t
zh)11!^58q})Rp(*v{N%+uf@N*9bkV`%Pr{UEZnmV9-ccgec*QeWc7lxk@Z9;>sY5U
zZ34@2JvGKS1EzPQ6WFi&CE|Ch*zSd!jcf)oyGm;UN*s7hiZ_qdkpqVfvN{<1X2CbG
z-5~z-y4m0~2VyVLZ9*{zBnHFoXTN~m1ma)0C=PdXxTKD`ad0%4&$;nAm<Fjq#w#;j
z!Fo^Y{@l@M)C9Iq#d)Rqi9Ey|eIWI76#rV)*)_r4#!<`w)@N!H1CHPAH<`h5X$6GO
zuX5Figom5Qj!$4aL3UunDY_F^ZAqvGtFezYweneU2ClcyZw=0HL^kWTsV+uddG8@<
zxh0VwY%WZ#a`b0S*Tax7SMpqdA_r50k7m6X1P+5up1R<$0jce%vDg^!fe)U~L1Ngj
zouZiz%ce{4^MBsQRD<J-HEqJqD;kVo_b<303pO8{{!y{f5Eu=C(GVC7fzc2c4S~@R
z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7
zfl)9T0;3@?8UmvsFd71*Aut*OqaiRF0z)$dCeIB~zIk|q*6-gr3XZ!awO6-42j2}l
zEvH}O@L^ZgdtaWLU*7MlG`;eJmc`E53YvG#R6<%`D8A-iq?EAan&RGUHSMO0OSEpj
z?ogPsT2A(XnwYYY?m?3?hgg)S?6fuDy|EFz-g1u6eDjbrZ}ZnDS1E{v%PF?z>s#n=
zs#M#dGQj{8a`vJ#%+5?sHD9+YLB35YQ0<(spu*mjS5)pDbJ4EYJ3(WGsi&sdhL6hs
zAAU2FILl@#V9lVu(4x)!k=zwS+fNqSyBMyS2k7ln;+EZ@{A+=_Qo&7L3#%9Z4EW|B
zG2at=Sn0})Vxv~pChcUdCJl`}I~Afl9koFAUa1$Ca(c1FwDHhR&BoKym0w@KZvN>$
zo9fSXE!yqc{c6`Nx2rAA{G#$Dgi&L{Yd<qt(WM%uOXsP6Zn~n<$=jwH`+B>a>(l9K
z;cqMy&v3D5G468EkP_}z+Hg)*E3AK)`nJu|st5X}>l<cUsfhmDYmh6_XwfT|pzKg~
zMvhl|g))beykYi&uNoV4SAy5X^17{3vAn!pbxW|N!G*0mj9*DD(E7x;L*qxC7C0Pj
zOr~gq+#P*RO?ASDS7uxvtt})HaugZ&$*F?Gr##<a@_WN9Qy$ZP(}%OTlzrREE&Tqs
zs&3&CR{pht#iH={dL{8WVP@Yn+!aM9F`2w8a@Cel%hWh<X{KVbSCEnAtJlUnkLooS
zJaE_1EA3Xit8qrrX!biLxrbNHE;O`gE!Q}2E|AJ=l67aAZol6_jeeG7x%w&5Dl@x3
zg53{t!<Oy^;5`|?lDd?l_^g#~aao%gM>T4EHVe>vy{*h#k#)M7UU-s%<6LJ80mWdG
z*7@GbN!OmKf%M+}D`I{k($>gYN*f$Mp>@`Zp_le5wSS&u#&hAkmWr~d`TqQ)Iv33H
zRgF@)Rg|mWshC(ERoGLRs8G008oY*9MRBs?X_4Q`?v|g_+?D<-{r?#)rgg+oasS6<
z%HP(VSMJT9sB()>Sw+Njy?%&BoT{2V57_O_|G%j3O&3=G{A9YpnN?|uJmphW>s_MF
z!X9rmd*t-jpx#$fE9$kV(#P%os*ZJ%8V>uUOlES`DDuCRQ1gruQ@U93O1WQI(QKuC
zl+=lv8%>hixXi;k%oH2nE;ZL(9;?x8ZmsHdfy<DE>!f*e{wKw4!RD$@;<%J^<>o6C
z26L+@?6?Nb3zm~t=*m3i(`b`z(>nJ!U4vimta-`XyQ*goyQ*?1SAyMo?*D1^%Dp-k
zv*)EME%St=4N&;yaj`3|@D7qc{o$$th=1^$qsk;lV?_}A{QKwTGuiVL8CCkV>u&y$
zJ=iF$RJ!E5R^+|2T0ai^DGE=Q(9&X@VSY(9OzV8Dib}p^gVBs@-sWec<aDyXgeZQ$
z+b5~)<*B$V?g}_=R((}c{*=Qj`@+0k>7)G(5s)2C>@UD^e_yppd&Bc)&H1?<8tF6t
zDb>cT(YY6=shXPlU-9uWPL-=O<;+%gxalM?ztI5cQ;v*P*(mqU#I0bxl2mSuiT9J=
zO8a+Iss^kyFyOLYp{-pltl06DS^3>U8`Jfl9+|WHr0FE?by45*@{QhQ@r6pF_vY%X
zDSoOI>vu&_l(9p1jqOg=e=M`rdsZo{E}J-4lJ#qm!ZZ_cjqAUB72mIr1*f$Qav^GS
z%{_E7)cX{d+-Fk$K4YQkst+HuR_At_@tvBYtjEx=_2$%hGsmdU+Mi07=~Q3&WnR#c
zr14NuRFlnyQQh;}HjQ1n?#kP*7+M(iwVNMO(N*PI$7B|?;*-K;<()b&JXBRqd|#k9
z{nj)ScT-N4A0b*6mDiS<UbDZd@K~ryHF!CjLA$(&<^;Ji<u`X{DI`T-FwpnoQ#X|0
zH+}zJNkhYDw(=X{f67T~>P(mI@HG;;wO(D{Ls`xKZ>ZYb3k~W=X3kb#+Oc0}+H-A-
zNpJbIbEY<EFKX%qrw`K;yc%z8Ma|b52r4pbGAJu4y;Trn@>g-Wy<a`%>`WteOC$4i
zjw$M?MYg){p0b-=x~{3dr)q`tBHntFg}>s=yq)hV*lt^AwpQSsx|Ip9;j)5>@;$$O
zH7y-ysCheXFkjkbrFyHI(c+?EgN3T-S_4*5PP1LvyylJiAqJl!-fK=(eWrORkIO=#
z=brh=m&X*XAG-=)lRjjc_2!O3Se>rf<U_CJn+;jbXML(Q2skBaoN_=w!Q;>ma9S{#
zXROTnTSD<w)N0l0#|O0K${18iZ_F{@d*q6=<v|+-_B%2bLLYk6YXy&~m|bg<U#4KK
z!Zq`}%FTo&>gsFkmF(`^)Njmg1c$?xs7dCv>r&0)6jrNB-ttjxYSc7s`_rMgxz1Fn
z_i%@tRgj>fZ})q3k0ws7JR=#f{agK<6sP_<Y5u%WOKFB;vWh(0P4$#pHLB<DG@3rm
zd20U6Wwl02kAzn3S80vY;T?wWRn)<D{e5%U+^2P?hA&65s<hJ<<@*Oa)v}d))MstX
z*5Y=au5q}BU16`o0<}|a;ffyeJPOA5^Avf%>zMC*b6(|jR5REN+sj%?hbIRbS#fGw
zY}{d`((R+ArqUju&?6_V{q;<RIw-IFH#w)V@QtCmMnJyFhNct7&Bh&?qF2sq?<l-x
zwl>;A)2ueZ%=6V9bMATR8pXyu7LWhUQ|>7FrMT?Wa>X_G%~aJj4yoSS;Q%(vVVj!@
z$V@lO4lw_m;a+7BzbD#E?c`N0<wkK1HO<8>YV)3}s#)xut@PaN1h}5*wB%NM<GmfM
zSJ`7WSPw`a2uG$!f%Df>Uwbtc?)hr@4T>sjoGaBml_b<dG8U;A#hq1o&g81LO7yFe
zF!OS-TIPGq>YR?MU^jrwfZ^LlENYs<C15pv{~xKG`ua!V!rTL3H-Oa4dLpC#;p9(M
zR(2KD1n$Y;axQRBgqlfyf^uqCG`K7>OJP^H-{YsM-0!7!*u)QPmf6?2V7EVxa|Y`P
zSkt3E<!+gpU9KM7?57o;V6`B7@(%MV@!aZEGcI@q*0<|loC>egbTxAu8E`rK&*qET
z(|b$7?&s}V4K@>`FXFPR>iYX8$}F*u<n0Sx6&g63;AVp4k#X>E6|j5t7amvc;r*hP
zb$x@HXWkiwhp!sdj57RHd~bKj!_<pU*H?S7okvA|tA-j^&3(9=y<0-S?gObyE<K}E
za_xia%!;L|PTN<j+@4<yRwwZ0Em+O!WzwoY-)Vx`m&}aSdQ4}ioL1lls{!ekXpaKB
zbx!ki`A|=GFdHOZR=i9l(}4lZcR%<-WucLVn!;Ttu;2UGHNbxO^KzQX-2Ulmwr*0&
zv64k#J3xB3JyKR~wA=%(|5Y^YRO>IQs^9i+0Lv#=CCbg4p`dyynh(scY>ANr>0z^c
zqvlr`4HgS%JS*QHcOPu`lJ}wDboZ5Gi7I=Dsv6scPO#hO9V}N{q&PuUCBq5K2bmGo
zgBkAA;)B5Q6`B1iC*^j5-3?Ovfk6dqzERF|rFMS-wL^0bsn{!IsugXVq4xBiiCRH<
zCRlz#V~X0&NiV@>fb@M~TdyLh{S?l>?yyPanD+$LPUX9*8;-St#Z7LhtGZPjQQdt(
z80;UAz6s8TV1I~A$dd)}L#K7A-unMUrS)aAs;t*cRY&uWVEa@fWfUz%G}R;yR)f<V
zNG%MnI-(*0Vn6tFQrYN4A6Pwz55gahrl`3(%uza7k)W=*vsksqTLx?&sQwQQJ_&XQ
zr;avQ&u8t&V0X%z$SJrOPE`HRH&-Q2MguGdGOJap93D<zPO^f{0f~dKteY)3zCi5d
z_tL=TfY=}`ap|>E-R5w$c*lES{e3Lg)wG#rgU!u+yBF*qj=n<WEZqjBhNT5yz1L%k
zRat5y!0d^|6)Ku()6{pn%>&yxd8VJ54GXX8M>AJ-zS0XSDLt3emrZ-F_Cx2A>V_ms
zwHt?{RF>YfR5l2b0NVj_0|*;l?g0B2#0KG{-gU|wUY-G~S)_SEZDXaOdbh<~)qk%b
zX{<7AhH@wCIn{HLO6tA0V$}{Eo1nD(<{hw`y0-<WDTTiU`^RKjowAMLK4nSId205N
zVqo(?=C^qms3mxuR{i8}r<`EbsFL*ZyYe;rt4e#^ofLQvDXQ%Gw^Q|$yO!$XkWQuO
zqc4@c?1fd=i)5;`r~Oymd}EUO`T2)bZwCDV+gGceqI#tC2bc{q8-#B$iGb}o^I$QU
zp2U%&7N56LtxxESiqn%hs`EPf)VDn8S2!CAQFmj;4Y2<1?TTuCJmxBl&)+N6@yn>)
zT<50vaizPO`GUPl33^A>d*tlZI}5GBdHQ*w7MKRvSGt}_CCYp%m|w@XPB}qYR_(td
zi>mB`i%Oq(v(*$Ap9Q<)_53wpdp2HVQ@eDJQEvXL^<Z(38Ivz9Q>~nTRc$rn9JTAJ
zB5M6Uoa&(ZKTYd4IBx#kZ33GQQVYY8?_0rg=`lya=1eIyQCsuGRH>O=Qnjn~HQ25)
z{?}l+%SK#aT5oQ)a>tRgO6h7d)Ph$q$>pt{sCsP2eO0FbX0?L&Wneww7Ho1Q0)N11
z>azO{a5`MGrxL6mW;cj_AYP|BQ6gB~%#a1eZ6zm5!Te=^KdY{vxmoSJiN5MJg*~dt
zO8eCAEpS&YPdcI+P`*RWa~`Bl1evw=up8KJjc7GBr!zOfY>@bhZ-olMyZ0;0=w_?k
zo(V~RAaM{rdiVm|O#xe#!FtU1bE><$?*p?zYWj1vz-s<a3|9*7nWlVUTA-R==0~L?
z8+yR?(Xvy^mCt{F1Xln2OD4E33GKY9_Re#rR$E^eSZ~CX-9{j@Hst)&oNw%_w5_7c
ztbebnTH$haaKE#nVXNw;R~uCeRPt5$1AZ$<wMJ>L^p(@G=9E%OeN(P@tGrd|ezB9W
z`t=Ca--??J9<e<(zTtCNsrQ+v*@xOQrp#S0Rau|qtMbpEW>(mCRc(i9k3p`GwMwo7
zvuRw+ZPN{x`Q`UmFV|2Ho}~D9=}gskYd&bREwIr%a++29!NhCIFKQ}Hf36iXKEE_h
zJzd?{EUJB#!KGcv>hZOX;ByPU2=XiS%?(pyJ~PMcx#28>7w7EE+|TY;n<r*u7%wZY
zqaS3aS+_S=&c;JZJE7*7Qp*2xa+7zeo0OXPYORPptg3y_#jKQ-U*%~@wf6tVlU4U9
z+JWPD<@HPr_o5RfxB6Q&qmEgtbgfyZW@)0TdhWTpN?fX`+MnN?YD?E1P#64Lq>-GW
zt(>}`P`yj3Thr(0QdQS{8v`k2c4d1Oe?#RJjOJXvUdnMkXXQ>`7FWHgQ?9Q&=d9{`
z{TQ=pQKl*hbJ~>O6lkbfsJ$=<O%F7BAgH3feZGX2X_L6I>v=_Gjq_YO1rm8m3$89#
zF|N-sJ(#uJ%y}uI&0;4SYJ5J5+r-fQp>fr1Z{@bzrr>tv&vRvJcY8d|Zv1jpR#<-5
zWXkS6%9&gRM(5wksh#f#*5aF<p_=FQ)<E%>uhE2nx!|;*vu25^;<{AjZ#8VHcjmU6
z+8t5RD%C4Bi`yDz)W5P{H!ng;OP}MX>Z=Lbs^6Cfm|frbK_OeEN$vCw6P0z^j>?Dc
z&s3YkyGupAda+5+#R(c+B~4nB-qx8OeJ`l47oMc}cIyJO3rrTql24{7cbflJO<ws|
zA@F{O>4r7@hM+X){P~LV$^HI{?u{bqYqzSJs`OT?g?wu_jhO17wcJiwExxxM?9PZi
zn-#NKKB#2m1cK8*`VLXG;%7fpR~jBtlb@%iD4TIn)irj9>hXX3!RkR~CtjYe^dl_N
zATFF)<@I(~(-(@)3L5#>HQujcR+pGzV4!w@QAKb<tV-Z1d6Q?XmZtCLi5hymZqZWQ
z_CVR?vzwZAP?6@;`FoA{qtjK+2RW(j3O7^j*B4OxvZUT*Qt}olljc|>5e;@z>9gll
zZ}sb#C9X2m1lcS7f3o5GY+f^uWO)^ze=ce~|4J3D40M&YH7l#`dLIDxw|3%O-EUR$
znu&Q6)q>McYF<y%Hv21o6r2uiqgqYRA6Tsx@PSh!nCrQjT#}daN4xvRtB$Bhw&i-M
z#+{H+>#%cEyST1Ee(T2FinX6Nt6Lq@R-5p>LG|OTUbFR*n^jhM1cT4QSW>fArJN5k
zo|Nz~NHv1{iN=+`KTMy`FwkCqMNs+R=II7Z`V3~j<6mn}JJ_cvKWmqm{(ml&$E!@V
zuWpzoTcbV|T$a2rZwHr~=Rb+5227l$Qf`-_vUq=+j!JBUroZzOt(aH#O1E|ttL<E_
zuNv=oPx<jOPUZg9StdL3c4<jZpQX97uu1vu$xrI~cl5zyk1H0N8*67wQ(Bhjt?KNx
z*5E+yZgAPtR8*kebK$;m&#wt)wlgPdZ(F!sZS%`gy)}vVRRcpsb<3ZZYE6H7TBZB!
zY3=afO{%T>pVY(Cx2gyl-ILAp(vhFVcus>U@4j+Z;9eC_TzKf_sA>G&qjJh;C)gi(
zY{_abRPSi>Iq8~8=RQ~S7uv7IeZ<U^!@pB!1#hyh{3|mv&e&$nO3#Iwara#{tHM3i
z_Wuo6(s~hL7QK<d>|-pG%Jkww6Nk)c$`S>tx`!WzD+QKE=!^X4G@cm4uX6J%mw9{4
zYp@@0huW)de%@tJ=ohKwZfUE!=$(veV0NTo$_58@i&>Y91J2ISG|1LbKTyJ>qCfGf
z$w~Dv!y36|>di8X!EO~<F-JXi=MkmcC7QbK>zAAT&{EP~dfQbyJ(b(c$nl`6gwIaR
z%`=~yr0-v%rn<9R^*LvlvPF=GDp&q<jewn&hB|pprgzHSHOng+bXR!nHVNI;qfuNN
zplWO>WE`US+@z<yR?Q)$%XF$$v&I)`U&E}q%8GU?G&ECXc-5@if0))hJgU6qd5c+{
z!Vj~c6&nryNc=K!y1mnM@{NW1QtDz_2RIzHW5p899!lGqec5?KB=E7dTH&L2#<x2w
z6j!;QF{$B;P}Tdr%+%S$%kUigOv97|0*Xg=Jq72HJ)gAIS$|6?MfTnU`?c{hr)v3P
zeX}zhyA;hD&Z}+-+G>`QprG~lW0CUro0pZGt5>URU8bzIKsH-5mVs5}<JCsJs+^^!
zZz5(Zd7s~J%CsZjbXmbfW!W8EYL{EPjC(bot7sqErFuDKmXg_$5S4WKOpRY!G3vX;
zmumXW^H!O{tF7euz}u`%exb_$!U)xU57IO?+8HawIWIDHoziah<DY^iPq(X9?TYIf
z8-5EIh$x7f#YOK_X{`(}3yEsc?0l!A>{b)5cHvT^T6QG6`s7A$?QFKi;4w45^p&bw
zyH6`yvY5*~JJhXG`uU3Ki2@~+#BJhco}cG~+aZ6eS`996tTgjv->sa*^$%QTeOz}z
zn=NskCYQih)#a1-Yl^4Js5wu5X!P@`kdf0OS?#?uA8SVx$(g>~$)zE^(8Bb^!B(@M
zYpOL_C+q^3i9X>^z+w692#+$T|F0L84>s4PIteVkbKNO$`J!bv1)Mir9kNu{EwxpA
zf1gcFp80~BeVZPr|F8H<>455!!d=RKX)S7&FAKqK-!;cqfN4;k0%2}`Be0o%)0Nb&
zm26b|H(5eu>g|{6JqeH1J=Nq?au`_E!bA6|I<>K=JUZQ`mS$TeXa2HHy-&3c>;{k-
zFg)dmwd%1$VPLl=*>R|Zs{T^=(cS|#AEbu;UZMKyM-uAI;onuO)tSNV9EtZj>T~NE
zl(jsWRSI36Db86vMSaWf?J9a(->9wJ3mH53bg2(qW_52@1k<c1uB!j=Y*gER@gq16
zKxXSLUkbJt#Lm8JqZBH)U(H(gBG~RDKewo)anDuz+L{LrFY!HY>bqk9fZeZB-X#Ol
z1Hy_4KUM56MJuVDnWDIlZ?00zdJDLjAbDhbTx}EB{ao)9m5%LLtH!2mptkz<WW~CD
zb!yq+%N5y~BEjKYAdx5E7vHP4WbQ^qO^4;GdmC8cW?tgp1-lPqZt*2fl@EMZ)Oz=p
zs=PQ9q*`t+307BQJ_qc_zVoUotMno5K0m>gYQB@-DF@y$1^Wl2-*C!2uzGeeHRT3x
z8!#IruBAF#d5RM+n16j*o7$IMH&xGH{tM>Ey2*g`gZlrEO<UCG@a|Gp_;L{3)&%K&
zrTIlgdqpj%{#SV9bxkEJtV7)(|1MY$sQ>@$t-R_HO-T9d<@^Bb-&IFfs~=%?0kc<c
z4pZ8<<to^&C5FXdee!H?R3^+!R};wj1or#8vpdx0{ApHwx|9{nSGu@cIq~@$upW>b
zU^wY)3|Rl8JAEoaU#!9AgXAVee*lZ8H>^<lZ2ebt(N1-h-!E3GeM=Np*WI{O_1FAc
zV6%ROzECsyX9;G5^rf^ms7fA@0NV-TYlU^FPWsoaw!C?&>PK-wusS}2(<)CLW~+V9
zIuDL(kh)9EUf}ouu|Zg=>!|87odt?<3wNt=d-ki`l6ni)s~Rbzz-HH^`bQEmMgvlR
z!<bbO)c?2M{a7i)agNIVw=Upx2$BQgi#-PF8*7r3U-$;7p9$hsOKJ-T+r72BTIua$
zL$G@dH6VSbYI#R6AJqSs3X)YdRJftiYx@{%Kgg_Bsd8}GfY{l)KBKtX{vl*62_(1R
zHDX)@#E<B!QwiZUh4=rb=hv&LvDARwxw5bc9QMzzX(=C_SFZ5Q5z>#bYiCoPSQrJi
zkMXFGijbI;Mi+-LSgqR|MKxWanJO-vt?E-00#&tAY&1%5^r%a|@=}#Lvqvpg^MlG<
zuMTC&R!CnR<OUGdpRpKh9*7OXzvFe4W@re4-LW9~otkg&6!k`Jc@_8AOJK3Z`!iMB
zSLdtE+T5ka?|MwFI<i?IEJhq09?3J_sqK{r0f)hRAq6FCVKJqt=4;emsvZW1A;^65
zzgJaPPEJ*g{&z|4n(-a^#c9<l*X*w<HY?v!th$n~=;OnxHdU%mwXu7La_6#WrH8T8
zReDbSQd4u{QCq$Hw0h>cnW`&$7{T^2IqX*HbgcrjL1xG9Jfir<Q3=dH%#{wNZ4aMQ
z^DB{6FMqsD%_s4XO7GIc>XMmfl;@RvRDHI4q4M+%8DMoM_Xeo3cC1!aQ_53ti!D*z
zmo!(=C8bL(ncG~6d8wnu#)mu9+2^-{!$7fV9+(E%xm?jeewr04c$}QKCQLOTQ%y}U
zJX`I+pLRu0kxsRV$wpxFTU8)qYlm8wtJ!PDD(bmF#u7kgv_G~}^;v#ZZR?(?s*Jsh
zRYiAqgZuy2*D-_RX3at=u=yahFwD!b3mnd-s=vVIOF3Gp9WxSF?wz?!g;miOELP_8
z2Ta=~eglX9#$BI4{eKna8|`Y78vB$a=H6C)JdsCLTIaqhqcLP0!S`s8vS&*=*zD_(
zTjhTroU43U9x;Xov;T%;u1byaclCaUcz9TV<ScgEg87S-<keoAIH><L?NMnlbW|-&
zU{LqFqo~H*CaxO6qp7}lt1j4WAU!@Ex4`b3`|pw}mp?n04HB<^9jk16Sxfo<BVo0q
zlU-nQLE<3%+$sjle^zCu61T<=tVf5-RlVSu2AB;}QzNniY=@4gm%_xz?TUr(`P6IK
z(iL-Bs}*7uFDNZLwOolgN)W6b*8d0T17SJK?O^lwRJViU;GWeXRUVCrir+jps!F~(
zrRErNQPpDSY()=wNZq^dJ|Z1@zMKWllOS_I*nRJQupQG(tkhDU_o>aV+N#X*<B@vD
z^TS{}Bzj+|o`~5BR#WpQ0qka*htt#@uTBTcgUo>8%_^O$cEXWhHN5=SmCBwrNCuZN
zg7t#b%-qtVo>0c8ZmIEDwQ0LN*d6(YRn?0WcYxU-HRj8E)sO4VRb3HxR4sk-YPepI
zoW&b1IS|c!^^W@I>~ytf36MGgB<ABa9c(X%y~cn=>52aiwag2M@ftSmscK*4Il<v|
z;efe%aL_vp_iyo2QI*uU1-s?O+fHR)ZvnWOAaikH&#p?f<0e9C7k120l4?y;yPh-|
zUjKvis9*F{>kb!C+K{tAHDCs0%oZfRjbjNoErZxzd|Ha<f;Xz^y9R>&%+{s~wwrbT
zEU=p|2cA)Jl;{A9Z4mBOyPzVdvOx|pe{lE9B(NOFtmD5p;cQdCUgf5jTfk<1dEc()
zD0E1*Iq^7L9;AMqwxf#K_eOAd$jKUk>l=``ZET?m!{&6b_;s=MDuF7K)mMC(2zC!h
z-g)zWl}PJQ1rQsC7ya&5-*IOH+ziPJD^ct^euq(2wKze|Yd0iqinHzp%Lj=f>SDL7
ztza>jnIL-E$J1bY-IGtK+%RSW#}P<;`l3r<am8<|lyokxSJiPZQVF~~M@=$>L0x~H
ztLo2>kHBUnZ``RCbDIs!2I>3G|4`|qTQ)dcK>V3I4yYcGn5-rp^hiY@2a@jgExxXj
z>Dmd`15)R2!3efby*2{uN1=!%s?wYjlwS0nQG5DcQMI=hQvZVL|6|iO!_5cj#fHVU
zTY|$ECTAxL8N>SjS4;iwnn$W)u2o=rK=J~Pkovx-c`evYVkb@D=D(ghPsQoVB^6V@
z=U_RIdSQnuuv<ZFeE50eNw_&Mb?3Jm!_B#Iz(MVre=dqU4T3f+rReQe;y-5yRyX<P
zKb7N~`@r_yVY#dD=JY>xS0hevJOv)gQ9CLtrc(W;RsE|#n#!SxIvO8$Pf)k6icr1%
z?7CWRUzf^hy*%Zpe`P4{`098RY#z)_aSvORHZ6eouSjvL+Rlaw^=|EU)!PEM!ETG~
z3sH59^;a_psZrarYPZ^pdjX19PGo}36z{vI##wp~oK`!{)Re1ay%cY}S)!&Aa0M(6
zvO~8@PIb}CL*P8wQm&@*{<fSFsQw4ZMb?~?cl#o&_OuBSR>$ism8(8oRJq~xNbQk;
zfZE>YyVT#jcT|<%umEh{p6iQL4)A(_!v>@ugcqNJw9ET%iGsz;`6jEaloD1C^~zCO
z{^yU%THX_2f4m7V1e@VdQvg=C@UxEExvFZ_8?2EEom;t8-2#jhWVHO%<arnr553pZ
zXyjR={wlB*tXFv9d@v2NFT{PZ5~CsyxPHD<?yh?HbBx-9nXzhHIMS54%k9*{oBYA%
zvr9qJsX*2awY3-QmCsIfg`1&U9IYCm@JOxc*BjLfHRo03GxOkbAUnR@5dfPHV#Ba>
z+FNiKo~VPgy^p>rREzn}qx>soql%%O4A@SAH3DFBX8JOL`LU_r!1aHnMh}?p>(s8w
zT-vPq&3C$Ln&)A#SlDc1mFKT_g3T^|y8s-Qd!C*KhZW3j5PdfLp9-h_8+FaeHefqI
zVjwKJiyzFNr*m6XRYX~R!k%`Oq7Ck<0on)DoZm>Ry-8iG`uv!*`oG^TVEaINa)j1{
z&2RH7QeEzN3v4$?9E3sjzxpF}wQHZt!RCU*K)7?CIh_Cc7bjSMQJaeTw}LWwdhV3l
z0Jj6AXLtWywe6*eiiWAKV1I$?|6+4Q{syTz_1FolfA14<aG1OQKMdwO?{-w{dACOK
zpRku2o0*RKMu(|t_6GV&jgov|J67tRRx6*P16J$eribE({o5e(g$MjMs#Tc3QB&L&
ztr9%NSVK)!L?hQqQDwtQ1vMQ(9W|~11EpqTNMFuc@rC+<uixN)1i1}_eS%)7#=m<A
zw)3EwKG>dSp;ci2gXBdwo>f=c8>9YWnyT8-zv|#}P1dhfz0uPZKL0=Gz-4u>3pZ4S
z=SZqA7t;irrT$?X*sR@?c7Vesy+d8&%cA>gPB(erX8-=i26s<G>;XlG050`&k6pp~
zUBjj*mnnQuo3IbkA4=p|r2dV~8XSg(N>9LL6v%xUkGH8B&pM?9n*TpH^}M2wY5-V2
zNDdi){kcGK)ea%B`>*i7Q+#sox7wWT#cG@j-znMY-2m%pJf019Ge{qI+A_6-##E)m
z)Cje{-w;26#P@z(4_C9sW|P9|BuVvU%hl8vJw7XkDb|6_5oR_9`%(3yjB1PI5wKWx
z*f}-spe~i&xwc?AkXbtKA?@i8PLYcDO@hJdK;k^Y`YI1*L*^t8N*z$UP{^<L$<PcO
zR_jFj!G32Byr~k&a!CEEETUfx(i{0=waRjh9pL`|J}D>Fp7f9EFU2;4(-Elu@0XaR
z`Zbyx?gx;bS=*nePdm>7Rs-TcyT%Q+tLRrF*siUc7O8sqEK~cp@r!Et?8%A+{1WOO
zYV4{DHu=K+qI?aKPeJ-&`1~I`u)N(Pc~zDN24J&5a?VV%z~W-<;YtGSI%>y4dR4xa
zGphUS3Re&0{jK`B!USyp*O&V0Q>Dw`;UaJ@Q>D`XDp)Vb9M$*7RNeZ|s?9&epw=*7
z5v*?So;cM_doQVlYwm{oah=jtFx!mX51bay=P9TqF1(?{W+S0q;&onisY4{#KHn$n
z74DP;VV(a6*$2XVYwVRLO@OTB0r5fj)|4*w9*rKw`j#o`Y@MEJ-~VKR(-~;~Ut{%h
zuzT-|K*FbeyOkP952*j|#(qil)^{CMj%<6d{UGsWALqj3AvUB99%djpWXziP0BkNy
z%;`g=a@)g6YUQi0f%VK%->t@TcQf2vkh<%eM3h~Z9#`t~DgwK~iic6PL=Do`pKFn(
zBDN+#BQkp-SpVx?`_<T2s;Fv~JXB{i;!@?7oTM?M;<mb&=vCE~6SdWKE>BVMJt?C6
zge47Z7RY=MPV84iao44ttCj!UhOAw2;?+=}@${U!Z0KQCh6ELGUS9Khg=)`(^=guy
zg6bWr+Um16pHlGu3t9Ve$W2Xs*Ty(-Sl)W!r>wOxOJPc%u)4`V#9A4~Qa-iC_MxgD
ztNts^ob*e%S<yx1{qubA{6EM&FX9W-4$Kl$1I_=3vs$V2JW5uTlJrzR(>6<O#y2sI
zZEQ<aFQ!8Jj?aEfRZZgDj^dwv7a)DUyWP9M;zjxY)beVg)ZH47sNLz?t*X_ftRdPn
zUGeEoZm_)7*MnesF7I5m_jB*5`gFcjxYAGs4wGpfTh%yUn<;(JoS-4=#-{N#{24fY
z?(O*xra|t_`gc&dyEqseHoM~Is!GmYp!WOKCbjfq6I4W1`_v}vhm@I5oFQW*()rBl
z8E?CkZ(Qq#n{nWcq*_hCz4{S{nQGnwY-)_%$HD#o8&f>MX{)6z6Kp=nJ{X?R2k9eJ
z{4@od<E^ww?MiK$@*yq-)s%!nu%FhRs|L&6YKDw)RQ>l=@%zH2>O1|LnuS%C!jzcb
zs+Q}Qt1g`USC!+f0$A<s9y^slQv<NQFuU@)AmgMkK8QXgGGBFxmxIQF)l0zoKw=<#
zN^UZkAF6Dlc4BRh`XaL!m7VgBRp;BNsXskcp(eTQl`2zMnfgCQ#N6ZI0%%)}^{?vv
zwn=ciH5blOT&8<KdEer7YKLC!0h<f51BA<e%!KowHB14=m)fMs>IT>Nz-*8j=LR0Q
zn(a^5D{;SbP#3)WTxsj!mEdwk_=t`2nKh8H2ax(vJQ@O{Aut*OqaiRF0;3@?GDBcj
z&K#B0%}X@bcd}~uf8fxzT-66&gJi>&tu^!9eU*R*OcsCrb<|%RnyLxv|9?r>QL|WK
zs<`ZYrrIv|=PLi4{%RiCuwNUr{|^*`x5`^p6ge%y`~UtY%+_DYs9*+~|G&f2Wq$nZ
z3o~1(i7LnEKUU>AeZ<__RZ{)VtOxpadfOB$P4vtz{)jg}!)~Uq^g4@1^1dI+v!psy
z<}h|?3GVu?$tpELYahcgWhI?OX1BfbO&+Yw&{(@u!Cd~^Wy7b}9&4rN<e8gvU00IZ
z)1wk{-%q{tZ=$*H^os^klIrGjT8^qqAJ8ynjb_&tpTVOATL0%1!w+6l6wf?Y-v8YV
zlP}TxH1=1XRV&vmH~*<KUsW~diB@;8g8Ge)<tj^L8l{Bn(lzZfCYr5%FQ#>|r%P?3
z8iU$dt7~d+9!*#NxlTbzdqbD{DfvDP_Gkyq#jV;(Q$zFB9d!<AN=GrOy(?L%x2-T-
z^>!hjLDp?~i;J8x3Rx$(6s9FIE9obkF^FF}RYjY%&wR~OP31jq*{Uas8q}N9ChFfR
zT4pruDTmg_6KYz^mOe89&Hul7G7Y?rWXZy%YM<`sn-%HFSR7pJr}j*GF1U?%aQR#l
z>p705@+vH*f!f;@7v%6+sOFqgO?Y)prC_hLd0(`eT90PA+2VI13fYhSO%_ZK*KYDR
z*GfoWRG-SY&v5AuR+FN*Mvb0{`?dKw&Vk$24HIrF?&n-+R%a!vHK$>Zc|?Dwapan%
z+Q!kvT9f`SS3453L2dP05A*9gKgiF|o2xXjdx0`&{{Ob*boGGETa-&*Ofg$1v`C{e
zp<1hFyN|iM%{_4ce<N?Gg~+n~#;Y$KRi10O0^D{DQrv3F=KtO>QHx7!M$l>bw3jUM
z77nY`-yfP_s%ZUBGqByu?1kGJO_BEJD$9eJz-{M%po^+gMHj1OR6+L8EU=JPKVTrO
z7W2(p-B;?Z^30wB$%S5@)DE2PQ2m|6psd^~p?ogYNi`<qp59&_F15cMCrz)uJg$)X
zYokVPN}9&`ObhV-zkQPPRgEUPnnVctnel!SGu$e4Uh@ODrkdykYjqcapPH@{4NY#Z
z4pn1lX;Ry8^OB0WOP1<{D>K3SG)1Ck8(mz_Z5EJpRV`#qvw6~;m6}aTZmQ{KW(LzW
z+sz$y4k=!^Vxa<B{}<V=rBoQqt*rECzCI`qnMYsM5s93mslx4{b<fL8OGDq=d{>{m
zI;j6Y{aUaYgG`UoUUx6``&kwi5e|<O#exHj^_?z&_YRbF?o)nf5vD%d)(e~-ZdWO&
zJv~~ZzWh(B!r|sQ=BM)|)Rv#B0;hdY|Nrlt8QMz|R%lrXFHoD`lC8O$<APZ>le|{6
zWvlYc-K<9QmkXNrZ#k#ElUZCre%3B=(D?s6uYKS&n8|Mh?u&!=|AE|{e~C-ks(QV0
zxm|{Wyj8FE<R(3hn!8q7$I@0SZ@Vw5JK^p;HRESHl$;cDRhRD1G~FM0S$oHsNgC%`
zc$M64pHq9Xca2H+6;|a-`D`XPbuK7`O=VKs_FG5)r2BWR&*$V-X8Y@E^ez8s<|N%`
zu30r(d(WJ7mF2%?8tf|#Q!STd)}DF$rPhsgylPuaqO~Q8omJbGGHPxUf%JQy@<z#j
z+jw2Qki%cqs}!;a$nKA)ns&rf?ed+f>f5F*RN;!$P?wxMNo$t5kh#&szv{9VR5Z0-
z%bD-r&7fU-e4F;-#hhjx6PUF$zW!BzB6w0G=xUsL^0E8M?e#t8`cpm4eY=mTtl!mX
z;yB4hEjwYQcFA5_Rn;q+x;Fx!ni?9+SKZdV&wT#MATv<^|JDEVYSqfj`uCsx(O4CE
zO|?SXN9khbOMS~FrW*d~|BQdQoz*<GmrcWSrJHKSi3_G#yNZl-qvAD`R!maU7tvI=
zFu1G%+W+_dri+eQf3Eqj34L0}c(Sw>=*%^DoE)gm^d?wy<2*&v(l_E78Np4;Rrkcy
z7z5H&XLO5eO4vvlnf;P9tyVp)QM$B8r{mE>)Au~w8iK6H)udvrj5N1!m`z^&R$caO
zui2}>|LU<fSPZh}Dl2FNXKS2|2vS?%P--@DL9XibN*)X5R33|sYbF}#=X9H2e6+(X
zM|+aKOQ)U2oDOGghOQsx8s|OD&mVJEUTPDezGU_;li=$CDy?U9O&>pZS67)TZnnL2
zn*QVGLPntV{}N|J!R!Bo^Butbf96Sg>V>|t+J>u>l#Ts#!2AEQ)Q!}$)E}7(hxclF
zIh<8#f6uC#;9{zlY@Mp)nJBF(pnE{cfQwtVou%3wH2-gPVWnB@fi$x{J(HD`p0KNJ
zS6*YfI=@HlTRdbQp@h#!RVe+4hUMb9nh$zDfbEx&ic^~qf79%uMyv8&W<O=Y-Nu?n
z4~VFVSMZuG&f~WT-uqE~pUo!C$>vivHMhAK8FZdBmo+m~`*g(9TxutqX0nxl+Bz9=
z^*gcl>d)k?HQvvir^&j+6P#y)=2fW4h1IBbb!RK;G=5gSvu24|Q?H9!)vIE2*>9f8
zWv;EtGg2H4JJT+jtEQ`}o@RIg?%y+7?bOm;QmSdQ;-C`i89A+|fwt-<qDdw<*0>r!
zusf<HpZ{JfIzH1Zw`abFa;c`-j(MBS<!A2E3{P5W&h=6kJiZ9(|MT4Vuj;<GS@rPR
zHQ@5r$vFy~he7!l7hWg36FkO+E^j#50<YemKM`Z9AiZwS9w@fyLe}p*`rNIQ^?a40
z`W`N@87ULzfN79AWZd~$OyRc5Q}7%;vN(+GdiAS(Ur{WIUhcPZz-6C;@g%Ume>zfN
zx<PuDQW?WWuwQa5|A1+jIUu@LVH1iuATb!WWIv_&&H}P#5F{qK*+hYP^9ry%AU+6B
z<Gu-A=f?KsHkby9{eHU-Y)|u#WSnM$)M3NMzmI_RVv{3PZ1ah=c<uL@6|U5F>m=OW
z$mS;sZdH`uzX8RswplT7^His(VRh%?3(I8O**}289@#vn8&j1kSLUOb2a_*#d#U&)
z>ne&n8qSN!-<RZr+woEmGd(;}@lfzxkOp>BU+W^d9VaT3&z+ir)ee|lFgkVqWpLdI
z<3D_j7}pNzSqwMhf^v<**Jc5wJ;J$Qw`l1d1=BEnCp-}IJ}@yJB|)%X1lvU6Y8~us
z6p!~n_UFUY5li=VYrx%uO^@eBcDOz0^655HQS?UL2vuDE(+KQ7bhR*kN>wk49+)_Y
z){mHoQw}7LjEz(b!R}Jwa|XxJp{>DSKC+sfYD-YW3O;{DF+YsC0Y#p}WR>F9#q0`N
z9n+L%u2EGwy>>HP{qLl+c>M*lH|sv1QoxNUu-dQB85OTT7F1kkw+Oy|7o_g}QXz#1
z-N$gV0~fP^{Q#0%@gZFbMB~ES=lCcEzM8CLUd0X9gREBi8K>eYuJ1~q{r@oWa_bVX
z+TGbR!T!Cz)C0_i$s^MSGazfQk;Ty2&#sE#)Q_$f#Mk$k54IP?2H`Kh+lbOXJJAaq
zjvzf&XPdzJ-sjeL6th5bFwCVmMd{@nZ?JsUu@?&8)wRI(sPQ54A51Me?SCK)tPdof
z<fR0U8xWruY!ZT6XAq<3f+S}B@Ii|K>}GWRbyJh!?n@E8uju#sKS~*cuAUJ8e|{g_
zJY=<kHjuStATd{lCvY_z-CN;okQ@x3(_V-YR{D8sz<nr~JT{uyJqFx<0LeW%tqM2y
z@F@<wdQ*?Af~y7T-H?~3`l>e`Y}U+c7geUmoL4G1*ahZqP+y7FtXS5!N{w$^v4%0o
z%(H(aRa_>{RI%`T3%6^*v=d<Ujrx1RVv|(3z;@_4Xsht8e+`zKtdRm11DOHCN81y?
zZ7modnVxT)pwuPX052D|F&<E27L)~>1yh%z51D6CpPvLbJK%>EnC*Z10z7ZP)T7f6
zuZx4#o|kC`)65Y<Si=NeA2NSgNC?=^$YLP2h~;&#nIJYc?Cg3&u|$*~#Xn5H+`wkd
zt}avBv(pUBhUwii`yt#vFVl^bFYR&$^Vu($!p-=(;+b+#$1gY^rnm3ZM`h6YKQKNr
zt!mYv^w9#+uBx!+2HSaBG(si$%z3c;0~*iDo3MF+*?vk4(lEV=PZ$*BOm>6qYWp%z
zX;W~xYW3A6@U&j}S05|~Q;$psJz5J_2NGXw51DHOv9C)fg6%TD&8)a9JzJUAO;>64
z#u~6$ZzE9atMsQUFybpFY5_RkuW!@^>j9ZLOXisJ+C}wXb3uGU_))VNT<z9o#5xFM
zeW3OKApX3bXK;H!d>BrS?@(N|0<!NPCI;&Nm+iR?Hyb97OxyM=VfYCqzh;v<O1Qzq
zvC-iV6u{=d_Wz%lXQk}6Aq^Zh5h1SNcz&g2fZ`TY3COxRn0bPXf?zX1{O-+#iq~)5
zQQo|$Sng-eEahvvz9~($)RSEmYX>%~k=+<<R=OPOS`v_6WL$V;rZTMlFWV)gTy(Ki
z3BCT;+v%lZapEsHeS`G=OXz{S!`osz*glXrHq0GdtFp>#w(@^|JFt7u)jZHzr~E%<
zmI`S8A0}tJ#2>6yN>~j{AF!E@HBLe5pW36Iy#f+L#~L4}D^;xU!0S$AJ;BQx!S=$$
zp6E`6`xhpLjXrVh2wdOc3lqTXxm`EGG&c3OS>CJo%VmK52I~J$dt?o^C*ip?n8v0D
zCgyZp5AL6gLI#5BLHgFrT?9^VAa+LGWU##;HnI3qfF?%$@Zie^tadXqx53373`3QF
z|FFav4#e2Mwe}5G{mV2i!o|~D{)5@p4E>7Y7e8b5pMY;D++84h|JQn||LfAwEK_UM
zP*y6^-Wewa-k<uqYOgxiuRF@E9Cj9`C(KpfX59)N|F@G|t>!tyL+$S+TlIB&xKxEF
z@oL=;-3lHz0hxEJyj7t#OBp-{>deV#5aoWubaC5M6}Jft7CV(A&CLv3)J|<&uJ&+c
zvqf|2Bek;;Yz7BPE~%LP3pLXZIc+X&b4Pym;c{go)eME#83D@geWz(WG|o|f{db#s
zZFQ{5O7Rx+dMP{8ri>1i!=B&F&!$%!-B@r<>xq`QMN4;}GK06P!v5WV)Ey7<S*Y!2
zFk0~JkU8_Y7>(55-;Khu%{8YnFIV^8`#`aA(HZa<lTVSDLPNEw+3Z$P&1GA3)bDLz
zwCKxWSK40Hp!G!ii2A+g{VLNMBb3d|rmL=qJz@5)s7^h;K}`9sU9@^zN{o8pCdio;
z-(0vgRQa1U%kFZjU0y4qEd2AKrn^7C`jwPrYUV3G>v1_pDl;BeG<dW1uEp>9obnoe
zXXU3QGAr|#DHu9wG;8#qSG4$7)~j+sPe$pK(>ygtp(gzq6Q3IM)`)7Ub7g5crRbX8
zP*qcIejBHG_ARS&(sdE_yE0PdfyO&5zL+gl+Z(+D9QQR_T1>7wC7Eg@&Nh7_Z=|xS
z)YwwH?3Y@A;wQC-lJCty=l|KQn+x6($~K3^M7h*RTkTVq`u*Aj^@y7xMxl#6Ob+p^
z(?|}s*D=ru0*?`Oo%L1KXa8cxlrvl7#-Ab!pS$cP`%`3f9J_SYi~5t*ZD(v(o9mTt
z(U;M#c%M@gtS9WLu<8RoFAdqf-l{Vnd^0t_U#r2U(5vwwMasPPu!RO_|G&O$zr~|m
z4%7cGM^(M%NUM0Q?^EmxTV?j7-PK6+uz^<6#C#QvmZ|b5no2Yd6?vPVxoV=>BqV5V
z_aa><+IgdLOcS4a99xL;1BDx^tw&VVuWS#pD4#u9>B7-@>Xi>%)Oz~3LHqv|1g2)I
z9a-N4cGnS|AVt>J*~*ERrz^f<;L*2FTC3DGTg$v_!$hUm_a~`^+~!er`@a`_4&SVO
zt;($79A@Wzwwj5k{WmDN+N05&8>E)nzFB?Vl4SLX`phODV=t;@NH@qQFKktq73QdB
z=N@D>tG!Qr+O1NPq?ymnf?rpvi#l#L*V&k?o~7ci=6%l7;MHD5i;T7Jm7TqpDD(U)
zm9u3&qjF(GxzcG?Hn6?(iYj#;eEp%ux=ut(US+-duC$lt@639Y7f;%xT=bIBT-DN1
zJ<-)hy*Dw`VyEgSWhMt3lMP8bC9QWqR*iFBt2TcVyT<7cS7mrKf|U$k_sNyINovH3
z392Rdi(52a-l(?dPboOR8t_Z2{$JazWj*7C)|}79>gQkOYc81?WiA#!Tm5_Fb7jTN
ztwv5kXUwO`o!54JvQi=OgrKC8i;L>nwI-S}?V57)LLuv}Vf+6-Tv!P&A5FzrHNQor
zsQ+K7tKL{(p!!kky^c~gxB8)5GAfUkaVqv^IhZ{upQK$IA*6YKKct<j>f>$tcE%5t
z`%P7*&mV~@9`!X>3uY454~w3vwP)U9<$};F8ne%QG39%sX|d_)A#GuWM78_>oAtAw
z^r)@hFj0%^gP&HCiKg1yza={9JKw8vN1Lei?}e;)%k$EaxtkEKy7$al)z6C0!FgrT
zLN(=S^O|&i^>0_7`(8^qS#!R+?88$UPq)gO-_Bm4F^@%FQ(Q;E+(9!^bBEh*t(h0u
zE$#&=sT<XpXgt2`qxS1jl!o^-X5|&H>MYdEkC~fRN2{oayf%H1x=2lP-FEH655pBX
zwK#Q4X3jA2j{c?Wce~j_`41m>{QqlBxa#x2-}Gd|UDOkQuT`7Lv0L@=Qx*d$8AUaj
z5C$`&L(|l)4Hs&Zbt<Y&to&qF+c(YVf~v5Z-$DuX_+myikwRnD=x^(k8!O&w^T}pg
zd<zrSl&y8pikP<6{NUc_>X8@2)GMc_n<xC3rm=Igq}mo8eKiH;yQ*)D<JA2ELyh)I
z-7{1CKUK}@X}ZpfI7aizslqDv{wb+D@7imuermtjr=n(!-4}M5d)_~;e%p5eI9;CF
zzf8S^eW993<~{STF+Iv#ns!?3Pg-m7sA#o8olTxu>Ypv<rfOI9Jzdi^*mm5~*j=A)
z(HeNfocC0wYO+4R#-rjt;Pd}ZdFPu=EMBQT|GSj=>Y|nUHzJfk=l>~gXZix3Z>&wc
z2QHhpYD`yV<~aeLtEyyM13t%OQRRGfr+^L%UFoIj7yC_A<j=LM`ei`&$COU^t*+&+
zq##hXM4u@m%{*&TvPxdJnfc~}edg0dZ>p?kf2j7#GtRUtbCKG&6Mf+Ffa|@G>Y}b)
zs$INsTD=+(>J~yzRqR%@s;TOWSUfM(Qdx47S^1-(f@+21J=Gg~o|!&Voo<nTMO-6y
zTBYVsi7++u`{72s-mlHSo?Wh%=CRGZsEk>Cp;wmLC*d9HdAy(07&Er06$ek%JU?jz
z*uJ0>)yfY!+te&s%w<2MiK=ntPBFK8m8mR#X_k5CM`IPx`oF45JtNf#t1UKU&QUI1
z?WHnh-VQn8WDCtt&Fj@vZ+5EL^UY8TbULmebA!XwwQ!2DpQVJxp(o3<WVA}n?(FW=
zSg=^jJoB!l#qF>A)YYm_SX9MrQ5Gp|Py>zs?^bbE(~D(O3B11(-VO*l#I0%U=Af>{
z_gFJi_qHLZ9_JM_(VCNUN@ZjH6SJ}nd1}=++B87(|BHn=RYO`|$UQZI><3%U#G-ks
zqYOU(A2ugT#qryG<4sP>l@^7y>Fv3@#YAyiv@%D>I<wx#{NQ@0p|C+UuUf_Y&7@wH
zn^%wMf$WnF?KKs%Z#Dy+|Km9GnfjdFtCa=iBvmqxJ8MN<IH}njXsk6SG)jfHLeT6^
z1*ZwOwY$c(gB51>4j&AD-pkZ<{(sJF%ELA#i@PQ&CTF*)>FRwqtIF7+FX&!r78)$C
z?lQB>&}ZoZtzYf8K<EF+t3BKZo{!PKpCNz4Ow(A?;JUir_FJmAK6IE>)Z42Tn|Nwz
zFzr>#xYMn&hdV-fdWyB?>mQ3v6ByrWo_Z~-b}Ms>vX``)TH=>(a9L7pcR}r5bh|oZ
ziVV1Hl`<|+D=nF-xvKbr%8A6Oy4R(-RAZJt(DSP5HjnwKAn*6;H0b<a`I+*H`dzK8
z>h9|*!C|u}m0Pvwte?8#0du|1{kx4CPcPKaj^fg)xT<ZkLg|a*mOH#)ce51psxbub
zH2vZ<&Ah6!LN)N*WUxCHZhT^-^7o{1(q0o2owilVmL-Yi8jJbC<uYISDzo*PKU6KI
zC7aH6UMiom`JvI4r52id?r~`a{FGOl^&;0$(9P3WO3GFJ*P^vr&l|YaK<EF0;wMMq
zuZh#!1)8x^uI4k=)*DT$V9=hOVXc+*SVOIpAzPK9W-{3QAhn?Rf9AlO%5HIxb^7}b
zUp6^&`hfa(H8HIf?%8H%WIDllA@ov&x%|KN#z8vlDuv4+=TSCY$S~2pmtnZ|<2rB|
z;P#D0F6fM$x^tGcX=iDt#twspracF@X(coZsjTibQ3TchB?5C*%9G!#=D9PPgX|Yp
zvV)W(u^-hgXY_-^<&=~wc>bU5?0tpg^YfLxo_|%j$8=S<ZS_Re(DzSFo1T7E2(<Ij
zcphe@5h|Uke`ZyhV&UgDRhiY_Of=ruo90xq8%*#k)eM@~qWXR3M73ug2Q<aEZZneF
zu~hZyH#xPpGBZ`AjcS!%ZWRExeb24DYmmBPifQ9TZ8cV_0<*r!O`5GfGgT%%Qr2g#
z-U|+^?T5A~J&EH|+>olSd|^Ym;%4I=;JgrY%tJ>>qFA$5vPHu*hD}RGAi~UEpHm%F
z|JQbug3TAqFI4OAIdA4^eNj<4=qotSg8UA`^`WQKPJe*Rp~M9WsjZ#x03043XC|Aq
zeQi}edQA#krh?QOh}dc`iv6ac_}NsAqcvCK#*gc!#g|@bzC7rxbU-cLurzprndc*2
zQ2j3tn*Rr#|8qw@4V(r+`a$@Gc{{wX0h|B-wcc5~^J|Ct5jS?NzR)OT#_b;3`O_i$
zf*&vARNYggVJe%RskOKUvPbm%`J-xb6^tgKzeMCJ>v>IDtXt*oTQ{iu&STYUx!R%S
zllVvZyIGI=H2p-A$2V7)%`$h=(mZ}g<-!y$y{|5pl(TkpXzkWk(3*JbuF9j2F<M+Z
z(^NI4@7H*wHwm2fAG~0emy(;TzH(-@O6F(Cp1EV5*Hjh7j%#iF)~z=8y_V9$&yH&D
z2hFuo{$Dn2`c|f%+xJ;LCu6JW>Dm=qi~q!EH=D+q&3tuHQ(JbCn!)o48UYixs7cmb
zR8Dd`Z|1?=W47$pCsom&XC~`-6I5m8MYX=({iHHU&`oE8MT&8;eW~iDB?0D(iZ7dh
z=Ko(*&s0-Y=G5E9G)2SuViwqq&-m8p-F0Bmc#t7#GEHc)W?rnX`VvbU)rB)Ro6HVl
zGTiZzO+(W2o~pixrt-uCn>4OlpO$aoUZ!2ONzy!oO<T)fZk?udU8~u%+NEkc^7S-j
zf6XxEzYN(sqFlhJ8ek)&Dv<e6;}e^qq5R90rY^?wHN;Oc>3n$AVcO7lP+h6_m8zjd
zr=jXb9aArTd-bo|&zkzz|5nqQ>7x&t|2Kd0Mm^EUP%UC2yIE_fpX&b_4Rb-)Uh|@`
zL;AXEPA2>Q*_lNw?$=9XTCKMH_fM_OPo&M4n>Co7|F=VlEA^6EasM)tMMtDnQeVfJ
z=7<)lowz4!_GG1>-em(`@R@VZ^*4g^eE!ab;PT;HJcoMUHhHZ8#|-7)W(8&!iW}t>
z3NqEMR`HmxS>>a7z2m4#afiFg3h(8r6B(W=u2H$C!I9alq)@U$C$>e^?9pL8<(xI=
zOt!=~n&yZ1C?65GRh^r@$t3>oG*!ksZs0s|Z`MZTOFV@dY^-zDUnTAYm)(=n%~gN@
zIbf!GV87CtkSR*Le~W7h%ss2JH&fbF?iP>vq@;6dm+wB(DBql+sa3tmAi;BynaiXy
zRW2bJv%|fYG)_)pR9lf&re>9NR&8;>SM`lnjv6*)d|FAbnH2YPv8he@;-XqEr=qCQ
z_)bN;yTWYMWM(yf?;dcO0Gj{zRM=v$?5&D<I@fNMuD6}w`ag&-PwRuJHQ0R~qO4k;
z$GFt<6y1#<Mamf+x>u*Q$w^PEZ(_GeWmTv8m#0dm>v#Mz;|QCj`TC)&d2XYwa`D3+
zB~bl;CF7=wUM!oc`j=pE7-xKVr~cNBTPs>{uLh6&L(RV_Y2f~UJl`bEzjHUKcs#gm
zo)IRi!L~|73snC<_;W+;9>ZO=X-xg<t3PR~@+5d_z2V=c4LbktR|z-h{69r5^IayI
zcdb=7FI}L&|81h_#=L%&nVd7tcO}0!Tgoe{{yj26-6&qv;v1Kby0F@D{eJoRs<KDI
z%;GbH%t7`469q2t`G27Pf7rD3+H9&%G`?j|*Xq%cP;b#<F~2Q4%cOGtCN&O~+2&fo
z7Y$cx$Y{;%IBtGXTu<cy_c_J(Z*>}>*{94;-K#TvW|L)pef~X-*o(PF%M15vf7gAd
zQMWf&{_5fbTA=g)X6jVS8<)&AH7-!qI%Suw{;aswe34X-@@pdo?G3J)8p~uasiovb
zDnGq(MLpn7vYGr{PR&=`FO}{0K2^6io~>Ro_o3>~bqZ=9e|u|)e3_@&|L3CmGzL$l
zTdi!G=aShp?IyFS-+Lpi-_tuy`NFOo1C#1@i}&`1%D&&u%2ylMDypoVX|Tg-n#S!*
zE6q)QRI64*Co1c7>#IxOO4dKA-(p;pyHE4|1R<^JRhvyXUG-J$FRjupkjPW+d19dc
z$oZfdzg?6?O~_ld%adk;>$#7|TunGu3YlK~t76(?X{#2Tu+5^y@xHpIbdEZQ)+}?-
z`G3w;;bx%o|Fq^WFxI_pp|v9Rgr;>$pSq<itC2@Hn~7psi~9UYo3(sYJJl7}r7AaF
zGE&=p@4Q)CAcs~`6PJZMdyg?kj*ITg39B^O|EOv3B=D-=Fxv)N|EKuFWxM*XxeJs*
z{r`l}D;naRX{zdG$4x)(f3B`^Q&KCd__LYp&a)a#XVxg*JSl2Xbo8jnIk87-Hp|zj
zRxH#}%2_tqOzoqjk=ca`jhS1-RU3Y?$gh$7t#P?vj+x!F{TeqJmY5lf34{CpYA=?8
z`~Q<ZYpLEaFHk?}bjv)5ElK6bq)+MxC044{^Z!?yKA}m`E9{%vM3<xLxi_Y&#folM
zY~f;3{nHz!w71((|L%(Y%1H;Wnmye-P02Z}LgUkmGBwYe8U|-pr73?*^;WCOooAZs
zoNFd+GtEFNBVTLpatSr2Jsuid`8PBc^zAVAYrC#?yG%ii$#$p8j+h;48fu?SLFfPP
z+qciyO|!>LO-f87)+ogM<KxpBOKfdbO0|3qCfuHA?%uDfsu4X$xj4*5k@Kae8mRx@
zEByq#{_oypZ{7KJ{TfY2{WX_{Mro=ZDL1cvpaib}zv^u_(~CW*ethN?waYzx7LBZH
zm7J=JOcwOIC@xT)r`G4_q|O!As3D-D4m$r&NikSdU0-UIhQH=DwVZWF%uV)6s4G0+
z1fTymtL3(;*mE82A2r@uC98I;2SqY!W;n2#cW(KqwMY7|67R|*hH+mK%vs}FK<ocN
z=l@CkS|FlUe&eOq?2^}VT+bG%eahjLhV}o?e-cw;R6nNtv0$t62{8km6YXl6^DWc0
ztf%!TTdnuj@#@l1_e*@Ea{l%#B@2<qrV*S>+G0XMnr{uKsMw2|s#l)8WRj!9ujcgV
ziV3LypD^pAn$OB2eG`Xp?fiWWO859}G<(D~O@BrnFjr%6)^3|>r7rp5klu+m57mB!
zw(1<;kfkLsn?-GDT9wZ48aefQj%Dh1g$2|Xx)sW9zI;&5E;vg=_%(-GcR>KSzLnSJ
zRa=reNjrN5iw3CvKeXYO`m}@&?TqBDW+yk=Xf9&y)XbVW+05*(l@3ejES+<yUFOqg
zf6&N(m91fKv`K@##$98r{|V(=d#;(Eam_G4Uf8StQgfrJ<hLVgi@FSSK>h!X3y<si
zKDRW9mxxhm`>0{@|KvPVQ2k$@@lSQyG$Z{h%Vf3IRm@fY`o&x==L5gN>S9xMH{MoL
z`-S3~MhvW)JNM5}v*-*k<L+WHx_In_y71Qs^;u^7R2ulctKHyIQ0eAbrNj4&*`nsq
zQ!SpgZ?#h7z0G}>8)`gCcF>H=nPg_d^I2n3@)osIt=Cofm$a#UI66(EB|^+d-mcS3
zCuFaN<8NbK!R`XH1?vPg9ymv;yUy5Pd_^nEEadnFjih7h<}n=4)F<63HK@*?sIa8(
zj)rHph5ErgUgk$1ELS<JBxa$Tmu<m%vC<%73ZL1R``+d&3RCpGISMsAH+|9C7xBSd
zxMG?)4`YYgt!Gm-<Rel{LFfNHiq|xYi8`tNZ_^{QlZLJOU9DG*rr%hkID6?DaQ&9M
zI#4r0YK_wCqn4^X*PXRqiAbw1{u5;mI{$C~tmEneGAb5fOZI8LI`md$;`LV588&L_
zt}`C1a&5BI+>)8AeDVH0y@?uc&CVQOsd97XS+ig9|IGGl8>z^z^jF`gt!|pL>Au<*
z6-kYsw<VSSSngE)=OLxxB_gePYo3fI`}Z2vTTkRv{hzG_o&T>a>Rzd=eQ38j-(5kq
z<=1OX4OxUOQmiC2g!r~<imf!%u+vy(sGB<5T+wBfn)&BO^Lz(k&2}j{^=;Au8kf#b
zQa`u<w)!ohEX{&kH?7qtS(V?`_ozlMny&WtM6ldPzLlzG%;BK(|CKMjPB91F(_-+N
zNhR&ZOvCyHRSS#5=afz}JW&Dl{}*iQ)y_&4)7;0-s=nV+Q*&*Ko!XOiNhXQ;j>bn^
zHfx>YF3?ubb~m%_d!jDNyvFPl-(&NON&K2mGyE;;Kbxq$E>l(q)&E7)Z>#mMzNr$u
z&l%kQ0kz)+a#m}`2*{|tXrHNl{?ljhSWiXAT21X86I9Kz=9p{hTdUNZHw4}Pqh!dS
zscNypRO!yL>EJwcSL~FQ@DCMu|G)0_3gwr!3rs-f9x664NXTI}z2S3Mu}|C3{C&eB
za2x07?paDlg65d#yUMHGVHO43w|uX$X>61VsQy>fe(+vxfleB@|8FmSUUOQwtmcd*
zPc)169#Bm^w8iXKYo3XlK(G1?K3DUqe0`%HSu^cta=PZpEWed>?}@4;-94?6cu&Rr
ziO^}oynFY}uImM<=Irb@`hB8E>rk7GrpBI~irc&=g8MKj|If)Phv=GCa9L}v=B`)S
z^FhJfIF(m**`(K6*2dQA0vd~zFEmCd9}nu+(Eg!f=JCTo^ES^{)qq|d6-&0os)rr7
z%7fPb@7SBFs{gQ1%}%}u+=i_b%2yYf#;94JvQ(8>Gem#2#Q~MOuFnlN_v>1`I=5cI
z@6~Df#A{cSvOWGA1WYJVjXAmi+;_d&%&Ka9d4<~bY9WJMb~$5y?p<0(wtv;~m5Bw1
zX^~?t*xkaaZ&U-^)68DRGgv&CtgABl@C9)FA0=gGENR4R62pGor0GzQN;cCPix<h?
z!Ex2z$8Y{~^(<9YwNKza<j(09#vE^SwH5ZKYqI^mtny28kKv4W^NlA*IclyAv(owT
zc`kg6s8vPU?8N&mnkOz_HLqJ9YJA$bS9`nkGEHw=8^xotEULNBPlMeLiuZ}#3zXJr
zJ1Si|{sXLkMa&@+gJMC=RV6nyQ>A~J%81_sr@iMl?wQ}6S!q()wMaQ<n<ThB+NnIv
zG|+diA<w-Ous>7GH55u5)m8n9Pnn5s3DI)eDQ}j_-><bZ+g+7m-6~~J|35awSY`4<
z8CBKUkzhAW?A;Ew_wSDnYVB{W!Qqmm7Ne?r!CE!x7K?JpB}P?~6Z2FK$~WpCoK~)y
z>!)j`#&uB1Ur9@YNugJL=3B^kr#t%x)efm6#`dbaO~V!(HK;Z3)ZD=RN41wbL)A`B
zLNnUg%~)3IlFES;Hnlm&{K0n0c;th_*6oYEkw&_S>1`b|l?Lx7vp0JlXh<j*s&NOM
zGDr=q0lSm&(mj<Jy#opyJNuLrmaJ2Ln=T8^3qC=cb+$e|tD)`CqxJVhi>Bk-H)cAU
zGS#+i-J<OBXCgQqBy?_7_tEP&_o?SrwYdHb+<yXvJqYKEv8lrP|C99(sYvXCoOARe
zo7ue9bFK1?{_o(h1*!cf?XUexc)8Y#{bH(dR(4ub{63j##fF0V|0>yhR}598ILw8Y
zoYVep`y4#~4Z8nl`pOWvevlc+{eRf~KMHPFwEq5<)hHIM(KK7YuV!l8ty6VM8JrH!
z-=3v1WnHJK&4%eZ*E6!gW&XdgO{(U)2TV#N_bZF+Vm58DZdEiEE>qRIyIOy$$qlXY
z2X|HE9hua2<=LCY`Bs}Nh|bliUzM-gn-Ho0Z{c*6_lEA;pOklKC5kAiykFj+rPQ=i
z^@6#PhL{iJ?hw{lZE|fBJ=9s2-dEaq3UU_1ii_Q<_0Mf|c(#eEWp=kJ>GMxkGmK=^
zn(J3!#w@O=8uja;=B)E2W=qyLYgJ~?&|ax~-puB>oMz1RI~x6VFVvfAbW}IaovGq=
zb*VXT8=JXN%L6shO%*10vwc*5ws-0zu2`Ycza~}J`NkFFu%=5Y#+s+ir!x6~?*CCz
zix*MrGHub1pWdS(;KmQ0%l21qGf0zaSGUhpHw`hnpz-x%h8l<dJk{+FW|;j_w=nVz
z`K4afxIsl<L{oLA7K=v0;uOV|t&?^56(^Zj1u$t{`g>ICjWLt?l>LiT<BJ<LbXRzr
zDT+hR!VjEwPvw67NtNj7RvKxW+6;I7FEZg5V%9jVFkNR}Pm7uGqD*yr?_`xJiH(Mv
zB+i(w@?lZyWU?~<d8SdLb?tmm{V$(2(N3dSVX4|8O-b|PR*kCOZu%CiUrv~B;A%8D
zw>il4FROt0qsdMB>h%{i4lUZF_4ej6bEn`FW}yB57bY61F6h%YovD3Dxr=d~sXgBe
zRo>5bW?B<u44!c1g41tf@dj|&+0b`TmG!rT;&&AfwfOD3v^5Gclox6o2iG0u$HG)6
zY`kV(ac;GS`uyE04clE+bmmB@{;XN4$o|4fL*%oS64Mbo-QMgtvo{g56<4;LHI@CN
zWWK(gMJ02+k*deOGbW$%%2XTY7=g=$#k=*D&&KI%ELB{pb+C*doEF7bu2RigD`pNl
z|JO})o$^8RAdQPE*HynBGB>^Zn9Jg5(MyeqY(iQk>iaaxmcKN7eIwS~$1O!A@m#an
z@sLQ3=*A8;rH#C5?_KAq&5b{xE@pC7v-DiEX3}dWCBZAlRfX<vRMGbRu2Ay2Ri$Ar
zqq)4{eATxTZh-w)yn;zZ)4#y5PMF!^l~B7%>1r=EQ2lTF+g$70<jLUh`M2FpbEdSL
zddp=!lW*nEjeeez)%vXEul;4wVUzhzv6`Dx7MsnVRc>C_^-zQDPN>C`--lI-ANDAN
z&i_x1+oGx$%cd$l(*s=Ig7Pm2M?XCX<}dtP2)5&4`5*P~k6Ki0N)<Jlz3-}(W}H@I
z4E(Ib)$jtGhrQx9g3EA_-k0HPHIVlIDNGhyrC~Q;S%qn_xCYzPP>mZthm}jWozvLx
zav#|2;`KjN=WKxFd-a3{aF|>AZq%q)0*U{1R*16&&7UP|WGwouF0x)jUF=7kn*Ra`
zjd^tsH5iNDt7OP^s+-lcX*}jXr`Gi(P$OT-MTITeT47a~uzHi`LydqJJ?h*lC7RQu
zrl?z;*sNYrv0j5G>J`|Z^^P-@5^9bqru;uAFLCgQ`aw=Ujjz#RYJSptHF)?Scft4>
zuL7$D`SDi>yIRJ+YwG)E^nmC8L1Il!2i3MN`wuSnyO$WLpNwbJIJWN@IDGzdn<#R<
zQ&L}Z<GZTA?hXy<FGY$x^_Mh^Jv71o1?dOj%M&1H9>=hFfW<$qaaYPqQ&tvV(xc(7
zzh3jrUJ=Dc%XjLpn^V<6<Nq<{SHbqJ$Z1il<bJFX>+(kZ-9{C7T!iZ`(_C~ZRKv1O
zP$MSws%o@~x`yN1O<;d~s#pjP{|z@cY8K@!R@yZCjYe0Ks+vi|0`*X@9<@jH^EI5_
zMS$G_a$C9aRQUM+#Vv|py(+hLHMGNyYpBmm&{)Xjs~&IBp;mh&Ld_*%HC&(MbYDf#
z`G0Tk#ewZzVfO&67i8vL$Nd^l)gb57oc$xFsWa`S#;k`Y)m~{s>Zo*kZgt~Hd%^lZ
z>OpwUn<n+`HI`uWCr(JzWNTtXk&l-ZR@MAer#9jBUQHo>H5Flh$XS0rm)XJd{~$A5
z-a^hf0<k?3990r8Pgk5FZmbEK|9^K%T;uI2$o_}}QOi}<wTWwj#{XGMnAD!ATvl&5
ztDza<uBCY+B3OMUPnnwhS5bAYmFLxtPCTtK!;?$RgMGOQ*GmB<(D;9H<9@JTC$o5{
z9iFVJnQLpOUil|NbLC>lnS2|!+*FUra{;RXxdnvZPT>Kk8Qr6+z~XlbOf(j5{h)Rx
z(OF$--yQW8G6L$k8~&^Cc$X+fS*U^4fb?;29|4EMV}tc@F_5@m59BN+5c`E@kE%tP
zI#^x9jNO_wo~Kk-UGjpfS4iRmvx62a0;i#evotkj7eMx6GCh5-!Ri~Qc~ap&SPo>~
zcZ(Wu`UA0H7}oy>@jor9RWw`7r1bBZ5jehH?RgKj8zko<Sqe92S-6j;&5{VXT%GZL
zRpps3np0kgtNq_x3wD$0bQbj&yVii&lfErgF0f?LxW)EGBlEbkhTtz-b=Eik)MUL%
zG*_!%RgQW7RO48_oQ6;4N>$a8J?i^sH>$B$imA=#k%!v>asyM|eU($6&VkE*5T7k^
zp2~e6W3U?UE9&a%8!I&m-Z!iBbMFF+36>UWwo2~SSgahYX)kZ0vGD+p`hqR9RMkC9
z)Xd6us_AGiP|lQ5QUB6isjj9GqduQ+fyQiQNPYzM|KI(#(LBI#Q}YR%ms0h*Tur5&
zU(|okR8uy)5T#sq+gju1+aGG@r`Cbv-#A2BQ~$sswcE=Z)WsO@sJ{!ip}v08dbOUz
zGMX#*wQ58)SgW@lHUWp>s~*T1u!;dv8o^JmYHTW94R-H?6Yd%&@;d5s=FC)I&&IA<
zdEqQrT=V-BRd0iKMXfn8>Ls;J8k2KaHO#JascRG;*I2@7t66f&OznbExLT>Br@As*
zq^5e(D)rt^?clI~U~^EjI)5T~J%7-d>1xlbr8L!CmMDjxouRp=<hHt2^g#_KjTB9p
z@QdoJe;0uDgwK`*)6?p+G)kxEs%d@mP~ZA34V)&NUa)FD6#t_3@a;{_hUT9dd5d;}
z)q>&!g!eAx)&QOV@5BEPJg;>AG>eK=LaX{2j$I074d>N9+}@x*ar1Ze48bzxPyTkQ
zy!DVXxHJw5YVdTQRaD=fqH#zpTm4PMY-LB~m+Gw(7K6=q329c_a??vgK+r*LxisYN
zkXVir%8T}CgTts%#t2ORd!?t=dI@rtAjmz>tly|ED9uzWcz;pjc$pa34<PZ`QAKKg
z>(^+`O!};m!17IX21B7nlx4j7#TE(m<R8N7S5;lrcBLe%AN`=DeyU1SW4#At&7t3g
zeX5}O|7Pyvs()o5^?}^Vy<j)0l(wqyOi0vND$=8o$b#6L*M3i2<GU|pZ50Q5vHGiu
zB-Ow5@|s0$Y?`f7!eG5Tw*tU)p%9CDmm{~vt3S2s-~G8Yzs9Potg4k$$rEE%S^4uR
zIQ>X|w*aR(kUv&$+^%`imRpU@xLey%DHlBczxptX=7(<ys)@T!Se&_8q`F+hObb;1
zfBv{aHAh8QiQh#BtbgOJE-ja5%<%p{clQOQs}~o6`#>HhAqF8$wq~II|Fzj^787Rw
z0rz`2t*<FH?muo3aHm4eX{r^t&(rd7t696TlLct}|KGVIsy*H}lq^;<sMv4q(ad&s
z)e!n~P;=6+e=3|>H_WRyHJR>_oT_%YFy4Yy<)~4-NT;^ogE)&xkNA|!V-G6JPWY`d
zrKZG!p{dNM&_~lEwQ-5+%o!=h69NRa_RX0F8vj?|KfeOphXM8fKQCWw_I10OCexzB
zD%byqTgXJtQaPRbQcH_tu6pA4ROR3&k;)IPma6~HF*o0RHcMmsx>D8U;m1^i{s*hx
z_-Cf{bDe^6Sio<UCChs?;%{9Br=e_Cb4|Ar1q}~}pQ`>2A^KOCtW-WJ#u={Kqh~21
zy-MNoc0a}a9C6C8_-YJI4jflc-M`cV6c_(vCabJ8`K7wTfX6`oq^wEU{a&r;4~*K~
zulm7ZI<24z?Cxnh8PtAtb(tSq>S!s|bx)-=7_$EF!uNe9R=ulDmri6g^V!y-Y%*ho
zWq9WdaGaZ-3b636;Z%*-C=8B=o&#Mb1@EV5Ys{Riv2f=+l?{)U8$DybX0mW=pyomq
zcb(g3KElg-q0p)3Td%&>4EX-i!s&OR$&K|EI#*Xd)X2H_Na=IRBvt0@4Pf_!`~a>0
zl^$MQ4c7lEHOMT-lwUJff?dn8y~KQ<Gi2|rzmcG2rh2LAk0;KmHzv*oyU|ZV!(4yW
z79+?1*<gRNy=GBpl(?jF_unk@cK%+iUu|{f@&^rccCBYpWlDah1nU3yzqC^Mqq#w)
z$GQfz{!ek;LnUz9tXO#veE%=VP5&C#sI0ydt(<tKRC(pS5b*uK;q&|SV`8~fIp!vs
zSJcTWPU#NQu%6nb_N44FI8FLFn5jM3Rc-2;$zvWPR$$1pf1&0#R(`eK!+TXfYx!%u
zi*7MFAI_t!x3@|4^Ywk;I)!cBZ1DO&9WyzT4zGT*xn9nySJap-Dy;Q1Ub;4^^<|$j
zaJ@Sd9998pEGnS+e;wC2C7Y&$O7Gd#!S-73zOEy6%0eUQ;Ww?>%6yu!-m(^oYwA@~
zO1CRdpU?vCCkRYvQ=dKKrbYUd7Ue@6=fGnEAb)~z{?$^I;3RhN+Wf1Wr<89uL)NP+
zA1$^pPOw&<Z1@)(wji~eBAm6OAI52aNRCx5khIb)Q0_BdRdP?`3gbHEJ8Z{|mg@Ui
ze68Tq0qy^fc^e42|5vGeRv}zJ$c#1am%-&Uh;5Jp*%RDql&6)`q^_aARYddh4<=Pj
z?)y4yd^xHhHK6{#J<lYwm-+X#=Y8!4m#24wXQ&9J9x=UZ$Ez~aW*xZyzf#>x%|m3i
zeoy2#Ej#IjDp_w>)Y1$3&3x+STFm)&Ogp$gTlIoGvq7$Bib|@rzV@NbT-w_{?@}>j
znWkfrlC8RI*%bA@GYi4#YV+lTa?7)7)Klj^Qd%~l8!YxI_mFCW>vWx5hR>=??lY+r
zmEKdc{<%*x<6?_>>-I@1uG{u#uH{*2-jr#jbuTnPTXh?|#meb98m246G`F-%sk6<|
zQBS`#TZM<?v_;)tJqx9o?P?vB*G>1l_EcTxZ3e#ow>4#z?z|QQ(|RXC6^AM=%h%JY
z%s}=3!S7+J?Uu9kTi5T^c=hinIPIheRv6U1ex%mOE@CF&pro;TuAbU@t-q=&a&6`g
z;*7>e-b_+&kXBH=d*4pol~GPTVR4G`h8Jx*Tdg-*Y@Q{jrJAx%OF`VhqQ=chm3h+%
z^*Kd>=4rnmXW6CH7OH^m|1G>zuYUc{T%!%kvds=yg{aTsxvuj+Hr8BD_^I0N5;oO7
znRmw8@43vkwPdND(vz`RdOBNUn!_1T|6eY`t5W09H#_wwOY|*_+NY?x>{7LS{FcYE
zCqvUv)=|k^{1LxJuiF!S>7{2h;_TOI&D?$3!dFt?B5lcPl^6$p)&EC?&34StQLfvw
z#cT;@o~nj!uX#eTvBBF<j^H}yYV;Ox{bl037To{e=%A{4`RFa}xu)LAF6#W?JR=kt
zth(0eiN%GVf7LggXH~uSBuS<87_VxW3YXGTo%8Cfb6ga48Gq_c2-t59>i>T~o@0J_
zlCZ_Tm5Y=g$GEF5eJ5s?ux_@hnV<(a&GIc>q9k~DoqEHXOIkcPt-$t!#u(n*wy=0!
zsHF^A|F=6aRh@ahpW3A0GP7IPxGX=owreOIf2V05`b)hf^OTXuhuIbd3mQ~Doe{IR
zu+mAxHvWLx`b<u>RPzuur~HZPcG>D$N{dcvfad>84?j~4;02wd&!dpDu14j~Cm{<^
z|NqJg$lc}D91$umK?O!P_wZTP@?BOgP}`yk>i=I*RMo1Gg7o7SJ&V`4y3<nqn{|e%
zY`K)leLH?FOJ;5DITn3pZGPu8uBcBne|2c4h5R*j4GVT3%f&TYK>dFuQ2k%3=%=Pu
z<ET0@<1081fb!!g9u0xf5Eu=C(GVC7fzc2cSs~#3cQUxYip$q_TTjTXVto#7D}mIM
z*)XfA)R%+X4wpHY!R>jFI5x~8>i}-o!sIq^tE(MNTmsfRW%4D}yDEFsChICF2Y&EZ
zt@%-k)vPI(&Z%l2W<xOtW*&%6ytzpAv<R!pyaHjk8c%N{uvvfJaj3Ih5mvsL4!Mhr
zd9uFx(&v!71g8Fl+=Br!1BQLvA!{07d}MmgX+zZyKi`4f4idl2P^Qkic>~-|kl1yH
z8nC?8k@awKmQ2Xm)StIP_K}0s;lhG`Qeb!AxO@O?j_ylgaNNvO4+Hn3VfxO#Wrdp$
z6NAx~A{W7WVSE_<bC;;PYef{ePw4%W2SZJWzl6%J1Nvb5VS0XtLC)n5uwYh8dxE%g
zL1@EDMK2R^xY{c=k*dOX9o73X+7-Wj@&mgWWX4^wpQ?YI)8OhrVk@Q=tG>?-QtSEu
z1?(1(7z{5{y{ee%^Fg_Lu{+p|W7SK*{%*XZpk7x6xt}NQzJcob3wyxsUs?`13ml}M
z$uvsEcE2)sj`~uplA0S^np*QJ$Ufrk-Mwm7ldqt-1EdFrZ;Ob5?Sb+4{es+y2;#FV
zp90(U{b#spa;2d9#a1m9am5yJ`2RWo0mV$-$q0QQy&(J|)dcLO!v&sTb3kH87AdP#
zKe>ToCrBI@zE=R5s|3j#wTq~oo)85N9}pjeCtDs<2JQb-d}XNiWg_IR2#`2v{~w43
z;W;Zy)y~vF?zso?K{$)OQPtueq<;+JBjcIBHiGA*&sOYJ6WLd%ge(VQU)?ZGE;zdl
z?kA8K^8UXoxv1ytuXv`Tp4;f7dM(ii>~GgYtJS;?GlIhgw*N0(s20V%ypQj}YC&d$
z@afNmaDO{q+ON9Q+*~c~hN7ZIjk%gy!D98ahkS}h4d;XPSTVGM<7VEb!(cv0KQ=5~
zct=(6A|&1ansQn#NYYGwo!uh%K5>wqxtuSRTdpDQV^jTuC<k^PmItQ=kUDJmj+ddT
z+}$MAX*}0a)Xje12zFm}ae<=rYERYlo(*6=$c&=jkg`zaV++{7%q?e?p9Qv~=m)72
zoi|Nc=oMt$FNlu|uk2}5i`x58$;(g%C44>a_F%*-$PCx#V(Mv2SE#C8cm}p}!}jZN
zw}Irb;k*7NN{iDWcm9Fo(=#Dw#%%fqDMLVF==j;Y8>)L}$%EZgeIQ7|BnxtO2}g$k
zSPWe~GC$k4QB_bEa>tvFqy^ZWF!>l0$USf{K8U`ic2p^OA|wxTziL#ub{ul<97qls
zb29wJnO;^-4Z(2NPbbK|O2}q_*mErI!_^91y051Eh47h1)2b0?`Gd?r#ugH{!RDPV
zfQ;qB<PKh40QY;BKjdz!Yrlk55AT8Ofr6>~ShESNzwzI4aK46#hjt?FD}jk2)8-jQ
zvc8<ma64w}^r<*L-v|yP{(#?D(+^BP`uRWqt&XWMcfADbeJ4C!dG9lR)wm_Bin&78
z%1{4l!tH?R2hly83@SOFcv0L85{KbivV1B@;@=e?{G0%<Pn_mX1e@g|=&H)YBBD5B
zLN?gW#FGnE|NfT&r{$-9-C!}8IUss_pbLf=NF0Pe>$oUCcw!9J3*vXJS*MDA{!ewt
zY_R_16m598yxk6|!$9}{g7o3T)1pkl?tX1~5p0gmLx_D<S}Wn^f%F$h<lzht33td`
zGDsaV{&}uU)wLoDJcnnu8uPr2+~<##*7eK-+Yi$>Q#BXN|5n4M@;DGt2PIw=Q?Pe|
z-17rc`y;tQdH<(2)$2Py$Y-lG!NVwXQJf0s{$G$97>-=jqU?3)oC>z{f09@Jl?R>w
zvuXE4uo=#uuYmoXxqhPRbjJ_i_+{~6Qhqxf8i&2bV09p~rzEUZObh-24wsKYeJaJz
zeyHePfZQu9n{iPkZ5cZ_OhIbV@ynNI!TMn03kx9kBZBz%iy-Os!gnF%O<bC)*Tf~1
zj_SFC!{uPTDvFs8_wmEs1JVn^u=9Un-azsuNKE#_0VR)Q{PhM%9Sm#Fod<VcR-1;(
z`2&!<1Yz<&m(K>@|J#=9r4n}n5;h>Yts8fP!wkfp@V!B0vl0i2`<B$KRetp3ABrD9
zYC#xu{||`X`|h|(eB)biJq_Z+@YM~|WNOr>qUb&U3DJ&H6e<OW5lsE99mT4B)y>MV
z^MAr)MO9(v|G@75Wi&)RKWTHE1d92W&)<N%|E1$q<<;B&D>H@aD<lM*QPG|LRuy#q
zPiG@9*xs<{&0sn_eJhxTxdWZn_`65B?BO)98qFPYs-OOKtFA*k|F6CxUU6CVDY)IT
ziNSF8{=bmBl+n$F@uMC}DakZ1P@2xPA0CGwxk86!U^|!YIwK#nR!Vu%JIKA+AUTH<
zkb9J`ghKA2>wS1i@mXLyihhtf(EY#443ImrL3~_Tb!WE<==`5O5~$_dgPdRR^nt9G
z&8kUtzRXhPzII4GdWyFk>{eto=<GKUH|4oX&%)L1<=z8!uiM*Da2`fi58|)U=TW&Y
zmjI6Almh|^(W@cny!9sR0E>at;KI+}u2Z(%rVO?t@at6+e|&7lJhO6H!9=CvJ&^fG
zT~P*QYsU#-z3Arq&GUx)0bOo+5EDG!VRG_Zkh3dbd>H+-N*=D()K_1%W?Lv$`(Wxp
zwCCsfIK#^Q2Bc5RCGZs`Y?P}1!}SQ{8-m@wda|B!hnpUXI-l?-VE*rQTj6Kef$Yp`
zJ|gi?`@EJnzo1g6s)LT*OCj()v)ySf^#^|@sxh)US{|RfTmH#PKk%O6Co^v;zVBG7
zoc8`C*gx*ZQ?<=hu7T$R=d6~Ky;a_-c%$tB*bI{<A;Yl3^JY(f^C+f%d1kRMst-Je
z6Dc-Tv8g-L^6rcT4Yq4Vh9GyCF1v0X#=pek47-`4wp^e}hjp;LX<5B$@ShrOeE|b?
zPW@2uJRzr-v&C^13A2aJOH|#T9krOgj=^}V&LJI*Qcg>dIToQhN?#r?RCu@4)^egu
zoRR#ohZaVek&3TguQY!1X@hpfK6&u@KiN{t!EQc&Y_h`P@M~tv%GtFZmA+8;%gt!H
zuz!nU=lVI?XN8Wb^K=WSe+pQsW}-Dy?R3LLbGDBf>dF~G%DU{Sss@Qw;5OC4BU{w2
zTAo(WEe+R_zBmCqha~^5P3t`mx0>e7`Le}wRtC~viWL8g-7(zaxx$jGd5LljW45BR
zY?tb@wdzKf^tv@OkA4J)P4_2Lsk}60<tTGe!=UGPO`fY8XmfRJ(`+|S0MGkQS_3%`
zW9?!`g)?nj7Lzj;SZ>-XtypXiX>TU-oHgBAG0W`e=`~>Wr%#;$hYKidTqekY*-_EU
z!Snz5$=aqH``dMlHsq>L_;6N%@o|x{aEYhsiXTU`qB6>KKU`sir>%46BrSGsJ)t!t
z#L@Dfu8irICN;fwA3OE!nfv5=#RL=%m%jkJA7qdBbYI0ny&A>sc@E(AhI5y{c|-P9
zjeEzoXr={BvN$;ba^~cWH<K(EF|RcZ-aJKdZNf=#`us6P#$wB+24iKHSa5n1aE?;i
zcj>f}QR;H@>339g(kl%to`1>IU0B1V_NbFVm1)6JwJyua3R-NY3Xg0R!T$SNdk>t(
zH_ug2m7k}k@c(DD_}Z)<aC+VFltr~IOIi)I{*TeM#2}F0M``+QF|gY!)-`KP4Q^Bo
z4_ImlTK{+YPLN#hXEyUVkz?i`=ZG1#9h{}L>DWE_jsK@B=(c@UovdwdY9&^IQufSj
z3^cdzs*)<uu{Zt1dCFW@Y_0r0jqMiudezlGD$P?nar2^K*=k;}JButO)mE;Wt^BCl
zS@G(NFDhGi_JZ@mg3ru)w{PpKEj`Yoqh~xt?F-jAi=5zc6^2kw^;&Nguv--@6*Ugn
zHe2rReyAL6EDerpP}qa;tfR6@Q~YLt{o7jArBroB6C97V@|u=qHk)MYPd?R7{V4&i
z(;rQ<);`##qjRj@NhvnGO8eHL7K<slGqqF~y-+=CJ<E9Q#SIp}UoFt#`j)Mj$+!f3
z*6r@K3gEZ_+4X<tE^r)ww7($=;)B-zf#@f?v$dr(7HNu<yiohB&#J1=wnx`p3v!R$
z<7J#mD_-fEJ0u>}y;OZ4oL^@1O;MimNDAx^$=oBR(Lz<qS9(pA=4`rYVBc{NY@cFO
zn39#kF0-|#ge>P<Z`8ebw?^GJjNicJ4wvdgp>H|~+nsc-%AHY+(&p34E8~?tH}MR(
z3|=;IuH@f(UPYbbQ#JO#K&+{H%k@`HUd~FlTSY)|$$chO)%&SRT0I)tUtXU#pDDjk
z#ZvsM)^ZgI3*D7+I!}x3bVDlMSy%|AswHpRs41e$uk^C$D%kC3o(EV8bM6GkOGz20
z`GKxXg`9;Jx}f!cN6a$yqnDmDEjuEl6l~gW*?;RQ*pFPByOsBU+G${xxlwcHynWy>
zjLPjctZ~j%P5I?**3*7n^+YzL&)9uO(tJ){m$7o}R@EahGt@xy{~f<Y)eL_Jsz#Pc
z>2f^^w%lzSug$S}fp*O5hhTS<3R|g3-8g41@9qyC%V6tXt^iv9r+=|V&D(mWv6kjP
z^ABk|RCnpm*S%;}X1-JKxoSq&5e0UKIMY>s=a_eJ)G92SCT79FW2vFgb;+=`exh7`
zO`_%#)-Ls+w7p<E*e|$R#=GyZymsc2;m!;F=6mDpESw#640HFlYR$}7)QX)GU>S9t
z-Exaizr>^dJ&GbbH-Y_reRF`h?Sw|zEi*hV(x;~wW>v|7*RMDW%mA11ZhA@Jv^#O{
z7Ud4c-{8HfsUP-%?M!FyR=9gGz;a)!l#0;07?qGz5rs72=Zbzw?W%dx?9^J#J}D}M
z7aNp)>9e?az(xM>f;fxEZy7A?b%fM2**aAhB(5<FoSvfeuIjUV@8J%)h~%#dObb-i
z9^9U!J%3*dIK9l-)~~oM;E{z)Jrn5sAJwNDAomAfxyWa3@LboD`JJuiB(?~xj(4-w
zzM35}+Qs+?>^7y#-z*;buG7@syh5(X?2vr6gQ0Rq@KTj{7gg>5Um)}Ciqkc~V>+Pq
ze}5#ql~zk;TkLytUg=Eh8L%0kypymz#8_>8yk*7LXqDMFKB#$25l|BT*{&^e5OVM2
z4v7q~8FQnTnf~8=-Nb}-ispoPPMuJh?dJKXyS3_GZ8ZN`tZr#$a8!*aKEN_jOidDG
zE(j|%&QwekPFCA|G6dXK0Evy_(GVC7fzc2c4S~@R7!84u6#_jsAm<XR{#OFGW#@@O
z`eC<Z#ngJ<tx;s-jaAz?KS=#LM}r#Z{GaXnrNDjO33|uWu82U+w03dRLupHVGlaDB
zd8SWRb6>!xuDC5)Mc{jg#=#F}8hdUFDOZ09R7;%lO0~PmRjIa89Nd<9n!>DMev}pL
z2ILrM|2|i>!sim$&O2fU!S;Mjhyl0LK<cc1{#V!dlc4@-dZJoroieze^X6!`dby;6
zGVK1JcURaoc;uPXj?J`F&rycVH`TE~?zGUpp8@W_#hLnRxL4j(+p`=o208OQVh@PH
z{1r;oTg}z?D<y&5V(ZwbP;sA4y=4hxerw^JrRpia!@=&ant-^EVoTHmRpVKwl>AR$
zkl(K$rF<-ZDp)_r?a27+&jpH|eKz1e+0x4!6=NCs)F&O@tTt6$Luu#zZD2izI}!5$
zFArW*Q;EK$<S<EBZR_rOusq1TZ8JWB!vMtAj;mKX`CUxCMub%@q-CeF#hF&Hx;=Tx
z;PB(pPgPy@46+x$J5E6T;LLm#)7y?<IgtJw(NAFY?{>~rJnxeQW`o2xE#g+`KJ5eM
zOWpjhX2{y97S9Gbzpp<Ta&|v+;7t{!*jwtH7xLil0qOl_utVk2vQyyu|K6ngs#acQ
z)HwS84me&x@+)6Ss#bBL>Iqu@TRmnMA6(6RDM*@W3xR}3P+OC#O67UAvb;O0<+CR%
zI{y?`&zrJTW#jD<xL?-ULE7CQJ7742I|M9WRHLJs5MK;;=MuRFu=wqD%1S5AZmIfe
zJW^3;<5M@TidNru!$eKxNdP#_e(qCOpMDOq<^yEriF2kZZ6bTYdO`f|79F)#177tR
zAKt3M<}|NbyQm6De^Kjc+6cFAmQ*-+?8A)R51baK$?>Y)IjXL_D_%+caL;|!WqB@O
z`#|^qJvnCq4tJ3G*zlDD&y_YFUkX+aldrq5PF+YtM6vS2I(5#yQEK1+WP$rvp!5H&
z?@tH2ccXw1SdaNuRk-;aNAIZai8EHcZ-Uq>+%o?V*sUP5+1Rn)7X^|tKJO1V2gLtT
zWvBG%$vU+|TKB>JmW;TkX2X3KY$r$_gcZ-lD;p~cs<1tr57s;XP^jv;9>^II$Aq`5
z9M{Ozh|FFH)<0kMiW=|pc9pjsKh*zmR;xT;G+kp27o&z^{zcUjMb_#+R$W&~{a&Xc
zYNP-*3uHbB2hV}*TLQ5`ST6jH^311UV0WaNc&ewEJW^+l|E<biWDU;Cld|@zK3%#+
zZE_fcdWE&3I<t9?!bdsC-S*tG{M5f{c7ek(qTXLwYh#u|T(F4x28(#Ool~sltLjVm
zseY{bue931S4BO+QW<pq56Es1p2^;*#wg0K20H)W;O0h^a=tuOMK5c02j7WmCLCHC
z``_uS-kS%B^Hhh~sx0b|wLu`gAiVCwDzIG@*6YCZeGVb@rn(OGy+*s#b|0OfTC>zl
zBjeaB#j`UZYgLXy@4ef!VzS!bgGW_;I$tUrvef{m0dwwaYPEmoC@N2ytD(A0M&pkC
zJ#e^q=lloLAp6YEolyyntp|rqn#Cej-TZTEY5aTC{yy5LBHVXc?bbWUJoAfIL$G^-
z!j;uQ`~RaCZGoFHBUVzaoy%3-J!^~FVb9O1{h>G1LF50C8}@?p_`%OjVDmxt!LWm<
z4mjO?koE_g!*u4YTCG!za!Z<ws^wP5eh#PWko6(Eii*H?KRT?Zl6z1=bw?Uxzp#yo
zh??&6a@C@lZEDX-AZx4s7YC@Euv-N-8)nyXW?!&=7#~EZUOcCIeYw9zuEi3t{U9+A
zE@^@Aby%#_lszY?e@~EAnY(I>s>cBn_2phIYR~di)sB|6sOR(lg`4?wJtLTXZ~?2D
zaLz3hyFusw9nh>(+wtiMiakGEA$xK`cD8Qn1IL%w(V6OBQ^Ub*kodHd>~KBPTs@Q|
zoek6*PWvluJq)>*26X<<b-r0}^&mar3s$S773|eWesod&sE2@7lHXVGSq!HG(zIl4
z64h3m@i*t+wLz_mZLh{n6JZsta>$w=vu|?hi;W~yjIB~N_O3hyKK~D7UU0mR%I6GD
zllaNcl>_HY*OR^AZ34RgcjNqDX4gDh%)TAksGPyFSJhv?(0r%=T&2DG`@m<O-gt4&
zv{Y}E*}7c`@=aE!)fYwHQGT5fpzO7yS?k=Xdzz-L3$(6y%c$REE;pMN#%yA6L0{vR
z470hoCY#~q#m-u+zs1cro}Hi~CAv=K@)m1#6%}D~>Bl_=3=&(+-oC9;FM2n}aLfOn
zTD<3OYJ%?n-6blboltX3A@<{C1)taFP1aAb)nKaYRAX=1Z|1h`m}=;&%bHz<EgFp3
zRVoY?pH-ga8fd<qInDIW<L8>QZZ1*Nx;I66L1~}byI^t9`G3ksQ(V-geXKRi%>*@_
z3@6F{w#ZjM=~1J}a$i-QTRdHN!IDtb34y!yzv^5tj}j4<*LeICeE#nysb2lKZ53+q
zGnmcGuc^y_a$KOgtyfL`(gk-t4VP-8)1kN3Z3Jbt8fVyo&;L#6ng^b@c$*)u?yvL1
z)Z)Q4^V)weRl}ZZgX8s!qMEVXP6Ly!j~yoM+r`z+Y!tPKXN}fikA0|e+KkB@bpLOC
z!EV!yKi4bqt=?-?$)B#}DEm%x#=^~Nl5vv_llg0nPt?9wU-MK`yR3I5X#c;`qM%*M
zO_>H}k@wbX*8FHT&*Z;nG~F~oyR9optEaO_?f%Q-s*Q6mg2(n&`#)7o6l?<D|Lc;T
zqwe7(s(kb7A(KxZa@Ee~Owjt)=Wo{OVx<9^|6jeh(!B1Kyh+5S4{HA(JW)@sN|f`g
zyJp(u)nYjL(g}^%9lnZg-&kZL{?1T8QS-;N(?mp*VHJznhnOo`BJIzWrke98nzy`D
z@wZo1?XHeiWA^!G2HO9B^4bP9)~{WvQAKam84HC}{{M^?S#;u$n)~q$YGM*zN?n=!
zYMN`kRSlP})@?bXsdh`s-%MQNmSU{k8;#us)*5>hOTp*=&Ns1B*ScC^Qm&I?$|-YH
zUwYqkP2Xit)Mk4+tLtS=*KCd2X%rsdq2{{kyP9IxYn7SD6;(B#yO_>u?-NV<aNLM-
zeu?S2xH$E$WszoYoF{6AZJDDQ-@>D>w1M5+L*TFS&)hYNPvW?gj_oi}y|AHNp}Vvd
zoENnD&2;9aC1|<@wP{|SB(4>fQ)srQhee$w-AmP#IoNddGY+K}ZS1P-CXVKDcMX&t
zeBWm5wg7T3SBkg2@*G|*`2D}T^3JLLsNqr9$`X@bb(q6^NwBC|=Bj$|csi*5zvLgK
zZDn7qIde0k`t^Dq4b59uO?^Y&Xzr2zt5CWj(~vdel-cb%er>LA+2HekfAvmM+3UOm
zoCf!Oo&c_kUzoQmf3&|LwD8CpdD!{CbKXwV{#<RY{&bIqR_QMt<yI3l?Xc<kYKbTQ
zDV@JPOZ9q-oT;?V9j%p5veZHM|2`?LQWHopFlh^NSM$7`YSLoes?g}vrEcWOtrvPR
zSL>VCbk#5Mq8c4>Y9^8qU(HT^i_~JbH&s^=oupf~NI=yobcI%<Ub`0eJ$u!qJN{_#
zmW!+Fu{_ks4;9e>jsI`Hd{AEL2$x34?XxOV`ylni28YvXrX8(XHSX!4^Zylcp8i&6
zu3w||R&TQ@pYj@YUFR3-9=;Du4_;=_{wKOo+vnJEGi{z-nykh=>PH*uHCFt4rslJ#
zT~)Kg!@OqeKC^$m(i*!aT{XEsd7Em$FKaE({lBLF!gcI|q>PPMW~(Ndr<flxS#6Ru
z%~R#qi6v@m-|y@GQV`cX!!|`#{8^h4%i`I37L!6WWVZh{`M++uW<l*yb%m>@>becH
zOw(_f8Wyj5u5nf_QEi2Ii)!4#CE)Xa=kD35%_uEyzVo4#mYbKSrWETDvsH!4>edT<
zG%x1=HVv)Yt$weyTzQ&?i`tw`xvEDGh-=#9+Zl@TmYH(=y`*s?MP6q|>wDAQ=bjqr
z7X51N>z^9+ckM9kbKj_b!=%k@(Uv6D$v@QfC(l(@;B1rE&_1?QT_E9z*|pczYLyWM
z<`2X7n6Lk}T>qnaifPy8$z~Jv=Ih;^`%caBs<*a!Y`ppLw@=LaCrnV=u|QFMeR3!G
z{@=F5Nv01hZm7AZ+MDUl3e|hEOvMOP|NnZk16=Pno)FXko&V>R!me()dXHAXLI+hY
zzcjN?t8Ep_@_wk@5y&tPxL2qtEXJtzZ0au+Pw#_jk4|k>S|e?w`9s=5x$3H|Zjh6T
z*~VR;lozkxZ@RR^!|c|fH7Y#U=c+jyEHmL?@KakWdsGcH{{KcyQCYb{QuBq^aSi=%
z^TG2EY1h=%{y3j7Q$4UBwEs_0*Hu_kXluQ?<sK7LhxtC{bKGaD&34t**y5d{X~w?W
zpnol|xk#Uon(n=&W?RbsYdB9}(U|`^Up<z0gZc`FGa9?LIBNJkchfr4@1s0#_XM^4
zo9b#aZz(7S_`X*C?Y7g*?p3B*(UYBKe%arZ1Y=uO>)t3Dx@P8>i{wbD7N~7e>Ur+3
za9eP*me*b>O|w%i3RkmPwbXO;G?LQK8GBA^GFr0bnAWs22Q=AA|C`KBtygae_+ZK?
zt7QJ)f1zfd>U{Iopm5bXg*z(K=E|vtFt1g2U)!v*fkz!2)}!nHK&=)C9$o)8y8aK8
z5=PhmF))m-{{y822p(PkH@f~0loE#H`ahA#01b=3eVRrGcs0ERZfZoV<pZC^z`4mn
zbH<jtDzA3hnV)Km)aXu_t_eE-=gfi(_4_tH>PwkUsVlf9D<4SOqG_sK3OfHsF}QJ)
z0%-l8(U(pWpSMR<_GF0Z8}4N?{Wn=c^|64Ixq=g~nR1zv`rn`yb=$55=KnlTs!yLN
ztUved1697|J524*-7))gcA;GDfqwO5xfI2{E3YV-89QkUrcc%Ioa3z(!6>hu;hboe
zD)QNwvFN?3Xs52ZNo|v1Z15{huL<Gi(JXtx=l_MKEY(PPqh{XH{Kmkb=#bgVNurwC
z3pI^YvOTn$AFc<T|0C~N(4zIuJ6`F|u6gp+t7T0S8joli%C1(wbNaOz=={H94_md4
z7&&NI@#w3CUY;vp|H)9j{}H>Hd*XBr(EWb}nK~N0|KF-BTzam0@W>Wbp@an*!Hvb5
z`-&9QCArqCyw}OsxVW@R!?L(Wy~3nMZ$ew7%Eos}1`8t;Es8H}Q!wJRkmuE2A+Mgp
zX27#0SL2$6y7|)-W;M|Hf6OZL)E~{9qF4P*$#{)ljK<TBZp|!rB@@-j5-O_BPTFVR
zvZ@4rn4!M4HOeeH=ZpC|O$YT%D=+Yxx5wqLjhBR%n`{#ZHMx90Ld`hN(c(ghoJQQ=
z8|q#vx6MH3|5r}AXSz_QM!9yAlJNtLi5lmeT{HvkzEt-+RA%_8c$%?eZN0i`;S$Y%
zi_U@1|6jIrrdoAezFB#ApXMduUUSRrhemR159xIAd1<=oIcV(f)l~1f6K~#^(XN!1
zCZM6?^<3%C%W0|yZ5cIJsi>=!WT%_X`qZxeO5l;^RgE~a#nH@~n&nzbjdE?~LX)~o
z9>#oDn`rz4bpM}Hujmg`&z+MDcedB5doFfTJ@SV|uIZAVCSx+M*=K?E8r#J0m@e2m
zT^rQ@|Cq!KZo^hh7FLr9Sf^fayajyzzmb`?MxNjewUhdn!RP<!h2^UYh0AHIT~x1D
zzT~~a+>Wc@^Z$fuX6xA}tyS7mn`4%Ff1zR}gRzF6*izMx$7boP9bi=Ebmdfgw@%h{
z{vru8Hhy`7nmt@v#g87VEq>pm5#VpDalBs3*dQQTeSzi)RgYFL6|QUX>N>`rrV}F+
zB*Nq!jh_lAnnj#8*O;I1-0Yuwv&OGQ4-`+X-lFf^`rquIe66an-xu)xKli`fP_=0~
zsJK_%1$_R0_3v{!&Qo5iuN8LHoG#3!dGI5L`RsrmHPHEg=J%V-<Uc!UIOiv+Mg1u;
zm#J8z6#w?1vETLvsouc1YN~y+)sIhG1G@iDjz=R{x#sLDwJFKxH2!C(sjrRFH@CbU
zr=AkJUi;{!h0?p{8LH`K9o9O#*hMoVkyGProu5X*<|MPv@hw`H-*PISc))46gny6O
z+<))2rXB24ke{^+eEz>w^<*vO{b};K!fxuHa(HE6=l{)CU#nKWnoqghE<^e7gA3Yf
z3-@aXhH+?3>FiRrXKd6yIb)J~+A%&=(E0yIDvC_E&T7>vbUdkX>Fz_77vhK2rqwo?
z?Ax28cIxg-Q2$^4?!`iN=Oa7x3}Y{7^<N58QMK>We8h9b#OGp~`OMm7T4&$osQ)zY
z*Sl4qpzb90N&8N=pq7Kta<#SJu4)$)NULwY#G~$Vut$A@v!N{L{NHSSb&Xf{L26l-
zR)ELK@<nf`IyP+6y5b!LI{#n6ac!N(5e6k~754_Svi&KVF>Y*{6=A$)QpZkdv$^o=
zy!|d~&V4UjBXU)cMl`dUh9_gD#?5)9Du)<6%|-XUG0QL8p?>9AuBn|DyL$1?Gg_ea
zfAT_tx|3f1Gk&v6Th+&BlljpZiKd|Qe@-2BRTKN(tarQXpyr9=Thu#yj;I|d4A9>%
z>7t%`Ld$f<o&6e)B>|de{%ltBGY~MFGb73HN97jvn$qR!p!<KGID4z<Y>QR#>o(Gn
zmu)i-XGzz3Q=F@nIem`V(Vl9Jik)vY&U9*<>8WdL9$Uk$zG;%b+HvEjYFaGC8p~9s
z8djT`m??Y;)_CQ8NJmGT)lB$tlKNEtP3q6rx*Mm|e==QeBBtrle9CNRY>IkJ7!&CH
zANj@;J{qSiWYvRSFE{(yFh?b9{W5b;zuV@z7X=JrXU;T@UcqU8HFT!lvftO#nm#Yl
z;=WyKZkZrqe!pg(T9?;WjbkczOhD)VFL3;1`s#|3hQ}rsGg*-<di+kkM$){y6g8)s
zfcxM2$NIqM|4$HGsv^(lqj{HahDvu4llfWCKt=BxiW;eAJIvS4kJEg%K|nQQ&l^>j
z6;0}*Yz3;Cj(nOc72hd6tE<*?S9LJ^BKBORtRu|q)9EHN-s)#6zVp|sOJ}v2L`M6o
zXTLVo2-$d5aY55H)w#>IsUNw$NQ322pN56dQ&l-RU$sy%FLTiSKcMq}^3TbuU$p$B
zCa~(GNwC*?bIFI()#r72Y35CxuilmyU?`9$Yc3wCtj?{$YJR6+m4>~)r+RWwp~kDH
zo7IhcUaBipNog+J%>ugrNBM1$nQE`nM>Wv;zt@M_)b5-V1)u-_dfNrF1=CAaLFfNv
z9N;r7yuHV~D)YNi>1r?V{6P5w9qrhF9*s*})6_d!rfY~c#;M)w`eA%zahK7e1sPgu
zR$sNkOzxR}QDN11Tk3Ciqc`1LTluhtyyGA9FeV=5%7ty}p!0u%)Qr^CY8+MS_XvXL
zAVBRJWNai@2+kW*JvPZlDViz&a})*J&*r}qJU6NI9Wu^{tY`UmTkyO!vKWZ%&(EN;
zt|Aj`-t8&usu$A`cO}~tn}W>^*XaVwp9tVnl$o;+Je~;B3&RahPN*Ce+6PusB2c1w
z>gykc4!#1gUXYp!PQ+fV$+zBu-3St2#|*h69>fM=xfw^5PcK^_cXQ!cuvs86Wc<Pv
zvR|U_bPsqA3M5x*_6Tkdh<}HlUrF#M58Mur7$N*Pyh?d@y9YRYL=TlK$vEA^YX5={
zRVd<@mi>X-eXHL@wY4S^JkJQTuX^%6B@GWV<!igDP{NGS3Ub~#NZlkpFO?ZFCMf2D
z#4ntOoQJS-A;ixhF%Vw=#8~-+w;0_2ATbzTYB?Wn&XqGU7-3)-h?w()>4DMnJ2~Ng
ziH@^SUcYS<Siiy|PgNhzDH!$zw_@J24KoWLEny>slBS~<zd{Kcm|jao$lm0cCxXE4
ziZIv$rs48mEx~+8$$q7B%XunYwJ~Zl%Dh$2{@DQzgT)mMV0m}{GB6*e4@4UlL-xjk
z*pqWy!Rpi|#Hw(Ki-6T0D_^KIeU=`0y*Ee=GPbD~1&77vwRvjn9{ON*<^D5D|2`n@
z{=Od_2Np-xM=YCf6J-D20UgMmMr``zeGupSPI6^Y3ov$4t=!OpH5{?&hl%yC4gmWd
z#z&`1&5OZl09_o!U-8cjY+qKZl=8m&kT5>nQwC1|XIr*_<>Is^fz8ifaT^@2*Cw*5
zwn#ztct!}$P{}%aMcHe^2XMXvnMn*j=b5K?=u@#OqX*)=4W^C3YE4((E4`FeQPtia
zsUR_N8rWW#dAhOF!S1>BeYwh%I!m=6P7N?0CJ&-7ZxvCqNUE31a=oasbv|U@KS+GG
z<s~I~B}CY+=lY{^=6$tNkbRo6AI~P0JCc@QKd)i70^50E$7wJP(hI}+C+8~f*l|Jm
zv#&Ea9QK{R09GTaaY#vKD`Xu#Of86ZX3$q{Tz*>7a?&al!+-OY+T9l`J9N)cy|E2)
zhoh)us?vXc5wIU3)BV9|ZHdJdc)l`T{#$vP8Z+3A4^^V7Dmsuda<<xZ6;-!`ss*0R
zV7<RGxxns`ypGrp{9RHK>>uHukUg5S?=Y#IWmZ&CDM$g^19BfOtfJEo)(ewws&T{6
zBfJK(Rz&pH9t=5{zK$C{V0XdzAbQqUJFtJc4JU$WkQh2PRJfu1fH_1(D>n(ep8_Vo
za`{bFy(67qKmSxauDEa|pGu9Rl*-#p-xUvEx~F(K39|pYI9^@l?RpNdI+$J%Jv(7P
zhMitvYATM`%Td&-KS7)g5Nj0-_Mh%=XO$~AA?tTQW}P#tKnbHP)t@R8d%G1<&6X*G
z&i@Cgf#K(0GU0hc!9!Qm?!j91qgFRGCW@{B&;L8jWzzVxeW$Xbkh@v@Mrrk9(^54p
zcFtDF>eK><-{Iwu^K+8pS7}5l7=z~n=d6~Kzg6C<a5X~2MEIwy^7(o5^%fZNnD|P&
zD{tbpGGj~O1n&#<y3wH4@qz(#|BuSKH$~t%&xtPWCf!*Ap!<L2Vmx@k>lg&(B$Xbi
zZqPE8EmzNbp{*$?-KAQYH`VlLt2C(puOgw+Z`RfS(m;VFKy!WVY%@c>25`8XK2@&X
z`#Rbz$y&(Z-~t}AIY(}&Tl2UZ3ODT5n%)zr4!Zv@z*1WiH2<G9W2xM|=YA%a)v7g|
zm1We<v^at1vBf_qYY9C6ruI_cjA|Blgt9HaxY|E$bJNxtC%|qyu~S#EwsW@HZjW~=
zu1}|{N~gQ1E1!?oFqLou=aub^Y#N(xZB-B9X;j^zuL1U>uGcmF#Zwy0mtRy=jBv7*
z1NHy;mx$|ocZz5fR<wid1;r0&|DWIw2fbNGHXF%Ft<uQZXQbKKXKTz_ZKXWn(_wH}
zyG><LOTKZ=)F{}_T;~hB+UX^ez<xKr$7)>UG2htojKA?tiE`CEyJ_Ye@8*KtB+GCF
z9R3kA_)Xs!^eaAz5jW!6TCBOkZK=lTYfWn1Uuz8tkDoAlyF*)DYE!djdq5RfAIMFu
zp$(=jpF}ik58pN0b&SpEe3-6|QmVLyE5DKYy|;%|qgXD%-7h)aR|$0g?*+bN;IKN|
z+-)*VeU|$B*?%;CdAFNZctX|!-8x=iX69^QQejx9dTZ}`aN50V$7os-ven?gWKXbO
zXJ%H}n>Ws=d#W~?y7y1iD0wAhD$DT?JpaGC7jpma@dHbhJ+~*REt?OX|5xr--wk&6
zMlODES_QfP-rogav4fc}!1w=x#IkP%f#YS?&PS$ImnSGjJ?2o~Zq~07yYh?PnN?|u
zsRg3S(+(~)agDAqjbp0Pf0*K~*=6mj`f_TZ`V_xO>bVQDjP}2}q?+@JRb`WTyW*dZ
zHp-Uw%uPV^|Enx-86CLqZ7NeEqHdg$WF{lur7r!rLHS#9iT<TV5pY<6`u{xtO66}_
znSuNN4%rOg{(pvCrA|Stt9sS!7!5Y<XX+bxjLfEUtXF-dut9kO^ILEnE^G`{OW9Uu
zmND_9($ALd;5-ZR69~s8cBr5Ja1}frbTCIlZQs^q;IOpYt#9U)|4UVlVY?Qn{|{1I
zaqE+2CHoKc%Y5_I6gsD=AMo8~n#y(`bpMZ1LpPT}K;u2rxWzNI9xK=@#JmlZI3XmY
zT7Q3oCP*&`gZlp<8iqeDs#P*u%%psGy|{M4##oIB|JXH~*<F;+UG3B=5reE9e!PrR
zar*lsCckD1X}L%EgU4_twA!hy?Ym^W^rVt%#JdD=|G(Yfjq2Sw6?&Cnv$VDzTcUWN
zN<t%FHO|<$?X=l?el;zVA0ny;PWS8H%TQE(m)ENuVkWP-^@*_R>3xCPP8Id4igMy=
z?UBZ+6PyjD^SpFqo%qGnk8nWhC6IfM>UpSSI=t7sr2j*0?t3kT6s<<J{^<c)U2D&q
zN*=J)@bY`0abH}^^wTC^ZRwH}ZO((U&6HkFQ5Sl)N`0Pkn|iC8r~0C!Gn7{Um|>Rv
zT-7Xp!(r7;TYj4yb6BEk`?V9CPw#Rq*Xd!|Xq0p;PU(VxzWD)(Xp?L9R~3@(Z&SJ$
zq^TFmld17UcMUjgdadQsKhJbk^<3%;6YbN7)SvCTtllUrt7`5$)3kp=n!)XJTT~~N
zJF0>D|4b2(b-Hb=v$VhEJvA#2zpu$WTSKexL6d2ar;@t2m%YYr=kKQTd>(`2eyJZH
zxIQ^Bp<2Dgew9HN+Yi&*`**7Izc<ry@oF-aI=oy>Unp2jXw^)klNQ>h^A{aaFYa|S
zd!l5iej{PJKIr^^uC;s`m(u2`dIxZU)9<`#Eat%+@6C#@Md{m1+L(ky^_yj1W71PI
zyrY(HlAzg@yWY%{$J0!DO}gsM<x|unR4u{#{2qRuW_qz_zuIS?#ikytQ}h<_@PX5>
z-AM^>S~&2}15EEu{iU4vRbNv|#z$UJw9@Ph$1eF}%MYq?dN7%L?Tyt4=2)qG{~Djl
zqq(|jDmn(r?+T<ej_uV_oPV1`?`x2Psjtv6Wh<M_rpmcTOr@K9lvkbKqnZ=!W1PVE
zU)AjdtNPE|k_tixnN*iPD^ioU;?+nufZUa!zvH^H@m9qBzpLsT)m)}VsTNBdHlDft
zm)UF!W3^L?N}6vzv#YPIPcxX(XJnSibWru+TN^X>Og4>3^{uM@!h6&j<LcF-&l##c
z|6HWOCjJn7{_pqh>B_5qLG~G*5Uo`CdSi~+zBlI;7hSsp&eyh=wUq8n3o=+^r)ciw
zsiB~<j|Y7J-@Ef^S~Jsbg5$FLz%ljKb5ROA9w;0C4(~AJ&v(&O>|Uq!xBj$Ay4VbL
zp0op|N1~USxdxn5_b7}tk4Rmw1iJrs*{S7<^TmQyC;wTkj9mY(|0JaOefc@nU)CSA
zKG`dS$NwLQ%4w;@cd4wjSZyA-<g7}DhbXB2S9Hz&udH;sN%0n&CAe(;A2d}n+pGe-
zo(Pnu!scYDthpj#67N~7)Hc~q|E!W2`2N2c)1%CPd^iP8H)-))%9AQrm@hTGt9tRk
zWN?|c_Oz>+-#k9^GwfyxbK9P)|68+2dGE?A%C|n+YjGqW*E~MeO6ydzwX$UKVY5Z$
zRwg&+dugboTAJq_PB+|OKT(U1Gtzvg!BHiitF|h)`9)N_#XHTD99RtJ+-ERPTwJ2M
zfYZq+xMr_b*N@|%^?&j~k9LCdC+PhD+55#zzu)K7*q`a7qAxFRe%Mb|^}{Sbt=&3P
z)TFH?Rd#=mP!@Kw(#+5OZz{|DNOQ50n3@qAn`(Rjzv_>KU~oSsRViO}@~Y_?^ZVq%
zWn*e(z8cHwUd;-f1eI0Ej(V-Do~p1sw>AjS(YCnADWkA`y^sR0_6o&woZ1E{zptyi
zm3o5r<b&@2D?00^&cDuDKTnF$_{-cqnq|$;H4E97oA`!$DqP&H26ngBSy@%fa|UMp
zwjvhx%ZruUr6A{taB{9S*6z$VnQDE|BvDaVnN3R1Lgiv2IIiX&2{OO3>X@p@d&oZP
z`=SesWq&TwI<tpG>)_u#D%#V28m=+lZ2W<#NMp<V4cf0la^Y=L|NEQG8kuHlK3a9l
z{ON8NV{L|c+De}cv??qmRP=A$QRcjQ8XUKvG%&Gyfimd+zp3@bV0#y`WSBnv_(Z+u
znYdPj%XYJJzfa)uPU7MZ^DIjZlaHzEmG2*c+`)0KOW&+NNXF=w`e|@D_Pt?|FAHl{
zegEo->2ih#nl|NWX4~18YDGJ5R9?x~s|Y&(Uzh!na%<Eom3Im@;4t(R*a~*{$=0K4
za~tP^+eLmSzp6YA<5xXZd{yCz${J<4@T)544f^!{*LJJk4RkV-Dyxt`zpF&!*Lr7-
zxuTMw`+pVlPDHAzJF1$fyuW68Y+JIy>+}@Ob2n^N_uP_DbqU*|d2DW=vHP1Y6{)~w
zsudesRlf5-R4Dqq2b_<qqplgQT7ALvW9u{(^Y1-on?ILpw(gQp-Bd1MFxjCP?9T0n
zwt&a~m$Cg<ys)8Mu}(Y@oEHk~<8_{dq-b(wzShj{V9;Wjdcy3AH=p`s?v2WaGM9kO
z|00;KX2kT|e34MG(w@G%;5-Wodk}sarKAcw|KDZPZ&l6iRB$}hz0EdHJR_mP9wrUX
zuQR#LwM84swYpfJsf0Mr(X3!$FuQQ+iDtBAt8%jNPQxf2YjdHd0QmYpkomr1S>QAX
zVuSEEISFvx0%A9@zW|N@E8KQ*)}E>|N8QGiMa%Lilgjsa9i1Z_ki9jJmvO3on&f0^
z6|hsg?BPjpnZ~|#wOZ(>J`={aONy72i%o9oTu`WKov6Z_bY3rRO0rfRccF4_<OkKY
zd3L6fOw#5X+?Q%+I{sFE`}3h*-<{>k^S4dbdN#jLOG!#VdDrA1EepvYl{U}a8j;m)
zV81_j!7P6=rA>Wa%tNJ=+Qne89jw1px9d#QR&eQ3-8OBZa$%*B>gRKgT6H<`W>1>0
zs?PW_MML#xtl73FtF=~7uhn*OdS#|;eL%Cte6RY!yj~5***$70Ia`(Y$W1UWWRx-2
zOkAMyyG6~kldDAKcZ0U}G~qKU%&*yXJw92ReC#MtH7Px1{$NX}={5VSiiRmO)O2{a
z>wVp_SVO?F6`V%Dl=kQ!KmK3cO6s!7f|O3pciwMR*R8swveaXrsbBS7!v`#P)w7E?
zf$slPKYV+W#`0rwN-|IPXlwSgn2VfO(AsW$SM$4-i@B#-fa<Xg%9{UF7MmVUy#@B`
z94CGi)9L?JLifMcINSfz@TEtt>8HihH9l%;>BRcjnfWKIRWGg$Q2F1r&&Z{k*X%*d
zOI6UFEDd#>>c2kU(a)NztZ-HOmU>v<dA0aYkIZ`SeNkDtYnA!k`$iVKw>cWHT?sNf
zBl*f~ElZfbpbNi7<h8$AJ63Hrzw^T1{QSQiN-AnrswOU{Ozx%qRIV|bV!FN9ST#mz
zx|#4&C;dXk9B}>bShp2i?*|8QgUg3kZXD`dMXK5bY$nQQjShp$7qf=*D&0Bz%;mDp
zG+lmasBDh4Rz4His4|h^sp59GlNt>CdP+fDdvs1!FERt2|DSYU)GXX|rkV4eZsjF+
zm{n%|ax}ev?u5z&gO%Vig3+-_;p?nejppAQHLpKe1uk>eTyImE@2(2E|5xQMv!7C-
ztGMR5fK-+96?;wVX09_g^fFgB@_wliS(T`HFRsS0Cv=(l@@2v*YuH)M_hzwb{&k<K
zDr{Py+8Eib_E6`nx?w!4<`Ov$t;0R+3N>#8)yxlAsfaVJRrq{XQn_l&OS4sznbls$
zxq-_B+sj(Y_j6hebB;_jm%DocbpIc?{+Dqo))HF{xl`-lbr!9OZ)d5k2w^elQWP_8
z)NRq?j)~QpzNyUg_k>oBjcLkery6IO7bH*COs?E#e*U;CIDIWUwOq+->lzi%{eSIW
z^}+eBPdHXW>daKl6XiaNH+Ejvir&-&Zll#6U{*V8dP!Ar`e*Y4C2dO07nH$yzMQLF
z$zp}6B6q1GxZZQ{wb4p_$_thQo&N(W19+G7gUhIZ%`@~PRVSK)&i@ggwBI~3bvoFO
zdk=1v=P@j^cr&S2MSFu0_|B7__m|Dq>})ZYwz&fymk2Y=ln2fKAIn><)zM_4)ZKMR
zL!94KvAo9CTqe^5Y<|k=mFDcaU50EqJG5B~*x=_Ue4bLNaM`2QB3fa+!P#Tt7H5JB
zly<(lX!tYfo!0fLF4eldxr)tAOkg+bHR~!$F&{FW^Hp0@FXXji-XaCCIZwHKwdQ50
zsR=|ns-0f^8Jvgg+1$;}H9^KsYZV!ks$b6oi~n4wptkA6Ql*=}mT8)OYXpZY+q*Wf
z`HW4hN)PLw>w&_hb*-s^?2|SNH7$ANH(9Nq{=Z^M`DKHI<Nlg{bMAt}2BZ#z&rImo
z7d^~vGFkJ6rpL0+sug*WU_UeL`2;rCaQ#gA+*5Mq>yNiv+?gq;#4rJ}&#Cekzp2}^
zv!>sIZ-LccXqpZ756Dc*cdizNzt<}s*<B0HOW(>L8Y?^6X*;RERn44zMeep`o{?PB
zXXEfWd77Ckb#yA-5#{=1r!!^;8v8US?U%5a->h!3a-+BI`n~pQT&EKhxmW8c^2<Zc
zr2*L!_V=JT==`4}b*<ofVS`hM8N>G(YRO_6nyTrG%#(ku1do6BglAbWe(W}>XqyU-
zbCBL2S}x`u7i^45{zKyD#f;PPZR;JB3RQ2J9h$?W?XjfM{Ji@Wov+qS%By=#l*H@a
zslK_eK~dpIy+X#DSg>Dn=N^*=nbne+1J2_CmEXloI*O#VH!&#J?M+qDJsA&<x8p{<
z;QIDtXbd>)H?0x^mp?)>rr<Q`c>0lSF~fGV#R50Xw(j3%@W^I|X84UPIoqmP;IWEp
z7U3o#4_m<O<b|!^{NUG-YbK*_BUO{}#KiZZsd+UMk9^J{c8iBw7OORW3{#a(SZL6(
zj1lb4B|(zP7Aj)OTSD9wLG%C5^;U!JRhao)C-l^A#T$Q~XgTZ<R{N^M3RZu1pR3xW
z9V@_YeVcSo)#G)pMRst2V*Q($;J60)8H6<+vnZYZa24#Hb;sVwMIV8j5%*tR-QrQ}
z6ovO{656-UodL(ei+=)IUoRilD(4PTn6p4#OFsO!x!|jZ;Ql}7<|d<6RU6DLL}aub
zpR5GU|AWu}i7Jl)r$LZinZIJd<_LQfi-PztEc))eme%ud4Vk+f>VErPDc5b`(|NRL
zE;!sl=l{HCs5d*bW3kS@L;T<{2k8rb_8naIOiWm45^a>Fv{TkzDI}<0KmXzyuzlZ)
zOO(X6sF{5IV{CEo<a`|~vo^Kab=P!f1wT+)vi`3&kH3_5$Dj2|w!Z6im-IpQ34+XB
zHgT?$WZD#k7;PDi*7u#@_yUQ~f3`p^?e#J36&{rebKh$z>UhmkGCm%!)wVLg>_cw4
zvWxHyO>U`IW@W|x+NqiTI%no2S+FR*QDQD=1iRVe0;Im0z9ZYh-(U+kUTQbQm`yjn
zuJpF?AGnRRK_f*^@xB$rUo$6JXn5<v<I3hnz5Xs|F3nQ26<{}UIIDu=KaH=+^vKQ?
z3JxN7z~Xa{mz%C{_-nX-A)j1d@&#2;{~zRjQ2oD7n@MM3ZmUJSxQSLwSclg6(Ai*j
zOpX2{%XB}-+$^sLoDT9I-vgVK@@ujJ|EwuS?r&$CSFV&(D10ziXS*S!KTzGfNZ#Si
zc@vQd6U?kH@04|6++-%+$e{M-{t1ICdHoVU+e<YL1e{Y||C|M!9*@Z?TCC*WX3_V#
z&0xdnPbR8jeC8I30tU6ZR+_@KA(}s~saxFoDPpljs9*8`*BphqJ^W@_Z|=y8?Q1kM
z|M*nqNouuu&su(iD_2dx?bqL>hr!`0d{P%&Zh-vg8NUr2j=%5!2m33}X`iBokgdhP
z%IiwuQ<zk8?%tNyOsJQyT+O7K_5T9ck7rVa3^Fb+HRs=TPX6AXL*^P+`pl-PH>w?M
z>`_^IG~e_}sDzT!+|P2L_5UC{*xz#~-BNP}r!f%!Ztp+&BgV?0`+t<qgiL|EW$lYi
zrj8Yd&C5@vsz-hP40daW#}5NpbuF;l6eqA)Y<+rGYm4tO1($Vt3RlwX<#SdnROtD-
zNb|d4Hn>d_QIfA#eqkZloeTlzmFBbgg6IFYuQ~-b17!c^vy+S%Z{=H@QaYtvy4p+m
zO<;hc<+}q~Ir1Xl@+?zc2JEJm_e~}juHP{he*9X);KF*XEb-%J6DG20?%-%OJ8)aZ
zLe#5Uu`^Q8BJSoc36QxUtgovm51Ri6u|fE1^Z~G0AT};6d+k5Ge7<FZI1BW8Rs^_R
zflJ@dp9UyqKbUty@ypfuU_EYj?aI^oBb7G4TLM<&yfz#xj&7FbM}I}%%f}$$g)Yg!
zz!0mQDIeSOM={BU3$ErkBjjvH*#3X+Zf3BY=N}nm-$gNC_V(wEV0)2`v=2Rr)eRsu
z*J>{*O@50wgR}V78YT4ef2YrN0h{gY&H&DHAbp>mK7#FO-Chb81Bqk9=flmwYHIJV
z2Ad<G*bCN+O)aTnc{cdZ2|uYcL&;V!0c<bK?8+zcVEfPdPFC7Kn-T0bn4GF6<ZN9K
z-<!`!IWhqfN7gGLc@QLi{>%i0PU}}F`at5msdmbs{eK`fG6wDc1D*ftI&CeuPJzj#
zNkl1`+3~>RMy2*J*vx)a8@PK@+8bnN*Rz7%JL!U<;zWtn%5!`WXRa#k`wrFvGZUFE
z;_rf+0}?OLYKGg@)2IM8)9dyTg}l3o$}5ueWtxSw!0PK-x!~$R`c$PM^G_f)41ZdE
z9ITc_rwvSl#NA?#$v--7kD?bOP7L<XX_AvTSdG&!#l?^_o6*hc(dbdkySyAlA87v{
zyKx`bf9UFA>;G?cvtX46sTWb+0{0V$kBn8MQO~GOyR;Xq)^-;M*xx%Ye^GGTkcKjs
z8ukWqCI`$MvCLB_@*j-t;qLFyxuDRYctU<tlAh9qoLfq1EQb|OuidPm(OwCj_Xykj
z6>OiwMZ`JPFmsXVuNzJ)?MQ~CCw4wLrO!v-E25wOtDq30pzxCuUIw1u_!!PkUCs-(
z2iZ(y_O#u_O54^kE1OqwqlDugcPFr4IO@fe?mDxmfcF2x^!0sG0;^rKFdiIMtLmqM
zX_!1ReP)U*PC1ymr?NA^=E3-c=(VyLD1KUS98#VV()Z;jH$1+S=kSC5cW~8PobGE-
z;#R(TwhgQ<>(~p04hLVb+LZmr2)Z}^Qi0OlznoxqhDx0zsNTXV3v3U_5C8AV%Iw*R
zKi)v<(D74kez5zH#T(NR<sh<nHaDU^OZ{?BDMnZUrA$Ls3u9|8g!C(5d}Mm^?#E!i
zB8!381)sa&e*f`-3*0^ciQg(Uz^eXwxj2R&ZZFuRahW$--Eu3p)@oUP@K{Xh85XUL
zcIQ;j?b&X2JWN|{@wY=77CUDv=Z0&mg|xnqPcVg?H=$p_p*gKMP5b8S4zPXEVSkkO
z7F8LaImDtIs#mX9=DWf~aa*)<yZ!>Rj~S~?L2m2U@=(?O!Vez*7vkFr?i-(xY&KOb
zOaQO{Kjx*X{_)oy@c#dc3|}>k-!*B*C>d&X1)fy1_GUAykG*I7=GH~^QiTGutkB&C
z=bt^)ti650?7WGbirLpXmB7PG)T~49o1M*ms2|MbZKkBfrq1qAYIw1)O3QUcGkE;J
zW&>pWU;BQB{G(gJ#yiE?HKKo?Qhh!v)$E{Lo~pq#2CatYavBe!_bBh-j!>?$JEL*@
zaJ#8Xe3PbCd9d0Y-rq{cRdUo`{9XtSbBE<e)c#7WQEv)g2W}VgiRY`U8wzV4i@B;6
z@_m7B))z(9Caw>9H)nO4$Nf=N5OXq><<(xHq{jJBPuFs{x^RgjxKI9?#Y$CU?{&57
zRT+ABytf!VpRiKnPu&#Ff8CrW>`al0ZcQ5CcIt@__td(s&NSuv>Sb>HYME+)=VWla
znx7IdzL|H_SSXFhWZK^Ss+0H}%?r%lfyX2ox%<r4YyMCbk+L)Wrs1wApK``X@P~+2
zerB2Ggm5dhByBUpOnW0^nM3j#%knR2ZCPNbc30yJI9^w2Nt-gP=+YDwQZ#=s&DzLn
zKA*OM@(<03$3IoShaXZcIMD@mKPY}LMae3I=KnLIAa@ZtzdmWw;x<X0>!gyF;R0K;
z{+Bzz?kN>jH*f3SW?aG(rY33M2^M>N!`s9*(8loL-#g%NWcth^AK86cjipx0^wcFI
z&9&kUrmfrUw9+4%DBH-NRsfCvR~LLy*?n9_&0qAhncCuK%3_>x;I{UC&Sv!*&Nbk0
znei@BP3F`+RbDnOrT<Gdsxqf@sX82aqH7g(Th&7Aw5eV92}PEa1dZ&^*VWhd>w@?H
z=X~6*cHsDZla>X3rmJ7v)UW8v)~uhkNo`r@8MVn9O*O67&NIqdv`;lrbdK7~ycwYJ
zf5orTg5WgXQ4nls`fHi#ed&*CT=&Dx)}_g6&QhA9^5dwX{*?qsSpD<d3)=s$7%zWR
z<-&$?g;QJEz~d1`+EO~J_cCj0+I-bm-m*zE?3bUJgz0Iu|FMUao6?to&0nc<OEqlG
zakGe)rAq#?m%---gTfw!r+iqicKX9ra2@4yl3&&M17!Wju?J0N+PwU#1_dJEJ{3qU
zv!$%I+p@VD+@A#1yo?$(TDs1dX04CVyu!Fnsfk<7&||KHnJya#Z2VuWE1gHBsV@<n
z20{8k_?w&rIE+E;a>m(UHn*&V_B(zzb#=)-p!t6#cC~Wt$}kCVyd<PbtM1vvXDU)~
zLMx6DG7h=({ub3fx9i4J^)IQeFA@jO|Nm~0QD1*cKrcbwNo#>#u<B>ka&^vdH<MG%
zGt74M_G;zv?N|G}eU0v|NPp$5`vzKh{AyaR?d__<OG30VW-V0Pw5eC)dN<^JmIp7G
z<pa0XtGAX{t6VT-2B)>Xk44nfR=w5A?c-A4;Cn#HcwN7`8n3n1>CzjflUQ}sFT7XK
z=wIJr>Kxsvwaw5;`!Gw8*}c{f&Hat4>eb%d8s(3ct8Gi<QvJGZy_tu5rCFck4fPo1
zKPFXg?yD+J{H7_?!l%k{VxmrMLyECAlaT6K78Ub?Q|CeR|H@v^1J!b-DeATSo1t;K
z>=(EU**WK^-sU(Cjfo}(CO4JzHFftisynkBRm%&RWYXTf$$+z?O~Xy^5vc#K>TDbU
z+W)U`#HvsGP6CJdO}A&7KVL;^E?ebkmJ+y0EqmP=jRX67Ow%<WcSX&WEK}X8s-w!g
zYL&)&TQkG+{o70@9*Wk8cf6;)erkg$U&;&hZ-;o*M5fL+OgW)zI?d8q{m0Lnrib6p
zQ?q>K3Y!0y|9tMC`us>i^}S#2nwncbQe7_0ZO*~dX0FwDO5gJQ4wHlK&SqvI^?GYw
z1*<2V{h>82kI8%wcfHxQonmT@fjiWM<2RTrIwGxdTOrVN9`9B)O?C#efPxOaP91h|
zo%3%+KRC~4Uv5!n{Vk!iU$9c`_gM`sX7yPro6-}^_8z$+-M3(>T5AfIx%b`s8ceCp
zs*m3_sfY(9tAWP<U$Sk{Xiz+$RLrBI>)I_~_UN#l@-s#;Q;{A&)7`DVl<nUcsx7e3
zH}O8~p(faW4Lla*a9l*i({-yx-K<^><y(f}@&BAJMrxY^x0*dK)B^SY6_?5^(O500
zs6L7Hok`6~MsrycIrV<?evK!l+?q|IZ3c^M3e5I&URI6x#&4D-xlrS>z+Cm}!))po
z-W#av{rA@>vzx9lUHFt{(rYH=i4U06rg+$?_SRb|zL}Dun#dPsRy<2c?d;WNaQS&u
zP)t>5S+POLLU!}tWd~FW)V3&t?*Bb0;Hi~hzX}{a?W?9~CS6>rzLq!0_@fhpQC4WW
z*3&&VHLK%xo2<5(tN#C%w&}zVFU@wD8EZQ3kTO5%XRccOut#OtspTpjnKtT^|EyLy
z{&G7w55V^SZ{p(yn;qr~S=+2!vQrgw{$JJ6PpX35QtEn7{K4n{UE9Q|2D<;}dPTn)
zKlJ_|7qs*L+Byz{!?q;PP%Y;2TQ$)6fA2gcHM~}GXfSuZRJq@zt#)k33{}wmKfdD6
z)j;?EFbAwuKc_qw>?Tk-0>g(k`>I;EoCcR0g?}5tWsAd`Jh1s7HSFfw)g{B^)Sq08
zQp<>F1lLuYK1Qj7&i{km|Kq#&uzJ<c7*)YWb@e^cf5CcT_y0Uy2)Pd_`jWWDs~3H0
z>%uO9^CHM>9(KgKSJ(S7%E$kRtIsq157rO5|0jI%L$yZvYOs2qgI(%TCk()0h<yLg
zZnsGA`G27Me>!b@<*u<$1N#MJK02NjWeWBm=>DGx39r?rJFZasaVA$O<MLLp9(JZk
za9a;_|Ib>9ZndWe=P0<nSgRT{F9R$OGVjwQ$XQ+>HfPOw#myU^s4-ertAg(T$=_lG
zR;POyvM$c`_yOf<juXIQO;<mv39nC7*>MJPS3F4ng`!trInez-KbfV$Y>@b?)t{B8
zos|Rg&3Tro)nv!3ZHuv1N520DbpD_6#cAs3_x~8{tWvon+YdhfPe(>vHT>&l^?%PQ
zz+niw|L5%OCo1Uo|G53zrEdQA4LEK=_y0_@RspAzr@h7CxH!D3PBpt_zS^{Z)nGpJ
zT26J){Xg12d%%3zjEgE0f&{^OK<<a(3p&DJ`TuD`s)0J@;IIYBZBSVN7GLVdAZz~M
zy{gW7My0}=OzJDMt<^c)mDL2Q)WK%`P57(!<v3!$Md@)d)npM{uwIZkz1z}MSNU92
zbCwNM-7!HG9A0yE8&oxxEK@_i|7S*uIN0u~toy<Jr#*{h)c%z&QhJ)WUae@_Tb0@R
zI$--i=l>nO*Pwd-0RH=bj;qy!)x+-pNtBCF2i^bU=eI)re~SS8{6EnBKeLuT1iN?n
z;(K8Cy-o}T^FjT8ZLR$(M=gC-KZ+yH6>F6$2genm`+shJht&0;Z~$S@{Xgu>=c>W(
z|M@xbrdoC%<eWm-{XZ&cn^h8uw3U`*L(Zwpy>v-s$34hi4EqHFDzg?iY3%Qx3r+_z
zdwSIxufJEBvgn$6g0Z-&-^+51rPJ1_H}{IF?KXR*7SYqMGWSt|(j3qAVE=&J0K!E+
z$|(HlCod~cJY@k6uUE<}>PM~isY^M%P_fnd3l<Z!nyIROrCY7JcD-616EFDuKi?x(
z;II;6msLM;;sn?|b}#NJYHiF?I8iI4?tEGpEDy5pv3!r}?)xmNp!<Kk{!LbP3tkI8
z{||QmPw-N8H7);Ds-XLS@@F!ud`tVM!s#lk9#$=`)*vOJaZ)B+RpGUw8tDF?<w9Ro
zxZPL6%?927!xo+i=I;xEoXZwrdt9wc&tCm(Mu*yy1AeNiZ$&jg_x~(wP*wxo|FcGA
z7FfN6M~2$$h+gpdf3*ub!C`Wj<EYyE4V+3{->NlK-9M}EZrKM8gUGiVz%<A|QPWwK
zRh6B<<=mlWFIDctO=^rvZE8!T)+<MzT&<RO*cEL49!*Fds{eCH?fkzTN|l`Xa5MPi
ztW-AznW#Toxk2@Y{%X}t25Z3Q|H1D6xi8=bHXmdk49}I61ec9>`626sx;1vG9WxSF
z-srGZB_bJe$7-OyIoOU@zaVE%M)}I9bR0RWGVcbbI_UnNyH9?pUSDFanmF4>O{wDu
z*qw#dhgG)RO8}b<vI}(oPh?UNSPsO8;q%MasYG29)$sF~1GWPuenqSu%=c*ZQhUxF
zsXj?$r%L%rCsmELYU(G&ZPmU698yjF8LNK!EhK(G`o7P<4-W67Lk4QaYiz)5kT~f6
zAJF-K-2Zsg%2ZZ@%>{{ru;s}fFduaP&;1{eeMlXpE$V?T1#tc86}!OhhTZ?;_f#2t
z{@)pudGet9e?aH|&3N<^oPR-PBIC(7B*5lt@uS|`WmW!GWmyQTTDl-)-K?FXG}yl0
zOOUxdWHVsw9$Cnq5VKpos_mzlK=r?}GPj_5z<<d7qlYf<R#AQ61upA8U3dgG@9uGb
zwc>4vGiG6CmHpqZ!nyMfSPkFw3~)IqEb<VX=Rj)SvQ1Ds$}Xzr>nj9S0}?Zw=BcK;
z5^-nR&a7!_^W8I)HP&r~s{yG4;fJ*);C|AgyR7Q_?p{KXpU@ly=U+*cR`eEprD_><
z1|ELi+uy40$}<9|on?=fsBxXv1&1NXTo^XUgX~Fy@p}qU&tHUzqtgy6#no=#U!WR1
zzgJ23_aCr3K>dH1y1=iHbxr&`S(I-t)CJ4K<dEso6BpoezXBe@-40V@@X`q^FB$Vl
z^~oh^uwP(uDy@)nDnWdCDLJq@5POSnk}}&?PB5Qony#v+u8Jz_dC0y8kUR*3`u`{1
zmV?tHh!4XJE3%dU@}+^r-D`MMdI}b(J(?&7xA)0ac5ocS)PrcTNlVps+}QvhzbmVQ
zoQDdM1L4FIf~x=iPF8I?^ABte`(8--8=KP#_pj7OQLvpL^)URZp&2YM%Hyq^9w`Ag
z|Jf77op{T-<yAY&e<<ao{8Cl;<gdn?e?evLY)H7|q$sFqnJx#11IWzp{0|jn8kT_F
z0OH%esaBf)p&cyWxmXlT>u+AI^4}4%{uiVMgx#NPMe%c}{&%puRvgO)>#@IG4R^cT
zS#Pi$NFOfT!xsow4_3nsX?wVRJ)(A=T@7p|Se$`@!Tb9juzAnce@77q$+sl3C|_;4
zq&$KD6<n@Witt%5AiZ0f_`v3Z*f2a#Z5iC0UEVuYjhZ0!JWSpoXfrrpSM<4p)x{}F
zD8D+)4-XqHO*44@@=}I`Vd33ZYNA?I%5VGI)K9-{SKC=;qxv`bvZ`F*M0k9{%m>j*
zJ21nqhsg#UUhNG(Rm)=!s-<14P)c!xlvUbi0#*99JF9YVsHo1^DGOGobR!7tz8qUF
zFdt+t3^yOuQ}efc4fp$hu6-(*54Xb81WX-_uKY1qWl@<FSp7DpSmlKkx0P=_U{G7H
zS)uacg0TAA$kR&U-jIFM;&G+Q5qu}XcEj{DTR`d$TZThm@d^I>R4XD%)E3vrs%rBv
zD4TilgTri53}nsY`T3CaC^e5!)tU)%U#a$_Jh1($Um)w8)*dTY-=O$j?L^WOu>UM4
z)`4l5og2Sufz3Rm>8_IZAW}8`U@qA1fbWaIG)Nt9Viq{wPTrji7Q5T0iDCw`P=cCF
z-ACmqJKPjgCTvELE8GC7$3f<xW9yY+V71P7(^O-=^C;EoC#cL`8w<AMZ3bk238#EL
zm=EgzcN{sZJol*@SWiRYPOv{Knjfhga6SiClf%7Bd2f{}*sU<TULA()Nr&-4w9@X|
zN>3P{s9ks23pNWR2Ex3`>%si3o-<Xi@#%rttMinUO)Zqwx;xcW%T66saq00_o0236
zwhyF-?J?wz;np8*s)D<X!EOSH!?3j22^D3{DPXf;V)ly>cZ)mjg_KWGVyo2xq7T6J
zPrZ2pZU;!;(j$=jjzMe~zQ}+&*V(tL5xk}cB)?aeP2-eqEqrWnj>#qUrbnW>AURU-
z<m2<ecQ%3a?BD$p%x;~uQrlSQI(*!)R=8ddB)=}L6g=i~O;|(ouGJE-T97y{+`jI#
z=7OA3usSuHx7vJ+cfju56E+RS?=?#vX-&KAsC&ML8Lak!MF38JycW*{y8&cpDElI~
zI<_6P5+E_xrqAH?{_)Bkuvs865Pmgv6N;VD_gke8t~mvckC(3>!0%iFsfFP>J8igK
zAhG9>JYf5teYg##L1HlcIW-1sFNhy>MjOR$kl5Ch@jCe?iC}%7v{b-tW<|aC+9hv%
zwRY~x28)Yc4u{+4VQ>a)2IKPMV7ZG+{(xzaSs-j@d_W_lzZb0L-tj)TK9Jml&C9^z
z%&`-6oP#rT7H+SB$0bNU+a(&#H(UJ}#qS*dj)3)o)YV3s!`Wq8!rJrQ{J`-Ck^^Be
z)mPdvD^BT5&XfeN;RA_*uv@w;*sU7bu5kSz@$_2A+y;p4IwKOS2E<0j-^>N!?%Dq3
zzGU+8czC#i)ZJI@K+(5vGvZE9)!jR_r&uVV*a^}D!djXiF~l2_zk|c!QFptJJo{|0
z{M|W+!Sq`f$o*aG4s(FhKFADY?2w)Y4)d=p)8TplZ;-rN@kLLt`WJg$H6l03!`&+W
z><U=D>wOb&nnN}V8(Z-Ge7Ih0@*pu8=U#YSm0j+q`Stu=u-id$_^{cbK5=~V=yE2H
znNZ>#Bro*-BG|61<jG)vfyB_UbM05nO-mu?w4%!q!)K{B1@}Afsr9{|3U{07#(p?k
zb2;{OknL2atFJH@9JcE+AA#c^X8y}VYh?fG{{@Rhr&a5?<zS{6m|9%)zt!8+u81rK
zn~zOR=*(Mixy`C(aCWeIp!PL~<M6Z((|dOQdhM#~-f(|i%7m1mAo=S(M|Ez076HeD
z%*jn)HSsGZDpiNBfY%fI47<Vcb7b8#-Q5rFg4yon?piO-%4x?0_=5F-%qp+gsQb-t
zBABnbm`QB^QWmh=SnvM;hsT{??7Ax|XKH--yb^35NDn$bRU`$r_t(@nU~@q7Et}`U
z*&seP%<)?YY-Z<<4?2I&)v5hG#sn@O+kU(Ui(}J+EcW=sdJJ`sL(XgaE;2%~yCgb9
zQz>B;SS<4Ebg=)B%^}3TGWP;FEUeF*1cxI?{T%M2+I@nuU^78{iTD6`o&@n>Sj3z~
zCs%nNSYOtR<5JAaA>{}A+)LU!_w0tdWxwZba2Ue$A=CGH1XT(St7@*i(4ukJNlQI;
zt`K-G=0NN{jX5?eRE>X~Gu3vSr@l;~Q&Uy!sPez`>1xjW+{)g2s?|UEXejIIs%x<y
zT&ZQvDFtprJx~)<QchWEv_V)=#b)9o{q-j$jBofHR*@|4H5K2;W-2$kQYlY%mzsVd
zzuDgoXSJpM2leNg@G7Q@x0x(tbvFHUcA>mP-FC&d88?((X9OtU-zcH^Wa3PX%Y3Dp
z_Wic%JF8xpFo$#)9qHbxnz;Lb=^cGTgO@GW)oYJTG_6@0p_2O{P&sGkU-g^iyk@;q
z)eR1Yb(wx%m8`KQ@~}Z-^;*q3o?g)Te+u9C25BYK98<JwJSlT>eWNjBlZj?@tc-f?
z^gX8W{|i;*XLo4*E=$)4y`TWD<KMXLS391mWb$vDzJ~a7XVsjvh03cw6sU7~Xn@ZD
zS9(&$p&=1?NF$_ak=nYI-U@wXY#QB@Sk<*RaH${Y@zZycKCZl!qt)Q<mku*$6?XY)
zl^5kc9%xYht$9xGjE9!`FaGzYw{qtxxju_jS$A%=%Bxj;22$HM7<NWAYVr8xX{?;P
z(Wt(Ak@BT`jNtUbbGupnddM9Ut(&rDl^ILa7CqepUiWlCZJkk`LWJ>@J}G0#X)jgV
zO?Q|rWzy1^weGFTROdCO|99R~IpJe#!m&<C`RbZahM&*Is(VbyQr~YIqCQd4&|uoB
z`-U41HY)x7A*CTF@20M}E>(FFd!_Q^Oj*;#t3@>zGrlpkTyoR!DqElS-|Jo)3%#4v
zOEr4c_U%||dVS{y`SW~XijvcP71wDysvbM$p#d8IXEl&F>6tQLWAo-+n%Y@MP2Mgm
z)BvsjdoIOhcECg4xZml9T2uIPHPHG$lQLG*2Hkjr+2*rU`!&}p^u1wG=v|qkk=C7J
z()?txa?|A}Ci|rVH9`CTJiB&*^F`g=m8wZIwbV1_Y%~R({~vO&K>ZI}uxd+yf%;<Y
zCZ&#d-<55S_p5`}{~y`NAoK3$YE|wXtCVm5uGK&K_nwMQ&07<rszpjW=l@U-t=Oa5
z+Y8x;w@zPJeRe>Wu|4NvlfK<427P?`n!?@h)Z!0ZRZn}eN-f`A-^g{+eYF(DO0|7#
zYn3-RwW`VoUooE5-X~$J{myVdi@r%k=4y3$>8YkC&g*M%JmgWnZ?V<Df@y|n=3h3Y
z4?YW&pTuz~yxQLeI{#lu(Pbfc{kOSxoX)DYLiHJ(|I|f}9@m)WTxWXp(npmO4_uWd
z`))9a()g%S&7-Hzq~C5fYu_4WDce-zRYz1L+H$=V&mXZ<Ipf6zKK~Dt&lknE%L%@b
z)6gvtRmq9^VCp^LfLdNx6F3cl*8eH*e5mz$S)YbqUzd7EroYDGw_&C(%(vAY5?oYX
z1+W@8E|oF8U7`YB{}0;#zk9i-s@e--u>P`V-@ti(jr(OVeg2aeyd84tAdl9BX^rZ+
zp(ixN_9-eGYEIH(=KT#G|JO3tQxX3-*_eL<v(}fnqMD%d|23i|6knR|G<r9gLoNT{
zM<cKKD-?^D*sAYZK3`w6XMvVATc_&$juRTY*IOIw26C8jK6|5ed4q_$e)(D5CzcCT
z)hFK23fPvXc_;F%YUi@+S{&<T)K{)lRxjH88{F2*^U{&y<5W?Xa@?poW5ES*`--E~
zQ}uY_A<fzLFV#W&|DCkiH7rBsY3AizF`0gMjYgcFyypIg^Gx{Of7bkIHA8z*@m$lo
zHP6&j_ls#R4Q*3ZShrrCWeS&S;d@rI)q8wQ_ezLse4lmG_?!P`wM7{`I*BV*DBJD-
zr_1^x!l-YWta4}P1T(KQ@y0cu*i<-#r>J~h)Tb}i8LY8s))KW??U%}m_KgNxmnW&;
zEIe*3ot~xfpt4WnS**Ca_o~e%$vi6!gkBqJoH!`}I{#0_=eV<4Pseu!7a@6_CAOkw
zFCASpC(p9hGWzIkI&tc4b;$$f8taP^O%mo$(x`|zpnQPSK&{;MhVr>GAr0%Dtp?VU
z_nPQMA66In@<LnrUY&{hCM|X5JVkZIjUh(UKgF3SO>EXso0)5>_WZuO>Go&?(Ek62
zht8{i-)5v98r5#fX>wN8@cBJcP2qSm;rZ+ayMi-JPB7P-TK>JJcRl@u#)hm|t<E(s
zP4_BXFjYPst@dcOl!gF@j`7^H+{*qO9wwH5=BRVukv45rZ8Zo8;WbP-AfPaBn<==C
zUUvAtMugNFCE?YJmFM4TROc(_Q<bq?WO{~UmqL)xeDyzTJ<NV5`D*Yj%T^VZ$x*dU
zlTiOzvr=)g{!$IMQU}EimD;)yFIY^q{OeR+-83`}-N0+g@j5|e$)!(f9jo^meLi(Z
z&8(mcoGx^xXsB@AlF`_@?y=I7rrGLA>N6B2j0)6BJ5QT}=Kn$G|82b9sJ8R0y?S<_
zim|aYli8|vJ&k$FOEuqmN~m`qKCZ8H_nhgB&Yfx&zK2a`oqVGHtM7n%)#lUcFE}&R
z_q1AQh)61EC0;$K@l1q4+4{gOmE~DIYO%lj<RAHXt1Z_}FtvM?srn>fhH1gFQWeJ|
z4ys4%{_2bV<}z~^-UmPb@6sYmExGnQHT%F#%G<w6Y1sYlROf!yZgft4nxW;^6`E~F
zmue-XC7A@OGHK3v_t#`zHM3dZ_c)EVXBK8F;y6_^bT6ySjk%&C%+;k1TK~^=6mq@_
zsEo)kY1itx_DKEmhX^f+`#Rvg7=_lSwAwt+tKOARHQ(H%qyC3wl@{pyUpvJpRnYoB
zw^CVf_}M*wul4xsah({iwaULrxRrwAeUy*fd~CeU>WEt8E-r&vzsgN+_#9SFuRU*e
z?q?Brjl<dJ&1$L>Qp|-<MyN(Q{{^pk`l!Ca^s3nuGijSU^6%?(G><M8Q(iQmNA<|o
zGM%4A@>-$G=4yKf>1jm0&M=dH^}<+5kz3>N;|FE|@3jqo#J$$~5XWPlnlwXY(uNKt
zhNp|w6DB2@&-ZmSWMzG1_VV>#^{p@W8P@c*>g-pO*IKuWRdLJxb=q+YtyO|!73C#j
zADFy%Yt))(>8w^P!*2fMYqRQ~orT(44i;)mGigzOvrIrb_MwU<qkDnr<G&_a?z)Yt
ziy!?{I(P4=TH%B>s;*C`s|B5IR^NDUk%n%7I=DTy(UL=BW51BrrZ6A1KiZr1C!O1?
zT>j&P!9k-b<_{AVDt~%(UcTDERw-CZ%D^sch5F`fW^+(n_$>Oal9#5e=Ap`K;Br{U
zcnRAMt-tO2v{Fmo8?R9MqO>C?9^5XiJ$+B@t}c_=LB?kDS07KRN*~e&k9Xd=%4EW2
zJ=Nru>{63}i67O3r!ZJ}@T7qA)o0~jX8tvts+U=pnx2^*s~VeXX*|_&pB7)&9j#yG
zE7j5$Rv89P;4#iV(WG&+OF_G9eTSNIgFLwX(^#!%Hsk9ettYQz&F63CGQR(Lr>=hE
z2d#g?&FY_K&Qx;{`UZ|$kRNoso~x|Wc2qHDW(3=Nu2b4H{b;U6^h{yxs7+F4_A!wA
zz3yyGFn?!o%=pwjJN3>_nc(!qX)0}|tJP_E=#?xu9F@E^6l{2eH7qWRoAuo}q<ODI
z&8%atf_C@aC>0y|)8KkXVs)5`_lE{GTYo8YwZ+d=y1ZV3%lBE#ff}ItALJ&*>bGj2
z+}SleB-bePA5>FY;mN78?bRWD2S-oUwZ|FEOjga3sxz(A4BfR!UHiMBf!YB^RmQ7p
z)Tfj!Hn}5v%XIF8UV}GgirOopE!4yk=c!lta%w&KHqm(g341m9bwAVtReq{U8`UZ+
z>`phGl)Oc1W4EAjiokZ$iX~UoKkhwjw$qeV>-K$l)$ETC41{GagTu<aSxjYJijtD|
zy3eY9s;`uo6J5ad|1yiKx=T-O(VW?%q?PxtS?dM&9kY7VDh(d*5=Gqukh6oHZ1Ggz
zuxqM$>a=sp>no(e`^!N71mS#zsqp)Mw3H1rI!;{&hpp@85c7vC1J#n)e(S)_|0&%m
zs?!=Ot#v|<McqLyQd70Q-|URQPVFm<>y+C52pI;-JTx<?kky%Xuul<m{%^~AFV(-c
zYvFy9PdU8OFU%ol$ARo-&bCq&>AtO^5-+T)w|au+JmofR@4P80PJLzCwFh{?@$z^X
zr^+k=2h(G#gmhGR--F}X=1i(a-F9vh_n4__oFXMAD_`g<eQQrp=c{2gI8lF1JAHbq
zN~N2d#)QXJCZ`XZn9Iq3&@Kw<S8raqSx?m{L{)yLpsw{@EA4=dlT@7S6Ln^pacF$<
z-=*=}eTw?6j7B-Gs>kw`f-#!$%XL)0{1gF)he4CLx}`*<w%oQPQ2np;Uz$<9b#JiF
zw^n|$4K4*5ou@RkZkW9?{Sfy=Cw7&)?qMYk^Nk<0wcZ#BXg*zkNz>nPs=7=Quj(@e
zPV<v{H<~%h-`4nhU&b`9W~*wH$_{Y-Z+$IS?^^UB<C)ysRhE{VHlKaA#{_i#Z%^wS
zRptYy^cR16uhnw;5I9`Eo?mHT8R)OEvEYk|;s;x;mr<-5kK<0M&9rYZ)%!KiaK4_o
zrn6)(`2L^iD<eSb{}kU&&Cs=3*J8fYa*1~I>}2iND;Jwx<PTR@UK^(+ZZy|)e;Fq@
z5Av6<QK_-Mto-7~5>2Z|lMMe`-8JQJdZ+2ybx)VKbem~vM7pNMmoBwmJEM)w{~k1*
zcJ#OUTK-_O?h6hYQxeV^<k~$}^tlnKDZAZPV|Cb6vwl5sHIATg^VG`+%|()}3~Yqw
zm`>m+HZ$K_qi+|ZshP9rv$onjW%C(VpO`IC;#EJUGh1D-?U~7r89K_76hcjVFMLvK
z{!nQaTqtgEb2En#DDC*fLiS9r=&jXY{Vk#LDr2?UlfqqEZ(7!<9Xo#Ctiq^6S%l}Z
z+AQHL^EX+?w4Bzqt1|y+R5|CqTFu+{yxQu!!dkh<MO3-3r0acS@i03t(5cL>?_{>(
zwWnEmx1O?=<_5L$=~*UsZmOxRw{}n$nPj5;c+DM^8C!ZZkF65Ys$K2|ZrAE<ysG59
zh6l9&A9VkZ%Je2V&3`(Z)ebYSH#u2qYkos>od)A!6RnjuIyLp~>@Z++W-`}JJEVGA
zkl8%`vZ|Jy&2shSN%rcG#kn-T9TL}UK3<?%ekV-3)mL35w%AbZ74vyjQ2ie#y-%ek
zBHHXkfs$$p%PVl1aAoR3m93&2hOwJU%<U%Sf%gA{%imvp={gIJD1zPR#owl-_DM!V
z@wJZe-&5C(E}9+DF0{I@9oM?R^l3Dw#?8}QW(7_@=3k37wfLu{m>*+JQ*HCzqzXF!
z*W5NnU0vgls;dDXI9+hYI)n3qYYX-<#L)Roa%b{FluoU^04{4)j)tqc-qTPMy6C3F
zl^O(Z?}OqGhWS{&f#=kHPp(p0aHCcEhR<Qei~YUIFUl^eCfXUOrM`?)obdOh>f#H1
zYPxkFm9sfMsjU0ds(L5L9NeFSnFFGo7fn=^aDvRK-U(U*cEeeoV(>g6NWPq9y6WG@
zIx7CEh;w9@yR1?%<lt54X}F`dJ<maT_1aacp<%Vki%$PmJA4^(*49O-GH{+_zPAiq
zZ%$ZSq*#6fvbP^(E(r52-3*oku~lYBs-BPApmy2$I9P1Q9&vSz5GCa`yFY^YpJ!}Q
zV!0FqHuLoUKDa-$;~{5RychNYi>)a^?9l+3i;V4*-N11jvwVWu<;-RkK9`?h^LAf)
z1l9{vTOg6AG&w~JEVgO3h5TgY25`85<So2v!FoV!5Pl$f3LI8fBd4hyV`2j9S#n?k
z*dCC)YHc9c{o20@RTeB_gNuXYj!!ZN$8XH5{i=)JZGp>`@!V2kybGBl^L{3%<apUY
z#daDqX#Jl;9V2AE5p@2~(`o*yHTI}$4vw0Ms`~FeuBsQFq`3G%rP6;#E`{Igwu0U0
zwZ2a=U+uW^&H4Ype$cSU1lxb7bfuc|VaWV{<5v~cy=9GH`P^ss!St4GGnJ~gOj9{<
z`v%y5yAo!ru3ocQHGY{OSU-nyfLf|9=DLf6VU1wFPrWayqVBOBoMu7(4sJu->6@1@
zRV`yFi_*1&XH<T?->25KgkL$?4YEJ0q#|E+Vj~OKPLRH-Ti>f{NkP`AfcR~EH<Uk?
zp8(r&MZ5?cexK{7C~e*_OZm&pY_QwF>bzIM;}677e*sw!k@t-WtVVZcI+zB@X>HlB
z%3_3Aj|UR-`cS95>WGS@_3p>2MLW9G=GH9*hbNClFxXrfp#@3~vsBesGk2=Jeijdw
z1L-s1msY*9$_*Tba<ftQ0nYrjQ8AZORcTY>a<wbr@^C+b^ntKUyQW;scT2Ed5Fdn-
zpGtuJ2x4CkK+J)I_@Rn-)%2qjRO21*fz6tBzgF$VvnH@x*hH3t#ge8$<}3T>rh(()
z@%PEfQK}ok;RiB9bPwc?Vi21v#Xxm4i@Ms8r4yBd=jW?dzw=Yy@o9!q`V|%xRYxfW
zkDX><w|F=>!ov=v@0|i<KPHF`!e_2;R{i@{3as`KPq&(c)Dtz=?Lx{$*TTSV0;#bv
z;#HX;Xr=zLH5=^SWZ@ZLf6sozr*u;ovG)O_mi6BU<#tENdQK2OK`cR4y=9^5++~on
zC_rK$?Eh;e*iNrw+m(3Om#ey)6sm3&>Q#{VQK<CGI7sDrP^98sImkNKpD)Ul*=!_~
zKeU)Ar(9KnyK8qqtnw=MP2f1Ld<)6jcX_8Mw%jgNk|_xRyY<w|2TI$c4b=i7dQ|on
zTvt_8VgQG&y5lZ|sHwbQy%iP0U^B1rD1gJteBWX<{x6QIGwzmw`3qNF2D=sHZV>)C
z(GhGmh;8*UOTq1#hSH_x62%3~kh5Vy;{Q?&z-GFff!vRNPjM~SE|44u^H0oBw#nYD
zx^?MY)s>g#E2kU~P`u~>x!--Q6JkvbNG&q{R`L^UuA1F)m35Bml)fIwSNvH7IqN;o
zjuGsxes{>3UvDC2gZWz@v#We`RaQ>37XZiGj>=q>No;q)?r9A%RMJ`h8q5aS<9KX`
z3for5K6?-!hO0V^70h!wROc^(oLdhQ-{b8M4jZoDU0`>C#Mi9oQ_km_q^6xeUtzb)
zHraX0tJIiRBi5re$vy?A>2Hr#Do?nz8?NuT2&AmM9p|8YefJ-@n?Po|+jWB7zU)ku
z+B3D!U_Ek@T`G64W`WrtHIm^wz-+z8TUGY#FjTW-M4YRfQD+X;f8tgo*lv(|5Dx9U
zsupzPt7g~3W)1Ie3$^<C+rj<+_5Dm*-269Ggr=-F53PHqUMR$;WpALb)ZQHpTK})w
zc+6IvKj62rfET+KSD%~?X#79*O}XN&@>V66Z*xqNKWtPz;yzzL?a)V)c8@@nC5(aQ
z34M3Xn7dx8P8Q@+XH9!xZok7^?LyOT{ahhy6@wmTv*4|2=AiNa$BA_s4i9vdW=VA@
z-;F(|J?*o%=7k$<+EvPG>gK-=n)Rqhm@KNXQE&R5X}<i^GsBe+<h8bNRGJsvXHYq6
zw@Go+5jBnb@*B)0<1ZTKw3?aE<(#Hrf6US70Q-6E%dyUy8hdssY+%>c))16do)aOW
z@S4-gbfL>mO<#Ut^|XKt^RH+8RPtv()O!5qw|dCeb7~4ZBUC~kCuo?j5Hxe1R;=lK
zzCz{px^-$yX<6zQE1Oi$Tk@(oUY69zIm@s4!#7(aIYnDJT8Lf4<6)9!U|_5omokUG
zQY5qTeZe$?YpIqNE1s1patN3!fX@Fjw3}+cvigj=In#9WX;G#si!#nAzbVjAD~(Um
ze>Qi9v6kpN%{hkMS|8%uP0qdFuDpqDuQsUuFK1^|kDSzPc7ajCBA;uSTFmAd;JnIQ
z$ZvAiaHC0M)PIv1{_<+;*_T=H`|nl%d9F;YV_lNDe+{R~tpG8zas?NanFppDE2ea7
zUEZXp^}xbY{khdq!}lt?jX(Xpu0H!ko3`m)E%mz^XOzFyu&JJy*<^O4k6nvH^Mg6}
z!e-+JBRyShH$^Rutq;^EMIKTs6kBC}edh;-Y?UUprRlONp!NUHcv?Z@|0;56siv3j
z?or?Qp+n2wq~FZoXAfxnU$N`nXY)d*1{3$nwQ7mI4r<9O|0)P_N14su{lu`gsZrDU
z^A%;#`G4{?OEuV8)|#C*-K3GTzsHPSZH6}J{2yV761BX4!72h3d8&76rPPbNXPbk@
z|Gjs0g4X}5wycy<f4=^|qHM-R)sjsu>P$Zc)wlGzD=}{nQLWA8P|ntt*Z=LQsB+mm
z&+NO1isFF>Pc^<T_^c*e?q=|OmVwHSGA*^lf{P}shcwLgooX|<bY!B|hY8MVT1$8}
zI1>UibqqHe-w!*fmc3z_+P4?Zs=Jp;sQI((G6jwQt8}Uxn?_zRJ)3!4{q7<@^E5$q
z&7fm@RDX*)8XUV*VeXMEuM&}+ue@^AY{eI$w^R!j+bZ?4Zvc;l{m!t}4Pj%_%=p)$
zdA03`rr%UE^YZg+)z!*2s7Tito1H(fS}kJZW;L@PAI#r#sw;nSaxz(UL`AYK*Go0-
zgpArnId#zZzx>vXyA^-6H>)cwOVgO}y+L*Vsa|u*jUUuLeTW2)AuXv{t0MogN;`3S
zm)81cP3jgsW*Ua;)|owdsiduc;-HeE<~BniF$Z(@-ig}ZZJ#TAzuPCN)3!?W&Ic{+
zs~e`t)~HWah4%jyKHA?9KL1HfwZp(k<=1&bl|qwTo#)rMG)oR&(AvT(siOMJN=I<^
zBhdK2@=1|!<x5IyOiRVSYH^$~*IZfHq`YO@O!fFDt4%)q;a6MoXdmeOKgIR4=BU{Q
zyXvo~QPI|y=ux(u_erB?%L$VyJJ*;$WH;09m0?o<Wcxyo`Ky82<~5!=$IdL!x+u3-
z_09=i9lyOy>eX#;)j6EF)dh|2$v)+ck`K6aS7Ysz=c;#)2ZPJ(usv;RD<&V;z94%M
z-2Zo9)uV0{yh>a8ajV%Lzwa9F<2p6vgX+vUyRCIJ{<Z7;-&$n8L_J4y&dhX;$U-$u
zqbrOWk2h>q-n)3Q`Gq7k^B&Xb>bLh5nKtZfQkytqy7u9R;Y#7B=IfpneP`@gFQl^F
zz}kXmJ)7zK=lMz-M0He~C8YE>=$2{;zSmG+^iD=q=)phz%A<|yM;8c~{!i-Fd{94M
z!`3%I?dv5^vq$@+jjmo3)ljq%Q=etFPi02`1oc<G50z@(9oD(N=%;x^9fP(C$3`tN
zLuqqnwU_ES6K87{L>ZVpVqOJW|F0@^EJ0N!)>GB)<tL4g0xu2utVPYr>Qyvfo(t4{
zGd09)4quPP%uD*}9o7}bA&bS#++=$-%>6mdug1<%Uwya9VDemL#aFT`G@jb)soQPQ
zF+ciXxk{-buZ54Gy2YYvt_D%lCz~oBVliK%7^J^(Rk_CF(g5vOt=rAtnAw|ulrLA4
z{BEJKiOtz`#|#}M*Ryv`a~4lfm+?Dk_IGxV{w1AW(EY!PO3S~3$AVZ|_%uP||B4+C
zRQnl4v~8HQREw54nO}LfS}E_`1obU%zMAJ~C}_=%*`(4QZKL{a#SJyjLT|NX-!jc7
zUUDk@a<Y2Mqvn|%Rq0eY`6JgXbTym#3d;>Dn<xEIYdrJEBueU>+N_4%>X%byDYeXT
zRJAec)wo-qqN(RGNz-qhx5|dA+f-)nsR!TxbC=mqdGSGijfJ|*Y9=deO%46tn%DIU
zYRq5XteJLkw}!cWy&?bnN^`NqnQELjHO-fu?a|~r_Eqhh(Khv?{T1q4Bp;|hDo@cY
z&|aps`XsCJ@+Es!w}?cly<b)&oBaHQ>e6s-bGuiWD)$`C%{@QQS2lRfr1Fg=*RUs@
z--096PPyxCr;3TrPWio==e0j~H)(PSd{xZ|Tc;UtvP@0-mzK$y7)#?>{?gk1vvsuN
z_Z>3*_1a7$G+>F@$FS|@_BRVO!!{XMa2rLdTt3dM23r5$ynTaO|LU763dbS)A0A!k
zSKWGTlBU!l1r4_Ae>5AePXq5IHrhW$>)iAjRb72E^EiG{b(wolH0%xZl`FRRgZBT+
z$yYF_=Lt5dxJ}5?^bCFr+W!x3S9q_PrZQ(i3wZy3OV(XIo%d5sK>PpFHa;|SW~nxN
zqv@ygc&>!nzCCH?+7C5U>~opHWuE7$N2b~OvqAg+<v*CHX}sAlp<=O;LFL1|)mp24
zg|yag+@y8EyHBN8WP+K($9fa>rY?<R0cXr+9A`4r|HY~`yI$UWGH08zquy?nua}C{
zOX8)?D~x>%c-CAq>xw<8ZnC$^P-d^HcF|4+@EFSW8JgfR%*md66fAYDO!QT{G;W_b
zuI8C~(adezG1YB%w`qB0&QpIPus~&rOrzvtjaE(DCreGM!ws}VCT>=%Rh_JSPm)Xh
z^;$WVpX(Hqt`*NwXEJZoP`e+gxqRI$Irme$G%mM4((KepRm<RE({to2Ry9er)%SX~
z#++GCTz;LYjcm1ntzu6|l75h*nELzaC(SHsE-QU<T%g(=*{2@6^S$1rf+i!enbw+i
zGhDTb7!yo{A{Hq<@mvLVKWjIadQsXpQ~v5n=8v8`s1=(*`b;-%qK&uToo>?Bu+ilF
zoDSuw9UILjp17r^RWG0#mBL_t<Cm-2yVUvM{r{!f6OA5ag=v<3@z*+7e^>3wvi*ir
zA8;G*l#Nw)yrQCA|BMS%|10volU3bV7HPUZTv4m~L%R8%M;DDYHu-7$eVL&pzP3dz
z{jk3p!{Nu^@hhd-MoOoRpM&=Qt8nkwtsby>i*oAKwI)?Zgf!}B&D1Jd+GsZ6fHbK7
zSD5!8%AEJld*ctmma5)c@2P{@qd%4?n!3HnH00fPL_@Ffz0&jdEOO_Iuc|*S5jMME
zzg<IqK9||+AaCu2W+4?Rc1~~`+3LV7)mqlWYB|<2=H;^|D;{kTS6{lHLCrp5uDbj@
zHE_E~qgYKnCA?A1Mq!DPp>c-Ft54ag53iZ)?U}Eq_Hd@X+4qbNrKe)wHJ)=dX-t^2
z2)zIQ)U*V(pUV?WSG=_}jajE<@V5Dh<^zUeHM2Gyb={}jT2%=<j990%s8x0CR%`yt
zt!l41Pxbir4)Ff`v+u=>H(gdU6MV2!?Zv{qW<fJfXx@}xrV^dkqrXn~q*-=Wo#K3N
zPH_Eyb=72Lo2G+`zUqG9yui6)tImcSn>5#clF`&!8LlM}*=CkL)l=Q~$$G`hLYu+n
zuTfg8wjkrE*_v0Ym5L`Tfam`~;n#a_z9P@$F7=D+3c&esQKzYzS#++tN0XGYoTRw<
z{H+#h&eNxZ(>`eaKjrB>?VQ~gG_^h`s|PojX;?=TnKA2GX<cDlr=XfY#W3m3HnWgv
z-P${u#X;x)i3vMYsq~fJ&;r>to#mJ^==^`!{{OGMZxpSn*Mrahl{MU<U3x@JW2<qx
z))c)2Who|UZJ*C;K=r@U1&cn_Teg{|C+8i~n!lJ)12q4y8GcF4DwN+OBKLzb%M*Lh
z{(l7?M<ccA&sXZ*aJ-{+Ao8Qi)ucd;z$P1$*HNp@jJdqDzVC@rHGDZ$cU!QzDs$IM
zE$?;RT6c~-Q_WlbUi&oHU)7p=Db4LWh18c#oGWR4Z;^b2371Bpk&>$9nkV4C(Dzj{
z)Z*P(wR1kEg6e<8=%z{PN51dZ+AY^?y24jLW5T~+jq<IVO?#STwfAm1uf1nyznM+7
zsMfR7mFiy?KhwDG)~x>f#VwVxQ%dF=ckDE4C}LD=YP)KpTwtsA)9$L);fLX>ZyE0C
z)Xe^F<e0WuRVakR+)HA)$@}N|%9s0}t98q^=sk*((u_~#QG0WDmck|hbN$7~V>Ko%
zcxLi(%OlM$KNXGgS|2r6hubDrhPsB+zc1F9m;_mWuIl~`H2yFD?zgaxU6j1}9=#b_
zJmsQVhS6Kir1uu9&sp83S+^$2>`9f1#=9+Es$$;8YF`c7RS(LUXr6dxX87s+6Vu6z
z%$n?RnK~?v*G(^0*=aOBS)(S)XlCS=e8-gY61T?E8;)iVBVE;{_SAv)|0@*rWNY*<
z*{XKu?o_jlWj|GOkH(rCi5)fn`Rb;=Qdg<TuIDLcwz0SLLicB@|5wY=p3dTEKF_Al
zEPl7T^0T6e>T{Opn%?fLP^tf0V*2o*hkE1&IWw0WCVl1UYM}aG!BAWjT>l@5iUW=R
zD?TjnQ!n73tmV-ESJ`)6BV_;o6LWRnGF9^%4y!d8&*!S1uSiq%HCm|FI%$>CL4F&}
zOF_EIjh@wdVK<x1s{Z?`_{kqLJuQB~G{KrpValw(s!1<rn>bGpRl64_p$<C#ud>}s
zc}o_fW-v#whNco1I9*?RG(oNL^l3BD{ePhI|F$HDX$CH6Q`=q0Zu+WR!F;=%vbz2O
z8O=5O)@n}qSZg5R9cLD*Y@^nHW3}16)0&#!x&Eu2y5+8(6Wyks!rG{DtZItJnh!Nv
zNw1j{)x0{@?u7<`?*CIvNj6Y3b*VP9dzGoy_<BCL{48^ARgv^KWZ<*N+PufuQWaGH
zgYw!O<HuTFd3~A{Y0dJizLr{M8&<0qb*(ZkpPgr9x$CXg{TCCpj>tVSsW(`!&Z3`h
zIx|GTe4Xw+&HPYz^Jg7dDxmZKg^$>%1c^>mOB7C4{nW<;9{+#wCmUSug6sfct|be>
zZDSA{giBi98O+R@3@$@K{I?(F)f7}b!1;&KF$yjYl7r#bst3Vq4_Ra#lzFAp!E2X!
zZyD%jPGZtOQ`rR8`*6LX(UGXdYST|WR60;{NyTz`sQ#bdI^g+Fm^mQYn4wwqe#9B@
zI1Y%v+4PuBf8cX)`E@_9OJ7w_NKbR$Tjd)Cvfw!+keb^UlvO>Zsp*$GXsL6#D;eG|
z<56Q;-KV{b$xQu7{#vm8+jq!-X^{Fwg`D7Wx25f(>iW}VU^Yk`88>aZqf(g`2ktk4
z#QoK6)p<8>0FSqT_#iBKTN`X<eZd9g!<K#EaW=7SW?-?csp??0JEV`G_%Z3>Uhq11
z=7m!YPqRYy7HTJEgY_)8VgiSw>>C$t_w#;meP`dYg3V%)ZPb4z!T@fsf$Rt2&kc`s
zP0a4<t&A`Nhr8?lh2U^eS<I_`@WVZ@*h$?!u%4yn)4+U?J{_Y56^HmYI&aP0!FGbg
zBvM(yY2nqo1*&Cr_rc<+HB$|4+Md?E+%61{`_KXb)#)n&4f`_M6^~^g#%QZQepHRP
z;;UM-YdSpK8l5Bcf{KdNdj5Y=@V}R7*c`k7E(aR_Z!7iC`E#rlY**66e6St%PnFdr
zk8KD0|Iv(m{p7>v!D)B@e-3!stvnd6Ji}!nSYPW^d9~FYO=>4qnZf!6re>=-&z=GH
z3&>ySSmzWYN}Q|ZAoeP`PXDicL+P{r&&<nuW;*-S^0Qb~K3rx5o28aH5$<-7nIbGJ
z)!yewg2i5Xl&bh7I_bPfHIaXPV1b^RN(4AgJzwuq-67Vg`}EIou-icT2;olGS#bAN
zPdKNqYUv2}_rAhOU~whhER79GJHg}s@`)S3_JGWUjsJt#v0{R1!C{c|WkGxp7SnN7
z<yzMbR}T`?zo`TcL(up?_rC<iS;wRdj?QR>=Rc4d5C)C^Pg-G!qW8lE$e1=rUhx)W
ze<+9z!f%^6)m2^#tJZT?f&CqG{H1}{VMcHsxxXK>-(rhIJ2+i-EO!Q{OQV&DJ10SA
zv<DWecfS<Zi$B(HnD;7QHAQ*8>P`EjiaaY5)V`ZEtEWBWQ#@)oA8bCmK{z;$^h7Z4
zdjpvT!#?LcRL|{&+&5W$XuI0GJUR9DTT<X~1*vJ&Wi!~A{#$A3ghcgSCw76wtL8j{
z`+KXvZ>65U`}En0Ex_q>{e#uu_y_3)VM(r4$`d>qb!L2=r0i(wqAwVF1#Ctdhc(!J
zI#SaV&vq_RN$=SJ=7aPY-CPXzzwUtya6F5$s_8D<B8y@!NF8YWpZ!INifrr!uzHYK
znfo`e`5^YLrR-pKD6^be)ZT|m?`A{xQ72{|)B5K-P22Ns54in&|7a#y9Z1jHJ;I7t
z{*>xD9I#YPj1&f&+qp^r?lzE`>)TI*#h;d28GYN~Zg4bPO>U(px30mL_i%Gg*yJf^
zw<Q{`OAiIB1?khC8Vi<RZ4nO+EA`|~z0*8jRVPYF>Q^5KQfM!L?B8XL2!@9*NFNAC
z-F*et17c5OI-&Q_*hN)PcL&)0e|?p~G)NAFV@>3hF10s+?Ok_mn!fbH^#*(9DXW;9
zII8K)P0(eQn4t2p`w5C3kUkK$5PAwuKVfB+aQi^wApA9Po8Hso-QY9|;)C$MZJPSO
z*Vlo|*vSH`z;=Sfy+0%JVbtDZYWMtZVzux1-IHK@)N7daJ3m790zSR#qT!y?1{MdI
z1;YDHj)KEP^(JH<03-&&Uwsh!#tME3f%R-LlQgJ!2njDMu^(XfiWD}0^*0La1m`)B
zUJx$1{tYZHSO;0FbtwXJeu>)o9jf!TODZfn$f0JizDk`*R7(wX|KBZMHnoEPmsQP@
z`qi4f>r_Byc)mnEw=j?UIyk+A71gVKkei}5ziO+pAZv=oO5JRYi)P$vt$K&lOi#{I
zn_Z=$<|8#%t?pBj%6(H0jXVi0aJddL6NaN=F01B6F9y5u_)SP2$yuBV56gvb1vU21
z?N&F}Z&q_Y4>=1nlyisrl!r}9*V;4Hvf0uV{VOyy3}X7!y22N!yPYZkn`LANxi{3_
z&J!FSQfnhL_^<Y;{olD4>~@g-y^7M{{0Cy6IyF^=J0oBH)^W&w&zhohYU_hus#R$k
zfYnE2cdCco)&sjg;oSu#kRA~JxX(pRP25DWE$ojHvtyge>z9Z<vLJaF&XhD&UXsuV
z4$mFYB5Ip1>{ngj!K*HH#X+V0oDSH{do&Bc^*(6)-+rgJT9k9Ka^-~rRda2`e(&=s
zh_isWckEXAv?x^l+Jxn5YyFzkUiSR}n=|R~PB3l1h)-ELB^~TX^Ua&pw%$`#J=Nn4
z_7BKi3-S(w)wSA$s7`$i+4Bw(7o8oUDx3kiYu8GzRej|h8MWd^pVUC({}(MFXBC0g
z|7|!ds-buYvcDXp4}_g&ZdN@~%meEGEB+P~R*l-oqoF1Z*}Iusm8f`%<Dlw7BZwJS
z9*Tp*<GH`C#^b+`^W0Bo+o`;7y9IWaFC*kEC0or?%Jah(sCECbR}<n_Q@*lLMg1r1
z71iks!C=1#Tz;y$qZi_LkQ-n)?EPx6`4>5-sS1SNfY)&`sT^SQO(suPZr&=dCMU|M
zYA!FXUX)m;?rh($+QqUM9A?*kv8YGdBEsd#`}eAgCL`8XR9t9MGb{{H|B#!mB)1|R
zZ04<{|5W7f@vBepMBFjk&Djlhr|gO6U^m3c@TzfXO;q-rJXgJY36EMw>Km|L)kqon
z`ftu^jVB>vcQAjvlVk+X|6A<pQ(g1gRc-sc{b2WP-MAZU-Wto#>I)3_s($yeRqwcI
zt9J6i1#p_@vDmM&u^V#c2Fr3towoTGq^t+k{~2NHlsDC8sm<9n4{RRDJ|`C!uv<an
z|8BzDz-*8>2!qD|K{N=<Z4-m*0rA(r?N=+?R;UJ>|91(Tpk}8d0M0L<@qdt-r5o<4
zF4pZ<Sr&L6>|aH(<EnzrZ@_W%Y~L~EX_p!__P0UiG-cda)Hi<Gth#99BlS6_B-OaG
zXKV1UQqh>NQL1W@*{Cifdsp?3&~p{*3P?T&nGeE6Irb=S*8lFM8Wms-cI#SBL3I_4
z2kJ8=EYu8rJHc-Ad|jy~vhS6e!Z{W7xek8nFIdmWuYLkKw}5SZnYtuH1URq9-SAY_
z+L)!d;pG*zzc-k`?g!b&*t$ct-A7pU;_8b^NooI7ckH!OxMqJ<f%PtAeMH~sooaX9
zepCaE{~y~BqWVE~iK;_jmHId4OKR_Kn`_Kn<f*!<S_yRikJ7$Z3aVz$A!#0DHmLr;
zZ15dym+eVNd&leztGWhnr25<G*6OmMTx$9<IvVB%XH?G!Ur_25jaJp%2T4nHu{+hg
zrYfjSo#rpUBn)!@wRY`NwQ!jrr2zXy8ja~(8rB8}!Qm1v=mQQ5ke%n6*Qo66=>_wX
zib~a{E`F#s*@jpBR)?}$<yuEIf1P}=dmMK_*FveqsvqfWRuxsb12zj}MszTPT1p|S
zx{y<_nub${+O#W|K<ED`ewYDij|8on3N|047KUTIA!VupCuHr5S*M74>sCe;x4k7Q
zf6BtacHVddNvB7cO~L%MeZnf1U2|1ABd@CU9N|~izLuk=R2`<`nGvAYIKvg}9?;#J
zS)WS5X$)kSx^KJ61Gx@xISb;$@ZQp?s$1?((l}$?3)Tk{w>|?oH_P<YKUIzD^XlAP
zsmi-uwy7$8F;+ibvrf%(r@Wf}j>YOMT^ewGjir!#V~WqRt3FbOoK*la@AN8t6^jK2
zlrAhVQM;V_1Fjw<*XX$s%#Y3bqxw$?vQFS*w7a^)uP1P~w=ebpt8rZFp`!OUTXnmP
zhWe5R%F^~LBV<_EMASg{|AqeAg_8b3W`pn)Lk)14i6wV~%ZyXZg36%!U-^}#N=()@
zWu7~cDxmuRAfFT1zW0yB;O3y$|Ew4Pg2PXIvAC*wrkCmspTkP)Be$#S_(JYoc^<;3
zbfyJ*j@XiZuz9y<<*BUPqXRb+<R%a<I4-T)xavAs&5~xwS}WDfh_y<dy6mdw9A#Da
zKG>^hHWzY7bbatOl@+m&`(#08%#Qt_vOl(3<-s*o6}{4H;QR^_2Vs6iJ@7c`nd2)}
zH+DZ(k^9dBmoLp>0NV>vcl`Ne#YM4xD#j&<`yhDYf2thNc?k|Lt)6z(7ZIrIp+Nek
zMKh~_>VFWssG~&DaTVkoB@iDUzWlX8b^iWjRi}Mr3TKbygTv)(&rPsfLHg`Z>#9!7
zj8r_v^;-E{avM0jLE`B6+PgFrcF$#CH@>?DIU^XRro)t9C8whh>|e{%PgN8Rb(FKV
zodwqoAT`g|BhFM>e~uUI29UVTPDAB|{}{mh*#hy3Nu8`JF`?VRe2_c{uhVu^Hh%`$
zI|<^$aL1b83ZXSDV6`7UO;&FHnXPKEIu-7IArVwJg7m5P)q>T7*dY9;0dj^3hz-Ko
zd(>1uPF|(5>U}=gFFJEifa6W!3Su4R8D2;jg4DzCZf?kVl6$tPE5~f9gomwj?NPA)
zyG_yx!fR|*g{GD(*YKZFVG{3B{hAxBD1Fu$Y!8=5it38_kopN`=H-TZuwD>9Ext-Q
zwCszLsLCB>XRRc#_`S4WN*9Y-z-*8j5a!u;TLwhK@Z>H%m8Nss6&@P3DA!)xr{u2y
zIqwft|1%rELeUG;gG{r2`h=pU)leJEkFx)+8h+Pa1y=uq^qj0|0f&9*tPXG(zmk;)
z`wb*_=;kai`{s7UzCMsRF?iul%yWV|1-ilR0O`MB^;Ts|-Eue|BnHDuS<}I8Vqey(
zyh0ChzPZDuWs1*S<<z#C34+~pvOQ15QE7_8GtNHMS2j1~`?qzfNk8UL<(V~2xgtJ6
zW%YvHipKLME4i67fYrmyj`Y}$!oTuxx&jX;<P0p2cWtVkIgeF$Jh-4N=?rNHIsELA
z-BsA4to<%o_1>`(mG!zN@-IsK!0v8e6r~dR+zy;>49<6f&24B`R4upwSvL!^lYQ=9
z6`^Fv*dK`h?Lw+DsQw4BLHOKG3FRWypJ2Vb8ve@4buP-2T^6WlZ8lQglk-*es=1ib
ztmQ6X`%b(}RXq9#QeJ}ef^gxr|KPA<E>Hx=U%f7q>eW{|svbU?D!%#rN@u=7&R4VD
z7YCM)ei08g>!iqe<)4@KDue2O$Fw_OGwvOO)Ft1Qg4K!xAE~Zlhm02#uS3LJC@YIx
z*H^@uip}8{m7V(Mfc2lgFRl7^=5;Xp)bvg;t@E{1<@YyJ#g^rF!1jXNJW*0zdA+>9
zYKzB06+ia`<)lUHz-|M{f$+Cmkp3fx4Z|uA7{KOrOb!66{TuNY9PZ&I?aE$0%fM#4
zzu^bdT!DRH`c1@aaJYWef}FQg@G?=^{fnKFY3NynPq~M{YSw*XRBpe&8>|jy*PT<C
zV@Anhswz4{j;i}i4}$H7sV)8c2F!oVG*M;qBFK7O`TsYRk7?+u*6n<vyg;{6*)HBi
zwM|P5?0%5G@T-t<RhHU$Dl5`a+Zv$ze{=Sis9fmI0h<d_55h8En&5oFzeQlT^Jnf;
z{dyR2o>hGGcCa`|JqT|)+^Q<QAxNRhW+wRVfn}$bE3&Rf+-(9<^Yq?ol}~%-Yi`{3
zL;ZG2m{y=aWPIb2*k-NRm&vLd6$Q;F@?TdwZYZf~Z=kO{k0n{{_@6#SvGu#us?r`S
zwkY>&Zoez3bMtivI6k<$FQ_!c9XA4<|F6B3OFwd_BzXQ`h_}N`+}GC3B7d@S#L-CA
zZ?Y`r6Y8Q=OIG~U1MUCsn7hyPsiY2Q{$I{_R+<K=%o3E7R1sf%SBw9mrPlhaFIw`G
zmMQ0K5HvGYTVuSEb*sj^KjCKI%O@MO{Q9D4U$D^ZWSE!ozuYq_W^Nkl4_<FG`~6VB
zz<6)C*`2x`^^`eThQfAX+J^6)z;ir3G0EULgx7UnWDfmyGtszRrg1^_h#JeiJhQ?L
zKUHtmoYMLb60AOB@<bK8z6PoG`>Qm6`OBD^2S;nY=e?+AvbI{~j@D5%{yX~0p!xqy
z;U@KN4I2$c&pu7i{{NoL0ySm>4y|cZ3)C!UTIv1Dn628$P^m9;($)OAGp{`K{C`Eq
z-IDs7?B$j08C%S5-t$w+N#CaGWV=({d%cN%;7tRgYKc<KwNGobwom3ZaZ}(><jQ{z
zcE8IsPIXK5P}3cIP0dx#KT(Z28V;_v51gnno^;&8#KF<tWcdOO<)0VA%&Sgxs@+&5
zsM`Brl3D7+Dz#NtGQjiyz3fYk7ETn>dg|P&#Zdl2ZBj{qp|HqnqnY=W)idmqwF`D`
z0H6PFwZvc9GQY$$hS5;V>ou>r_kr0)?0?#|ooWtfRbAFq%Q>2%rm=TBcwRb9w@~R)
z6lDG1BjtMV`M;|#b(_?EQqo{fc&wFnO4jUW2?wbEuP`xH+5D@3pz*_&1{ED87LDYp
zM7bX?s!aoLdK;=Yx@by8+bLvpPnFxLw^+R(UBFZ+qd~)Gex>Q9{aRWH%|a@3qE>?Q
zlwI>v)gvCW)tYk4!E<OV_g|}hyLv$N!<+r;w`B}fTo)`<URBSjo>I`Eww-yl(tq=6
zl~%!CRjI{Ide8p*sd={)n6?{aDs|@RY0kW|RzvMtK6w5=T*Xsu&bvLv7KR$867Rq0
zr?Yctb@ybe&0~A1c0gJ~>!axlqm8}oYDKTksi_(%t8RI4SarUfEI1ry`Fj{y?zS{#
z3#(I`T%u?e)cIR;@s<V^NBL9w4z1Z{Wz0L|LG%Al;<%Kmw>v0-=Km*Jg@DILR~B5<
zG0VTMc_h+A^PhXO*5i2sW>z=5)IszA?YV1BUjBAh$(guR&7qgkJle5a@$O14`22qp
zBa4d69v$^Ijy>S~c#Ab!t#P`Yy43Xf${eql%!91-)pFd9gVR1}{{Kh#QSHm@6Ey=>
zzo}X5Nmbv^^3>GXx=>61#6iWgm*fl^%#zKV4OeMDR<H-p{~M|BseE%50P8=VzXqJ<
zzCq9b2krlVct=U4+%7}mndLj}4TAO>D|SuPVzUfX6cKUO?&vH7_5YRHj$cx(IJVy8
zQF*Ucj*u~U{{OC%jQU#kg~ls`?kESP_!>WdB&yhYH&5+o$RoW!pMPq7$_-TISS+id
z5@T(0hRe+CDaS!g)_8VRlkf|=GSA#pCngAM%NT9cdRIC@HD%FtEtXS%R7;iYHH{5k
zs)6SJAG}~zIF#k9aj23<^#Jr->k<hj^;ZkBw3g1YR-YUmp}au4SABiYPpvgqf=!vN
zkEjb>?9r&$Gtsnjy|=bd6pPNWdDl$;y)o08eg260RB0AXuL<wemKrTn@wq&~EYU60
zZ0pU7s+vxwCe3A9YC+d@v_H)0P-SoB)qV2V+IVa7bJhIlPiDWh1HkkDlFKF3?dNRP
z+ws6n)1-=5^}7gz3X@>7zJFDR#*2G%joXu%wHoieRDZiiN{#(-no0dbJHs23w`#DT
zn5L#LqN(!rv#REG>(dHOi!F3w!*-f&+PhUN{(z#^{<kyDYSY}+l_HjE&MR#+G28b;
zUD<lO(#AJsY9AbytJZBu)-+kKVrU#R*Cb3yPO~t&N9US~t*Q0wB#kWwv(y&Xg&6%}
zt}*2<eWae)was*YkE{CO#Hsq#`4bgvd)qaNzn)OzpZMDJFCUZI*ZVun^e0xC_urkU
zKVRdMNxaq`Qx%6@dSAaRR=-#`LwiTtH#4L8zTo-)soM?Ir81mMWZmbge2-mjqU>Ox
z?xMNU)Gp9nKj2Ctc>lk6uoHOxzk@*mJpcduaF4q8m)%-ZMK3D9$`l6A|645MQ?E@t
zV0Q4=dCmC=DXQ<zDyiQ67Nzzq^0N}7RkCKQu&MGo({`PQ?$Tz9&kL$-%C#{SxiH!E
z<VrD>TPt+cLLN*sNo}@J(_VF24K)9Msg7Uy`8sjUHL`pfHR-D0vQ;OuLv2&2B>4P)
zg{LQ#WT$s&F0)ytw#ALlbjGBkX1#_6>ZYpZn)jaF)>K^UZm=M(%gjYaLG55@u32i4
zou*OS5w&$WhU#A}`qbz6_iAXg$ZD=zFkee>Zk>#k#tpUVU~x5{*Yg#PbMLF>mR~ZJ
zH=M6loL^&>>_1xtH2<$&w%x!>akE*ocoKa6Urbh5d(QrKnr46GRq~BWw4Ptsr9R19
z+xUX9htZ6yGTN`#Ez~;sS<*DDvQz!tZZ%Vp_UmSM{W!Js%y*i_lnJOPPgGX|&Hra?
z^;dIW+pL;-R0=%Ca*hMCPh&He4VZ73XAEv<Y|dS&HgCJ6qJ&tqn(e+Sb)K(NLF505
zi+um79`)a>#6FQ(okw;Lxcv2e3F#+*+7lq0xd<{gY`=G_+C^hQb<q01eavkd=QgKl
zIPN;4+&-gMjmtz$Ey}W7VNxcO`l8rx3IVwW8ZT2cz-|DU0mB8nTUEcBLgsn9S3%lr
z{tp|#ZUm|6ydkgQr!!UE#$vu&zyW=*d%V<ks23b$Rz7rgGPvLS`h${&27|bo?a5^I
zD>2UCagH6oA!lxC-_HP-lg3+1G~yC&t6A|P&evQU;|Cs30oikxL0V~EV2FCf<5gh&
zl8kvO_6sD{C9NTQ233BXRBxJD2zLL>Cd3{K>D_s%YT_nJg};6(R#gb9HhhB2!+^|0
z#)ooGDgQ`;?Ac+d^Hk<`JFoWe!XGvL3);%bZ%?T;*1VBVWkK92)qnh`+O$b6%Aaj4
z)#|F6!EOSXXWWqmZv##;UZTR~8LG~kc26yn_n?AB=s~bL&6T^sbcyI%CG}V}upey}
zOR0<WZc%Ydn+Xrw+sk;sa$?V&m3KnV*aq3b_D)r0p(A9E-1a5r>TAEWsJTvsjBia*
zg52*0TK~W495*;T{!~<f;{~LbL)BU})RP%3{_j+Ys*;tW#^TkGvp7NZzZ-vxYV<$I
z*!aCoSHbc3Mp;3luPy=XujwaOD_-rB0=w(W#s^??*QGV7cGf>sJNZc+JpS(`V5`1N
znoV9vuN>@`GY@JMITzT1^?=*}!@jB}VEImachzX`Phh)1a$nXR1KU;mWu>w}@H^F{
zrT10VXX>l-T<%n7(-Kv4{M8RO>sg?i`d0_YUNDe8kzM{OuEK~jIQL72simxPSKp$m
zqw=cC9USH#h5A$@oW#`Y9w>v|4$`;j1mq5><)<Oz|HTh`)pS%+mFH<Ts(U|lQ!^1r
z2HU3^DWkv;ucelF*cGl9r2mUxG`Jpa%k@&K(?6-~HT4WwJxC6Om4Dn(KPz3XoUXoF
zosqK%H2x1x^Xo3g$h&Fm1BZdSy&hQ411HEi$+9ML3I<$xs<(p=sA|Nwf#pDEA&vjb
zRM}5JamQ63$XEhMUF(whVE=;HAiSSbS<&5eJ6!DOqXTL`xOu_p03;8>#f1lzr-h2E
zPL#X|Rwo|5P}SpQ7}$LodT$jq4)th!GM)vt^Hlr`wPLFes<ndh8iE{eRAv_L(s(&h
zP2;4{Q`O%aD%AJLG^oyw{jY3$sT6D$$b2x4sRo+|W-~A_e7(Y^@@E(1&h8BlRMiF6
zGiq%9rKVPOq#W$FC8oM+x=Yy9ubp5}-*Y=&-Rxbs!hfD8U^Dt}7OBsFe;b@`a##8&
z`+7Z4)-g0zFFN4@x6^mMuIl>i2Pz*fGATvYNU0?)W>LLne^s%8Rax=Cj5I~Lqg&KK
zb+ZNg3k6x3cPdIur&S|gma4O9{#HBJ?xx`}k6BIl3uGSd&8F?DPc?7C&5qr9L?P?x
zF0~7*H54k<A$L4&U8<$-Fk_xN|7l_M`e;kF?#baAX2t!=+c!K?z4((+{>cX(^~U9=
z6*k8mQd>D$3^e|)aIbw0*gs0WuhgoRxF}88aYUo*o4Ll;xsW~L_N-RmumFYG&tM}}
z^(}M2{I#0OYC4a2)lZbNsLx_nS7T~Bt|stB6`WQ+?SPzNsd~;&eapli)kh7N)W1tg
zg45m7E$M1`E0Wahw#lfO^fs&Qy}<%DuVwEha5`2?S_I~U?1SOI8*hQbS)l-OPRPtK
zOLhJ&i<Q5>HCFZV+XxPm%2vod1?pb?VE&`SdMc~emZ^HT7^-LQu~u3c@1VAJqNG~6
zeXiR62*_Bu=H!<u>f6P^{`ddFp`4^XL*ZSwI9NZ-ZV;`SFQOWn(5CS)_7vEEATbaw
z@a|LJx^|j^TW6*k!^-pO55I0!UAb9I%{?t#-RbyRwHqtr)%4#jQ2(8;r*1HX5A25n
z{*ZMu6}jJ371vJzmo*@>HcYvxY<pQt+1T`=+A=@L`aF<02(zuY2(~x!wz%^3U7TR|
zsRV9PSN4RAQ-RdTFxY_A^hbY^dvldt`5r^Qy7Y!1MW<3Vg;Nnbm6n}auJkqoaz-&o
zJqTYqRj(*>)KBw$gsIwU@yS{ma|^)h1wIRiY5g@QRn={3HxF8IU&T@Hx(4X{pDx!p
zRi)ES3NHH>tClRAqOjNKtHzJ)4*2?i(EWdQCxuL0Pc$gn9M;nJXbdtrte~oVR?W<O
z%bQJRvpr@hetdsiLD_AydF_r~m9}seaG%F(%?4AJkAKV_eQj5mvdCL~@!#{xdskjj
zUST7lRrjn~vu>e>*1Q=Zs)fcp=3B3?G10VqseUsp)O^Fje#2=lhqRt+<eHzpWTz~7
z-bCeNn2&1kiV*YlYa<OieYnkYUbCs*pLf#m`^+g?6WN+HwOGv*A2r_vxApc-yDp!-
zfY~&(Y?k`^=L)LTX0qm!T+&o`h<0dAJf^QMQ11YapFgW!Xt>O0HFJ+?)BN*SN6p4U
zT>0;-r7CX@PgVr=|0f<>ull5*Pi@QVBN{907!*PG|E(}E)(lMLP<yj0UN1f8nabsd
zCHgrZL@d_E%1YO@ESKQbUZKEVAEUpo)K-1}tY~vkeB3=Nrn=JPmzw!yWBvTC%*Jd!
zv6>|o*;=z?icLV{|JN)1!0s-a9ibMNt!c);_PM#e`3+@<W@T{NsuPkl){luX>2_l@
zm9yEZ?3vnZ;VdVpCiU)${PLHMp#6VpYZ;${`>3Ut*BND9?a<1+zfg0wwzKNIf42-*
zUo<xE@Yt@gaa*kR4L@yF(EWc$pJyn_Xgo00jz6K<oA=N>ZZ)5=@rz4Z@g~Ar@o_5^
zN?wJkwp?!kyC38S(EWdJmdscFk!PYhZ?=eHl02Kq^Mw)`D>_`Y^uJq~slVg}r_&Ql
zmYKJCWSZ>e(NHeR4F!wU+AlN>I@N7BKh_=G7pS|#BKy#!K(%1WN3)KMHq8Q;471e}
zT)^x9SMv3O{cOUNuOg&&N@d}^G_V_dA3s(pT+%9?-7cX%w-K@zSoC;<s+>r;n(otm
zO4}=IROG_1s!ZP0qxWR-301AiezR{kT$G-ynxWx8@2mR4h}YmWIaP>Nt$MeHX{DdB
zspd;#gJ}u9noVn`tMaX@RaO4#u9^0Ez0uFh?5e?w6xC)HuTz;k<GbAazgNIv+a1kn
z+_g~L%t(8N$_lPUX2Gh8nmICaRP7d?(04J3HP=zoRQ#E{MyWW=Mlor1hth=&<%*84
z62SJB{4Up7IfYl#ZXSo$ZQnzhOsDsmod`Xn20H&o#Zv=p{<YK&HAW3<bNOXXDn8QU
z;61ybuwN)3tC()1rz)VL4lZk(4CPgobfUp&xH8<){P5{!l}XFZ!2NfS`Ujyw+7lx#
zY1W%eRdx9!tGQ;Un^`UEbkO~OiXT_aH9W>`WbR?#2-^P#E}NCTJQX_=dcbLLBe$>u
zX#d|D==pyj^Zq4Y0N2lamZsWoc3o4Sx#+5<UBGlzxs7SsjtNV_?f~8YC$)N(>Fwn8
z+Rm#)z-7Ih>toeZQ}axkpQ<S-A5=BHsdGW$+LvzC!#>;fWV2Ri`Lk%L{_wU_OZy>W
zlIOu_{{8JT?Tq|WD(`pQ)jhTHpVEos2(5_@&05`}E0jCBm9!?hm8-fwZq%qcF%eu=
z=6UJJD?d7*zQ4ItNoPhISd4p{mRj3=U2VrBkaddzbyw8Bb$Mz1ulixSzeY{9ZkfG?
zoxh-2Ow<Cc5C1c?^`3n&OFf#V8D8n7vFneD#&TgswccH;Ro=)gH8-BeVg7aYM76XH
zuS_yDI8|h~9?+U5d`4xG>nxqsp_7bm-%wR`YMx;3#{)V4=XFA)+Pk|m_0-x9Y3!Aq
z2oAFa@y+_GbqCaMFI;XqjagK4%1chQdWme+dd{DwtP9%>r~eYwIN`+ty8lo4xK*kK
zsQ+)>a7&xZZlZa#p`;e~mCKq^7p|K{H<YQ?rtH`F{A8us;@)qn7ar6rYF#N(HJ$!n
zr7Cl>hJTTfq2l6B(;IW#G_)rg>O8BSY?dH2PhGv}qROAd2Zm>!Xqr6~6jDpyEM{JP
zAV%HT?Et9%FTd}%ng*9^pPEvgrnz3(a@CDHnJj!i3tH@2aaq6P*>sb`8K2FZ_HWV4
zwR)!(v`$Fdzkjy5?%8c-EPW4^s#Y<owwCpP_W!ALFt0I9{T-$1<JxXECm~z^y|ov3
z{hYAm5pY_#p8~nRV}{N`HM!j<v{v!VP<Hw@!R-B#mGTEJ#i;t+T5c|W?7N1EGMg%o
zik0&2JHo1ADqKpFG$(8Po0O}x*GN=PLbKWIb;4;S;|bqPUH)G%V|*a1q&s7WN_^8I
zljdGw)tc77Dxm)V<=MxTwv?u5Y!lYd%Kq&HE^|5_)vJDF(+BPUQ@+dWr&MINO+!@P
zT5bIW8B<}C&E`sCdg>R2q%@x}2xuPLdfuQ#pWVFpd%nsy{pDuI*xNOPO4h0UalEI>
z$8=3?;Wsvole3O$=my`^WZnBm{!ECB+T$C4Rm7RrD)1&Js)To*2jBnqX6=3O*d<51
zmWoUylVL{VVe_uQJu0yE|2uEiX{m0#4i2AlkxMn_Sj4M4-efVkX4z*XT-~De+1pP`
zRy5TlGGe>>;+{ye1*Z$mzvr22Rt9rhM4#;hum1;~|D)#mT~#lZP4)C^$oYq$JUfa<
zLtr!nMnhmU1V%$(Gz3Og2!PK20r|L<;V!ss5$u%!ZW}+nyj$I3zqiVnNA~Jh{)%Zx
z>$Iwa=Kr-Lc7xCVQMH*1KK}=#*NsIV+^z?)K{)sES+E%&1H05+0+y<4>V8+5{X}20
zf=5adbpDTxfvx(+&7tahpYf}$oIFh(bpB5>1Dj@L76-Wf1u_GMB~)Ll#mX^&-DcsO
zs&?w@AI1J~#C=iQ4qVqLp5v#XEM=yydlxePc==(8hB8~fii@DD8V`$zQj6Akjdkwg
z>c<>ZG$e1#0QWPW`a#xQ^BjkaN!`d6(ad^Psb2by4J;3`|Ktiiu)QGmVV)9|*>+qS
z+6=8=`K~qV)Ouw$t7|(?1&<x|nx<%&zhncuA9Vf?NDl~0)vZ;N);^=Gc4msw@(WW`
zpDlo#n+%de#ukOQR6*zefW%%~UR1qhS)#5ZbWq*Df<dKAs71ZCCQ?b@Gi04kfkd7X
z)2l4?%iQZ!{-ialXY7HTmj}|@n6ChKABdgduwJ!yrn<&N-Ou3jf0$>@1k2?#LdLLE
zlyp`79U)`!UKVrJ=V$P$HP4645rFi^d|m`r4?6!xM@SdU28(B1Q=N4fGR}ADEvtqT
z?^QKhHz^I!`hU>*KOi+y)g`JW0*Aoo|Lj*r?6uUe@=^;4*$JBeSM<zTqn1DMg~lS@
z$Kdn=I{zn7bFSL@zYza`&i?_KRhjfeqs>GE+(%;ZU{c9^4Y|AeX>T!DAHT*i)x1p=
z>h`l5)IsO}IPQ7{KL2O+L&(~E(D^?Hz9QBiTPntY)2rfH$bCGmadm3Bvv0%K0PVS=
z2sT%j=eo-Kr9$eHE*Pp8ah_A(Q(>VYxnrYRMX)6}UF=)LsG*e#xi=H!PUAgNYF8&H
z!^0`6%tW2tmr=v<b%2^hDKA)j<t;6>hY}amLFfN~^nmco@{8a!^x*<zADnUYY4vYe
z^Herm_R%<Ztz6ysI3$ih^Z%^JgVaIi|A6$t@XzJ5l~y60{}X3Y0(Kv4{eP6&WbpYv
z6+hH88vm)Q&tiHFwhwgv&y_&PeAX`RhhX=0Z-lIcmo<@7Og?@@Ei}JZO?tm8*gTMZ
zE+*%}dO_>|LFfN~#9$bC{eOXdB-lKd+^rqOY9W@U>ag|ymwNW8ADFTcY%Xm5zu1<Y
zs?$Qn)h_OX?0Yjh|5+`~><>6DjP-Y^1cjJro?CSr9Cy1J7OT&f)lrX(J*IJfX_Q)@
zY_Vo@<ui>wK7BR5wQd@h?(R_I%nw%OyN0@6JHUhoY#z*Ao*4nEc`hnocZe4-YLsqT
zt|9izUVXnb<Q(tFuu^rWf0xxgzj<nW-!7uj_&rJWapPOC8682Q8j>##gVUuQ<2lto
zPDj)}C7x3E-gyHo53<8#YO~tJNBwGfj0cqhd*sx^^pC27_Wu_;EK^yLFk9vLnmOus
z_#MIL|FD&stL<2PMlDFwQsY9Bkj7cZHyTsF^Qy<*UZegjuwCi2FQfX^JEib&joo=f
ziShJo^=`X6Do-3CW4~tB-_%_h?KFapzEWSft5Ut!<`4M%pS!%)p!<JS^O<jf#{U%$
zI&W2HtNW?8_U$qibB^n3H@3AaYdCeN->+g)-YQY6X|?={#uTeP;BZ-7E(0#tK>o>U
zGE@^^VhiR=9{i@3b?A)xvg7;J+l<uJOXj4iTg!%k_3UF@4|b<8JD-Nx@<g@i=`9-H
zB_+Y*bv2T6)o%oRRDb1oP<@lhEcGiFR%n3E|BGG?nb&$2;0-n(WFHK_OoZG^{^#H;
zu(>}NSTwHGrl}lfxT?DSh9y|+FhdvEZsul48czAKS0(qLf|}Epv+BzVCMqATv{qlS
zS3`ZN|2s9mN+Gb?^-Ee*&zo3-<7*?^9~IE~Kj-d9g84AJK{Q|DTD5LbXU+SQv%&g6
zVj#@TA)sLyU8$5e@2*<A%R&tcg-dD@9scTJCwVj?HWaB(JFTl8upmg|tuvQK(rYGg
z88b!X6xe(b|7taE8zpdl0+|Im|7V%*0hI}%H`TKYH>23Y-VN~^==`52fsiu#=}RXK
z&b70_W`XoO7_ox&te&i=(&473`uc*D#;Pyu$}atTlzmj+s)FYK8|@(Tw9mg}g2xBN
zpz91z`$5*u`>YlNyAfm_2p?&)R_~eWsM=U;0FJwdpYEz`;qH*B7hSHv!FdX721p$;
zjxD<e&ZD2tu2Q_QZLw<bqGt-ZlY7(_8}zF^sK2PZ_`Iq7`o>op2J4?GYJ57X2wMO5
zyxB-iRRS`{fo#@mRY#@gzKrlNs_5cZRjk<pj_)U2pOuVrUaBfuN2|0wM9fjiy`8N5
zQo~f~ytA`f?u0|?9eW<D*iVsB`}<Z*wWdoI>^AQ?I$(N}dbnJ8RH3~0&Dn~d-W^a4
z<Us5f`d;k<_A|&m*?wJ0nQD=0)1R&e-ybH)7^6}WT@DW`)$Iorgwr;H%{;c*7cTzP
z)(gy5J2M4r7RxcnUE3ftL0G?n9W3tUu~fY)F$5f*>@(hj%amd<h`+0H#gyXubl~v}
z()T2+P0?kxrwYR%PW2?47jhsm5We8(tDc*!tUBeXI>arXeu2}3`Bo9tqXy;RaNB+_
z5G*!vVUwzmoFLf#KmV5~O|^_uJ2@Y+_YGv$lns#mI6X5;6h6IytSyb4*QQo*D_xc8
zKRY;1SUxK%cC?*Re$w*{Y~S`cNIEiR=2K)XdZYq6|7TKE6xd%NGjC?FDy?vBQ`HMk
zQb;^^OLg`X7u5rzzrpgL`TuI3ex*kZkafrcJ$+#D-NzNwn4Uo9fsJ16SCviA1c%QL
z3&^>j;_=xE@#;M)f((!~{x>^wRgzTxseUzv#PQDqE0lvJ=7Qq^<OUe#;gto)Wy{Rx
zC}De^#S_lIAi=FD)ICj2w!c-8?F6IJ{`gr+2J)(4zgs@quBNvwAM8eunKsXVtGbsx
z2g|X)oFl_oov5t5=a&4`j9Nt|5y;+`I|=^s*W+Z>mp%Uu4tJ0~mkvmnf!HAId^#O0
z_mHUpY_@*a5`^28w<`s^bb<XZ`HUMJ=7)9V6jvQl0pC^WkojNvq@6!l-`0)0!StaT
zMn&hRiHb|NZdMW6aSy)N;9#S$Qrqj<U^TDz-3O2FzwomG+YOrkUzPG+?cwxv75x--
zuzeu6A!A1yHx%)vTF9CjkecFTIk-I_{?5;v72M8<D#Sb91N%kvE2jpN%4M+Kp!0t~
zYL-?<gVSf|Hprdkd8MYR-F6LN_kiRpE;FjhotmREQ_W7*$bdm<dZU(#w$3c2mv1;!
z-4z^F>#g_|bf$ADUG-d~c+kuRY|qT9RB#x9^n>svTXDE~AU=bikurC$J=h=H{w`5z
zlANfb@q|TrXMj0a-Ju5pYH{^fRi4(Ft1L`}?45eAJqaAgApOGUr^#)~YgC_63)%M>
z*qf>%^g0^sFOYnf#dQ@yYYP=`Rt=@BwhYzhfjMBi{N67D)4p~A%CBk?l^4B}0lPD-
zaH-<Nn=GnZe|?e<QnLd4;md<~<<cuV)Ry08QsCaA0MF|n^DjzBE30HOsyx501a|M0
zV~{-+y-TMn@FW~l`6&KC(RtMuB{8`fiVRJaiVr_p$v13mRCSydC$AJ42-Y*}3}l~A
zrP*c08m3hWQudk}0&S}mtzSUatpCkMy&F2G^90!K@2ak72xfbzzxGv;zsM0Gf8IM?
zWzH4Iy3^@~pA}30{ZOBpa|rAQkXv7g{|EcAa`txBx}R4SuiFQz%vrZY<-M)93h4fy
zJ(*{~<}UVv?2iNKfni-wQ*f9T&E6|@O_5t=KcA|y?Ka4qC!f6q*j&f!@4;%SVqYut
ziRh^;jQ3RZ5L&Hp@uPxTev*WWVH&II=Hwr0ciK6XoA?mtZ~UF23{LwXyKb$QQ4CnM
zNxtnl;#^&jI0y^8QB{9>a5XrMg7_dT*U_Ugo$o9-98F&>0MkZuWfXt9NvkBeUzI;l
zI9qNpXRPw_9%iuLKzd;KXi=u3qzdF5`VBtsRr?>9f&CF{P_5dQvkt``|09qxM~|~y
zMYLf(*u7exThtTl5N9Qu$QOe3-;undD&qQ3y?t{N*gYWoo=yD)4x6tvR$v+=Hi}0>
zU^E0qLtr!nMnhmU1V&~EOuL?_qV~{CqiNbx^;yx8TGG#T!D}(@&5zeKQa`0~a5b}q
z`tlEI3jbR*EOyRTe&-ynrgXYVku7kMnq%t`mDw$aG)|eX1g{4Ih2R4<G38HR8Nh3l
zx>j`R&oNwR+T3NW67YVrd8Y}RdA%X8N>;}X)k9yTEG*UMsd92EgV#LOB%Co#em>Fs
z47-`a!yTVh3!i!@&ywm;x;=|i)7@~MhPFzjX7ZkL6=`o}^NjR4CW|{itL-qDVs79p
zZ<P1$w${by1?E|20+bJ{+bMge`l#m>^q6~Xx@&myimthX>>PEc`29v|*CuGn^{)rr
z|Eu`>N-WsTcaLtDnc?uow9rdW!<4sI&G<^C`IE$V$`3Ej&^&6>tIjo*Ph~cDgmUB7
zAL`dv_L^z0?AB<zC9isvBTMCE#X7a`t03bE?*$6gmrvr=(AaQVU4E68a-~qdx_+FC
zM$@)UY9~6?^fKn}Q$DONZ*VVXokdfvlazo>pFHUN|2uqP2Bu9->KY54niti@Dc-j`
zqs(3USJk}xk>0Oc3C3pU5;V5I->fC-bHv0&`MY8aQx14+Y{r*&YR|9CFgyIX+oE9B
zbJbkuP2lz=TlaS3g&Q<Y6gq`Xd(>Vjt68#H>O=>rt4!IYqOBVTI{#PIFDVGT)+BJ(
zO5-ZN(^^5tCulsHeM;@3DudC~7C950`&^owwZ_`5Z7ae3;LvtqCH`mY%vgJlX&mjb
zvhWvbHqN^KK<n=oWliCxxoU^^c&i=q6SDxtzma9S{Ce5viZRbOs%)Iyr(XMEp0bW+
zjOqLjt?Jv~Y|>=EF~f{y@jP`<|9{iNAd86ot4+>JO;WX*0XcIoPm{-tCzjPnun{u;
zTC>AnzT?h(^=%$6%;v2*ruoLN(d^T=wb~->&y}WKYEp`03sHXFwLtZYPo-MSj=!Mu
zf0Y(|{iEi)VW#R_2VOPU{eP$D-&foJahdA!s5*szOC~9U?*Ba-9imtKXqk$5#%i;I
zZZ4(&51y!(bUs(xen}gAj#l|bE!AJL3rzeJlFS}Is4|$o_oGJW=3Qzbno8<Q3o0~L
zPfs)s>I+ocvGIV4z7wa)<*1h`!pDAq)4u!R%|^$s@|j&vyP|f>d8L_eyPU?gdI#0&
z(^K`WR9~4ZW#uc?-L_WZ`B$poY5Y*RFqm6${T_92URap8QQNLUR>Rz1P3!xTUmD93
zYRu1H*{b?K_OSAkL(jqHKkWFZ`qzoe0(Ac$w`Yz~WP&@mt$(oixst*MLv;Zab#NcS
zXcCXg`N?tWvUg<_9bMYZ4Qqtd9yGUt(>|#Gufu*qi+k;B%@r2g)XcuhYF_`k+4P2B
zhlbxrcID%{n2gS(^_tgMh-!0v%LbcwvMy5P#9j`t{-)H&N}%<BFU;G)cI&q>D}7p2
z3ubrzGt^=iUaZD`^PEPqXSJ$|Sd`Axx4vq-es2Wr|5G+!v)Oc$_HOOcw`bHr{r^fk
zd(|_XLZ*DaQmQ-p>`hy&TNS<vu2;MI%uP==f1T!q4+~URIE!k$KE22ES>0^&X%E(G
zPbjifTd&opUoqWXwMucXrdDZ=);fs=s{3E0X)<mvRjpR&Ro`$W72J<voz*5+P^6@;
z{zOgHL4*g~f46COSA9Hbs`fpmQ1#62R;BHnE7YIoCu!;=bC^q%yiq^yEvwP|?zUOu
z*Cm?1%4@ZZ_s=mu*HfsWnXy`ZRk50St0AkpwXdbhysOF<Y}<>>FU`89S{Yk!+OmsL
z&FEaRR^o~k%KqLuI!^peCLMPADk^zA7F(E~nO?KMsvM9Qs2bj2t#`5~Nu%2Mjq2l&
zPNf=EQ+=Ic3H5m<6{cTTrf8Uz-d4AE+pA{fc+Jf1gt3vXd7S$Eyk<4~zoBXqCKRbJ
zKPIO%P0m^S!&f$o)>WdKdi~!tEG9OZe@)X<UwG`W`m_`K%(4PMsqgezth~}*ObvAZ
zUt;+^^|aI*hK4y4OjDAL)f3x!bPl%PH(Pq9Pwigzay14gP9sMHH?vb$&#F6`%rtM9
zb3`q7swnvW-|fAh)q0~(srjF&GT+tbqH=d`mqn08q{WOEMF#bzxn|8$)67GraqIU?
zXi_(tQ>B%&U&^9yd#-s;HM=Uy2{ZMr#j&QhJ1dkozWZX@?5V1rf34rlQe9P_Z|!FA
z`M;M|>VU^3PkvVg)9OzORZl3L(K^5Mpd$Y(PVm~^vU!Z^pRRaVu&XO-6ms&a%$8rI
z61^^6?PtwO#o!_}4WG9bijNMs>Sh}pG1IucO8M)1Uo&T!QuDpFIm%v|^=f>F%uLnh
zcBo0q-UXL!t;^S|Z18tipTSbE<&qQ$w)b8=pBn$MMdlYZT2+?ZWLBEjy<L6(D<1Xs
zg^XtJCgxidE>u@PoK&x|X`z6|`aix#sqK%<YbC|hzHR$pCZo4bUE-;f`sX%Jb%Fei
zYJcPO)iWnrY5Hg1(*W)NlL`)2U3+;Y`2IgF-PNiLzaq^+_x}}H_=Ejdyn;zpUFMA8
zG2sLY5!*S+rK`P^r_9?SC;4cDX4$q=>W>+76kAr#(a_;BRm%?SH(B<B*Z8vc7R{=s
zahgea+fC!S4y%j)W;H+6^~YSgC|cut??Q`|y%H)NCBGDxom#FUQutF%FP2TEEyNY<
ze^8k{ibq3WGz3ONU^E0qLtr!nMs^6i_@RQ*F7Qj}0k?@r36+YZ1z<e~b=WmM+hoA(
z&+h7)TK9Ir%}({p1@~h`x9EcD(+-B3yi>WsY-GDG)?GxINBro-t-0p72Dtx`JIh*g
z_Rd7Go+lMM;AVsL7cu?N*!`hYGhP_7w=_(Bist3Zf#CTOkUTQ>o(Q=o79_T|W)q4z
zATb!;clQ9;FEBo_^y9=nu)e<U@>*qcE5PoB>DBL82eyy#0OU+jnD|$&ry9TiE`h84
zowpy%HtPKa?z@20iT-iaoc<>n+`j?wLHNPeiyFtc)HRukAmO^BBm%DfuT45Q+*DIs
zz-Dz!NdvnJqz{BsS0!owSlfXj1`-G1b(<jP*K0bmY2L^!*C<qq2D=+12f|NyAZs5%
zY;-)+PaJF?Nc?``QMmceyF4{khO@xsK=PmFZB_fV{xpghNc_8$16WLK`%V=9g5+RW
z*;Ej$2F52u8#oJr)!zttjbi58wPs*(kp9B=LMl_4&TAAcl7@?c#6h^}W+jGLX>$=+
z?8M0nDvKU21pBqsR1Pc#(lg&jR<rLI12`T)d}RF17Ii;7Os-@31GxM4eVGWKPc<`V
z2b({2&T_asOb<4C)~jV;_1NTKVnz4m;A-yhmTL%FErz=triK_g=Mv=nH(YAnmaoMy
zmv{0Ku-$oPv%oY+Z|1^<DC&w&%c`G{@`R_o$xM*-x+i-Cu;y)$8KmGVEge#_HV@(U
zA6jRl!Is{sl73%T`OHpscwRy_llgC@)>eKWaGZ8!Frv7(a?eAR5SPthF#!t`%~RJQ
zcQ=6a3-`{4+XLbof7=6A_b$c>MSN>_qvrEKDX{q<b;x+_={c%i{mfu}An~nBHQ{<|
zO*dh6Z^12fEjGEe;P6sq->nk1Ckw9s%gbEVf3?wIerhNyn2t>}N3mN$;RRTHxju)M
zto&h(YNjv^Yd2e!>>C%<Ehiz?$JsiHXzo^r?C%7*ZIadou-jg3_XF$SqQI<KvX4zY
zrM@354pM^)mj$BUQ-ez#NWAGdx5_6qbFi6z9$!~c<Z1@n2a@}Gv>na{@o`~yTSS=!
zQ&Vf=uF=X63pN)f79hr`S$hF;Kh6U4CGd0!lP5&C)RcnVV}7w3OoP<x1@mgQ=$6Cz
zAhE;AEbx2};)C#`_&N=r!;m!e==*0CF_m1f8l}%VT5PtD!1{e>&co}bZ_lUTEj!FJ
zjN}<sE(E6)ke!z%CxO=ggU!64e-ULZ3P>IqgYN%9=2yH^REw9L1=c(HV6WPHH81s3
zG3%9cg{)PXj`@Sdk=29P0b06hmlPmp4ukmUxFB$+YS-42s)0|Oz~kQ_dC>hoAUZ!j
zP<@v6OXZXEyufBx@13X~%^w2xukX@lU@?$77-s#^4)-Ta3`Q?*`Ju98p)Xi3NKECx
zY>fMVvhPC9E#IND4(^uSAFILk7%p867X#_XhTk1@1?N{UWyl)xr5`K7_LY6CM{!qy
zL>^AFJXJWsdeF^3te~pyS^-&yXSW)%?j2bTOLwmd(_Pg2R*tbj?ly?JwoujN_Z6_)
zKzf8WtW;dO{RNzF=_sLcbylnT^_?FS`CgX8?GOI3OZ8JI<P3h0J`j#v)S~+8jknr`
zh5x|g#2_&kPG0#}G1O5&#U%i8zJAWyBVd2mJL;;pUpfd@f6P-)^?ZvA*ben)iC`L}
zKb?P<N`VdJEX-0)XSGMqx2k^DHUzs-Hshk2)uOLpHb^Zx_US%~5}up;x#8wI>xqKd
z`P{myGc%>t7k^u?eE9=pkNXBA#5t%SGmShUXP<-EFbul?2gEi=PX(s~5Zm9^Nm*J9
zaz-(TPYC~;iD;kxuC7o!ow)++CYZjT%V#Tt?*D1a^-^2H0I3f_a$7ed&NZFfTc_4h
zFM^`aEM}5wn?2&bCXgN&mNk(BmnEBK$f|ENi$@6`keaI-rpdL7PJ#OgBzAER;@nn6
zp;DCaR?eBPe!sI9e*aJ6Pepau{XYj*=z`<9vk4LIAoH$GLOoNu;_O<u`^#1yRMoGI
zRx4Y5P`P-Gx7tjnt?IWKj>uipW&zvVQU|%4z^!9Cm=7`!8@~05SIs@M7p!KDvzxlp
z*CzFXHZzoRO>SA5>Xlh1;daY0e}MbjML`A}PT0%_iJ5mzSFL%}s`}*DEU+IzVjwK{
z7E(7qT(?sBb&8|ftbAR#nQyxx<-o#sN5OV}?VqmvEU+CT4q|s6QMOZ`1D=-wnTZQG
zONoQ;|7lf39rK0V|AVft=(U>qe3_-HeeLpKHy;<oUx$G7zlpf1sN<Cm4o48*?#>Lb
zd)J6L;#_aG!~Ue2_Y}z4M$>OBQfzvt4c1fB!2%BpkXhKUkJ4P#5MjuDFS^r?p~S(*
z=3W#&NdNcW3MyL5`oQL9-rJ{=x)`-g1*rjHzj@wpKY;kiIG!1?HX0_Lo0^4Hy_Lax
zuw5W^-cBOwOZ6s!!xF^Dg(Yv^$Ejwk804OZ&F>h&X#!@(^#4rYauLLj6-Wn%pY7Q&
zRs9E<aQi`WKH-o%FVmR0z-mBZAPn38r~iWq+^#Y6G6nmGrDuaG==^^ZAr`gWOpfZS
zG<8Ae|0!r#vxDk?Rmlg()%XK`gU#|pyZ=XM_hPUa1==lY&LSJZ_y0u9iO`U^9j^h}
z{};S_f|}H{P&Ls0zvMa1YM}jpFA6j@epHBo-2ieI3{SZ2p_;iHa=xAGsYtN7$9P}B
z!|BX~Z|VWQ^VJti#Hl6AL;6O_&sV53l{zZH_Wyk@7T4Gv&Y-5cp+lWH8j=TK`~Pko
z+yYLY%a*xl1g>AC#<df2PBzGX!S~wmuvX^|P}!#(sDA8r5;$Bz`~OY^o>Q}%^#JS^
z3&s|8ox3Jr_apEB+vD>ebpD?rX#d}qEkP<-=dOYM05TsLfBm^Y33>mYarb>y=9-;q
z=dS4~^ZY3R+iCe%2^`*_{eL$XO;yVjQ&*CCucW%P1hK!)`x!4-Ey!%q5LT5v_dL|?
z7-ZCbo-0#ZEDp(Qzm_<F)vVhtpc**`a_)|3_+7P$R`*mUibKxQ2I;rwg^Y!O_W#Ar
ziwElgiK~BCP%YUh4CdFUg{s?f7_04WdJew-CtC$_za!}WAJ(6u;QfCfJuo~Wx<NH<
z6$^O(-^Cx(Ra)<|YDi9k?1=#F|9jrHShd)DGT3a`{=X|P)itUr4}<eGX#ZcO#RafD
zZ2zD28y1xphU?X~ok&wFpFLTLp<e>L|L>v|A2_T*`~O<~1>tcCbIUyj$UeZu7x+{q
z6Ai)UgXFYUEd}d4ZI+?@;*O@;l$z<P%lTfZy`NOB-v8QCRh}QRKkexZarK1^h`mBj
zUka;5yf^?(4<K{Sy_lf(aD|R~nMsDsf%A~Pir?j4sob%aLfQY9ED2ffZpQ8h&dYP<
zJk?h0%vQ8GK2v?eH5=94N=#t4gU<ig+xtqD(EdN0>zlypVf+6y;$Nsw_}-vu<YcdI
zc`p)l{vWu`0PX)vs)O9+nKefotfyih<cuEB{=e>E8)b$*6E)k+2Cy8+PiI0Mz+ppZ
z{~u!^WDhV%4+w+y|NXt@4?6!BoQ5atPf<%a0@>HQvaks(HuvZT)!i*^Dw3xlcc(qm
zR#UY-x&$1ab}=iJRJK)UY=xfxug1DbO~8pqEq}vdb=DQ^YOD)pX_VSXY7`V)S83jq
zsJ`?uzuKftt;%<AJ^;HH<OUF)!zqu#pYCx{rRj(%*nd0nMb&#ZK2(2Xo3F~gupXS3
zO?Va5UX;F4`@e`&-Cti;9d!PG(R#=ko~<8))S1plfzwUyN*^WA{Xc;L=haS2$b-`-
z$i7ZdCsmPlCY9+-`<04*wy7S}y`cg+{~xsf4<vu*>uj~|hV|h4e=`4VQZ@|RuadYa
zN&WVm>1sWO3L0OcPN^Oc3kBc*BlPZ(O86ec{coWCe-+!Og6(2)>H+&p_6Co7{q9zE
zSw%f{L*CP>N59H|_y4Us6sxjF!bR0y7jn;&LDVv}scD*^^Z(>qH?@KF&S~gV+nKyg
z-ZOl$M(HyljgxP8gTqC1BH~`Q-3;%PwXz$)VWVRttj50JfZEodT<Qz{Oi-Qky-lrZ
z2V^hZs~fgpckOr-sDAShqiXf{3t+QAe(nxjs=B<4LEYuEvfADU)78upAou^k_WxZ{
z!W?tnKJ76$oL{eov=LAEJX7Q5-K@OY|EY?neGb^JHg-rlg^gj`d}mR4`eBMHU&=Z)
z(EdN+lpM8}p`TPX^YEzU_l1Go^ZVNZmHUh-;4}tu8)*OEEVi3)K1d9NAA`<ke$${K
zF{2Hv4<rV{9;T4}sb+2WRG+nNR+oG&uX_LXCRL?f#_AzDOVwJR?NW97F+;uVA2VFv
z<61}=J*j7=3gfTkaJxYJ|3K&eznc73^;*wQxO$L1a@pI!e9->CHHDD#8ht7v)Z2B;
zz;Ops^N<HpU%~eO9ShI}o&N_u|KBJ4i7M#)|1HWvaP=TPu=*dw#)i!^jO2VdA@`_)
z<Y(*jsX9L22sQ&&|AWM_;nX9mz;OnX`)_p&bWb-}?>ph?Dxms5ZV9UrsQ!QYR}&r%
zF!dn1hm%1y=MyiAIUsQuz9q}2nk4>R>A}wl;Ib1W?lgBI*eqE6KVw2R*v`b03)TMq
zht!p@`X8hohPMa0fYrhH$n<9&7nKK3jKS#)Bo3<onFS&3PY@r3t3zgk^&{8+Z?`W2
zi-GEYkQ#g#x&8;K(|HK7uSyGY&OJyBghBN`h{lE`+)?*8!_<K4f7gmAaJg@{8gh3J
zOb$fnK7Xvdu4f)N+(3K~o~fD(=7Z}0$ALy*Hb^}2s+gj^OFy{G2JwF+H>m9Y)P_?3
zgTylz#i`EW-38YN5{q2aqT+Sw9BBU^Jib8UAPlPiRjnG7H|>U;*#@isGuKa4o9_4l
ztR7VVznu<^L+Jj0klrZ?Yn9T1e}K~o=#1UsXFpVRFF@)QQ2n2_4ExwENbk#+XTkbF
zY!F^pFbf=KAol&DmtgjV??Nh@xHQ$SiAyLS)pG~?>0mu%PYy^O2tVA%4^JN;J_yG|
zEsz7#Z;pb^0kLI298gBy{|^$whqdR<gS#)QO@m<l4>Jo^|AY9T`hT+$YX1dP|37;2
z5AH6IIv57k{~-3>cgI!Z8{dN4$RIup6RQ6eg%EeDf%Jgt|Gw&G71;j&@K{kbw+(6F
zuz}V8jD{;v+_E`N0)>D1{0(^czI42*vU>Y}6{b*q#e{$}s=CwPs!g@jlk05c1;<TT
z^ky&}p1u`KgWP}(Yy91#Quc5fSdHcmIkiv!y4BX%EduWgg{iNvh*w%x4cW5|5|d3t
z+)umzZ#X<|Kyv6f>Y<dfO!ER|Q2mcA2dn>=?mD9ov{p(5bpIbr-r)q~KH@8(r@{X1
zeRxU<wErI_52Hc#zcK^l3~ZPfHd=LOw`#=-4<%6jk1YQn=NC9nA&Y_7Y*tNb^JSK*
z^tH=_^Y|%V$X#t9Ib8Tn#7zaR(zD=j1gY7}y$2jFZf`@uW`M-d@fv*|)%$V@;5bI!
z|KFRi11yHF9-06A?K&0PZOULf0>47e35UslZ0^PI7pVR(-UGQuOIMUZ#oBQKST9Tu
zj0V;JFg7~9JP2{GBTP=73$k|-#)r{Qs}OBKQ(t|xnr)$2?SrWY(V+StL}SD5H;lk(
z0aX7Z%PUnw?tcP_3FI4s!yi`vgT!DMRR4q6zt?R=N!uW?Q9K#~qaiRF0;3@?8Umvs
zFfv2H%o1{U?UE}u!F|M==Y<tO<NwA7*;L$Q8dT2PMuYGFJ=&xQZeu+Aff#o{AOGjS
zvKy=hbU$jMv<>+F-&KM<YMst0;C7wUnm-B;ro9BW3&VvV=gOqz?@~D=jJOLK77Is%
z7pk;JD1-GSWJ3CLJB*jY+jm!9)~H^;XsXI}s9njxN&+l?bBBoPv+K5CHprYQJnpIv
zH?)<et7U@S1rmc{qjf96eUR!PA+;^Bx~jqkZg72juS4#f0jXV{yGAkf>MoTz>bv0b
zQ8NWpHP}qReIwzDJ*rdNA$O^R)WdM_wwcPX`+w`!=_v&3U&U|(Of51!W2>HO)Q1e!
zyWWfn8FtNJ|FcLs!|lGdja_w`^%I3v=VmBfOqc==OJw~pb_{p8ieOJaI8OhnBF-xU
z$$Q_DR#<4n4c04s>bHux@fF1wxliEm0Lj1lhPaQ}&4&r>29S7I@iAr56LY}+cr0>Q
zamDwy$~)R8!sS6~K==PX)+s^}1Bq|_&ZgM&(h@BGTc}a_+nocdr*Hm&+nJ?R2X+rg
zEet=s(Fj%#<6rpTg~EU9wLm5R-6IuYzw=;!^1ssq%P-=C%t3<m?WwW?i-FiM{G%3f
zmrNV8l47pFS9sX+G9b<dnR!G|HRx7=!sDVND!xBDRb79mE1SQ019s!jEorJXdl7w}
z^8EP<PrgF-VS?;fb%0grQrHdUGc{`!r!;wj!^`8>9OW%(U0^my4G3#qTL?}=AT|gW
zFuJK2RnAj5lNX{g?ZX;5vxfU%w}Zz2ciyc*(F;<Kj-#~I!Rle+`RUwXzHRn@)$1!F
zQ11V&%HRT<C*}hg``Y2S033!Oy{jEpgZnI^5^Z2LATeTbA0K8swmmrmH|M2^sY)Cx
z_PeleS04h0Z}O|ZN`{t@y@m5_&MCZDte}=$E(VUL#5dnnKK++boTUFjRe!@Ih3Qgj
z)b`nht0w;ar|jOiRAs^JiwadT_hgj~n^D5Hs&o<9PcVPny6hyIUt|srpN4mGsuA0t
zs$LT0RepGH4p?ka-hX+Svc<|Np>tKzmhDpMSz{n)`FSqbA9WYL!0-Ppza|e}ryO+E
zOVzCv5x+0aA6L=*2070QWT(TTa|)pGe-ImlgXH^_W;~Ms_sJa3cPJ<SbX7LLeqH6y
z-i?a=?GkEn?0QNS7F)sLvGhB?lH)=|+63t@{l60|w<sTS|L<$N9V%hB6IBIN9h9&6
z?ois906918*)>QSP*R25jmu>;QTfjiJ@Ea%cZK(Z&9J(08tkvl`&!iE%0$$dsszA&
zPJ!cyJ8D37=Dt<~i-Xu7O{b~M*7yU~@8!s#+H4IubLP#;0&uvm%UuLEQ)>w%j6r&|
zymS;*Oy;Ymx6f6UdgrXz&9ea<)*v|$J};vHHy^}D#vyk0V71@3Emn^C&ZBtgXsWVe
zZ!_2qwMlQm^c<TDU>bD)?{3v{g-m;Eu%2IMT$SCT*GqPnsVY@ELhdu~+oz^f@Wv7)
zP3Uq$=D3jUefy<RNi4Zj^-|y-uvsv<lR1zxM$JPKRiu2R!S)tJH!21CD62}Qeo{7C
z+^E8plc~DK4{`7A(%Kzhb@}gARO)SSgToGH-t<M9iqHMoR4M`x_x*z8zUfVX+q-?9
z4Oo8z&tcW@|2bebNKJPu<oq8H8-%A@IjXLB?W4Fvc{eyugU0`vEFf#ZK=L3wI{ps|
z1qdD;{~sOy2c?A3@qY$}(eZy!N`T<e@&D2Be^5#oj^qEKomW*Wx72HPGu3La?mw-y
z>>Mlje34(F{904_a@4lZwy`)I)Tf^IR7^`%?5Og3*JWyXpEj#xKUoVp|6i&1qoY>G
z3<(`;PATQ+iyn#()Wnp2a#x#H$xl}8zQbmqJ6FwYaobdtw_WTOf-9TMnY&)9JQDn)
z_UulT#Ya15wZ>=p2Dw7k%1noO%{`x;Hs5fWU+!j}y+(WX1SLT^N#)ajYqU4~_R}=W
zd#}aQ|3Lj<`vtSKBPUD(PQ6sKs$;d#X!>dRKSfGgUqjJ?X+ff@yo`eK&vr?Tv&)(+
z3^IfbfBpYwKJ!$R2CLUKqc1iqw6&LIXlm@)so;D-Q(Hq&TKVYCQ2Bx*9A>e1`ZZ_l
z6jkRi=e02V%&jWiDXAsmc}`<@+zHiB5i3<sD}7f#G<k=am2Zk>Ab+E3K;K(c`7a*o
zZv$A>;!;i3(%yYlKi_^&bIn{&jpP(<B^&2_jjQnwH4?U}sh`T^*LRaXu6$Rr+VJz+
zdludExRuw>%2Pb}Lr%Gq-@@?rkywp_+CS#gqD+-*k9Vs~U~E(UFSJBIG%Unq-{Sk4
zemeIxgSHi$HfEWs=pO6UE|ADmoOmK%{ROADd4TwS3ohGMHNHFd!0p4Yi~gIae=0E*
zJKbyAI_0d|tM$h%WW%;={5)5ts=eObqGV5-ijLwQvjd;KRJ!ggGqHNkrd<}cSF^rb
zN8RqbrjeS^B$Jz_VVa8Dg>_sNgVYt*rK)_ZVN+?YSa0s_eOJRg!P`RZyMf8~uhVpE
zB?Pq`tX8OdZAnq<IsC```pysX*(yz{zveDb4tpxBn%S5FTK}i)+i}h8g~1~=u7&+t
z%_py#ZxCGsTK}iemUh5myELDv>fv_vP6I{t<duKrw;h>oZogo<(bcdon$DlEC^dDh
zSMcDe(^xb?!c23)IjuRT63w+<w(E$rKUWrfT%p=Mcd6?B{H-bvH?ye=Dp**kEq<o*
zETtH<{!gVU^u79G?ItDJjEgGa&8_O^68+St^;s%M+2^aRnte){BgabrqNRjt_$d=}
zg~WYIGaGm`m)z)5+s+$eaAsAS^0cZZ^*M5%O@FKrF;}|AV0iwunieaYj{4uJ-_>v2
zda9wmDa7P5*9JAusrBkgt9Pn=_E1&rXVNm8l)Ob!q>srY<i#?xC_e!Wql|^-cRj2%
z^@5UBt1>SbXlMCacqGd!U%k|-%<}NE(y4E}Y6XjJm3kwJz~>HqQNN(`Rc)<Chh?mm
zB-b*{ulM;Zct6XkZx!QG`#5Eb`S}B@RsZNLRD1jLo`vgrP9@)u(@jC=|F-3Nsl=U-
zQ9b!wNMp9GmjdYgUr%;h^|V)_8WX-ZsHlkZSlqK(tri<-tYgV*Cby(!tumjMhW6LC
zDy?@fr>JZD&d_{y&e2?4Y?szOD>vo)72QUydp4T4#%<Bs$t<oA^EOcO_T5FQb?fxB
zuWpzoQ=>jr<x>u?4CwwpVUJ?b^Pj|27}bv{i*(;su6d)W^ZV3i^@)cTXszFPU1?cD
ztIp>B?V$C4N+(6am5<xFnY}mX)_HShkNV2OCZ)5p3)J0N>`kZYUs97==w$Nzk*G5N
z)|2X&9J=(?G~Q^wy6vJGs(ni1kEn>*Op#Ixv9!<HqSvpgYv^v&f6>LH)_mPqM^=<s
z`{{($s_z&+>8$5oukPK*u8}$4QzP=>Mp?<UDGDndNocnGS*q$cSqZ$B>Ocvj`o&`h
zw3|i#sBiE+pyaqkNJIWWw|4j!YxCun`!v4ZIH4IN<7j>>=!<sg?iD&p5(ySfy84<=
z7GKiv`no{l?(6^Rv-ERRJI_>FlvtlK=li)_<5FUonV+Dh+M4U!I-v7^d=gUh&Rz2{
zwGw=<JcIR*MgM(6vzkwADlyigs&h|w=|}xj((-2i1zP{7GO5VaV52*;#_KgZOqNwu
zX*Mrw&{%bBin{#&0JA;QbByfm9%-!Fr=YGcqN%FsqpKd9^+)M+Z=8-|L8rwn6&0-?
z<`G)VNB^47>pP*oJflx@(T^H4|G-wz`ac!^Yp+z5ZckHr-tt6ayFR<|{<RV2V*KKo
zB^vCyPn8_ZI1dMF+zOUfk6-!FWYT(nv+L%$8ea@q%##G9RkanQ3}p%?DrD@8R~NaW
zqH#y&zWMu9PPGlv_7<rzH!V6c{~CxqQ#Vg5tTDgGC}F^%JYQ2K)K)t<zQjW0S)oN^
zl#+Uczk~*VYpE%->Pn@r3cJj<?X6PxiCbe{GndD}uD#tDbpG$HeQDr5Yu|4?)rgQ<
zqdeC@LT&e>Y1+(Hf@+Tvg)DZ42`W|bwX5ILnrC5YbXD{4!3nCqdn{Ft?X_0lHiKC$
z*)>Bmv0$t61H~VDt2AesXH800IUc^$T*^|+{KH2XHE(Hi^|UKjOyAkeSBsmpUA^~k
zhtd{BYt<bMUp0QKFl!ZkvDEaN=dEn`ZHlVwVg-xmg<48yLZ&D=@IO~Kep09I<0@#@
z>*a2-kME}XG@h>-dpHv{)U)}GPE850m>aQ1%}UF{V#Y*OEsF|0jY$XYsQ*biuRig^
z4GpzJj9O-|W@|lieWAQG*;-B5b+TIQ?>_nHX$)!$4qP_3dzGoO_m+r-=jZuK1;!Do
z<?H4cE&A1Fp?dF>vdTUlm7NC!6lZ^!t-WO49}O;nuPWv0MVgLN7&KDyub3RZ%Vpvo
z$gZtXQm%Eqf!kbv;vx;xoVR8bZ1xsRJ@%SkLz*lYrWL6ceLt!?H|C0p$=*-ulmDz%
zUaNQ<oCiSlKd5bk3!7&c$@+34&Muy<)2HJ29CFS*f52~)aT9biQje?xkJZ7%|63hX
z0oDH?{yX96%6p&jtHv#1RRq=lPycG7jDvvGfN&2dgG$aPUKDdc;xK$mmQN)~{JY|V
zpA*32O(1cnxsdxbT?AcKd00dgXH3WjkC7yvT&Vi@zYMsZeCpQ?76Yk=;q8Gg7-BHF
z&pIy351tr<^@7B@)~r)y7L-Nte|5-guzpbe4^jidZ?{9%FT>9N2Z`arp!y#q_uBFz
z*c_dQ5c{gMR-)Jgs{cXiuwe;z$k}G-a-jO(wIT{U&ThAw0c<|9n%w7)mDcslLy3c#
zs<~i3sQ!N(Xar`1^dw#tQ?PgGhr8)Va)a{zPi?B#q4hsV4G3p0ic^`xy9=%lBo?`-
zMcM1pITg_Pe;__G2G#$nRt-v<c0=Y8K=nUJK6Cv<)#;8O!12rC!KD0lIy4S@i{bGx
zC1I^%TJR5WI{7Hnr&9dvhl=inCa@bp^?%wjb__p*%m(3?FCpi5gV-RvumG`F>weKo
zu)Yi5g_JjOX{ufmmry#Y=MMJM!FtHqk03oD{BR#X+&v&Z2**S%0FUj)yg3Rs2P7u@
z;eZmT{s*y%!P;}@f&B$io7JYF0;>N(Y#9Ezd^Y&}e^~tw69?7*FgB?E->k%e;yzIQ
z|LDm-xVvC#K{V+6e-OR*-Eo!p#<$?{Ll7T^Vf8<Z528W!KZpilMWIq~7=hRzd}~Ls
zYF~A;GVJ{S@K{k*w+(6FumRQoAT^AJD^T3BIZgtFfBF0kxcgr^UR7Sb{l79(sJ=o%
zz!??Y>2Fo1TI$JmHu8e)4U66kro+=AW9K0Av0;tBdz8x_P6Mme+##p>>0h_%I=e;S
zv3!{N`igkPW!0zPcFQIr_O$N*8xD^fn0jP7>Y<d9O!ESz=}h}k)D=1`1KYWD*BSYs
zwNmi&|3P{kPC(W|UkN=8wzK!)DaB`jka-Z0JPgO~Jff(~@Bo~~VPe>5)t%ic6)QXx
z_eh|YZx3=HXTqcFVY6ydoiDRgxvyOwZ1ySMa<E&`)gklWMBJ3;Dm@EVx0ibl*uQRX
zL&0VstA(-G=<}%DmxIi)g6jY1)vjQ*y$O)<ahN)6^z*mt;OGAbe!U7V8(``_Hus|N
zLE_5_CMp&0Sq)B4x}prq){cnv;vjh#2G##CHafjL2r&l=lauFy+}Q==!|10~h<zre
zzWS;)+d{$N0h7l@gX({5;vg~i8%E%?0IL5%d>B@${tpf_5MLnQ5FGxiC+jJ9xaooY
z3K9olpK!>XD!<olMM>LhUPJblfz)8bp!)xLUJ=|5-uv|`-mi9o-BNzSA1;nfFM9nS
z`{9xDro)i^l4;zZR7GF8syWRxR{rxMRi*M*1=vh<bGlu*RSnt@d(zP5zc6a5hCA}B
zPVtAVHwDSV>i>v~@oI_JZIpMsgX{?|>Pb{9oCw*UR;%TPF~>A#1Ek#mG6NmI5Uf-w
zEBAxz36&^RL$CkeMq7a8&$wTMhl|=r$h?h=!V$O_y4f)PVT1MHID67%2sVf7^&T*7
zeE%|vUVQ8SeP&>;K?j-N$FW0|!1{ls=5GaH`$6hK_;e#=UBS15da6r1+Trm7s{f5u
z5NqVO+<mOLOTHPr{-1g21hAVy`d57GRK`;Ot5kDa;Hv)>K=uEAgKn@H`sR@G<aCyk
z>T#t;aQ^uwu}CHH#8<FeHr++Uhx@OqDku3k!Q$N^CseLDpO^ciJ00vFrmII(J$SI!
zb0BxdPM?huzDH*|g6#*%?Y;)tPm!_LQRVbeGc{#vRi#+JcCh--vryX<YXfe9^B+hb
z48!XG=6b{&@aKDSip$vzz;1)7!$-Sc_yqPlNX@bQFx3y&df@H@i7l|b1iJrEqAk};
z)nX0gE<up^*Wwg-IauX7Q#E9nGm5_B!o@0k0<rJ4hK>K%aD=K!T8m=%3$*^9F9EVH
z1!NZpC*4P!uce5#{+}hPTCG8HC20LWIPB^*?!oK-g|-*KV*L83=Ti7bLCzomnGM2f
zOCWREAhu!mC6%c&7O1}A(o__V<x|;kpcl0MUsgT`60V#Xkp0&2pQ<q24$=?9m7m>I
z{~S*P$HhCd532lo96{^<QOZcASC>?H+84mXZq{AMUOSK(a{u-4n%}|Jpqh|es``GG
z3fK;i9uUr+{2R>Q&bCyE>vg4S+Jb{%K1eR$Y6dvIWor+B?R@Pq73cbY?U@_EVF%KO
z3;zkp0ImO5obCjfJBO*mRsSn&pQgG;{i2HYr8!`;I}aej9i|tZ2Ce@;+sFg=!|ReS
zU_V?vI~8a6#CV=o?KpTBoM%DxzxvJquv$02S6~|54rIP^`&XsKlOcE2&ASFECt&i}
z>i_*pSxP@AK=uI1FfUg2d4V|72Br^0|K9*Pivq+($K~r$+i)QHo7%lt-FhtZ64+lL
zHS)*bsJ?p%xvL1o$Av$*L+WU3@+-6!fX%yEd=N~-)JQ6>f`|E9wkmLZ9e-J@vZDfW
zR~1a1@}D<geF|-TU^Ostbo$JWzu+_n6A#W^2_BPy@zLpxZ@z)!8Cm?t$ID<do_jHB
zHU~h?H)2?6srllx0(jpjNbS{b6|lM2mP5`qn`(ST{l;{5lyjy)>d<lNvU*LH4#=4-
z!afVtzIytD%}h+02c}`_^P6?l_jpcLzZ%a2wr6#LvW9QtU5tJVSUtBa*u5b3H;vg~
z^&mDn-f-d#c+3P{oDg3)>@!%O_eWNZI=eq$HcXF<8szTLJKI}O?A_=4R6YLm47iw*
z`dm00rZ+q#M&tLyBPiyD>{rs*w*91LZlC}-EW>A-!u=L=@DA9FZK{wn88^;AJ@+mo
zFjhUMVIDZ0!0dq0S`CU|drosqReKlSrFml|A3WWG)a=loixU1IF&Li8*#p)K;@?i}
zhqJfaG^+2P`xedz$(L`LrQ#@td5+(NQ;_vTN9KG(Q4i7s!yXq2pKk|K3!<;wmjJu@
z-<0FvZ~=*dFza;4o?Z|;@ZE8>38L@RS22o%)q=!ec*5Fh46%!cCBSCw`S4eLmwYi;
zzP8>7?&e*+Oj=jDC&KwKvq1EP<D6i<AT~1Iy=*nu9FUl0Hl$4mVrMRg+!5Mn2)XwY
zB!&;)7lh;md}?yE?!(i|LAiJ8H=qB)>Q;Pa5F=;V%K`TjO#PH4kaYRAuMqAA(PYS-
z)i5>M)fZKielx+tDeNF*pWhDug&5@!Ob;ovcae%xUDHpv{ns`@($u4AdsMWqBHV>+
zCRbdB#!gcYaDH3wx&ZDrkeZC=l4_ipkTc`n7e}d=$LGW4J^r3V34?FT^1yQDpU<L%
z`I;R)nv+;wqNoL#4Z@rgXQ>!*7=rbI_-ykJ!r3zFQ7Cy7BoD&z9nUrVc#6St6CWun
z<%U7degMgt@N89kv~dR5|Gtukz;^okw86~;sS%p=9$ZG;RXd?xRxYloZvRBh`&x_I
zqf70YDgpLzx0jcm)7)_%a<4o{pL_01uz4%){|D19Oo^KRqvohO%66id4^oScBkm*4
z?fX~;DKpX4!1x#TyjOYnzY%Pv!8-x9U~$O3k|6ml-?YKvAT~a{aysHZS(v(<HBZ&g
z=t0h}gNa>RDybeY8L~%s;-&()-(m8EXj4Wma9FH=zZb3-bdULwP&<O|bjrxks9-w>
z)(^60Nl`brZa??BN#kdEBUqf}GNSwksX@mI#}W6JqsxK#p#A?(vQNU>!}befR67qs
z`j`ukLe2^Ssl$fR_x~^JNmC7SH2|lrYO6lgNgut`4mn#Y-MqR(CH=G=I8C9Ob9z>*
zYJLjt^X^g?{#KdsVyd!)RwKL}2HXG7E;dOmv+|OX;8Dmr_K)Y6sa?*h1Ka<`3A5kh
zauN4lVJpsqDhbl*V7(wWCh#z-qwoKpzy>)xdG;wkxLfS=AbUoM+5dk=;1)Pf?!5#V
zqc{=17i{0GddPZnki8&`um3-@JqD}?qz)Nhv7ev@>;Er(1sQ`umIv+sS4-OjwjU<G
zw2TqVSBPDpDzp=FMhHkQ@v4}jZwX{f1;npz*`xBP?+Cd6-_daoZa>>5N6h{Istl^>
zC42Gi|2Go31U4gE2r?e=Z|y|2m;d6yX?xc3hbpWIkp8*if-_)sAUDLj*s1g`dJV1v
z+7Gm-vh3ZZy5<pNK4Zdi88xO<$QodfT6FyVb}&4?KyrUhAl5faJ%+@2!ls=nM<N!h
zJvsJONn~d#*eyIW`ccws%ymSYA7l;)!}kC0*)<og_M||yvc}mYaM*&>5W>7C5o?F{
zUO%8(8}S0}Z;;-e?Y7FRj;KiLs9jVo%qfD)A@Bcp3lmkV$c3C;1=0h;`?vC|ete@2
zR}T_{VOak^RHaxA+x~x<c1<~UbIAH|n4a^W#Nd8>a_1?E|AOvsQM;9M5!C+&hjmhj
zBz*rro5*son9NHF6t~P>2iXSzGk?>ihj95-y~e74IbN%t=MGa)5SycFUZbH7+W)`L
zk`rui!jV5<I(7dRu-Py(&}qk(D%Ii(kbJMWB2>-)+<s91A3QGsQ=7nlOPNy&aZch#
zw<x$fE;!laHGj&kGb*c?*;H@mw4kW_;SJf-q_p6xVsR?F>Y{g$IW3T#i!Vdg%+J>5
z0oMsGf9;gnY$Q<hgVgN~h*h5I`xBhzL3~`en@dCuwEsWZAN9OST>JlJ@+PS5F;-XI
zuapM1GxFDNc)a4W<5F{p;t6KNnU=8q|2OVefa3w1zQPras%~izH-Yy5n+QYZZcf-w
zBWPd3CP`J;{{R2|-C%dZ?7+7FU#jt$?6>V(z~;u?xTRdJZ~$vx@bx0xdtJJIpsvXW
z+1C^N8ms@VC8fgCNcg%`HK(2r;4%ZG2N(V&q6T&!HhEz^$oXr$+KBVUK<a9izX#9h
zg4j=!Amz21b(M<y<b^0^@s~mFS9_s86U7}3BDP?4ht^ty+Y4Ks+kw-Z)j?N{?CM#n
z)ALi+1@ptyMXeUAgVz5s=uTE+a=WK|tubAFqkjiD?QwkYRJ-}Q13uPxZ!P4EsZ`kp
z^|hf>)h%8bs<?^1(lGruUBi2=k;=Zim1?g|cB`NGzfmRh<Qw(5CcKJehi_<@dP2^d
z2Kfbs^<yTf&wX8_QOkTlrM~m6T70*(TK6_Xup2ccCa7%gJgD*c#$0u;DMIS|cFj`>
z^T<;AYQI-K%jUY;O1@szo68f`EUZ*C{_I|>&cRouJ~s#APnOCmaJ>CHEu{EA_OQ~0
z>(e!gcbr!X-=3&0$|ea8qi-Mc!C?Y&i#OXeB@gBW>gg-Cs{h(?M`?DEnrhik4fPpI
zYQW)Ftg~NzwMHRWz1vd6_<ye>ySn$zc2&^)KmSf9wZA<F!EOhc3Bm$F8^G!C`CNAO
zyGPE+TYXumSaKm&`9}9?b-^9Z8u}NsmG8*RSNC_SP+Ot&MX9^URRuKvznt4x{hCs<
z+Ovb|8d_>$@U(WiN)+sdY2p$p6=`m2{Qo#K`0L^|-rez5l~ITR+by!+T_gFeqGF!@
zEww^DJ+Rorw~Ff9ZeCK$b?yN3!EOs<P`|5jMv;GyhMeUKSFrfANDkG4HzsO3b7Ixy
z+ntq^Wdqa?O-fW-{&|rGX#W4^moRX+g6{wObqaj`pYkuM^<ce)EkUX`+OpJ-P0;|a
z|KoeMM}4{2HjRnzB-Evym#HLICCVi!T~JH6-Vau9#d{K*zTPcX*7zF=nOlE$F-Fbw
za1~gd_by~FR6=W=Dreyjwc|(CG|FdBRw<2jRxkY`uKK%O0~{`yhm2G>3Lx$W`HTI`
zCUBf7E;^#dWs|39x^#zn)6=hte}me<<w(iIYhZn~`l@O*o+9ebbMrLhx3Fn6=Dk%<
zV3$+Rl9~s$|A+f#^&-g`YHnhXGymQuCaWgjV+Mym$esz^9qL(@N7XN_;#Jqxft2N4
z^L^D)h27OBDXa(k4W#b(KFF9$M<rz6Q?cA^jnj=xY9419HJ1P9(`dL@qJHtNlER4x
zu1Z;Q&(u%yR)gIS(hI^%OMfVW*8f@Weyp_a(o~g)YKOt@gU$ax{`^&aUfCC=>!}yi
zRWH}6otW<omR}P3NGYiAKiI6FrI0+eDFM>o2hIOmZ$G2%qq;(k?bs5qdXQZ$>%CO#
zgCS?JW&ODV4lmgKf5`Ly&b42`VVF001z1hKyP;~h`g-vCKae?QMLHTCdXT=)%0kFG
zjkBh6)cU$7sD<8!^hx~Lrl`+X`VY37wKz)ksz8@U+p|e(+sq3TJx*;=e{!`_ox^Im
zM&0F7Rj0EbHRQVIXsDPhP`x4>pwYSepPKgJGitUQ|AO^{>;&Q2Y3-^VQi$_qJYQO?
zer`PmR--@drTVsA#u|r&|EoVTgY=(bj2$)1*4$H<HaV@aqP$&wmECu>>l~0dq5ntg
z)fY&92dDQbj>76+*x#%3-i}g#D%215JIKCMH`&zJgxyqs^Icr&+N6!@{2OPff!6;`
zj&4zW@Sspl*#3}u&;KUy`akQ1Eb4I{G3uwk+)(f1Y1D8K_R>&0?Wg`#Lsi4%N1#g5
z#MA1%;RoSngXaHhPajavsyU<f-@Y5{2k}FT)SuNZRevf~q5ebciu#+kVH&6M!<0Vj
z-cWtEd!cf0^<Q<?^}%X&vA5JCz2B*IzNuF`^=*N=11Gzh(Wj?sy1x&rq)z{#achyi
z#?>uV;I#aDwlBOddgktLHQnw*8e3~7s3?6hQ13qZL)}<IMg5J@HuW>#m#WX>h4l9)
ztab(KwYbKuaf9!&+JEgE>L0#A_URqpWT4T!w@AH#XRk))j0fuL4+?33#{VZPZU?88
z)M@iISbs}^%V1C#E}8yWSt1-Vw|8lEw7T8tnW{m#dKyLxDpZO)+*R{TeuKj#E~-{z
zvQD<5O;8*-ETZG&R1P!tsb1CJuAX=~UHP$<l=|YdJL+2HPt~||Ji%)2&DyAPoe6TL
zpa1JVHPHNjMUsKq_q9^scm>&SbpN&@e^R8n^1U}22VcGf>jQ~T?QT{SzxP}{^6(;6
zmq}%6+=-<c2HShoqf_px8|ST2-z_4f{`=Z~^<ToP)Vqc4G@gktfc+;?2kDcW)GtzH
zY2j1b_vXA}%O_^l9lT$`>;E<$&(mO*)CPwi$Q}?*exwfOzl+$ZA|7w7E^#qlRqe)A
z^%ZfK)lDb91iSTP2IL-)i9&rUrB$UWH(EEV*PilL-O<IbI#2PC3TXYGquo@n`@a}N
z`r9f6n)?&8*6&-^R%1VlZRY+~=WqL7-TZFXVZP2TA}(*=4&Ax74L9}es-AFI&o&dZ
z>=e`3HOG*_s={pJF4N2Jck)ii+4X<M&Rug^w(YvonP}~Lfzv`TIn`WM<gCTNa2bmQ
zev%f=XXjaXYHL_bzb|INTU2KuC)Q@+mT-TsR3GzRoib0GWxpp{Yvfp23%uvH<<h!t
zlVquE7uLsWE4ruOW><RN-hNJJyD8gV?bXz>-4`h(Vz+1V3R{1vn!V2s+UzU$xMK6R
z^s%*1tG6vfzOd~>v2QjpKIeDudZxBl^_7}Uys@usdOnxkhU%4j)SBn+b}j1Km)-ei
zU)A2}c5gS{+Z(WY)xMnd+BP!DjQjl0``J0zF1MR*T)*#ynWVktv@Clg*?|4?i`w^_
z&&t~OXF1RQ#ZhAWOWt_zm%A5a|1bQ)-djpCw%1HOZEYUT->XwOb+7i^bGELUEw+ck
z5^eXW?6!Hb@Y^1Tj2Tu5wY|3cKTg{F^a#K0n@MuE>N-qzQEc6NW$goOS8P?>dwG6^
z?P}K_dpcHH@A>-Na^I8%qPC}s^!9waA!|Fwn1A2x9lHBAb>6nUC&X+g-{xV%(3!XY
zK-RK-k2CuAomX$Szp;JUzEUSK`*kYv2aL+wY?d?f*z2_l*?ni-vzMdm^<Je&^?mW(
z@Aq&|u(B0cZLoLFk~Z7sO;>E=T)*yKw)VR1KF(Qt;<TD<=NakSt(mO6@4}bGc5c5N
zY~Gcy?A5;IVZBeIes4f=^S%unlWjP*Iq&_SK7Fr=;E}!D-=6O|?eK2zt;Z63vv*hR
zwUqVQeapUe&+_S?_68s3-|Jd+W$(w%m%CM3`u9$*HQl>jr_a`V-zwWRTNU?Ci<h>I
zT+g~s@XgKLG9T0S2=Csr_i@auJ*LrIdmo?avn}bkw72k==3ZTfg?k=&N9~<ZWMq3{
zh3DQArYCK+ncwZ{)@ro%<~qCgZQhYxPiN-aW?#48+u<;I@45GSR*N+j?RjUQY<rYJ
z)lT5%;k`x9f_6{RCG2K?W3Ud)4c*(p@O+QgwPstsps%*iAEfR*UGKcl&fnhdK*sF7
z@pU|VCCxAIExV9qCnaKJ+su%+FMxmfp6^dL*qX74?)wxn)%Lny$lkMar1u#uQnj1?
zKi@j}uk^n1>i2sE`FLz<CGXhSi5;+M*kZ7^+e~(E;AK18K40E_dLIm|rSD9z<$Tw)
zyO&qPmN`;lZ~e4;HtUsU+0Iqr-Lt%H%HE%)zim#cG+NJ@zI-oRsGRM}2Zwe^RtehP
zV0PQfT)1OTANP~JB8w;N{ilD;>Q{I7-f{z`eJ|R#?En5}quonGHT%lF>idIa>#Uis
z8toI}ziU@7c5Yu~pxyraqK5le+^_9+s<~(TgROh7;r+#XeLT<FYzo`7_pHFv-I^XV
z_eO5pwl{j&cbn`cg?)J)v39O^9@_aghU~lW>9ft!xu5rCv$EUeU%$Lh!_~xY&HvOr
zJNqZ@%TrU`FMf8C9an?w{)H_zcFu8k_j9`z*)wHru$#8e!2X=a&OMw=y*8ZLOnWvs
zwCs5=%DYeIYN;)U=BvH8H%jfbpSW}H_br09ae7<#`wBnb_b5Wp-v6qfed}$*{i_~K
zwqtxeXFp4rsclh2&i-TE)wY${$LxfrEwm5RU^yUJ_1EUsoYQ-^IZ5t!I=R9Aj=<Hu
z9kUttS6(;Rm%OcX&$4IryH+pl*&E05&*mUwsqM;grF|LOx7yDC%VJmW>~Aw`YRg`y
z)#A2Ff$aNkUpLx&effVIgY!DOI}f$ncHZgV8>s)sHbpya@8w{Yy%k@7@Bd}MvOjpj
z!u?Uc$M^aASK80);<29;vv&W!^p*B1R`&Ms)-UY}z6#o%s>-&#5>dXl_Ko)5GhbiX
z+?V6sW5U;K%RXtzZrLN#_D=ZCWSf6yqAlyf+qUKnnfu>V@$T2^VX?pLWVAnd!-0JV
zei_+D&zx&FH@VNQBtUZC<~s+hvR_8oaxDJ2JAcO6J%2Vz?c4F<+U}VTXYE<7YiE1s
zOpER9z#99LjJyZ>#Juc-jo9`J8Rzf&9sPO#zf+w1>-$aivk27LtIyTg&l0t6|B`je
z_HI7h_NTUK+5hbmvp;oo@jl`EEC&osO6)47`1YSVb<<9`&~RVt3aPzZc@g_=FN@jt
z%u3Vt#{Y(Wg%kYtX@$?;+kVg8Zqdq3`%c6z+T*inwN0;v;9mFNEPJAyd-nEZx7yD8
zG1K-<kix!AH?Hn!%T=)1awvCi_-w(wt&XjG>ux=>(f*aPS5^J<UVpijHp%+)_xk*h
z*~hY`(Z=-`|Gv+YF4(M)PT89j|8uX$oa1{_5>)qvik{wk<kNw@6}i#2@1}_F`&9FA
zZ<J`b-F@|wdz+*7?2T}EVe@{P$X=0s4SO^%-q_1Izi=;Sx15c*uCi_Z=RJE{->uu*
zDwS(9n~%-bS0dHc{Ft2WWtD|{FPTo*JL6=_9u}Y8J*!Jt_tp11?ve3vu`N5JYRh@P
z$>u}HLR)#Qh`kqgE!iExu*0^YaFy-5@40*B{_ES${?=^!AZn5ALjxAuAhpMP`Hh|S
z3hv?CyFNJB_WT`3dkX;;dk5Xg`=_opIv`th$UccPYroi&5_|hoeEY4}so9@fm$+ZN
zRMg(bGu%E-$Yg)U<-+~H6js{x9-O`ZxSEk&80!-|Hc!|6OCQABx@I|BuX0wm72R=o
z&zmE*HnXl~+H`L!v}F>SvG@4I$2KP}3E2kcOWUx{@w45k{Lwz%zsycZI>P?6b=Urh
z7cSa0CT-cj%KF_to0wPoUD@sT>+deuAG^lP?i<e%+qth7+Sy!J+&A0lh}|xZxBFH_
z+wN=nGSlwIgeV)8Yl(Kda~|yDFM4EKtTDlE+s-w6um8{AcQxYJo>`9jY+GYZ_T?_u
z-=9{t&~|Hi*WUH<jr+AYm)g&<Y~0`NnrOfK#WlO&5Bv5XUEXBRcRkW>?gQq%8~qjc
zZhZ0D_T+ip-Cwr!?d@pXv{%hUX`jBCjm`S<7q;B*|L?teRM1w(jdh>xP6OKn(Pul&
zPEEDZZEM+^_WhD=Zt$bMt16i6qKu#InJ7_g`~2OGy#*In*ak$z*w}9UyXVvDVB1w|
z%=TzZFt&9N-(s8me4b^n;iG+D=FLA4mOp7<_wQqN&zG<qP?)sW)??a6`?G&1?ynZJ
zwh!SCx0m^>vNv<4$o@+IHMYA0&g|vN%-)}tE@vMRe`wDNo1A^^I+OO!V3ykdD&y(C
z6K0J2h4)Uin-d{@pe5bj?(f9O`|`T?*^69Uv)}aq`~KysUiNF(iSBhtSJ->%(WAY~
zuZZn+5l^vYuCm%YVc|JjxwdV#dv8tN+n8LuS2FwUK5oezJD%ng+uYOY_B_`u_BYII
zwJR$#+|PD5*#6Gsj(r@@_Uv1{m}lRU`9XHCN{aS!EZ=Xv!$fp%tH4RC?29e7aU!es
zvb;&#6`=mw_KwO@n^dXFy>qmd?rCY5W2=>Cv*+~6-n|zNU$+fd9<hf(oWbtpf7`vj
zKa2Nm@D<v(z*2W#QOmczlerdHDXw?i_xN|H?FPjny9GBm_BGCK+o%5N(cYl=P1ao|
z7xvbFd1>3gyM9kz@j~0U?SXr3d8PKv-8aQnPuP5)j!WC#fa{EV-|?>A8$FkE-y)mv
zy^F0m_LZ<m>|5}K)AnA7fKAND(!Dp1O|bpJG;wc-^3y&0UjDJ}{w!@9aZ1Kc^re~Y
znFsf5cWgOt`@rm&Ew{b<{t53&>>s9m*yqV3dthpKti4F;5Boz;w%VOLvva?C=$-vj
zr~cdbR;tsEU1Y&t|0&D%$y~ABS95*R-p_N+>~d_7vs+ixxcA*+y?ye3lWZ?Y-`JaW
z)5CVN<@vpHjHcMG*mQaCf`wIkMejP>#zz*|2=TJ)(e&2eYdz`VzH}=NyM2-6cH8HA
z?qgY!X>+veg56@d1@=YH6%TaH$h3=iIB);EDf{=VUB6^c$(`M{`MqtnQx0FT@#S-|
zaX<avX5;&~y-w>^@4C29XK&WIM|=0Jkhe3vpKHtLv3{@2DHprL^XJ>>JNE8PRZ-ig
zE8S?zbWGDW>gPP$Mc=RQePB4v_Mxub-ax+}dn0deu|2B4&{n2+rEN=b*xvcpGpwyY
zaP7So<h?h0&WAk$``!2cys~!hjN7husy!=hC0(O;+v@$?+gqr!uh;X}-UyMLeLjw1
z`-B7!><PM)wr|=MrG4jKd+yUX=U{i4MQtC4^2@#7jaBz{z4EuY<v3|Cd%&8#C8uKc
zGG4o6yVv82t@P}rwy&md+$(l!<zCMT3-;goZ@2e<>ub9Ow^Zy+^y~Kj&ir7P8FFdg
zuhVU|tV?D0i{3Kce@1(X?c)QhZT$K)t&F{%TFv)3z1QwVq3uIW+r4MD+U$+KJ#X)V
zm+gD!Gn?B~Tv=^>P-Two!!I*!zh^A7WjDyRU3y7qk4O=-?JUo;HhR8hwzf%ocUzWj
z+GC;i-*%xJr|qxH^X)WQ4flQDvBNf}Ex_*0UyFU&pRVj>+0<n-U;o@*`CjgQ>n1Dh
zwd+2%$IFz#?(N5AwsH)M?NnC&*fVwAls#K6<m^p){CPJkXY1}4wVSOC<M{XKE6lgG
zQg7O8_jcZ%zTeaLcG%b0uz$(kEA?Qvt%d1ETiMA6_R9X;vG-#1VjET+?|s*mcI>?+
z-?sNi$g;hOvz7NHJk;LZyn6cHFjlGE#?2fyPWBbH59Z$4>-*{Wo@XzVY?*JIu+1^P
zx9^65*glCLta~lJC)$K&CfXU^ZQEORu-9(c0gZk7x-EMOl_%}(N|4^CFI&Hl!B*17
z#^c=HuZjovO^Tgu8~vckuEF=BO@6J4UBe-XeGvwEdyZGlv7Kx*+4ii&S}X6HEVe#;
zI(ur*J+lebH?sZ5^wU<`Mr&XD+x)%aZq<8gUT@ypXrr`O!Nk{Yx#7#bhi(?^`N_6%
zUzsN7KAWF6_7=Roy65HM6?=E@*=-}>_s=Ho>N4AEt0*hBWykmS>jdw4{`8Km_AI`A
z`TckHii>&dy1klz@42QToBh`p+0M{!+xu>@^PUG*y0#1F@$LPy#&*y4N$s`|J&SDP
zy%yUs->kNMHp6+}BB8jwhxMM=8ZG7BpZjLI?YuXYw#MGMHs`O(?OQLQzQ=Zr!M^=j
zOndLmY2RC8+i$Z$Vg25%`QL1(2OZwKKH<{d?$tf^3VKuQ1orN?%Z*uU=cL4HA8<i^
zzu`UU{e9B!ZEU`m?vHpZx&M2w!huD(N&7EedSO4`n%nL~qQrsEC+FEQP3JpcyL+#_
zgN%ax98R5mKYmH<YugdJ@3unH-s-qYyIIX@`<9rh?qhj1b)Wgo-hF8wy!Tc49NqiO
z__6h@`Fm|G?7eIXgF9@3%^Yl|Sf1V6!&z^e^hm{aj^VAnb7b7?+|FFytJ%HD`uT<v
zHpb`v>@|HfWzUb;xP2nKQ*75}%k5j4e`3$PlT-Ev_1f4yVk@^VzH;79G;7&@zhl4m
z6}acy7YT{&H`}w*RxjUUU&D>7dyRLe>^s@GZr@+`x%;Hwh1l&-D%w{vZRbAT+O~ba
zW}UQIdV9WIv(E)<ewj~pS?QMciWPbXVh^(K`*Og+ZdK+6d!d}r{kLbX*~hnd`u-wM
zj(wsv6K%FyHSWDAyMHg&J`tNuM;BO0+j`jUe7w_E&-~1ux3T(r+NRI3)zCDtS*IPe
zck9n-doI=n?ag_}wC~Ni#kQ|@vDo|%-@13n)cboEpI^5-K~8GV<HQV`km(%zf~|P$
z&M>iAD?LAB^J1!+-Fn62HlER4HVc-W*mwFM_kN2v3-*Qep0l$*cVVCAs{?lTin#XM
z98|MgcTC2<HLr2+sodKA$^XOb#bzD1t^X&u@6o2Sb{E5#?Iv<(?2nvjYv1s1^WLZz
z!TY-yC+=&>&#}9{hRrT<f`*;VgxR*O>052@y{z7MP9omUXbIQ8h%0aQns4FT*HTz#
z>w0a)zGo@(?S7uE-1j=O(oWqyZQnW8K--00JM65(PwZWmS#JMaD{Q~LbBEocd)oWE
z!~5)}ZaK3r^+t@{zX_A}?@t!6)#Ki>XHFmQ-fnYG8&379dy^!^_H}*SxA)gBKD){L
zr0wJ*IQK1)oxE3dhlXwcOetI0j&QrGV~jS7M0fA;ICW+3E>FkZx96<dQ<R`?ca#0l
zUbV&ZE#I7Vu(i7<ZL8USZI6Y@=Dj~oDewDnk6~ZtE3Z8+`}+5sUCX%7%Ab8t$+EMy
z`PZNB`4q6-D(aYmUB?D-TfRq~d;hDP+}pNFbl<J%3-`FJ+P6neUS!{nIE@2bH)h+p
zum5g$;mrR1x0bE5%X-9V=XBC>-^O=+b``>=`(z|+>}Lq|@4w0@>~Ma{(|s}<)%Q#N
znZ2J^H);RJXW9GoIU@I2=X2~giZb5Y$<DBE>FeEl8D^{5Hg&n~vhSI(C$##}-hQj+
zw$5uG?cJ$x#zyL^@IEdjvwh;>dHWQ<pWgdGd-A^GYfJZWFUsCmw<^(2gS*A<qfF$!
z6_YD$%$85v%W(XUwZT=kJzF}K?lGO&yjSkZj=c}&oZH(utJO9s*xojM=7jwX-e&t^
zzwF&tP^!6qdU%rkRW+vlvL9{i->&Pmzd3L5{&mJ#cJu$fvpXW2ZF}Fwe;>2B%Kit9
z5A8NgiQE7GDW5%WSnmGE;mh{5?Qh<<Gwr|KGQLv#>^xpOxm4x-byFVgjqtx=t7PhG
z-x|4l|GZDRc8$$b_iO(Y-&=L`!|vLR+WW2v8SI-L|Ig<7N9Vo#@7CJPSTANb`QmSz
zxL*Q$>&_S1xt^c0zu@Mo{mb6=?n|7rV*j2-2?yTeYxW)f&Ae}-s(^jjWKp}0B>p{<
zrit!b@SAn7o|K7g>9sF=64Ih=)vTZHZMxCESFo(wW^;7PUXho3_lBB_+D?!=XDhX+
z-6n@G&X$X3&)(P71~xX{jka$~tM_y&C)nJLSYRg`Be&<HQ<d#xF5|tA(}nGtCJEc!
z4RJW&u6W08L%8Su*;_j7>p#obMqX^PUG$}O&#I`~dpy<BY#*GCu=%lfp0&{_mVKUl
z;(M1^pWiFSB)z{i=jgt*J}UdUSlH}O7EQL>t+LHdGDm0M)gakD1#SE7f@@Rkw%_`(
zckQjDJwN&0+D>`9Zm%U5pY4peb$fptyRnx)j$@zB)5&{$JE!hV;#;%VCMjsY$ixHt
zzs{SxU(d$a-efxS{@Y*w@2gT5vHP?~#6DiS(tbi`lie$>$NQ3>a@dJR-rlRbLSmn6
zhvdGWE(vy=t*iH`?m4t~%|h$FkD?duU9?1XFKfb_y(|{LtXF)VWy8p`&Njy+eecam
zG27*#T6-5~_1iSgIBKI%B(V2V{Q}!%5=pkDKY46B@9XT{yjXCbP2oN34~v%Vy?1nq
z?bbi@>^AQ5-5*oB#^$usR=bcQ?)}Xgm3Fi5t=u=S-h01>9GiXEE(?2`Oqu;L5C7Pm
z@X)qzpCoKwcA&&&`8TV*mG9>6yWL;3UoJp*-`qC;JwLOK@0HptXSdI;-%h;EdH*GL
zzI~Z@SJ_QmH`Q(;1GoK(=N9`!!ld@ISNrUf`OmOl)vV4=$E9cA`fJ5|&&e;^C$(|T
z-n{u;b{lJF*&X)Wx#!u+FMFr9H(USNsb(|j*SbB4lke<xjm+NTUL9>Kq$6neW|`$)
zbJL)`J+}jFGv`O`{hP+SZ%QZA-q@3zdv8=u-Lub2%_eS+<KB(0X6<2NT5pw5!n`jq
zhI#Me+OxJ#!q@CId;7xX8`mV8%%-Kb&nq6=dY-h|`@3Y>9x3a$djr-q+Hbue=Fsu#
zy8UMBEBm<<v+S+nJoe9DC}V%%vf91{++GJ<-0#_Xcf{GH>gU;Bum8O#Oxb*obL;ax
z`zIIdd#5j9`>a{dX62GYd#&D1-^V`D;=nwPx%Rhfxb4>7U1i@IcW-}Sb@+avgO2tW
zf6cTvyS{dROywrK&_bsD42C8)lh+pR*N#4J%jB+TSN}-Q{;+55{?grCb{F57?wj%>
zX79DN!uFqj&)v6ma>qW&?(}`Ne4P8DZA9#L*?h1$B+a>hqiNs1MVF@9rkB|4d$#Yo
zmCI)-+qV(Ed!5C2to?5_*!p%B?_JY6Z})O`mpz^{_U$(2y0DieYth~d-Ku*(zsTCd
zwe^Tisp6@<%MY3DnlW?w-lyNqY>V!k*kjP<XRG^8Yj4=<TwBqr?Y5lCrF&1Wm}K#>
zTFp-PQ|;c#!B=eMM0pQP_}^{k7`)y7f{XV)!TBlsS^Zb+mwk2JR)6<L`*RkW2d0WH
z+dt*yeVc^%J$tilA6nmfX<<A0_0_$P*;Mxi^HlBq@3Um@RMS&7*}PJFH}&qfz9DC`
zH{=@QzE{(<Y-P4Cuyso}-XpB5Xm?$Gn@y}o{~pa(3->bngzVYa|IjAMn|t3vNtV6o
z`%c=-Z@q7GcFKi4KT<es8eSgR`z<19Z~Xh`dz^#~_hkJy+AEv0dpFzbiF^0G7PhsF
z=-+!Zc;cR7uIjy?pUGM;{h@BVvsYkmtjy#+_Y`;T5#GLhZ(Ew#Ua5)GZB;i0*=`gT
z*}aWv$=>tZ@9c3BeY2PUldtU+4%fY5YmeJbUuIx)aO!=VJD#n3*FNmD?fPGCbEm`7
zUixy_-udZ#2j=he+`oHPu>GD|&I4b!+wIr6#j!7_Bh7B+8ZG<P&$jOU%gedXcbcmm
z&zZ%xZ*FJo<4HWYw>D^+t<$CoyYl#jd#^uh+_y}S*{(U9ckhYA&-Q4xW!T;o{%@-_
zb*b%?vNwD0-D2J+_UqW*i>Zuz9@xCGzsdjC?x(NJf$hI$+WDSiwNq-^xc_;_#r^TW
zChTWh@XT&*h?0HN*({qCIdc2<<#p}7Is3xy$5kh6R8tH07W+rq=+DjCJNr$Ht#iTp
zy;V1*_WFl@wD#+6-o-SrXD`F)Hrw}iF6|9F>$P_ai;%6P$_886K<0frn(o?9uw~zG
zmaKO`bIQj3#{5hN{!e_cZ`#f_`|sDR><>TFwdWL#x7)kdb>A*!MSFF%jW*Y$bM4bA
zo9(=>?${&O^JibScY&Qn#o2v+HcR&|T<^HIJ0{Jx__eLAQD2G8g<iJ3%=cLAs{TB%
zKEJeM*QKd@_m)5Xy2q0Lh;^*A@IE7v#J%df<@YW7k+OHOe7KD{<IdfV($f1DmF?VH
zv;4{4&QjKW3~#M%{p@zxWI4Ut8}c=9?>xDNy(>h7_VLD_-u>WO^xo4)X75R{S!D0J
zWwL$Mw;KC|gA4YrxMF<Z*o7#EvWouwil6xQ?>Vl$-~Xr8fnAFO?LzLh??38qV1NE-
z=U$N`&-V)MS+lS9hL_!*>wEWY(0XE5EgH8kJmi(lp|7la^=z-(b$xZU`&GYmUzM@-
zzEk&h@2gGkv2$9;xbOC5hW*XqGJ8wJoNak_+}|s<Xx?70m<P6+^1`-#ZMC+dJ7(Gz
zE?c?xpM$JzTfdQ=SbERi>!n@$l73s*7Ft;C`;b>*6*G0gz5^|)dlMtS@3nf_w)Y5&
z<zC-KZ}v3r_`LUPeDdDxH(9oiC-~U@KFnaZPHoB_ACdOG`MV1DCcP-NOO;Tzi}2rN
z7ZrKh*5nwYecrNP`_rOC?aedn_idiNW`ESe?RMR6_WN2i66~IZpWGYmq`!amJloy!
zcbM$=pMGQ)eL~J&?Cr8$fwO1q{r-H~{wyQ2eHH>c_l3*e-hbJP$$s^Pxc!F~3)=ez
zSMS@NHO+1Xe}?_FxrTOKik16%_p9uSlV7~IT*}zy_sofQY_l2n-QOOwFYbe!otb2a
z-KCK2w*OuEY*~M2?RztUV_$jJx4qv4R@>;!+p)Ja^QP^=9Z&Xp>ay=EcCN7%`nO^C
zhfuM-?`FQ*+kWqZjewx^z6i5DyTc~1+v%@bxx3g+*LLa2*1h=$<M!p;$lEt>e~WEf
zjOV_;%e?mql$q|kcf-%FV-N3M)%R8VYOAgHKH9~+_f((6zQ|~Sz3b;b+WXx@(pn|^
z>RyLnuf29ps`tK3xo?|PW3+do-x^!z*rvVm^C#Hcnf-l_>U<{~RUNfG9ytqaZLgc{
zjZtpeo5W~-VD6gF`#3N6??1@fzyC=`hrL#Bhb_;7U;FnlAF@xlrm%lwL7)94FPpvZ
zuTQjQ%~@*wYSPlZ%25+-`WDaLdv-_B-jk;P_a4@sX7f+#<{m-CPkTkzN7<f}s<e4{
z>&)H_KbiKO3vb$+KH;aWXdZ*zE<sW2R|SVH(;`{+S>0S@yXK7MJ}I6rHVfL!ZJjn2
z?Uk~5Y%@27%Wl^qyFGu}G;M!wI%@OmvhCivvwZfNB&hEFBl%=6_ia<#k1IrM_&3e9
z{n31DPj!j;Ue{9Qy(NA;)*n8|@3X5|xA)zVLwjGp*SDSDx@oWDh2wj9{#~}=sQR?`
z=?ZyUkC5p$e3P>GKI~m%rBZU(`b2}It=V1Cy)&vxZ9835_c5w}xB6Ylu=j@L7h8Y(
zMw?5vT(;E$mbO2N*V#H+R@sUk7qD&ocV+L)N8Wp9UzFMx*r2*cwMyHz?v?J|yv=3S
zb`vh`y|3(O`^)~)-tF?rdv7Gx*kt5o*u}m)XLmN1XWuoZD>m!B`tA0tblO*+H)UTy
zP2xV@wJG~P>TcNQc$~|2o5R|@nfpR^Kk0aGoBjUto-0!q*)rzw+XkN3wA-99d+**(
zrG1GOtb0?FB=)VDuyW5NKG}UG0=|1xev9sX^1H(33Bz%lE%uyy9TxHK6KdLLyH+R1
z?xgrnJB$3Ob_a_M_szS$VDH8!+;+1$boT3IH`+ekKF!u-#Z;@aSEubU4Qbmm{pwEJ
z#wlWZ>$P^-xb^nf=9n1RPCYHT+lyz#UiT*)2et*b?F+e}v;XW78GEs{HFlh@t?mD;
zn!ewBh4jAPG7k24u8Y~vEx2scX*bc<!@7NM*Y@dqo96!Bv(Y5QMs1Sm-Wq)tyWjU-
z?ooEVvHR6~@qK-|Y5Sb6FWu{Tr`JZHX~Evp%9m|7l(_C)|44D~0gu+bNxyH{sGP{%
zy}+W=_H$3Yjr`~RHjU-W_WTanvU{VW*gnbsCwD*g_O{i#chEL-pX@&E-Ld<6u8Qru
zDs;+@@!b-uGh1HoZL~13^J6=>cl93zdv)a>wo|Vf@6}l?x>u&DaZin3u<g#x4Eq-4
zN$=fcP_b9erEbrzgll`^b}8(&TKRepuW_aA2J^N(b8lMio0Id~_KVM>Jqor>dk=-h
z@3lRAaF3#I+TL?nUiPVh()N3|_Smr;{Iu^$w8H+5=cW74rSIPtR3d6;Q@_#feF5iz
zV6zANRkMTbK0e%Pw?@5nzm2k<{YB$<`&Oql*#CN&U@uv*d4FT_DLbF7|7<-%cG$()
z9I-VKmawnr6}DY7Y4yH!q8IJf+`GST^Krg?Kcj!{y?U!*ANx*IYhIUQdml{buoZb7
zXX_f!Y8&>C->z!!HrwRR3pVTdb8ORe*4Wl@i&$@5FS@T?)@X0%GJD&MBTsF;H}mXs
zXnJQ`cVf$)6Z4MS%6BAMFJ1I#uNBwMy*_E$d-I%oY&j-s+V0=fxLd1)*Y2GBL|cvz
z*KFnFR@*qrxbG2@+q|cdcf#I_v#j<MORwMK<>I{OKZnv@j!X|*?N=Q81m^bbmwnN;
zFY)M=J$6=JyFZC~*<ae<Wyd&E$mUJn5j*>bUACrEV(pxHp6@+5k;^J5R(&rAw}I7V
z=dIRit<P-N{1M(~Yhb;Xb5;G`ed|nZw>~|vr`e#!Hc@=}-ct+j>^-<{<(^vevONps
z=UM-}uDdVtoa630HsZFbaiaUI;~K1=oH@PsL}cyWcfVHbNwV0vcctw@n+tjZcH6q{
zS_duswwFzTXJ3o?58Jq?NwzKC#y0Q2>FnWsYO^P@Ja&(|=5$-D57TTG2v4v{*)3?#
zlj>_<d#iDuXu`t%Pg<4jxx$$C&DgffPWrRt{<6MXb{9TJ?XSB2W=}Ymv|Zx0E?d2I
zOZWa?(zq{eOP=i+N%wtcWjXf!W8Ay<hOg>g`$gOL?&Qq3pB6C7es;m8{fly6?PLFN
zY+q!m(q5a|`}-S?2prJf=eD1rPV2xn#tHlGRQ=iO)3nz9f(y&On+&V$Dka7CSKSu2
zJ9^>s{uaOW_K&+)?0st@ZhOX=(>5h{vhB65jW*M#yI85#Fz?&`c-EfZS0nf6rLDF7
zu3~8acZz`Z31j;MoBpoce`4hs`=d=W_De2~+vn<X#C8ow?fx~3zu8??>$eS@H)~gu
ziLK27h1R{zYA^Rx?7m|&v$1UNgYcr=d7a$0vwU{${pG)R-%F#j`?B55>`t=G*vHSR
zW5@n_)4oFthxVzo>e>l6*4wR}7iec_FR|b3d8mCcSD0-;{C&H<wyW%FuK(UE^J>=q
zW5H&2?mjyE1-;(dg(a8Tx7_mCue^)x!2D;^_G|Jg*r)Q%+W+wLy#208UH0`yPTI!5
zXtQ_s-@f-nU7n5b`Bk>>%Kq#LXWh2v)g`5U>P2sD3j{CO%-$|$({+6I-UB<2+VZg9
z+xwo|Z(nUEpKaJ)`F-}Enr&8oxwqGA>Mq--Cj$0u-^#SFwARyh`=d>^oAu7x1oI!>
z^WEj{-og*x_cX8Cwf6|axxIolkN4L7SKSw;J8Ab1t*<uf#?SURos+N=3vbvPV!w9p
zS(oLuxqE-xJYZhDcV*BKn;#s@ZR=Q!ZO$Ev*jIG4%XZHWw|)1V)NG#bciT53w`nia
zru}<&)%)(d@-cH?R>f^wOFPT`g&ueJuJh`)GZCI;%Rf7Be`Sr3eYx{oyXr_@`vvzn
z_SQ(Z?z#E9-!^OZ+`SVrAMJ@y-M1%Vy1<_F)8}ljF}3ZzU?gOtV8CS;cX{r<>+hyo
zhwNRsS1szU&1~g)du_HZwo$w1weRiTiFU@vKUz<o>}vb>vF6_6hgxiH&R*Q>U-iKD
zxW2x1-obr)_s;meC+3FuUeVItJ$?s|?8$z7Zr>L6Cc6m&-F9lZi}p@>yldZ*^-XpY
z|K7CQw)Mikm2(C6Yph*qyElw$kMfGmwhBj=?!9oV$(HfMb{k=?T3g|MH``mLb++46
zs%>BYox5+XPMh5}rgr<QmTUGs>gKik<E>)<igoe+yON#u-_B08*R;B~fA5(+wzkX%
zcbn+Q@69SYyO*;{*fyCpc5jS8`QBHzFWClqPT14^d5*0&f9u{b;mUpAV$}Dfu>RQd
z=>Ph?%N3sOwO}^hSGbGaZkgp;8;^$(dskTqABdHhZXd?oW&d5<>_F??mVGteS_jzJ
zr0lDn$shO<(y%}0w53D;3;BI#)GPNG`X}u@HA8J*T5YQB@#xCE3+FG~Z7+3iPmO_y
zt^D`zduldauvNVqy*DLAckhAB^|s$OKHqzpMQfk5=kC25H%Hp-+BD5>fue<-v!IW?
z)1i&~IbN>WA6zkW|NA&5hegMJ><yjOWAB(YZGYVag#-P3LI)0V8}7^7YP)Ym*<!nY
z>9uw|OW5~bbZOeRo=I<C*_kQ(PNq-WXSQ8-pTBC8{lC9g?NeS&*zX^gvA^?U;QpVL
zuKUH`>+Wwm@zbunFlK+Srq=$*B!RugY9DNV?*3%^gzwW{>1AQITUYk)SrGDfujB;v
zeKoQhY)V2U_f<tN-}6x<XixiN&AlHJWo=X~rtWR!&f6RH(qQlQ)q8D}HJ<EUzNByO
zDUn;d16k(TTxKZTyF!l1mM>|Ct=5uhHW#IL+w8YnVtr@Jg}wC~Wo&l|J>R?7uWRp?
z9Pz#HIIh`F@v^n8WZ|_HN)^~Q#b%vtey4@)2JWyuzFc>0jn;14>yUMC?*~DVeb-m;
z?eqAcY4d6BM4NUS?mZE|o%XiBT4i%=jn=+*+XMIN#;e#K^AO&*^e+2el{-K8PHAM?
zyF|RhMked-UctzBw*Nx|ZG?^dY=fM+_JwX;w72);i(PSMoA<Jus<z$hs$=7E!QIwm
za^K#G-bS|l!K!=LKNjA%Z0_?t+tjb`nR?~<o_Rm!?&VhD*~{CnwRfUvs_oC*$9uc%
ziuPQrP_j*$rM6dIN7K%j_ngh<%bqrV+~Rh(mtEXDsmyxstsS%NKL7UJ-?!LlUo3ym
zKJ#U>?Rj~e4wO5w9XPmll6}g<K>KPheLDqKe%m>d|LtLjS+SRALX)jT_h0K9x3p~e
zITq}ZPkXUv(f7!`zq_{C7P6n;J1=Y9KH0e*w!12MZ9T-x_wG!U-y`qsx+lIVXpb;o
zg6-vFVtYINboQkmX|r3RF@JAkUDPf;NBeyXHiX-}a-6#F$S2NyZV3{5U!=(IOXHTb
zEwY?zyRGPhP3B+2J?AC#ZBP1H@71hcXxqY2VDtLn^}VM8*4vzrpKGg;eRuEKJ5%>M
zPTgbM)Ben6wh@n=rlziK?%}n2Lu1d|CdBaA+;bD!&(5-X-|Uks>~f=j@8N%!yYIMH
z%--!n^81W7ZrjJL`E~D<nQ!f87tgj`k|1Oo<bTtKLyysJOU^l)o)u?pAFJ-MX5(qJ
z$&$IX=eoV&Uc+w=cC7MK?VMk-?YCFnU^lI?*RJ~CBfIVeefv1$T<qc;g7&fLl<d=M
z`fqowc#Yl02%ZCbU!32+Wg??P?CrVx7fN2-zb5K{o%x#0b_*`n?0<Z8)!xMuMC^V|
z{<_yBrE~9uvn_jNf8DdWBOJRo=2nKy?(M7hKF|5Sr{!7wz8CxN+L+bn?)5TZvrP==
z-uFM_*WR@K)qB4;%j}!UJlU>c?WMhPv%>5j@0Q#5>}R*#p}i0H-SYgqZ-w@{eV-=#
z?ztmA$8OfG>HCaRzS%{E2=Ch@6}tCyjp+gATQUb;tkJUn*2%qJ-rCEaAu7mjD-*N5
z-^N(`)^DF}ADQ0WtNQlwUi;uRdw*}W+#6`EZL`hI*_LCS-=3X3*K7;Lm2Hc>tM<AV
z&fe>`k9TjJ#Jj!w4nEjhW@E7T#_vsgr^xTIRXyOj=fpFieU})1?lyd-Z!37>sZIQ4
z@4ebHTlPNH+ibi2^^HBUzdCKt=XTj%TK3+?u(Nxw)+`&FP%k#?_=+7i|5ml__S-&l
zZ{flhd&{~W+33|A-J7GrW%J06#g4Dsd0$ZEW!viXHM>LhMeZ$*^VzHXkHz}zi3PTk
zZ_e3UcKXL&78lihT>p}7H##!zdo8Q6*ZYaw-o1QNY+A2gwAnAWVDDK=-o3L8?(BV6
zq_J-&qmZq9zM5UhwGDgX{aNfLn+V#Z>|MR*dy%@`14(8(-FhwCMyXx<ycw(Pe*O;L
zx6knU-irUL_7!L?-p6@=*S>AnefPD!&D*yiV&lG_FMX|dzX{*9(U)i6dt(DzR;Hf4
zo1JIw?a%GrTk&qe9^tn2dk@(4?UA^hyf;EW*tYyJgJpjD*1e09uI_!NFxxhM5u+{7
z{BOIXSZ424S^Ca~FPV4Wvh}C;9#6Swdwxp89v%ZFo6ZO&+oetS_I#RbvA4Nq=iW>A
znD@p^nzQ$Vi~Zg|WsCP-o40e1o&ba0w&I4pJ4`3<71BzwW!3cGdx=ZHYI4@hJ@a-)
z?@hY>VDAyJe|uK@3GSOVf9KvB#mUyo-fXcsSnRRaYM0yI@QrHwR@BDWy<1?tuj!|-
zt=9~hy+@iK?>SN6YkRZDZm+~^cAEt(ZMO0)3vCWAI=T1DnT>nRk4o5mR&%$L`Q)>2
z_wE?m8~pnFI#SJSIdAyxJ^#(dwywo;-!k>Z)(d2h?Tt1&WAiojq^<jTQ`_?|SL~?@
ze!j=Iewxjlg_-+~T6XR;-0W?)_2B${+P`}CF6P|4Z$hByzLJ1R`&J3I?UQoay^l>~
z+TP2j{P#r&P1zIVeP{2*W7_r&9~RmD-YvKHNX+`ZcF#HXyfB%yZ;RNQJ;e`a+Mdnj
z+SeB+X3HBOwO3zO-nQuW>b=gL5AB@WefIJnJiNzg`VHF&n%y?9p6uSULHM=JIqpK+
zi;KJW-ZNQdV_79<^SzRJZxD~-KFRd1J*iik_bl1Cb8pe*pL;i|2ikt+IBMhi<ka4%
z?FGB^Ki=DYcGs`H%3rtdS$x37{#oT`JI}LE_D*)5WS4s5!=96p<@;YVsM?)pn!JCG
z@mss6$17|ng;?5tJZiUZUf2t}lO|K`PMdb@UG9`=x97%b+ogic`)eLB>}#9ZW~Z_-
zcHhK&Kf6<sUUnZ7|LiT&-mv%fgGu|s`V{sx7bx3(T7Jz|J!t2i^eI9%Zdavjwcqzy
z_iUJMJ^NCj%?8%H)+OEi`(oCH+lqBR*&7_TW6z0aDz-^_3459IIPBcnIQHfK`mje~
z%@J#%xxD+bcHXnSXTh`YYq0SCC+|1xV>%&audkG`f9>}|yIpGq?Ly}}+MVz#wzIwP
zdoOGL8@uFVA$!?G{A~`fe%`Ap{bY}*@;}>W-43=2JvVI31C{qInbK)v7-F<H*eb|Q
zGlBDfjoDh;GiUDG9h_Xb-zJ)2|H)IL`{yuk-M_SEg8ha4#WtHN^zAOQFSflP`^{#1
zrnK!_{@8s#dh%?0Hp=WP<xsInQxf0juVu3Dh4-619m@sx1$Q^uUOG8rPdCeqJs*ST
z*zS>vwB334vhBwNhP|q}X?t@V4({976uR$$r;;6KoQLf-)uMe8Yxml|UO2-}>Cnx6
zqBYm-+E=Z%HHe*JtK>S}cGl%{wzi!Y_D*@f%jWsN8au)JJ9kFUxo*?(=<eQ>0PDR`
zdz)=HcrD-aMSkkuea@A8PquQ{ee28HTN|ps?|<&ZJ@pT-?AZ|Lvo|-0ZJ(f0|K9kH
zp1lW3%=TW;@3!6kTyXE+Z_919=1jB6?ccFiEwFR%>b_?CO97J)JosC;fA`bdd!NK|
zA2_Nu!|pd{!Tt>*+V-c8n%iq_Szu?ba(eglW4ZRd&x7p`8A<Io{<zG3<D>igpL6=|
zjeoIdKl{;X`=@KJ-Oqn=njPc1oPBJ(#kLPW+t@EiSZ(L|y>{P@Wm5ZAv@_UmniRfo
zt`Fn>LMBN&(Wncy)8&@!espQq-Xy(=wi?r>?$vj?V5=#3Y_GF<_wEl?m-d#_y4twd
z&D%Sprp9iGR_b1*x+8m7x9RQaN>H?OKX-Z`XLZlMB3)Oz@P$uyuYXu=JE8rL?WDu|
zZSPfB*rv)|wfVP++16Nc;@)WiXZHU6$ZdN~u5gdi<Q%*6>gu)}@AdY5_j$ax*>}_4
z!jN#g0BOH{3dTbFuJU`@{AqXHJ6*bRU+^CVJMUS~ZLj#u-FM*G=e@J0RNJk0+-~>p
zQtQ4$tJm)12^QGrRbF6IC!4)@dMo!H%kRhc9J(;cHeJizhTmS>?#YL!J-^;Bwsli!
z*mp2tiEWM|$3CmGd-wf$x?rCP<5auWy|s4EmlF2H3EJ-~uFbaVoF8u+a>dN5Me5+*
z(+jxwN<`}K?UKD{bLofa9*(cpwmws)?tORl<lbZ+oxQs%0`{`c++zEqbFJ-*_8eR3
z78BckD?8ixRm!&Zy}q^wlvnKG_*ZJ<bB)QaUe9GObKVo19yVsX-tJJ_mogH2CM?R|
zYx%XrMsvcuJ+Z3-_O9YNZ_935XFEw~rmd%K`<{}!jka421ovM2)v%YBtJ7wZL#~bR
zw+!2Tu{&&KzRlcw@htbgJG<`fEtsUTcS%Rk-lqL*`-HX$?OReQzgJZG!=9u)eS1wN
zO72})p>AiU%DC&}fd<=)azFQKYuE32p!s)K(fz%9TPMx3eW1g-_abw{URibNJwd6}
zd$pZwYy*>b?9JXLu-E5K)83~n_iaNyFx#?q9N%+yDWA3Lrj>h+@0+mqT=Q)kMeQB7
z>oiZ=ieByAD;9ChmgV7(Jszvt_uAd4v;B~9Wbcugi*0VqSh)AzUp3pGsq1WYH7D-f
zm#S*(CO2o#M!)ZS73JpH+5}tg&XTyax1jIsF7KbOY>duK+51tG)z0_9m%VIHwfBkY
zx7f|IFWUF#<y2c<OGVqMoiptAt(<N5M@VR&%L;=%DJQe{@mlKI9^84(M*6<#-U%;y
z_O5VZvAtgN*Y<6FhV9iQn{4gFa`r0eHSb}KJh^xIG`YRn-j(}YUTw9BOx(TigNEh4
znVFOK?Z~yV{cKya@B51hc3qWX`yv8DY*USo?S1j4!}d2@zwN(YtM~ZnEVGq8%xU{z
z;?BKWm&fmYnEPn&Oa^5;rRc+Zi$XN^x^UR+ojG^rUY};By%IaF>`_1CVqI8Uy?43e
zrv0ak)$E?`4cs^Xv!{J<iTD1@c5}PLHp%_!8GG&Lb64#@p1fec!;G|jSwFVhDyAIW
z%Qi`7@4E`Ny{*Lodp1?_?cFs`Yu}>}UbgE*uJ75ufq8HEnSQ&^!h7tZR&U(bzNBmK
z!7g3f4_=(MhxX^&{(LI7Z_bI(eQ`&R@4tTg<UaNr0``|XtLz%=j@s!bFR^7b+`VsN
zzS+J7Mt5wdEZSn%d&a}|yU;{CtycMc|7>pT<q;Fz*SEiY@1@-{ZLaQ=+xPd`dK>Zg
zN3G8a-Pp4v>b9Lkz^r`%p(pkj2FvW*#CvXU{pBb7ekV<`TR#2#-sDMN_kNtVX8#|C
z&Gu_WTldFi{jk@QWIo{W;KIIeNA&}zGu<4Dh06Bda}YWpy<@WN&OdMWt~HalRm)go
ztFeCF-p?mY_87H(vTkSEv0FD~maW5yB->LZoAxzli0oUxf4QBY?D@UCiyHS$mr}Lc
z`ci%0o@d|ee5U@hJ1ca`c9r+Oy$|>P-+RJ2!&bm3%=Y)2PMiOSn{2+vp54pYB(Z1L
zB-4G1Q(E`#JTiUX>LW|`F<CL~En`#Pm$*IBuD4xcua{xb9y7bneUcy6?E9$HWA~)J
z)n=Wh`aaG_Uu=Z9)b_nAJG=MBK?&Ql&o|hfF5=zCzUIQ7#NW-<lW$(JZ(~@t@8^`3
z{YIr%_PIVO+IQAT<-mu8`u$(u9k&lIWZiH0?Y8YV4`aKAIVbi%<UVF+`y_Y&TjdS5
z>#j)e+r?UD-*I^2e%~qj_B)*8_V;akWG(icVeiMn1GYNHoA>66RqTzrcw_G;UWUDC
zbKlr{XC&G#<cQw2LZ)NC{~T_6RqHmpKZ^YJn+tOGi<K_5w@S0y&lU8-?#}IM`<2>L
z_jmnJw7V(PU^B-|aNpATI=ee>U)sBH$)i2txu^CVnsnJV<b&+KP#%uGYm>Zf*L8?n
zZ+LynR%u;_?cPb3_lgDl+1s=A;9l9K9Jbmy9CociM{SG##O;-Tym{}c7Z-O+OGw!o
ztDoBQbeZCA4*yUa5#O@C=9UL+mK)95E2mqv_pI9!yEnZVdv}#**r{HBv9JG&qwT&G
zCHo39!}py@RJE=8wA;>V;sHCQnpRuZ=;`~nzMruF&Ceb?iQ6purwD!7-)!^2?&6Jp
zJC@5K_A^+y_onmo*>+v|WSikXdC#ubYit&LTxgphcFB_G+Woy5vx01&S^eCT?=fS)
ztHUw-Vi|`6?GdU6?z4(Hlr0cCP`>}ye!<n!4mxj6?Ta`aYdbq%#E!?#-ge;^sr|_;
zN%q(7Htb{k>t|POIeC97bAkQR$HI2Cxy&}g_R_YCZZquj(S5qt(<XoKqb*K*?G6a;
zTj?xgvoF$h@0x#m_SpV<v3HA?!JguI%(f?<rSBEdir(8Yee+%sn@hHi57+Nqw^+#T
zWYcAv<<k87Can#!<KSf8A0nP&_pwP~-*SmiyR(fy_AC)OWcPE=-F>1V_I9!?OZFXL
z`C}^{Xk_E?ce-`Wk4d(f&u-Zo`X|_|^kKBinq6nxs<hWuVs6h~x7_qS+^Uv)zW2}E
z+unI`ugG(ry*)~+ZFh^y?EUqz)0Qi;&vtRqv%QUOEB16OW7_AqpMT$CUaoyRv$omP
zeb=+)zg=Ou`lR^2zBLJZ_nlYTyV3Hyt(@H~yObH0cF%Y^ZT)&`Y^@o#?fbCWY5x|5
zV|Ht7TlQZFG_~8)_hYw~weh|(rqg>aF)g=!y?o-{jTbxia`t)d^{M6D=g)M&#(u_!
zy*FoB?mx)5Wq%xh&3;h<5&JHW_jYT1jQ6KslHc$6M0!6@>jwM#H!s*9@tnMSzRuOX
zHIXlD*q$ld7WMM(JG@C`kIf?<o50lq`xd<_w|Q8?x7Ukhv(4I7zP2i>pYKU|oW9rm
zrMm4aoyE43goO59n<izaym^hSlgiY+f-hd!&ddI0%kZ?#_V>}&y%84#ZFdyDv{^Y>
zXz${*lD+=-J*;La3+(wbYwrGZ)hK&I-zNKQ(Q)=M=g!-Su6wY*dzI~erT;r^d!B9H
zduY;HyXuPf`*;+?_RWZ{vip9{-R{Z#<o)~77uvmNJFwUBoA-W^HYIyYZsmPlmv`AH
z@JQIcK7V2F9kWndyDb-OrwJO{&b-0Ach;nE+ts!qwzJ(8_qL^k?d6p4wcWe^%U;nh
z+O|s)|Jm56=UDs2D%dF8&ELxsGJVf#xA`_TpQh|<TpDbrz!|?!$JuJ12jkp*K6BRY
z+jL{kzJIqn_x1#u?rWR7Wnb2#4Yr~S-t67e`+ZOPW!Amkix%wNxA)<mdpVD7u3o-r
zb5^l+@An7ycFSgD+36{tvnju!yw~W_g+1YVcD9YSQhV(>X6@4{J-D~7Y2n^3BkR5T
zA#8S`E9UQ;8#`gQ`|_T>?~)bvy{l`tjhZyiHrC9+ws?DnZSd!7Hc^lH56nzkWbg9P
zVZWro)%{KV?e?qx3LdDpWId4SeahZIRb~I%*2(trZpUnQE9vijtNGNX(BPb{#}@9r
zS~*+yY8{Q<$HTDSb~Q8KUddeywue5?uoYV%Z}Uq$*Y4fLnbsR6h4<Ck)$ctfH+k>L
zHkEyU{ETfGUDoXFec7`wx$?ojSAI2nKOJ?qZz_Li+cx>2%^7#A13nYB+nR7pv-`hY
z+J5U{xqZUxxa>Y&>9%22yKGYyscvhw)nxBOyPUndPY2jqGx_g1y`yHw?_ZKOS4-3O
zR^Rz<{p)z2t!o>{KAU$7_TE}$xp(rHZkydktTxkH_uDcyi`$jvPqOz)wb-BW?)m-|
zPkrpwzrWc(QF_t-y6tifKL5S;U(VjLKS-p}R_p|aZQk`;dy{)O_A>W-@AW*Fyz5?)
zfSvMsGut(Fwzf+Z?%1B0z_yR^r}Ey^xCyrJy7%ol;C{mPoSv|4*RwKPrr#QNt7|{+
z(e0JpD;21?N9eMI&Ad~dw%eKIceAZ>+{^K|#nw1?*`CCOMteTjiR}B&SZDjLt$5G1
zqto{Ma8kGB-N?54W?JN)**44eW=iSrGo4;+6LwT$Z#k>rz7vY8_OkyuzW3v`roEqJ
z3-`n|Y~1^-f^+XC#dCYCb@X=kl+WGUZ1#K4rYkG$n4N|8*Br^Ud&n$rKaI=Bw)O9f
zeHZR;wwn>vy8n{@N;~<PH}}0Ne!YLkMP7&Hrp9(HPrdBZpE(|QxciX3=;ah!VKsI;
ztDoZgGfubfpLyfqo*hk^dy9DzcJDc4X1jsq{GR`JqxSMI%-QSRW^b!df6a#RUbJoG
zof)>%nR@p*^Jwp@*wbh$soS*gf%{Ke?(ns?ZjU+bs*iT>wT?`&i#Ptf@9aZ|13%u%
z*iHQ*WN+(Nwg1Dig#C$(s`gvBXYYS?d4c`snfCiP*I%+OeE)E-vS*jA^CnZ<;;LUZ
zAAk7TS{<skE&0K}k0te(O{Z(q-j<~ew(}>v-xKMgZ(Am=v3FzI#JvS)Yi+k*VY1E3
zoooBK;PPI9b`iUvw;p!3^8EHMBp=x4-CVSPVy)Hww>dZWSNhoOkAJ#npPKN~eR8aK
z_62Or+`o9&wEarW5&IWLgxFWESF=ym65M}sE8o87YuoJJ^Xb?dJIu5<UBkMU@0Xz6
z>Rh(HyMM{p>ZdKYdAa<Fb>0D<z3cag?fsf4y~k6vYj?WiVf$sX-tBk0z;1u#Cc}ZE
zkW+TNPqgh1XgJvA@@?6dFLu-3V3O{>J8Q)Dm71s6h00m)vt?x4$FVBa?vDE+TP?o>
zwzKy7+g)IpYn!d$y>Fp+%f4IJrrEK^9@{<ZRrjtbQqs2TWIgvjP5!&@-whu7tF9CG
zzuGj>uKKb3KApG1woC5c*ju)VZQqsaO13_01Z)fL&D?u)%><i^hga-nebu!$gP&*b
zg^Kq!+>A_iR?b3uJ?=*Cn^<CM=Uv0LZ}!Pawzc(c`!3tPvrV1nzt>uZ-R|O@v$ndY
zPwzeOcK6;7SM+xuExf;X)3XbEdOru-N-PYq&gt&7x9Ip~xBRS-T}Z38{Z!YkeXqo(
z?4Nq|!2YkIo%Ukcs{8)$+rO`BdbzEtz=6HJOV8LC1kc<P>a<}`@ryZol_#Xz#9YbW
zE2q)7x37H99;*qx_V<3b+M8Qv+OfV_w}1cQiS`Z2d<VK79o`?`ENP$pbk6>|4taaM
z$khE@?8W=SHrd(h>=3ixefsKtDLyWH*Mm`et)3d&DNT?&(AdUrZzs2EkDye~UIx*7
zcHb0}_x7E=VCR3rcc1>DAA1}YzS<{nEpH#=^}TjeCYtX5Vd~+q!d&sd4aZsgyXIBx
z*PX9%;MS&X`)~C;-#?|~lD(gI(|(?rQ}?gBW_7@A$LxJ+(X;L2<2&}t-nQMpF^}Ou
z-N#&ezv|cazgiaUR~0*Gmp1dkJ_)h=Hq!4p_bH_2@7?^oVBev-leQPm_1R5Zm2Njn
z_xj$hA8+<P{y)*~y=0HwAK8_5>wnJPcgWFuU(i)XJH<l7eYNbIcJg|)c5HrM_a0Qa
zy>~%Y@Lnfxr9IVEr}xf1xzl>%hGkYwCJeSmmc{Hn((-f9ji#CQg6e(yroOpgms+!K
z|1PeMeR0dJ>~1>Tvz_$cd;jgqru|<cqW70{9kpkD(Y8PN4$pz5#{K)JC|KC9N`JoJ
zB&OcpezBA-r{fPh=?$y)FDuElmCTdcv#foyt<vKbo0MZ&HhD@>dm9rpY^QHsv$s`D
z*>2L$&wE;muh{rsJiAAtjCo(@MH#yv3F~cA4*#_7SL)o$dQsE%UZ<}uSK^L6v-dFC
zc~`Et4-Z>tXEXJj9gAkDt>kUl{eJV~_qXkvX7?qC^?-2KNxRZ8i@oOe`)vci{MxIY
zrN2kU`>;*11j|0t<pTS9{`uGh{rR$c=MmX`e5UL7DedmH&zLx8pI&RFeRFQ4{fxWk
zZH_4)-q#%cU|+PY;l6iIckMCxqqcYcT_rp1aF)H-W(4hB81Q26R$pCPLD}lPMrx6J
z4jQ;yKYv+m8~bj~{<NQ|_Cmr@`{!@c*{>`x-G0W4mi^}!-`(HFkZOPS$K3rE?w$5g
zX8rr_L@n8!;ILqyEvtZ?fPee`WXELtx13FT%T@gCI;YCnyUo16_tvc?`$X&F_Wf9{
zv~OzEZ9BhkS3AC7O*@Wn?EBLd^X%9rM(@2L++>?CJ<;}D$^)B63wn3&|FB@M?#xfS
z=VS})^Rl?Ur>c)*-z1G)do-;l?LD;e@!poBCAO?rboY2iPqyVad(^tX(qw1K)%&)0
zU$fbIB^cR0xVU`pmAtz)f1bA3lpRsBRciL!yM3SI-Ycu_?J05jW;12R<~{p)8uw1%
z&)<7K%5>kx2Mv2m<DTy2>DJvlX}6iJ+w7RVSEk<Dof`LLk9FuTn~3V{J!*m>d;c%k
zxHFE+)z&=Zv&~$K-Il&n<@P@73$sm^TCsQC&MA9tu9MogE0=%o8nw@RZ<$@UG4q&a
zJLjM9KIKl)y{#+6_MCj*xBJ}hUR!U6aO-@|?Y3*rEZ*12pSn--<Gj5aRZ?yLeoNW+
z^ta{S(|r$ZzZ$aJm26mR=Nh%za_iZ>w&%@Q_Z`#nx8BFJ-`0om<lajUkMI35(Phu{
zd&~DG^NCo!_F>p(#bsqT<1w$j|Dt1i?Wfe*r5xG3@8S9;dwD<W16+Hv_n8#*?tjF%
ze{c7Vn>MCv|JmMs!nya9Q|;dA?|<wSG`eEr6T`cAw!?$HmlTfdp8Z*M@BS(^TjPuD
z`yLi#+5EA(w>SCH-aWOVM|RtqX6}8nLV8~W+ff_&Mlt)$or(vTcDC9rwh%us|Ag&+
zX~SrHF`sJt!z<_PKXhu1UFthkduvI7y)mogZGFr3?Tt$e+iiT|@ZRu@AlnrF#kMz2
zHrv*&{k3O9S>xVzmc@IG1D)+W^H15nVf?-Khr-dl!Q8C2<|)tjcFtL4%c%Neuc61D
zJr6et?vvWswfn@`*L!~(G}uO^m+Xz%anZ)WIB1{x#Ob!Le@@(Upg6`R-Co|-=z*%O
zyo#x9v22j-Q@)utlZCF@e!m~Jx9j*?o6|Zhwo8xA-22vc{@$C5O>Mq;SM80RICpQB
zaJRK$tAH)5vF0A3gnxU}!XEB9CZxFUzlPKvyXq&lTmN3ON$69vUADA#cb3g!TmG&a
zd#kcbY}c<iy0<|_a_{Fuk8BU}yx;Tv?kro6ua&!#+#PJ^?^wF`+2gIY>D|9=9EAT{
zU%tM>w(7@({afRf+e<zv+du#57TZ}j1o!#Oy}a*-2H*bqFD~z!x8?4>7e>8ydKXIV
z8x2bB4*Wf|KSE-Wz2`hrdoKQ?yY1@2?YCRM+c(EUYrk45!+{KEZQJYJ>-TcL+r4*@
zt+d@?y`{U;uYTNf;J?70br#CjC33%QyFV|nO@9?|;QAIOhwD}w_gAdww7=apeZTz!
z2mAByX4&7$aX;YiZfl<s_hVmY;sSe--zWD+ysg;Zs-?1DZ^>r6W4kQvCIA1itK;D~
zpc_5I{=ugh`+vT-cE33(X}dRf*50G{*=(0@*ttg|L}c%r-wbx9cf#zlt9h+Wqc`oH
zJY$K?Ti-lehLAaX=S^I>H+N#8%_f^Idv@Puwd+VKwrw=!*tcrS2OF!O%r*z|+xH1o
zcH4Kp{cESieP&<%p-<NDrpwsRJ-uPSlGUlb-=vrA+dMOL&kBjzwihPc-P@3qyjLTt
z#g?^gnazn86?<2FU159npy9r`+NyTbSQz(u^xxer`ANy<?{R0_>C4aT-QBOer$YX$
zE&u*hn?s)7w(Ne4yG`W$ZT*UqcXK{lYNhmR&mPM??`#ey1lv}<<F}PFy0lm4&6U09
z!nOADY9HGhnEYf9?=b`0*-_eid6%>8%QL9hqyB-{*5~<o+eHiwdlN4lx7|7`d+)c6
zXKfy>;M)5(TEUuQ;d2{d0Vi9l2QT-?_)oHBkC|e7^~6$}gEIeZu4O9NoiF#a+AqSf
ze_rbjyYsxxb~j7z?oqkpxzE;e+1`%byn9On%xvqVrrB1jw%A@yxL|AQ#<BO=IX~Ni
z6%TAL>Nf6Cxt(wO<;LmVhuDLxbm~oZE%>;=a%EHeUX@DreTG)w_C9v~W0NtHX`jP)
z*?oUBe(p(;IBm0+-N^Pv9+UMft}fead#~=Dx_kScy-BkBzI%SR-L&M#-b4=Rz1wC#
z-`BY7@xINE>g*nQ#oAx^D|uk)+l_YI7eDNCTBK!vxwhK+fUxiZvqRPUd|ol^6^Q8F
zXD~V5Zl?0$-S6hN?W=5@vZrru_P!(67w>!BV!CfpbJ*T}u0eY)HcZ}I5c_wJn?c^b
zgzW2k&uo~v*YE4Yy^?NEZJ(c?wRdGrtIa}liM=h+LVHbFm)LIg`?mYxlohrQ>bv%I
z)ywW%ec{_)9VPQUmb-2D@?VnJtL|{nCa7YP?U}^uw(^B{_ujc8zxUg-Xxo!rK6`(B
zYTB!QiOD{tdZx`{o>Y73=?3=F-!1o>>Gtj`loq#l;`Xw8W@xs*>+{{cvhqQ8zQVQp
zE0zZBPkZ&mKH{y~fuF}i>=*6o+3%F=e4zKpT|1Tz6Wb%Jd-l1s*zb#4`o!jJ5ucs+
zntgjYQd{<N+`VZVw%x_fTv>l#oJ`cd1#v?A%s<Yv-ReEbwr?imzQbNUcCJg_?p;~F
z*fwml^1i9bEc+4;x!H9toM&sq@otZQgp2KduC+GX6)mj!EBD!27v9=))S7SK<&4_B
z+b{pwd*}4uy(O1A_pcJswmWZ@W%q2uGP?!RtL)}|ShTOQ>x!+;Whq<l=_31uS?BK0
zz9(*9T*PayQCPgca?gMJpPkS5H|I{;fA=G^{hpbh_Ol#WY5!whrENomkgYo(x7|%E
zOY6n&SMRNRd&_pwJK4P%jPGsw-?Qv{^D1Dk)Z+X5#bjmdzo#48tE|$nUwKaWK#jPz
z?S*S9_M+E2_Y2yyI_S=yYdd*yt&QgqF+286i)}8o&ayRGld;#a_~2d<ne%(EJ1p9J
zQ2oGe>qF7DYbVIr1U35YmAqbT+qS=Z&ns8;Jtu#e+m^hQ*}HX_;@+PHXZE<y$*?*3
zf4Ysb)mhtz>>u~==dRy7_d28Pgu8QXi+|49v*@6)ZNQK0y>5$p_5^32uytI*vZuTK
zxs3#evdzQ`-))#P`fR_Oy6&CaA#E#neYdUULRs7T+v>Y>7qQxIUA}4W_Sh{p%O$4l
z?a!IH*RM`!pT65V8>vKpo6EwCc11s#?W7C@?HuHp?Zjim_Pw+bwc|)pvtyP`-uI@y
z)h<lbe&1&K{rf`Zy6$@-U1i&K@W<W`!=HQB?B?6OQE29#D{DAy`_iV_hDfpPsgU&9
z8_E!DTjsxN*OKRd_WqVNv)#P^wT%~tsIA5GlY9Bf&e$g1oVNG+1hu`9S!Z@H<z?P?
z<6@hw|He0a9~|SbdwP3=tpcNutvugF+sTTv_VzOFuvrl+xNny@r`>9;DK?jw|Jdf%
z`PwGTId1FH#I{e{)p^e<)q2YxE^K?hIk(sfS~BnJYq_zv@p0eYNm{G+E@9)gUGV(&
z-g~NY`zB{P*gkj6-s|z#axY(Yifu=H_}*z-JNHaWlC>#l{%Cvp>@^!PDP=pMo+W$r
zj&Ip@qj<O7Mg4X59p-!YGjey>EA%znudD3dSGup!zLovkzP9|CcB_K??T$qL+50qE
zZ_g)IEn5kHp}ku_UD#{T>a^El?|qv+SNd(-=j!dX_Wrdu;%xsuV<EKz!d2hx3fH&p
z_scuAf9J(WyGhgA?dP5q-JieD&7Nt&QM(JL!fm$X>+bzr=e>tF=lI^h6Kd8f_c{0N
zDV4JmWKy$Jd3bK`{qO7dmL52{r%ivC&A*=_`${kJ?Oor=VOwayzHh@Bfqmb3e(m*~
z?z;EI!!>(<I&j$QYl`l7eR#&s`(CEqySbP5uF6r^FLl^qZ$FpKzEuMA?RrYZ_Psr;
zV}JW}-~RfyA@&zLe(&Akq`BAGaqfPF4OVvN(j*Uvm=xLlvbV6SdCF?{{Y<A_qFVc2
zpFL8xEYrem@^m@&eb%{WonUf*kK5|`wmz4b?Q-%s_ige0Vk7AkYJ2&&<KFI5YWq0$
z%kMc9duVTP&b+;6zpL!KKErKa`MQLCPtPvfr?Pv(zIf9QHtGLt_g%TTY_Fo9-oBXA
z#dhtk4*NtEwe~$Um$DTvlG?ZMT)D0OnikvXzM^}VZ<4g>-?x9y!T6)Li`ysKUXA%+
z!w|{3r%;${-#0n8Jstewd-|E5?>?n@(Dtwh(_Z8AOncXD54U}Mq<Jrkl+3<+%M5H+
zAL7|7kYjKA_N4CKqz%XRR^8;<`^i_)_Ik0nO=;rOT?MuLc3Oou_8#5hv{&IVlkMKg
zbNAFOnQpszak6c5;?g~96Dw?Mm>ybj1{&EuyExG%v-_z{ognMJ_vul#E4K^V6jdkM
z>g#mduJHA@EtvnuHhJ@;J-JJc?7Q*nz`ohMqV`;MSL}W|@3PbQKW$&$Mw$IvjhpwG
zZf~~R`K{AdX&w9ikAnAXFTJnWx9Pm|{+Tl*?ape8*lSM_+b3llVAnG3zRjQIt$Ww*
zwA{zmd~(nAwR>zIvX}0)H%hRXxcsE;s}_d6rZ-jg%0>k2Er?%a`?9UVHp20Vt;g?|
zwpJ^eZDx9!?`tW}*!#$Wf3J1&U0bmUQ}-%LKHC4Ze3kwF_gnX#XL`4PLvWUT-mIMm
z*8D%cU*Z2#J7v?d{j7@@?!B~Q{XV~aXZBtftF<%qoVZWXp?_Z@-y%D9hj6=y71{gx
zPaD}?v7BWWcPnU*?~!|U=6^cuEO)8yf7o8SKe$B3KCGmEpY^@t`_Hzd+a~PTyl?6U
zrTyj>7xx>^RoW+hGid+1r>pEgm&w}4hnL&6`c>PP#_-$66+PJRbKltZ(B&6<JMS*p
z^SWKaPUpeTJ>t!0_iU?V-OG2F!`9U1hE2=i<+dvpTkb!eIM2S`S>!<3?fv%uCb-(4
z++%3pu}We8(ue!^|Fqm|cWc2h`<14>)<3sI+5X9p-gnI=*Xr!U1lxoiY<u3m*tB>0
znjJP@h4uFC;IG>AO|pJ3!`6>`W-Dp$%jJ}|wVJ=k_R!`z*1ebNZTFO@?|t=8Wgpuk
z`#m3RC-41jqG7Gi5Mg`nM$_KejxYB5OHbTm^Jc*w<>oNk$x$qO15Re_E!sKT#;5ME
zja1<}o5zWHwr4Je+3u(~W&39OsXhNchwpK%ve?t|XxU!V-pP9f`C9iLG2glO0W<%;
zBEz=bW_8<b8vjS{o%npq-mf#6_dQck-S>%s*>dA+=Y8jnZP;7&Ib_d?`dND!pU$wI
zK99?;%VNnMbN6i9NXZF%3wFouiNCse@5)z8_X?-%v|Z7yv-hUW9NTwi&g~V@-nZ8)
zE6(=bNsGOpGYtnbc6f5$RAj}<?SI?Xn@rpoV`%?H!<?N{&uE{ryz&3_?B)#n+>91T
za+?g4AFc^5wEEPpV7cIDf<?ro1(xpiX**fh8t$6-ZSD@C^ZL7*cNgrsay-mxpxifV
zFMUFw{}iw8UteFt`u7YbwwpQi58qbQ{<x;rz|Or%yU&Jo%j-ExS~hbH)=}Fv6S-$u
zb~iO!*e?8KAy=__$NE*`+oEQ#H}7?LwoNa5`VN5<y<H;P)>;y4@0M&q%h&rQEcd^X
z*>!1&tkqWTmYv1<=XTEHDcQNSx?q=Nhmz&itc_MueQz!GLSlA&7MyFbUhItV+*L9=
zpEk|1_<c`wSG_gY_KioDTCDhW*@9SmiB%WbG1EdhXPV_d+f6&qx?Z%L_l9TJl24Mm
zc$}wLa`1TX;(M6Avpstk(fLhbMwIILn0>2~^V1a)I-?n1R6kQUP05nAyd<eyWqnRd
z@|2Zw<iQJS#Jca%xAPXCJ^4*UjQ5#)Xx}p1J)?8G=_5Y#>5ng&@5p#>61Z>e&Kff(
zOQpXJJAUX!Sa$y2V6yAF+K#&qRCaM%*IFuEeQ7qGsejiCLA#x2Zk{6AAH=$Eq^Y~T
zoy9Vg^YONgm;Y~H+-<nE>qCqMV+yN<$UD8AH7OU&R4zW<Iz`@e`=rA5Z843KM&Z>_
zMhc3DbRWe2HC}un*udH>$?R!b@K%`v4-J)r*P9X>ejXxymh0C#?`#R4v}>A<t(EBH
zyA~&HZFau<KiNWi<&vGL&-ty)4VGFQe=|?$cYB(&hTD9FwL6|HUb?7CwA%cUa^mh@
zzSAx%WEM96k$z_2Ax^BlMmcjVjl{VvJKfcGID3ig3|*12b6q%xh3A79%g?!!ccfVw
z?b;{%o9Oa@SbK?8cb4Ug<)+ol+x2GVSmh{A*~uCDX@~IUBRe&ew0Eg~&)sRTTy&Rm
zeiG5?K66WeMGWKS?Zv|XE%Zwsn5t*Y*jc<TddCkl(Vdp76L+5ZB5aYqh>d9TwASUy
zF+7b`RyuV}CxL@?UHSI}CH@%|nrwb0GNrdK>&83_SCwE3QYY5G!$qB<cDreV3DdUI
zAC7N37TjZO5IxUiik`>z^N&2uZ?w3WMyu~JlWjU|Ol|*WZTMo9Fz51)UpbR^r0o;6
z+$+()J++^~N?xLDS60jki?!N!c7;{M4p@4owttB=kL&2S?Ke1%ZkxJzvjy7{8;g8*
zP7D8It9LlBUT>=Ydhd=>oA~X|T63sg{}SuIQFWss01AN&T|ujlD;q50OcJb|-V5zw
z_~f~(cTM!piR~+Q*4+=>DK)uxm&l~!M3>KgKNoTc-L~U(*A(Tta!recYprMRhWNKE
zjjYGn53pY3WPPE?>1=k43uN~|;I~ryEDy__+wp(l`W>2|Pw$Z0FSy;Sv(Yln;o8oF
zQ_L--l&|kNb4y~-msgKHe<~fhm1ppUNmy6+HM<_iv2MMUkK=XDri3YTKlr*qr?N*=
zcxSu;wZr(?JV{Flp(_S`Z*Q2Jvd=Or%~dtoZz*XSEO*z;q5hjuw#tN^cLnwh`f}~W
zgeMl+1yjsI8n5j1iTQ1@q)grHV*REa0@cjB?rci1NHJ=#JZkj8a>j%emLCi@Zohu}
zrX`=}(w&Ja=XO5&)M9BM8Msrb{fx!u__@2jHgyo4Kd2oae3y6Vxbn^4RC#Ene&m`K
zjkQ%A+v;i;>ug;eWn`ST(UkRsp23qW2UA1VJoCo}AGWvp{xnt0s57dNxMRNZd5;Op
zrTIJJxfIMdP6)HG{&~gJ^b*sy32naHQ`K)9zo}WN@3_<7bmMgnQ>RZ^<^^xtO%6q!
z+^SnpLUcPKIQ^03Qr(;#%0_xS16FxkiQjG8vC??ct_jb+n{S#ayQ41Vj^$jTE9TS=
z7tSar%O<AdmS+svtvW+~>~Oo+xa;c-i=8Jwzu8t)ad6l2PLZ8`C*BhsZw0S(O`Qbp
zZQE^q(KJ|GYKNIU*S3XRO6Jyz1!k-2cWuqT<Z1puXA#l<__tQs;MgZM!-$Cyx+j)3
zY243uG)c<7p;7+ph4vc7qs9(OxAo-LPu@c9a3R*bpZ6jSL25U;_Zei=xa)N{*=Xx-
zPSBA$)~a7q%%KtU{olqjPdvbCK<Y*co?(7>2iN8*i-gQ=mW`KoTLex1yyF`Cr=2VR
zueIR4w{>e{tfhrWYYoxmpL$I5PL8HGrh*w~EuAbT?U3EuwByM10&|u(fjdp#@3$zv
z9<@_lZKmapODm05U#!$y`6^WBkfyKlC+GY621mE+_MKU`$!~R`V(GohdInz?Xbkpv
z5Gguf-a4<-GHo-{&XBzm%qO+}-mZPr-Lmqzjz#6|^j)^5;^yA(8Hvu9gWbQxy3hE^
zMN9D`6D&n9zB3nVeZF;<_R^jI*GZTk%>HeDV7uF{Qp5K<lOz489xm~`IaY2$c9wYt
zPL?}!wwOmIaP5jY{?NQMO4hQtNMKjw>w~-IJN8rE-kkM})@!*FExt1ITZSwXwb;kS
zymOrf_fBOG{$0{nJ9e6^y|imm!dJ_`lOi>AO5OGUWwYwde6xOIzzipS?H!JWmwPhQ
zqmPv72NY%*lqIH7y`4jBylH!t?)bP{cBg6GQVVy%Wu_|*KC(#j+P3}J4qc1K+J<Jg
z|AlOuv+bDa#L!oIQ;un}sqr!!?MO4!(^HzI7bv2x^K-=#J=a?gwbKRSHeFt`deE1r
z1(W&ALRP=tF5JAt!hm7nwl8y5Se#C?+0p(;+d|IB!(waNOcTadlbytd-_A|vErM6o
zZdd5}WU)i&keR&tft~N?Y~GO?th&=hUTIhSw16F3r}t1j{SoWFfl_y5@7EpoR<E)=
zvb|%cw(e}p-}^7`6#0;_v;V?t^KuX6T>|q3cNTk19C6_?Q1LO??OndMXUAhv6U$&5
zww<>ZrCMD4X1p{1E6c7YHe1avUFft766Um=-%>na{aT?V8d{(*{PFOjPLp@9-qasj
zn#sjebfi|?(s^#QQiY3W*SbxgChLIIP!mV-|JbpeXR8I1VcgFBkG7fpYG^mRk#=>9
z)V+3#2)2WkznCJ;dQF3O-km;k$NQ3QOJ6Soi<*!mi&~zTou3z%?0i~fY;kFR=C0q?
z*<0`THCuUp;O^nww1P>XxRJf+;X>Bu?EajyJ{@7zv0&o?xqH<;Zcb6JZ_FU}V8&&F
zD-`z!ZqoeU`(3ek*8+*3XE&~_TDeZPO?swUb<Pg8>C?QFtIK|a?>rsM{@5F)V0lqT
zW*f(RCCdqgv$s9Snr^YzGi3+!`t+T5qH4^`w?*vyxa<hg?eX`o8cp2&CvR`%j4;;e
z&ec(uo@rbwCTtWO-fdo9EMZzKCt~FF<CMWi0YmfWetwpd`ptHVIo#dx?YhfOeQWj|
zxn1?<MQnZB*HpGxd|)ghI={XBQENG2(_4#+VdXo%njbgIXq&#%wO4V+KKH;K2lh@h
z_xMt?Giu2kqRpdroV^j$v1)E!V=?uPljR42OFO#v_U~N&<j?k<vli?$H5c9WuGf3#
zCNY80`iR;APbc#j53Dd<tEy_r!hL5)cEc+3>!)}ukCx7`nDijaBJ!W&F7ev8=7Fg#
zR8N0X6kE;Mb|+c(6^Cv=8})p%bvV=Z@<*$;<w-p@*W#1gWxV!;X^qZts@wZd{)VOK
z61kn5BVSuAiOI6?72dt0c<Sbz+PgOHe6dSrr*e|IrRYTx)@86;n_j6}wd4J_EoReR
zXz1Ruon_Hiv(Z%Rpx&0+EC$m#KF18b7wH($DLO~IHO3$Hx6OHeZM&4xI*V}4_8p0i
z96M7^oUpK#5VO?JO58c6^We^S!+%syS0gRpclJy$f3f$#cJ=_J?XR!dn!Jpcu!u`d
zHT%{gYw_fJoaL{Hemnfqa)|B^Qak)2EXyq#j&p3k`dQRs?g1u~qA0JO*XQ?deRk*8
zj>mB?Om43}xWmkJ{%AU&cEFFCHzGqo@@$uJ;L&Ca1|Fkrg~vIz8GYw6Q<yAiqPlvP
zao)wo9n<Hk8=L*yPV_phfeJ*2O+Ss-ygaw=V*eM@7pG6_WN>nC-Q32s?I`C5WBJ34
zCXBJ>CVUlF^oey}-3cDEb=6Oe-IdGDo}04nXezjC{Ce6Mvw!cojIt^?b}ZQtVNq8j
zFzCaDSpO2M?!*ZWEB?xbCM;8Ko6r8$Xnu1|t68S2mqoZp=#KA+EL#($rFYGo^2j_a
z`KslDGp0MfGIm&69S+|)$M*D27rt#4Umb&Y)<;Y4@?UMZ^DbZ9X#a!QfFD&i8Ul0-
zfdyjI&8~kwVs0dsx$S7#R}06m2^L(tjd$ugN|?@keq_hWH9}_nr=o}+ry<tq7XA}0
z7y71{+j~#m!6@KpzIW<Av$((6mYK)q?l`vX<90{IJv;O7-XnTF1F`mgReEUt@e!A$
z)24olr?USonj|wVq~@mV6!BhVuAuc|=K;4|^CG#7VbkthedVX+y{1M>pYV&8|Lmi7
zE`6G~OK$h^ok#lD?+n!o*!5>p$S%)?BGzk4KU=9SzrRB_%5Rs2iG}6F=XN`f3ou&g
zM9$pBXf@69L-L7TLa{YOm*XFLryF)~SD1Gg#F|EH+L$$e>NewZw>N$G!exUq)3@#Q
z8uBLZoQ|3i8*c-pPS!lkvSsO9^Jk)NMqlr_ZMib{xM|P|1(Of|ubFhEYwQrPj@rq*
z{4mk$OLQ$YETwP!-*)KdBa4@emOGkV6n5Eccx%DCd(w{7*`d31vvMqI^CnuwS>Lyu
zv8ThlBCW$bw!&>k(-KyTZ=PWmHg31JtDn-`5jt05m$Ze_ut}!_6~EfEmCaMtJWN|p
z-r5nAa?z5ly3+iK*2|q@mk*mCi`uv2Pho`RugtRT1LeLivkse@sO&V8mio2TmoLKH
zb^Gk?pRNmT_ZQi`)p{boS?4BgQ!Pto^ZfVQ482!ZYW>SHGgf)rr`x-oNw4Kg^VY6h
zA(L*oTFsOBVWxQtj_D1QKL*O)D{ib7#wDqit%hQjdv4a72HX+Z^-U_poNNCRi#IaY
zcOAdtzGK_B$3&NF#QLKpbAwg*v1H4qR&G{8E~YzqIQ(`U?Own0v9;UI(-O0HJr?@9
zE6-8TnppFQRi|dM-YkCNrESxXE!pA2y}|sv>3s9AJ=1s8<yxBW{NQ8R{`|K2R{y^{
zh&6AxsAE=R+@)VU%X044jvZU=&unM-q`EWqu-cAW&8?;xdhd5i8W-$1bg7Bz>A)*y
zgW3HKRukLh1v^S+tlcr|Oq}UU#YN`bpAMN_Nxrr%JM5}O#m7XV>kne%<GUWaX}?<V
zj`WmV^B;-_4X<2{vY5Af<JKFxH_Z0CnC)QF%Q5+$yVcxBGs^t5+F28=`1&0JRt-B8
z&Xid&?h7=RGh1Sjdv4MWqX$p7F(!x+-TvciPq7F;J=^s1{pOuvnR1p=cej~2tDW4j
z_LGoBlV`_{$*2BX$QGCqZ62}y9Vm6vYWU5Q-_EwESY@+QNp6>gZbGAle)xm!hBaR;
z%y=Sp>Kxm={pP&&MEh5|Yu8SX|8p%ip3b$rs^GmtEOOnBB}Y{(#J^lI*Hm1*^UB5B
zI~<qsSS9b0wo-p3Xc;MEwCnos0IO>YzwT79$=H=#FlVRwtu?!J7T&S+J#~-h^xSsR
zZ99hshxv8SeP&PMwwX#yHZxau-n}Dm1-FGg-<NIOj+_?%y*tgPJ+wA`7|d+0?B#C3
zzV3zPk@i#z>(_j{CYzg@ct5G!c}?Eh(wi%oX#Y|>-lQe2SSfw0+j;+`$*#Ev&stuo
zwc90l^2fH@k-fV%Oj)usCilV4pt%RFZq$FW+~8tk;p}8@$?|{s4uQ4<J8$^3?szc4
zb!U-u>8^QEu{)*8rxP81#QI~$`z6~B1*e;HOnhN}?Vg0$BGWpHL(&yy{82m17*@`(
z=%2URbklr$qQ~XMZca9w9hh#`e4xziThEd0|4%lU&yx@_n_Mbp6u2pJ+xzgZW_dq0
zZ6nrw_ES<U_WzZzylw1j;UlGQHtqS0ouvY77V%|Aw)Hb9?NV73XrWLkL3Fq@%-E`R
z+ttb->YkRNx1{p6Ek_-6Kfft5Hmv@m`MxgLc(KlU1C!0tgT5a?Z1@e7x>C7gR@#9L
zJL+dxSbkMw*s<up`L1Jbr!A+}d+pl$`O40d-s^W2oL)$Dyb<e<4T)OTeEQKABDchM
z#+7<oK6|ISOH4RwSA^@1UAB*Scm1zv-(|Pt3DxZ-)_sFroydYu3ZEpEI*j-4QBm5%
zEN!66rZLe~L3-i=T@|ZyYqb_tcgYp~U#u`!RoLpjr_2u7XDa494%nJqlz6Z``HGZ<
z3~SF$qeapdhYUCDJQ}co=ypUT*BP@fODio(R?pnFqV|l*xySwHD%@YTH_dmpaO@1)
zc|2Ife9qrv7QBonEYdm6w+HJ?v6Oh=Y;p9$o*g-f61!|vm+r{h_uXvYe0$4cCrPUJ
zLxpOXttzwb89KN<H{Vsh%Y?mBW9yQzS!N$tHtl%g<zle$?(dyFxvT?L&kXi-MQ!(e
z+ahb3%lX^<rBTVQkIUy;aheubN*$BmWwO1-;&e{ZuIazlSU&i2iR$Tq+Wy$R=GgYH
zvkMH)+r6~-6{T(7d@s@@QE-V_tI@yhi6UFJM~T1LG4pqW#WFs2t6&2ibI<f2JMyMn
zv^;f>!D327->y9$KkwXKXtwj{jn9^h_V)(Nzt0Z4E1KUok<*A1TcxUTS$WPzAvu9?
zF)dxMPcmn_XDD9W$|?W<=qKqH1}`n&F8E{d_A#5)QO(^u9Ym+?w9Q|;^SG^`g|~e6
z&Wl_ryOLO|h%TR9Xa6>z@Y&pQ(!~Ya-gT}p-?}Aar)1uGa|MNU7L`gRJLgHx+qvBL
z#(;$jwbRR}dDufha;cTY*9dcq?sa*VYdY<AT6JC8o*!tq^GTVB#eQGsorhw>ERQXj
zz7@M}OwpHXR$85?n!N2MSG$!uf5gt0N_)0Tzd5m!^?KB<24nG^<r9wVGPv$ZbUMYf
z0lP><<Jt9`EV0@R;TD=KA{v_4KONRsD*jNmW=D|PqgChh^Q>w#Rxm!$tl<%~{+yO+
zp4zm>Qs;pD&b8q?cI~`+*HV{b_O6>BJ9b7-liR&iUXJSHIoMr2d__;6nj`nHYK^=~
z$P@)r>zmr~k+O;h>u;?7y+l!7@~oBOxlg;73(NW|ue&a8;wy4t?ZGSddi<_eG^U<4
z(7C*Or9s$2g)Oyfe08!y&KT}^*Gl#MM`2~U=ApkzEg!ZwZdXh_zg@X}()N8{wwms}
z<hISWUuD<Jzp~~>FFzu>{JS}4x0&n5Rhu(>I?Y*i4_N$<c(*O!SL4pr=@u5U8?(3T
zu^+KG|7yCqB&(X$r1kH2?9&e2bxoAV>fgPo7D7z<JLThjch1X|-|=Z<#!f!o!$jM=
zxRl58Uihya!V|w(%seA(6)|g?#pWeBySjXzS@`5x@60H*wPc)iis<&_@J*4}edZeb
zM9s~7uX^sDnTn?zU#XiU|53}CdPv1_b+`JXy)zXh!UR>%uV_-gb|6)2uTj3%va)H~
z&*t3I+VnwFJ5qR^)^^MHT5|>VYAxLQK|8C~N&9ZFgwo`QdbKB7OloEu`qVcq*{trL
zd{q62#z~b8k&{&(|94cMy?ei!QSvtJ=`QSAjz5KUXR?Ot*vcHz_U6;j5ozMlRpemP
z*=Mpq=b-Zh9Sif_hPsDZjsNU=sGom1RNsqR$hhjvPW_@q>kY3P95rgXaou1-b}P~A
zUZYrIwLjHP)S7W|m)2}2Wvw}8shZj$`C9!e?`ZOAr)$l7BC9p&?=#J#77^+zXH8Js
zD8E(BNW)N-<qMbUne8`Kuh^|r-BX&O7IgZm>iHjzYU}RKHtV|`Wm>Rgrg`?k?dAdR
zlFc;C7MQb^shO?ym}7SM&K2`}Jq70TmR?d{%+;u|_t|uHH)aP-pVPjY9}BZIf92g$
zuh$PyJA9E<^ZfNzjkEu^Yo^&u(73uoKufT;Mx)`;A@%M%-!zUt?$SK`<D>?^cE6@j
zq=ly3o`s46ol#(Hn))72gTz7n)%Q-SfaI$Mrm5>oEYX--b5(t-@Ntb<mv?I@?-tY4
zRJf_WINd;9-guM7<(DVarDtTT-Dy~?5u1~(W-#@TYRbNuN+4P(TUWJc-%OS7*%0y2
zLrc}FI~X<ca)i`VcxP!;b*<Dm$a_rVoXr8X!n-nR|J$uJR_?#7zVhr1HD%8U>iv6X
zDyhUiQNN$VqP}3?Ocfc=3F;>MW-3{I(^2*9tWlp_z^}tnBBHg7aklo~4mF*OX|dWJ
zWs|hm*ZkBz?(eVtS#5^SNtV^x56+#}xu?HNYhUC69hIkgx`|6lwf!B9b$vKbYhSyk
zuVcDfOJ~#JKibDi=O{)@d#?Ch<+KWj2Jss+f)zm+nct?kUo174OQG_>Yf%txnPe@0
zshC&XewBjE1>?)ID;=sO+#H!?SvprK7xQz<ZDv+blx1~L;*4NbNEMo-pkQ_bOov`N
zr~=}@x0W{Es}*MKFsH<nhjW>UI+vS~V7;Mf=FDEB*Oxm?_V9Wdo!`%C?BI1ydCAE;
zYKxf+R9tFRRRW*CS3bu5L764vg7S;cB`SQXf-2Eb50pRsaMs_rT1D^mltzQ+s|pM{
zRgdXiS>$anb^S8EzdKnCqGE#e7ihNYFZeBMdS`E!@fGejCcZC1Oc#jznk3k&n5cCA
zHM;z5lIfZg`KC3WE}1U9-lDSPd6)9d_14P$>!*U*$YLOI%Yx}jCxT<l;-0pfJ!@{a
z5V@miao}8^8GC)0#l@RnOtTtRn}0tmW$u<U!Tf&sZL_NtQ%(9UCz~8+J#2VYV6y4f
zWm`@B|1g?QF)lPwe|5(6UD82QwR2qR%=IhO`)|srg|eMczi>rW{na-e<=b^D)OTlR
ztF&cjt7zX3QYW_DFmA6fZB3Fh^VxISOnkqFd71M=v(8)*^J5R{O_yKxHoyMV(p;l5
z%6!Ir1@+CHHR}J1<|;`~J*3|CO-DWPn~rMYHy!n)eKS?&$NQ-@|MgNYPZHFZ<(j5f
zE*P$7%T=YD!``o7!z`?O@)(Q$jSN@)zJi~6y$dGl@3s7(!oJp8T`Mkx7sTH9=@t)&
z&U3W_(=c{pn2V{<QB@=Tr<Y9<1364~muxXvu;qwRL!+wcJzpWCx%-vP0zavmu58_4
zy62UpnU&RM^KB2}%$*;uG2_zYHgB4}+vM&7Uh~jR6=nwQ+U5>Pt<nOCcZ8iXgXBQ?
z)XfQ^E9#!fb;%0Jq@-UHJ-X+J;+?OmisB`E)!vkT)|mL}m-4+I-Rf5qyH%auzEZCc
zy{NX}ZJxTBQ=D4F98>k5raej%d1i_^PghW#U>c~#$j_<3G|^mznZZu!^DGg?(+nxf
zAUdUep0Z!tQQ^K%ixlU+{4QTQ{hVC=rGK&lU5Az2HP*<M%kENf(1}sByz@?r|8SH}
z@OMY8wTIHRL|=Ajr%OK2I<U=1`$lS$_AK5Mt%Wiw+CJJ-HB;~KYKFX+q#5yHie|Rc
z6wN7W%v!momo%cEdTTTU9@LywsIIwe!(yG+uhwhd^Piz}D{8h*@3M2+%@aj+@@^m2
zo^rlZXVptK9pxfP9oesY)aCpcH41hcs%hSspuX-ct47atMGecJE$W`k+thk5a%i~m
z&rqMQoo#kobCr=~6~Eb+$bH6I%c@Kr^rFo?Uz{;(TQ|w%Qs*7BtoPxjCv-JTnf7oR
zU%mO$#QGw$iSKns6RwzvCK^^>O`Ko2nXGDSGg&_4w8@8q!s=pQxm9*5O;?Y3^hUMj
z_FWa9FX}3r{i;-=x5=nvT<uZWsk}g0DJaE^y=uDI$6KWq9qmud#sB;_wOP?=Arq}`
zHcel^e1^mnGrNU7X4BF)=(Bk68c5~G8*y<58g*MW87#e&WmMe3ZD4DoVp#G+*D(Cr
zD}x!P({<T8uV@vdSm?GYFzJS6`RK%)e4?XKyj)i&y-O!^rH-!s)d-!ONm6F!y3<S(
z(@M;){q8XfWw>r?J@Ja!(=~I<j`18ZmFF)q588g!?5xCXwb+L$s`;J&RW!Y7R6ouO
zSG`tpO!)&(i7NNzYSks>-&HqNWGGs3_ozMf2~}-5C#3o)e1__oJ<Mv$PA^n7TwAAV
z%P*)}Vyvb-GiIX7+ZK5>=?RO}gY|c-pS`D`VdO8T@zc~!Bj_EAS`{z1dQe`3dJDs0
zwcQc&=5rW!8Qt|=VeY+|-~8EPKC@JVA7*ShUglG)-k7eRB5W~j&qnjaY+3!jkHvb=
zQuz#b$n+Xi+o|dCU$isioPSmC#_!YmF-rpU-+x}GUo)Z0Y~>Ygv-5GOW-HoQP5fh&
z%v;;o&8+U~n5#{bH?z_<H@g-0$E<tW7qun1!Wu{2b=97oWY(-bu~kER#Z(P%-Kpv`
zjXKq;x->MrJ*TL<Ts>{Ts-Ix6y#2M&sTnVf7Tw%t5aIR0C}`INgUnC!3?G*m80EkH
zZ}72uhBos;d!7HYpKHnWU(s6S=Ag}3t)hLq+gf{}z!B|vTKBcC9M;#CH5S#ro|vh%
z=t#DfJ<~tUgoZB~oqyD|9u<UZ&R@)^W&Yk@ld)s9<~qSIGWz#_Nge*KrT8Z$TjH^X
zi}=)qCbE;;Sj6q-<S5SPyC`XLHclp|aH{t1atV!=*YmXhT%WAd`K43y`~E~7Ma8*V
z)Awa-ZCRtHowu+_t1~xIF;ir%3W(OawoDO(L2M9q|MX1}M1%0lbyjN2FSDyTv~O3n
z>Zno@h?h|P#5!3OB<7XOttxx+tJ25XyQ*c0Ka6&kCz-6v*Eb1&yxTx(9lP<0xIUAa
zrtgjUHl8qA?Gj<k|INW@Ui%%*t~UjmQzHsACWLZo3f$hQA$|U}#)D%FS|6jTG;Pdc
zH5YL|(y(>>q{eXOm^xb<kLnf~9gT?p2h^w9t7(+%o20%`T3S6!Qd@1df46%4&Lne}
zaAh-%rT*qp4985r?A9@7i3>63Ed5}<DBIT5$-T?`T}+Z0H{X1v*=aT^vW8r$@7~Q;
znpgcpMe~`t>i^@`sutXzl;4{jQ#l-AqLj_Vq1~3&q!oJRhBohaXYKbpJG6Z^`)f~d
zv(i2uqOE;GN=C<RO}}<>7Ket*J5dcqQ-1aIqvzGl#FRBQ8h_FVczRM@sP>uKouhj+
zR#mK0?<!oS^5tQ$s<!`qmB7vNDzW9cO8PlTs+%<3RaHMVss3ZTr4*_^Q(nkkT}EsV
zx7v?CPVy7p#)_t#50X70sx33qLt8;LR8+0Rc$svG5T|VQx?GJ*?*G))?yu3%brsgw
z$Gb`0&HayRx7ITCW)Wufw)^brN26QSfB%y<*_*n;q(M^9oLTvbnUhVn$;y-^W};k@
zCeQWvoASzYo95|Tn=aeAPmA9uQR~)ZCaq?!MosH)8#I=ypV55q!&`H@+&j(vtNS!}
z&R(o(VDnM+U8s|abG(4+cDoqW%ZHs+WgmK}Fjqt?Z%tjUx;I-`Rr95$vhP)O^%O5g
zjmHz-syeWjsPnojtM$c{DLbAxsh+S=Tg_@um--5w3u^1Oo>MDLxT1a}%uC%}?z{R^
zNnVY^@0&E{TOCy06LCsyyOzKD)M@9{e$4PzePGq2uB`Q6rC%XZE%)98HI4-hY5~h8
zs%5TeP}$V<NzK7%oywJFNksvP&nj^XzKF!$6BGFnazy2&C8OZ?TWjPM&hSZeYp~0-
z@_ESUJlrf_963#mwZ~2MP5MrC&tnJFe?;$5-!NTC^=T%jipJAG^@6%hY6h|Fnk{RY
zwcPI>R$Z$*QDem!ZOsI0cMYzHXsrahW$N`Q;u@=icr+ZgWoo{=@lvgNnzrWDHyP?K
z>9f?&98*>^Kf|f1x=lwtO>3EYdqb-lsGlV6IYG^B-%Q0MI}`OENq^K#_RUn(-#1e+
zzq3Y7#&d$2VeAt%zCR08M9#LU-b%WvWa#o<>B6M<njqTB!BBbg_t%=c7_KVS*?TE3
zQ@f_sahFre|J;8~&Rq_gitp<+*6`VD<whUYIC5S{i}|OLrtSwZ&47zR8dZfmHP-M6
zt4rT-*VuSGPlMs?JB=;kyc&yZB-Oco9n(nR=hv`&xL7~6I$ZZ6*Hir_p~d<(dzthj
z(>CdEG~A#sFn^-nx#IWw>rCDCIYeKomK|EH$bV5t_4G3ZRom{Hs)tUjR(X1lM=5cN
zi0b@?7b-GgLaG7fKKfgv`1IC<Of}F+@HY_rGGA}np=bl?3$prJyZ-1EO*o@})k;MF
z_P$GI-7glJ^i7R6Uwg33-1rBR*+p(=v!2~qW>4<?HVtuGX}+kv!JIKkLDo2ahVmJw
zRnk`j<c0b5<W!bDSSwku`mo|3=WDWm_vgw#wB9JcJ?ydUM3YI1vA*0g@pTh~ckG{_
zIQtN@=+%iH;)mIcWR>Sz$$I_0D$^gnS!ttdmQdn@Kq>1PrX03`ZzOI@EfLG!dRakF
zrHieS)kdl53A5mgJ@tCGpZM!Xmqh7*@?NCtp2(zc9QR)@Tex3e+>J}`!nT=u%u2R;
z9WOLw_lZPmlwRsp%B!ERsB6cls`OJt*&#wvZK=G8CZCF;lzsST&5EvdW?Vbon3~wf
zn@?F*ZtB6(ZI=I_&U_JrwYj;@O;g>|HfE9HM@`jNcdIKoy;48Df2Jacy)gEPT4N52
zTGf_KpuUspksKDa^GScy%3pP=w6UnG6iKX6taBAq+aUQ{#hde<QphJ4)yo~7s%&Y3
zN{g4QR0*G)sBU_WOJiQadbLUYMe6mp<kSmiu2l=^S*)J&-%IVNTal{&w6$ukAENZU
zJp^>;cqHn-)Xver<-nr%YpR!itW2Grcj`R7)Qd6t=i0yOxj4Vp7yiD^XxZjJMsGK+
zFk+S7WRzRlZ=`d0mVW7>C}Vc>TL!h#(~P9FytGbqPE;3s7pY~o`Lw1g<5Z0e|Ex7S
zR_szMJ<+Ewx9y&0OlGQPR!^{&{h~6h2b|w^SZgL~eXri2nWp(fhoNYuRxG!imcPm@
ztvKChnmXKGriX(Sj3jdxn{4EYFg7<3HxY5qGG23B*p#bRz<5Dfl4*U;3X_)tt5mAr
zUsk)c^S8?5Q>iL}9nX~5XE`Y?RO?i!*<+yu6A#$GOM7LJzs{kroI08{hqT2`^6T_%
z->Cihh>*@BZaJOD>rJ$ackI?q**8;Jcj_TE5RDJ}KQhv|8+=^-`$<7{zSxWEavPti
zZ&EYYNY4#a^Lr|*<|^*1vAkJSUAODK+5XptCh|or7Jsjln_oBmW72WxfO(bice9oR
zRkM$hedfm-In6J*i<pO{O)+s?)NUTTe3O~sxfiCs3VLQuUQ5g##L1d%pOa&rH=o6v
z*#6POV=ZzG@v>_7S-&bNIGf9C`I032T;jYck93EE@iI}hh!tXLT^`3}v5y8}ioSZr
zX(V2;(0Ge4tI^@cVncfk7ZU}SXNJqRtu$7W+-<zfX@k-4y4A*~vsRhBx$k5mx@4X4
zz4?3wue@qZ-Ocliw})RbU6b<7Xzy_~(@)2?8#BvPn6gA$7~2_?nXsIAXLx)fv&qyo
z*G&@SubZBq=4SM>ILLHrXPdG8l}&0tj&D(~Ik!_SamErgi)B9Q>6`l1bnSkqS|0na
zdi3WL)#8ULsvUAW&9V%>nkHJ=n{Vg)Y1UMyWw!Y64fDGBT;{i9)|s9C5oB(h^Vw`p
z#9FnhnQK+`j&!Q(?zySj__s$@@bgY}>C_t4^8Z(qJDC=#vsE8aCU)Gy#dM>YK;99v
zxNm&sOCx&CLh4?a?YE6K`{0~vx-ux++-$YJ*)QQ4=9hopRN7p(T?j-k535p<Fw&R(
zlFz2Ja@#KXtu{e&Zu}1vK=i6<qNZ+>-kZD)Tw^*@^snj3Ws+t$z17Ta<(@Y+TvuzV
zylREn_U9I64*yG){hzq0mE?w~eCOMy+&iIN`BP(yvib6S<x*#6l}qOqf!R^2vy5Wj
zSsK}{?Kfn$xM#5E=WOEwz5v5-1t~@yPc|AQvc56OXkBgG^}tCxe)k4V;l*KE^J-sc
zL_Xx!mfjYqDWMpq-S@dubLNUBt^ONTT3Sx$G>_XJ&^*^=qRGHsuW``ENyB(jqo#;L
ztY)(zlUDz!3mQCj-!+tsQ<Wkg9aP&<E2VbJcE0)*)642E?@p@E=|8P9{lHWe7x7u@
zj6WIF|Lk@$^g8id$4(&G&|^uv;iJ_H^^3VK80-pRGj!@?)PJ@}*XZG<07J*<Y~#0L
zyA4mijxhPoBWN=1@HC@vhYTadcY#J5ZP|=mC$O57%&9OgU81bDD|@Y~Tc4GdjhTX$
z<gfLbzj_0-Se*^Du4Rg8uI8-KTJ-ge#$HJ!-AmcWv|JUW^|ohi)BVjpP3PBrab32Z
zk9DtIc%xH!bE@u%DZaY$%xBem|0!!+aJsDeTIslY;Pj>H{Crc@7dJGjOI=x~dR<;z
zU9ccoU7BIKqWjHVDv}&NiW?j6D}wkSjLZkIiA`6FzWSJE-O@K>od4c*7XK1c!P6C{
z+%`sLuNg0ye!TI^MB{CjX`j;!lQ^wOI_}&ZTAOyf&^~zQm3GvxL><W$lXUDdj%t;a
zGw2vB7twJ!aY}pBg72oxN9G#MQJ7;M?!skW<UGmrey_UO$0$X!g$|LX`SDB4D+`yH
zeRnW0kuvZ#`ueQPc>lTehOW#>#%i`{#tSc3n+U&^GD<FdV7w(m&DgRlTq{@jw@#kV
zLaj}v&6?^lYjkd>UDurQqe8pgZ-@3xT{|ta!XEA9u;*I&U;H(lof@=6!a23F7jkQM
zTW!{KH2<jidlk3V<GW8a*G;&hY0tb!Wzv?1YRf*qQ>}`fq`p06t$JhjDfOxo+g0{E
z+*jGS%t~F_?}XYB4oCHVuZ!yE&RD50W4fbJ{OYB~<1i7;yLK1Vb%Vmxo><M%2)-qv
zA^AelbWXIg*?!SQW`2d*rcT{=O+6QCnVAT=nkJk*V|KC2!dTdSmubqrnTjy1JN1wX
zjEzq3&dyd}@z+bE?wgK^+|)yAiQjb8QvZ6X744g;4idAwDW~4JZ>DP6ne$51f1Okf
zc#)z6VsEmsQv$JlGFK{XKKM&1J-J88D|4k1d-YP4fA_yAUCUppRC9hKn7yJiUI``!
z63=86SGsz3p4tl)YmG3wsp_XAJT#ZR3DA6fCr<NM^lbGIzIL_yj88QF1hr~_+MJ;F
zC8+%gqCxG<BWG`@F3Dj5jpM4U$YD_fiEsKFX&SiZx9O(J<HkwntxYa&lQw%RR%`S(
zR@<bUpVjP2-eglrPiwQ>oNIEA6jmua*RNIc|EsNj_g0Q#O;v~bE7j|&zxT~o`gWs9
zO(%AuiqPK!npY;-sek8g)zVqIR(tOu4o#<2cCCc@r!<_F#cFCPFVvLRxTG0T5o7jr
zZL_Jvo_i)U7V(&N$jmk6Hd$%x5%bn$X~9p^$v!8{l9k_>c?TG4o{wbIRP8*Zku@<(
z)5)k@lU4MQW@YAbjk_i5G>+%))l}*d(v1J6qYT0qv$Iu@X%HKPVd7Uy_nE(Eb2K%z
zFg0Uh$uxa<`jaW+sxGtBYnsf1Rezc$Z0RxA+Tv~g<I_aLm+x6j{Hy{^mKDr2Io$Bh
zSi(%mWXn57qjr}U#*-$Q7-tEp8DH<*sJ>&SzWT4ITWSf@PpBuE2x=HJ3ut68nyES1
zNUQ7p71qeg;!vL|AESQg+B?;aGb7X<&dgG~P&ikuq%=?MPsmiYdxl<WVx|mgUwvk&
zg4}xLd6XK6#)l`f`k4yYvYM$0J~!c&>oz*H)XMZ<fTgKn`UBJLd(WGcJkB%OU>$1A
z_Hve?&+Ifks|j{Sa>`nU@2@BsUXwRB5D&j-=<hXIpZ(uPBd=TA3_EQ0Y3co0qp>Zv
zUuUiNYHiD(En3r>{j~jlXXq@wJXbqZakh3%o~ZW5E2`@4Y$wz|_pen2xwGxBm+G?T
zQEFNDgVY!O^#Y51&(2oatol%4UO=*7gHf}LO1B;FehVhq<JuENJBlXDw{tJzy|m<<
zyp-M-F)>4Sb&jq$4F$(e^)PcSO&L)iO^&8S&6a~x)n_?Isjb`eQKOu(LnHF!Nz=>g
z&KaA}nq_)Ey~#N2jiaelZHKAM{Trr>xA~ereZ9-%(T+ybXd5}*mj)*^tIjv-Tv@kB
z+cUC4XWcVH9f!krbS!zdX)}~s=qyRRqFvyA$y7O_+ce%#!1NJYfstJ3HM1#;CYzXc
zw3;2%+-JJrx0R{#(_XVZ>8V;1uBdBuEUVM1z8R`{c2A+E<>XGSibrZ1^U@l$%Znqm
z&TzcaeEN5q%ER|+g8zQ8D&A}jlwB^eOO7WtT>8#?2jx=}9txPA>Qmm)%Pafk36py8
zqZSQG&k1TTlK!YY?y6B+uy3ZKqRl@QhLug~o4$0bgT(!&K9=s{7f`!d>?m(JD^_&k
zLS31-`>HbW_D^IDLq*k6mQ9!5D8(UrBC$%f$VXWH-|;DGzmICEXZr}NUu#TPzwy09
zRp$6UwWF-=>VNOgQ!D%MS*^%<xq96NJ++<d)@jIWSfo)stw|%xdAa)k%h77*E@)^7
zncr0R*mm4x*Ft}@_|?127R8)1D>!(<Y`4jFv(PUpCh5gJ=B(EBW?|Bf=I1A-C>cFJ
zpbDlnS1Q5iT}i)`Kp4aaVH2&H>ep7sX%wahsAqkdufE{iGW9=xiW+ZktW#fegkSx2
z@&@%wZ&s_vvPh__O`E6w*7AdLnbKwTP8JDu9jgz@{L|*CPq6%;Tx0n`SyL=o{m#$7
z8YjD*G!<4AYpmluthq*KpXR2GoLWDtcr|l<k~9*2p4OD+)7Ly}&7@)_FQBlsWUnG4
zZ>2Jbo|d>$MfYF5Vpx~I(*L#7l|k&3eKXZQ^siNy4ii)Z;fvYXU>Y<RaOCU_P`e*g
zwyPFD{HrE%_<;I0b`!N4Wk!t$oZr>U-m7R_%IH>$xX-Ru@ApIfLfQ(o%?{y8vA0Fk
z|HY;#nb|Zdy}bKTr8hNKS!u;hWu-eRs)1hnl@{dXsoa}yrFySsic-@hFR4lQ<wRd~
zDl0CC*(24wz)SVeyX*3<E4$^lXL&01PZv>RdbnS$kda?)_romZHPtJWsy5}T9r?$n
zR#doKO{7p=byA_a>O4zN<uGABMTXL5CDEr$Dj@m{XS>S9`G!i)6+e`E`1UD-*vfYE
z^qJH2bYFj1q+ih&tG{LspFRVJr~cj-^Yjl|I_jnFx~{MG-(CNd(pHr<M`To*cKRtr
z7FnxCo&Kg0V7FD})pA?q+dA#4<}VJYSUXQsk_u`w>8pQXI<Z98?C6PeX5~|FnKryF
zHLKZGWs>xKx0!^ai&@|HF0+02)~j+Ge^T4ESz7J2jj-BQmba=WYJF7~e{WS2ndGVt
z61V1RR|`#1RX?V>Uj2BYkVfpPE1LclbsEC6g4KP!($#ref;FZ;O4aa~$Dn!Y-)0TZ
zh3pzL4a79qSSPAmZ+@osr0J8Iv|NdLD?_ZBa?xcqHMW`Rv)O;>-|^V0+jVQT0hdXO
zfyTiSeYU`610H`J1J5#9{d1A04SsGAGziz^Q`~cLk;=N9AjRa}v%wg|2H~KlXNn*i
zB;I0esXp`XVf8r%tJPKCo2lQ5n5q`x&Y`jN+e7tV=?of<9m~}n9N5)71GG%Ieoryo
zXpmx(C3Df}P$#SDwAkrJ^91EhEdM?-E!=qC*lV+_=`^1XWBJehW}@zgOhvDsG?~AB
zp4p8fmZqh*TTSz}oi#f+JJw{wEF-foQJV}3%>4CxbQuiQtSt@h-)J&8kS=a;t?-aR
z@UK?=kZvhMt|f^Ei<@|~HXo|g46w}7EI5{<+5K8Uv*Bs2meEu>&1jd)nsWS)G_@{H
z(A>i$ubz99S>uCCyZWcu8k$FQ9X02)g=se25LGW+G+Aw1^%srx_i{ACrD`=!TAtB(
zW!0?lm-oHqj1DF(gA2M^S3W6f>?|lyf2wm{^N59>=ERrnT2%ppTE(;4G|SY&G(q}Y
z4LUS+g+w&|uHK~ioJB|DfRDeXZ#9>Z%?=kM$>M3oiq)+~nF)uD_wY<Gw!D*Wyd&zX
z(c`P_#;j&PjnpKf)b{4{s?UD*Lhao$M~#{P`!r<TS8MQl#3)a7X;I^>wbY3Fx>emR
zgHJ6?jYs|b`(<h|t98`A?V7LlKh8!yuirs+&Fd1?=bvV&%FAC=`+C}5nSaGyHA(SK
zWee4XDhdy>Rm%+)sJyIVP~}=NQ?>6?jI!v)7b-f++#2S0R%oc)>{m(3J)t3{&Z@C?
zhM~I0!W9|}hdNbdub8NvkO<be!^dH!w``ixu|0XF$2=DrKTeP^^_y&K`h1#~*;Ix3
z#)>yv&4detO?M0Kl(e^fF89x3mPB5=B45H|d%2+2d18kgSY@WzxXSpnG>D%(!6T7;
zi{D7W!_e5_-&w=Id|q0Q_Z%^7xVY7jYnH5W^5IaU{QH51(R0fUrmqk-_KRvY-5q_?
zWM8GC>7{cAO_i8mnRHA!Va&QY#O(k2$0h-bUCfd^Y}KT5STqh>EmM2;>y!G+tKZf0
z6__+?G`_2zzn!CWl<AfFuPc0NGmli5ZhLBHb|t&YRC7jy$-#7I^Qli+O%I>QGE+%A
zZT57Nve~TDYs{M7HfSfD{iRj@?xR+QQKMGFwVj&%n$xrtQ=+skUs|YDZ@WyZ(>6pa
z>X)b5q^&Y)#`AMkIp#>HE^fW7_CqjFo#*UiRquWkHNT#0)%j|9>WRTC^nR_orJwnZ
z-{9REMuP{_*6D{EuhRd*=c6}wNtwZV8*%+s-#7a2i@41val4upDlae%Vcc!<>ovdG
z<ZWf93JRfS`mu&)i<2&z&M^FIMr=Py_fWKI;{FAq*R=HIHqQ=~v|M^gL3hf0af72w
zs;5u<5>A?XP3cmhoUGGx1(m<M4u~llotIZxxkiqi*+S8xrc6rl=|`o>b7kdRFUBZ=
z<b>ads&hLlsdsQsQ+s&ET7A{Jc6H%})76ugD5@w-nyt3SuR;CN-Q#L0_w7_tf)<FV
zhsnx@a{5XHu3Ii2_H(wF`|DTAVy{07r7JB~yez{kquZ#imw94^9{-L2z0LnE^m<%b
z^iRbc)veg9qNg0TSU=FSRnNXolj!qb<Uaq<4dH*QEs+?f>wl(0cYFLq-M<=Zbk$z8
z>0T@q)#YBWM0Z7cknZUPkG0qto3!pt5Yh6LpQm}?)I*KuhJ0E}xD7Rf7iefz2QJfO
zET5^VerJP@=EjLy-^4|9%wHYRHu!%++x#wz&N0LLI{A()I-ErfIyNVFX}3&})sdMq
zSDWv2i1xKVkF{J~RkT|C{Ivg0>ef24vqt;4OPv;{(G9KIwE<?Yu3R-+$CzXO`KgQ9
zBB32-hO2zctqXh19R8b{_t+mc%|Bvcw(fnL`r>a(HOx0ISL4>GR9|?iPtDM#PksB-
z?dl#{{ptel5*kmF=c}ijTBxYvCZHC*AWhLU^}S-lmMFy)mG2cz7c5eIb4FC{=)IFF
z9)i(|J+}XizC=lz*|N?wd6s+F@bzXv)2Y*VjYIZHn>H-yG2P`5YEqRaYbp^YsQ$8l
ztp<p`n4PU^^-V`5;+u|I(!QCh7qhd~uRM=Zf7icOZH4qnbrz+3d7COeRafuZY5``}
zYO3)|)srn3tDQf<p**`uLDk3hl*&aub90}AF=m~|UYp&jC^2)s!)`w9t+Dy7$ARWs
z=SiD=J;7lXbvVrIpY?RP#@4s8Z`k&!@^9Rw@+PQW@<W5G8n<|ayt&*;xid<o@?mrJ
z<Zpl9ZhWN3+2rY$tp=9O?K<V}3`~CS;xh7|<6?5UH`hqG-rjIsUAZBw|AS7W_c2oT
z%~a8ydPp5a<HIZdda0>*p9QH?1+hWl016jSpEqUSOwigORnWL;`%YPv&|)XGJ2UpG
zu1uMse&zlG^;H|@sq1p`sVw{_th#z*mHLPJYic6={mSPR=Lj!Zx=-=%*TeG59vR6W
z5EPX=$DyYPqCw)!DJ}Bo?#1RGkeJzmUNzq{`_$EAKB`KrKdPGM9;v1y>#y23*HBG5
zXQSHBw+<?^A9ky5*6mhn_S08)UA<gw`s5akbF*h@JY2R%BO_(0O4I!|RgpU$8v5OL
zz-{t(AIvovJtu(2-&dZ!p=uf?s9F*GL=D6S^|wJ|^TxMdnY1u*n{GRsY^w2biOGeo
z=VtH4(o7pv1<a;R;x_A^eb9`T>8Kg8<F-9I7j)|qu4(h{?9n~2+DOlK-zlB*+VOf}
zIm)_Q+LCmQ{(9)nPc_r^31E=jbyG&6-*l5y?Jax0Py0LMjUu?kLE<2>f(y4L>uz0@
z2BqOU4U5%5Gzdq0(*ccfD1&GaA2h}dqP2MRbR74c(6W<%r0s9utrN`BsdeM`DxFCS
z?6v17EY+Tpouj?*FPFANuAMet;WZ8E_DwnpC)R08hOE=x&aR{r%ClJK?c@ZVBg?#W
zCQdWgA=bYRS2t))lV(%>KEX--K&ps(j{i-~osIJ}nxCFjpYvmjdb{}wEvuJVTGxAd
zOhvEynO+vXZoJF0-|(DWlG((jEyf3JZ<_kBZZv(;6m2S1u48)U*K@55)hO+=J@#6L
z2i|D9R6A?U+qyt&L)(0<P%Ay{rgiHz=S^FnHPMD0JT8)3Gh0#P_Dn?(efae(#R>an
zD$WLt2`%PUQ{O*Rk$2Kh!vmAz^<*_O4V%oh3_Jrx4Ytm7GH7{MWSIX@#bEu#Zo_3u
zqYcAeSes>hon>OVIL*{jS;@%dbAefI_EpohXYQKyJ{LC){Itxhc=ANE2fG?{UuRls
zf4H$$XRX#5oupVr-4|6?bvKmd>%0od*9lx{sjIoTTZgOppw#2UIvK^9a=Eh_+vPws
zhmD^sh`p)mhSH4?ZrO=;SxU{^x4GVZQIW{`+0O$SwdXy*RPy=!-RxTJiv@-Euu3ZD
zK4-I4v=Cr^IZ^3D`AgN@89S83zcecKdNC=3#8&+hR0d&?xSYDHQsiEL9rLc;nvbV_
z)Moj-K&RJ4NN4?oSvvK*7<4>lZPPAi{-(Xk;W22QL=oS38EpKF*n0Z=y%Kr#qs*!T
z@@xthDtkmHeCSi^x7;d!s(rO`1Gko}*;E<j&lNWna%b#N`MfPg@o&Up#n;C_D#F+>
z8YB*4&pak8dP!-z<j0OoQTeT%oPFu%#qt$y3vCGMkch9B6bG?eF8>!jVfjIIujL1|
z54n^1LF{IQcD0R`A5@wZ+SNh)^Ohe}K>R=df|WPQbg9k^OIH?qI!F0_>mo4wlIdz?
zb4wl-kT{5aYaX|pMo5RU|NnA%5T0UxN8w!OcKPdko(h}#Hz|fEKa#t~psd)&dr+HC
z?TpUkiT2t)f8?~Ut^26$e>6usZT1dr&hr*J*C(lIZ@#lpd#Xdbx^8ib3b$H_y5S{b
z4WmgP)mmG>sr%i~QBU;wq@LS!R$c3#nTAnPsP@YDVOm?im}_rcuwLu7OR!d_`!+4X
zw0tdxPIqnN=0vTRd97N1zjmtZUb|G*)M%n=v|xkMvk&tnYocZ;8SB1ReB%0D(dy+X
zMfux43VAWBbU9-Tbl=Ob)(fn=rgv52rQU`sF?vk_Z}eF0H|f1I4b_*B*`l{?;y$%I
zX5t#pGL6+v=sT--ZriKQz;aOCf}c;V<nk?*!gn3&n}7aR1C1T6`0J&b_)SL{M5FWH
z^{)lhE2<WAwUq9$o#idxKU*=nZMw36@=9sNXS<Y6h%JzxW+o<^-G5SLpR$nrT;5C*
zA&GiZ$*A|H+pl~#ITm%$^t-pB>6b0MCUPlUX6DyrO!j*ynbm*j(GS(&(|MV7L{BM3
zOwaXpf_`oGdc6&5&U%X@HtC8U_t!fUEvmoa@KklrAB$8=^iwseXG~IGpYl>=!*yp(
zlM-dMAGJ@_CZBz-#`}DyDyZB9VNkh=jA3Fhy3X^7YOzaz#)l_4YSZ5<sps<xs?M6+
zpe}WoNj*Mis@gj#R<$<SHq|N5wi(}I?=iNzp=FwIbhEM6$qEzZB~MJ!tAb5_L=>3J
zTN`O2RNi5dJwaPtaPB`fn+Z{BKi7Ow{k4u)ebu{0HKCMFWf11^_fWYZ9ickOBgLq0
zO{w0?x_Kt+4)>Y7mcMMkmwMHt|M+SBr!I4h=bg|uiTVEAXu+0kicj-gg<kx9C%NMd
z4<FOA31EJ2D!=sG`ahy-Tk2%qm^_jd)~V45>06|sd*`3}vsQD>_B-L4Z=bbmwy6D6
zubL8}R$lL~nX_b$#`kxvdM!IH=&arKK+n{9pPtB3cRiDXXY?4nIrVj>pVjqU9jm`s
ze39Pr@T=M){FAjFXZPxyEbh_C5L3|Ef1*z()JIZBYr||E>1}B`Ce|CZ)s}8jn?9{c
z&FtGe)oSUVssaA9RoA8+RIf~#teP9HtYYH0Sbfp6R+VS=-_<mAmT2fsXVO@%FjsTl
z?d_U%-A6S=Hyu(xxqP15l|{~)Y_soc{GZHYCgZTd(6>&}^apc=sej&P6VLtLruMTY
znrO}4YVx?((Cp*3qbBdCo;T$Q5jFoh=d0O7**T`wjzZ@8(IRG*jy+~~es!2Fn9gAO
zhuzGq^_H0am6<)-`A_EQ?UobO)AUKwbLhy`bNHpLCzbnB*Y3z-J+~?B`Zk5Z>R&GG
zSI@harSfw9D)q^7mTFUr9o6eTma1`x=cv_+IB2lO)T&2tDe5!YYwB`Th8g5{ZPpJh
znxOB>5N@!l<GP+xTaw;Q*<X5RY?tcAb-h=Mvo+K>m}#Wy5`Io2=!UPx->cFZ^NoAe
z#qKsLyL{|akGj;RzPM(lw%)ImI*;9cY2D?$q%~)@jE<v@lXi8DkdCJIZ|$F3u4oAb
z&DUO$*`veTyi4n0V~zGt{UzGBtiNdAVEd`PPiU4-STmE3yt||h_xc{~8*Lxd92hLr
zcbcA1JF$7Ly2FY?>hn}is^7ivN9}0QPc`imP3mF$&#7H)Ijyy#_K^<fyN{YZ3>?}{
zKR@b(S>D%nWKht_*zcgDs6A1uWX2<%rI%J})Hb?kS|9(ZvH7v9R`y{Rt#*bsEwyLs
zH13%#&}h|C*IM8+L38J#t(sTaH8u5=R5f{Yo@jIzxM{qraMjdY!lG&Malb}In6u^`
z)8iWNZ+WOM?q$&E-?~6;yUPUihK|i@`~K`yukV|zF8WPd)u?~H`j@=tYKek3#m~IE
ztstlFCJw@}ulW_qPUcGFl$*&4@Gg?8UDGES`{=lA8Lx~=lh_+^-;L=C>cWolSFc$s
zZhfL4_bDS$$@58{Jczzlv09;UhK2h3n767wi{`66_TQs=-<?bSpGB>D@f3HBpYa#e
z-Uk0rU02nhoEke(Eqi9R`t^2iHQoKj>a))rQnR(0sLrOPr&4=QUM(O+Pi6WwKGg}E
z*Q+PK=}`an;hmayzoF`l&_`-I{xNEIQ+w6UDf6o@-I}Vd)N4WfeF=WaD$8ziDs5*~
zRV$h7sOpt!p{isVpz{Cbdew407PWP6GgV)(Oi*sy$)b0zP*r;t&vjip-&?vmE_ZY^
zHs<M6iTu}{#M`YMDY{HIElNzsxqg+J-|J86Gj8^)zj~mj8F|oNbFp)Q=7ORIb*oF$
z)U3U)Ypg%xsG-)WEcIcVr{t;Lw@M(kiENXy#~VA@eYYZ%es`S};L@p8{l_V;R{o<`
z{Xu}aMz`)Zb+za5n&<cRYgQDm(wrrGT0Nv*MqQ;KNK?+FQ)AlFUaeyBcpcMs{aVr!
zHfhPXMQMxMT-SPkNJ)G7p=9lHzfW40-+pQFE3=vOmGYVfIUP1LQD!#(wm{N!z2;-H
zOc_z*%(r1?q2h+7rrWJebGWTkX1|J4?K~N&R#RA^-WYO6y}Rh4`W2Zqs(JouipL9z
z)!qJ2RoltjtlgdCs=3LSPiMWeqt<eUO0Cm>g|%AVWNNRHDb#w=TBz;I@=dGi%Qn-C
zL32z$W`vp>Eblfeb=_>XnSsT8yP&t3WM{hBRQ-8o4}Y&RlMfWrem~1o=N*@%R^FsC
zt=_+0+IP-u*76tLqZL$;qb<@uL+hWxKdqaa8no?KHtT%5G)pV`{RXX-XZds@zA<U{
zi7wPuv1!oRZZDzDymp~>)aC|LX-`eFggyGE9SgUcq%`j^$*-_6^$l2U@~(4&8N>ci
zlbc7Tm@e~vt@UBwO3k#q6s_{e-I`8Ec{Np?|7ymxS857;tJbtVK3$XbtEc9(Ns~3M
zuVT{pw(F$E=LzrC9fTD%6>?=XTbFie_TRKow`gCZap~Z8^{tF;YEg6hRM(|7Y1GxZ
zsV83lqxRvSgGPb2l6q?WBeh_uT(uormZ@o~x2VZ?C29)KoU0VFZikx4pOc#MzgX2>
zPn)Y3FPNa#7wDjVW|M{bjGH>@-hyFj_saLGzR&bkJE9e+c2|6j%9%r3)!H^itF5(~
zr~YK+3AMGBCg$@uiI_P*t2e7`JZ(BhRKe_(eVJ+7;|8;L>))AG&X6-(ETUj8!EJ8t
zsvTh>u{+zmB>S9c-p)!hp(b|o#<laz3k2*<lWVt||Hx=I<9KbME)sNCy;|?P$~*bF
zs#_}L)bHM2uj;U*SUsh+LCvBsN%dQPmU@EKQX{5D0n_|#+e~KvFf&atR5Q5~HqC@@
zzlveB(Js?%d$>)Kxb~Rb4VF-z98s_SM2kt?Y(t;MrX`y-{F9Gr9ML$bx*>A1>f`^8
z8nburR~Inx(lxFU)h=r_)U|rDQMZiyf{q3A9-V1BB6L#@pV9I7QmmU$EutIt=%>;T
z1{V2mZEF>#$fPKN=n31-s#sKHDVU4fDW+dqp#)+_UKi9gx*@2iqdHe7ZL*L~RoQc0
z^{QB%KZ)mc6}E}#G1sipwYtAoSJ1G)<ig}>#xt@cjJH%98w)rzn=I0KXmsr72BXGh
zSti?^@0%=Vx@%(2yFu;xr`u}(<L;=cl<=s}k<U@}x6f7U`}RfU^OIgxxkd%G;O!b}
zx4t|#eX@J0+4eIcW?I=#O}1a$V1C~v)U-BN){G(ifthO1f74&x{pL)(A5HEuoHv@F
zeb2<|P`~NsJ4;L!+I5>S&eJj0mV9IqR5Qo)h>EExXx|cOzY++8#&cjaNF2n6?Pv1p
zXI5|IS*9`n?jyCZ)t^-#F!ZYYe0f<lt!=)_?|}7cKk6;iUm3Tko!e8Qe{b?h9d2=c
z{X34&ba%d2(x18Hwr+pzG<}XuRl2u=n)G|;KGl=j^g;8t;Cd~Na%HW_!U~!n-??ai
zj!V(@41T2bkkLr{$R{<e?MyD(8d;hed<>g44z1x-i}?OR<6*M1hQ`f1>ZKWzHEw=e
zt@`D0kUHDLG>yb>I%=SPHE4_kL>oZY3>-Oo1GG*+86@WMO-E&Q&Nn^BbA@{PFI)8Q
z&T-cF{b8n`U#Fz6J$I%azw2H7nEWQa>NQ^aZ>Py<ZVOe?s9NKxVL4Y`W1FJ5hSK4i
z8bapR)n9zQr=GH(LGy(~g8Gvf4kP}&UIUHGH6~?09+(`O>0|U^yRGSTb55h}U!NJ@
z`n=1e>&#!{<vNSCET&m#26ODw<aII8oE@}PBRrx(Gp(mjW5Xp!&F$yXHK!&m*5KGa
zQA5z|s)m=BtA=90BFz`87Hg`oywaTbi(SKU+hesS98)yUe!Z^&+8@U(*`}s5rBQXK
zgRI)|4^vdVryf>MFk7u^a%z#v{>e+!qlKO;Pio*Yy7lO+$=&&(rYrM~nY@qUGO6%p
zHQoK1!?@J2!GvG-h@syC9pf8WVum{l`t&a}_Zwxu-E6pjimAb6wq7H_*J}+^_p%u5
z&=fVA#j@2P$X!-5cdMA@v#Ap_R0KC^d~@y8P-tn<yi@y7qiyQ~4ZFUTnmMy~Xk6T5
zt)A9jt`_K^uJ-uyJmsRvOiEEH|5by+<WxT@>#DqQZ&bEh|4R9??I(qvKrOYEk9Mm5
zuRSlja$Soe2rI04trWV+S3J|*STR8<R^gXctor?j(rQMh*QhW2HAn4hu(kU4n<1+H
zhi%n7J*w4y99XEjG*48OKjZ{xd=0#h4z!nU!oHbGAo@n*R?s+%(vr-DV1CsJZ@n$Q
zG<2?QQP#^)o1?dQs-f<7=^J|SoPE0M)@{-KeesFztF*bgNsGfwyCqu9p6Yv<y(!l=
zeNj2l+~@H|)5W^`OugeQ%?}lwG`*-j&D{PhlTM&bi`LpNHQJ1?#IzS)SfIVR;+WR_
zee<;*L}_S0HVV_W$&b{&;B;B%%*{PoRt{Y{LW#^eA)oGQo3GN<d7-jFXIJ}VZR?+o
zI!k*rbxx=~Gi$h{ZhFMX(7e;h(A;EenAtYN>1MNd7MmKZ+ih0+YLWSI@nz;e%)`_a
zKTJ@X|Aj^6y8C0b{r$S??e#0vUg^$J+46Ils_|<{_1Kea>VIe1Y3oL_XzTy)(n@#q
z(2{*PU9;OeODlqpNlWx^m{z!Vv6g+Cp4MVMK2yU_q9)VMR+;Xrk}?UtaLA-+@oJN{
z?`<ZZ%r~2U?iMrAj(%nGU~!I;;m0Rx{jmlr;n(ZcHCDe;=Xu1S!S!f@VpF)2+U^v2
z_4(I4Kx=A=+K(uF>$*zn@@eXo5z6YF0Ua7epO<Mwxa`sBTdS-V`&&fygG7+VyDAZl
zoe@<k50o#f|GwL<Dx6@ZA}jbu`A)?~)mqCR%JHJCDmJqxsOBtaS2fXjsin8$uGa1}
zAuVBTD=menADRmD=W6ZVGDlNJTT;u5+ea(OT~sUPT9wgUty2BkW0FQ%eu;+Hyyh6b
z{^DZL+~a0+@{_oMY2-?yQ%-*j_syPS(wo*}*7ZBxbfQeHsjaKNDc?tLlL>3L8YTU_
zVs`xU2a}x_yv;uFC2B7**3`C0-mdlCtz2u5(m~B*{5@K3$4s?kJleH(yqu<0k(Z{m
z;KXmex;2M%ChJJ(N9~T+_t1T#yQg@C{`ZwZdg7-2dP-?m^#0ne(aYd;G}#@~Y2f?X
z-Yjaeq}kz!1fw}$nayS|us3o0BW;o=!)7*jcZZ2-yP?*e@79_=Vcc3SYg{$&$XwNU
z-OsN1oz+_N<^m4QB$Jhz1v^e?ut_tiG0qRvNN%-JJNz(H-FZ@m`hh3E)j#x0sI=&}
zt10kysMZ*sQ_ZzZGD_epG?{+1$f*0sC!HTI?~Ow*uQa^&YQOQ^i*m*ry(Snw*6cH!
z+PlP9$w|)mVUm#Po4tQce9xqsDt(`1TJzG}l-Xy8NxKWP*^vkhQ%{|C)y2D?sJ~y9
zpk{h%g8HT72i4a(>{S=t-=S(8+@|vVZjyTU+e>OzwTxyzUUQnbW(Jry-MwkL$Di9|
z!i{corq_2(4<DOn8q6qRrZ~&pG}vsjy6M*m>aSb`G`q9UsGG&MYjnr|)_5YJuW2YF
zuTlDUj@r?4+*&N|C-r7rJfP2~^hRIe(j0x6%DMW2<?Ho#CGXYKQz$T)yro(1@Y5ao
z&C}x*g<l;~0nv(8OBI!?mMVhiGt=W0zr6bbW`p=U_divaQxH-6Ue~WS$LFX<!1@WQ
zTPCn-CLdg&zB6Kh`rjNj<(%K|R0VG-YrJ(A(WusH*2v%4tG=b7M4kJylE&Pgd+Ns%
z_%t?2*lNV<K30F{bHS`>gSy#vjs~;Zd6lMj*IqDHKB8cD{)vOx_ms<KSC7h@>J~+r
zMb70`-zKs|HL&ZCiZSzjRVI$xs(-DwsG4{FRb5l<qbh#cSUKOQLN)4eq}h@+y{2jH
z=H{-&u4Z3$8Jd|mE;Msc>^D{2b<V7Dfsxs9ZU*zUADPToTbP-jZ&+*b>%Fpt;0puu
zeR4A_-Z@_}yJahG;e1`f{OBD9i)P-11~*#+^kzTbWw2N()<D(vgnm@bI{n0VQ3jVU
zT+lzhd#6DdN0h<J^G+t_;uR(d_QA$CjMR(^Ugnxa94$1~^JX(q@;Pkug;mt_dJU(^
z8`r-^IXiC~w@k=2TdbvGo_uVhv4G!Nv(_1^1|@zrW~<zqOg)56O;Z-P8~oe8NH^)9
ztG=kzCcSv6efo1c*6YomV`3nf{YCGxoW4QZgGc&Z;u|%WvdmMxdo52Zuw|Z#n1q+c
zr+eiZB@?tY?|+lhT0C`*y8Hdf8nv&!X`j8fR{QO(N7`?U>~y|f?a=N|d!_BL-$~o4
zPD<zIy&c--k}I@ZoLDTZ^wyiXaCun7t!Fp4yX|WJvO&Z`;PFfI{cA$a&wk7@kDff)
z+}7ugSq9@Ivkl9im^wKznLSi`Xr`;X%=G-i{buWxXP8~w>}qx)uiGrQs7Xz`qE5}?
z+jAw8K6|x$PdBRi-t|?ESlqAL^ovhrYJ#h3u2Q8+y+MO&=GikUrA>R4HD7b6*PUxt
zSBqY(9&z@J%8Qd#$~#j<)TN7$s7dMVG+y_N+r-<uz{FwKCF8sqLZ*v^e;CiWxWTxu
z=fBC6L`7qR>o%sKb<f1Ce@;C#%lJCu9g|h3mYV+D$89!um9*(@X?IgL^KHf-DsGxE
zybUzHkYZ&z;jEzAyki2YkykdTzA@UZK4IA_b^o6t8aeL|t5h(YRqV=HsjjuhO0D4X
zbd#E00%l=LolHAjicLN@oH65=+ilWnJ<H^Cm4n%_Yh5O8mJiL|7Aoq^%Z|~L65OY^
z(~v_?)nI|%>X!L>2Jcqr39L!e-_`s{@92YFdgT)H40q4wHL@vaG>YQfWOP?~qv7&D
zJVxE&jRx9Z9~!Z6d@($~>3|VvJQFl`1)7@z(YSCL*CF-KOe@u;woFysa8Xp_xRRrK
z#QKTqT(^8Q9D><23@=Ys6};7|4jS7h2AeW|R=eupp>edcL4Dn01I>-+JTwKHQZ?5f
zT&%vHp;T?Q(t8a<jfon2BsvryS?yH;(Z)q56+tu%BlG=R->b!cR#lT?VOO<}mr(1N
z9;4PF<E`vu$fTxz_JUHY54YMD10m(=&Mmqs^4(f74|eGWH}dKQeCp8ouIaA(+kA_z
zRoOf3$H_}|`L^xQG1!)%78kWcLss5J?fBJDHN}Xh%3mxtsOYxxss*huRXZdQspc7T
zT6Na5X0zu@*PGor_tI>p!8TJ)VMFuf2H|GEJ50=te=Ri??O$WYzh|=90i*TGcU%^$
z>1W8R%zw$Fa(e$hWsfKOltJRVCIzT$R{98Lchp`|xs<G@&M0QBDz?X29mM_@ze~M0
ztU;-M<1RHeF>BR|N!n`F0^An*k&<R>)^l1kZA&s+y1>Gmg;~mCqMN40<;MGF#@(_O
z?I-@4v!DN<ne)I_>ki91&7(iAXf9{{t9jR}RV(?Rp_am$KCMu3GcCc`|C&l4T~*!F
z3sl8}g;dQxMk#Ne7pwBj=!?qPBbh2kkDgLKtIDM^=^3BOV;>u}t3gbvQJg#~33Dc>
zN9*lUKii+8p8bhm@xZ}4)e|$*)Y;GNQq%s=uJP-JyM{olm-=h@!x~xKJ2hlwo@sn?
zU9Y~MGe<2fVx7jS59S)*B-YAV_Zg_Z)-#kTUG`rp%ez@+_CF5k;)KPD<$UjzBy>K?
zrpb0IKD@-IHTQ6_7N^S<&08P)HJ{5^YC66Z)2e*at;x}>q%}v2Nvrzk6itcsr<7ME
zYN-0wURSZb7OmVpr(Q+hUP|S(nw;7mE_Stfdm_}l&wf{(E!v}F_+3Y@+b%}u>D%R6
zJ{%^x606yDT7O^F`Jd0H_ok&ur>AqV&XY7j^OQyArpB-NOg9+mnQTn<HS4sJG(E>E
zZO*rt+qCmdf!PU7UsI+%j;abKA6347S)|fhy;oUFeTua2gdnik&BI$%UN#me`Fy#j
z{FP^y$&z=<X3FlIrdfe$#>;ngn##Y>HQjJY-t5VXEhc9DY^EI1LMC&zo;5XEAZk)`
ztkCROS(WKOiTNh8Zu~McD|}~av+$C!Yui23vSksbptYHAZ`El`u{Tlg>#R}BsbT@o
z`{ubhtH<yqs6SfKq<(n!A(iyK6V&%N>uBhyYO255vt0fCS8fe!7E6sJtsUypYmTVx
z%lxgLa!gL0@yG>}+W8!Y-%cGgS(V6cyoBesQDtX{k#YG{lLC7c!%K@)Omli1Ov=~(
zP(M-RtD(^9uU^BVsQEg^LvsbUz2+JYZnZBFy=sNJuQjH=P|<k!@`hU2W<!lA){W}d
z!`(HrCi!aCv}I_zn(R}LDO;jCb;Vna9j!AolJ_-BzJ1fLXw?2e0)#E6OjPI&6p;Ec
zgF{YbVUfIpzOMA@n!9rIcB`qId&Q~EHPcfS6`!T<sd-$z(v(Rf=5>Y2q5kE{EARbL
zUwJ)O?LAAArhfM)O?82H8jUYyHE+#r)vyzqtGUUYMN`)-LGw?*BF(wVlA7H;VcHkx
z%vS%H^H_^PC{Q!@cCO~BtTruYM?USDT#}l5{yx;4B^#~zPPtEeMd>%K-m24DmaA>F
ztY>v=-h6&QEBpNk%|}0;Xr({-qWPjwT}wJlP!$v>AR2@l_svuU(M9`aD)vFyAhGsu
zx0R~`^wna+!_+=rnyp?lf0nwv@eTEPo0ckDBuOg2eW9$rYu6#wJyTwqa^*FfT~sx-
zSkLjwT=?QQ)2(fm7CM*SnS5Tq**vLK!0g7FHnX3W52U>}_((Ud5>ZiSzoR0UW+$Y2
z?7Py$V0{Vu+6;wOL0N@IYyT;P-@U5H;_yQHj+L|G$<-e6&jJ~goh;Smtl!%zcTHlK
zKY8ViilTd#((H{=njvvD>WeRWYHn!wrG7$ELW40oPW_8%g!+^<tm@(d3)I<#q}5w>
zuBfx`{j5Iw#ve7-85h;uU!|*m=b5VUAbF{Jp3Ne)-OqY7+O-SSTP|EQF@I}iz&!W8
z$pMS)#;+c;85v)nZ0vnW$>hzuX-1X)Ql<`S_9iD&Kbq;hE;iv#JZjGJZ@PKnqrWCX
z-$X1PJXmS=bY7CF9Cy9B8rKdpW3P6#YwbdcPYmmnEk)zizZI=g3Txk@w&L3?)zk^I
z<c~zYR@UsAru4LsO?&gM$Li*HCu>!mn5cc<qCso&1}AORE+1_x2?H(0n_4>aG{v>~
z!uM+K{%}e?BW|tc!H~ro7OIRIM$68sH%{Z$s1jPPuC%5_<H7Z<>b1wr4fCDt^^cxq
zHfAlnYt$|kY@n3HVr<)(U{GAK(C~4ai{U(87Q>Z-drdv|Ih!rixn*W`Ny5x@N153(
zH7m2PDXUG7TbwZUk34HSm3xL+fmW`3gQ=q0AJ4a{?&})W>LWwcy|TZn?bQoVJ)%EL
zO<HEHoW9gFwFm$GOw&wfn0>hKWEPU*U@HIPp{Z&}h}r9T-DYPV+&A688f~_X?}n+|
zga)mHT~D-{j)Z6$%$%yp)ETas@&CH!*S(IKm2WR-=Bt!x3L9_GJh)a#LykjIBjuZs
zx)V>QM$5z&jn&QEn%QessqfjnO|8Y^p~laEx$37j*lU>C@M_*@*VQOZNYQ-6Hd9l!
zcaNr@e1t|#??QEj)s>pln;vUK&i}0;5O-d~>*rLBBcgno5fPU(L>&w?k5@Wq`1Isy
zNbrPdnkBbtEPC3cv7&2-M*FTM8YK~2nhMVUHJ+Ze)ZB0>Q^WX2rN*H~In9L*^E6f$
z+G!s#?9e>#(W&!1+fs{tca4sLqL7ZwEqR^xC`oPeE&H{-B<E^t`>xR1d~u8R#`7Xt
z%UnCO&McUywet1}t=UO4wf@V>X=nfK)@tcqqqXCUin5hqp$drJsac{d_xPR?jBobZ
z9V`x#TgdoHP4t(KGTZVN^~BE&>c^%oRp((fSO2w+SEc#NK9wq8E{(;^_f+mNhN!)G
z->wwj%&ziCrAgh6QBVEk1zvUL3G-BLPv5Mn=D0vTyL_hFSt8eaYu$OQ9KKFeC6rrK
z%`=Tz?TovKy4KMds&kn4C?C;eSNmH%NA1&vZE9Jo+)Um-YA`*z`ip7rlcQ#*m#3Om
zh%1;)E}3GoDq*rIuiQ1$meNh8OfF}X{wGc61<_M<#FbT<o=AQA{9Ez<w{`Lfw{}QJ
z7amht&FCsulqqZ8|MZs8TQ*;F#}#&F`YoSKdnQgcyC>ytesYVKsr%Ni=5A-o%r%^x
z)sue3Y4CVXPy?+glAd}<1+<3;RMvULK2g(L-K`E1uWy>Iw79oH>5_bql9b*vI4$O}
z56o|#eo7T2H#wip$mw6P!LBW-hT(=64c@e97)I-S(Le2-V`v)hVi;7i)9^&$Rzo8L
z6^l<IoMzv@n^;`CV`!c$vch~$t%P~cJ1z^~Di-tEw<0XAEOxT+Fx68{vC314iIG-~
zEdQjkPNGajX4!6)8Ik^~hFbeoqW5SjU-hk4TB3Sh@rwOLA#ocn)s43&%YyLSgl#gW
z>K)3}alJw^YN9GfZ{3n5cAtZQ@j9KqPmQ&|ernK3_x`W*`^Qf0=!?H~c4l$wh^Hs$
zoXeBZStuB+b0D)@%WH+OW`OY_&82c&nnxo_H4Z%B)H-BSt0C=pOS4Sxu!g9Ok%rs-
zAoX2S4K%JdEC!u#rS`Octt#lep_g?lKx_KcK<5&z|LdhXh2fND$0BLX)xV-N6R%rq
zzM3zh$;48n8JGJ?W0t(C#-YeI%?H<(YKX4CqHZLyRBi2s1ode1#j1BLxK*QN<<;xr
zvehP=SgXwCs8H`;c12CgINEr9jHpp$!&hV7n;VU#@(YZ<u`rs<sCjSXVl>If{M>Zo
zt7_@S%d{REU)&sJcr#(D@ya|UqoOU9Ms<ZQ#=qZln3xH_F<k!Tp0R;NqVbK%+q47r
zEYRv`n5(TK-KL#=X1<n?d4_i5{!LnUJsP!N>VMY?X5!Lzs8Z1l(s`~mLr79LE4x!i
zVOg4v^qM0&R}M((%62}~PS|%tM<;8h&h$meCf%nU%`_GrHt~>qYP|5@A=AR+tj3cq
z1WYgHiJM*e(P%p3-zignb0_UPk?d;X4Jw)@uYYLVJIb#m$1zjw%7Vq3(;XJ88MLm{
z673V$x+Rrv>QonEI=f%iT)4&1>}KX=lY6(b&G+RmG@az7VS1!{ohh?~lqvIy2D9gB
zvrH0QCYmq!b<)gt!!$GZ^abYAn|00AU5d<>-d=6a(ClXRP41r7y}glIH@ACfIn>|M
zoX_W~S$0NDD@7_@^JdQq&1+ZgYWnnU*VJw2HC+<SW~!61$n04muQ|`H>n8i>$(U!E
z%rkj%*3fj~$2il_9Sx=_3K{B)ge^4=Pq0<xYH3tmedB~m=h02dfxePzw|I=zWR{#%
zD>f2UW52ad?c*UU4G_(9+(0dDw~v}vm%f_fvw5nv=lsC(*LRhwg61Vaa}*#9nxBBt
zAaM{MHeV58IZd5+vaH6|o=Eke`12Z1^iFHs5U|qhSKXujKWwFHQYV{cle?M5{7vVT
zkKJ(*O8hUY^mQe(qV+RhS+9<NDi+og!0b$swMwVcloULjG&MHNI;ehmnvMEB_m66M
lT<z-K^=sA7EY(;4{eFSE7qgeTin^Q{F?~jmns3k(>jA%NpeO(U

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index
new file mode 100644
index 0000000000000000000000000000000000000000..d338a78f056628943e0d7a59899e11fa5444b717
GIT binary patch
literal 961
zcmZQzVB=tvV&Y(A;O0(BElD(tFUT)XE#hDlV$fhQT~;k<#lp^Ogb-9<(O@w>WoOgF
zz%7)VpI2s-qMwwRSj@pF#3IDRp`gH_(ZFb$JojM&H><F|V@hH!T&9B&Bz5_gR<8h)
zaJ(TxoCzcz+LL~aolQPFwJ0w&hl5c_K!}Nh8HyRfCU-DvG;o@(-(#uG%PJ2u8jCU>
zkg}UI);9?;$%C!IqDB;?CVPk2LpFBdctd0#I4E?;X*4jIx~>RW#f$0%2Zas=kd$AJ
zPr4A27aSBiltAKF-@j|*WK+QE1qC4{4hMw}6^(`iET%#23!?d16|wmNs_qF3SmB)<
zJ`NEkMGRj+RW0EJt2!33+KPcoG$l2!I2Gm<Rv~7N1{Q^mC%hUB0;Vji{LT#gNGf1f
z!<2miD=Yo`Jcgf16l@(#riBk=K)Qb~CmWj-+-sa5??A1X!l%*Dz+w85;b8?2s}#&+
z6cu7%6_f5y(Gg^l0vnGaEeDpqBbon?6+Jq*6gsA;X=s>sX;u{RphgFm!UnL^o<i40
zf=JQ9rEmi*p0fPuISv#zgMvheiGxd_VXB%&!vaQA->aVv^C7z#p<=5#SjC>p$sdH_
zZbnF7)c{M&RCmQPaPz0<<R>NO#21&O7I1J1F(@>A)zsiHozOD*J_7>-!%Id61||^W
y(DO~%Aes?G9Qd?w0|O&-2E%JM5Em@gvh&&A4{RKN+4w<1_;L8}hHjNo_uBx6l+IZI

literal 0
HcmV?d00001

-- 
GitLab


From de0ed5a2bf01342d4c23357d3abf39eb68bb0b2a Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 6 Nov 2018 15:44:39 -0800
Subject: [PATCH 0112/1554] Add quantization mnist test to BUILD

---
 tensorflow/contrib/tensorrt/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 64a2fd50c3..1721305d6d 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,6 +491,7 @@ cuda_py_tests(
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
+        "test/quantization_mnist_test.py",
         "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
-- 
GitLab


From 47cbd92e296ba18149fb78b87726426475fcd2f4 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 7 Nov 2018 09:52:33 +0800
Subject: [PATCH 0113/1554] 'op name' -> 'op type'

---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 65bd568f6c..e6c3916e5d 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2669,7 +2669,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
              &transpose_nchw_in);
 
   // We use same name as original node, but change the op
-  // name.
+  // type.
   NodeBuilder nb(mklop->name(), mklop->type_string());
 
   for (int i = 0; i < mklop_num_inputs; i++) {
-- 
GitLab


From 64329f85785c190467798f25a746291a813f192f Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 7 Nov 2018 09:56:58 +0800
Subject: [PATCH 0114/1554] Add a comment: storing the output slots of input
 nodes.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index e6c3916e5d..01f1aa9078 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2672,6 +2672,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   // type.
   NodeBuilder nb(mklop->name(), mklop->type_string());
 
+  // Storing the output slots of the input nodes.
   for (int i = 0; i < mklop_num_inputs; i++) {
     if (mklop_in[i].first == transpose_to_nhwc) {
       // Fill "x":
-- 
GitLab


From 3ea1267b9758fdc5582948805cdd852b09f21f6b Mon Sep 17 00:00:00 2001
From: dianlujitao <dianlujitao@lineageos.org>
Date: Wed, 7 Nov 2018 13:24:28 +0800
Subject: [PATCH 0115/1554] Install abseil headers to cmake shared library
 build

 * Since commit 5f004516 tensorflow::StringPiece is replaced by
   absl::string_view, so abseil headers should be installed to shared
   library build to fix compilation error for out-of-source build.
 * To cleanly copy abseil headers, disable in source build for abseil to
   avoid src tree been polluted by cmake generated files.
 * Meanwhile, remove _build suffix from abseil_cpp product name since
   it's confusing.
---
 .../contrib/cmake/external/abseil_cpp.cmake       | 15 ++++++---------
 tensorflow/contrib/cmake/tf_shared_lib.cmake      |  4 ++++
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index 4546dbdecc..b0fee24448 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -31,17 +31,17 @@ if (systemlib_ABSEIL_CPP)
   message(STATUS "  abseil_cpp includes: ${ABSEIL_CPP_INCLUDE_DIR}")
   message(STATUS "  abseil_cpp libraries: ${ABSEIL_CPP_LIBRARIES}")
 
-  add_custom_target(abseil_cpp_build)
-  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
+  add_custom_target(abseil_cpp)
+  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp)
 
 else (systemlib_ABSEIL_CPP)
 
   include (ExternalProject)
 
-  set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build)
+  set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp)
   set(abseil_cpp_URL https://github.com/abseil/abseil-cpp/archive/e01d95528ea2137a4a27a88d1f57c6cb260aafed.tar.gz)
   set(abseil_cpp_HASH SHA256=84043ed402d2a2a6ba4cdddb7e85118b1158fd81fe4ac3a14adc343d054c1e2e)
-  set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build)
+  set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp-build)
 
   if(WIN32)
     if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
@@ -80,15 +80,12 @@ else (systemlib_ABSEIL_CPP)
         ${abseil_cpp_BUILD}/absl/types/libabsl_bad_optional_access.a)
   endif()
 
-  ExternalProject_Add(abseil_cpp_build
+  ExternalProject_Add(abseil_cpp
       PREFIX abseil_cpp
       URL ${abseil_cpp_URL}
       URL_HASH ${abseil_cpp_HASH}
       DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
-      BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES}
-      BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release
-      COMMAND ${CMAKE_COMMAND} --build . --config Release
       INSTALL_COMMAND ""
       CMAKE_CACHE_ARGS
           -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE}
@@ -99,6 +96,6 @@ else (systemlib_ABSEIL_CPP)
   include_directories(${abseil_cpp_INCLUDE_DIR})
   list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES})
 
-  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
+  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp)
 
 endif (systemlib_ABSEIL_CPP)
diff --git a/tensorflow/contrib/cmake/tf_shared_lib.cmake b/tensorflow/contrib/cmake/tf_shared_lib.cmake
index fdf522f1fd..c1bdc35fc6 100644
--- a/tensorflow/contrib/cmake/tf_shared_lib.cmake
+++ b/tensorflow/contrib/cmake/tf_shared_lib.cmake
@@ -145,6 +145,10 @@ install(DIRECTORY ${tensorflow_source_dir}/third_party/eigen3/
 # unsupported Eigen directory
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/unsupported/Eigen/
         DESTINATION include/unsupported/Eigen)
+# absl directory
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/abseil_cpp/src/abseil_cpp/absl/
+        DESTINATION include/absl
+        FILES_MATCHING PATTERN "*.h")
 # mkl
 if (tensorflow_ENABLE_MKL_SUPPORT)
     install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mkl/src/mkl/include/
-- 
GitLab


From 81f02368365e096c27f37adc7c9af08905855ed1 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 7 Nov 2018 14:28:10 +0800
Subject: [PATCH 0116/1554] New algorithm applied to "CheckForNodeFusion()", to
 cover the pattern "A->B->C;A->C".

---
 tensorflow/core/graph/mkl_layout_pass.cc | 57 +++++++++++-------------
 1 file changed, 27 insertions(+), 30 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 01f1aa9078..fa32a3e061 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -915,17 +915,17 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     if (node->type_string() != "Transpose") return false;
 
     // If "Transpose" has multiple output data edges, also don't fuse it.
-    if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
+    // if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
 
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
     // Note: this constraint will eventually be removed, if we enabled this fusion for training
     // in the future. 
-    for (const Edge* e : node->out_edges()) {
-      if (e->IsControlEdge()) {
-        return false;
-      }
-    }
+    // for (const Edge* e : node->out_edges()) {
+    //   if (e->IsControlEdge()) {
+    //     return false;
+    //   }
+    // }
 
     // If "Transpose" has input control edges, don't fuse on it.
     for (const Edge* e : node->in_edges()) {
@@ -2736,12 +2736,13 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
     //
 
     std::stack<Node *, std::vector<Node *>> work_stack;
-    std::unordered_set<Node *> visited_nodes;
+    std::stack<EdgeSet::const_iterator> current_neighbor_stack;
     auto node_checker = fi->node_checkers.begin();
 
     Node *current_node = nullptr;
     if (a != nullptr) {
       work_stack.push(a);
+      current_neighbor_stack.push(a->out_edges().begin());
     }
 
     while (!work_stack.empty()) {
@@ -2759,33 +2760,29 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
           return make_tuple(true, nodes, *fi_ptr);
         }
 
-        bool all_succ_has_been_visited = true;
-        for (const Edge *e : current_node->out_edges()) {
-          if (!e->IsControlEdge()) {
-            Node *candidate_node = e->dst();
-
-            // If the candidate node has not been visited, push it to stack.
-            if (visited_nodes.find(candidate_node) == visited_nodes.end()) {
-              work_stack.push(candidate_node);
-              ++ node_checker;
-              all_succ_has_been_visited = false;
-              break;
-            }
-          }
-        }
-
-        // All successor nodes of current node has been visited (no match found),
-        // pop the stack and mark current node as "visited".
-        if (all_succ_has_been_visited) {
-          visited_nodes.insert(current_node);
-          work_stack.pop();
-          -- node_checker;
+        auto &current_neighbor_iter = current_neighbor_stack.top();
+        if (current_neighbor_iter == current_node->out_edges().end()) {
+            // All output edges have been exhausted, pop the stack
+            // and roll back to the preceding node.
+            work_stack.pop();
+            current_neighbor_stack.pop();
+            -- node_checker;
+        } else {
+            // Found a edge not been visited, go through this edge
+            // and get the next neighbor.
+            Node *neighbor_node = (*current_neighbor_iter)->dst();
+            work_stack.push(neighbor_node);
+            current_neighbor_stack.push(neighbor_node->out_edges().begin());
+            ++ node_checker;
+
+            // Increase current_neighbor_iter, which is at the top of stack.
+            ++ current_neighbor_iter;
         }
-
       } else {
         // current node doesn't match, pop stack to roll back.
-        visited_nodes.insert(current_node);
+        // visited_nodes.insert(current_node);
         work_stack.pop();
+        current_neighbor_stack.pop();
         -- node_checker;
       }
     }
-- 
GitLab


From 4ff0bd48a7e06e775d86e14228e8ea9e9578f95a Mon Sep 17 00:00:00 2001
From: fcole <fcole@google.com>
Date: Thu, 8 Nov 2018 11:52:54 +0800
Subject: [PATCH 0117/1554] substitute backslashes with parentheses (line
 continuation)

Co-Authored-By: zldrobit <zldrobit@126.com>
---
 tensorflow/contrib/image/python/ops/dense_image_warp.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index 2ac7ff2a6b..13c70ff0e6 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -60,7 +60,10 @@ def _interpolate_bilinear(grid,
       msg = 'Grid must be 4 dimensional. Received size: '
       raise ValueError(msg + str(grid.get_shape()))
 
-    batch_size, height, width, channels = \
+    batch_size, height, width, channels = (array_ops.shape(grid)[0],
+                                           array_ops.shape(grid)[1],
+                                           array_ops.shape(grid)[2],
+                                           array_ops.shape(grid)[3])
       array_ops.shape(grid)[0], \
       array_ops.shape(grid)[1], \
       array_ops.shape(grid)[2], \
-- 
GitLab


From 3e19356fd840069d6dce9aee692ab189708d951c Mon Sep 17 00:00:00 2001
From: zldrobit <zldrobit@126.com>
Date: Thu, 8 Nov 2018 11:58:54 +0800
Subject: [PATCH 0118/1554] delete redundant code

---
 tensorflow/contrib/image/python/ops/dense_image_warp.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index 13c70ff0e6..6e0105d069 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -64,10 +64,7 @@ def _interpolate_bilinear(grid,
                                            array_ops.shape(grid)[1],
                                            array_ops.shape(grid)[2],
                                            array_ops.shape(grid)[3])
-      array_ops.shape(grid)[0], \
-      array_ops.shape(grid)[1], \
-      array_ops.shape(grid)[2], \
-      array_ops.shape(grid)[3]
+
     shape = [batch_size, height, width, channels]
     query_type = query_points.dtype
     grid_type = grid.dtype
-- 
GitLab


From cc46242fc750feb8da443d6c42ef2e2cf66cb38c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Fri, 2 Nov 2018 15:45:36 +0800
Subject: [PATCH 0119/1554] ENH: register double for grad op

---
 tensorflow/core/kernels/conv_grad_ops_3d.cc | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc
index 9271bfdf04..7fe4d8e93d 100644
--- a/tensorflow/core/kernels/conv_grad_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc
@@ -1070,6 +1070,7 @@ namespace functor {
 
 DECLARE_GPU_SPEC(Eigen::half);
 DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(double);
 #undef DECLARE_GPU_SPEC
 }  // namespace functor
 
@@ -1859,14 +1860,7 @@ class Conv3DBackpropFilterOp<GPUDevice, T> : public OpKernel {
                           Conv3DBackpropFilterOp<GPUDevice, T>);
 TF_CALL_half(REGISTER_GPU_KERNEL);
 TF_CALL_float(REGISTER_GPU_KERNEL);
-REGISTER_KERNEL_BUILDER(
-    Name("Conv3DBackpropInput").Device(DEVICE_GPU).TypeConstraint<double>("T"),
-    Conv3DBackpropInputOp<GPUDevice, double>);
-REGISTER_KERNEL_BUILDER(Name("Conv3DBackpropInputV2")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<double>("T")
-                            .HostMemory("input_sizes"),
-                        Conv3DBackpropInputOp<GPUDevice, double>);
+TF_CALL_double(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 
 #endif  // GOOGLE_CUDA
-- 
GitLab


From 29303bc712888e3f0d57e804c93bb1e7c58368a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 8 Nov 2018 14:19:30 +0800
Subject: [PATCH 0120/1554] TST: use cached_session instead

---
 tensorflow/python/kernel_tests/conv_ops_3d_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index dd90676f09..b3e73fa841 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -413,7 +413,7 @@ class Conv3DTest(test.TestCase):
       elif data_type == dtypes.float16:
         tolerance = 1e-3
 
-      with self.test_session(use_gpu=use_gpu):
+      with self.cached_session(use_gpu=use_gpu):
         orig_input_tensor = constant_op.constant(
             input_data, shape=input_shape, dtype=data_type, name="input")
         filter_tensor = constant_op.constant(
-- 
GitLab


From 3d974edb6e62e3020e7c83bb28c76a0db277f87a Mon Sep 17 00:00:00 2001
From: zldrobit <zldrobit@126.com>
Date: Thu, 8 Nov 2018 14:20:08 +0800
Subject: [PATCH 0121/1554] change error messages after splitting condition

---
 tensorflow/contrib/image/python/ops/dense_image_warp.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index 6e0105d069..df48f0e430 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -78,11 +78,9 @@ def _interpolate_bilinear(grid,
 
     with ops.control_dependencies([
         check_ops.assert_greater_equal(height, 2, message=
-                                       'Grid must be at least batch_size'
-                                       'x 2 x 2 in size.'),
+                                       'Grid height must be at least 2'),
         check_ops.assert_greater_equal(width, 2, message=
-                                       'Grid must be at least batch_size'
-                                       'x 2 x 2 in size.')]):
+                                       'Grid width must be at least2')]):
       alphas = []
       floors = []
       ceils = []
-- 
GitLab


From 9ab76bfe5468f858ce71443603aa16725bb42247 Mon Sep 17 00:00:00 2001
From: zldrobit <zldrobit@126.com>
Date: Thu, 8 Nov 2018 14:27:53 +0800
Subject: [PATCH 0122/1554] fix some typo

---
 tensorflow/contrib/image/python/ops/dense_image_warp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index df48f0e430..72cd7fe219 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -80,7 +80,7 @@ def _interpolate_bilinear(grid,
         check_ops.assert_greater_equal(height, 2, message=
                                        'Grid height must be at least 2'),
         check_ops.assert_greater_equal(width, 2, message=
-                                       'Grid width must be at least2')]):
+                                       'Grid width must be at least 2')]):
       alphas = []
       floors = []
       ceils = []
-- 
GitLab


From 35f9eddd98f3470e779b8834800ccf66174318d1 Mon Sep 17 00:00:00 2001
From: zldrobit <zldrobit@126.com>
Date: Thu, 8 Nov 2018 14:45:37 +0800
Subject: [PATCH 0123/1554] fix some indentation and typo

---
 .../image/python/ops/dense_image_warp.py      | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index 72cd7fe219..554f875cad 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -70,17 +70,21 @@ def _interpolate_bilinear(grid,
     grid_type = grid.dtype
 
     with ops.control_dependencies([
-        check_ops.assert_equal(len(query_points.get_shape()), 3, message=
-                               'Query points must be 3 dimensional'),
-        check_ops.assert_equal(array_ops.shape(query_points)[2], 2, message=
-                               'Query points must be size 2 in dim 2.')]):
+        check_ops.assert_equal(
+            len(query_points.get_shape()),
+            3,
+            message='Query points must be 3 dimensional.'),
+        check_ops.assert_equal(
+            array_ops.shape(query_points)[2],
+            2,
+            message='Query points must be size 2 in dim 2.')]):
       num_queries = array_ops.shape(query_points)[1]
 
     with ops.control_dependencies([
         check_ops.assert_greater_equal(height, 2, message=
-                                       'Grid height must be at least 2'),
+                                       'Grid height must be at least 2.'),
         check_ops.assert_greater_equal(width, 2, message=
-                                       'Grid width must be at least 2')]):
+                                       'Grid width must be at least 2.')]):
       alphas = []
       floors = []
       ceils = []
@@ -187,11 +191,10 @@ def dense_image_warp(image, flow, name='dense_image_warp'):
                 of dimensions.
   """
   with ops.name_scope(name):
-    batch_size, height, width, channels = \
-      array_ops.shape(image)[0], \
-      array_ops.shape(image)[1], \
-      array_ops.shape(image)[2], \
-      array_ops.shape(image)[3]
+    batch_size, height, width, channels = (array_ops.shape(image)[0],
+                                           array_ops.shape(image)[1],
+                                           array_ops.shape(image)[2],
+                                           array_ops.shape(image)[3])
 
     # The flow is defined on the image grid. Turn the flow into a list of query
     # points in the grid space.
-- 
GitLab


From 917bd04d77a6872e8ecf31bbe0a1143dc98b6e8c Mon Sep 17 00:00:00 2001
From: zldrobit <zldrobit@126.com>
Date: Thu, 8 Nov 2018 14:54:53 +0800
Subject: [PATCH 0124/1554] fix indentation

---
 .../contrib/image/python/ops/dense_image_warp.py     | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
index 554f875cad..7930b8317b 100644
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -81,10 +81,14 @@ def _interpolate_bilinear(grid,
       num_queries = array_ops.shape(query_points)[1]
 
     with ops.control_dependencies([
-        check_ops.assert_greater_equal(height, 2, message=
-                                       'Grid height must be at least 2.'),
-        check_ops.assert_greater_equal(width, 2, message=
-                                       'Grid width must be at least 2.')]):
+        check_ops.assert_greater_equal(
+            height,
+            2,
+            message='Grid height must be at least 2.'),
+        check_ops.assert_greater_equal(
+            width,
+            2,
+            message='Grid width must be at least 2.')]):
       alphas = []
       floors = []
       ceils = []
-- 
GitLab


From 2ee1c5a0d6b99574f48e7f0de9b94559d824b022 Mon Sep 17 00:00:00 2001
From: Tongxuan Liu <tongxuan.ltx@alibaba-inc.com>
Date: Wed, 7 Nov 2018 14:44:12 +0800
Subject: [PATCH 0125/1554] RendezvousMgr & CancellationMgr are already
 aborted, shouldn't send RPC call any more

When RendezvousMgr & CancellationMgr are already aborted, following RPC couldn't handled
by CancellationMgr. At the moment, remote service is already closed, client would hang here.
---
 tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc             | 8 ++++++++
 .../core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc   | 9 +++++++++
 2 files changed, 17 insertions(+)

diff --git a/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc b/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc
index 94f522c04e..fbccbead03 100644
--- a/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc
+++ b/tensorflow/contrib/gdr/gdr_rendezvous_mgr.cc
@@ -170,6 +170,14 @@ class GdrRemoteRendezvous : public BaseRemoteRendezvous {
     // Record "call" in active_ so that it can be aborted cleanly.
     RegisterCall(call);
 
+    // RendezvousMgr already aborted, shouldn't send RPC call any more
+    if (!call->status().ok()) {
+      done(call->status(), Args(), Args(), Tensor(), false);
+      session()->worker_cache->ReleaseWorker(src_worker, rwi);
+      delete call;
+      return;
+    }
+
     // Start "call".
     Ref();
     call->Start([this, call, src_worker, rwi, done]() {
diff --git a/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc b/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc
index b8cb538503..9fb920404f 100644
--- a/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc
+++ b/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.cc
@@ -244,6 +244,15 @@ void RpcRemoteRendezvous::RecvFromRemoteAsync(
   // Record "call" in active_ so that it can be aborted cleanly.
   RegisterCall(call);
 
+  // RendezvousMgr already aborted, shouldn't send RPC call any more
+  if (!call->status().ok()) {
+    call->done()(call->status(), Args(), Args(), Tensor(), false);
+    session()->worker_cache->ReleaseWorker(call->src_worker_, call->wi_);
+    call->wi_ = nullptr;
+    get_call_freelist()->Release(call, session()->worker_cache.get());
+    return;
+  }
+
   // Start "call".
   Ref();
   call->Start([this, call]() {
-- 
GitLab


From 623b4fef5351713697ba1bd54a4a60945a6e2afa Mon Sep 17 00:00:00 2001
From: "Jiang,Zhoulong" <zhoulong.jiang@intel.com>
Date: Fri, 9 Nov 2018 15:12:56 +0800
Subject: [PATCH 0126/1554] Fix memory leak issue in MklFusedBatchNorm

---
 tensorflow/core/util/mkl_util.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 7988364716..8f40032924 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -1616,6 +1616,9 @@ class MklDnnData {
         cpu_engine_(e) {}
 
   ~MklDnnData() {
+    if (allocated_buffer_ != nullptr) {
+      cpu_allocator()->DeallocateRaw(allocated_buffer_);
+    }
     cpu_engine_ = nullptr;  // We don't own this.
     delete (user_memory_);
     delete (reorder_memory_);
-- 
GitLab


From 74142f67926917061134678234ebee9683562f77 Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Fri, 9 Nov 2018 08:13:19 +0000
Subject: [PATCH 0127/1554] Fixed old tests breaking because cond_v2
 implementation preserves nested output structures even with singeltons

---
 .../python/kernel_tests/control_flow_ops_py_test.py       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 98f8fd4e96..71e4ccddde 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2097,8 +2097,12 @@ class ControlFlowTest(test.TestCase):
                                         [tensor_shape.unknown_shape()])
         return gradients_impl.gradients(r, x)
 
-      r = control_flow_ops.cond(math_ops.less(1, 2), fn1, lambda: x)
-      self.assertAllClose(9.0, r.eval(feed_dict={x: 1.0}))
+      #placed lambda function return tensor in list and set strict flag to True
+      #as cond_v2 implementation preserves nested output structures even with singeltons
+      r = control_flow_ops.cond(math_ops.less(1, 2), fn1, lambda: [x], strict=True)
+      #cannot run eval() on list object so use sess.run() and save output
+      result = sess.run(r,feed_dict={x: 1.0})
+      self.assertAllClose([9.0], result)
 
   @test_util.disable_control_flow_v2("b/116340060")
   def testGradInWhileWrtInitialLoopVal(self):
-- 
GitLab


From da089db8e6c868f7fab11e45cbea182daccdb393 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 9 Nov 2018 09:47:51 -0800
Subject: [PATCH 0128/1554] Fixes for @aaroey's review.

Fix MNIST integration test imports and add the data dependencies as a build rule.
GetWeightRange will now only be called in int8mode no calib mode.
Make precision_mode a member of TrtCandidateSelect instead of argument to IsCandidate.
Make precision_mode, use_calibration members of Converter
Remove QINT8 from ConvertDType. Remove QINT8, UINT8, INT8, QUINT8 from RedorderCK. These are no longer needed after removing QuantizeV2 and Dequantize support.
Refactor ApplyQuantizatioNRanges() to make it easier to test.
ConvertQuantize will explicitly look for the different ops by name.
Add checks to Relu6.
Add unit tests for new functions in Converter.
Add comments to clarify some logic
---
 tensorflow/contrib/tensorrt/BUILD             |  23 +-
 .../contrib/tensorrt/convert/convert_graph.cc |  19 +-
 .../contrib/tensorrt/convert/convert_graph.h  |   8 +-
 .../tensorrt/convert/convert_graph_test.cc    |  10 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 178 +++++-----
 .../contrib/tensorrt/convert/convert_nodes.h  |  17 +-
 .../tensorrt/convert/convert_nodes_test.cc    | 315 +++++++++++++++---
 .../contrib/tensorrt/segment/segment.cc       |   4 +-
 tensorflow/contrib/tensorrt/segment/segment.h |   2 +-
 .../tensorrt/test/quantization_mnist_test.py  |  55 ++-
 .../test/quantization_mnist_test_data/BUILD   |  14 +
 ...f-00001 => model.ckpt.data-00000-of-00001} | Bin
 ...odel.ckpt-14070.index => model.ckpt.index} | Bin
 .../test/tf_trt_integration_test_base.py      |   2 +
 14 files changed, 476 insertions(+), 171 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD
 rename tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/{model.ckpt-14070.data-00000-of-00001 => model.ckpt.data-00000-of-00001} (100%)
 rename tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/{model.ckpt-14070.index => model.ckpt.index} (100%)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 1721305d6d..64c0fecf87 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,7 +491,6 @@ cuda_py_tests(
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
-        "test/quantization_mnist_test.py",
         "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
@@ -510,6 +509,28 @@ cuda_py_tests(
     ],
 )
 
+cuda_py_tests(
+    name = "tf_trt_quantization_mnist_integration_test",
+    srcs = [
+        "test/quantization_mnist_test.py",
+    ],
+    additional_deps = [
+        ":tf_trt_integration_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/keras:keras",
+        "//tensorflow/python/estimator:estimator",
+    ],
+    data =[
+      "//tensorflow/contrib/tensorrt/test/quantization_mnist_test_data:quantization_mnist_data",
+    ],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+)
+
 cuda_py_tests(
     name = "tf_trt_integration_test_no_oss",
     srcs = [
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index b9c6dc7fde..b1443e7791 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -82,10 +82,11 @@ std::vector<int> GetLoadedTensorRTVersion() {
 }
 
 TrtCandidateSelector::TrtCandidateSelector(
-    const grappler::GraphProperties& graph_properties)
-    : graph_properties_(graph_properties) {}
+    const grappler::GraphProperties& graph_properties,
+    int precision_mode)
+    : graph_properties_(graph_properties), precision_mode_(precision_mode) {}
 
-Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node, int precision_mode) {
+Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
   // TODO(laigd): move this set to TrtNodeValidator where it should belong.
   // LINT.IfChange
   static const std::set<string> candidate_ops = {
@@ -134,14 +135,15 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node, i
       PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string()));
 #if NV_TENSORRT_MAJOR >= 5
   static const std::set<string> quantize_ops = {
-    "QuantizeV2",
-    "Dequantize",
     "QuantizeAndDequantizeV2",
     "QuantizeAndDequantizeV3",
     "FakeQuantWithMinMaxVars",
     "FakeQuantWithMinMaxArgs",
   };
-  if (precision_mode == INT8MODE &&
+  // In INT8 mode, we will always apply the quantization ranges provided by
+  // these ops to the relevant tensors. This happens regardless of the value of
+  // use_calibration.
+  if (precision_mode_ == INT8MODE &&
       quantize_ops.count(node->type_string())) {
     is_supported_op_type = true;
   }
@@ -885,11 +887,12 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   segment_options.minimum_segment_size = params.minimum_segment_size;
   segment_options.precision_mode = params.precision_mode;
   tensorflow::tensorrt::segment::SegmentNodesVector initial_segments;
-  TrtCandidateSelector candidate_selector(*params.graph_properties);
+  TrtCandidateSelector candidate_selector(*params.graph_properties,
+                                          params.precision_mode);
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
       &graph,
       std::bind(&TrtCandidateSelector::IsTensorRTCandidate, &candidate_selector,
-                std::placeholders::_1, std::placeholders::_2),
+                std::placeholders::_1),
       // Input validation is already done by TrtCandidateSelector, so we don't
       // need to check the input edges.
       [](const Edge* edge) { return true; }, OutputEdgeValidator(),
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index eaa1edbfe4..366d69115b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -35,17 +35,21 @@ namespace convert {
 // supported by TRT.
 class TrtCandidateSelector {
  public:
-  TrtCandidateSelector(const grappler::GraphProperties& graph_properties);
+  TrtCandidateSelector(const grappler::GraphProperties& graph_properties,
+                       int precision_mode);
 
   // Returns OK iff 'node' is a TF-TRT conversion candidate, which will be added
   // to TRT subgraph and later converted into TRT engine.
-  Status IsTensorRTCandidate(const tensorflow::Node* node, int precision_mode);
+  Status IsTensorRTCandidate(const tensorflow::Node* node);
 
  private:
   // The TF-TRT node converter used to verify whether individual node is
   // supported. It will operate in validation-only mode.
   TrtNodeValidator validator_;
 
+  // Quantization ops are only converted when using quantized precisions.
+  int precision_mode_;
+
   // GraphProperties of the graph whose nodes are to be validated by
   // IsTensorRTCandidate().
   const grappler::GraphProperties& graph_properties_;
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
index 925de7885f..e2a7c40f30 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
@@ -93,17 +93,17 @@ TEST(TrtCandidateSelector, Basics) {
   grappler::GraphProperties graph_properties(item);
   TF_EXPECT_OK(graph_properties.InferStatically(true));
 
-  TrtCandidateSelector selector(graph_properties);
-  TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node(), FP32MODE));
+  TrtCandidateSelector selector(graph_properties, FP32MODE);
+  TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node()));
   ExpectStatus(
-      selector.IsTensorRTCandidate(incompatible_matmul.operation.node(), FP32MODE),
+      selector.IsTensorRTCandidate(incompatible_matmul.operation.node()),
       error::INVALID_ARGUMENT,
       "transpose_a is not supported for TensorRT FullyConnected "
       "(op: MatMul), at: incompatible_matmul");
-  ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node(), FP32MODE),
+  ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node()),
                error::UNIMPLEMENTED, "Op type Sin is not supported");
   ExpectStatus(selector.IsTensorRTCandidate(
-                   matmul_with_incompatible_input.operation.node(), FP32MODE),
+                   matmul_with_incompatible_input.operation.node()),
                error::INTERNAL,
                "Failed to convert input with index 0 to a TRT_TensorOrWeights");
 }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 456840e537..b8aa49506b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -95,9 +95,6 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
     case tensorflow::DataType::DT_INT8:
       *trt_dtype = nvinfer1::DataType::kINT8;
       break;
-    case tensorflow::DataType::DT_QINT8:
-      *trt_dtype = nvinfer1::DataType::kINT8;
-      break;
     case tensorflow::DataType::DT_HALF:
       *trt_dtype = nvinfer1::DataType::kHALF;
       break;
@@ -637,16 +634,6 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
           ostrides);
       break;
     }
-    case tensorflow::DataType::DT_INT8:
-    case tensorflow::DataType::DT_UINT8:
-    case tensorflow::DataType::DT_QINT8:
-    case tensorflow::DataType::DT_QUINT8: {
-      Reorder2({k, c}, static_cast<int8_t const*>(iweights.GetValues()),
-               istrides,
-               static_cast<int8_t*>(const_cast<void*>(oweights->GetValues())),
-               ostrides);
-      break;
-    }
     default:
       LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
                  << DataTypeString(iweights.type_);
@@ -799,8 +786,12 @@ Status TrtNodeValidator::ConvertConstToWeights(
   return status;
 }
 
-Converter::Converter(nvinfer1::INetworkDefinition* trt_network, bool is_fp16)
-    : trt_network_(trt_network), is_fp16_(is_fp16) {
+Converter::Converter(nvinfer1::INetworkDefinition* trt_network,
+                     int precision_mode,
+                     bool use_calibration)
+    : trt_network_(trt_network),
+      precision_mode_(precision_mode),
+      use_calibration_(use_calibration) {
   this->RegisterOpConverters();
 }
 
@@ -971,7 +962,7 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor,
 
 Status Converter::GetWeightRange(const TRT_ShapedWeights& weights,
                                  float* out_min,
-                                 float* out_max) {
+                                 float* out_max) const {
   switch (weights.type_) {
     case tensorflow::DataType::DT_FLOAT: {
       auto inp = static_cast<float const*>(weights.GetValues());
@@ -987,9 +978,16 @@ Status Converter::GetWeightRange(const TRT_ShapedWeights& weights,
       *out_max = Eigen::half_impl::half_to_float(*result.second);
       break;
     }
+    case tensorflow::DataType::DT_INT32: {
+      auto inp = static_cast<int const*>(weights.GetValues());
+      auto result = std::minmax_element(inp, inp + weights.count());
+      *out_min = static_cast<float>(*result.first);
+      *out_max = static_cast<float>(*result.second);
+      break;
+    }
     default:
       return tensorflow::errors::Unimplemented(
-          "Data type not supported: " +
+          "Data type not supported for GetWeightRange: " +
           tensorflow::DataTypeString(weights.type_));
   }
   return tensorflow::Status::OK();
@@ -1030,21 +1028,24 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
         this->network()->addConstant(dims, input.weights().GetTrtWeights());
     TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
     *tensor = layer->getOutput(0);
-    // We need to set a quantization range for the output tensor of the
-    // IConstantLayer. Here we set the range to [min(weights), max(weights)].
-    float min_range = 0.0f;
-    float max_range = 0.0f;
-    TF_RETURN_IF_ERROR(
-        GetWeightRange(input.weights(), &min_range, &max_range));
-    // Avoid setting range to 0 because TRT will throw an error. If the weights
-    // are zero then the range doesn't matter: using 127.0f should ensure the
-    // quantized weight will be exactly zero.
-    if (min_range == 0.0f && max_range == 0.0f) {
-      min_range = -127.0f;
-      max_range = 127.0f;
-    }
-    ProvideQuantizationRange(const_cast<nvinfer1::ITensor*>(*tensor),
-        min_range, max_range);
+    if (precision_mode() == INT8MODE && !use_calibration()) {
+      // If we are in int8 mode and not calibrating, we need to explicitly set a
+      // quantization range for the output tensor of the IConstantLayer. Here we
+      // set the range to [min(weights), max(weights)].
+      float min_range = 0.0f;
+      float max_range = 0.0f;
+      TF_RETURN_IF_ERROR(
+          GetWeightRange(input.weights(), &min_range, &max_range));
+      // Avoid setting range to 0 because TRT will throw an error. If the weights
+      // are zero then the range doesn't matter: using 127.0f should ensure the
+      // quantized weight will be exactly zero.
+      if (min_range == 0.0f && max_range == 0.0f) {
+        min_range = -127.0f;
+        max_range = 127.0f;
+      }
+      ProvideQuantizationRange(const_cast<nvinfer1::ITensor*>(*tensor),
+          min_range, max_range);
+    }
   }
   return tensorflow::Status::OK();
 }
@@ -1064,44 +1065,45 @@ void Converter::ProvideQuantizationRange(nvinfer1::ITensor* tensor,
 void Converter::ApplyQuantizationRanges(bool warn_missing_ranges) {
   // Infer ranges across marked ops
   PropagateQuantizationRanges();
-  // Get all tensors from network
-  std::set<nvinfer1::ITensor*> all_tensors;
-  std::set<nvinfer1::ITensor*> tensors_missing_ranges;
-  for (int i = 0; i < this->network()->getNbLayers(); i++) {
-    nvinfer1::ILayer* layer = this->network()->getLayer(i);
-    for (int j = 0; j < layer->getNbInputs(); j++)
-      all_tensors.insert(layer->getInput(j));
-    for (int j = 0; j < layer->getNbOutputs(); j++)
-      all_tensors.insert(layer->getOutput(j));
-  }
   // Apply ranges
-  for (auto tensor : all_tensors) {
-    auto it = quantization_ranges_.find(tensor);
-    if (it != quantization_ranges_.end()) {
-      float range = it->second;
-      VLOG(1) << "Setting range for: " << tensor->getName() << ": " << range;
+  for (auto pair : quantization_ranges_) {
+    nvinfer1::ITensor* tensor = pair.first;
+    const float range = pair.second;
 #if NV_TENSORRT_MAJOR >= 5
-      tensor->setDynamicRange(-range, range);
+    VLOG(1) << "Setting range for: " << tensor->getName() << ": " << range;
+    tensor->setDynamicRange(-range, range);
 #endif
-    } else {
-      tensors_missing_ranges.insert(tensor);
-    }
   }
+  
   // Warn user about tensors that are missing ranges. If TRT fuses some layers
   // then these tensors may not actually be required, which is why this is
   // just a warning. If we are still missing ranges even after fusion,
   // Builder::buildCudaEngine() will return nullptr and we will catch the
   // error at that point.
   if (warn_missing_ranges) {
-    for (auto tensor : tensors_missing_ranges) {
-      // Note: there may be some warnings for "(Unnamed ITensor* N)". These
-      // are tensors which are created internally by TF-TRT. The ranges for
-      // these unnamed ITensors are always inferred from user provided ranges,
-      // thus there will also be a warning for the range(s) the user missed.
-      LOG(WARNING) << "Quantization range was not found for "
-                    << tensor->getName() << ". "
-                    << "This might be okay if TensorRT does not need the range"
-                    << "(e.g. due to node fusion).";
+    // Get all tensors from network
+    std::set<nvinfer1::ITensor*> all_tensors;
+    for (int i = 0; i < this->network()->getNbLayers(); i++) {
+      nvinfer1::ILayer* layer = this->network()->getLayer(i);
+      for (int j = 0; j < layer->getNbInputs(); j++) {
+        all_tensors.insert(layer->getInput(j));
+      }
+      for (int j = 0; j < layer->getNbOutputs(); j++) {
+        all_tensors.insert(layer->getOutput(j));
+      }
+    }
+    // Find tensors with no ranges
+    for (auto tensor : all_tensors) {
+      if (quantization_ranges_.find(tensor) == quantization_ranges_.end()) {
+        // Note: there may be some warnings for "(Unnamed ITensor* N)". These
+        // are tensors which are created internally by TF-TRT. The ranges for
+        // these unnamed ITensors are always inferred from user provided ranges,
+        // thus there will also be a warning for the range(s) the user missed.
+        LOG(WARNING) << "Quantization range was not found for "
+                      << tensor->getName() << ". "
+                      << "This is okay if TensorRT does not need the range "
+                      << "(e.g. due to node fusion).";
+      }
     }
   }
 }
@@ -1446,7 +1448,7 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
     }
   }
 
-  if (params->converter->is_fp16()) {
+  if (params->converter->precision_mode() == FP16MODE) {
     weights = ConvertFP32ToFP16(params->weight_store, weights);
   }
 
@@ -1549,7 +1551,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
     return tensorflow::errors::Internal(
         "Conv2D expects kernel of dimension 4, at: " + node_def.name());
   }
-  if (params->converter->is_fp16()) {
+  if (params->converter->precision_mode() == FP16MODE) {
     weights_rsck =
         ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
   }
@@ -1986,7 +1988,16 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
 tensorflow::Status ConvertQuantize(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
-  if (inputs.size() > 0 && inputs.at(0).is_weights()) {
+  if ((inputs.size() == 0) ||
+      (inputs.size() != 1 && node_def.op() == "FakeQuantWithMinMaxArgs") ||
+      (inputs.size() != 3 && node_def.op() == "FakeQuantWithMinMaxVars") ||
+      (inputs.size() != 3 && node_def.op() == "QuantizeAndDequantizeV2") ||
+      (inputs.size() != 4 && node_def.op() == "QuantizeAndDequantizeV3")) {
+    return tensorflow::errors::InvalidArgument(
+        "Invalid number of inputs for ", node_def.op(), ", at ",
+        node_def.name());
+  }
+  if (inputs.at(0).is_weights()) {
     // TensorRT will automatically quantize weights, so we will ignore ranges
     // for weights.
     params->outputs->push_back(inputs.at(0));
@@ -1994,21 +2005,24 @@ tensorflow::Status ConvertQuantize(OpConverterParams* params) {
   }
   float min_range = 0.0f;
   float max_range = 0.0f;
-  if (inputs.size() == 1) {
-    // Get ranges from attributes
+  if (node_def.op() == "FakeQuantWithMinMaxArgs") {
+    // Get ranges via node attributes.
     TFAttrs attrs(node_def);
     if (attrs.count("min") == 0 || attrs.count("max") == 0) {
       return tensorflow::errors::InvalidArgument(
-          "Min or max attribute not found for quantize, at ", node_def.name());
+          "Min or max attribute not found for ", node_def.op(), " at ",
+          node_def.name());
     }
     min_range = attrs.get<float>("min");
     max_range = attrs.get<float>("max");
-  } else if (inputs.size() == 3) {
-    // Get ranges from inputs
+  } else if (node_def.op() == "FakeQuantWithMinMaxVars" ||
+             node_def.op() == "QuantizeAndDequantizeV2" ||
+             node_def.op() == "QuantizeAndDequantizeV3") {
+    // Get ranges via inputs.
     if (!inputs.at(1).is_weights() || !inputs.at(2).is_weights()) {
       return tensorflow::errors::InvalidArgument(
-          "Min and max for quantize must be weights not tensors, at ",
-          node_def.name());
+          "Min and max inputs for ", node_def.op(),
+          " must be weights not tensors, at ", node_def.name());
     }
     // Min
     TRT_ShapedWeights weights_min = inputs.at(1).weights();
@@ -2022,7 +2036,8 @@ tensorflow::Status ConvertQuantize(OpConverterParams* params) {
     max_range = weights_max_ptr[0];
   } else {
     return tensorflow::errors::InvalidArgument(
-        "Expected 1 or 3 inputs for quantize node, at ", node_def.name());
+        "Unknown quantization op \"", node_def.op(), "\", at ",
+        node_def.name());
   }
   // Store ranges for tensor
   params->converter->ProvideQuantizationRange(
@@ -2047,6 +2062,16 @@ tensorflow::Status ConvertQuantize(OpConverterParams* params) {
 tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  if (inputs.size() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        "Invalid number of inputs for Relu6, at ",
+        node_def.name());
+  }
+  if (inputs.at(0).is_weights()) {
+    return tensorflow::errors::Unimplemented(
+        "Relu6 is only implemented for tensors, not weights, at ",
+        node_def.name());
+  }
   // ***************************************************************************
   // TensorRT does not implement Relu6 natively. This function converts Relu6 op
   // to available TensorRT ops: Relu6(x) = min(Relu(x), 6)
@@ -2075,8 +2100,9 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   // broadcast
   nvinfer1::Dims dims;
   dims.nbDims = relu_layer->getOutput(0)->getDimensions().nbDims;
-  for (int i = 0; i < dims.nbDims; i++)
+  for (int i = 0; i < dims.nbDims; i++) {
     dims.d[i] = 1;
+  }
   TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
       tensorflow::DataType::DT_FLOAT, dims);
   auto weights_ptr = static_cast<float*>(const_cast<void*>(
@@ -2091,7 +2117,7 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   // ElementWise Min Operation
   // Min op is a nop for INT8 execution path, as the input tensor
   // to this layer will only have values in range [0.f, 6.0f].
-  const nvinfer1::ITensor* tensor_l  = relu_layer->getOutput(0);
+  const nvinfer1::ITensor* tensor_l = relu_layer->getOutput(0);
   const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0);
   nvinfer1::IElementWiseLayer* relu6_layer = 
       params->converter->network()->addElementWise(
@@ -2117,7 +2143,7 @@ tensorflow::Status ConvertScale(OpConverterParams* params) {
 
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TRT_ShapedWeights weights = inputs.at(1).weights();
-  if (params->converter->is_fp16()) {
+  if (params->converter->precision_mode() == FP16MODE) {
     weights = ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
   }
 
@@ -3082,8 +3108,6 @@ void Converter::RegisterOpConverters() {
   op_registry_["TopKV2"] = ConvertTopK;
   op_registry_["Relu6"] = ConvertRelu6;
 # if NV_TENSORRT_MAJOR >= 5
-  op_registry_["QuantizeV2"] = ConvertQuantize;
-  op_registry_["Dequantize"] = ConvertQuantize;
   op_registry_["QuantizeAndDequantizeV2"] = ConvertQuantize;
   op_registry_["QuantizeAndDequantizeV3"] = ConvertQuantize;
   op_registry_["FakeQuantWithMinMaxVars"] = ConvertQuantize;
@@ -3132,7 +3156,7 @@ tensorflow::Status ConvertGraphDefToEngine(
 
   // Build the network
   VLOG(1) << "Starting engine conversion ";
-  Converter converter(trt_network.get(), precision_mode == FP16MODE);
+  Converter converter(trt_network.get(), precision_mode, use_calibration);
   std::vector<std::pair<string, string>> output_tensors;
   // Graph nodes are already topologically sorted during construction
   for (const auto& node_def : gdef.node()) {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index ec9cea38d5..78749124a2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -395,7 +395,9 @@ class TrtNodeValidator {
 // Class to convert TF nodes to TRT network.
 class Converter {
  public:
-  Converter(nvinfer1::INetworkDefinition* trt_network, bool is_fp16);
+  Converter(nvinfer1::INetworkDefinition* trt_network,
+            int precision_mode,
+            bool use_calibration);
 
   //////////////////////////////////////////////////////////////////////////////
   // Methods used by the TRT engine builder to build a TRT network from a TF
@@ -425,8 +427,11 @@ class Converter {
   // to add TRT layers.
   nvinfer1::INetworkDefinition* network() { return trt_network_; }
 
-  // Is the converter operating in fp16 mode?
-  bool is_fp16() const { return is_fp16_; }
+  // What precision are we targeting?
+  int precision_mode() const { return precision_mode_; }
+
+  // Calibration will be or was previously performed on this network?
+  bool use_calibration() const { return use_calibration_; }
 
   // This should be called on the inputs and outputs of any layer we create
   // where we know that the quantization range does not change during that
@@ -480,7 +485,7 @@ class Converter {
   
   // Gets the min and max value in a TRT_ShapedWeights
   Status GetWeightRange(const TRT_ShapedWeights& weights,
-                        float* out_min, float* out_max);
+                        float* out_min, float* out_max) const;
 
   // Registered op converters by op type.
   std::unordered_map<string, OpConverter> op_registry_;
@@ -513,7 +518,9 @@ class Converter {
   std::vector<std::pair<nvinfer1::ITensor*,nvinfer1::ITensor*>>
       quantization_infer_;
 
-  const bool is_fp16_;
+  const int precision_mode_;
+
+  const bool use_calibration_;
 
   // Batch size of inputs to trt_network_ added by AddInputTensor(). During
   // network construction it will update this, use it to verify the batch
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 38df6995b6..77cf3e7ef0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -133,11 +133,12 @@ void ValidateWeights(const TRT_ShapedWeights& weights,
 // Fake ITensor implementation for testing purposes.
 class FakeITensor : public nvinfer1::ITensor {
  public:
-  FakeITensor() {}
+  FakeITensor() : dynamic_range_(0.0f) {}
 
-  FakeITensor(const nvinfer1::Dims& dims) : dims_(dims) {}
+  FakeITensor(const nvinfer1::Dims& dims) : dims_(dims), dynamic_range_(0.0f) {}
 
-  FakeITensor(const std::vector<int>& dims) : dims_(GetTestDims(dims)) {}
+  FakeITensor(const std::vector<int>& dims) : dims_(GetTestDims(dims)),
+      dynamic_range_(0.0f) {}
 
   void setName(const char* name) override { name_ = name; }
 
@@ -166,7 +167,12 @@ class FakeITensor : public nvinfer1::ITensor {
   }
 
 #if NV_TENSORRT_MAJOR >= 5
-  bool setDynamicRange(float min, float max) override {}
+  bool setDynamicRange(float min, float max) override {
+    dynamic_range_ = std::max(std::abs(min), std::abs(max));
+    return true;
+  }
+
+  float getDynamicRange() { return dynamic_range_; }
 #endif
 
  private:
@@ -174,6 +180,7 @@ class FakeITensor : public nvinfer1::ITensor {
   nvinfer1::Dims dims_;
   nvinfer1::DataType type_;
   nvinfer1::TensorLocation location_;
+  float dynamic_range_;
 };
 
 TEST(TRT_ShapedWeights_Test, Basic) {
@@ -405,7 +412,9 @@ class ConverterTest : public ::testing::Test {
   ConverterTest() {
     builder_.reset(nvinfer1::createInferBuilder(logger_));
     network_.reset(builder_->createNetwork());
-    converter_.reset(new Converter(network_.get(), /*fp16=*/false));
+    converter_.reset(new Converter(network_.get(),
+                                   /*precision_mode=*/FP32MODE,
+                                   /*use_calibration=*/false));
     weight_store_ = &converter_->weight_store_;
   }
 
@@ -432,10 +441,19 @@ class ConverterTest : public ::testing::Test {
     return converter_->GetInputs(node_def, inputs);
   }
 
+  Status GetWeightRange(const TRT_ShapedWeights& weights,
+                        float* out_min, float* out_max) const {
+    return converter_->GetWeightRange(weights, out_min, out_max);
+  }
+
+  void PropagateQuantizationRanges() {
+    converter_->PropagateQuantizationRanges();
+  }
+
   int batch_size() const { return converter_->batch_size_; }
 
-  std::unordered_map<nvinfer1::ITensor*, float>* GetQuantizationRanges() {
-    return &quantization_ranges_;
+  std::unordered_map<nvinfer1::ITensor*, float>& quantization_ranges() {
+    return converter_->quantization_ranges_;
   }
 
  private:
@@ -660,6 +678,82 @@ TEST_F(ConverterTest, AddAndGetTensorOrWeights) {
                "tensor/weights my_tensor already exist");
 }
 
+TEST_F(ConverterTest, GetWeightRange) {
+  TRT_ShapedWeights weights =
+      weight_store_->GetTempWeights(DT_FLOAT, GetTestDims({2, 3}));
+  const std::vector<float> values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+  memcpy(const_cast<void*>(weights.GetValues()), values.data(),
+         weights.size_bytes());
+
+  float out_min = 0.0f;
+  float out_max = 0.0f;
+  TF_EXPECT_OK(GetWeightRange(weights, &out_min, &out_max));
+  EXPECT_EQ(1.0f, out_min);
+  EXPECT_EQ(6.0f, out_max);
+}
+
+TEST_F(ConverterTest, ProvideQuantizationRange) {
+  FakeITensor fake_tensor;
+  // Assymetric range
+  converter_->ProvideQuantizationRange(&fake_tensor, 0.0f, 6.0f);
+  EXPECT_EQ(quantization_ranges()[&fake_tensor], 6.0f);
+  converter_->ProvideQuantizationRange(&fake_tensor, 1.0f, 6.0f);
+  EXPECT_EQ(quantization_ranges()[&fake_tensor], 6.0f);
+  converter_->ProvideQuantizationRange(&fake_tensor, -8.0f, 6.0f);
+  EXPECT_EQ(quantization_ranges()[&fake_tensor], 8.0f);
+  converter_->ProvideQuantizationRange(&fake_tensor, -8.123f, -6.123f);
+  EXPECT_EQ(quantization_ranges()[&fake_tensor], 8.123f);
+  // Symmetric range
+  converter_->ProvideQuantizationRange(&fake_tensor, -6.123f, 6.123f);
+  EXPECT_EQ(quantization_ranges()[&fake_tensor], 6.123f);
+}
+
+TEST_F(ConverterTest, ApplyQuantizationRanges) {
+  // input -> infer1 -> infer2 -> infer3
+  FakeITensor input;
+  FakeITensor infer_1;
+  FakeITensor infer_2;
+  FakeITensor infer_3;
+  FakeITensor not_infer;
+  converter_->ProvideQuantizationRange(&input, -5.0f, 5.0f);
+  converter_->ProvideQuantizationRange(&not_infer, -100.0f, 100.0f);
+  converter_->MarkQuantizationRangesAsInferrable(&input, &infer_1);
+  converter_->MarkQuantizationRangesAsInferrable(&infer_1, &infer_2);
+  converter_->MarkQuantizationRangesAsInferrable(&infer_2, &infer_3);
+
+  // Input range should be inferred along the chain and applied to tensors.
+  converter_->ApplyQuantizationRanges(/*warn_missing_ranges=*/false);
+#if NV_TENSORRT_MAJOR >= 5
+  EXPECT_EQ(input.getDynamicRange(), 5.0f);
+  EXPECT_EQ(infer_1.getDynamicRange(), 5.0f);
+  EXPECT_EQ(infer_2.getDynamicRange(), 5.0f);
+  EXPECT_EQ(infer_3.getDynamicRange(), 5.0f);
+  EXPECT_EQ(not_infer.getDynamicRange(), 100.0f);
+#endif
+}
+
+TEST_F(ConverterTest, PropagateQuantizationRanges) {
+  // input <-> infer1 <-> infer2 <-> infer3
+  FakeITensor input;
+  FakeITensor infer_1;
+  FakeITensor infer_2;
+  FakeITensor infer_3;
+  FakeITensor not_infer;
+  converter_->ProvideQuantizationRange(&input, -5.0f, 5.0f);
+  converter_->MarkQuantizationRangesAsInferrable(&input, &infer_1);
+  converter_->MarkQuantizationRangesAsInferrable(&infer_1, &infer_2);
+  converter_->MarkQuantizationRangesAsInferrable(&infer_3, &infer_2);
+
+  // Input range should be inferred along the chain.
+  PropagateQuantizationRanges();
+  auto ranges = quantization_ranges();
+  EXPECT_EQ(ranges[&input], 5.0f);
+  EXPECT_EQ(ranges[&infer_1], 5.0f);
+  EXPECT_EQ(ranges[&infer_2], 5.0f);
+  EXPECT_EQ(ranges[&infer_3], 5.0f);
+  EXPECT_EQ(ranges.count(&not_infer), 0);
+}
+
 // Class to test various op converters, using both a TrtNodeValidator and
 // Converter.
 class OpConverterTest : public ::testing::Test {
@@ -688,7 +782,9 @@ class OpConverterTest : public ::testing::Test {
 
     // Reset the validator and converter.
     validator_.reset(new TrtNodeValidator);
-    converter_.reset(new Converter(network_.get(), /*fp16=*/false));
+    converter_.reset(new Converter(network_.get(),
+                                   /*precision_mode=*/FP32MODE,
+                                   /*use_calibration=*/false));
 
     // Reset other related artifacts.
     scope_ = Scope::NewRootScope();
@@ -830,6 +926,11 @@ class OpConverterTest : public ::testing::Test {
     }
   }
 
+  // Expose quantization_ranges_ for tests
+  std::unordered_map<nvinfer1::ITensor*, float>& quantization_ranges() {
+    return converter_->quantization_ranges_;
+  }
+
   std::unique_ptr<Converter> converter_;
   std::unique_ptr<TrtNodeValidator> validator_;
 
@@ -1134,69 +1235,177 @@ TEST_F(OpConverterTest, ConvertMatMul) {
         EXPECT_THAT(output_data, ElementsAre(2, 3));
       }
     }
+  }
 }
 
-TEST_F(ConverterTest, ConvertQuantize) {
+TEST_F(OpConverterTest, ConvertQuantize) {
   {
     // Input list is empty, should fail.
-    NodeDef node_def =
-        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {});
-    ExpectStatus(converter_.ConvertNode(node_def), error::INVALID_ARGUMENT,
-                 "Expected 1 or 3 inputs for quantize node, at my_quantize");
+    NodeDef node_def = MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2",
+        {});
+    RunConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Invalid number of inputs for QuantizeAndDequantizeV2, at my_quantize");
   }
   {
-    // Missing attributes, should fail
-    converter_.Reset();
-    NodeDef node_def =
-        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {"input"});
+    // FakeQuantWithMinMaxArgs attributes are empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_quantize", "FakeQuantWithMinMaxArgs",
+        {"input"});
     AddTestTensor("input", {1, 2, 3});
-    ExpectStatus(converter_.ConvertNode(node_def), error::INVALID_ARGUMENT,
-        "Min or max attribute not found for quantize, at my_quantize");
+    RunConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Min or max attribute not found for FakeQuantWithMinMaxArgs "
+        "at my_quantize");
   }
   {
-    // All inputs are tensors, should fail
-    converter_.Reset();
-    NodeDef node_def = MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2",
-                                   {"input", "weights_min", "weights_max"});
+    // FakeQuantWithMinMaxArgs ranges set via attributes, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    ops::FakeQuantWithMinMaxArgs::Attrs quantize_attrs;
+    quantize_attrs.min_ = -6.0f;
+    quantize_attrs.max_ = 6.0f;
+    auto quantize =
+        ops::FakeQuantWithMinMaxArgs(s.WithOpName("my_quantize"), input, quantize_attrs);
+    const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
-    AddTestTensor("weights_min", {1});
-    AddTestTensor("weights_max", {1});
-    ExpectStatus(converter_.ConvertNode(node_def), error::INVALID_ARGUMENT,
-        "Min and max for quantize must be weights not tensors, at my_quantize");
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(ranges.count(output.tensor()), 1);
+    EXPECT_EQ(ranges[output.tensor()], 6.0f);
   }
   {
-    // Ranges set via attributes, ok.
-    converter_.Reset();
-    NodeDef node_def =
-        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {"input"});
+    // FakeQuantWithMinMaxVars ranges set via inputs, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto quantize = ops::FakeQuantWithMinMaxVars(s.WithOpName("my_quantize"),
+                                                 input,
+                                                 weights_min,
+                                                 weights_max);
+    const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
-    AttrValue attr_min;
-    attr_min.set_f(-6.0f);
-    AttrValue attr_max;
-    attr_max.set_f(6.0f);
-    node_def.mutable_attr()->insert({"min", attr_min});
-    node_def.mutable_attr()->insert({"max", attr_max});
-    TF_EXPECT_OK(converter_.ConvertNode(node_def));
-    TRT_TensorOrWeights output = converter_.GetTensorOrWeights("my_quantize");
+    AddTestWeights<float>("weights_min", {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", {1}, {6.0f}); 
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
     EXPECT_TRUE(output.is_tensor());
-    auto ranges = converter_.GetQuantizationRanges();
-    EXPECT_EQ((*ranges).count(output.tensor()), 1);
-    EXPECT_EQ((*ranges)[output.tensor()], 6.0f);
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(ranges.count(output.tensor()), 1);
+    EXPECT_EQ(ranges[output.tensor()], 6.0f);
   }
   {
-    // Ranges set via inputs, ok.
-    converter_.Reset();
-    NodeDef node_def = MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2",
-                                   {"input", "weights_min", "weights_max"});
+    // QuantizeAndDequantizeV2 ranges set via inputs, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto quantize = ops::QuantizeAndDequantizeV2(s.WithOpName("my_quantize"),
+                                                 input,
+                                                 weights_min,
+                                                 weights_max);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights_min", {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", {1}, {6.0f}); 
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(ranges.count(output.tensor()), 1);
+    EXPECT_EQ(ranges[output.tensor()], 6.0f);
+  }
+  {
+    // QuantizeAndDequantizeV3 ranges set via inputs, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto num_bits = ops::Placeholder(s.WithOpName("num_bits"), DT_INT32);
+    auto quantize = ops::QuantizeAndDequantizeV3(s.WithOpName("my_quantize"),
+                                                 input,
+                                                 weights_min,
+                                                 weights_max,
+                                                 num_bits);
+    const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
-    AddTestWeights<float>("weights_min", DT_FLOAT, {1}, {-6.0f});
-    AddTestWeights<float>("weights_max", DT_FLOAT, {1}, {6.0f}); 
-    TF_EXPECT_OK(converter_.ConvertNode(node_def));
-    TRT_TensorOrWeights output = converter_.GetTensorOrWeights("my_quantize");
+    AddTestWeights<float>("weights_min", {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", {1}, {6.0f}); 
+    AddTestWeights<int>("num_bits", {1}, {8}); 
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
     EXPECT_TRUE(output.is_tensor());
-    auto ranges = converter_.GetQuantizationRanges();
-    EXPECT_EQ((*ranges).count(output.tensor()), 1);
-    EXPECT_EQ((*ranges)[output.tensor()], 6.0f);
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(ranges.count(output.tensor()), 1);
+    EXPECT_EQ(ranges[output.tensor()], 6.0f);
+  }
+  {
+    // QuantizeAndDequantizeV2 Range inputs are tensors, should fail.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto quantize = ops::QuantizeAndDequantizeV2(s.WithOpName("my_quantize"),
+                                                 input,
+                                                 weights_min,
+                                                 weights_max);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestTensor("weights_min", {1});
+    AddTestTensor("weights_max", {1});
+    RunConversion(node_def, error::INVALID_ARGUMENT,
+        "Min and max inputs for QuantizeAndDequantizeV2 must be weights not "
+        "tensors, at my_quantize");
+  }
+}
+
+TEST_F(OpConverterTest, ConvertRelu6) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {});
+    RunConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Invalid number of inputs for Relu6, at my_relu6");
+  }
+
+  // Get the NodeDef for Relu6.
+  Scope s = Scope::NewRootScope();
+  auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+  auto relu6 = ops::Relu6(s.WithOpName("my_relu6"), input);
+  const NodeDef& node_def = relu6.operation.node()->def();
+  
+  {
+    // Clip tensor values and set quantization ranges, ok.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_relu6", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(ranges[output.tensor()], 6.0f);
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {-100, -1, 0, 3, 5, 9}, "my_relu6", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(0, 0, 0, 3, 5, 6));
+  }
+  {
+    // Input is weights, should fail.
+    Reset();
+    AddTestWeights<float>("input", {1, 2, 3}, {-100, -1, 0, 3, 5, 9});
+    RunConversion(node_def, error::UNIMPLEMENTED,
+        "Relu6 is only implemented for tensors, not weights, at my_relu6");
   }
 }
 
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 80acab9ea3..4f64b7a952 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -389,7 +389,7 @@ void ContractEdge(SimpleEdge* edge, SimpleGraph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::Graph* tf_graph,
-    const std::function<Status(const tensorflow::Node*, int)>& candidate_fn,
+    const std::function<Status(const tensorflow::Node*)>& candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& input_candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& output_candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
@@ -414,7 +414,7 @@ tensorflow::Status SegmentGraph(
               << " (excluded by segmenter option).";
       node = nullptr;
     } else {
-      const Status status = candidate_fn(node->tf_node(), options.precision_mode);
+      const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
         VLOG(1) << "Not a TF-TRT candidate: " << node->name() << ": " << status;
         node = nullptr;
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index 802daec6a6..b6ae1cf462 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -53,7 +53,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::Graph* tf_graph,
-    const std::function<Status(const tensorflow::Node*, int)>& candidate_fn,
+    const std::function<Status(const tensorflow::Node*)>& candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& input_candidate_fn,
     const std::function<bool(const tensorflow::Edge*)>& output_candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index e648c3388f..d160cb3380 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -1,9 +1,31 @@
-import tensorflow as tf
-import tensorflow.contrib.tensorrt as trt
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
 import numpy as np
-import argparse
+import os
+
+import tensorflow as tf
+from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph
+from tensorflow.core.protobuf import config_pb2 
+from tensorflow.python.keras.datasets import mnist
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
+from tensorflow.python import estimator as tf_estimator
+from tensorflow.python.estimator.estimator import Estimator
+from tensorflow.python.estimator.run_config import RunConfig
+from tensorflow.python.estimator.model_fn import ModeKeys, EstimatorSpec
 
 INPUT_NODE_NAME = 'input'
 OUTPUT_NODE_NAME = 'output'
@@ -67,7 +89,7 @@ def run(is_training, use_trt, batch_size, num_epochs, model_dir):
     model_dir: Where to save or load checkpoint.
   """
   # Get dataset
-  train, test = tf.keras.datasets.mnist.load_data()
+  train, test = mnist.load_data()
   
   def eval_input_fn():
     mnist_x, mnist_y = test
@@ -76,7 +98,6 @@ def run(is_training, use_trt, batch_size, num_epochs, model_dir):
         map_func=preprocess_fn,
         batch_size=batch_size,
         num_parallel_calls=8))
-    dataset = dataset.prefetch(buffer_size=tf.contrib.data.AUTOTUNE)
     dataset = dataset.repeat(count=1)
     iterator = dataset.make_one_shot_iterator()
     features, labels = iterator.get_next()
@@ -90,7 +111,6 @@ def run(is_training, use_trt, batch_size, num_epochs, model_dir):
         map_func=preprocess_fn,
         batch_size=batch_size,
         num_parallel_calls=8))
-    dataset = dataset.prefetch(buffer_size=tf.contrib.data.AUTOTUNE)
     dataset = dataset.repeat(count=num_epochs)
     iterator = dataset.make_one_shot_iterator()
     features, labels = iterator.get_next()
@@ -115,27 +135,27 @@ def run(is_training, use_trt, batch_size, num_epochs, model_dir):
         predictions=classes_out,
         name='acc_op')
     tf.summary.scalar('accuracy', accuracy[1])
-    if mode == tf.estimator.ModeKeys.EVAL:
-      return tf.estimator.EstimatorSpec(
+    if mode == ModeKeys.EVAL:
+      return EstimatorSpec(
           mode,
           loss=loss,
           eval_metric_ops={'accuracy': accuracy})
-    elif mode == tf.estimator.ModeKeys.TRAIN:
+    elif mode == ModeKeys.TRAIN:
       optimizer = tf.train.AdamOptimizer(learning_rate=1e-2)
       train_op = optimizer.minimize(
           loss,
           global_step=tf.train.get_global_step())
-      return tf.estimator.EstimatorSpec(
+      return EstimatorSpec(
           mode,
           loss=loss,
           train_op=train_op)
 
-  tf_config = tf.ConfigProto()
+  tf_config = config_pb2.ConfigProto()
   tf_config.gpu_options.allow_growth = True
-  estimator = tf.estimator.Estimator(
+  estimator = Estimator(
       model_fn=model_fn,
-      model_dir=model_dir,
-      config=tf.estimator.RunConfig(session_config=tf_config))
+      model_dir=None,
+      config=RunConfig(session_config=tf_config))
   if is_training:
     estimator.train(train_input_fn)
   results = estimator.evaluate(eval_input_fn)
@@ -163,7 +183,7 @@ def get_graph_def(use_trt, batch_size, model_dir):
   # Convert with TF-TRT
   if use_trt:
     print('nodes before:', len(graph_def.node))
-    graph_def = trt.create_inference_graph(graph_def,
+    graph_def = create_inference_graph(graph_def,
         outputs=[OUTPUT_NODE_NAME],
         max_batch_size=batch_size,
         precision_mode='int8',
@@ -180,16 +200,17 @@ def get_graph_def(use_trt, batch_size, model_dir):
 class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
 
   def testEval(self):
+    model_dir = test.test_src_dir_path('contrib/tensorrt/test/quantization_mnist_test_data')
     acc_tf = run(is_training=False,
         use_trt=False,
         batch_size=128,
         num_epochs=None,
-        model_dir='./quantization_mnist_test_data')['accuracy']
+        model_dir=model_dir)['accuracy']
     acc_tftrt = run(is_training=False,
         use_trt=True,
         batch_size=128,
         num_epochs=None,
-        model_dir='./quantization_mnist_test_data')['accuracy']
+        model_dir=model_dir)['accuracy']
     self.assertAllClose(acc_tf, 0.9717)
     self.assertAllClose(acc_tftrt, 0.9744)
 
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD
new file mode 100644
index 0000000000..f38c2c5f4d
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD
@@ -0,0 +1,14 @@
+# Checkpoint files for quantization MNIST test.
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+filegroup(
+    name = "quantization_mnist_data",
+    srcs = glob(
+        ["**"],
+    ),
+)
\ No newline at end of file
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001 b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.data-00000-of-00001
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
rename to tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.data-00000-of-00001
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.index
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index
rename to tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.index
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index a6f51640b7..8804f2bc8f 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -542,6 +542,8 @@ def _AddTests(test_class):
         # supported yet.
         continue
       if not dynamic_engine and use_calibration:
+        # Static engine with use_calibration=False will be static, so we want to
+        # test that. If use_calibration=True, only dynamic op is supported.
         # TODO(aaroey): construction of static calibration engine is not
         # supported yet.
         continue
-- 
GitLab


From 8068b6a021e75dc20f5de5902a684e94c3a4b5ef Mon Sep 17 00:00:00 2001
From: Ubuntu <pythonanonuser@gmail.com>
Date: Fri, 9 Nov 2018 21:54:37 +0000
Subject: [PATCH 0129/1554] Fixed typo

---
 tensorflow/python/kernel_tests/control_flow_ops_py_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 71e4ccddde..95e4c4ba14 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2086,7 +2086,7 @@ class ControlFlowTest(test.TestCase):
 
   def testWhileGradInCond(self):
 
-    with self.cached_session():
+    with self.cached_session() as sess:
       n = ops.convert_to_tensor(1.0, name="n")
       x = array_ops.placeholder(dtypes.float32, shape=None)
       c = lambda n: math_ops.less(n, 10.0)
-- 
GitLab


From 1bb3c61392634bf6a2b6ad5c1c3dc554204fc9fc Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 9 Nov 2018 17:00:11 -0800
Subject: [PATCH 0130/1554] parallel_for: add converter for MaxPool3D,
 MaxPool3DGrad, MaxPool3DGradGrad

---
 .../ops/parallel_for/control_flow_ops_test.py | 20 ++++++++++++++++++-
 tensorflow/python/ops/parallel_for/pfor.py    |  3 +++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 171369b724..d44966078c 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -794,11 +794,29 @@ class NNTest(PForTest):
   def test_max_pool(self):
     x = random_ops.random_uniform([3, 2, 12, 12, 3])
     ksize = [1, 3, 3, 1]
+    strides = [1, 2, 2, 1]
 
     def loop_fn(i):
       x1 = array_ops.gather(x, i)
       output = nn.max_pool(
-          x1, ksize, strides=[1, 2, 2, 1], padding="VALID", data_format="NHWC")
+          x1, ksize, strides=strides, padding="VALID", data_format="NHWC")
+      loss = nn.l2_loss(output)
+      ones = array_ops.ones_like(output)
+      grad = gradient_ops.gradients(loss, x1, grad_ys=ones)
+      grad_grad = gradient_ops.gradients(grad, ones)
+      return output, grad, grad_grad
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
+
+  def test_max_pool3D(self):
+    x = random_ops.random_uniform([3, 3, 2, 12, 12, 3])
+    ksize = [1, 1, 3, 3, 1]
+    strides = [1, 1, 2, 2, 1]
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      output = nn.max_pool3d(
+        x1, ksize, strides=strides, padding="VALID", data_format="NDHWC")
       loss = nn.l2_loss(output)
       ones = array_ops.ones_like(output)
       grad = gradient_ops.gradients(loss, x1, grad_ys=ones)
diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py
index e6f140a941..d789dc65b1 100644
--- a/tensorflow/python/ops/parallel_for/pfor.py
+++ b/tensorflow/python/ops/parallel_for/pfor.py
@@ -1303,7 +1303,10 @@ def _inputs_with_flattening(pfor_input, input_indices):
 @RegisterPForWithArgs("Conv2D", dims=[0])
 @RegisterPForWithArgs("AvgPool", dims=[0])
 @RegisterPForWithArgs("MaxPool", dims=[0])
+@RegisterPForWithArgs("MaxPool3D", dims=[0])
+@RegisterPForWithArgs("MaxPool3DGrad", dims=[0, 1, 2])
 @RegisterPForWithArgs("MaxPoolGrad", dims=[0, 1, 2])
+@RegisterPForWithArgs("MaxPool3DGradGrad", dims=[0, 1, 2])
 @RegisterPForWithArgs("MaxPoolGradGrad", dims=[0, 1, 2])
 @RegisterPForWithArgs("SoftmaxCrossEntropyWithLogits", dims=[0, 1])
 def _convert_flatten_batch(pfor_input, op_type, dims):
-- 
GitLab


From c806b163c4d52cf80daecf2d63e3b76c7dc696e6 Mon Sep 17 00:00:00 2001
From: Nutti <nutti.metro@gmail.com>
Date: Sat, 10 Nov 2018 10:41:09 +0900
Subject: [PATCH 0131/1554] OptimizationPass::POST_REWRITE_FOR_EXEC after
 Grappler optimization in PartitionedCallOp

---
 tensorflow/core/kernels/partitioned_function_ops.cc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index 71e506e5e6..72310f33ae 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -166,12 +166,6 @@ class PartitionedCallOp : public AsyncOpKernel {
             OptimizationPassRegistry::Global()->RunGrouping(
                 OptimizationPassRegistry::POST_PLACEMENT, optimization_options),
             done);
-        OP_REQUIRES_OK_ASYNC(
-            ctx,
-            OptimizationPassRegistry::Global()->RunGrouping(
-                OptimizationPassRegistry::POST_REWRITE_FOR_EXEC,
-                optimization_options),
-            done);
 
         Device* cpu_device;
         OP_REQUIRES_OK_ASYNC(
@@ -184,6 +178,13 @@ class PartitionedCallOp : public AsyncOpKernel {
                                            device_set, cpu_device, &graph),
                              done);
 
+        OP_REQUIRES_OK_ASYNC(
+            ctx,
+            OptimizationPassRegistry::Global()->RunGrouping(
+                OptimizationPassRegistry::POST_REWRITE_FOR_EXEC,
+                optimization_options),
+            done);
+
         std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
         OP_REQUIRES_OK_ASYNC(
             ctx, PartitionHelper(device_set, std::move(graph), &subgraphs),
-- 
GitLab


From 7d96d6fbd3cdbe215c9dce78f8227ef273b5d37a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 10 Nov 2018 20:28:31 +0000
Subject: [PATCH 0132/1554] Fix ValueError by image.transform in eager mode

This fix tries to address the issue raised in 23654 where
in eager mode tf.contrib.image.transform will throw out
```
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```

This fix addresses the issue. This fix fixes 23654.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/image/python/ops/image_ops.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py
index d4fb99a017..b25a6f7b57 100644
--- a/tensorflow/contrib/image/python/ops/image_ops.py
+++ b/tensorflow/contrib/image/python/ops/image_ops.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.contrib.image.ops import gen_image_ops
 from tensorflow.contrib.util import loader
 from tensorflow.python.framework import common_shapes
@@ -271,8 +272,11 @@ def transform(images,
       raise TypeError("Images should have rank between 2 and 4.")
 
     if output_shape is None:
-      output_shape = tensor_util.constant_value(
-          array_ops.shape(images)[1:3]) or array_ops.shape(images)[1:3]
+      output_shape = array_ops.shape(images)[1:3]
+      if not context.executing_eagerly():
+        output_shape_value = tensor_util.constant_value(output_shape)
+        if output_shape_value is not None:
+          output_shape = output_shape_value
 
     output_shape = ops.convert_to_tensor(
         output_shape, dtypes.int32, name="output_shape")
-- 
GitLab


From 38455b2e111fa1acef3dd7dd00517b3fc1f1c38f Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 10 Nov 2018 20:30:22 +0000
Subject: [PATCH 0133/1554] Add test case for image.transform in eager mode

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../contrib/image/python/kernel_tests/image_ops_test.py   | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
index 4997c31a7f..ebf8a8adb3 100644
--- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
+++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
@@ -281,6 +281,14 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
             value.eval(),
             np.array([[4, 4], [4, 4]]).astype(dtype.as_numpy_dtype()))
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_transform_eager(self):
+    image = constant_op.constant([[1., 2.], [3., 4.]])
+    value = image_ops.transform(image, [1] * 8)
+    with self.test_session(use_gpu=True):
+      self.assertAllEqual(
+        self.evaluate(value), np.array([[4, 4], [4, 4]]))
+
 
 class BipartiteMatchTest(test_util.TensorFlowTestCase):
 
-- 
GitLab


From c50685de2d680d7e76e4b586e14138f33272a9cb Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 10 Nov 2018 20:33:28 +0000
Subject: [PATCH 0134/1554] Pylint fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/image/python/kernel_tests/image_ops_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
index ebf8a8adb3..ba5cdfebf9 100644
--- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
+++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
@@ -286,8 +286,7 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
     image = constant_op.constant([[1., 2.], [3., 4.]])
     value = image_ops.transform(image, [1] * 8)
     with self.test_session(use_gpu=True):
-      self.assertAllEqual(
-        self.evaluate(value), np.array([[4, 4], [4, 4]]))
+      self.assertAllEqual(self.evaluate(value), np.array([[4, 4], [4, 4]]))
 
 
 class BipartiteMatchTest(test_util.TensorFlowTestCase):
-- 
GitLab


From ced3c110f02148bab77d06352ec0465fd87962c0 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Mon, 12 Nov 2018 10:30:55 +0800
Subject: [PATCH 0135/1554] Revert a change: accidently comment a few lines.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index fa32a3e061..ac3c817982 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -915,17 +915,17 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     if (node->type_string() != "Transpose") return false;
 
     // If "Transpose" has multiple output data edges, also don't fuse it.
-    // if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
+    if (node->num_outputs() > 1 || node->out_edges().size() > 1) return false;
 
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
     // Note: this constraint will eventually be removed, if we enabled this fusion for training
     // in the future. 
-    // for (const Edge* e : node->out_edges()) {
-    //   if (e->IsControlEdge()) {
-    //     return false;
-    //   }
-    // }
+    for (const Edge* e : node->out_edges()) {
+      if (e->IsControlEdge()) {
+        return false;
+      }
+    }
 
     // If "Transpose" has input control edges, don't fuse on it.
     for (const Edge* e : node->in_edges()) {
-- 
GitLab


From 17f1bdd9dda6f690df6fc298ed5c884e197a5d99 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Mon, 12 Nov 2018 13:08:50 +0800
Subject: [PATCH 0136/1554] Some modifications to "CheckForNodeFusion()": 1.
 check for match before push; 2. use "nodes" instead of "work_stack"; 3. get
 rid of "fi_ptr";

---
 tensorflow/core/graph/mkl_layout_pass.cc | 70 +++++++++---------------
 1 file changed, 27 insertions(+), 43 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index ac3c817982..a5d2008a37 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2725,70 +2725,54 @@ Status MklLayoutRewritePass::FuseNode(
 
 std::tuple<bool, std::vector<Node*>, const MklLayoutRewritePass::FusionInfo>
 MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
-  const FusionInfo* fi_ptr = nullptr;
+  // Stores matched nodes, in the same order as node_checkers.
+  std::vector<Node *> nodes;
 
   for (auto fi = finfo_.begin(); fi != finfo_.end(); ++fi) {
-    fi_ptr = &*fi;
     //
     // Make sure node "a" and its succeding nodes (b, c ...), match the pattern
     // defined in fusion info (ops[0], ops[1], ...),
-    // aka. "a->b->c" matches "op1->op2->op3"
+    // a.k.a. "a->b->c" matches "op1->op2->op3"
     //
 
-    std::stack<Node *, std::vector<Node *>> work_stack;
+    // Stores the first unvisted outgoing edge of each matched node in "nodes".
     std::stack<EdgeSet::const_iterator> current_neighbor_stack;
-    auto node_checker = fi->node_checkers.begin();
+    nodes.clear();
 
-    Node *current_node = nullptr;
-    if (a != nullptr) {
-      work_stack.push(a);
+    auto node_checker = fi->node_checkers.begin();
+    if (a != nullptr && (*node_checker)(a)) {
+      nodes.push_back(a);
       current_neighbor_stack.push(a->out_edges().begin());
+      ++ node_checker;
     }
 
-    while (!work_stack.empty()) {
-      current_node = work_stack.top();
+    while (!nodes.empty()) {
+      auto& current_neighbor_iter = current_neighbor_stack.top();
 
-      if ((*node_checker)(current_node)){
-        if (node_checker == (fi->node_checkers.end() - 1)) {
-          // We find a match, break and return.
-          std::vector<Node *> nodes;
-          while (!work_stack.empty()) {
-            nodes.insert(nodes.begin(), work_stack.top());
-            work_stack.pop();
-          }
-          
-          return make_tuple(true, nodes, *fi_ptr);
-        }
+      if (current_neighbor_iter != nodes.back()->out_edges().end()) {
+        // Found an unvisited edge. Goes through the edge to get the neighbor.
+        Node* neighbor_node = (*current_neighbor_iter)->dst();
+        ++current_neighbor_stack.top();  // Retrieves the next unvisited edge.
 
-        auto &current_neighbor_iter = current_neighbor_stack.top();
-        if (current_neighbor_iter == current_node->out_edges().end()) {
-            // All output edges have been exhausted, pop the stack
-            // and roll back to the preceding node.
-            work_stack.pop();
-            current_neighbor_stack.pop();
-            -- node_checker;
-        } else {
-            // Found a edge not been visited, go through this edge
-            // and get the next neighbor.
-            Node *neighbor_node = (*current_neighbor_iter)->dst();
-            work_stack.push(neighbor_node);
-            current_neighbor_stack.push(neighbor_node->out_edges().begin());
-            ++ node_checker;
-
-            // Increase current_neighbor_iter, which is at the top of stack.
-            ++ current_neighbor_iter;
+        if ((*node_checker)(neighbor_node)) {
+          // Found a match. Stores the node and moves to the next checker.
+          nodes.push_back(neighbor_node);
+          current_neighbor_stack.push(neighbor_node->out_edges().begin());
+          if (++node_checker == fi->node_checkers.end()) {
+            return make_tuple(true, nodes, *fi);
+          }
         }
       } else {
-        // current node doesn't match, pop stack to roll back.
-        // visited_nodes.insert(current_node);
-        work_stack.pop();
+        // Removes the current node since none of its neighbor leads to a
+        // further match.
+        nodes.pop_back();
         current_neighbor_stack.pop();
-        -- node_checker;
+        --node_checker;
       }
     }
   }
 
-  return make_tuple(false, std::vector<Node *>(), *fi_ptr);
+  return make_tuple(false, std::vector<Node *>(), FusionInfo());
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-- 
GitLab


From 9b90cbd5f4ff02160def3cffc34d56ceaa2da916 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Mon, 12 Nov 2018 13:18:09 +0800
Subject: [PATCH 0137/1554] Apply clang-format-3.8

---
 tensorflow/core/graph/mkl_layout_pass.cc | 29 +++++++++++++-----------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index a5d2008a37..d46e7165ff 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -22,12 +22,12 @@ limitations under the License.
 #include <memory>
 #include <queue>
 #include <set>
+#include <set>
+#include <stack>
 #include <tuple>
 #include <unordered_set>
 #include <utility>
 #include <vector>
-#include <stack>
-#include <set>
 
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
@@ -514,8 +514,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       csinfo_.conv2d_grad_filter_with_bias,
                       GetConv2DBackpropFilterOrBiasAddGrad});
 
-    // The fusion patterns in "finfo_" that show up first will get applied first,
-    // for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC, A->B->C->D to ABCD},
+    // The fusion patterns in "finfo_" that show up first will get applied
+    // first,
+    // for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC, A->B->C->D
+    // to ABCD},
     // since the first gets applied first, the final graph will be ABC->D.
 
     //
@@ -903,7 +905,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   Status FuseNode(std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
                   const MklLayoutRewritePass::FusionInfo fi);
 
-  // Fuse tranpose(to "NHWC") + mklop("NHWC") + transpose(to "NCHW") into mklop("NCHW").
+  // Fuse tranpose(to "NHWC") + mklop("NHWC") + transpose(to "NCHW") into
+  // mklop("NCHW").
   // Here "mklop" can be any MKL-DNN supported op, such as Conv2D.
   static Status FuseTransposeMklOpTranspose(
       std::unique_ptr<Graph>* g, std::vector<Node*>& nodes,
@@ -919,8 +922,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
 
     // Check if has out control edge. If true, this is a training graph.
     // Currently we focus on inference and do no fusion in training.
-    // Note: this constraint will eventually be removed, if we enabled this fusion for training
-    // in the future. 
+    // Note: this constraint will eventually be removed, if we enabled this
+    // fusion for training
+    // in the future.
     for (const Edge* e : node->out_edges()) {
       if (e->IsControlEdge()) {
         return false;
@@ -1835,7 +1839,6 @@ void MklLayoutRewritePass::CopyAttrsConv(const Node* orig_node, NodeBuilder* nb,
 
       new_strides = {strides[NHWC::dim::N], strides[NHWC::dim::C],
                      strides[NHWC::dim::H], strides[NHWC::dim::W]};
-      
 
       new_dilations = {dilations[NHWC::dim::N], dilations[NHWC::dim::C],
                        dilations[NHWC::dim::H], dilations[NHWC::dim::W]};
@@ -2698,8 +2701,8 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   for (const Edge* e : transpose_to_nchw->out_edges()) {
     if (!e->IsControlEdge()) {
       const int kTransposeWithMklOpOutputSlot = 0;
-      CHECK_NOTNULL((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot, e->dst(),
-                                  e->dst_input()));
+      CHECK_NOTNULL((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot,
+                                  e->dst(), e->dst_input()));
     }
   }
 
@@ -2726,7 +2729,7 @@ Status MklLayoutRewritePass::FuseNode(
 std::tuple<bool, std::vector<Node*>, const MklLayoutRewritePass::FusionInfo>
 MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
   // Stores matched nodes, in the same order as node_checkers.
-  std::vector<Node *> nodes;
+  std::vector<Node*> nodes;
 
   for (auto fi = finfo_.begin(); fi != finfo_.end(); ++fi) {
     //
@@ -2743,7 +2746,7 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
     if (a != nullptr && (*node_checker)(a)) {
       nodes.push_back(a);
       current_neighbor_stack.push(a->out_edges().begin());
-      ++ node_checker;
+      ++node_checker;
     }
 
     while (!nodes.empty()) {
@@ -2772,7 +2775,7 @@ MklLayoutRewritePass::CheckForNodeFusion(Node* a) const {
     }
   }
 
-  return make_tuple(false, std::vector<Node *>(), FusionInfo());
+  return make_tuple(false, std::vector<Node*>(), FusionInfo());
 }
 
 ///////////////////////////////////////////////////////////////////////////////
-- 
GitLab


From e9853febd5cf870523ddd99ef6e4fa45cd7047bd Mon Sep 17 00:00:00 2001
From: Anders Huss <andhus@kth.se>
Date: Mon, 12 Nov 2018 08:59:58 +0100
Subject: [PATCH 0138/1554] fix issue in expanding of mask in keras.backend.rnn

---
 tensorflow/python/keras/backend.py      |   6 +-
 tensorflow/python/keras/backend_test.py | 118 ++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 0be6302a98..017491ac76 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3337,9 +3337,9 @@ def rnn(step_function,
     assert not nest.is_sequence(input_t)
     rank_diff = len(input_t.shape) - len(mask_t.shape)
     for _ in range(rank_diff):
-      mask_t = array_ops.expand_dims(mask_t)
-    expand_dims = [1] + input_t.shape.as_list()[1:]
-    return array_ops.tile(mask_t, expand_dims)
+      mask_t = array_ops.expand_dims(mask_t, -1)
+    multiples = [1] + input_t.shape.as_list()[1:]
+    return array_ops.tile(mask_t, multiples)
 
   if unroll:
     if not time_steps:
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index d8aa3e9b52..edb2af7e01 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1223,6 +1223,124 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
       for s, u_s in zip(additional_state_list[2], additional_state_list[3]):
         self.assertAllClose(s, u_s, atol=1e-04)
 
+  def test_rnn_output_and_state_masking_independent(self):
+    num_samples = 2
+    num_timesteps = 4
+    state_and_io_size = 2
+    mask_last_num_timesteps = 2  # for second sample only
+
+    # a step function that just outputs inputs,
+    # but increments states +1 per timestep
+    def step_function(inputs, states):
+      return inputs, [s + 1 for s in states]
+
+    inputs_vals = np.random.random(
+        (num_samples, num_timesteps, state_and_io_size))
+    initial_state_vals = np.random.random((num_samples, state_and_io_size))
+    # masking of two last timesteps for second sample only
+    mask_vals = np.ones((num_samples, num_timesteps))
+    mask_vals[1, -mask_last_num_timesteps:] = 0
+
+    # outputs expected to be same as inputs for the first sample
+    expected_outputs = inputs_vals.copy()
+    # but for the second sample all outputs in masked region should be the same
+    # as last output before masked region
+    expected_outputs[1, -mask_last_num_timesteps:] = \
+        expected_outputs[1, -(mask_last_num_timesteps + 1)]
+
+    expected_state = initial_state_vals.copy()
+    # first state should be incremented for every timestep (no masking)
+    expected_state[0] += num_timesteps
+    # second state should not be incremented for last two timesteps
+    expected_state[1] += (num_timesteps - mask_last_num_timesteps)
+
+    # verify same expected output for `unroll=true/false`
+    inputs = keras.backend.variable(inputs_vals)
+    initial_states = [keras.backend.variable(initial_state_vals)]
+    mask = keras.backend.variable(mask_vals)
+    for unroll in [True, False]:
+      last_output, outputs, last_states = keras.backend.rnn(
+          step_function,
+          inputs,
+          initial_states,
+          mask=mask,
+          unroll=unroll,
+          input_length=num_timesteps if unroll else None)
+
+      self.assertAllClose(
+          keras.backend.eval(outputs), expected_outputs)
+      self.assertAllClose(
+          keras.backend.eval(last_states[0]), expected_state)
+
+  def test_rnn_output_num_dim_larger_than_2_masking(self):
+    num_samples = 3
+    num_timesteps = 4
+    num_features = 5
+
+    def step_function(inputs, states):
+      outputs = keras.backend.tile(keras.backend.expand_dims(inputs), [1, 1, 2])
+      return outputs, [keras.backend.identity(s) for s in states]
+      # Note: cannot just return states (which can be a problem) ->
+      # tensorflow/python/ops/resource_variable_ops.py", line 824, in set_shape
+      # NotImplementedError: ResourceVariable does not implement set_shape()
+
+    inputs_vals = np.random.random((num_samples, num_timesteps, num_features))
+    initial_state_vals = np.random.random((num_samples, 6))
+    mask_vals = np.ones((num_samples, num_timesteps))
+    mask_vals[-1, -1] = 0  # final timestep masked for last sample
+
+    expected_outputs = np.repeat(inputs_vals[..., None], repeats=2, axis=-1)
+    # for the last sample, the final timestep (in masked region) should be the
+    # same as the second to final output (before masked region)
+    expected_outputs[-1, -1] = expected_outputs[-1, -2]
+
+    inputs = keras.backend.variable(inputs_vals)
+    initial_states = [keras.backend.variable(initial_state_vals)]
+    mask = keras.backend.variable(mask_vals)
+    for unroll in [True, False]:
+      last_output, outputs, last_states = keras.backend.rnn(
+          step_function,
+          inputs,
+          initial_states,
+          mask=mask,
+          unroll=unroll,
+          input_length=num_timesteps if unroll else None)
+
+      self.assertAllClose(keras.backend.eval(outputs), expected_outputs)
+
+  def test_rnn_state_num_dim_larger_than_2_masking(self):
+    num_samples = 3
+    num_timesteps = 4
+
+    def step_function(inputs, states):
+        return inputs, [s + 1 for s in states]
+
+    inputs_vals = np.random.random((num_samples, num_timesteps, 5))
+    initial_state_vals = np.random.random((num_samples, 6, 7))
+    mask_vals = np.ones((num_samples, num_timesteps))
+    mask_vals[0, -2:] = 0  # final two timesteps masked for first sample
+
+    expected_last_state = initial_state_vals.copy()
+    expected_last_state[0] += (num_timesteps - 2)
+    expected_last_state[1:] += num_timesteps
+
+    inputs = keras.backend.variable(inputs_vals)
+    initial_states = [keras.backend.variable(initial_state_vals)]
+    mask = keras.backend.variable(mask_vals)
+    for unroll in [True, False]:
+      last_output, outputs, last_states = keras.backend.rnn(
+          step_function,
+          inputs,
+          initial_states,
+          mask=mask,
+          unroll=unroll,
+          input_length=num_timesteps if unroll else None)
+
+      # not updated last timestep:
+      self.assertAllClose(
+          keras.backend.eval(last_states[0]),
+          expected_last_state)
+
   def test_normalize_batch_in_training(self):
     val = np.random.random((10, 3, 10, 10))
     x = keras.backend.variable(val)
-- 
GitLab


From fc541f794a27918ca6233734ab1f443489ddab5f Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Mon, 12 Nov 2018 21:40:44 -0800
Subject: [PATCH 0139/1554] Fix the coding style issue

---
 tensorflow/python/ops/parallel_for/control_flow_ops_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index d44966078c..7d13014687 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -808,7 +808,7 @@ class NNTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
 
-  def test_max_pool3D(self):
+  def test_max_pool3d(self):
     x = random_ops.random_uniform([3, 3, 2, 12, 12, 3])
     ksize = [1, 1, 3, 3, 1]
     strides = [1, 1, 2, 2, 1]
@@ -816,7 +816,7 @@ class NNTest(PForTest):
     def loop_fn(i):
       x1 = array_ops.gather(x, i)
       output = nn.max_pool3d(
-        x1, ksize, strides=strides, padding="VALID", data_format="NDHWC")
+          x1, ksize, strides=strides, padding="VALID", data_format="NDHWC")
       loss = nn.l2_loss(output)
       ones = array_ops.ones_like(output)
       grad = gradient_ops.gradients(loss, x1, grad_ys=ones)
-- 
GitLab


From b8417cde632f191a6a86c61bec7b828371d6fe62 Mon Sep 17 00:00:00 2001
From: NEWPLAN <enwplan001@163.com>
Date: Tue, 13 Nov 2018 15:15:42 +0800
Subject: [PATCH 0140/1554] fix spelling error

fix spelling error for documents.
---
 tensorflow/python/client/session.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 06c66dda9f..22303e95c1 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -828,7 +828,7 @@ class BaseSession(SessionInterface):
     nested list, tuple, namedtuple, dict, or OrderedDict containing graph
     elements at its leaves.  A graph element can be one of the following types:
 
-    * An `tf.Operation`.
+    * A `tf.Operation`.
       The corresponding fetched value will be `None`.
     * A `tf.Tensor`.
       The corresponding fetched value will be a numpy ndarray containing the
-- 
GitLab


From 8cbd3301ec04145e7b90118ae5e0c44c75fd5c06 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 13 Nov 2018 07:34:20 -0800
Subject: [PATCH 0141/1554] Adding a priority option to kernels in order to be
 able to override Device selection prioritization. For the regular case, we
 always prefer to place on the GPU but for tf.data pipelines when no device is
 specified, we'd like to place it on the CPU actually since that is what the
 user usually intends.

As we extend parts of the tf.data pipeline to run on GPU's / TPU's etc., this will become more and more necessary.

PiperOrigin-RevId: 221264064
---
 .../core/common_runtime/eager/execute.cc      |   4 +-
 tensorflow/core/common_runtime/placer.cc      | 146 ++++++++--
 tensorflow/core/common_runtime/placer_test.cc | 252 ++++++++++++++++++
 tensorflow/core/framework/kernel_def.proto    |   5 +
 .../core/framework/kernel_def_builder.cc      |   5 +
 .../core/framework/kernel_def_builder.h       |   3 +
 tensorflow/core/framework/op_kernel.cc        |  15 +-
 tensorflow/core/framework/op_kernel.h         |   2 +-
 tensorflow/core/framework/op_kernel_test.cc   |  71 +++--
 tensorflow/core/framework/types.h             |   2 +
 10 files changed, 451 insertions(+), 54 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 3cdda3ed75..a708033c65 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -192,7 +192,7 @@ Status ValidateInputTypeAndPlacement(EagerContext* ctx, Device* op_device,
 }
 
 Status SelectDevice(const NodeDef& ndef, EagerContext* ctx, Device** device) {
-  DeviceTypeVector final_devices;
+  PrioritizedDeviceTypeVector final_devices;
   TF_RETURN_IF_ERROR(SupportedDeviceTypesForNode(
       ctx->prioritized_device_type_list(), ndef, &final_devices));
   if (final_devices.empty()) {
@@ -202,7 +202,7 @@ Status SelectDevice(const NodeDef& ndef, EagerContext* ctx, Device** device) {
                             " :\n", KernelsRegisteredForOp(ndef.op()));
   }
   for (Device* d : *ctx->devices()) {
-    if (d->device_type() == final_devices[0].type_string()) {
+    if (d->device_type() == final_devices[0].first.type_string()) {
       *device = d;
       return Status::OK();
     }
diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index f8d933b45e..01e4072f60 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -47,42 +47,51 @@ const StringPiece kColocationGroupPrefixStringPiece(kColocationGroupPrefix);
 // returned list is sorted by preferred type (higher numeric type is preferred).
 std::vector<Device*> FilterSupportedDevices(
     const std::vector<Device*>& devices,
-    const DeviceTypeVector& supported_device_types,
+    const PrioritizedDeviceTypeVector& supported_device_types,
     const Device* default_device) {
   Device* filtered_default_device = nullptr;
-  std::vector<Device*> filtered_devices;
-  for (const DeviceType& d : supported_device_types) {
+  std::vector<std::pair<Device*, int32>> prioritized_filtered_devices;
+  for (const auto& supported_device_type : supported_device_types) {
     for (Device* device : devices) {
-      if (DeviceType(device->attributes().device_type()) == d) {
+      if (DeviceType(device->attributes().device_type()) ==
+          supported_device_type.first) {
         if (device == default_device) {
           filtered_default_device = device;
         } else {
-          filtered_devices.emplace_back(device);
+          prioritized_filtered_devices.emplace_back(
+              device, supported_device_type.second);
         }
       }
     }
   }
 
-  auto device_sort = [](const Device* a, const Device* b) {
-    auto a_priority = DeviceSet::DeviceTypeOrder(DeviceType(a->device_type()));
-    auto b_priority = DeviceSet::DeviceTypeOrder(DeviceType(b->device_type()));
+  auto device_sort = [](const std::pair<Device*, int32>& a,
+                        const std::pair<Device*, int32>& b) {
+    if (a.second != b.second) {
+      return a.second > b.second;
+    }
+
+    auto a_priority =
+        DeviceSet::DeviceTypeOrder(DeviceType(a.first->device_type()));
+    auto b_priority =
+        DeviceSet::DeviceTypeOrder(DeviceType(b.first->device_type()));
     // First sort by prioritized device type (higher is preferred) and
     // then by device name (lexicographically).
     if (a_priority != b_priority) {
       return a_priority > b_priority;
     }
-    return StringPiece(a->name()) < StringPiece(b->name());
+    return StringPiece(a.first->name()) < StringPiece(b.first->name());
   };
-  std::vector<Device*>::iterator sort_start;
+  std::sort(prioritized_filtered_devices.begin(),
+            prioritized_filtered_devices.end(), device_sort);
+
+  std::vector<Device*> filtered_devices;
   if (filtered_default_device != nullptr) {
-    // Put the default device first outside of the normal ordering.
     filtered_devices.emplace_back(filtered_default_device);
-    std::iter_swap(filtered_devices.begin(), std::prev(filtered_devices.end()));
-    sort_start = std::next(filtered_devices.begin());
-  } else {
-    sort_start = filtered_devices.begin();
   }
-  std::sort(sort_start, filtered_devices.end(), device_sort);
+  for (const auto& prioritized_filtered_device : prioritized_filtered_devices) {
+    filtered_devices.push_back(prioritized_filtered_device.first);
+  }
   return filtered_devices;
 }
 
@@ -472,7 +481,7 @@ class ColocationGraph {
     // The intersection of all device types supported by this node,
     // and those of all of its children, in priority order
     // of the preferred device.
-    DeviceTypeVector supported_device_types;
+    PrioritizedDeviceTypeVector supported_device_types;
 
     // The merged form of the device requested for this node, with
     // those of all of its children.
@@ -511,8 +520,8 @@ class ColocationGraph {
       const string& op_type = node->type_string();
       string devices_registered;
       for (const auto& device_type : members_[id].supported_device_types) {
-        strings::StrAppend(&devices_registered, DeviceTypeString(device_type),
-                           " ");
+        strings::StrAppend(&devices_registered,
+                           DeviceTypeString(device_type.first), " ");
       }
 
       type_to_devices[op_type] = std::move(devices_registered);
@@ -565,8 +574,9 @@ class ColocationGraph {
                                 "' does not match any device");
       }
 
-      for (const DeviceType& d : member->supported_device_types) {
-        if (DeviceType(assigned_device->attributes().device_type()) == d) {
+      for (const auto& d : member->supported_device_types) {
+        if (DeviceType(assigned_device->attributes().device_type()) ==
+            d.first) {
           return Status::OK();
         }
       }
@@ -623,24 +633,102 @@ class ColocationGraph {
     return Status::OK();
   }
 
+  static bool HasPriorities(const PrioritizedDeviceTypeVector& device_types) {
+    for (const auto& prioritized_device_type : device_types) {
+      if (prioritized_device_type.second != 0) return true;
+    }
+    return false;
+  }
+
+  static bool ArePrioritiesSame(const PrioritizedDeviceTypeVector& a_types,
+                                const PrioritizedDeviceTypeVector& b_types) {
+    if (a_types.size() != b_types.size()) {
+      return false;
+    }
+    for (int i = 0; i < a_types.size(); ++i) {
+      if (a_types[i].first != b_types[i].first) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   // Updates target to contain the intersection of the device types in
   // "target" and "other".
-  static void MergeSupportedDevices(DeviceTypeVector* target,
-                                    const DeviceTypeVector& other) {
-    DeviceTypeVector temp = *target;
+  static void MergeSupportedDevices(PrioritizedDeviceTypeVector* target,
+                                    const PrioritizedDeviceTypeVector& other) {
+    PrioritizedDeviceTypeVector temp = *target;
     target->clear();
 
-    // Iterate in priority order.
-    for (const DeviceType& device_type : temp) {
+    // Generate intersection with priorities.
+    PrioritizedDeviceTypeVector target_intersection;
+    PrioritizedDeviceTypeVector other_intersection;
+    for (const auto& prioritized_device_type : temp) {
       bool found = false;
-      for (const DeviceType& other_device_type : other) {
-        if (device_type == other_device_type) {
+      for (const auto& other_prioritized_device_type : other) {
+        if (prioritized_device_type.first ==
+            other_prioritized_device_type.first) {
           found = true;
+          other_intersection.push_back(other_prioritized_device_type);
           break;
         }
       }
       if (found) {
-        target->push_back(device_type);
+        target_intersection.push_back(prioritized_device_type);
+      }
+    }
+
+    // Sort the devices by priority order.
+    auto device_sort = [](const std::pair<DeviceType, int32>& a,
+                          const std::pair<DeviceType, int32>& b) {
+      // First look at set priorities.
+      if (a.second != b.second) {
+        return a.second > b.second;
+      }
+      // Then fallback to default priorities.
+      auto a_priority = DeviceSet::DeviceTypeOrder(a.first);
+      auto b_priority = DeviceSet::DeviceTypeOrder(b.first);
+      if (a_priority != b_priority) {
+        return a_priority > b_priority;
+      }
+      // Finally just look at the Device type strings.
+      return a.first.type_string() < b.first.type_string();
+    };
+
+    std::sort(target_intersection.begin(), target_intersection.end(),
+              device_sort);
+    std::sort(other_intersection.begin(), other_intersection.end(),
+              device_sort);
+
+    bool is_target_prioritized = HasPriorities(target_intersection);
+    bool is_other_prioritized = HasPriorities(other_intersection);
+    // If neither are prioritized then we just return the original i.e. target
+    // prioritization.
+    if (!is_target_prioritized && !is_other_prioritized) {
+      *target = target_intersection;
+    }
+    // If only one is prioritized, then we respect priorities of that in the
+    // intersection.
+    if (is_target_prioritized && !is_other_prioritized) {
+      *target = target_intersection;
+    }
+    if (!is_target_prioritized && is_other_prioritized) {
+      *target = other_intersection;
+    }
+    // If both have priorities and agree then we go with that. If the
+    // prioritization order is different, then we just fallback to the default
+    // i.e. what the DeviceTypeOrder suggests. In that case, we also set the
+    // merged priorities to 0, so that downstream merges work correctly as well.
+    if (is_target_prioritized && is_other_prioritized) {
+      bool priorities_agree =
+          ArePrioritiesSame(target_intersection, other_intersection);
+      if (priorities_agree) {
+        *target = target_intersection;
+      } else {
+        for (const auto& prioritized_device : target_intersection) {
+          target->push_back(std::make_pair(prioritized_device.first, 0));
+        }
+        std::sort(target->begin(), target->end(), device_sort);
       }
     }
   }
diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc
index 69f1611c1d..009f905f10 100644
--- a/tensorflow/core/common_runtime/placer_test.cc
+++ b/tensorflow/core/common_runtime/placer_test.cc
@@ -164,6 +164,13 @@ REGISTER_KERNEL_BUILDER(Name("TestDeviceEnforce").Device("FakeGPU"), DummyOp);
 REGISTER_KERNEL_BUILDER(Name("Shape").Device("FakeCPU"), DummyOp);
 REGISTER_KERNEL_BUILDER(Name("Shape").Device("FakeGPU"), DummyOp);
 
+// Op that has kernels with device priorities specified.
+REGISTER_OP("TestDatasetOp").Input("a: float").Output("b: float");
+REGISTER_KERNEL_BUILDER(Name("TestDatasetOp").Device("FakeCPU").Priority(2),
+                        DummyOp);
+REGISTER_KERNEL_BUILDER(Name("TestDatasetOp").Device("FakeGPU").Priority(1),
+                        DummyOp);
+
 ////////////////////////////////////////////////////////////////////////////////
 //
 // A PlacerTest method has three phases:
@@ -285,6 +292,251 @@ TEST_F(PlacerTest, TestNoConstraints) {
   EXPECT_DEVICE_TYPE(g, "n2", "FakeGPU");
 }
 
+// Test that a graph with no constraints but using kernels that have a specified
+// device priority will successfully assign nodes to the device with higher
+// priority
+TEST_F(PlacerTest, TestNoConstraintsWithPrioritizedKernels) {
+  Graph g(OpRegistry::Global());
+  {  // Scope for temporary variables used to construct g.
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* input = ops::SourceOp("TestInput", b.opts().WithName("in"));
+    ops::UnaryOp("TestDatasetOp", ops::NodeOut(input, 0),
+                 b.opts().WithName("n1"));
+    ops::UnaryOp("TestDatasetOp", ops::NodeOut(input, 1),
+                 b.opts().WithName("n2"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "in", "FakeCPU");
+  EXPECT_DEVICE_TYPE(g, "n1", "FakeCPU");
+  EXPECT_DEVICE_TYPE(g, "n2", "FakeCPU");
+}
+
+TEST_F(PlacerTest, TestGPUInputIntoPrioritizedKernel) {
+  Graph g(OpRegistry::Global());
+  {
+    // Scope for temp variables used to construct g.
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* input = ops::SourceOp("TestGPUOutput", b.opts().WithName("in"));
+    ops::UnaryOp("TestDatasetOp", ops::NodeOut(input, 0),
+                 b.opts().WithName("n1"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "in", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "n1", "FakeCPU");
+}
+
+// Tests that a GPU kernel colocated with prioritized kernel respects it.
+TEST_F(PlacerTest, TestGPUInputColocatedWithPrioritizedKernel) {
+  Graph g(OpRegistry::Global());
+  {
+    // Scope for temp variables used to construct g.
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* input = ops::SourceOp("TestGPUOutput", b.opts().WithName("in"));
+    // We colocate n1 with in.
+    ops::UnaryOp("TestDatasetOp", ops::NodeOut(input, 0),
+                 b.opts().WithName("n1").WithAttr("_class", {"loc:@in"}));
+    // We don't colocate n2 with in.
+    ops::UnaryOp("TestDatasetOp", ops::NodeOut(input, 0),
+                 b.opts().WithName("n2"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "in", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "n1", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "n2", "FakeCPU");
+}
+
+REGISTER_OP("CreateDatasetCPU").Output("o: resource");
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetCPU").Device("FakeCPU"), DummyOp);
+
+REGISTER_OP("CreateDatasetSP").Output("o: resource");
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetSP").Device("FakeCPU").Priority(2),
+                        DummyOp);
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetSP").Device("FakeGPU").Priority(1),
+                        DummyOp);
+
+REGISTER_OP("CreateDatasetRP").Output("o: resource");
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetRP").Device("FakeCPU").Priority(1),
+                        DummyOp);
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetRP").Device("FakeGPU").Priority(2),
+                        DummyOp);
+
+REGISTER_OP("CreateDatasetNP").Output("o: resource");
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetNP").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("CreateDatasetNP").Device("FakeGPU"), DummyOp);
+
+REGISTER_OP("IteratorNP").Input("i: resource").Output("o: float");
+REGISTER_KERNEL_BUILDER(Name("IteratorNP").Device("FakeCPU"), DummyOp);
+REGISTER_KERNEL_BUILDER(Name("IteratorNP").Device("FakeGPU"), DummyOp);
+
+REGISTER_OP("IteratorSP").Input("i: resource").Output("o: float");
+REGISTER_KERNEL_BUILDER(Name("IteratorSP").Device("FakeCPU").Priority(2),
+                        DummyOp);
+REGISTER_KERNEL_BUILDER(Name("IteratorSP").Device("FakeGPU").Priority(1),
+                        DummyOp);
+
+REGISTER_OP("IteratorRP").Input("i: resource").Output("o: float");
+REGISTER_KERNEL_BUILDER(Name("IteratorRP").Device("FakeCPU").Priority(1),
+                        DummyOp);
+REGISTER_KERNEL_BUILDER(Name("IteratorRP").Device("FakeGPU").Priority(2),
+                        DummyOp);
+
+REGISTER_OP("IteratorGPU").Input("i: resource").Output("o: float");
+REGISTER_KERNEL_BUILDER(Name("IteratorGPU").Device("FakeGPU"), DummyOp);
+
+// Test reference edges with one node having prioritized kernels and the other
+// has no preference. We should respect priority here.
+TEST_F(PlacerTest, TestDSWithPriority) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetSP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorNP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeCPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeCPU");
+}
+
+// Test reference edges with one node having kernels with regular priority and
+// the other has no preference. We should respect priority here.
+TEST_F(PlacerTest, TestDSWithGPUPriority) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetRP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorNP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeGPU");
+}
+
+// Test reference edges with one node having prioritized kernels and the other
+// has no preference. We should respect priority here.
+TEST_F(PlacerTest, TestITWithPriority) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetNP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorSP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeCPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeCPU");
+}
+
+// Test reference edges with one node having kernels with regular priority and
+// the other has no preference. We should respect priority here.
+TEST_F(PlacerTest, TestITWithGPUPriority) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetNP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorRP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeGPU");
+}
+
+// Test reference edges with one node having prioritized kernels and other node
+// can only be placed on GPU. We should respect the constraint then.
+TEST_F(PlacerTest, TestITGPU) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetSP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorGPU", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeGPU");
+}
+
+// Test reference edges with one node having prioritized kernels and other node
+// can only be placed on CPU. We should respect the constraint then.
+TEST_F(PlacerTest, TestSimpleIteratorOnlyGPU) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetCPU", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorRP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeCPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeCPU");
+}
+
+// Test constraints with agreeing priorities.
+TEST_F(PlacerTest, TestAgreeingPriorities) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetSP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorSP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeCPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeCPU");
+}
+
+// Test constraints with agreeing regular priorities.
+TEST_F(PlacerTest, TestAgreeingRegularPriorities) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetRP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorRP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeGPU");
+}
+
+// Test constraints with different priorities. In this case, we should bail
+// and just revert to default.
+TEST_F(PlacerTest, TestConflictingPriorities) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetSP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorRP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeGPU");
+}
+
+// Test constraints with different priorities. In this case, we should bail
+// and just revert to default.
+TEST_F(PlacerTest, TestConflictingPrioritiesReversed) {
+  Graph g(OpRegistry::Global());
+  {
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    Node* ds = ops::SourceOp("CreateDatasetRP", b.opts().WithName("ds"));
+    ops::UnaryOp("IteratorSP", ops::NodeOut(ds, 0), b.opts().WithName("it"));
+    TF_EXPECT_OK(BuildGraph(b, &g));
+  }
+  TF_EXPECT_OK(Place(&g));
+  EXPECT_DEVICE_TYPE(g, "ds", "FakeGPU");
+  EXPECT_DEVICE_TYPE(g, "it", "FakeGPU");
+}
+
 // Test that a graph with device type and reference constraints on
 // some of the ops will successfully assign nodes to the constrained
 // device, and colocate nodes with reference connections.
diff --git a/tensorflow/core/framework/kernel_def.proto b/tensorflow/core/framework/kernel_def.proto
index e16c2ae73b..358621dc0f 100644
--- a/tensorflow/core/framework/kernel_def.proto
+++ b/tensorflow/core/framework/kernel_def.proto
@@ -33,6 +33,11 @@ message KernelDef {
   // won't be used unless the user specifies a "_kernel" attr with
   // value matching this.
   string label = 5;
+
+  // Prioritization of kernel amongst different devices. By default we assume
+  // priority is 0. The higher the priority the better. By default (i.e. if
+  // this is not set), we prefer GPU kernels over CPU.
+  int32 priority = 6;
 }
 
 // A collection of KernelDefs
diff --git a/tensorflow/core/framework/kernel_def_builder.cc b/tensorflow/core/framework/kernel_def_builder.cc
index eb86f18ff0..fcacc3bebb 100644
--- a/tensorflow/core/framework/kernel_def_builder.cc
+++ b/tensorflow/core/framework/kernel_def_builder.cc
@@ -66,6 +66,11 @@ KernelDefBuilder& KernelDefBuilder::Label(const char* label) {
   return *this;
 }
 
+KernelDefBuilder& KernelDefBuilder::Priority(int32 priority) {
+  kernel_def_->set_priority(priority);
+  return *this;
+}
+
 const KernelDef* KernelDefBuilder::Build() {
   KernelDef* r = kernel_def_;
   kernel_def_ = nullptr;
diff --git a/tensorflow/core/framework/kernel_def_builder.h b/tensorflow/core/framework/kernel_def_builder.h
index 32dd21f94e..d74453cf60 100644
--- a/tensorflow/core/framework/kernel_def_builder.h
+++ b/tensorflow/core/framework/kernel_def_builder.h
@@ -64,6 +64,9 @@ class KernelDefBuilder {
   // "_kernel" attr.  May only be specified once.  Returns *this.
   KernelDefBuilder& Label(const char* label);
 
+  // Specify a priority number for this kernel.
+  KernelDefBuilder& Priority(int32 priority);
+
   // Returns a pointer to a KernelDef with fields set based on the
   // above calls to this instance.
   // Caller takes ownership of the result.
diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index fe71196979..e2a177569d 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -1091,7 +1091,7 @@ Status FindKernelDef(const DeviceType& device_type, const NodeDef& node_def,
 
 Status SupportedDeviceTypesForNode(
     const std::vector<DeviceType>& prioritized_types, const NodeDef& def,
-    DeviceTypeVector* device_types) {
+    PrioritizedDeviceTypeVector* prioritized_device_types) {
   // TODO(zhifengc): Changes the callers (SimplePlacer and
   // DynamicPlacer) to consider the possibility that 'def' is call to
   // a user-defined function and only calls this
@@ -1104,12 +1104,21 @@ Status SupportedDeviceTypesForNode(
       bool was_attr_mismatch;
       TF_RETURN_IF_ERROR(
           FindKernelRegistration(device_type, def, &reg, &was_attr_mismatch));
-      if (reg != nullptr) device_types->push_back(device_type);
+      if (reg != nullptr) {
+        int32 priority = reg->def.priority();
+        prioritized_device_types->emplace_back(device_type, priority);
+      }
     }
+    std::sort(prioritized_device_types->begin(),
+              prioritized_device_types->end(),
+              [](const std::pair<DeviceType, int32>& a,
+                 const std::pair<DeviceType, int32>& b) {
+                return a.second > b.second;
+              });
   } else {
     // Assumes that all device types support this node.
     for (const DeviceType& device_type : prioritized_types) {
-      device_types->push_back(device_type);
+      prioritized_device_types->push_back(std::make_pair(device_type, 0));
     }
   }
   return Status::OK();
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index 37a437136b..9f4c57e880 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -1237,7 +1237,7 @@ Status CreateOpKernel(DeviceType device_type, DeviceBase* device,
 //           * def has all attrs specified (e.g. using AddDefaultsToNodeDef()).
 Status SupportedDeviceTypesForNode(
     const std::vector<DeviceType>& prioritized_types, const NodeDef& def,
-    DeviceTypeVector* device_types);
+    PrioritizedDeviceTypeVector* device_types);
 
 // Returns a message with a description of the kernels registered for op
 // `op_name`.
diff --git a/tensorflow/core/framework/op_kernel_test.cc b/tensorflow/core/framework/op_kernel_test.cc
index 83dda6579b..d8001cd071 100644
--- a/tensorflow/core/framework/op_kernel_test.cc
+++ b/tensorflow/core/framework/op_kernel_test.cc
@@ -102,6 +102,27 @@ REGISTER_OP("Test4").Input("i: float").Output("o: float");
 REGISTER_KERNEL_BUILDER(Name("Test4").Device(DEVICE_CPU), DummyKernel);
 REGISTER_KERNEL_BUILDER(Name("Test4").Device(DEVICE_GPU), DummyKernel);
 
+// Kernels with different priorities.
+REGISTER_OP("Test5").Input("a: T").Input("b: T").Attr("T: type");
+
+class TestOp5Cpu : public tensorflow::OpKernel {
+ public:
+  explicit TestOp5Cpu(OpKernelConstruction* context) : OpKernel(context) {}
+  void Compute(OpKernelContext* context) override {}
+};
+
+REGISTER_KERNEL_BUILDER(Name("Test5").Device(DEVICE_CPU).Priority(2),
+                        TestOp5Cpu);
+
+class TestOp5Gpu : public tensorflow::OpKernel {
+ public:
+  explicit TestOp5Gpu(OpKernelConstruction* context) : OpKernel(context) {}
+  void Compute(OpKernelContext* context) override {}
+};
+
+REGISTER_KERNEL_BUILDER(Name("Test5").Device(DEVICE_GPU).Priority(1),
+                        TestOp5Gpu);
+
 static std::vector<DeviceType> DeviceTypes() {
   return {DeviceType(DEVICE_GPU), DeviceType(DEVICE_CPU)};
 }
@@ -185,10 +206,10 @@ TEST_F(OpKernelTest, SuccessBothCpuAndGpu) {
 
 TEST_F(OpKernelTest, CpuTypeRegistered) {
   NodeDef ndef = CreateNodeDef("Test1", {DT_FLOAT, DT_INT32});
-  DeviceTypeVector devs;
+  PrioritizedDeviceTypeVector devs;
   TF_ASSERT_OK(SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs));
   EXPECT_EQ(1, devs.size());
-  EXPECT_EQ(DeviceType(DEVICE_CPU), devs[0]);
+  EXPECT_EQ(DeviceType(DEVICE_CPU), devs[0].first);
 }
 
 TEST_F(OpKernelTest, CpuAndGpuTypeRegistered) {
@@ -196,24 +217,24 @@ TEST_F(OpKernelTest, CpuAndGpuTypeRegistered) {
     // Try a node def of an op that is registered for a specific type
     // only on CPU.
     NodeDef ndef = CreateNodeDef("Test3", {DT_INT8, DT_INT8});
-    DeviceTypeVector devs;
+    PrioritizedDeviceTypeVector devs;
     TF_ASSERT_OK(SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs));
     EXPECT_EQ(1, devs.size());
-    EXPECT_EQ(DeviceType(DEVICE_CPU), devs[0]);
+    EXPECT_EQ(DeviceType(DEVICE_CPU), devs[0].first);
   }
   {
     // Try a node def of an op that is registered for a specific type
     // only on GPU.
     NodeDef ndef = CreateNodeDef("Test3", {DT_FLOAT, DT_FLOAT});
-    DeviceTypeVector devs;
+    PrioritizedDeviceTypeVector devs;
     TF_ASSERT_OK(SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs));
     EXPECT_EQ(1, devs.size());
-    EXPECT_EQ(DeviceType(DEVICE_GPU), devs[0]);
+    EXPECT_EQ(DeviceType(DEVICE_GPU), devs[0].first);
   }
   {
     // Try a node def of an op that is only registered for other types.
     NodeDef ndef = CreateNodeDef("Test3", {DT_STRING, DT_STRING});
-    DeviceTypeVector devs;
+    PrioritizedDeviceTypeVector devs;
     TF_ASSERT_OK(SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs));
     EXPECT_EQ(0, devs.size());
   }
@@ -221,11 +242,23 @@ TEST_F(OpKernelTest, CpuAndGpuTypeRegistered) {
   {
     // Try a node def of an op that is registered for both.
     NodeDef ndef = CreateNodeDef("Test4", {DT_FLOAT});
-    DeviceTypeVector devs;
+    PrioritizedDeviceTypeVector devs;
+    TF_ASSERT_OK(SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs));
+    EXPECT_EQ(2, devs.size());
+    EXPECT_EQ(DeviceType(DEVICE_GPU), devs[0].first);
+    EXPECT_EQ(DeviceType(DEVICE_CPU), devs[1].first);
+  }
+
+  {
+    // Try a node def of an op where kernels have priorities.
+    NodeDef ndef = CreateNodeDef("Test5", {DT_STRING, DT_STRING});
+    PrioritizedDeviceTypeVector devs;
     TF_ASSERT_OK(SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs));
     EXPECT_EQ(2, devs.size());
-    EXPECT_EQ(DeviceType(DEVICE_GPU), devs[0]);
-    EXPECT_EQ(DeviceType(DEVICE_CPU), devs[1]);
+    EXPECT_EQ(DeviceType(DEVICE_CPU), devs[0].first);
+    EXPECT_EQ(2, devs[0].second);
+    EXPECT_EQ(DeviceType(DEVICE_GPU), devs[1].first);
+    EXPECT_EQ(1, devs[1].second);
   }
 }
 
@@ -412,11 +445,11 @@ class OpKernelBuilderTest : public ::testing::Test {
     }
 
     // Test SupportedDeviceTypesForNode()
-    DeviceTypeVector devices;
+    PrioritizedDeviceTypeVector devices;
     TF_EXPECT_OK(SupportedDeviceTypesForNode(DeviceTypes(), def, &devices));
     bool found = false;
-    for (const DeviceType& dt : devices) {
-      if (dt == device_type) {
+    for (const auto& dt : devices) {
+      if (dt.first == device_type) {
         found = true;
       }
     }
@@ -445,11 +478,11 @@ class OpKernelBuilderTest : public ::testing::Test {
       EXPECT_EQ(code, status.code());
 
       // Test SupportedDeviceTypesForNode().
-      DeviceTypeVector devices;
+      PrioritizedDeviceTypeVector devices;
       if (errors::IsNotFound(status)) {
         TF_EXPECT_OK(SupportedDeviceTypesForNode(DeviceTypes(), def, &devices));
-        for (const DeviceType& dt : devices) {
-          EXPECT_NE(dt, device_type);
+        for (const auto& dt : devices) {
+          EXPECT_NE(dt.first, device_type);
         }
       } else {
         Status status2 =
@@ -562,7 +595,7 @@ REGISTER_KERNEL_BUILDER(Name("DuplicateKernel").Device(DEVICE_CPU),
 
 TEST_F(OpKernelBuilderTest, DuplicateKernel) {
   const NodeDef ndef = CreateNodeDef("DuplicateKernel", {});
-  DeviceTypeVector devs;
+  PrioritizedDeviceTypeVector devs;
   Status status = SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs);
   ASSERT_FALSE(status.ok());
   EXPECT_TRUE(str_util::StrContains(
@@ -582,7 +615,7 @@ REGISTER_KERNEL_BUILDER(
 TEST_F(OpKernelBuilderTest, DuplicateKernelForT) {
   const NodeDef ndef =
       CreateNodeDef("DuplicateKernelForT", {"T|type|DT_FLOAT"});
-  DeviceTypeVector devs;
+  PrioritizedDeviceTypeVector devs;
   Status status = SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs);
   ASSERT_FALSE(status.ok());
   EXPECT_TRUE(str_util::StrContains(
@@ -603,7 +636,7 @@ REGISTER_KERNEL_BUILDER(Name("BadConstraint")
 
 TEST_F(OpKernelBuilderTest, BadConstraint) {
   const NodeDef ndef = CreateNodeDef("BadConstraint", {});
-  DeviceTypeVector devs;
+  PrioritizedDeviceTypeVector devs;
   Status status = SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs);
   ASSERT_FALSE(status.ok());
   EXPECT_TRUE(
diff --git a/tensorflow/core/framework/types.h b/tensorflow/core/framework/types.h
index a05dea19ec..c0df193342 100644
--- a/tensorflow/core/framework/types.h
+++ b/tensorflow/core/framework/types.h
@@ -104,6 +104,8 @@ typedef gtl::InlinedVector<DataType, 4> DataTypeVector;
 typedef gtl::ArraySlice<DataType> DataTypeSlice;
 
 typedef gtl::InlinedVector<DeviceType, 4> DeviceTypeVector;
+typedef gtl::InlinedVector<std::pair<DeviceType, int32>, 4>
+    PrioritizedDeviceTypeVector;
 
 // Convert the enums to strings for errors:
 string DataTypeString(DataType dtype);
-- 
GitLab


From a466bbdb048f66b678400b2c207768522bfc9a89 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 13 Nov 2018 07:42:12 -0800
Subject: [PATCH 0142/1554] [TF:XLA] Delete unused producer consumer queue
 class.

PiperOrigin-RevId: 221265027
---
 tensorflow/compiler/jit/BUILD                 |  19 ---
 .../compiler/jit/producer_consumer_queue.h    | 132 -----------------
 .../jit/producer_consumer_queue_test.cc       | 139 ------------------
 3 files changed, 290 deletions(-)
 delete mode 100644 tensorflow/compiler/jit/producer_consumer_queue.h
 delete mode 100644 tensorflow/compiler/jit/producer_consumer_queue_test.cc

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 162a137fa7..5f25e4626a 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -544,25 +544,6 @@ cc_library(
     hdrs = ["union_find.h"],
 )
 
-cc_library(
-    name = "producer_consumer_queue",
-    hdrs = ["producer_consumer_queue.h"],
-    deps = ["//tensorflow/core:lib"],
-)
-
-tf_cc_test(
-    name = "producer_consumer_queue_test",
-    size = "small",
-    srcs = ["producer_consumer_queue_test.cc"],
-    deps = [
-        ":producer_consumer_queue",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
 tf_cc_test(
     name = "deadness_analysis_test",
     size = "small",
diff --git a/tensorflow/compiler/jit/producer_consumer_queue.h b/tensorflow/compiler/jit/producer_consumer_queue.h
deleted file mode 100644
index 7c8c04152d..0000000000
--- a/tensorflow/compiler/jit/producer_consumer_queue.h
+++ /dev/null
@@ -1,132 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_JIT_PRODUCER_CONSUMER_QUEUE_H_
-#define TENSORFLOW_COMPILER_JIT_PRODUCER_CONSUMER_QUEUE_H_
-
-#include <deque>
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/mutex.h"
-
-namespace tensorflow {
-
-// A thread-safe, first-in-first-out queue.
-template <typename T>
-class ProducerConsumerQueue {
- public:
-  ProducerConsumerQueue()
-      : capacity_(std::numeric_limits<std::size_t>::max()) {}
-  ~ProducerConsumerQueue() = default;
-
-  // Wait until the queue is non-full, then append a copy of v.
-  void Put(const T &v);
-
-  // Wait until the queue is non-empty, then remove and return the head value.
-  T Get();
-
-  // If the queue is non-empty, remove the head value, placing it in *pv, and
-  // return true; otherwise return false.
-  bool TryGet(T *pv);
-
-  // Set the capacity of the queue; the queue is full whenever count() >=
-  // capacity().  The initial value is the maximum size_t.  Requires size > 0.
-  void set_capacity(std::size_t size);
-
-  // Return the capacity of the queue.
-  std::size_t capacity() const;
-
-  // Return the number of elements in the queue.
-  std::size_t count() const;
-
-  // Implementation details follow.  Clients should ignore.
- private:
-  mutable tensorflow::mutex mu_;  // protects all fields below
-  tensorflow::condition_variable non_empty_ GUARDED_BY(mu_);
-  tensorflow::condition_variable non_full_ GUARDED_BY(mu_);
-  std::size_t capacity_ GUARDED_BY(mu_);
-  std::deque<T> queue_ GUARDED_BY(mu_);
-
-  TF_DISALLOW_COPY_AND_ASSIGN(ProducerConsumerQueue);
-};
-
-// ------------------------------------------------------
-// Implementation details follow.  Clients should ignore.
-
-// Wait until the queue is non-full, then append a copy of v.
-template <typename T>
-void ProducerConsumerQueue<T>::Put(const T &v) {
-  mutex_lock lock(mu_);
-  while (queue_.size() >= capacity_) {
-    non_full_.wait(lock);
-  }
-  queue_.push_back(v);
-  non_empty_.notify_one();
-}
-
-// Wait until the queue is non-empty, then remove and return the head value.
-template <typename T>
-T ProducerConsumerQueue<T>::Get() {
-  mutex_lock lock(mu_);
-  while (queue_.empty()) {
-    non_empty_.wait(lock);
-  }
-  non_full_.notify_one();
-  T result_value = queue_.front();
-  queue_.pop_front();
-  return result_value;
-}
-
-// If the queue is non-empty, remove the head value, placing it in *pv, and
-// return true; otherwise return false.
-template <typename T>
-bool ProducerConsumerQueue<T>::TryGet(T *pv) {
-  mutex_lock lock(mu_);
-  bool got_element = !queue_.empty();
-  if (got_element) {
-    non_full_.notify_one();
-    *pv = queue_.front();
-    queue_.pop_front();
-  }
-  return got_element;
-}
-
-// Set the capacity of the queue; the queue is full whenever count() >=
-// capacity().  The initial value is the maximum size_t.  Requires size > 0.
-template <typename T>
-void ProducerConsumerQueue<T>::set_capacity(std::size_t size) {
-  mutex_lock lock(mu_);
-  CHECK_NE(size, 0);
-  capacity_ = size;
-  non_full_.notify_all();
-}
-
-// Return the capacity of the queue.
-template <typename T>
-std::size_t ProducerConsumerQueue<T>::capacity() const {
-  mutex_lock lock(mu_);
-  std::size_t max_elements = capacity_;
-  return max_elements;
-}
-
-// Return the number of elements in the queue.
-template <typename T>
-std::size_t ProducerConsumerQueue<T>::count() const {
-  mutex_lock lock(mu_);
-  std::size_t num_elements = queue_.size();
-  return num_elements;
-}
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_JIT_PRODUCER_CONSUMER_QUEUE_H_
diff --git a/tensorflow/compiler/jit/producer_consumer_queue_test.cc b/tensorflow/compiler/jit/producer_consumer_queue_test.cc
deleted file mode 100644
index f61260c6e5..0000000000
--- a/tensorflow/compiler/jit/producer_consumer_queue_test.cc
+++ /dev/null
@@ -1,139 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/jit/producer_consumer_queue.h"
-
-#include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-namespace {
-
-typedef ProducerConsumerQueue<int> IntQueue;
-
-// Insert integers between low inclusive and high exclusive into q.
-void PushRange(IntQueue *q, int low, int high) {
-  while (low != high) {
-    q->Put(low);
-    VLOG(2) << "Pushing " << low;
-    ++low;
-  }
-}
-
-// Push the numbers between 0 and 999 inclusive from several threads in the
-// pool.
-void PushRanges(IntQueue *queue, thread::ThreadPool *pool) {
-  VLOG(1) << "Adding 20-36";
-  pool->Schedule([queue] { PushRange(queue, 20, 36); });
-  VLOG(1) << "Adding 7-20";
-  pool->Schedule([queue] { PushRange(queue, 7, 20); });
-  VLOG(1) << "Adding 36-501";
-  pool->Schedule([queue] { PushRange(queue, 36, 501); });
-  VLOG(1) << "Adding 501-1000";
-  pool->Schedule([queue] { PushRange(queue, 501, 1000); });
-  VLOG(1) << "Adding 0-5";
-  pool->Schedule([queue] { PushRange(queue, 0, 5); });
-  VLOG(1) << "Adding 5-7";
-  pool->Schedule([queue] { PushRange(queue, 5, 7); });
-}
-
-// Pop elements from queue using Get().  Make sure that exactly <high> elements
-// were present and their values are all integers between 0 and high-1
-// inclusive.
-void GetRange(IntQueue *queue, int high) {
-  VLOG(1) << "Testing Wait";
-  std::vector<int> results;
-  for (int i = 0; i != high; ++i) {
-    int r = queue->Get();
-    VLOG(2) << "Waited and got " << r;
-    results.push_back(r);
-  }
-  CHECK_EQ(queue->count(), 0);
-  std::sort(results.begin(), results.end());
-  for (int i = 0; i != high; ++i) {
-    CHECK(results[i] == i);
-  }
-}
-
-// Pop elements from queue using TryGet().  Make sure that exactly <high>
-// elements were present and their values are all integers between 0 and high-1
-// inclusive.
-void TryGetRange(IntQueue *queue, int high) {
-  std::vector<int> results;
-  // Give up if we don't get all the elements back from the queue
-  // in 10 seconds.
-  int timeout = 10;
-  int r;
-  for (int i = 0; i != high; ++i) {
-    while (!queue->TryGet(&r)) {
-      if (!timeout--) {
-        LOG(FATAL) << "Can't find all elements in the queue";
-      }
-      VLOG(1) << "Sleeping for a second...";
-      sleep(1);
-    }
-    VLOG(2) << "Popped " << r;
-    results.push_back(r);
-  }
-  CHECK_EQ(queue->count(), 0);
-  CHECK(!queue->TryGet(&r));
-  std::sort(results.begin(), results.end());
-  for (int i = 0; i != high; ++i) {
-    CHECK_EQ(i, results[i]);
-  }
-}
-
-const int kNumThreads = 15;
-
-TEST(ProducerConsumerQueue, GetRange) {
-  IntQueue queue;
-  {
-    thread::ThreadPool pool(Env::Default(), "test", kNumThreads);
-    PushRanges(&queue, &pool);
-  }
-  GetRange(&queue, 1000);
-}
-
-TEST(ProducerConsumerQueue, TryGetRange) {
-  IntQueue queue;
-  {
-    thread::ThreadPool pool(Env::Default(), "test", kNumThreads);
-    PushRanges(&queue, &pool);
-  }
-  TryGetRange(&queue, 1000);
-}
-
-TEST(ProducerConsumerQueue, ParallelGetRange) {
-  IntQueue queue;
-  {
-    thread::ThreadPool pool(Env::Default(), "test", kNumThreads);
-    pool.Schedule([&queue] { GetRange(&queue, 1000); });
-    PushRanges(&queue, &pool);
-  }
-}
-
-TEST(ProducerConsumerQueue, ParallelTryGetRange) {
-  IntQueue queue;
-  {
-    thread::ThreadPool pool(Env::Default(), "test", kNumThreads);
-    pool.Schedule([&queue] { TryGetRange(&queue, 1000); });
-    PushRanges(&queue, &pool);
-  }
-}
-
-}  // namespace
-}  // namespace tensorflow
-- 
GitLab


From a326fdb4028b3bf7a08f0420c10571a29d55263f Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 13 Nov 2018 07:47:27 -0800
Subject: [PATCH 0143/1554] [tf.data] Add experimental transformation for
 applying `map()` on GPU devices.

PiperOrigin-RevId: 221265601
---
 .../api_def_ExperimentalMapDataset.pbtxt      |  5 ++
 tensorflow/core/common_runtime/function.cc    |  8 ++-
 tensorflow/core/framework/function.cc         | 24 ++++---
 tensorflow/core/framework/function.h          |  1 +
 .../core/kernels/data/map_dataset_op.cc       |  5 ++
 tensorflow/core/kernels/function_ops.cc       |  3 +
 tensorflow/core/kernels/function_ops.h        |  1 +
 .../core/ops/experimental_dataset_ops.cc      | 11 +++
 tensorflow/core/ops/function_ops.cc           | 16 +++++
 .../data/experimental/kernel_tests/BUILD      |  1 +
 .../kernel_tests/copy_to_device_test.py       | 38 +++++++++++
 .../data/experimental/ops/prefetching_ops.py  | 68 +++++++++++++++++++
 tensorflow/python/data/ops/dataset_ops.py     | 14 +++-
 13 files changed, 182 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalMapDataset.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalMapDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalMapDataset.pbtxt
new file mode 100644
index 0000000000..e9619edcac
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalMapDataset.pbtxt
@@ -0,0 +1,5 @@
+op {
+  graph_op_name: "ExperimentalMapDataset"
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 6775695fa2..7eb622dc11 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -46,7 +46,11 @@ namespace tensorflow {
 
 // A few string constant used throughout this module.
 static constexpr const char* const kArgOp = FunctionLibraryDefinition::kArgOp;
+static constexpr const char* const kDeviceArgOp =
+    FunctionLibraryDefinition::kDeviceArgOp;
 static constexpr const char* const kRetOp = FunctionLibraryDefinition::kRetOp;
+static constexpr const char* const kDeviceRetOp =
+    FunctionLibraryDefinition::kDeviceRetOp;
 static constexpr const char* const kGradientOp =
     FunctionLibraryDefinition::kGradientOp;
 static constexpr const char* const kNodeLabel = "Func";
@@ -1633,9 +1637,9 @@ FunctionBody::FunctionBody(const FunctionDef& f, DataTypeSlice arg_t,
   this->ret_nodes.resize(ret_types.size());
   for (Node* n : this->graph->op_nodes()) {
     gtl::InlinedVector<Node*, 4>* node_vec;
-    if (n->type_string() == kRetOp) {
+    if (n->type_string() == kRetOp || n->type_string() == kDeviceRetOp) {
       node_vec = &this->ret_nodes;
-    } else if (n->type_string() == kArgOp) {
+    } else if (n->type_string() == kArgOp || n->type_string() == kDeviceArgOp) {
       node_vec = &this->arg_nodes;
     } else {
       continue;
diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index abd0930ca9..9d0933e680 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -149,8 +149,8 @@ class FunctionInstantiationHelper {
   }
 
   // Builds index for nodes that can be used as node's input arguments.
-  Status BuildInputArgIndex(const OpDef::ArgDef& arg_def,
-                            AttrSlice attr_values) {
+  Status BuildInputArgIndex(const OpDef::ArgDef& arg_def, AttrSlice attr_values,
+                            bool ints_on_device) {
     bool is_type_list;
     DataTypeVector dtypes;
     TF_RETURN_IF_ERROR(
@@ -169,7 +169,11 @@ class FunctionInstantiationHelper {
         strings::StrAppend(&name, "_", i);
       }
       NodeDef* gnode = AddNode(name);
-      gnode->set_op(FunctionLibraryDefinition::kArgOp);
+      if (ints_on_device && dtypes[i] == DataType::DT_INT32) {
+        gnode->set_op(FunctionLibraryDefinition::kDeviceArgOp);
+      } else {
+        gnode->set_op(FunctionLibraryDefinition::kArgOp);
+      }
       AddAttr("T", dtypes[i], gnode);
       AddAttr("index", arg_index, gnode);
       result_.arg_types.push_back(dtypes[i]);
@@ -564,9 +568,11 @@ string Print(gtl::ArraySlice<const NodeDef*> nodes) {
   std::vector<const NodeDef*> ret;
   std::vector<const NodeDef*> body;
   for (const NodeDef* n : nodes) {
-    if (n->op() == FunctionLibraryDefinition::kArgOp) {
+    if (n->op() == FunctionLibraryDefinition::kArgOp ||
+        n->op() == FunctionLibraryDefinition::kDeviceArgOp) {
       arg.push_back(n);
-    } else if (n->op() == FunctionLibraryDefinition::kRetOp) {
+    } else if (n->op() == FunctionLibraryDefinition::kRetOp ||
+               n->op() == FunctionLibraryDefinition::kDeviceRetOp) {
       ret.push_back(n);
     } else {
       body.push_back(n);
@@ -638,10 +644,13 @@ Status InstantiateFunction(const FunctionDef& fdef, AttrSlice attr_values,
   const OpDef& sig = fdef.signature();
   TF_RETURN_IF_ERROR(ValidateSignatureWithAttrs(sig, attr_values));
 
+  bool ints_on_device = fdef.attr().count("experimental_ints_on_device") != 0 &&
+                        fdef.attr().at("experimental_ints_on_device").b();
+
   FunctionInstantiationHelper helper(get_function, result);
   Status s;
   for (const OpDef::ArgDef& arg_def : sig.input_arg()) {
-    s = helper.BuildInputArgIndex(arg_def, attr_values);
+    s = helper.BuildInputArgIndex(arg_def, attr_values, ints_on_device);
     if (!s.ok()) {
       errors::AppendToMessage(&s, "In ", Print(arg_def));
       return s;
@@ -693,9 +702,6 @@ Status InstantiateFunction(const FunctionDef& fdef, AttrSlice attr_values,
     }
   }
 
-  bool ints_on_device = fdef.attr().count("experimental_ints_on_device") != 0 &&
-                        fdef.attr().at("experimental_ints_on_device").b();
-
   // Emits nodes for the function's return values.
   int ret_index = 0;
   for (const OpDef::ArgDef& ret_def : sig.output_arg()) {
diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h
index 40ace6ef81..4cc1b858e3 100644
--- a/tensorflow/core/framework/function.h
+++ b/tensorflow/core/framework/function.h
@@ -379,6 +379,7 @@ class FunctionLibraryDefinition : public OpRegistryInterface {
   // Ops created for function arguments bear the name given by `kArgOp`; those
   // created for return values bear the name given by `kRetOp`.
   static constexpr const char* const kArgOp = "_Arg";
+  static constexpr const char* const kDeviceArgOp = "_DeviceArg";
   static constexpr const char* const kRetOp = "_Retval";
   static constexpr const char* const kDeviceRetOp = "_DeviceRetval";
 
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index d64114e70e..ab20b83298 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -244,6 +244,11 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
 };
 
 REGISTER_KERNEL_BUILDER(Name("MapDataset").Device(DEVICE_CPU), MapDatasetOp);
+REGISTER_KERNEL_BUILDER(Name("ExperimentalMapDataset")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("input_dataset")
+                            .HostMemory("handle"),
+                        MapDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/function_ops.cc b/tensorflow/core/kernels/function_ops.cc
index d5c09712b2..cca3cfbd7c 100644
--- a/tensorflow/core/kernels/function_ops.cc
+++ b/tensorflow/core/kernels/function_ops.cc
@@ -69,6 +69,7 @@ void RetvalOp::Compute(OpKernelContext* ctx) {
 }
 
 REGISTER_SYSTEM_KERNEL_BUILDER(Name(kArgOp).Device(DEVICE_CPU), ArgOp);
+REGISTER_SYSTEM_KERNEL_BUILDER(Name(kDeviceArgOp).Device(DEVICE_CPU), ArgOp);
 REGISTER_SYSTEM_KERNEL_BUILDER(Name(kRetOp).Device(DEVICE_CPU), RetvalOp);
 REGISTER_SYSTEM_KERNEL_BUILDER(Name(kDeviceRetOp).Device(DEVICE_CPU), RetvalOp);
 
@@ -105,6 +106,8 @@ TF_CALL_bool(REGISTER) REGISTER_KERNEL_BUILDER(Name(kArgOp)
                                                    .HostMemory("output")
                                                    .TypeConstraint<int32>("T"),
                                                ArgOp);
+REGISTER_KERNEL_BUILDER(
+    Name(kDeviceArgOp).Device(DEVICE_GPU).TypeConstraint<int32>("T"), ArgOp);
 #undef REGISTER
 
 REGISTER_KERNEL_BUILDER(Name(kArgOp)
diff --git a/tensorflow/core/kernels/function_ops.h b/tensorflow/core/kernels/function_ops.h
index 0f51eca163..9ddd495603 100644
--- a/tensorflow/core/kernels/function_ops.h
+++ b/tensorflow/core/kernels/function_ops.h
@@ -22,6 +22,7 @@ limitations under the License.
 namespace tensorflow {
 
 static const char* const kArgOp = FunctionLibraryDefinition::kArgOp;
+static const char* const kDeviceArgOp = FunctionLibraryDefinition::kDeviceArgOp;
 static const char* const kRetOp = FunctionLibraryDefinition::kRetOp;
 static const char* const kDeviceRetOp = FunctionLibraryDefinition::kDeviceRetOp;
 
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 088d1865dd..9733cf2776 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -75,6 +75,17 @@ REGISTER_OP("ExperimentalIgnoreErrorsDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalMapDataset")
+    .Input("input_dataset: variant")
+    .Input("other_arguments: Targuments")
+    .Output("handle: variant")
+    .Attr("f: func")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .Attr("use_inter_op_parallelism: bool = true")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalNonSerializableDataset")
     .Input("input_dataset: variant")
     .Output("handle: variant")
diff --git a/tensorflow/core/ops/function_ops.cc b/tensorflow/core/ops/function_ops.cc
index 6edd86b3ad..8e86dd9f78 100644
--- a/tensorflow/core/ops/function_ops.cc
+++ b/tensorflow/core/ops/function_ops.cc
@@ -35,6 +35,22 @@ output: The argument.
 index: This argument is the index-th argument of the function.
 )doc");
 
+REGISTER_SYSTEM_OP("_DeviceArg")
+    .Output("output: T")
+    .Attr("T: type")
+    .Attr("index: int >= 0")
+    .SetIsStateful()
+    .SetShapeFn([](shape_inference::InferenceContext* context) {
+      context->set_output(0, context->UnknownShape());
+      return Status::OK();
+    })
+    .Doc(R"doc(
+A graph node which represents an argument to a function.
+
+output: The argument.
+index: This argument is the index-th argument of the function.
+)doc");
+
 REGISTER_SYSTEM_OP("_Retval")
     .Input("input: T")
     .Attr("T: type")
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 0141ac730f..c9b11a2c38 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -38,6 +38,7 @@ cuda_py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python/compat:compat",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:iterator_ops",
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index adfacf1c9f..cea8bd6f0b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -28,7 +28,9 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
+from tensorflow.python.util import compat as util_compat
 
 
 class CopyToDeviceTest(test_base.DatasetTestBase):
@@ -294,6 +296,42 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  def testCopyToDeviceGpuWithMap(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    def generator():
+      for i in range(10):
+        yield i, float(i), str(i)
+
+    host_dataset = dataset_ops.Dataset.from_generator(
+        generator, output_types=(dtypes.int32, dtypes.float32, dtypes.string))
+    device_dataset = host_dataset.apply(
+        prefetching_ops.copy_to_device("/gpu:0"))
+
+    def gpu_map_func(x, y, z):
+      return math_ops.square(x), math_ops.square(y), z
+
+    device_dataset = device_dataset.apply(
+        prefetching_ops.map_on_gpu(gpu_map_func))
+    options = dataset_ops.Options()
+    options.experimental_autotune = False
+    device_dataset = device_dataset.with_options(options)
+
+    with ops.device("/gpu:0"):
+      iterator = device_dataset.make_initializable_iterator()
+      next_element = iterator.get_next()
+
+    with self.cached_session() as sess:
+      sess.run(iterator.initializer)
+      for i in range(10):
+        x, y, z = sess.run(next_element)
+        self.assertEqual(i**2, x)
+        self.assertEqual(float(i**2), y)
+        self.assertEqual(util_compat.as_bytes(str(i)), z)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testCopyToDeviceGpuInt32(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index a55b8bfb76..d34f9f25bd 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -540,3 +540,71 @@ class _CopyToDeviceDataset(dataset_ops.UnaryDataset):
   @property
   def output_classes(self):
     return self._input_dataset.output_classes
+
+
+class _MapOnGpuDataset(dataset_ops.UnaryDataset):
+  """A `Dataset` that maps a function over elements in its using a GPU."""
+
+  def __init__(self, input_dataset, map_func, use_inter_op_parallelism=True):
+    """See `Dataset.map()` for details."""
+    super(_MapOnGpuDataset, self).__init__(input_dataset)
+    self._input_dataset = input_dataset
+    self._use_inter_op_parallelism = use_inter_op_parallelism
+
+    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+        map_func,
+        self._transformation_name(),
+        dataset=input_dataset,
+        defun_kwargs={"experimental_ints_on_device": True})
+    self._output_classes = wrapped_func.output_classes
+    self._output_shapes = wrapped_func.output_shapes
+    self._output_types = wrapped_func.output_types
+    self._map_func = wrapped_func.function
+
+  def _as_variant_tensor(self):
+    input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
+    return ged_ops.experimental_map_dataset(
+        input_t,
+        self._map_func.captured_inputs,
+        f=self._map_func,
+        use_inter_op_parallelism=self._use_inter_op_parallelism,
+        **dataset_ops.flat_structure(self))
+
+  @property
+  def output_classes(self):
+    return self._output_classes
+
+  @property
+  def output_shapes(self):
+    return self._output_shapes
+
+  @property
+  def output_types(self):
+    return self._output_types
+
+  def _transformation_name(self):
+    return "map_on_gpu()"
+
+
+def map_on_gpu(map_func):
+  """Maps `map_func` across the elements of this dataset.
+
+  NOTE: This is a highly experimental version of `tf.data.Dataset.map` that runs
+  `map_func` on GPU. It must be used after applying the
+  `tf.data.experimental.copy_to_device` transformation with a GPU device
+  argument.
+
+  Args:
+    map_func: A function mapping a nested structure of tensors (having shapes
+      and types defined by `self.output_shapes` and `self.output_types`) to
+      another nested structure of tensors.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    `tf.data.Dataset.apply`.
+  """
+
+  def _apply_fn(dataset):
+    return _MapOnGpuDataset(dataset, map_func)
+
+  return _apply_fn
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 59389a24f7..3836a68e7d 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -61,6 +61,7 @@ class Dataset(object):
   collection of elements (nested structures of tensors) and a "logical
   plan" of transformations that act on those elements.
   """
+
   def __init__(self):
     pass
 
@@ -201,6 +202,7 @@ class Dataset(object):
     # a 0-argument function.
     @function.Defun(capture_by_value=True)
     def _make_dataset():
+      """Factory function for a dataset."""
       # NOTE(mrry): `Defun` does not capture the graph-level seed from the
       # enclosing graph, so if a graph-level seed is present we set the local
       # graph seed based on a combination of the graph- and op-level seeds.
@@ -1777,7 +1779,8 @@ class StructuredFunctionWrapper(object):
                input_shapes=None,
                input_types=None,
                add_to_graph=True,
-               experimental_nested_dataset_support=False):
+               experimental_nested_dataset_support=False,
+               defun_kwargs=None):
     """Creates a new `StructuredFunctionWrapper` for the given function.
 
     Args:
@@ -1798,6 +1801,9 @@ class StructuredFunctionWrapper(object):
         default graph.
       experimental_nested_dataset_support: (Optional.) If `True`, the function
         will support `tf.data.Dataset` objects as arguments and return values.
+      defun_kwargs: (Optional.) A dictionary mapping string argument names to
+        values. If supplied, will be passed to `function.Defun()` as keyword
+        arguments.
 
     Raises:
       ValueError: If an invalid combination of `dataset`, `input_classes`,
@@ -1832,7 +1838,11 @@ class StructuredFunctionWrapper(object):
     # TODO(b/110122868): Enable this support for all `tf.data` functions.
     self._nested_dataset_support = experimental_nested_dataset_support
 
-    @function.Defun(*self._defun_args(), func_name=self._func_name)
+    if defun_kwargs is None:
+      defun_kwargs = {}
+
+    @function.Defun(
+        *self._defun_args(), func_name=self._func_name, **defun_kwargs)
     def tf_data_structured_function_wrapper(*args):
       """Wrapper for passing nested structures to and from tf.data functions."""
       flat_args = []
-- 
GitLab


From 994ee1d86673955b297774bf9abc4e3dee0f817c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 07:47:28 -0800
Subject: [PATCH 0144/1554] Delete Map/Reduce support from OneDevice and
 Mirrored DistributionStrategies.

PiperOrigin-RevId: 221265603
---
 .../distribute/python/cross_tower_ops.py      | 12 +----------
 .../distribute/python/cross_tower_utils.py    |  2 --
 .../python/cross_tower_utils_test.py          | 11 ----------
 .../distribute/python/mirrored_strategy.py    | 15 --------------
 .../python/mirrored_strategy_multigpu_test.py |  5 -----
 .../python/mirrored_strategy_test.py          |  3 ---
 .../distribute/python/one_device_strategy.py  | 20 ++-----------------
 .../python/one_device_strategy_test.py        |  3 ---
 .../distribute/python/strategy_test_lib.py    |  9 ---------
 .../contrib/distribute/python/values.py       | 10 ----------
 10 files changed, 3 insertions(+), 87 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py
index 6b2fe0acb2..b5b349aa64 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py
@@ -155,17 +155,7 @@ def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn,
   all_values = []
   count = 0
   for v in per_replica_value._index.values():  # pylint: disable=protected-access
-    if isinstance(v, value_lib.MapOutput):
-      v_list = v.get()
-      if not v_list:
-        continue
-      count += len(v_list)
-      # Sum within each device before aggregating across devices.
-      # TODO(yuefengz): Check whether it helps to use accumulation_fn here.
-      v = cross_tower_utils.aggregate_tensors_or_indexed_slices(
-          v_list, math_ops.add_n)
-    else:
-      count += 1
+    count += 1
     all_values.append(v)
   if not all_values:
     raise ValueError("`per_replica_value` must be non-empty")
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/contrib/distribute/python/cross_tower_utils.py
index 35324d15d4..50b3cf31e5 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_utils.py
@@ -667,7 +667,5 @@ def contains_indexed_slices(value):
     return any(contains_indexed_slices(v) for v in value)
   elif isinstance(value, value_lib.DistributedValues):
     return contains_indexed_slices(list(value._index.values()))  # pylint: disable=protected-access
-  elif isinstance(value, value_lib.MapOutput):
-    return contains_indexed_slices(value.get())
   else:
     return False
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils_test.py b/tensorflow/contrib/distribute/python/cross_tower_utils_test.py
index a991156ca8..e46240abbf 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils_test.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_utils_test.py
@@ -106,17 +106,6 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
     per_replica = value_lib.PerReplica({"/gpu:0": t0, "/cpu:0": t1})
     self.assertTrue(cross_tower_utils.contains_indexed_slices(per_replica))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testContainsIndexedSlices_PerReplicaMapOutput(self):
-    t0 = math_ops._as_indexed_slices(
-        constant_op.constant([[1., 2.], [0, 0], [3., 4.]]))
-    t1 = math_ops._as_indexed_slices(
-        constant_op.constant([[0., 0.], [5, 6], [7., 8.]]))
-    per_replica = value_lib.PerReplica({
-        "/gpu:0": value_lib.MapOutput([t0]),
-        "/cpu:0": value_lib.MapOutput([t1])})
-    self.assertTrue(cross_tower_utils.contains_indexed_slices(per_replica))
-
   @combinations.generate(combinations.combine(
       mode=["graph", "eager"],
       required_gpus=1))
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index e90c510aad..2d75024e7a 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -554,21 +554,6 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
   def _call_for_each_replica(self, fn, args, kwargs):
     return _call_for_each_replica(self, fn, args, kwargs)
 
-  def map(self, map_over, fn, *args, **kwargs):
-    # TODO(josh11b): In eager mode, use one thread per device.
-    index = {}
-    for i, m in enumerate(map_over):
-      d = self._devices[i % len(self._devices)]
-      with ops.device(d):
-        l = index.get(d, [])
-        l.append(fn(m,
-                    *values.select_device_mirrored(d, args),
-                    **values.select_device_mirrored(d, kwargs)))
-        index[d] = l
-    # TODO(josh11b): Need a values.regroup equivalent that handles MapOutput
-    # in addition to PerReplica data.
-    return values.PerReplica({k: values.MapOutput(v) for k, v in index.items()})
-
   def configure(self,
                 session_config=None,
                 cluster_spec=None,
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 0dbf6ba056..1fd18e09c0 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -78,11 +78,6 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     self._test_minimize_loss_graph(
         self._get_distribution_strategy(), soft_placement=soft_placement)
 
-  def testMapReduce(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self._test_map_reduce(self._get_distribution_strategy())
-
   def testDeviceIndex(self):
     if not GPU_TEST:
       self.skipTest("Not GPU test")
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
index b5d393fd0d..bea684e77c 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -40,9 +40,6 @@ class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
   def testMinimizeLossGraph(self):
     self._test_minimize_loss_graph(self._get_distribution_strategy())
 
-  def testMapReduce(self):
-    self._test_map_reduce(self._get_distribution_strategy())
-
   def testDeviceIndex(self):
     self._test_device_index(self._get_distribution_strategy())
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 1b4251b761..a0d8f93887 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -25,8 +25,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
 
@@ -119,23 +117,9 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     with ops.device(self._device), _OneDeviceReplicaContext(self):
       return fn(*args, **kwargs)
 
-  def map(self, map_over, fn, *args, **kwargs):
-    with ops.device(self._device):
-      return values.MapOutput([fn(m, *args, **kwargs) for m in map_over])
-
   def _reduce(self, aggregation, value, destinations):
-    del destinations
-    if not isinstance(value, values.MapOutput):
-      return value
-    l = value.get()
-    assert l
-    with ops.device(self._device):
-      if aggregation == vs.VariableAggregation.SUM:
-        return math_ops.add_n(l)
-      elif aggregation == vs.VariableAggregation.MEAN:
-        return math_ops.add_n(l) / len(l)
-      else:
-        assert False
+    del aggregation, destinations
+    return value
 
   def _update(self, var, options, fn, *args, **kwargs):
     # The implementations of _update() and _update_non_slot() are identical
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 3fb9227392..95f4cdb786 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -35,9 +35,6 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
   def testMinimizeLossGraph(self):
     self._test_minimize_loss_graph(self._get_distribution_strategy())
 
-  def testMapReduce(self):
-    self._test_map_reduce(self._get_distribution_strategy())
-
   def testDeviceIndex(self):
     self._test_device_index(self._get_distribution_strategy())
 
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 98cdb17b8c..3c0c10430e 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -189,15 +189,6 @@ class DistributionTestBase(test.TestCase):
       # Error should go down
       self.assertLess(error_after, error_before)
 
-  def _test_map_reduce(self, d, in_graph=None):
-    with d.scope():
-      map_in = [constant_op.constant(i) for i in range(10)]
-      map_out = d.map(map_in, lambda x, y: x * y, 2)
-      observed = d.reduce(variable_scope.VariableAggregation.SUM, map_out,
-                          "/device:CPU:0")
-      expected = 90  # 2 * (0 + 1 + ... + 9)
-      self.assertEqual(expected, observed.numpy())
-
   def _test_device_index(self, d):
     with d.scope():
       expected_devices = [False] * len(d.worker_devices)
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 9b0aa9c990..a162973535 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -1287,16 +1287,6 @@ class MultiWorkerDataset(object):
     return MultiWorkerDataIterator(iterators, self._worker_device_pairs)
 
 
-class MapOutput(object):
-  """Map can result in multiple outputs per device."""
-
-  def __init__(self, l):
-    self._l = l
-
-  def get(self):
-    return self._l
-
-
 class MultiStepContext(object):
   """A context object that can be used to capture things when running steps.
 
-- 
GitLab


From e37caf727925e4be1745b5a5316bb293a957506b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 08:18:39 -0800
Subject: [PATCH 0145/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 221269882
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 43 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 43 +++++++++++++++++++
 2 files changed, 86 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index bfcc92dcb0..6f5034ced5 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21898,6 +21898,49 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalMapDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "use_inter_op_parallelism"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+}
 op {
   name: "ExperimentalMaterializedIndexDatasetHandle"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 430212ee1d..915cd26db1 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10361,6 +10361,49 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalMapDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "use_inter_op_parallelism"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+}
 op {
   name: "ExperimentalMaterializedIndexDatasetHandle"
   output_arg {
-- 
GitLab


From 0644e35142505c1ece3f052aa03389ca921263d5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 08:20:38 -0800
Subject: [PATCH 0146/1554] Improve depthwise convolution implementations on
 TPUs.

PiperOrigin-RevId: 221270135
---
 .../compiler/xla/tests/convolution_test.cc    | 291 +++++++++++++++++-
 1 file changed, 285 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 211d004ec8..3d91fc451c 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -721,8 +721,6 @@ class Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid : public ConvolutionTest {
     ComputeAndCompareLiteral(&builder, expected_r4,
                              {input_literal.get(), filter_literal.get()},
                              error_spec_);
-
-    auto filter_r = filter_r1.Reshape(filter_dims);
   }
 };
 
@@ -731,6 +729,151 @@ TYPED_TEST(Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid, Types) {
   this->RunTest();
 }
 
+template <typename T>
+class Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 4, 4, 512};
+    std::vector<int64> filter_dims = {3, 3, 1, 512};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/512);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(2048, static_cast<T>(18));
+
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 = expected_r1.Reshape({1, 2, 2, 512}).ConsumeValueOrDie();
+    auto expected_r4_relaid =
+        expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    auto input_literal =
+        client_->TransferToServer(input_r4).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4_relaid,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_, &expected_r4_relaid.shape());
+  }
+};
+
+TYPED_TEST_CASE(
+    Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes,
+    TestTypes);
+TYPED_TEST(Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes,
+           Types) {
+  this->RunTest();
+}
+
+template <typename T>
+class Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 4, 4, 5};
+    std::vector<int64> filter_dims = {3, 3, 1, 5};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/5);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape));
+    iota_int_init_value(input_elems, 1);
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+    auto input_r4_relaid =
+        input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape));
+    iota_int_init_value(filter_elems, 1);
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    auto expected_r1 = LiteralUtil::CreateR1<T>(
+        {static_cast<T>(6864),  static_cast<T>(7296),  static_cast<T>(7746),
+         static_cast<T>(8214),  static_cast<T>(8700),  static_cast<T>(7809),
+         static_cast<T>(8286),  static_cast<T>(8781),  static_cast<T>(9294),
+         static_cast<T>(9825),  static_cast<T>(10644), static_cast<T>(11256),
+         static_cast<T>(11886), static_cast<T>(12534), static_cast<T>(13200),
+         static_cast<T>(11589), static_cast<T>(12246), static_cast<T>(12921),
+         static_cast<T>(13614), static_cast<T>(14325)});
+    auto expected_r4 = expected_r1.Reshape({1, 2, 2, 5}).ConsumeValueOrDie();
+    auto expected_r4_relaid =
+        expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    auto input_literal =
+        client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4_relaid,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_, &expected_r4_relaid.shape());
+  }
+};
+
+TYPED_TEST_CASE(
+    Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes,
+    TestTypes);
+TYPED_TEST(Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes,
+           Types) {
+  this->RunTest();
+}
+
 template <typename T>
 class Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Valid : public ConvolutionTest {
  public:
@@ -786,8 +929,6 @@ class Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Valid : public ConvolutionTest {
     ComputeAndCompareLiteral(&builder, expected_r4,
                              {input_literal.get(), filter_literal.get()},
                              error_spec_);
-
-    auto filter_r = filter_r1.Reshape(filter_dims);
   }
 };
 
@@ -796,6 +937,146 @@ TYPED_TEST(Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Valid, Types) {
   this->RunTest();
 }
 
+template <typename T>
+class Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Input_Batch_In_Lanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 4, 4, 160};
+    std::vector<int64> filter_dims = {3, 3, 1, 160};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/160);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+    auto input_r4_relaid =
+        input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(640, static_cast<T>(18));
+
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 = expected_r1.Reshape({1, 2, 2, 160}).ConsumeValueOrDie();
+    auto expected_r4_relaid =
+        expected_r4.Relayout(LayoutUtil::MakeLayout({3, 0, 2, 1}));
+
+    auto input_literal =
+        client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4_relaid,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_, &expected_r4_relaid.shape());
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Input_Batch_In_Lanes,
+                TestTypes);
+TYPED_TEST(Convolve2D_1x4x4x160_3x3x1x160_Depthwise_Input_Batch_In_Lanes,
+           Types) {
+  this->RunTest();
+}
+
+template <typename T>
+class Convolve2D_1x4x4x160_3x3x1x160_Dephtwise_Both_Batch_In_Lanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 4, 4, 160};
+    std::vector<int64> filter_dims = {3, 3, 1, 160};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/160);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+    auto input_r4_relaid =
+        input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(640, static_cast<T>(18));
+
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 = expected_r1.Reshape({1, 2, 2, 160}).ConsumeValueOrDie();
+    auto expected_r4_relaid =
+        expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    auto input_literal =
+        client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4_relaid,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_, &expected_r4_relaid.shape());
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x4x4x160_3x3x1x160_Dephtwise_Both_Batch_In_Lanes,
+                TestTypes);
+TYPED_TEST(Convolve2D_1x4x4x160_3x3x1x160_Dephtwise_Both_Batch_In_Lanes,
+           Types) {
+  this->RunTest();
+}
+
 template <typename T>
 class Convolve2D_1x4x4x1024_3x3x1x1024_Depthwise_Valid
     : public ConvolutionTest {
@@ -852,8 +1133,6 @@ class Convolve2D_1x4x4x1024_3x3x1x1024_Depthwise_Valid
     ComputeAndCompareLiteral(&builder, expected_r4,
                              {input_literal.get(), filter_literal.get()},
                              error_spec_);
-
-    auto filter_r = filter_r1.Reshape(filter_dims);
   }
 };
 
-- 
GitLab


From 0d2edbe63cbaad650598afad50cc14a727405729 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 08:23:59 -0800
Subject: [PATCH 0147/1554] Add unicode_decode_with_offsets Op to TF strings
 package

RELNOTES: This Op adds the ability to decode a variety of input text encoding formats into Unicode characters.
PiperOrigin-RevId: 221270527
---
 .../api_def_UnicodeDecodeWithOffsets.pbtxt    |  87 ++++++++
 tensorflow/core/kernels/unicode_ops.cc        | 207 +++++++++++++++---
 tensorflow/core/ops/string_ops.cc             |  24 ++
 tensorflow/python/kernel_tests/BUILD          |  12 +
 .../kernel_tests/unicode_decode_op_test.py    | 156 +++++++++++++
 .../kernel_tests/unicode_transcode_op_test.py |   2 +-
 6 files changed, 453 insertions(+), 35 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_UnicodeDecodeWithOffsets.pbtxt
 create mode 100644 tensorflow/python/kernel_tests/unicode_decode_op_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_UnicodeDecodeWithOffsets.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnicodeDecodeWithOffsets.pbtxt
new file mode 100644
index 0000000000..15fc8747af
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_UnicodeDecodeWithOffsets.pbtxt
@@ -0,0 +1,87 @@
+op {
+  graph_op_name: "UnicodeDecodeWithOffsets"
+  in_arg {
+    name: "input"
+    description: <<END
+The text to be decoded. Can have any shape. Note that the output is flattened
+to a vector of char values.
+END
+  }
+  out_arg {
+    name: "row_splits"
+    description: <<END
+A 1D int32 tensor containing the row splits.
+END
+  }
+  out_arg {
+    name: "char_values"
+    description: <<END
+A 1D int32 Tensor containing the decoded codepoints.
+END
+  }
+  out_arg {
+    name: "char_to_byte_starts"
+    description: <<END
+A 1D int32 Tensor containing the byte index in the input string where each
+character in `char_values` starts.
+END
+  }
+  attr {
+    name: "input_encoding"
+    description: <<END
+Text encoding of the input strings. This is any of the encodings supported
+by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`.
+END
+  }
+  attr {
+    name: "errors"
+    description: <<END
+Error handling policy when there is invalid formatting found in the input.
+The value of 'strict' will cause the operation to produce a InvalidArgument
+error on any invalid input formatting. A value of 'replace' (the default) will
+cause the operation to replace any invalid formatting in the input with the
+`replacement_char` codepoint. A value of 'ignore' will cause the operation to
+skip any invalid formatting in the input and produce no corresponding output
+character.
+END
+  }
+  attr {
+    name: "replacement_char"
+    description: <<END
+The replacement character codepoint to be used in place of any invalid
+formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+be used. The default value is the default unicode replacement character is
+0xFFFD or U+65533.)
+END
+  }
+  attr {
+    name: "replace_control_characters"
+    description: <<END
+Whether to replace the C0 control characters (00-1F) with the
+`replacement_char`. Default is false.
+END
+  }
+  summary: <<END
+Decodes each string in `input` into a sequence of Unicode code points.
+END
+  description: <<END
+The character codepoints for all strings are returned using a single vector
+`char_values`, with strings expanded to characters in row-major order.
+Similarly, the character start byte offsets are returned using a single vector
+`char_to_byte_starts`, with strings expanded in row-major order.
+
+The `row_splits` tensor indicates where the codepoints and start offsets for
+each input string begin and end within the `char_values` and
+`char_to_byte_starts` tensors.  In particular, the values for the `i`th
+string (in row-major order) are stored in the slice
+`[row_splits[i]:row_splits[i+1]]`. Thus:
+
+* `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th
+  character in the `i`th string (in row-major order).
+* `char_to_bytes_starts[row_splits[i]+j]` is the start byte offset for the `j`th
+  character in the `i`th string (in row-major order).
+* `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th
+  string (in row-major order).
+END
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc
index dd4415711b..fde579dd9f 100644
--- a/tensorflow/core/kernels/unicode_ops.cc
+++ b/tensorflow/core/kernels/unicode_ops.cc
@@ -33,6 +33,8 @@ limitations under the License.
 
 namespace tensorflow {
 
+namespace {
+
 // This error callback is only useful for finding illegal encoding errors when
 // we want to be strict -- otherwise illegal encodings are replaced on read
 // with 0xFFFD and signaled to the callback.
@@ -146,40 +148,64 @@ class WrappedConverter {
   string name_;
 };
 
+struct ErrorOptions {
+  UChar32 subst = 0xFFFD;
+  bool elide_replacement = false;
+  bool replace_control_chars = false;
+  bool error_on_malformatting = false;
+};
+
+Status GetErrorOptions(OpKernelConstruction* ctx, ErrorOptions* out) {
+  *out = ErrorOptions();
+
+  string error_policy;
+  TF_RETURN_IF_ERROR(ctx->GetAttr("errors", &error_policy));
+
+  if (error_policy == "replace") {
+    out->elide_replacement = false;
+  } else if (error_policy == "ignore") {
+    out->elide_replacement = true;
+  } else if (error_policy == "strict") {
+    out->error_on_malformatting = true;
+  } else {
+    return errors::InvalidArgument(
+        "errors policy must be one of 'strict', 'replace', or 'ignore'");
+  }
+
+  int32 replacement_char;
+  TF_RETURN_IF_ERROR(ctx->GetAttr("replacement_char", &replacement_char));
+
+  if (replacement_char >= UCHAR_MIN_VALUE &&
+      replacement_char <= UCHAR_MAX_VALUE) {
+    out->subst = replacement_char;
+  } else {
+    return errors::InvalidArgument(
+        "replacement_char out of unicode codepoint range");
+  }
+
+  TF_RETURN_IF_ERROR(ctx->GetAttr("replace_control_characters",
+                                  &(out->replace_control_chars)));
+
+  return Status::OK();
+}
+
+inline bool ShouldHandleFormatError(const ErrorOptions& error_options,
+                                    UChar32 ch, bool format_error) {
+  return ((error_options.replace_control_chars && ch <= 0x1F) || format_error);
+}
+
+}  // namespace
+
 class UnicodeTranscodeOp : public OpKernel {
  public:
   explicit UnicodeTranscodeOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
-    string error_policy;
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("errors", &error_policy));
-    if (error_policy == "replace") {
-      elide_replacement_ = false;
-    } else if (error_policy == "ignore") {
-      elide_replacement_ = true;
-    } else if (error_policy == "strict") {
-      error_on_malformatting_ = true;
-    } else {
-      ctx->CtxFailure(errors::InvalidArgument(
-          "errors policy must be one of 'strict', 'replace', or 'ignore'"));
-    }
-
-    int32 replacement_char;
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("replacement_char", &replacement_char));
-    if (replacement_char >= UCHAR_MIN_VALUE &&
-        replacement_char <= UCHAR_MAX_VALUE) {
-      subst_ = replacement_char;
-    } else {
-      ctx->CtxFailure(errors::InvalidArgument(
-          "replacement_char out of unicode codepoint range"));
-    }
+    OP_REQUIRES_OK(ctx, GetErrorOptions(ctx, &error_options_));
 
     string output_encoding;
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_encoding", &output_encoding));
     OP_REQUIRES_OK(ctx,
                    ParseUnicodeEncoding(output_encoding, &output_encoding_));
 
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("replace_control_characters",
-                                     &replace_control_chars_));
-
     OP_REQUIRES_OK(ctx, ctx->GetAttr("input_encoding", &input_encoding_));
     // Make a temporary UConverter to ensure it will create without error
     // at execution time (and to warm any data caches the converter needs).
@@ -228,7 +254,7 @@ class UnicodeTranscodeOp : public OpKernel {
       Transcode(&(output_flat(i)), input_encoder->converter_,
                 &found_any_format_error);
     }
-    if (error_on_malformatting_ && found_any_format_error) {
+    if (error_options_.error_on_malformatting && found_any_format_error) {
       ctx->CtxFailure(
           errors::InvalidArgument("Invalid formatting on input string"));
     }
@@ -240,12 +266,12 @@ class UnicodeTranscodeOp : public OpKernel {
   // out-of-range inputs.
   void TranslateCodepoints(icu::UnicodeString* s, bool* found_any_format_error,
                            UChar32 ch, int src_bytes, bool format_error) {
-    if ((replace_control_chars_ && ch <= 0x1F) || format_error) {
+    if (ShouldHandleFormatError(error_options_, ch, format_error)) {
       *found_any_format_error = true;
-      if (elide_replacement_) {
+      if (error_options_.elide_replacement) {
         return;
       } else {
-        ch = subst_;
+        ch = error_options_.subst;
       }
     }
     s->append(ch);
@@ -292,16 +318,129 @@ class UnicodeTranscodeOp : public OpKernel {
     }
   }
 
-  UChar32 subst_ = 0xFFFD;
-  bool elide_replacement_ = false;
-  bool replace_control_chars_ = false;
-  bool error_on_malformatting_ = false;
-
   string input_encoding_;
+  ErrorOptions error_options_;
   UnicodeEncoding output_encoding_ = UnicodeEncoding::UTF8;
 };
 
 REGISTER_KERNEL_BUILDER(Name("UnicodeTranscode").Device(DEVICE_CPU),
                         UnicodeTranscodeOp);
 
+class UnicodeDecodeWithOffsetsOp : public OpKernel {
+ public:
+  explicit UnicodeDecodeWithOffsetsOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, GetErrorOptions(ctx, &error_options_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("input_encoding", &input_encoding_));
+    // Make a temporary UConverter to ensure it will create without error
+    // at execution time (and to warm any data caches the converter needs).
+    // This instance is not used.
+    std::unique_ptr<WrappedConverter> input_encoder =
+        absl::make_unique<WrappedConverter>();
+    input_encoder->init(input_encoding_);
+    OP_REQUIRES(ctx, input_encoder->converter_,
+                errors::InvalidArgument(
+                    "Could not create converter for input encoding: " +
+                    input_encoding_));
+  }
+
+  void Decode(OpKernelContext* ctx, std::vector<UChar32>* char_values,
+              std::vector<int64>* offset_values, int* string_length,
+              int64* next_row_split, UChar32 char_value, int char_length,
+              bool found_any_format_error) {
+    if (error_options_.error_on_malformatting && found_any_format_error) {
+      ctx->CtxFailure(
+          errors::InvalidArgument("Invalid formatting on input string"));
+    }
+    UChar32 decoded_value = char_value;
+    if (ShouldHandleFormatError(error_options_, char_value,
+                                found_any_format_error)) {
+      if (error_options_.elide_replacement) {
+        return;
+      } else {
+        decoded_value = error_options_.subst;
+      }
+    }
+
+    // Emit the char value.
+    char_values->push_back(decoded_value);
+
+    // Emit the byte offset
+    offset_values->push_back(*string_length);
+    *string_length += char_length;
+    *next_row_split += 1;
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor* input_tensor;
+    OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor));
+
+    // Go through all the strings in `input`.
+    const auto& input_vec = input_tensor->flat<string>();
+
+    std::unique_ptr<WrappedConverter> input_encoder =
+        absl::make_unique<WrappedConverter>();
+    input_encoder->init(input_encoding_);
+    OP_REQUIRES(ctx, input_encoder->converter_,
+                errors::InvalidArgument(
+                    "Could not create converter for input encoding: " +
+                    input_encoding_));
+
+    std::vector<UChar32> char_values;
+    std::vector<int64> offset_values;
+
+    Tensor* output_row_splits;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output("row_splits",
+                                             {input_tensor->NumElements() + 1},
+                                             &output_row_splits));
+    auto out_row_splits = output_row_splits->vec<int64>();
+
+    int row_split_index = 0;
+    int64 next_row_split = 0;
+    for (int i = 0; i < input_vec.size(); ++i) {
+      const string& input = input_vec(i);
+      // Convert input strings into unicode values. Output to a list of
+      // char_values, record row splits and char_to_byte_starts, which are all
+      // the fields needed to construct a RaggedTensor.
+      out_row_splits(row_split_index) = next_row_split;
+      row_split_index++;
+      int string_length = 0;
+      IterateUnicodeString(
+          input, input_encoder->converter_,
+          std::bind(&UnicodeDecodeWithOffsetsOp::Decode, this, ctx,
+                    &char_values, &offset_values, &string_length,
+                    &next_row_split, std::placeholders::_1,
+                    std::placeholders::_2, std::placeholders::_3));
+    }
+    out_row_splits(row_split_index) = next_row_split;
+
+    DCHECK(offset_values.size() == char_values.size());
+    Tensor* output_char_values;
+    OP_REQUIRES_OK(
+        ctx, ctx->allocate_output("char_values",
+                                  {static_cast<int64>(char_values.size())},
+                                  &output_char_values));
+    Tensor* output_offset_values;
+    OP_REQUIRES_OK(
+        ctx, ctx->allocate_output("char_to_byte_starts",
+                                  {static_cast<int64>(offset_values.size())},
+                                  &output_offset_values));
+    auto out_char_values = output_char_values->vec<uint32>();
+    auto out_offset_values = output_offset_values->vec<int64>();
+
+    // Load output tensors from intermediate value arrays.
+    for (int i = 0; i < char_values.size(); ++i) {
+      out_char_values(i) = static_cast<uint32>(char_values[i]);
+      out_offset_values(i) = offset_values[i];
+    }
+  }
+
+ private:
+  string input_encoding_;
+  ErrorOptions error_options_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("UnicodeDecodeWithOffsets").Device(DEVICE_CPU),
+                        UnicodeDecodeWithOffsetsOp);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/string_ops.cc b/tensorflow/core/ops/string_ops.cc
index 352253135c..8fe8e5db52 100644
--- a/tensorflow/core/ops/string_ops.cc
+++ b/tensorflow/core/ops/string_ops.cc
@@ -259,4 +259,28 @@ REGISTER_OP("UnicodeTranscode")
     .Attr("replacement_char: int = 65533")  // 0xFFFD unicode replacement char
     .Attr("replace_control_characters: bool = false")
     .SetShapeFn(shape_inference::UnchangedShape);
+
+REGISTER_OP("UnicodeDecodeWithOffsets")
+    .Input("input: string")
+    .Output("row_splits: int64")
+    .Output("char_values: uint32")
+    .Output("char_to_byte_starts: int64")
+    .Attr("input_encoding: string")
+    .Attr("errors: {'strict', 'replace', 'ignore'} = 'replace'")
+    .Attr("replacement_char: int = 65533")  // 0xFFFD unicode replacement char
+    .Attr("replace_control_characters: bool = false")
+    .SetShapeFn([](InferenceContext* c) {
+      // row_splits.shape == [input.size() + 1]
+      DimensionHandle num_row_splits;
+      DimensionHandle input_size = c->NumElements(c->input(0));
+      TF_RETURN_IF_ERROR(c->Add(input_size, 1, &num_row_splits));
+      c->set_output(0, c->Vector(num_row_splits));
+
+      // char_values.shape == offset_values.shape == [num_chars]
+      DimensionHandle num_chars = c->UnknownDim();
+      c->set_output(1, c->Vector(num_chars));
+      c->set_output(2, c->Vector(num_chars));
+      return Status::OK();
+    });
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 6b990d3a92..48639a4518 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1154,6 +1154,18 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "unicode_decode_op_test",
+    size = "small",
+    srcs = ["unicode_decode_op_test.py"],
+    additional_deps = [
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:string_ops",
+    ],
+)
+
 tf_py_test(
     name = "unique_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/unicode_decode_op_test.py b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
new file mode 100644
index 0000000000..9401e77adf
--- /dev/null
+++ b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for unicode_decode and unicode_decode_with_splits."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors_impl as errors
+from tensorflow.python.ops import gen_string_ops
+from tensorflow.python.platform import test
+
+
+# Account for python2 and python3 execution of the test.
+def codepoint(s):
+  if isinstance(s, bytes):
+    return ord(s.decode("utf-8"))
+  elif isinstance(s, str):
+    return ord(s)
+
+
+class UnicodeDecodeTest(test.TestCase):
+
+  def testBatchDecode(self):
+    text = constant_op.constant(
+        ["仅今年前", "中国进出口银行与中国银行加强合作"])
+    row_splits, utf8_text, offsets = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8")
+
+    with self.test_session():
+      self.assertAllEqual([
+          codepoint("仅"),
+          codepoint("今"),
+          codepoint("年"),
+          codepoint("前"),
+          codepoint("中"),
+          codepoint("国"),
+          codepoint("进"),
+          codepoint("出"),
+          codepoint("口"),
+          codepoint("银"),
+          codepoint("行"),
+          codepoint("与"),
+          codepoint("中"),
+          codepoint("国"),
+          codepoint("银"),
+          codepoint("行"),
+          codepoint("加"),
+          codepoint("强"),
+          codepoint("合"),
+          codepoint("作")
+      ],
+                          utf8_text.eval().tolist())
+      self.assertAllEqual([0, 4, 20], row_splits.eval().tolist())
+      self.assertAllEqual([0, 3, 6, 9, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30,
+                           33, 36, 39, 42, 45],
+                          offsets.eval().tolist())
+
+  def testBasicDecodeWithOffset(self):
+    text = constant_op.constant(["仅今年前"])
+    row_splits, utf8_text, starts = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8")
+
+    with self.test_session():
+      self.assertAllEqual([
+          codepoint("仅"),
+          codepoint("今"),
+          codepoint("年"),
+          codepoint("前"),
+      ],
+                          utf8_text.eval().tolist())
+      self.assertAllEqual(row_splits.eval().tolist(), [0, 4])
+      self.assertAllEqual(starts.eval().tolist(), [0, 3, 6, 9])
+
+  def testStrictError(self):
+    text = constant_op.constant([b"\xFEED"])
+    _, error, _ = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8", errors="strict")
+
+    with self.assertRaises(errors.InvalidArgumentError):
+      with self.test_session():
+        error.eval()
+
+  def testReplaceOnError(self):
+    text = constant_op.constant([b"\xFE"])
+
+    _, utf8_text, _ = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8", errors="replace")
+
+    with self.test_session():
+      self.assertAllEqual(utf8_text.eval().tolist(), [65533])
+
+  def testBadReplacementChar(self):
+    text = constant_op.constant([b"\xFE"])
+    _, error, _ = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8", errors="replace", replacement_char=11141111)
+
+    with self.assertRaises(errors.InvalidArgumentError):
+      with self.test_session():
+        error.eval()
+
+  def testIgnoreOnError(self):
+    text = constant_op.constant([b"\xFEhello"])
+
+    _, utf8_text, _ = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8", errors="ignore")
+
+    with self.test_session():
+      self.assertAllEqual(utf8_text.eval().tolist(), [
+          codepoint("h"),
+          codepoint("e"),
+          codepoint("l"),
+          codepoint("l"),
+          codepoint("o")
+      ])
+
+  def testBadErrorPolicy(self):
+    text = constant_op.constant(["hippopotamus"])
+
+    with self.assertRaises(ValueError):
+      _, _, _ = gen_string_ops.unicode_decode_with_offsets(
+          text, "utf-8", errors="oranguatan")
+
+  def testReplaceControlChars(self):
+    text = constant_op.constant(["\x02仅今年前"])
+    row_splits, utf8_text, _ = gen_string_ops.unicode_decode_with_offsets(
+        text, "utf-8", replace_control_characters=True)
+
+    with self.test_session():
+      self.assertAllEqual([
+          65533,
+          codepoint("仅"),
+          codepoint("今"),
+          codepoint("年"),
+          codepoint("前"),
+      ],
+                          utf8_text.eval().tolist())
+      self.assertAllEqual([0, 5], row_splits.eval().tolist())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
index 2908e2bfc5..4406907a4c 100644
--- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for string_length_op."""
+"""Tests for unicode_transcode op."""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 814013519857b614bd431b604bdb2af6462e8039 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 13 Nov 2018 08:26:18 -0800
Subject: [PATCH 0148/1554] Don't memset copy on write allocated memory.

PiperOrigin-RevId: 221270787
---
 tensorflow/core/lib/png/png_io.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/lib/png/png_io.cc b/tensorflow/core/lib/png/png_io.cc
index bc52180265..e8dbcb97b9 100644
--- a/tensorflow/core/lib/png/png_io.cc
+++ b/tensorflow/core/lib/png/png_io.cc
@@ -92,7 +92,11 @@ void StringReader(png_structp png_ptr, png_bytep data, png_size_t length) {
   DecodeContext* const ctx =
       absl::bit_cast<DecodeContext*>(png_get_io_ptr(png_ptr));
   if (static_cast<png_size_t>(ctx->data_left) < length) {
-    memset(data, 0, length);
+    // Don't zero out the data buffer as it has been lazily allocated (copy on
+    // write) and zeroing it out here can produce an OOM. Since the buffer is
+    // only used for reading data from the image, this doesn't result in any
+    // data leak, so it is safe to just leave the buffer be as it is and just
+    // exit with error.
     png_error(png_ptr, "More bytes requested to read than available");
   } else {
     memcpy(data, ctx->data, length);
-- 
GitLab


From bcf22255bdffa682139e9dc3f83550efd6de0cfd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 08:39:38 -0800
Subject: [PATCH 0149/1554] Add header to allows kernel libs to embed
 compilation options as strings.

The new header file platform_strings.h allows kernel loaded via dynamic library
to use TF_PLATFORM_STRINGS() to embed their compilation options in the binary.

The intent is that in the future, TensorFlow will be able to identify which
kernels may be loaded safely.

PiperOrigin-RevId: 221272554
---
 tensorflow/core/BUILD                         |  16 +
 tensorflow/core/platform/platform_strings.h   | 345 ++++++++
 .../core/platform/platform_strings_computed.h | 735 ++++++++++++++++++
 .../core/platform/platform_strings_test.cc    | 178 +++++
 4 files changed, 1274 insertions(+)
 create mode 100644 tensorflow/core/platform/platform_strings.h
 create mode 100644 tensorflow/core/platform/platform_strings_computed.h
 create mode 100644 tensorflow/core/platform/platform_strings_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index a701b38d4b..b7ec445e93 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3421,6 +3421,22 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "platform_strings",
+    srcs = ["platform/platform_strings_computed.h"],
+    hdrs = ["platform/platform_strings.h"],
+)
+
+tf_cc_test(
+    name = "platform_strings_test",
+    size = "small",
+    srcs = ["platform/platform_strings_test.cc"],
+    deps = [
+        ":lib",
+        ":platform_strings",
+    ],
+)
+
 tf_cc_test(
     name = "platform_env_test",
     size = "small",
diff --git a/tensorflow/core/platform/platform_strings.h b/tensorflow/core/platform/platform_strings.h
new file mode 100644
index 0000000000..c9d29e6362
--- /dev/null
+++ b/tensorflow/core/platform/platform_strings.h
@@ -0,0 +1,345 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_
+#define TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_
+
+// This header defines the macro TF_PLATFORM_STRINGS() which should be used
+// once in each dynamically loadable TensorFlow module.  It embeds static
+// strings into the compilation unit that allow TensorFlow to determine what
+// compilation options were in effect when the compilation unit was built.  All
+// compilation units within the same dynamically loadable library should be
+// built with the same options (or at least, the strings should be embedded in
+// the compilation unit built with the most restrictive options).
+
+// Rationale:
+// We wish to load only those libraries that this CPU can execute.  For
+// example, we should not load a library compiled with avx256 instructions on a
+// CPU that cannot execute them.
+//
+// One might think that one could dlopen() the library, and call a routine that
+// would return which cpu type it was compiled for.  Alas, this does not work,
+// because at dlopen() time, a library containing C++ will execute constructors
+// of class variables with static storage class.  Even code that looks
+// innocuous may use optional platform-specific instructions.  For example,
+// the fastest way to zero a region of memory might use optional instructions.
+//
+// One might think one could run a tool such as "objdump" to read flags from
+// the libraries' headers, or perhaps disassemble each library to look for
+// particular instructions.  Unfortunately, the desired flags are not present
+// in the headers, and disassembly can be prohibitively slow ("objdump -d" is
+// very slow, for example).  Moreover, a tool to examine the library may not
+// be present on the system unless the user has installed special packages (for
+// example, on Windows).
+//
+// Instead, we adopt a crude but straightforward solution:  We require
+// developers to use the macro TF_PLATFORM_STRINGS() in their library, to
+// embed the compilation options as constant strings.  The compiler's
+// predefined macros pick which strings are included.  We then search for the
+// strings in the files, and then dlopen() only those libraries that have or
+// lack strings as needed.
+//
+// We adopt the approach of placing in the binary a fairly raw copy of the
+// predefined macros, rather than trying to interpret them in complex ways at
+// compile time.  This allows the loading binary to alter its interpretation of
+// the strings without library developers having to recompile.
+
+#include <stdio.h>
+
+// Aside from the header guard, the internal macros defined here have the form:
+//   TF_PLAT_STR_*
+
+// If a macro is removed from the list of tested macros, the major version in
+// the following version number should be incremented, and the minor version
+// set to zero.  Otherwise, if a macro is added to the list of tested macros,
+// the minor number should be incremented.
+#define TF_PLAT_STR_VERSION_ "1.0"
+
+// Prefix of each option string indicator in the binary.
+// After the prefix, such strings have the form:
+//    [A-Za-z_0-9]=<value>
+// followed by a terminating nul.  To simplify searching, this prefix is all
+// ASCII, starts with a nul, and contains no character twice.
+#define TF_PLAT_STR_MAGIC_PREFIX_ "\0S\\s\":^p*L}"
+
+// A helper macro for TF_PLAT_STR_AS_STR_().
+#define TF_PLAT_STR_STR_1_(x) #x
+
+// Yield a constant string corresponding to x, after macro expansion.
+#define TF_PLAT_STR_AS_STR_(x) TF_PLAT_STR_STR_1_(x)
+
+// An empty definition to make lists more uniform.
+#define TF_PLAT_STR_TERMINATOR_
+
+// TF_PLAT_STR_(x) introduces a constant string indicating whether a
+// particular compilation option has been turned on.
+//
+// In gcc and clang, we might imagine using something like
+// #define TF_PLAT_STR_(x) \
+//     (sizeof (#x) != sizeof (TF_PLAT_STR_AS_STR_ (x))? \
+//      TF_PLAT_STR_MAGIC_PREFIX_ #x "=" TF_PLAT_STR_AS_STR_ (x) : \
+//      TF_PLAT_STR_MAGIC_PREFIX_ #x "=0"),
+// but some compilers (notably MSVC) place both "foo" and "bar" in the binary
+// when presented with
+//    (true?  "foo" : "bar")
+// so we must use #if to select the strings we need, which is rather verbose.
+#define TF_PLAT_STR_(x) TF_PLAT_STR_MAGIC_PREFIX_ #x "=" TF_PLAT_STR_AS_STR_(x)
+
+// Include the #if machinery that sets the macros used below.
+// platform_strings_computed.h can be generated by filtering this header file
+// through:
+// awk '
+// header == "" { print; }
+// /\*\// && header == "" {
+//     print "// Generated from platform_strings.h.";
+//     print "";
+//     print "#ifndef TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_";
+//     print "#define TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_";
+//     print "";
+//     header = 1;
+// }
+// /^#define TF_PLAT_STR_LIST_[a-zA-Z0-9_]*\(\) *\\$/ { active = 1; }
+// /TF_PLAT_STR_TERMINATOR_/ { active = 0; }
+// /^ *TF_PLAT_STR_[A-Za-z0-9_]* *\\$/ && active {
+//     x = $0;
+//     sub(/^ *TF_PLAT_STR_/, "", x);
+//     sub(/ *\\$/, "", x);
+//     printf ("#if defined(%s)\n", x);
+//     printf ("#define TF_PLAT_STR_%s TF_PLAT_STR_(%s)\n", x, x);
+//     printf ("#else\n");
+//     printf ("#define TF_PLAT_STR_%s\n", x);
+//     printf ("#endif\n");
+// }
+// END {
+//     print "";
+//     print "#endif  // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_";
+// }'
+#include "tensorflow/core/platform/platform_strings_computed.h"
+
+// clang-format butchers the following lines.
+// clang-format off
+
+// x86_64 and x86_32 optional features.
+#define TF_PLAT_STR_LIST___x86_64__()                                      \
+        TF_PLAT_STR__M_IX86_FP                                             \
+        TF_PLAT_STR__NO_PREFETCHW                                          \
+        TF_PLAT_STR___3dNOW_A__                                            \
+        TF_PLAT_STR___3dNOW__                                              \
+        TF_PLAT_STR___ABM__                                                \
+        TF_PLAT_STR___ADX__                                                \
+        TF_PLAT_STR___AES__                                                \
+        TF_PLAT_STR___AVX2__                                               \
+        TF_PLAT_STR___AVX512BW__                                           \
+        TF_PLAT_STR___AVX512CD__                                           \
+        TF_PLAT_STR___AVX512DQ__                                           \
+        TF_PLAT_STR___AVX512ER__                                           \
+        TF_PLAT_STR___AVX512F__                                            \
+        TF_PLAT_STR___AVX512IFMA__                                         \
+        TF_PLAT_STR___AVX512PF__                                           \
+        TF_PLAT_STR___AVX512VBMI__                                         \
+        TF_PLAT_STR___AVX512VL__                                           \
+        TF_PLAT_STR___AVX__                                                \
+        TF_PLAT_STR___BMI2__                                               \
+        TF_PLAT_STR___BMI__                                                \
+        TF_PLAT_STR___CLFLUSHOPT__                                         \
+        TF_PLAT_STR___CLZERO__                                             \
+        TF_PLAT_STR___F16C__                                               \
+        TF_PLAT_STR___FMA4__                                               \
+        TF_PLAT_STR___FMA__                                                \
+        TF_PLAT_STR___FP_FAST_FMA                                          \
+        TF_PLAT_STR___FP_FAST_FMAF                                         \
+        TF_PLAT_STR___FSGSBASE__                                           \
+        TF_PLAT_STR___FXSR__                                               \
+        TF_PLAT_STR___LWP__                                                \
+        TF_PLAT_STR___LZCNT__                                              \
+        TF_PLAT_STR___MMX__                                                \
+        TF_PLAT_STR___MWAITX__                                             \
+        TF_PLAT_STR___PCLMUL__                                             \
+        TF_PLAT_STR___PKU__                                                \
+        TF_PLAT_STR___POPCNT__                                             \
+        TF_PLAT_STR___PRFCHW__                                             \
+        TF_PLAT_STR___RDRND__                                              \
+        TF_PLAT_STR___RDSEED__                                             \
+        TF_PLAT_STR___RTM__                                                \
+        TF_PLAT_STR___SHA__                                                \
+        TF_PLAT_STR___SSE2_MATH__                                          \
+        TF_PLAT_STR___SSE2__                                               \
+        TF_PLAT_STR___SSE_MATH__                                           \
+        TF_PLAT_STR___SSE__                                                \
+        TF_PLAT_STR___SSE3__                                               \
+        TF_PLAT_STR___SSE4A__                                              \
+        TF_PLAT_STR___SSE4_1__                                             \
+        TF_PLAT_STR___SSE4_2__                                             \
+        TF_PLAT_STR___SSSE3__                                              \
+        TF_PLAT_STR___TBM__                                                \
+        TF_PLAT_STR___XOP__                                                \
+        TF_PLAT_STR___XSAVEC__                                             \
+        TF_PLAT_STR___XSAVEOPT__                                           \
+        TF_PLAT_STR___XSAVES__                                             \
+        TF_PLAT_STR___XSAVE__                                              \
+        TF_PLAT_STR_TERMINATOR_
+
+// PowerPC (64- and 32-bit) optional features.
+#define TF_PLAT_STR_LIST___powerpc64__()                                   \
+        TF_PLAT_STR__SOFT_DOUBLE                                           \
+        TF_PLAT_STR__SOFT_FLOAT                                            \
+        TF_PLAT_STR___ALTIVEC__                                            \
+        TF_PLAT_STR___APPLE_ALTIVEC__                                      \
+        TF_PLAT_STR___CRYPTO__                                             \
+        TF_PLAT_STR___FLOAT128_HARDWARE__                                  \
+        TF_PLAT_STR___FLOAT128_TYPE__                                      \
+        TF_PLAT_STR___FP_FAST_FMA                                          \
+        TF_PLAT_STR___FP_FAST_FMAF                                         \
+        TF_PLAT_STR___HTM__                                                \
+        TF_PLAT_STR___NO_FPRS__                                            \
+        TF_PLAT_STR___NO_LWSYNC__                                          \
+        TF_PLAT_STR___POWER8_VECTOR__                                      \
+        TF_PLAT_STR___POWER9_VECTOR__                                      \
+        TF_PLAT_STR___PPC405__                                             \
+        TF_PLAT_STR___QUAD_MEMORY_ATOMIC__                                 \
+        TF_PLAT_STR___RECIPF__                                             \
+        TF_PLAT_STR___RECIP_PRECISION__                                    \
+        TF_PLAT_STR___RECIP__                                              \
+        TF_PLAT_STR___RSQRTEF__                                            \
+        TF_PLAT_STR___RSQRTE__                                             \
+        TF_PLAT_STR___TM_FENCE__                                           \
+        TF_PLAT_STR___UPPER_REGS_DF__                                      \
+        TF_PLAT_STR___UPPER_REGS_SF__                                      \
+        TF_PLAT_STR___VEC__                                                \
+        TF_PLAT_STR___VSX__                                                \
+        TF_PLAT_STR_TERMINATOR_
+
+// aarch64 and 32-bit arm optional features
+#define TF_PLAT_STR_LIST___aarch64__()                                     \
+        TF_PLAT_STR___ARM_ARCH                                             \
+        TF_PLAT_STR___ARM_FEATURE_CLZ                                      \
+        TF_PLAT_STR___ARM_FEATURE_CRC32                                    \
+        TF_PLAT_STR___ARM_FEATURE_CRC32                                    \
+        TF_PLAT_STR___ARM_FEATURE_CRYPTO                                   \
+        TF_PLAT_STR___ARM_FEATURE_DIRECTED_ROUNDING                        \
+        TF_PLAT_STR___ARM_FEATURE_DSP                                      \
+        TF_PLAT_STR___ARM_FEATURE_FMA                                      \
+        TF_PLAT_STR___ARM_FEATURE_IDIV                                     \
+        TF_PLAT_STR___ARM_FEATURE_LDREX                                    \
+        TF_PLAT_STR___ARM_FEATURE_NUMERIC_MAXMIN                           \
+        TF_PLAT_STR___ARM_FEATURE_QBIT                                     \
+        TF_PLAT_STR___ARM_FEATURE_QRDMX                                    \
+        TF_PLAT_STR___ARM_FEATURE_SAT                                      \
+        TF_PLAT_STR___ARM_FEATURE_SIMD32                                   \
+        TF_PLAT_STR___ARM_FEATURE_UNALIGNED                                \
+        TF_PLAT_STR___ARM_FP                                               \
+        TF_PLAT_STR___ARM_NEON_FP                                          \
+        TF_PLAT_STR___ARM_NEON__                                           \
+        TF_PLAT_STR___ARM_WMMX                                             \
+        TF_PLAT_STR___IWMMXT2__                                            \
+        TF_PLAT_STR___IWMMXT__                                             \
+        TF_PLAT_STR___VFP_FP__                                             \
+        TF_PLAT_STR_TERMINATOR_
+
+// Generic features, including indication of architecture and OS.
+// The _M_* macros are defined by Visual Studio.
+// It doesn't define __LITTLE_ENDIAN__ or __BYTE_ORDER__;
+// Windows is assumed to be little endian.
+#define TF_PLAT_STR_LIST___generic__()                                     \
+        TF_PLAT_STR_TARGET_IPHONE_SIMULATOR                                \
+        TF_PLAT_STR_TARGET_OS_IOS                                          \
+        TF_PLAT_STR_TARGET_OS_IPHONE                                       \
+        TF_PLAT_STR__MSC_VER                                               \
+        TF_PLAT_STR__M_ARM                                                 \
+        TF_PLAT_STR__M_ARM64                                               \
+        TF_PLAT_STR__M_ARM_ARMV7VE                                         \
+        TF_PLAT_STR__M_ARM_FP                                              \
+        TF_PLAT_STR__M_IX86                                                \
+        TF_PLAT_STR__M_X64                                                 \
+        TF_PLAT_STR__WIN32                                                 \
+        TF_PLAT_STR__WIN64                                                 \
+        TF_PLAT_STR___ANDROID__                                            \
+        TF_PLAT_STR___APPLE__                                              \
+        TF_PLAT_STR___BYTE_ORDER__                                         \
+        TF_PLAT_STR___CYGWIN__                                             \
+        TF_PLAT_STR___FreeBSD__                                            \
+        TF_PLAT_STR___LITTLE_ENDIAN__                                      \
+        TF_PLAT_STR___NetBSD__                                             \
+        TF_PLAT_STR___OpenBSD__                                            \
+        TF_PLAT_STR_____MSYS__                                             \
+        TF_PLAT_STR___aarch64__                                            \
+        TF_PLAT_STR___alpha__                                              \
+        TF_PLAT_STR___arm__                                                \
+        TF_PLAT_STR___i386__                                               \
+        TF_PLAT_STR___i686__                                               \
+        TF_PLAT_STR___ia64__                                               \
+        TF_PLAT_STR___linux__                                              \
+        TF_PLAT_STR___mips32__                                             \
+        TF_PLAT_STR___mips64__                                             \
+        TF_PLAT_STR___powerpc64__                                          \
+        TF_PLAT_STR___powerpc__                                            \
+        TF_PLAT_STR___riscv___                                             \
+        TF_PLAT_STR___s390x__                                              \
+        TF_PLAT_STR___sparc64__                                            \
+        TF_PLAT_STR___sparc__                                              \
+        TF_PLAT_STR___x86_64__                                             \
+        TF_PLAT_STR_TERMINATOR_
+
+#if !defined(__x86_64__) && !defined(_M_X64) && \
+    !defined(__i386__) && !defined(_M_IX86)
+#undef TF_PLAT_STR_LIST___x86_64__
+#define TF_PLAT_STR_LIST___x86_64__()
+#endif
+#if !defined(__powerpc64__)
+#undef TF_PLAT_STR_LIST___powerpc64__
+#define TF_PLAT_STR_LIST___powerpc64__()
+#endif
+#if !defined(__aarch64__) && !defined(_M_ARM64) && \
+    !defined(__arm__) && !defined(_M_ARM)
+#undef TF_PLAT_STR_LIST___aarch64__
+#define TF_PLAT_STR_LIST___aarch64__()
+#endif
+
+// Macro to be used in each dynamically loadable library.
+//
+// The BSS global variable tf_cpu_option_global and the class
+// instance tf_cpu_option_avoid_omit_class are needed to prevent
+// compilers/linkers such as clang from omitting the static variable
+// tf_cpu_option[], which would otherwise appear to be unused.  We cannot make
+// tf_cpu_option[] global, because we then might get multiply-defined symbols
+// if TF_PLAT_STR() is used twice in the same library.
+// (tf_cpu_option_global doesn't see such errors because it is
+// defined in BSS, so multiple definitions are combined by the linker.)  gcc's
+// __attribute__((used)) is insufficient because it seems to be ignored by
+// linkers.
+#define TF_PLATFORM_STRINGS()                                                  \
+    static const char tf_cpu_option[] =                                        \
+        TF_PLAT_STR_MAGIC_PREFIX_ "TF_PLAT_STR_VERSION=" TF_PLAT_STR_VERSION_  \
+        TF_PLAT_STR_LIST___x86_64__()                                          \
+        TF_PLAT_STR_LIST___powerpc64__()                                       \
+        TF_PLAT_STR_LIST___aarch64__()                                         \
+        TF_PLAT_STR_LIST___generic__()                                         \
+    ;                                                                          \
+    const char *tf_cpu_option_global;                                          \
+    namespace {                                                                \
+    class TFCPUOptionHelper {                                                  \
+     public:                                                                   \
+      TFCPUOptionHelper() {                                                    \
+        /* Compilers/linkers remove unused variables aggressively.  The */     \
+        /* following gyrations subvert most such optimizations. */             \
+        tf_cpu_option_global = tf_cpu_option;                                  \
+        /* Nothing is printed because the string starts with a nul. */         \
+        printf("%s", tf_cpu_option);                                           \
+      }                                                                        \
+    } tf_cpu_option_avoid_omit_class;                                          \
+    }  /* anonymous namespace */
+// clang-format on
+
+#endif  // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_
diff --git a/tensorflow/core/platform/platform_strings_computed.h b/tensorflow/core/platform/platform_strings_computed.h
new file mode 100644
index 0000000000..6a17f3bfc3
--- /dev/null
+++ b/tensorflow/core/platform/platform_strings_computed.h
@@ -0,0 +1,735 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// Generated from platform_strings.h.
+
+#ifndef TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_
+#define TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_
+
+#if defined(_M_IX86_FP)
+#define TF_PLAT_STR__M_IX86_FP TF_PLAT_STR_(_M_IX86_FP)
+#else
+#define TF_PLAT_STR__M_IX86_FP
+#endif
+#if defined(_NO_PREFETCHW)
+#define TF_PLAT_STR__NO_PREFETCHW TF_PLAT_STR_(_NO_PREFETCHW)
+#else
+#define TF_PLAT_STR__NO_PREFETCHW
+#endif
+#if defined(__3dNOW_A__)
+#define TF_PLAT_STR___3dNOW_A__ TF_PLAT_STR_(__3dNOW_A__)
+#else
+#define TF_PLAT_STR___3dNOW_A__
+#endif
+#if defined(__3dNOW__)
+#define TF_PLAT_STR___3dNOW__ TF_PLAT_STR_(__3dNOW__)
+#else
+#define TF_PLAT_STR___3dNOW__
+#endif
+#if defined(__ABM__)
+#define TF_PLAT_STR___ABM__ TF_PLAT_STR_(__ABM__)
+#else
+#define TF_PLAT_STR___ABM__
+#endif
+#if defined(__ADX__)
+#define TF_PLAT_STR___ADX__ TF_PLAT_STR_(__ADX__)
+#else
+#define TF_PLAT_STR___ADX__
+#endif
+#if defined(__AES__)
+#define TF_PLAT_STR___AES__ TF_PLAT_STR_(__AES__)
+#else
+#define TF_PLAT_STR___AES__
+#endif
+#if defined(__AVX2__)
+#define TF_PLAT_STR___AVX2__ TF_PLAT_STR_(__AVX2__)
+#else
+#define TF_PLAT_STR___AVX2__
+#endif
+#if defined(__AVX512BW__)
+#define TF_PLAT_STR___AVX512BW__ TF_PLAT_STR_(__AVX512BW__)
+#else
+#define TF_PLAT_STR___AVX512BW__
+#endif
+#if defined(__AVX512CD__)
+#define TF_PLAT_STR___AVX512CD__ TF_PLAT_STR_(__AVX512CD__)
+#else
+#define TF_PLAT_STR___AVX512CD__
+#endif
+#if defined(__AVX512DQ__)
+#define TF_PLAT_STR___AVX512DQ__ TF_PLAT_STR_(__AVX512DQ__)
+#else
+#define TF_PLAT_STR___AVX512DQ__
+#endif
+#if defined(__AVX512ER__)
+#define TF_PLAT_STR___AVX512ER__ TF_PLAT_STR_(__AVX512ER__)
+#else
+#define TF_PLAT_STR___AVX512ER__
+#endif
+#if defined(__AVX512F__)
+#define TF_PLAT_STR___AVX512F__ TF_PLAT_STR_(__AVX512F__)
+#else
+#define TF_PLAT_STR___AVX512F__
+#endif
+#if defined(__AVX512IFMA__)
+#define TF_PLAT_STR___AVX512IFMA__ TF_PLAT_STR_(__AVX512IFMA__)
+#else
+#define TF_PLAT_STR___AVX512IFMA__
+#endif
+#if defined(__AVX512PF__)
+#define TF_PLAT_STR___AVX512PF__ TF_PLAT_STR_(__AVX512PF__)
+#else
+#define TF_PLAT_STR___AVX512PF__
+#endif
+#if defined(__AVX512VBMI__)
+#define TF_PLAT_STR___AVX512VBMI__ TF_PLAT_STR_(__AVX512VBMI__)
+#else
+#define TF_PLAT_STR___AVX512VBMI__
+#endif
+#if defined(__AVX512VL__)
+#define TF_PLAT_STR___AVX512VL__ TF_PLAT_STR_(__AVX512VL__)
+#else
+#define TF_PLAT_STR___AVX512VL__
+#endif
+#if defined(__AVX__)
+#define TF_PLAT_STR___AVX__ TF_PLAT_STR_(__AVX__)
+#else
+#define TF_PLAT_STR___AVX__
+#endif
+#if defined(__BMI2__)
+#define TF_PLAT_STR___BMI2__ TF_PLAT_STR_(__BMI2__)
+#else
+#define TF_PLAT_STR___BMI2__
+#endif
+#if defined(__BMI__)
+#define TF_PLAT_STR___BMI__ TF_PLAT_STR_(__BMI__)
+#else
+#define TF_PLAT_STR___BMI__
+#endif
+#if defined(__CLFLUSHOPT__)
+#define TF_PLAT_STR___CLFLUSHOPT__ TF_PLAT_STR_(__CLFLUSHOPT__)
+#else
+#define TF_PLAT_STR___CLFLUSHOPT__
+#endif
+#if defined(__CLZERO__)
+#define TF_PLAT_STR___CLZERO__ TF_PLAT_STR_(__CLZERO__)
+#else
+#define TF_PLAT_STR___CLZERO__
+#endif
+#if defined(__F16C__)
+#define TF_PLAT_STR___F16C__ TF_PLAT_STR_(__F16C__)
+#else
+#define TF_PLAT_STR___F16C__
+#endif
+#if defined(__FMA4__)
+#define TF_PLAT_STR___FMA4__ TF_PLAT_STR_(__FMA4__)
+#else
+#define TF_PLAT_STR___FMA4__
+#endif
+#if defined(__FMA__)
+#define TF_PLAT_STR___FMA__ TF_PLAT_STR_(__FMA__)
+#else
+#define TF_PLAT_STR___FMA__
+#endif
+#if defined(__FP_FAST_FMA)
+#define TF_PLAT_STR___FP_FAST_FMA TF_PLAT_STR_(__FP_FAST_FMA)
+#else
+#define TF_PLAT_STR___FP_FAST_FMA
+#endif
+#if defined(__FP_FAST_FMAF)
+#define TF_PLAT_STR___FP_FAST_FMAF TF_PLAT_STR_(__FP_FAST_FMAF)
+#else
+#define TF_PLAT_STR___FP_FAST_FMAF
+#endif
+#if defined(__FSGSBASE__)
+#define TF_PLAT_STR___FSGSBASE__ TF_PLAT_STR_(__FSGSBASE__)
+#else
+#define TF_PLAT_STR___FSGSBASE__
+#endif
+#if defined(__FXSR__)
+#define TF_PLAT_STR___FXSR__ TF_PLAT_STR_(__FXSR__)
+#else
+#define TF_PLAT_STR___FXSR__
+#endif
+#if defined(__LWP__)
+#define TF_PLAT_STR___LWP__ TF_PLAT_STR_(__LWP__)
+#else
+#define TF_PLAT_STR___LWP__
+#endif
+#if defined(__LZCNT__)
+#define TF_PLAT_STR___LZCNT__ TF_PLAT_STR_(__LZCNT__)
+#else
+#define TF_PLAT_STR___LZCNT__
+#endif
+#if defined(__MMX__)
+#define TF_PLAT_STR___MMX__ TF_PLAT_STR_(__MMX__)
+#else
+#define TF_PLAT_STR___MMX__
+#endif
+#if defined(__MWAITX__)
+#define TF_PLAT_STR___MWAITX__ TF_PLAT_STR_(__MWAITX__)
+#else
+#define TF_PLAT_STR___MWAITX__
+#endif
+#if defined(__PCLMUL__)
+#define TF_PLAT_STR___PCLMUL__ TF_PLAT_STR_(__PCLMUL__)
+#else
+#define TF_PLAT_STR___PCLMUL__
+#endif
+#if defined(__PKU__)
+#define TF_PLAT_STR___PKU__ TF_PLAT_STR_(__PKU__)
+#else
+#define TF_PLAT_STR___PKU__
+#endif
+#if defined(__POPCNT__)
+#define TF_PLAT_STR___POPCNT__ TF_PLAT_STR_(__POPCNT__)
+#else
+#define TF_PLAT_STR___POPCNT__
+#endif
+#if defined(__PRFCHW__)
+#define TF_PLAT_STR___PRFCHW__ TF_PLAT_STR_(__PRFCHW__)
+#else
+#define TF_PLAT_STR___PRFCHW__
+#endif
+#if defined(__RDRND__)
+#define TF_PLAT_STR___RDRND__ TF_PLAT_STR_(__RDRND__)
+#else
+#define TF_PLAT_STR___RDRND__
+#endif
+#if defined(__RDSEED__)
+#define TF_PLAT_STR___RDSEED__ TF_PLAT_STR_(__RDSEED__)
+#else
+#define TF_PLAT_STR___RDSEED__
+#endif
+#if defined(__RTM__)
+#define TF_PLAT_STR___RTM__ TF_PLAT_STR_(__RTM__)
+#else
+#define TF_PLAT_STR___RTM__
+#endif
+#if defined(__SHA__)
+#define TF_PLAT_STR___SHA__ TF_PLAT_STR_(__SHA__)
+#else
+#define TF_PLAT_STR___SHA__
+#endif
+#if defined(__SSE2_MATH__)
+#define TF_PLAT_STR___SSE2_MATH__ TF_PLAT_STR_(__SSE2_MATH__)
+#else
+#define TF_PLAT_STR___SSE2_MATH__
+#endif
+#if defined(__SSE2__)
+#define TF_PLAT_STR___SSE2__ TF_PLAT_STR_(__SSE2__)
+#else
+#define TF_PLAT_STR___SSE2__
+#endif
+#if defined(__SSE_MATH__)
+#define TF_PLAT_STR___SSE_MATH__ TF_PLAT_STR_(__SSE_MATH__)
+#else
+#define TF_PLAT_STR___SSE_MATH__
+#endif
+#if defined(__SSE__)
+#define TF_PLAT_STR___SSE__ TF_PLAT_STR_(__SSE__)
+#else
+#define TF_PLAT_STR___SSE__
+#endif
+#if defined(__SSE3__)
+#define TF_PLAT_STR___SSE3__ TF_PLAT_STR_(__SSE3__)
+#else
+#define TF_PLAT_STR___SSE3__
+#endif
+#if defined(__SSE4A__)
+#define TF_PLAT_STR___SSE4A__ TF_PLAT_STR_(__SSE4A__)
+#else
+#define TF_PLAT_STR___SSE4A__
+#endif
+#if defined(__SSE4_1__)
+#define TF_PLAT_STR___SSE4_1__ TF_PLAT_STR_(__SSE4_1__)
+#else
+#define TF_PLAT_STR___SSE4_1__
+#endif
+#if defined(__SSE4_2__)
+#define TF_PLAT_STR___SSE4_2__ TF_PLAT_STR_(__SSE4_2__)
+#else
+#define TF_PLAT_STR___SSE4_2__
+#endif
+#if defined(__SSSE3__)
+#define TF_PLAT_STR___SSSE3__ TF_PLAT_STR_(__SSSE3__)
+#else
+#define TF_PLAT_STR___SSSE3__
+#endif
+#if defined(__TBM__)
+#define TF_PLAT_STR___TBM__ TF_PLAT_STR_(__TBM__)
+#else
+#define TF_PLAT_STR___TBM__
+#endif
+#if defined(__XOP__)
+#define TF_PLAT_STR___XOP__ TF_PLAT_STR_(__XOP__)
+#else
+#define TF_PLAT_STR___XOP__
+#endif
+#if defined(__XSAVEC__)
+#define TF_PLAT_STR___XSAVEC__ TF_PLAT_STR_(__XSAVEC__)
+#else
+#define TF_PLAT_STR___XSAVEC__
+#endif
+#if defined(__XSAVEOPT__)
+#define TF_PLAT_STR___XSAVEOPT__ TF_PLAT_STR_(__XSAVEOPT__)
+#else
+#define TF_PLAT_STR___XSAVEOPT__
+#endif
+#if defined(__XSAVES__)
+#define TF_PLAT_STR___XSAVES__ TF_PLAT_STR_(__XSAVES__)
+#else
+#define TF_PLAT_STR___XSAVES__
+#endif
+#if defined(__XSAVE__)
+#define TF_PLAT_STR___XSAVE__ TF_PLAT_STR_(__XSAVE__)
+#else
+#define TF_PLAT_STR___XSAVE__
+#endif
+#if defined(_SOFT_DOUBLE)
+#define TF_PLAT_STR__SOFT_DOUBLE TF_PLAT_STR_(_SOFT_DOUBLE)
+#else
+#define TF_PLAT_STR__SOFT_DOUBLE
+#endif
+#if defined(_SOFT_FLOAT)
+#define TF_PLAT_STR__SOFT_FLOAT TF_PLAT_STR_(_SOFT_FLOAT)
+#else
+#define TF_PLAT_STR__SOFT_FLOAT
+#endif
+#if defined(__ALTIVEC__)
+#define TF_PLAT_STR___ALTIVEC__ TF_PLAT_STR_(__ALTIVEC__)
+#else
+#define TF_PLAT_STR___ALTIVEC__
+#endif
+#if defined(__APPLE_ALTIVEC__)
+#define TF_PLAT_STR___APPLE_ALTIVEC__ TF_PLAT_STR_(__APPLE_ALTIVEC__)
+#else
+#define TF_PLAT_STR___APPLE_ALTIVEC__
+#endif
+#if defined(__CRYPTO__)
+#define TF_PLAT_STR___CRYPTO__ TF_PLAT_STR_(__CRYPTO__)
+#else
+#define TF_PLAT_STR___CRYPTO__
+#endif
+#if defined(__FLOAT128_HARDWARE__)
+#define TF_PLAT_STR___FLOAT128_HARDWARE__ TF_PLAT_STR_(__FLOAT128_HARDWARE__)
+#else
+#define TF_PLAT_STR___FLOAT128_HARDWARE__
+#endif
+#if defined(__FLOAT128_TYPE__)
+#define TF_PLAT_STR___FLOAT128_TYPE__ TF_PLAT_STR_(__FLOAT128_TYPE__)
+#else
+#define TF_PLAT_STR___FLOAT128_TYPE__
+#endif
+#if defined(__FP_FAST_FMA)
+#define TF_PLAT_STR___FP_FAST_FMA TF_PLAT_STR_(__FP_FAST_FMA)
+#else
+#define TF_PLAT_STR___FP_FAST_FMA
+#endif
+#if defined(__FP_FAST_FMAF)
+#define TF_PLAT_STR___FP_FAST_FMAF TF_PLAT_STR_(__FP_FAST_FMAF)
+#else
+#define TF_PLAT_STR___FP_FAST_FMAF
+#endif
+#if defined(__HTM__)
+#define TF_PLAT_STR___HTM__ TF_PLAT_STR_(__HTM__)
+#else
+#define TF_PLAT_STR___HTM__
+#endif
+#if defined(__NO_FPRS__)
+#define TF_PLAT_STR___NO_FPRS__ TF_PLAT_STR_(__NO_FPRS__)
+#else
+#define TF_PLAT_STR___NO_FPRS__
+#endif
+#if defined(__NO_LWSYNC__)
+#define TF_PLAT_STR___NO_LWSYNC__ TF_PLAT_STR_(__NO_LWSYNC__)
+#else
+#define TF_PLAT_STR___NO_LWSYNC__
+#endif
+#if defined(__POWER8_VECTOR__)
+#define TF_PLAT_STR___POWER8_VECTOR__ TF_PLAT_STR_(__POWER8_VECTOR__)
+#else
+#define TF_PLAT_STR___POWER8_VECTOR__
+#endif
+#if defined(__POWER9_VECTOR__)
+#define TF_PLAT_STR___POWER9_VECTOR__ TF_PLAT_STR_(__POWER9_VECTOR__)
+#else
+#define TF_PLAT_STR___POWER9_VECTOR__
+#endif
+#if defined(__PPC405__)
+#define TF_PLAT_STR___PPC405__ TF_PLAT_STR_(__PPC405__)
+#else
+#define TF_PLAT_STR___PPC405__
+#endif
+#if defined(__QUAD_MEMORY_ATOMIC__)
+#define TF_PLAT_STR___QUAD_MEMORY_ATOMIC__ TF_PLAT_STR_(__QUAD_MEMORY_ATOMIC__)
+#else
+#define TF_PLAT_STR___QUAD_MEMORY_ATOMIC__
+#endif
+#if defined(__RECIPF__)
+#define TF_PLAT_STR___RECIPF__ TF_PLAT_STR_(__RECIPF__)
+#else
+#define TF_PLAT_STR___RECIPF__
+#endif
+#if defined(__RECIP_PRECISION__)
+#define TF_PLAT_STR___RECIP_PRECISION__ TF_PLAT_STR_(__RECIP_PRECISION__)
+#else
+#define TF_PLAT_STR___RECIP_PRECISION__
+#endif
+#if defined(__RECIP__)
+#define TF_PLAT_STR___RECIP__ TF_PLAT_STR_(__RECIP__)
+#else
+#define TF_PLAT_STR___RECIP__
+#endif
+#if defined(__RSQRTEF__)
+#define TF_PLAT_STR___RSQRTEF__ TF_PLAT_STR_(__RSQRTEF__)
+#else
+#define TF_PLAT_STR___RSQRTEF__
+#endif
+#if defined(__RSQRTE__)
+#define TF_PLAT_STR___RSQRTE__ TF_PLAT_STR_(__RSQRTE__)
+#else
+#define TF_PLAT_STR___RSQRTE__
+#endif
+#if defined(__TM_FENCE__)
+#define TF_PLAT_STR___TM_FENCE__ TF_PLAT_STR_(__TM_FENCE__)
+#else
+#define TF_PLAT_STR___TM_FENCE__
+#endif
+#if defined(__UPPER_REGS_DF__)
+#define TF_PLAT_STR___UPPER_REGS_DF__ TF_PLAT_STR_(__UPPER_REGS_DF__)
+#else
+#define TF_PLAT_STR___UPPER_REGS_DF__
+#endif
+#if defined(__UPPER_REGS_SF__)
+#define TF_PLAT_STR___UPPER_REGS_SF__ TF_PLAT_STR_(__UPPER_REGS_SF__)
+#else
+#define TF_PLAT_STR___UPPER_REGS_SF__
+#endif
+#if defined(__VEC__)
+#define TF_PLAT_STR___VEC__ TF_PLAT_STR_(__VEC__)
+#else
+#define TF_PLAT_STR___VEC__
+#endif
+#if defined(__VSX__)
+#define TF_PLAT_STR___VSX__ TF_PLAT_STR_(__VSX__)
+#else
+#define TF_PLAT_STR___VSX__
+#endif
+#if defined(__ARM_ARCH)
+#define TF_PLAT_STR___ARM_ARCH TF_PLAT_STR_(__ARM_ARCH)
+#else
+#define TF_PLAT_STR___ARM_ARCH
+#endif
+#if defined(__ARM_FEATURE_CLZ)
+#define TF_PLAT_STR___ARM_FEATURE_CLZ TF_PLAT_STR_(__ARM_FEATURE_CLZ)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_CLZ
+#endif
+#if defined(__ARM_FEATURE_CRC32)
+#define TF_PLAT_STR___ARM_FEATURE_CRC32 TF_PLAT_STR_(__ARM_FEATURE_CRC32)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_CRC32
+#endif
+#if defined(__ARM_FEATURE_CRC32)
+#define TF_PLAT_STR___ARM_FEATURE_CRC32 TF_PLAT_STR_(__ARM_FEATURE_CRC32)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_CRC32
+#endif
+#if defined(__ARM_FEATURE_CRYPTO)
+#define TF_PLAT_STR___ARM_FEATURE_CRYPTO TF_PLAT_STR_(__ARM_FEATURE_CRYPTO)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_CRYPTO
+#endif
+#if defined(__ARM_FEATURE_DIRECTED_ROUNDING)
+#define TF_PLAT_STR___ARM_FEATURE_DIRECTED_ROUNDING \
+  TF_PLAT_STR_(__ARM_FEATURE_DIRECTED_ROUNDING)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_DIRECTED_ROUNDING
+#endif
+#if defined(__ARM_FEATURE_DSP)
+#define TF_PLAT_STR___ARM_FEATURE_DSP TF_PLAT_STR_(__ARM_FEATURE_DSP)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_DSP
+#endif
+#if defined(__ARM_FEATURE_FMA)
+#define TF_PLAT_STR___ARM_FEATURE_FMA TF_PLAT_STR_(__ARM_FEATURE_FMA)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_FMA
+#endif
+#if defined(__ARM_FEATURE_IDIV)
+#define TF_PLAT_STR___ARM_FEATURE_IDIV TF_PLAT_STR_(__ARM_FEATURE_IDIV)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_IDIV
+#endif
+#if defined(__ARM_FEATURE_LDREX)
+#define TF_PLAT_STR___ARM_FEATURE_LDREX TF_PLAT_STR_(__ARM_FEATURE_LDREX)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_LDREX
+#endif
+#if defined(__ARM_FEATURE_NUMERIC_MAXMIN)
+#define TF_PLAT_STR___ARM_FEATURE_NUMERIC_MAXMIN \
+  TF_PLAT_STR_(__ARM_FEATURE_NUMERIC_MAXMIN)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_NUMERIC_MAXMIN
+#endif
+#if defined(__ARM_FEATURE_QBIT)
+#define TF_PLAT_STR___ARM_FEATURE_QBIT TF_PLAT_STR_(__ARM_FEATURE_QBIT)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_QBIT
+#endif
+#if defined(__ARM_FEATURE_QRDMX)
+#define TF_PLAT_STR___ARM_FEATURE_QRDMX TF_PLAT_STR_(__ARM_FEATURE_QRDMX)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_QRDMX
+#endif
+#if defined(__ARM_FEATURE_SAT)
+#define TF_PLAT_STR___ARM_FEATURE_SAT TF_PLAT_STR_(__ARM_FEATURE_SAT)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_SAT
+#endif
+#if defined(__ARM_FEATURE_SIMD32)
+#define TF_PLAT_STR___ARM_FEATURE_SIMD32 TF_PLAT_STR_(__ARM_FEATURE_SIMD32)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_SIMD32
+#endif
+#if defined(__ARM_FEATURE_UNALIGNED)
+#define TF_PLAT_STR___ARM_FEATURE_UNALIGNED \
+  TF_PLAT_STR_(__ARM_FEATURE_UNALIGNED)
+#else
+#define TF_PLAT_STR___ARM_FEATURE_UNALIGNED
+#endif
+#if defined(__ARM_FP)
+#define TF_PLAT_STR___ARM_FP TF_PLAT_STR_(__ARM_FP)
+#else
+#define TF_PLAT_STR___ARM_FP
+#endif
+#if defined(__ARM_NEON_FP)
+#define TF_PLAT_STR___ARM_NEON_FP TF_PLAT_STR_(__ARM_NEON_FP)
+#else
+#define TF_PLAT_STR___ARM_NEON_FP
+#endif
+#if defined(__ARM_NEON__)
+#define TF_PLAT_STR___ARM_NEON__ TF_PLAT_STR_(__ARM_NEON__)
+#else
+#define TF_PLAT_STR___ARM_NEON__
+#endif
+#if defined(__ARM_WMMX)
+#define TF_PLAT_STR___ARM_WMMX TF_PLAT_STR_(__ARM_WMMX)
+#else
+#define TF_PLAT_STR___ARM_WMMX
+#endif
+#if defined(__IWMMXT2__)
+#define TF_PLAT_STR___IWMMXT2__ TF_PLAT_STR_(__IWMMXT2__)
+#else
+#define TF_PLAT_STR___IWMMXT2__
+#endif
+#if defined(__IWMMXT__)
+#define TF_PLAT_STR___IWMMXT__ TF_PLAT_STR_(__IWMMXT__)
+#else
+#define TF_PLAT_STR___IWMMXT__
+#endif
+#if defined(__VFP_FP__)
+#define TF_PLAT_STR___VFP_FP__ TF_PLAT_STR_(__VFP_FP__)
+#else
+#define TF_PLAT_STR___VFP_FP__
+#endif
+#if defined(TARGET_IPHONE_SIMULATOR)
+#define TF_PLAT_STR_TARGET_IPHONE_SIMULATOR \
+  TF_PLAT_STR_(TARGET_IPHONE_SIMULATOR)
+#else
+#define TF_PLAT_STR_TARGET_IPHONE_SIMULATOR
+#endif
+#if defined(TARGET_OS_IOS)
+#define TF_PLAT_STR_TARGET_OS_IOS TF_PLAT_STR_(TARGET_OS_IOS)
+#else
+#define TF_PLAT_STR_TARGET_OS_IOS
+#endif
+#if defined(TARGET_OS_IPHONE)
+#define TF_PLAT_STR_TARGET_OS_IPHONE TF_PLAT_STR_(TARGET_OS_IPHONE)
+#else
+#define TF_PLAT_STR_TARGET_OS_IPHONE
+#endif
+#if defined(_MSC_VER)
+#define TF_PLAT_STR__MSC_VER TF_PLAT_STR_(_MSC_VER)
+#else
+#define TF_PLAT_STR__MSC_VER
+#endif
+#if defined(_M_ARM)
+#define TF_PLAT_STR__M_ARM TF_PLAT_STR_(_M_ARM)
+#else
+#define TF_PLAT_STR__M_ARM
+#endif
+#if defined(_M_ARM64)
+#define TF_PLAT_STR__M_ARM64 TF_PLAT_STR_(_M_ARM64)
+#else
+#define TF_PLAT_STR__M_ARM64
+#endif
+#if defined(_M_ARM_ARMV7VE)
+#define TF_PLAT_STR__M_ARM_ARMV7VE TF_PLAT_STR_(_M_ARM_ARMV7VE)
+#else
+#define TF_PLAT_STR__M_ARM_ARMV7VE
+#endif
+#if defined(_M_ARM_FP)
+#define TF_PLAT_STR__M_ARM_FP TF_PLAT_STR_(_M_ARM_FP)
+#else
+#define TF_PLAT_STR__M_ARM_FP
+#endif
+#if defined(_M_IX86)
+#define TF_PLAT_STR__M_IX86 TF_PLAT_STR_(_M_IX86)
+#else
+#define TF_PLAT_STR__M_IX86
+#endif
+#if defined(_M_X64)
+#define TF_PLAT_STR__M_X64 TF_PLAT_STR_(_M_X64)
+#else
+#define TF_PLAT_STR__M_X64
+#endif
+#if defined(_WIN32)
+#define TF_PLAT_STR__WIN32 TF_PLAT_STR_(_WIN32)
+#else
+#define TF_PLAT_STR__WIN32
+#endif
+#if defined(_WIN64)
+#define TF_PLAT_STR__WIN64 TF_PLAT_STR_(_WIN64)
+#else
+#define TF_PLAT_STR__WIN64
+#endif
+#if defined(__ANDROID__)
+#define TF_PLAT_STR___ANDROID__ TF_PLAT_STR_(__ANDROID__)
+#else
+#define TF_PLAT_STR___ANDROID__
+#endif
+#if defined(__APPLE__)
+#define TF_PLAT_STR___APPLE__ TF_PLAT_STR_(__APPLE__)
+#else
+#define TF_PLAT_STR___APPLE__
+#endif
+#if defined(__BYTE_ORDER__)
+#define TF_PLAT_STR___BYTE_ORDER__ TF_PLAT_STR_(__BYTE_ORDER__)
+#else
+#define TF_PLAT_STR___BYTE_ORDER__
+#endif
+#if defined(__CYGWIN__)
+#define TF_PLAT_STR___CYGWIN__ TF_PLAT_STR_(__CYGWIN__)
+#else
+#define TF_PLAT_STR___CYGWIN__
+#endif
+#if defined(__FreeBSD__)
+#define TF_PLAT_STR___FreeBSD__ TF_PLAT_STR_(__FreeBSD__)
+#else
+#define TF_PLAT_STR___FreeBSD__
+#endif
+#if defined(__LITTLE_ENDIAN__)
+#define TF_PLAT_STR___LITTLE_ENDIAN__ TF_PLAT_STR_(__LITTLE_ENDIAN__)
+#else
+#define TF_PLAT_STR___LITTLE_ENDIAN__
+#endif
+#if defined(__NetBSD__)
+#define TF_PLAT_STR___NetBSD__ TF_PLAT_STR_(__NetBSD__)
+#else
+#define TF_PLAT_STR___NetBSD__
+#endif
+#if defined(__OpenBSD__)
+#define TF_PLAT_STR___OpenBSD__ TF_PLAT_STR_(__OpenBSD__)
+#else
+#define TF_PLAT_STR___OpenBSD__
+#endif
+#if defined(____MSYS__)
+#define TF_PLAT_STR_____MSYS__ TF_PLAT_STR_(____MSYS__)
+#else
+#define TF_PLAT_STR_____MSYS__
+#endif
+#if defined(__aarch64__)
+#define TF_PLAT_STR___aarch64__ TF_PLAT_STR_(__aarch64__)
+#else
+#define TF_PLAT_STR___aarch64__
+#endif
+#if defined(__alpha__)
+#define TF_PLAT_STR___alpha__ TF_PLAT_STR_(__alpha__)
+#else
+#define TF_PLAT_STR___alpha__
+#endif
+#if defined(__arm__)
+#define TF_PLAT_STR___arm__ TF_PLAT_STR_(__arm__)
+#else
+#define TF_PLAT_STR___arm__
+#endif
+#if defined(__i386__)
+#define TF_PLAT_STR___i386__ TF_PLAT_STR_(__i386__)
+#else
+#define TF_PLAT_STR___i386__
+#endif
+#if defined(__i686__)
+#define TF_PLAT_STR___i686__ TF_PLAT_STR_(__i686__)
+#else
+#define TF_PLAT_STR___i686__
+#endif
+#if defined(__ia64__)
+#define TF_PLAT_STR___ia64__ TF_PLAT_STR_(__ia64__)
+#else
+#define TF_PLAT_STR___ia64__
+#endif
+#if defined(__linux__)
+#define TF_PLAT_STR___linux__ TF_PLAT_STR_(__linux__)
+#else
+#define TF_PLAT_STR___linux__
+#endif
+#if defined(__mips32__)
+#define TF_PLAT_STR___mips32__ TF_PLAT_STR_(__mips32__)
+#else
+#define TF_PLAT_STR___mips32__
+#endif
+#if defined(__mips64__)
+#define TF_PLAT_STR___mips64__ TF_PLAT_STR_(__mips64__)
+#else
+#define TF_PLAT_STR___mips64__
+#endif
+#if defined(__powerpc64__)
+#define TF_PLAT_STR___powerpc64__ TF_PLAT_STR_(__powerpc64__)
+#else
+#define TF_PLAT_STR___powerpc64__
+#endif
+#if defined(__powerpc__)
+#define TF_PLAT_STR___powerpc__ TF_PLAT_STR_(__powerpc__)
+#else
+#define TF_PLAT_STR___powerpc__
+#endif
+#if defined(__riscv___)
+#define TF_PLAT_STR___riscv___ TF_PLAT_STR_(__riscv___)
+#else
+#define TF_PLAT_STR___riscv___
+#endif
+#if defined(__s390x__)
+#define TF_PLAT_STR___s390x__ TF_PLAT_STR_(__s390x__)
+#else
+#define TF_PLAT_STR___s390x__
+#endif
+#if defined(__sparc64__)
+#define TF_PLAT_STR___sparc64__ TF_PLAT_STR_(__sparc64__)
+#else
+#define TF_PLAT_STR___sparc64__
+#endif
+#if defined(__sparc__)
+#define TF_PLAT_STR___sparc__ TF_PLAT_STR_(__sparc__)
+#else
+#define TF_PLAT_STR___sparc__
+#endif
+#if defined(__x86_64__)
+#define TF_PLAT_STR___x86_64__ TF_PLAT_STR_(__x86_64__)
+#else
+#define TF_PLAT_STR___x86_64__
+#endif
+
+#endif  // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_COMPUTED_H_
diff --git a/tensorflow/core/platform/platform_strings_test.cc b/tensorflow/core/platform/platform_strings_test.cc
new file mode 100644
index 0000000000..9584333bb8
--- /dev/null
+++ b/tensorflow/core/platform/platform_strings_test.cc
@@ -0,0 +1,178 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Test for the platform_strings.h header file.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/platform_strings.h"
+
+// Embed the platform strings in this binary.
+TF_PLATFORM_STRINGS()
+
+// A vector of strings.
+typedef std::vector<std::string> string_vec;
+
+// Append to *found the strings within the named file with the platform_strings
+// magic prefix, and return true; or return false on error.
+static bool GetStrings(const std::string &file_name, string_vec *found) {
+  bool result = false;
+  FILE *ifp = fopen(file_name.c_str(), "rb");
+  if (ifp != nullptr) {
+    static const char prefix[] = TF_PLAT_STR_MAGIC_PREFIX_;
+    int first_char = prefix[1];
+    int last_char = -1;
+    int c;
+    while ((c = getc(ifp)) != EOF) {
+      if (c == first_char && last_char == 0) {
+        int i = 2;
+        while (prefix[i] != 0 && (c = getc(ifp)) == prefix[i]) {
+          i++;
+        }
+        if (prefix[i] == 0) {
+          std::string str;
+          while ((c = getc(ifp)) != EOF && c != 0) {
+            str.push_back(c);
+          }
+          if (!str.empty()) {
+            found->push_back(str);
+          }
+        }
+      }
+      last_char = c;
+    }
+
+    result = (ferror(ifp) == 0);
+    fclose(ifp);
+  }
+  return result;
+}
+
+// Print the platform strings embedded in the binary file_name and return 0,
+// on on error return 2.
+static int PrintStrings(const std::string file_name) {
+  int rc = 0;
+  string_vec str;
+  if (GetStrings(file_name, &str)) {
+    for (int i = 0; i != str.size(); i++) {
+      printf("%s\n", str[i].c_str());
+    }
+  } else {
+    perror(file_name.c_str());
+    rc = 2;
+  }
+  return rc;
+}
+
+// Return whether str[] conatins a string with prefix "macro_name="; if so,
+// set *pvalue to the suffix.
+static bool GetValue(const string_vec &str, const std::string &macro_name,
+                     std::string *pvalue) {
+  std::string nam_eq = macro_name + "=";
+  int i = 0;
+  while (i != str.size() && !tensorflow::str_util::StartsWith(str[i], nam_eq)) {
+    i++;
+  }
+  bool found = (i != str.size());
+  if (found) {
+    *pvalue = str[i].substr(nam_eq.size());
+  }
+  return found;
+}
+
+// If macro_name[] is not equal to value[], check that str[] contains the
+// string "macro_name=value".  Otherwise, check that str[] does not contain any
+// string starting with macro_name=".
+static void CheckStr(const string_vec &str, const std::string &macro_name,
+                     const std::string &value) {
+  std::string value_from_str;
+  if (GetValue(str, macro_name, &value_from_str)) {
+    if (value != value_from_str) {
+      // Output everything found, to aid debugging.
+      LOG(ERROR) << "===== value=" << value
+                 << "  value_from_str=" << value_from_str;
+      for (int i = 0; i != str.size(); i++) {
+        LOG(ERROR) << "% " << str[i];
+      }
+      LOG(ERROR) << "=====";
+    }
+    CHECK_EQ(value, value_from_str) << " " << macro_name << ": bad value";
+  } else {
+    // If the string is not found, we expect value to be macro_name.
+    if (value != macro_name) {
+      // Output everything found, to aid debugging.
+      LOG(ERROR) << "===== value=" << value << "  macro_name=" << macro_name;
+      for (int i = 0; i != str.size(); i++) {
+        LOG(ERROR) << "% " << str[i];
+      }
+      LOG(ERROR) << "=====";
+    }
+    CHECK_EQ(value, macro_name) << " " << macro_name << ": not found in binary";
+  }
+}
+
+// Helper for AS_STR(), below, to perform macro expansion.
+#define AS_STR_1_(x) #x
+
+// Yield x after macro expansion as a nul-terminated constant string.
+#define AS_STR(x) AS_STR_1_(x)
+
+// Run the test, and return 0 on success, 2 otherwise.
+static int RunTest(const std::string &binary_name) {
+  int rc = 0;
+  string_vec str;
+
+  if (GetStrings(binary_name, &str)) {
+    CheckStr(str, "__linux__", AS_STR(__linux__));
+    CheckStr(str, "_WIN32", AS_STR(_WIN32));
+    CheckStr(str, "__APPLE__", AS_STR(__APPLE__));
+    CheckStr(str, "__x86_64__", AS_STR(__x86_64__));
+    CheckStr(str, "__aarch64__", AS_STR(__aarch64__));
+    CheckStr(str, "__powerpc64__", AS_STR(__powerpc64__));
+    CheckStr(str, "TF_PLAT_STR_VERSION", TF_PLAT_STR_VERSION_);
+  } else {
+    perror(binary_name.c_str());
+    rc = 2;
+  }
+
+  return rc;
+}
+
+int main(int argc, char *argv[]) {
+  tensorflow::Env *env = tensorflow::Env::Default();
+  static const char usage[] = "usage: platform_strings_test [file...]";
+  int rc = 0;
+  tensorflow::port::InitMain(usage, &argc, &argv);
+  if (argc == 1) {
+    printf("rc=%d\n", PrintStrings(env->GetExecutablePath()));
+    rc = RunTest(env->GetExecutablePath());
+  } else {
+    for (int argn = 1; argn != argc; argn++) {
+      rc |= PrintStrings(argv[argn]);
+    }
+  }
+  return rc;
+}
-- 
GitLab


From 7561099fb65c18bd091751b60fc45550fc5d4805 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 08:46:54 -0800
Subject: [PATCH 0150/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 221273389

---
 tensorflow/go/op/wrappers.go | 1324 +++++++++++++++++-----------------
 1 file changed, 662 insertions(+), 662 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index baf43f84f8..9b59c03e55 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5547,6 +5547,120 @@ func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.
 	return values
 }
 
+// MapIncompleteSizeAttr is an optional argument to MapIncompleteSize.
+type MapIncompleteSizeAttr func(optionalAttr)
+
+// MapIncompleteSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapIncompleteSizeCapacity(value int64) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapIncompleteSizeMemoryLimit(value int64) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapIncompleteSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapIncompleteSizeContainer(value string) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapIncompleteSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapIncompleteSizeSharedName(value string) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of incomplete elements in the underlying container.
+func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncompleteSizeAttr) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapIncompleteSize",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MapSizeAttr is an optional argument to MapSize.
+type MapSizeAttr func(optionalAttr)
+
+// MapSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapSizeCapacity(value int64) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapSizeMemoryLimit(value int64) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapSizeContainer(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapSizeSharedName(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of elements in the underlying container.
+func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapSize",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -9441,147 +9555,290 @@ func BesselI1e(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Transforms a Tensor into a serialized TensorProto proto.
-//
-// Arguments:
-//	tensor: A Tensor of type `T`.
+// MapClearAttr is an optional argument to MapClear.
+type MapClearAttr func(optionalAttr)
+
+// MapClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// Returns A serialized TensorProto proto of the input tensor.
-func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SerializeTensor",
-		Input: []tf.Input{
-			tensor,
-		},
+// REQUIRES: value >= 0
+func MapClearCapacity(value int64) MapClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes acos of x element-wise.
-func Acos(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Acos",
-		Input: []tf.Input{
-			x,
-		},
+// MapClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapClearMemoryLimit(value int64) MapClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// UnbatchGradAttr is an optional argument to UnbatchGrad.
-type UnbatchGradAttr func(optionalAttr)
-
-// UnbatchGradContainer sets the optional container attribute to value.
+// MapClearContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func UnbatchGradContainer(value string) UnbatchGradAttr {
+func MapClearContainer(value string) MapClearAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// UnbatchGradSharedName sets the optional shared_name attribute to value.
+// MapClearSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func UnbatchGradSharedName(value string) UnbatchGradAttr {
+func MapClearSharedName(value string) MapClearAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Gradient of Unbatch.
-//
-// Acts like Batch but using the given batch_index index of batching things as they
-// become available. This ensures that the gradients are propagated back in the
-// same session which did the forward pass.
+// Op removes all elements in the underlying container.
 //
-// original_input: The input to the Unbatch operation this is the gradient of.
-// batch_index: The batch_index given to the Unbatch operation this is the gradient
-// of.
-// grad: The downstream gradient.
-// id: The id scalar emitted by Batch.
-// batched_grad: The return value, either an empty tensor or the batched gradient.
-// container: Container to control resource sharing.
-// shared_name: Instances of UnbatchGrad with the same container and shared_name
-//  are assumed to possibly belong to the same batch. If left empty, the op name
-//  will be used as the shared name.
-func UnbatchGrad(scope *Scope, original_input tf.Output, batch_index tf.Output, grad tf.Output, id tf.Output, optional ...UnbatchGradAttr) (batched_grad tf.Output) {
+// Returns the created operation.
+func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UnbatchGrad",
-		Input: []tf.Input{
-			original_input, batch_index, grad, id,
-		},
+		Type: "MapClear",
+
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
-type AvgPool3DGradAttr func(optionalAttr)
+// DecodeCSVAttr is an optional argument to DecodeCSV.
+type DecodeCSVAttr func(optionalAttr)
 
-// AvgPool3DGradDataFormat sets the optional data_format attribute to value.
+// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func AvgPool3DGradDataFormat(value string) AvgPool3DGradAttr {
+// value: char delimiter to separate fields in a record.
+// If not specified, defaults to ","
+func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["field_delim"] = value
 	}
 }
 
-// Computes gradients of average pooling function.
-//
-// Arguments:
-//	orig_input_shape: The original input dimensions.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
 //
-// Returns The backprop for input.
-func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "AvgPool3DGrad",
-		Input: []tf.Input{
-			orig_input_shape, grad,
-		},
-		Attrs: attrs,
+// value: If false, treats double quotation marks as regular
+// characters inside of the string fields (ignoring RFC 4180, Section 2,
+// Bullet 5).
+// If not specified, defaults to true
+func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["use_quote_delim"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
-type ParseSingleSequenceExampleAttr func(optionalAttr)
-
-// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
+// DecodeCSVNaValue sets the optional na_value attribute to value.
+//
+// value: Additional string to recognize as NA/NaN.
+// If not specified, defaults to ""
+func DecodeCSVNaValue(value string) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["na_value"] = value
+	}
+}
+
+// DecodeCSVSelectCols sets the optional select_cols attribute to value.
+// If not specified, defaults to <>
+func DecodeCSVSelectCols(value []int64) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["select_cols"] = value
+	}
+}
+
+// Convert CSV records to tensors. Each column maps to one tensor.
+//
+// RFC 4180 format is expected for the CSV records.
+// (https://tools.ietf.org/html/rfc4180)
+// Note that we allow leading and trailing spaces with int or float field.
+//
+// Arguments:
+//	records: Each string is a record/row in the csv and all records should have
+// the same format.
+//	record_defaults: One tensor per column of the input record, with either a
+// scalar default value for that column or an empty vector if the column is
+// required.
+//
+// Returns Each tensor will have the same shape as records.
+func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodeCSV",
+		Input: []tf.Input{
+			records, tf.OutputList(record_defaults),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("DecodeCSV", err)
+		return
+	}
+	return output
+}
+
+// Transforms a Tensor into a serialized TensorProto proto.
+//
+// Arguments:
+//	tensor: A Tensor of type `T`.
+//
+// Returns A serialized TensorProto proto of the input tensor.
+func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SerializeTensor",
+		Input: []tf.Input{
+			tensor,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes acos of x element-wise.
+func Acos(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Acos",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// UnbatchGradAttr is an optional argument to UnbatchGrad.
+type UnbatchGradAttr func(optionalAttr)
+
+// UnbatchGradContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func UnbatchGradContainer(value string) UnbatchGradAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// UnbatchGradSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func UnbatchGradSharedName(value string) UnbatchGradAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Gradient of Unbatch.
+//
+// Acts like Batch but using the given batch_index index of batching things as they
+// become available. This ensures that the gradients are propagated back in the
+// same session which did the forward pass.
+//
+// original_input: The input to the Unbatch operation this is the gradient of.
+// batch_index: The batch_index given to the Unbatch operation this is the gradient
+// of.
+// grad: The downstream gradient.
+// id: The id scalar emitted by Batch.
+// batched_grad: The return value, either an empty tensor or the batched gradient.
+// container: Container to control resource sharing.
+// shared_name: Instances of UnbatchGrad with the same container and shared_name
+//  are assumed to possibly belong to the same batch. If left empty, the op name
+//  will be used as the shared name.
+func UnbatchGrad(scope *Scope, original_input tf.Output, batch_index tf.Output, grad tf.Output, id tf.Output, optional ...UnbatchGradAttr) (batched_grad tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnbatchGrad",
+		Input: []tf.Input{
+			original_input, batch_index, grad, id,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
+type AvgPool3DGradAttr func(optionalAttr)
+
+// AvgPool3DGradDataFormat sets the optional data_format attribute to value.
+//
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func AvgPool3DGradDataFormat(value string) AvgPool3DGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Computes gradients of average pooling function.
+//
+// Arguments:
+//	orig_input_shape: The original input dimensions.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The backprop for input.
+func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "AvgPool3DGrad",
+		Input: []tf.Input{
+			orig_input_shape, grad,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
+type ParseSingleSequenceExampleAttr func(optionalAttr)
+
+// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
 //
 // value: A list of Ncontext_sparse types; the data types of data in
 // each context Feature given in context_sparse_keys.
@@ -16696,45 +16953,163 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp
 	return op.Output(0)
 }
 
-// Returns immutable tensor from memory region.
-//
-// The current implementation memmaps the tensor from a file.
-//
-// Arguments:
-//	dtype: Type of the returned tensor.
-//	shape: Shape of the returned tensor.
-//	memory_region_name: Name of readonly memory region used by the tensor, see
-// NewReadOnlyMemoryRegionFromFile in tensorflow::Env.
-func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name}
-	opspec := tf.OpSpec{
-		Type: "ImmutableConst",
+// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
+type SparseTensorDenseMatMulAttr func(optionalAttr)
 
-		Attrs: attrs,
+// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
+//
+// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
+// is transpose(conj(A)).  Otherwise it's transpose(A).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
+	return func(m optionalAttr) {
+		m["adjoint_a"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// StringJoinAttr is an optional argument to StringJoin.
-type StringJoinAttr func(optionalAttr)
-
-// StringJoinSeparator sets the optional separator attribute to value.
+// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
 //
-// value: string, an optional join separator.
-// If not specified, defaults to ""
-func StringJoinSeparator(value string) StringJoinAttr {
+// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
+// is transpose(conj(B)).  Otherwise it's transpose(B).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
 	return func(m optionalAttr) {
-		m["separator"] = value
+		m["adjoint_b"] = value
 	}
 }
 
-// Joins the strings in the given list of string tensors into one tensor;
+// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
 //
-// with the given separator (default is an empty separator).
+// No validity checking is performed on the indices of A.  However, the following
+// input format is recommended for optimal behavior:
+//
+// if adjoint_a == false:
+//   A should be sorted in lexicographically increasing order.  Use SparseReorder
+//   if you're not sure.
+// if adjoint_a == true:
+//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
+//   order instead of "row major" order).
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
+//	b: 2-D.  A dense Matrix.
+func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseTensorDenseMatMul",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp.
+type ResourceApplyRMSPropAttr func(optionalAttr)
+
+// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, ms, and mom tensors is protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the RMSProp algorithm.
+//
+// Note that in dense implementation of this algorithm, ms and mom will
+// update even if the grad is zero, but in this sparse implementation, ms
+// and mom will not update in iterations during which the grad is zero.
+//
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+//
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
+//
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyRMSProp",
+		Input: []tf.Input{
+			var_, ms, mom, lr, rho, momentum, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Returns immutable tensor from memory region.
+//
+// The current implementation memmaps the tensor from a file.
+//
+// Arguments:
+//	dtype: Type of the returned tensor.
+//	shape: Shape of the returned tensor.
+//	memory_region_name: Name of readonly memory region used by the tensor, see
+// NewReadOnlyMemoryRegionFromFile in tensorflow::Env.
+func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name}
+	opspec := tf.OpSpec{
+		Type: "ImmutableConst",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StringJoinAttr is an optional argument to StringJoin.
+type StringJoinAttr func(optionalAttr)
+
+// StringJoinSeparator sets the optional separator attribute to value.
+//
+// value: string, an optional join separator.
+// If not specified, defaults to ""
+func StringJoinSeparator(value string) StringJoinAttr {
+	return func(m optionalAttr) {
+		m["separator"] = value
+	}
+}
+
+// Joins the strings in the given list of string tensors into one tensor;
+//
+// with the given separator (default is an empty separator).
 //
 // Arguments:
 //	inputs: A list of string tensors.  The tensors must all have the same shape,
@@ -18017,119 +18392,6 @@ func ResourceApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms
 	return scope.AddOperation(opspec)
 }
 
-// Computes the gradient for the inverse of `x` wrt its input.
-//
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReciprocalGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
-//
-// *NOTE*: `Minimum` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Minimum",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MfccAttr is an optional argument to Mfcc.
-type MfccAttr func(optionalAttr)
-
-// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
-//
-// value: The highest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 4000
-func MfccUpperFrequencyLimit(value float32) MfccAttr {
-	return func(m optionalAttr) {
-		m["upper_frequency_limit"] = value
-	}
-}
-
-// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
-//
-// value: The lowest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 20
-func MfccLowerFrequencyLimit(value float32) MfccAttr {
-	return func(m optionalAttr) {
-		m["lower_frequency_limit"] = value
-	}
-}
-
-// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
-//
-// value: Resolution of the Mel bank used internally.
-// If not specified, defaults to 40
-func MfccFilterbankChannelCount(value int64) MfccAttr {
-	return func(m optionalAttr) {
-		m["filterbank_channel_count"] = value
-	}
-}
-
-// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
-//
-// value: How many output channels to produce per time slice.
-// If not specified, defaults to 13
-func MfccDctCoefficientCount(value int64) MfccAttr {
-	return func(m optionalAttr) {
-		m["dct_coefficient_count"] = value
-	}
-}
-
-// Transforms a spectrogram into a form that's useful for speech recognition.
-//
-// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
-// been effective as an input feature for machine learning. They are created by
-// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
-// higher frequencies that are less significant to the human ear. They have a long
-// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
-// is a good resource to learn more.
-//
-// Arguments:
-//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
-// set to true.
-//	sample_rate: How many samples per second the source audio used.
-func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Mfcc",
-		Input: []tf.Input{
-			spectrogram, sample_rate,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // AudioSummaryAttr is an optional argument to AudioSummary.
 type AudioSummaryAttr func(optionalAttr)
 
@@ -19161,165 +19423,22 @@ func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...
 	return op.Output(0)
 }
 
-// DecodeCSVAttr is an optional argument to DecodeCSV.
-type DecodeCSVAttr func(optionalAttr)
+// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
+type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
 
-// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
+// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// value: char delimiter to separate fields in a record.
-// If not specified, defaults to ","
-func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["field_delim"] = value
+		m["seed"] = value
 	}
 }
 
-// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
-//
-// value: If false, treats double quotation marks as regular
-// characters inside of the string fields (ignoring RFC 4180, Section 2,
-// Bullet 5).
-// If not specified, defaults to true
-func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["use_quote_delim"] = value
-	}
-}
-
-// DecodeCSVNaValue sets the optional na_value attribute to value.
-//
-// value: Additional string to recognize as NA/NaN.
-// If not specified, defaults to ""
-func DecodeCSVNaValue(value string) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["na_value"] = value
-	}
-}
-
-// DecodeCSVSelectCols sets the optional select_cols attribute to value.
-// If not specified, defaults to <>
-func DecodeCSVSelectCols(value []int64) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["select_cols"] = value
-	}
-}
-
-// Convert CSV records to tensors. Each column maps to one tensor.
-//
-// RFC 4180 format is expected for the CSV records.
-// (https://tools.ietf.org/html/rfc4180)
-// Note that we allow leading and trailing spaces with int or float field.
-//
-// Arguments:
-//	records: Each string is a record/row in the csv and all records should have
-// the same format.
-//	record_defaults: One tensor per column of the input record, with either a
-// scalar default value for that column or an empty vector if the column is
-// required.
-//
-// Returns Each tensor will have the same shape as records.
-func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeCSV",
-		Input: []tf.Input{
-			records, tf.OutputList(record_defaults),
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("DecodeCSV", err)
-		return
-	}
-	return output
-}
-
-// MapClearAttr is an optional argument to MapClear.
-type MapClearAttr func(optionalAttr)
-
-// MapClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapClearCapacity(value int64) MapClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapClearMemoryLimit(value int64) MapClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapClearContainer(value string) MapClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapClearSharedName(value string) MapClearAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes all elements in the underlying container.
-//
-// Returns the created operation.
-func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapClear",
-
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
-type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
-
-// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
 //
 // value: An second seed to avoid seed collision.
 // If not specified, defaults to 0
@@ -19523,6 +19642,119 @@ func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, deleted_key tf.O
 	return op.Output(0)
 }
 
+// Computes the gradient for the inverse of `x` wrt its input.
+//
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReciprocalGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+//
+// *NOTE*: `Minimum` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Minimum",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MfccAttr is an optional argument to Mfcc.
+type MfccAttr func(optionalAttr)
+
+// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
+//
+// value: The highest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 4000
+func MfccUpperFrequencyLimit(value float32) MfccAttr {
+	return func(m optionalAttr) {
+		m["upper_frequency_limit"] = value
+	}
+}
+
+// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
+//
+// value: The lowest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 20
+func MfccLowerFrequencyLimit(value float32) MfccAttr {
+	return func(m optionalAttr) {
+		m["lower_frequency_limit"] = value
+	}
+}
+
+// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
+//
+// value: Resolution of the Mel bank used internally.
+// If not specified, defaults to 40
+func MfccFilterbankChannelCount(value int64) MfccAttr {
+	return func(m optionalAttr) {
+		m["filterbank_channel_count"] = value
+	}
+}
+
+// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
+//
+// value: How many output channels to produce per time slice.
+// If not specified, defaults to 13
+func MfccDctCoefficientCount(value int64) MfccAttr {
+	return func(m optionalAttr) {
+		m["dct_coefficient_count"] = value
+	}
+}
+
+// Transforms a spectrogram into a form that's useful for speech recognition.
+//
+// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
+// been effective as an input feature for machine learning. They are created by
+// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
+// higher frequencies that are less significant to the human ear. They have a long
+// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
+// is a good resource to learn more.
+//
+// Arguments:
+//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
+// set to true.
+//	sample_rate: How many samples per second the source audio used.
+func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Mfcc",
+		Input: []tf.Input{
+			spectrogram, sample_rate,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
 //
 // The Hurwitz zeta function is defined as:
@@ -19593,95 +19825,38 @@ func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 		Type: "FFT2D",
 		Input: []tf.Input{
 			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Inverse 2D fast Fourier transform.
-//
-// Computes the inverse 2-dimensional discrete Fourier transform over the
-// inner-most 2 dimensions of `input`.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their inverse 2D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.ifft2
-// @end_compatibility
-func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IFFT2D",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp.
-type ResourceApplyRMSPropAttr func(optionalAttr)
-
-// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, ms, and mom tensors is protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Update '*var' according to the RMSProp algorithm.
-//
-// Note that in dense implementation of this algorithm, ms and mom will
-// update even if the grad is zero, but in this sparse implementation, ms
-// and mom will not update in iterations during which the grad is zero.
-//
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+// Inverse 2D fast Fourier transform.
 //
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
+// Computes the inverse 2-dimensional discrete Fourier transform over the
+// inner-most 2 dimensions of `input`.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
+//	input: A complex64 tensor.
 //
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their inverse 2D Fourier transform.
 //
-// Returns the created operation.
-func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) {
+// @compatibility(numpy)
+// Equivalent to np.fft.ifft2
+// @end_compatibility
+func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyRMSProp",
+		Type: "IFFT2D",
 		Input: []tf.Input{
-			var_, ms, mom, lr, rho, momentum, epsilon, grad,
+			input,
 		},
-		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
 // Returns element-wise remainder of division. This emulates C semantics in that
@@ -20760,67 +20935,6 @@ func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values
 	return op.Output(0)
 }
 
-// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
-type SparseTensorDenseMatMulAttr func(optionalAttr)
-
-// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
-//
-// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
-// is transpose(conj(A)).  Otherwise it's transpose(A).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
-	return func(m optionalAttr) {
-		m["adjoint_a"] = value
-	}
-}
-
-// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
-//
-// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
-// is transpose(conj(B)).  Otherwise it's transpose(B).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
-	return func(m optionalAttr) {
-		m["adjoint_b"] = value
-	}
-}
-
-// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
-//
-// No validity checking is performed on the indices of A.  However, the following
-// input format is recommended for optimal behavior:
-//
-// if adjoint_a == false:
-//   A should be sorted in lexicographically increasing order.  Use SparseReorder
-//   if you're not sure.
-// if adjoint_a == true:
-//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
-//   order instead of "row major" order).
-//
-// Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
-//	b: 2-D.  A dense Matrix.
-func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseMatMul",
-		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Deserialize and concatenate `SparseTensors` from a serialized minibatch.
 //
 // The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where
@@ -26322,24 +26436,6 @@ func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in
 	return op.Output(0)
 }
 
-// Computes the gradient for the tanh of `x` wrt its input.
-//
-// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
-// is the corresponding input gradient.
-func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TanhGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Reduces sparse updates into the variable referenced by `resource` using the `max` operation.
 //
 // This operation computes
@@ -26381,6 +26477,24 @@ func ResourceScatterMax(scope *Scope, resource tf.Output, indices tf.Output, upd
 	return scope.AddOperation(opspec)
 }
 
+// Computes the gradient for the tanh of `x` wrt its input.
+//
+// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
+// is the corresponding input gradient.
+func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TanhGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Outputs a `Summary` protocol buffer with scalar values.
 //
 // The input `tags` and `values` must have the same shape.  The generated summary
@@ -30655,63 +30769,6 @@ func LatencyStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, o
 	return op.Output(0)
 }
 
-// MapSizeAttr is an optional argument to MapSize.
-type MapSizeAttr func(optionalAttr)
-
-// MapSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapSizeCapacity(value int64) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapSizeMemoryLimit(value int64) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapSizeContainer(value string) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapSizeSharedName(value string) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op returns the number of elements in the underlying container.
-func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapSize",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // SparseToDenseAttr is an optional argument to SparseToDense.
 type SparseToDenseAttr func(optionalAttr)
 
@@ -33940,60 +33997,3 @@ func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.Data
 	}
 	return values
 }
-
-// MapIncompleteSizeAttr is an optional argument to MapIncompleteSize.
-type MapIncompleteSizeAttr func(optionalAttr)
-
-// MapIncompleteSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapIncompleteSizeCapacity(value int64) MapIncompleteSizeAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapIncompleteSizeMemoryLimit(value int64) MapIncompleteSizeAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapIncompleteSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapIncompleteSizeContainer(value string) MapIncompleteSizeAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapIncompleteSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapIncompleteSizeSharedName(value string) MapIncompleteSizeAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op returns the number of incomplete elements in the underlying container.
-func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncompleteSizeAttr) (size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapIncompleteSize",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From 8e85fe418aa40e5e4d4d4700dd491f4cbef4b30e Mon Sep 17 00:00:00 2001
From: Peng Yu <peng.yu@shopify.com>
Date: Wed, 22 Aug 2018 00:05:19 -0400
Subject: [PATCH 0151/1554] add tensor forest inference kernels

---
 tensorflow/core/BUILD                         |   3 +
 ...i_def_TensorForestCreateTreeVariable.pbtxt |  17 +++
 .../api_def_TensorForestTreeDeserialize.pbtxt |  17 +++
 ..._def_TensorForestTreeIsInitializedOp.pbtxt |  17 +++
 .../api_def_TensorForestTreePredict.pbtxt     |  29 ++++
 ...def_TensorForestTreeResourceHandleOp.pbtxt |   5 +
 .../api_def_TensorForestTreeSerialize.pbtxt   |  17 +++
 .../api_def_TensorForestTreeSize.pbtxt        |  17 +++
 tensorflow/core/kernels/BUILD                 |   7 +
 .../kernels/boosted_trees/boosted_trees.proto |  14 ++
 tensorflow/core/kernels/tensor_forest/BUILD   |  53 +++++++
 .../kernels/tensor_forest/prediction_ops.cc   |  93 ++++++++++++
 .../kernels/tensor_forest/resource_ops.cc     | 135 ++++++++++++++++++
 .../core/kernels/tensor_forest/resources.cc   |  59 ++++++++
 .../core/kernels/tensor_forest/resources.h    |  63 ++++++++
 tensorflow/core/ops/tensor_forest_ops.cc      |  79 ++++++++++
 tensorflow/python/BUILD                       |  22 +++
 tensorflow/python/ops/tensor_forest_ops.py    | 110 ++++++++++++++
 18 files changed, 757 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestCreateTreeVariable.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeDeserialize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeIsInitializedOp.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreePredict.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeResourceHandleOp.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeSerialize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorForestTreeSize.pbtxt
 create mode 100644 tensorflow/core/kernels/tensor_forest/BUILD
 create mode 100644 tensorflow/core/kernels/tensor_forest/prediction_ops.cc
 create mode 100644 tensorflow/core/kernels/tensor_forest/resource_ops.cc
 create mode 100644 tensorflow/core/kernels/tensor_forest/resources.cc
 create mode 100644 tensorflow/core/kernels/tensor_forest/resources.h
 create mode 100644 tensorflow/core/ops/tensor_forest_ops.cc
 create mode 100644 tensorflow/python/ops/tensor_forest_ops.py

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index b7ec445e93..ea99f14aa9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1038,6 +1038,7 @@ tf_gen_op_libs(
         "batch_ops",
         "bitwise_ops",
         "boosted_trees_ops",
+        "tensor_forest_ops",
         "candidate_sampling_ops",
         "checkpoint_ops",
         "collective_ops",
@@ -1187,6 +1188,7 @@ cc_library(
         ":batch_ops_op_lib",
         ":bitwise_ops_op_lib",
         ":boosted_trees_ops_op_lib",
+        ":tensor_forest_ops_op_lib",
         ":candidate_sampling_ops_op_lib",
         ":checkpoint_ops_op_lib",
         ":collective_ops_op_lib",
@@ -1340,6 +1342,7 @@ cc_library(
         "//tensorflow/core/kernels:batch_kernels",
         "//tensorflow/core/kernels:bincount_op",
         "//tensorflow/core/kernels:boosted_trees_ops",
+        "//tensorflow/core/kernels:tensor_forest_ops",
         "//tensorflow/core/kernels:candidate_sampler_ops",
         "//tensorflow/core/kernels:checkpoint_ops",
         "//tensorflow/core/kernels:collective_ops",
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestCreateTreeVariable.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestCreateTreeVariable.pbtxt
new file mode 100644
index 0000000000..fe2ccd9da6
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestCreateTreeVariable.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "TensorForestCreateTreeVariable"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_handle"
+    description: <<END
+Handle to the tree resource to be created.
+END
+  }
+  in_arg {
+    name: "tree_config"
+    description: <<END
+Serialized proto string of the boosted_trees.Tree.
+END
+  }
+  summary: "Creates a tree resource and returns a handle to it."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestTreeDeserialize.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeDeserialize.pbtxt
new file mode 100644
index 0000000000..43dbcb7b42
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeDeserialize.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "TensorForestTreeDeserialize"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_handle"
+    description: <<END
+Handle to the tree resource to be restored.
+END
+  }
+  in_arg {
+    name: "tree_config"
+    description: <<END
+Serialied proto string of the boosted_trees.Tree proto.
+END
+  }
+  summary: "Deserializes a proto into the tree handle"
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestTreeIsInitializedOp.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeIsInitializedOp.pbtxt
new file mode 100644
index 0000000000..f9c7a67888
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeIsInitializedOp.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "TensorForestTreeIsInitializedOp"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_handle"
+    description: <<END
+Handle to the tree.
+END
+  }
+  out_arg {
+    name: "is_initialized"
+    description: <<END
+Whether the tree is initialized.
+END
+  }
+  summary: "Checks whether a tree has been initialized."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestTreePredict.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestTreePredict.pbtxt
new file mode 100644
index 0000000000..e8d9270274
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestTreePredict.pbtxt
@@ -0,0 +1,29 @@
+op {
+  graph_op_name: "TensorForestTreePredict"
+  visibility: HIDDEN
+  attr {
+    name: "logits_dimension"
+    description: <<END
+Scalar, dimension of the logits.
+END
+  }
+  in_arg {
+    name: "tree_handle"
+    description: <<END
+Handle to the tree resource.
+END
+  }
+  in_arg {
+    name: "dense_features"
+    description: <<END
+Rank 2 dense features tensor.
+END
+  }
+  out_arg {
+    name: "logits"
+    description: <<END
+The logits predictions from the tree for each instance in the batch.
+END
+  }
+  summary: "Output the logits for the given input data"
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestTreeResourceHandleOp.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeResourceHandleOp.pbtxt
new file mode 100644
index 0000000000..bbf5c51d64
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeResourceHandleOp.pbtxt
@@ -0,0 +1,5 @@
+op {
+  graph_op_name: "TensorForestTreeResourceHandleOp"
+  visibility: HIDDEN
+  summary: "Creates a handle to a TensorForestTreeResource"
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestTreeSerialize.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeSerialize.pbtxt
new file mode 100644
index 0000000000..aac2afa0f8
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeSerialize.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "TensorForestTreeSerialize"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_handle"
+    description: <<END
+Handle to the tree resource to be serialized.
+END
+  }
+  out_arg {
+    name: "tree_config"
+    description: <<END
+Serialied proto string of the tree resource.
+END
+  }
+  summary: "Serializes the tree handle to a proto"
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorForestTreeSize.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeSize.pbtxt
new file mode 100644
index 0000000000..6b85b0ed6c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorForestTreeSize.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "TensorForestTreeSize"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_handle"
+    description: <<END
+Handle to the tree resource.
+END
+  }
+  out_arg {
+    name: "tree_size"
+    description: <<END
+The size of the tree.
+END
+  }
+  summary: "Get the number of nodes in a tree"
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 18d70422a0..078d65dc69 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6695,6 +6695,13 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "tensor_forest_ops",
+    deps = [
+        "//tensorflow/core/kernels/tensor_forest:tensor_forest_ops",
+    ],
+)
+
 tf_kernel_library(
     name = "dataset_ops",
     deps = [
diff --git a/tensorflow/core/kernels/boosted_trees/boosted_trees.proto b/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
index 1ab72af059..4e9bab3e21 100644
--- a/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
+++ b/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
@@ -12,6 +12,7 @@ message Node {
     Leaf leaf = 1;
     BucketizedSplit bucketized_split = 2;
     CategoricalSplit categorical_split = 3;
+    DenseSplit dense_split = 4;
   }
   NodeMetadata metadata = 777;
 }
@@ -70,6 +71,19 @@ message CategoricalSplit {
   int32 right_id = 4;
 }
 
+// TODO(nponomareva): move out of boosted_trees and rename to trees.proto
+message DenseSplit {
+  // Float feature column and split threshold describing
+  // the rule feature <= threshold.
+  int32 feature_id = 1;
+  float threshold = 2;
+
+  // Node children indexing into a contiguous
+  // vector of nodes starting from the root.
+  int32 left_id = 3;
+  int32 right_id = 4;
+}
+
 // Tree describes a list of connected nodes.
 // Node 0 must be the root and can carry any payload including a leaf
 // in the case of representing the bias.
diff --git a/tensorflow/core/kernels/tensor_forest/BUILD b/tensorflow/core/kernels/tensor_forest/BUILD
new file mode 100644
index 0000000000..df035506f7
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_forest/BUILD
@@ -0,0 +1,53 @@
+# Description:
+#   OpKernels for tensor forest ops.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+
+cc_library(
+    name = "resources",
+    srcs = ["resources.cc"],
+    hdrs = ["resources.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "resource_ops",
+    srcs = ["resource_ops.cc"],
+    deps = [
+        ":resources",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:tensor_forest_ops_op_lib",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "prediction_ops",
+    srcs = ["prediction_ops.cc"],
+    deps = [
+        ":resources",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:tensor_forest_ops_op_lib",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "tensor_forest_ops",
+    deps = [
+        ":prediction_ops",
+        ":resource_ops",
+    ],
+)
diff --git a/tensorflow/core/kernels/tensor_forest/prediction_ops.cc b/tensorflow/core/kernels/tensor_forest/prediction_ops.cc
new file mode 100644
index 0000000000..877dfbc020
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_forest/prediction_ops.cc
@@ -0,0 +1,93 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/tensor_forest/resources.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/util/work_sharder.h"
+
+namespace tensorflow {
+
+class TensorForestTreePredictOp : public OpKernel {
+ public:
+  explicit TensorForestTreePredictOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("logits_dimension", &logits_dimension_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    TensorForestTreeResource* decision_tree_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &decision_tree_resource));
+    mutex_lock l(*decision_tree_resource->get_mutex());
+    core::ScopedUnref unref_me(decision_tree_resource);
+
+    const Tensor* dense_features_t = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->input("dense_features", &dense_features_t));
+
+    auto dense_features = dense_features_t->matrix<float>();
+    const int32 batch_size = dense_features_t->dim_size(0);
+
+    Tensor* output_predictions = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, {batch_size, logits_dimension_},
+                                            &output_predictions));
+    auto out = output_predictions->matrix<float>();
+
+    if (decision_tree_resource->get_size() <= 0) {
+      out.setZero();
+      return;
+    }
+    auto* worker_threads = context->device()->tensorflow_cpu_worker_threads();
+    const int32 num_threads = worker_threads->num_threads;
+
+    // TODO(yupbank): This was from contrib version.
+    //  This cost would probably depend on the depth of the tree we have.
+    //  We will need to run it on a number of trees of diff depth
+    //  and see the num of cpu cycles
+    const int64 cost_per_traverse = 500;
+    auto traverse = [this, &out, &dense_features, decision_tree_resource,
+                     batch_size](int64 start, int64 end) {
+      CHECK_LE(start, end) << "Start exceeding End";
+      CHECK_LE(end, batch_size) << "End exceeding batch size";
+      for (int example_id = start; example_id < end; ++example_id) {
+        const int32 leaf_id =
+            decision_tree_resource->TraverseTree(example_id, &dense_features);
+        set_output_value(example_id, leaf_id, decision_tree_resource, &out);
+      };
+    };
+    Shard(num_threads, worker_threads->workers, batch_size, cost_per_traverse,
+          traverse);
+  };
+
+  void set_output_value(const int32 example_id, const int32 leaf_id,
+                        const TensorForestTreeResource* decision_tree_resource,
+                        TTypes<float>::Matrix* out) const {
+    for (int j = 0; j < logits_dimension_; ++j) {
+      const float logit = decision_tree_resource->get_prediction(leaf_id, j);
+      (*out)(example_id, j) = logit;
+    }
+  };
+
+ private:
+  int32 logits_dimension_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("TensorForestTreePredict").Device(DEVICE_CPU),
+                        TensorForestTreePredictOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/tensor_forest/resource_ops.cc b/tensorflow/core/kernels/tensor_forest/resource_ops.cc
new file mode 100644
index 0000000000..aba97ea990
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_forest/resource_ops.cc
@@ -0,0 +1,135 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/tensor_forest/resources.h"
+
+namespace tensorflow {
+
+class TensorForestCreateTreeVariableOp : public OpKernel {
+ public:
+  explicit TensorForestCreateTreeVariableOp(OpKernelConstruction* context)
+      : OpKernel(context){};
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor* tree_config_t;
+    OP_REQUIRES_OK(context, context->input("tree_config", &tree_config_t));
+
+    auto* const result = new TensorForestTreeResource();
+
+    if (!result->InitFromSerialized(tree_config_t->scalar<string>()())) {
+      result->Unref();
+      OP_REQUIRES(context, false,
+                  errors::InvalidArgument("Unable to parse tree config."));
+    }
+
+    // Only create one, if one does not exist already. Report status for all
+    // other exceptions.
+    auto status = CreateResource(context, HandleFromInput(context, 0), result);
+    if (!status.ok() && status.code() != tensorflow::error::ALREADY_EXISTS) {
+      OP_REQUIRES(context, false, status);
+    }
+  }
+};
+
+// Op for serializing a model.
+class TensorForestTreeSerializeOp : public OpKernel {
+ public:
+  explicit TensorForestTreeSerializeOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    TensorForestTreeResource* decision_tree_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &decision_tree_resource));
+    mutex_lock l(*decision_tree_resource->get_mutex());
+    core::ScopedUnref unref_me(decision_tree_resource);
+    Tensor* output_config_t = nullptr;
+    OP_REQUIRES_OK(
+        context, context->allocate_output(0, TensorShape(), &output_config_t));
+    output_config_t->scalar<string>()() =
+        decision_tree_resource->decision_tree().SerializeAsString();
+  }
+};
+
+// Op for deserializing a tree variable from a checkpoint.
+class TensorForestTreeDeserializeOp : public OpKernel {
+ public:
+  explicit TensorForestTreeDeserializeOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+  void Compute(OpKernelContext* context) override {
+    TensorForestTreeResource* decision_tree_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &decision_tree_resource));
+
+    mutex_lock l(*decision_tree_resource->get_mutex());
+    core::ScopedUnref unref_me(decision_tree_resource);
+
+    const Tensor* tree_config_t;
+    OP_REQUIRES_OK(context, context->input("tree_config", &tree_config_t));
+
+    // Deallocate all the previous objects on the resource.
+    decision_tree_resource->Reset();
+
+    if (!decision_tree_resource->InitFromSerialized(
+            tree_config_t->scalar<string>()())) {
+      OP_REQUIRES(context, false,
+                  errors::InvalidArgument("Unable to parse tree config."));
+    }
+  }
+};
+
+// Op for getting tree size.
+class TensorForestTreeSizeOp : public OpKernel {
+ public:
+  explicit TensorForestTreeSizeOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    TensorForestTreeResource* decision_tree_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &decision_tree_resource));
+    mutex_lock l(*decision_tree_resource->get_mutex());
+    core::ScopedUnref unref_me(decision_tree_resource);
+    Tensor* output_t = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, TensorShape(), &output_t));
+    output_t->scalar<int32>()() = decision_tree_resource->get_size();
+  }
+};
+
+REGISTER_RESOURCE_HANDLE_KERNEL(TensorForestTreeResource);
+
+REGISTER_KERNEL_BUILDER(
+    Name("TensorForestTreeIsInitializedOp").Device(DEVICE_CPU),
+    IsResourceInitialized<TensorForestTreeResource>);
+
+REGISTER_KERNEL_BUILDER(
+    Name("TensorForestCreateTreeVariable").Device(DEVICE_CPU),
+    TensorForestCreateTreeVariableOp);
+
+REGISTER_KERNEL_BUILDER(Name("TensorForestTreeSerialize").Device(DEVICE_CPU),
+                        TensorForestTreeSerializeOp);
+
+REGISTER_KERNEL_BUILDER(Name("TensorForestTreeDeserialize").Device(DEVICE_CPU),
+                        TensorForestTreeDeserializeOp);
+
+REGISTER_KERNEL_BUILDER(Name("TensorForestTreeSize").Device(DEVICE_CPU),
+                        TensorForestTreeSizeOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/tensor_forest/resources.cc b/tensorflow/core/kernels/tensor_forest/resources.cc
new file mode 100644
index 0000000000..9f8ceb9620
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_forest/resources.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/tensor_forest/resources.h"
+#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+namespace tensorflow {
+
+const float TensorForestTreeResource::get_prediction(
+    const int32 id, const int32 dimension_id) const {
+  return decision_tree_->nodes(id).leaf().vector().value(dimension_id);
+};
+
+const int32 TensorForestTreeResource::TraverseTree(
+    const int32 example_id,
+    const TTypes<float>::ConstMatrix* dense_data) const {
+  using boosted_trees::Node;
+  using boosted_trees::Tree;
+  int32 current_id = 0;
+  while (true) {
+    const Node& current = decision_tree_->nodes(current_id);
+    if (current.has_leaf()) {
+      return current_id;
+    };
+    DCHECK_EQ(current.node_case(), Node::kDenseSplit);
+    const auto& split = current.dense_split();
+
+    if ((*dense_data)(example_id, split.feature_id()) <= split.threshold()) {
+      current_id = split.left_id();
+    } else {
+      current_id = split.right_id();
+    }
+  }
+};
+
+bool TensorForestTreeResource::InitFromSerialized(const string& serialized) {
+  return ParseProtoUnlimited(decision_tree_, serialized);
+}
+
+void TensorForestTreeResource::Reset() {
+  arena_.Reset();
+  CHECK_EQ(0, arena_.SpaceAllocated());
+  decision_tree_ = protobuf::Arena::CreateMessage<boosted_trees::Tree>(&arena_);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/tensor_forest/resources.h b/tensorflow/core/kernels/tensor_forest/resources.h
new file mode 100644
index 0000000000..34f61b2c0c
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_forest/resources.h
@@ -0,0 +1,63 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_FOREST_RESOURCES_H_
+#define TENSORFLOW_CORE_KERNELS_TENSOR_FOREST_RESOURCES_H_
+
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+namespace tensorflow {
+
+// Keep a tree ensemble in memory for efficient evaluation and mutation.
+class TensorForestTreeResource : public ResourceBase {
+ public:
+  TensorForestTreeResource()
+      : decision_tree_(
+            protobuf::Arena::CreateMessage<boosted_trees::Tree>(&arena_)){};
+
+  string DebugString() override {
+    return strings::StrCat("TensorForestTree[size=", get_size(), "]");
+  }
+
+  mutex* get_mutex() { return &mu_; }
+
+  bool InitFromSerialized(const string& serialized);
+
+  // Resets the resource and frees the proto.
+  // Caller needs to hold the mutex lock while calling this.
+  void Reset();
+
+  const int32 get_size() const { return decision_tree_->nodes_size(); }
+
+  const boosted_trees::Tree& decision_tree() const { return *decision_tree_; }
+
+  const float get_prediction(const int32 id, const int32 dimension_id) const;
+
+  const int32 TraverseTree(const int32 example_id,
+                           const TTypes<float>::ConstMatrix* dense_data) const;
+
+ protected:
+  mutex mu_;
+  protobuf::Arena arena_;
+  boosted_trees::Tree* decision_tree_;
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_KERNELS_TENSOR_FOREST_RESOURCES_H_
diff --git a/tensorflow/core/ops/tensor_forest_ops.cc b/tensorflow/core/ops/tensor_forest_ops.cc
new file mode 100644
index 0000000000..b4b6ba318e
--- /dev/null
+++ b/tensorflow/core/ops/tensor_forest_ops.cc
@@ -0,0 +1,79 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <vector>
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+REGISTER_RESOURCE_HANDLE_OP(TensorForestTreeResource);
+
+REGISTER_OP("TensorForestTreeIsInitializedOp")
+    .Input("tree_handle: resource")
+    .Output("is_initialized: bool")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      c->set_output(0, c->Scalar());
+      return Status::OK();
+    });
+
+REGISTER_OP("TensorForestCreateTreeVariable")
+    .Input("tree_handle: resource")
+    .Input("tree_config: string")
+    .SetShapeFn(tensorflow::shape_inference::NoOutputs);
+
+REGISTER_OP("TensorForestTreeSerialize")
+    .Input("tree_handle: resource")
+    .Output("tree_config: string")
+    .SetShapeFn(tensorflow::shape_inference::ScalarShape);
+
+REGISTER_OP("TensorForestTreeDeserialize")
+    .Input("tree_handle: resource")
+    .Input("tree_config: string")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      return Status::OK();
+    });
+
+REGISTER_OP("TensorForestTreeSize")
+    .Input("tree_handle: resource")
+    .Output("tree_size: int32")
+    .SetShapeFn(tensorflow::shape_inference::ScalarShape);
+
+REGISTER_OP("TensorForestTreePredict")
+    .Attr("logits_dimension: int")
+    .Input("tree_handle: resource")
+    .Input("dense_features: float")
+    .Output("logits: float")
+    .SetShapeFn([](tensorflow::shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle shape_handle;
+      shape_inference::DimensionHandle batch_size = c->UnknownDim();
+
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 2, &shape_handle));
+
+      batch_size = c->Dim(shape_handle, 0);
+
+      int logits_dimension;
+      TF_RETURN_IF_ERROR(c->GetAttr("logits_dimension", &logits_dimension));
+      c->set_output(0, c->Matrix(batch_size, logits_dimension));
+      return Status::OK();
+    });
+}  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4fe92262ba..cfc25513ed 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -132,6 +132,7 @@ py_library(
         ":subscribe",
         ":summary",
         ":tensor_array_ops",
+        ":tensor_forest_ops",
         ":test_ops",  # TODO: Break testing code out into separate rule.
         ":tf_cluster",
         ":tf_item",
@@ -1618,6 +1619,14 @@ tf_gen_op_wrapper_private_py(
     ],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "tensor_forest_ops_gen",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:tensor_forest_ops_op_lib",
+    ],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "summary_ops_gen",
     visibility = ["//tensorflow:__subpackages__"],
@@ -1941,6 +1950,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "tensor_forest_ops",
+    srcs = ["ops/tensor_forest_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework",
+        ":ops",
+        ":tensor_forest_ops_gen",
+        ":training",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py",
+    ],
+)
+
 py_library(
     name = "sets",
     srcs = [
diff --git a/tensorflow/python/ops/tensor_forest_ops.py b/tensorflow/python/ops/tensor_forest_ops.py
new file mode 100644
index 0000000000..42f3cdf324
--- /dev/null
+++ b/tensorflow/python/ops/tensor_forest_ops.py
@@ -0,0 +1,110 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ops for tensor_forest."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import resources
+
+from tensorflow.python import ops
+from tensorflow.python.ops import gen_tensor_forest_ops
+from tensorflow.python.training import saver
+
+
+class TreeVariableSaveable(saver.BaseSaverBuilder.SaveableObject):
+
+  def __init__(self, type_name, name, container, config, resource_handle_func,
+               create_op_func, is_initialized_op_func, serialize_op_func,
+               deserialize_op_func):
+
+    with ops.name_scope(name, type_name) as name:
+      self._resource_handle = resource_handle_func(
+          container, shared_name=name, name=name)
+
+    self._is_initialized_op = is_initialized_op_func(
+        self._resource_handle)
+    tensor = serialize_op_func(self._resource_handle)
+    self._create_op = create_op_func(
+        self._resource_handle,
+        config)
+    # slice_spec is useful for saving a slice from a variable.
+    # It's not meaningful the tree variable. So we just pass an empty
+    # value.
+    slice_spec = ""
+    specs = [saver.BaseSaverBuilder.SaveSpec(tensor, slice_spec, name)]
+    super(TreeVariableSaveable,
+          self).__init__(self._resource_handle, specs, name)
+
+    ops.add_to_collection(
+        ops.GraphKeys.SAVEABLE_OBJECTS, self)
+
+    resources.register_resource(
+        self._resource_handle, self._create_op, self._is_initialized_op)
+    self._deserialize_op_func = deserialize_op_func
+
+  def restore(self, restored_tensors, unused_restored_shapes):
+    """Restores the associated tree from 'restored_tensors'.
+
+    Args:
+      restored_tensors: the tensors that were loaded from a checkpoint.
+      unused_restored_shapes: the shapes this object should conform to after
+        restore. Not meaningful for trees.
+
+    Returns:
+      The operation that restores the state of the tree variable.
+    """
+    with ops.control_dependencies([self._create_op]):
+      return self._deserialize_op_func(
+          self._resource_handle,
+          restored_tensors[0],
+      )
+
+  @property
+  def resource(self):
+    return self._resource_handle
+
+
+def tree_variable(tree_config, name, container=None):
+  return TreeVariableSaveable(
+      "TreeVariable",
+      name,
+      container,
+      tree_config,
+      gen_tensor_forest_ops.tensor_forest_tree_resource_handle_op,
+      gen_tensor_forest_ops.tensor_forest_create_tree_variable,
+      gen_tensor_forest_ops.tensor_forest_tree_is_initialized_op,
+      gen_tensor_forest_ops.tensor_forest_tree_serialize,
+      gen_tensor_forest_ops.tensor_forest_tree_deserialize).resource
+
+
+class ForestVariables(object):
+
+  def __init__(self, params,
+               tree_configs=None):
+
+    self._variables = []
+
+    for i in range(params.n_trees):
+      tree_config = ''
+      if tree_configs is not None:
+        tree_config = tree_configs[i]
+      self._variables.append(tree_variable(
+          tree_config,
+          'tree-%s' % i,
+      ))
+
+  def __getitem__(self, t):
+    return self._variables[t]
-- 
GitLab


From cbb06439c8b11dac918b3e83d1ac0fda1c346a99 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 13 Nov 2018 09:39:04 -0800
Subject: [PATCH 0152/1554] Correctly handle
 record_summaries_every_n_global_steps() when eager execution is enabled.

PiperOrigin-RevId: 221281420
---
 .../contrib/summary/summary_ops_test.py       | 30 +++++++++++++++----
 tensorflow/python/ops/summary_ops_v2.py       | 13 +++++---
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py
index 4d1807130c..10e4556dac 100644
--- a/tensorflow/contrib/summary/summary_ops_test.py
+++ b/tensorflow/contrib/summary/summary_ops_test.py
@@ -152,6 +152,27 @@ class EagerFileTest(test_util.TensorFlowTestCase):
       self.assertEqual(len(events), 2)
       self.assertEqual(events[1].summary.value[0].tag, 'scalar')
 
+  def testRecordEveryNGlobalSteps(self):
+    step = training_util.get_or_create_global_step()
+    logdir = tempfile.mkdtemp()
+
+    def run_step():
+      summary_ops.scalar('scalar', i, step=step)
+      step.assign_add(1)
+
+    with summary_ops.create_file_writer(
+        logdir).as_default(), summary_ops.record_summaries_every_n_global_steps(
+            2, step):
+      for i in range(10):
+        run_step()
+      # And another 10 steps as a graph function.
+      run_step_fn = function.defun(run_step)
+      for i in range(10):
+        run_step_fn()
+
+    events = summary_test_util.events_from_logdir(logdir)
+    self.assertEqual(len(events), 11)
+
   def testMaxQueue(self):
     logs = tempfile.mkdtemp()
     with summary_ops.create_file_writer(
@@ -279,12 +300,9 @@ class EagerDbTest(summary_test_util.SummaryDbTest):
 
   def testDbURIOpen(self):
     tmpdb_path = os.path.join(self.get_temp_dir(), 'tmpDbURITest.sqlite')
-    tmpdb_uri = six.moves.urllib_parse.urljoin("file:", tmpdb_path)
-    tmpdb_writer = summary_ops.create_db_writer(
-        tmpdb_uri,
-        "experimentA",
-        "run1",
-        "user1")
+    tmpdb_uri = six.moves.urllib_parse.urljoin('file:', tmpdb_path)
+    tmpdb_writer = summary_ops.create_db_writer(tmpdb_uri, 'experimentA',
+                                                'run1', 'user1')
     with summary_ops.always_record_summaries():
       with tmpdb_writer.as_default():
         summary_ops.scalar('t1', 2.0)
diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py
index 18cefb8e1c..fac9055ef1 100644
--- a/tensorflow/python/ops/summary_ops_v2.py
+++ b/tensorflow/python/ops/summary_ops_v2.py
@@ -43,8 +43,9 @@ from tensorflow.python.training import training_util
 from tensorflow.python.util import tf_contextlib
 
 
-# A global dictionary mapping graph keys to boolean values indicating whether
-# we should record summaries for this particular graph or not.
+# Dictionary mapping graph keys to a boolean Tensor (or callable returning
+# a boolean Tensor) indicating whether we should record summaries for the
+# graph identified by the key of the dictionary.
 _SHOULD_RECORD_SUMMARIES = {}
 
 # A global dictionary mapping graph keys to a list of summary writer init ops.
@@ -59,7 +60,8 @@ def should_record_summaries():
   """Returns boolean Tensor which is true if summaries should be recorded."""
   global _SHOULD_RECORD_SUMMARIES
   key = ops.get_default_graph()._graph_key  # pylint: disable=protected-access
-  return _SHOULD_RECORD_SUMMARIES.setdefault(key, False)
+  should = _SHOULD_RECORD_SUMMARIES.setdefault(key, False)
+  return should() if callable(should) else should
 
 
 # TODO(apassos) consider how to handle local step here.
@@ -73,7 +75,10 @@ def record_summaries_every_n_global_steps(n, global_step=None):
   old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False)
   try:
     with ops.device("cpu:0"):
-      _SHOULD_RECORD_SUMMARIES[key] = math_ops.equal(global_step % n, 0)
+      should = lambda: math_ops.equal(global_step % n, 0)
+      if not context.executing_eagerly():
+        should = should()
+      _SHOULD_RECORD_SUMMARIES[key] = should
     yield
   finally:
     _SHOULD_RECORD_SUMMARIES[key] = old
-- 
GitLab


From ea2180d77d61942de039a522ee973114657db7c8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 09:39:39 -0800
Subject: [PATCH 0153/1554] Remove sparse_reduce_max, sparse_reduce_max_sparse,
 sparse_reduce_sum, sparse_reduce_sum_sparse, sparse_split symbols.

PiperOrigin-RevId: 221281522
---
 tensorflow/python/ops/sparse_ops.py           | 12 ++++++-----
 .../tools/api/golden/v2/tensorflow.pbtxt      | 20 -------------------
 tensorflow/tools/compatibility/renames_v2.py  |  5 +++++
 3 files changed, 12 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index b98c7f5f65..80d93f3de1 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -705,7 +705,7 @@ class KeywordRequired(object):
     return "KeywordRequired()"
 
 
-@tf_export("sparse.split", "sparse_split")
+@tf_export("sparse.split", v1=["sparse.split", "sparse_split"])
 @deprecation.deprecated_endpoints("sparse_split")
 @deprecation.deprecated_args(
     None, "split_dim is deprecated, use axis instead", "split_dim")
@@ -888,7 +888,7 @@ def sparse_to_dense(sparse_indices,
       name=name)
 
 
-@tf_export("sparse.reduce_max", "sparse_reduce_max")
+@tf_export("sparse.reduce_max", v1=["sparse.reduce_max", "sparse_reduce_max"])
 @deprecation.deprecated_endpoints("sparse_reduce_max")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
@@ -956,7 +956,8 @@ def sparse_reduce_max(sp_input, axis=None, keepdims=None,
       math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims)
 
 
-@tf_export("sparse.reduce_max_sparse", "sparse_reduce_max_sparse")
+@tf_export("sparse.reduce_max_sparse",
+           v1=["sparse.reduce_max_sparse", "sparse_reduce_max_sparse"])
 @deprecation.deprecated_endpoints("sparse_reduce_max_sparse")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
@@ -1007,7 +1008,7 @@ def sparse_reduce_max_sparse(sp_input,
   return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
 
 
-@tf_export("sparse.reduce_sum", "sparse_reduce_sum")
+@tf_export("sparse.reduce_sum", v1=["sparse.reduce_sum", "sparse_reduce_sum"])
 @deprecation.deprecated_endpoints("sparse_reduce_sum")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
@@ -1062,7 +1063,8 @@ def sparse_reduce_sum(sp_input, axis=None, keepdims=None,
       math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims)
 
 
-@tf_export("sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse")
+@tf_export("sparse.reduce_sum_sparse",
+           v1=["sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse"])
 @deprecation.deprecated_endpoints("sparse_reduce_sum_sparse")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 078b471a4c..6157121b97 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1120,26 +1120,6 @@ tf_module {
     name: "sparse_concat"
     argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
-  member_method {
-    name: "sparse_reduce_max"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sparse_reduce_max_sparse"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sparse_reduce_sum"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sparse_reduce_sum_sparse"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sparse_split"
-    argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "sparse_to_dense"
     argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'True\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5ea2fbcc4c..5975eadead 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -385,6 +385,10 @@ renames = {
     'tf.sparse_merge': 'tf.sparse.merge',
     'tf.sparse_minimum': 'tf.sparse.minimum',
     'tf.sparse_placeholder': 'tf.compat.v1.sparse_placeholder',
+    'tf.sparse_reduce_max': 'tf.sparse.reduce_max',
+    'tf.sparse_reduce_max_sparse': 'tf.sparse.reduce_max_sparse',
+    'tf.sparse_reduce_sum': 'tf.sparse.reduce_sum',
+    'tf.sparse_reduce_sum_sparse': 'tf.sparse.reduce_sum_sparse',
     'tf.sparse_reorder': 'tf.sparse.reorder',
     'tf.sparse_reset_shape': 'tf.sparse.reset_shape',
     'tf.sparse_reshape': 'tf.sparse.reshape',
@@ -394,6 +398,7 @@ renames = {
     'tf.sparse_segment_sum': 'tf.sparse.segment_sum',
     'tf.sparse_slice': 'tf.sparse.slice',
     'tf.sparse_softmax': 'tf.sparse.softmax',
+    'tf.sparse_split': 'tf.sparse.split',
     'tf.sparse_tensor_dense_matmul': 'tf.sparse.matmul',
     'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense',
     'tf.sparse_to_indicator': 'tf.sparse.to_indicator',
-- 
GitLab


From 0b37348107817c3f90a9673f917bd42fcc997d79 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 09:52:27 -0800
Subject: [PATCH 0154/1554] Quantile loss and estimator

PiperOrigin-RevId: 221283515
---
 .../boosted_trees/estimator_batch/BUILD       |   1 +
 .../estimator_batch/estimator.py              | 214 +++++++++++++++++-
 .../estimator_batch/estimator_test.py         | 159 ++++++++++++-
 .../boosted_trees/python/utils/losses.py      |  41 ++++
 4 files changed, 407 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index 14b6fc4ac2..d3b23d949e 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -132,6 +132,7 @@ py_library(
     srcs = ["estimator.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":custom_loss_head",
         ":estimator_utils",
         ":model",
         "//tensorflow/contrib/boosted_trees:losses",
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
index 38d19976ef..99ecded653 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+
 from tensorflow.contrib.boosted_trees.estimator_batch import model
 from tensorflow.contrib.boosted_trees.python.utils import losses
 from tensorflow.contrib.learn.python.learn.estimators import estimator
@@ -26,7 +28,8 @@ from tensorflow.python.estimator.canned import head as core_head_lib
 from tensorflow.python.estimator import estimator as core_estimator
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.losses import losses as core_losses
-
+from tensorflow.contrib.boosted_trees.estimator_batch import custom_loss_head
+from tensorflow.python.ops import array_ops
 
 # ================== Old estimator interface===================================
 # The estimators below were designed for old feature columns and old estimator
@@ -414,6 +417,101 @@ class GradientBoostedDecisionTreeRanker(estimator.Estimator):
         config=config,
         feature_engineering_fn=feature_engineering_fn)
 
+# When using this estimator, make sure to regularize the hessian (at least l2,
+# min_node_weight)!
+# TODO(nponomareva): extend to take multiple quantiles in one go.
+class GradientBoostedDecisionTreeQuantileRegressor(estimator.Estimator):
+  """An estimator that does quantile regression and returns quantile estimates.
+  """
+
+  def __init__(self,
+               learner_config,
+               examples_per_layer,
+               quantiles,
+               num_trees=None,
+               feature_columns=None,
+               weight_column_name=None,
+               model_dir=None,
+               config=None,
+               feature_engineering_fn=None,
+               logits_modifier_function=None,
+               center_bias=True,
+               use_core_libs=False,
+               output_leaf_index=False,
+               override_global_step_value=None,
+               num_quantiles=100):
+    """Initializes a GradientBoostedDecisionTreeQuantileRegressor instance.
+
+    Args:
+      learner_config: A config for the learner.
+      examples_per_layer: Number of examples to accumulate before growing a
+        layer. It can also be a function that computes the number of examples
+        based on the depth of the layer that's being built.
+      quantiles: a list of quantiles for the loss, each between 0 and 1.
+      num_trees: An int, number of trees to build.
+      feature_columns: A list of feature columns.
+      weight_column_name: Name of the column for weights, or None if not
+        weighted.
+      model_dir: Directory for model exports, etc.
+      config: `RunConfig` object to configure the runtime settings.
+      feature_engineering_fn: Feature engineering function. Takes features and
+        labels which are the output of `input_fn` and returns features and
+        labels which will be fed into the model.
+      logits_modifier_function: A modifier function for the logits.
+      center_bias: Whether a separate tree should be created for first fitting
+        the bias.
+      use_core_libs: Whether feature columns and loss are from the core (as
+        opposed to contrib) version of tensorflow.
+      output_leaf_index: whether to output leaf indices along with predictions
+        during inference. The leaf node indexes are available in predictions
+        dict by the key 'leaf_index'. For example,
+        result_dict = classifier.predict(...)
+        for example_prediction_result in result_dict:
+          # access leaf index list by example_prediction_result["leaf_index"]
+          # which contains one leaf index per tree
+      override_global_step_value: If after the training is done, global step
+        value must be reset to this value. This should be used to reset global
+        step to a number > number of steps used to train the current ensemble.
+        For example, the usual way is to train a number of trees and set a very
+        large number of training steps. When the training is done (number of
+        trees were trained), this parameter can be used to set the global step
+        to a large value, making it look like that number of training steps ran.
+        If None, no override of global step will happen.
+      num_quantiles: Number of quantiles to build for numeric feature values.
+    """
+
+    if len(quantiles) > 1:
+      raise ValueError('For now, just one quantile per estimator is supported')
+
+    def _quantile_regression_head(quantile):
+      # Use quantile regression.
+      head = custom_loss_head.CustomLossHead(
+          loss_fn=functools.partial(
+              losses.per_example_quantile_regression_loss, quantile=quantile),
+          link_fn=array_ops.identity,
+          logit_dimension=1)
+      return head
+
+    super(GradientBoostedDecisionTreeQuantileRegressor, self).__init__(
+        model_fn=model.model_builder,
+        params={
+            'head': _quantile_regression_head(quantiles[0]),
+            'feature_columns': feature_columns,
+            'learner_config': learner_config,
+            'num_trees': num_trees,
+            'weight_column_name': weight_column_name,
+            'examples_per_layer': examples_per_layer,
+            'logits_modifier_function': logits_modifier_function,
+            'center_bias': center_bias,
+            'use_core_libs': use_core_libs,
+            'output_leaf_index': False,
+            'override_global_step_value': override_global_step_value,
+            'num_quantiles': num_quantiles,
+        },
+        model_dir=model_dir,
+        config=config,
+        feature_engineering_fn=feature_engineering_fn)
+
 # ================== New Estimator interface===================================
 # The estimators below use new core Estimator interface and must be used with
 # new feature columns and heads.
@@ -437,12 +535,41 @@ def core_multiclass_head(
 
   # pylint:disable=protected-access
   head_fn = core_head_lib._multi_class_head_with_softmax_cross_entropy_loss(
-      n_classes=n_classes, loss_fn=loss_fn, loss_reduction=loss_reduction)
+      n_classes=n_classes,
+      loss_fn=loss_fn,
+      loss_reduction=loss_reduction,
+      weight_column=weight_column)
   # pylint:enable=protected-access
 
   return head_fn
 
 
+# For quantile regression, use this head with Core..Estimator, or use
+# Core..QuantileRegressor directly,
+def core_quantile_regression_head(
+    quantiles,
+    weight_column=None,
+    loss_reduction=core_losses.Reduction.SUM_OVER_NONZERO_WEIGHTS):
+  """Core head for quantile regression problems."""
+
+  def loss_fn(labels, logits):
+    result = losses.per_example_quantile_regression_loss(
+        labels=labels,
+        predictions=logits,
+        weights=weight_column,
+        quantile=quantiles)
+    return result[0]
+
+  # pylint:disable=protected-access
+  head_fn = core_head_lib._regression_head(
+      label_dimension=1,
+      loss_fn=loss_fn,
+      loss_reduction=loss_reduction,
+      weight_column=weight_column)
+  # pylint:enable=protected-access
+  return head_fn
+
+
 class CoreGradientBoostedDecisionTreeEstimator(core_estimator.Estimator):
   """An estimator using gradient boosted decision trees.
 
@@ -606,3 +733,86 @@ class CoreGradientBoostedDecisionTreeRanker(core_estimator.Estimator):
 
     super(CoreGradientBoostedDecisionTreeRanker, self).__init__(
         model_fn=_model_fn, model_dir=model_dir, config=config)
+
+
+# When using this estimator, make sure to regularize the hessian (at least l2,
+# min_node_weight)!
+# TODO(nponomareva): extend to take multiple quantiles in one go.
+class CoreGradientBoostedDecisionTreeQuantileRegressor(
+    core_estimator.Estimator):
+  """An estimator that does quantile regression and returns quantile estimates.
+  """
+
+  def __init__(self,
+               learner_config,
+               examples_per_layer,
+               quantiles,
+               num_trees=None,
+               feature_columns=None,
+               weight_column_name=None,
+               model_dir=None,
+               config=None,
+               label_keys=None,
+               feature_engineering_fn=None,
+               logits_modifier_function=None,
+               center_bias=True,
+               output_leaf_index=False,
+               num_quantiles=100):
+    """Initializes a core version of GradientBoostedDecisionTreeEstimator.
+
+    Args:
+      learner_config: A config for the learner.
+      examples_per_layer: Number of examples to accumulate before growing a
+        layer. It can also be a function that computes the number of examples
+        based on the depth of the layer that's being built.
+      quantiles: a list of quantiles for the loss, each between 0 and 1.
+      num_trees: An int, number of trees to build.
+      feature_columns: A list of feature columns.
+      weight_column_name: Name of the column for weights, or None if not
+        weighted.
+      model_dir: Directory for model exports, etc.
+      config: `RunConfig` object to configure the runtime settings.
+      label_keys: Optional list of strings with size `[n_classes]` defining the
+        label vocabulary. Only supported for `n_classes` > 2.
+      feature_engineering_fn: Feature engineering function. Takes features and
+        labels which are the output of `input_fn` and returns features and
+        labels which will be fed into the model.
+      logits_modifier_function: A modifier function for the logits.
+      center_bias: Whether a separate tree should be created for first fitting
+        the bias.
+      output_leaf_index: whether to output leaf indices along with predictions
+        during inference. The leaf node indexes are available in predictions
+        dict by the key 'leaf_index'. For example,
+        result_dict = classifier.predict(...)
+        for example_prediction_result in result_dict:
+          # access leaf index list by example_prediction_result["leaf_index"]
+          # which contains one leaf index per tree
+      num_quantiles: Number of quantiles to build for numeric feature values.
+    """
+    if len(quantiles) > 1:
+      raise ValueError('For now, just one quantile per estimator is supported')
+
+    def _model_fn(features, labels, mode, config):
+      return model.model_builder(
+          features=features,
+          labels=labels,
+          mode=mode,
+          config=config,
+          params={
+              'head': core_quantile_regression_head(quantiles[0]),
+              'feature_columns': feature_columns,
+              'learner_config': learner_config,
+              'num_trees': num_trees,
+              'weight_column_name': weight_column_name,
+              'examples_per_layer': examples_per_layer,
+              'center_bias': center_bias,
+              'logits_modifier_function': logits_modifier_function,
+              'use_core_libs': True,
+              'output_leaf_index': output_leaf_index,
+              'override_global_step_value': None,
+              'num_quantiles': num_quantiles,
+          },
+          output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC)
+
+    super(CoreGradientBoostedDecisionTreeQuantileRegressor, self).__init__(
+        model_fn=_model_fn, model_dir=model_dir, config=config)
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index c155128c0e..1ede129925 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -25,6 +25,7 @@ from tensorflow.contrib.boosted_trees.proto import learner_pb2
 from tensorflow.contrib.layers.python.layers import feature_column as contrib_feature_column
 from tensorflow.contrib.learn.python.learn.estimators import run_config
 from tensorflow.python.estimator.canned import head as head_lib
+from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column_lib as core_feature_column
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -47,8 +48,8 @@ def _multiclass_train_input_fn():
   features = {
       "x": constant_op.constant([[2.], [1.], [1.], [5.], [3.5], [4.6], [3.5]])
   }
-  label = constant_op.constant(
-      [[1], [0], [0], [2], [2], [0], [1]], dtype=dtypes.int32)
+  label = constant_op.constant([[1], [0], [0], [2], [2], [0], [1]],
+                               dtype=dtypes.int32)
   return features, label
 
 
@@ -77,6 +78,51 @@ def _infer_ranking_train_input_fn():
   return features, None
 
 
+_QUANTILE_REGRESSION_SIZE = 1000
+
+
+def _quantile_regression_input_fns():
+  # The data generation is taken from
+  # http://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_quantile.html
+  np.random.seed(1)
+
+  def f(x):
+    """The function to predict."""
+    return x * np.sin(x)
+
+  #  Training data.
+  x = np.atleast_2d(np.random.uniform(0, 10.0,
+                                      size=_QUANTILE_REGRESSION_SIZE)).T
+  x = x.astype(np.float32)
+
+  # Labels.
+  y = f(x).ravel()
+
+  # Add random noise.
+  dy = 1.5 + 1.0 * np.random.random(y.shape)
+  noise = np.random.normal(0, dy)
+  y += noise
+  y_original = y.astype(np.float32)
+  y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1)
+
+  train_input_fn = numpy_io.numpy_input_fn(
+      x=x,
+      y=y,
+      batch_size=_QUANTILE_REGRESSION_SIZE,
+      num_epochs=None,
+      shuffle=True)
+
+  # Test on the training data to make sure the predictions are calibrated.
+  test_input_fn = numpy_io.numpy_input_fn(
+      x=x,
+      y=y,
+      batch_size=_QUANTILE_REGRESSION_SIZE,
+      num_epochs=1,
+      shuffle=False)
+
+  return train_input_fn, test_input_fn, y_original
+
+
 class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -341,6 +387,56 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     for prediction_dict in result_iter:
       self.assertTrue("classes" in prediction_dict)
 
+  # One dimensional quantile regression.
+  def testQuantileRegression(self):
+    learner_config = learner_pb2.LearnerConfig()
+    learner_config.num_classes = 2
+    learner_config.constraints.max_tree_depth = 3
+    learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
+    learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.tree_complexity = (
+        1.0 / _QUANTILE_REGRESSION_SIZE)
+
+    train_input_fn, test_input_fn, y = _quantile_regression_input_fns()
+
+    # 95% percentile.
+    model_upper = estimator.GradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.95],
+        learner_config=learner_config,
+        num_trees=100,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_upper.fit(input_fn=train_input_fn, steps=1000)
+    result_iter = model_upper.predict(input_fn=test_input_fn)
+    upper = []
+    for prediction_dict in result_iter:
+      upper.append(prediction_dict["scores"])
+
+    frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3)
+    # +/- 3%
+    self.assertBetween(frac_below_upper, 0.92, 0.98)
+
+    train_input_fn, test_input_fn, _ = _quantile_regression_input_fns()
+    model_lower = estimator.GradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.05],
+        learner_config=learner_config,
+        num_trees=100,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_lower.fit(input_fn=train_input_fn, steps=1000)
+    result_iter = model_lower.predict(input_fn=test_input_fn)
+    lower = []
+    for prediction_dict in result_iter:
+      lower.append(prediction_dict["scores"])
+
+    frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3)
+    # +/- 3%
+    self.assertBetween(frac_above_lower, 0.92, 0.98)
+
 
 class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
 
@@ -489,8 +585,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
 
     feature_columns = [
         core_feature_column.weighted_categorical_column(
-            categorical_column=core_feature_column.
-            categorical_column_with_vocabulary_list(
+            categorical_column=core_feature_column
+            .categorical_column_with_vocabulary_list(
                 key="word", vocabulary_list=["the", "cat", "dog"]),
             weight_feature_key="weight")
     ]
@@ -509,8 +605,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
         # Weights for the words are 5 - cat, 6- dog and 1 -the.
         features_dict["word"] = sparse_tensor.SparseTensor(
             indices=[[0, 0], [0, 1], [1, 0], [3, 0]],
-            values=constant_op.constant(
-                ["the", "cat", "dog", "the"], dtype=dtypes.string),
+            values=constant_op.constant(["the", "cat", "dog", "the"],
+                                        dtype=dtypes.string),
             dense_shape=[4, 3])
         features_dict["weight"] = sparse_tensor.SparseTensor(
             indices=[[0, 0], [0, 1], [1, 0], [3, 0]],
@@ -534,6 +630,57 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
     est.evaluate(input_fn=input_fn, steps=1)
     est.predict(input_fn=input_fn)
 
+  # One dimensional quantile regression.
+  def testQuantileRegression(self):
+    learner_config = learner_pb2.LearnerConfig()
+    learner_config.num_classes = 2
+    learner_config.constraints.max_tree_depth = 3
+    learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
+    learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.tree_complexity = (
+        1.0 / _QUANTILE_REGRESSION_SIZE)
+
+    train_input_fn, test_input_fn, y = _quantile_regression_input_fns()
+    y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1)
+
+    # 95% percentile.
+    model_upper = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.95],
+        learner_config=learner_config,
+        num_trees=100,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_upper.train(input_fn=train_input_fn, steps=1000)
+    result_iter = model_upper.predict(input_fn=test_input_fn)
+    upper = []
+    for prediction_dict in result_iter:
+      upper.append(prediction_dict["predictions"])
+
+    frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3)
+    # +/- 3%
+    self.assertBetween(frac_below_upper, 0.92, 0.98)
+
+    train_input_fn, test_input_fn, _ = _quantile_regression_input_fns()
+    model_lower = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.05],
+        learner_config=learner_config,
+        num_trees=100,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_lower.train(input_fn=train_input_fn, steps=1000)
+    result_iter = model_lower.predict(input_fn=test_input_fn)
+    lower = []
+    for prediction_dict in result_iter:
+      lower.append(prediction_dict["predictions"])
+
+    frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3)
+    # +/- 3%
+    self.assertBetween(frac_above_lower, 0.92, 0.98)
+
 
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py
index b5ebaf1999..f8da20a54c 100644
--- a/tensorflow/contrib/boosted_trees/python/utils/losses.py
+++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py
@@ -48,6 +48,47 @@ def per_example_logistic_loss(labels, weights, predictions):
       labels=labels, logits=predictions)
   return unweighted_loss * weights, control_flow_ops.no_op()
 
+# MUST USE WITH HESSIAN REGULARIZATION,
+# This loss can have zero hessian, so it must be used with l2 or min_node_weight
+# regularization.
+# An example config is
+# learner_config.constraints.min_node_weight = 1 / num_examples_per_layer
+# learner_config.regularization.l2 = 1.0 / num_examples_per_layer
+# TODO(nponomareva): make it multidimensional so we can estimate several
+# quantiles at once.
+def per_example_quantile_regression_loss(labels, weights, predictions,
+                                         quantile):
+  """Smoothed loss for quantile regression.
+
+  The standard quantile regression loss is quantile*(y-y') when y>y' and
+  (quantile-1)*(y-y') otherwise, y' is a prediction, y is a label. The impl
+  below is this loss but squared in the region where the loss value < 1.
+
+  Args:
+    labels: Rank 2 (N, 1) tensor of per-example labels.
+    weights: Rank 2 (N, 1) tensor of per-example weights.
+    predictions: Rank 2 (N, 1) tensor of per-example predictions.
+    quantile: The quantile to use.
+
+  Returns:
+    loss: A Rank 2 (N, 1) tensor of per-example quantile loss.
+    update_op: An update operation to update the loss's internal state.
+  """
+  labels = math_ops.to_float(labels)
+  error = labels - predictions
+  square_loss_right = array_ops.where(error * quantile < 1.0,
+                                      math_ops.square(quantile * error),
+                                      quantile * error)
+  square_loss_left = array_ops.where(error * (quantile - 1) < 1,
+                                     math_ops.square((quantile - 1) * error),
+                                     (quantile - 1) * error)
+
+  unweighted_loss = array_ops.where(error > 0, square_loss_right,
+                                    square_loss_left)
+  if weights is None:
+    return unweighted_loss, control_flow_ops.no_op()
+  else:
+    return unweighted_loss * weights, control_flow_ops.no_op()
 
 # This is classical form of Maximum entropy loss, that is twice differentiable
 # (sparse_softmax_cross_entropy which is what we go for is not twice
-- 
GitLab


From 59a6fb8a8de84aff258ee7ec9d21966600c159ae Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 13 Nov 2018 10:04:13 -0800
Subject: [PATCH 0155/1554] Export tf.compat.v1.wrap_function

PiperOrigin-RevId: 221285685
---
 tensorflow/python/BUILD                        |  1 +
 tensorflow/python/eager/wrap_function.py       | 18 ++++++++++++++++++
 tensorflow/python/ops/standard_ops.py          |  1 +
 .../tools/api/golden/v1/tensorflow.pbtxt       |  4 ++++
 4 files changed, 24 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4fe92262ba..63352b82f1 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2972,6 +2972,7 @@ py_library(
         ":util",
         ":variable_scope",
         ":variables",
+        "//tensorflow/python/eager:wrap_function",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
     ],
diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 48266437ef..2b39e99a4e 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
 
 
 class VariableHolder(object):
@@ -45,6 +46,7 @@ class VariableHolder(object):
       return self._fn(*args, **kwargs)
 
 
+# TODO(allenl): make this checkpointable
 class WrappedFunction(function.Function):
   """Wraps a tf V1 piece of code in a function."""
 
@@ -77,6 +79,7 @@ class WrappedFunction(function.Function):
     return pruned_fn
 
 
+@tf_export(v1=["wrap_function"])
 def wrap_function(fn, signature, name=None):
   """Wraps the TF 1.x function fn into a graph function.
 
@@ -109,6 +112,21 @@ def wrap_function(fn, signature, name=None):
   assert float(f_sub(1.0)) == 3.0
   ```
 
+  Both `tf.compat.v1.wrap_function` and `tf.function` create a callable
+  TensorFlow graph. But while `tf.function` runs all stateful operations
+  (e.g. `tf.print`) and sequences operations to provide the same semantics as
+  eager execution, `wrap_function` is closer to the behavior of `session.run` in
+  TensorFlow 1.x. It will not run any operations unless they are required to
+  compute the function's outputs, either through a data dependency or a control
+  dependency. Nor will it sequence operations.
+
+  Unlike `tf.function`, `wrap_function` will only trace the Python function
+  once. As with placeholders in TF 1.x, shapes and dtypes must be provided to
+  `wrap_function`'s `signature` argument.
+
+  Since it is only traced once, variables and state may be created inside the
+  function and owned by the function wrapper object.
+
   Args:
     fn: python function to be wrapped
     signature: the placeholder and python arguments to be passed to the
diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py
index 4f1662ab08..cc779e6901 100644
--- a/tensorflow/python/ops/standard_ops.py
+++ b/tensorflow/python/ops/standard_ops.py
@@ -51,6 +51,7 @@ from tensorflow.python.ops.control_flow_ops import group
 from tensorflow.python.ops.control_flow_ops import no_op
 from tensorflow.python.ops.control_flow_ops import tuple  # pylint: disable=redefined-builtin
 # pylint: enable=redefined-builtin
+from tensorflow.python.eager import wrap_function
 from tensorflow.python.ops.control_flow_ops import while_loop
 from tensorflow.python.ops.data_flow_ops import *
 from tensorflow.python.ops.functional_ops import *
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index f4dce81659..9904e264eb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -2296,6 +2296,10 @@ tf_module {
     name: "while_loop"
     argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'name\', \'maximum_iterations\', \'return_same_structure\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'None\', \'False\'], "
   }
+  member_method {
+    name: "wrap_function"
+    argspec: "args=[\'fn\', \'signature\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "write_file"
     argspec: "args=[\'filename\', \'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 7bdceecdb4cb6df818c3e16a9ae6017a323346cb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 10:12:30 -0800
Subject: [PATCH 0156/1554] Update TOCO docs to point to the right place.

PiperOrigin-RevId: 221287313
---
 tensorflow/lite/toco/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/toco/README.md b/tensorflow/lite/toco/README.md
index 91f6f618a3..bd8f8282f0 100644
--- a/tensorflow/lite/toco/README.md
+++ b/tensorflow/lite/toco/README.md
@@ -8,9 +8,9 @@ the usage documentation.
 
 Usage information is given in these documents:
 
-*   [Command-line glossary](g3doc/cmdline_reference.md)
-*   [Command-line examples](g3doc/cmdline_examples.md)
-*   [Python API examples](g3doc/python_api.md)
+*   [Command-line glossary](../g3doc/convert/cmdline_reference.md)
+*   [Command-line examples](../g3doc/convert/cmdline_examples.md)
+*   [Python API examples](../g3doc/convert/python_api.md)
 
 ## Where the converter fits in the TensorFlow landscape
 
-- 
GitLab


From 00cd6b4edba956b6dd6e26013a6449c598f5c674 Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Tue, 13 Nov 2018 10:24:50 -0800
Subject: [PATCH 0157/1554] Clarify time unit in grappler ChannelCostEstimator
 and VirtualScheduler.

PiperOrigin-RevId: 221289534
---
 tensorflow/core/grappler/costs/virtual_scheduler.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index ba50e55538..424994125f 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -1022,7 +1022,8 @@ Costs VirtualScheduler::Summary() const {
       bool is_cost_accurate;
       std::tie(cost, is_cost_accurate) = op_costs_.at(item.first);
       VLOG(2) << "Node: " << item.first << ", Count: " << item.second
-              << ", Individual Cost: " << (is_cost_accurate ? "" : "~") << cost;
+              << ", Individual Cost: " << (is_cost_accurate ? "" : "~") << cost
+              << " us";
     }
   }
 
-- 
GitLab


From 13fa6ecddff7931717dfce47598e88f48edcb6b6 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 13 Nov 2018 10:45:57 -0800
Subject: [PATCH 0158/1554] Fix data dependency bug

---
 .../tensorrt/test/quantization_mnist_test.py        |   3 ++-
 ...f-00001 => model.ckpt-14070.data-00000-of-00001} | Bin
 .../{model.ckpt.index => model.ckpt-14070.index}    | Bin
 3 files changed, 2 insertions(+), 1 deletion(-)
 rename tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/{model.ckpt.data-00000-of-00001 => model.ckpt-14070.data-00000-of-00001} (100%)
 rename tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/{model.ckpt.index => model.ckpt-14070.index} (100%)

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index d160cb3380..2586d936ef 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -200,7 +200,8 @@ def get_graph_def(use_trt, batch_size, model_dir):
 class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
 
   def testEval(self):
-    model_dir = test.test_src_dir_path('contrib/tensorrt/test/quantization_mnist_test_data')
+    model_dir = test.test_src_dir_path(
+        'contrib/tensorrt/test/quantization_mnist_test_data')
     acc_tf = run(is_training=False,
         use_trt=False,
         batch_size=128,
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.data-00000-of-00001 b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.data-00000-of-00001
rename to tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.index b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index
similarity index 100%
rename from tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt.index
rename to tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index
-- 
GitLab


From d51714f8ab9c56ec0ab1bc7f0c7057da15af462c Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 13 Nov 2018 10:41:19 -0800
Subject: [PATCH 0159/1554] Disallow resizing with the NNAPI delegate

PiperOrigin-RevId: 221292529
---
 .../lite/delegates/nnapi/nnapi_delegate.cc       |  2 +-
 .../lite/delegates/nnapi/nnapi_delegate_test.cc  | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 9690c65921..4fe07004a8 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -1141,7 +1141,7 @@ class NNAPIDelegateKernel {
 TfLiteDelegate* NnApiDelegate() {
   static TfLiteDelegate delegate = {
       .data_ = nullptr,
-      .flags = kTfLiteDelegateFlagsAllowDynamicTensors,
+      .flags = kTfLiteDelegateFlagsNone,
       .Prepare = [](TfLiteContext* context,
                     TfLiteDelegate* delegate) -> TfLiteStatus {
         // Do not check nodes_ if NN API is unavailable.
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index 84a0a6a1d1..ca48af0c95 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -34,6 +34,11 @@ class SingleOpModelWithNNAPI : public SingleOpModel {
       interpreter->ModifyGraphWithDelegate(NnApiDelegate());
     });
   }
+
+  TfLiteStatus ResizeInputTensor(int tensor_index,
+                                 const std::vector<int>& dims) {
+    return interpreter_->ResizeInputTensor(tensor_index, dims);
+  }
 };
 
 class FloatAddOpModel : public SingleOpModelWithNNAPI {
@@ -97,6 +102,17 @@ TEST(NNAPIDelegate, AddWithRelu) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({0.0, 0.4, 1.0, 1.3}));
 }
 
+// Verify that resize attempts fail.
+// TODO(b/113110851): Verify success after the delegate supports resizing.
+TEST(NNAPIDelegate, ResizeFails) {
+  FloatAddOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
+                    {TensorType_FLOAT32, {1, 2, 2, 1}},
+                    {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+  m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.7, 0.8});
+  m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.5});
+  EXPECT_EQ(m.ResizeInputTensor(m.input1(), {1, 3, 3, 1}), kTfLiteError);
+}
+
 class FloatMulOpModel : public SingleOpModelWithNNAPI {
  public:
   FloatMulOpModel(const TensorData& input1, const TensorData& input2,
-- 
GitLab


From ee61c166bb984935caa71053fe765ac9a9a724fc Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 13 Nov 2018 10:45:31 -0800
Subject: [PATCH 0160/1554] [TF:XLA] New HLO pass that detects a pattern of
 AllReduce and CrossReplicaSum and rewrites it, to use an efficient
 CrossReplicaSum implementation that fully utilizes the interconnect
 bandwidth.

PiperOrigin-RevId: 221293415
---
 tensorflow/compiler/xla/literal.h             |   2 +-
 tensorflow/compiler/xla/service/BUILD         |  35 ++
 .../compiler/xla/service/ar_crs_combiner.cc   | 286 ++++++++++++
 .../compiler/xla/service/ar_crs_combiner.h    |  88 ++++
 .../xla/service/ar_crs_combiner_test.cc       | 415 ++++++++++++++++++
 tensorflow/compiler/xla/service/call_graph.cc |   9 +
 tensorflow/compiler/xla/service/call_graph.h  |   4 +
 .../compiler/xla/service/hlo_instruction.cc   |  10 +
 .../compiler/xla/service/hlo_instruction.h    |  17 +-
 .../compiler/xla/service/hlo_instructions.cc  |   5 +
 .../compiler/xla/service/hlo_instructions.h   |   1 +
 .../compiler/xla/service/pattern_matcher.h    |   2 +
 12 files changed, 868 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/ar_crs_combiner.cc
 create mode 100644 tensorflow/compiler/xla/service/ar_crs_combiner.h
 create mode 100644 tensorflow/compiler/xla/service/ar_crs_combiner_test.cc

diff --git a/tensorflow/compiler/xla/literal.h b/tensorflow/compiler/xla/literal.h
index e791048b4d..fa9a71af4c 100644
--- a/tensorflow/compiler/xla/literal.h
+++ b/tensorflow/compiler/xla/literal.h
@@ -301,7 +301,7 @@ class LiteralBase {
   //
   // Note: It's an antipattern to use this method then immediately call
   // MutableLiteralBase::Populate on the result (since that results in zero
-  // initialization, then reinitialization. Conside if a call to
+  // initialization, then reinitialization. Consider if a call to
   // absl::make_unique<Literal>(shape), followed by the call to
   // MutableLiteralBase::Populate can be used instead.
   static Literal CreateFromShape(const Shape& shape);
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 19b5c1ca25..3fc804b9f6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3513,6 +3513,41 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "ar_crs_combiner",
+    srcs = ["ar_crs_combiner.cc"],
+    hdrs = ["ar_crs_combiner.h"],
+    deps = [
+        ":call_graph",
+        ":pattern_matcher",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_pass",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "ar_crs_combiner_test",
+    srcs = ["ar_crs_combiner_test.cc"],
+    deps = [
+        ":ar_crs_combiner",
+        ":hlo",
+        ":hlo_matchers",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+    ],
+)
+
 tf_cc_test(
     name = "map_inliner_test",
     srcs = ["map_inliner_test.cc"],
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
new file mode 100644
index 0000000000..c11452a6fb
--- /dev/null
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -0,0 +1,286 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/ar_crs_combiner.h"
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/call_graph.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+
+namespace {
+
+namespace m = match;
+
+// If the argument instruction is a CRS in the sequence
+// AR -> Convert -> Add -> CRS
+// then return the AR in the sequence.
+// TODO(b/117554291): Rewrite this to recognize more general patterns,
+// not just the specific one of AR -> Add -> Convert -> CRS.
+absl::optional<HloInstruction*> MatchesArCrsPattern(
+    HloInstruction* instruction) {
+  HloInstruction *ar, *convert, *add, *crs;
+  if (Match(instruction,
+            m::CrossReplicaSum(
+                &crs, m::Add(&add, m::Op(),
+                             m::Convert(&convert,
+                                        m::CrossReplicaSum(&ar, m::Op()))))) &&
+      ar->users().size() == 1 && ar->shape().element_type() == BF16 &&
+      convert->shape().element_type() == F32 && !crs->all_reduce_id()) {
+    return ar;
+  }
+  return absl::optional<HloInstruction*>();
+}
+
+}  // namespace
+
+absl::optional<HloInstruction*> ArCrsCombiner::WhileFromBodyParameter(
+    HloInstruction* instruction) {
+  CHECK(HloOpcode::kParameter == instruction->opcode());
+  HloComputation* computation = instruction->parent();
+  auto caller_instructions = call_graph_->GetComputationCallers(computation);
+  if (caller_instructions.size() == 1) {
+    auto caller_instruction = caller_instructions[0];
+    if (caller_instruction->opcode() == HloOpcode::kWhile) {
+      return caller_instruction;
+    }
+  }
+  return absl::optional<HloInstruction*>();
+}
+
+std::vector<HloInstruction*> ArCrsCombiner::GetAllTuples(
+    HloInstruction* instruction) {
+  if (instruction->opcode() == HloOpcode::kTuple) {
+    return {instruction};
+  }
+  if (instruction->opcode() == HloOpcode::kDomain) {
+    return GetAllTuples(instruction->operands()[0]);
+  }
+  if (instruction->opcode() == HloOpcode::kParameter) {
+    auto maybe_while = WhileFromBodyParameter(instruction);
+    if (!maybe_while) {
+      return {};
+    }
+    auto while_instr = *maybe_while;
+    auto init_tuples = GetAllTuples(while_instr->while_init());
+    auto body_tuples =
+        GetAllTuples(while_instr->while_body()->root_instruction());
+    if (init_tuples.empty() || body_tuples.empty()) {
+      return {};
+    }
+    init_tuples.insert(init_tuples.end(), body_tuples.begin(),
+                       body_tuples.end());
+    return init_tuples;
+  }
+  if (instruction->opcode() == HloOpcode::kGetTupleElement) {
+    std::vector<HloInstruction*> result_tuples;
+    for (auto tuple : GetAllTuples(instruction->operands()[0])) {
+      auto tmp_tuples =
+          GetAllTuples(tuple->mutable_operand(instruction->tuple_index()));
+      if (tmp_tuples.empty()) {
+        return {};
+      }
+      result_tuples.insert(result_tuples.end(), tmp_tuples.begin(),
+                           tmp_tuples.end());
+    }
+    return result_tuples;
+  }
+  return {};
+}
+
+bool ArCrsCombiner::TupleElementsComputeSameValue(
+    HloInstruction* tuple_shaped_instruction, int64 i1, int64 i2,
+    absl::flat_hash_map<int64, int64>* visited_pairs) {
+  auto tuples = GetAllTuples(tuple_shaped_instruction);
+  if (tuples.empty()) {
+    return false;
+  }
+  for (auto tuple : tuples) {
+    CHECK(tuple->opcode() == HloOpcode::kTuple);
+    if (!InstructionsComputeSameValue(tuple->mutable_operand(i1),
+                                      tuple->mutable_operand(i2),
+                                      visited_pairs)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/* static */
+bool ArCrsCombiner::TestInstructionsComputeSameValue(HloInstruction* i1,
+                                                     HloInstruction* i2) {
+  ArCrsCombiner combiner(/*num_spatial_partitions=*/2);
+  auto module = i1->parent()->parent();
+  CHECK_EQ(module, i2->parent()->parent());
+  combiner.call_graph_ = CallGraph::Build(module);
+  absl::flat_hash_map<int64, int64> visited_pairs;
+  return combiner.InstructionsComputeSameValue(i1, i2, &visited_pairs);
+}
+
+bool ArCrsCombiner::InstructionsComputeSameValue(
+    HloInstruction* i1, HloInstruction* i2,
+    absl::flat_hash_map<int64, int64>* visited_pairs) {
+  if (i1 == i2) {
+    return true;
+  }
+  auto uid1 = i1->unique_id();
+  auto uid2 = i2->unique_id();
+  auto min_uid = std::min(uid1, uid2);
+  auto max_uid = std::max(uid1, uid2);
+  auto it = visited_pairs->find(min_uid);
+  if (it != visited_pairs->end() && max_uid == it->second) {
+    return true;
+  }
+  auto opcode1 = i1->opcode();
+  auto operands1 = i1->operands();
+  if (opcode1 != i2->opcode() || operands1.size() != i2->operands().size()) {
+    return false;
+  }
+  if (opcode1 == HloOpcode::kConstant || i1->IsCrossModuleAllReduce()) {
+    return i1->Identical(
+        *i2,
+        /*eq_operands=*/std::equal_to<const HloInstruction*>(),
+        /*eq_computations=*/std::equal_to<const HloComputation*>(),
+        /*layout_sensitive=*/false);
+  }
+  visited_pairs->emplace(min_uid, max_uid);
+  for (int i = 0; i < operands1.size(); ++i) {
+    auto operand1 = operands1[i];
+    auto operand2 = i2->operands()[i];
+    if (!InstructionsComputeSameValue(operand1, operand2, visited_pairs)) {
+      return false;
+    }
+  }
+  if (opcode1 == HloOpcode::kGetTupleElement) {
+    if (i1->tuple_index() == i2->tuple_index()) {
+      return true;
+    }
+    return TupleElementsComputeSameValue(operands1[0], i1->tuple_index(),
+                                         i2->tuple_index(), visited_pairs);
+  }
+  return true;
+}
+
+void ArCrsCombiner::GroupAllReducesById(HloModule* module) {
+  for (HloComputation* computation : module->MakeNonfusionComputations()) {
+    for (HloInstruction* instruction : computation->instructions()) {
+      auto ar = MatchesArCrsPattern(instruction);
+      if (ar) {
+        all_reduce_map_[*((*ar)->all_reduce_id())].push_back(*ar);
+      }
+    }
+  }
+}
+
+void ArCrsCombiner::KeepProvablyEqualInstructionGroups() {
+  for (auto it : all_reduce_map_) {
+    auto instruction_vec = it.second;
+    CHECK_EQ(instruction_vec.size(), num_spatial_partitions_);
+
+    auto instr_0 = instruction_vec[0];
+    auto add_0 = instr_0->users()[0]->users()[0];
+    CHECK(HloOpcode::kAdd == add_0->opcode());
+
+    for (int i = 1; i < instruction_vec.size(); ++i) {
+      auto instr_i = instruction_vec[i];
+      auto add_i = instr_i->users()[0]->users()[0];
+      CHECK(HloOpcode::kAdd == add_i->opcode());
+      absl::flat_hash_map<int64, int64> visited_pairs;
+      if (!InstructionsComputeSameValue(add_0, add_i, &visited_pairs)) {
+        all_reduce_map_.erase(it.first);
+      }
+    }
+  }
+}
+
+StatusOr<bool> ArCrsCombiner::RewriteGraph() {
+  if (all_reduce_map_.empty()) {
+    return false;
+  }
+
+  auto computation_is_addition = [](HloComputation* c) {
+    return c->instruction_count() == 3 &&
+           Match(c->root_instruction(), m::Add(m::Parameter(), m::Parameter()));
+  };
+
+  for (auto it : all_reduce_map_) {
+    auto instruction_vec = it.second;
+    for (auto all_reduce : instruction_vec) {
+      auto parent_computation = all_reduce->parent();
+      auto convert = all_reduce->users()[0];
+      auto add = convert->users()[0];
+      auto crs = add->users()[0];
+
+      if (!computation_is_addition(all_reduce->called_computations()[0]) ||
+          !computation_is_addition(crs->called_computations()[0])) {
+        continue;
+      }
+      HloInstruction* other_summand = (add->operands()[0] == convert)
+                                          ? add->operands()[1]
+                                          : add->operands()[0];
+      // Remove the AllReduce and replace the CRS with:
+      // AllReduce - (other_summand * (num_spatial_partitions_ - 1))
+      TF_CHECK_OK(
+          all_reduce->ReplaceAllUsesWith(all_reduce->mutable_operand(0)));
+      crs->set_all_reduce_id(all_reduce->all_reduce_id());
+      auto new_shape = crs->shape();
+      HloInstruction* to_subtract;
+      if (num_spatial_partitions_ == 2) {
+        to_subtract = other_summand;
+      } else {
+        Literal partitions_minus_1_lit = Literal(new_shape);
+        partitions_minus_1_lit.PopulateWithValue<float>(
+            num_spatial_partitions_ - 1);
+        auto partitions_minus_1_const = parent_computation->AddInstruction(
+            HloInstruction::CreateConstant(partitions_minus_1_lit.Clone()));
+        to_subtract =
+            parent_computation->AddInstruction(HloInstruction::CreateBinary(
+                new_shape, HloOpcode::kMultiply, other_summand,
+                partitions_minus_1_const));
+      }
+      auto sub =
+          parent_computation->AddInstruction(HloInstruction::CreateBinary(
+              new_shape, HloOpcode::kSubtract, crs, to_subtract));
+      TF_CHECK_OK(crs->ReplaceAllUsesWith(sub));
+      TF_CHECK_OK(parent_computation->RemoveInstruction(all_reduce));
+    }
+  }
+
+  return true;
+}
+
+StatusOr<bool> ArCrsCombiner::Run(HloModule* module) {
+  call_graph_ = CallGraph::Build(module);
+
+  GroupAllReducesById(module);
+
+  KeepProvablyEqualInstructionGroups();
+
+  return RewriteGraph();
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.h b/tensorflow/compiler/xla/service/ar_crs_combiner.h
new file mode 100644
index 0000000000..f6a7ef76ec
--- /dev/null
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_AR_CRS_COMBINER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_AR_CRS_COMBINER_H_
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/strings/string_view.h"
+#include "tensorflow/compiler/xla/service/call_graph.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+#include "tensorflow/compiler/xla/statusor.h"
+
+namespace xla {
+
+// Combine an AllReduce and a CrossReplicaSum when they are close to each other
+// in the graph, to use an efficient CrossReplicaSum implementation that
+// fully utilizes the interconnect bandwidth.
+class ArCrsCombiner : public HloModulePass {
+ public:
+  ArCrsCombiner(int num_spatial_partitions)
+      : num_spatial_partitions_(num_spatial_partitions) {}
+  absl::string_view name() const override { return "ar-crs-combiner"; }
+  StatusOr<bool> Run(HloModule* module) override;
+
+  // Helper method to allow testing of InstructionsComputeSameValue.
+  static bool TestInstructionsComputeSameValue(HloInstruction* i1,
+                                               HloInstruction* i2);
+
+ private:
+  // If the passed instruction is a while parameter, and the while body is only
+  // called by a single while instruction, return the while instruction.
+  absl::optional<HloInstruction*> WhileFromBodyParameter(
+      HloInstruction* instruction);
+
+  // Returns a vector of tuple instructions.
+  // If all instructions that flow to "instruction" are tuples, return them.
+  // Otherwise, return an empty vector.
+  std::vector<HloInstruction*> GetAllTuples(HloInstruction* instruction);
+
+  // Checks whether two different elements in the same tuple compute the same
+  // value.
+  bool TupleElementsComputeSameValue(
+      HloInstruction* tuple_shaped_instruction, int64 i1, int64 i2,
+      absl::flat_hash_map<int64, int64>* visited_pairs);
+
+  // Returns whether the instructions i1 and i2 can be shown to evaluate to the
+  // same value. Handling WHILE requires recursion, which may cause us to visit
+  // the same instruction again. To avoid infinite loops, we pass a cache of
+  // visited instruction pairs.
+  bool InstructionsComputeSameValue(
+      HloInstruction* i1, HloInstruction* i2,
+      absl::flat_hash_map<int64, int64>* visited_pairs);
+
+  // Populates all_reduce_map_.
+  void GroupAllReducesById(HloModule* module);
+
+  // Looks at each AllReduce group in all_reduce_map_, and keeps only the
+  // groups for which it's safe to move the AllReduce later in the HLO graph.
+  void KeepProvablyEqualInstructionGroups();
+
+  // Performs the graph rewrite that eliminates the early AllReduce and turns
+  // the later CRS into an AllReduce.
+  StatusOr<bool> RewriteGraph();
+
+  int num_spatial_partitions_;
+
+  // Map from all-reduce ids to the all reduce instructions.
+  absl::flat_hash_map<int64, std::vector<HloInstruction*>> all_reduce_map_;
+
+  std::unique_ptr<CallGraph> call_graph_;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_AR_CRS_COMBINER_H_
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
new file mode 100644
index 0000000000..9d5eaf63cc
--- /dev/null
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -0,0 +1,415 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/ar_crs_combiner.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace xla {
+namespace {
+
+namespace op = xla::testing::opcode_matchers;
+
+class ArCrsCombinerTest : public HloTestBase {};
+
+TEST_F(ArCrsCombinerTest, SameValueTestBasecase) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
+  %p = f32[2,2] parameter(0)
+  %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %constant.f32.2 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32.1, %constant.f32.2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(
+      i1, module->entry_computation()->parameter_instruction(0)));
+  EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestNumOperands) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2,2]) -> ((f32[2,2]), (f32[2,2], f32[2,2])) {
+  %p = f32[2,2] parameter(0)
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %tuple1 = (f32[2,2]) tuple(%constant.f32)
+  %tuple2 = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32)
+  ROOT %tuple = ((f32[2,2]), (f32[2,2], f32[2,2])) tuple(%tuple1, %tuple2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestTupleElementSameIndex) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
+  %p = f32[2,2] parameter(0)
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %tuple.1 = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32)
+  %get-tuple-element.1 = f32[2,2] get-tuple-element(%tuple.1), index=0
+  %get-tuple-element.2 = f32[2,2] get-tuple-element(%tuple.1), index=0
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%get-tuple-element.1, %get-tuple-element.2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestTupleElementDifferentIndex1) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
+  %p = f32[2,2] parameter(0)
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %tuple.1 = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32)
+  %get-tuple-element.1 = f32[2,2] get-tuple-element(%tuple.1), index=0
+  %get-tuple-element.2 = f32[2,2] get-tuple-element(%tuple.1), index=1
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%get-tuple-element.1, %get-tuple-element.2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestTupleElementDifferentIndex2) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
+  %p = f32[2,2] parameter(0)
+  %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %constant.f32.2 = f32[2,2] constant(f32[2,2] {{2, 3}, {4, 5}})
+  %tuple.1 = (f32[2,2], f32[2,2]) tuple(%constant.f32.1, %constant.f32.2)
+  %get-tuple-element.1 = f32[2,2] get-tuple-element(%tuple.1), index=0
+  %get-tuple-element.2 = f32[2,2] get-tuple-element(%tuple.1), index=1
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%get-tuple-element.1, %get-tuple-element.2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestWhile1) {
+  const char* module_str = R"(
+HloModule foobar
+
+%condition (x: (f32[2,2], f32[2,2])) -> pred[] {
+  %x = (f32[2,2], f32[2,2]) parameter(0)
+  %constant.0 = s32[] constant(0)
+  %constant.1 = s32[] constant(1)
+  ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %constant.0)
+}
+
+%body (x: (f32[2,2], f32[2,2])) -> (f32[2,2], f32[2,2]) {
+  %x = (f32[2,2], f32[2,2]) parameter(0)
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %get-tuple-element.1 = f32[2,2] get-tuple-element(%x), index=0
+  %get-tuple-element.2 = f32[2,2] get-tuple-element(%x), index=1
+  %add.1 = f32[2,2] add(%get-tuple-element.1, %constant.f32)
+  %add.2 = f32[2,2] add(%get-tuple-element.2, %constant.f32)
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%add.1, %add.2)
+}
+
+ENTRY %WhileLoop () -> (f32[2,2], f32[2,2]) {
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}})
+  %init.tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32)
+  ROOT %while = (f32[2,2], f32[2,2]) while(%init.tuple), condition=%condition, body=%body
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_while = module->entry_computation()->root_instruction();
+  auto body_tuple = root_while->while_body()->root_instruction();
+  auto i1 = body_tuple->operands()[0];
+  auto i2 = body_tuple->operands()[1];
+  EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestWhile2) {
+  const char* module_str = R"(
+HloModule foobar
+
+%condition (x: (f32[2,2], f32[2,2])) -> pred[] {
+  %x = (f32[2,2], f32[2,2]) parameter(0)
+  %constant.0 = s32[] constant(0)
+  %constant.1 = s32[] constant(1)
+  ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %constant.0)
+}
+
+%body (x: (f32[2,2], f32[2,2])) -> (f32[2,2], f32[2,2]) {
+  %x = (f32[2,2], f32[2,2]) parameter(0)
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %get-tuple-element.1 = f32[2,2] get-tuple-element(%x), index=0
+  %get-tuple-element.2 = f32[2,2] get-tuple-element(%x), index=1
+  %add.1 = f32[2,2] add(%get-tuple-element.1, %constant.f32)
+  %add.2 = f32[2,2] add(%get-tuple-element.2, %constant.f32)
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%add.1, %add.2)
+}
+
+ENTRY %WhileLoop () -> (f32[2,2], f32[2,2]) {
+  %constant.f32.1 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}})
+  %constant.f32.2 = f32[2,2] constant(f32[2,2] {{3, 4}, {7, 8}})
+  %init.tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32.1, %constant.f32.2)
+  ROOT %while = (f32[2,2], f32[2,2]) while(%init.tuple), condition=%condition, body=%body
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_while = module->entry_computation()->root_instruction();
+  auto body_tuple = root_while->while_body()->root_instruction();
+  auto i1 = body_tuple->operands()[0];
+  auto i2 = body_tuple->operands()[1];
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestWhile3) {
+  const char* module_str = R"(
+HloModule foobar
+
+%condition (x: (f32[2,2], f32[2,2])) -> pred[] {
+  %x = (f32[2,2], f32[2,2]) parameter(0)
+  %constant.0 = s32[] constant(0)
+  %constant.1 = s32[] constant(1)
+  ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %constant.0)
+}
+
+%body (x: (f32[2,2], f32[2,2])) -> (f32[2,2], f32[2,2]) {
+  %x = (f32[2,2], f32[2,2]) parameter(0)
+  %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %constant.f32.2 = f32[2,2] constant(f32[2,2] {{3, 4}, {1, 2}})
+  %get-tuple-element.1 = f32[2,2] get-tuple-element(%x), index=0
+  %get-tuple-element.2 = f32[2,2] get-tuple-element(%x), index=1
+  %add.1 = f32[2,2] add(%get-tuple-element.1, %constant.f32.1)
+  %add.2 = f32[2,2] add(%get-tuple-element.2, %constant.f32.2)
+  ROOT %tuple = (f32[2,2], f32[2,2]) tuple(%add.1, %add.2)
+}
+
+ENTRY %WhileLoop () -> (f32[2,2], f32[2,2]) {
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}})
+  %init.tuple = (f32[2,2], f32[2,2]) tuple(%constant.f32, %constant.f32)
+  ROOT %while = (f32[2,2], f32[2,2]) while(%init.tuple), condition=%condition, body=%body
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_while = module->entry_computation()->root_instruction();
+  auto body_tuple = root_while->while_body()->root_instruction();
+  auto i1 = body_tuple->operands()[0]->operands()[0];  // %get-tuple-element.1
+  auto i2 = body_tuple->operands()[1]->operands()[0];  // %get-tuple-element.2
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, RewritePatternArConvertAddCrs) {
+  const char* module_str = R"(
+HloModule foobar
+
+%binary_add (a: bf16[], b: bf16[]) -> bf16[] {
+  %a = bf16[] parameter(0)
+  %b = bf16[] parameter(1)
+  ROOT %add = bf16[] add(%a, %b)
+}
+
+%sum.f32 (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
+  %p = f32[2,2] parameter(0)
+  %constant.bf16 = bf16[2,2] constant(bf16[2,2] {{1, 2}, {3, 4}})
+  %constant.f32 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+
+  %cross-replica-sum.ar.1 = bf16[2,2]
+      cross-replica-sum(%constant.bf16),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%binary_add,
+      sharding={maximal device=0}
+  %convert.1 = f32[2,2]
+      convert(%cross-replica-sum.ar.1),
+      sharding={maximal device=0}
+  %add.1 = f32[2,2]
+      add(%constant.f32, %convert.1),
+      sharding={maximal device=0}
+  %cross-replica-sum.1 = f32[2,2]
+      cross-replica-sum(%add.1),
+      replica_groups={{0,1}},
+      to_apply=%sum.f32,
+      sharding={maximal device=0}
+
+  %cross-replica-sum.ar.2 = bf16[2,2]
+      cross-replica-sum(%constant.bf16),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%binary_add,
+      sharding={maximal device=1}
+  %convert.2 = f32[2,2]
+      convert(%cross-replica-sum.ar.2),
+      sharding={maximal device=1}
+  %add.2 = f32[2,2]
+      add(%constant.f32, %convert.2),
+      sharding={maximal device=1}
+  %cross-replica-sum.2 = f32[2,2]
+      cross-replica-sum(%add.2),
+      replica_groups={{0,1}},
+      to_apply=%sum.f32,
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[2,2], f32[2,2])
+      tuple(%cross-replica-sum.1, %cross-replica-sum.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Tuple(op::Subtract(op::CrossReplicaSum(), op::Constant()),
+                        op::Subtract(op::CrossReplicaSum(), op::Constant())));
+  auto sub = module->entry_computation()->root_instruction()->operands()[0];
+  auto crs_after = sub->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  ASSERT_EQ(replica_groups_before.size(), replica_groups_after.size());
+  for (int i = 0; i < replica_groups_before.size(); ++i) {
+    // Somewhat verbose way to compare the replica_ids, because EqualsProto
+    // is not available in the open-source build.
+    auto group_before = replica_groups_before[i];
+    std::vector<int64> ids_before(group_before.replica_ids().begin(),
+                                  group_before.replica_ids().end());
+    auto group_after = replica_groups_after[i];
+    std::vector<int64> ids_after(group_after.replica_ids().begin(),
+                                 group_after.replica_ids().end());
+    EXPECT_EQ(ids_before, ids_after);
+  }
+}
+
+TEST_F(ArCrsCombinerTest, OtherSummandNotTheSameDontRewrite) {
+  const char* module_str = R"(
+HloModule foobar
+
+%binary_add (a: bf16[], b: bf16[]) -> bf16[] {
+  %a = bf16[] parameter(0)
+  %b = bf16[] parameter(1)
+  ROOT %add = bf16[] add(%a, %b)
+}
+
+%sum.f32 (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %add = f32[] add(%x, %y)
+}
+
+ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
+  %p = f32[2,2] parameter(0)
+  %constant.bf16 = bf16[2,2] constant(bf16[2,2] {{1, 2}, {3, 4}})
+  %constant.f32.1 = f32[2,2] constant(f32[2,2] {{1, 2}, {3, 4}})
+  %constant.f32.2 = f32[2,2] constant(f32[2,2] {{3, 4}, {5, 6}})
+
+  %cross-replica-sum.ar.1 = bf16[2,2]
+      cross-replica-sum(%constant.bf16),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%binary_add,
+      sharding={maximal device=0}
+  %convert.1 = f32[2,2]
+      convert(%cross-replica-sum.ar.1),
+      sharding={maximal device=0}
+  %add.1 = f32[2,2]
+      add(%constant.f32.1, %convert.1),
+      sharding={maximal device=0}
+  %cross-replica-sum.1 = f32[2,2]
+      cross-replica-sum(%add.1),
+      replica_groups={{0,1}},
+      to_apply=%sum.f32,
+      sharding={maximal device=0}
+
+  %cross-replica-sum.ar.2 = bf16[2,2]
+      cross-replica-sum(%constant.bf16),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%binary_add,
+      sharding={maximal device=1}
+  %convert.2 = f32[2,2]
+      convert(%cross-replica-sum.ar.2),
+      sharding={maximal device=1}
+  %add.2 = f32[2,2]
+      add(%constant.f32.2, %convert.2),
+      sharding={maximal device=1}
+  %cross-replica-sum.2 = f32[2,2]
+      cross-replica-sum(%add.2),
+      replica_groups={{0,1}},
+      to_apply=%sum.f32,
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[2,2], f32[2,2])
+      tuple(%cross-replica-sum.1, %cross-replica-sum.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_FALSE(changed);
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/call_graph.cc b/tensorflow/compiler/xla/service/call_graph.cc
index bdd5069632..7987343bfa 100644
--- a/tensorflow/compiler/xla/service/call_graph.cc
+++ b/tensorflow/compiler/xla/service/call_graph.cc
@@ -325,6 +325,15 @@ bool CallGraph::IsFlattened() const {
   return true;
 }
 
+std::vector<HloInstruction*> CallGraph::GetComputationCallers(
+    HloComputation* c) {
+  std::vector<HloInstruction*> callers;
+  for (auto callsite : GetNode(c).caller_callsites()) {
+    callers.push_back(callsite.instruction());
+  }
+  return callers;
+}
+
 std::pair<HloInstruction*, HloInstruction*>
 CallGraph::NearestAncestorsInSameComputation(HloInstruction* a,
                                              HloInstruction* b) const {
diff --git a/tensorflow/compiler/xla/service/call_graph.h b/tensorflow/compiler/xla/service/call_graph.h
index cb56f4789d..05c7c99873 100644
--- a/tensorflow/compiler/xla/service/call_graph.h
+++ b/tensorflow/compiler/xla/service/call_graph.h
@@ -236,6 +236,10 @@ class CallGraph {
   // FlattenCallGraph.
   bool IsFlattened() const;
 
+  // Returns a vector of instructions calling the passed computation.
+  // (Often a vector of size 1.)
+  std::vector<HloInstruction*> GetComputationCallers(HloComputation* c);
+
   string ToString() const;
 
  private:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 26786ee950..b8f7726c39 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1900,6 +1900,11 @@ void HloInstruction::set_while_body(HloComputation* computation) {
   called_computations_[kBodyComputationIndex] = computation;
 }
 
+HloInstruction* HloInstruction::while_init() const {
+  CHECK_EQ(HloOpcode::kWhile, opcode_);
+  return operands_[0];
+}
+
 HloComputation* HloInstruction::true_computation() const {
   CHECK_EQ(HloOpcode::kConditional, opcode_);
   return called_computations_[kTrueComputationIndex];
@@ -3243,6 +3248,11 @@ absl::optional<int64> HloInstruction::all_reduce_id() const {
   return Cast<HloAllReduceInstruction>(this)->all_reduce_id();
 }
 
+void HloInstruction::set_all_reduce_id(
+    const absl::optional<int64>& all_reduce_id) {
+  return Cast<HloAllReduceInstruction>(this)->set_all_reduce_id(all_reduce_id);
+}
+
 const ConvolutionDimensionNumbers&
 HloInstruction::convolution_dimension_numbers() const {
   if (auto convolution = DynCast<HloConvolutionInstruction>(this)) {
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 818d4ede0f..b6bbc560d9 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -883,11 +883,15 @@ class HloInstruction {
       return false;
     }
 
-    // Use an explicit loop rather than ContainerEquals, because copying around
-    // std::functions may be too expensive in some cases.
-    for (size_t i = 0; i < operands().size(); ++i) {
-      if (!eq_operands(operand(i), other.operand(i))) {
-        return false;
+    // Two AllReduces are Identical if they have the same all_reduce_id.
+    // Their operands don't have to be Identical.
+    if (!this->IsCrossModuleAllReduce()) {
+      // Use an explicit loop rather than ContainerEquals, because copying
+      // around std::functions may be too expensive in some cases.
+      for (size_t i = 0; i < operands().size(); ++i) {
+        if (!eq_operands(operand(i), other.operand(i))) {
+          return false;
+        }
       }
     }
 
@@ -997,6 +1001,8 @@ class HloInstruction {
   void set_while_condition(HloComputation* while_condition);
   void set_while_body(HloComputation* while_body);
 
+  HloInstruction* while_init() const;
+
   // Gets/sets the true and false HloComputation for Conditional. The setters
   // should only be called by HloModule or HloComputation methods.
   //
@@ -1435,6 +1441,7 @@ class HloInstruction {
 
   // Delegates to HloAllReduceInstruction::all_reduce_id.
   absl::optional<int64> all_reduce_id() const;
+  void set_all_reduce_id(const absl::optional<int64>& all_reduce_id);
 
   // Returns data on the window in a windowed operation such as
   // convolution.
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 4c765aa375..ed3b2f1564 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -370,6 +370,11 @@ HloAllReduceInstruction::HloAllReduceInstruction(
   AppendComputation(reduce_computation);
 }
 
+void HloAllReduceInstruction::set_all_reduce_id(
+    const absl::optional<int64>& all_reduce_id) {
+  all_reduce_id_ = all_reduce_id;
+}
+
 HloInstructionProto HloAllReduceInstruction::ToProto() const {
   HloInstructionProto proto = HloCollectiveInstruction::ToProto();
   // Proto3 is so sad.
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index d43a8973cc..0b07341cb9 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -252,6 +252,7 @@ class HloAllReduceInstruction : public HloCollectiveInstruction {
   }
 
   absl::optional<int64> all_reduce_id() const { return all_reduce_id_; }
+  void set_all_reduce_id(const absl::optional<int64>& all_reduce_id);
 
   // Returns a serialized representation of this instruction.
   HloInstructionProto ToProto() const override;
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index 6152cdc609..bf5eb42e2c 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -1067,8 +1067,10 @@ XLA_UNOP_PATTERN(RoundNearestAfz)
 XLA_UNOP_PATTERN(Bitcast)
 XLA_UNOP_PATTERN(Broadcast)
 XLA_UNOP_PATTERN(Ceil)
+XLA_UNOP_PATTERN(Convert)
 XLA_UNOP_PATTERN(Copy)
 XLA_UNOP_PATTERN(Cos)
+XLA_UNOP_PATTERN(CrossReplicaSum)
 XLA_UNOP_PATTERN(Exp)
 XLA_UNOP_PATTERN(Fft)
 XLA_UNOP_PATTERN(Floor)
-- 
GitLab


From 340483dc457df4270b63467b74ba1d9e14989add Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 13 Nov 2018 10:46:48 -0800
Subject: [PATCH 0161/1554] Add back support for strided slice with longs in
 Python 2

PiperOrigin-RevId: 221293686
---
 tensorflow/python/kernel_tests/array_ops_test.py | 1 +
 tensorflow/python/ops/array_ops.py               | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index c90794c789..c5b25dc8d0 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -636,6 +636,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       _ = checker[..., bar:bar2]
       _ = checker[..., bar]
       _ = checker[..., 3]
+      _ = checker[..., 2 ** 64 // 2**63]  # Test longs in Python 2
 
   def testTensorIndexingTypeError(self):
     with self.session(use_gpu=True):
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index bbf7d166bf..78a6ca0e4c 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 import sys
 
 import numpy as np
+import six
 
 from tensorflow.python.eager import context
 from tensorflow.python.framework import common_shapes
@@ -431,7 +432,7 @@ _SUPPORTED_SLICE_DTYPES = (
 
 def _check_index(idx):
   """Check if a given value is a valid index into a tensor."""
-  if isinstance(idx, (int, tensor_shape.Dimension)):
+  if isinstance(idx, (six.integer_types, tensor_shape.Dimension)):
     return
 
   # Optimistic check. Assumptions:
-- 
GitLab


From dd783b82916fa9da90ed31b63f040151c7418d4f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 11:01:33 -0800
Subject: [PATCH 0162/1554] Add definitions for OpKernelContext::Params
 constants.

PiperOrigin-RevId: 221296559
---
 tensorflow/core/framework/op_kernel.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index e2a177569d..9e2d98b70a 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -255,6 +255,9 @@ Status OpKernelConstruction::allocate_persistent(
 
 // OpKernelContext -----------------------------------------------------------
 
+const int OpKernelContext::Params::kNeverForward;
+const int OpKernelContext::Params::kNoReservation;
+
 OpKernelContext::OpKernelContext(Params* params)
     : OpKernelContext(
           params, static_cast<int>(params->op_kernel->output_types().size())) {}
-- 
GitLab


From 72c01582a37ef838820ae3cde33ea270eb0fae03 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Sat, 10 Nov 2018 18:27:33 -0800
Subject: [PATCH 0163/1554] Fix some formatting/compilation issues

---
 tensorflow/contrib/tensorrt/BUILD             |  4 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 24 +++---
 .../tensorrt/convert/convert_nodes_test.cc    | 75 ++++++++-----------
 3 files changed, 48 insertions(+), 55 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 64c0fecf87..f30c31a789 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -521,8 +521,8 @@ cuda_py_tests(
         "//tensorflow/python/keras:keras",
         "//tensorflow/python/estimator:estimator",
     ],
-    data =[
-      "//tensorflow/contrib/tensorrt/test/quantization_mnist_test_data:quantization_mnist_data",
+    data = [
+        "//tensorflow/contrib/tensorrt/test/quantization_mnist_test_data:quantization_mnist_data",
     ],
     tags = [
         "no_cuda_on_cpu_tap",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index b8aa49506b..ca06152fac 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -425,7 +425,9 @@ class TRT_TensorOrWeights::SimpleITensor : public nvinfer1::ITensor {
   void setLocation(nvinfer1::TensorLocation location) override {}
 
 #if NV_TENSORRT_MAJOR >= 5
-  bool setDynamicRange(float min, float max) override {}
+  bool setDynamicRange(float min, float max) override { return true; }
+
+  float getDynamicRange() const override { return 0; }
 #endif
 
  private:
@@ -818,7 +820,7 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
     if (i != 0) output_name = StrCat(output_name, ":", i);
     // We need to check the name before setting it. If the input is one of the
     // engine input, setting the name here will overwrite engine input
-    // bindings which will cause runtime error. 
+    // bindings which will cause runtime error.
     if (output.is_tensor()) {
       const char* tensor_name = output.tensor()->getName();
       if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) {
@@ -1074,7 +1076,7 @@ void Converter::ApplyQuantizationRanges(bool warn_missing_ranges) {
     tensor->setDynamicRange(-range, range);
 #endif
   }
-  
+
   // Warn user about tensors that are missing ranges. If TRT fuses some layers
   // then these tensors may not actually be required, which is why this is
   // just a warning. If we are still missing ranges even after fusion,
@@ -2077,23 +2079,23 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   // to available TensorRT ops: Relu6(x) = min(Relu(x), 6)
   // ***************************************************************************
 
-  // Input Tensor 
+  // Input Tensor
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-  
+
   // Relu operation i.e. Relu(x) = max(0, x)
-  nvinfer1::IActivationLayer* relu_layer = 
+  nvinfer1::IActivationLayer* relu_layer =
       params->converter->network()->addActivation(
           *const_cast<nvinfer1::ITensor*>(tensor),
           nvinfer1::ActivationType::kRELU);
   TFTRT_RETURN_ERROR_IF_NULLPTR(relu_layer, node_def.name());
-  
+
   // Large range of relu is problematic during quantization in INT8 precision mode.
   // Setting dynamic range of relu = [0.f, 6.0f] helps with quantization.
   // TRT only uses dynamic ranges in INT8 precision mode,
   // and this does not affect the FP32 path.
   params->converter->ProvideQuantizationRange(
       relu_layer->getOutput(0), 0.0f, 6.0f);
-  
+
   // Create a constant layer to store the floating point weight i.e. 6.0f This
   // tensor will be broadcasted uniformly during elementwise `min` operation.
   // The constant has to have the same rank as the input in order for TRT to
@@ -2108,18 +2110,18 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   auto weights_ptr = static_cast<float*>(const_cast<void*>(
       weights.GetValues()));
   weights_ptr[0] = 6.f;
-  nvinfer1::IConstantLayer* const6_layer = 
+  nvinfer1::IConstantLayer* const6_layer =
       params->converter->network()->addConstant(dims, weights.GetTrtWeights());
   TFTRT_RETURN_ERROR_IF_NULLPTR(const6_layer, node_def.name());
   params->converter->ProvideQuantizationRange(
       const6_layer->getOutput(0), 0.0f, 6.0f);
-  
+
   // ElementWise Min Operation
   // Min op is a nop for INT8 execution path, as the input tensor
   // to this layer will only have values in range [0.f, 6.0f].
   const nvinfer1::ITensor* tensor_l = relu_layer->getOutput(0);
   const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0);
-  nvinfer1::IElementWiseLayer* relu6_layer = 
+  nvinfer1::IElementWiseLayer* relu6_layer =
       params->converter->network()->addElementWise(
           *const_cast<nvinfer1::ITensor*>(tensor_l),
           *const_cast<nvinfer1::ITensor*>(tensor_r),
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 77cf3e7ef0..257394c0a3 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -137,8 +137,8 @@ class FakeITensor : public nvinfer1::ITensor {
 
   FakeITensor(const nvinfer1::Dims& dims) : dims_(dims), dynamic_range_(0.0f) {}
 
-  FakeITensor(const std::vector<int>& dims) : dims_(GetTestDims(dims)),
-      dynamic_range_(0.0f) {}
+  FakeITensor(const std::vector<int>& dims)
+      : dims_(GetTestDims(dims)), dynamic_range_(0.0f) {}
 
   void setName(const char* name) override { name_ = name; }
 
@@ -172,7 +172,7 @@ class FakeITensor : public nvinfer1::ITensor {
     return true;
   }
 
-  float getDynamicRange() { return dynamic_range_; }
+  float getDynamicRange() const override { return dynamic_range_; }
 #endif
 
  private:
@@ -441,8 +441,8 @@ class ConverterTest : public ::testing::Test {
     return converter_->GetInputs(node_def, inputs);
   }
 
-  Status GetWeightRange(const TRT_ShapedWeights& weights,
-                        float* out_min, float* out_max) const {
+  Status GetWeightRange(const TRT_ShapedWeights& weights, float* out_min,
+                        float* out_max) const {
     return converter_->GetWeightRange(weights, out_min, out_max);
   }
 
@@ -1241,21 +1241,20 @@ TEST_F(OpConverterTest, ConvertMatMul) {
 TEST_F(OpConverterTest, ConvertQuantize) {
   {
     // Input list is empty, should fail.
-    NodeDef node_def = MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2",
-        {});
+    NodeDef node_def =
+        MakeNodeDef("my_quantize", "QuantizeAndDequantizeV2", {});
     RunConversion(
         node_def, error::INVALID_ARGUMENT,
         "Invalid number of inputs for QuantizeAndDequantizeV2, at my_quantize");
   }
   {
     // FakeQuantWithMinMaxArgs attributes are empty, should fail.
-    NodeDef node_def = MakeNodeDef("my_quantize", "FakeQuantWithMinMaxArgs",
-        {"input"});
+    NodeDef node_def =
+        MakeNodeDef("my_quantize", "FakeQuantWithMinMaxArgs", {"input"});
     AddTestTensor("input", {1, 2, 3});
-    RunConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Min or max attribute not found for FakeQuantWithMinMaxArgs "
-        "at my_quantize");
+    RunConversion(node_def, error::INVALID_ARGUMENT,
+                  "Min or max attribute not found for FakeQuantWithMinMaxArgs "
+                  "at my_quantize");
   }
   {
     // FakeQuantWithMinMaxArgs ranges set via attributes, ok.
@@ -1265,8 +1264,8 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     ops::FakeQuantWithMinMaxArgs::Attrs quantize_attrs;
     quantize_attrs.min_ = -6.0f;
     quantize_attrs.max_ = 6.0f;
-    auto quantize =
-        ops::FakeQuantWithMinMaxArgs(s.WithOpName("my_quantize"), input, quantize_attrs);
+    auto quantize = ops::FakeQuantWithMinMaxArgs(s.WithOpName("my_quantize"),
+                                                 input, quantize_attrs);
     const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
     RunConversion(node_def);
@@ -1284,14 +1283,12 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
     auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
-    auto quantize = ops::FakeQuantWithMinMaxVars(s.WithOpName("my_quantize"),
-                                                 input,
-                                                 weights_min,
-                                                 weights_max);
+    auto quantize = ops::FakeQuantWithMinMaxVars(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max);
     const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
     AddTestWeights<float>("weights_min", {1}, {-6.0f});
-    AddTestWeights<float>("weights_max", {1}, {6.0f}); 
+    AddTestWeights<float>("weights_max", {1}, {6.0f});
     RunConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
@@ -1307,14 +1304,12 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
     auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
-    auto quantize = ops::QuantizeAndDequantizeV2(s.WithOpName("my_quantize"),
-                                                 input,
-                                                 weights_min,
-                                                 weights_max);
+    auto quantize = ops::QuantizeAndDequantizeV2(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max);
     const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
     AddTestWeights<float>("weights_min", {1}, {-6.0f});
-    AddTestWeights<float>("weights_max", {1}, {6.0f}); 
+    AddTestWeights<float>("weights_max", {1}, {6.0f});
     RunConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
@@ -1331,16 +1326,13 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
     auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
     auto num_bits = ops::Placeholder(s.WithOpName("num_bits"), DT_INT32);
-    auto quantize = ops::QuantizeAndDequantizeV3(s.WithOpName("my_quantize"),
-                                                 input,
-                                                 weights_min,
-                                                 weights_max,
-                                                 num_bits);
+    auto quantize = ops::QuantizeAndDequantizeV3(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max, num_bits);
     const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
     AddTestWeights<float>("weights_min", {1}, {-6.0f});
-    AddTestWeights<float>("weights_max", {1}, {6.0f}); 
-    AddTestWeights<int>("num_bits", {1}, {8}); 
+    AddTestWeights<float>("weights_max", {1}, {6.0f});
+    AddTestWeights<int>("num_bits", {1}, {8});
     RunConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
@@ -1356,15 +1348,14 @@ TEST_F(OpConverterTest, ConvertQuantize) {
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
     auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
-    auto quantize = ops::QuantizeAndDequantizeV2(s.WithOpName("my_quantize"),
-                                                 input,
-                                                 weights_min,
-                                                 weights_max);
+    auto quantize = ops::QuantizeAndDequantizeV2(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max);
     const NodeDef& node_def = quantize.operation.node()->def();
     AddTestTensor("input", {1, 2, 3});
     AddTestTensor("weights_min", {1});
     AddTestTensor("weights_max", {1});
-    RunConversion(node_def, error::INVALID_ARGUMENT,
+    RunConversion(
+        node_def, error::INVALID_ARGUMENT,
         "Min and max inputs for QuantizeAndDequantizeV2 must be weights not "
         "tensors, at my_quantize");
   }
@@ -1374,9 +1365,8 @@ TEST_F(OpConverterTest, ConvertRelu6) {
   {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {});
-    RunConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Invalid number of inputs for Relu6, at my_relu6");
+    RunConversion(node_def, error::INVALID_ARGUMENT,
+                  "Invalid number of inputs for Relu6, at my_relu6");
   }
 
   // Get the NodeDef for Relu6.
@@ -1384,7 +1374,7 @@ TEST_F(OpConverterTest, ConvertRelu6) {
   auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
   auto relu6 = ops::Relu6(s.WithOpName("my_relu6"), input);
   const NodeDef& node_def = relu6.operation.node()->def();
-  
+
   {
     // Clip tensor values and set quantization ranges, ok.
     Reset();
@@ -1404,7 +1394,8 @@ TEST_F(OpConverterTest, ConvertRelu6) {
     // Input is weights, should fail.
     Reset();
     AddTestWeights<float>("input", {1, 2, 3}, {-100, -1, 0, 3, 5, 9});
-    RunConversion(node_def, error::UNIMPLEMENTED,
+    RunConversion(
+        node_def, error::UNIMPLEMENTED,
         "Relu6 is only implemented for tensors, not weights, at my_relu6");
   }
 }
-- 
GitLab


From 78c12aa260765cf8b8b24c3ec394cf1663d70a26 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 11:17:26 -0800
Subject: [PATCH 0164/1554] Move contrib.all_reduce library implementation into
 core. * We forward they symbols so existing code referring to
 contrib.all_reduce continues to work. * Also update
 contrib.distribute.cross_tower_utils to point to the new location of the
 all_reduce library.

PiperOrigin-RevId: 221299803
---
 tensorflow/contrib/all_reduce/BUILD           |  27 +-
 .../contrib/all_reduce/python/all_reduce.py   | 841 +----------------
 tensorflow/contrib/distribute/python/BUILD    |   2 +-
 .../distribute/python/cross_tower_utils.py    |   2 +-
 tensorflow/python/distribute/BUILD            |  35 +
 tensorflow/python/distribute/all_reduce.py    | 860 ++++++++++++++++++
 .../distribute}/all_reduce_test.py            |   4 +-
 7 files changed, 902 insertions(+), 869 deletions(-)
 create mode 100644 tensorflow/python/distribute/all_reduce.py
 rename tensorflow/{contrib/all_reduce/python => python/distribute}/all_reduce_test.py (98%)

diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD
index a513aa1e7c..f6c6560c1c 100644
--- a/tensorflow/contrib/all_reduce/BUILD
+++ b/tensorflow/contrib/all_reduce/BUILD
@@ -9,8 +9,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "tf_py_test")
-
 py_library(
     name = "all_reduce_py",
     srcs = ["__init__.py"],
@@ -29,29 +27,6 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:nccl_ops",
-    ],
-)
-
-tf_py_test(
-    name = "all_reduce_test",
-    srcs = ["python/all_reduce_test.py"],
-    additional_deps = [
-        ":all_reduce",
-        "//third_party/py/numpy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:state_ops",
+        "//tensorflow/python/distribute:all_reduce",
     ],
 )
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 25f4b4b8d3..238cdaf8a7 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -18,842 +18,5 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-import math
-
-from tensorflow.python.framework import device as device_lib
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nccl_ops
-
-
-def _flatten_tensors(tensors):
-  """Check tensors for isomorphism and flatten.
-
-  Args:
-    tensors: list of T `tf.Tensor` which must all have the same shape.
-
-  Returns:
-    tensors: a list of T `tf.Tensor` which are flattened (1D) views of tensors
-    shape: the original shape of each element of input tensors
-
-  Raises:
-    ValueError: tensors are empty or non-isomorphic or have unknown shape.
-  """
-  if not tensors:
-    raise ValueError("tensors cannot be empty")
-  shape = tensors[0].shape
-  for tensor in tensors:
-    shape = shape.merge_with(tensor.shape)
-  if not shape.is_fully_defined():
-    raise ValueError("Tensors must have statically known shape.")
-  if len(shape) != 1:
-    reshaped = []
-    for t in tensors:
-      with ops.colocate_with(t):
-        reshaped.append(array_ops.reshape(t, [-1]))
-    tensors = reshaped
-  return tensors, shape
-
-
-def _reshape_tensors(tensors, shape):
-  """Reshape tensors flattened by _flatten_tensors.
-
-  Args:
-    tensors: list of T `tf.Tensor` of identical length 1D tensors.
-    shape: list of integers describing the desired shape.  Product of
-      the elements must equal the length of each tensor.
-
-  Returns:
-    list of T `tf.Tensor` which are the reshaped inputs.
-  """
-  reshaped = []
-  for t in tensors:
-    with ops.colocate_with(t):
-      reshaped.append(array_ops.reshape(t, shape))
-  return reshaped
-
-
-def _padded_split(tensor, pieces):
-  """Like split for 1D tensors but pads-out case where len % pieces != 0.
-
-  Args:
-    tensor: T `tf.Tensor` that must be 1D.
-    pieces: a positive integer specifying the number of pieces into which
-      tensor should be split.
-
-  Returns:
-    list of T `tf.Tensor` of length pieces, which hold the values of
-      thin input tensor, in order.  The final tensor may
-      be zero-padded on the end to make its size equal to those of all
-      of the other tensors.
-
-  Raises:
-    ValueError: The input tensor is not 1D.
-  """
-  shape = tensor.shape
-  if 1 != len(shape):
-    raise ValueError("input tensor must be 1D")
-  tensor_len = shape.dims[0].value
-  with ops.colocate_with(tensor):
-    if tensor_len % pieces != 0:
-      # pad to an even length
-      chunk_size = 1 + tensor_len // pieces
-      if pieces > tensor_len:
-        # This is an edge case that should not come up in practice,
-        # i.e. a different reduction algorithm would be better,
-        # but we'll make it work just for completeness.
-        pad_len = pieces - tensor_len
-        extended_whole = array_ops.concat(
-            [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0)
-        parts = array_ops.split(extended_whole, pieces)
-        return parts, pad_len
-      elif (pieces - 1) * chunk_size >= tensor_len:
-        # Another edge case of limited real interest.
-        pad_len = (pieces * chunk_size) % tensor_len
-        extended_whole = array_ops.concat(
-            [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0)
-        parts = array_ops.split(extended_whole, pieces)
-        return parts, pad_len
-      else:
-        last_chunk_size = tensor_len - (pieces - 1) * chunk_size
-        pad_len = chunk_size - last_chunk_size
-        piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size]
-        parts = array_ops.split(tensor, piece_lens)
-        parts[-1] = array_ops.concat(
-            [parts[-1], array_ops.zeros([pad_len], dtype=tensor.dtype)], 0)
-        return parts, pad_len
-    else:
-      return array_ops.split(tensor, pieces), 0
-
-
-def _strip_padding(tensors, pad_len):
-  """Strip the suffix padding added by _padded_split.
-
-  Args:
-    tensors: list of T `tf.Tensor` of identical length 1D tensors.
-    pad_len: number of elements to be stripped from the end of each tensor.
-
-  Returns:
-    list of T `tf.Tensor` which are the stripped inputs.
-
-  Raises:
-    ValueError: tensors must be a non-empty list of 1D tensors, and
-      each must be longer than pad_len.
-  """
-  if not tensors:
-    raise ValueError("tensors cannot be empty")
-  shape = tensors[0].shape
-  if len(shape) > 1:
-    raise ValueError("tensors must be 1D")
-  prefix_len = int(shape[0] - pad_len)
-  if prefix_len < 0:
-    raise ValueError("pad_len longer than tensor")
-  stripped = []
-  for t in tensors:
-    with ops.colocate_with(t):
-      stripped.append(array_ops.slice(t, [0], [prefix_len]))
-  return stripped
-
-
-def _ragged_split(tensor, pieces):
-  """Like split for 1D tensors but allows case where len % pieces != 0.
-
-  Args:
-    tensor: T `tf.Tensor` that must be 1D.
-    pieces: a positive integer specifying the number of pieces into which
-      tensor should be split.
-
-  Returns:
-    list of T `tf.Tensor` of length pieces, which hold the values of
-      the input tensor, in order.  The final tensor may be shorter
-      than the others, which will all be of equal length.
-
-  Raises:
-    ValueError: input tensor must be 1D.
-  """
-  shape = tensor.shape
-  if 1 != len(shape):
-    raise ValueError("input tensor must be 1D")
-  tensor_len = shape.dims[0].value
-  chunk_size = tensor_len // pieces
-  with ops.colocate_with(tensor):
-    if tensor_len != (pieces * chunk_size):
-      # last piece will be short
-      assert pieces > 1
-      last_chunk_size = tensor_len - ((pieces - 1) * chunk_size)
-      assert last_chunk_size > 0
-      piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size]
-      return array_ops.split(tensor, piece_lens)
-    else:
-      return array_ops.split(tensor, pieces)
-
-
-def _ring_permutations(num_workers, num_subchunks, gpu_perm):
-  """"Generate an array of device index arrays, one for each subchunk.
-
-  In the basic ring reduction algorithm there are size(T)/num_devices
-  data chunks and each device process one chunk per tick, i.e. sending
-  one chunk and receiving one chunk.  The idea of subchunking is that
-  each device processes num_subchunks smaller data regions per tick,
-  and the ring rank permutation is different for each subchunk index
-  so that a device is potentially sending to and receiving from
-  num_subchunks different other devices at each tick.  Where multiple
-  independent data channels exist between devices, this strategy
-  supplies a method of using them in parallel.
-
-  Args:
-    num_workers: number of worker tasks
-    num_subchunks: number of subchunks into which to divide each per-GPU chunk.
-    gpu_perm: an array of integers in [0, num_gpus-1] giving the default
-      ring order of GPUs at each worker.  Other permutations will be generated
-      by rotating this array and splicing together per-worker instances.
-
-  Raises:
-    ValueError: the number of subchunks may not exceed the number of GPUs.
-
-  Returns:
-    pred_by_s_d: list of lists that maps (by index) from (subchunk, dev) to
-        preceding device in the permutation for that subchunk.  The
-        device index of GPU i at worker j is i + (j * num_gpus).
-    rank_by_s_d: list of lists that maps (by index) from (subchunk, dev) to
-       local rank of device d in the permutation for that subchunk.
-  """
-  num_gpus = len(gpu_perm)
-  devices = num_workers * num_gpus
-  if devices == 0:
-    return [], []
-  if num_subchunks > num_gpus:
-    raise ValueError(
-        "num_subchunks %d must be <= num_gpus %d" % (num_subchunks, num_gpus))
-  rotation_interval = max(1, int(num_gpus / num_subchunks))
-  perms_by_s = []
-  for s in range(0, num_subchunks):
-    full_order = []
-    offset = s * rotation_interval
-    for w in range(0, num_workers):
-      default_order = [(w * num_gpus) + i for i in gpu_perm]
-      dev_order = default_order[offset:] + default_order[:offset]
-      full_order += dev_order
-    perms_by_s.append(full_order)
-  pred_by_s_d = [[-1 for d in range(0, devices)]
-                 for s in range(0, num_subchunks)]
-  rank_by_s_d = [[-1 for d in range(0, devices)]
-                 for s in range(0, num_subchunks)]
-  for s in range(0, num_subchunks):
-    for d in range(0, devices):
-      for t in range(0, devices):
-        if d == perms_by_s[s][t]:
-          rank_by_s_d[s][d] = t
-          pred_by_s_d[s][d] = perms_by_s[s][(t + devices - 1) % devices]
-          break
-  return (pred_by_s_d, rank_by_s_d)
-
-
-def build_ring_all_reduce(input_tensors, num_workers, num_subchunks,
-                          gpu_perm, red_op, un_op=None):
-  """Construct a subgraph performing a ring-style all-reduce of input_tensors.
-
-  Args:
-    input_tensors: a list of T `tf.Tensor` objects, which must all
-      have the same shape and type.
-    num_workers: number of worker tasks spanned by input_tensors.
-    num_subchunks: number of subchunks each device should process in one tick.
-    gpu_perm: a list of ints giving a ring-wise rank ordering of GPUs at
-      each worker.  All workers must have the same number of
-      GPUs with the same rank ordering.  If NVLINK is available, this should
-      be a ring order supported by NVLINK edges.
-    red_op: a binary operator for elementwise reduction.
-    un_op: an optional unary operator to apply to fully reduced values.
-
-  Raises:
-    ValueError: empty input_tensors or they don't all have same
-    size.
-
-  Returns:
-    a list of T `tf.Tensor` identical sum-reductions of input_tensors.
-  """
-  if len(input_tensors) < 2:
-    raise ValueError("input_tensors must be length 2 or longer")
-  input_tensors, shape = _flatten_tensors(input_tensors)
-  devices = [t.device for t in input_tensors]
-  (pred_by_s_d, rank_by_s_d) = _ring_permutations(
-      num_workers, num_subchunks, gpu_perm)
-  chunks_by_dev, pad_len = _build_ring_gather(
-      input_tensors, devices,
-      num_subchunks, pred_by_s_d, rank_by_s_d, red_op)
-  if un_op:
-    chunks_by_dev = _apply_unary_to_chunks(un_op, chunks_by_dev)
-  output_tensors = _build_ring_scatter(pred_by_s_d, rank_by_s_d,
-                                       chunks_by_dev)
-  if pad_len > 0:
-    output_tensors = _strip_padding(output_tensors, pad_len)
-  if len(shape) != 1:
-    output_tensors = _reshape_tensors(output_tensors, shape)
-  return output_tensors
-
-
-def _build_ring_gather(input_tensors, devices, num_subchunks,
-                       pred_by_s_d, rank_by_s_d, red_op):
-  """Construct a subgraph for the first (reduction) pass of ring all-reduce.
-
-  Args:
-    input_tensors: a list of T `tf.Tensor` 1D input tensors of same
-      shape and type.
-    devices: array of device name strings
-    num_subchunks: number of subchunks each device should process in one tick.
-    pred_by_s_d: as produced by _ring_permutations
-    rank_by_s_d: as produced by _ring_permutations
-    red_op: a binary operator for elementwise reduction
-
-  Raises:
-    ValueError: tensors must all be one dimensional.
-
-  Returns:
-    list of list of T `tf.Tensor` of (partially) reduced values where
-    exactly num_subchunks chunks at each device are fully reduced.
-  """
-  num_devices = len(input_tensors)
-  if num_devices == 0:
-    return []
-  if num_devices == 1:
-    return input_tensors
-  shape = input_tensors[0].shape
-  if 1 != len(shape):
-    raise ValueError("input tensors must be 1D")
-  num_chunks = num_devices * num_subchunks
-  num_ticks = num_devices - 1
-  # Initialize chunks_by_dev with splits of the input tensors.
-  chunks_by_dev = []
-  split_pad_len = 0
-  for d in range(0, num_devices):
-    with ops.device(devices[d]):
-      splits, split_pad_len = _padded_split(input_tensors[d], num_chunks)
-      chunks_by_dev.append(splits)
-  # Reduction phase
-  for tick in range(0, num_ticks):
-    # One new partial reduction for every chunk
-    new_partial_reductions = [None for _ in range(0, num_chunks)]
-    # Compute reductions with respect to last tick's values
-    for d in range(0, num_devices):
-      with ops.device(devices[d]):
-        for s in range(0, num_subchunks):
-          rank = rank_by_s_d[s][d]
-          seg_index = (rank + num_devices - (2 + tick)) % num_devices
-          pred_dev = pred_by_s_d[s][d]
-          chunk_index = (seg_index * num_subchunks) + s
-          new_partial_reductions[chunk_index] = red_op(
-              chunks_by_dev[pred_dev][chunk_index],
-              chunks_by_dev[d][chunk_index])
-    # Update chunks_by_dev with the new values at the end of the tick.
-    for d in range(0, num_devices):
-      for s in range(0, num_subchunks):
-        rank = rank_by_s_d[s][d]
-        seg_index = (rank + num_devices - (2 + tick)) % num_devices
-        chunk_index = (seg_index * num_subchunks) + s
-        chunks_by_dev[d][chunk_index] = new_partial_reductions[chunk_index]
-  return chunks_by_dev, split_pad_len
-
-
-def _apply_unary_to_chunks(f, chunks_by_dev):
-  """Apply a unary op to each tensor in chunks_by_dev, on same device.
-
-  Args:
-    f: a unary function over T `tf.Tensor`.
-    chunks_by_dev: list of lists of T `tf.Tensor`.
-
-  Returns:
-    new list of lists of T `tf.Tensor` with the same structure as
-    chunks_by_dev containing the derived tensors.
-  """
-  output = []
-  for x in chunks_by_dev:
-    with ops.colocate_with(x[0]):
-      output.append([f(t) for t in x])
-  return output
-
-
-def _build_ring_scatter(pred_by_s_d, rank_by_s_d,
-                        chunks_by_dev):
-  """Construct subgraph for second (scatter) pass of ring all-reduce.
-
-  Args:
-    pred_by_s_d: as produced by _ring_permutations
-    rank_by_s_d: as produced by _ring_permutations
-    chunks_by_dev: list of list of T `tf.Tensor` indexed by ints
-      (device, chunk)
-
-  Raises:
-    ValueError: chunks_by_dev is not well-formed
-
-  Returns:
-    list of T `tf.Tensor` which are the fully reduced tensors, one
-    at each device corresponding to the outer dimension of chunks_by_dev.
-  """
-  num_devices = len(chunks_by_dev)
-  num_chunks = len(chunks_by_dev[0])
-  if 0 != num_chunks % num_devices:
-    raise ValueError(
-        "Expect number of chunks per device to be divisible by num_devices")
-  num_subchunks = int(num_chunks / num_devices)
-  num_ticks = num_devices - 1
-  for tick in range(0, num_ticks):
-    passed_values = [None for _ in range(0, num_chunks)]
-    for d in range(0, num_devices):
-      with ops.colocate_with(chunks_by_dev[d][0]):
-        for s in range(0, num_subchunks):
-          rank = rank_by_s_d[s][d]
-          seg_index = (rank + num_devices - (1 + tick)) % num_devices
-          pred_dev = pred_by_s_d[s][d]
-          chunk_index = (seg_index * num_subchunks) + s
-          passed_values[chunk_index] = array_ops.identity(
-              chunks_by_dev[pred_dev][chunk_index])
-    for d in range(0, num_devices):
-      for s in range(0, num_subchunks):
-        rank = rank_by_s_d[s][d]
-        seg_index = (rank + num_devices - (1 + tick)) % num_devices
-        chunk_index = (seg_index * num_subchunks) + s
-        chunks_by_dev[d][chunk_index] = passed_values[chunk_index]
-  # Join chunks at each device.
-  output = []
-  for x in chunks_by_dev:
-    with ops.colocate_with(x[0]):
-      output.append(array_ops.concat(x, 0))
-  return output
-
-
-def build_recursive_hd_all_reduce(input_tensors, red_op, un_op=None):
-  """Construct a subgraph for recursive halving-doubling all-reduce.
-
-  The recursive halving-doubling algorithm is described in
-  http://www.mcs.anl.gov/~thakur/papers/ijhpca-coll.pdf
-
-  The concept is to arrange the participating n devices in
-  a linear sequence where devices exchange data pairwise
-  with one other device in each round.  During the gather
-  phase there are lg(n) rounds where devices exchange
-  increasingly smaller sub-tensors with another device
-  at increasingly greater distances, until at the top
-  each device has 1/n of the fully reduced values.  During the
-  scatter phase each device exchanges its fully reduced
-  sub-tensor (which doubles in length at each round)
-  with one other device at increasingly smaller distances
-  until each device has all of the fully reduced values.
-
-  Note: this preliminary version requires that len(input_tensors) be a
-    power of 2.  TODO(tucker): relax this restriction.  Also, the
-    number of elements in each tensor must be divisible by 2^h where h
-    is the number of hops in each phase.  This will also be relaxed in
-    the future with edge-case specific logic.
-
-  Args:
-    input_tensors: list of T `tf.Tensor` to be elementwise reduced.
-    red_op: a binary elementwise reduction Op.
-    un_op: an optional unary elementwise Op to apply to reduced values.
-
-  Returns:
-    list of T `tf.Tensor` which are the fully reduced tensors, one
-    at each device of input_tensors.
-
-  Raises:
-    ValueError: num_devices not a power of 2, or tensor len not divisible
-    by 2 the proper number of times.
-  """
-  devices = [t.device for t in input_tensors]
-  input_tensors, shape = _flatten_tensors(input_tensors)
-  reduced_shards = _build_recursive_hd_gather(input_tensors, devices, red_op)
-  if un_op:
-    reduced_shards = [un_op(t) for t in reduced_shards]
-  output_tensors = _build_recursive_hd_scatter(reduced_shards, devices)
-  if len(shape) != 1:
-    output_tensors = _reshape_tensors(output_tensors, shape)
-  return output_tensors
-
-
-def _build_recursive_hd_gather(input_tensors, devices, red_op):
-  """Construct the gather phase of recursive halving-doubling all-reduce.
-
-  Args:
-    input_tensors: list of T `tf.Tensor` to be elementwise reduced.
-    devices: a list of strings naming the devices hosting input_tensors,
-      which will also be used to host the (partial) reduction values.
-    red_op: a binary elementwise reduction Op.
-
-  Returns:
-    list of T `tf.Tensor` which are the fully reduced tensor shards.
-
-  Raises:
-    ValueError: num_devices not a power of 2, or tensor len not divisible
-    by 2 the proper number of times.
-  """
-  num_devices = len(devices)
-  num_hops = int(math.log(num_devices, 2))
-  if num_devices != (2 ** num_hops):
-    raise ValueError("num_devices must be a power of 2")
-  chunks = input_tensors
-  for h in range(0, num_hops):
-    span = 2 ** h
-    group_size = span * 2
-    new_chunks = [[] for _ in devices]
-    for d in range(0, num_devices):
-      if (d % group_size) >= (group_size / 2):
-        # skip right half of a pair
-        continue
-      left_dev = devices[d]
-      right_dev = devices[d + span]
-      left_split = array_ops.split(chunks[d], 2)
-      right_split = array_ops.split(chunks[d+span], 2)
-      with ops.device(left_dev):
-        new_chunks[d] = red_op(left_split[0], right_split[0])
-      with ops.device(right_dev):
-        new_chunks[d + span] = red_op(left_split[1], right_split[1])
-    chunks = new_chunks
-  return chunks
-
-
-def _build_recursive_hd_scatter(input_tensors, devices):
-  """Construct the scatter phase of recursive halving-doublng all-reduce.
-
-  Args:
-    input_tensors: list of T `tf.Tensor` that are fully-reduced shards.
-    devices: a list of strings naming the devices on which the reconstituted
-      full tensors should be placed.
-
-  Returns:
-    list of T `tf.Tensor` which are the fully reduced tensors.
-  """
-  num_devices = len(devices)
-  num_hops = int(math.log(num_devices, 2))
-  assert num_devices == (2 ** num_hops), "num_devices must be a power of 2"
-  chunks = input_tensors
-  for h in reversed(range(0, num_hops)):
-    span = 2 ** h
-    group_size = span * 2
-    new_chunks = [[] for _ in devices]
-    for d in range(0, num_devices):
-      if (d % group_size) >= (group_size / 2):
-        # skip right half of a pair
-        continue
-      left_idx = d
-      right_idx = d + span
-      left_dev = devices[left_idx]
-      right_dev = devices[right_idx]
-      with ops.device(left_dev):
-        new_chunks[left_idx] = array_ops.concat([chunks[left_idx],
-                                                 chunks[right_idx]], 0)
-      with ops.device(right_dev):
-        new_chunks[right_idx] = array_ops.concat([chunks[left_idx],
-                                                  chunks[right_idx]], 0)
-    chunks = new_chunks
-  return chunks
-
-
-def build_shuffle_all_reduce(input_tensors, gather_devices, red_op, un_op=None):
-  """Construct a subgraph for shuffle all-reduce.
-
-  Shuffle reduce is essentially the algorithm implemented when using
-  parameter servers.  Suppose tensor length is n, there are d devices
-  and g gather shards.  Each device sends a n/g length sub-tensor to
-  each gather shard.  The gather shards perform a reduction across d
-  fragments, then broadcast the result back to each device.  The
-  devices then join the g fully reduced fragments they receive from
-  the shards.  The gather shards could perform d-1 pairwise
-  reductions, or one d-way reduction.  The first is better where
-  reduction Op time is low compared to transmission time, the second
-  better in the other case.
-
-  Args:
-    input_tensors: list of T @(tf.Tensor} values to be reduced.
-    gather_devices: list of names of devices on which reduction shards
-      should be placed.
-    red_op: an n-array elementwise reduction Op
-    un_op: optional elementwise unary Op to be applied to fully-reduced values.
-
-  Returns:
-    list of T `tf.Tensor` which are the fully reduced tensors.
-  """
-  input_tensors, shape = _flatten_tensors(input_tensors)
-  dst_devices = [t.device for t in input_tensors]
-  reduced_shards = _build_shuffle_gather(input_tensors, gather_devices,
-                                         red_op, un_op)
-  output_tensors = _build_shuffle_scatter(reduced_shards, dst_devices)
-  if len(shape) != 1:
-    output_tensors = _reshape_tensors(output_tensors, shape)
-  return output_tensors
-
-
-def _build_shuffle_gather(input_tensors, gather_devices, red_op, un_op=None):
-  """Construct the gather (concentrate and reduce) phase of shuffle all-reduce.
-
-  Args:
-    input_tensors: list of T @(tf.Tensor} values to be reduced.
-    gather_devices: list of names of devices on which reduction shards
-      should be placed.
-    red_op: the binary reduction Op
-    un_op: optional elementwise unary Op to be applied to fully-reduced values.
-
-  Returns:
-    list of T `tf.Tensor` which are the fully reduced shards.
-
-  Raises:
-    ValueError: inputs not well-formed.
-  """
-  num_source_devices = len(input_tensors)
-  num_gather_devices = len(gather_devices)
-  shape = input_tensors[0].shape
-  if len(shape) != 1:
-    raise ValueError("input_tensors must be 1D")
-  shards_by_source = []
-  for d in range(0, num_source_devices):
-    with ops.colocate_with(input_tensors[d]):
-      shards_by_source.append(
-          _ragged_split(input_tensors[d], num_gather_devices))
-  reduced_shards = []
-  for d in range(0, num_gather_devices):
-    with ops.device(gather_devices[d]):
-      values = [s[d] for s in shards_by_source]
-      red_shard = red_op(values)
-      if un_op:
-        red_shard = un_op(red_shard)
-      reduced_shards.append(red_shard)
-  return reduced_shards
-
-
-def _build_shuffle_scatter(reduced_shards, dst_devices):
-  """Build the scatter phase of shuffle all-reduce.
-
-  Args:
-    reduced_shards:  list of T @(tf.Tensor} fully reduced shards
-    dst_devices: list of names of devices at which the fully-reduced value
-      should be reconstituted.
-
-  Returns:
-    list of T `tf.Tensor` scattered tensors.
-  """
-  num_devices = len(dst_devices)
-  out_tensors = []
-  for d in range(0, num_devices):
-    with ops.device(dst_devices[d]):
-      out_tensors.append(array_ops.concat(reduced_shards, 0))
-  return out_tensors
-
-
-def _split_by_task(devices, values):
-  """Partition devices and values by common task.
-
-  Args:
-    devices: list of device name strings
-    values: list of T `tf.tensor` of same length as devices.
-
-  Returns:
-    (per_task_devices, per_task_values) where both values are
-    lists of lists with isomorphic structure: the outer list is
-    indexed by task, and the inner list has length of the number
-    of values belonging to that task.  per_task_devices contains
-    the specific devices to which the values are local, and
-    per_task_values contains the corresponding values.
-
-  Raises:
-    ValueError: devices must be same length as values.
-  """
-  num_devices = len(devices)
-  if num_devices != len(values):
-    raise ValueError("len(devices) must equal len(values)")
-  per_task_devices = collections.OrderedDict()
-  per_task_values = collections.OrderedDict()
-  for d in range(num_devices):
-    d_spec = device_lib.DeviceSpec.from_string(devices[d])
-    if not hasattr(d_spec, "task") or d_spec.task is None:
-      assert False, "failed to parse device %s" % devices[d]
-    index = (d_spec.job or "localhost", d_spec.replica or 0, d_spec.task)
-    if index not in per_task_devices:
-      per_task_devices[index] = []
-      per_task_values[index] = []
-    per_task_devices[index].append(devices[d])
-    per_task_values[index].append(values[d])
-
-  return (list(per_task_devices.values()), list(per_task_values.values()))
-
-
-def build_nccl_all_reduce(input_tensors, red_op, un_op=None):
-  """Build a subgraph that does one full all-reduce, using NCCL.
-
-  Args:
-    input_tensors: list of T `tf.Tensor` of same-shape and type values to
-      be reduced.
-    red_op: binary elementwise reduction operator.  Must be one of
-      {tf.add}
-    un_op: optional unary elementwise Op to apply to fully-reduce values.
-
-  Returns:
-    list of T `tf.Tensor` of reduced values.
-
-  Raises:
-    ValueError: red_op not supported.
-  """
-  if red_op == math_ops.add:
-    output_tensors = nccl_ops.all_sum(input_tensors)
-  else:
-    raise ValueError("red_op not supported by NCCL all-reduce: ", red_op)
-  if un_op:
-    un_op_wrapped = []
-    for t in output_tensors:
-      with ops.colocate_with(t):
-        un_op_wrapped.append(un_op(t))
-    output_tensors = un_op_wrapped
-  return output_tensors
-
-
-def _build_nccl_hybrid(input_tensors, red_op, upper_level_f):
-  """Construct a subgraph for NCCL hybrid all-reduce.
-
-  Args:
-    input_tensors: list of T `tf.Tensor` of same-shape and type values to
-      be reduced.
-    red_op: binary elementwise reduction operator.
-    upper_level_f: function for reducing one value per worker, across
-      workers.
-
-  Returns:
-    list of T `tf.Tensor` of reduced values.
-
-  Raises:
-    ValueError: inputs not well-formed.
-  """
-  input_tensors, shape = _flatten_tensors(input_tensors)
-  devices = [t.device for t in input_tensors]
-  per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors)
-  num_workers = len(per_worker_devices)
-  up_values = [None for w in range(0, num_workers)]
-  up_devices = up_values[:]
-  down_values = up_values[:]
-  # First stage: reduce within each worker using NCCL
-  for w in range(0, num_workers):
-    worker_values = build_nccl_all_reduce(per_worker_values[w], red_op)
-    # NOTE: these reductions will not run to completion unless
-    # every output value is used.  Since we only need one, we
-    # need to put control dependencies on the rest.
-    with ops.control_dependencies(worker_values):
-      with ops.device(worker_values[0].device):
-        up_values[w] = array_ops.identity(worker_values[0])
-      up_devices[w] = per_worker_devices[w][0]
-  # Second stage: Apply upper_level_f to reduce across first device at
-  # each worker
-  level_2_output = upper_level_f(up_values)
-  # Third stage: propagate within each worker using NCCL Broadcast
-  for w in range(0, num_workers):
-    dst_tensors = []
-    with ops.device(per_worker_devices[w][0]):
-      broadcast_src = nccl_ops.broadcast(array_ops.identity(level_2_output[w]))
-    for d in per_worker_devices[w]:
-      with ops.device(d):
-        dst_tensors.append(array_ops.identity(broadcast_src))
-    down_values[w] = dst_tensors
-  output_tensors = [v for sublist in down_values for v in sublist]
-  if len(shape) != 1:
-    output_tensors = _reshape_tensors(output_tensors, shape)
-  return output_tensors
-
-
-def _reduce_non_singleton(input_tensors, red_f, un_op):
-  """If input_tensors has more than one element apply red_f, else apply un_op."""
-  if len(input_tensors) > 1:
-    return red_f(input_tensors)
-  else:
-    if not un_op:
-      return input_tensors
-    output_tensors = []
-    for t in input_tensors:
-      with ops.colocate_with(t):
-        output_tensors.append(un_op(t))
-    return output_tensors
-
-
-def build_nccl_then_ring(input_tensors, subdiv, red_op, un_op=None):
-  """Construct hybrid of NCCL within workers, Ring across workers."""
-  def upper_builder(y):
-    return build_ring_all_reduce(y, len(y), subdiv, [0], red_op, un_op)
-  def upper_level_f(x):
-    return _reduce_non_singleton(x, upper_builder, un_op)
-  return _build_nccl_hybrid(input_tensors, red_op, upper_level_f)
-
-
-def build_nccl_then_recursive_hd(input_tensors, red_op, un_op=None):
-  """Construct hybrid of NCCL within workers, Recursive-HD across workers."""
-  upper_level_f = lambda x: build_recursive_hd_all_reduce(x, red_op, un_op)
-  return _build_nccl_hybrid(input_tensors, red_op, upper_level_f)
-
-
-def build_nccl_then_shuffle(input_tensors, gather_devices, nccl_red_op,
-                            shuffle_red_op, un_op=None):
-  """Construct hybrid of NCCL within workers, Shuffle across workers."""
-  upper_level_f = lambda x: build_shuffle_all_reduce(x, gather_devices,
-                                                     shuffle_red_op, un_op)
-  return _build_nccl_hybrid(input_tensors, nccl_red_op, upper_level_f)
-
-
-def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f):
-  """Construct a subgraph for Shuffle hybrid all-reduce.
-
-  Args:
-    input_tensors: list of T `tf.Tensor` of same-shape and type values to
-      be reduced.
-    gather_devices: list of device names on which to host gather shards.
-    red_op: binary elementwise reduction operator.
-    upper_level_f: function for reducing one value per worker, across
-      workers.
-
-  Returns:
-    list of T `tf.Tensor` of reduced values.
-
-  Raises:
-    ValueError: inputs not well-formed.
-  """
-  input_tensors, shape = _flatten_tensors(input_tensors)
-  # First stage, reduce across each worker using gather_devices.
-  devices = [t.device for t in input_tensors]
-  per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors)
-  num_workers = len(per_worker_devices)
-  up_values = []
-  if len(gather_devices) != num_workers:
-    raise ValueError("For shuffle hybrid, gather_devices must contain one "
-                     "device per worker. ")
-  for w in range(0, num_workers):
-    reduced_shards = _build_shuffle_gather(
-        per_worker_values[w], [gather_devices[w]], red_op)
-    up_values.append(reduced_shards[0])
-  # Second stage, apply upper_level_f.
-  level_2_output = upper_level_f(up_values)
-  # Third stage, apply shuffle scatter at each worker.
-  output_tensors = []
-  for w in range(0, num_workers):
-    output_tensors += _build_shuffle_scatter(
-        [level_2_output[w]], per_worker_devices[w])
-  if len(shape) != 1:
-    output_tensors = _reshape_tensors(output_tensors, shape)
-  return output_tensors
-
-
-def build_shuffle_then_ring(input_tensors, gather_devices, subdiv,
-                            red_n_op, red_op, un_op=None):
-  """Construct hybrid of Shuffle within workers, Ring across workers."""
-  def upper_builder(tensors):
-    return build_ring_all_reduce(tensors, len(tensors), subdiv, [0],
-                                 red_op, un_op)
-  def upper_level_f(tensors):
-    return _reduce_non_singleton(tensors, upper_builder, un_op)
-  return _build_shuffle_hybrid(
-      input_tensors, gather_devices, red_n_op, upper_level_f)
-
-
-def build_shuffle_then_shuffle(input_tensors, first_gather_devices,
-                               second_gather_devices, red_op, un_op=None):
-  """Construct hybrid of Shuffle within workers, Shuffle across workers."""
-  def upper_builder(tensors):
-    return build_shuffle_all_reduce(tensors, second_gather_devices,
-                                    red_op, un_op)
-  def upper_level_f(tensors):
-    return _reduce_non_singleton(tensors, upper_builder, un_op)
-  return _build_shuffle_hybrid(
-      input_tensors, first_gather_devices, red_op, upper_level_f)
+# pylint: disable=unused-import,wildcard-import
+from tensorflow.python.distribute.all_reduce import *
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4094e52169..ce81b1b38b 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -623,7 +623,6 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":values",
-        "//tensorflow/contrib/all_reduce:all_reduce_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:collective_ops",
         "//tensorflow/python:device",
@@ -632,6 +631,7 @@ py_library(
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nccl_ops",
+        "//tensorflow/python/distribute:all_reduce",
     ],
 )
 
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/contrib/distribute/python/cross_tower_utils.py
index 50b3cf31e5..d408f77cc9 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_utils.py
@@ -21,8 +21,8 @@ from __future__ import print_function
 import collections as pycoll
 import threading
 
-from tensorflow.contrib.all_reduce.python import all_reduce
 from tensorflow.contrib.distribute.python import values as value_lib
+from tensorflow.python.distribute import all_reduce
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 1c3d8ea67e..216dcc8587 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -7,6 +7,41 @@ licenses(["notice"])  # Apache 2.0
 exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+
+py_library(
+    name = "all_reduce",
+    srcs = [
+        "all_reduce.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nccl_ops",
+    ],
+)
+
+tf_py_test(
+    name = "all_reduce_test",
+    srcs = ["all_reduce_test.py"],
+    additional_deps = [
+        ":all_reduce",
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:state_ops",
+    ],
+)
 
 py_library(
     name = "distribute",
diff --git a/tensorflow/python/distribute/all_reduce.py b/tensorflow/python/distribute/all_reduce.py
new file mode 100644
index 0000000000..bd7c45ae27
--- /dev/null
+++ b/tensorflow/python/distribute/all_reduce.py
@@ -0,0 +1,860 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities to construct a TF subgraph implementing distributed All-Reduce."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import math
+
+from tensorflow.python.framework import device as device_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nccl_ops
+
+
+def _flatten_tensors(tensors):
+  """Check tensors for isomorphism and flatten.
+
+  Args:
+    tensors: list of T `tf.Tensor` which must all have the same shape.
+
+  Returns:
+    tensors: a list of T `tf.Tensor` which are flattened (1D) views of tensors
+    shape: the original shape of each element of input tensors
+
+  Raises:
+    ValueError: tensors are empty or non-isomorphic or have unknown shape.
+  """
+  if not tensors:
+    raise ValueError("tensors cannot be empty")
+  shape = tensors[0].shape
+  for tensor in tensors:
+    shape = shape.merge_with(tensor.shape)
+  if not shape.is_fully_defined():
+    raise ValueError("Tensors must have statically known shape.")
+  if len(shape) != 1:
+    reshaped = []
+    for t in tensors:
+      with ops.colocate_with(t):
+        reshaped.append(array_ops.reshape(t, [-1]))
+    tensors = reshaped
+  return tensors, shape
+
+
+def _reshape_tensors(tensors, shape):
+  """Reshape tensors flattened by _flatten_tensors.
+
+  Args:
+    tensors: list of T `tf.Tensor` of identical length 1D tensors.
+    shape: list of integers describing the desired shape.  Product of
+      the elements must equal the length of each tensor.
+
+  Returns:
+    list of T `tf.Tensor` which are the reshaped inputs.
+  """
+  reshaped = []
+  for t in tensors:
+    with ops.colocate_with(t):
+      reshaped.append(array_ops.reshape(t, shape))
+  return reshaped
+
+
+def _padded_split(tensor, pieces):
+  """Like split for 1D tensors but pads-out case where len % pieces != 0.
+
+  Args:
+    tensor: T `tf.Tensor` that must be 1D.
+    pieces: a positive integer specifying the number of pieces into which
+      tensor should be split.
+
+  Returns:
+    list of T `tf.Tensor` of length pieces, which hold the values of
+      thin input tensor, in order.  The final tensor may
+      be zero-padded on the end to make its size equal to those of all
+      of the other tensors.
+
+  Raises:
+    ValueError: The input tensor is not 1D.
+  """
+  shape = tensor.shape
+  if 1 != len(shape):
+    raise ValueError("input tensor must be 1D")
+  tensor_len = shape.dims[0].value
+  with ops.colocate_with(tensor):
+    if tensor_len % pieces != 0:
+      # pad to an even length
+      chunk_size = 1 + tensor_len // pieces
+      if pieces > tensor_len:
+        # This is an edge case that should not come up in practice,
+        # i.e. a different reduction algorithm would be better,
+        # but we'll make it work just for completeness.
+        pad_len = pieces - tensor_len
+        extended_whole = array_ops.concat(
+            [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0)
+        parts = array_ops.split(extended_whole, pieces)
+        return parts, pad_len
+      elif (pieces - 1) * chunk_size >= tensor_len:
+        # Another edge case of limited real interest.
+        pad_len = (pieces * chunk_size) % tensor_len
+        extended_whole = array_ops.concat(
+            [tensor, array_ops.zeros([pad_len], dtype=tensor.dtype)], 0)
+        parts = array_ops.split(extended_whole, pieces)
+        return parts, pad_len
+      else:
+        last_chunk_size = tensor_len - (pieces - 1) * chunk_size
+        pad_len = chunk_size - last_chunk_size
+        piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size]
+        parts = array_ops.split(tensor, piece_lens)
+        parts[-1] = array_ops.concat(
+            [parts[-1], array_ops.zeros([pad_len], dtype=tensor.dtype)], 0)
+        return parts, pad_len
+    else:
+      return array_ops.split(tensor, pieces), 0
+
+
+def _strip_padding(tensors, pad_len):
+  """Strip the suffix padding added by _padded_split.
+
+  Args:
+    tensors: list of T `tf.Tensor` of identical length 1D tensors.
+    pad_len: number of elements to be stripped from the end of each tensor.
+
+  Returns:
+    list of T `tf.Tensor` which are the stripped inputs.
+
+  Raises:
+    ValueError: tensors must be a non-empty list of 1D tensors, and
+      each must be longer than pad_len.
+  """
+  if not tensors:
+    raise ValueError("tensors cannot be empty")
+  shape = tensors[0].shape
+  if len(shape) > 1:
+    raise ValueError("tensors must be 1D")
+  prefix_len = int(shape[0] - pad_len)
+  if prefix_len < 0:
+    raise ValueError("pad_len longer than tensor")
+  stripped = []
+  for t in tensors:
+    with ops.colocate_with(t):
+      stripped.append(array_ops.slice(t, [0], [prefix_len]))
+  return stripped
+
+
+def _ragged_split(tensor, pieces):
+  """Like split for 1D tensors but allows case where len % pieces != 0.
+
+  Args:
+    tensor: T `tf.Tensor` that must be 1D.
+    pieces: a positive integer specifying the number of pieces into which
+      tensor should be split.
+
+  Returns:
+    list of T `tf.Tensor` of length pieces, which hold the values of
+      the input tensor, in order.  The final tensor may be shorter
+      than the others, which will all be of equal length.
+
+  Raises:
+    ValueError: input tensor must be 1D.
+  """
+  shape = tensor.shape
+  if 1 != len(shape):
+    raise ValueError("input tensor must be 1D")
+  tensor_len = shape.dims[0].value
+  chunk_size = tensor_len // pieces
+  with ops.colocate_with(tensor):
+    if tensor_len != (pieces * chunk_size):
+      # last piece will be short
+      assert pieces > 1
+      last_chunk_size = tensor_len - ((pieces - 1) * chunk_size)
+      assert last_chunk_size > 0
+      piece_lens = [chunk_size for _ in range(pieces - 1)] + [last_chunk_size]
+      return array_ops.split(tensor, piece_lens)
+    else:
+      return array_ops.split(tensor, pieces)
+
+
+def _ring_permutations(num_workers, num_subchunks, gpu_perm):
+  """"Generate an array of device index arrays, one for each subchunk.
+
+  In the basic ring reduction algorithm there are size(T)/num_devices
+  data chunks and each device process one chunk per tick, i.e. sending
+  one chunk and receiving one chunk.  The idea of subchunking is that
+  each device processes num_subchunks smaller data regions per tick,
+  and the ring rank permutation is different for each subchunk index
+  so that a device is potentially sending to and receiving from
+  num_subchunks different other devices at each tick.  Where multiple
+  independent data channels exist between devices, this strategy
+  supplies a method of using them in parallel.
+
+  Args:
+    num_workers: number of worker tasks
+    num_subchunks: number of subchunks into which to divide each per-GPU chunk.
+    gpu_perm: an array of integers in [0, num_gpus-1] giving the default
+      ring order of GPUs at each worker.  Other permutations will be generated
+      by rotating this array and splicing together per-worker instances.
+
+  Raises:
+    ValueError: the number of subchunks may not exceed the number of GPUs.
+
+  Returns:
+    pred_by_s_d: list of lists that maps (by index) from (subchunk, dev) to
+        preceding device in the permutation for that subchunk.  The
+        device index of GPU i at worker j is i + (j * num_gpus).
+    rank_by_s_d: list of lists that maps (by index) from (subchunk, dev) to
+       local rank of device d in the permutation for that subchunk.
+  """
+  num_gpus = len(gpu_perm)
+  devices = num_workers * num_gpus
+  if devices == 0:
+    return [], []
+  if num_subchunks > num_gpus:
+    raise ValueError(
+        "num_subchunks %d must be <= num_gpus %d" % (num_subchunks, num_gpus))
+  rotation_interval = max(1, int(num_gpus / num_subchunks))
+  perms_by_s = []
+  for s in range(0, num_subchunks):
+    full_order = []
+    offset = s * rotation_interval
+    for w in range(0, num_workers):
+      default_order = [(w * num_gpus) + i for i in gpu_perm]
+      dev_order = default_order[offset:] + default_order[:offset]
+      full_order += dev_order
+    perms_by_s.append(full_order)
+  pred_by_s_d = [[-1 for d in range(0, devices)]
+                 for s in range(0, num_subchunks)]
+  rank_by_s_d = [[-1 for d in range(0, devices)]
+                 for s in range(0, num_subchunks)]
+  for s in range(0, num_subchunks):
+    for d in range(0, devices):
+      for t in range(0, devices):
+        if d == perms_by_s[s][t]:
+          rank_by_s_d[s][d] = t
+          pred_by_s_d[s][d] = perms_by_s[s][(t + devices - 1) % devices]
+          break
+  return (pred_by_s_d, rank_by_s_d)
+
+
+def build_ring_all_reduce(input_tensors, num_workers, num_subchunks,
+                          gpu_perm, red_op, un_op=None):
+  """Construct a subgraph performing a ring-style all-reduce of input_tensors.
+
+  Args:
+    input_tensors: a list of T `tf.Tensor` objects, which must all
+      have the same shape and type.
+    num_workers: number of worker tasks spanned by input_tensors.
+    num_subchunks: number of subchunks each device should process in one tick.
+    gpu_perm: a list of ints giving a ring-wise rank ordering of GPUs at
+      each worker.  All workers must have the same number of
+      GPUs with the same rank ordering.  If NVLINK is available, this should
+      be a ring order supported by NVLINK edges.
+    red_op: a binary operator for elementwise reduction.
+    un_op: an optional unary operator to apply to fully reduced values.
+
+  Raises:
+    ValueError: empty input_tensors or they don't all have same
+    size.
+
+  Returns:
+    a list of T `tf.Tensor` identical sum-reductions of input_tensors.
+  """
+  if len(input_tensors) < 2:
+    raise ValueError("input_tensors must be length 2 or longer")
+  input_tensors, shape = _flatten_tensors(input_tensors)
+  devices = [t.device for t in input_tensors]
+  (pred_by_s_d, rank_by_s_d) = _ring_permutations(
+      num_workers, num_subchunks, gpu_perm)
+  chunks_by_dev, pad_len = _build_ring_gather(
+      input_tensors, devices,
+      num_subchunks, pred_by_s_d, rank_by_s_d, red_op)
+  if un_op:
+    chunks_by_dev = _apply_unary_to_chunks(un_op, chunks_by_dev)
+  output_tensors = _build_ring_scatter(pred_by_s_d, rank_by_s_d,
+                                       chunks_by_dev)
+  if pad_len > 0:
+    output_tensors = _strip_padding(output_tensors, pad_len)
+  if len(shape) != 1:
+    output_tensors = _reshape_tensors(output_tensors, shape)
+  return output_tensors
+
+
+def _build_ring_gather(input_tensors, devices, num_subchunks,
+                       pred_by_s_d, rank_by_s_d, red_op):
+  """Construct a subgraph for the first (reduction) pass of ring all-reduce.
+
+  Args:
+    input_tensors: a list of T `tf.Tensor` 1D input tensors of same
+      shape and type.
+    devices: array of device name strings
+    num_subchunks: number of subchunks each device should process in one tick.
+    pred_by_s_d: as produced by _ring_permutations
+    rank_by_s_d: as produced by _ring_permutations
+    red_op: a binary operator for elementwise reduction
+
+  Raises:
+    ValueError: tensors must all be one dimensional.
+
+  Returns:
+    list of list of T `tf.Tensor` of (partially) reduced values where
+    exactly num_subchunks chunks at each device are fully reduced.
+  """
+  num_devices = len(input_tensors)
+  if num_devices == 0:
+    return []
+  if num_devices == 1:
+    return input_tensors
+  shape = input_tensors[0].shape
+  if 1 != len(shape):
+    raise ValueError("input tensors must be 1D")
+  num_chunks = num_devices * num_subchunks
+  num_ticks = num_devices - 1
+  # Initialize chunks_by_dev with splits of the input tensors.
+  chunks_by_dev = []
+  split_pad_len = 0
+  for d in range(0, num_devices):
+    with ops.device(devices[d]):
+      splits, split_pad_len = _padded_split(input_tensors[d], num_chunks)
+      chunks_by_dev.append(splits)
+  # Reduction phase
+  for tick in range(0, num_ticks):
+    # One new partial reduction for every chunk
+    new_partial_reductions = [None for _ in range(0, num_chunks)]
+    # Compute reductions with respect to last tick's values
+    for d in range(0, num_devices):
+      with ops.device(devices[d]):
+        for s in range(0, num_subchunks):
+          rank = rank_by_s_d[s][d]
+          seg_index = (rank + num_devices - (2 + tick)) % num_devices
+          pred_dev = pred_by_s_d[s][d]
+          chunk_index = (seg_index * num_subchunks) + s
+          new_partial_reductions[chunk_index] = red_op(
+              chunks_by_dev[pred_dev][chunk_index],
+              chunks_by_dev[d][chunk_index])
+    # Update chunks_by_dev with the new values at the end of the tick.
+    for d in range(0, num_devices):
+      for s in range(0, num_subchunks):
+        rank = rank_by_s_d[s][d]
+        seg_index = (rank + num_devices - (2 + tick)) % num_devices
+        chunk_index = (seg_index * num_subchunks) + s
+        chunks_by_dev[d][chunk_index] = new_partial_reductions[chunk_index]
+  return chunks_by_dev, split_pad_len
+
+
+def _apply_unary_to_chunks(f, chunks_by_dev):
+  """Apply a unary op to each tensor in chunks_by_dev, on same device.
+
+  Args:
+    f: a unary function over T `tf.Tensor`.
+    chunks_by_dev: list of lists of T `tf.Tensor`.
+
+  Returns:
+    new list of lists of T `tf.Tensor` with the same structure as
+    chunks_by_dev containing the derived tensors.
+  """
+  output = []
+  for x in chunks_by_dev:
+    with ops.colocate_with(x[0]):
+      output.append([f(t) for t in x])
+  return output
+
+
+def _build_ring_scatter(pred_by_s_d, rank_by_s_d,
+                        chunks_by_dev):
+  """Construct subgraph for second (scatter) pass of ring all-reduce.
+
+  Args:
+    pred_by_s_d: as produced by _ring_permutations
+    rank_by_s_d: as produced by _ring_permutations
+    chunks_by_dev: list of list of T `tf.Tensor` indexed by ints
+      (device, chunk)
+
+  Raises:
+    ValueError: chunks_by_dev is not well-formed
+
+  Returns:
+    list of T `tf.Tensor` which are the fully reduced tensors, one
+    at each device corresponding to the outer dimension of chunks_by_dev.
+  """
+  num_devices = len(chunks_by_dev)
+  num_chunks = len(chunks_by_dev[0])
+  if 0 != num_chunks % num_devices:
+    raise ValueError(
+        "Expect number of chunks per device to be divisible by num_devices")
+  num_subchunks = int(num_chunks / num_devices)
+  num_ticks = num_devices - 1
+  for tick in range(0, num_ticks):
+    passed_values = [None for _ in range(0, num_chunks)]
+    for d in range(0, num_devices):
+      with ops.colocate_with(chunks_by_dev[d][0]):
+        for s in range(0, num_subchunks):
+          rank = rank_by_s_d[s][d]
+          seg_index = (rank + num_devices - (1 + tick)) % num_devices
+          pred_dev = pred_by_s_d[s][d]
+          chunk_index = (seg_index * num_subchunks) + s
+          passed_values[chunk_index] = array_ops.identity(
+              chunks_by_dev[pred_dev][chunk_index])
+    for d in range(0, num_devices):
+      for s in range(0, num_subchunks):
+        rank = rank_by_s_d[s][d]
+        seg_index = (rank + num_devices - (1 + tick)) % num_devices
+        chunk_index = (seg_index * num_subchunks) + s
+        chunks_by_dev[d][chunk_index] = passed_values[chunk_index]
+  # Join chunks at each device.
+  output = []
+  for x in chunks_by_dev:
+    with ops.colocate_with(x[0]):
+      output.append(array_ops.concat(x, 0))
+  return output
+
+
+def build_recursive_hd_all_reduce(input_tensors, red_op, un_op=None):
+  """Construct a subgraph for recursive halving-doubling all-reduce.
+
+  The recursive halving-doubling algorithm is described in
+  http://www.mcs.anl.gov/~thakur/papers/ijhpca-coll.pdf
+
+  The concept is to arrange the participating n devices in
+  a linear sequence where devices exchange data pairwise
+  with one other device in each round.  During the gather
+  phase there are lg(n) rounds where devices exchange
+  increasingly smaller sub-tensors with another device
+  at increasingly greater distances, until at the top
+  each device has 1/n of the fully reduced values.  During the
+  scatter phase each device exchanges its fully reduced
+  sub-tensor (which doubles in length at each round)
+  with one other device at increasingly smaller distances
+  until each device has all of the fully reduced values.
+
+  Note: this preliminary version requires that len(input_tensors) be a
+    power of 2.  TODO(tucker): relax this restriction.  Also, the
+    number of elements in each tensor must be divisible by 2^h where h
+    is the number of hops in each phase.  This will also be relaxed in
+    the future with edge-case specific logic.
+
+  Args:
+    input_tensors: list of T `tf.Tensor` to be elementwise reduced.
+    red_op: a binary elementwise reduction Op.
+    un_op: an optional unary elementwise Op to apply to reduced values.
+
+  Returns:
+    list of T `tf.Tensor` which are the fully reduced tensors, one
+    at each device of input_tensors.
+
+  Raises:
+    ValueError: num_devices not a power of 2, or tensor len not divisible
+    by 2 the proper number of times.
+  """
+  devices = [t.device for t in input_tensors]
+  input_tensors, shape = _flatten_tensors(input_tensors)
+  reduced_shards = _build_recursive_hd_gather(input_tensors, devices, red_op)
+  if un_op:
+    reduced_shards = [un_op(t) for t in reduced_shards]
+  output_tensors = _build_recursive_hd_scatter(reduced_shards, devices)
+  if len(shape) != 1:
+    output_tensors = _reshape_tensors(output_tensors, shape)
+  return output_tensors
+
+
+def _build_recursive_hd_gather(input_tensors, devices, red_op):
+  """Construct the gather phase of recursive halving-doubling all-reduce.
+
+  Args:
+    input_tensors: list of T `tf.Tensor` to be elementwise reduced.
+    devices: a list of strings naming the devices hosting input_tensors,
+      which will also be used to host the (partial) reduction values.
+    red_op: a binary elementwise reduction Op.
+
+  Returns:
+    list of T `tf.Tensor` which are the fully reduced tensor shards.
+
+  Raises:
+    ValueError: num_devices not a power of 2, or tensor len not divisible
+    by 2 the proper number of times.
+  """
+  num_devices = len(devices)
+  num_hops = int(math.log(num_devices, 2))
+  if num_devices != (2 ** num_hops):
+    raise ValueError("num_devices must be a power of 2")
+  chunks = input_tensors
+  for h in range(0, num_hops):
+    span = 2 ** h
+    group_size = span * 2
+    new_chunks = [[] for _ in devices]
+    for d in range(0, num_devices):
+      if (d % group_size) >= (group_size / 2):
+        # skip right half of a pair
+        continue
+      left_dev = devices[d]
+      right_dev = devices[d + span]
+      left_split = array_ops.split(chunks[d], 2)
+      right_split = array_ops.split(chunks[d+span], 2)
+      with ops.device(left_dev):
+        new_chunks[d] = red_op(left_split[0], right_split[0])
+      with ops.device(right_dev):
+        new_chunks[d + span] = red_op(left_split[1], right_split[1])
+    chunks = new_chunks
+  return chunks
+
+
+def _build_recursive_hd_scatter(input_tensors, devices):
+  """Construct the scatter phase of recursive halving-doublng all-reduce.
+
+  Args:
+    input_tensors: list of T `tf.Tensor` that are fully-reduced shards.
+    devices: a list of strings naming the devices on which the reconstituted
+      full tensors should be placed.
+
+  Returns:
+    list of T `tf.Tensor` which are the fully reduced tensors.
+  """
+  num_devices = len(devices)
+  num_hops = int(math.log(num_devices, 2))
+  assert num_devices == (2 ** num_hops), "num_devices must be a power of 2"
+  chunks = input_tensors
+  for h in reversed(range(0, num_hops)):
+    span = 2 ** h
+    group_size = span * 2
+    new_chunks = [[] for _ in devices]
+    for d in range(0, num_devices):
+      if (d % group_size) >= (group_size / 2):
+        # skip right half of a pair
+        continue
+      left_idx = d
+      right_idx = d + span
+      left_dev = devices[left_idx]
+      right_dev = devices[right_idx]
+      with ops.device(left_dev):
+        new_chunks[left_idx] = array_ops.concat([chunks[left_idx],
+                                                 chunks[right_idx]], 0)
+      with ops.device(right_dev):
+        new_chunks[right_idx] = array_ops.concat([chunks[left_idx],
+                                                  chunks[right_idx]], 0)
+    chunks = new_chunks
+  return chunks
+
+
+def build_shuffle_all_reduce(input_tensors, gather_devices, red_op, un_op=None):
+  """Construct a subgraph for shuffle all-reduce.
+
+  Shuffle reduce is essentially the algorithm implemented when using
+  parameter servers.  Suppose tensor length is n, there are d devices
+  and g gather shards.  Each device sends a n/g length sub-tensor to
+  each gather shard.  The gather shards perform a reduction across d
+  fragments, then broadcast the result back to each device.  The
+  devices then join the g fully reduced fragments they receive from
+  the shards.  The gather shards could perform d-1 pairwise
+  reductions, or one d-way reduction.  The first is better where
+  reduction Op time is low compared to transmission time, the second
+  better in the other case.
+
+  Args:
+    input_tensors: list of T @(tf.Tensor} values to be reduced.
+    gather_devices: list of names of devices on which reduction shards
+      should be placed.
+    red_op: an n-array elementwise reduction Op
+    un_op: optional elementwise unary Op to be applied to fully-reduced values.
+
+  Returns:
+    list of T `tf.Tensor` which are the fully reduced tensors.
+  """
+  input_tensors, shape = _flatten_tensors(input_tensors)
+  dst_devices = [t.device for t in input_tensors]
+  reduced_shards = _build_shuffle_gather(input_tensors, gather_devices,
+                                         red_op, un_op)
+  output_tensors = _build_shuffle_scatter(reduced_shards, dst_devices)
+  if len(shape) != 1:
+    output_tensors = _reshape_tensors(output_tensors, shape)
+  return output_tensors
+
+
+def _build_shuffle_gather(input_tensors, gather_devices, red_op, un_op=None):
+  """Construct the gather (concentrate and reduce) phase of shuffle all-reduce.
+
+  Args:
+    input_tensors: list of T @(tf.Tensor} values to be reduced.
+    gather_devices: list of names of devices on which reduction shards
+      should be placed.
+    red_op: the binary reduction Op
+    un_op: optional elementwise unary Op to be applied to fully-reduced values.
+
+  Returns:
+    list of T `tf.Tensor` which are the fully reduced shards.
+
+  Raises:
+    ValueError: inputs not well-formed.
+  """
+  num_source_devices = len(input_tensors)
+  num_gather_devices = len(gather_devices)
+  shape = input_tensors[0].shape
+  if len(shape) != 1:
+    raise ValueError("input_tensors must be 1D")
+  shards_by_source = []
+  for d in range(0, num_source_devices):
+    with ops.colocate_with(input_tensors[d]):
+      shards_by_source.append(
+          _ragged_split(input_tensors[d], num_gather_devices))
+  reduced_shards = []
+  for d in range(0, num_gather_devices):
+    with ops.device(gather_devices[d]):
+      values = [s[d] for s in shards_by_source]
+      red_shard = red_op(values)
+      if un_op:
+        red_shard = un_op(red_shard)
+      reduced_shards.append(red_shard)
+  return reduced_shards
+
+
+def _build_shuffle_scatter(reduced_shards, dst_devices):
+  """Build the scatter phase of shuffle all-reduce.
+
+  Args:
+    reduced_shards:  list of T @(tf.Tensor} fully reduced shards
+    dst_devices: list of names of devices at which the fully-reduced value
+      should be reconstituted.
+
+  Returns:
+    list of T `tf.Tensor` scattered tensors.
+  """
+  num_devices = len(dst_devices)
+  out_tensors = []
+  for d in range(0, num_devices):
+    with ops.device(dst_devices[d]):
+      out_tensors.append(array_ops.concat(reduced_shards, 0))
+  return out_tensors
+
+
+def _split_by_task(devices, values):
+  """Partition devices and values by common task.
+
+  Args:
+    devices: list of device name strings
+    values: list of T `tf.tensor` of same length as devices.
+
+  Returns:
+    (per_task_devices, per_task_values) where both values are
+    lists of lists with isomorphic structure: the outer list is
+    indexed by task, and the inner list has length of the number
+    of values belonging to that task.  per_task_devices contains
+    the specific devices to which the values are local, and
+    per_task_values contains the corresponding values.
+
+  Raises:
+    ValueError: devices must be same length as values.
+  """
+  num_devices = len(devices)
+  if num_devices != len(values):
+    raise ValueError("len(devices) must equal len(values)")
+  per_task_devices = collections.OrderedDict()
+  per_task_values = collections.OrderedDict()
+  for d in range(num_devices):
+    d_spec = device_lib.DeviceSpec.from_string(devices[d])
+    if not hasattr(d_spec, "task") or d_spec.task is None:
+      assert False, "failed to parse device %s" % devices[d]
+    index = (d_spec.job or "localhost", d_spec.replica or 0, d_spec.task)
+    if index not in per_task_devices:
+      per_task_devices[index] = []
+      per_task_values[index] = []
+    per_task_devices[index].append(devices[d])
+    per_task_values[index].append(values[d])
+
+  return (list(per_task_devices.values()), list(per_task_values.values()))
+
+
+def build_nccl_all_reduce(input_tensors, red_op, un_op=None):
+  """Build a subgraph that does one full all-reduce, using NCCL.
+
+  Args:
+    input_tensors: list of T `tf.Tensor` of same-shape and type values to
+      be reduced.
+    red_op: binary elementwise reduction operator.  Must be one of
+      {tf.add}
+    un_op: optional unary elementwise Op to apply to fully-reduce values.
+
+  Returns:
+    list of T `tf.Tensor` of reduced values.
+
+  Raises:
+    ValueError: red_op not supported.
+  """
+  if red_op == math_ops.add:
+    output_tensors = nccl_ops.all_sum(input_tensors)
+  else:
+    raise ValueError("red_op not supported by NCCL all-reduce: ", red_op)
+  if un_op:
+    un_op_wrapped = []
+    for t in output_tensors:
+      with ops.colocate_with(t):
+        un_op_wrapped.append(un_op(t))
+    output_tensors = un_op_wrapped
+  return output_tensors
+
+
+def _build_nccl_hybrid(input_tensors, red_op, upper_level_f):
+  """Construct a subgraph for NCCL hybrid all-reduce.
+
+  Args:
+    input_tensors: list of T `tf.Tensor` of same-shape and type values to
+      be reduced.
+    red_op: binary elementwise reduction operator.
+    upper_level_f: function for reducing one value per worker, across
+      workers.
+
+  Returns:
+    list of T `tf.Tensor` of reduced values.
+
+  Raises:
+    ValueError: inputs not well-formed.
+  """
+  input_tensors, shape = _flatten_tensors(input_tensors)
+  devices = [t.device for t in input_tensors]
+  per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors)
+  num_workers = len(per_worker_devices)
+  up_values = [None for w in range(0, num_workers)]
+  up_devices = up_values[:]
+  down_values = up_values[:]
+  # First stage: reduce within each worker using NCCL
+  for w in range(0, num_workers):
+    worker_values = build_nccl_all_reduce(per_worker_values[w], red_op)
+    # NOTE: these reductions will not run to completion unless
+    # every output value is used.  Since we only need one, we
+    # need to put control dependencies on the rest.
+    with ops.control_dependencies(worker_values):
+      with ops.device(worker_values[0].device):
+        up_values[w] = array_ops.identity(worker_values[0])
+      up_devices[w] = per_worker_devices[w][0]
+  # Second stage: Apply upper_level_f to reduce across first device at
+  # each worker
+  level_2_output = upper_level_f(up_values)
+  # Third stage: propagate within each worker using NCCL Broadcast
+  for w in range(0, num_workers):
+    dst_tensors = []
+    with ops.device(per_worker_devices[w][0]):
+      broadcast_src = nccl_ops.broadcast(array_ops.identity(level_2_output[w]))
+    for d in per_worker_devices[w]:
+      with ops.device(d):
+        dst_tensors.append(array_ops.identity(broadcast_src))
+    down_values[w] = dst_tensors
+  output_tensors = [v for sublist in down_values for v in sublist]
+  if len(shape) != 1:
+    output_tensors = _reshape_tensors(output_tensors, shape)
+  return output_tensors
+
+
+def _reduce_non_singleton(input_tensors, red_f, un_op):
+  """If len(input_tensors) > 1, apply red_f, else apply un_op."""
+  if len(input_tensors) > 1:
+    return red_f(input_tensors)
+  else:
+    if not un_op:
+      return input_tensors
+    output_tensors = []
+    for t in input_tensors:
+      with ops.colocate_with(t):
+        output_tensors.append(un_op(t))
+    return output_tensors
+
+
+def build_nccl_then_ring(input_tensors, subdiv, red_op, un_op=None):
+  """Construct hybrid of NCCL within workers, Ring across workers."""
+  def upper_builder(y):
+    return build_ring_all_reduce(y, len(y), subdiv, [0], red_op, un_op)
+  def upper_level_f(x):
+    return _reduce_non_singleton(x, upper_builder, un_op)
+  return _build_nccl_hybrid(input_tensors, red_op, upper_level_f)
+
+
+def build_nccl_then_recursive_hd(input_tensors, red_op, un_op=None):
+  """Construct hybrid of NCCL within workers, Recursive-HD across workers."""
+  upper_level_f = lambda x: build_recursive_hd_all_reduce(x, red_op, un_op)
+  return _build_nccl_hybrid(input_tensors, red_op, upper_level_f)
+
+
+def build_nccl_then_shuffle(input_tensors, gather_devices, nccl_red_op,
+                            shuffle_red_op, un_op=None):
+  """Construct hybrid of NCCL within workers, Shuffle across workers."""
+  def upper_level_f(x):
+    return build_shuffle_all_reduce(x, gather_devices, shuffle_red_op, un_op)
+
+  return _build_nccl_hybrid(input_tensors, nccl_red_op, upper_level_f)
+
+
+def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f):
+  """Construct a subgraph for Shuffle hybrid all-reduce.
+
+  Args:
+    input_tensors: list of T `tf.Tensor` of same-shape and type values to
+      be reduced.
+    gather_devices: list of device names on which to host gather shards.
+    red_op: binary elementwise reduction operator.
+    upper_level_f: function for reducing one value per worker, across
+      workers.
+
+  Returns:
+    list of T `tf.Tensor` of reduced values.
+
+  Raises:
+    ValueError: inputs not well-formed.
+  """
+  input_tensors, shape = _flatten_tensors(input_tensors)
+  # First stage, reduce across each worker using gather_devices.
+  devices = [t.device for t in input_tensors]
+  per_worker_devices, per_worker_values = _split_by_task(devices, input_tensors)
+  num_workers = len(per_worker_devices)
+  up_values = []
+  if len(gather_devices) != num_workers:
+    raise ValueError("For shuffle hybrid, gather_devices must contain one "
+                     "device per worker. ")
+  for w in range(0, num_workers):
+    reduced_shards = _build_shuffle_gather(
+        per_worker_values[w], [gather_devices[w]], red_op)
+    up_values.append(reduced_shards[0])
+  # Second stage, apply upper_level_f.
+  level_2_output = upper_level_f(up_values)
+  # Third stage, apply shuffle scatter at each worker.
+  output_tensors = []
+  for w in range(0, num_workers):
+    output_tensors += _build_shuffle_scatter(
+        [level_2_output[w]], per_worker_devices[w])
+  if len(shape) != 1:
+    output_tensors = _reshape_tensors(output_tensors, shape)
+  return output_tensors
+
+
+def build_shuffle_then_ring(input_tensors, gather_devices, subdiv,
+                            red_n_op, red_op, un_op=None):
+  """Construct hybrid of Shuffle within workers, Ring across workers."""
+  def upper_builder(tensors):
+    return build_ring_all_reduce(tensors, len(tensors), subdiv, [0],
+                                 red_op, un_op)
+  def upper_level_f(tensors):
+    return _reduce_non_singleton(tensors, upper_builder, un_op)
+  return _build_shuffle_hybrid(
+      input_tensors, gather_devices, red_n_op, upper_level_f)
+
+
+def build_shuffle_then_shuffle(input_tensors, first_gather_devices,
+                               second_gather_devices, red_op, un_op=None):
+  """Construct hybrid of Shuffle within workers, Shuffle across workers."""
+  def upper_builder(tensors):
+    return build_shuffle_all_reduce(tensors, second_gather_devices,
+                                    red_op, un_op)
+  def upper_level_f(tensors):
+    return _reduce_non_singleton(tensors, upper_builder, un_op)
+  return _build_shuffle_hybrid(
+      input_tensors, first_gather_devices, red_op, upper_level_f)
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce_test.py b/tensorflow/python/distribute/all_reduce_test.py
similarity index 98%
rename from tensorflow/contrib/all_reduce/python/all_reduce_test.py
rename to tensorflow/python/distribute/all_reduce_test.py
index 304fd7fb8a..b1956c75b7 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce_test.py
+++ b/tensorflow/python/distribute/all_reduce_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for tensorflow.contrib.all_reduce.python..all_reduce."""
+"""Tests for all_reduce."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -22,8 +22,8 @@ import time
 
 import numpy as np
 
-from tensorflow.contrib.all_reduce.python import all_reduce as ar
 from tensorflow.core.framework import types_pb2
+from tensorflow.python.distribute import all_reduce as ar
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-- 
GitLab


From 9a6587e229e87d4388b04e85df54ae68b72b288f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 13 Nov 2018 11:26:34 -0800
Subject: [PATCH 0165/1554] Remove dependency on conv_ops in transpose_functor.
 It only needs to depend on conv_2d.

PiperOrigin-RevId: 221301703
---
 .../core/grappler/optimizers/data/BUILD       |   1 +
 tensorflow/core/kernels/BUILD                 |  18 ++-
 ...conv_ops_gpu_3.cu.cc => conv_2d_gpu.cu.cc} | 119 +++++++++---------
 3 files changed, 75 insertions(+), 63 deletions(-)
 rename tensorflow/core/kernels/{conv_ops_gpu_3.cu.cc => conv_2d_gpu.cu.cc} (91%)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 89e95067b8..7593023ff4 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -628,6 +628,7 @@ tf_cc_test(
         "//tensorflow/core/kernels:cwise_op",
         "//tensorflow/core/kernels:logging_ops",
         "//tensorflow/core/kernels:math",
+        "//tensorflow/core/kernels:nn",
         "//tensorflow/core/kernels:parsing",
         "//tensorflow/tools/graph_transforms:transform_utils",
     ] + tf_protos_all(),
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 18d70422a0..0f843ef125 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -195,15 +195,27 @@ cc_library(
     deps = ["//third_party/eigen3"],
 )
 
-cc_library(
+tf_kernel_library(
     name = "conv_2d",
     hdrs = ["conv_2d.h"],
+    gpu_srcs = [
+        "conv_2d_gpu.cu.cc",
+        "conv_2d.h",
+    ],
     deps = [
         ":eigen_helpers",
+        ":fill_functor",
         ":gpu_util_hdrs",
+        ":image_resizer_state",
+        ":ops_util",
+        "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
+    alwayslink = 1,
 )
 
 cc_library(
@@ -1673,14 +1685,14 @@ tf_kernel_library(
     ],
     visibility = [":friends"],
     deps = [
-        ":conv_ops",
+        ":conv_2d",
         ":ops_util",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//third_party/eigen3",
     ],
-    alwayslink = 0,
+    alwayslink = 1,
 )
 
 tf_cc_test(
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_2d_gpu.cu.cc
similarity index 91%
rename from tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
rename to tensorflow/core/kernels/conv_2d_gpu.cu.cc
index 46167db3a2..8d2700dca1 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_2d_gpu.cu.cc
@@ -433,7 +433,7 @@ struct TransformFilter<GPUDevice, T, int, NDIMS> {
     combined_dims[2] = in.dimension(NDIMS - 1);  // output filters
     CudaLaunchConfig config = GetCudaLaunchConfig(out.size(), d);
 
-    CHECK(dst_filter_format == FORMAT_OIHW)
+    DCHECK(dst_filter_format == FORMAT_OIHW)
         << "Unsupported output layout: " << ToString(dst_filter_format);
 
     ShuffleInTensor3Simple<T, 2, 1, 0>
@@ -998,78 +998,77 @@ struct NCHWToNHWC<GPUDevice, T, NDIMS> {
   }
 };
 
-}  // namespace functor
-
-template struct functor::ShuffleAndReverse<GPUDevice, float, 4, int>;
-template struct functor::ShuffleAndReverse<GPUDevice, Eigen::half, 4, int>;
-
-template struct functor::ShuffleAndReverse<GPUDevice, float, 4,
-                                           Eigen::DenseIndex>;
-template struct functor::ShuffleAndReverse<GPUDevice, Eigen::half, 4,
-                                           Eigen::DenseIndex>;
-
-template struct functor::TransformDepth<GPUDevice, float, int>;
-template struct functor::TransformDepth<GPUDevice, Eigen::half, int>;
-
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint8>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint16>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint32>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint64>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, float4>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, float2,
-                                                     /*conjugate=*/true>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, double2,
-                                                     /*conjugate=*/true>;
-template struct functor::SwapDimension1And2InTensor3<GPUDevice, Eigen::half>;
-
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint8>;
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint16>;
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint32>;
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint64>;
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, float4>;
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, float2,
-                                                     /*conjugate=*/true>;
-template struct functor::SwapDimension0And2InTensor3<GPUDevice, double2,
-                                                     /*conjugate=*/true>;
+template struct ShuffleAndReverse<Eigen::GpuDevice, float, 4, int>;
+template struct ShuffleAndReverse<Eigen::GpuDevice, Eigen::half, 4, int>;
+
+template struct ShuffleAndReverse<Eigen::GpuDevice, float, 4,
+                                  Eigen::DenseIndex>;
+template struct ShuffleAndReverse<Eigen::GpuDevice, Eigen::half, 4,
+                                  Eigen::DenseIndex>;
+
+template struct TransformDepth<Eigen::GpuDevice, float, int>;
+template struct TransformDepth<Eigen::GpuDevice, Eigen::half, int>;
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint8>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint16>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint32>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint64>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, float4>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, float2,
+                                            /*conjugate=*/true>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, double2,
+                                            /*conjugate=*/true>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, Eigen::half>;
+
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint8>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint16>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint32>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint64>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, float4>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, float2,
+                                            /*conjugate=*/true>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, double2,
+                                            /*conjugate=*/true>;
 
 // For 2d ops.
-template struct functor::TransformFilter<GPUDevice, double, int, 4>;
-template struct functor::TransformFilter<GPUDevice, float, int, 4>;
-template struct functor::TransformFilter<GPUDevice, Eigen::half, int, 4>;
+template struct TransformFilter<Eigen::GpuDevice, double, int, 4>;
+template struct TransformFilter<Eigen::GpuDevice, float, int, 4>;
+template struct TransformFilter<Eigen::GpuDevice, Eigen::half, int, 4>;
 
-template struct functor::ReverseTransformFilter<GPUDevice, double, 4>;
-template struct functor::ReverseTransformFilter<GPUDevice, float, 4>;
-template struct functor::ReverseTransformFilter<GPUDevice, Eigen::half, 4>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, double, 4>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, float, 4>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, Eigen::half, 4>;
 
-template struct functor::NHWCToNCHW<GPUDevice, double, 4>;
-template struct functor::NHWCToNCHW<GPUDevice, float, 4>;
-template struct functor::NHWCToNCHW<GPUDevice, Eigen::half, 4>;
+template struct NHWCToNCHW<Eigen::GpuDevice, double, 4>;
+template struct NHWCToNCHW<Eigen::GpuDevice, float, 4>;
+template struct NHWCToNCHW<Eigen::GpuDevice, Eigen::half, 4>;
 
-template struct functor::NCHWToNHWC<GPUDevice, double, 4>;
-template struct functor::NCHWToNHWC<GPUDevice, float, 4>;
-template struct functor::NCHWToNHWC<GPUDevice, Eigen::half, 4>;
+template struct NCHWToNHWC<Eigen::GpuDevice, double, 4>;
+template struct NCHWToNHWC<Eigen::GpuDevice, float, 4>;
+template struct NCHWToNHWC<Eigen::GpuDevice, Eigen::half, 4>;
 
-template struct functor::PadInput<GPUDevice, int, int, 4>;
-template struct functor::PadInput<GPUDevice, double, int, 4>;
-template struct functor::PadInput<GPUDevice, float, int, 4>;
-template struct functor::PadInput<GPUDevice, Eigen::half, int, 4>;
+template struct PadInput<Eigen::GpuDevice, int, int, 4>;
+template struct PadInput<Eigen::GpuDevice, double, int, 4>;
+template struct PadInput<Eigen::GpuDevice, float, int, 4>;
+template struct PadInput<Eigen::GpuDevice, Eigen::half, int, 4>;
 
 // For 3d ops.
-template struct functor::TransformFilter<GPUDevice, float, int, 5>;
-template struct functor::TransformFilter<GPUDevice, Eigen::half, int, 5>;
+template struct TransformFilter<Eigen::GpuDevice, float, int, 5>;
+template struct TransformFilter<Eigen::GpuDevice, Eigen::half, int, 5>;
 
-template struct functor::ReverseTransformFilter<GPUDevice, float, 5>;
-template struct functor::ReverseTransformFilter<GPUDevice, Eigen::half, 5>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, float, 5>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, Eigen::half, 5>;
 
-template struct functor::NHWCToNCHW<GPUDevice, float, 5>;
-template struct functor::NHWCToNCHW<GPUDevice, Eigen::half, 5>;
+template struct NHWCToNCHW<Eigen::GpuDevice, float, 5>;
+template struct NHWCToNCHW<Eigen::GpuDevice, Eigen::half, 5>;
 
-template struct functor::NCHWToNHWC<GPUDevice, float, 5>;
-template struct functor::NCHWToNHWC<GPUDevice, Eigen::half, 5>;
+template struct NCHWToNHWC<Eigen::GpuDevice, float, 5>;
+template struct NCHWToNHWC<Eigen::GpuDevice, Eigen::half, 5>;
 
-template struct functor::PadInput<GPUDevice, float, int, 5>;
-template struct functor::PadInput<GPUDevice, Eigen::half, int, 5>;
+template struct PadInput<Eigen::GpuDevice, float, int, 5>;
+template struct PadInput<Eigen::GpuDevice, Eigen::half, int, 5>;
 
+}  // namespace functor
 }  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
-- 
GitLab


From f17806effec7ebb3618ac1426317baea1aac34e5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 11:31:51 -0800
Subject: [PATCH 0166/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 221302763
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 51 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 51 +++++++++++++++++++
 2 files changed, 102 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 6f5034ced5..17ced64765 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -76917,6 +76917,57 @@ op {
     type: "type"
   }
 }
+op {
+  name: "UnicodeDecodeWithOffsets"
+  input_arg {
+    name: "input"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "row_splits"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "char_values"
+    type: DT_UINT32
+  }
+  output_arg {
+    name: "char_to_byte_starts"
+    type: DT_INT64
+  }
+  attr {
+    name: "input_encoding"
+    type: "string"
+  }
+  attr {
+    name: "errors"
+    type: "string"
+    default_value {
+      s: "replace"
+    }
+    allowed_values {
+      list {
+        s: "strict"
+        s: "replace"
+        s: "ignore"
+      }
+    }
+  }
+  attr {
+    name: "replacement_char"
+    type: "int"
+    default_value {
+      i: 65533
+    }
+  }
+  attr {
+    name: "replace_control_characters"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "UnicodeScript"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 915cd26db1..da5d4c152c 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36929,6 +36929,57 @@ op {
     type: "type"
   }
 }
+op {
+  name: "UnicodeDecodeWithOffsets"
+  input_arg {
+    name: "input"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "row_splits"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "char_values"
+    type: DT_UINT32
+  }
+  output_arg {
+    name: "char_to_byte_starts"
+    type: DT_INT64
+  }
+  attr {
+    name: "input_encoding"
+    type: "string"
+  }
+  attr {
+    name: "errors"
+    type: "string"
+    default_value {
+      s: "replace"
+    }
+    allowed_values {
+      list {
+        s: "strict"
+        s: "replace"
+        s: "ignore"
+      }
+    }
+  }
+  attr {
+    name: "replacement_char"
+    type: "int"
+    default_value {
+      i: 65533
+    }
+  }
+  attr {
+    name: "replace_control_characters"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "UnicodeScript"
   input_arg {
-- 
GitLab


From 0483da87c2b5f99061ebbd8059003a89ff1818e1 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 13 Nov 2018 11:34:55 -0800
Subject: [PATCH 0167/1554] [TF:XLA] Bump open source llvm revision to r346485

PiperOrigin-RevId: 221303452
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 6d3562caef..29559927bb 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -473,11 +473,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "a22a9b4c3af50a52ba0015b6987bba7202c3ec8e1d40ae76ee7d7643638936ae",
-        strip_prefix = "llvm-b4ace5f3454131a3070ef7c11e19e42fc9a80b4e",
+        sha256 = "016e265dc5978f424f23a7893044bb41c68cd58a1c993b1fac99187d92176ae4",
+        strip_prefix = "llvm-5e1b99cf3b50e2d013904a4ecc096cee77d91b86",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/b4ace5f3454131a3070ef7c11e19e42fc9a80b4e.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/b4ace5f3454131a3070ef7c11e19e42fc9a80b4e.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/5e1b99cf3b50e2d013904a4ecc096cee77d91b86.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/5e1b99cf3b50e2d013904a4ecc096cee77d91b86.tar.gz",
         ],
     )
 
-- 
GitLab


From 7a8542339016b07eafa531720efdc72d13435884 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Tue, 13 Nov 2018 11:39:36 -0800
Subject: [PATCH 0168/1554] Fix the final checkpoint handling of
 AsyncCheckpointSaver.

PiperOrigin-RevId: 221304429
---
 tensorflow/contrib/tpu/python/tpu/async_checkpoint.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py b/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py
index 1cf7f9fcf6..1b09ce173a 100644
--- a/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py
+++ b/tensorflow/contrib/tpu/python/tpu/async_checkpoint.py
@@ -80,6 +80,8 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook):
     self._summary_writer = None
     self._global_step_tensor = None
 
+    self._last_checkpoint_step = None
+
   def _set_steps_per_run(self, steps_per_run):
     self._steps_per_run = steps_per_run
 
@@ -137,8 +139,7 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook):
 
     last_step = session.run(self._global_step_tensor)
 
-    # Save the last checkpoint synchronously if needed.
-    if last_step != self._timer.last_triggered_step():
+    if self._last_checkpoint_step != last_step:
       self._save(session, last_step, asynchronous=False)
 
     for l in self._listeners:
@@ -174,6 +175,7 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook):
       logging.info("Checkpoint finished for %d into %s.", step, self._save_path)
 
     if not asynchronous:
+      self._last_checkpoint_step = step
       _save_fn()
       return
 
@@ -183,6 +185,7 @@ class AsyncCheckpointSaverHook(basic_session_run_hooks.CheckpointSaverHook):
         logging.info("Saver thread still in progress, skipping checkpoint.")
         return
 
+    self._last_checkpoint_step = step
     self._save_thread = threading.Thread(target=_save_fn)
     self._save_thread.start()
 
-- 
GitLab


From eab659a777642a1fe7e5df6331bee38c5cf5228d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 13 Nov 2018 11:43:54 -0800
Subject: [PATCH 0169/1554] Re-enable autograph in tf.function.

PiperOrigin-RevId: 221305231
---
 tensorflow/python/eager/def_function.py         | 4 ++--
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index cad6721c70..52830d41bf 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -214,7 +214,7 @@ class PolymorphicFunction(object):
                python_function,
                name,
                input_signature=None,
-               autograph=False,
+               autograph=True,
                experimental_autograph_options=None):
     """Initializes a polymorphic function.
 
@@ -503,7 +503,7 @@ class PolymorphicFunction(object):
 @tf_export("function", v1=[])
 def function(func=None,
              input_signature=None,
-             autograph=False,
+             autograph=True,
              experimental_autograph_options=None):
   """Creates a callable TensorFlow graph from a Python function.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 6157121b97..b1e4c32523 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -738,7 +738,7 @@ tf_module {
   }
   member_method {
     name: "function"
-    argspec: "args=[\'func\', \'input_signature\', \'autograph\', \'experimental_autograph_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], "
+    argspec: "args=[\'func\', \'input_signature\', \'autograph\', \'experimental_autograph_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\', \'None\'], "
   }
   member_method {
     name: "gather"
-- 
GitLab


From 336e80e235f55873aa5df5818bdad6b12eb9579f Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 13 Nov 2018 11:45:39 -0800
Subject: [PATCH 0170/1554] Move Keras to use global batch size with
 distribution strategy.

PiperOrigin-RevId: 221305528
---
 .../contrib/distribute/python/keras_test.py   | 25 ++++++---
 .../distribute/python/one_device_strategy.py  |  5 ++
 .../contrib/distribute/python/tpu_strategy.py | 56 +++++++++++++++++++
 tensorflow/python/keras/BUILD                 |  4 +-
 .../engine/distributed_training_utils.py      | 24 +++++++-
 tensorflow/python/keras/engine/training.py    | 13 ++---
 tensorflow/python/training/distribute.py      | 27 +++++++++
 7 files changed, 135 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 0db5844e4c..790b683e58 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -218,7 +218,10 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
   global_batch_size = 64
   batch_size = global_batch_size
   # TODO(b/118776054): Use global batch size for Keras/DS support.
-  if with_distribution:
+  use_per_core_batch_size = (
+      with_distribution and
+      with_distribution.__class__.__name__ != 'TPUStrategy')
+  if use_per_core_batch_size:
     batch_size //= with_distribution.num_replicas_in_sync
 
   if use_numpy:
@@ -234,11 +237,19 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
         'x': x_train,
         'y': y_train,
     }
+    # TODO(b/119318587): We should not require batch_size when distribution
+    # is enabled.
+    if with_distribution:
+      if use_per_core_batch_size:
+        predict_batch_size = (
+            len(x_predict) // with_distribution.num_replicas_in_sync)
+      else:
+        predict_batch_size = len(x_predict)
+    else:
+      predict_batch_size = None
+
     predict_inputs = {
-        # TODO(b/119318587): We should not require batch_size when distribution
-        # is enabled.
-        'batch_size': (len(x_predict) // with_distribution.num_replicas_in_sync
-                       if with_distribution else None),
+        'batch_size': predict_batch_size,
         'x': np.array(x_predict, dtype=np.float32),
     }
   else:
@@ -263,7 +274,7 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
         'steps': 20,
     }
     predict_batch_size = len(x_predict)
-    if with_distribution:
+    if use_per_core_batch_size:
       predict_batch_size //= with_distribution.num_replicas_in_sync
     predict_dataset = dataset_ops.Dataset.from_tensor_slices(x_predict)
     predict_dataset = batch_wrapper(predict_dataset,
@@ -877,7 +888,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       predict_dataset = dataset_ops.Dataset.from_tensor_slices(inputs)
       predict_dataset = predict_dataset.repeat().batch(5)
       output = model.predict(predict_dataset, steps=10)
-      # `predict` runs for 10 steps and in each step you process 100 samples.
+      # `predict` runs for 10 steps and in each step you process 10 samples.
       ref_output = np.ones((100, 1), dtype=np.float32)
       self.assertArrayNear(output, ref_output, 1e-1)
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index a0d8f93887..f1aac56c46 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -58,6 +58,11 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     with ops.colocate_with(colocate_with):
       return next_creator(*args, **kwargs)
 
+  def make_dataset_iterator(self, dataset):
+    distributed_dataset = values.PerReplicaDataset(dataset, [self._device])
+    # TODO(priyag): Return distribution strategy specific InputIterator
+    return distributed_dataset.make_initializable_iterator()
+
   def distribute_dataset(self, dataset_fn):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), [self._device])
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f5b4531ba8..7db5f108ee 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -29,10 +29,13 @@ from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -214,6 +217,59 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
 
     return enqueue_op_per_host
 
+  def make_dataset_iterator(self, dataset):
+    """Make iterators for each of the TPU hosts.
+
+    We first unbatch the users input dataset and then rebatch it with the
+    per replica batch size that is calculated using
+    `global_batch_size // num_replicas_in_sync`. The currently supported cases
+    are as follows:
+    `dataset.batch()` is the last operation on the dataset.
+    `dataset.apply(map_and_batch)` is the last operation on the dataset.
+    `dataset.batch().prefetch()` are the last 2 operations on the dataset.
+    `dataset.apply(map_and_batch).prefetch()` are the last 2 operations.
+
+    Args:
+      dataset: The `tf.data` dataset passed by the user.
+
+    Returns:
+      iterator: InputIterator created for each of the host machines.
+    """
+    # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
+    def _get_dataset_batch_size(dataset):
+      """Get the global batch size from the dataset object."""
+      # pylint: disable=protected-access
+      if isinstance(dataset, dataset_ops.BatchDataset):
+        return tensor_util.constant_value(dataset._batch_size)
+      elif isinstance(dataset, batching._MapAndBatchDataset):
+        return dataset._batch_size
+      elif isinstance(dataset, dataset_ops.PrefetchDataset):
+        return _get_dataset_batch_size(dataset._input_dataset)
+      # pylint: enable=protected-access
+      raise ValueError(
+          "Unable to fetch the batch size from the input dataset. `batch` "
+          "`map_and_batch` need to be the last operations on the dataset. "
+          "The batch operations can be followed by a prefetch.")
+
+    global_batch_size = _get_dataset_batch_size(dataset)
+    if global_batch_size % self.num_replicas_in_sync:
+      raise ValueError(
+          "Batch size %s cannot be sharded evenly across replicas %s" % (
+              global_batch_size, self.num_replicas_in_sync))
+    per_replica_batch_size = global_batch_size // self.num_replicas_in_sync
+    dataset = dataset.apply(batching.unbatch())
+    dataset = dataset.batch(per_replica_batch_size, drop_remainder=True)
+
+    worker_devices = [
+        (self.get_host(hid), [self.get_host_cpu_device(hid)])
+        for hid in range(self.num_hosts)
+    ]
+    distributed_dataset = values.MultiWorkerDataset(
+        functools.partial(self._call_dataset_fn, lambda: dataset),
+        worker_devices)
+    # TODO(priyag): Return distribution strategy specific InputIterator
+    return distributed_dataset.make_initializable_iterator()
+
   def distribute_dataset(self, dataset_fn):
     worker_devices = [
         (self.get_host(hid), [self.get_host_cpu_device(hid)])
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 6f38d822e7..7dfafd8289 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -3,10 +3,10 @@
 
 licenses(["notice"])  # Apache 2.0
 
-exports_files(["LICENSE"])
-
 package(default_visibility = ["//visibility:public"])
 
+exports_files(["LICENSE"])
+
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index f939b7565a..7e3147bb32 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -368,7 +368,15 @@ def get_input_batch_params(first_x_value, batch_size, distribution_strategy):
   if not num_batches:
     raise ValueError('Please specify a batch_size that is smaller than'
                      'the number of input samples %d.' % first_x_value.shape[0])
-  steps = num_batches // distribution_strategy.num_replicas_in_sync
+  # TODO(b/118776054): Use global batch size for Keras/DS support.
+  # The Keras API supports using the global batch size which is currently only
+  # supported in TPU Strategy. For other strategies we use a per_replica
+  # batch size so the number of steps required to run needs to be divide by
+  # the number of replicas.
+  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
+    steps = num_batches
+  else:
+    steps = num_batches // distribution_strategy.num_replicas_in_sync
   if not steps:
     # TODO(anjalisridhar): Number of replicas in the error message may not
     # convey what we want to the user. Is there another terminology that we can
@@ -387,11 +395,12 @@ def get_batch_dimension(iterator):
   return dims[0] if dims else None
 
 
-def get_batch_size(num_replicas, num_samples, steps):
+def get_batch_size(distribution_strategy, num_samples, steps):
   """Calculate and return batch size for numpy inputs.
 
   Args:
-    num_replicas: Number of devices over which the model input is distributed.
+    distribution_strategy: The current DistributionStrategy used to compile the
+      model.
     num_samples: Total number of input samples in the input numpy arrays.
     steps: Number of steps that we run the model for.
 
@@ -405,6 +414,15 @@ def get_batch_size(num_replicas, num_samples, steps):
                     'Some samples will not be processed as expected.' %
                     (num_samples, steps))
   global_batch_size = num_samples // steps
+
+  # TODO(b/118776054): Use global batch size for Keras/DS support.
+  # The Keras API supports using the global batch size which is currently only
+  # supported in TPU Strategy. For other strategies we use a per_replica
+  # batch size so we need to divide it by the number of replicas.
+  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
+    return global_batch_size
+
+  num_replicas = distribution_strategy.num_replicas_in_sync
   if global_batch_size % num_replicas != 0:
     logging.warning('The total number of batches per step %d is not evenly '
                     'divisible by the number of replicas %d used in '
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index cb96e3e5d2..289ef7f9a0 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -893,7 +893,7 @@ class Model(Network):
       x_shape = first_x_value.shape
       if batch_size is None:
         batch_size = distributed_training_utils.get_batch_size(
-            self._distribution_strategy.num_replicas_in_sync, x_shape[0], steps)
+            self._distribution_strategy, x_shape[0], steps)
       # We need to use the drop_remainder argument to allow for a static
       # input shape which is required for TPUs.
       drop_remainder = self._distribution_strategy.require_static_shapes
@@ -932,13 +932,12 @@ class Model(Network):
         x = x.batch(batch_size, drop_remainder=drop_remainder)
 
     assert isinstance(x, dataset_ops.Dataset)
+    if self._distribution_strategy.__class__.__name__ == 'TPUStrategy':
+      iterator = self._distribution_strategy.make_dataset_iterator(x)
+    else:
+      dataset = self._distribution_strategy.distribute_dataset(lambda: x)
+      iterator = dataset.make_initializable_iterator()
 
-    # TODO(anjalisridhar): We want distribute_dataset() to accept a Dataset or a
-    # function which returns a Dataset. Currently distribute_dataset() only
-    # accepts a function that returns a Dataset. Once we add support for being
-    # able to clone a Dataset on multiple workers we can remove this lambda.
-    result = self._distribution_strategy.distribute_dataset(lambda: x)
-    iterator = result.make_initializable_iterator()
     with self._distribution_strategy.scope():
       K.get_session().run(iterator.initializer)
 
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 95104ad577..f515df0de1 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -541,6 +541,30 @@ class DistributionStrategy(object):
           "DistributionStrategy.")
     return result
 
+  def make_dataset_iterator(self, dataset):
+    """Makes an iterator for input provided via input_dataset.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    global batch size. With this assumption, we will make a best effort to
+    divide each batch across all the replicas (one or more workers).
+    If this effort fails, an error will be thrown, and the user should instead
+    use `make_input_fn_iterator` which provides more control to the user, and
+    does not try to divide a batch across replicas.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `InputIterator` which returns inputs for each step of the computation.
+      User should call `initialize` on the returned iterator.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
   # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
   # Dataset API such as make_one_shot_iterator and make_initializable_iterator.
   # Extend to implement more functionality of datasets.
@@ -1148,6 +1172,9 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     _require_distribution_strategy_scope(self)
     return ops.colocate_with(colocate_with_variable)
 
+  def make_dataset_iterator(self, dataset):
+    return dataset.make_initializable_iterator()
+
   def distribute_dataset(self, dataset_fn):
     return self._call_dataset_fn(dataset_fn)
 
-- 
GitLab


From 236fc65a99cd27acbd7a425cc7ebab1d3571b9d2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 11:47:33 -0800
Subject: [PATCH 0171/1554] Add ConfigProto.experimental.use_numa_affinity and
 related changes so that CPU devices can be constrained to single NUMA nodes.

This change will be effective on a platform as soon as it has real
implementations of the functions in core/platform/numa.h which are not
yet provided.

When ConfigProto.experimental.use_numa_affinity is false (the default)
then behavior is unchanged.  When true then the runtime will define
one CPU device for every visible NUMA node and each will attempt to
execute all of its threads on that node, suppling an Allocator
to its ops that provides only node-local CPU memory.

PiperOrigin-RevId: 221305866
---
 .../core/common_runtime/device_factory.cc     |  7 +-
 .../common_runtime/gpu/gpu_process_state.cc   |  4 +-
 .../core/common_runtime/local_device.cc       | 73 ++++++++++++++++---
 tensorflow/core/common_runtime/local_device.h |  7 ++
 .../core/common_runtime/pool_allocator.cc     | 14 +++-
 .../core/common_runtime/pool_allocator.h      |  1 -
 .../core/common_runtime/process_state.cc      |  8 +-
 .../core/common_runtime/process_state.h       |  4 +-
 .../core/common_runtime/threadpool_device.cc  |  2 +-
 .../threadpool_device_factory.cc              | 27 ++++++-
 tensorflow/core/framework/allocator.cc        |  2 +
 tensorflow/core/framework/allocator.h         |  2 +
 tensorflow/core/framework/device_base.cc      |  6 +-
 tensorflow/core/lib/core/threadpool.cc        | 21 ++++--
 tensorflow/core/lib/core/threadpool.h         |  6 +-
 tensorflow/core/platform/env.h                |  2 +
 tensorflow/core/protobuf/config.proto         |  5 ++
 ...nsorflow.-config-proto.-experimental.pbtxt |  8 +-
 .../golden/v1/tensorflow.-config-proto.pbtxt  |  6 ++
 ...nsorflow.-config-proto.-experimental.pbtxt |  6 ++
 .../golden/v2/tensorflow.-config-proto.pbtxt  |  6 ++
 21 files changed, 179 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/common_runtime/device_factory.cc b/tensorflow/core/common_runtime/device_factory.cc
index b43c718817..b94900114c 100644
--- a/tensorflow/core/common_runtime/device_factory.cc
+++ b/tensorflow/core/common_runtime/device_factory.cc
@@ -127,7 +127,12 @@ Device* DeviceFactory::NewDevice(const string& type,
   (*opt.config.mutable_device_count())[type] = 1;
   std::vector<Device*> devices;
   TF_CHECK_OK(device_factory->CreateDevices(opt, name_prefix, &devices));
-  CHECK_EQ(devices.size(), size_t{1});
+  int expected_num_devices = 1;
+  auto iter = options.config.device_count().find(type);
+  if (iter != options.config.device_count().end()) {
+    expected_num_devices = iter->second;
+  }
+  DCHECK_EQ(devices.size(), static_cast<size_t>(expected_num_devices));
   return devices[0];
 }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
index 3e95374fda..1a43d2c8fc 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
@@ -166,7 +166,9 @@ Allocator* GPUProcessState::GetCUDAHostAllocator(int numa_node) {
       !process_state_->ProcessState::FLAGS_brain_mem_reg_cuda_dma) {
     return process_state_->GetCPUAllocator(numa_node);
   }
-  CHECK_GE(numa_node, 0);
+  if (numa_node == port::kNUMANoAffinity) {
+    numa_node = 0;
+  }
   {
     // Here we optimize the most common use case where cuda_host_allocators_
     // and cuda_al_ have already been populated and since we're only reading
diff --git a/tensorflow/core/common_runtime/local_device.cc b/tensorflow/core/common_runtime/local_device.cc
index 873182371e..f1fcca194e 100644
--- a/tensorflow/core/common_runtime/local_device.cc
+++ b/tensorflow/core/common_runtime/local_device.cc
@@ -18,11 +18,13 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/local_device.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/eigen_thread_pool.h"
+#include "tensorflow/core/common_runtime/process_state.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/platform/byte_order.h"
 #include "tensorflow/core/platform/cpu_feature_guard.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/public/session_options.h"
 
@@ -30,23 +32,52 @@ namespace tensorflow {
 
 /* static */
 bool LocalDevice::use_global_threadpool_ = true;
+mutex LocalDevice::global_tp_mu_;
+gtl::InlinedVector<LocalDevice::EigenThreadPoolInfo*, 4>
+    LocalDevice::global_tp_info_;
 
 struct LocalDevice::EigenThreadPoolInfo {
-  explicit EigenThreadPoolInfo(const SessionOptions& options) {
+  // Wrapper so we can provide the CPUAllocator to Eigen for use
+  // when ops need extra tmp memory.
+  class EigenAllocator : public Eigen::Allocator {
+   public:
+    explicit EigenAllocator(tensorflow::Allocator* a) : allocator_(a) {}
+    void* allocate(size_t num_bytes) const override {
+      return allocator_->AllocateRaw(64, num_bytes);
+    }
+    void deallocate(void* buffer) const override {
+      allocator_->DeallocateRaw(buffer);
+    }
+    tensorflow::Allocator* allocator_;
+  };
+
+  explicit EigenThreadPoolInfo(const SessionOptions& options, int numa_node,
+                               Allocator* allocator) {
     int32 intra_op_parallelism_threads =
         options.config.intra_op_parallelism_threads();
     if (intra_op_parallelism_threads == 0) {
       intra_op_parallelism_threads = port::NumSchedulableCPUs();
+      if (numa_node != port::kNUMANoAffinity) {
+        // Assume that CPUs are equally distributed over available NUMA nodes.
+        // This may not be true, but there isn't currently a better way of
+        // determining the number of CPUs specific to the requested node.
+        intra_op_parallelism_threads /= port::NUMANumNodes();
+      }
     }
-    VLOG(1) << "Local device intra op parallelism threads: "
-            << intra_op_parallelism_threads;
+    ThreadOptions thread_opts;
+    thread_opts.numa_node = numa_node;
     eigen_worker_threads_.num_threads = intra_op_parallelism_threads;
     eigen_worker_threads_.workers = new thread::ThreadPool(
-        options.env, "Eigen", intra_op_parallelism_threads);
+        options.env, thread_opts, strings::StrCat("numa_", numa_node, "_Eigen"),
+        intra_op_parallelism_threads);
     eigen_threadpool_wrapper_.reset(
         new EigenThreadPoolWrapper(eigen_worker_threads_.workers));
+    if (allocator) {
+      eigen_allocator_.reset(new EigenAllocator(allocator));
+    }
     eigen_device_.reset(new Eigen::ThreadPoolDevice(
-        eigen_threadpool_wrapper_.get(), eigen_worker_threads_.num_threads));
+        eigen_threadpool_wrapper_.get(), eigen_worker_threads_.num_threads,
+        eigen_allocator_.get()));
   }
 
   ~EigenThreadPoolInfo() {
@@ -58,6 +89,7 @@ struct LocalDevice::EigenThreadPoolInfo {
   DeviceBase::CpuWorkerThreads eigen_worker_threads_;
   std::unique_ptr<Eigen::ThreadPoolInterface> eigen_threadpool_wrapper_;
   std::unique_ptr<Eigen::ThreadPoolDevice> eigen_device_;
+  std::unique_ptr<EigenAllocator> eigen_allocator_;
 };
 
 LocalDevice::LocalDevice(const SessionOptions& options,
@@ -68,15 +100,34 @@ LocalDevice::LocalDevice(const SessionOptions& options,
   port::InfoAboutUnusedCPUFeatures();
   LocalDevice::EigenThreadPoolInfo* tp_info;
   if (use_global_threadpool_) {
-    // All ThreadPoolDevices in the process will use this single fixed
-    // sized threadpool for numerical computations.
-    static LocalDevice::EigenThreadPoolInfo* global_tp_info =
-        new LocalDevice::EigenThreadPoolInfo(options);
-    tp_info = global_tp_info;
+    mutex_lock l(global_tp_mu_);
+    if (options.config.experimental().use_numa_affinity()) {
+      int numa_node = attributes.locality().numa_node();
+      int num_numa_nodes = port::NUMANumNodes();
+      DCHECK_LT(numa_node, num_numa_nodes);
+      Allocator* numa_allocator =
+          ProcessState::singleton()->GetCPUAllocator(numa_node);
+      while (numa_node >= global_tp_info_.size()) {
+        global_tp_info_.push_back(nullptr);
+      }
+      if (!global_tp_info_[numa_node]) {
+        global_tp_info_[numa_node] = new LocalDevice::EigenThreadPoolInfo(
+            options, numa_node, numa_allocator);
+      }
+      tp_info = global_tp_info_[numa_node];
+    } else {
+      if (global_tp_info_.empty()) {
+        global_tp_info_.push_back(new LocalDevice::EigenThreadPoolInfo(
+            options, port::kNUMANoAffinity, nullptr));
+      }
+      tp_info = global_tp_info_[0];
+    }
   } else {
     // Each LocalDevice owns a separate ThreadPoolDevice for numerical
     // computations.
-    owned_tp_info_.reset(new LocalDevice::EigenThreadPoolInfo(options));
+    // TODO(tucker): NUMA for these too?
+    owned_tp_info_.reset(new LocalDevice::EigenThreadPoolInfo(
+        options, port::kNUMANoAffinity, nullptr));
     tp_info = owned_tp_info_.get();
   }
   set_tensorflow_cpu_worker_threads(&tp_info->eigen_worker_threads_);
diff --git a/tensorflow/core/common_runtime/local_device.h b/tensorflow/core/common_runtime/local_device.h
index 226f121bf3..f305c212c5 100644
--- a/tensorflow/core/common_runtime/local_device.h
+++ b/tensorflow/core/common_runtime/local_device.h
@@ -47,6 +47,13 @@ class LocalDevice : public Device {
   struct EigenThreadPoolInfo;
   std::unique_ptr<EigenThreadPoolInfo> owned_tp_info_;
 
+  // All ThreadPoolDevices in the process associated with the same
+  // NUMA node will share a single fixed sized threadpool for numerical
+  // computations.
+  static mutex global_tp_mu_;
+  static gtl::InlinedVector<EigenThreadPoolInfo*, 4> global_tp_info_
+      GUARDED_BY(global_tp_mu_);
+
   friend class test::Benchmark;
 
   TF_DISALLOW_COPY_AND_ASSIGN(LocalDevice);
diff --git a/tensorflow/core/common_runtime/pool_allocator.cc b/tensorflow/core/common_runtime/pool_allocator.cc
index 66dc8f3322..6b40fcc4c7 100644
--- a/tensorflow/core/common_runtime/pool_allocator.cc
+++ b/tensorflow/core/common_runtime/pool_allocator.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -258,7 +259,12 @@ void PoolAllocator::EvictOne() {
 void* BasicCPUAllocator::Alloc(size_t alignment, size_t num_bytes) {
   void* ptr = nullptr;
   if (num_bytes > 0) {
-    ptr = port::AlignedMalloc(num_bytes, static_cast<int>(alignment));
+    if (numa_node_ == port::kNUMANoAffinity) {
+      ptr = port::AlignedMalloc(num_bytes, static_cast<int>(alignment));
+    } else {
+      ptr =
+          port::NUMAMalloc(numa_node_, num_bytes, static_cast<int>(alignment));
+    }
     VisitAlloc(ptr, numa_node_, num_bytes);
   }
   return ptr;
@@ -267,7 +273,11 @@ void* BasicCPUAllocator::Alloc(size_t alignment, size_t num_bytes) {
 void BasicCPUAllocator::Free(void* ptr, size_t num_bytes) {
   if (num_bytes > 0) {
     VisitFree(ptr, numa_node_, num_bytes);
-    port::AlignedFree(ptr);
+    if (numa_node_ == port::kNUMANoAffinity) {
+      port::AlignedFree(ptr);
+    } else {
+      port::NUMAFree(ptr, num_bytes);
+    }
   }
 }
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/pool_allocator.h b/tensorflow/core/common_runtime/pool_allocator.h
index 5b4623ba10..8be9c7b678 100644
--- a/tensorflow/core/common_runtime/pool_allocator.h
+++ b/tensorflow/core/common_runtime/pool_allocator.h
@@ -150,7 +150,6 @@ class Pow2Rounder : public RoundUpInterface {
 
 class BasicCPUAllocator : public SubAllocator {
  public:
-  // Argument numa_node is currently ignored.
   BasicCPUAllocator(int numa_node, const std::vector<Visitor>& alloc_visitors,
                     const std::vector<Visitor>& free_visitors)
       : SubAllocator(alloc_visitors, free_visitors), numa_node_(numa_node) {}
diff --git a/tensorflow/core/common_runtime/process_state.cc b/tensorflow/core/common_runtime/process_state.cc
index bcaa37fc8a..c8d5bf100b 100644
--- a/tensorflow/core/common_runtime/process_state.cc
+++ b/tensorflow/core/common_runtime/process_state.cc
@@ -72,8 +72,7 @@ ProcessState::MemDesc ProcessState::PtrType(const void* ptr) {
 }
 
 Allocator* ProcessState::GetCPUAllocator(int numa_node) {
-  CHECK_GE(numa_node, 0);
-  if (!numa_enabled_) numa_node = 0;
+  if (!numa_enabled_ || numa_node == port::kNUMANoAffinity) numa_node = 0;
   mutex_lock lock(mu_);
   while (cpu_allocators_.size() <= static_cast<size_t>(numa_node)) {
     // If visitors have been defined we need an Allocator built from
@@ -90,8 +89,9 @@ Allocator* ProcessState::GetCPUAllocator(int numa_node) {
     Allocator* allocator = nullptr;
     SubAllocator* sub_allocator =
         (alloc_visitors_defined || use_bfc_allocator)
-            ? new BasicCPUAllocator(numa_enabled_ ? numa_node : -1,
-                                    cpu_alloc_visitors_, cpu_free_visitors_)
+            ? new BasicCPUAllocator(
+                  numa_enabled_ ? numa_node : port::kNUMANoAffinity,
+                  cpu_alloc_visitors_, cpu_free_visitors_)
             : nullptr;
     if (use_bfc_allocator) {
       // TODO(reedwm): evaluate whether 64GB by default is the best choice.
diff --git a/tensorflow/core/common_runtime/process_state.h b/tensorflow/core/common_runtime/process_state.h
index cac312d849..06a283c9a6 100644
--- a/tensorflow/core/common_runtime/process_state.h
+++ b/tensorflow/core/common_runtime/process_state.h
@@ -63,7 +63,7 @@ class ProcessState {
   MemDesc PtrType(const void* ptr);
 
   // Returns the one CPUAllocator used for the given numa_node.
-  // TEMPORARY: ignores numa_node.
+  // Treats numa_node == kNUMANoAffinity as numa_node == 0.
   Allocator* GetCPUAllocator(int numa_node);
 
   // Registers alloc visitor for the CPU allocator(s).
@@ -94,6 +94,8 @@ class ProcessState {
 
   mutex mu_;
 
+  // Indexed by numa_node.  If we want numa-specific allocators AND a
+  // non-specific allocator, maybe should index by numa_node+1.
   std::vector<Allocator*> cpu_allocators_ GUARDED_BY(mu_);
   std::vector<SubAllocator::Visitor> cpu_alloc_visitors_ GUARDED_BY(mu_);
   std::vector<SubAllocator::Visitor> cpu_free_visitors_ GUARDED_BY(mu_);
diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc
index 6404d8bc6a..ca7ca5443c 100644
--- a/tensorflow/core/common_runtime/threadpool_device.cc
+++ b/tensorflow/core/common_runtime/threadpool_device.cc
@@ -93,7 +93,7 @@ Status ThreadPoolDevice::MakeTensorFromProto(
     Tensor* tensor) {
   if (tensor_proto.dtype() > 0 && tensor_proto.dtype() <= DataType_MAX) {
     Tensor parsed(tensor_proto.dtype());
-    if (parsed.FromProto(cpu_allocator(), tensor_proto)) {
+    if (parsed.FromProto(allocator_, tensor_proto)) {
       *tensor = std::move(parsed);
       return Status::OK();
     }
diff --git a/tensorflow/core/common_runtime/threadpool_device_factory.cc b/tensorflow/core/common_runtime/threadpool_device_factory.cc
index 6a900c02c0..c06a4035a7 100644
--- a/tensorflow/core/common_runtime/threadpool_device_factory.cc
+++ b/tensorflow/core/common_runtime/threadpool_device_factory.cc
@@ -18,7 +18,9 @@ limitations under the License.
 
 #include <vector>
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/process_state.h"
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/public/session_options.h"
 
 namespace tensorflow {
@@ -28,8 +30,7 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
                        std::vector<Device*>* devices) override {
-    // TODO(zhifengc/tucker): Figure out the number of available CPUs
-    // and/or NUMA configuration.
+    int num_numa_nodes = port::NUMANumNodes();
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
     if (iter != options.config.device_count().end()) {
@@ -37,8 +38,26 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      devices->push_back(new ThreadPoolDevice(
-          options, name, Bytes(256 << 20), DeviceLocality(), cpu_allocator()));
+      ThreadPoolDevice* tpd = nullptr;
+      if (options.config.experimental().use_numa_affinity()) {
+        int numa_node = i % num_numa_nodes;
+        if (numa_node != i) {
+          LOG(INFO) << "Only " << num_numa_nodes
+                    << " NUMA nodes visible in system, "
+                    << " assigning device " << name << " to NUMA node "
+                    << numa_node;
+        }
+        DeviceLocality dev_locality;
+        dev_locality.set_numa_node(numa_node);
+        tpd = new ThreadPoolDevice(
+            options, name, Bytes(256 << 20), dev_locality,
+            ProcessState::singleton()->GetCPUAllocator(numa_node));
+      } else {
+        tpd = new ThreadPoolDevice(
+            options, name, Bytes(256 << 20), DeviceLocality(),
+            ProcessState::singleton()->GetCPUAllocator(port::kNUMANoAffinity));
+      }
+      devices->push_back(tpd);
     }
 
     return Status::OK();
diff --git a/tensorflow/core/framework/allocator.cc b/tensorflow/core/framework/allocator.cc
index 84cee5569c..89c49a2ad0 100644
--- a/tensorflow/core/framework/allocator.cc
+++ b/tensorflow/core/framework/allocator.cc
@@ -96,9 +96,11 @@ static int64_t TotalAllocationWarningBytes() {
 void EnableCPUAllocatorStats(bool enable) {
   cpu_allocator_collect_stats = enable;
 }
+bool CPUAllocatorStatsEnabled() { return cpu_allocator_collect_stats; }
 void EnableCPUAllocatorFullStats(bool enable) {
   cpu_allocator_collect_full_stats = enable;
 }
+bool CPUAllocatorFullStatsEnabled() { return cpu_allocator_collect_full_stats; }
 
 namespace {
 // A default Allocator for CPU devices.  ProcessState::GetCPUAllocator() will
diff --git a/tensorflow/core/framework/allocator.h b/tensorflow/core/framework/allocator.h
index 8c23604625..531ea73e89 100644
--- a/tensorflow/core/framework/allocator.h
+++ b/tensorflow/core/framework/allocator.h
@@ -383,10 +383,12 @@ Allocator* cpu_allocator();
 // If 'enable' is true, the default CPU allocator implementation will collect
 // AllocatorStats. By default, it's disabled.
 void EnableCPUAllocatorStats(bool enable);
+bool CPUAllocatorStatsEnabled();
 
 // If 'enable' is true, the default CPU allocator implementation will collect
 // full statistics. By default, it's disabled.
 void EnableCPUAllocatorFullStats(bool enable);
+bool CPUAllocatorFullStatsEnabled();
 
 // An object that does the underlying suballoc/free of memory for a higher-level
 // allocator.  The expectation is that the higher-level allocator is doing some
diff --git a/tensorflow/core/framework/device_base.cc b/tensorflow/core/framework/device_base.cc
index 9108c32942..78ace480c4 100644
--- a/tensorflow/core/framework/device_base.cc
+++ b/tensorflow/core/framework/device_base.cc
@@ -34,14 +34,14 @@ const string& DeviceBase::name() const {
 }
 
 void DeviceBase::set_eigen_cpu_device(Eigen::ThreadPoolDevice* d) {
-  // Eigen::ThreadPoolDevice is a very cheap struct (one pointer and
+  // Eigen::ThreadPoolDevice is a very cheap struct (two pointers and
   // an int).  Therefore, we can afford a pre-allocated array of
   // Eigen::ThreadPoolDevice.  Here, we ensure that
   // Eigen::ThreadPoolDevices in eigen_cpu_devices_ has increasingly
   // larger numThreads.
   for (int i = 1; i <= d->numThreads(); ++i) {
-    eigen_cpu_devices_.push_back(
-        new Eigen::ThreadPoolDevice(d->getPool(), i /* numThreads() */));
+    eigen_cpu_devices_.push_back(new Eigen::ThreadPoolDevice(
+        d->getPool(), i /* numThreads() */, d->allocator()));
   }
 }
 
diff --git a/tensorflow/core/lib/core/threadpool.cc b/tensorflow/core/lib/core/threadpool.cc
index 42689a6c3b..e929ff45a1 100644
--- a/tensorflow/core/lib/core/threadpool.cc
+++ b/tensorflow/core/lib/core/threadpool.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/platform/denormal.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/setround.h"
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/platform/types.h"
@@ -54,6 +55,9 @@ struct EigenEnvironment {
       port::ScopedFlushDenormal flush;
       // Set the processor rounding mode to ROUND TO NEAREST.
       port::ScopedSetRound round(FE_TONEAREST);
+      if (thread_options_.numa_node != port::kNUMANoAffinity) {
+        port::NUMASetThreadNodeAffinity(thread_options_.numa_node);
+      }
       f();
     });
   }
@@ -83,35 +87,38 @@ struct EigenEnvironment {
 
 struct ThreadPool::Impl : Eigen::ThreadPoolTempl<EigenEnvironment> {
   Impl(Env* env, const ThreadOptions& thread_options, const string& name,
-       int num_threads, bool low_latency_hint)
+       int num_threads, bool low_latency_hint, Eigen::Allocator* allocator)
       : Eigen::ThreadPoolTempl<EigenEnvironment>(
             num_threads, low_latency_hint,
-            EigenEnvironment(env, thread_options, name)) {}
+            EigenEnvironment(env, thread_options, name)),
+        allocator_(allocator) {}
 
   void ParallelFor(int64 total, int64 cost_per_unit,
                    std::function<void(int64, int64)> fn) {
     CHECK_GE(total, 0);
     CHECK_EQ(total, (int64)(Eigen::Index)total);
-    Eigen::ThreadPoolDevice device(this, this->NumThreads());
+    Eigen::ThreadPoolDevice device(this, this->NumThreads(), allocator_);
     device.parallelFor(
         total, Eigen::TensorOpCost(0, 0, cost_per_unit),
         [&fn](Eigen::Index first, Eigen::Index last) { fn(first, last); });
   }
+
+  Eigen::Allocator* allocator_;
 };
 
 ThreadPool::ThreadPool(Env* env, const string& name, int num_threads)
-    : ThreadPool(env, ThreadOptions(), name, num_threads, true) {}
+    : ThreadPool(env, ThreadOptions(), name, num_threads, true, nullptr) {}
 
 ThreadPool::ThreadPool(Env* env, const ThreadOptions& thread_options,
                        const string& name, int num_threads)
-    : ThreadPool(env, thread_options, name, num_threads, true) {}
+    : ThreadPool(env, thread_options, name, num_threads, true, nullptr) {}
 
 ThreadPool::ThreadPool(Env* env, const ThreadOptions& thread_options,
                        const string& name, int num_threads,
-                       bool low_latency_hint) {
+                       bool low_latency_hint, Eigen::Allocator* allocator) {
   CHECK_GE(num_threads, 1);
   impl_.reset(new ThreadPool::Impl(env, thread_options, "tf_" + name,
-                                   num_threads, low_latency_hint));
+                                   num_threads, low_latency_hint, allocator));
 }
 
 ThreadPool::~ThreadPool() {}
diff --git a/tensorflow/core/lib/core/threadpool.h b/tensorflow/core/lib/core/threadpool.h
index 3da7dcb632..90c9f29447 100644
--- a/tensorflow/core/lib/core/threadpool.h
+++ b/tensorflow/core/lib/core/threadpool.h
@@ -22,6 +22,9 @@ limitations under the License.
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
 
+namespace Eigen {
+class Allocator;
+}  // namespace Eigen
 namespace tensorflow {
 namespace thread {
 
@@ -37,7 +40,8 @@ class ThreadPool {
   //
   // REQUIRES: num_threads > 0
   ThreadPool(Env* env, const ThreadOptions& thread_options, const string& name,
-             int num_threads, bool low_latency_hint);
+             int num_threads, bool low_latency_hint,
+             Eigen::Allocator* allocator = nullptr);
 
   // Constructs a pool for low-latency ops that contains "num_threads" threads
   // with specified "name". env->StartThread() is used to create individual
diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h
index 5732271f15..7374fccdc2 100644
--- a/tensorflow/core/platform/env.h
+++ b/tensorflow/core/platform/env.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/platform/file_system.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -395,6 +396,7 @@ struct ThreadOptions {
   size_t stack_size = 0;  // 0: use system default value
   /// Guard area size to use near thread stacks to use (in bytes)
   size_t guard_size = 0;  // 0: use system default value
+  int numa_node = port::kNUMANoAffinity;
 };
 
 /// A utility routine: copy contents of `src` in file system `src_fs`
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 4689af06af..174b588661 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -413,6 +413,11 @@ message ConfigProto {
     // Any positive value sets the max chunk size.  0 defaults to 4096.
     // Any negative value indicates no max, i.e. one chunk only.
     int32 recv_buf_max_chunk = 4;
+
+    // If true, and supported by the platform, the runtime will attempt to
+    // use NUMA affinity where applicable.  One consequence will be the
+    // existence of as many CPU devices as there are available NUMA nodes.
+    bool use_numa_affinity = 5;
   };
 
   Experimental experimental = 16;
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt
index f7491649c2..a1083d732a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.-experimental.pbtxt
@@ -20,7 +20,13 @@ tf_proto {
       label: LABEL_OPTIONAL
       type: TYPE_INT32
     }
-    reserved_range {
+    field {
+      name: "use_numa_affinity"
+      number: 5
+      label: LABEL_OPTIONAL
+      type: TYPE_BOOL
+    }
+     reserved_range {
       start: 2
       end: 3
     }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt
index 53b532beab..b505d81350 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-config-proto.pbtxt
@@ -143,6 +143,12 @@ tf_proto {
         label: LABEL_OPTIONAL
         type: TYPE_INT32
       }
+      field {
+        name: "use_numa_affinity"
+        number: 5
+        label: LABEL_OPTIONAL
+        type: TYPE_BOOL
+      }
       reserved_range {
         start: 2
         end: 3
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt
index f7491649c2..caa72fe5a6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt
@@ -20,6 +20,12 @@ tf_proto {
       label: LABEL_OPTIONAL
       type: TYPE_INT32
     }
+    field {
+      name: "use_numa_affinity"
+      number: 5
+      label: LABEL_OPTIONAL
+      type: TYPE_BOOL
+    }
     reserved_range {
       start: 2
       end: 3
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt
index 53b532beab..b505d81350 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt
@@ -143,6 +143,12 @@ tf_proto {
         label: LABEL_OPTIONAL
         type: TYPE_INT32
       }
+      field {
+        name: "use_numa_affinity"
+        number: 5
+        label: LABEL_OPTIONAL
+        type: TYPE_BOOL
+      }
       reserved_range {
         start: 2
         end: 3
-- 
GitLab


From 1f151edc6cf1bd508c93b9bbf3ee3c20995247cd Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 13 Nov 2018 11:53:36 -0800
Subject: [PATCH 0172/1554] Remove dead code.

_USE_C_SHAPES has been forced True for a while now.
(June: commit 1d74a69443f741e69f9f52cb6bc2940b4d4ae3b7
August: commit a473f435cf7345cb9dc2efacb471a7f318141a9b)
PiperOrigin-RevId: 221306957
---
 tensorflow/contrib/layers/python/layers/embedding_ops_test.py | 1 -
 tensorflow/python/framework/ops_test.py                       | 1 -
 tensorflow/python/kernel_tests/array_ops_test.py              | 3 ---
 tensorflow/python/kernel_tests/conv2d_transpose_test.py       | 2 --
 tensorflow/python/ops/ragged/ragged_tensor_test.py            | 2 --
 5 files changed, 9 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py
index 124515e5a6..8015a571e1 100644
--- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py
+++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py
@@ -691,7 +691,6 @@ class EmbeddingLookupSparseWithDistributedAggregationTest(test.TestCase):
       index += num_val
     return grouped_vals
 
-  @test_util.enable_c_shapes
   def testEmbeddingLookupSparse(self):
     vocab_size = 13
     batch_size = 10
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 0fb17081e7..d7c5a1a6e5 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -701,7 +701,6 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(g.get_operation_by_name("myop"), op)
     self.assertEqual(g.get_tensor_by_name("myop:0"), op.outputs[0])
 
-  @test_util.enable_c_shapes
   def testShape(self):
     g = ops.Graph()
     with g.as_default():
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index c5b25dc8d0..5452ec4804 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -1098,7 +1098,6 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
           [[True, False, False, False, False], [True, True, True, False, False],
            [True, True, False, False, False]])
 
-  @test_util.enable_c_shapes
   def testOneDimensionalDtypeWithoutMaxlen(self):
     with self.cached_session():
       # test dtype and default maxlen:
@@ -1109,7 +1108,6 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
           res.eval(),
           [[0.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0]])
 
-  @test_util.enable_c_shapes
   def testOneDimensionalWithoutMaxlen(self):
     with self.cached_session():
       res = array_ops.sequence_mask(
@@ -1121,7 +1119,6 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
            [True, False, False, False],
            [True, True, True, True]])
 
-  @test_util.enable_c_shapes
   def testTwoDimensional(self):
     with self.cached_session():
       res = array_ops.sequence_mask(constant_op.constant([[1, 3, 2]]), 5)
diff --git a/tensorflow/python/kernel_tests/conv2d_transpose_test.py b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
index 6f9992a317..95a17cfb44 100644
--- a/tensorflow/python/kernel_tests/conv2d_transpose_test.py
+++ b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
@@ -23,7 +23,6 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
@@ -293,7 +292,6 @@ class Conv2DTransposeTest(test.TestCase):
 
         self.assertAllClose(cache_values, value)
 
-  @test_util.enable_c_shapes
   def testConv2DTransposeShapeInference(self):
     # Test case for 8972
     initializer = random_ops.truncated_normal(
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index 61bfcb6809..f66ca102ef 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -1041,8 +1041,6 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.from_row_splits(values, [0, 1])
     self._TestGetItemException(rt, slice_spec, expected, message)
 
-  # TODO(edloper): Remove this decorator once c shapes become the default.
-  @test_util.enable_c_shapes
   def testGetItemNewAxis(self):
     # rt: [[[['a', 'b'], ['c', 'd']], [], [['e', 'f']]], []]
     splits1 = [0, 3, 3]
-- 
GitLab


From 199ead85e823e9b9bdc47c826eea8122f3d24592 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 13 Nov 2018 12:03:15 -0800
Subject: [PATCH 0173/1554] Fix bug in If lowering and make
 {Input,Output}Tensor.node non-const.

Prior to this change, the If lowering pass would always use the first
output of the predicate op as the predicate. This change makes it use
the correct output. In addition, this adds more OutputTensor plumbing
which required making the node field non-const.

PiperOrigin-RevId: 221308751
---
 tensorflow/c/c_api_function.cc                | 22 +++++++++----------
 tensorflow/core/common_runtime/lower_if_op.cc | 10 ++++-----
 tensorflow/core/graph/graph.cc                |  8 +++++++
 tensorflow/core/graph/graph.h                 | 13 +++++++----
 tensorflow/core/graph/node_builder.cc         |  2 ++
 tensorflow/core/graph/node_builder.h          |  1 +
 .../kernel_tests/control_flow_ops_py_test.py  | 10 +++++++++
 7 files changed, 46 insertions(+), 20 deletions(-)

diff --git a/tensorflow/c/c_api_function.cc b/tensorflow/c/c_api_function.cc
index f68f8a3e90..28b9f8df9c 100644
--- a/tensorflow/c/c_api_function.cc
+++ b/tensorflow/c/c_api_function.cc
@@ -392,26 +392,26 @@ Status ProcessInputs(
     EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) {
   input_tensors->reserve(ninputs);
   for (int i = 0; i < ninputs; ++i) {
-    const Node& node = inputs[i].oper->node;
+    Node* node = &inputs[i].oper->node;
     int idx = inputs[i].index;
 
     TF_RETURN_WITH_CONTEXT_IF_ERROR(
-        fn_body->graph.IsValidOutputTensor(&node, idx),
+        fn_body->graph.IsValidOutputTensor(node, idx),
         "Encountered while processing input ", i, " into function '", fn_name,
         "'");
-    TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(&node, idx),
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(node, idx),
                                     "Encountered while processing input ", i,
                                     " into function '", fn_name, "'");
 
-    input_tensors->emplace_back(&node, idx);
+    input_tensors->emplace_back(node, idx);
 
-    const auto& iter = input_nodes->find(&node);
+    const auto& iter = input_nodes->find(node);
     if (iter == input_nodes->end()) {
-      input_nodes->insert({&node, {idx}});
+      input_nodes->insert({node, {idx}});
     } else {
       auto& indices = iter->second;
       if (std::find(indices.begin(), indices.end(), idx) != indices.end()) {
-        return InvalidArgument("TF_Output ", node.name(), ":", idx,
+        return InvalidArgument("TF_Output ", node->name(), ":", idx,
                                " appears more than once in the input list");
       }
       indices.push_back(idx);
@@ -428,16 +428,16 @@ Status ProcessOutputs(const TF_Graph* fn_body, const char* fn_name,
     EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) {
   output_tensors->reserve(noutputs);
   for (int i = 0; i < noutputs; ++i) {
-    const Node& node = outputs[i].oper->node;
+    Node* node = &outputs[i].oper->node;
     int idx = outputs[i].index;
     TF_RETURN_WITH_CONTEXT_IF_ERROR(
-        fn_body->graph.IsValidOutputTensor(&node, idx),
+        fn_body->graph.IsValidOutputTensor(node, idx),
         "Encountered while processing output ", i, " from function '", fn_name,
         "'");
-    TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(&node, idx),
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNonRefOutput(node, idx),
                                     "Encountered while creating function '",
                                     fn_name, "'");
-    output_tensors->emplace_back(&node, idx);
+    output_tensors->emplace_back(node, idx);
   }
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/lower_if_op.cc b/tensorflow/core/common_runtime/lower_if_op.cc
index 44a2478e3f..9738006f5c 100644
--- a/tensorflow/core/common_runtime/lower_if_op.cc
+++ b/tensorflow/core/common_runtime/lower_if_op.cc
@@ -76,7 +76,7 @@ class CondBuilder {
   // The identity node with the same outputs as the original If op.
   Node* lowered_if_output_;
   // The predicate of the conditional.
-  Node* pred_;
+  OutputTensor pred_;
   // Node corresponding to pivot_f branch of predicate switch which is
   // the pivot node that dominates all nodes in the false/else branch.
   Node* pivot_f_;
@@ -102,7 +102,7 @@ CondBuilder::CondBuilder(Node* if_op, const string& then_fn_name,
       name_(if_op->name()),
       then_call_builder_(NewName("then"), then_fn_name, graph->op_registry()),
       else_call_builder_(NewName("else"), else_fn_name, graph->op_registry()) {
-  TF_CHECK_OK(if_op_->input_node(0, &pred_));
+  TF_CHECK_OK(if_op_->input_tensor(0, &pred_));
   then_call_builder_.Device(if_op_->requested_device());
   else_call_builder_.Device(if_op_->requested_device());
 }
@@ -113,8 +113,8 @@ Status CondBuilder::CreatePivotNodes() {
   Node* switch_pred;
   TF_RETURN_IF_ERROR(
       NodeBuilder(NewName("switch_pred"), "Switch", graph_->op_registry())
-          .Input(NodeOut(pred_, 0))
-          .Input(NodeOut(pred_, 0))
+          .Input(NodeOut(pred_))
+          .Input(NodeOut(pred_))
           .Device(if_op_->requested_device())
           .Finalize(graph_, &switch_pred));
   control_predecessor_ = switch_pred;
@@ -140,7 +140,7 @@ Status CondBuilder::AddInput(Node* src, int src_output) {
   TF_RETURN_IF_ERROR(
       NodeBuilder(NewName(src->name()), "Switch", graph_->op_registry())
           .Input(src, src_output)
-          .Input(pred_, 0)
+          .Input(pred_)
           .Device(if_op_->requested_device())
           .Finalize(graph_, &input));
   then_call_builder_.Input(input, kThenBranch);
diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index b9fceb6a31..466310d874 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -285,6 +285,14 @@ Status Node::input_node(int idx, const Node** const_n) const {
   return Status::OK();
 }
 
+Status Node::input_tensor(int idx, OutputTensor* t) const {
+  const Edge* e;
+  TF_RETURN_IF_ERROR(input_edge(idx, &e));
+  DCHECK(e != nullptr);
+  *t = OutputTensor(e->src(), e->src_output());
+  return Status::OK();
+}
+
 // InputTensor
 
 bool InputTensor::operator==(const InputTensor& other) const {
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 6a224ca4a2..90a25d4e3f 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -59,6 +59,7 @@ class EdgeSetTest;
 class Graph;
 class GraphDef;
 class Node;
+class OutputTensor;
 class VersionDef;
 class WhileContext;
 
@@ -189,6 +190,10 @@ class Node {
   Status input_node(int idx, const Node** n) const;
   Status input_node(int idx, Node** n) const;
 
+  // Returns into '*t' the idx-th input tensor of this node, represented as the
+  // output tensor of input_node(idx).
+  Status input_tensor(int idx, OutputTensor* t) const;
+
   WhileContext* while_ctx() const { return while_ctx_; }
   void set_while_ctx(WhileContext* while_ctx) {
     DCHECK(IsExit());
@@ -287,10 +292,10 @@ class Node {
 
 // Represents an input of a node, i.e., the `index`-th input to `node`.
 struct InputTensor {
-  const Node* node;
+  Node* node;
   int index;
 
-  InputTensor(const Node* n, int i) : node(n), index(i) {}
+  InputTensor(Node* n, int i) : node(n), index(i) {}
   InputTensor() : node(nullptr), index(0) {}
 
   // Returns true if this InputTensor is identical to 'other'. Nodes are
@@ -308,10 +313,10 @@ struct InputTensor {
 // that a single `OutputTensor` can correspond to multiple `Edge`s if the output
 // is consumed by multiple destination nodes.
 struct OutputTensor {
-  const Node* node;
+  Node* node;
   int index;
 
-  OutputTensor(const Node* n, int i) : node(n), index(i) {}
+  OutputTensor(Node* n, int i) : node(n), index(i) {}
   OutputTensor() : node(nullptr), index(0) {}
 
   // Returns true if this OutputTensor is identical to 'other'. Nodes are
diff --git a/tensorflow/core/graph/node_builder.cc b/tensorflow/core/graph/node_builder.cc
index adaee47935..a91e6dd057 100644
--- a/tensorflow/core/graph/node_builder.cc
+++ b/tensorflow/core/graph/node_builder.cc
@@ -29,6 +29,8 @@ NodeBuilder::NodeOut::NodeOut(Node* n, int32 i)  // NOLINT(runtime/explicit)
       index(i),
       dt(SafeGetOutput(node, i, &error)) {}
 
+NodeBuilder::NodeOut::NodeOut(OutputTensor t) : NodeOut(t.node, t.index) {}
+
 NodeBuilder::NodeOut::NodeOut(StringPiece n, int32 i, DataType t)
     : node(nullptr), error(false), name(n), index(i), dt(t) {}
 
diff --git a/tensorflow/core/graph/node_builder.h b/tensorflow/core/graph/node_builder.h
index 31fb590939..b1dc2ae92f 100644
--- a/tensorflow/core/graph/node_builder.h
+++ b/tensorflow/core/graph/node_builder.h
@@ -50,6 +50,7 @@ class NodeBuilder {
   struct NodeOut {
     // For referencing an existing Node.
     NodeOut(Node* n, int32 i = 0);
+    NodeOut(OutputTensor t);
 
     // For referencing Nodes not in the graph being built. It is
     // useful when preparing a graph for ExtendSession or creating a
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 19b067e449..5b78113f4f 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -750,6 +750,16 @@ class ControlFlowTest(test.TestCase):
       ]
       self.assertAllEqual(dense_gv, [0.0, 2.0])
 
+  def testCondPredicateTensor(self):
+    """Regression test for lowering predicate from non-first output of an op."""
+
+    @eager_function.defun
+    def foo():
+      return constant_op.constant("foo"), constant_op.constant(True)
+
+    r = control_flow_ops.cond(foo()[1], lambda: 1.0, lambda: 2.0)
+    self.assertEqual(self.evaluate(r), 1.0)
+
   # TODO(b/117945658): reenable
   @test_util.run_in_graph_and_eager_modes
   def DISABLED_testCondAutoControlDeps(self):
-- 
GitLab


From 2a481c3c2dd1526f2fdec528d15dee53d2e91a1c Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Tue, 13 Nov 2018 12:08:40 -0800
Subject: [PATCH 0174/1554] Remove outdated TODO

PiperOrigin-RevId: 221309785
---
 tensorflow/contrib/distribute/python/metrics_v1_test.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index c28ab41651..8289f7e212 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -72,8 +72,6 @@ def _regression_dataset_fn():
       "predictions": [1., .75, .25, 0.]}).repeat()
 
 
-# TODO(priyag): Add TPU Strategy to this once metrics aggregate correctly using
-# ReplicaLocalVariables on TPUs. Submit http://cl/208914352.
 def all_combinations():
   return combinations.combine(
       distribution=[combinations.default_strategy,
-- 
GitLab


From 7f94b0d43b5ef2d89890dec7b306cce1cc044623 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Tue, 13 Nov 2018 12:26:04 -0800
Subject: [PATCH 0175/1554] InputIterator to be used for distributing input in
 Distribution Strategy. Design doc:
 https://github.com/guptapriya/community/blob/replicator/rfcs/20181016-replicator.md

PiperOrigin-RevId: 221312642
---
 .../contrib/distribute/python/values.py       | 202 +++++++++++++++++-
 .../contrib/distribute/python/values_test.py  | 132 ++++++++++++
 2 files changed, 333 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index a162973535..ffb7b79839 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -23,10 +23,12 @@ from __future__ import print_function
 
 import collections
 import contextlib
+import operator
 import weakref
 import six
 
 from tensorflow.contrib.distribute.python import input_ops
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import multi_device_iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
@@ -1287,6 +1289,204 @@ class MultiWorkerDataset(object):
     return MultiWorkerDataIterator(iterators, self._worker_device_pairs)
 
 
+class InputIterator(object):
+  """An input iterator, intended to be passed to `DistributionStrategy.run`."""
+
+  def get_next(self):
+    """Returns the next inputs for all replicas."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  def initialize(self):
+    """Initialize the underlying input dataset, when applicable.
+
+    In eager mode, this will create a new iterator and return it.
+    In graph mode, this will initialize the same underlying iterator(s).
+
+    Users are required to call this if
+    - This iterator was returned from a call to `make_input_fn_iterator` with an
+      input function that returns a dataset.
+    - Or this iterator was returned from a call to `make_dataset_iterator`.
+
+    Returns:
+      A list of initialization ops to be executed.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+
+class InputFunctionIterator(InputIterator):
+  """Iterator created from input function."""
+
+  def __init__(self, input_fn, worker_device_pairs):
+    """Make an iterator for input provided via an input function.
+
+    Currently implements PER_WORKER mode, in which the `input_fn` is called
+    once on each worker.
+
+    TODO(priyag): Integrate with `InputContext` when it is submitted.
+    TODO(priyag): Add other replication modes.
+    TODO(priyag): Allow taking input function that returns a callable that
+    returns nest of tensors.
+
+    Args:
+      input_fn: Input function that returns a `tf.data.Dataset` object.
+      worker_device_pairs: A list of (worker, list of devices on that worker)
+        pairs.
+    """
+    if not worker_device_pairs:
+      raise ValueError("Cannot create iterator when no devices given.")
+
+    self._worker_device_pairs = worker_device_pairs
+    self._is_eager = context.executing_eagerly()
+    self._iterators = []
+
+    for worker, worker_devices in worker_device_pairs:
+      with ops.device(worker):
+        result = input_fn()
+        if not isinstance(result, dataset_ops.Dataset):
+          raise ValueError("input_fn must return a tf.data.Dataset.")
+        iterator = _DatasetIterator(result, worker, worker_devices)
+        self._iterators.append(iterator)
+
+  def get_next(self, name=None):
+    """Returns the next input from the iterator for all replicas."""
+    assert self._is_eager == context.executing_eagerly(), (
+        "Iterator should be created and used in same execution mode.")
+
+    index = {}
+    for i, (worker, worker_devices) in enumerate(self._worker_device_pairs):
+      if name is not None:
+        d = tf_device.DeviceSpec.from_string(worker)
+        new_name = "%s_%s_%d" % (name, d.job, d.task)
+      else:
+        new_name = None
+      with ops.device(worker):
+        data_per_worker = self._iterators[i].get_next(new_name)
+
+      # Ungroup these per-replica value so as to get a flat map from devices to
+      # values.
+      for d in worker_devices:
+        v = select_device(d, data_per_worker)
+        if d in index:
+          raise ValueError("Duplicated devices in worker_device_pairs: %r" % v)
+        index[d] = v
+
+    return regroup(index)
+
+  def initialize(self):
+    """Initialze underlying iterators.
+
+    Returns:
+      A list of any initializer ops that should be run.
+    """
+    assert self._is_eager == context.executing_eagerly(), (
+        "Iterator should be created and used in same execution mode.")
+
+    init_ops = []
+    for it in self._iterators:
+      init_ops.extend(it.initialize())
+    return init_ops
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  def _output_classes(self):
+    return self._iterators[0].output_classes
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  def _output_shapes(self):
+    return self._iterators[0].output_shapes
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  def _output_types(self):
+    return self._iterators[0].output_types
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  def _get_iterator(self, worker):
+    for i, (w, _) in enumerate(self._worker_device_pairs):
+      if worker == w:
+        return self._iterators[i]
+    return None
+
+
+class _DatasetIterator(object):
+  """Iterator for a single `tf.data.Dataset`."""
+
+  def __init__(self, dataset, worker, devices):
+    """Create iterator for the given dataset.
+
+    `MultiDeviceIterator` is used to prefetch input to the devices on the
+    given worker. `MultiDeviceIterator` doesn't work in eager mode yet.
+
+    Args:
+      dataset: A `tf.data.Dataset` instance.
+      worker: Worker on which ops should be created.
+      devices: Distribute data from `dataset` to these devices.
+    """
+    self._dataset = dataset
+    self._worker = worker
+    self._devices = devices
+    self._is_eager = context.executing_eagerly()
+    self._make_iterator()
+
+  def _make_iterator(self):
+    """Make appropriate iterator on the dataset."""
+    with ops.device(self._worker):
+      if self._is_eager:
+        # TODO(rohanj): Enable prefetching in eager mode.
+        # TODO(priyag): Measure the performance of this approach vs calling
+        # get_next on the original dataset N times.
+        dataset = self._dataset.batch(len(self._devices), drop_remainder=True)
+        iterator = dataset.make_one_shot_iterator()
+      else:
+        iterator = multi_device_iterator_ops.MultiDeviceIterator(
+            self._dataset, self._devices)
+    self._iterator = iterator
+
+  def get_next(self, name=None):
+    """Get next element from the underlying iterator."""
+    with ops.device(self._worker):
+      if self._is_eager:
+        # Batched dataset case.
+        batch = self._iterator.get_next(name=name)
+        index = {}
+        for i, d in enumerate(self._devices):
+          index[d] = nest.map_structure(operator.itemgetter(i), batch)
+          with ops.device(d):
+            index[d] = nest.map_structure(array_ops.identity, index[d])
+      else:
+        # MultiDeviceIterator case.
+        data_list = self._iterator.get_next()
+        index = dict(zip(self._devices, data_list))
+
+      return regroup(index)
+
+  def initialize(self):
+    """Initialze underlying iterator.
+
+    In eager execution, this simply recreates the underlying iterator.
+    In graph execution, it returns the initializer ops for the underlying
+    iterator.
+
+    Returns:
+      A list of any initializer ops that should be run.
+    """
+    if self._is_eager:
+      self._make_iterator()
+      return []
+    else:
+      return [self._iterator.initializer]
+
+  @property
+  def output_classes(self):
+    return self._iterator.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._iterator.output_shapes
+
+  @property
+  def output_types(self):
+    return self._iterator.output_types
+
+
 class MultiStepContext(object):
   """A context object that can be used to capture things when running steps.
 
@@ -1299,7 +1499,7 @@ class MultiStepContext(object):
   """
 
   def __init__(self):
-    """Initializes an output context.
+    """Initialize an output context.
 
     Returns:
       A context object.
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 268393ee80..b4b56eb4e4 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -568,6 +568,138 @@ class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase):
         multi_worker_iterator.get_next()
 
 
+class InputFunctionIteratorTestBase(test.TestCase):
+
+  def _test_iterator(self, input_fn, worker_device_pairs, expected_values,
+                     sess=None):
+    devices = nest.flatten([ds for _, ds in worker_device_pairs])
+    iterator = values.InputFunctionIterator(input_fn, worker_device_pairs)
+
+    evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
+
+    evaluate(iterator.initialize())
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      computed_value = evaluate(
+          [values.select_device(d, next_element) for d in devices])
+      self.assertEqual(expected_value, computed_value)
+
+    with self.assertRaises(errors.OutOfRangeError):
+      next_element = iterator.get_next()
+      evaluate([values.select_device(d, next_element) for d in devices])
+
+    # After re-initializing the iterator, should be able to iterate again.
+    evaluate(iterator.initialize())
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      computed_value = evaluate(
+          [values.select_device(d, next_element) for d in devices])
+      self.assertEqual(expected_value, computed_value)
+
+
+class InputFunctionIteratorSingleWorkerTest(InputFunctionIteratorTestBase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testOneDeviceCPU(self):
+    worker_device_pairs = [("", ["/device:CPU:0"])]
+    input_fn = lambda: dataset_ops.Dataset.range(10)
+
+    expected_values = [[i] for i in range(10)]
+
+    self._test_iterator(input_fn, worker_device_pairs, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testTwoDevicesOneGPUOneCPU(self):
+    if context.num_gpus() < 1:
+      self.skipTest("A GPU is not available for this test.")
+
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    input_fn = lambda: dataset_ops.Dataset.range(10)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+
+    self._test_iterator(input_fn, worker_device_pairs, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testTupleDataset(self):
+    if context.num_gpus() < 1:
+      self.skipTest("A GPU is not available for this test.")
+
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    def input_fn():
+      dataset1 = dataset_ops.Dataset.range(10)
+      dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
+      return dataset_ops.Dataset.zip((dataset1, dataset2))
+
+    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
+
+    self._test_iterator(input_fn, worker_device_pairs, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testUnevenDatasetBatches(self):
+    if context.num_gpus() < 1:
+      self.skipTest("A GPU is not available for this test.")
+
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    input_fn = lambda: dataset_ops.Dataset.range(11)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+    self._test_iterator(input_fn, worker_device_pairs, expected_values)
+
+
+class InputFunctionIteratorMultiWorkerTest(
+    multi_worker_test_base.MultiWorkerTestBase,
+    InputFunctionIteratorTestBase):
+
+  def _cpu_devices(self):
+    return [
+        ("/job:worker/replica:0/task:0",
+         ["/job:worker/replica:0/task:0/device:CPU:0"]),
+        ("/job:worker/replica:0/task:1",
+         ["/job:worker/replica:0/task:1/device:CPU:0"])]
+
+  def _cpu_and_one_gpu_devices(self):
+    return [
+        ("/job:worker/replica:0/task:0", [
+            "/job:worker/replica:0/task:0/device:GPU:0",
+            "/job:worker/replica:0/task:0/device:CPU:0"
+        ]),
+        ("/job:worker/replica:0/task:1", [
+            "/job:worker/replica:0/task:1/device:GPU:0",
+            "/job:worker/replica:0/task:1/device:CPU:0"
+        ])
+    ]
+
+  def testOneDevicePerWorker(self):
+    worker_devices = self._cpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      input_fn = lambda: dataset_ops.Dataset.range(4)
+      self._test_iterator(input_fn, worker_devices,
+                          [[0, 0], [1, 1], [2, 2], [3, 3]], sess)
+
+  def testTwoDevicesPerWorker(self):
+    if context.num_gpus() < 1:
+      self.skipTest("A GPU is not available for this test.")
+    worker_devices = self._cpu_and_one_gpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      input_fn = lambda: dataset_ops.Dataset.range(4)
+      self._test_iterator(input_fn, worker_devices,
+                          [[0, 1, 0, 1], [2, 3, 2, 3]], sess)
+
+  def testTupleDataset(self):
+    worker_devices = self._cpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      def input_fn():
+        dataset1 = dataset_ops.Dataset.range(4)
+        dataset2 = dataset_ops.Dataset.range(4).map(lambda x: x**2)
+        return dataset_ops.Dataset.zip((dataset1, dataset2))
+
+      expected_values = [[(i, i**2), (i, i**2)] for i in range(0, 4)]
+      self._test_iterator(input_fn, worker_devices, expected_values, sess)
+
+
 class MirroredVariableTest(test.TestCase):
 
   config = config_pb2.ConfigProto()
-- 
GitLab


From b3cbc96f85f6232f191227bec9743a7ec4e84c33 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Tue, 13 Nov 2018 12:34:58 -0800
Subject: [PATCH 0176/1554] Add extensions to the SlurmClusterResolver in
 preparation for integration with Distribute Coordinator

PiperOrigin-RevId: 221314065
---
 .../python/training/slurm_cluster_resolver.py | 62 ++++++++++++++-----
 .../training/slurm_cluster_resolver_test.py   | 25 ++++++++
 2 files changed, 71 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py
index dabe2fe1d3..f590ecead9 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py
@@ -53,7 +53,8 @@ class SlurmClusterResolver(ClusterResolver):
                gpus_per_node=1,
                gpus_per_task=1,
                tasks_per_node=None,
-               auto_set_gpu=True):
+               auto_set_gpu=True,
+               rpc_layer='grpc'):
     """Creates a new SlurmClusterResolver object.
 
     This takes in parameters and creates a SlurmClusterResolver object. It uses
@@ -74,6 +75,8 @@ class SlurmClusterResolver(ClusterResolver):
       auto_set_gpu: Set the visible CUDA devices automatically while resolving
         the cluster by setting CUDA_VISIBLE_DEVICES environment variable.
         Defaults to True.
+      rpc_layer: (Optional) The protocol TensorFlow uses to communicate between
+        nodes. Defaults to 'grpc'.
 
     Returns:
       A ClusterResolver object which can be used with distributed TensorFlow.
@@ -107,8 +110,9 @@ class SlurmClusterResolver(ClusterResolver):
     self._gpus_per_task = gpus_per_task
 
     self._auto_set_gpu = auto_set_gpu
-    self._job_name = None
-    self._task_index = None
+    self.task_type = None
+    self.task_index = None
+    self.rpc_layer = rpc_layer
 
     self._gpu_allocation = []
     self._cluster_allocation = {}
@@ -157,17 +161,15 @@ class SlurmClusterResolver(ClusterResolver):
     cluster_rank_offset_start = 0
     cluster_rank_offset_end = 0
 
-    for job_name, num_tasks in self._jobs.items():
+    for task_type, num_tasks in self._jobs.items():
       cluster_rank_offset_end = cluster_rank_offset_start + num_tasks
 
-      self._cluster_allocation[job_name] = \
-        task_list[cluster_rank_offset_start:cluster_rank_offset_end]
+      self._cluster_allocation[task_type] = (
+          task_list[cluster_rank_offset_start:cluster_rank_offset_end])
 
-      if self._rank >= cluster_rank_offset_start and \
-          self._rank < cluster_rank_offset_end:
-
-        self._job_name = job_name
-        self._task_index = self._rank - cluster_rank_offset_start
+      if cluster_rank_offset_start <= self._rank < cluster_rank_offset_end:
+        self.task_type = task_type
+        self.task_index = self._rank - cluster_rank_offset_start
 
       cluster_rank_offset_start = cluster_rank_offset_end
 
@@ -188,9 +190,37 @@ class SlurmClusterResolver(ClusterResolver):
       A string specifying job name the process belongs to and an integner
         specifying the task index the process belongs to in that job.
     """
-    return self._job_name, self._task_index
+    return self.task_type, self.task_index
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master string for connecting to a TensorFlow master.
+
+    Args:
+      task_type: (Optional) Overrides the default auto-selected task type.
+      task_index: (Optional) Overrides the default auto-slected task index.
+      rpc_layer: (Optional) Overrides the default RPC protocol TensorFlow uses
+        to communicate across nodes.
+
+    Returns:
+      A connection string for connecting to a TensorFlow master.
+    """
+    task_type = task_type if task_type is not None else self.task_type
+    task_index = task_index if task_index is not None else self.task_index
+    rpc_layer = rpc_layer or self.rpc_layer
+    master = self.cluster_spec().task_address(task_type, task_index)
+
+    return '%s://%s' % (rpc_layer, master) if rpc_layer else master
+
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in.
+
+    For users in the Slurm environment, the environment property is always an
+    empty string, and Google users will not use this ClusterResolver for running
+    on internal systems.
+    """
+    return ''
 
-  def master(self, task_type=None, task_index=None):
-    if task_type and task_index:
-      return self.cluster_spec().task_address(task_type, task_index)
-    return self._cluster_allocation[str(self._job_name)][self._task_index]
+  def num_accelerators_per_worker(self, session_config=None):
+    del session_config  # Unused, since this is set in __init__ manually.
+    return self._gpus_per_node
diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py
index 9aa7df745e..7c76e133fe 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py
@@ -67,6 +67,31 @@ class SlurmClusterResolverTest(test.TestCase):
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
 
+  @mock.patch.dict(os.environ, {'SLURM_PROCID': '0', 'SLURM_NTASKS': '3'})
+  @mock.patch.object(SlurmClusterResolver, '_resolve_hostnames',
+                     mock_resolve_hostnames_output)
+  def testSimpleMasterRetrieval(self):
+    slurm_cluster_resolver = SlurmClusterResolver(
+        jobs={
+            'ps': 1,
+            'worker': 2
+        },
+        port_base=8888,
+        tasks_per_node=1,
+        gpus_per_node=1,
+        gpus_per_task=1,
+        auto_set_gpu=False)
+
+    slurm_cluster_resolver.task_type = 'worker'
+    slurm_cluster_resolver.task_index = 1
+    self.assertEqual(slurm_cluster_resolver.master(), 'grpc://t02n43:8888')
+
+    slurm_cluster_resolver.rpc_layer = 'ab'
+    self.assertEqual(slurm_cluster_resolver.master('ps', 0), 'ab://t02n13:8888')
+    self.assertEqual(
+        slurm_cluster_resolver.master('ps', 0, rpc_layer='test'),
+        'test://t02n13:8888')
+
   @mock.patch.dict(os.environ, {
       'SLURM_PROCID': '0',
       'SLURM_NTASKS': '3',
-- 
GitLab


From 43ca9103278400b4f2fd8569a9a278fb9c927af0 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Tue, 13 Nov 2018 12:35:36 -0800
Subject: [PATCH 0177/1554] Implement additional features in K8s Cluster
 Resolvers for Distributed Coordinators

PiperOrigin-RevId: 221314173
---
 .../python/training/cluster_resolver.py       |  6 +-
 .../training/kubernetes_cluster_resolver.py   | 57 ++++++++++++++++---
 .../kubernetes_cluster_resolver_test.py       | 16 ++++++
 3 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
index 40b1e667ee..3583501a45 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
@@ -25,7 +25,7 @@ import six
 from tensorflow.python.training.server_lib import ClusterSpec
 
 
-def _format_master_url(master, rpc_layer=None):
+def format_master_url(master, rpc_layer=None):
   if rpc_layer:
     return '%s://%s' % (rpc_layer, master)
   else:
@@ -128,7 +128,7 @@ class SimpleClusterResolver(ClusterResolver):
     else:
       master = self._master
 
-    return _format_master_url(master, rpc_layer or self._rpc_layer)
+    return format_master_url(master, rpc_layer=rpc_layer or self._rpc_layer)
 
   @property
   def task_type(self):
@@ -307,7 +307,7 @@ class UnionClusterResolver(ClusterResolver):
     """
     if task_type is not None and task_index is not None:
       master = self.cluster_spec().task_address(task_type, task_index)
-      return _format_master_url(master, rpc_layer or self._rpc_layer)
+      return format_master_url(master, rpc_layer or self._rpc_layer)
 
     return self._cluster_resolvers[0].master(rpc_layer=rpc_layer)
 
diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
index ddae64839f..a705b99374 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
@@ -19,6 +19,8 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
+from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import format_master_url
+from tensorflow.python.client import device_lib
 from tensorflow.python.training import server_lib
 
 _KUBERNETES_API_CLIENT_INSTALLED = True
@@ -41,6 +43,7 @@ class KubernetesClusterResolver(ClusterResolver):
   def __init__(self,
                job_to_label_mapping=None,
                tf_server_port=8470,
+               rpc_layer='grpc',
                override_client=None):
     """Initializes a new KubernetesClusterResolver.
 
@@ -58,6 +61,8 @@ class KubernetesClusterResolver(ClusterResolver):
          'ps': ['job-name=ps-1', 'job-name=ps-2']}
         ```
       tf_server_port: The port the TensorFlow server is listening on.
+      rpc_layer: (Optional) The RPC layer TensorFlow should use to communicate
+        between tasks in Kubernetes. Defaults to 'grpc'.
       override_client: The Kubernetes client (usually automatically retrieved
         using `from kubernetes import client as k8sclient`). If you pass this
         in, you are responsible for setting Kubernetes credentials manually.
@@ -65,6 +70,7 @@ class KubernetesClusterResolver(ClusterResolver):
     Raises:
       ImportError: If the Kubernetes Python client is not installed and no
         `override_client` is passed in.
+      RuntimeError: If autoresolve_task is not a boolean or a callable.
     """
     if _KUBERNETES_API_CLIENT_INSTALLED:
       k8sconfig.load_kube_config()
@@ -82,16 +88,37 @@ class KubernetesClusterResolver(ClusterResolver):
     self._tf_server_port = tf_server_port
     self._override_client = override_client
 
-  def master(self):
-    # TODO(frankchn): Figure out a standard way to pass in the current task type
-    # and task id via Kubernetes.
-    pass
+    self.task_type = None
+    self.task_index = None
+    self.rpc_layer = rpc_layer
 
-  def get_master(self):
-    return self.master()
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master address to use when creating a session.
 
-  def get_job_name(self):
-    return self._job_name
+    You must have set the task_type and task_index object properties before
+    calling this function, or pass in the `task_type` and `task_index`
+    parameters when using this function. If you do both, the function parameters
+    will override the object properties.
+
+    Args:
+      task_type: (Optional) The type of the TensorFlow task of the master.
+      task_index: (Optional) The index of the TensorFlow task of the master.
+      rpc_layer: (Optional) The RPC protocol for the given cluster.
+
+    Returns:
+      The name or URL of the session master.
+    """
+    if task_type is not None and task_index is not None:
+      return format_master_url(
+          self.cluster_spec().task_address(task_type, task_index),
+          rpc_layer or self.rpc_layer)
+
+    if self._task_type is not None and self._task_index is not None:
+      return format_master_url(
+          self.cluster_spec().task_address(self._task_type, self._task_index),
+          rpc_layer or self.rpc_layer)
+
+    return ''
 
   def cluster_spec(self):
     """Returns a ClusterSpec object based on the latest info from Kubernetes.
@@ -130,3 +157,17 @@ class KubernetesClusterResolver(ClusterResolver):
       cluster_map[tf_job] = all_pods
 
     return server_lib.ClusterSpec(cluster_map)
+
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in.
+
+    For users in the Cloud environment, the environment property is always an
+    empty string, and Google users will not use this ClusterResolver for running
+    on internal systems.
+    """
+    return ''
+
+  def num_accelerators_per_worker(self, session_config=None):
+    local_devices = device_lib.list_local_devices(session_config)
+    return len([d for d in local_devices if d.device_type == 'GPU'])
diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
index fbb26e803d..9ab1fcb309 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
@@ -109,6 +109,22 @@ class KubernetesClusterResolverTest(test.TestCase):
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto))
 
+  def testGetMasterWithOverrideParameters(self):
+    ret = _create_pod_list(
+        ('worker-0', 'Running', '10.1.2.3'),
+        ('worker-1', 'Running', '10.1.2.4'),
+        ('worker-2', 'Running', '10.1.2.5'))
+
+    cluster_resolver = KubernetesClusterResolver(
+        override_client=_mock_kubernetes_client(
+            {'job-name=tensorflow': ret}))
+    cluster_resolver.task_type = 'blah'
+    cluster_resolver.task_index = 1
+    self.assertEqual(cluster_resolver.task_type, 'blah')
+    self.assertEqual(cluster_resolver.task_index, 1)
+    self.assertEqual(cluster_resolver.master('worker', 2),
+                     'grpc://10.1.2.5:8470')
+
   def testNonRunningPod(self):
     ret = _create_pod_list(('tensorflow-abc123', 'Failed', '10.1.2.3'),)
 
-- 
GitLab


From c456dbf48500c7e04e9f2e6570f2ae4d7b8eeb17 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 13 Nov 2018 12:41:03 -0800
Subject: [PATCH 0178/1554] Lower cond_v2 gradient computation to original
 control flow implementation.

Prior to this change, we would only lower the forward pass (oops).
Also creates identities for while_v2 grad outputs so we get pruning.

PiperOrigin-RevId: 221315130
---
 tensorflow/python/BUILD                       |  6 ++--
 .../python/kernel_tests/while_v2_test.py      |  2 +-
 tensorflow/python/ops/cond_v2.py              | 24 ++++----------
 tensorflow/python/ops/control_flow_util_v2.py | 24 ++++++++++++++
 tensorflow/python/ops/while_v2.py             | 31 +++----------------
 5 files changed, 39 insertions(+), 48 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 63352b82f1..9bd6ddd083 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2093,7 +2093,9 @@ py_library(
     srcs = ["ops/control_flow_util_v2.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "framework_ops",
+        ":control_flow_util",
+        ":framework_ops",
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:function",
     ],
@@ -2118,7 +2120,6 @@ py_library(
         ":graph_to_function_def",
         ":pywrap_tensorflow",
         ":util",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:function",
     ],
 )
@@ -2145,7 +2146,6 @@ py_library(
         ":tensor_shape",
         ":tensor_util",
         ":util",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:function",
     ],
 )
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index dc1bcb78b8..3314e3849b 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -262,7 +262,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
 
     # Gradient pass.
     grad = gradients_impl.gradients(ret[1], y)
-    grad_while_op = grad[0].op
+    grad_while_op = grad[0].op.inputs[0].op
     # Get the TensorList output of gradient While op containing the accumulated
     # values of grad_y.
     # grad_while_op.inputs:
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 998c3e08f6..b50e4a5702 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -25,12 +25,10 @@ from __future__ import print_function
 
 import collections
 
-from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import function_def_to_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gradients_impl
@@ -108,23 +106,8 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
                                          false_graph.outputs),
         name=scope)
 
-    # Set the flag to enable lowering on the `if` op if necessary
-    # Lowering allows cond_v2 to avoid some of the limitations of Functions,
-    # allowing users to specify devices & colocation inside of cond_v2 branches,
-    # and enabling non-strict evaluation & partial pruning of cond_v2 branches.
-    # This brings cond_v2 closer to feature parity with tf.cond.
-    #
-    # However, we do not lower `If` in the XLA context because it is easier for
-    # XLA to apply its own optimizations when dealing with un-lowered `If`
-    # operators than with lowered switch/merge control flow.
-    #
     # TODO(b/110167197) this approach requires cond_v2 to have at least 1 output
-    if_op = tensors[0].op
-    if not control_flow_util.IsInXLAContext(if_op):
-      # pylint: disable=protected-access
-      if_op._set_attr("_lower_using_switch_merge",
-                      attr_value_pb2.AttrValue(b=True))
-      # pylint: enable=protected-access
+    util.maybe_set_lowering_attr(tensors[0].op)
 
     # Return identities for each output of the If op, rather than the output of
     # the If op directly. This makes pruning work if the output of cond() is
@@ -199,6 +182,11 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
       output_shapes=_get_output_shapes(true_grad_graph.outputs,
                                        false_grad_graph.outputs))
 
+  util.maybe_set_lowering_attr(tensors[0].op)
+
+  # See comment in cond_v2.
+  tensors = [array_ops.identity(t) for t in tensors]
+
   # The predicate has no gradient.
   return [None] + tensors[:num_grad_outputs]
 
diff --git a/tensorflow/python/ops/control_flow_util_v2.py b/tensorflow/python/ops/control_flow_util_v2.py
index cab1d7b02e..2eec423991 100644
--- a/tensorflow/python/ops/control_flow_util_v2.py
+++ b/tensorflow/python/ops/control_flow_util_v2.py
@@ -19,10 +19,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework.func_graph import FuncGraph
+from tensorflow.python.ops import control_flow_util
 
 
 class CondBranchFuncGraph(FuncGraph):
@@ -90,3 +92,25 @@ def unique_fn_name(scope, name):
 
 def unique_grad_fn_name(forward_name):
   return "%s_grad_%s" % (forward_name, ops.uid())
+
+
+def maybe_set_lowering_attr(op):
+  """Sets the flag to enable lowering on `op` if necessary.
+
+  Lowering allows cond_v2 and while_v2 to avoid some of the limitations of
+  Functions, allowing users to specify devices & colocation inside of cond_v2
+  and while_v2 input functions, and enabling non-strict evaluation & partial
+  pruning. This brings v2 control flow closer to feature parity with v1 control
+  flow.
+
+  However, we do not lower `If` and `While` ops in the XLA context because it is
+  easier for XLA to apply its own optimizations when dealing with un-lowered
+  control flow operators than with low-level control flow primitives.
+
+  Args:
+    op: An `If` or `While` Operation.
+  """
+  if not control_flow_util.IsInXLAContext(op):
+    # pylint: disable=protected-access
+    op._set_attr("_lower_using_switch_merge", attr_value_pb2.AttrValue(b=True))
+    # pylint: enable=protected-access
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 7b0f0ed4fc..c35431154e 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -23,7 +23,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import func_graph as func_graph_module
@@ -245,7 +244,7 @@ def while_loop(cond,
         name=scope)
 
     _copy_handle_data(body_graph.outputs, outputs)
-    _maybe_set_lowering_attr(outputs[0].op)
+    util.maybe_set_lowering_attr(outputs[0].op)
     _maybe_set_maximum_iterations_attr(outputs[0].op, maximum_iterations)
 
     # Return identities for each output of the While op, rather than the output
@@ -343,9 +342,12 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
       name="%s_grad" % op.name)
 
   _copy_handle_data(body_grad_graph.outputs, outputs)
-  _maybe_set_lowering_attr(outputs[0].op)
+  util.maybe_set_lowering_attr(outputs[0].op)
   _maybe_set_maximum_iterations_attr(outputs[0].op, maximum_iterations)
 
+  # See comment in while_loop.
+  outputs = [array_ops.identity(t) for t in outputs]
+
   # Set None as the output gradient for tensors with None input gradient
   # e.g. TensorArray handles.
   # outputs[0] is the loop counter.
@@ -792,29 +794,6 @@ def _copy_handle_data(src_tensors, tgt_tensors):
     custom_gradient.copy_handle_data(src_t, tgt_t)
 
 
-# TODO(srbs): Move to common utils for cond_v2 and while_v2.
-def _maybe_set_lowering_attr(op):
-  """Sets the flag to enable lowering on the `While` op if necessary.
-
-  Lowering allows while_v2 to avoid some of the limitations of Functions,
-  allowing users to specify devices & colocation inside of while_v2
-  branches, and enabling non-strict evaluation & partial pruning of while_v2
-  branches. This brings while_v2 closer to feature parity with
-  tf.while_loop.
-
-  However, we do not lower `While` in the XLA context because it is easier
-  for XLA to apply its own optimizations when dealing with un-lowered
-  `While` operators than with low-level control flow primitives.
-
-  Args:
-    op: The While op.
-  """
-  if not control_flow_util.IsInXLAContext(op):
-    # pylint: disable=protected-access
-    op._set_attr("_lower_using_switch_merge", attr_value_pb2.AttrValue(b=True))
-    # pylint: enable=protected-access
-
-
 def _maybe_set_maximum_iterations_attr(op, maximum_iterations):
   if control_flow_util.IsInXLAContext(op):
     # Store the maximum_iterations to use in the gradient pass.
-- 
GitLab


From 545aa1f94184edfe471120d4929c1fb02290f85a Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Tue, 13 Nov 2018 12:41:31 -0800
Subject: [PATCH 0179/1554] Add Distribute Coordinator compatible methods to
 TFCONFIGClusterResolver.

PiperOrigin-RevId: 221315210
---
 .../training/tfconfig_cluster_resolver.py     | 119 +++++++++++++++---
 .../tfconfig_cluster_resolver_test.py         |  52 ++++++++
 2 files changed, 151 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py
index 7bbd189d03..95aad0de13 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py
@@ -27,13 +27,98 @@ from tensorflow.python.training.server_lib import ClusterSpec
 
 _TF_CONFIG_ENV = 'TF_CONFIG'
 _SESSION_MASTER_KEY = 'session_master'
+_RPC_LAYER_KEY = 'rpc_layer'
+_TASK_KEY = 'task'
+
+
+def format_master_url(master, rpc_layer=None):
+  if rpc_layer:
+    return '%s://%s' % (rpc_layer, master)
+  else:
+    return master
+
+
+def _load_tf_config():
+  return json.loads(os.environ.get(_TF_CONFIG_ENV, '{}'))
+
+
+def _get_value_in_tfconfig(key, default=None):
+  tf_config = _load_tf_config()
+  return tf_config[key] if key in tf_config else default
 
 
 class TFConfigClusterResolver(ClusterResolver):
   """Implementation of a ClusterResolver which reads the TF_CONFIG EnvVar."""
 
-  def _load_tf_config(self):
-    return json.loads(os.environ.get(_TF_CONFIG_ENV, '{}'))
+  def __init__(self,
+               task_type=None,
+               task_index=None,
+               rpc_layer=None,
+               environment=None,
+               num_accelerators_per_worker=0):
+    """Creates a new TFConfigClusterResolver.
+
+    Args:
+      task_type: (String, optional) Overrides the task type specified in the
+        TF_CONFIG environment variable.
+      task_index: (Integer, optional) Overrides the task index specified in the
+        TF_CONFIG environment variable.
+      rpc_layer: (String, optional) Overrides the rpc layer TensorFlow uses.
+      environment: (String, optional) Overrides the environment TensorFlow
+        operates in.
+      num_accelerators_per_worker: (Integer, optional) Specifies the number of
+        accelerators (e.g. GPUs, TPUs, others) that each node has.
+    """
+
+    self._task_type = task_type
+    self._task_index = task_index
+    self._rpc_layer = rpc_layer
+    self._environment = environment
+    self._num_accelerators_per_worker = num_accelerators_per_worker
+
+  @property
+  def task_type(self):
+    if self._task_type is None:
+      task_info = _get_value_in_tfconfig(_TASK_KEY, {})
+      return task_info['type'] if 'type' in task_info else None
+    else:
+      return self._task_type
+
+  @property
+  def task_index(self):
+    if self._task_type is None:
+      task_info = _get_value_in_tfconfig(_TASK_KEY, {})
+      return task_info['index'] if 'index' in task_info else None
+    else:
+      return self._task_index
+
+  @task_type.setter
+  def task_type(self, task_type):
+    self._task_type = task_type
+
+  @task_index.setter
+  def task_index(self, task_index):
+    self._task_index = task_index
+
+  @property
+  def environment(self):
+    return self._environment
+
+  @property
+  def rpc_layer(self):
+    if self._rpc_layer is None:
+      return _get_value_in_tfconfig(_RPC_LAYER_KEY)
+    else:
+      return self._rpc_layer
+
+  @rpc_layer.setter
+  def rpc_layer(self, rpc_layer):
+    self._rpc_layer = rpc_layer
+
+  def num_accelerators_per_worker(self, session_config=None):
+    # TODO(frankchn): Connect to server (w/ session_config) in the future.
+    del session_config  # Unused, we do not connect to another server here.
+    return self._num_accelerators_per_worker
 
   def cluster_spec(self):
     """Returns a ClusterSpec based on the TF_CONFIG environment variable.
@@ -41,12 +126,12 @@ class TFConfigClusterResolver(ClusterResolver):
     Returns:
       A ClusterSpec with information from the TF_CONFIG environment variable.
     """
-    tf_config = self._load_tf_config()
+    tf_config = _load_tf_config()
     if 'cluster' not in tf_config:
       return ClusterSpec({})
     return ClusterSpec(tf_config['cluster'])
 
-  def master(self, task_type=None, task_index=0):
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
     """Returns the master address to use when creating a TensorFlow session.
 
     Args:
@@ -54,6 +139,8 @@ class TFConfigClusterResolver(ClusterResolver):
         master.
       task_index: (Integer, optional) Overrides and sets the task id of the
         master.
+      rpc_layer: (String, optional) Overrides and sets the protocol over which
+        TensorFlow nodes communicate with each other.
 
     Returns:
       The address of the master.
@@ -64,14 +151,9 @@ class TFConfigClusterResolver(ClusterResolver):
     """
 
     # If `session_master` is set, just use that.
-    tf_config = self._load_tf_config()
-    if _SESSION_MASTER_KEY in tf_config:
-      return tf_config[_SESSION_MASTER_KEY]
-
-    if 'rpc_layer' in tf_config:
-      rpclayer = '%s://' % tf_config['rpc_layer']
-    else:
-      rpclayer = ''
+    session_master = _get_value_in_tfconfig(_SESSION_MASTER_KEY)
+    if session_master is not None:
+      return session_master
 
     # Return an empty string if we are the only job in the ClusterSpec.
     cluster_spec = self.cluster_spec()
@@ -82,11 +164,8 @@ class TFConfigClusterResolver(ClusterResolver):
 
     # We try to auto-detect the task type and id, but uses the user-supplied one
     # where available
-    if not task_type:
-      if 'task' not in tf_config:
-        raise RuntimeError('You must either specify a `task_type`, or your '
-                           'TF_CONFIG must contain a `task` section.')
-      task_type = tf_config['task']['type']
-      task_index = tf_config['task']['index']
-
-    return rpclayer + cluster_spec.task_address(task_type, task_index)
+    task_type = task_type if task_type is not None else self.task_type
+    task_index = task_index if task_index is not None else self.task_index
+
+    return format_master_url(cluster_spec.task_address(task_type, task_index),
+                             self.rpc_layer)
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py
index 468161d2aa..3db6d5447f 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py
@@ -133,6 +133,58 @@ class TFConfigClusterResolverTest(test.TestCase):
     cluster_resolver = TFConfigClusterResolver()
     self.assertEqual('grpc://ps0:2222', cluster_resolver.master())
 
+  def testTaskTypeIndexRpcRead(self):
+    os.environ['TF_CONFIG'] = """
+    {
+      "cluster": {
+        "ps": ["ps0:2222", "ps1:2222"],
+        "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
+      },
+      "rpc_layer": "grpc",
+      "task": {
+        "type": "ps",
+        "index": 0
+      }
+    }
+    """
+
+    cluster_resolver = TFConfigClusterResolver()
+    self.assertEqual('ps', cluster_resolver.task_type)
+    self.assertEqual(0, cluster_resolver.task_index)
+    self.assertEqual('grpc', cluster_resolver.rpc_layer)
+
+  def testParameterOverrides(self):
+    os.environ['TF_CONFIG'] = """
+    {
+      "cluster": {
+        "ps": ["ps0:2222", "ps1:2222"],
+        "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
+      },
+      "rpc_layer": "grpc",
+      "task": {
+        "type": "ps",
+        "index": 1
+      }
+    }
+    """
+
+    cluster_resolver = TFConfigClusterResolver(task_type='ps', task_index=0,
+                                               num_accelerators_per_worker=8)
+
+    self.assertEqual('grpc://ps0:2222', cluster_resolver.master())
+    self.assertEqual('ps', cluster_resolver.task_type)
+    self.assertEqual(0, cluster_resolver.task_index)
+    self.assertEqual(8, cluster_resolver.num_accelerators_per_worker())
+
+    cluster_resolver.task_type = 'worker'
+    cluster_resolver.task_index = 1
+    cluster_resolver.rpc_layer = 'test'
+
+    self.assertEqual('test://worker1:2222', cluster_resolver.master())
+    self.assertEqual('worker', cluster_resolver.task_type)
+    self.assertEqual(1, cluster_resolver.task_index)
+    self.assertEqual('test', cluster_resolver.rpc_layer)
+
   def testZeroItemsInClusterSpecMasterRead(self):
     os.environ['TF_CONFIG'] = """
     {}
-- 
GitLab


From 0cfa4c157d317ccb4ccd1f6c85773f973d628695 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 13 Nov 2018 12:48:24 -0800
Subject: [PATCH 0180/1554] Optimize the allocation of a host-memory scalar to
 use a single heap allocation.

Previously, the public Tensor constructors would always allocate two objects on the heap:
1. A reference-counted `TensorBuffer` object, which may be shared between (shallow) copies of the created `Tensor`.
2. The data itself, aligned to a 64-byte boundary.

This change packs the aligned data and the `TensorBuffer` subclass into a single object, which can be aligned and freed with a single call to the allocator. This improves performance when constructing a large number of scalar tensors, by spending less time in the allocator.

The improves the performance of the `Dataset.range()` benchmark on my workstation:

* Average time per element (without modeling): 80.076499 nanoseconds (previously 112.806258 nanoseconds)
* Average time per element (with modeling): 254.111791 nanoseconds (previously 287.067080 nanoseconds)

PiperOrigin-RevId: 221316413
---
 tensorflow/core/framework/tensor.h       | 69 +++++++++++++-----------
 tensorflow/core/framework/tensor_test.cc | 11 ++--
 2 files changed, 44 insertions(+), 36 deletions(-)

diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 0d58ab3875..eff1dc3736 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -118,7 +119,7 @@ class Tensor {
 
   class HostScalarTensorBufferBase;
   template <typename T>
-  class HostScalarTensorBuffer;
+  struct ValueAndTensorBuffer;
 
   // Creates a tensor with the given scalar `value` in CPU memory.
   template <typename T>
@@ -875,38 +876,42 @@ class Tensor::HostScalarTensorBufferBase : public TensorBuffer {
   void FillAllocationDescription(AllocationDescription* proto) const final;
 };
 
-// `Tensor::HostScalarTensorBuffer<T>` is a specialized `TensorBuffer`
-// implementation for storing a single scalar value.
-//
-// TODO(mrry): Evaluate other compilers or approaches to aligning the value
-// so that it can be used directly as a tensor value. For example, in a C++17
-// future, we could use `alignas(EIGEN_MAX_ALIGN_BYTES)` to store the value
-// inline in this object to save an allocation. However, this is not currently
-// widely supported in our compilers.
 template <typename T>
-class Tensor::HostScalarTensorBuffer : public HostScalarTensorBufferBase {
- public:
-  HostScalarTensorBuffer(T&& value)
-      : data_(reinterpret_cast<T*>(cpu_allocator()->AllocateRaw(
-            EIGEN_MAX_ALIGN_BYTES, sizeof(value)))) {
-    if (is_simple_type<T>::value) {
-      *data_ = value;
-    } else {
-      new (data_) T(std::move(value));
-    }
-  }
-  ~HostScalarTensorBuffer() { cpu_allocator()->Deallocate(data_, 1); }
-  void* data() const final { return const_cast<T*>(data_); }
-  size_t size() const final { return sizeof(*data_); }
-  TensorBuffer* root_buffer() final { return this; }
-
- private:
-  T* const data_;
-};
-
-template <typename T>
-Tensor::Tensor(T value, host_scalar_tag tag)
-    : buf_(new HostScalarTensorBuffer<T>(std::move(value))) {
+Tensor::Tensor(T value, host_scalar_tag tag) {
+  // A packed representation for a single scalar value of type `T`, and a
+  // `TensorBuffer` implementation that describes (and manages the lifetime of)
+  // that value.
+  struct ValueAndTensorBuffer {
+    class HostScalarTensorBuffer : public HostScalarTensorBufferBase {
+     public:
+      HostScalarTensorBuffer(void* data) : data_(data) {}
+      void* data() const final { return const_cast<void*>(data_); }
+      size_t size() const final { return sizeof(T); }
+      TensorBuffer* root_buffer() final { return this; }
+
+      // Override `operator delete` so that calling `delete this` in
+      // `core::Refcounted::Unref()` for an object of this type will free
+      // the enclosing `ValueAndTensorBuffer` for the tensor buffer.
+      static void operator delete(void* ptr) {
+        port::AlignedFree(static_cast<char*>(ptr) -
+                          offsetof(ValueAndTensorBuffer, tensor_buffer));
+      }
+
+     private:
+      ~HostScalarTensorBuffer() override { static_cast<T*>(data_)->~T(); }
+      void* const data_;
+    };
+
+    T value;
+    HostScalarTensorBuffer tensor_buffer;
+  };
+
+  auto* value_and_buf = static_cast<ValueAndTensorBuffer*>(
+      port::AlignedMalloc(sizeof(ValueAndTensorBuffer), EIGEN_MAX_ALIGN_BYTES));
+  new (&value_and_buf->value) T(std::move(value));
+  new (&value_and_buf->tensor_buffer)
+      typename ValueAndTensorBuffer::HostScalarTensorBuffer(value_and_buf);
+  buf_ = &value_and_buf->tensor_buffer;
   set_dtype(DataTypeToEnum<T>::value);
 }
 
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 925ebc4945..4fa9d1df67 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -854,15 +854,18 @@ TEST(Tensor_HostScalar, Basics) {
     EXPECT_FLOAT_EQ(42.0f, Tt());
   }
   {
-    Tensor t("foo");
+    // NOTE(mrry): Use long enough strings so that the contents are dynamically
+    // allocated, and the absence of a call to the string destructor would
+    // cause a memory leak.
+    Tensor t("fooooooooooooooooooooooooooooooooooooo");
     EXPECT_EQ(DT_STRING, t.dtype());
     EXPECT_EQ(1, t.NumElements());
     auto Tt = t.scalar<string>();
     EXPECT_EQ(1, Tt.size());
     EXPECT_EQ(0, Tt.rank());
-    EXPECT_EQ("foo", Tt());
-    Tt() = "bar";
-    EXPECT_EQ("bar", Tt());
+    EXPECT_EQ("fooooooooooooooooooooooooooooooooooooo", Tt());
+    Tt() = "baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar";
+    EXPECT_EQ("baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar", Tt());
   }
 }
 
-- 
GitLab


From abdf645b56312e43e8d4870f8655fcb784a68409 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 13 Nov 2018 13:29:42 -0800
Subject: [PATCH 0181/1554] Automated rollback of commit
 091af8981d7d6bc2b2355849d841efb8a1063d75

PiperOrigin-RevId: 221323463
---
 tensorflow/core/BUILD                         | 13 ------
 .../core/common_runtime/direct_session.cc     |  4 --
 tensorflow/core/common_runtime/metrics.cc     | 42 -------------------
 tensorflow/core/common_runtime/metrics.h      | 27 ------------
 tensorflow/core/distributed_runtime/BUILD     |  2 -
 .../core/distributed_runtime/graph_mgr.cc     | 21 ++++------
 6 files changed, 8 insertions(+), 101 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/metrics.cc
 delete mode 100644 tensorflow/core/common_runtime/metrics.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index b7ec445e93..4f998cf9b3 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1672,7 +1672,6 @@ cc_library(
     name = "mobile_additional_lib_deps",
     deps = tf_additional_lib_deps() + [
         "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/time",
     ],
 )
 
@@ -3002,16 +3001,6 @@ cc_library(
     deps = [":lib_internal"],
 )
 
-tf_cuda_library(
-    name = "metrics",
-    srcs = ["common_runtime/metrics.cc"],
-    hdrs = ["common_runtime/metrics.h"],
-    deps = [
-        ":lib",
-        "@com_google_absl//absl/time",
-    ],
-)
-
 tf_cuda_library(
     name = "direct_session_internal",
     srcs = ["common_runtime/direct_session.cc"],
@@ -3028,12 +3017,10 @@ tf_cuda_library(
         ":graph",
         ":lib",
         ":lib_internal",
-        ":metrics",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/debug:debug_graph_utils",
         "//tensorflow/core/kernels:function_ops",
-        "@com_google_absl//absl/time",
     ],
     alwayslink = 1,
 )
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 3a23f084a5..52c1cd2691 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include <string>
 #include <vector>
 
-#include "absl/time/clock.h"
 #include "tensorflow/core/common_runtime/collective_executor_mgr.h"
 #include "tensorflow/core/common_runtime/collective_param_resolver_local.h"
 #include "tensorflow/core/common_runtime/constant_folding.h"
@@ -31,7 +30,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/memory_types.h"
-#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
@@ -456,7 +454,6 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
                                   CallFrameInterface* call_frame,
                                   ExecutorsAndKeys* executors_and_keys,
                                   RunMetadata* run_metadata) {
-  const absl::Time start_time = absl::Now();
   string session_id_meta = strings::StrCat("SessionRun #id=", step_id, "#");
   tracing::ScopedActivity activity(session_id_meta);
 
@@ -712,7 +709,6 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
       exec_and_lib.graph->ToGraphDef(partition_graph_def);
     }
   }
-  UpdateGraphExecutionTime(absl::Now() - start_time);
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/metrics.cc b/tensorflow/core/common_runtime/metrics.cc
deleted file mode 100644
index 2736739b1a..0000000000
--- a/tensorflow/core/common_runtime/metrics.cc
+++ /dev/null
@@ -1,42 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/common_runtime/metrics.h"
-
-#include "tensorflow/core/lib/monitoring/counter.h"
-
-namespace tensorflow {
-
-namespace {
-
-auto* graph_runs = monitoring::Counter<0>::New(
-    "/tensorflow/core/graph_runs",
-    "The number of graph executions used to collect "
-    "/tensorflow/core/graph_run_time_msecs");
-
-auto* graph_run_time_msecs = monitoring::Counter<0>::New(
-    "/tensorflow/core/graph_run_time_msecs",
-    "The total time spent on executing graphs in milliseconds.");
-}  // namespace
-
-void UpdateGraphExecutionTime(const absl::Duration running_time) {
-  if (running_time > absl::ZeroDuration()) {
-    graph_runs->GetCell()->IncrementBy(1);
-    graph_run_time_msecs->GetCell()->IncrementBy(running_time /
-                                                 absl::Milliseconds(1));
-  }
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/metrics.h b/tensorflow/core/common_runtime/metrics.h
deleted file mode 100644
index 5dd4caf5b7..0000000000
--- a/tensorflow/core/common_runtime/metrics.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
-
-#include "absl/time/time.h"
-
-namespace tensorflow {
-
-void UpdateGraphExecutionTime(const absl::Duration time);
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 9f09122434..3a534dcfb7 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -425,11 +425,9 @@ cc_library(
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:metrics",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:worker_proto_cc",
         "//tensorflow/core/debug",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc
index ee39062e34..3944668028 100644
--- a/tensorflow/core/distributed_runtime/graph_mgr.cc
+++ b/tensorflow/core/distributed_runtime/graph_mgr.cc
@@ -17,7 +17,6 @@ limitations under the License.
 
 #include <vector>
 
-#include "absl/time/clock.h"
 #include "tensorflow/core/common_runtime/build_graph_options.h"
 #include "tensorflow/core/common_runtime/constant_folding.h"
 #include "tensorflow/core/common_runtime/debugger_state_interface.h"
@@ -26,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/memory_types.h"
-#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/common_runtime/rendezvous_util.h"
@@ -388,7 +386,6 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id,
                             MutableRunGraphResponseWrapper* response,
                             CancellationManager* cancellation_manager,
                             const NamedTensors& in, StatusCallback done) {
-  const absl::Time start_time = absl::Now();
   // Lookup an item. Holds one ref while executing.
   Item* item = nullptr;
   {
@@ -446,16 +443,14 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id,
     return;
   }
 
-  StartParallelExecutors(
-      handle, step_id, item, rendezvous, ce_handle, collector, cost_graph,
-      cancellation_manager,
-      [item, rendezvous, ce_handle, done, start_time](const Status& s) {
-        done(s);
-        UpdateGraphExecutionTime(absl::Now() - start_time);
-        rendezvous->Unref();
-        item->Unref();
-        delete ce_handle;
-      });
+  StartParallelExecutors(handle, step_id, item, rendezvous, ce_handle,
+                         collector, cost_graph, cancellation_manager,
+                         [item, rendezvous, ce_handle, done](const Status& s) {
+                           done(s);
+                           rendezvous->Unref();
+                           item->Unref();
+                           delete ce_handle;
+                         });
 }
 
 void GraphMgr::StartParallelExecutors(const string& handle, int64 step_id,
-- 
GitLab


From 795c27127a8358af6d058563b043190a2dc02460 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 13 Nov 2018 13:31:36 -0800
Subject: [PATCH 0182/1554] Disable failing tests

PiperOrigin-RevId: 221323813
---
 tensorflow/contrib/cudnn_rnn/BUILD         | 2 ++
 tensorflow/contrib/distribute/python/BUILD | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD
index 670b549432..a4c1e968b9 100644
--- a/tensorflow/contrib/cudnn_rnn/BUILD
+++ b/tensorflow/contrib/cudnn_rnn/BUILD
@@ -63,6 +63,7 @@ cuda_py_test(
     ],
     shard_count = 6,
     tags = [
+        "no_oss",  # http://b/119506830
         "noasan",  # http://b/62067814
         "requires-gpu-sm35",
     ],
@@ -91,6 +92,7 @@ cuda_py_test(
     ],
     shard_count = 6,
     tags = [
+        "no_oss",  # http://b/119506830
         "noasan",  # http://b/62067814
     ],
 )
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index ce81b1b38b..8e0866c505 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -524,6 +524,7 @@ cuda_py_test(
         "noasan",
         "nomsan",
         "notsan",
+        "no_oss",  # http://b/119349471
     ],
 )
 
-- 
GitLab


From d90652139f23d7e5b7e2e2bdc2cb3a274f6b951f Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Tue, 13 Nov 2018 13:49:23 -0800
Subject: [PATCH 0183/1554] Only export div in TF V1.

PiperOrigin-RevId: 221327019
---
 tensorflow/python/ops/math_ops.py               | 5 ++++-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 39b1ca8993..4ff7a8e3a6 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1091,7 +1091,10 @@ def truediv(x, y, name=None):
   return _truediv_python3(x, y, name)
 
 
-@tf_export("div")
+@deprecation.deprecated(
+    date=None,
+    instructions="Deprecated in favor of operator or tf.math.divide.")
+@tf_export(v1=["div"])
 def div(x, y, name=None):
   """Divides x / y elementwise (using Python 2 division operator semantics).
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index b1e4c32523..7afff0a028 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -648,10 +648,6 @@ tf_module {
     name: "device"
     argspec: "args=[\'device_name\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "div"
-    argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "div_no_nan"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From e63f6f9da5d62060a9ab257d70692b06e1b80e0b Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 13 Nov 2018 13:57:32 -0800
Subject: [PATCH 0184/1554] Remove symbols from the v2 API. Included here:
 test.mock test.compute_gradient test.compute_gradient_error

Also removes checkpoint_v2 arg from test.assert_equal_graph_def

PiperOrigin-RevId: 221328602
---
 tensorflow/python/framework/test_util.py      | 28 +++++++++-
 tensorflow/python/ops/gradient_checker.py     | 55 +------------------
 tensorflow/python/platform/test.py            |  4 +-
 .../tools/api/golden/v2/tensorflow.test.pbtxt | 14 +----
 4 files changed, 30 insertions(+), 71 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index fd55ad2af9..0a08f3b5bd 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -114,8 +114,28 @@ def assert_ops_in_graph(expected_ops, graph):
   return actual_ops
 
 
-@tf_export("test.assert_equal_graph_def")
-def assert_equal_graph_def(actual, expected, checkpoint_v2=False):
+@tf_export("test.assert_equal_graph_def", v1=[])
+def assert_equal_graph_def_v2(actual, expected):
+  """Asserts that two `GraphDef`s are (mostly) the same.
+
+  Compares two `GraphDef` protos for equality, ignoring versions and ordering of
+  nodes, attrs, and control inputs.  Node names are used to match up nodes
+  between the graphs, so the naming of nodes must be consistent. This function
+  ignores randomized attribute values that may appear in V2 checkpoints.
+
+  Args:
+    actual: The `GraphDef` we have.
+    expected: The `GraphDef` we expected.
+
+  Raises:
+    AssertionError: If the `GraphDef`s do not match.
+    TypeError: If either argument is not a `GraphDef`.
+  """
+  assert_equal_graph_def(actual, expected, checkpoint_v2=True)
+
+
+@tf_export(v1=["test.assert_equal_graph_def"])
+def assert_equal_graph_def_v1(actual, expected, checkpoint_v2=False):
   """Asserts that two `GraphDef`s are (mostly) the same.
 
   Compares two `GraphDef` protos for equality, ignoring versions and ordering of
@@ -132,6 +152,10 @@ def assert_equal_graph_def(actual, expected, checkpoint_v2=False):
     AssertionError: If the `GraphDef`s do not match.
     TypeError: If either argument is not a `GraphDef`.
   """
+  assert_equal_graph_def(actual, expected, checkpoint_v2)
+
+
+def assert_equal_graph_def(actual, expected, checkpoint_v2=False):
   if not isinstance(actual, graph_pb2.GraphDef):
     raise TypeError(
         "Expected tf.GraphDef for actual, got %s" % type(actual).__name__)
diff --git a/tensorflow/python/ops/gradient_checker.py b/tensorflow/python/ops/gradient_checker.py
index 1665219c80..3926ff6900 100644
--- a/tensorflow/python/ops/gradient_checker.py
+++ b/tensorflow/python/ops/gradient_checker.py
@@ -31,7 +31,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -266,7 +265,7 @@ def _compute_gradient_list(x,
   return ret
 
 
-@tf_export("test.compute_gradient")
+@tf_export(v1=["test.compute_gradient"])
 def compute_gradient(x,
                      x_shape,
                      y,
@@ -329,9 +328,6 @@ def compute_gradient(x,
 
 
 @tf_export(v1=["test.compute_gradient_error"])
-@deprecation.deprecated_args(
-    None, "init_targets will be deprecated in TensorFlow 2.0",
-    ("init_targets", None))  # Do not trigger warning in V2
 def compute_gradient_error(x,
                            x_shape,
                            y,
@@ -380,52 +376,3 @@ def compute_gradient_error(x,
     if j_t.size or j_n.size:  # Handle zero size tensors correctly
       error = np.maximum(error, np.fabs(j_t - j_n).max())
   return error
-
-
-@tf_export("test.compute_gradient_error", v1=[])
-def compute_gradient_error_v2(x,
-                              x_shape,
-                              y,
-                              y_shape,
-                              x_init_value=None,
-                              delta=1e-3,
-                              extra_feed_dict=None):
-  """Computes the gradient error.
-
-  Computes the maximum error for dy/dx between the computed Jacobian and the
-  numerically estimated Jacobian.
-
-  This function will modify the tensors passed in as it adds more operations
-  and hence changing the consumers of the operations of the input tensors.
-
-  This function adds operations to the current session. To compute the error
-  using a particular device, such as a GPU, use the standard methods for
-  setting a device (e.g. using with sess.graph.device() or setting a device
-  function in the session constructor).
-
-  Args:
-    x: a tensor or list of tensors
-    x_shape: the dimensions of x as a tuple or an array of ints. If x is a list,
-      then this is the list of shapes.
-    y: a tensor
-    y_shape: the dimensions of y as a tuple or an array of ints.
-    x_init_value: (optional) a numpy array of the same shape as "x" representing
-      the initial value of x. If x is a list, this should be a list of numpy
-      arrays.  If this is none, the function will pick a random tensor as the
-      initial value.
-    delta: (optional) the amount of perturbation.
-    extra_feed_dict: dict that allows fixing specified tensor values during the
-      Jacobian calculation.
-
-  Returns:
-    The maximum error in between the two Jacobians.
-  """
-  return compute_gradient_error(
-      x,
-      x_shape,
-      y,
-      y_shape,
-      x_init_value=x_init_value,
-      delta=delta,
-      init_targets=None,
-      extra_feed_dict=extra_feed_dict)
diff --git a/tensorflow/python/platform/test.py b/tensorflow/python/platform/test.py
index d084870b25..943832af7a 100644
--- a/tensorflow/python/platform/test.py
+++ b/tensorflow/python/platform/test.py
@@ -46,9 +46,9 @@ from tensorflow.python.util.tf_export import tf_export
 if sys.version_info.major == 2:
   import mock                # pylint: disable=g-import-not-at-top,unused-import
 else:
-  from unittest import mock  # pylint: disable=g-import-not-at-top
+  from unittest import mock  # pylint: disable=g-import-not-at-top,g-importing-member
 
-tf_export('test.mock')(mock)
+tf_export(v1=['test.mock'])(mock)
 
 # Import Benchmark class
 Benchmark = _googletest.Benchmark  # pylint: disable=invalid-name
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
index af3f06d8de..72ce733044 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
@@ -12,26 +12,14 @@ tf_module {
     name: "TestCase"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "mock"
-    mtype: "<type \'module\'>"
-  }
   member_method {
     name: "assert_equal_graph_def"
-    argspec: "args=[\'actual\', \'expected\', \'checkpoint_v2\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'actual\', \'expected\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "benchmark_config"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "compute_gradient"
-    argspec: "args=[\'x\', \'x_shape\', \'y\', \'y_shape\', \'x_init_value\', \'delta\', \'init_targets\', \'extra_feed_dict\'], varargs=None, keywords=None, defaults=[\'None\', \'0.001\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradient_error"
-    argspec: "args=[\'x\', \'x_shape\', \'y\', \'y_shape\', \'x_init_value\', \'delta\', \'extra_feed_dict\'], varargs=None, keywords=None, defaults=[\'None\', \'0.001\', \'None\'], "
-  }
   member_method {
     name: "create_local_cluster"
     argspec: "args=[\'num_workers\', \'num_ps\', \'protocol\', \'worker_config\', \'ps_config\'], varargs=None, keywords=None, defaults=[\'grpc\', \'None\', \'None\'], "
-- 
GitLab


From 5b1e910313ea29cf2f8679311634fba6c00c0a31 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 13 Nov 2018 14:14:03 -0800
Subject: [PATCH 0185/1554] Replace calls to CreateNewUnverifiedModule with
 CreateNewVerifiedModule when no XLA code changes are required.

PiperOrigin-RevId: 221331867
---
 .../xla/service/buffer_liveness_test.cc       | 24 +++++-----
 .../xla/service/copy_insertion_test.cc        | 44 +++++++++----------
 .../cpu/cpu_instruction_fusion_test.cc        | 28 ++++++------
 .../service/cpu/cpu_layout_assignment_test.cc | 22 +++++-----
 .../cpu/tests/cpu_eigen_dot_operation_test.cc |  2 +-
 .../cpu/tests/cpu_external_constants_test.cc  |  2 +-
 .../service/cpu/tests/cpu_intrinsic_test.cc   |  2 +-
 .../xla/service/cpu/tests/cpu_noalias_test.cc |  2 +-
 .../xla/service/gpu/gpu_hlo_schedule_test.cc  | 10 ++---
 .../service/gpu/gpu_layout_assignment_test.cc |  8 ++--
 .../service/gpu/instruction_fusion_test.cc    |  6 +--
 .../service/gpu/multi_output_fusion_test.cc   |  2 +-
 .../xla/service/gpu/stream_assignment_test.cc |  8 ++--
 .../xla/service/gpu/tests/gpu_codegen_test.h  |  2 +-
 .../xla/service/gpu/tests/gpu_copy_test.cc    |  2 +-
 .../xla/service/gpu/tests/gpu_index_test.cc   |  2 +-
 .../xla/service/gpu/tests/gpu_ldg_test.cc     |  6 +--
 .../xla/service/gpu/tests/gpu_noalias_test.cc |  2 +-
 .../xla/service/gpu/while_transformer_test.cc |  2 +-
 .../xla/service/hlo_computation_test.cc       | 34 +++++++-------
 .../xla/service/hlo_cost_analysis_test.cc     |  6 +--
 .../xla/service/hlo_dataflow_analysis_test.cc |  2 +-
 .../compiler/xla/service/hlo_dce_test.cc      | 10 ++---
 .../xla/service/hlo_memory_scheduler_test.cc  |  8 ++--
 .../compiler/xla/service/hlo_module_test.cc   | 16 +++----
 .../compiler/xla/service/hlo_ordering_test.cc | 12 ++---
 .../hlo_subcomputation_unification_test.cc    |  6 +--
 .../xla/service/instruction_fusion_test.cc    |  6 +--
 .../reduce_precision_insertion_test.cc        | 28 ++++++------
 .../xla/service/transpose_folding_test.cc     | 10 ++---
 30 files changed, 157 insertions(+), 157 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
index aeee543e84..cdc13d86b3 100644
--- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
@@ -117,7 +117,7 @@ TEST_F(BufferLivenessTest, ElementwiseChain) {
   auto log = builder.AddInstruction(
       HloInstruction::CreateUnary(vec_, HloOpcode::kLog, exp));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   auto liveness =
@@ -164,7 +164,7 @@ TEST_F(BufferLivenessTest, MultipleEntryParameters_Sequential) {
   auto add = builder.AddInstruction(
       HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation* entry = module->AddEntryComputation(builder.Build());
 
   HloSchedule schedule(module.get());
@@ -213,7 +213,7 @@ TEST_F(BufferLivenessTest, NonElementwiseOperand) {
   auto reverse =
       builder.AddInstruction(HloInstruction::CreateReverse(vec_, negate, {0}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   auto liveness =
@@ -247,7 +247,7 @@ TEST_F(BufferLivenessTest, OverlappedBuffers) {
   auto add = builder.AddInstruction(
       HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   auto liveness =
@@ -289,7 +289,7 @@ TEST_F(BufferLivenessTest, OverlappedBuffersSequentialOrder) {
   auto add = builder.AddInstruction(
       HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   HloSchedule schedule(module.get());
@@ -336,7 +336,7 @@ TEST_F(BufferLivenessTest, RootInstructionIsNotLastInSequentialOrder) {
       HloInstruction::CreateSend(recv_done, token, /*channel_id=*/1));
   auto send_done = builder.AddInstruction(HloInstruction::CreateSendDone(send));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build(add));
 
   HloSchedule schedule(module.get());
@@ -373,7 +373,7 @@ TEST_F(BufferLivenessTest, TupleLiveOut) {
   auto outer_tuple =
       builder.AddInstruction(HloInstruction::CreateTuple({inner_tuple, exp}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   auto liveness =
@@ -393,7 +393,7 @@ TEST_F(BufferLivenessTest, TupleLiveOut) {
 
 TEST_F(BufferLivenessTest, EmbeddedComputation) {
   // Test MaybeLiveOut and MayInterfere for embedded computation.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
 
   auto embedded_builder = HloComputation::Builder(TestName() + "_embedded");
   auto embedded_param = embedded_builder.AddInstruction(
@@ -450,7 +450,7 @@ TEST_F(BufferLivenessTest, TupleConstantLiveOut) {
   builder.AddInstruction(HloInstruction::CreateGetTupleElement(
       inner_tuple0.shape(), tuple_constant, 0));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   auto liveness =
@@ -576,7 +576,7 @@ TEST_F(BufferLivenessTest, DependentTupleElements) {
   auto tuple_root =
       builder.AddInstruction(HloInstruction::CreateTuple({add0, add1}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(BuildDummyComputation());
   module->AddEmbeddedComputation(builder.Build());
 
@@ -646,7 +646,7 @@ class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest {
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
     // Build module and get reference to entry computation.
-    auto module = CreateNewUnverifiedModule();
+    auto module = CreateNewVerifiedModule();
     module->AddEntryComputation(builder.Build());
     auto* computation = module->entry_computation();
     // Create fusion instruction based on number of tuple element 1 users.
@@ -802,7 +802,7 @@ class DynamicUpdateSliceLivenessTest : public BufferLivenessTest {
     auto tuple_root = builder.AddInstruction(
         HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
     // Build module and get reference to entry computation.
-    auto module = CreateNewUnverifiedModule();
+    auto module = CreateNewVerifiedModule();
     module->AddEntryComputation(BuildDummyComputation());
     module->AddEmbeddedComputation(builder.Build());
     // Run BufferLiveness on 'module'.
diff --git a/tensorflow/compiler/xla/service/copy_insertion_test.cc b/tensorflow/compiler/xla/service/copy_insertion_test.cc
index 7446bc7cc1..e4e9d7ba05 100644
--- a/tensorflow/compiler/xla/service/copy_insertion_test.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion_test.cc
@@ -94,7 +94,7 @@ TEST_F(CopyInsertionTest, SingleParameter) {
 
   EXPECT_THAT(x->users(), UnorderedElementsAre(tuple));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   InsertCopies(module.get());
@@ -114,7 +114,7 @@ TEST_F(CopyInsertionTest, SingleConstant) {
 
   EXPECT_THAT(constant->users(), UnorderedElementsAre(tuple));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   InsertCopies(module.get());
@@ -127,7 +127,7 @@ TEST_F(CopyInsertionTest, SingleConstant) {
 TEST_F(CopyInsertionTest, ExistingCopiesNotRemoved) {
   // Verify that kCopy instructions which change layout and exist before
   // copy-insertion remain in the graph after copy-insertion.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
 
   auto builder = HloComputation::Builder(TestName());
   HloInstruction* constant =
@@ -181,7 +181,7 @@ TEST_F(CopyInsertionTest, MultipleConstantsAndParameters) {
 
   builder.AddInstruction(HloInstruction::CreateTuple({constant2, x, add}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   InsertCopies(module.get());
@@ -217,7 +217,7 @@ TEST_F(CopyInsertionTest, AmbiguousPointsToSet) {
   EXPECT_THAT(constant2->users(), UnorderedElementsAre(tuple1, tuple2));
   EXPECT_THAT(constant3->users(), UnorderedElementsAre(tuple2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   HloInstruction* old_root = module->entry_computation()->root_instruction();
@@ -238,7 +238,7 @@ TEST_F(CopyInsertionTest, BitcastParameter) {
   HloInstruction* bitcast = builder.AddInstruction(HloInstruction::CreateUnary(
       ShapeUtil::MakeShape(F32, {2, 2}), HloOpcode::kBitcast, x));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(x->users(), UnorderedElementsAre(bitcast));
@@ -261,7 +261,7 @@ TEST_F(CopyInsertionTest, BitcastConstant) {
   HloInstruction* bitcast = builder.AddInstruction(HloInstruction::CreateUnary(
       ShapeUtil::MakeShape(F32, {2, 2}), HloOpcode::kBitcast, constant));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(constant->users(), UnorderedElementsAre(bitcast));
@@ -283,7 +283,7 @@ TEST_F(CopyInsertionTest, BitcastTupleElementParameter) {
       ShapeUtil::MakeShape(F32, {2, 2}), HloOpcode::kBitcast, x));
   builder.AddInstruction(HloInstruction::CreateTuple({bitcast}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(x->users(), UnorderedElementsAre(bitcast));
@@ -310,7 +310,7 @@ TEST_F(CopyInsertionTest, NestedTupleParameter) {
            ShapeUtil::MakeShape(F32, {42})}),
       "param0"));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_EQ(HloOpcode::kParameter,
@@ -351,7 +351,7 @@ TEST_F(CopyInsertionTest, ElementOfNestedTupleParameter) {
   auto gte = builder.AddInstruction(HloInstruction::CreateGetTupleElement(
       ShapeUtil::GetSubshape(param->shape(), {0}), param, 0));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_EQ(gte, module->entry_computation()->root_instruction());
@@ -388,7 +388,7 @@ TEST_F(CopyInsertionTest, AmbiguousTopLevelRoot) {
       builder.AddInstruction(HloInstruction::CreateGetTupleElement(
           ShapeUtil::GetSubshape(select->shape(), {0}), select, 0));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_EQ(gte, module->entry_computation()->root_instruction());
@@ -1295,7 +1295,7 @@ TEST_F(WhileCopyInsertionTest, InitPointsToNonDistinctUsedByTwoWhileLoops) {
 TEST_F(CopyInsertionTest, SwizzlingWhile) {
   // Test a while instruction with a body which permutes its tuple parameter
   // elements.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape loop_state_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1362,7 +1362,7 @@ TEST_F(CopyInsertionTest, CrossingParameters) {
   //   |  / \ |
   //   | /   \|
   //  (p1  ,  p0)
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape tuple_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1395,7 +1395,7 @@ TEST_F(CopyInsertionTest, ParametersAliasing) {
   //   |      |
   //   |      |
   //  (p0 ,  p1)
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape tuple_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1428,7 +1428,7 @@ TEST_F(CopyInsertionTest, ParameterWithNoAliasing) {
   //   |      |
   //   |      |
   //  (p0 ,  p1)
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape tuple_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1461,7 +1461,7 @@ TEST_F(CopyInsertionTest, ParameterWithPartialAliasing) {
   //   |      |
   //   |      |
   //  (p0 ,  p1)
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape tuple_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1496,7 +1496,7 @@ TEST_F(CopyInsertionTest, ParameterAndParallelOpsWithPartialAliasing) {
   //   |    |      |
   //   |    |      |
   //   +-- (p0 ,  p1)
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape tuple_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1534,7 +1534,7 @@ TEST_F(CopyInsertionTest, ParameterAndOpsWithPartialAliasing) {
   //   |    Add----+
   //   |    |      |
   //   +-- (p0 ,  p1)
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape tuple_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1569,7 +1569,7 @@ TEST_F(CopyInsertionTest, SwizzlingWhileWithOneOp) {
   // the operation (instruction) on the element makes the live range of the
   // respective input and output elements different than if the instruction were
   // not there (as in the SwizzlingWhile test above).
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape loop_state_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1632,7 +1632,7 @@ TEST_F(CopyInsertionTest, SwizzlingWhileSharedInput) {
   // the while body is a single constant (both loop state elements are the same
   // constant). This means no copies are necessary because both loop state
   // elements are the same so interchanging them is a no-op.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape loop_state_shape =
       ShapeUtil::MakeTupleShape({scalar_shape_, scalar_shape_});
 
@@ -1693,7 +1693,7 @@ TEST_F(CopyInsertionTest, SequentialWhiles) {
   const Shape loop_state_shape = ShapeUtil::MakeTupleShape(
       {element_shape, element_shape, element_shape, element_shape});
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto builder = HloComputation::Builder(TestName());
   auto param_0 = builder.AddInstruction(
       HloInstruction::CreateParameter(0, element_shape, "param_0"));
@@ -1783,7 +1783,7 @@ TEST_F(CopyInsertionTest, SequentialWhiles) {
 TEST_F(CopyInsertionTest, WhileBodyWithConstantRoot) {
   // Test a while body and condition which are each simply a constant (root of
   // computation is a constant). The body constant should be copied.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto builder = HloComputation::Builder(TestName());
   auto param_0 = builder.AddInstruction(
       HloInstruction::CreateParameter(0, scalar_shape_, "param_0"));
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index c95a514ca0..c77d5988ba 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -321,7 +321,7 @@ TEST_F(OpcodeFusionTest, Exponential_Reshape_Negate) {
   builder.AddInstruction(
       HloInstruction::CreateUnary(result_shape, HloOpcode::kNegate, reshape2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   RunFusionAndCheckOpcodesWereFused(
@@ -370,7 +370,7 @@ TEST_F(OpcodeFusionTest, Broadcast_Negate) {
   builder.AddInstruction(HloInstruction::CreateUnary(
       result_shape, HloOpcode::kNegate, broadcast1));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   RunFusionAndCheckOpcodesWereFused(
@@ -410,7 +410,7 @@ TEST_F(OpcodeFusionTest, Exponential_Negate) {
   builder.AddInstruction(
       HloInstruction::CreateUnary(param_shape, HloOpcode::kNegate, exp1));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   RunFusionAndCheckOpcodesWereFused(
@@ -429,7 +429,7 @@ TEST_F(OpcodeFusionTest, Reshape_Negate) {
   builder.AddInstruction(
       HloInstruction::CreateUnary(result_shape, HloOpcode::kNegate, reshape1));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   RunFusionAndCheckOpcodesWereFused(
@@ -447,7 +447,7 @@ TEST_F(OpcodeFusionTest, Reverse_Negate) {
   builder.AddInstruction(
       HloInstruction::CreateUnary(param_shape, HloOpcode::kNegate, reverse1));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   RunFusionAndCheckOpcodesWereFused(
@@ -489,7 +489,7 @@ TEST_F(OpcodeFusionTest, Exponential_Transpose_Negate) {
   builder.AddInstruction(HloInstruction::CreateUnary(
       result_shape, HloOpcode::kNegate, transpose2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   RunFusionAndCheckOpcodesWereFused(
@@ -498,7 +498,7 @@ TEST_F(OpcodeFusionTest, Exponential_Transpose_Negate) {
 }
 
 TEST_F(OpcodeFusionTest, UnaryMapOfExp) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
 
   HloComputation::Builder builder(TestName());
   Shape shape = ShapeUtil::MakeShape(F32, {3, 4});
@@ -517,7 +517,7 @@ TEST_F(OpcodeFusionTest, UnaryMapOfExp) {
 }
 
 TEST_F(OpcodeFusionTest, BinaryMapOfExps) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
 
   HloComputation::Builder builder(TestName());
   Shape shape = ShapeUtil::MakeShape(F32, {3, 4});
@@ -542,7 +542,7 @@ TEST_F(OpcodeFusionTest, BinaryMapOfExps) {
 }
 
 TEST_F(OpcodeFusionTest, DynamicSliceWithDynamicUpdateSlice) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
 
   HloComputation::Builder builder(TestName());
   Shape full_shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
@@ -573,7 +573,7 @@ TEST_F(OpcodeFusionTest, DynamicSliceWithDynamicUpdateSlice) {
 }
 
 TEST_F(OpcodeFusionTest, MessOfFusibleNodes) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation::Builder builder(TestName());
 
   Shape full_shape = ShapeUtil::MakeShape(F32, {4, 100, 10, 100, 50});
@@ -712,7 +712,7 @@ void CreateComputationForDotAddOutputFusionTest(const string& test_name,
 }
 
 TEST_F(OpcodeFusionTest, DotAddOutputFusion_1x50x19) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/1,
                                              /*k=*/50, /*n=*/19,
                                              /*add_extra_use_for_dot=*/false);
@@ -725,7 +725,7 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_1x50x19) {
 }
 
 TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/19,
                                              /*k=*/50, /*n=*/1,
                                              /*add_extra_use_for_dot=*/false);
@@ -738,7 +738,7 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1) {
 }
 
 TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x19) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/19,
                                              /*k=*/50, /*n=*/19,
                                              /*add_extra_use_for_dot=*/false);
@@ -751,7 +751,7 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x19) {
 }
 
 TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1_multi_use) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   CreateComputationForDotAddOutputFusionTest(TestName(), module.get(), /*m=*/19,
                                              /*k=*/50, /*n=*/1,
                                              /*add_extra_use_for_dot=*/true);
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc
index 2cd52e4a18..6c61b64758 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment_test.cc
@@ -73,7 +73,7 @@ TEST_F(CpuLayoutAssignmentTest, DotWithConstantRhsTensor) {
   auto result = builder.AddInstruction(
       CreateCanonicalDot(result_shape, dot_lhs, dot_rhs));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
   ComputationLayout computation_layout(computation->ComputeProgramShape());
@@ -114,7 +114,7 @@ TEST_F(CpuLayoutAssignmentTest, MultipleDotsWithSameConstantRhsTensor0) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       result_shape, HloOpcode::kAdd, dot_a_result, dot_b_result));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
   ComputationLayout computation_layout(computation->ComputeProgramShape());
@@ -158,7 +158,7 @@ TEST_F(CpuLayoutAssignmentTest, MultipleDotsWithSameConstantRhsTensor1) {
   auto tuple_result = builder.AddInstruction(
       HloInstruction::CreateTuple({dot_a_result, dot_b_result}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
   ComputationLayout computation_layout(computation->ComputeProgramShape());
@@ -192,7 +192,7 @@ TEST_F(CpuLayoutAssignmentTest, DotWithConstantLhsTensor) {
   auto dot_result = builder.AddInstruction(
       CreateCanonicalDot(result_shape, dot_lhs, dot_rhs));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
   ComputationLayout computation_layout(computation->ComputeProgramShape());
@@ -232,7 +232,7 @@ TEST_F(CpuLayoutAssignmentTest, DotWithConstantRhsTensorThroughGTE) {
   auto dot_result = builder.AddInstruction(
       CreateCanonicalDot(result_shape, dot_lhs, dot_rhs));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
   ComputationLayout computation_layout(computation->ComputeProgramShape());
@@ -353,7 +353,7 @@ static void AssertCorrectLayoutForDotOutputFusion(
 }
 
 TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_0) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+  std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
   TF_ASSERT_OK_AND_ASSIGN(
       DotOutputFusionLayoutAssignmentResult layout_assignment_result,
       RunDotOutputFusion(module.get(), TestName(), /*m=*/1, /*k=*/50, /*n=*/19,
@@ -365,7 +365,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_0) {
 }
 
 TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_1) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+  std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
   TF_ASSERT_OK_AND_ASSIGN(
       DotOutputFusionLayoutAssignmentResult layout_assignment_result,
       RunDotOutputFusion(module.get(), TestName(), /*m=*/1, /*k=*/50, /*n=*/19,
@@ -377,7 +377,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_1x50x19_dot_idx_1) {
 }
 
 TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_0) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+  std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
   TF_ASSERT_OK_AND_ASSIGN(
       DotOutputFusionLayoutAssignmentResult layout_assignment_result,
       RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/1,
@@ -389,7 +389,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_0) {
 }
 
 TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_1) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+  std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
   TF_ASSERT_OK_AND_ASSIGN(
       DotOutputFusionLayoutAssignmentResult layout_assignment_result,
       RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/1,
@@ -401,7 +401,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x1_dot_idx_1) {
 }
 
 TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x19_dot_idx_0) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+  std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
   TF_ASSERT_OK_AND_ASSIGN(
       DotOutputFusionLayoutAssignmentResult layout_assignment_result,
       RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/19,
@@ -413,7 +413,7 @@ TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x19_dot_idx_0) {
 }
 
 TEST_F(CpuLayoutAssignmentTest, DotOutputFusion_19x50x19_dot_idx_1) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+  std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
   TF_ASSERT_OK_AND_ASSIGN(
       DotOutputFusionLayoutAssignmentResult layout_assignment_result,
       RunDotOutputFusion(module.get(), TestName(), /*m=*/19, /*k=*/50, /*n=*/19,
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
index 691b3c7bee..f8f5f392da 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
@@ -50,7 +50,7 @@ class CpuEigenDotOperationTest
         /*entry_point_name=*/"entry",
         /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static};
 
-    auto hlo_module = CreateNewUnverifiedModule();
+    auto hlo_module = CreateNewVerifiedModule();
     hlo_module->AddEntryComputation(std::move(entry_computation));
 
     CompileAheadOfTimeAndVerifyIr(std::move(hlo_module), options,
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc
index d201a151d7..e30f95311f 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_external_constants_test.cc
@@ -46,7 +46,7 @@ class CpuExternalConstantsTest : public CpuCodegenTest {
     builder.AddInstruction(
         HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, constant));
 
-    std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
+    std::unique_ptr<HloModule> module = CreateNewVerifiedModule();
     module->AddEntryComputation(builder.Build());
 
     CompileAndVerifyIr(std::move(module), filecheck_pattern,
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
index 773336c7a9..9b10c49f4f 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
@@ -91,7 +91,7 @@ TEST_P(CpuUnaryIntrinsicTest, DoIt) {
       /*entry_point_name=*/"entry",
       /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static};
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(std::move(computation));
 
   string check_lines{spec.check_lines.data(), spec.check_lines.size()};
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
index f5419b7063..a7702c2aee 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
@@ -56,7 +56,7 @@ TEST_F(CpuNoAliasTest, Concat) {
 
   std::unique_ptr<HloComputation> computation = builder.Build();
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(std::move(computation));
 
   // Now that we have an HLO module, build an llvm_ir::AliasAnalysis for it.
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
index 6d3aed15eb..5f857a1a54 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
@@ -44,7 +44,7 @@ class GpuHloScheduleTest : public HloTestBase {
         .ConsumeValueOrDie();
   }
 
-  std::unique_ptr<HloModule> CreateNewUnverifiedModule() {
+  std::unique_ptr<HloModule> CreateNewVerifiedModule() {
     HloModuleConfig config;
     auto debug_options = GetDebugOptionsForTest();
     debug_options.set_xla_gpu_disable_multi_streaming(false);
@@ -79,7 +79,7 @@ TEST_F(GpuHloScheduleTest, SequentialMatMul) {
   HloInstruction* dot2 =
       builder.AddInstruction(CreateCanonicalDot(f32_2x2_, dot1, z));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(dot2));
 
   std::unique_ptr<StreamAssignment> streams = AssignStreams(*module);
@@ -139,7 +139,7 @@ TEST_F(GpuHloScheduleTest, SequentialAdd) {
   HloInstruction* add3 = builder.AddInstruction(
       HloInstruction::CreateBinary(f32_2x2_, HloOpcode::kAdd, add1, add2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(add3));
 
   std::unique_ptr<StreamAssignment> streams = AssignStreams(*module);
@@ -209,7 +209,7 @@ TEST_F(GpuHloScheduleTest, ConcurrentMatMul) {
   HloInstruction* add =
       builder.AddInstruction(CreateCanonicalDot(f32_2x2_, dot1, dot2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(add));
 
   std::unique_ptr<StreamAssignment> streams = AssignStreams(*module);
@@ -288,7 +288,7 @@ TEST_F(GpuHloScheduleTest, LatticeMatMul) {
   HloInstruction* d40 =
       builder.AddInstruction(CreateCanonicalDot(f32_2x2_, d30, d31));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(d40));
 
   std::unique_ptr<StreamAssignment> streams = AssignStreams(*module);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc
index 8cc76c872c..2ffc8bfb49 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment_test.cc
@@ -61,7 +61,7 @@ TEST_F(LayoutAssignmentTest, Elementwise) {
             HloInstruction::CreateParameter(1, ashape, "y"));
         auto add = builder.AddInstruction(
             HloInstruction::CreateBinary(ashape, HloOpcode::kAdd, x, y));
-        auto module = CreateNewUnverifiedModule();
+        auto module = CreateNewVerifiedModule();
         HloComputation* computation =
             module->AddEntryComputation(builder.Build(add));
 
@@ -148,7 +148,7 @@ TEST_F(LayoutAssignmentTest, BatchNormInference) {
           {operand, scale, offset, mean, variance, epsilon, feature_index},
           kCudnnBatchNormForwardInferenceCallTarget));
 
-      auto module = CreateNewUnverifiedModule();
+      auto module = CreateNewVerifiedModule();
       HloComputation* computation =
           module->AddEntryComputation(builder.Build(batchnorm));
 
@@ -217,7 +217,7 @@ TEST_F(LayoutAssignmentTest, BatchNormTraining) {
           batchnorm_shape, {operand, scale, offset, epsilon, feature_index},
           kCudnnBatchNormForwardTrainingCallTarget));
 
-      auto module = CreateNewUnverifiedModule();
+      auto module = CreateNewVerifiedModule();
       HloComputation* computation =
           module->AddEntryComputation(builder.Build(batchnorm));
 
@@ -298,7 +298,7 @@ TEST_F(LayoutAssignmentTest, BatchNormGrad) {
                  feature_index},
                 kCudnnBatchNormBackwardCallTarget));
 
-        auto module = CreateNewUnverifiedModule();
+        auto module = CreateNewVerifiedModule();
         HloComputation* computation =
             module->AddEntryComputation(builder.Build(batchnorm));
 
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
index fb77bc4b8e..2b060b03ce 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
@@ -117,7 +117,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastReshapeOfDotUnfused) {
   auto reshape2 = builder.AddInstruction(HloInstruction::CreateReshape(
       ShapeUtil::MakeShape(S32, {1, 1, 1}), dot1));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(reshape2, computation->root_instruction());
   EXPECT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true)
@@ -134,7 +134,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastTransposeOfDotUnfused) {
   auto transpose2 = builder.AddInstruction(HloInstruction::CreateTranspose(
       ShapeUtil::MakeShape(S32, {1, 1}), dot1, {0, 1}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(transpose2, computation->root_instruction());
   EXPECT_FALSE(GpuInstructionFusion(/*may_duplicate=*/true)
@@ -723,7 +723,7 @@ TEST_F(InstructionFusionTest, AvoidsLargeFusion) {
     sum = b.AddInstruction(
         HloInstruction::CreateBinary(shape, HloOpcode::kAdd, sum, param));
   }
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(b.Build());
   EXPECT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true)
                   .Run(module.get())
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
index dc221f22a7..d16c87ba5c 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
@@ -580,7 +580,7 @@ TEST_F(MultiOutputFusionTest, AvoidsLargeFusion) {
   //   ...
   // where each of the (pi * pj)'s is represented as a fusion node so that
   // multi-output fusion will pay attention to it.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation::Builder b(TestName());
   Shape shape = ShapeUtil::MakeShape(F32, {10, 100});
 
diff --git a/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc b/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc
index f2ef11e1e6..31a5d7a8c0 100644
--- a/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/stream_assignment_test.cc
@@ -30,7 +30,7 @@ namespace gpu {
 
 class StreamAssignmentTest : public HloTestBase {
  protected:
-  std::unique_ptr<HloModule> CreateNewUnverifiedModule() {
+  std::unique_ptr<HloModule> CreateNewVerifiedModule() {
     HloModuleConfig config;
     auto debug_options = GetDebugOptionsForTest();
     debug_options.set_xla_gpu_disable_multi_streaming(false);
@@ -55,7 +55,7 @@ TEST_F(StreamAssignmentTest, SequentialMatMul) {
   HloInstruction* dot2 =
       builder.AddInstruction(CreateCanonicalDot(f32_2x2_, dot1, z));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(dot2));
 
   std::unique_ptr<StreamAssignment> assignment = AssignStreams(*module);
@@ -76,7 +76,7 @@ TEST_F(StreamAssignmentTest, ConcurrentMatMul) {
   HloInstruction* add = builder.AddInstruction(
       HloInstruction::CreateBinary(f32_2x2_, HloOpcode::kAdd, dot1, dot2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(add));
 
   std::unique_ptr<StreamAssignment> assignment = AssignStreams(*module);
@@ -120,7 +120,7 @@ TEST_F(StreamAssignmentTest, LatticeMatMul) {
   HloInstruction* d40 =
       builder.AddInstruction(CreateCanonicalDot(f32_2x2_, d30, d31));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build(d40));
 
   std::unique_ptr<StreamAssignment> assignment = AssignStreams(*module);
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h
index d2f30ae7bc..d917320e36 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h
@@ -26,7 +26,7 @@ namespace gpu {
 // Tests that verify IR or PTX emitted by the GPU backend is as expected.
 class GpuCodegenTest : public LlvmIrGenTestBase {
  protected:
-  // Like HloTestBase::CreateNewUnverifiedModule(), with a flag for configuring
+  // Like HloTestBase::CreateNewVerifiedModule(), with a flag for configuring
   // the ftz option.
   std::unique_ptr<HloModule> CreateNewUnverifiedModuleWithFTZ(bool ftz);
 
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
index 268b48a1ca..a1ed849904 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
@@ -46,7 +46,7 @@ TEST_F(GpuCopyTest, UseMemcpy) {
 
   std::unique_ptr<HloComputation> computation = builder.Build();
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(std::move(computation));
 
   // There should not be any kernel prefixed "copy".
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc
index da8e513a2c..6814be779e 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc
@@ -51,7 +51,7 @@ TEST_F(GpuIndexTest, CompatibleUseLinearIndex) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       ShapeUtil::MakeShape(PRED, {5, 7, 2}), HloOpcode::kGe, param_x, param_y));
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(builder.Build());
 
   // Check the optimized IR as the unoptimized IR contains dead udiv and urem.
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc
index ea1fee040d..3019215c01 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc
@@ -48,7 +48,7 @@ TEST_F(GpuLdgTest, LdgForParamRead) {
       HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, param));
   std::unique_ptr<HloComputation> computation = builder.Build();
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(std::move(computation));
 
   CompileAndVerifyPtx(std::move(hlo_module), R"(
@@ -73,7 +73,7 @@ TEST_F(GpuLdgTest, LdgForNonParamRead) {
   builder.AddInstruction(HloInstruction::CreateTuple({add, square}));
   std::unique_ptr<HloComputation> computation = builder.Build();
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(std::move(computation));
 
   CompileAndVerifyPtx(std::move(hlo_module), R"(
@@ -95,7 +95,7 @@ TEST_F(GpuLdgTest, LdgForNonParamRead) {
 // reduce in the foreseeable future.  But if that turns out to be wrong, I give
 // you, future reader, permission to delete this test.
 TEST_F(GpuLdgTest, NoLdgWhenSharingBuffer) {
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   HloComputation::Builder builder(TestName());
 
   HloComputation* reduce_computation;
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc
index 14285459b5..ca0a78034d 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc
@@ -47,7 +47,7 @@ TEST_F(GpuNoAliasTest, Concat) {
 
   std::unique_ptr<HloComputation> computation = builder.Build();
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(std::move(computation));
 
   CompileAndVerifyIr(std::move(hlo_module),
diff --git a/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc b/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc
index c7f5112764..2dce7749bb 100644
--- a/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/while_transformer_test.cc
@@ -29,7 +29,7 @@ namespace {
 class WhileTransformerTest : public HloTestBase {
  protected:
   WhileTransformerTest()
-      : module_(CreateNewUnverifiedModule()),
+      : module_(CreateNewVerifiedModule()),
         induction_variable_shape_(ShapeUtil::MakeShape(S32, {})),
         data_shape_(ShapeUtil::MakeShape(F32, {8})),
         condition_result_shape_(ShapeUtil::MakeShape(PRED, {})) {}
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 1e7a6e197f..8b50cfa9ae 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -65,7 +65,7 @@ class HloComputationTest : public HloTestBase {
 };
 
 TEST_F(HloComputationTest, GetEmbeddedComputationsEmpty) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto negate_computation =
       module->AddEntryComputation(CreateNegateComputation());
   EXPECT_TRUE(negate_computation->MakeEmbeddedComputationsList().empty());
@@ -73,7 +73,7 @@ TEST_F(HloComputationTest, GetEmbeddedComputationsEmpty) {
 
 TEST_F(HloComputationTest, GetEmbeddedComputationsOneComputation) {
   // Create computation which calls one other computation.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto negate_computation =
       module->AddEmbeddedComputation(CreateNegateComputation());
   auto map_computation =
@@ -85,7 +85,7 @@ TEST_F(HloComputationTest, GetEmbeddedComputationsOneComputation) {
 
 TEST_F(HloComputationTest, GetEmbeddedComputationsDiamond) {
   // Create computations with a diamond-shaped callgraph.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto negate_computation =
       module->AddEmbeddedComputation(CreateNegateComputation());
   auto map1_computation =
@@ -119,7 +119,7 @@ TEST_F(HloComputationTest, PostOrderSingleton) {
   auto builder = HloComputation::Builder(TestName());
   auto constant = builder.AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(42.0f)));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->MakeInstructionPostOrder(), ElementsAre(constant));
 }
@@ -134,7 +134,7 @@ TEST_F(HloComputationTest, PostOrderSimple) {
       HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, constant));
   auto negate2 = builder.AddInstruction(
       HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, negate1));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->MakeInstructionPostOrder(),
               ElementsAre(constant, negate1, negate2));
@@ -170,7 +170,7 @@ TEST_F(HloComputationTest, PostOrderDisconnectedInstructions) {
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(42.0f)));
   auto constant4 = builder.AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(42.0f)));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->MakeInstructionPostOrder(),
               UnorderedElementsAre(constant1, constant2, constant3, constant4));
@@ -192,7 +192,7 @@ TEST_F(HloComputationTest, PostOrderWithMultipleRoots) {
       r0f32_, HloOpcode::kAdd, constant2, constant3));
   auto add3 = builder.AddInstruction(HloInstruction::CreateBinary(
       r0f32_, HloOpcode::kAdd, constant1, constant3));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   auto post_order = computation->MakeInstructionPostOrder();
   EXPECT_EQ(6, post_order.size());
@@ -217,7 +217,7 @@ TEST_F(HloComputationTest, VisitWithMultipleRoots) {
                                                       constant2, constant3));
   builder.AddInstruction(HloInstruction::CreateBinary(r0f32_, HloOpcode::kAdd,
                                                       constant1, constant3));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   // Visitor which keeps track of which instructions have been visited.
   class TestVisitor : public DfsHloVisitorWithDefault {
@@ -257,7 +257,7 @@ TEST_F(HloComputationTest, DeepCopyArray) {
   auto builder = HloComputation::Builder(TestName());
   auto constant = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({1.0, 2.0, 3.0})));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   auto copy = computation->DeepCopyInstruction(constant).ValueOrDie();
 
@@ -274,7 +274,7 @@ TEST_F(HloComputationTest, DeepCopyTuple) {
   auto tuple = builder.AddInstruction(
       HloInstruction::CreateTuple({constant1, constant2}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   auto tuple_copy = computation->DeepCopyInstruction(tuple).ValueOrDie();
 
@@ -376,7 +376,7 @@ TEST_F(HloComputationTest, DeepCopyToken) {
   // copied.
   auto builder = HloComputation::Builder(TestName());
   auto token = builder.AddInstruction(HloInstruction::CreateToken());
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   auto copy = computation->DeepCopyInstruction(token).ValueOrDie();
 
@@ -393,7 +393,7 @@ TEST_F(HloComputationTest, DeepCopyTokenTuple) {
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(42.0)));
   auto tuple =
       builder.AddInstruction(HloInstruction::CreateTuple({token, constant}));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   auto copy = computation->DeepCopyInstruction(tuple).ValueOrDie();
 
@@ -440,7 +440,7 @@ TEST_F(HloComputationTest, RemoveInstructionWithDuplicateOperand) {
       r0f32_, HloOpcode::kAdd, dead_negate, dead_negate));
   auto negate = builder.AddInstruction(
       HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, constant));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(4, computation->instruction_count());
   EXPECT_THAT(computation->root_instruction(), op::Negate(constant));
@@ -466,7 +466,7 @@ TEST_F(HloComputationTest, CloneWithControlDependency) {
       HloInstruction::CreateParameter(0, r0f32_, "param0"));
   auto negate = builder.AddInstruction(
       HloInstruction::CreateUnary(r0f32_, HloOpcode::kNegate, param));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation =
       module->AddEntryComputation(builder.Build(/*root_instruction=*/add));
 
@@ -505,7 +505,7 @@ TEST_F(HloComputationTest, Stringification) {
       2, PrecisionConfig::DEFAULT);
   builder.AddInstruction(
       HloInstruction::CreateDot(sout, x, reshape, dot_dnums, precision_config));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto* computation = module->AddEntryComputation(builder.Build());
 
   auto options = HloPrintOptions().set_print_metadata(false);
@@ -540,7 +540,7 @@ TEST_F(HloComputationTest, StringificationIndent) {
       2, PrecisionConfig::DEFAULT);
   builder.AddInstruction(
       HloInstruction::CreateDot(sout, x, reshape, dot_dnums, precision_config));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto* computation = module->AddEntryComputation(builder.Build());
 
   auto options =
@@ -576,7 +576,7 @@ TEST_F(HloComputationTest, StringificationCanonical) {
       2, PrecisionConfig::DEFAULT);
   builder.AddInstruction(
       HloInstruction::CreateDot(sout, x, reshape, dot_dnums, precision_config));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto* computation = module->AddEntryComputation(builder.Build());
 
   auto options = HloPrintOptions().set_print_metadata(false);
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
index 6a15b3440c..ff32faf298 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
@@ -387,7 +387,7 @@ TEST_F(FusionCostAnalysis, LoopFusion) {
         HloInstruction::CreateBinary(r2f32, HloOpcode::kSubtract, mul, clamp));
     auto tuple = HloInstruction::CreateTuple({sub, sub, mul, c1});
 
-    auto module = CreateNewUnverifiedModule();
+    auto module = CreateNewVerifiedModule();
     auto* computation = module->AddEntryComputation(builder.Build());
     auto* fusion = computation->CreateFusionInstruction(
         {sub, mul, exp, clamp, add}, HloInstruction::FusionKind::kLoop);
@@ -429,7 +429,7 @@ TEST_F(FusionCostAnalysis, NoLayout) {
   auto add = builder.AddInstruction(HloInstruction::CreateBinary(
       shape_with_layout, HloOpcode::kAdd, c1, broadcast));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto* computation = module->AddEntryComputation(builder.Build());
   auto* fusion = computation->CreateFusionInstruction(
       {add, broadcast}, HloInstruction::FusionKind::kLoop);
@@ -472,7 +472,7 @@ TEST_F(DomainCostAnalysis, DomainCost) {
   auto domain = builder.AddInstruction(
       HloInstruction::CreateDomain(tuple->shape(), tuple, nullptr, nullptr));
 
-  auto hlo_module = CreateNewUnverifiedModule();
+  auto hlo_module = CreateNewVerifiedModule();
   hlo_module->AddEntryComputation(builder.Build());
 
   EXPECT_EQ(hlo_module->entry_computation()->root_instruction(), domain);
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 6422346c10..e8eb7066f9 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -43,7 +43,7 @@ using ::testing::UnorderedElementsAre;
 class HloDataflowAnalysisTest : public HloTestBase,
                                 public ::testing::WithParamInterface<bool> {
  protected:
-  HloDataflowAnalysisTest() : module_(CreateNewUnverifiedModule()) {}
+  HloDataflowAnalysisTest() : module_(CreateNewVerifiedModule()) {}
 
   // Run dataflow analysis on the member module. For convenience returns a
   // reference to the generated analysis stored in analysis_.
diff --git a/tensorflow/compiler/xla/service/hlo_dce_test.cc b/tensorflow/compiler/xla/service/hlo_dce_test.cc
index 6c8095d397..1fa4259a3e 100644
--- a/tensorflow/compiler/xla/service/hlo_dce_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce_test.cc
@@ -59,7 +59,7 @@ TEST_F(HloDceTest, NoDeadCode) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       constant1->shape(), HloOpcode::kAdd, constant1, constant2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   EXPECT_EQ(3, computation->instruction_count());
@@ -110,7 +110,7 @@ TEST_F(HloDceTest, DeadParameters) {
   builder.AddInstruction(HloInstruction::CreateUnary(
       live_param->shape(), HloOpcode::kNegate, live_param));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   EXPECT_EQ(5, computation->instruction_count());
@@ -150,7 +150,7 @@ TEST_F(HloDceTest, ControlDependencies) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       constant1->shape(), HloOpcode::kAdd, constant1, constant2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Add a control dependency between two instructions.
@@ -175,7 +175,7 @@ TEST_F(HloDceTest, ControlDependencies) {
 
 // Tests that a dead call instruction is removed.
 TEST_F(HloDceTest, DeadInstructionWithCalledComputation) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   Shape shape = ShapeUtil::MakeShape(F32, {});
 
   // Called computation for the call instruction.
@@ -323,7 +323,7 @@ TEST_F(HloDceTest, CalledComputationWithNestedSideEffect) {
 }
 
 TEST_F(HloDceTest, RemoveDeadSubcomputation) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   HloComputation::Builder builder(TestName());
 
   HloComputation::Builder subcomp_builder("reduction_subcomp");
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
index 984a6266ab..3d8482065a 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
@@ -65,7 +65,7 @@ TEST_F(HloSchedulingTest, LastUseScheduledFirst) {
   auto sub = builder.AddInstruction(
       HloInstruction::CreateBinary(vec, HloOpcode::kSubtract, add, negate));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   HloMemoryScheduler scheduler([](const BufferValue& buffer) {
@@ -172,7 +172,7 @@ TEST_F(HloSchedulingTest, TuplesAreAccountedCorrectly) {
   builder.AddInstruction(HloInstruction::CreateBinary(r1f32, HloOpcode::kAdd,
                                                       tuple_elm, abs_abs2));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
                           ScheduleModule(*module,
@@ -218,7 +218,7 @@ TEST_F(HloSchedulingTest, MultiOutputFusionAccountedCorrectly) {
   builder.AddInstruction(
       HloInstruction::CreateBinary(r1f32, HloOpcode::kAdd, tuple_elm, exp));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto* computation = module->AddEntryComputation(builder.Build());
 
   auto fusion = computation->CreateFusionInstruction(
@@ -252,7 +252,7 @@ TEST_F(HloSchedulingTest, HeapSimulatorAccountsForSubcomputations) {
       HloInstruction::CreateParameter(0, r1f32, "cond_param"));
   HloInstruction* zero_vector =
       cond_builder.AddInstruction(HloInstruction::CreateConstant(
-          LiteralUtil::CreateR2<float>({{0, 0, 0, 0}})));
+          LiteralUtil::CreateR1<float>({0, 0, 0, 0})));
   cond_builder.AddInstruction(HloInstruction::CreateBinary(
       ShapeUtil::MakeShape(PRED, {}), HloOpcode::kNe, cond_param, zero_vector));
   auto cond_computation = module->AddEmbeddedComputation(cond_builder.Build());
diff --git a/tensorflow/compiler/xla/service/hlo_module_test.cc b/tensorflow/compiler/xla/service/hlo_module_test.cc
index 3ae67e4e5e..620cb7e01a 100644
--- a/tensorflow/compiler/xla/service/hlo_module_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_test.cc
@@ -63,7 +63,7 @@ class HloModuleTest : public HloTestBase {
 
 TEST_F(HloModuleTest, OneComputationPostOrder) {
   // Create a module with a single computation.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(CreateConstantComputation());
 
   EXPECT_THAT(module->MakeComputationPostOrder(),
@@ -72,7 +72,7 @@ TEST_F(HloModuleTest, OneComputationPostOrder) {
 
 TEST_F(HloModuleTest, TwoComputationsPostOrder) {
   // Create a module with two unconnected computations.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation1 = module->AddEntryComputation(CreateConstantComputation());
   auto computation2 =
       module->AddEmbeddedComputation(CreateConstantComputation());
@@ -88,7 +88,7 @@ TEST_F(HloModuleTest, TwoComputationsPostOrder) {
 
 TEST_F(HloModuleTest, CloneTest) {
   // Create and copy a module with a diamond call graph of computations.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation1 =
       module->AddEmbeddedComputation(CreateConstantComputation());
   auto computation2 =
@@ -111,7 +111,7 @@ TEST_F(HloModuleTest, CloneTest) {
 }
 
 TEST_F(HloModuleTest, CloneHasFusion) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
 
   // Create the fused computation.
   HloComputation* fused_computation;
@@ -154,7 +154,7 @@ TEST_F(HloModuleTest, CloneHasFusion) {
 
 TEST_F(HloModuleTest, DiamondComputationsPostOrder) {
   // Create a module with a diamond call graph of computations.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation1 =
       module->AddEmbeddedComputation(CreateConstantComputation());
   auto computation2 =
@@ -174,7 +174,7 @@ TEST_F(HloModuleTest, DiamondComputationsPostOrder) {
 
 TEST_F(HloModuleTest, LargeConstantToString) {
   // Create a module with a single computation.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto builder = HloComputation::Builder("Constant");
   std::vector<float> values(16, 42.0);
   builder.AddInstruction(
@@ -194,8 +194,8 @@ TEST_F(HloModuleTest, LargeConstantToString) {
 }
 
 TEST_F(HloModuleTest, UniqueModuleId) {
-  auto module_a = CreateNewUnverifiedModule();
-  auto module_b = CreateNewUnverifiedModule();
+  auto module_a = CreateNewVerifiedModule();
+  auto module_b = CreateNewVerifiedModule();
   EXPECT_NE(module_a->unique_id(), module_b->unique_id());
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_ordering_test.cc b/tensorflow/compiler/xla/service/hlo_ordering_test.cc
index 2ab8aa57f6..3ca77e60cd 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering_test.cc
@@ -53,7 +53,7 @@ TEST_F(HloOrderingTest, InstructionsInDifferentComputations) {
   //   %c = Constant(42.0f)
   //
   // This results in a diamond-shaped callgraph.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {});
 
   auto builder_c = HloComputation::Builder("C");
@@ -126,7 +126,7 @@ TEST_F(HloOrderingTest, InstructionsInWhileComputations) {
   //   %constant = Constant(1.0)
   //   return While(%constant, body, condition)
   //
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {});
 
   auto body_builder = HloComputation::Builder("body");
@@ -176,7 +176,7 @@ TEST_F(HloOrderingTest, InstructionsInWhileComputations) {
 
 TEST_F(HloOrderingTest, ParametersDefinedBeforeOthers) {
   // Entry parameter should always be defined before other instruction.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {});
   auto builder = HloComputation::Builder(TestName());
   auto constant = builder.AddInstruction(
@@ -209,7 +209,7 @@ TEST_F(HloOrderingTest, ValuesInWhileComputations) {
   //   %while = While(%constant, body, condition)
   //   %add = Add(%constant, %while)
   //
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {});
 
   auto body_builder = HloComputation::Builder("body");
@@ -407,7 +407,7 @@ TEST_F(HloOrderingTest,
   //   %dead = Constant(123.0)
   //
   // %root should interfere with %dead.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {});
 
   auto builder = HloComputation::Builder(TestName());
@@ -455,7 +455,7 @@ TEST_F(HloOrderingTest,
   //   ROOT %call = call({%c}), subcomputation
   //
   // %root should interfere with %dead.
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {});
 
   auto subbuilder = HloComputation::Builder(TestName() + ".sub");
diff --git a/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc b/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc
index 11994d99c9..c1073911ea 100644
--- a/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_subcomputation_unification_test.cc
@@ -66,7 +66,7 @@ class HloSubcomputationUnificationTest : public HloTestBase {
 };
 
 TEST_F(HloSubcomputationUnificationTest, UnifyIdentities) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto builder = HloComputation::Builder(TestName());
 
   auto callee1 =
@@ -103,7 +103,7 @@ TEST_F(HloSubcomputationUnificationTest, UnifyIdentities) {
 }
 
 TEST_F(HloSubcomputationUnificationTest, UnifyAdditions) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto builder = HloComputation::Builder(TestName());
 
   auto callee1 =
@@ -184,7 +184,7 @@ TEST_F(HloSubcomputationUnificationTest, DifferentParameterShapes) {
 // Regression test for b/31466798. Checks that entry_computation is still valid
 // after unification.
 TEST_F(HloSubcomputationUnificationTest, TwoIdenticalComputations) {
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   for (int i = 0; i < 2; ++i) {
     HloComputation::Builder builder("pow");
     auto x =
diff --git a/tensorflow/compiler/xla/service/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/instruction_fusion_test.cc
index 39904bd54b..6b48312649 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion_test.cc
@@ -117,7 +117,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastReshapeOfParameterUnfused) {
   auto reshape1 = builder.AddInstruction(
       HloInstruction::CreateReshape(ShapeUtil::MakeShape(S32, {1, 1}), param0));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(reshape1, computation->root_instruction());
   EXPECT_FALSE(
@@ -133,7 +133,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastSimpleReshapeOfParameterUnfused) {
   auto reshape1 = builder.AddInstruction(
       HloInstruction::CreateReshape(ShapeUtil::MakeShape(S32, {1, 1}), param0));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(reshape1, computation->root_instruction());
   EXPECT_FALSE(
@@ -149,7 +149,7 @@ TEST_F(InstructionFusionTest, PotentialBitcastTransposeOfParameterUnfused) {
   auto transpose1 = builder.AddInstruction(HloInstruction::CreateTranspose(
       ShapeUtil::MakeShape(S32, {}), param0, {}));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(transpose1, computation->root_instruction());
   EXPECT_FALSE(
diff --git a/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc b/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc
index 16fa80d53e..efeec96571 100644
--- a/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc
+++ b/tensorflow/compiler/xla/service/reduce_precision_insertion_test.cc
@@ -54,7 +54,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeUnaryInstruction) {
   HloInstruction* b = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, a));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -81,7 +81,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeUnaryScalarInstruction) {
   HloInstruction* b = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, a));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -111,7 +111,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeBinaryInstruction) {
   HloInstruction* c = builder.AddInstruction(
       HloInstruction::CreateBinary(shape, HloOpcode::kAdd, a, b));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -140,7 +140,7 @@ TEST_F(ReducePrecisionInsertionTest, BeforeZeroInputInstruction) {
   HloInstruction* b = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, a));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -173,7 +173,7 @@ TEST_F(ReducePrecisionInsertionTest, AvoidAddingDuplicateInstructions) {
   HloInstruction* d = builder.AddInstruction(
       HloInstruction::CreateBinary(shape, HloOpcode::kAdd, b, c));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -205,7 +205,7 @@ TEST_F(ReducePrecisionInsertionTest, AfterRootInstruction) {
   HloInstruction* b = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, a));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -242,7 +242,7 @@ TEST_F(ReducePrecisionInsertionTest, AfterNonRootInstruction) {
   HloInstruction* c = builder.AddInstruction(
       HloInstruction::CreateBinary(shape, HloOpcode::kAdd, a_cos, b_cos));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
   // Confirm expected graph before adding ops.
@@ -295,7 +295,7 @@ TEST_F(ReducePrecisionInsertionTest, ShouldReduceOutputPrecisionIsFalse) {
   HloInstruction* y = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, x));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected graph before adding ops.
@@ -321,7 +321,7 @@ TEST_F(ReducePrecisionInsertionTest, InsertionIsNotRecursive) {
   HloInstruction* b = builder.AddInstruction(
       HloInstruction::CreateReducePrecision(shape, a, 8, 23));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected state before adding ops.
@@ -348,7 +348,7 @@ TEST_F(ReducePrecisionInsertionTest, SkipRedundantReducePrecisionAfter) {
   HloInstruction* y = builder.AddInstruction(
       HloInstruction::CreateReducePrecision(shape, x, 5, 10));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected graph before adding ops.
@@ -376,7 +376,7 @@ TEST_F(ReducePrecisionInsertionTest, AddNonRedundantReducePrecision) {
   HloInstruction* y = builder.AddInstruction(
       HloInstruction::CreateReducePrecision(shape, x, 8, 23));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Confirm expected graph before adding ops.
@@ -402,7 +402,7 @@ TEST_F(ReducePrecisionInsertionTest, IgnoreOpsInsideFusionNode) {
       builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
   HloInstruction* y = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, x));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Manually fuse the kCos operation into a fusion operation.
@@ -438,7 +438,7 @@ TEST_F(ReducePrecisionInsertionTest, OpGetsInsertedInHeadOfFusionNode) {
       builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
   HloInstruction* y = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, x));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Manually fuse the kCos operation into a fusion operation.
@@ -485,7 +485,7 @@ TEST_F(ReducePrecisionInsertionTest, OpGetsInsertedInTailOfFusionNode) {
       builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
   HloInstruction* y = builder.AddInstruction(
       HloInstruction::CreateUnary(shape, HloOpcode::kCos, x));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
   // Manually fuse the kCos operation into a fusion operation.
diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc
index 7a565bf076..17cdaa74fc 100644
--- a/tensorflow/compiler/xla/service/transpose_folding_test.cc
+++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc
@@ -172,7 +172,7 @@ TEST_F(TransposeFoldingTest, FuseDotWithConstantOperands) {
   HloInstruction* mul = builder.AddInstruction(HloInstruction::CreateBinary(
       add->shape(), HloOpcode::kMultiply, add, sub));
 
-  auto module = CreateNewUnverifiedModule("fuse_with_constant_operands");
+  auto module = CreateNewVerifiedModule("fuse_with_constant_operands");
   HloComputation* entry_computation =
       module->AddEntryComputation(builder.Build(mul));
   HloInstruction* call = module->OutlineExpressionFromComputation(
@@ -247,7 +247,7 @@ TEST_F(TransposeFoldingTest, FoldConvDimSwapTransposeRhs) {
       conv_shape.ValueOrDie(), x, transpose_y,
       /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2)));
 
-  auto module = CreateNewUnverifiedModule("test_module");
+  auto module = CreateNewVerifiedModule("test_module");
   HloComputation* entry_computation =
       module->AddEntryComputation(builder.Build(conv));
   FoldTranspose(module.get());
@@ -302,7 +302,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeRhs) {
       conv_shape.ValueOrDie(), x, transpose_y,
       /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2)));
 
-  auto module = CreateNewUnverifiedModule("test_module");
+  auto module = CreateNewVerifiedModule("test_module");
   HloComputation* entry_computation =
       module->AddEntryComputation(builder.Build(conv));
   FoldTranspose(module.get());
@@ -362,7 +362,7 @@ TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) {
       conv_shape.ValueOrDie(), transpose_x, y,
       /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2)));
 
-  auto module = CreateNewUnverifiedModule("test_module");
+  auto module = CreateNewVerifiedModule("test_module");
   HloComputation* entry_computation =
       module->AddEntryComputation(builder.Build(conv));
   FoldTranspose(module.get());
@@ -428,7 +428,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeLhs) {
       conv_shape.ValueOrDie(), transpose_x, y,
       /*feature_group_count=*/1, window, dnums, DefaultPrecisionConfig(2)));
 
-  auto module = CreateNewUnverifiedModule("test_module");
+  auto module = CreateNewVerifiedModule("test_module");
   HloComputation* entry_computation =
       module->AddEntryComputation(builder.Build(conv));
   FoldTranspose(module.get());
-- 
GitLab


From 4b543f449c0bc49d0c9939eb02c17e281851c8ea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 14:31:34 -0800
Subject: [PATCH 0186/1554] This CL allows EagerTensors to support `len()` like
 numpy arrays. This should allow matplotlib plotting to work correctly with
 EagerTensors.

PiperOrigin-RevId: 221335277
---
 tensorflow/python/eager/tensor_test.py | 17 +++++++++++++++++
 tensorflow/python/framework/ops.py     |  6 ++++++
 2 files changed, 23 insertions(+)

diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index f61d847817..d0500a413d 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -128,6 +128,23 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     tensor = constant_op.constant(numpy_tensor)
     self.assertAllEqual(numpy_tensor.ndim, tensor.ndim)
 
+  def testLenAgreesWithNumpy(self):
+    numpy_tensor = np.asarray(1.0)
+    tensor = constant_op.constant(numpy_tensor)
+    with self.assertRaises(TypeError):
+      len(numpy_tensor)
+    with self.assertRaisesRegexp(
+        TypeError, r"Scalar tensor has no `len[(][)]`"):
+      len(tensor)
+
+    numpy_tensor = np.asarray([1.0, 2.0, 3.0])
+    tensor = constant_op.constant(numpy_tensor)
+    self.assertAllEqual(len(numpy_tensor), len(tensor))
+
+    numpy_tensor = np.asarray([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0]])
+    tensor = constant_op.constant(numpy_tensor)
+    self.assertAllEqual(len(numpy_tensor), len(tensor))
+
   def testCopy(self):
     t = constant_op.constant(1.0)
     tt = copy.copy(t)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index aaa12bf71f..84f7dc5c7c 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -890,6 +890,12 @@ class _EagerTensorBase(Tensor):
     """Returns the number of Tensor dimensions."""
     return self.shape.ndims
 
+  def __len__(self):
+    """Returns the length of the first dimension in the Tensor."""
+    if not self.shape.ndims:
+      raise TypeError("Scalar tensor has no `len()`")
+    return self._shape_tuple()[0]
+
   def _cpu_nograd(self):
     """A copy of this Tensor with contents backed by host memory.
 
-- 
GitLab


From cb7cd47376dbcb3faf77547324fcd0a5538a7e27 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Tue, 13 Nov 2018 14:46:24 -0800
Subject: [PATCH 0187/1554] [tf.data] Adds eager coverage for data/optimization
 tests.

PiperOrigin-RevId: 221337925
---
 .../optimization/assert_next_dataset_test.py  | 36 ++++-----
 .../optimization/filter_fusion_test.py        | 37 +++++----
 .../optimization/hoist_random_uniform_test.py | 34 +++++----
 .../optimization/latency_all_edges_test.py    | 55 ++++++--------
 .../optimization/make_numa_aware_test.py      | 12 +--
 .../optimization/map_and_batch_fusion_test.py | 12 +--
 .../map_and_filter_fusion_test.py             | 27 +++----
 .../optimization/map_fusion_test.py           | 28 +++----
 .../optimization/map_parallelization_test.py  | 21 ++----
 .../optimization/map_vectorization_test.py    | 11 ++-
 .../optimization/model_dataset_test.py        |  1 +
 .../optimization/noop_elimination_test.py     | 14 +---
 .../optimization/optimize_dataset_test.py     | 51 ++++---------
 .../shuffle_and_repeat_fusion_test.py         | 22 +++---
 .../kernel_tests/range_dataset_op_test.py     |  2 +-
 .../python/data/kernel_tests/test_base.py     | 75 +++++++++++++------
 16 files changed, 207 insertions(+), 231 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
index ed719a0ce9..9b8248a78d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
@@ -21,32 +21,27 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class AssertNextDatasetTest(test_base.DatasetTestBase):
 
   def testAssertNext(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
         optimization.assert_next(["Map"])).map(lambda x: x)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(get_next))
+    self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testAssertNextInvalid(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
         optimization.assert_next(["Whoops"])).map(lambda x: x)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          "Asserted Whoops transformation at offset 0 but encountered "
-          "Map transformation instead."):
-        sess.run(get_next)
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            "Asserted Whoops transformation at offset 0 but encountered "
+            "Map transformation instead."))
 
   def testAssertNextShort(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
@@ -54,14 +49,11 @@ class AssertNextDatasetTest(test_base.DatasetTestBase):
     options = dataset_ops.Options()
     options.experimental_autotune = False
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          "Asserted next 2 transformations but encountered only 1."):
-        sess.run(get_next)
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            "Asserted next 2 transformations but encountered only 1."))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
index 80a0d879dc..fe2c104e94 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
@@ -24,7 +24,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -58,6 +58,7 @@ def _filter_fusion_test_cases():
   return tuple(tests)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(*_filter_fusion_test_cases())
@@ -72,26 +73,22 @@ class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     options.experimental_filter_fusion = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-    with self.cached_session() as sess:
-      for x in range(5):
-        r = map_function(x)
-        filtered = False
-        for predicate in predicates:
-          if isinstance(r, tuple):
-            b = predicate(*r)  # Pass tuple as multiple arguments.
-          else:
-            b = predicate(r)
-          if not sess.run(b):
-            filtered = True
-            break
+    expected_output = []
+    for x in range(5):
+      r = map_function(x)
+      filtered = False
+      for predicate in predicates:
+        if isinstance(r, tuple):
+          b = predicate(*r)  # Pass tuple as multiple arguments.
+        else:
+          b = predicate(r)
+        if not self.evaluate(b):
+          filtered = True
+          break
 
-        if not filtered:
-          result = sess.run(get_next)
-          self.assertAllEqual(r, result)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+      if not filtered:
+        expected_output.append(r)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
index 9f7fbfeba0..e86b19438e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
@@ -22,10 +22,12 @@ from absl.testing import parameterized
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -58,23 +60,29 @@ def _hoist_random_uniform_test_cases():
   return tuple(tests)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _testDataset(self, dataset):
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
     previous_result = 0
-    with self.cached_session() as sess:
-      for _ in range(5):
-        result = sess.run(get_next)
-        self.assertLessEqual(1, result)
-        self.assertLessEqual(result, 10)
-        # This checks if the result is somehow random by checking if we are not
-        # generating the same values.
-        self.assertNotEqual(previous_result, result)
-        previous_result = result
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    if context.executing_eagerly():
+      iterator = dataset.__iter__()
+      get_next = iterator._next_internal  # pylint: disable=protected-access
+    else:
+      iterator = dataset.make_one_shot_iterator()
+      get_next = iterator.get_next
+    for _ in range(5):
+      result = self.evaluate(get_next())
+      self.assertLessEqual(1, result)
+      self.assertLessEqual(result, 10)
+      # This checks if the result is somehow random by checking if we are not
+      # generating the same values.
+      self.assertNotEqual(previous_result, result)
+      previous_result = result
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   @parameterized.named_parameters(*_hoist_random_uniform_test_cases())
   def testHoisting(self, function, will_optimize):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
index 7144d834f9..c2665cf69a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
@@ -22,10 +22,11 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.experimental.ops import stats_aggregator
 from tensorflow.python.data.experimental.ops import stats_options
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase):
 
   def testLatencyStatsOptimization(self):
@@ -39,22 +40,18 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase):
     options.experimental_stats.latency_all_edges = True
     options.experimental_stats.aggregator = aggregator
     dataset = dataset.with_options(options)
-    iterator = dataset.make_initializable_iterator()
-    get_next = iterator.get_next()
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[1],
+        requires_initialization=True,
+        num_test_iterations=1)
     summary_t = aggregator.get_summary()
-
-    with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertEqual(1 * 1, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      summary_str = sess.run(summary_t)
-      self._assertSummaryHasCount(summary_str,
-                                  "record_latency_TensorDataset/_1", 1)
-      self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4",
-                                  1)
-      self._assertSummaryHasCount(summary_str,
-                                  "record_latency_PrefetchDataset/_6", 1)
+    summary_str = self.evaluate(summary_t)
+    self._assertSummaryHasCount(summary_str, "record_latency_TensorDataset/_1",
+                                1)
+    self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4", 1)
+    self._assertSummaryHasCount(summary_str,
+                                "record_latency_PrefetchDataset/_6", 1)
 
   def testLatencyStatsOptimizationV2(self):
     aggregator = stats_aggregator.StatsAggregator()
@@ -65,22 +62,18 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase):
     options = dataset_ops.Options()
     options.experimental_stats = stats_options.StatsOptions(aggregator)
     dataset = dataset.with_options(options)
-    iterator = dataset.make_initializable_iterator()
-    get_next = iterator.get_next()
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[1],
+        requires_initialization=True,
+        num_test_iterations=1)
     summary_t = aggregator.get_summary()
-
-    with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertEqual(1, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      summary_str = sess.run(summary_t)
-      self._assertSummaryHasCount(summary_str,
-                                  "record_latency_TensorDataset/_1", 1)
-      self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4",
-                                  1)
-      self._assertSummaryHasCount(summary_str,
-                                  "record_latency_PrefetchDataset/_6", 1)
+    summary_str = self.evaluate(summary_t)
+    self._assertSummaryHasCount(summary_str, "record_latency_TensorDataset/_1",
+                                1)
+    self._assertSummaryHasCount(summary_str, "record_latency_MapDataset/_4", 1)
+    self._assertSummaryHasCount(summary_str,
+                                "record_latency_PrefetchDataset/_6", 1)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
index 6191a7db08..2386dd5f11 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
@@ -21,10 +21,11 @@ from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MakeNumaAwareTest(test_base.DatasetTestBase):
 
   def testMakeNumaAware(self):
@@ -34,13 +35,8 @@ class MakeNumaAwareTest(test_base.DatasetTestBase):
     options = dataset_ops.Options()
     options.experimental_numa_aware = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertAllEqual([x * x for x in range(10)], sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(
+        dataset, expected_output=[[x * x for x in range(10)]])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
index ddf3cbbcc3..67f3ceeabe 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
@@ -20,10 +20,11 @@ from __future__ import print_function
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MapAndBatchFusionTest(test_base.DatasetTestBase):
 
   def testMapAndBatchFusion(self):
@@ -33,13 +34,8 @@ class MapAndBatchFusionTest(test_base.DatasetTestBase):
     options = dataset_ops.Options()
     options.experimental_map_and_batch_fusion = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertAllEqual([x * x for x in range(10)], sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(
+        dataset, expected_output=[[x * x for x in range(10)]])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
index 3b4ca62340..a898c38440 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
@@ -24,7 +24,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -62,23 +62,20 @@ def _map_and_filter_fusion_test_cases():
   return tuple(tests)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _testMapAndFilter(self, dataset, function, predicate):
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-    with self.cached_session() as sess:
-      for x in range(10):
-        r = function(x)
-        if isinstance(r, tuple):
-          b = predicate(*r)  # Pass tuple as multiple arguments.
-        else:
-          b = predicate(r)
-        if sess.run(b):
-          result = sess.run(get_next)
-          self.assertAllEqual(r, result)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    expected_output = []
+    for x in range(10):
+      r = function(x)
+      if isinstance(r, tuple):
+        b = predicate(*r)  # Pass tuple as multiple arguments.
+      else:
+        b = predicate(r)
+      if self.evaluate(b):
+        expected_output.append(r)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   @parameterized.named_parameters(*_map_and_filter_fusion_test_cases())
   def testMapFilterFusion(self, function, predicate):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
index ec63ad7200..47a1b0896c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
@@ -22,7 +22,7 @@ from absl.testing import parameterized
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -62,6 +62,7 @@ def _map_fusion_test_cases():
   return tuple(tests)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(*_map_fusion_test_cases())
@@ -75,21 +76,16 @@ class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     options.experimental_map_fusion = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-    with self.cached_session() as sess:
-      for x in range(5):
-        result = sess.run(get_next)
-        r = x
-        for function in functions:
-          if isinstance(r, tuple):
-            r = function(*r)  # Pass tuple as multiple arguments.
-          else:
-            r = function(r)
-        self.assertAllEqual(r, result)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    expected_output = []
+    for x in range(5):
+      r = x
+      for function in functions:
+        if isinstance(r, tuple):
+          r = function(*r)  # Pass tuple as multiple arguments.
+        else:
+          r = function(r)
+      expected_output.append(r)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
index c95f7b2eb1..042b9ce54b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
@@ -23,8 +23,8 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -58,6 +58,7 @@ def _map_parallelization_test_cases():
           ("AssertWithRandom", assert_with_random, False))
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(*_map_parallelization_test_cases())
@@ -68,21 +69,9 @@ class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     options.experimental_map_parallelization = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      for x in range(5):
-        result = sess.run(get_next)
-        # No need to run the pipeline if it was not optimized.  Also the results
-        # might be hard to check because of random.
-        if not should_optimize:
-          return
-        r = function(x)
-        self.assertAllEqual(r, result)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    if should_optimize:
+      self.assertDatasetProduces(
+          dataset, expected_output=[function(x) for x in range(5)])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index f10b66ff69..470de580e8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import bitwise_ops
 from tensorflow.python.ops import check_ops
@@ -319,6 +320,7 @@ def _generate_optimization_test_cases():
   } for x in test_cases for num_parallel_calls in (None, 12)]
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _get_test_datasets(self,
@@ -366,7 +368,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
                                                      num_parallel_calls)
     self.assertDatasetsEqual(unoptimized, optimized)
 
-  def testOptimizationBadMapFn(self):
+  # TODO(b/117581999): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationBadMapFn(self):
     # Test map functions that give an error
     def map_fn(x):
       # x has leading dimension 5, this will raise an error
@@ -394,7 +397,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
         base_dataset, map_fn, expect_optimized=True)
     self.assertDatasetsEqual(optimized, unoptimized)
 
-  def testOptimizationIgnoreStateful(self):
+  # TODO(b/117581999): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationIgnoreStateful(self):
 
     def map_fn(x):
       with ops.control_dependencies([check_ops.assert_equal(x, 0)]):
@@ -420,7 +424,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
         base_dataset, map_fn, expect_optimized=False)
     self.assertDatasetsEqual(unoptimized, optimized)
 
-  def testOptimizationIgnoreRaggedMap(self):
+  # TODO(b/117581999): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationIgnoreRaggedMap(self):
     # Don't optimize when the output of the map fn shapes are unknown.
     def map_fn(x):
       return array_ops.tile(x, x)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index 5b49bdf453..35dc9ecbce 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+# TODO(b/117581999): Add eager coverage for the following tests.
 class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testModelMap(self):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
index ddcd7f4da4..d957e8007c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
@@ -22,11 +22,12 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class NoopEliminationTest(test_base.DatasetTestBase):
 
   def testNoopElimination(self):
@@ -43,16 +44,7 @@ class NoopEliminationTest(test_base.DatasetTestBase):
     options = dataset_ops.Options()
     options.experimental_noop_elimination = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      for x in range(5):
-        result = sess.run(get_next)
-        self.assertAllEqual(result, x)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(dataset, expected_output=range(5))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 739b6a9bf4..510b197ddf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -24,25 +24,24 @@ from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class OptimizeDatasetTest(test_base.DatasetTestBase):
 
   def testOptimizationStatefulFunction(self):
     dataset = dataset_ops.Dataset.range(10).map(
         lambda _: random_ops.random_uniform([])).batch(10)
     dataset = dataset_ops._OptimizeDataset(dataset, [])
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
+    get_next = self.getNext(dataset)
+    self.evaluate(get_next())
 
-    with self.cached_session() as sess:
-      sess.run(get_next)
-
-  def testOptimizationLargeInputFromTensor(self):
+  # TODO(b/117581999): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationLargeInputFromTensor(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
     dataset = dataset_ops.Dataset.from_tensors(input_t)
     dataset = dataset_ops._OptimizeDataset(dataset, [])
@@ -54,7 +53,8 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
       sess.run(get_next)
 
-  def testOptimizationLargeInputFromTensorSlices(self):
+  # TODO(b/117581999): Add eager coverage for the following tests.
+  def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
     dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
     dataset = dataset_ops._OptimizeDataset(dataset, [])
@@ -78,13 +78,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(1)
     dataset = dataset.flat_map(flat_map_fn)
     dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertEquals(0, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationThreadPoolDataset(self):
     dataset = dataset_ops.Dataset.range(10).batch(10)
@@ -95,14 +89,10 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
             2, display_name="private_thread_pool_%d" % 2))
 
     dataset = dataset_ops._OptimizeDataset(dataset, [])
-    iterator = dataset.make_initializable_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual(list(range(10)), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[list(range(10))],
+        requires_initialization=True)
 
   def testOptimizationNonSerializable(self):
     dataset = dataset_ops.Dataset.from_tensors(0)
@@ -113,13 +103,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     dataset = dataset.skip(0)  # Should be removed by noop elimination
     dataset = dataset.cache()
     dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertEquals(0, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationNonSerializableAsDirectInput(self):
     """Tests that non-serializable dataset can be OptimizeDataset's input.
@@ -127,12 +111,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.from_tensors(0)
     dataset = dataset.apply(optimization.non_serializable())
     dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-    with self.cached_session() as sess:
-      self.assertEquals(0, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(dataset, expected_output=[0])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
index 36582f449f..f1d00a59c4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
@@ -21,9 +21,11 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase):
 
   def testShuffleAndRepeatFusion(self):
@@ -32,17 +34,17 @@ class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase):
     options = dataset_ops.Options()
     options.experimental_shuffle_and_repeat_fusion = True
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
+    get_next = self.getNext(dataset)
 
-    with self.cached_session() as sess:
-      for _ in range(2):
-        results = []
-        for _ in range(10):
-          results.append(sess.run(get_next))
-        self.assertAllEqual([x for x in range(10)], sorted(results))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    for _ in range(2):
+      results = []
+      for _ in range(10):
+        results.append(self.evaluate(get_next()))
+      self.assertAllEqual([x for x in range(10)], sorted(results))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
index b71e6b2ea4..9fc79707d0 100644
--- a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
@@ -56,7 +56,7 @@ class RangeDatasetTest(test_base.DatasetTestBase):
     start, stop, step = 2, 10, 0
     dataset = dataset_ops.Dataset.range(start, stop, step)
     self.assertDatasetProduces(
-        dataset, expected_err=(errors.InvalidArgumentError, ""))
+        dataset, expected_error=(errors.InvalidArgumentError, ""))
 
   def testNegativeStep(self):
     start, stop, step = 2, 10, -1
diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index edb3eff3c1..ca853da3f6 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -38,7 +38,7 @@ class DatasetTestBase(test.TestCase):
     self.assertAllEqual(a.values, b.values)
     self.assertAllEqual(a.dense_shape, b.dense_shape)
 
-  def getNext(self, dataset):
+  def getNext(self, dataset, requires_initialization=False):
     """Returns a callable that returns the next element of the dataset.
 
     Example use:
@@ -50,19 +50,25 @@ class DatasetTestBase(test.TestCase):
     ```
 
     Args:
-      dataset: A dataset whose next element is returned
-
+      dataset: A dataset whose elements will be returned.
+      requires_initialization: Indicates that when the test is executed in graph
+        mode, it should use an initializable iterator to iterate through the
+        dataset (e.g. when it contains stateful nodes). Defaults to False.
     Returns:
-      A callable that returns the next element of `dataset`
+      A callable that returns the next element of `dataset`.
     """
-    it = dataset.make_one_shot_iterator()
     if context.executing_eagerly():
-      return it.get_next
+      iterator = dataset.__iter__()
+      return iterator._next_internal  # pylint: disable=protected-access
     else:
-      nxt = it.get_next()
-      return lambda: nxt
+      if requires_initialization:
+        iterator = dataset.make_initializable_iterator()
+        self.evaluate(iterator.initializer)
+      else:
+        iterator = dataset.make_one_shot_iterator()
+      return iterator.get_next
 
-  def _compare_output_to_expected(self, result_values, expected_values):
+  def _compareOutputToExpected(self, result_values, expected_values):
     for i in range(len(result_values)):
       if sparse_tensor.is_sparse(result_values[i]):
         self.assertSparseValuesEqual(result_values[i], expected_values[i])
@@ -70,24 +76,51 @@ class DatasetTestBase(test.TestCase):
         self.assertAllEqual(result_values[i], expected_values[i])
 
   def assertDatasetProduces(self,
-                            input_dataset,
+                            dataset,
                             expected_output=None,
-                            expected_err=None,
-                            create_iterator_twice=True):
+                            expected_error=None,
+                            requires_initialization=False,
+                            num_test_iterations=2):
+    """Asserts that a dataset produces the expected output / error.
 
-    if expected_err:
-      with self.assertRaisesWithPredicateMatch(expected_err[0],
-                                               expected_err[1]):
-        get_next = self.getNext(input_dataset)
+    Args:
+      dataset: A dataset to check for the expected output / error.
+      expected_output: A list of elements that the dataset is expected to
+        produce.
+      expected_error: A tuple `(type, predicate)` identifying the expected error
+        `dataset` should raise. The `type` should match the expected exception
+        type, while `predicate` should either be 1) a unary function that inputs
+        the raised exception and returns a boolean indicator of success or 2) a
+        regular expression that is expected to match the error message
+        partially.
+      requires_initialization: Indicates that when the test is executed in graph
+        mode, it should use an initializable iterator to iterate through the
+        dataset (e.g. when it contains stateful nodes). Defaults to False.
+      num_test_iterations: Number of times `dataset` will be iterated. Defaults
+        to 2.
+    """
+    self.assertTrue(
+        expected_error is not None or expected_output is not None,
+        "Exactly one of expected_output or expected error should be provided.")
+    if expected_error:
+      self.assertTrue(
+          expected_output is None,
+          "Exactly one of expected_output or expected error should be provided."
+      )
+      with self.assertRaisesWithPredicateMatch(expected_error[0],
+                                               expected_error[1]):
+        get_next = self.getNext(
+            dataset, requires_initialization=requires_initialization)
         self.evaluate(get_next())
       return
-    repeated = 2 if create_iterator_twice else 1
-    for _ in range(repeated):
-      get_next = self.getNext(input_dataset)
+    self.assertGreater(num_test_iterations, 0)
+    for _ in range(num_test_iterations):
+      get_next = self.getNext(
+          dataset, requires_initialization=requires_initialization)
       result = []
       for _ in range(len(expected_output)):
         result.append(self.evaluate(get_next()))
-      self._compare_output_to_expected(result, expected_output)
+      self._compareOutputToExpected(result, expected_output)
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next())
       with self.assertRaises(errors.OutOfRangeError):
@@ -132,7 +165,7 @@ class DatasetTestBase(test.TestCase):
     try:
       self.evaluate(next1())
       raise ValueError(
-          'Expected dataset to raise an error of type %s, but it did not.' %
+          "Expected dataset to raise an error of type %s, but it did not." %
           repr(exception_class))
     except exception_class as e:
       expected_message = e.message
-- 
GitLab


From 9c2afd44aa6d6bd697be17b2eb80ef32338ec15e Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Tue, 13 Nov 2018 15:01:48 -0800
Subject: [PATCH 0188/1554] Delete `num_replicas` API from
 DistributionStrategy.

PiperOrigin-RevId: 221340692
---
 .../contrib/distribute/python/mirrored_strategy.py   |  4 ----
 .../python/mirrored_strategy_multigpu_test.py        |  7 +------
 .../contrib/distribute/python/one_device_strategy.py |  4 ----
 .../distribute/python/parameter_server_strategy.py   |  4 ----
 tensorflow/contrib/distribute/python/tpu_strategy.py | 12 ++++--------
 tensorflow/python/training/distribute.py             |  8 --------
 6 files changed, 5 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 2d75024e7a..70855d8f01 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -658,10 +658,6 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
   def value_container(self, val):
     return values.value_container(val)
 
-  @property
-  def num_replicas(self):
-    return len(self._devices)
-
   @property
   def num_replicas_in_sync(self):
     return len(self._devices)
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 1fd18e09c0..2a6561e6b1 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -88,11 +88,6 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
       self.skipTest("Not GPU test")
     self._test_replica_id(self._get_distribution_strategy())
 
-  def testNumReplicas(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self.assertEqual(2, self._get_distribution_strategy().num_replicas)
-
   def testNumReplicasInSync(self):
     if not GPU_TEST:
       self.skipTest("Not GPU test")
@@ -1210,7 +1205,7 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
       # values on each of the replicas.
       self.assertEqual(2.0, self.evaluate(dist.read_var(replica_local_var)))
       # Assigning 6.0 in cross replica context will assign a value of
-      # 6.0/num_replicas to each replica.
+      # 6.0/num_replicas_in_sync to each replica.
       tlv_ops = replica_local_var.assign(6.0)
       self.evaluate(tlv_ops)
       # On reading the replica local var we should get the assigned value back.
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index f1aac56c46..622194e8a2 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -152,10 +152,6 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
   def value_container(self, value):
     return value
 
-  @property
-  def num_replicas(self):
-    return 1
-
   @property
   def num_replicas_in_sync(self):
     return 1
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 790b37f860..438b91bc8d 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -449,10 +449,6 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     session_config.device_filters.extend(
         ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
 
-  @property
-  def num_replicas(self):
-    return len(self._compute_devices)
-
   @property
   def num_replicas_in_sync(self):
     return len(self._compute_devices)
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 7db5f108ee..5ef59bf74d 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -148,7 +148,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     self._host_device = self.get_host_cpu_device(0)
     self._tpu_devices = sorted(device_map.keys())
     # Only create variables for the number of replicas we're running.
-    self._tpu_devices = self._tpu_devices[:self.num_replicas]
+    self._tpu_devices = self._tpu_devices[:self.num_replicas_in_sync]
 
     # TODO(sourabhbajaj): Remove this once performance of running one step
     # at a time is comparable to multiple steps.
@@ -335,7 +335,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     self._outer_control_flow_context = (
         ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
 
-    replicate_inputs = [[]] * self.num_replicas
+    replicate_inputs = [[]] * self.num_replicas_in_sync
     replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs)
     del self._outer_control_flow_context
     ctx.run_op = control_flow_ops.group(replicate_outputs, enqueue_ops)
@@ -443,7 +443,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     if values._enclosing_tpu_context() is not None:  # pylint: disable=protected-access
       if aggregation == vs.VariableAggregation.MEAN:
         # TODO(jhseu):  Revisit once we support model-parallelism.
-        value *= (1. / self.num_replicas)
+        value *= (1. / self.num_replicas_in_sync)
       elif aggregation != vs.VariableAggregation.SUM:
         raise NotImplementedError(
             "Currently only support sum & mean in TPUStrategy.")
@@ -513,10 +513,6 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     del destinations
     return tensor
 
-  @property
-  def num_replicas(self):
-    return self._num_cores_override or self._tpu_metadata.num_cores
-
   @property
   def num_hosts(self):
     return self._tpu_metadata.num_hosts
@@ -527,7 +523,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
 
   @property
   def num_replicas_in_sync(self):
-    return self.num_replicas
+    return self._num_cores_override or self._tpu_metadata.num_cores
 
   @property
   def between_graph(self):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index f515df0de1..f2f832f314 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -920,14 +920,6 @@ class DistributionStrategy(object):
   def require_static_shapes(self):
     return self._require_static_shapes
 
-  @property
-  def num_replicas(self):
-    """Returns number of replicas, for purposes of averaging across replicas.
-
-    DEPRECATED: use `num_replicas_in_sync` instead.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
   @property
   def num_replicas_in_sync(self):
     """Returns number of replicas over which gradients are aggregated."""
-- 
GitLab


From a17b4fc17b454c5150dca32d2b0828970a7afa06 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 13 Nov 2018 16:15:11 -0800
Subject: [PATCH 0189/1554] Use tfe.defun instead of tf.defun in batch_function

PiperOrigin-RevId: 221353553
---
 tensorflow/contrib/batching/python/ops/batch_ops.py  | 12 ++++++++----
 .../contrib/batching/python/ops/batch_ops_test.py    |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/batching/python/ops/batch_ops.py b/tensorflow/contrib/batching/python/ops/batch_ops.py
index 55faad983f..3e4d0dc1ce 100644
--- a/tensorflow/contrib/batching/python/ops/batch_ops.py
+++ b/tensorflow/contrib/batching/python/ops/batch_ops.py
@@ -18,8 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import function
+from tensorflow.python.eager import function
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.ops import gen_batch_ops
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
@@ -101,12 +102,15 @@ def batch_function(num_batch_threads,
   def decorator(fn):  # pylint: disable=missing-docstring
 
     def decorated(*args):  # pylint: disable=missing-docstring
-      types = [arg.dtype for arg in args]
 
-      @function.Defun(*types)
+      @function.defun()
       def computation(*computation_args):
         return fn(*computation_args)
 
+      computation = computation.get_concrete_function(
+          *[tensor_spec.TensorSpec(dtype=x.dtype, shape=x.shape, name=str(i))
+            for i, x in enumerate(args)])
+
       with ops.name_scope("batch") as name:
         for a in args:
           if not isinstance(a, ops.Tensor):
@@ -123,7 +127,7 @@ def batch_function(num_batch_threads,
             f=computation,
             in_tensors=list(args),
             captured_tensors=computation.captured_inputs,
-            Tout=[o.type for o in computation.definition.signature.output_arg])
+            Tout=[o.dtype for o in computation.outputs])
 
     return decorated
 
diff --git a/tensorflow/contrib/batching/python/ops/batch_ops_test.py b/tensorflow/contrib/batching/python/ops/batch_ops_test.py
index 01ee8703a9..9109b9c1c9 100644
--- a/tensorflow/contrib/batching/python/ops/batch_ops_test.py
+++ b/tensorflow/contrib/batching/python/ops/batch_ops_test.py
@@ -219,6 +219,7 @@ class BatchOpsTest(test.TestCase):
 
       @batch_ops.batch_function(1, 10, 100000)
       def computation(in_t):
+        self.assertTrue(in_t.shape is not None)
         return in_t + 1
 
       inp = array_ops.placeholder(dtype=dtypes.int32, shape=[1])
-- 
GitLab


From c4a6eb5857a4da600763a22ebc2a8fa1b19317cb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 16:36:30 -0800
Subject: [PATCH 0190/1554] Add DCT-I to tf.signal.dct.

PiperOrigin-RevId: 221356811
---
 tensorflow/python/kernel_tests/BUILD          |  1 +
 .../python/kernel_tests/dct_ops_test.py       | 52 ++++++++++++----
 tensorflow/python/ops/spectral_ops.py         | 60 ++++++++++++-------
 3 files changed, 80 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 48639a4518..002d5fbca7 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2630,6 +2630,7 @@ cuda_py_test(
     name = "dct_ops_test",
     srcs = ["dct_ops_test.py"],
     additional_deps = [
+        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
diff --git a/tensorflow/python/kernel_tests/dct_ops_test.py b/tensorflow/python/kernel_tests/dct_ops_test.py
index c9d0167608..27fdf8775e 100644
--- a/tensorflow/python/kernel_tests/dct_ops_test.py
+++ b/tensorflow/python/kernel_tests/dct_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import importlib
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.ops import spectral_ops
@@ -40,6 +41,20 @@ def try_import(name):  # pylint: disable=invalid-name
 fftpack = try_import("scipy.fftpack")
 
 
+def _np_dct1(signals, norm=None):
+  """Computes the DCT-I manually with NumPy."""
+  # X_k = (x_0 + (-1)**k * x_{N-1} +
+  #       2 * sum_{n=0}^{N-2} x_n * cos(\frac{pi}{N-1} * n * k)  k=0,...,N-1
+  del norm
+  dct_size = signals.shape[-1]
+  dct = np.zeros_like(signals)
+  for k in range(dct_size):
+    phi = np.cos(np.pi * np.arange(1, dct_size - 1) * k / (dct_size - 1))
+    dct[..., k] = 2 * np.sum(signals[..., 1:-1] * phi, axis=-1) + (
+        signals[..., 0] + (-1) ** k * signals[..., -1])
+  return dct
+
+
 def _np_dct2(signals, norm=None):
   """Computes the DCT-II manually with NumPy."""
   # X_k = sum_{n=0}^{N-1} x_n * cos(\frac{pi}{N} * (n + 0.5) * k)  k=0,...,N-1
@@ -81,11 +96,11 @@ def _np_dct3(signals, norm=None):
   return dct
 
 
-NP_DCT = {2: _np_dct2, 3: _np_dct3}
-NP_IDCT = {2: _np_dct3, 3: _np_dct2}
+NP_DCT = {1: _np_dct1, 2: _np_dct2, 3: _np_dct3}
+NP_IDCT = {1: _np_dct1, 2: _np_dct3, 3: _np_dct2}
 
 
-class DCTOpsTest(test.TestCase):
+class DCTOpsTest(parameterized.TestCase, test.TestCase):
 
   def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4):
     """Compares (I)DCT to SciPy (if available) and a NumPy implementation."""
@@ -106,26 +121,39 @@ class DCTOpsTest(test.TestCase):
     tf_dct_idct = spectral_ops.dct(
         tf_idct, type=dct_type, norm=norm).eval()
     if norm is None:
-      tf_idct_dct *= 0.5 / signals.shape[-1]
-      tf_dct_idct *= 0.5 / signals.shape[-1]
+      if dct_type == 1:
+        tf_idct_dct *= 0.5 / (signals.shape[-1] - 1)
+        tf_dct_idct *= 0.5 / (signals.shape[-1] - 1)
+      else:
+        tf_idct_dct *= 0.5 / signals.shape[-1]
+        tf_dct_idct *= 0.5 / signals.shape[-1]
     self.assertAllClose(signals, tf_idct_dct, atol=atol, rtol=rtol)
     self.assertAllClose(signals, tf_dct_idct, atol=atol, rtol=rtol)
 
-  def test_random(self):
+  @parameterized.parameters([
+      [[2]], [[3]], [[10]], [[2, 20]], [[2, 3, 25]]])
+  def test_random(self, shape):
     """Test randomly generated batches of data."""
     with spectral_ops_test_util.fft_kernel_label_map():
       with self.session(use_gpu=True):
-        for shape in ([1], [2], [3], [10], [2, 20], [2, 3, 25]):
-          signals = np.random.rand(*shape).astype(np.float32)
-          for norm in (None, "ortho"):
-            self._compare(signals, norm, 2)
-            self._compare(signals, norm, 3)
+        signals = np.random.rand(*shape).astype(np.float32)
+        # Normalization not implemented for orthonormal.
+        self._compare(signals, norm=None, dct_type=1)
+        for norm in (None, "ortho"):
+          self._compare(signals, norm, 2)
+          self._compare(signals, norm, 3)
 
   def test_error(self):
     signals = np.random.rand(10)
     # Unsupported type.
     with self.assertRaises(ValueError):
-      spectral_ops.dct(signals, type=1)
+      spectral_ops.dct(signals, type=5)
+    # DCT-I normalization not implemented.
+    with self.assertRaises(ValueError):
+      spectral_ops.dct(signals, type=1, norm="ortho")
+    # DCT-I requires at least two inputs.
+    with self.assertRaises(ValueError):
+      spectral_ops.dct(np.random.rand(1), type=1)
     # Unknown normalization.
     with self.assertRaises(ValueError):
       spectral_ops.dct(signals, norm="bad")
diff --git a/tensorflow/python/ops/spectral_ops.py b/tensorflow/python/ops/spectral_ops.py
index 4dcc90aefa..9d914513f1 100644
--- a/tensorflow/python/ops/spectral_ops.py
+++ b/tensorflow/python/ops/spectral_ops.py
@@ -165,37 +165,48 @@ irfft3d = _irfft_wrapper(gen_spectral_ops.irfft3d, 3, "irfft3d")
 tf_export("spectral.irfft3d")(irfft3d)
 
 
-def _validate_dct_arguments(dct_type, n, axis, norm):
+def _validate_dct_arguments(input_tensor, dct_type, n, axis, norm):
+  """Checks that DCT/IDCT arguments are compatible and well formed."""
   if n is not None:
     raise NotImplementedError("The DCT length argument is not implemented.")
   if axis != -1:
     raise NotImplementedError("axis must be -1. Got: %s" % axis)
-  if dct_type not in (2, 3):
-    raise ValueError("Only Types II and III (I)DCT are supported.")
+  if dct_type not in (1, 2, 3):
+    raise ValueError("Only Types I, II and III (I)DCT are supported.")
+  if dct_type == 1:
+    if norm == "ortho":
+      raise ValueError("Normalization is not supported for the Type-I DCT.")
+    if input_tensor.shape[-1] is not None and input_tensor.shape[-1] < 2:
+      raise ValueError(
+          "Type-I DCT requires the dimension to be greater than one.")
+
   if norm not in (None, "ortho"):
     raise ValueError(
         "Unknown normalization. Expected None or 'ortho', got: %s" % norm)
 
 
-# TODO(rjryan): Implement `type`, `n` and `axis` parameters.
+# TODO(rjryan): Implement `n` and `axis` parameters.
 @tf_export("spectral.dct")
 def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disable=redefined-builtin
   """Computes the 1D [Discrete Cosine Transform (DCT)][dct] of `input`.
 
-  Currently only Types II and III are supported. Type II is implemented using a
-  length `2N` padded `tf.spectral.rfft`, as described here:
-  https://dsp.stackexchange.com/a/10606. Type III is a fairly straightforward
-  inverse of Type II (i.e. using a length `2N` padded `tf.spectral.irfft`).
+  Currently only Types I, II and III are supported.
+  Type I is implemented using a length `2N` padded `tf.spectral.rfft`.
+  Type II is implemented using a length `2N` padded `tf.spectral.rfft`, as
+  described here:
+  https://dsp.stackexchange.com/a/10606.
+  Type III is a fairly straightforward inverse of Type II
+  (i.e. using a length `2N` padded `tf.spectral.irfft`).
 
   @compatibility(scipy)
-  Equivalent to scipy.fftpack.dct for Type-II and Type-III DCT.
+  Equivalent to scipy.fftpack.dct for Type-I, Type-II and Type-III DCT.
   https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html
   @end_compatibility
 
   Args:
     input: A `[..., samples]` `float32` `Tensor` containing the signals to
       take the DCT of.
-    type: The DCT type to perform. Must be 2 or 3.
+    type: The DCT type to perform. Must be 1, 2 or 3.
     n: For future expansion. The length of the transform. Must be `None`.
     axis: For future expansion. The axis to compute the DCT along. Must be `-1`.
     norm: The normalization to apply. `None` for no normalization or `'ortho'`
@@ -206,12 +217,13 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
     A `[..., samples]` `float32` `Tensor` containing the DCT of `input`.
 
   Raises:
-    ValueError: If `type` is not `2` or `3`, `n` is not `None, `axis` is not
-      `-1`, or `norm` is not `None` or `'ortho'`.
+    ValueError: If `type` is not `1`, `2` or `3`, `n` is not `None, `axis` is
+      not `-1`, or `norm` is not `None` or `'ortho'`.
+    ValueError: If `type` is `1` and `norm` is `ortho`.
 
   [dct]: https://en.wikipedia.org/wiki/Discrete_cosine_transform
   """
-  _validate_dct_arguments(type, n, axis, norm)
+  _validate_dct_arguments(input, type, n, axis, norm)
   with _ops.name_scope(name, "dct", [input]):
     # We use the RFFT to compute the DCT and TensorFlow only supports float32
     # for FFTs at the moment.
@@ -220,6 +232,12 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
     axis_dim = (tensor_shape.dimension_value(input.shape[-1])
                 or _array_ops.shape(input)[-1])
     axis_dim_float = _math_ops.to_float(axis_dim)
+
+    if type == 1:
+      dct1_input = _array_ops.concat([input, input[..., -2:0:-1]], axis=-1)
+      dct1 = _math_ops.real(rfft(dct1_input))
+      return dct1
+
     if type == 2:
       scale = 2.0 * _math_ops.exp(
           _math_ops.complex(
@@ -266,12 +284,12 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
       return dct3
 
 
-# TODO(rjryan): Implement `type`, `n` and `axis` parameters.
+# TODO(rjryan): Implement `n` and `axis` parameters.
 @tf_export("spectral.idct")
 def idct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disable=redefined-builtin
   """Computes the 1D [Inverse Discrete Cosine Transform (DCT)][idct] of `input`.
 
-  Currently only Types II and III are supported. Type III is the inverse of
+  Currently only Types I, II and III are supported. Type III is the inverse of
   Type II, and vice versa.
 
   Note that you must re-normalize by 1/(2n) to obtain an inverse if `norm` is
@@ -281,14 +299,14 @@ def idct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disab
   `signal == idct(dct(signal, norm='ortho'), norm='ortho')`.
 
   @compatibility(scipy)
-  Equivalent to scipy.fftpack.idct for Type-II and Type-III DCT.
+  Equivalent to scipy.fftpack.idct for Type-I, Type-II and Type-III DCT.
   https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.idct.html
   @end_compatibility
 
   Args:
     input: A `[..., samples]` `float32` `Tensor` containing the signals to take
       the DCT of.
-    type: The IDCT type to perform. Must be 2 or 3.
+    type: The IDCT type to perform. Must be 1, 2 or 3.
     n: For future expansion. The length of the transform. Must be `None`.
     axis: For future expansion. The axis to compute the DCT along. Must be `-1`.
     norm: The normalization to apply. `None` for no normalization or `'ortho'`
@@ -299,12 +317,12 @@ def idct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disab
     A `[..., samples]` `float32` `Tensor` containing the IDCT of `input`.
 
   Raises:
-    ValueError: If `type` is not `2` or `3`, `n` is not `None, `axis` is not
-      `-1`, or `norm` is not `None` or `'ortho'`.
+    ValueError: If `type` is not `1`, `2` or `3`, `n` is not `None, `axis` is
+      not `-1`, or `norm` is not `None` or `'ortho'`.
 
   [idct]:
   https://en.wikipedia.org/wiki/Discrete_cosine_transform#Inverse_transforms
   """
-  _validate_dct_arguments(type, n, axis, norm)
-  inverse_type = {2: 3, 3: 2}[type]
+  _validate_dct_arguments(input, type, n, axis, norm)
+  inverse_type = {1: 1, 2: 3, 3: 2}[type]
   return dct(input, type=inverse_type, n=n, axis=axis, norm=norm, name=name)
-- 
GitLab


From 07a248c2c4edc2f91a812eb8ac2f8f4f80771d5f Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 13 Nov 2018 16:50:30 -0800
Subject: [PATCH 0191/1554] Enable autograph by default in defun.

PiperOrigin-RevId: 221358807
---
 tensorflow/python/eager/function.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index c429dd359b..e4a5fefe76 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -724,7 +724,7 @@ class PolymorphicFunction(object):
                name,
                input_signature=None,
                attributes=None,
-               autograph=False):
+               autograph=True):
     """Initializes a polymorphic function.
 
     Args:
@@ -1137,7 +1137,7 @@ def validate_signature(signature):
                     "a possibly nested sequence of TensorSpec objects.")
 
 
-def defun(func=None, input_signature=None, autograph=False):
+def defun(func=None, input_signature=None, autograph=True):
   """Compiles a Python function into a callable TensorFlow graph.
 
   `defun` (short for "define function") trace-compiles a Python function
@@ -1470,7 +1470,7 @@ def defun(func=None, input_signature=None, autograph=False):
 def defun_with_attributes(func=None,
                           input_signature=None,
                           attributes=None,
-                          autograph=False):
+                          autograph=True):
   """Compiles a Python function into a callable TensorFlow graph.
 
   This function supports adding extra function attributes. See detailed
-- 
GitLab


From ac79b5c1d28fdca9e3f35783f5f76ef9de5f2fe9 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 13 Nov 2018 16:55:33 -0800
Subject: [PATCH 0192/1554] Remove symbols from the v2 API. Included here:
 tf.train.NewCheckpointReader tf.train.replica_device_setter
 tf.train.init_from_checkpoint tf.train.ProfilerHook tf.profiler.*

Profiler's API depends on v1 style graphs, so we'll remove this pending replacement.

PiperOrigin-RevId: 221359471
---
 tensorflow/python/profiler/model_analyzer.py  |   6 +-
 tensorflow/python/profiler/option_builder.py  |   2 +-
 tensorflow/python/profiler/profiler.py        |   8 +-
 tensorflow/python/profiler/tfprof_logger.py   |   2 +-
 .../tools/api/generator/api_init_files.bzl    |   1 -
 .../training/basic_session_run_hooks.py       |   2 +-
 .../python/training/checkpoint_utils.py       |   2 +-
 tensorflow/python/training/device_setter.py   |   2 +-
 tensorflow/python/util/py_checkpoint_reader.i |   1 -
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 ...flow.profiler.-advice-proto.-checker.pbtxt |  12 --
 ...ofiler.-advice-proto.-checkers-entry.pbtxt |  22 --
 .../tensorflow.profiler.-advice-proto.pbtxt   |  41 ----
 ...graph-node-proto.-input-shapes-entry.pbtxt |  22 --
 ...ensorflow.profiler.-graph-node-proto.pbtxt | 191 ------------------
 ...low.profiler.-multi-graph-node-proto.pbtxt | 134 ------------
 ...er.-op-log-proto.-id-to-string-entry.pbtxt |  21 --
 .../tensorflow.profiler.-op-log-proto.pbtxt   |  38 ----
 ...low.profiler.-profile-option-builder.pbtxt |  93 ---------
 .../v2/tensorflow.profiler.-profiler.pbtxt    |  37 ----
 .../api/golden/v2/tensorflow.profiler.pbtxt   |  39 ----
 .../v2/tensorflow.train.-profiler-hook.pbtxt  |  30 ---
 .../api/golden/v2/tensorflow.train.pbtxt      |  16 --
 23 files changed, 12 insertions(+), 714 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt

diff --git a/tensorflow/python/profiler/model_analyzer.py b/tensorflow/python/profiler/model_analyzer.py
index 5f19eac043..4b2d9052b7 100644
--- a/tensorflow/python/profiler/model_analyzer.py
+++ b/tensorflow/python/profiler/model_analyzer.py
@@ -122,7 +122,7 @@ def _build_advisor_options(options):
   return opts
 
 
-@tf_export('profiler.Profiler')
+@tf_export(v1=['profiler.Profiler'])
 class Profiler(object):
   """TensorFlow multi-step profiler.
 
@@ -306,7 +306,7 @@ class Profiler(object):
     print_mdl.WriteProfile(filename)
 
 
-@tf_export('profiler.profile')
+@tf_export(v1=['profiler.profile'])
 def profile(graph=None,
             run_meta=None,
             op_log=None,
@@ -381,7 +381,7 @@ def profile(graph=None,
   return tfprof_node
 
 
-@tf_export('profiler.advise')
+@tf_export(v1=['profiler.advise'])
 def advise(graph=None, run_meta=None, options=_DEFAULT_ADVISE_OPTIONS):
   """Auto profile and advise.
 
diff --git a/tensorflow/python/profiler/option_builder.py b/tensorflow/python/profiler/option_builder.py
index 2ad7adf769..9d8f7683a6 100644
--- a/tensorflow/python/profiler/option_builder.py
+++ b/tensorflow/python/profiler/option_builder.py
@@ -23,7 +23,7 @@ from tensorflow.python.profiler import tfprof_logger
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('profiler.ProfileOptionBuilder')
+@tf_export(v1=['profiler.ProfileOptionBuilder'])
 class ProfileOptionBuilder(object):
   # pylint: disable=line-too-long
   """Option Builder for Profiling API.
diff --git a/tensorflow/python/profiler/profiler.py b/tensorflow/python/profiler/profiler.py
index efbdd1ba68..5f62690b54 100644
--- a/tensorflow/python/profiler/profiler.py
+++ b/tensorflow/python/profiler/profiler.py
@@ -49,7 +49,7 @@ _allowed_symbols.extend([
 ])
 
 # Export protos
-tf_export('profiler.GraphNodeProto')(GraphNodeProto)
-tf_export('profiler.MultiGraphNodeProto')(MultiGraphNodeProto)
-tf_export('profiler.AdviceProto')(AdviceProto)
-tf_export('profiler.OpLogProto')(OpLogProto)
+tf_export(v1=['profiler.GraphNodeProto'])(GraphNodeProto)
+tf_export(v1=['profiler.MultiGraphNodeProto'])(MultiGraphNodeProto)
+tf_export(v1=['profiler.AdviceProto'])(AdviceProto)
+tf_export(v1=['profiler.OpLogProto'])(OpLogProto)
diff --git a/tensorflow/python/profiler/tfprof_logger.py b/tensorflow/python/profiler/tfprof_logger.py
index e651de32ea..6ccd0e0ff3 100644
--- a/tensorflow/python/profiler/tfprof_logger.py
+++ b/tensorflow/python/profiler/tfprof_logger.py
@@ -188,7 +188,7 @@ def merge_default_with_oplog(graph, op_log=None, run_meta=None,
   return tmp_op_log
 
 
-@tf_export('profiler.write_op_log')
+@tf_export(v1=['profiler.write_op_log'])
 def write_op_log(graph, log_dir, op_log=None, run_meta=None, add_trace=True):
   """Log provided 'op_log', and add additional model information below.
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 5699d86e6d..60a5540b83 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -68,7 +68,6 @@ TENSORFLOW_API_INIT_FILES = [
     "metrics/__init__.py",
     "nn/__init__.py",
     "nn/rnn_cell/__init__.py",
-    "profiler/__init__.py",
     "quantization/__init__.py",
     "random/__init__.py",
     "saved_model/__init__.py",
diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py
index 1efabcd854..60db654e9c 100644
--- a/tensorflow/python/training/basic_session_run_hooks.py
+++ b/tensorflow/python/training/basic_session_run_hooks.py
@@ -976,7 +976,7 @@ class FeedFnHook(session_run_hook.SessionRunHook):
         fetches=None, feed_dict=self.feed_fn())
 
 
-@tf_export("train.ProfilerHook")
+@tf_export(v1=["train.ProfilerHook"])
 class ProfilerHook(session_run_hook.SessionRunHook):
   """Captures CPU/GPU profiling information every N steps or seconds.
 
diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 857da431db..2ed7d7f29c 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -101,7 +101,7 @@ def list_variables(ckpt_dir_or_file):
   return result
 
 
-@tf_export("train.init_from_checkpoint")
+@tf_export(v1=["train.init_from_checkpoint"])
 def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
   """Replaces `tf.Variable` initializers so they load from a checkpoint file.
 
diff --git a/tensorflow/python/training/device_setter.py b/tensorflow/python/training/device_setter.py
index be80c36571..5874a1ff41 100644
--- a/tensorflow/python/training/device_setter.py
+++ b/tensorflow/python/training/device_setter.py
@@ -130,7 +130,7 @@ class _ReplicaDeviceChooser(object):
     return worker_device.to_string()
 
 
-@tf_export("train.replica_device_setter")
+@tf_export(v1=["train.replica_device_setter"])
 def replica_device_setter(ps_tasks=0, ps_device="/job:ps",
                           worker_device="/job:worker", merge_devices=True,
                           cluster=None, ps_ops=None, ps_strategy=None):
diff --git a/tensorflow/python/util/py_checkpoint_reader.i b/tensorflow/python/util/py_checkpoint_reader.i
index 1c73f7f06f..a1b98a2a75 100644
--- a/tensorflow/python/util/py_checkpoint_reader.i
+++ b/tensorflow/python/util/py_checkpoint_reader.i
@@ -165,7 +165,6 @@ def NewCheckpointReader(filepattern):
     from tensorflow.python.util import compat
     return CheckpointReader(compat.as_bytes(filepattern), status)
 
-NewCheckpointReader._tf_api_names = ['train.NewCheckpointReader']
 NewCheckpointReader._tf_api_names_v1 = ['train.NewCheckpointReader']
 %}
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 7afff0a028..018245b879 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -328,10 +328,6 @@ tf_module {
     name: "ones_initializer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "profiler"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "pywrap_tensorflow"
     mtype: "<type \'module\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt
deleted file mode 100644
index e09c44cc9c..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checker.pbtxt
+++ /dev/null
@@ -1,12 +0,0 @@
-path: "tensorflow.profiler.AdviceProto.Checker"
-tf_proto {
-  descriptor {
-    name: "Checker"
-    field {
-      name: "reports"
-      number: 2
-      label: LABEL_REPEATED
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt
deleted file mode 100644
index 8746243549..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.-checkers-entry.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-path: "tensorflow.profiler.AdviceProto.CheckersEntry"
-tf_proto {
-  descriptor {
-    name: "CheckersEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.AdviceProto.Checker"
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt
deleted file mode 100644
index a8a8858ccd..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-advice-proto.pbtxt
+++ /dev/null
@@ -1,41 +0,0 @@
-path: "tensorflow.profiler.AdviceProto"
-tf_proto {
-  descriptor {
-    name: "AdviceProto"
-    field {
-      name: "checkers"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.AdviceProto.CheckersEntry"
-    }
-    nested_type {
-      name: "CheckersEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.tfprof.AdviceProto.Checker"
-      }
-      options {
-        map_entry: true
-      }
-    }
-    nested_type {
-      name: "Checker"
-      field {
-        name: "reports"
-        number: 2
-        label: LABEL_REPEATED
-        type: TYPE_STRING
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt
deleted file mode 100644
index afec73f537..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.-input-shapes-entry.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-path: "tensorflow.profiler.GraphNodeProto.InputShapesEntry"
-tf_proto {
-  descriptor {
-    name: "InputShapesEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorShapeProto"
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt
deleted file mode 100644
index 3c83177005..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-graph-node-proto.pbtxt
+++ /dev/null
@@ -1,191 +0,0 @@
-path: "tensorflow.profiler.GraphNodeProto"
-tf_proto {
-  descriptor {
-    name: "GraphNodeProto"
-    field {
-      name: "name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "tensor_value"
-      number: 15
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.TFProfTensorProto"
-    }
-    field {
-      name: "run_count"
-      number: 21
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "exec_micros"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "accelerator_exec_micros"
-      number: 17
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "cpu_exec_micros"
-      number: 18
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "requested_bytes"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "peak_bytes"
-      number: 24
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "residual_bytes"
-      number: 25
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "output_bytes"
-      number: 26
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "parameters"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "float_ops"
-      number: 13
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "devices"
-      number: 10
-      label: LABEL_REPEATED
-      type: TYPE_STRING
-    }
-    field {
-      name: "total_definition_count"
-      number: 23
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_run_count"
-      number: 22
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_exec_micros"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_accelerator_exec_micros"
-      number: 19
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_cpu_exec_micros"
-      number: 20
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_requested_bytes"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_peak_bytes"
-      number: 27
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_residual_bytes"
-      number: 28
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_output_bytes"
-      number: 29
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_parameters"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_float_ops"
-      number: 14
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "shapes"
-      number: 11
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorShapeProto"
-    }
-    field {
-      name: "input_shapes"
-      number: 16
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.GraphNodeProto.InputShapesEntry"
-    }
-    field {
-      name: "children"
-      number: 12
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.GraphNodeProto"
-    }
-    nested_type {
-      name: "InputShapesEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorShapeProto"
-      }
-      options {
-        map_entry: true
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt
deleted file mode 100644
index 2b08a05437..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-multi-graph-node-proto.pbtxt
+++ /dev/null
@@ -1,134 +0,0 @@
-path: "tensorflow.profiler.MultiGraphNodeProto"
-tf_proto {
-  descriptor {
-    name: "MultiGraphNodeProto"
-    field {
-      name: "name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "exec_micros"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "accelerator_exec_micros"
-      number: 12
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "cpu_exec_micros"
-      number: 13
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "requested_bytes"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "peak_bytes"
-      number: 16
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "residual_bytes"
-      number: 17
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "output_bytes"
-      number: 18
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "parameters"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "float_ops"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_exec_micros"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_accelerator_exec_micros"
-      number: 14
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_cpu_exec_micros"
-      number: 15
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_requested_bytes"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_peak_bytes"
-      number: 19
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_residual_bytes"
-      number: 20
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_output_bytes"
-      number: 21
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_parameters"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "total_float_ops"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "graph_nodes"
-      number: 10
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.GraphNodeProto"
-    }
-    field {
-      name: "children"
-      number: 11
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.MultiGraphNodeProto"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt
deleted file mode 100644
index b3adc50c7e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt
+++ /dev/null
@@ -1,21 +0,0 @@
-path: "tensorflow.profiler.OpLogProto.IdToStringEntry"
-tf_proto {
-  descriptor {
-    name: "IdToStringEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt
deleted file mode 100644
index 7510c566ba..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-op-log-proto.pbtxt
+++ /dev/null
@@ -1,38 +0,0 @@
-path: "tensorflow.profiler.OpLogProto"
-tf_proto {
-  descriptor {
-    name: "OpLogProto"
-    field {
-      name: "log_entries"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.OpLogEntry"
-    }
-    field {
-      name: "id_to_string"
-      number: 2
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.tfprof.OpLogProto.IdToStringEntry"
-    }
-    nested_type {
-      name: "IdToStringEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      options {
-        map_entry: true
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt
deleted file mode 100644
index 19ff38a390..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profile-option-builder.pbtxt
+++ /dev/null
@@ -1,93 +0,0 @@
-path: "tensorflow.profiler.ProfileOptionBuilder"
-tf_class {
-  is_instance: "<class \'tensorflow.python.profiler.option_builder.ProfileOptionBuilder\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "account_displayed_op_only"
-    argspec: "args=[\'self\', \'is_true\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "float_operation"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "order_by"
-    argspec: "args=[\'self\', \'attribute\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "select"
-    argspec: "args=[\'self\', \'attributes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "time_and_memory"
-    argspec: "args=[\'min_micros\', \'min_bytes\', \'min_accelerator_micros\', \'min_cpu_micros\', \'min_peak_bytes\', \'min_residual_bytes\', \'min_output_bytes\'], varargs=None, keywords=None, defaults=[\'1\', \'1\', \'0\', \'0\', \'0\', \'0\', \'0\'], "
-  }
-  member_method {
-    name: "trainable_variables_parameter"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_accounted_types"
-    argspec: "args=[\'self\', \'account_type_regexes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_empty_output"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_file_output"
-    argspec: "args=[\'self\', \'outfile\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_max_depth"
-    argspec: "args=[\'self\', \'max_depth\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_min_execution_time"
-    argspec: "args=[\'self\', \'min_micros\', \'min_accelerator_micros\', \'min_cpu_micros\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'0\'], "
-  }
-  member_method {
-    name: "with_min_float_operations"
-    argspec: "args=[\'self\', \'min_float_ops\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_min_memory"
-    argspec: "args=[\'self\', \'min_bytes\', \'min_peak_bytes\', \'min_residual_bytes\', \'min_output_bytes\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'0\', \'0\'], "
-  }
-  member_method {
-    name: "with_min_occurrence"
-    argspec: "args=[\'self\', \'min_occurrence\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_min_parameters"
-    argspec: "args=[\'self\', \'min_params\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_node_names"
-    argspec: "args=[\'self\', \'start_name_regexes\', \'show_name_regexes\', \'hide_name_regexes\', \'trim_name_regexes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "with_pprof_output"
-    argspec: "args=[\'self\', \'pprof_file\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_stdout_output"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_step"
-    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "with_timeline_output"
-    argspec: "args=[\'self\', \'timeline_file\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt
deleted file mode 100644
index acb61dae9f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.-profiler.pbtxt
+++ /dev/null
@@ -1,37 +0,0 @@
-path: "tensorflow.profiler.Profiler"
-tf_class {
-  is_instance: "<class \'tensorflow.python.profiler.model_analyzer.Profiler\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'graph\', \'op_log\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_step"
-    argspec: "args=[\'self\', \'step\', \'run_meta\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "advise"
-    argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "profile_graph"
-    argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "profile_name_scope"
-    argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "profile_operations"
-    argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "profile_python"
-    argspec: "args=[\'self\', \'options\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "serialize_to_string"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt
deleted file mode 100644
index 7b4d3ac522..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.profiler.pbtxt
+++ /dev/null
@@ -1,39 +0,0 @@
-path: "tensorflow.profiler"
-tf_module {
-  member {
-    name: "AdviceProto"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "GraphNodeProto"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "MultiGraphNodeProto"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "OpLogProto"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "ProfileOptionBuilder"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Profiler"
-    mtype: "<type \'type\'>"
-  }
-  member_method {
-    name: "advise"
-    argspec: "args=[\'graph\', \'run_meta\', \'options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0\'], "
-  }
-  member_method {
-    name: "profile"
-    argspec: "args=[\'graph\', \'run_meta\', \'op_log\', \'cmd\', \'options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'scope\', \'0\'], "
-  }
-  member_method {
-    name: "write_op_log"
-    argspec: "args=[\'graph\', \'log_dir\', \'op_log\', \'run_meta\', \'add_trace\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt
deleted file mode 100644
index 4df6c4156a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-profiler-hook.pbtxt
+++ /dev/null
@@ -1,30 +0,0 @@
-path: "tensorflow.train.ProfilerHook"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.ProfilerHook\'>"
-  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'save_steps\', \'save_secs\', \'output_dir\', \'show_dataflow\', \'show_memory\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'True\', \'False\'], "
-  }
-  member_method {
-    name: "after_create_session"
-    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "after_run"
-    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "before_run"
-    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "begin"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "end"
-    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index a091daa298..89d9270b27 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -136,10 +136,6 @@ tf_module {
     name: "Optimizer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "ProfilerHook"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ProximalAdagradOptimizer"
     mtype: "<type \'type\'>"
@@ -224,10 +220,6 @@ tf_module {
     name: "WorkerSessionCreator"
     mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "NewCheckpointReader"
-    argspec: "args=[\'filepattern\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "cosine_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
@@ -244,10 +236,6 @@ tf_module {
     name: "get_checkpoint_state"
     argspec: "args=[\'checkpoint_dir\', \'latest_filename\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "init_from_checkpoint"
-    argspec: "args=[\'ckpt_dir_or_file\', \'assignment_map\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "inverse_time_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
@@ -288,10 +276,6 @@ tf_module {
     name: "polynomial_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], "
   }
-  member_method {
-    name: "replica_device_setter"
-    argspec: "args=[\'ps_tasks\', \'ps_device\', \'worker_device\', \'merge_devices\', \'cluster\', \'ps_ops\', \'ps_strategy\'], varargs=None, keywords=None, defaults=[\'0\', \'/job:ps\', \'/job:worker\', \'True\', \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "sdca_fprint"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 21177b1a53f7ff3a8f964f4ecc73dc009ae9f2fe Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 14 Nov 2018 09:30:31 +0800
Subject: [PATCH 0193/1554] Format a wired comment made by clang-format-3.8

---
 tensorflow/core/graph/mkl_layout_pass.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index d46e7165ff..dffd7ef4a4 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -515,10 +515,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       GetConv2DBackpropFilterOrBiasAddGrad});
 
     // The fusion patterns in "finfo_" that show up first will get applied
-    // first,
-    // for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC, A->B->C->D
-    // to ABCD},
-    // since the first gets applied first, the final graph will be ABC->D.
+    // first, for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC, A->B->C->D
+    // to ABCD}, since the first gets applied first, the final graph will be ABC->D.
 
     //
     // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D
-- 
GitLab


From c98cdd38b74d07aec0fdeb17acbbc2bb950e4f22 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 13 Nov 2018 17:28:57 -0800
Subject: [PATCH 0194/1554] Add distributed_runtime dependency to
 c_api_internal

The distributed_runtime header is referenced directly from
c_api_internal.h, so the dependency should be explicit.

PiperOrigin-RevId: 221364208
---
 tensorflow/c/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index b8db1b2144..edd0c30a46 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -60,6 +60,7 @@ tf_cuda_library(
             "//tensorflow/core:framework",
             "//tensorflow/core:lib",
             "//tensorflow/core:op_gen_lib",
+            "//tensorflow/core/distributed_runtime:server_lib",
         ],
     }),
 )
-- 
GitLab


From 94cf4b8cebaae0215fd231353b41cdf38d5d4793 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 13 Nov 2018 17:29:54 -0800
Subject: [PATCH 0195/1554] Remove tf.saved_model.LEGACY_INIT_OP_KEY from the
 API. It's legacy and very deprecated.

PiperOrigin-RevId: 221364326
---
 tensorflow/python/saved_model/constants.py                  | 1 -
 tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/tensorflow/python/saved_model/constants.py b/tensorflow/python/saved_model/constants.py
index 0addbdc968..f696d4815b 100644
--- a/tensorflow/python/saved_model/constants.py
+++ b/tensorflow/python/saved_model/constants.py
@@ -40,7 +40,6 @@ tf_export(
 # CollectionDef key for the legacy init op.
 LEGACY_INIT_OP_KEY = "legacy_init_op"
 tf_export(
-    "saved_model.LEGACY_INIT_OP_KEY",
     v1=[
         "saved_model.LEGACY_INIT_OP_KEY",
         "saved_model.constants.LEGACY_INIT_OP_KEY"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
index d57936a2f1..c83b569176 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
@@ -32,10 +32,6 @@ tf_module {
     name: "GPU"
     mtype: "<type \'str\'>"
   }
-  member {
-    name: "LEGACY_INIT_OP_KEY"
-    mtype: "<type \'str\'>"
-  }
   member {
     name: "MAIN_OP_KEY"
     mtype: "<type \'str\'>"
-- 
GitLab


From bf383cf4b5037952cddaef30d71aaaaf5fa64917 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 13 Nov 2018 17:47:53 -0800
Subject: [PATCH 0196/1554] Exposing feature_column_v2 methods externally only
 in compat.v2 mode

PiperOrigin-RevId: 221366646
---
 .../sequence_feature_column_v2_test.py        |    2 +-
 .../python/feature_column/feature_column.py   |   28 +-
 .../feature_column/feature_column_lib.py      |    1 +
 .../feature_column/feature_column_test.py     |    2 +-
 .../feature_column/feature_column_v2.py       |  100 +-
 .../feature_column/feature_column_v2_test.py  | 1574 +++++++++--------
 .../feature_columns_integration_test.py       |   38 +-
 .../golden/v2/tensorflow.feature_column.pbtxt |    8 -
 ...sorflow.keras.layers.-dense-features.pbtxt |  175 ++
 ...ensorflow.keras.layers.-linear-model.pbtxt |  269 +++
 .../golden/v2/tensorflow.keras.layers.pbtxt   |    8 +
 11 files changed, 1360 insertions(+), 845 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt

diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index 5ecd85807c..904d149fdb 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -1279,7 +1279,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc.indicator_column(categorical_column)
+    indicator_column = fc.indicator_column_v2(categorical_column)
 
     _, sequence_length = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': sparse_input}))
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index b7a6a88535..92e308070e 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -230,7 +230,7 @@ def _internal_input_layer(features,
       return _get_logits()
 
 
-@tf_export('feature_column.input_layer')
+@tf_export(v1=['feature_column.input_layer'])
 def input_layer(features,
                 feature_columns,
                 weight_collections=None,
@@ -365,7 +365,7 @@ class InputLayer(object):
     return self._input_layer_template.weights
 
 
-@tf_export('feature_column.linear_model')
+@tf_export(v1=['feature_column.linear_model'])
 def linear_model(features,
                  feature_columns,
                  units=1,
@@ -746,7 +746,7 @@ def _transform_features(features, feature_columns):
   return outputs
 
 
-@tf_export('feature_column.make_parse_example_spec')
+@tf_export(v1=['feature_column.make_parse_example_spec'])
 def make_parse_example_spec(feature_columns):
   """Creates parsing spec dictionary from input feature_columns.
 
@@ -807,7 +807,7 @@ def make_parse_example_spec(feature_columns):
   return result
 
 
-@tf_export('feature_column.embedding_column')
+@tf_export(v1=['feature_column.embedding_column'])
 def embedding_column(
     categorical_column, dimension, combiner='mean', initializer=None,
     ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None,
@@ -919,7 +919,7 @@ def embedding_column(
       trainable=trainable)
 
 
-@tf_export('feature_column.shared_embedding_columns')
+@tf_export(v1=['feature_column.shared_embedding_columns'])
 def shared_embedding_columns(
     categorical_columns, dimension, combiner='mean', initializer=None,
     shared_embedding_collection_name=None, ckpt_to_load_from=None,
@@ -1085,7 +1085,7 @@ def shared_embedding_columns(
   return result
 
 
-@tf_export('feature_column.numeric_column')
+@tf_export(v1=['feature_column.numeric_column'])
 def numeric_column(key,
                    shape=(1,),
                    default_value=None,
@@ -1161,7 +1161,7 @@ def numeric_column(key,
       normalizer_fn=normalizer_fn)
 
 
-@tf_export('feature_column.bucketized_column')
+@tf_export(v1=['feature_column.bucketized_column'])
 def bucketized_column(source_column, boundaries):
   """Represents discretized dense input.
 
@@ -1258,7 +1258,7 @@ def _assert_key_is_string(key):
             type(key), key))
 
 
-@tf_export('feature_column.categorical_column_with_hash_bucket')
+@tf_export(v1=['feature_column.categorical_column_with_hash_bucket'])
 def categorical_column_with_hash_bucket(key,
                                         hash_bucket_size,
                                         dtype=dtypes.string):
@@ -1317,7 +1317,7 @@ def categorical_column_with_hash_bucket(key,
   return _HashedCategoricalColumn(key, hash_bucket_size, dtype)
 
 
-@tf_export('feature_column.categorical_column_with_vocabulary_file')
+@tf_export(v1=['feature_column.categorical_column_with_vocabulary_file'])
 def categorical_column_with_vocabulary_file(key,
                                             vocabulary_file,
                                             vocabulary_size=None,
@@ -1437,7 +1437,7 @@ def categorical_column_with_vocabulary_file(key,
       dtype=dtype)
 
 
-@tf_export('feature_column.categorical_column_with_vocabulary_list')
+@tf_export(v1=['feature_column.categorical_column_with_vocabulary_list'])
 def categorical_column_with_vocabulary_list(
     key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0):
   """A `_CategoricalColumn` with in-memory vocabulary.
@@ -1548,7 +1548,7 @@ def categorical_column_with_vocabulary_list(
       default_value=default_value, num_oov_buckets=num_oov_buckets)
 
 
-@tf_export('feature_column.categorical_column_with_identity')
+@tf_export(v1=['feature_column.categorical_column_with_identity'])
 def categorical_column_with_identity(key, num_buckets, default_value=None):
   """A `_CategoricalColumn` that returns identity values.
 
@@ -1616,7 +1616,7 @@ def categorical_column_with_identity(key, num_buckets, default_value=None):
       key=key, num_buckets=num_buckets, default_value=default_value)
 
 
-@tf_export('feature_column.indicator_column')
+@tf_export(v1=['feature_column.indicator_column'])
 def indicator_column(categorical_column):
   """Represents multi-hot representation of given categorical column.
 
@@ -1651,7 +1651,7 @@ def indicator_column(categorical_column):
   return _IndicatorColumn(categorical_column)
 
 
-@tf_export('feature_column.weighted_categorical_column')
+@tf_export(v1=['feature_column.weighted_categorical_column'])
 def weighted_categorical_column(
     categorical_column, weight_feature_key, dtype=dtypes.float32):
   """Applies weight values to a `_CategoricalColumn`.
@@ -1726,7 +1726,7 @@ def weighted_categorical_column(
       dtype=dtype)
 
 
-@tf_export('feature_column.crossed_column')
+@tf_export(v1=['feature_column.crossed_column'])
 def crossed_column(keys, hash_bucket_size, hash_key=None):
   """Returns a column for performing crosses of categorical features.
 
diff --git a/tensorflow/python/feature_column/feature_column_lib.py b/tensorflow/python/feature_column/feature_column_lib.py
index 3b818f18b5..68a2712425 100644
--- a/tensorflow/python/feature_column/feature_column_lib.py
+++ b/tensorflow/python/feature_column/feature_column_lib.py
@@ -20,4 +20,5 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long,wildcard-import
 from tensorflow.python.feature_column.feature_column import *
+from tensorflow.python.feature_column.feature_column_v2 import *
 # pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 1ae510250c..7f4b1902e1 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -30,7 +30,7 @@ from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
-from tensorflow.python.feature_column import feature_column_lib as fc
+from tensorflow.python.feature_column import feature_column as fc
 from tensorflow.python.feature_column.feature_column import _CategoricalColumn
 from tensorflow.python.feature_column.feature_column import _DenseColumn
 from tensorflow.python.feature_column.feature_column import _FeatureColumn
diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 9b4a7e882f..8d27c96669 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -165,6 +165,7 @@ from tensorflow.python.training import checkpoint_utils
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.util import deprecation
 from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
 
 
 _FEATURE_COLUMN_DEPRECATION_DATE = '2018-11-30'
@@ -258,7 +259,7 @@ class StateManager(object):
 
 
 class _StateManagerImpl(StateManager):
-  """Manages the state of FeatureLayer and LinearModel."""
+  """Manages the state of DenseFeatures and LinearLayer."""
 
   def __init__(self, layer, trainable):
     """Creates an _StateManagerImpl object.
@@ -302,7 +303,8 @@ class _StateManagerImpl(StateManager):
     raise ValueError('Variable does not exist.')
 
 
-class FeatureLayer(Layer):
+@tf_export('keras.layers.DenseFeatures', v1=[])
+class DenseFeatures(Layer):
   """A layer that produces a dense `Tensor` based on given `feature_columns`.
 
   Generally a single example in training data is described with FeatureColumns.
@@ -318,7 +320,7 @@ class FeatureLayer(Layer):
   keywords_embedded = embedding_column(
       categorical_column_with_hash_bucket("keywords", 10K), dimensions=16)
   columns = [price, keywords_embedded, ...]
-  feature_layer = FeatureLayer(columns)
+  feature_layer = DenseFeatures(columns)
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
   dense_tensor = feature_layer(features)
@@ -333,7 +335,7 @@ class FeatureLayer(Layer):
                trainable=True,
                name=None,
                **kwargs):
-    """Constructs a FeatureLayer.
+    """Constructs a DenseFeatures.
 
     Args:
       feature_columns: An iterable containing the FeatureColumns to use as
@@ -344,13 +346,14 @@ class FeatureLayer(Layer):
         `indicator_column`.
       trainable: If `True` also add the variable to the graph collection
         `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
-      name: Name to give to the FeatureLayer.
+      name: Name to give to the DenseFeatures.
       **kwargs: Keyword arguments to construct a layer.
 
     Raises:
       ValueError: if an item in `feature_columns` is not a `DenseColumn`.
     """
-    super(FeatureLayer, self).__init__(name=name, trainable=trainable, **kwargs)
+    super(DenseFeatures, self).__init__(
+        name=name, trainable=trainable, **kwargs)
 
     self._feature_columns = _normalize_feature_columns(feature_columns)
     self._feature_columns = sorted(self._feature_columns, key=lambda x: x.name)
@@ -371,7 +374,7 @@ class FeatureLayer(Layer):
       with variable_scope._pure_variable_scope(self.name):  # pylint: disable=protected-access
         with variable_scope._pure_variable_scope(column.name):  # pylint: disable=protected-access
           column.create_state(self._state_manager)
-      super(FeatureLayer, self).build(None)
+      super(DenseFeatures, self).build(None)
 
   def call(self, features, cols_to_output_tensors=None):
     """Returns a dense tensor corresponding to the `feature_columns`.
@@ -515,6 +518,7 @@ class _LinearModelLayer(Layer):
       return predictions
 
 
+@tf_export('keras.layers.LinearModel', v1=[])
 class LinearModel(training.Model):
   """Produces a linear prediction `Tensor` based on given `feature_columns`.
 
@@ -522,7 +526,7 @@ class LinearModel(training.Model):
   Weighted sum refers to logits in classification problems. It refers to the
   prediction itself for linear regression problems.
 
-  Note on supported columns: `LinearModel` treats categorical columns as
+  Note on supported columns: `LinearLayer` treats categorical columns as
   `indicator_column`s. To be specific, assume the input as `SparseTensor` looks
   like:
 
@@ -547,7 +551,7 @@ class LinearModel(training.Model):
   keywords = categorical_column_with_hash_bucket("keywords", 10K)
   keywords_price = crossed_column('keywords', price_buckets, ...)
   columns = [price_buckets, keywords, keywords_price ...]
-  linear_model = LinearModel(columns)
+  linear_model = LinearLayer(columns)
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
   prediction = linear_model(features)
@@ -561,7 +565,7 @@ class LinearModel(training.Model):
                trainable=True,
                name=None,
                **kwargs):
-    """Constructs a LinearModel.
+    """Constructs a LinearLayer.
 
     Args:
       feature_columns: An iterable containing the FeatureColumns to use as
@@ -695,7 +699,8 @@ def _transform_features(features, feature_columns, state_manager):
   return outputs
 
 
-def make_parse_example_spec(feature_columns):
+@tf_export('feature_column.make_parse_example_spec', v1=[])
+def make_parse_example_spec_v2(feature_columns):
   """Creates parsing spec dictionary from input feature_columns.
 
   The returned dictionary can be used as arg 'features' in `tf.parse_example`.
@@ -754,10 +759,15 @@ def make_parse_example_spec(feature_columns):
   return result
 
 
-def embedding_column(
-    categorical_column, dimension, combiner='mean', initializer=None,
-    ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None,
-    trainable=True):
+@tf_export('feature_column.embedding_column', v1=[])
+def embedding_column_v2(categorical_column,
+                        dimension,
+                        combiner='mean',
+                        initializer=None,
+                        ckpt_to_load_from=None,
+                        tensor_name_in_ckpt=None,
+                        max_norm=None,
+                        trainable=True):
   """`DenseColumn` that converts from sparse, categorical input.
 
   Use this when your inputs are sparse, but you want to convert them to a dense
@@ -854,6 +864,7 @@ def embedding_column(
       trainable=trainable)
 
 
+@tf_export('feature_column.shared_embedding_columns', v1=[])
 def shared_embedding_columns_v2(categorical_columns,
                                 dimension,
                                 combiner='mean',
@@ -1019,11 +1030,12 @@ def shared_embedding_columns_v2(categorical_columns,
   return result
 
 
-def numeric_column(key,
-                   shape=(1,),
-                   default_value=None,
-                   dtype=dtypes.float32,
-                   normalizer_fn=None):
+@tf_export('feature_column.numeric_column', v1=[])
+def numeric_column_v2(key,
+                      shape=(1,),
+                      default_value=None,
+                      dtype=dtypes.float32,
+                      normalizer_fn=None):
   """Represents real valued or numerical features.
 
   Example:
@@ -1094,7 +1106,8 @@ def numeric_column(key,
       normalizer_fn=normalizer_fn)
 
 
-def bucketized_column(source_column, boundaries):
+@tf_export('feature_column.bucketized_column', v1=[])
+def bucketized_column_v2(source_column, boundaries):
   """Represents discretized dense input.
 
   Buckets include the left boundary, and exclude the right boundary. Namely,
@@ -1190,9 +1203,10 @@ def _assert_key_is_string(key):
             type(key), key))
 
 
-def categorical_column_with_hash_bucket(key,
-                                        hash_bucket_size,
-                                        dtype=dtypes.string):
+@tf_export('feature_column.categorical_column_with_hash_bucket', v1=[])
+def categorical_column_with_hash_bucket_v2(key,
+                                           hash_bucket_size,
+                                           dtype=dtypes.string):
   """Represents sparse feature where ids are set by hashing.
 
   Use this when your sparse features are in string or integer format, and you
@@ -1248,12 +1262,13 @@ def categorical_column_with_hash_bucket(key,
   return HashedCategoricalColumn(key, hash_bucket_size, dtype)
 
 
-def categorical_column_with_vocabulary_file(key,
-                                            vocabulary_file,
-                                            vocabulary_size=None,
-                                            num_oov_buckets=0,
-                                            default_value=None,
-                                            dtype=dtypes.string):
+@tf_export('feature_column.categorical_column_with_vocabulary_file', v1=[])
+def categorical_column_with_vocabulary_file_v2(key,
+                                               vocabulary_file,
+                                               vocabulary_size=None,
+                                               num_oov_buckets=0,
+                                               default_value=None,
+                                               dtype=dtypes.string):
   """A `CategoricalColumn` with a vocabulary file.
 
   Use this when your inputs are in string or integer format, and you have a
@@ -1367,8 +1382,12 @@ def categorical_column_with_vocabulary_file(key,
       dtype=dtype)
 
 
-def categorical_column_with_vocabulary_list(
-    key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0):
+@tf_export('feature_column.categorical_column_with_vocabulary_list', v1=[])
+def categorical_column_with_vocabulary_list_v2(key,
+                                               vocabulary_list,
+                                               dtype=None,
+                                               default_value=-1,
+                                               num_oov_buckets=0):
   """A `CategoricalColumn` with in-memory vocabulary.
 
   Use this when your inputs are in string or integer format, and you have an
@@ -1480,7 +1499,8 @@ def categorical_column_with_vocabulary_list(
       num_oov_buckets=num_oov_buckets)
 
 
-def categorical_column_with_identity(key, num_buckets, default_value=None):
+@tf_export('feature_column.categorical_column_with_identity', v1=[])
+def categorical_column_with_identity_v2(key, num_buckets, default_value=None):
   """A `CategoricalColumn` that returns identity values.
 
   Use this when your inputs are integers in the range `[0, num_buckets)`, and
@@ -1547,7 +1567,8 @@ def categorical_column_with_identity(key, num_buckets, default_value=None):
       key=key, number_buckets=num_buckets, default_value=default_value)
 
 
-def indicator_column(categorical_column):
+@tf_export('feature_column.indicator_column', v1=[])
+def indicator_column_v2(categorical_column):
   """Represents multi-hot representation of given categorical column.
 
   - For DNN model, `indicator_column` can be used to wrap any
@@ -1581,8 +1602,10 @@ def indicator_column(categorical_column):
   return IndicatorColumn(categorical_column)
 
 
-def weighted_categorical_column(
-    categorical_column, weight_feature_key, dtype=dtypes.float32):
+@tf_export('feature_column.weighted_categorical_column', v1=[])
+def weighted_categorical_column_v2(categorical_column,
+                                   weight_feature_key,
+                                   dtype=dtypes.float32):
   """Applies weight values to a `CategoricalColumn`.
 
   Use this when each of your sparse inputs has both an ID and a value. For
@@ -1655,7 +1678,8 @@ def weighted_categorical_column(
       dtype=dtype)
 
 
-def crossed_column(keys, hash_bucket_size, hash_key=None):
+@tf_export('feature_column.crossed_column', v1=[])
+def crossed_column_v2(keys, hash_bucket_size, hash_key=None):
   """Returns a column for performing crosses of categorical features.
 
   Crossed features will be hashed according to `hash_bucket_size`. Conceptually,
@@ -2890,7 +2914,7 @@ class EmbeddingColumn(
 def _raise_shared_embedding_column_error():
   raise ValueError('SharedEmbeddingColumns are not supported in '
                    '`linear_model` or `input_layer`. Please use '
-                   '`FeatureLayer` or `LinearModel` instead.')
+                   '`DenseFeatures` or `LinearModel` instead.')
 
 
 class SharedEmbeddingColumnCreator(tracking.Checkpointable):
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index a26b860056..2a133e2892 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -238,7 +238,7 @@ class LazyColumnTest(test.TestCase):
 class NumericColumnTest(test.TestCase):
 
   def test_defaults(self):
-    a = fc.numeric_column('aaa')
+    a = fc.numeric_column_v2('aaa')
     self.assertEqual('aaa', a.key)
     self.assertEqual('aaa', a.name)
     self.assertEqual((1,), a.shape)
@@ -249,53 +249,53 @@ class NumericColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.numeric_column(key=('aaa',))
+      fc.numeric_column_v2(key=('aaa',))
 
   def test_shape_saved_as_tuple(self):
-    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
+    a = fc.numeric_column_v2('aaa', shape=[1, 2], default_value=[[3, 2.]])
     self.assertEqual((1, 2), a.shape)
 
   def test_default_value_saved_as_tuple(self):
-    a = fc.numeric_column('aaa', default_value=4.)
+    a = fc.numeric_column_v2('aaa', default_value=4.)
     self.assertEqual((4.,), a.default_value)
-    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
+    a = fc.numeric_column_v2('aaa', shape=[1, 2], default_value=[[3, 2.]])
     self.assertEqual(((3., 2.),), a.default_value)
 
   def test_shape_and_default_value_compatibility(self):
-    fc.numeric_column('aaa', shape=[2], default_value=[1, 2.])
+    fc.numeric_column_v2('aaa', shape=[2], default_value=[1, 2.])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column('aaa', shape=[2], default_value=[1, 2, 3.])
-    fc.numeric_column(
+      fc.numeric_column_v2('aaa', shape=[2], default_value=[1, 2, 3.])
+    fc.numeric_column_v2(
         'aaa', shape=[3, 2], default_value=[[2, 3], [1, 2], [2, 3.]])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column(
+      fc.numeric_column_v2(
           'aaa', shape=[3, 1], default_value=[[2, 3], [1, 2], [2, 3.]])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column(
+      fc.numeric_column_v2(
           'aaa', shape=[3, 3], default_value=[[2, 3], [1, 2], [2, 3.]])
 
   def test_default_value_type_check(self):
-    fc.numeric_column(
+    fc.numeric_column_v2(
         'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.float32)
-    fc.numeric_column(
+    fc.numeric_column_v2(
         'aaa', shape=[2], default_value=[1, 2], dtype=dtypes.int32)
     with self.assertRaisesRegexp(TypeError, 'must be compatible with dtype'):
-      fc.numeric_column(
+      fc.numeric_column_v2(
           'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.int32)
     with self.assertRaisesRegexp(TypeError,
                                  'default_value must be compatible with dtype'):
-      fc.numeric_column('aaa', default_value=['string'])
+      fc.numeric_column_v2('aaa', default_value=['string'])
 
   def test_shape_must_be_positive_integer(self):
     with self.assertRaisesRegexp(TypeError, 'shape dimensions must be integer'):
-      fc.numeric_column(
+      fc.numeric_column_v2(
           'aaa', shape=[
               1.0,
           ])
 
     with self.assertRaisesRegexp(ValueError,
                                  'shape dimensions must be greater than 0'):
-      fc.numeric_column(
+      fc.numeric_column_v2(
           'aaa', shape=[
               0,
           ])
@@ -303,20 +303,20 @@ class NumericColumnTest(test.TestCase):
   def test_dtype_is_convertible_to_float(self):
     with self.assertRaisesRegexp(ValueError,
                                  'dtype must be convertible to float'):
-      fc.numeric_column('aaa', dtype=dtypes.string)
+      fc.numeric_column_v2('aaa', dtype=dtypes.string)
 
   def test_scalar_default_value_fills_the_shape(self):
-    a = fc.numeric_column('aaa', shape=[2, 3], default_value=2.)
+    a = fc.numeric_column_v2('aaa', shape=[2, 3], default_value=2.)
     self.assertEqual(((2., 2., 2.), (2., 2., 2.)), a.default_value)
 
   def test_parse_spec(self):
-    a = fc.numeric_column('aaa', shape=[2, 3], dtype=dtypes.int32)
+    a = fc.numeric_column_v2('aaa', shape=[2, 3], dtype=dtypes.int32)
     self.assertEqual({
         'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32)
     }, a.parse_example_spec)
 
   def test_parse_example_no_default_value(self):
-    price = fc.numeric_column('price', shape=[2])
+    price = fc.numeric_column_v2('price', shape=[2])
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -325,13 +325,13 @@ class NumericColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([price]))
+        features=fc.make_parse_example_spec_v2([price]))
     self.assertIn('price', features)
     with self.cached_session():
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
   def test_parse_example_with_default_value(self):
-    price = fc.numeric_column('price', shape=[2], default_value=11.)
+    price = fc.numeric_column_v2('price', shape=[2], default_value=11.)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -347,21 +347,22 @@ class NumericColumnTest(test.TestCase):
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString(),
                     no_data.SerializeToString()],
-        features=fc.make_parse_example_spec([price]))
+        features=fc.make_parse_example_spec_v2([price]))
     self.assertIn('price', features)
     with self.cached_session():
       self.assertAllEqual([[20., 110.], [11., 11.]], features['price'].eval())
 
   def test_normalizer_fn_must_be_callable(self):
     with self.assertRaisesRegexp(TypeError, 'must be a callable'):
-      fc.numeric_column('price', normalizer_fn='NotACallable')
+      fc.numeric_column_v2('price', normalizer_fn='NotACallable')
 
   def test_normalizer_fn_transform_feature(self):
 
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two)
+    price = fc.numeric_column_v2(
+        'price', shape=[2], normalizer_fn=_increment_two)
     output = fc._transform_features({
         'price': [[1., 2.], [5., 6.]]
     }, [price], None)
@@ -373,7 +374,8 @@ class NumericColumnTest(test.TestCase):
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two)
+    price = fc.numeric_column_v2(
+        'price', shape=[2], normalizer_fn=_increment_two)
     transformation_cache = fc.FeatureTransformationCache({
         'price': [[1., 2.], [5., 6.]]
     })
@@ -382,7 +384,7 @@ class NumericColumnTest(test.TestCase):
         price.get_dense_tensor(transformation_cache, None))
 
   def test_sparse_tensor_not_supported(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     transformation_cache = fc.FeatureTransformationCache({
         'price':
             sparse_tensor.SparseTensor(
@@ -392,19 +394,19 @@ class NumericColumnTest(test.TestCase):
       price.transform_feature(transformation_cache, None)
 
   def test_deep_copy(self):
-    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]])
+    a = fc.numeric_column_v2('aaa', shape=[1, 2], default_value=[[3., 2.]])
     a_copy = copy.deepcopy(a)
     self.assertEqual(a_copy.name, 'aaa')
     self.assertEqual(a_copy.shape, (1, 2))
     self.assertEqual(a_copy.default_value, ((3., 2.),))
 
   def test_numpy_default_value(self):
-    a = fc.numeric_column(
+    a = fc.numeric_column_v2(
         'aaa', shape=[1, 2], default_value=np.array([[3., 2.]]))
     self.assertEqual(a.default_value, ((3., 2.),))
 
   def test_linear_model(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price])
@@ -418,7 +420,7 @@ class NumericColumnTest(test.TestCase):
         self.assertAllClose([[10.], [50.]], predictions.eval())
 
   def test_old_linear_model(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc_old.linear_model(features, [price])
@@ -436,7 +438,7 @@ class NumericColumnTest(test.TestCase):
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column('price', normalizer_fn=_increment_two)
+    price = fc.numeric_column_v2('price', normalizer_fn=_increment_two)
     self.assertEqual(['price'], price.parents)
 
     config = price._get_config()
@@ -457,67 +459,67 @@ class NumericColumnTest(test.TestCase):
 class BucketizedColumnTest(test.TestCase):
 
   def test_invalid_source_column_type(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', hash_bucket_size=10)
+    a = fc.categorical_column_with_hash_bucket_v2('aaa', hash_bucket_size=10)
     with self.assertRaisesRegexp(
         ValueError,
         'source_column must be a column generated with numeric_column'):
-      fc.bucketized_column(a, boundaries=[0, 1])
+      fc.bucketized_column_v2(a, boundaries=[0, 1])
 
   def test_invalid_source_column_shape(self):
-    a = fc.numeric_column('aaa', shape=[2, 3])
+    a = fc.numeric_column_v2('aaa', shape=[2, 3])
     with self.assertRaisesRegexp(
         ValueError, 'source_column must be one-dimensional column'):
-      fc.bucketized_column(a, boundaries=[0, 1])
+      fc.bucketized_column_v2(a, boundaries=[0, 1])
 
   def test_invalid_boundaries(self):
-    a = fc.numeric_column('aaa')
+    a = fc.numeric_column_v2('aaa')
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=None)
+      fc.bucketized_column_v2(a, boundaries=None)
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=1.)
+      fc.bucketized_column_v2(a, boundaries=1.)
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=[1, 0])
+      fc.bucketized_column_v2(a, boundaries=[1, 0])
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=[1, 1])
+      fc.bucketized_column_v2(a, boundaries=[1, 1])
 
   def test_name(self):
-    a = fc.numeric_column('aaa', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc.numeric_column_v2('aaa', dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
     self.assertTrue(b._is_v2_column)
     self.assertEqual('aaa_bucketized', b.name)
 
   def test_is_v2_column_old_numeric(self):
     a = fc_old.numeric_column('aaa', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
     self.assertFalse(b._is_v2_column)
     self.assertEqual('aaa_bucketized', b.name)
 
   def test_parse_spec(self):
-    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc.numeric_column_v2('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
     self.assertEqual({
         'aaa': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32)
     }, b.parse_example_spec)
 
   def test_variable_shape(self):
-    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc.numeric_column_v2('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
     # Column 'aaa` has shape [2] times three buckets -> variable_shape=[2, 3].
     self.assertAllEqual((2, 3), b.variable_shape)
 
   def test_num_buckets(self):
-    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc.numeric_column_v2('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
     # Column 'aaa` has shape [2] times three buckets -> num_buckets=6.
     self.assertEqual(6, b.num_buckets)
 
   def test_parse_example(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 50])
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -526,14 +528,14 @@ class BucketizedColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([bucketized_price]))
+        features=fc.make_parse_example_spec_v2([bucketized_price]))
     self.assertIn('price', features)
     with self.cached_session():
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
   def test_transform_feature(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformed_tensor = fc._transform_features({
           'price': [[-1., 1.], [5., 6.]]
@@ -544,8 +546,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_dense_tensor_one_input_value(self):
     """Tests _get_dense_tensor() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[1])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1.], [1.], [5.], [6.]]
@@ -563,8 +565,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_dense_tensor_two_input_values(self):
     """Tests _get_dense_tensor() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1., 1.], [5., 6.]]
@@ -580,8 +582,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_one_input_value(self):
     """Tests _get_sparse_tensors() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[1])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1.], [1.], [5.], [6.]]
@@ -598,8 +600,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_two_input_values(self):
     """Tests _get_sparse_tensors() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1., 1.], [5., 6.]]
@@ -617,8 +619,8 @@ class BucketizedColumnTest(test.TestCase):
         self.assertAllEqual([2, 2], id_tensor_value.dense_shape)
 
   def test_sparse_tensor_input_not_supported(self):
-    price = fc.numeric_column('price')
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 1])
+    price = fc.numeric_column_v2('price')
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 1])
     transformation_cache = fc.FeatureTransformationCache({
         'price':
             sparse_tensor.SparseTensor(
@@ -628,8 +630,8 @@ class BucketizedColumnTest(test.TestCase):
       bucketized_price.transform_feature(transformation_cache, None)
 
   def test_deep_copy(self):
-    a = fc.numeric_column('aaa', shape=[2])
-    a_bucketized = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc.numeric_column_v2('aaa', shape=[2])
+    a_bucketized = fc.bucketized_column_v2(a, boundaries=[0, 1])
     a_bucketized_copy = copy.deepcopy(a_bucketized)
     self.assertEqual(a_bucketized_copy.name, 'aaa_bucketized')
     self.assertAllEqual(a_bucketized_copy.variable_shape, (2, 3))
@@ -637,8 +639,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_linear_model_one_input_value(self):
     """Tests linear_model() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[1])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       model = fc.LinearModel([bucketized_price])
@@ -662,8 +664,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1., 1.], [5., 6.]]}
       model = fc.LinearModel([bucketized_price])
@@ -691,8 +693,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_old_linear_model_one_input_value(self):
     """Tests linear_model() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[1])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       predictions = fc_old.linear_model(features, [bucketized_price])
@@ -716,8 +718,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_old_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1., 1.], [5., 6.]]}
       predictions = fc_old.linear_model(features, [bucketized_price])
@@ -746,7 +748,7 @@ class BucketizedColumnTest(test.TestCase):
   def test_old_linear_model_one_input_value_old_numeric(self):
     """Tests linear_model() for input with shape=[1]."""
     price = fc_old.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       predictions = fc_old.linear_model(features, [bucketized_price])
@@ -769,8 +771,8 @@ class BucketizedColumnTest(test.TestCase):
         self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
 
   def test_serialization(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
     self.assertEqual([price], bucketized_price.parents)
 
     config = bucketized_price._get_config()
@@ -801,7 +803,7 @@ class BucketizedColumnTest(test.TestCase):
 class HashedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10)
+    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
     self.assertEqual('aaa', a.name)
     self.assertEqual('aaa', a.key)
     self.assertEqual(10, a.hash_bucket_size)
@@ -810,25 +812,25 @@ class HashedCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_hash_bucket(('key',), 10)
+      fc.categorical_column_with_hash_bucket_v2(('key',), 10)
 
   def test_bucket_size_should_be_given(self):
     with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be set.'):
-      fc.categorical_column_with_hash_bucket('aaa', None)
+      fc.categorical_column_with_hash_bucket_v2('aaa', None)
 
   def test_bucket_size_should_be_positive(self):
     with self.assertRaisesRegexp(ValueError,
                                  'hash_bucket_size must be at least 1'):
-      fc.categorical_column_with_hash_bucket('aaa', 0)
+      fc.categorical_column_with_hash_bucket_v2('aaa', 0)
 
   def test_dtype_should_be_string_or_integer(self):
-    fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string)
-    fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
+    fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.string)
+    fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.int32)
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32)
+      fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.float32)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_hash_bucket('aaa', 10)
+    original = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(10, column.hash_bucket_size)
@@ -836,19 +838,19 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertEqual(dtypes.string, column.dtype)
 
   def test_parse_spec_string(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10)
+    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.string)
     }, a.parse_example_spec)
 
   def test_parse_spec_int(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
+    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.int32)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, a.parse_example_spec)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10)
+    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -857,7 +859,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a]))
+        features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -869,7 +871,7 @@ class HashedCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_strings_should_be_hashed(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
     wire_tensor = sparse_tensor.SparseTensor(
         values=['omar', 'stringer', 'marlo'],
         indices=[[0, 0], [1, 0], [1, 1]],
@@ -888,11 +890,11 @@ class HashedCategoricalColumnTest(test.TestCase):
                           output.dense_shape.eval())
 
   def test_tensor_dtype_should_be_string_or_integer(self):
-    string_fc = fc.categorical_column_with_hash_bucket(
+    string_fc = fc.categorical_column_with_hash_bucket_v2(
         'a_string', 10, dtype=dtypes.string)
-    int_fc = fc.categorical_column_with_hash_bucket(
+    int_fc = fc.categorical_column_with_hash_bucket_v2(
         'a_int', 10, dtype=dtypes.int32)
-    float_fc = fc.categorical_column_with_hash_bucket(
+    float_fc = fc.categorical_column_with_hash_bucket_v2(
         'a_float', 10, dtype=dtypes.string)
     int_tensor = sparse_tensor.SparseTensor(
         values=[101],
@@ -917,7 +919,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       transformation_cache.get(float_fc, None)
 
   def test_dtype_should_match_with_tensor(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket(
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -926,7 +928,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       transformation_cache.get(hashed_sparse, None)
 
   def test_ints_should_be_hashed(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket(
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=[101, 201, 301],
@@ -940,7 +942,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, output.values.eval())
 
   def test_int32_64_is_compatible(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket(
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=constant_op.constant([101, 201, 301], dtype=dtypes.int32),
@@ -954,7 +956,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, output.values.eval())
 
   def test_get_sparse_tensors(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
     transformation_cache = fc.FeatureTransformationCache({
         'wire':
             sparse_tensor.SparseTensor(
@@ -969,7 +971,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor)
 
   def test_get_sparse_tensors_dense_input(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
     transformation_cache = fc.FeatureTransformationCache({
         'wire': (('omar', ''), ('stringer', 'marlo'))
     })
@@ -980,7 +982,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor)
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
+    wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
     self.assertEqual(4, wire_column.num_buckets)
     with ops.Graph().as_default():
       model = fc.LinearModel((wire_column,))
@@ -1002,7 +1004,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((4.,), (6.,)), predictions.eval())
 
   def test_old_linear_model(self):
-    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
+    wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
     self.assertEqual(4, wire_column.num_buckets)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
@@ -1024,7 +1026,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((4.,), (6.,)), predictions.eval())
 
   def test_serialization(self):
-    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
+    wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
     self.assertEqual(['wire'], wire_column.parents)
 
     config = wire_column._get_config()
@@ -1043,104 +1045,104 @@ class CrossedColumnTest(test.TestCase):
   def test_keys_empty(self):
     with self.assertRaisesRegexp(
         ValueError, 'keys must be a list with length > 1'):
-      fc.crossed_column([], 10)
+      fc.crossed_column_v2([], 10)
 
   def test_keys_length_one(self):
     with self.assertRaisesRegexp(
         ValueError, 'keys must be a list with length > 1'):
-      fc.crossed_column(['a'], 10)
+      fc.crossed_column_v2(['a'], 10)
 
   def test_key_type_unsupported(self):
     with self.assertRaisesRegexp(ValueError, 'Unsupported key type'):
-      fc.crossed_column(['a', fc.numeric_column('c')], 10)
+      fc.crossed_column_v2(['a', fc.numeric_column_v2('c')], 10)
 
     with self.assertRaisesRegexp(
         ValueError, 'categorical_column_with_hash_bucket is not supported'):
-      fc.crossed_column(
-          ['a', fc.categorical_column_with_hash_bucket('c', 10)], 10)
+      fc.crossed_column_v2(
+          ['a', fc.categorical_column_with_hash_bucket_v2('c', 10)], 10)
 
   def test_hash_bucket_size_negative(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column(['a', 'c'], -1)
+      fc.crossed_column_v2(['a', 'c'], -1)
 
   def test_hash_bucket_size_zero(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column(['a', 'c'], 0)
+      fc.crossed_column_v2(['a', 'c'], 0)
 
   def test_hash_bucket_size_none(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column(['a', 'c'], None)
+      fc.crossed_column_v2(['a', 'c'], None)
 
   def test_name(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
     self.assertTrue(crossed1._is_v2_column)
 
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 10)
+    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 10)
     self.assertTrue(crossed2._is_v2_column)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_is_v2_column(self):
     a = fc_old.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
     self.assertTrue(crossed1._is_v2_column)
 
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 10)
+    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 10)
     self.assertFalse(crossed2._is_v2_column)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_name_ordered_alphabetically(self):
     """Tests that the name does not depend on the order of given columns."""
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
 
-    crossed2 = fc.crossed_column([crossed1, 'c', b], 10)
+    crossed2 = fc.crossed_column_v2([crossed1, 'c', b], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_name_leaf_keys_ordered_alphabetically(self):
     """Tests that the name does not depend on the order of given columns."""
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d2', 'c'], 10)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column_v2(['d2', 'c'], 10)
 
-    crossed2 = fc.crossed_column([crossed1, 'd1', b], 10)
+    crossed2 = fc.crossed_column_v2([crossed1, 'd1', b], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_parse_spec(self):
-    a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed = fc.crossed_column([b, 'c'], 10)
+    a = fc.numeric_column_v2('a', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed = fc.crossed_column_v2([b, 'c'], 10)
     self.assertEqual({
         'a': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32),
         'c': parsing_ops.VarLenFeature(dtypes.string),
     }, crossed.parse_example_spec)
 
   def test_num_buckets(self):
-    a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed = fc.crossed_column([b, 'c'], 15)
+    a = fc.numeric_column_v2('a', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed = fc.crossed_column_v2([b, 'c'], 15)
     self.assertEqual(15, crossed.num_buckets)
 
   def test_deep_copy(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
+    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
+    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 15, hash_key=5)
     crossed2_copy = copy.deepcopy(crossed2)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2_copy.name,)
     self.assertEqual(15, crossed2_copy.hash_bucket_size)
     self.assertEqual(5, crossed2_copy.hash_key)
 
   def test_parse_example(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
-    price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], 10)
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 50])
+    price_cross_wire = fc.crossed_column_v2([bucketized_price, 'wire'], 10)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -1152,7 +1154,7 @@ class CrossedColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([price_cross_wire]))
+        features=fc.make_parse_example_spec_v2([price_cross_wire]))
     self.assertIn('price', features)
     self.assertIn('wire', features)
     with self.cached_session():
@@ -1164,11 +1166,11 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval())
 
   def test_transform_feature(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
+    price = fc.numeric_column_v2('price', shape=[2])
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 50])
     hash_bucket_size = 10
-    price_cross_wire = fc.crossed_column(
-        [bucketized_price, 'wire'], hash_bucket_size)
+    price_cross_wire = fc.crossed_column_v2([bucketized_price, 'wire'],
+                                            hash_bucket_size)
     features = {
         'price': constant_op.constant([[1., 2.], [5., 6.]]),
         'wire': sparse_tensor.SparseTensor(
@@ -1187,10 +1189,10 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([2, 4], output_val.dense_shape)
 
   def test_get_sparse_tensors(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
+    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
+    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 15, hash_key=5)
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'a':
@@ -1228,9 +1230,9 @@ class CrossedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_simple(self):
     """Same as test_get_sparse_tensors, but with simpler values."""
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
+    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'a':
@@ -1258,9 +1260,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
+    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       model = fc.LinearModel((crossed,))
       predictions = model({
@@ -1322,7 +1324,7 @@ class CrossedColumnTest(test.TestCase):
             id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
 
     t = _TestColumnWithWeights()
-    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
+    crossed = fc.crossed_column_v2([t, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
           ValueError,
@@ -1351,9 +1353,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
+    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
           'a':
@@ -1433,7 +1435,7 @@ class CrossedColumnTest(test.TestCase):
             id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
 
     t = _TestColumnWithWeights()
-    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
+    crossed = fc.crossed_column_v2([t, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
           ValueError,
@@ -1462,8 +1464,8 @@ class CrossedColumnTest(test.TestCase):
     Uses data from test_get_sparse_tesnsors_simple.
     """
     a = fc_old.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
+    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
           'a':
@@ -1488,9 +1490,9 @@ class CrossedColumnTest(test.TestCase):
         self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
 
   def test_serialization(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
+    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
 
     self.assertEqual([b, 'c'], crossed.parents)
 
@@ -1566,25 +1568,26 @@ class LinearModelTest(test.TestCase):
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
-      fc.LinearModel(feature_columns={'a': fc.numeric_column('a')})
+      fc.LinearModel(feature_columns={'a': fc.numeric_column_v2('a')})
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
-      fc.LinearModel(
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])
+      fc.LinearModel(feature_columns=[
+          fc.numeric_column_v2('a'),
+          fc.numeric_column_v2('a')
+      ])
 
   def test_not_dict_input_features(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = [[1.], [5.]]
       model = fc.LinearModel([price])
       with self.assertRaisesRegexp(ValueError, 'We expected a dictionary here'):
-        predictions = model(features)
+        model(features)
 
   def test_dense_bias(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price])
@@ -1597,7 +1600,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[15.], [55.]], predictions.eval())
 
   def test_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1615,8 +1618,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], predictions.eval())
 
   def test_dense_and_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    price = fc.numeric_column('price')
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1688,7 +1691,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], predictions.eval())
 
   def test_dense_multi_output(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price], units=3)
@@ -1703,7 +1706,7 @@ class LinearModelTest(test.TestCase):
                             predictions.eval())
 
   def test_sparse_multi_output(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1725,7 +1728,7 @@ class LinearModelTest(test.TestCase):
                             predictions.eval())
 
   def test_dense_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       model = fc.LinearModel([price])
@@ -1737,7 +1740,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], predictions.eval())
 
   def test_sparse_multi_rank(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = array_ops.sparse_placeholder(dtypes.string)
       wire_value = sparse_tensor.SparseTensorValue(
@@ -1759,7 +1762,7 @@ class LinearModelTest(test.TestCase):
             predictions.eval(feed_dict={wire_tensor: wire_value}))
 
   def test_sparse_combiner(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1775,8 +1778,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [5010.]], predictions.eval())
 
   def test_sparse_combiner_with_negative_weights(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights')
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast_weights = fc.weighted_categorical_column_v2(wire_cast, 'weights')
 
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
@@ -1796,7 +1799,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [-9985.]], predictions.eval())
 
   def test_dense_multi_dimension_multi_output(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       model = fc.LinearModel([price], units=3)
@@ -1811,7 +1814,7 @@ class LinearModelTest(test.TestCase):
                             predictions.eval())
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -1821,7 +1824,7 @@ class LinearModelTest(test.TestCase):
         model(features)
 
   def test_dense_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc.numeric_column_v2('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       model = fc.LinearModel([price])
@@ -1835,8 +1838,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], predictions.eval())
 
   def test_dense_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
@@ -1856,7 +1859,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[3217.], [4657.]], predictions.eval())
 
   def test_dense_trainable_default(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price])
@@ -1867,7 +1870,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(price_var, trainable_vars)
 
   def test_sparse_trainable_default(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1880,7 +1883,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, trainable_vars)
 
   def test_dense_trainable_false(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price], trainable=False)
@@ -1889,7 +1892,7 @@ class LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_sparse_trainable_false(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1900,9 +1903,9 @@ class LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price_a = fc.numeric_column_v2('price_a')
+    price_b = fc.numeric_column_v2('price_b')
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
@@ -1936,13 +1939,13 @@ class LinearModelTest(test.TestCase):
       self.assertIn('wire_cast', my_vars[2].name)
 
   def test_variable_names(self):
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc.numeric_column_v2('price1')
+    dense_feature = fc.numeric_column_v2('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column_v2(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)
     all_cols = [price1, dense_feature_bucketized, some_embedding_column]
 
@@ -1966,7 +1969,7 @@ class LinearModelTest(test.TestCase):
       ], variable_names)
 
   def test_fit_and_predict(self):
-    columns = [fc.numeric_column('a')]
+    columns = [fc.numeric_column_v2('a')]
 
     model = fc.LinearModel(columns)
     model.compile(
@@ -1983,8 +1986,8 @@ class LinearModelTest(test.TestCase):
     model.predict(x, batch_size=5)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -1997,9 +2000,9 @@ class LinearModelTest(test.TestCase):
       model(features)
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
+    price3 = fc.numeric_column_v2('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2013,8 +2016,8 @@ class LinearModelTest(test.TestCase):
         model(features)
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2029,8 +2032,8 @@ class LinearModelTest(test.TestCase):
               predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -2047,14 +2050,14 @@ class LinearModelTest(test.TestCase):
             })
 
   def test_with_numpy_input_fn(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc.numeric_column_v2('price')
+    price_buckets = fc.bucketized_column_v2(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     input_fn = numpy_io.numpy_input_fn(
@@ -2084,14 +2087,14 @@ class LinearModelTest(test.TestCase):
       coord.join(threads)
 
   def test_with_1d_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc.numeric_column_v2('price')
+    price_buckets = fc.bucketized_column_v2(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2117,16 +2120,16 @@ class LinearModelTest(test.TestCase):
       self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc.numeric_column_v2('price')
+    price_buckets = fc.bucketized_column_v2(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc.categorical_column_with_vocabulary_list_v2(
         'country', vocabulary_list=['US', 'JP', 'CA'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2163,7 +2166,7 @@ class LinearModelTest(test.TestCase):
                               }))
 
   def test_with_rank_0_feature(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -2186,7 +2189,7 @@ class LinearModelTest(test.TestCase):
         sess.run(net, feed_dict={features['price']: np.array(1)})
 
   def test_multiple_linear_models(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features1 = {'price': [[1.], [5.]]}
       features2 = {'price': [[2.], [10.]]}
@@ -2254,18 +2257,21 @@ class OldLinearModelTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc_old.linear_model(
-          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
+          features={'a': [[0]]},
+          feature_columns={'a': fc.numeric_column_v2('a')})
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
       fc_old.linear_model(
           features={'a': [[0]]},
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])
+          feature_columns=[
+              fc.numeric_column_v2('a'),
+              fc.numeric_column_v2('a')
+          ])
 
   def test_dense_bias(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc_old.linear_model(features, [price])
@@ -2278,7 +2284,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[15.], [55.]], predictions.eval())
 
   def test_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2296,8 +2302,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], predictions.eval())
 
   def test_dense_and_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    price = fc.numeric_column('price')
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2397,7 +2403,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], predictions.eval())
 
   def test_dense_multi_output(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc_old.linear_model(features, [price], units=3)
@@ -2412,7 +2418,7 @@ class OldLinearModelTest(test.TestCase):
                             predictions.eval())
 
   def test_sparse_multi_output(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2434,7 +2440,7 @@ class OldLinearModelTest(test.TestCase):
                             predictions.eval())
 
   def test_dense_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = fc_old.linear_model(features, [price])
@@ -2445,7 +2451,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], predictions.eval())
 
   def test_sparse_multi_rank(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = array_ops.sparse_placeholder(dtypes.string)
       wire_value = sparse_tensor.SparseTensorValue(
@@ -2466,7 +2472,7 @@ class OldLinearModelTest(test.TestCase):
             predictions.eval(feed_dict={wire_tensor: wire_value}))
 
   def test_sparse_combiner(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2483,8 +2489,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [5010.]], predictions.eval())
 
   def test_sparse_combiner_with_negative_weights(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights')
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast_weights = fc.weighted_categorical_column_v2(wire_cast, 'weights')
 
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
@@ -2505,7 +2511,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [-9985.]], predictions.eval())
 
   def test_dense_multi_dimension_multi_output(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = fc_old.linear_model(features, [price], units=3)
@@ -2520,7 +2526,7 @@ class OldLinearModelTest(test.TestCase):
                             predictions.eval())
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -2529,7 +2535,7 @@ class OldLinearModelTest(test.TestCase):
         fc_old.linear_model(features, [price])
 
   def test_dense_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc.numeric_column_v2('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       predictions = fc_old.linear_model(features, [price])
@@ -2543,8 +2549,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], predictions.eval())
 
   def test_dense_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       predictions = fc_old.linear_model(features, [price1, price2])
@@ -2562,8 +2568,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[3217.], [4657.]], predictions.eval())
 
   def test_fills_cols_to_vars(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       cols_to_vars = {}
@@ -2576,8 +2582,8 @@ class OldLinearModelTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[price2], [price2_var])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2', shape=3)
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2', shape=3)
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [6., 7.]],
@@ -2603,13 +2609,13 @@ class OldLinearModelTest(test.TestCase):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
     # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
     # creates a Variable.
-    apple_numeric_column = fc.numeric_column('apple_numeric_column')
-    banana_dense_feature = fc.numeric_column('banana_dense_feature')
-    banana_dense_feature_bucketized = fc.bucketized_column(
+    apple_numeric_column = fc.numeric_column_v2('apple_numeric_column')
+    banana_dense_feature = fc.numeric_column_v2('banana_dense_feature')
+    banana_dense_feature_bucketized = fc.bucketized_column_v2(
         banana_dense_feature, boundaries=[0.])
-    cherry_sparse_column = fc.categorical_column_with_hash_bucket(
+    cherry_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'cherry_sparse_feature', hash_bucket_size=5)
-    dragonfruit_embedding_column = fc.embedding_column(
+    dragonfruit_embedding_column = fc.embedding_column_v2(
         cherry_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -2634,7 +2640,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertItemsEqual(input_layer_inputs, output_tensors)
 
   def test_dense_collection(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc_old.linear_model(features, [price], weight_collections=['my-vars'])
@@ -2645,7 +2651,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(price_var, my_vars)
 
   def test_sparse_collection(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2658,7 +2664,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, my_vars)
 
   def test_dense_trainable_default(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc_old.linear_model(features, [price])
@@ -2669,7 +2675,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(price_var, trainable_vars)
 
   def test_sparse_trainable_default(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2682,7 +2688,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, trainable_vars)
 
   def test_dense_trainable_false(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc_old.linear_model(features, [price], trainable=False)
@@ -2690,7 +2696,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_sparse_trainable_false(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2700,9 +2706,9 @@ class OldLinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price_a = fc.numeric_column_v2('price_a')
+    price_b = fc.numeric_column_v2('price_b')
+    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
     with ops.Graph().as_default() as g:
       features = {
           'price_a': [[1.]],
@@ -2736,8 +2742,8 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn('wire_cast', my_vars[2].name)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -2749,9 +2755,9 @@ class OldLinearModelTest(test.TestCase):
       fc_old.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
+    price3 = fc.numeric_column_v2('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2764,8 +2770,8 @@ class OldLinearModelTest(test.TestCase):
         fc_old.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2779,8 +2785,8 @@ class OldLinearModelTest(test.TestCase):
               predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -2796,14 +2802,14 @@ class OldLinearModelTest(test.TestCase):
             })
 
   def test_with_1d_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc.numeric_column_v2('price')
+    price_buckets = fc.bucketized_column_v2(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2835,16 +2841,16 @@ class OldLinearModelTest(test.TestCase):
       self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc.numeric_column_v2('price')
+    price_buckets = fc.bucketized_column_v2(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc.categorical_column_with_vocabulary_list_v2(
         'country', vocabulary_list=['US', 'JP', 'CA'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2880,7 +2886,7 @@ class OldLinearModelTest(test.TestCase):
                               }))
 
   def test_with_rank_0_feature(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -2901,7 +2907,7 @@ class OldLinearModelTest(test.TestCase):
         sess.run(net, feed_dict={features['price']: np.array(1)})
 
   def test_multiple_linear_models(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features1 = {'price': [[1.], [5.]]}
       features2 = {'price': [[2.], [10.]]}
@@ -2922,10 +2928,10 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[25.], [105.]], predictions2.eval())
 
   def test_linear_model_v1_shared_embedding_all_other_v2(self):
-    price = fc.numeric_column('price')  # v2
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    price = fc.numeric_column_v2('price')  # v2
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)  # v2
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)  # v2
     categorical_column_a = fc_old.categorical_column_with_identity(
         key='aaa', num_buckets=3)  # v2
@@ -2958,14 +2964,14 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([0.], bias.eval())
 
   def test_linear_model_v1_shared_embedding_with_v2_cat_all_other_v2(self):
-    price = fc.numeric_column('price')  # v2
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    price = fc.numeric_column_v2('price')  # v2
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)  # v2
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)  # v2
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)  # v2
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)  # v2
     shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)  # v1
@@ -2994,7 +3000,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([0.], bias.eval())
 
   def test_linear_model_v1_v2_mix(self):
-    price = fc.numeric_column('price')  # v2
+    price = fc.numeric_column_v2('price')  # v2
     some_sparse_column = fc_old.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)  # v1
     some_embedding_column = fc_old.embedding_column(
@@ -3035,9 +3041,9 @@ class OldLinearModelTest(test.TestCase):
         'sparse_feature', hash_bucket_size=5)  # v1
     some_embedding_column = fc_old.embedding_column(
         some_sparse_column, dimension=10)  # v1
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)  # v2
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)  # v2
     shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b], dimension=2)  # v2
@@ -3065,13 +3071,13 @@ class OldLinearModelTest(test.TestCase):
         fc_old.linear_model(features, all_cols)
 
 
-class FeatureLayerTest(test.TestCase):
+class DenseFeaturesTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes()
   def test_retrieving_input(self):
     features = {'a': [0.]}
-    feature_layer = fc.FeatureLayer(fc.numeric_column('a'))
-    inputs = self.evaluate(feature_layer(features))
+    dense_features = fc.DenseFeatures(fc.numeric_column_v2('a'))
+    inputs = self.evaluate(dense_features(features))
     self.assertAllClose([[0.]], inputs)
 
   def test_reuses_variables(self):
@@ -3082,7 +3088,7 @@ class FeatureLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity(
+      categorical_column = fc.categorical_column_with_identity_v2(
           key='a', num_buckets=3)
       embedding_dimension = 2
       def _embedding_column_initializer(shape, dtype, partition_info):
@@ -3095,16 +3101,16 @@ class FeatureLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column(
+      embedding_column = fc.embedding_column_v2(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
 
-      feature_layer = fc.FeatureLayer([embedding_column])
+      dense_features = fc.DenseFeatures([embedding_column])
       features = {'a': sparse_input}
 
-      inputs = feature_layer(features)
-      variables = feature_layer.variables
+      inputs = dense_features(features)
+      variables = dense_features.variables
 
       # Sanity check: test that the inputs are correct.
       self.assertAllEqual([[1, 0], [0, 1], [1, 1]], inputs)
@@ -3112,13 +3118,13 @@ class FeatureLayerTest(test.TestCase):
       # Check that only one variable was created.
       self.assertEqual(1, len(variables))
 
-      # Check that invoking feature_layer on the same features does not create
+      # Check that invoking dense_features on the same features does not create
       # additional variables
-      _ = feature_layer(features)
+      _ = dense_features(features)
       self.assertEqual(1, len(variables))
-      self.assertEqual(variables[0], feature_layer.variables[0])
+      self.assertEqual(variables[0], dense_features.variables[0])
 
-  def test_feature_column_feature_layer_gradient(self):
+  def test_feature_column_dense_features_gradient(self):
     with context.eager_mode():
       sparse_input = sparse_tensor.SparseTensor(
           indices=((0, 0), (1, 0), (2, 0)),
@@ -3126,7 +3132,7 @@ class FeatureLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity(
+      categorical_column = fc.categorical_column_with_identity_v2(
           key='a', num_buckets=3)
       embedding_dimension = 2
 
@@ -3140,16 +3146,16 @@ class FeatureLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column(
+      embedding_column = fc.embedding_column_v2(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
 
-      feature_layer = fc.FeatureLayer([embedding_column])
+      dense_features = fc.DenseFeatures([embedding_column])
       features = {'a': sparse_input}
 
       def scale_matrix():
-        matrix = feature_layer(features)
+        matrix = dense_features(features)
         return 2 * matrix
 
       # Sanity check: Verify that scale_matrix returns the correct output.
@@ -3167,12 +3173,12 @@ class FeatureLayerTest(test.TestCase):
   def test_raises_if_empty_feature_columns(self):
     with self.assertRaisesRegexp(ValueError,
                                  'feature_columns must not be empty'):
-      fc.FeatureLayer(feature_columns=[])(features={})
+      fc.DenseFeatures(feature_columns=[])(features={})
 
   def test_should_be_dense_column(self):
     with self.assertRaisesRegexp(ValueError, 'must be a DenseColumn'):
-      fc.FeatureLayer(feature_columns=[
-          fc.categorical_column_with_hash_bucket('wire_cast', 4)
+      fc.DenseFeatures(feature_columns=[
+          fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
       ])(
           features={
               'a': [[0]]
@@ -3181,7 +3187,7 @@ class FeatureLayerTest(test.TestCase):
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
-      fc.FeatureLayer(feature_columns={'a': fc.numeric_column('a')})(
+      fc.DenseFeatures(feature_columns={'a': fc.numeric_column_v2('a')})(
           features={
               'a': [[0]]
           })
@@ -3189,117 +3195,118 @@ class FeatureLayerTest(test.TestCase):
   def test_bare_column(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
-      net = fc.FeatureLayer(fc.numeric_column('a'))(features)
+      net = fc.DenseFeatures(fc.numeric_column_v2('a'))(features)
       with _initialized_session():
         self.assertAllClose([[0.]], net.eval())
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
-      columns = (fc.numeric_column(key) for key in features)
-      net = fc.FeatureLayer(columns)(features)
+      columns = (fc.numeric_column_v2(key) for key in features)
+      net = fc.DenseFeatures(columns)(features)
       with _initialized_session():
         self.assertAllClose([[0., 1.]], net.eval())
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
-      fc.FeatureLayer(
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])(
-                               features={
-                                   'a': [[0]]
-                               })
+      fc.DenseFeatures(feature_columns=[
+          fc.numeric_column_v2('a'),
+          fc.numeric_column_v2('a')
+      ])(
+          features={
+              'a': [[0]]
+          })
 
   def test_one_column(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
-      net = fc.FeatureLayer([price])(features)
+      net = fc.DenseFeatures([price])(features)
       with _initialized_session():
         self.assertAllClose([[1.], [5.]], net.eval())
 
   def test_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
-      net = fc.FeatureLayer([price])(features)
+      net = fc.DenseFeatures([price])(features)
       with _initialized_session():
         self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
 
   def test_compute_output_shape(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2', shape=4)
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2', shape=4)
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
           'price2': [[3., 4., 5., 6.], [7., 8., 9., 10.]]
       }
-      feature_layer = fc.FeatureLayer([price1, price2])
-      self.assertEqual((None, 6), feature_layer.compute_output_shape((None,)))
-      net = feature_layer(features)
+      dense_features = fc.DenseFeatures([price1, price2])
+      self.assertEqual((None, 6), dense_features.compute_output_shape((None,)))
+      net = dense_features(features)
       with _initialized_session():
         self.assertAllClose(
             [[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]], net.eval())
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
           Exception,
           r'Cannot reshape a tensor with 2 elements to shape \[2,2\]'):
-        fc.FeatureLayer([price])(features)
+        fc.DenseFeatures([price])(features)
 
   def test_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc.numeric_column_v2('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
-      net = fc.FeatureLayer([price])(features)
+      net = fc.DenseFeatures([price])(features)
       with _initialized_session():
         self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
 
   def test_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
           'price2': [[3.], [4.]]
       }
-      net = fc.FeatureLayer([price1, price2])(features)
+      net = fc.DenseFeatures([price1, price2])(features)
       with _initialized_session():
         self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
 
   def test_cols_to_output_tensors(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       cols_dict = {}
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
-      feature_layer = fc.FeatureLayer([price1, price2])
-      net = feature_layer(features, cols_dict)
+      dense_features = fc.DenseFeatures([price1, price2])
+      net = dense_features(features, cols_dict)
       with _initialized_session():
         self.assertAllClose([[1., 2.], [5., 6.]], cols_dict[price1].eval())
         self.assertAllClose([[3.], [4.]], cols_dict[price2].eval())
         self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
+    price_a = fc.numeric_column_v2('price_a')
+    price_b = fc.numeric_column_v2('price_b')
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
           'price_b': [[3.]],
       }
-      net1 = fc.FeatureLayer([price_a, price_b])(features)
-      net2 = fc.FeatureLayer([price_b, price_a])(features)
+      net1 = fc.DenseFeatures([price_a, price_b])(features)
+      net2 = fc.DenseFeatures([price_b, price_a])(features)
       with _initialized_session():
         self.assertAllClose([[1., 3.]], net1.eval())
         self.assertAllClose([[1., 3.]], net2.eval())
 
   def test_fails_for_categorical_column(self):
-    animal = fc.categorical_column_with_identity('animal', num_buckets=4)
+    animal = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -3307,11 +3314,11 @@ class FeatureLayerTest(test.TestCase):
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
       with self.assertRaisesRegexp(Exception, 'must be a DenseColumn'):
-        fc.FeatureLayer([animal])(features)
+        fc.DenseFeatures([animal])(features)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -3320,12 +3327,12 @@ class FeatureLayerTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError,
           'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
-        fc.FeatureLayer([price1, price2])(features)
+        fc.DenseFeatures([price1, price2])(features)
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
+    price3 = fc.numeric_column_v2('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -3335,31 +3342,31 @@ class FeatureLayerTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError,
           'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
-        fc.FeatureLayer([price1, price2, price3])(features)
+        fc.DenseFeatures([price1, price2, price3])(features)
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
           'price2': [[3.], [4.]]  # batchsize = 2
       }
-      net = fc.FeatureLayer([price1, price2])(features)
+      net = fc.DenseFeatures([price1, price2])(features)
       with _initialized_session() as sess:
         with self.assertRaisesRegexp(errors.OpError,
                                      'Dimensions of inputs should match'):
           sess.run(net, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
           'price2': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
       }
-      net = fc.FeatureLayer([price1, price2])(features)
+      net = fc.DenseFeatures([price1, price2])(features)
       with _initialized_session() as sess:
         sess.run(
             net,
@@ -3369,9 +3376,9 @@ class FeatureLayerTest(test.TestCase):
             })
 
   def test_multiple_layers_with_same_embedding_column(self):
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)
 
     with ops.Graph().as_default():
@@ -3379,23 +3386,23 @@ class FeatureLayerTest(test.TestCase):
           'sparse_feature': [['a'], ['x']],
       }
       all_cols = [some_embedding_column]
-      fc.FeatureLayer(all_cols)(features)
-      fc.FeatureLayer(all_cols)(features)
+      fc.DenseFeatures(all_cols)(features)
+      fc.DenseFeatures(all_cols)(features)
       # Make sure that 2 variables get created in this case.
       self.assertEqual(2, len(
           ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)))
       expected_var_names = [
-          'feature_layer/sparse_feature_embedding/embedding_weights:0',
-          'feature_layer_1/sparse_feature_embedding/embedding_weights:0'
+          'dense_features/sparse_feature_embedding/embedding_weights:0',
+          'dense_features_1/sparse_feature_embedding/embedding_weights:0'
       ]
       self.assertItemsEqual(
           expected_var_names,
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
   def test_multiple_layers_with_same_shared_embedding_column(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_b, embedding_column_a = fc.shared_embedding_columns_v2(
@@ -3416,8 +3423,8 @@ class FeatureLayerTest(test.TestCase):
                   dense_shape=(2, 2)),
       }
       all_cols = [embedding_column_a, embedding_column_b]
-      fc.FeatureLayer(all_cols)(features)
-      fc.FeatureLayer(all_cols)(features)
+      fc.DenseFeatures(all_cols)(features)
+      fc.DenseFeatures(all_cols)(features)
       # Make sure that only 1 variable gets created in this case.
       self.assertEqual(1, len(
           ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)))
@@ -3426,9 +3433,9 @@ class FeatureLayerTest(test.TestCase):
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
   def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_b, embedding_column_a = fc.shared_embedding_columns_v2(
@@ -3449,7 +3456,7 @@ class FeatureLayerTest(test.TestCase):
                   values=(1, 2, 1),
                   dense_shape=(2, 2)),
       }
-      fc.FeatureLayer(all_cols)(features)
+      fc.DenseFeatures(all_cols)(features)
       # Make sure that only 1 variable gets created in this case.
       self.assertEqual(1, len(
           ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)))
@@ -3468,7 +3475,7 @@ class FeatureLayerTest(test.TestCase):
                   dense_shape=(2, 2)),
       }
 
-      fc.FeatureLayer(all_cols)(features1)
+      fc.DenseFeatures(all_cols)(features1)
       # Make sure that only 1 variable gets created in this case.
       self.assertEqual(1, len(
           ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)))
@@ -3486,14 +3493,14 @@ class FeatureLayerTest(test.TestCase):
       del shape, dtype, partition_info
       return embedding_values
 
-    # price has 1 dimension in feature_layer
-    price = fc.numeric_column('price')
-    body_style = fc.categorical_column_with_vocabulary_list(
+    # price has 1 dimension in dense_features
+    price = fc.numeric_column_v2('price')
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    # one_hot_body_style has 3 dims in feature_layer.
-    one_hot_body_style = fc.indicator_column(body_style)
-    # embedded_body_style has 5 dims in feature_layer.
-    embedded_body_style = fc.embedding_column(
+    # one_hot_body_style has 3 dims in dense_features.
+    one_hot_body_style = fc.indicator_column_v2(body_style)
+    # embedded_body_style has 5 dims in dense_features.
+    embedded_body_style = fc.embedding_column_v2(
         body_style, dimension=5, initializer=_initializer)
 
     input_fn = numpy_io.numpy_input_fn(
@@ -3504,7 +3511,7 @@ class FeatureLayerTest(test.TestCase):
         batch_size=2,
         shuffle=False)
     features = input_fn()
-    net = fc.FeatureLayer([price, one_hot_body_style, embedded_body_style])(
+    net = fc.DenseFeatures([price, one_hot_body_style, embedded_body_style])(
         features)
     self.assertEqual(1 + 3 + 5, net.shape[1])
     with _initialized_session() as sess:
@@ -3531,18 +3538,18 @@ class FeatureLayerTest(test.TestCase):
       del shape, dtype, partition_info
       return embedding_values
 
-    # price has 1 dimension in feature_layer
-    price = fc.numeric_column('price')
+    # price has 1 dimension in dense_features
+    price = fc.numeric_column_v2('price')
 
-    # one_hot_body_style has 3 dims in feature_layer.
-    body_style = fc.categorical_column_with_vocabulary_list(
+    # one_hot_body_style has 3 dims in dense_features.
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column(body_style)
+    one_hot_body_style = fc.indicator_column_v2(body_style)
 
-    # embedded_body_style has 5 dims in feature_layer.
-    country = fc.categorical_column_with_vocabulary_list(
+    # embedded_body_style has 5 dims in dense_features.
+    country = fc.categorical_column_with_vocabulary_list_v2(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column(
+    embedded_country = fc.embedding_column_v2(
         country, dimension=5, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -3559,7 +3566,7 @@ class FeatureLayerTest(test.TestCase):
     self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0])
     self.assertEqual(1, features['country'].shape.ndims)
 
-    net = fc.FeatureLayer([price, one_hot_body_style, embedded_country])(
+    net = fc.DenseFeatures([price, one_hot_body_style, embedded_country])(
         features)
     self.assertEqual(1 + 3 + 5, net.shape[1])
     with _initialized_session() as sess:
@@ -3581,18 +3588,18 @@ class FeatureLayerTest(test.TestCase):
       del shape, dtype, partition_info
       return embedding_values
 
-    # price has 1 dimension in feature_layer
-    price = fc.numeric_column('price')
+    # price has 1 dimension in dense_features
+    price = fc.numeric_column_v2('price')
 
-    # one_hot_body_style has 3 dims in feature_layer.
-    body_style = fc.categorical_column_with_vocabulary_list(
+    # one_hot_body_style has 3 dims in dense_features.
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column(body_style)
+    one_hot_body_style = fc.indicator_column_v2(body_style)
 
-    # embedded_body_style has 5 dims in feature_layer.
-    country = fc.categorical_column_with_vocabulary_list(
+    # embedded_body_style has 5 dims in dense_features.
+    country = fc.categorical_column_with_vocabulary_list_v2(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column(
+    embedded_country = fc.embedding_column_v2(
         country, dimension=2, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -3613,7 +3620,7 @@ class FeatureLayerTest(test.TestCase):
         dense_shape=(2,))
     country_data = np.array([['US'], ['CA']])
 
-    net = fc.FeatureLayer([price, one_hot_body_style, embedded_country])(
+    net = fc.DenseFeatures([price, one_hot_body_style, embedded_country])(
         features)
     self.assertEqual(1 + 3 + 2, net.shape[1])
     with _initialized_session() as sess:
@@ -3631,8 +3638,8 @@ class FeatureLayerTest(test.TestCase):
               }))
 
   def test_with_rank_0_feature(self):
-    # price has 1 dimension in feature_layer
-    price = fc.numeric_column('price')
+    # price has 1 dimension in dense_features
+    price = fc.numeric_column_v2('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -3640,13 +3647,13 @@ class FeatureLayerTest(test.TestCase):
 
     # Static rank 0 should fail
     with self.assertRaisesRegexp(ValueError, 'Feature .* cannot have rank 0'):
-      fc.FeatureLayer([price])(features)
+      fc.DenseFeatures([price])(features)
 
     # Dynamic rank 0 should fail
     features = {
         'price': array_ops.placeholder(dtypes.float32),
     }
-    net = fc.FeatureLayer([price])(features)
+    net = fc.DenseFeatures([price])(features)
     self.assertEqual(1, net.shape[1])
     with _initialized_session() as sess:
       with self.assertRaisesOpError('Feature .* cannot have rank 0'):
@@ -3658,7 +3665,7 @@ class InputLayerTest(test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def test_retrieving_input(self):
     features = {'a': [0.]}
-    input_layer = fc_old.InputLayer(fc.numeric_column('a'))
+    input_layer = fc_old.InputLayer(fc.numeric_column_v2('a'))
     inputs = self.evaluate(input_layer(features))
     self.assertAllClose([[0.]], inputs)
 
@@ -3670,7 +3677,7 @@ class InputLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity(
+      categorical_column = fc.categorical_column_with_identity_v2(
           key='a', num_buckets=3)
       embedding_dimension = 2
 
@@ -3684,7 +3691,7 @@ class InputLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column(
+      embedding_column = fc.embedding_column_v2(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -3715,7 +3722,7 @@ class InputLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity(
+      categorical_column = fc.categorical_column_with_identity_v2(
           key='a', num_buckets=3)
       embedding_dimension = 2
 
@@ -3729,7 +3736,7 @@ class InputLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column(
+      embedding_column = fc.embedding_column_v2(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -3766,26 +3773,27 @@ class FunctionalInputLayerTest(test.TestCase):
       fc_old.input_layer(
           features={'a': [[0]]},
           feature_columns=[
-              fc.categorical_column_with_hash_bucket('wire_cast', 4)
+              fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
           ])
 
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc_old.input_layer(
-          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
+          features={'a': [[0]]},
+          feature_columns={'a': fc.numeric_column_v2('a')})
 
   def test_bare_column(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
-      net = fc_old.input_layer(features, fc.numeric_column('a'))
+      net = fc_old.input_layer(features, fc.numeric_column_v2('a'))
       with _initialized_session():
         self.assertAllClose([[0.]], net.eval())
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
-      columns = (fc.numeric_column(key) for key in features)
+      columns = (fc.numeric_column_v2(key) for key in features)
       net = fc_old.input_layer(features, columns)
       with _initialized_session():
         self.assertAllClose([[0., 1.]], net.eval())
@@ -3795,11 +3803,13 @@ class FunctionalInputLayerTest(test.TestCase):
         ValueError, 'Duplicate feature column name found for columns'):
       fc_old.input_layer(
           features={'a': [[0]]},
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])
+          feature_columns=[
+              fc.numeric_column_v2('a'),
+              fc.numeric_column_v2('a')
+          ])
 
   def test_one_column(self):
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       net = fc_old.input_layer(features, [price])
@@ -3807,7 +3817,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1.], [5.]], net.eval())
 
   def test_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc_old.input_layer(features, [price])
@@ -3815,7 +3825,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc.numeric_column_v2('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -3824,7 +3834,7 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [price])
 
   def test_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc.numeric_column_v2('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc_old.input_layer(features, [price])
@@ -3832,8 +3842,8 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
 
   def test_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1', shape=2)
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       net = fc_old.input_layer(features, [price1, price2])
@@ -3844,13 +3854,13 @@ class FunctionalInputLayerTest(test.TestCase):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
     # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
     # creates a Variable.
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc.numeric_column_v2('price1')
+    dense_feature = fc.numeric_column_v2('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column_v2(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -3874,13 +3884,13 @@ class FunctionalInputLayerTest(test.TestCase):
     # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The
     # EmbeddingColumn creates a Variable and the two SharedEmbeddingColumns
     # shared one variable.
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc.numeric_column_v2('price1')
+    dense_feature = fc.numeric_column_v2('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column_v2(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)
     categorical_column_a = fc_old.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -3926,13 +3936,13 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[shared_embedding_a][0].shape, [3, 2])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc.numeric_column_v2('price1')
+    dense_feature = fc.numeric_column_v2('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column_v2(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -3959,8 +3969,8 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10])
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
+    price_a = fc.numeric_column_v2('price_a')
+    price_b = fc.numeric_column_v2('price_b')
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
@@ -3973,7 +3983,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 3.]], net2.eval())
 
   def test_fails_for_categorical_column(self):
-    animal = fc.categorical_column_with_identity('animal', num_buckets=4)
+    animal = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -3984,8 +3994,8 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [animal])
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -3997,9 +4007,9 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
+    price3 = fc.numeric_column_v2('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -4012,8 +4022,8 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -4026,8 +4036,8 @@ class FunctionalInputLayerTest(test.TestCase):
           sess.run(net, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc.numeric_column_v2('price1')
+    price2 = fc.numeric_column_v2('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -4043,9 +4053,9 @@ class FunctionalInputLayerTest(test.TestCase):
             })
 
   def test_multiple_layers_with_same_embedding_column(self):
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc.embedding_column_v2(
         some_sparse_column, dimension=10)
 
     with ops.Graph().as_default():
@@ -4078,17 +4088,17 @@ class FunctionalInputLayerTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in input_layer
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
 
     # one_hot_body_style has 3 dims in input_layer.
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column(body_style)
+    one_hot_body_style = fc.indicator_column_v2(body_style)
 
     # embedded_body_style has 5 dims in input_layer.
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc.categorical_column_with_vocabulary_list_v2(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column(
+    embedded_country = fc.embedding_column_v2(
         country, dimension=5, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -4134,17 +4144,17 @@ class FunctionalInputLayerTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in input_layer
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
 
     # one_hot_body_style has 3 dims in input_layer.
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc.categorical_column_with_vocabulary_list_v2(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column(body_style)
+    one_hot_body_style = fc.indicator_column_v2(body_style)
 
     # embedded_body_style has 5 dims in input_layer.
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc.categorical_column_with_vocabulary_list_v2(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column(
+    embedded_country = fc.embedding_column_v2(
         country, dimension=2, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -4182,7 +4192,7 @@ class FunctionalInputLayerTest(test.TestCase):
 
   def test_with_rank_0_feature(self):
     # price has 1 dimension in input_layer
-    price = fc.numeric_column('price')
+    price = fc.numeric_column_v2('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -4225,7 +4235,7 @@ class MakeParseExampleSpecTest(test.TestCase):
       return self.parse_spec
 
   def test_no_feature_columns(self):
-    actual = fc.make_parse_example_spec([])
+    actual = fc.make_parse_example_spec_v2([])
     self.assertDictEqual({}, actual)
 
   def test_invalid_type(self):
@@ -4235,15 +4245,17 @@ class MakeParseExampleSpecTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError,
         'All feature_columns must be FeatureColumn instances.*invalid_column'):
-      fc.make_parse_example_spec(
-          (self._TestFeatureColumn({key1: parse_spec1}), 'invalid_column'))
+      fc.make_parse_example_spec_v2((self._TestFeatureColumn({
+          key1: parse_spec1
+      }), 'invalid_column'))
 
   def test_one_feature_column(self):
     key1 = 'key1'
     parse_spec1 = parsing_ops.FixedLenFeature(
         shape=(2,), dtype=dtypes.float32, default_value=0.)
-    actual = fc.make_parse_example_spec(
-        (self._TestFeatureColumn({key1: parse_spec1}),))
+    actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({
+        key1: parse_spec1
+    }),))
     self.assertDictEqual({key1: parse_spec1}, actual)
 
   def test_two_feature_columns(self):
@@ -4252,9 +4264,11 @@ class MakeParseExampleSpecTest(test.TestCase):
         shape=(2,), dtype=dtypes.float32, default_value=0.)
     key2 = 'key2'
     parse_spec2 = parsing_ops.VarLenFeature(dtype=dtypes.string)
-    actual = fc.make_parse_example_spec(
-        (self._TestFeatureColumn({key1: parse_spec1}),
-         self._TestFeatureColumn({key2: parse_spec2})))
+    actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({
+        key1: parse_spec1
+    }), self._TestFeatureColumn({
+        key2: parse_spec2
+    })))
     self.assertDictEqual({key1: parse_spec1, key2: parse_spec2}, actual)
 
   def test_equal_keys_different_parse_spec(self):
@@ -4265,17 +4279,21 @@ class MakeParseExampleSpecTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError,
         'feature_columns contain different parse_spec for key key1'):
-      fc.make_parse_example_spec(
-          (self._TestFeatureColumn({key1: parse_spec1}),
-           self._TestFeatureColumn({key1: parse_spec2})))
+      fc.make_parse_example_spec_v2((self._TestFeatureColumn({
+          key1: parse_spec1
+      }), self._TestFeatureColumn({
+          key1: parse_spec2
+      })))
 
   def test_equal_keys_equal_parse_spec(self):
     key1 = 'key1'
     parse_spec1 = parsing_ops.FixedLenFeature(
         shape=(2,), dtype=dtypes.float32, default_value=0.)
-    actual = fc.make_parse_example_spec(
-        (self._TestFeatureColumn({key1: parse_spec1}),
-         self._TestFeatureColumn({key1: parse_spec1})))
+    actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({
+        key1: parse_spec1
+    }), self._TestFeatureColumn({
+        key1: parse_spec1
+    })))
     self.assertDictEqual({key1: parse_spec1}, actual)
 
   def test_multiple_features_dict(self):
@@ -4287,9 +4305,12 @@ class MakeParseExampleSpecTest(test.TestCase):
     parse_spec2 = parsing_ops.VarLenFeature(dtype=dtypes.string)
     key3 = 'key3'
     parse_spec3 = parsing_ops.VarLenFeature(dtype=dtypes.int32)
-    actual = fc.make_parse_example_spec(
-        (self._TestFeatureColumn({key1: parse_spec1}),
-         self._TestFeatureColumn({key2: parse_spec2, key3: parse_spec3})))
+    actual = fc.make_parse_example_spec_v2((self._TestFeatureColumn({
+        key1: parse_spec1
+    }), self._TestFeatureColumn({
+        key2: parse_spec2,
+        key3: parse_spec3
+    })))
     self.assertDictEqual(
         {key1: parse_spec1, key2: parse_spec2, key3: parse_spec3}, actual)
 
@@ -4322,7 +4343,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     self._wire_vocabulary_size = 3
 
   def test_defaults(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -4334,22 +4355,28 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_vocabulary_file(
+      fc.categorical_column_with_vocabulary_file_v2(
           key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3)
 
   def test_all_constructor_args(self):
-    column = fc.categorical_column_with_vocabulary_file(
-        key='aaa', vocabulary_file='path_to_file', vocabulary_size=3,
-        num_oov_buckets=4, dtype=dtypes.int32)
+    column = fc.categorical_column_with_vocabulary_file_v2(
+        key='aaa',
+        vocabulary_file='path_to_file',
+        vocabulary_size=3,
+        num_oov_buckets=4,
+        dtype=dtypes.int32)
     self.assertEqual(7, column.num_buckets)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, column.parse_example_spec)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_vocabulary_file(
-        key='aaa', vocabulary_file='path_to_file', vocabulary_size=3,
-        num_oov_buckets=4, dtype=dtypes.int32)
+    original = fc.categorical_column_with_vocabulary_file_v2(
+        key='aaa',
+        vocabulary_file='path_to_file',
+        vocabulary_size=3,
+        num_oov_buckets=4,
+        dtype=dtypes.int32)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(7, column.num_buckets)
@@ -4359,16 +4386,16 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_vocabulary_file_none(self):
     with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'):
-      fc.categorical_column_with_vocabulary_file(
+      fc.categorical_column_with_vocabulary_file_v2(
           key='aaa', vocabulary_file=None, vocabulary_size=3)
 
   def test_vocabulary_file_empty_string(self):
     with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'):
-      fc.categorical_column_with_vocabulary_file(
+      fc.categorical_column_with_vocabulary_file_v2(
           key='aaa', vocabulary_file='', vocabulary_size=3)
 
   def test_invalid_vocabulary_file(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4384,16 +4411,18 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_vocabulary_size(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file=self._wire_vocabulary_file_name,
+      fc.categorical_column_with_vocabulary_file_v2(
+          key='aaa',
+          vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=-1)
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file=self._wire_vocabulary_file_name,
+      fc.categorical_column_with_vocabulary_file_v2(
+          key='aaa',
+          vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=0)
 
   def test_too_large_vocabulary_size(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size + 1)
@@ -4411,20 +4440,24 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file='path', vocabulary_size=3,
+      fc.categorical_column_with_vocabulary_file_v2(
+          key='aaa',
+          vocabulary_file='path',
+          vocabulary_size=3,
           num_oov_buckets=-1)
 
   def test_invalid_dtype(self):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file='path', vocabulary_size=3,
+      fc.categorical_column_with_vocabulary_file_v2(
+          key='aaa',
+          vocabulary_file='path',
+          vocabulary_size=3,
           dtype=dtypes.float64)
 
   def test_invalid_buckets_and_default_value(self):
     with self.assertRaisesRegexp(
         ValueError, 'both num_oov_buckets and default_value'):
-      fc.categorical_column_with_vocabulary_file(
+      fc.categorical_column_with_vocabulary_file_v2(
           key='aaa',
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=self._wire_vocabulary_size,
@@ -4432,7 +4465,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           default_value=2)
 
   def test_invalid_input_dtype_int32(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4448,7 +4481,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_invalid_input_dtype_string(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4464,7 +4497,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_file(
+    a = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -4474,7 +4507,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a]))
+        features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -4486,7 +4519,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -4509,7 +4542,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_none_vocabulary_size(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa', vocabulary_file=self._wire_vocabulary_file_name)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4530,7 +4563,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
                                   id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -4549,7 +4582,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
                                   id_tensor.eval())
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -4568,7 +4601,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4592,7 +4625,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4619,7 +4652,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     # 'marlo' is the last entry in our vocabulary file, so be setting
     # `vocabulary_size` to 1 less than number of entries in file, we take
     # 'marlo' out of the vocabulary.
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size - 1)
@@ -4642,7 +4675,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4667,7 +4700,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4688,7 +4721,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc.categorical_column_with_vocabulary_file_v2(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4713,7 +4746,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_file(
+    wire_column = fc.categorical_column_with_vocabulary_file_v2(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4739,7 +4772,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), predictions.eval())
 
   def test_old_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_file(
+    wire_column = fc.categorical_column_with_vocabulary_file_v2(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4765,7 +4798,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), predictions.eval())
 
   def test_serialization(self):
-    wire_column = fc.categorical_column_with_vocabulary_file(
+    wire_column = fc.categorical_column_with_vocabulary_file_v2(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4790,7 +4823,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_defaults_string(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -4802,11 +4835,11 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key=('aaa',), vocabulary_list=('omar', 'stringer', 'marlo'))
 
   def test_defaults_int(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=(12, 24, 36))
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -4816,8 +4849,10 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     }, column.parse_example_spec)
 
   def test_all_constructor_args(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32,
+    column = fc.categorical_column_with_vocabulary_list_v2(
+        key='aaa',
+        vocabulary_list=(12, 24, 36),
+        dtype=dtypes.int32,
         default_value=-99)
     self.assertEqual(3, column.num_buckets)
     self.assertEqual({
@@ -4825,7 +4860,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     }, column.parse_example_spec)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_vocabulary_list(
+    original = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
@@ -4836,65 +4871,65 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype(self):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_list(
-          key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'),
+      fc.categorical_column_with_vocabulary_list_v2(
+          key='aaa',
+          vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.float32)
 
   def test_invalid_mapping_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key='aaa', vocabulary_list=(12., 24., 36.))
 
   def test_mismatched_int_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'dtype.*and vocabulary dtype.*do not match'):
-      fc.categorical_column_with_vocabulary_list(
-          key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'),
+      fc.categorical_column_with_vocabulary_list_v2(
+          key='aaa',
+          vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.int32)
 
   def test_mismatched_string_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'dtype.*and vocabulary dtype.*do not match'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.string)
 
   def test_none_mapping(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary_list.*must be non-empty'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key='aaa', vocabulary_list=None)
 
   def test_empty_mapping(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary_list.*must be non-empty'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key='aaa', vocabulary_list=tuple([]))
 
   def test_duplicate_mapping(self):
     with self.assertRaisesRegexp(ValueError, 'Duplicate keys'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key='aaa', vocabulary_list=(12, 24, 12))
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
-      fc.categorical_column_with_vocabulary_list(
-          key='aaa', vocabulary_list=(12, 24, 36),
-          num_oov_buckets=-1)
+      fc.categorical_column_with_vocabulary_list_v2(
+          key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=-1)
 
   def test_invalid_buckets_and_default_value(self):
     with self.assertRaisesRegexp(
         ValueError, 'both num_oov_buckets and default_value'):
-      fc.categorical_column_with_vocabulary_list(
+      fc.categorical_column_with_vocabulary_list_v2(
           key='aaa',
           vocabulary_list=(12, 24, 36),
           num_oov_buckets=100,
           default_value=2)
 
   def test_invalid_input_dtype_int32(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc.categorical_column_with_vocabulary_list_v2(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(12, 24, 36),
@@ -4906,9 +4941,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_invalid_input_dtype_string(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=(12, 24, 36))
+    column = fc.categorical_column_with_vocabulary_list_v2(
+        key='aaa', vocabulary_list=(12, 24, 36))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('omar', 'stringer', 'marlo'),
@@ -4920,7 +4954,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_parse_example_string(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -4930,7 +4964,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a]))
+        features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -4942,7 +4976,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_parse_example_int(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=(11, 21, 31))
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -4952,7 +4986,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a]))
+        features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -4964,9 +4998,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc.categorical_column_with_vocabulary_list_v2(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('marlo', 'skywalker', 'omar'),
@@ -4986,9 +5019,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc.categorical_column_with_vocabulary_list_v2(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('marlo', 'skywalker', 'omar'),
@@ -5004,9 +5036,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_tensor.eval())
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc.categorical_column_with_vocabulary_list_v2(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': (('marlo', ''), ('skywalker', 'omar'))
@@ -5022,7 +5053,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         default_value=2)
@@ -5045,7 +5076,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=100)
@@ -5068,7 +5099,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32)
@@ -5092,7 +5123,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32,
@@ -5114,7 +5145,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32,
@@ -5138,7 +5169,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_list(
+    wire_column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -5163,7 +5194,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), predictions.eval())
 
   def test_old_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_list(
+    wire_column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -5188,7 +5219,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), predictions.eval())
 
   def test_serialization(self):
-    wire_column = fc.categorical_column_with_vocabulary_list(
+    wire_column = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -5212,7 +5243,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_constructor(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
     self.assertEqual(3, column.num_buckets)
@@ -5223,10 +5254,10 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_identity(key=('aaa',), num_buckets=3)
+      fc.categorical_column_with_identity_v2(key=('aaa',), num_buckets=3)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    original = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(3, column.num_buckets)
@@ -5236,24 +5267,24 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_invalid_num_buckets_zero(self):
     with self.assertRaisesRegexp(ValueError, 'num_buckets 0 < 1'):
-      fc.categorical_column_with_identity(key='aaa', num_buckets=0)
+      fc.categorical_column_with_identity_v2(key='aaa', num_buckets=0)
 
   def test_invalid_num_buckets_negative(self):
     with self.assertRaisesRegexp(ValueError, 'num_buckets -1 < 1'):
-      fc.categorical_column_with_identity(key='aaa', num_buckets=-1)
+      fc.categorical_column_with_identity_v2(key='aaa', num_buckets=-1)
 
   def test_invalid_default_value_too_small(self):
     with self.assertRaisesRegexp(ValueError, 'default_value -1 not in range'):
-      fc.categorical_column_with_identity(
+      fc.categorical_column_with_identity_v2(
           key='aaa', num_buckets=3, default_value=-1)
 
   def test_invalid_default_value_too_big(self):
     with self.assertRaisesRegexp(ValueError, 'default_value 3 not in range'):
-      fc.categorical_column_with_identity(
+      fc.categorical_column_with_identity_v2(
           key='aaa', num_buckets=3, default_value=3)
 
   def test_invalid_input_dtype(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('omar', 'stringer', 'marlo'),
@@ -5265,7 +5296,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_identity(key='aaa', num_buckets=30)
+    a = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=30)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -5274,7 +5305,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a]))
+        features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -5286,7 +5317,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
@@ -5306,7 +5337,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
@@ -5322,7 +5353,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_tensor.eval())
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': ((0, -1), (1, 0))
@@ -5338,7 +5369,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_inputs_too_small(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(1, -1, 0),
@@ -5354,7 +5385,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         id_weight_pair.id_tensor.eval()
 
   def test_get_sparse_tensors_with_inputs_too_big(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(1, 99, 0),
@@ -5370,7 +5401,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         id_weight_pair.id_tensor.eval()
 
   def test_get_sparse_tensors_with_default_value(self):
-    column = fc.categorical_column_with_identity(
+    column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=4, default_value=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -5391,7 +5422,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self):
-    column = fc.categorical_column_with_identity(
+    column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=4, default_value=3)
     input_indices = array_ops.placeholder(dtype=dtypes.int64)
     input_values = array_ops.placeholder(dtype=dtypes.int32)
@@ -5419,7 +5450,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           }))
 
   def test_linear_model(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     self.assertEqual(3, column.num_buckets)
     with ops.Graph().as_default():
       model = fc.LinearModel((column,))
@@ -5441,7 +5472,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((1.,), (5.,)), predictions.eval())
 
   def test_old_linear_model(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
     self.assertEqual(3, column.num_buckets)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
@@ -5463,7 +5494,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((1.,), (5.,)), predictions.eval())
 
   def test_serialization(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
 
     self.assertEqual(['aaa'], column.parents)
 
@@ -5482,9 +5513,9 @@ class TransformFeaturesTest(test.TestCase):
   # All transform tests are distributed in column test.
   # Here we only test multi column case and naming
   def transform_multi_column(self):
-    bucketized_price = fc.bucketized_column(
-        fc.numeric_column('price'), boundaries=[0, 2, 4, 6])
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    bucketized_price = fc.bucketized_column_v2(
+        fc.numeric_column_v2('price'), boundaries=[0, 2, 4, 6])
+    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
     with ops.Graph().as_default():
       features = {
           'price': [[-1.], [5.]],
@@ -5544,23 +5575,23 @@ class TransformFeaturesTest(test.TestCase):
 class IndicatorColumnTest(test.TestCase):
 
   def test_indicator_column(self):
-    a = fc.categorical_column_with_hash_bucket('a', 4)
-    indicator_a = fc.indicator_column(a)
+    a = fc.categorical_column_with_hash_bucket_v2('a', 4)
+    indicator_a = fc.indicator_column_v2(a)
     self.assertEqual(indicator_a.categorical_column.name, 'a')
     self.assertEqual(indicator_a.name, 'a_indicator')
     self.assertEqual(indicator_a.variable_shape, [1, 4])
     self.assertTrue(indicator_a._is_v2_column)
 
     b = fc_old.categorical_column_with_hash_bucket('b', hash_bucket_size=100)
-    indicator_b = fc.indicator_column(b)
+    indicator_b = fc.indicator_column_v2(b)
     self.assertEqual(indicator_b.categorical_column.name, 'b')
     self.assertEqual(indicator_b.name, 'b_indicator')
     self.assertEqual(indicator_b.variable_shape, [1, 100])
     self.assertFalse(indicator_b._is_v2_column)
 
   def test_1D_shape_succeeds(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_hash_bucket('animal', 4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_hash_bucket_v2('animal', 4))
     transformation_cache = fc.FeatureTransformationCache({
         'animal': ['fox', 'fox']
     })
@@ -5570,8 +5601,8 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_2D_shape_succeeds(self):
     # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready.
-    animal = fc.indicator_column(
-        fc.categorical_column_with_hash_bucket('animal', 4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_hash_bucket_v2('animal', 4))
     transformation_cache = fc.FeatureTransformationCache({
         'animal':
             sparse_tensor.SparseTensor(
@@ -5584,8 +5615,8 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval())
 
   def test_multi_hot(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
 
     transformation_cache = fc.FeatureTransformationCache({
         'animal':
@@ -5597,8 +5628,8 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 2., 0., 0.]], output.eval())
 
   def test_multi_hot2(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
     transformation_cache = fc.FeatureTransformationCache({
         'animal':
             sparse_tensor.SparseTensor(
@@ -5609,17 +5640,17 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 1., 1., 0.]], output.eval())
 
   def test_deep_copy(self):
-    a = fc.categorical_column_with_hash_bucket('a', 4)
-    column = fc.indicator_column(a)
+    a = fc.categorical_column_with_hash_bucket_v2('a', 4)
+    column = fc.indicator_column_v2(a)
     column_copy = copy.deepcopy(column)
     self.assertEqual(column_copy.categorical_column.name, 'a')
     self.assertEqual(column.name, 'a_indicator')
     self.assertEqual(column.variable_shape, [1, 4])
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_indicator = fc.indicator_column(a)
+    a_indicator = fc.indicator_column_v2(a)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -5628,7 +5659,7 @@ class IndicatorColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a_indicator]))
+        features=fc.make_parse_example_spec_v2([a_indicator]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -5640,9 +5671,9 @@ class IndicatorColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_transform(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_indicator = fc.indicator_column(a)
+    a_indicator = fc.indicator_column_v2(a)
     features = {
         'aaa': sparse_tensor.SparseTensorValue(
             indices=((0, 0), (1, 0), (1, 1)),
@@ -5656,10 +5687,10 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_transform_with_weighted_column(self):
     # Github issue 12557
-    ids = fc.categorical_column_with_vocabulary_list(
+    ids = fc.categorical_column_with_vocabulary_list_v2(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    weights = fc.weighted_categorical_column(ids, 'weights')
-    indicator = fc.indicator_column(weights)
+    weights = fc.weighted_categorical_column_v2(ids, 'weights')
+    indicator = fc.indicator_column_v2(weights)
     features = {
         'ids': constant_op.constant([['c', 'b', 'a']]),
         'weights': constant_op.constant([[2., 4., 6.]])
@@ -5671,10 +5702,10 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
-    ids = fc.categorical_column_with_vocabulary_list(
+    ids = fc.categorical_column_with_vocabulary_list_v2(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    weights = fc.weighted_categorical_column(ids, 'weights')
-    indicator = fc.indicator_column(weights)
+    weights = fc.weighted_categorical_column_v2(ids, 'weights')
+    indicator = fc.indicator_column_v2(weights)
     features = {
         'ids': constant_op.constant([['c', 'b', 'unknown']]),
         'weights': constant_op.constant([[2., 4., 6.]])
@@ -5686,9 +5717,9 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
-    ids = fc.categorical_column_with_vocabulary_list(
+    ids = fc.categorical_column_with_vocabulary_list_v2(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    indicator = fc.indicator_column(ids)
+    indicator = fc.indicator_column_v2(ids)
     features = {
         'ids': constant_op.constant([['c', 'b', 'unknown']]),
     }
@@ -5698,8 +5729,8 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 1., 1.]], indicator_tensor.eval())
 
   def test_linear_model(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5718,8 +5749,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], predictions.eval())
 
   def test_old_linear_model(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5737,7 +5768,7 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], predictions.eval())
 
   def test_old_linear_model_old_categorical(self):
-    animal = fc.indicator_column(
+    animal = fc.indicator_column_v2(
         fc_old.categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
@@ -5755,22 +5786,22 @@ class IndicatorColumnTest(test.TestCase):
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
         self.assertAllClose([[2. + 3.]], predictions.eval())
 
-  def test_feature_layer(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+  def test_dense_features(self):
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
               sparse_tensor.SparseTensor(
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
-      net = fc.FeatureLayer([animal])(features)
+      net = fc.DenseFeatures([animal])(features)
       with _initialized_session():
         self.assertAllClose([[0., 1., 1., 0.]], net.eval())
 
   def test_input_layer(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column_v2(
+        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5782,7 +5813,7 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[0., 1., 1., 0.]], net.eval())
 
   def test_input_layer_old_categorical(self):
-    animal = fc.indicator_column(
+    animal = fc.indicator_column_v2(
         fc_old.categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
@@ -5795,8 +5826,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[0., 1., 1., 0.]], net.eval())
 
   def test_serialization(self):
-    parent = fc.categorical_column_with_identity('animal', num_buckets=4)
-    animal = fc.indicator_column(parent)
+    parent = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
+    animal = fc.indicator_column_v2(parent)
 
     self.assertEqual([parent], animal.parents)
 
@@ -5865,10 +5896,10 @@ class _TestStateManager(fc.StateManager):
 class EmbeddingColumnTest(test.TestCase):
 
   def test_defaults(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column, dimension=embedding_dimension)
     self.assertIs(categorical_column, embedding_column.categorical_column)
     self.assertEqual(embedding_dimension, embedding_column.dimension)
@@ -5888,19 +5919,23 @@ class EmbeddingColumnTest(test.TestCase):
     categorical_column = fc_old.categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column, dimension=embedding_dimension)
     self.assertFalse(embedding_column._is_v2_column)
 
   def test_all_constructor_args(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
-        combiner='my_combiner', initializer=lambda: 'my_initializer',
-        ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor',
-        max_norm=42., trainable=False)
+    embedding_column = fc.embedding_column_v2(
+        categorical_column,
+        dimension=embedding_dimension,
+        combiner='my_combiner',
+        initializer=lambda: 'my_initializer',
+        ckpt_to_load_from='my_ckpt',
+        tensor_name_in_ckpt='my_ckpt_tensor',
+        max_norm=42.,
+        trainable=False)
     self.assertIs(categorical_column, embedding_column.categorical_column)
     self.assertEqual(embedding_dimension, embedding_column.dimension)
     self.assertEqual('my_combiner', embedding_column.combiner)
@@ -5915,14 +5950,18 @@ class EmbeddingColumnTest(test.TestCase):
     }, embedding_column.parse_example_spec)
 
   def test_deep_copy(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    original = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
-        combiner='my_combiner', initializer=lambda: 'my_initializer',
-        ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor',
-        max_norm=42., trainable=False)
+    original = fc.embedding_column_v2(
+        categorical_column,
+        dimension=embedding_dimension,
+        combiner='my_combiner',
+        initializer=lambda: 'my_initializer',
+        ckpt_to_load_from='my_ckpt',
+        tensor_name_in_ckpt='my_ckpt_tensor',
+        max_norm=42.,
+        trainable=False)
     for embedding_column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', embedding_column.categorical_column.name)
       self.assertEqual(3, embedding_column.categorical_column.num_buckets)
@@ -5943,15 +5982,16 @@ class EmbeddingColumnTest(test.TestCase):
       }, embedding_column.parse_example_spec)
 
   def test_invalid_initializer(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
-      fc.embedding_column(categorical_column, dimension=2, initializer='not_fn')
+      fc.embedding_column_v2(
+          categorical_column, dimension=2, initializer='not_fn')
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_embedded = fc.embedding_column(a, dimension=2)
+    a_embedded = fc.embedding_column_v2(a, dimension=2)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -5960,7 +6000,7 @@ class EmbeddingColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a_embedded]))
+        features=fc.make_parse_example_spec_v2([a_embedded]))
     self.assertIn('aaa', features)
     with self.cached_session():
       _assert_sparse_tensor_value(
@@ -5972,8 +6012,8 @@ class EmbeddingColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_transform_feature(self):
-    a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
-    a_embedded = fc.embedding_column(a, dimension=2)
+    a = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    a_embedded = fc.embedding_column_v2(a, dimension=2)
     features = {
         'aaa': sparse_tensor.SparseTensor(
             indices=((0, 0), (1, 0), (1, 1)),
@@ -6025,10 +6065,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc.embedding_column_v2(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
     state_manager = _TestStateManager()
     embedding_column.create_state(state_manager)
@@ -6088,7 +6129,7 @@ class EmbeddingColumnTest(test.TestCase):
     # Build columns.
     categorical_column = fc_old.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6147,10 +6188,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc.embedding_column_v2(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
     state_manager = _TestStateManager()
     embedding_column.create_state(state_manager)
@@ -6207,10 +6249,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc.embedding_column_v2(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
     state_manager = _TestStateManager()
     embedding_column.create_state(state_manager)
@@ -6277,10 +6320,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc.embedding_column_v2(
+        categorical_column,
+        dimension=embedding_dimension,
         ckpt_to_load_from=ckpt_path,
         tensor_name_in_ckpt=ckpt_tensor)
     state_manager = _TestStateManager()
@@ -6324,9 +6368,9 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6350,8 +6394,7 @@ class EmbeddingColumnTest(test.TestCase):
       bias = trainable_vars['linear_model/bias_weights:0']
       embedding_weights = trainable_vars[
           'linear_model/aaa_embedding/embedding_weights:0']
-      linear_weights = trainable_vars[
-          'linear_model/aaa_embedding/weights:0']
+      linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
         self.assertAllClose(np.zeros((1,)), bias.eval())
@@ -6375,7 +6418,7 @@ class EmbeddingColumnTest(test.TestCase):
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
         self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
 
-  def test_feature_layer(self):
+  def test_dense_features(self):
     # Inputs.
     vocabulary_size = 3
     sparse_input = sparse_tensor.SparseTensorValue(
@@ -6413,31 +6456,31 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
-    l = fc.FeatureLayer((embedding_column,))
-    feature_layer = l({'aaa': sparse_input})
+    l = fc.DenseFeatures((embedding_column,))
+    dense_features = l({'aaa': sparse_input})
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(('feature_layer/aaa_embedding/embedding_weights:0',),
+    self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
     for v in global_vars:
       self.assertTrue(isinstance(v, variables_lib.RefVariable))
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
-    self.assertItemsEqual(('feature_layer/aaa_embedding/embedding_weights:0',),
+    self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in trainable_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, feature_layer.eval())
+      self.assertAllEqual(expected_lookups, dense_features.eval())
 
-  def test_feature_layer_not_trainable(self):
+  def test_dense_features_not_trainable(self):
     # Inputs.
     vocabulary_size = 3
     sparse_input = sparse_tensor.SparseTensorValue(
@@ -6475,26 +6518,28 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer,
         trainable=False)
 
     # Provide sparse input and get dense result.
-    feature_layer = fc.FeatureLayer((embedding_column,))({'aaa': sparse_input})
+    dense_features = fc.DenseFeatures((embedding_column,))({
+        'aaa': sparse_input
+    })
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(('feature_layer/aaa_embedding/embedding_weights:0',),
+    self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
     self.assertItemsEqual(
         [], ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, feature_layer.eval())
+      self.assertAllEqual(expected_lookups, dense_features.eval())
 
   def test_input_layer(self):
     # Inputs.
@@ -6535,15 +6580,15 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
-    feature_layer = fc_old.input_layer({
+    dense_features = fc_old.input_layer({
         'aaa': sparse_input
     }, (embedding_column,))
 
@@ -6556,7 +6601,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in trainable_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, feature_layer.eval())
+      self.assertAllEqual(expected_lookups, dense_features.eval())
 
   def test_old_linear_model(self):
     # Inputs.
@@ -6583,9 +6628,9 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6661,7 +6706,7 @@ class EmbeddingColumnTest(test.TestCase):
     # Build columns.
     categorical_column = fc_old.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6717,9 +6762,9 @@ class EmbeddingColumnTest(test.TestCase):
       return ValueError('Not expected to be called')
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc.embedding_column_v2(
         categorical_column, dimension=2, initializer=_initializer)
 
     self.assertEqual([categorical_column], embedding_column.parents)
@@ -6764,9 +6809,9 @@ class EmbeddingColumnTest(test.TestCase):
 class SharedEmbeddingColumnTest(test.TestCase):
 
   def test_defaults(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_b, embedding_column_a = fc.shared_embedding_columns_v2(
@@ -6788,9 +6833,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }, embedding_column_b.parse_example_spec)
 
   def test_all_constructor_args(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
@@ -6819,9 +6864,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }, embedding_column_b.parse_example_spec)
 
   def test_deep_copy(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     original_a, _ = fc.shared_embedding_columns_v2(
@@ -6850,9 +6895,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       }, embedding_column_a.parse_example_spec)
 
   def test_invalid_initializer(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
       fc.shared_embedding_columns_v2(
@@ -6861,11 +6906,11 @@ class SharedEmbeddingColumnTest(test.TestCase):
           initializer='not_fn')
 
   def test_incompatible_column_type(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
-    categorical_column_c = fc.categorical_column_with_hash_bucket(
+    categorical_column_c = fc.categorical_column_with_hash_bucket_v2(
         key='ccc', hash_bucket_size=3)
     with self.assertRaisesRegexp(
         ValueError, 'all categorical_columns must have the same type.*'
@@ -6875,13 +6920,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
           dimension=2)
 
   def test_weighted_categorical_column_ok(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    weighted_categorical_column_a = fc.weighted_categorical_column(
+    weighted_categorical_column_a = fc.weighted_categorical_column_v2(
         categorical_column_a, weight_feature_key='aaa_weights')
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
-    weighted_categorical_column_b = fc.weighted_categorical_column(
+    weighted_categorical_column_b = fc.weighted_categorical_column_v2(
         categorical_column_b, weight_feature_key='bbb_weights')
     fc.shared_embedding_columns_v2(
         [weighted_categorical_column_a, categorical_column_b], dimension=2)
@@ -6892,9 +6937,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
         dimension=2)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    b = fc.categorical_column_with_vocabulary_list(
+    b = fc.categorical_column_with_vocabulary_list_v2(
         key='bbb', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2)
     data = example_pb2.Example(features=feature_pb2.Features(
@@ -6908,7 +6953,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a_embedded, b_embedded]))
+        features=fc.make_parse_example_spec_v2([a_embedded, b_embedded]))
     self.assertIn('aaa', features)
     self.assertIn('bbb', features)
     with self.cached_session():
@@ -6928,8 +6973,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
           features['bbb'].eval())
 
   def test_transform_feature(self):
-    a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
-    b = fc.categorical_column_with_identity(key='bbb', num_buckets=3)
+    a = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    b = fc.categorical_column_with_identity_v2(key='bbb', num_buckets=3)
     a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2)
     features = {
         'aaa': sparse_tensor.SparseTensor(
@@ -6996,9 +7041,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7059,9 +7104,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7100,9 +7145,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7166,7 +7211,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
         self.assertAllClose([[94. + 13.], [29.]], predictions.eval())
 
-  def _test_feature_layer(self, trainable=True):
+  def _test_dense_features(self, trainable=True):
     # Inputs.
     vocabulary_size = 3
     sparse_input_a = sparse_tensor.SparseTensorValue(
@@ -7224,13 +7269,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=vocabulary_size)
-    categorical_column_c = fc.categorical_column_with_identity(
+    categorical_column_c = fc.categorical_column_with_identity_v2(
         key='ccc', num_buckets=vocabulary_size)
-    categorical_column_d = fc.categorical_column_with_identity(
+    categorical_column_d = fc.categorical_column_with_identity_v2(
         key='ddd', num_buckets=vocabulary_size)
 
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
@@ -7252,7 +7297,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }
 
     # Provide sparse input and get dense result.
-    feature_layer = fc.FeatureLayer(
+    dense_features = fc.DenseFeatures(
         feature_columns=(embedding_column_b, embedding_column_a,
                          embedding_column_c, embedding_column_d))(
                              features)
@@ -7274,13 +7319,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
     shared_embedding_vars = global_vars
     with _initialized_session():
       self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval())
-      self.assertAllEqual(expected_lookups, feature_layer.eval())
+      self.assertAllEqual(expected_lookups, dense_features.eval())
 
-  def test_feature_layer(self):
-    self._test_feature_layer()
+  def test_dense_features(self):
+    self._test_dense_features()
 
-  def test_feature_layer_no_trainable(self):
-    self._test_feature_layer(trainable=False)
+  def test_dense_features_no_trainable(self):
+    self._test_dense_features(trainable=False)
 
   def test_serialization(self):
 
@@ -7288,9 +7333,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       del shape, dtype, partition_info
       return ValueError('Not expected to be called')
 
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity_v2(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity_v2(
         key='bbb', num_buckets=3)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7306,8 +7351,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
 class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     self.assertEqual('ids_weighted_by_values', column.name)
@@ -7319,7 +7364,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
     self.assertTrue(column._is_v2_column)
 
   def test_is_v2_column(self):
-    column = fc.weighted_categorical_column(
+    column = fc.weighted_categorical_column_v2(
         categorical_column=fc_old.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
@@ -7327,8 +7372,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_deep_copy(self):
     """Tests deepcopy of categorical_column_with_hash_bucket."""
-    original = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    original = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     for column in (original, copy.deepcopy(original)):
@@ -7341,23 +7386,23 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype_none(self):
     with self.assertRaisesRegexp(ValueError, 'is not convertible to float'):
-      fc.weighted_categorical_column(
-          categorical_column=fc.categorical_column_with_identity(
+      fc.weighted_categorical_column_v2(
+          categorical_column=fc.categorical_column_with_identity_v2(
               key='ids', num_buckets=3),
           weight_feature_key='values',
           dtype=None)
 
   def test_invalid_dtype_string(self):
     with self.assertRaisesRegexp(ValueError, 'is not convertible to float'):
-      fc.weighted_categorical_column(
-          categorical_column=fc.categorical_column_with_identity(
+      fc.weighted_categorical_column_v2(
+          categorical_column=fc.categorical_column_with_identity_v2(
               key='ids', num_buckets=3),
           weight_feature_key='values',
           dtype=dtypes.string)
 
   def test_invalid_input_dtype(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     strings = sparse_tensor.SparseTensorValue(
@@ -7372,14 +7417,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_column_name_collision(self):
     with self.assertRaisesRegexp(ValueError, r'Parse config.*already exists'):
-      fc.weighted_categorical_column(
-          categorical_column=fc.categorical_column_with_identity(
+      fc.weighted_categorical_column_v2(
+          categorical_column=fc.categorical_column_with_identity_v2(
               key='aaa', num_buckets=3),
           weight_feature_key='aaa').parse_example_spec()
 
   def test_missing_weights(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -7391,9 +7436,10 @@ class WeightedCategoricalColumnTest(test.TestCase):
       fc._transform_features({'ids': inputs}, (column,), None)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc.categorical_column_with_vocabulary_list_v2(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_weighted = fc.weighted_categorical_column(a, weight_feature_key='weights')
+    a_weighted = fc.weighted_categorical_column_v2(
+        a, weight_feature_key='weights')
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -7405,7 +7451,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
-        features=fc.make_parse_example_spec([a_weighted]))
+        features=fc.make_parse_example_spec_v2([a_weighted]))
     self.assertIn('aaa', features)
     self.assertIn('weights', features)
     with self.cached_session():
@@ -7425,8 +7471,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           features['weights'].eval())
 
   def test_transform_features(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -7458,8 +7504,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           weight_tensor.eval())
 
   def test_transform_features_dense_input(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     weights = sparse_tensor.SparseTensorValue(
@@ -7487,8 +7533,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           weight_tensor.eval())
 
   def test_transform_features_dense_weights(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -7516,8 +7562,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           weight_tensor.eval())
 
   def test_linear_model(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7546,8 +7592,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), predictions.eval())
 
   def test_linear_model_mismatched_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7568,8 +7614,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         })
 
   def test_linear_model_mismatched_dense_values(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7592,8 +7638,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           predictions.eval()
 
   def test_linear_model_mismatched_dense_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7618,8 +7664,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), predictions.eval())
 
   def test_old_linear_model(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7648,8 +7694,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), predictions.eval())
 
   def test_old_linear_model_mismatched_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7669,8 +7715,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         }, (column,))
 
   def test_old_linear_model_mismatched_dense_values(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7693,8 +7739,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           predictions.eval()
 
   def test_old_linear_model_mismatched_dense_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc.weighted_categorical_column_v2(
+        categorical_column=fc.categorical_column_with_identity_v2(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7719,7 +7765,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), predictions.eval())
 
   def test_old_linear_model_old_categorical(self):
-    column = fc.weighted_categorical_column(
+    column = fc.weighted_categorical_column_v2(
         categorical_column=fc_old.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
@@ -7751,9 +7797,9 @@ class WeightedCategoricalColumnTest(test.TestCase):
   # TODO(ptucker): Add test with embedding of weighted categorical.
 
   def test_serialization(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc.categorical_column_with_identity_v2(
         key='ids', num_buckets=3)
-    column = fc.weighted_categorical_column(
+    column = fc.weighted_categorical_column_v2(
         categorical_column=categorical_column, weight_feature_key='weight')
 
     self.assertEqual([categorical_column, 'weight'], column.parents)
@@ -7837,8 +7883,8 @@ class SerializationTest(test.TestCase):
       })
 
   def test_deserialization_deduping(self):
-    price = fc.numeric_column('price')
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 1])
+    price = fc.numeric_column_v2('price')
+    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 1])
 
     configs = fc.serialize_feature_columns([price, bucketized_price])
 
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index e0478ee357..9ee6a0a27a 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -33,7 +33,7 @@ class TestDNNModel(keras.models.Model):
 
   def __init__(self, feature_columns, units, name=None, **kwargs):
     super(TestDNNModel, self).__init__(name=name, **kwargs)
-    self._input_layer = fc.FeatureLayer(feature_columns, name='input_layer')
+    self._input_layer = fc.DenseFeatures(feature_columns, name='input_layer')
     self._dense_layer = keras.layers.Dense(units, name='dense_layer')
 
   def call(self, features):
@@ -49,9 +49,9 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_sequential_model(self):
-    columns = [fc.numeric_column('a')]
+    columns = [fc.numeric_column_v2('a')]
     model = keras.models.Sequential([
-        fc.FeatureLayer(columns),
+        fc.DenseFeatures(columns),
         keras.layers.Dense(64, activation='relu'),
         keras.layers.Dense(20, activation='softmax')
     ])
@@ -70,9 +70,9 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_sequential_model_with_ds_input(self):
-    columns = [fc.numeric_column('a')]
+    columns = [fc.numeric_column_v2('a')]
     model = keras.models.Sequential([
-        fc.FeatureLayer(columns),
+        fc.DenseFeatures(columns),
         keras.layers.Dense(64, activation='relu'),
         keras.layers.Dense(20, activation='softmax')
     ])
@@ -94,8 +94,8 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_subclassed_model_with_feature_columns(self):
-    col_a = fc.numeric_column('a')
-    col_b = fc.numeric_column('b')
+    col_a = fc.numeric_column_v2('a')
+    col_b = fc.numeric_column_v2('b')
 
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
@@ -114,8 +114,8 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_subclassed_model_with_feature_columns_with_ds_input(self):
-    col_a = fc.numeric_column('a')
-    col_b = fc.numeric_column('b')
+    col_a = fc.numeric_column_v2('a')
+    col_b = fc.numeric_column_v2('b')
 
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
@@ -137,13 +137,13 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def DISABLED_test_function_model_feature_layer_input(self):
-    col_a = fc.numeric_column('a')
-    col_b = fc.numeric_column('b')
+    col_a = fc.numeric_column_v2('a')
+    col_b = fc.numeric_column_v2('b')
 
-    feature_layer = fc.FeatureLayer([col_a, col_b], name='fc')
+    feature_layer = fc.DenseFeatures([col_a, col_b], name='fc')
     dense = keras.layers.Dense(4)
 
-    # This seems problematic.... We probably need something for FeatureLayer
+    # This seems problematic.... We probably need something for DenseFeatures
     # the way Input is for InputLayer.
     output = dense(feature_layer)
 
@@ -163,15 +163,15 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def DISABLED_test_function_model_multiple_feature_layer_inputs(self):
-    col_a = fc.numeric_column('a')
-    col_b = fc.numeric_column('b')
-    col_c = fc.numeric_column('c')
+    col_a = fc.numeric_column_v2('a')
+    col_b = fc.numeric_column_v2('b')
+    col_c = fc.numeric_column_v2('c')
 
-    fc1 = fc.FeatureLayer([col_a, col_b], name='fc1')
-    fc2 = fc.FeatureLayer([col_b, col_c], name='fc2')
+    fc1 = fc.DenseFeatures([col_a, col_b], name='fc1')
+    fc2 = fc.DenseFeatures([col_b, col_c], name='fc2')
     dense = keras.layers.Dense(4)
 
-    # This seems problematic.... We probably need something for FeatureLayer
+    # This seems problematic.... We probably need something for DenseFeatures
     # the way Input is for InputLayer.
     output = dense(fc1) + dense(fc2)
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
index f06e798953..f6e165bd7a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
@@ -32,14 +32,6 @@ tf_module {
     name: "indicator_column"
     argspec: "args=[\'categorical_column\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "input_layer"
-    argspec: "args=[\'features\', \'feature_columns\', \'weight_collections\', \'trainable\', \'cols_to_vars\', \'cols_to_output_tensors\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "linear_model"
-    argspec: "args=[\'features\', \'feature_columns\', \'units\', \'sparse_combiner\', \'weight_collections\', \'trainable\', \'cols_to_vars\'], varargs=None, keywords=None, defaults=[\'1\', \'sum\', \'None\', \'True\', \'None\'], "
-  }
   member_method {
     name: "make_parse_example_spec"
     argspec: "args=[\'feature_columns\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
new file mode 100644
index 0000000000..35b31bd3ee
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -0,0 +1,175 @@
+path: "tensorflow.keras.layers.DenseFeatures"
+tf_class {
+  is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.DenseFeatures\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'feature_columns\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'features\', \'cols_to_output_tensors\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
new file mode 100644
index 0000000000..c727d5fc2d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -0,0 +1,269 @@
+path: "tensorflow.keras.layers.LinearModel"
+tf_class {
+  is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.LinearModel\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "bias"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_spec"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "layers"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "state_updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "stateful"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'feature_columns\', \'units\', \'sparse_combiner\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'1\', \'sum\', \'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'features\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "compile"
+    argspec: "args=[\'self\', \'optimizer\', \'loss\', \'metrics\', \'loss_weights\', \'sample_weight_mode\', \'weighted_metrics\', \'target_tensors\', \'distribute\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "evaluate"
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'verbose\', \'sample_weight\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'1\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+  }
+  member_method {
+    name: "evaluate_generator"
+    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "fit"
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+  }
+  member_method {
+    name: "fit_generator"
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_layer"
+    argspec: "args=[\'self\', \'name\', \'index\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "load_weights"
+    argspec: "args=[\'self\', \'filepath\', \'by_name\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+  member_method {
+    name: "predict"
+    argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\', \'10\', \'1\', \'False\'], "
+  }
+  member_method {
+    name: "predict_generator"
+    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], "
+  }
+  member_method {
+    name: "predict_on_batch"
+    argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "save"
+    argspec: "args=[\'self\', \'filepath\', \'overwrite\', \'include_optimizer\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], "
+  }
+  member_method {
+    name: "save_weights"
+    argspec: "args=[\'self\', \'filepath\', \'overwrite\', \'save_format\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "summary"
+    argspec: "args=[\'self\', \'line_length\', \'positions\', \'print_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "test_on_batch"
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "to_json"
+    argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "to_yaml"
+    argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "train_on_batch"
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
index 9d7e5bb8c7..3b4724ef10 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
@@ -124,6 +124,10 @@ tf_module {
     name: "Dense"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "DenseFeatures"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DepthwiseConv2D"
     mtype: "<type \'type\'>"
@@ -240,6 +244,10 @@ tf_module {
     name: "LeakyReLU"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearModel"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LocallyConnected1D"
     mtype: "<type \'type\'>"
-- 
GitLab


From 87fad1699b2b4323d2b7c5c3324bd512b1b8e3a6 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 13 Nov 2018 18:26:50 -0800
Subject: [PATCH 0197/1554] Disable CAPI.TensorHandleCopyBetweenTwoGPUDevices

PiperOrigin-RevId: 221371293
---
 tensorflow/c/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index edd0c30a46..d4defaa185 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -209,7 +209,10 @@ tf_cuda_cc_test(
         "//tensorflow:darwin": ["-headerpad_max_install_names"],
         "//conditions:default": [],
     }),
-    tags = ["noasan"],
+    tags = [
+        "no_oss",  # http://b/119522529
+        "noasan",
+    ],
     # We must ensure that the dependencies can be dynamically linked since
     # the shared library must be able to use core:framework.
     # linkstatic = tf_kernel_tests_linkstatic(),
-- 
GitLab


From 27b14271731036f8e044dd8b3a6db75514a240b3 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 13 Nov 2018 19:02:23 -0800
Subject: [PATCH 0198/1554] [XLA] Fix TODOs in hlo_evaluator.

Remove TODOs that say "add DCHECK back", because we've moved away from (D)CHECK
in XLA.

PiperOrigin-RevId: 221374452
---
 .../compiler/xla/service/hlo_evaluator.h      | 10 +++-------
 .../xla/service/hlo_evaluator_typed_visitor.h | 19 +++++--------------
 2 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 07f8d0aad4..8c90252896 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -222,14 +222,10 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
     const auto shape = instruction->shape();
     const auto* operand = instruction->operand(0);
 
-    // TODO(b/35950897, b/27796129): add DCHECK back once implicit broadcast is
-    // removed.
     if (!ShapeUtil::SameDimensions(shape, operand->shape())) {
-      return Unimplemented(
-          "Implicit broadcasting is currently unsupported in HLO evaluator "
-          "Shape Mismatch: %s vs %s",
-          ShapeUtil::HumanString(shape),
-          ShapeUtil::HumanString(operand->shape()));
+      return Unimplemented("Shape Mismatch: %s vs %s",
+                           ShapeUtil::HumanString(shape),
+                           ShapeUtil::HumanString(operand->shape()));
     }
 
     Literal result(shape);
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index ebed875eb4..6c3baa6f18 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -161,9 +161,6 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
                          HloOpcodeString(hlo_instruction->opcode()));
   }
 
-  // TODO(b/35950897): many of the stl functions used in the handlers are not
-  // overloaded for every XLA primitive type.
-
   template <typename NativeT,
             typename std::enable_if<std::is_unsigned<NativeT>::value>::type* =
                 nullptr>
@@ -2723,12 +2720,9 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     const auto* lhs = instruction->operand(0);
     const auto* rhs = instruction->operand(1);
 
-    // TODO(b/35950897, b/27796129): add DCHECK back once implicit broadcast
-    // is removed.
     if (!(ShapeUtil::SameDimensions(shape, rhs->shape()) &&
           ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()))) {
       return Unimplemented(
-          "Implicit broadcasting is currently unsupported in HLO evaluator "
           "Shape Mismatch: %s vs %s vs %s: ",
           ShapeUtil::HumanString(shape), ShapeUtil::HumanString(lhs->shape()),
           ShapeUtil::HumanString(rhs->shape()));
@@ -2757,17 +2751,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     const auto* rhs = instruction->operand(1);
     const auto* ehs = instruction->operand(2);
 
-    // TODO(b/35950897, b/27796129): add DCHECK back once implicit
-    // broadcast is removed.
     if (!(ShapeUtil::SameDimensions(shape, lhs->shape()) &&
           ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()) &&
           ShapeUtil::SameDimensions(rhs->shape(), ehs->shape()))) {
-      return Unimplemented(
-          "Implicit broadcasting is currently unsupported in HLO evaluator "
-          "Shape Mismatch: %s vs %s vs %s vs %s: ",
-          ShapeUtil::HumanString(shape), ShapeUtil::HumanString(lhs->shape()),
-          ShapeUtil::HumanString(rhs->shape()),
-          ShapeUtil::HumanString(ehs->shape()));
+      return Unimplemented("Shape Mismatch: %s vs %s vs %s vs %s: ",
+                           ShapeUtil::HumanString(shape),
+                           ShapeUtil::HumanString(lhs->shape()),
+                           ShapeUtil::HumanString(rhs->shape()),
+                           ShapeUtil::HumanString(ehs->shape()));
     }
 
     const Literal& lhs_literal = parent_->GetEvaluatedLiteralFor(lhs);
-- 
GitLab


From 9a9c1f31499f65503961c86e4dcc6eb3552c8775 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 13 Nov 2018 19:06:26 -0800
Subject: [PATCH 0199/1554] [XLA] Work around GCC bug (?) breaking the build in
 buffer_liveness_test.

PiperOrigin-RevId: 221375006
---
 tensorflow/compiler/xla/service/buffer_liveness_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
index cdc13d86b3..40825a7871 100644
--- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
@@ -611,8 +611,8 @@ TEST_F(BufferLivenessTest, DependentTupleElements) {
 class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest {
  protected:
   // Builds and runs a computation (see test case computation graphs below).
-  std::unique_ptr<HloModule> BuildModule(const bool update_uses_tuple_element1,
-                                         const bool fuse_gte0) {
+  std::unique_ptr<VerifiedHloModule> BuildModule(
+      const bool update_uses_tuple_element1, const bool fuse_gte0) {
     auto builder = HloComputation::Builder(TestName());
     // Create param0 Tuple.
     Shape data_shape = ShapeUtil::MakeShape(F32, {8});
-- 
GitLab


From 8619905e714cc7af160094580494cd39d26994f4 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 13 Nov 2018 19:29:01 -0800
Subject: [PATCH 0200/1554] [XLA] Comment fix in pattern_matcher.h

PiperOrigin-RevId: 221376722
---
 tensorflow/compiler/xla/service/pattern_matcher.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index bf5eb42e2c..f196d9b7f5 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -90,8 +90,8 @@ namespace xla {
 // are provided below.
 //
 // Example nullary instruction:
-//   Param()                        == Op().WithOpcode(HloOpcode::kParam)
-//   Param(&a)                      == Op(&a).WithOpcode(HloOpcode::kParam)
+//   Parameter()                    == Op().WithOpcode(HloOpcode::kParameter)
+//   Parameter(&a)                  == Op(&a).WithOpcode(HloOpcode::kParameter)
 //
 // Example unary instruction:
 //   Abs()                             == Op().WithOpcode(HloOpcode::kAbs)
-- 
GitLab


From 84cdd143a8bd0efc3cf64bfa41a2531c1c1a74aa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 19:36:09 -0800
Subject: [PATCH 0201/1554] Fixing device placement for variables

PiperOrigin-RevId: 221377325
---
 .../boosted_trees/python/training/functions/gbdt_batch.py  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
index ab5713fbe2..85020c5df2 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
@@ -1257,13 +1257,12 @@ class GradientBoostedDecisionTreeModel(object):
   def _get_replica_device_setter(self, worker_device):
     """Creates a replica device setter."""
     ps_tasks = self._num_ps_replicas
-    ps_ops = [
-        "Variable",
-        "VariableV2",
+    ps_ops = list(device_setter.STANDARD_PS_OPS)
+    ps_ops.extend([
         "DecisionTreeEnsembleResourceHandleOp",
         "StatsAccumulatorScalarResourceHandleOp",
         "StatsAccumulatorTensorResourceHandleOp",
-    ]
+    ])
     ps_strategy = _OpRoundRobinStrategy(ps_ops, ps_tasks)
     return device_setter.replica_device_setter(
         worker_device=worker_device,
-- 
GitLab


From dce5102ea087107894f558b1fcb6b72a46873364 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Tue, 13 Nov 2018 19:47:37 -0800
Subject: [PATCH 0202/1554] Add XLA compile API tutorial to book. Cleanup
 notebook.

PiperOrigin-RevId: 221378352
---
 tensorflow/compiler/xla/g3doc/_book.yaml      |   6 +-
 .../xla/g3doc/tutorials/xla_compile.ipynb     | 301 ++++++++----------
 2 files changed, 136 insertions(+), 171 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/_book.yaml b/tensorflow/compiler/xla/g3doc/_book.yaml
index bcfbcc3a22..756b932332 100644
--- a/tensorflow/compiler/xla/g3doc/_book.yaml
+++ b/tensorflow/compiler/xla/g3doc/_book.yaml
@@ -11,7 +11,7 @@ upper_tabs:
   lower_tabs:
     # Subsite tabs
     other:
-    - name: Guide
+    - name: Guide & Tutorials
       contents:
       - title: XLA overview
         path: /xla/overview
@@ -27,3 +27,7 @@ upper_tabs:
         path: /xla/shapes
       - title: Using AOT compilation
         path: /xla/tfcompile
+      - heading: Tutorials
+      - title: XLA compile API
+        path: /xla/tutorials/xla_compile
+        status: experimental
diff --git a/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb b/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb
index a83e3f7859..2a83092805 100644
--- a/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb
+++ b/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb
@@ -1,25 +1,38 @@
 {
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "name": "The XLA compile API",
+      "version": "0.3.2",
+      "provenance": [],
+      "collapsed_sections": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    }
+  },
   "cells": [
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "f4TSNCvpENrW"
       },
+      "cell_type": "markdown",
       "source": [
         "##### Copyright 2018 The TensorFlow Authors."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
         "cellView": "form",
-        "colab": {},
         "colab_type": "code",
-        "id": "vamNSA0vEP-m"
+        "id": "vamNSA0vEP-m",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -32,139 +45,84 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "code",
+      ],
       "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "xD_ydfejEV7H"
-      },
-      "outputs": [],
-      "source": [
-        "#@title MIT License\n",
-        "#\n",
-        "# Copyright (c) 2017 François Chollet\n",
-        "#\n",
-        "# Permission is hereby granted, free of charge, to any person obtaining a\n",
-        "# copy of this software and associated documentation files (the \"Software\"),\n",
-        "# to deal in the Software without restriction, including without limitation\n",
-        "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n",
-        "# and/or sell copies of the Software, and to permit persons to whom the\n",
-        "# Software is furnished to do so, subject to the following conditions:\n",
-        "#\n",
-        "# The above copyright notice and this permission notice shall be included in\n",
-        "# all copies or substantial portions of the Software.\n",
-        "#\n",
-        "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
-        "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
-        "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n",
-        "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
-        "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n",
-        "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n",
-        "# DEALINGS IN THE SOFTWARE."
-      ]
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "e1oSi4lHFt3z"
       },
+      "cell_type": "markdown",
       "source": [
-        "# Welcome to `xla.compile()` tutorial"
+        "# The XLA compile API"
       ]
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "b7noD9NjFRL-"
       },
+      "cell_type": "markdown",
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/xla/jit#turning_on_jit_compilation\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.sandbox.google.com/github/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://www.tensorflow.org/xla/tutorials/xla_compile\"><img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />View on TensorFlow.org</a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
+        "  </td>\n",
+        "</table>"
       ]
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "v9YbsuLZaBXy"
       },
+      "cell_type": "markdown",
       "source": [
-        "xla.compile() is a new experimental API that compiles part or all of a model with [XLA](https://www.tensorflow.org/extend/xla/).\n",
         "\n",
-        "Please run all code blocks in order."
+        "\n",
+        "Import TensorFlow and the XLA library. XLA contains `xla.compile()`, an experimental API that compiles part or all of a model with [XLA](https://www.tensorflow.org/extend/xla/)."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "45kUPj5ZFrRa"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow as tf"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9NMQFjroSMns"
+        "id": "45kUPj5ZFrRa",
+        "colab": {}
       },
-      "source": [
-        "Imports XLA library, which includes xla.compile() experimental API."
-      ]
-    },
-    {
       "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-Uggy03rSGJm"
-      },
-      "outputs": [],
       "source": [
+        "import tensorflow as tf\n",
+        "\n",
         "from tensorflow.contrib.compiler import xla"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "GZVNiRmTDV-5"
       },
+      "cell_type": "markdown",
       "source": [
-        "Define some necessary constants and prepare MNIST dataset."
+        "Define some necessary constants and prepare the MNIST dataset."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "f37TSEGvGX4_"
+        "id": "f37TSEGvGX4_",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Size of each input image, 28 x 28 pixels\n",
         "IMAGE_SIZE = 28 * 28\n",
@@ -174,17 +132,17 @@
         "TRAIN_BATCH_SIZE = 100\n",
         "# Number of training steps to run\n",
         "TRAIN_STEPS = 1000"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "TiVXchblG5hK"
+        "id": "TiVXchblG5hK",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Loads MNIST dataset.\n",
         "train, test = tf.keras.datasets.mnist.load_data()\n",
@@ -195,16 +153,18 @@
         "images, labels = iterator.get_next()\n",
         "images = tf.reshape(images, [-1, IMAGE_SIZE])\n",
         "images, labels = tf.cast(images, tf.float32), tf.cast(labels, tf.int64)"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "x_ZehpZP-SfS"
       },
+      "cell_type": "markdown",
       "source": [
-        "## Defines build_mnist_model function to construct model\n",
+        "# Define the model constructing function\n",
         "\n",
         "Following code block contains a function that constructs a simple model with one dense layer, including both forward and backward propagation.\n",
         "\n",
@@ -212,14 +172,12 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "ZbhJl_WvGa3g"
+        "id": "ZbhJl_WvGa3g",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "def build_mnist_model(x, y_):\n",
         "  y = tf.keras.layers.Dense(NUM_CLASSES).apply(x)\n",
@@ -228,47 +186,41 @@
         "  train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
         "\n",
         "  return y, train_step"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "7Jh3lyQHDfM9"
       },
-      "source": [
-        "## Uses xla.compile with build_mnist_model function to enable XLA"
-      ]
-    },
-    {
       "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EtDwez_1gjzv"
-      },
       "source": [
-        "Following code block wraps the model with xla.compile(), which allows the target function with provided inputs to be executed by XLA."
+        "# Enable XLA\n",
+        "\n",
+        "Use `xla.compile` with the `build_mnist_model` function to enable XLA. Following code block wraps the model with `xla.compile()`, which allows the target function with provided inputs to be executed by XLA."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "kYpCXCdRHNuN"
+        "id": "kYpCXCdRHNuN",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "[y] = xla.compile(build_mnist_model, inputs=[images, labels])"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "4giQh62IrZGF"
       },
+      "cell_type": "markdown",
       "source": [
         "When compiling the graph, XLA replaces all the graph nodes constructed in the target function with a few XLA ops.\n",
         "\n",
@@ -293,62 +245,62 @@
       ]
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "TPGas4jjFLZl"
       },
+      "cell_type": "markdown",
       "source": [
         "If you were to print the constructed graph now, you will see that it is not much different from a normal Tensorflow graph and you won't be able to find XLA ops mentioned before. This is because the actual compilation happens later when you try to execute the graph with `sess.run()`.  At that time, Tensorflow triggers a series of graph rewrite passes that actually generate XLA ops, which compiles and executes computation when all inputs are ready."
       ]
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "EZD1m_n1DxAF"
       },
+      "cell_type": "markdown",
       "source": [
-        "## Trains and tests the model"
+        "# Train and test the model"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "qe28bAHNHUG2"
+        "id": "qe28bAHNHUG2",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Creates session and initialize all variables.\n",
         "# xla.compile() doesn't work with Keras model.fit() API or TF eager mode yet.\n",
         "sess = tf.Session()\n",
         "sess.run(tf.global_variables_initializer())"
-      ]
+      ],
+      "execution_count": 0,
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
       "metadata": {
         "colab_type": "text",
         "id": "qgsKmz3n2UiW"
       },
+      "cell_type": "markdown",
       "source": [
-        "Following code block trains model.\n",
-        "\n",
-        "Note that evaluating `y` also triggers its control dependency node `train_step`, which updates model variables."
+        "Following code block trains model. Evaluating `y` also triggers its control dependency node `train_step`, which updates model variables."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "_GxF6jTRHVuA"
+        "id": "_GxF6jTRHVuA",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 34
+        },
+        "outputId": "fbf299ca-02d5-4e95-f9fe-8f3c0432d132"
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Feeds training dataset\n",
         "sess.run(iterator.make_initializer(train_ds))\n",
@@ -356,18 +308,31 @@
         "# Runs TRAIN_STEPS steps\n",
         "for i in range(TRAIN_STEPS):\n",
         "  sess.run(y)\n",
+        "\n",
         "print(\"Model trained for %s steps.\" % TRAIN_STEPS)"
+      ],
+      "execution_count": 21,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Model trained for 1000 steps.\n"
+          ],
+          "name": "stdout"
+        }
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "dHlQlRSRHXD1"
+        "id": "dHlQlRSRHXD1",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 34
+        },
+        "outputId": "9c3677a2-ec84-406f-9d2c-d722844f3093"
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Tests trained model\n",
         "\n",
@@ -378,35 +343,31 @@
         "correct_prediction = tf.equal(tf.argmax(y, 1), labels)\n",
         "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
         "print(\"Prediction accuracy after training: %s\" % sess.run(accuracy))"
+      ],
+      "execution_count": 22,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Prediction accuracy after training: 0.91\n"
+          ],
+          "name": "stdout"
+        }
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
       "metadata": {
-        "colab": {},
         "colab_type": "code",
-        "id": "ynJQIuzjHYOb"
+        "id": "ynJQIuzjHYOb",
+        "colab": {}
       },
-      "outputs": [],
+      "cell_type": "code",
       "source": [
         "# Cleans up session\n",
         "sess.close()"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "xla.compile() Tutorial",
-      "provenance": [],
-      "version": "0.3.2"
-    },
-    "kernelspec": {
-      "display_name": "Python 2",
-      "name": "python2"
+      ],
+      "execution_count": 0,
+      "outputs": []
     }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
+  ]
+}
\ No newline at end of file
-- 
GitLab


From b2b03984eec354ec9b46d88449e61e7365e993d9 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 13 Nov 2018 20:34:51 -0800
Subject: [PATCH 0203/1554] Start using Keras graph functions during eager
 execution. This means that the code paths between V1 graph mode and eager
 execution are now 99% the same (identical up to the part that actually builds
 and executes the graph function, which is in backend.py).

PiperOrigin-RevId: 221382254
---
 .../python/saved_model/keras_saved_model.py   |  27 +-
 .../saved_model/keras_saved_model_test.py     |   6 -
 tensorflow/python/keras/BUILD                 |   7 +-
 tensorflow/python/keras/backend.py            |  45 +-
 tensorflow/python/keras/engine/base_layer.py  |  53 +-
 .../python/keras/engine/base_layer_test.py    |   8 +-
 tensorflow/python/keras/engine/input_layer.py |   1 -
 tensorflow/python/keras/engine/network.py     |  26 +-
 tensorflow/python/keras/engine/sequential.py  |   8 +-
 .../python/keras/engine/sequential_test.py    |  23 -
 tensorflow/python/keras/engine/training.py    | 497 +++++++++---------
 .../python/keras/engine/training_arrays.py    |   4 +-
 .../python/keras/engine/training_eager.py     | 123 +----
 .../keras/engine/training_eager_test.py       |  28 +-
 .../python/keras/engine/training_test.py      |   7 -
 .../python/keras/engine/training_utils.py     |  71 +--
 .../keras/engine/training_utils_test.py       |  28 -
 .../keras/layers/advanced_activations.py      |   6 -
 .../python/keras/layers/convolutional.py      |  10 -
 tensorflow/python/keras/layers/core.py        |   9 -
 tensorflow/python/keras/layers/embeddings.py  |   1 -
 tensorflow/python/keras/layers/local.py       |   2 -
 tensorflow/python/keras/layers/local_test.py  | 270 +++++-----
 tensorflow/python/keras/layers/merge.py       |   3 -
 tensorflow/python/keras/layers/noise.py       |   3 -
 .../python/keras/layers/normalization.py      |   1 -
 tensorflow/python/keras/layers/pooling.py     |   6 -
 .../golden/v1/tensorflow.keras.-model.pbtxt   |   4 +
 .../v1/tensorflow.keras.-sequential.pbtxt     |   4 +
 .../v1/tensorflow.keras.models.-model.pbtxt   |   4 +
 .../tensorflow.keras.models.-sequential.pbtxt |   4 +
 .../golden/v2/tensorflow.keras.-model.pbtxt   |   4 +
 .../v2/tensorflow.keras.-sequential.pbtxt     |   4 +
 ...ensorflow.keras.layers.-linear-model.pbtxt |   4 +
 .../v2/tensorflow.keras.models.-model.pbtxt   |   4 +
 .../tensorflow.keras.models.-sequential.pbtxt |   4 +
 36 files changed, 554 insertions(+), 755 deletions(-)

diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
index 27b5b6d22e..4c8db94d6f 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
@@ -25,7 +25,6 @@ from tensorflow.python.client import session
 from tensorflow.python.estimator import keras as estimator_keras_util
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator.export import export as export_helpers
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import models as models_lib
@@ -307,31 +306,11 @@ def _create_signature_def_map(model, mode):
       serving_only=(mode == model_fn_lib.ModeKeys.PREDICT))
 
 
-def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph):
+def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph):  # pylint: disable=unused-argument
   """Assert model and clone contain the same checkpointable objects."""
 
-  def get_non_optimizer_objects(m, g):
-    """Gather set of model and optimizer checkpointable objects."""
-    # Set default graph because optimizer.variables() returns optimizer
-    # variables defined in the default graph.
-    with g.as_default():
-      all_objects = set(checkpointable_utils.list_objects(m))
-      optimizer_and_variables = set()
-      for obj in all_objects:
-        if isinstance(obj, optimizers.TFOptimizer):
-          optimizer_and_variables.update(checkpointable_utils.list_objects(obj))
-          optimizer_and_variables.update(set(obj.optimizer.variables()))
-      return all_objects - optimizer_and_variables
-
-  model_objects = get_non_optimizer_objects(model, model_graph)
-  clone_objects = get_non_optimizer_objects(clone, clone_graph)
-
-  if len(model_objects) != len(clone_objects):
-    raise errors.InternalError(
-        None, None,
-        'Model and clone must use the same variables.'
-        '\n\tModel variables: %s\n\t Clone variables: %s'
-        % (model_objects, clone_objects))
+  # TODO(fchollet, kathywu): make sure this works in eager mode.
+  return True
 
 
 def load_keras_model(saved_model_path):
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
index a65b2ce466..0619c4ca5a 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
@@ -29,7 +29,6 @@ from tensorflow.python import keras
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training
@@ -463,11 +462,6 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
       clone.compile(loss='mse', optimizer=keras.optimizers.RMSprop(lr=0.0001))
       clone.train_on_batch(input_arr, target_arr)
 
-    with self.assertRaisesRegexp(
-        errors.InternalError, 'Model and clone must use the same variables.'):
-      keras_saved_model._assert_same_non_optimizer_objects(
-          model, model_graph, clone, clone_graph)
-
   def testSaveSeqModelWithoutInputShapesRaisesError(self):
     """A Sequential model that hasn't been built should raise an error."""
     model = sequential_model_without_input_shape(True)
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 7dfafd8289..9f85fe5dd7 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -318,7 +318,7 @@ py_test(
 
 py_test(
     name = "advanced_activations_test",
-    size = "small",
+    size = "medium",
     srcs = ["layers/advanced_activations_test.py"],
     srcs_version = "PY2AND3",
     deps = [
@@ -369,7 +369,7 @@ cuda_py_test(
 
 py_test(
     name = "pooling_test",
-    size = "medium",
+    size = "large",
     srcs = ["layers/pooling_test.py"],
     srcs_version = "PY2AND3",
     deps = [
@@ -496,7 +496,7 @@ py_test(
 
 py_test(
     name = "recurrent_test",
-    size = "medium",
+    size = "large",
     srcs = ["layers/recurrent_test.py"],
     srcs_version = "PY2AND3",
     deps = [
@@ -763,6 +763,7 @@ py_test(
     name = "model_subclassing_test",
     size = "medium",
     srcs = ["model_subclassing_test.py"],
+    shard_count = 2,
     srcs_version = "PY2AND3",
     tags = ["notsan"],
     deps = [
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index dd9b0c07e7..aeb9bd4cb0 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -376,27 +376,22 @@ def learning_phase():
   Returns:
       Learning phase (scalar integer tensor or Python integer).
   """
-  with ops.init_scope():
-    # We always check & set the learning phase inside the init_scope,
-    # otherwise the wrong default_graph will be used to look up the learning
-    # phase inside of functions & defuns.
-    #
-    # This is because functions & defuns (both in graph & in eager mode)
-    # will always execute non-eagerly using a function-specific default
-    # subgraph.
-    if context.executing_eagerly():
-      if _DUMMY_EAGER_GRAPH not in _GRAPH_LEARNING_PHASES:
-        # Fallback to inference mode as default.
-        return 0
-      return _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH]
+  if context.executing_eagerly():
+    if _DUMMY_EAGER_GRAPH not in _GRAPH_LEARNING_PHASES:
+      # Fallback to inference mode as default.
+      return 0
+    return _GRAPH_LEARNING_PHASES[_DUMMY_EAGER_GRAPH]
+  return symbolic_learning_phase()
 
-    graph = get_graph()
-    with graph.as_default():
-      if graph not in _GRAPH_LEARNING_PHASES:
-        phase = array_ops.placeholder_with_default(
-            False, shape=(), name='keras_learning_phase')
-        _GRAPH_LEARNING_PHASES[graph] = phase
-      return _GRAPH_LEARNING_PHASES[graph]
+
+def symbolic_learning_phase():
+  graph = get_graph()
+  with graph.as_default():
+    if graph not in _GRAPH_LEARNING_PHASES:
+      phase = array_ops.placeholder_with_default(
+          False, shape=(), name='keras_learning_phase')
+      _GRAPH_LEARNING_PHASES[graph] = phase
+    return _GRAPH_LEARNING_PHASES[graph]
 
 
 @tf_export('keras.backend.set_learning_phase')
@@ -3123,20 +3118,20 @@ class EagerExecutionFunction(object):
             updates_ops.append(update)
 
       # We set the update ops to run at the end by conditioning it on output[0]
-      if updates and not outputs:
+      if updates and not self.outputs:
         # Edge case; never happens in practice
         raise ValueError('Cannot create a Keras backend function with updates'
                          ' but no outputs during eager execution.')
       with ops.control_dependencies(updates_ops):
-        outputs[0] = array_ops.identity(outputs[0])
+        self.outputs[0] = array_ops.identity(self.outputs[0])
 
     # Prepare graph function
     # TODO(fchollet): can we restrict `captures` to variables actually used in
     # the relevant subgraph?
-    graph.inputs = inputs + list(graph.captures.values())
-    graph.outputs = outputs
+    graph.inputs = self.inputs + list(graph.captures.values())
+    graph.outputs = self.outputs
     graph_fn = eager_function.Function(graph)
-    graph_fn._num_positional_args = len(inputs)
+    graph_fn._num_positional_args = len(self.inputs)
     graph_fn._arg_keywords = []
     self._graph_fn = graph_fn
 
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 23419ae150..9f875cdc3c 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -183,16 +183,6 @@ class Layer(checkpointable.CheckpointableBase):
 
     self.supports_masking = False
 
-    # Mark if a layer supports using graph functions in the eager
-    # fit/predict/evaluate loop
-    # TODO(kaftan): merge this with the _static_graph_friendly flag once
-    # enough eager function bugs involving control flow / tensorarrays have
-    # been fixed,  and static-graph-friendly layers will almost always work in
-    # eager graph functions.
-    # We conservatively make this flag opt-in for now to avoid causing existing
-    # custom layers to crash.
-    self._can_use_graph_functions = False
-
     call_argspec = tf_inspect.getfullargspec(self.call)
     if 'training' in call_argspec.args:
       self._expects_training_arg = True
@@ -200,7 +190,7 @@ class Layer(checkpointable.CheckpointableBase):
       self._expects_training_arg = False
 
     # Whether the `call` method can be used to build a TF graph without issues.
-    self._static_graph_friendly = True
+    self._call_is_graph_friendly = True
 
     # Manage input shape information if passed.
     if 'input_shape' in kwargs or 'batch_input_shape' in kwargs:
@@ -222,17 +212,6 @@ class Layer(checkpointable.CheckpointableBase):
     else:
       self._initial_weights = None
 
-  @property
-  def _is_static_graph_friendly(self):
-    return self._static_graph_friendly
-
-  @_is_static_graph_friendly.setter
-  def _is_static_graph_friendly(self, value):
-    if value not in {True, False}:
-      raise ValueError('`static_graph_friendly` requires a boolean value. '
-                       'Received: {}'.format(value))
-    self._static_graph_friendly = value
-
   def _init_set_name(self, name, zero_based=True):
     if not name:
       self._name = unique_layer_name(
@@ -298,8 +277,6 @@ class Layer(checkpointable.CheckpointableBase):
 
   @property
   def updates(self):
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.updates not supported in Eager mode.')
     if not self.trainable and not self.stateful:
       return []
     return self._updates
@@ -366,9 +343,6 @@ class Layer(checkpointable.CheckpointableBase):
     Raises:
       RuntimeError: If called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('`get_updates_for()` not supported in Eager mode.')
-
     # Updates disabled if layer is not trainable and not explicitly stateful.
     if not self.trainable and not self.stateful:
       return []
@@ -471,9 +445,6 @@ class Layer(checkpointable.CheckpointableBase):
     Raises:
       RuntimeError: If called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.get_losses_for not supported in Eager mode.')
-
     if inputs is None:
       # Requesting unconditional losses.
       return [x for x in self.losses if x._unconditional_loss]  # pylint: disable=protected-access
@@ -772,10 +743,10 @@ class Layer(checkpointable.CheckpointableBase):
               # Any issue during graph-building means we will later run the
               # model in eager mode, whether the issue was related to
               # graph mode or not. This provides a nice debugging experience.
-              self._is_static_graph_friendly = False
+              self._call_is_graph_friendly = False
               # We will use static shape inference to return symbolic tensors
               # matching the specifications of the layer outputs.
-              # Since we have set `self._is_static_graph_friendly = False`,
+              # Since we have set `self._call_is_graph_friendly = False`,
               # we will never attempt to run the underlying TF graph (which is
               # disconnected).
               # TODO(fchollet): consider py_func as an alternative, which
@@ -989,7 +960,6 @@ class Layer(checkpointable.CheckpointableBase):
       # use `compute_output_shape` manually (these users will have to
       # implement `compute_output_shape` themselves).
       self.build(input_shape)
-
       with context.graph_mode():
         graph = func_graph.FuncGraph('graph')
         with graph.as_default():
@@ -1606,6 +1576,23 @@ class Layer(checkpointable.CheckpointableBase):
     """
     return cls(**config)
 
+  @property
+  def _static_graph_friendly(self):
+    """Whether the layer can be called to create a static graph.
+
+    Because of nesting, there are two components to being "graph-friendly":
+      1) all inner layers are graph-friendly
+      2) the way they are composed is graph-friendly.
+    We denote the latter as "_call_is_graph_friendly", and define
+    "_static_graph_friendly" as being the combination of
+    "_call_is_graph_friendly" and "all inner layers are _static_graph_friendly".
+    For atomic layers (no inner layers), this is just "_call_is_graph_friendly".
+
+    Returns:
+      Boolean.
+    """
+    return self._call_is_graph_friendly
+
 
 @tf_export(
     'keras.layers.InputSpec', v1=['keras.layers.InputSpec', 'layers.InputSpec'])
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index 704589349a..798775b6a5 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -81,14 +81,14 @@ class BaseLayerTest(test.TestCase):
     inputs = keras.Input((3,))
     outputs = DynamicLayer1()(inputs)
     model = keras.Model(inputs, outputs)
-    self.assertEqual(model._is_static_graph_friendly, False)
+    self.assertEqual(model._static_graph_friendly, False)
     model.compile(RMSPropOptimizer(0.001), loss='mse')
     model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3)))
 
     inputs = keras.Input((3,))
     outputs = DynamicLayer2()(inputs)
     model = keras.Model(inputs, outputs)
-    self.assertEqual(model._is_static_graph_friendly, False)
+    self.assertEqual(model._static_graph_friendly, False)
     model.compile(RMSPropOptimizer(0.001), loss='mse')
     model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3)))
 
@@ -102,7 +102,7 @@ class BaseLayerTest(test.TestCase):
     outputs = inner_model(x)
 
     model = keras.Model(inputs, outputs)
-    self.assertEqual(model._is_static_graph_friendly, False)
+    self.assertEqual(model._static_graph_friendly, False)
     model.compile(RMSPropOptimizer(0.001), loss='mse')
     model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3)))
 
@@ -116,7 +116,7 @@ class BaseLayerTest(test.TestCase):
     inputs = keras.Input((3,))
     outputs = InvalidLayer()(inputs)
     model = keras.Model(inputs, outputs)
-    self.assertEqual(model._is_static_graph_friendly, False)
+    self.assertEqual(model._static_graph_friendly, False)
     model.compile(RMSPropOptimizer(0.001), loss='mse')
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       model.train_on_batch(np.random.random((2, 3)), np.random.random((2, 3)))
diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index 4e96106004..590b935d40 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -84,7 +84,6 @@ class InputLayer(base_layer.Layer):
     self.sparse = sparse
     self.batch_size = batch_size
     self.supports_masking = True
-    self._can_use_graph_functions = True
 
     if isinstance(input_shape, tensor_shape.TensorShape):
       input_shape = tuple(input_shape.as_list())
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 9b58180e3d..febc224689 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -139,7 +139,7 @@ class Network(base_layer.Layer):
     self._eager_losses = []
     self._scope = None  # Never used.
     self._reuse = None  # Never used.
-    self._can_use_graph_functions = False
+    self._call_is_graph_friendly = True
     if context.executing_eagerly():
       self._graph = None
     else:
@@ -258,10 +258,6 @@ class Network(base_layer.Layer):
 
     self._track_layers(layers)
 
-    # A Graph network supports defun-ed eager loops if all of its layers do.
-    self._can_use_graph_functions = all(
-        layer._can_use_graph_functions for layer in layers)
-
     # Create the node linking internal inputs to internal outputs.
     base_layer.Node(
         outbound_layer=self,
@@ -282,9 +278,7 @@ class Network(base_layer.Layer):
       if layer.is_placeholder:
         self._feed_input_names.append(layer.name)
         self._feed_input_shapes.append(backend.int_shape(self.inputs[i]))
-        # layer.input gives an error in eager mode
-        if not context.executing_eagerly():
-          self._feed_inputs.append(layer.input)
+        self._feed_inputs.append(layer.input)
     for layer in self._output_layers:
       self.output_names.append(layer.name)
 
@@ -301,13 +295,12 @@ class Network(base_layer.Layer):
     self.outputs = []
     self.inputs = []
     self.built = False
-    self._static_graph_friendly = True
 
   @property
-  def _is_static_graph_friendly(self):
+  def _static_graph_friendly(self):
     if self._is_graph_network:
-      return all(layer._is_static_graph_friendly for layer in self.layers)
-    return self._static_graph_friendly
+      return all(layer._static_graph_friendly for layer in self.layers)
+    return self._call_is_graph_friendly
 
   def _determine_call_convention(self, call_argspec):
     """Decides how `self.call()` is invoked. See base_layer.CallConvention."""
@@ -557,8 +550,6 @@ class Network(base_layer.Layer):
 
   @property
   def _unfiltered_updates(self):
-    if context.executing_eagerly():
-      return []
     updates = []
     for layer in self.layers:
       if isinstance(layer, Network):
@@ -641,9 +632,6 @@ class Network(base_layer.Layer):
     Returns:
         A list of update ops.
     """
-    if context.executing_eagerly():
-      return []
-
     if not self.trainable and not self.stateful:
       return []
 
@@ -895,9 +883,7 @@ class Network(base_layer.Layer):
 
   def compute_output_shape(self, input_shape):
     if not self._is_graph_network:
-      if context.executing_eagerly():
-        return super(Network, self).compute_output_shape(input_shape)
-      raise NotImplementedError
+      return super(Network, self).compute_output_shape(input_shape)
 
     if isinstance(input_shape, list):
       input_shapes = []
diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 5ce4ca4df4..26866d4714 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -120,8 +120,8 @@ class Sequential(Model):
     return layers[:]
 
   @property
-  def _is_static_graph_friendly(self):
-    return all(layer._is_static_graph_friendly for layer in self.layers)
+  def _static_graph_friendly(self):
+    return all(layer._static_graph_friendly for layer in self.layers)
 
   @checkpointable.no_automatic_dependency_tracking
   def add(self, layer):
@@ -190,8 +190,6 @@ class Sequential(Model):
       self._layers.append(layer)
     if self._layers:
       self._track_layers(self._layers)
-    self._can_use_graph_functions = all(
-        layer._can_use_graph_functions for layer in self.layers)
 
   @checkpointable.no_automatic_dependency_tracking
   def pop(self):
@@ -213,8 +211,6 @@ class Sequential(Model):
       self.outputs = [self.layers[-1].output]
       self._init_graph_network(self.inputs, self.outputs, name=self.name)
       self.built = True
-    self._can_use_graph_functions = all(
-        layer._can_use_graph_functions for layer in self.layers)
 
   def build(self, input_shape=None):
     if self._is_graph_network:
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index ea8fdf675a..54db0f3566 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -362,29 +362,6 @@ class TestSequentialEagerIntegration(test.TestCase):
     y = np.random.random((2, 5))
     model.fit(x, y, epochs=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_sequential_can_use_graph_functions(self):
-    model = testing_utils.get_small_sequential_mlp(4, 3)
-    self.assertTrue(model._can_use_graph_functions)
-    inner_model = testing_utils.get_small_sequential_mlp(4, 5)
-    model.add(inner_model)
-
-    self.assertTrue(model._can_use_graph_functions)
-
-    inner_model_two = testing_utils.get_small_sequential_mlp(5, 7)
-    self.assertTrue(inner_model_two._can_use_graph_functions)
-
-    layer = keras.layers.Lambda(lambda x: x)
-    layer._can_use_graph_functions = False
-    inner_model_two.add(layer)
-    self.assertFalse(inner_model_two._can_use_graph_functions)
-
-    model.add(inner_model_two)
-    self.assertFalse(model._can_use_graph_functions)
-
-    model.pop()
-    self.assertTrue(model._can_use_graph_functions)
-
   @tf_test_util.run_in_graph_and_eager_modes
   def test_sequential_model_fails_with_dict_inputs(self):
     num_classes = 5
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 289ef7f9a0..4c56e1eea7 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -121,11 +121,8 @@ class Model(Network):
     # initializing _distribution_strategy here since it is possible to call
     # predict on a model without compiling it.
     self._distribution_strategy = None
-    # This flag must be disabled upon model mutation, such as changing the model
-    # layers or recompiling the model to use a different optimizer. New function
-    # definitions are generated whenever this flag is disabled, ensuring that
-    # internal graph functions are always using the current model structure.
-    self._built_graph_functions = False
+
+    self.run_eagerly = None
 
   def _set_sample_weight_attributes(self, sample_weight_mode,
                                     skip_target_weighing_indices):
@@ -264,9 +261,9 @@ class Model(Network):
           # If the given metric fn is stateful, call the fn and return result.
           metric_result = _call_stateful_fn(metric_fn)
           metric_results.append(metric_result)
-          if not context.executing_eagerly():
+          if not self.run_eagerly:
             _track_metric_tensors(metric_result, metric_result)
-        elif context.executing_eagerly():
+        elif self.run_eagerly:
           # In eager mode, if the given metric fn is not stateful, we invoke the
           # given fn or its stateful version based on the given flag.
           if return_stateful_result:
@@ -330,6 +327,40 @@ class Model(Network):
                 return_stateful_result=return_stateful_result))
     return metric_results
 
+  @property
+  def run_eagerly(self):
+    """Settable attribute indicating whether the model should run eagerly.
+
+    Running eagerly means that your model will be run step by step,
+    like Python code. Your model might run slower, but it should become easier
+    for you to debug it by stepping into individual layer calls.
+
+    By default, we will attempt to compile your model to a static graph to
+    deliver the best execution performance.
+
+    Returns:
+      Boolean, whether the model should run eagerly.
+    """
+    if self._run_eagerly is True and not context.executing_eagerly():
+      raise ValueError('You can only set `run_eagerly=True` if eager execution '
+                       'is enabled.')
+    if self._static_graph_friendly:
+      if self._run_eagerly is None:
+        return False
+      else:
+        return self._run_eagerly
+    else:
+      if self._run_eagerly is False:
+        # TODO(fchollet): consider using py_func to enable this.
+        raise ValueError('Your model contains layers that can only be '
+                         'successfully run in eager execution. '
+                         'You cannot set `run_eagerly=False`.')
+      return context.executing_eagerly()
+
+  @run_eagerly.setter
+  def run_eagerly(self, value):
+    self._run_eagerly = value
+
   @checkpointable.no_automatic_dependency_tracking
   def compile(self,
               optimizer,
@@ -391,9 +422,8 @@ class Model(Network):
         ValueError: In case of invalid arguments for
             `optimizer`, `loss`, `metrics` or `sample_weight_mode`.
     """
-    # The correct graph function may have changed,
-    # already-built ones must be updated
-    self._built_graph_functions = False
+    run_eagerly = kwargs.pop('run_eagerly', None)
+    self._run_eagerly = run_eagerly
 
     # Validate that arguments passed by the user to `compile` are supported by
     # DistributionStrategy.
@@ -403,9 +433,9 @@ class Model(Network):
         raise NotImplementedError(
             'optimizer must be an instance of '
             'tf.train.Optimizer, not a %s' % type(optimizer))
-      if context.executing_eagerly():
+      if self.run_eagerly:
         raise NotImplementedError('DistributionStrategy is not supported '
-                                  'when eager execution is enabled.')
+                                  'when running a model eagerly.')
       if sample_weight_mode:
         raise NotImplementedError('sample_weight_mode is not supported with '
                                   'DistributionStrategy.')
@@ -417,11 +447,12 @@ class Model(Network):
                          'DistributionStrategy.')
 
     loss = loss or {}
-    if context.executing_eagerly() and not isinstance(
+    if self.run_eagerly and not isinstance(
         optimizer, (tf_optimizer_module.Optimizer, optimizers.TFOptimizer)):
       raise ValueError(
-          'optimizer must be an instance of tf.train.Optimizer, not '
-          'a %s' % type(optimizer))
+          'When running a model in eager execution, the optimizer must be an '
+          'instance of tf.train.Optimizer. Received: '
+          '%s' % optimizer)
 
     self.optimizer = optimizers.get(optimizer)
     # We've disabled automatic dependency tracking for this method, but do want
@@ -434,8 +465,10 @@ class Model(Network):
     self.loss_weights = loss_weights
     self.sample_weight_mode = sample_weight_mode
     self.weighted_metrics = weighted_metrics
-    if context.executing_eagerly() and target_tensors is not None:
-      raise ValueError('target_tensors is not supported in Eager mode.')
+    if self.run_eagerly and target_tensors is not None:
+      raise ValueError(
+          'target_tensors argument is not supported when '
+          'running a model eagerly.')
     self.target_tensors = target_tensors
 
     # Set DistributionStrategy specific parameters.
@@ -445,6 +478,8 @@ class Model(Network):
     if self._distribution_strategy is not None:
       distributed_training_utils.configure_and_create_session(
           self._distribution_strategy)
+    # Initialize model metric attributes.
+    self._init_metric_attributes()
     if not self.built:
       # Model is not compilable because it does not know its number of inputs
       # and outputs, nor their shapes and names. We will compile after the first
@@ -493,7 +528,7 @@ class Model(Network):
         skip_target_weighing_indices.append(i)
 
     # Prepare output masks.
-    if not context.executing_eagerly():
+    if not self.run_eagerly:
       masks = [getattr(x, '_keras_mask', None) for x in self.outputs]
       if not isinstance(masks, list):
         masks = [masks]
@@ -524,11 +559,8 @@ class Model(Network):
                       str(loss_weights) + ' - expected a list of dicts.')
     self.loss_weights_list = loss_weights_list
 
-    # Initialize model metric attributes.
-    self._init_metric_attributes()
-
     # Initialization for Eager mode execution.
-    if context.executing_eagerly():
+    if self.run_eagerly:
       # Prepare sample weights.
       self._set_sample_weight_attributes(sample_weight_mode,
                                          skip_target_weighing_indices)
@@ -555,145 +587,146 @@ class Model(Network):
       self._collected_trainable_weights = self.trainable_weights
       return
 
-    # Prepare targets of model.
-    self.targets = []
-    self._feed_targets = []
-    if target_tensors not in (None, []):
-      if isinstance(target_tensors, list):
-        if len(target_tensors) != len(self.outputs):
-          raise ValueError(
-              'When passing a list as `target_tensors`, '
-              'it should have one entry per model output. '
-              'The model has ' + str(len(self.outputs)) +
-              ' outputs, but you passed target_tensors=' + str(target_tensors))
-      elif isinstance(target_tensors, dict):
-        for name in target_tensors:
-          if name not in self.output_names:
+    with K.get_graph().as_default():
+      # Prepare targets of model.
+      self.targets = []
+      self._feed_targets = []
+      if target_tensors not in (None, []):
+        if isinstance(target_tensors, list):
+          if len(target_tensors) != len(self.outputs):
             raise ValueError(
-                'Unknown entry in `target_tensors` '
-                'dictionary: "' + name + '". '
-                'Only expected the following keys: ' + str(self.output_names))
-        tmp_target_tensors = []
-        for name in self.output_names:
-          tmp_target_tensors.append(target_tensors.get(name, None))
-        target_tensors = tmp_target_tensors
-      elif tensor_util.is_tensor(target_tensors):
-        target_tensors = [target_tensors]
-      else:
-        raise TypeError('Expected `target_tensors` to be a list or tuple or '
-                        'dict or a single tensor, but got:', target_tensors)
-
-    for i in range(len(self.outputs)):
-      if i in skip_target_indices:
-        self.targets.append(None)
-      else:
-        shape = K.int_shape(self.outputs[i])
-        name = self.output_names[i]
-        if target_tensors not in (None, []):
-          target = target_tensors[i]
+                'When passing a list as `target_tensors`, '
+                'it should have one entry per model output. '
+                'The model has %s outputs, but you passed target_tensors=%s' %
+                (len(self.outputs), target_tensors))
+        elif isinstance(target_tensors, dict):
+          for name in target_tensors:
+            if name not in self.output_names:
+              raise ValueError(
+                  'Unknown entry in `target_tensors` '
+                  'dictionary: "' + name + '". '
+                  'Only expected the following keys: ' + str(self.output_names))
+          tmp_target_tensors = []
+          for name in self.output_names:
+            tmp_target_tensors.append(target_tensors.get(name, None))
+          target_tensors = tmp_target_tensors
+        elif tensor_util.is_tensor(target_tensors):
+          target_tensors = [target_tensors]
         else:
-          target = None
-        if target is None or K.is_placeholder(target):
-          if target is None:
-            target = K.placeholder(
-                ndim=len(shape),
-                name=name + '_target',
-                sparse=K.is_sparse(self.outputs[i]),
-                dtype=K.dtype(self.outputs[i]))
-          self._feed_targets.append(target)
-          self._feed_outputs.append(self.outputs[i])
-          self._feed_output_names.append(name)
-          self._feed_output_shapes.append(shape)
-          self._feed_loss_fns.append(self.loss_functions[i])
-        else:
-          skip_target_weighing_indices.append(i)
-        self.targets.append(target)
-
-    # Prepare sample weights.
-    self._set_sample_weight_attributes(sample_weight_mode,
-                                       skip_target_weighing_indices)
-    # Save all metric attributes per output of the model.
-    self._cache_output_metric_attributes(metrics, weighted_metrics)
-
-    # Compute total loss.
-    total_loss = None
-    with K.name_scope('loss'):
+          raise TypeError('Expected `target_tensors` to be a list or tuple or '
+                          'dict or a single tensor, but got:', target_tensors)
+
       for i in range(len(self.outputs)):
         if i in skip_target_indices:
-          continue
-        y_true = self.targets[i]
-        y_pred = self.outputs[i]
-        loss_fn = loss_functions[i]
-        sample_weight = self.sample_weights[i]
-        mask = masks[i]
-        loss_weight = loss_weights_list[i]
-        with K.name_scope(self.output_names[i] + '_loss'):
-          weighted_loss = training_utils.weighted_masked_objective(loss_fn)
-          output_loss = weighted_loss(y_true, y_pred, sample_weight, mask)
+          self.targets.append(None)
+        else:
+          shape = K.int_shape(self.outputs[i])
+          name = self.output_names[i]
+          if target_tensors not in (None, []):
+            target = target_tensors[i]
+          else:
+            target = None
+          if target is None or K.is_placeholder(target):
+            if target is None:
+              target = K.placeholder(
+                  ndim=len(shape),
+                  name=name + '_target',
+                  sparse=K.is_sparse(self.outputs[i]),
+                  dtype=K.dtype(self.outputs[i]))
+            self._feed_targets.append(target)
+            self._feed_outputs.append(self.outputs[i])
+            self._feed_output_names.append(name)
+            self._feed_output_shapes.append(shape)
+            self._feed_loss_fns.append(self.loss_functions[i])
+          else:
+            skip_target_weighing_indices.append(i)
+          self.targets.append(target)
 
-        if len(self.outputs) > 1:
-          # Keep track of the un-aggregated loss result tensor.
-          self.metrics_tensors.append(output_loss)
-
-          # Keep track of stateful result tensor and function for the loss.
-          mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-              loss_fn, name=loss_fn.__name__)
-          result_tensor = training_utils.call_metric_function(
-              mean_wrapped_loss,
-              y_true,
-              y_pred,
-              weights=sample_weight,
-              mask=mask)
-          self._stateful_metrics_tensors.append(result_tensor)
-          self.stateful_metric_functions.append(mean_wrapped_loss)
+      # Prepare sample weights.
+      self._set_sample_weight_attributes(sample_weight_mode,
+                                         skip_target_weighing_indices)
+      # Save all metric attributes per output of the model.
+      self._cache_output_metric_attributes(metrics, weighted_metrics)
 
-          self.metrics_names.append(self.output_names[i] + '_loss')
+      # Compute total loss.
+      total_loss = None
+      with K.name_scope('loss'):
+        for i in range(len(self.outputs)):
+          if i in skip_target_indices:
+            continue
+          y_true = self.targets[i]
+          y_pred = self.outputs[i]
+          loss_fn = loss_functions[i]
+          sample_weight = self.sample_weights[i]
+          mask = masks[i]
+          loss_weight = loss_weights_list[i]
+          with K.name_scope(self.output_names[i] + '_loss'):
+            weighted_loss = training_utils.weighted_masked_objective(loss_fn)
+            output_loss = weighted_loss(y_true, y_pred, sample_weight, mask)
+
+          if len(self.outputs) > 1:
+            # Keep track of the un-aggregated loss result tensor.
+            self.metrics_tensors.append(output_loss)
+
+            # Keep track of stateful result tensor and function for the loss.
+            mean_wrapped_loss = metrics_module.MeanMetricWrapper(
+                loss_fn, name=loss_fn.__name__)
+            result_tensor = training_utils.call_metric_function(
+                mean_wrapped_loss,
+                y_true,
+                y_pred,
+                weights=sample_weight,
+                mask=mask)
+            self._stateful_metrics_tensors.append(result_tensor)
+            self.stateful_metric_functions.append(mean_wrapped_loss)
+
+            self.metrics_names.append(self.output_names[i] + '_loss')
+          if total_loss is None:
+            total_loss = loss_weight * output_loss
+          else:
+            total_loss += loss_weight * output_loss
         if total_loss is None:
-          total_loss = loss_weight * output_loss
-        else:
-          total_loss += loss_weight * output_loss
-      if total_loss is None:
-        if not self.losses:
-          raise ValueError('The model cannot be compiled '
-                           'because it has no loss to optimize.')
-        else:
-          total_loss = 0.
-
-      # Add regularization penalties
-      # and other layer-specific losses.
-      for loss_tensor in self.losses:
-        total_loss += loss_tensor
-
-    # Set metric attributes on model.
-    self._set_metric_attributes(
-        self.outputs,
-        skip_target_indices=skip_target_indices,
-    )
-    # Invoke metric functions for all the outputs.
-    self._handle_metrics(
-        self.outputs,
-        masks=masks,
-        targets=self.targets,
-        skip_target_indices=skip_target_indices,
-        sample_weights=self.sample_weights)
-
-    # Prepare gradient updates and state updates.
-    self.total_loss = total_loss
-
-    # Functions for train, test and predict will
-    # be compiled lazily when required.
-    # This saves time when the user is not using all functions.
-    self._function_kwargs = kwargs
-
-    self._fit_function = None
-    self._eval_function = None
-    self.train_function = None
-    self.test_function = None
-    self.predict_function = None
-
-    # Collected trainable weights, sorted in topological order.
-    trainable_weights = self.trainable_weights
-    self._collected_trainable_weights = trainable_weights
+          if not self.losses:
+            raise ValueError('The model cannot be compiled '
+                             'because it has no loss to optimize.')
+          else:
+            total_loss = 0.
+
+        # Add regularization penalties
+        # and other layer-specific losses.
+        for loss_tensor in self.losses:
+          total_loss += loss_tensor
+
+      # Set metric attributes on model.
+      self._set_metric_attributes(
+          self.outputs,
+          skip_target_indices=skip_target_indices,
+      )
+      # Invoke metric functions for all the outputs.
+      self._handle_metrics(
+          self.outputs,
+          masks=masks,
+          targets=self.targets,
+          skip_target_indices=skip_target_indices,
+          sample_weights=self.sample_weights)
+
+      # Prepare gradient updates and state updates.
+      self.total_loss = total_loss
+
+      # Functions for train, test and predict will
+      # be compiled lazily when required.
+      # This saves time when the user is not using all functions.
+      self._function_kwargs = kwargs
+
+      self._fit_function = None
+      self._eval_function = None
+      self.train_function = None
+      self.test_function = None
+      self.predict_function = None
+
+      # Collected trainable weights, sorted in topological order.
+      trainable_weights = self.trainable_weights
+      self._collected_trainable_weights = trainable_weights
 
   def _check_trainable_weights_consistency(self):
     """Check trainable weights count consistency.
@@ -721,21 +754,24 @@ class Model(Network):
       inputs = (self._feed_inputs +
                 self._feed_targets +
                 self._feed_sample_weights)
-      if not isinstance(K.learning_phase(), int):
-        inputs += [K.learning_phase()]
+      if not isinstance(K.symbolic_learning_phase(), int):
+        inputs += [K.symbolic_learning_phase()]
+
+      with K.get_graph().as_default():
+        with K.name_scope('training'):
+          with K.name_scope(self.optimizer.__class__.__name__):
+            # Training updates
+            updates = self.optimizer.get_updates(
+                params=self._collected_trainable_weights, loss=self.total_loss)
+      # Unconditional updates
+      updates += self.get_updates_for(None)
+      # Conditional updates relevant to this model
+      updates += self.get_updates_for(self.inputs)
+      # Add stateful metrics updates.
+      if metric_updates is not None:
+        updates += metric_updates
 
       with K.name_scope('training'):
-        with K.name_scope(self.optimizer.__class__.__name__):
-          # Training updates
-          updates = self.optimizer.get_updates(
-              params=self._collected_trainable_weights, loss=self.total_loss)
-        # Unconditional updates
-        updates += self.get_updates_for(None)
-        # Conditional updates relevant to this model
-        updates += self.get_updates_for(self.inputs)
-        # Add stateful metrics updates.
-        if metric_updates is not None:
-          updates += metric_updates
         # Gets loss and metrics. Updates weights at each call.
         fn = K.function(
             inputs,
@@ -766,21 +802,23 @@ class Model(Network):
       inputs = (self._feed_inputs +
                 self._feed_targets +
                 self._feed_sample_weights)
-      if not isinstance(K.learning_phase(), int):
-        inputs += [K.learning_phase()]
-      updates = self.state_updates
-      # Add stateful metrics updates.
-      if metric_updates is not None:
-        updates += metric_updates
-      # Return loss and metrics, no gradient updates.
-      # Does update the network states.
-      fn = K.function(
-          inputs,
-          outputs,
-          updates=updates,
-          name='test_function',
-          **self._function_kwargs)
-      setattr(self, fn_name, fn)
+      if not isinstance(K.symbolic_learning_phase(), int):
+        inputs += [K.symbolic_learning_phase()]
+
+      with K.name_scope('evaluation'):
+        updates = self.state_updates
+        # Add stateful metrics updates.
+        if metric_updates is not None:
+          updates += metric_updates
+        # Return loss and metrics, no gradient updates.
+        # Does update the network states.
+        fn = K.function(
+            inputs,
+            outputs,
+            updates=updates,
+            name='test_function',
+            **self._function_kwargs)
+        setattr(self, fn_name, fn)
 
   def _make_test_function(self):
     self._make_test_function_helper('test_function',
@@ -794,19 +832,20 @@ class Model(Network):
     if not hasattr(self, 'predict_function'):
       self.predict_function = None
     if self.predict_function is None:
-      if not isinstance(K.learning_phase(), int):
-        inputs = self._feed_inputs + [K.learning_phase()]
+      if not isinstance(K.symbolic_learning_phase(), int):
+        inputs = self._feed_inputs + [K.symbolic_learning_phase()]
       else:
         inputs = self._feed_inputs
       # Gets network outputs. Does not update weights.
       # Does update the network states.
       kwargs = getattr(self, '_function_kwargs', {})
-      self.predict_function = K.function(
-          inputs,
-          self.outputs,
-          updates=self.state_updates,
-          name='predict_function',
-          **kwargs)
+      with K.name_scope('predict'):
+        self.predict_function = K.function(
+            inputs,
+            self.outputs,
+            updates=self.state_updates,
+            name='predict_function',
+            **kwargs)
 
   def _get_execution_function(self, mode):
     if mode == 'train':
@@ -1043,6 +1082,9 @@ class Model(Network):
     is_x_eager_iterator = isinstance(x, iterator_ops.EagerIterator)
     is_x_iterator = isinstance(x, iterator_ops.Iterator)
 
+    if is_x_eager_iterator:
+      self._run_eagerly = True  # TODO(fchollet): support using graph functions
+
     # Validate user inputs when data is given as a dataset or dataset iterator.
     if is_x_iterator or is_x_eager_iterator:
       training_utils.validate_iterator_input(x, y, sample_weight,
@@ -1051,7 +1093,7 @@ class Model(Network):
     # For eager iterators, when we have to process multiple batches of samples,
     # we will standardize the data when we actually loop over iterator and get
     # the batches. For now, we just return the iterator as is.
-    if is_x_eager_iterator and steps is not None:
+    if is_x_eager_iterator:
       return x, y, sample_weight
 
     # If input data is a dataset iterator in graph mode or if it is an eager
@@ -1157,7 +1199,7 @@ class Model(Network):
                              'TensorFlow tensors. '
                              'You passed: x=' + str(x) + '; y=' + str(y))
 
-        if context.executing_eagerly():
+        if self.run_eagerly:
           target_tensors = None
         else:
           # Handle target tensors if any passed.
@@ -1169,7 +1211,8 @@ class Model(Network):
                      loss=self.loss,
                      metrics=self.metrics,
                      loss_weights=self.loss_weights,
-                     target_tensors=target_tensors)
+                     target_tensors=target_tensors,
+                     run_eagerly=self.run_eagerly)
 
     # In graph mode, if we had just set inputs and targets as symbolic tensors
     # by invoking build and compile on the model respectively, we do not have to
@@ -1177,7 +1220,7 @@ class Model(Network):
     # part of the graph.
     # Note: in this case, `any` and `all` are equivalent since we disallow
     # mixed symbolic/value inputs.
-    if (not context.executing_eagerly() and is_build_called and
+    if (not self.run_eagerly and is_build_called and
         is_compile_called and
         any(tensor_util.is_tensor(v) for v in all_inputs)):
       return [], [], []
@@ -1185,7 +1228,7 @@ class Model(Network):
     # What follows is input validation and standardization to list format,
     # in the case where all inputs are value arrays.
 
-    if context.executing_eagerly():
+    if self.run_eagerly:
       # In eager mode, do not do shape validation
       # since the network has no input nodes (placeholders) to be fed.
       feed_input_names = self.input_names
@@ -1259,7 +1302,7 @@ class Model(Network):
       # Check that all arrays have the same length.
       if not self._distribution_strategy:
         training_utils.check_array_lengths(x, y, sample_weights)
-        if self._is_graph_network and not context.executing_eagerly():
+        if self._is_graph_network and not self.run_eagerly:
           # Additional checks to avoid users mistakenly using improper loss fns.
           training_utils.check_loss_and_target_compatibility(
               y, self._feed_loss_fns, feed_output_shapes)
@@ -1347,8 +1390,7 @@ class Model(Network):
 
     if outputs is None:
       # Obtain symbolic outputs by calling the model.
-      graph = K.get_graph()
-      with graph.as_default():
+      with K.get_graph().as_default():
         if self._expects_training_arg:
           outputs = self.call(inputs, training=training)
         else:
@@ -1508,7 +1550,6 @@ class Model(Network):
     """
     # TODO(fchollet): this method may be creating reference cycles, which would
     # lead to accumulating garbage in memory when called in a loop. Investigate.
-
     if data_utils.is_generator_or_sequence(x):
       training_utils.check_generator_arguments(y, sample_weight)
       return self.fit_generator(
@@ -1621,7 +1662,7 @@ class Model(Network):
       val_y = None
       val_sample_weights = None
 
-    if context.executing_eagerly():
+    if self.run_eagerly:
       return training_eager.fit_loop(
           self,
           inputs=x,
@@ -1779,7 +1820,7 @@ class Model(Network):
         steps_name='steps',
         steps=steps)
 
-    if context.executing_eagerly():
+    if self.run_eagerly:
       return training_eager.test_loop(
           self,
           inputs=x,
@@ -1879,14 +1920,13 @@ class Model(Network):
       if isinstance(first_x_value, np.ndarray) and not steps:
         steps = distributed_training_utils.get_input_batch_params(
             first_x_value, batch_size, self._distribution_strategy)
-
     # Validate and standardize user data.
     # TODO(anjalisridhar): We don't pass batch_size here for some reason. This
     # means that we end up calculating it twice which we should avoid.
     x, _, _ = self._standardize_user_data(
         x, check_steps=True, steps_name='steps', steps=steps)
 
-    if context.executing_eagerly():
+    if self.run_eagerly:
       return training_eager.predict_loop(
           self, x, batch_size=batch_size, verbose=verbose, steps=steps)
     elif self._distribution_strategy:
@@ -1943,12 +1983,12 @@ class Model(Network):
     x, y, sample_weights = self._standardize_user_data(
         x, y, sample_weight=sample_weight, class_weight=class_weight)
 
-    if context.executing_eagerly():
+    if self.run_eagerly:
       outputs = training_eager.train_on_batch(
           self, x, y, sample_weights=sample_weights)
     else:
-      if not isinstance(K.learning_phase(), int):
-        ins = x + y + sample_weights + [1]
+      if not isinstance(K.symbolic_learning_phase(), int):
+        ins = x + y + sample_weights + [True]
       else:
         ins = x + y + sample_weights
 
@@ -2002,12 +2042,12 @@ class Model(Network):
     x, y, sample_weights = self._standardize_user_data(
         x, y, sample_weight=sample_weight)
 
-    if context.executing_eagerly():
+    if self.run_eagerly:
       outputs = training_eager.test_on_batch(
           self, x, y, sample_weights=sample_weights)
     else:
-      if not isinstance(K.learning_phase(), int):
-        ins = x + y + sample_weights + [0]
+      if not isinstance(K.symbolic_learning_phase(), int):
+        ins = x + y + sample_weights + [False]
       else:
         ins = x + y + sample_weights
       self._make_test_function()
@@ -2040,9 +2080,9 @@ class Model(Network):
                                 'models compiled with DistributionStrategy.')
     # Validate and standardize user data.
     inputs, _, _ = self._standardize_user_data(x)
-    if context.executing_eagerly():
+    if self.run_eagerly:
       if (isinstance(x, iterator_ops.EagerIterator) or
-          (isinstance(x, dataset_ops.Dataset) and context.executing_eagerly())):
+          (isinstance(x, dataset_ops.Dataset))):
         inputs = training_utils.cast_if_floating_dtype(inputs)
       else:
         inputs = [
@@ -2050,18 +2090,17 @@ class Model(Network):
         ]
       return self(inputs)  # pylint: disable=not-callable
 
-    if not context.executing_eagerly():
-      if not isinstance(K.learning_phase(), int):
-        ins = inputs + [0]
-      else:
-        ins = inputs
+    if not isinstance(K.symbolic_learning_phase(), int):
+      ins = inputs + [False]
+    else:
+      ins = inputs
 
-      self._make_predict_function()
-      outputs = self.predict_function(ins)
+    self._make_predict_function()
+    outputs = self.predict_function(ins)
 
-      if len(outputs) == 1:
-        return outputs[0]
-      return outputs
+    if len(outputs) == 1:
+      return outputs[0]
+    return outputs
 
   def fit_generator(self,
                     generator,
@@ -2171,11 +2210,6 @@ class Model(Network):
     if self._distribution_strategy:
       raise NotImplementedError('`fit_generator` is not supported for '
                                 'models compiled with DistributionStrategy.')
-
-    if not self.built and not self._is_graph_network:
-      raise NotImplementedError(
-          '`fit_generator` is not yet enabled for unbuilt Model subclasses')
-
     return training_generator.fit_generator(
         self,
         generator,
@@ -2242,12 +2276,6 @@ class Model(Network):
     if self._distribution_strategy:
       raise NotImplementedError('`evaluate_generator` is not supported for '
                                 'models compiled with DistributionStrategy.')
-
-    if not self.built and not self._is_graph_network:
-      raise NotImplementedError(
-          '`evaluate_generator` is not yet enabled for '
-          'unbuilt Model subclasses')
-
     return training_generator.evaluate_generator(
         self,
         generator,
@@ -2299,11 +2327,6 @@ class Model(Network):
     if self._distribution_strategy:
       raise NotImplementedError('`predict_generator` is not supported for '
                                 'models compiled with DistributionStrategy.')
-
-    if not self.built and not self._is_graph_network:
-      raise NotImplementedError(
-          '`predict_generator` is not yet enabled for unbuilt Model subclasses')
-
     return training_generator.predict_generator(
         self,
         generator,
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index a2a13b9bd6..9f3f3c7c24 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -248,8 +248,8 @@ def model_iteration(model,
   targets = targets or []
   sample_weights = sample_weights or []
   learning_phase_input = []
-  if not isinstance(K.learning_phase(), int):
-    learning_phase_input = [1] if mode == 'train' else [0]
+  if not isinstance(K.symbolic_learning_phase(), int):
+    learning_phase_input = [True] if mode == 'train' else [False]
   ins = inputs + targets + sample_weights + learning_phase_input
   num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                    steps_per_epoch)
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index 9131df5cd0..9cb752a5c6 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -20,12 +20,10 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
-import threading
 
 import numpy as np
 
 from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.eager import function as eager_function
 from tensorflow.python.eager.backprop import GradientTape
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -39,10 +37,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 
 
-# A lock for assigning polymorphic functions to models in a thread-safe way
-_graph_function_building_lock = threading.Lock()
-
-
 def _eager_loss_fn(outputs, targets, loss_fn, output_name):
   with backend.name_scope(output_name + '_loss'):
     loss = loss_fn(targets, outputs)
@@ -171,56 +165,6 @@ def _model_loss(model,
   return outs, total_loss, loss_metrics, aggregated_loss_metrics, masks
 
 
-def _maybe_build_graph_functions(model):
-  """Constructs polymorphic functions to use for fit, evaluate and predict."""
-  # We lock this function to ensure thread-safety in case users are
-  # hypothetically trying to call '.predict' on a model in multiple threads
-  # at once when the graph functions were never previously built.
-  with _graph_function_building_lock:
-    if not model._built_graph_functions:
-      model._eager_process_single_batch_graph_function = eager_function.defun(
-          _process_single_batch
-      )
-      model._eager_model_loss_graph_function = eager_function.defun(_model_loss)
-      model._eager_call_graph_function = eager_function.defun(model.call)
-      model._built_graph_functions = True
-
-
-def _maybe_graph_function_model_loss(model,
-                                     inputs,
-                                     targets,
-                                     output_loss_metrics=None,
-                                     sample_weights=None,
-                                     training=False):
-  """Compute model loss, using defun if the model supports it."""
-  if model._can_use_graph_functions:
-    _maybe_build_graph_functions(model)
-    return model._eager_model_loss_graph_function(
-        model,
-        inputs,
-        targets,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=training)
-  else:
-    return _model_loss(
-        model,
-        inputs,
-        targets,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=training)
-
-
-def _maybe_graph_function_model_call(model, *args, **kwargs):
-  """Compute model loss, using defun if the model supports it."""
-  if model._can_use_graph_functions:
-    _maybe_build_graph_functions(model)
-    return model._eager_call_graph_function(*args, **kwargs)
-  else:
-    return model.call(*args, **kwargs)
-
-
 def iterator_fit_loop(model,
                       inputs,
                       class_weight,
@@ -334,14 +278,13 @@ def iterator_fit_loop(model,
       })
 
     # Train model.
-    outs, loss, _, aggregated_loss_metrics, masks = \
-      _maybe_graph_function_process_single_batch(
-          model,
-          x,
-          y,
-          output_loss_metrics=output_loss_metrics,
-          sample_weights=sample_weights,
-          training=True)
+    outs, loss, _, aggregated_loss_metrics, masks = _process_single_batch(
+        model,
+        x,
+        y,
+        output_loss_metrics=output_loss_metrics,
+        sample_weights=sample_weights,
+        training=True)
     outs = generic_utils.to_list(outs)
 
     # Calculate metrics.
@@ -456,14 +399,13 @@ def iterator_test_loop(model, inputs, steps, verbose=0):
         m.reset_states()
 
     # Calculate model output, loss values.
-    loss_outs, loss, _, aggregated_loss_metrics, masks = \
-      _maybe_graph_function_model_loss(
-          model,
-          x,
-          y,
-          output_loss_metrics=output_loss_metrics,
-          sample_weights=sample_weights,
-          training=False)
+    loss_outs, loss, _, aggregated_loss_metrics, masks = _model_loss(
+        model,
+        x,
+        y,
+        output_loss_metrics=output_loss_metrics,
+        sample_weights=sample_weights,
+        training=False)
     metrics_results = _eager_metrics_fn(
         model, loss_outs, y, sample_weights=sample_weights, masks=masks)
     batch_outs = []
@@ -553,9 +495,9 @@ def iterator_predict_loop(model, inputs, steps, verbose=0):
       x = x[0]
 
     if model._expects_training_arg:
-      batch_outs = _maybe_graph_function_model_call(model, x, training=False)
+      batch_outs = model.call(x, training=False)
     else:
-      batch_outs = _maybe_graph_function_model_call(model, x)
+      batch_outs = model.call(x)
     if not isinstance(batch_outs, list):
       batch_outs = [batch_outs]
 
@@ -630,32 +572,6 @@ def _process_single_batch(model,
     return outs, loss, loss_metrics, aggregated_loss_metrics, masks
 
 
-def _maybe_graph_function_process_single_batch(model,
-                                               inputs,
-                                               targets,
-                                               output_loss_metrics=None,
-                                               sample_weights=None,
-                                               training=False):
-  """Process a single batch, using defun if the model supports it."""
-  if model._can_use_graph_functions:
-    _maybe_build_graph_functions(model)
-    return model._eager_process_single_batch_graph_function(
-        model,
-        inputs,
-        targets,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=training)
-  else:
-    return _process_single_batch(
-        model,
-        inputs,
-        targets,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=training)
-
-
 def train_on_batch(model, inputs, targets, sample_weights=None):
   """Calculates the loss and gradient updates for one input batch.
 
@@ -684,9 +600,8 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
         if val is not None else None for val in sample_weights
     ]
 
-  outs, loss, loss_metrics, _, masks = \
-    _maybe_graph_function_process_single_batch(
-        model, inputs, targets, sample_weights=sample_weights, training=True)
+  outs, loss, loss_metrics, _, masks = _process_single_batch(
+      model, inputs, targets, sample_weights=sample_weights, training=True)
   if not isinstance(outs, list):
     outs = [outs]
   metrics_results = _eager_metrics_fn(
@@ -731,7 +646,7 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
         ops.convert_to_tensor(val, dtype=backend.floatx())
         if val is not None else None for val in sample_weights
     ]
-  outs, loss, loss_metrics, _, masks = _maybe_graph_function_model_loss(
+  outs, loss, loss_metrics, _, masks = _model_loss(
       model, inputs, targets, sample_weights=sample_weights, training=False)
   if not isinstance(outs, list):
     outs = [outs]
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index 76aaf1643b..bd25af9ec6 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -20,10 +20,9 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.platform import test
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
@@ -51,6 +50,7 @@ class TrainingTest(test.TestCase):
         loss,
         metrics=metrics,
         loss_weights=loss_weights,
+        run_eagerly=True,
         sample_weight_mode=None)
 
     input_a = keras.backend.zeros(shape=(10, 3))
@@ -111,7 +111,7 @@ class TrainingTest(test.TestCase):
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics, run_eagerly=True)
 
     inputs = keras.backend.zeros(shape=(10, 3))
     targets = keras.backend.zeros(shape=(10, 4))
@@ -129,7 +129,9 @@ class TrainingTest(test.TestCase):
     x = keras.layers.Input(shape=(3,), name='input')
     y = keras.layers.Dense(4, name='dense')(x)
     model = keras.Model(x, y)
-    model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse')
+    model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001),
+                  loss='mse',
+                  run_eagerly=True)
 
     x = keras.backend.zeros(shape=(10, 3))
     y = keras.backend.zeros(shape=(10, 4))
@@ -147,11 +149,11 @@ class TrainingTest(test.TestCase):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=(x, y))
     with self.assertRaisesRegexp(
-        ValueError, r'provide either `batch_size` or `validation_steps`'):
+        ValueError, r'must specify the number of steps'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=validation_dataset)
     with self.assertRaisesRegexp(
-        ValueError, r'provide either `batch_size` or `validation_steps`'):
+        ValueError, r'must specify the number of steps'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=validation_iterator)
 
@@ -160,8 +162,10 @@ class TrainingTest(test.TestCase):
     model.add(keras.layers.Dense(4, input_shape=(3,)))
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     model.compile(
-        optimizer, 'mse', metrics=['mae',
-                                   metrics_module.CategoricalAccuracy()])
+        optimizer,
+        loss='mse',
+        metrics=['mae', metrics_module.CategoricalAccuracy()],
+        run_eagerly=True)
 
     x = np.random.random((10, 3))
     y = np.random.random((10, 4))
@@ -178,7 +182,6 @@ class TrainingTest(test.TestCase):
 
 class CorrectnessTest(test.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_loss_correctness(self):
     # Test that training loss is the same in eager and graph
     # (by comparing it to a reference value in a deterministic case)
@@ -191,14 +194,14 @@ class CorrectnessTest(test.TestCase):
                                  activation='softmax',
                                  kernel_initializer='ones'))
     model.compile(loss='sparse_categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001))
+                  optimizer=RMSPropOptimizer(learning_rate=0.001),
+                  run_eagerly=False)
     x = np.ones((100, 4))
     np.random.seed(123)
     y = np.random.randint(0, 1, size=(100, 1))
     history = model.fit(x, y, epochs=1, batch_size=10)
     self.assertAlmostEqual(history.history['loss'][-1], 0.6173, 4)
 
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_loss_correctness_with_iterator(self):
     # Test that training loss is the same in eager and graph
     # (by comparing it to a reference value in a deterministic case)
@@ -210,7 +213,8 @@ class CorrectnessTest(test.TestCase):
         keras.layers.Dense(2, activation='softmax', kernel_initializer='ones'))
     model.compile(
         loss='sparse_categorical_crossentropy',
-        optimizer=RMSPropOptimizer(learning_rate=0.001))
+        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        run_eagerly=True)
     x = np.ones((100, 4), dtype=np.float32)
     np.random.seed(123)
     y = np.random.randint(0, 1, size=(100, 1))
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 3cb24255d1..8ba19859d0 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -1794,9 +1794,6 @@ class TestTrainingWithDatasetIterators(test.TestCase):
     model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
     model.evaluate(iterator, steps=2, verbose=1)
     model.predict(iterator, steps=2)
-    model.train_on_batch(iterator)
-    model.test_on_batch(iterator)
-    model.predict_on_batch(iterator)
 
     # Test with validation data
     model.fit(iterator,
@@ -1923,8 +1920,6 @@ class TestTrainingWithDataset(test.TestCase):
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
     model.evaluate(dataset, steps=2, verbose=1)
     model.predict(dataset, steps=2)
-    model.train_on_batch(dataset)
-    model.predict_on_batch(dataset)
 
     # Test with validation data
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
@@ -1985,8 +1980,6 @@ class TestTrainingWithDataset(test.TestCase):
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
     model.evaluate(dataset, steps=2, verbose=1)
     model.predict(dataset, steps=2)
-    model.train_on_batch(dataset)
-    model.predict_on_batch(dataset)
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_dataset_with_sparse_labels(self):
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index e563b7a23d..61d40d1caa 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -138,6 +138,9 @@ def convert_to_iterator(x=None,
 
   """
   if isinstance(x, iterator_ops.EagerIterator):
+    if steps_per_epoch is None:
+      raise ValueError('You must specify the number of steps (number of batches'
+                       'to draw from the iterator).')
     return x, steps_per_epoch
 
   if not _nested_any(sample_weights, lambda x: x is None):
@@ -224,9 +227,9 @@ def standardize_single_array(x):
     return None
   if x.shape is not None and len(x.shape) == 1:
     if tensor_util.is_tensor(x):
-      return array_ops.expand_dims(x, axis=1)
+      x = array_ops.expand_dims(x, axis=1)
     else:
-      return np.expand_dims(x, 1)
+      x = np.expand_dims(x, 1)
   return x
 
 
@@ -1053,9 +1056,11 @@ class ModelInputs(object):
     self._inputs = inputs
     self._is_dict = isinstance(self._inputs, dict)
     self._is_single_input = not isinstance(self._inputs, (list, tuple, dict))
+
     self._flattened_inputs = []
     self._input_names = []
-    if isinstance(self._inputs, dict):
+
+    if self._is_dict:
       for k in sorted(self._inputs.keys()):
         self._flattened_inputs.append(self._inputs[k])
         self._input_names.append(k)
@@ -1064,7 +1069,6 @@ class ModelInputs(object):
       self._input_names = [
           'input_%d' % (i + 1) for i in range(len(self._flattened_inputs))
       ]
-    assert len(self._input_names) == len(self._flattened_inputs)
 
   def get_input_names(self):
     """Returns keys to name inputs by.
@@ -1074,56 +1078,29 @@ class ModelInputs(object):
     """
     return self._input_names
 
-  def _get(self, return_single_as_list=False):
-    """Returns provided inputs, potentially transformed.
-
-    Inputs are returned in the same format they were provided i.e. lists
-    are returned as lists, single entries as single entries (unless
-    `return_single_as_list` is true), dictionaries as dictionaries.
-
-    Args:
-      return_single_as_list: Returns a list of size 1 for single entry case.
-    """
-    if self._is_dict:
-      return dict(zip(self._input_names, self._flattened_inputs))
-    if self._is_single_input and not return_single_as_list:
-      return self._flattened_inputs[0]
-    return self._flattened_inputs
-
-  def get_input_values(self):
-    """Returns input values passed in."""
-    if context.executing_eagerly():
-      for i in range(len(self._flattened_inputs)):
-        v = self._flattened_inputs[i]
-        if tensor_util.is_tensor(v):
-          v = cast_single_tensor(v)
-        else:
-          v = ops.convert_to_tensor(v, dtype=K.floatx())
-        self._flattened_inputs[i] = v
-    return self._get(return_single_as_list=False)
-
   def get_symbolic_inputs(self, return_single_as_list=False):
     """Returns inputs to be set as self.inputs for a model."""
     for i in range(len(self._flattened_inputs)):
       k = self._input_names[i]
       v = self._flattened_inputs[i]
-      if context.executing_eagerly():
-        v = K.placeholder((None,) + tuple(v.shape[1:]), name=k)
-      else:
-        if isinstance(v, list):
-          v = np.asarray(v)
-          if v.ndim == 1:
-            v = np.expand_dims(v, 1)
-        if isinstance(v, (np.ndarray)):
-          # We fix the placeholder shape except the batch size.
-          # This is suboptimal, but it is the best we can do with the info
-          # we have. The user should call `model._set_inputs(placeholders)`
-          # to specify custom placeholders if the need arises.
-          shape = (None,) + v.shape[1:]
-          v = K.placeholder(shape=shape, name=k)
+      if isinstance(v, (list, float, int)):
+        v = np.asarray(v)
+        if v.ndim == 1:
+          v = np.expand_dims(v, 1)
+      if isinstance(v, (np.ndarray, ops.EagerTensor)):
+        # We fix the placeholder shape except the batch size.
+        # This is suboptimal, but it is the best we can do with the info
+        # we have. The user should call `model._set_inputs(placeholders)`
+        # to specify custom placeholders if the need arises.
+        shape = (None,) + tuple(v.shape[1:])
+        v = K.placeholder(shape=shape, name=k)
       self._flattened_inputs[i] = v
 
-    return self._get(return_single_as_list)
+    if self._is_dict:
+      return dict(zip(self._input_names, self._flattened_inputs))
+    if self._is_single_input and not return_single_as_list:
+      return self._flattened_inputs[0]
+    return self._flattened_inputs
 
   def as_dict(self):
     """An iterable over a dictionary version of inputs."""
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index 7b217cf373..d6a92dec7c 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -167,9 +167,6 @@ class ModelInputsTest(test.TestCase):
     a = np.ones(10)
     model_inputs = training_utils.ModelInputs(a)
     self.assertEquals(['input_1'], model_inputs.get_input_names())
-    vals = model_inputs.get_input_values()
-    self.assertAllEqual(np.ones(10), vals)
-    self.assertFalse(tensor_util.is_tensor(vals))
     vals = model_inputs.get_symbolic_inputs()
     self.assertTrue(tensor_util.is_tensor(vals))
     vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
@@ -181,9 +178,6 @@ class ModelInputsTest(test.TestCase):
       a = np.ones(10)
       model_inputs = training_utils.ModelInputs(a)
       self.assertEquals(['input_1'], model_inputs.get_input_names())
-      val = model_inputs.get_input_values()
-      self.assertAllEqual(np.ones(10), val)
-      self.assertTrue(tensor_util.is_tensor(val))
       val = model_inputs.get_symbolic_inputs()
       self.assertTrue(tf_utils.is_symbolic_tensor(val))
       vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
@@ -194,12 +188,6 @@ class ModelInputsTest(test.TestCase):
     a = [np.ones(10), np.ones(20)]
     model_inputs = training_utils.ModelInputs(a)
     self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names())
-    vals = model_inputs.get_input_values()
-    self.assertEqual(2, len(vals))
-    self.assertAllEqual(np.ones(10), vals[0])
-    self.assertAllEqual(np.ones(20), vals[1])
-    self.assertFalse(tensor_util.is_tensor(vals[0]))
-    self.assertFalse(tensor_util.is_tensor(vals[1]))
     vals = model_inputs.get_symbolic_inputs()
     self.assertTrue(tensor_util.is_tensor(vals[0]))
     self.assertTrue(tensor_util.is_tensor(vals[1]))
@@ -209,12 +197,6 @@ class ModelInputsTest(test.TestCase):
       a = [np.ones(10), np.ones(20)]
       model_inputs = training_utils.ModelInputs(a)
       self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names())
-      vals = model_inputs.get_input_values()
-      self.assertEqual(2, len(vals))
-      self.assertAllEqual(np.ones(10), vals[0])
-      self.assertAllEqual(np.ones(20), vals[1])
-      self.assertTrue(tensor_util.is_tensor(vals[0]))
-      self.assertTrue(tensor_util.is_tensor(vals[1]))
       vals = model_inputs.get_symbolic_inputs()
       self.assertTrue(tf_utils.is_symbolic_tensor(vals[0]))
       self.assertTrue(tf_utils.is_symbolic_tensor(vals[1]))
@@ -223,11 +205,6 @@ class ModelInputsTest(test.TestCase):
     a = {'b': np.ones(10), 'a': np.ones(20)}
     model_inputs = training_utils.ModelInputs(a)
     self.assertEquals(['a', 'b'], model_inputs.get_input_names())
-    vals = model_inputs.get_input_values()
-    self.assertAllEqual(np.ones(20), vals['a'])
-    self.assertAllEqual(np.ones(10), vals['b'])
-    self.assertFalse(tensor_util.is_tensor(vals['a']))
-    self.assertFalse(tensor_util.is_tensor(vals['b']))
     vals = model_inputs.get_symbolic_inputs()
     self.assertTrue(tensor_util.is_tensor(vals['a']))
     self.assertTrue(tensor_util.is_tensor(vals['b']))
@@ -237,11 +214,6 @@ class ModelInputsTest(test.TestCase):
       a = {'b': np.ones(10), 'a': np.ones(20)}
       model_inputs = training_utils.ModelInputs(a)
       self.assertEquals(['a', 'b'], model_inputs.get_input_names())
-      vals = model_inputs.get_input_values()
-      self.assertAllEqual(np.ones(20), vals['a'])
-      self.assertAllEqual(np.ones(10), vals['b'])
-      self.assertTrue(tensor_util.is_tensor(vals['a']))
-      self.assertTrue(tensor_util.is_tensor(vals['b']))
       vals = model_inputs.get_symbolic_inputs()
       self.assertTrue(tf_utils.is_symbolic_tensor(vals['a']))
       self.assertTrue(tf_utils.is_symbolic_tensor(vals['b']))
diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py
index b0dffced3e..a2385dfdbb 100644
--- a/tensorflow/python/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/layers/advanced_activations.py
@@ -54,7 +54,6 @@ class LeakyReLU(Layer):
     super(LeakyReLU, self).__init__(**kwargs)
     self.supports_masking = True
     self.alpha = K.cast_to_floatx(alpha)
-    self._can_use_graph_functions = True
 
   def call(self, inputs):
     return K.relu(inputs, alpha=self.alpha)
@@ -118,7 +117,6 @@ class PReLU(Layer):
       self.shared_axes = [shared_axes]
     else:
       self.shared_axes = list(shared_axes)
-    self._can_use_graph_functions = True
 
   @tf_utils.shape_type_conversion
   def build(self, input_shape):
@@ -193,7 +191,6 @@ class ELU(Layer):
     super(ELU, self).__init__(**kwargs)
     self.supports_masking = True
     self.alpha = K.cast_to_floatx(alpha)
-    self._can_use_graph_functions = True
 
   def call(self, inputs):
     return K.elu(inputs, self.alpha)
@@ -233,7 +230,6 @@ class ThresholdedReLU(Layer):
     super(ThresholdedReLU, self).__init__(**kwargs)
     self.supports_masking = True
     self.theta = K.cast_to_floatx(theta)
-    self._can_use_graph_functions = True
 
   def call(self, inputs, mask=None):
     return inputs * math_ops.cast(
@@ -269,7 +265,6 @@ class Softmax(Layer):
     super(Softmax, self).__init__(**kwargs)
     self.supports_masking = True
     self.axis = axis
-    self._can_use_graph_functions = True
 
   def call(self, inputs):
     return K.softmax(inputs, axis=self.axis)
@@ -324,7 +319,6 @@ class ReLU(Layer):
     self.max_value = max_value
     self.negative_slope = K.cast_to_floatx(negative_slope)
     self.threshold = K.cast_to_floatx(threshold)
-    self._can_use_graph_functions = True
 
   def call(self, inputs):
     # alpha is used for leaky relu slope in activations instead of
diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py
index 0671a5a36d..d1b03b8822 100644
--- a/tensorflow/python/keras/layers/convolutional.py
+++ b/tensorflow/python/keras/layers/convolutional.py
@@ -120,7 +120,6 @@ class Conv(Layer):
         name=name,
         activity_regularizer=regularizers.get(activity_regularizer),
         **kwargs)
-    self._can_use_graph_functions = True
     self.rank = rank
     self.filters = filters
     self.kernel_size = conv_utils.normalize_tuple(
@@ -1916,7 +1915,6 @@ class UpSampling1D(Layer):
     super(UpSampling1D, self).__init__(**kwargs)
     self.size = int(size)
     self.input_spec = InputSpec(ndim=3)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -1983,7 +1981,6 @@ class UpSampling2D(Layer):
                        'or `"bilinear"`.')
     self.interpolation = interpolation
     self.input_spec = InputSpec(ndim=4)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -2054,7 +2051,6 @@ class UpSampling3D(Layer):
     self.size = conv_utils.normalize_tuple(size, 3, 'size')
     self.input_spec = InputSpec(ndim=5)
     super(UpSampling3D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -2109,7 +2105,6 @@ class ZeroPadding1D(Layer):
 
   def __init__(self, padding=1, **kwargs):
     super(ZeroPadding1D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.padding = conv_utils.normalize_tuple(padding, 2, 'padding')
     self.input_spec = InputSpec(ndim=3)
 
@@ -2175,7 +2170,6 @@ class ZeroPadding2D(Layer):
 
   def __init__(self, padding=(1, 1), data_format=None, **kwargs):
     super(ZeroPadding2D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.data_format = conv_utils.normalize_data_format(data_format)
     if isinstance(padding, int):
       self.padding = ((padding, padding), (padding, padding))
@@ -2280,7 +2274,6 @@ class ZeroPadding3D(Layer):
 
   def __init__(self, padding=(1, 1, 1), data_format=None, **kwargs):
     super(ZeroPadding3D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.data_format = conv_utils.normalize_data_format(data_format)
     if isinstance(padding, int):
       self.padding = ((padding, padding), (padding, padding), (padding,
@@ -2375,7 +2368,6 @@ class Cropping1D(Layer):
     super(Cropping1D, self).__init__(**kwargs)
     self.cropping = conv_utils.normalize_tuple(cropping, 2, 'cropping')
     self.input_spec = InputSpec(ndim=3)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -2475,7 +2467,6 @@ class Cropping2D(Layer):
                        '((top_crop, bottom_crop), (left_crop, right_crop)). '
                        'Found: ' + str(cropping))
     self.input_spec = InputSpec(ndim=4)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -2609,7 +2600,6 @@ class Cropping3D(Layer):
           ' (left_dim3_crop, right_dim2_crop)). '
           'Found: ' + str(cropping))
     self.input_spec = InputSpec(ndim=5)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index e5c37be0aa..8031272e25 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -81,7 +81,6 @@ class Masking(Layer):
     super(Masking, self).__init__(**kwargs)
     self.supports_masking = True
     self.mask_value = mask_value
-    self._can_use_graph_functions = True
 
   def compute_mask(self, inputs, mask=None):
     return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1)
@@ -125,7 +124,6 @@ class Dropout(Layer):
     self.noise_shape = noise_shape
     self.seed = seed
     self.supports_masking = True
-    self._can_use_graph_functions = True
 
   def _get_noise_shape(self, inputs):
     # Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`,
@@ -326,7 +324,6 @@ class Activation(Layer):
     super(Activation, self).__init__(**kwargs)
     self.supports_masking = True
     self.activation = activations.get(activation)
-    self._can_use_graph_functions = True
 
   def call(self, inputs):
     return self.activation(inputs)
@@ -379,7 +376,6 @@ class Reshape(Layer):
   def __init__(self, target_shape, **kwargs):
     super(Reshape, self).__init__(**kwargs)
     self.target_shape = tuple(target_shape)
-    self._can_use_graph_functions = True
 
   def _fix_unknown_dimension(self, input_shape, output_shape):
     """Find and replace a missing dimension in an output shape.
@@ -488,7 +484,6 @@ class Permute(Layer):
           'The set of indices in `dims` must be consecutive and start from 1.' %
           (dims,))
     self.input_spec = InputSpec(ndim=len(self.dims) + 1)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -540,7 +535,6 @@ class Flatten(Layer):
     super(Flatten, self).__init__(**kwargs)
     self.data_format = conv_utils.normalize_data_format(data_format)
     self.input_spec = InputSpec(min_ndim=2)
-    self._can_use_graph_functions = True
 
   def call(self, inputs):
     if self.data_format == 'channels_first':
@@ -600,7 +594,6 @@ class RepeatVector(Layer):
     super(RepeatVector, self).__init__(**kwargs)
     self.n = n
     self.input_spec = InputSpec(ndim=2)
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
@@ -929,7 +922,6 @@ class Dense(Layer):
 
     self.supports_masking = True
     self.input_spec = InputSpec(min_ndim=2)
-    self._can_use_graph_functions = True
 
   def build(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape)
@@ -1029,7 +1021,6 @@ class ActivityRegularization(Layer):
     self.supports_masking = True
     self.l1 = l1
     self.l2 = l2
-    self._can_use_graph_functions = True
 
   def compute_output_shape(self, input_shape):
     return input_shape
diff --git a/tensorflow/python/keras/layers/embeddings.py b/tensorflow/python/keras/layers/embeddings.py
index 5d805ea684..28d8ef252a 100644
--- a/tensorflow/python/keras/layers/embeddings.py
+++ b/tensorflow/python/keras/layers/embeddings.py
@@ -116,7 +116,6 @@ class Embedding(Layer):
     self.mask_zero = mask_zero
     self.supports_masking = mask_zero
     self.input_length = input_length
-    self._can_use_graph_functions = True
 
   @tf_utils.shape_type_conversion
   def build(self, input_shape):
diff --git a/tensorflow/python/keras/layers/local.py b/tensorflow/python/keras/layers/local.py
index 30b83eaf50..33d09a1660 100644
--- a/tensorflow/python/keras/layers/local.py
+++ b/tensorflow/python/keras/layers/local.py
@@ -154,7 +154,6 @@ class LocallyConnected1D(Layer):
     self.bias_constraint = constraints.get(bias_constraint)
     self.implementation = implementation
     self.input_spec = InputSpec(ndim=3)
-    self._can_use_graph_functions = True
 
   @tf_utils.shape_type_conversion
   def build(self, input_shape):
@@ -430,7 +429,6 @@ class LocallyConnected2D(Layer):
     self.bias_constraint = constraints.get(bias_constraint)
     self.implementation = implementation
     self.input_spec = InputSpec(ndim=4)
-    self._can_use_graph_functions = True
 
   @tf_utils.shape_type_conversion
   def build(self, input_shape):
diff --git a/tensorflow/python/keras/layers/local_test.py b/tensorflow/python/keras/layers/local_test.py
index 300e7c9654..2397a607da 100644
--- a/tensorflow/python/keras/layers/local_test.py
+++ b/tensorflow/python/keras/layers/local_test.py
@@ -28,39 +28,42 @@ from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 class LocallyConnected1DLayersTest(test.TestCase):
+  # TODO(fchollet): investigate why LocallyConnected1D
+  # fails inside a graph function in an eager context (fails with error
+  # "Incompatible shapes between op input and calculated input gradient").
 
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_locallyconnected_1d(self):
-    num_samples = 2
-    num_steps = 8
-    input_dim = 5
-    filter_length = 3
-    filters = 4
-
-    for padding in ['valid', 'same']:
-      for strides in [1]:
-        if padding == 'same' and strides != 1:
-          continue
-        for data_format in ['channels_first', 'channels_last']:
-          for implementation in [1, 2]:
-            kwargs = {
-                'filters': filters,
-                'kernel_size': filter_length,
-                'padding': padding,
-                'strides': strides,
-                'data_format': data_format,
-                'implementation': implementation
-            }
+    with self.cached_session():
+      num_samples = 2
+      num_steps = 8
+      input_dim = 5
+      filter_length = 3
+      filters = 4
 
-            if padding == 'same' and implementation == 1:
-              self.assertRaises(ValueError,
-                                keras.layers.LocallyConnected1D,
-                                **kwargs)
-            else:
-              testing_utils.layer_test(
-                  keras.layers.LocallyConnected1D,
-                  kwargs=kwargs,
-                  input_shape=(num_samples, num_steps, input_dim))
+      for padding in ['valid', 'same']:
+        for strides in [1]:
+          if padding == 'same' and strides != 1:
+            continue
+          for data_format in ['channels_first', 'channels_last']:
+            for implementation in [1, 2]:
+              kwargs = {
+                  'filters': filters,
+                  'kernel_size': filter_length,
+                  'padding': padding,
+                  'strides': strides,
+                  'data_format': data_format,
+                  'implementation': implementation
+              }
+
+              if padding == 'same' and implementation == 1:
+                self.assertRaises(ValueError,
+                                  keras.layers.LocallyConnected1D,
+                                  **kwargs)
+              else:
+                testing_utils.layer_test(
+                    keras.layers.LocallyConnected1D,
+                    kwargs=kwargs,
+                    input_shape=(num_samples, num_steps, input_dim))
 
   def test_locallyconnected_1d_regularization(self):
     num_samples = 2
@@ -113,30 +116,61 @@ class LocallyConnected1DLayersTest(test.TestCase):
 
 
 class LocallyConnected2DLayersTest(test.TestCase):
+  # TODO(fchollet): investigate why LocallyConnected2D
+  # fails inside a graph function in an eager context (fails with error
+  # "Incompatible shapes between op input and calculated input gradient").
 
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_locallyconnected_2d(self):
-    num_samples = 8
-    filters = 3
-    stack_size = 4
-    num_row = 6
-    num_col = 10
+    with self.cached_session():
+      num_samples = 8
+      filters = 3
+      stack_size = 4
+      num_row = 6
+      num_col = 10
 
-    for padding in ['valid', 'same']:
-      for strides in [(1, 1), (2, 2)]:
-        for implementation in [1, 2]:
-          if padding == 'same' and strides != (1, 1):
-            continue
+      for padding in ['valid', 'same']:
+        for strides in [(1, 1), (2, 2)]:
+          for implementation in [1, 2]:
+            if padding == 'same' and strides != (1, 1):
+              continue
+
+            kwargs = {
+                'filters': filters,
+                'kernel_size': 3,
+                'padding': padding,
+                'kernel_regularizer': 'l2',
+                'bias_regularizer': 'l2',
+                'strides': strides,
+                'data_format': 'channels_last',
+                'implementation': implementation
+            }
 
+            if padding == 'same' and implementation == 1:
+              self.assertRaises(ValueError,
+                                keras.layers.LocallyConnected2D,
+                                **kwargs)
+            else:
+              testing_utils.layer_test(
+                  keras.layers.LocallyConnected2D,
+                  kwargs=kwargs,
+                  input_shape=(num_samples, num_row, num_col, stack_size))
+
+  def test_locallyconnected_2d_channels_first(self):
+    with self.cached_session():
+      num_samples = 8
+      filters = 3
+      stack_size = 4
+      num_row = 6
+      num_col = 10
+
+      for implementation in [1, 2]:
+        for padding in ['valid', 'same']:
           kwargs = {
               'filters': filters,
               'kernel_size': 3,
-              'padding': padding,
-              'kernel_regularizer': 'l2',
-              'bias_regularizer': 'l2',
-              'strides': strides,
-              'data_format': 'channels_last',
-              'implementation': implementation
+              'data_format': 'channels_first',
+              'implementation': implementation,
+              'padding': padding
           }
 
           if padding == 'same' and implementation == 1:
@@ -149,34 +183,6 @@ class LocallyConnected2DLayersTest(test.TestCase):
                 kwargs=kwargs,
                 input_shape=(num_samples, num_row, num_col, stack_size))
 
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_locallyconnected_2d_channels_first(self):
-    num_samples = 8
-    filters = 3
-    stack_size = 4
-    num_row = 6
-    num_col = 10
-
-    for implementation in [1, 2]:
-      for padding in ['valid', 'same']:
-        kwargs = {
-            'filters': filters,
-            'kernel_size': 3,
-            'data_format': 'channels_first',
-            'implementation': implementation,
-            'padding': padding
-        }
-
-        if padding == 'same' and implementation == 1:
-          self.assertRaises(ValueError,
-                            keras.layers.LocallyConnected2D,
-                            **kwargs)
-        else:
-          testing_utils.layer_test(
-              keras.layers.LocallyConnected2D,
-              kwargs=kwargs,
-              input_shape=(num_samples, num_row, num_col, stack_size))
-
   def test_locallyconnected_2d_regularization(self):
     num_samples = 2
     filters = 3
@@ -226,64 +232,64 @@ class LocallyConnected2DLayersTest(test.TestCase):
 
 class LocallyConnectedImplementationModeTest(test.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_locallyconnected_implementation(self):
-    num_samples = 4
-    num_classes = 3
-    num_epochs = 2
-
-    np.random.seed(1)
-    targets = np.random.randint(0, num_classes, (num_samples,))
-
-    for width in [1, 6]:
-      for height in [7]:
-        for filters in [2]:
-          for data_format in ['channels_first', 'channels_last']:
-            inputs = get_inputs(
-                data_format, filters, height, num_samples, width)
-
-            for kernel_x in [(3,)]:
-              for kernel_y in [()] if width == 1 else [(2,)]:
-                for stride_x in [(1,)]:
-                  for stride_y in [()] if width == 1 else [(3,)]:
-                    for layers in [2]:
-                      kwargs = {
-                          'layers': layers,
-                          'filters': filters,
-                          'kernel_size': kernel_x + kernel_y,
-                          'strides': stride_x + stride_y,
-                          'data_format': data_format,
-                          'num_classes': num_classes,
-                          'input_shape': inputs.shape
-                      }
-
-                      model_1 = get_model(implementation=1, **kwargs)
-                      model_2 = get_model(implementation=2, **kwargs)
-
-                      copy_model_weights(model_2, model_1)
-
-                      # Compare outputs at initialization.
-                      out_1 = model_1.call(inputs)
-                      out_2 = model_2.call(inputs)
-                      self.assertAllCloseAccordingToType(out_1, out_2,
-                                                         rtol=1e-5, atol=1e-5)
-
-                      # Train.
-                      model_1.fit(x=inputs,
-                                  y=targets,
-                                  epochs=num_epochs,
-                                  batch_size=num_samples)
-
-                      model_2.fit(x=inputs,
-                                  y=targets,
-                                  epochs=num_epochs,
-                                  batch_size=num_samples)
-
-                      # Compare outputs after a few training steps.
-                      out_1 = model_1.call(inputs)
-                      out_2 = model_2.call(inputs)
-                      self.assertAllCloseAccordingToType(out_1, out_2,
-                                                         rtol=1e-5, atol=1e-5)
+    with self.cached_session():
+      num_samples = 4
+      num_classes = 3
+      num_epochs = 2
+
+      np.random.seed(1)
+      targets = np.random.randint(0, num_classes, (num_samples,))
+
+      for width in [1, 6]:
+        for height in [7]:
+          for filters in [2]:
+            for data_format in ['channels_first', 'channels_last']:
+              inputs = get_inputs(
+                  data_format, filters, height, num_samples, width)
+
+              for kernel_x in [(3,)]:
+                for kernel_y in [()] if width == 1 else [(2,)]:
+                  for stride_x in [(1,)]:
+                    for stride_y in [()] if width == 1 else [(3,)]:
+                      for layers in [2]:
+                        kwargs = {
+                            'layers': layers,
+                            'filters': filters,
+                            'kernel_size': kernel_x + kernel_y,
+                            'strides': stride_x + stride_y,
+                            'data_format': data_format,
+                            'num_classes': num_classes,
+                            'input_shape': inputs.shape
+                        }
+
+                        model_1 = get_model(implementation=1, **kwargs)
+                        model_2 = get_model(implementation=2, **kwargs)
+
+                        copy_model_weights(model_2, model_1)
+
+                        # Compare outputs at initialization.
+                        out_1 = model_1.call(inputs)
+                        out_2 = model_2.call(inputs)
+                        self.assertAllCloseAccordingToType(out_1, out_2,
+                                                           rtol=1e-5, atol=1e-5)
+
+                        # Train.
+                        model_1.fit(x=inputs,
+                                    y=targets,
+                                    epochs=num_epochs,
+                                    batch_size=num_samples)
+
+                        model_2.fit(x=inputs,
+                                    y=targets,
+                                    epochs=num_epochs,
+                                    batch_size=num_samples)
+
+                        # Compare outputs after a few training steps.
+                        out_1 = model_1.call(inputs)
+                        out_2 = model_2.call(inputs)
+                        self.assertAllCloseAccordingToType(out_1, out_2,
+                                                           rtol=1e-5, atol=1e-5)
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_make_2d(self):
diff --git a/tensorflow/python/keras/layers/merge.py b/tensorflow/python/keras/layers/merge.py
index 0ded0e42ed..f295af3fe0 100644
--- a/tensorflow/python/keras/layers/merge.py
+++ b/tensorflow/python/keras/layers/merge.py
@@ -40,7 +40,6 @@ class _Merge(Layer):
 
   def __init__(self, **kwargs):
     super(_Merge, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.supports_masking = True
 
   def _merge_function(self, inputs):
@@ -369,7 +368,6 @@ class Concatenate(_Merge):
 
   def __init__(self, axis=-1, **kwargs):
     super(Concatenate, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.axis = axis
     self.supports_masking = True
     self._reshape_required = False
@@ -467,7 +465,6 @@ class Dot(_Merge):
 
   def __init__(self, axes, normalize=False, **kwargs):
     super(Dot, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     if not isinstance(axes, int):
       if not isinstance(axes, (list, tuple)):
         raise TypeError('Invalid type for `axes` - '
diff --git a/tensorflow/python/keras/layers/noise.py b/tensorflow/python/keras/layers/noise.py
index e7c0478513..cb7cee3ebc 100644
--- a/tensorflow/python/keras/layers/noise.py
+++ b/tensorflow/python/keras/layers/noise.py
@@ -55,7 +55,6 @@ class GaussianNoise(Layer):
     super(GaussianNoise, self).__init__(**kwargs)
     self.supports_masking = True
     self.stddev = stddev
-    self._can_use_graph_functions = True
 
   def call(self, inputs, training=None):
 
@@ -100,7 +99,6 @@ class GaussianDropout(Layer):
     super(GaussianDropout, self).__init__(**kwargs)
     self.supports_masking = True
     self.rate = rate
-    self._can_use_graph_functions = True
 
   def call(self, inputs, training=None):
     if 0 < self.rate < 1:
@@ -155,7 +153,6 @@ class AlphaDropout(Layer):
     self.noise_shape = noise_shape
     self.seed = seed
     self.supports_masking = True
-    self._can_use_graph_functions = True
 
   def _get_noise_shape(self, inputs):
     return self.noise_shape if self.noise_shape else array_ops.shape(inputs)
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 3d3bf647e6..7a91693128 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -145,7 +145,6 @@ class BatchNormalization(Layer):
                **kwargs):
     super(BatchNormalization, self).__init__(
         name=name, trainable=trainable, **kwargs)
-    self._can_use_graph_functions = True
     if isinstance(axis, list):
       self.axis = axis[:]
     else:
diff --git a/tensorflow/python/keras/layers/pooling.py b/tensorflow/python/keras/layers/pooling.py
index b8d6b03664..72a9c1d629 100644
--- a/tensorflow/python/keras/layers/pooling.py
+++ b/tensorflow/python/keras/layers/pooling.py
@@ -58,7 +58,6 @@ class Pooling1D(Layer):
                padding='valid', data_format='channels_last',
                name=None, **kwargs):
     super(Pooling1D, self).__init__(name=name, **kwargs)
-    self._can_use_graph_functions = True
     if data_format is None:
       data_format = backend.image_data_format()
     if strides is None:
@@ -231,7 +230,6 @@ class Pooling2D(Layer):
                padding='valid', data_format=None,
                name=None, **kwargs):
     super(Pooling2D, self).__init__(name=name, **kwargs)
-    self._can_use_graph_functions = True
     if data_format is None:
       data_format = backend.image_data_format()
     if strides is None:
@@ -427,7 +425,6 @@ class Pooling3D(Layer):
                padding='valid', data_format='channels_last',
                name=None, **kwargs):
     super(Pooling3D, self).__init__(name=name, **kwargs)
-    self._can_use_graph_functions = True
     if data_format is None:
       data_format = backend.image_data_format()
     if strides is None:
@@ -599,7 +596,6 @@ class GlobalPooling1D(Layer):
 
   def __init__(self, data_format='channels_last', **kwargs):
     super(GlobalPooling1D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.input_spec = InputSpec(ndim=3)
     self.data_format = conv_utils.normalize_data_format(data_format)
 
@@ -705,7 +701,6 @@ class GlobalPooling2D(Layer):
 
   def __init__(self, data_format=None, **kwargs):
     super(GlobalPooling2D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.data_format = conv_utils.normalize_data_format(data_format)
     self.input_spec = InputSpec(ndim=4)
 
@@ -804,7 +799,6 @@ class GlobalPooling3D(Layer):
 
   def __init__(self, data_format=None, **kwargs):
     super(GlobalPooling3D, self).__init__(**kwargs)
-    self._can_use_graph_functions = True
     self.data_format = conv_utils.normalize_data_format(data_format)
     self.input_spec = InputSpec(ndim=5)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index 8ccba990bd..e90629407b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -69,6 +69,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 27aa91a645..4f734ffa98 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -70,6 +70,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index ccff809f2b..7a6a18b501 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -69,6 +69,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index b0fc7f97f1..f8dacfc886 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -70,6 +70,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index 8ccba990bd..e90629407b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -69,6 +69,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 27aa91a645..4f734ffa98 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -70,6 +70,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index c727d5fc2d..8e0f0e3826 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index ccff809f2b..7a6a18b501 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -69,6 +69,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index b0fc7f97f1..f8dacfc886 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -70,6 +70,10 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "run_eagerly"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "state_updates"
     mtype: "<type \'property\'>"
-- 
GitLab


From 4932558aec1d1cf5c248c37da796a0f7e24b50e3 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 13 Nov 2018 20:43:08 -0800
Subject: [PATCH 0204/1554] [XLA] Use TF_RET_CHECK rather than returning
 Unimplemented for invariants in HloEvaluator.

PiperOrigin-RevId: 221382865
---
 .../compiler/xla/service/hlo_evaluator.h      |  7 +-----
 .../xla/service/hlo_evaluator_typed_visitor.h | 23 ++++---------------
 2 files changed, 6 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 8c90252896..d751f40fff 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -221,12 +221,7 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
       const Literal& operand_literal) {
     const auto shape = instruction->shape();
     const auto* operand = instruction->operand(0);
-
-    if (!ShapeUtil::SameDimensions(shape, operand->shape())) {
-      return Unimplemented("Shape Mismatch: %s vs %s",
-                           ShapeUtil::HumanString(shape),
-                           ShapeUtil::HumanString(operand->shape()));
-    }
+    TF_RET_CHECK(ShapeUtil::SameDimensions(shape, operand->shape()));
 
     Literal result(shape);
     TF_RETURN_IF_ERROR(
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 6c3baa6f18..b79412e4d9 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -2719,14 +2719,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     const auto shape = instruction->shape();
     const auto* lhs = instruction->operand(0);
     const auto* rhs = instruction->operand(1);
-
-    if (!(ShapeUtil::SameDimensions(shape, rhs->shape()) &&
-          ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()))) {
-      return Unimplemented(
-          "Shape Mismatch: %s vs %s vs %s: ",
-          ShapeUtil::HumanString(shape), ShapeUtil::HumanString(lhs->shape()),
-          ShapeUtil::HumanString(rhs->shape()));
-    }
+    TF_RET_CHECK(ShapeUtil::SameDimensions(shape, rhs->shape()));
+    TF_RET_CHECK(ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()));
 
     const Literal& lhs_literal = parent_->GetEvaluatedLiteralFor(lhs);
     const Literal& rhs_literal = parent_->GetEvaluatedLiteralFor(rhs);
@@ -2750,16 +2744,9 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     const auto* lhs = instruction->operand(0);
     const auto* rhs = instruction->operand(1);
     const auto* ehs = instruction->operand(2);
-
-    if (!(ShapeUtil::SameDimensions(shape, lhs->shape()) &&
-          ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()) &&
-          ShapeUtil::SameDimensions(rhs->shape(), ehs->shape()))) {
-      return Unimplemented("Shape Mismatch: %s vs %s vs %s vs %s: ",
-                           ShapeUtil::HumanString(shape),
-                           ShapeUtil::HumanString(lhs->shape()),
-                           ShapeUtil::HumanString(rhs->shape()),
-                           ShapeUtil::HumanString(ehs->shape()));
-    }
+    TF_RET_CHECK(ShapeUtil::SameDimensions(shape, lhs->shape()));
+    TF_RET_CHECK(ShapeUtil::SameDimensions(lhs->shape(), rhs->shape()));
+    TF_RET_CHECK(ShapeUtil::SameDimensions(rhs->shape(), ehs->shape()));
 
     const Literal& lhs_literal = parent_->GetEvaluatedLiteralFor(lhs);
     const Literal& rhs_literal = parent_->GetEvaluatedLiteralFor(rhs);
-- 
GitLab


From 37b4d2ef3f90f53f8050f557038b65ff6d1a06bc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 21:15:19 -0800
Subject: [PATCH 0205/1554] Introduce C kernel registration API.

PiperOrigin-RevId: 221385497
---
 tensorflow/c/BUILD           |  46 ++++++++++++++
 tensorflow/c/kernels.cc      | 118 +++++++++++++++++++++++++++++++++++
 tensorflow/c/kernels.h       |  92 +++++++++++++++++++++++++++
 tensorflow/c/kernels_test.cc |  99 +++++++++++++++++++++++++++++
 4 files changed, 355 insertions(+)
 create mode 100644 tensorflow/c/kernels.cc
 create mode 100644 tensorflow/c/kernels.h
 create mode 100644 tensorflow/c/kernels_test.cc

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index d4defaa185..183faaefa9 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -174,6 +174,28 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "kernels",
+    srcs = [
+        "kernels.cc",
+    ],
+    hdrs = [
+        "kernels.h",
+    ],
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = select({
+        "//tensorflow:android": [
+            ":c_api_internal",
+            "//tensorflow/core:android_tensorflow_lib_lite",
+        ],
+        "//conditions:default": [
+            ":c_api_internal",
+            "//tensorflow/core:framework",
+        ],
+    }),
+)
+
 # -----------------------------------------------------------------------------
 # Tests
 
@@ -304,6 +326,30 @@ tf_kernel_library(
     alwayslink = 1,
 )
 
+tf_cuda_cc_test(
+    name = "kernels_test",
+    size = "small",
+    srcs = ["kernels_test.cc"],
+    linkopts = select({
+        "//tensorflow:darwin": ["-headerpad_max_install_names"],
+        "//conditions:default": [],
+    }),
+    tags = ["noasan"],
+    # We must ensure that the dependencies can be dynamically linked since
+    # the shared library must be able to use core:framework.
+    # linkstatic = tf_kernel_tests_linkstatic(),
+    deps = [
+        ":c_api",
+        ":kernels",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:proto_text",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 # -----------------------------------------------------------------------------
 # Python API target
 
diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
new file mode 100644
index 0000000000..ca69345264
--- /dev/null
+++ b/tensorflow/c/kernels.cc
@@ -0,0 +1,118 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+
+#include "tensorflow/c/kernels.h"
+#include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+
+// This file forms the basis of a stable ABI for third-party kernel
+// implementations. It is crucial that changes to this file are made cautiously
+// and with a focus on maintaining both source and binary compatibility.
+
+struct TF_KernelBuilder {
+  ::tensorflow::KernelDefBuilder* cc_builder;
+
+  void* (*create_function)(TF_OpKernelConstruction*);
+  void (*compute_function)(void*, TF_OpKernelContext*);
+  void (*delete_function)(void*);
+};
+
+TF_KernelBuilder* TF_NewKernelBuilder(
+    const char* op_name, const char* device_name,
+    void* (*create_func)(TF_OpKernelConstruction*),
+    void (*compute_func)(void*, TF_OpKernelContext*),
+    void (*delete_func)(void*)) {
+  TF_KernelBuilder* result = new TF_KernelBuilder;
+  result->cc_builder = new ::tensorflow::KernelDefBuilder(op_name);
+  result->cc_builder->Device(device_name);
+  result->create_function = create_func;
+  result->compute_function = compute_func;
+  result->delete_function = delete_func;
+  return result;
+}
+
+void TF_DeleteKernelBuilder(TF_KernelBuilder* builder) {
+  DCHECK_NE(builder, nullptr);
+  delete builder->cc_builder;
+  delete builder;
+}
+
+namespace tensorflow {
+namespace {
+
+// An OpKernel whose methods delegate to C function pointers.
+class COpKernel : public OpKernel {
+ public:
+  explicit COpKernel(OpKernelConstruction* ctx,
+                     void* (*create_func)(TF_OpKernelConstruction*),
+                     void (*compute_func)(void*, TF_OpKernelContext*),
+                     void (*delete_func)(void*))
+      : OpKernel(ctx), compute_func_(compute_func), delete_func_(delete_func) {
+    if (create_func != nullptr) {
+      c_kernel_ =
+          (*create_func)(reinterpret_cast<TF_OpKernelConstruction*>(ctx));
+    } else {
+      c_kernel_ = nullptr;
+    }
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    (*compute_func_)(c_kernel_, reinterpret_cast<TF_OpKernelContext*>(ctx));
+  }
+
+  ~COpKernel() override {
+    if (delete_func_ != nullptr) {
+      (*delete_func_)(c_kernel_);
+    }
+  }
+
+ private:
+  void (*compute_func_)(void*, TF_OpKernelContext* context);
+  void (*delete_func_)(void*);
+  void* c_kernel_;
+};
+
+// A KernelFactory that returns COpKernel instances.
+class KernelBuilderFactory
+    : public ::tensorflow::kernel_factory::OpKernelFactory {
+ public:
+  explicit KernelBuilderFactory(TF_KernelBuilder* builder)
+      : builder_(builder) {}
+  ::tensorflow::OpKernel* Create(
+      ::tensorflow::OpKernelConstruction* context) override {
+    return new ::tensorflow::COpKernel(context, builder_->create_function,
+                                       builder_->compute_function,
+                                       builder_->delete_function);
+  }
+  ~KernelBuilderFactory() override { TF_DeleteKernelBuilder(builder_); }
+
+ private:
+  TF_KernelBuilder* builder_;
+};
+}  // namespace
+}  // namespace tensorflow
+
+void TF_RegisterKernelBuilder(const char* name, TF_KernelBuilder* builder,
+                              TF_Status* status) {
+  using tensorflow::register_kernel::Name;
+
+  tensorflow::kernel_factory::OpKernelRegistrar(
+      builder->cc_builder->Build(), name,
+      absl::make_unique<tensorflow::KernelBuilderFactory>(builder));
+
+  TF_SetStatus(status, TF_OK, "");
+}
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
new file mode 100644
index 0000000000..db51b2d535
--- /dev/null
+++ b/tensorflow/c/kernels.h
@@ -0,0 +1,92 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_C_KERNELS_H_
+#define TENSORFLOW_C_KERNELS_H_
+
+#include "tensorflow/c/c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// --------------------------------------------------------------------------
+// C API for TensorFlow Kernels.
+//
+// This API allows developers to register custom kernel implementations for
+// TensorFlow.
+//
+// See c_api.h header comments for a discussion about API conventions.
+//
+// Users wishing to extend TensorFlow with new kernels will call
+// `TF_NewKernelBuilder`. The resulting kernel builder can be registered with
+// `TF_RegisterKernelBuilder`, which will allow TF to construct user-provided
+// kernels when necessary.
+
+typedef struct TF_KernelBuilder;
+typedef struct TF_OpKernelConstruction;
+typedef struct TF_OpKernelContext;
+
+// Allocates a new kernel builder and returns a pointer to it.
+//
+// If non-null, TensorFlow will call create_func when it needs to instantiate
+// the kernel. The pointer returned by create_func will be passed to
+// compute_func and delete_func, thereby functioning as a "this" pointer for
+// referring to kernel instances.
+//
+// The TF_OpKernelConstruction pointer passed to create_func is owned by
+// TensorFlow and will be deleted once create_func returns. It must not be used
+// after this.
+//
+// When TensorFlow needs to perform a computation with this kernel, it will
+// call compute_func. This function will receive the pointer returned by
+// create_func (or null if no create_func was provided), along with the inputs
+// to the computation.
+//
+// The TF_OpKernelContext pointer received by compute_func is owned by
+// TensorFlow and will be deleted once compute_func returns. It must not be used
+// after this.
+//
+// Finally, when TensorFlow no longer needs the kernel, it will call
+// delete_func if one is provided. This function will receive the pointer
+// returned in `create_func` or nullptr if no `create_func` was provided.
+//
+// The caller should pass the result of this function to
+// TF_RegisterKernelBuilder, which will take ownership of the pointer. If, for
+// some reason, the kernel builder will not be registered, the caller should
+// delete it with TF_DeleteKernelBuilder.
+TF_CAPI_EXPORT extern TF_KernelBuilder* TF_NewKernelBuilder(
+    const char* op_name, const char* device_name,
+    void* (*create_func)(TF_OpKernelConstruction*),
+    void (*compute_func)(void*, TF_OpKernelContext*),
+    void (*delete_func)(void*));
+
+// Register the given kernel builder with the TensorFlow runtime. If
+// registration fails, the given status will be populated.
+//
+// This call takes ownership of the `builder` pointer.
+TF_CAPI_EXPORT extern void TF_RegisterKernelBuilder(const char* kernel_name,
+                                                    TF_KernelBuilder* builder,
+                                                    TF_Status* status);
+
+// Deletes the given TF_KernelBuilder. This should be called only if the kernel
+// builder is not registered with TensorFlow via TF_RegisterKernelBuilder.
+TF_CAPI_EXPORT extern void TF_DeleteKernelBuilder(TF_KernelBuilder* builder);
+
+#ifdef __cplusplus
+} /* end extern "C" */
+#endif
+
+#endif  // TENSORFLOW_C_KERNELS_H_
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
new file mode 100644
index 0000000000..e706c7c1d9
--- /dev/null
+++ b/tensorflow/c/kernels_test.cc
@@ -0,0 +1,99 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/kernels.h"
+
+#include "tensorflow/core/framework/kernel_def.pb.h"
+#include "tensorflow/core/framework/node_def.pb_text.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+struct MyCustomKernel {
+  bool created;
+  bool compute_called;
+};
+
+static bool delete_called = false;
+
+static void* MyCreateFunc(TF_OpKernelConstruction* ctx) {
+  LOG(INFO) << "Wow, actually got into creation";
+  struct MyCustomKernel* s = new struct MyCustomKernel;
+  s->created = true;
+  s->compute_called = false;
+  return s;
+}
+
+static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) {
+  struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
+  s->compute_called = true;
+}
+
+static void MyDeleteFunc(void* kernel) {
+  struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
+  EXPECT_TRUE(s->created);
+  EXPECT_TRUE(s->compute_called);
+  delete_called = true;
+  delete s;
+}
+
+// Tests registration of a single C kernel and checks that calls through the
+// C/C++ boundary are being made.
+TEST(TestKernel, TestRegisterKernelBuilder) {
+  const char* kernel_name = "SomeKernelName";
+  const char* op_name = "FooOp";
+  const char* device_name = "barDev";
+
+  TF_KernelBuilder* builder = TF_NewKernelBuilder(
+      op_name, device_name, &MyCreateFunc, &MyComputeFunc, &MyDeleteFunc);
+
+  {
+    TF_Status* status = TF_NewStatus();
+    TF_RegisterKernelBuilder(kernel_name, builder, status);
+    EXPECT_EQ(TF_OK, TF_GetCode(status));
+    TF_Buffer* buf = TF_GetRegisteredKernelsForOp("FooOp", status);
+    EXPECT_EQ(TF_OK, TF_GetCode(status));
+    ::tensorflow::KernelList list;
+    list.ParseFromArray(buf->data, buf->length);
+    ASSERT_EQ(1, list.kernel_size());
+    ASSERT_EQ("barDev", list.kernel(0).device_type());
+    TF_DeleteBuffer(buf);
+    TF_DeleteStatus(status);
+  }
+
+  REGISTER_OP("FooOp")
+      .Input("input1: double")
+      .Input("input2: uint8")
+      .Output("output1: uint8");
+
+  {
+    ::tensorflow::NodeDef def;
+    def.set_op("FooOp");
+    def.set_device("bar");
+    def.add_input("input1");
+    def.add_input("input2");
+    ::tensorflow::Status status;
+    std::unique_ptr<::tensorflow::OpKernel> kernel =
+        ::tensorflow::CreateOpKernel(::tensorflow::DeviceType("barDev"),
+                                     nullptr, nullptr, def, 1, &status);
+    TF_EXPECT_OK(status);
+    ASSERT_NE(nullptr, kernel.get());
+    kernel->Compute(nullptr);
+  }
+
+  ASSERT_TRUE(delete_called);
+}
-- 
GitLab


From 37ce39a37f4e8f0b21cc88f2eb147004584b6c8f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 13 Nov 2018 22:29:22 -0800
Subject: [PATCH 0206/1554] Fix output indexing in depthwise convolutions

PiperOrigin-RevId: 221390830
---
 .../compiler/xla/tests/convolution_test.cc    | 140 ++++++++++++++++++
 1 file changed, 140 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 334d5f1367..a5e9cfd0cc 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -798,6 +798,146 @@ TYPED_TEST(Convolve2D_1x4x4x512_3x3x1x512_Depthwise_Valid_Output_Batch_In_Lanes,
   this->RunTest();
 }
 
+template <typename T>
+class Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Input_Batch_in_Lanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {256, 4, 4, 512};
+    std::vector<int64> filter_dims = {3, 3, 1, 512};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/512);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+    auto input_r4_relaid =
+        input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(2048 * 256, static_cast<T>(18));
+
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 =
+        expected_r1.Reshape({256, 2, 2, 512}).ConsumeValueOrDie();
+
+    auto input_literal =
+        client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_);
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Input_Batch_in_Lanes,
+                TestTypes);
+TYPED_TEST(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Input_Batch_in_Lanes,
+           Types) {
+  this->RunTest();
+}
+
+template <typename T>
+class Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Both_Batch_in_Lanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {256, 4, 4, 512};
+    std::vector<int64> filter_dims = {3, 3, 1, 512};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/512);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+    auto input_r4_relaid =
+        input_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(2048 * 256, static_cast<T>(18));
+
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 =
+        expected_r1.Reshape({256, 2, 2, 512}).ConsumeValueOrDie();
+    auto expected_r4_relaid =
+        expected_r4.Relayout(LayoutUtil::MakeLayout({0, 3, 2, 1}));
+
+    auto input_literal =
+        client_->TransferToServer(input_r4_relaid).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4_relaid,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_, &expected_r4_relaid.shape());
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Both_Batch_in_Lanes,
+                TestTypes);
+TYPED_TEST(Convolve2D_256x4x4x512_3x3x1x512_Depthwise_Both_Batch_in_Lanes,
+           Types) {
+  this->RunTest();
+}
+
 template <typename T>
 class Convolve2D_1x4x4x5_3x3x1x5_Depthwise_Valid_Output_Batch_In_Lanes
     : public ConvolutionTest {
-- 
GitLab


From 88b2369fe7c4451c63ff0599f7477897dabff2e0 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Wed, 14 Nov 2018 14:34:05 +0800
Subject: [PATCH 0207/1554] Break very long lines into 2.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index dffd7ef4a4..bac434886f 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -515,8 +515,9 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       GetConv2DBackpropFilterOrBiasAddGrad});
 
     // The fusion patterns in "finfo_" that show up first will get applied
-    // first, for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC, A->B->C->D
-    // to ABCD}, since the first gets applied first, the final graph will be ABC->D.
+    // first, for example, graph "A->B->C-D" and finfo_ is {A->B->C to ABC,
+    // A->B->C->D to ABCD}, since the first gets applied first, the final
+    // graph will be ABC->D.
 
     //
     // Add rules to fuse sequences such as "Transpose (NCHW -> NHWC) + Conv2D
-- 
GitLab


From 619c775f7f19b079a35360b50b993f1ce44674c1 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 13 Nov 2018 23:05:08 -0800
Subject: [PATCH 0208/1554] [XLA:GPU] Refactor the 0-2-1 tiling implementation
 to support more use cases.

The current implementation of 0-2-1 tiling only handles two cases, a single
kCopy instruction and a kLoop fusion instruction. This change adds class
TilingScheme to support more general tiling methods that consist of tiles and
block of tiles. It also restructures the existing tiling implementation to
generate the kernel based on a TilingScheme object, a few function objects that
provide hooks for HLO specific code generation and other parameters. This
change supports our next step in extending tiling for reduction.

PiperOrigin-RevId: 221393383
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 744 +++++++++++-------
 .../xla/service/gpu/ir_emitter_unnested.h     | 119 ++-
 .../compiler/xla/service/llvm_ir/ir_array.h   |   8 +
 .../xla/service/llvm_ir/kernel_tiling.cc      | 112 +++
 .../xla/service/llvm_ir/kernel_tiling.h       | 141 ++++
 tensorflow/compiler/xla/util.h                |  13 +
 6 files changed, 842 insertions(+), 295 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 87b6cd640a..9976b521b3 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -69,7 +69,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/sort_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
@@ -88,6 +87,8 @@ limitations under the License.
 namespace xla {
 namespace gpu {
 
+using llvm_ir::KernelMappingScheme;
+
 namespace {
 
 using absl::InlinedVector;
@@ -1188,7 +1189,7 @@ Status IrEmitterUnnested::EmitColumnReduction(
       .EmitLoop(IrName(reduce), index_ty);
 }
 
-static std::pair<int64, int64> ComputeTilingSchemeForReduction(
+static std::pair<int64, int64> ComputeKernelMappingSchemeForReduction(
     int64 depth, int64 width, int64 kWarpSize) {
   constexpr int64 kTargetNumElementsPerThread = 64;
   int64 x_tile_size = kTargetNumElementsPerThread;
@@ -1322,7 +1323,7 @@ Status IrEmitterUnnested::EmitRowReduction(
   int64 x_tile_size;
   int64 z_tile_size;
   std::tie(x_tile_size, z_tile_size) =
-      ComputeTilingSchemeForReduction(depth, width, kWarpSize);
+      ComputeKernelMappingSchemeForReduction(depth, width, kWarpSize);
 
   // Round the width in tiles up to the nearest multiple of kWarpSize, so that
   // the use of shfl_down is valid.
@@ -3146,31 +3147,6 @@ std::vector<IrArray> IrEmitterUnnested::ConstructIrArrayForInputs(
   return param_arrays;
 }
 
-int IrEmitterUnnested::ConstructOutputReducedShapeAndCastOutputIrArrayToShape(
-    const HloInstruction& hlo, const std::vector<IrArray>& output_arrays,
-    absl::Span<const int64> reduced_output_dims,
-    std::vector<Shape>* output_reduced_shapes,
-    std::vector<IrArray>* output_in_reduced_shape_arrays) {
-  int64 num_outputs = 1;
-  if (hlo.IsMultiOutputFusion()) {
-    num_outputs = ShapeUtil::TupleElementCount(hlo.shape());
-    output_in_reduced_shape_arrays->reserve(num_outputs);
-    output_reduced_shapes->reserve(num_outputs);
-    for (int64 i = 0; i < num_outputs; ++i) {
-      output_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout(
-          ShapeUtil::GetSubshape(hlo.shape(), {i}).element_type(),
-          reduced_output_dims));
-      output_in_reduced_shape_arrays->push_back(
-          output_arrays[i].CastToShape((*output_reduced_shapes)[i], &b_));
-    }
-  } else {
-    output_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout(
-        hlo.shape().element_type(), reduced_output_dims));
-    output_in_reduced_shape_arrays->push_back(
-        output_arrays[0].CastToShape((*output_reduced_shapes)[0], &b_));
-  }
-  return num_outputs;
-}
 
 int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape(
     const HloInstruction& hlo, const std::vector<IrArray>& param_arrays,
@@ -3230,304 +3206,502 @@ llvm::Value* GetBlockIdx(llvm::IRBuilder<>* builder, llvm::Type* index_ty,
                                 "block.id.x");
 }
 
-// Emits code to process up to (tile_size/num_rows) elements in a tile, given
-// `emit_elem_function` is the function to emit code to process one element, `y`
-// and `x` are the coordinates for the first element to process, and `index` is
-// the index for the origin of the tile. Emits bounds check to ensure that each
-// processed element is within the boundary defined by `tile_width` and
-// `tile_height`.
+void EmitFullTile(const KernelMappingScheme* mapping_scheme,
+                  const IrArray::Index& tile_origin_index,
+                  llvm::IRBuilder<>* builder, llvm::Value* y, llvm::Value* x,
+                  llvm::Type* index_ty,
+                  const std::function<void(const IrArray::Index&, llvm::Value*,
+                                           llvm::Value*)>& emit_elem_function) {
+  int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX();
+  int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY();
+  int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
+  int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY();
+  for (int64 i = 0; i < tile_size_y; i += num_threads_y) {
+    IrArray::Index source_idx_y =
+        tile_origin_index.AddOffsetToDim(llvm::ConstantInt::get(index_ty, i),
+                                         KernelMappingScheme::DimY, builder);
+    llvm::Value* y_loc =
+        builder->CreateAdd(llvm::ConstantInt::get(index_ty, i), y);
+    for (int64 j = 0; j < tile_size_x; j += num_threads_x) {
+      IrArray::Index source_idx =
+          source_idx_y.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j),
+                                      KernelMappingScheme::DimX, builder);
+      llvm::Value* x_loc =
+          builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
+      emit_elem_function(source_idx, y_loc, x_loc);
+    }
+  }
+}
+
+void EmitPartialTile(
+    const KernelMappingScheme* mapping_scheme,
+    const IrArray::Index& tile_origin_index, const string& loop_name,
+    KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
+    llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width,
+    llvm::Type* index_ty,
+    const std::function<void(const IrArray::Index&, llvm::Value*,
+                             llvm::Value*)>& emit_elem_function) {
+  int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX();
+  int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY();
+  int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
+
+  for (int64 j = 0; j < tile_size_x; j += num_threads_x) {
+    IrArray::Index source_idx =
+        tile_origin_index.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j),
+                                         KernelMappingScheme::DimX, builder);
+    llvm::Value* x_loc =
+        builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
+
+    ksl->IfReturnVoid(
+        "x_in_tile", builder->CreateICmpULT(x_loc, tile_width), [&] {
+          // tile_height_bound =
+          //   ceil(tile_height / num_threads_y) * num_threads_y
+          llvm::Value* ceiling_of_ratio = builder->CreateUDiv(
+              builder->CreateAdd(tile_height, llvm::ConstantInt::get(
+                                                  index_ty, num_threads_y - 1)),
+              llvm::ConstantInt::get(index_ty, num_threads_y));
+          llvm::Value* tile_height_bound = builder->CreateMul(
+              ceiling_of_ratio,
+              llvm::ConstantInt::get(index_ty, num_threads_y));
+          ksl->ForReturnVoid(
+              loop_name, /*start=*/llvm::ConstantInt::get(index_ty, 0),
+              /*end=*/tile_height_bound,
+              /*step=*/llvm::ConstantInt::get(index_ty, num_threads_y),
+              [&](llvm::Value* y_indvar) {
+                llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
+                ksl->IfReturnVoid(
+                    "y_in_tile", builder->CreateICmpULT(y_loc, tile_height),
+                    [&] {
+                      emit_elem_function(
+                          source_idx.AddOffsetToDim(
+                              y_indvar, KernelMappingScheme::DimY, builder),
+                          y_loc, x_loc);
+                    });
+              });
+        });
+  }
+}
+
+// Emits code to process up to
+// (tile_size_x/num_threads_x * tile_size_y/num_threads_y) elements in a tile,
+// given `emit_elem_function` is the function to emit code to process one
+// element, `y` and `x` are the intra-tile coordinates for the first element
+// to process, and `index` is the index for the origin of the tile. Information
+// about tile_size_x/y and num_threads_x/y are stored in `mapping_scheme`. Emits
+// bounds check to ensure that each processed element is within the boundary
+// defined by `tile_width` and `tile_height`.
 void EmitTiledElementalCodeWithBoundsCheck(
-    int64 tile_size, int64 num_rows, const IrArray::Index& index,
-    const string& loop_name, KernelSupportLibrary* ksl,
-    llvm::IRBuilder<>* builder, llvm::Value* y, llvm::Value* x,
-    llvm::Value* tile_width, llvm::Value* tile_height,
-    const std::function<void(const IrArray::Index&, llvm::Value*)>&
-        emit_elem_function) {
+    const KernelMappingScheme* mapping_scheme,
+    const IrArray::Index& tile_origin_index, const string& loop_name,
+    KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
+    llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width,
+    const std::function<void(const IrArray::Index&, llvm::Value*,
+                             llvm::Value*)>& emit_elem_function) {
+  int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
+  int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY();
   llvm::Type* index_ty = tile_width->getType();
-  // Emits a constant value with index type.
-  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
-    return llvm::ConstantInt::get(index_ty, c);
-  };
-  // Adds `addend` to the given `dim` of `index`.
-  auto offset_dim = [&](IrArray::Index index, llvm::Value* addend, int64 dim) {
-    index[dim] = builder->CreateAdd(index[dim], addend);
-    return index;
-  };
-
-  auto emit_full_tile = [&] {
-    for (int64 i = 0; i < tile_size; i += num_rows) {
-      auto source_idx = offset_dim(index, index_typed_constant(i), /*dim=*/1);
-      auto y_loc = builder->CreateAdd(index_typed_constant(i), y);
-      emit_elem_function(source_idx, y_loc);
-    }
-  };
 
-  auto emit_last_row = [&] {
-    ksl->IfReturnVoid("x_in_tile", builder->CreateICmpULT(x, tile_width), [&] {
-      // tile_height_upper_bound =
-      //   ceil(tile_height / num_rows) * num_rows
-      auto tile_height_upper_bound = builder->CreateMul(
-          builder->CreateUDiv(
-              builder->CreateAdd(tile_height,
-                                 index_typed_constant(num_rows - 1)),
-              index_typed_constant(num_rows)),
-          index_typed_constant(num_rows));
-      ksl->ForReturnVoid(
-          loop_name, /*start=*/index_typed_constant(0),
-          /*end=*/tile_height_upper_bound,
-          /*step=*/index_typed_constant(num_rows), [&](llvm::Value* y_indvar) {
-            auto y_loc = builder->CreateAdd(y_indvar, y);
-            ksl->IfReturnVoid(
-                "y_in_tile", builder->CreateICmpULT(y_loc, tile_height), [&] {
-                  emit_elem_function(offset_dim(index, y_indvar, /*dim=*/1),
-                                     y_loc);
-                });
-          });
-    });
-  };
   ksl->IfReturnVoid(
       "full_tile",
       builder->CreateAnd(
-          builder->CreateICmpEQ(index_typed_constant(tile_size), tile_width),
-          builder->CreateICmpEQ(index_typed_constant(tile_size), tile_height)),
-      emit_full_tile, emit_last_row);
+          builder->CreateICmpEQ(llvm::ConstantInt::get(index_ty, tile_size_x),
+                                tile_width),
+          builder->CreateICmpEQ(llvm::ConstantInt::get(index_ty, tile_size_y),
+                                tile_height)),
+      [&] {
+        EmitFullTile(mapping_scheme, tile_origin_index, builder, y, x, index_ty,
+                     emit_elem_function);
+      },
+      [&] {
+        EmitPartialTile(mapping_scheme, tile_origin_index, loop_name, ksl,
+                        builder, y, x, tile_height, tile_width, index_ty,
+                        emit_elem_function);
+      });
 }
 }  // namespace
 
-// Emits a kernel for the given hlo instruction using a tiled 0-2-1 transpose
-// algorithm to improve the memory access patterns for the input parameters
-// which have a shape that is a 0-2-1 transpose of the output tensors.
+// Emits code to process a tensor element in a tile for the given kCopy HLO that
+// performs a 0-2-1 transpose.
 //
-// For the purpose of tiling, the output tensors have a logical shape of three
-// components 0-2-1 while the relevant input parameters have a logical shape of
-// three components 0-1-2 in the order major to minor. The x- and y- dimensions
-// of the tensors are tiled in square tiles of edge length `kTileSize`. Each
-// thread block of `kTileSize` x `kNumRows` threads transposes one tile: each
-// thread copies kTileSize/kNumRows elements from the input to a shared memory
-// tile, then the otherwise "regular hlo kernel" reads from the shared memory
-// instead of the original input.
+// index: The index for the first output element in the normalized tensor. The
+//   normalized tensor is the resulting tensor after collapsing contiguous
+//   dimensions that play the same role in the transpose.
+// y_loc: The y coordinate within a tile.
+// x_loc: The x coordinate within a tile.
+// kernel_info: Other information to support the kernel code generation.
+void IrEmitterUnnested::EmitTileElementForCopy(
+    HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
+    const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+    llvm::Value* x_loc) {
+  llvm_ir::TiledParameterInfo* tiled_param_info =
+      kernel_info->GetTiledParameterInfo();
+  // TODO(jlebar): Add AA metadata to this load.
+  llvm::Instruction* load_from_shmem_buffer =
+      Load(GEP(tiled_param_info->GetBufferForParameter(0),
+               {b_.getInt64(0), x_loc, y_loc}),
+           "output_element");
+  llvm_ir::IrArray output_array = GetIrArray(*hlo, *hlo);
+  Shape output_reduced_shape = ShapeUtil::MakeShapeWithDescendingLayout(
+      hlo->shape().element_type(),
+      kernel_info->GetKernelMappingScheme()->GetDimensionsInElements());
+  // When the output_reduced_shape is a 0-2-1 transpose of the input shape,
+  // the 0-2-1 transpose is achieved through EmitWriteArrayElement.
+  output_array.CastToShape(output_reduced_shape, &b_)
+      .EmitWriteArrayElement(index, load_from_shmem_buffer, &b_);
+}
+
+// Emits code to process a tensor element in a tile for the given kLoop fusion
+// HLO containing parameters that are 0-2-1 transpose of its outputs.
 //
-// This is similar to the following CUDA algorithm in TensorFlow:
-// https://goo.gl/MStRV6.
-//
-// `kTileSize` should usually be same as warp size. We currently choose 32 for
-// `kTileSize` and 4 for `kNumRows`. The CUDA algorithm uses 8 for `kNumRows`.
-//
-// TODO(b/33320379): Here each block transposes 1 tile. It may be more efficient
-// to launch fewer blocks so each transposes many tiles.
-LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
-    HloInstruction* hlo, absl::Span<const int64> reduced_output_dims,
-    absl::Span<const int64> tiled_param_ids) {
-  // Parameters for the tiling algorithm.
-  constexpr int64 kTileSize = 32;
-  constexpr int64 kNumRows = 4;
-  constexpr int64 kThreadsPerTile = kTileSize * kNumRows;
-
-  // Construct IrArrays for the inputs and outputs.
+// index: The index for the first output element in the normalized tensor, that
+//   is the resulting tensor after collapsing contiguous dimensions that play
+//   the same role in the transpose.
+// kernel_info: Other information to support the kernel code generation.
+// y_loc: The y coordinate within a tile.
+// x_loc: The x coordinate within a tile.
+void IrEmitterUnnested::EmitTileElementForFusion(
+    HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
+    const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+    llvm::Value* x_loc) {
+  llvm_ir::TiledParameterInfo* tiled_param_info =
+      kernel_info->GetTiledParameterInfo();
   std::vector<IrArray> output_arrays = ConstructIrArrayForOutputs(*hlo);
-  int64 num_outputs = output_arrays.size();
-  std::vector<IrArray> param_arrays = ConstructIrArrayForInputs(*hlo);
-  int64 num_params = param_arrays.size();
+  GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_,
+                                     GetNestedComputer());
+  FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(hlo),
+                               &elem_emitter);
+  tiled_param_info->set_y(y_loc);
+  tiled_param_info->set_x(x_loc);
+  fused_emitter.SetTiledParameterInfo(tiled_param_info);
+  TF_CHECK_OK(hlo->fused_expression_root()->Accept(&fused_emitter));
+  IrArray::Index untiled_index =
+      kernel_info->GetKernelMappingScheme()->GetReshapedOutputIndex(
+          index, output_arrays[0].GetShape());
+  const llvm_ir::ElementGenerator& output_generator =
+      fused_emitter.GetRootGenerator();
+  llvm::Value* output_value = output_generator(untiled_index).ValueOrDie();
+  if (hlo->IsMultiOutputFusion()) {
+    DCHECK(output_value->getType()->isStructTy());
+    DCHECK_EQ(output_value->getType()->getStructNumElements(),
+              output_arrays.size());
+    for (int64 i = 0; i < output_arrays.size(); ++i) {
+      output_arrays[i].EmitWriteArrayElement(
+          untiled_index, ExtractValue(output_value, i), &b_);
+    }
+  } else {
+    output_arrays[0].EmitWriteArrayElement(untiled_index, output_value, &b_);
+  }
+}
+
+// Emits a block of tiles, given a function object to emit one tile.
+void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
+                                  const KernelCodegenInfo* kernel_info,
+                                  KernelSupportLibrary& ksl,
+                                  llvm::Type* index_ty) {
+  KernelMappingScheme* mapping_scheme = kernel_info->GetKernelMappingScheme();
+  absl::Span<const int64> dims_in_tile = mapping_scheme->GetDimensionsInTiles();
+  absl::Span<const int64> dims_in_block =
+      mapping_scheme->GetDimensionsInBlocks();
+  absl::Span<const int64> block_sizes = mapping_scheme->GetBlockSizes();
+  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
+    return llvm::ConstantInt::get(index_ty, c);
+  };
+
+  // Emit all the tiles for a given dimension in a tile block.
+  auto emit_tiles_for_block_dim =
+      [&](const string& loop_name, const IrArray::Index& starting_tile,
+          int dim_id,
+          const std::function<void(const IrArray::Index& tile_index)>
+              emit_next_block_dim) {
+        if (block_sizes[dim_id] == 1) {
+          emit_next_block_dim(starting_tile);
+        } else {
+          llvm::Value* starting_tile_index_for_dim = starting_tile[dim_id];
+          llvm::Value* block_size_for_dim =
+              index_typed_constant(block_sizes[dim_id]);
+          llvm::Value* block_id_for_dim =
+              b_.CreateUDiv(starting_tile_index_for_dim, block_size_for_dim);
+          llvm::Value* last_block_for_dim =
+              index_typed_constant(dims_in_block[dim_id] - 1);
+          llvm::Value* last_block_size_for_dim = index_typed_constant(
+              dims_in_tile[dim_id] -
+              (dims_in_block[dim_id] - 1) * block_sizes[dim_id]);
+          llvm::Value* num_tiles_in_block =
+              Select(ICmpEQ(last_block_for_dim, block_id_for_dim),
+                     last_block_size_for_dim, block_size_for_dim);
+
+          ksl.ForReturnVoid(
+              loop_name,
+              /*start=*/index_typed_constant(0),
+              /*end=*/num_tiles_in_block,
+              /*step=*/1, [&](llvm::Value* block_dim_induction_var) {
+                IrArray::Index tile_index = starting_tile.AddOffsetToDim(
+                    block_dim_induction_var, dim_id, &b_);
+                emit_next_block_dim(tile_index);
+              });
+        }
+      };
+
+  absl::Span<const int64> reduced_dims =
+      mapping_scheme->GetDimensionsInElements();
+  const bool block_contains_multi_tiles =
+      mapping_scheme->GetNumberOfTilesInOneBlock() > 1;
+
+  // Emit the tile with a given tile_index, by calculating the tight bounds for
+  // each dimension of the tile and then calling emit_one_tile.
+  auto emit_one_tile_for_tile_index = [&](const IrArray::Index& tile_index) {
+    std::vector<llvm::Value*> output_tile_bounds(3);
+    for (int i = KernelMappingScheme::DimY; i < KernelMappingScheme::DimTot;
+         ++i) {
+      int64 tile_size_for_dim = mapping_scheme->GetTileSizeForDimension(i);
+      // Only last row or column may not have full size.
+      llvm::Value* is_last_row =
+          ICmpEQ(tile_index[i], index_typed_constant(dims_in_tile[i] - 1));
+      int64 partial_row_size =
+          reduced_dims[i] - (dims_in_tile[i] - 1) * tile_size_for_dim;
+      output_tile_bounds[i] =
+          Select(is_last_row, index_typed_constant(partial_row_size),
+                 index_typed_constant(tile_size_for_dim), "tile_bound");
+    }
+
+    IrArray::Index tile_origin =
+        mapping_scheme->GetElementIndexForTileOrigin(tile_index);
+    emit_one_tile(tile_origin, output_tile_bounds, block_contains_multi_tiles);
+  };
+
+  const IrArray::Index starting_block =
+      mapping_scheme->EmitBlockIndex(index_ty);
+  const IrArray::Index starting_tile_for_dim_z =
+      mapping_scheme->GetTileIndexForBlockOrigin(starting_block);
+
+  // Emit the three dimensional block of tiles.
+  emit_tiles_for_block_dim(
+      "block_dim_z", starting_tile_for_dim_z, KernelMappingScheme::DimZ,
+      [&](const IrArray::Index& starting_tile_for_dim_y) {
+        emit_tiles_for_block_dim(
+            "block_dim_y", starting_tile_for_dim_y, KernelMappingScheme::DimY,
+            [&](const IrArray::Index& starting_tile_for_dim_x) {
+              emit_tiles_for_block_dim("block_dim_x", starting_tile_for_dim_x,
+                                       KernelMappingScheme::DimX,
+                                       emit_one_tile_for_tile_index);
+            });
+      });
+}
 
+// Emits a kernel for the hlo instruction using the given kernel mapping scheme.
+//
+// unnested_hlo: The unnested hlo instruction for which the kernel is generated.
+//   Currently, these hlo instructions are supported: kLoop fusion, kCopy.
+// tiled_param_ids: The IDs for the parameters that are 0-2-1 transpose of
+//   other tensors with the same dimensions and need to be tiled and tranposed.
+// mapping_scheme: The tiling scheme to use.
+// kernel_generator: Contains function objects for code generation, such as
+//   element generator, block prologue and epilogue generators.
+// kernel_info: Represent other information to support the code generation
+//   of the tiled kernel for the hlo.
+LaunchDimensions IrEmitterUnnested::EmitKernel(
+    HloInstruction* unnested_hlo, absl::Span<const int64> tiled_param_ids,
+    const KernelCodeGenerator& kernel_generator,
+    KernelCodegenInfo* kernel_info) {
+  KernelMappingScheme* mapping_scheme = kernel_info->GetKernelMappingScheme();
+
+  std::vector<IrArray> param_arrays = ConstructIrArrayForInputs(*unnested_hlo);
+  int64 num_params = param_arrays.size();
   // Allocate shared memory buffers to store the tiled inputs.
   std::vector<llvm::Value*> param_shmem_buffers(num_params, nullptr);
   for (int64 id : tiled_param_ids) {
-    const HloInstruction* param = hlo->operand(id);
-    // Add 1 to the minor dimension to reduce shared memory bank conflicts.
-    llvm::Type* tile_type = llvm::ArrayType::get(
-        llvm::ArrayType::get(llvm_ir::PrimitiveTypeToIrType(
-                                 param->shape().element_type(), module_),
-                             kTileSize + 1),
-        kTileSize);
-    auto* tile_base_ptr = llvm_ir::AllocateSharedMemoryTile(
-        b_.GetInsertBlock()->getParent()->getParent(), tile_type,
-        IrName(hlo, StrCat("tile", id)));
-    param_shmem_buffers[id] = tile_base_ptr;
+    const HloInstruction* param = unnested_hlo->operand(id);
+    param_shmem_buffers[id] =
+        mapping_scheme->GetSharedMemoryBufferForElementType(
+            llvm_ir::PrimitiveTypeToIrType(param->shape().element_type(),
+                                           module_),
+            IrName(unnested_hlo, StrCat("tile", id)));
     VLOG(3) << "Added shmem buffer for parameter " << id << ": "
-            << llvm_ir::DumpToString(*tile_base_ptr);
-  }
-
-  // The 0-2-1 shape of the tiling scheme is the reduced shape of the HLO result
-  // for the purpose of tiling. Calculate the logical output dimensions in the
-  // tile from the reduced output dimensions.
-  std::vector<int64> output_dims_in_tiles = std::vector<int64>(
-      reduced_output_dims.begin(), reduced_output_dims.end());
-  CHECK_EQ(output_dims_in_tiles.size(), 3);
-  for (int i = 1; i < 3; ++i) {
-    output_dims_in_tiles[i] =
-        CeilOfRatio<int64>(output_dims_in_tiles[i], kTileSize);
+            << llvm_ir::DumpToString(*param_shmem_buffers[id]);
   }
-  const int64 num_tiles =
-      absl::c_accumulate(output_dims_in_tiles, 1, std::multiplies<int64>());
-  LaunchDimensions launch_dimensions(num_tiles, kThreadsPerTile);
 
-  llvm::Type* index_ty =
-      GetIndexTypeForKernel(hlo, launch_dimensions.launch_bound(), &b_);
+  CHECK_EQ(mapping_scheme->GetThreadsPerTile() % kWarpSize, 0);
+  LaunchDimensions launch_dimensions = LaunchDimensions(
+      mapping_scheme->GetNumberOfBlocks(), mapping_scheme->GetThreadsPerTile());
+  llvm::Type* index_ty = GetIndexTypeForKernel(
+      unnested_hlo, launch_dimensions.launch_bound(), &b_);
   auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
     return llvm::ConstantInt::get(index_ty, c);
   };
 
-  // Cast each output IrArray to its corresponding reduced shape and keep the
-  // reduced shape live during IR emission.
-  std::vector<IrArray> output_in_reduced_shape_arrays;
-  std::vector<Shape> output_reduced_shapes;
-  CHECK_EQ(ConstructOutputReducedShapeAndCastOutputIrArrayToShape(
-               *hlo, output_arrays, reduced_output_dims, &output_reduced_shapes,
-               &output_in_reduced_shape_arrays),
-           num_outputs);
-
   // For each tiled parameter, cast its input IrArray to the corresponding
   // reduced shape and keep the reduced shape live during IR emission.
   std::vector<IrArray> param_in_reduced_shape_arrays;
   std::vector<Shape> param_reduced_shapes;
-  CHECK_EQ(ConstructInputReducedShapeAndCastInputIrArrayToShape(
-               *hlo, param_arrays, param_shmem_buffers, reduced_output_dims,
-               &param_reduced_shapes, &param_in_reduced_shape_arrays),
-           num_params);
+  absl::Span<const int64> reduced_dims =
+      mapping_scheme->GetDimensionsInElements();
+  int num_shapes = ConstructInputReducedShapeAndCastInputIrArrayToShape(
+      *unnested_hlo, param_arrays, param_shmem_buffers, reduced_dims,
+      &param_reduced_shapes, &param_in_reduced_shape_arrays);
+  DCHECK_EQ(num_shapes, num_params);
 
   // Calculate the starting element coordinate within a tile for the current
   // thread, (y, x) from thread_id.
   llvm::Value* x;
   llvm::Value* y;
-  std::tie(y, x) = CalculateYXCoordinateWithinTile(
-      &b_, index_typed_constant(kTileSize), kThreadsPerTile);
-
-  // Calculate the index for the current output tile from block_id.
-  const IrArray::Index output_tile_index(
-      GetBlockIdx(&b_, index_ty, num_tiles),
-      ShapeUtil::MakeShapeWithDescendingLayout(PRED /*arbitrary*/,
-                                               output_dims_in_tiles),
-      &b_);
-
-  // Output tile origin is the index for the first element of the current output
-  // tile.
-  const IrArray::Index output_tile_origin = [&] {
-    IrArray::Index index = output_tile_index;
-    for (int i = 1; i < 3; ++i) {
-      index[i] = Mul(output_tile_index[i], index_typed_constant(kTileSize),
-                     "tile_origin." + std::to_string(i));
-    }
-    return index;
-  }();
+  std::tie(y, x) = mapping_scheme->EmitThreadYXCoordinate(index_ty);
 
-  // Calculate the input tile origin from the output tile origin.
-  const IrArray::Index input_tile_origin(
-      Permute({0, 2, 1}, output_tile_origin.multidim()));
-
-  // Calculate the current output tile bounds in each of the logical dimensions.
-  std::vector<llvm::Value*> output_tile_bounds(3);
-  for (int i = 1; i < 3; ++i) {
-    // Only last row or column may not have full size.
-    output_tile_bounds[i] =
-        Select(ICmpEQ(output_tile_index[i],
-                      index_typed_constant(output_dims_in_tiles[i] - 1)),
-               index_typed_constant(reduced_output_dims[i] -
-                                    (output_dims_in_tiles[i] - 1) * kTileSize),
-               index_typed_constant(kTileSize), "kTileSize");
-  }
+  kernel_info->SetLaneId(
+      mapping_scheme->GetNumberOfThreadsForDimensionX() == kWarpSize ? x
+                                                                     : nullptr);
 
   KernelSupportLibrary ksl(&b_, llvm_ir::UnrollMode::kDefaultUnroll);
-
   // Curry a few parameters to EmitTiledElementalCodeWithBoundsCheck.
   auto emit_tiled_elemental_code_with_bounds_check =
       [&](const IrArray::Index& index, const string& loop_name,
-          llvm::Value* tile_width, llvm::Value* tile_height,
-          const std::function<void(const IrArray::Index&, llvm::Value*)>&
-              emit_elem_function) {
-        EmitTiledElementalCodeWithBoundsCheck(
-            kTileSize, kNumRows, index, loop_name, &ksl, &b_, y, x, tile_width,
-            tile_height, emit_elem_function);
+          llvm::Value* tile_height, llvm::Value* tile_width,
+          const std::function<void(const IrArray::Index&, llvm::Value*,
+                                   llvm::Value*)>& emit_elem_function) {
+        EmitTiledElementalCodeWithBoundsCheck(mapping_scheme, index, loop_name,
+                                              &ksl, &b_, y, x, tile_height,
+                                              tile_width, emit_elem_function);
       };
 
-  // Adds `addend` to the given `dim` of `index`.
-  auto offset_dim = [&](IrArray::Index index, llvm::Value* addend, int64 dim) {
-    index[dim] = Add(index[dim], addend);
-    return index;
-  };
-  const IrArray::Index input_index =
-      offset_dim(offset_dim(input_tile_origin, x, /*dim=*/2), y, /*dim=*/1);
-
-  // Copy input parameter values to shared memory buffers:
-  // tile[y, x] = input[index]
-  emit_tiled_elemental_code_with_bounds_check(
-      input_index, "input", output_tile_bounds[1], output_tile_bounds[2],
-      [&](const IrArray::Index& index, llvm::Value* y_loc) {
-        for (int64 id : tiled_param_ids) {
-          IrArray& input_in_logical_shape = param_in_reduced_shape_arrays[id];
-          llvm::Value* shmem_buffer = param_shmem_buffers[id];
-          // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
-          // global variables, so LLVM can't infer much about it.
-          Store(input_in_logical_shape.EmitReadArrayElement(index, &b_,
-                                                            "input_element"),
-                GEP(shmem_buffer, {index_typed_constant(0), y_loc, x}));
-        }
-      });
+  auto emit_one_tile = [&](const IrArray::Index& output_tile_origin,
+                           absl::Span<llvm::Value* const> output_tile_bounds,
+                           bool block_contains_multi_tiles) {
+    // Calculate the input tile origin from the output tile origin.
+    const IrArray::Index input_tile_origin(
+        Permute({0, 2, 1}, output_tile_origin.multidim()));
+
+    const IrArray::Index input_index =
+        input_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
+            .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
+
+    // Copy input parameter values to shared memory buffers:
+    // tile[y, x] = input[index]
+    // Note that tile_width and tile_height are flipped here because we are
+    // reading a transposed tile.
+    emit_tiled_elemental_code_with_bounds_check(
+        input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
+        [&](const IrArray::Index& index, llvm::Value* y_loc,
+            llvm::Value* x_loc) {
+          for (int64 id : tiled_param_ids) {
+            IrArray& input_in_logical_shape = param_in_reduced_shape_arrays[id];
+            llvm::Value* shmem_buffer = param_shmem_buffers[id];
+            // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
+            // global variables, so LLVM can't infer much about it.
+            Store(input_in_logical_shape.EmitReadArrayElement(index, &b_,
+                                                              "input_element"),
+                  GEP(shmem_buffer, {index_typed_constant(0), y_loc, x_loc}));
+          }
+        });
 
-  // Wait for all threads to reach this point, lest we copy a value from tile to
-  // output before the other thread copies it from input to tile.
-  // This is `__syncthreads` in CUDA.
-  llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_);
+    // If shared memory transpose is needed, wait for all threads to reach this
+    // point, lest we copy a value from tile to output before the other thread
+    // copies it from input to tile. This is `__syncthreads` in CUDA.
+    if (!tiled_param_ids.empty()) {
+      llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_);
+    }
 
-  llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x);
+    llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x);
+    kernel_info->SetTiledParamInfo(&tiled_param_info);
 
-  const IrArray::Index output_index =
-      offset_dim(offset_dim(output_tile_origin, x, /*dim=*/2), y, /*dim=*/1);
+    const IrArray::Index output_index =
+        output_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
+            .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
 
-  // Write to output[index] by emitting code like normal, except that values for
-  // the tiled parameters are read from the shmem buffers.
-  if (hlo->opcode() == HloOpcode::kCopy) {
+    // Write to output[index] by emitting code like normal, except that values
+    // for the tiled parameters are read from the shmem buffers.
     emit_tiled_elemental_code_with_bounds_check(
-        output_index, "output", output_tile_bounds[2], output_tile_bounds[1],
-        [&](const IrArray::Index& index, llvm::Value* y_loc) {
-          // TODO(jlebar): Add AA metadata to this load.
-          llvm::Instruction* load_from_shmem_buffer =
-              Load(GEP(param_shmem_buffers[0], {b_.getInt64(0), x, y_loc}),
-                   "output_element");
-          output_in_reduced_shape_arrays[0].EmitWriteArrayElement(
-              index, load_from_shmem_buffer, &b_);
-        });
-  } else {
-    CHECK_EQ(hlo->opcode(), HloOpcode::kFusion);
-    emit_tiled_elemental_code_with_bounds_check(
-        output_index, "output", output_tile_bounds[2], output_tile_bounds[1],
-        [&](const IrArray::Index& index, llvm::Value* y_loc) {
-          GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_,
-                                             GetNestedComputer());
-          FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(hlo),
-                                       &elem_emitter);
-          tiled_param_info.set_y(y_loc);
-          fused_emitter.SetTiledParameterInfo(&tiled_param_info);
-          TF_CHECK_OK(hlo->fused_expression_root()->Accept(&fused_emitter));
-          IrArray::Index untiled_index = llvm_ir::GetUnreducedOutputIndex(
-              index, output_reduced_shapes[0], output_arrays[0].GetShape(),
-              &b_);
-          const llvm_ir::ElementGenerator& output_generator =
-              fused_emitter.GetRootGenerator();
-          llvm::Value* output_value =
-              output_generator(untiled_index).ValueOrDie();
-          if (hlo->IsMultiOutputFusion()) {
-            CHECK(output_value->getType()->isStructTy());
-            CHECK_EQ(output_value->getType()->getStructNumElements(),
-                     output_in_reduced_shape_arrays.size());
-            for (int64 i = 0; i < output_in_reduced_shape_arrays.size(); ++i) {
-              output_in_reduced_shape_arrays[i].EmitWriteArrayElement(
-                  index, ExtractValue(output_value, i), &b_);
-            }
-          } else {
-            output_in_reduced_shape_arrays[0].EmitWriteArrayElement(
-                index, output_value, &b_);
-          }
+        output_index, "output", output_tile_bounds[1], output_tile_bounds[2],
+        [&](const IrArray::Index& index, llvm::Value* y_loc,
+            llvm::Value* x_loc) {
+          kernel_generator.GetTileElementGenerator()(unnested_hlo, index,
+                                                     kernel_info, y_loc, x_loc);
         });
+    // If a tile block contains multiple tiles and shared memory buffers are
+    // used, we need to wait for all threads to finish using the shared memory
+    // buffer for the current tile before we move on to process the next tile
+    // and overwrite the shared memory buffers.
+    if (block_contains_multi_tiles && !tiled_param_ids.empty()) {
+      llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_);
+    }
+  };
+
+  const BlockPrologueGenerator& block_prologue_generator =
+      kernel_generator.GetBlockPrologueGenerator();
+  if (block_prologue_generator) {
+    block_prologue_generator(unnested_hlo, kernel_info);
   }
 
-  // For multioutput fusion, emit a tuple with all the individual outputs.
-  if (hlo->IsMultiOutputFusion()) {
-    llvm_ir::EmitTuple(GetIrArray(*hlo, *hlo), output_arrays, &b_, module_);
+  EmitBlock(std::move(emit_one_tile), kernel_info, ksl, index_ty);
+
+  const BlockEpilogueGenerator& block_epilogue_generator =
+      kernel_generator.GetBlockEpilogueGenerator();
+  if (block_epilogue_generator) {
+    block_epilogue_generator(unnested_hlo, kernel_info);
+  }
+
+  // For multioutput fusion, emit a tuple with pointers to all the individual
+  // outputs.
+  if (unnested_hlo->IsMultiOutputFusion()) {
+    std::vector<IrArray> output_arrays =
+        ConstructIrArrayForOutputs(*unnested_hlo);
+    llvm_ir::EmitTuple(GetIrArray(*unnested_hlo, *unnested_hlo), output_arrays,
+                       &b_, module_);
   }
 
   return launch_dimensions;
 }
 
+// Emits a kernel for the given hlo instruction using a tiled 0-2-1 transpose
+// algorithm to improve the memory access patterns for the input parameters
+// with a shape that is a 0-2-1 transpose of the output tensor shape.
+//
+// For the purpose of tiling, the output tensors have a logical shape of three
+// components 0-2-1 while the relevant input parameters have a logical shape
+// of three components 0-1-2 in the order major to minor. The x- and y-
+// dimensions of the tensors are tiled in square tiles with an edge length
+// `kTileSize`. Each thread block of `kTileSize` x `kNumRows` threads
+// transposes one tile: each thread copies kTileSize/kNumRows elements from
+// the input to a shared memory tile, then the otherwise "regular HLO kernel"
+// reads from the shared memory instead of the original input.
+//
+// This is similar to the following CUDA algorithm in TensorFlow:
+// https://goo.gl/MStRV6.
+//
+// `kTileSize` should usually be same as warp size. We currently choose 32 for
+// `kTileSize` and 4 for `kNumRows`. The CUDA algorithm uses 8 for `kNumRows`.
+//
+// TODO(b/33320379): Here each block transposes 1 tile. It may be more
+// efficient to launch fewer blocks so each transposes many tiles.
+LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
+    HloInstruction* hlo, absl::Span<const int64> reduced_output_dims,
+    absl::Span<const int64> tiled_param_ids) {
+  constexpr int kNumRows = 4;
+  KernelMappingScheme mapping_scheme(
+      reduced_output_dims, /*tile_size_y=*/kWarpSize,
+      /*tile_size_x=*/kWarpSize, /*req_block_sizes=*/{2, 2, 2},
+      /*num_threads_y=*/kNumRows,
+      /*num_threads_x=*/kWarpSize, &b_);
+  TileElementGenerator element_generator;
+  if (hlo->opcode() == HloOpcode::kCopy) {
+    element_generator = [&](HloInstruction* hlo,
+                            const llvm_ir::IrArray::Index& index,
+                            const KernelCodegenInfo* kernel_info,
+                            llvm::Value* y_loc, llvm::Value* x_loc) {
+      EmitTileElementForCopy(hlo, index, kernel_info, y_loc, x_loc);
+    };
+  } else {
+    DCHECK_EQ(hlo->opcode(), HloOpcode::kFusion);
+    element_generator = [&](HloInstruction* hlo,
+                            const llvm_ir::IrArray::Index& index,
+                            const KernelCodegenInfo* kernel_info,
+                            llvm::Value* y_loc, llvm::Value* x_loc) {
+      EmitTileElementForFusion(hlo, index, kernel_info, y_loc, x_loc);
+    };
+  }
+  KernelCodegenInfo kernel_info(&mapping_scheme);
+  KernelCodeGenerator kernel_generator(std::move(element_generator));
+  return EmitKernel(hlo, tiled_param_ids, kernel_generator, &kernel_info);
+}
+
 namespace {
 // Returns true to indicate it is safe to use the tile based shared memory
 // transpose implementation to implement the kernel for the instruction.
@@ -3562,8 +3736,8 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
                                   ? ShapeUtil::GetSubshape(hlo->shape(), {0})
                                   : hlo->shape();
 
-  // If the output_shape is reduced to 021 shape, find all the parameters of the
-  // hlo that are in the corresponding 012 shape.
+  // If the output_shape is reduced to 021 shape, find all the parameters of
+  // the HLO that are in the corresponding 012 shape.
   std::vector<int64> params_012;
   optional<std::vector<int64>> reduced_dims_021;
   for (int64 operand_idx = 0; operand_idx < hlo->operand_count();
@@ -3600,9 +3774,9 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
   }
 
   // Each of our shared memory tiles has 32*33 elements (so ~4kb, if the
-  // elements are of size 4 bytes), and CUDA has an architectural limit of 48kb
-  // shared memory per SM.  (This is increased to 96kb in Volta, but we don't
-  // use this, in part because it eats into our L1 cache space.)
+  // elements are of size 4 bytes), and CUDA has an architectural limit of
+  // 48kb shared memory per SM.  (This is increased to 96kb in Volta, but we
+  // don't use this, in part because it eats into our L1 cache space.)
   //
   // For correctness we need to ensure that we don't make more than 48kb worth
   // of shmem tiles per block.  And for performance, we'd probably like to use
@@ -3610,9 +3784,9 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
   // gpu core.
   //
   // We say without benchmarks that we want at least 3 threads/block,
-  // corresponding to 3 shmem tiles if the elements are 32 bits wide.  We choose
-  // which params get the shmem transpose treatment arbitrarily; it's not clear
-  // if there's a Right Choice.
+  // corresponding to 3 shmem tiles if the elements are 32 bits wide.  We
+  // choose which params get the shmem transpose treatment arbitrarily; it's
+  // not clear if there's a Right Choice.
   //
   // This is only sound if tiled transposes are the only place where we use
   // shared memory in fusions.  If in the future other fusible ops use shared
@@ -3666,10 +3840,10 @@ Status IrEmitterUnnested::EmitConstantGlobals() {
     }
 
     // These globals will be looked up by name by GpuExecutable so we need to
-    // give them an external linkage.  Not all of their uses are visible in the
-    // LLVM IR (e.g. TupleThunk) so we can't give then a linkage that merely
-    // preserves their names (like available_externally), we also need to ensure
-    // that they stick around even if they're "unused".
+    // give them an external linkage.  Not all of their uses are visible in
+    // the LLVM IR (e.g. TupleThunk) so we can't give then a linkage that
+    // merely preserves their names (like available_externally), we also need
+    // to ensure that they stick around even if they're "unused".
     //
     // We may have to be more more clever here in the future if we notice that
     // we're keeping around too many globals because of their linkage.
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index 334c0b3c20..aaad385d1d 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h"
 #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/thunk.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h"
 
 namespace xla {
@@ -47,6 +48,94 @@ namespace gpu {
 //
 class IrEmitterUnnested : public IrEmitter {
  public:
+  // Parameter block_contains_multi_tiles indicates whether a tile block
+  // consists of multiple tiles or not. If the tile block contains only one
+  // tile, there is no need to use atomic operation to accumulate a local result
+  // to a global result to implement reduction.
+  using TileGenerator =
+      std::function<void(const llvm_ir::IrArray::Index& output_tile_origin,
+                         absl::Span<llvm::Value* const> output_tile_bounds,
+                         bool block_contains_multi_tiles)>;
+  // KernelCodegenInfo records the common information to support the code
+  // generation for a kernel to process tensor elements by blocks. A block of
+  // tensor elements may contain one or multiple tiles. The code generators that
+  // generate code for tile elements or block prologue/epilogue refer to this
+  // class in their prototypes. If the implementations of such code generators
+  // require other information that are specific to the HLO instructions, the
+  // implementations need to define and use derived classes of this class.
+  class KernelCodegenInfo {
+   public:
+    explicit KernelCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme)
+        : mapping_scheme_(mapping_scheme),
+          tiled_param_info_(nullptr),
+          lane_id_(nullptr) {}
+
+    void SetLaneId(llvm::Value* v) { lane_id_ = v; }
+    void SetTiledParamInfo(llvm_ir::TiledParameterInfo* tiled_param_info) {
+      CHECK_EQ(tiled_param_info_, nullptr);
+      tiled_param_info_ = tiled_param_info;
+    }
+
+    llvm::Value* GetLaneId() const { return lane_id_; }
+    llvm_ir::KernelMappingScheme* GetKernelMappingScheme() const {
+      return mapping_scheme_;
+    }
+    llvm_ir::TiledParameterInfo* GetTiledParameterInfo() const {
+      return tiled_param_info_;
+    }
+
+   private:
+    llvm_ir::KernelMappingScheme* mapping_scheme_;
+    llvm_ir::TiledParameterInfo* tiled_param_info_;
+    llvm::Value* lane_id_;
+  };
+
+  // A function object to prepare for the code generation for a tile block.
+  using BlockPrologueGenerator =
+      std::function<void(HloInstruction* hlo, KernelCodegenInfo* kernel_info)>;
+  // A function object to finalize the code generation for a tile block.
+  using BlockEpilogueGenerator =
+      std::function<void(HloInstruction* hlo, KernelCodegenInfo* kernel_info)>;
+  // A function object to generate code to process one element in a tile.
+  //
+  // hlo: the instruction for which the code is generated for.
+  // index: the index for the first output element of the current thread.
+  // y_loc: The y coordinate within a tile.
+  // x_loc: The x coordinate within a tile.
+  // kernel_info: Other information to support the kernel code generation.
+  using TileElementGenerator = std::function<void(
+      HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
+      const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+      llvm::Value* x_loc)>;
+
+  // KernelCodeGenerator records the code generator objects that generate code
+  // for tile elements or tile block prologue/epilogue.
+  class KernelCodeGenerator {
+   public:
+    explicit KernelCodeGenerator(
+        TileElementGenerator tile_element_generator,
+        BlockPrologueGenerator block_prologue_generator = {},
+        BlockEpilogueGenerator block_epilogue_generator = {})
+        : tile_element_generator_(std::move(tile_element_generator)),
+          block_epilogue_generator_(std::move(block_epilogue_generator)),
+          block_prologue_generator_(std::move(block_prologue_generator)) {}
+
+    const TileElementGenerator& GetTileElementGenerator() const {
+      return tile_element_generator_;
+    }
+    const BlockPrologueGenerator& GetBlockPrologueGenerator() const {
+      return block_prologue_generator_;
+    }
+    const BlockEpilogueGenerator& GetBlockEpilogueGenerator() const {
+      return block_epilogue_generator_;
+    }
+
+   private:
+    TileElementGenerator tile_element_generator_;
+    BlockPrologueGenerator block_prologue_generator_;
+    BlockEpilogueGenerator block_epilogue_generator_;
+  };
+
   IrEmitterUnnested(const HloModuleConfig& hlo_module_config,
                     const HloComputation* hlo_computation,
                     IrEmitterContext* ir_emitter_context);
@@ -205,22 +294,32 @@ class IrEmitterUnnested : public IrEmitter {
   LaunchDimensions EmitHlo021Tile(HloInstruction* hlo,
                                   absl::Span<const int64> reduced_output_dims,
                                   absl::Span<const int64> tiled_param_ids);
+  // Emits a kernel for an unnested HLO instruction.
+  LaunchDimensions EmitKernel(HloInstruction* unnested_hlo,
+                              absl::Span<const int64> param_ids,
+                              const KernelCodeGenerator& kernel_generator,
+                              KernelCodegenInfo* kernel_info);
+  void EmitBlock(const TileGenerator& emit_one_tile,
+                 const KernelCodegenInfo* kernel_info,
+                 KernelSupportLibrary& ksl, llvm::Type* index_ty);
+  // Emits code to process a tensor element in a tile for the given kCopy HLO
+  // that performs a 0-2-1 transpose.
+  void EmitTileElementForCopy(HloInstruction* hlo,
+                              const llvm_ir::IrArray::Index& index,
+                              const KernelCodegenInfo* kernel_info,
+                              llvm::Value* y_loc, llvm::Value* x_loc);
+  // Emits code to process a tensor element in a tile for the given kLoop fusion
+  // HLO containing parameters that are 0-2-1 transpose of its outputs.
+  void EmitTileElementForFusion(HloInstruction* hlo,
+                                const llvm_ir::IrArray::Index& index,
+                                const KernelCodegenInfo* kernel_info,
+                                llvm::Value* y_loc, llvm::Value* x_loc);
 
   // Generates the IrArray for each input of an hlo and returns a vector that
   // constains such IrArrays.
   std::vector<llvm_ir::IrArray> ConstructIrArrayForInputs(
       const HloInstruction& hlo);
 
-  // For each output of the `hlo` instruction, constructs the reduced shape for
-  // the output with the given `reduced_output_dims` and cast the original
-  // output IrArray element in `output_arrays` to the reduced shape. Returns
-  // the number of outputs.
-  int ConstructOutputReducedShapeAndCastOutputIrArrayToShape(
-      const HloInstruction& hlo,
-      const std::vector<llvm_ir::IrArray>& output_arrays,
-      absl::Span<const int64> reduced_output_dims,
-      std::vector<Shape>* output_reduced_shapes,
-      std::vector<llvm_ir::IrArray>* output_in_reduced_shape_arrays);
   // For each input of the `hlo` instruction, checks its value in
   // `param_buffers` to find out whether the input has a reduced shape. If the
   // input has a reduced shape, constructs the reduced shape for the input and
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index f4b05f29c3..1540a40ef8 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -108,6 +108,14 @@ class IrArray {
     Index(absl::Span<llvm::Value* const> multidim, llvm::Value* linear,
           const Shape& shape);
 
+    // Returns an index that adds `addend` to the given `dim` of the object.
+    Index AddOffsetToDim(llvm::Value* addend, int64 dim,
+                         llvm::IRBuilder<>* b) const {
+      IrArray::Index index = *this;
+      index[dim] = b->CreateAdd(index[dim], addend);
+      return index;
+    }
+
     const std::vector<llvm::Value*>& multidim() const { return multidim_; }
     llvm::Value* linear() const { return linear_; }
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index e5fbdbd51b..286b987d51 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -113,5 +113,117 @@ IrArray::Index GetUnreducedOutputIndex(
   return IrArray::Index(linear_index, unreduced_output_shape, b);
 }
 
+KernelMappingScheme::KernelMappingScheme(
+    absl::Span<const int64> dims_in_elems, int64 tile_size_y, int64 tile_size_x,
+    absl::Span<const int64> req_block_sizes, int64 num_threads_y,
+    int64 num_threads_x, llvm::IRBuilder<>* b)
+    : b_(b),
+      dims_in_elems_(dims_in_elems),
+      tile_sizes_{1, tile_size_y, tile_size_x},
+      num_threads_x_(num_threads_x),
+      num_threads_y_(num_threads_y) {
+  DCHECK_EQ(dims_in_elems_.size(), 3);
+  DCHECK_EQ(req_block_sizes.size(), 3);
+
+  DCHECK_EQ(tile_size_y % num_threads_y_, 0);
+  DCHECK_EQ(tile_size_x % num_threads_x_, 0);
+
+  dims_in_tiles_ = ElementWiseCeilOfRatio<int64>(dims_in_elems_, tile_sizes_);
+  block_sizes_.reserve(req_block_sizes.size());
+  absl::c_transform(req_block_sizes, dims_in_tiles_,
+                    std::back_inserter(block_sizes_),
+                    [](const int64 requested_size, const int64 max_size) {
+                      return std::min(requested_size, max_size);
+                    });
+  dims_in_blocks_ = ElementWiseCeilOfRatio<int64>(dims_in_tiles_, block_sizes_);
+
+  VLOG(10) << "dims_in_elems_ = [" << absl::StrJoin(dims_in_elems_, ",") << "]";
+  VLOG(10) << "dims_in_tiles_ = [" << absl::StrJoin(dims_in_tiles_, ",") << "]";
+  VLOG(10) << "dims_in_blocks_ = [" << absl::StrJoin(dims_in_blocks_, ",")
+           << "]";
+}
+
+IrArray::Index KernelMappingScheme::GetReshapedOutputIndex(
+    const IrArray::Index& output_index, const Shape& reshaped_output_shape) {
+  DCHECK_EQ(output_index.size(), dims_in_elems_.size());
+  Shape output_shape = ShapeUtil::MakeShapeWithDescendingLayout(
+      reshaped_output_shape.element_type(), GetDimensionsInElements());
+  return llvm_ir::GetUnreducedOutputIndex(output_index, output_shape,
+                                          reshaped_output_shape, b_);
+}
+
+IrArray::Index KernelMappingScheme::EmitBlockIndex(llvm::Type* index_ty) {
+  llvm::Value* block_id = llvm_ir::EmitCallToIntrinsic(
+      llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, b_);
+  llvm_ir::AddRangeMetadata(0, GetNumberOfBlocks(),
+                            llvm::cast<llvm::Instruction>(block_id));
+  llvm::Value* linear_block_id =
+      b_->CreateIntCast(block_id, index_ty, /*isSigned=*/true, "block.id.x");
+  return IrArray::Index(linear_block_id,
+                        ShapeUtil::MakeShapeWithDescendingLayout(
+                            PRED /*arbitrary*/, dims_in_blocks_),
+                        b_);
+}
+
+IrArray::Index KernelMappingScheme::GetTileIndexForBlockOrigin(
+    const IrArray::Index& block_index) {
+  IrArray::Index tile_index = block_index;
+  for (int i = 0; i < block_sizes_.size(); ++i) {
+    tile_index[i] = b_->CreateMul(
+        block_index[i],
+        llvm::ConstantInt::get(block_index[i]->getType(), block_sizes_[i]),
+        "block_origin." + std::to_string(i));
+  }
+  return tile_index;
+}
+
+IrArray::Index KernelMappingScheme::GetElementIndexForTileOrigin(
+    const IrArray::Index& tile_index) {
+  IrArray::Index elem_index = tile_index;
+  for (int i = DimY; i < DimTot; ++i) {
+    elem_index[i] =
+        b_->CreateMul(tile_index[i],
+                      llvm::ConstantInt::get(tile_index[i]->getType(),
+                                             GetTileSizeForDimension(i)),
+                      "tile_origin." + std::to_string(i));
+  }
+  return elem_index;
+}
+
+llvm::GlobalVariable* KernelMappingScheme::GetSharedMemoryBufferForElementType(
+    llvm::Type* elem_ty, absl::string_view buffer_name) {
+  // If shared memory tranpose is needed, we use square tiles.
+  CHECK_EQ(GetTileSizeForDimensionX(), GetTileSizeForDimensionY());
+
+  // For Nvidia GPUs, the warp size is 32 threads and the shared memory bank is
+  // organized into 32-way. We usually use the warp size or a multiplier or a
+  // the warp size as the size for tiling. This may cause all elements in the
+  // same column of a tile use the same memory bank and therefore shared memory
+  // bank conflicts. Adding 1 to the minor dimension of the shared memory buffer
+  // can reduce such shared memory bank conflicts.
+  llvm::Type* buffer_type = llvm::ArrayType::get(
+      llvm::ArrayType::get(elem_ty, GetTileSizeForDimension(DimX) + 1),
+      GetTileSizeForDimension(DimY));
+  return llvm_ir::AllocateSharedMemoryTile(b_->GetInsertBlock()->getModule(),
+                                           buffer_type, buffer_name);
+}
+
+std::tuple<llvm::Value*, llvm::Value*>
+KernelMappingScheme::EmitThreadYXCoordinate(llvm::Type* index_ty) {
+  // Calculate (y, x) coordinate of the thread in the 2D view of thread block
+  // defined by (num_thread_y, num_thread_x) from thread_id.
+  llvm::CallInst* thread_id_raw = llvm_ir::EmitCallToIntrinsic(
+      llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b_);
+  llvm_ir::AddRangeMetadata(0, GetThreadsPerTile(), thread_id_raw);
+  llvm::Value* thread_id_int =
+      b_->CreateIntCast(thread_id_raw, index_ty,
+                        /*isSigned=*/true, "thread.id.x");
+  llvm::Value* num_thread_x =
+      llvm::ConstantInt::get(index_ty, GetNumberOfThreadsForDimensionX());
+  llvm::Value* x = b_->CreateURem(thread_id_int, num_thread_x);
+  llvm::Value* y = b_->CreateUDiv(thread_id_int, num_thread_x);
+  return std::make_tuple(y, x);
+}
+
 }  // namespace llvm_ir
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index 5ea05b3188..fc1fa9c3d2 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -46,6 +46,147 @@ IrArray::Index GetUnreducedOutputIndex(
     const Shape& reduced_output_shape, const Shape& unreduced_output_shape,
     llvm::IRBuilder<>* b);
 
+// A tile is a spatial subdivision of a tensor. We group tensor elements into
+// tiles so that we can launch kernels to process the tensor elements in blocks
+// of tiles.
+//
+// A kernel mapping scheme describes a method to partition the tensors accessed
+// by an unnested HLO instruction into tiles and blocks of tiles, and the
+// associated information to use hardware threads to process the tensor elements
+// in blocks of tiles.
+//
+// Currently, there are two main use cases for a tiling scheme. First, we
+// implement kernels with 0-2-1 memory transpose using shared memory to improve
+// memory access pattern. Second, we implement reduction to contiguous
+// dimensions in layout, with or without memory tranpsose, to achieve better
+// memory access pattern as well as to reduce the need numbers of executed
+// expensive instructions, such as thread synchronization related instructions
+// and atomic operations. For both use cases, we can apply a normalization to
+// the original tensors, to collapse contiguous dimensions for the same purpose
+// and produce normlized three dimensional tensors. For this reason, the tiling
+// scheme class only needs to handle normalized three dimensional tensors and
+// two dimensional tiles.
+//
+// The current implementation of the class is somewhat NVIDIA GPU oriented. This
+// situation can be improved when there is a need though. The idea of 0-2-1
+// transpose using shared memory can be found in the following CUDA algorithm in
+// TensorFlow: https://goo.gl/MStRV6.
+//
+// We use a thread block to process a tile because we want to use the HW thread
+// block synchronization primitives to synchronize the processing of all the
+// elements in the same tile. A thread block can be viewed as a two dimensional
+// array of threads, described by the number of threads for the Y and X
+// dimensions. A thread block (num_threads_y, num_threads_x) processes a tile of
+// (tile_size_y, tile_size_x) as follows: each thread in the thread block
+// processes one element in the tile so that all the threads in the thread block
+// together process a subdivision of the tile that has the same dimension as the
+// thread block array. Then the thread block moves on to process the next
+// subdivision of the tile until the whole tile is processed. Therefore, each
+// thread in the thread block processes
+// tile_size_x/num_threads_x * tile_size_y/num_threads_y elements in a tile.
+//
+// There are situations where we want a thread block to process multiple
+// tiles. We can't group those tiles into a bigger tiles because we limit a tile
+// to a two dimensional spatial subdivision of a tensor. For example, when we
+// use tiling to implement reduction with tranpose, we want the partial sum
+// produced by each thread to accumulate values for more elements before using
+// shlf_down and atomic_add instructions for further reduction, to amortize the
+// cost of such expensive instructions. The concept of tile block is introduced
+// for this purpose. A tile block is a three dimensional array of tiles, of
+// which some dimensions may be degenerated to only one tile.
+class KernelMappingScheme {
+ public:
+  enum { DimZ = 0, DimY, DimX, DimTot };
+
+ public:
+  explicit KernelMappingScheme(absl::Span<const int64> dims_in_elems,
+                               int64 tile_size_y, int64 tile_size_x,
+                               absl::Span<const int64> req_block_sizes,
+                               int64 num_threads_y, int64 num_threads_x,
+                               llvm::IRBuilder<>* b);
+
+  absl::Span<const int64> GetDimensionsInElements() const {
+    return dims_in_elems_;
+  }
+  absl::Span<const int64> GetDimensionsInTiles() const {
+    return dims_in_tiles_;
+  }
+  absl::Span<const int64> GetDimensionsInBlocks() const {
+    return dims_in_blocks_;
+  }
+
+  int64 GetNumberOfTilesInTotal() const {
+    return absl::c_accumulate(dims_in_tiles_, 1LL, std::multiplies<int64>());
+  }
+  int64 GetNumberOfTilesInOneBlock() const {
+    return absl::c_accumulate(block_sizes_, 1, std::multiplies<int64>());
+  }
+
+  int64 GetNumberOfBlocks() const {
+    return absl::c_accumulate(dims_in_blocks_, 1, std::multiplies<int64>());
+  }
+
+  int64 GetTileSizeForDimension(int d) const {
+    DCHECK(d >= DimZ && d <= DimX);
+    return tile_sizes_[d];
+  }
+  int64 GetTileSizeForDimensionX() const {
+    return GetTileSizeForDimension(DimX);
+  }
+  int64 GetTileSizeForDimensionY() const {
+    return GetTileSizeForDimension(DimY);
+  }
+
+  absl::Span<const int64> GetBlockSizes() const { return block_sizes_; }
+
+  int64 GetNumberOfThreadsForDimensionX() const { return num_threads_x_; }
+  int64 GetNumberOfThreadsForDimensionY() const { return num_threads_y_; }
+
+  int64 GetThreadsPerTile() const {
+    return GetNumberOfThreadsForDimensionX() *
+           GetNumberOfThreadsForDimensionY();
+  }
+
+  IrArray::Index EmitBlockIndex(llvm::Type* index_ty);
+  // Returns the index for the first tile in the block with the given block
+  // index.
+  IrArray::Index GetTileIndexForBlockOrigin(const IrArray::Index& block_index);
+  // Returns the index for the first element in the tile with the given tile
+  // index.
+  IrArray::Index GetElementIndexForTileOrigin(const IrArray::Index& tile_index);
+
+  std::tuple<llvm::Value*, llvm::Value*> EmitThreadYXCoordinate(
+      llvm::Type* index_ty);
+
+  IrArray::Index GetReshapedOutputIndex(const IrArray::Index& output_index,
+                                        const Shape& reshaped_output_shape);
+
+  llvm::GlobalVariable* GetSharedMemoryBufferForElementType(
+      llvm::Type* elem_ty, absl::string_view buffer_name);
+
+ private:
+  llvm::IRBuilder<>* b_;
+  // The number of elements in each dimension.
+  absl::Span<const int64> dims_in_elems_;
+
+  // The number of elements for each dimension of a tile.
+  std::vector<int64> tile_sizes_;
+  // The number of tiles in each dimension. It is computed from dims_in_elem_
+  // and tile_sizes_.
+  std::vector<int64> dims_in_tiles_;
+
+  // The number of tiles for each dimension of a tile block.
+  std::vector<int64> block_sizes_;
+  // The number of blocks in each dimension of a tile block. It is computed from
+  // dims_in_tile_ and block_sizes_.
+  std::vector<int64> dims_in_blocks_;
+
+  // Number of threads used to process elements in the X direction of a tile.
+  int64 num_threads_x_;
+  // Number of threads used to process elements in the Y direction of a tile.
+  int64 num_threads_y_;
+};
+
 // A class to represent information for tiled parameters to support IR emission
 // for 021 transpose.
 class TiledParameterInfo {
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index 8ce7416474..b015f4328a 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -387,6 +387,19 @@ T CeilOfRatio(T dividend, T divisor) {
   return tensorflow::MathUtil::CeilOfRatio<T>(dividend, divisor);
 }
 
+template <typename T>
+std::vector<T> ElementWiseCeilOfRatio(absl::Span<const T> dividends,
+                                      absl::Span<const T> divisors) {
+  std::vector<T> ceil_of_ratios;
+  CHECK_EQ(dividends.size(), divisors.size());
+  ceil_of_ratios.reserve(dividends.size());
+  absl::c_transform(dividends, divisors, std::back_inserter(ceil_of_ratios),
+                    [](const T dividend, const T divisor) {
+                      return CeilOfRatio<T>(dividend, divisor);
+                    });
+  return ceil_of_ratios;
+}
+
 // Rounds the value up to a multiple of the divisor by first calling CeilOfRatio
 // then multiplying by the divisor. For example: RoundUpToNearest(13, 8) => 16
 template <typename T>
-- 
GitLab


From be2e12057eb40c360c419f1fe1fc238a22e0501c Mon Sep 17 00:00:00 2001
From: Anders Huss <andhus@kth.se>
Date: Wed, 14 Nov 2018 08:38:43 +0100
Subject: [PATCH 0209/1554] fix coding style issues in new rnn tests

---
 tensorflow/python/keras/backend_test.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index edb2af7e01..99b53e69c3 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1248,18 +1248,18 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
     expected_outputs[1, -mask_last_num_timesteps:] = \
         expected_outputs[1, -(mask_last_num_timesteps + 1)]
 
-    expected_state = initial_state_vals.copy()
+    expected_last_state = initial_state_vals.copy()
     # first state should be incremented for every timestep (no masking)
-    expected_state[0] += num_timesteps
+    expected_last_state[0] += num_timesteps
     # second state should not be incremented for last two timesteps
-    expected_state[1] += (num_timesteps - mask_last_num_timesteps)
+    expected_last_state[1] += (num_timesteps - mask_last_num_timesteps)
 
     # verify same expected output for `unroll=true/false`
     inputs = keras.backend.variable(inputs_vals)
     initial_states = [keras.backend.variable(initial_state_vals)]
     mask = keras.backend.variable(mask_vals)
     for unroll in [True, False]:
-      last_output, outputs, last_states = keras.backend.rnn(
+      _, outputs, last_states = keras.backend.rnn(
           step_function,
           inputs,
           initial_states,
@@ -1267,10 +1267,9 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
           unroll=unroll,
           input_length=num_timesteps if unroll else None)
 
-      self.assertAllClose(
-          keras.backend.eval(outputs), expected_outputs)
-      self.assertAllClose(
-          keras.backend.eval(last_states[0]), expected_state)
+      self.assertAllClose(keras.backend.eval(outputs), expected_outputs)
+      self.assertAllClose(keras.backend.eval(
+          last_states[0]), expected_last_state)
 
   def test_rnn_output_num_dim_larger_than_2_masking(self):
     num_samples = 3
@@ -1298,7 +1297,7 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
     initial_states = [keras.backend.variable(initial_state_vals)]
     mask = keras.backend.variable(mask_vals)
     for unroll in [True, False]:
-      last_output, outputs, last_states = keras.backend.rnn(
+      _, outputs, _ = keras.backend.rnn(
           step_function,
           inputs,
           initial_states,
@@ -1313,7 +1312,7 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
     num_timesteps = 4
 
     def step_function(inputs, states):
-        return inputs, [s + 1 for s in states]
+      return inputs, [s + 1 for s in states]
 
     inputs_vals = np.random.random((num_samples, num_timesteps, 5))
     initial_state_vals = np.random.random((num_samples, 6, 7))
@@ -1328,7 +1327,7 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
     initial_states = [keras.backend.variable(initial_state_vals)]
     mask = keras.backend.variable(mask_vals)
     for unroll in [True, False]:
-      last_output, outputs, last_states = keras.backend.rnn(
+      _, _, last_states = keras.backend.rnn(
           step_function,
           inputs,
           initial_states,
@@ -1336,10 +1335,8 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
           unroll=unroll,
           input_length=num_timesteps if unroll else None)
 
-      # not updated last timestep:
       self.assertAllClose(
-          keras.backend.eval(last_states[0]),
-          expected_last_state)
+          keras.backend.eval(last_states[0]), expected_last_state)
 
   def test_normalize_batch_in_training(self):
     val = np.random.random((10, 3, 10, 10))
-- 
GitLab


From ad452dea61eab4e6441182b5fb5188349f5b2d54 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 14 Nov 2018 00:36:58 -0800
Subject: [PATCH 0210/1554] Add a test case for computing an inverse
 permutation with Scatter.

PiperOrigin-RevId: 221400568
---
 tensorflow/compiler/xla/tests/scatter_test.cc | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/scatter_test.cc b/tensorflow/compiler/xla/tests/scatter_test.cc
index 7e1f4aa0eb..32de0fdf78 100644
--- a/tensorflow/compiler/xla/tests/scatter_test.cc
+++ b/tensorflow/compiler/xla/tests/scatter_test.cc
@@ -129,6 +129,42 @@ ENTRY main {
   RunTest(hlo_text, &operand, &scatter_indices, &updates);
 }
 
+XLA_TEST_F(ScatterTest, TensorFlowScatterV2_InversePermutation) {
+  const char* hlo_text = R"(
+HloModule TensorFlowScatterV2
+
+update_s32 (lhs: s32[], rhs: s32[]) -> s32[] {
+  lhs = s32[] parameter(0)
+  ROOT rhs = s32[] parameter(1)
+}
+
+ENTRY main {
+  permutation = s32[3,4] parameter(0)
+  reshape = s32[3,4,1] reshape(permutation)
+  operand = s32[3,4] iota(), iota_dimension=1
+  updates = s32[3,4,1,1] iota(), iota_dimension=1
+  iota = s32[3,4,1] iota(), iota_dimension=0
+  indices = s32[3,4,2] concatenate(iota, reshape), dimensions={2}
+  ROOT scatter = s32[3,4] scatter(operand, indices, updates),
+      to_apply=update_s32,
+      update_window_dims={2,3},
+      inserted_window_dims={},
+      scatter_dims_to_operand_dims={0,1},
+      index_vector_dim=2
+}
+)";
+  Literal permutation =
+      LiteralUtil::CreateR2<int32>({{1, 3, 2, 0}, {3, 0, 2, 1}, {2, 3, 1, 0}});
+  HloModuleConfig config;
+  config.set_debug_options(GetDebugOptionsForTest());
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(hlo_text, config));
+  auto actual = ExecuteAndTransfer(std::move(module), {&permutation});
+  Literal expected =
+      LiteralUtil::CreateR2<int32>({{3, 0, 2, 1}, {1, 3, 2, 0}, {3, 2, 0, 1}});
+  EXPECT_TRUE(LiteralTestUtil::Equal(expected, actual));
+}
+
 XLA_TEST_F(ScatterTest, SimpleR4) {
   const char* hlo_text = R"(
 HloModule SimpleR4
-- 
GitLab


From e0c346f3ccb22264bd75f291c6b03314c33cd2ca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 00:59:24 -0800
Subject: [PATCH 0211/1554] Add tracing to XlaDevice::Sync, to help identify
 bottlenecks

PiperOrigin-RevId: 221402670
---
 tensorflow/compiler/jit/xla_device.cc | 57 ++++++++++++++++++++++++++-
 tensorflow/compiler/jit/xla_device.h  | 28 +++++++++++++
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 5c1b55cb57..738bac54cc 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -218,6 +218,9 @@ XlaDevice::XlaDevice(const SessionOptions& session_options,
 XlaDevice::~XlaDevice() {
   VLOG(1) << "Destroying XLA device " << jit_device_name_ << " " << this;
   mutex_lock lock(mu_);
+  while (outstanding_asynchronous_operations_ > 0) {
+    outstanding_asynchronous_operations_cv_.wait(lock);
+  }
   if (device_context_) {
     device_context_->Unref();
   }
@@ -384,6 +387,7 @@ void XlaDevice::ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context,
 
 Status XlaDevice::Sync() {
   VLOG(1) << "XlaDevice::Sync";
+  tracing::ScopedActivity activity("XlaDevice::Sync", /*is_expensive=*/true);
   std::shared_ptr<se::Stream> stream;
   {
     mutex_lock lock(mu_);
@@ -391,7 +395,15 @@ Status XlaDevice::Sync() {
   }
   if (!stream) return Status::OK();
 
-  if (!stream->parent()->SynchronizeAllActivity() || !stream->ok()) {
+  Status status = stream->BlockHostUntilDone();
+  {
+    mutex_lock lock(mu_);
+    while (outstanding_asynchronous_operations_ > 0) {
+      outstanding_asynchronous_operations_cv_.wait(lock);
+    }
+  }
+  TF_RETURN_IF_ERROR(status);
+  if (!stream->ok()) {
     return errors::Internal("XlaDevice::Sync() failed.");
   }
   VLOG(1) << "XlaDevice::Sync completed";
@@ -441,6 +453,49 @@ bool XlaDevice::RequiresSyncOnCompletion() const {
   return sync_on_completion_;
 }
 
+XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
+    XlaDevice* device)
+    : device_(device) {
+  mutex_lock lock(device_->mu_);
+  ++device_->outstanding_asynchronous_operations_;
+}
+
+XlaDevice::AsynchronousOperationHandle::~AsynchronousOperationHandle() {
+  if (device_) {
+    mutex_lock lock(device_->mu_);
+    --device_->outstanding_asynchronous_operations_;
+    device_->outstanding_asynchronous_operations_cv_.notify_all();
+  }
+}
+
+XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
+    const XlaDevice::AsynchronousOperationHandle& other)
+    : device_(other.device_) {
+  mutex_lock lock(device_->mu_);
+  ++device_->outstanding_asynchronous_operations_;
+}
+
+XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
+    XlaDevice::AsynchronousOperationHandle&& other)
+    : device_(other.device_) {
+  other.device_ = nullptr;
+}
+
+XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle::
+operator=(const XlaDevice::AsynchronousOperationHandle& other) {
+  device_ = other.device_;
+  mutex_lock lock(device_->mu_);
+  ++device_->outstanding_asynchronous_operations_;
+  return *this;
+}
+
+XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle::
+operator=(XlaDevice::AsynchronousOperationHandle&& other) {
+  device_ = other.device_;
+  other.device_ = nullptr;
+  return *this;
+}
+
 XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device,
                                                    const char* jit_device) {
   // Any op assigned to the device that isn't rewritten by the graph rewriter
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index 49f53b477e..dc8f49a9c9 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -164,7 +164,30 @@ class XlaDevice : public LocalDevice {
 
   bool RequiresSyncOnCompletion() const override LOCKS_EXCLUDED(mu_);
 
+  // A simple RAII handle. On construction the device's
+  // outstanding_asynchronous_operations_ field is incremented; on destruction
+  // it is decremented.
+  class AsynchronousOperationHandle {
+   public:
+    AsynchronousOperationHandle(XlaDevice* device);
+    ~AsynchronousOperationHandle();
+    AsynchronousOperationHandle(const AsynchronousOperationHandle& other);
+    AsynchronousOperationHandle(AsynchronousOperationHandle&& other);
+    AsynchronousOperationHandle& operator=(
+        const AsynchronousOperationHandle& other);
+    AsynchronousOperationHandle& operator=(AsynchronousOperationHandle&& other);
+
+   private:
+    XlaDevice* device_ = nullptr;
+  };
+
+  AsynchronousOperationHandle CreateAsynchronousOperationHandle() {
+    return AsynchronousOperationHandle(this);
+  }
+
  private:
+  friend class AsynchronousOperationHandle;
+
   xla::LocalClient* client() const;
   Allocator* GetAllocatorLocked(AllocatorAttributes attr)
       EXCLUSIVE_LOCKS_REQUIRED(mu_);
@@ -227,6 +250,11 @@ class XlaDevice : public LocalDevice {
   // True if the device requires XlaDevice::Sync to be called on completion
   // regardless of status.
   bool sync_on_completion_ GUARDED_BY(mu_) = false;
+
+  // Count of outstanding asynchronous operations which must be zero on Sync()
+  // completion.
+  int64 outstanding_asynchronous_operations_ GUARDED_BY(mu_) = 0;
+  condition_variable outstanding_asynchronous_operations_cv_;
 };
 
 // Builds OpKernel registrations on 'device' for the JIT operators
-- 
GitLab


From 71e0b26b568842846c5fca73f687387d7c7f6db8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 01:02:11 -0800
Subject: [PATCH 0212/1554] compat: Update forward compatibility horizon to
 2018-11-14

PiperOrigin-RevId: 221403204
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 385fd431f4..6a20e63e89 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 13)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 14)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 656b2fe018a7940595121ea08d4a1ddf29fa65d0 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Wed, 14 Nov 2018 04:51:28 -0800
Subject: [PATCH 0213/1554] Remove `DistributionStrategy.worker_device_index`.

PiperOrigin-RevId: 221425367
---
 .../python/keras_optimizer_v2_test.py         | 17 +++-
 .../distribute/python/mirrored_strategy.py    |  3 -
 .../python/mirrored_strategy_multigpu_test.py | 82 +++++++++----------
 .../python/mirrored_strategy_test.py          | 20 +++--
 .../distribute/python/one_device_strategy.py  |  3 -
 .../python/one_device_strategy_test.py        |  3 -
 .../distribute/python/strategy_test_lib.py    | 12 ---
 tensorflow/python/training/distribute.py      | 30 -------
 8 files changed, 64 insertions(+), 106 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 46a1cf41c5..e4e7717f8d 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -36,12 +36,16 @@ from tensorflow.python.estimator.canned import prediction_keys
 from tensorflow.python.estimator.export import export
 from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
+from tensorflow.python.training import distribution_strategy_context
 
 
 class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase):
@@ -142,11 +146,11 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
     if context.num_gpus() < 1:
       self.skipTest('Not enough GPUs.')
 
-    def create_fn(device_id):
+    def create_fn():
       var = variables.Variable(
           2.0, name='var', aggregation=variable_scope.VariableAggregation.SUM)
       # grad for cpu is 1, grad for gpu is 2, avg grad is 1.5.
-      loss = (device_id + 1) * var
+      loss = math_ops.cast(_replica_id() + 1, dtype=dtypes.float32) * var
       optimizer = adam.Adam(learning_rate=0.01, beta_1=0.2, beta_2=0.2)
       train_op = optimizer.minimize(loss, var_list=[var])
       m = optimizer.get_slot(var, 'm')
@@ -156,8 +160,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
     devices = ['/device:GPU:0', '/device:CPU:0']
     dist = mirrored_strategy.MirroredStrategy(devices)
     with dist.scope():
-      (var, m, v, op, counter) = dist.call_for_each_replica(
-          create_fn, args=[dist.worker_device_index])
+      (var, m, v, op, counter) = dist.call_for_each_replica(create_fn)
       self.evaluate(variables.global_variables_initializer())
       var_val = [2.0, 2.0, 2.0]
       self.assertAllClose(
@@ -233,5 +236,11 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
                           ]))
 
 
+def _replica_id():
+  # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
+  return constant_op.constant(
+      distribution_strategy_context.get_replica_context().replica_id)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 70855d8f01..62619a5756 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -662,9 +662,6 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
   def num_replicas_in_sync(self):
     return len(self._devices)
 
-  def _worker_device_index(self):
-    return self._device_index
-
   @property
   def worker_devices(self):
     # Make a copy to prevent users from accidentally mutating our copy.
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 2a6561e6b1..73614a000f 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -78,11 +78,6 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     self._test_minimize_loss_graph(
         self._get_distribution_strategy(), soft_placement=soft_placement)
 
-  def testDeviceIndex(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self._test_device_index(self._get_distribution_strategy())
-
   def testReplicaId(self):
     if not GPU_TEST:
       self.skipTest("Not GPU test")
@@ -103,34 +98,31 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
   @test_util.run_in_graph_and_eager_modes
   def testRunRegroupError(self):
 
-    def run_fn(device_id):
+    def run_fn():
+      replica_id = int(self.evaluate(_replica_id()))
       # Generates a list with different lengths on different devices.
       # Will fail in _regroup() (if more than one device).
-      return list(range(device_id))
+      return list(range(replica_id))
 
     dist = self._get_distribution_strategy()
     with dist.scope(), self.assertRaises(AssertionError):
-      dist.call_for_each_replica(run_fn, args=(dist.worker_device_index,))
+      dist.call_for_each_replica(run_fn)
 
   @test_util.run_in_graph_and_eager_modes
   def testReduceToCpu(self):
     if not GPU_TEST:
       self.skipTest("Not GPU test")
 
-    def run_fn(device_id):
-      return device_id
-
     dist = self._get_distribution_strategy()
     with dist.scope():
-      result = dist.call_for_each_replica(
-          run_fn, args=(dist.worker_device_index,))
+      result = dist.call_for_each_replica(_replica_id)
       reduced = dist.reduce(
           variable_scope.VariableAggregation.SUM,
           result,
           destinations="/device:CPU:0")
       unwrapped = dist.unwrap(reduced)
       self.assertEqual(1, len(unwrapped))
-      expected = sum(range(len(dist.worker_devices)))
+      expected = sum(range(dist.num_replicas_in_sync))
       self.assertEqual(expected, self.evaluate(unwrapped[0]))
 
   @test_util.run_in_graph_and_eager_modes
@@ -138,13 +130,12 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     if not GPU_TEST:
       self.skipTest("Not GPU test")
 
-    def run_fn(device_id):
-      return constant_op.constant(3 + 5 * device_id)
+    def run_fn():
+      return 3 + 5 * _replica_id()
 
     dist = self._get_distribution_strategy()
     with dist.scope():
-      result = dist.call_for_each_replica(
-          run_fn, args=(dist.worker_device_index,))
+      result = dist.call_for_each_replica(run_fn)
       reduced = dist.reduce(
           variable_scope.VariableAggregation.ONLY_FIRST_REPLICA,
           result,
@@ -274,8 +265,9 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
   def testVariableWithSameCanonicalNameAcrossThreads(self):
     self._skip_eager_if_gpus_less_than(1)
 
-    def model_fn(device_id):
-      v = variable_scope.variable(1.0, name="foo_" + str(device_id))
+    def model_fn():
+      replica_id = self.evaluate(_replica_id())
+      v = variable_scope.variable(1.0, name="foo_" + str(replica_id))
       distribution_strategy_context.get_replica_context().merge_call(
           lambda _: _)
       return v
@@ -284,8 +276,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         ["/device:GPU:0", "/device:CPU:0"])
 
     with dist.scope():
-      result = dist.call_for_each_replica(
-          model_fn, args=(dist.worker_device_index,))
+      result = dist.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       # The resulting mirrored variable will use the name from the first device.
       self.assertEquals("foo_0:0", result.name)
@@ -449,14 +440,16 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       self.assertEqual(3.0, self.evaluate(v1.get(devices[1])))
       self.assertEqual(3.0, self.evaluate(dist.read_var(v1)))
 
+      def replica_id_plus_one():
+        return math_ops.cast(_replica_id() + 1, dtype=dtypes.float32)
+
       # Update using the assign_add member function.
-      def update_member_fn(device_id):
-        update0 = v0.assign_add(5.0 * (device_id + 1))
-        update1 = v1.assign_add(7.0 * (device_id + 1))
+      def update_member_fn():
+        update0 = v0.assign_add(5.0 * replica_id_plus_one())
+        update1 = v1.assign_add(7.0 * replica_id_plus_one())
         return update0, update1
 
-      update0a, update1a = dist.call_for_each_replica(
-          update_member_fn, args=(dist.worker_device_index,))
+      update0a, update1a = dist.call_for_each_replica(update_member_fn)
 
       # Update "sync on read" variable.
       self.evaluate(dist.group(update0a))
@@ -476,13 +469,12 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       self.assertEqual(3.0 + 7.0, self.evaluate(dist.read_var(v1)))
 
       # Update using state_ops.assign_add global function.
-      def update_state_ops_fn(device_id):
-        update0 = state_ops.assign_add(v0, 11.0 * (device_id + 1))
-        update1 = state_ops.assign_add(v1, 13.0 * (device_id + 1))
+      def update_state_ops_fn():
+        update0 = state_ops.assign_add(v0, 11.0 * replica_id_plus_one())
+        update1 = state_ops.assign_add(v1, 13.0 * replica_id_plus_one())
         return update0, update1
 
-      update0b, update1b = dist.call_for_each_replica(
-          update_state_ops_fn, args=(dist.worker_device_index,))
+      update0b, update1b = dist.call_for_each_replica(update_state_ops_fn)
       self.evaluate(dist.group(update0b))
 
       # Update "sync on read" variable.
@@ -613,7 +605,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     components_sum = {}
     components_mean = {}
 
-    def model_fn(device_id):
+    def model_fn():
+      replica_id = self.evaluate(_replica_id())
       v_sum = variable_scope.variable(
           1.0,
           synchronization=variable_scope.VariableSynchronization.ON_READ,
@@ -624,14 +617,14 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
           aggregation=variable_scope.VariableAggregation.MEAN)
       self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable))
       self.assertTrue(isinstance(v_mean, values.ReplicaLocalVariable))
-      updates = [v_sum.assign_add(2.0 + device_id),
-                 v_mean.assign(6.0 * device_id)]
-      all_v_sum[device_id] = v_sum
-      all_v_mean[device_id] = v_mean
+      updates = [v_sum.assign_add(2.0 + replica_id),
+                 v_mean.assign(6.0 * replica_id)]
+      all_v_sum[replica_id] = v_sum
+      all_v_mean[replica_id] = v_mean
       c_sum = v_sum.get()
       c_mean = v_mean.get()
-      components_sum[device_id] = c_sum
-      components_mean[device_id] = c_mean
+      components_sum[replica_id] = c_sum
+      components_mean[replica_id] = c_mean
       self.assertIsNot(v_sum, c_sum)
       self.assertIsNot(v_mean, c_mean)
       return updates, v_sum, v_mean, c_sum, c_mean
@@ -642,8 +635,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     with dist.scope():
       # Create "sum" and "mean" versions of ReplicaLocalVariables.
       ret_ops, ret_v_sum, ret_v_mean, regrouped_sum, regrouped_mean = (
-          dist.call_for_each_replica(
-              model_fn, args=(dist.worker_device_index,)))
+          dist.call_for_each_replica(model_fn))
       # Should see the same wrapping instance in all replicas.
       self.assertIs(all_v_sum[0], ret_v_sum)
       self.assertIs(all_v_mean[0], ret_v_mean)
@@ -1205,7 +1197,7 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
       # values on each of the replicas.
       self.assertEqual(2.0, self.evaluate(dist.read_var(replica_local_var)))
       # Assigning 6.0 in cross replica context will assign a value of
-      # 6.0/num_replicas_in_sync to each replica.
+      # 6.0/num_replicas to each replica.
       tlv_ops = replica_local_var.assign(6.0)
       self.evaluate(tlv_ops)
       # On reading the replica local var we should get the assigned value back.
@@ -1469,5 +1461,11 @@ class MultiWorkerMirroredStrategyTestWithChief(
     self._test_minimize_loss_graph(strategy, learning_rate=0.05)
 
 
+def _replica_id():
+  # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
+  return constant_op.constant(
+      distribution_strategy_context.get_replica_context().replica_id)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
index bea684e77c..7c7026052d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -40,9 +40,6 @@ class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
   def testMinimizeLossGraph(self):
     self._test_minimize_loss_graph(self._get_distribution_strategy())
 
-  def testDeviceIndex(self):
-    self._test_device_index(self._get_distribution_strategy())
-
   def testReplicaId(self):
     self._test_replica_id(self._get_distribution_strategy())
 
@@ -57,11 +54,11 @@ class VariableCreatorStackTest(test.TestCase):
     devices = ["/device:CPU:0", "/device:GPU:0"]
     dist = mirrored_strategy.MirroredStrategy(devices)
 
-    def model_fn(device_id):
-      assert isinstance(device_id, int)
+    def model_fn():
+      replica_id_str = str(self.evaluate(_replica_id()))
 
       def thread_creator_fn(next_creator, *args, **kwargs):
-        return next_creator(*args, **kwargs) + ":thread_" + str(device_id)
+        return next_creator(*args, **kwargs) + ":thread_" + replica_id_str
 
       with variable_scope.variable_creator_scope(thread_creator_fn):
         # Create a variable in this scope.
@@ -80,11 +77,16 @@ class VariableCreatorStackTest(test.TestCase):
     with context.graph_mode(), \
         dist.scope(), \
         variable_scope.variable_creator_scope(main_thread_creator):
-      result = dist.call_for_each_replica(
-          model_fn, args=(dist.worker_device_index,))
+      result = dist.call_for_each_replica(model_fn)
       result = dist.unwrap(result)
       expected = ["main_thread:thread_0", "main_thread:thread_1"]
-      self.assertEquals(expected, result)
+      self.assertEqual(expected, result)
+
+
+def _replica_id():
+  # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
+  return constant_op.constant(
+      distribution_strategy_context.get_replica_context().replica_id)
 
 
 class MultiWorkerMirroredStrategyTest(test.TestCase):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 622194e8a2..d623798d0c 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -168,9 +168,6 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     del var_list
     return [self._device]
 
-  def _worker_device_index(self):
-    return 0
-
 
 class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
   """ReplicaContext for OneDeviceStrategy."""
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 95f4cdb786..79f9c39cc8 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -35,9 +35,6 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
   def testMinimizeLossGraph(self):
     self._test_minimize_loss_graph(self._get_distribution_strategy())
 
-  def testDeviceIndex(self):
-    self._test_device_index(self._get_distribution_strategy())
-
   def testReplicaId(self):
     self._test_replica_id(self._get_distribution_strategy())
 
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 3c0c10430e..9fee75a476 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -189,18 +189,6 @@ class DistributionTestBase(test.TestCase):
       # Error should go down
       self.assertLess(error_after, error_before)
 
-  def _test_device_index(self, d):
-    with d.scope():
-      expected_devices = [False] * len(d.worker_devices)
-
-      def mark_devices_fn(device_id):
-        self.assertLess(device_id, len(d.worker_devices))
-        self.assertFalse(expected_devices[device_id])
-        expected_devices[device_id] = True
-
-      d.call_for_each_replica(mark_devices_fn, args=(d.worker_device_index,))
-      self.assertAllEqual(expected_devices, [True] * len(d.worker_devices))
-
   def _test_replica_id(self, d):
     with d.scope():
       expected_devices = [False] * len(d.worker_devices)
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index f2f832f314..f1d2db0647 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -951,32 +951,6 @@ class DistributionStrategy(object):
     """
     raise NotImplementedError("must be implemented in descendants")
 
-  @property
-  def worker_device_index(self):
-    """An object mapping worker device to an id.
-
-    This might be passed as an argument to `call_for_each_replica()`, as in:
-
-    ```
-    with distribution_strategy.scope():
-
-      def fn(device_id):
-        # device_id is an integer. `fn` is being executed on device:
-        #    distribution_strategy.worker_devices[device_id].
-
-      distribution_strategy.call_for_each_replica(
-          fn, distribution_strategy.worker_device_index)
-    ```
-
-    Returns:
-      An index object, or the integer 0 if there is only a single replica.
-    """
-    _require_cross_replica_context(self)
-    return self._worker_device_index()
-
-  def _worker_device_index(self):
-    raise NotImplementedError("must be implemented in descendants")
-
   @property
   def between_graph(self):
     """Whether the strategy uses between-graph replication or not.
@@ -1232,10 +1206,6 @@ class _DefaultDistributionStrategy(DistributionStrategy):
   def non_slot_devices(self, var_list):
     return min(var_list, key=lambda x: x.name)
 
-  def _worker_device_index(self):
-    raise RuntimeError("worker_device_index() method unsupported by "
-                       "_DefaultDistributionStrategy.")
-
 
 # ------------------------------------------------------------------------------
 # We haven't yet implemented deserialization for DistributedVariables.
-- 
GitLab


From ada03a97ab77bdd58cd96d3a3fdca490278165d4 Mon Sep 17 00:00:00 2001
From: Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
Date: Wed, 14 Nov 2018 18:34:43 +0530
Subject: [PATCH 0214/1554] Fix a TypeError

We cannot concatenate string and FailedPreConditionError
---
 tensorflow/contrib/tpu/python/tpu/keras_support.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index 08f58a5f5b..ac72e3f55d 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -132,7 +132,7 @@ def _tpu_session_context():
 An error occurred connecting or initializing your TPU.
 
 The session has been reset. re-run keras_to_tpu_model to create a new session.
-""" + e)
+""" + str(e))
 
 
 def setup_tpu_session(cluster_resolver):
-- 
GitLab


From 0c8fa0351a2da1706e179571a9637dee55966e64 Mon Sep 17 00:00:00 2001
From: Vadim Borisov <pyybor@gmail.com>
Date: Wed, 14 Nov 2018 14:13:36 +0100
Subject: [PATCH 0215/1554] Update README.md

---
 tensorflow/contrib/quantize/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/quantize/README.md b/tensorflow/contrib/quantize/README.md
index a1f2b59026..9085d9fa71 100644
--- a/tensorflow/contrib/quantize/README.md
+++ b/tensorflow/contrib/quantize/README.md
@@ -28,7 +28,7 @@ Since it's difficult to add these fake quantization operations to all the
 required locations in the model, there's a function available that rewrites the
 training graph. To create a fake quantized training graph:
 
-```
+```python
 # Build forward pass of model.
 loss = tf.losses.get_total_loss()
 
@@ -51,7 +51,7 @@ The rewritten *eval graph* is non-trivially different from the *training graph*
 since the quantization ops affect the batch normalization step. Because of this,
 we've added a separate rewrite for the *eval graph*:
 
-```
+```python
 # Build eval model
 logits = tf.nn.softmax_cross_entropy_with_logits_v2(...)
 
-- 
GitLab


From ad0aff393ee07e60a8154c229876c090f4f48e29 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 14 Nov 2018 08:55:14 -0800
Subject: [PATCH 0216/1554] [XLA] Fix TODOs in xla/service/gpu.

- Reassign TODO assigned to xoogler.
- Remove ambiguous TODO that isn't clearly suggesting we do anything.

PiperOrigin-RevId: 221451978
---
 .../xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc   | 6 +++---
 tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc       | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
index 8751e3a9c2..364f69a69d 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
@@ -453,12 +453,12 @@ void GPUBackendInit(const HloModuleConfig& hlo_module_config) {
   // * 3-6 gives similar results as 2;
   // * >6 start hurting the performance of at least dot product kernels.
   //
-  // TODO(jingyue): The current threshold only considers the numbr of IR
+  // TODO(jingyue): The current threshold only considers the number of IR
   // instructions which do not accurately reflect the true cost. We need a
   // better cost model.
   FeedLLVMWithFlags({"-bonus-inst-threshold=2"});
-  // TODO(b/22073864): Increase limit when scan memory dependency.
-  // This helps to reduce more redundant load instructions.
+  // Increase limit when scanning memory dependencies.  This helps to reduce
+  // more redundant load instructions.
   //
   // The specific value is currently large enough for s3d in shoc benchmark,
   // which contains a lot of load instructions and many arithmetic instructions
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index de04ed85c3..48a860ba41 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -810,7 +810,7 @@ std::vector<uint8> NVPTXCompiler::CompilePtxOrGetCachedResult(const string& ptx,
             // binaries are not available. We don't want to spam logs with
             // identical warnings in this case.
 
-            // TODO(zhengxq): we should implement a LOG_FIRST_N and LOG_EVERY_N
+            // TODO(jlebar): we should implement a LOG_FIRST_N and LOG_EVERY_N
             // for more general usage.
             static std::atomic<bool> warning_done(false);
             log_warning = !warning_done.exchange(true);
-- 
GitLab


From 2a555cd91f7cdc6d384e6887ddb6af7843bdb996 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 14 Nov 2018 09:02:07 -0800
Subject: [PATCH 0217/1554] Actually disable logging in fuzzing mode.

PiperOrigin-RevId: 221453081
---
 tensorflow/core/platform/default/logging.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/default/logging.cc b/tensorflow/core/platform/default/logging.cc
index 34db490106..ad3feda701 100644
--- a/tensorflow/core/platform/default/logging.cc
+++ b/tensorflow/core/platform/default/logging.cc
@@ -135,10 +135,10 @@ LogMessage::~LogMessage() {
 int64 LogMessage::MinVLogLevel() {
   // We don't want to print logs during fuzzing as that would slow fuzzing down
   // by almost 2x. So, if we are in fuzzing mode (not just running a test), we
-  // return maximum value so that nothing is actually printed
+  // return minimum value so that nothing is actually printed
   // See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
-  return tensorflow::NUM_SEVERITIES;
+  return 0;
 #else
   static int64 min_vlog_level = MinVLogLevelFromEnv();
   return min_vlog_level;
-- 
GitLab


From bee955699c66d9a579f70b1a035d032a9973871d Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 14 Nov 2018 09:46:07 -0800
Subject: [PATCH 0218/1554] [TF Grappler] MetaOptimizer now takes a
 ConfigProto.

This will allow us to start reasoning about more global config options in
Grappler.  For example, it will allow us to move XLA functionalization rewrites
into Grappler, because we'll know when JIT optimization has been turned on.

We'll also be able to selectively disable particular grappler rewrites when
JIT optimization is on.

Additional changes:

* PartitionedCall now accepts a ConfigProto instead of a RewriterConfig
* Eager context now passes the entire ConfigProto through.

PiperOrigin-RevId: 221460335
---
 .../contrib/tensorrt/convert/convert_graph.cc |  6 +-
 .../contrib/tensorrt/python/trt_convert.py    | 12 ++--
 .../common_runtime/graph_execution_state.cc   |  7 +-
 .../grappler/optimizers/meta_optimizer.cc     | 47 ++++++++------
 .../core/grappler/optimizers/meta_optimizer.h | 10 +--
 .../optimizers/meta_optimizer_test.cc         | 63 ++++++++++++------
 .../core/kernels/data/optimize_dataset_op.cc  |  6 +-
 .../core/kernels/partitioned_function_ops.cc  | 34 +++++++---
 tensorflow/core/ops/functional_ops.cc         |  4 +-
 tensorflow/python/eager/context.py            | 55 +++++++---------
 tensorflow/python/eager/function.py           |  2 +-
 tensorflow/python/eager/function_test.py      |  6 +-
 .../python/grappler/cost_analyzer_tool.py     |  9 +--
 tensorflow/python/grappler/graph_placer.py    |  6 +-
 .../python/grappler/layout_optimizer_test.py  | 20 +++---
 .../python/grappler/memory_optimizer_test.py  | 65 +++++++++++--------
 tensorflow/python/grappler/tf_optimizer.i     | 14 ++--
 tensorflow/python/grappler/tf_optimizer.py    |  8 ++-
 .../python/grappler/tf_optimizer_test.py      | 17 +++--
 .../python/kernel_tests/signal/test_util.py   | 14 ++--
 .../python/kernel_tests/while_v2_test.py      | 11 ++--
 tensorflow/python/ops/functional_ops.py       | 19 +++---
 22 files changed, 255 insertions(+), 180 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 26d54eb156..f95ffe4100 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -264,7 +264,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
 #endif
 
   // Create RewriterConfig.
-  tensorflow::RewriterConfig rw_cfg;
+  tensorflow::ConfigProto config_proto;
+  auto& rw_cfg =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
   // TODO(aaroey): use only const folding and layout for the time being since
   // new optimizers break the graph for trt.
   rw_cfg.add_optimizers("constfold");
@@ -287,7 +289,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
 
   // Run optimizer.
-  tensorflow::grappler::MetaOptimizer meta_opt(nullptr, rw_cfg);
+  tensorflow::grappler::MetaOptimizer meta_opt(nullptr, config_proto);
   TF_RETURN_IF_ERROR(meta_opt.Optimize(cluster.get(), item, new_graph_def));
 
   if (VLOG_IS_ON(5)) {
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index bb81fbf93f..d149dcbd56 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -323,14 +323,16 @@ def create_inference_graph(input_graph_def,
           output_collection)
 
   # Create RewriterConfig.
-  rewriter_config = tensorrt_rewriter_config(
-      rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
-      minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batch_sizes)
+  config = config_pb2.ConfigProto()
+  config.graph_options.rewrite_options.CopyFrom(
+      tensorrt_rewriter_config(
+          rewriter_config, max_batch_size, max_workspace_size_bytes,
+          precision_mode, minimum_segment_size, is_dynamic_op,
+          maximum_cached_engines, cached_engine_batch_sizes))
 
   # Run Grappler.
   transformed_graph_def = tf_optimizer.OptimizeGraph(
-      rewriter_config, grappler_meta_graph_def, graph_id=b"tf_graph")
+      config, grappler_meta_graph_def, graph_id=b"tf_graph")
 
   # Optionally write the transformed graphdef as SavedModel.
   if output_saved_model_dir is not None:
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index ab619ef619..0d36930324 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -552,10 +552,7 @@ Status GraphExecutionState::OptimizeGraph(
     return errors::InvalidArgument("Can't optimize a pruned graph");
   }
 
-  const RewriterConfig& rewrite_options =
-      session_options_->config.graph_options().rewrite_options();
-
-  if (grappler::MetaOptimizerEnabled(rewrite_options)) {
+  if (grappler::MetaOptimizerEnabled(session_options_->config)) {
     // Adding this functionality in steps. The first step is to make sure
     // we don't break dependencies. The second step will be to turn the
     // functionality on by default.
@@ -638,7 +635,7 @@ Status GraphExecutionState::OptimizeGraph(
     grappler::VirtualCluster cluster(device_set_);
     GraphDef new_graph;
     TF_RETURN_IF_ERROR(grappler::RunMetaOptimizer(
-        item, rewrite_options, cpu_device, &cluster, &new_graph));
+        item, session_options_->config, cpu_device, &cluster, &new_graph));
 
     // Merge optimized graph function library with an original library.
     // Optimized graph might have new functions specialized for it's
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 82c88bb06a..6975fa715b 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -127,8 +127,10 @@ std::unique_ptr<GraphOptimizer> MetaOptimizer::MakeNewOptimizer(
 
 #undef MK_OPT
 
-MetaOptimizer::MetaOptimizer(DeviceBase* cpu_device, const RewriterConfig& cfg)
-    : cpu_device_(cpu_device), cfg_(cfg) {
+MetaOptimizer::MetaOptimizer(DeviceBase* cpu_device, const ConfigProto& cfg)
+    : cpu_device_(cpu_device),
+      config_proto_(cfg),
+      cfg_(*config_proto_.mutable_graph_options()->mutable_rewrite_options()) {
   DCHECK(cpu_device_ == nullptr ||
          cpu_device_->attributes().device_type() == "CPU");
 }
@@ -570,32 +572,35 @@ void MetaOptimizer::Feedback(Cluster* cluster, const GrapplerItem& item,
   // Nothing to do for MetaOptimizer.
 }
 
-bool MetaOptimizerEnabled(const RewriterConfig& cfg) {
-  if (cfg.disable_meta_optimizer()) {
+bool MetaOptimizerEnabled(const ConfigProto& cfg) {
+  const auto& rewrite_cfg = cfg.graph_options().rewrite_options();
+  if (rewrite_cfg.disable_meta_optimizer()) {
     return false;
   }
-  return !cfg.disable_model_pruning() ||
-         cfg.layout_optimizer() != RewriterConfig::OFF ||
-         cfg.function_optimization() != RewriterConfig::OFF ||
-         cfg.constant_folding() != RewriterConfig::OFF ||
-         cfg.shape_optimization() != RewriterConfig::OFF ||
-         cfg.remapping() != RewriterConfig::OFF ||
-         cfg.arithmetic_optimization() != RewriterConfig::OFF ||
-         cfg.loop_optimization() != RewriterConfig::OFF ||
-         cfg.dependency_optimization() != RewriterConfig::OFF ||
-         cfg.auto_parallel().enable() ||
-         cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT ||
-         cfg.debug_stripper() == RewriterConfig::ON ||
-         cfg.scoped_allocator_optimization() == RewriterConfig::ON ||
-         cfg.pin_to_host_optimization() == RewriterConfig::ON ||
-         !cfg.optimizers().empty() || !cfg.custom_optimizers().empty();
+  return !rewrite_cfg.disable_model_pruning() ||
+         rewrite_cfg.layout_optimizer() != RewriterConfig::OFF ||
+         rewrite_cfg.function_optimization() != RewriterConfig::OFF ||
+         rewrite_cfg.constant_folding() != RewriterConfig::OFF ||
+         rewrite_cfg.shape_optimization() != RewriterConfig::OFF ||
+         rewrite_cfg.remapping() != RewriterConfig::OFF ||
+         rewrite_cfg.arithmetic_optimization() != RewriterConfig::OFF ||
+         rewrite_cfg.loop_optimization() != RewriterConfig::OFF ||
+         rewrite_cfg.dependency_optimization() != RewriterConfig::OFF ||
+         rewrite_cfg.auto_parallel().enable() ||
+         rewrite_cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT ||
+         rewrite_cfg.debug_stripper() == RewriterConfig::ON ||
+         rewrite_cfg.scoped_allocator_optimization() == RewriterConfig::ON ||
+         rewrite_cfg.pin_to_host_optimization() == RewriterConfig::ON ||
+         !rewrite_cfg.optimizers().empty() ||
+         !rewrite_cfg.custom_optimizers().empty();
 }
 
-Status RunMetaOptimizer(const GrapplerItem& item, const RewriterConfig& cfg,
+Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
                         DeviceBase* cpu_device, Cluster* cluster,
                         GraphDef* optimized_graph) {
   MetaOptimizer optimizer(cpu_device, cfg);
-  optimizer.set_deadline_usec(DeadlineMicroSeconds(cfg));
+  optimizer.set_deadline_usec(
+      DeadlineMicroSeconds(cfg.graph_options().rewrite_options()));
   Status status = optimizer.Optimize(cluster, item, optimized_graph);
   if (!status.ok()) {
     *optimized_graph = item.graph;
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.h b/tensorflow/core/grappler/optimizers/meta_optimizer.h
index e599a9201b..a06da4394e 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
 namespace tensorflow {
@@ -28,7 +29,7 @@ namespace grappler {
 // Run the other grappler optimizers based on the specified rewriter config.
 class MetaOptimizer : public GraphOptimizer {
  public:
-  MetaOptimizer(DeviceBase* cpu_device, const RewriterConfig& cfg);
+  MetaOptimizer(DeviceBase* cpu_device, const ConfigProto& cfg);
   ~MetaOptimizer() override = default;
 
   string name() const override { return "meta_optimizer"; };
@@ -65,7 +66,8 @@ class MetaOptimizer : public GraphOptimizer {
                        GraphDef* optimized_graph);
 
   DeviceBase* const cpu_device_;  // may be NULL
-  RewriterConfig cfg_;
+  ConfigProto config_proto_;
+  RewriterConfig& cfg_;
 
   struct OptimizerResult {
     string optimizer_name;
@@ -85,7 +87,7 @@ class MetaOptimizer : public GraphOptimizer {
   std::vector<GraphOptimizationResult> optimization_results_;
 };
 
-bool MetaOptimizerEnabled(const RewriterConfig& cfg);
+bool MetaOptimizerEnabled(const ConfigProto& cfg);
 
 // Run the meta optimizer.
 //
@@ -93,7 +95,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg);
 // during constant folding; if NULL, a new device is created for doing constant
 // folding. For performance, it is recommended to pass in an existing cpu_device
 // when possible.
-Status RunMetaOptimizer(const GrapplerItem& item, const RewriterConfig& cfg,
+Status RunMetaOptimizer(const GrapplerItem& item, const ConfigProto& cfg,
                         DeviceBase* cpu_device, Cluster* cluster,
                         GraphDef* optimized_graph);
 
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index 6105bf27ba..b60aa25667 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -134,11 +134,13 @@ TEST_F(MetaOptimizerTest, RunsCustomOptimizer) {
   CHECK(fake_input.NextItem(&item));
 
   TestOptimizer::SetOptimized(false);
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
   rewriter_config.add_optimizers("TestOptimizer");
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
   GraphDef output;
   const Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -151,13 +153,15 @@ TEST_F(MetaOptimizerTest, RunsCustomOptimizerWithParams) {
   CHECK(fake_input.NextItem(&item));
 
   TestOptimizer::SetOptimized(false);
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
   rewriter_config.add_optimizers("TestOptimizerWithParams");
   auto* custom_config = rewriter_config.add_custom_optimizers();
   custom_config->set_name("TestOptimizerWithParams");
   (*custom_config->mutable_parameter_map())["foo"] = AttrValue();
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
   GraphDef output;
   const Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -171,13 +175,15 @@ TEST_F(MetaOptimizerTest, RunsCustomOptimizerAndCustomGraphOptimizer) {
 
   TestOptimizer::SetOptimized(false);
   TestGraphOptimizer::SetOptimized(false);
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
   rewriter_config.add_optimizers("TestOptimizer");
   auto customGraphOptimizer = rewriter_config.add_custom_optimizers();
   customGraphOptimizer->set_name("TestGraphOptimizer");
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
   GraphDef output;
   const Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -190,11 +196,13 @@ TEST_F(MetaOptimizerTest, RunOptimizersTwice) {
   GrapplerItem item;
   CHECK(fake_input.NextItem(&item));
 
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
   rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
   GraphDef output;
   const Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -205,13 +213,15 @@ TEST_F(MetaOptimizerTest, RunToggleOptimizersAndCustomGraphOptimizerTwice) {
   GrapplerItem item;
   CHECK(fake_input.NextItem(&item));
 
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
   auto customGraphOptimizer = rewriter_config.add_custom_optimizers();
   customGraphOptimizer->set_name("TestGraphOptimizer");
   rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
   GraphDef output;
   const Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -222,13 +232,16 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   using test::function::NDef;
 
   // Enable ony function optimization.
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
+
   rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
   rewriter_config.set_function_optimization(RewriterConfig::ON);
   rewriter_config.add_optimizers("function");
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
 
   // Define function library:
   //
@@ -394,14 +407,17 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneFunctionBody) {
   using test::function::NDef;
 
   // Enable function optimization and pruning.
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
+
   rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
   rewriter_config.set_function_optimization(RewriterConfig::ON);
   rewriter_config.add_optimizers("function");
   rewriter_config.add_optimizers("pruning");
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
 
   // MyFunc defines two Mul nodes inside function body and two corresponding
   // function outputs.
@@ -505,12 +521,15 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
       &allowed_optimizations);
 
   // Just record properties of optimized Grappler items.
-  RewriterConfig rewriter_config;
+  ConfigProto config_proto;
+  auto& rewriter_config =
+      *config_proto.mutable_graph_options()->mutable_rewrite_options();
+
   rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
   rewriter_config.add_optimizers("GrapplerItemPropertiesAccumulator");
   rewriter_config.set_min_graph_nodes(-1);
 
-  MetaOptimizer optimizer(nullptr, rewriter_config);
+  MetaOptimizer optimizer(nullptr, config_proto);
 
   // Define simple function library with two identical mul functions.
   FunctionDef mul_func_1 = FunctionDefHelper::Create(
@@ -605,7 +624,9 @@ TEST_F(MetaOptimizerTest, OptimizerTimesOut) {
   GrapplerItem item;
   CHECK(fake_input.NextItem(&item));
 
-  RewriterConfig rewriter_config;
+  ConfigProto config;
+  RewriterConfig& rewriter_config =
+      *config.mutable_graph_options()->mutable_rewrite_options();
   rewriter_config.add_optimizers("SleepingOptimizer");
   rewriter_config.set_min_graph_nodes(-1);
   rewriter_config.set_meta_optimizer_timeout_ms(1500);
@@ -613,7 +634,7 @@ TEST_F(MetaOptimizerTest, OptimizerTimesOut) {
 
   GraphDef output;
   const Status status =
-      RunMetaOptimizer(item, rewriter_config, nullptr, nullptr, &output);
+      RunMetaOptimizer(item, config, nullptr, nullptr, &output);
   EXPECT_EQ(status.error_message(), "meta_optimizer exceeded deadline.");
   // Make sure the graph was reverted to the original regardless of when the
   // optimizer timed out.
@@ -625,14 +646,16 @@ TEST_F(MetaOptimizerTest, OptimizerDoesNotTimeOut) {
   GrapplerItem item;
   CHECK(fake_input.NextItem(&item));
 
-  RewriterConfig rewriter_config;
+  ConfigProto config;
+  RewriterConfig& rewriter_config =
+      *config.mutable_graph_options()->mutable_rewrite_options();
   rewriter_config.add_optimizers("SleepingOptimizer");
   rewriter_config.set_min_graph_nodes(-1);
   rewriter_config.set_meta_optimizer_timeout_ms(1500);
   rewriter_config.set_meta_optimizer_iterations(RewriterConfig::ONE);
   GraphDef output;
   const Status status =
-      RunMetaOptimizer(item, rewriter_config, nullptr, nullptr, &output);
+      RunMetaOptimizer(item, config, nullptr, nullptr, &output);
   TF_EXPECT_OK(status);
   EXPECT_EQ(item.graph.node_size() + 1, output.node_size());
 }
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index 213ee7c601..f90dcb95e3 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -226,7 +226,9 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       (*meta_graph_def.mutable_collection_def())["train_op"] = collection_def;
 
       // Create Grappler item.
-      tensorflow::RewriterConfig rewriter_config;
+      tensorflow::ConfigProto config;
+      RewriterConfig& rewriter_config =
+          *config.mutable_graph_options()->mutable_rewrite_options();
       for (const string& optimization : optimizations_) {
         rewriter_config.add_optimizers(optimization);
       }
@@ -264,7 +266,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
         }
       }
       TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer(
-          *grappler_item, rewriter_config, ctx->device(), &cluster, graph_def));
+          *grappler_item, config, ctx->device(), &cluster, graph_def));
 
       return Status::OK();
     }
diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index 71e506e5e6..89b74495c7 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
 #include "tensorflow/core/grappler/utils/functions.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 #include "tensorflow/core/util/ptr_util.h"
 #include "tensorflow/core/util/reffed_status_callback.h"
@@ -50,12 +51,29 @@ class PartitionedCallOp : public AsyncOpKernel {
  public:
   explicit PartitionedCallOp(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
-    string rewriter_config_serialized;
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("config", &rewriter_config_serialized));
+    string deprecated_config_serialized;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("config", &deprecated_config_serialized));
+    string config_proto_serialized;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("config_proto", &config_proto_serialized));
     OP_REQUIRES(
-        ctx, rewriter_config_.ParseFromString(rewriter_config_serialized),
-        errors::InvalidArgument("Unable to parse rewriter_config string as "
-                                "tensorflow::RewriterConfig proto."));
+        ctx,
+        deprecated_config_serialized.empty() || config_proto_serialized.empty(),
+        errors::InvalidArgument("Provided both 'config' and 'config_proto' but "
+                                "only one should be provided.  Note the "
+                                "'config' option is deprecated."));
+    if (!deprecated_config_serialized.empty()) {
+      OP_REQUIRES(ctx,
+                  config_proto_.mutable_graph_options()
+                      ->mutable_rewrite_options()
+                      ->ParseFromString(deprecated_config_serialized),
+                  errors::InvalidArgument("Unable to parse config string as "
+                                          "tensorflow::RewriteOptions proto."));
+    } else {
+      OP_REQUIRES(
+          ctx, config_proto_.ParseFromString(config_proto_serialized),
+          errors::InvalidArgument("Unable to parse config_proto string as "
+                                  "tensorflow::ConfigProto proto."));
+    }
     OP_REQUIRES_OK(ctx, ctx->GetAttr("executor_type", &executor_type_));
   }
 
@@ -506,7 +524,7 @@ class PartitionedCallOp : public AsyncOpKernel {
                        FunctionLibraryDefinition* flib,
                        const DeviceSet& device_set, Device* cpu_device,
                        std::unique_ptr<Graph>* graph) {
-    if (!tensorflow::grappler::MetaOptimizerEnabled(rewriter_config_)) {
+    if (!tensorflow::grappler::MetaOptimizerEnabled(config_proto_)) {
       return Status::OK();
     }
 
@@ -530,7 +548,7 @@ class PartitionedCallOp : public AsyncOpKernel {
     // TODO(nareshmodi): Consider adding and using the more generic GraphOptions
     // proto (which also contain the OptimizerOptions).
     TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer(
-        item, rewriter_config_, cpu_device, &cluster, &out_graph));
+        item, config_proto_, cpu_device, &cluster, &out_graph));
 
     std::unique_ptr<Graph> optimized_graph(new Graph(OpRegistry::Global()));
     TF_RETURN_IF_ERROR(ConvertGraphDefToGraph(
@@ -562,7 +580,7 @@ class PartitionedCallOp : public AsyncOpKernel {
   }
 
   NameAttrList func_;
-  RewriterConfig rewriter_config_;
+  ConfigProto config_proto_;
   string executor_type_;
   // Contains maps from device names to handles of function partitions, keyed by
   // FunctionLibraryRuntime pointers. (Because this kernel may be instantiated
diff --git a/tensorflow/core/ops/functional_ops.cc b/tensorflow/core/ops/functional_ops.cc
index ee14a851eb..5e0bdd888c 100644
--- a/tensorflow/core/ops/functional_ops.cc
+++ b/tensorflow/core/ops/functional_ops.cc
@@ -226,6 +226,7 @@ REGISTER_OP("PartitionedCall")
     .Attr("Tout: list(type) >= 0")
     .Attr("f: func")
     .Attr("config: string = ''")
+    .Attr("config_proto: string = ''")
     .Attr("executor_type: string = ''")
     .SetShapeFn(shape_inference::UnknownShape);
 
@@ -235,7 +236,8 @@ REGISTER_OP("StatefulPartitionedCall")
     .Attr("Tin: list(type) >= 0")
     .Attr("Tout: list(type) >= 0")
     .Attr("f: func")
-    .Attr("config: string = ''")
+    .Attr("config: string = ''")  // Deprecated in favor of config_proto
+    .Attr("config_proto: string = ''")
     .Attr("executor_type: string = ''")
     .SetIsStateful()
     .SetShapeFn(shape_inference::UnknownShape);
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index e3fef524bf..2f6b038dda 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -25,7 +25,6 @@ import random
 import threading
 
 from tensorflow.core.protobuf import config_pb2
-from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python import tf2
 from tensorflow.python.framework import c_api_util
@@ -86,21 +85,21 @@ class FunctionCallOptions(object):
   Eager functions are functions decorated with tf.contrib.eager.defun.
   """
 
-  def __init__(self, executor_type=None, rewriter_config=None):
+  def __init__(self, executor_type=None, config_proto=None):
     """Constructor.
 
     Args:
       executor_type: (optional) name of the executor to be used to execute the
         eager function. If None or an empty string, the default Tensorflow
         executor will be used.
-      rewriter_config: (optional) a rewriter_config_pb2.RewriterConfig proto or
+      config_proto: (optional) a `config_pb2.ConfigProto` proto or
         a serialized string of that proto.
         The config used by Grappler when optimizing the function graph.
         Each concrete function is optimized the first time is called. Changing
-        rewriter_config after the first call has no effect.
-        If rewriter_config is None, an empty RewriterConfig will be used.
+        config_proto after the first call has no effect.
+        If config_proto is None, an empty RewriterConfig will be used.
     """
-    self.rewriter_config_serialized = rewriter_config
+    self.config_proto_serialized = config_proto
     self.executor_type = executor_type
 
   @property
@@ -112,24 +111,22 @@ class FunctionCallOptions(object):
     self._executor_type = executor_type
 
   @property
-  def rewriter_config_serialized(self):
-    return self._rewriter_config_serialized
+  def config_proto_serialized(self):
+    return self._config_proto_serialized
 
-  @rewriter_config_serialized.setter
-  def rewriter_config_serialized(self, config):
-    if isinstance(config, rewriter_config_pb2.RewriterConfig):
-      self._rewriter_config_serialized = config.SerializeToString()
+  @config_proto_serialized.setter
+  def config_proto_serialized(self, config):
+    if isinstance(config, config_pb2.ConfigProto):
+      self._config_proto_serialized = config.SerializeToString()
     elif isinstance(config, str):
-      self._rewriter_config_serialized = config
+      self._config_proto_serialized = config
     elif config is None:
-      self._rewriter_config_serialized = rewriter_config_pb2.RewriterConfig(
-      ).SerializeToString()
+      self._config_proto_serialized = (
+          config_pb2.ConfigProto().SerializeToString())
     else:
-      raise ValueError(
-          "the rewriter config must be either a "
-          "rewriter_config_pb2.RewriterConfig, or a serialized string of that "
-          "proto or None. got: {}"
-          .format(type(config)))
+      raise ValueError("the rewriter config must be either a "
+                       "config_pb2.ConfigProto, or a serialized string of that "
+                       "proto or None. got: {}".format(type(config)))
 
 
 # TODO(agarwal): better name ?
@@ -152,14 +149,12 @@ class _EagerContext(threading.local):
 
     # Default rewriter config corresponds to turning all default grappler
     # optimizations on.
-    base_config = rewriter_config_pb2.RewriterConfig()
+    base_config = config_pb2.ConfigProto()
 
-    if config is not None and config.HasField(
-        "graph_options") and config.graph_options.HasField("rewrite_options"):
-      base_config.Merge(config.graph_options.rewrite_options)
+    if config is not None:
+      base_config.MergeFrom(config)
 
-    self.function_call_options = FunctionCallOptions(
-        rewriter_config=base_config)
+    self.function_call_options = FunctionCallOptions(config_proto=base_config)
 
 
 ContextSwitch = collections.namedtuple(
@@ -897,21 +892,21 @@ def export_run_metadata():
   return context().export_run_metadata()
 
 
-def function_rewriter_config(rewriter_config):
+def function_config_proto(config_proto):
   """Context manager for setting the grappler rewrite config.
 
   This config is used by Grappler when optimizing the function graph.
 
   Args:
-    rewriter_config: a rewriter_config_pb2.RewriterConfig proto or
+    config_proto: a `config_pb2.ConfigProto` proto or
       a serialized string of that proto or None. If None, the default instance
-      of rewriter_config_pb2.RewriterConfig will be used.
+      of `config_pb2.ConfigProto` will be used.
 
   Returns:
     A context manager.
   """
   def _set_options_func(options):
-    options.rewriter_config_serialized = rewriter_config
+    options.config_proto_serialized = config_proto
 
   return context().function_call_options(_set_options_func)
 
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index e4a5fefe76..e863cf57bc 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -260,7 +260,7 @@ class _EagerDefinedFunction(object):
           f=self,
           tout=self._output_types,
           executing_eagerly=executing_eagerly,
-          config=function_call_options.rewriter_config_serialized,
+          config=function_call_options.config_proto_serialized,
           executor_type=function_call_options.executor_type)
 
     if executing_eagerly:
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 2af08689f8..ff1d947bf7 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -102,10 +102,10 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       _ = x * y
       return x + y
 
-    # The default config allows everything.
-    rewrites = rewriter_config_pb2.RewriterConfig()
+    # The default config allows all rewrites.
+    config_proto = config_pb2.ConfigProto()
 
-    with context.function_rewriter_config(rewrites):
+    with context.function_config_proto(config_proto):
       t = constant_op.constant(1.0)
       self.assertAllEqual(add(t, t).numpy(), 2.0)
 
diff --git a/tensorflow/python/grappler/cost_analyzer_tool.py b/tensorflow/python/grappler/cost_analyzer_tool.py
index e6229e1856..7dbaf449ca 100644
--- a/tensorflow/python/grappler/cost_analyzer_tool.py
+++ b/tensorflow/python/grappler/cost_analyzer_tool.py
@@ -25,8 +25,8 @@ from google.protobuf import message
 from google.protobuf import text_format
 from tensorflow.contrib.fused_conv.ops import gen_fused_conv2d_bias_activation_op  # pylint: disable=unused-import
 from tensorflow.core.framework import graph_pb2
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
-from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.core.protobuf import saved_model_pb2
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
@@ -79,10 +79,11 @@ def get_metagraph():
 
 def main(_):
   metagraph = get_metagraph()
-  rewriter_config = rewriter_config_pb2.RewriterConfig()
+  config = config_pb2.ConfigProto()
   if FLAGS.rewriter_config is not None:
-    text_format.Merge(FLAGS.rewriter_config, rewriter_config)
-  optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, metagraph)
+    text_format.Merge(FLAGS.rewriter_config,
+                      config.graph_options.rewrite_options)
+  optimized_graph = tf_optimizer.OptimizeGraph(config, metagraph)
   metagraph.graph_def.CopyFrom(optimized_graph)
 
   report = cost_analyzer.GenerateCostReport(metagraph, FLAGS.per_node_report,
diff --git a/tensorflow/python/grappler/graph_placer.py b/tensorflow/python/grappler/graph_placer.py
index 654013b23c..9c05ad8179 100644
--- a/tensorflow/python/grappler/graph_placer.py
+++ b/tensorflow/python/grappler/graph_placer.py
@@ -19,8 +19,8 @@ from __future__ import division
 from __future__ import print_function
 
 import time
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
-from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops as tf_ops
 from tensorflow.python.grappler import cluster as gcluster
@@ -54,9 +54,9 @@ def PlaceGraph(metagraph,
     cluster = gcluster.Cluster()
 
   # Optimize the metagraph to speedup the placement
-  rewriter_config = rewriter_config_pb2.RewriterConfig()
+  config = config_pb2.ConfigProto()
   optimized_graph = tf_optimizer.OptimizeGraph(
-      rewriter_config, metagraph, verbose=verbose, cluster=cluster)
+      config, metagraph, verbose=verbose, cluster=cluster)
   optimized_metagraph = meta_graph_pb2.MetaGraphDef()
   optimized_metagraph.CopyFrom(metagraph)
   optimized_metagraph.graph_def.CopyFrom(optimized_graph)
diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 8cc971c61d..7b68d5e80d 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -1443,11 +1443,13 @@ class LayoutOptimizerTest(test.TestCase):
 
   def testGradient(self):
     meta_graph = _simple_metagraph()
-    rewrite_options = rewriter_config_pb2.RewriterConfig(
-        layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
-        min_graph_nodes=-1)
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
+        rewriter_config_pb2.RewriterConfig(
+            layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
+            min_graph_nodes=-1))
     optimized_graph = tf_optimizer.OptimizeGraph(
-        rewrite_options, meta_graph, cluster=_get_cluster())
+        config, meta_graph, cluster=_get_cluster())
 
     found = 0
     for node in optimized_graph.node:
@@ -1458,11 +1460,13 @@ class LayoutOptimizerTest(test.TestCase):
 
   def testDepthwise(self):
     meta_graph = _simple_metagraph(depthwise=True)
-    rewrite_options = rewriter_config_pb2.RewriterConfig(
-        layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
-        min_graph_nodes=-1)
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
+        rewriter_config_pb2.RewriterConfig(
+            layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
+            min_graph_nodes=-1))
     optimized_graph = tf_optimizer.OptimizeGraph(
-        rewrite_options, meta_graph, cluster=_get_cluster())
+        config, meta_graph, cluster=_get_cluster())
 
     found = 0
     for node in optimized_graph.node:
diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py
index 03b42f6453..98cbb1a4b6 100644
--- a/tensorflow/python/grappler/memory_optimizer_test.py
+++ b/tensorflow/python/grappler/memory_optimizer_test.py
@@ -49,11 +49,13 @@ class MemoryOptimizerSwapTest(test.TestCase):
     graph_size = len(mg.graph_def.node)
     nodes = [node.name for node in mg.graph_def.node]
 
-    rewriter_config = rewriter_config_pb2.RewriterConfig(
-        disable_model_pruning=True,
-        constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
-        memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL)
-    graph = tf_optimizer.OptimizeGraph(rewriter_config, mg)
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
+        rewriter_config_pb2.RewriterConfig(
+            disable_model_pruning=True,
+            constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
+            memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL))
+    graph = tf_optimizer.OptimizeGraph(config, mg)
 
     self.assertEqual(len(graph.node), graph_size)
     self.assertItemsEqual([node.name for node in graph.node], nodes)
@@ -72,13 +74,15 @@ class MemoryOptimizerSwapTest(test.TestCase):
     mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph())
     graph_size = len(mg.graph_def.node)
 
-    rewriter_config = rewriter_config_pb2.RewriterConfig(
-        disable_model_pruning=True,
-        meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE,
-        constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
-        memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL,
-        min_graph_nodes=-1)
-    graph = tf_optimizer.OptimizeGraph(rewriter_config, mg)
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
+        rewriter_config_pb2.RewriterConfig(
+            disable_model_pruning=True,
+            meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE,
+            constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
+            memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL,
+            min_graph_nodes=-1))
+    graph = tf_optimizer.OptimizeGraph(config, mg)
 
     self.assertEqual(len(graph.node), graph_size + 2)
     self.assertTrue(
@@ -127,7 +131,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
   def testRewritingDefaultGradientNames(self):
     """Tests that rewriting occurs with default gradient names."""
     (original_metagraph, _, _, _) = self._GetMetaGraph()
-    rewritten_graph_def = tf_optimizer.OptimizeGraph(
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
         rewriter_config_pb2.RewriterConfig(
             disable_model_pruning=True,
             constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
@@ -135,8 +140,9 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
             layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF,
             arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF,
             min_graph_nodes=-1,
-            memory_optimization=rewriter_config_pb2.RewriterConfig.
-            RECOMPUTATION_HEURISTICS), original_metagraph)
+            memory_optimization=(
+                rewriter_config_pb2.RewriterConfig.RECOMPUTATION_HEURISTICS)))
+    rewritten_graph_def = tf_optimizer.OptimizeGraph(config, original_metagraph)
     self.assertGreater(
         len(rewritten_graph_def.node),
         len(original_metagraph.graph_def.node))
@@ -153,7 +159,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
     """Tests that rewriting occurs with non-standard gradient names."""
     (original_metagraph, _, _, _) = self._GetMetaGraph(
         optimizer_scope_name='optimizer')
-    rewritten_graph_def = tf_optimizer.OptimizeGraph(
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
         rewriter_config_pb2.RewriterConfig(
             disable_model_pruning=True,
             constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
@@ -161,11 +168,11 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
             layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF,
             arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF,
             min_graph_nodes=-1,
-            memory_optimization=rewriter_config_pb2.RewriterConfig.
-            RECOMPUTATION_HEURISTICS,
+            memory_optimization=rewriter_config_pb2.RewriterConfig
+            .RECOMPUTATION_HEURISTICS,
             # Checks that name scope "gradients/" also match sub-scope.
-            memory_optimizer_target_node_name_scope='gradients/'),
-        original_metagraph)
+            memory_optimizer_target_node_name_scope='gradients/'))
+    rewritten_graph_def = tf_optimizer.OptimizeGraph(config, original_metagraph)
     self.assertGreater(
         len(rewritten_graph_def.node),
         len(original_metagraph.graph_def.node))
@@ -182,18 +189,19 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
     """Tests that rewriting occurs with non-standard gradient names."""
     (original_metagraph, _, _,
      _) = self._GetMetaGraph(optimizer_scope_name='foo/bar')
-    rewritten_graph_def = tf_optimizer.OptimizeGraph(
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
         rewriter_config_pb2.RewriterConfig(
             disable_model_pruning=True,
             constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
             dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF,
             layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF,
             arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF,
-            memory_optimization=rewriter_config_pb2.RewriterConfig.
-            RECOMPUTATION_HEURISTICS,
+            memory_optimization=rewriter_config_pb2.RewriterConfig
+            .RECOMPUTATION_HEURISTICS,
             # This should not match anything.
-            memory_optimizer_target_node_name_scope='r/gradients/'),
-        original_metagraph)
+            memory_optimizer_target_node_name_scope='r/gradients/'))
+    rewritten_graph_def = tf_optimizer.OptimizeGraph(config, original_metagraph)
     self.assertEqual(
         len(rewritten_graph_def.node), len(original_metagraph.graph_def.node))
     self.assertEqual(0,
@@ -298,11 +306,12 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
         0,
         len([node for node in metagraph.graph_def.node
              if 'Recomputed/' in node.name]))
-    rewritten_graph_def = tf_optimizer.OptimizeGraph(
+    config = config_pb2.ConfigProto()
+    config.graph_options.rewrite_options.CopyFrom(
         rewriter_config_pb2.RewriterConfig(
             min_graph_nodes=-1,
-            memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL),
-        metagraph)
+            memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL))
+    rewritten_graph_def = tf_optimizer.OptimizeGraph(config, metagraph)
     self.assertEqual(
         9,
         len([node for node in rewritten_graph_def.node
diff --git a/tensorflow/python/grappler/tf_optimizer.i b/tensorflow/python/grappler/tf_optimizer.i
index 39ca71e99a..daa5bc9444 100644
--- a/tensorflow/python/grappler/tf_optimizer.i
+++ b/tensorflow/python/grappler/tf_optimizer.i
@@ -34,8 +34,8 @@ limitations under the License.
   $1 = &temp;
 }
 
-%typemap(in) const tensorflow::RewriterConfig& (
-    tensorflow::RewriterConfig temp) {
+%typemap(in) const tensorflow::ConfigProto& (
+    tensorflow::ConfigProto temp) {
   char* c_string;
   Py_ssize_t py_size;
   if (PyBytes_AsStringAndSize($input, &c_string, &py_size) == -1) {
@@ -46,7 +46,7 @@ limitations under the License.
   if (!temp.ParseFromString(string(c_string, py_size))) {
     PyErr_SetString(
         PyExc_TypeError,
-        "The RewriterConfig could not be parsed as a valid protocol buffer");
+        "The ConfigProto could not be parsed as a valid protocol buffer");
     SWIG_fail;
   }
   $1 = &temp;
@@ -67,8 +67,8 @@ limitations under the License.
   #include "tensorflow/core/grappler/clusters/utils.h"
   #include "tensorflow/core/grappler/clusters/virtual_cluster.h"
   #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+  #include "tensorflow/core/protobuf/config.pb.h"
   #include "tensorflow/core/protobuf/meta_graph.pb.h"
-  #include "tensorflow/core/protobuf/rewriter_config.pb.h"
   #include "tensorflow/core/public/session_options.h"
 
 
@@ -94,7 +94,7 @@ void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* dev
 
 PyObject* TF_OptimizeGraph(
       GCluster cluster,
-      const tensorflow::RewriterConfig& rewriter_config,
+      const tensorflow::ConfigProto& config_proto,
       const tensorflow::MetaGraphDef& metagraph,
       bool verbose, const string& graph_id, TF_Status* out_status) {
     tensorflow::grappler::ItemConfig item_config;
@@ -110,7 +110,7 @@ PyObject* TF_OptimizeGraph(
 
     tensorflow::DeviceBase* cpu_device = nullptr;
     tensorflow::GraphDef out_graph;
-    tensorflow::grappler::MetaOptimizer optimizer(cpu_device, rewriter_config);
+    tensorflow::grappler::MetaOptimizer optimizer(cpu_device, config_proto);
     tensorflow::Status status = optimizer.Optimize(cluster.get(), *grappler_item, &out_graph);
     if (verbose) {
       optimizer.PrintResult();
@@ -127,7 +127,7 @@ PyObject* TF_OptimizeGraph(
 // Wrap this function
 PyObject* TF_OptimizeGraph(
     GCluster cluster,
-    const tensorflow::RewriterConfig& rewriter_config,
+    const tensorflow::ConfigProto& config_proto,
     const tensorflow::MetaGraphDef& metagraph, bool verbose,
     const string& graph_id, TF_Status* out_status);
 
diff --git a/tensorflow/python/grappler/tf_optimizer.py b/tensorflow/python/grappler/tf_optimizer.py
index a73a4a98fc..e72667b6f3 100644
--- a/tensorflow/python/grappler/tf_optimizer.py
+++ b/tensorflow/python/grappler/tf_optimizer.py
@@ -19,22 +19,26 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.framework import graph_pb2
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import pywrap_tensorflow as tf_opt
 from tensorflow.python.framework import errors
 from tensorflow.python.grappler import cluster as gcluster
 
 
-def OptimizeGraph(rewriter_config,
+def OptimizeGraph(config_proto,
                   metagraph,
                   verbose=True,
                   graph_id=b'graph_to_optimize',
                   cluster=None):
   """Optimize the provided metagraph."""
+  if not isinstance(config_proto, config_pb2.ConfigProto):
+    raise TypeError('Expected config_proto to be a ConfigProto, saw type %s' %
+                    type(config_proto))
   with errors.raise_exception_on_not_ok_status() as status:
     if cluster is None:
       cluster = gcluster.Cluster()
     ret_from_swig = tf_opt.TF_OptimizeGraph(cluster.tf_cluster,
-                                            rewriter_config.SerializeToString(),
+                                            config_proto.SerializeToString(),
                                             metagraph.SerializeToString(),
                                             verbose, graph_id, status)
   if ret_from_swig is None:
diff --git a/tensorflow/python/grappler/tf_optimizer_test.py b/tensorflow/python/grappler/tf_optimizer_test.py
index eca0f67982..1321e1cb0d 100644
--- a/tensorflow/python/grappler/tf_optimizer_test.py
+++ b/tensorflow/python/grappler/tf_optimizer_test.py
@@ -17,7 +17,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
@@ -45,11 +45,12 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     train_op.append(d)
     mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph())
 
-    rewriter_config = rewriter_config_pb2.RewriterConfig()
+    config = config_pb2.ConfigProto()
+    rewriter_config = config.graph_options.rewrite_options
     rewriter_config.optimizers.append('constfold')
     rewriter_config.min_graph_nodes = -1
 
-    graph = tf_optimizer.OptimizeGraph(rewriter_config, mg)
+    graph = tf_optimizer.OptimizeGraph(config, mg)
 
     self.assertEqual(len(graph.node), 1)
     self.assertItemsEqual([node.name for node in graph.node], ['d'])
@@ -68,9 +69,10 @@ class PyWrapOptimizeGraphTest(test.TestCase):
 
     # Optimize the graph.
     mg = meta_graph.create_meta_graph_def(graph=g)
-    rewriter_config = rewriter_config_pb2.RewriterConfig()
+    config = config_pb2.ConfigProto()
+    rewriter_config = config.graph_options.rewrite_options
     rewriter_config.min_graph_nodes = -1
-    optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, mg)
+    optimized_graph = tf_optimizer.OptimizeGraph(config, mg)
 
     # Check that the nodes referenced in various collections have been preserved
     self.assertEqual(len(optimized_graph.node), 5)
@@ -110,9 +112,10 @@ class PyWrapOptimizeGraphTest(test.TestCase):
 
     # Optimize the graph.
     mg = meta_graph.create_meta_graph_def(graph=g)
-    rewriter_config = rewriter_config_pb2.RewriterConfig()
+    config = config_pb2.ConfigProto()
+    rewriter_config = config.graph_options.rewrite_options
     rewriter_config.min_graph_nodes = -1
-    optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, mg)
+    optimized_graph = tf_optimizer.OptimizeGraph(config, mg)
     mg.graph_def.CopyFrom(optimized_graph)
 
     # Check that the nodes referenced in various collections have been preserved
diff --git a/tensorflow/python/kernel_tests/signal/test_util.py b/tensorflow/python/kernel_tests/signal/test_util.py
index f2c4d0dc8f..0a8a621c3e 100644
--- a/tensorflow/python/kernel_tests/signal/test_util.py
+++ b/tensorflow/python/kernel_tests/signal/test_util.py
@@ -18,12 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.training import saver
 
 
-def grappler_optimize(graph, fetches=None, rewriter_config=None):
+def grappler_optimize(graph, fetches=None, config_proto=None):
   """Tries to optimize the provided graph using grappler.
 
   Args:
@@ -31,17 +31,17 @@ def grappler_optimize(graph, fetches=None, rewriter_config=None):
     fetches: An optional list of `Tensor`s to fetch (i.e. not optimize away).
       Grappler uses the 'train_op' collection to look for fetches, so if not
       provided this collection should be non-empty.
-    rewriter_config: An optional `tf.RewriterConfig` to use when rewriting the
+    config_proto: An optional `tf.ConfigProto` to use when rewriting the
       graph.
 
   Returns:
     A `tf.GraphDef` containing the rewritten graph.
   """
-  if rewriter_config is None:
-    rewriter_config = rewriter_config_pb2.RewriterConfig()
-    rewriter_config.min_graph_nodes = -1
+  if config_proto is None:
+    config_proto = config_pb2.ConfigProto()
+    config_proto.graph_options.rewrite_options.min_graph_nodes = -1
   if fetches is not None:
     for fetch in fetches:
       graph.add_to_collection('train_op', fetch)
   metagraph = saver.export_meta_graph(graph_def=graph.as_graph_def())
-  return tf_optimizer.OptimizeGraph(rewriter_config, metagraph)
+  return tf_optimizer.OptimizeGraph(config_proto, metagraph)
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 3314e3849b..5868b0d226 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -135,10 +136,12 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
 
     def GetOptimizedGraph():
       mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph())
-      rewriter_config = rewriter_config_pb2.RewriterConfig(
-          constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
-          memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL)
-      return tf_optimizer.OptimizeGraph(rewriter_config, mg)
+      config = config_pb2.ConfigProto()
+      config.graph_options.rewrite_options.CopyFrom(
+          rewriter_config_pb2.RewriterConfig(
+              constant_folding=rewriter_config_pb2.RewriterConfig.OFF,
+              memory_optimization=rewriter_config_pb2.RewriterConfig.MANUAL))
+      return tf_optimizer.OptimizeGraph(config, mg)
 
     g = GetOptimizedGraph()
     self.assertEqual(len([n for n in g.node if n.op == "Enter"]), 1)
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index fecd7ddbf9..57542e3c7b 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -19,7 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
@@ -1027,9 +1027,10 @@ _rewriter_config_optimizer_disabled = None
 def _get_disabled_rewriter_config():
   global _rewriter_config_optimizer_disabled
   if _rewriter_config_optimizer_disabled is None:
-    rewriter_config = rewriter_config_pb2.RewriterConfig()
+    config = config_pb2.ConfigProto()
+    rewriter_config = config.graph_options.rewrite_options
     rewriter_config.disable_meta_optimizer = True
-    _rewriter_config_optimizer_disabled = rewriter_config.SerializeToString()
+    _rewriter_config_optimizer_disabled = config.SerializeToString()
   return _rewriter_config_optimizer_disabled
 
 
@@ -1048,7 +1049,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
       the signature of `f`.
     executing_eagerly: (Optional) A boolean indicating whether the context is
       executing eagerly. If `None`, fetched from the global context.
-    config: (Optional) A tensorflow::RewriterConfig proto, serialized. If
+    config: (Optional) A `tensorflow::ConfigProto` proto, serialized. If
       `None`, all optimizations are disabled. Currently only handled for eager
       defined functions.
     executor_type: (Optional) A string for the name of the executor to be used
@@ -1076,10 +1077,12 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
   if executing_eagerly or len(tout):
     if f.stateful_ops:
       outputs = gen_functional_ops.stateful_partitioned_call(
-          args=args, Tout=tout, f=f, config=config, executor_type=executor_type)
+          args=args, Tout=tout, f=f, config_proto=config,
+          executor_type=executor_type)
     else:
       outputs = gen_functional_ops.partitioned_call(
-          args=args, Tout=tout, f=f, config=config, executor_type=executor_type)
+          args=args, Tout=tout, f=f, config_proto=config,
+          executor_type=executor_type)
     return outputs if outputs else None
 
   # The generated binding returns an empty list for functions that don't
@@ -1098,7 +1101,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
   # When running in graph mode, the graph and function graphs are optimized
   # (i.e. run through grappler) per the session options, so we can disable any
   # eager-specific rewriting.
-  rewriter_config = attr_value_pb2.AttrValue(s=_get_disabled_rewriter_config())
+  config_proto = attr_value_pb2.AttrValue(s=_get_disabled_rewriter_config())
 
   graph = ops.get_default_graph()
   f.add_to_graph(graph)
@@ -1113,7 +1116,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
           "Tin": tin_attr,
           "Tout": tout_attr,
           "f": func_attr,
-          "config": rewriter_config,
+          "config_proto": config_proto,
           "executor_type": executor_type_attr,
       })
   outputs = op.outputs
-- 
GitLab


From 9ddc629fed9ecc3ba8ed6cb4d4c00d40793adc45 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 14 Nov 2018 09:47:42 -0800
Subject: [PATCH 0219/1554] Change signature of tf.expand_dims for TF 2.0.

PiperOrigin-RevId: 221460560
---
 tensorflow/python/ops/array_ops.py            | 51 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  3 ++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 78a6ca0e4c..8925af6664 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -80,7 +80,7 @@ def identity(input, name=None):  # pylint: disable=redefined-builtin
 
 
 # pylint: disable=redefined-builtin,protected-access
-@tf_export("expand_dims")
+@tf_export(v1=["expand_dims"])
 @deprecation.deprecated_args(None, "Use the `axis` argument instead", "dim")
 def expand_dims(input, axis=None, name=None, dim=None):
   """Inserts a dimension of 1 into a tensor's shape.
@@ -134,6 +134,55 @@ def expand_dims(input, axis=None, name=None, dim=None):
   axis = deprecation.deprecated_argument_lookup("axis", axis, "dim", dim)
   if axis is None:
     raise ValueError("Must specify an axis argument to tf.expand_dims()")
+  return expand_dims_v2(input, axis, name)
+
+
+@tf_export("expand_dims", v1=[])
+def expand_dims_v2(input, axis, name=None):
+  """Inserts a dimension of 1 into a tensor's shape.
+
+  Given a tensor `input`, this operation inserts a dimension of 1 at the
+  dimension index `axis` of `input`'s shape. The dimension index `axis` starts
+  at zero; if you specify a negative number for `axis` it is counted backward
+  from the end.
+
+  This operation is useful if you want to add a batch dimension to a single
+  element. For example, if you have a single image of shape `[height, width,
+  channels]`, you can make it a batch of 1 image with `expand_dims(image, 0)`,
+  which will make the shape `[1, height, width, channels]`.
+
+  Other examples:
+
+  ```python
+  # 't' is a tensor of shape [2]
+  tf.shape(tf.expand_dims(t, 0))  # [1, 2]
+  tf.shape(tf.expand_dims(t, 1))  # [2, 1]
+  tf.shape(tf.expand_dims(t, -1))  # [2, 1]
+
+  # 't2' is a tensor of shape [2, 3, 5]
+  tf.shape(tf.expand_dims(t2, 0))  # [1, 2, 3, 5]
+  tf.shape(tf.expand_dims(t2, 2))  # [2, 3, 1, 5]
+  tf.shape(tf.expand_dims(t2, 3))  # [2, 3, 5, 1]
+  ```
+
+  This operation requires that:
+
+  `-1-input.dims() <= dim <= input.dims()`
+
+  This operation is related to `squeeze()`, which removes dimensions of
+  size 1.
+
+  Args:
+    input: A `Tensor`.
+    axis: 0-D (scalar). Specifies the dimension index at which to
+      expand the shape of `input`. Must be in the range
+      `[-rank(input) - 1, rank(input)]`.
+    name: The name of the output `Tensor` (optional).
+
+  Returns:
+    A `Tensor` with the same data as `input`, but its shape has an additional
+    dimension of size 1 added.
+  """
   return gen_array_ops.expand_dims(input, axis, name)
 
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 018245b879..665b3f5d81 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -690,7 +690,7 @@ tf_module {
   }
   member_method {
     name: "expand_dims"
-    argspec: "args=[\'input\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "extract_volume_patches"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 0df8b0f376..232ad48ac4 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -31,6 +31,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Maps from a function name to a dictionary that describes how to
     # map from an old argument keyword to the new argument keyword.
     self.function_keyword_renames = {
+        "tf.expand_dims": {
+            "dim": "axis",
+        },
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
-- 
GitLab


From 93a754b28b9e77bbce881425ff003978de2cb1a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 10:18:48 -0800
Subject: [PATCH 0220/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 221465977
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 93 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 14 +++
 2 files changed, 107 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 17ced64765..13416c179a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -40319,6 +40319,52 @@ op {
     }
   }
 }
+op {
+  name: "PartitionedCall"
+  input_arg {
+    name: "args"
+    type_list_attr: "Tin"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "Tout"
+  }
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "config"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "config_proto"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "executor_type"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+}
 op {
   name: "Placeholder"
   output_arg {
@@ -71870,6 +71916,53 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "StatefulPartitionedCall"
+  input_arg {
+    name: "args"
+    type_list_attr: "Tin"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "Tout"
+  }
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "config"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "config_proto"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "executor_type"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "StatelessIf"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index da5d4c152c..ac83059aec 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -20383,6 +20383,13 @@ op {
       s: ""
     }
   }
+  attr {
+    name: "config_proto"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
   attr {
     name: "executor_type"
     type: "string"
@@ -33556,6 +33563,13 @@ op {
       s: ""
     }
   }
+  attr {
+    name: "config_proto"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
   attr {
     name: "executor_type"
     type: "string"
-- 
GitLab


From 42be3a9034e3c1d4c2a90f2c495eecff970d44d0 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 14 Nov 2018 10:35:17 -0800
Subject: [PATCH 0221/1554] Stop defaulting number of eager threads to 1.

Defaulting to 1 is good for microbenchmarks, but bad for larger functions.

Before:
entry {
  name: "MicroBenchmarks.benchmark_keras_model_subclassed_fit"
  iters: 1
  wall_time: 907522.916794
  extras {
    key: "examples_per_sec"
    value {
      double_value: 1.10190054873
    }
  }
}

After:
entry {
  name: "MicroBenchmarks.benchmark_keras_model_subclassed_fit"
  iters: 1
  wall_time: 805292.129517
  extras {
    key: "examples_per_sec"
    value {
      double_value: 1.24178538861
    }
  }
}
PiperOrigin-RevId: 221469005
---
 tensorflow/core/common_runtime/eager/context.cc | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index f23cefb33d..583ae64edd 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -32,18 +32,6 @@ bool ReadBoolFromEnvVar(StringPiece env_var_name, bool default_val) {
   return default_val;
 }
 
-std::unique_ptr<thread::ThreadPool> EagerThreadPool(
-    const SessionOptions& opts) {
-  SessionOptions opts_copy(opts);
-  if (opts_copy.config.inter_op_parallelism_threads() == 0) {
-    // Eager defaults to a single thread when no threads are specified.
-    opts_copy.config.set_inter_op_parallelism_threads(1);
-  }
-
-  return std::unique_ptr<thread::ThreadPool>(
-      NewThreadPoolFromSessionOptions(opts_copy));
-}
-
 }  // namespace
 
 EagerContext::EagerContext(const SessionOptions& opts,
@@ -61,7 +49,7 @@ EagerContext::EagerContext(const SessionOptions& opts,
     : policy_(default_policy),
       devices_(device_mgr->ListDevices()),
       rendezvous_(rendezvous),
-      thread_pool_(EagerThreadPool(opts)),
+      thread_pool_(NewThreadPoolFromSessionOptions(opts)),
       pflr_(new ProcessFunctionLibraryRuntime(
           device_mgr, opts.env, TF_GRAPH_DEF_VERSION, &func_lib_def_, {},
           thread_pool_.get())),
-- 
GitLab


From 7ee436b8027136c73d4fdfaeafbbb084caff646f Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Wed, 14 Nov 2018 10:42:01 -0800
Subject: [PATCH 0222/1554] Fix incorrect link to xla.compile colab example

PiperOrigin-RevId: 221470285
---
 tensorflow/compiler/xla/g3doc/jit.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/g3doc/jit.md b/tensorflow/compiler/xla/g3doc/jit.md
index ded1e582b2..b61a0358bc 100644
--- a/tensorflow/compiler/xla/g3doc/jit.md
+++ b/tensorflow/compiler/xla/g3doc/jit.md
@@ -86,7 +86,7 @@ on uncompilable operator, xla.compile() returns an explicit error. This is
 useful if you want more predictable behaviors from XLA compilation.
 
 Please see
-[xla.compile() tutorial Colab](https://colab.sandbox.google.com/github/tensorflow/compiler/xla/g3doc/tutorials/xla_compile.ipynb)
+[xla.compile() tutorial Colab](./tutorials/xla_compile.ipynb)
 for how to use it.
 
 ### Placing operators on XLA devices
-- 
GitLab


From 0cd83fc0d8e247d390cf46fb526fd33886fced03 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 10:46:29 -0800
Subject: [PATCH 0223/1554] Use low level API to transfer literal to device in
 local client.

PiperOrigin-RevId: 221471132
---
 .../compiler/xla/client/local_client.cc       | 24 +++++++++++++++++++
 tensorflow/compiler/xla/client/local_client.h |  4 ++++
 .../compiler/xla/service/local_service.cc     |  6 +++++
 .../compiler/xla/service/local_service.h      |  5 ++++
 4 files changed, 39 insertions(+)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index f96b6c9c26..aaa5d6989e 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -310,4 +310,28 @@ StatusOr<int> LocalClient::ReplicaNumberToDeviceOrdinal(int replica_number) {
   return local_service_->ReplicaNumberToDeviceOrdinal(replica_number);
 }
 
+StatusOr<TransferToServerResponse> LocalClient::TransferToLocalServer(
+    const ::xla::BorrowingLiteral& literal, int device_oridinal) {
+  const ::xla::Shape& shape = literal.shape();
+
+  TF_ASSIGN_OR_RETURN(
+      ::xla::ScopedShapedBuffer shaped_buffer,
+      backend().transfer_manager()->AllocateScopedShapedBuffer(
+          shape, backend().memory_allocator(), device_oridinal));
+  TF_ASSIGN_OR_RETURN(auto stream,
+                      mutable_backend()->BorrowStream(device_oridinal));
+  TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice(
+      stream.get(), literal, shaped_buffer));
+  std::vector<::xla::ScopedShapedBuffer> replicated_buffer;
+  replicated_buffer.emplace_back(std::move(shaped_buffer));
+  ::xla::TransferToServerResponse result;
+  TF_ASSIGN_OR_RETURN(*result.mutable_data(),
+                      local_service_->RegisterReplicatedBuffers(
+                          std::move(replicated_buffer),
+                          absl::StrCat("TransferToServer literal of shape ",
+                                       ::xla::ShapeUtil::HumanString(shape))));
+
+  return result;
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/local_client.h b/tensorflow/compiler/xla/client/local_client.h
index e49451ca97..ddb36680e8 100644
--- a/tensorflow/compiler/xla/client/local_client.h
+++ b/tensorflow/compiler/xla/client/local_client.h
@@ -129,6 +129,10 @@ class LocalClient : public Client {
       const Literal& literal, int device_ordinal,
       DeviceMemoryAllocator* allocator = nullptr);
 
+  // Transfer the BorrowingLiteral to the device with the given ordinal.
+  StatusOr<TransferToServerResponse> TransferToLocalServer(
+      const ::xla::BorrowingLiteral& literal, int device_oridinal);
+
   // Copy the data from the device contained in the given ShapedBuffer and
   // return as a Literal.
   StatusOr<Literal> ShapedBufferToLiteral(const ShapedBuffer& shaped_buffer);
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index cca3755617..2180ac845d 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -220,4 +220,10 @@ StatusOr<const ShapedBuffer*> LocalService::GlobalDataToShapedBuffer(
   return buffers[replica_number];
 }
 
+StatusOr<GlobalDataHandle> LocalService::RegisterReplicatedBuffers(
+    std::vector<ScopedShapedBuffer> replicated_buffers, const string& tag) {
+  return allocation_tracker_.RegisterReplicatedBuffers(
+      std::move(replicated_buffers), tag);
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/local_service.h b/tensorflow/compiler/xla/service/local_service.h
index 3b4f0b5083..f56ba32b04 100644
--- a/tensorflow/compiler/xla/service/local_service.h
+++ b/tensorflow/compiler/xla/service/local_service.h
@@ -63,6 +63,11 @@ class LocalService : public Service {
   StatusOr<const ShapedBuffer*> GlobalDataToShapedBuffer(
       const GlobalDataHandle& data, int replica_number);
 
+  // Registers a vector of shaped buffers of device memory, one per replica, and
+  // returns a corresponding handle that can be used for talking to XLA clients.
+  StatusOr<GlobalDataHandle> RegisterReplicatedBuffers(
+      std::vector<ScopedShapedBuffer> replicated_buffers, const string& tag);
+
  private:
   explicit LocalService(const ServiceOptions& options,
                         std::unique_ptr<Backend> backend);
-- 
GitLab


From 2244653d37c20731760464c02b71c4394d8de65a Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 14 Nov 2018 11:02:18 -0800
Subject: [PATCH 0224/1554] Remove legacy code that is unused.

PiperOrigin-RevId: 221474322
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 74 -------------------
 1 file changed, 74 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index e2988f5f2a..431e363612 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1067,80 +1067,6 @@ struct LambdaFactory {
         return nullptr;
     }
   }
-
-  template <typename T>
-  std::function<T(T, T)> binary() {
-    switch (op) {
-      case OP_CATEGORY::ADD:
-        return [](T l, T r) -> T { return l + r; };
-      case OP_CATEGORY::SUB:
-        return [](T l, T r) -> T { return l - r; };
-      case OP_CATEGORY::MUL:
-        return [](T l, T r) -> T { return l * r; };
-      default:
-        LOG(WARNING) << "Not supported op for binary: " << static_cast<int>(op);
-    }
-    return [](T l, T r) -> T {
-      LOG(FATAL) << "Unsupported op type ";
-      return l;
-    };
-  }
-
-  template <typename T>
-  std::function<T(T)> broadcast_r(T val) {
-    VLOG(2) << "LAMBDA VAL : " << val;
-    switch (op) {
-      case OP_CATEGORY::ADD:
-        return [val](T l) -> T {
-          VLOG(2) << "LAMBDA VAL : " << val;
-          return l + val;
-        };
-      case OP_CATEGORY::SUB:
-        return [val](T l) -> T {
-          VLOG(2) << "LAMBDA VAL : " << val;
-          return l - val;
-        };
-      case OP_CATEGORY::MUL:
-        return [val](T l) -> T {
-          VLOG(2) << "LAMBDA VAL : " << val;
-          return l * val;
-        };
-      default:
-        LOG(WARNING) << "Not supported op for binary: " << static_cast<int>(op);
-    }
-    return [val](T l) -> T {
-      LOG(FATAL) << "Unsupported op type ";
-      return l;
-    };
-  }
-
-  template <typename T>
-  std::function<T(T)> broadcast_l(T val) {
-    VLOG(2) << "LAMBDA VAL : " << val;
-    switch (op) {
-      case OP_CATEGORY::ADD:
-        return [val](T l) -> T {
-          VLOG(2) << "LAMBDA VAL : " << val;
-          return val + l;
-        };
-      case OP_CATEGORY::SUB:
-        return [val](T l) -> T {
-          VLOG(2) << "LAMBDA VAL : " << val;
-          return val - l;
-        };
-      case OP_CATEGORY::MUL:
-        return [val](T l) -> T {
-          VLOG(2) << "LAMBDA VAL : " << val;
-          return val * l;
-        };
-      default:
-        LOG(ERROR) << "Not supported op for binary: " << static_cast<int>(op);
-    }
-    return [val](T l) -> T {
-      LOG(FATAL) << "Unsupported op type ";
-      return l;
-    };
-  }
 };
 
 template <>
-- 
GitLab


From 00774b2d5e7843596158159ee1e213ae40d6ed52 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Wed, 14 Nov 2018 11:02:37 -0800
Subject: [PATCH 0225/1554] Sample change making a kernel unittest work for
 both TF 1.x and 2.x.

This change makes all the tests that don't check gradients in relu_op_test.py
work for both TensorFlow 1.x and 2.x (where eager execution, resource variables
etc. are enabled by default).

PiperOrigin-RevId: 221474407
---
 .../python/kernel_tests/relu_op_test.py       | 259 +++++++++---------
 1 file changed, 131 insertions(+), 128 deletions(-)

diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index b0f2796ede..68243f27c0 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.compat import compat
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -55,52 +56,52 @@ class ReluTest(test.TestCase):
             np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
                                                      0.9]])))
 
-  def _testRelu(self, np_features, use_gpu=False):
+  def _testRelu(self, np_features):
     np_relu = self._npRelu(np_features)
-    with self.cached_session(use_gpu=use_gpu):
-      relu = nn_ops.relu(np_features)
-      tf_relu = relu.eval()
+    tf_relu = nn_ops.relu(np_features)
     self.assertAllClose(np_relu, tf_relu)
-    self.assertShapeEqual(np_relu, relu)
+    self.assertShapeEqual(np_relu, tf_relu)
 
-  def testNumbers(self):
+  def testNumbersCPU(self):
     for t in [np.int32, np.int64, np.float16, np.float32, np.float64]:
-      self._testRelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=False)
-      if t in [np.float16, np.float32, np.float64]:
+      # Force execution on CPU even if a GPU kernel is available for the type.
+      with ops.device("/device:CPU:0"):
         self._testRelu(
-            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-            use_gpu=True)
+            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
-  def _testReluInt8x4(self, np_inputs):
-    if not test.is_gpu_available(cuda_only=True):
-      return
-    np_relu = self._npRelu(np_inputs)
-    with self.cached_session(use_gpu=True):
-      relu = nn_ops.relu(constant_op.constant(np_inputs, dtypes.qint8))
-      if np_inputs.size % 4 == 0:
-        tf_relu = relu.eval()
-        self.assertAllClose(np_relu, tf_relu)
-        self.assertShapeEqual(np_relu, relu)
-      else:
-        with self.assertRaisesRegexp(
-            errors.InvalidArgumentError,
-            "Tensor size must be a multiple of 4 for Relu<qint8>. Got %d" %
-            np_inputs.size):
-          tf_relu = relu.eval()
+  def testNumbersGPU(self):
+    if not test.is_gpu_available():
+      self.skipTest("No GPU available")
+    for t in [np.float16, np.float32, np.float64]:
+      self._testRelu(
+          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
   def testReluInt8x4GoodShape(self):
-    self._testReluInt8x4(np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]]))
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest("No GPU available")
+    inputs = np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]])
+    np_relu = self._npRelu(inputs)
+    tf_relu = nn_ops.relu(constant_op.constant(inputs, dtypes.qint8))
+    self.assertAllClose(np_relu, tf_relu)
+    self.assertShapeEqual(np_relu, tf_relu)
 
   def testReluInt8x4BadShape(self):
-    np_inputs = np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]])
-    self.assertEqual(np_inputs.size, 9)
-    self._testReluInt8x4(np_inputs)
-    np_inputs = np.array(
-        [1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1])
-    self.assertEqual(np_inputs.size, 17)
-    self._testReluInt8x4(np_inputs)
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest("No GPU available")
+    inputs = constant_op.constant(
+        np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]), dtypes.qint8)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Tensor size must be a multiple of 4 for Relu<qint8>. Got 9"):
+      self.evaluate(nn_ops.relu(inputs))
+
+    inputs = constant_op.constant(
+        np.array([1, -2, 3, -4, 5, -6, 7, -8, 9, -8, 7, -6, 5, -4, 3, -2, 1]),
+        dtypes.qint8)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Tensor size must be a multiple of 4 for Relu<qint8>. Got 17"):
+      self.evaluate(nn_ops.relu(inputs))
 
   # The gradient test for ReLU is a bit tricky as the derivative is not well
   # defined at around zero and we want to avoid that in terms of input values.
@@ -202,15 +203,15 @@ class ReluTest(test.TestCase):
     self.assertLess(err, 1e-10)
 
   def testGradientScalar(self):
-    with self.cached_session() as sess:
-      x = variables.Variable(100.)
-      y = nn_ops.relu(x)
-      loss = y**2
-      optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25)
-      train_op = optimizer.minimize(loss)
-      sess.run(variables.global_variables_initializer())
-      sess.run(train_op)
-      self.assertAllClose(x.eval(), 50.0)
+    x = variables.Variable(100.)
+
+    def loss():
+      return nn_ops.relu(x)**2
+
+    optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.25)
+    self.evaluate(variables.global_variables_initializer())
+    self.evaluate(optimizer.minimize(loss))
+    self.assertAllClose(x.read_value(), 50.0)
 
 
 class Relu6Test(test.TestCase):
@@ -228,23 +229,25 @@ class Relu6Test(test.TestCase):
             np.array([[-0.9, 0.7, -0.5, 0.3, 6.0], [0.1, -0.3, 6.5, -0.7,
                                                     0.9]])))
 
-  def _testRelu6(self, np_features, use_gpu=False):
+  def _testRelu6(self, np_features):
     np_relu6 = self._npRelu6(np_features)
-    with self.cached_session(use_gpu=use_gpu):
-      relu6 = nn_ops.relu6(np_features)
-      tf_relu6 = relu6.eval()
+    tf_relu6 = nn_ops.relu6(np_features)
     self.assertAllClose(np_relu6, tf_relu6)
-    self.assertShapeEqual(np_relu6, relu6)
+    self.assertShapeEqual(np_relu6, tf_relu6)
 
-  def testNumbers(self):
+  def testNumbersCPU(self):
     for t in [np.int32, np.int64, np.float16, np.float32, np.float64]:
-      self._testRelu6(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=False)
-      if t in [np.float16, np.float, np.double]:
+      # Force execution on CPU even if a GPU kernel is available for the type.
+      with ops.device("/device:CPU:0"):
         self._testRelu6(
-            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-            use_gpu=True)
+            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
+
+  def testNumbersGPU(self):
+    if not test.is_gpu_available():
+      self.skipTest("No GPU available")
+    for t in [np.float16, np.float, np.double]:
+      self._testRelu6(
+          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
   # The gradient test for ReLU6 is a bit tricky as the derivative is
   # not well defined at around zero and six and we want to avoid that
@@ -297,25 +300,27 @@ class LeakyReluTest(test.TestCase):
                                                      0.9]]),
             alpha=0.1))
 
-  def _testLeakyRelu(self, np_features, alpha, use_gpu=False):
+  def _testLeakyRelu(self, np_features, alpha):
     np_leaky_relu = self._npLeakyRelu(np_features, alpha)
-    with self.test_session(use_gpu=use_gpu):
-      leaky_relu = nn_ops.leaky_relu(np_features, alpha)
-      tf_leaky_relu = leaky_relu.eval()
+    tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha)
     self.assertAllClose(np_leaky_relu, tf_leaky_relu)
-    self.assertShapeEqual(np_leaky_relu, leaky_relu)
+    self.assertShapeEqual(np_leaky_relu, tf_leaky_relu)
 
-  def testNumbers(self):
+  def testNumbersCPU(self):
     for t in [np.int32, np.int64, np.float16, np.float32, np.float64]:
-      self._testLeakyRelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          alpha=0.2,
-          use_gpu=False)
-      if t in [np.float16, np.float32, np.float64]:
+      # Force execution on CPU even if a GPU kernel is available for the type.
+      with ops.device("/device:CPU:0"):
         self._testLeakyRelu(
             np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-            alpha=0.1,
-            use_gpu=True)
+            alpha=0.2)
+
+  def testNumbersGPU(self):
+    if not test.is_gpu_available():
+      self.skipTest("No GPU available")
+    for t in [np.float16, np.float32, np.float64]:
+      self._testLeakyRelu(
+          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
+          alpha=0.1)
 
   # The gradient test for Leaky ReLU is a bit tricky as the derivative is not
   # well defined at around zero and we want to avoid that in terms of input
@@ -391,15 +396,15 @@ class LeakyReluTest(test.TestCase):
       self.assertLess(err, 1e-10)
 
   def testGradientScalar(self):
-    with self.test_session() as sess:
-      x = variables.Variable(-100.)
-      y = nn_ops.leaky_relu(x, 0.05)
-      loss = y**2
-      optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2)
-      train_op = optimizer.minimize(loss)
-      sess.run(variables.global_variables_initializer())
-      sess.run(train_op)
-      self.assertAllClose(x.eval(), -99.9)
+    x = variables.Variable(-100.)
+
+    def loss():
+      return nn_ops.leaky_relu(x, 0.05)**2
+
+    optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2)
+    self.evaluate(variables.global_variables_initializer())
+    self.evaluate(optimizer.minimize(loss))
+    self.assertAllClose(x.read_value(), -99.9)
 
 
 class EluTest(test.TestCase):
@@ -415,22 +420,24 @@ class EluTest(test.TestCase):
             np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
                                                      0.9]])))
 
-  def _testElu(self, np_features, use_gpu=False):
+  def _testElu(self, np_features):
     np_elu = self._npElu(np_features)
-    with self.cached_session(use_gpu=use_gpu):
-      elu = nn_ops.elu(np_features)
-      tf_elu = elu.eval()
+    tf_elu = nn_ops.elu(np_features)
     self.assertAllClose(np_elu, tf_elu)
-    self.assertShapeEqual(np_elu, elu)
+    self.assertShapeEqual(np_elu, tf_elu)
 
-  def testNumbers(self):
+  def testNumbersCPU(self):
     for t in [np.float16, np.float32, np.float64]:
-      self._testElu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=False)
-      self._testElu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=True)
+      # Force execution on CPU even if a GPU kernel is available for the type.
+      with ops.device("/device:CPU:0"):
+        self._testElu(
+            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
+
+  def testNumbersGPU(self):
+    if not test.is_gpu_available():
+      self.skipTest("No GPU available")
+    for t in [np.float16, np.float32, np.float64]:
+      self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
   def testGradientFloat32(self):
     with self.cached_session():
@@ -517,22 +524,20 @@ class SeluTest(test.TestCase):
             np.array([[-0.9, 0.7, -0.5, 0.3, -0.1], [0.1, -0.3, 0.5, -0.7,
                                                      0.9]])))
 
-  def _testSelu(self, np_features, use_gpu=False):
+  def _testSelu(self, np_features):
     np_selu = self._npSelu(np_features)
-    with self.cached_session(use_gpu=use_gpu):
-      selu = nn_ops.selu(np_features)
-      tf_selu = selu.eval()
+    tf_selu = nn_ops.selu(np_features)
     self.assertAllClose(np_selu, tf_selu)
-    self.assertShapeEqual(np_selu, selu)
+    self.assertShapeEqual(np_selu, tf_selu)
 
   def testNumbers(self):
     for t in [np.float16, np.float32, np.float64]:
       self._testSelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=False)
-      self._testSelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=True)
+          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
+      # Force executed on CPU in case GPU kernels are avaiable.
+      with ops.device("/device:CPU:0"):
+        self._testSelu(
+            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
   def testGradientFloat32(self):
     with self.cached_session():
@@ -599,46 +604,44 @@ class CreluTest(test.TestCase):
     t = nn_ops.crelu(f)
     self.assertEqual([50, 5, 7, 20], t.get_shape())
 
-  def _testCrelu(self, np_features, use_gpu=False):
+  def _testCrelu(self, np_features):
     np_relu = np.maximum(np_features, np.zeros_like(np_features))
     np_neg_relu = np.maximum(-np_features, np.zeros_like(np_features))
     np_crelu = np.concatenate((np_relu, np_neg_relu),
                               len(np_features.shape) - 1)
 
-    with self.cached_session(use_gpu=use_gpu):
-      crelu = nn_ops.crelu(np_features)
-      tf_relu = crelu.eval()
+    tf_crelu = nn_ops.crelu(np_features)
 
-    self.assertAllClose(np_crelu, tf_relu)
-    self.assertShapeEqual(np_crelu, crelu)
+    self.assertAllClose(np_crelu, tf_crelu)
+    self.assertShapeEqual(np_crelu, tf_crelu)
 
-  def testNumbers(self):
+  def testNumbersCPU(self):
     for t in [np.int32, np.int64, np.float16, np.float32, np.float64]:
-      self._testCrelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-          use_gpu=False)
-      if t in [np.float16, np.float32, np.float64]:
+      # Force execution on CPU even if a GPU kernel is available for the type.
+      with ops.device("/device:CPU:0"):
         self._testCrelu(
-            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
-            use_gpu=True)
+            np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
+
+  def testNumbersGPU(self):
+    if not test.is_gpu_available():
+      self.skipTest("No GPU available")
+    for t in [np.float16, np.float32, np.float64]:
+      self._testCrelu(
+          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
   def testNumbersWithAxis0(self):
-    with self.cached_session():
-      crelu = nn_ops.crelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0)
-      tf_relu = crelu.eval()
-      np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1],
-                           [0, 3, 0, 7, 0]])
-      self.assertAllEqual(np_crelu, tf_relu)
+    tf_crelu = nn_ops.crelu(
+        np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=0)
+    np_crelu = np.array([[0, 7, 0, 3, 0], [1, 0, 5, 0, 9], [9, 0, 5, 0, 1],
+                         [0, 3, 0, 7, 0]])
+    self.assertAllEqual(np_crelu, tf_crelu)
 
   def testNumbersWithAxis1(self):
-    with self.cached_session():
-      crelu = nn_ops.crelu(
-          np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1)
-      tf_relu = crelu.eval()
-      np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1],
-                           [1, 0, 5, 0, 9, 0, 3, 0, 7, 0]])
-      self.assertAllEqual(np_crelu, tf_relu)
+    tf_crelu = nn_ops.crelu(
+        np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]), axis=1)
+    np_crelu = np.array([[0, 7, 0, 3, 0, 9, 0, 5, 0, 1],
+                         [1, 0, 5, 0, 9, 0, 3, 0, 7, 0]])
+    self.assertAllEqual(np_crelu, tf_crelu)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 67593ee405a919f1eec048bd991ddf22dcb1ac22 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 14 Nov 2018 11:07:12 -0800
Subject: [PATCH 0226/1554] Add a new enum for reduction type in distribution
 strategy. Currently supports SUM, MEAN and ONLY_FIRST_REPLICA Allow accepting
 the new enum in `reduce` and `batch_reduce` APIs. Change some callers to use
 the new enum.

PiperOrigin-RevId: 221475510
---
 tensorflow/contrib/distribute/python/BUILD    |  5 ++
 .../collective_all_reduce_strategy_test.py    |  5 +-
 .../distribute/python/cross_tower_ops.py      | 85 +++++++++----------
 .../distribute/python/cross_tower_ops_test.py | 40 ++++-----
 .../distribute/python/minimize_loss_test.py   |  3 +-
 .../distribute/python/mirrored_strategy.py    | 28 +++---
 .../python/mirrored_strategy_multigpu_test.py |  9 +-
 .../distribute/python/one_device_strategy.py  |  4 +-
 .../python/parameter_server_strategy.py       | 15 ++--
 .../python/parameter_server_strategy_test.py  |  3 +-
 .../distribute/python/strategy_test_lib.py    |  8 +-
 .../contrib/distribute/python/tpu_strategy.py | 11 +--
 .../contrib/distribute/python/values.py       | 17 ++--
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/distribute/BUILD            |  8 ++
 tensorflow/python/distribute/reduce_util.py   | 58 +++++++++++++
 tensorflow/python/training/distribute.py      | 61 ++++++++-----
 17 files changed, 230 insertions(+), 131 deletions(-)
 create mode 100644 tensorflow/python/distribute/reduce_util.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 8e0866c505..24bcb98e09 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -85,6 +85,7 @@ py_library(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
         "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:tape",
     ],
@@ -105,6 +106,7 @@ py_library(
         "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/eager:context",
     ],
 )
@@ -151,6 +153,7 @@ py_library(
         "//tensorflow/python:distribute",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/eager:context",
         "@six_archive//:six",
     ],
@@ -343,6 +346,7 @@ py_library(
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:util",
+        "//tensorflow/python/distribute:reduce_util",
     ],
 )
 
@@ -669,6 +673,7 @@ py_library(
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/eager:context",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index e3d919dd0d..219c5f531c 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import keras
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -128,7 +129,7 @@ class CollectiveAllReduceStrategyTestBase(
           with ops.control_dependencies([fetched]):
             # TODO(yuefengz): support non-Mirrored variable as destinations.
             g = d.reduce(
-                variable_scope.VariableAggregation.SUM, g, destinations=v)
+                reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
                 d.update(v, update, g, grouped=False)):
               after_list.append(d.read_var(v))
@@ -224,7 +225,7 @@ class CollectiveAllReduceStrategyTestBase(
       x = distribution.call_for_each_replica(model_fn)
       reduced_x = distribution.unwrap(
           distribution.reduce(
-              variable_scope.VariableAggregation.MEAN, x,
+              reduce_util.ReduceOp.MEAN, x,
               destinations='/cpu:0'))[0]
       x = distribution.unwrap(x)[0]
 
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py
index b5b349aa64..994ed345d8 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py
@@ -24,12 +24,12 @@ import six
 from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import values as value_lib
 from tensorflow.python.client import device_lib
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import device_util
 
@@ -150,7 +150,7 @@ def _simple_broadcast(value, destinations):
 
 
 def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn,
-                   aggregation):
+                   reduce_op):
   # pylint: disable=g-missing-docstring
   all_values = []
   count = 0
@@ -164,12 +164,11 @@ def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn,
     with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
       reduced = cross_tower_utils.aggregate_tensors_or_indexed_slices(
           all_values, accumulation_fn)
-      if aggregation == vs.VariableAggregation.MEAN:
+      if reduce_op == reduce_util.ReduceOp.MEAN:
         reduced = cross_tower_utils.divide_by_n_tensors_or_indexed_slices(
             reduced, count)
-      elif aggregation != vs.VariableAggregation.SUM:
-        raise ValueError("`aggregation` must be VariableAggregation.SUM "
-                         "or VariableAggregation.MEAN.")
+      elif reduce_op != reduce_util.ReduceOp.SUM:
+        raise ValueError("`reduce_op` must be Reduce.SUM or Reduce.MEAN.")
   return reduced
 
 
@@ -179,15 +178,15 @@ class CrossDeviceOps(object):
   def __init__(self):
     pass
 
-  def reduce(self, aggregation, per_replica_value, destinations):
+  def reduce(self, reduce_op, per_replica_value, destinations):
     """Reduce `per_replica_value` to `destinations`.
 
-    It runs the reduction operation defined by `aggregation` and put the
+    It runs the reduction operation defined by `reduce_op` and put the
     result on `destinations`.
 
     Args:
-      aggregation: Indicates how a variable will be aggregated. Accepted values
-        are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`.
+      reduce_op: Indicates how per_replica_value will be reduced. Accepted
+        values are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`.
       per_replica_value: a PerReplica object or a tensor with device set.
       destinations: the reduction destinations.
 
@@ -201,17 +200,17 @@ class CrossDeviceOps(object):
       per_replica_value = _make_tensor_into_per_replica(per_replica_value)
 
     validate_destinations(destinations)
-    return self._reduce(aggregation, per_replica_value, destinations)
+    return self._reduce(reduce_op, per_replica_value, destinations)
 
-  def batch_reduce(self, aggregation, value_destination_pairs):
+  def batch_reduce(self, reduce_op, value_destination_pairs):
     """Reduce PerReplica objects in a batch.
 
     Reduce each first element in `value_destination_pairs` to each second
     element which indicates the destinations.
 
     Args:
-      aggregation: Indicates how a variable will be aggregated. Accepted values
-        are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`.
+      reduce_op: Indicates how per_replica_value will be reduced. Accepted
+        values are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`.
       value_destination_pairs: a list or a tuple of tuples of PerReplica objects
         (or tensors with device set if there is one device) and destinations.
 
@@ -231,7 +230,7 @@ class CrossDeviceOps(object):
     for _, d in value_destination_pairs:
       validate_destinations(d)
 
-    return self._batch_reduce(aggregation, value_destination_pairs)
+    return self._batch_reduce(reduce_op, value_destination_pairs)
 
   def broadcast(self, tensor, destinations):
     """Broadcast the `tensor` to destinations.
@@ -246,11 +245,11 @@ class CrossDeviceOps(object):
     validate_destinations(destinations)
     return self._broadcast(tensor, destinations)
 
-  def _reduce(self, aggregation, per_replica_value, destinations):
+  def _reduce(self, reduce_op, per_replica_value, destinations):
     raise NotImplementedError(
         "_reduce method must be implemented in descendants.")
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
     raise NotImplementedError(
         "_batch_reduce method must be implemented in descendants.")
 
@@ -276,19 +275,19 @@ class ReductionToOneDeviceCrossDeviceOps(CrossDeviceOps):
     self.accumulation_fn = accumulation_fn
     super(ReductionToOneDeviceCrossDeviceOps, self).__init__()
 
-  def _reduce(self, aggregation, per_replica_value, destinations):
+  def _reduce(self, reduce_op, per_replica_value, destinations):
     if check_destinations(destinations):
       devices = get_devices_from(destinations)
     else:
       devices = get_devices_from(per_replica_value)
     reduce_to_device = self.reduce_to_device or devices[0]
     reduced = _simple_reduce(per_replica_value, reduce_to_device,
-                             self.accumulation_fn, aggregation)
+                             self.accumulation_fn, reduce_op)
     return self.broadcast(reduced, devices)
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
     return [
-        self._reduce(aggregation, t, destinations=v)
+        self._reduce(reduce_op, t, destinations=v)
         for t, v in value_destination_pairs
     ]
 
@@ -323,20 +322,20 @@ def _group_value_by_device(per_replica_values):
 
 def _ungroup_and_make_mirrored(grouped_reduced,
                                destinations,
-                               aggregation,
+                               reduce_op,
                                num_between_graph_workers=1):
   """Ungroup results from all-reduce and make Mirrored objects.
 
   Each all-reduce result will be divided by the number of destinations before
-  Mirrored objects are created if aggregation is "mean".
+  Mirrored objects are created if reduce_op is "mean".
 
   Args:
     grouped_reduced: a list of lists, each sublist has components for each
       device, paired with a None. It is the result from
       cross_tower_utils.aggregate_gradients_using*.
     destinations: a list of device strings for returned Mirrored objects.
-    aggregation: Indicates how a variable will be aggregated. Accepted values
-      are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`.
+    reduce_op: Indicates how values will be aggregated. Accepted values
+      are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`.
     num_between_graph_workers: number of workers in the between-graph
       replication.
 
@@ -346,7 +345,7 @@ def _ungroup_and_make_mirrored(grouped_reduced,
   index = [{} for _ in range(len(grouped_reduced[0]))]
   for d, per_replica_reduced in enumerate(grouped_reduced):
     for i, (v, _) in enumerate(per_replica_reduced):
-      if aggregation == vs.VariableAggregation.MEAN:
+      if reduce_op == reduce_util.ReduceOp.MEAN:
         index[i][destinations[d]] = v / (
             len(destinations) * num_between_graph_workers)
       else:
@@ -557,13 +556,13 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
     self._agg_small_grads_max_group = agg_small_grads_max_group
     super(AllReduceCrossDeviceOps, self).__init__()
 
-  def _reduce(self, aggregation, per_replica_value, destinations):
+  def _reduce(self, reduce_op, per_replica_value, destinations):
     contains_indexed_slices = cross_tower_utils.contains_indexed_slices(
         per_replica_value)
     if (_devices_match(per_replica_value, destinations)
         and not context.executing_eagerly()
         and not contains_indexed_slices):
-      return self._batch_all_reduce(aggregation, [per_replica_value])[0]
+      return self._batch_all_reduce(reduce_op, [per_replica_value])[0]
     else:
       if contains_indexed_slices:
         logging.log_first_n(
@@ -576,16 +575,16 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
         devices = get_devices_from(per_replica_value)
       reduce_to_device = devices[0]
       reduced = _simple_reduce(per_replica_value, reduce_to_device,
-                               math_ops.add_n, aggregation)
+                               math_ops.add_n, reduce_op)
       return self.broadcast(reduced, devices)
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
     all_devices_match = _all_devices_match(value_destination_pairs)
     contains_indexed_slices = cross_tower_utils.contains_indexed_slices(
         value_destination_pairs)
     if (all_devices_match and not context.executing_eagerly()
         and not contains_indexed_slices):
-      return self._batch_all_reduce(aggregation,
+      return self._batch_all_reduce(reduce_op,
                                     [v[0] for v in value_destination_pairs])
     else:
       if not all_devices_match:
@@ -595,11 +594,11 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
                             10)
 
       return [
-          self._reduce(aggregation, t, destinations=v)
+          self._reduce(reduce_op, t, destinations=v)
           for t, v in value_destination_pairs
       ]
 
-  def _batch_all_reduce(self, aggregation, per_replica_values):
+  def _batch_all_reduce(self, reduce_op, per_replica_values):
     """All reduce algorithm in a batch."""
     logging.log_first_n(
         logging.INFO, "batch_all_reduce invoked for batches size = %d with "
@@ -630,7 +629,7 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
 
     reduced = _unpack_tensors(reduced, tensor_packer)
     return _ungroup_and_make_mirrored(reduced, per_replica_values[0].devices,
-                                      aggregation)
+                                      reduce_op)
 
 
 # For compatibility with code using the old name of `AllReduceCrossDeviceOps`.
@@ -713,7 +712,7 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps):
           validate_and_complete_spec(spec) for spec in all_reduce_spec
       ]
 
-  def _batch_all_reduce(self, aggregation, per_replica_values):
+  def _batch_all_reduce(self, reduce_op, per_replica_values):
     """All reduce algorithm in a batch."""
     logging.log_first_n(
         logging.INFO,
@@ -761,7 +760,7 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps):
     assert not remaining_grads
 
     return _ungroup_and_make_mirrored(aggregated_grads, destinations,
-                                      aggregation)
+                                      reduce_op)
 
 
 # TODO(yuefengz): support in-graph collective all-reduce.
@@ -795,7 +794,7 @@ class CollectiveAllReduce(CrossDeviceOps):
     super(CollectiveAllReduce, self).__init__()
 
   # TODO(yuefengz, tucker): is indexed slices supported by collective ops?
-  def _reduce(self, aggregation, per_replica_value, destinations):
+  def _reduce(self, reduce_op, per_replica_value, destinations):
     if cross_tower_utils.contains_indexed_slices(per_replica_value):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
@@ -803,7 +802,7 @@ class CollectiveAllReduce(CrossDeviceOps):
       raise ValueError(
           "Eager execution is not supported for Collective All-Reduce")
 
-    all_reduced = self._batch_all_reduce(aggregation, [per_replica_value])[0]
+    all_reduced = self._batch_all_reduce(reduce_op, [per_replica_value])[0]
     if _devices_match(per_replica_value, destinations):
       return all_reduced
     else:
@@ -819,7 +818,7 @@ class CollectiveAllReduce(CrossDeviceOps):
 
       return value_lib.Mirrored(index)
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
     if cross_tower_utils.contains_indexed_slices(value_destination_pairs):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
@@ -829,7 +828,7 @@ class CollectiveAllReduce(CrossDeviceOps):
 
     all_devices_match = _all_devices_match(value_destination_pairs)
     if all_devices_match:
-      return self._batch_all_reduce(aggregation,
+      return self._batch_all_reduce(reduce_op,
                                     [v[0] for v in value_destination_pairs])
     else:
       if not all_devices_match:
@@ -838,11 +837,11 @@ class CollectiveAllReduce(CrossDeviceOps):
             "destinations are different.", 10)
 
       return [
-          self._reduce(aggregation, t, destinations=v)
+          self._reduce(reduce_op, t, destinations=v)
           for t, v in value_destination_pairs
       ]
 
-  def _batch_all_reduce(self, aggregation, per_replica_values):
+  def _batch_all_reduce(self, reduce_op, per_replica_values):
     """All-reduce across all workers in a batch."""
     if context.executing_eagerly():
       raise ValueError(
@@ -883,7 +882,7 @@ class CollectiveAllReduce(CrossDeviceOps):
     return _ungroup_and_make_mirrored(
         new_device_grads,
         per_replica_values[0].devices,
-        aggregation,
+        reduce_op,
         num_between_graph_workers=self._num_workers)
 
 
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
index 3e274ba67c..2e352360a4 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
@@ -30,13 +30,13 @@ from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import values as value_lib
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.training import device_util
 
 
@@ -143,24 +143,24 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
     for destinations in all_destinations:
       self._assert_values_equal(
           cross_tower_ops.reduce(
-              vs.VariableAggregation.MEAN,
+              reduce_util.ReduceOp.MEAN,
               per_replica,
               destinations=destinations),
           _fake_mirrored(mean, destinations))
       self._assert_values_equal(
           cross_tower_ops.reduce(
-              vs.VariableAggregation.MEAN,
+              reduce_util.ReduceOp.MEAN,
               per_replica_2,
               destinations=destinations),
           _fake_mirrored(mean_2, destinations))
       self._assert_values_equal(
           cross_tower_ops.reduce(
-              vs.VariableAggregation.SUM, per_replica,
+              reduce_util.ReduceOp.SUM, per_replica,
               destinations=destinations),
           _fake_mirrored(mean * len(devices), destinations))
       self._assert_values_equal(
           cross_tower_ops.reduce(
-              vs.VariableAggregation.SUM,
+              reduce_util.ReduceOp.SUM,
               per_replica_2,
               destinations=destinations),
           _fake_mirrored(mean_2 * len(devices), destinations))
@@ -169,7 +169,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
     for d1, d2 in itertools.product(all_destinations, all_destinations):
       self._assert_values_equal(
           cross_tower_ops.batch_reduce(
-              vs.VariableAggregation.MEAN,
+              reduce_util.ReduceOp.MEAN,
               [(per_replica, d1), (per_replica_2, d2)]),
           [
               _fake_mirrored(mean, d1),
@@ -177,7 +177,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
           ])
       self._assert_values_equal(
           cross_tower_ops.batch_reduce(
-              vs.VariableAggregation.SUM,
+              reduce_util.ReduceOp.SUM,
               [(per_replica, d1), (per_replica_2, d2)]),
           [
               _fake_mirrored(mean * len(devices), d1),
@@ -281,7 +281,7 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
     t1 = _make_indexed_slices([[3., 4.], [5., 6.]], [1, 3], [5, 2], devices[1])
     per_replica = value_lib.PerReplica({devices[0]: t0, devices[1]: t1})
     result = cross_tower_ops_lib._simple_reduce(
-        per_replica, devices[0], math_ops.add_n, vs.VariableAggregation.SUM)
+        per_replica, devices[0], math_ops.add_n, reduce_util.ReduceOp.SUM)
 
     # Test that the result is semantically equal to both the concatenated
     # IndexedSlices with and without duplicate indices.
@@ -302,11 +302,11 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
                   "AllReduceCrossDeviceOps",
                   cross_tower_ops_lib.AllReduceCrossDeviceOps())
           ],
-          aggregation=[vs.VariableAggregation.SUM, vs.VariableAggregation.MEAN],
+          reduce_op=[reduce_util.ReduceOp.SUM, reduce_util.ReduceOp.MEAN],
           batch_reduce=[True, False],
           mode=["graph", "eager"],
           required_gpus=1))
-  def testIndexedSlicesAllReduce(self, cross_tower_ops_instance, aggregation,
+  def testIndexedSlicesAllReduce(self, cross_tower_ops_instance, reduce_op,
                                  batch_reduce):
     devices = ["/cpu:0", "/gpu:0"]
     dense_shape = [5, 2]
@@ -317,19 +317,19 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
 
     if batch_reduce:
       result = cross_tower_ops_instance.batch_reduce(
-          aggregation, [(per_replica, devices)])
+          reduce_op, [(per_replica, devices)])
     else:
       result = cross_tower_ops_instance.reduce(
-          aggregation, per_replica, devices)
+          reduce_op, per_replica, devices)
 
     total_indices_with_dups = [1, 1, 3]
     total_indices_without_dups = [1, 3]
 
-    if aggregation == vs.VariableAggregation.SUM:
+    if reduce_op == reduce_util.ReduceOp.SUM:
       total_values_with_dups = [[1., 2.], [3., 4.], [5., 6.]]
       total_values_without_dups = [[4., 6.], [5., 6.]]
     else:
-      assert aggregation == vs.VariableAggregation.MEAN
+      assert reduce_op == reduce_util.ReduceOp.MEAN
       total_values_with_dups = [[0.5, 1.], [1.5, 2.], [2.5, 3.]]
       total_values_without_dups = [[2., 3.], [2.5, 3.]]
 
@@ -502,26 +502,26 @@ class MultiWorkerCollectiveAllReduceTest(
       for destinations in all_destinations:
         self._assert_values_equal(
             collective_all_reduce.reduce(
-                vs.VariableAggregation.MEAN,
+                reduce_util.ReduceOp.MEAN,
                 per_replica,
                 destinations=destinations),
             _fake_mirrored(mean, destinations), sess)
         self._assert_values_equal(
             collective_all_reduce.reduce(
-                vs.VariableAggregation.MEAN,
+                reduce_util.ReduceOp.MEAN,
                 per_replica_2,
                 destinations=destinations),
             _fake_mirrored(mean_2, destinations), sess)
         self._assert_values_equal(
             collective_all_reduce.reduce(
-                vs.VariableAggregation.SUM,
+                reduce_util.ReduceOp.SUM,
                 per_replica,
                 destinations=destinations),
             _fake_mirrored(mean * len(devices) * num_workers, destinations),
             sess)
         self._assert_values_equal(
             collective_all_reduce.reduce(
-                vs.VariableAggregation.SUM,
+                reduce_util.ReduceOp.SUM,
                 per_replica_2,
                 destinations=destinations),
             _fake_mirrored(mean_2 * len(devices) * num_workers, destinations),
@@ -530,7 +530,7 @@ class MultiWorkerCollectiveAllReduceTest(
       # test batch_reduce()
       for d1, d2 in itertools.product(all_destinations, all_destinations):
         self._assert_values_equal(
-            collective_all_reduce.batch_reduce(vs.VariableAggregation.MEAN,
+            collective_all_reduce.batch_reduce(reduce_util.ReduceOp.MEAN,
                                                [(per_replica, d1),
                                                 (per_replica_2, d2)]),
             [
@@ -538,7 +538,7 @@ class MultiWorkerCollectiveAllReduceTest(
                 _fake_mirrored(mean_2, d2)
             ], sess)
         self._assert_values_equal(
-            collective_all_reduce.batch_reduce(vs.VariableAggregation.SUM,
+            collective_all_reduce.batch_reduce(reduce_util.ReduceOp.SUM,
                                                [(per_replica, d1),
                                                 (per_replica_2, d2)]),
             [
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index c6562463ed..5d3b5d8922 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -25,6 +25,7 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python.single_loss_example import batchnorm_example
 from tensorflow.contrib.distribute.python.single_loss_example import minimize_loss_example
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -484,7 +485,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       self.assertEqual(distribution.num_replicas_in_sync,
                        len(distribution.unwrap(loss_output)))
       loss_output = distribution.reduce(
-          aggregation=variables_lib.VariableAggregation.MEAN,
+          aggregation=reduce_util.ReduceOp.MEAN,
           value=loss_output, destinations="/device:CPU:0")
 
     unwrapped_output = distribution.unwrap(loss_output)
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 62619a5756..bf065713a0 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import shared_variable_creator
 from tensorflow.contrib.distribute.python import values
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
@@ -178,7 +179,7 @@ def _call_for_each_replica(distribution, fn, args, kwargs):
   return values.regroup({t.device: t.main_result for t in threads})
 
 
-def _reduce_non_distributed_value(distribution, aggregation, value,
+def _reduce_non_distributed_value(distribution, reduce_op, value,
                                   destinations):
   """Reduce a non-DistributedValue `value` to `destinations`."""
   if isinstance(value, values.DistributedValues):
@@ -190,21 +191,20 @@ def _reduce_non_distributed_value(distribution, aggregation, value,
   # and equal to 0.
   if value == 0:
     return 0
-  # If the aggregation type is MEAN or ONLY_FIRST_REPLICA, then this
+  # If the reduce op is MEAN or ONLY_FIRST_REPLICA, then this
   # essentially means that the same value should be on all destinations.
-  if aggregation in (
-      variable_scope.VariableAggregation.MEAN,
-      variable_scope.VariableAggregation.ONLY_FIRST_REPLICA):
+  if reduce_op in (reduce_util.ReduceOp.MEAN,
+                   reduce_util.ReduceOp.ONLY_FIRST_REPLICA):
     return value
 
   cross_tower_ops_lib.validate_destinations(destinations)
-  # We do not support an aggregation type of SUM if the value is the same across
+  # We do not support a reduce op of SUM if the value is the same across
   # all replicas. We call this as part of assign functions for MirroredVariables
   # and summing up identical values across replicas is not clearly defined.
   if (len(distribution.worker_devices) != 1 or
       not cross_tower_ops_lib.check_destinations(destinations)):
     raise ValueError("A non-DistributedValues value %s cannot be reduced with "
-                     "the given aggregation %s." % (value, aggregation))
+                     "the given reduce op %s." % (value, reduce_op))
   # TODO(anjalisridhar): Moves these methods to a device utility file?
   devices = cross_tower_ops_lib.get_devices_from(destinations)
   if len(devices) == 1:
@@ -588,28 +588,28 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
           cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps())
     return self._cross_tower_ops
 
-  def _reduce(self, aggregation, value, destinations):
+  def _reduce(self, reduce_op, value, destinations):
     assert not isinstance(value, values.Mirrored)
     if not isinstance(value, values.DistributedValues):
       # This function handles reducing values that are not PerReplica or
       # Mirrored values. For example, the same value could be present on all
       # replicas in which case `value` would be a single value or value could
       # be 0.
-      return _reduce_non_distributed_value(self, aggregation, value,
+      return _reduce_non_distributed_value(self, reduce_op, value,
                                            destinations)
-    if aggregation == variable_scope.VariableAggregation.ONLY_FIRST_REPLICA:
+    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
       value = value.get(self._devices[0])
       if isinstance(value, (int, float)):
         return value
       return self.broadcast(value, destinations)
     return self._get_cross_tower_ops().reduce(
-        aggregation, value, destinations=destinations)
+        reduce_op, value, destinations=destinations)
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
-    if aggregation == variable_scope.VariableAggregation.ONLY_FIRST_REPLICA:
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
+    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
       return [self.broadcast(v.get(self._devices[0]), d)
               for v, d in value_destination_pairs]
-    return self._get_cross_tower_ops().batch_reduce(aggregation,
+    return self._get_cross_tower_ops().batch_reduce(reduce_op,
                                                     value_destination_pairs)
 
   def _update(self, var, options, fn, *args, **kwargs):
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 73614a000f..23379a72d9 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
@@ -117,7 +118,7 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     with dist.scope():
       result = dist.call_for_each_replica(_replica_id)
       reduced = dist.reduce(
-          variable_scope.VariableAggregation.SUM,
+          reduce_util.ReduceOp.SUM,
           result,
           destinations="/device:CPU:0")
       unwrapped = dist.unwrap(reduced)
@@ -137,7 +138,7 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     with dist.scope():
       result = dist.call_for_each_replica(run_fn)
       reduced = dist.reduce(
-          variable_scope.VariableAggregation.ONLY_FIRST_REPLICA,
+          reduce_util.ReduceOp.ONLY_FIRST_REPLICA,
           result,
           destinations="/device:CPU:0")
       unwrapped = dist.unwrap(reduced)
@@ -157,7 +158,7 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     dist = mirrored_strategy.MirroredStrategy(devices)
     with dist.scope():
       reduced = dist.reduce(
-          variable_scope.VariableAggregation.SUM,
+          reduce_util.ReduceOp.SUM,
           1.0,
           destinations=["/device:CPU:0", "/device:GPU:0"])
       unwrapped = dist.unwrap(reduced)
@@ -912,7 +913,7 @@ class MirroredVariableUpdateTest(test.TestCase):
 
       with self.assertRaisesRegexp(
           ValueError, "A non-DistributedValues value 5.0 cannot be reduced "
-          "with the given aggregation VariableAggregation.SUM."):
+          "with the given reduce op ReduceOp.SUM."):
         self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
 
   @test_util.run_in_graph_and_eager_modes(config=config)
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index d623798d0c..8ce8e11589 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -122,8 +122,8 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     with ops.device(self._device), _OneDeviceReplicaContext(self):
       return fn(*args, **kwargs)
 
-  def _reduce(self, aggregation, value, destinations):
-    del aggregation, destinations
+  def _reduce(self, reduce_op, value, destinations):
+    del reduce_op, destinations
     return value
 
   def _update(self, var, options, fn, *args, **kwargs):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 438b91bc8d..49733dd7a6 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -22,6 +22,7 @@ from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import values
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
@@ -307,24 +308,24 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
             "Cannot reduce to another worker: %r, current worker is %r" %
             (d, self._worker_device))
 
-  def _reduce(self, aggregation, value, destinations):
+  def _reduce(self, reduce_op, value, destinations):
     self._verify_destinations_not_different_worker(destinations)
     if not isinstance(value, values.DistributedValues):
       # pylint: disable=protected-access
       return mirrored_strategy._reduce_non_distributed_value(
-          self, aggregation, value, destinations)
-    if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA:
+          self, reduce_op, value, destinations)
+    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
       return self.broadcast(value.get(self._compute_devices[0]), destinations)
     return self._cross_tower_ops.reduce(
-        aggregation, value, destinations=destinations)
+        reduce_op, value, destinations=destinations)
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
-    if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA:
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
+    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
       return [self.broadcast(v.get(self._compute_devices[0]), d)
               for v, d in value_destination_pairs]
     for _, destinations in value_destination_pairs:
       self._verify_destinations_not_different_worker(destinations)
-    return self._cross_tower_ops.batch_reduce(aggregation,
+    return self._cross_tower_ops.batch_reduce(reduce_op,
                                               value_destination_pairs)
 
   def _select_single_value(self, structured):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 81a23c8903..ec61f62dc7 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.distribute.python import parameter_server_strategy
 from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
@@ -473,7 +474,7 @@ class ParameterServerStrategyTestBase(
           with ops.control_dependencies([fetched]):
             # TODO(yuefengz): support non-Mirrored variable as destinations.
             g = d.reduce(
-                variable_scope.VariableAggregation.SUM, g, destinations=v)
+                reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
                 d.update(v, update, g, grouped=False)):
               after_list.append(d.read_var(v))
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 9fee75a476..31d7490290 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
@@ -26,7 +27,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer
@@ -114,8 +114,7 @@ class DistributionTestBase(test.TestCase):
           before_list.append(fetched)
           # control_dependencies irrelevant but harmless in eager execution
           with ops.control_dependencies([fetched]):
-            g = d.reduce(
-                variable_scope.VariableAggregation.SUM, g, destinations=v)
+            g = d.reduce(reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(d.update(
                 v, update, g, grouped=False)):
               after_list.append(d.read_var(v))
@@ -169,8 +168,7 @@ class DistributionTestBase(test.TestCase):
           fetched = d.read_var(v)
           before_list.append(fetched)
           with ops.control_dependencies([fetched]):
-            g = d.reduce(
-                variable_scope.VariableAggregation.SUM, g, destinations=v)
+            g = d.reduce(reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(d.update(
                 v, update, g, grouped=False)):
               after_list.append(d.read_var(v))
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 5ef59bf74d..ae5c55a346 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -31,6 +31,7 @@ from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_
 from tensorflow.contrib.tpu.python.tpu import training_loop
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
@@ -439,12 +440,12 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     return _create_tpu_mirrored_variable(devices, _real_mirrored_creator, *args,
                                          **kwargs)
 
-  def _reduce(self, aggregation, value, destinations):
+  def _reduce(self, reduce_op, value, destinations):
     if values._enclosing_tpu_context() is not None:  # pylint: disable=protected-access
-      if aggregation == vs.VariableAggregation.MEAN:
+      if reduce_op == reduce_util.ReduceOp.MEAN:
         # TODO(jhseu):  Revisit once we support model-parallelism.
         value *= (1. / self.num_replicas_in_sync)
-      elif aggregation != vs.VariableAggregation.SUM:
+      elif reduce_op != reduce_util.ReduceOp.SUM:
         raise NotImplementedError(
             "Currently only support sum & mean in TPUStrategy.")
       return tpu_ops.cross_replica_sum(value)
@@ -459,10 +460,10 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     else:
       raise ValueError("Multiple devices are not supported for TPUStrategy")
 
-    if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA:
+    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
       return value[0]
     output = math_ops.add_n(value)
-    if aggregation == vs.VariableAggregation.MEAN:
+    if reduce_op == reduce_util.ReduceOp.MEAN:
       return output * (1. / len(value))
     return output
 
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index ffb7b79839..e2931fcacd 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -30,6 +30,7 @@ import six
 from tensorflow.contrib.distribute.python import input_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import multi_device_iterator_ops
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import device as tf_device
@@ -373,12 +374,13 @@ class MirroredVariable(DistributedVariable, Mirrored,
       if self._aggregation == vs.VariableAggregation.NONE:
         raise ValueError("You must specify an aggregation method to update a "
                          "MirroredVariable in Replica Context.")
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(
+          self._aggregation)
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         return strategy.update(
             self, f,
-            strategy.reduce(
-                aggregation=self._aggregation, value=value, destinations=self),
+            strategy.reduce(reduce_op, value=value, destinations=self),
             *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
@@ -614,12 +616,13 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
       if self._aggregation == vs.VariableAggregation.NONE:
         raise ValueError("You must specify an aggregation method to update a "
                          "TPUMirroredVariable in Replica Context.")
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(
+          self._aggregation)
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         return strategy.update(
             self, f,
-            strategy.reduce(
-                aggregation=self._aggregation, value=value, destinations=self),
+            strategy.reduce(reduce_op, value=value, destinations=self),
             *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
@@ -1549,6 +1552,7 @@ class MultiStepContext(object):
         The aggregation method is also recorded in a dictionary
         `_last_step_outputs_aggregations` for later interpreting of the
         outputs as already reduced or not.
+        # TODO(priyag): Change aggregation type used here.
 
     """
     if distribution_strategy_context.get_cross_replica_context():
@@ -1650,12 +1654,13 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
       if self._aggregation == vs.VariableAggregation.NONE:
         raise ValueError("You must specify an aggregation method to update a "
                          "a variable in Replica Context.")
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(
+          self._aggregation)
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         return strategy.update(
             self, f,
-            strategy.reduce(
-                aggregation=self._aggregation, value=value, destinations=self),
+            strategy.reduce(reduce_op, value=value, destinations=self),
             *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 9bd6ddd083..a6e5c110b7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3590,6 +3590,7 @@ py_library(
         ":util",
         ":variable_scope",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/ops/losses",
     ],
 )
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 216dcc8587..13d797d987 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -155,3 +155,11 @@ py_library(
         "//tensorflow/python:training",
     ],
 )
+
+py_library(
+    name = "reduce_util",
+    srcs = [
+        "reduce_util.py",
+    ],
+    deps = [],
+)
diff --git a/tensorflow/python/distribute/reduce_util.py b/tensorflow/python/distribute/reduce_util.py
new file mode 100644
index 0000000000..8df3923daf
--- /dev/null
+++ b/tensorflow/python/distribute/reduce_util.py
@@ -0,0 +1,58 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilites for reduce operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import enum
+
+from tensorflow.python.ops import variable_scope
+
+
+# TODO(priyag): Add this to tf.distribute namespace when it exists.
+class ReduceOp(enum.Enum):
+  """Indicates how a set of values should be reduced.
+
+  * `SUM`: Add all the values.
+  * `MEAN`: Take the arithmetic mean ("average") of the values.
+  * `ONLY_FIRST_REPLICA`: Return the value on the first replica.
+
+  TODO(priyag): Add the following types:
+  * `MIN`: Return the minimum of all values.
+  * `MAX`: Return the maximum of all values.
+  """
+
+  SUM = 0
+  MEAN = 1
+  ONLY_FIRST_REPLICA = 2
+
+  @staticmethod
+  def from_variable_aggregation(aggregation):
+    mapping = {
+        variable_scope.VariableAggregation.SUM: ReduceOp.SUM,
+        variable_scope.VariableAggregation.MEAN: ReduceOp.MEAN,
+        variable_scope.VariableAggregation.ONLY_FIRST_REPLICA:
+            ReduceOp.ONLY_FIRST_REPLICA
+    }
+
+    reduce_op = mapping.get(aggregation)
+    if not reduce_op:
+      raise ValueError("Could not convert from `tf.VariableAggregation` to"
+                       "`tf.distribute.ReduceOp` type")
+    return reduce_op
+
+
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index f1d2db0647..4ed2fb1925 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import threading
 
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import reduce_util
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -755,9 +756,13 @@ class DistributionStrategy(object):
     """Combine (via e.g. sum or mean) values across replicas.
 
     Args:
-      aggregation: Indicates how a variable will be aggregated. Accepted values
-        are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`,
+      aggregation: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+        DEPRECATED but still accepted values:
+        `tf.VariableAggregation.SUM`,
+        `tf.VariableAggregation.MEAN`,
         `tf.VariableAggregation.ONLY_FIRST_REPLICA`.
+        # TODO(priyag): Rename this argument when moving the method to
+        # DSExtended.
       value: A per-replica value with one value per replica.
       destinations: A mirrored variable, a per-replica tensor, a device string,
         or list of device strings. The return value will be copied to all
@@ -771,23 +776,32 @@ class DistributionStrategy(object):
     # TODO(josh11b): Return an unwrapped value if colocate_with is a
     # single device.
     _require_cross_replica_context(self)
-    assert aggregation in [
-        variable_scope.VariableAggregation.SUM,
-        variable_scope.VariableAggregation.MEAN,
-        variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
-    ]
-    return self._reduce(aggregation, value, destinations)
 
-  def _reduce(self, aggregation, value, destinations):
+    # TODO(priyag): Remove this when all callers have been updated.
+    reduce_op = aggregation
+    if isinstance(aggregation, variable_scope.VariableAggregation):
+      assert aggregation in [
+          variable_scope.VariableAggregation.SUM,
+          variable_scope.VariableAggregation.MEAN,
+          variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
+      ]
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
+    return self._reduce(reduce_op, value, destinations)
+
+  def _reduce(self, reduce_op, value, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
   def batch_reduce(self, aggregation, value_destination_pairs):
     """Combine multiple `reduce` calls into one for faster execution.
 
     Args:
-      aggregation: Indicates how a variable will be aggregated. Accepted values
-        are `tf.VariableAggregation.SUM`, `tf.VariableAggregation.MEAN`,
+      aggregation: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+        DEPRECATED but still accepted values:
+        `tf.VariableAggregation.SUM`,
+        `tf.VariableAggregation.MEAN`,
         `tf.VariableAggregation.ONLY_FIRST_REPLICA`.
+        # TODO(priyag): Rename this argument when moving the method to
+        # DSExtended.
       value_destination_pairs: A sequence of (value, destinations)
         pairs. See `reduce()` for a description.
 
@@ -796,16 +810,21 @@ class DistributionStrategy(object):
     """
     # TODO(josh11b): More docstring
     _require_cross_replica_context(self)
-    assert aggregation in [
-        variable_scope.VariableAggregation.SUM,
-        variable_scope.VariableAggregation.MEAN,
-        variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
-    ]
-    return self._batch_reduce(aggregation, value_destination_pairs)
 
-  def _batch_reduce(self, aggregation, value_destination_pairs):
+    # TODO(priyag): Remove this when all callers have been updated.
+    reduce_op = aggregation
+    if isinstance(aggregation, variable_scope.VariableAggregation):
+      assert aggregation in [
+          variable_scope.VariableAggregation.SUM,
+          variable_scope.VariableAggregation.MEAN,
+          variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
+      ]
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
+    return self._batch_reduce(reduce_op, value_destination_pairs)
+
+  def _batch_reduce(self, reduce_op, value_destination_pairs):
     return [
-        self.reduce(aggregation, t, destinations=v)
+        self.reduce(reduce_op, t, destinations=v)
         for t, v in value_destination_pairs
     ]
 
@@ -1154,9 +1173,9 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     with ReplicaContext(self, replica_id=0):
       return fn(*args, **kwargs)
 
-  def _reduce(self, aggregation, value, destinations):
+  def _reduce(self, reduce_op, value, destinations):
     # TODO(josh11b): Use destinations?
-    del aggregation, destinations
+    del reduce_op, destinations
     return value
 
   def _update(self, var, options, fn, *args, **kwargs):
-- 
GitLab


From 6d11cff78085799c847bc676955a8933fd1fa3a2 Mon Sep 17 00:00:00 2001
From: Nick Hamatake <hamatake@google.com>
Date: Wed, 14 Nov 2018 11:24:31 -0800
Subject: [PATCH 0227/1554] Add encoding data to support core MIME/Unix/Windows
 encodings, including core ISO 8859 sets (Latin 1-5, Cyrillic, Arabic, Greek,
 Hebrew), various EUCs, and common CJK encodings.

Data is hard-coded in order to achieve a hermetic runtime, and to simplify the build process. Data is in little-endian byte order. Trying to decode/encode CJK or other code-mapped encodings on a big-endian platform will result in a runtime InvalidArgumentError ("Could not create converter for...").

PiperOrigin-RevId: 221478869
---
 tensorflow/core/kernels/BUILD                 |   1 +
 .../kernel_tests/unicode_transcode_op_test.py |  54 +++
 tensorflow/tools/lib_package/BUILD            |   2 +
 tensorflow/tools/pip_package/BUILD            |   1 +
 third_party/icu/data/BUILD                    |  46 ++
 third_party/icu/data/LICENSE                  | 414 ++++++++++++++++++
 .../icu/data/icu_conversion_data.c.gz.aa      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ab      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ac      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ad      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ae      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.af      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ag      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ah      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.ai      | Bin 0 -> 177696 bytes
 .../icu/data/icu_conversion_data.c.gz.aj      | Bin 0 -> 177689 bytes
 third_party/icu/udata.patch                   |  53 +++
 third_party/icu/workspace.bzl                 |   6 +
 18 files changed, 577 insertions(+)
 create mode 100644 third_party/icu/data/BUILD
 create mode 100644 third_party/icu/data/LICENSE
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.aa
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ab
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ac
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ad
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ae
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.af
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ag
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ah
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.ai
 create mode 100644 third_party/icu/data/icu_conversion_data.c.gz.aj
 create mode 100644 third_party/icu/udata.patch

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0f843ef125..5fd0f5d5d1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4800,6 +4800,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:string_ops_op_lib",
+        "//third_party/icu/data:conversion_data",
         "@icu//:common",
     ],
 )
diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
index 4406907a4c..4ad5ee4103 100644
--- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
@@ -378,6 +378,60 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
 
       self.assertAllEqual([b"AbCdE", b"HiJkL"], transcoded)
 
+  def test_cjk_encodings(self):
+    strings_ja = [
+        b"\x5c\x5c",  # Yen sign
+        b"\x8f\x70",  # kanji character "waza"
+        b"\x83\x4f"
+    ]  # katakana character "gu"
+    strings_zh_cn = [b"\xca\xf5"]  # simplified "shu4"
+    strings_zh_tw = [b"\xb3\x4e"]  # traditional "shu4"
+    strings_ko = [b"\xc7\xd1\xb9\xce"]  # hangul "hanmin"
+
+    expected_ja = [s.decode("shift_jis").encode("UTF-8") for s in strings_ja]
+    expected_zh_cn = [
+        s.decode("gb18030").encode("UTF-8") for s in strings_zh_cn
+    ]
+    expected_zh_tw = [s.decode("big5").encode("UTF-8") for s in strings_zh_tw]
+    expected_ko = [s.decode("euc_kr").encode("UTF-8") for s in strings_ko]
+
+    with self.cached_session() as sess:
+      outputs_ja = string_ops.unicode_transcode(
+          strings_ja,
+          input_encoding="shift_jis",
+          output_encoding="UTF-8",
+          replacement_char=ord(" "),
+          replace_control_characters=False)
+
+      outputs_zh_cn = string_ops.unicode_transcode(
+          strings_zh_cn,
+          input_encoding="gb18030",
+          output_encoding="UTF-8",
+          replacement_char=ord(" "),
+          replace_control_characters=False)
+
+      outputs_zh_tw = string_ops.unicode_transcode(
+          strings_zh_tw,
+          input_encoding="big5",
+          output_encoding="UTF-8",
+          replacement_char=ord(" "),
+          replace_control_characters=False)
+
+      outputs_ko = string_ops.unicode_transcode(
+          strings_ko,
+          input_encoding="euc_kr",
+          output_encoding="UTF-8",
+          replacement_char=ord(" "),
+          replace_control_characters=False)
+
+      result_ja, result_zh_cn, result_zh_tw, result_ko = sess.run(
+          [outputs_ja, outputs_zh_cn, outputs_zh_tw, outputs_ko])
+
+      self.assertAllEqual(result_ja, expected_ja)
+      self.assertAllEqual(result_zh_cn, expected_zh_cn)
+      self.assertAllEqual(result_zh_tw, expected_zh_tw)
+      self.assertAllEqual(result_ko, expected_ko)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 7aaa845ae9..1cac5ee138 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -112,6 +112,7 @@ pkg_tar(
 genrule(
     name = "clicenses_generate",
     srcs = [
+        "//third_party/icu/data:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
@@ -180,6 +181,7 @@ genrule(
 genrule(
     name = "jnilicenses_generate",
     srcs = [
+        "//third_party/icu/data:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 3a863d3c52..fa372dcd74 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -132,6 +132,7 @@ py_binary(
 filegroup(
     name = "licenses",
     data = [
+        "//third_party/icu/data:LICENSE",
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
diff --git a/third_party/icu/data/BUILD b/third_party/icu/data/BUILD
new file mode 100644
index 0000000000..7db21566e4
--- /dev/null
+++ b/third_party/icu/data/BUILD
@@ -0,0 +1,46 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+# Data for core MIME/Unix/Windows encodings:
+# ISO 8859-2..9, 15; Windows-125x; EUC-CN; GBK (Windows cp936); GB 18030;
+# Big5 (Windows cp950); SJIS (Windows cp932); EUC-JP; EUC-KR, KS C 5601;
+# Windows cp949. Data is pre-processed for little-endian platforms. To replicate
+# this pre-processing (if you want additional encodings, for example), do the
+# following:
+#
+# First, download, build, and install ICU. This installs tools such as makeconv.
+# Then, run the following from your icu4c/source directory:
+#   $ cd data/mappings
+#   $ rm *.cnv  # there shouldn't be any .cnv files here to begin with
+#   $ grep \.ucm ucmcore.mk | \
+#     sed 's/\(UCM_SOURCE_CORE=\)\?\([^ ]\+\.ucm\)\\\?/\2/g' | \
+#     tr '\n' ' ' | xargs makeconv
+#   $ ls *.cnv > filelist.lst
+#   $ pkgdata -m common -p ucmcore filelist.lst
+#   $ genccode -f custom_conversion_data ucmcore.dat
+# This creates custom_conversion_data.c. You will need to change the target
+# :conversion_data to depend on your custom source instead of :conversion_data.c
+filegroup(
+    name = "conversion_files",
+    srcs = glob(["icu_conversion_data.c.gz.*"]),
+)
+
+# Data files are compressed and split to work around git performance degradation
+# around large files.
+genrule(
+    name = "merge_conversion_data",
+    srcs = [":conversion_files"],
+    outs = ["conversion_data.c"],
+    cmd = "cat $(locations :conversion_files) | gunzip > $@",
+)
+
+cc_library(
+    name = "conversion_data",
+    srcs = [":conversion_data.c"],
+    deps = ["@icu//:headers"],
+)
diff --git a/third_party/icu/data/LICENSE b/third_party/icu/data/LICENSE
new file mode 100644
index 0000000000..25b6eb9d34
--- /dev/null
+++ b/third_party/icu/data/LICENSE
@@ -0,0 +1,414 @@
+COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later)
+
+Copyright © 1991-2018 Unicode, Inc. All rights reserved.
+Distributed under the Terms of Use in http://www.unicode.org/copyright.html.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Unicode data files and any associated documentation
+(the "Data Files") or Unicode software and any associated documentation
+(the "Software") to deal in the Data Files or Software
+without restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, and/or sell copies of
+the Data Files or Software, and to permit persons to whom the Data Files
+or Software are furnished to do so, provided that either
+(a) this copyright and permission notice appear with all copies
+of the Data Files or Software, or
+(b) this copyright and permission notice appear in associated
+Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder
+shall not be used in advertising or otherwise to promote the sale,
+use or other dealings in these Data Files or Software without prior
+written authorization of the copyright holder.
+
+---------------------
+
+Third-Party Software Licenses
+
+This section contains third-party software notices and/or additional
+terms for licensed third-party software components included within ICU
+libraries.
+
+1. ICU License - ICU 1.8.1 to ICU 57.1
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright (c) 1995-2016 International Business Machines Corporation and others
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, and/or sell copies of the Software, and to permit persons
+to whom the Software is furnished to do so, provided that the above
+copyright notice(s) and this permission notice appear in all copies of
+the Software and that both the above copyright notice(s) and this
+permission notice appear in supporting documentation.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY
+SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER
+RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder
+shall not be used in advertising or otherwise to promote the sale, use
+or other dealings in this Software without prior written authorization
+of the copyright holder.
+
+All trademarks and registered trademarks mentioned herein are the
+property of their respective owners.
+
+2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt)
+
+ #     The Google Chrome software developed by Google is licensed under
+ # the BSD license. Other software included in this distribution is
+ # provided under other licenses, as set forth below.
+ #
+ #  The BSD License
+ #  http://opensource.org/licenses/bsd-license.php
+ #  Copyright (C) 2006-2008, Google Inc.
+ #
+ #  All rights reserved.
+ #
+ #  Redistribution and use in source and binary forms, with or without
+ # modification, are permitted provided that the following conditions are met:
+ #
+ #  Redistributions of source code must retain the above copyright notice,
+ # this list of conditions and the following disclaimer.
+ #  Redistributions in binary form must reproduce the above
+ # copyright notice, this list of conditions and the following
+ # disclaimer in the documentation and/or other materials provided with
+ # the distribution.
+ #  Neither the name of  Google Inc. nor the names of its
+ # contributors may be used to endorse or promote products derived from
+ # this software without specific prior written permission.
+ #
+ #
+ #  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ #
+ #
+ #  The word list in cjdict.txt are generated by combining three word lists
+ # listed below with further processing for compound word breaking. The
+ # frequency is generated with an iterative training against Google web
+ # corpora.
+ #
+ #  * Libtabe (Chinese)
+ #    - https://sourceforge.net/project/?group_id=1519
+ #    - Its license terms and conditions are shown below.
+ #
+ #  * IPADIC (Japanese)
+ #    - http://chasen.aist-nara.ac.jp/chasen/distribution.html
+ #    - Its license terms and conditions are shown below.
+ #
+ #  ---------COPYING.libtabe ---- BEGIN--------------------
+ #
+ #  /*
+ #   * Copyright (c) 1999 TaBE Project.
+ #   * Copyright (c) 1999 Pai-Hsiang Hsiao.
+ #   * All rights reserved.
+ #   *
+ #   * Redistribution and use in source and binary forms, with or without
+ #   * modification, are permitted provided that the following conditions
+ #   * are met:
+ #   *
+ #   * . Redistributions of source code must retain the above copyright
+ #   *   notice, this list of conditions and the following disclaimer.
+ #   * . Redistributions in binary form must reproduce the above copyright
+ #   *   notice, this list of conditions and the following disclaimer in
+ #   *   the documentation and/or other materials provided with the
+ #   *   distribution.
+ #   * . Neither the name of the TaBE Project nor the names of its
+ #   *   contributors may be used to endorse or promote products derived
+ #   *   from this software without specific prior written permission.
+ #   *
+ #   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ #   * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ #   * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ #   * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ #   * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ #   * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ #   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ #   * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ #   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ #   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ #   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ #   * OF THE POSSIBILITY OF SUCH DAMAGE.
+ #   */
+ #
+ #  /*
+ #   * Copyright (c) 1999 Computer Systems and Communication Lab,
+ #   *                    Institute of Information Science, Academia
+ #       *                    Sinica. All rights reserved.
+ #   *
+ #   * Redistribution and use in source and binary forms, with or without
+ #   * modification, are permitted provided that the following conditions
+ #   * are met:
+ #   *
+ #   * . Redistributions of source code must retain the above copyright
+ #   *   notice, this list of conditions and the following disclaimer.
+ #   * . Redistributions in binary form must reproduce the above copyright
+ #   *   notice, this list of conditions and the following disclaimer in
+ #   *   the documentation and/or other materials provided with the
+ #   *   distribution.
+ #   * . Neither the name of the Computer Systems and Communication Lab
+ #   *   nor the names of its contributors may be used to endorse or
+ #   *   promote products derived from this software without specific
+ #   *   prior written permission.
+ #   *
+ #   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ #   * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ #   * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ #   * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ #   * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ #   * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ #   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ #   * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ #   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ #   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ #   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ #   * OF THE POSSIBILITY OF SUCH DAMAGE.
+ #   */
+ #
+ #  Copyright 1996 Chih-Hao Tsai @ Beckman Institute,
+ #      University of Illinois
+ #  c-tsai4@uiuc.edu  http://casper.beckman.uiuc.edu/~c-tsai4
+ #
+ #  ---------------COPYING.libtabe-----END--------------------------------
+ #
+ #
+ #  ---------------COPYING.ipadic-----BEGIN-------------------------------
+ #
+ #  Copyright 2000, 2001, 2002, 2003 Nara Institute of Science
+ #  and Technology.  All Rights Reserved.
+ #
+ #  Use, reproduction, and distribution of this software is permitted.
+ #  Any copy of this software, whether in its original form or modified,
+ #  must include both the above copyright notice and the following
+ #  paragraphs.
+ #
+ #  Nara Institute of Science and Technology (NAIST),
+ #  the copyright holders, disclaims all warranties with regard to this
+ #  software, including all implied warranties of merchantability and
+ #  fitness, in no event shall NAIST be liable for
+ #  any special, indirect or consequential damages or any damages
+ #  whatsoever resulting from loss of use, data or profits, whether in an
+ #  action of contract, negligence or other tortuous action, arising out
+ #  of or in connection with the use or performance of this software.
+ #
+ #  A large portion of the dictionary entries
+ #  originate from ICOT Free Software.  The following conditions for ICOT
+ #  Free Software applies to the current dictionary as well.
+ #
+ #  Each User may also freely distribute the Program, whether in its
+ #  original form or modified, to any third party or parties, PROVIDED
+ #  that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear
+ #  on, or be attached to, the Program, which is distributed substantially
+ #  in the same form as set out herein and that such intended
+ #  distribution, if actually made, will neither violate or otherwise
+ #  contravene any of the laws and regulations of the countries having
+ #  jurisdiction over the User or the intended distribution itself.
+ #
+ #  NO WARRANTY
+ #
+ #  The program was produced on an experimental basis in the course of the
+ #  research and development conducted during the project and is provided
+ #  to users as so produced on an experimental basis.  Accordingly, the
+ #  program is provided without any warranty whatsoever, whether express,
+ #  implied, statutory or otherwise.  The term "warranty" used herein
+ #  includes, but is not limited to, any warranty of the quality,
+ #  performance, merchantability and fitness for a particular purpose of
+ #  the program and the nonexistence of any infringement or violation of
+ #  any right of any third party.
+ #
+ #  Each user of the program will agree and understand, and be deemed to
+ #  have agreed and understood, that there is no warranty whatsoever for
+ #  the program and, accordingly, the entire risk arising from or
+ #  otherwise connected with the program is assumed by the user.
+ #
+ #  Therefore, neither ICOT, the copyright holder, or any other
+ #  organization that participated in or was otherwise related to the
+ #  development of the program and their respective officials, directors,
+ #  officers and other employees shall be held liable for any and all
+ #  damages, including, without limitation, general, special, incidental
+ #  and consequential damages, arising out of or otherwise in connection
+ #  with the use or inability to use the program or any product, material
+ #  or result produced or otherwise obtained by using the program,
+ #  regardless of whether they have been advised of, or otherwise had
+ #  knowledge of, the possibility of such damages at any time during the
+ #  project or thereafter.  Each user will be deemed to have agreed to the
+ #  foregoing by his or her commencement of use of the program.  The term
+ #  "use" as used herein includes, but is not limited to, the use,
+ #  modification, copying and distribution of the program and the
+ #  production of secondary products from the program.
+ #
+ #  In the case where the program, whether in its original form or
+ #  modified, was distributed or delivered to or received by a user from
+ #  any person, organization or entity other than ICOT, unless it makes or
+ #  grants independently of ICOT any specific warranty to the user in
+ #  writing, such person, organization or entity, will also be exempted
+ #  from and not be held liable to the user for any such damages as noted
+ #  above as far as the program is concerned.
+ #
+ #  ---------------COPYING.ipadic-----END----------------------------------
+
+3. Lao Word Break Dictionary Data (laodict.txt)
+
+ #  Copyright (c) 2013 International Business Machines Corporation
+ #  and others. All Rights Reserved.
+ #
+ # Project: http://code.google.com/p/lao-dictionary/
+ # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt
+ # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt
+ #              (copied below)
+ #
+ #  This file is derived from the above dictionary, with slight
+ #  modifications.
+ #  ----------------------------------------------------------------------
+ #  Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell.
+ #  All rights reserved.
+ #
+ #  Redistribution and use in source and binary forms, with or without
+ #  modification,
+ #  are permitted provided that the following conditions are met:
+ #
+ #
+ # Redistributions of source code must retain the above copyright notice, this
+ #  list of conditions and the following disclaimer. Redistributions in
+ #  binary form must reproduce the above copyright notice, this list of
+ #  conditions and the following disclaimer in the documentation and/or
+ #  other materials provided with the distribution.
+ #
+ #
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ # OF THE POSSIBILITY OF SUCH DAMAGE.
+ #  --------------------------------------------------------------------------
+
+4. Burmese Word Break Dictionary Data (burmesedict.txt)
+
+ #  Copyright (c) 2014 International Business Machines Corporation
+ #  and others. All Rights Reserved.
+ #
+ #  This list is part of a project hosted at:
+ #    github.com/kanyawtech/myanmar-karen-word-lists
+ #
+ #  --------------------------------------------------------------------------
+ #  Copyright (c) 2013, LeRoy Benjamin Sharon
+ #  All rights reserved.
+ #
+ #  Redistribution and use in source and binary forms, with or without
+ #  modification, are permitted provided that the following conditions
+ #  are met: Redistributions of source code must retain the above
+ #  copyright notice, this list of conditions and the following
+ #  disclaimer.  Redistributions in binary form must reproduce the
+ #  above copyright notice, this list of conditions and the following
+ #  disclaimer in the documentation and/or other materials provided
+ #  with the distribution.
+ #
+ #    Neither the name Myanmar Karen Word Lists, nor the names of its
+ #    contributors may be used to endorse or promote products derived
+ #    from this software without specific prior written permission.
+ #
+ #  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ #  CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ #  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ #  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ #  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+ #  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ #  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ #  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ #  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ #  ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+ #  TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ #  THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ #  SUCH DAMAGE.
+ #  --------------------------------------------------------------------------
+
+5. Time Zone Database
+
+  ICU uses the public domain data and code derived from Time Zone
+Database for its time zone support. The ownership of the TZ database
+is explained in BCP 175: Procedure for Maintaining the Time Zone
+Database section 7.
+
+ # 7.  Database Ownership
+ #
+ #    The TZ database itself is not an IETF Contribution or an IETF
+ #    document.  Rather it is a pre-existing and regularly updated work
+ #    that is in the public domain, and is intended to remain in the
+ #    public domain.  Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do
+ #    not apply to the TZ Database or contributions that individuals make
+ #    to it.  Should any claims be made and substantiated against the TZ
+ #    Database, the organization that is providing the IANA
+ #    Considerations defined in this RFC, under the memorandum of
+ #    understanding with the IETF, currently ICANN, may act in accordance
+ #    with all competent court orders.  No ownership claims will be made
+ #    by ICANN or the IETF Trust on the database or the code.  Any person
+ #    making a contribution to the database or code waives all rights to
+ #    future claims in that contribution or in the TZ Database.
+
+6. Google double-conversion
+
+Copyright 2006-2011, the V8 project authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aa b/third_party/icu/data/icu_conversion_data.c.gz.aa
new file mode 100644
index 0000000000000000000000000000000000000000..b68a2c6516f8183e805c509a9139cf63d1ee3fa5
GIT binary patch
literal 177696
zcmb2|=HOWCelD7UIWxI5J~=<HEVZaOGe0jrC9xz?FPY)Z-{|$azGd^+@12hi+>pPk
z@n*^~%R_GQCsb}LBzQO6-j!h@)#5Wvg~>?sL&8CWKliJT{obQov~KJ7@>N=a^Phx%
zeJg5g{CLX5LW#$+%kBR?zW--){ZIS<-T#l*|4;w_=ls9r_22CO-<<#Nng73|^Z(tf
z|2*6NulW9-&;LKF|9AHP5B+~%&i{}9KmWM=|Bw0qzHG1me*ORL{Qs}a|Nnab?_2%f
zv-|(d-v96P{=cj5|9f`+|C9KC&({CDYX4{T^Zb8j@Bho+|9AKPO56YO|GwY<bG!aW
zd;S0We?H3pZ~y;6{@)AvlRw<+f0zG%*8l%g{{O%K|CQJO&Hwk-{?Enzf7}0mz5n;(
zx-b5ZuI2yBum4p4-*;co|Fipl&EEg(?Ec^3|3BQ`|2u!t)aUvC{^bAr)c@r{{{M&f
z|9yM^?_a;%|F7}?U+@39`u@My_y2&rd->!3ACvEXK3@NK*Z=GFpKr@of6@Q{$p8PJ
z|9{W_+x`FH`G1e<e{HV+cmCgT`~T1X|INR&`qciPr|ZA>uP^_#{?2pzf5-oSjsN>}
zf9=!%i|c>K+x~lf|KHj8zhC#){W-n=$7K7zo9%zf|9iCm_vZiq&j0`USpWaG{D1%C
z>pp+~|DpW<kLmwE%>MuQ<LvAI|Gmop|FQmO`ToCW|9{&4|C#*%fBUb_t^f0S|IhZH
z;{QKLO|Sp)_Wu9+;*~q&KmY%J|KGEZ_5Yr&|Nlw;|D*T+zsdi9r(geDyjtM@qtN@=
z0{{Q52U+y!{r|)Je|`V|Z+rdL_y6tx_ur4HFR_j1o$_bTJj=7`k>_h{-~WIAyVmCN
z)_IzH>&`t_{ux*qcmMDI#jbr<eoemrbJqETdn0S#Uwr+#?sDiprS!Syw@kZr*3|V{
zrEPra)!2D<vFCTbuDW~A)^p$Y6JM_${}r?Te(ujSub-A}@77K({Q5j**Yu*w(=*l`
zx_eD2`_=D@tFL?Bl`sGO`ResWjnV3#U!4E6>+birVrSo8i+jJxU4fr}PTc>m-`;7P
zhwdr<d^PfUO|NI{ojZH?ZvOUU_xgMHD&4={mCW6n=XZVA?^kQqtHr#2w=z3+z2S+;
zo)aWL{f(;5$a+{Ny>IQA_1br9d5={;o48|cT>aeNSDRmIU0Qwcgl*aR7|&TZzdxTR
zGOy>S=_%7+b2ew6+v;|9J=^DOuHd7W=G>V4(Ddb}mtUUVUY|el`|(g^%}@K@y_-2T
zCfa&)+4(<v_9s=Dii=+V8+Y@2{NM6#f24jqdb2yHTDt$)o1HHI%UY}Rc799WU%l4x
z?)$LW`R`8#o%*M=KJI?yV&%^#W!}oYyuN>K^)AWu`|@G&f#0jY*M9wN`XO?1+SwEP
z+}C83?)&$xy7P_Zj^8nRmb^TceXY3sr`h~fZz{h^Eh=2UJN%<9)61^i>u0a$JaD#X
zQ&vr!{_($Z-__SY(VF$F;Fk3iw_jg%U;VrG(zL7eLqYlOw+m}@7hSKnKKz<n{eI=1
zV^#mZOTDRkKL7r;e;Yp?uYOzn-9B@h$=^F~zkcWP%k6(39{xCOk;|uPtK;6ix)d6p
z`n{|&b!x1y@`~s^ahIL$=l}Q_D`)@FD9iTcLv8=kv+MK!?pR&x`}~`6`-_ipZ?o-}
z2FI4FgnpZwC1-zc_oKjo?=gY*>*aSp`X1Le|MKbM%U|7omwIJii2wc2y>35EAMRLO
z8_{o@dP;tgs#?nbIPdS)f7hJ+DXhEqSXR16zV6;-$?xUwul(N_l_ekjtN({zZheP*
z?o0l!et&0Q{B5KBPoVg`?p68P<+b0xXJ?+jcSE<XWZM6G@4sYa?hpRgSpJLo+}_<w
zzxiLgR385B?xpj$jO*jP?KevQU3ky)^CSB@?_VDFrd9jD^X(1~`e}3Y>LvZWXP*y#
z+d1=BW!-0^{`h-7x0jsZH~oBe!CXUbuVw#=yq*OcE0_71U3>ZV-5baLcn^U;8Q%Bb
zIJ}?y;3N0(vxmR^$TNOveRhuQ*N@C4C;KO>&d&RJ$LHU{?GIDs_In#=7vGbgrD=ch
z%<8Gf-(NbD_tept=bXP8bL4Mplg-oHXGhwvo_Y7w<MK?m;FrJtiTrrj`!@aJ+4-|7
zYyTauHeSE)`?thpj=So9nDpOYD!KJaU5;OU`>|hlqhIA$AKb26yW6e)`?O>BC$0CU
z9kJH^{`thb`458j-3Hl~xorFV%V+!zy5`Sb`F+{vAFBC}r&s<tlXk29?gK~T`+X1p
zuC(Bvz5k~E{oW&=_EtP-KX1oZ-20;Z-p2oVvt3M_r2EhJu9?4Rcm1(9yKj~pObopL
z_<3N^e68;Hr4x5<lh1n~oc-s=Quc5BThnbGJwG7W|Nfz%{cDDJX-`grBMg1ZqIc)%
z)h07se_z1%bGG~Y?}~bNo?YVpf5)))51-3~xOc&y?{E8eSozP7$d2>obAO0QduRUS
z{aa{y?)4SkwLdwTf8BJs^!T~C36F2h0^NE|({Dd2c{82=?LE0~%Om4I3C>SHr+HY<
z|90Tx>3^KUe-j<gM&F%v<?f^QdAjbJk%wfymND<N?)41&rX4C;le${y_mwU6_cyP3
zciK2ObiRb>-CM6)xn?hTAo=)mp2z2)d8?}A7Hb^7H0_t~(Jz~-C3h@;uqrynG_N9*
zUw^&iR|%b8d%YK|^mR>s`trHO(-);ZjQ>CF`~B;Y&aJOK_0o5IAG6<Izx#XW<6YeI
zYv0^X>=(=4FMWUSUY-@%-=F@sXFkU_@8$1JJCzsz{&;-9N^^_mdEKdAKd<h+^Yy(L
z|Gz)8wppkz?|r@fe5BKk=9&#>p9>bBDO+YA)U3PN>*D%rHcvjQub1Dy<v=)Vyl746
zhtPvi#{62=3;g<*axV8@GO4nAwpo9ok^QH~_F2pRe>waKt_se0y;|YJ`Ov?QCqDaf
zz9;JExr(3Re#hG9^{<_L`tq&6*{e5w%Jy-L*u87wU*m<R=dfP+dHq_RO1z`skJ9xq
z|F(u^|Gc_6B%{1^?eFxw>Gi%>9&i3q@bhcM*NUR|SCi{<?<Umcp15^&Z`GOeF{|o%
zOG8tu`{(no-jMG3)kY_}dG`GEcFW&rZFkymJURK~pF5}i$hGmdRu^m&+g5+xc0$UH
z7i;SG$(e_1pH18E(KkQ3d`I<-tUonRWW2td|H~#fhxZ3Be{Apk{=a;3E7v}Kw{Vlt
zpSt~$_VShICF{1I&UzB0{_T8B+Mk*|_Y|wA922!)zI6wy!gGoBYlF|d-}H5h?c=mR
zXQEwiKi(?3&(>&t_=oc`>3`-#+uAPsD7rs>{RCSN&Ew~DO&-qg|H~z}1cZ5Ke=6R8
zuw-lW^gGQh=gOxA?QzeJ{rLIgsZbji%j4(wu(JHQ^Ql2(bHTCqF~xu8hzop^J0xWv
zf0=vj2c61k_0M{LeJ)=7JuYbfzq`*@H?8N|r}bmi10+U#GOO9bJNB3VexCUL{?mVv
z@8`eOE#6<>|NHr#`gPmSyxe<qyV~XG$t(AWS7e*+HJyELmha=e2cl2ET>J9RwW5C?
zHs$+1-@fc~^y!~}TR%r%{s|VYtAF|FT1jR0u08#qe|t~gzU*}L<)>?(e)@Om=~|FX
zZT8N6`%7xGckQcxynWj1-`>}^FMAz*`Rm%Jzy4i%yY}U`YbDj$JNND{sh%I7@4J2b
zwA;VEw{Kr|JNok9wNL;3yYzSM%YWBO>a%y&&$yNE`+fVe-_fW4{#|-J`tswoFF#)U
zR8jx(<Fzjp*;RYiulhb+F5kC*`?BZJr$7H)`aJsb=d~|?Ui(y2|MKUxFE#NO^L5XE
zpB|U*yMFt!>(QrQ|6N-D_xB@c%7&*|5M%!_E+<>NOUIu}$4l-r`V+5kczylSI;4C6
zQUuN!uh%SiRb^fK_uHqBkNfv?)r<bq`V;zN^@sW?hxY&5{1_?p!SUfA-DtwCeNSo|
z@6^3(V!KQ>IbYP+)g`j)+tw-k+9~XIhfYmcy~%sKoU!KI8`GyQO`lrbyJT{Y`V#IZ
zrxmN}@~nPnZavyAA{Kp7Hhk8S1v8iQy-ezzX57bKT(pMS_EGO7B^?#Z{CgMk<F;Gr
zPt%wZvf}fkxznelEsoJFtGck5U1XyCZa;CWSPxPCm?|fJE87`cc9}jc+m(@d@zSDy
zPZKBG-gwR0eQ0`4eL>{oxmRZv-CV8n?|IkZ29N*6?5iVxZMz*m`-|w+(=}87+I}`I
zn{-TR=AAdzl^Rz5!5#CCeqQ?grKy&ep6cZE;PY}T#UCA?@~Jt#<n24&dHiu74JGCX
zmb^1pHId_arL?{zME;V;QPHSrLRpjYQr_J*;M?l|Qs!Of=Ly=@HfKdczyJCzwQo^n
zOvvYd-s;o#U-+hZ{EUa+PY+4=)=cr0SAPHc;VS?7>-WGfD!QtcFO%O`*xvp>=hphH
z{V&TG`&>_Y&EmGoTXceAXm4iK-o=yrg33SL2wV1hlj<ZVrN;+*zT1UNH?3aM@-tzB
z9$R$O5znr5p6jQYWOc4hcwXb7944xNwa&D^E&bQi(s@s$>;)MgO)QAq`S-%huchVZ
z7tFol<J5W7c;!yJxXrJ&9^P{-@Bi9g(O=46te^KT!n%6>&Am76WJD#szwPijd5A^V
zC%kNf>}z*V)yy-u7HRIUUevN?YSY!LJA$q`NUjmI%{~<$s%n~A6#6>ly|qcm%2u`$
zr;>jaf3&!mQnYYMpU>Txb1OD^?>#ZIYt#CW_A^bPobRV-O#gDoe(Uppk9>cvZo0pa
z>E%gp$GEPKmpbY_#N)g^tSwp;yJENZ`P$5z`fnL;3%=j?p(urY#|_4HK8M^2u10L!
z9dVH}pu|Y@>!KGXI##^h)9Mc<Z?OHoOo3(Jcd?A)-i|dJj=njYw129}{iUDuV#_sm
zdhd*ldb_4L_gYZG^k=(5t}R`yCtDh*cFC*WLq4|ZMUcbvXFEgAEzR0zVv$v)X)^s!
z!@G++OeCfsYx>J)zhwHg3FlbMoc7#YwLvE+sEZ?gy4b~~rX>$2O89U8y`Jmw!fS!g
zLuP!)kX*i``gPjrwEt<z3C+{n8S?Mu+^oJlrT@ZSee1Zp_x!q(=8C>u`g=WBdFA0t
z6W83W`X9P+T~h4PU0tQ3uf18@@=g{u`_;2@+^G+}Xvga+F2<^OhjrPj*{l2yekyrX
z{O7dLzJe<!{SWDF2&r0mBWUB&B=4i&)*PMi_iMR$^$FGN`}HE59hWJ8x;1NyhneFx
zmn>b6wW^P@I(I}Vy_m+aYpHNfuJe4Y<NJ3zguPsIDYA7lSGwpqtvR6<{#+|c0$(mV
z<l5TJB`&I`6%+C!Fl5(?l7r&AxWc3Sn9r~OQ9MDpMK$<e5<l<%Ezy^Q-W*x_I4hI+
z0$Y=Cg6hmA6FQtG$fw5&RT^4Ks)S57QK}Zz@Dw={GJU09xKNh&o6yd^S{uVvziDj^
z%vgNEZDFsIbgS7uRkQ!bH~#O}`e7QnVf&?zZw<dsep>#+&-k$Yy?^(v{0m<56<q3|
z)B=NBM1L1kOu{?+Pj#C2r+=!^+&{f&p62zL{FUd9<g2BAIM{jKCnoKY;pT_F!ux0M
zxO?RFGOowP!qaD4+&Oap8FT&Y9k<VbqGNSI*uzDKSnKC54R~OGd*Oe00PGY!@%44g
z@BFG$+TUKEU)(=eC-zMTqxH&DyJQpjk222mw3|Ft!|b5)^QxE`>VGRuH`wl9_V)aq
z?SIRk{w@E#f8OnhyH|xA4z|DQ7P{M)_v!U#e^+hRex982b@j{jyO&1a=Kt|-npHr7
z`Rcki;WKyKy61WAD&KsAxOA7>_pX$`f3mYmuQI+)wZGDpg-!Ifp;sBp6xnl$`yB)K
zObTa@uiyUr*;eMsDXv*J968E@c2<4$^R%4k`{BzQ2KCii4^F*qjbp!Aw|3*)pyda(
z4Sz`q?k&73VLMBX|8CXad$k%*RBwM<x9*6!=GMniZ@lZTJk|EnajRdMuurOUlg+l}
zdA3guzYhL>to*x9KtR`9o4D}c$%(&9@84^T-+P1g+jfcH=bt=YQoy;!<?<8Lmg)_&
z?Jiz;fACD?FOJgxcQ5LEvF%p7@NS8o>CR8tGRu@E@3DK7edhc9HT!?93BH&n=b0S#
z<?F2{(rT;=R@Tgq`C;dKa;yEBZ;RKhUot_bxPIxf3eENB4?NiW@7hAqjmg<>LYx;!
z&y~=+J#|mg$|tT-A1~UQP7)3OYn^g3+xD#lyY*eaeWHm^w3cXE$J*peeVjJ+#X0Tc
zi_?5Jhs#gZQB~HQ8Kb-IlEQ1g&l{Kj&pw`T_ho*P+KctqqA!a``Z(&J@?=!wzg$)x
z?5U`Dz_;jXbLUje6D8-j+%TQK>)C}q(?u<-ZB@VN1$Q=H(t9S(f3w8ObMoO#-_CO;
z7jsT_+d9-r&GK{RpSJ8y&^=r6WilJGyKm|JH{BdlrnT~^&-S?mn=8Dk+RP@s5}LQb
zPjAwx%{~WD$%R(FxF>ci-=@0ut8DBm*FSy}cU5n`rR`Jq{cECir`gx*HMiq;X2@6f
zo}KsD)%sF$`-I3ZDe7J8HvNC2xnc5}v!0Xd{^v+JuWeZ9;;8;@fv>Li+E+{cH>&4-
z=elwAy}W4uuc;g^)kW`GEM&K8MxOq>$MFB<1h?{M=AP->vTe`G-uH-f`^<Ch%!4Nt
z`j*0Lo%Y_yJUYKFD2OvDu-0^{jYvV_n;j}%6(?><6y4e1Zr1Cy%GAV^x$M`4RX*O0
znNz~%eA~ZGQum2#L(korvXTznRmM6|3D$X+E;#V<u07}c?fJ3L=O<q0yxF_4N9y>)
zxW4MQT6stQ&Hb|bMgH~s8~@u)GD_t;cSdvL^XcpMT#4M4H!aAkt*l07KfkU+*i*hK
z3yd3t7pk3WxX2m4By{Cl`<JZ3zw6R2ip}S4$(VVId8^Re4Q}6UZXe%#sL%JP_Uurb
zhx;zb|6Tjb^2_!&`)4g|JyYeuc~|V@rL8|TJgqa8_`e;xE9_Jl{_Rufg2jK!U+9~(
z@J(CedVPVcl=g;<!gm3@yS!pdh1PMM|J3wMVv(7GaaVYrx~58VMy8VArbCgoWwF(f
z>AJ7a3C&Z95p?lcU6#L9CN1icC%1X#!)1pK-K~%JE?g;H?f5#S+Qgl)c*)!~!CIN@
zDzbJLjRFs8Ev%R}?b~8!ZIkuY@3w6>uyqc!uy|i{ZsRuj-!}Wcs7}#Q?aocusOi&m
z!{@$?9*3sUjw?~KT3GJrpXp0X(n(!Zk#aQhB+IUw)(`IODc^P9PMFDi<-S|ms(Tjv
zbNg^`uXal7pGWT$bSLUv+WRx5@6g}B-#s^6ZfDE8e<OLS=(kk=$!`|Kd7iYJ<)QuZ
zU45a;eBX8N{}xVenJc+v$_KU<;moc*OMF>g6=Y<8sE>LXTy8dT@3;2b`**It`#7-l
zkN;bfVE$eI%<ksj%HOqW#;PQCp4+9b-v?QDM@`+z7c9KnfK_egmU`YzL9O4WC~}oJ
zD5(7Cm))Bl@0aW3`rB9VX{2jj*jsZ^sl*r8lG|A?Eq$-Wy?o0IMyCxkDpuTLFKe-W
zYa9DJ;G&z+jns<Rd0E_9w<eoCHWHOCd~aD5@HHXBIV&af;W8VmE22vIS-;M{GMGGf
zSLyOgs@t|5dbXThNylo6OMAx43v#i#Sv$TxyH;wrWg+ifsi^GVXGO|;BR~AQ{pGu&
zm52KKIOg5=y?c)NS-svo@yns9LPuW)hn4N)6OGc0k8zs)_)^^WS<<sZ;%~kBk-uK@
zZBw`Q)76iTSk)h$ZM*Bo#&vHEKh_w#c`f+1T5}=WR(|m?#d@wM{ujk|J=R)RyXZ}V
zp8>=48;whCUrc;(OX$o&wc7Hc&~-N|^e1^uyL6-B^^-}p6|-JnwmVaHM`6jWqhe?E
z=FHhV;cHY(p>ehOdzC5vy-ibhpWJ$h&E-y-_|$;R+>of7-)~-*=JEcV7e6^T$w^OR
z2cx|4o|orerwM%ZEMZmLp}@lKen8>*E4>r3xj$vh1=^f5(=W$QXjoZjm$8128Y6G%
z;u&>1=Pt~$RI&fgI5}&tKU-*ketM5sP3+<Y+FB=B=jxY-PgukrxcU6NS4>y0AKX0k
z^||RKOTv$@>y_B%k{oTats`@Xd$xIPP=xK;Eq`xnAIVE)zMa2+a$eOpqq|(I7bI)w
zCQWx-k+%FKi?;H#(sh~_*7i*<`)MI<_jlI$ozv!;EuUIqF!Q@hh`N^g^mDFr6HNEs
zmFSY2CE{7+|2Z}~v|{gEpLJK#_Hu-;`c!?U%xYS&QeNJggypZU$W5=cdso6A8~SA1
zOX<U!l4fjM+0%-(sw1qWy@Tvmz4$m+i{;Hf!#USaXXbNQ-);W-YkT}o!JE12ktdU{
zUl!S2ma$z<=N{u(0|_rx0b}LHR}CGiD?Zzb)V*Xhw_gy#w%R$;Vt?psraR4cp`AWX
zolMsYcFszA_wdoA_NlD#bJc$znQs-l{LqwF8T%Z1CM{H&@r@_R<m`p)%?9t+CI4nL
zxN^ie<ligC9f8*`pAG5Uc%kzDj3lXL7TN#RuUW=EQ#%s2I`i4Ci1#e&e8(qVuKl;U
zOD*1g4?Ac2`(*!To*HSXt=<n;_|17(-JveCH7|@yZTf<mrD1+m6<HfY9vEreJ9E(D
z)|S3+ER$B2b@r}uIde)X=7q-fM>nR=+;JsgpVjR4z9$QX=PK8PoqKb1rfsQLcFr_|
zHFJ5GEaMmOWfv`dqG*!Wc=2shr}ioS>G3v;JC~}-aj$#+=*sgs`7-}_53EWR`mrdC
z;mfR3sftz3?Dg$?t~kzbm}~B`W;NGimuY(^Okr5!G>xe)<)eS(7soJzd+uVk6MpQT
z=(um@Bp0U3q2l4<M_0WQll|pt!u+Ly@7uD=*&CPYS?~*Ts%h`IbK1G3rP)dQ=JDM}
z=C9xX`_H@es*9UTzVCW}Hq}$_)r*y}z2&C&zF*wH{X~$p<-2a<Z*P-(LRU6DNdETN
z`@FnTVMB|!Mu$+|IX7Q%&Lc+)962VO$=s&iA@pJbpRBPa<KvynMOYX5CHT6A_#S-|
zlDlmF3uT*}ww<*p8VhWlryl1m->>*#P2p`z#q3+DU$`p5JhH{Kzj<*8o_x>p_i>Kz
zN#};Qb?aJ0XDDO_8F?~yE^&URB(q9mzR1t0@_)}vxPI)KX_mF_f~ft1B(C<&Toz~7
zUErA0(a`O&`i}fMh2O_SZ3`}ZNYk^mHoZPi@zkW1>v=9tRo0zgae%Q#DR%a@Q_F5$
z;&^qIXJ5mn>|c|8K5Xf(T=F_(&a7p#u8K+sd$=)Q>Y8>psq6PUPl41nZn?~3+9u)=
z?}{3F>y|$H{j<pKyF&Wp+R~XqC;d8qXbL1Q=WSW-v^`Xy)4h2Pcfh|ICg(Zbzqx+w
zy=*dPR|Q|!mg0Z!U6*e88D-SK=JeZZH{&_mg=T*|*Ru&L^Cqvl{NyRigTPm>a}I1a
z^p=-BRH!TvS+r>Go7(0pKc1`YE4JIe^^w_!Nw;eQo+ZvW^5eC(so_rkeK}JiGZXX=
zUr9IrWi+R3vEJn;uR69!wnn<eOBtM*7dx9nHejig?9#&fFMFOl?tK@zoJY@~A^%En
zto2;Bys8ihhI2-nj4Km9uKv~<KI@>C-UHc%#}{QQ2Fp+4&8_;{WZ>PDq<ywCL3W=R
z<L+skAweHE&)Poo=j_CKEA>e80+xrj{_<o^3$OosVN$->;UyN5!byv53!0B+TwJH&
z^<lBCg*9W^t<xC}2^x}jRGxP=NiNv^dBcRO<|}h|NOi;?Smw@j?G}5x+%B6V?{(U2
zOseJ{3CUsqB&K@HHJ>kVQS*HdOV_Dvd$x4cRw?qwFN<J2ve1vcWrdy#gV+;}-$FMs
z*8iGYqVSqq<R#PRd27TLEOl@C6Smo)_0(a}9aEy(3qwrXF0vnax8NG%S!++Lr;b~`
zvCryoe}3(xmcTStp)JK%AKpJWansA71<P6L{xwXP6EOd@6-V%lN6UB^&l!JOlamws
z>Gx94Wgl*@e>$o0JZt4g&r;UdoyImQ*3JDlch|7{yq_>ZcGo87@Ul%E4l>39UbXj^
z>hJD!esWx?z(`=?g@e8?L<8GeE9W;(vPwy6pVguya+O{6`;8^X)Sh~&JDy=%J3~QC
zc*Deo)ogz*vzY2@Zp_`~HziI`lieb~RfO-Iif!GSxy-Y2?StCvnB2l2t@c|a)6eHP
zUGc>xj?aJ9nUzx)2kr{>S!Wtp=KNhNMC!*SJM*ayOBG$FKd4ojdH>+S_<E}o{_GWx
z?zD%xG`(xMJ3*zN>uspQX)c!Nr-_Yno7e39bY|i{`I|F~nm3(VwqyZ!*U`7G(j7s^
z*{+LeHD&HtaCC-;N7qZ{WRB_IpLAz@4)JN_&sdNov`pf;)~&u3H%uNDxN6S(zCvMF
z)!vDDhZ(LK@R?fh>N#t9>1LWvSn3@0q?sc`@E~{OTc;1ZB=VOCU3ZG8(v0}yXEY`E
zaoX}(eqk49cA2bi>X~*WJcQBJ<*Qb~9-(!!UOs)LR%_Yw*(W&EP5(mLiD{haeXm%q
zv+pqDSyy@~tLo*S&0#J^jZaU!S@g;!AvHLOjl)I!-NZSES+0Gl<<QaC_KYuY9UsTK
zZOohBw9MABp4Xhd=W#)}dEvFc3H(_}(_M9V=diCdNEDtDwXXD|OoR88ImR5`ce2G3
zw_ZHprKhp`h;zo`zgrv`o2(R)?q`>kC&j;z=QKE-?C^#6!8FF3<~%y*o^6@!<YKeR
zbC&Qe_X(YcRUR9iyfH`Y{`M1o`)B>L(=$u_>sobw+ntjsGjGg$WF5QZlZTqV?v3VU
zCJmZK|0nc(n0G2QO^!`v_R*8ilsEnC;CX%W@0${<*09=+P4{M<oX~O8X#b+GcfOcM
z%dD`P%D(Dj;NKhDFaCKj&$TV;i#6|@P=EPfJykiCkDqF7THkv;PpVt2l8;Gs#eRp6
zlf{-xHWV6&>Myt4aCgm@MjqyCD-I~+ws!7KE?*F*dpzq}>T=-~;Q_)X1*&&vYFFtx
zY;P;@{`&RPSA(-Lv*dm3Ii#edBp*qw>J$;ESbx#is6R_+R<=W~pdpi&Vi&v1Ckgij
zmwP(j$={r_=hv3YS?A;=_A)Pjo&3IpKkqc#ukUZZ@6Sw-@UmXbo%;W7k-go`WwqTb
z&s;C<?GY*GeyW$5{6>G*|Cm#YZ&x3_6{?uNTtZ-jQR?5df7r^W2_65J_T(vN{lW{}
zYhQVuHEFuAeU-}LcB9N!eh2z@J3Qr?r6ZGd-}R%0a_+1}ho-aa48HPH=p*mNzYaIG
z7IJ9$8h#MTZu4E%a3a=I;+y=#3%pa;aUG2eygkXtleur|x(E9-JW`hK3cZ-H_WaSs
zl}|Q{YKboVcyE4MbHS#0-ql@gZE0}_ww?T9a^<Vgx!)69#Eyn8xTPs|<HcKni7{Wx
zyX5qgx;M+IJe{m`TS#!_%TK?er>@kvto*WZQ<sa2uWGkiBu7-)p#b$M>7|!m2{ByX
zmGL5ht7-0{&C5GpRW5XiUa&S=(Q!L#X>{hgjfbW#sO>h2F6Dcv6MpgTwC8*o9}kAQ
z&)dCVzP3oYX`$|n(ylpi+)NP`FGVD6W_xDXX9-+8HBn*z_ne@n622*+PS5*d0wV=W
z%uW<fKKA0GV~Bif(1j_gPdi$T?%(?JZIV=LsYc5q{g<&i+IrpJqLl7Vn|9@&H)E*D
z3Z@UA4Rw-o{lb?tERuDqRB(_CIDa?gd&@n!Cok(bboI27mOH+mrPG!nIyGkD4~O0B
zx@07D@2_@}4zmhyR`|5y#jHk&uve?UEND}TQa{ONt$x+cDzrq;`fB*BcOlbbRu`Lb
zy>i*rcJT`1+wWf7E^8(_obKiBSbFqAf6bATcT04(=&?UCT9Vxod4&In$prl@gDJ)@
z*Y2A3d(-EM^{$5D)q!h|-qeUx5e|=G=X?A5RPl?OI>(|}HuD{u!fTo)JXzB2=~OcT
zk0nR0K1nn+jJ$bqhvD2^4INjHt(@+%)HC$*lj3!1)0Lyoy=U6V`gFHKi}<Zs!bw&?
zKQUHXAK2*<;J<udAJgUU%U`}0W%<q{B62FiCc;Q!&a%!L;nK`S-n#+}?mkZli}F<x
zR|tAJC3cqSdKZ@$U(f7oDyr+)arN8o$*;8K#8{T|M&17M*<|M)g~e;9BrU4?SgCV+
zx=6t8-5j|W=W}Mbns;ry6;t!J=jip7vo9F8Meo{po2B*M!vAYdg-@_tRdBgTl-ZT3
zc3t5uvnqk-Gk0?;*-ke7ed?EZ$mbImeoEfj7wMOJx5di$TC~qbi?53N<(HjnUZrlh
zlJ8-@z)tb!JilVTWN%}V-?TgDt6j$TpKs13U)mRRbXoD{g`fT^UH^RT%Vq~f|La>F
zq&`*6+TyQpM6_y;Q(L%P*y5ALclZxVyUsZ*$}=f#;m4!Cn??OT1+v^=m9CTua564-
zyml^BHCr`ZK3_L%+Z?8ME4E1Wyr{c3{r&IC_S_1OKcQ!qi<_*>Kc;BnwrjbQkNcP1
zr@ut_I<j55(X!X$9`jD4`3H~9*v!qeC-bQPTrbgsESX0BLMz{MoA|#->R`W_tMT<~
zs#<ZxT9++sIjv=f&$ch{QgS@;%uef#*T=hyRzFj^7Id>Yo_W#@{-PqUyTV7!x85po
z37c|dN?JlLr^n7h%fklhljkjzy5?f5VaPq%;cxiHI*~W$<|ux;cGolQ>vGB2H`iYZ
z<fvMqzh3@P^&0uhC3e~WojKXIsT_27(6nn)D-bs}Qn+xWfk*z^nf!=p3->$CuJt<L
z=eKW;X@tJuh3Oo-PIm6}c3i3!JtsukqVr&YpWU}7C3EJ?+LYp@@F!wQ!skZIE{h&L
zrR9e^*RVag@8h#^d3=HVbdD!_Z=b2|U9IZ2@UR}w-Bn6_>D%+(<@{UE_bQ;*w|{B)
zEuZr1S&NG7#TH-ITvfDUzq+i%Bb_61xr%dIHU%+H{W~Yr%ZT*|*IvEv$~!LcJkZ}g
z_an<L_rxWgx`GGR^lv|qG4;#blGGQ!_Icmo{-5iUDZWzX?+4j1iO6P_Ybgg?7C31!
z<ov(Fz;S8%vXv2>|HEslSFc_CRkcQbO8j~5uE)EQjrVh#xnw1|9_31FxbZjju9GQC
zsh~?Tzt0(=Yq^hlC!EuK_#iIIU!#7(7e#MI?cL@Uf)nnr&Rf8u>7?v^+Q4RRyR4zp
zYxkE+l*L}25VMN3(vp-F$csP3c`|Lq;uRMq->*BmPGzBQ(+h_v8QHl#dMo#O6&UZn
zbjnyopdgNaOT<Mh$8_h|s0{@ZrfpQ!Jz~z?P~vE<AMlkmW0P=&+RSXdxwhO|QLNrd
z+1%DDPiH<kxzK67j^2C?i;9R!j?JoT=c)cmk9@}xzwv-xx#Q+YVLjuV%!S9**Idtv
zyRfT0@6=(nmZbJcqOOy+Mo-=<JgJqVZQ7Jc{}sKyHi`92opy7-zW22CHUHLTh4KA!
zNk4k;sng0MK9?K(FEv*DoU?xwE8nyy-Tjl!94>mXI_l#wp`^R*z0O5}y4H===E`An
z7w%9Ezp?1hj<WM-I2R>FD`p-!(CC)7dTYw!$qSbyEo8r?zU|SZ8%i@@O4Uu6JyWgQ
zeAD~wPo^wh(&;<dV8$E8BOQ;PeRK<%e0SY@e{TCtGaZhopZ58+$-}+O;}LuAl<uC)
zpQh9K@0@<7c74+nZkN6*`gw1Q>=?4!xtg+L13&Dlo|W=m`K*lUO)0$$$@DV~L8j~s
zr*!)+=-fA0oW7GY+#^t~!-VC{YuPCxf8QGP@!A;{3jf=<xT^i_eTCMm+G%g}x+C<A
zC#A6jn=xm6efdWElC{^grEjyRHw&7~oWAU{_un_4cr83$CMt{G<}~Tp)%WA{wx&x*
z_dWP$b#lfrHXVzdug;#%-?rvwovHP-oHHuVxL!YsNtkMI<c?mDnz?j%N@YlKf!AKE
z@>N`pnMI4Ed*AP6)t>O!NaBP*q4mS;@EhCM^snz+bxx91GjQUy67QNbmsO(^o@xiB
zn0IuVU1eNt`e6T>`@v^D4{j2zFqMwzoO`%W^C7!i<krxLv&$Kun}6%Sz_==H^_8tj
zJ@;&OUV4|k+5hCT`bp13>lH!+VjQ=w*=F%pZ$``oo$31=H@3Kn&bUA6QebOJR)g&C
zBE_1COLkQl>)xAsH1*~g=iPxH*Tw9aD=y*_wDDr?^Wcu3kM1{%zp7jKPJG)Ox0z?w
zhAO9vS}d4#^VEkp)_JQB6o=-8-ET?X{Bmxjm*TY0FY-Azlz7W>S*p_nFLl&B&NOax
zl2?7`dBF9m(ur9StFAUpn|mmBX;R+l9_i_=?MLT`%S0B5<%vaDFO<3T>3N>%dN(7T
zoT9t?qT6ogY%yzZ^?mkmRpj=hYr4NHJ^k7%_teH+dv<<<N$4csIfvs^Bm*8<S--vZ
z&g-)IwOoPCm#41%8ah{ObB)J;hKZX3CSSL5EZUmzL8XhkdCKN<Yp-p;d~&Nzc*H&R
z8%%2JCn~s3lG;~!!Miu-o$sxX>z5f!%uaG%emv#Ok;ExAj@J_===64V9?4y`yPEUC
zK9{Pf11{E$yqjHn!nRy_u;ccp%fA|gv#Qn8-fLNU7au&T+AS%kckqs<%he+~i((g4
z=Lz`vW+m}1)s=XDa{D#kS2<sHCtS99qQ)%eTmRO=`P;rwp(RZ3m));^=l3SLoN4*~
zH<=7?B*XL~SA|#?Emk|{XJWnChwZaPi02H$_^GRpTy61pE{uOPzl&{J#0pKf)t$W3
z8{*a;SnbKUx;y>Vqs6<nez~~Y$aMFEEz@Gx>)4vumn_V_>d9tq-B#sMo5HbVEq_|-
zqWJj_6H=;Ujx0<+VlAB}>#@eFD&u#Wt4&F$+_DQdr8u^3bKbI5ciuIJ<KA-IX7VY*
zxjVR2tZuG3b!tWY%bU(sza|u(Qf6~u-nYe5$3#h7WX-%#u0#Bu6W9(av93GiD>P#@
z=k*CEm6pqJ&u%{*t-X!uWbDoxlRV~T7yR|OHq}3DWBZZjCZ*oHQZ6jF_x^ENZ5y5c
z_0aq?YtN=$)z^HknOVT7(I371IhV|<O=4fC*<?DOn5OHP6E1vm>fNuKTAClNyS`~j
zP93|^{##2~)ldAh^jtNGJuCd`k*d~)`)6ls3I&_Gh~~XM&cb|oN4L;{RKYe2$s6I)
z+FkK8{BK)yvA8|iaC4$(vXHleKL_t7l?+#YV@}m>$(d)CiW#qn-|_yimS*hp_@y2*
z(;9_RrM~ig?^r(7%-Y=e(As%-mswA{Gu`KC^P<x;mrL(E9@rYb`?Sanl@<5HgFH=m
zSflRfE@evGS(~wdug%KF<=Myem*<HknnsAfUTO0s%;?a~&X~>;iGIIMJ+ET*J2_8k
zSXMg)#hfh_m$|dWBuMl53+3p&Jax%8HWf!ynd@AW7F*8y*>l=e4_6=aM_c9x<e%L2
z`Q$gg7rx3@q+}SceLW;3CiC@0!^y)Fr9v)!xqhJPbXs`Sr{2?34i|NWWd3;5YA(4!
z>$1%0mg5^0?>#7Gi|O?hU(WfGO}P2(f$Wb_f!76$V?rNG<bL4tnL9i4Dx3baR$GqW
zi;`D!&+p1`I=-pmV9ve3`HwqIICl1L;F&CZb6H*Yxn%8`YcFg*(v&kVRl)Mr2UF&2
zsZtX>r%m<=l)LHK*j^#>)k}#*@Olhe`o=Sj5xcf`x!Q?wEHp{mogXd7WmvHES+vn~
zwxUg~-u*e}JZ*|S7Ja(BdetuVY2JlX&t#`=cHr()iQexsdDjfTYo%qo=J4~eua>y0
z&}?<@Z<B20dYSHDbp^@2Z=YR!;3ZbHl(RU=oKdzh==rlJ&zcGy4f*P}JXn&g{$jW3
zCO@s%!$liT$8?FD(K9RCXqZ;K>~d($+orOdhO{|6MoP`{H(H%$DHhI7yLyH%^+<$)
z_{plHS0xtxSoHGsiHQ3@mZlV^D+`)Fe!{@UWjUkc@AXNmHn=~_)H#~9FEefBOxvo*
z#oM^1uKRQ5n6_i2LTYxz+pGUf#FW_9+BatOAGPG%^ZG*k&)KV1yjF<Z+>-16@71y?
zvs#1VERN1Sbk3ZaWtK(S)J>Jzo83;XnbfxC=Nh5@)YCIXoJ>#Onsl}G64$)!nb(V?
z)Slc7&|1T8Wd52@YI%dNvC{G@yIQVHT-!9mQRzFo)HVGr298U5E~Pp=*d%E+`(ul%
zbeNdShHVpeDlQUUU-WSCQzQ1`MA0*=eRp(lI+@)JQeX4L?^LQBuY1$Z$tzs;Ui~4J
zRm1x^xgtgLhGN9&u(=LDRE^Vqq^nLAy5lps$xdv-e$S@@a$!#sH+=|JF}yBz?Jcjv
ziM{`YuWxlRIQz@w=(ek7zb|`y)-zf)SI>9v!zulaw#OHrI`rnLQ}=tPmT42ZRlocb
zdaF|NtmF2>)jAnUndcW+ng0spXZ#d-gT<3cl&f&V4DNX)3y-Y3x%2-!^$i-#zGi`X
z+>Q6=?M_PBa+vqus)Fa+i^@VTuE;p}<AnD7A~VzABY#Dt<+^{q_@HoTfqso#T)>>)
ziH}y<P4qjw&PT|rLt+1$JFgBsz0&G6Gi0_tv#ijD8SRpa1)gcPIZaJ+)4z)9t$(8Z
zyTN3kaK~j;4VP!iSMM~GZVMAwyYEec@YVA6py`zg*D@AeZ9cx?w~gqtqFHYRB4h#*
zH?ypg<rMXA|H)mb-W=?z`Q5Nz(rw8%=E=`*L`MYL{Ji2hJ#^)bD^s_9+>@$!a`zp-
z-7j^D*-pl|a@8dT6mB~=`|q5Xwh4QD)mCp<-7tUl+=d3z!vPmobN8JQc5JDul+(_!
z*{yb4YI4zEiM!X289$IZ_H=iQpOI-rE9Yy4yX$f*yBnO<WO((KJDM%$i(=|&4p#ig
zTK4MN&Nm<U%6{#Lxp1f0s9qtc#$&I_!+ZYQ=Wo~57ibAmZn}9cX;bWp2UFe{@CyIZ
zc6{0*eaT>we*?p`173X-m1mn)>aoaZ|2C4_eZwSKFiBu$ddwP|hm)V_tkG`I>ig;R
zvM=<0ewt^NO^1nnez%aYo#jH;sc-M-F3g&Cu-dt&$t;^K;y}K#4fmd|3BTW5KUJi`
zoV-aV(DQ>fhd=k1XYbw_1RFVD65?swdcR`&vvp=2lg})&{d4kzk-52s_q3Fkt9cf%
z&)5<4?)HQ#zhb4d9PT-s?>IggC2zK=`j?V+-t+F(>^T=j78mi9=>=T~ILYhtD$2EJ
zx$L^dMPDthJx>>YS|t*H>7wgOzu4aPe%Cnf_cy;8&CWSF;qimU71PvzKNMOY;9SRK
zvwFXCz5ZX(zdL7krscVmAB{Y@$%CCSRj}c}x{}T6cXPbWH(&0x7ro;@$?TFgmkX<i
zgZ;TPwUT0@V%uh>@J3C!yD2!n{XZjjY*5A-U5Pm<X>8TSpO-DP$T3fLE1L9m&c)f>
zPpwv8{hG0DUQ+wx4_<4h`{q2-xxlNR&bTN*CuX*IR9fsAorOQd{v>XVK9$wC_12_i
zw-&d*o1J<xEtKoK{K<JoE^coA)qcKV?^-v$C#GCqcQg7oF$x52>e^Oi+gE4BeqZJF
zlPneSmXCp@H#|a3ZfGxvcV&$I_N%~xqwqk4%hQtOV$3szvo_cWw=FyPwB&}O=E3);
zd7_Wpb>x)E_-as-|3tFDyWg=^_TKWJUG2yIY*i1h;I5dHdGFZ0br0nAX4^O(ofOp<
z^00EtwvxSPZr{khc5h?Pl8JtoE*Pj3@>{4pnH1d4s^eE2b(VYUqb<AoQm>_J)T(&T
zHn}~^*J&%0eucv8**dDNj}_WKiaZN@B&>JvjnUCVJzT6ud!}A{7O(!Lm+O|s>sM18
zzBSGbW<AFx8*k`R`f#gM`!z{M!FTfP_vF8d8wOl!*8BaJm1*JrZ&I0h8Sx@-R(_fC
zN0RqYz^}u9uRP#s?AOfDR!k8(@=!Nv(_)_on>w2xYTkT5<>2Ef3UyA}Cs?Q2dOmHN
zdC8TDeaeoR8*-bnPTdsq=8WGNsqVsNeM)9#(Z%zR;@?PecYm7tKwJFQd#k7=!W|oh
z=WQ0wQ=V?2>LSOloqp)+Lu;*ro)2HydrnK~4BBLLZ_2%U0@YWJzU5hPRE002(?06Y
zHRj60E$u01rnkI2{re}YP`^F9?o+>C9?#0(>^GWuapp-KzAO2s_r;t`ea8`}5IIvT
z(reP2kH@(9tN)hCTbz{%e4Q3t<zbUxI?1v>WgqLZMUior=4os;oqco5hm)7zur9dS
zw(q9K{bSRs;%ai;R=D%8QP<@CeD>*qJv>|X9Xa)Illju+2eZF)mRJ<#vMrmmT6wCL
zO4YP^Y7cgP*SIP=(^yL^FrX<)I=t76`{`51tmlp1;tcsyyJ|hZ-h3ohzDrJ%^X{2x
zUsb%#KfD$$D66!7diFs`h3L=so>#AbRR}3lR*^fkS!3ZRjhm8IVLRq?aWB!`+;Drt
z+G>f5BAeCE8yL-(OO5uHZ=dpc8UN<WM>W^l_4;otoN2XnlZeRI)I;AN9Zqj9c4F(k
zyyQ#bTFuu!4;3_a??~L3bI0equI~!R7t>R@?k}1m_J}P$T-g2Ul>M%vQ9D8={9Q0p
z)WCbgnTtNFqkHpQF0S0$_1MxQ&3MVpO`Rf>dJ`3o7Pwv({UReeW9E#BVd4qbo(MGQ
z)b|Nq;yWU3vF}z<-<>&1?;{FCU(MgHm7RT~O)l(URpj=$$4l5gi?lM$P><YN<Zb`u
zN#gI)&7x(STiAlx_6EF?Or3mI<PB3mTH1uJ?dy{M|9QlDR{EF32}MQ2^BT74Tsq5B
z-qa-RFK=EtK~C7|@L3BJiH<X|*{vo*4?pu-g<ZVj^6h%ozJ;?6POw-WedUs3b7#e}
zKMOmq{+8yod%SfyZ@!a^!OVL{H`&L|Ub{g?Ir7HAb8$X#qQOgdl{LQO(i8r2)rjN&
zWP1mV`eKHY{H)K^_f6oMxgcNd!@`rUw?ccopRS*}WOk3aE?=YUy#)(*vHlO?OP_OI
zg}r{I1kZ^#Y@2pzZS1@H_)SKmp$a#9PQ+xLdIx!4HtB`UFJGkWl4APldwo+BQ(~7a
ztJ&*>u<&(UyV#dRhg{TM^y1#cUFV}N{#uqcd*Y_Ftuia}vU59wrY#D2QDw&6CM8%?
z;4iR=>GH|5nl9$)K8sDSO`9EewB<`f!!@~U2KDzdpP!hnzA<grp4~U|@;SDt-Z0+g
zn8APIU57>M9Zx^DH-!gEw!|=<xE6CSb?W81bnbgQ3IykW5ZC8FGWn&AV3wa*o94-@
zeEf=&c6P35Q@v?-e7VFj&F8a%%j_#Y&V6)5P5sG~HT6o5?^iyzX1kno^8D2gqAsRE
zjCZd3->HpEQONh%voFDUQ7EJ8V^`U(Z5A(8G!m!Yew-y{x?SPHw2d!SuheqNMP3Sw
zl9-`u5+-)(M(fv^HFMLRUwpLbhC*VW(cFZ#Niu5|GW<{v+`jt8q0Eh&^%aa{f6V;T
zJbP}^+|yoiCtlW`dEK?-<C@7=JP$7Pwsza!w8GGL&RjE%ERlb``#LoEb{;ageJ8fj
zBv0@1E;WZUVUz7MJ|&*1bny;cc=E%u=VC<(AM$>PEnUg*NbgePJ7Li@;gfOm!;eMm
zT`Y99Gib^7XYZD(n5$1XxoOjj^9S<ur>|MIsr03gszynKdD(}?7H&J0Hyc%bA2_xj
zaD942J!{jnW${Xu8@xAtY@Zvfy6opgBhHo1PG9D%$vjkh`SJ7Lvo1_;P>ft^|H|X|
z>gIz#+Ga33K9>@auw0g@if!(lDYt$HTQu+Lxs&XWca|wLqh$Hzl~=16ZfI{QRII$w
zJGq%h^u_ZJUsJaGobY_(ui?vonCmoW-}T>XHYHf!lu(rpx&4wQy}f~F{*<4a{+((~
zag+3LjcPvdtml}0Lf$#El=J5qPPi9G&E3cRkoAyH$cNNT6*Kd;S%wx>n60>zRLxNR
z`efM4({?T?ig63VHs|se&6s+-#-hD;XYPf~xhFrqR@t~-{zj9mZ|dxmt5ogRA6(jV
zcxl5y{>g_~i*`ul<R3dT`(>>5_YY~^@3juyn3!-xS0tb)<ObKm<1DknW4rz|@oBM6
zZEJdeuc;||;#wtD#X|1NBTL0Hc~_(`9!xv8ExkfCx-map_|v(vnH8-EYJEIzJHE@j
zz#?Z>lDEK@aYN|u2JR`z8t<gE%(|v9m-20>VzoSND%EuK-)Wb1yB?$#U6ZmtdCRgQ
zcB{X>mch?gpY0<rtKasiaX8ARRXZ{3T<46f3oEwq`Ijw_=Plj4rZ2zxMO9?%|4mQ7
zrpAl2UpEs8-pl!Ri!(3x)pIY_r}Xd{RWx6*m=aQU=uwMM;(nw1USd})vla6fs@$45
zH6nNRp=YP!wjQb!TXAl7%lB1&)89`Hi(sCuEn>FNJ>#kf_l}&k6TI%QpJRLb{#wba
z15R6y7nJrFZ}$q>zA9w<t0_m-s?OEcw0w2=Ykm8|yR#14*?)<zDu40*k^Xj$AVsd3
zzfbb*UgI0gt8w1k=4M7oV&D7ErA%7k$)`V9-Bk1~_!Ro)%jtW{(FZroVP{)<xuEo@
z$=8xoVvAT3KlicS-urRMQ<IMpv;6CMqgEVYsn59Fws_Myj-w)Hy_hfUxPQobdY$#Z
z=&~nn93}5gr6_j?&2r(IZN$0VcUjG;TjeLOdK~!cxYjkk#CNN~ytjoy-g_g;UBmTP
zWH+gvdaJ8%a?@oB&xx!@0t`IX#~j$?%C0CHPRQ85B#~>A)6%jJ8dH8O$+0)!m59hs
zR$IZc^k0`p&diXQfU27oIvLB)N3OmxSwG`^YEVIBzG_E!@TV7xqlKNmuW!ELXr(vd
zmdvVO-%K31XSNnee%oPjMqkBqHbaiV7vmF7muv3i#+0s`CY)!-cI8n)?S$#4%|FLx
zzYsq0*gWs9?{D7bg?Ep{PsqG`e(lYcOYWPN&rk9Ev(GU0fSgQAr-mtOhP=QJ0i`^H
zst4B?PA1;Tsueup&Ro9heyzNfVC~B!9;=zhEWMQ6K7E_M(awF7n8lRk;&XK(`Ok7p
zWW1G;Kc~0yp2ELPn(lixcjuaXOn)4tWgapi=f`Z@>zn%5<h-A(#FV;h=7qKA9?m#y
z@RGrX_fcNa%-+^6o|D_u&wSf5WAFCV(^ogyK2O`7cSi1}&hIX+{sKqy!+#X>PsMbI
zHTIsAP4^PY)!cvbZNSff`|Y+rB7VDmk^KJl#&gD&oDx;(+>_hh-g{)TartFVhNlfY
z!aG7l4*!T)C4Sn>jFbJO>8Z~a54rrZ#pBM!I%rG^bUt9kyv@p}a$kVj3Iz%N)jHxF
z+OuQCgg)#}R4BNSufY~8;Bbxm1RImSY~c&diZD@kYu(FT0-sMh?(qFnmX_RhlWnfU
z^dpi>175K8m`kZ&DKYyPtk`5_(-WGr*6EI7jFe7Y^Qy`Sqx^8IYes#=JN=j6VcWO5
z?e(hGWWGZSOx(E=R!qC?_<&12+}ZoM)wziBD~{qD^wuBFNNtb1Jx``oc!|GXm)0AH
zr`(|$_q)FKooZG;lW3DKv48XX+S>DJ=X^4ho^Y6Si%eEI6tc-@_m=0JE0ebzZ*OpU
zw?y>P(v8xl91<UkX75qVZ2P+>xpbnU=&B<dg*K_aIBLROdBJd7_O-O|+h>g?+OE5-
z(qolkKkIk9migKnA9$`69J(%X{Mf5p1Cd;Zn`Y5_O=dscUMYOFw{{zss*aL>SIg^a
zS>J*!C$DhNIk&*5yL8s!#5_-%c^hW4B%FBh<(c*w?ddEP>-YD)>8gpUOW0(_<*Fg{
zZrk@j)rjjp4_<n|xb<S1qW&6T$Gr?Mgp3}?_*8KO9Db|Acl#~N5uJanZ)8{_ghj5h
zxO84Qug${delfN0nbR|;C!fE#nD9KCBCEh&p8mIUTlV%H+4o=g`9?oEFl(3jkvo5F
zb~e4@vdRc~v7ApqEo14zj#rnD`Spd*cyq~*DNNbVvSl})=XBjh*Ra|1UfwBpwN%-%
zbWfxE$y;50FXp{*(VyAp|LpSQ^V>w??oSqXHnN`g=7%?{=Ubb=uBBHa!Z!QFCC2!u
z^e8APwPpnDT$yWpaQ`Vy*93R7M^2$-JEyU-&D`_w#jY7%3ifp#pHda)e1DT8`Td^1
zWv*(E@3dvNJ|^&aE_PD(e8F0Jq3216v*Y`QzT?hE8bWOjroMTjb6ZUVgd}fiuk}&#
zwA}5x{MY4wN2S7Lm7Ih;OqPBVz2a=Hdoi}DN=mhDeawU<pU$kB*7xG|VFhWMEk6@B
z<)0N6y0|XjgN7Y1gQbH+$<?Me;j`wd*|!IrTsU{9!=#2P^X$V`wg+?dE=w*8;?Y$6
zt&?`p*|qk*gxmC4!H>N-7<C<W>@sCp5B90)Jme`~-fR->Yt+hi+TdvN4~@$5+Ky{_
zVFH|%TImH76&u$r$lSGYablInjlPS<N8Q7X9^74+-=!&T5cuh_^MoDjqCLUhv%SA?
z>RUhCU2`G6?CkRi%G-sx{^*2hJef8Bo3I5#j?aRUhH33Dr||hE2c~Um?kJfP81X|!
zPW{kHmP>APp1<S2@mKM7374qxf?$nfD>Y}S6xKVeN{Di`Y4Yv+u})LsgZ`YgffAQ8
z0&kk^R95vreCF!9NmHUc9&v18$)9xnfPk0P+yregzp_g?3h^QnSUyOuVhK{7*|fo;
zNMeEf$(5R&2Xn+2Z%%MG@#mGa<h?X!Zk*rbm7JeE)0ZV~-WmGeDDJ=3gkugX`9v*$
zvWskA_Tm`Jx5H7(mUgrp3VglzW|E@C_ni3dGwV-h*&X8Il9bux$5OWO!iwk>R}!a}
z)>uWU$kdrnxlr3zHv8H40-g4r%pcog`iey~9YehuB8zm^vxv@WP@c?ud|5&9j5V>V
znfoUj-h3$hZu+eA^Ul4SX7K#>HV(!Y2VVH?xS!%KqNyV!BP^^u>FbvIq^9}@QxpY1
z7zk<v9{J~48D$wzvLtj(x6!>Aqq_J*YuCmyx!l+!=V$Sz_R&U*RGlqz=13ObRX?%e
zOTvtuJTDuS?S#!1RTgib!LZ1C`mN$brOlCN-kdmIA%5Z6Ti;5zS3O6o3lrBL^|=0T
zy7%TQYiH!zol?({WS#tEZ|0`#u(F#5j;)?^3ktebTm|lN>r`!2^%GhXvAOn7!l~HW
zD>q(PME-7VNNek0_leq;W|AJ3u|8sXL}q$Flakrph}swp-gj$$wn^?yVdI~_&f-jh
z^z41#zNsyHW;AzEZjMFll5K_hYFWE)ZhrUrpy@7K-UCY?R_txuV)9^m+xH7;2bO9+
z*?D8_Ik_8&tF>ap;!3$}lC}o9)lO^vZ)S2y`IJGr%Vwp>&1IF@*QV#?Jg63qwYk5n
z@0v^i)3P;{S3c~!Jwtu{lBL3z7al&R<zwrYX2c_M>PS$^2lspn*SLyX5zJO5cb4&d
z(hAwK>lLpcpNh4X`o(Fx*3Mp^*S9d5^QxctgxM-hZg20Nj+VYH_iJXC(C*26n(Pbh
zd~dW$rJ7xw&|I*6XV|vclXKMhIc^>^zG--OcAd+n6Wt0n%U1YC#^;1e{4-#V+xzN`
z7w@SZo6Qxw)^cR5?k@W<ow>g>{A&I??ZUqv)8bz(3B5HZY@437+zG|>Gt)xEnv)_n
zO8nR6I~wHk(VK<y=k<tt9h`xM*(@BELMH?hFDzT{!|8Na_l2cnarfFK2^_CIZVCRX
zO5}9mc&y~%FY``#_Jo|qsG_Jn?yV)ZHm+&!Go~Hy{Giz--@doQbBEF1@|h`{<DZEL
zru2MEvHrb1@breSrwxvp=;+DtY}lp}9JTB;hpTZ;SV;TRWWFC;gOejdwj9_q`{~59
ziYK=-ZC)D3s6O*kR$HG|wLtX5WMlPsJF8Dxv-`8pJ1<<fF=UqOyR-LBwLEuNYFp&7
zoa@C|CC9UeO1zE#W*UoJ+|Ij+M<tN&h|hXKOPht0wmx4kw8ZmrHGdeF3-cRgo!4To
z^1B6=aTlrnv33-k?{JXwt7NpOtHdr2Np@~2#^awZ$6b@_Gj98|VXuV1L5cV73_lMo
zyZCq8(nV9Iai6}Hm%4fCjP~ocJsWa)Houfec%K!eAv0%2XB&G<OrY*9U;i1}-mP8&
z3!?V8t-rZzldNn>eu~$g)v^A+A0;F%PuB=md}6K=t7yC-B)aB_5U;*Ujk?yNRM%N6
z*4%xRQjt09#fCq>3vDcJmCD3)mfDEctnw=^4NF<ak|R*E%OYL<`5Tj*n}RpP_^xnO
z+??^McQ!}u%`3CMNBIUA3$d~}f1VbeedGMLR^7*T#!GkI*l;e&=jqXqxD65=it9X8
zWY7FDcKa$RYH>R+&40}+Z^PHiE`Gn6p|dvj8S8;7ZTybwkNC}!)zOdsDnGa3$%Qj7
zTZ+8g!VRv!y{07Pzq?5AM#B3p)=#@0=&UR`BEDi>rS3wxM=On{N7jCR<G%akow^_D
zn)lYPl{)B_!)0<Mc(vz~F89k_ksfnTNC|8RJ(6p^b4rF{nET3g7Dc%%ei;(wCbt`1
z_T;@}-jM1Qx$NKv^RV2<Q}S*wzU%$`=v_+toSYf_#$Rr!o!!j&+l+1Bo_N3IXWy*3
z87(N>`*>4j>(;0LPMW`+mtV2ZooRokSmGVAUFkV%CcRnoB+l<)t+LHu1r9#-RXdI`
zT5$g~xxaCJ8qc=RGmo&u-DA4Rr8~iyTUoiIUswB3Q_AgSw>KI{bKYj`S^l_0;c8V9
z(<{!0hxcwO){)@<EP847iWf#d3|6vmWX@pnJn-oKZl8;{eQ!TL`#UMw`d)m<FKN-4
z9BiJvXEqiF<@Ba|YaFnusWW%kcQ!iNxLIFN;=1Hnj!xhBTEP$T7BjUp9M8smpS%8E
z{+pPlXOENmm+qdXdF+eNdx;4bgL9&HO}ydC@tH|9>cjisKatH}3QM{d_uDH6h<VLA
zzS?}l-WS~8?tC!e+-IDu;~X$0Y3G{ioe3Pb1Eg*)mONE<R>Oa79N+ar0YQJ-Yem*E
z+~uBZ7@D8=_THT*Yg?XPxUu1gXmN);`wng=)jdpeYfruV^2FifA~$cZGeUgN!p&PB
zzi2%9Y)w(_^$+T|KL|TdVJLZC8J_#}Y{hO-iLdA6+7=5Y=&g=g{9AI~hsh^nH`=Hv
z%(8#;Y1gZJVbc|lADPLelQCg=VrjpcaCFQ@llMi}mG*>i`L<T^hix^wZ`Ui%u6sM<
z>&{!XXQGw`_C9O*&2l~@EvoO|&m6JZZ1;z}{&9kJb=$coMhYIWi6}YquI>G&>z6lh
zY&+y+#`e>)_Rodp`LpX_+pGVa6D^skrmNV=b!=~^%@Hxzck9}lPwe~G*K%yLkSoK>
zZPyqtUy5X2$CRn1-Qo3RnV0M8OP5?#ua_$L<or^5ewt-QovLde>yr;}0-s7SJa^mu
z`LtECQB@=Bkw?`JZgkJct)9N_Q1z~;+dkJEIu@@!6sr93wC|pqE1iq%re9sTaN)ti
zl!>M8Vb@<B@Mf%9{IJiBIs1@RjFIR5#d6U%xU{A8u9tl|Z2I=0;|{rZir-iFe0GZ#
zG<spT^ZD9oH?BV~{AJY0Hfb#n*9pI%g1fmp&07xgId!}UE?D^edw%UEPfo?urum|s
z%U0xEiVbf*%YE}{W6Ul$zPhO!Lr(iY;>h8ZZu})|lT*EBNAi<Ln(5I;JJVLrV)CuG
z-jh2|E<erXpkkb!qVGnD3?(1O<q<Mn5lKm>yB4(`;C=7oks@Ztro2hTDJ*hJ=VHse
z9p6~L<w%C}&Ai}UCZd=ebaCUi2L`Sw>tFuosM}g~WOI}F{nV%f?gyuR_Run7JsP#;
zb{W&+YZE?meyU#@)aCf~t<&~Dhkm{=P43j4bkJtf_gUOY&nGQ*RQ+2iGf!2+%yrLp
zp{~6z9?#7FZGG<i?N>+7M5`4<?YOh)N>7Q?*F*DWAJY2Kq}S+pP?-5Q`=9)Y<<4R$
zlKq<T+#v-KcRz&m=02OZtL0Q`AKy-vc@}K@OrGv$zWvEX-23w0*YA_v_PkQOc(iW8
zn%k-DNx%NCn;aXW{P%#Sw<6b_f}p)?Pk8>`a)2SlYhU!Ln(ZOAeQN9L51jmSz0+##
z@5o9yK7+H#k*9c5Ux}#gD!Kmm$%HEh=I$t+7%ug5@@;RQJni;r_X1`#I99wo$#q9~
zd$oGn(yeDivp36n&Yza?eBt+Zr-a@f$#}axz`*uxTgQrX(WiDLoDu&e@JO#J;ny?M
zFK5gfcORM<{`K(DqHX77((bK!DSC-1_t4bzj;UAPl>Of)ZfX&;uTNtcW1)$K>)Psf
zN_`XLXSU7EZEnq~n3H+PEM&rmB~Jab#ddA_wT!Rnbqhy;L-1SG9sk4Q8#jn8YGu_*
z{5tb!*VdzRxT22D)p-?naN@1w^ZthETTC-$t}MHdWNo|ijr;3OvnOibRIO9WDsr4Z
zUw@Cea-{GBqrXf}J8y*Pv+SDA{P&^r>rGpv)=#=svFW?~{AU&`6uo!U?k-I9j$HR4
ze0i_-P6uPJp!w=~pN?jza_3dvi#_;s)@0+DbsE#7?-WZe_j2<L>T4*;*p}ICV!ZbZ
zdv%AQ{|d8fYs%_6RU|(yu&6#_vgDnQtI+iuajLwMbyhJo4jmr%`B`n;rycd?)F^$y
zF1w!P=6e0q=Ea^Vq8lGw2}}><+^lkU*W)E8qpJ$tqPIMpxaP+95<{QRUeS<hErZAF
zZTCi>KRNI68~Zg*$08=g-@F~orBpOSP2vIz^NaEi*TT%Cd(Jrgv1;fE-SDI!&Hjl`
zZNvw@O=b-WJ+`;h8rasS?<in)lu!_1jydU;!QA5>u2)f*vqXE-t1k+X3)Wc~%sMwg
zt(+yg@M7G)37cM8JQoPm39JiWE$0~3so-vvz5SDL<l`sS{$=|(->%WI%9mTTG~(B1
zHGxU_QAa9uHZM=%-hD^OJ-E-cIGnRS#A;Q6TiRWXrf#p<p<JHbDGBzm9&eI(?w?|j
zoO;;s?02DM*MxW86gjvwYK8G<kL7{!b9^U%PJbma`To|{u5X7fHXB}Pos+6)6jQO_
zLhPJLs-JlHwsPLxY4p@m<hkp?txfN|CscRji95|uEy*uVmZ@G`w5;^p7QWS{e+5Dd
zxlSB*v@@vF7m=AWdv1I4r&^ike39DzQ?0zqyC)nxQ2#;w)9--Ngomn|UwCL|EbzBJ
z_hg32M45Iub|v-9sW0|f%v+(?F|qk%tuycGkFKTd36@+PpA{ZyG!}C`EdKC*(Q%iv
zo)xp@Twa{yTjCre=XrdZ*|EK%)|2{vuFWotnk6N#V*Gi+=Hdc7C%0G9KAAIhe?Pep
z%XcmEuF$!QHC-+~{@%AN+~sxD+jq`9AI&v=-iarN_?g1rdP<xNJM;J6kG)6tTOF`4
zbY?a8yKu_!&jHcak8eJ>RJQ~jJ#_0)^0tB-b7y3-F6wMLClIvwR?S;3*}C=vQevl9
zG|7lsI4@OiJ=(w3Va4(rZ$G5IOg-`4d}hF|;K{sYo?TxV=TFF<f9r4wlVHuVt!m3(
z_GjvzIQel+_GHG>qS3E9cK<o8_Hw$Z)B4Q$(w{5mEN+|SRK{xgTl7p@XvZX(mQ!5c
z?&YU3sm6ai_L0r;B+G$&n_Qngm8{xv@}j{c=AwfAk#7|i?GatBYZSGm;H&w^JlBAA
z+3yxd?8tv|nqTRx-*Rp@R<+KQl`j*HDCM%v;-8^svf9ryIwflR`APauT23adV%m2p
z@rHF*@ZB>R6}$5G$xZVLSF_vb)90Nda@J#E@5SKW6U)qgdw;Vo%TXxf@qhBuw#0DF
z%u*TO&3;8K+c#D+Wm_y(On6+iv@fCa@UBdU^jq4qpBJbp&E6MlUcR?R_T1Fjo;#oY
zKD*>LPte-gZPMp5ypMIfYH&Jsbc)`!l<+F8h`gJkCCjy=y5#nq-Bj>UX*0u2=7<F$
z1}m-f4s?YYb6wh-bu}zDdOz3H(@9&GB^}>$Omj&>>Z3~`QoH7^IF`@r+nKL_=&W$l
zD#7Ygby=(>oguelB`&ZXzIc^Q>MgUBW2K8*8gmw>RQ@C$jr}~mJHvlA)JaTGWIp5=
zo>h^bE1kyRV^Fu@fammtmU$gNn42!_$O>p&>+mnqwZv&EqX0)w!>4;+X1v^&I_raH
z!bQ({iV2P-eyto$sWS^qgso~f6?f##+4p))ormd#X}`6798`{%izr@K_`54PCeEfr
z?!`ei)o{i42UrjKFm#9PYYn(zHd(kg)i;l^$>G!?Y0fP+A5H&z73g&z+f*aE;nvh0
z5);~E9tgdS4By8o%lU7{LAB!2gRj16U7g6#_arIA<%;5z2tHl8#rGuG(h^c;ei58k
zB)FwWQ9pLZm+nc?HhcbDcq+_kzTT0ka)(p?3pXcDk*ypI`<^(aapp%0-F~-UVM&|6
zjCae2?nQQuuS;Afw5-!+l)L@>nb4c;>#z9=^gmk8Ir3<OqSs@!*Sj=!Y2M23aFF@6
z_lK_42JP(jf|v<&Z4}Z}PqB%!sEeeFKfTtzR+%xEbMuUj29AU3%a#f7Y)o92cx$4C
z6iXt@q>~k5$|iS}o-F#8)U|VJhxqYRk3^zaChiiNQSa2UH`th$H|t|!sV=)jX4t)^
zsx6nlJB7{u7PjfZ4o1^w+>>8%Ti=^|{K#TY)5$e%xjr9mygryJyYlos-V)w*lYLJ<
z4R#cmd)Z{6?LNoNk&-_)EnK2yR93rFnJrEB*Pa=twz@9m;tS7NRm!$Z<Y!RGgLQo3
zJVoU{ckO6AE!z9)_jGNK*Ol!Pw`6ym-nTqtVzk-c9((?rMNj{(`S6&<c!pZGY_@8<
zT;JBsy+y1i)1xcDR@urj#Ob@se9Tr<i8?Rhas9I6eMLLj$@`oNgZ9n*w?)pvreaET
z$gA{}B_+kLZ5}TA{K8STZ(BmV!<I)2K8HxhHKlVlUCB~Az2uMqYxY*>6GwA2UABoR
z-DWaNUa-?Y<i(W<{jqHKe5NKPrp$5<$uzUf?mqK|p+ABtT>3+p-KrpQ9@7&{U$c&%
zKFgRt>u<A(bIsX*j<aO@O}_8{Sv#v%e?^1*nx`gOUC;9u*_n#iPT(xuzINk|N4~jQ
zd3}paI={VX<7<EUm`Adw=rvoU%N4P|Ykzew3*P0!*L*a^`hI=N@y;`S5%b<|yvb!c
z?LWh_rse8#CxsgeoC3=J>fU*D=vWjpQ}jJop;=EQOIPGDznN=wNbp-=ncqCwvwE}3
zrhVM_cxQ)vp~NYr=2YV?f_7@rld{$yd*QQMY=Ozl`bPbO5AEJwv3q*KVpG&~ks~rP
zH}9<NtXi$3B$B<4YkLFtpG$R9Ws7x{ZPlIcm^bfSAtPRv*1Yn7*PU%g&Tn%#7I5QE
z>b9WUe%?>=*Ez&SpJ!%2%q4LAh*{UsgHmSKeyU`tl>BF|vAF#v;!UgW+wQeC-MS~b
zV|hh(J50`-v+=pi4PzUvU(cuC?YfcbSg7zR>3+6W{zm0lFQ@;Vc%#%YF-V;~)%8zA
zZEo+AKCMp<MNI1iy-rv^E!;LMQ}n(4ydCa-50>2IySB;sd&=*y)7QGi^rLjmZmiUu
zcji%qU%<i@o+ml_Lvp2(o>i~jtZot|zip}Lxt{8E34#%EXS*am&9VK?b=R&~c76VT
zU9N~g&Je>x`|~*dZG5szW!Jo}t!6(}7P>1Oeyyo|PxI!Rf8EJ(kG6+jjm$OH;O%(5
z;wIB1Kc_ao><9ic-dylX>-3+w(jubzTfKqTY?e7j1}qnEd&y4UTV)`^bh`WT&(hPL
z%Jbr#Bcc?aYP;0eCOw_0yEL@HA-!$#B<|a83wWAeh<LEGt*_`0oZBa{dGiIMk1R*`
zz2MLI)_mf%WJXQ#soC!i9WF3>XZvu;vh^C%1s=WYT*`9nMUmW$Wvtw)^HM@STOYY{
zqyD3Y&zk8ys}tFuI?P(WL7L^o2KA(*o$Aq-d6=?z_RP02jJ+Xqyv1B|d!4^q*DTXA
zuOlmCBp&Fn9Mvp%$@Q9L|EiYR&c+NYvt0jaT~vN({Y9>ub*~Du_#B>ahW!&9`dDY^
zmCVk)y27UK_Pg1-dzf#zF5@!e3OjV9JDDN&oY*6+mP1$KOBQZl_H)vbM_;$U3{zLB
zzSg{~Td^#8N6U>at+GY09fjqMmb)5=*>z;xVQt%F_BU3`W~X!h6{qlx6Ezz&UTjcq
z+cLd$!?dk`WdGc)>zIBpYq#<|woSd3&R*^+U)Q;O&{0e%k0?EEFTNv2P?ULw#*tk@
z?g`@jj8nF8hXu9gu0PSlR=I0)kX`V-iPsGuukv?aw<;#Iye~`YJXg5tZdZ+OTy<;P
z*ylP&|Cd^J;`nSXo-6;9XX~DD+q?NDvxM~QuP^P)SQCFvTDU=Um$uWknA0o%IVCkD
zAL900^0-^=h^&_P!LK%XQ_qGme7*g!`vhOOuu4tur@W&__C)e9q<Ac8S;w;UuIp!m
zuHw3%!Y?YfJbP8NhwJCxV<&yiGao#<sfJPT$48s0oB5GNTiQ}4>E9HKDBB}`#@)G6
zvwdd%4;>G^e9ii82lWNE=dO7Wd1iIQO{K_GIUbwa>IvI^E9{xEw7_=Sf=!1vB`g>C
zb8}(%rn!?l&s-{X_;7WP=jLmSM}w9$PS=m0y<(>E57zWE=Q6_7Zl}$>o?kZU%AZ?|
z2h5G9Rb7}<7A_*m_~PNhUGYn#v)(gE9?7&)JAatx{)Tl&Ht4q}?(bywa6HAQ$L?;r
zCWP^&YTuD<lW#KE73}@BW5WsW52t3|H=OJ|PbR>gQ<dxf%fx7wYWr=E7du^g5dTK@
zWTWQlpL0(geq?ikL*~pSLp|-D{;3w?Q8N7-s!w^T@tpr6TIu4KKKo0`?7h!h4vF)h
z5f}aJ`iOJVT@BUU-d<{rpN%aa^`6~l+<lp^Z|66;CDS(F6l<Bd{(bEHZ}-=3IT$z5
zNAFC<#EChXcK4eeh7<>Wt#lH-7NGoQpILpHxy-F7=@YF=S0-Fh=iB9JWPUQ}!L36#
z%QwyCs5E)NXZFCrOy;koKKsqILx&%k1pc%+mVViw|C#4%UANs)>X`=@m>S1i<(VZ~
zwk6=3+K)Z5{--pP#pE{Ml+J!MW2Rv6lzC<I?wrjkeA-dS`p%~=gez(F+zHEfuyjVO
zTyl5Sg6z%QX<92jB;I-Rq2lly`F^gaT;Z4UbX_*z;@m5#GB>t1AnB62wwUdk@5wv5
z&&_(jrsbOM)kVv)B16BX7oKqsnqk!vGxg-^Y<v6vVIczMI^ypG71qTDT-wsLYHIVE
zV8_zwQ7HwjQ{`63oIBLzzgOsX#P;9Qg{<!VT_*EJj`41!N#-Q>IlpbSUVJaQsd*^#
zr|(XuYlob+aQJIpH+etTYh}cy6!ANqYr`K_Oj~j1%@1y?RVs^DKdVUepS}Lg?9IRO
z%*5~9XQ{cmX7`R;4@DT`w;AimsD$|K$hx`ZlIWY_Vz$l4!|FDF-2SBX#LukF?9<bf
z?^)c_y3N2fd&lyo_+ybwO+u1QiVKB~85g#xrZTV|j(Aig{%7qwpThkvd*grgie1^7
z{%wlUm)H5vcUHU!RAyT8V}0zR?%cSR)ZI}ncV}GqGCku^Y*2l5sdl(a;p|xq!9HJ9
z<{fT1HYXq|aq7fa9>+^hto<vQM45XpN!sK$Wxh))FF$j^)7f{{pCB>!z^X3UTi=g<
zOH$k&n|%Jk_3y7Ujqmk`wu)T(^6=YQN8u3dxZNvV!dsgoqN?0E1&kjZ@qOA}FBx;(
zCnm9FMZs^I85-3qLL9lJ@=e0*A2aF6l$zeDp6Z#qVtJ~M#F8}I5;OljTr<uelV0~^
z+LUcpG8#&`4U8hQK21965XvLhm07Skn2mLFYuVdNQ*Qs+A$Pmxo6zO&44IMVYK-5^
znpGyT@5-G?AvMz@kN!9{srug1l{-yRS$#e_)U&J3n)I;YndPC|`mBv?FC#Bqy3cCe
zJ3%|~>Ff{t{0^M0?b17$Iz36U!zpCNBAuTp(+-EUzr1qaOKqjRcx9+=U#^PCf?F3?
z$-VFktPK@aJa8l>beZmNgLGMSrMgqXHLkjAx?=OC-iW#F=-*s?Bl&30iWKf0fmPWI
zf*$%$zlL7B{bK*Z6KXS6^1Q^(JrR4j=<AuM?c9g@6;;koD^^nQ?Qni7y~udZiie%k
zz5U&<MSRHO`?5VCZ;Cb7JYJW@34)g*uiX`~+b1TobXBHcp|AN$ksnPGYrTKB&y29O
zVS8@P*uG<LqE7a`Re8H6Rz}<n;%ZL49T6kOcjDt-*?ama^G;u2`E;Sx#-!%4@)2j(
zwTrx~+qbW{Cb{06d3FQ0fOMEhp^}^9=KGUs_lw(~-`Dr)@XFVw2@`wLEbaxHYUeMw
zm6m_fD@>%sy{D^XeWi;0tDX0cIB@5byH54EzHY;Ar^8Cmw)?uqs)@=xn%#3JzvPsi
zfK*1u9;TRSJu!#hr*d`%JysE$-nQ(oQP_bBz125#x!c4aZ#`Zop|ZjItBk^-C$l&%
zUEgH3V9A`5ipSTdE_}U7=GMQvWgkT^1#~-4tk7TmN4Huh&TMMLG`7ttFV1&-yZce{
zuR+zuMGp#Ib>(hocu~7iSz%YN-nQp!ZFcZT=$?M$9IR_EyWjv<_N8}wZF!$B-z%N6
z+iLZ`O>&9#`zmu;=JC!?Ur@0!L*?+q#a9CkTRzeA3iCgb6Ll>qREBGYyB9-|lYoPS
zj+Loh;-5oDx($U+7dt9+rLLTC<ZNx`yJ+K27v{yKEfDNI#x~=~)7gy&pKLFCz+<YG
z-@sP;@>}dfTV3A8c84uaZOC4p*tj+~Y_e>fs-W8B+yho!JM=mItu}OL+O92sVGtMQ
zUlXBwF{(Q(?2SsR(6=ua1m5u}v>#WIJs`yVB5A`L_6x3#HLoLf2}e9GJbtn}catS+
zbHwY)-zzqsKYQejh|$F!hY)3vs~22Lk7YerD3!6nzb;XS)%ymo?t)~^9Zu}K*3C#{
z)j71W;;pBY`@B{K!KW`d_Vt~PZq#_wyz8Kp>8gqOi+n8B3SYXuChl9=ti1=@lhO~1
zEZ_JyRdBZ#gN&8>8rk1ro5~zI9yaxzzr8YV#~$vu=PhFXPqut@P;JXUFhhai8_$_<
zBGcPdPt5DAjQC;`(E5nc!kw{^<EO!gqYu))hng)5EH!$)n_+3H!HzRm&T6brDRKEG
zx}(kEAP?L2_eBTNCpm?ko*TX2v7h_xGAD`8a~0mT{@;3~#607GvZRbBo08ng6|6<~
zQ%bLWu?hUWiaSmD!qrUfr6*1P959u>;&Ckg-2SSa3qDIrR6dwsEm`C7?BQg?_LTp(
z^K159oR@Or$wZ#BOXM6JM0&y(rVIBZMLpO%X(3m$rpA{rhKQwYdW#My9Mvx0s(mwy
z-FH{u+=HTLPe_PL)^5BPKdW)un%8Z`R=?w4%DsA*e~tHZNwvZIYo*f`tg~!(*NnMh
zB5rVNl8pMyh41A~_8sLl-R)4yZCWK{yL%qXs?~nSh1G)(r5?L1_=4+8NvqqYqcP1V
zCThy8nl5@ztJ0GB;Fj&LpU57USUW*n+1M<)_J#I~He(&GCmllR`co@<&m42nd2?gk
z$4z%*7cFr};GdhX(iXEO>1<g{SE1PN(o<|ZI5wo6Yz=yB|Ly0LSi9S6^;dcCa%8Jc
zn9Y{ZkgfKk=iX+^75CaQwL~(viR(*!oEbmcaX#CMRZZvKZz#UtdNry0)|$B&YhF}%
z9T!>i`raGWQ@ada9G<tx!gtB(mCO+#%x=>fOwIURlf{%$RXLItn+sJ3ziVABvTrKu
z(-4DYZTpU~Zn(;^Kgu)U^#nx)kHAB|(|q_$msljfc;WQrqurU>NiV#@QiT^Ce!$}$
zD9`d{`GeoPqN<ew^_m}-ybJrpaLFUJYv#Jy-{$geS}d5UvCdeXm!0=$#I$2m45s{@
z9^u3PXP?;JRpAu^^Dfoj=5NSkDCYQkEhJQY*(H`$AvIh0tF!Kx&x&pGjVbV{ESr&*
z`ZFV_<G@)RE@4g;7oX3m>%LyH*3)P`Y7}pv->tboJ7Bik&O@6YsWoJIakXz-b@Xl1
zofPpkCLF(%t3SS$<om^#_AM&n!*Q>cO&N7M4tHDMopn0xc8Sl@=-|;#fysFePlIli
z=6`V6;C=c2%tMw7(hllfELmOXo#D$~BRI|Z%4|J_V;@(re{DR=aU=83)T#B)^^X>8
zY})j5%DeVsb5E`jsC&tB?&ShYS>0;(sLM;wa#i2p=lf!2_J0{SBj?$9E23{GJUSA>
zC4cR=!5L+TWjFHIa4F6|UTG?&(pRE;UFfUC5B&?W8xP8c771<O-zn!1rJwp)>;sP?
z6U*citM4~mjGbYiGvfklzhu_CYmz$*7$dfIw>*sfus`d_)42}s^c`kzy`$tCyQMJD
zSJg4u?y#PzfyWijmdg=KpBP!ptc{%eYyMr4534vgo)fM6$NHy$+1i$y`|z><P5e6^
z3(pMRyF_)?oQ=O5xo6!L+&r20Wym@vwUu>I4bcJY>*x53uX&YP_@`A%;wFRt1($s~
z&+h*epCf7!WF|AEcPr1kxR7r%o|Lj*=J7dYYjk1rokLHx`hRu5kDjR3KCS0!%%n;i
zaqYZa?k`W<diyQj@?P5FG~1a!b|<w=ZZ>{8<KbrW(e+uIUQbD5$K^uaE%(}G55C|r
z+kTTNN~yZY=VVTXrgYY+wgqyctUq)1PGK}LX_(L5`Q?-^uiBbplX_1GttmW|_hV|v
zr!w{4^mPh_KTkZ#T{myHtU_VY=0I&$ZC-7ggoNIQEw7WVJ2WaRtemjUpun@QfN$ZZ
z*Qy)e#x7O9{CD@CTUT1Tyb6AYG5Kp4f12~gc(+OB;!R<9(@%+o&1e?jJifueAhBW6
zgX6+J-`x6T<W@7t&pc67u=do{nk0c7y@x?5+Q%c>3b_5-gPHw~GK6e=)jRc7B7^*~
z?ltT=?`6K74Xio3-DFY9|7d>gkJ7=Lj`hU(K6~x8W?ueM34x{U&DCP>qMVmC<wtG2
z|8a-0x%Qryhr>4LHfZOCYoC45y2fzY8kcE@uD?DY(8tcUQYM{su59*F)!*}M*SV=_
zZ29tE_?8);q8X3P!Z2foc~%D|=NNrBuwQUBqwdQY$r{|c>`u-t3wci!e{ka3Th!ox
zx_I-CO8a8d$oJZn&D;#u4=*}Ay<W2Ioo3FPQ)$d1{?U1BbA31~t{kq5cUb&k<)pR7
z^Cj2&<Sg$=pA%ycrZFk>m7c?;2YKqRzf28HtXkl;z~@Mf$etsHW{wj&q^>!I89R2W
zB?L>{FP_)tEpp+};s;3<c8i7PzBOD@`9$VAb3Y$ver3~Qhe=irK}L2{cnnnS)qLjX
zO<)veo^#};IrplEhbBt}b?2<V(9aiR;!t~KZ)wa1hNC5u9Q}B&ZSqS!wjnsvlqdA@
zEU#4(DzgLH6E_Peg*7**T#!(Dt0HZwVlyTEc5M6B<{t%j&%0F@@u|u@U(NVx#}4`U
z3t##UWKC{NDyv(5W!B=b@>35&_;@s>b551*UvPQn4Vh&(3WF7AHCAv`9o#K&Ag*EJ
zy2agrS3+-jNi=sI5nQ`%?d+Lr&8IB-w4>y5+xksYdnZj@DI26<=>BEWIgfmmjKJMr
z-`-ulIkf&u?P0cOJPWV7b#ZP?Vp=-mk=8Xn%|<7Y#R6I{cvPj&Gw~Q$nEXzxXqEjr
zE9mZq2y>}xw!s45i|$`9c4Kv0x@=>cfhA+ou1mcRQxbhnyYw$u{-$S|j6vLv`*)=5
zA2;|%SifJw>~oT}R&m9(eXm<=gk$$RD(P<WT=4j<-G*Y%Yoar*7Fl`b-t7N+D%JUq
zw!!MtS+3%jm#T?BDb3?K?RbCH)16Fvbv~Xwe&JbBn}&qKjP?5V{>z!{3v?QCzQ1Sr
zahk1n&)(}fIVTkQ0*rt6Uatzznz2@@toXu=n8!QcUw_3VGiwo3>&=sita;3jj1HvU
z(7Gd^mR{?=eGT^>?iYFoBtJ~MsCQv*L09d><c|fC`HOEJQRJVd;<TpB^~8%K2JepO
zy*vNGf7;Xu_f?pdCBHJ`4c-t`6DV1)8BjIlvEJLCO1tMC(pus6@by!XIedY&7R&P+
z1>@uN?rmLD#OW9Q_RB+~1?jQ+O2W5y_}ILvxwr4SV$D;no~t()zuh=ITm9_EL+A3$
zE##gUUFGDE_loOpX$p6`t-Vl3?elW({GgKJORrBREm9D8$nxE}K1ZHe*gU7|S(d?N
zuLX^tezI;fQS?_^(q7}6aERrJy0Xe0xpQsb_UIW;3tmvtt-WR2m3ouYrK}1w&e~mZ
z`RUkM#IxZ+pSs%?qlKsPF5YF#jX9^RYsRsH`_K!aPv$IFJlS(kEqyAo(<IPv^6Gdc
z-N)Cmn?&Czar+pw^76{Ky~y?z;IliQG=+7?!s9CJ0X0oW?``|x(|vx=kCv*oX(5M1
zb2dE_l?phN{o_S<{`L>vi&i)M(0UeepyK4cyrt7)SY}%sGnWd;Zd`m*JFV)L?80>>
zLZ`(||Cs7ko{HKw|M*qTb91KmteLKpJnc=f*rr9TPu#tNEsk7e2zB#a=-YBYvP1hg
zyV^M!HrbFjzf`v`@h%jVITCg7qPf+v_l}IItD4U4+9am^X+!3tH49IyXMU%@^|QW>
ze7kk~*AkgGM?|82X@6k;zRCL2GUog;gH4A*%Yvp#RAj$d-#Ysd&(Sjl)74f9OU(5X
zJ18C@7?V`GVh&%)s^SN4o#ZFq*5A{^9p*LVQSH4*W^3l8(}lUy1rxF)zs!}cD_CS8
zslE9z=Y{DHwtc!BcleF><o+7dKVpaNV&<5L>-_WEExbVQaD8uMPKAO)=BKQT>`1H4
z+9giB{bBQtzkbeox=yiWJD)_y^|Jz6;eBN<v)8|y>+PQIsqSgsos!_U<U(u8zpNRn
zO`kO+?><}?`@>^DgFxBAlZUg8oZb=j{3XMtS8V$l4w@)1mwi%tb2nA<|5J%Y)0zF^
z`*c^8Sg`6%e&A^pP}VVl`H^AW)E-^c#{m|iyMH`7)Y8-RaP`AY{mrM#0?M8)`MplT
zV8b)6O$V2WJh=SgU8nY|!nbV4N?aSgboVz!{+GFxz_wsgpm9v7^Pj-N-tZ5++OyTF
zpSg%X-_+dk+U{NZL-XcpqkNwo-d-|uA6l@nx2|MhxV5o@(|j-A!?-+=$4cLsj=!=!
zClkB3F`D)5rxyOA)J1Kc%yX5RmAccJXS_bh5HhVYsoW=QhkYYo{K~oOZ>mQ#f11?g
znpCS&5-8-YztL-3dZ@#?gQsS=ZC6_{Wo}8?bH)?14j43l;!CjK@l#^ThTnk~R^^qh
z$(y|6!b2mIw6BUYJ5=v$#QivWeNEbxHJdgXX?@-$XqL8pm*t%|OqGlqd9|nWn*6!6
zq3C8o`j&lbj+<WGd*a*HM8#J{O!rJ1_p5~;n11hNj?U`i4jF>)ekdI^DEypo_C>_!
zXJNd>H^1>5)|z~JSKse*_a|#>{IOW~Cg)}wGf(j0U46fgo#jd`cqeC;D(RuPI8-~o
z;?4cbHzYlo;<>rE9F66Ydu^oMTB=;EyDEe2d0)u!xx0Io-_(1XvvFZd#nSX6aYw%C
zh~8hY$n;*^rTfOwA9rj}OkKT0AatW`zVOy(6LZfCFFNBDw?icJACnxPW9iieG2fa4
z6gU07wN~U2m-mzWK*xsquxFe2!gn#PUhTJh;m&tkSkuy%u6JRb!NtcTzTCa6)zxO=
zszUqO)6WMSKfK}a$BtuZ{7PzUZ&z8(X$rECy!!Z`QPy^o^=v!l{1v!$JHcJ@d189Y
zqZ?)=p;LUEBiNqKSUB^kK!o$jYux%7u5T_)J$`iM+=XGE^B&86iLKbUZrQ_i4&QiI
z>7L@N&19dNp?7mdfJfi+&v~}5mfv%`w9>9>&fe9$t5&(a$=JOs^E}`0|2a=8S-8FK
zy*xI#J#FXLS579@cXB>+TYP@Xc<Rd4l{XjfzT@y!LcM?T_JvE{uCnX2$~(K^%+j;p
zHnEy5&|y{nlhnC%OHs+o-t+yt_RrQU=lGPA`1;thH+xmvuZvX|GX8jT?2L@au9E7~
zAh+bgr4nu#UaD%Eha1)$RXw`U+41SB{F3Dl1a3;DoMfx~a#MSgz>iAioLMI>@3uCc
zf2F13hV}F(kIp@J=;}API@P}PNRQ~(dlTl){?4&WUS^%t0_{TPw_E<K`x@^XGXF*A
zI-Nt=T=A>JqOJe81uNHF6%AFcS*LJ&*Y#P6Cx6Tge4KT%uJp~N$4SQ*Y<#C4{$L%$
z5z8e9wcq;d2sS>Sw9Z1~_T&#|y`CAL5kGyo&qLiN&^^?5qT<Kcje$EK-<W5}_{A$B
z;bc|X%qK=4zm^=Sia(b7j^ALtXAtAbS%urXcR%txd+ua7=Le}%%YHDf7Dy94bZVM_
zR=4;=xp%L^>*fAmSd_nFabN}S{nEozoi6*=il%IrUQ$=RT>jX1wanr;O=mrm>CC#t
z8GE0&btIegT(D$g@G24K-g)!%<V#6U@~lehEQS1*@k9%lFY64QIk9w-*CK%)w$j!Q
zp|3w^K3R2=Vd-3xBxdb(=|?42SmrH_(GQv6*q*1#Qf997lH&-ke@xS>PmXW&WaRxt
zs!PHdl8yxjA2TsYzuj3BdVs^Yt66gCp`3cZn*|B{E9bE0-aa>(!OY2IIiEu&&#ARJ
zic!my4+?olmv8udN31M(!@D}C8A*2)_;<{EXAzaTdAmx<zkn;;j>~O#NY!Xht7Cev
z`sIg^*%ZMaw$`r$mF6;hKjpdReaoo@DHZ~yr<x^}NWC+j*O47=^5aRcW8Oru2ho<R
zO2qRbKI$!F5%+v?Z(+o-4t={jj~>5JV~}4rwdG-~M%l^*l9nc)j0)ucu}@q4XhGtn
zjTvWF+&FnipsUg`N%u#`(T$rd_KGcEw5f#4s>qL3`OHfTn+rR1cD6@dl8lvz(+&_@
zm$>A1=*2+iJ<DdF<8qi$BmDT3-p#=6`~oYVaekQcCRw?f>G!TDV%&Fk9@g6a;g71o
z4aQKhFKcZd>CQM2+Y=$md3A4V=&7YkB0u{6{E;8D?38P^prQAbxm?YL47bEy+%~$N
zC?NRw=joiXisn5&dvAwuu$n743O?%VS^lYYa%Eu6#O#2IOE(sE<n9etsJ|P#^G<X3
z@5>p98P}G6y|yuh`9Mm5OC#$Y>-P3MEAdtT+$J<6-Z-TpHlgp!RfY|lHttE;p)INI
z&Gv56Lz5$cXJbA;O^y`poqg`Y)|-z+MWs5TPm1bn7w}m4a8J#aN3*J*y?*xmdI9UP
zmq89ihjm$dZY<OO@>y(3K(CyH!<>sxUnsFIoog|}u36tIW_4}i>$b1AKWsLAlYZLb
zsOYy;yW5j0=kt1su2np~?;P7}DQ#`(<DYKs`%<nTQ5i8`<?|Yy@C8c>r=H+?w(+<7
zNjJ5S6>C&B?heVg!{VzRzO+v+wR`mw(=}UMuAK}0viFB;<wy6=+PnODSF9`4^j*GW
zvdu#um6Ne2^EQdtGut!&F-pyUJ|jkadD{!E>k)_4wy<b0JYZK%<XGp{vE4cT3iF|j
z-lgBxJ0`VnU_Np7;tq+x4Guj#mp}CWvYBYf^wmdbe;$ul{lSHkLicdIpHkqI?7u!(
zCtT^plYK_+MTL`}G_Jn+;E8xdi^Zf*Jqp(rik~Ph)|n-AdfD-7_Ya((l(-~O;-p#Y
z$*>L$>sW(nhZeg{$f~<A`?5wx+fMf^ht&ta+E4nc@bH+%ll@`}>BcHIdAjCFZ{Oss
zdRS}OJL82{oX>M-C>(Bk|0F1h-!ts)ses0tPg1rSHCp=rQMS~bpuCbzEtEIxrou-v
zj!&h=aVeMG98@Q7co>iwm7@IA=z4feoY~B&x*-{13!?sJJ~j4Nem(u+wH1<mpZQk)
z{&?rd2`RRvQudofFK(Hh`o80)U(aUInVZeOzS(PayPwINr}`$7(3e$~d2RmfZ}wW5
zN9&#XCs)y}@wIII^s_tD*=OF=2;bkS;iRK!H0!NIaD;-Kx5EX!eI6_J>i8S4kr35U
z7MR>HZP)tV^(xAt3_>@(!slr@`McdIIG9<W{y^lhkAn2gZG|^)g|9l06(Z<xz94{W
zN9M7ujEak6SCl_(aN4x6Kj*jGXNO(O7$UsQq^GWZJ|)*D$|qDLdY<U+%i3kz%SHBX
zddss}@+I#UMj=zPcT8K9OAfK8IGO}r*r9u)IbUQ&16%51$8%mmningtIWBM$d9tZ6
za_6dZucjp$OL?aHy$H}g;m>vJAnVzOO&VKF{v>N|Si?2x=|rYSQn$o6@!Bz(S?_v%
zl<n0~p$SRL?>nu@dCeh^)|wEz+u`G1CYED167wcH)`?tJPK}Vx+tKhh_}%lv6M{eY
zr6#+Vc~x7eG0zC)-=JxrvO%L{i_YWQ3xrQaPBRR-s1cpmwf}5O!pEiNhrOrszgu*u
zbzW%oy?-k%P0pEC)W~3a=)t~ShrM#7GcG&&J4y42DVs&TtJ#0*`W?>O-wGMuy_A08
z$G1gYL2+hfS@k`qeJnQ~{$BX`3hxvN#zPS|7D!6++>uy$@w%mZ*mE21=OwEaFrRo5
z@Mu-uN|&Q;vAf<!D{Hs5i5bc~4wfzrJ$RsmN3pcoElle2tc6VsWpfVxO1R2vspMZY
zN3OQ&k9_lkl;C?p%&M>WCui(F<-Lv9zl^b9TcQ>RM~22(v&(1HUT*%r;bC(L16yH_
z-GolwxXrGwuE({trY)(ntK@55Abpy3&O6^-k)@@}S?{da{I;y`m%D`wTl=={ic_xc
zaW<*GnIl)9=(WIo)6v7b%Z^<OIlwQswedYmbxzPZt|XUC_j5gkUXBTog<N6634#YF
zw|=nMC6dzqZ<|nJ8>_RQ<}aVbiNOy}>ov%)Sa;mFA@F+qna15&xvSP)a$?SNT4hr(
ztMy`<(^Kz*ooa2%gP%`Hkk@fcy=rBs_w~U9ojDiv9b94;E>)epGq-oM#LmAR_cfX|
ztIq#ZPWGyt;pMS=hEey=<t!|0=E0kUbr;#N?oIeVV?%+@*~%D!2)4zZnbSg6=i17s
zxHoi_Y&Gtx<_=!ddiI7;i0)DC-AD8u$UUuiapUBHw@(>9y{+)kdhD~4QFyV9k4wn*
zNr58CM>R56em?M7D}&u9>{eEyDXU1L&XS+IthWjYw@fiA>DJ{bZBd&dy1FkydBKEj
zF$Yskjy>Q#*P*}J_MV@f@P9>_FoSuS1uicfjEeJ(o*LOMdF`pyacs(sRI5zwuvn&<
zyZ)^zVk*49tm18^zShkN>}ri_?s}{3WrUq~>+G>8)!U+=H<?X5<oH$Pw*I%b^;rb6
zQhu2Aa_YY1x_K#g&-FEP>J=~U*j3t{<5KU*Qtr7nTvSW5#LX^L|FW|9k_&vRU(arf
ztI)5O_X*2*)U#_w!x9Ar>)C%6#9dBS?E9|NFiE0D;%|xuvzvXS;mPOIrk<RvY+jJ-
zbLeMVWyE9F?&Zgi-A_2dX_-60k#WQ1$~qfWwT24zJxf=<;JV(eCYKU*uEdhpd;aMI
zuBB%ilP{b<elhCwMGpRJXS;)Y<L!6<nQ_c?O+riBjHi;FXFGl$v)=WwZ)Q#XjwU%#
z3CEX#2PEA?L`0<vB~OcT99fyi`NcMI7R#zNOD0NaYI@1_%@$r@u;W>b`K(Je{JZvV
z&lF#%CCGT>SgEGF+hT3AgUmk-BByTgGTUUtv!x_K++j+Rz*5_O@mHK-riM>vZ~Ha-
ze1~J^m!&fWatd=-EZh40;|Z;ZrELF<<}hu3Vo)U<pMS|Jx@NV=*ZJ<-uEuQJ8pD^p
z`zcpW>-6;v@oSzvW_Y~qJ=eofqkofnJSsY4Sai<clis{D%uzY{C)){^zKShn=@W7c
z-QBwj8H2+<ofP?HclmN9W9kRBZ;q2?HC1olxuveWZ_(NXlh>wCe#>kXJ%8DY&1r}E
zZ|W`EteJGEBW?ARwET5ic`7a&Zhq)4STbpvy56*|!j7bxMIp<NX`Xs|MbzcX{M8b(
zUL2FZU6i%poLOFt(B<jvPma$1*QO@3poWRvGGlpcyGQ5|5qYf(>%U%L__t-FIOo~;
z`qkwpR~@;a;1Tjs+2~aLFYT#|TGrM(rytpPwaZD7`|zVQ)3(6JRt(JXQa3kv#%=rO
zX~)3Ao4)$<feUGD3uZL4ZPDJrs?Eya*i<i?E24QW*ilWZvO9jM%Jui}ZZ6{dZgKRO
z3#;mmpn{&>(?)NN-tn5V^Xhk7C(eEoAZ)eV*7>Ra^BXrkbbo|=n9n5I^(QR+0e82S
z+WD~3qi*^gc~=gVo>_3n_3P?8&sQqzw+mNrse7$`^Zbc)<22X8@*_*v3;Ax}-Ms0~
zMW)Im=I3ksv>yv;+^|_wc=(x|+?(Fq2}h+<zi)omw1Z`{T>8e@4Ua@4-fYjWPcALJ
zwcvc-PcN2ay=~FDee0L@9KL>5bcS8*&6fqNg(5dybuu1oQC*;1DSGEv_>sP=OV%?8
zGxy(|P*C>gan4&=naKFEza^38$Bt~>QZlQtC2!IRjqa}pat?(Z2+cW=cH)8HyCoGF
z>cJ<&^(;Jjo=B|rytDQC$|vpnGAoYzGM*J-HWM`I_FnyYWq<28?x%{}K|6M?x%#X1
z>LhRWT+3%=Z!^yahGncuFn3xgrIN}Tx#sNb*y?+6PnQeMd;0pd<N?(V{~ZfA?d#pQ
zHR!Ms_ng-L>u;m>N{hX=%UrcMQzO~DLHPRF^K;MdODg8{oiA#>=$VK9^1#HH-G@T%
zG;3Eq=;HBvDx|9?ZqRD0ykAiM?1F<%mx`5LZ*DTwwAM`y=RV@9Ss?Q6=B%YQykEtR
z=3QIJc87UKvcKnn{*zH#-JA|q%(<QCETegQ-OdoM*wbs@23KbI+^)Otoxw0sTt?)p
z^0Rtv7Llwor;e?;<N3+ge$TAS509U0o!lLJcP00-S0?Y=_XY-<o?5aYe`@cVn-6wM
z%4U?EzV>G3v#EKv_B3{zv!C_~+hWh-*xYHmB;&JR-u?~S^Akf0ezHlHDwqogpN;x`
z_Dznl`mH0<`(C&z=4^}0JhpMR#yhhwM@4hqTLgZoyTfrzJ86aXNeNx;6QN>1ram-r
z++#HHDaX{gZ=D!xm$mkm@A4>EU66YG^{k+In~&}5dlBkfd%w&7ai{W+9^0ftCjwr3
zxV5M_ByjqNcHKS}soej8X$HfTfbW`jHbs{(ty##s<$u`S^Sd@*Y+AZ<uR%fIhwKME
z+-7+PTXa1#7z3D3oJ?zCmRZGY$*X=K^2blFBL)>MY39f3n<^86#m&BQI}|?@&Pa^V
z-@9<np%wwo`9fP7r8I)wyV<!9v{ao__Kp8k@G$zd(^2-Pb5msHU;ka;(bDm7)6>hT
z+4E2LTuQBTZ)<4On)xAaO0iCY`{X?}PvR;*TkbOuvc8?V?Zu|QJ9}PFb5yjJbr)~F
zaBs<Bg-MP}C9nH9E-4gG|J*8mry#Vnbj?Gn<9Ax8Exl{PIqBt-sEOH<f()}FU$P}z
zE$h6o?%kr^J#!cB-P!iTU5|aA#FM#sUlsVpmx{ISk~VmA<gDkkM4n6ulU?uHT^8<V
zQ*FE2()Q4-d$~xwuE+whEYr~6fsA@d`BDcIEH6yW(@|s1OFI29=Z?zlzDt*9p9ndb
zmUr?`l~M63uAPZqM-)VtrJO8W9Wi@JxQKbz=AIy5@fSDme)v1FMpjgs)jRAB<1x>^
z;IIjL+NN5`ueZvmWtvIWsoY*;XvE58Zuz8i(;cOsZ@i|ZtH}GVI`A#~WLSO$x7LRH
z>=vu!jIw=I=NoGBEqr3E^5sg9vC*@a=@;$Tr!q<@hJE7XKEnJYedjczh6fyC(i^UH
z3e;sT+F%k`;MbuiEVxB@*O%X`CKQDBUObbSa{L3oY*gLlr)AR}=RKW#IcL`;p$C0a
zbKk7DzPY@7n`hXzw_;p8eKoW7wT(J-4PHGiIHe#p-Bdw_(R_C9i4{xl?K1X^ni*7h
zQstzFZ&}pbU;iZXJ^U;tpX9qR@2uSGTPu0jZjvobj}Buj5z&rJ_uutq$1AN1bK~~i
zpQHayJbXeA<4UIA#e!lcbEBqaay`~;tMVxLd}V&w5t%T(?j@6^HqF$LlvMwg?W5DF
zQu*Rj-s43<2F}N`t8TpGI<23uM&wz3Wkm}+ORbvIq1Ei)rZ*|o&pWqmjbyS`)$+@C
zPWtSV*Ze8x5OVrKj7Fwz{l%kvvt=h7u#4s1a`D*2BX52QnTH*HQF~BgYr?`P0TI!b
zBIzeD9w<H*`@Di}#v0dUenyU-Zbmjv+OjICk{L$tOqe1H_nurEq;Ktg(Bt|uU6c2x
zDy&w&XX`M%ANB2F9(Vnt21(D%wv<&D`Lx$Q@UU9*#fepUO2?u6&cE#i_gNpu^UpbA
zrF-(1_Q@#CfFo%(rS}=SxB3NeWQApN$<Ac#NIibF#Xy%?Bxk{PCx^?tGejDMtzT@6
z@?f=DX`|hiF-P6H<36*ohWoDIES;k|CVo8^Y;;aP3%Q&YcG=OSD|x}|#@8qH@^@><
zB>7(R+IUv)l;+z?6OlK|qGoiwU(A=7dMoR?{?(RwK8w~dJkU>8JF6*lj!o_oqmvYe
z8~c+9Qy=X+oyyMnQjakr$kD-ygG2COZbf*b>gt+TT8u1{{<&0I?U=QAr7F93^&PM7
zV>X98%OAg$&S?Mp+2pXKn@fQ0nzwJ`i@fR@xt%9x{BC%rq*+<AsxX&{n@zdt{W+_K
z-Wkr;{7>bl$fsED{rG6%oVSyW&ZgfM)V!BsdO6~3V4kt^t!L~1?o3~O<A#>+sUy4V
zO#)&c6+QQy<lR(%;o9N+OH6M*toE5G)u3^D-}*h=+Ppr=QePt2Y}1yseRvvrLP_eP
zRE`>BPDA>F+DY%~ul{h`JEv%E&Wn|KDTfPMV;@=1a%_CIB5?J>UZG5Lo5Yz-*D{J3
zW<LA8@8^!NtO}(jRUb3ON88^moFk^=>3ys(f42IHb-c+T^R}l{C`#9BJiV7M?QXNO
zu=~OU-8SVc1{wE@T3iwbI8JYtk+%HFb-<-9r#7N<lf-g`iz|2=R5rYvd(^>oc|zKB
zJ41zEpN}lg*%;~nT6(5+rrXrn7n0-*>?W^lP(Kph)W1!sAlaKwq`kO_&FARdITDV3
zYO88D`wAqqS8Om_rO=ney<SfJuZw}HqI3$gn~rt+n%y5(>*@P1S~C6DX3=C8SA|I{
zHYGAm)!-7@pFHy&`#0k}+u1Ho8`eektzRS+qx^5Bbl!S)4Yt?$58sy*{?jR`6?fa^
z#OU>G(h(_@n<gIHHt0q_nw0yy&-}^hU&-N{O!RKDs5zdHTYh<_?&WWt7LT?#xp95F
zxk~8d8PgdvBd46+d{{_w>D3%VHKS*lZ(dGp(o2Zt%gNTMxbpWM*DPZZXU1pW3>i&D
z5)z-PGM>7&W5=qSr}_5JUfau1o)LW~CBNo)#1XTdUd;#3>IE!bU>uRFJbefMua4I4
zNS341Ry}ji-7xX3&<w6KmbV@j-B{Mbsc~oF-o67)<=dEI<NdmWH&4G?vEuLSjQ_qI
zvxL|qc8EJ_?7Yn9t?{kykzVSp!ZTaij-=d@O;=Pqe>g;GCRg~Zv+`c^Q+8&bRyh1g
zLg#hEO_8$Yi&Qt~#$6OX7q<G&O6LIQqBLuclVvJN8}~=FiHPy?d7iy=W=6xUcSgzI
z-#c)8dL}4TKizYich6s`zB6n*&u;IjOwZ1@lQGa?H=SIQ=@t~ep`<nH>-UMWqFT5A
zEZjZg=5d2%8x#)Zim$2;;hV&F{Mt><wRu^a9~?W&)op&pzU;;U$7`>I&T&1y@jy-5
z(%Sr~Xi~ghas9{hudknTmSC|qJ}+=DiFx@=``^zU7!MV`EU}*#ZF^wZmYapsm$(Ig
zunb>kpzEu6Wcv-f$hiKm4vFkS;aAVPMV{f`e`rT>ZrdtXshFq`fscE-v)^4Ts=N0t
zi>oleS7o)_YVkQ4XBSSqaHpWSQAW1-ppfjlK*ui@H$1;@*y+$akG(s4)-9#d<Aqso
zWUaTJkgaY@o86Zt+;^-?|7>4E^o^<;WlK%ETW9w=<)|!X%1by>EX7bB!uI{hrvF<O
zEnT4?!tv)$Q{5Wzjt3D7u9i35*>m{4@47eai;KH#E1o~p<L#L7C^G6z(S8S$+<)t4
zoege!dw9~BdfT<v#V+osEe>i(KYVmXp3SS22Tqf-5|bLAXSiFIl)Qc=o%Y|E`GbWh
zZ(PA%+v71spHrAMzo&@#?byk9-2BXygR{O?t)C&a(8B!v)6JGU*3GNUO*`!)`A%@w
z)6a>1OQo;hX6`<;s{Q^yX8wuhFRpx$TK&Q!f06bMX+3tC3nhW4*1TJE-N@Q)UEyTq
z<0b3(m}L5er8?M_WTmYCe=?9~vh&+kvqIHVd8%EDvh?(P-Y?77-|VhE`@GTZElu~O
z(iV69T$6b7qk*c~gXKBA35ywJU0-IJYDP;qsJ%UUq4_Y^x@;@iC)+tQw$(?ltoCfo
z`1bAMMA6$54wSKele(RFlIu)x>f@@AiWOZdH+tflFRnUw%4X%WTM5T5EY7)CKYRPd
zWzL^Agk4*c6}B*F*O@z46J9;|wrRFoO7>FCr6!MgYY%R;t$5?1<jekX;{QkXyDP7~
z-!i*D`*Ka>XQ3C%TC0nszQ=BGVO_T&eqF+Zi}ROBi|)2ymAW%S*I&ygwly($#xXXY
zYX!S++_)=M5|b>S^QOl4om=RJ*&kQSoHw1Y<0p@^)|^=t*LY4;#qhaHw5BaMbxq2;
zRAl!dwFTnGxaY9knSM!k>5VmWuk6&Cw42Sc?V!t{c}e^~o`fE`nK(@`@q3WN(l0jW
zw02Kw-6FA_k8?inrLd?Sc2OC=nvM6Jj>n%fxUeK@g_%<6Q$3D?ZS^4zS8|pwzbO|u
z`5n_9r`2)yCLE1iYwmYitl7(2X1c7UO1GeVN8sAdP4YU%9%AN)c9p(<Q+}Guntg@V
z2A5LBVi!9eg)fgfj(ujibk=D_*`oco|A#m|_gsGNGv6ZZY?Ii7Upz!_Ms+9kt&y0w
zE_lA@qbIGiw!E7(J<@2U(#n~CRh3S>PSe^YTTu4<{FiEN)9TF{>PM&6O6jX|pHn@f
zqUN;mrS>h}o!2+r@UD|tuG%DgL&JA_!XNKTBIkNolM|;m$F$D-S|i8bbh2ynNv^Fs
zxF+u4oatxk&tRXZw69?Etdj=a2EGrRKZsX!H{Ch9F;-mV?;gH?6AO#Cy!^sBci)@?
z>OL?2q#j|}ReMNCp;B_;*PQ+}H~G{yN1l*nc-&;>-taWw!cD;^vP>Dl0hgDoPE~m!
zDt5X1=$9nNopsBn&D$U|MKQ>kVcLR}ZnsSf8)d{S#O~afv17&t9kaWxuJ2hGbkzd7
zJs&rfO!wdxSag$Z*G;z#mL^fUZyi#%-`qQE>nhiWQR|u-4(zk-YAz6Me-^}g_<G)f
zZTmACM1Fs`vDVgbQfam7s>QQ7uVyV0o0^$1pUE$F!Ssf^C6k3jPi=E(KgC|6X!Tip
zrNBzwPo+Q9TyonT-Yx6V78Mab5t6+4Hd|{B_bI1KcaoN0OM7%fpkLHwq5i>!$rF<V
z_wXM25m=hQn5g(J;)JP~%7>;KbJbdQUiz{oA#sVoiSE;^bN6U$K6B{i!|sx4=0AH<
zoLRCDa!TFH*G`<T#dclv`j)GSlU8?jCP&=XnP%9(<ci4Vmr*bNBpO5<*xPxzS%iP`
zHNV}#-W-=YxNeoOg@^dAbo+BdOqBiV>pzMO7Zz>0(>iOhQ@7o&^}aKc55E5K$du_w
zx0qS3jZ?{mIp@-kSVv5oZ2sFPZ@O|v&LoLV+a12R^-P>NmrXb4R{4x$ty<nZ9Y#AG
z^3}SOd6RB#+2$(2!jgZ(b8F2C2?McQO<v8&;2-XRhuUwo&T?7Mu;aipFM&mz7ag1I
zRKGV@-(9j=GdX&vLzh#$lTVa>)Skk_?f!?Oy$|30KF{Fvq~_E~Wk>%`h}1c)w9NCC
zr|Ap5S;6^fPiGr8rTN7!+b=L*IH=5FK3~QKR!%Dg+vi+f{$E}iJUrerZ{0E0Sf)kS
zA36rkoO(h;YU)p;`6nb5Y*h^t<EvCfLzbjXUixHBifq6pVXtZYcP6|rpDnmLp2_TK
z8s8$(;D;@?8=0;!o!2@zdyed`li%+iQJi-~DL822^$jI&?PNt8QzviCdC54@LE6y#
z=GI-TzrC2Aes5#*?D`%2#ohDr?X8aw#$21aZ%uD)q<hr08JR!a3Xgf{-*0z0%l7vo
z$Cj{dS{p7d+3S?pX7ytE+H)6z{8s3xZ~1H<7*+ZA5zDGA0gIe=++r?jv_ElS*`-cT
zoyALTOrHJY%Kc-_a+&2(^DAs#9X0nk-gaGScN+8EcRDjVy4YV$Nc+)g-Z7IWZ&_+%
z!{Niinp(@3AKY!@8(frm{>qBoDt@Z%D@6Esdv7dcyKPX&5qe{qW1{nMHL=O2?R_8P
zcj#<-ClIH0n9Y>Azc_ZTllmpjJ<HOsZGLft>EMhU;Wo8C+n#jpMQfkBKG>4B>taNn
z#L_snV)m0kOcPTXV!nAFJ~C(Jo}A<*F-@mt`#s)w@^JEW(eu?I306kC(~`GKoQa58
z+@<ujHhFKD_`zd3HS<oWytlC6^w+Fld^hVK*D_Ye>#+hwdB08-tjxZ8pS8cwIDOlt
zLwT92zcWdv?4R<`WMXdEiFB4V=4)r9Da`w;#xI_@g(GwOOT$UWO`160n(ee+@M+oP
z5)BLM_yEV?ZncykhN+jo$dts0Ph?!Vao&tSJltx>pI({pli@MTb7vJ7pN4OHj`mvm
zWPbYmW3JIv^MzNJX>LC%`ecsB>ZFUI2NvzvDv+6GQc^J2dFmk#g=q%LJdrXlefa0h
zdUR^x!$Q}63U}9RoVa#f6YH6V>J^7i*?;3*`djU0Scrtii>@ZeF4ORYRYuKDmkzVW
zZ@$c<`pv)~;vl#7lnI(2gjz%umZ(j+lw)=B-N{X|^HZ$NA_KgHZ3-Lp{1RL_Cgfc6
z{}N);a7*ez1@nycvt!=g)x2&Lo_FL$?uS#nbye{{CC#}@rrQRbd{G?lR4J-m<|E{P
zB{W~D@9NDr;*muT9)>3t<~-4leyR*Y&C1cHE&6xQB$&;9Y4*AM?6rMo=ZVbvc|2`f
zfasH_(nlp@9!TtWNK?(OVfgV@gzI%-jZW*f;xabvo+SZ%n(G$)+PBq(<JRpOAG2nb
zvlAvOKaqJj?cHyyFCmjmLjruwyDmA%^=(_ARph_Q@boPo20abO57AQ=OV7UUw?k?|
z!{rdU@IxN6B*FqqESTr1y$O&zSet*`=IF*{LK9c9Xocm3D_mUq{9#Aniz1Q9v2I)o
z4>8I2EI;Zp(|Jl-`|o2_hxcuC2^I~`e3)^#OZ29e%ubfTV}JR~u6MoE+Sir4Q0hup
zE6eK%ti4Nu7tLJv>G_%|F9cG0oBGVJuMFvs$yRd>i4YX-<=0=eL^AI6HKiHX(r2*!
zTNY}x?5)S8O__%`sqO!IZPL_QVUaquSDD95U#P9sGMQSd!d2~kP5R#3s4w-t%U^r)
zOkhk=SK%*oSSMgBT3PA(D>m0Jzi_|knzeItPo1*5aWcyD1y9AK=b2B-na(b*U@l9u
zPCisD%DlRo^(Pm@=`Gif`rdA_*Vy>wdQiukq-Et&TR5(`GcgOuuKF>5g6uMD1^EkB
za??1aC3OT22o=Wti4fr0b9B*`pTbK&-uUkRH8Y;)<c$h*B^w|4^K(-sFl@fTy-j1$
z<D>HPmx`8GHU*xNo^z=u(u7Z6X70xMI;t9;ed#~MbMEZ@W6+l!>$P>+Bk9G5E_CdB
za!R6ee(1T^51Xd$&P&;4<G$58$ns@1hp5I(-d`!&J<HzQX)DNBu)E4zE_W~2)@a5@
z-!k*=q?j#lvt6@trf#sv;`tlD+&{j}|NXPeJ1toM&Nv|V%`E?UBER+A?fSnuTh5)D
zEVeVUbyLxiTkk$6YR)}tzqahAhm+3UY5XtWWb?gsmuUSZVUn;d`;eRQ8OuihUzh8A
zbs~?Poset0d&euati6d17dK4I2-=h@6!x=%r#ySw?z#03J=XmUS@Y4R@1op>w)64c
zzZzEmSsSu3xSKP$zj=?l<MO+<k%0zPFSv9~eZ-{g@^WuI+fuaVR;D}O^Z$wGqYlJs
zrN1~_C?vMx)F;vQ4RgCP-6FylaDJVd`}zmNvZcZ|Hz!Z(vdQq8pZVLsd&X>`zlHJ~
z+-ijz59?mprq6g-{oEX<!>8`8D{_lF-w`o=LXEo0ea<9fMFxM41K$-bbof7WUe~=<
zl|1eC<-nJRV<u|ko{oxonVG&X%w@(LjmgPut}D4((<+(WJKyGixS7S-cDSQM(|o0g
zr|RmBOPi8<i)BLk7}|?8=P0*6R`i$Wzw_it#f|huS9%vr;m+RZpeU~KW=H3y-F!lP
zQx4V1{j*8T({Atiar#~J0h2{Lu9W;_d+y!0VaIXPO;7pk9M;Ti{&wxtygBnMXB_*s
z=kVT$wgAK5Ya4P!dt+Hc7%R;k?%VpN@2)(+Im>>d(}69S`KRY@*)+LYF#OxIV><&h
zFUA&B+MhpIK9gTKx&1_H_Tkx=qYegCuHF`Hk)WQt|MJR6o%j3S?7pXCwfn^eE|>Ji
z^0Le4@?M4B@BSY0?pFMyt5@XPrzd_-Ic%U8v)6o+cG|Zqr{4YCFx5Zx@fPb7hn7Wk
zl_%y$u{BIKjAy=_wrzIg<FsYb-}kOwyTk0VL2~)*ldOB*o;@M(J168tSDUs}T5!<@
zdF84HTy6$FH*cqX|8`@}rrbxKD<8V9UKoAt!rxm8Cf1J=^hFP*^f%m!_7|AdxqUY4
zawi@RR;zc99xO|kxz{r(Y5rt$&mW@C_PssYBOqQPb|^w;qxZSR0k$reWB9md^n0;M
zKTlb$v+TRKQO~YDx`*B;=4Xe?p7e8>RNJKTp83<0ytj|j6n~j$XdN&*yd+JE|KoP2
zS`MDA<sV9I^L1mY`%frX-qM?H=*>}h#$o!;8`rB%%yu2ycKyctoas+?#IL<)ZXM&M
zc&ozk?6w5!$R8TF4)okmWv;lntGj>mX7T*gFk7F65fZ0XNy^zs|1Ww_{-)j6KvsUI
zSM6Jmk0;yWxudFPR>@sB$~>#kDV?uu<-u#FH`cJtj`@Dg^u|55-;Wj~n7_^F-zveL
z;>_?(?t+eMZq5Nmy&}GMEQ|6t7~XU}Vc=@<=R)Mp?B6mq%L;mqhPC`DeIF%Ub>Y*h
zgK|r<K8jrkD5z6a+qcHOJx^xMmwl(F+P2;Z^ia=Eo`1S6X2bj3c}Zt05;hk`80+1T
zDfYd7FOo^}WSh?>#i=<RtQVI|-8rk!LDVOIclr!vvuVN3>-GzqO?{{PG|ArUUXy}`
z;8|;*D@QVKw0xVr;Y;ugua>Xdyf^r*dHqdAr8nu~3;rGMlJ{=uJ(hdhxtnwTkIQ>E
z3JHI?o%DD8>uGEUl^1)dROL+O@BG5jvq9rn-&c-3oZ2lO$%is-Z&_L0>U^<8tjbi_
z_mVHu#&h!D3i9S(ez-@yao_zonX?Aj=Ord6EDFh5^Zmzl{#0dA_Gy*IPm{toakokA
zXD#!MzBx;Kp1;;AKT%<ojoUs{&*`c7!1LDn+c^iDLfLyC)aBBx&c%Oonh|H5))S%H
z{MBG?!8`tQUj%igMo!NuZH-BqcYN#e!rreBuGk1S%}e{U`{wh6&DY8{yKi}RgYzsy
z^=95Ctrc+!ZnnjoS_gH`rCvK`l=`WA@;z-uY1Jp2W>y;RIq^y|&Ss^>SEXH#gExN@
z`N*UB@(PD;ie&eS8PnHJeBz|aG;_0?cyV>%ySA1c_cvY4S}%IH<f&!;%SyA5`Mg(e
zx94y2DlB5~V5t^o4@@^T@h?nRoF;TnVba!?NudpE(kJa*p7OeZ{d%^`{;1T1<)_m>
zi+(uqw6j{WPSCW{cJ20yGmnM6ZjkdiHznX*=GEv$&&}s?KAm_o<=N3Iac3^1GL_Ce
zn9eujU|YuS=Z9;Yu5a9I_n`Hx;_R8{*%)WU6ny&;eb-pk!|HcElYaZZ#I}sWQ-^dH
zHWV)Wp&NJ2N9W9%ZyOuGN8D+8yS6Z99&ekE@Pw5MOWhBwopwyx{ES#$%KRJ0&8`Yf
z%L!#HoFZxXE@86F1P|re#-<B}_4wZYH@GRQBoSsPloTDe)Xc!&%*{VBNlx&tez4s3
z@<)%h$ObK$`H!b#RYcZ>mCJ6bi<$qhVzd3R<>ZBSRUWtCs;<Xd=C1E`iZS-Sb*9<u
z^OD@io{itq=YP|hm(w88RMWVAa__#`Iniu-y?Sm_au#iMUGRSIZfUm9<z9F7Q%zp~
zFp#^~JhS<vu<0Z59k0cvW~_0VsKP1~$9(TAC&%7kh1q?D8`wkhq$7Xa{8ieYxXm;v
zT<w_zkEnH&cGwlIXO%Sx+$VQOoKj@s{^EHuVZ&B7t_gmP-v!=iOc%e`-}OB3o5Qox
zeB-S1*6mBZ4qvG3{(N(@-X2bW|2e{wPR)opC-?Ddk3&w6W_0P1?eFi&ocYh|Kkr$w
zvbn+k`=)<aJy_keo@<}h59%@6wIgnKY<MF7<MWRCJl=pu(`!#n-FNcW%k$ShIX8-Y
z^-i67OTTMt<o2fyo)Z^!y%W2udn)Gp+f}zix2K+3e$M=E?ylQYBJa)m#lQcu|Cevy
zPj|Gw{mT5m{?CQx^*sMw(^vS-4wigp_2N*n^o`9*+dnV){ryvZ&xCo`_qIK~R`&na
zsns`2mkHTF@lMR}TeeKSNJsLr(aA;j-ao6|{4ZOX*Yh%e^Piv?U-x*e@|+!8-#X6K
zojyPMqW^Z0(9HLH%oAdB{-3`6^!)ASLb*BbCx(YaEefl>@;BzU-}I^T_f%|jwSN+n
z+^%}7P4wR%)n^Wp%lFCuezO0K?df+<%zodj|55Kdapkk~+uvEgD!pxYe4GA`AF;_>
zLwe7Jb=6(F{QlSG-?c?yY=yUlO#j|krrkeH$-kR#>)nNSCoWsMrOaJUcUSTM4;N={
z4|STu)O)QiNAvHhX^!2wZof}{ty>tI_WZ4(!T0_dmtyaG=Q(bhsJQXZFaGs&=ilGB
z&%J#9yW0_NyRY?qS{&;g7hg7K{}0<m7cTg1uU`6>U*F|6`#HOr-#>pBwfH7m`*WfG
z&S?3ZzTe*hqP0)Ysyf0n=hf^gt7J9*V9TPFFQYd;wNLYzev<p-q(kNBzF9B+b-(`3
zzO{ct_T4?RA^p$4vwu?BTFsODHrYC?ZM`Tyf5C}!FZr+D@&E3=*?528-scN?g?F5A
zK4Tx*|BvCl$llnGo!cLB&i}EIH{svOs!hDHQ?xcPu4d>KT)SiWTu0Y5L5J9HtEPYR
zTf6nOn)^EYJ54|Im7Rq+H6B;$swhdDSQndeZgBrSang3{pSm1L6E39RVzyqkP>thh
zT2X!TOAof+$~w2hx^&*$Y+U=I{q7`HL0@I<OViD6_x#;*Vs6a(<jPXp)aoA|YL!yY
ze3Yub32jMcm@8eBv7tKP>8hY<7jGRb_`Q^`J$R>3%7#@Yzqh8W{C8mC%QD^LhZVF_
z7KB=?V3WJ+Bl327_?~;G_vvomD(dcVC?ovKSu_7o*F}Y0Yx3lG<lV1}%Q16P`0(b&
zM!O;z>2HsEx2JcjNV(U(DynKY9P#S;mD{NwgBE-|c{K2Jr=@za(bu@Tdj@)YH#|Kx
zL;aP-X|AJJnl7CBrd4^egYUcY+57}|bvc_k>sD)Dw|M7p$6{OKb=k13^Cgv<D*^?o
zcKq+GxKXmU=c|eP^1Bbp(=_#7h-|tjc;(}V5?lW@xfzYuEw9aVRk|iJiMJ?WYqH%d
zUX9NebJy?o<#LsJvBf`tZQJ%{iya>>sy$+GuZ;};c+2In$Bqf#&5zt&G<V<X{jXd$
zna5<X?p+kK{ll?MlJg9s&AHWAHC7~fAKP@DvtYW9-HGel+#Z?E7x}BGmQ@yF+&*#k
z4*9)PoP+(^|Gi4T7HhG8){a{j{WjV+?71A{U$f^CZ|};RvpgUCeY#zBJ>U83KMcN#
ze-@c8;;A0gcEYclwV5|<-;Cv4XVll*r#w0P{af%yt-{2UaUY)7CT`Py;l<uzbNnPT
z-{W;Z*EqAZt3)p6;0u28F|9IGaNj?x*ZC*<?#>DHvn=EInc$ng^3Bhsu_DLM|G2Tg
z<b>%=b>n$o0;FHoadxp<??0};dHN|&=WAh72h*SAT>JKP^55>IJM81tf*yY>O+V+S
z@{+G$+WER&9M3ZY7CxAI;P0-2s$&+brm(q~UH)PJdE3fb#f|shYtHnpb1OLYH}6*9
zTMeaZXI;Hzf+s85PO9#>x!3u0vcPA@>fUWvJSXgmNs_-k*)H4t==>#2UReP~>@L@q
z&5D?_vu*n;$ApmQm(^FCO`piNSI+4?yU?<|y9{ml_RRYp!ZXKscGe{O-)H9S+FtN`
z=e{kM+V40`xiRm}rAeiiBqepd>hGm%-SeN{Rke*jm2vUWCm+7at!?H%zt*?0H-tCu
zx54Lgw`a^}RF=D+cIDfzSbrH=tv#3YcGRv=%Q@1yqnF>eEZNdbcU^qgtKZv`pT3#t
zr?X(q<j%7#KP2@QIcIN|Gma58PK&gWP}a@g_{}K%X@2+n%7<a!W_G5#DqO5JK3NbE
z{%c!x^wG#;O-mA;3;sBsnep}ERpzuv!Q%59!%gml=5nvSXT5P-O108nnWL5u^|A#y
zdcu#r)vB9z$<6!AFSmDHZ1WH7sCJ*|+~cal#_x5I`>S!xs<3<Wb5k!Hw3uXBm2TX}
z?_Zr3ym^{T+rL|XCW+mbTb6rb@53qA=dHeB`eXY+?WI;XJI)<TFkN)*^!;;nzn5(f
zF9`b6@FstQ`LbiT^-b@Y{EC_^u2;f;bg71DwQ&uvV>H|MUL}>?90h00_AKH#+Tqcc
zcWBM#WcC%|-||^!U1SirvtHS*;mw=FH_O<6r_5XHozYPHbZ_6eg>pW3BzS8ZuIFgX
z5<c0z=YjO~eR7jx4#`X^HL0J~8m$>uC9Ljj95DNT!aqr$h<)p4+}r(EEv9fW3+FD|
z)0}}xuWvtma;4^y`z%wo+Vz`OUZ3vYzgIM)Ag;UYvZ~ZpKF0#PqsRE)bb9EuStxhB
z^S!}SIpd6f$Kkh4^Edkc((>-r|90f)g>V_?&+m%D{zas}^{>g#+*3Yh_T627p9zPZ
zyU2dD`B9|vJ*jnXG==L*Tz7t0Q8jnt*%+Tgi+W$bii`QgRrHek&E)t#7ZbkUzeRT+
zH4sk7m;Aj}bVYCG4B1dC<4Hx%N6b79*b4Ta-)7@EYx^zX-Mzis0`vB+sJxkF^Xk*P
zs%y=A;|or`ZFc+ITl$DMjCn)MdB^1)Yu7AYZSmli@Y5gX3(xx;o_QfrBazueC2|GB
zx}`IucC>jvT4wY|_)DYBlMSX$F~al3<CZ^i?sF{jFpRmDQXT2HTk+VD--~S-vzyl(
zRm`7${rVwGk;OXi3v(v+IV&5Tc&sr0gk0}AMmF7*$FjOPTMk>4se0V_!XxinT6J>w
zg?DT|X-SK7{>9}?-Fh>Aan;7z$Ctm!oE|Xy*2-C(ak;Ih(*+r1E+(Jf_HcsT&4=b&
zVqbNiOFg+_g;uLf`eC03eMKq9`V{xglwE!3;Kp>Z-No-6buA|G-#u^O`zEbAvaD?F
z>Bk56&3s$$EB$=k#@o|5qxpR%D^JTd%X-bTD_r{5=ZaU?pLx#UiCC^+sdxOolEJl<
zxEUwCek{3Tobx@zT4NpWS(~Ffqi25eId>%O$<0GSwZ~Rxira2HyLIBT4j#+L3i@#t
z{)du;c0Y6zz2Uq1b;L<isTW4o-O2tA!oMF^tzP((nJ-LFDZh0=nzQbr2SU=XK1nXo
zE!{7tb5`{HgXS`G6|M7cRyelD*NEDfrfs`&(n{d_6U9_%LFG;6v$I`fXZ}^FicgsT
z<6X7;RAYxb*<pVD=^vvm^vyE1nejvYedX-HE6b$#s=i!&5VYzpW0myTf&kA4SJV7%
zgd7dNae7Z}(!;y}z9Y{X8b9s3FLU$QZ1Lj?`4`@}|B+0rEnr&OcuG)V;$^i(5}W#0
z?!Dpmrmg1a(z1=G&*uNS%%WabzxjJ8n|OMMOzLEgb!zFWYuCP#%(0!z`1;WMS8-G4
z9<2EN;#S1gIN6lOW1_d73(TLp`MspztgUOMXP*$5|H(Y!!GampX$vOJu_|*nSe8+)
zvPk`2_E9@k_Zgp7+!TG1^s*!3bXiL8_RCDaQ<zkbD&^%U{=Bfz!Fp!rD&;RLSMXK+
zz98oqcdC5N?2Emh-z@Y|yj*qY_S>D?#GAsttiBv9uvOu}Qk_{TvzaUBHL&l#dgH^D
zHRnWbXuQ|@l>W;^XkFQNler&MtyE<jZS%b%E}YLy^#2me#Ns&h&B@I_m-D>V;V-N@
zE_!dV&EBK7_Maa<HfNqHcsO<MrbW&gSDKC|wg0=Bn?3b~w~qZSg*9%w<)-LMn@?I3
ztQWU=$MPJV?3{0&ik4dze|mJq?alJjceS$O>p0E7FL8Jm{jhzTYll~z%<|XHjfOiX
z9k6&Tzc+7HlKqWM&wicxHhK0D%bkTyEI0m#hx5F8qk7Zq^_0C2%w8|CP+BHu6uhxR
zr=9H%^F5PK)j=JG_s_o0I5WHal$yZn&2w)~Q&sqOeQiaDdex`Z>b>7~+ui+_;qPC+
z(mAvA?e>>n66%}Refv1)W_S4E1mX5$PE6JOwbyTpPm7iRsJCA8`t#2<bI)3&KA7p<
z6MXFCCav>t7KAgdxiC%n`wE?{3t7#6Fa+#tmRs^Rs<`C8TEn4&p38qN-Ov2C&2zRr
zF-P=sO}(){*Mg1xd)^)ht(?8%PL5mLrm*(8H|M9llY8U*UFpl&$lt-TCuTjo%x$M!
zeU|-NeE6EDn)ZKx2F&*^IQzim-k;z!FI};DuARQkC!Mx8&5qn+dhgdJlb`dIF5ax1
z7O^~HF7q|b=H54QF&`OCvcfyhb}x?bn3kF&IqODGaNClp?dQJBY1BQudFO~_<*f&6
z4HmChc=yhP>QCFQf7yD}d~bgLSINR{wJB0Nw{Cyqys)GAkgaNs@ZRJ5m4)4|J~5V=
zl(muno$nf@!m8@&GPa)I*eX|8<_d1CRx>-i_wE#h)+&bG#&Vh7>DD?2HDlK}H`yO=
z^U&EZbN7)(LZ*{#=lj$@vcWU`Z+zTYyT3s~J#FoxD`$h#nXN9gg}x2_&Je)Q&X>2L
z`crsD9OFGr=~MoX=GH`XJYTCD=d}HtTc`j-$C8V`uWk-HYs+XTx`Jbm!=s1y`OLZb
zgf^esmbOt?I_0?kdjB^~S2S-kEAA8Dyi1|6@4NSI+3S6lhi5J?>q%OF<;~1rHmv_N
z-=AIGRe7qc?_I2^+4d`|kH^>PZY>n5+cw|Xxla7%))``+(&<Nfgx?=g3rL)sw2SYf
z-_LJdr`~Dq*nhz0rR`?tzngr`^YyK+ecN{SU**dH!N*fLk8M3-ky?Er?$w08Alpl9
ztNQ1kvD~{>^X<YT8=T&yWIb|v$Kp77@=o<fS#NJzB{%78m{6m{d$|5J_d%P&W2~Q#
zRCNmIJ+oms=X~;uuT@9=uKu%2bhkQBQrmVV$*J7Xb;kBb3oJUe+zFeL{hM3Y%;bE3
zx22Mr1J9I~=Etp<PO>f5>(DwP`G`k`L*bxPq+NiZ<HpCON;^7~ZoGc;_}Z-XqTU(#
zVz;8z|IKyHoMC;!VELM+x0kJ#TO@1Uza`?`tpDVPw+qwP#fg$r+E+YS7dtiMc|!Br
z>ylZ&Ou2QY%on{ZaGt|(;^&10%M*3^RQAr>7U64AE0)G3SACS<t7*e>r+N1b3Up^F
z+H}OsT4?$9T9~Ql!orKP6<>ImONFO69g^ICOrYgOmf^mf{J%|&yUb%gPJZ*^*bN>L
zg|<(p%QgonEtOuV9x;*a*e2!-6&9Wq7B`Qtd&I<?QLZ26E0nvwCENJE`Yd<D^eua@
zJ(ZOGz1pujcSGf@*AKop*XXNtovn#nf1lCujH2d>X<Hl_Z+y${jLy2+p73$Wjr}z-
z&)1f|pO_=$x-eklSKF?Q9cTZFmfhU8xYGIHS6?~X+6v#F&yOD8$oKB<hk3z=tnVrp
z?(p<=zN^-Gc-@E8e=N%;y1d-$pSeT3TDnSTOI&03{HL5JW|!Yin_{lM_4b8H59W%W
z4lI`rnUTXM)VRi6I@IjqUh~y=iwx%OzWdjagY!uCa!&Eq^GTjxuIx~<5!t&<R{Qk1
z#&45;Xn1~MU7vMAc7L2>#mo&|g1px@dLA}<aqZr@(|P-|kBPX&_OB`l7CN-_tec|e
zJ?HSRQzzaE<Nd`tH}dMP!#eqXt7bg7*%LBxMquLnQ;IUHq}pxn*GzGIs>vGNu5d1C
z$tg~eg`U%N`tSFMRC!kCn#9eYm{s+ctLfU8UwaOxEwQP*?y%LveN&UA*nA5k1;MjB
zDzDCa6x;U4UVOD5=hcR*{z*%_SWmj{neuGW5vivFk7~{P6w>S>r$_(0cE0DnO~}Tq
z?xek+ynGWoyIGbpi*Y6F(H1Ga7`Z{E^3sl160ZX@h5v{2n0bB?dF}Af)uefKnb5oU
zH?6;3>$_}pwW8l(;Zw<pm(qHdZuq!;<+JP~4cATDJ&YdiJz|sTb)x^*o!)Bph}~b6
zq_S1dr?kF)I9dF|($J)=xJhzHRj$wbP_{98yG_N*yTM%dPA=Xga%-c+KDkE=xSkt*
zl$mV*&dG7(hLhnerLJpcFPgMUSp1;W5$Bl}Li4ZuRpU-|RjCQ^eRVR_$1wKv>YU4|
zx4ypdwO_h@s*d{i6-B-?R{WkDyz2Q~6W6~d1luZRCO!<_dDHQ2-6KVFxyjSMPiGQu
zHCtw}Lf$-U$5dYbmAfaDty_^<zAJol;O_H9YUi@jvrAegC0&(f<qVpy+~<+THQOlZ
z@f0c1!oWTJeSc-QRv%k8-|Jgncb8{p|I~u1b^cS@AJ6!`Cdff~{)r>zA9<!qWPf*4
zOe))0I*sMt18#voZ24QKYCX`NsL<6clB*TAThr(9g|$ouPc8PlmUy?{zr>ZqmEoPs
zRlJH<bmj|z+YkQ+{JH*=Q>V(!ag8B^)A{ugT64E7)zfZU7_sU3|Bwdvua8>#3_iXn
z=sS9XV@_q+n>^RnFMIBpCe13B3{FUYyYSk~zdlZa->#*`Y`;EZqj1QQ1F@neza@6G
zu=K25H@j-~ze96**D#29EOwASS0yCVyLX!-lUmtr-}UdN%YI+3X~o*LKyG7U^Vf5V
zGq${R`y-mWOI3E+>v_KYCexM+TU<NJ$$Bq4sObFjnaa2CygzpDx7Nngc;jiilV=op
z{LDW3a{KLB&rbyZ*|J-?LxB6%+swP&ccnKpE;!n57<-!UPS?z5KMa^YUH&BUQX{vL
zUFft~%_)Yk`F<(vll9u4pUgSZ$)IBS?cqwjH@ShE4dXX{Z{4A_&E!Sax6p~N!r!QP
zOkekqRqIm3?GI^Z&65}TT11A;UT7=3|L?@Dg4Q`@ewJxF?Lt4Qxc}u5IJ9(@{6xWD
z=29K}hy8?98ejVIr74!&3N$U)ecH0JTTFx9^wu8#v=<k3<}&0Y@ZVna)|55%{J&SN
zKTo__f3_&{+x2IO$@Ooa*>gpErkACz2wpr*^3xV2wuEZkYpy!aD}<&W{x)m3{3`Qk
zvzZTVeSK69v5RqkGW#E^FnRW@6^XB<8{gUZO}k(D`;zpggH<)fw$9EX8;b&BUN+=z
zV&==+(RgrMVSi~`+FAp@>y?%(7c4d2KRHr)>h_Q)|J2_8<+k1Q@>=J?z@^_O-46F(
z<h%IVgQs$-|INBL@y;%7$rELDFTM5S(B^qd|4I0#U7lxZZ7_4H$o0*3{d1z<K6|)y
z<+QUsYNoQY-us70iRQk4*<q@rU-sH@uj<S=X``UmQ4_1rS4Yj*k>U^~n8Y1a8GFDz
zWoumWF+0n<-*??qn|7`1`tpFY;!N`@9xlBxDKALq_QUBHRc^}`@N8JaxS`1J=G)nl
z_a4fu&T9SIWOn)E`QOTWZWvxv;8@*Mv*`Jnsf)hXpURlzxvI9Ts>ggmNLbCCzq_Pr
z-M6ex|4^QtRqJu<%ZlBl%J!j)=TA6&RpQU}=@V3nBO;>QdAJ$7C$_wGUZS`ETG;h{
zNopdJ=2~ZVpPTV!rrOeXj~)e`xPBvhy{=Ko7R6VuQ=|H4eyI9gW;NkxMdhPBN8igF
z7Ps`%ez9Kdn|JPw=;DB@UCTa&9S}-Y>1#C%P<V3B=2DRUn{D&!ay0LqS5EMs8NB3t
za5vj&E3d~^n^!YRb#u(vl(W)JVe$>reaWJ`!x*LF(tjtc+7RC>Cte-a^Ksjj<2~-7
zjn}kJynoGnUYjjLzia333mZCS-pqEtGoxyOasKC=x|uisyuP#R{fo=7Ig0%!_bJw{
z+4(yuB}?!8V;1J{gX_%M_xvbI2-mKzeW$lZD85>8OKsb#U&(KuaW~v}W%zj4l-IA$
z-tmp^S-ZnW+~-x?tHUOV)vEiqF8ek0e9WmMAwkED0_U$uS&{TzCNAk=#sZC^8yD=N
zIp%9@ivIJkOE%oguVR+L|Cu+OooBcBZfW3B-*z!=BER%X5vi2!t4Tci4O^?uJ(+T@
zLqutLXwNQr&)JzbE?qf1X<^K~(7ie_#$2r>&pQowUz>91qL5&3YG2!){a;k~9(Z?B
zE@p|mpP2hzR?k~e-kY?heGBV+D`IGUR8oO^<=x47dP_I#nz+<nq~z0@wvcmT`vpC>
z-KgCYrywSGPigX+8<|Vj21i&;PcqM1Ey}&Vb?P0fWufo9PaH~GG26oUefe(YH`}Le
z)AZOlJ6LD`!@E<~NQrm;?GiK6?f9L4=JrpS#I+aKNG|U+pBaDk829&G=hJfc=Ew(&
z1j<~UE}^hQ_tvM!ciN|4d$)S?zGkkwuMfoqy>bwoFjG79o_o@N(XG=$w%GZeTqbwA
zLi^2EvG$Ih4_oH`l9;H++Pq=Ed%9};!MPXtbxpoqv;BTE<lWQxH)7hiuDKB-9JRFn
z$iFN}|4VGv$6xY_2=Uzf%~YkbErN4`$;^QB4|;kOBTpFEtgEitc(N_SJO22M!#%pJ
zJEJ}aZK}}OR;9DH=23LDNyqYq#eA2Nz5ESiZm?WuE@zoB)AnPR#g?vf&y+Xrzg3~V
zRxCZlS%xi4y;z&qT|;uNZrIG5$s04d>s2CWukM?;X6{%1<dwhnvc3Ct^y|i>9(vp3
z^ID#U&5?~RS>n5U<KoW8pX=^eTv1R~sq7Z^3E60tmuc5|xA=4Z{ud|O(zh1AULmx3
z*}MZwHe~C|*WJ=+KgzmWKKgEc9-GEd32$+y>e3Y_p2-~9AIY)JbBky*hk1IfaOh3x
zG7jgXLSG)dU(j+n(n4(w_p8ol`CaGsD4bnlwWE4d;&!I&7qu)oF^_)hwp=c=6|zm)
zeD0-8SWXFR1j`=pmO9JlY-UMP6S(ib=ZU<NbLQFETeVlxVqGWNZ~FG?pw+qWaj}V?
zR+;|)5%hJ2u1&y~$&ZY^`bFP3Wc$x{<UQbDoU&-rp9w}cFF!k+p0=f?ZmRmWjjZ*N
zp})M^-Dh`2cImT!xxY4QZ|BUlZU(CrBm2y26OA3#Yu+o7?)tn@HX^j?58vJEH8Usv
zH9eC3Uo2GMU-LWptjboG|Cx;|rWBVw-aBJ!_KUU`0fkqCA}0UVX+0Enqe{|*ZPk}8
zIvX_}sNL8iVO9Bb#(~|@%dVY~_pmnnk+pJC!3oa(>kJ_(lF>>}8vEpJ>v-I@3UAo4
z(dC;<u(;SOj;BS|PN7$sJ2xzwQ(&qs`Q_|`g<(4LqGC1Pv@ZRrx@1C#314$*;1}67
zJ43FtyUh8y$;M&#P4}tS4I4L?ZmwKr`hCtd^}Ft(NglH;uXG(ZlHL7_XLj<9_!Wf?
z;hAhk72a}rQU@w-^2kM8SQa-sZvV*%WySf`)9?IoySzKy?u*&?s##rm>0h>M`1-~d
zRe#Jl)9{aHxuX51_1hozFL`nBm*C!824-<Lq^jf6qaw^w_BJ-Hl(<wqvu$O(_-z0D
zp4m1lO80lNZ2YBQW>#O)dn$?j#p%bPDrMO(oDZCo+;S)A=EP*9{7=h{9KZL-QjK-W
z*^K9>)m`Jh{?X+UlAm&Vt9f&oj?}SF32!bu`eU{I+}xRd%kECq{W9Z4VB?bfxb;>l
zpXK(Ql-sBhHOEKb=$iRGt}#(xbKSScpHmU5i+#8Iy!^gDYMf!;*ehc7-wLv6hvx)N
zI%Vj-fJgkTGrt~N)92i3@5Ddl;gzrd=3UQ=YgT2;s~0odP-C;;?>?7f^$#U;V&^@c
z{H8y6P1H(dm4er2pKe_||9kd}U*T=*dG=r9vGX<m{I1xvzGl*sOG?~)_l~gbPnjbn
zdF{B6Z0>gUGbe5@{#8=*uIVt}{pdpfZBOz}?*H{N(%#idukc4|yT$M4u_^Q7wd=0F
z{^RvdZrk&^*VQWLthZf0dhPM`#dldecCkPHvi$!3Uq4O#<bS`6w6~RBo~}GM{^ys%
zIp5`<|2>|WIw$N{<=^8LYkMxAvRQlP@~Jb=!!2XyET38#H4{vHE-9QBcJ1rO>h=E$
z&;9)Naf8HBzv<UrUfQ*L3z)Ww-L?D_n3GZ`Yk&E#@qPRA9QNn7yn8P%dgp@u@xR6Q
z?61FE!?)x|<@V|Cz6I^9zjirULhSy#bltf#x1Vcz{rLL!-am=Pb4sVapRc-Up7s1~
z8%qA{`}gSYd5M|g)&J_w`QO$4uJcc{|H=8v$zAWy$CtX-f1ADJ{_1J-uk#$cDPJ`A
zcfS4S^D94yrzWhI$j@W^Uy%Q8_4xv`<JIhb-}&a;D|=9re(pQpocFR8_pJ-+Za043
z%lusKzTu1hLNIz*bVug-b({Tm6Wkta&M*Eq^W52*y1Cn4Wv<@#`Q~f>-GToXew6wl
z|K$41u-N6&Z&PxXJHIViF_&*^)vk-@Zg2nm>Ft(Fe%iTn179tlS{L<l`PBZ9M-yHz
z`%!u8)$+A<w_YvRogBQo`tk8&FTyQ1|KEICAo`A4^}kMQp(_{KHG}!4s`H1<-?L|)
z&d-05b%B>1Ki=Q+U-8(hUvqwn?|b?3QbpZ*Uzx-IGGg2R{k*pG?fHgKn|bzaPJd6O
zKdzUQ<cqv*FSzSzb&$uis@mCR`;V3XI&avl*-+)tQWbHi>P2(aVZN^ul)nZje=V5*
zHsnU$jfJ(|lK%?ApINp%cT{<1+5FtDS^i`3oY|lMM15R6pR2kg_1?SJ^{UMA%;7io
z-}+fvu<)k+HHq!~8JpiJE6oqxRCnj?_w`x-{w_Z!&~xSco1{N)CKP{N5ZtHtfqQn(
z`^)E_OP8O0yxX^b{uej-;799T{_eY6Z>s!{dGVdg??X?Q8y`7t`}l4>%bSif=^wZL
zw_)s`Az*K^_&d*>dWq2aGYaqD&#zNW7MlCS;3a#~dk8W=a*rPg<#+6r@7OzEcq;^b
zH<<S(*6VKJ=Pg10+Hbc!65d+A<=UUmpf~|X%)G0+yTOdVTQ2!)|J`$H@%*qVpE*17
z_W!8<6R*|mIiV<HR#C<*A!qR~J_*{NA&4U@&1aAAd6rd1`Igu8-)CPb_!Z$V|9au>
zS1;#0Up>oy@9O2{Z`GdoJ@;X(lbPyvVafN`KR5sW^Rq4ch5aAFTeXL3et&!WdrG2>
zw*94)>Uc)`;tkS;OY2_Vu2l%Cs|&Vw`}{8<_~mh<zbU%=tJ@?#y}xL?Ki+uV%oTz6
z@BAxi`^)rbV{O}mN20IprdXRF|9RE_<L~R0yB<&ayU_h^==94mmS?9{F7tk4{FbBh
z_RX|gjQP<y;<tlV-IaWQ=<^?|wO1~GT0Bp1rTreEb3ArWZ@u@vy-LpH+qv_ftgn6T
zHgd^ke4fnx$vUsZ>~7Eex8*OcYt}4o`MLYhlFPUI@1NW*@2=4p`bxg+LfyR;+b+o8
z*neuK>0_UbAOC88TdgEgr*!zITfY30&rv@_OTX`*!gJ7jssHt#;@7`SeP|!owpr<)
z@|MTDPETx|QvKNXY5VHpd6nAjXBQWnottLaXLQ5n>bYVkn`<l8L)uOSc59_Kug+Im
z;BI5BTCZ39Zq={7bMIT_=TH27&q)5=W|?=JOJYQSJ-?`&#crAhM(++@`11bpi~lZ{
zUrbL8xPLq_N&2~f6uYX-$Bin>Z5Q1wblLs5qwKig+rH=?8-MN3Ta3U&|CgF=%3kxp
zMH+~R+&=H9)!H?ePl1VDUm34urp^rm6Ma!qTo3y{Ti@S)`tRm@_P76To<Hwjd>r4C
zt5J0+?7vD&zRXu?c7I&BBKyqxe+9jBz4CeYhTNO=moK-6_uKl+DgPWK&)xnT(Qs}4
z&z<{<S6$g`ANI(2e`xuvf1%9h{-;QOEBL~!_s!hHaNYjXiKli(S<d)2aowp+udayR
zFieUNk1Wi8|M8A+-P!XE+tX#(YU@HezrA^>eXiqg_MG^?d%tV@_4D5N)3NW^xlO)~
zyle8r6YMARhJO3DEb{l<7(<rFkH5Z)+Vk(v4-a+O^E}^j4n6p!u(0P_^7~8u7QYwX
zx)<5M&o0D%|NZCr`@h&a9{U~i>NA_qW8r)Lm&)Ch4%zH4)i`?Iea^#Oe+A2W@-LrX
z|B*TV@!wYU4{klj)bCEH*jp9%-sh$ArXPVb|NHBhnp$7l{p_>;nx}h0{xQ!fxO`9P
zd$zFS!{R>G`|EyJh6}84|MSPB*8WB2tHsyy?YI4^b=!C^Smkc>pSL@g-@5hp^F5h0
zEh~3_KKH@4cD?b3*Twbwas%$Iy7MT)Typ2V=bxh67oBQ<o@iTq`NlI37CQ^x6m5n2
z{F+~Xi|@p)`1o1+*vr}ZTvNUms{UA9`)l8A<sEn2=N(*ae?+I&PVtbDv;Fz@oc!~j
z{}x_)&Z|51_j5}wsn0pa+upAIRQF$I{_3AQ4W8a~{Iz#x&+q4Z>eu~F&k^EQ>h+U*
zpug^YeEQ<5BfoRQC%p~+cfLBd>UEq~`Rn_4r~k5Y{=)3@qkR8#X1!qdEk;2rH2Yil
zrp(tadn($NIdS2gr>D2Cz4P{p|MlZn)Wi-&yOw|cyiaoV_jl(@?f)#fB((i?<o&O6
zZhoD^om=}Rc&}#f<~u=WcK-cS_j-5locGVJ-Q8Jo{ny`pmGf5S-;dt0e3AUj)9UBn
zE1vCn_v5Z=cjbR0_BN@XzaQ(&Jp1^`j}LS2`&245Gx^!u-TAoZQ0}#zx8KcQt~xO;
z&!qm%k?p)w-_QBgCoMGB_u|=Edm29c_<Yt|?(46K?{C)M4(+R{|5&p|oh?vdQS;xr
zvvvG4jyj+EJx%$3`RA1fLta@t`Ka++gY&pdPnB)I|N3){b={BIG8VIMxwCfuqbP2@
z;C<iU+J9i=SM-0$6WJN>w#2Raw(6f3hWGfkRi^)*_T<9bC;wy)cU*kucr5Qw@BR3^
zAGhTew-=feyX_7)Xp{O^DrbK#W6SU8p2I5>>>h64^ZL%cuS(DF?yRx5Z@pz}GVeKm
zj;-C^ir+Pl9~<SaktlyDcf98NI}f2p$0QdP?oYJ4zvlk?s<OU{FJ?;LE!O<DUibU;
zzaNEP+8dHx{yh2odo$<sIVS{5X2+ZEFIG#gRamj7?OEyh^4iH&eeI_NAI0C_%46|$
z-Q@f^b$4=)=53$n;?aMbXHQt({~tnI>s|%#TzRo%SGe7jUGMgWFn<kGx*zNL{qC2{
z2YXnrn7>om^*QYNuKI7)C*Ey5@%_j2;IGjOKV3UocfanpQU2p!zZb<`oLtV7kSF~`
z(%$?3j}=e-ug5)@s{8iOneXQd|L<8Hdw%=n{8M@Ve{ZfTU%tHG{MGk@4aaY5->o^c
z=HUFR-`Wzhr_cMkIPr#!*w4+!j#<2pG<^F{rMTAE;M&d^U)OY%Rh!K)^-yKfb30eU
zf7UAY#q#od!sX}6p6}s)%76c3ulDS&&XP?_gTMc6U3WPl^ZoAwYYtxaw=}Sx`Xt!X
zN_Ne1pZ)cFYkz&_^+~dO_kQyGe@nOTe*0KD)A>h*`t<XWx1L*mILou@e(2**-zT^n
zng7h$^7o5h6Dqgvj@Y%icZake_mpM9ueKyiZniw1c{iw|t;Ouel)0~!vzM^_J}$kv
z?nlAr%O6&rSiaFx^1XOfb$4Bs-TrN#mz=1yx9d2uC;sD|Xtn=W|3y!pe_^H4@~~yS
zayu-$qTb)zJO6*R&$E{i2l$GQn%w0$`^&`Tu^gAYZt>i$zBVg9e_pq~uKvnC<&qiq
z6E}bV8=9YdtKzEM{##Z53ts2exn29Xky&1DTj8d=qEB{we{uho+Pck+TPk1k+b`Pp
z`NVE{zl(Na-?zk>TAXN&7Zul8{;jI+Zfur)i^bZ_>}H7@_RpS?kzDZl&m!&G-PcbQ
zZkFvhr%{!k{`Z~H+dsPm_mtjWQ$Dk5)t+~=-~MzH|5mo_@40V&t~nhqOTGm9hri1`
ze*N_yE3?^!i|54kPJX$^!&?9SQ7zjOulN1_rgp#SXi=k}w7_ZY+NzrMk7CbDCyUtJ
zJNZ5A*zU!VH}b>8DzD9bUTZjm^<Ub3xu?c%GUdnbD(b&WF~6{U{^z%4duM;#>GuD2
z?f$x~hq(>+R!z#$-Sp-2>b<=u&Go-EGKFU-hi}ZQp18wq?`-)@{<}ZsR32OZcFi68
z<9j|&KQG!d`Sb$AGY<BauVO3h_MeJ9So8XxmE^nlzkjNZ``NvjIXlbbwPkfj(UTw5
z^Jd>WGWF})E5FywIqo^>)V(Ro=706Cw7<wElC$dZ)W=&pXUzBgakPK-_J95PPd>ls
zik~k!|9jUR$yc!tw_7}GDtvl<t+njo@^q<k-G4j(-mh{z#i_7<!F>5;XR;qf%B-00
z=KEb;{<Ldf)0uhOKEG8rW;I)~mp^L$ak2Eab*1y)hpftPQSYt$JhN7IO3{)H&5|z{
z_`b}1c`0ql@9NgaH%`5efBNT_;SK}l+L{e)lE=+5zxW^d`*qvCO=}nV-oARf_C{~m
z*{{zYzkXqx{p{ZJC+zH@3v$1^U3`CAzGv(28fDR>iua0KGs4S4<h`$2|G$0y^G><>
zrJEL?dM_n?RN}jF-0MHHKCZ4S|I_Tb;{LT?+ve;~_xOG|>Fck5TPL{RDe5^Nu=f7y
z`7@)}@mqcTc=Y$rtLLWYJu5vQGE+O@(f0R+Dx7b3yXoBDb0FYKSO41j9qY_(CGMoy
z*Tnr7e9xP1W6K&|u(Gz+NzA_6`In&5zZIM6d|H?nUn@^>QH<Tkt<+yVZArC2ozL#p
z>E9ppe|F_tZgnqu#b4Qk550%G`wsJcKg?Wx`0kVV+Vv}!$_LbNKl*8HU1R<$f$2fX
zlg|fUhWLJW&ni^1{iR;{VC~b7C#4?querHt`R2@%=I-HNzQ%q&8@zZ+^`-hC70V+b
z`wchlv%65A{_o3##F}XfuNAjl@xLAUzfaBCz9u%_-7)d#r>Pzv?upt>*r@TXZpMG}
zxaw(+*4d9c_8+_b_}AR26P=t(my}<f_?~-y-$p){8SicPshefbf4HfAdi?I`PqwRt
zFJ_suUsYZ4oc8_gVV|e}T#;>DRUjMI_;vl}w)B4nu5LF@^e4w#_j$g1k+ZA+`>pex
zLHli)_AOSk`}H<bZ`--jgj>HSS_dm#Hvh5FJvom3(ypc%H}-$}S^CL#-|9t=e4l)=
z`}R8U{Nif)UpDXF-ww4cxNQ;r*J8qx7vIZDuivpNeKbMi>*VM2<$vvec5GMS@x`|n
z<$iF`XT0(99pgqhMYlPxKVR>beL8K=4>!yHd;RyfCK<nZ8!Pzaj^y|Ahc(&t_Ho~D
z-<^NuMsIVsrIqQe8@K*mzF6kAKl1(V<Xam*|Gm80bN}nw-y5u62fWq3+rQs?=Dgqq
z_4__nuI6s4lX1HH+TY%Gsbnwz>`#B>j>#?gz3b(nuV*L9SKmx`EXt|<JNwS9gg%jL
z9p5w@J9>ofZf9Kleb=|0)%F`%|75h?knj7Ex2@k_?(O83<#$j2IWghZUe2EPlNQwd
zaQtJv{(beO?xz;@KT?wzuU2oY{orlI|6y^u+m1^&*IcWf@PdDD)Yru(rLxDRS92}u
zKKkkHp8JNo-^X7ox)FK(xX)YbU%`uOLINASf8VqD;<eMTwaTIMt6u4)RWr=*%kKF%
zoAYy|%E9f+0;=vNGT%PCRPOAu3tk%X+ZXNY%X5ymUpDo}j{vn5tbc3bTdO6nI?LQI
z`c`@5@@HY~2TvESt(%))^MBUCz&_?BeeX+OS@67Qx0b!`u8`qBGkWp!<}W3aJsyj!
zdVimXJ?UeSdu+y*&tD9l+Dkv|O@0@B%YE~|y!*jM^H+-A{xJK=rLSyvKI<OT3;F)*
zjLWsxvwOBZ|9Gw-;()t#_xicYALgaVYfRCb_V97e&l8u{H!qUO`eV3GS%3e&zXqRs
zzVOTGKAzFmo9t|scY5PR``zvz-u!;E-Q29t|JItflTUql%xrDD?z`moeQqD8e$ah*
z$Ns&O{hEzeZylc|C%3t9@5`CT?)au}xFy;8$y4U@u8a2N&o->q4e9@N+~;%G(U{BT
z|1^`#>UUdDE86^vv9mUu``u&r57+-KRCaTJ87Po_|LK9}Yp0y<<u~f6deJ@0Y5Iy~
z0S^Ta+MKW6|NdRny!M2-&reiusg|glah8ACt(eQJ<flIUKKcEtuzQim3_CA~eNe7D
zb6(N-zV!8l9}mm&_<drYYPRnE&ZeK7qF*&veNjIvlymx4`i$w{%6XR8J=#^@v*Cxu
zxiioH)vf7!+?i3saMXTc$md^jHs$9({qzetD_oOs*!STdkMyg`ZR?IHY}@R<#ecVZ
zZ)ka|LQ#6w^XnJY`2Kz5s&bV+bkp+n$s6yo&Rs6!y3Q{p`1scurQ>(s-^lKH-SepZ
z?&sgTi>ib_7wpY5{rR)AV1tn3%I7*3vzVpVNw80w!Co?heaj4XRkLaP?GNtCzbC8p
z{CeioABX-v{CO(wa{jGn&F9WJ{JlN%)6H~#Ho;@ck0p}(i*Em^pR;@2(+#}k7waPW
z>TYmPu9(dy`+c5;zjV9#%;)#+9z4!o+jLw1)VX_hd+(>0)ZE*DNwe-=?w-=s^{ZaW
zHB_IuWg9f#r2Bd0>bmQ7{2yz#SzfL^HFt5ro!t4FvgZ3%PnL~+x%`@ZnT(wL8~1Hj
z<6DA@V)o7~mzDkcugyi!;PC#-x8H7;>g#%bapmhXJN>Q1&9?R>&HrOE_qT4}$-*Zd
zk7X8_Kf3$+uHo&Z{(s9RRzH{dUS>1zefodHA9CsSvg&E4U*uo7qkQIBU*pdg=`VW%
zZXG;eV>dnfcZpd~@N=uhnx7BmN4~3kaeCvshnCHsew^v(Km2D+^%J{X%Zp#DC4PUt
z+_Ixksr+&J*`>3RvR8FT*RHS6x-ZEmBfym(Uh}T<?yBnR2VbpIUbXX4@L6q!m9zS`
z#T@!}!L@JSn?2D#7cj33ekN_QOwME1vKze5j-IOCv;S!9>&&`2dEYMYJZ@2Xr~mfc
zk0(z2vTb^%@l3S(!@A#c*O)hc`F-7H@wK`c2g{xv_nrRj{cnk1Y1W-@_MfOcFWLB=
zdBMD=X4x+;UHR9S{p;98*16BGTL~?TUTqvR^~_`MifcErUa5b|xz^ye?bdyD>H8W5
z*K4+aYi2vSb${pHU+JxPO4E1Uf0l54Be$L3x^JHCKbrq}y^V<5acK9e#az-_6TaqL
z{CZ)r)O_XEx9q>a?W*Pd@9z6vuj)lex!bklnQzng?c2Dw@k;%tCo(tfE*>uY-Tv&g
zzVI2>+~t>FPd@)YVeO}O$&L5i7RX&~{`^kc?!DZC=75%_I6keS@XL9<XQMMsbJV_k
zo)_tE_F!#VyTe}%<FgA}u3u8WeeCPIc*{?6GEbQmO;}&I99w32{_g9`b+6}Z{c_m-
z=F|5VHP^a*m+y18jEJhQzQ5gYe|ltX-H(dEH$BPwk3ZYZHAjB`-R1d`cI&?%@}7Ss
z?)Po?o$ar~GTwgwaK_kX{_|(QFVrxv|8uFcn(dR^=76So?>&yIze>0oab@+smtAdP
zDOaPupDkU?;6C-r=6u=po~!2-I;g(-?6F>|{f^!{-tw#8XPsLg;b&a7zfOBq{u`qy
z`=7skWqI#M-Be>PV_C=Ry=T|G_c~_NdS!j{$@AxDo>TbR&$P~bfl1WP-No-(3xEBq
zY(D0$UU_%^)1E^+M5FJX`}uTn!qouwJ2&$S5B&&J+ma>Fwwi78E|q&VH7|biTb%k=
zrgZlGZJDF#)>`}XSFenD7+@v0baAA#*6M85uD#bPmu^0icl_$?Epqv$Uw=Q>-Lu8>
z%-fq)+W)>q*Uh}YN$%>SkKe@H?{43|d-=<}Yt<(6w%YUjyf-V~{#N!;nYVAJNzYYY
zw0*JWSGLvrA79F5T~h!1bKGkM%WT;zd92)<ugyMvb+<<Yi|pko^KaxVI+edx@AvF^
z_sj)09r`})^XJ8JLIUsqS)Mwc{N&ogfR#0Mb~%*{F*4=Om$$Y&`&z-gM7}V&?$ven
zf*4ubU3<S)a|OQ$oqOr;yAY2PeOB#Rzjg9<TT9;B9Nw$^>)O3bXMbP4r}InhKHpni
z_k8znpDs>`wf%WA`APNqUycHsZ|*7!nQ{2mXDf5Nh?0}{UhO%|>AYzE&)|J8z6(A7
zXR$r|PTXU?KfbG9uPuN1?(W^V-^)&1<f>??UH@sS_3^u30`Kp;`H8>!NksX|FB=u}
z|MJ`JeqVP<>EDAN?uWlMiyXW?p?3b%j^&@<P418M%(}np*WDi+taG>X_gwqaV*jS=
z*hL=szhO-8pQe1+X7$@WSI_oUdHb1)SI^g~-McTf>$kzXcYC*PKKD7gdcC~p*TVm+
z@;_(Q*3Ff<`7GN0b+mnbMfM%m_$z@o()QM!i(dTan%#W&=DTyw|C`$P;Ln!Z_e!r{
zul=l^qHF!#V*fME?=SL8_k4XFeJpGG?*;ew^uOZo&h5RtJ$c372<5&-rtjj8m0In8
z_5Jrh=IDvH&WEpvd()b_E7<<@+=tCO`{nkDR_?COzJJ_p%D%1VcUkvKRPFi87u+-F
zT-39Jv!f30s{c2m?cvI2zTqq8%<tz<;<)Iyt?$wI-+ybuKRDjbFE77i|GS-YKQDXI
z`7_72ud3TOwQsj+*>m&TeeYj-KmBe{JMpftjMVepaY^@<l$ys?$XXuy>ZvzPNo0$Q
z+V-f!6Sr5KuI0CWdu!@KnIl_j=d9gv@!vM>^fhG{^Vh0vuPCpXv~JDopKt%ydnDJ{
zyuYw9DqvT!!|k8W5p^6t*O~k6Z8##M^X6LE7Soya^}iL(qU)3e8X4u7r~myS5&e5l
zwz1vORS(y^WN+P_eUxwewVU6n{;$}eueG#xU*YPd#}3xzU;DoPdvkbH)VJ5S(%(cb
z-LrG!V=IBelkIgOf2_|v5?dsEsmuJ-nVm^1`>Lnk$m;p3dsvWT-OsyahrHeYK3t$S
z|Jbe1?{?cg@2Xhb|9SIV$@u-@sWyL9LZaU<{=Va|=^J-<v$EACJ<4;V|Gqo5$9nd=
z`Q_5?b!8qC>=#7+z5V)&`N@-OMa4c%<NxU9ULRbkeLdP{_SwkwlQZvqn))SiQE$3M
zSJmWzyLMjpY>u|<=lPpc{{P;Rke_eM_SN6p`@`;z@XvF{gp_UXo1CzF`d8&i$N9VF
zI-k6!+L!-ayw=qI^^=0%y9<BER9su^Y1i+NRvWO%ZrUf)qmJ9ACSRMls%3)0-DZ*N
z?tOwOF~0dyuaawDoe8v=fBo*-;NQRGGXDlE@x7|9G0(YKp4I8rANBLkl7e^dY+lWr
z@zs}0{5pHp{O1Y%()0a9&lk^?&p&=jUG<PgP5JBJ$36%C=?J$wboTn&R|@i$3ID&`
z{B>KdP;1Xq!TZzMYNl7@{ahU6IcM$bnwO$qs!!)T)L9?>&HKM?ciW#pty`a@#X}ul
z*)tdGnfd))e%4)o{wu4Q$L?<Y9lj#}sd+xb!OPqJER)>%PH>0+-_reGGndWVE>}MB
zU!l+AZ<*;bp2z<zxOtT?f3v)7=1CjY-#>o|?##V*_~rXK?tBH@%E#`$*>gO%R%-iu
z_Jg0S|Fvw2oGoAZ-7Mdm$NG1ROceXc+B@$`AK#q2x^|tm{Di8fzw`dj(Y<Sb`tIi4
z_g)7#UU@(N?Yl^Z{;#~B;@L&_xvnz2{_HI4+Fy2_*JA(nuD$>G(PF>8o}iz1wtru|
zNnY~7W8LHad2RRq$Glq4d+yzY=Ip&YGa}fZ&1DZ<#~x98<@Yo7$ulY$u9P}Fty;W3
z&TOy!|Nn3HMs9m!u}<aJ`i(!>f2aKuu5LFt#BgbAxl2mwy<My0)}8orRyk0}bt5BB
z!$XdnvnEgD<NJ0zNZL<N?yRKc*ZX^pety-p^sMKe1+%7{HM_U3`1hZ;>6@jCx7OZz
zwPXG6{dc>kfB%*nlsRwVjK8;!Mk-w2d#ilY)I9Yywc5;$vl&mlYQJ#OQrAJz@oJjC
zX?@7&`scEp^UNl>mt>z)`)CkuxwEae=3}b(a?@>2g@@iml`&;I&e^&3c3io^-ovgD
zZ}N>w*l)_@RfSyM;MtpG+R<v&>3_adKmW^5LoGq^n+ztcT8kW3nFlvX#hdBgzIkuC
zWK89acC(*%zn0aT=ik_MxIn4n-|PvBjl1k$zq@DaW*uwGDtpGJv~boYx64wOWm9-Q
z-dj8GqTj{uPhabv$jguC7L~bfdH6@$;jgm}cDOlT<Snp?&zP{!@f2TJcG#Zga+#G!
z7rXU~?5jxm`r(H7G=9tG#){hY_SxretjP<yvQ717)0=&{7S|pnFRz=!_Gj~tHATPv
z+_Vmov+1__a<(Y%`!x5kG`q(V^B+Dt{`%G&M*BS+r|V7b>R;dS{RP*ji&I|QnRF$$
zV#@m7N%P-LnA9kpR;FpY@B8m3HOqc8-8(o#N!c*i_LJ%V<zCajf4au!8{M}<Zto>?
z8HOX1o=n+zo2g3e@U{%DoXbBK*!QvsJU$?J_^s;iX2Ye&`xnQywZ8uQdx~+%iu5<>
z&zJu*WPSbW<VpTo3q^C$w_%>PQ)i!=vGneSCBK;xi|;#zFa5vZS9z=VX<-kq+0MIm
z6zmE=?NQwPZh_x4<>{-6dfq3u_!JwgZvNw&{O{G{b%#zETBQ8&k@noe-#^(RvBLOb
zaYfddb*Fb6xPLNC#YlU_+STmEc^B)A-<{^znK|>#N2bc+#5Z+`9Car5UpP5!^4WAM
zMkap8<L8PIGP!HtiYd*L{eJhf(PHlB-da+YKJ%WtFILiSTvzqZf73s+-CufJ6mMIH
zg{}-gxkBLH(b{|WW8U`$h37lHxu8(j9lzj}()knDWU6)dy)NOenfoE$H{0#|PPU6D
z)onc<`Sl5|T(Uy$?fsc|;!il$+Ph_}Y-TL~vOmoEf?wob@5@_t=kcz(wq^IqcHV$n
zH{TfM%nf?OFJ5t&dy?qeePsp(3#+-#OJ@s8ZohrpRd|BOo?LguwT0XNbKRV~TsdjM
z`}M9l>ziX3x7NGPHohCX-k{~<A(rKaLc+^AxN6s5ijDthIs2fpMNh5%vXi#eyV~|z
z9GexpNJ7`8)9ixR9}B0nR;I4CqO9H*b~<E;R?X7>7qE!2;^G2x#rY4yl3!@o)*pU;
z#3gL)uW+TkJpUQ_*6n>8Th5q!edC^&|18#3x!W;V7O*^<@{C2N*>M}A$E*B^g)z&R
zEX~(cR69*xwyoxd`vd*m_e&cNzS-HgXXoR{>wlhH3Y=AZbKl*o@mjlUY<IO^wpekm
z$L;?tk*yIEo<%P^V`%@}C+fhQ**kt8WBIOD=ySLJpUCnmpJmxvzI?xD?bsG#Jd1n5
z%{6(J(u*F4b?Hx=Fsmb9_1r2DIrC?FZFb+I#FKlkSg0;Ld3Cp%aJ!J{vr^r_8^;qe
zob&i4-QSg8P+YgG#f^9C=Gk%=8Gm%Bb;j~Nn(VvRcVA6&c-y+~iPonZ)0Q2cxi~M#
zrpN5*8P<(g?8<iE`}gK*i0Y*O7o-2A{+Jz?tXlo*;eBoAn^B%eJ|s+>ee^<kV6xmk
z1^I#&-zz_#em?$NMSGLK$lT?2AyfB7&HubN@yf$#J<lu3%EZm8=j1;Oem=cplm5Er
z`{(EUC`|cpa%fF&#H!>ktYZ6iPx|omTA=h?nW)9hdBGaVimT>YygDW<efa9LKNi~(
zkMpg0SlOGX+Z&fUIs3$%yeXO7uUUm>)PE{Fl705hDa)@>!KRB%kH0jk^!%Sb^X>Ie
zj<!%+58HiAKUXc;QLcaE%U7*wY8O3Kr=MMw)5SUe#tW4Nmb}wUuS_%XK59Mft@Z9U
z*#*hVy0o9)XgJ8mQ>`U9!?E_gIrFNxWhYGLOj{w5w?pp*lk#(E{&#-6ITfE?YWA+E
zvk_k{_c7G%!r9`<(eExfuIK-?`SYU8{x`S0AG1wtul-gyOG19P@n7@&w6(QcCN6%u
zA*grR;bN1`*B2+x%HvyHC@(#|L1l+V{)ZhoT~%{U&%E~%zO9oNcy*u6<W*J1F9qh_
z$#QRA^5Mqe*O`p-??|eDzx?!+CFcRZ_T}eZpK5mSjeC<De(>Z+;iUPDHY*k<nyk0|
zAouCRyhP2q+@i8S67~%7{QFlp<SA=j+w5v_&caO7DyA%cN5SM*_L+|pa`bL^{Lg)J
z$jkg#$1R`u-+3;ows{+Wx~RWsa`zX1{CnD?nLF3k#%*qz7!t4L6Rf2C_4BW`^Uggy
z^NoK#U$HA_N;JRd$_EB99TCTFs73WoZk=^N_K}gNEXUO1sZsM23tm2du<ZE<8(BNv
z?N=KbrsywU_U6!v4>NzosT)s!EvNt3xBp6V)f0Bd3ya^|{46-x#V(b2lJS@M;kWAy
zdGjo`O{#giOmw+%_?><4gSPpv*E##vS?yw(?N4`qH%Yd$U9Z9={X}<0Z@eL)xTWD;
zWh|f9Zwqs=dt1B;o>=~h3(@s_`7NXH#lfWIza#f%+$jEQIbUtJi~FXPjyxtlEL|xp
z=FhXdyjtYVoS%g{Q|;;}O+R3I@K_0_;g1@%$cUt8zAN0H7HWwfFp-}WY1d=E`tr`X
zc_&|Y1>N3uTQ%z6iIjxR-xPP(7dCP4-Qm1&cfAHzbl!j8Tds3N%<q)?T`<}a{qXYD
zlZSU5J7HMaUifQM=f|txlDBf7e08Wd^!ulod(vJ`7oWtGxOC?Xz4N83j_apBIUadj
zv*c&(NvD;MzgHW}q<xp0{Nk4Jv*SDWym%P-VnNVye)YwgrN_(OOkcP&llx8Xu9BsX
zRh=}%r|<f5+pa=k`AhTs{GKBpdquY<9@Bldt(U((K7fDws~G?4E}bLq*L--f{8`#f
z)6!$v>(cWAx7fT|kbC&e#5wP7o@mh9)+>5^u}J&8a}%^H|Gjcs^d>s)#S@P=KBk^d
zYf9fN>bjn|qVkGT`2@in&gk=o55gmJ*k&$q-~07x`{Bw>Zd(oO7EB8*EjpMUClq+<
z#WvO(EQ?m3dK;g=O8C1`q~V=Mg)8->IwUiTCMqm&O#C**R_MEJ>i*;3YrAT{cxTx#
ztei8kJ6pWIzo+?y<NKPqGmoBcfBfv}^@QEoA9lYix+y#3^oxnb>t;z#Rb$t!ee!(n
zPY2z-PaNO1=B$`{Yods$rLyqhlNZasPm&jCvHi6D*RKx_i)Xsu+`h}$r*yTs#dfnZ
zlO${Bw`ZB&bTXLHRWmcr_^iXJwO8!Z_=BugJC-IkOih&EvfL!DW{vRO=U48C2X=*2
zEj#CONcN-C|7RJ@GtYGX{4aD_b;7jAnx6X$JgrPpIy1v`p1+lgRTKKG`D<;fx4`_U
zl8v70CVvxNe)j;6ljrv{4l=t`tQIWUzGwI9*qifyaI@dc7p}N+dch7pnf}b<x3c;S
z&K=sP?_G9qs+n{4mDgOs&tvX?{X2J-`HMHl<Art>+I_OS*fr(J>+Bo7yDu*}W&Ld9
zt@u~FIK<XPtU0mvL1J%>nw#9^bvsu2AOABa=47(|Q_njN{OnUjvzE@N`L$z@_40Y@
z&olTcPyCux%Hh>oV{g3qTAA<6#t$XmmYz8_^Jm6s7NuWH<89s-bZg({FmC@lk>|?E
zx=D%2Qn#HH8v@k#en0DeS0Y(W^Wp0ylYiMfW_pz(cl_m|w6A_S7Z;ql74fR3%$m32
zbIHS*M_+%uC6{!qSwA~RXSwcXzxR>rP2arymi7Ky>%Yt46Wadd&5wCE|FLN1Vb9ME
za_n-~*3Fo9OkjiCc3(rrmeX|?GwLMb-UO|l93H!MUhGHjP3)q(`J`IY7e`Jxc`SDK
zElcL7Gp+QCbyn@}5nC?GKTB7T>G{_w32~K|ZLN2ksagNYjlMO{)PB#-&Phl1oi5m<
zKJkaq`d{r!i<Yw<Tv_G0@@mLE+t8K`eMb+qhwN7S_$Iz)d*V5R#T$)}xNDhM%e?pQ
znw$JF&?4ES?p|V^(c+KReEX-`+9dJYxSjSf5$jr07hrU1;${6ELd6}S?C%zSKU;U`
zx%au!cZdI~s<Jw~cyfyUxb26gx|o8E@0=Sg-e?9rFA12Z#m2RvvhTgi<Y`xKzn^i+
zGKJ;wzoR$3Lxm4rF<Jj?QJCGt1KVvPPo8?h<95DM|8hZRZ~3lu-=;6<iOakEb(+S!
zD*fDL>B4jFuUn9tvyoxf_0xY3ExoEQ`Stjm2gTKz56_)na_!rs-`ghd-o(CZ?ejO+
z<Xb)8oxSBOs(#)_fB9S=p~J2Dtd*K9=E@FjMh<892S^&2@AA9ip!&_`g!-MAlW*>~
zIbU;Oy_}?0=&FKQGlH8Qwj8Yx_A0XUzvm-$v`1VZK)+;roPyKdWncN{n*=1UuuDE6
zcXskrrT)K*tP3Zo2yb~*+qdb`j=yRqKX&>ZFa9Rnaw7543x&HcOl{T0-(H&|80UNI
ztfcJwzI$oh?`B+naP#s3$*#@bt0(-<Iej%$zv%Ho(;q(5Rd$Czx9rZnB(tD#b;q3#
ze)ktxx4c!km>bo6J0o>X$F77Eat3cRp7A+&O+WrZJmlzv+y`8@0^*7}71&cIhQyy}
z-4MM;q|bEWGT~Ev9T`?$KmFLtug+ZNpj^jz`X^7<nVUV=y|!Ig*S~MN*dy~hPnPD0
zdel1XzxHbGoYLdfZ@T86)j9w1pVYAnncLn>yK`9i_+5S3xyLVkpPQ?>>Ee#{%4`c?
z)^IUT@V<0jCEP96cUExTdy)QixA-pVrcIE(`th9I{oI9N%j3c$U5{#sWX8<mn!f1N
zB-zFLR%w@?-Jkm9#?<-TyC)nzHZxrI>k7FIUz@$AB$b{$qg7n=Ai-z0=BXLXqJ}x`
z&8xe9Uf*|OYG(BsnU|NZv>#S*VLtP8Pp0I_ur+(V)>l|OyQaqaeA%sQtInPaE~=9`
zKihkYnP!D?%VeL$MgIyG$=o(teKf~)`!f;YUGX*<*H^4s{@dBmmGe-BtzUWUM;E^<
z4clM+DLVJkv*p^uU-vHmX-_Z^og#6lBrTm|$x_WSw}o+{-mA)!jxEW2xAW$9mwl$L
z`^Ag6VrR*$TqpfAZE0?@{@RSKTmKiyNW^@c`!aujz@x=|Uw3{=-ckB#wyYw*TV_YB
zlj;uRLihBv4!>%l+y&f~b*Wux&jo)Pml^2Ax1J8)<u5z)VOm0(#V1$4W3JcU7%kkd
zYdZb%G~1P?%lGXzUBDoB`{-r+wK4Y;vbW7Np8fcZblxmeFUHH}esRU|x>A?ppYWf0
ztfZ#+^gfT)&uc3m^{naon0sl5n2U?i{`qf@J{P#fen$MU0GHQicA*9z-lOOJcC5`Q
z-0Gjg=f3;$*&-kHxdt-Zm5=lV`8?R6<MCyQrXKI&TVJkK#l+v+zM5^`NtHcrhZcxj
z$cQ#%UOqFBgYkz$v&Vk5AgjI?lL}tHsJM3V;OEw*rzTyG@Dpsw3~^KaT(i~g=cHuq
z%+U0Nb(aq4@UM<o_i9d8de#wD_K>hm`KKeRBw74l6m2TeSUcIyEW1kMe7@?ow~^{H
zf*TL97T>yd<<@jr|Ly0*&lO9iIJ+knvj5fkkPstR5fc8TrlgB?qgDSY_d<2S*IOex
zEEd+JG>e(s^4|E+woriOWxks&PyN^F8sgF(dM4c0`d`W1w3xB9Lg4I^MSah9zA$*5
za{K)f)py5!-@eR$^3JzAC(TYiD_s?FyYH^A#Vq-Gi&Km1%DX}=m|UtBK6bMFInmf+
zcKOGJcX!V%n-}NzU2Uh&^o7R`?5;~KOq}*D&OlkNe$|F6>-{XYfAmW(sy=_?XLw<}
zWJ7ni%A7lwrk=>FiSb#OwV^w8LqK!Z;d8rJ=X@=?zIxWJnX!9>7rs6>t@g;VHEy$}
zD`~#9TDmiFozlgdJ8N^E*&P?%y?dJ6hJ3~2uUF;sF4|hRc;V}ug*Ptm)GYbtUH&)e
zd*`}Rk)^-BPd_R0=JNrmZQW%TQ}goJo8m7$FV(J>l>2onTKVlciCq2Y*#81yx8)1J
zvi8fKjXL|{{t*#X!`T6UyhPUizH0u`+w;pDT@`D&uWNsERn|}cz4Teo%e>#VO59$L
zA1PkaJb1*~^mU%mUi;a*PDd`f@@wPiscYBnw%t>*&HFM-<iFI@=W<To(iY~Px&O|J
z6o&3TubS^??w@J?_g(AF+UUSmKmV?}MTc4Q6mMC5Yun|UmLwOe*nX*We?*X*a#v-=
z!G13{<4s$17Cd^eYyD4=ATM*qS-zXf(#`thPb?I_=e%h1(pQ<6UP?LcpK*MZu#RQQ
z1*iNOKj&V0C6{3QN8r+j*F5Wd%;Gkm>wP)L;_9LMH~4rDtnxYR>$K9-?_{dkyp}GX
zNw0IVmmj&YqS<!pjG*+-Ruf-bo)u=?V;4Q+^&Z)aAI_QGPPlcbeM>*{alOD}cHFDa
z`b;oV4GPz{WcXZF_3?_&trcGTuSmWtX{-wQ%VDhip2IRkDC6vbwv^Z{xwA6W`n>L}
zePtH1+&_N%`u#c_Z@=yM&ECwxZ|v%`(TPpqeC9^k$#D*!Y&#!>vTrZ7-oMvXEhhZS
zUX9b)8BcRRPCr-vz4M!lvCWdxCz~w=pLQj+i^_59S#Q3l9DI1ix|?O^PwzH1xaiq>
zZt>mAZ@NlK-)#9k>tX69{w{0z_2ym2H<_r%&QrT!vA4DQ*G8kWm#@E++WF|s*U9OH
z9DI*;#MpmL&2xL|(SP`Ta66OO{)5ZQqV9j)_;uYQy9ZM(&uFdk`TJLjKk&NKW^ZF@
zKmF~;e;=0H<|xZs-FNk_UZj2b>%|5+W)&ibn+k)smMcy3Wv=+#cKordsj63F{qefO
z)&;%$H%&Kg-Y(Q0tD!bGVqe7}-<sHe5092V+_CHK{>0*^lVA189d`TPz5Irc;e^t7
zgBa6&6};ij`i=5ebKitbd($jE;rxugPaiq`mesF3T$I03zS=U_^6|lPbG!Sp-|j{o
z{`7xRm%sGGjOk|<?A*Tp;#}nmA96d(Q&!*de`5FL!OK0#<$7<HpT6}%U8<B<Z?UHA
z!;%xyvCnt3o|l{QyZQEW4z;Jd(`xb$zc_aCcfNr4y)N5}cPonSM_pVj*mJfh=iJoV
z8~gZgFO`{NXj#5uWAYpJxf|6vWFkuE$rh&FmiU{(KkMbD%U*BJY&rFG*X%ydW8Zy0
z6xyV}=Z_P3({xBZXKgr>o~eNCq?b;8X7>)B`#){dgX{@E9@_kt{ApwDq*Ax{a-H=%
z!`Dh}a`lGaRo}Fn?U23t%VsifTZAa%%wsi1y_PAhsW~f)%Us*9Z+x9=V(Kd%eXBBS
zQwqz3X-QX2%~?%5JU2!Z>VH33(bnMhE+cTxfnt`75O=-DX_3xBou<LJ4uv&2m~5>I
z3@rG1e&OenjA=@p<x#9MX-yl=G`BuqQy0R0+-6No=Z?rri908)l2eNIKT^acE7rB}
zV%M|@2CFZZJhORw$13WDu;EVav*9hFs#>>hM*o&5&o-DdBZOh?+kICzy?x7G?x?@~
zZMLMEjJ)1oC;i{YIHW#o4PRY7cSi=(s@8k_kHm|9l(FpG7{wnv?Y-ZbW~N`C+$DAR
zesj1up820BFiCQ|sYUm*n3b{ovqaiXzJA8EPHe83ztBOmD=SwhYMt9^X)I};DN|kW
zui3e|Q6~KH-UTIX3v~oHUuJZd{UkAQQg^n*zgpAf1}EBU<aZjJuT{G2r>A!<Ubx%(
zSpRj^Z&{|2PSrOH?*v_8GYPtB8l+{GKI^*A@tYfiU)O4{miJpZSJUabt-3eML+SpN
z@#nIl9(;Isc(&=MjtcX2AL8_0*9U%3Fc<AA3f}1dOltRoKWb-xzI=G!f1&Mz1^VxM
zWZx}mja+(gp4PS6H@}Z{WgXVnQ+=8L|L<$jvi7{?QijYb%}1vGJ7{#ZJg@fV6U!sL
z0vDG}J+h<sTB-WXCmXgJ@V^QOT7UEJ-hwI1Ra<vHVct1=tAUF)TlPY!`rqj_)lbT{
zXRgkD^yJ=~I&sDLeDgyQLG@1#&P|xR{dn2#Iq!e(a9HuMN-<J@1OM4y2HMe<x9@s|
zm)IKK^l8mpa>C5_(TT5DEg#)C{PC^YFDg?iSLapC=k!}<PWO8^*4%n_(4}~(bxMtf
zC(FxNucWXaE@f6m1>!|FbS}39X+){i8J_=lRdGxHjE!-Z|M6~GBR1(K&$0v3fsXr>
z{+Qb@&{nqM{(Dbm`rG||n;*0NPQJCku)1~GLQk!?)8D<}*uF4hmiVJN->&8S2*2{K
zzCu=N>g#})>+H7#OESjn;#$;dX54)5_|qBl8h_qmt-h%_-E!L(<LbzTr2(^&b5;7*
zh)kP%cu~=k(2oL^@6)cnoNoB`X7gF2-yTozs_}~4^{Rd<dA0U^uC3-5Z^^56S*m-C
z%<g~KXT8>Q^7(zn(_hABe!g_^n&j1+b7w9oeQh1~=Ki|E|2a=yT(?Pj%VcZMG~Fh!
zOZQ!NxOLF!l=KT1RtBGJ4%_^&xG~e`(02Rt`$RA1xW4nOjQ)IdlB(}w4sR#lWxcD<
z_<W4fcL-a2wse#BZDH%T*6;qf^3IxBVp}jV`flpB3Cf{Y*7!`E$Eof$J#;s>uHRwP
z3wsw$ytKOG-g$HBGWJ<M%eKCKVl?-O#JZc2wQifXMP9!gV%oL!xSe|8_hnC}Ua3gm
zVKd>-@%_ac<~<IS4*Zw-XHE0{Cxxdy42~Y%@Sl6x|M2Nye;4iecq%GX&pF)sLFwmJ
zEyw=92wcAAlETuPS6-Igd~r`*D)M^s?K}4*xNb0geY5Jd^Hq;mwOb0@V#TK9eYcpf
zMl2(C?UMC8yR1yzRP-h_pW40J$dkKtv3WJO)+hV2Q}a@J&xA8CUi<W$Yk$PsYkLo^
z(uvw^HF<5_=DZk&s<!wCFVkLmUOV~h!8*0>1$#I|%Xm!hmDcUZS(<NAy7i5|(DT2}
zGxQmySKbd>u)NgN)HJWH*KX?C=&Fk9tM4Ygzwmx*>F4d&>UVoRY2wPN$Z3mbYrf%B
zuvViza8YTL)$X5n+i$B0CK%o<f3W$RdS*#Z_K}y-=^}2*4!5mt`t7bhp0%|3hGX2M
zYkg_?kAIz)`<@|UwsiHi{W3}A#U{bU8y0LnmLXMHe{c1b*O#YVXE4b1E4Ot!eKp~I
z__qk(^x3EEue&g?%KXoJv+nr@hc8JFX1ON&S@nHsnzK7f@n4JJ!#8f7RhEl$toQ1h
zH*M%I&Aw7lzR>^Gy&q9K_8#@GzWqQ}HhZnsy9LLl@8pv{u_tbRZ^6TY9%;_m^0m?d
zlN={~-Q&AGXvx{NAAPO%S0*Pfe{XsJ$P>rF^!e5&Vj2Q=n(h(jPP15j*YL=1UOt&p
z_lVoGvpRT|K5*1LaN_60-UmPQs&34jdnr?E&Z}sJqlaIH_g%0^XVdlFBRa#o@~X~E
zaapm-)29?>H<_2JW^cc{C;qIqNwKDk<NefJW&7vd%X_xGePB6p_O7H&(!OH;+zeKI
zxpU4fntmtyf9kj9;zAE;Lv!o7MW2Eeik+Ul{fXth>b=W$P42ud-g-;8)njFCeBG}4
z6CB2yuM|F=>vuMSP1|xiyR(G5ePn4tYo*A?3#oT>(^TKAlf5|W`L5G)zxE2*rQ0_j
zer?zE)Ga6X?#&m^!jE<ziMTG;`OoUj<)VZg+N-tYw>Sk|o_}`TnnLR(XQh;wZESwY
zO6=z=k5YZ^xKI1}BvDDWJ-Z9aq}`bne%P%F+rL2R#G7=sk2gC%o~i3!|MBCU<L-|>
zmOeRZq1Jxabh7JLx2fAK7Cp^ZaZ_bG9WK7u`r+w!OtE*C?$eX|)L(LEcd?~!XNUSJ
z{gomrY=u#U8&<s2SgCM4y{TmHM>)d+vDJU>2KR@YESsF?YP`(oM0enUh+v`I5dD4U
zdpEs!asTBzp&sX+;|7nJKHE<<p2@m|eRuu5zFi*`7l-Znv*fbrN3LlTjE|i(*#0ou
zV%0&u#X><Ef9Kwde*QFU+2f@X=PND~onph{*b~tGukE3;(+h`2C#@My|6k1!6Jejm
z^~Is)m_O57lQyA+)q%0Qf-Yyd^oAUHnv%!J7u{d~ZzpqnVaTe*-)<*mHwiAiE&11J
z{p@EYaZX}M=Du3#Qan#zr9I6!y5{ibj_=_U+h299x%#YV;-7nFYh}zcR92ri7tWo1
z@9lr<JH_9h^f&(6Y-Cy-`e9Y+bF2N~{$=i7Ukau~o;z7pup%izUbFYGdu~Y3l1=v_
zs^)ZsX$tK=!!J_uOLY4kwcNuGkLFyp*vkKY(WD)7dtby{JAXqfQ-ZPmCEI;h*{6H1
z-HA#Fed1c8o@&OSu<*8R_TJ-yn=NIlRX$s~vTDeGmlN0RZpv$ob69#=dUZ)G|M7bl
zzaNZ0^X$_<pLte`m5(KNB(jvYep}@Gr0iR|@!tQbtnt3fyR*K!OgMjWs)&Z4W1w1>
ztEk=K*2nU7YBNvHbN>@o_icsI@(yJYeZ!ii^@f32tp_h%`TJtyfok6Fq+{3p7QMS8
zVSoEY`TUtLRkzRn@#2lVtgdtWgtmOcg<)x33zKyx+}(7K-AMl;$KjI^+g|?Y@z{Uf
zWK*Tqwapz*O_hoaw&Yu7Dc?G;Ud#OJ+yARvbB-RJm-qg0RE}8AvQi22`7hRMry1t&
zn8e|_x>L94lS`{lcjfVutn9VNSEa2!vmvTX{Fcmd(GQ1qu?MG>Y)t3R40trNq0A+H
zM(QRzk8k{^*Im3A_cXeowri!BYHeQr8+q#xamiHg`>T58Y@f}R*?FjyZ>jWduHFUd
zmu|c|lK6k;hWYc%Wy?Hk^1t8S5W%RcRe#>Y!Loi1r}x|Z$q(OThw5yJlzZ1-v{JI?
z#vk|h4|+Uf^V9m;cV9Z#eYt6|`>oY)MP2{RSetiipBu;8=$98d(`{{EsKutJ^Iq=_
zyO)~$Vv68YPHl_ZuF~b3kKKHyudZk2n!(s_JF9WaDTx(Fb-#Jf+_CwVo0V{VDesd>
zUHWHQ)~;?bd_DI;bl@vLoygjY&L1`u7x!oEp8l(sCvyIYXXTR}e1sFZy>GsXf5}+4
zv$(iP=j6dNyfynKI&%CAS$4HdXv4`9S7mLDo7vO)Z+rT!X_j5Q_fDC<1phz5(08vz
z7T<nqd5`0~&GPt+?GqlnpC~f<`l6MBKGy{%KcAdG>F(TL8H+2bdSZqD-Y)0q?l4*(
ze>Q|+{Q|$@eN9zTi)H=8N}86fy5ktif5MeLYucf+k;zWMp}Z_YJN9eEX>WUUJ|oU+
z{^W%rCT!3BY&Dqj*Ycm7uXm{Z@a=`_r7dq~A1yfT$Ii%TymNi*g}3HX_21tvn%k21
znB$jV$n`^oGcK>U(9W~}c-!IFue>kePvZ=}?4454n$lZolz4SV1^<lAcRBx_Oqo*h
zes-{T@X-lg8Btx^4+-3L`!Z=>(y`0Oi<x#_5PK;ob$Hu?%gviLB|Glq%Q?!fzRSDj
zP@ho|2gl=A91?TB@7c&OOZ{{4=lC~?#XVD*a+2n2t@+}_=Nf#usNkB5uG)6-(nsgE
z2z@=`a;-H_@+R-~>8&SiH(16;JYAHL!m)8m{|){=?z4v@Bz>iBs&#$*E;6-kTVsy>
z>lcQSr@}YCIAFcDQ9atV_}<CC`{tYW`PYAmj(T2ucFy$FPsjD{Tz>XJ@yz5J9p}aI
z@?oJ<&l=5+wkq|$Z?s44${UldQcu~Wxh6kb8K11_G`D!~P1TUTDSkPhl6g~7ZZ1Fk
z)9mjrbKx|fR}(I;*ZaoDt{*C&E)pm2cRV|UQ9A#N%*498m$puJwDY$*sl0sFwi$ox
zJvP-|*j@MP*oLark>{_hwz6I-qn$i&e_-98`Id#JX3g?+{E^N%!=+=rO3=)f>6OI+
z@1iEUOR_Gyw<eOeHmGoRn77*We6>RNeS1UtmSu|un+eYk_*vtAYa!o;$iLNI|BCj7
z^46Go)J`y#?J~YAV{B<;w3#<m!~a*>8{^VFJuDLLONt-d+P&QLqp$1*@ld^q9yfiv
z-fT;|8}`QVPY1u_hCj33Y0qMs+smZ(T3|~0N6ub<+h-@T=2V?YD_T5(HE-@yU9H`p
zG;8Lb^j`IK{%@;1g;(vlc{aY)&edkOC!f{WE*)`B?_blc@9b6G%l?<2*&?r<v$0Jx
zN^0?R@BK&4UYhxN??mHujK8aDmX_R&ex9+M?Q=w`wYY5GhsBRQZ=P5y_CDRPGA6j7
z)Wm&L(XvLZxv!l*ZWR7o?3OCt{a0h|>C=6)FMK|{Y9Ghpug={Qro8o>*s!sx+WcJ0
zv$D|g&Bqe&$z%&0&v`s8N=Gf(z{g|NvfYK%?z(p$3hj>E<Cu5tRo30K3E#_RNc3{*
z+zw~*UbrOX=kq&?|1WYhPUliu7k}rA(mv_`J*OLsS8;5e@b=ip5S!l)&omGJo;thb
z=7Fq>wVVecEc4e_8gol>aa+1_3HiL0j_&?pV12t|(at?<ev5ctz4ysSJD_{_-bo)t
zHZ7TU+4gZ_=-JbgEpKTruFqEdos<$bQPQ4$dbX<eiyk}a%cq|g_dgOn8?)^j<L`*c
z$8UHZW64hWcGS3XciieXPxfjirElBkS*BmS(Er=cJ1@NUWN$p5Vk-Q*`?Jk#@2^j~
zr<CvhzCUqOhWDQO<5Fg;mb2;QtX|>$am{>V|0^bccd+g{9{%XLu1tu~uL)V@a%G(T
zSvE_y9C~?SO~SR}t%2uv`PTO5N9_`bFMgu-(8nOXEt2=B+2?rf<?UzIJgdBN=;UPu
z^Rl@U-cH^AKySUsO}Q|MH%AIBx7u^5Z9DLH`%S%TI&}++{I)gBd3$R~p6HJ^nb*sA
z*Kg#PeY@>oSK6Kwr*?-UpYnTl|7~4<w>s+jMvI>-pDRxG&^4b>(ReXwr=WBBQL#Dl
zM@#O$Z0joZ*)os&8|!7;uGn`=>u<k1+Wt3b%4P9is*~6E3A$SsROp%;H*f2EpdcH1
zWM$f3{RM>y%r@Lh?C(s8R{wN&+uh>(*)h7t$8)FNirRkuB~#(r#4U<F;^K4GSslun
zEHs5>(v_`wlRr*toxSvO#@|moIckf{jAyNr<`-8kxG^=we}nk;-vz7&>01Q$IHv4i
zk2E;6;gPY$@Alste&!h5y_h<)=A55V_LY;~ayn<rwg={4Q9gLhk869;)cM)(X5UJA
z%r2py*nIYi=+4=@n{9X8K7P|-bHn-O-Z^<EYW=&)V?E#R{T-w8ZhfIfykE@zBf(#r
zTOM0pKK*sohQq7Mf^7}IxGgA>O%DxZ-?`+2U3>VlIkU@@jjN`fN|H+to>%kp{^TR4
zRc~JCUcE74BG1}M&KeUq1J2#PsC&<g`TLvKS1%=mOf`G@@@(F;h#Id=(&4Lf4*cA+
zdm`U+l{2r?^>1a%Zdq~Zl*oE>b1~E7+RvDDbbiP0tPy<j>csSI`cbQ2pQ)>xDqba1
zb1y?bW3ThNtyzL*OT4BWlRQ5A*1RRhZpW;+Vq!ffr~Vgz?=B<NJ@1^}Uw`MqpC@|5
zvT(}^&#xZq_gF5vylrya8S$v=G9q*9>mTo(_h)Zz#-l}ld$Oj#UB|$G_T!wa+@fO-
z|J_@5X42i=e%V3W%w}g=M18)hY2oAUuztJAb05<!vm$oCY%BNudPIHuhL|t5*Vud`
z`}e-LRMmSXV<~WR*;<C(HQ5Qh4-TGs?h||RdFS%Fr(Fe+{6|j3y%p72x}^2bnc3$8
zc%;w&<XwA2Kc(f*<M`!aH_KnNPky2B^82qX<-eXSxmM!tl5V%Y^V9JcDesjnnm=#Z
zv|9MC!s8hi=l1It2^)W2-d%iNXyQ!vWhQ&y6<+Z_ZT#x$Z#ABr$~mv!ZxK4b-;=HR
z!5xDeg${3b&X<Wf|8j%!^HVSS>~j9kUg~~RI=PJbfa~{yn?3iYoDqxjl(ovgc=lo9
zhRZ$N`M>z<cV_b_r`?_Uf7w~L_4_qX)@{GNWT))?Z(KH;&-q#err1`ooblV3U=z0X
z)kEb!LOWyL+&?|(=O(`MYlI7DnWk!Pc$%iRIN9RhubHKiGj1xCCqE6c*vwRA{5wTD
zF07`;TS<HOj0MTd=Xpe|y&7{Nc+<O2b8o~{?q2IBC8?Hv?aXq?;<OVQ_xH`ntY%F(
z<iFf#O@u=4j-JPN=bSsO%kjcRGu{49`jl{UBlauCzN)kRI{FX0PJC1RaQczydrs^r
zI<}C<|FYV$%#O|VHAz3SpU0j)8W(%ydF8E(&iqrA#ilMvG*`UWIsHiNyZ1jFPi?)p
z&g<Q7{@v9ZrNqpZeyj*mj(**~_mB2x=KRv~gBQYAPZ!MoefVSIt8x+j2@xtEY$EmE
zeM&l77(1!{rS_GVPa<xutEt#$xqacBHC6xW^2;}u$rprY1*Cc%pRjYbN$7l$P8aon
zQ>zY~x>&yHc&S;?BEj!#Gu>k|W)#1w`uxuSTWHSayGqPaX}hP0#0Ni3;b?q*>84uo
z%Vow9FU~(KtGae(_Jed;T}eLC_i+#BCY2b<-r;EupXB+uW4_9#RjvR2EUn+CV>C6b
z^H{R4qXtJ$k$u|n%*Njy$#rWyqo1ETzv7DW#f=#$g~6ArXXMv9GaTStCZO=%@2P=G
z`Gl!{CF|DOhJ|wdS~xE|q(AtTz}H8|+-H<L{qyKSP-lzgh1{Dm+gN@-{3he<Q7(Vi
zCGSm)f>U0ybE~%C83~4|SBoyHW%B8=>`|ZYlNlbfxWCTHPf@YM{?YX{PbD<NzGjEK
z+CJl7+dn>KL)NsCXp7>q_519bgsXp@y54)!DecU|KaIZ<i_dX==B;?Ya=Od@Lqg}C
zZvWE1v^x9C?HoOi>hmnKR-BRemQmN;&9iM<^vCKFv5<4^2VY*QT=Q{8RmO4Q_N>R<
z@@&e#Zmy~{w^&-&_e5v*)BOINq7EM`)5|-yaegU@xZQT*Z0?LH1`0FgDBp20WV>=I
zT|Uo2UMt^7Ry$pt<HPKWwJyT%wp<T6XdE~1Z|@sjpJzXATqu?J=)G=}-ivFSyRU|I
z<W&{;U*ldU;oH!?%`!Ak<kjApGq>oz;9JetDD8IoL&6`aBW!oi9WgkfJYz=+ufk;R
z7iGz7mTlvixam9Jrz<b7ua}m;Q|o-ZK+yi7d-u~-QzE5w?mArl*AeXP9<}oLzE@Y)
zS6L{SFd6zzSUbJdd-=sHm!|CyWB;I-w$)K-Nn(n^6Y<E&eup_T6y>*M_pewN;Wo>1
z*IJ*$bMjLPWx^hZ{FZi|Jkvk;lL@o%vMF9E!KRarv(4e%eDIR);*3i+RX3ilk!!P^
zyD4AKYPR^Z<vFK&6}unvOgwpc;;TXrKR)%Urc(Rrb_3?s#xt!<_;yX5$WdwX>|NG}
z@&`eoo*QhGC+k)kJ=9$(B6gRds&S&z(?>U-J^emk-|EJL73}A;^hI_kiK<_F5U#W^
zZ-K~z3$NzPE&uYfGH-j^@wr!g>YiPXopD8P<-yFICs)gFuZ&u*-M^^#dRS0SX`{b-
z-_6T2H8We~=Ioq0`SInj{$DOHm%fp^<^ERD%;o-6a~|K%zwVuSCAF^PtD5c=tNCkZ
zuYGk;^4=ej+w7C>YJW`F(!6IXn`=#^^RMtJ1-qwZ@0_?dXqVnLk=^0fewwB>ZFS|G
zlBs!1Oq}=I+rz8MvS;m`_H}mJ)F5VIIkUC*m)RbeH}%Dw|2}$hhxeYit1ISqfB9=$
zH`NH2gF74YHqSS`I4|stti<g#mTl^e7wXqvmbvDZ%C>RJtjV!fGRs+KANtk4?Na;4
z{pXDtl>Cpi`W%bkS(f}}T1HLLGQ-4L`INKr=aT<zJbJfW_&oo<(<#a!7B77Q?ybDB
z=JL&VkEi=>JfyzwspUC#&z~FRerH<bp7J~Q*|Sr}U$2;%IrI9K>#6aPN0X<<C-kOy
zE6mK6UngGbIoIc@)y5*(_Xcmhs`W1y@s%I8^ZWPWDvPwx+_zS$A9ISHsl5G{;&$2d
z*#f(glX7QSn|Zd?ZL01ua$Kd}Jny}($1UNyvI}-;$KLsNyUdt=Ky#hst>^llo}Z8Z
zJn-5|MWwB=Y5mcIyH}K^$;kF}cucxB>3dh$k^@QSKFLVOR~JZL@etm;Q|NV6?(A=m
zI{#Ldd!{Zns!mtQ4&J%@VXL>(z0V!t>1I8XN?YR=JI2m??0#{YBU|Sr&hjZf*BV@|
zZszIgoos!ks+4=LN6vSNZ<cF4=U$KS_n)VG&F5Zv6o0U*OityyYn5g^U3>dn10p)O
zQg{51tbg@hGNf(k%G5KrGnRXm#D17DWz9qd=dZtvBm|8m+6-q{t>Ag?e*fi>cX`gL
z>m`#eReJ7q`F`wG>!)9f6Rn)z9!{O(<N8K2*W*dbrftF3@?*A{T&pvQznG>}dOpou
z=grn*_g2_cv{Z;Q`5!gAYZMY9*Zbu4VNF%36<4oB$JnXN{kN8}_iirl+9Ub%l@p^S
z|3%i!K9tdMZO!)|YpVA8K5hTeo|*Umyw)4@OQO3bWo?<(eZtbntK``2L)-ex?zODU
z=HC1CQRb!<t(-aAU!Gjuaf&_9FUZ&^RFBzB{K@jV0~4*i_Xb@J5jX#}LGSOS^lj1~
z;+p$!{r=J=J7drPC)PI~|K6Y`|4(J=#`{|>%)ck*zmL22b+&Eqie;wVCsUPA`YzuX
z=KB4}!QVV}a=$0^Tl=!t`hFD^<yhgWUf9~U@b3-F2)C_ybDnPq@J#rcSU&$W+p!JJ
z*MiONO6)R^k?H)OckxzQ^F=3lHtX6?vevs3s;;JeT@<>#FTnl63#QbN7Mmr@qVgW?
ztBtkhU`(&syyma8`nU7{ADT)Z6n9&D&TUy%TII~Mf(tVEOTx@2w3RK-OW(fEaJJHg
z&6<l3s;mxG?oaDJ5UO9Z%&_~<lXce8mHU(@oqCtH;qP37&CD}9-fTax?Cswz7t`+P
z1^-B4xYzXGdj3+Krx&NLGM(vv_3RqM{n6d;R=UsHkh%Nlle#M}8m=$$;(6;;uz%LF
znTuz-uK8)Or1txnw6FQ`JKT0`xj5yXlIfqDcOJLbcGlS)o+G-|`lk5RUp>BS?%Jsw
z_uJlnTYc-fr_m+7KOW4w`zJ;An(7~p>iyF-FPxiRe>&mc^5t6Zi(E>#o_N~h<kx*B
zHu&`Oz1{5Aj<ViSdu0Foubmv{etOIHe;cJT3uaGzy)E(CP3=q{<=6SrZWrw5%*~EJ
zs=)s0v&E{*S#zw*q@ShB@}*a@G0aT4%O*bmV59#Gms>Gk9&qF=-o$g-*Es&T{kN{m
zn}u)2{yO_mxz_yc%h3KXvvpDbKVF>ccjw_Y%NB=oYq}36mByCl^1WUC{O!j#ZOb>_
zh|~(ZxJch*l4FSHb({Yuj=Zj!UL-R$NB61s4Yh}N{;~VNU+d^&wWl`O)3WD@W6nbF
z9R{=W*O+x%L`bK9I%Kxt<X)?)ErmuW{@rdatKNEI+Ny5nbfp<rInM+HoIf!=PFL5x
z^yG{$9*@sz^``hlIJ8~&^Y4B7=-*ccv*d`Ab1dXOtqzrnKP#^zQ@Z=^mk-)6eLkyg
zy^-Ooe$XU&b!{*E`cj|OMoXGq?kwCl(f#`=ovRBE{f<3-?U#7ux^SUwrD@HtzCQ7I
z@zpCWkaNxVSG(OWM^32opZ7}2VBNf+^PgVm<<{*KxIR10JUQoz;W?@OfA8%K<DM<u
ze(mgCsmHT2-2LNMor|&E`>O2H>eSWiSE;SLC;jc)wVgKC&etvpx+LsvU%ffHf6Hs(
zKTFvk$5(P?A1_F`eYh>VTXxCj{b};WIj@xC@@_6s7MA|#cz4H5$FDIq`kPgr_xJyq
zu`Z+d;;nPX)`rer``asZ;m@~^CHa3!nD!(uF<d3|SnsaShp3vi?CGm7uG{zE$bTi4
z+oDajcivg7F#ga{?ceSA@><-q5VP_r#-}%xzTunab#<SH{KdE2Wo3Q$uL@bOkC(6i
z^zhoYc`@6kuKss5aqc_5WFwi>9p;ZJPbpt6-ty<^h2uZ0gtJn-_e{KMX77<_`fFQs
zR_hwkO-DmQc3+=n#U?9obam3Fb&X4pbzPo!q`BYip@66C<BGn!8Mh9m@vePYb^7qb
z9Dyey;!f2^9xgO`@a^QIL@w!;;-`FfyIPOFxi#&KzH02_0~z{~C%69fPF=uzfA!5(
zmp<>?^mFO%_c`}JE!TMMU|x4uDY;sCkC^_<*?S`TjawEP<|TAnX&+foy87mt40qji
zj+^UKQnatE`nTIRD9G39qED4tS9$u*yVYIVQQX}8c1KN*l^))3?UQ3j_Kp1elXI+!
zjvp56x%T4ZO~dx$_`6f*Wjb<8xgFxSH#FF1_fSFqd&#^#g)j09e?Rxp``B^#!IQV`
zv4U^x)}3&Z)ob4zw|2gMVL<V3$@N;AFJ3hGPyKu_^q}2Dj~|Y6bEcPRi<K|k{qrH4
zs{5+yIr`Pt^{%VTpZxHN{oKRtG6sioY!~nRX7tiKbPoH~{hnJT4+rWl^fH<5*7E(<
z!^H2Z+1Xc*I(DTWn~^m$Mq>G&9UpGVx}RROx~8iC*jtOW>C0m$uYIBQf?Gnrc?vuK
zY~$Gfb5tTM4!ys>^X*AlzVuHm6Amx7-#=~LluB!_)OC9O=GtZ2eC5L4_A?e5zrNmU
zcgV!q^(t$r;WwYf`%^<)gZ_qhRn4s|T+S7KK6YK0^WmG<?pNBGT{iVf+V!(#kwxO{
z`~z1GKHK`|^0wDrhUSdsv!};japbRLy`7!2|L5VY#WP=oI!ov<{XXt-#bzf@YE98s
z&5M~zN0W~J{(EM7cFh^xUvlc+`n&J4eeFA}SMI(#yYpw|jK}8$g=eZ4s(BW#nAb5Q
zHeWygp!&~Z!TYJ~8ZX;Dl`O8$|Ld3%a$;$_?97(a$`_9tUpz8PVwH`M@AjF0V>iVs
zmg}0{OM4$ycTDaS<L*Zul4h5Db4z2M&zMp7>-^a_hu*H?Df-R3tJ&m5{<k;B?r?|m
zBs@&pdFJM`v&MSzIpvmXpGYl?I2QfuTj`a&?dH!fzGybmW7^}GfA1iR=wacy_rYsS
zZca$txc_&dJ$G9@|M#fxGbcV?IPJQ}^r)O&SBiR{an1M9&DzxZmTzXlq2T#vcFC$4
zzY$0~Y*J*Jd*JcB++}yZe9iD~Ua<A_-7oq1Nn12;8Be}@p|n-jG<sY3fi3sSO*Y<n
z`BnPEqRqBnTxZA}K6~@%?UyOrdyI{vb!RQU;&uAtb*1pVYEtit@3_9b|JKLs_M>}O
z{|Bi~op#~I3QP4_n-$OHZLc}|d|~*V#);=I&f2cSsDC)-)#cVrfob{8UuK<suN-jL
zbMJ*eyC#~QKL1kd&Wm3i#YbLVjpf=`q27M^$EGViIoD@OK2eC9Ty}T6=DJufZd1E-
zt}2tBA2yD<i+^VvzA3~OpLol(ENpT3)6UDb2fjNMKFwIWe&5zTPkZ8fZEK=$wQk)~
zXl1kQqjSjJ(;hRzmzGv6bPaD0&An0_I_I+9$}k(FIVYdq^xL&uz}fo1qSmc`&I!-`
zqaQtf=OR=4ZTHisd1t!0SKN90L2;UQVbP3R#y(qv)RWgH7|iuwoc%)fa-NI&(u~tz
zDz<BhukL;Bf3v&ohvo5<>30sUVApRIJoZA(cw63zt+QpfCVABb{!zSD%wzhN<IxwP
zHp>_HE!$4LUj1tEAH6TrB%8ySr5?83e4S+W+WidA+(pK>C6<U7ac7=+HM8g2$3<yx
z>relkm6&>?bS>-FxT~uoH-CG}BV)|)Iz>yW>$1n}S*G*fuMwL+WB(-ut*YSIi4!8#
zS>5+5&doEJI!hsW^SPzw$IfsV{cpUnC{m$lacKTQ-}Y4&l5<RrFF$^!Z8`bQL(_mb
zNAWEgs*zTv-!rOxWvn#S4*DPPZFc`#Fvlg0F)Lzi$*Xf0!@i2t+>4#>_Qvw{{1o;X
zr}rFw+v@t~arL=3D@u|S>zDp#@DB*xKigJuskiIFNLQD#M^|F^op|-MK!ty@#+T^}
zxsFS=?U?4&+8J!RMO#ev!&{@rUp^L2R5+)6aaGUy2k#ze7b@`DCEVd(v8;B*TvaWl
zi_d1N7VcWV&8Jdo>O}w5b5~S_bUoY1abj9gj@!nUn=UNtzHPz8H|cJ~PrvU~d!8QP
z7F^U)Z0Pjj^~D0MlMnZ={&7NHc2V@?{NSmlwterp`$<_$b){v-Mdj=ViAfLqQ>&bV
z-bkK(bja`4YN-Xb#Y-+sR%UaYQ`!9KdCEzlT&uv#;m6<Bcm_BBm@-F;Mf%oLQ`@)3
zm+w~3YW_P@rvHof)Qk%~E<9rUcB-C@ioLV%SN+^>Zt+t$=S-K*Y<m2>=XR#rLXTDb
z%8d!ejSFuc$Zozg`$+=lbc1(y!g{`LUbSb#v1{{>&2cL&Zcbvem2AHJgzK={7mtmG
zl`}V+#9GVj-FMxyyp+pIq0qv5kG16itDo7T=S-?L$*$UcJv3^?_PeK#`Oj8w+PI=r
zB2(I~M<S%-Yxdt;!S2p-4@Lico9`~Of0NvslO+?>*D~K-c4<qu@^iIhqvgk3CkY2<
zrD@lF|IJmsZHb=V>6NQ$U;o+4W4}CWf7CX?`$7^sgky{xYzta={p<2~Ha1I}Midvc
zt?A~k(34p7x@xt^+|K5Q2bJeGzF4Xtl`sCeG%M!+wh79;{}Py*g6e#mK5tS=I@)Nx
zd)npNNddFn%uMg5r}RpN8AnZAC_gKAU8PCDafhXQmr8`h{E_<m*!qQ&dFA5dcj0}9
z)2gO;yvyh0TeZ3IUH$H@XYafTt~nT(b1yv4X{yc66MD1Oac}!!?(4H=YNg77xu5=>
zIve(9TewpBbGQF?(KBwXJ)-;JMd;tBt@>G&w>Nc*<(~Igcj&>gM`k}$IHta8y?D)<
z-|FU`+{ZV9`(ElU=&}-GS$thWT;OKTu2at@$`l#Ac{N|H&ElBL?K6CLcS`?ETrZ=f
z^ffMI>ca!ZJ`2r#KC=h9iiLlUzxedMqsuax)DR^m?zNR-H5!d=GugSlr!HN%v?#H)
zy8KPZ#;dyvE7C46d%NiM7lj9Q5)x*|PEOw^Bg@wP%0%?G>tm_eT#><xocK@PnKma=
z?pC3`%C6k`SBx|xe);c@zr3=j`t!mY+kZ{oF0`2KxlfMAlo?q^laA(gP5Ha)#jhOC
zQk@;`5p`QtUNs&H-nQU^S$=%u-re^05-&@p%Z6RqQ{3CoE5dD_zuoe-*U!Upo67A@
z&pmf`^G6x;+_OJrbn=(IJA5?OGUD}vhY}%kH}FpXo$|c&+nX=Nk4x3(MJ2JQZhkoB
zP)$PX#-2{jdtG%u(&z4TnyFj$*{dM*#rfE-&B}5czqER$d+m(q;l3;)`s}`DtKsyc
z@0)h3*0i2#%YL_V3EvU>Ta&t9N1b>hBJya@>zm)!2VQM%j(&9g@Rq}2mb><4g*6&K
zof$6iey(kpzMi|k?CmzW6J`IC-gfcy&;54r`PU0MZ-365dA;}bDbuBwBp1f<WNdw8
z8JDwi`lWD9(Tp?wtoOo$IIgWayz$!Q$@xj^FPdeBOuIef@^O*lr=+vXo?pyJ-W0o{
zXzH4&jYXbE&(}XI)RE5l^=xa+A5S;$IZRpJyZmgfzcrbmY#C?0b(ZQcv29BxCkAbt
z`PL}(%(smWvTlFRAKM_6x4bv@<_>di|EjFA<tMoI?1|VM8OSM;{<zIdn%VXC*&9#N
z{^qC`zY6V~bbd*WGkeyhssAD+Hs9PA%3`JC^f{#RoPXNqU3asO?z+Ipe9}cH^_q-)
z{$p>o`xmu-yy%@}`e(znKJORPbT{pquCHTMr4gsMK`DKf;`?u(LMEPD<SDS{Onr_1
zav|FSN$cmw*Ub1D&H21=)>b)hHbGABcCVvRQmF#5MZRz2J5EXeS()@~u0C&2QtHW6
z(FzZ1t38+Pa_vqY&H8j_W88Wzway@`t1$vk{#5j69Fedn*#F+JUWdzgzS!S~Is3B`
zC8s3x8??@x>g}y^?_v8diSV$wFFxL>eem*~O5yTEUd`{aeg3AKFHST2e591^!-)e4
z8%4~dvf1X=A6KpKS6)?_ws?8|Im5`#6`Mb|7H(h2{8i?e#L3WtcY98qn-+L;oA%L7
zC!X!O?yUaxV599zX8V_q?b<7QPIXmDN7bG1uUy4)ZoTPVC5>f<f1XLKE0Dk1b#lV8
zDIBk?lUEx*v&+);J+Sb_gBdc4>cXXL$EO_<@lm>F^6qHw>a~@(vLiydcrx3p9`hJp
zomb`<Bjq^z_1-V5J8qt@x8ENz{mv2X$rJ9YWN*&>buwUs{{E(EA%7n{3yKI5`+Su>
z&vDP8-zB#vCVl1pVZQ!t!mp|yZ)>-v^GG!%rD=Gdk^1>4O?vMSb<;a1Gqda7mu|N3
z>3-jHSn}4S%82j%&s?XhPd|M{>>K~WV=Plv&f1?<@X+bU=_LmbUUZy$VDjhLR&(X9
z-8C!FU%lYdJW;*4W1YD!H=5NHB;6*<-@fM`>%sg{T=P~+=T~2$tItiQO<B99R%YVD
zuMN9%W~A!PP!c~OBXUpHu)aZ`JH1k~WA*+f*D!USX}2n$1v)J$VSih%>-E?*%4YtS
zB{f^8^|@6n4mc#m_K~0YoxHUC{gB6#rAybQ6s?`g!lPET+?x5C<{6`#%NxaX=e}HX
zC1d6cJ~PQPA097XaAUfqppU>ht&iv5Uz1(k&X{}ETiNA6TqW<7(nmdPo{a%FwDy&-
zhh>`|+o^a(JF@B9Avew0=MSVAU)kz*Z2!#6N83$Tuv9(yqC4Y#<iveTujPIEmCtcT
z;ryKoYc8D>+|bJ{@iVFWe*4SzyZhOHDY8zhoYn8P_-|ptil3#uPMI^!=51MbI>hfo
zve3s_d~b?R-n?p=#mLFRroQvN+QRuSJzC!;zo?766SL;~>n*GMzs<f7a;Zbo<eZqZ
z#tFsq4o^-QJoCEV9p!tf_6w(o-8tKtnheWcZ@tO$$bX4g_V%YU=Y2htA+XX&dOqJ<
zp_3aIWtjWdNQZuVeOGVy{rF|2cfDOqgTCxdoAH=kc>6?KkFK=b4HDJL2WF<U&RBAM
zrXT<HC688TtjgNVCOYkn#QJ3lvWr~58f$MA$^W_X{}tv&Z_c?s+TYLItJ&qeY}pot
zV*&Q3_Pw;~4_fufcuR5JBf;O?-;T}cI}>!__G|;KU)k5EM*e+}VIte3?Zm+mUQoQr
zB)|EjEXVQ9+LsqBs`0Wo^L*p{=f^WFR^{?{T1e0PJlW1pf-~{pmgeX)OFr4Ih*>rH
z1w-SEu7zb^bd~R#i~m2m#{R`do3=?Ve|PNZR`ZPCdt1BRbj89v=bL9r+~(el>0Gz^
z&%@yJ^6DRzkEC7>dLC_jCOVaGsoRl@Z{1B_np++BY%-b4zpPH-?!mcPx8AJaj`2Bu
z$c}ybgeC8E@@01`Dwsx=>aP#>*zm!|w7l!S*9VD?h*c(a>kL@qw6494TiEyX?9$TQ
z%U#RF-I*4hULU=3H>ck28Aps~OC5AI^*ecw@%gEHf1VbbzL>Fe&o0mBCxW#rb(SkH
z`y%3=Q=BB#8WqpEW>TL(<?@%iSEwGBF5jI}EP1`OsOYni_t|eEH&y4=-m;$X@YvmX
z*5?@Zy1(9CpxeASDx|<R_*l;Lm4EljrOz}ryJ)@QT&+T*(T+3wS!%ztZT<XUZ9xrN
z`}cn;{vsX$Jc}zhO<zfS9pjjxWg;qcP0jsMbw$Ry`h4^4MxOt=4gZ+F%C+g%5x>8H
zPjuZ1_ODKvEB}d2ecmhIn>A@8gOKR_y72m02XC<}^OarvQmXRX_w+%RFM9LeJ(yTm
zm8R9QNk)DS^J9t7vuqQERv0T)@v2KlJc~>_7iY30za(4U_Cx2RinXRO>7p+cuda2e
zO^yqkd;ZV0#PF@B>u&GrRhR0zZQ6BZ<7NrN!lVdA$;9g$?<aL{j0)8bj9n2T^y;B%
z<g=aPvfBPLw=f1ji#hzKsp`~BvDEX)y2n-*hV=V6m#$TBirew(WbB!3KQ?nN3X?5*
zw(P3@<Ba8Mi*443UNqXB(W_XubXA9@*c8^(=?@%@3;3KSrS;scdfjs2$7h#gO*_<$
zw)VPtr!3~WlC#}D&0df{Rm$f{;}f=xFYIT%jM#3qPrU2!|Bb%)(yU^a)JE^THtlNe
zPQz}Aohud<zk3+uQnFDn?%}79+?||#4jvt`^FF)XHJR$Rw(X76lEqQyvzKL*8c*EG
zR=zD|x6#+M%2TsCCG6K6xBtIg^+!gZ%(dFRvSC)G(%F8dtMB}<vW%(@)>vTP{wU;?
z<TJl(w>kY1ESJ}9-hVgwpWHgB+|J3rzkZ9q>Av&x;n-*U{?y#RT>NH=>G6lv^B)?y
zaVy3?IIi%=;>_Xfo__CJe>Ap+mF!EN`6l?8x7C>{lVgcRS62I9QZ_nw>RZ}6$tt5;
zp4=s`)>T+Ke5lyO-7czoY-RP&wb8ugv6~+~ow>An-BI12XZAc<)$=&uK;;W-kBq+T
zjRL&SqAv;M9-A%Ny25I8p>}P0XRh0oOK(?9nvgrsNo03((;wx_?Q_Hy?R(>$yz9)m
zPQDL)|6T>2(_86SE^%{$rqdju66Z@RlQnI9G>Z&<CBH@-{Pm~krP*$`?JpKaU%%XX
zFTwEn;*TpjZ6`gcPh&D%eLwcxy?6IMOyw!?4(SyN53W=+xaxFaZ}!v|QR_1|yx{yK
zeZl+t2c?f^)29c-ywbnzuNm*j@O9eFt~A{sxsFB8{x}K!4m!c=Soc%u%3J>YFVmBL
z@~^+~)!V)-VUvsH%E(Tq80%waR-IcQr+jaLkIl>C=w*J>59#jm`SmBzJfi%`#^r$=
zTT(>Nm>Fp*maDA}cqnV0bW}O`mQ`|9V%6FBf9BP}`IfF$R;{O`x86DwvG|XfhTyr?
zmzMag|MJZ{<5c+4qPsi)mqypGSP*>ULBGq-_gvfj)eV!p^*Z~*9&9yt{k|{ZcjPt~
zxvnKk+Ao)QNV~l*+5Syp!9<POT8mvaO}M}Rs-#x`T>Z>D0gChHS$K;TUHG|RhWWm5
z<`kBtEZL!NO<b=!Ylwc#+2d}~S0h_7MIgm$Yq<9I-wxZq37p#|ZK2nZxXP;Yyyvx>
z&vq+Lw_UO~T=UM?XKxsqFD+75Nm~=OJlOyH%J7KZ^DE_U`|nBLD)cm1Rq39jUj9z?
zBa<I3F*i;)Am|qzR$mnU#Mk$Mijv#L)2?MN8`bwOxV_NICGWn=&kKoJTb9>EizGyc
zI963`yeY{m^p^Xsl$ZRr<-(7HonA}CtZ=J0JUvUe;YzJ~swKm3$DaQ@9g@wLmaFXU
zQcHE8yf-voqRn?!`nLw*ud2`HKeU`!d3MQ+xoh6%$-ei#v$?OL>FbNaeY&i2!c&xU
zW!`pgjN-3<7%tT!KU+`b@{C<+Y3x49owF+!M^x>I^_N_xbmVYjS=5)WU*;smb3d2K
z-*c~c&Ps2=oKhBXAGZfS%cjrjxi!;c>hh!JX$>XE3{P~+ac>PVoXxgVXtG;5$Lder
zi`Rr$UVM7$jA)-=wB?n_Q4TzAZ`SQNDu4da+-=3b!UV6}RCLYDxS(<7a%cO#9U}VI
z+Yc`-sJkaq?fU!1vbM9a-V1-LrYxU1U+B(DmqlD$p(VTSe(^l0TTyL&Buq6cxlVp!
z{A$s5QPWj6fs=L4w1>=4UMVuUBF28|%3Y@>acOQ_b>m#^juT}o?YL%1?7bYl(0FeA
z_k?cCm8FjkrL5Kqc3n}rE&HWr*@g`+bM9XB%{`bWBzh&q{-w&A^`U?D)eE<(2Kpt{
z72kOK_m$Dgs(XPP$#MVXbV|F*_N&+h3MNX;G1ZH&I(yStcJ8EWwwr#OU_9kEP3h<M
z^zYAvgkG-m*LihiyK`yuLX8l<x88eRa4^5|(Z9~i{IYkZyj0|`OwV^MiB-ijAC(9{
z|F>G_LHK{U_zY>@^3E`G;a{cRUheL0X^~S!?;A)Rw_l*XdF~UdJDHiwL<~0M#b_@p
z{LZ)CZkDOIVpq)ceG;L|MK4O<oS?14x-HM;T$8ck8XoCi;gkL?GL~Bz{`KC=uixsw
zWYz7MocNll-Areqb-dcl&E8&t_nVhKoo$->?6;P4(V4<md-xk0n`eAIAHH_|tk-cJ
zY^Ki-vicV@t;^0#l)L}0;C0OsvDo*)F^Bskrj&m6+FBoRa+b^5dXel)`JcXAU-CSC
zeOOMI&E)Ihd<$MqTDkx1dX>C)KR@l56}Cx2hu7eBxyTGT+gleh_RqcR8kcEh(|=e-
zTh&uqU6ac*;M#`umo2?DMQo>@{kMItA%9?+p5oE8*=h2;A7#=OC2Gx)o$=Sd>uuF7
z?U(oB{^q@1b$;_Axqs(gsx}_D85XCv$>eBFW$0g#W&hIeYScZfOeo%;r6RUj)3~+#
z*~z@4;c04mC$|4vu%+s7iNROzud~C=&$@iSa$?TjO&j>d=6~FLc}w;=oxZb?*Q;uM
zE@xVrZ*OP(c68&D*Z(X&s~TkOX;@m*)vRZ#u{%RySMnc*&!?6dt+ALpr{36%*ZEcZ
z$v1}&J(+LUAAOspn)`T?!5*VsQ~J5B*RyLD1pK~N<)U~Zrs9{Y{UqH+xsOYt^|UUZ
zJZR4K;+@aK%VNP@3nvR@n_d@~x@RYc$floDDx0RH?Mcfz|I704*0f!hRIYeEtTpne
zE&Hl5&t;nW#HXx({tC_5Gkx;$O9zkeR|+kuiL?+2$p0~=UR5>k!;^`v&aDp58eg1U
z7_5FnjPXOUr-jfF1HFvhZ{7r{871xO_;Pr=#U(?}3da6P-0GFQlkNt8E?jB;ZJV{{
z7US-Ik4FKAMBe0C%`4XWsd}~Yi?~$KBlY_2FMsAGXl{>wvi|;ZgH1k9I`4;@^2%-q
z%~|h%JobA<Np!_?$3@5D&*_$b5M%3nz4*c1D*0tD+2!^Bf4q8=+1t5ld1v|4KX0zA
z4V}u{*Wh@)i@9Zza>%r^yPqFp3i&!$guiss-<Ln0-QRnoNVAS%;jM@BkEPCZKf|D#
z6<l3!t8y#TYjyEIJBL?%l5w&ooUviYdd_^Y6so)4EAjYBaf_Po`_u*Bw@F9IOh{g*
zd_2g?@b}J{hI}83nWvN<zPx)@=D(8}Lc4T#cp2rz35!aYbgTV6_D*W%n-A5;?{3oz
z`_IR^`(a~w-`n=|*rUf@rZT<Xa@otwm1l`+5o-w>+krhR%^yAGcm3s8*KsbR@7#l)
zyh7Q2xewlV-kjriogZ$$e~sUL*~5QNY@0v3%pv36&N`me?hEfpur7{I+_vH0x70gL
zDMde@>fBX3zDC6NLjTFGqqoFzL$%i(E6FWxKmL4v$(6fb?%hfH@?qnVz;h0p50o6J
z{d}idv2un`0LNCJya$tyo2+fVvV{5a1Lxzn^(1Gtp7~MH_wtt8*_74k*`1am8}E7i
zzpKNq_;-OG`<#TCk+;`IXgvAi-SULL;ef|8IcX8A`Zss>+zQBh-6y9WR61$)4CB>%
zW_~KJ{y*jWMul^0wokoeGi~3ZkhQAITNNW#?I=(ZV+zyUzWrxej(YfBrTe+RqAxy3
zHkx|xPm$HF=gYHMUyEISbY*&u(SB*?1MQ0MlB-r_FK5Z>b3dluo4K=9>2NSx@}HEq
z7yT5{?3Zcn<JY>uy@65Ru07p^yK{D0B8%6>N=t4DhO&>g{Bxh_XmbVV9{nb;CEZf|
zT_oSX+@}p1KJ4yot3oF)P2#uOGb#OI#OFwx<KGYF@4hxEt3SPVdiFWN5T0*Gf;Hxe
z@4mC;hqvZ0>vi14ds6q#J^V2^gNI*(zw^ziulWW)(@knq_>6m)mRZa+X}f#(<(HG^
z!n5xch|Z~&*Hn`zHaJ;S#O}E_<=Vv8b!}Ba27G%K+-}(M=W5CQqdv`9pW~;ih{>y(
zM7H1k8!uxrX@`hhn8ouMhnd?ZWSdNS_V4Q9uluq$EB#eEua#d{xk1oqe$4+GSHaYy
z4tn=JgilF^o+v7Rdc9y%&L+<$`IqJXqTL3*t#t}34dj&FlEpgHMUFn+G*z$D?XPic
z)434YdAs>-QZmk)uU!##bK7A(IrrtB4^;Q85?L;(sMgc|`1)U0V{0X4@1GX=4q9PP
zKTCa$Tl4;O2^;(Md7j;eg65yie&4O)oU4&~>BEO*d;Iy|%{eKORqP`zwfBOIa95k9
ziNe$U>(ZiG>VIsh%R6~?jh(KfmFlT&{7vuK&d(KZ{~7LEFn{UU^O75vYsys`?K*sT
zNyhQAug*1VnEbDH9y;1}vFnxdYnzEuoi<C3oxeB#-b>Zm%$GJf%eX|FyXKY%O^)v=
z5}kSEZs$R}X2*xgv+gWwn&h336cTXkx$&~!k1~s$_qJG`O#GY_UhPsMRF^YN{P0Ki
z&j<2P7?*3kyV4uF(|Zo5-ldt>T|T5Oo95GF;(zYy&1dr0o*a&T)U)o{uCT}@&d26W
zI)C$D+<7+9WhN1FGEcWTXzfk?DWi5R(|vcPO!OKh<+YcNFO>{vZj0F-IBTMX;k8*V
z@f=?t%`3b8&PG#iQ{}tZl<<PrVVgg1yQJ!TdaBV@>-OZUnb$L$mx}~Og>o*MUB0oh
zG$Li4*42zvcfD#q@7{G{jnM4;2S4+@6LY83XIuY#krv*#Iqv0zbR{0kvu|e12s@ZF
zJ*`1xd;jx_f^EB(Z}wcmxm??6>A&ciF6nAh^R+^s?NaV2bKO>6Agh1JY}s8kLx$2Q
z<sW>}4Qt=twokWz?*2w4de!D5{r0<8>Fd_cd2M*}<m6@_q2o`MY+YKpo0osj1XV-L
zzI!L<$5@uGwl1&z<JNH6>C2q?k0Y<|U96ZAZJA$fmED@x-}BwtXWN>ci?0byX1<}`
z`yi&;xA4;@?PL6&7hY!GV*Yu7iE+k*4Z>?vBxGc(A3dFNk@>Rt7l#LG7xef!FYL7n
zTPCkCRj$LrkoU+ZtCM`^ySHn`KHM&6BhI}!=zD?Ru6Ktc`W63e&1_vZJKtQ7Y3D{u
z<;VoT%Z9giPt-M)4N{7|^RnoR*t)LwYnIQpugjl)wczgV*&@k(LH(b%8Ev^3l*Rg7
z@m|mArWY}rf6I4vE@5Tcx^jQ=c8)~LH@lQ?OW$H%eb*`Y>8a9nXVo^ket&UTd-e8y
z?xjxzGSX!D-<r7SWIJrryTM(TKWEupW3vtVkELDCyZxUe9H?sBcAu5EaaxV)nZsf-
z_nR|IZN%>`SiD}!+J>#G{od`)@ck)QZ%r!@i+%fK)^nGu(kF7Rzm@yEgs-pkIBVam
zNRtWwLv}S^c&zx}viNpc+}G?V9<2>4md3yT$UW`b>F+<!?(%Ki_44rM--~8;9ZN{K
zK4XbklFzkC>{6~pym8l_Jl-?)<WCpb=tXPRC$jMTm+E;ScX|KrpU!^WoLhHXN-Vr#
zxcmKlbEl1~Z|MHuyP9|5Zf?8j#JLyiHzn$|I_<sEefN3X&$Nvb@1Kd(y0vEhQTBV6
z9tlftoge97eS*7rH|Odvuk=)=-2AaO*LD_<+?<|AbJ{k!Y`1YsEdP9d^Q(8EW##9S
zZfY;wt+nX2k@lo!tyzB7o~n)QDrFzvX6@9D=aj$ex$N&eJw~O8Z!Bk=;hZ#gwd0(?
zN*~Enw(X~mY;wBrIlGQ8wS3o|O;L`L@6=DOU8bx&Q|qM9rx;(wWxqlhBj!)PZ_2vZ
zm8(ZL{^;z9g>o(VMZ(;T8S(rYdnQD-J*`L;?Tz4jGI;`*nB&3RB-^`J3j&s1uHP&w
ztZnl-Cn9x|;gh>hRnHv#csKfQae$9uXXO8eEsrf%9ebz8UDI$%$-*z-jNkOfGd~x%
zvZ*e7Yv@sIuPQY=R%=%zXV%W>JzJ{fCm6mh*u1E(Axo#Ceq;8Z?T7ZqC4MdSSC9SW
zF{L0}k8dgSTd{SirRtU?E;n_eYED)iySqAkKl6o2*WxdH-f^~<UGHknrdMfGPps;4
ztTnKf_o?TUb9!B&C%$gI-o)Uo@81>G-_Tw&`QxjTrF(*AYfqlMCg^=s*qodn=N@FX
zuat?gS|n$2A*nPwd1dfHqrHLRE9Y*%_C;^?q0=HWd0&MxeebHUuK#NNsbS)}M=$^W
zSTCv(n|r{we_o}0(z85!i4`UitE`vZzQVpeRBgFde$S@7iaC}mQ?71r-SgXU;@uem
zmm^h<I;+3VRd<XvH@kVr(ezV!$?9tzf6lN+-jiZv6S*0%bMJS~=<`QbsB3NAx-uea
zwo&QX!xt0!r><O@biD7`k1aE84K`V1NG+B-QSweG({5Jc=j=(+vrKZF3;mwj-u9H{
zh~rq!p?LUGo~`+09{aVqfBHP*|IQGUkjj}pFY1lFx9Thn+pkYJwB%$J>i4Zr`_8FX
zkoV!XSTsZ5!R=bI+Ah|QZ;QRU`G&j5!Q}U|KbPDrzx%BAs43!h*lKC?{`BvRva1E@
z_BS6!ZEI*;pSt#P+@-^vJM=Fra;D9`=p^~yPodghz1i*k^tU@MCs#cQ=8=u$JrrQb
z-M{PO<%$a>WxUp+yjnr-MKZ2FzhySxZ)p?$b@h3H?7b)t`?IeucT9Y?K-XI(+uM2H
z_l-h#=jO7%HmjTe?%2xSvq{dMPo40odbjTFyURzDrCxTq`{?zQm0b<53Q})B*vq#0
zfNn*vsFwE<URz0{CHqX<vSSw7Bql~VOEUeQnz+-_?u~ZCmWhj3n2C4nJ^r#gcIWrs
zt68{@Y}~kE?sxW=(#LEp&!<~@YV)n-)jqvFyUhQ=k~7xVW-uPR`>Q6xzWU7Tl?SIZ
z{oc3wxQ3Ta>!+jH3(mY-^KY85Y2n;8y*-cI*Q~tu>!jT88%dKQ7r8`mEWW()_if*r
z<wg7Y=dj%=5M(c5b0|M%%01;~ecz+f!%WUU-YuJ0?@{<@)tSjGCpRh!#Ppu!bmIHn
z%eS!CVa<DAKSo9GMuFx0U*Dfw`ponU3wH<e@s<jeTNQumUO!@AKba>Zv{0okCdJwK
zW^7tv`ilH>H52EBxW3bqI@>#2=*nr^`z!Ld&3ip5{{7B#rt?1UNJ;myNfCN!Gb^HN
zPp9C*Jo!1d16f|LX1nG$J?Y5L3vXP1xG(Rp6aDf&xbVB3zy1Uv3Ex!-c3NLoYffCB
zxJT^wl(qWz^NL@;nAo{&zm@GjgD;iSodmN|j?8_t=;ks0RyV#KkGB00Tb2>GRCy`a
z_5PrpxhW|Px34b0d#-F{clPItqANabI%it`>7QT5{A)(l2Tx48+deyi_u8(){hMPe
ze`trKOuWBbP5WcXPw&YyUawx)SH;gip=a&n&1)_n@!6X%D=nh3?CoX!Esy4!e3-f`
zg?E+E)dO`e<V))&DwVfAKl93$b#YL2?D-v=<=!9uX7)DgiX*qC{~t}w8~0n?zkN%e
zvq@bd?&PEmDL?Aw?@F2R_U{fCvCj&xAFpz;-f#F=$@+ty`|d3<)8*t`Kbu9JJv{Zy
zmveP2A!m<lt*R2fZ2!X6X5NEi^WXn{wrpDLms7@dJ8#e1HrMTOvvz4vYh3xBE8=^n
z{py><5u6(>ez9wPqGzvhWZBk}t2=j#$SuC@7A~3htz7x^Y_F>;lD5A-Qyf*n?0dr{
zyy^q@zjaFr`hQ-KG7czr;WtXFF5ndYvPrye+0v;Jmt4B;r-a>pQo}UM+Tw=g<4?XB
zY>qpBKm2TSYRbyD|5K%2hZiY!KF+au?6UhvM(ypNyWi>WmF7Bj;D)J2m*%G1xjT)0
zR+lH`|2?%s_iO!0?~L37*DC+req{PBqPETQ&#{E89E0o-1%C&gmMhn)q${J#ukM*Q
zL-X^!l@{CPcz=FYl4YEz@bh@>$tS8GZ9dCr-Aj1!O(F5vjAy@RB-UIyylhhMYc=)_
zMm<mTXB|GKqgOQJpz2)Xw9G#yofB<?yVA`9KC4aK9nQ@4#xd^F@|7PKBno64Vw%x!
zKI>>)LY2LOt&{K^?<J3fmd!~2`f%y|b9+7c-fZ`2yzacb;KnXN{xhFH{WH|lFBMpF
zYh&ru88h2W(z<Omy4*@XKC)(gk!9VHDS6oL^0eF2VoQbA+QuZfa;*qkqV&s;Z~fwq
zgwrNB->ANRmo+0-QtJKuT+huD_U`Cx`<DCHA*yfHRh0`lYjdac`!IC2ul_i{{i5~>
z<DD-rw^vtfdl+^{Aj^!c<Kdg1cV{X*>DWJETi_&#5bn<Q3(20!e3CD@4*q<eTWEhT
zFLLMWnTszOuT}HBA+m9+&(xa<R{G~!^$Sz>Pl_!5G3jvE<yS{Hf2_JL&K7el<r-Vo
z@!AV1YhUbJ(-}Ka;*6ZV(z(bj-*yz-cFSC*dVfu6gY;*``QH+<8CbQM_K0s^W$eEC
z!G-q=&Gs)peyeiTTbo#KGqG)*i5K{6T%tZ(D|gniE&hJ|%bd@v>fZ652rG-udl_t`
z|DDb1+L?z-Q|I!3wpy@cNztcg6EvPbO<8)_=$GqBp{qyday~!uVoGU>{cY|=*NuxO
z``kbA(cj)&HmXzp-wv%@`LBC)=h<({e0u2Q+_T>MmcKRM%<mrl^Y&uZ@=u&8Mvkcr
zG55RI312hbekIOdY`0%m)sbqk-RYYa9WFf4cJ0n%^KVhBG#{4Vz8<t`E$>`Sqic>y
znva<hdb<B!jLlf(xSWOQgX7~O?*&nJ)K8jm&yD?M^H<qh;k9qT*_y%w=j%e17G0@O
zPHum`$nvC(eQ4^-%b9Gy?_ZpCv|3|l)DMk~j_RB#>W(|VNMBFAd_7b|!*S_4yO~$3
zymoE;T&ri2vupW(_8OIM^A;wG%q--YG&g!_gWCkJ;J+cy?w*<+yZ(ytti+Vi3vPCv
z$?d6&<RjNQwmR>-VqAQ0*F@e_UZaH?2AhuE?DKiIyUU@_Z|k9%fm*^6p+$P3vd#Ub
z-Q`(#67TA29NcnpS!h$r*?GVA`g{3rOkUf4+gl(>^vQdR0M4~L*FD%6<<s(b`&R#J
zKVM007A$GfX??wy)9abg{vWRYuO(bQZXocY#Y22e*zw5i8w=kaI_G!lmz3U}m&=x~
zWx0`^pryLY>bgVqoeQnirpu%6TQq8ZmTLaTw|tM(x!R(;SN58$Uq9=4;_hvk7w4t+
znaqmYEqt2&@?ze^TXSc{mRLr-idHx4&dm(lG%w?g%~=yszv2YdsQBityO-9zU~xb7
z`LI`~?~`KF<SQ@a#l4rAo%A}9QnJhA%7eED(i8kj4bJc1W$}+cJRtFyOPZ)%*5bOT
z+}ZPQwmy&Zo3M2vb9W!h<aNPY1NA1Un(nHOZ+kYGWyvAmAGce)g!6LEj%?o<uk!QF
zjoB9K&Wi4gDsj#2`}yPH4@3LMxBMnuovFchKIM~N?lPZk{f{|S&rh0pZP`(Fk9X|B
zGym#7x+`k9tL)oq`%`yas`n;Mdc9@#&E_edTDt>mr#5DNu3GVURh2UP3cs8+V$nf~
zpMQVnjW>H3c7&(U?B0QsOSAZ1Kl8gA_jb$X$%<C>JWJJl1&y8czBndx@$Yedth6^Y
zWq#F^l~(f;U3ovAI&S^icG2oXkG3>V<P9$Bk+VymlW+We_DxGpX=C9v(|oPhGq2fw
zoac4tiS7{3>?sjZ`;6uV9En=GVR`EH(4DFG3)5mPM2{>>l-wQ2@_)H(aelgd)h2QE
z`Ood%2|n@K-uE?j_V0HW<L>C`e%q0f+f#FD$G&$DJx#1vod3%AcTKiQ)AR5tjPBfB
zhO>V<X7DFmx7jtrM2L0k8y=l&rg!5^uhg4p-*mnoXI#JWtI5RA|M`@AdtND<=+1ho
zI%zidg5}ZtRX=uo@Hfg)eSUnk?UtR9@^XG_Vq$x<uCu<bGXJ?Kl<(rY(`DPU*1iZ&
z{;u;P?)TN}8_KUR%)H9E&NgjI*1Ln@d<(i)yS;nU^=8`1Xd89LyYt(lPQ2MLy_el(
z(=4-d&w0LGUl_RFZ??;kWv`bP#o1?`3Yzazne(t>!}s;(>&mJ&epwK9-}lF2cJ=pD
zW9!fUPP^E3?ar}xm77fUckcUpF8V6#nhT2NcMohROyJkfOS#z~oflJ+f84g9$c0m~
z;=X799rgRmb{9-m{kuo^{i~yH{gqa$_oo|FF8_Yn_T(P+-=|NP?DF{DTv&eT=GKi%
zj_q=oxV+qWmDvW>_p>+YHmmmDeY@;(#_3nv*WEg)crjBWUhM3-^o)x9=MFEIf7*Fx
zmX`FYo+oz`fBQUnwS!@+ukPIKR-dcbgj2&;OIdH+6skAr*W7wd30JR-JHM<r8LPHu
zx*tBW<*DXPX0_^{=K`+jTy`(kp7m07@9L<#o?!~M$KU5oOI9m1-MHs6znHtR^@kf<
zs>5|RmltpO`%UM?{m+l$Cudy0>$dEm>FOi97EC;NdDeZ?=nH8UqQab$!+sZDdnbQ$
z?S`~<?`op*znps$KDo{I^Om1d{}o-;m{+kKeeU?NQ1jd@Gaa_;+l;^5Jn4U5Z}r3%
zdFla~84ovaTwXh2z5AnsXCAyh!*IXnOwn%5awXI6zr@ztwH?j9t$x{nd-|m5t3NBG
zKQcB7j9nRUZu<3z3h(4;o(>injZ!x~x|%lse9icI+f`YuqUo|zn%U2W9ahy^ys<Pu
z%jAUrtT$cnE{DC&tIjODzxUMtX3s_WLF=~5@T7g57-I3LY47_lIZny{KBR2&UL@Kc
z8~es8>OdFk=V$#3yTXNQe#ceX_-hI7bW5HuxymfzTN3M)xvh)N-{_W%F1!C(;C`Ax
zewUJ*cKXvNc~Ni0u9<rMQ8!h4pIK_^mNM&#boWo0wMCB~K8mybm{wNT9hB@YzW3Wh
zZDE_)8=P-xF>gKVq&{~?`(GXT=Q|FSl($Ygr89pv=OS*$`LAy;+FT=6y{Yp4Ii|w=
z_?(nQ=jLSj2%0=Qf9Sl@*`9BIUd_r(K61Y||ChxhhF<^m_g+7nJ8%1H$I5BNAAWM0
zukH&oDYzy2%+*ORwy47YbKo?#Rr$WD6}yje`E|da-D>Qlm14y*cTLeH$7`Rz{qPE3
zbyDEPjkBh2QgxGLyB}WPcl{36o1VnUE~nlVZ1yg@==5{>Y~{;8(x1;heptrkn%IRz
z9o9a*uiBQ2`agHyG%dJwb)DOmM;o~GUadOz`d@g^EKjEx)y}JjpJgxI=cV}am|g6#
z59=>n()VZVetS!L<BFW>*He=A>jmm+xAsr$v)a7-%1+~nyvh0>KCTUZX&SU4b0$ym
z1gq`4n`iC4*z_zaS>O20FWp9uk6l~TSoev%jcq%+cF*#ihi_Kd+3i|g(zL`y{O*(!
z+h$BU8uKBn=BeqISb-H^!=FxT`sDq9!LKAd^Va3w7Y{jlT>QjS**!a1yiZ>I5T?$y
z=Y;&4|FibJ-L2NFbiKGdu)@vr%#2ztzPrv#j#<jFan5;~lDzQ-v&{O|-b;>#X-4O}
zL$*by?B00vQKoly_9b)n*6Q9zQNP~SsPviMUFY*p&w2l=C%4)+?SIhyE@}1oqV(r`
zC$F_wb7S2$n^ljNC-LmNXv0>M%lZ1(Jc)^X!9i!j_Eg+1*b^~zk#)45%cY}F9~C*h
zn11)dA(vxqYmY0u)NKyQ+JE`6hyB%WKb5{HGc#ud$FQ$ox^CxW_7m6Ura6c$OkmcX
zaNK6jPOj8&-8*}?uQK+udf(jeTQ(+&NxV&nXVbL9N4t1-*(O<br5p=&?=ZJ|TBWwV
za>4so(-$zioA>u<pZoCrzu$a!(Zg8_7M`sXsN8?t@ptME15du?wM%z}G4m)qPwsj3
z;K6H2=GYZmC)qZB`;zHjz#9;}q;!|a<I>Mrx(Uhiey;r=@_X6TI*(+9W0m5Iub%n7
z)Z|XSO~%RtQd7+C6+eFa?r!n2qH`j0)@Ez9n0_pFm0A<Kvfx%|+@_;<P8r<r`v1Z!
za;H(m!m{f^UoWjuc{H1ceMO1idY*ghWnDJxR+8RWdVc1~f?2WICsw?wS-Dk6I#jRY
zeU$A7r?oLJwFN5^uAO8y?%ws)B<-8<<_T5TW{JBRtG~UuQ+kWf${jOzuaww0Df`xU
zt{XZN_9zA%in4fD)t<Q}!{2^w&8F;_ZAqzDb9z^#@BiDoHL=wH`lP1Q6I&Ku{CIML
zPPVM+Y=`}3@v&!r*Sbt*cR&4W(wX(v7e$Ni&fCVAKK0U)J+uG1cUK3-m!4I-xOVA8
z)mbT8-u|}LY5C^2F8{r?sV=cxwPH8>WS7aT<?c<lr`2zXU3J*0=Q-=X*IYL<qmBhv
z?9NQs`26>h#B**%=K72LCYydU4S$?BF>lA+H~%i3t1F*b9k;RjZRN=i<*nww9<Dhu
z`K+JZKAYO=xf>T*%dD^4($n=eqxJ4>AKQPOihCty3G{xn>Rcx%kUKrX`CQV55(XZ>
zcM~<ItxMXnxbNb$cM8hEC5Lxg$86s5s;ol0uH##{-I<!0l^V+7GZR;z`LIQEo6)1Z
zbvIkL&$zZIt2eCZZ_)z41>a1&_&qN#+*Tu{n|8L)SJGlV_k#_tWg<qg{5n_kH-|_+
z)$mi-FSjgMUOn@j)Q+t`C1WI*7asj|_U@Opmr@0TyYEi<(%IT6tuD)(Zjs4&ZNGB<
zr5m&K#qFgh`Z#>Kak(>dRf)`%XJ(QoZOy-)wzFL)E};{spnp7PZi&I%CH?anHCvcO
zlNf7w&*}aAt9oNq*4~hP@xNYGu8dg8;4WpAo_bpHhrM<utD3E6y6s})bkn>*Lp{5w
z{Tu6^+P=x_T)NitTo&WpyZ1K>%INJ`x;4sp_nEApQ+{>Yp7ENn%_;0Q`%3PKoh#z|
zSSC&j&YW+1O5o<P-B&8gzyH0vt2VXh`1#H$uQjIS3m;J_KRU;U&)|9H#MvrNGmpG}
z_T}%AyB?3s9KY>Q5zd=0GgtWCqu&eWrX_RDUo7?9<-hH|$kwGf=T&<m{I{L8czgTj
zJmvN8=kV0e|2(Ii;hzo13(-7|6)YA)Q$uCWm;HJ4jc41Z$6HoQtvO%Id%x`y>x5&E
zR<-?RogyzD{(JYX(}zl)KFq5<xo1U4?BdtQ3on1}kg^F%x=~Xkv*GGO$FIdzA<o+@
z5^@~ZsdaZ<UbA=_i|0h`@bj{=VQ<26xI%ejzc>0>UU@ZpRcK`Blb>>zjUM=JT-flv
z%se7ts?emP86KW0%L|o4x5$(pZ9d@9u;!fiiA&;>tA2R~_HwVRXtiFr^H%u8b?gl}
zR;Rxds1#05vgBF3Gf_86GV8~lo$Ry3(pW5S<sOxKCpWe8nosQu>yJ}xt{QFUTJ&A!
zyw0~To=4v?D9@KYxYxDs(bG+{m&Gmpd4226Nk5i;Dira}W4oUlzqd2{R|x+ER*~D)
z(nY14<m7@w`!gz-A0}G9SZA@hptYZyrSPN3k{QzJtIo8Z+jAuMbnUFTz{(3UN0`5w
z?)?4aF1z>TtbeS5f6rLuh}jgY)Xgg4Sy{p~yGJ#C(b2WB?>{Doa-RGm{P}g(s@WxT
zR(x?@eP;T$SxXMu-?iOQoA2p;ag9rBl+ElbZ+5012;3gA{WH_ORdStq+3LNSYT8-$
zGcU)tecqz5AS}jh-So`8(~lfGGh?lQL$=Sdo2Q<CaLRC1Z93L7qx<;2EvqN(thF}K
zeENOae&0=3Ii4T1<k_wewD?@^#3KfQcfPz)vyD?cVDRzf^uRes8m2g&a4}}Sd25qz
zeN0Wq6OHEHi*@Wm&#Wfj&zh^O^`lE)K77u>>is89J>K%mI==VH!pK$qpMU@T`Q7>c
zn`63L_<!-1*I!8Ox_Q&(>#o;@l{c;)y=#|QZ>pzOyJFtm_PpHM8SNWfc|4Q%%N9p}
zl$<;_e{-m$%z?!bQg;g!RlE<c<1-8JSzxgF-PEcw&Ff#wUIsIyMQlzoIHI#_ZCOFm
z1B1ZkEC+}5%ncJ?%~h=Z9DL#1C++Nz%-=63q|PjmbhB(;ygx<v-s-@Gwi2$>XWd<E
z7pvEG`NZznsI{5B%h#%2***KN`(GiIBiDi*!@OM%n%0D!j@&r?c(<2xU$AhlU*qHA
zp7d8WQd`3o?fA0W@>j~a^>dY+!>xBLQ#<_RxlQi8{?HRTfuWnqOtg~Uw(bx)u)Hg?
zbAsR^)jd@c_ozJz?R}`%u`WmIvUB{y6eq@0|B~)=O}(@_+BwMT^X^A)whQT8vaV&z
z$ep@)nf8<nci&y<Ov|4zCp^h9ao?TN7_78+^}3Cx+@~EceQ-NKw@3Qd1&=_NuC%zd
zbvrn8kHpDYg@pg^Xm%``DzjMF`(NgU2Ojgv{g>_axHHG-tKXhQ&ip~g`XAgBslOBK
z5WewLS$Ei-X_4=rzc+h*?yyV4T(_%RZ;0RS<(nsF<<fJgC#ZG7Weq;zi$`CCWz2j0
zW@nqRT|(Hh|0`7%&bvSD_1*VxckHaczLY1mU3|{Mw+>HBFYU6?t<$@lrIGjem97vo
z=kuUi_J9AJ9Zy=dta#kBSEla_L*B3d*RKmqG*#T~xm?XXXws`GR<9*%UG1m&#qM@*
zD|N|t4DMT4+S{2SBEawe(@*x<%edTI%R4$(+gAlXdFc@4R#=huJnnL*K-9ec%2hl@
zesiQnu3u}ZT7Jtmm&as##og6Yf-FuQGX8p4cG`)z2lshT)suQ0eKE>icy*fn)M@uB
zEY9~_cCWAP+#B#mQ%m}2z{Wb++sy5bvCeUyE#G_UZoO&vYJq9(zG=H=%C7$Z&s=n^
z`$?V8;YwOEZ_6$+J~~_T(NCr_K`vf5_Ua4a3%f$UiN3ggAx@_DxkcpJXBM$6K8crN
z7B)u)YaTvdxYWHpe(71Qtj4o?d-rVL$`tB;c9DVKV+U=Cyng$F@CPB`kJcrfbmL~O
zeA#uWWBb-eLDE^?>bGMm6}K;Tvu{k(V}1MX()+J@PSI9;DO!vCa=$RH+`PXd?ct**
zpE=$YS8aV|WEOk$(W#5|`+l#=G+lo8z|m{xPFxLq?5uov)1qQY3ketb{*CJ=-!(m9
z_Vw`f6Vu++wXT>o$#%ySe(u?_C-2_LdnEU{_`-FAs~1$Sys`Qszq;%YQ%Uvv!vXDe
zs)mh*H}eADN3Xbi*g(HW|N7t0LenE-<MVD@J|1eV9Vg$j&!B`^BPQ#*UxabV8oBcq
zLtZ`ly1TIb^uz=A<E`FXPdfa|{Q$o(N7wa?Wlgs=Pwp&P{Dnb-?{4j``YoY5oL6V?
z>wmC*Vyn)7`I^7?;+1;FGyZSC^Oi4b*W4G2)iNvJgg*ZID(?RC)!UseZv9v~Bj4zf
zU?bo5X6|Q)nEhU4+*aR~lYQ-(W9uKuEsObA-I>ev^TF$KZlSx&&*{gXVLDY>%@$x-
z_ww%3DSS_aHh;2SlXR>1&AY2oPvi17q|Oz(UhsW>SkAl6vMa5hysN3~&Fp<DGVy|&
zlH}#s*K$d0)m>~yIIPYdo3(y*)`0_Bf(IlX=fsv9J(-!eRBEbLK%s5c4v*)|YfLmc
zUcIuH{IkPv_K~b<U(Q{=eMRwDKyc@c+`A`tUf#>($i12I;~mj?OMGW4%ybTncI7@j
zXWydBhu%DXB*T92i`S=zK6@V(Z3&-|o9nmlzD=dR`U&~-%zJ7+EGmpzvFLa)+s4_6
z{fl#}H+=bWdE48bMG_~gek@}NFm)8)=EBrvS?Y4~ZB2)F(ed4Pf4Sa|oj3c4&!S9S
zy=h9(cbZZ+@yxWm@w8xz)XlDM+qN7_WW48V>Nc@O_rP|Z--3Ao_T`hdPMOgZe6Fr~
z;o}Nx>rIkQQ$$uwSsWK@onvP4M9-+(YV%^V{mCy*W*yzGvAuW3)A0EVrqpgY5jTm|
zG&|2DV#leY9bV^C7x2tYovJA`=Tpr2BMEx<`1^Tw8%2G5@l?XlF=wUsoe58uT(y4Q
z@M4So(}|5Yx9u#tns(7`bJyW;`$y-L9-n1Bll<!1s~Ztt4(!mY;n=jV(eIt~zmiA$
zMgMI&^y<$$W>=#HOD=D@x7Wx}<F1wDC9~~?U4?h^4KIf+YM$=qmXVaqv%{bv*?Xt4
zbw`#v|5~?~+eBWfMCn{k7O{%H>wY$Q)!t10{=VaTk9_e;y7VE@#PiF}bX%K?`W1&B
z>^AP$IJLVgdq&ViyJyO4yJk<k=~Mk*`T5a+*K1#REPH8TJgI2zk&JhhF8p$@l}?FY
zKEmG>a^`Hq6P0UCCuO)_bBAl^Nf;Qj9x9rBL;JJD7rq@Ov%L;|XTF%}bL`#>wOYR9
z<7w@Ig7dB%eV&@U;G3-P$Ndq)PTOO2V`DjeRWBZtow3$t0-ue5VMpsm_RXuFzh9Hy
z@n+7RNs=1V=4RhoSH~ikZO0?K<cLh{Gj17!V^`A(=JTeg9_-N*NS8@Aww{t(zVpnH
zGtFO?U*NI$Y4XCx?9}qQJ=*=J%+eQzU5%LbP)B~2TIb<vzN(@E|F{pA4I)l!34K|W
zb9&otz23ma(~j4tFZ^{w=3b<e?SU%?zTdE4WnEaD9vR_yi*56Bo7xW&uJTsT7f$_V
zYQa0d;D(p&rkmdiXKnmt5&BYfo%V7MOZ&gGB(J(vKR$T8hT%l6@Y&EaP6bI@L^X`o
zc%NbsX^Wn|VlIEym&b3`Nu2+oxARO?*oyx7iYmqh&swveu`(W(I~V%uV|l~;b=9xL
z_CKEVb;HUozv(i0dzZw0KI5G_Yx>1%IXl1C4Z+)eX0KSbIb^v*iD;<tT!WxJis@X(
zwyk(6XlmYXZmv5qXYTD4Sqz33WGy5ANS|_#)42K0-`(fDVNuWT`~QE}FKN<yczuy|
zgi*bQ@2?GA&!sNMu4on1HGG!0s+nn<oY0%=!bTb{HU}DAjh`G8m%FO5p={E^<6V>G
z`(OP{S-c`ec6a%=gRZ}l3d292Sa9%Z=-kI_2d4gxcropd@FG!Nul_B09}gF7`BA7Q
z(>~4khW0Ecm8WU(O~<~?zAvkET-4W)>+$*1GlQqw{prz^adrJrs9H72t6lr>ky8b-
z8#*Qw?v$V1?P?(2onR-?IB%Yl$_o$aI2(hbOLrAaDCJvv(s9Ye3q0&=Kbn-SUAQHA
z#q#Hq?>4{K<ZSosuHpJsQor9ui2E<^_Ov{tlXxn6&*xot+5)<QSEmH=s`f3vD3^c9
zdh>^#%Z)2fwv}(~Oxu%r+(*i&`1Y3STiG{MC#L(?9NDSldHPSH^xKQi7?PzPG(8NT
z|E8zBy<^F_z@j^z9-ToyKW@n0W!HXZ%AGTF%6ebld8sw~>ggPA`*r8bZst}7&5t;B
z=Jf1kmp8gOPdh&Sd-Em}zO*$FQ`cDS^$1-otIqlG(&JgS&#ig=<n?S@)-jyznkppn
zSd&|~mpS6>VWa8OPo6e$+qhx>akiFiS6(dAnY((~QEQ_KYHr-B-7_R_#YOcl^Xw^c
z>v>|T)8j1HnmFssArqZH$#)Xv6hd~Y9{crn`i==b%AfDdD(hogfB(5tsng<_@ol@U
z?o1c>sGsjS`RS|-*IR`r_>|gL-_5^qF-O-lH==)sH{XxFn|!mEO=i$6V-DT5-NUCm
zcC%8ic>hP~k59HcA7wG0z_mT(z~b$~I|4RLp1}O!|AxtTyG$5-%IkvOrL{!O*>c$D
z9KYr#E2HJUXA<`6M8zs=zkbK(t}}hlytf<P9<g2A{^~WOzQ<eh2&*ce{jRNR7JYtm
znK_*!vhCjwnb0$9F3ed~z^Qn@*TZFUkWu><_g3alj;e<)yw2GE+|7E`KB?84E*ttZ
z7EH`vVyU#c{m0^M!nS-NR+EpGpH5spRfT<%^@_M^6*fC_BdO>T@yqU8?|n~F+OdPR
zcx8@KyRM&a=&_LV5loDyyUp~aIYYMP<o)z9-~M<*>Y1NMRyM8mKb2VWE^B|yqwMs0
z)f4|}wOQM*zW!9!{m-iLoQCsrfp6tXQGF~kLO0&G6OEQQ6U`_UF*p51xA4|ppN}4f
zt4=)$Vw?YchGwehEz3*$|2}KuT*z`u?eWqXJxx42wodlu43UqI^_A@BRLzmk*%rF}
zkZI7id(E|5Z@$xezHWj_(XFjTi`3TcczgD2o#ne(0c$<hhn!6`d{KGPXGTWric>OQ
zF3W}AIGVKo%(4#4iw2?Ed4fGF=Wn0*t>n91%EcqvM&2q$8!y~EU~qe$-<_(?!pkg2
z3+FCMTfyY~V%qtww~OR<v41@4d98AnMI2X6&^pheBPS!C{k=4I!qW?T@|ePl&Hv6j
zShQp24T-mPcja!JFH8)&u|@0SzSj@?6oYT+h(Er)NrzAHV3x#WmdteK>2;cAyU)o4
zPx%+gH{U7g=ck)@jV?GR?6*!nY3{x57K_7WkLRMZ7VLWbZ^;^Sdnsc>_5F)HY~0VZ
z%&}P*^~_78y(vi}(KvLUul6pkW#7I(c*1FW$HyzP?z!H!t*e{oO=*_hHe+I4RXV>g
zPg~_e)#=A9Bcz`1o%=E|{YIIg>SLL8>t8?pZ!P-GZ@I$54G%YJzq0x&b@D*#(=_Ev
z6YJe;d$^f??$VIF@ISrE_NZN~>Xj9<KL*^sSQdRV_nE{W@6x~ezcTJ>d^bO^J3QK3
zr2KA1;(DLLsP_$llDBrw-%{{2Y?;>rwaqv5-@9gWs~pvnE1PN7Zs)IBps3M!+PQJT
z_w(#MF)>$H1^u`-O?P34&LRtsEVpNAPG+pt(|upOU$SqmbmH`9k>QtDU(08HdM01@
z`a~nm#*1m0fqZkt&aVH_>6Z|t{2_B{)YVSgtc(y<@w*S+*|bG9XO$fMIg5GCB;h>S
zCwqVI6^LDTWoxk4<VBoy>~oB7&Yi4jlvVzs?e&Vx_?)O?I%(cB<PQ29+~=#-IlDdX
zUP{S+#SObYo;@S;SjKLvFw^{L=d+~+zx>&6uu(Hu=t<44&aEH06b+i+Z}@j{*A}i7
zd#mQW(<)||{4<Ml`l@wG^Y!w*d9J1yz5J$io@c#H4cFFBHMMSE_4joIbye?+HoaUM
zTc+z(c0=k^K>8=wHDNOxR?94Xm9*lh_~yhHU1A-ki@&~B+5D<x*MnBSr*#ke-j*HM
z|Dr1P+@gQX`is2!PM%!Qa($gZ@XB(lnCK~;9tY08l3X#fZ+oB1Bi95~HJ2;#U4Cau
zgg$QJy6idU*%aH!vFUqKFK28ylmEnUPmkqCmp!~Ur!Lmc=I8AEkbIn(H6-<Q|1`B^
zsTG!Yk37h;kF+_qrlVF}Z>~g%%F#2i-%O+x7yrIi_-)avz{Wk(8FjDi)xWQm`;EKV
zde5gY6)yH*@92`VJbzw<+Gn)Ke=qEJDond&7(MBx``&rm_GGVpoOf?+#)?+!h_yRn
zOPAb<*uJSmHFEmfC(~26UdXI_d+PR`kXM;ucUUG|Z#;Z1;#FpW_7@p#(Rn=Q3kx6Z
z%zw|6*vga7bnv=Iq_x);w_@k@cQ%?D8D6U1KB;!eC5NEJ+|#byd3-qk*lsITLErbb
z8m_#5Q$lrQwPZ8eUi=X2urW5|?_A!0q-loHj^YE}8*i1iU!E6pY3+%At2?!eL?^#?
z$g6$4_VUs~-tOh=_}NTV!+WloNxd|+3_2WiYSG*Koz<V*TxUN?+h|+;X@#wK=j55U
zm)+Fc@FTb1b+P*GgJ15ql*##8%s6px_KEkCdRlK><azO}*3CqHf3nqsW!V*BFU~zO
z?_kQDX3e#vHlpvCXor*XqYabpwx8P}m6ow7Q-1mNqXnM%i@3jENL2r3!^@C-zVC1E
zmgl+0@4Sr_Ja_Z$+-*+{%Q_q<G{+aZ)ui5inp0ppjjw-&*9y)vjkD%tNUbxrFZWtf
zdqd_V^X93Z7Z-}BCwM3;E_xs$#&SV+&cX>b2G_L@FPI-E`(TZl+0xZVy9`&Z;#sHD
zZ<#hL+1L8Z(ue7#Rud8|A3dwwZuv57;^EiMeSeOcZB@H<A!uWAuj-?nUB|bavf+Dt
z@7dFu$CEzh7*9Jl@APrQ_;cNyDStKgO<4a|^84iA%(Uj<-Lt1|pLoA<W3)ibiMLN&
zX21Vi{PKEp&-&)SCNiQ~E?iG)O$z-=rrr5sW$fbC_Kokjsr30Ty~bKIKHX?M!f9H0
z=HzBG`Tuclk0N&_`6wTpGj+G{vgHdGy_zf-8}=;o&GgMh*)yh2__l`cz`N~x3>8b`
z*mGRH{}|aMwMFT9U*5zUnRK#sPFT76_NW=9Tcns?b)T}IH%V9Wt*e>cr|8o`NmGQs
zE;}V7Wj>|kdV}RhldocVPm;V!*xVkko{{$b##brUtGp6lDy>3o^~JJ{vc9eAb3awu
zy|wMki{cq^CXx>f=T>kkrml)OeW6*(e^RI6me%A8?unwBak3vzz2-T(!_)I+t;_Wn
z5t<E`(-Rll+00p{`zYuR_qPQft7Jn3yCpkb6=>?7xHf%duCC|DHpVSQJNZL|Zv}0e
zCSxdk_~(b$Y5OkK>{E9*W);gHxG7Zj^4d$0`zL8Osa#n*Y0vJ7-+%hAjCAd4y!<}i
z+hO*UuYP&mt0ZpAbeTqN=K4PQ<VRCY&Z1eFcN)~1<-g2Zt@UjCE3=tboqOlpTs!ZN
z$J%)luVt^<cWA1+XfJEt)DW9HQc|AVqrV<Gv;Rfr+|G5^-j(m#qSpSj<kZ$?rJU3b
zCo^;2wfWU+s$velJGXOv<($jfttA`2tl%z9y2RrxCH?8GRMj4x)#kN6yJtmA&gO~V
zBDwEsWa*CIf)lGv7MduBRvYc(KGk4rXx`~!wrawCecvPt!>3LUE8U%M-uiw;?Bml(
z=~C7^<L6I%;&$-Sy%UOeh5z<UF6aN_re7{v=4HJsZ2K9_mtR_5^m%^%x$Cj%de^d9
zQ|8WCl2mbvcloUu&lU@?FVNI7Dv{&)+jDig_m8``OlR%7+&|-HVnF!R#cZB0CLZie
zJ8y0!w5y%JBk8{J!ubn2O%?|VF8==G#|-D}a;>>b|D-A9vxK^>K6vW#)wkDfiBJC(
zdb4zMXy3b&&*oVDlwD^1-s(tG@3Dg3HDNcmUax&MedYhXdF%R?o{n|tJE`+2Z+XKV
z-#ulf@%et7H$!Xu|82?7@DrNsANc+<*Z29cCdWBTkFJjiPFljO`+c_RT2t#Cf7zAy
zTFp3`7@{V$KIMeOnXLv|j~B1ve>UgJEq|@n(%ohIQlGvxeS2_)`Jwu_3u|oO^myG}
z8p%FKiS6(nzLbIxwO{LYmGD2C7m>S?P2r4r_C#Oj>_wTrlh6K?mel?3ULf%2+r(KD
zzpQLwOaGFhk*>5q^GLzuGZmBf#2XvkcG>dw!^Y~TpX1m!nMi7EpH;d;_^d}2?<Q~G
zLsM1<dVLY975VUTzE$URzk}ZLd*chmKbXZ|yu0dar}O62472_%lKykk+{_v09rBtJ
za@6T=*V^ulGqyUt*?-H=oZ)Zk8}YOgFQ)8Ts=Sta^X@lUnOm3J`<#^OzPWAx^$kv?
zX>LMit@f|}G;{i?xY*VE9`;C{U-;*IjmH<hWo7eAW?rr-O8!5?eEP$4n`L8HI4@+}
zbWe2IG$a3XIqm7jyAne-*Xl^}Z7w%gTbem7cK6Q^lV{(r@)~A5TYgGx+GmHvmp+wO
zRPwss7VL1|^Lp#%`CpHz&ki*&S#0U@Y~3NbT@{Ds?N+MR-kjVQ@ZqX$>*xFX+L$EY
zrb$Ly9Xx3xbZF*H2Knk+$35-)ZSFPbB^-3SYICvT=7ZU>MiDi4Iy|q>T`j8XZuWlu
z+p@G>DL3AJnZj=zCEUGw;+ub8tID=rvn#1B*1P{kkvmbMtdmbiGkSyG<S_Qn3yqge
z7uaFF{?xm&W4{*l&$`~|esY3LqF>1wo@ZC&&g|FoQ<_|K<G|N1S7raN`aE$@*<QXC
zo2I1nE@Hm%=i9!Z+>q>Yd)B4X^YWG_HpFh8SD&`=_ssYuT4kD1@7~n7CVKD9RPW{S
zYF;;~Q@Hb*dyO;qLYqqdNv>|H@9tg-cz!1G{Q*AlH+OlC%+V8Da`)YrE*3T`R?n^B
z!iMWlUT=KvuJ9*r&y%KOF4yY24$HpP`S!@_B#+TKv*5R4T-r-g?yu`~IxAK7e0F7=
z{vJ!0t(D)7l(ns1e|y&N)yu@V&KAj-Zc<#kn)i3@EZMS&o6kBtPCWW<K??6*j<b&n
zW?%H%+U>7z7~j8T+uuC9`gh^`lOFy{D%))8Ip5;VZNAxMH{Cw(_~RM;?WEk6uAR)<
zIde8GU$j2_-ugF^FLzgmZRNVV_2QZmo?^K=8Oz7_EY4^It-pOT+{OF7?`+4EDkn{=
z;)h!n?~-3zv`NZs<$}X%ZIhW_Z;YPIV?V(u=vH9N&)y4sr#@C5cp0;wVN&$#YmbUQ
zy<ess5+s^xyL>^I*1RV_>zo!XzM|#7d;2Nppr>lQ%95^PA=BGSpRW+RH}z!iERJ)h
z0+(Lt-sXSY=c_ZHe_X`dLzefde6_U>&04kGG1>2zOKeneeMVu_#?2Xx|LT+H{oj7*
ze=FDP)!%-aF1*IR=5XXi-lv6+PVCvV>gCx#Hn%g=mRigy&A)!*eCPH3SH0EhWu6PH
z5wkiZ9k?P)Hu<OX^n^uSYpwWLzFN$YHo20t$TMx<BOmGea^d#tewRzxR{#3+D)9ND
zcbBXMdwRb!{@PYnx@y{^*`duJ&pp~&ykQNm(7C=>oX;c@;#Dtw-lMQHYyFBT9JeR3
zY%}?FuB@w&TS?-|=Qc&P5W%w>+qu68aj_>(bIko&=5gEI-)sA`&rB;HZ8JIjvQNwA
z>W&Sro68ox$xq5Xxlg$1rIuf9k<>)%TfV=OHLh*3IsW<Iy6LM9EVW7LixO<(e|JiC
zKkwuZZIe?<(|wm6Nb%=d_03+>n)CVH^UvyEGJUI$a=gCguyOfr_8rQ%O%2|**!H@c
zZ<zcrCS%{h^*5s1q~ATvzGZv(-oNiDp;kZas`oryC!O<-i|@;-#jC9IukL@Wu54$f
z;XRR0?006-g`}!Wf|Kr?{C@ShWXWaE9EEUEH{P@6p09qG6$L)M_2$#w>}6%TmzIAE
z6xl9ud1hMO^~H5tCo`YiV_DbadHlwT@1Kim-iIFXY0uS7j+mTP^6kxD_FH)#?4K+p
zJX1crJ6}6xTfN(}woJS2uOiJw-Uq*rE-F8@bS`^tcF(!5i=&SC$&~GgD7>W{JGt)4
zmDd3eviX#@tc_d3@z(nKCA+Mc?yNa;nisXonC@)&Cl<Xq)@j?W^D{2yA6k?t@BYr|
z;{(I>qS>Cy58T&nz3OMVM5$@!-{Vi!517c@RGsvuFU-IETjtTj+7<hrImfUiN*&N>
zSfTiVPv}YBoS4YR-HVSFDfAqiXLNh&mvx@|=ZThV|K8`hOWv>j_aBYqqX#{{*{%Pu
z=kLpJTdFP^YZqVkD=wOw=9#zUtl8<4CW+bU7unjYmd$#(cjBQbGnUu+cLx-`V^)dd
zdEy;m^l<mt3D+;AM+zICeX&_&2LEG&SEa#^oj0slZtB@=o#(O2^`6h&{3UG%&n}rg
z`LX%2XNya1g3R}(s9Am7uIx1bjfCLG$!oP<st3F%44#ry``pF6FSfhw(;v>xZ@Ufz
z9PB;*s`7j6^4O<ppPvdC^RIeh%D($|+jQH?m}g6WC<=#s?O7@&lX|7TIZ|FAs5IK`
z{29lMHD_jTJ<xJUxKn6{dHgC#jVir_=UZy;|224TwlzqP^Qo!-y#g`CU(7s{&aO{Q
z-c<7F@j-`(kH_kAe$28xD#(68Xts`W^$C$14#JCK=VT-tefa3qa?S;wCI+n8wQjot
zi`-jF6wVwjd9-@hJEoHw#fN@u(otbO(ew1*#Wzicb5^=AuU_$c;u)t>L;vra^wMLG
zK3W%k+2M@(OX;!)?^pg>eBEchAKP!a&<(*mZkn!N@_F&B_MZNwdmgL)j@$O<L-CU7
zCy#7>v+H5)jQJ<)g)(Qk-t8)xYb&!f+IFt|w3^}#SGHHFiM{OH=Re7|JLt$%KAp=q
zH8t;CpM3aJ*?Lj!pS~KK!_}X1_%fJ^9*b5DyCU)EWXIy20W;mZ&))rOt^Vr9!m@wA
zegE6`ZMc;ovn>1YVg@Frqu<{u+&K0_$XD3+^tKZUf_XWLyK<hrn(6fT%rA-XN3$N<
zu)naDKeIHs&O==Ncy8yS`&a)&MKk@;ocFk6_s-kR%XfEf{{C(Lt`qus;dfa4UE*JU
z`kB9d=cc`l%;)|Xyw{xko}s~Kmx0xWTyv}FQW1gj%6s3GBG0eMRhFL={ix&Gh1n-B
z>^zuoJ}`dzb&DBG(=G*=%|1ByV2}C1H#&|J%?z)7pRVvlq}D|IER*8llmzuW@5@JL
zw;eqqzw^>t+4v`=1{u@$E?F>V;>6o;?_6awIWXOIcYU{#$ZG?SSbfR5(!dp_hv&`m
z{Wm3ZVUI{yTd+0v<)~W=R?RcJ-~GSS>A%4j^VmP%ejZ7*VNsvKZ#%oU-BOlm-*f|C
z!v%WN_X|6}yC@ag{;PFseP8}g_w~!qUK2E#@!KivLRI|3uv_|OCYFJ++DG=qZ$JH3
zF?zn7ct)%B{Jio|$>Vb7Hh+%QR?e8&{;*i~|E~P4i`>>7t4cNR{i-|f$t)?W2amtL
zKQ`52;;XcMR*!wMO=iZGcpl(N+HPHZ@6U|K5C2F!U*q-FCUNzd4O{Q;<kwuMU0zdK
zvB~tP=ixY}zU|A|u5b2<l}vTNdX+b7-nxn^N7Hq$R-ZUqP~f(thOO&VV{=!?G^@?(
z=94Qwe=mBzQ+nBL=P!%0=FH^rYxwf2Li_B8Km1{r|A$@pUy^yx;S*D9+>8$?8<*7V
z+<WF4-&>=mn5IO7_xlnj7+jn6M{R5TG?$sW(-!tlK9+m@<XOh$Zo2~Bw9m;?6lR|!
zXq&rWc9Dhu`Il499d*_FGcDri*W`JgCw(^d-dsEF|F`~==fCrM*}i+Toyl6KZdq6M
z_WqA@>Axmh+No#$Op-Zw^qk?1HxXvfzsx>i_d$2>+QsSL&h`ex9)2GkT=}G8t6|x(
z80*us-T6`?S`J^bKA*z*`u<V3LYu#-&fj-GU3YJdV9O$_gLyr_tkX9}9`Elz#e2b5
zuHDczep`p&@|K-DSX*DQ?@)g0^f~vAzxL6te2ajkmn<LOy}a?>NnZJ|EwW#^EH_L~
zQrOKgp;2m1)b!cLnyVRiSFV2ZxMtzh=L;=M3xjOuh;Ya}?^|d0e%Wm2?b7Clj_TN4
zu9fT!eb5=5DYHEJ%e{*+#X&X`*{*!_p1+HKNk!VF$$6^ZZrd&Y%9HeKPoDr&S<c0i
z)!IqtUK~n`Hl5rz>#vUsQ~iwWl;08uH{9!<GpBsrx~JQh+*$DPROR+x-=f_Ah39Yl
zAH}-v?iS%$>;JNOu$=L_|N2qX1P<;@9@iswrPpK2XJ6lc+|qQ}ZA-=c$elL}c1o`N
z>TT#V@8a8F=Ud{%UTO-*D$b<{Z9825w4l6w^?kiNv2~m7>CX;H<~QXsd>eH}x9sab
zxrnWuD@`YNJiC4UYF2xTjn>qP=w*|&gW`*SJv@Bm)z@1gcNV?#cy)y}?Mklug`(2X
zO`#iOoo2r52}{nlC_lGt@v`u&_HXm9*1SA<LgK6B+TP1s9#v{AtvXOXCrA9^-6zM4
z7gdYZ&YGuf?OGxmrR$V>{)DaSx(#cOIa*4yCTq^pd%4Z7Res5dxm`6Cy|=d{oVc_6
zxa=2Wk;ij;xmVxm*ynK2WXFTKbCxkCwlB~+Ab#1N?K1CS&xMNYXVPB0=sWcEf%Kud
z%*EzwB?67bS#R#$=i77R>ZGgZ!en%e*1BwDn57iSTw8o<>amKe^UCxhKkt}1)!J$K
zoLQXTjFQ%$4O_jHC#?L&pO3!znRfeThntp9)t|mu_`AU6)orUS-)^X1^Lc^!#)O)C
z+n8j`#lN+??RfZZ{@HhCceMG0UrZ`4;M4Q5PM!bly~sgViGV4}Y_Ez9XE2-dOWa#C
zzo_#C3kTEPH_1C%7H2pV+$mTvZS&ob`Let}G?rVRSnPPC$|&lis`F2$s+e5S@Q0sk
z`&iv?d@SHzTNae6B>m-L+q+M<mhG-jjJg@ZYpZ#z^y#t}-|v0>znV)VK<|5%sHR`3
zzF_-RU%iMaMyGnK|BK4#fA;!5_j;}2tPs)Fv%YV*8awf(7=MJn^RgV}I_a%PZ>h=z
zEp)zS?zT{t_mS44X-j5H-SMmWZFbUT|1T{`?Kqy}M>g(U_9k)V1pSy(=XN}Kq-9fN
zb4-k(p2LFMkMm+!s)K_1WsfrNC3UB7GaWeZTm9PFoooMIN70j4a@{=Cez3kN`gZzp
z!kkYTPkdcge7!C{d!o}58y0g}_h)jsahFvZW_;V3zIa2o;8$5A%fIE@8=pQAeBYj;
zpkTJ@_u-gjA>DzTVl#y9Y!c&*)KAjc)4ujT$8zn<^PkJD$~&w)$<8kAm9+fxaKRV4
zGv>5>Fny*{buw3Tvg>mG;`D;YRXb!83QFdmJe>TAZR)AJc6;8e*XLws=~4}>DXCuj
zPWqWbHX}pcjhn}JyDs}BG_{GZWcKElZ6AviFYY*Ix8C8%g|*Y0j_%;+jIF<+dH2`y
zjfb9oc#&wZ&%OBR!&7`O6qj!6SjG3k^5y>EZ+qs6@t(J^JO1AD(cXg8<9@B{J2Xsn
z9&BFmeL;@<{R>kox0PnCJoVlrKYV?9`GtetW+&1Nq&QW~1rNS{Wp!)j(jKh@rD-R1
z^cUZK^5s^WZY}S!I?1yybSL}oILL2)a=W$gUYj2e(-(c4&XQM<tT1=py32vj?4KtF
zKNHHIU3vS{E6cy1eEVh_O}9+`lNKbkM{~`ebmddqJ-@tZ|5g|yy4|<ob%j;mm&bRF
z_~m|ndlIuX>s?p0VV`<;wTxS}iOkYQISC!RUsHdn|IfL!`?{>&<%dq}1xt3ZemuT`
zZ^n)O%4Y78N_!2dy!8G8tq03zv?<rs&MfnPy4CZb@uT_I7G3}N`E>S?G%4M=UEPvN
zi(k%q>ay>M#wDMXX)jh8wyaycz3T6k^;7TPJ^c38GnduwG0r?v0Zwy7bWi=#R(!N7
z{fC38=7PPmL$*A+&GF+R+kFl9g@2~M?DUBJSRT%_d;2~|^W`7%cCf2`{^#-c%eMnJ
zH!GgJQ?yG(|HoTV-Ag$E+2xrXHDBMY{bBEM<JsfhPZg`rTo(U%{KKKg1-+-$=HLHn
zduU>wz0PFDhcV}NU%j5T(jZ;h`0(b9wHs>O=CcdGKJ@apYSAM}?O8lr-Hx`mmt+XO
z>Gu}&auT>*v*cU%N#DNq)tQgk0?L-`IiaWW$uwDZZ(ig#rR>z7mc{(5+5glYDav0d
zb^UH+`$8XWzHsOL?;lqGI@~aA3-iGbim!GU-c`ynna(!l{*-vji7_|l1pE?>`Bs~2
zCZolD_FweW-Ju^+&EJce3APE?_efaky54=6^id(OI{(4{BAaLT;~ZVnibEDSe_OfV
z?v?lH&G)?~%hYWbO!c_FMWB?OOI7tf-`vNWqMeQ=EJ=N=c3;ZZ{OZLJO}Qm&C7)PG
zFZp*b=}d-7SxHaIw_Vu*+H7_nEXK1gUQ%7o62kSw`sI`imG!gUpH{!+lw<39po){t
zSW@AUZQFJhpNIRja-*$oPJX(o(st6Jt)G7R2i;jW``*>LPO<rddv5PsFaGk{67%>b
z-8*5YZumcRoce2~R>sszm5FuC)iY8vViT55KM^jt{rS~t8M%g@du~-$dp+MP?fv(~
z+`M0pPYWz)nKkE#-@I)Vr~bro?Vs{0p>0;i(GAb1%h{`{q)nN)$Ts+|-;;uEhcDjU
zx~pi5S<mLV7cWg@*FJXp=|A?!;<Ef3v!1Nhiep(eb6rLIo0<1^t)0OTF+<r#WPaj-
z@E2-RtT*4@;aVHyEBbt5)SQyV;(fmllz)C>csJ$$JH3T+=O(X9U1nKw*<jg9#xtFh
z7e$`za-Aw#ETQu(^Yhl2wXQp_{5~+XbKm`yvAZ<m?rt(KagV#9Tc2zGc4@TPojj4M
zyDxtKt67$1x@q;rw|QYRt0LIvwubeUXRqx%8ymZJ>s7hZr?X~!d$aPn&87SAZ(Yt3
z*EU+;ntW#VG{K5nZ|=Nn+r8;%zK^lK|LfzKbItu-Vm(&IieD?7yX2y7^qUhgbHB&r
zmy~;y@!LD?d9q}su-Imc+$lcNe3z#*7@XCRoWs+%Ec#}T=2Mr9>o+4}ZZckf$2RfU
z;e-{fcPw{rvcGrd=ryfwyS|6Tw*p^mUG((Hl)e{CJ!k&!tx(!vtbN*SO2lIR`+Q#~
zZpl8|FlDdY<eI;4Ue#wBpPbS-CpLNVdyew<7bY`bgsfe`6rz<EulbH~`|s|Ub!`?e
zEbq=;vF?q?v=^pxo}Efj@Lr~T{D@-1rs_{01ruZUC<T7@J$dZq{~gMgyPhf7`mB`-
zn?5t>1OMj2=bx41_2dE?WuD(m+4%gmlmFyv_b*<JH-G;>?8nk3t=RP}H4=YLE}2ys
zJVD9k#woXsP47>?_;2jGzQ0H2?P=3|*?FGZUxX?>*tdP>(^Y|;u9J1^ZfMD7=D%o5
za`XIKY461%%p5;e!@@Ns@~V`+N}}ej2|jl_XLE9&Z8-F-OlZ5MT#NmtzF#NbF1z1t
z>XGK28ZV!C{__@LC+<I*TOa$}da<;AtM~Gi`vqdHzDtWgmAj)A@#pLGUB+j76~07w
zN4!qAJla^)UT~mc%7gX4Ufr+UcXIue9_#5x=1h~+J@{y0^66ZilF7C%TNPWfcz$k9
zT;0hs>9fgIuQw98Pq(O>Y40-3^Q~GH+!<0>z;f|^GF#=WNR!B^vz*O0A5wd`Bev~u
z3D;gZ1I9OnIcpF1cdqX@U*qvfD%d7z*=L749sTdb*=uW8KRKLSH~;3|%+y__FK)bF
zCcJj>$t54dCdVhbKW8f5S>SQGH}Oj!r>*!Qck?ffeh)6%J=gJhqBnJ_@z1Z(b~~y&
zf1j@4>q(rHZM<^#&hlMPXCBWf`u69r!s)HKW!|z6zfAct)w)LBEWG2z^|x83Gh%KD
zE)nPEdHq4-%EGK&A?5k+k0hO%@G!Y@_vFWsym@DRj|t8`b87;_&pSJOU)=forrUD&
zOVik{&KE&_>w35SoBqr0-5cAUKpQbxW~J!DEjej!v6gLLi^|S#U--e~`z8G=H!m*T
z^K|;Uvg?oDy_)}^-+Jv=5%==4m{_Yj-`BkBo^6<OEAdy`vhKIqxwrUb`Ih}Oy(}ji
z{$<CSbSt-#kW=PGt8VVGJLkOYUdnY9>7+LQWmS<MKIY#2_ugb?@VQMZW;?dWBrom1
zxyJaX?%P!wg>Th`zrCLIonNaUd3TWBE1z}mvZW0QQ;&F-Om^+x|I=0U$EUsdFE-9&
zd7*yn?s;97^A@r@lgn=1-M_j1pR(vR>;ICz>y_>mhumaac}ApurpOJ;)oR=J$b1hu
z6R944E{pTznL`sctGPt&`o?B!d|5B$+}b%;m(ARKOR%`@V3~Nm|B4+sPo%e=og!Ua
z#wYaK=gOp$ziwIQ)F!?@zsIDWFC=Vz&FoX_R{YEex+Zblr>e`RtZUBI|2D?A54~EN
znI9OLrBkY#d+Dx@o_xPprRQVSwew>)%->q&^H^@liRb$`Rb;$`qD3#in!aP<>-`Iw
z<DE}sohbU+nZ4qVf6t%)VL#+oPq?-w+W6R%i(fbXuX$&`=+gYoIZbmXwX$t`#b3A}
zyXA?0)I}Mang4V8C;fhY-r$$=R58ogYl(^pD~}h)|8@TN{@aDB<-x@fwcnhb|L?iF
zlbyTrTl(G~=c_;eUv>R&KI@<DZkf^dgEOA)t~sRV{@cpxK#X#GyNulH4I3Q%Ev=ru
zh;dK8v9YsA>Cv-5x6^%PT^7QHYNuwlbPM^Z&haYZ`W<*o(r1T#{k54*-CTZ&n_Oq+
z3f8!brfZ!GvAkR$c<kkT`y#J6yQUEN_3|J81zIUAVprOwu!w(A$)U6=cXO97&Qtn4
zUUam82o+hzJDLlXg{+i3-Bn6loIu2+K95U+oU#)y2zpv6f#nt|PbzVl1XAAum*bS3
zxCA5&(*iee;u4T?U@dO$DJ3p$?kO)jbYv%96qJ;mxK!E5N@=mOkrmT%#f2#)F891Q
zfK`BrnDFu<Tg&FY6PG|r!Rnu0jD)CH@`jpK;xeu8M6fZt*~ZK2EP`nj(f&L3m^1Da
zVw$+=ftax03*QS9S}OW^?n|ajo*ew5LrFGplbTu1BvZYE>kYbJ>5DH^K6S5siE>wo
zi=(?piOa-3jTaqFAVNttaMv=+oJmRg2UksZHJs9YL6B88aEbD%D_gpJKW$ZgF*{Pk
z+q12sZ^eraE!n`wemolK7X(AC7A;cN0uif9T+Uj{E}G-$9`drI3#6*6Z^ey?ug}fg
z5q3aTUoS>zpJl&FRz^Xw_W{GJ>sN5ashsn6{&&Fkh2X!IunDg?16X4kT5j$VeXz*+
z-vNOS9O}i*!e8eb^D?bwSmj$ZpYx096aF<E{VEYztUC|4|C!0TDoc2xYs)*udWLTM
z3YI_XCYu<Zn`<jQGtfmv{u4XHd*wAzx!o_E|1r!GdA4@j$C--t4R387nJ;fra9{9B
zG?2l0(a8xJmxP6O?bdS-_h|a4clOPT&c1mQR?N|}Rs8ZUu&7&K^6B~csxwL^J}av@
z_3LtLQiR{h3vXt-7G^fYde}alv%*)FXUA)Q@qK=ls?Fgm_AgG+n_evXTqR+Z@{^lC
zxxBl@-Hg9K)cYqT`^EaTzxcI)o$P*!Ql3T(<qs!JDHd9Np)qK!eQbx?(<u_a*q<B|
zREXs}Gh4CN?vdK6=uL0t+jIWhzsTSUZ>MtglaNE2H#yrg_TT1oKbJP=Y|%Cj%eO(T
z-8+7$sJKP*N?r1w?U=M+HruYuLeD7=8&~|l{pf7H_Mh#C>a}aaJO0l9`~PJ2e9r}k
zY?V%@{&&2R!mN5@v)ELP|N8UI!>_iV_<X2zNy?MD=*F4|p(nQQHap9&`~G)x*QJIx
zT|yRWSDXVUyi&RF|J1*ovMv8~a$+~gIo4cT^*#N5WI{lc!vFc^u@V<1Z<_P3BTrdq
zKZlDF|A}|1CXx^0Bce`P%?eZgZ&UDB&y&SZwSHcRc-n>|2km?RZJy5%_`lenk^A45
z*&Jtu3i%ezQ(gXdqU^sba<3~UZRVDV)Bb*```?!xI=hvgcHdm({p*h2>Ti>7?0Ql3
zy5iL9A2D8cE<OKelfrRWc!sIivweL>HlE5%eE*5f`O3<zfAi(GsGc~HAH6@*@cJq1
zgPVEINO^B*Hj)tEpb@Xu<=5zPPvPxd9;-mR3%@QujjvxP8C09e7MRB-%(5*zHS^c!
zw$Ohi852HTop!F-evxUHWWIF4zd!n?=5w_Mt?8`S6Hfj=zuj~0#*h<AT`AK)eag6@
zwxG#crzG98j@3VIe!j#1IiJ#P<FBldt?G9XDUJ48U9$PbvdDitD`z;hFSB~ezfL$c
z<XP+;r!^eDhjhiwCvvB3`Re}U<#{QM*NarVR+hxwfAv<NHB9Z!rkI@6?Wg5e>^mP;
z73>?(Xm8ylH2ZJg^eAPf|5A%Y?LBvWp1eSNeWjQo+p-eoIYP@lWQFfn@h)wv=Ub^>
zyJHH=%K2V*#nm%J1<IRv-fk(}t-AR^{NnDWz59|TES$kISz!8m(YIHxrNk{;!t#AL
zi~ruQ>24VhEOwN%tIqD0)BE3X;neho%~w7uJWokFE#Q1=3(GXo#Cs8Ux7n@9c%E$;
zxjil`>z{+RxNoe>MgA>i8w0t^6nFiRn$E=GFg^a(0-ZWfj~Rj;oA;b-`xE)iqtMDn
zqVUDXzK`Z*;-~ftS$z>)y1(n(ez&i`f>T{R7}tj^aT8lwe{$KR73)qvc{}59{j+yg
z0_HXi^`d`PpW46Z0n>RVrj``0S?)~?oLTc&GV1?o{<!?){QRYS>dXHG&$3OaT4%q)
z%5H~IXTqF>AAdhDKW)Ek%2Vg{56u|88RkYbC-CyF{FI)%{<VHW=Z?B=2C@2OX=0k;
zXR1U0sT?$zw>3Q_r{necDYG=+M0;GF@^O7r{n^&cH8x?A53@J5F?^HY+|a&2e#Y`2
z_dDd6G^IHiBifodSA;T!FsisDOgx&!@jG1Xb0*vMm1{Qq2yfk9%>MprLu9%yqvWzl
zvo){wbUd`>Gjd?R$Z4GOuQI*=WQ5rD`I`=k|H<yysCH1_>R`!at{*p4O$GmjKi$r)
zV|?IVugJG2pQfEUKP%|<{KW}TkNpL6)E}w^Iz?;>=niKrp7&MN_2{WXcXo>HZRj!P
z{D104_|g5h7)~BOV88rNv91mi&%=0CWe?NSM$<o@d1|TuB7i-w<#OVI>siMqHn^`m
z#JJk?#Ock~><>=+kggaRD{xNn#4LkNsTXg2x{)Z;UnQ^VQY!K%n?p!dxyX0+i-MI?
z3KLfaxySsTEO21m?(WIH^JX!sDC|>tHusamlf9}ATXmOkO17|Ul|0{hqAKD?s;s8b
zBo=MIL#y?RJ+G<V^qc1I$>iI+CR$K?YL2#N(v<EKqMAt;r>X8dHO=I4`fjlkhhjD;
zF`LFJHTkR-xyV-dd=hJ<O3&712`TI^7cJ4);vueipiKGOg0E(p$*&F{;mKZEvfJV6
z1Ut1Q{5NktXlC7R_-3kuTuQo7p>k(GXV~7Y|4yvin07lWS!CtM^9uwIMM;JxRoyKv
z^ABP)NRbwu=`v|0$6mc8+s$iASAU9X6WgJF<WFeJ0bd@Kj;<fsiaKJu1-8l042^ho
z{p{`>k4HKdOT@n`+?>Mhoa%ii?3?YxzbuiuKQ<}dRf%L~4LN+lRpZ+PPxF*@yd9S(
z<vKTQVGK>)pmuYbm6=|{)odNk65c|8vjwKSlU0sQ&Rvn*eJU&F1oNd|0-R+F7Yf|D
z{z6nSZL4zPUZKaaLLYtQwwGUfyJN<IRXhJp>ANU>QQqi)^b?6Q#+kpIf?c#+Z?oUx
z)3NisJ=>&qt<-D(soPFVL{^`gm49fJl9ks)QRgQ<f}WE(dRUv%#7`FadCV3te*JIy
ziObm%C%??UbjP)Kqv!;dmR#PZ27A}-$5eiVHQnBK<Phg7ZSB4pS_MT<XSSdD_}ova
z_2e9F$xnfzg8NV3iK^STDRbS-h-2yQ87gO{Daq+b?}$2{ax3u~r~8F6^`=!56@DgZ
zPT<tM>=$!iKxD(-$u3z}Uhcdn_BqJr)-^6p!wpXRj(-sm<m-5tuDg)qz4%G_ZVR?+
zPH#3|?<&}@7bCo5O=kO7kDdccv+|Wf9NnbT)+{*2d24~nOpgy$5tEC3cXLFapQV@G
zux{qh=fdWZKbD_hZrZ_LVsxlw+E+8Dp2peBXH5Gxsk5}?#DmJR<p=&K`3W#L&8unE
zN!qBe%IV>rdpn)u4xH2WKJ?0EqUf?U#o6XM0X!~CYEzE(8tJ4oD6(a3b?V{zbgn=k
zrhc1@o-=1t-@5OC&!+9ztTM6iMO@p-jq4_z@95|3H;(K|v6cJrAlvN3I@RBfjeWXZ
z2M!2KIlKGx^0N~TolZMHp(@sg$G^q%g@1Ubi}^ND6UC38Th^Jl_VwP#ynZQ6apKc$
zy$Qi3>tqt6eNV(rQjj~=ki9G7co)x%tI9^bu3n;ZgDQ+y>l_T7=D5*RR8sYVM^_E!
znbjH*d(LcI5f&|YZ`BOfY?Iq-4ty%eIC48@|Ie@QShdWL{kavq?SD^&$oidEpG;mO
z7_6Xl)U@^C&n;5HAG#t_FV20Zd0I&6QBIq!VC?;cR}|-%dx;;oW!aSXIosrXty=I?
z@7>O+iT+Bhs)ki3#oIq!D_qsNNJH2*=|Jj?*Dm@J2P!i5*1Fs|z_m$bhlYChny*GF
z>}LEqOx|;%svQqY-QQ-Xxm)DR)EOI6bCcg++mW!b_qIikqvghlB0H7uslV;z%<?=f
zkp7H)T}Y^dVuVe|LV^7sHI>($W{)<~bXY9)?VWyM?}MK=9J4=fp15FNmR+dty`~L0
zOVe^UpN<xcJHIL0$1p@EvoKF0hxd*t&vge@gAIZWOK+|HH0e*YU`dv5pH{?_H&+~3
zjx1`lbh71nJSpSorGU9*VNKzevPyT|>bf42JN>@i<+|6ohSgu!nk_%t`Sf^Zv8BaQ
zJ<T6X(R!kLj!yQl&1bZXw5dH>`7q#DRmraV%GGNVSu`tFFq8-WHK;WG=y`N?x8Z@x
zaFe|r*Bd=|FnWDfzRVf(zKi*3+Nv}mj^e*3Y9ftP=J2j!?##M<t~-(?dDXH74AtA(
zx(eJwLxR8m%9<UQD;pseQakr@=EhVBpG&J7RX;cQJZhXX<q6}Ix6c_9yq~!B=R3OI
zn^ok=_hr(|9EKi+7_OGyM-DSZzNKmYKKRALm&svsqm+^u;}6y&E{ql%Cgo@-yD%sm
zESSriucDx$6eRfO=7e^SWle{8*%@1PHBaPt%83YZ8YE9*c){qyvS$gGS&)d9N3F6(
z>;@smw<6w5D>hA;r7$^uiPfK*4yO-sUzd@)nC#aO|7hNxg;fe%mp&g`%<xJ=FSNON
zd4tr|-iTaNp9!msWd10|{p$*RJe8p%rFHHTsTQ`=`S04MTuG_$)Vaufe6OW|>kjv%
z)>bDU0r}pOcV}Ko6+i2JFz&I}7v`648?T(2(c3WDd=8g|xT2uLNyYn8w`^83&;8G{
z_1(eeHnMEFMGU`oY0BK~I(Sj&kE4whf4{r8J>%Tq3u_YHSMUY&U1anUJn*<}>b$F!
zd~vJZI?P=1{YZ6!Z}87)LLbB&gcY=xFATI^c!6!{qT|ct4t~tsbmLn$>n--msFv3q
zOfxqX>Sl^f2rGZ_tVAL`FDoN*{bu1~{2a$v4oz4bUFT)$qth9uRW{wq{)3EQ{Gyy?
zGj*bhzf`;uHncNer?pFUno84}rUc=bv%f#{<TOR-HRPDFpX*H!h@GPGOr2}_(OFtb
zf(IwM)r1SpGFi=7C1m-&!S>;;Zl?nqzZl<)d@!&324~DBQSO!6X?s{7o`_>9oPOcK
zt*@>E%ro?Ni9dO;$5pYJ_hH@kkN;TutvlKut=x3*ziFd@-Hv}7MO7o1gi4;LGf!Ck
zdQR;Ew|=i<4}001rvFw^6o}Xy=&+RE!1YUUll&}&F4u%+_Jn6KR$C*!9yulM8lk<&
zR^8AmO{4Jks*Q{v6r@=~A|iB_uy4D&sH^P<gL>@l1veiC>D()rU?Z?UVUmb~GS>s;
z`dtZ&7^06GoCsx6bB&Yt{kB)|#vEIL=(fYjll@G4UYni0w5ckzcPVSz<SzS)KW#<x
zcSJLInCE>cXWp$?Z+7~9%Odv;;u6x1@r?!<vpsf~&bY^PQslsVWy|IVyB2Tee)jTA
z^X+}=SwANy{+=9P`0=*gs@hp={hKCczL~uAilpGt4(`Lb`B}?+o#sq%n4<N{WJ;d#
z##3GL_E{Oa2X99+3fSIWqIya;@qK5;Z=Mg1zSmdVPoHp;S498nf{$to%mf+*c~z(D
z<g=XKerz*uhT!)@U8~d-S3D7V_@tmz<KLnsUCH;hiF&_Vlxuje@?ECox}(w)+)K)y
zh%CH(Xx5)KAM+mAE?|zB!OD4Ho?D2#2eXmuijR>AqPG@soBi%szft#+Q1cG48U|5g
zOXjUJTt2&>y~40%+g7VnPpS=WMU_oE&bnYGV~~j5Ety#zru=U{WUn<6PCc9Uf7;$n
z^%)B$)Z1pQ&^UQpJ7|&d;TLH(;vEqMc`n}F*SH#2Fm%eNy;xyTcwo2KH@+KoX3Cm9
z^1fDj!ELeG8Et>m=1r)yj#2q@$Yj%7)q+C;2c9(QAFI-d(OZ(AxM*#!o<#AC&#g5c
zyO=u8*R;gC@*1AjPhRMgwD@F1%l5)FX=UMQPYmZCP3ydow!@MAazpSc$=>NsQ(IbB
zon#PtzOQxl!C5+7y;4(;`Yx-vvSrUyn}`|WGdAq=EE8L#=<z73d0V=t$oxr4r`#g^
zYaXOAO&31MzKB7_b-`(|vv(77M5MGuCcJ$6djoHvz>>tCu1C7MYAa<`*iVN{`KrS?
z>qV^ZwLa_A`?H%qckEYk>z}o`%3$#|iL_fnwbSMqJ~Sv_`esdB;rmiaKmCGo!54{Z
z+g`m3VzFqxz;S)L+M!3vAF`*~of1%DWz%F>Ys8@XQ7lV%_DUY+oN~q}xe2`-*$ypT
zv7v(Ax8=XR|Lt&q_0df`0jEV;>-;_)P!ktzn8uOy;=@Ia8;YrcRi7VKls>Ac`}U~A
zLA7E2iDTWRej9ilbZ?)E?c9)K!w@ZTK!2fp+=cE3W`YrkB73zCZu-<{er<kp=(Nhx
z<yF({mc)gA(oFdmdh^!RgIOm(Kbo`Gbo$SmVF4T0t^MkzlM@@@fAGikqHSVRO>Z%n
zbUN)hlEj%IV!udfYU2ZO4k=fa&89N0`H}onzwCH)daLc#qmfZcI-B-v>qv-9_{8(L
zox{oX+7l+u;G|f^j;0Ux&KYfgLwPt?GoQMXVd7TH=&6|SeClV}DkcSE$v3ad-*wMQ
zNL0>W)3Z;a;HW{mG3URy=9S6o=iQD8FmOvrXVTwS@$EzqllX5Xwfk#7yUv)>TAr@3
ziNkDvhlTPEnUn7}C{=!rdzTV@J}J}TK}W}$OOr#Zf;@`sB92@;cYQ&KV}Zz-xI0UX
zpM-YWtmjbwaUylcqYt`SkyoE8wwMT3=ZR-#hUzkuD17qVWA>oCqk^w-PFqaC-hA;t
z8|zA5R>jrs@|(Wyt=zJEiM0nJ*E~4Xu<G~4+U!4qca&yq$UZy&M*hS<Ssu#mE1sWz
zv|6Cx7?XUWQrCumCKDx&xNKOPzTuBzT=g3lj@;cGI^kXq=69Gy-j0|oD*efF$)jzt
zV&Z$IH};A}YP*znO=8G1UYaPp{?gM#?$%4<(FeNaH|$sa_$p?p&4wE~XLEbaZ`v@~
z-E|L`HuL4Tmvej{hG}qbteSUd;+FJcv$wAze;wJo?4ogLt;tvawVie=Kls-k;SBu~
zEq+RHqpN#Ewctck+2&>45<JK4-Uub>oD)b?c+PlObb6m!VzKeTctgdGb<5tqWpw7<
zqgXIq=Z%=sZ--0TMu(f$boz=NHWgDza40Kjw9&d@7N&kONIkx&b=rwtdla@W*|%oG
zo9Q<cx~g0pWDV+P=A>To5>EIxA%OSh(+$scFHd~4N-(ivzHhRMykgSB3Hduer#gHV
z?Ax6ibf+x!C$nJfp(v^H<p;{<JhXWg6TA3CR#KX{|1A|BrTK4#ch8x;O<M1c6sw&;
z)I`Pm)y&orkFWiSbN|K0|FDK}rvAaO=?Qy%r$)b+)Wf>$;3d8@abILPExelEswfyg
zx^SWK@pWDXo*9jd3zgPB;#)O&sm_6!VjrF~F!~0~SD*Ot6<e=%g8YN!fra5KzA-Nl
z{;;{LzT?nLMjw?<kEsu?Xhe5;><-=)cBe^7MDwEA(%)SkS^=eQt%oEwy|NdaXDg}b
zQ^a1#H{(Z6sKBI+s)d3=Lf_^!=sLON&Uo;=?5vK#%#s}!a}xLPiym$)>F96gIiho@
zzHNs|qq3jmT7zjmkyX2SivOHCw=0-~tzt>}jFZhZXIkDXd<s3PFuj@gop?c0Y<Iw+
z8{XB2e;%;A^?l8uw?{lRbap;%)c;#`C+w^}Lolmo+hen)2MI6UKd^2+8&D>fBhT6O
zYDzt$f(Mi9f=9RN3+_(oO%K=qlz!jCWVPO<BN5W?J}vFMuwcneXEyf<v3d!slq9?>
zY7XpO(K+>ipmg~AHx(8?D;(QO1g1Y=SUk^RQNbsk-;7=Ei}uM(RE?VvBKThXwt|uD
zZl{Zm)2eSAVZL&{p~gCp!$4`4_MUR4jq3zOOmf~&%=J8cdYiAqA%#T&^4_tHDe^zJ
zzllAxOZ{Q&q>r9Xn_B$cUrd&-jIO-7^D1La-I|`?-yXgEu^?a2MRD&RP6Khz{|B1G
zs(2g4^hK|_hrZW{ZI_dc=(&AweiO%sBkfPO2|d)9ANC`N;qU3*lyuimF)x)$L}rQp
zR8E{BC}32w_3HAZ`juajr~DO-){Tq5=delL#?wey=L*lO$R7;HeFZ0&%c=g=)mWc%
zxoLlArF)dDYQvZN23Hm(^acJ%+;{ue-7C+RZ+rD$FsXT)N4S%tOOyGNeuu0Mi3@ix
zNu)SV*$^Q8=To0&)=3kAo30JNpMRGW{J!A%mIKrOlooNsPnc6>wJJCMo&2-5Kc9C#
zKep@sYnF+O8okx?wJzE1yx4qw;&0B&&nBI2Rev#$r~fS5496V58&hsL-MjH)Q%*=;
z-Ied2N{gM68}~2kHvMC{d+(;{A31J_JT*>oIU#=Jcjo-{y-qp}JIp6dKKWf<NtxTI
z?_w=Kb5GOr_dLx1-_GB$e}j!m-9B+MOU3X1R<BQ$e}2$B;`U8B9m5w|42(6W?>i-i
zOyFPIV6>s_tf2DmJ!|BQt{>&mWLvv0wB_se+Ol&Dk~289nA};}9524_f9h@5xF?{k
zd@D=p=?_BXPZX1GuybBAFip9Uw4!*`jXH(PL2jIFjsLU@&z-u>^0)EPhA92{&Uu~F
zCoMbm{rhL>$t5@bT;YH5-MQwK?6&28Nmkpxf4hA5%<)v^TMH)YFHg5Ve=7WF#_eU>
za%JwVR`N@TVYe&S?_IU|rov5a6)~Ha9&K!|YRjs>3F++ayv@IPT8e<JZ@9-yop<hO
zf3K`;naZNV;Q0E)0-x#$C!g0%iG1+m<uBQ%J2UgeKG{AAKQ{4C!t2SYOw!$Jd<ADO
z{G;}L_xCktH}`Bf=l<+WUihuWs_!e6^JB`JZ~v>m|6l(1|MOLuKc_X?Pqg=}m;2$C
z;^&^T_J)0~_|?53WraU8|GfMo`D^j)JWj991NAQqb&{$zZ%38iiNEpoYTfZ`l5(zZ
zn@p?ztNcs(=V5+w{STgp%RerE?SJR>H!+S?zjNPh-+jN4+b8SB{^$~piq5yDf*Ttj
z@aMe!9iGB7)r6^D`O^L)dnH}I&wjhzr9}N-&gXK^Z~A}R<yM=;dLNVg>HN{V&ikO|
zpPNN}f07#S)yvKO^r`!3@c)EM>_^j{_#AhQ_Ly{^X@B3}bZ);-(G10p&TqcX^(QI*
zg8ieUJ@wjFO!ac6mmY23b^lqNbv>&PkNn@c-?R@0naJJbOsPGwso}K5kv`_kI$qa5
zDt{{<9NAd+WcKO)H#(md8!d7vi4NKBT6BD!y?=|xCr;_JZYL(JDU!ciQl?{2xA?d3
zJM+)q9fcVFt_6AV<b8IId&fR*vuk>HTx0d@^HPT_uFN+5_~G)8%Ri03JimDE!oCat
zQ~pky&#qGZN%-sWA3;o+At5Dx87uy9t>8!r46FQm^qETik*%Eek@XLpmF+hzI`Mwf
z-UsZ_to0AX!W1`gF6EsScqhMf|0m{r6ArE$F&}0t9!l8st6rpY?(}#@L+|qBljZk?
zE6fX-51qd<|NX=Em%*+~yw`n~PV*=hcsc9kV}TH#sKP@pR?eu{d1c9E?~G~BJN_u|
zz9<&8-A+<9H8VsrtnPjI`b+koqOI%O{)w#kyD7VQ)6K3p{VaxQ8yz0<KTQpu`C+$G
z!v|&Fw*it*pZ%D;!1821%SAz<j>qMKA`jD9Lk{~XUpxPI^3UKe=@Vyn@3)`hedym^
zp@n~D`Fzq4GBEUg^beGlKEL^Q)2LARqvt)H7yeBOD*GSxD?V|YvoWCjtNF+8fmg3<
zABf{UcI)zw-wKQ`|FzzlIpu~^efGuq$B&rG3I$$clvMrE=;XcT_{wcRL^nGwH*`F-
zcfr?LuXcY}_c!?KbHSkCKPCTo{@&ygnDRt#GXEjJTU=Xqu)UGjxVkKVM%~*F>cLx<
zW-r`0^IU9z`aAiHhAw=qPU`zOJ|$OKPW>G1JY7h{{_?*k?sB^OFR-l?lB-&H`RU>$
zW<mQ+To$bNSN`k|{>FXJv3}M+2cGHi6EA<*e4lM*|Ht!pKRZ83{wc_4`GWtS<X84t
zPE4Ao|2_48=zn*;e&?M<?eBP|8Jj%%xPWCMXR|ub;)p*SC(Irhbhkd<Ry5`N{HwtQ
zE2iC8`1kTx_vLn11b)t#Y2Q^_(Q-jy!f|EA-{p%|3fdP5@7#PhxMJGzYi%nzqdhbF
zI;$$*oK{Zw^ymUF(-et6Zr^ABSiL59Q?O(}v7=ci<BlEm6Z5|<-amE!qt`JabGv_k
zzgc(xQ#Y@y#)Q8je=Dc3pZo9N?Yl-Jdvm+T5_5<0<+oX8yir>5F6s81^9nP%6E=4z
ztmsbIaGCq7+2y;>9*Vx)A)r$Bdzn1P6qa<Yb56e$LN4zUoc;OEVs*tQ?jB#FL^mC)
zYrD5wYw=fhF8vUbo1g72mV9+lz0r^nscaX}oO~geKjm}LzmUHlE&hDld#BjowMNhH
z>tPMgqnOrJ=109rc6H{>2`;<6Ia<Vk-R|PT$h@tL9d=*0-S{y(ZqG*Tm$EF=I@c}y
zaJg<Ga}cwLjh*_OHdi(#fdyB69F{s%=jzWFV?RDmOLWrVxbx519&gjyGF$8S=f(dm
z{s%<fs<zvI$a+_K-7Xf1Gqt<o*H<cNzKNgle`%c0**e92s%xH|+_+GkL3G1hm)%GA
zUEH@|*YDEuk6WrAKk7MKWUlaeVb%U8d4ZMP?l0xOTWQY-HU8~$QJDYvgh#GL5!=*v
zu6Eh`;NJ}GO*TyH9{%WA)vH_0FZOTk`rOm}`|e)ZxlK}I<Mvw(*RQ%=u!wB1KQL>X
z!AT~Qn#$+}hqrjFyfvpUYi6ixso+Addl~G{^?Y9?zxuNH*OnLOKUaMDH{)jg_hWVE
zf4ykv^)FNTxw|*fBg<~X))V(-PYQ2aYSZ(#{ML!%pGqTbp7EWuoF=a^VfoezoD$Ca
zc@}w}3d{a3))H=A-jkePQDL$#NA&r#C9>aUY{-4EQ#ej@xt&`5AqkbF)QgEZ$v+LF
zZalcgnXx1@N4aao?T9*qC))jS4I9iw?f5R+O!WD|Z7K5U5T~tKMuCEY=0Q!lqZ(!w
z3p-j8`Q@TlOwZYv$h7A0jp!7$T-NQ6Pd%vH;IsQ)yHN73E=>()tpj1g36|o4_c|XZ
zJDX`OH0zpt{{6D2(PwJ9R<-yge4ZqE#87aDK>D*Knu14veK$Bd|M{fJ>pn{IOTLi0
zakNum0{e#7;;}-~${Xb#$lk9$bp2dI=-;1({y#Fm9=jT`GqOvJBRFJZZBFXpV;bA7
zT=ZvJlq#PO&Hnf0{{F@vK@zPkpHc;*xE;3&_;ELyD4l+7_0i)@Y<Gvr^bqe!57XDJ
zo0t9Otf|;?9mPqj0%XpV7wq2fw4;Al!I>V-x>v8yh{(+?O)E2ab>MCKsS~p{eepZ-
zKKR;A%?BELE-imBuV;5Z2e*rfEz>8}41+iC&7OAq{#^Cp+PqJ5TP}L~b#wo2`m_AN
z&r@x}+%pQF<Q)CGwEn?=IqjN<Pq&l=PO<B{<oV&U_~F|kKXSOj%vf1wH#z$<8ZKLL
z=QDHOLUy~qf=g^JB^U0NKYjF4dZ3lA*LqveUAe{nUfY*82rw3In;hqTDciHkS1H!<
z5Mz)gOK?+xz#E|lGBcP0eOugpB2O<{llSV&{3XYQZme8#eA1U2lb4*HR1!H!&ijSo
z$JzfHx?GP;Q&^@i$o#-kqpsxk`f~R+_6mb{3{CDaUj$VC&Yz<wWV&MAy-z8fw^*j%
zd{*mp#o2Lp*Brf~T_-Ykv?<CgI(75b_VxukY)-vw%iy%U`=n&@W!qV^qucvc4i)U4
zBUe;);-%v9_BpaeWo{*F=d`syopM=tuk3}pjDKY>JY>A@yku5`#?%In`3k~*{~oM<
zv!S+T`VM=aBhx#7n4F0EepAO=FjnGXyWm`jskYpIB~JGVIL_VF+*dgJZCdzu4##$v
zFX=ArZY3Em{ca_hF7w?=UhU7!ajBWpv0nJM+=dFFef~$-yM8zozF~W;>{_$5<Gjen
zncLKJ=2SmOe)8DcmUF(=!(i2#RU+Ts9Qblzy4Q~&&Umgn%LP)DQf6>Csxk@oZLm19
z;EB2B>U7mt4?rqg^rwDUDzq=;5J>U)Zyi2ALOADZDTS$YscBbb-}%UCn791uSq|=n
ze4Au=jul$Rs2zEAaBjp283%W!I-UzEr@plO<z!o~Q61azmwnsx9dqRlzGgPE3HW)X
zFXO<<jp7a;AM*)|#66cSz5X<3`sVES%oDHwn)@uBe|^@&w)-uH311_t6IhZ~oZ%9V
zHQFe&$e+oIm4)ZPrfc?r>ECa^+i|ye_v<njRV)8*E<e>@<&@`?B-|-nH@iflc8871
z6Q?iJSj-N#DA%*?iGBLu{yg)-H`^t4$k=YMo&Gq@Vczf4k1sD47ph{Ma;)M(-woeH
ze!(l<w_Eo<;<sCBXSdLM$qJROn|)h0fBM$ucE9F5&&_o%?W#Syro2^by>7?JY#>_A
zSdya^Cf{tBXmUs(;`QGjKN#J=ojb0PxaaZVj{AS@S8hnIdAi2JncMyNGgB7#1Sb1K
z%Su`HUse0auykrmF_Y1ahYU&CPF=fYBXvB#O$hVf<53>r;Pj1O<qN|t?*FkVmu{#`
z;(xJ4AgAX=%1LjHTYgS<P8+KminKfbNeX}B?^0#hxoT>kP)JO1Ip@7!@lkIXIcpeL
zY8L#@FUjJq{_^eJySw*yAHMi_TkwMS```8Ff4tRp_R06@Z+m{U3CbHj(muB5u-OMi
z<$WnfruY1qvhht}lh%(}g7bx!v~V`+iOKXu6x~c0{&usa)~wd-`|=hYw-%KshRQoC
zPk+o4ls9?QeeAF2;~UPRr5{4gr@mOWq1#&d+MUCd{pmGJTKcDam?*R_@Q}IJ4>zgf
zefcd7n^fu)KCzd6J95LAdGD+Xt24i6-IB?FakVAR_r=waZ(p25zr5m`a&R7p*Cvms
zvYTH|u^)_<{P;zniPh{}yz_&6{cZOBubU*Qk6byuScTQl!&YGTM&|FrO;?`gOxxVC
zxxQs)((w(-P4Pxn$|pai)v`7x9X;()mZ2hYPtkeOUWt>d%#9tK?Hi01oXqt6tCP+A
zLv7+_lllXZ7Eg3;q{@Hz$XoLGg21$%KaXcS=kH2Tow34bS>-<sc}Iu+EZiBVW;~VL
zZ1dh=XHRC|j>{byfj^sX=XAN2B%Jx(y@>y@nzn<w+m}T`iVNhGt&A3_%P)Q5lI~WL
z=8~`F$@Hsud29EJz=d6t>lg7^nJx-f-etNd-dy*6TC<2$nxy=S%_sSe%bD!p{S|&w
z^4YsNbLa8R-IM7e{L5ltKbSC$(pflJ*edd+?1kf=>>8`yDDFC`Dr9B3aPmgwbnQoW
zFP_w!Wd=Qz-TzACJ<GmEpH+<4I(^#&81LuI({4Gmmbq26VBQPa**Uj9y*aje`i#s_
zEB&ci%0DLmy1&AG>EkPV^etAMDVgtP<*0Iw@2Q>kri_D6G$z<_-?2|w`t)wufg8*I
zOj1^mn98}}yE}hw)rR7oTF0e39imJQd|HudtHn^kGEIWf;FSCpmXEWHcu%=c4*PQV
z-@W_0zOE0ys<(Ib5%V8B86PD6^%d6bee~aD+U&EFbRLx+Y-#CU`SEjyK&$~X_mbyK
z|9b0}I{%A)xL@?&_Ar6H>uauiuKdUTEbSJjLz$h-n&w5Ob^fa#=bf7Jb+1O<cA3>F
zn^ZR_rtQ^^f5_)Oz1Y?3!HSnZ)&|wCjSFqj2ywY}<0Q9h#HxtT=9+U0ZiQ@CX6vxl
z-nX-P68C?t-HsL8bngG2*Rg#1RcRlgzWIWTKO)?(=SA4hJ}{C0_7<y*$~!UEn@;3$
z{o?wOusqnH%ss)EN%G41w7MD#D=jwX))>ZRdwg$f{x@YsFGD0tv54xPHV>xaZsynF
zr<{_x(!b2D6`dJ4ahAwQ$BA!)moT67*r0b|I;-T?f2&XD?mF?wyDGaNIA_+m0Il_B
zm!G_Mk0o`}>Hq&u%q!|Yn=9$KcEQPMOmRkw78dSbE0p>m<yL^pUkj@<J-azom!4wS
zk$O<_<ep#mR&>}2UHrag6Q|RXG?({@Z$h*e<t9x29X)rWr)o=6=nR{WN(=ZV_HK22
zp!%_eN4cbz#f;-9ht>jPgL6;pI1f9vgsJE=vvT^0#jsXg)L{{0>PtH^!P}x@Vnp$U
zDLG}SoKE^ddPPz_YNm%IpHw~6QlC91CurK4MMkT3xpb_1e#(B?l=8F}3=7Om_P*|q
zT^YGe<kr+@$$wu+=*b>lx<BR*Yu^JkYlFBQE#Eu)ZmZqXc=luayPxmZD>$v2aN&XM
zN6$xTEC-hVygpCjqiei;*LfY8=bILNe0wzO(#BKY&Yyev_xpTl$J@$HUw$wC@A%vB
zhzrkqfy35EKJevkah5J+4!v^e{qwGAzl`SHk5An5t@6$sM~Qn%Yrbdi@wIcEa<|X&
zzM{|id-G<r&*014%r<xPqvLu)52e2f`OI6yxt_E4=0pQ)m%FZ)4srZT``KI^Di_(R
zrg4n-wL-pgiqfOlsBo^cr&jIZThl%9YGsbr&(bZXx}KY>KmB)+>t0{(thYFnWoeha
z<&$&HSsia5|NeE-eI1{yenGm0l<2|ZNs>0^i&XCm-adC=(S~2m`U+z8fx&@^mnVh)
z;ePT?P*%`!ugJ%50er{h%vRVHd_41=+xB9dyIDJ<2@6Ym;pF=~J(CySF%l|JZ&6qw
z9B=(gpS_0Z;TQY(gD$t7yBPj|t*_ym^t;2y+RQ9#`qDG+yd+9ztHiz2RC@h;_uo5v
zn$E47c`lAOC9AMILhz4!u)rD4CbJ7SyEF?<xmkQa5MN*<G*@}`4jboB)&K9b=!sVU
z-FoSIcf)Jmm`3MQ*+)7qcTe1ZQq8i__>}d-`IDHRzyE&m|2D2o(&zUI9y;pw&p~6I
zWIfxy_-o-)a}#%6t(TYky4E-#x9#^Foh$FGbKQ>3pQUx__Mh!9Y7!GPl62lR&V1D0
z`(OTt!%4QFL*@^w)4%;*+OV(e`uWq2mhYX9d4FNrS9+*LSp4H3&;L(&SA~85&)IwV
z882(FGaK`iPgW;aT#z}bU^}@uBk6juLh$LFpl6rE<}}L9$@iV$rMhpf)=~MtRc9}~
z>yB7zu(6m|XX*a?iJ|jie*bOITF-E9M_XUUCyftrjh5?9r&=%N*`4B%9X=ycXwQ%K
zbw7T~e~}NJacPrg?u-Tgyo~p44;<hv+o7te;=z7sf#e7FP^DE3cfBvXijRA=TGGpC
z+BV0(e|8ww)}H;}nD;Zk^?%rp|COv07Oidi<9%oQ!{`5RI!bozcZd*?OpLkGdSstR
z%0!l<OX{yIS@a<1;@(ZG&Yugu+R&lFQE$6Q_GrIM-+v)voxlTJKD<-=)#W=b#MUli
zG4iR^IkaW_K|UXWj`f8+g0uM!)jXOnduhwz86l~zF)Y6(KVjJ@@#)w5e-k$L++TI%
z*YxsE4(}V4Eg$k`H%$nY;=K1Fp`T$B`yP&>gvoLnvcBj4RS5W>zh8NgOW-3}(_In^
zB^I1#yv^fLVo>FHI?2VTzvxf{o9YRD3jv3zO+QMGtZs;BTU=`_sURcdsHsrRc%EM)
zP2gWdeZ8sZYzLjldLy2H_u~Zr<%$X<zvLH_->~cEj9F}Qjh8$Y#Z0f`%9HFA5qc*g
zuyBpH1M4QemH!z3Kb$U9)wf^KYTNnk=O6#Cx&QzDI)x)D@un>TE|0by$kY|wTCl%0
zktdCB#leE|_U*sZH%FhIQQ=Ue%f_d1ydfv?#E&;y(hfaP40!1-6p%YTD#3F@4x?F%
zs^FaG-QJOho*%g@tZ8pQX=+rGf$HNgUXN66&TE-rax!gFinQmVbBB8kJZDd8NGbU2
zo#8HYo5SY&1b>E~$z|`<BR4<joPI!yM}7C)D`F@6CM@y{oALCWmYC(q>`xQV9G_I<
zeo3;sJJPKrbM`ETvX>f>UzmTC9PHFS{f%9Jtxo<Vdk@jyZ&a8}jdt%yX`gt#Nj_xS
zz1HktHumYy%p%<8^6bh~VHZdc>pU~PTK>@E#K+E2q9?7R!!B$~QP9=qZ%SDb+xPsb
zvM0|iD~*$9e|PbV%Y9eeDXh}3`H7b`QHVdN&m&FSQdH=Zc86+EQG3^S$vJslIqzKE
z<PV*H;2&&vqN`hmi~p06hWE}p`3lDaZ)TP}bZqLG&DvnPsX)w8pt~l_;A6uc9+w2y
zrknHoEEqOMaNp55a>nU_uFp-;ml<-N61tY!r+$^k{$CTi*?Eg=!u?%$3jXe1zxw#D
z^ATI0TiG7ms2B2`Q8Q=JALb{oJx_Vw3cjG=SrGhmyYG{e4EywM@~PyPPWs)rsdjI%
zsELT$qoYst%A?lrX4_{mojGy3$($3D6(m!{w5L2-o%?#TdCGxztItJD?>Wk>zvrd4
zq4k*!zG*3w_3ydNY~3ijEo?@VbkoH||J(%y4gpWoi%*|64lD4QJn?SAi43JUlgf>1
zkEOSrR%>z<KDfy9Mu$LX``p6|PF9;=xZPOHw<O=FRkiqFQvHM@F^}&4b!?0Ed?aQ%
zIXY@3yWGRiagyKc4i&^*dwEDCzjW@}+~+s%?)l+s)UxJQ?bUbMtsPGu$sQ8X*W7h{
zSK1oyXRD0E=W3pL#8{Z&w0vKc@Uf%+0_T-_b_6`Bm>L#d+1<wdq_Z%i?17$xN{>ry
zShrqw!t}nk3$C_Avj2Rwt3u|$p=Obw*sw!-r_0*g88@XU<hzRV<m#Tx-m_MsM5495
z;uD|2Ht$ugCoN`jbKE(8DO;uWl2?KKMB565iIvgmw|=EZi~hc)_G{NW!D)q_8iv<%
zmp|DiQ1oqw=Fir)n#w1K7g(o6@kTba>1;JP#B=k1n}l2Y8fR-~WgSE3Z3h*z4oqmR
zI8nqh`OE~@%!bd29W8b#xpGZ>{E-XVPF!_nd8L?Kb>JhfnYgdgge7Y=7wJ@TT!_3B
z?WAsh#NK_gms;ixzsP{6Yre!xdJy6HN=fgYu%CQ`vg+&KR*gSAJ?E;V{Fo%2^I53t
zp8ChHiyq`|V#>%^aQn*PQ{rteiaM{yO;wGES;?fVBw)8=UxsJdU-zB2zd3rS&*z=k
z+?9A*Yksc%FO7wl*6mPz*t$IT)&8wBz1;*9TVBZTx#?HGudGFBXAPHp=cFYY*%sco
zp0s97(bwiFsu%K>w6wS$^1d|ldrG=ZXQTIm6@ILzgB=zzM(#~vbelD0?yA?#9HvX&
zSGB~X_!@e)w=Q)HYwvehP&+v$;pVNiw^UQu{(Bg_aM-h5WvzqN4E`yvH~##Uvcg<)
zxzTkMlbT%F69KE5^FC*v;op=V%9f=3<z7(7YCHSF`jczFzIpIewvc=G$v+yp%b#&i
zHox_-X1;Oe^n!>U<+$QE4rd!?oMGeD$hfe$z)>OW&F01*wzCAjwFT~4e5vctPlYw2
zi=9*s#)@_O-(769Va6n}MB6o13dxrhcX%((@O*YzuY1ScZ`UWUUhrRS$E}?Dx%_)J
z&RpWLFY5Xuv&y^^nu~<XzpeMb;d#*P<m`y|2|<T6PE-X6x0Z&bpWBf2HF52sVD0Xs
zT@CNAKiwdvP&~7wf#E7=yy3|y7guq$zE{bXchH~oDe2RR8(*2V4W~yv6pV15@UG@7
zyU-fezve+}6}@k##rqdNFPuCr<JZ=OrY6ZEmkKp*=G`)E&eZpO<g4oXq~R9_i<(wg
z1OF{O^(AgUJ!;CF?B!1;$lI0Fl^HpGu=OzeF!x=oC+9>diw_qj7RcB>`P|d&`N>0{
z@5k|?uG6M^lXuj3On0pLoNG{Aap*)`=tM)A?8fuLe2co2Eq9chnph~+trOK0bG*y;
z?W4`XVVsKGL4CDHOs16cFy07oP!hL0__S_o&*f>%kuMzPZI|u;@HA~k&u`(TWsmQ1
zOg^X7bMFIV|M%i0eEALa2l+Xbf-cN%xG%6rR<C!HTKA0K+-y$;+2(Vf*w@!9{^Zmr
zZV8^mm=C)rzM04Mf6b(Tr%`d~r*Hj<xR!kLSgLp4i<umk96qemXq|ie(Q3VAyDrM_
zJnuN?H>do3!`CM|{sksJ_!%qZAuq~Swom_rQd3dm%<vCf`tk~u;=jZ*cV3#k<U`J_
z>raYya+jX@b$!z02{->vyz*{WpJMoO9w8NzDK5Jk)aPl;K65Ti{AuE+f+yTAMTh4y
zuV+}@uwuh41xZKOdp?%}UpDXQ;!0;Qam?GSX~1E`$fh6@`=Li;|D}5(JhwHfuRHK0
z-nzNFp@?Cx?vB5Qj7%0UdUA8}lC@1i9=q%mM72w=P6|{xr}vnz$=G9C_~|1*Jf_(>
zI;;3DQu!_XCaHH-PwC_4J1OV435!hE+Gc!QUSU%2gg0mUCYb~@3Rx=pyG`2M@lo~r
z{g<h#!cU}PBX{Z5PnZ_?QMfX&@=1`U@sbaJ0#{hQUsItLwJK$&{h|im*X|7L2Q9oV
zxJ;N-T6EUzWXRGPr*=gJ@OmCo`>CO7{Iog8pW`)CPIJjVkFc47>orp=Tb77uTryN|
zD0-xRpfTkwC!d|5kBUj}3pcs<b1z2=C3)GcQOa7lY>P{u52H2TmnVV;quOqVd^vJp
z@(#<Hs~+s`nzu%%VDhq+W~!o3L=vML`YdNI<YU@B{d!`slU87gxNDT}NtK<(z1I$j
zOiW#WVOnL#lA0wqgY;JYa$NLqZl{RXA-1SFS`9TR3b!UL_K4J)X(P{|FYr#4Wy4#2
ztphCzLf+aHEbWf#PA}lp7k7N*9>DEcrPXj|(SdAh)A)v(s}ocuEZ_PlSMCcHn*WXG
z&YMq1)-HWx(l<#z=)`)C3m&yk<foa$XTLC6<NS7U!rAp&d<_aqbPjKO`bnbuopMH3
zhC}QF<|T2WCeM7OnS5XE@|Yy_t<-7D;!f45P1C;e8=bO05Tf)!F2gUL^UbMy$A7zj
z7TjHuf9loUrRKXh&cv29@x`Ci^Y*xZYsRJ{-}xsUO583q!)>a`i}cUUa}KT8SYrA>
zaM6KMACZrpp*1OqVv`itDsN8{c%!OtbJLR8c})|24k<5);bsay@{B>yLGhP@%4+vV
zztXeYr9T+R3humjV4Kc%A&F;<^B6DBPq1^S+gIXwV=v>Hj(Ha4QpbEdgeRVgEa<vw
zJ}E`Ya&D{gn-FJL?%Hc9xvyKEEM>bo{Xzxj*69mm7cH^75VUcBLx;D-!IZlUpQnY`
zeGLm#wLL#=$=~d1lQaCA61y$z>=Vx3S`s<QMc}33fpXPXc`nn!9E@jQ_X>Hjr(D7}
zRrgg&<FeTfhnU|<{}dIiFm^oY^uTg~T#BlWeAcCtUtanOtDN8$$;`Mu`@8$e8D?*T
zepX~HU0eTHak*7p=HnA~ovvbQ-*blNCM}w~G|0|sM_U=+Ipf?9=T!ep;OB6kXjl1s
z*IgNV>yGXaJHMaoySsS&4L+T)RAF&tyUYGY-8y*I#5WCb?hZ@_uNu6cGfvu299(zk
z+1Z<KEI!@3RNG~2!)4NNW@-Lg$44>3|4sy}O6eYw4DHF^k*j^H?xS0;5%1Bxa^D#y
zo`1H!a!OE8oP0<7rb{_iQgfUqPsv}FFYCM_;X_=)y@_#sDy=_{_<sJRG4pks_>$?9
zzkKj~ylL|GJifDAtGz@8${RK?m&nd;a1AjLbZ)y-=<!was_oWO=h(Op9bpKX;9J9F
z)ax53ba6=s&oUL|nK@Hu*iQb~?UAfhkz}^`#JPa39Ie@YDK38;TN`)BhbT_g6)`=w
zvHpbfrFWaRSPT5OdTEuy$>_<xNK;5d<Urnsn+~q`{3jlsbVc;#Mf(Tr7BdW%Sly7?
zP<2Xnx6Heu%<0|-cQTx6J-dp*jJ3kzq#$SM369c+(9hj9j~egw7@yK><nmG%th|0#
z=jFGa2Rn44--(;Lgx4C@n|*CAvYYTm@v(pD7rw}^CwMfpo!8y{?wxQXAgJ}y!L<VU
zU9Ij*VilLUp0Q)PD)>`tf3mZ-)2<0m<%>MFl{AX4mT|Qz-N4)+^MFyC;lqh9fxBAg
zD*j@&+pA<W@5;4&#@*R14^0n!(yA}ud%`a8NATxvOTIF`?`M-2^z0MTzVUXFuit_-
zOe-1!{;hi5%l+WIT}u7qm8sk(Hbyvn<P+<wxxgpm){^V!qO(P7BJ1>NZ9BpQ#pYf5
zs<J3f(5ojQ_Q-Rtxw=VNf|I*dZVCH4IOvJJ=@5AIce!Gnc{D4p@-cZ{v3Ug?9P?#0
z|Ge4ppjXB@#L4C3oy7f){%<%Qq_h9ci?GbL5B(w>!fD(1y++`tq==Q<9PLlPy5tus
zoIEgDe!;~>^SGt%c7-{=dC7N8*JW;cN#V8){R&EEkNh8Xwg+zc!5kx9vzu|@MvIFZ
zQWXDuz1g_N`t09xW@|jtRNNNsR=CDH(O&b@yUgQ;k9*6jz6#rmt504U;d$ldz16?|
z$vc?|uDn%VZ|H0EX)|-?cLp0Lrt+>Sp|!^sDAgE<RA1;b%(b&IIUV~wsl(V~sf?*^
zO}XI2RPPx=*RD7ISFZlt$HcPkYRv-qq^Q*Fn`uj0CfsSce*5Me;Uwd$PHg79-PeVq
zf3$@?W-!^k*?BsT^-nQFReg)(uL29(7))E9DvFz|*u^S2<CC;tt(agWTf<fblf3_q
z7h;b*mJkz<*!#$ix85r6`uAvsZ~4!I!slgj_TK)OD_Fm(zwB~Of#5E0nI4VbIWs02
zCR=wp*fa0HB$p6;*ydZ6>Bc|j&zHT4RduQE^jzwx!ZRtsd;g&b<K6woZW*Q)T3DYu
zH>1~b$67m9QGsipRu?|Md2Q|*Z}W&s<rx)Qe^#`3G#B@)&!{OYIoy8p<oDZZjrXSS
zRIB&%7I3n<)V@$A>F302)tfKy&2bQ0E_<ut_=9(X8x7B!Nv;)KbangFjOEepAt$#f
zyS7ZSu~xNyP(HsT)<^xoo^L;MldH-OeQS9!C$~sTe%73_Pv2S-Rxo;5aUHBTkV`do
zd&{D9=EJ@#d{N8Ff~Jd<*Zqh)>(%P1^4<B?kF6H7PkMd+H1nj_FW$U`yT2<<c+OEB
z9nw7Qz_|cRp*g>wY--zG5_9pRUB>CAh9~D64*Ll3vl}jDopFarr||Y1HLZ53(@UHm
zvzPIHX>7PCXjdL!_oRPLOhV_gmo2@%UHlug4}SS*Z#gsd;;EuN9+kE8RMzBse(7V8
ztW*5Es8mI<#>eUFj=HBml*)V-6{>K&7oJn1`0#ulN0`SJryZiLyVp1E^ZVBt_GlH;
z{5wsbS^2v%B3!Dc%~P-cus-g?HAj~FETPWpq*iR0ZTfUciPbFabG;V~g)~@$n{KK&
zUiY|t*sybYNYkl}yWMgak^%%z%BI|j{A#VKzA@YDsoFBemD|kDtNd~ks!g`s?71_~
zU18%v)xy4*tGnmC*(A<yX}HAZspp2pPTw4&AGz5lgvv@E^9(S)HoxbyWLWqXH-kdO
zn~qnFw!TbCYCNlO@OaR-mw(svO=ixwRg<@TdPK<QiU090$3i(9-x;lQ<evVZXXcfL
z24U5Q{)a<Fza^Gjn^a%^b&*KNjl+h^yLQae+aCC{{mBvmnVlW`lLXu6Yng`a7f<{a
zXR!Isa$XBlE}kuV;#}7h%_WLnPRcBss@X2SY~q8PnhlH|(a9I1gk*KPzByC}Dn4Fy
z=V61$hC3oRwsS4u{i&{(R}s0zf1gw1<RC%CiRYy(OuoxoD%b7(z#wWPD-qPhyrLvx
z)sN>#F5Fd~v3;s1gQ<)0Vw*)Lcq=#<c~?gCr=00tS@kz<rB3zk5WO8PeBmEeb5{It
z>`4}_dsdn)C->>S=6)gf9rBU^v2*5XU;C^zDX6J`#kFYJxQlx)_&r(0_5Q!s)rD7l
zFZ}b7d%5HK1p5yT5-Ut1J*Q3O>TmM882w5;<kX4@AJ+$cWdHBv?s4GatL?0(RM&jU
z{jY!e*4(8rW=wq!m+i04{*!s>dF-iJExr()*X3e+J%XNc%-Z^2;hWvVc*_Up|4sj)
z+I8WA*`(0I6%YN{zh)fY|3TyC$5{(_tETqY{#bEpe&5`Td=F-Bm4B_}M?#kx?ySCh
zlX+5Bf#<2!S7o<u7U8a29;5tJO;q|+^VQuV?SK3}2F_<bukoX~o^jRT)$5;^`dsUq
zc*Ss%)e|NS^-E5ktG^wX&8T_p*gxj4#f!eq*NraqIlb@imk;waH%{_7tH(7*PLuKL
z>d;k1VRs}Rc6~mWwE6$uOUi#V)BZ3eM9N+G@${nF|F{c}=G)J0Z~s&MD7<eP?@Etf
zOL?Ci;cR-fXu7M7$)lAUKeQCwTjN|&l#{dkjO~qpFDGvJ&n%1I;`puLL*AQRoW33v
zdcTa{&)eXYx<9<Bz<%522^MxAH2ySg`ZLQp$nmMy@`Y+~E^|e#{#2Y@`Jy0k5sw$Y
z+RU8aEHX3ay*;Ut^z7k<Eq`ZjI`!;Jdeigk{&l|h^L)?i*ZFRbzWjM}{N?G<mp`wL
zzYO9nju)O58Y6Yq*vm%1KzU2hu}B*?<CW<?9aonu(sX1IzO{!@=|U>UGtWJBoByo4
zvnSGN@1`a0=N6TxoYbB0@crf*HI?0$SN%(0eRh(*Lia-D?q?Gdl_Ph|yutNNK1T3k
z!p9p8>H%$+6zgZ%rDn=V8oCyV&eK09Em$WVV6)}df!b&PE*D;%Q<3Q5^n%4jb`k%l
zf6aB8pY#=tPhZe7IyB({|7;PL)0@7fcDy_N-@f~=Qn%X2=jC5#TX3ZsCZsH!5HA*|
zsXw#qe)fq*&HdJYL)+eM_~#N5KYP~=j{+0r&%d*Zn%1Ar3qSZ%tG?mKqlX*c+poC5
z{$i@bB-<zP5tBarXgRKK6SwdK&!UfC)jzkW<es^}<8o+%@oaC4)n7N}&FY$C-f}6-
z@N~Z5cH@s<C)9`k%q=Q;IzR6Ji=w~ob1rO_komU1ea_5J+$U8I@Ee?(zisLNJ%5`&
zx&LMPct!ouOz-`-%0<K#BSgM<O#h%>$a89UG`p5>2p4O8+n>nhT5GPSwyDB=PuH|h
z3z@ln@&&t~r%Qh5J@QrA=y*s!rNPwo$@Vv_t2$22I{mFOM=;SU>G0p+SD&n(yqLAX
zb<=uX>!QoM_10z{%ryMuk#l0jg*BRzQ@cX7R)owpQ-Ac3>1JAmVCoLRr<<l{Z@lb1
zU-ZjNwFn*yk&o)4`-%=twAuRZ)}bY#rSbC=@A5VNJ|1SMsG6Yr|MTxR9Y1yYTwil7
z(3UU!`6oCl^#5L`O`AOK{ys1vc(SO9)B3Kk`Joc`j<u=$ylY||-IgBc)^Y9s@q5$P
zdiEKey!SftkyDP-p8wr*kDQs~>LfVv>-RHX&a7O&J4o?J-{T*C3MXT`1B@gk{U%%O
zX#Q#RIkoN9AIrbl*H4@?s80MgN%Hm4kA)us*L<z|wJ?7cr(Y|7F3;4BJUk3iKG6;R
zw^v4+G#&XFDN=sGQf!6Hid4QB*`+Nl{wr_%Zd^OJdGB?DY{rSUQv;aR3P$yL<v88!
zYd<i5rqg+m8JgRww5AFi(EH-UZv1rK%M{n8-`V9Rnm7dsnZ0$~ZMU?dj%TlT=rxfQ
z1{YWyL!uSdPu;2<_S%)p@&QLskuKx<4GgdD7^&yHWM0~zb5itdYR(-Vj*w@EzTJ4x
zXO#QRNN{eDO;_I3*@21fPZ$?_1ihVRzR+-j%A);u7k12fvH5N;-%Y#QdTW-Xh~4~n
zakKQjt$h!B&fYn;v(qd<Ws&I|shJZ_tY|RM<62ZznB06kUXQ`<W4(sQN6mvl<`=d2
zc*;!Yu8I6GY4X)_rR0y_;vTGLpUsec@u4z@$kHEYvXzZ`-<3savscwTJm_#VN78NS
z71Ib-_1!HijW5k*zJ2plT|}JB4)x_{+ukUvD{k?5!?ny$v-9V|Q|v!9y!2Dgd#(AR
z^fAM+re4*3%BtcBgLWTV2i`Q1dl_#1U5TlsiwYbkwh2q}>Ul6eNC|)N)gm*sVf*24
zMT!z#ciOo&Z+QM^hVA>nsT(F9`^$K3X9m-@oese-Q~wDcy_Deeddsq+OA8fWXHUPn
zO8M#R8T`|hm9;LbJNV)9%MU?zxAIo?9<B+0vYuHYaa%^`J(&#&3LSsG96n+xHC1h~
zqe;Tg6i$xUg_dcKZ42Mcd-key)uy|4r`+Ceo&9rez@5xm>$jVKMjtdf*BN6}E;noU
z%Hk-_dnGG(hPg>~7%tAYFEzd6{VXiUa<;ta+2saXQue3tT&?Xt^PxW9K6Cw%Z?Wfj
zia0M9-O|7NsCm7?sf#Vkr|JBb=~=b(R7=|Bl-&v5%j46|f4X%xto7@%Y1eyv<90_*
z*syzJs@>UK9W(Y#cP?dY+qiXGQogr+YFVQ5hBHe;6<>Fn&3Sh81<PR*K8M{qE0^w^
zrXkE0w)()c^?PrsCmf2hoT*T7sOspto!c%RV>o-Mgh4n;XJbjec_jZH568%`%>oz9
za{d)8oxb?j@6=zFX9{!n6$EWgPH;c1wbpX^yBGJ0*3Ox7@79-R&f>GpWlOKVSzB`I
zMXvuer^wmo>w>)3^v;NWR$DNs_?ha=6{*hhv7gl!JM}OdwQo}Ky|{Yi!VfpM&)@lT
zp6C0^0gbBi&)VGna#yRmwi`67E`PR3&6nL>EGq5P%lKub3@ccFyY@C}=}ya<RG+K(
zcS=v(+)~~4Mb~CH$ljf%@+b9BQQw09Q#k%?ojoPm@7P9F?{$B(m~xp-PxBN>eBOIo
zUs!!{6jxVd<Mx!3g%_i_!sFHIb34LTs_BX{eckEr#C&v~kK~HC#oFqT54$&QzHHE2
zC;RsLO=GW~ljpWLsVzK`r)m57rt$P?&PDErOV@Jm?Ylfjy7sqf<f5;ux6IslIn8|G
zj3aStjK4g&aU@M7X`$B1nFbpYR;u|Q;@I<dij2nu1{NoqMe!mRII^6rUHQCCIlC`(
z)Op1%y0`7g-5L>1hJ!pZA})J;b~L(e{%cg(_ANf+xNqi3A%BjVwTxXdR~k2~2A?`m
zYG&@~CAK3sT1l0ojromwDznvs>Y_I@`{#MJ)f-O{{`D}${^VI<v!*hMpCLILzjep#
zG~1SH5+?IEE3)^y*tD#fN1gUd)t}+}@y(dmwE6wkb6ftdi*KLd=d?=b=%q!`x`8%p
zmaSG_KJVv?mCq;a+>%nbY|hSkQq^fiX)6xaO}y!{U0L$rbi=7Ha`ZQCH|kAyNl*6n
zTRUT8YerW7%%e{QFFBn)`1)Jiv8{r8qLeBPOVn=b7sm6dMiu2$9`z~gayqTEuXAqa
z+w_c3iT0Mv3?J7`@lP5zZ0q}T;&jDawhTAf?Q4<}6C`=3-~V2BR9dF{WXi3uTQB1a
zzq8%Dx10I?-p1{_tv3jB>+0Ws_hk3p@1NfK?eksm%EQUKuc3U6(cd!7AjvtUzUnUS
zkp=he+KPF<i+7vJ{&x|x*7UQ{cHgtEY=5{WVgHk5Cr^qW`Z)Kp_ObgrFH1O9+)nhD
zId(wQyUCFu_Rs8YnakZw`95(;w~pj)=A5**F!Wu9P2}UtFJ3R<d%eI~YPG1;|EX`q
z|5lsSZI{kk<b67OevH4x%-)H&&cwPrNY=T0?}q-*JcDAxXYS$4ijMa9PFjD&yYBTg
zo$YLnb488I(ii75y1kT2e<GEh!uGFa?zHQA={0ZD%JOflOU*mO&-?gdR(eOq+jDc)
zr5XsADH{IP)c!jCk(<PP5vlJXJ+nD`Ki}*-_Gr%B@=tT$cj|7PS+T#&_|>zL(tDa)
z#8OuNy|wfqdtBBWVc(4t_4#Z3)91#d&rM0{wUXW(WEHOTI`WLG_tx9;JV9^Mb9(xX
z*QG3esx{lTdP@I1spyb?-*2tgHWV6Edru9O*Ur0h>f#Qe&C4!pRW-ZZ+Hq5CyUek!
z0R97CvnHCQNT$uYnfLtY&KGC8&uq(K(qVtw7`HoT#kLy}(>E9E88uhEO?&Aw(;&A^
z@0!p0y)!i=xl9Ut))_k-Yd>=A<&$HVbCzy4U-!Rf?qy3U@69J$jvJ&T=UXj*+<vAv
z-O_6HW2@E5%o1O&nT9PlRM}`~Wn!2(ccHOWdU8q1uZzzdW~RPz47oAMdh*2?6Lhy6
zxgu2jJ9HQ0iKAZ3-M8wO{WIpj<EL*g9leD!Sik1^#wZE#>k%_A)a71!COs*8j*q{u
zZYg)~>V^C6epvC>r+4K|pRa#!`L5M1QF^Crwk%D1ZstvE?a9B+-!aPB_k6;?>384G
zl#brqdhGg=Z?AJ?d@^&+s<G;R_k3Q~ckbGpjp=%G*K_tZPW&VioSbvynKZL-R8m&v
z&X1b2P3HNXQ25e(sBW2J@w<~tH|wwa&$F~#@#r}w$!bCA=Atd}w{J?!xb#!}`cA3)
z5>wBXbRG)%wt36^OW#c2B>o9{EW!6%RN8-YV0!z^%$Q?0Ehp_&%?mnPc)H}9>bEP&
zC+_@Ix-O}FUsCzy%`IoEjZCxeJUbR-GB0`YC!XRPH+Wv12v{U~a|vg1y7hWt>*Asx
z^DO1(u6}v;*FWQLXZ9HyTyc`DFl92hQMjx8-_i1aOE;@5vvW#&)n?(jZ?jQ#dTP|}
z*&_BX#>cv4zIFeS)yoT>|K!;IfYQT9b(ihGnP%FXar#!{^u(1b;-#712N-_+^o1cX
zB0u1hbIz<>!8vS+nG*eO6Eg}HWo50tB#^cDv2VgmUajn3BF{DJ&ZyR$Y5QT=bG)EV
zTA|I#YlCz?^8ttMehZ!ckI`L>;Wd#e)8v|;YO-`E-MDm&MbTNn^{L{bt~pkBWn@zO
zx)%u_*Ph1dRHCrUonNIWYyLXtY8#fX4tqIPI?8>OdegK~kIDRXr+^-p@4*5wQLfbE
z>H<Yct|^DU9Xnh9?bH1;Xa8?K9UfozTYuAK^X)%<>prK+UWk3Xvc;A?zCvonh4WT?
zdG7+|Ruz{Ul$U*a_sQJ-)Qi5nFKv@|itXKJ{+XwK;@vCZuO`2kt@}^7@9IUNCo6LN
z&bWA<-8&=E|J>Hs23P<6esI;>#9-&^S&=P^8}}Hx$EI$*`2Jevt<qfw<*x}#n*G0d
zwlDH=+Y3?e*IMR3>MzB3nQuF_Q#jhFH*e1R+E07$+^U$heo3Tm>2!l`Lq>M}%KD4<
zz2C*De7N)C9jnQ7b@uhw^Za6-UHW?K>8`8E45wnf*Lkjw)r%^*ZnS#!vqh?cVb*gp
zHau`+iOEP?ad^7Ty?;ptma|;X9=vO&d@aWKN9M%yufn90CjPbKy1DT3>V>}*pWKj$
z<k8IyIlI2_r;$g5qKKwV#PK)GLGdiC)o<G#Xex1S5#I6e=W>gLBiD}nOs{GFdHh1W
zbaK7prDF#kzPx^LkHntUEB<p-*YV!?Y8Q7s?b?Hnrf$!zT++@iKW-)`|KTM6`oB^Y
z+{KN|t%cj4O^CL=CDxRiyL6`WgA1HWNjiLD&2bC8WUTic75p!-t1G9f-#OIe%wtoz
zpxeO=T5gk)6f`+pC1>oK=Kf>p->IVW7aea%x}nZebL&`#{}FxL6T!RYRNSt3@>8zB
ze1>9swdBWbuNjUMcKq-uV7844P-`hkb5Ps-#45h$;E}v9AH2>^>&-i5q!+Yj$)d|s
z!=wZ(jm#z$q#DjtZWdFFnfR{aX7nB<RxPIvC1d7!j|C4Np0u!a-N%l1iXCRI@>$Q6
zxQxCiZNJLn?!n#C$TdUhM?v0g%`?Vx)ol)cK2XTJ+%$Dbvd6ql<*NnVZf=y6e90oj
znklftD1-0l>r9?$#;wN{joh}*=4iXj!90mWMSR%~bA6*HbJG~M9(Z7?GQq=6BZ`;#
za*l4sr24G^CMqg@Zru@EW(J$-249i1%33@%b%vy-Qk?CyGXkz`S$!8gMcDdo9y?*E
zU@mFII_Zv~56|l`5s7Y2=fhI1%PJ@CmT5Y$;o8{~XAR$QTc51Z=jq$E`S6>oG7pd5
zu<rh}XoIfUk=8V;F3szQx6WBFw!*bADsH#O>f9x37n!}4NN81={D0o1PjfE?T@b!x
zp|C?E+bqj`uSUVQ-XFI#O8La4d<=bjenkdn%-TOwG^0QGlE&l3NAGR0^YV4l>RxE<
z!nD;v((ktal$#>o5BXGAFX8^QzFPgsvJawZVmFz?Hf%lghbPc+?XA_WOC8uhYMXg}
zySCUT;sD37{uz(m1a_sIOBPf<rx@fSGSkAX%OkmigJtH&*B5w?Se;Z-&nY%wP1u}$
zg2iXqB1wjwZr-6Mn6<c0KiOD6VcndXulnnLP42e~e5?8VkA=kp;e~ITAKTi#R`!&$
z=sCP0qhxOTN)6ki6QiOgp1t$%sf_*3gO4MY9^d_Q|K-cIvt#cr?zsJKi{0+!m&2s=
zp00n^rWYOVWi@NN<=M-L*In$3x!+Zki!4vK*sgzf|5Kj4Dc8<k-2JoibmMHfdtYSk
zv0s08bLngW|JloP)*b(HaoPTFcLQe!%D<TF>!15V{KcZ&iD_)|xmM}<pU$OSP|7km
zV_E&>)Uiz)7M_*a5I*nv70>z2kGI;aGv65#eL(3)#PWkXqZ;(*?7ryyYwPLOSzHq?
zPK*-depp`{7rR37S=Rn1nrlC|oqVQgyUQc`!^?xNv({alU9-3{%5Jxj`z1%YkL~N~
zP6yUTS!#ZE;6A_agv|-}^2fQ~%=;FZ&bR4KRoL)F$<1Km?9(6I=0DW<Rq<9)##QvN
zLGDwYH?C)R^Y*Wg?c0`?lxQ_;UDeAfS<BO>qBg58U*>tz_k2;AwNd5dTN^LwoZJ;S
zv-0ZRvd<~Si<r+$%=-|&_{q(i=NIpG@RjrPJCbEw@yVA{U{<o9#GWZHctmDxu;)$s
zpH{o2d}5U_e~a43K3|u+yv5s$OK;E9d*7#f_$KSg-xrsj^R2tXa%APnDdv*z%qO<g
zpLP5YuQolyZS5)MvL)x3HW~b}-eh>j`oUqI-~2bbzEr%Ly;etc%8YVPy$|IAF*%dZ
zC~aHtdXi(m;${9>(|wp;UN~|oc*Yk;i{7r6ei?^ew(2l%nt12Xto2N@PPZ=EGugx0
z?V!fa6<<BY&fc_B-M3&tqH6!UL+y%LKQy=en_E$}_{2e_{YTtSmQ3Y)u*}5c#9!w-
zc^rZop1+hhwja2<`kG<hH>I3yJGnfM^ge#AnDkYxld*83%+jL%dpi5}ncaVS{mx>)
zHNPe1S8RTrR~3+y@Xzwvhqi>nrnmFL?)Ug^<yyOYZSw1VWrkaAwufEHEnNT1w6gnx
zR!_*!L#{eui*}_x2$hNOKNxoM--JNb+q=#dhE~p+7#Mrh>1pCuwKtm43cs)KytH=n
z()W*NF8a6p#nO^kA?sIc>q=(d2>P@*I3TF#agsZ8_njM;$_w2s-)T;I;m0}GDgEBz
zwmfBPu7yGGYE~E@Fi5O5pY-XjL6*St^2~o_qR+jz-|U$i^+;{o#Ku=AHT(o+7~LOi
zllYb|{rKO^_YwS`+8ufJOx6*2EO4){=0jy?g1TFluzKs|i$7G_W)<sqtGV;oZ1j3>
zp(^Y=(Ok-TY9YHUf8vQ93sO8DOfYnl>b6n8#(31DOm|VcW`)G}JxMj4@w1p(X1;i?
z`D1#LotDNer90vmcZB<_&*ga~y7TZKqbBihVs&;$Uw<l_)IPoSo>INxv8z#SSCt;m
zJ?RxIvF6+^Ud9bS>>tO!Jh#cUMxEUzhs93w_gkT*8m8-#9d9+e9aj)z>`3CbO)-Da
zP`95&T52)R%$nC?VHv>=ir<nY9basC-?P-Xl{?Gix02a~n~T1#QDO|->$yVu<e#%m
z{|s*NyEMI5>P?t*&t3fBMInjsCt`suPsG<Q2vc;~X4&S^?e#&xq{T+@af{C3mI>D+
zn42~9PHf}gS!69SZKM4*!yoL)^#wX9c3~H7w^{CCZ+ztZh40uI3FovAJc^%<Tz3h0
z|B%`4RN~Dob8v^?Z2^m<uB?Q_jQm6%v!jL&K7N<kaiC-^N6dl<UJawFS?BwnEy>~g
zHQ8V8$;3Y{pV{T_9^CmVf6lhE6PA0(eO=oAY0av)I)TqSK9tJ;l#@FAe0u)!t@+D0
z?N~J3C#W(~(C>cE{sW>d(@XlJm%r7W-x<?X;AAHM%q_J0SKZ8WeEVnd8=p+mZ%N_O
zE6h%-&irol&bMB8*EF|pQu?#}OszTHb28fmge_mVGX=cr<UaXaKjfa8Z1v)oe|q~q
z&n>v^vh%0($!G6PZq3#DEFN{n^pLkm;N<D28xC@A(KVfYB|32lBg?wZL!!J=e|dFR
zA1xCzOtRm-Kil}#J&Denra@i~*LpV3KDn)P{@jLO^ZQSor{7yzRTnC@Zj=3vKYTU0
z>kj-WUKT!qA$Iz~yK^S|Iu_-;G@0+-=826HKHa<*ski^ZgH509s<ur@oHb*`l%ujY
zcBt80J*?!I{o`})?fW(7*8je)+xV%!L%8t&o&84hj+d>!{g5qx&WzQ|WOd%Y`o87Q
zIWwc!#xIr8wfROuKMFfc^2(c4_}IQPhOqrv)7qFP@E~D^(k-V1qwQv!S&e5)&fttb
zywZHpn)0G~TLZT23yrVj4*1tBck9)f?tHEJ7fNjAclEE1z3p{T;pvfop|?5COfQ&o
z?R8IKdX=et3io*jgSDT^e#_XMJ-|1aErw_Ayw3_gPSN5e8Pe>>3je+lJGfKqiIKdN
zUE4v~A9vK&JzXf3pS@FpNk*wj=j6eC4MlwQ%6In6+!HZRt$+4|id*x(ReJwCt7fso
zYC5k~E&q4fUD-yL58lhq-#0z<W5~>&7PoiNGwy%2yvshn;coWzX$9tW_ja^Ce=l(G
zs!pQmj@(~z_Q?&2syoBNSr)L)o7DbB){V0=vqk&Iq_^8<3ManqS`cXvuUNgi!EAHr
z+k@2?*sh&;)qGcWVbJ^oHH~X_p1l9Q@rj(B^s@fvyL~5wCzvsP{8kWTqOf0H<Tl6A
zY=hVH?-mJ{Xza_sSSfsG`wNyQ{|>8hcpF?2%DZkVlaR%<(t1h0cfflE+dxjf?w@aW
z9!yS-PLMa7G-s|8_njl>SLUx?7-pSYA(c}m^CPwVcjke7(;G7TZSoFho|={}rCq`F
z{nypC9m{sjS@6cB>19CL+@F{3ZrnbdcX#c^TdU8Wko~&u^FyO}-g~PokNYoPsw~^&
z?;<<BF2~tjY=L*o$$CESJB~l!E_f%r`R}TD=g`|pldk{WRgztOeLc4-lWoHr?u$#q
zmO9@&%=`R^@FqEhuXk^L=-=|Sz+g-8^Lsy*UE8>LRgk*<mrvn;cbhFblz(d3Z?Bw{
zcbop!9Pk&hJ2`jzm0!|HcW+gP*SN09cU$|KsiCL4*H37JRW~Dp`nH_47cSd!ho?Ui
zFAbltgI)2IvF+mUf3+&!-Lf~N!@oHAxYWrPeER#?czwjRjPw`$iN6`<pL!?#yU1Rq
z(d)ORshr1>87Cj)?maCYQ`5mFWAVnzEh1_0v$h`LzK2I{cN{l6T{<o5dEK^?W$P2J
zb30l}g&eQj@$Zkl{O(k_rD{J-HhnWNxFgML#j`k%f6iHVp*10Y9k0EKo~0T%so>FC
zIYxEA2^`y;-lhq9{=R;*Uz#~;k_Sh?bNAWI$7inz(d+NdG!8qmc=DeLttttZ%_m=7
zs$Up+?n2o+mx=<fZHa#Ku6rLAdXuxHaD!OuS24v!wvv^Xs~ER@mh+c9JmZ_`{VTIy
zJKVioHb>sHUgyypjrPg~r#CO`=>N;5wocG@dSOy+R@>>5#owRA3Gw;W9t^*enfD^Q
z`qmFMg&B@HyVGYX95Fk)!86fz<tnk?KR3+{PM&!0T{`olV`jPHEJkK8jx&B%cWs`(
zaB1kx-=(wNt}owzw(c?Gw&$tE-dlEm-C4T*<^FT~oVWDY{XJg$%W!4hoWF1Hewl5M
zVE8-wd1%r6y6;g7@|EvDe_5XUYkJaQZEaKE|4tqYTfcO_tY`l-d)I^ebE=N<y-g8X
zH~EP@!)LRnHT-9e`aO7i*Xj55+_@KxOP^Z(b!Pu-5SaVx@Ad=w88IIgtkR50um8>W
zz2&7sq)&Z@^M_-dDibd)H+!VqTM((gW17WguUA`nrtXPV?l|@L_*sc9ZnrmhTyN!E
z-O5>;cqm_7?L|P=<3{taBeUiNma5%9Vrp@8@wuXBE3Z7iS6pWE@x#xc&+BAv&)F1Z
z=4+w3{Mg2QkGHRsdGN$k<968WKd}#IpLn{fM9E5H-D+7a;gTA*jc*kW-07-gSaX-7
zySzbEzkzi#Zvk&gk(8R?gsyw5qE;(STcI7fPT=Ii{MawI($+4Ezt3y7J-_cv#mukL
z9~&wK?*93v;&Lpq<hSDgoyTmJU(sB&Mw&^k|IL<1zalDLpPEtrV9GTWPj$tehI9Ts
zRV<aB+^foPB(zmk#eRv~m)vOPLmyeThkw)Dotl16kRdbPNPfHJjdxQX<{L<JOEjNP
zNSF|l@^8=iM_x%4Hj^^8ExfjT%GBK(HeQ*=6R&^N>AC)mN88lA!@T0f?(Ir=DBB_H
zHn)IF;nIbi9sVAxC#rLt-mde9k!eop16dXs?nht#PS{rdBH@fj&ITcw8CxDRUfL!)
zFKhdm`_B()tYu{tzJGD<ZR4lH`6Y%H*B^e5eZhO>OSMz?Q#*mExzqMf+L@^E=)Q5T
zvj2YFeygmg>k6BD=gxFE*5Y<(XYjX6zwV`9X0!S8$4TXaqT=?+rP793?%Sul?8~xM
zPEj~;)9&fcw`*QcxK(&%`nMgq9j_g;e2%auDSuIZ@u%ewds@DLobBe1*Nat;vM+cg
zwOTOZ-J|_CLOSl|J=wdXsnz6vwMpmuV}Esn+pfO3pS2^uf-9na|HH{FQ{JzbdQ9s6
zvxKC{_7h@0rg}Zfw9d(>d*P`*U*+Db`R|&hyH5WjYGE&Uz4v}izx=I*Klto9CeGP@
zcjFXG+4V;Q!@6FyZhoql`{8-Si(M1SwUTzfz1&{(KCs|%dwxggBd)hM_9#7&zJ5aT
z#<EAvkw<xcchB|oyjorvyif1Nrtk%q|C}zV72!6&m^XjU!{_eNn*?uNk3M%^=;9Xs
zjy=Kg{7<LNj_=x-n5=p9g4K84E6dp3YqtoRKW5wRRloe{x@3zbCoK8)Hfj5B@u_+7
za_OCA-xuCtV*jDHy6#t%&Dq;#dA%Yg_vbF$C46eO*3@0SSFW41T4tR;@MqJzxwCGF
z95b8DT%-T$<VC-p{3CmNj%;dvvf1i$GH06k<0EPQCi<T~@cwe$R>HdEE$1$~H?67}
zNnx#*Zb|tC*GZgQ-q@pCrF6pQRHoq!(G|P+if(hP=AN*a@f5>KY2W#8w!F5?{mY-s
z$nSID;+Fd+8V62XUvb@Eqk1vlQ)?H8^-ug8D(8N!ZF;>uxHME&@B1%<df6lEeix;R
zyKzfN{m)f4Q!(1J$FJH{IDW}e$Lj%aPZ`aS(fMC@edTEh_ln#zc4m>T)$_{>Hf$~3
zBzWW3fiqn{x}}cIb$$43>9cjWmc01<?dRJ)Cm*Sq1#je6@GM^OdyD;<Un{KVhP}@G
zDdo4Xw3y`zr{Bhm+rJEUZrj9k*YbR@?d>U<dm|MK=Y5@T{CtB!OwLm|zukFC<^Okl
zTwQh9!GtT+sy=_usTUbF?t99L!d7Wcxi78sS}n{-^^w^b^$WZ={%eKF>m*vdn#7xW
z-9#cxv|??)TFj#RJqnk_lQui$?_4`~P1@F{Y!=%E4+QBf=AP+(Vp((8Rh?{0|IVCS
zCONfoZIOHCnspq!a@kDy^7ZehZ;8wm{;fBqkc;_;<7+|jZmIRGt=VN0A1|m&ZwdL8
zB_5nF(H^$?Q(D<kvz-C{ADE<7LJhAOFeJ?0nZ&T=;=GCPbkvvJeC@}dZo&6uQrzRt
z-LKN_Z<s2$`SH7;f`$2pI$y&S?WBEUd8W?3^jrFpJI8xrx9xK-XDWnPMzHp++L?A^
z7jOK&+~tp_y!XCQ<6UmD%}OA1+D|{v_DH*m>55N(PvJM^EAe^3wB>Y$p2Vk%>dp=Y
z!94yB8@719b`TI-9lZO2HoHQ!lHk|RQzphuRybHxE<dN)`q<Z-du5hPU9f1&o7nf$
z!>%X4{JuB+xu$vMmkqiPRj=P`+rrRisk&_1b)Ka%TMhEpXm*ObD~N12eK)&${<=kt
zHw1m&vNbIgT<F3g==pu0m$dE@yYpd+=H>YZ`Pi6G-BzjB+Qx9bWKHYOnD+;HpYHtN
zQ5Ji7{{g0V7k?+cxqC(~$Nb-e+huASSFN#6%$pXxyu3Zm*XFkPJ-gd8z9zcl9SSpV
z{pV$CCU&oCxu(S{FXPF+{PNpRhudFOHT@p*-D6eAt@744Z%o#2VmjT-yMpad^2xVW
zg2t=&ICx1fb)By>H%vxy;-pN$IS10`Z1Yh3dB9HMdbRQsy+c=W1s*Ls;I{4*<Ff5v
zZ$xdrwj@VO-!{uj;gst8ZrT2M?!}7kyE{)5@dR9NjC=H=Q$lrG)M@iaM-q+{iT_zA
zIeABkmsC}Huh`D;9n1FiC%su;q!?PczIF*~RQ>f&%`>NW^79>w{I;UrAn!c?;u+zd
zhtEHMw#|LsXGgP5wmj!%RSDUiYpzySx+lkeJv(aG0zclp!tZQ4V~lT}nHX%?nEftj
z&ugD$tcE-8b{?KMXQy|~w(Qo=eJ6LUWtLE#S@6w3*xc)R^2}|8v;0@O=Y~yQ(i5C3
zBan0A>K7@=a6UgttCbZm#ZnZfe;14iUfy`Q`DW?tC%+B%M4Y;kV>4a!@5H3Ql4q|D
zFO%B;`sU)(oNw<vpZv?>)=$&b_vdj1XU;z2P{*nipMF01+gGF8i7An*e4W?7N!m7#
zePZYAJ9qSs-*YX!DR9;$Hm8<p-Cik=FZyLigs;t+^^^JM#~nW2DMi^2I(N?rVZO~Q
zVsrS!H@{0eq%KB%urU8|ao3$?rH*|6lRV#T*tNI$?fT8n6@PQw3RnLnx$XUey|qnm
z*)QB%x51W6iAQED1Mm5B1^-+F7}7Vg<aHb5Ox$?QNO6ActH|j8+fJ`Ayp_Te<fdlc
z<twwVhG~X#-!tw8OGTB<4)Pl%<!c|EJn{dMXk*MY(@RyqE>6nTFr3f&Em5qmzKrjc
zs2<P#7c1-v_arx7UebEKQsT?KX);IoSynQu{mfXKF{Ab5(F;-04@3gb|1McncRFs{
zq1)zvz9<G&PZQA*jh|Nfsr$3Z<5_k(Zl`(DBXruQ?|8HQO7m3dxp#DOlPs%_JzSo7
zOi^ov^NtKvKhGD;_u87`A6%XH^HQo7@8h!^pUQY*Gyi`18u!N1ytSQc`j^|UwoHv*
zQ(3j$;&yU>^v>F>^w*O!x-T2oxd__6j(BA^YvI)D^<hSLXUfOiSQlj}eRf%9w$Z&r
z6R+I|rq^1mxUpkaZmn@ysLh`}@6Ue_+!Z!2#eH&%fAqIiU2l2cJpP`!L|0aElI5km
zGp4KB%kJ4NzWF1GtEzi-VgH-s-_>0nXzq?Z9{cXG+bmzlN{R1H(-S+n_5N3dUaE_q
z8p&}(`o{71!SAc~rOAH%8NreL)@uKnFG{B$=3KVnsb0aKe#`0h1dBg!9_{eH`1WqX
zb*=THo?ChZmhu-Gl>T5%$$e&N^7V&XG0WyT@Arp4PuJcx@A~%R=~1$~!`|I<{kXnZ
zJ~+OAOI-g&qu<&uy!!W6X)Bq<B}C?l7|-m>+w9MyHBofh1B>cu^O7eoe(;q$QG1!-
z#!~CM=|{|x)=5a^o;biFelsKg?c%(o6T1{Hy-PpfZqBW3E?a0@_-e_ew)wZS>bEUg
zy=>c$u(&TRf=9(-Z|{8gIz#R8-`$<2yCOfZ_H56tjlX#AHuI(Z7vGz2dYLJjXHmcT
z$Y}xTbN#~S?*E$D!e8F}X6td0P&eg^H&*|9R?@ec*;)S1{5j_)HRgo79nxKPXzR)a
z#kZvtraR3!cR(a!!J4Jc+JU>K?q+=QX4gL6PX519Zs)FSKcD`Vsr%T`Na?pOcb@I!
zmhF){cWdqRShLb#oBH<rHsMYA`+f_q-16`_^S+-8R_|kY-E_fTMCGaO2D`+&Tf&;Y
z<nzz?@VD@L59jm8c^=uX-iQ6StE-!_JDI6>#+gqK5;y$hFE5FE`yzXVs`q9=o?EMC
z#62_pZ87DK^v&akkNL@;5IFQO=tWDZBhStrXDc()%hz0HRBw`e&vYT|fsAtWo{uwx
zI+Y^ku|(eH@+qEfZzu2VeKc)y+r23hKGxX0xTmG~c&Cltr-W_4IbyQ!u5P`%)1o!j
z@7%m&^Sc&q{-Qp6?H{(y>?d!8F%+2!?vZ}I<<M^?zf0u{yp#JG74Nks&w2D`hso{9
zJcT-c5_%qrtbTQO)B3$lZHMlENSkOT79MzH#-D9dMdQCXzBd>9*B1V+c4p~@>t762
zZhgI7`ZG#=Uf_kY;7DDrdgXOFwL7K7J=g8M#kzMYf1vL@|BvM}c9mZ|+kQp0_9j>3
z6Wb;0Iu0%Tay@p+&nH|#*8};NwpTS=;op&>^N08D<=;ZrVq=aye7{zPxv|DvKjw0~
z^TYd$$u$heP6ogI?ONHjbkRNC#vcc7RR{hkcbFsB@Z)UWvY&V6GjDXffBkdm8pV{G
zr#gD=Z-3KOZm++WZmtl=CGK|QSJ|$x169s9<Nd>HjMkbS`Ng;E9{&?>*L$Mf_Gdp`
zTAOGQDAyElHsjlC-#sbY-KYF`{@@eiqW2c>eK%x#P5Lhrb8W?XSp~NOdzXuqc8c5M
zn;t0$)k{71*gSJWvO-6g#*41W*N>dyJ(IuSYn#vumt*?p+mDO>)Dfs>jym5_QN6!4
z@9>?x$A8Wlzo?COtNA0gWga)b$G_|gSt7ND<+iPE>v#IyOzu9O*(Nw$=vv>O1-f4j
z%t#k_|K$8pJ7WRf(_WP~s`H;2r}EvuZhQB=WP0_kqb>#QvFe9z{LjAHZ1PxW&l~Z_
zULTE)tI6HmWz1u*`*=xgcu9cObd!qfHz(bDp7Gct>*w=|&iGB$wIP|WBG><~W$`(^
z{!I0Di7lQL^4ALMeJ6*thc!QWFVZ&i`mz`<x$Vt*qBGO<)+A`3+EJf=<XF_ty0^@{
zooxvYzi;e3J>4|G$Rs_BNmTdAD(eOX^PKG`I&`}J-`^VE?fda)<{gb!Q|^~1>|Vco
zj=7#^vcLJ}bvs_%ESPxs-L&HBl;Z6(++tQAwB_gDT5>^N{nlNxGqpF(qrWfNy_7rD
zu;fRSo_$_ZY>I5Lj<Jn`=5=R<y9X0@KUiKH5p1#3WA2t4TE~CRD1Y&0LiX<T>NkZy
zgqr3%P0xL;{FswnH8#uC_o3pIUAGoYSKD6awO8^=O-#*ZOV*y`1uK5PG)>%H&3P<h
zmiSZ6#oN9q?@_<`wT|0Q_UMdi<?D%zepefQDZan(d0uvOO7)z|&kDEaS=BMF;=G@G
zR@ij?f`jpei+z7x`FK1hU(Lqj)9)IM^PgjH9SnUo`O~)2%Ua3o{jZ<*wm+RDp>nb!
zW?@qGyt~gi7L+SJQ@r<lMq>1{9o>)4=SysPe&CGod;R$*rB`gx$UFPu@vp}DdcPF-
z%^Ckk98yVmy=~r(*<bu}{(CY_=y_`uq<){v|F*{Zr^$Ef&sIv-UR~?CwBy}w8J)+z
z(rwd2E7w;>hunP~8(#hJs8v(_`U4e@mas3XHZjjywJK1u{H)zR{n&rcn7Geg)>QlK
zf8%!8k6XocH+EIsxW6g>ety?G%fnu;TQ`)yPqp)wHlLREFjC7>HUD}`DF5&8Ta)tY
z)AK*BD3;{rx4pCOrT@}T@*6*zEzOr<jB(GK{Ndy5U6B>L&sXf8n|)XLyQkBqCmBa~
z&AwnT!Rgxb%T4z!c2)L2s@y5b=zrmp|H7F9EcskVcS?WX_*iVy+$}|#XLWYp+r-ww
zUGn+ori~W&Y*Hs09uKS9uq1D-!u(HnA01n|H}#j_X2YMKde25$%sbwdcWr&*i){YQ
zHh$N?RqnfXdF$F|t9d?`&R@6otaxJ5e5dbg4F6sWteyWvtm6I0f*Z>&FXVpZA#x*B
z($??ZcE-YEoVx$NGH5({Z_Hrym#Izq;RmL+;JN=(7c7{2Z`qAK+|kGD4CAxD&c4m4
z&pmtH!TSA~rP*gM&3m40v(c8Z?#;Uw@5N92IV8Vp_ulzSx8J?{?)R5X);`A*#ANC@
zZg;Ku+@BE6?f*$^_qG#i-Pz`PH`*f?v&L7lOTB;o>#u@%@n1F*fuzqOg0=HP?|cwU
z+HU`H+SjbzNz=@W<wY+&n9}$6ugv+yFOSwq%v;3varN4UBP)YzWVQsVB<1dy8^YCY
zuz}0>*sG`&PEIWAH2yl4@6>R6sOUY>!f*4j&lkL`!>6x(!DE#F;pmcl?T-1&Hca)q
z$r4m7Q4u`h&C1iSuVpPe`tbY)zQPU1c8BlW+R>Nyu>R_qqsy9K+?^4w;wT=srb@x?
zgS6yl6SeOaY8>oVf*(Hmye~c0?;Y^s>)v>Vgx^_aX_B2P)2FMZwVcWq>D>F=_46MF
zlZUCFPPbjH-`HYu<v+vpoJZvakv`eq8K%Gb_U~@m=D2)i-4FY&pUVu|_hFXwLKWV<
z`*h>4+wFb*^_$7fhr#R3J-zoeOxSq&@3e1MU1W~b{{ESsAiJO8_2b8S`wxE=aQwgW
z`t$DHfA}9*)iPfHnYrrzqT7#UBl?xz*fYF78moBSPyF?l^$V=_H)b)q*B*Mg;+}2s
ziTmef>^xgE<ub>%+m%A>Vp&h`e$tFO5LIw}?jd*ngNiW+TxS$+6B1HQ-f}9e;Lhw2
zmA%JaH%-4ev)cH4m)XyJBW?ZZ%U2$=OMR|1@49ur9<yAk)pAzryT{-Evhq3g^z`QG
zdw)Lve@*qoDPKKD+l`f5zbwhD+5X0V@>R|yxhM9&Im3HypNvZsZxUb4h0A@5YJ*mu
zm#Nz0_UX-4<$l-WJ!iJ}N*c#}4|{!2>bBi^?mKx^&)3b)|NlmEbI-NJr{AL2_ub^&
zmi{M6`D5I){@?5JmtNtU(QY~G+s(51${lyk*}0cTZr3aCj6a@uqs`K47SFoxtB!8j
zV({1YEqBd^&jK-znpb{%n7Z-BarTpcFE5f0|L@^_`<ZNQ#E1F%*Y23c?3SDGl3C0m
z=Z~G~?ubtFE{5oXUtL;{HY}YqlTBt{OH`D{9?AD>N_O;LpTh6|+fG5}^V0b1jZZn3
z?Ems}bJ67JF#kE*_Fn3Ex=rwGuaeKi=m{^M6$Z&kMU-|_{0&!>En}+OrI#D`;Om`$
z=g;rHlrH~h(6jT`>08g*uC$*i|5meoskLu<iR{Nii_e|%^}a5={D;@?mcDlxoBJcO
zZbz)UuqSr){8xLwzwuw^^Ze79A1ju}x^1a-^2p^)whnn$cQX0w;nSiIS$H3A=Qw_<
zSNT<@?87fx^aai-<m~KCU-M`7Q$7974N_hA?xuNG?z4>-Q^<Pic0b%Za>uh>_KGom
z%RJ`n>AAbYJfkpo{jnL`68qC$+HNmvoB#Sk={o%bTUWpIXLeYsE%J5C#kn?`|1O*N
zbVx3<n0IgY?{#y^eSNl+eD;6u{@*>()#qwO=JGx7o<IBMlWKcGrRVGNQ^!_Lv3b|H
z`%S6e&iBb`b3ff?aP@ppC;8dOn<w-3hicKs3Z+KLZn=?c+525{!}8uF`M+MVc6n{{
zt%^TOo;vkzz3+Z7_qyB1f)2g+u`#kqzDET0Gg35+D%Wb~O0SNoSX=&N!>&x;`1$#<
zCCVyO^A$9Ms~)VY^(|9g#(Gp{_pYt?*2tD`K2yt{_4(e+QUL}}_w7547nY}S?w|Rn
zN1gv#_mP!%R2lo{?z|_OCCKyp{8y!QOM`w*yPojFtcm&b-Hq{i-)y`YayJ)6@4UVC
zs+V+~p7NpXi~Ltl2xriGy=ivkwX^eH-i(;sy8fF_?71EH@6FG-^X+O_K>d!T`@gB)
z$&1}{Ta-^;$L6N$ci#)?_a$52&8oGuE>QljtbQV_>T03=`^M~VuYd3QW|My)KHBD-
z(0Rp5>0cLgFVEAF4qy1ZMPP&Q{bRE#wlTcPdV5g4#%uq+^@|qpOU{#u+%-YncyV-8
zO-}O5<tJa>pYAsG>tpuCr@l`Xw&ShZE8RUav!=`Mv%!liJPVhlzC2lTzv|HDqc0Cn
zdw%@;vq}ABC&GSQJG^#z{Ab;&9lVm$de5|+=X$x%Tq*WgM&^9`x3k-KOGZ6x?~Ttp
z{*CF)y{L2R-iBzeTEu^B)w^vo<)#EUKTO~7dA<sx|MiBghr&&#H@g1b!O>EwX~%MC
z&b~9z3)AbT=y{on`LdRGHgU?nJS??f=Jo?0**@%1-162=`0wLdrERX$Uc2yoRLQ?-
zzjAJp)EB?*2@Cw0C;fQk&%65e*%zm8dQ=Oaus_(^<;_|6yQr1>a7X_vk<8hT7MN6X
zP58wVZEY0$>c+mgmrU-g{mZ`W``cTuzE0W2cwpPtDZAJiW=pKf{$?)3%+>L`MCLrh
zH@|~hcl(#xD3|OH6RF*u?pwZjUeN!-^WSu)Yref`{@3>R=Xp!No%yEoKI8Y(d9~}^
zer-rUEX}|@|9eiwoz01?2@jbI-b{A4oRYquwNEtvi)q1;@&khQ%<eTzeLGnCelS`r
znErZ0VE?!HL&vJuJ!8(fb2jo=zI>vx?IE|QgKocAq(f4VI-dL%eItUCbAMa!W7!;=
z*8lg!BG+vw)Jc52*O5=~xZl-k^Q*U>dz@LPsu{FnrH4u3Z60@j<^KYW@BeOjx?1e<
z9J{#Xxx1YL>t>m8G~@+*x%Xz}zgg>9R;lj48NAI%{_y7+7jI`-9s29Ao&R=hmt|d?
zfxmPAy=*P}FE4)fEzqsq)0rrFaP#%Q$FwJYFZrwa{ExtbmG!>wPp(Tpma`}A!NPZs
z|JGF;{?hB%CpJHP;@5PMuiyNZozv2_-_JPN|IDjkmX8r4$x3(i>gUe5x|Q=uzz;R6
z8%6t4BE<Y^kJW9zw8^vQ`QDfJHtk!_-Yft9;rGf5u|K~XyjW`IcfT(>X5EuzKl}LK
zZT^%S>+W@$>HLoc_avq3Ur22DecL-kp7~qkOY_x<ZFc;={>QJ}`Sk6|mxv#C7hL!1
z7fZkXmr+UDTR-fvfZzp-Pw@@gc^(Au-zk(X3F&xY5?ix9ZvFF1x08}L<#_Q<S@-za
z2EEs%@;8i59EqsV{d;N|&;PG0k6v0F^>3A)ooHr#HCM2-{Guy)$uGY)y~&#S`tO>Y
z`1gD7y3M^8Y@g=2<v_;8p6GjV!5fVx@0_x4o4u6DmyZ8jj~g?r3~%oB%H5rN<<d(Q
zE34@5D#>}fue>X)%~%*&+pBnZ*{}Sp!(mQ>b@nB;^Y%W!EgmcHZTm~KCLqt{XQl;f
z=12Wsjz6Z~e0|?>Qag{T5p&hv`CPjb9?1sSl_$KmtyuHtR`>3MO<x!8s<jDdUG}tk
z{i}{=qCfwJ^iIwH&Ka$x5*>d3<Kp=pd%w&s-MjJqns;ws`E7doy{XB5j``|8m-q0_
z{b7?o-^A+w%{@#1ihusrDEa67v$dB*=kK~?{QG;$q2-=8@0CB@=Q&^0Wxn#AKaBgN
zvj67=ZBs9N@Oeqj{^<D`wm0`h%>Obk`%hkw-10xXTKk=UR44CTq9MLNYJTFWe*Nq2
zPj2O3J6^S^cDr5WyOZDUOl9AA?nvc-%OK9A3wBCVxg~%6J8;+T+cV*n74I+2+ID-N
zBJXp_TXk18mF2gq&Z~63Sk-!dO2g*gp6mKP*tOVm9!<aUw)n366Q}c~n?KynD9}!O
z$^BSf`;UUSk=nEVob)vY9b1Llzxms;PB?w~SCUk5pqlT_5);9)BZBGQQ@%dX+NgYE
z&+9{4nGAaC()K#6i>!7zwzrzyhVSppH$RhJ1l^xs5@ery=i@0GeKy}|7vGhsG`#m0
zE>H;gkh1Qc(*+l{yJoxx?AW~bJzn&g`_3mlokYngf1V4ht+#brwD0Nyvri7`F>6%3
zpUS-s+_-tU!WRCy#r1obHDwN0$#JnAOD;88U)L1(e@~g%6Zca#v0DS(O)aN?-6Z$$
zz+BUeS94`2&YX3N{rj=^`rHzqTQ;3izk5JBI#BFbkHVi~V?*h*MCpe=E27@cs1kjB
zwtErBp_j)!wwj1_h+Y4hcsK0Iv+@-iO%9)%|0q3%@1xNnIjQX3JcTQgEDu#}n;T`c
zVy1}8u2zXcedE3vN6Wsa{`|aS(bAj$=gFR0`n+@6$G0*!diP{!Oz++yd5$O8P_%=i
zbxXv{`CPwPrcO`NW9M7;{y*!t|0|0$_C&t)e)wZc_^UI2c2*jB@1OmZUp#ZqV(TtW
zXUW_E_OlmbZ6e}%<1GXP`!!~sF<($oALmgUzc{ZfCI7>&D4}gF2R`L$KHFMn9xMN&
ziQoI<l9QWmM(G@_7Hyrj$hto$zP4ex_U<RaCtvFEO}ntvM6+h&+C{R@Za?-c^4PYr
zQ}5=Jz>Tb_8n2C)hS&A%JKH<u-v-l?>qo4UuVkLf(_{arIqUV$HMT$37-cQ}E%=q~
z+u6Blwtnlgb}ZDMR%JKmUhkj97M}!zH-DJiR&e5j!s;CEZ|sK0cdvYTEAXKIhE1`v
z{{<HpYyZi+@b6^LADN#0vu>W9>s)L8;^(XVtr6uN23vLQ=f6+;`a`9>^7;15Q@6do
zCwrQ^^0$tT>e=~)^PV1abTo5$D0C)e`DXppf}QDMM_#;^oG%@5=#}~2$&oi_@6tbW
z?(MUq+m-$EpIrEOaPgnpkyo#8bw0gw*=_sm&Cw6v&-irdfxfS7)su+(ABz(%|4uA<
z8loM1{pRP~N9X4s+xX?^@3N-*KeitEd@$h4?OoH&uYNtRz2*PcM<??p9SP2q`ue!H
z<4BT4uI;f)Q7%nP#xM8oU4H!3BbnX0M>g|Ha_={-{Pm#e<Gm-DmHP{S|9PhI>(G;z
z6IW?+DVbl5U8dRje$J#v(p=j-+P?Iu+`7;It5VZ^mO{<C--Q=8aqhcoSKrsjbNu|$
z`nc~JXN#Uz_uq@uEPv=7<a57HPWix5PR+~y{Ga~3G)qgn`Rq}x@Wx-Kj~~|#u+w@o
zKjpK|9=ZHY3`gJR8p+S`IDX0dD)avPTQ8aKs53a2we)&#clQqm=A+9eZol;U*5$8a
z@2o$5e<FC@i~q^go6P^$)N%7njsBr+enL_HbMMV-YT1X6I((Pgzng1`s9aTzab$2?
z>cfhUQG5NbO@5oJ{nw$J@BZ~>Hm;ou%!O1l^Bwo!E4A6nJW2o1Biq|4alZ2TW;4B;
zxBbgql(c_#&~2&XF6KX1UYcdI&p!6Gu}H-F+l8ua*LhA<-CZ}m;cn0RH(8sbAFBL+
zB6y>^Wc$suuRFhd`&NB9b9Rzi9p8dN&AnXPmT$Ztd+ro#f_PhS@k{9guXg{}QH^k0
zHp}~d*qn@i-4=r1<7Q_5U3*YOP4~%n#$Qb7^#=l8KD6>&mG@q8?Y)#Z(;0G2+t0S&
zNIDfX`MtlP^=z>eQMa!PG)lT;UxzQAp%q<P#pm()?Het<OJ#3Mc9d4V`CP{LuFJVI
z`i#==Et`Cn1m9v_nKgat+T{CkESuUbeW#{%tGOkda_G@jve?w2BGUcAnQQ$IPou3Z
zU3+9&cApCE*)Pp``lwrxEJHrylwaH1lb+W8S#tD%q?}^kcfR;p(QZ#Y{N}&^r?-0V
z7LJ9sQ}&g5edM<{4EP@Rlby+oS8<Korf1ss=WP7o?9;kNaqm=#H}7A?byt7+xz_WV
z@^J&PxB{MEH4BaQt8^Yar2S#T`Ff|op4RzM_vY0sTvxiDT~@v<u-kHK`ls!ZIp)7t
zU0u+AslSxp`zQCw-8ZJ+j=J~i--E87G3Bq;%{YCYzo?PF>dVFtSxcSMXU^9-F89<R
z`QcSB_LcS1ZU^n&RMPY_s$6?d)Jx~Xzn&~8dOLaQ?ybl9kN;-=XUL?v@^W>6%kOO(
z)9%R2mb=fM63TPzbxm$?b^MH3Ynqq+dj9NB(O1609jBjdJN|HM5w}?Wk8c4w|7KZj
zE_!`*bH0D~cQ>E#X*u&=#T@(ddj7QT?-$i7M5{h!q@VxfT=(916U(<J^O-)z?dZSS
zv|sc4h6fKBKg$T|ZomB2e8ryyPD^ZL^4oWuI~`!xR=uU@^w;07tQwB~y1S0~MW%Dl
z>3KWS)HgM~a?|Wj*igP{{&tHy+r#W)`7dvg-~1z0rtV+c)mR>T`v+N<F6AFA-+MQj
z<NHIui<eh8D{ni_67<Z>ao=t0U)N7^oSQRA>Y}0(*V!L@25oy>R>kN3`cq<byEm&|
z=wOb{9;OpDOIzmgT_|O_RB^BKR@>uEH@fa~nXS2_dMm&9#+KdsRkzF)SM*PH{Iuv>
zzqq;JiQ~uKPu^6keyLLK_pcp>*ROPW+RHNfzrH9^_<h&uCxTxTq+`8`RHDD|SC~n2
zntT7-m;AHs5PxF9clWBa><_;q^7_g*73RLZY?O6wLh_ECPd+_m)L2++|LlT3PYmya
zUF_yNnLVmHw0_S!yS<42^M`7YS!eB28_#}{easW!vGCISjZCN4S{Yfph9&*-_&i0Y
zNh*5h(-_+pJ!zG_uQg|?p19opwp#MNMdZKCt6_3m<lisZRHQvI?EklbHx>GqqK?n{
zmiBH7!z6R}<K~AZs7{z+lzTBoc%qKjA0_#ny?IMl<vy({cJn!s*7S>EleYdgPX$Tw
zE-vl#kke(V(Or?ek=q}uM9+MlX*KJ5N1bh8_71K~9sljteTewgdun~>ig_YgYm=LO
zMOI#1`%z`%3BKpImgl+5I>R)fb&=?<t7or2Q%vFbRwVs-)v7B8lIKkMT%}jQ_tW#a
zs^nC+>PM?j_wMuT{&A>$R+>s?)+q_Kh41@crY(&U%#4ZZc>n9XRQv4bN#@h9Z{W>a
z5MgCm`uG6Ht3aovuP@%;-&P$c%jA1#$BU5PyB6&17SXgR7HeIZwsOAx{eW3-_`KJB
zQGNEu#Bz`JEeD3nk=N5^Ki_px=*qf@7aoYoap&hAvlmGfFYzwR{t-1rK1X?rX7=ud
z9Xp;>sycJ`?7v<z_sQP1?}V@9)RxX?zray-db6(a-QOjWl6)_U8{WOHD){0&?XC0q
zQ!8zS+hU&WjqBBZ>b=GOyu<BJr%wvMig5n!{AUH*pX03$?xj7*pZ-{P`>~Ce++J`P
zi9h^(eQo`|xh$9Lm#3K<^;iC?u~uL5=kZ^w@4mO~${*h>ydb3b^>VJBO<nJ{7x^lR
z+ppeSCvE2!zAJ`1hu=ZnW75`3N7bJ!ROs*gF0l9P%zI5+Ji9wS-=2Ev9oK{MANw9X
zk99e1{6_utMbU4(mG={K82;^9WajoG_D6Hi{BVP72h4T-mVEr|8?LCk>cRJI+UhU%
z{ciC8P<?x<@rU+A&i4}%7EMTec;-koV}Lwk?e2t?FMkw${-m|z)a}EE_?kuMURe5p
zyTEYfPmKpV!)yF+zO$WozTxWgx7~Ypyn6VJ=Ur@Kv7_(Joc2G-{lC85y_XuHUpu|{
z?zFnA?yryjwRmMUQ}xS5xnhOFzpHolcP*=7IV2kA9J#*!S^sLqI=Nf5T_Oz0FYbLS
z{jYa5H}ik!|7E}Oye$G{-@jXN|5oK^+sIEBqTFBocPQ?6KX$n?<$kvP?CXEF&B|$R
zz5n%7@XfPa{99h9PQ2VAvt6`o4#)puQ_tJgTyec;KJQ+XYt6b}XP4ytYmfGb>@F4(
zSRZ*TsAH>%od1J~Zaur~)-HA4?U`{q+&M36UBQ~#$}{0*ziO7|Z{K+?{n|p0i7BSD
zkMBEe?zbg5@?`yU;nm;L9z8s5voY+jqSD8x&ok^+ywoW?`}O*n$xj|x`6j;?3oQ}8
zGtYBVm(kqo(<lEl3YfU}IETMlcYye93z2inpRnm$?_Rkxs@CATn)HP2#%{dFxy65L
z{$J>QZe8->3sO55&Rk(_Zzg)MNA2F!+aU%|8!uG{=e?fT_sgVLVE>F6xm!yPU*PFG
zbFR*Ef7F(16Ri!+Uv87)-ZbUz_GNh~`R@<^+f*&kHFM7nSC#cjhu8S=?0p+1q4W9C
z^|c=Jk2dc;t;GKFquIJ!Tdy!N=j5g@X&03@>;JfG_1)soe>d2EW~DFBUTsx>>zrP2
z^s1USQR+^=TYJr3JbeCpb8dZXdh$G1OCz7j_TAgwSJ=+f`2O}t-=yi!ZED~38~(4g
zTHYaXedqnKo%1)YO8jZ|^PSnxzY<frFB!ae_&mDtkzLcHxfe}b-<f<je|%th(A<Ow
z-!~`b`4)3(c>cSa68HR4e6*DM<~4mQe!G3y-kMjuexc58d**%l#p@MrzE}Srb+>H(
zu@60^VJ<=1UpcPa-czvk|GkRIt)l0z-*~ccQSOSDiLE}DEd!jqJ2+2^-qNcKJ1x|f
z%DF#sTG~mOUu))sS*|P4DqD4?X8VfRG^v#0#3JFHDL2|=wzGY6HNE}&>SfD)4;q&%
zo6MU1Q`#eXZb9-zewGIw`2h>Qf7V#Lc=D2`%rknA?q^(AYhygWQJe38=~H!PM?3eD
zHS+CBw$nCuD_hOnIj7;_E2&oYhfiYFKXbZlm+m=s`-47j{@jv(6N(>PnQ(dkdEH0F
zt<8QfFE2>QU68c=+q8AEQ6hi0B<<~*sp7fz)`zS_#gF>-OUjBBJ{uM{?7YJ3`lh&@
zwfOe-_3=f){(Gxf?pd7wcsL}c@%!SmL?*@H7{0FcYVGmg${h=3*R#&urX(u1=3U9n
zHxrJ{zkOBEd(#ASlb_S7^yRzjm^Mu0dcfkaNZ_nwO;_co#Jg+Td_#LoZFKkT4w%z_
zw`M_r{O9vO9^7T=nKh?dVqt!F{(_v>s(O<%jVJ0I({=lK{Jg-{1%KMP&-|TX_Ggbk
zwA#;KvKx;Ez3*IGc=%4{bpxl1-xz!H(lnX(@YK2FU$@VWIs7oWHaK1H=J%7A5+uHS
z_&nJnVcq*K?_|TUBMx4HMzf#!C*MqNR!Y4YR(Y@M&LfT0GgF-Mxz@H;KF;G6-}UfM
ziP83T5~pspudAt?bz;g3KK1-r_jWnFOETLsr#1Igf$*YJX0eCvr=(RTJ-D`5H(2_x
z-SQb<xBs>={rPORreE)($l4!$=enBND(|r_ecSXdr`lcMY4^;xf1Wc3tZP{{`ASu%
zklu^um9zeQUbe&ex&Zs}SH>qTPAosn@tbk$o#f{od#Y3<_1bT4y`s(_zh>t<<r908
zD|zJ}uFSEiKfM0P=dZS9`zJMZD}FS4HZRvfyl&3y+#i3Ba>}}vtWUemUlMkUH-7fQ
zcg=4mFAb~SyYf@JpS7@v+C=tF=2D~R%@_ID_)k>r5sTN0aJp+B@7rJh>gwmM`4hBD
zcD?V~c0<QMxZwA$t>>TS?K$F~8y#}{R!x84_mHl~6*Eo<W}Vo!?D~U%E6=LzTK27Y
zv({d`b;Z^bA+N0$eh6v$b)+P-WJYPGeclI?gE!3>Z_U2f_jLQhpko%baqBOYoW8br
zrK`WpOz{K1H@v>Xeq*=6tLbL?r!FP#syZA|Zv0dB%9~9(2OhlNXJpP%YW%3KTjb4+
zxrtfPj8{wFegF8pbp5uUyRz5oZQfz~=HKzyJ+WsqeIlP$Em_2ApLaR>{?0jXJ1^|!
zNm=*hR*Hd+ZQx>m*{X)AaY-G?Z7a*uZhpy2H`B~v-W04BA$#iH+jTqtY&pJe{;56d
z+s|)b%_tYXnDJSYSk=5Qxu&o0rtDkovsT8sWcACPQ{T$2xaY8W;<fX!`wp)&?k`F+
zdi^_e-Mp7_7L6u9*04%duwQ;`H?_*@?;(f1_snJ2Z_m>+Gq;ePy?xtqpV^L^_f<TO
zbc@zZ+;eaWTR3-YZ=VEX%#Gd0x6EIA?u=`>pZ%GjWj#q-zszFQHnzQaoRNF>o!Pg&
zt!qNxZd)^5c-Ngfr#WR;GM)INS7E^|f9Ca(HNUs$q<I+hzA3hhUg3OWa`a=nro8`4
z=T&W<lB#>#?#!#{PxY6-2v1dOJ@MjGk+Sz&fxZ$hiRp=1-{bP6{oiLYsmk~mzu(d;
z^zG)IoWK%`%XSx>ct7r4{!!<7sHy*rcWF(V54`^OaCg|RJzvk<zJ2FReeQd=b*;0v
z$9P{2@i+Igk=F`)rDe?>Tb7oqyYca>j`!U1a^nBAtas?wT%PDx?3lGi-ecYDXx+~n
z|3%Nds9>V|XXO%IU-L$Og*p}0dt$MNu09sXY5aY~E@kVJo%i0)ikh)D@9dgOzHS_K
zjKyJ(%ijMDO-f)(`MTqijz{&4qL)wC{FxJ0ySdROeBT|@+vch3CZsQPdYpSw{L}H)
z_G23_&ELj9Kl7YTWX;)U^Y^CDS7o`s-6ZMY!xbDd5p}Uk<v(&Q&C5;uZn5U;ny;S|
z8#l|9{960s%FEtEOV(LtKjE^A-=DX=G^zG_xpjVaok--DOpD3wI?@mKb$a}oetf2d
zr>LLzu0#4ScO+b&YETthav|}Kx|*!A*Z~nq?i%%nVnTP9?CA1;<X#tVo)gx)J>R$T
z#+QiiU*$gYJip|yg6nTd+v)YiGtYfKxBvLWqY?8qXdhG2tC`WJwtb^-qsLs`m-deo
zCQ2-I*%(=L^L%M~r_wg|wA`y_m)=qOUw7x8-8mznoYxP(?^-MRcxSD;qU`bQYkz&d
zlBPUY@6luSODm7-?&13o$S{4s?BQmeZ@E{Wa^_rKVXC)%`WADp-g}wi56`dKme>8r
z>iWxFyQ89ai>T}{kDgLv7++;)zb#VCxH&j#`@Xf`Z`|0s{M<%kT}{8l^S2L_ch~P%
zI%s+^p8NjgNlkuR1bzwUCQEJYEp~o-YvXLrtET_=a>|Al+HE-<tGMl3&&EaC3O8@9
zv1d4W{LatrJK~!j@NsN3k$s#P*!_9+X|Fq1pBLKv-*zx!%lEWpMz>gU!abySea`gn
z+5IrF&)sD6-osO;Urm>l*47kPsAy}P$yB*vPPXgxL$9*mr~lcn+ww&I_!e7+`|Mp;
zn>g0<{HWik=YPTV!LD24b*KLF+!KFs&#vh^)2{9M2Y$2NH9y!g_r<@B-SMu%1-2)a
z?wHKzyH7e~-kwkIM2+r#@km>^d-jzN{~j{0IKg_?zTuRDmT}%|<8m|Wz0%vS2Co-d
zYA$-G!|3-HYxRXY-8X-f{(DQm=lzY_l|pURUehm6u3#`%{`%Hy*88%{4};m?PkZco
z)t=`U@AR8p=c?=17F+d%#5Wu|I_HYV=hdl4ex!OD#dP=GU%}+_a=J_N^b>F6V*T@T
z-2G+h`0AKgYh^-&Z!eg=fq|)ba{kAc!OzcBzd!jrUGs6~(j7|gw_TXAxjtpb`iI^1
z>;4;^e)+X5SMS(==T`s2`7eD9ZwlM<o;9t#5%I!K;<|O&+iR}7ta+a$@zm*lonmJg
zWc!(Q*Bj@F+izW+^7`_P>R@N->AGU~8n4}&d&X<6S@my&oMPb(^0C2nx82l=OjAST
zS7uhXbTkP%h{?bG`|yJ5zwdqf5-sQTwOw`P*>qsvh5*~oA2!dN>-$)5b=f+LKf6OC
zGfyAq4}QNwak}O0y6CCi$9~7;e?J*F!^2>2?t-_<tLAyFot_sG`O`30x{m+Tj^l@F
zb@fDR9l3Q6d&(}lA~s=bt<>9C8N1pU){?hxEfu@@Z{?}0)34+t?TZy?wWwP4?waeR
zKZm~`XZ?9BDfi6I7uP>-nV!G=vBk#k0lvw*=2|RHD8K){Qg87)v(NW*Ud=DK)p3dE
zF#m}FK6~A`4@(c`zBnmfynVxWQ|rt3Eh8gVF8}h@NOR-k`oD7&6Mrwd#=Y(zo3vj3
z(z$kLYIWB3JDXJeH`-LY|7-QC_u6sKrRG;l{4_rM@5PF%C%-QLk}H*X`1{AL4&SA!
z-X7n)UF3;Cc)Dg^v9v_<*UF6#6y_~>eUt6)&zybtMDAR0{%vr+?e%8EBQs;pd|VZ_
zQLWBg^OuwFhmco^C+aR9e-^x_X&3ACDh;EH?+bsw(t8qicWv?(6^VWJ-#*rTonlv0
zpj<s;N%WrU)9Q{}z3sVj-hEB-*1ERqKd!8I&wF0MzTWEGx>x-lAFsOcbi;2xNxMh?
zOYU43&)jz9`0lTBxlVt3kl1(c$L9Sr7l&S1y0(7C?zz9a__h8%d;FWPJ=!_`gl$;t
zqG`V}?;mY1z3HaU*?F)0<?i&qLB9=N&k|3yvkW*|%=SC7uXL}Uc53gFUYE>5tH_nF
zC%ip1Nj>j@=l>Hy{OjEFkHq><+u)s@XH|M<ZsHWPKQWmx?uYMBP3hgeEk5x}Z)QUO
z%H1uj+&8~*=42XtXmJ!>`K)HMvwP(qv+236?>~D!T$W!p&HGD*^e?liZ*|T*Pgwdg
zPss0J)&C87k9oh{yT1HlK*`;`Z?&G^XMcHcQMFica$i%Ku0{F5M+@o?{<)*`BKqR@
z&AWZ?=1-`4`|p-le6EdCsPWv&Id|6|*!}&`+Fjef>VG$T_h|i(H(h%)ezkH&y^j;V
zb$?m@()<Jwxr5L2FSNb<a*yHprs`(dS6i#gPi@#<xMhOU^mo0}BDtk!-)Fvm=uyF+
z4YfgHy{v56w^Fa`8Cd)Ol1^WB|8P!J?<>b^Ne$aCWYuzSZeI6h^~x-X4`#3VzdL@>
z-Kx9oPxH=uOpW#Io5OjJ-08Qzx~cZf@rL~U^Ox<|<R-Q8ar4YvX9fSiKm6pkpV&9+
zc6rR#s`d8*Ur(?8dhki2{DSZO2NusZtZTm{xA=sg)64i~cc~arvFC?`x9+L63$jwE
z-D<1P_V@Ok(|%>KEF!!=ZTSjREya!&vcE93-M)ZLjNz~9rTDT>Rc8O#CWO5yef{KW
z?{nRQR-%8;^`^W1Kfn5e%T3#3ucH*+-GBHq%Jbyir{`1;M=YINpM2VOyJ_OO(+Y{n
z=R??dZ<If^3;4s^Zg%V5EyIaBgLbv<`mc0-&QX@xg)^FOCg-gCqB!q{IK#!`mn*hh
z>s8zTW8cP1o>=L1ZR=iJghq4oI`V9H<vDe~S3P!DNoK;Ddg(JC^lr-U+PRl=YGv$)
zKSqxuc9?EExBGdd^x>zyO{=`m27j5`$7ugFsxIV6dun>nzV}Do+UY+FUA_A7nW}^!
zP1nZnynpVmU)|ETl07$a>%Id5Z%^m%+O;*7E4HUP?_<!(GWicNCySaYW2>XzEx+p5
za)0lK3sYZT4d1Ezx$Nyhqw7+VmaXiw4nDZFCh3yN?#@eFtln>me)o3TU&HM4mD0hp
zLnoQ#AN&^Evr*Duenvv#?1xONHonVkb+k9%wf4BM{JA4X<3!%@YHiKj{deD)RR6sv
ze=|JH+$mQ+!NV$r?YxuUpV@DwRqsx(KV|i#hS~9W(y{HX=I@s8TJ|JbDdCjeMd$aM
z^sHnx_S@R|Z`JfWCDHOd_u8h8?YqxhDq5qlTPb~us71u)*=uhu=Bn^y-5V%TZm=Wu
z){NIm4)*p1q35sEY2Pw4u3U69YL-Lt&(zKSiw|{YJleU7Z*Kou-JO%a1X|y6D^~q;
zSwZbIue+Evr?BjYHs0%ZTECZdNHxS<`19u3j&-%MKfi^(-QLPP_2X5Sm>u^!yW+xq
z7OyN~)D+#hPv1sIujQBMlTWdCKa1wyI%RiolijSOV!ylb*T1jM`uXkevK6<|A3i$s
z$L<JYhU@M<sWT#1U-^~(UVHKT?0>ChIq@fR?Os?{Z7mlp7Z<bjJ29E*v8?#LF1v5X
zY?HU>iAIFKv2v4EO#HJ+>x<i(k97)pMM*(>8t=Zka<Yc6>cGjv|EArRGOVAyMcQBM
zwer#RGLemPSDt=!TVKJ~9I@q~TE;^Yx1Ex2f3Mp)HS%D$U8cmooDYk`y>}<u#OA!Y
z{x@5ATdjF=?x9;7f0P|-`?tDfo7eZupB;C;x8+@a&(*iWHuAxG6E=-atKNOC>bDpE
zH6bao|L6RMBi$uWxjkgO_AnoJomelYb1Q7VlUAg4bJVMa)%)j|W|w(i_wWhcY<%m+
zJ)ZKcYkwlEci+yHt~nsB?9$VBf4=ysqWX<)ufBY{^7{B*>o>>sdd;3Xm@oLDJ7v3t
z*?*2hM?-3vADr3US-m%|-0->76aILmjbC&2OTPMjv+lP3b*DSEI-mEj++N;y*Wm28
zg~`|K%O?l@TQ`GE<l@<n--G{1Mh8q1;k`e(=3#-=;|og6zvjxEX0W_J_0=Pd|F4Ut
zKQ&Tijk|x~ge6~bpvBkd)z5jNSMQ6>(D$hGSIOyFdEw@_(0=LHTMKSvU5mcHwpjAh
zx59WEo=V~TA5R-Z@|3sDIMuDCsPiJ_X!T+9)t}Ctj$OX*Nv3~I#P8dZm(mMt1vmBG
ztuqz8qt(Ab<M>3mSIMU~#s2<NamL<RS^9^n$+UI-)$cynbw>PuDkuGC-u!1b_x|Fn
zegFCGuaj(YFDvh|%U+g$(5m?AUhw>W!$pUy?N@9(UH9r-*-D`sdG&E!+aJi-mrFj{
zb^qXJnTvIi&u^B0%YSIO_P5P}&oZSivY%AhoO@_@dPhC)xmeHt2Ioa2{>jbyTs*yd
z<uRXt1NWbr9MV~5my@)y?Xoi8pDTBrCRDY>bG>&nf3tH__9vy?n+<PleSLI^a+&da
z2PM07z1&mV&wmYSHW7$_Z}fhBp!bTK``o1WAM?!cY}Vd=wL-sA+cY%a@5BDjdo~97
zznuKs_q@%7pQnXR@08M9uyEZ`4Z8!^4r{zKn>uZugzK*je<o%3+i}ZJyU(lfxv=h1
z#Pw~n=G&Y*UmE@HS?H&={5Q<D)PC=fy_|c;`O$5~_xogL#lKnlMauK&)Z4f39CwY~
z_40+x^XFB)k{9x9a;x^0cx(^S-jTIXiv2_2t?Hfy!S7|=Bv#AEt>doS(_8X+%Y);m
zKGpchi~rehp!+?Wi)ldghp$p`tLxufyYgmh*ZnHXZ7&#JGoFv@yK;Z&hr$ECe?G9K
zt>3!;<y9foZD!B%d~=^LE_wfX<Cci+L907YaZfnMR&~1fdei3Zr?c4(rCRFW4*neC
z@kg)aklhNybsw(S`7}o^_@3LipDpcCp-Ayp|GSS)mpR`Mb32*z{Qc)wwiX*BS62TL
z*m|7pj`8~N^B(d=lQ$P>>dsD$n7Q$u_dIR8UoVzsEzJu#{UcH8LP!6$_P?3sFV?Yb
zxw~rqtGBisW^M6`a}v@||8@4?&d55srbfXpz48>-!=A1C<KKRImFD+5qF~KCgUU1R
z*AHZd75d%%^G^5eC5Ed_`>pr&bsKFAFZaH$@gsUh#=Yh5^V)7!=Zo^&AD_1Q?3E>(
zq<@*VF8iAo9I^8DG6{PgTc68%^*2P$N-e!BWHE6)|5BUQW$(9Ly0Nl4{j_{<t?c3C
z2SN8QJzsfIng2&lR?55+pF;jdyfsVRcV+jskP3N@7gFXqyK84u&D0TSu3N{%xb)r~
zp7mS&e#?fPPP_6g<n9gExU0V=242{Cxn+g+7PBz><;BN&!)x{)_c!a>7npy;_^tWt
zg}GTda*LPp+UK91kbY#tuc(dgJCBQ<5Wn2}NmM0rWoOs>&co+Z<AZiD{rXRH&8|;v
zK^ud%dj^Y4{8xK->dR+8e+E~*JN3_|;dW}7*X_3Zuh;3c2K>3w(=v~-z_6kHX<Xvo
z1(!c3>Tlkb-BM<1_$P)nx_s~S*M;97+&jEILu~PRw#c(tA1>YB_`+`8v9iBKwe=e^
zqB|9SFTdFFVfAa}sEcoJc3qg@B6)u6+Qog=e6vsA<+@X>=PX?FEilJ*`nuG^*H^yY
zrJp>bq&{&)8<$l3w{*6?^5xH8Hv9TMfAKD@DfwTa^|6D0-W@);ZQGC6R<FBfKDf3&
zrdaQ+MEjr9ZhAbE))?Kme0=HFMdG2KjgHrT`}y{5S*+sfxVNEGOXWWlNsDbRsc3lm
zC+~pgtE+FfDSb=bzc>E;{ny{GE`MwN|AqC2rH(VJUe8amju)2u{ah|>{*%M<x4YLV
zeciVHeDu?^yzFaVR4tzVJ+NWE(y1?9;Z+K|OTH|N@;bdiINWaj^=uujyxDHSdk$rt
zSKAb={d-#y@3j;6^5z$wH+ekwUuxUBX?vz6&AjIQ`h$R95u-rHgXYe!%nl~P7nfT~
zxp?<)zK}n8H{bR(=?iNX2EIDAK2d2~q1v^_Grt|n?^(8Y(W`wcE9P|NT8h7(+#x?L
z_{xXdQ6ACt&mCL8NhE)9{HvmRt({A*@v-3HI-{lEWe@9ZXr2Gzz3jSe)ypa`rP;ll
z^^YNZ%9kQ7t^VJIM&Y|w@y&d%9g%Y>z4!ca-(Lda$-8T6H+x!h7R~nAKQH-z+2kPM
zYg3E&-&u0~wR!z!Pw_ujB0h@GE_oaJX!f(ziU0B@pQ^nuC4N6^xBK*!=My%jm2EqH
zy*xos{-Dyj<r@1~yVJb6WN+p#RQk&v@UhQJ;^*}A6%huO-x6(|CqMenCVDR=C23zB
zr>K1M5nJb@&8r_heQmvBhQ8;Q?YqQJ&Yy7f->d8G$ESbk-SuI2bi~PdyIz{jOUtcY
zVdJ*$w?lq^;DKq~6YbC6G)tSMu6}>=_IdxpQtrBbEZJ7eZ13D3?J`F+_RpE!xoh9t
z?y@%CqPyQtq^3{l>rVIAkNxH6pP%$S=hb1>(|aFTC;jXD&&wPYTUYyN(>n1?-EO<Q
zhVsO(_Ax3Vc^=7k7~UtHJ8%B`o>rdXqv_h#v5~*5+l=iPT`#HTP2U}0eR!vZm)>`F
zzq<PFQ<rVkDoo^l6fV2BHdj0AaID#$psQQ`OYQd0IC-?;_-m2*w{-oP?b2p=uk$b2
zzwMLn#oxbzY9_bMu9Meev=O_d8fq(f{l=%;PhXs0v01cM<QlJK9>dxE!;&S9yEop=
zR$(yLy>hrAy037XhSJUOIk(tTOvPeX#PQ92-M?v>dxKo}x^`(%m7kT9cd6%?PTR|5
z(mmOH_xg#yu1w1)6|!~hzWh}7vD8`7=d+LfSv`4$=-b(;_qaZ8Dfl{l!CJPn{@*kC
zU8kRz>7!F`^Kwsr=-KI;qVB13Ue2w2m#twuwM@PFR=m#*(NtGsA47(Smn)Ya_;z;c
z)kk}OWL}=SrixMYx5Yw!jsst^&wnjfHk5j?`rAi=Z{{8utZiA_+v|=Ozbn6;B`**j
zDYz?Nbw3x!1C!PNRQ=EFZvA7UaCN57<(*0_L2)zZ`6}$lsJZ*?uSDS4Ot;(H4{o2c
zs><<X&#t{&+1@6b-(L~>*CzNd<Ch7gFJ3So|K#`8pro|geyO)#v_aL(%j_3FZ7%rN
zuf8VyZ{eY+Yv+Z(tqJ<sug()%Z<?6DW843dLz8}29g@rZ^mcZf&4RZJU-1Vg7Im2i
zCjaA5{OVV*J@3BGiOs5qrU!rYE89M!YirD7LG#m-KbUhD&3r2TsFLlmxzI}U;ElB#
z6^<`2UX;GKyJFjY-4ma6o|^|7HhtWE;LcBbCoxB>xBT2jm*V}8HOTXyn|YzqSz7I}
z-Mi<{n^dazCB5#xTIu|G`>*3CE8oj!D87?ecks{A>+GvT|FM63Wu>{xXnWUl_TZQ4
zyA<Y~50b2_Eq1<a{B7f+>@@{i=j9{jL<ENRlz#0nytpVo@8d2;oi^6`;}7>O%g*V^
zh$#N>Nuu7{pjvxM^g6?ei*oYp|1~^*+Wa_o<mQXA!Zn4G>WNZ6>vZ4$`lXxJXxTO~
zdY@Q!JJ(FLqWkST4z9kJKVNr)oWsVFu=#U$9Or4?5phmy@|9A<O>u|IB%{ur;L-`_
zJ}~K6$LiBpEA#cfq{&Zn6t3+m`>@GR;t9`oR@oh!+9#U%Z~VA?QCz(2r>Mfu{qH89
zRdD%zTItcZ-P6zd*TuWNd~)!+`?hHteYTsLh`u-+B6C>9{>;&UqmRq%tX4lgK4aaS
z#W$=A*e~?EmGJK<V!2+GV|jAxw&jI=ImdEm+zvT@nalj-jMn2fqV7Bvo8Zi7^F8g-
z*0@FXy~iVu&EDg}R<Sag=kndKVuwe<FO7T-r!CQE+<bk%OKR8(IS;Y<C2@t?K3gL$
zhH)r6RowBolx-89U^wyd<99oQpT*Doz3GMZjOu-!kCKlb_7z%IX4YCGld>jN?fI>f
ziQN{m&zjSJ9B#6E{^CrZY{cy!o7_H5GWsHVlXvsd>6Hg{-T$9asF}0l&kiNES<#UO
z-vVamcfGq@7<TZ~?AJ*}lY%DX_$U8(yqx!#u=%v>MXa~yOc#?{Y;N<adRuhcYsYRk
zFJ+k*soVQ^XYVz4sq;}k9VNO+YuA*v?dz`>|Ixndr)||@_F~bsuRQ*{t{oSCUHNU-
zo|MW6+rEFnYkJg|%oTGi%`=`b&$YXM%KPOVcbn(FI5naA%*J0|+S~3tPf`AHC9fzy
zW#vc5zaPQ`^Ih$Eeh3}N`L(BX&W|fcTsQxcHlLBaV@KBA-K*b~d^ut<KQMOt%NUcK
zoomA%o#Hl<c(%Jns_fj9yB+(Fr=4=<zps?6ZT05W>+9OwCj5UR4@v1e{>&@h<q&oM
z+{<R$bJrB!Iv+jX{C4_g+Yhgrc9?{3J2Z2)<nG!BZ}%i#H!qR2y)tjcx32+*=C8kf
zcy7)wssA;1=dCq<T&XNuzd27f`^_<v%<$*^O3in?Uw!X7IgiVBPVDQBxBZ7d<%=g-
z?at$#_~`S)DR#3J_7)a)<hSm8FV^HbzfHI>gHLq2o#@Brc`ACfZol0#SBV$b++Fzn
zpxq94F=OM31vlntZ#($y=xJWfCC)ilD?Wz&-xIrU{VKo3y$802zhB#QS!JQZ$-DmP
zyKP;BMDD6@>6Kh}c)L&XJjqU56QlJm<;Q+(ov~3gw%2lJ*k5sTy_g4jl6&`;a9jz0
zF?D;tL#^!lfQrb{O~22De%3t_UbDi=^0Anl@4;o<i`QPhoogo7?I$L>#r2ryk{6zR
zCl2$iOuV`8O!fTLCb4sa=UvOsbTRwT$M2T1diBXri@q!8OIO{K)<1Ue@#l+Kp9&8g
zu(|B$ef&HdTfSRP>h-@?DeFE&-)l;>Hry?9$!K59|8HB`=JPoA+dazN_A4`f-Fxop
zH$Q(b5?uQ=Uq5;4ruXV<Yp-kHJY+ZV{YINt4DFj%vwExfTHcGEvi4fGM)l9E8%OWH
zRl2s~y%2YSPW5^#vCaHHkG(IGDb&7ma@o#TCu{!xp3s^2;r-6Kip4oKT;bwt{ySSO
z4f#G>SbWixg)O2t#8mZqmq~kXskK-Vwel#}H~wtr!=JByDM|Zvom=<NqZ`NDE0*VP
z5%Hf~!|RYzU|m=7V(09dJ=+#;J3dc8Nj7lDpNamv{NG*_{63lYdEzHG^EP#vU)y`8
zr<rT>vFiIim%jPmaL?4KyLykTWIKH2x6QvDN37RgVe0*K`&PmwP93rI)6>FO?uBHP
zM)=6DKl)=L+t1s~=i=ULrVGyARebWtP3hV8Nmu%%uX^3RmeBF~i-V<Y<$)eo@s4e+
zPb;U~H+fOFs^X~Xsl+oq^XrbXuXwa|@qD|saz4fPPI^yfcEp7IsXla&&HA2N{puf&
zQoc;xvspCu{Zz5W%r7T3Dpj)QuW!$H71x%ws{iiY_%otp<EFQ-qfgIMmEZ@7wshGQ
z&*g9Kea#@BDB@hV>B)ZnYhCNKmhF1BOxi<iownYc?rkgI6g`%eIwzdFrKBojsaT%T
z$GNT7ojXjmw>td0{pDj+PGjf3&eJb`9W3QH|6viR5?e2x7jV(>Z&t?IQ%7gXK8m+D
zE9=#{$+Tvz^z1c$&;0X!l;y;Fm&%E)t&b7)&Yo1l^f+{-dH1$k{m-htZpnE4qKjQL
z==6g%{wFf3!()#-Uw5rL?qTC{Z*vfTw~X9`FNzo6FAlwJJV~x=-S5fwesJ5g7;Opa
zF5wcY+k1AJUtU<OuX)=O1IH7>OJhZJ!;Y12SZ@9Hb#TIujZ43a&oy~_B)GrWxcApp
zmM5(b+^_2P@BH2vQ~Z2MWv^nc@1J}9bM_v5Fy)Hx)uiq2UuuJHO-&V%@3z(aY046R
z|CG{=$5+<=^v(-ss#CPuvD0nqqSalGe;&M7?`V=6eDmqUsYiMK_-Q0A6<g?-eEF{A
zg6~_tf4A^>kpH%KS=;JY_wU(kiJovo@&BTE_m6LR;~X!i{p`U`h0peu|M*<RJG5qJ
zywuElll*4+#uq|wT&C?6y=dFIWa}dD4c`8KEyg?aWLd7~{_H#Y-qC)U4KqXDbbkK2
zjn7wmYNV{5eP+>T!+W<cY}H=%=ZMMk_618<z1zORh=0XnHSZJoAGloY_TT-dv$c!I
zMoLGuqtfo=#ORe*MfP%E_$qMO&c37U^V#w@(pGo9)+&6RD}3WHt5;Z-t66H&kM`18
z61P`3ywtS&?%<%;@aUt@j4iSEnO!ry_ek7oW@r|<?{s|Wv1_|7nsDjexKVI=<3zLU
zKSkd~cE7uNbro-7Nn7{!y_xoAoc}iq{1nz&w)We%w+Y8Jqhymc*KE0V_gm4?zWDBS
zSM}4b+xrzhU$f5gR*OZqq7&bOXWj3AeGbU5ixu~?lr7(;T6g>3(#qb7Lq9jo7Im%r
zc7Nu7Gx6;1dvn(Z#MQ~WUAn?0+T-taqF+4l!pG+QAC!MwPYe(G&7CXnvHV(-{mn;l
z#ctvC=Ci(~%Oq}yaIL*<!V)&)+^?r5hdP$eHcrm{-nAnA^g~tVbrpBC)AW0--hJ2m
z_vCJ!-SfrsXTM8b$MN<1`Qrtf+7#E_(2F{k^>5)}=luZ!?}T)gSU#|i4^Ce9|Ff0<
z-|kPlDkScI%$mn+|2c{~y6@<g=<bg%df%(CH>szT7N}0VdF!z;UmpL8DarS}W(BzV
z91iN+aQVQ;yY=_eKiFIEa=OF6Ve8q||M#tb($4j_@pDW3DrG5&Wa)~+o<^>@zv69|
zFFrBzSJAXvPd*xJtMcXUn7?S<-%oF^i`_bZqi*T9*DoZCe^+jcUwZxS?3*$9C%#UL
z$v=5iu{i(C*K0BPXXiRUySLyy@2|v!!zXI;FMMaK%|Fk-`14-J`_f<b?ls^s_`>F#
zZ}8>YAp?j1&5l|nQ<~>jEnc`?vdyvWUdEf=WB1Zz*d*J(zV1C{cfx|lh_880_1Ax2
zt=j${I+Qu1<BIyD7s7>KIxX5dEq=Mk)GGBcPH6`-_a0oleR})Vb9)cokv_9`ug#jW
zcf8N;CA@q4G_hf)dBr!j^Y`vq@_cy1J8$p7&D+cQm~Gnc-jcSdu1=Lmc*#5MUc$?_
zQxhAOnnzSMFWnyA#yCyTk*6a=-r$H`&&JsKJ^Y0~Y!f>FN^tXZB-yv$cRp>LCpAlM
z<K>QdYJW_WRtP%xYGfS>WS&&SJgunCrTg&$c9*AYK2O;~p1Q?6WlMR=ma}wQox!q8
zDmwKqGY*_Bu-`GEC0L>^_uJ>?ANYIN|J_nP#}Zx`mQ?$1dHtW4bse@by&;S`o1e+L
zJD=S)|JTbpt+wEhyVhT})m{A@cUSvf-SvsqWebgB9BX^O6&Fwa)%oh?`2#n5=f9HO
z{qQZz%F6l=t;x%?xR+~HTb3OS?^>dfZ5#I8&UlF}=Ow$Pd{_Q1Z7Qu_-C(<K=gYJG
zPqixg<sW~UwrqQ{-i+n_&%c;`J@z)((Y*Bd&zHHcdb^7cuC#tP<KD7+JBsxzN{{<b
ze!TqNkIJGyS7hyd`{q|Yetz;}aQPn-%eq&-Gt||6?s8s!(z4P#ao!w*Qgxrn!prWp
zR2Ch&GS|Y_Z%&o)^OK#y<&R7(3t#!pQCIi7+j;rPk(K6|^X8b8s{2l!yzJhR%A!kG
z=9>8W&8nLG?4)OMd0p9V^)!Ld@0;J&^U3_Zv*FAUp5uPM&Lw_+wcNUVQdM8fe)Y5!
z!s;oZ(`Vb?RXp`eJwJC5gZjPtowKw#Hvc+h{O-u&`E~!E&#YAyaLE4mW!I;!HWBT|
z#|8iMD5>~1OMjTLe!`V|bq2K(_E{TGEUvA)Ul;u#pS525NA$t}v$(X@t=z=ypV?UV
z>!0u^roVlE(_h*5D;B@ofBxTQo?af_pu7b6HnY1ImUSmLE{rta^17|;uBDj7tZ&Yj
z>tx<oMA@6XZD02H!X|n5Hz0b8d=%f~1&1$`NX+7MKAk87rY|RESoSpYIbSZ!u;SA_
zUU;D}=JCRp7F%SFJ04&7(!xZxecEM<7bYfi+m>CvP$b0;rc0!v{2nj3eBq0sNiC1A
z{AI8BvO|U?PbGF0@z~z%Tc~~f!qns!rwqTOoOTZXbYZGZiE8`C0~bo<W}S0BZQ0M-
zw(RqTFP2+s_;mB1U-)8aQp=}1|M7zJ3ttxBdFXulxy1`3li0Rpr!N#qvV-Xo$tb<Y
z3r=^$s(mq1+EOcH_LT4K<&LCp7W<a0ywH*QP4MiS=Rb`2?)Q3}^YzF(RHQuX_MMIw
zcUoSk`Is--b+-4!IS{&W?q{ZwTg!70>UsXhL{RxsF$i5NuA9{%_;IJ;$DIZXe`hMW
zJ$Vj7Z=N4A5maU>0U@@M!dV@HoVzS!t&-&*nFuN?m4J{MSXgqGpyVz=>0KtiM>+)V
zf{<YBEVU$;t~{5nydz!jW~e2(^yIm;<Q-|rb85{y(wgVemUpBr&#67{NPC`3N8XW+
zJg3gQBb|9JU3&g52TTN&<4Tm1N|e(|gtJPN^GcM9N|eh=gsV!F>q?ZH{=|Fm|62=5
zCEG&+1g~ly^!+dS`~2G%`_KQo%*%Q=ps3h6(tJv(bE3V-y*~lB_*?%5+~9BBZZL(<
z`JsUb-|>h94cYcXg)$#4o>)xjbAD(d0*cH4E8f=20Y4<zYk8LTFZ^IAVt35(`N9%8
zuICF2<k;u7UwR(!Lxu|!PlXHH52b3v9CMt$utbt;I;YREz{w7I^W8jdx(D1mK5;ka
z6}O0D;WfPn)HZEB_?%@;GOzN1yWiTR9WV46yf|dCMv&_+M_XQtv-q;a-U%<<0!oy1
zc3JRdx44-%EtY<GvG>D{oq|etIVR<`c$WLz{xRd*$B4rm)Ajs?4V{)rJ6`VH@bZ(1
zpmNDBL7TfA^WV);OKJ%)Z(1Rpcx7s)lAA_ps*{JZoXo<K&&o4?3HtoySa#3XB9|rD
zzG<cW!7Kd%znVFY`}t%P9_m&O$*W`Wwr^T0zwlB!=LPo-Uz8($32N2OQA>geFV9qR
z0|~zi<z4*cUvc-}?-%Rm{ar52m6xaSs%=r8#_P62v*l;Ya{XSo!TuPN^GgE}a3J41
z6kx^E`Z&PCPg&0E_`(7iuH}UnGBxJ59m>?0bIeiR`DL<3P0wUozF!hvpz!5-UU;EW
zV{ZE)P$09{@-00N4rG}UNw4V(3naOw7hc%#@wJ07(<wWrQ*}PAZXVK(XMGo#9g<3R
z|L*V1`=9&e!Tq9tK*{3&Jg$Fp)uz^S|JVJq{X=-`|GXFbTmOswi~f=S5X_jb@yC1C
zZw`Gc|2g;Otm|8`M9THZrAe2PGCY^3es=fP_FBCvcj?o+pIWcEYzqrst+zdF_5T%@
zqE@d>u{^V!$BgUQwZK~58E?)`VZ6RN@}Ab|&bN6|pS@OJ(k_0#%8%nzjO*lce=e=E
zZp!-iGn3E#s_wyEg=w6x%4Zz>Ar`tl{=ZE1-QWMD)@N_rmHqA7R6Do%cLIE+pPaXu
zEq(DGyX5z0GF!JhR?UxEC4Fp0jenl=o!bnt@(C|Gb4xV8AF<o<`NhiIjM50*(zw0T
z?y!ZW{?y&tzb1<*s!0CU`9G%o8|=2o|FA7zx@}^s;zj<=23IbyiGQ27onMcCWo^N^
z?^o=0JbtmVHmC4YP<CyH^YvdhV%95t+4%R3(C0kv+#J!e=vNFg?7Oc_y8U3)uLVo_
z3Vv=Dk1AsKzHam4(1wziFHY809G_sN6Z_G&QX(#M<wpMe`k9mCau%Pq&22cjc=@-5
ze;bm#dznA*gnpTNI^AE^EJk*dD!=#lPhwjX=Y4j`&_B+&wuJrgmE&Q&<xxfMTJ{1r
zg4(yP{;*1R-YtgnpN#e&G3j~FuD|t3V$Htge2-UeNIF?tle(f(CSJbm)nn_bSB5?M
zrQs(J`(DmjdH6@n=cP8kJ~;WmIjrq`&1{!Z%G>7Ei#J?h)Jyi4J@&16O;nNeZU04<
z#m{P+9!|T{CiZ#tPlIQS%Xcj7GuyUne^i$Io%7es`Y+y@{x01^_v?(RJClA_erLX<
zyOsZa`s$hgGxZL$L_f0Z(fj_@=Jty$d7bwg@~YQ=e<S^#y$Kvn0(T}^``z4~{pP0o
z_vVFfJgjd&s4H{vmYJ3(|EX+)O?8$e|K~XOLgqgrbGN15DRnNAk9{$3m+Cpu=uZuX
z?OJUYe?EI?JwdB*#Y36vi=XB$Uab;$?a$YPM>jA3raQgHUzRQW%;p>ATC5AwRvx~g
z74X&hn%dh8zY}%x{mnK@zF&Ns{H;;MdiL*4?^90QJ0W8qq?zZD$Ly2p@s?R8*jloT
zXXWL6uTSPSB%ZP1DUjcNk*(sH;@1B6GQSNS(r>5KEeMrpbl&-QLrbEE+_eu9zwUC1
zO+T+#d7~lqMTL4)mV4DFgL6NO{ukaqDfZ*g$_4V~3+0RWAD^+URESw+7m~7GUSqmZ
zh|wfH=VxZ89?Jx|cUkejl<k~m_fYxmq5^dv`G(RHwsH-1Cu}cVarc)yF+F3Mt#vF%
zugz)aBKeei?N1szJe@ynn3i$*(&am%-f}NzY}Buvv#X@zWbFd?jC=q01sg@o-F)zj
z$DA);?ELO7Iq>A8gH8LFea6u}jP^ptio7o~*WC_ib?Q98vxikHgVF7Q`g*CTFsIIK
zJC|1J%qU%++Pk;u%BE%WDz>ay880+XS5WTfpPNndt~|aW)UA4LZ_Y%~Hp8iP3;SO;
zuUfIvHSWbs|Lc?66PM1rb$qJT>|1T78`jl^6uw`%^Si>AD|Q>a__sbWGjZ8+Y1=#J
zZrh8#n^Pl}JY5rgIOz6<3zLM}uNKul-<Ejw^6sZ^nO8SloAmocQC;K_F8iJ}Z)|t=
z^1PnF!xepWU#Zd;)8fC=P8qDbIpNOroE`s7dlq-w?qptk-nLX?-lIrG%YJVk8-ZUB
z_AIhj+{yngr}FWJoPEo;9dx}e{QHK$VXu|B8L2C__OB`G)7_~bV|!b7YyTNjFY9lq
z#m=ub<b2Htx0q<b@Z@69<qn5)^X`cAeKK6i$6d}-`lXdS_hqnr?6XhGTlwF$9}&Mi
z``3+E`+}xSz9RniP3Peot3I5V^!vuD%;`b>i8s&qXRf!a<~7}NPma0hP0hWF?ALDY
zNsidI@fP2Nxvds$p{qY<RNlUzbus;6bRVNq^XrqjIr}F2R^G__Iw9bmjn3_#n*t0f
zC1TWBijJ%POTCy_Bp)Lg^n2CZh0FPxSHHMD>Gy+Ibz864?eO~R5j6Am4K2H1g;~W6
zCzCz$lDX&FS+aLb5tgvnowz;n(S{t+<cLoVYTo{`A3W|Jn>OdT=H!4cG7g)h>LZJ!
zqvjg9e%7kZ5d9dL6~DF0AUaDv>P5!hEgKb^dY`^+=9c|E&1~C+@2M}HPa6fb9jUO<
z(=C;mnKMJtC@iZrgWGf)e|MPV)M@vQhdox?_9XG{DZ3@#WuG6OVAGR7!R~GC`NXL@
z<?>3;_65)5^?agOX~R*r^UK63SyPYCvk6jv_SyZ<?SPLnY;+ze*h`(5yz7xzQ|<!)
zLki)qM7$)H2s8IKNpy&9SR1T4HS*O-=D($UTfYQaSt<*y?K=|X{cgqOxz3*&;<Ywj
zzI6DGu(zDwm#TMLzwDcpxTf|-cb$*?G0Sfo`-=lIN@gA9UCDegXl3iPJ8d)1DQ@lG
z6m@>v6SEV^YmzMf9<#b7aNhB#VSw(`mfVxM4O2Jet$dz(Q@2*);e=X+Scj&qU-;J7
zz4-31?*UKN%UaEiho;#Gr6;eh)%>3LgIQ4cm*bO11xY;0QZHtlI{y6&V@kUo-xQt&
zr&M<Qy}{x9s^VV%)GdX_>Q3xvoW}V(;N$@T-pNK2j2X`8C2YIn{Ak0hJfHW=j~h%&
zwvl{1;hy;BInJL9JnTd2&e+6zzgyhGbdKd}?v&#*tYnnyJl<JsOG@cxx0_^Em#-vp
zJ#+Srev9euvrJCkOl{v%sUTNfB%c<0>dMix+iFiuxWyj1-P8M>&x09OJVk33^L~=K
zEv+%fM8#h_Q}C?2)N&(h-Mybh{>qdyPZd5q;f`Cx#FAG=&!<&dXtO)1&vky6*zwGE
zi*v!%ZxM2no|nFPDf?5Qdb&;di$3S7_!IL#etFWUaU$7t?la5dPwXzl#=W*)a(=CL
z+`re>N6yzC`aIztn^gPf$^|twlOBs`9%;Mg<^4_BFqpsQ@z?6@uPgIc7S5VH&8Ecq
z>y?GqzwX)o#Ky1OTek4z&B^UsHFZl>lK<SB+@5^!>lML?5iDj&ugh5GJAW$p!Mjp6
zny+*1ODXN#H?17EZAuSq`gZZ_Ox?o|ZtZFE&N-Qz<Tx$rm0`-e)z{U<Tvjff=lpD6
zRc(^v<XIfc`SWI(-RJsda4A&juJL!RsOL`aTk>OX?>c$HZ~a28V@LGYJ9~!d@A)3I
zH=x6rRp-k6`SlYo%0Ch~wxI5oNO5Cm)l|EtRmXJ8LT8Cc>MLEkx?$UKfBOkjvICY!
z$nIYCk%@c3e7TFqBn@kWr<`bJ&q{pFSZ3bZYphf0`)NXTWzL0PP5e{OeAaJLI{g1<
z?Aj|GzkF`-_3e0ZuyDa)4zc$ZQ~fSWJbczE&MR1N=^Gho*u17=qLP{0y>AvVwT7FQ
zoZ7pg+A-uXU$)$V2$sr^pFKQQyX)9&ZL$nhJ8@2kc}uv&jXkTb8C6#oDgB&&*XoMQ
za)*nWqLGWYHSA<Pcgn?J<_mS<w^v`hWldb@keec;Jo9<<vzOB=1ybe-O9n~un`uts
z;7JMo;NjMP)!k4)$%XUHI?nyI+qhmVXZO|$cs6lLg$lcX=SO$Ru#-<-+2?H$dLp>!
z@Q+Pz^=qydM84&@_2+n7RqdrSK4KZ7)l0>LHmpp!T*k%1Z#Tiscw2LD<3xe0Ia?~b
zKl(pg@zsc_>T%|F5tdbQOjk{AJ(;1*B;XWUm!%@|>G=Q9fG;msuGq$vzS?5j?CgpT
zh6TK<^gbtk{gW;m-V@0>{jNx8*uxZ2_b|p3QHwCaWKpd!!z6Ro_Rx@s*5*)|h@<tP
z55iq*LRVb(HH;Iz6B-zMb*D<tkt~rNp@Px%wwIirvuLkwv5AYkeb6tpOl!0M%-U0L
zg0G3BUYi{0pJqDqYD!q~rdgA(&Y86+IKAxx_iD??vnR9s!iu@RzQ5o5<$-4B;im^4
z7JX2$ZBR1qTh_B!^Kr(h8*Y;7!EO)FSo0furB6AhQM2en=OJ~j`5HBg3R0t#Y>Jeb
zHoDHxa%G9{{&1qVW4o9j*CH_ujo5%)3kn<`HXUkZU*&gY?(rAroJ-X^H+>GdTe;?)
zO?bK78f*V+e?uIvCA$~z@BPJ6DEo3oH~Z?svd@eCGUhqjWXN<fJuf}doseWYZ*$<%
zz=>*GD>g`-&$3y~xkhcntb`>@uQ}iNC73o|WeF3G@J%ocoGcvapHjZagD*^Y^4x`w
zZnS^?H2-+P*N%Q3?nd_AMH{r+U&#9!EW9G<BYW_Qpr0(`Rly)x$E&Hn1`DsI3I?rd
z`|yfmL!ZEFj!o~bun1?#9=s-)b60cI)T;tTvX0k9eGL{~7p##zcwMkgmhpyQldR(n
zQAhcOH-x)v8*d1AAc%$O$6udhUtOr=8QCi=c_xeFh=Sb5st*UVIF7_!nWPcTv{79k
z|6cc#v~rFk^BmrA!U&#B`M8_1AlYg-kv}is4d+p0Rc{unJ-2U-x-emhwx>r>iU*_O
z(?*g1fp@}hb~jA_vFL(gK$!bQnbg85dnWE!`b#q)&h4WFr>56_jW>&Ox+krvPx6s0
zO}jGl;#A?s78fUbtqgqUY{`F?i}|R<lv@*isHysK@;ucQSs#+%_OdC7^(a@DsL1Lq
zcF9f57yfq|{;iL2yt_St@8sWxgN)i-ej?MuuJA^49*8(;U%;RraCc+T^NmHF3VRg2
zI;V!RZ<k)E#Qw3D;qAst<r{;_6NHnOzBy1bp`X!TSbHb$B-^koy|5^~M_%jxAC&Cc
zy7_C7S=_GhnNc71J}^$WE??N6r7EMm&b+NrFwd=f>gJwM9|k_vu#+xpK6OOwd^*49
zYX_4cs}z??h$^@83-t;93+5HnN3nclxy898Se04%h1vw)|GmL3U*&$J|6~2W_oqjk
zO#Q)d_a!VfekZizWPTnzxZCBNZ0m_L%C-u&is>$1y}5rrpYRNt+OYc8tN1H3C)5S#
zI9NKksQ;*UQtwMXTKV^U{^9+if67Aw_yzCmlufXU{XcE`yD5j6!~X^Bs0z4L8nD#=
z@I{BkzBWek>hi6ub9K(j$gerJQA+$Q*V=8|g2JiCgoJ{%bTl^{*(dbbYn`^`I+pFZ
zC6eW>RkNHJ?Nf7P`tLJ_rz?5)GW_`b<6s2mI`vTYR{qcx&tCPd-odMBxru=*{1!96
znCFSumGw3i`!f@#F_->lzS8=9fy2hm2ZuvKc9~k9oT0sR@`@kco4$C3yj5NGX%gqU
zulDv8->xoRcD8Vy+(r9T-%3^2S3)LUZ%(a$QuzN6@2Ble(5`X4_7D9v3*?qdK9{t3
z#PmA0oq3Jb-D6&C;sIN)_UjzFzj2=4)otr<=>)V#v{tJ7-V*TJSGBfsM?Gtm`GF)~
zy#T&hr<?6gPSm=h)v-dv)wIBBrOsp4Q;f$}bY>lwO$ucfKJM}R(5vNTkA5t1P1$O_
z;@Gp&jz1x1BUZ_Z<Vf*z2(PX_!v8Sv(dsL9k7jMXwb_SVdB&DQ)7s0lu68WlwfnCC
zL&2m~oRa5W8*Evb&nWbMe%txQm;Qy;ye;7^SouyWl>ha^tKnO>w(1G9u5rt?XzW|{
zsAE>#?cy_sqW{UXtcw<9J(R!a*4itL*RrQ2-fp*SRm|I4x!cQQwXW}6`I|;zk6te6
zQ4f~rbmgD%#z#<BMsC$FrmNwuWo`n&F?x}$vq}|L^VX;=d|Vh=9{cN-!jWAw&4Zji
z@GtJ0tTdI&eO}Z5`AZv)+%KH8ibLj{QRrXA{jyRzRd-66%TEalcpetHRs3<Djh_Ek
z=01U0a~sduH;DQwFsL~vaGJkg;3ZSDg*!La;4@3y8GqT1>4*E4H=RBBVecvL_%F_`
z8mza3@(re1F(_``+dO&i<-6=^_8aCuVG5D|&b;ZR)&7~>M=cpL65VWTx4m8x*x=fj
zdvV@{+DBJSjW;djxZs<@Jfq`w>w{cxckUy-1?s-j>fC0VyfqDPu-&R`&T#*?!(o?S
ztkcBjmS`pSdhgSG+uHx|pivW3)W=04+XUv`*LqP@s<5P>dEJu-%j_;Tb|rGD7;Zh)
ze0y`=7p^<4`x57^ZeO)}FB@;`13}>ouAiURCvB{Hy^FcE_aEC$dDdCxrvkV8)h*Oq
zyu2~^?faD-Cq+3!w`>abk1v0GeudwiEp?yRDi|a}EK3)@K6Rz3_1K|+yA?)RERqtt
z-<vFHn;&<#)LwVKi!VnW_u){xEdJFGZyxjx5&rEY@<QX|$%1=ryTm10XNmFrWLM&n
zoqW!TS61=N<vn%m;=5d{x>J9JFqG|BTVlZCvDP?G;n&F$i$!*JuO(hRZU}cgDq9+z
zXmOyqw5Ih~1#9{Btm<jvy!SFsGv1#ycS79!nK5fm@9DdjX`X-Mn88w2Kb3RGzfN{2
zEqKLIa-3_60aMbUoQqSo&w9^dE4x>|LQB;t_rSK5f9+SKerjYZH}PysnfSrEG&FY6
zs;_G#ZiL@kBd|dBN$wq^QrlPiH^2CGdBq|*-2&~m7QB-ZcG$gkx$tL&8N=(}Jab)n
zCkZXo{`=B0WSPZTEhgOy9*-Av2Iz|YX+2)kYUl8*&;8NNnim}RIs+<>=i9OsT8Ped
zmdkEZwinv*>U|a0)lZGrR+b7C?$5gYJ5$`^^4zaaCWU%D{5Ye~;tKo1#kWKyZ?whk
zXg<z#eU4bu*FDyoW2BU?B`szwZ|m5frOjc{`nn@C^m%{6tu&d}SC8+Rm0=;}-8@av
zM&P#l0ndkmVJoaxPZInQczto`W!Ckn@5~xB&WYrH<C-1ug~wG)=K6y}4^};y$R7Nx
zvzF!TJionj!#0*?YRx;Ppl0x~x@q4nLmvJ=J;I%O{(Ey69v%D`KL6AFvXBQ?ypO#6
z&lW7^Qnj!;omHG`sgR<B$sIrTE4=KiYtFy2;LD1&J<;mI#n6((;B+@}eZXz^EB~#g
zoTv-=b*Z!C<E`p>%TsNCDEl7c{-*TDGkDoj-@SjOZmeG+&nY6uoW@<SA@i=|{O3Mg
zS1(LCEcL*IQDs;2iqjVZ4u(n`@%2~#cFD6;K88O+Cs9+YNOR8quBkQ&DXU+Hvu3Sp
z`fxsqLDhK2l*5H#6{`}=zB}0PxUwBwc`33wtthRA;bx-b*2f8faq-5ns;k#Xy9>uy
zB}Sx+J0zz)<aLt_-8z$ViITR(FP-<Y*1!H}rf5_@cj#cg^X^W%0*k|s&MhsUTRWCT
zY^i7!Nq!>9S>(&y@kRWDiSq3e^&yXRjo3Qey<1N?&e5~k&mZ-uN#nz_{nHI^bV+dC
z6g&R6V3MiDR^1I>CLW*mc+T|=)||$5m(pLPuQK*4UYBL*A?6~tZeBLeE0(|B79#c6
zj4!G%^U0OQ?oHXwcT#%8)pN;*HZJHf&1!9}IKVFZ+SC7%)Ca*81)&n(TmN;PC{?Nw
z_c*o7ENa%|b=sdI3uBDixDUv!tzG@uk)`s2<YK+5w8!c_hc+&dWSDQjZ8-Vlqlb!;
zrI$ALly+XxzV4Sk_Y6B{e!$DpwSM!~TKK-=oOp0!!z`Y%=$gCL1(r`1K6oWG_s%Ev
zi5%VY^*?HC2vHPUez5W`e`zIu-bLo;dVjbdzZO{9>-<$gJ>&<2c>7_KJrWaEO2shn
zJ>Z{FxM<!JFMhqZj#1J->Xy%C5bs~(rEu}R-(#H<y%Q^(kDq95w!FL7!g`y+mZ1JA
zUZ%oN91~d{>D|(5IAPNA{((9tv(%-B9nMcncuoDxl6r6D`)}H|?%%<y=ll5Qs21!N
z-F4(h0-y1@{2x43yY9UH%EBI2Ewy83joMX@cdE_R2X}fs?wWs4B|uB^_rkc?D1lXL
z0%{fiMD`pxcgOg~W9wdjh9V2&xfa*DKBnYM+40K3;J{x-v7-^S{)va(9=0#8=JdRx
zC6xHb+T}x$ew!=PQ?)q0^mmRrqMrJKOv2|&+HG6@zEds#ar2S7?%#U9>AKdJ<PH=#
zMr^iHeS4wuiY)7grY<3dC*3>#PqsJmjM1OKXev~;FkM~!Ec1<pSyCr6b1d=-e1x6Z
zbtIRvcU+oMXnM?Z`+;5D3D4cmyFB3D74hK9ir`rP4u$E9tb|=%uIYVw_k%^tYNPP$
zN{wCS!aonaoO9>NN#;lFAB;EZ=^T;?Rh{|Fnj=f%MSf+I{HY~7g<IuUW~_F7d&H`i
zlig4t)lBUn%ZH{4A1O9Iw|Pf+vN>0zEfHTQw8VPHy0{%DFS1Bql`Wg`ZVq3Qq<NXJ
zjiZs%I@WYSjYOG9Q6oMH$!kASH^{9G5OVPh(kS~>VzqP2yfw=XUt4J$(ZTWhvBJ%X
zzkjN1D-`Eik@8P*(^Qwaj-MR)t4_^XaEpC{>Ls3rz0Ug_=EY<+H1WL=DRZn5yeRH_
z)~_aBR_@iAoVPq&LVH~fzBlB^5PF%<Z=+z9GCg3@6oJOa4+Qx-8m87S-c}xwG3mh6
zg6B_eMqXbP#dv`?dU?5x%f5LWiG`C&?aNm^Sv5__CCTx`!Bd$BB6+*KKP65)w#vHm
zK+o3|KREbLZIC%4p|DZN)p*ujtKP3G)?4TX=|=Ml3YS!T+HvBQv|!6#X_wimZ$DY>
zoXF=?x=HcGlqF_T(wjEFpZ6v$<?qWDM+2MB8hcOGgsfEUNzpp$?A8$4#Z+=KY0;7_
zGp19MBr=!FXnw2=%9vhxt01dU#B=H)H3>nMf*T?_<}6MM$6wcJ8(dx)F#o_DnMJzB
zKP@=?pISzrXL)ekW7dj1dxvkHoGeM{UKKNs9^;l{sJHJ`+qKL|Y|e|bA<uJ)olGV&
zcRtcnU_BOhULlU}!h){}&Ih<T+7ym>YVf@b6MGY67xInqzVy$v)kixvab33-y2t%O
z;z-RwLpjeQf}2xh?n`@G_5WC8DX@5M?nGNY{tq@ujFlbLa`*dY@MQ-#zG1o6S~Bm<
z_V$M7EOSH7&f+%xP<qwI=}%}%Y{fS*Hx11rBCRYM+fH#Zd|trfXE0wfoz=j+*Yfo<
z%LBfu>wX?vEN=W_B}WE($!0%=>oS`9H-t6cPu`eOGUc#Cl*3vJAJ20$6g^wqy4Dpv
z6>UGT^j0(T+AmVdr6P6;rK?u9Y&oDii!<-b*)K=D79CF%^v!#%A~~nhPh{T1uI(Ep
zPMu_9b*XgI`|M+$t0q*m=xtlKfjj-VlA<H8i>qYg5^oDx%_Uw>Hrr<A^^2~aP!RM>
zu;I}O)&!<G6Q)1d#rIU@?_AB_>J_PPvvbZl-{Np*VrlaAbQW52z(Ms<&fF7vCu4+N
z?p>X<-9s-t(=^;gHJIt(1+(gDPqXGOIg?vADcSO5*L;K1`YM+dG=yjSNlHBau9|!9
zjQWY=TBqA9EFTB7o;x|QZbf_MA{kxg>y25*3jKnP7pk+gDBV0@#Wrt~_4H;I??nsd
z2}LRSCCvA#uec=mRpPK3i=}Ad>fo)B3sg2-mSxS|EukDx_4VD!$<yu$?4G#$>ADjs
z@A)R&X<s4do>a2<RrB#F)3bcCmUpFA+OTf4Rl9mrthh_w<)B61heL`Szk-FbSbw<l
zyiQraU2TVzO0LDh>0x(0>pVA{NZ6=iBqV5SeoOO6tESqzF4yR0>%9)Ar*(c~ns9^d
zB)5_T!^(_bO8(+=N~i2`ns!K5zH*(?3;qSIi<?xIZu7mxFJUft`PYRjGRtQiioLW-
z_S}^zP2PWE`*$cTvb}lGyZZYrzrC*O^d$MeubIqnM!7XHG%aJ(!O8pXUpTbmf`r$x
zuLm-{??ib71vaSJExy$`arg3dA`3hotDadihtcu5<*Ac%PJZNTDi`xuFnbZF;;(|n
zFScdbRx8^W?C1GCQYuuMxQe5HGRMLB{F`Q!s9S7({G@@UZ>yY-<m$J%n_WDzByzUS
zGySl2p~4)&Dsv%`zDvKfE^@{>=>*;os;TIDA=59)x!UYS2J5T1J?1u7ISiI8ZoHt)
z%cS$XRK!3mQDC;piaeDsnk%JiIX=W$ToswA<d8TeZHkZJM-%>uJe*O>Vl<M=DrPyq
zn`aO{Q%dE4$i;*Q5``^cCJL2L6qBYFzTjIU$EPg&*`fbRQkB$;%3ZD>o-AH6ed6V$
z2lJ+1;dT^r;F_>Ybozs(1>cIkHs1EX>(OY@Vj$93P+0L)i&fAgB&2?VSK*W=7J-{g
z`ctOZeT)|HpS5T0JKH%Q8xz_6TupBs-Oa*L)~(LYvc2vdr)Pj;QSL|8KLIKp9e!N;
zTK5*LU$W%9x0!a(R1x|9>ie1R7wJy_>Gzm>!ufrEwW1O8Os=r7{@Kh^%5Jr(_^NT?
z^Ls5$R>lGW3RNp6i*RrKbVH%D%el?9yvJegLiSGA854geF3wQsa%$M9WV!K7YI{<1
zh?~f>^>>-dTIM`lvd`&V<FY<In{NI^0!Lh(-xRa2-@W!z@1%wEm;B^1VcxJ~!GWIb
zEDn4O2TyZ8u9OLLy}Yk@yIjf%hn2ERd{=Q87qB!|ed5jjz<i)mUL@esJ(fI<9g73s
z2zYLC_6yQv>kyvAqGrv=_}-!6YlxQU%sByP=Ef*)_%-FbrIrW3XBVeq>k0SikpUcz
zozbrEcEo&tQt4v4D>Ff|Q(CT;UC}&nI@3?~Ck;WXm&ab>ys)%zy1~iI4~2IaPmoeq
zk3Qj+aNg_tJQa71Ck~(d6nDL4zP>c0K`mo_$vaEA{vD}GFL~~`Mk$CeY9<yImd?^l
zX}HHWZMS_uc><H6;n$k-BssoBvpE7i^_ez1&vU(h<#}wjjG+I+2a5yRr<86g>Z`ou
zP|e^TF5xow?TVud{1Wy(;P}5sO>si2iriyX)~`P-A2t^rD%~Mhr?5iM>+uwcl_78V
zY^}q-FNir~a!Eq%i^Rd(FN1aja|^mGG+QcQD$aF>>%7bikHE^Ms%n`FXR6G0nJFo7
z$T3I!gi4NQ)`{RdQ*y3omRoto%N=P_n|V9)2;(UkZ|;su50(BNdRKBF$jZs4_n}a%
zvxJ~$H|O1oiGF8OPKK|OXj5Rjv1lXn*Wc|a;>Q|4+vr_Nug-kme`5jH?Ou0=twt9f
zd7YcFry}SBN7vgKiCoSzt-n6*D>^2-L~m-uJgx~0f9CK7%Z5H<{hfH<_fL5dFW<_2
z{&H7RbR1qM^qpHFe1kJ}t_)L(yp0wAoRUeQ#rm8!8~jDhBTrlmOl;m8B2|3h%Ld0g
z-jkFZ<f;v>voG`8RIuRELzYG+qxFX#@fF&fnBt)5ta#gLerBJRaBoWPy8<s31=F2{
z>TzO$oyUA0M6I^m%b@d0K+WT%UVO(I4)!-T<;^Uo8qXvI_jpxBwjaA3nKk3fB9FzF
zz6IDcrf^#DX@6Rf#CyB(*V)5QHrhz6Xcf47d}sEel})pn737!YPsurDwMp)$;S68y
z0<K-tG#+w3*kdL*kA3N}xY<*stRn<H|2-)^@>?hOj$>lc6Hc93_aDS>D(7-5G!Z)`
zYjpL^?ho5D-W=v`pA!<hvyMsRMr4!b=FrwTEg`~BX2w0qpK>HtWrKS9^Gt{5D<5?^
z)!5i9GKg-xGyl^DEnn8W=@Z3U`yDS?cPKRTt#^E`z4dIK)S`~;%d3+XPI&C8^lQZt
zt`-+AV?Rj~*H1Gi#T|GZ?!_X>Xql*XX1&sd7sq5yv3rKhIbzW<i_2C=Qr100Gx4$K
zO99ue4T;tTF%I7}mTnVaSybDo7OOb-$<GB(nfe$EjU!(aE_Z3t68tp1G>Nb1$<03h
z4uu70mt-ZGf9QSGBywlE$!v>L3fq_zl8hw1t~A^fUUiP+cYvnWoJ+1+Nt+(s$(_ft
zcyiP_g&3E&L1FP+*9<aU%e{iOH@i&XpTrk9Pg&LVTIRFK7MYISFO+V6W^-fxdZ8_e
zsekiR#%d2Y=Ra(^%bPpr`Dyy4c<Bmja&fV6TbfiD{8sZ|Rjlz?P^QSrSmc+|>5$U7
zYTB~X*00{YVw}EH_JYY)7xk?N)IYYXSBCEpeI0foRYdwoUX19QhKwz<B|MTIe7-PK
zFD>ep#*IByEjkBl8o9k#nq6E1J=mQ(LozNaYd21AQ&m5!kg>!grtZ?T$1GoKgIL(!
znOzV`aSusc<@!1@z(Yf1LI_uVOyn~Mua{>`ay*3I7F^w{SM1EXDSU<S#3nVC&wjy<
z?yLK1yzXilU773KeRyW4Nikcc^SPyMYE8E86`uF~u1cP?x|QIuoBeI77`yYblHyyB
zn&!ATPpF#n_{pLhrOT)2ZT!`7IFb3q|1Y)^=P7-g)*4{D-uS0dq+q}jm!_MJJk2|`
zMe1i(c^>LI5LYQROHAs?1obe51deo#iPkIgeUHtWo0c4ATCSeTP(AHqc<bd4m#(I7
zE4(ODs=eW3uR+L?u!vKW88|tX1h7;_{kfd=b4kmJ4=b3yO0Ouoy|TJxq59IKLi45G
zslPH#YC5uh+`ssXmX{ifp~ccN$vl=L6XKrD-tzR8<n^Xg2BFGa$GFWW{jzoWZn$Vk
zNxz}lN@M=`m-ik9Oq}gv8xk8cS9Wrri=buF9;K~Z3u<MQb0=}yI?kE*WZ5f^Or6-<
znmq3tn;Rzk9_IUH9A+rYZ&`3gJ=w@)TENQ%Y$?l~j>g%_JA7L3mPhhNncG7*t+|S2
z8VOfzL$6Md?pQn_?)2$JRm`=^H8$|sef&^0!H-2tX2qn+T***|PfuF8lx_=jztJe0
zCpvZ3w-B$FO#Bb>o}az6^o050)6*0`d{`$jQOQi*Sa%Y0;Gvt7)n-o=Yf_&&OFd!7
zyB%wiJ<Wev-2K&5?Wm<bZ&FJ28f}?0)kQlT)N7t{Yqlmi$@gEFQsTB?*{RugCoIu2
z^!n9V{A<JIRTuJD+}B)>^x9l~;=XFFp3?i2%S)M0c|O~s8mUroWP(+N)TPIZf>LK5
z(`wC0o@g6udtUWV{}FbPzh3vGOO~;59F3X4G-HeDQ=Q#C2^@io<|=QTJN*Yo+G!z;
zD{sB@1x{&}y~=pGznOcB!#XhwR`GM<JOV4(_~(@R{o6TPpEt+8z(6*JQ{Qv$Vvbbd
zaE@ZjP5ORG+Wx|EmmO!zJ5Gpd3`<qAWqw*Iv)7<NggwG}j-cQbtyjl%wiI1_ctb=|
zNRcb()k2Q>9A|yS*dLxapzQSZ99QAD4==8i`fM@aeAO}WO3Uu9yDNT8UiM6+(rM*o
z##-kkX~JC>W2;xc63yMPN!~K@@4@ET*FN6)U8?T#p(t=t$jZ(mt?$1yOuF=K$|Xzo
zNY@Knc3;%q@qtOi=BR;Wk%D2TjT#4E>9)*LrBExsZiS%PVWsLHvaCF3U9eIQzTCXy
z&F-+9mkU^CdoFVbDoSr!FL;-w>8%21!j@$pvX+SO8F$`aa*Uxp<ks7+cU~Lg#5$KW
z+Xt+8;(O95T5Hz2)SV~aUgB^y_^5iat8e0p*|xT-iU+u?o@}(a`t<1$HSH%81Wc}W
ze&U?6Xr7SZoJrpTR_#8?aOY&iJ@HvZo|l`d8${B5kFDQQWm&mme#ey}rtE98j8||^
znzZ4F_awoK_tTgAsurd_SaafgJM*1*rSF`7Cmud8!WQy5D>Kd5m#5EpK_hqj%fl{9
zt+rg-b6s@JZO>}G7ZU@6w(WSH`9FBF%+sBMmMr%qIr%kSu^m0|wtDiLnRidGe0=5f
zhrVTP%fHQ+WO^g?sN^ui`dt;_%I{7s*{w45e9|gEA+_b=-J(mm745|`dm}YimOQWw
zaN*v<_F!3#PVmlWC$<$$GFPAAGv&k1%MlsJf9%=f|9~e!O0xLp63xn^Oil@<=lXm;
zGv{?HboxDG^uDj?uWw=DIw_P}>1xH72NItJ3>l3Z*~+d+rwCp1-2Cd~U6C;U1FWk{
zvljWRDevGsrLt>>Tk9%`7b$bkaaswz+w$z|7XEerR4<%$T+31=ouk3@<I=9?G|4Gt
z2br3kgta<dkBgoAJMoBnK#Rxa(BNf-Oq*7GS$zI<=MCkVLQTPD&D(ahE;7^ls(rid
z@r)@mC50811PNzZWr%9p9Vld0oHf0|;H~~<zh#^s9lt$VFsWHxc&mZ+)@^(Cp7IV}
z^<nXZ10tMLFL{5DnYT#hL~_-weh1EJDf`QUu6w#jR0cIqGcgzR-*wCS*qX`mnv3+=
z=B=Mubf-(SpksrczM*pA>$b@^J?}gd)X*~4C~fT0Ha;{@vQu`8tL>Tv)tUbpqi1H_
zSg_$lu1QIvXwLmbF%Fy^0;eWqFFreO+dsYkY%7<qo-jS;@HCFRnOs5|92IYZX1w;?
zeAK~Zooj(egKLI@zv0YD?+XmnZG&I%H?>Z=pf@SL;ihOy_!a5Rvpl@*HwP9!;pVlk
z(#%Q}b60ETV$i?(rOxAsr@{9vA)lF)^&QP(teAfs^U1JM^sv{gXy8`(%KBjO*{?_D
z>c^fFJz-VhBCLG;MW+&T>+QJftGteAr950|v9xqckHQlr9=Azcg;9qJ7g)^kOIBhQ
zy&vfzy<(?ipL<q;AD2(}hJSpEudaPDqyLiFsWo$ydQ=n>r{)C<J+v$}Q8}tBy4C)G
zaqDf<iIX4jo`1e_v*YowLre3fPULBq+*O$M`mm@`E$hX{bEdrY{Stn@<m2|4T820O
zH7PV(Gke~fTyxoRW%LuLfU|uS&PH1_+ar{M^Rl|<1>F|NzO*Y_c|&2`OXFjYPtTsa
z_{!{}yrpM(P4~~vJfy+Lzo}Z~{}g}W$8&gZrrdU(_+;*4!v^!kzjU~?-A``wsDE@P
z$Lps`zS@nZlOJ@fJ@r*2?KUrHKD1vbRq?>Y{6*)4Tc>U~<q%QVp?UMK!%1P!exKsb
z=4&0UEQQND0-s4(@3i6fc8s0wDm{tEfq8p>a`>d_D|9EGNR;0)N9Vy^E_Ejx+Zh3y
zjc&fYS{=m1>@D^#v~ug|XA2f(S8RW4Qy}E-qs*8WsNmkyZxptx>5`RT*|7)nE1M%0
z&fxd^KJ#%=&}J*SSD((cbTM0<cWIM(%>RVpz_*D)xBZjdq*9|bK7V<(Bm2A1%SV#=
zGfuyD^4KgKyhrGTTK0w9yHjIcFa`4qRRskw`=@HIzghOBPR+*gW=clvB>wH+i<`f*
zS462-$~Kx@dUQ}ROMKRhXR7-)ub!%_Hp}A2ih{PUC#R?vb3d{Zk`%O5w0>vD>35KS
z)}gN!thPeeikF}IdoOTRRl_yw6JlOWo^5VX#r(4S#5&hCFIiV>5ZpSMqcU9Oo=(?^
zJBx$&iY%0px>$VMaF(l@V`+m+OWvY6Th=OkXX<gZ6>OPoH)*eT$OCoRXH$P@+xXg^
zl;8EN@mEpkgyN?1P>BYnWl|w|ikJOXo>Ba17NO#G`N@739)}|{mPl(nInE((BJIHA
zekNUs?LrK@T!_j1+X@V4RhB#U-I~d|eT}n|bAS4@TCc<pEa#OiFIlYZe5iHpQ{O|D
z6S?f?tZc4juPF@JEAq>u)T#1}quqqK&LDv~SDwx93F5mnw<6L?@@lciRDl!WY(?6F
zpE`6sGpY`){m_#3a7oVg3qKZp=+Tt%U6AS!D=|~_{Q?F4m*xJlEtA$;UOE%kw_f^4
z&DFTv<608h?^LI?XczreiBX>Od6DPAPS*Y@0V#nzmwc-4hOjefM^53M)Lm*YmCYeV
zM(B#*hQe)!ZG~4mn1`IWHSs&kKf8)IbA{xjQcJUXtP6EzRhwnAW<KHBTKiOW;huFt
zOU_iP&+6|{Ig#zNYUw5>`JWC&%obiV9xbR)ojl#=(z5q%p-iVIF8>;5(f!HHVfV_e
zN8REz^A{AZ&Ux9R_V#}2_IH|V+p1i33W6&;_b;w1T-+8Jw##JKiohjD_N!jqDW7D+
zJeSYt;=P$X{g=x^7M=0g*%1D6vQY(V|BCLu1CnuddwI&8o`{*sM>gvQF{WJlpD^*@
z28$9Yjtv)Hux}AsD3rF?g=@)`MN3TEOH?Pixdtv-Y9q?6E1;2cTBO`igSR9~|Fgg{
zfj>)_cr3IRZ*P44eC2tb2obIh4eqjMVh4Xt{j3}ly~9RM&eQIae&@tlXR6#6`7&+v
zoIQ(?oxh6hE>n5`vF;NU6F3F-ITomVT6Cd_O*-_TvB1+$D)tVmWsXXuKGFK5`z0^D
zht+{qGwI&ycPtN9%v?R`#OX_3f+1~>YE@=>hhKBMapJPb{A(SINvmC=&GoqqCr>;v
zLBMFkImf~k92a)2nq+^qZ;r@xMn@H`)&-3pmd#zxUMa*IHjAbG<9dtlThcl9^qxPY
zVYo=*Tr1o5NqH<C^B(Lh`gFmz)5YJ@-}&9GO#zXPRt2f6Ok}q@%oF7DE3%K$pTQWI
zCnK_piAlNe<~xzZM5|Vd3%(`)iw#1jNX#x-eB@8i_54Vw9U7idCX)P;F<gc#5|^G+
zO|Eiuxp>w0v9-$A@Vk*Bvu`j2$)zs}`r32ko=CI0)HyLO;XOSuo3?Pi6S&*FG1p}2
zZPoXSUx}UZt}<Po6tnZrmG|?0GnAFhd*Uy!SV8f(UhVFi>Os$*ywK)V*E9}YHaEC9
zYu%Sye&6OBo-#U{RJKq>U$gSSG|Qz46F(?z-jsaZ(c|UR<ylKhYL%??K9nxmCwn%=
zpJVr;52mcvaYi8=Hj6rbRmn^?2$gA1JDX+}XMM=XV&+nX8J#-!Gmaf|Xj+obq05ux
z_^Y8LIa5a_;E3NopN}aFC)5Iyr?1l8;Bu;S6-(Cs1U1!}O|kE}Y81GY^;vv#!w>Fb
z&)rr0f9jnb(y=qAdgi9yb1G!IoI11EicO&Kkw%&4fmtbYzCGjn{#D>9>!W|c>DzOX
zG(snz*I!`Z6|3}d@@1yU83jcyhb)h0cl$Xtr5v;AX>RB_FmH#M&5f6vIohp{cG+lp
z9h{@$^-uON1DEOPvq39VD|h`fdcu%lr4@KiO|B_l>s;~qUko*!dp2%A5vx-0O6%ma
z=1u8uRhHK;Z*6q(d6u`JeS@|_HB-9Fq?bah`F>8mY?qnRl@rd}6!IGyZEKg^`axt~
zMM|I2b)`dlXS-La3Y=*dIwWVxZ8xdC?$+BicUU*H_RSAn<+AY8>dJ~Ymz>```Q5qU
zhN9&>X5Se(E5)8VKGWQI=8nD18V7Z;ZYF2@luVYy+Jb#&lrMf!;i(dI{FpmM`*hoa
zLkSl%y(~}YefFGtal`bp4MMjZiXY1<o$Gw`(p=*fpZS4_(;hxsS~}xh|JSGSGC5!W
zJFpgsE)Zbb6Ewv#cjI3lcVn3|lGBS)3$AZzV+>1633N|6VUckpR$-Y+zL|62jjRvC
zwG#!jI2;c3&t$ur^uVv`@4Gh-{g~EHkYDk{Jg#-p()9_Pl6NY1PMY|ReQ#h{EbB3i
zQpu3~HVb=ah6$N!6J1w`oQf$DPp<qZc5=yigWT*%(jrqDIs~O8iw_wuPcQ6Q8NlRb
z@7?;W(V_8ecuTKPl>xs_nUYP{%W#>@Gy#Ws6Yr-tcI;w4$MWgG#JGHyD?7Gvd}LK{
zGL>eP`ugzP9NCZF=VJfofAL6LF!{4V`2NYqx7(EYM;;8+Wtw?r<p~WQp2Hq89Ot(L
zU)geb`qb&y=6yMu;KEt+#nG|+N5T)zjD!sJ=-n(V?)Rn7nEt<1k`sPH>eK713VVKj
zu$trewbd@_;FF*Q#eO?JXv+yr(3k44*(@vi%0P7CjgKY1%N4^_PhMVR^0Cp-Qpw+q
zZC_)+@kVY-&6}(Ezb%w2G%;1^_4hY&|E+YK!*7P;+|7NHeRLJM=A9`v^3UYC>zVgv
z$=NHO*$*6c3bLJCSM}|V``v%by)zwbKhIbi!?nhpf&I_{tBFCbDP9X0jP7nbaPi5J
z51u`Tnl7nl-{lWGcYNNgQq7l1tM2^Fy4U_>;v%j|91E0=_dFB560j)q)BEK+xcQlQ
z`LBh$*iYDyba{u>za=~0Yr30w9v9Bd^5a_g!Q<@3o55^h$9cV?8Bcobn&hjz(Cs3>
z?c#|?5~6wT2uL0}8ELVZfn%vczY&*JW6G(|3iE6-RSTP5O}y}P$_KG`*Tt%TtH@Rb
zIcc8aw!Qq=piGEu-<4j*Jxr0QS^@s6R_sZ!?VQy<S?~9jw5iq--`2}K%(zi}N_pcY
zL-jmktHRESEy?O%_|=u}%-yl$WsB10e`y;sPOOtuXV?+7%It(aL!|2~^V&&kW`A>A
zlb;=?)@7pdcE%@-950jT)?Wz$5*rp)q)Kmj-SpgZ$r`PoKJlH2%7wBOTA{~nmLDqK
zpm3bUp}T>fMbJukuKS11pU!y~%an2-zT{+iozgYK|CpG?!z13>D}}RKg&hO^S?m^V
zKhW9joz*BL5z)c#uY7;bnQFfK5;N|w7ppeZxaK8#sw&wZ6Eo2h?49U#^!CX@%V`=t
z91EgOA2h%9b+dFq!_yjtm}OiBUzC}vHs9XT|FvYZ%H5(>6V%Vnmv}5IXT=jSrDNmw
zJCAx~ZnUw_t(|y6XW`GUb4-NnJUN*ke%3v;^|MFF>J3Lc4Fz^HWmxR^b7_O@oj`%;
zqLAH5nzFh}E;+g{YnFJWV3Dzg{gB-&wlJ2xXZ4TCEM6!v)ptu=n_AN=bN%MDS*w-u
z58OX;BBDJ;+W+44ub(Xztynhs(cQ^S+^wH%6FEvRIV=`C^wGW9tmDDStW_4T4_%zr
z8E>@QaeCf$KCAQW?#ml5@LkE|xY<@=6el!ig<p}cx}Ni*)?X(dm~D|-)$b%4w{@BA
z!Xm4czw9zy+A_408Y2^*ZJ)gEs)fReJ$5ULtTlBV?tbA&*}kD>NqAtuyQV%q=D25^
zyCXH{biMKa{ByG64nNIs5uIO4E}qkz-Lp!$O>XML*NHw;QUhFj3r|e?KYznwyTXZq
zf4k4Lr<>HDa^H6OQNF>eimA`u87*Eg*VI?e|CI2>o^@q6R=)bSyYi0b^%AG2?5BbY
zYeM_3T3>sqY4uG@uJpLWGv-Jp<sw$Cdm23UrS6CMwwurBo}N}eO;EZhx#L8OersE1
z*Uh6Q$5%M@O>ABDhlT0eqW(3phsxdK><qFweBLlV(UP5g!|_;6SsFXv#)iB@8VZlO
zYqqWu=@j(mfAN&hK&z+HvAw{{f!piz|FcZJ@232YimqDR-;;lxDZpVVhvqzU!4nhS
zBp56-HhUh!HJzLDxRV=O*rMGw20tGxW&X->=<xOni9Bx-x!jEPmkR9Zo3X7=wD3eb
z`z}!y#Rc~94Vp?@CwD5<J+Y4{nJzKOYxAKS4LzJ89zw6yH99TKZ=CJBOwQ3z`GMbg
zu{lhwZgnX?Bxmfec$06lg2Bq4^>z6CpBvwt<qA4CVP1pdc}ttJ$bTDM?woaHjZI02
zP`kRHwQnuY{%sc@pE>!SM_xj@lWE%#iQI&&i9v=^cPGETpVHRbx@(=&Q2`gDn&g;U
zXAO6V%0x4n9to-m%x2u3kkylNwwz7OMv^zX{F!9?#T{aAr%lORm>IfIFe=bv!66CX
z7osh{O@1s`BG{HJS1+2^XEu%fki0?DH9=9Y^&f69cZNDJUSn!)oK^l}i)8DCjd5q3
zo-8xiZJ@;{!9FMIgOl|Xr?bqf51LKr<lTFg>n~H+ouj^<j=6D)Gt&fLC^MT+{Cucc
zE2_n-=Z^A3>-J5aHWAY4jGIKOjK%*g3vqAbxDw)M?D|*4$5A-+9((QkkCyV!+WP|{
zS`#B3oj%B{V_kB3a=Sq4SDufL=cs(oF@K}W#hND=ups!>k<&6$#bq3pc?WI2@I=Z;
zGqOo$`OV-1x@{A+WmA`J>HM~9z49wZ=gjw}buxbQ1iLCdIsSh?X|tdI49}566^Wcx
zZI77mIy0wU*KpfeF;!Kx;~4kJLr$#;2Q|3Ym9D6tsBL_RId?JtESq;a=bNlw-{0(a
z*-6#X`kBP>`FfF6OMXmx9I{Zge@kwwm7AE2j+N@BX=O7nJBsKh7f+hp8luS*s3Ec7
zSKzs4<wY0bnwU?`Sop{4h1scVHp&jc*CisYcc*?260-{4pxE0Pwdt4nX0QD4&{P+$
zvop@l593R*{m@wSr)3R$_d;#<&_8?Tc0TlTND~#3x7aDVcJGDqMHj9f-eNa%QGVnX
z#-F0Mx-U2RU4QywRln2W*eO9bblJL(N(H{j^*V57bIY_zt!z_V%|AYvy0BL%J$$3X
z4Nmpbt)CxkKOnaEjd@whr=U-tR_W_-D(BtInqcTK+0*-Bw8_#{pVvGQKcVEx(P$)l
z%iG`CU<MQGpJ%Op6W9ISu)lXtZ_4sr!a<vrUmj5XrMo=OPs8u~Op#Z7iRX@cyy&}X
z-Kv?$!YuyvL|ONgB#ng&nz-F}&O2o=tJz}nOwZE?#r*b~C@fKZeQu}P+gK6i8R45m
zxMqITc+0kVa#{SWIZl7&4^I<y*|I(}VgJvmdY$vC60Fu-%rto}<{@z@iIqvyB(!zr
z`sv-%7cXYN|6*a@LT9IKD{givc>aEnW0rQI_p)!F`5~*FyNgX3^td*JNmzJ>A5lDi
z(B>`==f!I}lbD>{!upvi#ZJ6D7%ZNjsGK6P|MZ>vCo3+kT(f4*EUmew#(%#1J+`%(
zx~`i$=CggU?Y%?ZqM|xl1!=Y0=dE6UaV1mh(q*AiIbRYz7L*y!-Mlx8b4}>A3l6nw
z&Sr@QXsW3Brb@~;+`2t?n)6nJwJ+YxO4#R`HBmU-+3M+$e9vC3S24??A`B({)lQk{
z3(vp#G+ZlU$)d8rISjj>H`&ger@pl%>w4sYxLMJbSy|<qIBVurMOj%pD;F-`>eyi6
z*kQ?itEVFDi0ekVYo`=kc!X5detS9Rw=NDS;o_7j{pq<Z@zz0emXbTEh7Ml&*L7Y0
zUG=Se8#{mIOfmO)zhYxc4le(4p{%u`)$j7$ymKlGcr0I<TE{(`Y@~DZq=i8&?@ZyV
zlBRja&x3mO)}L16yXOABc)fz?ylp#nPY>L%d_|9K)y0Es{ReC%GmZ<MIHrC{U}D*h
z+m5Ld-hA%6^63iueWTTyXOx)l`35yAIosd<-xxO4@8kO&D}Dd^{x*!Xj%G}?;ImEr
zw@7=<TqTvN)9Dk__vYO$D|?%)t>EmcZ*1h-HeGmg_z&Z*wci&UH9CDvAs{^@$8CZN
zr$mPQte*|9&+%<zSt-dM%QvO+)yC|J*7J&fip6(a%2I!=z;^J`?oF4gwN51|1pV*p
z=HH+fm>`v^QInzY@#-G-H)$+)8;%xi{nXW>*|e|vY5hgt*}t39xZ+A#qx5sQ%1_;u
zNqpFCRP(e^+O#yV>~M+St<+z;4^Q9=T|MELyhc_EgV@n^PO-DLc@#6<4SA9IP1Hi<
zZoxa3ybJkGW)HVMJmlokZnC_M$J<QuUe1Xhk<+=YW=_jx&z}6in@y&5Cf90>R}JR7
z^F>S3rsY4&+I>&;VxiWHvmY)qy^mmhpg1>7m9uN|xjia><~&h8m@qGm+by4^*CMkc
zP}b&Z-<7#>@3w|qd8uX;x3B48!l^x6)zQrd&SmVG8OyPo>vq$pyX_0tF<UJy3+KtN
zJfMAA?$@bzAJ$lKxV49UZ1DWAClnmy)%yNGxLwZYRT*2Buc_ku>J}%<^RSQqQ0U2(
z-%Z4<H~-4~yLGL2sNM8%?xtO*RqE_Tj(uLRS8rc-m<fNJ>=n+6JseBsnPn|8HOch3
zEBJ5yA|@Rtv9l98w>-^zP*Rz&Y;Ew?qJ(G7D-RgVG|zkUMfm>8*v&VrcebvRc=Pr{
z$m%0;M!W8vwLkVoVRM?*>osSX9ByhaY-&DR=(dSxU+Tur<yz;mG*&ak?p^lVezV#B
zCWZHr(O!zYXG3r4D1O)RzU!0B8MYyK_t!-Sf1b`Wyes)Q!9Jd=K``SWxA=^?r*z}4
zo={LZ<1)*e?NC8{Lf-NZT!$WSdUvaJr_P=~e#<)imu`2^T{Gd}oLB9^wOgB7_o{BP
zXsz+{eW<BYeCj~;_ULMn<LvKODQrksP?)>y7yDxk1J5<*o8_NwUw6P_liHRDUXv*$
zCbdr)BaSe1wB6@!{pz}4>1o%l4*g=y4bkn1;Tn(KYXdlxS^RI7{tz|f6g#jyVD)MK
z<#sC`&wD#zk?x_6OVbqU!+v)?J>jXPWK!~+fh&sDr{5~#OTYQ?3#L1ciZAgv_Etak
zzRVkgMLX-3)YU&;+t@SrVVTstOX61yXP75Xdo+9Np;-cpqrN-6)?;DQyJWmEHb1Xy
zlg5Xxw+lEP?XHP0KeXs}>W7`5pOp8^bNHNX%fI#f=>ug!mVFzU9(?gTy^2|6@%A|P
ztFt0FLb6Qb!b;wHRO__Oy%ekP$Tf#g*37_Z>&>%@W!8(Ye=p1T&s1C36ML}!Xyi|Y
zX<<K`rY6@e?{W>5Hr;gSe9^PQE|G^7TOO?3#`pF0mZnt(?_V`se=&F73{}flQ3^8Z
z*G^el2A$Ac9Oe*h#(jF>%}ujU|1^Ioy?fDwg*<s$)-mk{W+z;QyALdyIO|&To5}C~
z-xE>TzT%}ypTep`YrZX-bv|id%HD&?BJwj1Xnx=-crUu%=i<U_>-F!Z#{brBp7JHh
zK)C(+%5b0g5~iF7W!4+5+t}m(`&0_krka%^U5zhugw;IsGC1C<nQavP+L^%iNx#@J
z+DES4FQ+}2p*p>%SNQ+KlC7<i_wwG$-C{ORd;-Tyl^@@Bd+cZb_1)^%_Ur6dpLX8g
zAY8dOMYm$Zyyi&b;HR(jY<CAMo4oA3#TD2m?2%BXS2D#~-8bt;%6kjTz!%C<cRDp5
z^dHg>+^oFcLHy}V<#z{MS<iPon|NY{SI$heR{eR)Dk|o!o!6HBqA2MG=W%N%nfJeo
zUaDUI*L*+a>bmSLmCENk6(ZOCy<q(5WkeT8o081F)#k4bxCMVxIis-Xe&-JN8r`oU
zv0o3eE#R_Pt01=3^|f)pJ71MqJ8m66t#>yiZk<`FS(m;`ID<;oX=#2Hk$@SU=hVv5
z)&AW1n&<fSs)+b+h4)SB_cf=x^3OfyeBsg29lLA<&M$br@=u)TiN@KYHv;%|>r8tb
zw`Bae`scU!de?{QvHVX~I)>knU?@(r*zqUGWKVPF1CLc2eO^sx`e$DC5S%~rYSZ*L
zp1#{IL_WNFC(OlqL-wz{>80$i7l$THoZOTe!1`)Ku!y$6jEYyk6GdgRBE16D{d4nn
z`9yL0FFk#RZ;Gkbg$;X;Jt^f@EKe1a+BUm)Ged~s$+>qQy?nwNYP|o&mKBe7ojI+&
zYt8nI&1be*uH5v|+`#a~+0O@Mc72Kwblti6{B$LI)oCYm=Ks{4?5$N}JyZB%q-4th
z%^4}5XPfQ|Jy-hnond*_e1(EWt{e6rla@&M1xhWiS$S@Ht3#>Om1k#aFFjIqZ*&jp
z-W7B3<+;>?&lPT=1*=v+)6=)zaa~|f)d|)u4ok$kUu^lYZTYrE7L8WZBY89@zqY7&
zUr`yjh;?T~Y4*}XA_`G)T>loo{%CVwh}ERia{HO4R@GfCJu|zjw^j&<rL*`?P1~ch
zkz0%@wtU7%Ew`)Li6X|_m*O|vm?G+x_Q6roKeS|(tLu|blk`9CnVxB{-1xR?Q^WQ^
z^{;1y6*$B{uoQF6^pWj+e)gg65&1&dc^o+#geN=*KmPn$UeTqiMK>;XsUKJGSKc-G
zEtkVC?XuH)Q+oKd3b=B-!`$kYcYn~^zHRO&?vM!)!gJTWaC>6macS07X`P3HnhSz=
zaJQI#>GJS8Y_IvF^yfy_EoINTPB+~4+r+M)%l63s_1_1Z`F6egwP&^XB%WuBKCQYQ
zDU-c>^`VW5d_gSRr}HuUUyIj|ZEksX)Vlx21IG2$Cv+rsG1+Tcy;`PSz@K)B^}~y;
z>vEp-iqDfu_xDvzn)rT^{DU4<rO3|*m+xDAI?gXVj_=*Pusvn_HS*^^XTPA-5wW^y
z`KOi4euBv_3*IkP*%;loN4!NNhIO9Ki`=RsRR{GZamq{%zrJ_ZJJE(|*R4-T-1mOj
z$mQYLx!~4Eix1)D^ImFcoR?{|6nJ>^vf_>4hlzp7P0RWUpGdH<Y?0zl3+wB=bnwa1
z&>su-YTj+WA)>CF;&!^J!~Knc)jF<8ye^{dp)*1j?%Hv2uKtPhp*oyzkI(4$&zIHs
zw(irtJjcxk#mYK__0MuP$?uU^J%@j`xtrONN6*#g#?KVJ(X-5F&i6H3@g<eKzs}St
zXGPS!%;Q>Mc5ijS;prcvLrylYoay_jOG4t~<y}`M`+vMR=Y;>fb<v5o3wFM#YFn|@
z=;Q1TX?@<VKLt0Fub!Il|55aZbv^C2p%bkhmn>hM$3NBYSog()!IIrtI%dshjweLD
zO<Js>o@QthcfDSXby}aw)LH(k&lXN8IdwStw)33#C%@mQU7pfr!)$pw^kQd_rhnQ&
z^<%R<^>_Qt>U<KL@nPKxd!FB-q3h>mDQs>$)IK@ca548*0k*EHDZlnFSswM!MQv`5
zr+T!|Eze0lipus0t$b@^E}dMyTQ@N7tE%?;-JAC+l~wvq-X19&_H|cUo3XxV#9D>D
zw@Rh1Bwuk)P-@*hvBWFk{Ew!L^m$#J^#xu1)9;?2GQ%^Ft-*jVuerH#pKbElich6w
zi-mPvA8v5mwr0b(;9Rk-k4)Wn&oIB-_|nZV>4@{~#{yjbB^Qz&9MI62GBr??buYhS
z;haZS#TPCbt+;;D=vVmu>u&`3LUb%*AJ3B9vh-M2qSiJk=J-n14GWgbte;o1i)o*j
z&CKI(#3J`;$6r3J^smrmO_yTNgL?~l4!&H;)x@=JL0QhRH{P8u3niCpSzK`nbNqUD
z?zeR{MP0?$>Q7aww77|Iial94d#h8)k_{d9dxZR*KX92pTx{EI6!asSy=BJM1m7PD
zi#Js+(^}S*uC>ZKzUynn2i}0w6a3~Ky!6o7`P1{1gNKeBx+TJXKQhJdc*;YOjUmT$
zBYDDd&(tLz(mek?m}S$PX_AF*l8cSB?r{tL@nkn}e>r!$(h=$ZkC$?O3|gvC^nc-(
z&sX1RetzJ)xAm`m=I@U4k}oF)r8RUr>4+r78ER#!vuW^HzARl{z466fyPGx!H(!36
zC)d}sbX%Ee<%RZy_v-WG&M)BNvn^fT<aa<RVClp4)7P<`JmbD5vZb&42V3y^ijBTM
z9(>(WBKGCg&gs|Y%<(k*w1GojTyUAIv7dgn-Qs6=mU-BII4N}H(dkDEA7{N~xTg{j
z=)zxiPptBU6;J<(po#Lr`&HHCL~7>EaeW{*^UCGWBRWe;BMXcx?r=W8)W7-S+}h6e
zC6<o<(TZO$zTUm->7uFOTgvC8eXY#OxtVR^d47$$^am-M#i7?(dPO#Oh;_VU`d8H}
z6QTUVYj4z?^p<dQ&a${gE01hCsm*`C%x2q(J?xnqjvVu=v-#m`+wqD?es*APSJU&7
zW|sFA#j`k$KbC**<Ia+#ev<2yy6t&C)@$l9do-t&?E2jOb>(B8^d}ySY^;afdG3lQ
z-`yzVveU7wahCgfg%>NPX{^pDkjhSzebt>`o0`b7S~0!TyU8Mm^^1*xYK(^a&7=o&
z#kOixy^nTDUhOnDuj+Tsuin_oA1c=~%~$@K{w}oCjO+aJrs~Sb6H|j*&o4QsRedSM
zk~e&+_1ztfy?rIC9nNX{nHtUOR5z5X_*po?Zb@C(cbmzahI`g(v`kMp{(9qTjomc{
z{A==>%gQP=AB0a`c5RRTk%kA|?~WIWO^6fC*>g=V;8)APNH$wtrjRSuK~EQkE@<);
zjTEuY7j@@#sD3EIkf7OocGn?kw$=ikq+cI&s<OBiNiCR?>0CXr!K+=wPo!AhBlL^P
zdGouEna?^caE<ABHvjaT16>na#I&vl2f1>s%09I4+UArv$M=EnmtQXrjo^E|HKXAS
zi=Y3KZ}uV+Bg(QI75$ozUg^2C_n~vZu|+(u9M;+;{&;!xzV*L6M*esEe@yzZZ_kdt
zKWXM0HnTYBW$u#bU27MivSANvcyhHX|Fb8tGaJJ#SvMs+a?N(=zpJq9%o<LS1v|v-
zZe|7Ymy2D}k$p4E`&#|9QZD-=7h+8vw?6FQH_GGw@+i?HRx#@9oI*!I)@xj=F2B-U
z9%&OcA&J8_@i>#*9&gc`e72@HI^N%ltpE9H>ZwDGGnrm@sZ1;pPEF#xyKCwX3ylTG
zbKAS@1o^tlmMes~I{mpjgL(0uh_>(*?7@#aHoIoze1BCQ{Ca}>58I{gs};1HV+|@j
zj)_nGc;UNF-kCd&shgcywe4hP@y!;i-hMO5asAAdI~N=X{I<f9@7uZbnu*^ZCqG}j
zVU5z6mu)u>Y(KHa;;y8%ZHk{MbH(+h&aG-j(`NFwzR<4G&e{=~zIkFVhuQwB7kbN=
z?|r?yaING0!wb2XemmWnqPg~PDEq036-~~fDYfT4MI+uE4wlr}QWVJA%DU&#Lm|P}
zT4p{wcOB8Zez%D+%lD4d$47gw)qLO$nHTF(zfWgV$`R&bo%V#H)W%=y9daWk^!H7^
zD>VQ4X}#hN$~qm>Lcf@Ndld51S!HT0Th)!=zajA*FOK@LzFZp6n78-YnmLP>`(0dq
z$(@z^vwrjr2jjgAj}4}NUv)z-?rM$T!t6s5$-4#iFDiId932;wS-t6AW9N+-tt-~X
zJ@Z~GoAZ$Au=;-+rRwH`;!Js&j9(&@HZz>@EvZjk@agm#LEX5di&`R#g_Scm&HW_t
zagMHz;XElvn=dJ~V(k+nmUtKlO7LFQE4V4SN?1Jdp<|_AnwaYZXMycIZeM*|wvcOT
z*d-O0WYreFMJxQ{RxM)cn73>5@ro!{tGB|JYcs{W7VNnDtvKt#E7|vp?x-%f<*DIP
z)nFL6S?huB`=p*lUn0KFY?`-w-kW$A|I6Li^Ecg}x!2$6P>q19HkZ_w{;r*!VUZV-
zg5POvHlDrNYK_<WFCtg+63q|fxW8N*V88p2qs6@afoz3k*2N`OJF=ZFv`@IAxU@QS
z%8kRqubnCu@!4PET61s164%8a_)aPGOU#%!$6-lkgl^|8iH)Y0SS*9~hsx^|ZrXNc
z&hlAHW=rf%P~ItNbxtw3%G7q}-8*V4Ex4<1bDWZBnpby2!6{*d)$5rdjlbTT$S}?Q
z!MA6%Qg5Sqb;Z1$D;BJ9NNl=Zu!AS=^&``_ahKOs#~VI<WyK+P<KVu6<e%M&uRk96
zu=35Sd{+L51Fl8ZUz4A?pLePA{xdB?$&lM+4VS;Xq^0pCvvqbWs!zRXeaEor_IIw$
zV#`j|Pvq7xQ}epss&ODFdF83z>pE;w^KPlm3(WDFa@|j)S>!37h-;hMU7M6*Vb8rT
zAGIg0(sh}BeDOzr9WIZ)J5!%jC~XayI8WPfUrA<kkm<y^hg(f}_M{gbIgyb5f=6zq
zD!<yiDXq<$ukdOZyt@{hE4*ZBsu8ypXO_hG&3Cm!zJxR;oL^(t5*H}4U&Pn9wVPq?
zvAP#0dx|=*ls4WHwz+1$#n(4*iMdZo&liuY0VxJ^J>DJ4yRuUCWrLFIBpqQ-O%3H;
zx^2wgFRy-@Bbp+5=G20O>I*q;n!Sr8gltYWRljB|pJy>u!%y{=YGTkUg}Jtm`h2*m
z=RV<G_4=aHV*U+wdy-C_%(_w|*?;JW?{+U)n`v*iy`QVv=#_Q#Pt^zQw(1M^&eykE
za?kC)^E0))_C{-HU|Ctmw$+dBglt;wTqp8%)w2Z^F`{1K>f2=04oqY$RTedz`2C&B
zrI&$)ejXFuGffxki5K#0X#JAozwnlf34`Zz|F5SaPhUHGfGI91<KFe@nR~@t4X!`0
zd$n^l)5*U}^FJ-p<C2@QcSU8|fjE25(^tdoRF)n*;pq|`b7-=$`=niBI%|&CT#b~y
z+goz`hGTD=<eP^l|7G~U*4rC=xnke0ZEN<-s?+RZ=}%f*=xg(BpONdus){OG=K1ZH
z_@6qza5<6@)D|p%ZjE~MgUn^sQ<E;X+WWn3I2|;}$i*&I?d<N*uCOq+pbD-1U-)@6
z<pi&bTsL`FBk(}<|DOlZ*P0(6S=-=QRP6R(<89xPhw+O%?!EIfTQaNcW_pCiZ<R@#
zeee0j$z0<3#xC6UeGS(pt$^hfnhQJpZB+RzZ!+gkKm7KM#QOj@CgUe}ES&=0=--=s
z^iAB~E|0q!Iu<w29%8pYC^Cg5OX=dwjONg8&l3VOr6TP%9&x<)?7{Zt`$p+Wr@c}u
z@3tjh-|SViG5cEC`sKg681g5X&$hMQV(KTvUwxgS<|UgC$0pe?iZ>iW6|`c{o%O3N
zYv0QrIwAB!2>)BT@Yu?SSBv#}O{6`i?@0Q>YSMFkPvuQM!v&UyS8lWXuCQa$^3z+d
z&k2s3oALE1_x%Sm+z)I{IHIM0+>))o{qKV7#xE*gtYlhsERyx6mPP1>6%9<E3LbpY
z@xK(W^5gM!0i_?iPks_9c6`06N|}B03$7^#@+Sqb3Z31<pkBbv-Z#Z0RjGBRN3fby
zxN7ST{#)k{<}uqwDs66Q3NM;v%DZEa!OHEcglrOOwRY{DzE<S@?8z!NFO7CC;kGYa
zYAF^y{dMAjQyOsx)T^JDXD$?yla}A}w;`o+QEc#{sh7_)H}}qAlrp;7<e<GoMDq0}
zJ0oH4+ZJDs<+pyC(#t5bXj|+OTjqI@{$@uyCl>6n^IyGSgYSB#Ih`Jp_o@_DPTFDP
zX?2gSGa^DuqRckb@j{Nrm&4JSi_${QNv%*YY}~@5(36z^bNe-R;Xq~9C#HP6k|G|g
zSg^<DqN~kCajT0Pl_$onEnhTOr^3K=bImnr`*~}gJ(2z{k;!{+@)EhmLq1lalX7cL
zo_+l}VAm~`OP*%;i?$wT4_(4x64Sn<)I%iW`tor8J<E=rKGevq@@Z?|RyA*n*^`!8
zzvMD^NqPERw*S*3(T17|8;_h#Qpeek$Z;;Z_iMUkiFNvg=FUseu2WCv=x(fd_Bg@X
zHCCW7`=MCW<X1{Z>MB;<ys>-UMc<!auU-4tRy;B3nalh>m4`V>rqS|z{mxH+y5+b}
zG||sI*8FwBXPy&_c2ss6y-4Yk>s0gTWb@?d^layOwn!)P7gx#2S*3q2uB{Gzuwl8@
zDJgHuFFe=suhy(OkSS@hFZTY`c>Qp}^=*>>Y))oq+?1Uex!W?>Y?_C^I@j#Wo@e+c
z%kKMZV88N_hf<91--#;Ar|g+{-dWV;H>d?CS7rZ2_OW&8HBXHhPqt1}`RcXnlIBaU
zyGvFFUtW2(=vB`242!uuI>%W;7tCvB^!(Jf?)-}PuXbMlD*eULdpRFRTi({+C6(@y
zPG|O2X!=fIXNxPejCgc5x3lO;RX5XpzkPFSHYWtQm?T&U>9aU72(Dk=qPu(6?U}tI
zMS>P$J4Kc&WrbTPbozQ2ggGs^{b`ecL|M3=KI5(1Q|}gd{@H%F`6Kc{BGCu$bNvhZ
zfLrMQZ7zZOrBllu&z`@}GEVqXT)$@i*Y&HiT&!-qJld<zlQ{ivh*tYY)8j97ee;jU
z82HPdf4R0N|GCYf;#c1+OP|lFN^Z7zS;!fEtg!5nQTfE~hmm2IAD=Q&59Kx68{}SX
zuxR1;62Dtt{M4QJzq(pfo&I+9@#?*5X?d-|GwhDnR@p{K1TI}S_Z!Ed3*wjLKWER2
zH;w%!r#9h3B3svk=i%F0BCNVke%(@<HP`R5#JM$s+S`2>Mu(X?ulgRm{=9`AOWvfF
zd|M7bi8!e2$9hx!)idWE&*w$kep>p$vR?0U_B`(N<%gu|=3Te<fBjZ@ZDx*3U6W3~
zMN{?s-ktJw@7lw*xJ{B)za~|b?0oP1`W-f<@0HJ8b}6oYvthc-s(FU|hdTS>PcM>o
zmziz5qan23T4&*Gu`4@Y_J(QgJv+az|MkjDoy$v>eSCZ6*;<F)4f`uEpV6+q>6m%y
z)iF-*%Lb{DZQT)KzFF%}mMrZR`R@COP2|nIZ|l05-<Y0TpIK+C*Sb^mcEaB+bDtDj
zD$fr4GACP4IOP3riwCP7$R6}m)6>ezyY=nWvvB)|?yeRawsT&rI9TWs@hH0guuX8{
zyp;y$Lw1}AIq~^<<(Gv%>z!_`Etu0T^7oI`ckN@|4|gVioDu)L?&#;BigyvSmPe)g
zJ9CQXC@;U=rnKlmW9!{Jf~>)JPHnM$``9Knk8RrLrkoV5+i}ZP%Qr2yJ+<Xy^fQxG
zzGB~0yXjqOt9>k<{$lxZ%jwO3rl=3etULHJEtu~btgj6|R;(8Ns6};oa^|wgEs4@|
zeB@pC|9OzQ<9SSm$=uc*u8+$kKHPXK%kw7SPI}gHBY$-<&BfnlJ^o{OMxadSVraL0
z`FqyW^D48<4o4J7T=-!oWaYg2xXr=iHvie5+bg|GkKR?YCc}DC=eotWE0up|D!<;A
zmA(Cpu;{W*p|m{rSra|i$yXe{HoNm;%^457_y?I$XAkTRR~Eak@VR9Aq#E{@^YS*&
zIsfv)CBeOh%dSp6Gqc*p`4!7iV?Qoq(K^5UM6DUiL%#-JTX`;7ODt_sMOF9Z*?g<`
zxGpdHF?V&q#z=QRMZJy5hO5jUXk@Tlb2%EGw7&kxQJr!vSM|=lJ71plmlboH(Cw71
zI*s+|TjyCFTHE_YI<M<pIG+5d{M<}a4`It`>Tw$WCB<`I?nu^}v8?I2PvY*Kp2wEv
z&QD*q@r=lqnk#LY>aCK3=Wl;Mr02(A`}WP0WpnSYxZ23JbY|HCPZ7Bi%i=xDJAN}v
zT*mx%y~us$hqh&QYX4r&w}13U^ULY$>$xuGN>vN~d-?46VguGM@z3X`N8VHZTE%c{
z$-xc#?mbgsSHD~QxRBNCO}X)6{`6UA|JppbEBI=D%jy@aSC>rtEHZb8@!B5axp79#
zo_}rlc06JIbJs_W>H1S2_B8hP%k!I`EZKba<c_V&4{WrWa$#=%{n@7Kck{IUcwN@^
zl~wj|6h2@)eQ$E#T>JjLn;dsUn=~jpsnk~4UUl#77Ro5=xH5fu=NnJ|eUlpl{5sPm
zoLtMdrO&FJ|7E$=%da8#y%tWj(6$ayKfFHGChN&TVa8*AKjwa3_guKGbw}7d^Qm)h
zOn-iG_7U#9g-#N$?)2?%a9^<h?7<x$)yp;VZ>@WFbL+bW-D_=SD+-<q=g+V!J#6xR
z(zCZ$uZnF_aNv3;cj@neMj16B?!^<?Y%5ar_ZdbcGUkR#1nz0rrMp9NWt`6H*U`4?
zBbLVhEd0>s|Kakk{nNj%JsF$6ocYiBDVLi$)UJH>a%wXFe`tRHyo+BCnlHCz@ONAJ
zU~i>dyZoLBlXA?M=eOHTxiIUxZQ`!2%;CI7HGa0Iw{ERyk3A-Q&UE6xerp@yZzrUM
zR!_9ev+cH+TpBp{-Sbb+zO_gn><i!h?9BpU$GtPW#KX37^NZMD+OGcjvzqfQ-`7cH
zcTASdZke-A=Tp~gIngYoy|=z<?Y^__mGJfq(brWAPaoC$KRdSYV_5>n-T&`2o*u9@
z`0X8j-s0$f%NtCG?pnvLVO(GLJl-k%2p`YRC!z^w^*kJF67D9wf59FrUy|ZflkF;T
zz*Metrlhs~E4c|L4!q!evbS>0!&B^bd-e7|>78(5`p+w}VlyY%R$hFS#B#TL!EJ$x
z!vf8h8aABzl&Dd2>HCG3odv~4U)*0*uy~kXXMe}**5Ka0`PUpxyJr(y4W{kZc_3(e
z`LSds-@DFRHWMd_2W{$i;oif`?YM)RbrRRyw>tw9?&^Ez9+6NJP|izU^nIuLtb{E$
z{vKVhPSF3zf+xL&afUHAN|!dgoE~d<hpEPDmW?u(bn1DIiH}|$`S{s|K{d9yb3^3M
zr5}>+#V|HkNS{0>Q67-pKEX!4Aly1?uG8X_R=1m?Jt_}YW<R(wH&}2^*RJRnf`XNO
zhM!nvrM~ja`W?HFlSy3h1b4;q`8%fb*Ibx$IXmj!&3%>)r)y{L6#mQdVM_4U?dsf<
z><VfOD%cKRa}$_dae{kRr@H^o$#R=Dqa7>t<%{ym&OCck!`smKz4X%LpL^B?-FCXT
z;tB8Dv(dAkio8+((2&N{_IPQ8!t5laNyqHso^|^Oo%mR}iLD~X#WL=KV5HocqS9&8
z-zn~L<vX~Mn_DZZ|J=9MowF}p*0>mMbaCRx1?QcwHFv6SR&euHnz-P0+iPPXHBni|
ziwTPY<x4UTPdz*J-Q}%H_w=L-B0C!IMU))y`DDB#g-gzDw?J1-<dZ6)$%;%?DKmB-
z*&p}iGracyFCQ8p@O%9!%K}T@N9Wf4cW4c)H{udlIZr&+P@X$iYghQhil@wVUZ*xs
z`E+|0mzvNo_fw+T49}(V_NdA0|61uJ6&x(sp<-;@+P+h*nO(8g`{akp*GI+c|84HE
z7Y=#V8T;?|?0@zX1i#-8*z`gF-P8L$^~V3s{xE-xmvP!YR6ZmlCqOsYCTUK%?c0C(
zuV4OO*|YoK?QhqA|Nocv<G<43l-aXBO!~m@*Bexmp0d`={P~1r?Ze%-kBgh{3t6z5
z^J`Rw*{h1#Up#fcaX%|Ehzd@Ry;eTMc!f`9=Kr?J-}@&X+%7ZETP<|Laqqo*FSHxl
zZu?Rt-SBEd>gStv3|;}RIg}FR<J|-}>nc`;HR#AK(&Bmf^=ZSuv>)+P7V2hi^=jG}
z!n&5rb?G7B1OMFbybPG|{&`7vn1$8vhw<^-z8gBPnGhbmmT^tt{nv56;j89-T;n$H
z)YT6mCLR54;?7$?Zk)f;bViK-7b!2JoEvpls?TpxpX}?q>6lQ#qw_0H``hIioL@b?
z{Ki&MiNA}r8P(pb`n=+8qRD}cJFZPvUj1Lz;b*mrU-qjNzKQ;FF58uF)y~<oIHIqo
z?$4j;YsG)gk6s-1{F>HTv&IiwG~%yba?roMqPVV9g~hmJ`o2}GGMZlJ%Kmx3#ctj;
zSGj_hN4*mY{;1yBTYSw-=yBit_Ddfp1-|XHP7dkhUHj(xl(ni;Z<Xb)eQ@V!s!yx;
z`$V06qS1d2eS9dDJ-c<$+waVZ(JOhGw{&j}=Cyqrp6j8pdzNzczExZAoeOwkTKE3k
zu?o+m*<Z2@a;~O)-5$ldn4Mjx+Lix-nP6(^y1z^T%pS5Uy2P%SWf{&9>z}qUY)!C)
zawnq{^Zhy5%(?pox}!SxeLXOF)z7T-KjtX~`xFJACwu$64VR7m<8AYIds@MQ4x2On
zuhkgWg#50!yLVm8g6;?L0&Yq<#ln$KJjAwcx7gb|#ad5MZQI4GlbZ9RES49gYPBEU
zTH7NmD!<q5%J#$mKfF1{_i1BeO^!l;@~<ClPfOp*9KV`BgJs5(cU#uA{d~K9@1ekc
z<BDR7*O`_5&imxHY+27*=otMwipxCV(F$>sC3jwJR-Tc#^W*NS8#|mOSG{04ZY#a{
z<1S|#(W*;)HRW44v!4ImpJ@7@S5k&Ues@7}@q)Tl*WXNh%3pOMN`{4R(FOGrYszK>
zCNFZFa$)=Y@Yt6YOqq{1ywUqShvn=xO96$_>O;GkY**}G#cMbB;e%Qxt;JD0s;dOv
z=rgI$V%$5ipv!uK44+=WhprnBA4_QEG{0kDeRZzxi}eIq`%slxh3^*#y%I2Eztk)z
z!Nb|&bX7&}3!A>o!p%alMnwzG--zp!j6K5Uz5f7%oz1=M)e3fhG@q+okbhfdAfR|v
zY43+cQa^P>ckSHR*YkUF*|rel%3m3kF^ir#P6+7_i+Q>E^`gj_*3P(;9rle%D`OWw
zexG-5=DZNjGZN<ey#*#F?u?tgM()`H?^irm!yIO@Sgp*Ld*a!+aDVxoHucymO5E1#
zp7K4P71fY?x%^18*~+kq^QL$%2wYlz<XTCnBS+nitP`tijvvxuIqCOb=U0*N4Xu<1
zvtr~t3!44RQriDi8#8=uasL14FNeCivEc>5n;l%69Ku?+^Ht1?v)`H;lf+|j#$?x#
zo1uz(ckTK7qA8duAzPg%l;5>~@@p2IN|~4)%vOhvE}1)5?zouVn|0sT+H9>jJMs8J
zR*72=b+(1;t*J_LD?7NiNw4>g<Sd5k9m$#+Ob0gx)@Ei+nSFx!4&&iR^Lk`Aw{f>^
zl$yD3edq0mM|Z2wE#rMB_DnIr*6_7wRCVCXN3Ztx6dzdE`mJ+~gy57reGg8CH*PI^
z##+vQC!zgzl$OAeSDW^H_UoA+_4==%$UVyg26v9Vs0eLt*t_Zp+lGkEr`E4B3)>h}
zYQAdiUxwUuH`Z@^UV8XL_m+Lm#b5VT?6|R8ER!KUrpj<$2eSgx>=n!VzE3u?xx&5u
z;kAi}&v)ca;W+YGYti9ZH&)o_T~BZ>k+1BWcyJ+C*jBy9br;rZS1;1po{_s*FEXLo
zULb6G5TDb8Da9vW{&9Be5a*j{xUS>9D@(8So;QaMG(@tR*{+p2^vhz=9NrM-Jy8#5
zt1pS%`?X<hf$o*=yi=lAE}Wflqrp8u>dzBdw-VhoGc@M;+<)j9vOCZ`HTU(A!1)IA
zSC+|@zrPeY>$=&#1D92|h&3#K#g*vyy6C#-)6(^vZ#2?&dZkz#aI#K|y&ASRPn$Pd
z&^qsXQ@p_YZx>e<6|{Vnv0XInp+j@Ww!m8MgDd4{oRSqkm3uXD>UOVB>vW!+Sa6`i
zKU?hp!>R}EaT~oKq^@Sze&BYSqQ@iCS)z|@Z`52Woaq-_;Lv(sa@oXnPxTiF-Pz^Y
zzjwZ<&Ca@+cH6eD<B`ugE}?0Cqw;uy+|`4-%sL-Gd=M0V>(2u558mP;^VQ8u8~C>^
zzCYvnT#o}~_QIia++qWD_sA4lxc_jlmYJ-cuh3M}v+t7V1?|cJKBWm9@#*bvXFYr#
z(Q2e_VD@d#&OD<}!s*l2UHxL<rnceT7tM7WdKX8q7;!M<$o;UB*V28T_{Q(<W)3Tk
zb$dfsgcvs5mfmnced-p;6E|mtW;Bbi3-o8iD#*DkzxCv**GD!3f8G%1`|csWL9ezN
zZx>&stQ;S0IQ=SDZX2^^B#ZRhioZ3nDi7qYI7l&9TTE1Z#Oqpc_WPZMPOlvtwk~om
z|8eSO@c#mt17F2vta#TW(DL=QWe5jv-{qK+auK`Am`8kbro4RNJ7>vJ2B93CdCYZd
z7fL@+h^prEcRk+gWbb)~DaKxD`Ccm#>m+rV?FanX^Z#|(B-k`ew0QO4ldIp}1I)*J
z??^msUjJHZ<Eu!WNsGhFue`0b33yq0;wt;1E}p~*e9j3a1|ky~|GRcc%6<&F+dMI6
zlJdNU@A@0{^d#KWv+lN^+dj8PNYJIVNJ;16^yUp}Q6d$q+Z;PnL<~$P25z`AD{mE3
z+3M9!PfC@{)FoKw={86z7;LIL5Pz;ao}KG(%!IuOJwi@-+YT#c3B9<*m3Hvq2ENWc
zOz&T>@ttR-B;d+Z{9eS9!(y#dnc(^d9@$|Bn8KJ{D-EtN&zwBjgVW^d%oF+odIhgF
zs{&6ZoUgoGd!F5#{ji+a-LCep_9d$$S!Z_@Hry*nRrB;y$hb2zdWVD)yF(G*_k)Zl
zujU3_Us&+DIo?e{C}D?7+`)<ET`6KRoOus+&Y##gg<;X^!|Mb1eacr{x8~1quri1}
zq;XK|issjnlWe<I=gO8mKECOZQ_S+Cw@&IlunX%?U^@7$)9y_PbAL(RI`&!YZ~2di
zzxc>o)Fz>NIdXQ~jUOr1jORXmc*eR#hU@n+g;wjw_X-Y*vUU9aU~$se!p-EPOY>K@
z;z9{icBTA}GVHzQxMy^-H!z(PX^$;_{Lr}XTH}I)It|k6zStKiaI_q{u%ba|!igov
zvrgJhjy#Z&b;f_e{-z9P$K4zKc0Tv{SEI1_z=ssmcLE#YWwTX&t=!yceRl)z0daOC
zE<O#`>H8<mJ8(vbmF?J_4@Vj+l;<}H8(mm@yR&QBdD~^K(~ol2HhL?#D3){wE}ofJ
z**E<k=i>bw4$((9$$n-IWm)jotNwb&Vf+08OBV#q)OWeFlIJeB$$v(1tG0bBxVwcP
zuDEihJBz99_NtQ>&*OXM6g$oOaHumZlEw2NvqkHo(<>u&8uzcR7W&gzfAeqQR#R@H
zplI(SR{BRLSsqYm*y5FtFS>_;WdVEO*6u>K=$!|-1yu@p{~Yjkk=#2;JGAn$&TsKI
zOgtBj`!aLQno~F}wl>H!zm#5)o8f<WwY?;3&yH{Gj%_Ov)!j~n%;Dlsa|~3L$YOjj
zPvWlVPk||19!uYSI;H<$+k-B9CGp9#b(O5=ytB3We(-ciVPa3#qCS<P2TR1XD+9#s
z_DQUGu>7_=+h+&KGydVpiZ&?=!p@Ykt~^qB{nOO?504~-PW-v$;<Q+u^;W1+OyUwb
zqm^gB`S%OW3Qg&2QMC4w6TX$2VAJxGsl}S@W>5#;ofhwcb(aehj`2?_c(Cr3p69Co
z+u*}~hg?7IaVVL0Z0UlpJbUUHj;QQ9Ipd@7f;eZc&mY#>upaz>U?G40Wsg;tlv$jQ
zEf5S?@kvBZCY5da(|D0R>3ND>>$3{94C>e=xtTmv`TTfx8fcyDRD8eS(D?&r!_pTR
ziHlXUm@8Dv@v%xXN((wJ*iiCS>+5HitcJV^frUa7O!;0jU!1c-aGek9`W0mt1j6KR
zb-8gzr-;nznyj6C-1{A;u7nNa{{yK<<gAPDt1fl$ne1_H5fjG~XTJ+lCqEp1E<Ul9
z;qr&atm%q@;)kRYw@myvJ?k;6qKct+(z=7u4D1gB4y3KSRL<TK${*(P^kE-Umg5A=
zN2{72R%?Abl%c#~>Y<Mp+&E>}$_zUgr}OG78u+CfHe67O<WYLs?9(>AkCidEMUd(7
z4TdNIYaWg>(IyfM63KOziW|i~wEq63u(**Ue8MAva-%=o7iX-y?kXewp}~|-g!A<-
zedgwC6_fX}_SLc<JyF=XU*X?{m##C}n=W^lr|Qh&va%3pI4<C2GU?!o8?rpSf{_n6
z8C=3vu5l7O-2A9dCA#i_uC!Ru?kWC}8=jv`DqxR%AoAyc?Wzp9cCEmfFFvPCelJ?c
z-QIY4?v)<@t)JOa{a5oENZotVHRH!*mdH&9KhFz&_vZ4%%O7TBTsV0rZHDTD%?>vu
zE=aD=o)&lPdN89Do6PhlD^GM@oy{oIX?pjxS6cJKRYr%8R0as~*tw`)IC;CZ@Y}mb
zrWFSsmzBs}bE)lc;cL)!+Nm$8sUV|yVD&oQ3(*NXf?ar2U5f&Y8M_X9>pz^WmNbLu
z(e|oW%CDjyxRoxO<}g#XrfvF3{zEL`U!p~B&3cybz>wo@=Niib)r^n@mdCm0Ha_#a
z^Nfe>@a#0+8Rs6GWhYoMseO<s(RrZjvEWLqQ%JbTdtu-A?CL$;{w?g^m}1vxToJx2
z`i<>{UgNbpTc7h4iD%3TIq-VQ5l-%Y<|^@vPQL6Pcd-XE9=PIoOoH8FUnhrgB-drO
zrqvaJp$Ej6cXIkA-8ku6u(`rO*kV!n2BAgVCsp@}JS}*<mBDpljN_uK8eMWREIQw9
zDmY99PdH};X*w=1V>r^C-n=8w(I~b+V9_xf{gi9#Rw^@ht@c&0nwhM=aHZnfu*bF>
zyRYu`Gg_<X*%~C2CCVif$sy<3$E_an!gGaVC-dLEA#WZi{_*Z|TvYCQfk`HHlG1^E
z*1JsX3v;Y3=gX(8tcbhQZn5jL_wr5RbB`I{aF^9%;JKJ}XYGxzGlDj3KT>G2m!-`*
zzU!r<rP~6n`L_kF+7u#hbbp&RZ_=f#(9ca7uh}EEcRlNy{G{f=!`P{>9=d9#PRm-)
zWf~uHe7g8V8@}=o&zr4_w@6GfJ<9&^#_Ps6u}A-zzEV#O-fFs&O-6rn+^Vis;%k*;
zt}0y6`ebcds_)fSkmzvW98=2G6b@#aV`WCi;zj3}9kX6||HwqS=_lRUX0Ps@ccmii
z8l%weUF*3l9FKkHy-*nT=bd2T{443Pn@`y-@NoDNej-cYvhIT`tC(F+OTO4@#20WO
zEKaZ3vA}MF_7iEQjn~9d&7U4LW|=AHvL!rK(e#e-#S*sN4O2@dD6(r_&5~!5bbcWv
zaer^uqt(+E|I>Kze}l#qQx-3#)j})GW-Sz4f2Zl#8IGfjKYlOYd&>A~@0VcFGxCM&
z(hYb*P9OTE+acmTBfkIkMPUWbO?OXe6|9&V$l-8shxXhFg4&Y<>ccu839g*VP@}|?
z-@Bu%z~n_pn6zk8-!+9TTOFsHIRtWYxV&-5c)(x4ZuH<xLv)7cW;c$z&Po~Ad*@po
zVtC}n<Q-t!xGLRkXKT-DrpKKt&u;j6Sc2Jf{_?WZodTN}gKGHqF?gAB+_PxerT6^c
zGA2IzQ%|$gSG{_zx=)}xGBhk>V_3|M1=)=?72%9Gniw2d6jsb+VAC-0GL>ZBBDpY(
z|HdkI)1(CcgW?P-OY$5lawa^mF3S{+yePo%m+{%ceTtgfY&V7_1yy)f7D~uC&kC4*
z&L;4H7|YFw&}|G#Chkj}71WPRSkO4jWdkeYBlRz*SsSmNyfAk`bHkAfHL2nWFBzVQ
zWR^18TCw^~tz&Jo+7T@9MAFBtBt^)9!SG!wL+!kSN&>8v1{^(w-fNxSI>q{Lx^ksy
zeuztgtkU5qA@8uXvz)<!yI*#!@i_4Ih`S#*Uu23<xk7_0i}yJu$%!8pt!!1^^r<z>
z^*|WU8P6imUF?@!Zbi)V>fqv@x{PJ=0aFJZ*Ef1a$qum-&5T^)1Uvk$n5j(`RA)}F
zN%wI}2|451pq<7b@g`u4;{yYkLtB?SB-@`}_@a|BTl}adOTg|4->=xExOTE1TlR3S
zvf65u-rzf$I;B(Vm}maY`P96SkJ+*B*~*8hD(@BF1TF8A&=u(7pC=}i!L+t8+uK}m
z?;Pdb3_J%;OlH5xb-3fumph)uf(qB~9n@Zsz2Wjh&&mK^c9HMR$34C;=Js>!DHe#h
zTFSbrID{eP9HWP)0e8pO82;)7yJIK$Zoc?CrRpkI?%pY<kJimq3Kz<7%40b8lC{pK
zX_+yHjbO)}1!@f&AADOj%SnR!jK&8Ql?j|~8Vs`zoY>>A-Jn2Y_rkiEe5SVE5Y7oQ
zKSc_^m5O;Y-)x%Y$n)9EQis8!<BX%?ekn8M<y~tJ-e8XZWVf<;mCC7`2b?Odg{|~@
zaAudsR37$oU#?^{OtF#_lGWIpr|*80m63JN0u5;cxdUPyYqnKh?~CKF(G<ET7#Vm@
zE+YFIPrOWmRzt9jU~j6|bB^^>nZz~KpE2w(I@G{<{r?21<Q|KdI7d-Y4bd5YI6^n+
zJ@9fUXOr&`)p#P#b7FPKypY(`7Y8Z?9u)L5oJ(0?&+l`xIr+0=@_HVN2~SUQrpB$<
zCUnkg#?OT;7g!|MPvB{HZ;-2!*}0YZueO=L%}hT-V~HBuua$3w7_^(Na+u^c%N>?p
zb$-v4!%vKlOrGHz)8?9LFoR1*M88O?F|OWW0)vZ$pJ=Z3hS>)u%#5*<-FL*PL5gwv
zQ_p9z3cdoV6I2(~?20q7cXY9D_xNwIJYh-L?1p0IFbTzf4-AAHa@AgmYsqui$sYV;
z<SU+a-tTOIDwFcYxL!S(_;XAw`$cA4sMP$qkh8vVR@09)W@Q(iDJGg_I0#qrEnMay
zy+CU1%u5`z_FehXEOV3lGlScdWqOhcZBYrj9b$^D{Zj5r_gw0pw64WFOv30!v(__T
zmbga2g=R-M3~xzwT6E3h-FndL`n7A3#m7p8E}FPT&-IdWa(wWAQI+}v$4UEts#c%T
zz49z6*&yfC<8#Lz=CDp!S;f9aq(`A}(}iWKZnM|VxS*H1%~Z+qDZ8mx-L>}`g_?H?
zvW_GNoVdC^v)$pg^(7O>U29ILaz4<xkp9c7aGti6*Vc<Xe;OPnsjJQ~{bcZZrC#u}
zl385Kl@<P&glUBBQ)2o&w_xTf1J2lGXT>g*%)Q-HSY(!{620&ttMXOl(}&o*j1F`*
z$BT4)S+$+5lHc$x-!Ud1{VN7r%vOjVx+iXU_jRgB&X)#VVFyd+c-NiM*JLMnUEA-L
z&lYo~KXO5^y0Vd4-Kv!EC45U(DPPDr;8?}xH|22Iv47exOjA<T^&T#e^=8aIm~F%t
zk?P|4ZAod<gso-!-!M7OFb;eWmZ|Ud=Zi+1#nBS)RsTPJYu9Mnd}ChAR}Po2oPC}y
zM?_ywcWU6VIiG3DeK={k>4N&8H35?}8`NJu*Id`S<6t0*jANF{PL0MtHY<uc4m4kA
zo#ztSx2M%De&PaVu}iKYeHkY%a-QN8-Dr1DV}gvYV~offPbG&dQzUNvXMHEi@~XSb
zBu2LRvY42Gi>K`7uE0r5Os6@{wme_Ke?jBZDW<qzeGb{J+v2!_nEx92@S1V%cq6ct
zHO1`?OUbg?DNm~Xjs;&=J|V{C9k*if0@3cDdt7d=ny_9eGFDt_4@ZQ}N^^$DM#ayK
z=U&$cY!5Bf-(mj1T&kb*;9HlpUr%p!v>#FUBBpt>&**g7f$GL1G8^Q27i<VlX>QzV
zbFfF?RRY7-?4KLvZSro(4HH_X`u_CG<*xjN@~!tey#+*_#KTT<>^zqncYNj+h70GI
ze_#AC`$qq7?jPq*emcs?T;<hz`1F<{K9313Dg_A(gAY7wVD1c(-KcQnQoCa8tP`&H
zo<w-oID0-&nUkhaAm;G*&6N<BBBckZn`9T+EHJ#EBFq;(_gi-H%;U3T3y+;^lw?g}
ziH<wY@@9s{rb3r~n-7d_K@%+&xbGHb<Pgg8C^&dzQfHjlyC)ORMKUcqqH4gn$8|{>
z`#U>z{-`+45RZK))HrWUGSk>nB(#9NfGb9TYhTi4Vf`k1Bgspza@G|{Ih<n5K9qHc
zG1Dq`_x8)t;yg{;ltNQ4K1r!lpXp|sai#6Pje(l)(GI=LC5$DW+~((Z$1%B1oA<+r
zr7NS-?nH;DiZ7F>(8@_Gr%P5SopyR%>7O#Q>{qaa$hEe)a|5hPq_*-Kr580F7M>mY
zq~+a{qti}yUN>}AP`=wC9$)h0!@P)$`j;(zCa><<c<%iBpmgnVgLu2wt#P;9s&eiL
zvpRB$mN9F+GOW-DTdGv>Ad&5ZW6BA?dkR5|-)0E7o_TpNF+uzK<m=4LPMI$EJ3Xup
zFf+Fu=wAP17R$6ll2OW)$25z+%s%j|X}7|TkE>$Wamw^&o!*f1Y2t%t2Vz|$U2=9@
zTcji#<q&wY=kXRHKhB)itOK$)6uMXa6YS`_-EA_x`r=(>&#504$xps+yyuyR;QvPb
z8Ckd1g*vk_ntE4V^b|6?do@DdroPUzL|{Q<<I<SQkI%#v{=|Ogd#E=rJZJ`U_Oib#
zE(UcheVr1J*_xrbQZee5`@~Nt9=%iW@)hw56$$(tQfGWQk5R2O;dS=1`I>$o_ITV?
zQCM(|WwPd$Xx>LQCO%vLbS6J^5PraM(LjesVu$RnlUcR_FW-CV&fl=s(@M8lS)H?T
zZTf=mRmZNZknda}Bz1~KNmx8gZIz_<oYzc~x^6PLzbQCd!gRGKJ6&|?5-(w`4gX41
zuBj=W>YsC`Oa7+R)OCW{uPQq7C%%p{KGNLlZcr%Wbm~YD`_psMKSFnyInJ2mQORnN
zta8l7GuL;rW1y1RTH`p~l`9HdH&pMqldHM^jG>lb&#QC(5^}2X?z!I2Up6lDy4%Ap
z`eVoTcN}Ihuij^v)F;#wl&<kpcyVsxdizyrDuGkx9}Jth-d3i!W=fh1H`7%P{^MDp
zVsVE~@7z|+u%jq9?AfGH^GPdipUT+~vSjv|Wg5<G3>j;y1kYwJnXa^_n<4B?=I_9K
zw;-YP>0-f6lZ1WK7F{vE{7OJhnqhwPB}bjKy-h3fZo9C)42y}@iScfjy!L_IZmV5d
zxrZ!lB@Zc0aQ#2opnh*#-zVn;o6KBp@esEM*0nnxsBYXneTGm@mrE$qnpIh=zfQ9I
zAliA(cWSvO&*Y%<i&g5+oju6;W<l6evpFf+f$#mQ70<Ey9?59g`!dSp(t`u8L5moq
zWmB%&E%XzY=40NM^k`w}mnUgL&EXdAOB}cEXFkMqv#w$0LAh@w?W#W=qqXk637y$}
zYrTU8%f3}Y$6RbX3zlxUs(Ev_`jL+n%g)cb5_l(i$z;Q$4Wi;5_8E>-&IE`>e{a2D
zqW6W7ozFsebH*FR?eXsCZwjCPcG@he)U(6pnT^fo&h=MKK17JB&UXsgeddTV`;&!w
z3`(D;cs{n%y?4v4m$6s7Q1bf9myIiVj^>x_zG1nQ{fAJ`DPiTk9_)D?iB9UXgtaEQ
ztQO_EbIChzZ)fu|@0w>E%3I|>y<krIozC~p{``Z6pzDtD3B|u|#b38D`|sg?=b&Y2
zc>5pOh6aDvYe_5JQy7%mx2Ny8{?p!|zTWtQ{g1G39nuy-)jhqB=Um&+lV3Ui-ksA*
zFH&DUZHp>A_HB)D!V{BiKQ{hXU=DZqqCfxnQHLVy{M$(>=f2zD3yIzFcz@=RcOq$B
z6F5$+*jN5cg_Ymw!IQ=a)d%W=Mt!ILAHTPI>B+uVQH$T_dD}!hy7#|tu1j)9_uGy$
zk)hdZZ#_S+|C~oDM?it)Mcayfg0sIG&-B!s*`&0$M7GMXZpGqDi-INGuY4-lut#S?
z(w>Qus}EngXC`Knnb^}QF}o!}x4l*1g9g_zjl{?1Zk*}Pi(cvoZ3*T6&=z~{&k<i9
z<NTw=KH?E>2j6~V=*axM_-#}&`*q<n3lAAsiS(p@e0yr!3Gpi@vt)!lPOn+?yDjy&
z`oEVS!@ZiX=B()xJ2KJW<VvN!7GAFeg-#at$f-q{7uLqHz8AfF(3rRPXy?Hfd5T$c
zroVg=vcLI9{0#2<X#u5%4yJwi&3{5aS414!{_8ydJ^T5NUyrF}7C3DYcKfsJ@$AKq
z;&0Bsxc;Plw#VYP$#V`(o%COLhpf+zHnS*U_OizZ@?>uGCZ@?u5p!pM^uj^M`0><B
z|9qK0)@wJr$!qu~&$%SAd}Zmyg=K-SVww7a{;hs))54gz%IAM-@1MlB1J~T&yFQwJ
zW{bhvc?&E5vn$C68wDuxg~`0zzi|EkWjsm0R%%x3>_6gj&_Da){$1|-CI5Bu$mv~}
z^V)iQhr;pYu2tI`*6Yu%Qun(o+wqXk`{(PF1>u+9M$9qToNyp@-E>C<-8nzs%X5VO
zv=Y?16Y_|iZ&k#M=W$1p+Gl#tnfp?1;(kfVTZ=Z%+&+W#_S<|Vertm)V<od_1$~zf
zpBTFAou<vT+x()d{bY&Q&4W&wvm$b`5<ce{+3tOLj^CPPisZtM!cOk<Yz875S${dV
zByzuLuc+JGarFJ>1z}?A&p-O2sJFrOnX`wTbWhw7lci#X>koz=`*NZrK1C)zc&lR7
z`;Hsc3NiXlUya=K4}VsYQDhRl<q&mbH?w1fWc_rO2NTQvj@@#d5X*AI*2O00-pj@k
z$(8rHv>$zV;E~Coc~F9D;RUYOJx71%&ydMCc37XYjH~$U)Ds#7)Ay7`=v8ev!ZvsH
zF0HvUz2)}bb^RZI(fw?H?$7tz7ymQA&G?;bm#^Wy`K{eY82uaF><&ET%}km#Q*7DP
zOyvfrj-$z21QHYvI5n^`tTk1wPIS_3iMnydYr#~9)qRKF|GqI}=SEiD4bzUPnS5Kh
z(Z6@A`L%liyCoBulx`a=TlUE@L29PjGMlctX%~Hp1zi)JM4dzvy%snv5ZeF#@&8FF
z|9A4Z8Pz{uH;-q!^t`B37Tf0vuD-PB4eJj@Hn$4_I|NtGotVsHE0eyN<!L_?Lu=u;
zfPIVBiY90s2x(Z!P#kh#WkWFIGA19^8Egi;2~r1ScxrAiENAq2Y*-SwtTD6Zr=%{+
zX`%BGGZtw*Otd|c6#S!+O=g$VskGa(a``zjc9;tVC@gSrcwDd0a6pZf>67r6MqMrz
z2@V6H1QDi=7Kxi%JQ@snxde~d{)rcUd;j!>L%BP*+xTvO-tqTFfm!BVKf$oY2d|e^
zCaRbGPLus!`<7wLWw~wNKL5SW<ZigRSKNPN<dGi#*llwU9m;yL?IW+f+k)1*iX@AF
z-H*S-pGy#aV14Far0I;U^$SjXlgx<=boqb$)s&Qv``zv(w{hJMuL`(5Y0d@}br&^W
z&g`N?vw8xSF!n_;K6Dp<W1DLmD?2^0<a9^Wgoi)UAI_NM>1(Tb;f|<_z9z@wJF&`_
z+rRy<<&ns~mY)0Y%~_k?7>jOQJJ%NP<x9o9HywI<X`k|2-@msn2S%K%-*EWf^5-!r
zf_FchUGzOZ<8=J_1fQkeep)uFGv6DZd*rS0XX|#`CCk2FzI?8zX3=}jTCQAnxe05S
z4k&L3I`r||@wK^30eLR3ZqIh_&#jT0W74d*+rM)E1kV0Vv;BXyC#GGV-@10ceOH><
zHTg9&`z2d?&7UuO{j8Vg&ey}K*OzZu`^!Q6e-6Lyvz397w^wi4%-eO~z<=+~TS*Q7
z+}XYxHn*l$EjYE`+4=c*lN9#Q`c0Pq@1MO;Ryz5pLD&Bvy~nqCBy%{JQ&Ki4&dl*R
zCN*c{46zN0tT`SBS4BCbvL<jHm=U!0Kg%2@16F9V31)ou-lzTp`~N7H|Kdgtm2E48
zrll>rqFci+_-dQxVXmu!7xt|^AG>K;Tk^gIEprb)-Ff<x+`(3_j)@E7vd_ge7*(yE
z&U9UBmG};)H2D?pXRvmO?{HdH;6L?*&V@Y(p1S(}esA{GZT?a%t=xZ8uYCD#EMY2{
zTyM9Tw?1a-#Ho&dR$6HBoax*xTeso&Efu3}AJ271{%Gpm?$J8)zwE6l-(UP&5BKeN
zWeNCNTAH%O;>WeOySKeyyZGqR(ba*SGYcm1#MQnx`l~PS(^#g}Q)!u3&2gE?2a_UJ
zTCCE~n5V_m8YT1hka4o>Oe4O)#E`Xef>$n`nZm%$@Q?MS?i-Hn8eM!R59u8{#qfI1
z4?W*w=N6sHGYX&6vf!isg#|~sb{GYUd4Fuw_@{m;uU|C%<Hkd^c7McgC2d^sCgkDI
zOtZc7v;KQO`5VaA9sYVs#FNEcVn@o49gjP#_oq#DOYs@m`C9~|D*}`&&lp%o?RGlq
znzmUp^#9}{zG?T_Q;*j3z2D9gvGw1nw5DwRfBdhu{HtSIbEQ80fsWm`c&Fe0J9)%j
z-4?ra-sF?U%B;^0uT36Zab54iD)80*%8i+!mx4CwIMoH*;X0DB(dp}g4(l!fhS1rM
zqRyQ8)2g!~cH`0x_s_cRoR^o%8|MF?(8iW?$bMs#LyL{3)udMb<-g-6FmE?8dfGXC
z4ya_)%(oLfCMg~#%6RdTtLJoq*rpQ33rDvIf8_Ys;Jbz;f^&mVg6aXU22()?)g!0W
z!kC^s)e*RT$Nuq>`eUB|doKSq_WPAS=gWDEm-7oRfkfH1o_x5Z{@BZZk|qo-lmAJE
zER6;gp*^3<g*rc+wRkzZ@RIrCCHcoZ@AJsGIeIk&$uu12O%a~T0EZqXzIh97*#@M_
zt|)z<sc^udVas{#$M+&K>}(Q*C3!^Cryr<RQgztxqbSrlKVw(IBze}!bJ*H<?XJ04
z+|lR!McV3xxyyFND&C9f9eU1RBo-YO_}j|&i~C||N1xLd;YG6r_O`rGagkrRn8}ZI
z4wnI}5WURsY{7qlyX`M*UG6LH;zJmw@<n>m<@~}$?^*7)wd6TFZCA9s%iC4w>@;6-
z;pqt%QaxT6sgy_x?c(8#ZR7jV!m<8_=&lxyTJ@H7icZ%BDtohkO|cL@wz4U|<G|eW
zK};b`K}=jMB6s?8esl_eOWno>rf4w^gB|jbDn~q4Ok_}IbYtpak>E5CV|VD7=<CB!
zar=G7@dcc#l6J4OpRnV9#?k-gDGZZ@C;oZ3L}bE$=G71CEtqvK)GxZov{&^~bkx7}
zW#!6`>Z2}LIexPjaX4R+{b1V4r?-EXr5D@_dv-l*UTVIG*!&}|+3anlFZ|E2MZ9{q
zSAcEf-)xConbV)0_N}!u`PqD*P3&I%rL^zoQx3e!O4Rf?I`Ldd;60(E@;qGYPaJvo
zA$(VVuX^EQ&pQ=rDsPPc|1N!e>BaA!%9cmJFaO`P@qZ1QK@-PIy@~86v<_FSmGs>?
z??hIeU_$z;MSaoE2d{}QS~dB?^+P#=elL&Yikn@GEt}`I&|jTt|6@IiU!`UHv{*0k
z+zpfK7oTzbq0Q#}PcBsjJ!Tzsd=GmLuRoRL(><4C`}#NTFMJ*>QW7bjv37!F{*_tD
zdxBZ3K3#I!zH8}Gr>UQxmT$a&?dq!)b426*Tb*j(6e0NH)R83}8tgCQo=EI14`1JM
zq1vo%W}VeBgXwz(Tzc6~Dg<qb$YK!sckXA_{;P+wydPZsm+<q{j=X~c!tCdK9^7@?
zKH-AZ>hrgxntE^9ba1FAMZUSm;OgM;P*Q7h$8<;IWAncsciZM^+p}2d!^Mopott*V
z_41uAyBYo5!}rDc&kK89<|<Em;5kpDq<XoJNw?nb)D64*ZkN5j`$%=7L5CRYRkvWr
z*Jib~tHN%c+?3Sh6LNh+s>5}y<Vc&IyrUvp=iYp0zk2h}6*G6Nc(cuS(bAiebL^G`
z#iwLFa&TKGlNzna)+*6{b^1)<!<kpVC$3W|=hnJm;h)ZuuwwJ665H36w_|kPc)2M_
zonCPCEKi$9hU~oa5B;KY{JM)nTdEG9jm%u5B>Y>@_)gldR1qFl(TB6X8vWIE$u5$0
zkX@X%Zbkp+jf@)u=IE?7`0Day@8Y8O2U^v3#Qwa~_WrF(TiL=1eJ@or7=7lS)2?HF
zW7w>C>QLcx=}it*CVC2025Jt8SAGlk?L4;DP5rt{l9B0wV&g*R{CT=3CEONwdMm31
z`OoP%;55(o>;?B->lrS-E0oQC+-b3L4y(h$yNMH5d`*(&U6Rpdv^e&RUW}-0<s{9A
z8a}_6zXhI`ow<AGhnyv|Hmz6UakYJ(x1{|!v)dGImGFD1Hr9{j^u7JrB>kp2sLd-p
zzH~lsW3Z{-91%0svlmT!A84Fd8lK{#%I#;jY|n{FmU>$cq)Wf9{Ctu3x8DJVI~R)L
zw%sbP-xm9O$8v*j>W5RV@CHgBIHYlL+lKq4GT#lS%)PmaL$<}D=H%7>@<;i5F5m8I
z=e+)If^SHN)hUr3=CLole3rS!axPyGHr2EA)Q+}gfnV32l-rdX^Q>Wt(X`UX6E1aH
z*s(c$oyd@%aWbLUH@@obFDpe0`DZP%VnRHnM)OMbq$kHQmm93y9$KAZ5_f3E*3)~0
z798LAUDTRS*<E7w?7N>AP24(>bK}~2oqeZI9N95(Qqkdq*UAc47@f{t)OUM%;Dc>y
z`yMBx+Z|AOP~?4Pw@KC^rZoYf>M_>l*99FPq~DjXGfGO>C#Lr3(%RjB-JUJWG#63t
zyL;^Si<9C@4tN}V&~uw*|C-K9ze}(6SVShvJf6EJ$AWG1@}Fw6ZITYm*wy|uA!U<t
zLb%tpo!4%zGc4QNqsyM-=QjILQQYs9!KS%C%rCy&`1xgJ^qt0|vu&6yb5EvU_l-3Q
z)tT6KgX8@)Y2NxZKa$xLI9L5`Z@nsWP^fRtvbwFokB)4p^vbiI>RY!rH12h-Lhj+X
zYvB>=%C1cpmzjNHr{2+RvnE~jsnnB=d}lNJ?DZqP?dAP1Q>qTGJALoY9k+X5*E`HP
zURL5)peD2@&}`<Cw%J<p3PmS2Do!ss9sFp`(nQvM+-d^-TpuFS%=sl;-bcrrjCOq=
zd3%RJuc++G7v(c0j7}?_-1y2a=wJFGDb6Kr^RLc(U7L`nyXEW;#(=wpk&VZ>WMoU-
z9azt4{Yu#@XOwd|?aF~R^Qd&8AJH>5?F^oEoBtb=HKUn?^sSpg3Y)8(;_WypikfqU
zu3r42dz9m=io}x}Z5kzivlA8BjvmYTTshHKYui1x7Sn`dCVU@a4+;5P+&Fb@;Df03
z!t0V28<;Izzn#%0rFZ>mpArfE;z)^GOCDS+D+<c{Ui9$p5&cGgqiZT1XWG8lh_4cs
z*ciR&x7~r)jnU6`EU`Si@P$K6^^}Q@hj(=ZZ+?7h+fr#h-jui$$tWGq#%zA^2^-BW
z&Qs6XyMs?q*M7}=ahq997mZYx<?M3NyXD2ZJLDVdEJXv2@2>M@^#2we{(0ikLj9Rd
z%O*`JP0IXo{P(mCALTDZT{!kj*fR1~Z7xfR-pOY^l1j2#Vr`zxvrWajWH+6hIHjyc
zKJdyi?p3-OJ5n#zesoyyhN1ZH)!jF3XMfHS^;q5?zT|?npxeq7j(7M^iyfA@UwmNW
z0~v$gzn;!gw=miN@!S*ROYwT^1!pU)ZH#rkvn%*^t30Pm!Uav!W9gDTv!18NB{;1K
z_`o38Jt-sL<f4;%^3>RBQ|_!}&ziF}UouE|nhl4=hTlaIKAZ<5cmnM{du@<Raml+c
zesD|f{|O>15)E(lB)*yHm-w+}N&4sVFXE-@H*fmhjefn|^RAIwUR=P_QwI*t>&y<U
zIbgQo&7}qB_V%+VPT{)g8{cy7y0G<@V9%)Ot=2|Sx34Nr>%ac&P?zuu*6G?0e(%eC
zv+|{s+9E!i3jgH+_c_uI?E0hmT_@AJob{l`x0PGpvQN}X+5gCV!kdI}t}XI{nrf4S
zBA2DU&nV3OxR+b1JWTps*_o6!Z9NV~n+;D|kBVf8y>Xu#6qUEQZfRtWdsSWByxmov
z*B^U7oAtbhUzO{}_t0D1GNqNTN;WF6&6>tkddKGOd*AaR{1eaGKVUe1RrR;gpEFu}
z=YDTh-fQ%)DDi2l&8`Wx=B0jnJkn3C<Y*O~X6W(Y2k)`Mz?Bl0pS*g(lDeHYT+`fV
zb0jyvV_fX3+N8pqQX_@homT3vHVSg_s=VH}<oBLqdqri$j+;onJ-p+Qxec?ymj}N3
z+}HKE`}NLkk5}ZhERXViV0a_r+3D~Hb4pV_&-TmfI~Ue%t9&8qkmpTFmRMi0MzI-7
z#SKhj(iSTSn>^sGT0P@W#=}Js90{g9>E8raSZOnKpP09$Q!15Xp?lG~-Fc3BcYYs=
zo!Hv=dBV!(Uo1SG+BPedsvol^@tbLfzpy_0_Ef?RZoTAo&HZ2g#XND#SbER*r1$0d
z|IVD+Ce`v~&Jv!fJESwtH!k2_`=E8+cg3Zfn4A|pciPCcEqi0L@2!AE-*l!=Ycafb
zZ5CJX!AoD#Ls??@>|~;s{BAzQ{EM++(TYOVck(wETz|0D@$dl&ZP(T5tk)S|PYPDq
zExbI2TXp|y!vIUun&SM>nTxMzRVKd(5Bzx{bm!a(MFlPg$!Q6tdvlikWRjC??E2+k
zTeD!{rU^T}BP7nA;qooWyL_+t+odRp1%Heh4cGnp!_Jpw5+vUCuP5iG=62i6AI>um
zF1@HX|6yQ`c>4waML}DS9#gq>N+Xeh+kTG8QolpNv0JRY(ocOp*Q<AI)i%Rvt4bLe
z&dmvX>Y!@5bhSCpjYSOcKWd+D+I(Y2FMG$?ss9;wY$^#oeg3Jczw7%=*7KW}GyeQk
z#<PEwY21~7T^(tg|J!eAxNDSA@$7znK#A$+qZR4KYA)fCI;VrDoxG!Y)Is%T#*Y40
zw)8jBB@Q<%r}OYTomp7;C`z71*vH>oqx9+x83EpPa}!rT+LYJsy78xO)y52){Z~RS
zo%|qmIpRCxQjx~Ymh@v9?kp?gKbvug3FiuIc~v<*A>c1F`*iKujLbfI%qlTWM&VL2
zQIA$T>o-jfs<^Uch4kb5o$E7uKJ&kuylvM0sK{42vHv&RRdM3n_eGtJ>y!8f*2RIB
z)?PDm{o8-%S6Jnot;fv6PFkf}<ovS8EPS{k@9Hdv1IjkxbJIAz)6Z|`Gx^`}^w|E8
zD>KBut6%mE=xs3EbDP&7ELHwdtL7ZDn^pHtGGy_!%zeG$#O1iU<D1?EyL^7VYx)-M
z$H9(VTkmACx%A}jHqT+0@I3GT=EoN<_-M_(ILYkxoo{{4{<3em&aC7r;-BleZl;)}
zO=GF<|JG?{JcZ7;3*y%O`jA>=B%mjKV!_?z_jor=$Yt{uSQ*k^GSlPB`~13q^Ls;|
z2WZZ`eq8F-yX`;IommYvgBE=C{LyS@>nvRB9k+q+$JHlAamRMv<BFPmraA11?v?r5
zbD4Edd#Uq(mh77AvHL*r)#7I-v$vjeUi?pe53i5tJ(+m91!?PdN#`y3*Bi-o?UV22
z1$OEO1OGK7-Dmfmp7yHAaj){r+~2p}oc~_C;Jlia<#ew1EFU8e_<o$ZeFuN*J@W#Y
z{t{gUbEA9TJgX0xAIbc+p?mtO%1JxZ*6Ypcj(+T1b3ZylTB-kqj-v1(qXO$KzkQsg
z4}U)QQSfA4$AbF(E6WebPCZ}S@!nijMpVM*WOa;uz^nLa0_M`Leycokj{dRfR(P`i
zaOoXu7E_zq)8xwhS6s8SI5%zg)Z2R&iODO=Y|{NR&)jUqR{J~WV=D9SZ{}MjW~V%p
z?W6f=j%^n>zFuGTw(<G2>J1-$c-%7Q-oa@eQ(P_Gf8?&()uolskI1B~V_mx1%x~r6
z6T22iecB}Uv_RhB<ZmX{$**fq-+7yGjKO5rb3fys({{e)VvZ>KWE{Y-?C}>@zX_k(
zeoitvDSz!MJ3IfJ$aoub_3akBT~0ck$en%7>=Dnma-n|BJNvmkbW-KtGdFST`0d@Y
zw(zu>XD0Kp>;F%jjSKpAQ|F6SWsAy`r6~{Jho3)HULSP+`NH4Z-Ug{}OrPN(Epnp$
zinZkVq{(;qKg;jgb>eERm)F((w+<DxmPDGjK5FAtX?=aR$$fjLtDs0;P=CM!Aujdy
zE2)(k?;}qw_S?<TQ0zYGZ$L-D;}853#6?!WKDgO6bG5~xR%UJI9FFZvoina5b%{^s
ztaeacqI*?*^@be@PL9vhWt$#96uo4(=}blXlO=D9UtCbwZkl(g``o;I{VE?-HjTEU
zF53j1GPtaDMdYTZy_h|HtN-~5Mc-LlQf9^3^J%86a;iQp9TRpV=L^s2Dcrx`F)r_L
zGH-U8v_hlA`uOErb7gMD6y966|D04v#-{gCn_TBjD+_#cz^w3L)_Ha=hx%K&T}cJ0
zQ2}3S^d)Ltc8l^QEYnj7$_wGD*}l{9i_bEi+6m(Ft6LZB{Pt_noxsWj>B+bI4)2|-
z!BQ3Ab=Ipe>}jWIWSjKi`a-sVnaYiivx>_y=ln9;E%_}oCFEJfY8~$_EjxE>tGb8g
zzjl4sdg=Z`4&(b951%=5pjl&^@T4H!2jAEaZdx`)=b@=!ruEr5uKeB{JrC-3OsiP_
zWTjrO=27$5l$&!uJ^5O=>Y<4E-13~{?C+b4IP5}J)YVTAliD%QXs`N`o$R7pckbK7
z5<aQ@Lsj|_R{70AXOzkg?rhF-f1|oVLQRAF%r)nmet+-8-3k@4NME}p!Da#bImMY`
zJ?5V#)`^@yw63aQa`w_a0WYlnmRz}=YTjrVtNDCu=DJM|D^D&dugW~;`iFCi*OAmo
z0sl4&tT>#l_I}M2y+`xhp5H$FD$RIq*3?Z4eoR#Cs@r+a*5UFdm+8-oS3CV<irtyh
zays!fuW#k{s2ht@B8@GDS~IF&JlbCL_q6!Wy?3%d?~f{rwqm)HI#YY{48@8M46`|t
z&KvkDd0CV;8TVTM4}Py}pYXWyTJ5G=(whBdMTa=oXByfid=@%$nf1Thxy52Pty+4o
z^|Q>|-C}PP8nq%S)<eQgq~}62>yiYA(=G~c)a*MFAKH1V_B=hY#LM1}&APPq#$@|4
zSvkw^({8!!wiG^D^`J;@!g0<i=Bs*lDStQg&z%4Hgc8s7o7u)1I#0R$BClyG<O!Vq
zyJ5{IhM&o21+L!jWk0r|&Z2?4=tk!Hq#dX0B64On?=}dz{QX3=%yntD&sTo!IUDl2
zc3bC`+fw1Tjac4`&)9!0tMW0!AM;=9UhYl0_Fu|l<tcf;1^*M;+GbvMVD|m)x+>8z
z<Kt}6O@8rn#dcUNFHp|8(Rc62tV7o(Bs~6|Sflu%AwnT)-tP9l>!jy=JNxCo;j4sx
zpDh;uu4aUE%y^L~rc_w=uVA6u$}e@172#2pYL92j=@pCAO`Bd|{ygPk!rI+_FE-cS
zH}i<O6P@zjQm3+RM(h{mD7h5N7f}g&quFdPEBiNCJ?zcC@I`yA_C2L9+-&>oEV^WO
z&*`4D=iW5I>9vXn=59Hx%BCCnM$afv#YVI2eaF0<7k3^cTrRruBK25Sgwe9!C(iWz
z3@Yf9W8hcXqob}W!Jo5H=Am$kwSd^ROOFHD<sRK`*0~%$_Z)ABMs&Q~A~(y;f(B-v
zx9xSi?=H3Pc3lV$<58*5_KV6D4{n873x5?}b>+h>_O5S!Aqy{EP4<`4y&!$!x$}bw
zbNSshp8ZK!|GA`zN&Q0g>5r{*mp7fVT0Nz5nXZma<c;TN8N)xF?v+RsVyf8awSCdC
z>~MXLi8Ypa;h(m2g=KUkY+3xk!EPEK+Xk6Q9}<2%e(38p`PkeeD}Q^5TK&ITG5x_4
zgY&cf4>C^M*IJewljd@$>M48Hh50M5Iz_L`>ssF3^I7%WlmO|M+Za+C1aEK(&sF)`
z!(ZzjoMiC8r_!>}wJ6p5$)touv5iJoYAa5MR3Glyp~D^UIwm9_wTfNWY|hgqby3r9
znp_uLRj7B`X_x-i;K$Xr&ik}yzRULcWb7*;pQE}%^QOuS58D+F82yff9ayWf+)ZHm
zTIs$#-bJSIvovg%>9H?9@-m&reAd0ghf=40nGn!@^6tCP$<tUjPTPLzG3)!_Mn3oF
zP7iY*9zOWiDr(;2wF=2`YE^tk9!~l8`mkr@le>ay;)%cH6QrLiYE0;hsI`91D*1H%
zhxb8cp%<E-uWVe%#qF4WB0x0W@zY&dzAJ?v#Uo?wS{`i*nB;cjt<^ophpYFwoqZZC
z*Rp@@%S{|lgI*;++izy^X>ZByHKy~M@*lSNFWhPr7!tLEV`XvT(W;v-jQ&0-OLKfL
zynVszi&<VbXZ)*Ud!ux!{*CRmpQo!L^ottWUS3U*)N?wgC=%IHRHiB!93;x6;>75z
zq3t<&uM)%kt*oE?N;YY099wvwpQ}lJ@_~M%8?i3_fwFli1;0e1o@m%kU#p^Lw^3x>
zn`ev5KR(*_&GYYSZzrAG6O?22U$w0ioAdQ%c-YdMt#5w5-^%s=>2lYmqmCz7o9sMN
z4qZ-fx6v>RdH3nw`D0=CIU=g;R)pVky5!0fTxtJN@0sW<)>oE%VTw7oghE|36JO5n
z%3JYpQ~bk2&0e3c8tr~nR5b6?6tBHKo9CTh5%pB##L<E;alPLXAD)rWm&>w~t6uNH
zEgLayhx>|0i<BA`m~#rRdNO(AW+B-ryP}o_$=Y`=J0k6yT>m9{hJpG*<GmaVOxwcl
zJk8pZ@jdx<Mn#ci%BrUCJkxFK9DH(v#SbtXe|z)Nwj|p<_bNGfR>-aYui0#{=7!-!
zPh%ctcHN7LI`wRmrk!OxvQT8#7sVq?76)Q4UGKiKWXj=!=UX;Txtz23V`Y}((g$Cp
z-*q29u|i~4isgZo4{y3`^1tRP(!97}>07;zGjfwVZtUMAAsyoOykYL9_}7k?mQGmK
zEfZGpqp$s`=`SxPcd^~Slpb>RzvDV4+$bf!PSbz>B&Nxx=ilh2mzJn!S)8*u7JuP(
zLch-`gYzAp7anY}l9XtB`qaR0?vV_aKTBSP<fZuL7fV^SU+nnO%y}t8HE9oT8%t~5
zr-Py&e=QerpZZ8m$H4#jY7I{<es%qYedUW53#`msUVY?iJj3m*?cdKm;hJ;Yr_zu$
z=s0(oa8q%^Q@z6qGZIhM7zoTXnBW-d&0_cX)Av){Sz4PTTMfJ=`vvAME;#+*pia3}
zqqKs?_w38-FKYx`@MzIqpjq+JZuRq^i+^<rT;&tiyB}CSPr7M!P{_xQizXM{XKc=%
z^)}dEYWCD4yuHz`#~-#_ic2lGZCYy-A3p76YT|dfxVv^!k6+mEd@hgI;dfJNb>7@O
zl02KaVM5dDFEN^dPj;3cNa&0{s^^@=XzLd7{jTELx!=P|ME~5HswP~W`tDta;<0P*
zTCRxIsGs?qIhkSJ?KgF+tEIIXLic`MvamS!7WeYpnY$M4Iw7JyO{`zy*yG#n2G>;9
z=&qiun{&g$<n5~3o#!=lv}NX0<n30H*joD1wvmTLth_aI$x-FcHNnRMulgx_H%Q+Q
zFzR)4dGWHPOY{Xp<(p#GY0Djq*%)hjow$R~$W%&)CpoT9D6i<a%$eN!dhf-otKJSX
zUax8W+vxkw<ND5r0g6+1XuVOpdDCX$mkHl{7kKZqyYjTiNopFmPV)P8Y!O=~s@^qQ
zpUdPJ9Fsfs@5!h84QA}fU%Rhj%g$I2R>f!Bx)L{+yjR?>n|=Dy+TU;D{$Du9-tPX8
zjUn%~(YDn0qC9%s^HOhiY}gb0<m&HxUp<)Le-yc~#-Zlr8+nVatJlpZOezWAav<li
zw6MBxMb5(7v~_b|HL9E36;zb#RhSUhecIS2P{wbP)@&KMZP5{hHXml#`7gcwqvdA1
zxzYchpMJX<Onnt+x@D!fbZXD$y~?x7&;0%{ZROQxl21D)l-QSX_m;`cdOmZf-(D^q
z{+ViK@tU4*Hx@kai@E!x=tkplotEQ26L=UGAJv`Df6C?*WBeu-t(_j(*Y;ek*SNQR
z+V>g$lTR%)k@s;o>-%4~h+*<Qy+=#Wi)0vG(3$+sxmE8>?3ae*3CA}j)qlR1_u{#6
z>5uSdE{Qd|9JdYIcgKe8wRxt&t$u2SPC?SYXInarI?T?joB6ybx5T5?YVJjq>CxBw
z9Yvm2Mr?b?D8y`fMe^5T|KpR*c-Aj`chV@h?n3w0{HF<G)8#tU-dXys?GHFUwbnxI
z;^Cyas}qXsWBQ!;<eJIsiZ!0?JG;wa;)VmVn?863&Cz)-)F&IYa`U+mKKm0KPHfd5
zp9rUKQjdFP#Ts}b)?PnWVACe2IuY^4O;dI6My6_(pV|;*vuF`(^Iq4xk1pNJn6!d_
z^{*o;f*0PUlzHq9*I#=_&AjyBYsP24ZI#ZP@NMx~Z8AOno7t;FtxbE(UQCW!);aTY
z*TZhUtg6;U(Q~)m;+?puXKCACkIqSwp+>8#{Dj;XXJ$`6RVrDe>Hh4}BJIr{qVab{
zGriW81@Z>HF5GtWu*0K|r%qPOdtD1VX@BXu%lX|tyC*j3T%YkHerCU3)xFvcp{4HJ
zyX<*011dSD&g52nbtq(|&YhaopNkrf9G*O1qH*rAQ>~&=Ofyed`nPCVZgm!C3;Noa
z>~T4;X6BdO%r+bvHVKPf<-V7_GR<tNvSi%6C&B4!PMk_SeImp#jqU4pcI(ec+2@yL
z)}Jp~vM}!UuRR8}Ei7VbIZwHNPRb4{-Tik;s_8oIgAeZRIPWSxx#ZzXyE*e%T^>!m
zqx7ah<i4q;x%bMdPgNg|JqQ$Jo4h4<*T;tTWv=Q0x6f{sTx6d7_?)lV4G-zXff5<q
zKaIB>-QMSLc$0pbW&fVw?yxtJt=n7P9xhp%^=)gy)I(=Arxyo*NRO=3uDNTTW0tY9
z^-g<daK4jsgLCJEXH^mD^Gak|?5BR+&?0iTfmbRkbrYXJ-Sv*X+}9qAOv|lQcxI^P
zKk&X&-1q$HwXRy$i0d=8Zcotfe!X;NxA%rro#G6w0*>~b4c?0bHzi!qlB!cExfJUj
zYq%n}YyEj2`S(Vv9Ew@f-dN-;y0>pNN28e3<^r}9-knN0-ZR<l1#Y~RE>66EZ~hPF
z9lJ{1cX8|X@@(9fCsxSwJM`4-KjF%AAE&9lzgzn;D|f!LXx9<8I}V0n3yx_&nZ^C3
zMCVG!UI{iUsb7rjN@tEfF^YVwc#Wm@!=<}X$5)i>IX&n7h6MJQ8@ClbZ)9+ahnXKU
z+;hb3|Ce$O=l*9qG9Gbl&D~dhE^U#tb8IX7%x6aodKLN~pWn>stN&n%*WUIQdIfz+
zSGwozJX*WWG4;oWq$jcIU+z^unERdkOc1|0OWsUBzK%$-@HElp9jWWe%lGy-w|H(X
zoTvW8d*S6-%Z^8WzjHHY`x$m#=79Jjo0b&4@0}fDI@@$PzVp9Py%JiV;1KeC?H7ZO
zR?Ee&sAZ>ISYmF{^nQ;++oyFG9&M0dk@Q~2G?PE;)~-@HuWjt*ITo)pj{XgO8YOPe
z`0&$$-`CilA3hz$8~gSVXYISSJJ_?Hm!#e)PzX@?zN2fi#m6g++q-fO%{UeyBHGfn
zIDeP!+~+?nSl7HdT338vuTgP*<+mP}EBCJccARdi_J*aMRll-C@qCG6V#oW)el}U_
zYf;xbmi^WWzU0etXqJ*q{&%%s!Zy#tLSN7RoxV`BHt9uDooy$Fg5o{>`%4ox%=y_C
zsdMeV%!XZ|++Sa|@5r~<_c$-MJdDHYM5$Yw)%We}C)j!9-^-@yzm5LZUh)1bqp_vf
z^ZvXY0jI5&{I*l%-oAUo&F>D}9ag^|e)(SPe#zMUGl%N^-$n8H=c><4tF`QSy=&+0
z6){HJo==>WJKu$=m1F+%&*?K9PRvnPjr(;VCjS2_>kR!Kt10ikZ@d2TnV}1pY~l9%
zJkF&;`-AmdF11$O3Fgw8!}Z*tf4Ma$OZD^iK1HRi_r-+gc(3i>D-_qK_;XX4qu@KQ
zkLep<O0*<w^mr6&KmWOb{98Lu)${2WKJV{5pe~}<dMQ;gPV;>3p-lM;HR?=T9Jleh
zi{13qcymf=uIwg{k{~&@LcvWtb}Vd^Fckmzeskl@`JTC_zr1|koAh(nf<T$(-dAVe
zc}6z&EzUPmWs-W7bkq3o<rfeA5(@=4Sj~UCNBDTDhl$k`AJexwdP-MJ=S5Z?e%|C9
zIqS|ljb&{twE_2L^G<A^v;3L$eLcNJ6Z;-Mjd;1i_u;R&6Ti#n*R{P|UwP{Cn$7ct
z?(1t5y`0b3HTm<U&oL{-gTiMvHXcy@8GGD5Mr~@)%6a@X95XJu#ZBycz54W)Io<h-
zd}f=?(bP(vY3{UiPWP;>XN~i`77E5)Tz&OtOj?d@c_!bp#crh&*x%l1t!Fqm<4~Z>
W$(~&^Ijfdd+4F4RmVGW$<N^S@7JWki

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ab b/third_party/icu/data/icu_conversion_data.c.gz.ab
new file mode 100644
index 0000000000000000000000000000000000000000..d60aa92d675c85f95e811221bffc012d65e6c29e
GIT binary patch
literal 177696
zcmaDJmEo+h(|O9yn2ozQ+`TM+28f^3s&Jp0uE)K|?@M?0^p%mH)J$F;GLCIoYPF3&
z_6h6M68V}v44bAD_3C|>Nb<X+zj9J%x$)#(Cfi<L*SODG<q*4N?xyui<*yu4mx!r7
zZ2h`OyS7y{=If@p9rGWo4NhFWrz19W+RU)!!7DC&cab|Z|HcXy<CO_nWnR_d_vKV}
zpICM3_WoA)TMr&q%sg_c`sl+oZ_YhQ*r@Q)-XinSi!6gpQf(zc<+tCyD`<N4*eB8F
z>p7R-EDm30cdMQK_Q};VTSi)Exv17W=fl#U;-2j2l6{-k8GmZ(%SlBW&DMr5x4+>r
zb<_NyFPApxKiKu*Uq^kaF@uic`tFcZAyPF>Z{{7^y(BBH$bj>uYfDgEEN@xLi#PA~
zJ-%_o?&r2GD<=j`-~2@>TyyTHDU&80GcVc@*CH1=W8U6o9?N!tD|vwjCl$K?byW+A
z?7Ow5u>0G_9olS_LPaOPKd@LEoX(?vTDEQD(YoNzO-#O%Rl~TFcEv=jEnGhRu*Kb7
zSFL4=FaPYhUtT}??pf)8nBu1BEthuWYXt}0sZ~8&@cF^+%l~^$&U^2Z{m=1bwvWYg
zV^gaD5plOu$2vvRG(SE~i<z^^R?+T5U-3eLDvk9vPv5fgidB5Q@Fh2#E%}rUt7+E-
z<*7E6?OWz}J4VlXEByG~v-a6fudI(<6LE$sApg(*ef8UqE^FmgIsEAGgWCD;)@n;k
zZ4wo_y*}8k<MMywSC)s9HGOQdPh6b4F@dAz&k7rse!VL<lPjFoZrn6I`}J#?XHxSu
z7fpL>8IwHU$x_X0F7G!Vi%(qJ73~$T>0LHIwQbjyxn6~-4f8|U<Qo66Fnd1zCVs-@
zLQjz8dr6){HeBz@&uvNdiewG$ZtmT7sCQ41keD2cb_s`s`=RqM-OAq?7nLkI8`<k@
z+_*64z5rj3n_bi8jBC%9WcS+cJzw=AOkN`L#Y3CR`;ItWeHq-G!*PbyyMaeoe2SQ@
z&*s@h>}(ruE}1sZGuy26ID?P%RKx4Xj9m1C+Nv9NuDtZ`fX%OGUsimo%bm02zTZ1P
zJ-Mv!vXi^-{#aDFeCn;vmPgx+ZwfLOZ26mT@xX3*2d(wK(%q7=D+PAN*r*l1sa$zf
ziaRZckthDPPSgs2k4w+zyx%0zyP<c!99O8M&PU5*4Hq;vXe4cH*t&9ASlk}ocNU*}
zVi(@{{mf{hd(ci^t$c>)H*X}S2<544<zEo@;<CEiz3@}f<x>|s7jj+e;19Pwu<hZ+
znUDX9Eb8rP-FtIpMZiASJkQ(QZv;f2HG42jVc=Q4>kd0#bRRqGS@VP4w;4H|KBulT
zwfN?|s=Zzzio2=G;M0M;jRA6gsux_gaHUmgs+YS=D|`HApDO1Ay>p#R^3R>DFSN%>
zy~<T8dyq4)zM1n#zT1-T2XCt_IAWxFcrI(>MvL8hvlNg2zF0g%b<ejAOmPwGHy-qv
zR+~7bn$2ys?7a{ENy!l}6r9{A&Gkw@ouDmY?rO#BYsxt9cgswJ!qdmtrFl5ied8@O
zPk(Z)7tna<x9Q*>#>;Q>0(y3vXx%<`f0{4ftCwDTcVu4|KCS=etu&`kg`J<qU#8`J
z>i0dRc75g#iP*V1W=H9qus<SOXRon;$5U6nvpXQgBV4b$smJiqYImnw>bzf0EWf|~
zpn>(oo@zF|Ykw+lZj_rn_l1*Fq*Fxzhw|qQE#4n5Djp2{q>${l)H&+eA*B~5l&;uh
z+`OHevEpJRubQ@z+zzS89jP6r^#%LbbI$YUY<5~}xt3vS^()mJmY+{m75*!VI33yf
z*lgqCx}FA`9c`Ar@BdsWdb`VFa?EsBnU80rJeLI-R$R={*cQYRxj9eg;N-4gljp@Z
zGhb`iY(I6uD(JAC&66qFfA%bo&MztY$31z&X7#FvI|QzVzPz>9D|7df!1hny_A$R?
zp8b76!M`oJx=pOR4sl)zJLmjJc}nEz)U`p+--+LJy0z$0F<Y(lIUdp5EKO0BPO|I%
zKiluHv0kun$=k)NpI5FwVDzai<4vP@i^fsbslW49C$4JKe|7YLQ@D>_wBDD+{*F3!
zI+`aooY^28clJ)&^25R@*OzAP^jNm)JoAbt+o#mE?6CN7)<b)yj_PR#(=WTC{as^X
z-&%ill6@jl`01+YiyPrnt@lpqt=b$tk^N+z;mO^Xd*-dS`uR|TGlOjxbL<Vp-gUPZ
zPgIYK`1R>cy&1#nLn}&Um+We@t=!OA%FdAdP26YK9v9EY`_ytIKikxFCkO92^~5Vr
z>ag^q74w}nHGld&75&7&`p@Bc7OR&&bkgLR`~7X7->3KC^B4n9>O4;Ay<RZC#GGd-
zJE!B5dwIRwp1jKf?+a)CH_mImwBJzZ&g@6EH@Cf6o?2OQCGuA0oQKkz0#9ikJ8zm%
z`#)<=dh)7UIR~57N>f}XCp?ROW!$&YasTceOOL+{N%>g&&Q8huN|#aLkLkNE{@0oi
z|BC;jqK!w$E#=l1ix?{6m*10)JFL8C;&Q)v3!BOWw(+$X>Mx%D`%vPar_EMRnu`*>
z3Zj>B^@R6ivU2IXcy~!!hA%roSpLd&-}fu>in+GP3O<;!_xa}!ez!T<(_^)+E-bg%
zW_kY>^V98C@r%PB3FcSoo3OHr-~9eK_SEF}yptccnZGd7Z~HBrmf~iy%c{00=6q`I
zh3-}^$*Gd%TO9Y_T)|z;u}PkzmCMXv&V)w?+IG$QAvsa{V?%mYf!*iT^{3X}S+uWq
zp6N8suH)wa_A=QxbXP9T=kyl0v{W&AI;n&ED)-uDa@xuH*=67VIYkD(`nJ!pFEH+T
z&FiFRJJ~DWsZMFG`x?S<(m}K6<cY2Yso77KAA30MsD$iuLFo%3Y!&YPuab-HxSOOp
z&G!mSyZ4gm)QSHmmNEvk>{DDm@gV<$X)Bu=4{i8)b=zr$_sexAd~aNpr_0$l@k8Cp
zf0gd4Z?5Sbvu-Z?b~kf<+u8M>7bVD7T5-ISxG?eW%QlYOKZ}l6SpDGB`g-Eo^L1U(
zdp`7?Tgrd#+M8wt-gyoaU5go?KAHW@>Gg|)R<DYWoxA0js_y*gAg^z4*FpZ|t`#ZX
z@|&#-<IAR<5H<U8jKQ)nF?{+11GOD7k)faeYA0`=Hz9rhfv^JASoh^8-1wdB-3_{W
z(<YkVJ@V`6scZ5q_gc;s8}EI$S@zXV)vm9}KC^w+J32KRZ85Z)x9hyk+1hoQzA@ak
zTiou>&6k|AWbYrl&2t4RN>u#b2~140TXE@P<RU%&xTl#n{WkG^6P@RGTIk)yo#6{4
zwFP1#w9fFK>wo%kPKRPVU-Og5=dI<=%2UP9S=zn5J99q!gJ0ik*=BE&mpR=c7W?bN
zzMD%<&-PPb+8i&rt)yYjc@^{T4fX2vk;|s#74t-!={3L1`0Kap<8HxIn?7(qH8^=X
zs$6h`!$N0mpG~&q#k;3u-eD}0{3G!CYy6(PPq}O_45h!$a<R8byq_NwG{-(>iN@oF
z>WyYPrlD%Vx5aLAuHS#9eRuUmmb-fyjCQPY`}R}G;qRLF0l$j=KI}bn>6>urw4G;m
zEj^`d=owkYvCt>&`ZnD@C#%Y0org!Kuiu{-w~fa#*Hl-nZne*|pc8EG_N>@ay<S51
zPu7FWw>Pi!cwge&=<{ms)uyalA!~j|rBA;sps!xL`Btj^_2P7!jaI*7xAHsdi!Xdv
zv}CelgqU7R&3xl9O^%<2rjL%qzY{gNan^Wk4VPWl<m!(f(p{dOalWG05FE1RgNjwr
zZMQGXg7b?lpF|eixuIEdm(%3CNr%kC*|EQ4`W>%Z*RK<~++r74uueki0js@8$FW8;
zOZKI!yW_l8vNY-%>&?^D-~IJKdW^)8)sJ$DEmzKA|2Cy5>GXPa^$ELAO%U>YcqYDO
z&0#&e58O%f(|4aTmy*-@>iyK~V&K=kF}q$&RX5iZzdncS>G#C%^K`OLOIMa>C$8Ln
za&=MS&nb&`Nqas2%<8)N%iPSJS9uJhUk12!&pSP5*1LfFe`RM_t>*0(344Dovb%AO
z-`D+iFN9k*UKZQ#x7Xl$+WH9RC7CPEw`}CtVtrlc3zOkJV=i%zy(d=gu}xoDJ1KSN
z8^c6NwWS4}N59;<<QjKkg3pVnKTixc%bY2QjL_`O{Lxs%F4c8JTj~D?y{3td`j4h8
zJ8hxsbRqCt@78=Hsm!X%#mpC8Y%Y0p(4wO4&fc9juEvVCug<z3(;8a3!A?s3&%V;^
zCDpxu`|}OIbg1swwX0Fc%w550U*s!cQJWnq&os57AAeInACh}_QgpHVJekhVt{1w0
zPx5+~$p7x+^C^uSvcC^EWwRbT&*1X<sh(&@DF3nSjOHf(dt9%t?Pv-Vx#t@&LAL3d
zDaYN&ybZtd&bwD`+jacJzGpfdrXd^Jr`i}NUsanBxnYU4vf*2`+~)Z6Z=dT~>)rXa
z-8uL&Lx>}j``LWKh;(hag<CaWh4n2owYzT>cKSD8&%!>Fo36X(Z+V^{FyH61=U<x}
z^Z)g=shQnO*s|ll&Sv??k~b$EDCV9nZ2jrrisahQn)lUKn#F%!Cb<9XLd`<QXFM-2
zZ(qgxq4M7G<=rc@+uyu7BKmb{pP<&{xw*o-*WSIDP<Zm{*Z)QBT<O2BT3wvR^rBE)
zneB<*z9h4tcauIB)}%jI`XL|Vl@P9Qf0;nUukP7v&O9r#KIFctZe8TQ$aFiWi$8pn
zVgr8_#yRZ%dRTd9!;E*z<;}mA1T}qkYi#<`eDThwwoKkRbDuco=S0ctO?@pNnx~wu
zrTo;adTG|<V!iaGOKap#=*89_d6#_p-rrCUyRyBh?Wc_G?-nJs=zib2@^*?G`{LO(
zIl7Bh=vXja-&eS!;<n#~>#v^vbzl{j{e0Z7{N5kC{iTmKNB7zJU9~?{xP57@b<p(g
zFBuyv;tbuENy#t$dd_jr+6`^TjVw0*YB*>oeIWVC$xg2c$EP_ee{0~+{Jwd1ec#om
zzaCWoj;@P7EGuC4F>2Wv*>d0eiGOr<ubA$AG0))lmaw>g8{hhWyfdNdigmP!;NCM8
zf!7lXnAqMK<O*u*e?KE{(JL^0`u77O_U=3M9L44t${nr^<T&kC|L)kz&ME1SEE5#l
z?OwC(F#h_G*Vp;@=Vyx+?tN9#TytaI?+f#a9N#})aI}3Xi@u)EyC*KI-qqZXIuctv
zFL7CC!Bk19LvNBN%RjH_Ep|Qfa!>v>;~O^?|K0!a#PMrCgsS+LrEihl(Wawh!n<_h
zm&8}UXC7Q@nE7y>lDX7puVT5rzO0|t%d~9gE6>^bT=w?em~S`!me<c!{XI>yvPG#f
zN$L-4sjbu6#wR72w~tI&BP6YGz(jNUhuh!!9tyMw$*-&1);l-wn9-VNPLBSk*mXEk
zqn0+Zb^p$?QR?pJHkvoF|FKcN-#c#B=*I%=g8F41D^AbcIq!2dqju=w*EhR851pA^
z{`*q()~vbdmvxj5KE8JS(6KXok2cvR2(DD*=g_-i5)#i=6!x1p*M93~J)_#?&Jq8k
zX1MdblJ<0ZCH;)UWJ^+t<xFi8>nGL$Cpu3rolv^`*Tju4b3eO;d}ymU;a1cgp>IEN
z#nM=F-}-+IKBq;Nh3?mF{41nfX(PzCE$<UY_mV#rQzrbsHu+76>jAZoy{C%9xI@H`
zEyxsKGD|cllj$$Fl|knH7yfb!HC(eZzb|YwJSftvetmkU=8dazKd!OWg`NMsC8X=I
zg<uN@+v{W8iBf7?-Mg;2Y6dJ!U;F7!Poz5kf#c?Ut2258KAz{f#j5m`BmH-sWkBWO
z=vpb+PgBYddWY3LX1@AL>E-OJb=+-RI9BOi{4iDQz&f?c06$wLHbdcO%of%aQ`Tti
zejL_s|55iM`^o?A5;e0A>P)upl;{1;*3?|T{!xVM`)%4Pq0=1~d!L-`$ZTuM;b=0s
zcTIuE;iZD+@7~N=wQ{G`_3ew!x_nSd$-dEVWnOUNN$R}xl-)<fMUSP2-wikS<*#yN
ziHhtE`#w+i_OsZ{`RAQy#B{8RSL9iDSET*il<Bnxe^vjQ*)M6Lw_M_)aZ&kmy*IP#
zF5QxR|2W<2%<8#jJ6sw0Bsex5n{2N(f8NV&wHXzMEng_fWK}3z{I&g(u|v2oXv49Y
zzVQonqz-7M7}!X%=H9q<*P12RrCs4tYD<B-|5Ao^A7$BA%)Hpmq0@7G+Xt8P!U@eQ
z_%E$-^p-zUuxy%~SN|Vz*XjJZkN%Z!JH6?P+x(RuXL?pYSp4X9^xe10B5yw1-^f~D
zFLnQe#V(IK(ki~^f4`r$a<2Kc11gTQycTd8T+HWvH#d>f_@aPiv1I&qv1eBE#4W5v
zj{Gou`TmhJclk3m>(nOBkG-*_(!4XH9TJ1NtpY#s9${Oz;3jLYX6b3(%|@cCMu7q6
zZKG%JOMAiSzVhse`J%CoJI>X5ysGk2VHKC$%Qo@ueV@wD9hx!M!%|*skrpxElep@j
z(}jSyM^Cj}w*OEZ^ZxM32MiS}KdS$UG=F`haI$VSYn1i{tv%9<tji)_T=g`1Cj9W@
zwa<Dh<g4tuSr5H<$Gao<055YyT3qvPEuCfmKdrLQJD@C+*=p#YshYChK<d=)P0{m~
zTsUPE;MBHh&$SyoHw2xxSNT`v9pE{#nyL8#yVv>L$E=CBcl*toe5CN4V?(de#YHpX
zt{C~)aY{`$vki)>d7Jm>`EN-U<A0ZUW=P&onO3}rm+f@9<mu`#$z$`6JkXeW&8b~$
zx~5`A^==lYW?SuxEz>JbJxKZ_nmBJm@3#+$IhTzO)xEAf*lJTQD5lB(d4lHLsXH%4
zENfZHT&8<Y@Lq;3d(7jtY!+KMdQ<tYM@1|PFuQ%U`D$Sqo7r@ixApVm`Q>CcKe%N%
z_X>k`<f7Em^WRETUB0%|q2qKy`+4un^FIn2RH#V#+RO-ARC~Z$<U22C+D!R77nqa}
zrPduz`M8gH1OFaLr_&cM-H;TEwHLoVW9GE#s~aBe;k28Ow&IZMJA3t~%RjF)&R;QC
z@NWWVg@qCG+1f?6jT8HIVy9W~SL_y#S)pxfC|s%dXwRC?)^yL7#vKb%zkfI(8g8=l
z#=iU`0iP#xoetR~Cg8l-zu=x{aM3;YAhEocMJ_%k#C0q6eK>DwXzp`Ropm~uD`J~a
zojSwqPe0a9ky$3(XJeQWbSO1QWW|r}_p9gp<ksF6=y0>QIjE*kPn>tAUe>qbi$}fh
zigE|dTfX$l=SHtC{~wYL8Da;H73!q3w)8C6A~`?s)sb$41#<ZhR$qzz{B&yiYX09l
zPRSXto=?kdzp{J1Qp~cj6HlGowk7I)SDE|!z4NWs)vBM>p4<~~t>t7&R<HWJIqlKJ
zDf9j<iCxY%bM3Q_%ibNY<X1edes0&+(>1#{EUe!lI$`<bR?W$#8zjDUdUxy)$V=Yj
zV<Oj)alZS>{R`VBJiq+;ahZK`&m)_bc&Vp%7M{6WRrPMi<>33y|F-p3?GxmlnzMA?
z`Q1O#r#`NXRQ`IXeAS%#i3}lPZQf6=nLT<dcdF`i%U10RMmyvJcdVHj`Yc1ic~6Oo
z?%BlkQ{SxiijA||{8GI4eC_jjb1zlr+dJGUXHwnWxjFHP!9B-dx0aw-37N>GwMmA>
z@xSk$W3l_AQty0k;X5bpV;c8e?;Wpml>Xu-w(_&f5r2if;`JqoWhcKCNB%$8nC~uI
zE^oVi?Q9|T$C@0EE$6>&o7OLTGWBN4x$lj}iC*;qPo2NiB~QHcVdni<vtPSUKiZyq
zf3?;3gsa<q)WT~mqSbgJ`n*4d9n^XMJSQRKyYMA>7a`#jb0V*Q@F|k|R<VOy{Mb{Y
zeVrjIrbJ784?UbJ%lY|6KxXisV>3mbOD><!bk8^I-yWIcuAAif&%gZHXv{Ft>NR`E
zFE<e<>BpZBv3!zKtbYFGQpLj=pO3cX&a7eV`feD^?=1IuZ}JNPnaVFp*JXbx<o=w`
z$!fp%Lyfk?T7`4}<Br?LiaipDyZ+$dG{fnEzmIzr?|XV!_By|~cf`M69OYY&<X$&F
z{4UMu9DB@+oX=<fuIs=0a?#Ei=NpuTLpIIr+GTb#MDu-LN6i0}ozm|PIGULM@Y|hw
za6#;u!_sHiN}Esi^{xy0weQ!iYPs-)OPfR&Tt4FeYESxque*i4Q*95#tK8byzW#b@
zQ2NaFm11S-oPX=TzjU&zjQcBJ-6CzcY)|OZnMzu0++AnR9N5lneW75<8oT)ts}tK-
znQT{lwOjsk%?%}A^RJoB97eiFOzY-E7bkmX)c8jKi+UHkzW9Qkx?T3{$8~?R7j%{#
zRNZq=ns39cx8E*si^|3G{hIlSXUjX@s^T`O@CgsN-hN$?z#Tn(<Hgzy^G^z?Z}+Xr
za}YhhE42RZ%N=JO8xDWkRI}yFh81@Y->JLzw8injdi_&}IeVqQeG3(O{+?rXqQk=-
zKaQR1ijexusC>xb>HL+S&hOzlENdQpQ;g>ZgVA}-y@H|l|LStTag8lz+J5Xxdpjr3
z>;tBIINGL!9}+G4pBL!RZZq%no{i_6tzy#{lBVchc$)aV!8-PdtU{}ibK}kDnz#P7
zl-1pP`0Y&0I+h(__cni4`|PxdY2)*CR?~jB)i^jLwQX46VUU+nVYKez>IAEYcLiU{
z9BF$x_af_*Hm4tsaet(j{+&~>AU}!0e7}U$Hh%k4lQT_DniSVG)I8i}-{0u=-0<D@
zw)P_qJdJ6S*x&d$l%LYgpED!j$BqEY_tI7gekV;T&O|T$sH1!S#LK5fwQ32^Kk!uk
zTz6QIMcRArWu|96tGgy}`_5rGYw_60n!U4GUEwP8L>Hmbl*y?dFE+c*zV}s3wxay=
z!t||QlS(wt>6y-dGM`P!KgjI*=>r;1XMX%Pf6nULb-Vb=COo`s7s%jy<jDIEH@aQI
zul;W0*5<xg`mVk9U9g2<eP&)$YT#kT^Sf&Ae(g^F^}ww>c8wu_$7O@^^T#zOr7WB}
zC9baf!9LHe*Kbe$<6qz4mbp}C*(IJ#&UYem$5>|AB_y8QazTKxmAx-WwwUEk)T}CY
znPm!FB;wy&oOfJ$e^bV$6Lz2c)=k{BEqFSU%Jfb7;_U&SCe@@nHT=s@Sr~2YSn`3%
zvEkP1ttU>iX?B|^l*TVvvQFEP#ap%V-Lfs++^v#|*1D@b6h#zT9!*^vSnC?8!ga^w
z8}s?g%htb@X<ODlyW-Q%kA3Y|*!h(2um8TQ`y<!yz8Hy5&f@+~iig*po|%2e>9t4i
z6VL2@3Ev~1{CjAVWi9JiSkI~2S=`pjHo>R)MAVB$;Z-k7_nPDfT~!Ks#CUYw=Y4x^
zzq#JpdwF~5PhAz;!?h}!l?PbFGF%cBR^{%kdXSxe_4Q7@JI@z*sY+L$oTG0#Ve6#N
z^M1bdICn{9NrgSDn<1}9*u81Y&lkAe{dbASXu5rX$7I>oq(ilLPHs4T*yH7)Nvrg9
z8t=I0zMr#0<ntEUiDxgfo4ud((e9tMuEV>92OqU3ZI|CYgPE~!=Zn*KZSE}+7N1&q
z=gQF~4AWn+zBuH0>g}c-Pvaaq%sAuNxa*7TWNI8TBA%b}kmVNkyv_PmqIpVUP1CLH
zsXN72g?`#5rME>iX3pay{PG#T`Y~#ur|xWdU9$hO!R<94SIv$1agq0zW$T=k;x}!l
z2mD*~#PH10Jq7Iims*dtK77QQwqu^|$2^a6@4Cg?TzBlxXIh)N=Tpe(b1MrEo;ud*
zC!Xl5cXR%F+X=ZVnnQIb+nT-0KEju_=Z5g6Im@i&dMq{dTMiY)o|P+G>X%mhMex)q
zhTM-uk-J=ffBU@e^8-P@qm9eAIrLUNjPR3LmOeqx<E;J8xX$I5`C`JouKh}xzF<b{
z$HsF9lh&`uwCvcOojoP^;<6vX#yxMN*X?Yp63LTHvvOV|nXJfdaxj(WmX?g<`G|@e
zbC$#zD2e*E{`k{&$ZOx$yKFjwSu?DBcHZ-osNA!+DS7paOHOf@%OY;Aon<M1{;e#Z
zoY;#w+*3}qM5T1B*d6|^xoyw1<9&Ct?pMifH`*=qvu}meH@^H7Rhv1163Q!u7JA=F
zSD4x?vi#ttg)e>YHGkjl_C{exqY_IpzaP^avnyPsu?ug-Rs3n4+p^N!+4{Bn<dbIe
zi!-#`Hnh6F<|uh6Wp(fCPm$8H><@O~4_@5KaM^RuP&7Tj<ZJYX>4&%85x=>82g`1o
zbln}=xAwl2{wTn1=eF-e_S<uFL-;e)^)E+j+?ii}!MM#i*67tbsTQ@%;@7{_{3uG(
zljqQB^UbnuaOyw(MWl9lz}m<a$A3FSU(nuXnYlgHCsW?SqE&Nh-@7>R2;GnR)0aMa
z#PPPi@@Aln;_Sm)5;7K~M&JEjBE;~e_}^US^*J4SyqPE3eKRbp%k})SXMJmP{#{}H
zJ4vRYW#`>#yO$ndFO2wNEb->libsj}6&KvlzVdW!E$_ZF_qQ)}l-ay@Ugb7v@2rJ~
z@>}PW{o8Wn`|lIiR&Pj{V}2@0J#f?1$4~9HN-5hNG1rzisreG{vio9@vEY@=me~!f
z+2+b#DUMT}Ju!maBs=b%WNScv<g@2*KSfXZbN_6E@ClXwA!0GH&+cr$uwdz@#ELS#
zWl>uLriZVv`s!`Dv**caJKZOWPY;QE?K5Wl&FB2j>l6FGBSw?rg^Svp%mPaHx+Krd
zFk((UEmZ0LD2B6PYH?}E;Uujm8iLZ_*zI_n*aA<SD@<I_*rYRM!s1DrdKR4t-rKGd
zBem_Ud12a;d*OR8PLV3xCGe@$sjMWW$GB|DgwIy1?o6A}^L2yo)?aO0is`CfjlOXo
z-j)*QF~Me1jD_o(>)$ijmtEnRUgNaR^H%NC3ESQ*JoNd4zlOH4*;mCotxN9i?Av~&
z^Ww6?O|~WMXIEM`-7>lrHP5$whjh*URiAYqFL*AKxbRR*`QMne4Hl(Ig%>OR1s)5?
zdEMjDnHDs^+`VPqKDFo7DF!d})t>JVezLP=McN-3)zwR%_bqWg|NB%!({tll>1;bi
z|D5rBdO9{~#$W!7X|C@wF3<FLm7i_JcuH9PH?y1n(?UJp=>=0gOSJf<n?E@8oZ7|C
zQpE85x7v!jX16(}hcusho{O=cySG{5ot0;TqQ9BH>3w!#1}?oDuM1uomrdQXWD@`7
z2i7;PoXecE@wjx1isyFucQ1LxKMM6|Uh6d4dwgoo!nW4wt#zNO{u|sdjARN8xLfx9
zGOOt)F}wGgk}k0k0Tr=3zN@F0xrh|;hFZkGf0I@A`_?P9Z(`zH9qq?FAM=-=>v7!p
z`<~3tqVOMgJvevmaWzoAzv|-lN3KahMk^}6FAKR@uFp|1SCa31_X%Ui<FOidE$8lX
ze`=(2t>n9$QCr(}LwlL(IGchUo|n%qa^yIu+J36KIHvXG>QD3CJa>iq^V{^^?^xK@
z{3Y{r=Ucnt^piWwo~*o+{cv5Dgs!%AV6V@y#5#`M>6Mp~<$S}B&d@10W;>IT>=J0^
zJl(5TuJW|`wQbS!dnZ;2$L~oHHGcN#C)>l^|Au1MCF?y>pWf+GaypdyP=kkaf@03o
zO1(cxZyUq&XUgg?xXEO4@7VG)iW;dkcW;(lyqy#*UJxp`TQBjRY}ErcovOJy;`iUg
zu6KA8_?2yAV(1F%qjz=%pXL*&a58@VZPnRT(RZxxU08VPCZB55?A)w}eHzk=_CG3^
z%DlfH+h%=o`=$v;9!)8IxzEu>e!=}3sa3%<6V=`I9PZ4RHe<f}u60JWGgP<NN@~AP
ze%3YFJf?r%citDjv_zY0E6y)v{N3O<HGHK(Oce*~oa3JyzAseH-827;X~N?$oA25z
zS_=x5;&qCTB<k<6&~juCi|5#J^F^}N?3o*X7P_Z&^l_{%sN_rf<F=4@O2>{Pr&GDs
z{jFoXS!=bRVD{vI7s+~iG)1r1&k6B*WcH?iBI}nup+z?x=ez$nZ?5x#UHfX<)L(~7
z*OUh7Pif|Ed@<)3`<4lp0~1z+lmzilDVi#^_v?+KhrE+>cRJ}Vc=l<7;^Bj<1JWZ_
z8rL)%X1v~a^VYn7Q}(D#<$YTD#QCma+4%|EN~ft9{S?j+xSyYCw()e&-%ZTwcMp06
zoLpCw8^gjd^HZH=`O?oae)a|CYnNRJ35#Dl&spW2?dkK^m((1w_;RrI*ZEV~6OQCg
zD_*zd=za~~Bg&_4UWot2<oBp!vc-Zqb7!r(Y2G?D<fSH0@R_T!a!%)}t8aAFMAwGg
z6Y%BIFTdIr896_a$w6B=Y1xk3-MT)9A}1`k(;!y9SEYP?ZFJE6xgPUp8ck>U<P!95
z&(TEbE6lNc3Qy|ZCFS4X_R9O-v&o?MLT}hB@7jwebE3176D5-ho@mXH-}Z0sH0#Ok
zLHqje?f4j~Xa19qi}k%Cw>|IOm(szuW?k|uAEa7Jd(%F>d%xLencfk7Q{7<AyJ|ZF
zqAO0cN`H|qmtQ^2$oTZ3ST0HDmHcxiPx+R1Y|eGv$B6}}n9Vz0yJ~GcGPNo=-Qv^U
z11=YxYDHKc7=6xL)b`hav+%c0Z_dmwd)~)il1tNHGqG?{iuG=z$1=$ezqszPHP)@-
zRa)vDQJ2Xdt+g&SdX<R9w%0#-j~iui7_XRNveUovsq7N3?@dg{rDAvHtmO8dD6wMK
z)oAW7F=FzW?~glbl*)WKzH8Nl^^cvd%{<=wrR45RyZno9_mqFuRekezKKJB#{7d$R
z3QI(mCZ6NJYW^wVR%J$B<8P}K&*$b_UHSR9V*kUh8;y4vY~FVE>%mDi^EIX^+W+pp
zTg6-Xx327Q;PHI@<hq}ktoIGS9cER{J-pngJx;SpreV^HWl<}|j@_t!wuwE$)@}bC
z_d>ynDR)eFeLnklnilu+m7gjj8&|!17!W%1`@|l#M^?LbGr6U1@0b7J7_Ao+GozaK
zH_so*y{m1EpC7CLd9XBj`nrjEi{-azdJC6)ihaFoRriub;*k#zi0*mku*>}TnmKis
zVs&L6D;V<rei9_^b|T@vz(R3BCr!T0h-YP|<@H_vOQxMJsXz6;G`s!S^QR9dWy*&y
z-O==0Snu37<;y?M{MeUk*EdV=drwq@$fomB;x{KX_36#2nX7Z!)i3VHtPAr_3L8J|
zofWI8JVWpq^Ze?bvifpW^$XkXxptMM#>{IJTbi!seAh?WB9_Os*{OWD%Ni5?)9nk?
z)C7Mr8YkyX-|c<!RHEgl4ez<v|Lyena%yg`RBLu=n@sb6UiD3TR8!0TMpvzTo_ufO
z@>3Q0hYnv=pPsnkucPBUqlHqJf4$=i-lM&V(XMA>%zBymuiVR@8J>_)eOz$A{JO)l
znd<(WH*Q3;I2))ZbttpmTFTrmGk4VkQI!}~--v&nZ`!)tuKkeXzUIFB+0WUJKbXF$
zU@BVHwwx({>z(5#<L)2NKYiJN=c}<5zsr2h0HgTF@&|*0PCdW3@2J!X$=#XEg(e3M
zWwyCi-}$um*PM4(%}S*B9(CViUXhbI|Al|O?z_r2ay;Drd++_^*!#o0Ht#L}$49R?
zf1jIhJYYh7@wflR)BI~?9?5Te$?`5F?T*4k_HLz;$G!7p{rB|*FKJq0uxOL>ho+#v
zv&)=L-_x6L@a(>sSzqmX*VRj<Z_%7{>;L7+;nF5ct^68xF?;>upMGjyu2Af4<<vhH
zR~+2(fpvxLZ@vR}E@Z@IT#1iQV=?*J>-E!o3j5|m@q4XUjxoNQ7v!1n`HN!651Zb-
zM>aNGmuXVjA2>-aVCVhlZ7rAEgKF=7{QFL~HqJ!zrkz4axk*6({R)L;AJx8j-Tlk2
zrr5W*s^meXR!+9QlflmFcPi_IBm^I9x^VE_qXP}{8TX`1uVf3f7s!{dcrWeCpMT9H
z((i;r`|HiN_Ye2lDKT-qdHv>I`XBxk+V8mKOs<Bn;IC$6StP%1iooN_$e^X?{h~h1
znb+ue^VF=<Jkoq$Kd+ioe>wfdC#k0&m(KpVC*Y*eR<T*N;sSDPg16q83rlTdn5NhL
zJnL6l@ucq86KryC$$Kt3tNSr`UVh`5SC3g!AEuw3_-RYPGxjXqkMjzTdn~!0J5%!M
zp&akJh^nP6sRBFBO!;l3v(v`hQOqv(Reb8LfM5G&{e9>$#k>DB^JbMNi<h2@oA$Ec
zR)TT-Cj0X{7=@4ZrwZ-4x;HpP@YCl_CtQQ%pDmkUJB|O%7vpNDec>ni4o|94-NOFv
zJM$6tos9Z=NA6E!UA}km%nHA4`KcL8eNCo3e{6Z;_~wa44;TFPKk3<R)ADCk@cC8T
z@;Z6zw}>|h$5$m5x1K61WeYoO9DiNUYv<>)JZ<eK_Vkr0hsWOVu~@gS*PcP;)ywB=
z&L0%hj?9?qAM_=!rX}9JD8}Zqz7{`|W}k}sFYev%RP2TO?LOSyRZ!S&SI`~(te|u2
zI&SuruTEX4<>61gpf@%3W!9$^KDSE4wY-;C)ET+Gi+FPS_|(;f=9BANc)s2`d}!el
zR`H8$x*r?&eK53G=H>h2#cfL+7q!azU4g!bYxh>`PnqD$GFx`nW8Lt1dycO(ZRdXX
zy1M%0Z^eYd7SSg{7OvuE(>G>akIgNgd3nPn|B9r)=M4W#ayy>fTDSl94cD3LqO-5;
zu8WvydaqK_CY*`Geof`HIe#zjIPcT@)7B(bB23l1E~7@j@T{Sde};6}k4BNq^u_lr
zmY(UGu(oyfR&6W&mg$Abe*ZV`K3yKMv&TJP{<E38^LD<rG!wr3Sa<!Y%I4i$H)o46
zf8BmARR8;}QaQbx+T*A6Z4Tc2sokXiqS4^MZ8NW@>;7oGWU_SFeCNR5X*VN_Z}G~P
z99o)FJEu1L@wqb><rVnOF7FP%8yJ%MpOfdoyfge}SEasPTvhC#y({W-uExJzhgi<p
z-`6<x+R0JWcKzFla=+x~oIm9zEq3U$v{2RUZ))3{eD@z`6f4OWe)3qXENf;)gp=MW
zo~xN|y36O=B;VS$#ip%u{XFZvW;}B)c`tj@{v>K9Z)UOBp@vPLi!HL>SI++!_pDN7
z&5B<Fk=nX%{CI4b%k;i`%~xL@w`WKCgE{*QY%J$$%6;isZgXDYvp`wR+?VXh;f2r6
z=uPe3^YDykK;{2KN9DF|f3oYrk<VML-uYcm^vm0+V}2n=Z(_jXM?b4~U3&PXl;LpX
zIo6pQQcP3UUir1=plE*Yxy-}XdFj>0r+wzXkT6`j=<A~xkIZ?T@#`iz)g7zVdG5P4
z?Ut+jDmLD8j$N^qudN(sNfpL!Uh~Ufb-Q$7%keJ}y^TMZ)w2~%7Vx)*obV_XdiSNN
zdcE@74(|=qUdHj*FXBJ`ykuwlA)~EFt4+k)cHaIw)i>LE!@A#HZI`>R|45tB(f(;u
z)?}l@E!&P8&-n8;#{C`RU7zz(2lS=*6#aD)-IztCpLHKtYdPW4;cKm_Zwpt?zPIw+
zmMwPkjjn1G_D80#x_fc+>uoVB;(2qb=G>DxaB*r~Qtevq$!p`ZLfVg2cJa3Vyq3Z9
zwD6LQ#XHv*yCP&C)z&)(gunNBvP(k!*@FruxtlA?Vpek>{<EvBW#<W-D*mIXvpata
z<$K-eJ7x8kXWqYpr5~buU&<Y;T3MOSc<{yi?>*6>na_>*k8J)t{Y}oU_ZOda?v4}B
znRUJKQsu<vl`gy<YOVV3%1?aQBdAx_y7Tw@-1RLp^f#L?=3o19hwS+ty|$ox$M!5R
zeRAAu#V0QB`_pbe6=V3~QmJn>Z^x(ETi3@NUoyjLz4OF3-RXOc?uf5aa@J41>2`1H
zBW1%s_s?yZW5*WtMN&3s+S=7E&n=q*cEp69l3DX~+Ggjl&XkE3ORh*eHusz|UfRt0
zd&~Zh5fgQqPO{&%&^Oody0+|I_|g18-6=&<)3hR6B`Xsir|;Zh9w^r$v+$4dY73>F
zS1MDypEM^ld4E_JaKBSXRlm&YeiZjrH=ZW-uU}ut>C3g8GG8;<aFV9EhW>WT4Z$uN
z9o3&_pPqFg`O!L`e9h-(Pg@sHajiZ1^2~3Uga6lyA3w11u-=so6E<;+&a?55QIh8F
z5t(P4b|o%m+LUXtz3l#C>*TY46fFxmDLu`*`jJYU%(A}=?go1C^i1@Q4|5B+Qmj{Z
zdaH2RQl}+eH*cJAZECRkx#&Yy@%d#3e;*2;vRiw(tA0iK#Eu}13zy4f7M}dh9GxF9
z)u}V@jZNgAJuY#Jc^a5i-rd=0aMSBvWa|C2ze?E&ZqKF!?ma&tH>~E@@9>m{_J#Lr
z9k{OkzQpW&%))uAK2w2%cR)D*6_<3^X^Uf6oxCc`qH7L!zhyEI-}70``bP29^Fe3$
zSDpU;>fYbXh;^&)-`W_X`BB;9kmUcmF#Sh|p8Wc{kGtwgpS+3PPLG}W7o_H`tP%`+
zKf56)@6v`3R}0>ieJ!}Fc1w)!d#B*_ryphoNn8wgH*+U@{);uIzdlIi52<`T?bzP^
zQty?%D4R`PG$H7cq1vyPSH$#UwJ$8Se>-pfzZIY7|1w;Xe`H&EKu>UW@dBUAwFh*c
zuxC}BSE^YPC?3hl=rWBnyJxS%V!dYyk`j}zdrKVOmoK{Wj$C5HW83W8a&N`V8_kxS
zn0EZe75&83bIddBkAKSf68PM;P<1bB<&^5>Ydr2+2VA^iB^~}vFU(|r(j4}SF?(me
z%JE{o)mL?V%U-?rffk=?96nFu-`lu8^3S8yWis0Ni!xuNaN9Rmoxbp{Wy90V7d5ks
z7oOdhtR3=V*3moL-_{w;?b^A?I^28adpomDS1mO*pD#_GA@yVVwRe&0>jSRWO@6sC
zFyO*k@i4~6>{D(YiVpS5n(%%_i_wk#TXV~Ig&F<4kXUVFSJ}p<(7NT|yu(xH#ciAJ
zbU|Ua+|-B$DRDK{aE~sVFpE_Go1UzH*V%A>-Qaia?=AZS9_J5FSoo$j>zefJ?|Uw^
zoqAoQY4l}J-!=|m8Ixm9uFq~|+*Rk=vh>p<-%|n3|DH^lYo{MjAQYS3^IYJeL^!uq
z;6l6L81WzC;uro0vF*6)&hk|{IYeTs>5I7^oObkYovZ6(QBqi{a-4-zf}zv<#O;0e
z9Y4oj=rH3KdiUUPU(M1d?U#4Q9`OlY@UA}9UORHx>$N|trGmA}!#mv9Je;-gCzGG&
z>dCEloFAT_=6~w{3g??fj+Q!C_N@~<Bf)P~)ax~4wZRWR?Z+KA)V}Yma#*=1ph($C
z=%l&YvY(e$ZM#3eBl@Am;n>IX6z4B?c(TB|Zc5RDk6&-^7y3PgIe6aOpopLa=bo`m
zjoGTuC$P;{bNPcueg`<#aAZxs{_ld#%nsd6$2VO6G)Xl7q&RDuM2KF?|2-RYnmlh*
zyyFU&o8doQLO&paTlDsuG>4-BS^BGQSN-<zo4(R<OKaImhvf@a6z;sx!;^9J)9puB
zeb3+hz*YL3X??19=MJAW;RfH`zs<f~a3DgAFYLswMQ`TRwd<5uc*L1)?@<1|>B#>n
zn~ppboBK|5S&d1lWmMF&vkBbG4EMa48?<=uC!>c>-y6*>C-~}FtO$8_;1_Fn1fQSZ
znbzy3*V($6&fJ@9D`xcOkauLo=ciJ6N$U<c*@cVsRj40pRbR*7C~5b%R&0XlnaI2U
zy=LAzYW1C$W0n8)cUPB6e{Ebjb=UdDu8RLQwsKtfJh|YM)%VvES8kiV=Z|uY$KDMa
zukHUTmUdyejuD$4)9$=wAJ+ve5&Bqn^S*BITJDG2kE@n3b(gmru6#Rl!aokhO&tcs
zI_n=c&c4I`_xM#wx%-xN&-WH@I^lE4?!+(qcMp#9d??cmIl>)NyL<IAr?#e*`-+@*
zx`ap7cnd7Puzk(6X?Ax`K4YG`yV^Oj>{{6S7WbE2>Z!4Q_3~N%nsS;;7w`OZEOq9;
zU-zDIeaOu&o!YbQTB~GWtoTpw@WS(zQyF!qXwKiZGHmzE$Mg7FWxvX)B$*Y@JZdmu
z&uNcsZ})UqRQz0X&ti(x?0LJn9L4?FW0&k;|EjQI(I>a)hSeV0nV~!8Xq)~ke%W?m
zlHfCz^tIMUY?}5xc(lal-j|}MzjFA#@8Albu_q!YYnigv$`jun1ypwX9yydX^~%yz
zVf|I-ewgw%Z9W#hQ2*5J(hX)}R`;di#kSh*J$Jvech&Dn%ig;jcdUM!$FXVVKChb#
z^zTf0=Pn;D^KSR;AFF-^+)@9L%>89~(ua(B3Ei=)YL-2cHCD_y)2=u1a1PgPi6=J;
zqZeLdnldX#a`&ur5_*yPn^p!by0T%7_`_ZP+a>pUYnI-;^sO)SlB?FXt@@m%7bfia
zmJr1M$<8H{KT-Hm>lEf|+u!T;NM9)7Z>x?>ywox4yV<8kO*z|?rgue>dnO&{Jy(41
zo4xWCd6SK29vqO`{&dz6DREn8Zf^O`m1{-(Gkbn8zwOLbe{Zn!#`OK{Te@Zy-mm+g
zXy&Aub6d@z_DHVgWYv<+wIBU<q{r^t{{2XIlmE5nTdKKl-!tj^w)<V;ygv~Um)P|_
zxw!3L|N9`z3LY_q&lT<q6nB;fl<Z%Ab6fjci@tZ=Pb4nLe|YyQsAf;K+5Op43xy13
zN(81Ik_wG)y?uF8k4wer8~bKt91M)exWay3{h83hbK9;zE$F=Dsd%2-WXdfb3tOL>
zE#7Amgv;KqIMJaMb?)Y*{*sk1*J*8Ao9>u4!F8|lZ>~aijn4tGj8Qr4yCx(zH$B(*
z{#ie5^>0OUOQR_}Z%?r}bvy3g7Ohi<Wh1QjY+$Uc&iQn^ZD!k-=79EF&k_#(jeK!s
zbHT3XAy5A#M3jopyY=%1$Bg?T&&*WW&u%<zFWp*|C}wB6`q^Bq!o!B8;^kHQ4z5yp
zH<x933V+n91@mXkW4u;*^jO;2ENfv-<++D^EDm1zAl~u$7sJABbsG0>emMHOB=7q_
z+e4YZ9n4A|WPFcod#dr_oIp&Bq@0K~bLw;Xp4!##!&Q=t)=Nh*J8m&jPnwg%=2I>&
zy}kTHgJ&Q6YO{Tx>kj<cIn_pp?ePMqu<9-Y+gS<{N)lYt_Z{wge%$%fr@b}rM2xn6
zpPBHpdV&88?!O+V=XNwIIYuYQh0OXeCCySlrrz$w|Mh7HzUtX`#N_iWKF6cIOgMkm
z-WeLtcXU~w(Q4bGW0v_p`|}~zrSE4=)LDEtvhGH(?F9dreUo!c>*X>pdUAQ4wm4(Q
zHuodTE6Wtc8*WpNT~gQLJ%2pWRXq8g@VCUQl;6zLEh3EmKKxwta>mV#cTX)#XwxjZ
zc+-*j@0rG4KF8kk*#^w4_W6DnrKL*`lzh8z>;z-y^rwv1cI}sFezkupqt!Wf=Y&6L
zijA7*DmPSKza8Ty?5^}SB>CCfFKn;E4$qWd7S&eDxV^k!yXP*U2Ai@Bv+mQjK67Vy
zIH%6qvBhk!;ez?MZzasJNstJUdNxHZW@%$ylA>|ez5{9%3d_n9o_zL7zH_wDz{mF`
z<Kj2-5_eh)vN=30auK}X#Cu7sT|0W`;wLX7n6|%PdV1M|A9`gIcGYgyVL$pj(BOmK
z%c<8Z-cNbKWi7M3u3~w>!=p^yFVv?zZCNz8#WK$8`Wxo$nk71K?|gf@X5Xh-N4RGe
z{t&zFa<X*(17@cOAKgy5IsNn#`^Z_?e(cwyGkxxcN#<hP;${WA-A%S^_A6X>Q+Wb!
z%GF~(*}PWl;fxTo{iAg@U{k`1q@_E9GDYSd3H|0=&|N!WZ%ne$jjhj19zVb7bmUF_
zX0cl<?X$nAN_|^VJMZJs8E)^Q{m-2(_51%#U}CuB^uz0~JIvhq+kx}L1r7~Sz1@qh
zq#n@NbwJDS=uZ6!iB&;@l`%CM$4?ykaQe_qZr1g&$$Qr=%e_BU=fn+@rsLOdZxAUG
z4D`9ZT|;ci)AXX_r#%%^3t07jH#+>^d}IDpGyUW4b1PdnwZyTAowhi_bV8wBZ&ItC
zQ;XH@`geODy(-Z=r5L$y_cQBv+ccl{J}i4wf7tVu*h**dp9ho*e&u#7|NQJ?+Rk~x
zJKOb`+Vnm(1#Q(o{WQqyWP|s|%4p@UN_sMt=f2v?8}9VrXg_7#y(ma8M3kF<Z*Zo{
z`8QKvExNTSo6q)vO=QYd`4{~Hm!CgUSz2}f!<#+98PNxGcdhU%3b?zx>&Vjt;~3}p
zH`(@o{~WNkfA7B=Gi5KOT4m2LSug%As`=eac||Mt|C+)d#OFOdC2wAra>1Zv<Mm$8
zlil2l7aH5ES5&yg@4Lg0)O`8bOP)KyR|SI0VyY~3?tOX58{C-3bTV|~nvYF!UR=Eg
zzcL@?y(hxgv}2prL5;4>d+ykNk#^7C%fZ)vOunb(hpYF~KWi?z>dd*SJTchhow4r=
z;mQZo)E3G<W&hN9?9+@uO(q$EdLe&{P_CA#ZQbmlY%*&q7T@T2?KC^<Lhtg-LV5PT
z2m0lh8TTF1KbXMQ@MS^W_gPanwHPj6p!V3B+1c1udse>2Z~Li?F;dTF8D4uhPyc#n
z|F(yTv+B=n|MByN+T&Twy}XYfJ>Adp>NbzqJiAYJN56XA-qiknMe)~4vz30IG!(d{
z^!y&z>fXs(Tz<SN`{<s0o|?byw`C@1KJszN{Ic}I?PD3aCpPn@sqUH4%I$Xb+0xAm
z*WRf(Y{vgs>>h`Zg3dv<P2n-`AM=XZ9ou})aE(LY)VT#&cj~fB{3ln(WX)k#+VaV~
zbMd15ulJUgy}NS%%e>UGMGODB?>1kt@H4M${!gYdKefvyf4=N^x!<hWXi*TeS7H^%
zGS8YXy%$ceO*CI%px`rS+wPg~4xIes745X(*?+|;Q$Ouln)++QRHj#w^A|3$m9WWA
z@eO!5qx?+C-G<pe%nel)OVVfYG;MUhy@*+(TmEU}_GJex9(td*Tq&T|tGam0T!#I!
zAr+FWQ95BBN4B<n7f6hZ<UV@yoTb+DC)ZW-S#PxbvJfz@3HxPq!jAW0;6#bc-*eNV
z&R3tkpOpUh@K@Q)Mvd?1Zgne_Fa30H+V8B*sY0>WczsWQKk;(grRIJ2oz?r8cjcdc
zcl_sy$xRD(#0%Ul_~mxU`TW9n@kZ^IdnJx(2K=qJJRr6sopqh&Lc7ObZwvl1Pic7Y
z;mIq3a;6r`M~UIF|7Y2(Y5VABd!g<gpY&U+p8VVVhmY^MTveyO^WdjKS-WqlZ}Wb%
zv#Km9(oXV7^omH=6Q5tAsIg+Ep6JI_)+^r4v^16~^kOmj$iVv7=<!09`0V}btuH+L
z&-u~u{(t|C8#b_2#H{FS=|1<%=&HtHk2uRKbHqL7B&ED-owIk!=eXtxw$qJPZ<jdR
zd%U=A!(N4${I<g{m@iL$tR%WC%ErK?+tlDr#BJ$&M`p%u+7L8NF8)S@HGfsB26uAy
z>=Wva57zf*ue<Fj&OPzPHm0IeVlJQbr3704e3ze))4J>2Bh_8MEcs3~u(|0QPg=A0
z${&w+){T>jr&rBS<nQ{Edi>$iBvt>V_B9^`*QMU9E4~|W%p#EgP0VbQ+b923bqQ43
ztvPy0?~=E}mB7sR%hIE#`FXKy>fj5><!AiyQO~E=ptEoKId*GtKf}itKKC(4*xK`m
zDJ+`lRTm_Fuj0(s{Ct&||EDS!XJ{?_8`7$GpE+B#e$R%fJD9HBXezd!dws3`yEUHs
zf=`|a`Y5+zZhdi*i`@swRpNqSC+%0=X+0^e)N<Wh%Q7_P=G2wK>B~*+CZB(iprY@R
zx%^r8@!fXkkMJ&-R2^c}nHp#JtL#M@^SvAIzZNajyYei5*P>r%r560^xo9uoby%g^
zzU{+xg}>{K53szMe%LRmapUFnV!A4y3eta>yy*@-x~_c2!_)n8bKcjs+rNJmelBKi
z>iVg1hB0n>IsXpnT)iGDzVxG$t<-&Cv%I6bHC~(EQ?2Zs9vdiLCp~-b{SQ@q?-(b(
z`V<@g%{@b?YR`KwfoL|hrP+6uv%b^*Fx_+7`|_|KVS7K83R>O$VK+N9w|ZVvs_NmI
zOc@n^r=t6&>bt%+B_`|K?-pg<^;J$gLpLnIapFe(Da8jWyU$PBcmDH@yNeblO#B|v
z$oP_<ZO0S)i~!C48Cs>=bZ=bewtTM~_pC#<py|H&2dQJfxm3dUI?L{C?6sQR<}JXX
zuvhjrZ`9t)4!QDLYwrs`OOExm+rIp4VqW5DuJ@bG<9Kt|O%W)${O;PdNA5!Z`E%J>
zg^CYLY}xa8v#s#u`VF@KyLob#i?nnt693iOu%PBy($)_b12pRwn7x=79`sPB#p_33
z@`@Pct7|V_G17B)Fn=A$^8Ez=^k?VRt*klIaZYK&xhiX$U))#UNX5Avnm<u{%HE##
z)7JY7F0)*F!~D*mWY-q$T?>;dC!OS<EV=)-!9uORJr|Y+3d^iK(;LF%9{oZ1{mhS&
zoNK>bm2T77a;8AygQU0I6Z@cxQY%-fE`1>zd8OuEGyl=o?atX34%t~6WyP*~oY2-9
zarlx<6}#Mp`EIX#>n&znSngKkwm&mlum8=wb=%V&Uj9;7DKT2-R54@bd`-o9@~eM|
zd9@Xo&OG(wtlGyf?7CS;c%Pg;=qcL&v9fSh_vD-xivQeag$7ltY(0L8cSWCo-|ZxR
z-<9_tZWTz4dS-WJ+4-{#)4m*9;ovK^QmuBuoqYE9?viKHKU=#eWhC1N$b8v8F<f1L
zIm4y7A^+89{kvm-RVKmI^|#Oeo!cIKd1il0*_`9Aaqm{oi5@AP%yU*O^PLm<<wW4^
z6ecc%cPz@f9R6zD{eE2h>RrFCYN-3`v5PIRUUhckJhgv*Jr)zsOHMykv*GWH;)l(r
zc&2_c{5bhoNpOPeDK4J`Q}L@D3}PvNiYmm+K6b5}t#MU<sqxgVOX~OYgWmk?SS2^p
zN3bNQLgi!-t3~n8kb9E<r=~UJ{zzTur1**5YTJdhsvtpr*SZ6)4i8(rI6tmp%I7~?
z>mfU9!%4>o_5bWLGFqq99+fh$U<=?~(ATCmbA2g8dRxfMp!MZ$%Q`g|$S!-reJ7VG
zLfN~O0RmNp8Uz-lEMw7N(O_A@vcgFF)!lp<K}+_&-xUilFV)Ggoc{mNE#}##U#7oF
zPgu<qEqdxgzEUoW!1=@kXRn5d49bjdOg$_T-OLI{PN=CdJ@Ij>X%Id>hk09XW5nzF
z)c>VCpPzP}y37;0{ANy%bk8^Co9Wxme`<R5e2Ztrth+BwoP+fP)|7C5%WUly)eGAZ
z@i3{?do$Bjukh8oq8?_ot`=Pvwj=6cRKNj{NaVw`*4d(YVLKuprfFAPzcqi^W_#bA
zb+cCeH97wEvY(10C;xJW%P(&Ks$V94C3|`1Jpa#zw!Z6m`@U{D`qbr)m&U$@d+mH*
z_PPo^VidWOaCt?2Ta;Aa#N{hXEI;jecc|j5FmtHmdiSSEGKb9y{-2rm#jN?J;GZpt
z@jGmj7CCXJC8|#`$vmAgJ@eQ5b@7!S&ecWl)qJJA{!nl4y~lgrKjyp?-I3<{_Ohz_
zvYVIowLLx%{y6fn+3#&N>2|TZsz2P^y2@tuw2SedEqk{uisyb(box(6$AoQ*epj$8
z{r2(R0`FtRN21(kZU2(K<iROFhn|)V{p?NIJ+~*6J}AuR`;d6*_QcAciMuwLFRod$
z_*b%#!Q5}6u4(?Cb|)z1AAkC8Jx^P<)ZK}vrrAZf-s14JU$pN>=l)Oo{NK;kEq%S*
zP}@viWuxZLNt;+tiD}GUbIN4F%dauVjP|>#&8&Lh{^|G0@*m!h_ge*N^M4HYJ8+6K
z!Z`R`M&k582jvw{=xj7=nJ2caly72Wh;Eeh#3CWh=S%!IAD3)8WhXsXqwdO_Xg1>r
zo>luUKRB<Lo8oA+V_Ai`r_!Gy|L2n?eHXKRt^fU>e71SSJ(dR5^Gssf=9^U83&@E-
zId1U0)3eF;@l3gkseC4jFHikm5y=>o;Kyw?M{n9yRo%<tPrR;oymobYeAjb|){51q
zF1@u1=8U#t-<x#$+uunmDyILNqp{>v*dd>1+Yc|hsxGl_M)%=a%Tl%DHBRn7?5J}{
zTu$>kbJ~pk^DjUC*Tfop^Ky-m=(2MW1^psIiz|&IU-2BPeOpoB8To5ljeSeZv0nv$
zA14>iJ9si7!S=!S<)1Cx9)Hk(-1_eHuW6NiPw%Opf6!`cyN<tZ)}r^V)$^VC_vI8T
zPPO(c6W*Ju#OQtN@-yk2yT70Dt1WPFC~n`mL->qs)|=AAO^fI2-Z>=s;cd;PZJO4`
zm%c3ia=`EVo_*#$s_AzwKFK{NJ?Z$873Xgs-#nW$`@ym^Pj1xLMSsn({ba9peED>F
z?td93_xd#5pU=$h+t>Qz(A65nM>Do<OO;?fzen`KizKJwj?;B@+q@I=e4f8-RVxYp
z*w^&%<L`&j^OjFaVLWj5Ny)P*FFha4n|seZ*>~R=<Fl3e9hya&C05#RS`W7wd}^I8
z`il3~-hOZ4!^!bsD!Cya<|<ek_4&nFOfuBh3Rf!D*|PMdOgBf`4QB=O&t`uV^3J@v
zo$}-R``s#E?loL|=2s{p|IoT#Z_4JDoL25L>?}{X#5cC9c~?gyeodUnE;?mr-fw>Y
z#&`U?54-#1<`zlX6#Hy42;~cV_U%LDjQUAW`@gpTH{8FzP2^H++Vj)9?A|Wjv80Ii
zd6(YV$SEHc{=ankaol@(WgWBOodo0RxQ>HPJYw&jwQ}gVBzUfGEq`XpHSJt)<<&}^
z2<v5H2e0y<bJ`J>s8?OTEK#DwJnI3EL|)kMV_$pYZ_lnhrrIC+O2X$;M3sHPwW~XC
zyk1x2xnH&Jl;_c-EZ;sUZIIR5=v(+c@=aCuG)}|%WAXdU|HRIVeU@|DN>E>GVSCuW
zsisXa7gC)+>3*7FJ>~u6ebVh0_m$?$uFU^f92`~tyjFepCtkiUJ5p;V_r*qk`}_67
z-5c9o(k<NTPkO(|70>58@cZpb<~^HCeyiD<r@#N7pp&BgK)3Q%_x#EW?R$Tm7qzH&
zt^B;t^G;m*viHK`m8m=WXYZL`GxcsM$GMvIUhUlx+qF5}dXI)rd1dq8F40`@o6^>Q
z3%NR(uDULLXw>sr^!%HxK5a*q3Tp+6ojuLJ^ukLsgN{vhpQQ44IlkJF5hI@ZI)1uw
z3|raEA0PC?3$M1Oi%+|M^6>Fcg?*<EZcnS+?-42P%QEA-SqQ7$4DO9*W3#<}>^6Gz
z!y;<>ne9y1-$(nt-uA&=*<k94C+S-0QKfwwidgDa#KuiMJ)2GV@tKP&a{cv<4n5S_
zyZQ6#m<Qh6J)B>kO08PGCHmUdg?5|nrEKqXk-EDs=)kEe`S)*F?$6}xd;f+d{|t+~
z`<1h-$JMJ3{(4v7d(PVRR?DC8(3vTr5BskhvuzJ4YMbOQ`e~h$@Kl!j`(u|apXrsc
zNXRHxX#VeC370;awg{{I*!M>04*TQlX@<Wo*nTeZ{qylk@nQ*y?keLt+4pNxH;MjW
z<Cb~(Li3;a^ktXUoeux{g<toia&yXc!{U^a$DLHNZr@|&+4buAvbX<?bAL2{`@PvL
z#{Grr{3RDlwe391CFN}rWy>nU`4WoOZ(GCt`q}=g-woddOU!-t+Us$Z;Z>fz>GkQ;
z=ASzKp?pm`e_3(<!TF4vmenMuRHP?c+>?}C5V<4TaNg9l^F!CyoB8j3ChsWeC+Qsj
zHcj@__xB=qg%AAMn|Wfr<eaEEe;ZXIVjJG-@3h{{`TkD>@A(|l2?@uG1iqjBpkE-N
zQuxuVcwI+dV0v2<PmJ0QuMM_m9!H+oJNdeT_Sv;vhgdAl)-7D8{itN|8F$5Cjgy_-
zEmJaDn`Z6U`}s@jF;`o0H%+EfFW+b|?%H*AI`gLf`aai+R?V7~6@HKT50uQ$pX1`7
zu+;8VeX>uz@t<m=FE{y^W>@BjC9kijHB4DpJiX(KM9<q7ja+e`zubBx|DW}^KTBob
zKFwp-KZg2m`rgqsi6>xx;g6}-6-TPieA^OVG(EmdV^hS$@`<J1b)R3^cdWlR_tM6&
zwV54@K0ZnMGIhOG+ugdqp$xOFc5J$K|788roBJjIt>k&u&CMjvv-yTZcR~Yexy<cL
zcZ+XZFXh&c(~MiWmrLiE@dd{BO?$h0)J{FVSRwj0G-+c?Ip_OHdrxaj`>6FebX$1q
zQLcNUdyJdJ0=5_!@Jz{%?>u=*hwYW<g!K}CwE`C1`O_$L^pMjOjrFx`URMGZ#gt0)
z%4!5UcM7S#a!mbjE?|*OgZRm;2U8cibPBP0iGZY-RJo>T1T1P3QU&oBxbQ;wSKOt%
zrf38fy_~X{N42Dr-@%i?t7)N%fa)GjYX;SW0YVKzY%BO3JPoF>ENW4I${ZlXqiQo*
zqG1xl6c%Ss1uv!mp@vBb8k`_~Od!cd2)k*a3WF+-ya-s*#ghS~kVkdmZ02&dJnpth
z9#dQvs4%Hs3=nFZ!~tSUc_{@#*v_6TUP>VG42@0~2%A?`P@~hqlf`S%LKPth+XSSE
zNmWqe;Af910n-+n$nmOjYIr($GI>p!>f_?c<Rug^i3cteI=^EQ%M_IbDv}`a1`ylX
zQwhROfoKy7n6zekCxks|p$a2JuT*%}o=H1TsIN#C>QVT6UEDmuJwxYH$(v3?7L6~e
z?BBw7yjsBB`gdxC;#TuL2LnF31)kV3o#&fsbN$I<2WmEoPdzSK_%^X#I8fuJ_69A<
zJzi{^netlSaQzfD(AjxI`;OOzsX~84H##kwzxnr(MK@Rfp55it*1_YFqM#(L)TOwH
zL6EgW!@<SSMMPoIk6I;3E$?bU-_3b@CQk}v3hR2yeRM)l!xW|^BF<ihOQv)j4q}bi
zn5=koo#@NJ2m8bh%rn^?d?d-}i@0Rds$Bju>$$Cm-{y05iR^hH6K&`-{ghI`m0OCF
zlO}DS`+4Ow6Yf*lJ0xyzyrJ+QK0mEBBsp@`jRQ_GD{JMxn#;OhUF7>+I5MlMs`}#2
zd-{eO_H$GSMl6wkzrxI3Ynk8CD=TMM#d~ge{dJ$q;alnfZF_%a>;LllwK`|LfZN@3
zJ8u<Uzjy54x9m?}uCuP^juWpAU7O%q+#qf36ufZt3I{*dqF440m;agl(EM<I+y9WZ
z!ai9piPOgUp8~}fxSyG6)%sez$)U$nPfhN&x&5Tv_MH-&wMs4T?yD_4tud=-qx-RQ
zUl(g7ALBjK@OkT%nS2jkFzuh4`}=(I36cKS7w@jS!B^Jwp<9|mnc+T1t@)#vo`90X
z6^Gqq77G3^b@|q<EOeFQgjJGcUxD4+qsF1B>!tTd@83Rs?w@z}CvQyr^Ka|!8wZnr
zC+3LnZ=XKbWBdD$Q};N}z36qbUgGqT_xIL5x$Ct!`sVJbdqITz_D$v5)gYq3AogZs
zy87}*8%|$f6Jw8?Bh!6YFWIJxS^wULM;lIORFporDjg?VCd<2?Z%@_X)rZ$Det5O9
zzw&}inKW-c-=50Dvk&Jjet5RgJins&My^SWU%TG&p8RB?`H5*gj}xWtfBF%YcFany
zZ>5cheeBuSlBprei8az&J!VO|n+6@*9@u^6+=~@%?&&4#JT@C!t-5$|rr;09lCrcl
zNuNEk&E<b?zd3W~hu*EeZZ9}yUaoH2Hm5NsZG-2`;F@DxvmNK9|M8o7xrQxno}+jA
zgLO~mO_qF^e$UI~@Ee`)SI<gps(X1W^Z3@kx8L|kUG2%#O^!P~=Y~<*ww{%`$$6*e
z+%Zbq-t$s7x$g9w2S#Z-dNlQt`%ce!VwASCCsQwZ-sw3njM8@Xtkg?hcY4knqqN;U
zFZGi5ou2c-C~Z%Vrhf9d({sKUrR_cQHvdfZOyT%rXQmr|O+8aPb8`H#H`5LOrk<&v
z$=%S$>TsNCK{DfooeVD`7)o>)ti%}Rk|k`Qy#GZE$OLY2V~`mKdL9@vzdC!v&`O*^
z7G`2G<Asedi%^6M@}+C!Tb>opT)j3nd2ZK(o;QV6Pe0nu?vlU1JIdv_e(=>_{cLeO
zjMn}i5`H9p^f{(~+3i>V2d>4scUhS84mI7o!MT0Gxm7oe?UashvUNJp{&(iR&8@c=
zH@`VEP4}++)9TA79<RB1B4urVRdT<B(dOuNZ*+}MeoCtQd-nhT&wuQ$HoG6n)|Xt!
zyYHL*{OiZBa_o5esr`73)`g?|3l6KES+_>^=Vj41v+|o{{dsd{C%Aib*c{RQym0qc
zGuv%ux(UahzPQZHmr#4q>QHFR?N7Q1pSi-sHl(J>cYi3{ZJTg<dxNfaMC^fUGi%oz
zIj*a4Z(2TMbohou#@k|663)}|oVPuTKCn&n4Ug!Hd(G2o4N~3r@!p)3zp$m4b$7tq
z4{T>Dx#Hb-f`k~gwQuZcxH_|TLpoGowf2oo4Y6tGA6)FKlHT;g;`Ae{(`Pzg6lI;^
zui=R0y#D>T>0$B5GAplFe6L(%ziWbU@73O8$0{D~6SjT&|K#)h#`{eFuQGj{9#PNz
zN9>30hv)~}4}b*sv)17h@aHPIzJ{fKcem!__*WLc;!hadc^_9&Rl&^^t}G$&ImAF+
z!1%6xoX?;2Z{B`?@gv;iQI+oHc!omRN%gGT?jM=HAhkU1e#*U~pINf931>LZ#oWyk
zyHX!o{=crRXlYBo!l~__H`H-I{@=)RnfvE1-ao5n{fs|TVD0>6ea_SWTFw7*IbPqt
zI`v7qz%Q$W)%RX+nZREA(O91QbmDZWgU>(wy!URY)I#<9mi`ksYIwKD$XdL!X>z}I
znlIsY;Olb?YmAS_xRmb5J*TWE+qP40R=!X~>+yH9y;oFLN@^$GbAR?X^Pc-Qb@^xS
zGUv_PczA>Dr9YiHtBvcGieDa5n_MONInO9w<&A#Zk+>r@85=HtDcW{dr8wo~`JBgw
z_Bk;tQ}12BQCwxY+wXLl{O2cz-?L(trutod=Kq#m?R+8MCz<2Y{@pVR`;0rETV$OO
zJvnD#Vd=D|k1yKGRc%<k`OM>+bLO5s|1+b!?OWBghvzj)D_sBECgs;}+3?`b(eGQ&
z1WVfIorvpYI<<SJQGe;5HOD`HI4-}v^{?Uno1cSqCeL^HDf7K%?%n6sZf|VA>)%iN
zIm6Lq-}y!NQ!C${{oZ}Af40^6TQ!%je2JaL+qOG$9ecWr{9)OGT_3GKY>eWI|9;Y`
zf3xaij}sOr3wwkmSQH-5+qG}@&)CwxzYUule}0tz=&4=jwrcmX*E;XQ?|gk<@<QbK
z$|Kjz>Sy1$`7o_(6XV0heRtga;?`s;>@8a^vAS-XMfKG0`p4VLz17}_=H>5~c^u6N
zBBlH9cFb>@94<NG*mnN-v}LpI)mc7UJn?q>nU^26uS;y<kGD&c>iv`&7B_#t?7n{|
zUw__ynE%?z2)}EIacqBHKAhc?YV+sI+Yg8Hjr`7?%(lzkb*96!yYAxZ+UHvtXE^58
z*gx|Xuxe(@i??i#Z*MO&kyyFs=e|SdJ|FGaxZ<-^z|p(g>=r9&C*9GsKPvQA;Kr~2
z2Y&9Ti(9eVX?^z8LNN}<^zB>f5?`Nq;}T|X-=lfkSvE34`QX&`fB)U6vpA(UMU*>H
zxHs`v)cyh$*5g@|kNUmn5PS3M?(O~W<G*P<UUW;pVWrVArcbISJ_QTkhR@3=_+|4(
za$nrB8up&Bv^z!K2Wt1#Jhg3NT~%;m`G(sk_px^uB@3iYev@{qj_b7Ie2yEJ`@=TN
zM7^~Ta7*?Qb7o6S5vbhwe|v}V)7<|?(ek|O_ilJz;jsI<#OF1CTnY|4^c`oH+^gBP
zspWT_ppbgygIWKd#NO(c{Pp(k+&xTut(P_LbH2Nve)pKNg7xvd*AFeYYOO?0DIb&O
zmi?l8O*CtD*@E*0T?z`DeqK^$SX6)c<6Dl$*)sPp{g{@1%xK@a(~-AZY*$;RgsE&+
zn`!D9V|4yh$~CpkW-Gm)MJ_zYRr>PACF8lGuhs;l*Y5h_m)^Q7#?JaRw>SIYH3nz6
z4zJPIk>%NZ$UvKC^S%uw24}b)uQ53H>(QlyJ9_RySr*zon-5vM_V#AKyvE=x*X1?(
zy0Si-51DBDY~Ht}#NaH~=QRfBe|@@i&_?^r=0kfXJcqJuUTb@^pI)=!4A<#3>vd#x
zHXkz5*4eyoQ^|%iT(8$`IQQ$-rGr-5Yc?OUdcE44{k0KudTZ&m>(ltQMJCP`-4+=i
zZFM89H8<==T5ayjglP4@n|91Oy~cjJ-L5rV;)Or=o%v(Do^{&E`fE>r2YudmPbYM}
z_PJe8a<b3uda-8pxm|B^UZ30a;l!p-9qYXB-g=q(aboJ%iK#y)mj0cX$~Y;Nby6zl
zq*C5Vse+R>=xz2s``Sw7w3EtdFO}0lDzBqdPA93H&QdvD)UrG)aLe!Q28+ZMo(DbM
zC2?|-)X7aUCpXER+@)}GlhVmeDknFodEVMKb?K|r$VsWOlS&gOrKWEBYol7EwQW~a
zJgfWP52E6w532bKuT5C^`P`3HHotQYZu+s6o$*2V+6%f}fqeVr{#uytZ@#>RXXCHy
zbqRCgckhrAc(d%d{fG5kcinS>KCX_vbNKzjPZHZE^6oFwoEpFT+=Ua)JdS^E9p3*r
zd-1g30%_I#tCN(O51(3Z>%*{Lj$>2DyA}0s+n@jZx|)09k5?NNe%K~|W{!UuzyF$E
z-P<)eDw9lpUzam$h!1W(x7FGA)&1X@RdsI@FLkUB`kT!kBHpQD`8PX#!w*v}v+#o}
zL+ai)vivX&e^Gk$P|}Cjj<)N+T%CUZ*Tssut7i^>sI|2CTiw6l!1|N@$8P65y577{
z-2cRfP75uoeL)j-1oD3DW$%A*{Xp^xBmY&i9x46$zNUvWCAc$wcRbU6xv4(eBkuG(
zdGTmr)d}aDnj3$8e-YS{Q~B_^TykUl?jS2cZ_D4;IZC&G@zu24weOon9siXUXYC{F
zV<Vnimph^%Z20oa`mPl{ZUS6#OSXUA#P{dr`*?=^40Viun0~N+;C>)}K))fL;eS5U
zi#h52SJz%R`04e+&($;HAOD#(?M!8z{^#tu@wX>MUXz-xmAW$MP*lRo&}mtzE5lAj
zC9I5`wyJbx<o0uVp|69s>Y2X2l^DC~+S07ptZQ2@C$6;832hfWpS}EWP-(;}KJT-M
zZ%aV*)xwpQ>*n@(UoDh<y!6?QGAq5%<4a%dxFU1iZ~4+!TSDeen|1lt%Pp6#2bD%;
z`DHJ^98?;$>YVr4%C{wPtIl~}tz2oDzhrO7-07=62NlOz$4*;yI;b=<OD}slh+d`l
zdQB8mFg9yl_VU+3rLn8ld7mwPTN1l!-Rqt>kl^j0(&$xr-e+^)e%W6i`v(yY@PNr@
z`Jlb`pV#{RmtM<zumAtM^EWK0;7T9@{{<K<wEJi8j&uJkoO@m0XdS!oWZ}D<Kc1Lx
zHMaTkz2x}W8S(EJYCc6bKH!b_|6b@Se8lGM&$Xd-+&jvS)r6g&ZvD@#?zd3Qo!933
z??i5{_PBMO>!2*(yf0T$o-BPQC;xM6b@?UV$=AJR=BwW|*`2RGw^mfW?ZM5H>*Mb(
z3(sHu@2a2C!|eT&?#0$FuD9Klt-fFVUIX)azryhQFXDGk^t*H2ynp>fMb$SO828JS
z-I~5{pSlQV(!Vc%ZoL<Ld|i*R>-VpD|E|7cJ9PaX>+z8N!82EM1=yB+zws*mx2AX9
z+d~_7ZO{L*{%@LLo$j8Q%bRuU6vOS=-BiBL_PBH1{P*>BW&z(t*8kgjf90|DGYgH*
zBwGC{-+hZE{&(qxcdx5|U0)X%SpF^a|E=|tA78)6TrTxKcEb$u_4)hHok=p^^zq~8
z>yK~C)Z2dEU-$PXmRNYtBJg&*tK-hUHg@~}>HdlSvHe5-!~2K+PdxP7%l6&bsc!-|
zeBxNE@%U{=g8I?ThCIhJwGQxIikolsp!deo!ei1`1?`sY5;L497vbXi<IJL)KZB<(
z%ul%@c=dtg;R-REE&f`MBXj1he<pa^*Y<<|ipMkWhvZ6m)LSUA9G&=aweGx++#72o
z_H3D_oD}nc<6HZ@lk#r$H!dDcx2!s5&}3hqD1F}Pi~G^&yN|_w80+8CXMDE!;zXkb
zTOa)W@oE=)w&irW^8$JYm5sjs;Q6<qh0(Z=L;q^=w+7b7iF0pgKF<7n@5>5-nib4D
z+xsrG%yRP%o@cng*zws%AKQdrPvvuG&Rj3C$*EZpfB%P0#lz;n*z@nsIaS0Sl$P1f
zr!Q9jV`|0GtMg6uYf7BHdYB)~-Zn2lQBM8Lonx2RA3WEr@neJVr1ZBY3~Oo>Ck5K>
zi|aSA?B(ybGyR(xaQ@b<8VQ^1{>cZIyKUYaZ1?;eV=0UA8?B#@DkRSDStQq$Zu6z<
z)})O+J!j?nB%<doJMOb)&5!I;znnE|etkXT{O`!j&)G-*W?q`EuV$S6t-ZF-&~L)R
z2T!%HR?N8e>hH_zMTd9jpH1d|DSmI}W^w+-k6SAH=JY;1tkc!uuJ&ohqEDt<{^?X5
z%Zq(*c)jDJDxo-@_$!;9|9-W)-NcGnzD{tO{bYkIJNK7<XZLlSdsKSmZ|M5tZdq~r
z^rz(dspK#1StIMn9Z+u}b!+93`E2#Kl5bCBS==7+`eN{(#P@S+U+<nSyqP^=vxa)j
ztV0~9ui3N&pEXXGm+w1uz3Sj`smaGLUfJAprgnD9mgV;}+h?Dukx5hD_2~Zm?+3c&
zE?+G_IOosJi_Z=};}1P{*vP9~NY;O|J#VGW8sqGkd2aPm>7lvT;!oE^{mGmgS^cWg
z^yH_SCvG2H_`Ot=ci&+6@%dtkvCz-Q_cYJPO*;{(*z+cLrOCCWhTJnuvQqcB&fm{m
z@xIM|{bt>3wRU;l(YMazHES{-<$1W&+x@f9&DEO)_x1cZ{D7Z-g2>O=kNO4gO^W_z
zX;dEc?RaX+wuW8LXMCDxsTX+U+>8Fxmh*Ql{NTc{;pd*-GKq})^V6Kq+$jI$;=fq6
zEqnf|&6<`yJ5rAYPTEys_cft|T`BERrnI8a?wJ$$t!oX|M_T=~=sR9`#P#$?rfL1@
zFOE()c6rkl#krpg<z}9BIsI1SyxR?iM}^Y5*G!D1zs<B(ka#^qzhcg*PbCK|ex2yw
zkoTzn?7!2?H(zf*+qH4m%Qs4g`jRSA($*V)S+_QRpZ?OPA04hR8ZxZDwPJHm`$6Rh
zxf#p-Y?U{pPUK8z*=_UawDP5|jA_?Q<i2e)NVj}``HerXhM`4t!Vk$cGo5o2ZPq?s
z$?364Xr}r0J#IC7CZs>VWjX(O;L?RRB*SO!T-y7**8EFTY2}*DcdJf)+Vi70RUq0X
z&)8b|ev%KHgtn!ee@Jq|Pn*8y92dE}=4Nmjb3OQ(v~UM&w3@qIsf~|v@;rm|b7qch
z6SunG`n=q*Z;$P#Ire$XF7dLqG7EpJ{x~!3<U*^A$uT>3?RedE@Z9H}J@*pR3)Drf
z@veHjXyV_@<EF}Tb)0F+uX~y!+{3v4o?iY|rn0(Xx1h<!Gj|F*=SNI`vn}OQWo7Ta
zZ=tuY$JSS69M_ZIEqyGr@U<JuF|$P{zf~m3HEcBAQgdh8a>c$gL5DxxIeg)p?1Vz;
z!yUfaaa{i_diT^GGFp@DKKJ?A&xP_o1&!x1%KpAm`fW?__T!aPEI!>j^qNud_~XOh
z6zb*t>+`&Ar>{8ooOj*tIp=>XSS_}azg?61t*`fi&cB9B>f7ykH>*ax(R8V*iJLuH
zu42=#%g*T>r}Q(L71DH01-sqzlTvm`ozYmIYRAXi=Bglne*d${>G{uV=A@Oq;ST;F
zb@trnLfO5HPP57<a&$lEnQA$&?D-5q6XO>r?o1S^o$YD--Ewh8*>?Rr$=tTff3#LK
zYkHji{Oo*h_=n>~=ib!n^*uM*^m<n8^f%sXjczTfRg~_Zeyqjx+4Bsac?azex7-b1
z{L$i9a->oHS(Wp5BHKPzr1*J9TwJ#=TB7q~#g=uO-T#^J3v8Rzdwg+w=(OdfDes=V
z>pz{dnRkZpw@c2Eo33A!{QO0_|MnTVwEMp=*%=u9zL$Q_;alYN=fycL(~fH|Y<XHa
z_vYmq88N4cORrhl+>I@;TX*_|{?=m`udYj<)31KXcY@S;_xU~1a{be{?YBu-bMBz|
zC7s1{p7TDh(2@Io`v<EH_rEs=Qf09d?R7V=+UK&fuO#hli0?7Cw0kBDGD#(2bDr1~
z-*5e~rDyq^zN4Of{RcM`6ia=TZS#&7?zTVo`tr?`*BtWx5^C#>*R8XBUb*=9z6a+I
z9|)9JWxKbPwQHxfW8d+`r}iD~UB3D1Up5vyy)QB*Z+qQ8zP($PcJzm2`ZE9ed*S<@
zELzcbC)Mqb>7mkjA7%QTzX`lAyWZB$@~&>`xu%wnCa2HDq&=JC_RMPjHpWL5=@k!+
zK7U(!SDAPER!OUQ((~p@%5Ln)4LNn@?d)v%f<wP&op_uXYjphQj<aVnZQkBKaAMB$
z<e4*1*R1k>W+mS~bLZLRg=yD{?fjHKUteZ9?;PK@rJ{@Lb2E>;iHp8vP<1^`xa8#>
z%h?My7eCKw(wl3o-d9@oZjQyZrNz&WY&>^vuJCcI=sUSma{}Z04_ydswb=A2asRed
zo7{h2y3PJ3UYHpAxkl>PTjux>t9i#Jnx8p!D8n)8z&Wum-hO{x<{Z?K_wT=$FD|t+
z^!4h8Y0k}yo#WGgan%*6zORmG%38Dh*o_B{DPEIr&zvc?Y^o;XDNhEa!*1<U7+M*Q
z=2n1~fLvu1ShOpvjbSPj-b{ad;$wqv2on#B1c!k@g2Dj-P6dIa1Yu@IHjr4YwaS4P
z9z87*H;>FXwy$#vcsYnq-w~INPG27ukLDFS6ioyZlny-c@M(A<cE#See}B}6=M2Ao
zB;SAhS7Y10142u;1r*C3lrrXuaw(DC^5OezZL?&zLc_(U_X|Y+uVY9$cxg$(+Rcvz
z#V+dJDAGxr{$JeeH>BykpW$CU&w|gU2WQMoSi@tIaq!x!RPMKT&iu*zUO!`EO6|6H
zJKw+7){uI0`PIK$XPP<NF3f6OC%I8v_ZIgax5k|NpZ&9b-GBV8p5wvQK=HTwdv_K4
z)rjt%ZCKV^Yi{{y!?K%^XD1{*>-n+%-If0`4sUm?lHK_C(#&^zM0S5wDJ%7OBj<L}
zq-W!5@g#HBqj|G86d&HWTj$TB@8Spk?`AksGj&&Og3YUEZIK3ZPp^KLc%`=D`Q$uV
z^OGNrIDX*#5hB<4FV5z-{sYeI&7yK&+kXk4iobmPmS*|p(pj#HkMEo&cR8wJ;`*CQ
zeWtq}ZgKn<{^iYoM~0+#T}l5YCC#f#3RLKMw2SkW$cftTf2tCX-#Po);I?Dk=cPiI
zlP|=j-HT0=u{wM3mB~4_s={E;+qO9tef=EFn>(i}ik$g*B=4uGKi`kU`DHsLznlqh
zx@>nd`TNb%^c@$!cMJHlEZgw=<&m1oU0DHVrsbMz&%P@izGSwRlvSF>f`vM5bu-=m
zzVo~L{@12svx~jHN4&hf@Tk;=-_Hb`58l~(X7#-5Jmp7~^}aW?u~#LmD0^XcQ8)6?
z3YHy0{o?$y!vqVBSi%e{)?WCM>ih8N>1A%q?(*fO7+B7(3Mja__z&+a!^^F+4lc=V
zO+06;ul(cQ&G}cC?6ABYa%S(uvwzrh%$l|aJXn*o(2U9S_=dX<#yMU^n{~Q=_UyaB
zxcSU%$F=SAVy<1Er@x>j@%ozY?Xzx1pH<5XI}y0xcJP_vqkq!Rh%&iV8()3Im$&=k
z4(p{eH=ecmvQI;Fl~K3R$~58d&CFWwf4!Tz%lFzJhO;RimnT&!%&<LKlYXZ9kc{Qo
z<HFNjA3u9JU2VmtwI4ZG?@c_rBEV6h&G(1?f%TWqJZb7)mHYd^*CqvP{%cIXFL|H!
zxhlx`n`2$sk*h5|QJ?bT)}Zwp;hmxVOc(kqkN+#Y(V4%IJ>JND<8Q_;*1!5<@AjX$
z{N`_iCeuf8?t}3<Kk7DW{5slnL!3K4{!Q7|r0dJSGF_dNrhabr)i0_>qB5^HvE)wZ
zIl>?x%du_R&GYY8CfQZ03;jE^yXlkZ!i3FRPWRags)Z#_Z=TLq8o1*>&ovEU+u#0I
zmcGanXJ6~_VG`>)E$%tayPFsN-<141DgBb`%Q^99g!k0nn!J8hz@qv`Z+=gI^5g5x
zH`SZgocy$;g^6XJPPCcE&2xv+wS(SxZd77*?b^HI+oEG@TdyrhyE%*T&93gaWvflO
z&V@z_O;G56v~W}Q3+Z14C7M_2l(HSJHP|pFI7LcdJX;oKq!irq`?jrdfRg4K|NeJo
zDgA-j2iJGK%5n;t@FCWDDKBr%iroucoUgTgv^u|a-8$~gjTVppNxWq_6f#qRzhKdu
z6>gI_C$H63HwhJ+zp(7DcWc<XS<Hq1w>;+%xN!dJZ1d9pTb_3@WjV5ntk*BLyKy;#
zL7+xK@`rNdR?Ri7`_exC%4UvObIdR9YFe_X{q3xO?V`u8oaNkpYi4&Ady=+{(Qkh3
z3vXP*`@%CO={i0?w|3iN;l!<%bwvCAmu+F0wYT9D$BhfUhubt*KPfG6+v9Rvcg=yF
z6GRQ>2u<pl#Muy$>Nc-O%XpQ`0(S0J2K^ZAh>x2W>3zAUr0noLYT@#=1+&*(=)SdM
z;cKx6Vp3<hOrKfxWrarVVg4~W%*rQh#W$^*soEyRqJDEk{ieHVXmGgP;EZ=#A1fPC
z{ib<e(Cvg-c05srZ|AqO$V4}&@3hjLwTnOQ+SNUYzn4$7`{gk+Xh;36*9rcMY&+RB
zZ+*@<x=VD?EXhT)_;$?dT0hHo(JZ|kvoaUyWvxqcU6s0;ZS|tR*IK4sJL8tqG<DS@
zmno`UOBUAuXno~=Qb%KEV@uN#Hx-6<E%%9RFLas2Ij&z6@8;TD>d(U8n9kZZH6qj`
zY^r^r(v`+~$2-hN6t*pK=#1R0o%G*i_J`~DF5BnI&zO?3UrDyDx-nz3dDOO)`_}Ka
zz7GDazD=+?h&RW6>xFuWFKIIc7?#|8o#yBqb#P{0b>SD=qkb37=M+CKG<R%DW3%C5
z^mKJJIk#-xtkaqP7f$Wdk67gK=$5~=-bMN6TkPJY$6cGR${ffL$gH(iBkHG6>2==z
zro3-|7unnTKltaO?02PTQ3u2GO{M{jE$bF<pVYO7v9gi%C`akr9lTOrE{;tXH(Rb%
zXymm0(y6#jq1$)Ty|#BNF4_0n@sx=zo|Rnnf6EK4gIp}y0&gle%yCm%xcc7j<p=u;
zEIhttwoZQP`EA?$DX$hruW{J6>h~Mozf0Wy`|xCZ`g&8x|9i-Ld#PK!YV$=oqh5D}
ze^Xzqoss*!Zrum*SDshaa!6@p%w|2sX`vVS@JZyu@Q{brArF6VW;$?9Eh7I7!}7l_
zC%ddJ%0_=|-Y6`4pp!*8QB3)_<b*3*g=T*{_=Da0QmK;TR4$*@GdJ`s4el<w{8HtX
zhHG?8^7a+AwQ_T17Wux);Lh4IvGZ=<HT$>!9^AWrgI9hpgT&D^J(aU@(Q_7dSoYlD
z7riK=wMh6&keS8B3ODaHfphb>v%GIoaQrzT&_Kce$Z3uh8SBr!-;r~rViv2av6iF%
zGaol@t}IPKPOX)Us@hpA83I0TTFClh^P<gNaksABX)amaxcQ}op{(+@YaO<xQt8K^
z>8yPv*nei)6=tKaw|={CPTw}G%xS~npIf<wrs?16&d{GE>i;8e<|NT`-~OBkQ;>d=
zbiX@QciPtPcW?0J^~JMlz1r@6R?hVFt?NeIKZ6QSEGjzwWM@m)V-C^fg0E7va}6Gr
zg+I&;FU&jO<+hKdA*1y6thsM4>{kujyWpl6ucjiCb8GAENi9d>%D78eo%1|hCN&<o
zES0%gH!<Hgt#PZ0vZGk2ijD*WpRSqaNwzh|t({_|SyrtOJM`f7=Ltt1zr1>`P5Fl%
z(~a`u+B@QJPEI-YOgZvp%KFKoFPvxQYHZe>va@>X7ym00bhEduxv3y}I!Y&a<MyNX
z#F$G$w6A~2<jwxJEArvB3&O6GE_4g~v$7R_6NnWl(_Y|mVE2STg9QmJX-ahq`?m?E
z>ukQsS02&icI(O68GCN}9f)fB#Cld^g`lkHmyZHpE-fo^i`m>0YSye_^em4pH$`Xf
z&(!6SZ_{0pY{N^=eS5&O&8m6rxf>I!9bFx_cIDchi|9Ar_OfTK#cattLf0gW;}-SL
z)!k~4cy!H4i^UoIN9NtwqrYw5?ZB=7G6jz`rE)zxdOcesR`oJB%W09d5nFr&*VTQq
zx^+7xfAZd~t&48h#q^x$Dl9#!vFgUWn4W0|u8UfWT%T>Y<9Wj}ca=9CjRIvJYq+c<
zGwcJMgeLW{u(GmVe7nwTLsOSfmsJBVqf!pL=7McaQ(4$u*Csd}Y-!?T6<+0ahRa}P
zf)rQUlanWyd(OzpKX~=6<W)@r7ps=Y-cUZ3s+1?e4(6;S!8<OAOz>iK5BE8n!s($P
z^z-2AZB3ja4Eg8wP59n_&AunWF<2~PSy^e=OV9PUr&L|#d-By^TWn2?Nov3*o4+zO
zTW?H>4R1Q1Grv`8F<0XKu!nIAA}_t!`c_i3gz0LWs|3IIkJCkR$KHnb<gnfj+qgy{
z%GtH1ZJx`N{-!3u1Ih<npE$l|U2C4XH))RclCP&!mnSu7Uz@ygud&%ar>4~cH??;B
z<od7rd*;Llx!bW?Gk$R`lbhux&AH{O+mfW}l?l%l25%|o-RiEjrn}Dj_xg8{p}JY_
z^{KBWpLtySO+(_a%$kY~pN>s6pZ)%<nRKvRSg_nT|Lzad-&Sw45nFiaNYB!5x30y{
zGt<8FX6wgi-o@ve4AQ~^1lC?K>uuA#cV_CzT}vufGID-m5pj9lTYez2&RdFWk+NCg
z^jQ~qZ%ay5>|qPr>o_U<j`{-j81FD8an+0Z($z11t<wD7Fx^-&hGS9N0(P_I(q$cK
zc|p?_OgGyu&GLn9+lIvK41w#?EnC}k7glqca^Je*vp?O@`Hir@PxQUVuOfTar4`my
z{gXS!|4~5KD|~bEma~hjO)tMP<vw{;VB!YtmuC*IjaZh{`ugRW`#0Tp%+y==kfS)c
zVFH_OO!MN2rh>U4doK5gJ&x_sxRER~&rDYOQMSp7D>{z-LS89t9|A?rXmo^qJDJJp
zB{F5jhbyaH{M`<At>M*|UX>NLp+BQ}cH-g0_iXI@%9^I<Tkzj~u>0DBqHDqrHt8R+
zbu`y0yObE0u(f{0$u%dL5>|b@82{wLnbT`;dVZK^dt-6N`pK<F&wSha;PSkXrU&yb
zoiX08Kd;B(+5@F)4<1~7AbR7$m#YG=jz_h1Sueg;Fge8e$}G0IVTVoqr!A;SIoM&t
z8Jha%R6|(61{M{Kh=>ge+DpnETQ1lgy|s#E^YmIXQ;q$zDjlMnt^D^rd3z_K`F`}_
zqls;4*&g%V!-Qft7#bc;*<7HJ8(29*KYhN=t?fIPmHpT=zj#BAV7i>nk&G?ZPb!(H
zR?mKuYp-&#M`-ph`_s2>br*6=Gl`n92)s$n*|y?mqgl(xO-mI12+J*6YSz=inaX<c
z+P<Hr8GEXlu5n4-GB@)MaJ36GV9vVt>%QuPe<E$$)@|D0a?M6eX6=7B|6fYig^xOW
z9$3W~pm3h4Q|sgU56xR%tgbz~sUhTSh*SLhgG{U0s&6~b-@5X4huG|`=VxA6IzMgI
z!he$2Vr>7pZ$B2XVpY;i!8KdC9^d-*T<p@;o1gQ-Uf8dR*eqqOo#Yv(c>4j%W6Nqo
zX|Wd*bTmDrezC@%KM=>1<+48K70Y8g9mh_VkR>fF{KmH$^<-nWWm;&YMOa_IcGi-a
z<E+9BiPoe0PU~?BGv`cJy%Bb0GDFRx%67N5Be{R4gl>M(`FHa-|E<Sm6=T~k+V>~e
z|FROwT{Oe%<!j@Im$}lbqkEh!v>a}CYw1->xwl+hBYAPliF&c;i?%;F<8fitW~Iz&
zCpwRuDlNOn#kQxXV%AKCC_fK|WeaKoMRZmyUQpxO^w<A_({0m5o*mO7%FpN|+e?3N
zDbW-<z4*w&M+?nFc&^C=x2L;Ic=xAmC-3dd)pmb_ukYBFGHuef8!vU7H}D+fayMS+
zy7)&%{%+oV|5s!mOj$pP@pRhX6JaWgJ?<HapAO*aEk0Wk`{a3U;yc$_S1<7_@?F*H
z$Le?CMc>*Aj?RlhkG7bw?91pVz9{f)k)oQb^NUWl+Q4Snb6jdqJWosxw0kT*OXo#j
zZ|<RWSGdJQT-sS`1-HasU`%EAa};X$Eu3LF(R~8@+|3i?=BQa@yl9X8c)ec9ZA;P(
z5#?nsmgp|#yd81KXx4*{dkZp+UL09H$<OEd6`$)@bPD2(dENZ4$bFre6<A-@B5l20
z;7zrOYTPFE*y~q(`mfk>y$Lkj?6dJX>np|TINkG(y%S9nJQ#BLq7JbwOo}$}<vw!E
zO{G#-zr#T{W2Ww$3nGyxTc&gRPj=nCP4T?Mlb%Rk(LnzTSw8728g&k?Z!0qq$;mj*
zxn-`CidG#{ulnZ&rx#_P&dof_u_w1m;pROicC*{lm%muHc8?_Ut?ne1Gg4_Ax3g|s
z@U}6<WoIGh(*Lif&HN|8SfDeTEoHO+X%T@_8c~fIGm|0`U707niR^kkJ7w*QT}RWb
z7Nl7%_}3Y0QgmbOO3}jAf&Ch@<tDhOyb!&WDy^}_beaYin@w+p6r;_egh{;YoccGt
zZhLZ<U1F+g*tJYOW7~qeX3x2uc7JaQ@QIl4i^*?GgB6#X+^pGbXL}QGEl6Ck@ZiQ4
z{l;on$1ms7>KH|5M78TPPK|x1)A>;~El{%~?Mcs!y>B~=uloGo5_7Ogj+J}0oUXr#
zK&<2EsVUVhKVC`t^SBs3@0xV?DAT-f1HVI?98EWzv`oobxMSmySDOty#1gJaC3;sJ
z72>n)QLX3*o0+gNW`a^i@7rxx&#aoJpubA|N`CeF%AA|Q($l&=HiRv8TzyG6!7Xre
zpVmiX)<rkZnMW`Ak){{Hp>$VgOHTHTsAtWsD@twmr*?f~Irwt2sK>W!2KRO~?VG8!
zZ07Dz<5aKzE$XxP>S_cC&AmNKGcw^!&W6=9KdqeUk=!e?<z>j9H6Bv8`qCEP-M0P^
z<7&tDBv#Xh>Tztx&aiE1C}(s~z96K{<#wfU9@nlc?t;UoZWmZHU(w;s64UOuC&|w0
z>(zWcq?zmJq#l+T$0TI7?K0!h-Lph$&k`NSYYp*?PYsoBXp3JJsao*D;jrUgVNN|(
z0fCbH2D%{<J~J5i|Jr#(m~oT8+jsl79Je}WoKZh)Fu!9$%`C5_>YPrlZu5HLS$zT>
zjeo^Vo*eilQLF5><`!9Xmp79S-twOw!e<nIdYSeC)vDtmLc!mHBX9L_eq&cOc>4at
zeM6(CpO^T^I^24_z2l;~%)ALg0cmF?EgX5i3%@d3Fi*2u`^~x!RqWjwwU2BxFFi;W
z-7a=x<NnF&7XB=3E~lnx-PaCW|7Lel%GBg6!D9axB@;eXCVAiZ%AEdFX%C-)@!Q}9
z;+v}qSA5*E`egN`^`TB8Rw_p)9Y|<skYF%kNIT*jqoBCMh^bxNe@=`|;jOPn0<R0V
z$NI}0e#ff)DI_F@anHhk({4UGAR^GRR%LF-+Y8g!(o!C`tQ9ijE>YUkdU((Lj-E43
zGg-Y-yc}8?ZbVIRWWS*FvD`>>*-TA_8;K_;FPHb9*UfS$(6~*sF2G}^VUfm-RzH7Z
zzXwG#=JWCDv+$Q#Z~H59=Z4~M@1Cu<G_O4^bePFdq4-vx@zI9BaO<Zvvy@Iv+VQlv
zx^xfc5hmgKr6qBvH6~wdWBPuwslBx<R!qlw&#wPZ=F0QxW@+3CxPIrDzYV+M!zxa1
z)Atp{J(`R+Ze9_Vi}PsJG+e{S;XD1npN}dGNj@Daozta+=L_qLwfzgxi^&%6Fx0f$
zmdfr@;(2%e=O&@r#-yuLjHW%bnOXFsLs_-YVdq}fBaNHROL^G)DesANJDmC;_n+Jj
zv(!~BKZH4Rau`>gU|2q<rL#W8=B!5Ghb<2h8&#g?*)*h82Sm8bK6v|Q*V37q^RpAW
zv|8R8=;yVpN^6@jW0CO=c9nui&O3EX?1m0a+u2)Lxw6-6N?!b{c#-NhZ*i%r4##Hc
zX{k2;%UDk6C53a^%(^eH5pQvfx!&D>YGzv0H?fGH9gN)(#!6)@mzUktw=v;*KQHQp
zRANH^hWTz!JT+!~I4ySn=uVBtg^UV^75`-336Q@l<9)d4!=yRP&N788e_OWkW+q6@
zT*UCK@JXML<jezd2FVHs92^eLUmU8?=ppoiLC9Do?ZleKxo;lH1(gJGF>ctt=ZZ!V
zlRMwnBYe+Z{*wNAFv7*+@lED$p*O?7DyGbz+Pvb;tWDWFlpgiWxiE2#yUJRZ%L|lK
z_noMbx$4}><NLx`rg&EEEe6$sZJgG^;mr3ZPI8U55YA|oc4yi>OJ~OEnGTjG7wwwk
zXUJ*e`suT|=mF_d=IiJ4T6~=G-A32zz~%qUTpk(|TZB9{CZ<f9?DHsX;tp{K>5eJ2
zzR%+Vzlo(>J8`c;vb9J+Q(NOq_|MsS-C}Qr`6D(gR6Vt?$wWv~sjxKSuH6(#r_@vT
z-!vUNP$G7uH>lR#)bH}6n_t&9@6Ori^08~Xi{`l}5_?mP|9mrHI(0+ugR!Ep`P-IB
zGgXaWoyygY{IJD=Ki_4-aY-JNBab8wu0AKUWx+#Lk7n&}GoRNVd|$mrs?39lvE{@R
z502JM<wZ*I7W$KBD26U$nIYTF5+lSYVzzRF!tVB7<zSzaPyCG+y_i4qw{zQNJ2{tS
zXB1AE7(A^`t((ftcO(6&#wz~gC!akUP6oR_TD|_-LA%YhhrX`bE5Izrd5<Gbw(CvT
z{B0`Y9lKbc1RrP?%dumf?Pu6q=r24^?aIWD8x|xS-0j3}VDf$iL!L0#vW=6glHWZ^
zpZWi3%Be$#%A9R9Q}nsbCp#IkW;rc6bDS+mM{}8_+bO1<MylJm?u+kBPxnx7^=2{Y
z@QB>7vFX_JH?x9HE&8#mqJuLltjyC=?37-k3|IIBK7~(=?Myno?b`3>HP<$_PUPv<
zz2e$_z%lN_DUOBvDkjcWT7ETDu_t7MtZ3mhD-V$#jTsY688eb<HD*K<STd)%vj%!L
zuG*mx>{_k8DP&sCp)_B9r5x$p`geO1%oSWe*l5UYxUBewmHGQZhXWHC7AbfCW&Ry_
zvNR+p=;+(PX`2#U&pLg6(#X0<Y0I>wtt;)*!=I$A@_F26lRE#+zRB~7CS2KlWL?7E
z6)ID@gj;sBE9cefyC@6I2+S?bh!9)wb4$nfo0Axhu-o*kOXHd>BXzB{?Lk3&=ArXF
zduFB^eYl{zeZ{8Asb-Ec?;dreR%AYrmR|m=>Wi@;Lv+)^JN(Uyx>-W>7%I91w&#51
znR;Tud8uiCj;*}2<EMGnGO4G#o?cliwDMI(NaCq=8@0XuG<jyMo0;`QNNW1OV}dzP
z*smuqIP_upo?N#D>!z%f^{$wu=YRX#<-D^Zan+}~{rqo#W$p5|f6#rq<4=Z|@}Hlc
zeV6xEw?9+xxos2jV_nvBR<T3U-*iGxO+0l!a7K~5N6q=&l7&md>JOCN64F&)FVJE<
zW9jrT&h>>hEpIn3y%m)7)MlpE<s&A>tPUGKJFzZ0pnm4kix;BH5_zU%^0*2(`=8PG
zEU;X1SEDh$YfhQJc1bJK)&zZthfhQ&nm$TBa=N9Q^J(ssh1y%1PF`wSzNd8)lMnB^
z3AT5PAF5sFHrtcaaBITytFB^CqBIXZY(6ddOR~lGNcT>r#X`YPB2LvzWLV6cAl_PQ
zl6OBSEycX`uZfudib4~HnG8mZB4RpcB0k+SdURw)9rFRbW``A%woEy!I(6~!`v-#+
z-UWOslv3l+Fidtj_PO=7%LMJ)>sPMi(7)H7HvQv)dYy#Eo(B_!<chAEv@Xy%7VqnM
zXL6RO<29xXR;Jp-gI)7Q?WI3mIJ#_s%^L2IUq!DsZJKfUL%Q=Rrdcz$t(|`C*6qKc
zd<sg;$09e@MX&PNsrh6FvzCj57=y(YMb&41si(v`44xJz%@D8A3OQ*T%GVQit>eD!
z?+{bR3APGXMSgGLVtV%DOa`}UNm0_gf-t>BH7=gcZU&BjwLKYTO;p{^d*@iH=K+OB
z2D=}mtdU>a(Em1S%R-Ohz@A?+U&|JY?uaxi;9~c?c|do&y@$?>8H)3L0(Woll~Brh
z&BJimje}qKq1N34rd6R?kM>G7>bSNBoczKrG*fKZ3Pxvk_U)nx^-{j2x9(neyLoTL
z`lYcCg(sw_RAgvJwp84G+@tKzBCz;e$HJ~1;TB0p7HI}o9Xzv~Av_@e;QB7LP=>V{
zCk}_Q=kL#5vDoqMwGW#%K6K`6_cGvr+<kpZQG-q@qwUO3JB1pA90Hg)Y6Lhnw1VyY
zuiQ;3;gB{G%@>+3+EW{2Th~~u^jBn=#)=$6^^_+qJqb%%y0*<L(&+Ku#I;9hrtSuI
z{f#$1{58IrUfIa#C^X@~!iNuZ4|V<NVq(<s{?I<>@(HPrpE@d}KbkzuWctGLg7dvm
z*13%1ffK@}tjJg&$oR&{X&u+UZl-X{#?N!_|J4f#<enP%Xwm+FXN7&<JN|P%PFo{;
zNNR$I=#{;krp9er?vV-%Ni0_;?&wz!@@z^`O*yWcx^D_+Lx*4zTers}rAa&12hY5I
z`Ox>Hy8~)|L^yxFuj9eu_bGW_Sk0dAJYFhIa>A1PpW05`zT<@f^Tg##X0E*DIC<)c
zU?u;~)tZh1nO;jYJv}C0f3dig>sRy&ZdK_@{kYPKblv8C0^9H1+*|#_RCn(4uo9cL
zRCT5TpXg=EOw)8XNUF@=_;05BLe)R5I<0GUGx8YsG}(yq_kP+e&zh6CUMS?<HnmyI
zF`HFIm{ljA+PSshEyE%aUq;jPAj7XcPPUJ`ToySjVpb4qDbH)3WxRCC8Ck>0yTl!C
zwree3ysTl<l!gwKV3(Uae*Q45|0pJq?{tK-If>JdW7T<)`V~8-_&hV+U-8;fbedCZ
zs(<VGR$uY@KC#BrXH7QTW=`WcHHAU8vypXi@Ura+8`YG~n_BI?*A%+LK}GAv1nwEj
z>K073pLkVEV~56G2csw(iN$|Sx{jQ@HZLvZRKw(n>dO9()0%X8jMCnTFxf7X_ikNm
zac73EX7pKw-A=h7?}e<D-|&isuQG0Z<>P%(k&{osC^>~K?qr|q)(LO8PWHa?^;Txw
z@bysftp#mPZmSElm+iU~lr&#$FY}@Nhf)y<FDE#$tuFA^yO7Iqb!L!<3~Ty5g9$7m
z5zOrVmQ|<pN;iCE{rQk%n#*sUZ!fLvRYZ)1g!UZ%mOH&Vy2+-?hD)PsP1l@<DY++v
zC#Wm<v&`ry=ZqAVQ`_?X0q=KCR$;ER6y1#4G$sQ{%Z2qT6L_>+j8=HF8VXcg3NBD<
zh|7NQM6;&h`y&lOyXgl^ghS7W-$*m~JoS#O>4KpA*$o@QZ_LvU-_*6)wc^068?K4+
z*Ei3oIJZ%oui>v~nc$X)o%-RMTBqJFRP_yem8|uL)$o$ef-?sgj~;U2U3quundfhe
z@>XwWzgI5u;^r(}_Y2dxzAUv}Ic>({1tu3F4l7-CNWL)d8%rcJmzzk#j`WLvbTp?$
zY~_7&WkZ4E)LYSwT4yqyrChq$x@K}6Y*J!%<;qI)*?0J`=0cV8SIup5G;Um2)uz)s
zmt|w1srHt3&aV%gGVd<sGCHfYkJWt6yiM#!dIZ!345e64xk)YIx*lXEyQ|???$(JM
z(${yo-CC4)J?NQsCqswOn?Bw&DR0X#b@vY{PDk0z3Kp<3Oo<JY&+hfn_H1CaytiQI
zGphw|ix&G`4-)gKnCD|~y5Rm+f8#qhT;F<hsdi02ZK<&Py~I&vq0h1*KbC6!393G%
z#i0H;_Woy)Rq?Jn_K8|K7nrK8SiNy#>Mp7E<vsIq-p$m^nkVgX`}MES4x4>>-snbW
z>xBAm{!pIy>b>;+%}U>%2$UVVCh8p-tk~eh&=Zs?Gihnx`(si!)stsV(OcPNQRC66
z`hFHi;Q_&5zL|pUkBkKVJgfftT|!7%j`_et6XCw48U?drmdS)o7UEgj7q#BrSF5RO
zr}~ejKC4t21%BUKlXTtW_W7nIkwPDhoHTkWW=t@4j4n~JIeTK(N0Y^UJ>{&SUuUk|
zp|(OirQ<?Zlc>HnmzhmY&VSFdS57W6U<g)Uxam*2+t=qun13g)?qy7v-ZT4>E8`iv
zTdR^+zIz|6#_%otMsQ+yZ)AAZ4Bb?3ZN^RW+YQ!qM)K;W21_$Eep-<kHm7Ublw_{6
z9YL&XcgV46atpA(I=qfW<_1&pCDHAx8n$sL2rc;j_p12Eg!u_6{ok6F^9iLWOwL}r
zVfNXzBAZyI*fDXr{n=vJz!oj4Eh=(z)dD@!xwC@i->p~jPh-kpH5C)l(FkO`UBf5P
zpD45bSz(;4I{)X>bLUUov@v4e9hR4Gti>gkM%@lFalaAy@!l+tOnw_?Mprv^rj}Uo
zvhzQ*nI|v@zSdA{IJIok^g}l+mOXo+$^XRRLh!GJT7LsdJGC_!GIAy<O?mL^-Z`zC
z%^nUpR@bCk&6-%}*fsq$&Dvmm$kx<~J0h6r+LVMvvz*x1F)k74QGdm%{6HdZ0oQeg
zQydW)Ok%SVe6~8--{ThBqoWb=aSP)s8{UQMU&f2=@MV=b*IsjMYRU?anSnp-1RNIF
z9N*^<wwYaLLPYYR_r9VR=5M|!ao^!&+)CZ>tkcy|Hy53nrF}nq+k!Br1QyXXT8uqg
zE-epb4xfA=wB=Os()`uC@>d6S=V>x%{qDKiyl?HaU-|3@9E1C239+Ag$h?&0p39PN
zl5ajYC|TW3uXc2DYsy)-vMnoUq6}A}m4HZnsCUlJ_U~&B`G-AJzs;>S#iL?ofWg89
z$AeAfqM@N;YcwJ*=CG~K)Og5fF=45qH}j0c{LY(-(-XX6SMRnuadOJtwEg`%H~*{`
z<M|`|!&<0djZ1`TgU<;`54M@ElR{f=T?wl9b9F6ccq3<~z3Tn%h&k%PeA>HPe6=Up
zc1^iH<yWou%s?CEte`T{oWLiCp44sevX7k9xAIf9i0bbpYbH*3(0kYAdi|rNy0<uM
zOs@QzwZZG5Nt>FEKj)!YPIK3bd0!NAY_gG5o;Y#ygS#eYH|AEvT)cOxz2O!^>8xiH
zIHiR|e@#nUvPkCF<dy&V-mD2)e}s>#I-{$4>7#o=mjV<XT;NdH<dDLc#Q0iL^rzR+
zhNUj~=U68ET(p6K`9Sjpv9nI94~|VU;5xv_w7BVB(3OT>orwp8%uTB9XDP`>>YUwY
z+x*@0N$rZ(>qkO$9Q{{z>Il>p=YF}7{NeVMW7=Cks%+)iC&g9P8ow^hBJA7z6UnBB
z?YFD{5WQ#<J$*sgdsgKKDhE6(W-{tBN4foM(qZNIwO<*0Ip*UHXHABz<jMX;M@tsA
zT%0E-?IF(S`}vc6XmiF2_3KBQ7?Mt^O!Enw^{a4c+?}8-YvJhp$y|L);ycgkYGjIf
zMYWb~Q@=XNFp#S&<jL9-uP^p}xZu2vfx-Wv)7pd06?Jl)nnnj$-fix9d)4UQ;jRny
zEMgrjB9Rx`wh2hK8ZR$+Cc?5I;<wTpgJl5^&uC6b;eOhFXz7k0K|ZGImag!;X?lhy
zE4NxJLz5-IdjC1`Q{s*@Hzg=+yXpGoM{Do`g@nd(M$M=T2WL$STe$SCr-P8L<((BB
zbFOe{Zkn_(%!a2TgFAfbgskno5lvP~3*2nLrPRWt-G2iY&I?_UF8b=`JcVqw89n8?
zn^&q=8}KdJ$=4nG$#28KL)#kF*VtW{`XtWsfM}JrGt)1&gByxG*T}fDi5+1U33wzD
zo%u8@Y9~vWME;aj0n7ps6Mh(#P6^}7SlYI*XzgOHEz3_vPhikoFg>ZWSNO}ktn??k
zj7m9Iq}o>}?s(wxYpP31?}=Ffb`QhT4)&~J4HccD!4S+iMbKVa&S=pQ!~Qw0%kHq9
zKX!~e_}{nxIad{PjK7^Y_H<q0BJ;lN$1Qm=ccmi#xJ7RZvR`?{J?hl+J#(CHFn$#?
zXTR`=)vzXQS&g2HrSGRx>l#vDy>gS6+Qe`w@L-zz-X)ighOC@+#jPr1-_MKBLsl-o
z;&v!Q<kzj!{n>MlC9?e9vh(io{gI0*+|`b3<9(rD%{b)+$2(T#O=@ZrmRlP!i%etW
zI?WJ&wJx8h=$i-EH(~wCl|8Xn4+NaK5LfkQ=H+IEhKLuzIV)2>c_luax#JyM{7H)%
zHiZRZ%2M8u3L6{}8c%$m_&&KbWg-8ecems+HI38uonND|Id!IoaFSbxlww}X3By<1
zAuViL4%JbeHpxHVG8%`ZWw2a`C|)fSu<X!Bvp(B^g^~r?20?b}c~c&*IntY}&8D!)
z;Yqj1j70}BYhFJ(D0gO}*@cPMa=O<Gn#<X3X8g{<o?*$nWsajmQS8mmH4`}_lC#<W
zWHSHBtZpjrG)-$bT`@CIY4Nch8`k$+a$;g%x?LF7EmAlvD(x)4qC(E*m(W|mmgig4
zmY5%Wbmg*!k8$v`{@xBRoe50AUd)}WLeYohJip!NTOi<&%q^9jv&-}<%K?Yn)CYlo
z6*#u%@!trKUi-(2?blg0E%oq<i>Ehhzp^z{Nm<ZjY@nz)O`}NWm49I1k=rdQhXq0>
zl$$R7x`=gVh5gj~jEmO`KAYn>Wnmmo&$(My7oWJir!wdM$%)d(7ygW!qB*N7K{hDD
ziCMJoO1?ts{4EnjI1JL567&uKh3I@U5ia<4za~Dy@uS#=*TH_tj%6V?f)o9H>o#u@
zeEnwgry2S$8vC|$C#aex-<i6)=~9~5iV7YM-Oee@6;a!J-8rwgv%OMYyJs)ox5x%X
zB`u|s8u1hPFKp0rNR-sETB@oce?Y{gp_4)O<GT&77A|_jc_1Z>|G}R7YaWHWHa2g+
z!E;eJ-(LEj8ABGs?F&|)e><mWdS4Lk|6n8=*s}M#h(h6^G7Tn;6rN@3I{!Nj7CCmX
zYjE-`JmV3=F|ol^^*}_rMEbusx+m6H8Y>4XAK~1#jNzO`z{$*&+ZSwfNWR~p5TH19
zZsWSc)0mPYxP5MVmR#0$>J)yHYI8w;#h)!Ax>sK?Eb5=mue_^7>ehmpi|4h@II{V$
z>^ZJUS)8GjSI%hMV?FT5V+QNe=3|fMN3bqXZ8)hhaiZ#GPy*2BJsOb6_CxCND+X_d
zWsB4<yH#GWbMbTT&~=niOVp5E^5#wCP2Y!VPi0g6A2L3;Ht__n&LqdK2kUqplGwh5
z+z3s4ukt%TY(k`?;=T=&Tr(E%T+PW+Wmv@EBBogPb;`quf{zZ%86<z^Vd&}Ab<*L!
zlEao|=3K(2c_{W&5T7x7gl6iC8yAdEHdq8aV#t<0ajIqp^NodHJ{g;ZFh0Ke@LbPX
zA=c_u6`pCP3l<29n=IIyaO8ByC&5ye2G5G=A1-L{WnFBrPJ6Rl$+6gIQ`!{$A4d!R
z@ruq)x^p;?HOsD9KklWezrv15v-jVu^T>_aI??Fy5C0gJFcB%kW7B5;N#~!@c-OSH
zcj+31jZEKS_Pl;@Mwrv^;F4s``Ql*?EDisJ%7fxkn%Ns4A7xb5+b|>W@y*~zE}g;=
zc})JI>~eBjr%0~c!hBcN=*5**DF!9)PL-U!JRt&e#253KTwvZg-SA79``!e;IkpDU
z26ak~YzMWkn>&Xl&8zyWxk>9p{LHNij@KG=8n3$k>0uBLR8q=5w7W%3)#gk^Maq;D
z4D$l__fBKA-O6%S<3!|*D_yg+=lx<eueix@N4Z3q`|%v7_LLt9_gSwd_%BmE|JAKg
z>as)cN=6Tz8AicMGind5ntt+l>Za_O)~Vm-8cmKrt+MRQ+=+b$7|rh~Jmr|fvOvcB
zX~L@h*+K&Ctbe4H#aZ0+II=sMIrem)J$gp6Q$^{ZN$8UeA5{OU$7#i<{aeH?@@d8v
z-sunGjGP4eE2KV}JWO<&x~)evfTQHx-UPQ;$*CzKIvNoH6SiLoo3er-IlWZi%zdFf
zITbep+?^d=bTl@6Wxdzh%pS9}Me@e~%zU+JSHg|2_3BQUmF}$_d~HUAg5b-}^P9IT
z7&|AndAYi&CX|U;tC?Oj3viiTBr}zHWwMH8RF~A$J4=cj!`?S-;*KaNIL;d+_GE?T
z?n|pTI<Uvj|7#O2U;jtwPt;=_)*a03FQz!NUJUv4=Yl4fS<Qd7XvGbtg+6f$l6E)$
zvv8=Fd2n6&!Fg#d74IEi*;?#3Y+E#`Yq_Xt_?{<o&E%fQM@n&@v#6=id!zAVmDj$k
z!acJ0)EgO0J~MJ_akDY0nQd?kGx|O?B((C~c>&+AVM&fk>?vD!{Syi(WH{R1p<mIs
zTtGVL`$p#dp(`6!=|x>KzY&yrYF6v{*7L20-ELV-DKSV5DB;W&YgS{4d9nAY_Y#Q<
z@*3xBT9~&yvTgMAXmnh>rA>ErVC>Zmsg8_tx4NG%G`^SWaGGKA#G_XpE==Uiy0Llz
zJOAmcHXol(f6B6h@%toRe`(Ph>cI~=UjLL@S2)dW*-YKCv*j12bJuzNwBMBQq_~W$
z+i&sL#r7UG6F)4n|L0U5U$1;<={J{&Cf;YJt1F2)y67<7>DS#M5m9h{#RIj|h4-&#
znu~wiqUfJ2Cz8|qk@>u!dw@jK=~GHOL`o0YTyEd8XW4$6(jz91o4QLFn%Al{NX#@!
zR$1V}(D}1dXm|HDxsZJi?pCT5xMU?M#|9}cUhH;EtwUXwSvhpev`4L%7%#nHnm8-c
z+c;RM?%mZh0+9{A=57W-&eD>yTeqKivCz2b;-f{!T>IPG^E7Q&Zk7?;tJT%_f`6Xj
zhwY|Q?*vVGYM*+128Y5Vjo{Q%_h&X3F^Vv0u*AqXBpy7)V``8ZtQ~zVV#Ck;_fE;@
z&re=ybk^r*+>e+JsYGsr7^Z(Vm-06+5BZ{1m~eXP){dVY!YuWjJ!MjoNeZ)0eR=J>
z<$TKgpRLNXr^$q^zm;}Vc&my5r$>O*f{KQjmrH~$>b06(V3-~uxJfN!YJh86uVkgs
zClR$1!3=vCGF<)c1*LFZ<FNOO`_y>hoAQGlleSISvUJV`uemNuZe0y>Xtdstbbd+s
zZIe3zuX7lfw3t2yzLi>;`&M(e-kaSUt}pP6JDJRH6rUodvu2|JgSGyi0vYYbOXU_p
zf3q~X&92?Jv51XfGQSevl-h9l9KOv(9*)sh_PJ%n>XtH`<bLG(|7yO>yZJ}$PFky|
z+!RWFR&AHT`*M|7fW-q@(=J9<<_6h2$x55tM0!pLGukYyar)V!!|q?8W}^N$^VngY
z2irs(?4>I!ZACZl*n3Mz<V$xp<JyO@hplz8Vs&S7S=SngE;}<rRx@MA2Cu}aWlMM1
zX0RMsJ+DQSEBRG&=EAzmmt0KdpHYwz7hHD6<BYa#*oG?s=Q9~j{7U@u@~))h$t}w}
z&v&Nzv}Md^-|1=-Y_r6h=ezT|I}`aQUiLfpeg4TwEjFC`!g6J*-aQf)yJy_uDr-Ed
zuiJcH_0YfbI}Uh1Xq<fh=Dy@PZ$tCc6%C#iE8E^bTzl;J$NABIEb2L$#wtunw;lOH
zVxwN^o;Y)$E`fW|DWjm&Oijjc8}o&C89o%*`usof(qDAx(l~~n4s0QT$2@E+mWZb<
zS!HyI+t}!Z(~?z2LDw^zXLBkrOcIKevUQfZ;oepAcP(r3k6$*ERBzh7=GID3saSTx
zx4m*#xO~^85{qDwCB3!^t99ip<V6?W?deWdyV=IUAXu5JBfNs|d3*JtNt~5JuTHJd
zX7)L@yD6UaDffcN6*@7uvpDCOP5Hk1+lM^12kGV;b5(z+^*C!!d2{FNoPSbI&SpXE
z23KAk$xS)<alWa0y}!c|MwzG_6AsJGsx!~ujC>P!r6g(==YkDwk60s|Qym|%_V_5(
zs;_v!>!5t(n3`~8#5JMFm8KTXH|y>v_4utbnXt^mNAZ2bGeZs&&Jy3N8B5HxFWH(H
zc{ry`Xv;V#snlfb!hP<;*O_mM3)4H6>BY|DjObD=>-sD9>`*kT$x-X0yBQvcEnW7X
z<x7Lj6i20du6w#GreAR4ek=K~>el2vTVKygyBjyVRnuvsl;A7HL&2s_i=_l#Ccdq`
zT^;rM+pYCxE{mlOe2!Dt-<e=v@*=SDZH{j0I>CURIkzml<I5SMe+bLDIC>rMI<Tv}
z>5sxyrHia0&pAK;@z>$r$a7QWvhIHS6KgczyEM(5rnx1BOM9m9D&`eyRth<!_D<DR
zPHLV0JSi<uHiw}j`OKCxGpAWzHnQwHGq+`;iqhBPZ7xaiowXj;?=#c=uGUDhs;2Zc
z6^T5n4tub8fw)wVm6L{gV^PSIFaOu6vb%hC?9!jnG(lufvY2C&j`=UnO^ln;bnIvU
z?KzP${lt-`pWz$cDs0O0-(WFaHqLSKndiIj3-oLkRn5L--Kb`klFPtd+`{^H%Jq(2
z`I_a;#>TGEhRtHzvkSNMNZ9Kr9GRsRyzly<DH?~<LN%f;DmbWW2kldz)${Uc7c0X=
z$qqxsbsNG(--WXNEAqa)v3%yH*^-^HidIX1<uk3ziA-zleIb4?SbfieoK@P=nXMky
zQ#GTDibST|2s^$`c;>s74$cPcqdKbR1VWXAtmPy{Ds*SaOyHNF_~TV{ZTW;wv2f*C
z>&v!899*`~xyhXAn8RKHFU?OEE}vmz_4;J7b!nLOp_DI^I8}wUPBKjRX!Kq++<Z;k
z7KS?4U@Zlc1KAZf7#hRF6SY^^sGXUyfkUTYpPP<nTdVV_du+A7_br2`e`gAr@T-!o
zd{vjpA=mIPfmfeoHQb01Tk_91=>N^jXL@Eb7|j>A)8|_tRFv8~m9>J=YTBn`qO%0E
zlow06a0d(iIAFX$q2h<zuk0oJ**+I8u`Vnre{^C4`@}?krMq3qhvmL(nHhLsPx!;D
zvkL@w{?+kFu;{3e3|utX!zrg}-hu|h+njs{;yu}}^sJGLwDuC(vY&I^zP$;q2fLbh
z_Y1gvI}&8-mpbpp9pS4xUOu_G^5N~;q|QwtGqi%Ab!T+{KOQUkvdBBEerBYwbiQ)F
z=6&&`91)$GgMBx>CP@?>;0oFQNW5SB^yeSDdZUw$G|pr5XS`l<<A?y0tts2m?EL+c
zQs$qv6J5jYzg|O*?Fsk9r2z-N+~5$Iu<Voh^0(=ZGnpP=%>BP+#qvj`%r3jDWVS#2
z$R(}a{o~)Ehf}Zj7`dOGvZ3MA-nQ#Dd(`aiG_5`96Zz+?UP46bPk)9^+3lTkS7f})
zWRuHfiuO9|pqNm0+f0f%VCF`n<WzC3-EJ-%hKeq&Z(Z0H7%IO%A)<6LtigqK%hHLO
z?ae!yw`l|#&s{Oo?}~?d-0D<KO$W{hPor&05o}BfWm7#^j|Mu=T`^<Z76FG8x1`z9
zzcV|7#CSaxBqUA{xIN+5`7BP`m&|gPa@8Nix>P>iEt>vTsEJRdVzSzbb;3#fUz$!#
zV$@Y$9=TNL&@(j)R&@*3?<}m@X@@;8@Ty8{Winj3HI4Iokmii_f)#Na6;)DKEOB|d
zVd<F&|7A7tKllCm?Zm)4q38JaEdo=I80X&!RJvo4<9)MEwW{}{$CsOLH3L^4S$j)I
z^Bd>k4+kw$yEvDeNGvu?sN{@@dobbm)C=Kvdp`eFpSLcMQ}_{6jVk9clXK@-9;lsC
z+wE{qh`~c+VxSW94qg9(%f38{6AC25`~&tasB!H)72j~}#<eyf-b$$jW~Zt(4Rw72
z#dbH`&$=qV!6~sfLY--P>w!ex8~>|cZROTJvSs->^N&mFlPd1G#9u%1VaGI~M6Ws8
zh3iVnjUBZ<hOIs#{J+!c;gRNlYz-Rbsqscj3pO#|IvH|UY(FPwuF6dz0e6<r1NTiI
zTGaaoT$m*k|D|)&)0XLpOFJ}qTMN5p?D^2+`=RHr7uUU1kqtAlIhX&QeAhppb5mbr
zw~}b3*6mvicV*V<q=s{Nad0nozq)qmp?zUuE6zk*tX%VM;$g|1yMmWyYcd@Az0mw)
zO0oYQhH!_w>y}Q;O!8XdyR%7h3WJEG1KY$mPES6#OEmL3JmFp>|Jc9&s*Cy6gMVhM
z+g`u<Jad@g8TJ*&CfA1j&N<Pwr{{ZZML}lPC1W9xfG2?rfgDX;<z>GN9y;t_FL`ne
zH|O_w)fbh*hd!&ls1*LXhqLM)3u}Ip*LL+2!54Z?U3)AZ;lH9p_|O-%j56VMr#6+&
z+bjKD$gQk6Cefwv+ZrEPZI>k*HuoiEvMe~WpwFI3$*$!`gMVGq@usCG1<x`a<lgX1
zqq_I-ii0&LcRL*6G;BYrd+=PQCZo`FCmp||PKgIksa@M~J?7`WjjNBCq<Js;XnV2J
z>|*4#piAA#{V7Y1Of~Z0)_dz^GkwwuNk1Nu1kL-C=S(@puBunycXH`_7vF+Ko8MfT
z+SkYv^xk*llLQI>qu#R@6wL7b(CHiSL(SswH<ri)b<US(3#2GGKD;e_MUc_t(cG=8
zv}+g@7S_03I}@Mb%XXossg;$ROKgu$MPy*I>%1-<QQq|@GOzzu(I~BMvf(u~YdCl`
zZ{{9>Jz2cA>6N|O48oy%I#<QI#mgrK=&IVJE}oik`Gthqp-xe0zTIw`jxVPR)S8Lj
zm@TxW&Ng$Jgw36vsSh$|JxH3h;PPu$;RC9B&Q#0{v{UXsE9_$F5%A+{6RT;M$%04I
z6S?x1?09l_{ax3t?d|uQ%P!|~b90>Hu2z3D_jtO+vxN>y$_CB~4JW=I-mhq^%OViT
zyiw7mT|oBGx@2|T7-6pROHQi;7rC9gbYzNdMvY$&N8q9(M;Ujv|64hYZJCUxr;p{5
zQ#Hv)YEM|+@S3B+Z=UFzu%Kn$vB@tLW?Hy09{&8wd-1L1+S(IA^UD{rgtp{y3JP9Y
zB=^``YpUCw4F@(UKj8o5*yuQS+I;3MCVP%)vQ;f(aCp*vq^Xnj*r{{>Z{9w4|0tu#
zJ`J6KJ>@w?st-<ynSOh5q<F;|AzO1^#(74{yP2*nImD@Ic%YH7<A{mv!6~)Or-ODz
zh3lP?nD~t;ye`1g{VeZU6IRcyErJ@NN2fd$Q3XLJ6K1pOg4x%?g#ryWCAfrTu?sh-
zKUv76sp#$Wvq$Gy{hWfMVg3_Lm9x{r7<4rP0}i~o7@z6d*(DY(vPR=Zz#_$T=ii0J
z!s|I*y>d?el$!oAVwI2PD}lK?j-FTAYrwow&4sOkd%MHsDA%Qi3<CDS&x0%j{><f^
z5h1wfX?64m{>vx_wbXMn9NZgk@GPoBu3Bf_={U`I*Ago9xEEZUY8bZgrL)2`*A1!Z
z*E^&7?l?b4cL<qw?x1wm!pLfCjZYKizcNd#UYvH%@CDm%4UL4iZyqk)S{&AKGjT#;
z$AX2Q>e}>Y*<G!>8Mrn<yg}jI@>%WkZ+(^v_li$=%GaXebftLfJ9V2=FA9^J>LRN7
zWsWfFwPv(P%7hh299;bIMVgS_nXHUQ6aQb*UmLKyclqflVvkN_843Ok*rNYo#Su>T
zU;29D&wn@X+{w0Ehll-SL4ekN9ho45!!E}Uo=y^GFmaExWmV!=$q-u~>bSP?%Eao=
zYvi2Q8s271e8jZq7%NBcv<F%}Jv$yqmfcBu?-QAPoMGbDTkBH}D_)g7{ldrF-nyk%
z<J)FS&cE)@JdQ5q=Fm&cO0k}CSi*(JNchZmiHjm{YX06|V<NFwo_Al-qnjshGMMgR
zZF{lyXUOgQasEw<c1zw}8(W#zpA%;Kd(9Guj0t{c%(uo%?%1$SB|Gy-dQH~X{%Fqg
z|2mSV@3{0uLYyak{*yLk<pp(19#mgpS-kiE-JrMT;vapqR8MW>iBy`iK&Nx%%gbhe
z_qG{7@pz!`S7*}2p2~dv*nw2$wOtvZA@VZb79kA6o(BXAlpeS=crr|8P-FCAlE}|t
zn!{qjnV?bnztG><oQvV~oe2y&i+x#hU$NQz?sOA1S;gSPID^T6HGwNIgkd?8AL|S*
zgQ?YQ)5RNBx&*S$;j);T{r~6Yd1*P5x9v{39kWSy{>`uxN!Gy|=dRTF7Bt)a=(c4^
zh~w#3i@s~*UUYYx+}ksSTePn~3>1C)qj1aR;}`klqZzgDt|{4k=q#7H_MJ5an-B5M
zPTG9wte5$_6*q$X->g(OyDGf<mE^{$^KL}=e~?!%yCyvQIH$or<InSw6B1%3cfFnY
zd-Hvl<GfM{OMZ%KOe{a_u5nhWByAc)4^t0Qj~KsdSCh?#>Z%%xUEQu3mvk6sOt9f;
z75&1&a+Uwjt#cd(puN^En`B)4eD-xl1^r9e|L3vjy8MsF9t#|l%su%cNR7E=S%U{_
zh?YXIri+0_sKY9zAkhn}8X{P)Xeoq)gu)y)F-;P^u%;n{HAGt>dZmkjM!3T+rc0t1
z)-_bHUeQ*F&vY@+h;TT>BqVlWLqmtC%Nm6wRzd0GB8B2pRU@Pwk8ml432bcY5Ov8h
zzvd?6(|;i^ipc<^M!O@z@kslmXr**+kP@xPyaq-=?T#Bsf7~?ubQvWey1>TWZk)%#
zo88i6>a;(L^8nbEi6<CEwO2%d%>Q>gB-=*ihu3Vu66e;fjQ4U&yp7kI-VsdbZ4>1y
zQ<!(LImmLyJM-v=Ijx&TPlv7f99ZiXvGitFs`j+o+JBg9wb(XEoJi7Lx%_IxTFtA!
z0@pfTl?quafAv=I+5%RgRa5h>^~3HN*qC<IO8mzrZ-3^BTIEuU{y)sMzv=}U^5VBH
zP<a-1cUMc<xBm4%FZMGWJ;!`3Pp@O{-`7ijx$9p`&zzoaEqw3vSI%3hrIX8itBjuA
z{*!Qqdl`Gv>(pN@Z<5+QWz~+)@JP1Vz46V5Nrv*0*7=psAN;SD=-Rt+qxrVq|H|%N
zd=e*Yth8gwIqO@A{-5t`^tRMl6Z@uVsq41xv~L};risTLbq~y~{5Io9PP+5kl#DGK
zB*i|Q`%}MYqyJo4ljHKmzYUkq&uFxJ>zB8C;_v;PO2R81W_I{b-dEV$`Dkh9)r<2=
z(qAPkcWIKGRLvONb0AbuXt$_Q<i68Vf7kBym?o^gYI3Spr`Oa3wj4JfuCjM35l{Xm
zl;y4YyYNH&CbK6!QvXUXe-H;BgS8+1Tr99aDMM^}N~3v|!u_;`LDG4pue_9xJlfzg
z@xike;pe4(#vU`>H~rP|5GeS{BEblToEjPp%52MLdhVRbxpTWpgf~-^&;_C0Cq)%s
z`yCOsPHeQxG;m(uB`X<hT`%J!d+{qnna)Ky)8B98Ps)Yt7G8E}<L!C!2hVM1-`wn4
z9io$+_2J*ScP#TBv}YYRwRMs6TVwj<<fkaV6aSu0+?-wXwkL1P<DbjEnD&&In)bbA
z%P^}HnW;WexAAq=I{i6kw*}vL*66UjZ2xS7pQ5YeXWe-}fBS~t^K<TQI5>Nq(Z4;D
z|L^1c1UbwKRN?%a%usRfO}4W!cZ7A}r-vrS;pJyGMEvgfct^Ky>%%`gT&s@+s@+?h
z7H+Uga&5HPY|X!STfSeO%OsZ+H|waz3C)Kcmow9*pDOnGdt%}Pu4O54ZDPR(*C)NN
zmVWH?Ni6wc?|C`ZFiFt`dkao)YjrxWzu)^o_vehiZeIldI=wZ?vhTiltH(N7#>aTs
z^60AWou*>ltZ@ZREB|khUa@QIwp$zb_bthr@UHi*M(*9Ki*HYP`(T%BoM_9n+^|9;
zvsW9EUo?H#(`2L6V|t}u_vYHIo#n^X;_@>8P1&KqyRcY9@pR>qE0501Sh`&4l27An
z$Hs}fFFh9sKXzz+faH-odG&9mz2K5>vzs8cm#?NzE5BL5P0v2w=I8Xn3(*3*A98cf
z?Gvf9%R0WV|K9R<mz3{5WmWGKN#F0cKYH^%iNcK^Z%k46!lkr(=6aFlvKzXsr*>>&
zlv}C(G5t%BMDtm`C&pKLZkSf~vOjr~_swq)$HW*O##7lI%rR+vTkhtq+kV@>GL=nP
zK_Mr~^~S<`ll)#SKl&@NV@`+Q35$J_xdL3rI5i%)&i*9CFMr_n+0~cUuP6*ska>R6
zLU6h30=q(^|4R-x{QT2ldeF=8z?_gf1#2yMpH4rhxM}xEB}v9#whZx&_a_`ZBXI2A
zq*ET^9NR8*8<^BKu=6c?VVD+de?8>T4)N9g`(8>-&Gn1+SZ0@Zv2VRwjP@+)X|i`Z
zuS!=u@_MX#>r3K(L31aG-tYq7IX5^fch^bBxxFf%G;7Pj+xKiv=&n2Vu<<<C^XdCI
zzNS8H316pRa3h?bPp9?o-oP0<^S(CbSp?jzxSU^J<oIVILzm+l&D|?n)OS?OEIWGX
z$mL!ezfJ>_t71o%*{moH(`&r@@7L97O$+*_YJ0iPP;K;{vDWnBp7UI{LJW3mr)+QA
zeCE(|xr!rcp^gQc-YpgXXaDV=PHdU1=Ovf>xjNq*z8v_YUS|G8T=m4(Zy!5O^qlEh
z_4<C9eN38LQ^wZtz_+tBOHSI?N<L}hZCl{))OTx1^1Dja|GveWkAz;a4ak`mSQsL&
zIO$tpVPnyc&&M5Bhn78dm6+Z4+3wh}lIx!r{p=JvezTNm6Qg}dz}1${;Heg|S1x<%
zWih^89y2AoBc!Xdtm)LIjazyZYTnB)y8KP3mo+hRkpj2gRF~4N>5Jq(Eis?{^HavZ
zpt%YRTN(Z;xC(Q;TA0%;>$38ZQpy#-#CpG0-&7@)qi4++W4Rn+5A4>Ry5jzuA57<c
zt%G|?0(bmA&-OO$pty+Z%%!t=)s`!%@!e8j|5H4DPVmHcZbl0te*U_bdLZvgnYvQ=
zf{kfR5tUqA_5Q~N%5R9j<$LN@b$eAuf%4<h%YkBdw@Q}P9cozE{oj{)KmTLtkoU8n
zhD3>~m7FRSzkgb)Sj1qBRS}bnZwO<i+PA3^n_2mzkDSWAeBxe-V==E)QQ@sEu1ACP
z)Ne?)x1O?Pv)X;)hhBWZA$FeK;^*&~Y&YI?C2p4Q^0pRE-#@NTx3jI=pJ*<2lBu5W
zpu5k4!~egPM(s27a%k>T;}SP((3|}2T>9}9x)WIc`z&nT(LE=`u<K7L<08KasnfPF
zoN{*$VySz<^jZ1TCNtq9yWBTMHnz-=N!P#d`L_9~DxJBm>5bx>Ej#Wzy}QVEXHVqg
z59dw>R#|=55SK~O%H$MV+V|b>Pw{P*#>}8ES1b&J-7|&mh%fbv`BC0@X6uXtzSA~+
zI}x{kX;YSW1C!E8r!&C^9{yw7w&>=b2VLQ&+XFbx-IMq=O)6sgY0ae*$|ea-`0g(2
zX`t(p*2cf=+`4bu)5^RrOG;nmn^<0ZX|L>N;Wy2npJpaiT)J(4GIgg$x>@hCj0^j?
z?8^<_@)pO=HP+eSn#dk=%+F|!Y$;b4%c+H@0)Aidd)xJQ(b5+Vo5dx5*yVA)%!&-V
z(vZk@I*R-G!{j~dUKTTI82wpwCqRnfsK&jR(!8<>zvri>g_^(g@nHXLvGItI-<ADp
z^M0gT+V=|ND9XPnJ-h9$TJZFwrA$T}w_69UT|MhV{DZ`pu;g_w8+kQVA{lDycQJK4
z9}kRJlwfuGyWSHs-W{(hSGg@;EvfR_>3T~PTkD&~+~kP?fxoWz)^!K$4!KvU86&;(
znAxkeZEH$buBm3qNG^WX_rRAo@vFo=FW+rTGoI<*$=t0v>!$nr)fGM;Swg>E3yKp7
z*vsR+NjQ2!(vcRu@XU-OtKUC;E7`lT`{OQ~%<U`oy5GLoA|{?$_-Lv06P_tk8)H^f
z-m^-+ekA6K=Ks&9`qr%77|+L@_VGd}*B5o)Y%z`G6M3)1qbyFnoVxbbN3O|J&j$bd
zqES}EHtW+RttVXC`!)aln^)Ln;Md%%7A`iaW1s)&ci-p7+cAE>9Ql{A;)#Lr&ZWB;
zcfFjiDf1y;@zPsC)vI3xR!(O#b4^-yG;+haZ_5>*y1r1KobYO?keN59igzm0w1Pab
zd)$}LtFA2dKmOptiwW_UgA!WP)wI{leq_0QLFPsK6Mvdxn%?Y^xcTAaqP`t+EgI=F
z)>cKnx*DN9sWfi8(e|5>?OvhX-X{eN*BNW@ysUj^)Ye{mWP71Pw{36B3q>yR181Z^
zKQ_AeoVD&OYhLNAJoU)Qsd}4EPkFP^XNmrSLrblc#B_IExnRPuLU-RO&kf&qOnhg~
zblXpN?UxXvn&7DJ6|+tpU3#&zQswdGch}X#C7AM;NaP<04U#hVKR8$Kz^14BU%ctq
z+impfNsHs3n?IOW&F)!q%fQ(~qSVtTw0?q<$%LKEKQ9VMTxGu~F#C|9Uc;jWDreru
z>Y7~0I4Yt0JuR?mZdvz4rzSC_r|SI*ej!J<y~#{_^~>YUrd7N}x97&Fg=TT@$?!h(
zC^9%;MzC7v%9M?Jy}2emQBM?|U7-G9rnzCm1HFzjO#<({r{7TQ%xcUpip<^P!~Cyg
zb##vDxsLKpF0DPaY3EPAa1(Y+yYOst*0hU3-itP^HVok2n)RSJB(&vT^OU*!pE;ha
zu$$7cuP3)&Izx)7LQHYSwMzzGYRLv{X&+hN&lG=9qL>@+b;9XTYl3gZaxbr-YYRD+
z{w_NEQL#6JE1+`GZvV1l--5i{VnR6HH2+<ls^prxxh$dnyPAAdzi+?T1y^pjjhVAj
zbhMVn%)4vbKl{3Rf%tdJ-DlhbDz{`F`Tkj#MfJM#9_<}^hi6Kk-BF}*D1h6=Mr)=)
z)~D+)S`+SQ`sX!$H##{>KvVqNv$9ip6WLuG%yb#mHg9y<IgRI0{LYYdu{>S&YHqt3
zw|koVx@gSxI57Fu4kM0OGY-W*e$VT!cXoK1zx=><<*-s~tee&66pobAqgVTUEf3e1
zY<*i4(fhqF;s4*tl|7g4aztDgFG)Es`D!M&>k7u13~hYBCw#e-YTIwcc%!w7DKjVW
z&Y?$tJ<SBxzIJ@?{Ld_N=jsVnj;k;II=YhS*M{&G3ysqP|J@cPzvf?65G$;@XXE7M
z#fEE{yE@Zvls8>yycT)mqj1~Jz<rx}Y=Vm}F>Tr^_NAgxCO2ts$&R*b2E`U!T|v&d
zvu-6sHXWGq$<pe{>X$F_-{vg5o1b)S(cvo7W=Z=4XXl&mTgzG5+@#!O?(+WU+V;uX
z@|UW275+8ho_h2^-n19nqV}y#mh4=sxGpp{?CV1rAFs6KqJ1wmOs!iG@7v8b`(J9p
zYBTw{9S51+Ha>WGGtKGxhbgwZq_+IAyRMt%UScKry>z<W!n~KymTVKeu|vdTBd__f
zLvIWNnpSUXsM@_YaQ2~vtQkx}3tFm^uP3Qk-rT%A$#mI+Ii>4LEvikw`+6vxPN+JT
ze><&3Wy7T@XPTyO+37N|b?KV_Cr{42FI^iNWTeKi<4(c6Tl1D2G@P#|Zg=tM@=XQe
zNw*$#oMt$_>{5NS%(;W|H<wx)=rh}Boa+4{Fk_Cd+26`U!97RL2bu&wa*2s?n7Kdj
zy6$za8@H~B>ueD?C!(qzbD_p)f?nY&t*~_K&C6O0cgRfJc6!!Cm%XiQrWaqcoziVj
za(|(7dxrR><-QTOnGQ-6Z~l9gtFG-tozb6_5pOv+uR9mf#gZ_yq-*LIr_^OkL0n5h
z(<HLa`c1yIA$n)m+hvdH+qT52>wR#NY!1-+o+!3?UeZ1RJzv8mtDHg}y7{Ia*?xbW
z+rj&pr%!(^{d?=p&ElII`YU9Yh2IRa__U61Q%;1mii;Gp(EZ*cXEp1L-m+|&oxDgS
z&MfS0`Dde<d{4CldXv20>PoyfxwS|~*?a4x#V=0H3w+rp+_fpUEm7vq<;^Y+o;q%F
zy17t9+_h%A*ro|94HiUR^a!ak4zs<-o-|$KM&i}QI%{ICeAp_xe)E2Jv)z2vyKOT#
zrOQv<;8(BX(0r2X)P2DvYv0V^%Qx>Ia>_JoKFv6#@OVnhhKea|)6|bAt_;d_@jcRZ
zG&E*=`prXKit3R+r#KWOH;S>oR!mJ=?7d;CL71rbkC@HD4~==8)go`6F?Q-W5o4y`
zAtbSSo^P%9g;R0uN*t3m`piFeTJ7@bgwM<VpPjTR*0;K7x3=wRnPZK!nE%Kv<Kmz6
zO(aDi?3I)CY`z-`H#cj(Kg5vfUura6ywt~I^~@sEnI;(#_y62$n7Os!N$wY+4-2NH
zE8UIqJ@L&VefhW55=BN)FZ?#~hHpLE`!+6SqpI1;_YuVgPgJ-!wYZ<^oHFHB_|09q
zJb6b`ZNCSpZ+RT$yLqFrey;w(Q(~*uy$ZO|qBV0hTW!QPWeJ-Z4IM`eU(fipDR;Ba
zjZN?7y?txK8X>c@-{jV+CD%INeSE&d<e`_(BNJ5y^)7vtEta9#C5|rP4U_6uImk_W
z^Q!37>l%|kZ7XL@>n!B?>8SNcEcyz^x|LB*3)oWlT5}ld70ylBzkHwfMEx0?B!oBR
zn+WV)oSL}xe^SDy0|7f)8$EqZW?zVXeONg_wp->-+5763Nxo^D)8e+;ycS=+b#Z^K
z%Bs?o$!o8Qw*?eTNuGNmq>umq<h|yf8c$!Jzkl7vr(VmNeX7rJp8YndyLZ8Pfdgl!
zwisAFoiTgOgchA8iGLQ~)qS`t^o2XqR9pAG7IRcSWeayseB1B4oxQC3s?Q3obwY0r
z&e+8prKNU3FIrt?+ODJxd-{WnU97nuS<G4^JMHE}N4A$ixvbd=j}LX-lTYg`t3K9|
zVBqVs;rNtErDr>D$8Ty~a6G7sHQ|MX<df}zPu`hrNH9nhG+HXH_eSvKzn8C6w|mVC
zd?+chQA0`ZhOo13^p6P_X1@8{$8;>ily~~9-dw3pFM-Z4q9?PAEVx!@t}6Sq#O25q
z#}uunRa}}G#r9Y3?tPx?Dpa@j&YP&a{U7%pR$iO<ZofC*yG<4=xtTUfrB;hAsZ1=4
zwm2aE?5e8!f-70EHWTHqEXg@7aIE}jQR2glY0s8T)ofppTrM=z|3{Fp*u|7fQyl^?
zocdHZ!`mo-=Ojy=XM!)bv}qUVCcC(b8E-u~ZO)UcS{3}}`}IQRUOV?J>cji;*?LaQ
z+dC6(E}R>9JYe%1hyLP_!c!-@vyGDv|J}K_FWTkq|ATh6?pJcVHmu}{oh&Hv)9q2U
zV6EAq-$L4#D|f6j`0qPy!Q)?QbChH*akhNE`#LgrPh(2pqf4rF59d@L&U+rmkQ+QT
z=5}-OExq8sjp_`u6yNIk8}a1X=WsF<R5F#gu^-vk8D#EmaUw&pr_Yy7bmENEu+z`J
zNl1$AW0t(Wip_QD^baCCp6u#&Sg}3KZxiz(sf|8CQRdIf3p(!Zcvb8tIGtZHdKsg5
z+moc`PkuM}z0YoVb~&I^cSE|k__jAT8zmA8|BGxq>}yl7=B34tJ-6hzlMiqvT3!%)
zd)s8|m6DCO6g&N=`LesOnwICt`;OzX%P&dB{T`n;J-BHol`bD9Q+Df9V#BFtA8#63
z*6+GoxVBPy>G6N^jkSGm)BkTVut^Z!<+D#W`FoC#k*f9r-x;hrtWjRtpI4poX}X|t
zIp&|+M2pDVlM_D*IP1OH*kyE6FJ0KK!re6W@agXd<+hz&x#%t5r$eF{ujl$qIJ)Qb
zp_n5=OY;tkv>4bQ*`KTL_Uv$b=%iGCN#kY8vsUybYcG9N{l=w0AiyZK>({&myTx7H
zBsnH(#jxdWa-6idKzPcgXuaZG`<$+>#Nb4pnJjNEZ)-lVY5j_nsA-coz3~R!e0^Z~
z(JLPhDKvVsaA%yomRo+sV}t*G{VDz0+w!N})GPB<*b=!|Cro>bYtH0@i_{L?yYflk
z)a|(IdU|``D7%TOn@#_(w)^R{3lFdH94ZoA8Eq*(wTdI`m)?a}`r);EIJEAnZA#2j
zt5kAge6vt^`JsLD?tV-Zx^=PjjU4mWeKGOcGA@46vf7vXgx!Ozow}I5ovcorQf=?j
zAbiyz*JZ(lT*2JD6)swdlXSvvU!MB3FPxu!x#Rj{o+gD%{u(_?_?(VOZ;Q-*pq*Tr
z9LgcZagkSR#fcW5FLNt)ZOyD+%*EyKy~dL}E37Y8ut<^JYj4bvFAMJaGI^^d3RH{V
zNGVecJUPu|iSf6R;$yA%Gd0@|J!9uA4!xc6F!^Bg-P~^JBc>*^%OXl#kH^k5Xnfkx
zYAd|9!c8FGnCZLSs^qBUn<QqbIjq02Q1(o5Yfba3O$J8UpM`hCPSULGljwP~peKpx
z)q<|Q<uyn47D_cL=vjC$^2S+SJg9c+N72E%_O<yP!2$W~9GoI~37Q2GjB2W!XFS6s
zeD5Cmlloq$(S54NtggFh9q#LM=Ke_fV{|TI!t9oU1kZzpuM{>f)#Bmq?$F~n618+@
z&aIPmp`{O=PHBxx<NGb5E19+?;NGVP!9rVYK7P}U2t6EP7Wq~q{1sDK*ZkuzH%+zE
zux;!5$GiOCB(<Fe2050po9|voa#q@W*j$x6e!&gFjnC6`4r*vRy0~1*-~K>7KKAJM
zh}XF#ZQpNwUzal3CU0_8;HqPnl2S#gIW5EX#H?`JbUyC%As)Y@?Io}Hl+Oz4?7gqN
zl({@VeUrUQ*@O!_-uf8qbSqfB%p%08UN=+WQOJikx&IU<y1$p()Oa@Q(v7#42}`G^
zuM~a5c1`2WjZ=Ipr&W~y^a;zF5%}!ot27DMbx#y$HE}a^I>>bhI>+=a-f>2Io5kFZ
zUb@eGw^*lo-#ooi^5#Y!*6KUkB5v&T{PMk|BWG*rwA}bkfjJg7%f4;6vM+Jll`~JD
zE_fF*Y3GU=dbj0#A2v4ly|vN~iZ@d0(oT%s{B$P6${hdDz72{!S|Xn&ehu!~#lAi2
z{_aOfX8NVJ?|hDAh@G1E*i-g$D%;eYlB_<D6HYyMm*+VjH?>)2>i+a(>#Y?tYBP7=
zJ@)_AgS|<HKcu$pP(A0HtanUsU26JH8Ro-%o8H7ImQ87Lxe|7s{kDhPi^jQ+4!L{(
zI+ymMD{=BEzlQB87krzZvMrZ;cJ$79?V!w#zXJNN*0^k2s+Z`zF=$1BWlSXJE1k1z
zBX6Ia_IQeQ;(n)@WzT$mv@bec^GiBu`P06yN7;PmF87m|!q;GI^&$C2(~a6Y=D{I;
zEzi>yI0d|9%h}fb?MT&wIpWv(1=Wt)N`B+t^l{Z8<7GPU#2p#4Z!%t25$DT@EXurh
zx_x!hlpD>Ze@-m<%6I>tfZX|q^F?~*U((wqdnryS`<A$5z{4$bzghJ^TEXG8+c|Eh
z$OOT(Pu@y7qA_<{1ob1gi7l^De4!gN)w`=Tz3{a5CT?HJ_LWil)|`(!^vl_4!@L;7
zu4Npoo2FN9sC9mkCSEf8aczZfc=qPGmaR8BO_tSmT+ZuuySb=TN^Ih*?=4e(TB9DO
zD9-4*v9+;U$MXI<ug>_hfhE_kx3%&!ex2duB6{X&q})pv;omcg)uT3@b-5?e#dONe
zAouOTRnn~WvO4p{C&rlc?zbqID*M;#%4w0{`AlhZlxFRIsQB<vv&PQfD;Xw!*=nt|
zL2S~=+M>I98!gua$e;fG=HICw?8g|YlMEs?3RYHUYrNN=c(^4vHt+ROm4&)*FKd`@
zn%5I?$MSLgj!Ca~$<C_yY8F0kOQdT^XwbJ^d8g+w#~W~jv6Y0jOZmy~=V00KMt$G@
zg51<YJIs%IM|f?pS-O7es?HcC{S_r^veG$zMa<=UekpDDtXNyeYXO=8jBMw&w%m)-
zk*Yo?=n|v4Pbl%@O2PRDL^9;`A8Dl6+zmOvl5;*%zOVdYgh+JBu}zY)=h%0PaA}1-
zvi8_i#`n@pq;pY8`wnNuV_jYowpY*dF!`AO_RgD4k8T#u-t_wBX2S_PBNn%+yXi&#
z{Qe?3nXjEWdG|fF`*ZeqP1?S+byZ9|kC#=y#_aALO|2a*jZBu;HU`hVIZa{~r_(0!
zH%?9W10y_M{ylSg%KP#J1%+oT=R94i7JX>teqC?h<7u`@oGlA>X)@~Qo?PAW)1k&a
zDSyhBeftFG_;3{N_qg%e!t#O7jzm4{6Q?&6%$pEAo7dpm;nTBizJA~Gq@geK^Rfab
znf+#krH?E{&Gq_9x=kkbB+Q-IKi5rbP3aynfptuw0^*IU(q>DpUaK&}$c1rpK_a_q
zM%jc11yLVM**i;{0(Z;%#LL?YuKvJte|Ldp?iQKssZIAizQ*OawE8aCD!AEvkKJ@(
znK$$9g~e@Q`}+0P^Ba9}E1j8-zL^{|x#qOtq9<{z&sLvewp^QXYv197ZJYayl{iz@
zFX=54txY`eV)Ep}nai{;yq~P(rh9Wr=B#ONSB7q@GVL$EGL?^YUyJ_{@pPrDMRwWR
zf#w^xsjQ3_`gyI@`sJk0Vn=U2l$}_$>1D*+uooM?nC;$VK6A?d)d_c|rIkNjVe_a{
zVjKU`_p<McuP4v<$atim_D1QHc#^Z;GzYazeZ^Y$2NoCkCEcVF&;Q_hzBf!|f6V<3
zt>Y~h-`u*Nyl~CBbAJV9M)g@}Mp&Eg4SQnQb!6Mj^GAgi_yuWwy|p`wBjaRa-i#6v
zBY!@f&NFuATV*fbZ{7aiseXsVdJo+?cbDlcOLo>WMBL4?2=O_(>*NU~CC!_c{0}{_
zOTU~aeM0SG>>JZ(Q|B1o2rrxQcv{O*uDN%;F7U3q|Dh$(U$3uvO_);pjlyTXIc?XM
zD&13UVk~00+MhZ5{)&55919-J-X6Yh=4SqD>bK7;J#PJ5$nk2$)Jb1fE3-^HoM3XN
zZDWw5<f=oezkJxZDo^lr-Lc9SeA?iyB*4Gvk8Iz<86TNGAO0M@CWpgRYH_;A#j|BL
zqTPKnBj&WSCr=F7_4bNK<=<VpZ&{Ka2IujfYFwy)>6UEChLc<Sto|%BOlo4u=@l#V
z+0lEnSnrO?Jl8AFyE#nP)OhVuXZhTmJja1IH6;H2i7t<{ADeW<OrB<SCVL9~?wA?9
zocm;1;^#AdcS@Tc%+F4~>9g~MdC(@!i5kbWC+2UO^u0eZ((s4px6fzaosMOXmYd-*
zdBZmC^xz4l(sn|V#E-rAGGTq_c~i$Ac}~Cs{!8m@49h3|5WOhRx%#E+$q19kqp8XM
z&p3i!%~DW$vpM|x%gvjg@;%M$t8$Ioxp}_%!b9m<Tlab2c>SWeY)SZSHzS*Caw<pP
zoaAXPnd^N-%ZK9=>tu;(9ZI&l-ahD>%W>IYv1(5=bK@%M!UmbGe2QKv=f&>0aGjp`
zR_eBBY3tdu`m-&!1$;DZG?3}~V}8kA?67x$g1kyeS&z}(XK$UJs6V}^KjW<$qrkDY
z{Dg)58*MqHCoD~L%lddvZ{8cG#Y#1P-_9q-AB<{Ge|n&3<rIzGt&Li)g%$yZ`DS}v
zg^rXgky|>CH^ytxH=o@`*V3(3YHHqE9hAGgmVMH+4EBRZ=Gv#`O?^?<<hR*YpuWsu
zyY|8b5n@}WYVEs`V?6hi)|^i|8rIDng){d&H0*vT6TW_Nmp<nN+vpW@(^||nsJ1+N
zb0mD`&$2zYmWFcp+RV<F)U~JHc%$LP#P>eB6-ONmbqsDA^ZpI`x;BY>>&!hlWea}z
zIOrC{`Mgx0e8cYK=F9sQwq|WFNY<TNaAo1fj~5abC9Mq42+?`$6Q3TRd~oi!Z)?}*
zellHgVw0H4YUY5Z5ZyT(qL1EmNGk=N?(6TXOkLn@w)?U^cl4s76<aK4rzPgio~Th5
zd1hys=i!w`5#IA^rGM)mUz^@@RlzEPeP3;y#~opIWxlkYOOqFL$A%pJXE0-FqJ)gw
ziZ#s(zol^-`Mp@7J9E|E+&|h!_a?NMR~?BfHROE$wW*9LceT%P_8^;uGG6>H|EBld
z+4y3`qCWzcHzfPAb#~o%-1b0dQdH_H>p4;X4o+Caee&GQnNc6wnhu*6h;k`DKPS+3
z*5^-r3Wsn-bi{!k=gk?RSJ-n7YHm`Rao^Wp%&d`P{pnL4r=~85Xqc6wyD{@~w)xVt
zzI{nP=QJ5hRxb`oek1s^`33(8lXrzHWHZlx^E+qFrq-IOW#GoR&dVit!ri<c*0)?L
zNuAOse@F9f*5Zx*<~(E3lAc3iU0mPN{!Z6E$636Zlc)6HX_f6!E;1QH%o1xeA08~a
zo)UU+XQQX@n^YwM-t_^-;=4CAOk;Ei*nah>n#_mwt2CLk!V*r0Eb24~H`}~9Klh%-
zrK?kRCFO``yh!PqaO91S_9_dDt#_EGryb&*VmU3tK2EQd@n&96$)t@Ek*EDWJBH@S
za+l8jvj6S-720oZoao*5c{_XUx@~;n?)o9sZtD$_9wZ%oZe+5Tt!Rc=QpdrY#m<*M
zrCDYNmoQ&UHZuE^FvV>PM^${;mAJN-yOx}JIN`lm%>vHLOpD&sx>o1B*H0~X4Jwq2
zU-3)B?&6DwRu@`@)-I~Pe`3NxdpX7~iMc{2FL_NWT47Y^S040P?7H5T|NIK6%`cXp
zR?)oewNE`DFMw%^!d#PUoEsDulvY=NT&vTv>0Mv-Z?%^9E>m7l+4wrxcm2jUDbj(d
zA6IY$&6ZW3?i4NX_hYG8#C_F>wkI2=ynS5yL^?5tf%DZXoA@sh+#Dx*Uxz)MV%Z&)
zW+9oIa{BiI@rk*)lZy^deej^Abhb!A%M2Nd10AxHWVJ5d5Ek;A_~81EsnX#mWIduk
zStTXtYZ$g%O{q?sExTdUpVq`V1)pmt>~anM*))BIJKH)nOG*9}+P|L`ZQZW&$LAz>
z&y@*tI#leIi!_|{+i>zn!?ZOX%Uab={y!JD?q)h)W`2_A#VIB_|2tE@XWn{a<9%6d
zC+l)Sc9lzo>o%I4XkT<E<l|?b=3BMA3)VZlbJ}-vwMlpQif_e}CjGb1m+?s7x<q6V
zn^myl#{Y**bW1pMEjp{3Lyme)jsNzm;qkYe)?;;cNxaJT>aX8*?Fb9Kt0{D8d&2~!
zxCK3*qckTK{C%c)=JfB<i9dGsdOgvf$&~(LD#zv4N9V5{+reioG?n87kL87jDU#h!
zAN)D6cGWy{E}qO2-#D~>B}`I|Zf{;Gx-k90%uN1nZh_Y-y2h9FH;PVwtbTT}sJ`_U
z?>WoW4DW2RRGGKOQE|fI-+HHwWa1Vb?$~(pb-mGzSO3|i6v7X<Noabv1UKm1T2(9W
z+!N`UUhwW+hQ!3D(o>HH>QB6(7I`Y|r_jVjx5WNjTeIY2&GYS!4yIgll<u(?U0=}m
z!r|EU)-;c~=TE-l63XAC{BBe6Nr694C-0B>DL!GB;>vRAShcLhg#sIu#d*$@som(A
zx;Cn1ZCifsCU2W~g;oZWCd-LC4L|>IPmYQ>Fij;n`SXj*PlTtxIsM47^-V~s$mH-C
z(X;@SxeMQYc3?fdp+j>4OZcUG3$EOpGV4Q2N8k>Jb3vVFycyI!fBNs?!S+1mo5$DS
z2TP6TY+k6S_4C|@#hD2r1&Iug1Z*ZBW>TL1Rmr$Frsu#q#=MC6I~wdhPULQLh+Jd-
z()%4Rr^OkIzWgmSE?h7F_HOQB?UN@p-+8XPwnbUv8}kar$H$z7zRsP<($!yL6~wYx
z+3!lF^$!lanE_`y`Ewrm&Z;nCn&a)tFTw2PzesrY@s~{<{z{Qj-(E{M7Op8c{Nm8>
zm)nC*3a{APxnk}SE(@u8A!E}{HoY6YY9Bd-yC>+a`+D}wgmm653?Es3E%RVxj97l^
z#wDR`uAU6ZtOkB!>lzF+PF|j6)Z(KOa#N4t<7E-+T_QUxk|xaV$lbL-`D%rLCWF5p
z%US8G0+&sG?Y<zmquft!uE`EAX6FrFeqYpj+vPL_&tCiZY{slkL9gj$>Q=(mJKJ*^
z+MIm0xrBc)Q8CT2|FwP}V-EM7wl4Nns`mSw1+pVPr7TupPf0jGxio0s!6WO|CYH?G
zc8_b_`3tv&4NH<1^khVTJu_YI*OCC2jGQmhCnVTc$+^UwRa|&$(N&qP&vFdr#`~-m
zUf}cfs($DzyZ>v=`h0)NFJpMO>`>Hp{zV^|6E1FwWe;%R^q*~)nZCB|{-JFXQ=-io
z-AyXoXG&z8xM#R%i<0!RD;=RXrq5%37BD}ctK)UhGf!8ptNovk9aw!~*D1mEDt{KL
z`S?t%nqZi@JoGL9LJr?KZ-2G^KE2~&nbxf<D#9KQPu`fZT&Z&Tq>9c-vp#8QuzAEM
z_OZEc>=awWv`%2UPtW@gHez;WI$Aqjv>GRN@W?Gv*V=!TS?c<N<P$4|!#ftOuS{y>
zc;F^J(d%aN`F*CxYf7BHwD8-XoZoGkwV=ARFJbc?rss#`!lq2P-a18X@tbe_=I!5w
zqN`t?e#LJgruR;xJgPrvBbR*{Q=-=V6PGXDd*S2u{7T7=M|A~1qvu7~Y^+E!T=ja_
z<i)%zuQYDFD<8~kop>+#?R{;9LdLza@vYi5BG0Be{5@E;yQAnzz3c^_9jo&0Is^tA
z9u5x8^~{}9H1FTf{OiSs1x0%ewv;;m^mBFcRWeayYw@(1t@B~UESrCxCTjQ06bza(
zB~&E&r^)ur^z3VrQxPz!<WX+Zn14Bh@2X4oEUmUJ&VR%YWp}SwSR47{!!o<lO^TZ)
z>s5Fy%X@OI^L~|(s?kJi=83HRw~s^_%zbFi**-sH&wP$Gla0O!yT4fS=TFInvnnSm
zq;Ab$^hx;2`_AH{5f>*fkWHWV(otq5OQ7bgjt_Q+k4#Pc{XazJYZ+r=RQ9f&=@<Uw
z9_0JM>u`Lz%ZtA!=D2N`9mJ&m?a0d&0V+>CUu+b9cj&sw(@&e(O9ft-c8UjanJ@-K
z^cnY`Zn4PKc6XYm<*{)}$=<j3ZyZ|gZ4x8&vqL<-oZ*&$g^IvLUwa*<%X;pgKBO{*
zYBcI>;nm=l4BfB0DWWp|&_sP5g-4cRML7&70&h8VHn?Q5nw4Ezqg@@JuPie)AVI)?
z_no5HF0KjU@<xX?Ht5-&{m+#C<&c@wd!-jKf@UkOvLzjGI_fLAdn)tEs{OsIJ5%Pj
z7FH@+>Y2=ZVYacY+vGNDf?nafpU3!^Zz{|5@aZ4r*vIa(;<1^kpZZ6Qwg3f|qAen>
zN{{B`7=3L1xj8q;`vh;w)kmhvFQhyao=Q(htMRQdOW}$v_t|5Z{QNQRNsEP3@5eB>
zJk+0Av!O?v@nWEO`ON#*O4%LL3Qj!@QN6bQXOjQx9q~SGT~<72W+^UfIlAmj*oW&L
z7T$i=m%lUUr1Rt?-HnXrtu_AoLBQ>5hTshik7rLx?s{6t&1YEVn-HzJap{Q_o8SD{
zbls`(;Eo=dbLOXIb$yPt3qNKlmTpd3(pK!nHm!L6!S5^HP3iev6keB;8S*6JT7Bbe
z<zt(;|1q$vjq8lP5?*}c=1vtRJx4d4q#e_=?&+(zvMy1rQ+90LVK9f`p~K2Y=eQaD
z98Jad?tUhl?7Z!!lA(eU|Jx-8W-r&b)j2k`vG4-l%1`SwlP0tG7KP~R@#r&K%$>=4
zGpu0a$K?lII9(3xTCx7bbyX3O6F>ak?X8ow{q$k>A8&6f`6iB<ime4tEgd<6_bS!}
zZF)LSTwKX=j(12;v6si5#ev=OIe%X8%t)I(B~UE=NU)1;-!T#M1Yhx>hlflTb?>+N
zcxi4%%v7_OD=rtyC;nQyNYW_hRsKANe#iUu9{jq-TK~T2w|!LI9<_X7_bJWHk9@V3
zSCmYrv_+|y>=OQ1@v-K(W%<Q_re}P`<372vTw>BIe;8eO?Qc=sAEhbB-)(E1d@&=z
zZTXev*Lu&E=uA50uJ<HQtzq`&jSP2tpRQqArfi&<{J`hV|F;f{jW#McJkDBEVsv@!
zZavkLBD-!TAM)Jx-}=g<<<D1ynoe0}Z!){9{r>-}-*0@BjtrVRZ-ut3de)TmL+e+)
zU-r+eW4~IpfUL(V_Z_tz^UECOcidNsshKf1>1o-@e%A-@_DPg+Z-`hcn3iuFQOmEN
zVDs}+>y_xZb(S&{W+a;RDC{X<(@WU%<U#SDV@s<y)$W<{YZ8BI;||f&jcFTtj#zN~
zFq?6ytn!@-|00zh0i|@S#ACnKJ2jpXz3RGL;ZK=C+oSx~TNW2<D%CuHX!<$&#QVC2
zQ=Hm67o0p)zGH)e&YEM~+c(7iT=2JGwe#;v_htDfZydV1a(Q33TDoh_Yz_<c8+tq^
z%sDO}jZv=FcV^hFa{ADzHK%uZ&+xt0wKH<Z{|4RL=S90@J^wy?e)N3Rq!!ajZpSSg
z4jG9{bEF8a-u2;<PRrBQ+$u$5eRT_6nM-bG&03Z)YRM$4-M7drI?8W#xAx-xH!kJ2
zGcLy}`E4)BJGk}U4yXMh7a3EYs7{Ou?(8;u+-;t7MBUQhU(ufWqAw|*PVBt1u0)FM
z-|wOi_x>&V%`UFzF|mHyqpF?Bhp#_nd-j3%VsZJlOZ_<^SxuKj7O&s3?b&g5cZJ<k
zoi+>d<!Y>mSsa+JrJPj9F_l%G;r(0JCl{vfW4Y)$HNh@)&k3KhU$<7?`LXfij=<+4
z8Ge<W+#6c|E^z+L_C;q^{LS2a@zPacEH7fazC3+$h&826h}ku0W9|FH`~A(Ilq9zH
zt!%6~yvW?8Q(fPWg-P{ztK_?vMr$|{552s2^zMwF4%XnLZu!p7W>T$>x7a5Z%-O}G
zba|J3VYYf+o#m$4`u6J6w*RP7+V9KuB!Ai8;OteuKdw*|xR9y$YFbpx-@W(UJNh4c
zT{-aS_wxH9o*|bF=V`9{b8b!Qq;O*^jb&lygBQ$a^N4!z$iimcM6tdn%LR@H20WHF
za#0o7{+PR{A};^C?r+ENIH@h0cN`boW5%V!{_E&R{k~N`YzH0(wEv#XV6<rNW5)mY
z{_b5}X7PDhg}Rvh=|>;VCOPbK`gb?f#`I`#RL|pOCr|9N$~g2P{pRIENv;XWAH($h
zSoSOZ*<w6*>dh4o4q0nX&pFp!)g4~?#HaMWmb={Hng`sW?c0TAS=hwpd*<-Yh;qHW
zW2>0C*}9__tGilW$$6A!>}Cq7Y1U3qZNBKLH@TfbBH(XQ@{g4H{D(v~-PUr^mv}ej
za*XJM)mQTNOZG+k7OsBW*12gn$KSmo?#riDYE=K3-?UbcMJzddrLMoyMHan~8J)kh
zc%DAydVcZ|TcvOx=l%=7@+%AvtxP?<NPYek<+h&tu`vdgTh?5j!5iOqKjO~k$SsLI
zI!Vn+yB1p>*k0Ky_0#&*igfFr2W1-PH|}}Nw!}$IeX~h^((zZH_K0u(?3%iM(upg3
zzMQSzG4rR{{6)_c>i4^c+rINku}rwxk$bw%qMEnHiKSWVj{mLzgI!M}&j0w4vHEfF
zuI!@$&m84TU;eu|F}IWV-YWUh$;|R^A3fV?^V-H_$B~=8?yJvxO(}h0Qp$eu>yB9=
zTc6ykXPh?U@ZX+)$8LVUdeZjf`t+*Zi8i|*mzFj!Kl*vX-NSFhH+dQxW}T2S%&e*9
z{GWfBN%znXD}m)lC!dJ#wb;qOa>Yzb=_h6UGnO6Pvr##|;yP1xx<tCvRt=_idau<2
z<DL{<_-bl1QIOqIMxl!L^QnijFB(*@E`3z$b7BL_qmHxMr|!Mt-k-UH@p@eIa=!Vs
z$qMmpR+ldDGVmTRP<g@|Yvq%`=Dt2KD`k^#Z+z9+_1$igjEe;htv-5uXQ^eLCDST3
z!<{Ko)+#1lvQlx1mhZJKrN6C9N?!7cUt`JTg>jdI4sVIEwmp=|@+Py(N>^dGjgV$d
zUINek2lH-zpK2SCmK0Sn+wb_D$ErKZ9{kI6nJ4*k>*v{@ul06+E;LE(UJ;|$vQMx5
zz4U7Nn~8~Q?!WSWwor1rWx}>kGeb_eFs$96^w)Nwlzijzn6$dnQBDd!bmm`q`gYSE
z?P>n6m<~^t-*o$Hb9P0`^!@V7?E{M(c&B7E$gh&@{vY4j*1d}5koXam&DHOJomHJ7
z`0>|O!8@y@ILh31aoB`r@QWJ$QJyZee&JG|KdT>SY&C6-E~&gE_5SR{^yeK-a~^&E
z#Qg3?k#n8&9Q($34u7_;x0PGL$FyYOI|H8_<CF~RU=7x^$%;zCp|)jjHeKAR{V-yI
zpR2CGYo(V=Uu|Bpyk7Xw(0Y~e@AT_$8D$SDE=&mj@PF!`nvHI+Z+DsAS+T+JZ$#<t
ztIpAN_n*F(b(v-(;km1-{(G;nTuJlc!>8nGKZQ<y;u4$MqgEey_pwnebAC*QnMMC5
z_0z4V?Q2$RTzT`;FW$NB**o>ScD$0SWel$Iw0NamI?3T}$xt*seyvUHafa7R55Hes
z_jkj$r!RU%PTp@m@8f8;uhuNp+pYbc{;zp!YDA8({K?>Iow1@hrJ!WrcK*)Y?Dx<8
zn3#X___?KCfeB9bAN*PsmOfH+Q8imFddO|xAKUOrd6SI!CV%ZqVfdYLR82IQWycoR
zq8+_bD-5IdCw||cC2@V{U(QU+iT7u7<ld8f-R6}2XT@TV%9FJVXP#Tztgn3aZr`^j
z(*-U?*NSQB_|D%szjfWYXLpK%LzG^W^xs^$O#W&B@73!4v-XAf`#oV@_G(MB!}|4X
z=U1HbSY!8=<ywr;?U0#oFPR>WF>XwMRlK-PUe!tH>As-fZW9lj>+j1dm(BJ1Wbx!s
z(SoTnpEMtd{`{5KCA}!Bd;PluPwy|jn{k4ddEqL)-)%z75318b4=w6{u}|#GJ%-nB
z8RPDll+L`f=4x4CVtPnPM-hA6N59pUCJDOrclVc{^qKxn)wt!pec*%1N8joBUh*v6
zaXO%6{=a8i-()nss(dnI%A?m`m_I)Fnz3MZY>9qd_$lw8#)T|>?|-;G<*q9%RS=x`
z(KPFf`|QdFcK6orCKj?&{O>K>aj^A1+gXkLn>!eeXx%xgHraQoUVYW_wGm|saR#pz
zyKQ_YG`DB%<-eA7<;?Hit4kjBVp_It-@@IE@62y&&YH9G>_TZ4kEVrB<z>><FIq)@
z@za|<qfPvZ>p}<F54@J^_8qi6#iriU#>;qg_m<gm<%a8XDsIXYY`!!7&B<%^-1kN8
zYI_{Rb{thb->Y%!$JrD9z3Ob|>)Gu2maICu{N)KovB!SX*WS0z4STR|y7~XmedTXA
zn$-F=1)cj`|2oNoXWfO<2a+1{E-dz%XeU{4^6KuzqUH}?ah)`szVzJdSMSq5U)`_2
z@4R8l8@)8|sW%;TZhkPj+0k^<_RPGSr@k$DmojPHloi(d_RZK*q9X3Sp<!ZF{Nbd!
z7<v8A8xt2~-m$7E_;_UFgr66u?lV_#Vd#BvYd@oAS#tvm(=P8B6HiH&u4HyE{BiIl
z`?9mH9~Zs+{j7-ja=wX5__kWkHvb1g4hh%4uUkEF4cn5ZJO^6DR7~c+kDbKLbFZgg
zCD`Im+YjyyT{)6rH*@#7MV`=j^5j&J)rDh|7dJM3?Jn7LnqkTG%^&nUHtj2SzI?lE
z;iqKLJLc2gWq)eUTd{MYSAULTLcyu7doL4wr*}1;s}P^-|EW{>Kn8ogh&`XfYmt>Z
z81f!VtKRRt&hTgw58p(V64lz|3tU_0>UO4c-`Vx)oy<}#q4$masor;P`tih89$cdR
zl>39U_>bSmOHX)}OxKf=yDC~>{e1_+o~pT1*DX5YF)Owvs_uH~N5`EJVw2v#JaI1Y
z?6=q#ZsiB^_bv2{S;1LR-u!iC<3sl4`6cW@3di)!Gj=@RvWW2=yG%&@hlIYL8{etM
zO?><M#Dbm0U(ci}UkJK)@Zyh~e*HQUoS{Mwie6s&dcVJW-cFH_XVVVp-ddo-k}f01
zD7AU%FQ3&l)sN#ob{;bgeCB;K%2lt>>}}14d6Q&X=e%qBc&)AYqn>UYXY145PiOsU
zmM)R^e6D@2c&nGwj~Vly_shsee7t#C!1aPn#JPR4N|Akh_B-CMj=wzpO(pXUf&J&_
zum3sG+I~^m5hKI*drmHRzVb)-FWoB}ewAi&iL##&2|LkXnr@@IGw9xp(B9YARg*-H
z#D`XS-v4Ci_5V<u_*su<U9WhqW~bID@Ay!_zxH^zly{!UiT|&D<%=#}B2{p{fq7%Q
zdUXHAtGxW?Pb4Z=HHCM7(KA0cRhsegs=l*J_c4d>%fC_Oc<b7dwD{M_lXA7rd0jiV
z=~?=fc{($~V`Hs6f8SiXD3+BcR?(bm%C_~ZJ?q)-FWN1zKkvr;-7LXBnseSRbm@_c
zi<vCk-6+1xkS8eQR`-AN_`gC0A|YMp&Hd~p-ulI}-_LH$Iq|$w>WrSt)SiV0S7y7#
zZV8(HHt{6mRdG3?M8#k2ug<?K{C)4LNkQ@(@1IN_ou%8AmW%8XbK<_g(5L_O+7!8*
zJ1J8b1AER{GU(|rn@$dVy=X!3yLG~Da?A%iI4l0RoH3bsWZvS_r)Ezsu@4ONIg${6
z_wk&0-B)_Q@7XWCZ~gAZNBVLTEKcq?7ns+%y87-TyMF7+f(s7K2Y2?oPB_hXf@zn8
z$9JX(0Y!!2zFU(|sy;BD_`5Is%<@eM4#q4GxQaUtOpZC*oc;Bx(ec=aOLsO(U+Meq
zQlq)#URUHcCA9$W{RY40rPVx?o+~o_PW>61ZI0Y7P3(!gf(~R@N*hi*Yw2?&_KSY>
z=2MNA^UN=vE4i&Kl(9BMSnsoU=Ir%_`$`^qo0@C(Okn(eVnu}HnF&2vQ%^orzO5*D
zXg$05SxI(#R-^Q+`zs<(UtKi$_oSFjKZ~9BWb9lPe1G}$tVc2ek9yv(-gmx2f5FSw
z?NR}!uE-{H7}!=jeYsh@X~qiwzMF;q-`_U=&gb_%mObTSXIbo_Wv7>adiy{*(L($(
zD^qxxZ+VgRRPFh9-yZp{x$66!3Xz-l%gtZC_pX_@XSQebz1!UE846}EH?OUnE3R99
zskY2gVYUD7S8|IiZtR`3*6d|<`I!|z7}h;aO}w2I5_iq#P1woPUn(LEwpOX8H2IuA
zU>$$W<<0byY~AMj8t(n!wfOQ@Q(^U8#=sb^>F-_`#A^9peRTZC#*ewjKUmImeJU7R
zp7OVp-9y{<-!8u+kB{1Iub-nGyYinzt>@|f^zVkh*}iKZWoD}uH@<!S<wD+m@er}1
zs2!i$-W+`Hp#9auw6luw$-Cf^lX5#xxD|TtUF6|7HUERc&y>qkVjAl0w!fMEqvU1K
z&h5%grg3IJ-(LQCF?PkfCVjD6JWbPM9Gj%_PagX6Z&9@CM{ZWf9QEFL-=FVJ3jO-|
z)U#uHGh@DVf77!Un7{n@V?LdW3I*lgzfXUDY;(Yu;(d%I3nH}Hp5BzZ^Yq-4-J5?!
zC73GweD~j<Ez9A&?}6^SBHylYMb4OgwQ`k}sFb(r8<m*}nOU(vjyrtkcS%fS+c_!g
z->!98pEL??h^@LVmffW4-s|-seyzo>d&0$rHmP~GJ0G?A?YcFyKk}5p-bV?Rj%+{w
zy4{*Oaic_I?1tu=*iyq=JHGopQ+i*wF@I^wx5bO|yi+)&-e*e(e_tCfc_1eJ%GbiY
zX^+`l%2?Ij-Cu0x`ux{U!3C@8GxGMjFxPF^xsYMw>ASoYHV)yUhdo?F9e2<FQdxCi
zUT^i2%Aef+j$fs7QlnxnO=l3<zqW95nY3e~>rR_PlUo1g9SiK}&^EnPyfZK+{@%w&
z*J`&$8vcxYwK9DEtG_z)V<UGf$-iVX-?3}G7$e*KnR8FQeE+2C<?kb(Etzt7q_~2X
zX3L5+e-S;tK=A+Kr7wT!*T?_T_dfP<b$s;u>HmIv{xkI2@VRB1%&IWP-rV?#B_IB%
zAGlg^|5I!9L*6MxS{cbl|IPaQVoQxyU-H^B97i1lR;+wFyFw#v<>jqU?m4i{O22&}
z_UNkVUguKk{pC04ZBOt&dpg-YeB--itF7OjGdkOQ%l^UA-QUl;dx@7NJ3qgAHF@n9
z`{gh6w<q&nKIZ;VD<gyT>={Ab-zSd=uiJHXM+wgtSzC1jjUQ(vLROi+|FB58DDu1J
z-4{L&k2zjsk+QBlb+>%kuH<dnVGFuU^=k7UUCMp5`l7>k-oyMZRx6UrCQaTF!S(w=
z$y>ur!)Tiswx++$tn<1ORl5a`b=elN_RlV{*At(<v|I35Ov1JgD~pyJSaIKY$*LdC
zc(6J?BSb}DZT|O``*z*^r}g(O-u(Af_RmwfA67klEVz`zDVld)oK1L}4$Jfz|C-t@
z&XlyVCe}V=Z_Ah1R><=0;g|P}YxgW_e7e$9L9gY;)*Iih+9sWfjoxr#^-8ZrGwsUs
zYEG5hVX9Z~4vRH8D(vp%sH?4!$(r$-VQKLVq3b_;llYtp_E+9~n*Al|SJp(WuEuXW
z%FG{}b9lMdU(|Y8mu%?o;zf=}R%ZSzxup`%^z8nuX;MG9;_os~xVc6`Z0n8vOP>d8
zIPa?e5c2(So#g(TOLMu`@ce4+*!K6aTJG){`f<BY`OdA@ntSnq_(5N`6fK2hQLiV{
zSnsivKH)mIzQgT~|GH=AA4-2Yefajgzt!K~*BJ)AD`H99_$$qJ^6Iyoyw6@;3e{TU
zu;nV7d0FZWhsdY%940)zX1YI+<=Hn!PI1euJn1~!om@gj{pS8#offZtzus7CMa{(@
zitBeT^4J=m9lxCY?YY3NPn)g=nE&{AnqNn1!HL~p7iF1;ZI$-EI_uU`{TB(L;j^19
zOC@VQl;3!7?>5z(YnIG}V|POIf6Tbsll$-br_FgPuCptDivFt$jkhh_yt{H&Q(#j4
z#8velzP@AUZ4C4KWG7qner6xb&h<qFp{(I=e_5V2|Gcr{qQSk|HPNMZs}8rcg`VOI
zWr)@mJR$o1T3x}1BTIH3Tzk3t$D`XvZKul5wOekpqF{>qOS9;sJC8Yjs9pNQ%Xofq
z(47W_DaJ><!|UH)-0*Tu`3lDGW|cfszMc3GQ~Qm%`~1zriIST-1E;865=oD*bS}HD
z`XR$^pH{@brA1MfLJuFmxz%3R?qljhG1K^cZ9Arm?_(14$}ErFvHY{=YmTC{qs-gn
zWh~~i-7Vl%2)`eFXV;ISi#*&K@}j;f`*m4mK6I?!ZU1vla!6(6hM8<PcKNZdn&!Ov
zufjZO_vX;o+ExD_RM$+q#rb3Tx8>dUjpBbzTiTuFbLCg*#cj{hwDwBhG09B$^kj+b
zRF8Y5_bN7hetSgB&d^4$dY)eV-*^1cx8l^=FQ`Um1vSjrrITK?cjIoQHBuTSA;u!S
zp|8EK=4pvC`N&m&zshT&dg&`ut=favRp#>+d71gJD(z*mQq3^icU3P#LQ`3B>8<`J
zCtOrm-xY6PTd=}m?|qv)2R<|(-Y=cwKksZ+c<~0eMcP&sW&9SE*Fv(47xqXcp8aO@
zcY{rY)G58GGh!FqHnYf3veetXXZA9g(B%6TC6P5RR|Q!m{FOesPv2fMXw&Uy&(EEg
z(l6)<+@m_7c=cXQ(>-b-YZmR(ynmWaV8iaK4@;`z`qRumzr1n$?xNb#u+wk1Xx7}^
z5xhL{!O~T1RtMJxy}jBqBl~go+kWMBcY0a(-O5~4H1S>7r`d~Vx3X_Kdtu7!;#W5I
zD(kL)`TFVmXC6k*z^fglKNG4yzTWAaKKB&!Z((uiOp{NV3A3bUTu3#I-FAJc$n&|Y
z{jyV}eM>5j+uvub-gRl=6Q$PEchA(_*{kkl`^WpLQpHxex1SEak&B;Qf1NLmPk6<)
zAFp3Yx6a++(<Q#pe%G(xH}4#H<d{`8+1YPT)o-`IF6*YQUc5we?WP>D6@3#{P20ZO
zTm983#tC=3-+XGztx0y1{=esQ;g<smSC`K8`xN-XE}Y?X{h}M4`MW+Tt~$FZd$Y?`
zox*#Qm*3&|Z}4J*n$P^luh{OWcL_y(3t_&TKkJLaZmT&f0*f!TuKQzmERaK5V)vDk
zpWeQUxVrrYmzr4f(bo7QU!;VxoL}V}NiZ$CR4!%^A2u)FW`WjK-+<6v9~QgK*H}C^
zf=yssbl0af@{cEXta>@CcC8%af<@D<pEU~Zp1LN^Ec)@=V>_8=9NHqrHRGuGI>RJ&
zbM^0*Uw^!jZkM0)E@4Vx=~3<jM`x=(O}p+^EP9=7$<1}T&ni!^(`B1iSu}OojL$lq
zt5bv4eZHP5YT~hI--_>Q>~9jDvr288-+yH1l$x{q#NsnQO@7pxr{>ChnAgXC$FBAN
zVwunPF=mNb&i?e#yp7|~`#(XiW_w><T5$ZhX#kh~Etk*<v$VH|?R%49H1|Hs^!*o)
zWG=tl^SW}?_k@>b;=PVsclo9EPRjgdv!ZI@{_^f1#;euk_U`?cKZ%<-SnTJn+xBQ{
zS<?=-H^0|#3;bE7zvE!uie=pYm;PW3s#IRHGF0f*{}}68+tV5xmbpy{b`D^9T@=q2
zA0){wbJT50F!Qz}4T9wl&!5<G?JED|6H&It+xp(G<___W%F=UPp#3-aNL=@pcWK3y
zn=AjwEe+l=+xgh6vrl)KnFmw|hi0x?e&wcAlg{(cdFwVF-ca>^f_ml0SuB&>j+GyF
z+NbX=zL9H^xs>Lk-;UEASKqm{!+pv^!`m#HZ@cY3-cG-C?ul|cPb9Ow@2B)#lkONv
zZ@S68bN`Ef2j926im<xNyl2z?a%Z*$KQ{=R<-Yjg+Pmy;-{;MG!@+*-xT5FoM%}e0
z(dp5jOU#utO6~3AcdgM^Whni*V&y;ATT5m>;octp?=-L8xs`YR+Fw7ky(}hlp7frz
zAAdZ!RN2c^BOMUtHh00b_V|~2r(P{z9UrZ4|4;PQl7FZ3UMs~qGaoo}$Kz$jwzJ#j
zzC0MZHmNi(rTNY5A6IrR-EDfS(X07H(t|y1yt{6GyVK#2r4riQu~J>o{LqY?_SRod
zuB*kre-U^8{%YwLr}--kR{xH=lW%VG_d(D7o%eTVCAeLxXkWiavgp~2g<`5kbLH2v
zN#&oiFp+t&WVh~lt#_I~Z`H)^s_T<)(ckdSH(Nz<%dN6IHpkMWH%YzN=g2(gS*aP9
z+2W#`FRsR1TDh}n-Ktw&7mRg6W<Ptojo1A64-@f^xqXa5v8&#&-6>vi!{qw*i92^{
z#m!!p=CHkJY4h#9`xVpni!9YVci_VDrd$`@WcPcnB0rWqdZG6?LuAFXq=yYVOkXi_
zI;i~is&?7)iu;|jOWd@&>HfDqy_qdylFiwhpxXFiU4BA>%i5UKoY;d)PI~;24`+XT
zpZCx1#(MJ)@7ey{J+NQ;M>+HVya#B46)~KguS5RGl^wT|+E8}<8RHA9H&VARxNF=g
zlFC`)Wu0#*kpCkr^W_o0R3kOxsm4!t1~D+2zTN(~=7w>Petv?JY{eJ0cU$eJud8Sd
z(wutzOYFin`Gb+32TktAC2M`&X_4?sgi$5z;jX-3QA5|sKIPZ1A9h%-l)di2HjSy%
zLcaTi?q!)1ry)5_L%}W8ShPm-fyv9;`P=^H3oSF~{4~#}e#(PI3cW#}8ki(R*mkYk
z-E?tV@r8);*-xegZu>Mj#5;{^ewpfYyZJ`5R`~E~tO;h^vw62rmx9^TO~>}`nbBcC
zy~%iE`KrbjlQfl;6beM07H7F1v(&WlUX|_>etrQ*n|HCuRqF!HmeWW1xmtOPrc}M?
z3M<N467a2<?bQ4#)&sk&jl6F3_;xI5^qt?Ut1s^O-rVkI*CUq1^(<*u1n$1<E)SXL
zDz3pg{m}k(JR39;<HgtMFy<CLS7G{YoZGl)n&1MDZ|B}*Z~q>ef11tFR7KL_^sGs;
zDbE(OI=sISI?uUUTdy*9-LKNzte)AwuAY6!e$p><lKxuB-`6G9$?~5(cGX1UdGE6)
zPj1x2@06dB)^jLiuHl7bQ#PMpO!F;D*Y+*Wy)SM3Zc}nyJgeZYMX$MkzfOJr=;@OE
z-Xe##=C-`vULrW_<TqaKbx)dtI9>E=4>o1*G81H(ti6Kyr^?Nn0*wFE-Y|#1@H|rS
zmOCygvD~66NFb$mHY3|AHp|ru_Pxpc_Dg?RNzqxiZ)U%Ij9t8pxqO4ockkxA_F;bi
zu^OW*ljm4*Ubrr@@owU+H%wxibpN_}Y|>`DGv}qgRQUBb=UTdWzt2>f=C%D{#uJaH
z-KQQ&uVRhfUi{A3MRT^s(f$dn%T3kQE!_eO7_Yw6Tw0uR`}FDSH+xQb<{l_aJ;5^b
z!}aMsrAh~<By`VKwRW)8zB4bpesSHQxqIi#X-F*i-4q-z8L`1KwOyodb5PT<>pO!j
zAIWsh+4Fnx<)(ybQM(_V`7T>6x8`c$!&-|ozl)ZbCf+@^W2yCSkJiJJH$UFIy16jh
z<*U%zhuI(R#JdF6adijCJSq`no-X?!+~&DUi{%#`>m6?=%g#P_msh=GU7y71!l#bz
zDTRXPVpP@Z*$V}KT>V+};c3fh|3!^aE(NiCXJ7H%^|~YW_3hTgtnz@6<K>G3^rP;c
z=?JvEqV29Cb>4FF!UGFB_21rqW&5bbpVR5LB72k~lj^#4@72pgwuZ0dJFFa1-LJLa
zh_%tFBJY-q->g5SPyRORQH%=*Z(hjtrhJ#olH)NpPWl>Ew_{irDyr<5bZB4Lw<XT@
zvM=MDe;tU`FZrhXUD8OGeL~Qtdh>^wYP*(Lwk^7O@LI^ng_aXee>hcSx=n$lV(uaF
zFV04H`$bP=IA&d1TH9N?{PYgSvt8SojQ)LM_4Jal+&+6{)uC5U|0_QFf>Lh$pUu0%
z<n}ba^}4hF{yP8s&rH@QTU}=!e;<A3PwDgi{~aF>8mjK`vN#$v;o-aA*_nbapC%-%
zYnifYwyfQf^{@X$oidqd?R$UQUtNtYF2BF*FMGP4`S<%>h0%-}$2L6w^8UZw|1XF4
z=l}Tsh1bf;uIO#`PY$!UQ-1t5mXm+>`2E)(*0JZ+gR1^b`RiD@>AZZ)f$c{#L#uAT
z-@a|yy!ZZ(FP=1se_vUqXuo^H)#lTSmQ*b$;hM@{c`w}m`KMyGPuW~WXB;oZ|3A3q
z*~!$`moFYxVxE()v%B1C?tk%X5=qPLrXE_Qt*Z9x(!byLR2J!KRJ(4S$El+eon&^p
z_-2;Q?RmFm-3jSEc{B3szgPFxNZ<LiGwkcuu&+NK{ko%HY8#bN{yqBFKXvvW@1_66
zmmJ=^#&=JE=c3&FWm~r&JlA%oZT9;=`!9(LZeQOujicoCt!I;XtxQE9zS;fGe%k*D
z3`>}o_)Pn;`}oTj)i#Uf{I$Da%lGA{^K<9=-rwsg!tcKidV4R|c*>Q@th#k4|84V@
z=>NNGuKlu*bUEqYwZH#6Z!kO*7R<J#{+i;R66ZacZ_E`Z`|R`lX>a)3_*&q+S$7}x
z_|>HSJ^uXv?du=vUjM!HQ^o1_+oxN9<X2o@psRjklBW~PH`lD-z&)1s8hceUi&9Q5
z?=3xlX_ftctNU;MiQQfDF8z0`-cPld+-uu6X&!vHd(rY8Q58Q<eV_h)_vO&{`rmhN
ziC$M%f4uF^GK-_%|HVh2{m*Tl)6Q-harB;wQm&-OzrGt+j28W@^LC$B64aqqQ}uT1
zmuV9&Rn#UOPAKqweQ_C2zW>QLy33zlU;e3T;loMCr>p&AKa-dDJ^p*yuBDs#m!z&P
z`rkRN>CUof7jL19DQlMdovYs#F0@hYMA<AO%OaJO)X+sCpBi46{`>hi<ltceSr1h!
zl_>_lI3`}Z&z-^gjVFS2I-8b;sl5WHrssX1uk#jEsjc`||9Z>c_xC5L$TM;E>&mZH
zG}3PEnAuvcGO_6hi_zDLzAs#JZ@&@Jd-UhFpP$?E88s<u9c-CX#perhiofz$q;QAb
z_=JFwjgv!J=F+~T$Da~~*tW?jugTzKQL>wQWO8>#vzotRD1X${>2A&U6B$;v&*}T^
zyCwPetf~2{e(soRbhcfloQeP7=i~mf`f@xP1K-O%?wi87RAS!$O;`U<%5D-qlPs{Y
z<#_0#!^`HrTDpY&&S?g9hF?2Z#>}#eyIx?t>yy-lBUkxX@x*j<PKw|DTlnT#r4`Z}
zyx-4`<7*Vm%4JdO|NLI*!NLso6O|VFN``vVjn8~C`|MeFFmy35qqW1WgML3%Dl&s~
zf2oA|WaUj<{OXdi)EtRTF%vGo+&7)up2_ofNbmb|8Y^Sx3Tmt^TA6g%??LUGNq21?
zcSQ=S#OVa+rzrenW$KN%_+-g6KD%140F`|6>DioSi@lC+cTHfJexOLSt3P(u>4&=t
z-f?bNBfc}3dyle5>A45YvEOrQi}pS8DZiAWpBoU#T9nf_mxFEpZN2@f8*aXe6&1K*
zC*FMLUvb3eT<H_*uGHK;J-OI}A*xF7yM<b$j9cH1rN3k*<>hV=5Wji8G$HB3kGHjK
zUXNW*oKas~q`zrTv13li*VQ_0Muj2I5B)Ov`}JhluBGnj8>dQrn;<j$VnC|Dwg>O5
zL#yH|p3Q#tNa)U)1r6p>jq;DagydJ8?ckdDz~b&4?@h}z-e|JiDDv6KwAyQqgmkC;
zxq^u|ayTZxz4+&zg#Ya=(yd4K&w8(a(M+V`MnJvwn>7BM`?^{|t{J+$z4~oSV$I*Z
z{xD@tN|D)dmXjw~wP!tJh>W?pTq;o9ir>Yv?r(n8lUucO`POgVAFmVgO7n=Q?*%1~
z2T#7Py`7+Yd#jp$^OZ#hw3J15&+2!$zi;~DpRdek8a4GTvUS~XJZ|sXC-<hrFO0c0
z?f2Q-^L}Y(TyEy-|Gp;jZJOS!)bMKpTjumVIdW7wdRNAkX|3IA(Gy#J4f)vqy{h=m
z*%+yCsdjN%dLVn$nmKLvG^WKZ=B`#OpK$h@qIU4y+Ku09LfO?=_pVyD_ua;@-kSFt
zWnBgG?>pbD`K-Mt{_895Q<uM_U0r$LwEnEkf}zphL{gbg*Tgm?U6wdHLu6I6!08R8
zaz7Q$Ox_;1efx7K4+W2B{GvA}Ge+O45W96aW!Jk85|i@OBUoO=o4vWaY<BYXzpHz`
zo}X048y&i+E$CZq72~!$ckgJc3(Q?_xOrDxonynX(qkrHKCv<eOweq}YivHSFaCgL
zqy>|DoD&0o>G_n!a{5nmS4qe-onsI?`g-z;o`$VW2hP`8mASP4v0d{m<tAg&9Wkcn
z6QW#fZlNb{?qwC)5x->JcgIGLqjEwI4>_`5D9CT)^XRj>@j*&;?hRj6v3A}QG9OKp
z6oYSUZPorHx>2jD(`wV&m$OP-!?kq7T<TmlT(i4z^#@0Mz=d^h4#z8ZZ18z9Rn}m~
z@m<ranKd-pmI(26$nLQedL`Yu*<;;Px8pzVe!IoenthV>=%u~?xnHf$=a!ff-5`9?
zd}H|!nf2kkMkb3_oMPU|Eu$r9lgOpRE-+WfQDE=tJ6bunK8sGBYQpyL)ASqVua;@t
zf7jH#QboB}S4qWHb+>8EQAKre34!ewPjA)e`mG}MzFK)^bzoQN(ItA)SFIG^#ka6p
z`{`b|S70A^PD6aI_`79RjtuKE7F^sYWFoMSxoHdE?IHn5o21VFj&eQu$G-&!7?x}Z
z+Qeri@pY>U*J6`O*|;yh&LTfr3M5O<m+qOB*c7qxp|8S6-$J{9BTgqvH6HOP?&An*
zy6?6^X!(B0pT~?=G_vk<@UV!KJI~dm_i&-@LbXPA%{Lr^#~<$4b57FqM?U{P6CSfQ
zvYPj9MNX;F_PY0Dw$qn~cPF%-SJvu0vZ3Lp-Bk@&jq=4>H`&UYHZ^Z~(QE1P<9W>6
z|J@Nm(OV4X-xrCS%(C;&yj?B|(@!kAXYiOeu<2`dpy#Pc{A(8-4HnQ2b$$3|(JmF?
zbBn~Azii3$ZZ%pJUHvRAK+u<e$>qP(SGBCrRc<|@9$LY+yZtBkn}ogAADYis#+58F
z5?-_<H);hR+oTIqD!krHzf10$U^ip-t34~_U7dJ#c*I#;QmD~b%+Z^~*_~>&Ho))N
z<U46s_)TRR8Koi`Gd0hD-Q0Fi%ItQ)kN;hHs}w(rWa@LSb7Ep}FrC!CG-2tZqs#4*
zYu`MdlT=w|uCY*8%RqqR(prZ!mXkU=`g;^^ZV(q>Hxy_t5fRxX&1!Vz?%Km0YrfU8
z1x%`WG|ON1wAztf#~*1}R0W^tJw79URoA&~KHHbPS|Qxqa8IZ@^mpJk(UaR5|8K9<
zxy^YoFVjG6^$O{r348Wj_F*d3_Y#<_Q79?(WaaBO>$befnUNBwx^>yTD@QWUJ+lm&
z@SvC3Zq<&Pe)Au;Tux`VTDV6jBm2-Ig``c3J@uHP6uU2PP2hPp=VJe*WiNv6Y_YWB
zP`#AldhgJuKfRM0<Tay@3H_Dw($>D`T6=S!>g0WA6JCku`b|1Ff4_8l#(n!nwwp=Y
zuJq3}Sh%~g<}KUyBvnV%SreaZwT~&<lh3Q!t-^V?`oHag)>TzD1qu5!+9NMCIu-K2
zkYN4h^)=G#N<n7F(kq-gF`Ig7SOcd`{q|w0kFspn^U_D}Cj{&6zt6SgviWVk1x}xC
zlwDn3*uT~z@>JA;HwVi3bV{Xt+r)}ixm;WunzxqWVDv1u3F@;&-k58=f60II&XjEL
z;@L-<lI_=eWU~Fbv`v4HTyfOJ*P+R)J);*%crI%R&gfD4aC6p96E*%4`72S$Ze{11
zo+V5W4BDEy;>Us)S=Zm))DrH``7{0Lu_cFN<YujUvWj`qUj0nppWA|RTl*Y3mu*)!
z3Xl4{r2Djn)9M3LZ+C20NvyIGP3Nk#7d}xRduevkQkIb8hw?QSrti=2&$_OsF(v8N
zD&c1xO8HsG?(Acclz6@*=dPx6PAltjyO@PGscZCZA6P0E6Ev^gYQi?_oYj774&)_e
z{ArpdRkbrKfYa9W{;`W|EiU#O%&B!>VwXC}VTyOVYv80CDgV~bX4^SwO^{wgVcaRx
z)0ey})5;x=guTp>{9<_Z@#%~@)_L2mJ9XCQIP~}gzh58p;M!A5L%TiP_wOi{huYL<
zz7uA!_g<_o^Ly!b;h^oJ8j_l2^3#vm?O)Pfscil*OL*=2uD!FT)M?h&y?lCh_RCXt
z(~|TgI&&^e7i-mzne$Lca@U44T8&*XmS48yy!LaSc<TOv54)yuhieHky{{_$HeuH$
zon?;P;TbV9F2Q*hoh@qv5?6Rn37s$3BvH!aBADxW&F;Y&*R@%f)P%i4_DxNR%-nyW
zGJNUWlWE1PSQuS3bh`@GxnJC3)@2pXbTG|a{O_T#@?w|NPq|WmIyFNsww*qHLUf(c
zHpZ2eN<B9`7Its7Di5`GUKH)V;`1E7D;j)aPqR&Ar=NZDvpRgIaH-14BTFkhCUO>3
zZqgQB?|4EYXi19u`fH1gBHa#Gn$M`?nK9>PESvPXGyB=fE}v+Un>Kgq!H+IK=5+jh
z<}N-fWZKKdx0(v)q-P5p3+dh9UUxB-=i(kwPnqCZyv}Ox{BrIp2RJPKmGbkO;?(M*
z?W>s9_Vj8d-8{hl&ZqrnQLpg!cAcc1qOEI#J0jdlGX+l@uHo*C;0r$`ytQXhJ9AU&
z#`rHqmu0VtH6-z^jyk$r|5VcN;E!pW9wsetNENxU((?Sa)b6#lj}3S%mai2NIG0^<
zSxq=J(0`(zucza*)8<y&uPk5l^{#a7CfgGe-sYTYtvT%VN~gQ--Fo3VHOA@uE)71%
zW|+FC@?_tW-F#<DV^~f=58u46dygzy@M)n(?xmZD_OeJaSQi&vR#BO3bgy=i(Z|dL
zHqqdF`2wtWosP;)<#@5|zfiWT{nUH9mFFGaavSP!X?lG6XQsX;HbAFJ>C(>7sVbI=
zE4aIY-0pfvx_T~svgC-f<#aETbQaI>EfY`tH=RFy+7!hZ3oqW3^pKqWX3tf(%`*R1
zFVL7d)l7PONrd=|YOA~mS%o<>lSHQS?m5p?Vm!H#?{3LkpN+3|cWvdi)x5v2k>}~0
zuI4iW=T19mOev0==HzO5angk;H<BcMIaeA)hbA3=7BE#{H@BH|-#NQLk7=wiH+wQF
zPj+eAYC3Dy*!|R<<2kFpbzVx<iY+-Op3f)?Jnp!)YWdpk9d2^D6_a#czby=rxYc!K
zKhM&QwQBdv+*f+ttYH2mU-Lf5MrqzPHtEd|H6FfQI_<F3{EMkgG5gl7j@h?ExU~51
zqxE-G!oFTMTC}K$Ercm!wq?wv0F93OMbeDd#F?4(0-s9F__@f4KX7Y*62}(t@3mEb
z@3A(e+*fR@kQ9-vsJ%EvrTNl<>+_67y>7f0XloK#nl$zCq-8EyYf_5;D>W^a>Cg_U
zxqItdWy)TerOS?f&&>8@-OgBM<}>Zn9gUSqi)K9eYFl<^(ZTp1-$T-ctV<_N3rV;T
z`?KZfJPTGM@ii*dF=r>+6l%Y;_5PJAcr)VCB%42L7Q6K=ig_QpkW)0iK9oIVPHCR*
z$`fL1L;5A^kF5<k6cn`eTZ>1iZ-Ywo&Z%5JfmKth3MO8PzRKLS)6}=Yqh-q5r1J3T
zlR`WW-qV`2{G0#l>SbT#Gk35==1gGSkgRU@>ewIeMK?8;d-aQ}G3BQH@hY<Q30<TA
zfxoLZ;q&d_i(j)^BSkXZSJeBS%52o!;`_K@_G*S;pS=m6f0u}zRg;Ky{{NYwLH?;X
z%YlSPsv>z$JsNi(lDfEO^%BRMDJyR{EB53*3!fTW6{2x=lF7Nxm)(Wu2=#wlbkEE<
zFl5;b<rPf^QhUEXlYG6;+Duzz!P~uI2|FW2mAF!Lvp<W@-n40p@@#g)rAJ>)TPtVy
z=<kC|zuQgP_;<Q<-an*exIXoBoctWs1HD2UEmr>)kaXL2LrH;M_{URkJ~j3yI@%si
z{~a!S+69V~B>gpPi<>alKsWyD>jP7C_s8cjE>DSn^MC1&&E0hg9h2KyPD+M!F1nH*
z`SDWfq;@eCquVk8lQ%`bi8$>e&CB4eEP8?M*_s=tTf8hLig!0R2Rz~8V}0$;I&H_C
z1wAqfD;n=kWW4Ed;ZR4gkRgl9@<j@oE0iQJEQpw(rqHE4Q})YK#u#2zO$(=~)1E|g
zw~4k1Y%R@wyR1ug^QDzr0-7>*&BzeA{&DGs=L}uWRYK|8nQR2#Fs5#_n8(;JY_mpR
zL{+<9>-nT60sjYFTucw;m|TL6*eurlDe9*Ap+kl3kea?SE04Z$v~ZH92lGUMEZc=T
zk#h@<vY(g#>bR*|_Q3bJPu!l@P3$k+c(~-sm)9~&+h)#q${mn0-)`yA_wzKW48FdW
zzIg6`V8g?|S1L8WmPFm!yKTax1zUx+D_=ePXtA>StY(m!WBzX5H3D~%)b&}yC6r>N
zrg^Z>^SXE^H(u%o*O@6TtxB7^f9muW{onFDY|3?xlbJ5zH%kBB^KkvNZ$%dmpVGcb
zH>L8gd=GW^vMbHa6pa6Q>xs{)i!p9fk9h4YsF}(j>S=sUU}@2$RX<(#Pr4n<_QzBF
z)mz?X;}b%<LVIrQ{m6A}f||Y9y*0Vt8dz5UI-{1d=+HNF{S)z17qV&Xd9$y9C*8+s
zzKYYFMHg0k817rH#yt7t_i23!N3)GD`irT)6FaMa?Z_pImI{kGfh`sMdr$w6y(1y~
z+MxX8I-?ZLZGwllJI|lCz`5y}K<1;RyE!f_T6XK1_)D?o@`#)^KJ_>4l1^_Jnsl7@
zsAnz6JG$-S?FC2N?`x%qOvvwMn5A3h-m&t6)q;yz|EGDeFEx(b^5&L}s~>CVyOSF|
zw{8^ZJ#^XhF>BHf*PCkddlGip?wPz}?<0*)opW&=Z3TDajzlU>QabR-J)Gyu4IdNx
zMaCAK##00Ki?lcYQJZ<e_=>sj<o|d1b%K=k1~`UFT&tGKJNrodQIGbeW!!Z|^2^l@
z?ycsPVN$Uy>YB6ZywJx9Yqo7ma1JO`J}G<DV#{xZ$8$X#H?Gy4d@HvzOR4jKgvilo
zS%av8DpRe>nkS|A>})QcT-EkJ=kBie6Zh@-s-wyIiFef`=eAq#cW*lVnREN8i+SGn
z-L0C1w&_QCEpleHEl)on@O-!B`o<+I?4{x?7fSCmEfjg$5HF;4xgfgeQf_$XqJkE$
z?^APjhpag?YeiGyzP9z(SEW3i?7i~zoBex5zDD=n+N$bzXv>GLS9dp+o{|pDf9W0h
zDrncHGiP>u6g7?VSUibm(jslwMNzJQg<H56mdutB*}LD;th{_d(k#7MKYjO_@4j@y
zm^~ppP31(^`#j&e$)_9{FZKDZQVUm}YUq8CZI$Rv^U$?Qw$2kx_-eC)Hfh~^tHYwZ
zXv$V=lbZF(S3elf+qvpt_<|=I=A|b!uTOe^>wo^N--qPxZrpe)-g9Ns)x<4b_iw+u
z8|xU_JUwW|6P=f5k~J2tw+gDrI*}s&eC<r{#gmpc$Ci8E`d#w&$5R2-V|lD=*nUI>
zNyb^<T4{B}q0OR&-6-UguGO+X`@EhQJSez&&e`(qh6iuwnwV6~oN!#MRQr+(bMi%_
zlOenAm+7z0l9avkOWb&Uz!qN55Yb1MuNv>yU#W0Oak4>u;ro@V1VlM?xvK=LcCxIB
z$Y(y7WB1?l-!r)@3f{Z=?aC^TuDyGLKeanRHT~ST6Dpyn?OK=aisw_$c=T2D*0huJ
zi-kR%m<0K}C$#1S9qo9lS5m1`&ii%MT>GcptG*d6-X0$FY=+sYB{Noj+5TpF(c^bc
z+%NXLy2-R{eeG)3W2@``mvTK(me!qT#jkoXWV=be(yT4_7dX3!vMv<cdS`Cjt;<2D
zIP#(^)@n7n&Teot;N<pGnxQ_0bJ9#%%PGh98cs|3TeX0D^7eeL?V($Xxti-$<ubFC
z)#hp`q+hCBII(iW_vz=~NBCaZ<=-?%%{`%eUS5~^m&hgl4?UEYuRHtIQ)&5Z8Qmnt
z)o0x#;<Y|1HD@?{4|gqa39UBdn5(7{dYtWDosYy$uQRj%)=oOHsv>`KQCZs5OXsdG
zn7C-oqyD0>w{<>`wz)=bKd{>2$cyF&_pY$Fe_rum+o{@}x0XhpX5r1{|0K;S`Si5w
z#MWJGcJg1YPW>vxvaMCtq;Kb%CN15xeig>XuiF%Nhg?>=?#Yn4#DSaTqV~@p*|n?x
z2Ca-QEb;8R^h#v+ONCvTGrPr?E%LhksZjWN`>U|*eG~55$b1(UcIKM)El*c2INmW>
zXyXjI){q5<s{(d7Z3*m*KkKWyPoa}Rnf11bOGoUbYa8trSDY5iNjS)PBTsm}81w0M
zk`X8WP3rgF93*RII#aaZc3Me_cB;~`q8+I_SI>GG`q*@82KVXcWorZ~j;)O~P2?1Q
z>UBC;;9;&{6vN|AIsZpX>^)f`H*fuZe;2ExxZRrLMtr_zOpn!dtBfql7oA?C(%Nw6
zMOeD)%%>9J|4trVGC^+I`s&wWLMzUxoewv4pRi`i`e)`@;T_j@Ia#GkOg?fbt~Gp(
zwoc&Qn|-0xdy<?5_@5M;XgEdX7Ee;|zIUT%J@eiVx4T}Mv0FP9&;I-Rri-{cx5@Tb
zJ5DcUYCjz2t!_G3a?<RJi>G>iO!ZYsdl}8TH{0t{i9lT2j@-9K&m-iIvxfa?)0%Mh
z)ti8w{%$FE<9T~t`!yWQOO8wp=FiXxw38H?y)h~!Fk<RSp@l2E(oTvQh(@Yy6+O6>
z|9Im{@rikCz9+wOwM}x3{9kS4_y1ed<$WU7&m3wmU%0hH*7DJzSreqLY|<6k%_U`e
zy5rWRjV$ahn`#{Y>4jMD|Nl|kHZaGlS?j^+pr0lxSySxKEe_sl(KmyqtmSeXAE(9r
zv{&&Tem=|Ny`B>H{C?O8Ud}7~a;|Oq+a%UqrgP=`uJ79xua*r--}xzY`R~;ywE9-N
zR9p%W>G*ThWZzCksV&a8%qzmT`WD#7tk`t>F1t_e@2j`2ZF1zCxFhjysNWoxExx+@
zt|Xp071Oh@N$rl#^LMptzMIci|0-7;62?C(RJn8^L-pMD-^T0>$u_$~(|GHH-D5I&
zS62lK@UG=uS@BkSDZ6>Y_E}S=&nx+L=}|wQ(iJ<!)<SikeS5Z@TD^mzB%>=jbNk8C
z@Q%C(`reuu6%SH%Hm?>n2v*Cneo`)5Hpw_-;>Mew^t4MiZ_Uizy3X_Ok1J36?$w**
zZfmXHzLGgMB)$4PhtT{kgB-I*+UwWeKeoW_f`^(e_wNgNnHM(tI&3Z6w(IKmz~Z7q
z)2gi&>qk4kj{4}TP;h?Z?(N63_eUE|-jiPP`;TDoecP|rYKyYJa6Zl!KO!UesrIOk
zn&S2&H*bl)e|bRlWMaXU3C3m5v{j=+C-O6CD5jg(Uw*nV{YYqK<d^^F<(((j&s66(
zlgT*q`P0ErpM+0pcSUMn{pFT^lD_Hr<SN^npXc1Vxks{P>HTCDuV0sBc=qfsE>AK#
zSX`Oo?Wy{vNOF64mn_Sj9sYNPX4P&!8@%m<8e?_4zTS(ozu0~q+P~n=>gu&-aklk7
zwz}6oMry|?^{)5yxWT+*i%A!Uy73-YM!$z@=1fc%e@OUUX1i?2>btjEe|6uJr7I(q
z8SY3cZf@EZZ};l##ATfvQ~4uaoefhDbh){O%Vw9NrT(YwK3BJ`ws({epEzex&+h{{
zul<|UZl|l9>i9PGQNcWY`TI{_?+Z-Xlrn$z8{s883@&<Va4!pEsu$Sksx^22To<k7
zmv^{o<*x9#;MHk(N!c^8tYznlrJGiMi!<4ov%y@@XL@aD;QKP^LqRL=g?sU83cT%j
zCdgTAbxPgQ(R?$rm`p*Rx7M8MNgQ{AC&oW4Uh;l_TiWk>Asgk+74H|!{?DxM`AA6o
zW0w6pPthVy4zJxiJtup&?XUVdpH=76|M%u98$U?j@UZ!_=2lf**7e7i-5Fl>%NNf%
z<74%3abVGv^Udq$Ui|g=a{Z0^MLVi~hJXLlKj-;>Z*z{T54xG7J<9HTH!M_&mVV<H
zA-pU2;cYI@P!3rKvG||6zRy2UQKM=X&-Q`k!L!u{#xEUy`%jcwnZk5i;8@#*C*PF0
zgv{mM)wxwj9y%#t)#hNx(xxcT)4xIgz#f4oZEcDsyZ?4f&{=SRX`Ms+&3=ygPAf#1
z77MVoZoAALGyiW-LQTs(<uh^%zn%zMdiRT<TK|FymV?SKWF6meWV$n0AG*p|=x<^5
z_}C5Ms6L0HMK6LFUUD+3zmkxg@AtxtY3s~2-4`6b89Q9#*ykQ_vdQI6KSw*8jK#ro
z9Jy>0BsZkKTV*v}@Ltb5woTG2@)UM4KAqOXbvTScr1|cFvIP5v&xhI)+?O1<y>p+N
z4)aQ$7NZF#nI`b;klOJ2!R%YI3016f#3EVVINS0p@k-QgTF6w^6}aPo%az0CDl5O)
zGeompXZp&rLpZ?CVS>`*1%?R=79G%M+`eR?5u-Rq5_f{T)Q6+*Jj{<NbN%>iJk9sz
z;$D~M*_LeC`||eK{QbRq+FXl>3OA*A`{v1|C+Ba>xpeON$L%Ih<=@7?N!+?u^#tR)
z;}>2|K07&Panquh%eAMs^%b3V|KV?y^ZQc8p#{Nn-K=}d=Dpam?B*;Ro64+DSDvi&
z)H+yF;gZ(F#QbuKy!@>fmmaQvqRo3s%6>)P=jYE3wmPb@&6G$<njrD?`|>5;`RNl*
zHv8~Qx^iivgi7V2-%gA7-ar4I-CFI?$<LG7)%pG8u2`(}H_JJG_p!|1p2?>yy&}B!
zT~c^Fg-?Bd&ZlhIe=#z$E|;41ye3||QsLo~|McywyNZie$yaXp<(SXxe`Qa4M1D%7
z)!w{2um1Gk$nHIs>m=TAN?_fpw8?!YR+lpNUab23ENWiJd%g4ak1wXB$$za`pCA9`
zgX-0&1uAu2YQL+#7F@Wt<IkeX`&%Bb-!pT@>w~-5-b;O{tC0Hcz3a;t7pn#D47Io(
z%lcfjxa~Lh%R{U8FBsPJbaRVl@T@apnY`e#$&8f;jY4KghMlae>$@IdKl$I8y7lYs
zUtJkwuBX?TF}>S+l6si@%%w+Nbkvdz1^qTe^AuG^>pe?0l<&^d-L>__*;n7s-QFAd
ztvIIe++&#=W#y*z=l;#vKk>%#bSd@-*Z7e84=$Wr{aZm)xbM;Vf3`E3?p!-`pu^*^
zxz%L_Nj;^V%QNq7{<~jokEoQ^mdllEIZk=}Xw}oJ(2(%`a_{oQB&!!s+U+MjcF>PE
z-1^~X)!T{x-7bE9zW-aY`M><VOJ{ZeeP6!yklJj^{HH;A_ikS5?)}$4@y-1IQbr!h
z8r(bh?@G2xh;t~+?*5^kJNNhdXZLshEf!z-`sDRD*&nz5dd2?!Kfn2f#f<waWA^Pj
z_>iq#S|+c+#N6P{Ba6j?eGU7H7hmr6Gk>*e;e3zO5Vvi|o=WSP&-zpS>HW#W9(Q?D
z?r(VXN_u*JOx}s&xi;rd3iB46G?V>xmh0}zCwB|i?z?s5#XQz6QVuG=4ke$>GdOyc
zSKa)LVk&=USJB)O*JIDtrJb+oYO^)pIWPCKec|N)>+c@ulMxh`oZk@jhJ}?U@A9=0
z!F%ftXvG$o-&$mwk+$6?BkRJI_QgguA#MGqJ_JhIX+19VS+@Q(?+)pce?8v07r9TW
z7CQFz$v&0xCzJ1p_8tH5QNm8{@bP4ei{(E`dfxLcQ`=tcQ2F8&+j8lcyn>wvpR$Sb
z*4<chcy>v}WWM~M<9**A`daNVZx8Nbzj(CfVcRC@Q+XP@7s>>F$<wIj;5uH@a$Wn-
z%E^;FLl#Ku<Zak-P{>&$Z=*^4-GsvCrwdBjq-U)zKYveZ&fdgdMn3UvFHe7nlsUmu
z`$LD%B%}7>N40+Q$~`9^Up@D2#^m-t_K!|%bgdJAnZUjDdGCv(0cSolw_NTtKhM5N
zvUFYj#o5;jlK<Aduz$C@=1N{?s$bl*`+w`VPW$y<@6WEz-cRydzQoUZ@+P43e~tF7
zzsgAq|MB(wdslMy|Hl*myyyPQ-{V-4^FZ=hR88rg-j>IYG@LJK?J(k&f3ULq?%Mxz
zIF{@XT=Zk}`iWOUYqx&)km&HM%VT?);WhJ^sYmL$|19hN{y(;c=bBhqqUnO_wtvg@
zp4R_jd!ju5;_ClW7yqv~_wO<9&%|!-p2L5{f6KqoVOOt<5Pg2Xw{p&>sQ+Kf?9Q$a
z*ETr#M!WH^-}RmUepww!xGlThASjN9LsKDI>qOehnJE_~YIOhm9C~o{<n{j!Pn#zG
zm%n&=g5p2nE1%wH=dRFlnRI{c29A@=9_O#mI{5TIOZm_H(uy&+`2JisGhXr1r08z_
zeeM6=<}OFx|9^P>&id6KTK`|O{Zj7st3J`>zpuH+JR|;xS$Vh5=2ll{ykEFTq(!;#
zc3z(Kn++0q1v?J@{x7Kan$5kO`LHgZx8Y&lbNe^H|NmshmHjo`g%|H%slOia^KIU{
z_tF<n_N@H!Z((ge?~|W@JJ@ga%m4bmqUEyt$3J<qk9}@@$=;Fr+-JYskB@DVlV1P%
zSlg4hPCmcj^y6P29lkSb|NWu3?ECZYa$o))Jyk#VeSC4>sbBLWX8bR|ccARl_y7OD
z#~<BxYR;wqJi?Z0-R75nKhIc{w{YQA^Z&R0i|qe;<(ZV9&4M|X|3BV5FZcdm`=i@u
zO}$<<<<|e@?SJZDvmLQH-L|ym|FYlz`d{Y!S;*2hefz(Kc~|~#+k0r{#(B%MKQsH5
zFWvC+M?>c3Y6tBEA8)f|s?wJy^=6(twk2h8@TS1CA7_{t{5|^Vwrr!pDpNu8g00tA
z{cX-Do5Gt=@Q<DKgW;}U3mT?{zx%)aePL`%?6$Vui>k8TSZ<j=Kf8{@?rL!O*=N~1
zw3S>}%(4#ud?D*u^jW3VlF?uJRwM{~YTa#d_d!YXWnMq?9d{mh<t6Mq`czL3#4c(+
z&8ug=BjbWzch>3In|rIbo?cWnJ0yQ{R_UHwsey-61K)01v5m=kp<lT9j5`TM%_k?H
zIJfrRPCu_#exKg>1-<rjdwbQXY^zr+Z_&NErFU1GT0Wmr`h54dEZH?t?8R!GQW|+1
zcXS`+6*G^xlTg@vly}+;7v=*Ad}Vf-<#N&g=XQUYR@Qd1Wuwhaos+Xw|K53i@a639
zm2cjD`%%B+#RvO}`R4D7t#`=oI&_uy9@9Q)nX|k++b!N)1Sgx9Y|EwR<Q0gS8{A!V
zu<*w7v%l|na_!o6@F`n(5%XzXz4Ag(kun`pwzxLmdw%x!p33>=?>|4Qy|?H5?C+JI
z?eaH5=<-Xw>+%eC9(=+!S$!f%cJ9(E5C!*F6dklH&kN=^_z%|vHOczqB6f%Rd@!#z
z-{Hn1wr)@{bVMa=d&J$Nuh>>gugTl6>!4Tj{_iyvGnfBmJnWMt=YHtF#J^Ab>zmc8
z7~;P9n_1t#@&9An|3Kxcf0yf|kCjWEdvV$H_J)X4`(>{FZ>)Ry=YRTIR)w4?_4_<S
zp1Ws@nEyQA=k+D){K>D|Zgu@RVQE;$9oiuKU))D-QCYq5maeY9`jY+`X1hPX7wrD=
zZ;rX(t@;^ag7^P>E_!@-W6Yk^WNrWW<M-9S?Em7mftBaxkNqY!hVpA0eotRlzW+$>
zzji^sf>)>N?{F2|{r~9CUw?_S^?%tqD$C=NmF0FSi{viZy{bQnljXsQat)Sj<)?+a
zA4^Qr{VKiuJg@rUnfg~O_I~|l#QS8ck>BrISG}8(JKkE|UM&@C5g9)7h~D3*6}~s8
zH;Yyau-BCQ&YwHa@65*L?L3Zt*WH%PX+LMkdAjeNq5ss%RcFpG-<E%3{>>l9t_VDf
ze|x_}>a?D}z5vUcqb&J%Yx|Q{_gj5)+Oc0vvGMoT%WoV1GZ@@ZQkx@E_)1dy`IC9A
zyG_Ge_H6(9%$e!9qxO|guO624|DJc3_te`HH6Ox0UuQje>E-DeEXzKxF_hivsK!v@
z^-+JrtoGIk3LC@f_l6l?oAKn@|6iZ8&vQ2&+}(RBWwOS3Kbi0SQx0tVcPnKD@7>8K
zwrvhfdGcL|_kH-(=S3T4Dd?K{glrTps;Cc{Khx`a)u#N!d4C0ehrL|p=9+MH)AIhf
zQngzg4xcKS?D=>sYom`>Y<|DCE;dL0+9&2UIW>+J&R^V<zHD*Y6~5Vt{l@9)t<U)%
zXNmm3asADKBc2mC-%D~jT`iRU_v6mJ>Pu#?h*=R^A6wsB5~SELsfb6P>GHFgDS5Lc
zmZeqYxi*<@33tC^V`F`HN@nIX&a?IJ<zzKQQ_IfW+ElT1M&iM>1=A*-h)!qkV-KBm
zV;_s_hWPC>q$ja=c!y^FeO#i;!{c!C>32K%di&5TTw5nUJMZDHb5<#<wBp!R{|_2>
zPR;n2I&;y6F#8gX*@bMCf149$y*c}&y!YLj`g6B=!f)^J{FvRkeC69D#R>kJ78~^s
z3kU5@<UFo=_?E(^gb6|o7WYq^zg%Q_+pE29-J7}d-`P4g?mzi3?y+y^zpMM}_as(Y
zax3nZ?}`ZEytX&y@5YI9w=k%ji~C`k9K?OQ{){xYYxKumHa$0HMSh=fUbyA;#_CDS
z!;9`(3AfJPH}ldNkDM3hS~}YA-aNhci|q&56$`X7J4DZIxNW*T+~QBfQvd1Va}_%)
zLcOkhE@O@?yt8rB1{1r-s(tTDxy2^^6LS8$=G!gyCGme+c{A_ohqW$I=Qp)&o9pze
z^K;$IYs-ooE^ME7gmu!X+KoLYAD<Hw-SqOk=@~DXv}v2``>!qv{NeHK)RPIpGK=QD
zTqf~r!y2(qCt3Gun#ualad+anm2sfZPO+plBmP4B@&fD3k5)2EC;6CkeD0mhtGZ=^
zX@k_Y=Z~U4zum$$QLDN+bjfkAaJwVzEX)7y=Kp^ECU^0STWdM&ojP|g>=Rv)bSFmg
z>EAaSzUD6~4`W+<TeZ6|JNd%8?YAV-yEWTG>ZLB<$vH45u_1riD~_wZ3%6bF_kX;!
zk!@11*WJoY|MKcTr`EkZI(cqn;nc-9?_b}YsL)Y&gm+EX<!95lij0_3t8f21bv<s)
zk*YeS)T9G!vhwp{1=~-h3%{9sJo|s^cJb-V4%gP1zuWiy>0OI_NAbuufpwdYzb^eg
z`G#+ugwMT%Pa&7oTz0=Yv%~Sv-(>TM=?mv|X<z!YbJoO2=C+4?$B*d?Ikxe8AG0!=
zR&UyOd5b{AvUH8k9lqc7<YO6kZklw5(~qyXr=UN_;N&-%lpNk0YKmWT1f6VhUURQB
zv-n~(^UxaIj#JtbE^?o_eDGahU!&db_tUk_r<Q#_>w8k?c~)RgdTsF?M*lD7U*|np
zz_$Is{AC9}T&~eQvv+EPUYtR|Bvz9ZRd<bd)XZA8)0_8KtnSYHf8KnNI44>Ey!!Ri
z-7Xj9UZz%+&72qaR%Yv~z4<1kGtw@+)8;!BDCf#IZ_N**r;Xmbv`-!}TY7x9QHg5m
z|FCnjD_%G<W*O}KHtRgU#H`b2be6;z8$DN1?BA7^yGDfNn|bQBo3qbP_?5PAqTJ(?
za{k|r<%{(c+s%HSXE;54eqe5w<wFB?nTD<NWJ42LpH+R9e(-*G=cW8#4h%O;nC_KI
zE-5_3_Wb(Q>0hs0ntx5%Zknw2mSz8p7hBmpS5(QEw|lAH<7uYmHC5^+!pRd~9oQ=H
zakt#_c};gpH*e+qZMAphmzZ}3^K~YjPhR+O`L_0b`8PuK%acpC#~Vn;E5!R`e3s#l
zN$j2IF+Jv}<;}~hW<)Ppock-*|N4!ty^;NYk3}trV6uHbO>XY~YoA{(dROE#KV<jU
zy?38B_rF`{aw(}V?dYeyT@pDbO0G`+yl1)bdSlybTP-e4J}H?#ZRVbXPk6mQUw;^t
zWYb<K*u8w&+qZIO&dRR2!8bX%;NP?4-_8*QrR>boC3QRQ_n)(kP}?-=xN*{#!{KSw
zRn8n!Y<^5|{h58RU+quEqGM;*ukDNw+7rLg|HAQ|b-w#dZ=SC@Q(&)t?AJMGk1aBN
zlb)OUIc(CiN>;d2>c{cvh>(Z$tIJOxKkBvH%kcS(`uU9t@+A|WO8m^zOJcrNQqg*L
zL;2Hp9W!R!oOkJImEGimt^?;DUtrsBx@E?hn!|H4oO&;t6!xzAH2r#tUcH?(+qRTF
zJF4%Re~mq8P_TXf%}d(5KIzDuIXE?jMfm%Th#<cu!N#Xga27iC#6I#AIvM$OLc?s{
zv)fB&aB|8XoG4&?d)@YXOw3iG93n3-ZoPUhdB?PyFHNlHZY%fQog=q~GqvQd#ODpS
zfA$7Vz0tcXGEe5Kt0TY4=`S+`!o7;}fA}bhd+CVXP7A&vc3#rf`;*zGns<NBh=g3T
zR$j|~HTIq&%k4YC{dp7QHqHA#U&6gCmc8Pq^Qm?foz)B09jMgRzVP~3oSSx}n9gd}
z(|lj<?+q-j`}0=SGx6XC%OkCq<b)1CT;QUc9^5#g)ApFt9&_ifXUu&RXME48{?w)=
zarNuGLq9cc#Hs2;Y};@-d7;tgd0(x3Rz$o$VxVJ@yZ7Dg6<Sq+ryV|AdU>^F3E%XU
zWnbAcOA7lcr`)Z0e@(-sp7Cj6oG4G*$Iy6*#I2>4?mAt+Pj%Qk_H7DH4(VZkIAKe{
z&1I$g1sD6Ju6t$lc!R?H$W>{@+c{j#&gAhub}-rhK=G%}!e<L)KBc%dYp*fvKl^!l
za<VepZV}O-n(gx+ZNC|G{G;KC>gQYQ?w?<{t7d}|;}qo&nTOwhRMh!-^>@YbnH!Gj
zW%@*XZkg(=wOif0G(^V2*4nFEBK%|K?v7tauHD{e`&a7W?a){s>C+ZTVLz@MyIFGA
zedUAs4u^Sljo;L6J1es7P(FX2uY1_$_Lo}m?sc`lX2z&5$Zmh{c;>gb$${34Z6Oa*
z;@zw6G>Y_mZ=L8FZm+yqF3T#cUf+*ly?Iebr+3^!n;prAOLNxW2-tXZCfodN=MyRx
z|BvDbne<|-dag{6anqGK?#CCJM4atDeYZ{h|E81ol)|gt9G3qr8Jm3bmfy`+w}Y}d
z3tV?vgo`lzJ)YbDJLGW0rEY7*O(8ozOl~xIFI!lk$aE`Yqv*ey3<K?*AAaimUaKzH
zedP4i*=)gCS{IrgJ+c4t&hh2Fj;+x=CxU-R%K03gy?MsZt@{M4SAUOn+{kzHKzI0c
z^8(WZhQ5luo62{E?mErs!S;58tX)rSzsxS}MT-2>9hXJ>2RF~m@0H#Y&!)sH#&#+H
zN&oV;$}`(*r%a#t^|EqKo@0e7m&lvu`PP0al|D?4>Y+bw^lfHceY8bd;>1^*+BcJr
zPm?=hGB>t9Kj_FTf%G%`W8|E5pKq3_`6dv<ESo;nSNl~`)czK&Rmma7SG_FRQ;Nm!
ztSj67{Fj2E57#9wM(!&MWnQld|7Ni{l|y~**)KP?9Mv|yd2#8zIp_Jz@;*1DpD4{)
zfBvoh>1XvmE{-}nX3x|!f*L1ZJAL}%=e-vc>y)<%-dB}(*!4T$;mP@@G_wA>xn^Fy
zxM1_te^)-funPXa^77*JW3N6PH;#Y1Yktzt>!pkK=khJvBN=@s)mKWkc&pZ)6Move
zpXVI<x!m&ir;O?4U#y<EIPEwxJ>YWC3#WqroOX9juKU>E&`O%yf3qlOdsFvw?~@OI
zXMVmSw8-@DVZM8Zw({TNzdP6cQH$creNjG9_MgQUt~Zvg7VupYK3zBC>!azFhunXZ
z%V<yDyU?IU@ARF8ChGF+?FBL4r@75~pDwTb)HW{v`ZWi4drnjF-*;c}o}Itp`?)(c
zzp8R)i2t`M{-|6a$Y%a&aqwHiwRUIx1fC|$HSaIB+1MOreNf1*)ji+MK3GisRh{>?
z{nyj}9<(@aGxt>D?6iYy&P(gfCv&QGM;C5-Vf;H~(|>j4pJy(Ay?HKdvugRb?>A@n
z3){T$^GyHvtmZzS_jXHflOx9_*8S<Z!Ldfiq(}Ag;jT8Z)7$pcio2Irx>)_c9OPJG
zt0kzS-DmH{{*1jbrE*7S=|7$KyRN>|6jROkc24j211`1pqakY~wKwdW=%f1U!W>m&
zrK5)?C4J*snkkjk&BJ7V&+ClLQ}26B*K*G`PnFZ|k=P@xns}H;<@nvl2DblYO%0b!
ztBf>mu=)3BYtOcsWjjBFs5suS+mIRbqCot2#)On<rH{hiD=S;yjQ4vH_+)(!R{_s;
zA&aE$^eGqpf817e_`B3~&kS)h!@}*_W;-))J^egm?@XiJj#=58>SpSMAD!lNy?Va)
zf=PRi8qc~uxnzdKGvN;{F7qx{#BQGUdWSmaR~wclk5(@`rk9Q@w5m3-=;XRaigK#&
zeC>9=xm>UDG&}prwR1N<ob|`Q>fZeSD%*UsD_`!M@zT-cQq7;Yk*BBGEP9b$6rq}a
z<KDswjo%K>owwTsnW;{;Tk`0g#t#|pIA2NItEu~TpA3nzh}h=4?r@^Yx|Or98b$6r
znNrvj^Hg@}&&Pr+0w)>H&s`GQc3-Gm;C1#T?WMLi#TInj%;MCOc>UGt@zleS{yC51
z7cZHg6|wZYMgOATpKWVbrJtOw<-Ce-)8Z*}JJUl==bcc0ys~7;&K8#E8vCaBCA=z8
z$l9^KUO?Vm<x!E3*wahb89U!TX>`1(>@a=KfoJp7*tMKA=fuvN<9Ubgb(Wxf^3IT`
z0<ML<GcLDRZ#LXi_xE$pe=EzzH9z$8i_+rCW*AtVVV?7wZQtzg-{*Uz9xe$Hxc_5q
zyME`>DZaXmMt`}c$-JHZbI<ns$<9ZMF5mUt{LHp&Tj;!VJiV_?^G{9`IBb(1<y;ss
z<AI<$!-@XoY<Fe9Wc)ZPd)sJ%g}qh(x0(C+s=fZQ9R6#5?8)ux#@E&xZgRV_{oj#8
zZ%ua{eZN^;X40R61;WoBWbF8D_AJFn{POL3!ApmOo6j2lUTrUO;OJA46Z2l3vY4G<
zG^x_9>C~(Vdu!s1td`v=5?=FcM>MZZQQ0!{J1K{2u0>uv$`NyVs&De+6sDb)%Mvdh
zHkIF-bbr0O`}+T<6#wYoZBU!BCh2GCY0u+gvpWQ?i~G29HJ1PUU6@n9>AA~^#at(2
z9gl6QFnP7|^sRO5=lAV-x&HW~r5x9`?`!=tQAgFrEX7FGwNTYjaZ_uaq0$+q%DdNG
zkIm!1YNuP(bMDEG?NLW>^(9QxxvIYYgVU?VtQ(!vljQrI&u@s0mh^vr_Kc9s#2@i`
z8D8gadiCduINcOwoELZHjikxC70&wdmdrQ*o`3o=wl?1(r7vY-BNwCboTPM(qj3)|
zy}P(bmD$XG^H%565`0Vc+^V)b_2v6Rues}|-<hj*e9q^1D~YeWIFJ906Z|Q!Kha~d
zLtntmpz5^$x0Z>==LM^agsN$}9=!GIRbFbJOlOGhzwl#mlDCwT*>4^R(Pq6l``c10
ztM#^r0)u~>F8E(ie%Jg-`{ttm**~wAmvqK1I`Dfz>guMFH8V|iKg!AeH=|EsvLWA#
z&&pQq56#=|pZ_46@w?Du?cw}M&c9`9FPz_2U2&`M)tvGhcC5KU8y(A-rHyq=GB&^e
zDgE+y@!`F#xxatD`+a^*Ezc3l7cUnqKisY~f2-8;#q)EVpM7XNQNn3@>fX_mw6n^e
z6XQ1-82PJr>_7S8E_>NTrE`b(TD{H<|FPBdg5mwiMv@nMOwGDC%y!J1^7DgaUfMZ<
z=l<4-e|@L^Y|J&fe|+u%iDSF(ac`FRlr-bD;H;v}dpHjrtYp{TAkli+$L?F&`pcJH
z)gN5Asbd`NS|fDkwc*=|TSLxGbK2+~_V~Y3NI-{^Tt>fd;M(IMpRYb@X!#{^?`r*;
z0$wruJNfcWryqMB+;it@*0x!D7A72ME9f=6@-f?dL1Cw%dyAW`O41{hW&4UhOV_f=
zlyNi%_h_ADGuXKySvZ|z^Xas_n}2j>PfumOWUt2X@2OkSo{t9W&-W@mHhO3o<SgR(
z<v{_<0}*SL$5%E#Eo7dz@<726TQSQE$}5*`zU{a}Ay4t~ikeTsi`LpTCX0N%#=S^$
zR{TG=O~LBHR@a`rTX=Xvam!4p7&rT+`3wFZ51-HQ^uqg=XIAexWOeJ`c3$rmVvLzz
z#S<+y^G>T*`r*3Uoy$EZrn^cdwqF;#`XFuA;U;x|vtJhuiP!W_x4BtS|N8#)D|QLf
zmozOq-Y1d1NlG@iWQ(r<FN@19)&G7(SkHYsF>T|$_8IywUDHMG8e2pdZFz8aeQI%I
zUdH{n_`mfN@6UL@|LOd_d0j!T{;u(CIDTSPw6j89Jo6pxz=N83maZLM?bf$WpWnfj
zEXVUYIpJmDu3YZr!rSIXES7(Eb5e7edhV@i<IlD4zm%MP?XlL+-qCW~z8~T#Y$94^
z5o_XAj-2`TdZv8j37!7KGS_$3+F8Bl-MQ{}xcPpG1B*VNwiZ2WcsS<N_qMfB4~_EI
z&)vC4soQ+rhw3lK^Y+xeZS=3uI~VZk_2-=*z3S&$L|kL`QP{NJulw`y^!8(^9V<5-
zk8c#b>GuD1=9goen7gYStQM$SRhdN2yL0`t;_}x$&&9VN*|PJ@^{$^$3M;>GJmqFp
zyVJua@0G6ju=t#7$Gd}~_l?=&tIvh~+ADvt()p=&TAoaaWm!VrQL|SY=EbEi6DfNs
zJ3&8wvgQIO%{%$Wvkx(LZ{7K5#+6UK6(PG`RY>hf+8B61?b})Z5ByUf-~aJ?iE;bi
zyzdj{mzw!ymMh<oG4fe|=#ExorI^+H-Q54bHoE^P?k@fkzis#O>x-6sUf1_givRoE
zOH=oM{bA{uzSZ$d`n)y95``YJhx)5>@4axbdGvVA{h0rZ91bbAmd#A7o30-Cwf1=Q
zql8EO&D|STif5m8WcMg8)ZqW?{OzBBk>$RJuaqobnddb{<Xe5tvNw1(XR^`98|-D(
znu`LIEtho%l-KvlXggeZ^ffMHr;mK4*;&7NuYK!oRrCnOXMC11eYRqLWyc}*k4+gX
zs#nZSENi=V-tT?SsjCyd^nTBrzdlnTt4*b{-qxp7+WnRJ)F^|if{hm6wA74uIxde{
zoVcfdep~_H<@OAd&yoJ#Ri`Z^OyyLJC+ThO-qQcA>!{V~sW(!}KYadMpYfr_r^NoJ
z_52$lK}P2jWzN}nJl@DT;hx>+Lb=zR^OdJuEOKc5TK(YiZ@r7#U*4Vc?BSiOdSBRX
zuiNI5@_gE|4X!B>l?f3BJ4E+v{&B9(Wx~+~*A`#C89eoL_Pd>b=g4f!v3S@o>z;0P
z+syq-j9FDs`JK<d*4>YG^t*BOaE$Pr8J-UQ=dbP9{C82kqqf`jx)X9|-da_k%Ff~n
zUv<CvV)@>Fmm;lG?4`dJ8ir?|xhhr6!ea6Hw~NH1)Wfb@WgdsDjZ1I;7nXg+e&;#S
zg=}Sa;`?%U?QuS($gJQVlHThQxB1(W%LYNk8q?CZRk_VI=v@#v_m0GuO25@l6+?dX
z+{kSRcxYqcsrq%V-g?$`X{T21*(s-c?YGMOE2k7r@HwkIsJG(zs=vU+G%40+eZV`x
zg970;#g>~2=61!+x5=3KFhslTr2D!m{b!}zt+%G1D*n*&ghMxLo@w%_1;N$K#}E4j
z_3yVd>=N3)M)LW4-rSc5%uHV8>|B43aR;m7n;UZ`o(!Bl-T%5nRi=lIxsQn0L!)=^
z=iClaO50vL{~%}Cw90w0hdUL{SVX>G{FkBTpx4ukcB#UUk_FE$udG<qmQ(rWyXh8Q
zrYTw0%<cc!54Y_3eD4$QkH%Z~g+)!`pKO$1OO$=M$w*xO+fV+&A5Jzad_s~VdJ}7J
zN&1V}XFoNa{5yX2a(fp$@f8uVWk(r~iRfyUeK%bHRwsJ#>Rj7<*6$nZe|_a$G2eHG
zGS3U8lV$fWeXd@3ndkhU(k0wtZ_Y{nX1w>&vN%3xvPSvw@At#R_AdVwA9h&v<u4oW
z^%s~Ix%~Q6^`-Py+!H||k#FnX{eFIL;yG_NXPI<Gw_4xn3{AJ*$36X3z2Sx@Tjt!p
z2YU<Mdqho-tyldnGk0mAxx~7!&nx$yXG-Id7Q0pcEWWqu;IqdJHfk9?zuyO3XPY21
z-{Vrho7!i?+4-xVd+J}?A~H|l`}3O8w0Upar>+cg_@V#z=!Ag0k55X!yr1p1tNc&$
zg6ZWq7$4mFFEeZBrtfA^UyKT4)MSnKFRR;n-+Rli`073PR{va(lX7nFySeX8Q;wYZ
zRW;|L_P(;67hZ3Stz0(0bq-tI{!jO<KW(q~5I--b?|<a0oaH3f%`xHsOIPoY>wfWZ
zdu-*|<I_%<<*fO>`to_%`IlaLr_{V#>B+xo)}HB;J2jM?|NQu_dL-@QzfBh}{n^X^
z<6w`$?TeH86=nMVFPXX1rtXVPy8JRr-8nPt{Q5M`*vyD{Vzs_Rf4B37FX?;#|7med
zwVR-LjZ>vMDxWKgx54@4_KiL1wpY8W`wR-r$}4a8zHAdd{pq7<a--q#RPN-tRjpe#
zuh0H<HRbI6>3rWl$EVt5{jHUDo0+usj%~(zbB<$)_WypL*>@~dGJD<m3B|k5uYY~r
z<lp1WgKA0rj`b|pC3k-0**3M@{i|%V%H9Viw{E?UuDr>drD=S6?f&;iXJ(%KY^8m*
zFPCF?07KB9&n}5qI9d)JS+nJ5#`6t3JuLsdZZTC`9>4#)_(_Gw(i^$|o{4>RuyV7i
zg?p}<9Akc6tL4HU{}=KqbGi!NX;AjLp8v{S%wtpM$zM}$RjjD~wli@VM^o*h{om}X
zb*p5KKT*DZT6gWDpY7(aj!#+m<M<~zyR?6oXZ}2w(sw%lBu~wp&chZ7+uIc-ru&-y
zl`d^%d^69l<>n=yo(U1nPhvP19ckEEP^xip^Ff2-H|?AGq_ih@{Q9^z@#Q7)IlBZc
zO(l0NoHO%6MU$thdSM$Yo9drAEXSh~7aot6PTgAK^{H9G@<IN*%{%_z^iM3cu>O*<
zZNbuiGXwqHDw}gtj^FFAS+nWbt%Sgst<@Lj?tB-%e!f|hfA<;Ib6a?tc1L>2E&KT}
zVbjY+(l!d`ozEMtwJA6*Z1mY`NxExOk)@1=WvROL2KMfrxvpo9ciG0u9TUkqe|(FL
zaBvt)V07n|%u<bfGavSEGYhsH5#&!jd+4zPv+7@cA@1wOItLSq86%XhE{(qMs;zcM
ztJeJfEXK>(KASu?|2;ChZHw`3Ua90AoX4K}3a?mHS|h(xKe6}a^2XClPQKeev9O6e
zE?eN@!ukEZ?9_R0`1iaKTzf3$ijB`6uGe8Ix6Zm5RNHKG+PL?{tM<T;0smE-vX!zb
zR#ZtuGd(_^@|Nr30=c~^bMO8=xFSgBE$f=a%6*eNFLr&o!oJ)6((C)XV^XU=TP)er
zv2DlwXIpNkr7d8#t##pb%~;MipOIbhVNv3F<sZs>)AZC>Ebci^oogV-{l-P6nW6BP
zOxW!`Gp)nA{X@!5y-oa{|J=l7-{tP<N-N5^SIs;7XU5IKIne>@P1oFMU%vKs3ahyC
z?ykL_$!feFTkcMJExc~as@qA@6E~d=KQ&XQ#o>{f-Cq3@KbPMBV=?2QdPjfM(U8D|
z+pT7=zGl6Of2DJ^WZArwHBs;E4K-!rVqdOljy>EGD5%c-#d5FQ&*O3u-BI;?<%ce{
zPh8p~eC6I@LD^M|yG`5QzP@m2+Wx>Nvy_b8f7h3?aRmzMw|nf{pd}Gs%y7A2<*b8m
ztS6}MnwZ1*F{!VvU)tIE@AHkxuRffeY-zJi`*8WW>l^CNrbgHQwUuD^{U6u;FS$6O
z#oYH#_{MEVBCP!kEepP1v~Ws4nj$7pdNbrhk;8Ik%iNgFdeO2gLod}f$gisV+vcwo
z;A1f9&j;Qw|GvMJ_-;LA^~b2z%CLpjhFkswEJ#on{dv*4rZW1>)2aOt()uc0VFKr}
zbT>t@*66c$y}$f;?+y3nQkmy5yBc=Sh}#>e`b_s-@9X*IXEN5`Klc8l)xE>tzdz1#
zp1eZYFKkZF=d;hBzuYvz;qUh^UNK?j>t3Jueto0QW5b;}dD7SP3zr|?IREQcC#AA`
z&WFMZo>dw3oKNBD-Wj$0x%U5BuLGyHn@Dgi`krDd5hi#*+2XnS)?K$&%V$jFzm#C`
z>TC70-lzNIHqYx>*K<!|-8PN|_4|&BXxmQ8nLN2TP_ZrY7XPCTF>%fH&wK?hWU{C4
zI`;mj|Mi+ZFXxDbTv@)rSm~Nc-j2G`=GmXCcYjP3c(Xa#sWCJ>$f`Qo+aX%(`Fr`-
zlXhiRZGEo&FMgf%5$4z}_x{)O{wR5AzR>N>j*I2;N9?K@=W=zKN#_~KOPv&dD0KSX
zzn`|=8N152^tX1`u=Te|CmLAH?R4O|HnaA?5v%#BmWTeB^(+^>x2$h{s@vAhCxkd%
z_I+1bTKUFszQxn>fR-7X3vY&RKK$mSW9}#26&se`eecP?hgs;#o3xG(FE9De-_u@@
zZ8xb~jBWL&3l8VbR9)<44tjR?j-*$j>8XPWD?d!`y45dyp?Q5@_qo&FC+i$$_J~?-
zw4Q4D*Jgrl{kQj`KEc0V@0&Ak3RieOuXpy3Bl0(=T{|@4#a-s?i$^~yEd8-&%2)5#
zolC{u-JaRU@AcAn#<8aSmD?rcy_@8|^Y|~CC}vZ2ug~pD`1{|-Qnr7Z*YK!P@5?F4
zTQ9Z<Uj5=xllbe_MhSEN{dO$IDxTf7D{s|JT-*ImVbR+&&gmTMo?n-YUViwm&WGjp
z<~xedgfo2pQFEm#|4i|hxBCu1OPM!E$%jc=uvGe>*@?ETj%U{}OwIi1uM;QrP(XV9
zQ?~y1nsaz{Hq`D(+j9JmW5ho##>LOSw+D-}f9pT7zhGj_`p2i)R=b~gmwt#{*iI?2
z;Lg76jn50qt|y3o`pR)^R@l$_85563&MowdE_%tfNnpC{repmatRH`#FL>Vk({S1~
z<--f2dd;NHDpw!PaOT<bRJm@)n>%SgCNI4CVa0NrLt(Xb($CKvwEVVTMY-8>Pb!=3
z4cnhD*X-`TY!;jUJMrP;U-v%xPO)i>Td>c}UDV|Dm$svO<Eo!Gr@iKt{yt-c*X=(K
z`<$+J&*<dfJn?Vcz59(t=4ZG=S1g$;KX=xI?ER&4qz~3h{M$S+ID6i?ZM@2_B%ZgH
z?bB1U&)IN#vaQXYRED0J`^x8L9bUQmU046o?b07ywYRL75VD*Xc*bJ)E3sb_mrL(3
z<~hQ@r=+KVSFu<r#VzKZ@tZY~r}yq*na*BQnJvn*N>SyBwo+Q}PpNr~XP3OKJTXz?
z$kp?wqw@^K%liG2pV(?t7n^v6Z1qiCd#Lp<r}v+|@xgEAd?}b})fW}%_H1iM?={AF
z-XoiyD!)?q<`tMx_I^^c&9Q5~dYKJL5(iyh7J4KpY9}yg&Oh9+>Bc_6Rcx2u%$Zg`
zZ(rE$g}koc-|SO8oVoUr9@o*<>ODUXmhalS_FIU-dXAdr*vaku6T)4Tj!W~JWQn9k
z-}(RMdDFTct2+~(IxOaGt-JWvzc43E)qIDIYH)~~XH4Jpq9c#$40f;MEIXaE_G011
zc}G1i?U)-M{{66caLi=s-p9!XC*IunlH9mLPSY;6_?pN=(Soz;&$4<YPfQT_Vm`gB
zV$F0zrC#P%vFEkL%F{)9l=zY-@A>pNHEnlB+*wxcOX30Y0y*jozb4+B$n@vLqw7WY
z*6Sv1^g8M{#k^gl`sVMYrkfA`Gdi+pzNN>jiK71xi!~p(yzBM}^W*u4ZIfbh@7lh)
z$9n8Va=zUJ$G0|{9q*j#f4zOjA~RW@_e;;tuFzbQX{U6jh&%a#mCNo{n?*Tn%cB|h
z)^Gp6%<D^UB2SyaCCQV=_AEc%EXQAEGpql)M&6{e#xph*nz3ka*!TF$!MafPinX)d
zI&-U~WY>zXaZP``oa6H!^S3R#?dLNWdCQ;v6|!0E;)&}PI%nSbwOL3iy$UVf)w<Df
z+Tk_ROntrO-Q2HoU#eY}zxZ<3<_8LWbK<=p7SHOP&~WVXo6J**Uu3TNN*k@cQ{a9p
z;-%q7zg*tV$NQf9oYh(%S;dldJ%0C-eVI4!L@bgnthH}0yY#_c?|6>xkAIt-F6Wpm
zNr?`)Eq-|eOUj8|UpQ`fUoT$Y60<=^{EpknhzZ}aB9h{}FRr`6yNoZ~R`<Q+x}$S>
zFL{|M@9kKe@ZK)&L+0BH%06e~wtw^OXQ_JBx%%MiZ~Wo)YdN;8tM#|L{pWf*<4;wY
z`q!El&2Jx?_BV2wWM@u5uc_>-+}$&l$2OTZUgEeQIOW1xwU$4hT==%fN-^$@uIsG1
zkh%Q9rX%-sFRZ?ITT%W6x8kyP?n_pEGdU!xMP=?i=J~wNphj8g^rjh1Mjv8LeDcp8
z<^Pf|^5WlerPb#$qo+kq3%s{hHf!QbJ*MjNHw9Y_vt1kBWZo_1Qn+*YmeYSJx!<K0
zr#}k1dqr-YV0_E>LE(#o|77Q+ba&?W)lX$OH%mV6#nyex3$&`2SZ&|%{r8<84Vz@O
z_MYlnzi#3iwvVf(Ie!Z)*sxdIBrdM{l<<{;*faSK|F&F?-EvlJs!;uJ@q5`@WF(ER
zSGqgLducz4`h7h^iE&?R*yV{<*HRVLO<0!4zBwK->ss9AmtStb+wn(L@sObM`LAwX
z?}RkMG%ih_`p*7(*`t%QM6W+x8O6p~T<GT7$G)LN*{6mvW}E(1>t;)X3o{wl<gF26
zH)_~?a^2hI%zjl2$DS;*ace1f>vp`_+4iNFdiK@oH+<q{S9lM7>u)f6yTk3p_sfrp
z4eQ!HEM7QDuV~GDBHPkFA)n>zhw1E3DqpFa+Pq2ET=Zzi`BlvmzXweEb$x2x#=<oP
zlWtymYGf6$=9zidtG3=Rxp#ciy&G)4w_Mn9=T&OfYoWc*6V=MACaYaOSE<IkdRgh3
z4<gIiC3~;r?|I(u@~S&@W_tEwf9cnnmWM45-7C$X^Zixf(I0KgZLXNT@BC@@JkogX
z@nzk6uI{_{Q$E}<Zr+!N@sj_3Z4CWXDmFv^#WgGCsCNac&6gJDZp&Nm@IR`WtK!<9
z8%mv;Q6&+}{4=_@$LOWc)6ZX^J^$yPF8Mw8V%wi2X-~{3)MxnpQ<+myB<ktB-+!Jx
zFuC_8{7cmxTg%TM5A3zbmo0z5U72=T=pa{j<%auTZi$=sWE9EmRcTCRR7en=U%fqK
zvPb(JrB`LAZ#|p#OD6f#nrW3PogO9aRm<KBJG_z$uFBuM-+k(8)qeBYGoF1(XutXR
zl;G#mxhGHDb3f{%W`B6+r`eVwceW?JVpGk&AFjb55iR(nY#DFjuIOv~I|Ggs+nS|)
zof6v3WV*yunBz!b*L64bf=5$+)+U}8Se|~n?EANOYvQh?X2$Ih6Z!T+US8=ohufaj
zw_Gc?rv2JdY4^o&=eaM{i{tVZU#xBS6L@`H_W<99Msv;0%`%yVC$4>bo}9aWVq&NY
zx1HL4KKqD@54PeR%laa@L$3T~wRd_h(4+fg{rr77%iFKHe@V9cS@3q(em;AKfG?B!
zKgyasTVcq$_Kv@aTc^m3UV#n$UTT&)=jOzC`?&5?a>&2?I6&<WlR|F%vgJ~uOD*lU
z342}kZ@2ei{Mt~hqr!dRv*dJ_y@J(?@5uj{*!t$F>HF_?RrjR-SAAW5XYuhst{lm`
z*Y0=E?>9ZZ$}q63)@IkKS7rW<-49vW^e4udFIW~{FBZ=~wSJ;3OX`_W|I-2KN$kQ_
zGxqx*u}NbJSTv0@Y4PN9lapI|K1ntvTO2h$cD%ajx()A<c`J@H{5;R~bj9&?Ca?N;
zmasfii1~kgMyFqi#|6uloEiU2Gu8&*k=wZa>1&B6)(Y$9Pk3DWL)In7`EGxM>g?z}
z!pxKEDxZnz)-Bp0X_WEfo$?y3W#ZSbo7~(eeqJq3eF4WLkzyrAtFO`m^^@P)><v!;
zyn3!fhZ@tYxrVg~Z;!S-sd_tw-)NWM&Hq7huT8(tYTo>PsmlG*j7hCY(Nlw4+GF0C
zYfs{B68$$pTkP;e9{bC)-e2Ej_1fs<i|vXJ_h>GX)la*{lOS8QaJSG#d6)j<4}WgX
zjrp7~^Hc1aw^f%bXU@6OSZaLrD_=xJ`?=L!Pd`chWANGg{krhS5A**mwBwmB*yD8a
z^v>pIapf1@e_b%)vwmaa_tpBFx4oX@$ZkDz$#r3q#t9iFZ6|(<&1Ydc#6Rn~q@b##
zw$f5|V=wzD;zo0{?`hwClh<pp#x8B;Jh>MqVm`~Pf5I!T<ov1bbbrgl%<J+>w}gAH
zzn)^LS%3G(`kANa{d+U_{-ni@Z+|;mSF~yIo~}7*B!BSP`_2<`3j`niIQRIJ;a@w0
zYfJ3<l`LKMoz0(OS$<bC{qP&>IkgWAEW(3S*clj>e=5}p?YUCKo6sgD(Vcj(A+uJa
zLo*<##vt43uJMrv|8`pyJYV7_XP4)*%j<o%_1)?V%Wu73Zz*y3TH$gd76J2HmmMTd
z$35&Zn3j7oo6&5FPDI;&Ga13hX&qBvW^B16ai=+aj*L&!|J|W`>N&!fmmFv0j$vFX
zwXkUB)x4*Mx6QtaKU)(y>+Ex#6Tw@RQlD?1a(RaJq9a0;YNw>W$cR4MV(8whSl-t2
z{FVEiBfRIIc+9O6`zu#wu62CHc8ONMX9+JqRUJtZnSB5C-)Q@4%j=WlOTI5L-csi#
z6u>k|`c!0h|7zo1CQ=LR|38@b!+ldsUrxi*_Qrnsrb{YL?Qydw^&3BkUYWS@cyiCu
zdxseOzs4JP|2?93=JCIu#^)U6UjMt?TAubvTYfj&-1BZ424&YzZn-QazG%i3fw^Kc
zE}oYY-S**&r0#25?~SSRo;AG`EzaAZ%prGe+WqKP>rWeBe{EM1%CzV@GgGGE?M(-6
z#3sD?cW0ADjO@EZ2I9^}=gwIL@9|Mr-MxLo1(uq;f41&h?Np}!k<L<y<v$d6d*{zP
z4$~8_6+TXOcH(0eIC}j3tNy&+@a=U0%jzmFn9B3BI%hmxf8^=AIIgP~8@|3g_Oz2{
zh4Wkk4x7tnb4>WpZ=baPxy9UrUcYy~p5e3k4?~B4zg@Iin?B!cF`0g$<EoYIiS-5B
zxeB+QoW;M9|8(QA?;9&O>^RL>zVlDi$Lg+=r9HdWrUjn8*6`+I+VTz0>W;bW6>-mt
zFxN5==;0B3!8BvWJk1+Q+#3y9yGuLQDAgUA^hi$3M)GUX?w?z-KQ3N2J3aJt-3{$W
zUtdmtR(7rQgYu~wmgtt^EG2;%-^H1G`DTmQEIJjCwZFPiTlL^$=Kn2~q35PX&8mBE
zB`aGK{&u(InXKd|{UWkC8+2{sJl=1Xjn!S2Xtb+r$FG0s)ddbK?o7TY;hC(GTT_2(
zURA}u-HYG9D73$nWj}ZG?v9K*KW^?;NqF|ER6^UecK-Socgu|xGnVvyx1D?Ba<bvd
zHcf?PKR<HJeVTrG!^dfx10O3*(rG%bl$6(GD0AF}QRSqq%BC`dO9wSRR=XyA<UgAw
zbGycD&8&UjL|RV#(3n_LUSzv%g13QZja1>RfKSIh8uPth^RzQY@am&CL2p<-OQ)8{
zvHVDSqp?W%SnP@D{o51wUp+qmr2Ovf#`X2Na~Zt<a<}MZ_Ak60-`||SlG*!{^U8@I
zDr8R?zs|mvR$aU2^NcTb?~lBm)A`e4-{z?Kr_S}7t)BaNW6jrXyPmomePBOsp*E{_
z?(3_)rM9yQLcV>f3p#%&+5C=k`-Er5li#$Q?@FGUKeNnLPG;TtoM(G(`4*aHzn|K7
zH$Q7T2W#ZRxROo2vyM;NIb-$Fk~QBs7EgPaD_XgWvGbsmUu$2>GWN5(8;#a|=}oep
z_;_;f3}J;$8%tD@-mcui_wsaDe}LKEsp}3G@x)h{dAigVK5pK(JE*+uW+YQav&xba
zt(HC??KZ6wf9W-0;a)%M)pFh!)>ytiaYJx{Rr<|CUUl!E$}Ch^ykE)6uEX|d*UK$i
zl^4vk^?Z3&-Y=uua-BtRU2(U_!?oY6nmevGHO$(Zvi_p-4Mj;7m7DXuI@U3#x}Nyt
zT(&{j^r?m55rHKRVi$eQeyU`zcUd0seOt@&tyMm^^kuo8mP8#Y{62eY{>k3#yAQY4
zmw6pzR^n`%ao6Kj_!iyUoo}`YrJem(I#aiK?v0fR(xpfA*kh76U0ZQR<vL^Ef)}T<
zFGesoBuBlTH$(W<;kUv&&zC)1_4&NV#&yZM8Bat#ozwD*Z?N84vpz~?rH6ulz~w`S
zwo33n***35wr32lC-FJ%=w2|-;L!D%GdY?MF?0op=Wli^-)(v*sVwZ4^Qsv~mg*TT
z4d42%%At_CrCsy<hs#M_UF$epT5L2*wi)-`dp|MpdNJEYorQbt&Ykuu-!RvQ>9t&6
zlI<KVvlmY`1gai1V_RRlK)odG@Um0q3(kDCYAnzAc3JDW@~Zz$cXqa%+%oA|iM09Y
zf0Fn3zRLVP-*Zy!TlzfPf;fS9965)-9gphCdiHb4(Kp@SU2G?9D0H=ROi`5nVfW>?
z&F46NmmkrxwJB`pOTSx2oPVCS_sgRIJ*7+Gb-k~fA8lN-`r_HAu32AmR-~nET_7ia
z{a*J0&gR|jT_Nf|e*y~G*I2LNzWZKh?%yfxu2!m*e_1v8`F^y>*B;S*CRErdT31)8
zG5vA4)~mOP6HR3f)t8@Kz|<E0C#yYSX~xl(PmHtYuiM<$CsyL}MD&tZgnjO5>Fb#V
z2IVhyPp%1za9Q$f$@{lA%6r_4Og?Z<HmW<o{nq29z%zTcbr16WOZ&167vC|n%5Abv
zVOYs^_ur<XpUTGit{qzzKKU@qVz17`yW3Ltu^aPUsGtAv#Ab<)319hoCAM9WueHBi
z_dVf;t+~%%LH*>@6JlBBxD@&&6rQ=iVbiZ$LH$?6y!#VAwsF)(mZa3kPYKhnh|T|b
z{OKnF-c?thd~Lb;L(pN)`q|H4=-$k3&0(ml+!^udp;v|XEMCRC7U$2Mw`RHb)8XbH
zt?irZ&i}aT(O~z)U{kMCNJnPBp5HTjYrCl%43@jQ{dxayZ_A%LiwB3V#J}HutMLqn
z2ZPa<g<JPL`6i=2BiHuC2gX|29ebuImz~-nSMOI{7<b+3i|QkblTk(dr#aQ`|9@3#
zUirSEZ0_xgyBFK0wJS#aIygN@urGGUiF?xbf?oe!Uh(JLw7$%SM2-Gx^Q7%NHc75n
zyJb$K%F6F+Pt_IZ_UPaGu9W-N-GcqsS7V==r}5U`a`>NUr|28Z5tlgk&Rxa+UG`?X
z0?QwtcGZ;VzhC&pB1HUh)`Jb-rv22|f3qppT1NcCvxrL^O`JSS9VD0J&)wHy{O^RA
zTY7Zg^-f#ytFM;xI=HdNC$`tJ^n|R>Y1+ohw)2y%q;r4ltHXtV7r67UXfjqzkNtJ>
z&2CQaQ~y6dnU~vpZRfs;H(Pffn{{*A1snfTs|Bauc`CQ?Ez7Am{!T1<|8?an3x6MF
z-PM^^^<8hl1|?O2M8;KL#WkkZe|^99!|mUeuNJ$%x%EUUv^?rt)$S#2;Rp4u8J_zQ
z@#@)(9rtJ7f4a#|IqVR3r0+Z9`<B~|zPtQ)L1|Ls-8b<sOhbC+sOcWb?>6s9ZZ6qA
z<GtkX6O;a%p7WREjrvuT_NdcBt^6*d+}WhG9Tm=EM)z+0+pw;hNASV)y{kVtJDk?@
zn-L?}qjJ$edp7UxEk!2BLp1U|&wkszQ}lD=gyuamIy+^AU+k6qG`sfYv5WPyJHzVV
zvQ3vST=xFB*<z7h2THH;&pqYobNZ%}@|5cdpTa*jr=?BzJWv|f9VaB2`SF2Lk9Dl$
zjkET18hg8|Z;BPCbH=YfIA?O<2|2IV{+fAJWv7~%mvz)1w0OGUA$!u7u9UT@C)Y(j
ztrIEbxHxULO@(`h@=e!fL%+jcR`_}MmUO2ju8EvhrLul%PN1eo_+O#kOOK~S7@gf1
zDIdG0P~}$b4qZ03_?~Nrn@V%G{`XkA!Mvp)O>BDYW|5W^-L;bain{HQt)<H&Wx2cZ
zP0uV{&vZ02Or?6EOw#PNYN_r^OG;+R7=Esg(h;0fYSw*`_f+RTrFnWv4xdt0_3mV+
zJA8eyIcec<Up}cT=Uc2SMGj>L-+kDzO!}jY?YfHxw_o0Q>135v<#EfqWrtrf#Ab!-
zF6Xi3>D(dIzLvG!?nK#>l$KhdD9e1ukLs%{%_aA~SDSgQq{_wSvLlc9>4UrYm=$kl
z>|b#<{J0c<sa1V@ijl32BCp{iruvY-mW8W-TCEITeE;;4-05po?u67%DL%a7`+fWW
zwMt#D8~sxr$ltJdU?O!l^J#JeC(quRbPl$RbG!EU9htm+{hPqB&vUKoYEt5w<OS9K
z9v8DLxiDcz;Ln$dF%tbV-D^L-(f=?{eA(+9UfrZy?+z9v<j9_y{y5(A)IG*6XU->h
z9h=Gas?_xM8Xe)Ci|+61lX#ZeKIQw$$tw2i;_YAl^T|`_$@sitkNG*4=HD)=c}GOJ
zRn7&!N}IM#+s*V;PI^drLsx5`hU$O*#&!KEPrnEWr1O05sQWhc!sg0pb8i$rf0=gs
z+&f8|St`ms3-;ZQ{v}+o_4S3dKhiHB{dCT6-ty46@^w}9F^24CK2_x;os{<ru7CF_
zy<Y6_?={BL7RZY0DwW$5-QDzc;UA8w9kchJYjg^zobXg?LtT0H#<P32)Cmh#e^a?<
zldW66+(zoe`hyA&OS~=aHXfMn+7!=Pe@ymM#r~)N3}fFv-E+=tf`x9{v}dos3S}oL
z6_u3l@_t>oEtuy@yM=9<*53zWTf*dm|9Jd5`R)6XqW4MXLO=N|`Nq3-zWLYVhu^RL
zV4<WYe>L;eQ`>V-F1&I#={vfu$9Br<m*G_gCAWpvT)DAjda}{F&80=V%AJmsK6-G<
z@zYxE5aH=bLKE(@x2~ORE2+1pBgAZC{QQ|cFW1ORs3e{|mwrv@X_f7!-J5t_4~c(U
zXS8UJ$RYdp6)o3QwcEEzMt=U;E#5m#=k48y>!$4*N9Tm{RUGx;s;%SD`B|s`($BA0
zio?OZ%VdJT?DIOEbcf9Qi>~G06pmDoy*g`GUi$G}T;kRTCLRAOSF*X&{OR?L)wySn
z1uXe25&UWWcdsk|*)^F$B%@v37c{hc9{+H3+taPiF;bf^J$qSd-TdmhWc=<u@~3s>
z6~DAS`Sg8*)stuPS&82pgr9HUTr>CS>%RO62BJ$A{^Z%VIicU-_;p412fp_^E={ud
z%W{jIBW?d8<12-2oi^|PES8_e5%k7bdflD*?8;I-DOZf=F1ipT6Yx~<k=)b$=YG}5
zy#CI-CjF1Aw$vAyk9RD&WM3Ido=sutzqO%Xa?|?#f`=Y!^{{2T&rqHEaBI=TzbEcL
z-s-L%tET?1>-wCD=RQ{ROfK|o+qAC!j&pyXe|hiDn4ZF?``@#)8|of&)lI7Y`~9%b
zpI7N?r{$Vj{{K2hH-GliljW?*E_>vk-ObcA(qi6Ux5uwt@NL^p`xL*;eHP(9ElP1m
z8m*6Qv9cEQ-m*9A{4c(5+A)uhfB$?}*J<%Q*KIb(o*4RF`&4}W?7Y8XlRqAsBg~vJ
zd2(xd!N2*F4eoDdTV8Gvm$SQ|bV&&t&%Q@nSR9T$G`zH?gs;Vr&+k`s<)3c}>;=Cj
z2OXOI`tdVop~FhwJ*V0Cmk3SEZs)V-V(6S6H?gXiJ)pgIT5&hahls^*G;EZA&c1D5
zxJLH5MtsWl!pD*m_9&!vT$q>}ouAHnr~bgt!e^Vlu5%7ulQGxp_L=oN>JE6>EJ`gp
zaB9kjp0B@4Pu%#=U{=N^nlkCY#wIofg*%TQHdQ~hIi}^YY(^i4oyGxfkq4=tkMtkb
zGI{XS;a=ung~%uV@z-w~+)Cc+HEG6Te_nfkm2XS(Ss0A^J{*$yEw?JTXu<mC%H_f=
z?SH+FW&bLUPG&f^%SW5VXlJe5)LoJ;pOe0Q;?<Zg@t$*OWy}1x;uVJ6(n9i&`UNz2
z^e_7zc(Ov(zpm;?(qoO%=}CE)-Wu-{*I8S9=JuZ7SFUBpS|`8n`P(jNbvMhgXT$Bg
z6HjOEnQr`GPQ7m3`sFY0#p&1Iy>{RBMf(5Js{8kC*@Jbh7yrEbBEdqr>NT_7o|0x;
zgPJcz0yDm!Q@vngdq>ak#Lw&ri~pzo^E&fT@knzZ@BBp@>q=|qU9@O@cs+2Qp^mz`
z@hV9^{hAl@b2R(*F8r>yxOm&09doA}U3g}YYvt73`MbL7)yco9I-hw|1DwV7U3|KC
zYLD5P=ubOuyDYS+Y+kJXWq0_>O$DdZHFa9mbTck&J9GYaw&c0KPuWM`EtXRI7v0iY
zwr2DHYc*?ZvyQ2+b${Wst#qxG>PM^Wj+#$g9doC-F1yQe^pRVNmx+0q-(AC3-3B*O
zr}n=T*mz|Bb^|M)N#_lf&RoAVd0)n)FDa54Is6@;5=37udzld;Cf#?_cjF;VrL>Rt
z7j~#=$DLF<VO;aoQJP_ocG%bY>ux_vc@8wyDTU6HbxB=ed*GR&#h=)w108!7$OkYn
z%u8=~w>HT!j%l8*{`h}_wa2w-4FMqwZFsYy+I;$RgKzIO`|@m#7~A_JVUyhD_O07H
z^;5F-(-~RrD;1Bpxb!gde)ruu)A#1{`L_RlC2+6sk}SG+h;`CLPnopucVvzVYRLX;
zy~?}gwDht?@{*p5jxG|Jb4F~UM(g4o3WAcq66_y1uzuOcQ>XsW_DzsKpWKti)<?Np
z{;vOd$8t}8vV*Db4tr}Cx60y$cN9cIj)fnuJfv9t!e8>><GY7r<WEMgxa22mcI9cX
zr@>Np!+-1+4HG6xPrf|I?NZn8*!_XGBjr}_d-t!K#c0p9^Ab@ymmPjee63?TY8L&x
zUuEY(%}n)}iOh_7YnL6K@_AAJU#ooOIZq^ih^N1q>-;M|{^oQ0>&yQiS)=3@|6M`p
zlh1df=Dl;=tM<0H%|2Dgb+f==$Jg)ve+?&Y*eKxdEE~K1sQ{~L;a#7l=kMQE-Tr;r
z3p4wrlHIf7xNp_x@2pNc_VpL9sPX*o2W}h?sL0(|_)YcKOXa$9gV@7zwa2r23m&qx
zZk&DMa<T!F+0n*@yH1P!S#sw8%cJuBzB8WlANR6rwArONRavo>rS992FY_GNp1uE5
z&75V;jmITfCY=wS{ye?fytyx;-Pj<%;eAaH+p1_=%ad8lb=7`pZaGnUfn#%FMzYiU
zhp~H-=7yHXz1pzrrOwmCMPdK$@jZBB^5SWSm{{;b)wgFp9=#)x&B1E_;!u*dg}vBO
zV`~x3n*72)e-6DApOwd=;KO<C<%x&d60ON?KH-zNFLOQn=GJHO@MPf4f<V4k%UORk
zhkR3h@yzPuANBwHT3By9ubb#nyjaV6>4W$n4WB3K;a_(aO>6ft;T4ddvHyGK-TW?(
zD_hPOuI7Ee%{_5`)U}SYFE?LSh;5w2CHOXH#e&E$i+(e{zp0&Oa?ESmxr@o`@4M*O
zYE=h(4127#-Db(O9u_6FifG4rf!WpHHuX<@oH<cNUcl`{{plk4#(lGjlnpQb`m$t(
z;*y^V_fA$DRIg8czp>^71KXl3#hgTeD#d0ug_xE7PnM@<o^k(k@&<QM=*sO~^%({|
z4}x#KHt&%*AN=0&s=S#akNVH(U;WRI-(S8%#<{p~!t19O_8gt|S0GtFb;a@3^8a^y
zlsprA^8v&0N^bXc!FQt9m)C!`j7cfYv((=EE^(dTlI`<W&wDjX{=H3M<DKFa^{J|I
zY<Ip(uI+hHR&s6ggss1=&8lSIGoE04nehDiE%j>Cn7nCx^Zx3mdRa#Jbewf~vF_*X
zo&zC1k~+WNs?Tn}#I&I~(Cqiw*ae)aYWaZyXD`1tJ8z<)ks`yXze4!>{s|lA|KBA1
zRcYf_`TRe}&t5*`s2zNJX4kuw8TNj{Po8o%{xf+wd2a6h?0?@j>sMLUSDnk6&~@DE
z@ye|$?OKi(TQQ&XZGLf2cxuI+cJbq`+kRPc2{Kh@x25wed$4pyR&t5_^NyNhmR!w(
zy`OH?f8X=0y^Kk6|GOK{c4W#piKH2rzML@corZ(E>Y2~W4l>va%HQuXym8|~^4?!x
z7`RzF|Iaiy*YGS-?#id6*l%$?pZ<E*b1?jEU;E|r{Wa;!t(T-2oyjzg)Vwcl)!WZ@
zD(<J>u`LqZR{n)z;s2!U@_x>ER+w_}=$5s{*KZcQ|DJbZo7nnq*}BSmvNo7ca^4p3
z#Y^Mj`vYt)hvjGf%a|GXV(;Aj?B=(lc05a7yDM4o$<<P$u&{TPZ~CS4mtOYTHADW@
z+WkzabF#&zwXUv8SRVapzpInf+R882KYab77d!D&e7|X+{gT($bw1U8>XJIg=WCd<
zrNB}0>cNvMPL{?_)_$+?`1QL(Z}va4R4q3>`p=Vn)`JS&^}ihpkF~vhUMdl|POd}J
z^1sw|_QyVj??P3s+?Xo1{J!hEj!D`^Y8SWuQApqUzx3L_stp$wcd6cX53~No7cb)M
zx8}P+*W&7DAK3J+F6-W&;+bEcA};dfN0G)RjuqYMce*n(_<DFkS$Q@coOy1Wz3tS5
z3upRID_^qO&~R<X`>ooWZcQ(r8zx|@^Yq^h(WJ*`lUjXGZnxPd*IqbRHdMZO&y4*0
z_VwHA=g!t{R*lzmtEhBa_vc67_h%xWdT|r2_McBtKHfZkVatVXBYlmX^KXT{%Kz-Q
zdy)V0_ZbdshrHL9@7N>!ZPllLFB=((Y(1WTe%l*+>DOz|yVqvSIVK?O{Cax$(>FWn
zc-+FSU&_8@E&Ffcbk{2f<6{!$I)3?;_jTUoBb!R9-mFTkx2+4gGW&$6n3v3k_dnM1
z+g;wD!fbe^@R=0P5qZ4@Z@6+o7QDUhepjod<x_T4DaY)m?drQy%M)@O<Em_fUs-RM
z)@gC%5T{}DNr|3vgKOu%Oj=rW-~RIYy1%n*?|!$Q{;vJ;!n+Qs0y~90WtP^K_{l7{
zIKM#oe8#@=dp*K4eddI-6inQ^K6BOoM%Rb=|Lt6o?n(Ym<oJ1X$}bzap2~Bdlf-%-
zXL}qez4m>x&-zcZ+t!x_@B3|H-S#+q{`GjjhxHbEYM;%^|CUaE^5^uW)51Z&{=TtZ
zeM&fVu2r2O_sK6Zo~nn$J~vyqHy&W;5|myhpY`vS=k3Lxwr_FVa<<6b?8DuPJuiK3
zu|GV%?7{2p&vkfa??2YLCFS_p(lA@rE>WStUrGXh?W|5Fgq0}&wfp{~@&7iRmH+lx
zR$dQj&iwiRoVsGF#70i>S$TTt&!1Km<%!sr@N4Yvx6>-vxGhG<@@SWs&ri+yA3gr>
zPOg%7(Rs=)tatwIy}qSk`I}yqDazEwbg0JX`*JLH{BXZi<i!6gofT7jc1WA(ysf>@
zT>p5_6hA9FSKUPAe+f%DmL}Zc3YfV*%z)X3uRJpSeq6-ijqConSlp6X8@5DEx~MK`
zUgCe3sw#=}e+RF%I6hfk@>Q+!sz;J(k>5tSuw(JZ<D=_WJZ4os>;J{iZ1QUB`W{dH
z)kk{gH=VlsY5j898pAgSw_Z7HF@J;DSH(*V9iKH$e0<)1Y=S|Ah3&WVX?c9MtDde2
zPM-6YXX)DStF&Y*J})i%JN1&%S%D2aHU9ssd)@!*?^vIA<kj&#|LPX(G51b4yZCo;
z!=B?e<Wv^e#@-ASIQ)N0=%%~t=0^T2S+?`Poq|MnoQKKRudeU+&nsl?7QXOAV(Yzk
z>c3~cE8P3Oum17(#tn<)<}svaNZt$Js@&@I?v4RJe|N(~*JGu1={MYCt>=6)p3yDX
zn4Pj_p;YK?8Lf{i%uSwekzBBA$+7wNF;<7q<jhI*U;nD(Oqjp(j#KYgCC*iP9{+AK
zbJocd_Mb1bJgM1QdG2hA!sj)@qECxV6~4G|aMyb~MR6Tnt|Y>$bZ-8_ozuj39-k;|
zH(^R$lHuaG+G*EMWbdty&Uf1OVngslLq5xnubq2ro@IZ&Isbd4I@c$@*rN68Eo|*p
zH?N(*GwmzmEQ3ibtKafB3VgA1ig4{IYdP`x@s^mc(xpq|0{m@a|GfIQ_-gT@Iu6bM
z4t<Mf$oX%-K4nhW?(#=pXD4iqUF|wQXXA0UUAJG{eKq$-{do^7<4M`S|FzBg869rL
z$@u)(lgwFt&&p<o`>62GS@<`$Du+|Yr+=k)dfSEsmtQBg=7oDH_|9wJ6U}sEiNLm$
zohJ(vUI$z)SaOH=iAeg^o%bj1`njx^Biw%P?9@A(dhM*!_@=C^|DdcQt+B!XuZ9)#
z-p;q@rBuYLJOsJ&u4iofzjlH2p13a_KeSzzOcid<m6*M_FY5b5izm^1cP_2I%OzUj
zV1I$(bQ9z9s8gCA7Kv<)PaKx?a(sOr^)lm$p}q37yaLY9`Mw<qaYF8A6OU-l>wC3h
zN!qQQ6O#H2f9RbNXWxJEds$oa&J{{`Za5Y?Ke4enq9^|2gRGhRzu40(KNt4g^Os~=
zT_*oc`1`-?`FGA6guL;%sy1uYs$1Vz-(S7;f9ZNJBjMLCuj}sWO^n+6rsvt^+pp7X
z1o(Yq-7fBBi{R4Q<hZeLv6A?WwaLNkwGx)|l2zS)vPIpomi_3sI>gU>r^L6qW0U75
z2hI3$vA*|{@r!2`mrZ4A%?oR!IZ~~T1sv`Wd9?Z-`^>2M&o`WR|MOdC{r7X5wU=2(
zWt{kW(%$~n<k#meA9Ud3nO1!*@9Fj8(`k*L_eIaWe<uIqr_!|4z4xm<98TWvi0|}_
zU+&x&ZnD`%yX&Y*gu#rwgK`P0RGseVzKnfs+F5WjNl#j;k-1rO{_6?*Ezh`go7Oc;
z-JIg1FLyWPWXJy_OdoDsSatH(qok+Ts}9`kS}Sp|hbzNJ)Ar@%J<*4ig3q4qk_<T$
z!)37S_0#6VN949mROiY1eX>O_c<JHRM4wipZyUHDbM5@X^yYBv*}Kv5^P)He|A}yH
zig4|CvZGa3Cpp{T(y`P(N5lkoxSi1b{M-EWS=%|A{(by@J#CH5o?qq87jAC;x<Dwb
zc>1J%A9?N=NtvxTg^FsuH%$0tdM2z_BiL*MH+OwZ#*KwnW@HA4`W>vwKCU=-U3>b{
znHQ(-=Pr|wS@q$Q=W{!z?Bqwccc?GCc-hTAe$S%#@3DVvvt8<py5i#O6PJDr{%X>t
zxO7(9&G%bYW;~7fb~*CVEuHNRJpvJeGO;?mqD*o52O__=J*@dvR4m4$Rcv~0g3MQw
zq~5a(hRieSLrff2=jrpD`)3zY|GRzXA#>hi=D#d?-lk}}+~VCU-+y#L*73tvz9k>c
zz40gF=d6<IYxQ;}ljli1O`EsY;JoMIX-U&x*iF>a*&}mx!q1x-*DsbF@U{?_;jZ{A
zrG8<CUiO~(5100x{5fHZ_#cIe1I-b!@5Aj()&`tBcVXf=rQ1vX-g{U-cm5MA1I-5l
zx13+@xOatPwcpaH_p9f>{B=*k$$r|IZI7293OTE@LA3R*bef=+-_mQ>(`tTQoF{n3
z^6aYf6LU<ujm~jAI=!>j=$uLMgsf*DUu^R+IBT=eYNeHVwEwKfZ|^qOv`suLW2d)&
z<ND*ha>?H$K37*S`x7Dk%GD*+zdu(czI%81v0vXgcFFwYI4d^Aol~!AvQ&*$z@9so
z4;FB{Z?@PlA?E&TxtE&*-Z(DkSYz-h)b3n_#^Y+HBNNYfN7Ra6`*L!Qs8VC&-k+b{
zT?++{XPMbW2A=kxvt`{h3#*C3as?%uEg612oSb&!CPx{=WA*5|*}D7gc5+=d>uq~?
z#el{9N6h9`UmX0tu1>Ff6TED@;jI3HIfnKo**b4o_LnlZ_=a87C=676`dLI&@3T|8
zVcdI({%1#bU;p~*{^VW9t<sxMtBN~i_pGZdIjNYO`)ld(w}MB7G(TK>pXlyu)h>O5
zTWq4iYeW6jcF$gL`LdL(ubZ0~+8(g}cw>}LoL;$+^yV8g@0?pw`fQc4``n)|-en(O
z!m3?<=jqh==+)IXPMaLyTpX4pVXu@{eBh_D>y@dtkNy{=JU(=mMLjQGe4@Kb-hKD2
zdw%ceyRklfU(e=K8-En;uKS)ZE?eH`c4g}SQtRWtc3g=+GE08;b88o)Ylo_AZhaQs
zZ8p<y=f77L%UN8vS>`l)^w+x7WE7q=zwztcr!RNUe2P^%`tI`#sZG}M3mG22n~<ve
z<x$(K8xQ`y{duIBqfGrm|F_zNUZ>eTslpGsa#(Lqkzah~%!w~6#1?mLx^7#`ygcjv
z)_be>oq8Evc5vPA?Zx+cjrZ6HO<B@bb8$_AX}8wdiB|d6vSCjuPh|6GTt9yQr*vOV
z?mz4J!zHhiGB~u(GMy4WduXxevCYo2GsHY*O<j<-I-kYf{nM)o{oqZ94Q9G3sP16b
zynDa4FQc%)cIU|uJJa~bYvtskTsO<yU*l`?dfwTlN)LgvLN1%dgiG|}*LzxAI`nzM
zmDlYp7jzU&vboPrI`nwqlO_5p44*&b=^m>2%^noGQ7)5j!wxYXqt-u1t9DAtpUYC}
zGfR-WGm-t*_O{5y`;#)_O;opU?+LzpvHkdgj9}h#zeDuyrJOC)nihLz$1BVAQyH%n
zZf}aX#iV-D&~j(wwBAV!a?-|uGdbNYCm)z_+dPTe>SRUI8g*mdhnEX#J!UUlHPck$
zE$hL}*58wEYKlx`l3ksuopwn=X@V*9g^~lWJAZh(xwGuw*8h8N3D=XRV|%u~JAQ5R
z>es2tt_Qw+*b{iH^xgA$Z{CUM*X~X1`L;Fwo$WM+%NBdqv|UT7nOj)0>-ycjCttNJ
z+_3)l#=^bZ+(X{!sxP^7@5uLK?(b?Bm5E#KT)A`R?$*0`j6Xg$9sk8=GU;%~Zq~bb
z*VW281=T*ZUvH>BeqB;M#8xVPc4=N?Po&)Ey61=G%gnA@TyA8zldIKnU`_kf1h(+p
z`_g8KRcF^Ms#~`GtIg-O{-bC9@%?=JA?EO*XL*}*_Plztr;@Mm-0?FX3V*J@Y0J$Y
z`js)%^87>9Rpp{TUJ3_qJ?zha>A~E$o7c_WwP(_rYcmf%71U3A-<)$Rw`Q~8j{EJC
zg_CCI{@iQvn&<k>d1c#P%r4ZpmL%dQ!XF}dBh6>eT(NZy|2-dP-28BL$Hrf7`43Fx
z+<$d3PPy;?@YA)uK2Gb{J!|j3xLi?L9rt;j=vk>pdFIUTPI#DqQ{E-Rvvanh{i-*z
zK5vQ-{g7$NzoGur)XIE|JpYc_C-3@vww2rQso!&hDzDq09g-XG7afqht^8G@?atYb
zYJ&&weoRYolbJbBxXmW--;7)p@nku@`WII<r(Mtw3$6L(ejr}*dwHwIYbT}&65sAG
zV%8U1we0%)M>igKTb!*nIk@#t*0O8+x7VBxODfdbyFO3%?*p+D-_u#OlvnmFTK8M^
z_|Nav9va-TYBo;mD+(T}D_!r@$mn^srKXL=Zq~;sdY?UxJM@0oRa%koBw8kA(ld|m
zSL*q`$+)=u&*N2W`P0{0S+jW3>3r|c{R^MhynFppfB*jepN5|NKi=%zYu`5i_2R3G
z72SS5yU~4dQz5Tx*WZ`yYZY$KZ@71%(5J2A=7XDu6D|fSE>5^1pKj93r#`*NA@ik~
zNa{hOxA%`Hv*_u){p7tq*Xo|S#)PS#fBjSG;FsGS*OvR(ut)xP@jh{mxl8M1jSW8c
z@AhRrCAjK>=ITV2-|zcu{-6CWxJCHXwb}ORiALut%`(kA<ZoN6top)#*7(>@iN{ut
zz5o8O-Te99^~jA=@26`oH~)0^=6nx<_b)Ql)FvH0EqFof^v}rsoe!$!94*}&+w-gT
zZ`Bj|bn$K3`j5EYNkz0wFMnKmj$cODrnlnrU8_4rc}r$zvK*HwkMS`wvk%<FFI`i$
zWb&enf0xZ-kAH2;lQ?khdiCqVWB;Cs7(ZRluah5TVca0bsk-3IqNhDyBEI}N;lQq*
zFr$&*nSU1Fu59^`ckEYB*h~I2P?k@7QXabXTg8P_?p5o{_9?X(R;R{Zl?^qUVUgUj
z(cZE7=!Ne3FJ8}|W!K;NBc62n^+on0UzcA!&9{C2HLJB|=O<tL)RPyaeU`~X)?H}E
zvwy<p?ANX@TfHyrZvLF3KPEFwuok>E>A;^|8FE~O?y~a@e#<95d9mf$oA-TTRxMYn
z4;?)(yY;U8Y|gD)o;}QH?O5&kJy+wVzbNnaqYv+V*(GYuzW<{Bo;6>MxBi+Xx2<wN
z^V&Q77ZWetearZ3V$+8IUtid!ZHy0DGGEX|SMO}X`MNv>Ztwd4;!`+X@BX>f(3b9K
zrTj)NQGWk4=km(pDK$4XJWr1KxAI;Z%dV{@vM;YLSY4@B@#$yXg`I)-%je&9dUI^U
z#%pXV9oL^^p1t9bWJ3U3UoA^EkDK9(zcx8>K0U_5{(LvywY>e;;Gt9JKVzc*`itM#
z*&bxRkC?Ko<IfYtEptC}NGDwS{c>erK>yQS8qKjk4>}0l+wl7L8IPk`Z~fN2Q2D9#
zdd|JC7Sb#K#s)>~Z|Ci;dcdVGKjR=<w9jEH_A6T@Hl*?>oYvI68t*vmWRO$Um5KH1
z|EGlOKGQIW`M&G@boZ09-*p7+{XL=gi^0T`(@K7F#6I`9QS^spc?rX%r;Zibd(L0`
z`u(%hfpu%cqPAxLn)+ILh3+khmsR^q&FbgxsX2DJ$=b~}=l!aXy1jRM3T|Zy6`Vd?
z^X|c+pvuW7K5sd6cxmyreJQbg#phN$|0$PUBwO{o|9JCZtq)rYAE$SoEAe1I_#!a-
zqolCu+bi8FD;`+!zmc?fx!C%qyWaQejZcNNS;~?&N)((;)jq!Ab@D;m*hz=PCe7AL
z7oRG0Jm$s@=Im2-&)7fl7tM5D_Al;|>-Vd#zglliIpbOXV7cLBzj<outEa_p4|Pj3
zf7%diwKC)9lUoh40?}@_qWdL&{*v=6>hQB&?0HVq?9Y})o6<FfW!q1$w|F-3*b1eP
zcfuQbb^kwBF?IXFcWCYRx4jov7F;mtj`N#e=JNAh(B8b3xyMf2lJwlM#N%D$hC8iI
z-h1Q^pK^b&@9OoBy#<BWOZiuX&#qcmx8q&YW4-nu@g47sw78c)_l&OMIkfwY6kp<j
z9-kRMU+iDD#`w^_eT-8}-aq_!FSmJOpYh+rmu+AD&10*tRN~)sCFA+ZIx)`KXB&8D
zFR<3A_+z8Iau3^?GmIW)e?Ar6t(L2QyJ=~fgyG+r>S<!HVy~T$Q;5;Fc_h3~){^h$
zS_|*=gPWMIYsxOnGCE@UZ&$GW<b`Kv7%fi{SJ@^i`<?UTyt7Z19<2LZ{xa84>cd2(
z+UH*q{w`zquE=MR$-~6QVwc{2iTUKM$#-_W{~D357q(wr_g%yOS$#Y<zq&Ic@)sSS
zQ7<-q4gYsHzRMR3X2+Q=^_X}5pW@x<N7Hmu&A<H;>o-gK;aw;2-r;|s=<yuKt5V^W
z{J+EU9`Y=kFg@}x&-4owCL3Im>=?b<*lgC=@t+P~HrrUC$ktH&);`leshLTV8Am=!
z2|4cB`AX1{qs!={<)kCc1<Mb6a9rG-C}=x--3t$y3tL(QuWFw)2@YUR+2GFbex_x}
z&dT*~Cz`eFo&7$!$4n;gl>DvCwdYN%({q*E{v>o|tT=osjpNAsO?oemE&Qp!?TT?Q
zqvEpz9}j+?IAdb=wC;?1O=mX=Y}Q%5;r<G?caf*38E#T%oXNs!ceLWTZlB8gfW#^D
zDobU4?tXFmU-+%NJ+EfnYMF0vf4RA>gZ&&00qL1(A6<3{9+Wff6PlRuV#(#Gw^N?Y
ze<LhX62v`O=b_wA+pOjt4nd}sE00Hf4!2gE^uhUg@YxIBZKqZ`Pd++tg5K&`(^R#K
zgMR-t*mZ5nfyqh<g<ktE{&4@aXG692=SwzM@)RSYww>ked9iTHviRFSTs`Ogza<%3
z=A`&x{kwYWXl7sKDb8(oY@VE)AyPW8!^HpL;a=ek7q8=4=KS@GuHMqmo%1gzJ?Ger
zg_)lYwDUd<Z@FhIC^VJl{0C#5#mlds+I#l?w^Lu={H?p>Y3&^TtJuhL=LVl6JvzT`
zNao3_ms@{L)_B9SaPy?*zCQW6Ru2<+GXBI!xGXG@jhe6~jBkGE^$W6Z-4-M&J<qmo
zk^7ph-Lu)9U0L$;n>CK_W%aBsCkuW0y+B~r$9wgG-x)66`TF+Hgd>W*-M%KLv(qje
zKKLPOU*!8e8MenZq<`Oj_xBO&Vk4EqpU;%;PmX<l{!7Gxqb9mm+h1IHH23wJ#ub{C
zwtRoBqz{TlU)$9x#rywL;Xb)^@vHelst?wliP?IcMfrKV>sPDaDzWcpUOs#Buh;dN
z>#S$}v<h)~<g!Wl&aM6OHj5kW{ePRQZ~H5sU$Xrii#L<Uvp?U<?eaSfg?LM69)A7o
zSC-_-8Y}%{m-r-ay?@(#=C29+oUK(i{+^F>uP~p=sqr{vRg;X(!{aB{eN8Ff_3OR!
zG$&Qf^mye^{~tx0colE1otLBKVJ7(E)ynhpL_{L)yn3nEEz+=cqN~_@?IVjOiC=xM
zZZhx8k`7t*8yiC<KE|&8u*Q8#?4yJKX8Be`Rhh;a-njCK_x%rTEBUV`x8EdP{dzGa
z)#dBO-zh(sZQ~3~Uj24w@=a|?=(->CGUMCSh0lI9tcX|0sJry1yY^F};Ht~QjaI)`
zo)Jv?xg|!v)xSycaQUq7Us$(XVBh#}qwVQt)oG{S&Q;WKihD2h?`+AMJ=;r@FGX8!
zZp+D^x8{-8E(@N;Rr-F%ck%nj*Y+%(clP#~_*XCUlEofuY1#49#lAN0?z7#V62bK)
z6Uq(x{M4=IzInHVJLuc)VxPF>#(#4{pZw%B=51M?qi~n$s~yMj_q^Nx^OoOh^eEU-
zV{U)0(OUNZL9>Yu4^KU;(^u91*pbz5i(=B2zYEWooDJ#^biDR?caFy;G3h?z^=6-4
zTB4Gh+V~GAJZk%%U%n}9=ey*|&ltO9;x44*>^Pa?_v~0%M9$X0nv{ccC#To$y7N3|
zTlb9uNwqY`?38)QS!vcw*rnyaHl0X3KF#F*k?g+DC;raep3Z6Gv+DM_r;Y9pnEy#^
zi2HA{drz#&2I=pgm+8tp6JOe_e&edOzeX(6XWho1%fvSFYrNJ~WnmFoG{I<XfCVp$
z>#-NF>(^;_+<cb9?{4iCG-KoG>!xQfNG@7&<PgK!`{y|wk5rhLM%cUbeVHORNwxh0
z_YD_y-uu7bSxs}fJS%aP<y-dn?{5?xRjMBgKV$wDS%1eQ{_f)L$N%Qev6`zoeZPcZ
z>Q9qP&jWe#YBraMMTTgUWm`R$e)=vkc!#=5=>0dWKI!_09#2kf+&%A-m54^vnRkt+
zKOOu&f8&bArSnfeZvXzzn=Q@3Vn;UPdTTpHhdY)J3${I3V9m+1TxwSKN_%dfUDG~2
zZe_I&QH*|MmpMsg)6zd*8)`i5L?26E?5(=W`>u?!chcNr5e5;@H5S|MTpKZ?<WEEH
zckW1)=+2LfT@0tD+BBqgHn1F?{ATW#HBTn>#7x?6w)v_D+s<=F1(!6_CdRTYFnZ&)
zesy)^o85`J%TzCJx8}^ttE^XVO^z%7@WRRIgwkE}f{R_r^WN>;&Ck>(z%Gz(a4AUP
z#72!?#cfspKb>rQlqgx?trB!6=k@-nA0~YH)M=NoQS0qH_p+M0WQ)I6?}cuspR4|N
zZ-dTHeYXeGY$m>*nm_gQN~dp1hxXptt@c8p`1Zehb?IDHpOasfM!zp#{<|Xo^Mgl+
zY?Hs({!wat^=h*9o`#c`;=7JUub+NYPoC$e@wr`&yA77`xmmojEv$M`$Q8Kl(3QOj
z-!^7^N&Hdpl)cbWRm@lUQnSnYNAD*T9k1E*uBs(9HO_WMm!xv~)U>MeiW{C?%U#rR
zdO^s0o5kTgwLA)QPZ%xePjpJ;-r;}nOP82`RpojeucN#E+y8!fE2ZdJ&DsB7Ua7rb
zaYsHl+^HaG*0q~&ZvL?A-l*U9HE~A1$kZ?5IqO7^{X2YQ!iVXqe^L$W6rEC}uQLnY
zKT>h4KknY~RKuX~)~%d%k?Z0)CcB+>-4Z7zylH8qO6aOR*Zxfq-^BVoHM7c6AUm?<
z#I*kWXSK)rA1X$>#fkMzPwmbre5l_V_V3&K|98y`cGU{+auB#4(fiVJWA*iYktwrs
zBA>C}blPxIEQN<P@{gPUpZ5_L&LvKYSfBlLmQSV2{4UoELTZz9G*5JN{i?l_pZxZB
zRc+&r6o;*~XP=vRzR*kjs@He8Lw@sd(SX-Gf4%B8&a!#4v8PB#Ypu(j5{>Wee*|53
z$L2Xd3OcB1Uq6HO-Kp9e2`jS#|G55o$e?&6Qc2`P#>yM)LRT(KwtN=3D967*L~rKz
zIcttvsof2E;5^BUd!x+m$LkV;H%_iqVGw0m!0vMKOnLQG!EmcT8#=ZZdw*WN$1rVY
zy-~E5`HSAiyOJ{uUzXmJ{c$2zM)Tq2q>C?O7CY8VUFf+l=WF`0C@bMz=i<Tx!{zGU
zA4r?;X7A*l{HLqc=V)ZBS@9P4*K)tDE`Q4W`?0!SN7>ROt4TBS^YJ%*HFMUTKA!&9
z!tVUvEBA`G{4dov{OA}cd1n9I%9lKrr=(UseHUA|^!mNqzt>*>taAAMy*(P0dG$LQ
z|GfIN`nAI3o6iFUoRs2HAM+b5I;i3H$y+CTS!L4spE57q7V@6SPx0k<TcN^PUK{fL
zJp0PUPk7u+BrZxA8icHP*cZ3v_=4ukzkfR&pH*VQE^_+qSO4qTS^?FJMYq>pFi*Er
zI6Nh~ue1Jh&WcJ8){PqW%PQwFo1VWHz;yWY0{(NS7hOMHz3@Rr^wT$|T+fGDAJ(2`
zlvF)2czwKY-u0TjzHjH8^RTG7#JTTH{aqvbyjlD6oMd*E&SG~wb3HyUFRXKt=8LDP
zI)drtX?C&vT18Kf?|FXz^Eu!2<*Dh)&DH@<Cp0;3Gv<WlSVWz(EGY;q2u-y%ns%R=
ze|PMxulFvPJNksz>ozV)yqmxl(X`Xtc<t%z2>;rrPqs||vSvf6t;V<2)=U51dSSjR
zyQr2Y?@OWXyoS>{r(W*!F!E10IOS~er|&bT%#DrVpSV2Y6z7fIg|&-csm-W(8uwzh
z<#E5lx~(TK8%)+NUm}(NZ(?<m;))14t^0pg^#AC$p3%BEmGj2N>tS^qvMc{v*;}qM
zTxOBI?fsJZIx2G~w<(0`7>njSQhl=Q(G$ME2}f5}6<;eZTO!>uL;U1zkEo~5{+F9a
z_vr2|JA9v6V-st~ox9v+JDK9YzX`kM<s|ITc#^eukF%=7&B-ov9FFy{tjporu{zJ%
zVBy@<xy#R`E!r@dWl6i@ouii{<kj!1EZ*mNk@MK*h6n!tnXErN-`}+%QgW>V^X10V
zOwSrhSG@hV(ylG$`0Lh7v%6YXv2S5Hytlb1uIc>O#jMw)w0%qkxfb6l?A-dN>Wuoi
zt6NV0jb7bvsu9&(KWD<Tt5dHfF28a(>AJ+K6ImO7YCc<aN9~vYjT0AqS2#`S$qt*e
z_DSN#wkcm;|6R8K-IY#t$Bu3FdGCH+_gZ{K;^CX~TdPB@i!Yq#TI6EUxpa5^4<75q
zAuSTEyN}sspDC=Wedp7x{lC<GfyCOHsqv2)&aZ0gOKOkYdTpY?T=7>Dst5RmBig@f
zN~g;Pt``crI<r?$EmkHc;;>vsp0RyY!kxz(S02B4WyjJtYgZNYN$XB;a$O#uzU*15
zQ(<MmYtzg-`882frrrM<nO3#M_I2iMu16_N={qwoZJx@1>m)PRoAdWS7fL-das9Ms
z{fZg3*B@K{vOoXi%)_U%wi`9s?%BZ<%~<>YyzS>43r@4&;?J^e&1`<1{;0yX<)zi(
z_?y)^uJO0cvkKq7w%VO+c<AZkJrdkyZ)?-*OE|sf6&EFL`!*%cF8gO=R>RkxKQmcm
zmONXy=SpFBaa-$^tJ5An(vm*8qr9!hO`|(?-<|iXtC`b<dmq(4N%(epu9=>OQecCP
zltU-GZSu=G4IVYm^3y&zPuu!h+G5@3eVW!&>|)FFIjVe@iN3th@?_5@k+Pl=G4`d0
z8Wp>`PfllD_iA=COSp8Vp}%rpNX4(IJim&>tG`Y2$zS%_F6X51<Qbl4_Nxg?UFK}<
z{?IAD*kQuT_$@6TW($g~(v69gtr84Ba8L83Sn_nuU(4SuoM5ve=i9puCfT+Z-fmkJ
z1HQ1kIXYVJvJb1Ba^{kpp!wVDuTSsTzwO_o8!S<)H*XWln00m54QU-sMJxV-P4@Ra
zCuB<edH<J<q4UX<XKOC+Uiq&#<I4}fPoBGlEjBF-v)iaU^Fd+p{i<aRZ>9!#O<JvJ
z`)tRx-}g?*ES#jmGAHdJ+mDqyCI)VMZQ8FpS2OR+j>zD(KNSxd^S|9QUH#G1cD`+!
z{oZagJNr4_g0=VDm&=k$UP=Ya?OE6a#A~CM7tifWD2h^hxxnu2#S*KFFCF^JfB#Lr
z$Kv*GMe3=vsf879&uY1ST7ArX!3X)z=3Z~wQtlhuRE6xH%xiegWu@;g^J6pr%>S=3
z^O3~G<5T{xuAiTHJ+|~gp6NxSBI!h#1JCbYK6h<P&1tK1Pb3ZMvaOBSjY>ClO_GVZ
zwMLR@)(W|p_H654^+LI-oXeO_Hl92d&~=8hO!?bE8|jR5#-UH<yL{TynP974YQ+7z
z$|lHHYSniK+ma%~Tcu*(HlMO(ESKz#PW8M{=G(K7M{4rMJ%6XQUwUylNg{h+8DH1W
z_#8(2a>j^_7F%^3?e5IE{<O4c&w;0Jt7iU*d3uJ4=k|+^m66AOcT4y3esKtz@I+?c
zk~Fn@XZG+)tbP}3=Oa6*<&^;6v@}-L8jYE~8+q2RD86!I(TR6A_xxboJcDQRj~Sjt
zQZ{`e(&jr?XWm)z=+5UmzE_Rns?vK8m-SwobB5!%LS$I%=|Z253qn(wy1n}=8DnM`
z@9YvO{w@CY(N>1^$$OZm{j0iczm4g2?Sa&D$8F9Q{`cK^{$<P5=gfy6ZQ}J{-KjQj
z;&X4GTQh?H{;{m_`t#tA+o$c{IBg5K&er6=+*FglYrow>^?8%Hnpey@I_vMI%}1E#
z-?9(=f23|r$+k-uz0@yeH$P2#e(tzU@6XpgA#TAhH4^gc^D@m$c++NPoM!E2-J<aK
z#E$cBC+lQ?-i*`#^V&$6BRyi<TDH6I#F$t8ePPV5z3Sk($6Ismo}bgS+hB9Y+(g#f
zAG9T^qx<JFFu%M0cs29XbkW!8xkgT@)mG<YPS5+O{eP;~-fd;+M{a&qsOH)8=WnR<
z#%FrFMN>Q17aK>V&0!1va@u9Bv2(4>vE%-CZ&&?IJ^5?qLg|my-)3*;eg8yzuJFS5
zm-osDoT_veo*N{2<*v72M6_JKRk8W2_q^t}Uhj^-+;vIxqW@d@>krS(xDw|5d}(0u
z7rAp5e?vU3^?y3=aqj0$W;VNLtCwro-{k(sZ*cj$w{liTo$s<)HH_Bgt@kI!ub&Zi
ztB1qt^~;|rx6=NbR$o?GVx?Pr>8&S!*p4)h?J+N}e?Ijtl=-JLr{+P0?{XW|b~g0{
zPOsd0d`()}g+;|Hl5a+AIAZ^*IO;pwrM76th5n3FO!?W3joui{N-276@be4rk?c6r
z{QoRP?%VHtO#ZG|Uy_|zcuPk!=IwXp^H={%Pdmph@-{tY%|q4X6yr%5bwT&s9ohNL
zRrMVFz1-%Un$X+#mnU3OtiCeO<Jgu<Uv0TFmMON_m`MbAyBmBHym;uJ3v2S?RyN6p
zN_Vv)%A^*omQZz$TKYkfb;{Iz68xHK7g$f{ZgBHHH7%Lxlva;q<IC6>w*rgbp}h7d
zvc9>wF5WC7z3|h|+BpG=ay;`t&s)30)pPx~zmXIF#B(_rsoEG_R9)x7^m$jU(MgSt
zR$-R_P6c_+m@{7-URS+ouG)Kh{gL-qlQ=sQul&kTaQRuvwIZY{^mpy-ywXX9vhU09
zyr1>`?bk0KgjFB)MxOp4V-j`Ua#JDGYE7n7fgSBXS&EnJe*5gj5vzIX&B>P}ijr6K
z3YrElVd<*<Z8+%|$E~8xik4TS|Guz)KK)!t{0shr`cu5U`gLB#w_g?YySl-g?Nsy%
z6@K;Ki*LEw&u&wBAvDo1BE~gs$tqj#4~E@$tydX;IoGk<ea>fPJK^AW9`krKgc@@K
zkKJCkboO!M2L~hA?#k3Xe$d-&&bNP~?TknK9j9dsn(zMG^{PN@svx)W%bCk&{@<q`
zA8dCbva$ctUC;98Gp~A1|6l)RZsaVL-Mwq>C+q$cV0ivF|Bc#P^S0`+sY}B6UoGDE
zZ?4t39u6rB$%6+SC!8!x43RzV!IysadtKnn$9`Q~&T+M!*rB7pP4O%Dk0k5fgB?}H
zm5EXtDn7gxoStvWeNJBY^tl;-J&$cXZ}0vjfw8;c(5?IDbTwPvSy)KDsNKrcedCen
zp4y5W-+$*OAKqSEQ8Q~(<q5+^C;Mw#o&UNm@G5)0$?&K?GuPuUXJu1wd~OY@*&Vrz
z+jRZ5{>OYDZDoUWjvelIZs%L5RF^!v<w&pN8#kHc=7)@*o&|iov!-BA#LBhle=nNJ
zpEPs&Q82slQ_SCzkDXiZZCqb`GifK&r|)rxZXZlms5!&jH~svUrkZ5QL%+T+6Zxe6
zyH09W+TV}Li=QX>{NH0V<Jn6mBX)y+$par#ys9&I+U?Xf=sSF<`?M67y7Rn#$LWu@
zGnLQocv==|{(`~EzUijx_VXQ`v$n<eR{w10pImyT<HIilNxj;dy*KhcEq{M}a*y47
zneX=7YE_F}=Gv>szAcQEGTr}XL+!u7X(?;8e}4P-=kB9llY_PtUH!ahX2c}_uaC}N
zy;gVH|FymM_t*7nocDH{oL@Vw@Mn1Dl<k36idPohiuP3gKDTG{QaeeD<+X})g@a;c
zw~7A`$vMC{vqIcnlKbuC#7!FCI4{NbU65aTfBmb+GNOEQ@-Mo_@Rq&v-g*6i!sh0l
z-m31k(>=F2etCJ@-r~eB{+y|M^LP66yA*243HAR|zOJ+4IBWHNlM_d-)Zei>u_I_k
z_|Nn|ZX9i=_HfG0*ZaS-W!|nC>$-2=`F6SXM!R-WZ1dNt)nb>Q+ub+5XJ!ASV$#j0
zrru9X7y0Q&?X~#3Sy?3B;#*WoPHgA;#EAhZGq)zb6}H&^K`A%JbXQ*X<bK|RTNgI$
zeOi>h`_=TV&m_b8_1pGn91+-MBjmR}tK0psvdX@5scU(x-_QR$V`9-R{<#-FE-}k*
zIUD~>D01JqROOdC^Rfl(Y+t-Ln$A3D+xrV2D%TqS%n7aWdpxCVroGH+x2KoC+iiYh
z`}o5b3!A;Bg+AYwoo(H6>vKTJnaeu@*X}S&j+mj{Y<R6i_ju9firs%LN`EX(o^#f$
z?dHp+r)__yXTDtX+&A1{&Mx0i(~kby>>0QG@s;_)zdp}*3$}4TwrlU4W6mjaKJ(Zg
z7v9EyXP%T<uIKhWby_`hx+fNI?E8|rw0HGW<?8?8ub(I9+Sh%&^z-k=*GF^8E6#>X
z*iN6`RA2G`h|`KXt*@K2?;MI;I76qU?wbGHaNZE(8L6p>61ME!!JZ~vcQlOF8Soq5
zzN)3Ow(gev1}!a~y?JeJvRhZ@MbyhT6<iShSo-Hg&F{$0D~i_1+dr2##{@U_W@U)%
z4BX|{DUnvFnYnT9nNlP1eV^Z6&ib3qR<*xDeD$v_B|_5g?;Gv?FTLb!)1qG~Po4_q
z)tjHnKM}aT>(7*mgulio?kEH&NBncycOm@n<}V*@54?7_GH(uas|t%1zy9t1<RD+K
zhl{<ZGJbrwV9VVJPg=I7edYA{?Q-Do;!nHlHe|$wzESo&zUR&K3X$?S)s%*18LzXN
zMeU1robkD)bv@Ny+U3}__e*|Xxu8`3I_ppA>PJ8Kt*X;C?y5@qusih163dA7S2ow~
z-m|~jPhuYPdvoV`na!=`J7*kRQDGLd{?F#h!ZZ7)`771kQJ7)){^f^dT+a**ilxsz
z%BhRV3k*B9{%PV#JFdgeVwYB}EL?N(P_CR_c1im=&pTSiF^xa>O)a*JKK(fQ&px|^
zvl}+g`WWr3<z#ibveaC<{<llL<+s4{<1FW1?b<bQvkS9W^h}+MRhzfiU#?9|Kj^vh
z%aJ7auh)Lhmb&^f?ET02yVsTMm{-2x<g>I*&z}Xi%J05)gwf$%Ms)Yl%L+3#RP`uL
z-da0%wYAyjxUVT4Uh59O+z_zlvixlGTN#tQ?rjmVR{OQD;`5!!JG!qt`d)SLVMWG;
z5BX<q{ZA;bv$%RneP*?d82j-{)1IXW?7FphtM*KbG>gkSeb%lkx%2jpUSjZH`_$t%
z_xrf1n0!tyK4tS=e05RKzr76>G4^U+qTf$eKI;uljw-U#yV>;jirA9ZA?x&1e@yuh
zH2ci-`kga&PxrqQH~0SQ3?8Lpu9a!0zP`SbUGI9x>!U^K((EaxcQy%S*h}pXK6!Vk
z&*~X<9LG#;U+$KRWDGffu)p?+{GH3?uhIkTo2?{6P8R5Y<!$AeBvL(L=fi2|KbHOx
z{=4UpF@NZ~F4^TQ>2v)T_h(K$V4ZfQRo~^0@Z`4L0-HbEZ_fR1t~pchi}$r9S!UP0
zzx0>cJ)d~)-=j6^tvht8-miU8dU)q+hH_c+!&fVNOxK5{)h0&2d-qB=UjF+3<=Sx{
zS03<O>Z_Gmcj}&fxBI$9dh?c6uUPixb&zt@H?`)9O-H`!XFXjzeU@eP`k3tL7WM0+
zr-_Fu*BI@3pR=~~>xwA*Kff1uT?*Kr{ZN+gvwM)#&T`djtApZvI&=5Vl{+e$`Om{u
zxw0<E=;JDN;r#NsSzE(=A8lX$vG66&{s(_n_C@+s>{#z-c;eq%ldzv_Uw!a=68iG|
z3Ava*O_9!jSDt(}%jescKYmBsuD`TA!1uYf^mb8RPT%?a+qXJgoi6puUPt4fo-WJc
z;JCT-JP!Zccj@Z=T=@xa)iN0VEP1y2-k!C~C$TKvy*GDivHZ8=D?N9cS(<*_8(Dg?
zTx)%Wg}GtG3h~(;d|YQXGD-yA*uIoypQUSY>+g?JuY8KxeO6p<diLSpSu3%g=SA5v
zKA}BZv;PT1Xs+!mTe`Ps>ilCtS3?a}xLo|OYudh@anG(lZO!?!rpxiJp|EA)3cY9g
zIqL7v1m5y`DYbO>-1+tOEaqN^TmHN;UcS=cbbjlPwekP=%GbLb5c*~nI&I#%gq3qG
zH1FKZ+5YHk)#+)gGrr01{FPY~_}OZYz1@_@TMqQDc)Q3sdcOVd-|n+FDa%|x|8m!?
z`8;1f{k*DPwc-1@`T6UrcDDrV5`R2@YCZStlFJ5j=X}?mJlAL6jc|MYx_r6UVLM+k
z)a>a0KK0VS+?eG&ecMy+raB0=b?$z2*l)hPwtaZi>uW0eToVFp&K=#iv+jcAcWaT#
z)3-X$u75n=|G?^Z^%ZV69^Y2^eR=1((~_Uhbf5q6Zr{(mdimOe6W8*rcg^`%{Jp;D
z+{A>?*RemJKAdwoLcLgie^dD7^Q%kMjwZZpGTbip+V|-CAH4$IN^7iZuibVlpXycT
z+!h+U^Y~}U>?N*h(_d~m)^8F~ka&D=vSl-?daf7$^TvIb4wxTY#+zb4FX&#o@xJWm
zwd;Qw*<Gq=e^K6f^lZQ^{@$8*joXAB?iZer7T{0YHX}(&K<?w)(hpu6Z-_3O{n&QD
zU7RYP`uVMlg)ij8ziwWXDYy5qH%of#{ok=61)fW<Z>r?=e|Sx!tpCdT#SeI1Jq*5E
z7T0{`@Vq@{R^sZ{B(J<Z5&yt5tLNReTmM`+FZW$#t}2Lky21T4ea|LC`LiK=HFDy%
zR$jfZB>0}()zkcZh5jDDR9Bm?xE#kZ^;O38?7lCizBlVRF6t=k-_x~c&E-9lE^TRG
z{y0A_W>&sPMg3ok>Z6O+8t(ahb?fO>7iCu~uFW-Fqkh<G`PR=Bb3CWL`K+UG+x^>d
z_U5*4f2_ivgmeXHI@wyDTVMP-|K9U2mG_kAykxnx&g$~hKX+n(R!*7x|9teaPeEHH
zGQR6yYu-QSux0UduZ<h|v+e3?jJI5hS%3G*;qz{PO~iMfIc*=FS#fV~%+;7U1}Rae
zcL^nniwqWjvtISO=w~sb&gD67XSW6$?w_(c`1M)c=yjHQ@44;tTUYGgW;fIF&fZ7!
z`wZ5r%#fJ3Rbj6ayK;=~!JDW3u9mENcC_p4v;Oz@)7{oAJtw&T*XHM&I<n$y*gv<-
zHZ9Dz?+&<c|6u;f!!O-#@A_4CaDw@+hPWd?dnV0_2$((hl+={cfMXNo_sG0FaqIP8
zZ_X8O9=feu*K1Y3?e2;^lW(E!`vsqFvrA+<G`GX+EZ3^ov?<5Wl-`*UTW8sCdEfrE
zu3r5q`)W_?<RGsTJ^X!F1MI(x-Y8A1yzSZ5zU*AL_tMI*qP@bCxn|qXE{O^+vHG)R
zW_j)H{VS&h&o%uoDfVL1WT|=fk+rkEKYJBa#`|V(IOp<sNAhbQqf2(oe(cwNotpO8
znolzR-c#w1nva&x7T8-;7a{I1t{U}txzL~J|9^kGKln7S?eXH+qR&M!R|2+dmwxtE
zO!Ds|-R!;dXUF=ixztv&@7u&}i!*2PtMYE-Z`*%%t)cy)bG}F4w5jZS`}5JU4c|+y
z&G}ksYFWI@rg+(x5_MbSZ`D7QHBP;Mk}|9K+Jw8si@vxWXN=ofY50GmyF<Tw&d%%W
zAHR%h`t<ktI)%>4*QZ|2I#cAGddcpNcIlic<^Gv9KOTfQ#r@f*JM;H>Ham8g$wis7
zie-d+&2KKTJQsKFQ3+4C&GV;UOnd*Bg(pwAuJc@C)8AJUPN<2@`X;gUOM2?@h;B3M
z!#m=Xrzb0RGZ$Z*WLF?7@0sQ1`;?hI;(46op2q=ZX50QcC0AxCEteJH+I!PxzvVQ(
z>{{9Rr!=O%R=xM}M*X{Lm8b8E!g@NjMSIelwr@|WUHdq5M{DPva_{Bhig%w+YI}a?
zj>o#4(!9a9-M8e)Cz&2RxciCBQX?a~&=Rw0H={Iiw#$THD6LCVpL6u98MAKib$2dl
z!H_@C&wbb&82GW_YL3_3?Z#O%K0JE)bIVqn=DBJ+u5jlU?m3Y2b+XCj9jA{!+}U#E
zd$Ot5xmJODD*58qO-^UZ^n4GWd+Fqq2S=)QZ#^8m;YHnd*U6<_*Ya(b$6LNp4}TUM
zbN7KoM`hyOsF|Xo`UMttpLSkV`^D>i^agj!{x!z~?(RIhI($*T)#hdKXJ-HUbusZ@
zwcGUn3psK`D))u&`7$rgYtD7s?QNe$qaPRazmj{Jr04eb=<3ek=)beF<oOG>R;};M
ze1HG!?-kqQo}A`AzEr$@_v17DUcqt4*2W+64}X;WFGD5LbXobdx@~v=to%Qx-Fy3=
z4)47_!sXZG1FoD%<u<#${%YCovrS58KRO@!)0$Tt`?u!qz3(^P*{{2CS~u*`tlKwN
zyt@)m8Q*+vSJAno_gb%&uE!PV8J$>i{G^_`Y2It?*AKT>hJ7-7S^0lWqMFT7;kn7v
z%b!`MtJFqwPr1A~fA7w|+sE$z_Ua7(kgh1QT{7^a%ZoR;JLg3j+=)M2I!%x1-<#($
zho!dt@Yt>7Bdw6Oab^1VGbj7f{(rIi_ULr&vJ#P;-B)A2%#Y+a&y!@ehgWlr;qLGT
zyXcQMeSaNQS@PR0_genl%PW2}HtfmL&cFNLfA4hFwC_?Is|#=BI&b<s^Xx)(ixtzF
zzuzwQZOOB{6|w%$&rgxJ@0Kp}*<NmcHT&eduglWCrbY%mjeGrm!?AvyH~&rU?ub+i
zDT@4?w?FnSd-S#2Q#Ncq`?lHP@ar=VOViWm1@67}<=n*OqEj_pzQ#80pR#>YwpjZ9
zd+*knE@{v8x~dVnSmmsg&5bjg-8uU9EiKPkHNE+;_R09ZD9^J0f%?5M+kgJtwlm9m
z=8MFgPnG+<W6pcO`BQWLf`6X%&bL=z+}D1*twrjLkPYLCIBwqk7S^3X_9{;6BE@*m
zPdLAD?!Qv|<nFz$U!B^F3o;M47x_u@wq1$8tL#@~yP~PkFe_kJp#6Ob2RZlOy?st!
zS9|vaGo3m=<<Fkmhg#~@H`rtq|2Sc%bnfkbyQ5`KYmU8dz3}}>&A&G*)<oYanY`$X
z_VdYi{L)L7Zgvbk@BF#+W7FF0FU`FVo5h)x*uUSEwg2ATHQ(E1-SgMqul{si^wFY2
z*UrB?JA3|Z*SI~2l5yw%tSftx``kS>MkMLZ>NiVes=2h%zS?hFQ)CyTJUR5~qPA&2
zxA~pEc;Lp{+bsW0oc650WV|j+EaQ2>H|zYFI)3kSr%qFQV4bJ?>0e;gv4^jm%PRle
z4lSOr>X2gkuhgEo&z~CZxA6O1^0r`iP?bKn<otE3xAEuK?_WMOD`Mf1t4%#SpTGa5
zxt-<c$)b|~4UCM@+tc?Sj^ru-X0dmU=(F>04;R1mpH=r#Ce_Nu^mb+QVXM@N$q)B5
z?3~NPJ3Vsy67$^$Jgevbo)aM7Gv#ei<ni$EOJ3WQzdT9*JLy<p=ktA`tK1L2IU61O
zYb)1wt|h$6QolLBZ)a_mX8kl>JZK(QSe(I??9>k*x1ITWFWu76PBrP~gQGv}KTQ2-
zzWe)r1*ZIiWfwEIdhu@hWykK<)@v+v_}5B{TT6mAPo0%}CFh4zQeW2o`t7Ix?%1%-
z{Zjg@hvprRD@6CsNq+NR>g<(s*7Doqw8M|}G5`O-=C^u2@4~dlSKigmaVv~2UwJ=%
zdids-<+c{Ta>7sAi{8+vik?5;>S#LqrOT(}{`Jj%U~TzcZSvoeZ~G5UE?)7xpZDhN
z(2V^;Z>KDgSCb37_Bz7-JLBrAm^%>{uim!4f6y)@UVLG>?&rRV!Tj6rY+Cc-$^t{v
zjJn(L=L`QnOg;VKR)RskMVQB%7rm>c;&?=^N@|}`o>jiQ;KB5>-Ag`}zT*)r>6!m?
z_3K&gCg)$a$0zt~PMjw)J3e#nZ=IX5hh_hK5PBl>)beuU2dz2gOLA^D&JGXV7&tx7
zo5$@}DF5^RsSh{!AAQXGGySn~xV#z1(dKh}>$+xNx%@`_?X>0A?wP(doymbhXR@R=
z&-lD3t74*I#0v9!LRt52)&&J^zV>*#t8niAgF7|fCnqXR-<eXgll}OPd)vZgrrn>$
zkRJD-dNbQO<~Jd|vjr@?k6Wcy&Cxn8dF0D)UFoVdFAqO{+pWCpo}2D*@r~C~*{+|~
zEHHb2_K}Q5|Ngz7cWv9W&}(sZ^TNvd7ZvO0Y8@*&zs7FP!B6S8I4_*lY<d}TvpY4s
zx~kQ%|6%6ts0DKt_qpGl^O`x9_i*|tOaHXxZ2L@pK2G`4d(g<c_x4Sjb*x2qdtOdp
zdnB~^;bzWsy|VH|<8yn?<aeI_l9XD-cWuwq!kpM+?$Nu}T#mnT|DM0`U#m2oWwMu7
zB?}Z6$IjTl|IRbHmMzK6TW%lnKVP`EZS#`v=Fj;WBiBWBeN3)@GkI@hY@0>uhD)5E
z&hB{~u~E+Yz{{W4E0^AR%x-CSX|qI4FN>Mxt~=We-e;ZJ{^#B?Ii}hFYBE<{$$whV
z`*dl@4DbJst}orj@Av4KYGu@?poshP<0IBQoU8CrT4TP%9-DPfE=IjNyYKRz6DKC+
z8?D{-cG{zg$7$QG@>EZqpR~{_y7X^DwvKxDa|!RIUguJ4>+eZ(?aEoK(Ywe%)UvhL
z|C)4U-j&GRtL99W-=BYC+t1BCxp$vDbI<!OSNiy^tGe;snJ<q%ZCLep)%H5mxc#Zi
z;%B~8u~>KPb@((Bt~pCzpSbVUz%0%GN$!TV-1IUxx8i-TmMc6y)tZ}qY|jmY6KD1q
z8Ed}$GxM9jbl0coeanoEP9E6FJ8!DtVSe%LzieVYgmFz@C>Q$vTC^yC`e&b?>(>^D
zZU6jwc|G^Juj+j-FCF8pKOB3g==q7$2lugsdmT->ss8KrRNZNNCP&^pev<8<ci8XZ
z)Q!7r)uRf;9(=xeUG}ler%j#j6I5QWdGj&i`}R|U&rSZ`p897W*I9#<{L<&E7H;-P
z^|jqwdvC9W{qb9GYai53?X*1gMe@WK&#zB!vrRo!Qt{enadkuY_x#ig-oH2R77yRH
zxBV?!)%~{9d;UBzd{QHGIkvWcZpX{ohL;PMSHE~Em$&P?pxxs3+LKQ=N?TOle(w6{
zi`_v<Tfe1SP76%E`s>SyLLb&$?h}4DTr9jk^LXy>@(g)KDMtw|uFo3xIQIYjx99Op
z+czB_?Of{AQu6PGZ*F{6ek*@p{l;D8jk#RU%3nw&oy$8nb5-k<`S)i(($w2)pDXR)
zxq6@H@3+<aw{?Bdbh&D{R($)kFV$bq&4~~Wir=#Q-=$Ad(psfKx7F*f=KtND{4S`z
zed?XW9WORMo-SFp>)Pa~aJju#Z=X`HQg;2Oq4u~dTWw`VpA65mb>ULIFQcN*-4EDT
z6|?fo7k`=Mm%_udR-7)|`S$|PExuExCnn^`x9u?H{(9%Ry1A9hfA5#<KjrUDI?-)!
zZ}MBAyZM*Ilf0PHhK>JjHtxQ^X!knCb{;;XBTVm3ov{(UdE}I5o7&4o68)`rKI}Rn
zp0MTdHVgZD|Ld-ne=CoCm42yr>9Acz#mjTN<u7IL74kSgzrOKL%#!n3Kg`5<)c)}<
zU*dFa^M#E0OX~mj-)(ytzGzLr)lFA}|3>8>`+B&mrbc%Ah6@jKf9*Lvbz-rlo2BxB
zhr)7ywmb+>uiJflPyM8Ci~H}NyLtOB$5PvxrZ3aCI!>_tvSqvE?Trez&x(pCU)Xzi
zS9!(vDbodG4m=atny@Q!#iPzQGE4WLNx5UL73TbMP0BwF@v0whmfd@HeUG^Pd-3!8
z4nIBI`OM<fo!m2mEc5-_7tXp7GMn@I>X}PIRxjF{v!Hgp)yvzR`)u>RzIb}(W&XT_
z_qJ>Ntkhof^Y2^H7PVjYmsdQQba&=U=d{b#+QP9`7Z*>K{F}m--#+&Vv#-^~CD*3?
zU_5>M$Zf7m-)>%>{7&1RcPH;m*@?Du6kiAHKC|1qyLRsC9J~8x>weF^r+@wG%iNym
z?XOjCtUo2Aa_Xx0*=eW$oix3AW4q+nRi9p7%r9{_w4LO8p?}`&IF88scV&`fJ-Juw
z?GgK*IQPuE9(|7AbGB@4wv?~k{oUd0+wyB^J5?{$2ix6PGv7Rae{i7Hruuzp(OHYQ
z?01KopBCvduix~gwCI%G#ZTWR+^q`u{L1!h+T`fsN%`>|50<WO`?RHFRY9e2>Mp+~
z)5lVI=UcBnR7&pdG2Hpl@AB_k*?)XOpXE+%Y<zxkcRG{pUiY;Bij{A&-|PR|a&+e&
z{qMic{=6@fUi&9NP~+9fH(p(DYO}k7^cEk`u-~Rp6L)jpe9<`FPdneJ#>{&t6?@Gv
z<IU#pL3U>s|Ns4GN~`DB{<FL1x4Inm&;I^9?OdT}>h`D=-%mHM2=l4jlzuMd&a5r6
z*>@hF?^<&t*T46&Pxz_%uckHGRlm(yAaBlm((utCF}Z5vA6su7{PUnL+v{cO`(lHO
zEA8F|SuWBNeZ#S&KJ$6iY5qB@?m5Tod%8C1<Kfv0Uh}?rE^^_l;kBtJue%5SZh7<j
z^^Hn7(+el2ZcE?pEOcM;ex8C+`^_Vt@}3*XpL>-x<^HXGflY30w|~sk{$^1q_jUWK
zkgi{~p=VVl`t2<%TF2@1SzE?ZdJ&WL$?{3>RqlJPT`%9c?}liM+}D?`kN+z3-ClS4
znXLSBx9*jnMDyqTT62%l!MeX{tA+NDlzlZ_Zn+!myXUVwa$Y30>VIJUpV-)<|M>w4
zeEnwc#5`<#-abfrremM&c7NjcO2eY(so&;*-MoU2Z|ftSSJTVprX1RLJA6mZYxU9_
z3uEGD>n*$2eUV{*V5a}RGco6R3jS?u3)_AD_x!CN<g+Kt-85TYyKDC`)%R<h#GfXW
zM99hZ{a*Cc*DQFxIM?x_+IwyNj{{#V<GS~B-cRq{o1P~XiN=-PZ7-eM`S(oAz0Yce
zf3IoPc^-+4&dJh!W_bL7-M#(AO*{X7I@g|SQWp70i~F|ww{X_qUHdm|s*PUi@>@6h
z%jq52*5T<fvsx}DSAN;RtQ|CKyJB+Xz53|(ed}+fm}mS=G_d<_+rHZJ&ho|a4aHxl
zZ<-u7HS%A~+5a#9{jOJL3fKRhJb9Ci#s7OB_NB%!ooDU;&0HOEdVSvM>9OzT{J*g?
zYR!tS<-5PVW#jp*I{Wi)uKfJI4^{awYrX0%X8W0aIPEPsU-<TuPjh_E+-G-+mwacv
zv3B#<?pn2Fth-ot#iz%gJ9y;f_JaInJV$@;NS%9H{QC8`LIv^r-g@sk-*!A-xB0m7
z`$@grw(G6Jw>SmXcA8n6&$D@7`1{BLp&I=!{*R{x)jv6K-TSoTp%1fX9s2XAc<G-@
zTNch)+x2wrv}i-AquEdPOnm*!XI}01BD3msdH<SB*64BTS=fb73BKaTX8rTk6Yn%R
zt1X$j0qe!yt$XEl?RV(x|F>h`_6weiT;;d)>sRmY@LSn??C<kyWd;~NzNh>8s-929
zpLZ+nt-9(PaeS$Md-mn){=NSevum?${UP<FXu|2teS2#QjU;FPY&_kSe7Q?wM$C%B
zbu$^=8u!O&d|upkEAz?WZ$*0>Gv(9^gfkCcS-oR(Y<<wGmy^!?Ts&co$K~$5M`!Jr
z_x1TE{k6tF7yYPWeY<+exi0&f-_752w%Yw!>au)$pzT}X#8_UP>fXh?H*}rz-~A7-
zku_gBL-j~{r-$p8*;gMw*mHQ1>p9Qo4^(d~niaNt%aKKWVx?PFEP7fOQ@e0u=!W7`
zW_`~h!&+meNW@>4y)sv*=g*ld_Hq&?hQAGCe%zaPSANy~)|KnTzphW+ZGOY!r$(aw
zsY}<+zb~Dgx3i~8-l)#-OwI2r9vf%6e*BZut~}?}hNEY0``X`|KkavbMhyGMZ(X%l
z{10+IY<`w}^mpp<(>GqsKdf8z_^|KFxW^&-Kks^;e|>ECbSDu9?fB<epNb9qE~Tz2
z;PKweofW=u`uE9_ek<$dPA)wabo4=kcF_GT59eN+aA4Ywr_)3Hck0WpIJ`B)vR<r?
zKd3hUM9I?DY^%HJpQc~*nVY^nx>x@9>G`J;(jNXz<=>Y+^ZV-9uC(^25*yx_*6b1r
z`<HNL=|-!=500<DUQ+#Z=JOu=Tgx|;n&tED?|vTk%{=sGi{3W()%mjnSo@m4RR8`J
zBU$r#=?c-}ubb9aZU6W}NB1S)%{N>xgX-Q+7mM98C*S5<>A^iFy7AAKJ$f^p^=aLY
zO|kKtUazh&6xcKQklKWOtTUcG%isGl>zJ$V53N|4Rs3QM^%d#P$2s3SUb%dJ{dd2d
z7n_1VFTU_x=-&3~qz#j7<RlJW<1Ut2|KRAI&)a4P&;4-Q_j=fdKc7Rk)&JQuM>t2m
z_DT-l&u0zi_no_uFZ58PJnHAY?9?BtY~Q*4O1bdoX6RD+6~CS}9oyTqVv1YyYo-1#
zrH^;-H#b&z#p~3b`#Jbc_Q$6dylxp%2J1f`W}Pi_cj?>mB@ey)KV3O}{rR~suQKQV
zU2Mv{J9&-#m4k047+Umi-e0Z#ef8A7Zoa5*wKM;y>Ygilz`y5>$j;zhAIs*d`W<C+
zm9F%Dv1*dzNjBB0!h<_v=kJL9l%8*)Z=8Ag&HgFoOWNC3AD-8D{Krj=W|`Z84Au+e
z)~<S9XpvyO!1JB@&bi@#&zJr6oHcpN49VD(2Ts}k)0%#s^=q*DuF$%=sVmn1pzp1n
zQhPq#yJj9CZnI$SA6=o??O&LLPi?I$aZXcvxY>Ai&->YrDtF)NGF`4e>q6goPpvl^
z(@q|Y{q*PK&)}Q;{~cF6x&3U;+l)U~BsQ~Y7w;}SG&!TDcAd%k_vion$UUiE6Z5pA
zs&q@do3;&u4ELi{;paDVPfVO2dZ_eH&%*YHHaix6Nj@kVuI#b=igC;B&5;jFB{wfC
zJ;zXaOXA*HuY7@LzqJk>PgG>P5%cTUwuk-Izm7-8-n(URdhOavRvY8)s^m1SyBhd)
zmc8w_e~%^={C_@M$?%VEU9<K6nyE|giteo`UBa;E?f0uwD)^5+dsaOC;RdIb_M%-T
zTkb9}p5wrDKlX>a+s{RAPwurHo9()6z2B8XC-*fUKKIu??nj%s$m`lKr}gibM)Wf>
zM^8vU_u{wY&XtyLduu-NCEwrbvCZ(z?z5XV$7#2HUaJ4+vu;yKLfV_&IR!Cq%N%xm
zU0>X|wQbc$N3Oelm-6O0lv(v_ESNfjch0N&1*iZ0c1<`{uzT~ys*G>isyA+&t<|+@
zZnijD7PGzTrTFoPd{*V6qWjb8UzuEaJL@-p-5-@FJ7@2^ofXn;&A(uqz^dnSPW?8|
zRIdB|YQD`y)!+>O`|r4?)J1ppm84ZHwbx7fwXGul+Pms2e-H8W{p>0etD0_~bdHTz
zO)+}@{`b<1y!;3Eos`}C{_iaB(m(s+Ltd-v>07@#KIx;?`PXT3PbbT-kzbd5U^mBS
z5y@Q!l{;k5{7hA;G@Qbd#`^N>Yr}<~Uberw=~Vag*pHXLk{n;pWd8b;`|b0EjW-P!
zHoss0d}$o-`W8!_&pY|Qy?VSaS>}IHwE59r8`-|~e%$eGpQY@&?TQD?m;FvExi9eE
z`h5M)z5g~n(*5>t=>^kQ-ghD=&fR`={?C@;Pm+?y`_CQUq;F%naiN}hjQpxIt2wmy
zFs)<TY4rG&dz|^L!i=rwcPlIOUCJ|GDARuYnZu=Z_XPI6Fs*SlyKlL=aFf($_uRjS
z^?vF19hldadnfxw&B5of_U*QhvOZbF{oj1+jAGrY9lz_>q#wRKC0hAU+4(=SncI%f
znP>VsT`yT^srDb4cYk8KGxZ<sj|<x`zi<DU@&yycd_G=H@4B+zZhQU4QkQQv$x^@h
zzP`Tmv5t>fLoklL`r7B`(psJ2wcE}_o1Q8BeA>L@CzH`Vmb<x=?_KzK$7$L*!6RQ~
zGjxtWKK*^|#zp&o$|Z$L?oT}xULX5e_0_@Y`9GbErhQGXd~5dcZOWQ8=Y@V1{ob1Y
zuS@>x!>K0Ich8yD;hjDA_o0|333KoN3pVuLF*CY($LwOA$Xj-PY1NF^lU3xz&N0~)
z-23=<cK-BZb5>j2db;mHneOV%CjTN9Zuedm|98=x_nzNG5AFN^qgiqCkFL1yvqRSX
z`X+mFU($>?Lp|ozfA7D2lpf#lW_RCt)w1>Rdb|H#{Qt>}VS{Xg@a5v*m3yDo9{cgB
z^xgD;ru`pIe0TYME-hZ>OqcB0qHhZ#w4e9a{4`7c#%0qXmwiEg-I>4t4D%1ayt*{<
z*8SfOJ3pO%^gAm2p8~)3$IW7P*&puT{-tkt`)|8oh4|yk`~L>p?vtG<T3PdRT55dU
zA1n1u<!4F@&uwS@z35@-W2^VeH#S&KynWqJ@_&a??R}-5?1!)RcAq{zZT0`n7wYbN
z{{Oc!^xglXJ0rL6vz_uYD`v~vn`boQrd4Zixb^T?+=HKOH!9}y%yX`<+<e!ze8;mL
zWkIW-_g-Dy*`a-Y&4I(&_dnPi&f79Yx_{#HYvD^Dy$-ju`16g+)aJUzn&6bZrDkq<
zIq9qt^Ll<1oqyyTQ(GTu(S78+7|&Jt=Rw!b$+gv;Uj6Lwz16lx;{9jLJoL*dLJKOM
zT3`6a6xr=pUb8#-d#vkk?Rfbo(Tz2M7v3J5KmGTkf4|Eb&;MM1@BZ47sGaj#wB<cF
zf0IwR)^PIZ%;(3n)-u0M`*FqLxYs@Lj~m*&<Ie}Z_1rP<DF5<Gop*O$8h0g6nEyIA
zxJ_K}p7nw0p$Dg56?96pc<t;NwY_Z9+*d9!()L_0%7e@FP0rb#Ib3O}nj3Gw$8qaV
zXOG1uf2Xt@+U&B?|DL?+s{Ive_0>IV<M+i}nch|vTKx0Wy^Q-|6)Zo(&iF_Ey4vtp
zYt{SRdt#M;N{Lt3_%79-^rd^p?Y}d#E-pKH?yN*>uf+WsC%ZTwKfe66cI(sA;;HAp
z_xIfY|M_qK`k#|`t?Bu6`n%RG``z!$zTBO#zW>;|_<ijk%Q&CyY$*Ksbmh9AUh57A
zt^4ijA78xBetmc3^xs#bwY|QaFZ(dH<8{dKmkh7}F5dh8|IaO#o@*UrHLbB!(tp?I
z9DaMl>q%|fJ*#5hrrmv?d$#P_wBu2R^(l%qk_*4D>3_|)`ONI}kNZk1;{S;6bL2mL
zzUo^4esfLkWm~M)r+2<M`rO9Py3X?4gJ`bEl%@Q2V%B?Cz5IH<(mq`8uJOI?=BJMz
zzx?;mwmmcR?!9}-9WR+NJ77~uU(tqBUhBo*O#H=AoA~hcj*2~OjMfdmmIghFxqaM9
zac`c7e#N;@Z(}%j=h<=i&8%5C|Gn!6Q&#tV`P~=h?<uOA>*#y`?D_LPbI!-Ty2_=0
zjq7EpdzILG)rikl%75*?Z<-b!RF!J6@7x}PU8+kC2T$H||8Ee>{l*8}Ts6*2o73}+
zZ!3P?VPKNE^O*D;ji2u(eDRHu(mSz0U$Ww-661aC3w2?NWxmg26YD<qZJ7VP`d;C;
zn&)nJ-PEKdbDkV8PJO>*|7_C}so|IW+SXtDA#ry8?j<L-<`;kdo8$B1GIxWGwrt+a
zTNiKjJJqtW*hs&f^Ho}0OZH^>nqKX<g;7Gc8?Wz})w9*P?x&W?ec5Nf&(`GL{88{>
z`KRAc*^IS1msS0oW*DFKtolse^$DJ5r+2e%Uwz=&d8gk_2G<OKmcILGwfVOHmY>mS
zc4t3)|M&ZTSA+b#f2RGJPx~t`eRZ2Zm3iyW)qS-amwn9THIcpiB5C&R$7lK<&)E03
zJNNvb<l4W}(&IUfc|ESH?@Vv#Uc3H!>4s0U-0rQ@4c>U{_wtV&=38c5yq_9>_e6N-
z-fRBrUMgLA%i0<Cy*A>Z_tuE^Kbr!NcRihV%2k1<q|f>2tofaC?w9M{1-I%mJ#NXj
zJXh7VIY9Q`z0&nx{}`%x?KyK~=Ym%;5m(pzh>Cfd5L@E*^4Y~5l^dtZU7J1K<)v%5
zwf^zE>sEhy(~fU_du?)*bA`m0+Mti^-K%VFzMFr-WYgSN-!~X&-?xZb5+)sD%*`Kv
zMP%Q{PS#$2srx}Hrtt?v_6fhdA|}psr`Yk8{<4Vg*Q56D@cgFh^Kka$b!JlMw!fG9
z#CPc5$#ZM!U*GyRskS&d_y5f`^Uqa<XKP1?+}*>bTWj;Ip;K(fHtpv5pP4u5XY6jW
zvs?Ols)hFk`+S8DzsmodVD#SlsoG{*_C1xaPmQLY=Htmf{@~%VBX4HUbiICa#iKip
zr{y_|*SxasyKA+1#U@|>uWm0z_wHPg(t9dp*?Zp0n;#iBtMlGG#+p54$y)g)dr`yF
zS7WYEpRzbOBtG=X>9%vM&Mr%~RsHqdk<az%sKa`@e>3*H|DM?T=;5D=|CRLu+jrMQ
z%$>5nc=q{4X7SsjYiIC1R6Nx8y?%FDN>KWqq!sZN-}YWizNqy$>Bf2A3-`T^@*=N4
zzt3&6UTAMkc(4BSnNQtQ?te78eg4YF3(uu*XMN1p+w<=Ii*=_<1ApsVEB{#UZs8|w
zKk<gD<vspAFFVgKP1;-(KcmR%^nt{)*|K_$PWKl)o}DSyyFpLw$0G^pynE|seRTNU
zcm8|Xww-xT61JAzdv;^?0ZZP0-zpwdr&>I{c3$acoo?{GAMX~P*}gYH=2rdsZ69=v
z*4#O_`fG8}(dlb9?6F?oaE_Jv@9SblFI(%CWowtLs`!6J@8RsAL(}KE{8_{r=_6LY
zC$0GZDJ93L23w8=|E!5#oW}9~Q+rYG51%Ndb?cs_{;-YKOI~u?;_tPNtn|>xb6uBq
z-<h=_Cf_$wd7k6HI5q2%W&ix|wZ|Lo3On3hd1%)i@f~|2EdCkA9sj~mDX3MmyZipL
zKUN2J-#X8`a^JUB)#vHS*Yy^3#X87JrSA1*+I2ni`QLNqH}WzgkI!{HcXr0}o=c%z
zzZd@9GHvhGTVLPU-qu-S(zwv)o9*vt>+)?8ea)WoyEd8Mc${f=bGhZ^&rU1PY^^#y
z#diHI<D9>?zPqcxO53bYI-l|&J)}D0@4~fjyua=U)3txz_ie=~0pE2|zd17Pvsd;;
zrxaa1!D61m=KS@?r?b2|KBuO?&rPU}vs0=yZq6_Nu$OzE{=w;5ckZp98_BNkZg*e4
z=+@%Tyc^bSyyJH2>U)dAPg|JN-;}L9q;IbBBd(<|$Rl{kuGL}Lfw?Sm!VSNl&OUQ5
z>)S6i*7!9CHoe|+a^a?$_NMhAAHH(v-=Fm$S}0=1l=YRV-(nIjoSVJ;_Mh<m|8q|L
zoSgpuCTnS(4qvVEq0`@mek8TrzRI?52ZQaihq*Ot<99t@{^-`l);-O!;+lJR9lD--
zP&(n`t}C8v|F<3b%kT1l<z{`lk(64!<-Cva2G=h(Y+1A7`puou%M8uSi|qDU&yi<e
zbu09}S<wXRMH@1UKJ#APH)U=?mBQCazt6s(^UNau<N4BkWgANFI2|rze)aRs{lEL&
zis$l0Z%eJM<~jTQczVFR+Yk0w{d(x(z-yNC;MS_nXnS|D^;gYyD#mRLz4QByRE<)}
zp)b2P<vB|J>uGqb{-U-1#pZR-&p9~BaLjG)FY3Nh+a13B^NhF;5=X_?yf3v3Jg#B#
zrbz#dg58_EuKSi#HNC?(h3q?bwBUHN>B8b2a}M{nXBYjr6xn~yWr~5USo^-+&)>>+
zl>h&svZ>+mk`sQD4>28v!o`se5lJ_slg>nY**&;^STjl8%ferKPFj)JWv2e>qgxKx
z<jtLPSm>N%p;-6kh;y~BVi7tUA|4#C{ju^$mHFy#W|wcS+qk;m(B|Ic#;+kswsuG7
z9=+^)Kkxn1Z}sb=i|_ZCeoa04J;b8)(5nrWZ}{4s_|}G<nJD`1+>`Luhvqk1#4L0Q
zOt>YcS+qHG%ZOR%7DPSRav)MCA?i_y*m71s?l~Z_$Ol~95z}_eIhxvdnrS+#9(Rn`
z4gnAO57rw$>^3;`XoFP_-|oYOEI##(E?*h<r*Q38QrdE0!+`?TRs274MHaHQu`Yu^
zZNX=iuOnR!UVF6p@r_T#I==*-+ne!x-D|;Vxx#As=PyNeU!HE9_}A}cc%*rJ+5M`>
zjs461|9kg7!t>v9_hagZ_S+rW{^(b!;;i^+%eQLRo)+J)ug{&`{@(PGdH8<u=u270
zk3D<MAN~Bk(QW=pU$6a~J47FbPv4_gxj%pX-=$g^&nvTk>#r4S+w^Do<O}yN_Dycq
z-=@BAvA^qltsion`~E#(eW4%uN}WmYo21Xyhe6VLPk)PMMez#WzR}_s_x7J{Oiqhi
z?_Z{l`A+sr9Zu?>nPR&6R~OTY50lk-UzI6lFIjM(Ro4IV`t|P{-4`75FVQ`F;)s3G
z;>hmj3ct)59Ful-&-k^){3&bGdc(cZ%i_fpRw+)4-=!4EouQ?)Hz4qmr>lzSL^GRk
z*OW(9GvoK91c3M<E<u93dcJ5Io?fJMjx~EqUzdlL((LI9zdB#Dtms&xWU}>-mzL7X
zj>ivTL?^E5SfVtk>%yPU+AATfDPOq18@6_7gtVxwj0e;FJF@Tn3Jno#?fMYu?VIvl
z+p=**hm&hY-tX4hsD(;gq6=@nyH_OK0oHS%S~~C4Be(qlg0qC`MYR?M2)1@z(5cMd
z<2&=o3vn%{&(GF7xr%_qXHBmu{I@GeaBA0uJ5}#*WO6K2su6LVJa2Ecc`Dzk_!S(!
z&EMSSIfS&B7iud{(ppsI@oC1At`#9JOO>=l7k2)>_iwh9;zE$Rr{DV)IDo`WcHduo
z^U926Ai?k7??D8&Fx{wbkx{+7zk>I^g#OA7H&+v_MJttNX)OvAjPpG$|Ms4PNJm#j
zh)b}mNvO+WB`ML1A%eXifzKaScV(=CX^;}VxT3?&bxU#jthyVAf2YJepA$1%XW=ZZ
zWe=CEPn5l8arL)sNae9x_N8lNeoIG$zkSCVaKe9YUGmTWvTx>Byu81$<Zb4RGpnq^
zZpdnDSZltjT7G+H`Nh21Ri-|@H+I^rFgD)!deQp@r}plxdnB6wzO9o(HvREj<?e6y
zE{Cel)p{A?Sw6KS)N{F4&D-gF|CfZGT;SDqT>7lug6=ZcZ*s4$cnAJ{6g(~DT)<Nv
z`CREMeRn_egq~Ya#5T3~U7>FD*&2Q3_?Q#hmRIe&UFsoNyPjuy+|pOmkLkbde!PgO
zwC>d(mC$XuVZWw_Y0v*1_x)huPA1WePnVx8KmEq#W~eFes>~CgFaN&7zWeXXmB~8%
zIrF0v0_O*x6ZvNP_Lt@l%`cPt?6WRE+a(?*oAt`$w3*8VmR>$p?vpF4Yo6XH*ZL9n
z;f2V>DPL#hdEDGz@bScP`EMeJzg;T--=QwDH|1X(>wD81A%A^d9lw6Y@9Oz0r8<*e
z&8*+S{ZQZ9vP#KD@SlCp2g`X<^6X#c=i9yS(63&7KKx_hw8BO8+VkUUYh&-%)CBEI
z;(yI>KVN_C>g{>!N}s<j{uz7kTi(0$d;i$A)BR)L8P?fneV^SiA#%3Y&*jHo|0>R}
z`uEK`FSfq-*Ye}%&g$NqH{(v--SsCQpYP08pZqpWSLolI^QHH-e!pLPwSGTGy`A}I
zcFz63cl_p_`e?tLqMg)3f74I;r;pCh`YGSl@wGqc&++1k_0{u##7B$%?-oq)__+Rz
z%g^Xr-|Kg){bN`DV{!7!bBVw5M}KHfdAwh4;lJyyf6U!i)aQr%kUw=KUwHNZ-dP#@
z%Z}`KDgM)~yd}Q9;oJIa?e*qP|8ECh-5;R${{LUA+keihZ;W0xkE8nQhwb-{?yakT
z{MEbV^4)*e-m3rq=yPoSewRO&jsAU^zbxhdo{k?=_x;})bHrX|{=a1%KQ3=ndsbfm
z`|eBo(8u+wOaF5U)@LvKSzY_@)C7CAN&k1|{84*m??3m?`O?k%tsnmFII-VU>7TIa
zxzC$f1uOpuzKh@DK5st1{Nes&_eH);w`XW<TT?9a+hXta`_C^u=ogi}7{1?CfBycR
zTk=2F`rrQg+*0;>XNcjP@FgdG<A2@T{&s?sy`FDw`Qp1b|5){%5iHrm{rv9tNoyDU
z>-@i!yJzm^d+B#8Kd1&@-%$O%R_MXiwawq9IsXP$s0-{ld_(Q$Z;L~AX&cKepQ!3r
zJbnJ1_13(+S<9|$y?_0|BL9_pRQ=^XeF?pO@{I0=m%I<}RfX+wJLWV&wq3kaKliux
z^{xN)X2g`u6|ijjp4j<*hyJ%8{V9J>cy|3Y&r4K&c<PcJ?<VCveNW2kZdTqqIPrBb
z`=9mY?xE9utgk)u@K5~ih4p&z<?mL%DsA@fu@Cv}x$&F!$GOq>HotTGA7i;;vHmQl
z_mxq9<~aP*ZvS~F|JA<ArRCSY@B4S2W8%y{w!R;^d{IwA*yk=ZHu>19)cIGcXtQMV
zhT4Ya>9R&k{&4KL*?Us$&7ZGY^X*?hU;pjr>UhE9&(Ht-nH{@neM(%Rsz|t3;Zl*W
zVn4lf)?InzrTa|x)6|4G&Sz^AwoiHrW=)N|=QTaNsZw=Dc-7Oe14XJ4>sU@tjoT@@
zA%=6B_Q9g1;d!h&;Z0w?HpIQW8qN56?b~%MucyZC72OcWc}@FZ)za&EtU2LLZ@o6e
zzT6tkczbHXZlRp;s<&YW%2aQxXZbxfZolY;c+PLy2kRC;U)%843(VPDsQqAvkW6^f
zW3T)>To&tCj!%8C<C64tratY1m8ur&zdT#p@Y$;%j`N)M-b(EUd%hmfK3KVQdmd{{
zc++dIg4ma9qZzMHeXvU?CcNr(*n!gbU(BC=+V}O>_q9DA_m%!B-}|@r_0RWp?02mG
zSKs@$Z|R@l`*$Yo%bv6Q)X#;*@4TL`D}9&sy#3xz&3QXA=bhNSYPa~WWi^kM)jnEQ
z_h?=Hqje465-!a;e%yFTDc{_@KL>u!w3;_#@xNyNg%1xdn0=G=y1=5cn^GQUjwRf$
zzm?)p7~cB$>Fa<whZ+AF?@$W~+`m(Ellbi^f?q!`zjtx+%-ZbhD#jD{Y<zy9`Tw2u
zx6?|mnE&}dPxC<C{o7CN%^#mUwQpzgq^Y)+-{x2EvYP&Q-%FwUh5Nr=Rno7Pdu_k{
z>Yt)*N-t$%mhFylfBO0Py7#X8%sRzyR*IGj^V$43Z`&T1lKwKoF!266eb&=O|M%V(
zzEs}zhVx$DmNTC>@p_m2mz^h}`Rn#v4>7@aE_Sw$_B)5WO<X(io!{<vcV^c_JpHrx
zv&0kcHj6yFxz69%=kj!1p7GL8EuH;~;oLPl>|SU%?Rd6m*Zz~&+TVrM@b9pH@y2<F
zVd-zC%FAcn-V6W#wcxOr=e=t$TQ^Nx$S>yiXdeH?UDhc#pDeH!yQ#YDb_hG`pTzw)
zmu8DeE?i=~t#wCislh?{Z?_+5q~t`u6xsgzd;D_!b*~TeEuOGGcW20nYhr~_rLpp5
z>ninQQx1oo*<JNlcYa;q9QT<kC+XkZGPC;Yui3#iS9xE57FqQs_gvbisjuIQt4H41
zV0FOpUdru^Z%@B||NbQJv`Vp+!J~h>mPV8w`TD*5Oa8>(J-OQR$~4sXGH+kMKi<m!
z(urGfj9Vvfd~bhu&HLByudlnG@gyg@#x(pvxXJY!yZ%iJ-jb1$b9-NDc>L~v&lGMh
z@Bb5W>;IeQ?gx|~2!7zGV6kD6WBmW0GhyQNbY<V_$Nyg6nNzc^<<pz1OV$NGSYP;W
z#`FHKoh8>^-oIr3=&`chDbbIUooXg^_zM}X>zE(%X!4oko=d_SmkZ2Ku@iTwuU%*J
z=hSM28*a<`yey~v*LdEaJNZET+dmi2%Rd&*oVWj)`;*_Fue<Nk%lbV1sn_q%;k%XU
zE_L`Tee_hZTX`b%ZROXGKVMJwiaUD3-{_}jo9)djhI+@X>Zg8ox2@mezohP;)bswz
ze-ZONJ~2K0RR8Z+M}Lv)pCtnGRUS=t>Ct|qKJ|yn<oO|=!a6|O)$CV33AgNhxc}3i
zz;p6ng;&nq|LbS+9Q$YLm#pev&TZrV`%`<ph&|^K_0}It1papY06F4_`qV#5l->s|
z7nr}_Zsn8EzCB!@o|?x$@3*OcbNla))${Cc{ajqH)cg6mP5qkw6W`UtYK|Otud8|D
zzE^3Rw)6b`c9}=^DAi3m>2Fji`bg_&eEeHGP!L-)ZCQT0bLozMXI}g7o%Q+t*I%1s
z>%T=mDg0}@-u}Apr}+I|a*X{ADDeX4{;=13khA!@{lBlje%t?VSN?Z?`s4V$s|B~S
z7~0s`-?`KiJuC2iL-eLXZ}YsD$M!J4lh)e4<a6AO>dep6O72ZM7k#QyzYs)Us`TD@
zhdVaUb9(fpr_+jditoBJNpJg-)4ETK^h<WC#^!rok3RJ^?nPDp<LFCYrxooL-*s=&
zy6sC|>pm^hFWIenH_vl>^r^RTFUq`k-JO)Teah{<7v4@Q*{^yxU-P@X?q6^J7u(l-
zi4OavtG>SOwAcRBrS~^o%D1fSjSf4etNnP|)WCNQ(U%@i`?6!H%`w>m@2WeK=53$y
zT=(hqJ*u|(p68=aeU5uk>0NcN`iJh*O8qapRQKk2u8%(TI_^cOch%iVaoeX{-+STp
zy4boE(|0>n-|y~S`t14g-CdQ}f46^QF1c>>e)p`(^S|4xKR#af<Wc*i-wU6<%X%kT
z?ka9Ca+lw?JiK)M4(;6_l)d|eyl7!~1qfZQm}sx{Xx$eu`l?hPawPWS(b$i!KNr3O
zv40+o{dqL@x9iV9!E#m!5aN~i>h5}HYQh!y=5x0EAhu}Y9nr))rioWThPD-kw-sJ*
zyI2=;Bv$bl2r0Y$4iqe3Dgi=kCH}g*-VsfPX!sM*T6nz`qT!>n?6FvRh=ybQQzgph
zf;2onuJXX$_0HEMkRH?IKLNdk*Lw@Y`wFl3eRQ@x7Hfa(kD=(!=MUSv{=7b$A&;Dh
z{#<3<;C#jI;p^-A^Emul)DQhP7yS22>b2kpjvx1h4$Pi%^*^(G{Qr<Q|CN5+U+@ZF
z0@QsFk=XV}<IUP_Rli?<U9NvV{`TUlU*_MLAwF%(+{%xoFZ9Cy{<M#uHus#N^NxQt
zYizzAPJUYX`}f9AbNZ(~PkvfiTd~IG@57CszO)}dby;3c+rPeIjSa}P|3CfGdUEuf
zw<@o8`dX3sTQBY7n5MtqQL19qviN_W{p)Q{uTHQ0s<VFH(@;AQ^C|!NPp9PPJ)LTA
zb6UH;Vp=#To9y4)@L!Bw_2c*Z#kSe2Yp#Wzzx66I`sMrd+){QMut52R&D)G#L`F+~
zS+f@8(Rydb`tMgmzwVt8=lOTB{r|!z>AhdqY@BC$ZS%2HVCJ8Pk;y*~Z$9>^J-zoS
z|9Nn*K?+QejsFuF|L>W-L;aBO!_E)l9TRez-s#)b@2UI$%3k`V`Pq&7bJqX9_-*lf
z_ZR19{NB0UW`Fbht3NNaJ-HC}q)sa+aLKCp)d#A-=Vz?XKYfMYx514m-*#%!b@S@$
zIT`(Lep`O|eBpfXgOwh^PSfjZb^mT)iSSC#f5Wq_gzdI@bFVb(lkE5BdCJn*s*S8U
z8W;^6Sgr~%E0}IH4`7Q{Z}t^t^>pasw9~wCk3%x2k;lxwEZ{)!V#ld$rBX*7R+YXt
zUSa7FFw-&jo$J4Z%JmJ$Zms_OcVgC^?uQ4BDmF04t;rXi@Ii+`-P70H_N9N3&(`O+
z9aR?;O?u_?%JA@f7N1pGE2q}Z{==v7-+X;je`|t3gZ_hup+UTnz8)*4Uh&$d5-{ic
zy~k(%YWxoT{`l&*&F_!j;D6iv?%TgQt*j4cgPypw)cO6L^ULyU@x&WSouP*Dp@FL>
zt@^WI%PI!n{bmQFl&w#>UOpeIS@QRe>Y~ZfoiDt8UKK6%xFnFkVZb86#KX9(uGaUj
z!Pl!nS=~zeG~e$QDq%WWvC>(`>IfefV_>p?K&us(mZx;vuNxd5UB85GihsGscxi=w
z#J(-8%NY!>qy%(Uv->h$k_z2de)NmrN?sS{ivnJM-}UOSzFd>nwSoKOJLTJbZ_GDc
zJi(@H-L3yk<fc!G@Wi{$+jwv4r_7(gdq_OG`3>hL@e^tayX$j)Z@hcl?tZtd-L`Kx
zE+y;t``VPfzp*uW_q(HRveqSU&TLdY{!RLJ-kVh$XManQ@8vZqJ9on{d38^(LD_*D
zzp6IAJenph9sMR_<K&}aV%6a}Q8!aewoI45b(H_@{dTi=&e7i%p8a*f?60{)M()|=
z-<|}|{<zZj$CY>0S+{Ms-v4)3d$%2r?ecj{*$Ljci_$sP<bDp%+HbSfeBLb)|J=6a
zWe0<A-(Ok#{z~0<?X8#R-a77=o3Q-EBqi@I9nDQDS3E8%y0(kR29<aiDjaPPVGhdu
zZ~N(0`qq+FOE(+7I4<{PziHT;>${`lk8a=fOYPnF|G)RXzNJ6W@~y^qgE@^;E=%l<
zyVt)i{M_%2f|@g}zm@O&Hs87|ta8fDepzeL!qa+2&vmV<0^Q2qY)YOkyZg<J)YE#w
zg63(vSKmBb_a?k`<JI__x8l)1YwgS4gx|gRx^HiMWPYf`@3(O|r4qku<8r3%;ND}H
zn|!Nn3-8T!h2L)PykTXut?N(Uj_vN-+j`b2B=1k^{%hy!ZkL>W;>Ie&FSB&ErS+_p
zkThG${w;g`4dLz^yFOVzx^wY)pKbY@oyq>8f7;F*J$UKD9OZA)t8T1cwohJqlUd5{
zH+Q4VwoSS#n5nn*dNl7hKijKG`KC9cbHBaII{E!d`_sp37M}DlaGWg1xANiNkX+VY
z-|W>px5pjJ5Ae`#U#7Y+@+8}er_R3eS-))!Iaj`#V8)Yqc!HV8&K5C#Cl?MwMVD3q
zai<iHi4RrZy9Tu@mP!P*E2hd_YE#UWxYWkPp<8rff#WNKFZ-02@H*czSi)=WAj3IP
z<;Wxf-zQF<AOYuF0sE%5y!*n}=sNxHH^n6i8!vL~O`l-7`@$pt4tqtWqYLsmb(D@S
zxbP^b!$;Aq^@hlz;0}L9r=v?QJPPRuQ4DL15M3175w4hYbkT)JVI47wWvw@+CWUv!
zYL+d%@zv|1w(I(}I?h=}OB9?}721DY{o=Eiq59F8E};vw1H>}8uedTf<aQWqo?Buu
znblT#gT~}VXSUB0iJIeCsCIOwNNUg&&p<WpRDtv$n8>Dw;@$=mpH5BkHoJ1WEmQF8
z-~a#C*Zhy)|NsBBRX^8;?tjjB|Nr~c^GQpmMz4Oi^g^Gob=jL28y6q%lb05d7L?vJ
zxA<+<?Ru?N-jh&x@z$%U(!plSay$a3nJvljxH(~y;`&X7smZ6>uJEpu4l-Y|N<z#1
z?u=cArOvO~uJC3`Uou;=(jYf^Tl~khsl~HBpM-7NIW;Fwb$0a0I`=oym(7>ud)%L}
z?$xac`xIkaU(SloKXYTkM#W_3V{JXWl3HgZ)@aYP&tEz_`u;OBjT3A0idWyB^41}*
z79?<O`^x4&xAhdZ?>f6SFIbIX=hs}2#MNhUzm}ThdR<;-@;2}2_E{g-mes6Qyq|S)
z`>aoEx7Bd#FzsRD>R@JMS76j>VEf?UQqj6X!96!<hi2>SpdBki6TI#(`;xm!$maE1
z0d=P(92=Efx&`!|j#RX+5Pam;A+1={azk*Ddxw0WW=ZRAuP9hDakA-jTj)AfAlxaY
zlkGFBSgcbGNMNQwy3-uaiE2k?3FI!ioUgHEh43T44r|3vEjNT0`FGeGYHZOEY0gvM
z;31L}*x?p^f7zF}Y@l@0Dj}K_;$yY;fKsRqmu_5ZhiFn*N2Fp_>yE8obG`1r`=?#C
zbjOrOL0#UOdSFJIX>&+3l1Nfum&<yQ?n_dx1|?ixtZSR11Ur{B1#4L>MI{0cYKqPY
zLM2u_n8M|^5<>hHiO}(y_+n~OV3+HzQw_W?^Ig}hd~!YP@bCTMf9tlq|NlE~e*J&@
z{r~T0e+@Vz`*ru~H}>UkZfs0;@0YbVQ!rC9OId&A^t>n2)^?l@JCdsXXp?B+&8{6e
zu6va8eDZuiX~;D)Xr0-ToD-{ad~VLzq?ig$JS(MxG>;v-(hE*kc^-EsY*X}Y&6K`u
zwru4KP`b+XxIJOnp)b8Jr7xQ=fhD7LAc0%=Q4&#rZ2p;h%aXlQ@=n~Duv0O(b*8kj
zd0O6ykli^a?o8OK=-nDAz1b`+_r$FUTNSgNr`-x_y=k@6LtXsklSMc4Cwl1xopjaO
z>7gqA?0_p+{A^=)ND)ljfL-s>Nmr|#8GJEICc(s8Eej_4fy9d+C{2ZnpK0taDVpe~
zH;Y+(<rA2AB45ms$=xMI7iD&MsEV&VrSvy^$$u{4b4sicOBlM<=D7Y^@1ZYx_FY$6
zdF##CiDFupoLr+;cyNnmE>zMDF}Z)6AvAu~ub`{bSM3XVyEIEbwECr<>-?hSLNoGQ
z_b*DmxcSc37tyQsg?wGQN}oG6DB)_{rK1qlAd%4OmAAaF#$CD^bancxoR#;cy~<np
zZ`!MZm2%Tdc81LNF5Q*$qM3Kv!7tOT*8TfEPvBa<ZOHzw0e^2rKd9bqkdnBzwBi&?
z_3=Kf&97y7SHDiK{QLaF*M22Xw-(e&L+a&1xc`?l{5|JpUGjgQE21q2SM~2K>znEE
zn_g>fdZ=l1S98-{%}qBnjc#hbtY3e~`|+yu%fStQe|dega%Ho26|<HOU|qo!!nled
zR5Rq@tVL!=Z~FOmy(#J9DeclJ?OIdvEqK#zhS){hEmJ10^qaWWZ{kM3iCfRrX|*bT
z5cpx*BIm^4*nYtIf#Equew`QFYx^z)ztH?3Qo&`zI*;i*qr(f$FG3ZZdsyO_^c$xi
zSg-m;sETtB%Qfp1o@%aXwp<!J;#89F1aEk!Ipe#?n`$mLJ63La){urcjpQf68$M~y
z_$l(HhO5nv)mxr5q$N%#`AzVKZ<;fHi@d4jVzX!MmS;We@I>WiV+7-7rYzPgTsuT3
zcrSa-`gnfRSO0^5mM8oVHek?-V9+dh>aeq+nDHgkwOs23O^N{rg*+Oy+0F8j7c`j$
z9F$p+z#n43p%uZRS@870g@#3p?#z?eRCpt#E|~1nW?#k@!n;Fig6V?H1D6|CFb2mn
zhBI$s%i+Btb;0yOw!@8vjq4jWGiD}7aVxOhV%x*pA#Lzow7^^;|G>S5y^OWY|JYdk
zIR1D^%rUS?EO2_za)7f@m`R<bPL;<GB&G-#+b_hne5y2$AIBU43&nz{&yEkm4?WnT
z)^ZS{0VMuoD#(ySU_&-XTvb}Ot0<x1&;ukTSEn*J3$v-iOmTYfecQLn)?WSZ*?*Oc
zRc<Pt6yVfWIWr-ry+u)In#T;K#O@YDp=%y9lyAy$HmmeZ+~9UdhO=6QXVNJ%UQve~
zN)du<G|qxZjxd&L(T80Lp&PU!mb!Rt=89ph3thPSLg)jn0^uE;cUbb6_RqTD`$7%E
zd&g9swUYa8>$@u}55zvuD-hoi?CtzQ@rytOhYgDy6aTGNr{4z_UI=~wOD-Z+T(+5E
zjmr;sKT!Q3R1x?>^~<8t`)P-oxYAhKm4v)5?3f{PbJ+<u*KJyxA}7A>%85uhy(l><
z<+hTv_|07>p1E$*-_$r!x2LajVs1}PN{aU4rZoMGZ%v={G~PFT-D9UT^%~DC<BanQ
z9GlsC-x*z6cG0)?$b~hIyV-c9O>%gqPu#{UWu9?&L1pp|ZpAjXzK2GajyWo`uaz+=
zzHrXbnVna5%a%2-zfOE};g(}D`&(I)iVF`m##nM&cE8d2p4mB3NZ)9y{<dSAEU#@m
zzR7W#?xvhcsk3#YQ*UaWPK!LQce*Wd+nG(7({eZG%t?K%Gdo@Ow~lniO{ddpQQIzV
znmBFkmYij!Z?@#DNcGm8opm$p^s%UI*Ea3Ew(<I=gVS`k<!nlwtt*{-GwXC(^zppY
zZPDBAY<f8@cYDsB)YrPR^JRbQN*CO0I;|FSyRAFf^Q(65_Q=%WsO^C-w>`VX&htH4
zQ|Y4RX_W)po@*JsxaIH4zcZMJF{t5)h5W;nD>Of9UbO6QTK9ZD%T&+2FQK9fxB|X9
zhjRI9nxwAe+}7Xxj=$@Q$V%;?aI;;PUJBh(Es>voPoCv<Q|ZCB1m{id3C<g?3*>>j
zH9m~X7=jx(6Sxefo)gL8GGmQy(mg14byrKAlYZlLhSP5yYWGVsKD}15cTs^h*Y>0j
zku&FqKYKp)XJu;LzDC9iy6%@*bC>_DbPWzSDFq3cEH){14St;G&6vcM#Fca^k+&!#
zFY{H`vUM2{;?$oSX6+)an=8}glOA+{jdJb2{q^-fj$jkR^%`Hhg4Ulop>4Hy(PeG7
zUysaw@X6`1rgY`~YHs)NIjVhY&!V&1xAvS0iD#_H5WVn9H8<|!Gf@yRRkG4`X805k
z(KqL{^3j>^`Z^j57#}m8YP~6TLH9wlgGJ`YF0u6)5aQL!3fJE7D;23)aTgzn>Ou%x
z(e*1nc1f-GIjSACXAy*0z5J1=X<XprsVgg7yTi9Yh_4}g79G>h+G8XNGp|c*{k*R5
zD<DIc!Ogq)=xQ*?ycHi$g}}_yj@nbD6&Lte)HH5p7udYbAA`Ofyne|<G(SFP=^OXE
z{oCYYpTGX@?57rIl`qf4>K@d}b<k;tfPqz0EAPRe4Jx|lqB*BEElpT2qQ~mJbS9H{
zqxYP*a%xZRHz;lS&l$s_$0Xjk`atM|aM4{s0brI^f#?pdKUq%Z3uhnjec)ya%Kgj@
z7kpoU@;Ub%)^|+hTz6aFIh8kB_wj;qdn0?RoD=^-_FR)b;R$K$If5T}E?f?;D>4%s
z)m!77^cPNtW`52+8efE}W-1sw;dQ-N@Pya%-i9Z>7fqQ=zjaG72P89Oo%tbcTsLQl
z#saQGImUM%M9;`?l&BCF&`+2bqw+%eW@9bmU#4HIf4Dg$3<C6mj+Sl;K2vkV;c!Dc
zqad@|sqIa6lVop5Trgaa9P(Y^NJu!}93BtJ4Mq!69Zok)XEa>AvuM_CrJId&8Eu)X
z*#7WvNZp;v45|}!df(~wN>oS*nC{YMU&odLt^*ciAGqGI!7#?Mz)T_cz^#UDjJeGB
z*pBqRJk_|Hv6A^4+ZWy!(jUwf3UtExWcWN}HdstE6-}CEDyGFPw_iv}-NmnCj)3LE
z!b6W+j&t_MONQ0HutO2yoOqe*iqw}C0Y@S_qq^lxvuv|mvwXQG3d^b6oJ$Jzd8z_2
zLa_*9#N{kg#iFp+B}W{dfJ_vYTYfd$=}F58&WXZG>7Q>X80PP`e{p|-{<-$I#>)5k
zSN_{4pKSl6rdBhF;ZSi?fMNuPI8)mMLk12Rr{xDU3yLfv9*9^-M+#cC`ZWeWP%7yB
zoYi!8@wtwp4p-0HH+COzeV|pabVKM1uR_*!OyS@J;<a!#v?>UUWHN7*Zq0KtU$`4u
z5pdtpED_(ueV6qe6QuRvxUl_#^9x6BPEgI@->BX?&&hw`@&ll{VaZ(9c_LMuds`qa
zi0)~1m#^e+|2q3voxVrXItQ`MM@tOj6?m#GWP5L2Tmd6yzAG2W`Yz|^zPQXJ3r3{<
zm2JJXxBW6g*~JxeBkSE4mc4&if>36Xm45Ah;EKEGM*hKKq}qQhMxMlG<R@Ai>E$o;
zs`Jy2&?gUG{ysVL+`*bGgZWZtma~0cd9WsH!yb8`<!$Ck>|h>5V)C4WD(M@Z$;GAe
zKQnIrc_zWmI3kCCnQ`;WGYdZR-dXWldFE>d>jQinnBOq$Zj?=6xA3T+@Xz542xc&>
z+_vD}m4j@uITzqW#VMl~iOqVZFM<%n;n=JTOWEdTT<Bz*n_(a;Su)SDTry6F|JUgS
z`+3w9n%^0{&;t{PyPrDNOMaPtkR|0pEZf~wFj2A5=7mtRo*{ymk(`sDc+{z@RYW^U
z3k+8Uw|H<)yrRk$n{)w66!-A9#B%NyD%<;UP0RQG+BI9>d|u%y{%lHZ=(&G0J-S4@
z#Ja@0B%*5eIVo3hh#r`Bglmq9Vgc8hw`W&I9=#&HYNhAVD@&5Hf(%Vp&X^VQ=5+Fj
z#E)gm!}pv~3i)Mp-o7zD^o3T5=#C&ea9X!w;D~L_V=`|9m4#{ri$Mv$1k@q|6@!h|
zt?xv>-@jh=6h2kek7I@eOIuzJc_bWXe}It5@3j-EMb_L-4(gQ6i*Mr6a#WUJDLT7x
zFPoM}gQyE@pyL%kh7D3XL`;_7nAHD3`Ul&-#{37;KejQHao)JMRtt<o6qYG3Q(orJ
zeDVN;1p{9L^AQF^j|vG3&qAg-OAH--1SZ?~O^VQa%(T%XYvL8A{Ri|vbc;+6j+wOi
z?wgRB(9k&{nk!g(7YK9wzZF-m<+logL{u1)m<%s6uoWc<D5wcc?(jUKlC+7z=ZVTq
z5KG9gQ)0)ofVW$1KZVQ*2?Zlf2c9$DGq%fq;)s~YEGVp`X5&*a<AcG&#ISHa7oUz9
z0)^F20vjY=%(!5<@bT-tdoLcTD~LZ|cVhnbJzGyP-n-F1{mwO=qio6n9HLFF1&R;K
zE^M~vkUrFQK(k=dq63;A!2G#Ppn{JJRQ2Vv+wObM`26vYX-DJd?-6;JmTu2_kFToi
zVaqzE@GfmoMR;ca+Xd<WRl#x2lE-AO)eWi`FZe!CD-hoSstrYUao=5;-3TuKJ_u9<
zfSSQopl0xvDF>V%I3m@Atn--sw{N}>`~uVjhV+}b_JSHu5lOQSOg49^JUqpm^?qO0
z*NuV)rkR&i9-eNV#CLZ^gTZ4T%VNGut)A)EW|n>M`z{qKvl@al9eCbwzYzc6|6S^(
z^3BF(M$W}E{&zIC1>RvzNOCyZ(9I~xY{Zts^Fre4RAyynFE&tBIKe0()!}r;I++=g
z0mieeRgx6a51eb5cQMA&z+^$jfr|}G7(JP%v6=L~JJq<7F_alp6`qjVV7efiC;Z%o
zCIxFyRp@Z5VJl;9@~)zUJcqjtyA9W72rySO|2kIm^uU9LLyYarLhLGh5i+N_x=(q6
z>N~|ECun`8Zj*LIswZMc=W0|WEC8u3!xmgy;NGhAOd~KmaCPA8K<%LHpo%#kEY|vL
zju4+L3o;B+on|hb6`;2;#w@#V%FD7jvuvJzvwrIJb?ay4^Vy;IuI+fS;#79j)azTc
z)@O#sT{c_570&v3|JM1l|9+Yk_SAIksq9;9CZLM!0i;jE`i^P;u7cPN3Y`kQ3KJEk
zD$Fdfa&&4u;*c`wqKl7`n&9M)O4Z8h_f1FY3jF7tTH0<GUb`{>)(!b5#(T^7Jv#3C
z|IHS<s`C13OP&*~v=H9G`RA2WIa1xo&aiOz1?LBfAD}&@{cRVVUx0YvKD9rWEvR;E
zgw&3ZZW6dp4Xq!YvKyj!0=gNq3>|bWgTgOxSRFJd_!L-V;-Jg6LiPMME&<^VPAx~~
zD@jmwCqFJ{I--&^>C^>=tr82wz!F7&Rx%i}8n${}L=s!Fj=}JU(v=@ub~I+zuSsFI
zV{75zkTft_kP;Hkw}vM}@`I5=n!}lfnT)23cNQh2JDhKr&*;e9#Kyy`0ct;aGtXkP
z;q8zTFipsExGJ$u2Ht<(z?jT@>R7>3aN8-L`4QU@-Wk#X=F7C%L2aiW(gGF=1)wCe
zF~)L(#e%{Ej~h;K_JbNfeib^~_y4-VVRcYp21`fdanAmy-@@g&8Iu&CEg@mK({JDZ
z<CrK6N;jYs^04qw^3K0k-8V6V#BBV)VjJ)Ld!>A{5!61CQwNEKtpPQazzwqx7NC^0
zrO6kRs6Z+7`?aGR<u$j@`~C4!t4FT=ueuD|g&)0VSs&oovi|XH`4;YnP6q^xSa=hi
zAJod;_v=w%1vQ#PELhJmI=m3s#Sy~-X(Mf&_`B|!eLny0f9es!YdEj5+!o*Sj=#zD
zpp?PVo8X$``;XoB9Cuf$!^({xh~XATU2YpS22i;H?>;l_SL;>zA_5z15r?*?yVc61
zFQ0g?u;5Phj*IS6zkI#-XTCDe{kv@A{f<WM^$og-+h*uJ>*h>8Hp8gkndk<!1Yd{Q
z4d#r6OwU-ZaPIibbwhZA`U3w0^BWwhw*)rKaN7Ikp_}|kKSv(F{O<+I^OTo?#voQ%
zGu~|MVU%PxV=K9{xYw#nn>~r`49^P5vvWBOCm1KBn}qYR@JdJpm^{chaPGys-Uo^+
zt~vhVkb@1gJjiqiS=;<&VvWPKhBb`Q%)8iXcsW*=POzIK>mhyl6{Fz=vj@34uf<nL
zg9cU{zymAEk!NLWn-?`6<ZK3wr+|h>Ebd=3H?T-7X!|OfqyQaP*~)k6xMPC^$DFWg
z_xFlyOxdza`csbI;%8cZRh|W;DdM$4t%|jQ1#BQ@^A6~kO61xOLFPtbru5RM2S7tC
zFl)|o%{?{oSMOgB%f4$*;<YO6OPpKH*2G0AX66bcuy8GO+906Ea>u&-ck>3t9F93m
z>Y7}0nBLFb{nzn@hy^$~C>01+yub6)Eo>e8lTYe;S0?Zm{yx~Zw|F}vv}F;~xcb1>
z`FD3-I47&T)qRD<e|pp_*8@a(LJuMnQ4*XP$7U8ilUc$&`P>@gw-4r)y>p59EM_=C
zUm^a$zJ@x+|67We_ouM)v8nVV{ZnR>aw`;X=wuXS)?-WIIU%vZa6v){b7G>y;f6Lw
zer6>$AD$HwXSd$E!>Y;tg!=}l!xnd7&x_^TCn<p|DSd^60|y!!4d*DS@I*))-YaJ)
zV3?3};K+<Tcd%i}2aY#PFkEXZV0^a^Qb&QhYs_h&N^51fo9>o#fel<u(>Qg7!_?OJ
zyf|?{n8);BfN+oDK?&igQyLDb2ls?WJ&k-WyOdq-+0v`?zRI1cUK%>TRQ`<jjOhmY
z-BZ<*67Fd)j61ZarIPcr&@)w#xW=LIMO?y4YA(=*#-YP4?dw}QI0c23(l#4SmpJ=&
zscEr&+%D(cUfu87-`QV!Es(ok_o&!2QK!}fXlF_>!-v6X`328OkcPz^ruV)_K*?bT
zM+}Qz<+|V3-u9fTvA%BlvWe^drU_EXMfW+k&Swhuiq>3rA@sqm^SkHUMEAYcso%%g
z%h*=WWUl4?08}RhzEA`8q`)2SM(a@Dg}X1rzR&~J9A+%iO}q!cZM*$X;f6pChZ*Z^
zuVOA6)_b?}em7qTe&ET?ZNmzxGQNXUz{hBRY|MCRzNN}}qPa|^^CWYfO6SSuHkJI$
z>hg+GKHSD<LyWKcm{lyRF8L~G6T-biEOS{?dmiKclKypellV2n3-lCX5A13vV|))0
zSs`8kO(%hO^Ai#r4z74-)zR3($jvO}R{FmYluoqRVt7tSoQ)MQJPT=lOyRMRoM4pj
zRIZY3ce?ZL^w$2l2?9P$Z{NS0n7qN!?1qAHfl|5|(+A-{EcOTWKWH=R1;soN51hW}
z-ND2SoMt!v1}&Je-d2aFYzLcl4A-QF6@i>=2bpHn%$}PdG$F~+$FWS&tCo2U$NC4_
zg{vM<aC7x>=865h_3MWO{meIem+bsL>DfK+Tjq+HyJO~-+sEy?n)~MX9sak=*d-dB
z3Y{;0%iS1U?G`-S6^f#rgR7l_tDU3+KE=#Ao4Pzk;rl(V;;h*x-cQ@AzJ;-L?qAVm
z>)tQU6xlklkTWY~CFfn1*Gwg@Z@X@ZZq?3GZ(X;-S(C?Aw#!PSG(hyv<Q=sa?_QBR
z4?^$G)d__LEpO3d@D7~5NY68tNrzGF>zl4+v$nBaKJMf&cfYiw`hs`&84f#`2Q~12
z$lJ`|f4lXz9`BN$P2X;x%Gwldd~^EDH>Z8RrK<flV!AZLF+0!I<Ywp0H%ENRlGLIN
zxVR2*ZD3mSr0W=elzqnE1wWg=U6+1&QtV6$ciJXa<C{%0a}N4!OAyg%khQ#!l)V5X
z^FDRcX``E|GjrBE6hCt4Tsv3ow)W)OsVDcHN{LV1v|dR;hcjc-*2^jCuA0*pGm2J-
zCG(zIA;WX};0_s{GqbAIi{Gcbo14R!tI%A+<KHfD*g&@DvZFee+p8IJaT05q4_ZmA
zZGLFAY<Xx$;*q|m4_h6rCAKsNT1#weF0_`|(LB-m+RS;Cb;nqGH(E>Xo(v`K&s%0#
zp`KiBRF+;buU+EUyOcM6$KHh-h58)(kiy~L_Tfv;#mh&H<Chd&3U)g$X`7kC;eTGr
zHWw@xZq%0k@rbY?&%=`f^Pt4~6dr%5);04ED77ittk!Nfw2?eKS<y!F5R90u-EJ7O
zD)hJHtOBvUpRQghwl6V!>D}g4Ww_4#5Tn}y0WB8phjEH298yhf8x#w;<}A_ppxx@X
zF!%wi7WlB5Blg6G154*w@89Vj`u*6?Uz>R!_-+FasW08Z6yB(<y6$iqLmlI}Zq9Et
zGp0`e+z;<r2=9RPELz`P*$Nq1LF!q6M^>7h`b*u;E{d$WsIxa?THF@a5ED=RSxdz4
zg^8TK)SdRyb@LaHq*%a3mAx60;wqv8O}zDIEfoh#w(2f8q_Wq*Q@<`caK=*cw3n@s
zRf*H$ZZK*2T;iVm<<WLgh`@}c;@cXyCVxF77Ld3u?$5SG29f$c*SIHtJ(RN7AW}bv
zQS{lXqxsq(6<`5~97&4Xu$sJQt%lDf?&K%`ILw)g*q-p*ki07^WyA6`*EMm47EfsF
z!c_(#qHU`V1%%#E)vAjQlvu^(9Ev1W*4cGGOa!a|Nh)>`$Y%(>V5x_>t{-oA9SR5q
z^I&Fd$PC$&8G<5Y5F%PO37fHVmtZkAQGMkPEUxv$>RKGeF4`{o)qD24TcQ)+tqnTx
zd*SWPsXo6i&7E!(&1JyU3L4B4I`k&=>_3GNj%iJ*nV^B*N!3d){I5GMNy6yDpVKUF
z#E<=MJ$pbY^q<$PKT}P9YG-~5-!y;IU!AJMd*ZL~=mb4ozvN`R=j8p9)aq4#3#>U1
z`ao*~RFu=M<xO77oZxq_=j@P<Sdvt5Dm!<t4%h&u-}hM*5S7^O19L&Wp$oC_4v<I{
zhi!`-I6Z?01VN+5U!XIL{*AD9E@aXOJZ>Dde$U77OCLaJ(E=e8rc>*}@BPru{Gpv`
zp%utrcYysz2E*ddf;A2NA3V*33_EA&FbG?I&{K$W*wawS_?77$>yyitAKDgA0wW>T
zj>aZNc4nbhG8KmdJ;A7x>4*ck=UmFS=i!PaVC2G>v>@fc$&7U}$UPzEJh$?vC_SM9
zlQwO3<epH&YQq>y=u9HGC&V@fG<3X;5z!NRkf#&Q_k|bI6LJDi{s}X^hP%hu^DqK+
zHUyY2%sX_aWhZB`@H4eDzB7J%t(YBPwpMFlPE>eQq|?ooO*ayG-)t!Gyo8FF1S|v0
z7UqByaHa~MO56IZZ8M0WHpN$CHdrR|&4y#!4$q3WvQWERcauMK<(0oLet(?QHI-fH
z=RFQ+a#Ocnv$Fh7a9Kp~`W?&T3jOy!npbty_G?#lQoCxpw(q&HS##FDfex=slbt5B
z{>}0Ot`C$71b1-6s4>*Eam0c~k-;M?L65BZ_KU{!7BI~@TxwLYdFs@Ack>;Dca`$H
z<<6^TDsP0Pmbtscz{MdGKbI|hP8!r_1`V}{R6%;qjmr;AcJjX!`9kvpY)+aBJhsv(
zu4Cx?;H31TJ(}CymY!T;A~{uU@ueet*+v?^Y)d<3uWgaNmQpZ_$8Yk%%%lcW$th~h
zmr?>|*)^Mk6{dvDl1T7nTY7SViR6@J=O6gO%wUU{?3f8Q<E3%M<iJcLEnl~#os8E~
z0%q}eO?J#oTD_(}h24lPg$Fbj@gUj3<EEB*&2J3{o+UgHk{65~q?&~DUE$dwIl&k-
z{_^78g^ri&WHh7-OcXL5E;TG=^ktrRtl%k6HrpoVNbr>O3#kvL%e2`~vCZN2kiIO%
z2I)41^IhTHAw2;yBfW3qnw2Z&w17HIZ0g|t52)jl@MeQr@|BdHSv`T14YQ2geBW$P
zbG`!dH$v#xicZ;UNBFXD@nsu@__nP&8DJ{8O3nFd%8FSx5`Awcq6+bZ3R|zqv|fXx
ziYIjPLazHk3=_efZx?Xqd-ib_E%)QPCA%!YJG-q5l`6CURoC;k!t<>jVvKHqU#~uI
zsJHXgO95L=LUh5*Nv6~MPP}Nopjp5*hY6g3HegY8=~rf9R!!!LN%m{i>Z5*Z7KrXB
z+nphja7$h+PQjw(bfboj-fW&T-zGbmFTDMM<Lb2krQqJS-@@G&5NQfJc?DYM0Ggwi
z)_-Be&6e3sz6X~j1aHv1A(G?AfHHT*Qm4C6{EvZS!U^fFJr7SvPuipXr^4Ls5On-5
ziSMES^VPV-O*3;Z==zv{KHJh<$oRCxYuQ83)ek+r8IvBwLQ0}I#-B{zSif{9JyT|5
zV&mb__`&5NvB6*gs5{lpD7bhR`_3YT<O9bV`WR)ILBlZ;l4qwfYcAfYt;80^wuh%f
z(!h9^HhU4<8=e=E7mOdIJ6vd3crn&;f=NQA!{vr$jQ-4v*nD~)o@!hpwc0}AhK0h7
zf+bfi)Nfd*-zZq3ssKtgm$ldoH<&NTKX5-{os5P|frUchfkzETH?CQ=X7!3WkU1rF
zNK}FbO2VC<@Mzn@C!W-8>Qq5<Q(<$j&2_I-Vv|$1saq<#!gNLGic$^GNXrL{haa^;
z!zy5tAfqo=r$W<5#~jG$%bImEu+f*6<2P+IJbGkY_daQMZeLO=ve;|qJNNtiZ$mS4
zKTNvC+vo&ob~zn*!69bG!q*u5K*hB&_<=}8G+%k;rB_yZJLf}^OM&2z&=x%>aq#>V
zsPqZCrd1-kD+t{4mUlmt|9kx%b-lMf-s*a_ljbmM8#TYPD!f_xXjAc{ORqKBR($`d
z4j$drELpy;^&Pk@0u9<N01eiC5vaHj1g^0rgC?@ziySi3xokn>GL5i#PS7F;p{ki`
zAD-|A-ZOa08+va_O@G8(<vDYe=WsnZEp4baYbF~~QsF(s+QY5!oy$XfgWiHzhusb3
zj31evv0kZW1@)$n9&zUZPk9<Fc%ZG2c;HY&8zV2XoLeco6HmyL#goqoUO++ttR0!_
z<~*C{Q3jj#WKIKBR_7Y#GTJWQ?XJw$#Kr>};9bH98oD##1x<S<WbuUar65gvGA1YQ
zDpJVx$(A_caJykUV<GdiV?|FLzymbQzm9EMwrQ19ICvooxX59G6*(WY4_omsWk(u?
z1Eqtg;;2%&yPXo*=l36oo`roQ3EpWEd#!|578j@d{C356vbP)i@1q%0qksN&j5GTl
zeQ{Ozuh8{hUYkw1=ifN}K#}%+?e$;aXf@AL(UPXb2TvA1$x$|8?chAZLSSZMM$-(Z
zr7zCbJw-)Rd1o<&`?>n5T8kRO;MJ*z*RH7A9AAlqQk$0@n0k4_<r99+KF&VwU|9aN
z<Vne<%YqPSEMOdC$!yvDhUW+zhV#iC|C+Y{6B0Vb=G#cbNa>nj8<dgK(%Fmc#F&tB
z>b2ay-^!rOikOQAshNZ{oX**gE)p4QDPRfBzu+O+bI(qJ2U3;PU8YG_#eP|`+TzLk
z@1d!OcFWh=m#74(2c+DVPGGsU-SQp(q1FP$1P)11@94w6vxVwz@X?Xy3<@P&v7m(q
zMmhz8J2?JmIf*aqKG14i2WlHMuI|<@5#9BzICig|!MCdie9baI=sW*yU88%pvwJSg
z>bYQHb1G5x&4StQ<nK+@ECEl$e6t5l#yOQQ1Qq0<#sP<o#+L<4TjZGfOY<Z$erKGo
z<Fe8G0$PW}I&aJ71HlheKP<Z7{X+FiQS#q>{!RTj=k5OVbKKCbv2l9X)O`8u9O=pK
zEBDp9Ewy{GZ^_<6^682C8~$_NVXfl@&2qz=wag6{oDoaq{2SwgUuc4wwdiZ*K!Y?}
zReRl6+U?NT@Hl7{40(aZyhY-xFI7x?g@l3{W=OCpsh{uzPe*_z(I&mRFwbP(QY$1R
zqOdGL_txZNn?sHx&=o$dq*dC>m6lJM+cg6QU!BT4k(shwaTyr06}c5ZJ#fBZ0i!c>
z7n=yLh13Ml0=ao?F1$0O0!$ZV9k>drRZ8#2A72`Be0s?7=~wt5tL6-2R~MKnWIJ4I
zSj!m8ypOG=_i0q?X2wkBOKfX+Go(M5DdZi9seJq5$D~(a6coTI<)|*f=4a>ShuoIc
zUN0+qJ_wGu4mjUz*<{6Z_oL2LB($_4$01U-@f}jQysDTM)F7d;TKNK8Glan<z?@pj
zvg>i=DkS99c;v;Fw{}c>ZN0y~ja-#~KIpg3K9eVP9^t2MU-a%%aatgt(ZU5^MAPW}
zK*Yi`5Y(@)TY5pW1lA-e5R7nN@HBGO>CGY2(^u-94~m_)^jq))ugKLGkXB1fH45L8
z&T;qAo!+yeHJ^00&Sx^`x_j&+PwQpgK4Xa1zal$=U--p-H&#3Q*&m#QKn+`gDh`_r
z&Bpcv$v^#W^UeTOHQ+T9A{Cd;iBy5ww%}Elpso=l9UDPfxrwKwSMAZ<$);}OU5bP>
z9e7$MU+(Km*JSNzEKLMWB<fuX@&7#Sjl;f%TE^dy1vMJWWo6le(Gb@i_611?jyCi#
zN`s~VdS2B%In_9cQJp!6Z4FXWw&xjNr^F4(4aN)751fAiYswzD^y1x(j>hE|V=V(r
zA7p{k2zUmdw@`$o=s^y69x<Ew7TX?Bzi1a@d2-Z61vbdk(CMknt;~GjdBhDC(?qA4
zicOu2S|IsV=%Bi|_Mf%y0nnx`jOt}RD^F~g!In1pqCJ*RFTJTwG=4HMXHV(h0~5SA
zDNavu);rl2Gl_Sn%56~n@%vXJvY2#{r`eP4oD<?VE0TBG6u6ne(6R-B(0b*~)E@`b
zZ-UWGrXw!Q5+^zx8)i5_CJs+B8BCm<qAm!T0_RHdV$eCEnX+7A84T_y(gLk=>sSec
z>*jRL@hX!_g27qZ%N3VToZB{|ZAKp$zB~2u#6gw9^+(w4PO^bfnE`5*a^Pyi8p9aN
z52gw^2W~WMWK3l~$F>9%$l!XM?F=tux<x0P@6TQBkB$DDV91Eo(*b?ek+9t9qRI!8
zbC8e^W730-I@96yizt9vfPNJ^r&DWHtU>VvS+E4s8YzF04^;bLY4U+4`_hY_IzX2&
zoz_o&<NnV5W`Fq9<iy<YckOYjGQS7OwKMlP+%BrD7tm|rW^_IwVA8_dxc%uTY%_3r
zDG{I!=bO+L?IzKKs}e#Rv>Zein00B#h_2xZD_pA+5uD}w!0oSzmdA^0=XK5QNGY^0
z{Z=Yjy}N>W*M-;@as8r2Z{8;DU!Rh0@vZD58_&YOkX*@OlOdCp$zj_vkI5f8MZ4re
z@C(n|T#(WB=?A(&J%tOPEZHQvjO|XvY?=HUAKT>ee|&6H$d~xkrj&2-Y1xlL3o(`s
z!_z*-pU=27XI4#L*`LC`i8Y7&h1dt(1yKjKh9B70kjr?NX&-A#chb7%_F{cr|AQYy
zcp${W+N8qY`u5i=S#T^FsgFGncT9X{?c4fLe_5w{uYW8mt+{Ace9>rMhGcxox2^UZ
zwxERwU>{gAa=;c}fm|(AF}wCHzmtCB);|3;kGW@`ZB4C7l#Dm{ed)NL0f!oBIY;A+
z1#DZuvaf#}vf8(yPd~>@eA>s>)S82DyyXw7|6rNHAe_LY<F|2wn_?COs;-`@vZ`_F
z5-rcrBOy>|lD<`Od0;WGA+I5C4fFj2<;>94NS0gH**-oruQ2iaj)c8i?fFg}><e_*
z)lkCtn&}qnp2=UQ*-hf#AwEH0;ePyjV_PQThPl>jI7t7J-MyzTHZDDS|Lv+h$@8z@
zy!SrmefhTU*0+Dle%{kp8Mpmw#c{j!$@8yolzX4^w|v__>*)V;*Z$9Y{qfwr`t<kv
zlWp(a`~{b}d+qVuz5mbMt3O_~XY<$l5ScZ{cdtEey`I5pJ%iQmKXYUMoztsNpT2+d
z>HB+6-@p0v{hLqaWk0QvWU9fEaG7ZjWlw+TORad`v(0YO+|C&w*gB(i##9FJBC(s1
z9Z)EweELKF0-wXy%O1*J{;<yY78j^KWx4F|<j8CW!^Rm7J_>3A!XbSy6?)}LTmsCV
z!jsfwriKgFs#s4jQOtC@)Ut%rQ@HQc>JHFE>@1jAB&ZJwm!71ia`x4!Mv%%$YAU`S
zAZ0JB_fM)hdBEbk^LmZfbq^FH^L4&E>uJYj+dHFGox$--_j&G{xI?MX^-)}Fj7QwW
zJxZ0(@t{AeoWd8bJ}}jYBZhSy({)CN7h3Q|Hdzi2!i9Hn-et)H_q}2t=zUmy{c%iG
z-I@>1sqONglAj)SJ3Q^~`iakvZuk?y`*Y(W?f$}loHiQJ1|BqnLYASTueNENetX`r
z`pduaXZ`m*`1O*6eHb$mQtG#KVQtAwk(=p}CKSRI%&GTs?#4O6it4^@M^utPi<Vx9
ze|RryV6Y(Jz`=%=K%Xq|($n<~pb0Pd#AtFzIA0HshGc<}LaM{5hN+CYpk)(j4repo
zrFJyVVYFth0=2f_+Yx%--Rqixw%$e|tLO6FlkA`l8EEkhXdNnJvSfWhx+yAJs=#>Y
zZo?kNYVc}Q&^V%^!DIe+YuHech!68b@a_TFxKNb18joMLv&0<Gh6KpKqh$SuEK^MM
z=4#dhZ0N@7s|T>54dJUB(UErjFY}q_K}QYXZffHw(crsZ<QVHY8E-Ghdi1Iig7z7%
znDwJ--jAxV3!H2RRVK{H4KRIr)$+#-mevJACQW_@oMHziG~{|NWvFT3Z(uZFYi#5+
zU{pK6GDBs~1K}Dz=93OV4NMytbQrh~I8AwF6gFv<f>xmRA?=x~Ppo5@-N@^r+{?B>
zlh<<QboKjp`Zvk1-!(ti{#l*QQN?9Rt&3V0?R63?mpv)@d5ZKiUH=A`Z3&%UK<lgz
z25ksQ(gbyG_M9zRE@SY1d4lT($UckvPvUoIMT9N_?W_>p!4;#<0AGX@D9>cx`2B8e
z@VObw=NS51CeJH6wr4)m_c|Gs{O2bp8UFCvpm;+hhs$8<ttl^ZO{||9RGtd`G-dLW
zzf%@0J)_eZ%`k0tXIa(dfc&UezS4U$mvAs`m+uuealIhsc-6hII-Kc9z~l7`&NWx{
zWd#XLUB#DWx#HP^pm@htwy6?l?Uj0^?zAo9IuWwG<F-kNe5aMhD+Mi6nUEJ|Z$mCH
z_&w37TVmJD#hYca;?e@M$yqh3UuT3Q1T_XJDk}T12FzJEF>zAxZqN5WUax$J(fUG)
zLZs;b6ECpgZ~U9bKiBVFzxV%+fBx-96H9N%l(ipbI`u`odqY&h)&p6MSDEIz1%L4R
zb6F*isYpnN^;}=kl=xEtb(;3~4>UC}?Q76y&|ZI0>j&39tD{_tnpP~i(Q&KemdC1z
zA;udRa~Px#_yx(<u-`vep1xIiLGydIb+1o_#e2^By|3@dKMqEACO2!lnm1mKj7i6K
z-IMq=_m+8qQo2$)(;No%1DpknF>HDb+z*ugy8hxe1i|c`vj1~;AHwb-Py(*!*_h+&
z6CHoMaS>0{#=n~rwWqDk%nqBqIqBrAqtjX<Lh7cyRWMlN^CnXG@Tns;rW;q+sESSP
z)O>pG+P3JsYKwF2uhq5qcx-1)N}4d+=!EagN6UN)gO}~NRs6F~tTy<|+0`%JomZOa
z`atRYy4Dp!fvUT;&aHm+e3kqx71O_WR93v6y)J*3%bs0}`2HG9d2yon{t7djWn~u2
z%q-NbrGllUey^Or{L-8*6VXi1JBxU{1I0rZEz)<5y>kAy?*tZ+Wb5{;$2}jcQqh{a
zbM7In1%4~9TrXo^-OsS{$%1Lhcf}qE*H|505}MJn;9qLW>`y;dEfAWZr2DZeiN!N?
z(y9lU(vlC<rft_0U(XcQU=nsA>h{jnObsTp*AxZ&#j?Jsd6OnA`Cg>vgEa5<zw6&6
z>|;ziFJqE;p>5M~-#3N9^TI?GdfjSS&#v0c&{>+7G<WtKMvpbclZ_cVC%(JavnX<M
z+SIjqNvmgnn!l={&%xdE`;;Z6ldt(knPy%sxn6j`@VO^*roZG5^}lv6zkBZY|7Baf
z@bc|qX%A)f-Z>_lcGYL~hcdpre8p3G|0Fj*Da)4~p3&;Lb4yMyIl5WWLwUJ$<m{H;
z3uo7@jxm3K{8s<<XLS?JG%u;Sy8WyFU3RTLeeb{Pq8qla+yDFZy`A?!U3729#}kg<
zJN}O^u3EXeUWa8W1E@U%#_Rj$^v!{^)j^~kkBguCIneTI7|vEmS4dY#TXTM<XkSu_
z>F=Z7jOs;dMQTY9D3&x$J6#b3(-fg_XVJG68~@+<cZ~CLVqt`=S^M3G7EH_bJ3Nw$
z=Fm0J0mGQc>H`nEA6{TO%ygOQa?_=zO;HY3*_n<Ic(kdjIP4Nrm#SF$+K7aW2a+0(
zGIg_F+s+!|z2;mJD6^$i&v=)(ubuNhSFx?nH-nppbNDaYp80O@aPizQgXf2r@K3fo
z^UL7v;XVAq_P;V&i|u{>8T>r_hX1ntng0ds9N!zlcks7ZJMIu}sdn5U-SXRUhjL5x
zA?-Wjhvq8Ykvz0l@s8}FzlwJhljO8^`?pxT?g(zFcHI%(^4oPs@)4`4FZKz(lX<ys
z;yY;svA*QFM|oe#eUI<GaDsR4iih3LE->9Szf`LXB9i&<aule0mQN~C_xwJwL>@+j
z@0z^C+I?64lHYD{;+Xbb%S&^W@2X#_Re}?broI!I{~u%&*gP=t_~^m|3@g$vZhm$1
z%NJg`s(s6tg18npvVUxn$ehVhtC}R->3PK^O{(!i(*-BKxK@j!h5u^VKN4B`feH(x
z!Vtv$zm6$q^994b5;amYa`RU&_-45Lj+oW`#+;M;oO6zTVU4)irODI&a)ZShzS*oN
z6c#6GFN4N3(`Ba5PZV?wXFGHIfT509#367*Z$w1&v{c5cLaR4N=)z0Z*A`4V7q{s@
z{5Ik33@7d!u^T!Y=6dB!yD{fzTH{%!-K=HYZ^Ul+tCo4cIhVBkKwjfrCf|t$?n!gJ
zOng&pmR(?KXXOJOFRD{8$7@Uf`wy9nH~TcV9LQ|E%(R^KTySUmtjm!xVmovSA|Grz
zklOe<<k6-hPj&RTW43uz-QM&(_xy?XB`>_De4Meb;CfYg`{9Qr8IA9>U+(+lRX(jG
z-4jfFpIQ>{2_hzzm)AZ#H|d?u%X5=Jiok@D{`;DTI=eP6vG(4ze#vhy5OGTTZseu8
zs&^wd$!yl+2J2q4TnH)#k_fjeOOmf%9o+d&=Vl+{kDv05KPH4PHkZ8gE<vMW<<Z$X
znbDU&avv&~!PkBGl|Z6?<fT{L4-2fe9M}Mc8e#&|B+i14KWzgaftnWK5S3)ro!JP3
zJ9QRBBo&Hvvx;%+h=I;^O^|eNP!&tx2&>RmvtA4Cgha&!wnmNPX-{vgk(oU;`LMrr
zyhZm}rrE4!pvchOFyHIVv>S7ef&v~M8GPO{=N@f1kl1(_68y`B+Ew||FPh6Q02#5l
zC?~A%?5T|pE$8XaISPsi=)tajs(Rif?wsGJ6ohk@Pc49iY2{LDZ%E*6h6bM3uJwCB
zVFU@mOSP(Z!@)!xDAd70uk&)>q<7los*Mft`+lS!CMUK3u-8#|vNeCVmQK;9$R|%W
z=WOP$y>Ncz{O&r9qF3QxUVk%>c=F`4bU@3xt`O0ckE8<vx3?eMRD0p=@*{g!G59v9
zIZR%CKn=XKcFMUjqwlgMu5W+seBG{B>Aathk*#6jlVx=?n0#F4dR2N}2+F#>h~;?e
z)c&~*Rh9a8BtOm8)_!|;1H+Pi+alYUKD=D<V5L>%6P-<$1vUsQzvZiRbK#|?sLA~2
zlzwxwhQtRHJ6>hoDi#tEkbHYX5_?EY$@Gc4K(dDoSvw?Ck`;Py@r0-IC{M7`5WT9M
zKbO@Uh90xmvUq&W?`CIRvcn@wu=t>+<qMrnnF7Yvj7ydj3dK+E-0WHTNN0xBoBj!w
znc9_ibt-P_RNQ`2q*&>^nqlSD9J37}SH7qDzjMFczfs=WcIJ$i+gB~GyPLFnmrtnq
z%2#Tkcb`ZHs4jo?Tust<;;aL{_seVVyWXnhz3@*#a_;=;MGHNgC$`neE-?_v)VLI~
zWT}syYphqU=L?}XTU(-;RwyU<ty&;h!sPWYj7h(go5x4}+fw1rL97as^e5}DoOOL$
z>8u;|erN2|kIfU7KHu5-{K%o=qz9S_s#&UjPu;Tu{W7%A9KTnTl&6`X-XHSDOnugj
zFh3q;^=}h|jZbu5T--2ug@(XKNpE*nm!)ORTD)4q@tZ=AnOSpiIITUQnltt7e5Q=Y
zr#!XP6$64~Jxfna&z^d6UX#KKhLxh0g=;sR3vQUfHDQ&P#P@r42JxS|W4<kZ=RSG%
zlfN8)_eF0#*}mH6<QGT5W)F)?AxlD*=<K-Wvg63xNv2;rw(Z^e_KV7Hjxd%RynlH$
zx<bAdUp+r{k7vN;@4hchYX4_FiPAJVnIUv>ir@5(br!LOPn>qG{A^;`pC@HBYga(9
zcvg|(-7C%e7>)a$XS`(6*f77#O0-n#)grg)kkxuiH%vJsl%Tt6MplsdVXwlZ9*d2x
zb7m-MEx6V^!!c3mu8qsmU03%S2+X;`?ziTjnp-_nn7~#0GNy*9X991(isM~>PWpyN
zb5z*=LjnGbPS%{kZ(pVIKA$dpc~58RJwCRIS;`swymmL9aC-b^;T74^>n3~Vo!YfB
zt`4W!dz#uVFf=R9Wm<GeW#gHoj6Y@5D-J&~52<sGmvUe1mpmu4q9*UlYhIp5Esqth
z{%z8m_>QN*K67!n)Bl!#p3$ngQ?~?Oa(i{5m+!7kjH%T(K94T5OT2lh_OI%;{H^%<
zSLXM|-{s-||J7+ly*U2h`pOrL?J=&tuD<a+%a^AtOW;&seei9^A`jP<;-0!)A*$=V
z<{Ue`F7Dp;<&XY`Tv@fkcjBzN+$jq*gD#(sowQ4-l=GF>TJ5a&B9Wi07*g{&%s}f_
zmm09t@h)s%<mWmU60RXK|MpH((shmcRq1&tXvtC!@0}6dS+&7$->WS6b-!Dz@UVbt
zhfs>|-~E^W?_M)OlwrxtrZO%mtqf(GW922z=}EUF8+Y?1Xc&kyh%s|JNEu`}{_KCR
zF|{w~rN-oQFYXvhUvn1_bzyegopt8jnOJMBWzk1V+&<|kC$e^Ab}W3@JdJ<Wyv|(r
z$0adm%a`qx-Fx|>HJF&YS?HIC-GW9Dzv-MCm_BS!TEJ@e$~tSs;x?<zEq!LMESA6b
zsj{Bjns@igMPFGcF_WuYO+?mLbwRaCtdN{4%cRG@`X2x4yZ7qF+PjALC-*-%94lq>
z*IqHwX&n#q-QS6RPBUAKI8*JmyDGKGt)I;P%j3||bK5$-+WjpimHb?DO>u*A;_>SK
zEsFeie=lq-e_>twldE>Gw}FZ2&pNGhdPgUiNnc8Ho;>y7lv<D3yqTL7bInZBB`;+<
zTjgeawZ4$)Ts-aL(bgHR4N4AyR~D^seVL(Yv*dDH-gb#uJ9*B=v~9V0VNJcqa^8|#
zP?P>E+L~=!|K;zLH-3_Bi}+4TcUj%qd8^5F{j&Obi))yd1<jabsG9up{nGtQ``NlO
z3=ZBmmA<k!`$p&+tFo`V>Q-Jpaal!S#pwk`%Cb|pEeKkg%69oL?-Hrzswr`jI=$@P
zj~k~+X3T6XyXmmXaz*;e>5WyAE5O9m#wbaPuh-l^+`aP98{9Q*TPoHjewew{AVQx<
z<Y2;1g8&w0SAzhS=H$R%90e1Pfp?;DKQJnod349>dgtUQzu;+&Y!Vp_2V-1Y*d5zF
zr3_xGdM@`U=laer^=|4rwk<K2Qas%QuFGHN-B_Zy>xg#dwIz|0pIzCKX0TRLqp49z
z!YW23fi=)`Vwuk^!`apvTX&sY!#j66pQDlJeg2&v?_Zf4wcRJPRdcPucZG%gU6K`l
z!moT0-F}C0!SbYiL3h<yJs7z%wSy*JS!Cf_c;)wUHOt`1b2NPowhEeUoiuaSOP{kP
zYSM|pT1i=ljF)OG3pp~SYev*VpS1=17=1ZS5AI5c-Jll{o)!B5v`8iPg&uf$Hz?N}
zG+~_;#qR6dvP{kCbIjb}bGzKRs+rfsh?Hu+n)F8XfKUSG1_m8QG2Lfsi_hLOxwTv1
zLPh@ed$XTaHy3ET3o!6B<bA#TChxlh%NBc<4wFa$W??3+4I9lbrV6W?%{25~<~3RM
zn#*+GlAB+S-Ah?zGE>Vp<d(})81W_4da~qY7=PPOwOu)jPA@CBRNK7h<eH34QnS|_
zQ{&1rGM{y(SMB)XZ|*MyJ}4@<9O!5eyi_;Y^Wu|;DRz@WEtFy{F18BCTFjL84SDpX
z?}3ZDU&%ukaR{O23R38^!fqGyZuZNq_Z=O=+Z;F><2ajzm@>mmE>4(y^MZ?`U*6Z5
zFHZ8dIC3@$F{Mg){Bmz?Xm?n@#b~2hqeEwyXSS$_m(y~WjeG2_=Qg*;cJ`fo6z%1~
zC6`(nmKd-o9F!xyIHm}AT>nv!7wquk%RO0(Z_d?EZe4%z>dNnh)qy2j&tJNe?YUQP
z<@Kd6uU#t1_S~(P86F#UpjNBu)$=R-u1!0F_GtJ8&Z%|%{ZBzMbmFSI6~C6foW1PJ
z?1Wp)mqJZ~%`f=GIvAJ!KXc`)aq#!EKC#A|8J&&V>LgBmPYC<+ZEC`sfb&o5Z(WMK
zl)CGG>Z@PVSKf|amRCD-*V|O_Uw_L?1I^z%ImqsnW-R@>Y}36f*KU8U)|w$zCd!@^
zb9G&+R>Uu_@GS!Ev#u@au8vZ>zU56r)is@N?J$-lla@~25uRGK_SBQGsV7ynxI8vm
zx@yIp>i8X`c73Ujp6ARv6ODE%r4~7#`r@iFxjI1Y`W&A;%b9mFjdos2DVlupiIOm*
zdQrxr1-(n6cI-|mN<I1H*rb!Ts!W&493Qi;6*rk5!WBNnM^A0$9dD&?`<nYdOlR|v
zGRQozq;S$PPpRiqxA<MMdKvd1Z$ihy-bu$!J`&;9ycFiS+{4QCQc+~jhfr6xu!b$k
zVc)GkY+1Z4{0{S?1g0has%K|?S+%6_{w~%`d#C!k%b$MlEN}YGzBh{FZB5<x|4XB7
z%4!0-W-;aKE)ZT<rS;0(czWLZh+TV<?M0V+yjZOfG;z-&kI;#$o`kxt?TS+U63V(_
zQWR_UDTm-I%RGDwXFYlodF7wN&Dg-*Ddmj27-Adv7I0r+1)b5?wuleBHvhA!NU7F+
z<8!XfRR^Dh8|BDk#_eI8d~$MvcLO9}O}QnMbAHjcc30+$PVW^8R$p|>mwM@I75)5$
z@`1}+7?>B#zLD#ryuWD~?=7>8t80ryCa|5|9opBFdI&VIn0eum<K*{0Co_FG8rjOk
z$j&76CUXkYk&w-ThR>WGS{t~dOFLITSa>Wfh3mlKzVHM64f2eR+mwPC1eURc+$`2+
z++EGY+F;6<&2)=p4`;{b(0L8bbDWnncrs39`o(gFbA|Bb)H&};`@Dias4D~<SkPdW
zc#q}C&C43BF=cFxLeJj(oXqs=jda4H1vR@DGq+fBrV6dgab3}{Z^PGz|0=dHWk%>m
zC_8{Ep->a)OUJz&LT*f)pxmvWt^EJ07JHP+7LSaHD>_?bI4y;WRGwV;H`#7d!RA*^
zJY6n#jY~}L)a^;}4fwwJZM#`>`!t6xkCLPJe~12V7Etuy5N2u(VDu?J|Mu^yxqK@*
zConBx&~)IMz|h6u+OX(KSCBT7cw_g0R0e@vkY%Bb+O5JnxO8HrFKtzfm9CzhEgrW)
z&0(>X$mgpfTQ#$SudFNC#Sr_l<e-&i$wP2a2`e7=woQ_D;(rzYK=DJwz6V!lf4SeF
zdf-yK>)QR9Q_R<X`Zm4$SoP(qElE3`-pR<*m(t1Du;Azt<>t~Qc1;UzHZ9|dQp;#x
z@TvJ$J*%mFNNvDZ$F<&bS*^4#STr8H>5!$fV%dUuO;uc4S}PVWn3QC-`pSeYUJF8*
zY=veyYBn_G@A%1QZ2B{TLxz1`&|J?_wY>^2*>+9#m3ItS?HTW(Wxn!GN&W(}t<GC_
zXjK-vZ({nzwZ>n|{9&zA&ECSx?)TrnxaYbqp(f~~=h1Xifotk2{t@d6t-ddQ18(>0
zzir4l_af@)-;kczxVvlj%s+DRm(%AZA?*I8xv?AflpOt3H>vHB$kD+2zF%*JtiLLH
z`u9puez>&q$qKFuEE!BDMr(CU8FwuzJ9=mJ(xtDbeGi`0RyWTp9kgzCu2KKyw}+p&
zPTlqCk8pzKTvx7HJ5~PnXHOM1VbXeO%dujO`Bu?&!Qtr*1zXuT!u7wd-ZxWcn%H+u
zZH5^P!YrY3N6&^ahzVTTr>5z7{At+Jqqe8ItEY&IYjAsf&6b-KrK-tr<Hx38-nlK+
ztT#MYO}3n+mN9!l?v<50-75?GH#Ia{OtvtYp2l})&Ym5Xm4(HdG+vxqxbT~nCu4`}
zm8UCC6<#WM^?#p3Wntr|M@&zTvYzVZo+9eQ^dV8<?TnJXC6<$)`OGrTJibKs^4k?J
zSwPP6U#)g_X363umY3hoFj;Vgv0PREombqQscRS(Zdts=antclJG3hc!#_P*_vGp8
z#GN|>7=Jtyd70qJe|e7GH|Ms<8{I4N-yFY}8oA4%@+H&Lzf)EwX(YWc{rqgZk2&ju
zq$tKs1)gi?F)d+Oxx$mnHLyEHoDr0qm2QUaT+#J;6{x%VY4elNli6LjRJLe7yEiRz
z(}S2z5AOIMoR#3aK`(;+0_zK=5^zDJcR(Ov!cxUM0ZSHoH2;-!optJr&+Ll4hP4lF
zReG;4=f8I$u`{%<E9lJipesvPcn6BFRcZJlc5E@%rCj&mz1obuU)c(!rkRy4X6tN?
zzRv4?noU>xXzSILOdqzG6m$3ZiyvFgCE&c8{j8ixwMhEZY5cS7GOBeCaP3;V;5OH;
zwHy9vMLg176DA(EntS!t)~l}$WtApw-L)Z#wL|8Ikbv9k4U5c6xTUYP^1eRQ7V5wu
zY~W_KX#=13n!{XDn=|^O9oacLnZ#I4xGI)Bl?YqUy*ivVudi{=!+UKK6=DLq2~iGP
zXRHg&5c{B`5am-I{=q<R!JKJXN4vHg9M^gguahi3Z6ntK=L0Pbx0YTyuHwMxaG<5(
z(*B6ES`OSQH9jW3PN!R@f%p1(sVgq$$q{GU#^n(4{J@?q`x;|7SI?*ry_VH_GSqRl
z*p(IQuRnVf&Klz9G^s`Q7VkvU*8%Z&`!7u2tua-0)vC&{seNl^C5CDqU*#Ekb;&9(
z_Bv)RdyasVo#83RgQqyCIx=d3mk0=XzBF;o?YbqhRdbn0(o~@-3*Ud={QCL2`Mc}h
zdhOKiON%r568q`bu9SDX=ZnMywlC`AZ0OAduM>{!bDb2o@RL_6^No{nZ*-S#y*H`$
z0|+H<(E9GXcVBbAr-SMKmCwtSU$8VMNOjD)?A5_}WXDz}7G2fYaKqTB8Ql7Nj_R#x
z6AwGgUHq_h=LV^}kxO?reVb%=$gkqz%F>7POA{sb81RTq5aXQXGK<-NHMjd}Zu`|-
z6WqEN7;R9E+YuH!L!`TfA+vRX<e|3$CM_b2^8&^D0>$eB#o3N=9njZMs%`MT;+rR>
znK^Yy=%g^uDAi2So6K+fO&l&Uyn1~@Q(J=Nh)<SjrkL+eEnE5MXRW%Q4^4|?{4s}Z
z7xN@{a6Nxq_3ex;i_=nz61gU%fk)k!h<o)XyxU_?E;hk$hDqfm-qb$r+j|sey?4)W
zRGO>!``O=+DHoz&Us;!}=I8bG%KG(B(obr1g{Um@Z~^s3Lic;6o4o8<78*JE+=SH6
zb#3#!mTN96al5SI6?V>h+F9>uXMe9;5gWMsQO4sXo{c8Hb7rae`3kGwwAH_9`(9m>
zXQ@Gp+?DPF&JPZHud`5Rld@{I&Cu=HJ1N^e_k(}awM9|)yHsNvFMd9BqmuojnnKs>
zd8g_<<xSF@dNl=(o#ReUXWbGkw7ErE$WnBrs7vW*rZoyH{x4ws;+6eyWy!+@C5dnL
z6!`DjP#Gs-t=}`3^-UVDLzH4h+XB|6Q~X)>D?Ww?Ja=5mzEv({cL-b9Id1Let=@r*
z%O$xceB+Q($q>{O$eAY|c24Q7wAKEH#%q<{UYNmQvV|$^9CvjZWAa7*JF*><k1Uj(
zROYeE<&yf1Lq~5#G6r%NzL4~kR?VIAMPMfLOXVPKj$2wU)t%W?mG0E-ajLGe{%@2K
zXOwZz|L>exXO_2ZyR!b4?%P>hC&DW2kKSK2fAzkVzm_qr@LKB?2Rf%@N}BKj&I=4C
zsS7x-fp#T^Upe<($Xw-}DQHkApsyX&!(-xMT$Z#i?PH|eZK=~8T;81d1zB5PecZEo
z-OK0lUq3H<bni;c9!@UD(z`(m*9ERJS>e9c*g!LGhKRNX!^#^A(wbLYJ;fCLNH#=h
z-I9p8N~Y`2bqRhvR8^5^yJv%3+zf90H!ZA77A>C5vE=^5YZKk(vQL#etjRaIO@3v<
z*8}I+G}-oXGv9AbUEjcMd8I7i?yJU#Z+be_;?wqWF|4vrI=$QAQLMyAy`Bnjw>?}T
zJC+p2ab6eRq~5zkQRa0l>jk^cNxTZa%g)`BoWJo+Y{dIj-3qhAPG%{*olw%a1QfQn
z6kjrU@}D@nZu+%zrS`Y#dj8%>oxuCPPAjUTYyE@gE0_0Z2L%RC4BE8mkz!1Phytqz
zs7w#+UL<yO<*$FCQ!}RSSv7yPN@(f=q0mRGj=a4sQFf0tyUFxq-W1OVY6a2^EqP9$
z?JBVk^diL91i#RW5nsa%8S5}xVG|@VW3I-fDOXl{Y@RonZPS7+Q|~5Rd6xRoBHxB3
znrTg%>4vudd#^2;nX%lQ_v^By_j(ry?{M_=`244lO_Jx01iyYyyJ*Amqslx>=9<2h
z=lR^M+jr1SSL`Tv_oG%<Lx&SikJ;C*UwI&5dGdk|Hfh-iS?-7l%+^c#*;cdn^oxfb
z=U)A|_3Flk0M!Rx4wD<y82y<RvG{PVShSg!TRDYw2}dB)GM13ecYc8kPO}<J8FQH?
zvFLDSD7m)n+|+P5@Z>86*B8Pc+`af7r?Q@VrJ}h*<nmo1n;fnV5d+U%er2Z^5`!Jo
z*9IHJ#YO1qpP4HD?38!esk!|n89$~7Xeb6aEogDboI3GE50A3P;}yIg!sM^XUwAq7
zgXY8FsL2si9hSGaN3N;UNOoG%;=$Q3B&TX~=?~lL=dD*aH>{Yu=2~(`qaWv7p}Uhe
zXeEkfOXe&TZD<dBzgJuG_rh?kTT5!L{>}MqxFvI?Mb)xL%j<5Z&b4~9q4C#31s8kq
zzcF93udX+pu+C+bQmATx=vJ+);48~4d?jzX%S_{$@jW<j_M$S^x4*W&{=M$~)}M;i
zPgI4Us0yddpEu>^Mt*~Y*%qZ--P48dExH<1kojuY4EB|A*T2}uS^JhqPdmo_`C;qN
z4NZOq93Gl099L@Sw2F%z;#!gUDkv&rQQflVjuBk?U@>m)hpnw24mnjMD(WP#dZ@CS
zN?*wgSn0UdD>31rcEWRaEz=btnOr{A;>-4OUyfs4C-cf?MafIHo=Wj!d$=3|K02P=
z-?WQ=RqSc*H@Tt?4zD6&g{vbA4YcJM51m<ZY?74sD~*+NTI$v>2@`skYP?tV&tCC`
zx7J^Gn|5u<`hD{k&fiuxQSo)1R-A^PK)Tkyyl+eMHh<|#&6HAF9Cmlxg5`_E7+f1x
ztq7gCN-5NHZC8lsa?NwAHJN>bq%B1ka9$AEs{MDKh-2VxP*6>>Qpwl2bYcBuk!Aa4
z9ew`C((PEFP|u2u)klOf5<j2vw%ayq<{CYn7}0>@{8jH_SFYUtn5W58R`}{xnW+s`
z%N9kx(!chj+md118m^96C;X1bFmMNU&z@kW!;sm&K(Wc}!h*d9o8xb6S#`DZ@)^bt
zCQcW)UZr;{J9BeiZ_QlIBrtb{NpqFhwlz$A4O=`eC>_lFut2s+jYG?RMeQz;wAI|3
zueR=ebttYhas4U<PL>O-9=|+fZYgHAx4iSTR(U(Yg!%UG?u8|WQ}ljmZ(GCt`fBUf
zRZO#-z5g9uu~WiyV_YahliwthzFk|^hKr}I=B~clx^NZKBD0jsA)fOe?tS&{@$4CI
zg3E1Z?kQTwa*NGvQuTa)`A%6Eo}@GZ6-L9!Gj7jsw43Chw6#Iwh^C9Ci{^=YT#h%D
znMGYn=XFMPMxAtH-7sCwI!8gO!!4+HiPR6aeUdjA%T8U4HFSCFBI_aDA^B(9+-EP8
ze@d8Ty5CZf7Jk5D!8_-1#Qu7n!21yai46sD(>4VjT@-%cY3i{(C8vLXtGDoJ&)d5F
zV)XS7tUn60^Vd3OJjncd`OQTgg<bnMEU;*rBRpB*Op^h(l0V<2111-aa2PB5^crX$
zT62S&@AE~5&&oa%Y&jKdTDB?_tG*IUsC(nezEF9VQet0=r%;;bm&?k#SZC;0i#cX6
zruogV`aPGU_Bn@ySb|w=rpRNLlCHE%*Xml9i#~hV)|bK5Dk<{JZ-!0cw<A`i36GX^
zPFC}qWf83|6xM!tVYSaJgUmxqI47(5%##0kQN+(6mi^~NS+-dQl~>v-Cp~kYB_Hjr
zv_K-`(2}X1vn(<}+MoGeo6Ib%?mK6VdwpcWR)^PfOeTI3&|F)7sY=~<jmgGKQX0M+
zY*>QrYnL+VD1A`9@+GBuVs6VzJzgaTwi66e3Cslzd=ECx5n&RlY4ZQT30n6vhfQfg
zgZ_i=V!nM7>KuI>&#YEh(6Ii2_7ARoOlufc2d`0(>R?+WxLk1g=ji9#`(}ifzn$3V
zXKB^JnA9Mlz{XHgvEVTCQ{C6$yDfiPRCb+zn4exMSSnbm`0fqgZbmz=pMT=-OvyLD
zef#aVvcr2d>vx=eyya(R?)J~xe!}gRkIedPR1CfEoY8o)!HMsl$mdqpr-Dypc3eEX
zR_^JAcOnOM<@1V7N+W{XMT;X0+eMATc5<vxUld%jc=3$GU6yM)oV(2|>V7$wAIUqJ
z_orZU`|aBOu1Z@R%ztVh>wF=z<7=|j>t&8J*b|kG*tpJEzF2YxkBMfI<O5fSS^Pik
zydJzgmm&A0+REg>+xhRlWXdhE{-u4k*@XX6t)2h;|0Px27I*)BQ1G8#{(Jds)4Pi=
zpPin!uVnw(DwZ9Rm(R|4x8-G;3-2$f%inkQ7%gGj-nq7EnfmNy!T!^{9oHO4(y4g<
ziA#yiFm#j4eAUb6UFNI)(z%$v@_V?oj*G5~u1cJP$f7WzbuVL@Rw!r%Y6nU?Nq4ku
zdA7#P=z`4DpwdNUEoCib0(T?dy2x_KE3n;^WMDtwD0KJ$XHwIQ8tc8ko}6x-k!o#o
zWY0Ni-Rt|MAG{T-*v)zGY+|{^9kur__5Dsu3S3>-xcuX#&sR(Y5AjOmIhuq<IW83q
z0TIDbicbSQGFzvFd1SV>oZRTNJ$FapVmZdVdj-FZN`3rgFL!CJT@b{b;B-A<$H&8K
z<@3H3>|VTcLzLQ<1$9LUdInQ{vbS8BHpS_(;+b{>#UvTV!v_)<oD@uBpQX3<;Uy`(
zwGV}=ZT4+wsQfjHb%tl6R_jiY*Bl0OXH+kHKJSK$aMT3OX>se9bf!(6F|T>q^97lr
zoxLUte{buYt~T$@moKeT*>@SQwZ2>&lCz9c_rk9yQ{5NKOf4$ulAP(jQ|9d?cfq+b
zGiLdOUA8h^8+^I=$`=_^TX)~{fuGO#UiLDxW%oUAdpY>rmUX_bZ~RGW`?zSgnC(XE
z!;6-@wtR8s-Q_?3xRP4x=dgG?3Ym7!I&$XSmJY46Q8T2)L~<MtLWpTUkDmV1v`BZw
zl_oC;q4SV?*4B*j82O2p1ui?5TzbFcer!(5q9E>FTNc=IUp~+zq<iM~spG4c2iv@>
zoA7w?t8S6=ks3iUE|I6V{!+WN)bafPb<ekJTgmAEsoqicc#DgbQ<cz5GsjFZ?bAuB
zkv7%ZIX^$PD(~_=w6?KGC(ch?`*_k*4doQ()xibc3uU^eCVzahHbPTOe|c`t8vQ#i
z8v8%BuG79&c+_fL#B$N|*WP$d*PnA~i-I}-s#Ou8qN`UWg@$f0YdGh3^LhFqx0x%J
zC0z|!Gpj3fvTxhZL$(Lcc4TRV`5s-hGUBpG_N=COo_!X!rv&}dPd%I<vt+jZ>6aJZ
zMZA3G{e5LgzL$COwaejut8MRIIXm^MONrl|oR|BKtvNgW%=@Iew@bTByr#OXjGwWg
z%(i*`q4^cYQ}UY`)Ekb>(d}Ht6t0#g`Zh2!A@6gNIy+=lVUoh_msd5zo(m+fMlgmo
zh$gV!@OXKZ!K|UzqSs>XeYTm>9I_tOypMdI_)MDh$3if9zF4=-uM1CJ%iMGNv60)&
zVzTy`Moz)TSu5tOdD30-qO9Z9g{G!CEwfebsOW!j-}-OGypqt@6JPz^_3C8!*0`0|
zw2dUUGi>*p;>^K(iqYbAp3;|Yo<7|Jy`p(*3yNE%weAGHnEvYIb+HF#+~!hT%d0+x
z?#)><uO-y~(5kNiw!bT`ZeE|&8e;o%RhY-s$?INuhHq_JCFA;>d%4M+OC|@_Bou}2
ztyvQmBf8lIl-^k^Qj67mW*I~$$ugd1n#TI$Zq5(W;3{E>qoHdztP%}<+F=zxXI=IJ
z&S`29vm!E+mUe!1S{%CGeD(F&S6`pa@(o}6a@BLctFz8cJNN6*887WdlY=*&ufA4u
zb=LYSzEDe*gW_T-NlT|X&PvEkS}N*zZF2K75#JR#?uipGoxA=`VE262^Q)e()|RO9
zv%N0<TA6A6G%3c9agKAgglub=w4(EeOC6KXq)Jt$NmGOjSq)he7$X?OmPa^<2#C6<
zKJ+?t^(j;Q+$2{A)*B3F4ZJSQ7d<vg-tgGS_=dsSvs6IVMZME5cCi9mhhq@)1BIiE
zbq(@<g^*=iN@)<-RPW2;)9}rDi^H7etuC=D8|Q>BD@*@=&UoVYi}&Att1q{Z)#K)1
z=J!8zz<Gwjg)@m-H*CIUrysntX7lXQvd!o6;xx?lr_5g7aCX(sx#5~;ckQe^cYTjS
zUf9{Qt9C}3n*0AvVa!|LDVXLuLu|!^G|@Y2nr4R<SZnR5RJ^mMYj$YiX05z=%4Vyd
zq)pZ7E@1ZUmRP=BGQq7yQ&7$MOmOG7JMwBSGXxUdS~MHA^1_auEk03pZTDHLot1lU
zubDJEv}m(eUf9XAuVZJ<*nNGr!HOqop?CHf^{v{lj^ns_yv@#>xZ`H8pFR7UvpEau
zBvo^tn4gCZEKc@Wuw|O3Fz3}fYi3D{#nn8VBf@QR`I3vbu<qL0O>&HBQL{2Lmo1&y
zwk~hpR<rX_v(wl1GUwMcE-?S==Wn*UYu=f)FQ#6)ANy;!cYXQG_ve-+yVsX5+cCGb
z=)2QS<Hv6^zMea>eRJHq7vK8w-oN<v?)yX8(t9)imu;zZpIyGCGF|ZQ#b>d0_r3^%
z<FsMwl2zQ9;5;VKYXWLvo&EZ#D^o$r)0EkBRr7n5tBl`*7lTebO=^%(F=gOuIDFvY
zHU<li!iM=NrA_wc=XUf6GYbD<+3)eO!iAxb@3!Sp)});e?|zy;{b}Cs?b82^HO*)7
z|E?E)kju7*zg6c&!OICRofjWo?7Zag5@*$7O`l~3y#c?9g7O&ELS_Ue>b0J1`S$+W
zzq`^5R_O_hYGE@X6SG=(9({4aw)nlA?3*of^m$AU&)8Ug{>JX~H+L61e}8e|d|BrA
zp1svOUdZX~YVUq0+x#wYQT3D;>ASa7&v+rHw(I_$^8FmI<n(rJ_uTE?UAN79cYF7{
znKj=xT(r$8IsamJd;hzK&VFZgFV^sUHuO0$!`(4rL9A4p=C351mva8QmjAe0EqwQ|
z*Y^#VY}b_VYyaJI>b+d}uH#eRY1V&_bNzm0fAx%)aZFz}v%H97D(Sbr``GXMj>~&R
zUdYw&IzGSd*!*`NmskIJDObPC+;-RTh3^ViRLi`w`2U1E{o{&Li?#bV+1$D$;vanH
z_u9M6!QT&D*_-x4Zu+j{OW(y#e)n-@aaK_LjMtg#HcxvYw_?|Ip}UVmzFS;9|IO>W
z!&TdWQvQ&;%%R`Ed3}#sT)pR2h3MV-H!qx-uU|M|y6=YWnkqBfK<o7j6JG6XkmR*F
z(7s)I;(;Hg68qU6Ur+cW)f3<RG25Vy_n7|7!q0-1oim{4HEMQLiarr}B=RVvZq1)C
z&&9jeoXb+}^^y{;F`Z`uYJWG@n_d??;C98YOV&j_OFd0p_1OA@MJWb#ylm?Zc9=?R
zWK+JHute$}o75T$HsxyxM;=LSD3-qR<4){jw>f#P=E^@F2Xg;ms`u-(Ud_#Tn)k;#
zzPo$o|G&E_-}s2g$5+V}2~zhQrQ{a0g)1I@eEGu<6B|3v2Gv=*iSt@!3NCY-A)Q!c
z!1vRE^T-y4$qKI&*Ayp7wM5(9`C|4&N-n<b`1FSlQz|SZ<^0;(CqKM;+~OPa;m0RG
ztV)K9Jxr+(HR@Bm{-N)}haW~Z`*_y7wY5)w_%OAi;^Iz0X5T&o>qB2UDt|2x-llyg
zVmZe%j~QZ#Q3f_V>#f?{+aF#`tSFI)*!!uqdK0K+zU%MhxAWXBBdmX<-u&QpeuI#F
z*uBH2sv|1-wcjT`we9;P2jNHZYrjrRwOzBgy>v@|=ZDtdZHGLvdrS&196h|NB0`*h
zP14UMubRubUwmvUgM+`$soFQ~w;1EZ<{OSj6w;Xd9t$NfM(E7{+vwE3){oC{>irEp
zvoiLa?5Q<Mo`2?wy7AY}L$fvZyuC2b#45--5Cqu+*)Fktjx-BZv&u7)p8sN7XWs1j
z-SUn;(=X1OB`z6Jv)iwC=em3A)|}5a2->W8@ZbFge_3e>wiffyLs@#$SeG!q6uHCn
zrqDloNoKD}?oE@uQ*-mb)I5|4Ssvs6+v@47&i%97IN#gM%8WAhJzufh^zxRpS=W3n
zOJ!Cyt)8s$?;x*m-pqo_2J&;~_4}Q7UtT`z`OV0A^TPc6wU;lSdihsUl)umG&P-dw
z^Xra$DJhJy_j&K<^Sb?I<fAR~X3YEJyXMi?)@bHd!?o(mz8>5nH{)K+jCcP&y~*o*
zC;IZ1|M%{awab4Wi0<*Ujdx|-oxA3g!WE;n937%A)veR><#an|iAl3`h))!Yt=j3W
ze=kmV*Ow&kvW*?_B_EDT)fuw?Vhd{tZ<xAb>WZm3A&bmf%ogpDDwKL8RcNJI_Nb)p
z^&hhzy`|rh<~_>(dhNNLZqS@{nnqiBVq9X6L@ux1rDQ*?d)>wJ)~g?``uipQzx>M!
z4F(5t1(ypRG8N5hDB!-Ze0sX{>CL@Q_wBv2eR}xe*L&FG-@Q3~`oUAb{d-=;zWaQ-
zwyx4_Q(xhQqU9io>iMT@Yb(Vz-FZI!_aCdxy-#l*ExI2!&qU|@+ozs~KHVzbxqiKI
zWW>8Crzf{QJ$dL;N@C@v4K|Sx^K^b>@WfkW=Upi=UN+k>_t@0;pLcgp$U9$BoHplP
z;)2<xch|gpcII+VW!2S}VR^?&J};fEe>dl4nD_QsYxhkLFFSoIv9i6evS`B^ooQ2#
zern2nsM@R_!(yJi>*l4i)7~ZJACtHJ9{u6{f)|lrQ<k?^nmS))N;tBv;qS%66838L
zYD$wdIwz<s(-Cd{Y;>@4=7jcg){d1fE1#?qO5H7e&3#eWBC{hVo9=bo^4Kx)!)n2;
z%3GUXaJ*;O&CR=E@}#(1Z@eHUCqDpD%<Km^3qTtP0++9zHluq+Re9YU=6&{kn<ngX
z(AnNL`*3<}-1OYidp&u7rhVPB{r8)@b35Cscd_h9U3yR?$|2%F=En^U9~PY!Nw~XZ
zzen)toC#^c8I8-_XUG@MP@KALzk%zLiy~G@g{RWIXXpj`%Z4n^c$Q(c_`-o#I<uV<
zg*+c0wRBvpKPOaJ>&yy+r6zlCF0>T=`-SVXrq2q4sU~j&Ux@fFGnjp-<k8vP`wT9G
za81^zxn6YU@_#>e8`ll9Ul~k2^ls`Q9j?n7XBL|@eyR;=<eUQP%!9h+%A%5X5{oZ9
z@fTL^4>FMdl+nIeU=7nHD}}pf&h#H*kA))<xzfin;o(WU?p}U7FYkNF?&Y_w@7{^h
zZ#}=&!@*jL$I-gVBm7m2_X`MeIsN@}soEzNm#}!Qv&YLAyhKH3^|kuE6;%tIyu$N}
zN-N7m1)+wt^n;Sq9`JY<RH$0)5t51HtgHx6NMMX$3|ka-X~mJuKi5u#IIl=cS1f<W
zHQ`o|Lk9belfLE~CcA;Qe|(UwVYl}vl(1mn6KG!1ykhbaHBaA$%Lj@jEj9f<Octr#
zd=9jrh`C?hNPxTLyneho*WEq&=5KcI(!Kxob(L|E*`jlw^z#1CmS~r}sB>UVhXup(
zIqd;#Gb9tUT;$AV$|jbzycB%qK0_g~twmF)T_&+Cqi)3o!PUyMG&C-6Fyhs|{9*U!
zzplkyYJoFcHLh+5;?)Kbw?kLWj|kY#uvzohOLwsZJ6C(AGpc)K17FOU5tCTfdUL7k
zibHvfha-(19tz<u);+Vy;PjzCT|Zx5W`8kfMt0(+)|+B!(KGTk_BEt3rp3*w%t+t!
zS~x0jMn&c$!^OPwcXQ6)4nct#jA;qS1QXPrUGz^=dUfY{{-Ni2B2k{QYH4yu(-zE}
zk+x{rzQ*=Cr{_m{zE}vzu^+0kU-P}TLA?fw+?jl?u5oWDPFb~V=B%`=St~2Q#BN*M
zHu1&HiSt&bgO|zQ(ZBDkUgrcxTj~@F-m}YdRIn;BNjCN{$bmMGRDP(c6qMoXO4;)u
z<N_aqSOaT<A-JogDYhy0fcOW|8rJ`-I9BLc{qQ;jJ7UA?sGN-YEyko-ulZ`2?=zS|
z_lhpLf3W<6Y>hmd(sJc=g)|u4!OPCX_5+du4{#PpS)4pj@>$v+x_{?1O9V&61MwfJ
z|74hEI_3wzc)Kh`;?(-n1=mcfX1mM3vb>i#`{l8R<+Tr|e421Y$Mp1}KU~JzKDuU#
zUyd%^8mz%KBR<iq)l#%QB+;t%r-)j}46np%Et$e^OJ)SKfcrw3ht_ajRy#A>pm32q
zBX~&cQ_D@^W&F)9RXHC0yjBTMrua-S5I^)*F7HRg^J?`cVb5C2e;f$-F$W#JXqd=)
zBv4dL<mCmXyZ>H(J8!|%*Y{IHlDWUi@|Wt|<`TX$>~EGHdR|xdqRel5aa3iBZPw|J
zckhJVwBr|npiI%ZZ_-?%jGv3YzAA34H6!i|s4=yBp7Z(4hUW?IWD=MH8n_r3mF5Ns
znf^5Vt1MmLYQcK*=uO5CS{(Wu`cGCWEQ1V?#kn&qT2R2vHT|;d!HtU^L|o)^Fj;=1
z<Ce_SB~q4oOKu!2|8Q5cjHM*#%{rGOB}Yn*@O)tYldX`@AkV<w;n*d7gvD^~BNjvO
zk@6OI3zoe2Z7_XyvSNB;JVXEcpoZgXuiu<g8W%WioA1v(+IIO#=k(sqTqe&NW%ob2
z@*O|hEKQBeNvTo)<g4qq>+D|D9N6^zr_Sd?LJ1H4#;@6K%C_^;a%-zss}`-={b<hI
zHD@wUN*q>K{G!}p_Ez5a>fV`_OX?<E$P_uZxIn2jSYwXULzzbhcGgXp_*r9)$3q{k
z<v|whi{#n-lrL;sdD@&;+4oECqCDYJX|)GQ57uxk54D(lXr+kX(gMBK$(rvDANa{~
z%k1K|mAlMCzb!j;FXZOzjGCmCuh^>}uy8LAv)FoQr|7v=1yL(EnTLK;%P~*fof(k1
z>K$)%@z&++=QbBCYBkpNwVr%#he?|SZ@2g17`vA~KIsW@>_sU{LiOf$YWgj8`0AOS
zWF`DLVVbca>-rKmruS^QqRm^_{m$BaDEn|$a%NJLmGb7f?gd`UjE&|_eA(lZo?@kZ
zS#svzF9%l6^gCOUw#-;~XF^H1#9dLha#7|${kf}NhFqNxIX}Z;tzC6(otx=qyYSk1
zo28{V`rol=dR4pol}!28x8p@eci%g|2RAMscsBWi=MR+}Av4a|Exw1AImXMmUli1o
zo#<o|Dc~#SGMV*aila%S;H7@P+V3|Pz6dDMVB0QLR$8!{asF)kdw(j5A4{8==j7d3
zP+eZ~zOUhhM@!!}-y@MFTi)~~ylDC9?rv0a<xQW<3lEk<7yMGc*KtirvQdB3qjTcE
zO7Y+K@#oa<KQUkS=38uCY)ehy^AGP<>zha4|5kZ-U&h({v7gJu-|a5Ae|7He757}f
z>a8kY{O5Ih@xt;Cp-s<XMcyx}5I>r2HIKWtC9l6`=|gMY`<iSs^;VVN4SUW0-X`V6
zZT@)s7x!Gh>+J#wwQp8Qd9tMAx51`#Kkk5kEc34N3t5;SIw^RL>%#8td$GnlR81nv
zi?;bBTUau@)bl8J+{JzF?~MJsU*sf2mN?$D*ksVB)1%zIE-$t)S;)fi66d7!x*6Au
zd}dF-p6b17(kn)LliPXLIr+CUcjs*9aq1J?W+ZZQzTBl#1}PGs4JIVtI65WqltJnY
zBOl3U6C6(&zJ2uFUeXK<x2ZL&9ZwUV%>#kEd45YC(L0_ezMBUIe@h-wJf0}t&I5w%
zJiaeN=gXJ5fFfC_xT#%#T~ZI5`l0y~VzOsiYe}%k9NHkdw`F<D2F@QUIYu&>Qa4m`
zif*|6P^me$yV1AVx7qh|X2R#p51Ah_H)L*jSn}=XfghXYc1AlsJ6WU8Qo%Rt_vEQN
z!o?n?#ZKcA%Msa}lX@{L^+Hza`K;7)S*dK*FJ9?Q&Hu4jpYz$WUr*kt#3~rfntk>C
z`D<@~b{-J?llT0w^!GLFeBG~9c7_U+y?%Q3lGldqLf_VQ*>9b+%s=tl^W-b%BRA@X
zuGu)1Bk!xg+AEFh*W8!(y{NgTu3M}BeQMa#x&MCe>CSx@-0|q(<k!`Unsu_p60?>{
zZ&{^xRqNI6sVe{SdGeS1uZdaj&igq%q;K=BZYkEP@Ag4g4F4^@)%W|J?d!LpVGAEw
z%<E^LCiQct^u4LP`DfaS(`LKuw~fy)iue0x@w%VcebMumH#<$tl-4R9oVWg~n{@Fe
zLv`!oO~xR?8bp9u#_HBNRyk}rZQP{?(hsEX2un0h0K=FK<~4ag?rhkZSbX?d+1bO-
z$`0Q9u=AmDA{@qU2>#G;zU6Go*(NY7DiEs?s}akYrob{g^;OvEJZlSSJ83&<v*qi$
zmnxl^pf{uT;+rMrBIoyMrnqeJ_W0fQXzqpf=B()(%52j1{ix^@?mO$*#;(kI^48?f
z&2>B3)mcvoXe#qRIu)>CO;XpN1o5M%1q_^z@F*G|IU``A-OFIVC4c%|UD>UBL+e*3
z8XidzwMl6F`yf-U<49M6-HOT9kK-LG|IF1m#(z=%V1!++@_~1Zi+B6|Ty-#c{p3a4
z!mAE@#RS9~dY66^s=aVv+2Lh}m*sP?*RW*@vFEUXA(NS*BF_(&yPh{-#RwmVu$^bQ
zklBgF&qG6)XS2<|rjYX?^YA4*mAC5b)-yZ(FWuYyEMGA8=h=_N*C$A`{W3A`Pkv&2
zXYHnCTQ{AsT~rjJmvAgyM5^yonbgxaCa11vhUh&A)HAt${-eavjur7i&$KT^&6fL;
zvNGq}`iuMB{!9Hmyk_bBo$7WgEjwQ>vfzJOsd2%%Q*KcK$6-Z2cl*yV8hv$JL?yY7
zDN3~$$UG|i!Yt-g)NxW#t$l&a;}`ZuBGZ#)HVDje&f(i=$?{xau5$(7N6UB2VooI;
z7Zu&w6=WBca9mdOJ8qMp7}6dg`&lE?NMx<^4!)08*CbsOt&hAB*zA0PFVMQ>t-$u=
zxsD>coZs+mv}SoPu-Ewq-$(0jl1FMqPA>Lb<;C?-vt_x!Jg15{_xU1+gL(WFZCXF{
zES`NTfWu$$&*Y?vE|>zj*zX#<E2q3${9dm0d&VB83c0D@U+lThds5P6%7R}xS}s4l
z7l{R)@5ta1`G3A}`IR#+k5{&AQx{#ka35=|#N>sO&2xmzB;6&~9h-OH+2XcoA3Sf&
z$>qqA0>gQa?}d2EPW4cBe3F%L83YqQU-4YE{=>@+FE_k=)xF{6=Zt;<tFjN08{m-d
zT*LW>^9^TtXG047JACKfRyUo8aMg0ua+Wn4KQsoV{o4B*W^S0dA(HQ0)A^?JkaXX4
zw&<RP1vezA8~m7AvoAKZ+h&#Pw#^%h7j-|>Y+Eid5A6H4`XB*+N1LM`-lVVAa5l6#
z`a$Hdy=KR91*r6LiFu3buLl|KFi`!R^R=^ZW{#zeapHy}!L=JCbDFhGTrNA*ueto}
zcH>dup8@f`r;Ie1%{XUo5#Qi>qejT?<%h#+iN!n;8$iWgrBTh*@E;~O5YT4!87ame
zB{xdiS<Ph4WX<j_eEH!eA`tl2Hyb}NegMk)#fP7@o#mZ<`{2%p#lPnzfVq3mtL}aE
z%jWmH-QRAN=iYjMYsT4ITh4>fob$J4pTD)GJ}vP4qIs@1-4$XVwH~!9Hq8C2^(yqr
z>W*;1^-6KBdo=bIC0c2`n!CKKHk5mj#0plql?>b0<bJv0Y`s-7>}CA3wI)CBoIJSA
z>#tGDF8wp7zn0EDF#YlIKJQC~Qy-Q%Gv^5=?cz8qdP|$hEN2SyWRE4s_>U=>6ssU*
z&14_BqdWIyxk4%dP(=_ebZGhIxqP$vX0J141Nr&j%g;7Or&dZv9$M?OV=8C!%0oXo
z_k<(`uiq51+t=Kk^YF=(c|Olhrp)sL5kAi}Um46)pIm7)Q#@yycJ0CRgXss;U~uOp
z^BS`nvmCP=zs!`=%MPXOzO-GVA|XI#56dBMl|RcOZFU~#lc+Q1o}+M8c!}C!{S9*I
zwvt_I=Ucqc+H<V&dh>er1lJn^NzOfgTQsB(<#23PJlEzRt(42LRq<M#(-z)`X15&c
zCAoGeK5CmFos{S5@Toagq|EsaZ=?CIiRwmg(gj@a3)DIP;QeUM@<5=`nWL|etx!fV
zRlN0~K)bU7-$aX+M?20dbRAQa0hJ6NixY2}M^<dV=J$}rg-^k<;R(ZJ<^a9`%Z8^j
z4so84aVTauYxq9dZY`VHfnuG9EG>OsO*zAvr5&ssUNS68=2TZGWw^>1!X6>pP}*>f
zG3L0KpTir5P0T0!6s#NHuxt`O;is^h?}0VD&fL#a&wccIUh*{4>9cW;Q@7EKxd~-i
zxolu1*3R83&65+#nr?EYs85)?@}iWPLmAV}sWBI3UUxit&d}?*$Wo~r3^zGb!h;T;
zWAu9dXGw9*B}?0tY-SE+W$f%{&eh3RG9Br<sOeU{N?2=+&JBlEADSiB-V%7lyYP`f
z7N26qahuC=cla8uS>6fkOqPrFZ)++)(=7X_oa2FFLpz6@Q$@!^#g5}vHx&EY9psoQ
zJDw;`X^)U=s(jNd`>2xR#m0W8DwCklAVFK#FMImJw3aNNFi)wf>xJf`<sI`1OJdj!
zTfc~03U=`aDG<2`V(hhHdELJLmX5QaZU3Vhljn=Ax?jw%KVDz<@%Mb5+kr<?H5M)J
zh>LspCqdlx3q*^p>z6n2SL6DeASS=o0!b9laTK`}>~X%(jN9N!+i~Tq%a56D5ft}Z
zKWFydPustGwO!VhbzZNP)?YiJVoRH&Xtma}kTok+Eu&8I%B=VCylYXeY1!~xv@X0N
z_F?hWth=U<K7QPl#dR^Dqs_@gWYIza-kheh0fO_6y3e~hHBu*iW6agbp|+8c<uO+m
zhRQ~P3D-Sc6`~*01j+*6OSQxv*?ppH;#;NLu3@4NR!uQ`7v{TD?$p<Sb?rsAQ(iaD
z+%#|H=LyW>iyW_Aah!Vf;Li7F@8|t13ub&${CH~B<kj!hSTFHD<W20js=?iuySR9k
ziB;KzKG`Kgmf~j<3fQ+x{@T~KVgF%vX}cRA4z}@|)qpOHdUI$2xBR~|@;@i*f1R-?
zbI)l*>G`uSRB*q4z`G@4$CkJ2q;_%b`n7YDOW-0E%Xf@r!5slz8ZL7-oC{yRdh^wj
zzX~^5h0a}hj!V6@_>ktqo*x#<bNJm){d)9o?_{z1eN6Wc95;W!UGPV?&i($u{fWia
z7Dx>pNWnMr!^{t$Vt(ek#suTqyL{(b&bPn|`=+y?hR(FlY-ZeM(q^}fd2ZyerymBv
zXCV<~|K@VO;MsF7WWCPxsfKHUu1(2zz1gEBx7kMbpVjl<=dWl@-(@vLzkFBGvem_3
zt&ES)UU%-fHT$FfeP4VQS>0Rt__vkg@#=L!8?R5SpZo8+V%^hZee+#bXWvha*s*$J
zyHs@nL*QEPE9TPQBkkC$Yj!_;&3ixpO!Io9FTCyREozSUYpy6*d~WWk_Vo;l_b{JY
zypCb*bnl9XK8n!~qf}e<+4nnb`SkJiQ>HkF@Eu_*!e)eZgq^E>dh5G|fsM)exP?9z
zsRt{c#pr+bjm$q)5xntZ>{8v3TTW2ILUZM}GmPn}7P^NkCH>A6B()iD4$%<NaJr>6
z=T#$rjAGi{8&fCO$xbiab4*a~QTWf*40)~*oPVQzr-f;L@;|uL&%p2S$2UKtXZ{X5
zcyZP5wMFwIO!tS~yf^>omm1CXu24Ur)i!Eng71~?x$f!y5$Up4XqoMN*FD`5-??iI
z^|<@l^d!UQF{v)UaNtQ#L6Mt>muB##88wSI9{)PVFD;w4p-FDp<m5Lef4+Y5r*zTB
z>){$wB0r8^`T6=uK~<53t>v6~D@4y<Ki3Ot@ml3riOgMBw)@nBqn8w4E;|^gdi`Ti
zRojY~WUF_!@6Hrh?K~$oCGv@h$R#HqZpXL6hoU&L71y;M5L1fg$XzP-?11*Gb^24=
z_Szi3^;q}okH7zHy$q%%{+Bb|zAo<T)52#nOFh@GUcBao?f#dd^0}d|^Bvr${5c>m
zn1AF}ozT0ErrN?uA3f%Usl`0nz&S;yq|E<Ola}W?cBeS^wQu{IqwD|N{`L6Yw<)hx
zXFvS>)8*EeqTJU{Z(cjSaqaZ@X#MTm;^K18KW}1KE~|6f<DSdEYs<NUYQ;WIcK`P?
z=t|^~wH<=D*LFk;Zda=N#Q*r<fj=)zwX1f@eEnqc>S@6h+h<Q!h6b%`)?cML?|-^X
z?eT{}g{Kx<E?pJ8%6jRk;8nIuUj@tkeW?-f-$cVwPWF{XU{+wxzni}|e=UE0{@DCl
z``dM8e{W^xFY2&joVB;9)2{!UMSvXpuE!4Rc%S`P%n)l<5$ZbEc8Y7t0@JHE`=3vZ
zZ$J6_%T#frb9{n^|4Z3>>(%4a`z;I@3NQTlvi{x04S_vXIXWWD+7}+2V3=CFXp_Il
z+s;LQXKK8he9rI7yH>w_0lUSszBko5#z$RvG3RZl<rFPv-iK`k&O3JgZ~NEzZO!@%
z`7g}B-+%S@KGW(%?&atAd-17j|2VnrRh;sEHQ6H0uwciX^EbVm5U@dtFIWC<>zbyy
zOPa1ea5UHBaQ`^-xKhY^Ctu5}Q-VV+uTBk4m5jetRsONIeT(12!f%WQdsq$}`r`Cf
z=t@?K|E3$?o^<8<*UKe#`nY5$_6xXkh^n~MDwX6(C_7G&;Na<dwj+5{<BIc2b$$2d
z>vM>va49YA5UlF3QMt$7xm-CzQ2E`TdO_t^6H4S&WSjUbuLnAJ?+`a^;@|n5ar+l-
ztw(`J7I&~WhaPEemvgn<*EeIT&9=~YH=b|qc(1gt^2O|pdarhan4o5s&ZCpgpf;DS
z<ol}Gk@H*lzW$Q=`crY|$2I;wGk*84i7=VK{^gk5#{WLw=670Xe;2I%<ezZ1bE)M)
z6SvC&vz}JZDD<2AD_*K;Vp~Fr{o0<M6$YnHlvc#pEOPmIdHFAuu#_v$W9Q3<EDfDs
zq0M$YR$W4R`hf<GC0r+%c72iC;IHSams^lsu=>xhh4=3&h;(FWI&PgJ(Zax+RBw|Z
z<Mt|8P<2IzP>4&Xql$oPK$DP$A%~3Yv&w~RUOY-o8m&T8oH`vH1f~Wwd1;s$l~fk3
z=9~?Ne_QH2cPxkq(9sYR;qGER`m8Vd&{<jYFVeqw|1LXz+x+S3U3&MAZ;#oS|Mt_?
z*f#dqefwYk`#nAPN6PtCOq%N7<ebhd>-zL7M~3Oj^A+d$Baful7P&Qa2Yc}T<h<7A
zxOVcJBYY)E>{c6@w|1^`^ViJXzCqL4S5wSn1Eia${*T#AKKFo@+S%i<rsQm)L(2~?
z2Xzn+FSFL!@RHA`-GBanKEDgRn-_<d+}XQvZtC&dyOWoDuF31Xs=M)v>a=}tc2r+q
zw?m5kdE0s3`Ot0!s55PCA#5jXCu|0Rm#@El;Pb=hht3T+WKTbsZkeC3Ibm}m7@j}#
z^VqJr|L2@x?`=^&bgwNj|8U0hgOU37JFHyR2yE)8l5?}=Up%k<W8vc`QzY0<@E)Gm
zURZdz$Mx`z9f?yu8x&qVyukZ{s=(7l@=otH&IyW}_jF6kT>aF{)5de;fWTj;AIf}r
z5>IU`a-XtGa<nRnwP{Ekitngz+0T1!&4If28}<b3;bzot*(LBgX}+V#JEu9^f_*EV
zCfeLFkjs<cH|MFBUUqSvyn=_L&eRyqX-gxfots^9PhyHqPqXx~gS@TZ0y6kpzx_C^
zd}&JNm5+T3zu3)VcYd3ov8COq`p?qO%5&wfI8@ZVa4)HHSNF2=4@#VIZhFbSDQAtW
z7xHVo?YuM(OlZAbc`5Gni|EwtH@i+upYUfbpQ1+mqEB1TTiN{3J5YbQ@Wcm)KL>v>
ze_HfMo$nm?7sX5Na-E9K8|_$r3%s>gyw~0!uT;nJQ}Idr4EeVw7BH(ms^j>h_^JJZ
zyi<M0U&SBp_RSjq+L;ulFf8QYaCOjNc2X4h%d@CRL4;+Y3Wuv>kFt8|3WJV9w!lgO
zu9g6$79~Xy&V@c4txk8+1*UQ@OyE#;){t_#C~&GRz~YEa^NJb{R+lr%XHr*maHP6u
zs3gsG4DoAeQko*Na0Q2}tA=*a6raFPle1biW0n=nJe1nEcoN^%S(7BcMslW|u8|L&
zuC+=pRFRpto<Av(XT96Uy@56PPA~Jn+@BrrYx!{#0jvM3m9BmkJ=C(n@rFW<z`PgI
z9Cqq1tX=1p&ds*EUSf51XUc&oGTP>rVJq}SuT1B9<^A{ior2vvKk^2fJd+Nc`76ku
z`62JaCvw8SIRB;}O0ZgXr~XhH1m3*LI-hku>ulE9tbdydK5swUKcDydTlt4wVKveb
z^{Ykh*Z=Rboj<*DrAy5B`}=Q6Z)f~|KHtbH_>oWB`<x$HiqGvOEtHkc$82KHvfjwY
zwc<mIl>drPEmr;=pA|2(d&vJ@aKKLM^TnB=vs{0Ryb0dmf6$J#x>;qHqtIQ2&b$dO
z<{pcrRW9;Qdco%TMHtE`fyqCQlKgl1_0tNI-|y><Y)<{X_-_BhyL^xA)%%mHY^+%&
zxxQ)MSe`K7{k;E0JJ;VLZ-ck^AGG^@v--q`7r!-crT=FS*<u8>wEy1x$2sW_I<|L-
zUHO&s_2h~L6R&(*-DJP1$-d~1V5Crxb6ru&!G^cu2jx9K9KU=(<$=%!&)Bqt!{4$u
z{FSPI{`Z922JVY`F4iKiq}4qyd|9xH#g{3t?}*U$t}knqVjCOQEvfr(!@7j?sDsGM
zXxCq67mxf+mwOuB`zvhmzOyEUv+ZAmE!^jOt88_^eHMYw%{(oq1@r>f<cq!rDV}(w
zce{kWhlXF9gXCv{Tg)4cSiDW&N1y#2_Ick~<&xEn_p=^spQ({&`tf(lq8*_}^Nj=V
zq%GQU^+f(>j=NdUw$J+cH{EVw^wmFMru)yTm(1Su$ZY<FwWsQ4Z+v9;vn4O|`)v`i
z?Uk*0!I5`XecHb2&swee`5|K4cm2ur>c0KQX}i|nwO;lApN4D5v$+dsHw8bO{k9`_
z_nR|5PYbM^H5>o@d2q!iU9GY?&qt6s_{a}yO^t{k`MVcvq%-w$r-dYYnobL8_C+o@
zxdk`8Y;tM$Z$JO>VawSDFci>ei(sD5I2!_`&9*xf9tM?|uz{EjJ7wo^&gYyDt{7%>
z&er^52`)dm&0z5K!N?DhNg6p~IiRk%S<bfR%?~#}1ecsgA2A<W^zrG34S%Kb_8+^y
zzv*t$%k3KT7u0l~ntoU$zDE4P{z!q;3Wa~Wi>EHsc>j&tnlqv<BvfC_Ihe=S(B$~V
zIY%#uI0y3_eQUQPxK_yV++O<^?wjp8r4^;@mNXkI;Q7`)e{rbegsN}V{Cd9cCKdS2
z2)dHEDCB&;j&+Q*tySXv4|zgm8dmi(HPQln?+Oq9UM5}s`R9Ww=Zb30GwLG3U7Sb1
zNG?x4WZIoAW*!%_>%ra!dli1@v`@@7dt){|Pw#HbuDzGiU0mm_Iubm!;@{uZjkf>)
zU9Fl`f8WY*pSHL0gPM=}ryp!Qe0Ki-6ak|Ta{~i**L^s5E+9z2FSj6A`py+L^Q#J{
zRGr!03hdJP^uqS%Ro{$Qu~#}$f^BXs3k4H1TqKZ~$2$y6lz4hMoRtq<Krk(^sFV^s
zW{9csLWe;gx|v|(3wzMa1RLLzjAA=j<wC(k85G+ubQs)e=KY@V_|3t#>J1O&a(1vw
zze6z7`OU!6hiz{ZAea_dRK7cyrM4?$)@nZw=h}I<7A~*6wXE{ivZ&9SXY5+-v%^_>
z-mIC+pFLgq$vU)X`L)NLm(A=?cpTtfsMoSN;`#4`Uk@kTKKiz`De6u0pGA8-TG=eG
zDlC8J@wM49%w>hh#2}Yb%3ggsfinUm%}rvD{+T$_NTlG>K|gkD*__^8!M+7r;meCk
zvyRM_DqLPvrWOA+;{iM8RI5mXTdzN_^KY2C;`Dj3)J!Sf@cF@ljS@`_4}zIDOjy+Y
z`BrOVmZIy*ITb<sC+NLW%=xqN^YL?Qt-j2SF4dEaG1|d0SwvK%nXAi@_2~11FaDoZ
zHNTT5Z+JB`Xs+7T%uC;=tTL$#nYFp{@Am4_lXL#QzJ8$K-{glEZ|~pv*rl^xQ04gg
zmv7orR~UG-I%@HJ%J{I-<V4vPJxMLzpu`D(-)u;&F!>oGwd9h*RJN5SU^PLB6WkY{
zFEo2wpdzAH@$Fa6*UpozhMLNU|F*wYniwx_v{KQYb4TI_%RWB2nFo{;gbFxsEcs}$
z;oyma<dhx%`#v7O&J3E0JEkHf|F}x3W>!CF1f6X*oAh48n-O<&*5sLmnM<#p-Fx-y
zu~%uyrJFbIx*PFuY0067U<vJ-3qiAAI<dw)ofZ=xSvGm8X{6%hoAxum2_I5@*i_)S
zLm?(WN09Ll>%(aU&N~*~2*?S$(N@6Cdp(u=B>U}`L4h$^UEZ#RnO3n{U9aCqdl>97
z=X&(HnP(rTSE_)~53TO!_JLf|uY!tWw0gX6UjnN?t+LEpIQVEob>e5ILhgk-Of+>{
zA6;6q#YA<jvdr}MP2P$-O=NXjp9V$f`FeOS-1%>f+~zH+y8P+WFO_6^Ma`Np?NX^`
ztbgaFUyg3)AhWM?d=2^E-8U-GYAx27<ML4FP{|e(S>2<bgKXB@2M6l=7f)APvUEpW
z*XyMxS3LWyCG8L*v@&+~8@~lVLbugwCFPw8Em`UEclEt{>?e2K6gE*>q^!H;_{7MY
z-{;L|Ke~9;q#Vl?*B@S}k*PDEIVVd?|E~P{S>~alo02=L8J4a)$os2tW!%r#)6_XX
zKWXKA7WcC*aoOrs6Gd}^RtC*CO?6yRrM$N3NBPo&+Uv7p-hKc4qC7W1OY$&RZ)b(G
z-K%tmg01zNzgAA#_TkK$xcV~HMfGa;S2RprUalUSXS6|Na?ZPT2Nu0fDxJl(@$rr5
zzsVa**NW}B9A4@!J1tDrVomv^C&w2_WH4X*H9wV6KKJeZG?inLYY$GAIkkAhnw-{^
zcejLJIkkD>+7GOKw%pm1MEx8)CrOBl=WMInx=Pw|*6E46liJT{ghw9b{CXxSlx<<c
zx+}A-O%0}*on5MX`Jle6%L{MQwIUaTo_I~v-d{EKI%mv$k)5CROj%sinNnGzzjd9+
zNdsR!?xJh_MnQ_s%CY-aF3l`j*`OzSq0}o<$MnL|XFIpN(GflWv@&z*&)O#?wRWj>
zi%<P=oBDI@t3`VLQhZ<fnfa#IoV#~_>Yj%hj~6uB?Q}f!Y_fn_3uj{Uhb2l+!qj8$
zZ<!x+>*1}PTMln6&TP+oev0w(`32U@hZuJsG&Qf^{qr{O<=g3d63dHUw%mPk>EzPM
zzx}IhROj4DcCud>FQCUE-ogz_&v*8<Gc9ZTnI|A_n|&tiWb&alImNA=MYlZWE!R}=
zKBxQJ@!D^GgM%-EBTOQSH(%*Ub%`sznbW&9Vp6ROzrTM~uKzsVJuA<vf2@oM*qAWQ
zBU@FG%WwYnn=5uy1-7ZI<~iBiCtG~6yt;Mc7P$rLnhWRcvw5^+`s0|Hmh)al)xEqC
z`QS%}sI>91&na_^O}K1ATSA*XZb$sx*%WQ&r}Devk)J=uuBUV63u>Q}K7HqMx8Icy
zpTEg{O7EHETe&i;xreFs%j<}Cj|t7}UHaL=s*<Hi|7I+i|8ak4M&5hl&N(waT)*^v
ze#33&8~1j+pZ@WcN>u(-x2@rVQ|d$<Opf=h@b31i`xN4GcdFeol~?xG&(_=wyzooH
z{zT-r9?!5onLDu)re2$SYlfkD)+;yPyGD0kE&M&_ZocS+wU2hbJiRiYBFla5CUy1u
zm4-j6xK2IDUTSk{`TwVmmXc3jHhPFA1#|dmEV#Y-?Apen>SH@iFXagB|Io~HWR<|0
z4d-%1FTS1m?3+*Vd}H6I`CE-7*t_<A{%|fys_Ug)`mAWylGSxNq8HW%KL|U2=hgJv
zi%&b1zgL*l8gMM|DM(~yv~BU~^srf;@3-_dO!&|+bF$JL(T7_)YM-3`cJ8-<g_4_O
zv1W?i(jH-s%zgdawwtQt_A6~KTk_WDepl_s#Zk{4trlPEoZ`2nMCobx8P5uqyWy9f
zerW&9@jvJJ?$yom_NJkyx&vBr&1M(;k6gV)$YJWK*oPcjOXTeC+WLkaIO}rkUDy7j
z*UpCIy`H^oTh{Akv#rmZT<y0_F8r&PZbalVuBW%Iul>Hiab*qn?fo$_6CY$wjN0k1
zw6aR*X8sd*&V@~DVgr*o1maICPqbQjJ21eub@z#Nw{Lu}+{drBsj*rj^kI=s`E2F;
z=~8pM62;B_p9`wH_dUkN;kx7<V|%&40>dRQAF>`jQ|h&Q#l7nRPV=kvMgFv(W-i`4
zsnmA$Gp)aglb3JzV><LUurv6}>jzH-io<Sv=kW_N+H>a2+9eh7EL*Ru{SWR+PWH`d
zS{(3iv(<qG_S*T2=bN0<n5Vc}DQR!e=Ic987+;w?_r>Ndr7L#at=X7(Y4xwkObSYg
ztCzjoFLFP?d%OCxXzQ2PZkGO!RH<A&FHiMVnD5<HE2GW#pE@3R@SEz!K+ku-TsrhO
z{1!TNSM5T<*XKX77(bshJt;4&$NNia!t0d%>n3HYo10vHIQi8F&j~qUen+yNm7kBg
z`{dHWwQSS-{?)GZWc;z@Q+3eKzltHoJ1bXl&ORxtE+T9ud}yVX@y^dpKObE>x|BPw
z+uO{?JUq<8E9jH$#l%Uc-g`bRFYCUlp|<{X;$E4z+PSmW9<E>MFmJn1oH*b6{i{r$
zJH=07ua&4-H~)Xi6wT%@ef!SbN(|F(w&8iRYyHk?d3%>{TA7-8wOKcF(KXw*o4N9L
zZjlN(`0nvfcm1h*14GvBtYzQe{Y&WX-=5PeuGGvASg6IFEqY3O_ldvXbysJ4-n?|}
zZrA;(wY%lY_(gjXY)fUH9lCRVsY}^|Wt+3Am+m=WQ<lgr$#>%N%}q=0H0fCOyvq2x
z@!phs|1#pvGu?9Ctu<xwk^l0RUp30gwO(CIn|-jlCCpi!z3juw)he%_E-dt0-eJA=
zqU)AIx6d|R$L~a5>Gzu(#lqfoZOtc+=-e$V9=<m_C%&CIf11Ny^I2I>-|Ec$x3Vhm
z)Qi~cg?E;1Q#<##EbQ|H9)^=^l08{6_Gbql=UdVf?w+qyul!zhWw*h);GzvBH>(vl
z%LS}kBUgIs_m|g|)=T+zUSiYUc4XS?=Ldp2{1qJ(ujt1gJeuNl-*Ic)_31oERtv0m
zTB0SoDN6F*zrWkq%k58S9seDp;XHA5$f+z<k;6%HY=<H^VwQS^y>4}i$eeKX*5*<<
z#}Mw1I!87OTyS#fu5D?s6+Nq!8O+HPsO`G;lEmt(uOC=FI{ezz)a39lN2fp?*Y!3<
z?jom^8~D1GYJ{zdm;2A=ANQ%KHC}H%x9-DCS7Vb#P#d6olT4^K^K*AA`%Us}vtllD
zPR{##nRBw=jFNp%ER4R`GzqIuEnK3kulmGMIJ|a({`39kciY*oQFTb%yw2rk!PLUi
z|9+c7ng5;tQ4%yUtmWIGUpp=yo9{a1$D%26rdsZwjy(H(p=Z@R8?D<}FO~++UaP~h
z-nQQKhWb3#C_d}On^v?;P4lk5zg)}pB<JRtDy=@*Pm|g-_~%PKNjJKBW%(Ju)yYfN
z2CnD3ULx?yYxUiKXD>ZG^(M9B?(;Q2!**^JIMrMv5ikBA!?f+t{NRt96_u_f|BdXe
zy~G^naDz4L+6<1zUqu#f{$*p<TT=h|%#(EKUp{Y7^ZG2EQOl69aL3#I#a4#@?&#gA
zSAVnOh~Bs5|HFQ$E&HFjPX57;3r|mNvGvw}>8tr^Pjg+FvqFMmhifi(sd@YD)jbuf
zt>33{-U?m0x_s&CA5tsQeOJG{SQh?v+m7SwOulqjMw~3XU~$!X=fRZlnV!okjm`@H
zYO!1RUxi1gN;P9sl+>5AtZcQ-`<gafZea<0mmhiaM3!jLf4<uGea`XKlh%}^TrKzy
znkMF(%{Y7E%Jc&X4)zP=FYv#}*W0`EQ&~aYe~!=6&nA1T-%-1xCQ{6$5a6RBb~jQ$
zZ{8{KJqzLmYQ#jmKGz;_|6+OU;_32=yZ_ZZ=jCP{ywa%6c}~Ql_<{6vt!4|JIqd)c
zO};Q)e&62@Z`;?)vtC=X?2kfYzV(`9;kN=mGWLCZaa`wmKVt;jzIPvLH)pTAzSSUm
zn)~t19AbUj?jFA4y7u&>)pFS}KC@T%EznvVHgm$k4sT^oZIyc!K35m|>8fR&eG@c&
z-u2*}x63rQgv+kI8hGDDm)q~DMq<OvJ{|elOH%lwvadu(Y?Su;Y&reeywWAhnnIs1
zSjw{K@;`0o^M?OEpRe4u<I2Q$Gh(BU>f06<&3n6U(+|e1bK<XU&3L0@W#h90{oFDZ
z^Vu@r+Fto^y3uRTB*uBEe^;)&_gU}UwdZcTY(;)coc^=2WckOt-D@vM$u3>J_UH0&
zp=mwqlOt+`Tf!r*Y0aJ!)>~_mwY8`5s)mJF-Mx3l9$V#Zr)V?3yS8Iub;!wUj9r@o
z<QKm=^!1WV%={0ZjtJ@LwH^>r3g+-tH2e5wYQ%}DYBsGuM3_Xh@+<xaeC74~+<oxv
zA;H5JTAO9vw;G9@b+T|))$>f4zN}|ikIOEBS59lV59+a4E4g@0JK?d!Ii0WcS%5oV
z=;Hv@b-t74oKJK6bNcn0e<B{!7kT_S-CVEvZ)sIjSH0%Dr8}lSij~{=oa^rs!G=ne
z1w4Hpg#Ul?uxE|adS$2iZfV8zQ`<%E2UVP|Ygu*uyK>mkoh3}E_74hc9b3M2B;WU1
zcyf8T@hf53;;YLOigQFvV%j9N{`b|Ju*|bu@;g&Q=$q!ks2s0@?~48xO7!$kuzb0%
zPg`e!zq>NmC8w)bmTq~XBY&VX_@on)UGi@2eS2a8Ppx?W(MIXpq5Cxx!#}LLx?foI
z;~}qDISGdE2kJsQ{aIsemD%t2^V(kswbgtnS0d~l9<)eyW9;+g<p+<r9o}=(a<bt=
z$pyRDSe)f8*>720GU4|A%lEv#i`}cJYHR-&X|T_$Kf~shv89=J<y7}4R-IbA(q?;0
z=5D-}zTK)qb>;sH%=~wgU0%nn^~&6Is$-+YiOPaHrJ7TZdcN*05EP%R{_nv?^^z`=
z2~4F6R!&>FJ4d!PX*S0l$L)5kzu!J7Uz5u{ZT8cH8tXF7cLzz&kmx@BtIl}#^~Jk%
zL-&QAs?}2RSX-X>Pj5HJ;(If0@m?-T&YEvkedf)vGv7>Wtxhj@z7Z)b`-`tOJ+tJ-
z7fr<)s~WZ(wlB=T9-e9~aQLuu=d)-1=4bNP?Y!$;^pf|zCZ7+7`Wf{v!7&YWPWB7u
z+dW=X%>9e?Z&RJ){sr*@dYtAhw`;Y)GYV~W&iiHO=sL-E2rXW>L3GVI!9!Pb`Kw~n
z<JIe84_#C}%v?2#?P6<itc7itsR2vL7p9APE!HLXLazIsja|7>C)iclw0x;m-m1r|
zB$jQv;uCtu>z?Yu&nk0-A1-lF_b6=$dA2-F|M~UDj-^*uo_D*w$~<a$;O0HnX_Z%O
zRs<f~S32j~N2R`1nbBoWn%kU?fBo*eBK-J)@~@j8^_E22ePy;e7PX@<H0{!*$-95;
zefrk^YnI&p2e)(|uQRvo*nWpWFaLVv4e9uIMbCC$pTG5xS{#Sz8B_I>ss);<((eQp
z2h_i8jdgl)Z1L*qt{V;V%YI(sFf3qQ@%d<I<)aFgt=CsCSTUJRO4v-2CF|&v=EJjc
zCN;<R6+bXut5#UIc&=7fbwv5(_IF>s=09KEYGP=k!j&$5+vMy$o6ARdkGkYMxM{p!
zZM}Q*iOmtwvb%NtYQmQ;t@BHkE?so%<MWl_1=@R=%I6*kEiX4?e4{YWncd`{*Ob$O
z`8|(igYu_7t+rnK-uQj%27x^l74kcxp6@>(`Z+_($(<ukF>7JvvRTh7rB^<iIjii_
z_Lxo2HEi1_+6ta^lHqRDaoKc(y_RLOz}Y1Eubvn6<}59kb|{i7`H0GO{_ShMrW#DU
z9(W|B#Lpt+k=8~H`x73edZAA~WfaK;JPluJ9W`giv{!{I6MW8#1<mnxP2nnEx#mcq
zS+I2Cs@KbQ<-v%sJ4I7J%{;pQ%8i*_Cq&rwmxa|fU6#+O&WSr}EqF(_&|V>eck<`O
z%DoBmV`jaJ-)XYbWap;B%l?lq9b4MF+kD;2n9HW-$CjE-{U-c3XSv!tt{E?8MR9qr
zsyuLIO$zUs7px05a^Ft3+$WZx-+z{g^~=QKtxMmsn){!%@M+$<!BTilsppon5B_|Y
zXxp@j`TE4r%JaMC|5<NuE^aUIC82#D_ul_6_x#ma8Q0#qRr63+Q-0fVm!)y~!bzNM
zrSCkXB6#l3ms)>pkG|>W6Z?LCFWvTjDWko2UiOmsIrVw{{Lz(lsuOCIBim!;7TDh?
zR94tzbande4d;>;IzD-m?zgF}syjN+cH&acS^M2fH7A?t&5EqLu4#5};})CeU8S+P
z-g)1?>P%S5E?U;Mw9A-l!D-14zm%h@ub%#!;qzk8^etSa{+CXze`h?Q=DE^gPW>?1
zoy8*3OCGXHs~4Z~s!VaHo3;7QjQ`C)?Pt7V?mB3+Ba+QL{9a7wwhIBV(yOwaOLHrP
zAME~X65ykEa8g(C^UDX)J{~_RYu+M#=&b70IqK3`sljVYtV7ot6m|+vcPRAii+Xv|
z#q6q=zsIJPe%s`Y??isz&ch;o?q|Q($()5#AI!MdB6a0-na|3zkFPHC+?pAbJ1gXF
za`^6lzBiK=?ANMFFMelc9_Tyy)y7cM++g3?A=YLqm&x6VvGTRulG&YgYf)cAKGzSe
zWy+2EqK@}Oo}4wk5%8C5e(jUest48l@ij8-YUvC`yEV7ZF4~<Ll_kaen!WnL7l*vh
zzb`rGNUk{8RU0$2E$hMo{a=R^OEOrO{K}P?{QjuqKc2p~=Bl>04reTOopzb|V-M>U
z35EB6@Aj`ZiCOMcFneBt^W>bQ+*JK5#~Tl}svO{m{3aA~`0CbG!Hy-y9#0qftUbzo
z|81}5vMmSOnm(P`a_F}gThwQd?%!`aRI<|L{!i>G(p<l|^b-Gc?YR#xo`3sFWlh@a
zT{qj~Lj;dX=QHG=w~D&qY94lOY3l}~(D~PYeJOZwKPdCk>$5(8LalcFN~!p0WFvR3
zP-ykk-i1b<>whjUKX7XO<S!TAuTFmGe*Wm9_^!(zQ#hJj^^Lp(GdI=0Rocrp?bTYd
zbFPJ_znJ{5D!Li;tK*&Rsp^>-B2IH3+q7-}bIUNGJ6ChX(go9+qPVgu59qLN-ov->
zmhTA%0q?ELLUK=8@VIRKljSEQC#t1t<8|fYB9`s`a<yIor^`z8Ia#-gTnP#|&77t=
zV`;*)zZtpr0t37qbwv|y_|LxiYo2_Yt-zh2faLq4Ur#=ZKY4HUL!Te(&WFyIW_rm|
zrgv<~t^a;)oE%JDEr)_lRh_(M+)-cX|3dgz%Ra~ZZ=PJVPq?rk|HAIOZvsl*-B|eI
z-bUw(3tt2k96u+^{zYy(e_oGX!~eV{;hPs(t-fi;F4l@JFujm`<<$$Oyp3n??SK5f
z;<9<{yu5Eei<?)S{~Rc^V95<%j+aN;A}42FX)Mfr+HAaLN6!gnfvDECtKJ%{Rcm^p
zwf*SY{ui59d~3bAlf6rE!P&a{MIt44F9w8HbKN?&yW~vj`CqTC_QkmUz0U10)9shd
z?FCD@f~?yPEKEKl+#oD=OXGS>0(+<U(YmS4UpCJ?-X?wB-u3s+-3Mj88n3?Hx{Q}W
zP`uZ%re?kH$JDeng^SMJI51~PgJ{nsOLtj`z&x(tblV@#KD9pdKYXEi|LYm|E?eBb
z)_HS^$QkW;?nTL44w@cI{<K2v?jFZuj`yB7D)l|wG;8OEty7wQ7_#1E7W?{W;_6Hv
zWy=R`=YGV-yggzsn|`D3*DXQ0zy62MrcCnJP|Hr)bxo~1#ptv87h!PKTj?JDy|VA)
z+xBzsZf-nWHsxX8$Nt{i&O07Lr)xmd-L6}8n_`)6i*Ac@(VzJ4)waXep4OZ_>~Li_
z%c<rmiiamWZ1mgsdCAkCf4H|#YdrAty!^u(8GAQpNFI8s&T=M$eX**DmGc{(jm9kJ
z1#AUxeQoYxxg_8jxaRz_fad6}&jpWM6mWDt!{cb|a#ldmxuZvTqX4_g+@gnbxYc7V
z?jHIn=C`9@S#6Tsl&^f}))zFj8tcqS+<5M*os+h0*{0`jd=5tcmgfx3KD<wk@xvdB
zLJo~|i)nZ3%XdUBX4x9!VtTG<(u&1LPt9#s6rWwPvpe+2bk+UgzKj2dol@$)ruu??
z;g;}lSIv~4m(PBmd8)kfz1;`n9u9>^8y@G&Z}lt;|1NWJp>&Q|g7*G-M#gsz%>TDy
zX*++D?fD~ypS~JefBC(-chmK~uP&s9|N7<I#`THe$^5T}z4DJwULaGKZLnRFL%Y>w
zRmN`Bx36btm8E_$>MZ;_?d$zsrdd{EHrrklRVJRdGQ6HtVlK5v(rt=`!L=&Mqpumy
z_?jGWu;0%!rGm-xh7!+`guY@m?hV&93w}<@wfXM#S<mwBmTPmIzXztzepBofAN^GG
z$JhJ(ljco*8~D7n?pv*2_Q!X^PO%-OiZ@yfrXPypO4qD;+PHqtxz?=Q!YgK+t?NUs
zdSBFXUF|Q&>ObSu;#;zJ-)WS)?-pJgwB=HD*jlyS->$D)^`@|-EcEHR)sEpyUmmaC
zw%VvXG|l$TW)S^s-RZ`KOO86+@Cw~0vM=a|*TdI4v!=YV`Dda3JK?^=VaczR*PpKe
zWpkq!c8sYRl8q({csu+TnVn$%)n6!9wY)N6d82CL<PDxTo=g$2<DAbnn{75*ZSw}l
z8!9;hWg1o@bEQ{bE#<IoxyJs#B+pbqMu3lly<Enw;`s;e&(3e_w|$t_ap87wvB<J?
zBc1T_)RUS#N8}F#^v_zkb-HZ5y2kf49gZtxmrBfEx+yZxLb$?b$Bj(8`du4!w*7wc
zQ{Usq>K#mn7Jrn}_dLCDV!N+}&VsKoI*)EoHZD2AdCi`?Z1ZP*>xSnw{42IiX6Eo;
z;LADJO6JYyD>1qP+H>DX_PVdqT~nJp@0WS1&_UJpZ+Ig6uia3xVe2rGdF&gt^T38}
z=ZaqJv3$K^bNBMo+W+@y>dh<PaXav+*F$r|v->>0M~Xc!EL*X+V9lcNh=k)hA-gjE
z&Hb<-bgGkl`JQDnk9SDfOr6c1d_BG<_P1cm3GwX@L>1!>_it>wBj4@*+@tKz+&9No
z&ONzA?1X$)zUa4eo4NI46AWInZ?4-A(>A%@;Ag(t|Fci_&-`OP`G4A{`$}v%0oygF
zPucT&NlZ|Xo+kS%v#XOPWqI;HY&kDdl>pi~vKh3n#brI~{HJ`pX541p9L;B&&bImP
z)KT-B@2R<T$&{cVFU_wJ3l{|~G7?>wP|>+G{6<&T^bHl9{L>FqSh}sa>umhBAeVRR
z+lAX~=Ojjn6#tpaWHP_Lb8~7*z&Gh*YKzyqA2Cd{zM*)c4YaEINn%Pr%PoP-gB@I~
zlan`RH|2NiQ~cBRp*yT@LD2^9=j^+0^y^s8&~grxG_9?eX6$$4VnD3CRVE+rqdiky
zZLfsQx+m0qTDtRWqKx?Z>Q#KPbMj_N?FcJ$y*ID<;M?e6>!a=SOaK1a&UNE-(7NeP
zk9F<Xn}h4O9jgtU`#RP10n5XEUYYAJbRH3~y7=&w-(sh8>x`P#sDHm@{$$;J6Mbp#
zGuPhV&zZ3>u0$rjP2DwkZM*RH*2xz{XPE4>i`kpG`q-cC=QvsT3TH=8OS<#lN8xbh
z{X>fqbbt88JpOXy^P{IboOu&|2A`Dai&}N?>&r!AN>5x@ri6YM*l*7F(kpb$hCH?R
zN9>mc%Kgb|QQKT}DdAb<!9IEJ*m=SIyS2A$h_ij4ROii|pSEN2uVl-6$FEp>mfhgJ
zxyi%d(q68`VZt&sVWn`tkk#kRe4Z>0+bQJoN?_IFgi?-cic#$avX4qRZYU-luN5&3
zT_OP4GQ#pkV59R1zKvGT7)>5p9eFLV!TAUuWD`lUtc}n%=R19+TihkN?kbkG-;n+N
zVB!baN1$cdP3;15zh5ji@G+dzIobL1#E&eNU;awWdfr~$|L~skW?`Nu67EN*E=-u$
zRLS*BbH?(Bf1U?zSU-oPpIdBPKQ&EKZtKs6Dy~<RA?c2`{qJ;EEC(HSEf?VLXv_Ll
z<VrAvQS$FOBbax-&ZJtB>y;)*8T*ysfb$<iZB8v;)x$5mHodr{z0*V4<eU2DZC9OU
zRtkK{IFXuZ8CDyTWNH~CKJBx^fu)5>VarPc&7)@R(674U__#LBZ$iQDOBQAchW+M3
z=gtQ2th<vEb-eZaj)MCu_7}bq2|aqfD|wY`D`S#)gY9Y#|H(y)F*o+VE{!|>a?hpw
z&=)0<mDZN?&j|fi+@YNH>Fwm{YouSqxaVx$ut>uqbq&X<*lY1T*Vwe~|KzPzS?m#c
z{NziqXDc(Vem?Yd>HfOoKX+F%od{cUC&Ns0k-?+$SBwfxn+5(q{{G-vFzdE$=4%&~
z`~8~LU}e~|U|GzQNR5*Xj+q7fBIOj8i><1!TXur${G!>Kk(*wL>jrb#Kd$kgYx&{E
z>?>h>tevYJd{tP#$fP%)*80$@eD;NYwa}R*8R<40mqq1g@ZNkXysO&kS=sZg#hq^2
z6N~FSIpl@qo(Qfo*>Ev#_6E!3S>U~wQaN`{o8H*5Klg5XvwyRHaX8m(HfiQ?>zrMS
zFUg95wxisbb0luV%MFsSJvapwpdDym_7xVy^q7B{mwM;I{R{U6cJ189uP9rfZ>=~-
zz>lN%#@v08FYK)>d|!xv>He*M<Hg!t37Zo>Uz}NH=?W@f?gkxBJDg^I|M6G-t4W*Y
z?(_RrdG^UMXPqr?LXY!LEljvp_xe|rZ%#kcL{9GGJ?>B9p0P%AFMhAJMd365f^X%M
z?B=G#E%%GRDdcD4v2+tdrq*Tee}AppZmY{HMdUx5ZsdK<aL&e8)1RL^FZSn7?SnlN
zq|CL%_^n?XB~>p}b${wrnI!hDFnF_z)il0se&+j5_ZQys@p-oAaQQ=-=7!`YW~|;`
zLU*1VJR|kkSBcZtp!mhQ2a1ow+7{T?fB2d<c}xHG$g^*6^%nfP!Y?v?;m*Y5ic8G1
z6b06b#~(Y!S8?_3Y5(1?l62-;%rd+EK;`YFT|Zooeh}Jd=B`=S9eY^0x3^?H&%5(E
zCT@>JvS%zkdEfo1Zo>UD-`?ijKlANu&V8jvjV^*;f9#o4vT#<FM_jl)N6+<*^SOJk
zZ<@MLLAptYPqXrW?cF2%7vz@M22^hd+rWG9+h_g#XByvb{ygd7_l`Gj&YrLTY%pQ*
z4(a1n*ZLSHOk5z{l(%KQ%a&}W{Ej_}U)pX+H|2j(4s!V@Q18spS68Ab*>%vc&q>)i
zFv8tbH~;D>4<0^8izBB6c#`KD3Ou=V@w{?Z&9OK3N<~Wl)P%BxYUfW$KiPSiIW+1L
zb2sO`*Xd>ltFyPM9F3Z!rRzC$+8hSkqfwKRbS)Ww9rv8Z=fipZzWIaZOS(E0Lag_<
z{#9PN$h2kstQ-B0h0}vqgzT-^InVu`gpk?GH0{ermVRGv#r@%CTV>n4`cQGpqSfX#
zswYf8hAT))|MQLQUHek8pzvIZ%KIYo3n%;xE(D*}>}kDhdq6rRe8&cdDNPamQ$9C%
zZgQ?XC-Z+F>n**n8vft5FSnOXpYz(HK2G%72Zn<-J11us{!5%5AAhyz_5DJ#n5C1l
z|2jsk<+9Ock$!jLd`s5x+}DR#El=JS*(x#bW#qo*>D^g>WyLSu3V0bD@qB)%?C*rX
z&;6Xvofa3idokfvg=eUT-`W-h*Sx2W2VG^tbNG4l%$(Fe?f95<+Ol!+p{D}V)}KG}
zIVEvg+>FcG+ya#!cd<!_2;1J1ytee=iDVwJBWZE<XXYpQ=>)&X)ni>RDq?x`wPxJ;
zWc9g^K8Z|UUh(GiZ?Q-74!!$hZFzrU_kEMNsEPaZ_8ez^IK%Gw`TF+86^v*663e(Y
z2hW&$XbpG7;VC?C4tlcQQ+*>;_cmLotUBS_27Uvp){8&C`!29eQJHu2<Ba|MuYR&Q
zNpOGU)^A-|e(1^zW7A(gy>`Ki&vWLim4CCzK}@~yv{hWBP*>*0bJ-<r2D!_3e_Z+T
zSZMHCTbr3;-z&NOW|SOSBCIx7<3^g!he^S+d)O@_s}{8Q%`x#XpKI~*h^M-rRsN;Q
z$3H)8oDwxxY5A$m_x)ea<l+mNcK=5A>MsJC<;OmTS*Wty^<G?D*DO2h<C~`%G7&Od
zCr=;V@Fk}I!|C9R=9|vd`K<P9tU^{MWcM$-vdBq%`6+qH2fk0YmAl$3{<31~Tf2QK
zKQ_;Mxhl)_w8vWC{WZO3wIg=z^E7?*KAv-4+xoJE^nf=7<}zz~PAeFtT+d#$;=r=T
z<|nWBPLqs`p4?*nGyUPW^V`ePSA5+3>+}QJsj+p2g1eXhJ$!5O^|bvPnB+UV4o5He
z@NMawPp^6R3%2AuniSi7dv${^tB>{Jb#<$@)V`iSb4%r8ouI_4o@ob6=F}~y&XZd@
zX%64kE2rnqTv&hpj?RDa&%3VI++NoHXr)-XoWJ_rWh|P^_Ub|h3?7Jl4cc&e^3C80
z-i~^#mYc$#ml!8<U6?cLx6kIeVa8s@K1;9mbTw*BZ;Ii{s&w-&c$^>=^Q`38;)bse
z?zTRD%=sWDNW1iJ=c^-=gV$<^f8(0-XYwLff9BRgyStKie=b%xku95hFG4x#f`9Y;
zHe1u5kI%~UUNTwr^_f(Ezxv*pF_EU5<_dWFFhnc)KW~<3*z9PY*igOe7uV5em)^d#
z6PcZt!?C?Z+R2vToZ*+RkMB;H6<MsgtpC)ez8%;6W>v90DU#kQa@#3z)q<x%HS3Gl
z?ELpn^y}n6nNlw|Hdn=S{j)+@6Q0aC*K%!tcS+-=*@<>`b=Mz%-y|<?aqgnuf(t9|
z`SHlB8U>b3Z!d42yz}neK-+#3+pVjX+Sqa#e_8+JS47@sJ*6d&Ra%uhuLcUTtyjOY
zbMq&^T)&H)w{N6xaND^o{6^FQ{rh*SR);07ySH?L=HXQq0f(#qJX)}+Z!_DEXWZUz
zlb1eu9zSo6b*isYgN#47zs2jAULVI>hvz42<Q`BxD&_T~)$w=P?zOqAOIqAMwmn}c
z9vyDlR^<M|;*9--Liy}gwpV?7AEzDZTNa_ef4+_3o5t5mE`%FaJo@oFtu)k^|8k78
z?ad5D?gEYeKX!*#q;Btd|AJ*(7|Y7r|ISTYSLV5#D@FBSy7fn&CEBHb4Dw&Av#b?Z
z>$HQbQL9D6?BnT`o9}E{FT8(Oi1GBI>Xp-V%wJ1g=CVvRIOip__Yq6<U7>q=z7w@s
zwELK*dp-0to%ftozyH^?gNdgeIK8~gb$wAUdy!nz@}r@bp4_^s7tQ%t#<qp0MY5;X
zt##SPsH<hSKkx3j{jzcKY`!?IUoRItJsH1frpeZ**mDh8QMF7gi|x{De-&NbXR-08
z#FgxZ#G+cK2PGcQ!YA);;pJjn?sy}B%~!FyXsh%6w<+!Z?foA&fyVdS3ksfUvKa0t
zDR}un?y&dH<7aL2INpDI-&g9yx9nB;gym&N(iY2nmQc`EcMab8_*V2=0rAE!cb0GC
zmNne$ap1u*8?B1%pKcYK*gM>Qc`@|Q+$mpGFGv0UeNC_W4T~Jpw0)W{Y}8}#Ki%tj
zptk7!v{`TNt$cIt3A@Htarq^IO6<XFwq5GdcD{R#$?#0dc~#-v+*=s}EKev!%{5z)
z8*bBg@`7i|m5<R<56#@>U-aAYXY!7_XHGnRKR^G?$+|l>uXl0S<j<M&UMS@D<xSl?
zmKlA|n#cPp;q0d%?^UW_t(G}YynQ;^olkxKN3D%TKVlMMW(gH)*tuE%ZrGImyK9?!
z^C?3=?n76^gB?xQN?bm$tM6CP%K7Z>L5EFFPn&bGw`Yd-(yyQ2<zKsYw&njFoxa(f
z^~N9b)&8X%^H2NpWYS61ClSvD6iw&lJdts&x}&(%eZ!TuN{+SvH@!U2mgLPC9J2NA
z`SUK>lNc}EwmlN&`uRX~vGCg;zo!MRyB1f!dis)bi~pQ1ibAa(U#`|3QF?nW`tpsk
zjO~lEcc*L<^4=Nwc1oi~%ZXd{_j#T$1$TJ7*|RO}$=k%v=h`=HRCnxsT${7dZufhW
zn%NIu|513oEzjU~?w4<0(zThl|9-L7{LD3nP5nN1P90g>nxZYeeN*A<ubaMB-IVe%
zd#?89!Q45l4}bgLx8HP`OYQ0}u}hT`&NY|FDY*D9o_73X)^(*do~<iaKRHwC^KG8m
z)0^|8u9&{EG?QNOsW|nxkJ*CBa~<#0PP&l5Tw(Tj_MJ7&#RvUV8!h<$@$2^;UG>WI
z`IanEBVGAVhg!ngE7p8fzx+aN;UO*M?`ra&mim>+xO>go@_uI6PHthf&u{9(kJg0l
zxbro1%jvd;^Ucq<oo{<$_&tJI`9{ma_%ol>BbOS^pLO#0ujl7#!?RxU9=LjAdcWzu
zC%3JFzWzwx#nW<?E2jM0Q?^;{_75LTomD>b<L_0IIzzY3k~#LIeD{7&X|7`tt!@{-
zE_)a~y>E7Gzo-Joog))AOk~*|^Rs{A`p3)cPw-A<ZJ6DZ@$QP*a+&VnPQIjXfBB!j
zpZr6V(YI9Qo6WivwU^wZa@1wlXPv*pufF#2?4)R}T+J;{FCGeN>*M<F-+v-_|F<m?
z{&{~_Z{vCrDSQ0<eyu5<EA*x%C3dxHiY<$np_Qa5Hmf>4+hOA?PpQB2ZeQPj)y4Z~
zPUxd28=r0Ly%BXY!%1FoyR}Ko`-_z?mrVPUnUYu|r4eM<r!_rb%bzIz*V=ESw%eQD
zN?65b@#5@*Nk%hFyyk$=k$mTiSx3xldH3Y+wo){Cd%Y#8XywAZC`s{fJ>I#$7U|_}
zRfxHh<(U8O&e65oTBHwwHgRR^&N!Y?5cDGJ-jfsmX6LBq-o9`C(~s9=<sH{4QOfZQ
zKSXxePIhqTU;d%!U{6q0r!4PW2E8EZh_m6YQ-14j-`Do5bkE|77L%O5AB&E?|KB^|
z>%>{rFE<6wN?acLYWn2N6_*a|T)(d%+Hh*k@n!|Xtm*s1ck)kY+9$U;xZ=e>YuA|h
zKPE;N1jcdh))IL*S3I%hljpSZEN+&LpF0?ZT^c9fIFzw)>rt+n`E_Q!$Cq^6wpUtk
zU18EQd3!FMiz^J%mOVT7+GS>>OxS9Tr(7#%H7%JPoW8HyqFu4yl}UEBZnX8n1s7yj
zxVjoh#H1PQ+}JQB;P46mldYUP?*HF5SG;Y}#)5UX-RpPWa5!tP?cY=UP2_lSb#g<+
zvS7>ecN5;M<hn3}Ys)P?z4UsElSd`?$iHUSx0ohT_$=P#gJDxeY%$}s9*bqOvseui
z+oRTtGp)V#d3QqbE8P}_FXz80D+g-2s9w4smv(E_N!{bc4T>v{6c&g(JW9DA7guhe
z!ZK07?v=vATg*oUmUK+y%i_8{C)qr^)g-&w#JiqDbqo7p){kWpTZ$L<u)Hq$@JYq%
z1^eHl8Xu+O|4(8Unqh8p)_nHzygALUxR+>(AA6;A)^~HpjU<)bm5&bzTOT<7xBc_N
zgUbpto1>c-IbMA5_oKm{LVo`eALpApmwh(BtI<|2bLZo><Kdp2oBK6-&lg`>k=#%^
zWol>m3oXWNpR+4?jpJ<F>%-qlb3Zr9KKAzIh5MS{MbvdR#2I>-KjfKWb$iw?wv~DQ
z0ym649=P;hZ^fA@>F=^LIZU3K1#0g8etSmR^J`Q71oVC8xswxi<fG=3qUPC4-<6o(
z^DN-0T*Q0s-HUh6dp1t`;?HpA`te0djgN%m=d?ykC5p}yZ(8ukbob`{ed^pw(gEQU
zPYVCtW3(;DXTs`LqSw_Hr94|J6R2=q%+2-QFU9bsy({i~{aU)KcE;1h>;JA8z5WpV
z_Vc0}nMIo-%%-i|t9gfUd(bKK;|tCQeE;+%I;E#5rq{r$@oT02)NNmnSj9*g>YIo6
zzWtJ%xO3(m^TSofbMu$|IvcWo=IzPa%k@)=UOoxB+ibJIEUKpP<fBPPRp&o7Ew~%A
zEB0z(-+|h*RiCH-Wb_bVO}gbZW1S7-!};8Eu0Nf0aHDTc>(QV7PJ)UpEdtz!5_~^x
zoEiI(#qQ#*t9s0hOqN3N$5)@{lu4K_#x?E0?G;g+d^3J@9V&0R7Ghta9xu+M?vpP)
zLwVKLUzKcyCvS!esV9~F{yi<By7?_{jd#e_tc6bs|Ants{I)B1`r3@z3AU}@zSu5*
zZ(h6hW%gP9Gu@x}UGx7HclrCQedoTEE?aM0JNM=5Gmqab2`;#qW15}Gth2($k>m6I
zvzgcOcE!G!@iT`({bW(;B%6I}QjcCJiO4*EUH)FMJoA2Y?G1C3H_ZKhE5Y{b`%}WZ
z=e#PAQR%cfu=#bs`!}EcZz&hun69JcVtoGGVoiUMQxDJWoHy}ioTu7`YuC<n=C11Z
z`m@<Hc7yFD8Nb!9LlibMX-z!-dB*E`MXcRSY+r9_zvBJAZnH)H-|u3P|Ifa-yy5jE
zc8>FByVSzxTReTGvnH5zhlxvUjAd5t=O&xAyFaH)tLD*`iJRvAVCjxCLeHMIZhpSq
z>&xSBv8NP{o$`IQI{4wvNk7f1E_MoT(le7<e|);9-c_$mQ=8JzP2Y{;r*GdQ-C<Mt
z@AaoOo1!_{_<Ucwf4}6od)k5TEk`bjRRp=Vwr_am!p$uw!4|^f7aK8k26t!F{Ka+?
zcX~wm?418#Rf4_XAKf3xry74wEih_yc<NHUZ<k<WX|vOx+yD~|uTSdBez@)X=eF-(
zn^5xBSrDe%odDzcRmy??FFV-R*u7x?!u(6|;e-jxcP-ojfjb}TKI8>$0_y{90&A(W
zI&dMJ#h~t;qv%ela;tp{;xFjG5dR|j<l>1({yL$iG0A5YUmN7aUoD+0zVgEEPw$s3
z{^O?e`jYgyy}^x>zs+%uyk~bVra<h-N6vrC`d;{Tf0B6G!~OqwdW&y?@C|!T#+4R&
zZC_trO%N^Iwk4!oN9FU6315%#iM+nApqCclo1d8zGl!G+4TIys-_KKfQiC}A@B2?Z
zo>=+Jy+FqQeb5=HqKzuz$0Og(`Zz&gPuN>etxF3heqNq-^x3y%<@cWpNM+}#dOBJA
zp0Z^vtbM8(8Q3im=<S{0DjxU!QP1w^!&_`FHp*?k$NSJZzCm*7gh^gMwT~DY)PJ4z
zaObJZyPv8=q%hcS<bU#0^ZMc5E4w8oSt-lAPcUw>QhRDRLD%2e=D^$^@*j&<_8y8%
zKD+C`-ldi6!ukar%Nql(dqt&2an)?yzHQa?n0@6*|5n+5dwqY)I^juyX&ax5^v-IS
zswQwY{_*!8zr|fA*8jg#JDvNXOmXei6QAdEJE%BMwRyHMq<GCK!8nD!RtGVq$S=+>
zdiT9+P1o#7R-Dl4A*K|`k$OWbXsS`D=I0etR(h?RvM$U`3A7KrFGSi%;Azsl!<mv@
z1&W7SZ?tjEl5*ZBxh!a|$=0?dyD~Sat)BXE<%(y=0upDvnKRq+_v}l%X0F~pa}gL#
z+P^eU@oO{7!XsVD*48ZPuV40lGO!SO-?2;i^&E|te@>LCEH<~Aq$gUcv2x`JbN?`7
z)p|HGEMQtHtTET&C5xAOfK_AA!WoaXmpm*ntb9GW^UkU(^WtKTeY8EW-D5%6gCj=L
zvHhJtb&U&Un~UcNvx>&YWNM|fJTbi$b>sRynb!?ZEPk&F7k=x^`<C0>V!_QT`P!xH
zPuM3dyQr+r-amiWrZ2sd%N6cy4EL(%o6VEwQhqoqK27-i%c+OD-<n=KzkYpPqq*vm
z^qEa|t4@5rrWdFjws88MqqAKrr}{+(-&lNXv-UNK0M4s5hnG$f=N4@~YG%Tl{JY?v
z<yP(aDl$L6Z+)Ag`SQ)z`4um$I<1cirte)6JSi{v`Su^{`0g|^-&v8q^jA#zvbd*h
zTVxMyVc5mAaMPh<+vQHCP8IF9`<S;jm~G1axYB2Le@u@lI4qXp7^yz}!{6hM%uP+Q
z7h~sFw6FQ~XO6SWEP>UT3cMN2k0$6X^q9_;eOPA-XKSEB)zg>10;aM`-QIPKZ)srF
zqs$%d;HjxM0%h-REPO9e#$nwe>%{M2&wPmSVZ(*V4-fv$uiNwU<Fn&k>Dehi^Ot<D
zm)hYs!*7P)x;ZhOV%-<lynOLXH*Wueh&zG#H=oLi*GyZ_@e9<@a^9hM=kWCtbIn_>
z-MjJp>8~h#*N@j*y^~DtZ2sUny?9HF5JRZ;;t3OlTW1A-ZRVMyx{yEJBKqG_&97C<
zPduOh`k7sW^gQd?tS_D<o-cB$7g*1C^S7GwtzW;>(k(UyC2sl|_SN*f-6zje)+-L)
znH`pBa(JC&!esHs(UuOkUpuJZ%89ej+_^OW&8saMp?5P9=j|&vt`&Q=X}aodxuq8V
z8&%?t+Z}y#`qd|%)4#0G+09<^TGlyBHI^wSe5vb;mhA8+=8{T!p(`AsYMd^ozkPV~
z!TrDYAI}N)Y?{3O(<`+#e=b?H`+q(l+-OocS7nLHCF?w;NWll8w{|Ev$L@$ZTV*T!
zPyPFML+h8frk^O9sdns&z3IXe+D{%|&2zl{;Bsboob9`*a$jR|8s<$u)ELg$)Tqw8
zbc5OL$6O6Nx1^o5mu0O~edG1({>r{@I>(PL+cc%-(uu1N7cFRebXsR|X<}!F;*!=G
zVo6dv_q}2dzb1LGPR`#=DTbr)IA=?EXo9YC;=76iM=TQbniZQ1CGHsQczl2N>M*wN
zg3RBZOuJYs!O^O5z?XH$$`5AzT61_*J@zbY@L-7en-G3|qJ@zF-{C{{ih9TFcv{>{
zlFOH0J>^l-6FkeK#>Fx3N%e#FwAD7k8o_gX3Jo7~9GudqWPJ2+NXD`qGae<$-T$`!
zhYimqYkAKqmzSMuZH+?b_^8KOeY9H{<C~tK6?u8)jB`gy)Rq|=p7-%dPx_{O>Fq?M
z(-=?BTvOD3<@FPfwb|Kcq6B51vWAML{IGemU3Jch=AA3{I(HO3fAQwh#@zpNv)X0z
z8)wEZPMN8D{-w^hhSo)CuHBpB9*bF<Fui$o|G}G%Df^xVOMQ)4J!N|E?N*^xKkP!x
z_9!T_zFZNO>M`@_En8D#ez!vT=AVmCKE6HmxbU)Rt5@IqVgH2Z@ZOW}s$%P3%Ot3)
zy$G}u`1w`s!3|YU_FETT+CL9C_KS7u;pUe5(`&qCw!KegzR2Ko;KI|!mA}JZ`zjoI
zsoy$VYT2SA_m|~3E}qo3xm{D;Bx|3_w?pgp)^5-~WyN(nG;{VI|2evIw$yD+ugnV5
znSABkCC|fOEFbrq|5Tj&eb2`wZ=P$Mdvzwe`r2%#^giRyt!G4xf=^s2U1Ya0k>SH$
zJNt6e&31vJcVa!*6J=hryD%17w0+<(<$b}cyM2O#S4Leq`+XC!w#d~hUqr7t?cV%v
z@lV%>R}wCRVX}l@`@TE1QmJMhGvjz$WDoHrPM`35aanNM;=r`WnF-0NcNu3h{*4ei
z^q}oS+lR37<_~Qjp8fm&CG^09wz{JKux+uNvrpMy{I$|}`;N4a$DiJ>IeaJSXOHwV
zwpDo?pjA`64{sL7@o9TihHJiGB@=7Aa`l|@wde01-g)ZvA00k^k0!R|sr`3OU1pn{
zrJlB}SSIeF?BaVbF8|N}+tqDsxv3~-YU8Fk=Px|I%zNPT8g-?YoZD8kyuVz>$~tGB
zQ^C^9F)qTdnA}%fH$HtTrJ?)6G0WnWnk)@rQMrHje?F{!ahj~;T{GvWo4GT#?pzfj
zb8Gvm2PY0(aX96FrYH19W`fMxjK`;yx1Z9yX0rABDdV!N%=qJBwVD48yU!H1xRZ1K
z9qW0vPwQE{YW{{#Vw2Czo|(2YzC`BC_iqIk7YiHDS)VvZ<u=Q_2ZDitsSJmlE||Ob
zviN*oAh!DW&hWWQ%61hqx$75weB2pgxA5tlV9BXhI?rYYKU8J&{{EoC%;taX91h#0
zm9=ZO3U6<&eZ<YrAjcx|>yB~O{&i*l=3ZH`<+zvfgszn16aM^Mo-zM_pUb3$6=F&e
z9ElhG`CT>&99uMh?Ywf4T~2Sf4r(7uRH!>@(NgwvTBCqiQp6qh_xX|>eu_G+J4B8@
ze17Bvm*<LOcOO5P{Cr8>1cwAC^LMX~%{32O%kwQ!!Di`>DTn;I>?PfKG~1^nRK_JA
z+Zph(_{lZF822REiKQvioKAn5CB~Hz8@er9#?07fv!8>pWyRK{f?raXug~aO{Ka(H
zdc)e@m)U0uk1bfnC!BHR`Pr05{HBHM*ETesSjUrAGq?VYe&&vl^m{7TZ#-itu4*uS
zeI@P6qWQCo0^3ft-M)BiUb@!bkB*W-YnT2@5chl<R2uGnZFkNkF{v|?=NWu>p|D=1
z-Ff$-4~E_%M@svagdB9**J;+%Y{*|B<Dyx$@%5Hne41a6D6CnVa`f{wd5e7qx7q#Q
z7a6HFuR-v9)Z&6;>A!;Y6y=H}7H#mBv2ja%bGG^J>$gt5-nN&WOj0={wq6rUw36oU
zsIv9X|MqX1aPG5;3Y{%3GOr#Qh(tV93ydq|)_?tfN%Z;^2hN-SzuB8&w{q+8zi0bG
zoY>=k1kFC3`d{#Vbak;U!^GWFH!R`N*}N<0qoMkpm%D6*bMC#4cUt7-wA9LHA5#`T
zTj0GMr`7x4o~S)6q#=7zL-HbrQ>BzzVRBRWvuDq>uYY-cfA!)!`HMYV7JD4Id_ZJ9
z^D1YPRn8`-oIuFvwYkx2bDb3ahrCB+?;pM5vFN_XqWd2T@}FGDe{!LoL%jvGtNY2N
zlcwF?Vtd5<9Z#wE8A#3iXn4u?Y+I7^#)UU*++1a5ac&UwxpmA_wJoi@$J6=jxtoy!
zMV*S*x7fBs{}F6(Vq<UGC!z97L92aH(Vx$?qW%A##(!elSFp%ye^T+EKkAM8_BC%W
z@{9fY^Kdo)ecMGP8eZ++G|%^cX<2H^bF48`(n9r+rJ!F|K~T$PrioD&Zys3qpE6!B
zRpqK>Vy2p;`g%RHKgp#LQx!`m>Q-&8jc1st=PGJ@$Z^*^xl)&1TE}X_GJdQ%ywK~&
z9F55lcK-VuCv{%E89w*TobtlRhwHl5Ph;IV|E<LdN4tj}BAY+)T)eq?jmGr#FV5)9
zojotm{-(eC!nvv^{Ga7s-e~W%(@A4iU(T=GDLOw>ew#lLH9map?xf(a+W+LZov(hJ
zy+bVQT`bp1)-R_%zyD^idPy>)PwJhO3A_Kfd;MML>f-uGk5A;$>G<-b)oy23MMOm_
zaqU}hmF>9R<{;ick?Ad`GiTMBNqdHPYJQxxW<^0%*rRCG<YcQ)FHB9&U;DXo<{2@w
zz93z%XGL$U3^=AX*j$Tp{;Z)EbNF?S_+=*(?u$At#xfjj->#f;d2vbLvXhB(q>NU#
zb82CRyX>p3!wg5Cvv1UO*(z}Lk&5ljdyzHChCPq}uCfm@7T%LR;oCObD4}w{&{h6j
zIhvc6o|v{g>0#h{r$g)KEzS1pDtH-lR{4g{o0;y1I%_f+e6}5SU^&A2P-en>3&qJj
z=E6J8g?DbB{3NS@?+Cw<fMBqTkD#Gwx2Dmu4iCLIX?(}iK;)Y+%MJ7QU0MF^@~<@^
z{8Rt#3jOvv?B@FD)cxy1!c8Wyqzm4dn>QumPFq&^zvbCN8>37DmY92MoDocJn))P+
zm#g-uba{?Ijb6IVGq&&DGLF6L&31I?-aTHwZsxb;MQjTn>*g@uU*mDU^!%4uUp7xS
zt-Sy0#%z^25u7E}+K<;pKMvlrfp5o$a}id-Uy{A9X_)b?3ORG~>(pAAqYDKS-l*OV
zpL=xv6Z24(BNoY*cjwrwpQmt7rAy_R`O=#fc4g(d_giz<v9ljp^2F|;+tU}Rse!Aw
z!+vT!{wHL+*l6Wl)dnlyxoTo1Pxn8*_V8eTB%lAoTL*XapXSNfE@?L3aG&noCY3}J
z+qa(lr#p?*`(GAvtj}o`-xd+3{H)d2Ct32d<i1mD`(K}XmhyYmoI4h~eJ1CI&*IZL
zqIy*BeD5mNpk>qC7hT=fv}^rzu1>ePqW6t2oW2ln>4@p1_4c2)8(x)8nY%JCBDvhw
ziuL!*PS+JDS06t9$NOHbgZ+Z<Ev0ixd6~-<K&@_27n%3rTiJY#ymuBaU$rDTJ<OVU
zC(FxX=kX_U+=o~np3LNW*pyVLt`y->aayp@G$2RgjOz}?J3@8d9x-p<?>=v}e(448
zA3{4GDTK+ev2Sr%E3j=}p6N^8BOB{m1E%xrTc~?H>4|L3$A{hg@s&R|Ev{b};uK%;
zW7%SPC;6k(W%s=;Ua;;}_3B?;?N?77DDML?@69`L&%JraRfRm$j%eklYt;%@dy2ez
zAEoNwx#OD2ibuhMR-zlK)Q{ixHvXcq{|eWF8Rw>bJbut$^MJ<E*>BvYayw@`ik4oB
zi)i=kTfTJlX?7m&`<u2@E`D%EGpK5&^Q%D3(@x>X!=g6^EI+BR=%@YpysRDWGfNh<
zym4E6GlO$-jLE6B-RyNAHhjr?Qhe2ncN&wG+_L+Bd37r16<PhdyK{o*v$$RVTo+~e
z9S&1HxN*TNu3!<Hmr~;Cy=)10-c6gv$G2p5yt>#=#|zdrC)f0z4ic;3o}_V5IDK=v
zTxibY%lS`cTswH>NO4T8RrRt-rqNsbg?IPqZL@LMcf&1L+i1G4<j1#ci;W~D?PpsR
z&)@P%_IcyeXKN>YSO0PU-@bD{&Nqo~aoWN4QR^8aW23gqIsr%(J!PI^&C!GFBiz4E
zZ1fOwitI>LtZDtx%2p%K`(qnJPk%t}_RxGQy{Xp&H}Nqy>Rmk(wBYpO-lN}A=S_e0
z^^vru`73tL+ZXTe2{4}`@_f_W*D}o_AA$~eyT)==Xzp9eq<eH{$*h$JW;w2k;?Uc$
z#V)#b(XYg~9}HXb=D#X-W^i2+;KQ70X%xyi%l=dTMTdz2Dk7Y&hh{ls1l`%*xymXq
zbmf6RCLhAsOgN>%ZNc_Mphd_ftDFyfeC~3^Kjn{0{c3g9S6_Bdv_Bc<A~WSI<B`}3
z8`dCQ+l4#M^BC{cx4zufZ(Yf^PFTI;TC|VFRQ;n@7w+}&tjnD9FYNV-!`8AFmKE{q
zau(-=ipH99_;<C;{=@%c;Y^kkS$>WuIn%yaX6N$tzcjOBI6rH{v%f1Hel@6ToY?sH
z+EEAgzAp?qybgJvcf^O^n!A7V+<QAX-)p_SJ<YAn$tLty%KC>Echppsc+ZEfPd~T$
zk)%uT!4hFJ+1(72d}>p6A7*&Hj?3lOu}Kds7dgDSe&77p*Hw~RwGQlbGxLvl-+Fc7
zx%XkEj~VxUn)GpvVWhqINsjqao4$%!a?M-TU~}#i%iHASpK0^fPEmeo`0Rsdl9<$)
z&j)SfbtW7v*+27b`Hmftk><f~(sn}D3i)v?ZxKGk`LLy6Q`WKxjC-HGe_0g$A@W1y
z1~?3MD%bjOa{~w#F3sU6<@o=?s?uw2^tOj07k?dNh@IyW=XAZ4JN5dP_T=OCKhzg9
zx3F{ai_~rW(2%%dYggBbBf+abgqY5&zpup3q^hp=wYewAt?Ai*9Yyx;w~Zb@e=gUb
z|DSJRxtOW*4IclR1CuICUaaJ^HI?@-eYsN4_Uh8*tImcj*Q-=;|2Wlk2~)&cp1E5d
zuHjw2Hm^VU>8#3%t9?bQg~BE|=e&$MG<iv?=&Q{KL>w1Z)?IrQqHS5B{wOChEPLZY
z>!aFMWxGRPz4Yuiw2-W_QL*vf^}kXl{)kW7Q)}C$A=+opn@l{XwLAIohn&}!!k6CA
znNq_P;g$WgrkW>s{nd#JZsyqaubFxJO?3CYc~8Q{)w+E<ey-B#+07TQzR>C5Dd)_@
z^$b6kb4oNl*mCRn>&=%+t?uoz;<1`8$5GeIwDh=sA5$$u^92k4^+!(qSGsWbhtK11
zwM?tkJ3eVDRKJmY$~)=Y{hR+g4|uoc{#RwskuX|OW9fg%q5RXsM!zY6zoS>p{@@e*
zX1av5%E^o+%in#!zuhiZlxM#B+vuwcZGW!ZemLms!Pc7s>COvMlP&H__?fp|?tNI2
zT+#G$zI+Sl=-WTMj26!>?oQEQTixhy`e7xT-PHrjgKDxS&&zrg?E5EV-wOFRi$I5B
zbDU6AYM&r;ZTjCOJ7yh9<(jS;lQw7n{cW%M7Ad!C&y(+aRAJ&CXy|%WL^SxRp{r|F
zlUX-|$>jgrSH8RvJaxG!Z?MUkI*E=ZO|E4cGZrO$JKUhRTw?}1Yp{q<pn>Z_8;yt)
zbH3huwfTDb%n8@!kMY?|HB+ejHs3^Zle35b`;mn!UA|nu&k4GR;H>(?$sa)H%&plz
zb%SAHdER@E28VfbD&#uYr!AkHwrG0$?UO8tr$me1JiRZN{OtLR=LeXY{aBS-Sd9ZN
zygIVy%;BHqM$@)ldF8$8v&!XXYIFC@sEMtLS>q>_(7PdH=E~v}%c6&!(PymfI39=b
z`)}d3vFckMJE^AQmE`xMKSLCm!nU7!Uwlb>>aBSX{y9Bp2~@szuHxE{=ACm=mOii8
z8);BFOJ(c)!+dh<azsUBZ`vGR9_GfHBd{Z$QD~l5Tt%far$+kGeGA?^t@QJAtSaLY
zvADC~<Ja%Jc18PtO8of!Z&|zE*S}MEs^xZ^v$b7gpXs-a)7UG0=8f`82HN~(+m}~O
z{pGxIzlC^SSn1yEHLn>q-&^7RQ!-LJKt^?i;m5PHJ{vx{$uQyAk?$3;!H*SF9+|7x
zwfndGxBHfJf{))jc0jR4|HF(-k^2==H+aCXM^OoMbp3&}1K@M3rGG_h>^v&;A|DJv
zn-4VS_(=G?0I@j#w$wT9m$@qZn14R#d^6Q$pwpZ!S4rD}&Z(C+gTUK?caxqldIo+H
zQqwQ|l(miN>C+pFe(7IrUi$3)t<2Z=3;!Lw%e*K4iS32z4c{ByJKksB|0tj1fa0g5
zm#gJdRDS)_F0c4D@%4S)H5&UL&$-^Tg5mL>r0a{Tt}he*7}yv1RJd%i&f30&WxqCt
zc^>jwlNz!xVePl{)%Vg<6Mo;>;lII9b^WWA)>r*Y9G*I;W`9)Mk`r>y*8i=pOSFH;
zyVcFxiZ1D&nzV#H|LKu<_uu8R;SBeeXL+etw=jw>SQz$Y_J!@6T&Ea5)4OqN!JAJ9
zHmp%Sk{N7wGGapa<JsK{rk_3)Jdfp?;R@F*(*@tUY~N2U-754x<jUv!`RA@QJ(v1d
zb>n0CCKg4GjgEqaQ+C<UxnsdQb(4@o_T;^$axWX|g(h8J5;y0U-2VdwYIUbYh3-es
zyTNjG>hh!NyeaEzE}riw(mGT1_UWgfSv{hss^2nfd%oL$;juvDMKb?h<VMV6EIoSi
z|Jm3wv4iFU=UqRPmRK#fGZLtE{?S_}(Q!bLrCmbi?-|4J{tv%RpK~907d3tNwWsS!
zcV@>TG1IT_y0cYp@4{))jZdSa@>oE$+X9Ibff=A9?$gbfPWK)tQr>Wf<BGOL)%;kI
zRnBYp4qCaq7Fe4st0N3qk$ml<N~%Uy`wH3LJc|y<DwT79RwO@=T~z*N@ocLO&AR13
z<~}O#x({X?D(AYd^EIt@>axPMk?~9CzFmIpV46<mR=y;t0{415t}Rj}a=cLz*MIE4
zvpQnM(w#@`_S?_)@=hwrnHlkS28;N#d+x6fF570nu*}u!x?sug_eUnazff$Tz4n5W
z@ahFyH*fwfajx$1@7tB_Pi4z8m+g9LWfAZyb!uFx-^#NRmb>~HTV}-cMCA2f&f2;4
z<38P}X2<f!<-b~9m|b!x{p54s<1XuyMHy2XZ+xA%<k3eT-np7FGp6#d45?4<^Pl^4
zsRzHn^f%L|`ZqIgtd!j!-1VYhRb9@T!xLop?&aOD)$wBLe#M(srK|4GS#hb)@vqVD
zdY;E|dYAUPg<h)+a?P>5Vi0I*_+2LA{<5_%a@cPe`CNAieIglZ<GTFoI_Ymu`{un#
znf*w9*{yq3-kIx<812uW%Jpe`Xx-n~$xrj-B6hH~d=xnrEOEZ9Y+^bu$79W&<p%Rs
zUCG%${dtts=2f=0g9>eb-<)wOAi_V<)<0z5hk1V)I)BW2RL%8XbI)^&=vawev;K9w
z{-d0H^>gRZPS%qfc`s%Bm-P-Y*4uPXtykSFYPHSwMyqr8+?{+QtnThu-t2owEphUO
zk8fTq_SAVedBer><2?(`K5y`CKI?AY$R@u%X;(<ui@Q-}FKknONpNI;$Zo$E(=s=&
z<i?ERT8lZ%cFcBS^X1{!0~~xgC&;H)H*>Ml<;w@2eLH*L**u{`$Csq#ZZ^HAH~+2Q
zo$V$j^TKSOrg^oWQVzJ*U|%C@o_(UKL*`h7TB%&jEmrGmBC9o{*Z+L~`fF)nckHjJ
zl3&ywZuDPqU%u4Ipd&6TeXekw?)_EsJ^yv7>z*yXo+`?1vexEkhTq>CuB(3C>hAY*
z`+Ftt(uZOXKBL_NDO@|Y{7`?nK)>Z=YS-^wa?C&7N^A_`B~P0%l$|@dw!r_{e6M$B
z4u<^=*v2E2Kc&9F`c~2L1;Jri-cj`v{!6Q_>e`p2V-@Lm=5evog{c|LKP%?1b^pF+
z>*MC>8sCp?>)a+=-(Hede`%`JjP1eOAKJ`2x^;in(uX;%8CRMjepLIH?^FL9usVB{
zQBG0(mCP$l#+OVVtM_=_+R+oZc;!TE&iuDo5zLoO4PO3Oa{Jo;xDA$h#&Yt9(+)mM
z_I2hpJhn&w#*Ea=!)etrIu1EMa(>LHuReTCMECcBXY0QeO$xfsfBaBA-)`5l-Z!p1
zk!Q+3l31|dd%}Ch`_22<>-hh19e9|nx@@1qABUYht`7y;oMreLEm)E+Kh61W6}4g2
z)I{q;j%`ezKHmQH#80@|T6)>v_x2YpTb>Kd?-xJte2Er^qScWX7M#g*Pb>PgTgbAn
za9+|^xZtCQfn?V;#hCUAS@sQwe@cG#u)JM+WX_{g>CYGHRD7sN_kDdnW?D_EV1UHB
znmt*OLi!PMhg$bMHGTc@`t`$WigqVQMHK8jcK@XOuF?euUQf7DDEsu3?bWjvjXMoy
zR6Urvs*Zc=7tfDdEW&k+Uwv*XTYHyNNqBnV%|2<-|3|L>o6GZbZ__fd#L$N!+?zJW
zojFm;75H`2{ZgU-I@zhAIb2~2W3?(cnQDG{8>wbQ&Q|!dt^5T?+cu%pEkE8L57XaW
z=03fXO>4<q7ZK*8vn+UCHN8Ao8YALSw#BU7;rA6qE@R(tnd6(ZzPo<=dg|V?KlkMp
z<*s#pJ;z+x(M`_3G5vyGE{}Ql2P3`q&p~f}Liv{kW&30+$BLfZ{wj-0V#>~E>vmkw
z%KH~lmJyOWbIQqiSs&)Hhi&V>TF4&!Gfm50%KP}s-ND=|9?M+OWcng-&tS*G{R`&@
z_~^~e=~D|o_cd1S>E%Of#kNNY=>3gd5Pv7@=2g~a*1w?Rn-#a*dUA8>&8xaUVyceD
zd@MKOYH>Z8vHR_Yt;Z#{nK+c4f4cj)%imwSUT(Ojw{(}f&R>z3!o@yug`7c6U;2)h
zt8Z<5_VS>o$GttppU+us_{Y6scYVRP8wd2}{_^#5)m~^R_Q7QDfv9B84+=teZurIL
z$>|%uy`=k%DSCHS##Xyc#@xRzeQuttJty|z7p~{xVpmSbJQM$>DskfC&NlZCt+gwo
zB+q^Bkb5uQmSEoZJH_L+=)D*@UKg(`scV<Le6Z1OIm?yMkF)&}r~f${Sbx-3Am#%1
z1+B{-3E_2V#>?|djY^_7`>eh@`H;HJ4fDs*t;c(BCHwdkO9*Neosm+?SQDGBdTyVc
z%B__y;{E@kwA&9%J+vlleQILU+P0F?&<*Qq&#Fo*7X(G_oUfC!x=3-|sf-^1xdoeq
zil)q1v1>iQ!KR+qH5<=tY*r~Q-zj|WYkTCo|GWR~J-sq(vc{DqcNZ#d@B4CNna-O7
zS6{TbE?f9+qfT<;X75mgZ$}E6>V+3yO|iG%Sb5e;V%{4owRI1K&*n{1%6#-E{P+6f
zv*IUCIeTOO%1h?93A+EENA6l;*yQ8-d{f{(!}Bp>JkAC@7Y$kt3-CKT@CXV<mcD#^
z;qJu{ZRd*<|6Xs++#DszHD$g<Yvs{XH{Ac7E;t?V!RMectH03ftNdrT-F|hk<nFf1
zWjDDvD(@t?To&;SHu1f<#b@^2OB{EPJT#cFc*Uuoy<0POajxC9Z|>Do0bBkqDVTjI
zmutJ`t!1-fyLM^5T9)~3gYZ@UvcMm{vR8v9*K-|T!n)^%lPGhdW$ROsX~7YGiI%OY
zW#Qttw<jssE&nhlY_~_ERqIQUWulz*A%3gdT+bimbJvpVX^^d7^5)tv4$Ck1kL-*I
z`pCaJ?97U-1?;Df>^k6R5+Hwkf6=C$GjbT$Jhi;JX~O$yHR)CAK1JWlxo%B1vNUv9
z(p~Of!>}WH*MZ62`d5l)sOMCvy3KnU5Vo%1e16K=6+8aL)xZ4uK3zjyuBbmnZ&Fxe
zvTQEf{w=%fqFjw$uiELiBys9f?@Qb6sqj7cbjv$lHAc+%|HF{>^gD}IwdyW=5_Gcu
zljWK!>28BBCmpR`_Bm+O>b-k>KhVha>ascer-mGywq)bJ<CX!NHl;3Iq+hys_fNfz
z$Hn+FFWRg+^1MHX<NMO-kDTiD;$5bS#GYLpbn+1EuaHBwilLEtxf);9A1M{;Xt&K{
z)IY<-J)8Nn#kowK-s4lxq;}h1)wTBgV!id;sW?~W*)r}5VT*a~9$sD$aY29o+07CH
z6W2{yJ)gDYU&494-(O-Qg?6lXWYa(Qugkox&mV0*Jn^|xe!~32e(iJB*1p)jD`<sd
zRm+F1_m1Dce*D{NS1zeUt@XtzJ!(I<2z_&Y5mtX9L55BJ@#P5zE#9Azi%^&IHM*iQ
z)6ZbX+jF{$quIV!zBaNre&u^&K|;>0rJ9R17iaeS1-;iiT>0#xq|1G?N0uU|7n<-b
zEzl@+UMaoHd~VkJugu~~vA$Dptlf8U?awP~zg}6(f6>2pdEVbxNnN&TL!)lSLfNEW
zm%qu~n(6#FX3gccwcmtwlNNgFY%MChc5U17>FGD44(yuAwl<P?$F1VI$ClZ!vpw&5
zAR{#Ab3<95z@OdQmXzct*UEK&G0Xn6?X6|!&9hOWTD42!Yn3?d)3aTRMNDQbUUWgm
z^pSty>Fdg!nI?>dD}Ob<YV<nb{&4+@@Jre^roQ$nY~8Wxck#A}>jzx*e5HTwNmVW@
zNqYU~Mq%9VJ7-#E|Gj;d@$r*OtPkzfYWwe;`ugjVe8!IUtG4g%3+?;2Px1Bglpg|2
zY++OQPk)<ZR3;u?e*b!C>F!ne;aA@-yY<?NrS$ZvveVN<op!tNS;&{zeVD_3qjiVK
zqk!vQZm+&}RAtN33)7UO&uzY&<R7~`aI3pc!Si+-@0qJk`Sri{TD|Q0^Vs>D?Zl@S
z<rah%ND8hwDdY1jL20SR_bZwp6e2fY;2J}~*OkFvS598OE%@Ea;CCy98}5CKSQ}Tx
z`~8KD`}X$+Z)1{FE-#QS6}h(Z&6SmJu58r^y!f&*OCxXHp2V)L9glqXa)pOI=J>e%
zq^$b%i1m*)Cv4d9_)%Y4YD}MQi*#3RxVh{N(3$?A6I%9~zWnyBG-luR)7wjb?pl}k
z`uDDy#9t5F){B}?ne}kq)|ho2wYN^+pMF38c5N7gcWY-(-R?ZOJ*N~bbGBdqxmT90
zS<YsD^PLHg&sT65YnO(6?9(i|@L?9)`}8TZ!ZS;?i_Yfm;uUQAed%@91n=~~mosfR
z=O#&I?phM6+~UsIUUS>0x?6Qy(}yjNKfHb`=e|EGD93L4`P!Y6jOTOz1-!DH_++!v
zjE@T<pFZEEwd0dw^0}U0nISvE^y~Ki^HqBoI4A#oac};^mdN}+Pjcq|_<Huzfmiuz
zEVl&Mo~JVYeIj+vQ{HF2{Od_mUfL?gNZI_|y3cmPjt=t`Y=;(T<YX|${k^i=%=guy
zv-N`M0lstm3=><M1#&JJ2~RHDI_vz=xdH`FM>sdSEj0?&3jUn(@2~e%&4cC%-V?IZ
z&vrU$^!@xizx<8S3!@j`B5E$#%#>MDR<3vT7V}1~ms4h`1}`=051Oepb!DVcs9x|>
z7%^Fsv*)))Fk3L&=L36Mm||s?9R4<$>qy8>pIrNulV)noTzOJG)NHlK<Z>}4XAT}i
z1D1mVtj-cWwa*T&Jf|tL%wvX<V${me`MV+uSGOuwCH~!z(=Rf4B9p<|Mf04}<N{}?
z|J-0?ICs&?kTc3+b65VH+JEZ4jZ%>0Om{aq#mXfqb*e>k1?=ZPwb{k7X=U)nP_~Dy
z#hQwHLf<W%v#IOD<FwsoCs#lFez~W9>U?qIlz@Y23ueB1>9_7&Yunm7nQ6cO{hEDm
z+ft$VrD1Bdob5e2zh%=Hj&^$mCj=*FM}4}r@xfN%d1V{RX1N_#dLd+T`|yUF&9{>u
zJ(bOSZ}Zpg&u_c(eS6p4W}YwW91x{y=YR8OV{M21tSF1$3wi4~W-{5o%-A66xH&)P
zVRYoSS<(N4?fqLfWmeSO3bl_uetpwk>2LE@Pn_g;6Z`u|OnKR3v0qz*UOb<%`$4*8
zVmCu{$&NDTLakj<MN_(Fek|9Lj(_w<lYh<i5-al{o^xf>TC5Vdj%8jtDl#QFz|YZ=
z^{L3@-mhDXqP8y!+inzfopEop+3IMCwb2r5=g;_+m%1e{bxT&NM#S=jxlN|s51ZF2
z{@2(YYd5$4m;Sd^_E%4fU0wYzWcj}kR>zAOt31NRcx9Z<fB*eHdd<_HuMBN+`~n*G
zHuP>S5H^#yec@X$cLQixD&{7y8L!z?!{f7d#rBtJg_UZBm7aP~q7`QA|87P`!^;mZ
z`M7hea;$Q6IxfpfH!JyAE$e^NaP}j2X+@&3%pAt~jPn_h5AVMHon^kX+4kfb%a6;8
ztNzR_=@*%_?OJKXW2-fVvSDuhHuwMj{<NF>az@D7Am7KAwlC@{+pJU)8rd%!()T!6
z?$^Z@g_WVJGGnyZjJ7dVM^x`Ma+Kb3;m#GY6_H1_Y}C#<wMlGO#j%xtgzCdAY>paz
zIP}O{?#l0P@A!}3++RPH)A@0SobXk1cISZX)2C&(e0jK*E7R07+jW|H=&KoDVhW<F
zTGH>C?}}f#X`_Bl&fincZkvLi?1-PV+HPLfoaI67SIdKUnLJbJPuUW!wm&dDQ_H!u
zQMrumuigIJ6HBcO>h8~wa<*-syzI-fD#_(XdXM%>hXzV@FPSK`E%@5n;A?9qv%k)a
zP`T`C6sZzyl04Bztdm^@baMH#^u&6fjoa_tkT?H!Jjb$cGr!*5xu^13Cw;j2VdJk|
z52u4hqSCT%vd8l8=~}Or$MrbQ`40C&ts@Hs97<R+q*`W4l?PwW%ev+3yL)A(x#;U<
z)5HEM%&_iwsBp}tF67&liQMgdMJv8GxAcXtQD*34+0f1xWTB^ZN^V-n>Ij24j-xlu
z=3S3n5q)Rn{6&6u7x~>)^1CbL3qrSi?ne3Co#g#@$)-2Q^vk`qOTD$ANc&RAYl$T@
z<}a${`aVO$#7Cv~S<m{?NBsOj>rxaWK54I=I9u%e)-}_f?zT((xO#)tHO*D2E44nU
zzF@oir7_1T|Ex^?h8(3Ek9S`YoWYV8_gu^Nf9;{Dj>OI*8+IhkQ)&rwI1}7?sl;=a
zby!aI|Ib%cZr+cYzEJC*E%W#LGCN<dd2>xZWrN^~{N4XHP22e?yz^gd+s;_l>*j9T
zVvlY7wAAdIfOv?|QQOP8!nZba{=F%>#er*SpML2@-bLCRixt-Q%&YyoO+o0Z_owXt
zx8|^iu(0y2H}<c{`kSG;)N{hqs12KcaNVC|$Mebj{cQQkXB4m3uHv%U+>r4-y?;|-
z#eTg%m)1|}7H9juJi~i()3S$eU!+v2T`E;M-Fjumy8V-<NV4{3xy==Fy=$^!ibj>x
zvhw3co*fk9YYEYM6{7V@MD<lg)c@-|8e5uYe}1>vGA8*)gOl(N-tE$P=F{)!>3^2V
zGMyij_2qBN%Ce)J!E1Zv?@E|1*1wUNb*I2sq1HL^R)7Dq<H>XPY&jpl?w_~yzooOE
zEZrKk`k8v4Wbo=#nHv_KYx(PHl=IGiwJV(CB%s8dwtU^qmDSsBnp&66vD=-rXZ|YP
zug&7WnhuBm=3VpU=|AgjHas=^|Nr~!x9{Z#+4((f3u{e%#6EWx>TSv1&ay8#B`SPc
z=~}nD$|r-jF3mai{K^sTY4@-He8;=(`uoouVfo62J-h#VuXd>2A6pO;&UIyJxv#F^
z>Vs34K3Qe^({<?&^%*Sk%UAHeRz9kgd{#bEb5+SvIrX{67Tvk2bm97g{?-=@Rn(Wd
z^Bi9n=_|N6^j?P8zZEn0WL4T+H~$w9?{Vi^*BRlp%Pwm;{!VezD=3!mVG8G!Y+U<F
zW}48G+Yi!GPdrNv`Kicp-L;p0$HnztCvrBv%j2tGR`Yjak4#zfx$t!<U5_FRSq{1G
z=s44vHKWdWip!x&Chx8bH9}v_SEXHl5x#M|E8oY^qc=P9)@>;~yl2hotb#RNiYqfd
z)k{6mD9sG@Qw}<Nwe|Z3z3-pjFPnYZ=?SlHPps=odDoLc4trM{%q-m>Ewy6yr6ti)
zD{k4GY7^&Ds(ch)GTHB-%F!MX<DfG?7C1)wuP#5Ude_;}o3YfJA@p@=QtP$;b#1A2
zQ?uS~ifFug_{6Ndl?##<q<9=u7Ui9{@k;t?f4{VMzy1^mZT@#=c~Id(lL=Qiu1fN{
z6|G>BW1QUJc|bKG`QuU0ea1Ha+EY5&ezn}wVdbpdd|Jqi<80KL?CR!234P~ety^Y&
zwU<3~PImY4pbalC@N=-YI3IeL;b;Lm--L7LN8Z1@wW$Jcm_K@Naazvdvz&vck;f?Z
zsq1c?&Y+E}E3UigvIyyYNln>!(4D(PXO-9*qmH#{_umEArzi4lThcc5dDn$2d^i1A
zCoXz^>8#ajt6$qD`=o{5WZ3XvmGt&5KaXj-n?7x>R4-@>Qk=SWRmEp_`DQ(}6BF*v
zS}>PQ#`9wvpL$eYMrPL2{r<N^{BpFXteJRgA=~y>j<$sYt0yg7?^Tkws&6;ni+ME<
zt}eK7=GXUQ%hzy}ZJf9LP}1BAiIf|cS#Ce<R_Xnb=w4^8w^XlidHVl|sUIe_<)5@r
zyZ<8LR{i0}uYP=9xzNFNg1hJXVwOIJlN*kxXU<uiWLjk@6m0oxUx~;1%i+sjr)<+x
zf1h+sFYPUicwsJb`{|~gCET3q{P$D~V(x~m*})<`k>%(j{;eAqnwkCEck*{j>*Rd>
zBkdLpg7fVEd*|pq{9g0kVTSFC=kI?f%$jqwQGmlqf#Z5zMVD-oL(gFc8IDGUm|unR
zTc$J~zpu!3q*mgG*P(Ku&kt6!1T)FTSRRtj%746rh3EW-wrOn(qs&8Ix2<Dz{+hSq
zUTCc7{Hsl~A4d0nj@r(%J)*Yr)4sIg6FjPKK4_fVrFnP4%DkIXASh{p!AXlA5%qXS
zrNzpgeHxzXL2?U=mYfpQ_ETs!(z-F-CEePC{oM4Qz3)7~tv>(v(eACDcV=yH2~!dM
z{i18$+4o%%abCWR!uzAsGb5!kqsmUM`S65cccL@nqOOQ%ETzSJb_$DD72E7GRIod{
z`L!0av>eY`C+2EV<BAIf;vw&zD<ogPc~GzWu3qskz1x5ExT~*Lsck3_UM+a_!+)=x
zcd}N;haLTu=^zs3r{Ke7e?NVrNdL!jCf<4LfB()_UCn(saecn!roi2+gcFWx-Cc9^
zY>>{O*)`#l<2_xYwlR6n41Hv_Ug7r5zwVrik4K2-)vPLIkDR*QN^REUmnK(aoX>tt
z40s;5(Aa0ii`HM!Tkn?ri{!M?*(!V`!!+3b$T~4cUDmB3uRS;F-k$lSDP|SN(uC=&
zoL1^h(Q?-0NV9((Qp@r6Qk4DODEqq&@3K-gES5c3k$8TV{nb+et}l1~`S+N^__eU~
z+xFb|E6(q>ORrt`>&^sE-k<6T0>@8zGj}(*9{}ABbMX5wH}<_39X0u{X!3snT_^=T
zb4Z#$Wp+QaboU0c8#QZ|B{)4`lWy*@&M8T{c6Z7A#76K8SG#Y!Z~NJXvjqigHEcC)
zhq<A*qC9+X=flp2;A>cUXWzcgGW#~@X4ZUmgE@!Se2VRz`Z#Or1L1E=%02(h&}nPi
znD#&`rN~*=V(JUQ?;G0-4kr97F69d8-F1AC?as54@0Xgdc+h$FlGM{hKWenKUtY_(
zn6q2g;_%7`44)6R#4oA&@O^^gDwQSm`I^no<Zt~dbiRJ+(^ai`*QCD`-<{@I=N&(P
z4P!*7*|vGToG&uP?u%_&!!vj8mx3erUS5s8^-%31D{tbxshXRlKTM4Z4*X<(Y1cl{
z89^Uch@J|oTWodmcE>GVkr1n=f7g8QWo+k-$X&2>QqdLG#XC$-Sw+WipKp1-dHQsw
zbQk}K3+p6zWq%F*V0>|VW~W}g&Ux>m2Z_&@Obb|!gwz#1_gji=-V|Uyt>cu_6}y#e
z;Co@bIU+tp9+E=b2_ul;)yF%ZcQ)^AT^qR?5ry^56Q&DPc21nGP}w<Yx<O^<<mnET
zoNOOkWA^&-J(Ig&yP`VbJHxt;)y4PM7T;ex_3O1$RjaSxfAw{{U)}a=*O)3TyY?#n
zXnXPdy!<l{X34Gt^Xpx0YOm#ol}XNv;eYgDe}>TU|MNY=-0uHRQQ>QMkWng>(cs%?
z!SYzbJ>+D74IiVW%M*(UGD$@orxdl?H^@9Hy5s)i<Ohf1xsMHYh&{ep@;Ecidh6>i
ze=LLF95enj<>S&rUQ6{JT#EGFbZV8_`?H;=dR;f_>HZ44cW&*cKD+7RnUi{Vzo>h)
zYKnJM=FK@j{P<X!Zm|TXXMUP?@6V1`hc;E_8Tsbd_-wacSm>m>o$FHAd&{f$&HwHH
zbKtjg?afuXQ;%i^)W7BISk1u3ENXL)bCyZe!38W*dRyxPi#QFhP0^OSa<VQj;JD`Y
zRY_ON!!}GQt$67FPsRI@_D&t!HXC+@tLyUbHDu|ZJb7>9gurWJjRzBVbc*c%vG-f*
z`TgqV8n5|6&G#Jmu+e5x*{|xd`t7E>9y^OwOr9><$m8sGR<7r=viQMTvBn9qNo5?j
z6tmhZWIwN{TeqxofyWs>M{Ae20^6ML@HJYyyo+G(x^_|R?W`BFhsrzdgD-Ub?J!Y7
zZruifqsj7rL+*P1@q1|9nlEHgTI?8FecLZuG~}c8<BVNSfzau?N9C)`ms`wxR8bfD
ze6e-?)O1O?uRmX$cW<<l`V!u1XSD?+*Y)^jpOv0-p-sQm+(#8%k2U+iTDl&ubeqe5
zKG??pqs`IJBIko`&exelO3Fd_GU?|T4s%}&m>)U+MbXydE3*RU)|iLttP0<_!jDrm
z>tf83DXW8=EEMnlvHE#z`ak!N{!2~wo1R!UUEvh_ezp5*YsB}6wJ7hoZ+IxOq<xRw
zY-Lvympt>_pP$=q9=EYlJ!`xF`?0C1?dzwkwct#BQ5_Q0@S!~4==JwmieER^_c`-j
zuiRC)@4s}{<ZYJg^1r+{-P`qk>-zmB!m~^cD{oUU(BoLjJ?ZQb&1)qy72XEzYZv^t
z>^-mYVXe$B8@^r(ZFpg{{l!w}fC`DI`Ul%8e^tq)9^U_jDa$@w>B1`Y?(FyVzRuC!
z`>v|+DgFKT``4K(&x5Kw64(9z7kI3*YjgcR2hXUV>0-b47SG-gC$h0>p>Xonf|KvR
zB+h#A?}Qh}bf*nsPKQ_D5;?6H!GF-i`-V!6R&?lcwW#H4w=N$5P1k{XpLV=<Rmo=J
zYT%o^rE=EXa^4~J(#<;cvQ_Hkd7vAdWzC`{+rR3NwUafwo5}|&Af><8$P1V8{^<G5
z%`;ai6prqmc{KM`h`;hBhk9oav|sq&N5bcqA>{sgSDhO(SlddxR0Ll<VK(seQh)6I
z{L0MDiv3nyA6z6s*F1v`5CY>iW>%34eQF+go3r?gEWheKzc=;VuG6t!pMKdrr+#|y
zv!ZqP&;4EcH(_aUUHSEG*?V;NTYrz`&MdPno*N$bUgd4ZgY`d6*3LTR=h%}lRnxeC
z|HhWtPmc7RI{WjK*6R{?XT1)&AOB*G=;u^@3#ijx8**+{)p=3TsK#qn4__{sw`=)w
zzsK`)j_Y3t%in%wuFS-&y0d-WHi`95{{_4)th3x_P%qQ`*?eE-uM4Z6UQS({JSoqu
z!1`oG-(&%ms^v8{@9fWMrmf)Tf9q_%#$esYXq{7czR$nFnLRZkB6VuS){n<Ze596m
zF<kU*{B|%SzAh(Je;dofZ+}7qP3J0AZq<Kze8bs)U-zX%TOa(8bFgIM$8?GPt-q&P
zdcIn%If?a@@`U{zueaFG_|1B^c^7*b|2w${wkNc|wB3+quW|mu`_TNzeSrpN7CuFb
zBM&STWEOp}U;aZ`EJZ`8J>buJo{u&>Pb!RzyN>@avOf`^^RIl1`nwjS<vK~9?49TQ
z5I$7Yaaz%!{e+BDamQIji_MwqUj$lmT`rVY)Nr0?)$&SU_2Y)pj%(oQJEpRZ8;1Q6
zQy((E71-*0gRjt<<(<Gz=NEh%t@$dBTTQy9*99pvoGLmV7CP9td=xn5EYn|`@?Y?u
zSN%jA)`D`U%X0<RuNAU-^ws*Y%F;DYgZ!eON(FYG&?;WiU~~1&>weJ>(cgAhRn}h$
zea~$^&sC{&a@(`OqWSx$hQD1=GP7BuRVaC?&7Z)VU5fK2Jn@_JHMMa0qT6>=4sjip
z+O1}3(e8L+Nx$#Ajrw}B0p}hCw8a=j=Ei(5^LnXvDsS?y%*l_I&s**JP&wW%@MzOp
zFYnLypKDw+WXo*ej@L{)-uPHe`cbOowOu_{U%Jm6l<pMMTV&K<eqiaXEnnZKO1+$Q
z_Vo6ErcW!yw(tL{I)UN8PnP@oH?xv<{i^({@cd!x<Cqt`dDSJ=fA6Lgcvt)vSn_Sh
z6~5`W?yrj77UOQZ*Y56e|L-r=XMZc1pLcRQ1kHM<RMvGvGiiCpT=z)3TbnxOGHsi<
z@><htFj^hf=AJFKG5El;lx*|O>06CNHU=N@6SQ`HBeJ1aazjSU_Ddn#Gh((+OPGI5
zAi{cGwDmfPjysw~%TLTzD(`|Y*mnj$*~jqz*O^~WdX`>)zQX^^F8L=cZCf|rUfO80
zfS*6}L4)+2r0~rFwPL$x9bMPvR5aJA$e&sI&V!CKuS6!jMn+SfR&hwzK<|~3eywD3
zV~eU{ja3b3#A?@JCh6K_lZxO7OVjO3-8NkLF;!}R(Btr|%U-#Wl{t|s&uSIcSe?9?
z8EF%;|H7gvQXO56*7he3?75W5v}#s_Xl3+`pwAkn&!4RPG<R11yw^L{*eN<rbnNB+
z7ZcXzq$O1<yUz3dk^7hAcP@zw-1OL|Y|hC(<C}$Q*5{T4lt<lf+9J5r-RXwn8-X&7
rU8~eYzBZgUn`F($;a$S9nK9|X)5dS@ue<|x<u!z?dB4TH{=hc?RRElB

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ac b/third_party/icu/data/icu_conversion_data.c.gz.ac
new file mode 100644
index 0000000000000000000000000000000000000000..de9b69ff9474e0c9ccc799d40d092d2ab2ad98bb
GIT binary patch
literal 177696
zcmXrm;>(jOblCU*g~Fs2$yZokvhK2qnRD~m_Y2`}q1yVEQu}(P`D*2z*yry$c=mgp
zRmk=7FEy*XuFtcz$mjRF(fs+-0|DDSzbUrf%h%+_v#prayfrv}U9k4j<Vo+ZOn-f4
z`s*ojH5}|UI=3_U{!5#g=j6S}f1!Id)OC~9*Qdy6UR&N7HM4{1htdzF9ZEZNtf!UR
zjN2?UZC!AhU;o#CU+2bbFf9JkbhgQN>de<1pZ#X;^jr4vU8PB$`P{|w{n@)xpK+XH
zbUJ#$TVSutMQ@3{yVg1%F??!yE8@RSf4sr8i;a()kH<UMb<`;QO!&l@8fe+_^aS&J
zrs9q>iYDzhWSWXQ&MDfo|Bzu{;M~He`0I7n3m@+15pR^`TdL)KPwy3-x+^qgmuggB
z%z~_w#dSxI_%Ewsb!2Bhtx~wMP5!i0r7?@xab@$xTbAn1RS8^}7qwP=+sE(g?GBu^
z(+FwmI=J`$+k>z9%g!x5*LCfZJ(KUETf1-hFRfEA{@{A$o@>DUrQvQ<n>XZc6L;D3
ztLW*31wJo(H3RO?tvFov>Q|uHk7t}^oVs~Fy!*lgwcJ>){QYrj$-L*^UZ&oA8`(Tv
zaC)iinT@&I?RQ*xw_?Tg?>_GrolP#De#JM{A<V4iT*>@5pHoxS{a(Lv`JCvt(%JF-
zl?9&jmg%U)?r(hl>fzPi^>_X~>z|xIdH>f5C9T{u?#~Qf^*4TV^zkQQ=N}ng<&fjw
z^uNwP;eN*3pl^DQpIEGWab?@fIej+EcTPTOZ+SV^gf(i(o$F$L)%(1;S#lH;xEZ54
zZp^Z=xwWF~#yz6~)=giwd)H^by0Pxh+Lg)KuWrn;@?-4|7W*f1%lfFApVODg*FL2v
z7%gPt*(J08-D#sgvn*38{z)5^n3RUxus`Q!Bm;xdY};jO4R6?N>*I=^eEiyzji!}3
zzH-0M>%5Mhef-+9jkaIz&a&C2*1z^-a`vYcXZOl1kDX&Q$0!B@(*>r>%(c5=v(4@@
zlJ3ySUVXRq@+4oalS)_1lgyKhGMS=P_xFm{y?IQt|8EMeesyX3W-&g66R%cEFzlT(
z-@fSW%A00CUO&4&+b#a?d2_=<M?*~)o_=}u{$$Zt*Du}ImOo_irR>e8WxjUeEN$<v
z9n#u%f$@a%;!B?oe`)eO{=rqEiihW7zJcC!z57QO<kxRVmvwtNm#fmoqOQ)(>&nrm
z%EEQ+xr(ZM6V5YV{i)6uFJASmwLJA=*?O<K{h7N=PTLo$_eD%?zb-gI{Ls2vn&lxM
zmM)5xust|6t1@(-x$hh)iHzp5j%|m_tAGFAlaf%!aAno9Uspor&w7^RnzmJjz2#))
zk#lNkW`@BNZ{B4OdtG|pC0LWW`23tEn~3i^3^&exiF&&wc3b*cj@K)^mo=>2cJq`$
zipHPkjrCVntyN6T_V{>VXLZ<JU&-6Hi{FX0>9Cz*<k5UKX>C|rS96HWl8pIDuamPR
zQVndbR<a*_-YjI_{AZoNaYmj@*R1OcyrX8Ua8A5-<y3uOoZD50{}*4Z6|(wqwP-@~
ze%^m&#gC&8&9?Wu;k)kQ<5ww+ZApcXQ^UnuxVu=7E?ceM#MJfpyw=08h}gv0_4m69
z#LT(RzslU}w|@=S(WmTkjJF>jZIPXBc7j_me(f{oyIbrQE)M(QX5bfh=EbK!lM*LC
z)O>P}k2(JG_NsU0?OVN{{z*^jvFJ4P`TkR8`CF||+fIvYSuM7kQ~cDU&8N3*K7HY{
zO3I56G2P9Fzd3v|`{xip_0qM-YZ9zXtt;jn*Y`JHn;n_^UG@Xlr#b5vDsKA~#%Z<g
z)Vxs30*l8-YIQCe3ky0;o7-Kpj{j)`b94`H@#4k#`+PaH7V*tL(4Ob}d*fd9JB3Z&
ztUimj+PvNNJFee<^3}e5p?~!@pP%$|&10*NCK?H9jQ?l+xqate+BdU>+kfuy{cP))
zQe0ZU&vIAb4-bhajP6_Se%m&8+nnch8W*i+eqQKn)v>4Z-}|FezVBvRXYp+Qx+_1X
zRX*1I$GGp>mlac#LIW2D70%e|T=H`|)8){wrFC=dcRl_3IlkPG!%Xo^i-mC52CiWD
z_L)nZ84fALulye~-zjKD%B+-j@mG&8`$_sro@M4(wwTXx&6nd@Zzrw@dzG}+C}KIg
z+9tk@y4Mco>x&#b<a0gKZi2`zyUbUwO}>_PC2E!}y)iZIh~Gl(vpv2pF@F`Bm936!
z`(ril)QgIFaz5UMx<|K&Tn>UVPJi;42j)F@)hfI6b0XKgl6gy%+CM2ChNDf2i-lJ3
zcdZWXosg(Iq03<5!>)k#M;057E%kB_IXv~p{h*}8fTY9+%M4F2n>Wck7m=0|iK^f@
z{q)+${ioD-oDyFgef8ndr!x;feR%Y#<k7{+w(;SX@2=iFbadvyqnrm?PyRpjq(1Re
z{hYMY_gAlP`<VT5)(j2(zb%bZz4yN3l40;{iQ&zToE*`5V5+cN=gYMw%;lB}^>qgA
z842%J{PmOd{`EZFGxYYJKa=WA+tqR|Z~H5~X^YAa=N-1MZ<jSB-MD7%8{0Ga@swmO
z&(+Gy=bc!dlylL%x<bT{C1Jbt@@tttBEE*d7M=Qh>eeO2|Gzp%^**Y5wAgTCgKpj@
zOR<ex_y0L%cWnOlXPdlD<N89x)_7^&;?-L>-@<LJ!rQr5zP)O(UA~mPP{YIg^m%!`
z`ApKmj()qEeYK?QKhF~VaPii%jcYfuNM#@YG;OYGiexWeZR;}LNt=0!(<ir-C>XQP
z^qhZe>1$c`a+y2H4u!@2(ubtGq`Pjr))%jy9ewx0{R{pt)W1xY_TO#w(D<QoA_Q(u
z*zD5H`mm|sWje#rXP3^dHQRoxtTa1p=aH4(ExTsr{{Nl#{n*Xr8_ygym#k^@TVfp`
zZp0-id!bvGxlsR;!;%)J1ddw8Z><l?xsFu6k*ohd>CY(-2OjkRi^fBeB7TZ7rCybj
z9H(F46!BY7A7_5AAjYUbto5_{9EXQ2k7WL5rf!I3H|JiqNKgNdhQ96UkIj2N)a)s+
z*<;fozpP76ed1#S_b&#bAY{B&y~Vt9&9{DujNsrqPi-zSg|n`({d!~h-rUCg9Ib_i
z52PNQn)Q?U*!L$F{jYegxNY}#(NV5Y`vnc+G2y3Kk7&+b@#*K^G_S_Ur>W8@`L$9L
zbvCHH{LuEX(`NF*OFu50(|Z@YEzT}y`E~vDk8c~#*3PXy-0<mUntV;v?ui;}V)uTJ
zdS|e}?PKn)JfUN8>IJ*c-6)e?{MO3#_MFR0vuc=>ICE6q=~(PynJV^LdU_6DXV&||
zLp-rd-8U`iez$~m&9%)-5B^=zTKsCa+P=%8fhXBKI(Q6?T-?7rw5Tdr!QNg~@M49~
ztHTTJWL_O!C>O(-m1@xcN@Zbz1XJQn&()=xTs==r*2t%9===0^?(`lP?=O!ZMC_2@
zn4@t<ul3=j72kC7B)isVZmPTVX`<$lWdgIEa#jjnUG;=>Yvls7)>}P`HBSYs@Lg!u
z8f8*A`K1{1M)Uq-nrzPlZCf9RG<|-tKx<*j9$7b@5N+p;7Ojs&s)AE|pMF+vy%FNF
zJYcp{4%fzvdv2^YZyRrkWCeH5b;|2otbW<~LSP1qX5>wod&xTcUJLJG(Pg~eyw)lE
zzyk+nnRiCN-f(%|k!a96RP%mQ^!lJ>W?ENyk~`CnvQA%J;mVQ6#O408J=}>$q0YY0
zJbG`~|DN^zS(y*D?#Vbwic2h+RukppzS~Z?M?3c2A>k*NEI%vNR!ur<t9fP0jH^PM
zqk_I#ti1On;GRK#%;m(}372=KRk_qUMmpZSbyY|>A&TdNmam5Xmn+`PUs~V%+4c2W
z+?{{b>fadO?Y(SUzDv38*3(jRUG~eFiD#DYJ@jy=to*;er}nk$1S+Ht+;q6A7;`OW
z^O|cPda76K<`4ZBAsP@W`l8l-cir5tz28$)Zy#%Yx@G@I1O6|M6|A&0mPlQ3<XJV-
z_UpyWyo$P#BY)<lS6LLT7P_!PVMRdASI&J#&x4mXS<c_!d1Fb}u}K1UVjdPZxe_iz
zVBU4r8mSto94jz1v7VLqDY<VkpM31X(!duRBtcN=V8eL<?gV29{1*Afwc_yew)4ER
zqa6f-6N(Q%D=WzR;q$}i1{|`dA9_9|uI+3S6c!bT)ri$hQ(&LXI-7Sk@9eh^a(?9e
z$hiTA2R|)%+IZS~&mNXD>&>12|DJc2mw!Q9x})-4?<Y6CmsE4U6OJfZ?VDyU=1{&g
zy;$z}z4U#m%l<Cg`?q#e$st$ghTH3=|BaYD-(0eVhxMV0<BGsD>yrx#c)hNqZ2ovb
z%H?BEz@$y;3v;cEUc7X=?NYMMI4AKHyPiVy`Agq<guPVz>Up-F7XJCP^N$sy;_1mh
zj`}UTwPEUq*=w%;cRTT5&aT(SVphGaamfq4BVGuLRaLe2#oy3;-M(dN<GZ+pi^E?$
z68%-ydM;KjUB7?x^y4?9%2#b|w<>L&7rnAy>{V#|JSTnsh11<jqB2+6wwx|I?V!H)
zil%nyDX-d1LGf#@m}~d9zqzjYt>UQM_8X4{s$&wrZuN>?6C}R+OKIIB4H@^xKi;{;
zIIth*pTIqB`J&6}M-B=6b6V2<drQ*33K6ED4v!~2>*ZAhraDD*Rl0O|m;ZZX)ZKVP
zq$#+=Ptm6JhsdK~jsQicTBjZ^M=h6?0?VA{a20B?tP)shrx@0nAo}~nPURPR(W-MZ
zrY;iwx9_kN*Hx_-Ei1pgU6QX_tG!a?>5R09l}Qoai4iH8AY=q)crBFBa0%Wylj~ea
z(XvIxqH|ZiIa4N~VHf;!rr(4&?(<Fu2+le;JEWs7$Z(e5l8_0@CW){fpJZ|vjQE?o
zcZSbDb2x1NUOV^NdtWVIU;bnr<-Pl*_^y{?^TIg16?MLP`}|E>J8$v3(6ua1Gmk&~
zda`qy`m5Pb9~lZvujG2KxX1n7^CNS$?e8;}OZDHh4S62af63PKdjG4%)>G~USb4q_
zT6%nPdyro0-r08Z4!pjzgnRNlZ!5i*%a_DY@$-_OJLOB@B~?#$mA`!@)|XDdu#kM2
zyd-<#JXb5Rm*FK%_B(^_FYLE--Nh$+UQxEqseR#vM2!*&uDzOT{_nXcVl#8W)QR_m
z3*)X=#_!wE|1T~g<x3)m?bJ5aHJ5^xzqB*^`a;LpJJKa>!jum`%de%c{I7BP<h+f4
zv`=n3I^{Z##*Z??8Ere|F3hZ5J!ucGmO32Gos}WtCne&i<@7_PMkn`LLNeESw%Kgb
zmn*Nvr3F}Kg3zAC0Lzz|36~QuCw?}3KRt6lkJXzC<uBO3L^)4&%(yefM+%Ipq;Bxs
z;JI;UgX804zv}~xG6RfWX0{04xU*qnmyDFTYbu*-D%-vt&5Mkh7a1*HSQs4qOmWVK
znIC3uh=f5QJr3~}?nAr}+de$G*-}va|NKdA&iTBv->Uq4wyfM~?#d@?7TU%Ay^xdm
zdHt2S_1|-^pPT3FZ+E+;z{$Qo_SW+)N5Y=={nqzec{l&U+SqEdZA%5(yaFUxikrkv
z#2IjWkYst!<HvcNiJkqTb@Pk4jjMjwBs<&w`MB)hr3giyW`VZ<|8^%9e?C6jJO7{A
zL$ke)pIkaAt1cqk#d-A6rK9)7%2)mUqOc%AAi$tQMS_KUihtR^+S<+SUjHpLdtbHR
zzP54S4&|tOzka6~a<12(?056U^y6ycZ*#3n+6udaqT*`rZ+SiGVA17G`T7<ub@#2V
z=dANKx6E-~cJtxk8M_Q_=<yulPJXp2)%?hl1HTMSgKX>N?zMZCZrq;eWVm1G)C2!i
zDUNe<nkQd)@Zu)d6Nfh!i}|m06)Em(VGvgG;V@J@)*>O?<kMj!wRZNgSIai;T4`&(
z_4%@IcTB>}r#>(1x#s(JCsgQ-r!yy;WS8-LflQ|rj@94JEQpd^ryQBGqc1?H-id)r
zP@`pmK(iAMm!pP@qL%O#{sNzm9G<UczTL3o@&(1Bh)bC!{2gsu=dKFPiZ$v{{+F`i
z$&negSq{9}4x!r4OEbCLSK6Fa_GdljFE_C&<(1$n_f>KmSH+xF4*$%NvCyRTQjWXZ
z>41;3PF)Y#m-hSBwxW<@%Yx0PZh9$lu5QUbp5Kx#lRqt0GM~D0<ptksC(mxm)Db^@
ztk&hlxlM*MUZgGLe7yZj08j6<$?@i2!WeJHY@WhB{gXn>?9vu{+0925{IZQ%Ri+v7
z%bxLbZ;VI3OECZ6=j|(1cB#%e8`k7~+rzTxwe>YE`Nj4ja#y7re{9)aZhFwjbzR;y
zKjVL!@9vT{EuL;_+-xe`WO~?KdZYK*jC*M}(l1Ig`F*q&dS}+8H@|b^PqR|{^~)lB
zvg~Z)t4+6yRNV`Fp8ZN7ouwozNpt2Nz0^%<>%X?o=Hk^q*vwiRS5o@(#l%a0lzYBL
zzr650xnT7&lfKr~xk~wyAM9JB^gRClpO6cYre-#_38&w&{SfK%nR+6m+vloB*=(_7
z>1(pp*N(IPx}JVb{nLW$&sUt^ZI@nKx1l8W#F93f1^ldD;p_fL#-9)0(elmc#thX1
z&cVg3DmiwX^2Tf`IcJm;elbjDH@RVQBgAyQBh##l7Z+=sF@>NL@x~q2RUFneRy9^R
zU#^?-JGS@U<nVF(aFOfA!^P%H{CJtoyly}IU}d4EZ4iF&<X2{CVX-?K408{jj4C*M
z{0^&m&Y6UQ!_V5zwwx^~P^(d^QOi-wS@vGqOxW!5!N?DhA3#~#QguSY=7i1X_B5U^
zdX`boc~*Df&ZqP1w|tIz-SKb9tIn%gPd+6m+C+Bzmbp>kV##fgZCxjnIE!76=fXes
z75=YFTSa8w<b)p3{&CQ&mUW%!%9MinVTWGw?vh>bF8lbJ?^~vJZe6_5<nmeZhd=+;
z*z?T0etAv7m0j-+$zEM9IsM0$<y8s(_0t*WX~+Ig3X3;>wQJ_KeUT4VNPfHFAr<>&
zt=@crOz-QH-rZCRKd^0|{nnB{r&M`Ar9U+3V)%P~O2k4LPvvhf6yFG3yByH6JTP*(
z#p!j`D=c#oJ#_MoXMaiX|90nLjqTs3VyC;mD^7cTG&TG8)OjD=*jK&0e2>TJ)Kpo1
z%~fvyr?n(>=1e=0);Wh~uiix6ZE^*ovHB0c7FotHy-__B$DX8b_>F}{mP1U?c?b7I
z-Ii^|RR;r)mFr)Kb&g$o*D`i3kIO^>w=1`HM^%S=zt4K<clF_%_19kWc^qkEKW-p%
zq~O3I!R}g?6B89;)-U%6-?^>ga6u`r_}Kzpo^C6i(~Z5sd^r((Yoa7Ib{g?5EVn!4
z{c`EKoZ1apdp}Q%bkES-dTIOQw_hAOr*^FgiCCJXHP!!RwoiQWxAjM+1ZYm}S{dTE
z^u6`e0L^pHEO?Gh+-Y!O=>ogWcGb0gzxKSI{MIkH&?YrFd*%%}`$K8g&Oa<qpZm(M
zz^CTB`GC*nc|L#t{&~0S4ino)(QCb#)6Qia<1*7uHQ|~xRV<o!*Rum$H&-whhWNcw
z|8xD$pIillypYF1tJS}4O3A*bSKOa*|M;bu5l*LD=FiX$-khGK=)*I&bEmH12UWKt
zM>=L+iT2lCquzGvh}g_4CH`yLJFn_*%u#FG+FtPaYumiT5ogLfGv6+`n4gfc?R`C~
z-`CxT4F5!b|E+s>?X10jYM1$2|2eF?IJmsLYH^{&lKt&HKdkLyUb5NT)qh?nR&G8!
z&+al$+e<m)-pl9n^dv$b8-G|aw^MJn#%|S<ZN`h==$q&?UWhvs#1ZD`5H4Z3#XtB<
zlEHg6!<Efe70pu9nLRBuVgg$XJghDxTwaiW!Tg1Ei|-+}#O4joH+I&SzjeC5@co_t
z$?K~4H&}nMdc?=ERe)D~e+%;=M|QR^<}DC7+3(>7+W;%Uv;NF~AI)O@+f-M1{OHnd
zYdy^uA5Shj_SgK2^siN$3|1JJ95a8ucfFLEwAqQcRJ-5<+v|Sb(!c-P{M1d(54?wB
zx25yds!!bf!e_B@|2Y>f$$64br~GP}%=NzK<M;EIU;B7VTJ4t--f6rxJL%8$Yf8z7
zC#Y2aSl97ap=PzmoUo1hZ;gb@i)6|IK03WURJc|!=az^49S-kWjb28%1xcBW*XQqh
z)?9tZZ1pToN6}UJEVC!=dKWG8D(Oac?EU)q?R{nS|9>yoGHdmp&Pg^|{O43%4*3LT
zEn2bU*_|TwnGIGCrPKCR_??k3_)&83Ri<Q^!)p0oc`3f;OO|m*Y3XD}UJ9}I6rGW@
zVW+cPL>BvW367$fDzgHmY-TK*sQq~&pJ8QC$kl?GXLNn9P5ZTR`O0gr*mi$;!S7*L
zkl@k9%H@AmPrf?w#`6asD%ZbxWcPnh`k&^&i87&V-j9y>e7a>l@z~uWds&%^S-jHO
zXImFYE_$}VZmz3RLE8yQCv~k43nn{1P1!J?YrB|%fWNZ?FQci;6@egUA6`dOm(WE_
zuh@70&}C9OyuNz^Pdm?XN2!b7<~zo<-T$AW@u+Qr^kapyy6%n^E)NCT9@{B-w{7Qn
zZ``q6CV#g^(&BIYSL;l7KXj4I>*zXYbLoO!*Q*@nwqSSP^-+@Rj61>t`4sJ2{zSO%
zI~8z+Khgf1q{~G6mVW}joZs*tw0HS0@Xz^=y5n(!r8f-(j|ND}rX7<w{I`|Uu+8%F
zv6XEWSAyDiA6_VCeWt>2O_v22Mev9)tYq-`Xn(Z+w86YpJ<B8BAH8x|an<gSUEWKz
z(|3JZs%^JRc=v9jUo%gnb4F|RO`3Y|q|){u8!e@GZC)th!?{LZO7E?Os_Wj&s%_hE
z%zSiINYw9U;SQ5%_pL{DzNB*3-f>#CFh9ED+r_U>-dnvZm;JJPA=BM}#&yp1h7m>?
zaToS3kY{^t&7$)B(!~AYO@Gy=$;(SQ1zS6Ql)ZniPde}T-))bt-2Ql|R)anLnE7>{
zy02T_O_R&JEWc3ww9LM>viGm`<>$<Gcz)h<i_+0a4A0~mW_@3;cur;F@-Ewi=cSJg
zcD*v4_3bB<;p$xbYkOb6e;u<jDpY^P%zaxH-JQSA^z-a_WfA|1GnPv|Ym&@+-S*V}
zxaNOLpB`hK|CV<W?baSr5M{gB=cM;$$t8uUY%2{sS{=1`f)Xd#`DyXktu#4NS5hMh
z7T{_Qkw~%Mu-L?7DMZENH&O`|AMIX!K72Y+y|t@p%Z}H@*PgFB&+F*H_m|VMSbN!|
z#RnSQeK-?^KNv1jNSwT((0%#3DTnnAsXt8iC_1$4;If0uewu9fY4XG52dKUV;gTCA
z>_;_k%ZlA70mB!YZ_CbMlus65&tc1HJIp<sarVOpJU@8Y)os^i9iH}n>F=VLD+N-k
zCFWM$JIz^Qd3lAm=+}9R;}ZW!E=bvP>3G6|bL)SpOFOj%Z3%7KcB=o{i;cc#jSbo_
ztkN{+%=TG3!_?xyDtqsJA;~_!gwkpq)#AGLai8J){A$h}p^)5H^;1lCtupPdsxj93
zefQ15?JH7#xtuz_^ZbHRF6+!gk;3V>c+9-t>NM-GsoZ<{ar|Ag<-3EGk3~wKd-Y*z
z`+FJQH(|`XuO7%V-4K2Db=c=!yn<`wQ%X)}oJ#Pr*!$V$^+%l(nh&QG+-z2DPUWB0
zGmYn3x$EhKlL^%ax26Bq`MLR7?6c|ISKPmK)+WsVzU;c)R{poj^))H;D-IodK5xUv
zm$g4$uCQj~Yh-@#`S%aIz5MSt3xDv<J5%`IQi!kR@B@vU8#x=FOet+U-*$GTCy#Xd
z>5I4G7kn*^*~jOuFJrrgbNjWH=<GvlO&3a7{F%P~Irm+oKgYOFe>hM+N3%*GhNGYP
zVbYKN8#XBOq-gX#77nmK@#4v3f%S`JKNgBTjx@O*nd-iwqLJaZ{;6oTq@0fBhWyOw
z&K8V^#3lAE{b1U<UbDvkVRz8->%NH*j=3|Iltggk&X`*DVuJY@FIzvW*NK_i7FHkL
zU~=2w?AHw<w+-%o-5?Zg_@Q)3zH+T}rOS)(HZ9xz&AMmWJ7pKi|0rkhcj<b`e@KQi
zaZ;-RbCV-m-l^9qZZ_8LJI<GS?|Jd4<ynu_f{!xVHrKe$uRGMd_Dg}*`p<WRb3|pn
z&v>VD*J0QElI=U{wr>7oea`3C#~1sSnuotqop9XNLB1j9c|q%a-ny>_Th2M`c<^R!
zrpfk8jq5!RXbCS=f2_%KVN1Qq_Xn!6(~?sE&efX#&Si@uuj(rUj<grqLAf`|ZmThu
zduXt;9pq_FYI8JjuK)kos4M!9@;vK9H8&Jq<-7bi>-fue{iheH>vD5E4>Q#rW~y^`
zpPR_Izwpwpr~~<Tw9b9^lu!BhE%D#Cg~{hH2xiQfP#AF_f5xFT5*ppXk#iQgYfTQG
zIcJe)mzw6R4gAt_pLr%P>eYJiy~m?At#;e5BgeiU+{eWA@l{xD-iak`ckb}aZ*a~~
zyW*le>F2+PLK{492v}HMoO9B^UZdLVL#5OdTj?J*yqC_2D}u+RX7w{m$BIbhNafhg
za*>7)RP8$aXh+G9k{=eUt#hn$e14@~>U;R$g|F3v#ecnK$b&|$Htdw0BW-71lXv6J
zhMl#(-*;Z!9P-8cPsO(==>tKRRxgw}rf_7<vzoc@LY+(gfrhi@gD03lsuPP3KhwPQ
zeT%T2uo(;%dvVO?oX<KN0iSCu*>@=YVA{d7!(eD_A#EpZCv66SSEQZGH7|M1Pv4Xk
z#lLfX;_3&xbZm<x&zD>JTnTikTsSrHgh7gg5s&0C8=fZ;WrnFUjC>@Q@dUTM+z^=k
z=$c=jk+b$;H86bk;h}nQ^_-gOCk5Ht9eyvInm9etzWK(}8*h`QC(5TcT|JZLc}J)7
z<)j%pohRXhX34>{E^%QyZaZ!>81&u~k)%)~mSYBm+bT9bEbj6@oYuXR-N*H^x5Zw*
zd2vjyR2RiLy;5Bk*YrwtRh*H?JYKH3_5U|LKk>og%fT1Sm+qfvD3v%7xTMdfL*IGh
zulpA#@U&)cUlyEOS*2$fD!1&`=aA}od9ly$f7B_?w!5nLZ2w`Mh1N&j3G8wH(pSf}
zQGR8Lxa)`dqbE`|I@&erip=Z&rN(dSzQ3bE`FYBZt2gUI=O!t*3);GT5xDe!s({_n
zF26tT4|YF%^mCcNW#fY5%zrCx_N+fP$30=$@8Y@wQ?4J1kJ>xro$5M%7Rv7t_~v|v
zztP_1pTJM&7yKLTpB?0WXn*8?07HwA!W4#u92}_*YnsKo8Uy%RniQt6EL7oeb=2Ta
zn(G)M)AC6{gkzx#hpLl?kkd?o-#m+w6sK^njPZEtq$L)#aY9#Xh)#$Fv%SylRBh*@
zFtMN|KND}93fKvjUXr9J%KdcKQU0>PofA&A*8e{l0Mb`<cD+0{ljh8PkrcGB^LIwC
zp4wA~JL!g+TuzY!r<wvJS}rMwurIXfs1)#=C~&GJK&{0|aSGSM2#%*tYnnZ}+5*gu
zSgl&iGnH>)1;<n8H_aYh9RY4FMoJ<=3#W9{PI<amSH3jTaUxe&h~JVTrK!Rz=XgAI
ziF3~l3s}+w<7%k~Sx!8~x!EytR&K<s+&RkJ-kW*Po2{C<u;@@uz-RVlDJwv_pvriU
zPD_)XHNAc5&9<dC+m`=z-u!ukQM6$I$iUw>%T5GXPCV5a;+JCIykda|t1CoTr%bvs
zv$NT5`RyBOR>>~C6}U=f>8Zd~GO6|fi%xjBx@j2&O`Pb}6Oyz<NcsFd1yRYBCk&kR
zwD;R_W*y3%th|)}z=6|IUtdm(d?n1|%iQcQbKYO7q9tEOk8yTOui@4CE&hkp6DM!5
zlwh6R;(O@X<p-A^T&|J10jjMfW?o*_@*(rX%MBloY(BiqT4%#cKA-k}RW|FKGn4Z_
zB^uA0x$EGBjs1BW=iTJJu|tL}$He;%N~r-V2infQec*#Ux*h*ra?Y%k5>W;}5~es_
zJealY;qx;E%PK0uxSBt(xXzxaro45j>cTYHvrflvI4|b(=;qJ)6fx_;<PR6Emdx~e
zJw;6^H~6l#RQSTgS8b1;$z4`|a{TC2@5@v6IdGRroieL8+4=BUzO>nIo5xRI9QmkZ
zb^pBJWar{z#}0C`AHTEQduq+fN&2CmzCND&G{9PGVtZrQmW=1Do-0gweXCby&flE9
zm$&QScacAPqBwV#ziGcv`SEO1okY#y$drP<>-*wt`wsp*FW$@)EBM4{N9qIZ?%zK8
z@(=BJ-gIufdqY;OtiAJYVp91GB<3xMXydD$miwn>ByLXJ9QzSg&_xH>uFzT&x32OH
zU-0h3SKb-?|9AC}V2u97uR?M9*XwE;qn6kO=!P9X_=@=&``Xs*w{T@o4^CLPKz30X
z$L(9P%gWkU-c8IZpMk_Y{?1^l^_l(k2d5f|99}$EOV#L2x&Z4Zfs@V}{2OhKe5(JN
zlzQX5Lfz(*^JS)~OP}Xtwqts4k#wS9j-$wp{cPzAZI65txb0lPf6?}xaR*aP#|OnD
z?Go}yH5{K3xta}H8(SHtS$zs}k9&3E^yZ06yY_@koWAO#)yAjZcPlD|wuW?0U-cpA
zYwW6hr-fHKRlKheI?&a(k^i9}<0%GVl_#ZrEN3Q6P;)wAamXoi;SSG-CZZOaa~|aw
z``87p6_=4LYUy~Se0!q3GuvAkFAoKuWd<`3rF>ibSoY-AR0W@9n>bjDS0?Cv^_ncm
zJ6W*Tt2V%o-DknCBm9kz`TEUYP4+x?^VsCoFDB+sU~gG|LsPtMxq_8-LA%#v8;>9p
z4>QXXp<hgb4(xVZG5x5!K6kBl$=VB>osM_0Eqo^$d+40(?ta$V_I=JbYNbSH%wU(l
zlIWzW_AJmMR$Hl#<EP@Y<5oWu-n8$KXR7b`qxk8#mDU=b>j&A<%g8s){J%4fy6cAN
zhw&{*QV?ZcDdBOnp{#A?+{CQ(8Mh>tA(+oL8vDHE3AR3b<($D*5cAEijQy*2%k+P$
z+wy(!zWQ9A=k}6vqIOf6SIV3yGh;Ri;Q^ai8S*oPZ%L5?NZjM7!?(?QCp-*%8EiS>
zR3lg%VyA=LRlDOiHy*fnOW7v7c9LFGgT<V$b~9om1PnTqqObKz>If{ZKkFX!RfcWl
zUN%lmcFBEgEBDT8^UgcisBPJOPvZK?52h#BtXD7?p7{RN<bwZ;k2W_mj-@b4r!Y$Y
zo^m@vK#ya33-=*a1IHZ-cka{||L2|k);JG7KlO&WK<SR^tOURNzh>pEeHoqo>)Nb+
zTbJd7nRXMkqeW}8eeXwI{(gV|DHkIiNv_py>)!0okGp82pyRA4cC~i(uKdvNf74Uf
zt$nv|>%9EXk83wpukOq@{kT>w?xt_*gUAYw#f1f5qGx!t-DfCX&c*h2@$vl74{Ld=
zS2yOHept&Em;NC#^5!QCp2>%Ih?V`leLq%im*DIh6GWRh7k=yU?3|KiBBr@2Qk`>l
z3$I#BD`#S&Q^X8`YxfQFyX-c2-beuP#1Ik?W+qhh>+<4~cHi>5hnMaC1R8og^XKON
z)%&Gx%=mgS$5>uDtMvQTN=NTmRc9K`^3J|}aOcC~!_V8!^3H}fmiW%KoNqbbauxzb
z&dbc<L}?~HeW3G0r%}+18`><oeemXon;&j&xVhmb)0$Ks{qzS<KWw-i^={jY$y2?j
zKXc)~oWXkMs!d?Os@#rc6Pxxq-d~U}VCGZmdx$U5JhOJ%waFrzJSMTv?O4(Fy(XmK
zQL#c6*UWP>v-Qg#E#A_1>VNpecQtYKPs^-U<OomQVype%&H3tCZ)=gr`-_;Q<z15|
zBt6@??DEvkyHD)ZOb>e8D4gEyqWk64lD{VN(~~cqemJ2^$^4+u-Wrj-DVvttcc({-
ze)#>wf5r0|Q%q;7oQRveaiy1K|DmGm-E*h%)CC>*sF?jUJ9SIwxBP`xS(;yp!VGjT
z<W&5WY!BzH&w27U`>!_t*V|cFPiO908<`h*K6d)M=<PchPtK^Eyz<{NEB`C~sfR1}
zO|z&ix;dS7%QT<bZNJv!s6^@|Eq(0wmN}FypkmLF<K7K>Tuyj*?1|a`kv00_w*zY{
zq}lWNAIKdMEiz@+Im_T}eBct#1``CcPck8sEn*hqw6un$2xjRSOqKfzwa}G){jM+m
zS@KA};m+Vgk5nGpMjq(9U{xq#B$F<%f40wV4bUpbN5vc$6rGOOvIS1y3$$u^DX`pm
z4qu~{%c~jb0<5nE)+WoH)N=l4#qvgAWAa=_kuA<w_!6y;ycO7<EE}nNxRfpM1K&mK
zmiGeto&WGL+PHiWIAp*3&y9c5I}cs|e_o#d^6F1EQ>EVY_X~BzPH+u55Vfe4!LP{F
z-uuV;YX%!_Snk>vd<@TQ(n>!eto`TDo%!>%C(J&e*Yrd#{`dZ(id{QrOn!L$`m^;t
zZzNXUIDYk|=gG67KWG2jZ*y&b|NqD0cIHu~izlW3uGg=4b2&8riR8Nf@Bja<*Zc8*
zJ@-5njZCj8AKh=BdcObn^ZTEl-=F{WbL#&;*6Z%;fAkjm`Fr`d-%J13N$M}UFLxwg
z{LyyqkGI?1oqvDVt+$@IzxKp8rC)Dd{*@`!|DLcu@x*<<C;95P^5t)3E8GGxu&{sS
ztAmmJuWSV{L;Y7if_*Gs{#drcv3zwfLTA6b?{{ba+#U753-+@;`uFz8-zc&DwL9XO
z3;&maxPRUruHilM=k1XitB-%ocb_iq`!Brk!T+5P{%_&_cR&8ap7Zu;rc)h1|L2PR
zYoBa#t@7Lb(9P@a>#q;pTo-=!SM09W<xkg%*Q~j}ba&nKcfVqHzGjvGy<HfL_<y6Z
zRU__C`dU?6efrvK)tL3uOZLV+`dW4R`fJsM_0wDS#wmTP;`RK~zoh^A_VE0;P5*@R
zGye;H|0wodzw&?Q_9OM3e=0ivocJGT6L+_=^KV7x-xJ>->`|%T;{rlYste*S*~dMy
zzZK12#edG5xnnw)fWD~0t?GifD2F{)8Y)&YUcBG@`4Y>b{a-hQ=l`lMh`Z&m=SpM6
zN@lP)$D^e}N4!-#KFZDccuhFI`Ew9=(jNBbOC^tZn|4gk68H|%;JD{XbHz&b!vFoZ
ze!V*9?b<QD3oH@kwC74o#Y#@Y`_0DjcW16YI5S`Om#szX?x#B<4n!xV$KJ0$`ahB3
zf4$xh`wB;moj#5~g<?NWiv1`S_i<9(#(TdPUf)v1y`jQ$Lq+F?ijx~ExHndKZmj6s
zSaEV=1^1>K@BiM+FaM-^>XYs%5UC0#Nn-D<uI=7blb;a3Naw!wjqMtzzpH&yJ@sjC
zH%Pd1Q%&clocC|PR2Rf8(uoJ7`(TnJ_VL@dKHQ#re9F&QbNm1Qo}8cl?eqJ8pWlD<
zF8E(F)Bb;){r`LWSw8Oi{{HOa@6RsAx$OVmf1mG2K6}!>j=J9h`{$mxZ}+5{q0V|D
zgqbbyX}j>x+ns;jI`3m!^tTLz_TD>$EuSFrL$=<ZZwvRcE&5-!XuhAoy_lkxmMi|-
z`?CFuf7tY%+5Y&(bq79LUwB+TNBh|C`}5;gzbjvG{IAll+S`x6r}p2UUi*LF8!oQ@
z-<N~3P3?}~-7mHaf6rHcci-;LYu7KgJAZ;u`%kG3Fr)M5Z4~xbsgAF=JHe<O!FD=$
z-C@mrzc(Oc_vX3-nBn*4K8!6c_W#>Lo4Nl2ryjn1UOv8}X6yCs|I}(6KYcy)Bzxb!
zwdN03TmM&<){4EZyxPm?Pkfr^e|zzQgZr-s)#c3Dd#$_L^Y!!{YTcL4%g0ZCth@2?
zTJt?EWly4uD}1lkyynwCD64-GgnnL=k2_=?cWL*Z=M~??{_i)n{BLg?<@)2t`l_ot
zWfo6&64LKgiPQ4jQ#z?aQ}yE|uSZL#9f{WOoSr14e^w=KmFJ$)$rYOF7vqiB@2|Ri
z-#CBIuFsS1eH6NHc+WV0&(&+!AMA0d==uI&PfJCmbC%Q^2DP{&f9uy*hS$ye|99~s
z={PO7JtaLm?&rT<ntH_hY{&F@g5Q6y-D}cYu~O>eCDWvR_iuUM?wGzy@blig75leN
zKPIR@RXNVeea-hN|FUGw-zDsO7-ae5`j7qBckA`OXmHsRvZ&&v%f@^9Z<mT5_15j0
z9w!pN_r-su^MC5QxwW!Z{fjIN{`dOdzwCV*YWJI8crW`Uo4a=J_Y3-G4dZ7e?=v}G
zquDp*^`D=Qv(^7zlKH&UVp`?f=l%Mdo%X%q`e~-~#5?@t^mUW;Lsjn|Ypr>y_0vp$
z&s*;Q*$EY;&vTP1ZcF?-e&WM6qdjMN@~scwzA!=G+j*Z4@6CJP`anTYy{`p~TEQd|
z``h-vPp{72UT}Uo456_<Ze#yj_WalB`|<T1b^Er=um3*3zCXU{_w&<#et!S;^ZP&5
zj_Z5t^X2xN_kUCRwYUCHnL?elz-QMlx7&Zbb@}&A@jmkhP@VxX<C`advXbAEeECnN
z0&lkqgV0y8IOdCW)?oDe0-Rm{`$9bP#d_<D@x>3f^Zzv!_`6*gj6m$i+l9f1|FNmS
zV^igNYs0$V264=Wb=HP;)`kDSJ*?wB^7nS<Z%}5DfA@XUkGBvm|MzT#XZh-2B>xP~
z&d%M(mh`U-g#La@+{>2qw+xJ6?7aeyw)4L>6#!Qw!oRok|3+iC2JfG{;zslQ{Phc0
zoR~S^e%tl!;t{creSaKUoxN}0UWV6IhoAj_@QmBy{)wOV7k(Z)XI`=I;^+DYpL^%)
zGxUl#9g}u?+`8~1)5VH~8G8~<;w+@}6?(-d9h3HY+`9B5)8&eVnR_0Z#8pV?JM@Y#
zIwl?TxOL@6rmGbTv-TvK#_f>OPv{ljbWA$xaqHTTOxG(GX771u8uvp=zoA$B&@t(x
z<ULE@Wf#P0NaxS!6+d^3JL7TdmXB?>DiU+|959Q^kj~%HD}L`7cfsS<9Ut57R3zr@
zaWIctA)SAtSNz>E?uz7nQzw0YuxEpL+yiO-FTLh}j&avNW<BuH=|P2a!5#yPI0YGf
zkv?;t<J|3!S&w{ldQ{<DxMzb!T!M_gNuRmQ@w(SJ`Agp}-MW|2GQMGszQZ|jh3C=}
zKC_<q$@HX#v1lKo<>R$3$6nuFZJrO7;|9yIf#pnZZTI@U-|FpZ>-@Ff)?WM9ANM;f
z=l-m>(OYYoi}o=<4K!DH4$;K)qy}Ah^Zw9(;raJq7IRPd%nP%a!Sa5?oX2}vf4=o&
z&6`{L`Sbh#WhMVd6FE4YV@>20WhMLX$2os|^P=wmjkx=938lT}54qQ``}yk|)0)@3
zv;Q~G)_+i={QSSeb7|}Q4}Q9x|KE7tylx+ZNgRh%K1Z*3+c9pJ$E}M#wq2}vn6an9
zB+f%BU!zxi+A(gQ#e0vQsxBB!<X98=+qFsi*`Fh28PuF~^S;)9Y}p5#LGm5WeZ1H7
zr)+sq-W*UO_y1RuY5)J7{eQXrEZ@&h|M~g-*U#_&dJC+t{(sM=E?@4g>zC^PKi)e0
zDN}gX{bD=+M^O3gypR0>qWPl;>N<cMNbu&*Yu!DtR?**Y3-`j<|6$Fa{m%{|^%5X0
zBybl&9ol2acKcVB^ygdR9=4=EWl4WP{ei=Or8>ZcIJm|0KJ_EKOuz1M=Dr^o*`0x~
z|BF}u-<ExwOFOXX{paX!%w{uA|NkHHw1F>gqgUjrpl63yUb%Sf+M&tk6buB+%9|uE
znY#)$E3i5GB+XdV;VCd#LCw%7@Q6x>CXXXeu*gP^B*7y}ZK_=nEFk7#Ay=J;noU1?
zUubk{)hS!Kg-vW(<CWp`aJ7+(&>=0a+5bN*6IgEa-2TeSJ1d{8eDjmPR;!+?ahAaf
z&DL23D>yr6ZCIh%JL|&=&WW=OR%%Y2Rj`tCW>o0rMW+PqOcg_f-AtE;2>Y2Th6;z7
zE)5lqGgS-|PBUE^CY)!g7%p69x-?w4&Qvi%xXpBF1ozfmi)0cbgJ+pOj1*pEniv(l
z%JgBB@Fvs5=-^$Z52J+-nI^^rpE7+IBYep;F*f*?>BCszM_U*Ab!xfqxZYCZd*k|)
zI^Q4HTN->NZcJ(NwYbsJ;u~>eN}F%Rjg}7I88@bM`R=&U(&KyM#+1I?mFJiF&92Dx
zI-%F{#tB3qv+`cK%<^b*<Gk5AA(jI~fLXRtrJ4V~X0rBGZGCm^-{b46l2^}LyK5U%
zW8BrFVNW>}cMELdR@^NFBH*m*;OX%fG@w!fo3xQw)h`5E3cl5LZojYa=efYEdY@Sj
z6PoxE6>J3Kd=e&^8=I%)g$fzgD>EoFNKfjJ=uwy>5hLm#;3we6qvuu7RM1qAR1oN*
z+TkS-n&l`G$hk<!MYT<<YXwUn=ORI8VONu;j7wr23p<sh98H=sj$CjNbQEwDaO6?!
z;>euHB6vh#QHMf@LWdtutjI;qMZzxXZR_qfDOfp{B)wSF;V96oz-Gu3ctoMYQNa1v
zGL0mGBMKd!Ji#J{oJm4QRN6GVX0RBxDt07xDTz7iG({ZQ;3C+qAm*r(6tSp7)y7p|
zzo4vY;L&`S<0rHxO$g=m)u;(=^KgoA&HOBXPvlX^vp?-tQ>~}kOtt;L(Eq8wVt_E4
z>CymUK2ya&VKLLCfx>d8ib29^rb~l_^-L9mS7(KVSRPug){=E-#j+_`hgPaB$vU)Z
z*_NzBtJRKV9a^*OO4gyZYEQBbty}gb>(F{Nmh3|tmWgC93=5y4tds4vS<NQfX^Waq
zw%1m*m~5wQYB||n+tq5aopz}8WP9yYo0ILdOKnZI*KV~v*-m@Ru4o26<@i2p!g|i%
zvo5XY{69-z11IzBpbebtvnOoe<eq(L11JA%fsLHPvx7EriqD>~kyCp1rH#5@Z(hk1
zUhfM|F^IGSW%<ta5cWh*3ai~$>%HA?xAs8b`mnpV>i<NQ=3LzzdOZwk#Ohr)R>>%)
zh&z=jri6nC1S|j5CvGWKh?KZf88VAswn^aJzE}UgNw@r$adO?EDC2WMk*7udP&cQ~
zr?h9s&aCxPdX~?Uz>?rBB-kw2BHX;1iQ^FGq0WP@oPLn>1xZC-x*bYdjv-Aejs&<0
zPF2uyG)c->)Zr%JdyC1?B+x~<!%d)9VwQ#yhZ2WUhoXz1qV6PxCJ84IK^8%li=7L*
zm8={~nqC~aur*r1MID^b3^_pQs7<j;g2j*{$>Y9=D^HUIi(zMCmy(*JPt%Me2CjmW
z6_y#WfC?_94p)J0iD??3VynZI$6I7$%W{rK=8G1qtoqUWLL$;tXp`1L&erv(1Uxxb
zYvlerJlko`sX6}xmKQCbw0zTZ`!6g1XtHD-TChwc>(D|qm8?UHmYHN7TCC=hb!f@5
zkgP*X)l#xT&vt3K|G3f;;45);N|3L`)s_(7h^te=d@HWDMEK6QIwi_?$JLe?-y2t_
z#QFZX+LGWaacxSH?v+<vMJ=hmnb%s<eLJqTWcseW){^ae;#y0t@5^f~`Mw<2TMB(O
zueTKYdR%WQ_07EAQtsPvz0FE>mCVC9;ZLTC@xi}LAI1wanI$F!bD2F%5Ee2^ObnJX
zdzdJ!WR{o|tY!8vN!Z9NF*(?3+fuvPR=JP1Y9f^#kfH`$o~#tkRN_QW312g=F8zCb
zxv0gftE-}a_s6g9TU)j5)h#WE5!IpM(NmSN6n=UwZ+v6x92Y~SW=}Xj_4DlB|ND2l
zo@iYayW&>ss@RpcoY~b`cF7y;w&i+g-yR^C$H6XK!omEo^FtET<UKQY8Wyjf)bUfA
zQG)Tt1eYTTEDw)7Sj7M>t(t^egj<B0yCJD6h<j0om%vm7EyEB<IR;8hiv(RJZ*$=B
z6}iZ<NYF*OO|6TgiKB@li6hX(qg$mzp-sS*#gPS4NcjoORj@KFc`L?pk#mu-vp6I%
zbtHBuu{rXPTOED{Rfoy86jX<|(k^A0^!^vt`}zAm|G)RY4s5IZ^G-+faYmA?@0rK*
zI~z?5C#Z>CI^yzgri7=ka%PgM@0lm_I~ST5PEnh7=}5@Go5s_=)|Z9;@&9q)$aI53
zHJ?44!R3dJ%r_|1@Uh_vo_@$_uTQ4+qtF?<MAhnAJChS1Mb6kIrnaxOvpDfl?2KLF
zYX4e0n-d=;&e$cPX4lp^Iq^~IjWXkJ1yAc;s(;+)|M$GEVC|;+{CfNEem8huJmb6R
znf)JG-w98Azi`v^_Pg?1f3Gg9zk2KUi+>yL#&7$5+U&mj_v3&6?XW+<zwy5O_ur4}
z>Tk<u?YDmi5-qDeFuy<OegE~}x4$y3|9<@S-;Y)GyX_u){a3Nu{@%O%?d*GOWADr7
zf8Tza`QE$!?Y|$t{a0r5ps?QNUH|dFZz~Qw{#S8FzW?{_j}6EFeypqCbN~K3{)+wf
z_ult^zwRUV0VE>-{`>ah%y#ek`+q-v{O?=!#INuFfAPHb>3;Q!FYj$<e7|?P`ux+g
z>^1k~=YK!`{O{YJ4d<^Pdf$Kk_wCn=@!yYM|NF7DzS{P|>wiCX+3$Oozn;A&m0e5!
zOI1MSjq~1%p7+0Pjnmh%wXcy~c}Bcy=Z5+FayBhb3wMn_dv4ZV)7{#?UccV(t!ho>
zB}cJmrBT&;uf=WO`eysHjQ_uNu75wj=*#Xo7E9yH6P9#`xMs~VP}$UNqVj*I?cLcQ
z*wv3}D9!ZH<E$?bxaq<*ahIBmh`G$-qM$CPOl|gwdO@NqGY?HTp?7J*pXZj|?oW3d
zC=Bf5mI{ArQ+WU2-;EdBo-JD8v+ezzci(@uPwd>KBE?lSPq_X<g6N4*h3hOYMZVWQ
zHh<50AXM1&=TU)`-6kpz>sTKBKFhn4^NHGo*<n?}rsgZZs7<}M<<e}2Y^JrIOWVV$
zIOYiOwk_aX`$fP)v5U*GxuyA0+hteF*b610e&QCfQ!camhCeN71Pk@7IeLFJ&vr5K
z#b#R+yH2G@-xA1hUcehDB2?D3ocHYd`XAi+CuU|O&s%g*(_s05Ib6kQdBQut@|@?H
z?{!{9+41dnPrU=nezr8cP~!dT_++cR-p`g;H$$z^9xt!wz8$w`_XhsHIElS{QOLUa
zuG5xy`S}+8N;8aD_h;h$VzV{tRli>4I(_MglIcc|wH~#Ko4T&JaQWU)E%NyHc}_Z;
z)Qm|^g@*;ZuN+ZQd|3CDZ7rjGn@8+tX{NMyE9ISpUr)Jo-%$Q@Xq9j<>q@UpyuN3I
zm*`!Z&>}r6Vxd~ticZn2tX6?Y*DR%0l{@Dd?+TUuEZ~?{$v3z8-s}iz54B0&hj@M0
z@GcQpnSIEy@GxiK6p7hvyAy<7Duqs*rN-0wMv3?2|Mq38i{|aP|8A$0UeC=F34Sq?
z)+&`;Ji=wV)ya2_K;4t?X~Hj>6rS&tVvrKiz5*2w5QZ{4WG4N$2^VJI?bsJ^r(oHY
z=OQK^H&l#VUMcZ<?s(V{n~@~0Ah~q0jAV|<!b%TC&hN^RZ`zGMFiiXZc>aY(caF2|
z7ivx%P%e2O==539MBeBFhipa5g&K~t?L0P4>~e}G@=l)>nfpOXw}OOb*n*V4Rb=iL
zFyoi4P}tNdqEzZ@p^;~C=$T@UT<Fwb*1ipu9J7rRJyOIoJ^tC7oO+<hBX<ZykidG}
zDsYWYQcvMwo3anDrpv2Y1p#*oTIM*Pux!&x=a5l+E>M&PP4x8{%e36vm=vGLIGyv@
z@zAGH8N~7dImzjq;u9I~smjKVicjWTs>}Xp`ObF{Tg|%*GaMGJ3-s2}PfGiD$IN7J
zN(f&~-P0tVhoZe%j~16L{IhP#{nk&r1M0+2z4!VVtFfQ^X*tP)zZc*6Yoz>Icu%ZG
zUWU{E*cba>UkMb?-^cy8d||!HoeL?uR)CoQ`R<i<u2C|b=p<a$35rZ}V@BT<MNrv#
zwHzTyrP7IQDt8K?LjTwKRZoApNbk%2)_*fMct4r&=lQ%HDzg@z5`4Y~WFWGZBbM(W
zHv0sK!(`tGS*rY(ms<FLUx?+SIpqug>wP)j;;ZBgPT2eF+kJNPd_3O%<FWJd*fyWt
zJc`d1pV*Y#N<FdVv_O%Z(>WyJGcCSK&6n!7i7`)Kcv;VRMiofSBDn`4lRd;&c3m-n
zCV|g)F0<`6JYp%IP<UqIES1|AT2{7OymIcC#kt#s&r5oiz*mOlVqMaG5V6@VZ62{p
z6gkah5??9y{SsItxA2tUGPy@9_!YC%z;3V9JIg$49YPQhLQsRNUg(?>fM|KOaI=Q<
zjH-^sb}cK~&n@DCnQ7H+gre9lbFtteIq#{;KD;YCMP@;Qb*HJ}#WPB$W*)i{eCLIx
z%UO6RN%t9i6)~^k3iKC#>XZ0NGw+MYqO?PL6P-WJum0d@vhI|ic@>wXzhXLa>^)22
zvh@ErC-|4Ow$FX7UgK9fvU4PK8vYi#E<B}gKGEsZ%@?ivmw)&!@^9X$`@uh|x$6BN
zzF+yrwsn8-p-s2gmKEy-&v~aAV%K_V`MI4wg}asq{1nleZ}r^h=*xLdPlH#~yqa@b
zWO+5$RR4vaR#I27UTB}i-QX29TvK7$x>&zIz|wopJJpbVEvKfR>(D9OH9g>`kXF2v
z@}?s%<D8y)uc&$1b3$l&HRn|Qg`buhD|x56OoOYqh*05<P$BacY+QKg(-S%;gzi^y
zn!~KKH?iW}mm-$JX|De;4U`qY%#UlQ3w;uDigP;Q%~7eTsIT}$wPjDs1iA}$x9ris
z>-W^*kF80R$B8L1LjU5VbX8Kk6oamVDn@6gwk;Pq6pb&ru2M4fl@Pk=DmCdpfA8v|
zm<KyFj<cdN4(Qr_p1D)<jrF}cF_%0SXA6OMJ_Z7Yf^m{^tek9{<oXPX<a#+Le0{-@
zelB{q1V0ahB!dz6p*9<}1|No*oGI5Mud0PCy|l_?p;l?wR?g6sCO%e*?_E`OG!h*-
zxnIRk=G=5+zulxnX9?5)&JzWBA3E$XKIm}7#iICLu0i*|-T9Ae(-@Na3S8yb6xbA|
zIh38wyLE9^$rd%)wVrcL-)8t~t@U)z4odg6j+)%(_LxtTYw6_{rw1)23k6IT`f9N7
z^;&<Pzx$GlW$c5JKtFMl*r}IU{luS^I9?95jD5fpcuv5?d8#qXIf17<j>aLD&JdyC
zsm845M4tLM8i!g2Kkx}WCt?B?5_+oRxH-hq`$2;5?=_EWilry5@!SMyr0~xw6yv?b
zKdW=0c|^wa6IwySnkI?ME8bjf*t_fDuS56seo6~+l((o0uj!FsO?+Y|!|CSAQY3vM
zw?s{%V+M!v^T-JaO0PSc{GQlI3oAevkuN&eh;!|o@HuH=cv|g_zueJ|7Q+8!&Grj=
z<nTT`$HFWqocQF~g-h@Amh?E}_1)QH=%}zrxo1wT5#y$H8X2zVM5BJ)ZJZ(qVoq&c
zcVgF~3!;|hV!KkW?SD6O@8gJL)hVz3e7FB}=jH#p5C32A^PNBQzkC0u`ups!b^iWu
zx_@ud`zQCeZ+|lR--f&YXEf&TO?Y&^a>eWSx4S<rwJUpI{oAx?>%-fNi$4GOwqlQq
z#Tv7M2wjsv@hi<i)^gvciY?ur|Cy`%w@S`m`)@r*Pnm=+Q47jQi&biWV0FZ8igTdd
zM}t(kt&V5Xp7AfMe|O;DnV*v9c|y;w2>kQyKn$qmw{ZKJHFKXGSY-3^Hvi{O#+@Y-
zS9$mfx4D!}Y<pr~etRG1eqO_t%}ahwUvRSJew)Oh$VDquGbR)S{5dGuddl<VZGPvc
z3Mi@kj-AZZ$&a?nvz=;uGU>x@0cQ)hrw)+HJZ8${remR&dIczqWvhx-9ay<&j#kDh
zt>s*Ev`(#T0t;m>nxmF6OKUmj9JN!KP0Lmm%{l-Ua+)g4<le)fGQH*gKJi1Gfqopb
z0-kc5<X8MTWnZrTqB(j6Eoa&%Si=0UeWTD*o<?IAOXmwbq2~lFoX$C)n90e0l-Hf3
zl0%VS^@(DOP0NIKflmQWa!x0lp@JgkEe_Auo@97mZpQTa-YKHc8e+khX0z5g>JeK5
z{(L*&5Ey)7CMTlu61H$U7kr{}mInJoa5O+`hXuYS>rNFmDnGGloilTTXG+HvSE)$_
zhgUMWpQz-rJiqD`lx3Va{e<Gr^R~AgOnj$aX6c*oS)?fKklIAWAn{C(L%iw`k$DSG
zaYD>mUiBc@=7T24SR^Kr4T9m-pc?79B`D8vfa)d{8)4<jvu<u5-<+9XBE57%nb!`6
z2e)54{hN0PTKznIuL-JtmVfvj^3U(%?Ke(;RkvT4+5fjzI-bM#hjfrf-Th<YPQw4<
z4&C?uS<P9$+$5Z@AvQSF^D3m9(v<qH(pgDZEB-+OYKx%we9R`U=@z%VSJZG$Rerdg
zy{(|-)bs#LVZJWuK5!)wI#s#vn~+w#(^DU4fq6%}YmrRh7H}=&ekh-x=OM>b{e?x~
z&d|b7sv-NFFtw~+g`(w5YFpa**S})}=Rf>)sDA(Nm&Qs0?>P9?W^_bI{yVpK?gmB2
zlxxw>NwL!+#HXbO*k~I(cR$B<hIg?}I_onxmP4`+53&dgMou&4eV%_W_2r7>&cx`X
z8`FBMruA7(lh~%(mV0tV^2-&;Cs!nQGDaJ{ipbhJEn=-|Sa#>tC=_PUm-i~qG6nkz
zZoRedpTqoTn?T@f|3h<y>UVgvrzaNPJ-7X?M~HrzKv2gO(}xRgoU2<LV!I@Nw_wP5
zl!jIMZov@#kGJ!u%vu7<UHL8+GRFfzxhp4qzu;7AWiNFXOT`ROK7%uscNgti@N(50
zy@FjD%eiN*JN1$YEF`&Vj$%QZMmoo=g{LH$(mRUU7J!A8X{2+_T6t<FQ+ij?vIR3&
z&Cx6X3oShr$&}t%G;KlTsyV6!(=^gKXDvM?=wzmtBJ5<Q`b6aXxy{=<4yS}pHtpn@
zxXZ&?Xx`ua{HJOAuT?KPXSTDPS3c?AqJTRS<{z0|6$43iUg`07WwpwF7My6G_PRN}
z`S5fO@w0RPmTN4WQ14a0XYwM0XF1QSwex(6ze(Ja)-_(dx%NZbOSg!NQli~WHfI+!
zse!tGOB6qpRm$(4zg8|hTJ!)jgZI@e@#2)EKA&Y*B_dVh4|k<rPEkC$%cJ>{URTd)
z1vkNyE=8U@7H)01CK4sAs~mYfrGSk++018aq?+03BWX)pvW5DMoTq3$JAI@kQsa~W
zYu8UB!+5pJ1p?+E`q}9eDZWQH7k=4lz+B#{_G;1z?pdj3)%RGMSI3vFJi>EpLJ+U+
z)ESP##xpM6^!xftW7@gZDp^Xc&#tK!actPRS>en|gXITAK4&D!A!YbTCigSSDvL{^
zo(Wkb$tku%+s<*B{w8rQmYV+_?C1%P@maLmv&67dPwnDS6{Ux5$xQ0KNw1SsV$|L#
z<YgFS+*&jPUf;H9Zab5z@L12Qc%`%W=Enx5URT#OONnoOoKzOWW(s0zKT2|iFsD2A
zo?dY5Xu%G?tQehkRAy?xW2+T~l2>-{WX15bs3$*uwyM`Br?q>n!>V2%o|f*$0z-jy
zY=@3I?2zCPXMUKa5Q7lhePH6p-qSzWK%IlK=^uN!H~tQI%G0RK^3LhOIuM};B5<-a
zb4{EcXv#k2Xmrj}f(FP=^Z!h~p-k(Y9)xo2;c5(5{h-AX*Lq;JKm|llZO$s5*j1fH
zkX}9{5O<b42=}foYFV+`$0GD$CQ>7R)d9ggzn3y$yPO3k^T#gH3R=8n=0>i-@Q)u>
ze<%s%-;c(8wfchzsLb>0Sky8{JL0MeYUO$Rk*lTA?~v9xt0x!><ypX(ART=Zibado
zIZsv2D;F(V=X4HUO1d+<;L+n1s3H4Qr&0L{4`R&7PTuk95(!ro;ghNl`T3M)dqA75
z%1`dJ%$Z$v#n<MQ8i+X)HYgP?<7)ZH)m-tp+LN9Z={_BH6A<$ZsHqFq6CL`LL(yFE
ziCW8@DHCQ36bU=!Ii2w3+{rocf>Ra+Se+7dYC|4l3!k=fj(`)N;u8TUK2;DwN0x$O
zy{6q$hhMo9Jzm;bl`Dgi%;grviLpx*f1dyQ?6&WZ=E;xv`7q)pImFLPy0^1P0C^;-
z{@wRGF6a#^a1(2l#&WJ%pti;GuA)^7R<4?(RR9)Rb}Ex;d1ukA1(~bns1?l8Sk5^M
zBm^($Q5n0mmUGY1JN2?@+3KQQ2VO3kqn7~|QamNul(wR%?SSN>If@ye@tHYbp;bl8
z4$NFMM>AuYRyx-l%~La*(m+CjPJWq-1D)JZM>^-*EH0V_Pb-#3Hy`<td*3AI<CZ&O
zKg*|y$a;5~_uO52?(Wj+%s*}{hjuD|Ue0ld>);(%sVUo(d_)zuF-+=wGO0lOQD^$@
zgWLH_66;?qQsNQ`T+qSfDx$T>L6Ege<3?Kaj<eeq@NsRsDOF^ax7jo$S2XcS&Ow$N
zqSgT~8lCt>e%%rI<m&V0_}m-));s0P&0JUxov19bf9G;d_;;arQJlt?otjVFLRuz%
z=2H8y{GsH}1)Ueug(XGG4;O!O6kdEk=Es80tL@0CYyO$Zc?x0jV8<O6T+I&~%bKO(
z-cy&oN@nA)2}o`1?~^MRfrNZirngA<MbyIb;$>IMq%YCoPdiWzk3TF75=v3psdA^_
z$SY^);P4#HQ-X`{FVk{gW4A;r2$7zc=4rUEu>&WjHsvsRNbBN9vvBJvPQzb;E|!X6
z^2me9b&UO|I1TqIE;B#!rES(dk@G)$ixb|=xxeG#zeD$X`^}S-<}5#Z?q~1p1PLXX
zhg}89OewZjhfY-bs5xBQtk5%Ox$;4mYr?JbPE>BXAROs0BIW+3Z{i+Ku}#NZO^$oq
zGvMAk;j7a^@uE1LFQ?Y@TbCSeGvKQIvrzh?0AGu>FprB*%f!tKFC3TOCD><BwCxzn
ziCb2OPTW)zR>*c=C~IlW{r1GIzzp|u=bWrJm5JP%Se&|0d(%3N64!I^u2;C&u17NI
zlkdqWmA<{*taPvY@Z>8sebWBF9w@2rN$%dB`TxT)Nx4&zJ71@p*36Myccvh<F8fjP
z9sdxnLv00VKhu{gEmD->C|5gB>(4sfM)XOJ#%jq;X6_P}(aB6Hxw45*awc;cz4m?k
z=uVr(_J!wiJBp>TGG|Y+%sRdC<W(7~;umKEqvd=U=2~W*2y~um8WQL%x-;}iSfsP*
z=G##s%WYEB_>OMpN)VkIDf}dCtKaq3ig${aOBMU7UzEglJeM<y6@Rp&-S7I+igOd|
zc2D^JGst_!tBdUwUliqPJKW{*GrgX#cyssfor3dG57VWWRs34`xms~u-HAVO9n0md
z_D^3FYH~Mh%l6j-3Mm`{Ml2kXOf7tjPHhZ|&J9z}H9lT0lJn#Jq>O-D2A8|-#e?@9
z7U{S(A^q9YGuqz|yx(-<hhoZ~hCOAGDc1~A{s<V=IUMS>{QKQC^YE09&R)NT*j>UV
z&hl6Y8uL5(|GLQh|9QQCZXAf<?YTN><2T3r9ucmiwdcawWMbK5d|(6EKT1_IHlJ=-
ze7a%r?2OH)cll2*ov(L~2QpAPbGedK_|p@+{Kb`T9`!IvoO}W@Fwmy_+D=pU{afV+
zr`EQ&&d;5Z=`H<cvhlM!H`Fh?#4hXHB{6&5WtZ4BzSA=o=^b14+9Xgs`*P~66{k%C
z!xvvVx}wBTE4Igr0CQRAE<fQ?LphLnH~K*4^|*t~yX+F1>mn|F@u*wLMnO1prrIkj
z-z!JON@l9Pa{3>;ymQxu8u3~0jeM_^&eAw)?BK0^bjAt3(0{-G|G5tt8F+S6;}iG2
zx2k_He!h3F@W{=3<>kvg_xFllx_>w8(^9FQuk}{0S~BZ%?e;9S%t<OHi}fZpU%K0E
z6qoM(@_nA0<ofr{MH5%4#Ppu{5!-Rabhl!}zFN&rou#{VQ*C^WP+A$=b)S0wkbb;)
zl3?lg6*4c)&+Wgz_peq%CX?_<tx2Xo4P0I+@v>%4RM_9Eb75I}o^A2(vx<9mY5#eX
zSHM_ve$O4F$f^9ZJrqyA%M*W49#|=Q!c<tSt9&!}g3yaPLS~&j6RnoYPg&EVn6PiI
ze#5mTj9bo}M{VY8|CqYuuiwXaHI0j$E~FgV{h77ENkp`Xjg$T8?$4~4)eWcy!dBgY
zY9K7t4OnQJRywClV!ekR=lgTw2Lw;GgZfn!<@=Tt%?hXsK9%qE(^q3Z=Tq~gf7GVj
zZ}~Jkpw2kZPk2^4sAS!9+n>Sx#vabu#t#=%n5#Ni>Se@$$B7Xm-1nEUXf5ZG0X2hq
zTFQf&9&lKiE2gW-h`e9MqT$Z5Qxn?qX_+uv2-+N?tKf35yPZyWRG6zW^a&U(SqBPS
zO~t9gPA3+&$T*!)1QElLHRn<6)5fBf2@^l8`-JfMsmZ5aI-Q8++{vw2uJ}Z+<<8Uz
zy9J8Go!*fxX#GrV!*TB0`R6XZz1w(o!|~OL{o#k@)*rKts}6ncme7AaZq~lUcOSEE
z{0!#!peOF=n57&rZQkp3&tE;0KDYJ$d$nhGqz*jUS(kbv`dj+Jxn(;QYRc~3H4tk!
zziv0QLhR{@IFYE>zIJ!M>Q(0bR%(Yd4$nEmC?3(eVoSHlmW?&t=@}>eH?{F<>xTdR
z|8A9()(6iU6`i%y>u;2I#!kOc(rG)r{zh@9?DQK2mErGhcpWK&v=@SY<-g?HC2;w!
zbM@96c^;ulmT@{O?e=i~^gry~jnzlXoQ=HN?v}sTwQ+cTYp&F;%X4SG6%~Y!8&{-O
zh^R=MsIoY8qS8do;o4&5o;k-;m{M%H55emFDy8Qqo%~a517CE`5l5=@ovv}W#+|6#
zctLx!opomZ4FR`>=lyO$eG9Ap)79Vh7PL&<Y^vsP?V6z5IkTAzk=9&qPuz^maKF~)
zWL;Dyc5C9+#TWYKXmiI-EOlL|U9?W8#PynJ>$(#=H(d~oG#A;W`23jD_A|fM-EUSr
zTF?GA$4`iTAAh39$NRy*4yWb6`D&K)*zC=tf4firRk*UyM)bd`|IzvfE0G)$-sbMu
z)VQWyxo3_$dMihr+jintmxby@^K`yIn>!mXa7XHkR4G0`>ZG5tH{eAp$lFG*4+op)
za4f#izB~^xtnYTN%}IV9MJ*hq*KJPbPwqs#Xj>!Ab$3Fs^TPLW3Ok~WBrl{IEqI;c
z&}Ffr>9I!JV~zUq-2ylKpK&WlF5SO7mc#GHWS(~Rp5^R3$Ju4TXbzYpnZ5gi?CuYD
zBQxF{Du<x-g|d;!?*CpY3)enTJJ45i9EASte%|f3&*xXk%r7sEUS2M}q<%C%<dm7}
zWIGRuP<XGC|J(%6`HGY6oYiaFe17rF{32=e(zxVO`iUjysV1TGXM*H3MPnW3^oX`I
zhxli<${Y0?l!(=x(D^f=vPE(82j}#M?NEtH@>bmTDH`@EVs$4x>R#ylnXvHBgvcL?
z%0Cxw{@~p9$vN#|d(V^hbBFjPPk~K>Xb`JA@uK#G&YugBKNRD@8vKyhaX&%gZRH&g
zWs@q}rS5^OT=Pm`wH3q>0;{c3#OgE@`4yqV(LzOCfi6Z9V}$Z7oK8$|0+-(Mb2M9Q
zS|=<Q`Xu5s&*?-kN2Qjczv7c&D~L4+s6O4dyrQkT@8Lb(+uZ->KAe}lqx>-M!-S^3
zj>Zo)4>g#wwViSpp*^a(Lhs(+u)6qlpK4lSXJo{tHQNt-)k7ZYfBrUU0b{ComvrBZ
zR#ojhc$;>|!;Z*^B(Z60cC1}@N^tXrBRV=WbJ8Kx1JAd{Es$I#W614&hH1$vq}c?J
z$e~F$*Du{1?|tTw&}Ifr6UYR?=XrV;bVB=PuzKhPU4_q2zpcLT{a~YrtIFan@+O)l
z+*9BU@9GQU89k>OCDbN$hnT{e%!$9{zOc%OKi#pQFtm>wnGxa`9BLW*ASBRF)C9D&
z!%y@nSjhK*NuZyw3217<Pxz^c<K+-bun@<rCGK~ML|Y9Lm3A)a(wgWCUizXKv2S~A
z<C^YyM^e*Kdj0Qn??-obKWalv$T&|iW;qKQ(=&Fp48GtKdQQY5c#1LW*^s9`jbI_2
z&~ri--cvTSoDF)a)418i())r==s6`B;XJd4X~NUzqO6D7uU@lI#<)@WPR^mTYI@en
z(6Ozg-}i2}7PZVi3z~jvneDsdX3W&bOzt~ww#;4&ntu`mnY9`;N&=Q#1!~(pOcT!A
z1{xfJC<9GEft7KG`f<oW2CPA&*0f<g5iyrhO!3g;T<n!13>iVlxu3mhrHr@}%A#HH
z%HAO@_GC)MRsFnQy=tCLOV(BLHh$eJ$vwg8vZm~WLzlq=C2X#DJbSb|E0qgNer{A;
zS-d#px7-ZZ($lJ1Q$T|zfvT_tN!#8x-n_Ep&TOWaHXk&W=#?)GfG?|i-x#Uo?h(7r
zu5jnC7L+h$-W<?ZVD;X>uxrNztC*lr->cB2R5$jqr#1TguvNX&UhBYh_3`dI`8~5Y
z-RItNAw}(Eob17l&m9{in0HDYU43TGhg*}J41H9@S7yN{?7r=F<F-iRQIK5v`Oqei
zy4Ii!aF+isx0n^QbSGIMd~v|4S+E(w4YlhpGWAI?d$_&SaPMi1>2-o;`1<~fOyHES
zyoMju{MiLhvP8VxGPT{;xfSk|LbgC^?x|_317p^?Ekw@qaf?}H6hEoCocrw;qqN$?
zcgcm8Sf%zS|83p4O{}KshA-X`)9Zw42e-wcdlPyt1@=`uFEZs?ilo8oM3HIN>8l!3
zR>4QZ=gc)<vh&WQM@8OUuXQ!`wlCR19)~G&aiDu_p65cOVYT<MQJ@~(!?c~Mttb(A
zc*{zeovN&t?w^tVdFY9?_)4!;8PEk(IrpWvI;in&kD0-g`TeiJylLFQ8m*84{rA05
z3m6xRbtNk8^avFOby|0MI16oi-y3y6usvGOr?F&;vW>3zvH<AH!TYj{94?1g9=cqV
z$h8zvPT$_MqHcP^HrJQiC-9%tcw5WbfmqkI|KDbfxvLepA379GM1)b!bEAMmhFa{C
z0&ksigikC_`15?<s!183Lg~0GxV!n73Caj&g%!c5r81%jMlF>U3!J71H>rhKqA-w`
zXCklIM4lQ)l*BIBOJbkUK7k@pr*%#zLOFMGDTXUP(Q1jAI$<?PaL^3+59}NSJ!TXt
zT6TEdtGq9kY-aTEnLr-jhQg^Q)F4B<J0^0Pw-*#Qg!n3<Of1x9K3jdnweYZD9^b<>
zl^$3%?FbQhCIOnpZJF)7V<xmCIRE!0(2^XBLuVE5^t2#~*gUX^jQ7;dEPVyV9Prgz
z|Bu`6?I;TRbMWP%v#~oQkuvC=f|l943yRGH`OsX%<~;>e1MlQES56G+%u#Cfhy_n`
z6%>o-y#uv8`amqilFwOM?i`Tb`?{7pt<c`P*NIq=;PNd$BD@lPW_aiYEiI8dA$)u%
zw|O~e+1QCAh3^iX)m&S%{8{}I?co3X{}Y}z-QM0dOZ1ghMyceMT|7~-Y-`swhfQ}}
zeR{#w)C&VCT+cn-1#LAHbb*xj1D(KgLMGl*H?y3BFOa%mSR}c{>iE3YN86*go}O3`
z3Lbj{7h{{Fyevw36xROeU%%_4HE3m~#@|})<t}MWXI%UrDLeiV^INGdZt_Uk5MFm~
zF7UGDG;jam(Sy7Ms9wy^)zbF?hxzFRQ#tof4L&58H_dUiii}A8|2eF8UUx1j5%;?u
z%!e|IwcPoOCi|mUi`a0>DKSF#re1DghBRRA6!e_(e5p6b$+834j{nY&SILN5=$$Hb
zL}i3d31;;TeHzjj>}nZ$AtcmK)B-FN{M4lJvWunf1(Q%eVGG|Wmsxz(Vt!tD`96}t
zW^dPevr_?66g8X!+7|G*N;Vm>q;j0PpvBX8c`4WK1urA_9G&a%=bY=RNw3zeWr`F}
ziOn(=&P#Eg!0Tw{;;&p>g1j)7|E!3KF=%WvSM(HgDhoWexJ=94VmDX1vyoSRu4qBa
zX82V5H8Wvf?L~9c=G<-BtQhJG8`R$MkR#pvVUCB}%e8Bn`U=dT^{)N-TWS|}fXw1d
zcSf`X3JwdN%R6*)LeHh>XjYkrInWXG`gvP@7i6xIdH6<c4lhzA4O$sgP%NC62VNO;
z=T6IP=n~<`+brOP8txpRd017@<ZKIQayA$|IZH)BXAa9L6P!6LTR;SyCCOR8eg5aj
z_X0(Hiq8Yut&S|@1FhlaD_!9%^NPBHX@@`s8OR=v(7p=L3UI{qxJ~x#5SW-pg?Kol
z`w7dIIWuuDE`J_*dr8r<1A;|-is>qQx>q-#EiSh@ENFCC5WLPD!Ms!2rhIhwj#;N(
zI-LXABJ3w#^v>xVWJx-xjM9b<JGM^PEd(8Q<k&gv1S^q;Fg^)DSJaU#IA_w-wymr5
z<UvctCwyjDjowVlvoM-&*fOUbB)1D`lBxdPbmtDW5KE}6F}z)e(C}iF%*@jgkVSyI
zUHDFdwWO^mO3_Q^u;fQwz6}w2ZWxL%5WHUhj()qq*`TK<Vvd2A$HV*iKbnnG^^(DB
z?jl_*ky?WPK{|Rni-LHBPr4#!^ob{*LKg6?E=nn?1nEqNG*5TN^C?QHLdSLlK7~LB
ze7F?-2e}|*Wj3dxzUmXmP>sN+An0=Kkt3-1<o;q=PWB*?i9MHUO;jERz}Fsq&fvJn
zq3Fybd{Q&=Vc%6Pd9#U5OV<75Wh|BxP&5&DZ45<ixXovGiPP1Cjs?0+g*EeToMToB
z!8RHrYyzqUgcr>bKxKfpWg)lwp*yv}OJb4R{m@mis3S0lK^f!`82F$}LDA3eS?p7i
zoz8{1pjPemal8wQf_judYat9)xqv;($$qp9B-}E=m-A-J1YZsi!D;ELT{-7<{g;iK
z|98sIc&PEnJo``X$^U7+>qA_#ru^F%`B?g<tJIWzo{wjL2yfM_+k7k>G(1|j`B=+j
zNYnSnS*gbYPG0wyeDuyh-EQS0)s1~!O5~zBsu`e(!a1s^BAe1y7EL=4xoD27$!1Vf
zHm!G|jE5fQ{>bD5jgSpbb(@a`_<7x5VmP%Kby4P}h$-;y8WIC-=?#1}4(iey#A+O9
zzYRRJJ1{vt$jR^HcYg445%X*w3+TXe{xykHkxnPPIV(A#J1(Y7m@WX_hB3?pRi6C4
zRIJ6GBr*{+wse0@;h}}_$X_Yr5vcLEcJr|oXQyRXR2I(sKfmwqe|4Vq|Gzi?kGD5C
zy|(GTuF}mLid*0RWHtGAY@a)5l_+AWBkA|N_!GzEE`43(k<9<H8a%iKS!w^@b|=Hr
zmM^y8{evrnV0%7qe}BeA`I>o|kFDok^PNAi?8k35w2b)eozG4-Gu0bEvl9f7*OmRh
zBjfaR$&QB-Q<+byBK6MhUsVhNZ#3Nj+i?nEU|VShPVUf^X6mz^pS)J!8D<(e`NRZy
z!#is(n%)#hc>Ye{5Kp5dm$6H>;svKEVo~jiQQRvJ3z)>d-oQOauk;41?{%v;Ov}3$
z?Sia5#<I{DEQDpDF-YjS;L=K~L(-BGTLbR==g3BCVt?3e!}4|7C4OF&#U*~{xU&(9
zLyOBfF3!+c&Y5kVn2R(pR5!g{pl^ej%HonK%4_bnY=yP>LCQly`!<vbUu#2*v^^}Z
z<ao%D?QG=bZh7d+1o%+=9a$%Eu?$`_317JTVYf|7LCc)k6N+Dh7FdH=#=_u+#!l-N
z@Zvh~G6T@oJIIc``%e{Hz((8#jgf;{kS726yNWK0Tke2c`NB?lUMGCPt$f8N%RrO;
zntv}i7<oP4RIiu<ANs3*#|>&{4Af#sdf2PiG6x*dTc$Qcw+w;yS<L_K)Zs>*)c{k@
zy#8+|_Tm3^`5BK7KFxpl+El`$zWm|qtrA|45mxr4?^|59#d|#e(8euuPACY`0{mdD
z<Y<xc%2%}YUy{iF$3iV(u7Mul-49Jq=S-_`5Qpvq2T5Ia3G}^i;*7$P_d8fbMKK2W
zgwfY%f)>qymu*U-uF(W7ngI);uF-^cY}2_=>Oy$+_)zHF`z=gUC&LQHLQZ(CcwBYC
z9MCG;=fXY@I4oyRFotfusrU2lD2jmy3Dq4}<#0c76W;R*zLhZrTK99@9PF&A6N3L7
zYy_<g`g70`HXU=Ppapc)&r}crKJ*6*OOo^K=cgUKz9(+B&WUAP)FU&c?zU3F(@oW(
zGY(yeV_ha-DwLN2V$BjT1r2`rO|}wQ^le$GhR<8yR{<TfprdjB7jlO78EkFWaPN=;
zErieF5A1_#5P!-MSRMY9Be)vGLTsLNo~rEg)_IDuOK*#GpTSnRhI1lMIRbx&Jmm=f
z4PqfQfHuPF?w%i0X20`YUD3_IPd5ENnfiV5>v!tY@A}7aPn~>;-`6V|wR!j7S$X$-
zg@6B^Ux+PpGQ8QQwno=_VN98m=ONJIv^KS3nb7AJt2S(V(^7PkQ<ZbJ3)|BBW^3xr
z<^K22D*1Jv|KIw$54U}evCm(0C$_bm@p67~^A|CLVrC^3<e8M+4V)JKySQgKJf682
zu@!5=pZ%;p3lIyN!PO0PtuxZXW<=|1mWDcu?}{R)2Q3K3bI<*A+bi$XbIiy${9Gro
zX}(VKzjvq963QAo<xkF*{8wlF@`2##^orEAdJH+AZiBXBaU9t#!ez?8@{8h!_OM?9
zU{*$3*e?MKXA^7Ivmv>DjmECCgD<2Ux<7Bz^7EkGC4c9qxJKr3ol!NIekg{inDdOP
z!9>`!&j-DUYICN%6UtoV`BHbHnhd;T{;xZ+PIp_P@4M}@-_@U2`1_vy&jBa1iTUAQ
zx4L`vUb<ge8UKBOiAHSy_p)iOAFKAPuMQ~ktcZ;}u<+vYE2a+@?09(Lh4Zb9gp;Tp
zivLK1W~k$6il;aoCr|sceeuVchuAjXxTNzh#(o0-y-nsnQ^i+$+3nU*Oy$({?npC!
zsOLR>Q;DXx_}N3*1*pSi0wxA0Td_>;`aouK5&Z~I#bxXYnaPFsBcPMJ9k$=ys`*~F
zvrP)LTEFk|ykL0mNnT3W#Hgit$HM{+cjOVU<E(ufDuvDR!TaoT?zNnq4cckTG21-x
za~Nnc8<XYx04#P5G>MJI5?eJ#z`_Ysbz)n?XSsC${Ry_dUamV7d!(avt&|d$Cv~lF
zwkcC=eKz|-&FdgH6`7z_@sh32Zci|3+Kn3c=ZpFNh8<Y6;~~e{?VvUC?iu?ykyy(`
zS}z6mU8oVB)ehfDf8Vk~19ln+Vk&qK`@>+%8Lrw>&OOXpSUGd$Vr2DN?iqG1&z$Fc
z3s`R3^2|BI?nowo=`H7sxU^Rat_w@1D4XaTmrj8nMq$nF|CHnGX&+hD=`He81^X`4
zbX=BuwdJ+X-F($$aZep~J%8{TZG`vy_w4ya3ip~nRGqNBsGt~7Tc7|Lz5lLIqouyI
znosVOBkHWbjLlS}juCWb>oF7QJ*3Y|?!SpY_gBm<H$(A)lZfcdc-SIf+dq66u_9Ml
zeb39pB2RA4t!1Azk(2u<FJgBZxY4-q6vpl}P@^%utEgqpYfwK|`AH1Yj?EAA%Ka@g
z?|y#QskXHIp6x>ic&dEA$NIty4fUhE?k9F~S{lQaSUj|S&u8Iu&iBO5om$6P`z90%
z7eNjT_<wFUvv25An~&xEFY+IMkb7Km`}^&GhyF@@JR7v%Hemnn2jA{L{=4@(4`0pQ
zbJzaN|6slF<L%eG4j1OdSl8PxH$OKad#>(F_Cq&!>^%HA=E@%7E1PdusK5C9^jr1S
z?{)vBegCfuy4MiK^lR_^U$=vI?f+@t^=r4!yCt$SuUkELC~FFgdvrhL)Zc&4AOCx|
z(eB2Zm2cl~-~OXK^HZP1%ER}4XYOAtuD<!J(QoN*MiTQ5&N}+bafY~G*wOb%m+p&B
zUU^tQbHk30CBLhmeST(T<yE@w-QO$m_ul_0)A{#JckX?A>&fL&SKfQP{r|20?>dEh
z^K};F`^(o0pW5>Ag1_^^v{h-#?COl)Z&&>L>gV#$tLNO;>|DRKx4kG#_hC^=^>rP$
zJ!jIi%9b&uyGQGl#tZNKKhNq%Y)WN)q2G$S1+Hg5-@SObYHjMK)V^;A{+C_47c_U;
z&(LD)?_Bkzx91<K=gb!jZ!~*cdTsyZJNwmkU&}sqIzV&Z<u~uex5RHU|6BL$QT)m$
z`&*~&<e94}`Y*Ye{rUCT@;7__RsGFB`Mgbh)~8deB3AxwWsArzyWJ`lv0wtLmnPd=
zad)PYeN9QC-_{(`d-A*Ns>;;-c`GJxWe3d@-4MnYqJ84v1znyGJRjCT_J&{k^?1ft
zA%-@_Hs@qNUCRWEgo;BJcbD8_Y5euk{r11PTP!*+p5*&qUuzn5DgC_K*6QO=|Hse&
zzjga(Z;kU-?7n`FivNF|?tJONyYB7td%sD&dhae2)M{3B>2ki!w4G~@&#s?+@AMzL
zrej(rVUdN#Zx7}(K6DS7zC-Ti*?*^gelLEsxqrU(s?7_w=kSKTOn<Td=a+AGzgB)P
zx^-UEJzI9eVQtgP(N0loUUs>K1@fDQ{xsgR{^MUU<~+yxeKBRr+V*e$Xu4vP<5XYi
z`)^a0AHRDl>q3O{{fsw@+`MMp6zE>QJ*Zl0#r;X?La}DO$-(+6MIY8Lc%0_@j!R|R
z^or>ZujVb*VtnZ?a{Wluf~{;T_p0l+ZoZ&9+pV!YM5`*vO22-;@a*;P|J^$O+$QJ$
zJF}S$sT&yD**FfW&8qU;n%Do_Dvfd8+84D4yShJ_eE9uhr|YxGg^rJ90#aKSzH|Nk
z<+VfU!e!StKfCQ)_k>~VzjK=}GwytoP_V4Nb-&vG9jEx&wp_Vj@+fA>|Ij;CO$Yy*
zSv+9-{^R}elJiUlHUG)7?d)qgc>iw79Kj54-Qx^q5gCQifv-P!=1zFW?koJu+1Ypf
ztgi_rx-b2%&GPy8Ib+_my4=6h7B27prTs<urRS{1>zN;vaxZ;tQ!kkH@%rX#9@630
zH(MRIl-s`Sq1?g4CRJ1K8fZw#rfFI4KC4+eNtUDU>C5BS->6l}8rAjoAC~{dX!73h
zMb>6}hvWjz+>6J0m&}}-5U1R<?BZ+X7W?SftnbTPRzLbv-N-5wzv20g_I(eeRE)$*
zk}9T&op`BI_4BxDU1F4iS?~O>|J}6w+Ba^vdHeUX^A9De6z?uxDXn<Wu*9m(SUcN!
zm!*vRtHnRA8Axo)*{R0-uUPj$wlRO4fBfMa%XcsTwMXHOQM*I-UAK>Ar_F?d5^UtR
z@ZbHg`f;TJpZoRR68o1`kHTf`S=fK%OYiwESn~dL@4`9rHd@a<cIKnno|N)cTN9V$
zzBbKYe*BHmpO9VRtb6!GZZ_Uo{wQsEWunpj6J6GeFIp=VXRf{Jn!co);rX?PcdDko
z>?q0epIh^G)u#2Q7j&k6_%hcjrd>-kJox(;E$(BYk3#vj1XRtaeeiy&sQd1=s*Ei+
z|KHkCx!H|->Hq7~vR+<Ve0gs4%d-_5KDNGgFW2DJKhvDD=howYb<>XcMeSI-)n5O7
z{`Nc1m6sK7(=IQpdYLV0`^nvX+sS+P=Wl=gXN~9Fe(zru_rJ_M__e9p>m4`$Y56VR
z3V4rSH{E}3Q6GzimEE~B?rC;LpJ!J7e>?B{S;liLM#i5bKW^pyvErsmUH$G~GdVxx
zs~wujk<N5y-E&#Nb1v4xZ@sqczc=|xfK{4N@v9flAK&Q@o0j3vdUyUArL``*n^=3(
zy~?@&tf)=ZJlFG*J#C?f@$Cy`3*2}4=O5W0`Ml{&lKkbmTQ8ilEp0ykP4jGh^K$-m
zH};6N7D{RiPitQHw@+B`C_Gqmw%Ie;lHbnXOFvk<i>z2YNBj0???|=}X^s21PQ^!L
zI6J0Kx-u`gD%E@S=J}$Pf^DB9UM^gCp-nGuMHI8m?3O7%zOh{9W}dY8@jK3C;*1h~
zbLK`BZEMq;_U!Cc>#bW1SOXbXGF@8oIV3jwdsrCjl`g+M>REYBL6;`}ik$V@X6l^M
zAGaqPtmk2@2*_@Y>hJzpSl`-Jf8|G-)PbmKzo&Iq0$y{ot$#M>+x>ajmGTSieN;Zo
zk^X;M*ht@T({0HUcMV=-_@A@!2yk#-D1AG~*^yaPTqAmeF6$|Q262-R%k>+#|I>_q
zYUc2EUA`M*uRg<i<~a7{@1A{F!TR+Aqv)}p#tPRtny+N!SpRlb-#qDr&?`CFeN{iU
z&a__lKF8bj_SMsg6STDJL}jx#*z>%S)@plb+k9mQ=dz4vXKfFuN%wB@{T6$mS=a1V
zWb;DPh+CT(x+YetF{nsHH*YnK@^AiWGdcaH{E4+53?3%4O{RALWn?>__;`+4-Z}G<
zn^C7qBM+@+5?yP$^F-z*u7?*j{zW}q&|TPmaK%d}rRKjyhVj#adyet3Ox|kyVEd7F
zqZuU%586)N>icm(z({z(;sZ67XQo>#9;~xGGyQPQ&dcFVe{Yq1I&@;D+Udk)GHItW
zXKLN`o1whibk&!mkuR>5>WP}_I<H%^W1&l#u=w12u6)a)c5&Xazh)|YVWpE~hIW{r
z;9OV5Ywx%h3pGzju&}wY?uPu0J?ZDpTk?3>NB=4_i2Jbq`=8Gz9-Mh_s^7ixVWQ+0
zp0JuZl6_|iQtFZ)JU&y9(q{a5_RRY;m;G^@WNbWt=Fc;G&KErWU7TiT{CRWUpOyJ)
z6J|PuF<G8IHs3I=GfQ>CR0pw!H7uKiPIxI~KYD!TPY-Y2jfUBA?=vjdN62t$9danX
z^ZU1VX7>40M|hL3J^S(Z`*gNL!iig^$s6vS>Fq4}%j<XQitCSG)yqp3L_16Lmj4#K
zu4{0>_`v3qG7G}kRm@vtIb{rF4CdTAJmaJL{r+N^2{KVd7lk}lF$J+M@$wLzprsJ%
z@M-Gm-s8pl<sFSdeA)GXx-%Ls2H3APVoccLAYr-vAj^qex)1a$pY%7eFiJ2=Fs8E1
zE@yCKaC2ZhxJ)oZYVKN5%kpc^?$;8+nfFS)`1-jjpGAbBYwJO$<`iC?>;;k1Iwl?0
z4s2mtWg74{PUmDV&yBi(b91MuGtNGyp`Y~Gf9GZn;ok*JheZp1NT%N_j66S8O!50(
z@5hI^%I<T%yY(#PG4svaDcf~soJz87`2F4E+pgJP?}y$`4+@Ojz~#32!{tlTy@&N=
z^grfpbyn4z##nYJL7k^9Y5PX=FR?n#hi;$#TCQ3;F)hQTEwNvjhwII;mF)+Y?%i$D
ze75%9!I~{5#^O?C&AU4@8@DaD-ns1m)}?R1-b*vu=zg#`DcRj;%j)fx3ODwo>!n`#
zbbikDyG|D)Uo14O6~4kGZXBL)Zqtom`HnZ~rS~5BCoar9P~3lLo=5d<wFrKLmfOzz
zH{Uf|qOQdF+rfQ?%7nQurG4)7PkZ*QOnA<1i+Ed`HCq;KZV<S!{JX-7?5T6PL*Lwg
z!Os`G`{CWRM^CPKdA!lkKDm1P_luIp_Q(We+(>Tc{HB%q;7sH5L!3FA!>27w^8J53
zGjB5AI;VB+v(syeXPfi<{#k8QnfJFq=lR87haXfgy(#PYT`t0%`$zrMZ;W{dmz~`n
zFBFyaXx3)8+pl-un<7&rwKO^`;CK5;akJ@OeA^%YFZplhwD4v2oV6<#OuN3$;(&7K
zp^pVWWcl_VZZ_pTzIgs(FKY?OH~K%eS07{F)&KqbX1gD2P0s$=vtIn(w0C-2^1}X_
znNI&x6Y=lSy=Nb5AH6?+yYj%Yx4)meSPQ+CV2GRi)?MjN*oW26CMKMpHrsrA(0#qh
zGc%SyyzYJNwa=FHLwtQlV^_@(t@2dfr+a>`(NELPH?hj*dqwK)chxSN8NGJ%tPQ@!
zi_X5?I{9|XtXWN`yNst=2G5+mEdA{H`(D!%cb8|?I8QF!@?QO>3fq-t-UjWhdoHMW
z?aE32|2y`TSMU0eH(5ca?fV$ZO3sOw-B|J~{@uO#Prq)Ry6a}<ey*q&O4%9v?(ZmD
z$|WWprKyo{PIcbKPv4D?&Rnp_&u!M?`1Jj8-igr}s@F5UJRX(Iygd8ou~XZ-E*aZA
zem0xA>gUZ>rQ7Ql*R6FB-8uL5o7A1nuNjx;zvtV#mrq@0Tfy3ng`R&rzFR$d*3z<f
zf1FXw%L~t(9^EYdrK>#m;cs2$Srb`b9bBN=oR-5Bm-qc|+xznWT;Epp#+ce2{x!RK
zwpnlVg}$?^&n<hHA@wd&F8_z3oY)7Q3zF~e8%SKb8KoVyI(d4Y>DixeHCJ`M@A+HI
z?5~!0hH-+{i|6Uf*15;G-r(hKJ8*a7#cher&u8lXu*=F=$ZNYUaBcPSlY8wseHS~f
ztzLa}w>M{MMnJZ+apnt$YYdto$f3!g%y4+JtL}xUe%{0P?oVg9y}EY6Bc=%EEMAAV
zIcsHo6Lz18X*q4>$n-^LhcsLL;RnfoByGBj?>x_Y&8o6XIQ>C$;Dg^B6J7*L#BeqH
zCl+Ysa4hLN%N;CKkX@a}@j}8ZDsP4D=0oM5Usa`j<k`8wR;g3+Y|FY1-|I#;(|0~8
zdVS*8Pceu0X3Hn-JL>iPsM=N*|HEp$O2^LHvisJjum9)k@VLp$?56w+l>?Fn70lai
z$R~XF<*D~*T7GS_;a4|?NfG%@QmoRs2cIPEkL2h6ZIpQ6{HHQM83p#!AL|s8osYKe
zvv~V4>!sI0vwZioy(|ZKZ@)=fWN2cyZK;jm6F2#meQg)_FZkTCvB}f7cvU0Yq_(zs
zW<2+1F5humA~WWQD4WXrDE<UT6$YML*0*+e#V_9U@#~s)uip+v%VL{#i)ZM!Pqos!
z^E#Ewd75=sPf9*h#<GpCr+xW(Zr{z<Jkv^Aq<@=B%je8IJT3Q?{0!?Vy_n?3r+riT
zs-GX6)poG>#_r;OFRb>id0BMt_>Nz7x0TON6}OhquMp!moU~Vlo7<}5K<<-mN9^u@
z-+lD{&c{uLKJo8o2Y&iI>9X}<p)&;vCq$Rs<=W@Bo%ibO8Ggr$Qa-vVD9`yJ8+m1-
ztE`6WN#53a<%#EA?B33IdSLSLuZrL~7Sn~FXF8-Y&E!l;yO%EVBe&?xneD4L7$|?*
z#&y;vVvEp~RV!A1Vc++2x$+vex6k{Yu|7%P%C~rLx7zBO)XsnBJpat;tgb!n&G)2!
zPJwLmRI7-U&Bj(oBp2oMT0X65>$+KE`FBoms5ZaGoR2E}@9*4fytBV%&1u`i5n2k4
zk^9~3*2TvL#omtJac$F*UFW7e)IMBU^Gsz%{>{|imHDSGFrGBXxbcjGy(wNc`AcEn
z=etG=c2@hXnaB9r`b`rH+g<4pn~PoNK3=ogQl|S)vi6RA(Gvr2zvi<|OM9k1H7xm>
ztMKke+Ox9j#s+(zcFys&TYRNk+V|>#BD0Cj>H1d9SE`#NxPF-ax0WttTV|O(NiVI>
z(tK_F@y9(u@7FJKeiwXfzU3!3UDsbu^B!&%e-*rXbF~%ME8)Z`%f(bytho3pU}^g4
zwM*{Y+9DZ$M|8>V^E37>)K>`Ia-k{E)75s)+JE1}8+VJYefddjtwHyN6Xz{H@0jp_
zCtLqo%oiJX{rzUwB3`teyLVxo)UT3<f;-#Xt_AS(KIOdTyg-iWtjYx+FRSg(Z@m7^
zF@fn7U%<zmYaVOIFa$9Ku`jU|{mGZ<ymr59`}J7s8S5weJ5HS&qIrzvm2r=-7MrBK
zTK(qWRMo9KF_lKA>nnMJxq6$*#6+H(vA<@#cj9Kl!Hu7M=XkHLuMT^AZoP$=ZS+AM
z-Oe>nZq4M4iV0D&nYw>-sZ_P>{iiA=*F93$4u9NyEdOC{QZASE0-kl}o?T1t<w=~U
zcJFdQ`7tT;jwzEi2Ul0U`?;iYR?7T!yMl|771{4>D6E+hGwF=ofolIxM=nMB?ElWU
zW|D96)HA&2Zr?w5PM>p1$ZMU3C$ki`vWVp{t~gkALOC`4>7zpq;&cB#{g#_`qG{i!
z9+PS5s&C#lIeeDe!ISCVbbsr=St&I!(mhl2o2EVA_03Q={c*KmhMKnQq}-)>O~t(z
zZP)Xh_Pj4z_p*H1yT__kE9ZT;pQGAu?sod=<>DV(_8nRONNnwkR~-)e&CJz4*IqAb
zI~W}&Ty*VMe$u5g2O7IqWd5z^-0|u+AN%&0u&u6rzn*U?zjnAOts=o%e7<blC(Xmj
zIXjAt{VS>yKgU%@pM1$*E^)@H^Z%y^S=$FQ&8Jj6{NUZ8v-0T6)t6LrH{EG;5-(BP
zu_WF5MD!xBsg^aN?XO=x3O!V9Gux@K=I_+pP2Ei!!UU_%yzcK2oxb;_&Y$F_myS!P
zsrj9cytYMpZ}q}zGu~ewArAGazn}FzP>z)-@#L18Up-mo-odCH^Z3qxo)cRmC{v)b
zcE^`@jZ^b73^P1mDbC(<*#7R{CtG6*UYD82I~L!&{_=d-rt-6$OD=2W|Nr*n+9{{Y
zJGa_AdQy=0z2<du$jv<$WnR2^*LC<RpV5Pg#6Hy%HG2=6J^nWDnBFPfOkuujhu&(J
zl*oF|@N+NEynmu~*Bqn2pO2m6V`bUaIOSB=X}%`~GrylX_uHCh_u<;IUFSc{*zW$T
z?J~En^5NUTXJ>N?O<)W-Sj*1$sc_}<&n{1|9yN}e=37*KC_mC7efKuwYR4yYZkA=w
zHx4sUo_n`)_wM^kt5ogp^S#>}tiIm)&cC<cOON(fmmRn`;nM%dy$wGt_Z+tk+ZU$(
zzT{@~+Vy#E)2b&QzV5)@_4bSM_p&d04k%uqP;|y(!R(Ds%HzI<@Z}o(-ST`z+}%x<
z$-;{^mPQqw`PLH8d}7z!r}KC1ot+UN+-LSQBj@r*-c73#HqWy9Z@t;*KHr=*$7hC`
z2gVw2wO{ygR{ElIqH)vfwF08v1zH$~7OYygbY`UJWYdXp7jNFJG~cjfQP``rK-Q=G
z_n0k6laG$PGSxL!<JGA}>vUduHQZjh-}&a&f0^~yFCCbyv*uc8@x%=q;x^x%^YYm2
z1uBn>r=DQGJ-sBlswbV@>!;nRLvIYfyiH9tV4t!!rSe?OJzn)I#Y*nCs^_WI)tgT{
zsCDbi0e6jlR$+$iye*7Z#S}t!eBry3$00mPI7q>8$7;PV_ub_6em|QTy4$iOt8+rb
z?df56UQZO=QJuHY+Bko~H|sC9kCS80zbs!9Q?zW=rVD$YyNkYEwMb6u^+IW(1%6tu
zmNdDYH@P$SVvF-_u2TjEZrodDwl%R`xf{^7`MI|Fnf;c(+PHKpZC=fuw0n2pjZA~t
z_ljk{mp^-z$0V|4p^&uHd{!>;6<-}5b6<0I_}<7S{vzV;{8#%w>zXq@V12;)TJb{v
z?=xO+a*~hDDsu^#$|{`J%js1Ak}qic{@%;#`G$KJPA~l=bZUc_lwkAwqO4!PmJ}S8
z>ghN*(XhgZ&HotNYev4FgPAwG1^ttGzkZl4RV%pD{_2BUHx{&;o%Sx!_*d8B)ZZxc
zV9K1cM&F7mEd6=beSh<;H^N}+hYTN??#7vOC$?HoKbX*cw<XFrdxel*yYH?h-edfH
zM`s>BZRFsc!MFLLaz^H+O8;vm@7~x6`($`$mfd8zG}+>;^!MXSr#Vb5km8AJzL;z9
zgLm7FB{#b=H!XTH%Pxjxd3Zp}&nXX$c!IRIIBfXjr))ndY7d9X;l4PBtpDex*=)|X
zv}-$hU)$K6ee$x#ZqCfV%NJiW{<z)o**~FU497n5v0D~!pR2j|w)SSMtlXiS3;*vi
zFnif~#OP9HP~=Ll>31eHB#Wn7-^#hb)AxOy7LU@}ADNwYtamt;zjp9HusWzF?=6=A
z)62R=3#RhwUC-S+>8IGbc&&vpUk^P9Tkl-|^{LzS-Ok?KXV~vvX3unD^4#JV&;0M`
z%vlw(9B=-vumAW@FDi-ei|GZX2SGKHWz3m&=$hBePgIFewrApDzLFr``uO1oYni&^
z+g{Ii-gfra-Q0G$fTGe|>G!{X9(cZTSMIL&XJ_$#UZ)+Nx^1i7{hmpVr{(W&`}Mo{
zY{DYh#(v?eH)sEo4f1SIcATrZHUDd(%7n-OnORR~rq1pxP&*L$@p0R=jGJERO7jn%
z-&}o0=cvyU;S=eLY>U}F`^@)j5_x{9NjR?aTTEW@OcwU^uvW3lyW=|l<ShMlOh>zW
z<AVCrZJXJ+q;_shPn{Sx?M2Vj`Ed^>iX3+DGx$<>?6|p-0O*K92W#DiMApPDCydve
zecXBS%!5S-yPMWNn$B>Z;k;F`Lw}KlBHM~>E0(spG5jmE*vPkap5e!{jQ6HFul3t%
zRbX9Uz3Pr-%*RfS4||=DPMx5oU>ft;peXs+&4nlCzUrT)^ETX7QtNa;U>jFo@sA}_
zzXq@iIeZsQjj;K8<wBZNR5tSxw`nU(R(`0LT446!h~$iW90#%+u3s?GVbEOg$ZLL+
ziTDAjjGc@gjvkC2j2@264Ze2i`S-&wG-z2bC|f-3f|K=w(gTYbjUx=2R~pUGYOej&
z)*H{FGx4B^q|Zh3oIQdUc+N~bC^I*D)q5Qyo=FFlZYCT($S3)x_uvBw-9BF-lY|5F
zZm5Wc_o*``Z>rqfeQ1J&PUpd-8%efn?)**OAk5~RxP@=qvS$VxZ@!XDy~TcmrQNsS
z21~oA#dEDVmJ_QlbT6FFF#Ev<PQKX%aZv}^PW3SG1ni%Eqi&Y(0fxJ~FW9d5bH->n
ztCHm6WmnZE{F+=W{f+g%oA34SEO+N;mhc?CckdSGw7eZMJDQ@!)&IuSwr<Zol=(Y?
z-#{jACBq~s?z<<}zFBAMzGJHI;z_1O)1U5q_iNhn)loAWtd&Y$d=gr~vq3cdn#|!}
zokkxPi}y{^Ubf#c>s>kH+QXMBl6}1&pWfV?t~KlM)F&%Uc5ktgb>Ai>mE@iF^O(8$
z>!qm`iI-|OEYM)qxsdj|I!ht0qS#h1|JchaIl~{<c)i(lo9D^rPugEPD>ftLneu15
z*H-baJuF5-Cm1Fem;YGnq_$$_g0v<p@9Oe=_iM*2-OueUjc1?dkY^LuTphdT$@k|4
zxp^v;?4LKU5O%edIQQJR+X}>-8@VU6LA(FW+>_tiRj(fpdzIkGx?k<9-=_4y^Gna(
zU!Xj3{@T~&C6kXvbk?YTVtw-b-R#Q;!lymr3Cx?PY~DOq@zmYGNmf?sDn-}S=KJ5W
zUh_^bz>cHCb^@#KC!R9Bp0)yoJAo}-ucw|m;4aK{O4|4v+xhKtp7*6S_U28k-d$F`
zvv}{s(}vu?ubpy|<azaaMweN0)Z>Le+G5PJw}hO3R=>?@@9o4*`g0y$_wA`xShI1_
zM%B{`yk2wguiE->x|&_o30>(=JFC9#<zr*D;Cg#bhKs9f-=hr1KBGB>H+vcK<}bQd
zvmjGm<l)21(XSJD=4&)B<`0>&a1qbcRE-&hYgbI}G76cta1oDa>WY=Wi|3y*c3*L^
zt0~MOm(|iFA-B&!!PKm`w(#cirEC?vA8sY^y@*`9h*@+?Sj5lIla7A%2>O2Q!ZfdU
zpDnpBeqXHdDe#}{v&T6#f286!7`JZO_+)8=8Iy@{<B=Sxi-8$)n)m|zF8pKVx}L%1
z!KZOp^5<QS1I_vgbH7R4Yd9!t@Pqkz)>Ct?BfJ>}B0Te&c%7OyClrYB$TeMbz47nP
z-48tS2Op@s*?4YqtLUu8-wDmPl5Gz(S{xSoQMUf$MxGK~saej4J)5r0WaiadlViZH
zSz=eYIr7)MYYop6Cd+c1JG^t{tAARrSmqruH`pt<LiiV3w+XZU0kPShx9cwcw0>!K
zVaY?q8%g4<D$CUc5{e{>+jak!dE93)*!??b?Q6v+%O>ZmRM&T=u2|0#7twa8O#Vkl
zR_o2u<Y=eLj6Uys2H$dTJ*)gan<3Ts<)Uc@8Cl=Ywf`=k#<=WSVpjR&y5qMZ`3=^;
zOWAiv^ToNU0;|3Zwd(S->?dC)So#Xj)jDu{ciyRb&tv*Ysm;FyZ`ulmwC-K`-sZ~V
zsq=V!!&JSSXC%6w7qxnS%lyH*C1P6|+&nhSd0uwlw1|vRXEUqEmO0$+Y8kGvYcv!t
z-s|jL9NYY}XvX?ntse#ga?HP)l$(X+*NfX#XZlU;{q%3o+e6>KnsMKspUr0cS@4I2
zvHfqxqWRzU)@Sq7%v<5sWTk$3L+h7&dAUD6-chs=ez4|;mEgRCZ?-E;%}76fy)#Q~
z1!q9x=6vnIwJckOPB2ZF>JXMzCwAiedtHV|&J@)Nty<avC(EMmMd<u|-g7sv>Bv{!
zop~%FAq;CFM^ZCSV!R~z;|_;A?0B(<XMWxZs_oszc+BIS{cneT8+4x-6zsWr+P<`^
zUw-k~`5RB)72v$1eD<Q`x;-o!&5L>XR`_|CCe9H&&YONNef>}AnB(<7ml!C7h9vPm
zV~`M?y5s)#;{pGIeo7YJVOepPt#|93e4FN7yOY-}<l()Q{em-HqeywdMz$5X28`La
z3cOY(NjPt1yCT&f(jd|z0wN)FO3;%#H>X)`dbh&j!}iiJ8Hqo4(k7^DoVe}5Ad&F#
zZ9Gqg{%>Z5#w+I&n{uQ)93C~LIXpAt;4H~|P{7=NTSdQd7w?r?KljUm^_&dMZ)(ro
zk8$ss;P8x%$=qOvt`k$|?M0@IMjm&!aWJ1{dGq$rJ=e|W4nI|CJjHT>N8<Lu8*w+H
z!j0}khVQf7ki6FX?ANsdchBwnq%dP+>_*+SGMUyd?m2!>@A_Q1cTMbyqG#2ye8vUa
zd^dRR->{2kCPT=(gd+9H@8^a;QoBFzYH=gWns?`B&HIoi`@!M6->&itdndn3Ke;$9
zR4QKXO5ugQ&F|)xykGF!c<qd*54V2Zxr=G`g9ZPLx0VVoasI9QBj(P;sjA-^eUl$?
z*yS{toAF-gxvchP?fUKV*TugbTXwN%{TlOW%U4H#ack7&H`*;B9@8B0dEdnKJYVjt
zk8&?tlkj-))tNgx9&K>Y6U)eF-OsDH{POuw=JMwB1#%gizKI=ppK<5?`TKp_Dh?K(
zz54z6$ENh#NpX9>&sm)tQQ-M|c}H&2KG_V_`agR_YR&&XyZ5zN_E~w=xet4%|9=wp
z{Jna1npEGyInTXMDR-CWz5P3B(F*16s=Oe-bG50ml{X)LU!qpNyUsge<?_vn-!A|0
zxZ88(xKFK*NNL)~viox^W>4FG#{Pd^+Q+m7vu@2kI$PF$9!Gl9EY7Tjlfo87ab~Gb
zxcZ?mZJ*KMsXy1Ad4Idw|Nono>rG73ekN0xmR`xZ|NB)#^&TECo8;-c-vlk3ZgaW1
z&9I15RDa$s-?YEw{fp+=T0hNZ4m48wAQb1AF8Q<S+x<cVhnw$(w7$$aB%b4y^|)tB
z-om#}YuJ3>GpUQ*3dnF4*Wh#0Sb55L)}z%<As?%`ZhqU>{8=#S_xjuSZm;PP&DCfY
zZxQ}JFTckBx#g(`8qMY{7t7DTT=n|pvENHCyW4y8=@qnvZMS^#we&fQn3l=?`gM^R
z?v+}vj_|JiYHnF193rx1QI5U!2d>z_3tdfNS3FZCwWK0bE0ta^)XO+o+<sVcdf|`K
z=xoL%Y-@NwNIfvUa#*taE~qiIo!@l7sL~1}2F)8LG85F+#ql2CKEVC$B}YiWr|*i4
zZ|<F7XDhV4cjD?@;}yFtd7gZjz4OTLnffvsCH~fbb$=|`C%>f8>>77*KyHde>pPt<
z9DjsF0t&uHq({Hn67@E5>p3HxXAP%*sn+!yZ_u?AX3##f@gUo6g(Z%BtUM9^v#$v8
ztT@<^$23jcJ-<F9Qj<+MYs*5DInOs4<tgm>GDG~WgwHF>8}XYaKJ60MDA`c?{_VmU
z@s;umdws7l=Y0L<{B+}GZ;mzXuFf}Hl4s7Stzu$ZvrD&>V<I!p54*?%p$otCEC{Qx
zUaG)&NA}iB+rG;0EeUfXZ(O-KE$y35d#;`Nw>yG*+I(r(Gs|pq=Ujecb!Oe+H&$oY
zT}-*Sr%QD0|BqbyAA~HgBs{(#zw5tx`I^qmszis<Z|t+K&*VKfSLD>c2N_JlY2w$e
z&K2vpCggT8QzSgE@_%32-ToIxq^o4stT-HHwPxw{dYP#$Tees#wDT-xw&YoOS>W88
zTEE3FdgrM~-=7k7{;Q9KYkOdcge$)j7hj{nr0m;_7E!Ljx5OSw&-%*K$8%2nPI0RF
z(`^j*|Ej9iS4V%bKC|)95#tEs84LcoF-dZ!s7zo~P&T&DkG^Z%_=3AP{ciZS6EicG
zJ8EwDZF~_t$?LP>cu9@lA)hD46a5wDch-O=#loJ|mVK<PKX)cPZJ(9hIiB#e9Sc?l
z1x#g4)$c2KR+;wWeZkTNtWzC!DBH|9?81@|A-Fc3kNetDP}^Or$iTq}1f4)+kwcL)
zllwKkvlhF9TQ8O!C^%5?<PEoFNJ?zri>^7>o%oo)b6#uNa{Kq)rWomB50C1iN4+i#
z>w~;SBeb_%i71p=XYwGgvpRbEYo2ed?`|wO#TIIov3_Che8wZa414wlf8o<vam`~*
z_1X7XIt-?BI6(vED{T2X7AZ5?#qj*uYkbg!<<4e>nH#&yoS7_YWqlWKwr_Z{z5dso
zrdiT=tyZpHaJ=!~+RZ#`G9#FqS#R_mj}4L0Iqc+^$j`%eknJ?X?1biB#|_dpWFByl
z$Z5Ig=HsOFy7^`9q@On$4}4>5GY{C~YdHCYlumy4x3*q>m;CN4O><6uW!q%F;qN`)
zpE4z1*xA2VC+>DCXZ>2Q?Xt4kEYplvCE;@%ljdQG`)~4oMy)vME~%yhDwbtS-Y=;(
z&tjVZF1FPx8>1b+zkB1b;KyUe`vyg7cfqv)->&bcoSIL~^@-b^!5OEYV6x{aci8t`
zA!}cnFv>;TJ@{(=_1R(Sha23#m2LPwudR9~qtMOUC+{WXuqRgscqhD(=iD~=VE^m9
z4f`M1&947%W4t4-V9K8#?oqnYnfF3xto?pG_4{YrGsmYWhbP|qTP1owdY#W4-jh#5
z1LH~z91m*8d)mBjxntS7=Tf~%%7*#zKc?9Q$t!=|xP8^r^V7Gi(EgtMzFG5p$djsb
z_dQbkLZ0oE`?E+Ue_L|(yYI6nZa;Va$`7B`eD^<#)>N9jn6ty0L2c?w&wAt1g1nl4
zW|gdUwxtG}MT(WqPfC6<^F;W?`}@|{GyE)?Qk<k@8Ez;xkzL;CjA~o)h2Ogu^>6)k
z`M_@dWzTJ1tmoTTI!`o*)na!<aaYz-u4@m@Zx654{<<-slC}0pOw%Vty_K!Sr!W7t
zuI~I8!XH+iU70?0+xm}TYEdUeFHHK*G3WH}B7cv2jNw|VE*AD$+zZycX}{lEqG;Ch
z%#vqD2VM%4@7-#Wp)8PgUF-kT#hkzUve$6!WqDzq>bSP8EUofXJ%3*wQ*+e$dCj|v
zy21}A>8so4Yu-I+uzJOsn8y~{UFPj>9vUkaF0fzyId91stDnnk-*0Wx*s}0Kwt$e-
z%~PjC&((bl$+o-m*7dx=-6MRumRq0be5vRzs1Eop#3EeckhOEsBaOo+<cjC1?w$Hn
z!=H5z*R~sr_lh%Ch)&Q_40Tw=%zJL5rvA#8Ij<|uPi%PfEv$5nz+RasLEXK}Hm0W*
z8YZeKJUzO2t?iB7_4zs8ytykE+VGxDQ7fMpTeDuyL6=vH?^DnMPL>C(3apB*PFBaA
z^535hy;NNHe_rmL_r^&)AI=Fcv~f(^dguK{_m=gK3gQeR_+?cd>@gBcmEIXr@Qu^X
zI3_gNre|({|Kf=UQ;mN3u+4wD=GVPc!-|0AIY0l@s5Llm{k!wKj6+|4KhwRYe$IF0
z+3GQCKVSM%eR;1IPv6w|q-!QMip|C;AJWg2hP@AUEt@szTKWBlrEAxR#LW9){=?iX
zQ{kj{LF&1)udg#Ltb4IvLPlct5!s!-lJCw)&e+#qbiDuILc=OUwY;9><wAMJKQ3m<
zoy{^(x{<jj|Ii-$Dc3j8>kY{_&s+C&j`YNhVZZO4h|IZE@QKIr!dJr|7U{tOear4G
zo4a@8Gu~GpTD0~kHLGt)KQw>+$L5la|H^tQg$}(||1kGl#-FRro^PM!e*QB3#n}Vz
zEy^PjrufzOo1XQXye@lRKzZ2CILD14J1<q1|4@D|aPRCazLTf6o-KZUSUP>td#T;p
z+7nY3)H7Z`*qho=Yr&pfXnrDC`x@)q)Q0W{1%fhiP1bkwcCfzV>VN%Y|NDI`Th1=2
z{&GYs>#e75fneX|I&&GR=|?TxR!vekywK8P{<qiF%vv@rn|0W%3xww+NA1}0ST)1M
zS5^3pvDl5{GdVk~t#==ewOAf>k5$(A?zY-#kDgr+csDa4San{lm&A&I9}4N`n0j>%
z%N)>qHX+h_jzByAqk~R|IX@=IYwV2Plq>)Gw8HCGJUjCC9xXnx?d!jN_pe#scgQH0
zwcNVZ_j|OW_w>@)@Aj<xQD?_>u%zLm`LjD^zpK-(eEys}cfY-P!pUuh4S!+`KGxp*
zSZOxj@WW%_`6*ZXbdIuon#qx_G>5Yw=()DwITs7zD=ZUcI;5rT<9g626~+YaSg<N+
z8^>QbaZ3Ftczs5|bXJe9g;S0S27de;VWAn~%{fKW`ryvWFMXGk(`(Po_^~PKoul31
z1e=ruefE&7lSXsm|DIXB(J@Nw#h&KI-{0QzN2W5FxHwnx{Mveur6DC?wewoOt@C!y
z>(`PokkKkzb(Qfj<Kc;)iwe#$eYI2wb%;_}nHzYI_eUE`L-@+2T)LqLR<F5GQh(0x
z$alel@B`Uvb{y;aZ2rXR@X6@5HrwUAud*AiGpsMY;@rrY)o^`9*q1ZL^Y$=Xt}t4#
zkzw!Lk1D^OoYiH>XVvB_(`xvAoTc1K+p3u1ox;PdG8y_(dmJRKPF)CWJ7@Go=kU(7
zA5oQ48owP)Nazt`OFNq2wJ-3&R_zsEnNCOE*pukQqjn_2Im4po8k5?g45b9|bIWq|
zgzUC5&Fg#|c;#KJUX$*|lYAzb?>%Ol<Sdza@|LUWr9Pca=4*~yrWTx>(JN$9d~DkH
z#SV`;_iz4qaNCWf<%)*l&ko#D-J`X&kEMk%%kanY4WiFwgCl0~`2P{e-gV$*^TUE$
zyf?b++c#R6``$moTfh8z(h=U>Y`Y)2c^JsN`@i7<cet9vwGE>0&g@*c)hgjYheb|n
zc$iF1(_uA>oF*`<+t=Zo0rUNYN8Jsw1+yBDRWU%SGrf}gSC%KFF1`JqpL0)t(^<ve
zos-|mmb_o^<Ls3A+_L%=G9TZcS8g_L*0z^>+G_4v=XgHh=<%5kPw==uGiS3q$K#%6
zVZ49l&$+GJ-=ClP@XQ}Iu8MVA|Lu8lrY7xK@uTI&pQp}Tf2Jn&UN!rjzr|_$j6Pre
zSeUlM=x{Orm+g00edWu4{94$yK{;mT0!1aoIWI3%x2-Tf=+M+zaLB^%jc-CjVnd?n
z!(%KBiJ+6Q1O7#B`}g8DL$dz1ssHuF_de(BFYjK>n^*B5cHQ0=6?cu<^Vr|@9epxw
z_VfMId*4r4m^{lsSjPOPsrj5E1sVr}8D|&2oAR*o(~rZiR<w6MU0UlMv6J&uea+&B
zck7>Ch;ThR_v)|Rn*33#0^CxgS7msco-TOz;xzLX%hmCLyG^spZ*MBMGvq%aWA7wZ
zlV|(H^lSRv2ezA!K5I~Y+9r14fZ%cG{hV1^3l?6J)?WAVZcEG!JN6szRPP^@30NHX
zX|e4?E2EFQg*Ps|`~SrIeeE|3#P>XwefQQbPIv8|-}Tb3tmCh!DLQ&Ho@L3hJ&=%|
zQ=lC1x-lyM=I?W_PQ^0wU#VJ<<LGg)i+M?H(#vg2zsB1=x!SnZI;7|HHS1_?o!MU}
zq#Y`2W%=M-y^cR3-I+ylK6BYmCii`k6LyF5Z2K6kxQo@*xuw>o{lmS4MV<?fidOL7
z-2Cxi->rn*PLgcATsbc*Yh=npZB8X#Zpbvf5!PI|)w7vta#fzmVa6r9FF8eL8*F3C
z&NkTYR$cwB>M_Hkx+S^{%8?1Nv9sLH9pN*{kH~DEX?7#6J@X%%usPFO(QP-mGugJ?
zOzf9^c7wZD>CMK&a=Q)FfAg<T(bxI$q)%@7*?Zdk?_2z*$`md3JbeGU@mcq^-?{GW
z{rZ@1=LVaEF9&6IXMBrz?$pHeKy;2~iP6#9eaxwUdd^k7HgDUlwP0!e9;V8gKL;Da
zADq+E`lh&j!(8Eq*Ta1NOc2@i{96CN{<L?s1!4bAb6b7er`9(s&vA0vw(|PyUp)(K
z6z?+Mw%a;$k9b`9@%_KQ$?RAeRw#Sn!~?xKnd~ytPp=KAJll16dBF~mxRpzP`8(e1
ze_s@tTHm1lV8cJY*Y6Gf-Iv~1p3OMB>h%8h%a$A0?2uex5qCk+xA~gT)69}qCR5(U
zZ#UfJoBejf&F9PZh&LJ?NXs|=vD5oD&$G*y-$qTC=I5?iDV7y|{QYOpdeOyC^7C&`
zn;yO8_M8jzb#@<_8S&fb)-(P1+50@dPnG(9UHGTR6J^FemWpj5b&|cHb)h~R(r^6u
zYuR{)#fVux{TXDX-Jfs1mD>&G*(OW=o3r<A&i*sgd-Dw%bsz&ppmlb05|giOu!;JU
zrS<*Z+**bAMjb~=k4^i2_dI9#ev=vzj>U7=-YG50J8M2?Uqy9aVR7!!m7ni&-V1hm
zy79W?hI@C^xc5#l-L_0(%17n9AHP{W>W%h3J16ihZ}<~~&y4qtkL;E0I<K_&(z(ym
zzISzAIkQuGdriH($A{M&-|yp%`JE^{A>i`!Tbr$O&NR(S{J?81RB_VraNMlj@(Y*A
z$`rY+H)l%Sa3`hWPHVaI?4{8!JAdyfm{|R_`1G8`kF>N_Kb+Q+rQ>Wjr_gP^x#N?*
z3#YnPuUOOdDP+sFQ&lT2l(X&;ybyPev&LHRbL{Oq-`cA(R<s*4?A?E>{I<P3V^(r>
z)`43W*rgVjO>$UZ$&$b_t31HFaSd}6*Atm5K?};VPdXhxbbmAVoZ^7@6T~0<<tTk>
zYP>Ug*|TMDKYzF?yLh3^+^n}MI#Sd396oJz)$1?^bA?%FY`m%e`FW31H!P`)XX5(1
z`|JNg?KR5|X1yysbk`-Qb^47Z#T(yRH0(HOkjW;xeZ%U~CA%{>teAiBjIM^tCZ6ur
zqY;v7-)vYvtrY$CZI1iy+c)doQVQl&X3rI6kFI(5Xt(qICo7U|H%@o0dl|Pupwf4i
zucnEZhZ%R=`su$5)y{2w@45fA_w$NN4@7RH)X#Xse>088@6blYJ#jBXW$v=9s|)n*
z{<FEL>Ey-*yKk60@SAh}&yHo&jtOyIV7Ye1%J9jbD_Z-G*`NL0sx2%qNpE86<HM|e
zeDe=+@79R=Se8%`|I7QpH81}Lwxcf&E>aA8Bf9Zhf0TOr4s%1^CkcExQZsID+~aZZ
z=!U~HcCTiinYi+0PQh}|<buC#M}+yNZI*L7WU2I~#bnE}`&&xv#Gc!kI`XZ(wYl*2
z--FuR*UPQ^TNjo^^4~JQSInTg-1+hS)7kGfOnm2``u3mm?7r1%#*Wj!eJ)xp@#|9Q
zhhO==kITMYpXo7GyL#H;*n%D0>Bsf@J@*vr=<r{8%gy;9xFY%Z^O?%mO#X1^>zqGR
zsq#xo&eiTzlzZhPd8O*LrXGK7WJ3bm_P%|#>97}n@%taos;`v@&40&z=!Nvu#f3ZU
zZx{cFbM6zj@$amipTGa%@016^f2tlYUarf%*|tOfW!1slOYcR_YW$oWxS@90Qtk}~
zmD~Szy`N}&eu-)H!7_;g;}^U0OP*Xivsv;_i~FhvJ;&b9*{%B7^iR$4=PUaS1NYfD
zmd^W8J^e|NSMIWN=aNfql+EvdotiJP^1w{1*=>iSe!Vm<H4c^+cAXn}<!0d8n+?wn
zT#wuXS|}UI^62S)8_w+A``_gYU76%{ifxaSmO|XA&0HVX#P#`wPfJSG6q~T#W#0uh
zZ%2t6=TmN6_u`6*@N9qSBev)66waSKqW1R}F0g<5^FeuRn)@|pshGIwmNS_Pqzm5P
z{`l_f)~~Jmr@o$jVB!Je#*I8|Tcv+Bl<MBL&fBT)aprVg{A#l)$0OQmdsXuC>$bn|
zyHmgG`pSDZ6!_=N^In=KS(NqHM_`Z4r--MP4-TDds$LPW%llxUYsSf7<J&%aw<pV(
z%DV2bKfL5qL+F-0u^-n|-0f_3kQ93-QEVd2zI%sA)f4@-G4d9hx#OEoJ&O&z5OvW+
z>e?A;;R!<F&%Q05YMmKuExdp;<iWa(qrXFrt$oQRobV!*cWuS`^?O+~oSh}3E`Lgi
z*6Q57(MGo3UGm(m<F+MJtQVdyIdZo5Y_*U%m(+@OR_)4Xnwxa@E)&#<-q8In+go4y
zP~Vp?&(G~SrtSPBDR;pI<<Ls8JL{)0@#gvb(c+TTDOs?fdQO4j#Kkv$u-*FW^|?yT
zY{wPeYYQwBj;2<un06!Tvi*vi=5E)QO>A{*ZZ0lVOFg~(+N{G-&8rg&lyf%t^RJr9
z7t)&VzV*;0X`jE}HtKWx8Aj?wdW*ZBi+pj@rS|8ma_6LJiTA}zW%JHd9V#~1#m!cK
zP;EBn>B-K2j1_hXONSNhTk+&em#$`s+1+BPmwCG*ceba@|5$JN<j);B&Aq}$F3j7Z
zYFeH%r`c-z(?dKWQmz)tZ22ku)gL%q&TTm8Z}>x#&Hd=Z6*a5hhp&x{Om7y|6FkKd
zx^dqJ`!)0O0}niMKlyRi<1TIs-TNX({)SrpRITlu+573e&E3vfbyxV^Or|l;>$)5`
zb496z?)`iBXEWIH@tv$R`<5%#w(p^y#U0l2|I07DKN2nFy8QKnEsHLHUA$X^_jNsA
zf-U>$`&x?*3tI#Wo9l?JP-Zuhua>zx|I|a_g2TV6mqsN%m36z9_(az2-}%fNiDzWL
z-JZ?EGNJTw!4zhZg)`e6MG_9MRBvGJyvfQlBWk(mUA7l~x2Kxl-<2zvXSw(4wzop(
zUe#UM;48AM^1b8F+_n22*K6B<v{AG^_4~8rue#$eWqv--&A%;ocfIgWk1fiKeU1IX
zCp;9Mn>`6`;Z!gNRhGt|*Pg4slLK0Wp2jqjp=jH$rIx$+e)%*QGtO&$k#Anu6vdh4
zsJwb%QPK1dUID8gzjoQGwq<VLMWzYHhV|}sov(UyciyWJp3=pkpq&x5)cBwSi-2gz
zjD^$AX?$>7a8Lqrr_6;(yq33JQs370l-qDh%riK*>6o-5<0R%u%#(yCF<)vDf8X$Y
zGAG00MRHn2>(;b)t<ip!_@v=iPh%WYO6<a!T~;e|q<9Ua5==A}ESx*9d3L$V?xGo{
z1~vItf|O+wd#*8T<yB+2#@5TapiR+{A=@FDv6pr0mItrc6Ie{d*6d*D;ggV+nr&J5
z;qICud!{AlymBP=P2S>pB_aKF3ya^qY0DBOAIqMyQ}OSc%`09`R^xF{b-%4}ZNi7&
z3-|7g+IH~Sr*OU}0c{73CY(F?(CmL;wov3$r}P=QJ$qPwZS!2_a_+&);;H|-3pD+w
zDNg^hZ>jBz->v^QZkx%R(6ChY$Qy-c#TWl}H^^$6{6EMdD;RK~DLBq!YC<Nvx3$MD
z$4m3CS1U6xtGqo{CGqpc>gJ8IJ#Q6W=0AMf&0uKVSy|G<d*D*~do|m;zrW8rJvn`v
zLGAhNe3y!t%HMm{^uIb|vtxtyi@wZn+h$w;JMvDay|&;+%jSl<=jwBXr+;?574kFe
zPP^7)*@gEOD_h<9t{jq)`dQAE%U7tvGXJ=j%tw_qe^xwdKOhyi^1Z-a@5>cYi)P-s
z;&fg%w#3#$L*~r%!!?!4*=z1}7c48um+*D-3G;t{BXOUM+Ud+qGHR!1zxt=U{hv&F
zW#M02uJ+&7vZs%1C{O3+o1eSS^1!#s4RT(Ub-(7#Ie7T)PiKuUnXh*~x6Q9Pq^lix
zDgM^7H_=|*;&*tq7EHdl)qeTXc&-!P%-c8qu1nkVK(h9W%jdIMxvRhTs(aP#?LD}c
z{m3Wb^9g6yyDQ0e)_{hPpC5Yle!1`!4~5$+_UmSB-gEV3<cG7T6hGg4`;X_pi|y04
zrJom<vmH%1uKrH*cdqQ8FBY3`TxP7_zT*A+mCZA?EV}pJuX^iKd*;gVnJoW~MO`@S
zqWV_yaPW$myDYoE=AY->=K0R^<z>IHSDC4r=g!@E9sBI$f~iX*c;nSSu|Ba6+7VpN
zdbTjtFl<`V!Z7aJ;f5|2UDMagy{#2<HM>>!CRFI-yzEN<2^KO3{@y5>_qQ(l?9zpT
zymQQNgtOf@-OsS6Z-dE`PqXfBJE&cLX}@~gw8fg!SLXiVtF5p7+_&ab(=?sXGQQn+
z9*S>C&zUpvx#GE)_w0uk7-pYgWZk-MR%P?vgDe`&x+`~>AAY|klReCx`_0amk9s)f
zM}6LTtoXY0yJvOPO@VF2{dcdk$UH4HyKN&~G;eOTpl<)4zieXp!VV{MIc`s{JD-%4
znZIOvo!^{23-wKcKld%$oK^n(?WaA5D!U)86uP}SJL~<OZBgr{eUiQLcK$sXEvw7#
z4T2V0ell^dOpHIsxMWtU#r3r0)z?ll=C0f%_if?=>Bg!RIpH(3t=`++E#7qV#J8CD
z)is(13a@hR*`9rofAUA%uPAk?_}Kp@JF=S#7jI3p|GzW#W1e-}&YzN-qt$umRBiiG
zyXwu$ia8(ut-a8=%<%G`xt$IA?QC&bjWSYKwo5;XtvD^Ndp>rBdHdf_5B@IwIKL$D
z$Bmu!I~V$!XXq?wNLZL&Hf!z1=gHG^)EMKeHglLWvGQ72{g%B|&7mLg!a=g`?$=3C
z8<sVj`r0$fp1Ab>XPU?bEzdLNdkSR!%RG7b>B-wvOAf2sLiaAkol}}~*J``2@0(3;
zGHQ5d&-vbEE#z<VJ3PapLi+nv+c|sZ-nLfSu7CT?wOR5TS1w-We}32R%l|&g&oiFv
zJUxHX<2n|h2J?&(_FRG2mrm_UTk!Z|_F{1k^|w53mM1?(Y2Rf%F#FbzqQz^)H#!_y
z8*TWs;KSe0De2ED9h-%}m!F&bQ03j)9g#(wdmd&!e9LRc{p8TSoVAe|R@L5rm)ey2
z`!oETckrRHMcK9+k00E&tg&v-E|24@JG=YIjvl-C)4toE2YuLH#9PfX|NO&4mNr}S
z#gEJld=a(#{I`p(+A^XGZRYjVs&SlJYjG>{)-$g7338G)`t8r3mh4IETYmoN3gPps
z4|JsD?U4OBXZ{Jl$(#{#6)ArGZ?@lmYODKxQnTJN<43I*8z1D<?vDEFShZ~1M;-&V
z+;k_l3G?n7s+wD<?{KV$|JGj4ce7~n&lg^kO)vPT9keUjVR`(pPCS#wf=wSo%`8I1
zWQ5kMG=B4st_uIYsP5ZNE}e#F$I`AJ*?!WbaA~1O_0o^lrA^aw;~r<PxvN?Gpmp`i
zlU~*nE=S(`_i^v!cbB{7*}jjtu$R+<?bi0iN#D=s+86kly(xcSzojgm?_tow4?kB;
zdDymH`GfAuW8CvX@7wGusZX2z$oeQNZ*K6Pzo$<5+o(IapOnA9?R(4it$X7iPii<-
z7hL>ev;D7Gr?#EDA@|{1aQZ|yvCNdta~Th^d6m0OHi;;E8#dOceVTbfz2QoZfBH;|
z^QtPP*V5b6OX{@#9hlWwrgp%RQB`oAONHF_<Gpp$FNZ6|U)VYI`pU3rkJf0{$EzPY
z>elFZMe2n8cKtP-zZ`>OG89cM|M|?l!^zYf*QLB@k(BOnD^<by*6ZS@ueMk@-CRE>
z_4(J0jWRa^UIm|@DOx!}O(>=ETnyWDeZ3`OhnOl4NTwQJIk4`;>qBocr+litGC?6C
zqh?F&$!c!#Q^`8b3whd%EKWTLVq38_LFC1oBQtWWjOC{5J;-5+@pnHux082W`+b91
zL62=#FqgBw%UW@S?RC)9f_478-d?{NmuegGVmDJ9>mIMIVJk`>nG~{KVe_+8ek<Mh
z?81dB?s7YSS~tJ=9q44m9{4_as;uQ3X6_i1{i|HwXHTBs@o4An=Q1sx)7|rGs*Gpx
zZq5q)r+Ij7yTd}}Rz4A{8zxM<_+QvX%xhXExT1aD^xN|dzx=HZ)K$GUN%;_8#av(J
zZI=^ldXE_=sPaC$Z1qt0WTfW<SB7H=mlZze?%!oo%e}<hDE+d;le{mRPRufsIGmqQ
zHQW5bLAhH?rX{fQK0BB2C}$J1@1YajT<7*g_*Lwg+|Cx9v8A<J<-v)>iyaJdId>(m
zJgiA`YT6!J+`V3RTH<;42M2#mXSNc1b9X!MH@Akx3GZWaVidz>tx<f=^trR_hQ9v)
zf-ABc*xF+@Wc%&D_W2dBxO94nPQz@q>?enF(hVQgvWOn`v0QUy`)bkNH)0}cr;ny|
zX0D35{9Q_RI-kg@-@6a*J8qwMBvkwCl?T6jHJUf`_<d*JkzE$JzNG%!#aX9aO-tiX
zdfRkHpXA$l^VI5Jiwk4}+G-nA_?M<h7JYwL?)UBAC84d$)+T&z>n*-vk^e;?f`3c)
z+xg2rzFPQX(`56a?0YXC^~SAGaj-q$W2rNlJu-Rz2hZ6Bx~mSSSo&Tw<7a=hjW2$~
z{s%Mv{L0F`-S&PmXuRe5iSysTYW)0O+w#)-N9NDxp@u4-zJJfP`q1-S+^3;lS+I}A
zROpIFK)rF}nKLzMb(<eOK2wke+S$k~pWbF{95eI&sXymdz5o6kbh#flYs`u!`RjF;
z)wPY|h2s}<ab1a9@OskC`r8TlE=m^eR_}i07&y8&HGXiBu(Ds=Q*9$?pp!U<dG+;=
zpn2L&4m*t54@5-#F?29;Dsm`tD01In#F$`_U@_(4WX=G7(Ve^(l3gcTto!H7z#8%-
zCXj#j#&xFa8v+;h1U{_zvghU39e>Rk7i?ql;GH#d=YkE_ziUrZzpoYU-1{!n`b_No
zi8+g-zN>~@zv}KN9kOSG&mP}@(cCTzrMd1#@bk8I>)d(gY5qLHmgQBV{`KdU8q<p#
z^&Jnde`a(^lIhMIYi+)`BpwyFR<_du8qWIvuW(K?`r>wbHbdzqe(zZE#?%{p4@|{0
z&d1F=BJdzKn^!n-gCN`Pg0$>=jArKz?R44>rp(bT&R?u+&3lUZaYA$I58Vd4T!*R6
zOWc&2pWa~Z_?}^KQ)NRqi(6{{lzU1tuh_+&tr568TSd)!vbU<2q!E)=+eO1D+2)&u
zH)NY18b--Azcjod*Zk2iYJTxe?&}LK=*G`jTe>9eT!|RFaK<ZtZxdf-bAfr&Mcz9J
z_+C3==Qu-O|4paP;WM?-aT$K@zpfgjK3ANwJ$|xEgP4Q#_He(vg^BO`&L7#ja(2V9
z_wEPE*Cynro^F5Rx;B5me($3rXCJDyo(pcx`!DiKm&G%>Yff22jJo?mU#{M!LtVQr
zZwOa&V7=J&FY2B31ou^67P9UQ|5@@Te%Z~^8L{ivw3p0@IrDD)h410%aWW-V|F)Lz
zocrarPEE1H<L0?#8!yj}+9dy6E%fc>na!WwqCaTnetRzx;^ee2@j&dG?@3MbRu^x%
zx#28d@t@xOJeFHRXV!rhLA<N{{A=CQ3oO&OAN#XbVCm}#qRXo*ix{^&5&CeAE#G^W
zx=`kF+YI(ktZR$6AH5o+H@(*Pu~}o^hP{rjwoRR1c|+}E^@<JY1$!PO3az_T>n(p<
zu5SK%@jEId<q<jA76112J{14LbIrv_=xW@m-wpjNJ9tkOi-(=e7y8NYsZTau`C^}o
zHFuqjzw8{-710Z19-npDCw;Mc-=1fYjWb<mMJ}4T)FMBAPv;#r=c%`Ur%!v4pEmIs
z%RQS(2Af@;=RDqiE9F9lS@+bX7Spr$GJ92Cyml%`XUnvyM<ZfiKb^fMve`baIPl~4
zfL|XEp9<1ZuvfOfcci+1S@Gq!|D}KIwg?TpvF4PqRgL8WL*0_Ttpf92__8k5$%tYV
z4UAwrZ8XQP_WRjIHq*k-{h9mmNiEODMHeJ?Nu1gc+qk>^bbuUlNV2ZPhNvXl!zZQ9
zo(QPwRv*-w&b%tz>Ci`Eck{Aucjs)gVqR39p#J;Uxwn@RPKs5>)mK_BSYWp>FDJ-W
zIr3iC;p(FN8<A~qYSLA>S@-PUtb5hr+E1x(l2_yQccgb{-8z3N*egBcG^@@|<LTd@
z+%w2H@hRZ=cGXA9(=<3&F3dNXws0fQjz23Lk5&tAkI{W76XIMcb!x-=*Slp`ER<t<
z%2_lySyJJ`38o#39oJ^+?swHTINB<%;mn~u_4<!<9bfBMR_s-9Ouz5Dx|nUH%#3|3
zFPQ({Pn%>dII$s&dqP6QdE1iwzihWxB&PDFU1M7*H0@eKrn&gu$;k&<0_1zOcvkXm
z3){G~fpKx><TZ&mAAehCekZ``nggGSWrIqXVfv5#uB&gKV95V_`Nxrj!0-7Y9`{=&
zWUwawI;Xeq_xF=51yd8=&-r#vYbO8Py%VMfbhh4O^_g}!#nNZmWl(8rpXQaJCcNSA
zg37~8mbw!TGg<0PILx#&LLow`eQBXO=Y|V@ms}UV-tVk8<)NwF;hS!U6%Madl@M$f
zER+yje?_RLwR`Q0MTRmjRNiT>I<t6s3(M~Z8~FWxU*K4|G5skkx1X^9Yq7iW`-k@*
z&$gb@Ui(4q&Z)W8!P&p&ZcLC&5H8~@caPs-WBl^poAu|KSLC1J-2L#N)8T?Y6_$R5
zbz5qh|FVAPx+s1xwelQCUh(a9U##P!<hB2ISg+f3Yqs={k0<{A4E!W~`sWMz?E3nD
z6Tki5r8Z+``1W@TU;55ewF)VmFL3fvug*P*A65Hze*5dCWM95pwv<=le(jIf62{(!
z!arF)%{-B=G>4HjRq~s%lK;!6AHTcAp81k?j+MK7@*ypYY0>emXE|T!o_}#h=TfEb
zV`-t(&To4A!rj@-X5_PFEnK19y=&8oZdcuxEbDY$oq9CacaHje)+gBu<BGa3$Z0Sp
z{c`$!;??v8DVE<>yj0WJ>G*!lV#lxcX4b#vo#4&Q>0c1Hw8tmFN@Vu)#eGxvGoNXl
z9QM2RJ2(H^Pd(Z1*Dp#G3yIatcd2NL`L67|@T}mv69=WZ=6>#~Hu-vLn_O4%<u{v^
zSWRZ^Ui<BXcCo|dt(s?lyj^DUWu2{9^RIi6k8M>8`pdVpY+=v&9xlszx4nG-zK1$r
zPI$fzZqsMa|1a{(tk<`=oLSLibAWxH(|p0#y$%ay*X~%f`0d6Ut1s%@k(<7EfAY_N
zZyfd=*4c8Wu~>fU+O3=1HfDt%w3ynPzmP|?=3YU<@=0RfUwqAb@9dnjVm{lh8_65L
zuZ~!nv7%kfhhYuxoWPG3zRGta->tHmEF&)WD)Cv_`_BTWl(oHeOZO~|<dTYCP|$cx
zTx0h0rL4y@?k7(@c#r)pEB9Up-Ry^p*NOfrn^m!H`@4&Bx6ga~=vL-EvTNS;TX4%2
z?yFt@E@UciIytBBYgosz`pOS{sV^=>q>Hy!Cw=he;p-~h7FHp{X8zcMqdYd?)P&md
z*{XM1X3p(gJ5`UPhPSysvH1+U_~9dK6CMkDSQOb@+Au%#zRkUflVw)_nZ920fur&c
zCAat^4|?WQs?6D3msq47Gp%@uj=p;(XO!F$3v-qY%zM*q5=G|9dc;Uy@Hx9PZT1%a
zf`A@AFUQ@B!jlTT&q*JNFs!v#vDJJ3(jqv2`vS|^$=xOQi@w%oznsgugxhvzvs6LZ
zMV`exaS??pgx&kK7q*@|d}BqXmHHP`nSZU~Wq<B$czEnu)W)p|kDktLS{A65RrTau
zgYm<O4|V$gUj9)a+z_?E=8lre2RF74IWqUo+?;Yeb>nnq6XxYdeR)n@ig(KPeP*oE
z;QXk-=^XRr>;E60INCXNZkyev4d#C<9I{?qHa$OWzPMF9({!Vh8515@RV*$3-GBSv
z92fWH3tuyoAC$fwblN(#FSyS+`~2Z0QPQ)S#jn}2uL}JWCdPLq-ZSOt{u+xz7CYJk
z-@IHgN4e*{iQ(SkHs<@Lt50NBxH$QYs$za@mK3Lhy6p6f_uERgADonycCWo?9qZm;
zt+)1*qE>y4|NgvnVP1t<y4yqHZ72U-b5?Ht%X+@YX<yiES++k<x>*t`O*UToD}Vm(
zP4&C;+O%qxPW)azH#zD4Qy*i-w-?WR|63;Y>C_GBEq|)2nL(?yH{VeHP&lnx`h@=U
ziCPxlz4x4)eC4=Di?ZWcmaW%Q-p?1l;uEm>;bRw5;VUy`?P4t1LxLuoJv;I|X1~tP
zZ#KdYy;`&x*RfQnaqP2KKQxsgJpE~kc5G*X>WZyDOx+)JKI!?JFA;I>sL75$FKVv{
zxb@B}{M+kbE4E@K!zIby`^kk3iJXrwobhImV3U|;aIRPKJ=38WBm3Kz-#17F<O&FF
zh)TMvzV_Zvt1Vk{Clsdso4vna%JI3JMQty8c1=7iEws3D9;0d8n~Fq-(p6iX)`}Xe
zQjVFau-STg>79>;S8koK7B$ey+Qk&4x`fR`S8;AIPkHs4jLfHvbKP`#@A}WSbnxc=
zAZ4+2=~c_`=lApnT(&>|H*UV*ikHTd_Fj3ia;Zc1UG@q01TD8*bI9g(=shR1;M87i
z!SZu^SVCUc&3pbXtF_B|!MFO~U%DAoBvcl=8qH8?PBn;VE|$LNYtWTa`*3dI&8;>D
zQf#Xe4W!fSZ~e8-mI^zP;mYK8yNKzzbo#yi1l2_MoPf0ks%E|>->%rM42uwJ&a|u+
zIrY6fXsLwMiU^%km%6GW5BJUa9KB$^`MH=Qwi6^?&8wXAOPV$AuK8MalT$5QZbUUo
zhaFGMIM=pKywQ*O@goDh=W%PySwFsa;47JLxnXORnO%g!a@L{<g?84WjSf>e19oq*
znzsJ>F2P9`7k-RUYF=2Vtnz<(Sv>o%b~9tfE6?9t=I(6fypTEP!>QS6<r344SwAWT
z>^3n`zWTR!!8`XQ?hb3$WJXCzh_@Gh^g6!flG$RR%$Oq!Is_e6&2CwITsP(Cr?kY7
zn8|ulEAu7z+V9>t-1sp}dZFa4F2`Fk5%LQKC7;%A>HJfC=`rj6w{!2S+uBR&SClp1
zW6n3WkJPvDV&mO!YxGp@!~4m<nyP~@mP)uO$#j0WGv&`;52ice4J<K#^=B+dQxs~S
z%W(1RmR{i#%a#A%VX_tene#&`!+lk;+O5r6CEuHzwku0kr)i$EG?18eB%4DySFHN|
z^*@JBHz-*DDyo_58Wp)@s%q4QpG=>QT5|_4&VKZ{wdoM2X6FYU9fR(lw@MDTEopkC
zA<F6H%#yXx;cKeFK^K+>k_s~&PF*Y4tNMG0&BM)uduwXL_JHDJd>IF_LJWF#`l@-{
zd44Xs&*I$;hf@zMSH3;)7PPSAX4ts{kLp-f_<8V!h8>9b(5j^vdSFFE;KGewx}hnt
z3zv3r#oqjW?(eMi9~av2epsgL=$tNDdHuFzVNHGhnwQ6Jl|EtU-FlAYz}<%3j9Cp^
z&sjX<Q((|%&|LCy;dI?CcNnfLh+@tHubiH6TGAmVjb(}7F%=b1cf#`ZobAbh-X^vN
z=4{o)X~*-A9{hD9L6zx}f{KLH;-!t18y&ATG#W<CcsptDRnf4Rz*A|>rwtSj&XG9F
znR`M-C4!sz_w7YSMdvR(f0drCdUnI^MY@c~ejT53kt2mEEWq)GKv|Zi<U5_~_YGuN
z4F2r2WA;@z%$(R5vT+fU?%UmU?<-BhWOm$l{K(f;uB;O9GxFSO{W;fc?#29h{^~x9
z!OxV^>DyWqyJk;Oj6b?K%JDT{l>O<+C12y2l2~n)yE6$D1nddV;yIV=`OETnWE|50
z=DXG}@2$S}?ssUw{_FKwR_dEG6&OW%{;#dQ!{NZRx*BZ9{r!Dh3XNCKKU^=(7*%et
zlU-Zy#)*IzL7xm#JRYB3wlVV?w;S7U?s=un@6Uf`J%4ri{kp?b<NH4ynjNe^Wz9q1
z_1|C5idUa7(SfZ2yd~_!ne=D1YA^TOTL(OEwcPyZHE7|FvGIJ!x&(Xe@AHkh6}Y1|
z<lNSF1Z`S*l4JU38>`w1*4{NPrO!_8+pR0~!0D>Y|DV4uwSkraF=Tit+zDL9H91P?
z!p-yo`zGa3wPKH&<J0#g9xyhUv3c<mw~!i^6=Ek2S}85ipUn~MJ<s6Wy0+IBm?l{Z
zGqf?cIVazn$q;EBHR15p^{uVT4jx`ohB)YFRZGainOqg3UoSJftk`SL^me)8^J_C0
zBF(ROI-l$HU99-p;WEb)8J!Y^*A5GrE!nwcPgn)Ka5y)&_1%eGi!aDpmWxSE$$AjN
zygVv>q2p3EEz^RP6CHSYvK9m|w@Z0mVY6gzW$TyfymDX(o0jQ^<-TH45V>0^3lA=0
zdu0-^Tvtr0Co{$8!jg&Vy)K+(CKnbr%kICK@@0qkIu3I+4wnYGlN~8``X;gqWESsL
z_}utbCg*vAHT%t4xyQW?PtAh*F7Rw(J8a2uZf+~vVM~d37R}~ooF>0LP{5~DEHtzE
zgpA8G1tax|mK|ptQ=me7E(SZCQ!!GX=v&dRdR}gRdi6clmYTP6{@IPJvN{!FHScn{
zd-cRm3Ft6uTy*T@b1N};v~P`0U;0MLSKQykZN6`=5$>G5W}kl=+jZ`~PfhjdlCP%}
zOb)Zt|HzfM^H0^b(kG(l&1W#2SNq3*%43W2#Xc4bwS`Yg1(h$JV+z?}k>p=h|J9_i
z=6&<;m}4x}e$!$)DwQ(TK1u$tf-Zq?NLTw-{72}-sWaiA)n!~$bn<sZ$i}BX(>9Ku
z`Lj^y{z{$~3oQ<b{^ER-$IUlK+sfr!#hcjB_laGz*rWS)R~DbxG{={>{N=`y8WVwe
z9L_V3uRZecySQ8B8Y9L8iIA)lH`#<0vQMuu4~}8AxVlS{@rl*WwU6cW+@r1a)Xr_S
zKb!FEK=G69+uQf)*8Y3H=1uE#_V|D2!nXew<g(gee&M6a`N|b<ANjG)k@flB9>FjA
zH6ZZ9E0356wq1t<9CEqd%YIz9VojIPl}4FY_EA-q3xu?PIVT@AeUNu#?%~^y8TYds
zxW%xZF)leetKlMpGFQ=Koiv7^1)l6CV!WW$C<!c9!4urr-~Twd`Tp?-h3mdgayWf>
z*75p%b6lzqJiFT{y6vPDC$CPK-iLF4+5L^%etvh%Ykz1IvF>iBUWj4xme>t}Y|5z{
zg6f|?dsAJ*uw(nr1F2^`nWiyYMi?}6-o9rr|NriArg<ypM)Na2w9&2$D|vrA$)I8R
z>w9}2{}=n^BqYqi>SLB4Vcop*_OAN#b{j7k)UmxzH>lrs%08+}uwmntlY5lsJQw?4
zttR{-LuR_ep*W2h?$K-5g(A-<Z|BRdoWr^2+@D!&I?FE?<w^HWpJmJ#G3#5NwMoTR
zOW}rw#6L3I&i778_;ZINbFa*~W!#mb66Y=VA8E*Xv*=hJXDHvjI?rnwmFphlm>r7t
zdha-4(boyHlRgK0h?L$R_38TWeII?l-POrZn4@LqysmtslnGn$n{!>=O)USvv~y>^
z4}Dz9xFz+yvdNLZ3M}`UtJ00?*L?KTNsYGR{+~ZtYZlv^jhAKmKgs8@X;=QJwLjLh
zn=P)~_40>TtHLVG-t%VPcwbz`TKj9Q+kv`j(Ot_K-7CK-uh=ouN_Rrk-0k82ubaGc
zJQNnlZ}7(N@Vza|^7HN++TTh2YjXMXpCffV9py3m<Bxi9U%Q|9XMbwL>agm`yZ!tX
zZY)16y88LqiSHYnII5rjnsnIphS-nW>*Vz-zD>3XHwf7N^TYXVH9Jj>${FTeF0K50
z%yjDQRrUAto$KO1p9^Q-{_@=Nx$m#vj=G=yFD5E#-<)~BpMA4-x&Qt4y!wN)4z|gk
zFPh)^l{vNKPvuk2(}oSIruJ(8=5YO%j_m0Sig?F7oAcWn&0qWcwmh$pUpJ%Xym7i=
z+O(wLYqM9LnA>!J_PzVI?6UX$SBSi=ShRw7`lWZ=C-w(^V)?-;dgJ%ZsYl~9UfrB9
z?fIws&sR2uUQ}X#_hOBzy}ziA$%Cm!qn))rO-o|@dGq4`zX{VG_4MaU+&Eu2Q+i^<
z_S404R|Rd|u)hD?om;i@zV0h7eZ0RW=39C;`|(qir=R{_7~fuH?5{k1_4x*?{t0_d
z-MyY!)BBKTo&Gh&UR`U^s85y4-##;#C?U0(NjGxUo0#u4>fB22O%2<pah<D^&f|@<
za4_<|cc@^V;6I%up$9TJCST`io%i$0xwMzn{4;sg92|LBWm(%BPOn~_@2Oc`Y4>){
zDT~K{%QI!Hs?N@<IQGw+@l?S*O;7I`XBoBfnxAp79a_lXezRs{<#(IwaTnROPJNKR
zvX`0b>ct0>9M4sj$936m<~iTKI-nt$CDid!`kwib`<q3l+DuFLIq!Hp`UMx)3WZl1
z7lJ}o7~R*k@mIbpzi3+6$+SBQmdf)do+yocy~pi$Ran@gnVBBXzd2eR7L8j!cl!Bf
zN7wRlO6^$jck#7qr}cqrd$VM3i_Gc2;3M_>aGmRux|VO3FPJl_6r8huT3f1LxIuUA
z_wU8x_kXK?dC9>2uK48i6HKWb5BRQBHN5RFE`I-r`Q2YWj&*zYPB<sI=ylSfW!9%Q
z+}~IG`*J{ApV5K4Mjy97kd6%BJp1~E%{;%~&e(ln!Ja8OlMX*NUQyl3oM++j{Ag?W
zdgX${@wf7x+Qz*5x$EgY#_5)S9*6ARf0%uX<7pe$wKExP-}$hrZhf*bh9|bGar%Rz
zn<km|`QC3-B)_jIn;mk%>$kxszjIa3RxfZ!y7J{z^s4h3AKtPrnHKOOK7RgsxgV<&
zcK`kKa_+~MD|Y-1vQax0m$ahVtKQ*v<MuMOGaKIgoo61O^If6EHGBn=kj>7$jlWeF
zOmk9v>i$By;MuvU#`^7=zs?oJ`IgM9J@oB#iQkJAJ4}x3*_xHcUi72P@BNw!ewPov
z+L1N;T-Zdr-p@ZK_Pj0FFo$p2Y|mc1ih@;lUbG$3FZ$u?w>=~E_^-6G2WR*2wRi{j
z#7=qVpP&-$&F2%_#<(tW|J$yRk5%?1dB>I?U7lYr`e4b-lXi3GX7<aipZML;Lt{m8
zfODSd+LwC{HtagT`{OqgucB=6e<#bPo=%Ok{B=|A!`DZZVe{QK9@2QHp8mczZpW#r
z4}}wBPT7=u*`8gHdEuZ|Z>?2r{(2da#p>t!E~?K_U%ua@f|vE`t$%?}e{=XY-|I2&
zt8UnRanYvZx@PSYvs%p`KRU}YQz`%P&0>k~?Mmg(b61_=n4)g6CtdQ}TFsr(PtW#j
zd3)zg!k!0p=YIaH-19)LuKx4t+u1wf*Ne-kHc#WMUsCWqV0(JO|9KwYj(uJ4aoc^z
zeDy;UIm#bb`|!;+u-ezidvoo>ki!x`zH%BozIY_!$JCSqf2(eO|335R?8rr&PoD2u
zc-iNC%6-oBIu{GyzuV&>mS3h}yIy~eL!DUJzW6giXI%DiZMY(HML)#n<jx0A|J3|Z
z-_BOk+&lGXbYxfU)PH}AmEt?g7H-vfu}$U3W8V5JK|HLweJrAhFE&1ox@zn{@tEb?
zaM$xn&qYjT{Z$CN*Yrm!VBtINS-wXr*6KXrNweFocW-B>&X<{|PrSAlp0w~nn5(zt
z$wn=U&w?91mBid%Izcv6rt<Q+Z_e-UaWyAr=xp(INMb*@ng5(w_y_O4Ccird?{iA0
zZD*bPv-@|TSc$)xovvlJ)N>tK(N`JO>2{CQUaUOT)fQ3cczovU9meb{CUaFx{TkpH
zBx>-=p(4mz_4;Anp3sGl*tUZD=L}2OJbEWcDQvC(@o^rfK66&?fm;Euc_%cP=C0f8
z<Mz13R)0x@;0hV@#XJX%bk5vnJteH+&*OCHfJs1`gQV5fTMMu8Z7p+n)xy%8IhTj&
zZRyI)o|HsmUbkZNQ(vAlaHd^Okk1tm(pAw_^qI%|ja|x0!hH6dbuSjNF{dY;xFBr1
ze%r5YM?QMo^kwjvJfW(!e0Rvve9ewH{Wo(ubG1*ZndGr)rx_$~dl5fDOPlTDW|iDS
zha`RHtX*o5Al$NkQ)a^>US+-;7VVM0-5R$uul@f-aO?Dj-t~tBF8Lgkn(H2SYWjf}
z`xoxs6?NszL20EEH*%)jyneoP=6mG_+cZx(@KrXhJACO)s{RyS;~b6VVuj<MrpVl%
z!#ef1;MqK3-9%>xccyQQJdAU4PX-3eSoZel{i|nR=`d@p&Pq+4k()XH=i^qf@5<90
zZw1Azdp7lL*Sh^%mOcqvK2vIE`pwKuW|Go6z9+A~+8)oC_V$hH9?JvAEOYM2b?v?V
zFX4mCw{wZ_&z<0!?Y*ffV4lX#@`I85y|$~Loq2nzySc2tIy-m69O+BD7nC_M=G-$_
zsW(lSi{qO`#EIOQ`!<(MJov9EJ$KIUcYeY*#kkr(ykjb^e3;w1!{7_=C7Jl^f2<mN
zSd17_R3=QE*?;Db4gX=q>)&M}nP$%X899Gvy!wQh4(FO4Z`Yf*;M}xV?_aZ+3SVh)
zI~V$Q`Mt0v@M5v!-FrUYY_sgmxv70{p6!kM=T#0(Ww?IecxQoQ?ox#uhd7yCM}s0(
zdR&;Hpg3pgLu-y#9tJZKl#(1jE;xQ{Yrn0_&#DQcA!<Hr7xJVZ{OPvfoAUc-cFar7
zT`v`;x{1BxnWAU&OR(+GO$UioV>6ESeRl=a_Iz6PFM9G*t2eythvr%;WV`!@XRKkn
z5@p%Iu%t^YK>KPJ>wGQ`<@vt0&a<|waqEU=tYpesI!$WJ6^50JLCHr=OYTg19&yg#
zS=-8`Y!dcGmHPFJm-gv?KDX0+g%N}Pi4vJ~whe4rh8$P_A3hy_K<YvSLzd4JMTV8k
zm(ECT+8p_Q+fu&JvIDO<-@n;e@!I@}bI-*Dy`FD}G7OmZ@{2uQleb^y!bj$}T-%PY
zRc20@zTn!ywiCzLjW^}qIAC^cdPCOI(3v;vukC0EoxouBY|Yo_YolI%Dp|+6h-ZTx
z+vnshmTZfYH&~h7jJnyFki&EBSVCS7=V#NuOUuKBvZ^*NON{2(mil{P{u*WvKJUjH
zO!%B1w@&!B%P8)5m3(Vh#xsX})(IIg3FiAAp6X@bxxr$7YscRP?skpu%2&?EZLd9_
z!zh&NbpHFrIo}Q@%FBE^m?=Le#b@@0?Qe<_3jfPBup}3<iJ9~CS}JxvRA`Qvwy;Z@
z`RhTuilyxbx|)4IcrD9##Q8+9*{huKgH*MW;=zcDV0Hb&;K>S3KjRJNnI9C_i*JlO
za&u`>{^141LR)@>a@8K5U1z`S&XnpyvkUh>IPu`f@z<^HCl~B}a5J&^?I8{S8(+Fa
zr|NSkD1*`>=$Ms>(9;L&jtl<(H%Ic^nf$UE5LJ*?r~TP_dty!fIiB!m+XD_vm0Sm%
zgr52E<I;SIU3waiKhDp0c~-S8=H7mG;Sv=I4*wQ$A&CTugor~H(Kc~kA8&MHw#nuY
zp2P@-f|oqB?Bxv7f8@`IDmqYb;Df^sW5!KoS3F(wu1;9sBrUo#^hsFci!P^dkt&s)
za_!$#8G6_xlzER7=G_;uQZRMMYPiC&k}-&R30tbkHD?AlhwqJZ78@~c;(K8?p_oVQ
z#G3PA`**+QJ@w+n%=G;io_Yz1D3m%eX-auqJkWKGlk-T1f}8Xj;cJHz+#PKjSkLDz
zyWYsaJSU_1Y>O}RmW&6k%!X1kFFN$H<(W)aJW=-iX0hwD9t+fHTV%WIuC=+nWA2jO
z^SPg7P0r=~ll9<w?%(%kik=E-&-m`sFjc>2)5LaOmp$QE?px)EmFT~f)SGcCy~0aQ
zp0!;1<n06Z{JzEgWLn#Na=&q@Ipb`%Cs&T_Yh8GUWkJP-UE<Re#jn__f2(Ub7xFLD
z=>4(SOMegiZvV}2z*g;HrN-4n;X7Mwc0QH0J(tP;ZSR8cwL3zaz8&msb*@a12c;}w
z8MA|l!gn}1L6nTyLyPU}mwk?SXPHze9dB1!@a@EY#c;WR0;%a!&1H6M=9$R1>hP<I
z)cXnZ<0}1+-0W`1_DRwEr+UI#?;LZP$A@W*7aI3$V*Y4RDmUGF_D<W=6Lue8;B~`#
zd2jKJ9d`xpH%A0GXlB$HRPa?l-%x%pf=$AveQ)s%>--j$=I0yA!J_wV%eEJ&elL4_
z+`l;6y6D8(W49%0`(J}cd!|lWU!K4D3u`)Cc%AJj?)l3T_V0C_R)0_ZK!LUYyw<3U
znLA6<qm3Dh{XagO6QkL0vF&KJvD?nOg(-pVVx<dKD0iE0s!;AW=U(u}w!i&ix>)}9
zo9l)B6y|qknbpj7PfdUD8Z=by6Ci9A4?gyJrhZ0B>N9O)oAV|2rZ3%?nF!wX&J@W|
z#3u0L=<Rs!Rr@_kt?ay}+`ba`I)2j9{j0wgtUs3AGVjUOeB&LHC#g;-b+1sciEPuk
zAlZ4x;8cRfVadWf9c>0h^L8G5C0+ZW;R>&R%gQiz9(JDc7VWFIjzm1sPJIxNe}*wa
z_Jhm>HU2J(Eg_8Snc|jC)7lcUaB0_^>~yA#WlV0a=Xn2q<Z;b&NvqXU{=;?m1TSbQ
z?v|Fqb$|BsuIK%u)yK1?rF@=i-DBwl-V99!(-lh_rZIhB`p^``m7%#{sl&OsQLh%P
zGD<jbahlrwzLyOw*@*_?Zq<nfLg}nZ{VYqTvHi7-t21HbOqI}Sju3vp_IZAB?0XB|
zg67}38<u50<zB%Wb}&IKXG8ZPqdDF6JD0!sH`m21^!~}3X>5g?y3Yk{T@;-*!^P|-
zOIn_J^G<^s-T5`e${d|@qVE~*%C9N5XHtpaif-9be&&8h%k%#?csKH|-1qgz@kDJI
zwTHWzo~SW!zxyp;v|CbWga6{zLp&B|4n*eGystPk-E6Yh1kMCdnb;e^$!YfRj`tZ|
zr<FZE4!<{eo%?V&Y=^K=Ppfsd_=FgRM3#5X1<&uR7dQM$kUv)&dFRaMD_yhN)_(9x
z*W4m!#55s6dykj+*^NsN7g)S$GtXK%(T{<7+ac+#F25aSI~Z>$x_`_3`M+8g&;eR<
zCT~9ahu7?C@4sjE`2G8OnZK>|BDKwslXO5WS;zDJ^&!7o&+8s#KXKxW`-9hKYEqt6
zC!aO{ZpiQ?=Ed*72~D#&v(#49o<IDzxM>#W6Q8{`QgcA3=geH7)^O%b4Jf67cC(%d
zPpi|5vrCP<02*lm?M9qhDZl^Mv}F>9`U+B+ME0{Bn)!CyoN~vk-G|>V?kqTT<KBnU
z2N@1C9M(G2%L$?<Z1z=ixW>n7QB)RkpsX>TMdZPWo7Tb$7R=i@w|en=I{~p>O0T<|
z!k@%MzUW$`%@rH?;*?YPp#vqJ%}iBxZ54a&UBB>IuJJA7{r%=<y=-TF?$mvGz3Y3}
z5-}f!JG^_pe|Ihy%i@=uzg@JF<;17(^G7?sPq63So3Fela`83yd*655O;_xweq_I(
z!C_%@E1!r}fO_MmqY6v*ge2Zxzj~&_WX`gSeOyr%rZ$D1VdiQK*ZRI)j<nwqdfMR`
zo0<8ApOqJ1sxpL4_`OqwqpsBH&@tr?|0<_QPye-aTH3qS&8|1%-ZyP8eg2&1+rb0{
z9=1n&cK&*|@o=cl|5<(Ixex4fH4{ZzzZ7ZAJY2a!-1xpeqfU$kn~RCA@fM-?dTTsq
z&0MqKNL60U&f>_*c`qv7t>1KGT|`@g<gSo_c_QZ|9Tu-Ol34wj?c?3+-kUvz8ul7V
z=rkX__2JoW<CI6?j#<A?-m>{Sqt@zF{ic*c;RDC!wmaR|jGn>>YW>N4d$;44KGRnQ
z>E~Mt`J|t3D9kIVbNK%JYRR1s?X%8(NZ`xiTk@~TAmVT$sJS>-&)tAgx9*-rSy{ld
zrz{z3-=3W}cka~h$_}j?ZEk1BZI_wZaI8oy=Jtki=KK9&ZQrNW*aXb9T(H1kJ>TnB
zQ%z$}iX7OxeP7b+zk6FB=4?5-l_CFUlkWBmU+!R@oLh_&6&VW-@0l0dS?PR|cmDVM
z1NN0k(Y^*k8x|bkJ^j*=+vKyBM*Gt{e;=-sx!7g3=EZrhUVRDuig!xCGTx*wes^E?
zd+giRTls5@?y3J?dHQ$vYgYG@3!WaoUGAH^jBWGZzi%%7wz)WO<<pv;*S_6vG=*&J
zrtI%ry7s`<?p&vz4?QO2RxQbMTlILt-PcTSCw`F#P<{0+Cv^9((|3E7OU;7PfB!fj
zKCdnQ%r*PD37a_2h1~L)d-gznx?$R)xtvdOuKm7sz$Zl8QMbY9jY1r_G<Fbcn4{Zq
z|HLZg-#`D=Of$J}#&!O^@K3KN+*)_So}7zW_h;tj_iy|HzUjP_TO7-Cv|+j8!6YX4
zCY^UTZ<Snkk>ZMB;=HiaYhS^B&L6(jWp`$p$?We++jjlkeGP%D5n>7pdBPY@228eJ
zx2N__fSI$qP4cYjdbh9#HxHHzZ@$pxB)#fpTL#bc?@yN;lJnokqS3#Zr|(qJgR(1~
z!r)yK%I5CTw~|jt2Tr_w;@jr-$uk?yDt-t&WHCGGomM#O5-tzXs8zz2JSLC1&YC#n
zD%`Q1S0mVP-C=F--OY^0PT1{ipP<jEXnKI-07r(-ilq!a3_T4-i;Nj>pK;oL##W@d
z;k%H}mQxIS=6%?vxJSS42zUPD@2u`@>UMGZHeQg^@RGm%wKSHQd4H$y^TpFZJ%yT-
zXUd=Ljw!_-dKS9s{(SX@Il}Vp;l_@+=RpS!OIdz?H$T^Ni!q~|+l~mKw{w1|+*21m
z5f17OtKKrM`lD5>dTT02cvF<@j%>NBpp%bQuqvd7{f=v&xS~(?g!|oiOaAZo`?U`7
zK5Qv7yST4|r6jD}zDqwNHl3qRS?Kk`_uMO)FFo1nq5D_KcxA_&J@@>b9=y6Zo%vL9
z4)^cQZt2El*%`dNYZLodtiQXpkwrN1%HfA!_ndqA^w~10YD<d(>jLXn2P66U)|g%`
zkK42(c4P9k1C3Y3LuST3-N?UQckQ30`wYZ3|EV^0Sf%0YDyelUAhNA%Tgc667rL}>
z2f6;v%(%y!l`EQ++ZTVf^+vKqUXn>~vSl7i=$yIFB>IkPXL$35TwJj9*fPNbvN@M#
zeB()P`Y&Gnf7iO*o!PmI|Nq|jq+Rvf^ZF!hx2Ce{B|)3B-rYNs|8&)<7e5S(Cb#Gq
zcre6ChuoC9Ak~r=klno1EW$zAJ67jyoQnC(EUj<RcV?b(z8x_4(XOK(SA2Ts`YAnp
z;|h&EH>7pc*R>t^ZgQSeUR%rkebal<Ie)rd>|8xvF|U2c+(%_nl?ScAPAmRZ?$xEK
z`EA+V=jnxZM<2+n&~aNP5t)=yuQ+A*`xSS)Rr#h%L?)zkPE>UZYd71<f7#CWRfu4)
zA#+*EbnzRSQrr37{H|Q^*Z+NzLW6XT%>K`{wOn}*&Q_^}mAIP+NE%*R_U5u}Y(d(D
z=}Zqd{kQU$((s?6ny%6)GmYz0wC^U@^~IcAc@6JtOYZ#KJCQ+0GQwy>O3KMZEk3h5
zfqE(o(tYNq8{bT3Y3lZSIHfU6qonadt5Z2=ti~6+mZx_Zq<;&D-4x<G=dgSOe~;yn
z`{D5u6CVdMcsFWsr>y=Eda-+sbU<or&ell`tJX|dxJXd@Y82z$AN%xAUVhNJNZI1I
zgpNvM!&0|tQW04RD;>Rkv%lCEww+zcqHDV0*21NC9glEtx9-sS7Tq&B?MCDS?cWN#
zrVrk^oO}K_+CKjI-Ni12Hd<0y{5LPWi{0cExN*}qpTdsMj^`ehv0Kcal+kfpk?rwq
z)oPAHi=$U`R_Lr-CNVQ9g)d~@1Ap6tF+U>L`_yzDdGFn$W4P(S2_a<&!{d!h#V!6Y
z<CHzzxjPR${JbDX!+PF-kvl6i%%@m$UU1>R@`d51oPflFvw>`%bY4{P=1ANykmwC`
zXEWo;u@HG+lE^O2XC`yvo(Dq)`z+?Q0WX*qa$n%Oz;%BzlOy*9uKOH(0bDmU5*9nX
z@LH&SV+Dk<`o@j!^y9o?yTtGRv*-MAZA!JW%!i+Qv!BoUJG0j9zV#<V&RHopE=rWA
zZ)MwRcH**v|Kcd=E4LP0GgF8@{OI+8qKsx6UZavNU%s75e;e98@3!~+ul~&^s`F0?
z6z_h&#x7*ec8SR@l^y36FIxFxm5iQZz5XgCw)PoaiGMt&UUX}A?%^>oI&wlpIk?Bi
zz=+k3RqnNtWiPYaq4VjbhN6A?^>S<0WwdDWEtiN%Dp2>6|8xEA+`S*aU3vX8D)U!d
zJYU^>Yuz0x7bm%F>?`IwIMKs9NT*=mk+(wIH*Pg=|KfhZeb4(h%QxqzUoQ{b@yYuB
zeZ_>UueVRRDf;f;s-T&B1Opd-{l5FmSE~n|-$VCY>u0=sVIF&~^z7!j73Z70GG5qf
z?VP!ER$JFRHEr9VxFc8Y=1*@`-}-yWi>7mF-fRoc7CPU$F<Cb~Uv#y~fz^BNe`scq
z>=D})%dzSCxof-3J}%%}SbpGd<lhDxKFK*o7H7U(z4Y^0(TT@X4k|W3K6~FW<~VQJ
zQO`F0z`WbG)<+ea^%X=uL?!HGeWhozkLh^ik+?M+N$fwW^&A`bar-O|Slf4JK2vDe
zhVX-NPq{-D?&JEVwZ&ZCBxb2_?xy)%-(IO4`hGNg+NP81q~<y>ujgHrld$!o>drL5
zMW3ywF4+5xy}oL}!c)h@S1Z-8?B0L(r|bPa)_Xun{H#*;xgy@snOb^R>YjJV`((Tf
z;cON0>$NEJuXtp3?QiY(F7>>p2N_b`pFOO8s&laawS{urTbsB?y>0GUXN(oH-W*=K
zH{Wlrt>(=5y5s=UPZzD5Ub1+JWmbFmUu1tC)m37=wCQEd4nN_0Pg!OOS^LS>EU7$e
z>YZL~_p3z3`pDa52Ie8JUgUncqWSmjiPx(SC|+9B`GGU_+{fITvmgJ_c=aXp+{;<Z
zL`;`1XO-5HSou!yd}^JOQ|beK=gVbF&%BHK{N?E82HEQ-CxU`?q-I6cIGTI!ioA5N
ziR+uD?@uPZPAS7+chjhb`<vgn-%zc66LCcN?<WsS!)rU1*AxVv4CLH)*|4X7;enM`
z0^TVYtdrfko>Atjy}(YME#a-MViLENV~;#!xRK~Nbpb>31-FH~O@1m@!)N;Pi0e7F
zwu<%0E?`((y`fP$!#P-??#sXS1BrV@=LhTUOIB&%O?+Bnr4zG0g-1}Y@!`s^*?Hf;
zyG^^Wi0|(1ANQIcy)NWVb?_EH5uo7QBgp5-%$T!g!cwMbTyK^;CZ`5v%se*l?Y>l=
zncTM@#&WHQ`EPaq;Dm)iZLJ^gSZ7vs{$!f^yPD-`{KDOBDZP1K>=U2Jyl8x<aP9h2
zz56WvjfZu$cs7=AVAnnB@iatg)s+Pe$HX(vvpjJ>++6E%Kk}NlkFIH%E4TdrjF-oz
z<&~)SEq9ui81UNqbd}a#<NMPYrro}A=`mv?%iC9Tz8ka)_Wk@EP`GMGNTtp8oFBH&
zJei96)BH;JKeY`Cth=h&H1Xm`o0C5+Uqtyzco&^of9jD)vv7~kapx4B%@5LNR-D=N
zEcnPnk>+3@*X^_J-VzB8_EDT`xbNxbbi=JjPOPf<5^!7Ju!&bdTvF2LlB%27t(T1I
z7h~s12V7uYYxCrpAh)2#ak)u}mF)?Nk<SF(3unC5ocw@sJzKOf^QQx4ZfyJFx5}7k
zI~4T(PX71A<@BOvhra&Cy&Pfop9R?*50>v)bUrb2(WPG@yHeAwOTK*SGb?PrApE(#
zkL}T={}pw=%PQCp9^5{ecSHDzAVz6cEj<T4lio`Q*nT}`dU9x*Uc%yr-mb9q70rwD
zt;`R&wK9EsdGh;aF`dwjg6tpXc+NQaFS|y&wn`+WEOweyMec<N=GAK_G!#lDtatS0
zjp{0X)!)x9?VIz4pZj-9@{;@4T5jI6dvqdZN`m;ox?AhN+bmC3p7iNOM(16}C+lVe
zPpRf!wOLzCx-!el?A#IOUEgjhmCCc!{#vAZC;WB#<GJh++@4k5lM8>}Ha;pPWnA;g
z*?QmUsqYMyELSXb$?|(u=OOQ`B@yIUbM0B1<&&_@O~TAGHP=ljH#~ZRMfrwk8dpuw
z|5G8|Q}%SW+U0(&I>*#~{YlN%#@h>byjIN<57{w6^-<UQ+wxk^mu&2MC8WGeWfRw#
z*I!%I?`L`LVg0!4SWoss%lYh&cssv$e1CEMUD>AdYefvhZp8L4-xA;QGvoU<_Vt(f
zjI23&bxIxhrzoCtmgIS5{^9S>IZEDbnFY;N<!2UtDu1VV`C_^((}t9UlLvzKe2Ch1
zsqtyRcfKjs9DMs^YpjF2tT+{%_4j}DxjFmt%aGO1Z1XF!f~_r5D=sWz)zVAYdckx`
zlXS*Qs|DA(QdTdL+{U4AdE~A4pF1;?^4@Jx$#9%K#dDg<g?kp~zhsu0hRJi>z4E5*
z{IVC8{yQfyyk{sYGn-c#WF4{U!PY}{S`oPknpH;^^ZxAU5VB^;X>Th#5Lb6GdY(~a
zR>I20rEK@FXRM9o72lisE#0gC=G>${r855+_WjX!YM(~#eKqUrxySPb=M)L$8QAWf
z5w|UGE=SSnc~5uR^PheCG(x1hV2;(gMT>fqADP!r^KcgGZ9MfzES<~G(CFX^kq0gn
zpXZsJUB~X}9sFRkvwFnyTh9;OljAwuxlVMMX5RB7mlqyVD&NZNRBV>*+4bY!shxqI
z8&z08Em`#6>D-m4tC~um$mw=&Ptc8hom+Ux;(E~LT}wLZ-cDKg^~ITUyf6MbW?jGR
z!BA?S@S-u(j+-TxJ58|rXz7X1yWbvdR(O!Mp6%PWpD&)5-$>DkD`@=m!qwfzwdA1W
zMX6f}@7J8NkUDl%o@I7?+<KmXjfa&U9hq|Zq~pBC>DOdg`0iSlyiX84zRUf`jl!FL
z-x6M)E2}71UHix2v={3e(M=%-f*8G7bwoFXB=E87h-Rz|@)o@ys=JqevYmEY&Py&c
zts5&ccARQkov^s6myOkAAOG)~FC|ZTbxbuLPqy7@e#FA~^9<GOg!O^myir*R>l^p3
zEbb}W_V;I0*F9E!;WF*LzuDd|jCv-R{j&7=t#tLR)eCm-j=Y?=u$rgs?csA>9O@6s
z=1=XPuchUYU{mqy-PKROPlUMknM{j3ug*V>ds)>D#gA<&T{mrY<g>yxa~m&iSpMqY
zC)?M-61U&Y^%Si8P+(;+U;pC<_s>&YF4jLiKgH?r$xgRf88cRiF8)wxIkCfSmQcQ*
zO-+)@%DthB{$IM6`#m7NaNpifceT?WIp<}S>{7WGcVf@$oQ%D#AMXWuuXEzQ{CIlF
zh9ym`fyv<y*Vr<!M9TO4c^osVhkxz_Y4%WkgU=3}*^QMv^76`!OL`9;SlJR*r^R`z
zJzMtQ1xEvg!i(D`6<R0-DeF~lx^5Bl@7{xOttabGwVk)TaQRadXQ$l^P8N4IXW?XC
zeX+ymgJkEPYkn`<%*5k#TY>2|*V}|Gv2`WCnxzG9B`ytZ_e|Hh@bk-K?Ui>YCpq8v
z-Fn=AKUdO;$0c+7$}b))zxqRW*DdFOBiUcJF6a5imD{<|n01v-h4CpB#;9G3O0~D7
z@VBhHm7dVw>Q`}_`SI<<2!Evn0hzg<HJy`1<E*UGkNuxxv?csw*D)&w>sM>8EU=w3
zVOD&F`}h91i)_=Rpm}PV)Lu7n>EAlb0^F{D`p9SKBYKo=_D^1mE$<?>lzjX)XZbI`
zv@_yG>C5M><9N+`+cm}h+qd<R^$%j7%~?EIa<7YX^K3=UeHZmkdH3x-R=g|Z(b2NC
z$7_7@%*+n^sC3#Yyt-HX)TZd+#G)6U72j8#R~IXYW&6`^ENQ^rl`uQFWks-(Z=v<k
zw<68eUZ3^kv#%BAFDg!(n|mnQch2jsgP$jfaL>^%+^uz`Ni(2e-;tj}&%Ngy?|8O`
z@6aB(&O`;%BXRES<@q8fdat%FpTrO-!q0P*J12RSbywUIfeF2xMqBp(UitZo`^)0p
z=Xcf4oXk*T-L4Vq?68=JO|r)*f$0nz=YiFZNBKmpuf7tv^>OM$(Wh=rebNra3q<$m
z?k=}9-=%Y4#k(L6hUns-Ywo^|Y4T$-31jqTeZR?k51T0O^A}qL`!-sO?EQFZ$^E^j
z*Rts5cJJeOu+=G9bWPZf1$VFB(YG<V7E<u=U8bI6^Dn8lT#KK%MQ4EvGe`_{FTHy?
zrT(Hx?Zv0l%HKE1sdm3gFXCqNWt4lq`sIYXk#qFrbnAE<`5XKDPBF<d$usrqvYs>V
zJg^)DH<WKUQrU5u(+^yRfg0|Hn_Bj&zdm=K>G-Pqt5)bZxG?EFd|0q_zU#?{#c~U8
zr%SymypzD!U~yx<YPs5ig2#E6#oGLTE8X`ME7(&0=E#Dj^|jM@Hklk`Q_{7o-k+KO
zdd8aF=Ew4HNsC0kkk-+$F}D4nX+Fi8lP#~ombto*txt8QaBssO?(z-467KKq%FUI1
zW4UI-qdm<LtAj6enLX2w2y(P`S#k8_s)O&g_O-3HICawC3)lVnAIFa6oV&i~-zLNV
znRl5UYn$w2dYsI3s!r43wK~h3iGdd)Sl5W%3UW+Nd?8(XNOQ94p5xUMO8X}|91fM4
zARv3BQYdF-(_XLWRgZtGSg*G9a9cT<<w14UEZb?%{T{IN=vVQve7$RQi0f4o4{we6
zhr0`|xov$X>GkB{U9W_#%}du+?pb$q%H7VCz=h|lHy`}GdD`5v;|I@5_*KkZ+2zDi
zras9v@%>}h^XEZ9E<Bahj@7Q!ymfkapQMud2h9(fZzR4yKhHFu>3CIi&u?QthX*&b
z3PnFP+MZ?YQ(bfQV@9KXqq|0w*<)$@jO#DAy#6rV;*Pb)yjZoe<Asm!UC4NU;kAm^
z*;XCPlOmfNdh+%^{2)}A-&pCKYT^9H!{hj?>@N=6|3^iws8p$&^R>NZ+v!Wnj*Gdi
zjFY*ZaPGYmP{iDr!tuQI=bG*W(VUXCAEFO8TX9a%NDxllaEf7yt&fQg>s_yajhDN&
zbz29U&iM2pK63A8)l-cfa!*aBiE%7)-pLVstKi1HcP<9=RS&y;QdEmvTyFV0;=(kp
z7H!tl89tdRF9g<|kQQID`+~{0hphhh9{ditShzrT(P~R;j~Tlzav5hvS$VE+>Rjg>
zbCK`%)cc)!H+<e|IJa?0%3MBn@v;nSM)cc=2|GM*Unp<3;eC~7@}4cO+<+nNupXl?
zGvlI*Z|8U!OrBd(_W47?_t-aY<DRp;&JN9VXTO)DwMOOZ@6Yf0&P_UNeSJ#zwB6a4
z;$(uW3+LS0ZGCF~)Gb?gE5F@esNFqL@q^+A!@$Uf;D+D|;g?ejpULf7yhiCmXXr|f
z8~o1ml2nScKc#0J`JlU5J@nk0;9!AIEYCOkb_X5&{9RRZ+L9aE*3Z)m_Z|7^H*=q{
zw@`BDXVpA$iAbjYb)B&SleU#Q#2(}=xG-_wr+|t4TfXy5Q9s4KO0#2!A*h{nVp5|j
zv-h=tcPb3hY-T*C3L*+h1fI7o=1Ezv!m!1ek<X<4!oyAH-$^()&szNO!_he^R=eEW
zA{mPoUSM@9Uu)O)+b%7|ORaZ(pw78Ds$W*}Ot=y8x-Qb8R`gTIiQt<B^EJz!$i7|t
z+i#mdOJB_EluU&=8=NGAIaOc&KF_<1Z=tW&{l{FnAt!|+Umy3rY$DZ?ow3He;Kh^8
zc6XlcIG85&Gwhgm&+~Utw$E0@#TPo=l1j<Wd8yVY^Lbvi?fW$cw)Y!9;<C9s|H1-U
zr}uFx&N8=j--WjPmT0+MJimQu&GL0R&+c|EY5L`q>yoE4^Fc*O?RUMMj2)lM+x8u6
zo6+mD{j)%!-H`_p6<c;l6<lYUf9PH#+w=6s#|vg^?zu6mPP6+*h~SSFw`B*WgW!hb
zvrf_Va~2&4Z@1euO%fz#@A#wt!s2F&RF!i!tt_JEQl}m%epA^QFUCCe$WgPF&ztJ)
z%6P!TP$m<V_V+vNY!3{#z53yA-q&!^LbEq(n6o_d((PmBCmWPE+ptyk7+ItgoZJw9
z+W3K6UeFWmntg#AT94_PD>bOHOG~u6@RvBxSa?e5#SQbHTRd-1^z>vf<GOL<VeWG4
zN74?(8VUT>(k|Ca;_t3h-z%u}edVNs6K_xJmTB)*Y~m8#pZrYO`3&!-`Bp!dPgr{`
z-}=SM3F+@Enk9B<JIn6>P^FYEXVc*nZW2)Kz*9MEqN<O%>4^&}hvNkH@m|-;Yv^UG
zluFH*@bIsaSJU1nHK$&P)Gt~+@%R5Dxyj<(TMr*zo}{%lm~m6vi@q(Zk6S7~nK<!{
z+>0~Xv!m8#Pv3BtHM6}xxZ0*-!Iy2-vTykcx0gzA9qByl`f+3NrRS?H<R8^^d{)dm
z&Y=CMK*z>^)(rk5H7&*mb*I@bTl}l;x?jl5Hw^9Td<=gl|CzAn)%Wk$-xLZk&+HNA
zZN7T^=0f+DDd!j&Q>BX^J!jOOck4x_leymPm<wl<UZ1=nCUyQ}pU%q!Pj+u1-5J+~
zo_pWXdv>8|@(VZ1-R)0QG~6cHiB4S*GTkbvdh+@2*A*8;J#?P$=^gON`^#4K9x+27
zH}B%es4oZ8H}QQcIcxZgUApP}`9p1vX4w}eIkGkLEM=^m#PI7Id+LVg2LE5Uo#)-K
z(ZgZdoN3+bPO5V}@K{uD8~NYptF4HQaq`g>*A~9(V60<ZBeric>$89aKGwS-FI5hx
zb+WD(GZDKz_fDK#*4hcnKea{`JblV2Eh@I^#=RTzPla>@7RcZDE1q;|J&)+E6<=?w
zmFrF94Zm10^{-xe&l26xeauA~7v+=FJH0Plh~Rx=dLvZXoA=q>gnL&T-KI(1TG-vY
zuC?ul%`rQ{^p}O-Cs>wV50%-kX&RonYh$nXx2doH*~neJXPfx%jlz^gWzWlv>z?KQ
zP|RzeAtifWES%Y=uqWo>@^*_k{1HEwDxYchFSI_oP&p$b>26xr;hP_GZsz4UO<v<u
zXm|8y%z`7;TaUaIYWL<jrhX=Jq0n}388-owY1_`_EZ7m7l-0RXb=jjiJB~7R3P+@A
zSl{~G_)02{Z%yS<HtX%ySrtL<l?!Y-75*){ZJ7!xoCMXyY|pN2*?xGz!W$mXO_Zu!
znXAn_8opmr(`pPoZk=S~f9~QMzqcEwmw$M5VMep+@@X;`SfjKY^m2CCPOROxdBIYq
zCrrAo=L_=|Y~&Kzpx*d?Imb8dCyCLbZ{9|J`@NAlPJ2tp!MLYUS*>YWHxBoGcz8;m
z@kU0%V#b|;-n<v8)0cd9WzjXwTG`mkW+t`e%7<f;Mq<fw-)o%BrX+gD9&UYmYV(R6
zJ#SAL@5<i&=}+Xv%G<T+fw?00^*TJxEt9@=Ffrtc*bjjp0zYP~&;ix8A2i>{&v|3a
zYSC(Lt;JoyRlwE1dyOum9;4nvU5*_rJ6LwKY*$c7l0TfUl=^g<V)Bo(#mveFmM^^H
zX;6^7h$p3ajo_xv^^(a2?=&Z?GJCU40~IZ|vUfc{GSyw*J;(9T#b2%r_u2NzN)<<>
z{5Vz2{HD<KrZlI({3lMI&!6IZ9KyYTTUKm>w<2Tv>!v=*p3nPUowon`Oi@C?przT@
zOeQJoe(k(>8@(??xO%tWxAC8w(f%=a^6Qzo|BNn{&Wl>Gv3;JT)BDMZTUM<<Jn07C
zd<(;fJlEMZVrR5kvNI;1U$$8^>*%3XhI~ow&4pgCPHZrC3SRr?S(d^3W9I7`-@EOT
zwDyU9#eZdapRU@Rb8|StZtSvoUG`Kx(Q*Byve%ZXmrT!InR#zP+gq;-KRZ459d77J
z`u*?o)6T}9?>NpWTYIEBJIB8goY>>1*gYw!;j`kojjXGRgQ_x5U%4UlBVpbB87B^&
z<$CA0>dBpT=Vw1E6MJ!V>+*;<ZL>DJO-eGeiAnPDG*ms>a!$PDwb5QxHo>EkhbwYj
z3c8r$0}|i(H|uGv?wH8xwut-CKBnzGC#Nl+;2?9Pgi$<qf`(?Dz^czYA+t0fq2BY^
zKINoVUIXuI4SpepTN744%a~UC?M-?}-<to?$6}bjtv?l7oAUR*(S*59;vBIJVG?J(
z7jn1gIcPg*i%8A%JI1n?>q(ZvftuTO7xV2E%#zG!zn*z8h}BzECv?LW^YcGl-YQ+3
z|Fe1egNxJ5AHMfj%~<(S)=c$co3{Pb196MDnnh$Vd-Lj)Zn&cA6fXDg+p$QQ?Gnv7
zOK;DgyFp%5^q5TZn?I4Kt8c8YF$ocSAn-uop2M62A`e6!h}_dzp>sg<faV@f#^nsl
z8K!>_V~JskVTw6)_!yg^<wSdK0rhqIY)&>#JnPF&-OzrnsnVco$NRMJlfuCxypHXk
zot(a@9%!3<?1u;QCQ~7XTWqHy>TXt^y1u%RmtC5VHDU9DIVaO{w44Jb>eYTfsq<r9
zsm6iE7&(OxJsnJ}B5WODLJYec7+JXv{0cs2tQ`Am+Drk5h1v`3|FNIgI+@{us8hIt
zIoH{k7Pf`b6Qo=78uA2oG42d}!S+J$#opHYpKpJ3E&tF_>)h5A?AprpHuQ{6GDDZ(
z$JgBzT?c+fHaJ&od{fkv!eR%4t$UAWJx^~;4*nIvV^OG|#1bgo@oWZjznDeSl4%Dd
zZ*ODiGoC*EgP6%)NWNR}^6ipy>Bo{a9)EtUsj^~6;{y?yHyh?o;ZVy{kn3^hm1w!^
z_v)U82J=lRPKI3OR0ZkQG6(CHG6icW{ab~RMHdvsd`(QCsgVmCHplC<-IW&D!m#tu
zwC%>b%oQ?Ni`J?zl(a4oJu%fO-0ipH3$`b23#AuGx8ynGiR@zB1rp#}xcge$cH>iv
zE^XhgZ&g_D#eVag$HD658LMnW(oDZZnzfiqX>k{D6malMfBTb=eO~i`rcDi7+xe68
zK?RE<D4rh(-D_&_2Eht$CMCr>PJy4M+)Qof9kpKTDwn)XT=MR)>c2LlPaozxHg!rj
z@y#zjC16%_ZfE0%AW*~>a5pp>P3U#9ULfmaJz*}Cj?FL(4VZYfQcF}poGX?gR>9<w
zIL}2<LxWu`35y*VTXm*3hDvW}5t#GA+nhm4r*N_En*yo2mqHD_TxJ_Qi+?Pg9mV;g
ztt`VKFQ=XD1ski0g1J;Yb4l}s*b}kA7p<qKYR>&6(%yCB%c9r8Mb>I>JIlJ(opSbG
z<J{m}!F+m)a|4JGTw!Ps#Pw8@E8ppWro$R;|A)RD7Oaqx^T8phO$_2eGCTQn52zkc
z-DB&w$B?7{!eP(3n-=T*DG54}ya$?;eFP;YoOMWO*w$n;VeLeQ3uXaqMr!JpJm-4E
za=djY<B)x5#_@db)CCOg$}X#J3!59P>1}zSll;=`^Yc%~7Vr7$vFSAX-iyD#KVUtx
z#O>S-#}yR<&wRdLI%p}As_=ZlV}pwJPN{A8JO38@3mt5hy&*6|K3hFqQikpOxn1*o
z>Vubf>a^W;;eI<|e{&aqwa2usnY$OiZ)&?Or~134&-mrz8`q1O+i$$zoBvkmwZe&&
zu8VA+W_5U4MYaV?uRlJow@tNG`S87UW?R2lRxJFi^JS?Q-`0H}!_v>!9}Z7=!#R6u
z0mtFK={B?bET4<8MU}5Ml@ECkpS1WY^IxtR$J!S>U%R*Z!w(IG4fkiftoQx<(C2{J
z4&k)+G6`P(LkiW$SKH3$-h0mb;)9LRk4^2)SoHCUCg(2Z^LfZ)8c_1TE_KI=<U=OM
znzS}bS;+5^W_z4%@w`O-b(z#_S8GeI4PlSp{kze8Zo*l)>cY6})?YlYcYEKqIaW8d
zed*i1QyU67)utF~erah<lB!Afl9Rlk`S-}R3GUwC48NY=vy<aNR2ZYQ!72ZFznlJw
z?D*YUUR8T9bFIA9H`C4|KPF!dtiL?(S$0kImv@Gi_L~lV4$R2n;ml>-XKHrG=KaR7
z3rWJql}mn_&B<GEb$;I`1;)EQ+dMlm&wOUm6_AMOZnRbQzO(%30~Ve)lD(6Tr9b|>
z#Cmg9i1y-3Tvmc=KSN5Ge|v0tthMvJziWD&Vb6*!*SgGCCrJcmhM#cjWc<Ffv0?Mk
zCBl8CQWd$N(ZgemVTXS#YTV2AO^Q9jSI2VE^1~D7vdV0_ax^H1qkC<E-xE!-uiu^5
zU%t?Hv)ZNplfXp&UDnB|MROj76ux};@KBNDuZzsf%PPv}sBR5E;{U5{(-XEyRtJ7E
zv<n~CoNaVw(Q4bBGT?UcxyLMTjgM;?s#!jtdAxtqi)9P@%$Cfb#FX4G?-J_cu~AO2
z?26d-;5+A7o!&0p7k<RQN^wTit<|-iD}VetaSPP!Vs)D)aZq!och8#_FRlA3=Joq+
zx!+l9aebO<zeM+WA(n0THbr#XbTjup+5N-Cab7FO0u$zHwP{Hwia9_{jHr&EADbGF
z2jzIwT%NOI#?p<e0@+?)2)ZtDPNCVc%_GLy@A1kf6OL;(JF~U(@NKP9Zer3@GJI1b
z<9L_<kl*9eObg?d#~irizSdRe?fPZAe~Vm_IIX^jdsWTjJLe~U`Fc^#vDL^T>;~7*
zb-#XTG;>T?cwouSr-ii+0g1=%SnY5+YJ9_&=|sSWbnjewla_{qCR%3}HeK~9zb&V*
zW1q2XV^Qzh3x5_%F1_u{_Qdx%*X_NG-=uD2ZwY%-@%+Xv<4@&$-cuHOx2^k_<HIx0
zi0SpNO;KgnON3&+Oe)C@`_>%Nao1nd;kA%bUG9s=$0zUzN=`kse@FZt*Y~I1ZJttG
z*p+0qYw@2*8NMf39S48-97!`gUbf8Qc-=|s^N)=BiZ2?L8gtL<^Zfebkek>Y+eE#6
zf{%U*WqK{q?A+iOlcN=3D6_BnNrw9K&dz4doyqOD&&>R>b>gS3Q9qn?E#k#AcL~Oo
z?msM`w@+l|GYi4ug`wdGKW|MuHi=6ts`Mh;R_$Yl+A5beSDxIfP<C*i2<vVumsQsS
zmoC(?7yjLRM<6Xz_t)LvwpJSx);edq6B4U$^gel+5u119OF_Itj-HWLn$11AOa2_2
z%@*|c{ZcsmP4@O(_vN)}x*xWtZ}_*PvE_tw!qda|?(S7LnssjS0*1v*7jiCU&i7(Y
zJ#k>-<ioFzRhm9uX<>Hr$02K$QWKfB*Avf|Opj8TQ@G~s1a{2^!R@u||GsXQmU#Q4
z;NLon?)@)r`b)kM-D7#NdA@W)^YWf4eoQMiIZS0e@s-h$wPb19hj|InZ#klNvLvi*
zTIw~eU{T1I-JKV{9CMYg;gB-9d?2n_(QIEKZ;r`^ix=#2Cos&)yx=2lbbdik<+{m{
zi{>uhJxR3e{K+eijN3n)vw7CNd-p|S^%mP7-*p-u8h`jAD|PzY(c_P^_iBGCn4Ob$
zOMB4|_H?eddZyh%`6eE_u9Wk3njH8!?}d%wrTqojFUtEDcn1fc^L6_se%<r^@jN5`
zog1T%7o3}5{J$oFd8g)1+1=Wf-<W3qh?#Zi_!PT2hU;(K)-n6+QI__*qhjZs^p7&j
zJC44%8CdV}sXX(MOrQ1ArxMyKQ>NHtvh`2dykrZjXoB!&`5LujJP$R?4TG(hudjFz
zdAa80g+KXcR?Iy%>z7`~&8;ab4wP-tKX!i7vFTHGE?KTx>Y`sSRg)=pLr%iua>@55
zRy+PvZJ(YWmW;8RQC~Nqf6MQ!v)?mauCXfFXV+{mXd}DkmB+dnYBtvrA~-5PTza@C
zAf4+0b92W$+0W0)o?X7_ET!AAUg1WC)+t|KnaeX~INNh)s#$kDJf!H5<M>9QibHD4
z7Y4@JuZ}!<QK@x{opHZlLA^AG^$mrIIr>qtTa%KnS)66O%|7R<rF4jP&iRCJzS8}t
z#mm>5v`@5*vz7^aZLvZz*~U**BIfB2rtGPPA1qi5crRy*Kd;^3ZngN!qKRHPzU60@
zJV?D)do?o4QBLl<X@yNXPkLWLl5(3_QBB9R*0ch>G@jxM%VVZ(FwgI)^pkw=IWuDi
zQ=GQc>ZYY$>IW{L(Z02WX=j<xfn@7@ja-vg*0^rG8O>At!1SG`WyTJP|8qREGS*z2
zCdGNcJS`zPrldpDx%rpWkGu=_?)kpiQX+moc0PlJa!P-z-=p^z&Icb=-CVstxBs?W
z75n2o>^$k`RBWyvQ`@GkQ#HMPO%;FTxjPj{lXs}kDw`=aNl8sPZ~wZvBF)J?6F0m+
z>^OG<^SLbxmbR4`sD7KWA~@)X$5WetHZ#?WEYE{ud<{jTPil$#+|Qd-yeN_DSAn6#
zX4W4k?9Xz9@8b9MdA{vH<>fnNGJhucA8&t>mEgda>AA=>AUa8>Fz?2ef^uJV3B|C~
z^Or9MUT3fSygBRNi@j@ZEz><F;$CE>d91>wugFpIlhm{Hq~yXr&AyF(7PrzLcnYU#
z9Jmm7?(f$#4d1et*H(Lc$f`(Rv+#~5gYsd+M;D8ergwU8DEYACVEYjPzL!_0ACR<S
z-f9)`^?VJ>u_t`{WV7DRcQ*QP>YZPs?7fZm+wTe<<7Ry$x*_~Q?V{wva-!RLrk(l`
z>fN*4X6l7YQ=?XaCfB;kKfHbV*DW;Brv2rd7aumbwhFxP+i<l0`a;g?V~^Xod}{Bz
ztgo2Iwe?r?&b8nLJ`b{`+_if7*qT4FRZ7iC{!;gxEp8dxH>tH-yO*x>Z@Bv~Q@-2y
z-0=gM<rTM%CuPca2Uli1o~1Zj`d5y`k0%mLGE6duqz`p-X3jC+y`}%Crc~DZjs<@h
z#Fc}4-WarR=Ce6^_1_=6)m#=^3%hQ<==;?9sI2Su&I6~MCfo>G_D|?JH)!JJ$XlUi
z?jDyTQqNfYw0I6b-r^;+dh7FY9rMyPdrH<US;M(E<L8lymcraj&xIq3-20pUn&;hI
zIGdgKQP;d%$_E~6&hujO+&{I_+-lO1>IGkKp8D4JJ2viDV+Q~1(+9RU<~Q^IJ+pj&
zp?96tEBDFQf9&d;y8ZDIo{9Zzm2Bo37ot*IWbQqUHCemRviE)0LRmKFNJZ(ggIxjJ
zcTZXN;OJke*hlAow{rYgcfkJm#m^Vl2YR<Bcm+IHl>B`nYnk%=b;WiUdbEYk+rMpC
zJ4Hr+i}TURS*0s3CGzpT6}8y@LiGIP3u>&(0#5`T=X=Gf<GW(3$*BWJm^Or8xb#$f
z{g1OQ2hOQ%KhyR&V1d|_gK?_fHWw}@+$*ig;q<-|kjm=Kt7E!p%Y|LMZ%j8_f59RZ
zwZ#!61RD5YHGY}2GFwPdY=ZH#m8&iC47X|j)tQ>rIrnz+oyn&q-Jk!+s&GHCZq3)$
zneUBN*Yaky{Xe$(!u`Tyl`r2_B!Ax;DXVSqdFR#d(k1Er_qMC$^K{y(Y*SYM)!x2*
zZ~lRWQ&u<!xtzK_v;117uW<CcJ?j+Yg;pQBIn{@qnPYYB{W-z*Hu*7Un$r1B-m<jT
ztM%!$Qhhb0<YU_0<vKQnULT8{uCcz>=-8HjUvB<8UHQ#R`faT%>yF7B)e2~zb!nnN
z9*1wo&KHaOR{ipKoVhBdLc{-e#0#Elzgi#o_^xAW&F-Feb>FKiuUajlF8y2dO?+$q
zgWSJm%XAB>YItnU)v0f>+P(hPy!(6K6yILXzU|wJARfPW;aX~v@&?~cY*sWU9N-9J
zWsF;RHQ_J+tOpBE^c5tU9ISh5ul=-Qc^(Th=bX1!>MskPDd6M37k@nF`M(vGQ+c$^
zk~eD}&hZb4VOp`nw#2?8zI2xfv*GQ;(vYUq{To>=T_1ZGp7~jFZsGhQ_n)61J=VUu
zAgsCfr!!Nn{4s@>IjN6+g_QVNSlv&%F{${Ztp54Dm(iRxhZB!-JQJ`ubK_vS<6%*=
z7{}{D54^YEw%fcuCj973zv}J3h1J?x@7mth&2~LJ^_$`Q(^_etimU!qgv@YmWZEHm
zIIel;)EkkH-yXiX&T@X8*2$>vCI1yyl-!zXe&Lenv$qw?w{!0Jl;*-C8qO7)xKOky
zY)i>f9?`Y8!D&WfezE=HqqbKL1SxyV8h>7Y_Kxi4hXPV-w=Qy(W)n(%SiYyQTI%bp
zV@IslR!*_quq(LDSL(*vy7-!>Y;RUhzRkz08*5=R;n=Pu*<^+aw~ST$*qd)vzK}DF
zxcVg4@$B4u*0R_e>t6B9kz?FhvU_(?PUw~`p%*o~ZeKjXWmS>GyVc$yzkb!1861{Z
zj1HXbpIVXJB;ILq^ry)7zS<*6WskP4ZkfmDv*psfihXAeM^spqHEw8oc5L0mYOxPC
z%M%+U7mBK{xcF}M&cl59JMR|FaV=8!)Y2-Q($4Yi%}LH9A;P|<7pEMm-@w+K@S{`l
z9CPN*pP#uW&+xt#AC$w_=f7cYM2~)>N~_WI9kGwfI&ZUf$3H4Nc&dHNqxl;zC|f2y
zDUF)%c|7Hxrh<dOr^^k?EEn<oJ{}jeUB~Y`SFaD_+LQlIA6WI!)I_ezEK#2=jMpxu
zdj0Q{R~;B6(^}s?xN&LkzdBBZe&0<j`($e#U%4$&Q4qX}n|InVr=1O|YvPZ+VA&mU
zw@967lGVLPR|Spc?uXAFJF4!P<hOg){qUgOJ)dOfRc%ji=V6^S^YQ0<Su^MA#y*(O
z^Tzi?Fc;^7nN8~+H%4)9Z8;p6w4~|oc0D8WR|jt%IH#MTm9l!m!Hw26J$`<w5e}_e
zj~#gYf%oxhgHx{l;RnAvS%*Y8#5!cWcrE86tS6E5`-H>#7}F01f9_@6<2_`$<)oAH
zHa8}Y>l+mM*020=!`jK5<NA&L-rLskPn_Sku`^MlW6jLDQ@VV1Nh_?}w^qA%ce3KI
z+TwGE<*VYhUYb{Cb#<!YK6~HGP17{kK~n<?%$cVa&1{#vEXE!``$Qk7SgNwUGiWsL
z=v$Yn>Ka?^XI_B;a}t8r#63CpOIE*wdFH{E6~T*?Zf1Gyu?(zqEZuZiRa2~FpUhVE
zTmSc-^IX{U>JfM>TCC*UlYh3aEd0I%zFRLTyS&V>BjjR@no5=2iob?;8Wi}~JYT7j
zu|siLa@E6<t!M7lDjZ@q+fm%=I5+Kz`kd8`s><H;`<Z75GTl0B#mP30<r~i%%dJl?
z2$`iCv_G_LoAOL_>H<R+_u2kR47(hRSx>aS_ii-fTIR~rx6K4R64v_L{lMp(&pW02
z4)e~8@vnIwf4f@XK-0wVR-UOwAtw&3lsz{u^wfsrc~fraCY=ag8Fu7LL^IDl`@4cI
z*P0^VJn5I!x**HB(6P-;Qf=QZMdvfDbJZWR9;)**^e^14T*F{l?fN6uT*%>t-oo8!
zH>};R#y-3pJ2`2$^(C46z4iBMgTL|icSZe_u-aiO=J=2=G|YVG?v>U;SLM_V?VIgh
zPk1%a{)dIcq$H0nMa@Olr}`Y{_!rupth}0Sa&)1xko55fuJ0G+{ZHH8sd>iypHHFb
z(U3DM7A&_s{q|8o=V8@lVxXnpoHB}*+)|ND{eBOp@TO{|?ib+Vef(&zNz2NoFLe4U
z{)oL+XiHnPnc*O3^@>fue>~a6GU1?#a}C?P`#bf*Z!EUp_@V1I`^cdy^R_#!*fznM
zb6@c3s@@yXMSC(QJlq^F5LedDbCGqf*NYEYn_le+$fyxiWI8O+xG`8SpJB@4#+pSu
zVcb0?zg8AIzmY79=+=8JkkI{qhP>60g!Kn}kBZ#gc`;E<LR9?Q!Kcz4%ZyT1Oj&qq
z`O>ZXcq_G1R~YmjTdzEWZFBIV`Bvr!f>^zKbxc=mZR~B^&Mz(W*Cl+i`L~#9-y{VY
z@9(hHF4JmU$~Mg^!b#}FJJ)y5>bG0ueN6wZ^z$=lC4I)KJ=J_U*N^Z$%41n^i%FiT
z+~qvK!J`u=q?58bR{mVDO}jXw>#!!<V;{Cw%llf73c3#7JlT2gl_p=g*gwlcXP#rr
zPuvaqAa?YMP_R9CY4?#6O3EBRdyc+YqYMsb6)Wys{S`(PoqUhLtGf^OC(4;jlRY=P
zHUHJIm0R|{D_S;NE_e1fg&E%^Q!<&PK1n<=j7aghFjqRGwmFAwagU$l?d!2gGQ1CD
zQryokjO7buTfME}!Go&{tJ$kF&i)a(c&<&0xzc}fJd=a<mMzcE^lKl#Ewu0d9R_LE
zs8`$^tU96_LJkBa9v0oOMP0^Ub;I%4%0pA-n-rV4zG-a@fevqP0x_g{-x_nZ<RxXE
zkDc@OxKu<|3TQ>#v}3<!>=ih+{@=eT57RbGe&+w<cW|e!uH>#sP05Q^$8_&}f8$E@
zCDR|h9}a%r?PvMYN+A0?XL|VZ=hNEDm}R6c_8f9Q&CzBdlT^?Eo>5mUU@7_M)9Bym
zpA*U0>(X#^X4y4w6VQ6a;2s}CqoXI7l)Zb7oMQS@d#%ka%r)0?UaQ*dTQwikJ6EeN
z6Fa9hdxOsL(-PiXKT8VbY7gDb<J?oftt!EQ`>RFbF8<16FP<#SwYAvrlkqlxgwZC@
zlEX_a<p#56I4)+JRxzRf;|IBUU(%8eS8i*ctjg~F@AZY)-4-=Q@818h5aK($R-P#<
za}jrunO<|!N1?X(>&JX;griflRyuf3o+qgzx+$dMl39dS#ttvt&^NqUCk{M3?x*T4
zs*@Oh#MZ?^Fn_AfLylzyub1<HR^D9jVV36Qy`FK8H^=0LKXcJ+6^0V=FW-*syv+RR
z*!LwJ2bt4FGgm9y>OA{ysC7Q^<KjgtzXe$Q4LFyq!*-By{;fw@9TOGZ*d!#G=7Zp&
zKF6|uItMuZ?%CP0O80~22TMhUU(Yn%rhzhZ<3v>ugDGBiD(gO$KH(pS^5;#f<T0Od
zoZ+l}K)FuU#r&k%UH$j>a8xvJxV$dWR{hU2MFY@e-hoLIRhgx)1+p1!7Cjt3ZwcdD
zO`&~3nVd=m8#Z1jw1_C$^d#Zr!AZ`HeZhi<dFM4$Yb;>cDI#=01+?AZlmg=dZjlYa
zPUj5Hh#pG5p>Sq`N9qkbtv}CtFR<UY-rnN;z}Z1kD`mxmg+?7yqUFM>l1~Jy`o9sX
zQ&(SbjqRIM#H{%?MO!kMt9jp;&MF7Zo|vxK;p}a_?bjQ_t)CMN-dt^CeEwX5DP7D`
zh||!$F=xVze<yBvO_lHOT6XI8#b@Weu9nGZ?Gvsz_n<cY&>X{?#i##y+xOd^eLnp`
zT@Me-gGBBnin5@_?YD*%&Wn=5KWGM)vmCJHD>`+ioaJ_>-F;7I%eJ<>WhKT-XKS)8
zmpBt5d}hy_yIfm79%gwSJjWljeuGK*wBm)+CnbNME(voiE!=VBp-?b)kIO87t)&`!
zsyh#Eji_B<8~fDt@PeZXx;93$En16(cFgUHsmOAdoc6`x&WqW7yG{!2xmC333PZ2>
zD@VVpQYr_&9lueh^lMG9TeIu@sXC^)yF~t8S6rczabL}@M%t%SQNM5Iw--<PnCA5#
z^V7ND7k%|#>4f77=YxJf*|+9-ZtFyeE3f!d3NjB=F4=LoW-jmZ3=8c7wSd)+WxF4{
z-B>!$OPIe(q4}VUOp};`h*^x|;m{n1pbG`9yx|uLTHo7BZTwKHeWCVXn3{ik@guSP
z&^>O)+d`(_wwvE3ouRX5nrME^Q|WeJt49_uKFcY;TO0I(YoYe7g@**b^ZlP+xq5Nf
znMtAVo;GDHwYW2Da<qD|tdYqH0rx2Rg^6qVw9S4kZ4=8ia5>I><;4SOmfQ~_yqj{L
z7c`%Fa6&3t%BW<I!^G)b26s%p_MJWS=XxTenb3uARW@=?SHFJga}F%n5tt)Zb>_s^
zcVAlFSEU|Xe6BPjv*~%Th%HN`w|&P-sYyu_iZ>V?`8lIv+bps63xCeAhhEU^K6rA<
z%cgVjGD*$bIu(zU?SAw0{W-&=8mD&GU-KW;blqkZ)q}>|#Egx-Vuys+z0W8=aPZ*y
zro7cssRy5w3C~+;Rk*w0!1>P$dXBERarsNhxjrwC(8FmrtX1EZ3Gnmme6+OE>dDj7
z9}m>YZ%Dt8R&wI`%8He|Zz31+d^u2{*n1?0DZ3)bU2@GW6~>Cv6_-wCHBK(t<0ZOo
zPUZr?X)>#FB#u=7fBJp*D*jNb8`=r$H!S7O<+b%qlziWi*xdBO&0jKa?SlWeF5gc7
z+{z@nzx&wK&DG`GC4TwJED2PIj?BsANz}T-diax8OW(U7>mw_A{@&I8>yo^m`|^r}
zt(Q#V(&H>yE6TzhtxYyvIS>TSOPh}LUXucCdYZVD&1~0=7hK<t$hin!EZvlnayYhH
z_f1^;Z*iH*@B0*%ubFGUi_fsbMX$_yXK(k)7ZtxQSZeRS{m@PM=jJ6HKM(JYKBL=J
ze0<)Cj~8c#7MwXG@Hm*~*r{_Xl@8B)R3>&sEIiofVRM!Kta-vI%Pi(3Wlhq|w5;ms
zQk}os`HZept9Q)WF7{JC-#7iJ&1{c5r+RL>Mf{DJoX*><dGpy*lZ*W%Pdr}HvaR;Q
zf{Tj_+WVb(e3Qf{{5f=ao^Fu&1m3e!d%M5o2hM!9Z9#3b81Ea&h*@q-Z-SB%`qQf|
zB8xUWN$6z1Zn^cz9jBh0jD5Pl+PxWGG#%4&m?vuVAfn^L-{m~1A{G<x{+<8Vm^(r%
zz)jJcRVQ>q$jKmP@YHUL#J7T{H<r4XA8>P>9kOH948KQQ=L(fx#4c-I$`&VmBRgUJ
z#)oQEC)(UVn{4(-SL9x}(IOsq>+du5KY#D;dlGT@+xPPCldL1|*!_FwQ`8;xQ?_3#
zdCJb%GO^U6#fvsEv@^7Sc(i0&aP$M`2f+`7mG5X4-1>cm+y3B*oE3|%S9Ptl43HL_
zt!&4=KK<N6OOs7JTQ^R>cqw?=;$1sp9~E>SY+V-lNb8)1&B`Z_xC{jIm!8!NVe96v
z;N-8F=l?v)QQ_LRZ70fOe+e!&3|y__t>7@7|Az5~^o;8tjy-3)Q9SpEpyRsoLmRb(
z%g;;B(+H~dZ2i8ELrCM?W0^@0^W=Kn3|a!-i@dDf#}f7Yt9MzoILBEAiDss6cV5UI
zbC&KqR;1Ca?tWCRG-lhSX98NmZ0z1PfyIf(-yGfB{jElKaarP{?;=N7CAogY-Eb&L
z{JhEO@!?$oCU;h{RC4vMPi$VV^l;AXS)x+%&g;2b)=gmd-n;YP(zVhd%M>5AcIUN6
zs^41Zn7S$Cr&#jA6-|5D-V5#R-j*FObN##KvWz6nZ@yNt>^HJadfDplEZE34%_?$M
zRc3+Mn%?+*I_IWwa+Tk*J?q(XC2sBR>y}$hS=p~!Z;3vCK`wIR+G^Jm@9#1_6gzlA
zK-qAA_m%$LiyO1LCu+LIsUNg0FL^i5Vh-QjFT1_H&YYNE!dGamePh!9Sr2NuK5OPZ
zXHnd9Qsg=J-xi5G!K%k2b>CQSbDr^;^>u`g(B_R+s<9c*?UtxEeFi7vM>UD3Oly>q
zWwsbMs7#%HZs#ZC?q$jvNe7#6YTU4UV0Thxs=~F)9t{b5*`NLPnWEWj>kv?Uti))t
zx5fFMySh8qK5^aP#3Ir2SYYX~$vqm)2S1!WaO}W)R=pz%b383RKmV+-*3Yu+@*%ad
zi|KQVSiSo`HaQ>Vn{D<@kI#(DXEDRh4NJX$ZaF;HOZ!6H;Y`krC5KcdeGb_W&L+5E
z>86Hd<%ZKX_idG0v&pHAOVX}ADQw01ru@SmOE)dmu2`E9`>gPqC)0|pjZ2T2_i>x@
z{XVGn{GzPp;W!T7Rd*aymJ3DXhGaK${$Ms)aYJ_5#iQldBQ~6#zSs7`P1cRt1&qJ%
zBo$Blm~h%9_tU0!TmMDV{nr;~hlgAYza^l|+{0sNbmWA9vM{7sCt}gZenZ~+c{-!t
z!?T^Iz{?w=58b}6_dFeGosD?%dEt=H@9Ycr>l``x?eBJ;xNX||%|GydFfnm%oogdI
zmF<-9Q&tvcX*RPn;GLsw`>r(PG_y3AWw8Bj$-iI4S#a!H*}CmKDPJB<oLX{EfnAy}
zrTxM=&38;@AL4SBS940)I5AeUzIn@RDEYli+tHi#jc8Wq^Z8;KoVyz$mrm0HZI)!(
zX(q(5>yU}o1a05NjJpn+NKKFiF&yCw1Kv5^yZ3I%jjhr5-syEaAMHH#dfG-!mTw0R
zy?S!KtnlY%hV~C`?x~D?jC>D$AGU^{p3mA*`$e5)4$~Z_IftBgI9F`d(1&|l;&slC
zLpIHGRo#-NhqlJOC{a>rIr>nmYvbP2Uqhosjxj2j@7r*%+rr(|@R0b9&Ho(!c79^o
zAjEaZc=>^#TT${i<G&pelFu~QW5{-G`hp1y9=Po_Kjf|K=+DCI|5k&4%Edi~k}Vg_
zx96?C6(26Vh)t5mXvSilZOZ;1Y))!vHHN-c(NJKx-=-lGt#CtKH_7y89tYPqnY64M
z{`P<VYBsB?d-G+qPvg44m7L2RHD$g$>xre47%s5#hA(J1)OPzy`YJY^j~Va0rfErW
z>~?6^J}wZ!0veA#m7%fCnYpC-o%Dptvcw&qikZ1G9P*l38uGS;DVD2#Z`BS@YRgpp
zCZevT-JNWsHLGjZdhYObEIq*@_g<{fIiPqzagXMnf~P+>Gi<#7p#`)K7qk+W$q%+N
z%OF$nfa;!z1#SUnS1Nr_{h<1$q`9MRZ>mp+T`P;njhd%AGXnOr%w=IVS38xYEOS%Q
zU_)ZUVXjkCSsJ$W8BNHY#BhO+)nbAyuYljHI}J=5^!+SWdNVjqoFwA+Y6>q4Ym?F=
zOPR}>=0Xk^m>jt$a7XWA+U0bC>&dBU>l)SxY+>5TC&aMJ37#hI<*RKv?!sUubt5Cd
zA=%q}!^NY=%6ud*T+A!<e>HFZ@qCL*-^HeD^*^7+u9Wc5UAEHNdga3qeXXvEjBafb
zl1VICk_ksPOg{+PG3T53-ZJWP)FVwUnND}>8IS*f7OJW4*_%>PC-y+--l`Qk;H4C%
zfsqXlH~8HQXzO{n@mz0Lk@L>U2@lxTyl33xc&#7^ykdB@3WLeHZa($WQ<GU5$~bxp
zUKa$lA7FedAQfHFGI<w^z?H_ZhA<u$)}}`q%L;-x`h*!-bU~|QE=<v0pv|(@VO=W=
zHt)jM#<;C_Tey3{_T-?b*P7p=dsf=5HJH0dY4h>Lr!1nS)$io}pM13I6lAVIcq*#}
zXf;f$xShM?>g5{H1y9RAKo&e55lpdY)T!ayXSG>l4=bZ`VaRFA^w?jO0cKa~H&px;
znWBBFmqpebTm~>PZ!v6SKN04^kRih^_mBPL3MB@U{<lS-g+J?VIA^eL3eK2gB>#`y
zbNTW;hyC(pbQgfudg+PmVwknllJ!KZllX?f18%HEOD~9mS2%@p#WKcTyfnX7_2wns
z3qDt~5Nm(Hi-Ah@pS<n%`<J%+>AIu$-*#`Z2&}yOsUpNgG<NEC<5TllEn3=-KKoI1
z8pJYh6_3yVxtU@42Q_G(pDj?pRlr%mneTW&)nSg_rWWy=dj8kMeh7hJS>4{bYh>qf
zs)QZVuT|S<QCNB^;rUAyi5-T4V$u(%=Uq}*a5j)_Q>YNbEvr)w+dP>ZtVJF;lySXn
zC<CpF;d+aRL$L*-3#Pg;aYQL(Fcu|xhdYFGL^Z~6b27v#X0R5?f|fa5;62gnWbTmv
zzhKe4WqvW0y`R+X_^zF9VdmCndDi*D`(`D%`{z<h_uda`ySpe^{*ScxQRmnTd>zS!
zYaSd~DQNy7xwDt;Ew2oRIA^0eU$y6pw=V>Cl!(`Dza1PiFX_vYnH*nt&5P`PdgFFF
z``5X`9UIm^InQ^`()^p=S9jwV!rPA9X5G~jH_ZLMMB;{M*^kqLYww)-d+q2qn}3qW
z^8M{XPx`G^zLvXTDf^q{C$;8kXC_ZbaQe=jrnIko!d{LE4=t~%MPImG@VL#<&F^C1
z{gz{GeUmGnhFrU}rXtIEwf5ti5<8#9gwK)9ZF+oOaLT^OT?gkEGw`0uD41IKLvRsa
zUY78)?B!d;pTE2ld?(>jglk9a9KV1Ui?n>79hkd9tmk}X0*iax>KeH(mi|B9mF!Ij
ze{VMNyUV(Nv;7>RzqkfyZ}Yv~{Y5Bhx7B-wi{3G%IR}@lj;Z*#lhuFn?UFU@@BiK}
zd1H3v*RRzp16HlRCL8Mhd7t6LEn%5DWkyzOe+PIQ99jNf$)x}0VQYoce_d41^;>3y
z27Eg5;$xcq;m<SwCwcb-?|mm{$d|wE=gp&+J|ulyu|n3cWZo|6k1mOiWb7ATFx$6l
zzq|I7+a1>PJXtbce$AMAxgtXPj*rS=`G;~dcR4fmwRO+AEaQCr{<&qBqD3R$Y?qM#
z?={`-XleK3z^8XVYy7@>Znyoqxo`UzZk}Jh?Lhm1@)ti}o>mqYHD)<8<!H{q${iLx
z&ZiBc&VNZZ+`U=h+<EY@S3so9xsQq8)*SIOFgkid=!emU@U5@U2`PK`Jjl2XTF4T4
zh2vA|v~!<szD!%bYetu#?WCFKFDFa%vpkCyGoPq?_{rJb*Dg6UsMWFws^v9+*U3Km
z!OZt{;(?meq8!&2i0druf8G}No>%YP!E&}Z-nH2YSH%Qm_Uu~NBfh{)=aO~B*Df#T
zH;J3S&pBYhlB9g+yW%^S1}*kWvD^{5IWL()#X!?)tvXZr_D%nxcOhb?%7NEA;-475
zVfUU2YVL!Ex~A%cW~^-L?P^<{vxDtLTU)k6UMovOp22&zCvKnf8uBtG>pfXl`e(++
z6Fcfzr{%M+IrA_&QnKkbv$o(5k%=D0Q}!9KShSc6oUmiD05Mv{jpcA=b8rcL)T0(u
zT`h^Rn_y7$Xy-!>W%VbZ!Rw2GavJ_Z47cQ_H0I3Tqv0T}<j8a4nkPfXYz^j{np>D}
zDQ<z&XT>?VAZr*~S$HZLqZp$kRTxSd7l@v?>d<h<IQf`AYf&yJXLo5&&_-UK$+i%_
zUXy!gcERuFNRczm31TgiWoOhLc*49)`p4OR|GJvlk5|u*fBfUZv-EE+`n{t2=l-nT
z7yeOmpMBN2^7q#2B~_*FDox6ob@IYx<?D_s-S5-1X`fp(^Qo=u*Eh-CzG08Iq{#%V
z`MYzslC<IUeav0@>)&5L{BuJM|Hs`ACkbEdT&CgjT(95y!S2{fmkrGN^Ph;eyS*=T
zUNm!)<74kfN*WwV^`3L?mvf!?`$gl@B<)X^zE7O-+<05ovXg>WAI(thQ~aQqH{qaQ
z+e_J}9~}>Cx_R%O`Zl?upXY+ywgdgmUn_XLvv`axXY4*Hn6aN{u4nV!qnA1b?@s1a
ze{(AF+^k2pjpJK`Q=dzVvBy{*+t<gq)i{Vl*!7%%gp6ZKhxxzGmc@&ARV|-yF#U-|
zquO`oL%%Y$T=U;QIdkmy+<9!RmWw3rB*mJpE<VIw{A|XfpJ$%D*<0Ut=?Rm}LxF0y
zw=E~E&xsw+;eS?QQ2e=hrnJwsgetbl20H}~9p(2~I`>H0nYmA#P6V=lx_$rga(zct
z#<Zyh6O(4>8@hZw%EI0ERFpAB<(Xg6(<gThlsLMIZ7e*;{#j|Gv+(q)^UW5gyI%4i
zHfb-M*1TEr*iD`@2@+`rNe>FsjZG#BHBSiUlqxPsYJ2RQ>@sO8m(-RAQ~L!^?qoFM
zQtR|m1`W&wENF0wUe&N}#^F^SJZozNWw<T8gQridJhydb_ko#7H5p;&Vm@haEh>|c
zKR5MQyF{0Vzx`B&jMgscXsN&z6E~a^l|Q?qYiV0qMn*f^O{rUoYS+JBtW1xIW@l3R
zqRjhiLu2Re+F36oLoQB}YI!Y`*T3PM)cFPu@A><xpG>`3CVN%<=S08n6W*M86RK)s
z!~bgg^U2(Ga{KcN`EOh7jXFPP>6Hz~i%$vKU;SS3?#!asJB}y@CNICB{OZVM`N(S%
zZa62sznk<<GEk`0qif><tFk@w_$4k>-7@jIb?~%9<f%T>w#S~nQFAXjs(gQTKIr(p
zby}x`ANh8gvuk-T(VUoME~@ikzKC$=#uFSoqARUG%C3pIdU?H$;SQ-OeagJkK+}T0
z8#XnZ&k@-uE&I^-XW1PMjmK{*PMmB0{-AMhw6n2dk)lzHq}G9uX`eT(sP8cRqG<3T
zx6P=>u4;cV^Rj>E7T&+SX=gw(Z_VM0QODm4zh7g&;>dKFO*d7%H%PUAw7zxZCSPCK
zhfi0}glY><mEnCd;kdNGk59i`RF4NgfByGZYTGA)jK>pRG*6sZC-`lm`18<<LK!(M
z)ooXNT3<<siD)cX;ONcjl)0nvm=?dlils&l43D)l=68i#R+TMT9zUI*dqpctMmyIx
zt$UNLJQ&QhZe#>xFOHJFk#WH%nSGh?U84|&na{URG7+{_-+bYs=&FZbD&_XRh+WjS
zUQZ&pWX7(<*bMnCKT>KV58D~M{nHb8#7;u5?tc2@)WTIe@(S7S+;3mNzxnrPyCv^-
zDa%^#@t0?|+0^>(@X?1aR;(>g<FTpH_g@umGv!fP*G*g9>N>IZU>{!tq3W5-POX+I
z>N<GyV@{!{^r@_~udlRmP2Uj(TJRVeAekHa)l7d&s%z5|p>S^>UC^mG@0Nc``X0Rf
zjfS{T$ej$0@+mi6Y`&DpSG25qDb%pkLgMomy@c~e?%kXD<e0H!#nvXHh^=grJbP|A
zMCw@b?d!bvG^A+5lM6ErzbNl7%5UIpKelHw!>xh^q6<Ws9UCvOX0T?oI*D_{DwHr5
zxj%YecJsL#tF)+7I9KdJ6vhc&{SRfAzq5&4Xt<-r$#8c;1KSfbPlgwIFW&ka&)jr3
z>4&fC;rWk33Rixr$gBMArF!^r%+2UX@cQTE1)yP`<sX)P_;yG7oS?lcc>OcDt*iJz
z2n3N<e(N#nF}8hndbi`;15lz1pCZj^l?U1YC}1ih9JH^p!JPlC7~_Jq3wctcRTxbC
z)|$_WlK=NyEd^dHNTuJF=G(&mzgT#aqK3cv^NkNI{#v*7+bI5?w(!yUgO6>vWtXp#
zKL7aG$}caACu}{-ygq;DWQGeCmo00~&E#5m!NyIR{lAo-@3!-b2Y4^=I+-h&bKPaQ
z%i4Q*Q{jcT{CppeT5hsvp2mLrk;%2U#Z$J{cCk)n?RackSf<|R|My|#a;77ab1iDV
zus`4U=#P)g#!h9M`T`e~zC%X>`z&-nupdABkikL5?0E3k0+ZW94}|W;E!i<~xku&Q
z&ur<ooVzvkGE~Y=S)5KdpC5H~#`o`ggG$8fx|-H2FOW=J*uKDSyG-!k=WLfMzis#+
zBqC&TXN5-j6m|}o#D!C9v`?3xuIH4?owo6c%kP*L=i5DBMdf#dm-Zd{bwudu;<fzW
zpUH=>4)&hEZ1anx#82nGd!AGEzPY=6&eo;xB%YV7xfnXv`17%q_bk3VceH+N@yfRI
zS^8O~iE%eFY|U!zB-`h9S9xT<iFja~^H}=guk<rI!u<w6)gz7GnEXERWASI})W~|b
zLrPk`?#B0wdaSDseKxX*Xgs#@^!Z1(AM8pDivKe0M%V)-iHf<u)^)$QuFk2^+?daN
z-)zCs?+eQ(Wh|OhaaVg=d#Jm&`~R+94!(oQ*Bz7_A6T+XIH=u||7z>6oHvgqDf*Qi
zTBWhwU{6+V#EpB^e-t|3D=n5?#C|-CeLIiwK|P<1+aHRGcwhEaKjyAqFR(4z_Hsbf
zmiyl7&%a+fA!MtU#5F-w)|tg+XI#b`h9rxp8S5qDmM5N4=wM_kOWN@&ZHZIsv~BX2
zZO&`H4{go%zPdn_{UB50vNELuKeptx>gH~px8l2F%O~#j>E7%&LR+K0tG`ZM>~vK0
zz4Kg!ir5#N%`qPoWaq8F1Ug?i!Q10mx8Wt#A4jLXGYsd|<nk!&+`HCqnpdmVSGoM}
zn<eTiBAc9Fc$pf#X_sEedq?flruMM>FEtidE>7&;b=`;0`&@Fd6c68Wvtt(T7wB&J
z65`>;!%(aczCGKDkL7mCD(@T0&MLt+Rr}bCj(t%rdGgI+rPt#Q#Wb};{G!iw-L_Bf
zzyJ8`k6O^^L0xzC!8d<&K7NzB^l(XNS<a?L#lUt68D@U*_T+~TtMtD~T}-U$iau`F
z5j@AgU|)6T<J{?NN}C^O7Q799boLwLu7~C}Z8c>EI>K{>^8E~&dsi%TiJg?;*(GaV
zU&prH_3ph@HQW52-(GFh*Y|qb%azk&ZcJL?Eu?EQDe?8Wz2}N2{Z_pf6=FT-wers-
zosv@-%OyU~k*;W+m#^LNci}n9WA@MI2Dc^kwCXY4V~I}iuQ@*9xqw84>HoLc`WLpC
z$u{z^N%E}UU8=-mBU(4va+R4UL!4|@>4sM!tD9AiuYUbux!$)rr5=OovNu1Z4@+u1
z*m77d{Jy&Hg2t;(Tg47fPMKi1#ge5}XW>S!D^UxsIc=TRD4p|F@3O|0Ywj~s43{}{
zr9Dhx+kc_waMS0|V;O>{4oFPtDofz;a<x7Zppd%4K={bqAJ0R2Ya7gkSoI8RUd{Hg
zoMBR+WqQNk+4@1pG#N3OeUe<vl^YnJ`0L2babeW?8NNo;L`wcp(wvRP)rUXE<T~&0
zd2;QZc>L<dKAtjnAKx|Ea|_O~8E0OQI22hl>-n9A%!JJc?>>G$q2&9U=hdRCS8*(U
z`Rm6Y!(-cee*b>A#h*n@Xhr$*xf50UyUMyh_7)#=kUhPC=gdnlt$Ta_q_SN!b?u*C
zv3Q}`qKq~dsYyvb*Nz$KBo&s;%D8o6LdnI`=^;IiOTW6kmT0O@TBWtR`?Kb`-iJ<Y
zzJJQj+_XFw{m9o$h^cnjk!}ImH4^t;bxfP+;Vq==DYExZW>)Qg9nXykaox|BKAt~)
zM=bN!qgSK$M_l>Y_JrxXve@FqcB$<bBG&i${nnV_s2%XfdG^C2r;P*J4%!<uznI{k
zq84Ga;w%>@L+*uXj5pap^d#n+7ZWb-pC{dPZ~eWw{VP-$r1{L|<XfJ-e9(<inw2Yo
zXQrZ)&aR8j4c|0VRyKf6g1WWj!ZPlNRp2Gq@l6Xhz2NL$Q1M+m5-h?sO$)?mRBhSd
zy1xv(C_AY6q)AjZvovpxiAl`jtx73+R^27f^$&0sI&Zl6PUlnihWm?>yB}Mu{MOcx
zJ?r`g)^q<h*&O_R*71VKb(!Dh>h>PCwfj5$1a7@6n=~KPIlP!>=%icl=Jv`Um5$6P
z&1sq<<}YUi?T`+)n{`X_(s73EvIm>y3i`+jfAm$nzwx7IU5wYRxj}Ee7Vqm1FF9)+
z5b~_~$-Vh^)XzI4v45T}?J~WT<=-Ap2lKOX6N@)Du)A!qxz(sN$^Tu9)8&I^>mp}b
z@R>fHbf|pWBf;&=T!odg8~0AkUa;kZvlgr0OMS88DQ^lyn-++_)cn4YvpIT$=$prz
z4^$p;xBcnilE!O#H7kMb5pPDF!n;osAHHKbb$HK_Xd!*8qlf=6Gs<uI$LuNhRQ>3L
zS3b@51<d}=JB1mq{qvb}@KQqCmxL6xoSO$$ZDiy*mX>hxz@>&lx9go7&P8X|>O`{$
zzfGBNP@y?^Enl4aeepf-{3K;So4-#ONEW=GaPZ)y=A};4xFWPNGUpge#J*BxC}E6h
zJjN9h&?7#<sc`QyyM;-|zV>We_2c$y@p5b4r+O0olRHbgK1*_lhdpf5x*DYz(;B6n
zvU<xku7_Gzq846r+iF(T;ojZOtCiQvlCZe3mrZoLBh!Y92i(M^ciY>|ZrT!kueLyJ
zs?>Jtqrvb2r*i)p_S^*=`6bYegd%Yz?)dwmtc=qcrZY_caOmQ_w<bL1GEbX|R2>iZ
z_I+}woWyY9+X9}H<}DA+HyR$-Wxi>}$&f3Y3K~d5-buIMCzE620#Ive=>&!*@Hmqr
zlY=;Dc!`%q^nz$pn8I~8CWr79JD7F`y<mHR)cJ&L7Gztvd&2hf2j5AoyKCrIxB_|n
z?EJ-d63aFATnvP?rh+RT7H_d`OO|xqoqXpMc+4Dm{0!W&hI{CFL3vO0;Xpl&a3PaB
zFEse4sB^N&u6}$X#g-*a0X*WXb*h&o;qZYr-znl8GT<TKy%QL~^js(FFR=|_4Ph6K
zJZw0}yFhfo+c$?_7c;~vfRCUI-w=4vt@XlGCols#e8wa-K{^_=DUgx(M6d2$hPw(S
z%tebY#9p{9nOc_~yz%~{d)jaR*o4Mxm$*4M{mZ92_qQh>TH_$rEoTeb0Lfy<ro~;u
zQNxwbrJvUL89W`*2KD2i?)*DEaw5=ef)g)yH3nX5TQ>c`^aJ81e>rUCUfKCjLwt%h
zr&M0a6n4;X?l$&Rh1tyq7;;5Z6{6J{7vwJDNm+h@FQfeeAB;9}|J|1CbnK5V^I8$m
zK=u}<Ek+&;FPI#)CusLr&JnyWw6o@f>S1q&7hIy@3gMt;AjCsR?LvhT@uKUy%6WP=
zc(t`|)H!#5pP<gpe@hx#7D_ZWu9&zmVCqNj0<LKr{9rR4M-Om(Zj2PU*U}(tsKhkq
z&};z#_s4oJF&zBuI)4;WgeKN<JahKB!>@Erz^q-t;<w9LLzmvbVvhGlEwxL}C*~Z}
zOx`I!G5TfkC$>kkc^n;%DJVF<n5cN`gg}e%Z^b+Ph8r3UB%D|DC~B#&yQ!~Z%rV$>
zRC0M21K;nKaE6<jU)0wz!dm?eVG^5edLH<|#SzsIbMw;FgB_re`%O!sW7{ITn84dR
z!9%<)dHda1;+P!F+t`hcT~3sW&3R+-UcJlrXkB-8LP^~`t##2gwHH_Zv0D6j^XiC@
zHPA^4L+BwP!4=h&5C7=E_wmApwPOxN3xLLc!K1%7^j$Z*{QO<!vt8%Q3Ax?oT&G0A
z2Ldra?fk^l$asq<RUujt-uU0*2p-60-vSxXX5YemwlVC&f!mUIr9pdIr?LoaVOZ;M
zG<6aKI0J@9d@ghQ&ITH^p9>na-^B#pri+vnIW5Z$JnXr+mc?w>jT`%3ELpQH`rbK@
zo2!?DvtyatskpOAIc^1MjWMAj@61^2TJ}y)Jr6#l=FmMu8`qU<mTT;Z41gSpQoLXj
zc(?Mj4`nT9e?~TVS8y}wfkwL?>T*b&<5;-{sY}Pssg}oKRU>E=so%(WOJHAy!!|c2
zhuaR4C)^gwUf^3Od%_P&+j%m8_vv;uFL1cPw1sKQC6EA%uERQh4lU4xM2`T{Ze0g+
zj=K$aG)h1lRw3isU;)K1We#4cwnx`J+Hq9Yx~ML0`;Nc;Mv?wc-xg0fRn>J!ns+KI
zXarBZx@hi74bW<|M*qa)2Rt&@F4y>QGcfW&@PptAZ_uXJsMnfY`Jo|V;6B`}6*>n#
z*e}n~a6VSxa6)a*)3ze@g9my$jVADJYGZaPdZ5dEQ;U-!S2k7QwhAM7o35<WakI<_
zyWLY4Ftkr>HMTNW*uuOs-~|)p#MV|lfn5x19bPys<X)iNvd&>$&P%QrT%f!Oo@{!N
z+nPSF{7zBMuj%4_o4v|o-g(;Hbn9I4#e0M9mnpFvv5C1G{U59i30>(VS-n5R<=ShJ
z8Ei=(W&cfC7HIIgwDlQd^rX+}$GPV&zt#N7NhUY-*)&as<m1<uD*Cy09D8<Hl;!)%
z3p{-${lUq6hwkM*KPk=patfE{y@mPVY|T+?7vHJNY%k<s)AE##zA#&O_k~N$mM{K2
zSZSekoL6TjORr0J@H!21fd@7_ZaDl5k&?f;VY%^|ubTH4oZ7tb)`3pR_c{m83VdUg
z+{L8t?pP6i`L+0MwfW`Dyis$Ez*A~E+e=n#6;*J!aL9b(+Y28aneY9vQPoi>WSg_A
zb;yR0qgg%5=eR=>jh$aE{Jmyn>7gUmf(Oc5b>H^zs!TaCZA!a*wUA1o@rIUz?r%H`
zw5IAic`nYWkWbQY+!I)sbYu717oO|xM)S;xTxu(kbncL*Hml(`)7>1tEY)X9PBEVl
zz8F=_vQ_1<ZAi(>@855%J9_f#Yaw|>9!}Zi8gu6?VqADzS@gZGhd)bZ>xIQN9C=0c
zf1d`J33^<(QTcnuNrvi4pVQM>@1H4VKKSc|LQAJu?b#pijs$-7^H+B~IQc?{@LOZ+
zh(|B)3-@mAud7<O>nY1JmadIoPOfj`={C|a2?)QwCHn6?`RDP|@`PT8EL!_8<=Tcv
zDmRW*ZCO>G$=(~^mvOiI+F#Rax7WX)^8K}@^|!a7DMdYE=bA5E_?^q(etp5h2kcf|
z2bt2cf4<Y*vY$h$O<_{*{;H=&zgGJDm3qyaW!A>!-qgtcFtk>oH;`%0u6^;jPfX*P
zubnJ2%r1U)!8(&caPJS<RF7FFbk|qCu>G@kB7^>N&-afR4et9kr~F%>!?|sB;<@*=
ziwzt;vPF44n3S+vZOQa|zaCFl+jrM^)`5c=iNYc;b~!9(ty_CRgDXNeBz*tFZMW6s
zCUXCcI`~cL!8#p=kd5YFCiQ)r`tOaSampU&XXYus&Thh&b7z*UOPd+=hUIznk-o=5
zexim()^ealjhY`+-~9M;c0TJqmVH`U+@Ml3-#0`oq2`y;%0JU&o$Of69|+4=EM7i2
z;lVK%$rEgmk1vZ`KHAQcvRZ{<OX-xx9PlLPTg=I^4~vdJ*E8ChFPX7)B12Qre%&`4
zQ*YR5Gi^D?n=?Q4hSBLAsnZf?>m2?T*`~X>Cpt}X+exFG-qagRr+2VUE4;1!F0ZwB
zb<&<%PDF)&BcChUr*QH4yRw?mKSZwU_PI~pK0EBtdrdJ>ZOBmo`92|HppF0cMB;q5
zYnpx+X*Gu&1Rw$)4yyK-;d9SBk#O*)RS&yp`)}pnH4ER%=AElLd!a6Hce1^uEr(if
zaP|4`#!B9c%a5saFh6X2t~)31hG|L1-sk7Uif<U|n)O+#ztTu5@b9?8$a5%k`WE@e
zyFNb+E%<fj^Xzj0P8F+-_msY0U+)&hQ5BOF?0zFVN3rGNr2Cd^FE|p{_pP|@@nE;I
zP{+*)cE(%vXGyzG%RG8>`*J=N=Oa0Col6`ZWs6N{ySUlgOzL=MYOTzTO&>P7IHmZ#
zJT*Z!eb<pf4RaBHHMzMmLYwj<&(3=7?XhQHDo69-#7&2tg}&)F@4fE5eD8B53!zQx
zdAy~a-poyX_h*ij!M3v&`GJkSk8i8)3S3fl-za4>+gh<2)`tz!$4{DHyeg+#dYt2?
zNn;|jlXwPa`}DWI@7|d#wCnu#{=3AlKK^Y>Cr<l3M>aYk(?+Rcxw&O(#WJZ1rCA>v
zHrHlJJle*;+VAp%ohcTN((8ZCox86xZsx-tqr*2-4P6|{cJWy_GufPH(N0(?Vm2pk
zvV`>q;qNsrr*A$H`89K;<_#UbHPakf_WBlT?NAFYEx31KOL~t-!grg;FBC$yIkzql
zZMxRPu6rZM)%wAPC3C}T?Bva=177Fe?JD2p^up?dkksm?_if?ZB{`&i<W)SDy0zq@
ziPQq=c&Xi!*A({rP;H1R)C%dT_nZG^<I>XQOTI<C@F{xn_1@z}lf5J5-cH^2y_!>f
zGrP~s&*mYu{4>r8oAU2x_y730;zPk4yRGRvk5`(1z4ewc?Am?hHJAVQy?*j7LwkN)
zV002sq4klsGgj#EEw`A%G~aJ&&*~|fA2dI3iuyq-w)X`U8XvxuCZ#kAI{Sg=?2cSw
zw&&d3HKQxp(nyXccc<DajaB;RK3FYQ&g~VsKjDPHzR&H>)v>l2e_xcJpVVb^Jo!hd
z-CyZt$K>X;{rnLgn7;0Trb~m;v`vo>f1PWqCdw<m+Wz|t+f41MpPv=hbh_*JFZpzO
zl9@w~-2Y`Y-pm(ID^L70O|#jN4RkzC&V+`yW>@msz7=k3Zsui|jtf|)A|QS7z1(q@
zEpzA9v&r$r$-c3S&`MdU&8lW)^Q8N}(bg;K91Auwg_W0=u=&3~@FM3&^ntkJW_w$+
zbRX<)c;98d?Zsim-}*fB-m&b7+y3JE4CT7F%NJ)i**tFCdd>awOXmC+yW8LVe6X^Y
zLtqQvf~&h<=H&b@us_FD(aNMHa^c0r?+e$R;MhC$!>)Il`+w{5D`yLAId~<yohP?u
z>&Fj||L>5m(RcIj2>E(BcW=T>$6I-Y@4r^o$!YnCf){j@+fQ}ffAogz<12crf&yI`
zMVWhg`n0Mxw$12z+;YC8(L&}?LDxY|wsNa*%flr+H^29tK4ri6>e06%?UQ+wG7Gv6
zYc5+hBPvO2nfRx4!>!=5uZFXR6I}L{STE1f;5J%ra-Mg~dF3k+db_x%ceGS4=xO&|
z5-ef-<U>f5{M)%+7wzYW|A{rV-`mlc-`uJzdqw7?^!K)N@jbfbefmL`)zd$RaO~>8
zZ@s@f<D|kI&v`e#{jPC-B-**|*%59>wsxL44~A)b7V?N%OIek5Tc6J2mCn{Xarmz!
zcZ62L%7&#*(@w2lul*@apK)u*f~8Gi`WaulruCg`n#***=E>PE2A$9ifeV*fIT&1E
zz4~+F^S<5M((4?(dEdM(|63yF{qWqw$NhKhS6xY%vR0^(?fH$JLPjR1E*vOt*4iC$
zH_z7cR^4BpslC7Za>eJr+~vGAxqi-{+Eu4Odo;d%`~DN$+OFAh-<{v!sZ>tSxmU|x
zJu2u}d86t1hD{%;cklNvv_6{iqwdEKovzzA10rL5K^qA~%4&8$Io`oA{X>{zDx_Iz
za#rX;(}wfSMlL-2>+%rmNdkT{2Ojo)&TBlSF{hQKVOyKggx+fz4Q7zl6%Ei;77b+!
zEH8X}xPHzyyUU+hbM^=yNH>pi`F(6%E$d}pbA^DtNfr;RJy|5VQZyzkbZGnGX6HZ2
z#?ZDQcFwy`E{?CoKvNJ$tL5hXyu(l}`X+SWYtgmQ3m<VkvGQbi!M4>*!JOTZxuiKw
zYJ&7%Pf&ZdSC@C;Zl+zkoG$HBI&b9^zWdR+T~23@Emlf;)OP4(+oF?gAJ`rVwS(7}
zvN_#e#;}}W`G+*eR7O8WKj{1e=;SI;%N5+E>|k><FmP%Vd@?&*>}Ip_qw5kMyA|%w
z`DbA+5Ou;LmgVy^hOc)Vrl=(q^9pX=xu_sd!Ad}IQ=7w$Go6;^0`6xVZuo!DW{wkl
zlGRaYy8Hr1ktxegsYg+~4_D6=E>>II*ya(Oykw=~r7HrNSsf-@{g-Gi^5(s`wCzQZ
zvq0vrImOIPMg<~_OJ5eB<K^i35wKcB%5h<vMZoVPjX@d5>Xhyo*PPR=Ka?%0%C}H@
zalN9+AqT++oj?3q%Ks@W5_X)kvGo1q^T*0gZDc7pq+9EM{%z4k7sr-k_dYBxb3fg8
zWvMNHJ%8@Si_V3I&be%P=vYu&@uq)O_lEC5w!-P6)mxRWZ7*2%#5U=t%nW0bS-p3o
z1W(m1^OoGPtGwK|VQ;RMQM*KY^(!4+b+-eNzoOgf-+gdDYji(dNvq_2V=Xf?TUprK
za!IztjUQcW+4!r?Ca(5MU&^o8T7B!ikdC0mIa!H*7U`gC{ExEBIi5v+mJ7J1vA<x-
zWBHcHN=(xkcXVuaouS*5BkA285an`7X~ML^EgW-Q8+4MAB!#*=B3Sk*xrj2ZvhPT8
zU8AMsFImx*5bWHrQgPoEIVqkbcV5QD$0`DkwMscf@HQ@NI}nhZutJgPionavj(2z1
zw`A^P$x&Ez+2CaQj3>W09*mS|`CN34UvPnCkNs4wdkHln$z}$NFYVB)J;if-DbMYE
z!{FBf4tG1)&%XR!%xt9a;j+LFNp-uQ9}{OxaBx{6x=YBVMM}wuS8!ol!v&sXYx4`Q
zI=-2>RZA_h<vn=nV)Q(t#~XIKg)D5~dUC35b;G)h$!zH7C4I}j^84$BrhDhCVlIB$
z8~JAYj>9+3o&PLW{^P>TClRY`<w7#Ai|3Wklsg}|`$alvp(um0Gib#qi(Tv9?|&@q
zxeK`R!=R0)IndQ^upu8Y$mv+zQcq28Dt~#aoqsb>td8gL``)%9_sR(gkJ@hB*!O7~
zFAH0SHqyCJXR@m|6(3eaEUUPw&Z&?AT20|7oPWF|pJ6S-+7cm;3$k}Y{O?FTv*M1_
z0u9AN2VR?&f*BWTTi<C<kdD(o{O-u@>bYLBAH#Ze68V!4%N=X;?cv=m@&54nlEn=>
zGz~UsZq-)Wb&hk#de8a$-pC%9KlMiI#-C;D+VgYzWim}pY~DMu^oB)c_2bo&D@*0w
zEnm(&-`jRLVB_0Q-)^{jPt>Yju=LU2(1PjCnNqyGEwQu9ZJzMXH#)Y&>3b~W^xggq
zJlkc>yk8#OZD%TT+x~suVZYKR5ed;5$G2UWXi)s9LynJ2>XhsylblU<TbGC!_I3$b
zE4}~s`praxyvpzKkulpX<U;TL4f(g`{O*PCHv|So7V$Gm{kmuv5dEO~?eWcB5;GH0
zx+ZG6ErT--Ja<Wjl%fw^|1W(T;TKrJJu42pj<;Lyb>5t$9k9VEhll5pMinonCt6R7
zX4_OM#Bl4v0XEpU6pS{BSI__rVlZF~PpxiT09kK&%L93E%H&zbn!t>6vrG^j|JV~V
ze*8G-!ltHdAbL=e-Nz!Kk1?2MhUbriK8FMz95}+MX5R)bC6|3T^!&+r@PHKPn7;z>
zaaAb8iO{rlh<kqVufF32%$JS*9d}Mz@X*b@a>1FCi`g%<aVj%!a=i7y@;TeGb37-G
z{nTOJ;yb1BjmRy=w*t4|^fPW2R`4LH=(z?_jwsL=BO50}Ea<GUqjO?@fC@w6#!F8!
zzWrt?Iq`0&jFw+W-1a@Seqz3^;ZNTde@bWMd)V6tSy?$7<lvy9b|W1tqZ#+EKb5;<
zDspA__HxTgY2VrFeG25Pcg<-?VBKT=J-%k|HCs*D*@jl%l6su)o`3(?{H@Lo^}GHa
zi_C5p<lK;O{vce}6DuogZE0mzbZ7GA^bVV)f9L+Lvn$$gGnpm%?wx517hl`J{bbFB
ztw-)h#+|Ca$n!Jgx8IihmZtfoJQ=Tf??$DsUHtgxjfkRC7SC9$g}Lr(i^mvy21N#(
zS#aN<?}b2gkkz&&CONNVUNUVD`fIX%n|r1JLv^Uzzez$rcRqi0aY8Zo>7~5&(|T$T
zx=p`TzOr-Lmi-I<s<P(owiRXzewf0&{%6yrdF-jbZ%j#>s<vp8)gwbW_Qu8s3rn?Z
zG<OB26vuzAwBit{6Zy&T!$FM6rB1Cw`hu7MPm>gj>4^kZ7VfT?fCfi4&<Wi<Z`QIV
znK-lL$un)~Yz{S7*iq5zs*%zY+vL^MCY^A8p<(;9ZP8D8{x}+MaaQzZeWjhS_`rIJ
zM<*Ah+fO|Z)Od_*Yf4XNOoi61C7)c4XE?Zehi{0wc}Y|^>R=peb?F3#SF2Mvb?@5g
z9E99HG;t{#gwY${VwZm2e8cq@3I{LADbLCCfB#EPPb2rQjOVW7++NY^PwiRd6}|uY
zbSotu-_V$&hi-Aiiv&Y&&=F8}?(s1&0v#I7YN!2kLZ^SDf8+Usw>G^5WmxD%PSZd)
zI&p$pku|;_Hkx~!{u<oa9&O*b^88%s)BUnfl<yZgO5SOi?Pm8$mw(Dh!EY1#&rbJa
z6TV@rp}yziPQ__SU&`7<yp%e9+g7RNzu)oTV9d^oN91Nk<OI~6`jF+ekk@H;OP{W{
zU_d2joUvuK!%Y^(In#9oETmQ>D_vj6Q!ccianB;2u)UKL9<i0gH^lI;us#a_WdteC
z1U?WmMx%rkeCpoT3DDuoZO*JEtrsjN+>~lPcJOpt@pRthg6+&{LT~pnf|s(rYdYq5
z)FNBr`Okk3Y~tsb^c-$24Eey%ckSrLy%zT3%isQcpb;zM(w&@9S)p^T<EY?93k5Ov
zJ^voO2$#8Mh1&V{Mj5Z%!6LEm(-hv+6B7SEf;Z|~bF$_6Oli!SV0uSE$Shy-M3^T-
z#%m4cn`)d4$Qk{O$SuVc%xf9eO28KB^|A<Tak#+rWRpVxBY4%VE@*Au0?`wtAaBCf
zTA&TGv)-LrbaSeH$Cpj=Dc6)Pi$-ob!|mFnxyfnbsgol9DjG(Df}HIhZ>1rVA&0o<
zPX<?!P6sqWa1Sfv%nf=DPnH@lmwtK6yTPO9S!26J)%k)0a~>X0{ynFld9gar^HmaN
ze&%mx%(Z)d$Klc^=9K0&4~x(7s{E9iuyzu|g*Fk_WW68IQA^n2f^(g$SxT3cF{AtL
z(4G&hO<XU2dAohn6$h<?zrgjx3RFykqY3GRNH(y*yQX(i$>qH>te@rYvg`P*@0k2k
zv*YLT#`m5zI$0m=W^7P%NC{TDDyZ3J?_uZ?q-gYHLiG1$5xbNVhMIg#Nm6{9(}OGK
zBpkjYnReRXd#mO~`7;HHZakYqXRP`4@Q$#3{}0aA^^3n)&RJ`ndT#aKM5T9k4ou-W
zbcx5&2h`g1F~7N9K6!eBeC(fjOh#(Wm!3c2J(DZ(O!bVWiQQjb`EL)4{boAtevtH9
zUvF3OuQzp@F6B1#E_}QE&%*mUb2@~p`NH3oJOvZ1pQf-)>v_kQC41-HzsGD#^V{E-
z|ET-_;~$gmLivJk%&ix@MSDXRF70|>ULs>zUXs)-vpdQm`9(m{t!Eo92DI7Oc_=jA
zGi}{1E?Xs~pTk~a+Bfgm&C4M(woi|*t^d@kcY@n`&#getUGC*i-t@`bow(S#sKou;
zyAwBMOW()`Br?8OmZbZ{j#H<0m*&1jU$4&$^6Q&t6nF8)k3T1uJAHrgVUFy){|gUW
z`yc(|zxcu%PU{;lDuk;Wd0sr_xf@fmc)8E|M!pLd7^SKX>^)|#$GD}1?;_h)lZFcq
z8ZL%P{W`VRVb$KJiq6bY>QZ}-O^ZA^d-;t}ne97-SDrinTI~4ZMJqq0g`b=anyL$c
zOw~D8SXXY=s{Aa{YR;_%K06n50^|YB1FCz_nrnxm1q`anRPyg_S{&mMP`G&Eoj^I2
z89NkTytp}WPbDYMg|Gm&O{U<}OC8J*OWVPnuWz1AkUCTbTB}a=gD!Eeg=I2g<^;Z3
zdfl@D&k206^tyNEQALPJ=si32dOa<!eAt<+0&zcNQlEyhEKJ!5p5A~h;YXRoQH8b{
z4nJ4Wem_n2sZ&#zG%xfJOUL%}mZ==lcI=Ur+j&y5RT#GLPHD_RYaYCr44%1RSZe_*
zHReJ~jb9DWR!@{C17s$kbsgfsRY-P%<}c?L*Pm~BzWs5r-^SGzd4<2{e1;xs@aNZ@
z&&7d(6^j>c0ylfwK5RRG@)N#;W|1bS-WGpK`aLJQYzjB0R-VZe>r>4v!saY{YtQLA
z-*{X6t`~7O=cd36#Ke?y2Ky#|=mg^<Sd*><B;d$B;Wcv;WLhIzl@T<B<FF2G-pwHo
zan9=Y^C#CdE=vk8yp!}pW~r{}Qr)?WR(|?s=@o6|`mnbR(%_K>6@fopKutN;sgO-2
ziVr}C>4J_}t@gen?9>?Yu$JY4r)|o;f;Vm}mp-it$XPk#{@&ogqt-H?uQ;CDsQB{6
zgwj9BC#=n$$R2;UP^vAK@!s{Pf9@UNza%(uv&?PR8A0-!{>>Az`JHAk`{snhIUc!Y
zDkjp#v!>5VTB-U-qr0cb;nL^6RGY+w-e*%!w3#h&elINgev8ng#oyPdnBSaex!Yr^
z*%PnZui{;`4TUEumRea&JS^wBBICFFv8VIrWZm>SGj+bYttgAno@2A0o;h?Z$f3l&
zUBbzF)8@aE=h`|jZ0`H_dP7T@0Y`34;e^|=&n**7KPdTZ+T!qGL5{}ZAX&@%eRJ~v
zFR482?Q*@+XyvSP|6L|kl&o}b^Pg1FzJU3uCrie$g_7n|ZeHq}R<_}JO{gTt#&S=$
zkZY<3cDJ~;ylM9TCj%bcQ9T~ioV-%;*E8o29}Q-h=seinu$PHlM%Fawm*37V+v7$@
z{pUn=epu(tz#e_`lIW^6pq`P}G_8nL+&KkOpBG-3(+xST3bONfea2NiubZdmdnK%Q
zOqSJo8)x^qWL8(~(ZcSFr=OpHEmp3-zUDx1<@ZlJZ5lmx-+ioY{^6R~4rP!0j){zJ
zhvKzAd<#fwJ+srv?8sY@?ZJ2a40nr%-TwIVGh3Bkp|x~Y_hHKeC-+3&FJ81wB9i~h
zWAokV<*~1qgeShAteSVfbnWfEjh|W1wNL7uW)c6Oru53SxM$u$Ydoj^-*s-WYSZp3
zy94A`r^nsc9q`V7S{qAG-`kK!yYdwe>31K#@+>{}g_4Pa!>{xOJ~KCN6a6c`NiAQ}
zO#bhcosHKvbQLKx-Y~eqU(T}C-}p-P;f=N#B}zXPT120+-#*#V(j~jb>WY-@shy0y
zmJwe)nH<DHYfxvbRJ^8X)$w^-=2xAzQ)iB*EZrPi&RD&a`QcrMJSDyNVY4*!H<^67
z!X+8Ee)4Sv7xhC=&IYe^YU2vij^5R^KX+&P`~I%5<6UXr^iI68dCK-p3VyFnWL64i
zb}6q^_WbXSua_jO>|T0zp~CIWnzuiTw08whmF-@;yWCQG#fknN_p9YA{@x6@6(Fl`
zFBY}f`jVp6a*e$98NK~}b683~9R11g-1&|l=n6}bvM+i+Hy>DjAl*Hc@m>Dh&5NbZ
zTQMjHgD$W<0zRBW$8Zyr!KnAJHG2Qg;x_5NB6u&!#*-nVI)Dw-M>_W0%~g`{t_G*m
zhp!abRbF`M#K>Bf&ry2!cIC5EdlcVYmH3g>QDfS&Pbw+C_uzW#w1+usdOdCw-1pcw
z`N9Ra#&_CN0(n_PZ>_jc3tpt*5Wc7Ied1Sl#@P0&?#ooRZ{K$$Vf{f9rT7<5Vy0Ug
z@qF)nvio!p^L^eL^N+RvzJLDn?fn1m^{Wq^dm+30jmGrobIr@o!^RMf1y$N;NOQ0)
zaCekEv3MfGh2#Z1Pc}LjFxo19VfnP=g75`lCw~QhE?bZQ)2BrjxG!)!=_}}S?Pb`j
z@P+A9(}n8y8~5CZuEwbDZa6-eqF6GAt0YfkN@GqhOT#uShdM*rU*gacol6)&7kgY_
zbpi`8#wwPuqHbGL1TR4Wuh^&(zTKyAruz4M<Lsb-bsFYEC3hq={HKU>99t}XIDqYw
zh6=-%3=QT=9wCM*4y!xoHG^MpzTjM_K0&>8Uc)?rDyGVy7pyN>7m801Z;fk+6R2XW
z41B@*LiI&~{VehQED!5Vp06!(-8pf=<6{n$i+C0u{y%4Dva{p~&_=v$4d$C{oD8{0
z{opt7if<RgE{7LPPe3!>+%0+zppAID9A0oe@mi=2nfeA*2~J=E_|e-J=iaFMxfmQb
z56|zs81PPmU&!Z9gjPR$D)T{xFB%%ml_o+ARXkG~ZNxa#7O1z(bC@Sm#Z=|=g7b;z
zLiGjeEpZNUB2|o4jxSiBxGof5APy1`u=?YFZih`9dPAhZemU4b(yXZp(aMYqau@KV
zEJj^T10A&hZy$639U>p2QNmc#cmX63?hwuq+Zdx+0_wwnj}mtXM~qSQ-aGQ`_q@2n
z?_Gb)2hExjh?o8}a~F)n22EQiiH$|}-Sa7mB|qjDFJ2H6n5Xh$hhe}x4d)jh>?;ow
zSW5z`vV$Qj5hifA>NSAYB0!fWfHo3=Ho5Kst>xkZo&JHmv&gOdlhB;r44J1zr!D_j
znoFIMeF{3T-23Ur1vv^54RWlh3jJ<O4*iWr6OJ<-Wcku^LHNXECw~Wj4%<c>%`YGU
z?i1Zk`VRUWdmHv>d|~?1bb<Q>w^RFn%b!c>A1&2$iH;UhvI{IRJ?$z|c0AtO_{l}m
zsr~;+=`;=Io1nF#yr~Me6&M%rBAw+9$@bQu3kuLvmpDiOJ#~ST8)&Kky!2(4@DkH9
zm5{QuKBd<;Cw$8}QEjGXCv7yJcjfuEJL$=vHY@(VIpJT<iFz|N2Wg`PyqXu<WHOS4
zwkUGnn!uKOqS;K%N!n;3@5&2pF&W85TNI^lP2kHt(Qc;ZB5kyYSMy@qoQ&k4EsEN=
zCWz&p=r&Vxvldz`+j+6?NJe_n7G=X*6D8hy^pue+{NjelV<849&r5wOnffA9ks#WA
z(iU;9UY$($Nn4aXZ*jM}O#{&rRo+f@Que+zQRS`2q;FmeC#t^nm{g|XS-tqe2@q{P
zd5QJsaO+9h)<!ZD8FEiFoa0qJ$D3lI#CdB1>)+%P&1Nc2(n1S)XI|K8e`IIl6E78K
z=}imkd467KYsqkS+TzRAJ14_AXp5reEwxs+xGjo?w<bvBPIgi@zcoQ7*Q0Nn>%xih
zxgHbDR6K4kx^QBmnTn@%(Gpq9OEWo&@(`0s!l@h(cR?g~%3j!cF@VjeXi2>c*pu#k
zTdZ5%z&@L({MK*bL>1Y8&p{)JQk<9jS~A@af&31pl@mdJKeR<T@zz8Gkl(?y#|)6)
z4c>apC{yvNUU~scTMI4Q8<?TyQ=Vd7_9U`QZHkiQtqHuj9_?n&oTY^p@k(B7o05_2
z2lCql(Oi#ivuCc-LW_B4UTnLPk({?h(el;=*<6o)vuEzoLQ8lhFSUteCiiVo^t?4e
zHP>UZ*)vaRp{2YtFSUhaCa>F~7<p@gZm!34vuEDcpO(u0ywq2dnZ9p}a{R4{`fpE6
zFC$m@<=Z+JhKwhE)<w(r24txCTNf>t-FZ1q!(<YO?z;m@A15#O-N|%+w<Vmb_X&tr
zF2AMS>h@`ia`~-^)^B^Alq*5>nYr&gTzUR4z9B!g{v^8y<E0}Lq;pU7nyI)S|L{ER
zqmnew&x>s=FD3rnkv_rZ4DzT|oBek-qXQGvK@lE$Yl3#}iK*K>7f#g6Ju%(viTCX#
z7d&Q~J@JvgJWssUEpC&cxMbzAZ5k#*Exaf9_xVJ-JLU4!&v;*T+Go*Um2*ap({`@+
zSnjebo2Nd{&!nhw=CxBxg558zRNi@IqW`NC>b(!o?`2##(f^g-!ifQ|PN<hYnY=4F
z<4JGZ|G+o>r$nSG7x(Q6bicPkoU8XwfV<ZUWz{P)g<g4ZmM&?r3U<<Nb$hj3`RnD0
zKVMFD0@1%-p7`^{YvIIqFHd|ZNvW_hwwWtAPnI)S_N2|+mysuKm@LVWDl+T!oZYs@
z*ZI~m#r(@Nc4VHoW3nV$>XRw&&smm#K-N0Aai#Iqi~L?}G(o-9?bA}l-<KxTW=?ie
zetBs^N#==NTU-}Tbk97q*yYm_!|F>j>^kRoI$v5+E2K2@M@Ni%`nt#J`zKoVdc>A2
z(X$E?FaELkhWyfQrkO9h`bu2gcP&zmy*P8-3y<)UW!g%v7kizQe_xnb`{IOMNt(RX
zCVttQ?R{?!7ua|Fy~xYa>*Jy?BDJ!k?~Swjq=n*Ky=R=<jTS1;yfD+}g@;0k3a?cW
z+uooI6<(`JY=0*kdn}wN{o({~$rCoKVrJQ$jeU0<((@A1#eChxeBI5sKK<tTSqt%^
z#{|c?c&HyIu9|*W{^-9IJSj_FJpJIl(SCc6j_qs_sm`i47hC6L^L)8_Q*50#%~RAq
z4=N7kDH@)iFf+Z|N!k4TggNOa&TV#GIB|Zu$Aw~*7nYws7lFJx6?9O+i9VO6dCHFG
zC(ehN|KvH?d@W|lpOqkYoYz%$JU`b7q-Mc$j|;`iFL+$w1}9cclSyA?PFD4$*#1{}
zt*Ci^;*{qndWzGISsFc-nOWF(#v(m!j<WH&iIQ;PIWr4Asy=%yocQ&bM_#eY_R7T<
zeDZGpbKhvMH9h*=!~UfD5URDFK!%}>56HfA-)OI7;1KO7dE%M}Lx!{l^GyxZZOo8u
z&8Tfz&`N7i;}+=%21Mf)x+_hgMEH~R)ytn(zv+E-@{{!C;x5R6E1==17pwlbeV#L)
z)uP4R3cBQq>%DE<r+>(+aMf7mfTyXD200)DIOWle(ZBD8IbdH7|9%_OuE0RZ#Fg49
z<P~M$F{#h2&<ka}6rnfFG_y45ffg4_LDs1{tm|cISQk(TYO8~r=8#Qxh=Jd7#r*k`
z=V#o0?o#=Cjt-xwy4Lz=x&E$Yi&lU7W{G}=49Yz+pq16Z(0gQHD{dhdRf*d@_xWst
zt3zf<T!*X`Iyl7xIt}##=u)H=kP|@6KtsJpL2X-etP5{j_tg5;)#c6?e)_g}lSRm!
z6?^hPUYpgm?$mA2#GA|^X<5j^Tk}?NJN0_82N0hGqaN%6x%x)|vTz~p#~kC<&k}hS
zr&ynYCS_!wHhxlJT=3RW@&sGv;pcOt@vMC(b#4*Sje^CMI4CM~x7GU96_tv%cL`6u
zuA4u9vTN|g(8}L){(vqwTD%xC$VSrK;<IA8!@TZ07sPm8C|JDMCSaZl!xmc&=9>z+
zKjsuKUc{5~T7?1cwBl4J=!ht6R?$=-1aj7gG$%tWXjO?9`VB^qCGPK>?%Amq>ufuJ
zl6l2$XinOG{^X*SFIMfT_4}uDKoMmd#wXAQ1<+kS(7mi8phy8PUPPWhX+3XQ8P)%=
zZE;M%I+cta7XuN88%zLQU_|?DWF4C~e=>VOIThpRn|v*<jWWgOdb^A!tewno!7Pw%
zQ!wf<=q!B!5y)sY7j%JvB>cFB0LY9G!P6Qlf6pnF%iE2TjdoA}rn^fkA3XDey!Gb?
zC>;h@2s8P?7yj&EL0S0Y4K6Ak78h)49H4B(ZWJ~vY4aT8i#=P_4kt}c+^T7I`ee8G
z&+p%#rGDxSI#AV^e97*cu%)?y>$@3><uwQXS?o)YmEX|EA90*LN9JHz-UH@q9~oc&
zX!u)LbI{5zLH2Oy@d8spDS-{%2Xr4NA2Ez8<qJ3bwCQ9cBU^mi#p#J(QYGsCO0Xrg
z3C>7vc_@+i$D&Q5Pq7Wb-go!ncI&F=Hwqr!OIX@zRF&wi|K>%j?A{HV#ml}O`tX%4
zJ^#x$w(|TB-`U#Vm;B&se_sG1P+0Dd3rzUjAD4g#4A#Fw6a9^SuflHT|5U#-rSDbP
z&V`U01NVfybNYAl`TJ}4<L2DA?laGL@jc8deDn4hmM*KAQW@(w>+=1IPOlQ4t$bG+
zye-60w7<H15_m|SAy)d7!@6dchILJ4Te=s=CKWHKUc5l($=iT)j8n=^NB4hUurfMl
z)gQME&yR0^#@u{<s`BgQQ_s`>#%w(w`d^4I?p@pV>a7)5<>LP4l)X!xvCgk1EYc{Q
zeYNoXP+0pQ|8l6>77nWgyB8epoARud^U#DT&ydD^_BdQ%;MKL<^5n`SXK!nt^H+YZ
ze6xLzwf)@{+xPfB*(R_ky<Wh!`@zjz=WiPoP0D}Yc{6|YM@FZH$J^BZ-<Vwg=A`|#
zw@shVpZ&CR)gC98T!WYLYJU$espr`nW8s{!R&h&Nag3eJ^!L&WW}nk`Zdt2%$Sm&4
zCqdWnx5ZcYXEaZr{r^{B&fg7@s?tt*=BK|NF8ZqxnKFCpuf(-=5v%3TGVwF+n0@wJ
z;^x{LIq|vM__N<0j{2>^+w!(}bN#ww?Dg*y4=dhz^M&7f*QWDR>p9G=zh4rVoZWHq
zZq7IT*?$kG{oioYKBrHA_T$6Per&L`12Y$!#zI5y*u+%D%j^ID_MY4RSw5^I_|B=q
zviIWeUvKrUYOVgdO*H@8`Ku4E{tMQB`1(ib&#F351lYICp8G4&9qj&==IOuvcZYX}
zzkPW1eA@Eb8y7==#$44;gap#T`u|_n7yW9V`!CS%f5GMbr(VjJ{c^Ybd)W8C*7Eur
zlJZY`_{|@$p7Sv=y=IEB-JPEGMaSCbehfVKqriCoshRR+&)h9PA3pm>E4}W<O!=p0
z_{~4Bp7S#?y>`lGyE|*v7hP+g`!(>~uY%3{PDRR>U30hWzH{aIVwu=&&x=#%YyTJ6
z_vQg2q2!gli8;Ek|0$0^!Sm&|d+w~c*>1Xckw=8R`R(Vc>en4>FWCGn$2>MS@6X|0
z?u+*3r=PE?TX$>vhWfld{(pBSS3X%*Gd0-m&6NJ4Q|+@qh5G&65PU_#Ys2k1Q}d<k
zY)aS4U$g7mb!hpm=gaEXoqE2ij`Q~OwEr=^{QvGGxAr&I+nBC<Q~Xr^(AjgxgCDP|
zUH5Bx!H1&Fm+epew0L!E|EZr9Sus=3hyE4XcK+<Yol^3DcBWU@gGFyXKLrZ${^iF)
zf5&{)fB4{OT6*1-&xKb?>!yDGcx8vy{MElybLP+fTWo6gqu9T=ehNqwni%B$dslzk
z`6~X<!PDRUt5?kVR(R#@PmSfZHl}E9G!_i$)4Tn7Csg0J#OIIa+&*sEY;f+EZ@EcD
z_U3PeS9tBdK75*X)_zT`j49{q=V^~adG0)w@-(wMbH>)+`PwP>%nvKS?R+_Z^{1Vy
z;tze>dDA}pakKq*ndi%S?dR>PZM*XJ@rv8!9(V0<an^b-iC6x;>-FOw%d=l4eb&2l
z(o@n&L03BRX7rB!`|9Cyr=Pfguc&TU!#CMA+x_J)*foFs_L9k=V@|=67G>oc@0MSe
zcsCv1?lRxe&U1;mN}qE6pZot!=J0MVkK(nQU#Hz5%P82_z~S!Fp{$_td_6y-c-@AQ
z#~jH8NAfI^%w+<V`&LY}xcI=XlO^JEqD80joFjsMMf<X^Z*BX&x#*-+N#(xm$6D7O
z7oYojFeG!$@%f<A%xIZcvaMadgxVC|&AZxGUi$1gw?{(4&__R7y-mXM{)2Xz9LbwS
z8=fYdJ~*v--tB`O&4TStHIo>MZ-cg;9#yRPSrEsyigEV_)2;*O{~X$tFTU>A@~7wj
zFr8U+clFWL#`_C6-WFfB4}V%5x*wM5|36(;-@B#$#>J_%>;5=bJQF|Q_g`aq{f?LM
zPrvY6KVCfN<HNL?sm6A1dislwwa@+-dhW*tW4lw5@_Em~Ek7SV_eUeWZpX~{r)T)B
zKQEs1^I=-uRAc)$Yx;|>wa@+<dhXYT&330E<@2tETfRPg?w3aT#M!@UrYt^fykFtz
zgH!e`>2;289>9}ua!B90b5oe6G|&FGQwupQ8r!{GDf55BWBI2)+NJ+%9k+k;!`<@V
z!(;nTedM=(x_Hi~hnP9a?$pncp0BqJulXPE(Ec6sv;D&v<%YTa&g<mcWA^jytdlAF
zYw_e);fb%WC%itcczqxxUTaOe?=PqPIbnB@>fGv7g~_+9J||dDGMVU`dt#Z{lHmUm
zLaTTsueSAMIeUHWS6q2(LTK)ZRc1><|4Rt1=AC)9?M&8e5vh|`+ume3mu>as>Sf7x
zZriGO_?B3!+oY|Er*BP2&FywlzIbawX6}hCpfS3fw;tQdo<vy}t*w_id95!c+kM?u
z<?FwWPb5OZI7&ZX;pVN0*`R?rsQTT(8SExP>t#Ey$7z@dt$&>TZpzQ3zzmf*YlNU|
z=k>lL+3Q85z_k0Lt>Ij~M_zYKuuN4j=juI@?fz)1`M%4a)XJrV?-kVUd6;hFvE0n=
zpZ;>gr@Xh*ZEno#H_w^RsEAV9)cGYZTAcwB1(%4%y_{t|r!wb%RN!RSKYvAa%4bP_
zv;SJpjTMfdR3_(ddVvKYiGG>(LBF3G&yD3&9zJ+Fb6rc3-Q$B{`%nGk;V2BX3;*hU
z!_NHo^Q@W=#oH!<1;G`<yiE5Em+emd>|yt`sHjvET{!1_#fGozi|m-4P)pzGpiGnQ
z07-)dn?FkmJc1{9V|$PRxBq}0gW|&aosscRYuH+LKub~i_B(cbJ8NZ%ep@~HR(Rs?
z;|afyD}G<^{7%08jy>PbdYPjCR!{zsDv=UvWwyNdjjy?(v((4s7q9xAxs_zT<n#Xj
z$8M~dEA+{)>)3`e<##Q6XK4P_Jr`4V^B%YM@9_S!3+n%G++#oOZ*u1<{l#DZZhaU3
zUt-_4>0Lg*cRT&b-(U29&nfF0|Msq}H~(UO-r(MYZS}eD*Z<u<|9*X6_|Et9fB*hm
zp?zH2_S^c$e@ow;dYJj_U*$gg_1lXo=4`ExeZT(On_FTrQGCDW*Pi|-lvTTvJO0PJ
zyT6OlXIt|hJ8Zbm`scpwHTSh<Ys`7Fd;R<B<!`J_I}Ro9yZvY1_HB=Hv+p^572CCa
zZ4LL^=oR;?>LlMr?`GARb--=j+}-b8v?ENKU9E5Y`CjtLY293%-S<Kc{`z-!{<Q4P
z8$V~*raiZ@v$B=gQhU7V#oh8Bds7eo-Y0sw?7iNR_p<-}&HMA^pFUi2&wl;us=Zg<
zU;6iQbKLaU`rF6F_X(@)E|S^3J^yXnhYXJVovOd>(;lq7SN=6arvKQ-yB7QFXYKQS
z@YuigriyZT?7z=h{{_nOZe+yuKl}Q3-rs*ucPBm${akUqW`9m<bTQ`xq4!ZY`>UC!
z+>cItp~|y2$GJZKuE93B6APDp|F7C{=z_OI)y=lJZFSx^DmlMSTafK2@=JH&{Y|`k
zb3&&Dq;iH%<BfHis>-vsW#RPP7pdBdT9mvr7Uj5332@;A5jVS5L<^e6Chyv>8_Vl@
zMdojd(yK6+yIm_Rg}7X=tnHBMT4AXJX6+IUEbb5iD;EhY-T`KnYA<?rK~p4fyHb|M
zqGuLbAl9p}Ba#nWL8kD!X6)FWpX<71ZHLUStpP5#yD}Db$gph#v!VqpRdN<~Y!kic
zBxq>|VQo_?T64omP<Pjy0GB@L=rZeQ!CY;n;}dj6E^bqb(okZb0%qL`bK!Q~0ybq~
zhZxAvZK4;qD^+PMYF9dRuNp*EX)MZj{j#=0uIt6Zj(wsRods=0E;<X=YJ-FWT=?N~
za$PSjGUmH}dDFO=`yNO!Ob67wUtu7{P#qgZ4V?s)MGUtoxoIS|fQTH|p0yo<T@njB
zU~*1^$=XRdt~~)qIN)-ET@ukCVX&4X2`VCnMK+U|C#{8RLN@Ts+78LC84EjhiW)i#
z8jBb@3l?i9<++{-IKtx!lauV45e*V<Q%ci-%LN>{0}_U50n4>1rSY8Irj*uqwy0nm
zvVpm-YXXjNxvp8;A=(wO5JW@^W@{(4Durnzxuy$Yh{4nc9Lc(AdGx>Ty~O@~fA<Pw
zs5@$4DzdR?$t>rqGOLeee6Vc3Y8=)3@Kv(ql(QEfO+I{7{aANDdm*31d>KPK!;~5?
zq30_;o=ki!{P@^%wn9Fc`7)Mvh9xyoLeF1(Jo)gk@Z;m^S3W;s_;gb6iSXWE#*-}E
zC;Ia#&zEqwGwvu>`1!=+)6Bwh)sLrcYv;Eg(rG`xa)a?6`^UdSZrDq__<z0aO?~)}
z-G}ar{<GGe^0`xNvgiB%=k%@F-gz&JzgIA4P1*c_`oCOP_3e)f@7Y)#S;fBmp!3fD
zDtn*D+k;=5{}lKgf9Y&o{t}7*AC0Si_h<d#|L`+>ZNUH2{y%m*{W%Y!LoQV6`cDfx
z((z~V|8GA2I{){_-`ZdFop1TQzysFuU(cm~eX{d+<;3sP&-D4%Ex&T_q*Axw89Ax9
z6aCLrUi~{&c>a@#i-W@JJohQvR-bEllinZe=Avc)@xTwhn&X$n9xpjN<LHcc->p7o
z?kk_Y`E2Ri`kI1*nTI#le2|v<TYY!_4;M@Sd*<^yX7?o~`1np(YFqYe@1gURMSUFi
zBlj-J{>;N<{_tOZ{quc=0grk2$L*i{|6U}sbiG!*|GVUjoy&!<&0qCb;hNru>9-~R
zyiNFW`NZyDY2|;#BnxsEZ=K8j|F{0dko>gN-{xt%{$-2)nd@_-);Zk$<NGp!>ZCst
zYP0qntLO>;Z7(nWf1kj6X_4m>TelW(e^+$cb$0Z>g7R4XzD=+87s^dP;P|&~YD`1x
zgdgl*d%rG!B{G5UO1aJRx&Eg<J<>by_urq-$DYmYE<gKg+WnWU(RB_N)bm6HtWT|)
zoNjik<ma<6r#%bpw!dKwvMI{Cpfx4xikW@urkCH9wod816%-&^BdQSfbUD|?S0_Uo
zM2`iqc1&}6ZElhyoFJSa9M|09+u+;aTfxg@##qMqC_;bEw$IGFpPh_1$bR$mSF^3y
zZh?ts4HX_<c-$t?&cLoB137<rLIDS8-$TCBA;_{}$bIvmb1G_-IoJQ3HScfRhm&hO
z57=r=zj2hiXZz#lkA>u>6gu5z_KaV?PP!nDKW;+%z1NKP{ByWj45}7?ek*VOVDF2*
z1)cK74u_}it(^xtm89+6y?uVE9B=vRgI?;mK6vZgFs-V3PPy9cKk>W%{VeDFS9jn(
z|Bv5{_4W_mv;X<quwVW|{X53@^*?2Q^#A+$^Rjh)_@A&poB#h!eg5KKyH-x>oBU~i
z-+s7tb8?uNagp_bmp5ld?*G5r-o1wZbuE8SZT;(C|0-(I8|%z(e*C>ZU*mMV)4u;3
z4oS;wQs)1*sq9SfUsKD!*&#o~of+=Gbk+^qv$5yx{z}`Iym@!}bp>DCD7*4ludnyp
zv68K_`>vL)2)i8^nd_dgHFseyd)QW4=dad|cRx-p4Ss*%H^={ei{BmV`*-~6cz3?G
zPN~lB(KF?$`r~|CW!0|Kb!VK}*IEB-**n#e&tm4e_qWtu{^onz{O$Q0Q;(_1%`VHi
zwXyNIn%wM)mNUu0{l2#5AMYF~7C!H5WB!q6=JRFqW>@7s+Sz%1`QLT#a_7vyuD8~A
zliA66%6W$Rcay%~tBri*-)pb?&t=6c<45QJ=q|4_PrPw%UdXw5@Ba$-Z>;D!nye!y
z9P`$;^n2=t#~&s7kMH>J-1tv^Z^T}?ygIY$RPLa4MYsPmZ5Gt~)2^lcWvk?ujC;ZP
z`I`UEUkv-%9(wKSKFL7ljh~iJI&1bXAV2^4rBDCsc6Lqwz-(PNRp&(c(x2?%DRq(@
z$*<?PzvWH(_hZNZIgfsycDVoWZQ@ef`0wRaZ@;d4%-g{4S6_eq`R_^Y5nt+#f8X-h
zTJO!zdGB95uef=>?B~zE_mPLnnX;elmSsHm)_i-t%DMk<lJ?IvyxxB{zCHfe(PyzI
z_V-8BbMWoh|L2bBztUZA>crWu*Itr-QzB(v_xZk(<I$qK|Laq??*ExS_v)up?k`@~
zZvSR?_x;}OuW!6}o$_~Q>AmYo`aW;gtWociJ*E9SSN7F?>y2wCZ@+W6<@2F$fq|6|
zZ-4u|=v~d<yPMZpFaOQodOP?T_y50Z;_Eipw=DjDw9WAUYSB0Hp%&{O{0g7<;eRhz
z*uT^LKmJ#+{@1rvy!86emS6R+INwMA{`KH5-~Z=Ff3yqE+|Cx>t(5=tZrs|%f7Zpk
zYFqtgMTqAG?L3K&z<YvQr$2h%F!{edzeVTz={@E*qn~e^E3oNXhE`qWW8X*L-kab1
zUmu+Nzc~N6{hANI4}9QhuHpCobIjpc{eNRc_5IvGWE21HxqZ5zXIuQ0=tGB8odokZ
zPk&j}VfrTe*YS(0G3SnE>4_EZoOarPTXf@u)1@&wjobp=JLa3lu$^AfaY4%CIa7(w
zUoL(}Rr{y*lI~3!bN3(i__x<U$mPKwS=UecE)xy-T)>+llzO-is@EUyZ+LLON8Zpv
zzWKZ14*A9Z4J{O!*^S)pU1XP>GeLA$q{3x>N#<J*I(gQec+t&s?!<#$o_8l+^sl*e
z)H!2^%3^;b3)N%^r8|=@>PyN@K4>f%GkL53d!~#E_2z1$U+T@xMindKAMw`j{Nrqq
zwEn;6sruKS_&4mceiPfG|9gJRpZh9u;k)YlKYTZT>d#SMAAW9bQ2xiao8F&~b$vJ;
zbkoA?#fuD9M?D1zF|9eYT6QXH1Zza=^_K5tH@~Mdnmsh-xWRIR<wnbOg$MQN-Q|WC
zk2<$si1jHkyr}x6;exXhKc}q<h|%EO;9SA%#19!Z|3ClYh5y!%zVBC`doRBH?0@~Z
zi|@C;wEuNKK5x~l`W=V%OU~HfzG`24jiCSQulDV~>lZcU^X{3*_5Rd_INLkgm#e@2
z`2O(Uy@jvmI{x%7`0cO##&VV|Pw(5Qy5CH?=hi0Y-?<U{O~vP(u3x=&_56+ZzttI3
zZ=Pr?yM4Rohy8s8|J$|p7?)R<e$aEtj{T}0CjI_Zrr!7CN!4qvy{lK&Qkwo^_B8Y4
zTD$P)=7l;L$Go2SuSt0JC}oZHaiRBWeXS2{Z=7?zKR-gL@9GQtZkCgxU;hg)R1oC-
zAl}c~xuYTCM1SbN`4Kz%6Iwgee(aYNEn?z6^ux&Sf%^5o#zKmA-|IFTIG)yM-QcL5
z`Tr+Vi)@s{qE^+Y8H-vqqi!r}HH`9o+F~0ev82^CYQ~aQ&!`(qS_461u~8DsT2rG=
z7j)Lhf1cZOd`pPmjpbW;+-~$QY&URO^~cyh>G6K+Bd3qaE0z7*+kWGZvqZ;Q>E8*N
zhsE4-5|4_d<s=^8b}4n~d9LMCr@cu$DThkToAzefIsHGm5u2iqr5#B=G|5)y=H*GY
zIzQz(=j{GlIcweW*FN`Wo-_8jez?Te=eoDm{R?gFFRR=xx4x{Jv929T%vyKdK6mZ^
z4Od_7)(a@#;8i_czJB|ozpA_TJAS|ZyvFwRzXx~b&w9W7^Y8cPCaitmfAzm(!lU|o
z67fIN|LrzTG2WK8*iG|M=A-+i|L>mM^?Cc}^$+9s|KGRF{Cg#X@tgd-&zJYh{^$E+
z-JEc*`}6x<|M$85`uqKPjKsgM_CL3?IPMgGe*c&H|F*VyW%}p#|MmYT>uB)p=#l;2
z|8F(d=L)~EzhD2?{~xd9xB2h)e}ysNBA?~|zi!;{zvlnj`cL@^AjQw;|B*iS{B_)u
z{ZD^Ru6JJh`~3I#<NI0vUsG7|UFqN7?e|vyT~TqVZu_^{TjN*#ReKwrvES<V+?)ER
zt>2g4`!Zc6?)%?w@!L1<J$8Qo1G8Io&9}Tan@@kM?^*Bp%llHikn4-R!d}%{nQjgD
zZhWlNKgoBg`@<gHGklk%^?Q@Ploj0LJ3U+f<efujrmHo5j+^vz(wXTg$DhA#*elEW
zL+{X=r_=X-*fc$+>{QL0{oihXV}Ho^=<faXJKj#e{cUy5@4Z|9PP<j7n!CTDx>H5W
zJt<N}%;lL}#8D}aHFZaJxv#4@66L<8=*X{QcbHd2T<SR56LIVWYfr?k4%hC8brY8U
z+57eQ{fQF)ZG@7|y<Y#<5K^A~cAe$_?_YmzKV)~}wbX<ENg8e2^KV>}?*7<&?uNYL
z_Alic$MpWmc6}_@ICg)bM16bGia*mij#eD9Zdts~)%oXft5+N9Tq~OvG%Q#YbU668
zGs~p?mqT_O*nF>2L*V1Z_g{qd>|C1Hyj}TYkF(sKR%PG#+SBi6KKY)+-(QlrqoMg{
zrFr{)g=Se^9-j4&?OyTde@cCIH}~hXn=1S7Tb$UuX~nC*|NiVdvz=>-;leNF{7-&=
zVHKNJJGDmcOJ1rD6NA3V5hrUMv*sJ(-??=fDnIT!-aCEz@(+tXJ><`Q`16L-(aRZ|
zY@(`t3fmts`JdM{58rqvN&N2Z{debo?NJlF?T{*H7W(ec>#AGUmi6^w-xy+DQ@6&7
zmx-4-KUF)h@RW0Z2JbBaY3F|F2OKv13qx#g2DF%UZF%_U#Md26&W0yrzwn69Yfcwm
zxY+2!g*&b{Sf^?4dbC1kXZMEg4YE^tzp);DA-L|5zo>MDp!OEVd8-5N{ht?8{g0oC
zA<2B_cj<5IC2KzX-|%SnzK^#*tN;Jqy>`;S{_r2~IjcK*CChaV>^9hN-R8g0qrca+
z_neNX7A?^aITqu5CgDg^v`f{#Ko;l!=VD*ZUDvU6>i%UJ|G#qmHn{WqbKI5AOLfQp
z*)I6TKk4GX+OYeFXMbz7{#`Poc9Q3^jQt0tUpMS&{$DQia_tYj`d{(>+yAfRdjIQ{
z^<0Z@FDtT_%k6)z{$|>zy$w(Pf4qFJxa|Mu^q9Z(n-9h9zN<aqz5RZcHGhQGfBNjt
z9QIFQ{io08sZ7hWuK9DgV(R>Q#e}Ew$5#LO!_BDu-yrVk^LUPkpT}1J5xsYb$M&cA
zvOA*FKlabR{Jz+z=xKlGvv}86{~vNmuKp}<cjS?8kd5!2;6NVjS*@%7e9q|jl9w1?
zShI)8<BxERP~VBeZ&N}%*{1&aV;!jaFe%$@Y0K)*v%dWNuwwGNZK)3u45kKF%JH4c
zwzF9fCG^bs@lh`Ri5DI0B^xGBw7<V%f%M|Zfwqkjb&sCk=V1TU!uYr5af8t(4d*^1
zi<6rlpOky)=AZT?z-anJd(Rn$Ue{CB)IZ)La6k3o$+iCrpU>U@X1Bn<fBWr!y!qd%
zlek3k)OY<KXTJH@rd>-^d9eS?B~GKB?a%G?XZ<Wa_3Zzw>s;GTf7ad~_D|uhe_L$b
z>c5Vcg!_K7KRZ>g6MHQ0-$FmO;(x)aDNCL|FEm`RiHC99y4a1s+9kUir+hVeFT;E!
zfva(%Sh8mK!705x+xo6`e++4Js@sxo*tJQ*EwX2^QOu{~%cS1+?>><F;8pmgU&ph)
z_D8qgly_#?KXIM?{ebKCrv3K2CpvEYzvW#0j|uyor~Km!{8g;IQ8x5@yH?n9ef$5l
z=Puqq==VQp!{zD&7wse>5Bk0SUiSWeZhGBamiP7rN&l-_@8AFSIJ5S>XL_a6$Bidq
zU-#eM`B(GO?i-J8bN{!h{SoXp;oI%_lr4v>1HKB!Y*v@~RuefVuiyCRpZ5orZ@#kZ
zk^H1P|DRmD&lph8?(la$f5We%%l`8<c{S~i5x8Q1-%3I7`*znUb>B>$>1~_&d(*yP
z)1A(*&m`Z8yJo!YLG8)tYu2jPzfF%+q#d6>XN$t=mbTdU->2u!t@5|{C!3dY&`HKX
zHZ=Mh!~1zF<!le|v2A;hbbjlVHNTE5cTbD|{=~M5soQeC#=G+i9z1<6%fxE@&1HH*
z^*QzK_tRIi>)YE_v)?}Lw$e$o{<&6J(cit#g!i_*&D^20Z+^|Zn44#E1?s;i%QbG=
zw*1bfA9e0kn^OHtB==umYd-1c+zY24+12VtzmlDh8a!3_#nyD*%{vYsYMN`lPBZ%T
z7bUUgM<GF<Uj*dHd(G#4=3k<|P5u0)9nSqtGbhS=zgZl;dGEaF8vEzjAB^U`NQ~RL
zsClF9&+@K)?7jIPds$CCt4inF{HM<3`jw0CbGANEbt?+oe(%$7)q6qi?<-7ZRPt{>
zBUYX6viVP)@|K)U%ia7_)@wy=oAFzB|7UjH8$bU(%{u7&X35OZ{GW3>-yb}C=G@EN
zSr*qXC4DnI{r-A6f7&zQy;puGmjsFZ)=d97_tA@kr+2q|ZMd{}^CpWvz0>b|&T)j*
z)qnFa&f2~sUDoZ@i}HPLA3p8=p?r45y+`M*C)-Y1e_yiOqhoub+pWNzN9UJsN=cdg
z_uWkQ4Z+684k^jL`myl)W2+yLHxItepE$Ga$@U+g*Lj>tc>Y(y?wp2|Rri@qk9%MJ
ztGoB>%ZpF<mUYXWSJHo{@uqxVn8D}WYqW1`mzUlvJ{0+9asS`?T7$E&zViA{4cFWE
za9hj8pN%XL{a#)?ZFQIL(Y{m8Pgg!-;$V3E`{x~oWsH}#I2p1Jm_E1#+R1ytFHv(X
z^DV}=4rN|%8p<53CxF-aH$b;vFzjLgulMHy@4C>wEpY*~v@6^p+#wvY{)qv4wjwih
zVU+9z*%N;+U)sNiQ+7?shD!%e`m<|8S?3wFnb+`aFyi85uw}4quwet!jW&&cj2Uei
zZ6WLapGfmP@dRzdSODE#(g5C{adCIe9d6lM&Kn>9{&R0pccqs8t@e(%XA5Uns_AR&
zyLj^E@sqWCWbWSSzj*QH@r$)KvaxsjFW!`#JNM%4ne`%$?dc^IZvE;10$JwHyZBOa
z@4TPxf+aO>$J3uaerMmc;Ipxn{o?PNKLxVLF1W}jWxL>NztRhqwRKjqp91W=7ATuo
zIW1P-@+**K?!t?bQhOI}?N@ps(93u6h2Yv+5y#~jC0%aoGyX2-uyR|hY*OX)dxz6v
zeG@C^#pYZ3txbM8FLpMma{0Z(WwF1Bm8<l8#{=$yU%K6nXME}TR@m!yKcl42?R|#)
z_lz(7X7<j4pFR{$te07FG=0|bi&nqyM;v&+u_yocdF$wn`iqb4{v??F=e>OYYIBuG
ze}zR<v;IxlBN=*sMaxEYzi;!m{=cHI#xa8VrL4@G(+?KA&J`<-y0Y`k|4-kapU>MO
zVo;HFFqk_n%HZ>5=0)r-{9W^`IMb>;E;esaf5;)`exZMXyrVr+A<JKmLlaw!Ino8r
zDa>)SxV>NfB~Oc_%sGb$#tlpfsi)UG$Yedlbx5>D+akts|8m9e_mg@3XC0pW?ZXmB
z8_7Q(mwdHR{(tVzY^C?EXD6Os@;l$|flGgcP?S*Bwa-Ow3^{E$Z3JyOFH7n2<{v+q
zbJ|vE$=pa=wYj3NSI*#HwX%7OL-upV%$8|xsc-j)s57WV`g?COSt@&eoBRZ~)XdOn
zX{nix)3zmN+E{0${h#}@!9>BEdD9jJZ{bZ_6ulR3*uos`-ZEA6vWQOZ!Ao2^XAfNB
zv$=hL^H1ez1^*|c&3gOp=DT@d;_SQQYb$<<p8s}V_}cG$JB8cz?jrl@ZRH<4XPn3H
zarna)W`FA&f%D(r-@@*{|4F8}{XD&sm(?q89qoH@r~Y%D?f1`(nf&(i*dE^eS9h!4
zJi#RS=6^Sd2RHuzb4lO4XzKhQ!Ya|CsqDY@*DqS+>bv*7<D*U?133#h2RZ%^EMF>|
zI1dT6sA_mQOjFQtjF_~6g<bVU_J!*XYaHX77qPkWHp$-q-z6)e8~In*|J(9;YgzMk
zE3z3cGA&|t>UaPC;N^1mU|WuAfo};L+5d9P`=l!!_&K)z&h)_2l&cL=jHb+4Y!7vA
z&TzD7F6gWGza-focknV-iztV-Kv=;JrgYYG=e=%MT&kOAvOs<A@!(JSQCB86uJFIq
zUfstr^T%xd4JtOvW?W0pES;9Nz5UBaXQ}MIg*s=mkK4c7|MP%}iMRLd$IpK#3cjjj
zpJc$Rv!~eeeat26=`(is$$w(Z%$@d3EN<75>3T1hdRM)i^xONL{3k|RX!M1pX8K?2
z7rMNp%;m|y+5BG8+~svKl?UrVR;6c_Pisrhtemz?Ja6BU+4?V^-Df{DMf2>1XN9sq
z7tG}Ma^AefBzn2z<vluYwg1$6{*S-(fBKjIs(<S{=c?XkuZWtlytOiF#`4CUvgK1-
zG85*piCuj2yIp0Av&HB9q5|1CyA%BE$6v~>pPqOA^4rQ^w*IHT6vaHNbeOeld1h6W
zqrdr!%$4(|y?V)Et84ac`W^cvIi~V6mc7p06<1%%?{D{@>O_(DoBg*#4kf($xBahJ
z_R07A?Ui+2cG_FUbX19nu3f(I=zPhN=*6a06QpcCdgYcJ>)7Y!9<*3_<;96t_R2_^
z|2O?!_Mh$EJ*mnoFB-~v?)?+qp<3YmVA=uQ&S<9XM~bs-mK{sZvRSrl+9N|#xy{?X
z>hvF5>T*O2Y+I1A^2VA;YrZ|dKWWQzsr~AuHoMrX_}{r-Ihh!{t5LA0uVrpvy4zgI
z(zGix4?gT!es)2`!RP0>4v8LGvto0Ylt<Kw$!edLa93UGTV%Y#(rx`s%`0z2vdXwj
zty^a~f0f$U@|*GaflG&UbkAJcrt|j1r3pH`C;L((X0<KNjhNNE)HXt@WvOdKRLj)Q
z4O`e&KR5N5c3DA3_s}IJ9bYA@iGLeXEPIb!@>(Rm@@DsyH?CRVluG@3jxb+pS`@zW
z=ISeNLbJYUnO0Aowb$!xe^ZqmS5}$6sXbTLH$Btp>D!C!40vDFX#81`ZTbHx^YM>N
zKMzdg_gUS2?`f^Wtocj-MoJ}oF*2>5Icu-a*|=qC`I}_Dg_g*=tq<A^Gkl4_+x<To
z{Dr@o1o{hq-QwZj{pCy9ylF2pXWpEB<&ACDH#yVl{#kq7&)!??=5qGmY`Mezr@e!}
zKK^_u<o)dVVy`nd&a2a@o;+)>$Ju*})7}Se{=VX7^_4%bR@|)I!|xIN)g;tE_-mDg
z#Xe2>3DaJ_eWBp{nUnj*|6l*Mzo|cL6!qty{Mp%m4qndPB-+o{BG1jkE#Q4RJG0@V
z`m&2U9JZXcGr67kzwJ>=|ESJjD>r#_;YU~Dou`#2O?OYx@4t~(@AB(Y$DgP5?&pLm
zR6lq<nA-4rWn(z=dbT*<wQPyJhonSwv;J*aaXIMg!C>w#F%jJbQ3Bs8zWMWa$(dK*
zkvVv*qy718yGIL^<$f7ITB*$8E^uzajE^5Aea<eJ@3`Ommi)w>HcF+HN8)}f7fp9p
z(eFPIAvbZ;AE9+7y$UgoJNlL`$+c1azigt7r%Y|&>CDB`F1zXOUNS8<=(L`uy5aZx
zML7oZE8d*@)OehspLH2$)U5?eKmLt?#LR-FtWmQTEPWd(HGk>XzY$XNmOhP~HE*eI
zq}1f4rjb#TuYPW*id#?<Fl)+EU)T7no4T`Zx^8{5=v&)Q&l~5oXUHt+jI`$W;H$D*
zA<uW|&7%CQo90{pe9F9Oy7kShs4`#O-OI!#zx{YkJzH7c;#1@HXsiEEnJ>rG>+oK?
zl(aZM>*v$Uo3gV&s=K1fT&<mTI4_5tug=ZfauH-pR9T?zrHt>)ml||+Z(J_1^}Tl4
z>96JY%gb+;X1%<6$;4Lo@?|3gQU7PxQ5U<uj}^xrOZ~rVSzv$eOU31q<@40uKJA!b
z_BQiW-JXmpcmK)XOfvl^f7|jve*N+3>WQ;9E!V96<(|KMs?diSAhz{hN4t#KJ~f8_
zPk(Bk`}Obp8~=}4o#Xml{cZk>qg88)zq~m5>ud~<5w{h$6Ms{!%bDt#F;`Rc{;{lL
zujO1ZsYP`uxNhL^7MP~O^FZi8_5R-S_H7H+JH|KfW2@!;CH19LEzxEHtGuq<<wu4H
zqWXXDccXci9~s)pU49HBHh+Iuq5s&@R?fJofF;qdVTrXw;L~)LV?C>kSDiaB<Kvc@
zXC*?zvPxa&it?<fc*WHv${+7p!Sf*Ez+|-#75{vrD=taQ+tRlD`F)e_^m*)0gfB09
zb7RU==Kez2*H+I`|K3Qj`xp^5E9y&_?ehCo-=b{AU8~B1Y$q*z)UM()d2`VkYtBoz
zbk3eVRp}5lXX(kvTXU9fjGQ%dX=7y63|ZmLTMWIGH*YcW?%rgg>K(nwM8aCE*tTp|
z-_o}cQvFN6zSv(AAa&gT3$Lwxc-1#S+ut4YCMVkN?wA+r+|SGTjXBItA@{_k4xP8h
zF5B6>Jq#iqAHVEi!+Yei+`qSy1^&#ROl15Ie>9QV&OBkx(!GBwq<WSg{81Qm+*|g4
ziA>0?-seK!`<Fkgkm{TM_~|FcOow@FPcj|mef#_v#1y%2`@fIx(&@^mdDkB&2wU3D
zU9MUcwMgFk%NCb>c1!sgeCKEXn)pMSzb3nXZ{y_~(RF|S<@=mHre4w$_Um8yoBFo_
zPk8d`vgRlH+Ni&(urrl=?fBvG$1OsoURN|ZWlOzmwzyn4c<tqhz62L{gGI^`7nyrr
zFuHwVP_Akaysw&^|8-|7=dWp#KrOkdke7i=*<E~cdp|r^{`0fr{UpJ>+Lmbf{ZDJk
z{A~U_(a2x&sHuSE5yvk{31g1*D}QDNmb%RqmC!Z_3yE#q$dttDxMpI{hfm6T|D0Gl
zX`)}<lF9NfpSWB7EVT1<tNJ-HuI`DtXWH|=Gv)W{KR0Hw{q&q9dEj@xoq?~?oIf7w
z_Lo-nm%Mt<Y{Plk?$FNul6T5;f1TJmZQ{AwCB^sIX7YP%?)2}Qc&_Hj^kW{&?TV)5
zJehQ=Lu1i~r!HBZ7y5eD9lf>f_gVYCJbr10&f1fg=IFdVd1>CE-~A=Ol;{4l$ktJ|
zH@sJ5r?J-O^yBoOPkc(|rulSVu3y=0vg^koiJz^8o1`Uwwi=$3I{jn1?DeY$ZDf|H
zMn@Itoa%Yg?)J@D`S-$!e=nTypIRuE3F7V#b@N-@`(`?boD(K(xz_C5mRl$IvQykm
zi>h){>`gyKr=`@JZu%`*m~?Uf(q@xd|I7bBZmG3@XOMgU^0&VgF8+sLV&^w#pD#af
zIkRdX_q=WIGk;x_@4YPF`=;OR-2UD-ac*V$%DbmeynFh@yVNJwf_}Hx1niT(IJf7G
zn_HQ(^~z_aTmQcIKazLkR2)y_A))Y)Q@eJ45)a;DpFV$z@sUn@<9{p*j~Cl&>`{*h
z`MTm&;8qql&NeTHsR}xcUxFVos=n~PFddrn3wn?Lxl-ZR`}|6UPw#Osv8BStd+(MC
z2ku{DUvytYU)a7N-|_zB4eZRep;BLLUsPZCzTmy%{bswQt!X7epSSa@I%#@+j*R@r
z88Xq!&fjnk>yxa#+Vg&r<b74+s&$gK*Lx(lN*ZSwr(a1sv*PTF<7r=#jel*F{ClIP
zF2~K@EcyR_!^hdFZuVxI<!9gQKl|o*+P7q5d%tIqS7+YLKl^5T+PCPg|K0zx)Bc-o
z{y%H^|5-Qd&%Rlo_AT7l-Yczc%9%I&)4s)j{t|Eed;QG6*UvniKlAU6GyPL${=IqT
zU-rCz+0U+@JX4qR%>Cq<e>u-YZk?&~UG(?#=38OsS5J4y-2Kzk_;Yqzn!Cm3$Fq0-
zG2YBP=jO+=bCPDKd*!CFn}60$dv@jKnYy>S&zs+W`hU?eYVogipW}rf)s^7C(^eZy
z!o91vZV8|5&TcGtxkNYj`lX7+`)A#ZKl>+V*3H<nC${#;2ep*WiKyCWlISno_bxN-
zyUAmZeLoaAFQtI0jucznYnPL3b<OPC)5?sE?VHoe%#C-?o;lyqxO(!;c<aw=UY+^V
zHS=cmnK!m+--L{<n`hn^Gq!Gh!*DHM@RH!=*|q<rnJ0>#{dq+GcD>SvGS~GR7KKNA
z4Bfa$I^yB~@EZ3uFWACze*B7KYhrI<cjN34l2E<L^{eAX>_yo%Gn%3<O0Rjr8}@}w
zyQ*1q@4;KY&%Hlg;Rj6#;N)<n!l$Nv%?rM;GcIdqbVgm&T=POCtb|M3s&#9>&yDFH
z=g9TH{5S(a%$Ga=5-KRzFZpqXVE;*rIrC!-&z^ia@vM0N$?t_nlGAL`mKCMhq#r9z
z3plf4*P>Tl)!K5qf{uy$#|D1gx8DEO(aB;Tzx<i)_t5ihl++~I%a+@cEr0ncbJg#=
z)W81A<JUT!Yhr{VcdBf<G=Ke<=h}b2d+py_6!fyzVb+AH!G?mDyL8?@zd!HfHWQ20
z&!2jnyL3Zmt?u9NUjO%As$U;t9Qo66^QY#trSb7EuV0V67XR`_xYf6*@&?-W31Pp!
zLm1b7HblOBasU6M$y+uFKmV%#+QIvK`b?G7ee$Nui!I6~e9Oq3FpVuUbK*tDHE$Tg
z${4h*8%nJjrpg{T#g~xEzF`w{#7(Hod*(}%-p=?m`R%-AGJN$<%TIXO@Bd_-GS_tK
zRKB{WQm4Gw_ZwMMEUk}!c|ZKu*V1WU!>xWyozJgrFCS+0XDVO))AS>!&Uc93Rye}{
zv%w_QKlt01jyfMJ-7A-!lIP#}>wId~vR9d_=FPkKy7KPl#>|a7RW@z%j?Cn?w*H^{
z;P3kP^{4-tGg;Rq@vePe|2Vt+U48fj)zS&k7v6WS`MzmhZ{^{ZNB$F{FH~QszVLqO
zgzrW#Y&mQX*uJm@w-t081DQ6mB!U|c4c-^7r)*_&<?WIZG1bTlxUwLl@rKYw)+Cgo
zqPZ@;bI<gH4*kc!RQUMk9#5{aS$00;9OuH0=e-s8I*z9r)U#Emh4|?;yjvS_?{vjI
zkoXy%Ly|4V8fgM=c5eP|v}&D0OyiEG0@jCIN2YdYNB{KvU9l<g|6R^Ee?vFPPuTWl
ziJ`6S|7D@Jnf;j)E>^ykxB369xi3&|^_H3ohUd4Ozu}*MyQJ&E@nfGGGnecxzFq&h
zF>~F<DVyp~urECc&b9AsZMXkdON!dLbY`^Fj-{>9vwqwwQoeV%CP3=%yRYBw*91iE
zU-~opR(*@dd-<T28g`xdrn}srj%wMxA12JF#SeW7-B2U06VG;f`oT}R8){_N%x8bH
zI^m~Ygw+0}U!!O3Uo>ZObMmig+t#n#^+e@t{C`z;WBGzO#@Wa7RTKB|Ov`V6`a0pK
zQp7&avbfg2S2xvL%zM9M-<O@w?|KA(H!1ZG?!UU=uXELbEv4HZ+kAG+d^Jz)U76ej
zvv-+~=b61b&MCjW`p>(<J;$fm`rf##v{%M{E=%Ubd15a!JLj=U2%h`huHtd}^A;O_
zFwy+EOm2eM^S2h0F1tQyzVztct8Lbs_uhY8vpxG({i}c5-`4;BXTB`wF|!4`x#QmO
zPxnt(G#q8tnV0a9qdPs~-0yQLJ<lryssyS6Dgr9PEFUIuMlx+;N##5xG(lA^fN9Ax
zo5J_YkDZnBUw_cQd-=AC4DPOJS{zM#mF#tYX>{9ZeOpoUpCyq$uj<10h8w~O6CT9>
zwbnWLc;U=mgQk^p+D|w>>se)-b#BE@<J4uY*G1S>&hCA9`JZr$c0{z}{vcQF+doW-
zZk?FC_D_nj{mpa!W?v?Eb*+0AFK)3*rThKL3J0a-wKq4ePfj>~sIOC2+FQ9h^zVQ9
z>i^r`?dVI+Zunj$7SHkE=im4>&B=eZtX*?zmFU&2<wxGli+{E$>6Vg5-~$DQM+@`~
zt{+IhH>+u_<_!qJ<UDI%L#}XBi!ghP<c8z}eT??E%zfhxTq=UNW~~!w@!ZSW%A2DU
zurNY0$#ykY(}%(XCm7YM*||gv*V?yn6jyU|ZCj84A~q<gF&Wex_{eygJw|TB&h7Iy
zFfDuC;9##Mz<ZC;^vjPj<HgKB%=wtBvv0;1D6f&(kb7YJJ<q1ODmOeIy-_*nC-g?;
z+^Xl=2iU#Ub><d)m|Bo<sF9OhO=bp|@%Sm@jv8f|mIr%mCX^fSpF7Q5SLw#^Ty@TL
zcK(NZT0U=*;b~Rqd2S{Bi^-`V=1?K$aaEq_1~G>Ut}Xt;+VWsW%VVLw(^V6GJ&<LX
zRuHqT`Mp5PV?I8G;1|YUSFY9ADykV$rxo(=`f`8X+-BGPW!d?zFV|}96<xZ0(U-6-
z^&gJ%Ugllnx%B&jFLG1vHD8)F#YpLu)8YGDE5-RY?l@c|cTrZ1f8&n(kLtRub>3~*
zRhxR?DvMU}leVk8tJ!1j?%P$H+%=2S#60KBy4&X4^YiXMdvBXr?0#3a^!w$?3-A8z
z`u%#3-}mcx<IC&srm#JhUFIwy_fTlZvPW%442{Anee}eGv@RIg_GKDfPFZ%+XY$PE
zwq?e?=`+t5%}g~qoigpD&g6;S-%3*cgVBBqk!yP%o|S&KC-Y6M@lF28e+>e!PI_x_
zlW#JZ_^e#ZxR{-VxvBkgVr_o=$+8(oX1wBVn<kKA$?{q;Lf~K*$F!EiU0dEsBw4-|
zXxdPD;5p;sYHOdSkgu%HEiakZ@$ayE6?s!&O}bDd<3S;gYYrPYnnDHEFoDhTJNZao
zLaAFJ;a$U8=6!rWtbR!{+Q=G|9=Lk3Uv+}up2uD<3uKt`@9!!;;c|HUwbE~=KFhZ}
zc&Yu2(Td~Xb;fx19@&Dj1GgCSZ_E3~&v<lvx%8nAl?PsI<Xdux{a(V!B!iPl3$#B}
zIZNF0kFZbttNrlr9)2%pfqVWHc8Nb*o(fOfFEr1;!jAvW;m@4s|MS=Es+_Vpc!S%g
z9LY+zg0kMtxpQipO((6HsFVJ@Q6al!uJzlO%e7e|l}}1}UUS=IHfig_18TL5jMrSQ
zZEn81-<m!B(u<W-%^Blg2H32;yF25%-mUFZCpe0_epCPU;kI~ptyR<Q`dyhj*51qA
z$Q8N9>D~QW!<pK?>b~l}YB0#o>#Od&e1?zV%p{|uDP1S09J@Tz&u}h?vr0NF?SSqo
z=@aTTR^f+B*0t3Zc0Ut48*6Hwvuv%z&6grPmo+869o4nE{6_3<Z0Wp>yN^HocKB_(
zv{0Ex2YX$FOm@6Xl#jWK8GF&P+y3RIm0L~b<n6dqu=8Qj;iqk<W%bMVNW5yY<SSpm
zcjv=V-sC$Qj_~^5G5J0(@`f>Ybz0(iofSQ;Yd3CC72D=wsKEPRNyAj8IIbJ&59T)%
zGMzif5qaYiS9DzBY3-cKR^9ar7{$u=IE74_F<)BC{DAqzbE_H>9`8G>CO_Xn>VN^`
zaTXon4?bTu?@PGQkk8civt?P)V+-w%cgpuDoKxOYzVx#^Z{oBW>AaT($~sHc1p7E6
z)E>;dZ{B`5m2r0EbFDWqt#el|5S*0fIGJO4<#MIFhg3kQX<d7Rykero`~|+B41B=o
z1(&0R;lZsvpQRE{+FU5x@K*l(LC1s4&Ft*_0&*%XQ$IT-%K5Y|7oK*mC_%JoL)oEQ
zoZ8P5Q{Vfv&h4z>oooX|yPOutF*T%YJHPq$uUm)wE~Z~iznp&kx#*0ghsE!U?=!4f
zo>pD|xSKz9Z+FOhrK`I=zUl?-&A%ev`Kar?&c5J@u9tU;E-Tjdc^-B{JZeY(E3vD*
zQ>t>U>JP2Cr!>bdG5*emU5Um0y>im$-&W6h|8n2*-}Bzt?b`j~ozp*w-~Qh(-YwYi
zu;B2+w!^%f9@CwFt_j<@s=Vvf+(SP_%4Ql&c9fWFGAGaCZo#gHC5K<Oo#vhXJKiKF
zZ^xaaokt%r{C)QJp2t4(^*`6Ob5>j~e|}24zL=etKUwBn$6?oYcfQF7R2i#T{s<a$
zY+Z6nEbx3|SgpGFm$<;2+FDUTuHLKU4W-SBHagv9czS5D2*=?l#{9_WRT-XMn+huA
zojyb!$Y5N~x<>3n)PbCJ+AH-duCW~~3lnCMW|3xj=>2<fz3pv#Gp_H(OfxJWJZqS{
zw>q?J{i%1MfxER{g{iuPY0EhklpnY^zlED!Qo-Pyvdr{e&c^NQSQ&p<Cw!Q}9?jSI
zv$|MdO<RKH4f)vk3x};Xus)Q&ljN(CtrRI7zDaD-)(LA^-$zZ2UA-tS)NJ*sw9-p9
zFSt22m`&O`aZRrcN7V*n{zm5cYdTXPT29XLh_FwrRqwwnZqo8x*ufW47HpPd`cQM|
zGiQJL?=^PucOQJ~zWV%|+8nRv2BLSZrcO^>wPp{O@;iT-sypY}PNX@fe3rYihkfgH
z&Fz^|h6##Nv;1cH&GG@kUA4Pv`Hp2Uz5I6cZTD?)bDeklchnaCUOmI1@b{x{SKqF_
z9sWJuciF6-^OxpyCRzy}E;;)1Wed+`)#uAU8~KROoxdb~hTnOEx$_dOY!8=IwY~hp
zBRqN0W#cz(cX_SfAJE+uU2@;*&f|jq=C&tqlvbW9`2Dc$$0I-UIeCpYsx}^29_{np
zg8lu$H=FJh?L6&z`suXOy3=RtR6I4{jww5=Ykv4^-nAQhq_!P6_U!6~bg9`9huLo2
zICG@#L&Ti6FEiTpGdaAEK3sYK+mkkKW;2_U#p@dcnf-Vo3=<AD{AFt6w$Mx1q@A$2
z;VhFKw}kG4Ee%(h;<#s=&2;|Cdj7__Pt6K4;u~TP6#u_7S^C|Vw5u1kNL5e$WPPD{
z`SZFkJKMv%wh5Y<t!;Z&y6Vp>ZjJ*x8DH1^Ig$CpPS0AoVs(7_Vypj#Oc9n360>96
z3x#AZCR|LI%l2gtUmRm#!Mg2pCeKT(XI!neX4ZzRLn}F#KXbIR`20!k(uHuTXxHS`
z87q`~br)|;HF=ZT>!DJ6FyP@9mODZv-WK|aTWor!7zAx=-v5O~`R0|Zzt@Ty)PKC%
z#Z#+y-g2JNoV27RuBK(TUr$Lado8g?C*DWRSM9Fd-G5%f9^3t)P(Lr;hG(_DUys3D
zi9{>ILnTQqFOP6s?i7q^@#bCqGUW0~wcweYlUJ$*`!3^?n&oFYH_hm5%B+(nlQYj~
z1mC)|VP|4__dBt>y1Vk<zFXyguQi${?cMFui}rhLa{XXsx3j`DR>n3taQ?pZS--xl
zJ-uD;{>}>d(;?H3Z@yD-kWcQKRSu*29a*^@m2b`8|9y7MocZWu%YQr7&11ycPxIBw
zvNa#pdoMC~?xM}XC48+`)^6#{FLD^>@-CRqyI?b8NyC2TKeD>Z79TA!YdRO{Z$E9r
zxxNb*ldPnj(w%?wT5{~rf3UCNFVjD634;d?>OU@uf8i0Ydj8^#%-p$)H!GHywbX5y
z4$=_^*6|{TV=nK)&5Sur?^PC?R6KuRX@2SPZPxccJm<Jq+`fCRp4}sZ;k?uj>t)-{
zv0XUNc7gF9--q&s{fkQ`EJ>UMM&B0KluY5DWb%Sb<=`#O?B|KO7cL&$Fgb+>gj9GY
ztML5(<>t!qM3Le3GsY9m6WEpb-Q*^;6bqj#t()f=)P{=KVxKEAxH1?&{Md3-Sj<1d
zCh=p-QCS{q&ut5|S#BTP+|2%TuUZYO`S<gu))`EVP1IU{C?u|R<sL4r*Zb_J+vU1Q
z-&@&MzUrZFRz>!$+?vAf5@ywzv;B4&%q>cczZ1PHao7Hm!%l}EwH?*%+O3rrac9Gh
z#KPmP3+H`}^<A;+c^1=EmQ|b~!YkAsY@YG(@9~?G-;V0e%Fo@mwL0Xk%ha@VE8W8-
zQEe}4cz!?n=Xt|plg_7wF)Y7)%a6W%lYe`w;UeV&zL)3iakanvDPfv*(6o6kS@<q<
z$}AJMc+-7bKkoV2+4~<CFzNq4R#JbM)%??AUh6x}ubWCMHWto!Q4l!ge)SuR8u2d`
z5(Y0m7AUQe_>lCZl4XZsf_r{avv1CZPF}N(AJ=^rtPErJkuq4BTgDuHH=$U1#w)hj
zw-btm8Qa1X1I~9fG1N#IWF1&p7-9P0YC|M*-q$qK3q@?zR}<Dty$OA6@rBoH^}#gL
z8*A8Zodydv+h)m}Z#$@FdLfK$_SFN2%;M&^+~`S>VS2yBL7>8YuTw|p$HFL!Yg?QX
z<Q`c5cVw8yKf~_9uZFwKb?L^^Icd#jb2iMJc+fFiOigG02AkQEGiB6ZsHF8k;#Uin
zgoiDy!eV|A7W}gg9s1dJx#fUkNQ4Fd7P%bT2VC6hI&;|5gvHJk2}m_<C^&S8^O@S3
zIqg%0!~8y&CBAG~Dm=|kz#{Q&%hsD`PP|y4&2o!f{7cxEru+Yjf4$z6xb@}bM?5tJ
zzndO@^{~oY;yL$T>zwV0e)8|FO%Io3wY|K;b9tx4mDwx9?nl*Te_eYuW!3NeeVZ#U
zm=&I_=PAq=-{n=zzu58i;>^0Pck^~tzmkheud*tBRNeMoa$ow49|ms|7aew4`_y9d
zp^~)LmuI*x&lG)DqiDD3gIci3^?cixpH7H<-g4)Dk^5;`-SRDWE*80Gf1H<p@1<-_
zWA^2<ou7EQ@APW#DUf%w^eJ@yxWUiCKtV#l#zs`2Wv7MWtQ`BvZ__vZ_1QMRlgspG
zt=jJMo;McgOC&i>I2ClSPEG4d%J-RX%+wQ`_bpJYc^#X!X}3?v@{@YTH&dtNH@0n9
zUq1QdJHwl{YO~Ize4ddbr+&XlOvbiU>QKi$6}8z%QYO!kIecdEL!YwllX-?WWxmxP
zFMeymGiUKsw#MyYhXo#PXgIou)5ol_Q~y8;{|uMYuedEF`Mf1`5;wFQ6k_w1;5IM3
zq~ZfY%08ErUs&z%Tj!H>NP}~+O3x$%FS!Rz{z7)1KUx|VbDo&&?-<U(RK)pMr6##&
z{_~1drcc7rl0U`Oz^J5E;m_kI`Av*cTbl3v7v%W6-F?IRxG?`|SA<n_XLw!z@VfYw
zaZUJ-bso1KSO(Q*?vu$pFg<nGgyo65An<TW^{Sn!Z~wdRyLaxF!Q8Kj-=pv37w$j$
zSFAkZPGVs@s6M`Z{q{?_WuI+)o>$H|_t{|X^F*ujhu3>2N6qFm0d>Ic)Em!CGs^vQ
zZ?f6<>v!VI>*szf`+mK0^1Btg7r*P<b-R4Kd{6m|10Oirr6=v`?$J1M)c<~`_3Kvg
zi=eicuw2par*_k%^`!L7K`pX^oevi+w2tGBd!o4az0S9}N9#7<$QS#4Hfew43fJzp
z&z^K&5EroGskQjEp~T_C2eUK2C*CIQk9=d^{WopneVuQ!Teoe#k$3H5K*SsU?!T!E
zuj+i8*_xSG=(hPr+%?0Cy5D9V-M8sNl-TW4PLYv0`rU7zE~&mCzMRqaU?8J6tB&Z0
zkT1>*+qiycC9G-K%yf+_;`HxxRdcm$!w=*b@-;ZTnA^zi^gbetL-yM5iVcno+kR`P
z30VC<*s<}P%AsyXeYP_a1>mm#F{=k@O*^6|KTDkS%rU&-9ox$X7h0T!`HtS5Wg`%;
zF#%jhr>o7GS;%o<QYFLp-W@$p9Nrzg!@R3q`16BzjQ5)N6r>%R$vI7J&ddcXKD+vS
zf+Lq2`&;&9KiZCNo)_D^@bIh1^D<YAD|vR#l~ZK+lV<&!Z+*DT##o+dlRR`%`y)=t
zZJc78q*;9=ZT4;PSuOAR`rh~bd)*!V=ksyXvtJCpy}mV_OEBI_xzzn=vu0l4okcrb
z3%ehQ9pw!P%dlL0q%LIp`gc1%{Q*_nvwBW(=`Noc;b**YPFj-X*&{`>I%^GQrWt*j
z6n)NY<DA?i%iBkawsk(u5k7rW?OW^0x63a3F8>_mvuw7H`OJrH&21uorScq;Y9{$j
zI@Ov`p(M$9WXFg4qbB__yZ=1?lz-Y~dZ*s~qP2&m51Q^2TesmjZ~YyE6MG*lG7oor
zu;J=Ml?cz>J?rkUuU}EHW&eWqC5$qQVqL@CSSGW?Jlf*=i;2m=>wqR>F-whrfztub
zZM~VQ7nJV^yj>&kjDe3MLgAJogE&i$zz4?z?2O7Ra|AXx9pL^p^P0FolP=S~A0n)+
z0z5@t>M`?KZVFVX*I8S$vD|0eAkdO;r*~fX)cfGS#W#vSJY3-+@XX#i_BFr8;Znxi
ztToHudW3Vxx$#&8-oB`mco+#auVar$xY(jB-1jH<^Nuxn-Dmi^U7+YUmv_3HfZdOB
zh7S(%I`cM^9J&<A7dLM^L%#Z+xgZ{=xB83e2jUMsTK8nbi60wI{P+-Ed_c)l%yW~+
zrg^fB^KQ?Kky}`6IB9>u`QK%#?LWRoeu?~g^UF)&BiyrBzrUKz{WA0SBe#cP57QiX
zTO5mu*AZ2X1vS5C`CQuYW8R}`uEU>gkH(6Yhu&GlBeiSwqjjrJhJadUl|GYKsy+5S
zp{e}sXv!*3^Gxls&$5|4=8Ec*Zir9L1oM2%XXY8*YnkJpWI6xHz8#6>-4DeM>#i`V
z3{=k9;U4sE^2Zr2la>f8|H;23us--)>&7L?-i|4;?+SMwfBG$G@w8x%H{I58YMXu1
zW}b2Sv^U@4L0fd*ji+t#c@odslJodhzdxM)S)k$3?B5C%Ki(TWczED&Q@)&yPsI)a
zGoBv?pFS9~B)YB)eZ8dnf;f+%;tT;Fjze4I-Sgv0rfd#bz{Ae2BlDr)z!64a_Lw(k
zT%%HWME&}z#N=5LpEb;6&P!wGExVAw&a0iX%W1{jRGx)pYJzS5a^2sT#VKbiM6evx
z;s|rxz_W*yJ-T26>+zdShou&*6G-W0>3$MCUApP<9=B_!)y*wh{9f%X|E9#TH#t4h
z_IT!RdHdf>4p+)OtSu`&GyTh%MZf>JpV!$P@mw&X^5CWJ(uS`J+A=2=d5gtvdNIe^
zGya2BWQ-*H<jfO`yu?B`t(aq-Igu-QBA0VG$KJrX^CDd;8*C$OPCoJ!^PhM0@S>AN
z-&oxa%__XJW?zDe&a9M+D&1<4#wTZaUgH5da`Mlli73d4B?6*(+hKXNjx|fqUwPg)
zGxk))#wj0@G#?)gI@Z0kN8DRdzsz|1w8cOFU1mANa!BNm3e$Hv;Z9HI%bu4#+Y*g3
zq43)OIef{Mvd7kY!$)0C9fpm%bV1V1%&Dig%<wavnPqe}1(cLBef*5)rdgd$k-D4>
zY3QphpXpcr``x#>k%>RcF2CHPp0<2;fce5atHX~a)y#eJX52|AYkL_W(Y}jiNwwO`
z=O&U#UpXgyS@t$Y?DS5ZX~mH`&o@MzOWZi;@T1p^|2A4y?_B=0Z<=+`t}jnb#7}4H
z9hp|}l{fyL!86|UduJwp|Cl20|K(zewEwn`Fe2v8SJpSRKcw0CC2SsiIB~z`SArqS
zYQ+e?gH;Y6eZT!q+3yb_er~zkv2M@Kgj5!7<p`;Rw>ZSyH&``26<ouAJO1m@$nOFP
zp^(wNrj9bV3w%KaOw*h<$ThJF>U1P%y4dG4?&JSqm++_IGxIh63j2ib!4;3I(oO1*
z|1LUwTl%ks`lW->uC~ez@_V1Z)n>WQXkoz1x8;`SrOv7DZ&hv{Q39csbpj2{!hAjw
z7p|TEd(`nf^L%!D{)R*Pl>#FFHh3RXVn3q9-m=a_XE~$nLH<a7MhT^d?Q7T%ycDeP
z`Jr$-l3O8X8~?VSr<Uy7)-&;!&aa$bZ+`80lhpiDYBFO|=j|5h%wt{O7dKrC-y{51
z`cCKV<<iTfmr1MfUjDo6w)p+v8vl>eucoYO5A;22-BE0?#-FG3^WL-H&F<f>@Y|Im
zm-1O*cmCUbTYq0WH)lr0cZ+rJ+V1kU@0QB5w@I1wWAdq)OJ;~`o?Z4^;g9U~m^riR
zZg-vi$do4Qay<3Y>E^&4Z~tAk)xEd2eaAWLXIt;h)#lF@(O*!(c$8Zs&fz^%mH3nm
zOd0u1?UE1J7MC)%a4Y<g_Ltk>cfgwQJ<FNo4SEfw2ao#ec%Of|!Jvh!g{vjOqPgVY
zQU5b_Gv=`s%yrz!aEjGH??nV_d2wywG;Wp$j;q~R1@u<9uo~#G_i!h~Fx+SQAr>-^
ziDUZp<qoeax!-RRdTznR^xmZU%OAf}3=1p6zTIy9qw{g|v3|#o8M*>h|I6hV^>}{h
zUsEgyVDe}Z?tP__A>eXg7H78FoSEQC8dQ%LEDL7WH8OnRb=chD;s=GR56_r;$WCdx
zEW`(H`LXO5Vmn>LaUhYiT~%h9K}1tlSip(p8%``&U@DlEIKPD}%BF|m-089hQ$4R&
z@7S~R*HV+(vRb~{Ta(+8y=J$sZ3#JCvvpq}@B81<v6r?ldw*t$_V-D76W=M`U9`jX
zqwdiy-isf8+){A*(X^wnV!|ulE!ma&buO1}l;4sK`ZmknXD+)}lWlzQ8QV3^IVp*j
zCl3`(YJIB0{X5BMZsg+Qc~*BztUiC-?e_LzUHfm@cLh6~xpab^i+AkrzGV=?p|J3J
ziF4AQjxPt?ryYLIz1n^nYm08!%cB0%Z>LM^Ez6MI_Ta{YbN8#mvN=T`#~=L>eluU|
zw{L3o${hWvZ<jvOT&JHQv7GS?hdJ9Di4Vy#Z9Eo637+}FqLKSHS%;Lhcj?66RMpa}
z{PoCW$#Jg6s%u;<S}a;~WDQnr?>?m&o_c%5iBD7CE>=1nUdH=0rh%bH+8_@!Ho~??
z+8|%1j#t7g;cCNIrhD8w-UVNu&By<^c9oE9K3ml+lNxr3ix+R(^0WP!V=epmn&6M~
zVTJ#aS-6{C_aw-EuzbP%=gx!k4}B!`5_jz3-*=#p^LVB7${gjXxl1<nhD>QWRkmmf
zXF!XvaEOnE(ZdrqKRDRa<d{4UGxr@xI@HOjt~O^zL6V%GkA=a*J)l_;?*_w%ds-d}
zu{mq5$Pw-ctqwAMzw-Pd|NWiI&dYw?HAO0Mzsxz_yDAmQ`#$Qf+}XIb=~?b!P5x=Y
zayrXxBK#^h%=wgHdG%mk#2wE<_14CniG|aTzU|HwmkPTdRmlPBX_bawjSV$lW0Ldz
zG3URH<>eRd{5AW2dxq`z+jsJd_n-Z1_UrAt?X&$r_4D17T_;N>zie(>W_+h^$KQhA
z;KA<O!EHYm@jSDybviX6u;-c5dV9~A%YQ4&UoWY@{MYxpxukyGt_sJ(Uwc4xvyo&#
z!oix;CDUH&Opde&zFY8^*ZSVctp^&*pLZYLYf`h>kauxGZVW%Syj)x7-n%lvLC)Qo
zZ}fOmmux&eXAS3X?h31fR}FKS>-Z#Yd3hvn&vfU^%rWlGUAFP8iC>e{ZC_{0%r|CU
zyAND=__g6W^E2KV<`3>=qz8UxW8&kBuz2vWp_5sSZ-+&~;}_3$51eGwX5S-YP<-Ib
zy?Ii%gOjU;CO8P5<BPC<@YcV#kYiu>^Ml*wSBdZlp8LCGQnSJt>4JQjZ-0NxG(NKZ
zKMPCZz4LNq|EzmhnAv4y!2MxR4|&6e?;G6~Oxnj_l$iNGxwn=_+&)%Of|IerhViY<
zirNo9-)1u|Y;lldV&BXk^_;Ox>cAt;ZuK>DH<)DwIRDLDVc+|AO<_TE`@D5M64Tsx
zvsT#m_O4y{&ty$}?`O%H%lqZt%HKXNeDAoW+x+RelU(Od3;0vF+pgViy7Z2@TUX>=
zj@<roP4)RV_l$R+FWB|{gm1?>ABDSiFlb%>_p*9P{mXxs{)!%pz7zk@^KN`${n5Xo
z+x7F}CGt%hUwym!Ry+7<+g0YZdGVPy?rhkZSloT=&921a!_V4sFI`~Swfj}xjk_B@
z$KBbmJ8^kp`Qe9s?bdR6_MjB;;OMTsQ}v%uia4jbQ9sY0FI6aL(#>uaZLRp-a}tu`
zeb~$s?yR?Xc=W|ypJz^dY;My|>PVbTxt3@V@n+e|uWS0|UA@U*EuE0g{3fSiuJg=u
zZa4bxUA-v?5qc3lw|B#4!Ll_ae}4p&lpQ;F?uOy{QnMrH+HPFjZzY{<pRPG~?#AS0
zuis3VE4yiNf9aQv=iDkU`M&7h{6T@kolQsbL&||E7teia)!pk{R(9eX-_4tf*3!x8
z$!~IcPKh0yVAYavy&;~tjn~2~;buen`i5-gb+z6%Hsl}Je^0?$IwigN&2iR6o1MR1
zU;fLoCGmN~eC9U0$b;+sCxvnxJZs2b;P~LJN5vdbKE@qC7hjzl`|*#tT>rOz-3J@_
z(u~Z_7gRRSTEn@#E>>m31<@BNxhmO*tu=pYi~ZDgX>gsSGbv@FO14U*@-+@?DaD^U
zJ`7##*Y6+TJIMU@e+Az;wSuz7&Ac)d_CMyOJ11^cU9EfiX0XB$C2z6FO(!<ZaZi}4
z+O75aW^cxcBrmbxYlk<ce@_ui{cW>)-6MwE?Q7qg*UmC7QCR9C=H01dw)xJ!7X656
zhN3&xnx20cTlznEdP#Rl|I4<QZJviqw!i%LvhAp?*mnJ%8*jUB-xfWTsXOg*l+N<4
z5q{Yl=NwCluL!GJlNWe*(XVUI<Yrgr<lVTl;bO9J<1;(ZNcpYW?C)#mN~U`4-Yu19
zc<1;B+rxkT|C{}lp30~<>DU9UJ%1j5>OcKfJpa|6RS#<KoD|=)Zg(LoLtfmCv(md2
z51cN`n7Z$Q>-_N3yT6v182snmw)EgOlXDmUB}hx<I5+2JY~bbHHoq~>>_+L7_l<3P
zt~^-Fw*Knw(+S~{Z?u@R?}rCXe(;p-z45JwvCo!VoM8U?Zpwt}jQWG#eD{hz%xeBF
zv**PF*J|O5s}`i++gaXwaGl`|Q?{(D36~{uq#n;;WZ!+PV(u)#{kgyXEqP-n&7j;g
z+m7AsI7`Jfb)G2(0f#I&7pv4vdf;?OzjA`&W0M&%jO-E`_cabKmYvg3ujj1&QoN-1
z{+#9_298q&QA~=Q$&u1Z+=rF8o81^nROU==a61&inRBD~ka}L|{Ogxoi%okMT;{L>
z6KkjWFl)RNyY^+`d6Nl#%r8wQSg{}8_v73n>$QOv#)pg2+Mb@_IXzQynz8YlZs{=d
z4XWokrKSZNz3G0Rsdmq2+0Mtu0(RHGS$5fX*=(QhYwup!wUW2Z>Q2&+Yl)8zC(Gnn
z+$q?x`r$Wm)?kmNu0oSuR(HHpUd6<IuJm48r~dYcyp89qAF1BmvsBuB#_kVZkod3;
zkGspe@cZpH27RgW)tzgzi#8cGUgO;2x1fY+>EzGfD-}waRtkH}b$G=Rq(12{OO*PB
zs!6ghxU1HfJbCJFyXNHLuy^ZanLfB45EQ$ie!&W&x0Gq^=N}sWJ`uu>wM-j@JLWpP
zQh#xGf))P~KV<WYzOy(^ebdnPev`FHjr!(=2Rnsp*<T!qob_cxh4lOX=Vql|JmkTt
z{4{aNp84m$^{+R$HMJn>SR;ECQ>0LzH>d{gac1Z{&1}pb$+tx&r|}}sLE)EWaxbUp
zaCWcZ=w8FI>ym(*_W?EjTPiZ&`{&D^^RW>2nb&b}Nnk{Ze?*G^g|C+<uGd+1Ti<7X
zS?T8PmanrHo_|_hv%1&&^oPAX_fLP=Tf?-u=h116ebT!VXVvX;{dMij+LzInw~LjB
z-nkrlXVH%SM`A~HyP`wxN8NS(dLtyc#`65ZqIu0vZP-uG<eg?LrL){9;#}eeX|s2Q
z@@H#H?%$fJApAD(;@tyba?2HOi1%kfXCSw%^sW(X+xtg--nSjA3;uL`E&CXB=brjg
z?mKZ0zA<eV-?6vhHEWE%%DvJ>abgvFH=NNas7{<2`QZB@gUt=EC)*^{Pc(UAE4^LZ
zV6Vljhy!1l)?GK;p`&t}RYspji`ydZz<Z}du{V6L6_!UGczfu33FB_=8?hc+xTnN&
zta?4eTI^Y0>&)cFb=)Sg3tl=I#4cEs^gr{59-kNgW##H)A?vQIKQ!d|eomH+xu$PU
z|H*$6GUsa3bLSpC`rGeb`KNuo(_Egs*IsW~dG=)XF(3boe@Z6xg0t3Zdr!HZd+=-h
zgi9*?d<Wz?6%wASPm2$)YD*Mq^DH>nq@(ypiNQ%lU}A#Hp$N`o)t;$e6@)4#GPpHK
za2{8cNk3Vx_)}=%|LS6g^9TI4@z+<dY$(Y3Ut+5$f9=KvlP^2>?6%@bzixGTeu(@b
z_eN#8{39<DZhg^x8C_TU%-+|^clVELNsmC$BVvA|^WWFyo#On<cO{k{e&ratef_(n
z-99fnYqCGDJ-gj(-nCfM@~d|)m(9C6|J|zHiRFjiwau+83ZL6{mv^@`U%Ablx*e+@
zwjD0#yI<*h-)rjqm+Mx1Fu9wwYxT=@9zRNag%#>%My!1L?eyD^$`2*@q#tdq*tu+T
z#YywR3+9gD=J~&$mEPZThWEoirpZricPMq=ZO>NV`0XOY)SlYZ_Tk*I^4Q621%U_L
z7^OY+zKaVqOx-5)PJ93K&(^XBv{xBD)?W}KDfnh{<Na_q7I{}|ZN@OJ3e8)QJOWt)
zO>?*Xc>7wCanBZC=_uj+ozhGN;RoUv<5_!_+Wh|4G?9nnP&VV{T;cZh0!_cSOs;rn
zw55DzCda<+9>;=tt$gCFZ+>ttu)kO((7^tmy)3_~X1?6~zP=6TPkBa|CSGj`73TA`
zFnM@k55I>j(}s*gi#WUU81IEk9hh19fR``y%ijG9o--E~Ji8ZIm(O>KF~E(*Tm8oj
zh7`FLp9<TDUt6vVZ}7Qb)Ob3DSH;^QdgA^Dfj#pbGOm5*5J=)pscyDj|8nW~h5psH
z*48Ucjf?(T96uz=QW6$BM<KOdL+jmJ4ej8PiG{9{F5g@Js{Tr1SI?Ed|HN6{&Ia8J
z)s$MfCvD}upu1=PKJ-cXRo0*V|KS6hQn7VeY4xkt`Ku@Y+d3~#-Tmtv&baLj1=h>Y
ztX=VMk^L-Xu6o_0(dvixw_TjV<RGXRIzQTK<CRN0tRDJZkNk2~+jt{qDc9qS`FH+b
z;nAt}t5xjx$W2aVPnkBieoKV9o|%@|@)ldKa~s$n{o8E))uCK!PNHkZneuzv73c1~
zAruqBaLMlcRlA(4>FlD#GnP-^c~Ky^J|@<9ikJR%5s~1(n%6U<FV2x#7-Ap%vA8;D
z``4u>Qg^NJh@B|@_22fR9dG_8O|qSNzt+%snWRx-Tyg%aTHS3eN2hJLAvAk>323q#
zGyxuel%?zDZp3<)kZZ1{bF-|jrqtd35py@OwEe1VSb4<V#Nzg|e`C+un)2`1SzG-3
z*|)QAXWuSoyW_i{`T(?{uXfpYnZMMma|Uw-_kUk<_%Cn4vA?|E&F|bVs6YIdx4b;?
zQRS|OCEQs|6MScGNm<gY_~lPjo`(&4-Oe4RvX3wOvdxq(*`3JV9ni=js_z}p$Rg?y
z(D+2e!>{p)$i3xV+Qt8Nb?jaFZujK5HQ#(YKhHitcfsYOOM+*G?u$|KiJp0FM|n<}
ztL$aY#a~{QUMp`_?KpHO_MYK|!|KmND(+2>F8U>OLr0!>*7@&`#c$S{PpDm!eQY|P
zl5hMq>4=(l!8LCd|Fin><ni%ywt0otQx5TM{#AbJ)4teTO?%xty0Z@bH3;?J=4ury
zGv|7qa#XR#kKE7ab2n|j+mtu^OYrg;n%T9>X8)}^mZq{v{!?7M;`{kYzYb4)wzzKZ
z_WK35e))R7`ft(LQDjyUVV&@{VJZKertORm*gB*uuW>ApZpdrc#rTG8hcrWe!yd*j
zag0CM9!M|9XV}m9h;4%O*AoiqEc-dPzqe$_ThCCye1uoQ{J<TC66Pzs56l>DX&<=3
zkitBLH^8jnCPO;&g5EkehHGpI(hG7MwlUsetB`(>(eqiV@lyStHwk$?pE(ZXHS9L@
zo8(?K?+cFryGKozV;rMeJ=fMbGT+tr8nWDfxyvtag7bs7*<Q{Uetwo;FKOU+z=H9y
zw&UX-WoZS`1GhRACbloNRgUxM^ZEQFqHnW^(U;6(i%t>U^<E<P>V*#aWJq7WeQ#2?
z+U@C{hgti69WwAyJ1yAscUz4m)5MocUMi%CP1aNk_E|Q=hf%fbrO0I{2Q+p!HFBlT
zX?L4UuJ8+>(ch5C@L{|Md!h67YRe5RZNkjwf`$xsy)3yWy!o>4irE3?a}$jYr?j2q
znJnq9W;pjCxA{zv0C<JnnR6zxT9rXdd%Y}#j~Pl96-;{dM#+%-;CmO>XO(;7!vZ#H
zGt6dqGu3?oYoM^p*`g&0QVdHtC#X48sh*h0ZlP_?)WIpIRx#s2yxdCT+v1Pz8Ego@
zA~yHS%N;jg<=2((&dN&j-?{L-c8(rLFwcEKi_1UfS)_NoyRvxarN<`^1=JKCE&;<w
z^VTIXwLUNW{per!Ns-B#kH5=p-m(ADzg53izjM!i7se(Pdnd5e{i<x}_Vs!9CGtO4
z%zkMC8}c@uX_lE`b+=>}s8bEfYoC*R+`QlUt!B&Q&WbX=@L<)s3#(jKuWH`1Yssb%
z+2x_CZ0oIpS9@{Cl`aS?ofkIsRkitxth-lttu73IyzW?R?{2B@FWIteuT-u081Q)E
zG3VZPDSp#?6Wv3f=Z1RjU(I)Sl~H}T>c4yQvuv(bT$fz$^(%Jj=j!@ZA4485J?7mz
zUA}7N*Pz#6rj)*E+_{x?zgKSOiL3Pruk~2o-M!_0AOG&R>*{V>zn^DfGPmH~ErXdb
z_(3P<;`(1V{KO}F!r=L1Vt>CZZ=b9Ng5bpQaZS!d|N1Hm&?-9-NQyDCyjrqKobhzl
zjN*37awxGrF;4c`)t6$Ib(cj4Bu#NDp7LVnigOO<vWt2rUGLHGuaTIR&&e}q`qfh|
zYkhb=cbd)2nEW&U;-mYwqw@>iv;D}cvWt*(NMNyLx0M&MHK;21!tl~DXu+3FU*>!K
z@BA`9;+L>@z0x1`L-k63mN%dM!umj}A)8@!;H&p%9UBfXvan0YBxEzJ3p^ja;tR()
zHVbKn+=eZTSJ)z?zuGVrNI%GJ*v@##%{p4Rh2eHaIy-Yd^B!IXbB8+&Ma*Y-9n2kK
zx3su;d+5GeJG(9P%7$-JX3ts=rzAxj<dyz<XjidScUhb2(fSoBdJ|(7?QqGTc_+7M
zyXhzCxxc<`$m4&V_kYRy`v3b@Gc~L}G-uVFO{-%#R{sgy_UrwIjztSNc$lj+UDq#p
z(JHdHZ_($uN&2gkx>ufEZkBzcA@O(XTT!#Pgi{;dK9aXTy}y3h!i78&W!AYZn&{J;
zGCSSin#Wl!t5cenHgtVioz%Q;?e0r!Qcf3qx}(w^bg{B0_V!1f>3L6!pFW!N?}qLF
z=jTN@7#l?dIv5!>*jLCLC~7zz-f-rc%jD1NTm*eZF2V@^3HuXv?(ueU{vZ-r{k*-6
zVP)(({|R=y*MsN$eqQ{cEXkRvR=!YCV*h_<G1edJSq=z4@cUqO@HNYN;XUUpzBixe
zv{U~v$NhzE(0jq^{yWty_n|@$KHJZ4E;^D^Wz)dt_k7~SgU?y!e|}Kh1kv!IsOdDP
zp8AeCU(Yb^nDdrL)@f~x+*7x+`#wKScE2sEw_oJ>kCostYMEb0N{_wOlXT2=__}<}
ztjM)IFM`9)ugYJyX#VWHrwwk;<9~&3E)DBVQNDXJt4nK=*wH8v*7P^^ua|waOAf9$
zmKWf9`Ihap$h%Rd9RK%c@|f+9OPsyt{A8Q5*=%dh{m?j>cIT0L0RPd$dNTJ!Z+3Sp
zd6wnNtvY)q^h=K5x?ejhC)5d<=E=?Qh_T!MYM%1n9Og}LkC`dm&a!dIuyzv{b-X)&
zk(JH=`n7kz)*md${>ELLGhx~(8|k!L2K$c{E;W51<Hufq$KbBg+2^4C`O7xeHhr~#
zJE4W3wK2Ts9hXXnNrmN<@Bg{PH1pi5lKofzhJBB|n_s&B>R+>8x$m~m^fR7$?3LL}
zP+5Cx$-<W<mtTU48qn<NyKi^jzV-TYe*Tu-^AA?ETT7|g+8j!}ba+a8w9kFZnG0VU
z`kK$l^Qg_y>JAca{BwHAiajU1oCE$Bb-QwZuaIFgJivBa+&#H<@x~^j;D9u)U7n8S
z8|Pfh@ybv1^qrSyx94Q}oUOf2&YFEn6W*yex9s1>pU0oc&z3ejzSMO3v7C1sE+>|^
z+r}UFd$hRa*<_a3u6pB{PW^5V_D(mKelh#)=d;&}WB)vTaQbQ6W!+`JXS_?U`g8g5
ze7X8viI+gDfet$z{--CEyZD#pj<m_Ur1a#DA6eZQZoVcj;_8MolMQaZwONt0=%~x;
zyYCn6YyMkvZ-IB|;fpo=!EaB0_|VR7Z1eEs%&ofu?r6;v+@SeNF6;G+dG`E!!<KWu
zDsh~v=RQ|&#bwsgT?_ogue@xsiVaxK{c6hrX{H4mu0B5`8Rgikx8fqJ>8^q{Hzv=+
z_NO<^dCSnWpoHNvqu>93G7WDSx42s>Gu&k^;eTND>gGfV#s*o1GKQPsSNIJo-Zv?1
zkbO|baOdJVx$6oHS?n8R50o|BW-MSoA-ng_JsB2-{J*;&+zJoaAISEX)wcNw<D}^4
znlEfv=Pob!u}o%00LxS68T<wJto$88!mWq57xCTY+T~YS%jN0(VoF-|^&^cJZZg_c
z>Pp_SKFG~-{mA75dOdcN|DTDS{QJhu^V=-;E|x2~@~kRYN{X-ioA1=y$CqDZ`&*rF
z<=eN`w{ETPyH)FUl<t0Z@@>fL@{db@JrcZgce<l9`=h&j4>=}tLSg5#wnVFk2G2ni
zfXPe{6p?(HfArzcqZfbfx>2f8<0Bw8ZJWp29e*yx9$H|bv)@819#%P=eAJd{^)h%b
zNEZkv8L8gcFvCx9?$R>vv(ut<O((f7zrE<Ptjb#ZZvCs`%a82Yt<Pup-KQnpiFN;q
zY3kQsC~3BGZT#jimAPc1&!)9*-sWq6<?3*`O!6;~P<ng2^vj8D`A>iS3E!#oOlgkF
zAy1__DjxmunFrTzS<oOZ@}Y?-&caKt?83!TIVK)m#g-d$-QS#3=lJ{iY^SbL|B*VE
zZ_1nJuT{J$wMoWtep?P}<u;}XwvOLyPRvuNmfI<t;t%4n+!RhZ-~5JilR8Lv+hUF<
z@6R2)zU1zlY4=<YsA$|=A31;e{YuE1<qOZRJTEfdzRf**D_iJV$I$G6t5P8l=5Hyh
z`D^}xTNC9kza6a=G1p1&*|)VIJMix6T>cxz?`O4I-I-wU#asHNe&w^BI+NeXMS0sv
zZZn_MXc%+TF)>B|@YE@aFYj7)-IhOQr8u|X+>GoaCJ!WUeaWn0R^#1v%)?nl@O;?L
zFOqC7^Sz?PSAXNWuw3dkqxZ9j<tqe3OqAV?8A_Sm%LCWu{dsw8K8OgonLc->oA~6H
zYQ{ckGtL;yOys$Hd*eToKHkZa5FwUh4bF?pB(@%2A?x<mAgevly5|-ngi5@^2bF1+
z+dh5MCh0p;XL1kk^N%PgJp267s~JbXo_wSC-7|*AeAevMH#ckwIn#4>dr0xGt7mqc
z?YVtt(N5Ro(w&LM7vJ4q_fFnA`@GIQ-<laO3@$ztJIwo}r7(Q;?yIlz{?<;ctN3%-
z*7v)K?Vp!?%a6MhI_ylkb5*Kv>-3D;iA$%dl%GzHTEHGGvO|gWcaP|UE~c2k25qha
z*V!vFgs*lw|6yfW^zWYYamLjPjx&cpn9BcFtG#Th^X{%|foJ>vF*z*=XNV88*2;2!
z`eEIo(kqX5g$IIou5VYYkQLQkw8%ANOGnj~jxMpli%MEiF0S1xG6ccmIdzkngQnNl
zPv5#(+ibz<>RJtpgcI#mCvua-eb=5^7%<gop;og~C{ydIKb)a2h0A&xmARKO^IfR|
zHNsXN3+`Pmb^aw=*4`^sD?bK7T4hrDruPD4Lp{Y-)#R=Y)(P4DFibsS#kXTums&k~
zn0MT9ztwi9t=-wBZLP0ZxOdIyS**NH!LEscQ%QwmqJo=}Pn$%8$}uI*O&<y_rNCg(
zr3)op?ZH0F1lA;iH$8})_qoxaE8mbQ&+t46(o%1-n=vQL>gwT@kL|hbf8O7I_FCnQ
zS**-`Yz|UyJtQ|2C&xA{3QJtQHuTq`&2k5>GsH7D@CukU+>l_8IdE0NeZj+Hg?UTd
z1&kR^&R~~L`>ktIb)NBu*ObWGy{nt{pWgT>!y&I>=lm--c$pa1E&WsLyUu-s{eQ!t
z26M)H+V7ROJ2D*bWiaP|(6pGdFHMH&WA*I-IY+?<TlM$vOL*a)X5;jDqE!D4jt^=L
z-3IZNeXBXXP7U7HRS+-uPTQ&0b;WwWzGeChH&_xj%e@q3V#s0IP|0BS5VA-y!b*^D
zlPurYO>Y`}zwdezcW-Wij=nc@nZ(A!6J(h;7-qF6TJzjayu-&_-c!f4Yt`rJI~Kl_
z*mwAY?6LO-4ef$9JP!>Sr59I796M|v*RAwmO5)OAvrp<ANSnCj_F=<0?H_G;USv9)
zx4yTp`m*NY8F8_1e6k(<nX9BOTyB`>vU{nZZSRXrxB%N1lLg+kck6|`(^14uKM$#|
zvH;00XSC%7YuKgq?(6dQWF%#4eEMLf{ra!hbpOxqu$<#}YozYPZIhNrm}Li=89MyE
zP4?pY4|gBlKKz}1`>&^5vfudkT`;&CnY6S({H55@?PB+%c6=_E-!!A-<AtvFXB@RL
zfp1ysZvXxL?8VwhR|T!JdG~iG%F2gry}x~J!LA*U(E{1{vn8`$f|u*v-M72;|5^37
zpyhezPABcT`}pm<^}F{!{MYtd`d$8vGYNYRm$2pE1^4A<su}yH&FVS-_>FkoZT&yn
z;(z4s|8TqR;r73W@_)D9|IvECjAQ@pZYe?TxS*``)I}wiwoKo)<LrE`$$S4R)@NH9
zoM2OW_I6kP_S@Oz@~=+mDXwnG5i;7U>34Y>=gJjUZqrY#c;It1O)Db!SMt@BYnbeo
zI<9)uJLN?E{>Xpx?YFP2-f;V^MDTIBH|y#Z&W7GvFeQBDI;}m6H(3WI#umA6JfXFH
zagz0=tk4X1uC-Spr{4%{l%0BgVwm>aZC-~b^l!Yv{jvYm6OYwX{Z=zs)+cqH@ZNEz
zRQ)Hnqlw+++~+>q3zyvv$duN4`_Vji#fG;}i#N1A`*qDjS}OA~!<vON%}g%0x!t+|
z;#*5)IxFX9EaC0lvT))*ezgNyInJ%QD>n2tZ*F6|Jtyl#ya7{z5yL44&9yttOs=;*
zyX9vR<y>kTn8tf`>%v`TCCzPl7c8C!1vYPPzp!C0TMxsDKgWv&8nPSKG45g8AgPeb
zFfqKLkI{zhhXg}P!vw=~X2&^>&t{m+V#mfHsgTkzfzgFcLb5^Sl=EYm)eBurrgXFk
zU5uNrG+Q91U6?=X(9i#`6FxgXXg}~jnI%cq@(afzk45hjypF8<-Z}ICZlOQ^AMSUR
zI&aL=xW>EvYFpFYfDN-J=l<a4*r>1a+ez-?j}yn<@Sprw@Kmt>ru?M855KE_<hYTe
zQE4Xftv#jQ<6FDUZ>LQlrEtP=lYELT%MJcXe-&=>Z<1?S^!%IZpI4`%Z`i+P$k@+Y
z|L^kh?<c`qfG#{Y^3<4J^5dEJ`*qQiU-E5v9hR|6bXH#LvgpY#?ZjWt=`Nk$U8-$z
zmp$ojmGAcS#O+m=*WN2pExa=S-Nz-CptXosw}+L7O1`?k`A8kdAAujrEUT`c3*B3>
zDm?b9R90O2+11N_UVCOe`~4oN?NvKX^|!xkySsYb?%MCGUVpjKC<<*ugQ`X&<~b5i
zw>!LvVgJkb-THv>ck4Uv3-(|B>-XLIW%;iAnO5LEfdv<Nr%UT4r0qOfB>%cW{e`Dm
zlfpXX0`raUuDqT4#rnK-egvD`+T7A}+}2Uy$6s$)oKiYvP2Z%eft!AvymB#coz<yO
z_noga!ewJuF4itBSz1@J_}s=gclB2<Tx932SS(yxvgll9oPDR6l~us;@9P(HmzFH-
z)B7U-{MG-5tiQmq*x}G{oiX9K;~R!e%xCx<tPi|on8fVE=V0lOdgV{wtownlt+Q&M
zpV_eZUip>Z=Vm5uZ{K~?k%M6);}JFq>4uz!P5bTHKcAS}lQ2)P;R<8OZMh2@nZI0L
zuA|t$Nr9y!p0THXlb_O=eTP5yT~By=YsV+CHNtQ08@~VMI4Z!hPx#BZ3a8uk_Us30
zm_B!E%scUDPEgsD6@5Zi0~c9&g*^YTaj|o0iN?=KP9Hg5KB#N@eXY@|y;t_?^2NWt
z6rZ@!`SkS*2U*z_i<wJ9=KXzhhy70c%|55krk1;^I?hddmC&@xe|pEx8iyT!jLUDH
z*Va%n)~sm|JbiW&k5|jmM=N_5o_>274A)PRS2tSv;=)nh*FWZjS$Q7{IK*q3^KWA{
z!`$|Q!w>H-+ds!<a`mNk>(x6BO@01&VpBSooSH=89L3ewj&2B^v0gxHs<v@xMubAl
z%ry@To7K7JX;!E`^^xaG|2~!PcZ<XGwu8Ly|4W5)7%_X*+R5z5W;)ZU<;&y#uI|;T
zr>`gbU1yoJGpMO(@B1h1pZm^dr{-;v@q1twS70Rmsq?D%RJN&d72my>?PQ8yt*$Sx
zy1)8W()q*RchzamTy8fTT$Zn2+U<}MVH&<*rCy#&j6~Rl_RylKikD1lvmd>F6McF6
zGHb8zQ{KG<?U(>}5nJ2#@18!lXQ^Ctq5oCc(DIVd{w=!>zuIVezj)Ec8t>#yHfr+f
ze8=DMYU>^DI=$hM(V87RVCDh$!XGmVA_^tLV%kKf-><mz4pN%$*#Gb!+hf^x`5$K8
zy<bp&7_#G4Ex9!{!c{UXu+5*Z-E?ZAZ)!xaq<+@P_$!Z=PEasiIj!2J^M8H?*Kx_!
z+N-+Fek94=3w|o$zPW{;`H<+YOMm8myKLYgCVEHWs6r&qB(7suj;73-?vg&UL|7n3
zWa_aS;`gm;&zo$0X}wZl8JGCsjcxlreV6<qnOCwdM2y>Qjg!dX!iasB6m}XaM&zv%
z(Be6e@N$Lvd46*fj*@-zOSVYd)Dl?W@=RU%$CFp{LVFS|)*T3c_Q$8{k?P_@%!?ab
zzD>O!t}apjVxDYf%oeAkQQ!2{ZC-vY+f*1dFYxcBlAKlDsz>J*)W?5Ku~N8tbnC5M
z(tN?v`%*6y{FwLf+vV>*_l;}M7xkZRUbOC;>(2|p{O{79ls~<-W7dZIb*b%Z;>$ir
z->YA&{h><Wt3%cX7QSs!o8>aDF`um}sIlaDEUj^anYDUBLjPpe^aYXpCa(`_J~>cd
zR`A0jU<-Tu1g)>fvsk>DMgHclRc^SuBHei5bFcFZ8%@79W<R!A*ULP$cEKk8Ltoie
z{dUNeT%pUnHC1M7>Vj=NSEe1*eCkjuImeGLMvh&t|8dGe$rq-~wa?6-DLA}lSjW7F
zkHOmE4Z}v}HGPH87<TY|uwnr5j%@p9e4(E0*LjQMvzcqt7M$nlGiQHw#$m3cOg`V0
zS%rP($6p6$2sj=1rCw?Ghw;q(2kJjRRTYY?(^-@vIPuN3OG&O$YxI7p-k7oc1+V>v
z6#w4)DWP@ey3ZeRI&dN^w%9v7FJY&ea==Nk*q?jDI2eLBZGN`iv^J1Qkqk>){WoE?
z{~CQ8j@j9|Cftke)_azfsyxnMGp!U_nOH8~vF`A9AM==hMHdZQWqJbNo+)>>$v=~y
zte%-8^G)o-)F`LkeY!H{Esu|NXPnv1|J81(@-6L`Kab3go6lTr@~6Q0e0lP|+1bzD
zWIz3VN?FC(=h=BxEg6=4_Qwv>?3XdeuG_MF*|rP+R{h)d`}8~gK&G~nJln$~KR>+m
z^zhQt?cUS*r|ZbC3$@$#rp3eX`tvP9k>CHyf0B)M*u<(5?Vywt-#KCK8_tvS65duX
zsO9%FU_bTU;iug3ujLDB`M113$oRzJrksoo-;{Tam)7zaJl&q~wpKTtukXq^wMjWj
zC*>x+Q9KzoA%~IkOFZW{^&9iQGUz(zrhon^wQ?Q%O56Ltws>6rl9}Ckzb!Uqy4~Ez
zpXXb@YParKVwxnZ@Oa;$xvjrDS-F2a`c`&uPS~_%TbE?&W=46>vfBaT@$0E;2F5IU
zH1FEXWu<E26&8sHKhM>yy>y{u)yojjB5mJgv(9Mg=x*NMeZ8M=LP^6C<_p{Q{`NY2
z>aVWig&zwyW<)>V6|q;~mB6d?P=AgM8KJ-4ta!&<DK2?Fyh^V5b<=?*?VOC$wkfXN
zZx{M7b4SF}?~YT-A7w3g`izZ1gP+MTlk?k+@IQ;JW`8bgG45l1{6+U+^rQ;TYKz!T
zjR)12zKi^?P-3t3F1<PL)wl3s`Lm7x&g#yJHrf7S-`3yT&v~><`%RuL_wmcx%iI0#
z@2Yujbn&#&dezW)?QppbAFLnqvE@BJU$D)kHgM;{B4)nt2aQ?#1$llRnGsWG-dt{U
zBdu+<L_hzU(EZ0=eQiAcJ9Mwztn0@;WgdxM+_M2>ONN5S_qBJuS5|A5n!j1lU;QS}
zKH}72-3bA0`K`y<w(nZKQBL7&fA5XU&a>ZTRApQ*@xN^A>u@nvr$2Y+!&A&0SvwA}
zZ;#L0(Ga|y-%I^xPSK(_=O4eYy!cDKZ=Uem`bRG;FHTUCx|DBIETdzWP;<e8P1&M>
zM~QucOr?SCPA3kFJ{uN?Ck)-pI(!E#93C_DG28Gl{Nz%wJf(QVgW)N|ROX1js%_J`
z8o68{YuG~f7HsxQWS#QayxXLabM>2h{<U9w`z~#YUivutkZ0REi<g46irMx*o$oC8
zG3}Z6_qo?i=044ze!irsV#9&f%W0E3XQnX8U4A(4*CLU1cLEdx6df%5F6`buC-Zt(
z?ZJvY$9-N_?Q>dS+;p&3S<&D}Vjb6_S}n0N*VVV=8kwed-8tua?Q3gx@w1tGp44b(
z&iE6jaA5K7rYAG@9PNyHG(jvrGon)VjIWXTq*Z@dd9`?@Y$tl2YMU+m{q2^Z>CE4j
z?YD|vrT;p0>lJY>iy!B*FRlCesO_@7-u+#3GK+tndupx!+K}5et@fHZgRDXc!^Le(
zK5H00TNwMFVGuSxaAaj=66=fzMzyep&h?v^PAuS<#{NO}!S=27o4eNcq|6cAGxgjz
zRf9Qqv-aJ5y1n=IK{o+~U0eTlPP4b%GRJP;2XPaI>&yxK3v3R2VmQNW@_a(k|HcKs
z{9=Q4Ds9?#UQsZr@x6BALzXt-l@^ch{McPw!17YrO8(|gj`QwSb_H`i&T+=57tDQn
z;+_D*I^iAXlnvDl<{T(sI?TzXt}$mp!L`l4^DQ&d`I`f@=cn&ekLunmtGhE&Z>8Sp
zcaytr@0i<hS9?)m>R(oE)k)cjoO4T0UE&d2ntRHx^PM&?x9k;Zrmdeq>%m`{ys5EQ
z*zlsDG9`RdhyBc@es}W!8*FrF&e_d#_jvm2hG$GB;tO_8STDX{C&S5B*JB)}p(`&e
zGkRQBsCWMr*R-puX{#p3n*KNYd%JJ%_4#@Bck5=p+Vs~~b<$kbQ{nejemcZY*ylX$
zNA26GuW!2ENxd*#=Zlho?0yC2?sb2^YVJJAb?>@@fB2_bHvd{V>y08tO1YZ5oHl%@
z{<t{BmW3nPw_i{5P;l+RHx~Y3La~3Is(L-t<n!N{b$RdN#JLG+>1to5pM7hex%bHH
zYin|JXa0&=!x+EEV@_8|<omd5yQhE6d+WDrN9B&%U4N&pG1~60zZo<Tcw{lt<g-f_
zf@TmW{EU^Z0B;-n9)0Ki$!AaNxc`blN48V6G!l(Kn*uD-CYe~C{&nV4<+Dvuo7Y^^
zS#5gy*BQOkhhleitNz`aT=xA!<-}#CKG`$G(tQp$OUSjXR?%5)b$Zt6P0u#1*{Ngg
zr>CuVy6f~JuS1)dHm^CLqit~d_@vXvC!St)sw%MJ57(*PQQ?s<w7Bh0wRUX^2o%#g
z^~fOoWk~tvg7Pha-@RU>ealOe|HpgzaKgq7KbGIt*Y9|`r24e(v}m2}0u25|UB9|d
zl_p(b@OP_`v<z6GzQWd1FzbQk|H8v97ryi!T#+Q3zWP*EEqnOf;Et1}jglq|3Wxln
z9&YaPJ~63nij6Ac>i=1O8|`X1xp>%jD(Pwd6V{*5`MrJJRJ|r`7PjZg6TigzKTnyj
z;3Lm*ZMpK%Rr^={d$;ObP+;vjr;z&1*CRhqzdhya>q~!UyKVUV`q#Xd-!31SzMqFn
zo!!!4OYiGf&e5%Azqrd4mu3`&tXXY+*=*nX9;<U3QYXxo5q#}+_!Nf}?+KybflD1v
zGvq&hkh{T%?}OYE+Y8kj#6LX!QM52yzvOM}y1f^+iXWfOJ-2#c@%qKZ`Z6uzyS_Vp
zk9%<kETrG3f8BE5pXUtU<~0`uFuma2VE*c6^$W=#ydTUN9;`@bQ3>9_v^gL_jO&b3
zW2A;bck-Fmf1A#H|JQUgSAb<Ba}VE!>&K^Y?@bkc@p${Q16=2v4o2#8bwuC${B;MX
zL5PD5Lt(eT>24nuhXQS#-(Jm?X4zFN`<VZH(|n@FFey4QvLakhWP0nPO%}a({@;z6
zC?Qnv|4CfR<$Ct%vhJshPwK2H3sZi5cSW#6Tzb;he%FGf4?pg8IJCQGoon%e*F}fR
zGY>aOXFs%<{&dcoxScvxGbi<Jt+c$aw$gt-L$vXilG8<->yKM*`{ksfqke4P)*sUz
z>95Z^bX9lONfUdA;@{82&g$-ZKPC8V$d}!Ec^mK4eH1sV=sL!JYK@lX%WjT!J`qLA
zkIj#T-aBOY?E2>YSI^gfkuH&0U1;0>)aQhxqfxxrs@1t!yNps>?{nXuZoJW)#pAn@
zcK0$VC!bi;@4tiIRBO!3+NAp0|Nn~gY%#eh{}bbv<<I<=`|EV}%yXaC-V2&#dhF-c
z1NjF&{1MOJDA6dkz`wzc@fXVr;Rb&OyMyJc8g*<M#G`&WdB;~U=x_Pjx^3TuYpx7k
z47E&ODqY?dxs-8CQ{M7_nXk=FMWIUT^L`W4moxkne&F9=&-ibRdv?Qxi93XkfX&Qt
zkG9xU@{=bvyy4%rj~CV-wXB_6?>^sm{*KE3mzN)t<4m2*clp}Gr~fWLS)91*OKV%Z
zdre`rY}`BhyLQ^npIRA<F7&#dNO|&M=loM8T<0^u!;*969BXb{7VNWpme13|f^yDp
zqLKC`U%%VUycNIUS(B^!=Js_zEJQd&*Xt(teA2F|=6JlkJ^Vm)`g{uypXGcPN@bZ=
zZ+Lolz5m5qY(E|-eEfFfpvw1qN}^|doagCQn$2|oW~w;j<kZ&rXBu@G&$>P9eBHWE
zU=91UjEI*FOG2k#TyryRjmW~=FJ8plmQYwJ@nAuB+Wp0=f4eEpa{Bho_4AFV=Qin|
zJ7jPtX~r$nzmH1}zpP$XACvd~?&9Rmmu6FZS@(tRD=_Z9Huvt5!%A~wm)=$|v%D$R
zc&L|^X{z+^C%bB7O`dd~5xn~{_$%vUez^m$!rEOeD(1<U_@A4Yew<<1p>-#x9`@Fs
z#bs16Gem##DWmK-`LnMV|GdW5RZ(kdo|AV&jn{X<?*4aTcej__2Q9pvd)Rhm--&;h
zV|mSG-euV+zLuBX-ERHFBX!5&mwf6TCbpMe)-C@%_Z?_t!sv2}TJRR1^qJ?1euLJg
zO~0+bJ>KVHRKq#O`A@iJ%zIM*|4U}`5;jK${|8DU-y8F+SPrk9{F1xM&Y-T~4@0HH
z2L-DMd2@YkGwgPnm&@4npo-xQ<0keCb*vBM4pcTgXPm?yAeT@%rTD)jqlFy9&$X$6
zdo8;3`wm=E+7Nz3=G*I8eoi*+TR$gSzuQog)!(k!UmLsU(YmzI#k-fkYrCtvtF!FB
z(VyD}l9qhu6K|f56TJLR;BuWn@eLQ_98o!sO`m3`ZoNKRoBzi7(tUnAIs+SjtHnka
z$vPZiP_kG)vrOUhlMm&t^OMxp2_M@wJ9qwyzm{Kjevyo*{nM=$cRc;y=N(<Y9)Wty
z`M$r?mc9RSdge9NC+nAVON;HkC2~Ki#`|l2cxk`7;G+7RcdK?LmWSUhFROZ!ckymg
zMM%Y0p8LkOJ)srHo+zIF$t17P_GG$g{JoEsVb`YK*KGOx^a{`W=F@@w0Y_gk&uR|o
zx_HirZSK;)ue}bJ9WfN+7m$4`z;Li6X6Jr}wIEUF0&9kM3_BOgGc}Ym++(czzc1Om
zLjG~bgfD;Wc++<>-z&Ue-SD1aKl6#c%9a4BrfZCT>>O(a*y<gdOOISr-mbKAU+^ZS
zB~06{A6NV=6T9yIoMwwfhDp{La~2df9Rs&Z8aB)|`!4Rj!Y=5m;OhRET9((!k<9zT
zoAyQah43*3SU5g(S#qw3<BW4ddRyyext;UcHWVE?6?tA}Z{*CQR|>AZ(^<rK;LGQ~
z4^AygS{ZjRY3syU&YQN|R@MDI{=WO_PK6IAZ<hve>+BN$z?$UBe!O98lp+J0=8C*W
znuh8TtDdy)emB+bt#$tFtD<60FP-taJMq=+-TE$qT=S+q%q(O{=<nFM=9oUqt=510
zHhj4CYh||QF5L{tFBw-;7w*zAo85Z!`Z16A^E+m@KQ7<)CS?119;;&$1GP^Z?A&#C
z>hqJ8C9|Ey(k2|eqL)?_Sh@7Y=d5oJ{p0^UUADdE(~g>g-AgQ1&E)Rm{4l-ce63`j
z{hfjz+h6`@c&WFHS*EP%OKq&$KA*gu4?o$?`u%n9m;P<sPU*i7xyMGzNBs$%&$sW}
zy&B)xY35nZEdfevvfE4Fo_)0Mho5ZVyWcX6Q`P3=O?;<tcf*W~kGfi)7Dm^fzvO#Y
z;q{F~S?e|FEnj;s+<dg}$F*1Qw3+kd+&=G^Ep29=BNtu#`<dRZ{ELx^Enj(L7r%UD
zx9hjWGFzYO3lm;!ssGh-ShmM4`<{?(h0kGr<Gc4k`)JM{;x?TN8cdvU1hfDu613Ab
z`uUgTplviD_RKR)nKPnf1G{Py=2-6(n(yBCQ9(6o!dIKT&)brBPT$HBA6czqF6lX~
zwJu(90x#1V7svCP&b-SmSem#racT23fsFLp`rV4M#{Y9uBJXL|vT{gtescKDEycuT
z6q|Es)8-r7J0~}<Iz83wpvpzx9ri!xy{q1p|MH)2$DyBZgv4JoskSe<^NpWBdcysg
zPal^~a*4}#Uu|INxMW?c_S}27PFm!o#!puLe!ZSaozK2h_1&^h<+k;kWzEcYGp)K+
zQ>yx9^ZWyIYigYw{T2A%eNj@dpZB<-h_BHyur`0sJ>HVoKl{#ch)ucPskVFPvDcFu
z<=M^F_ME%=T|0TlJ`X81-pOqFS9c!%^5BK-`BjW1_g9>L#Tn+mp{nU>@dt~BLl!4$
z7D(o7J0-K?Oe2`+l5uzClaGF<9grJp88DR@MA&zHlRhdmv)QTr=@~`~IflxHyNngl
zik$Pkvb>X84(zzA@s)M*<qVFFXhF{1zOTI87yJ^16?t+A)eWx<LCc4FHupMJ$Q(-N
zWBAH&j@gEv!Pemu!zufyuZ{5qHVmH_PB5#0dma(#+V<6!6@N8995db*9I<KPe&$1W
zKCBS2o57jfxn7{Dl(%ik<cK8`BX(G17&3|&y?FJC?b@;$J+rSXQ+KR<tyEg~|E2Sd
zN58x4L(;WQtm&2(GoPS-qe$Rik-)!Aua9UZE?9L^i{W+jtlQd&CyO%vX|ZKg2kEpw
z(GS(Qc2u`bI!5Nb?|+NC$BUf1OOKi*#d0<-=Q|<#<kMHb$QzQ~xrcv8Wd!`(?X*IT
zrQ*_qhfhKmYPVGAxmst=Q@kL}@~e{Z&WCg6OpPbMcbgwko*SfQx9Q62?3kT~+Tr(9
z-|JRRS@S$L+<x(uxkouaUpo2eQsdKa7yaV9^nIsmcL;p5eVKom@7Pi9B|8#ZXYAjz
z>SajOp1C!V;tT~$2RIqh=am*NNfEH<-%<1Asj}wtn)Sl5rxgSGX7Dl!SUNtj>EKzQ
zR<Y``gCIl1W;w^XYz(<fTlVLE_nTnp_>5(yaEf2U&rNSq{x4f}e%1M9{>PVDN}c*O
zJ4yC?wu{L@<5Rwea}W1EcG-5Z;Me(L2OgQ9Nk;$w7~YK!oAvY2ze|6Y?*3eu_-ltj
zZOjZiJs)k`myZwCh})*_Im*&?Jlc2JEFaUEr+Q}kn9f`cI&UcW{SnA{Lu!|Oo@M#e
zTE9Jd$9sC(x%@cx!_!~tKh57Wr#mTrPtleO+A~_qTh^&btZ}uj(%++0k{iPwE+z6#
zyCz<nMdHf6C$&%8_zs@?K3CA7%Deyg-K2-zJNlBjpRcJ-e13N7-A@Xv%3lsX`M2mo
z!$LLZsok3d*qEl?U)lD!+0&83{DX&?u+@=+b<VT4rm<XJ_~!IImwg+$Z#%2p^-B))
zQxv`Pac=n6wX&;3CVaBHR1w)b{fbJ;uJT#zv99yvGa}CYxRZThero2qy70#t1@(n(
z^IEl={qN5Zn3Cysow4y$@7BV(?@zv(#vG$oG+lhv`tqp@*Y7!gmV0UCLKFQd&ss0-
z4cH~__1$UXruo-CU5=`YI_*DSZQ0bnTQ<!P`zD-fzxK%UZWE@KdqQt5@7Oc4UQSt;
zX;sm1?o<4uEBlYGg||jt)@|%xcqVQ|-Dz39?eTg08B)w+B(A-)VBl4|-|&f7Yw}B*
z6R*Ud@NPIZS;J~c-e+-vj$hUOvgyng)(p`Ob95VQ6&n6BoMTql&!n;a+2jnHmkOS-
znG6l=9C8WU{_1X3ytrT<Zx>raX~R{<2zT4vG7ho|r3_aY!`!W-Wm_IRZI~%)y?58s
zfK;V>(<lFayKg7k^;1(>g>>Yl!*(5BSejb8agvVgBZUUp+uCa)E^VH0jd5R}%l*hT
zu}O{VeZ%KB#oSO0*#G-9-=uv$v8(>C+PC?Z=(|Nd5yjlO%wD<M^g->jOMjmhh)*wL
zS}Jq-=-c+&{M%>jV_k3kmdE<|jQ2C|B(FNXUGM%bpWL5*=l>qqn_OQgXUui|*w*5u
ziAD97N)1HY?j)M`AB|kQ&L_FGz;?erhe=+BK*GJ8TFH)2GOy(1>umUZ8zMQ<6lb(0
z$fewf+Hj$=?ytGxhQ80*`rrB1o!q+5Zl$~Z!S@%>Suw9=h7R{`SbI!8t7g$L^|+(I
zm@*jlG5_IXuwnSXaL`|E#_P%U50AW&cY3~uv5Dd5@#*plwV%CTc)ObKnIvO@+=EJn
z7w?7px7U7(_$+bI3bfd%;(Mb?zQc88OZkhEH-9SBHT~dxRG$^4Ah_T7h=W4khgyy!
z@84!uma#O%v0MIYn&<G#=F-3DbLmUmi|jk*yI0s>vVK&(iCHD7W|OdrwZkT6i|n3#
z4GhtaO8eC(Rc}&GvDG@FcwpLu9L|$qqHn@mpKmp_GRKbxE+|(P<8GM9(6;ny<JQ)S
zs#tawE)m8FEMdB9`PS(FNYi=rHgc-kg*%haZ^-q``t|15ms9iqU9DNV`FN^%PwmII
zFRd?c7rP(yC)KpHb@LnhyNiCwae0{iDwCI5_9=cvqVe3c`rmm*a|?I<Ejavg-LjP@
zXXg99XSi(L5X_|WJ548@?FPTQNKt~{Zk~6-KPxva^mC})wDHlYh1wdCVk}y`6;91R
zSaz%5IBU3dUGHm)&pyoudz<AdPCVGy^n6pid{~j-13}%h=glWbFIHD)mx<ml*qr>z
zH23WXaT5#u-35=oD~T_zcDy0~@KDZ*^NY?eJHPJyLjS%pe{OTm`?@DIw(*MGX6wpj
zKH5J0_s2U1cfEJH!r-ISf4jxrKT;Iy->_r<>WU>tL4zpJ^LV~TS6<4px>^D{TmgKF
z+AN-{{!{hS&M$$iQJ#@HIrHuJd0TewfA(+I?`)rfa{jxvC2>olOSZq*S6Y#BSb4Wc
zkj<Tfij2c=+kY!;srYs7rFHGA$TbY2+73s#0ydrst~nRE=A%fq&C#6+izA+DhE;Zj
z&TI0sa=gS@@Tfv@TKZHqfsn23X0?8`^K!rL`2C`2DRcIva<Titf0q7=YCHH<Z|ds>
z3s=u{_`LS)+uiwZ|E;YSxXauAf0u{2YCyoBRc$ZCF8>vNYkW6p);H6;1-l-aTzx8b
zdVl1l*l&gRuYdSH-SsxBmfnQ|#YcK`U0y~tWf$oRXxxc7*DIiNCU2=@Yr)gHWp6jl
ztiGw5Cup$sgT~G9XL>Vi<Xs*mDly4+%@yR~;K{S8+@aPoL$Uir7iY6kaoUX7jw9R?
zD;PI(2Uaj{KHi?H+Y`)f{gWw7{LIcN^B;!Qu^mmiv%SXpTll}H#R`?pi%yFEQQ_Qw
zwe+C2x!fFHDVYPd9n}io7rbeVly?xj@}Ws8F5oaX`^2M#Q+7Ei@G~ZuoN}zHVo_ae
z^YEv9^m^t5-Ud^LYb(<AGkX{!cneG!t}#R~r|>Q?J#dX7#&Utqi;t|f`xYD*H<^3L
zG1c*+riN9kjsCBug{2%!%NghWD~^BX*RN2(aFCI$Ug+q2cZN1*nQd+%7dp740)H+!
zQK4RM(a#XZo*>&$+Hj3A=D5x7uZ3;aH4IJc0(rdcmltf%y&le?$I<jEe2p^0oXBsd
zEM{qH_3LXm#M$_*d|<HPwxE4=ZDph8wo{F~+nJ^F*md8u-95eTci5(nlTXfCQhS(p
zc|K>z`qPYO%o$Ic7oIf#c*6W^N8X{{{6jtYhkBmw;yZe2M^SQXz*0uh8?u|Lg;!VK
zDSJQrlEx0jN8hGPOqbbH^>1~2)Z^MqEsr9r9!FL^HUOdfQzVb{1}l_um<aD+oe(<T
zcmCZ8ovYa<uKf3sYqrFt@8Pq4Uds&fViK5FZ0+*4?R1D?piIQRzu9xnJ^B*+a?j=5
zgIfag*WGcr&Nu6r_rqh}58q}8P1`G+d^g>>LB7;gI^E^{f`=b2*N9mznAh-Y`74!6
zZ+WBUeDK`x*XzLF&+>;1AIxjmecp__tL&l2W{cI``xhLXxV+zicba;{JdVSh%N8ey
zi@a%S`uo1|?S1CEH$@qKvOL)=w{iZwgWqfp6)`>IoS~jDuc30=VgrMqvy+~`+H`Tz
z>*5#1FNMvs_b499osn>LQcu?0rBU81_kTQ~Hf_&Zuk3i6NnZjkC2PucU)t>URWWs|
zRnXeLkk^V=t30;K1+8rl`7$BPc1aZf%4^Q6L?>jiE{S4XX*YFEMSPgH!c{JhtxQ2{
z8?K0_&41R`+jf$xyhlI%ojC}--NhFE&imRjK{bwxyh>Tk9Z`K}S|tTm-3f1GV?V`q
zLBwfE$Srx<$oqDy_SJ^e{$BB`j8Aawu~pt*ZJFxkbKJFK-gP-*%DLp7{%Jl5m8SaJ
zE!-~ez7)RL*ech=EGSuKk0IN=8-H4qzJ6Vq-u?33+)@Sq3190RO*rx%INH?T*gAv5
z?8+hM=*|1qet*~f(SuXItK4aIMft}T5liEDuT3m@@2$F}R`>h(iQ=v6Ck6Y7_^ipl
zost;Nn4P!5jrZ%HD(9=o>w0!p9sVA)bFW$6`rg~8AAB>~vHE`M)K;(hS$?IzpY5O3
zd;2W!`AKinQeGdw`m1njUCdhf@Ylz$D)xYRd%RsL*76jzRqqK~mOpoyt#9?ObJ??A
zeF2>b863N>bmq5D8DhNGf9M8ST{q3F3lNY<(Tx7NVvcOsl>SpQ6J0r19{1i<R1p#A
zrgh5Q_2&v1#;_?9x`cvc!*ie5a`>D-kk&Apv4HJ_q{H&byZ;>HVM&l?&#he%aWPJ-
zy;F4kiLUSz*VQXCOhuP2v2yO)zd$iSIw7}V6XPj11L?N{44I4P{$+FGb!u2Qwg1#h
zFW2WQK-TRE3bX@RH+{tmp{uEFt2P%Oe78OF*E&7kkoYT0rQ-Beb}HPdsrqV>J#FVV
zGw*vJF7fi;{cs^x@Nz<{^nStGixD5zoK2`W*w7?s+2~U{cT(hZ;}ap-CQGLMTJmT`
z$I&?z`}#M%ik`n;XZ73JjN)xG*mJt1)lLW<O1&Kw`f}A?i#$u#L#y@APJh__UT5D^
zAziKeX3T+~pIx$Py}|IVt>w0d*fl@BrEYvn*`7Q)xO@Fnwy75br`!&{{XnF5d!cMh
zrh&o{4;$;ZrI$+M{1#rEG3(eBlW&jR?scUp`iA$HOnzxr`Q^Ih>|M2)yF}_eFWPx+
z4Vw9WRi4M)&{yyOdT+hT(7IMK{2bR&b)D{mMGP~TQ}`C#lq!7Mvtg0I^<uY(i!F&d
zv5E`{>>atRI~FlqFFw+K*e9W$-{PaL)yC%IH+Ah7d^;~ZdC%$V$_Y$2!kcc)-{DkM
zqb>a0UP><WJ3}ec70!U?g8BV5)hf4@Z^_*><7Z?r7t9y_X?E}~OS$l!v!CwV`uE$C
zVS#Xi-;swl4s*YDJo>^Bpzbi&<0xl>dc$0XZSDeJ#11v@-T(6Pt_aY<#TS+<?U=Qt
z_nXoT<2w_CL#|l(mTlg&aOx}jyEb?1?#?S(6e7H|OheUK->Yq2ay3^28>_=(yZgzf
zm*}L=*)-?iDxOohY_E+XW;o9My<k>az#(gmhrX*e@qH_KuxGN=&dG^uZU-&XW^s3F
z;JWR|_<G5{;-LGMl7U8(ShPQh2>Z89nv(73Hs!$awW)%?z6j0K_cHl8{o(1PpFbW|
zuc&a%74qtj37Nm>14pU5hP4x;W|r2(1+5mBUyFDxdKVb*U1LSL$dzVJzmLq)js`*Z
zt{r%0QsK=Ud_3V2d-Y3(`|LmLHgMlz?mnNeiG8xof$xp?Og&;uHgNDNNxxs{IamMv
zQgz$S|CKHq{_OT|%N5F+Z2tRkWP;uIQyx4raa<>xyw<lq+8S_Y(T{77TtoJBbV-Xz
ziLGDy=-buWknJn>ZT)ramG$cPTdMA?ku%pm^0sw(WpVfYZH4CL7w@>5zkFv{YTu(}
z=eC*Y(wYVnZH09Yb~qk$UYzx!fca`pPnvCuipJSj9@P^SD|a0&2^YSi*p^pmar(P}
z=!RGCZC@IlWo=5#+mdzB;(Epk3F!|SVlq>7xq{k`N}D*IZJy3<aKA&+#AXI(2HOhB
z%7p2H2jUp(1q-fPyvSI=(fhf#?RVngzZ(`BnwYftE|}A{Csh9U<X3Zf-m6roiyB8h
z)?+$1J-n55Kc|=fwf`x1{SQ<or3a+4<}FF{2ryBRa0*a=GMUk8$0nKkl8f&-mbR|0
z_Q@)$IWy<EeR)qJ({kx^E9Q93Z3{W!(Kc_n$wkeai6YxPxS~{8btg56dDe2BS*&tI
z#rKwLl*y);Nq2%KZ;V^ANu@FMxw%wOd-@OeBELzAMKd*iDxZ|<JbzwPrJg5E@r0ka
z%1rK}!>l_eG^d<6xME#`%KRXQ3gL<SeoPjae&64SsV#NA_S3G_F|~)EJ&G1O{#U8d
zf9n%>v$k#*C6=Y$Yxl%IVNzt}kaTFvkKFgi`_!a1p*t2cvtJzGmEEzBdGU;l&W4qY
z%h(o3z3s3!VUcHU*z)eSA6QjD)Bl}MAJ6Z;WY1E^@h6;R&*H*+;&Z+qJ!@8&?69lQ
zYJa4(`8NgEt;NjT1|HMIbap9JM^2cm<KZRtL~(c1mE?n;cqa9$Dtim{sCFK;J2!vs
z)0#;}^JKbhoF;{6pY)kx<HRKM`}zj|h&?qvSL6CZza725FkMLSFX!&h^Dj?NRl4vg
z|Hwtb{+LXUD>0^_N_S>;3P<?Z6qa)@>RSIU`gc+fkM{W)>dk2%zKE-P>A$L!krG-e
zzGNlGrx2E8&RzV!_CEUJ!_`%unquV2;;GogBH(D(uCVa%)FlN!WVDPb)3f}u_??bT
z?s%uu&TxK;jg!%rlo!+XI!opL>+OEqet^Hx-ie{}n%@KQLLL|G2Pf<QeEqnA<IbDM
zvdXXD7w+(4(6Bu0!Ju)`d{VEnWu?cJ&Mo4b4o}Ir;P2v4Tl?hQpFib`ayU0^`f6sO
zI;Gg!+`-^BBZK*kl+)dxzdvp|p!w&c@CDZG8haLfXgoCe<sn7RdX+m9KG+?a&JnI8
z)Bb^fSB*^Q1wn4b99Ged2A5kMHXqUyR-Y(m#9R20caIcPdfS2B=BGZ&Kd!4uV!T&;
zEOy4rPZtaBy}VTrmca7!W9qVpA0zDaPWNb>{#evtur9aLefIh7i}!3Wc)j`6XBn=a
zO_{$XqWh;_7vICW;l{tx>^EQ!J2z}*^UJf$&z&=~;LfRsTInkecXt@8`AxHkc=YGR
zqa7K)-1)lC%@lCAKIrKyGk@{+%NHXr&YA3DdHl2k!*QvDnr;PIg6TIJi~Sy)6Zk7r
ze(|!!@`#=Uxi<%vEA$03#xZt^AF!Tj@9f9--S}LfKWpu|nSUH*+UA`p-q!3=XR`X6
zT1Dbf?-%Cr26vM-FKyIioFS*@!+-X>315oAtX;0hSzfg@w@nTft+&4WvE=Z}i$3Oa
z%-nQ%+D_ZAE8fwZ`Y11^UslNbsO6=TpO@IgBwXZsb2YH<B8xPq*{cT^o6>i`(YR3Y
zwf^z1Uy2``ADkE1A^RdN;8Oa<<%z3aeg++PeamF;ot|rXeh0E<9ALfFn%R=s(s@K>
zAD^}Pj2CU4v**23X@H!G*E#o2%9O)uQ@K6G<&LLxfi{z>1+($`E}sEC0#EC5lF`)8
zmCJ&CmxG3((qKp68SY;OI!@6CbYkk=6syUXeV5NhKCQ{;oWa~gE9gdAHRFoAKIaT%
zxtjbY&jhc;K4UWT<P)FEGe7j6^qH*q@`=&RRHG#qe{qMO<ZqN`s&8`$-*x9h<ClUz
z?pGP_n9YCpHC*Pi#J9N~b5F(UG4B&G-Sv0Nw9j$J-d^EbchBj{9lm?p_=``3CxvdR
zzQMUsc+=)$i>b0xwyWla_P==lf7V~S_5Kxo!qpmqSKKZX)i1g<ODSi$@clPZ{rcg`
zuk|PGn<FD@&Zkz*{Zi%GyLeT-HfGt?PCB;!+YdX$`dY*`FL|-){K4t68Pjb~iM7g4
z%UWKgwQG5r`?vdLj(4A}oExpWtjpq!?&@C~{>+*;<Jsm<vTtWz(-%=+uQ-Xf;=NDx
zukr)wvOGV_E=^0zpZ@pj+g)?49v`sTEN@g<zf#3Gm1FW(tMZDR&9&89rkB@UTo~iI
z|K74<k4X<w>~jKBHZc9SG};^MTD)6?r&{30+}%gMPFPrDzw=Z3oAi7;pK!I?uQnC$
zFWZ|LJ$upL@7un9*Ig{nWY6-CqfzPp!rb5+l1aCFKVJRHwrlSL(Q~n(9+mF`L{_Q!
zSUYQM&MAx(W&QI%s@dDl=kz&i>+H!_U-XJZ?K^t<hU+O|o9nw&jW73|_FvxVb^ML^
zr%v(K9UQ6O9^E%8EN0tq_uF;mo{IK0ITtrb9lbGAWM&0#>y7_azm}e|Kb_&VPTN)c
z=7!DfQs<)9JgmB-{)Mr}`}WM(-fL!A?;E|Of^6NCv#joWipTlA5878Mux<UNk|!Oj
zHci+eu}3JqboZtyHR3CcdQKM@))s6`{v+!lZ54H|@k?UOw11C22l%GtwXT|^zQ9Xp
z`u_F_o2+YRr(I~h!SOC<TJvnLl#}+Qw+;qMd+IHJpgZZwR0{<gZtL2x%VOI*|5j~#
z84;7X=P<Y4bFUSN2d>TAsII+l&*A^`rh-?T&sTiV6zhHI+R2(7tn(MP7Kf^yzWwe~
z^b*<3#5;2NsTOZ;9iJ0j61HUC^%<voTAkOao_kj#8-3{ZgbQUiFLu_n%376lrRMnT
zPSP-ZE7oppn$5-Osv^BkywYXa<IUgoy}GxQv;7Wi+gNpZl8JeoyV2G-`<5$W$)bKb
zn^vv5wzn|njO@?Qvy1QeIW6_RF1Fy!gTBU@c88X}*=54sx`sJidiAusfvOYx|5!Y|
zfA3me_rq&(I_lc5ufIJrd0w)A@9Ex&Cw+cg_jr2AQ1^ak=gcLo-(4=foAzqzqteq~
ze-{RG2KPQ-4wuNktMW^tZsOUW+*3lYJiqY5dYjkfdoxXEmDh;ATjAGmY1Y?uim`dG
z&iX9<GOKE__ti+h$meqp2IX&^EK)3R;KTK`OTej1pmEZg-snlU6nx*Ddnls5$M4*p
z>fU&xkQ0k+d)5if3C$~4UR-didtcNR?w~_?bFMhvD{1%^vQK~2%GE+US1-P&XH&LX
zDQxnB%D23(FSoW{JnNn;o!3%+HT<pOxhFqYF5mDl=~3|AKAw&<bFN+49q1G7%~dst
z{bXjm?cC*}(VA0!t-i6?aD~mriCZRZ&AR1(x#`FHZ$~G+ZQHbJO>Wb1R-t?9tpVz;
zQ_nJ`-nevlau)mEz1{D^w=TUB68GKq<e9f}0#DmRR|IEfTL&9;+|J7XaeMtPtLHu2
z-aOi&!MtWsj%bbY+k@5@{$*BZeV>;(efEaNtW!?4xr=WFo!a)a!yt%t@{WS#2GKIv
zlO)RhYqauLU$*r;wQgJETiMk+ujOgA98sH@c6stA0k{9!)2~2Ew1$+pIZfQl-t?Y6
z{6lBM`fZoP<|gM_+&Xdp1nZ(#Y?<?$-7<m<Gt3@5_;zTz-u~8udq3r>hW&ca8?f@P
zV{|9CzenGj6VnwND{pNovb?@^v$Ch1{0^VxvA4~oOdOw2*_&wT6#P?y_t}2AoE6ia
zPFena%k4Qm&Zlm@6^=MPc~VVu?c5^K`<<#Li;^0Dh>Okpl(qfSm;4{m5sOuq-j<ly
zwMR<k_Q^#tPY*_RyxGX`>VbCaJ<hAEKL=maPoCTpdT8&v5P3fNNptMiK0TqaxUlp(
zyZ);C%ik*Y<u0k1ykiT0S@)Hq*Pi(=qIqu}Gx(mno7?kH{lV=LOBL=+uRWoAH#f>_
zzwKkY*)#e!UHpB^XGLIqtjV<7Yf=yMrW+fr&PkhVZ}$+?IGdAw+}L5){v|73s!Z*Y
zo)RH$mjBIWWu)Z_l{G=G=~7E=So)+F=eIYpSWc8-&D&U!eAo9>Mr*A2{H(H5ZYJ`@
zk-QfF(yj)*E}NtLdNu#A82Kdc^^Li<xy&1$x=t$B3d{?v3*x+x>G(^@_uDn?duAo=
z3Y?n+Zm@^Xon1EnY~Jk+r~6N<<SHLLxieGW%(L4l=S}77uPJvo@g^n5aSNPzXxt^d
zW$*q-v(UVV%4_}W_h_6l%rD-0E}%AVVeCbxJ+t~=?b_G$&879${iw$qy7#>a-~TUm
z%T$xM#U~RJZ=6${Sl52)^o6*q$15Ano^NSW^*gR1{PEYe-OrbP|KwNo>hj6m7jEp{
z^49I#*41}RLQ8YM-HG!nRrt02G5?jXPk4T>kM;)z(jQ4CCy}LDH_zqTU0(C)*WGm-
zmtP*9%4MUIEZ#kD?wXsImxYM8ZJZ$TNsx!@@RcUP+lH6V8me8oxca1@=;~irAG>YM
z6$oD_Zxk)scW2u{w|9nRSrVVZ<jfq!W6Q32ER8x)A|<nKYm;aCjXC}Kvma)e&R(`V
z*NAtw^Jd}Ax_YmC+ox`>wX9+BI#uA>?sTTAbmrM*Neoxj*$$tRnfxIl{TuI6=QVZ?
z8D_DkAMHsjFM3;c_M$_|F|#H5=L!!$UwZuK5!M~L|HQ@rb(*uBUUFsY>6<g=maUMR
zeD^wAS8DSWv$Y;<>!!6V&;9>%U$NBM`n~^`8MDkh{OxGz-jWlxU(Q^Yn{BZ&+V$cx
zF+1M1K^)&c=k9&}?PB?&W|{e)o)r9CA8iW_qoc<nEOk$mKT+1Uo}e1x{kvpSWSZXb
z=`Gn01^f<4Ze`!L;*E*+)d?jxub$0Z-4kfeR^E5BsPN3Q?Cno1-k5LHPLpvA3SaqN
zt7~_=A+HDf?U;vaw<Vua+kW77%BH|Fh5Y;_tG3C7JY5{Ls`rcF9Id9T^#?`XbQeX-
zU!U{SYG#)Cp1+Io*iX-O?bWhYKX_8^LPv3xxc;kC^B#E=&;Oyhceal6fqg#ze}}G&
z;tMO)3EHvl#iCssR?R=y?~-_LN#f_$bqaY!NnD-#m(5Q3A(bBS?(%V-%ukoAn`~Zv
zKUa9-HuH)dvmYLQHYG2Ce^!u0&eo`HR_nBVv%)`i%(`3Kbv)Q+o{`<+zq`dQLQ5Z;
zeJop}6m@D@t#3_CoUfxbV^P(@F#Rj5t6IbKkH1R2{&Z!y<MimVrw_YUUH#p1YR=MA
zQ+6&-JD=GZ+jdOaf@i1W<MI_zv$p?zda1O1+YH~bEf*Bul&pN7RTZ2Z+8JlMzV65?
z_Z*9zzwbS3&#GI-c4UkBv=>E}#Poho-@QC-)!C5UEAysT{mrbp`^Bm_rD*SbN#kc`
zHwC9CEVWXfwvR7A`*_cu`75-q#!L-)UAb+dhv~vqXMTR!6_Z|G|J<+LTJ~MZj`eT7
zI}h*IQz)F6{{PyHuIDy$_SI}YUE;r6V9u-^lW+2Edd>99d_MR2O0_%2Pkh2&CSP2)
znLYPV^yCYtvsv!_tId7}?ix5PxG8w6P{R67p7)}|4-3Ao{&ZtY(z+M3H!vT2JK6F#
z$EibaH|=v|I3f0{%T;&f&(QRub6FkDj$2>p^MB5|uy)Fxy>Dl!duK#dO#gXD(78zL
zP4$%P8w~5d1zTCKc&B<&;J)+CWBIF@P90@2%i_QJ(5Yqr3A2rFyyGmk{+(hQt@k|G
z#^>d~PX{X4jW{poOpH4&v1H{Hkyrn3AK6#??2TPxeOcHWtIuEkmaX4$`Du>Y(S38S
zPfKn0{c_p(@9UsF&pBpq&-{0N=}q^-$GO43jsyz@J(GMt(MtYYt9IJ=H>W11O*)Xj
zGk7BB?~mrD+x73?S$XIin`Lh4t>jFvUz@b$d@|X*kFP&n*Lgaw?KJC_Y4`tcdcI04
z`Lwp`2e;C<opo!%>Te(V3F=hv3r=VVduv$u_;jJQoLlApLy2-lirn4Sg2z3xZ-oeQ
z`?LT2n%lDV`1b6Z(;63_oOZu!Z-|AsHP;@28T$=t;+ro_V{rVWcHw+_h~4d3Gjghv
zc5S`*H%caJZ*!1<p!Bv^k4^8{Pc`_sqF7Mxz0WP<yfYg6H_VK1n=VkCu#(RtS8$i=
z#yGRjOPCVdKEL!k<v8JlmuQ=rLz(z&{?qHOZ|Uzn=l?$Iw5X%?l0Uy$)bvg->y-Yn
zjcM+#*+ySD6x5f`-|n<RMLs;Jjs4t$8T#uKsy{lFJZHG@evw&D+_KF={qJ8D-@8|^
zmuFAe25HaTH`BMSzVUhU{uti%KT_YUIQ6jpHEV9V_xnyW)zCY_-JAWwb5C*qJ?AF6
z{gUFl``fob_wMzbe!GOjdFvGQ$xHaB8q~K<b`CaUI9YM?@M7zGRS%!naG8hP?tZ#u
zZ_XDvHnpQ0Z`U5+c)P<Pa(}Pl(FNO{9&mgyckhK|bMIYx_i$^`EL9^bmphh`?Ci^K
ze^mZ>O+)OBO1S^GS-*?-%$&B~Q1s+fHtU0$IT!a7S{S%-EK^u?TX_0${=oL?+Y+jk
zpSN8-USR!J?{5C@JQfAr`8)IM?Kj+a)n=Vrbo#r-F~ReHYq-|iOo-3>lhE}z=he!a
z7uKYn{9&Bi@$2h%eaDK~R$WgLr~Xy_Rr2f3%%W=b8-?Y88&lri-70e2F)-5k;|_-p
zz0$w$`AThMKI`|?N_Ly^(!U<toKJ76U*Ww+SGs-Icb38j>zF>ie*LKUbdku4{duZ?
z1%4=gNauLNDsfzOiSkoZABkYq9<x6p>(?d;<|KFaa9%FEROE2YXnD8G9eMFguZ(r4
zB-0-pR{L(u?Bf5RLq}!*uYasLXKzl(@xA+$uf{HYYmL@NH!uF_sTIrb9X4L_u&DgE
zO!1EgpR}$qa%>L2lNjvD*!idIX0V5v`|Or|$0Z9MJq|b2_DuBnIW_r&K1=YV1lPJR
zT1`)m`^X&M{r8wTNB)v4FLs$RNXo2Q8?fK(;Ay?PZ|jzOOkBO?-?KgC%qd4N8?o1E
z*`41TQ&u7FQWo-bXYHPQrhleP@>Ua&4Vb+6nbxsu!FSSYm9~D1=q_9Qs=UY9`_ZlE
zd(6B<Edr1K*v<FIT+{XCBIVS5+}=Ue7b1gejCMV&xhz>GAp7;PU{u|l1N9t#1T4bW
zo?rE(s_)Qh$E}n0g@4!oTIne@DX&*F?NO#uN@khP?vDL=W=SI5IlVOjnLj3^uUCEK
zxt3kr-tEKGi3cBQnLq3<+G3WlNAW28lut%FI$v&YTzu8)*Q)&0$Jyq*n|%KET=SX#
z^u51j#+RwuL~i3*-j#LpX`tSfTC4H{r#-DKN_0dYZ!C@yopbJUUEu!Ws`t0vO%t|0
zwyU#F(NBxb<jGE_NBwRO`q%#7^haLVbGr3|1HDlz7rqh7i0<K^zclcz&;H||k5?U@
zWzm@sv+&=fyH#xc?|N9QtoFQJl2Ko*)xqX^p~Uxn^ZQAkPd{h6783Jp<qn%wDJ^Mz
z0SEG3cW>JM@OJb*UmvIAoLTdgtG`%D1SomlRcd^Eoc#}|sQ&)7%w?w-SJ*j|e>pQh
zbp@+^R#Ba$zL9OCs8B|Ca=@&9wb_fg%U3)NdAi5xh}+@Qm2(>W!!KN!d3VEB)i?g)
z=7E*ob|ulz`K6WTybCb8={swmtZbZ)rM~y1LoE?(W%-V-KRK^1wcTp%r+Uv&Z<CMj
z*YdJywin)VK7JVV!`NZ6thCnaijBK{BXT)Z-<@J){T(AWW6icBSDXUQ$H<$1Z@Qjz
z)%)ViwYH_-{GJpQ>|eG;XV3B3f9;wDyu!~sNtjV2;P%3)<&{H8yj{<Wd%M)z=ZMX3
zzqs?U#1f8x{VgBd_VGss&i3n3D|^3Tiq3k4b1S>~_AIxxPmgd7_$P5`w?<N_mgG63
z>2sL-++K<84%qg9V{hWO&i`^p|HsGlfEzjf92yUfNq+sNWhnD}BJ141&J~lHI=mS6
zcXn;bN$VGRbldS<8b7z`O5THOlJ;aCUzaocy+N3!#}XOkSNaUEg)+B_PgtFr9K!Za
zOm0E*xo+h{hOPHiLPItt952zEeb4jvl{uoW=7trOD{maUd%S-7+@MwZ=dM32Q8L%l
z_>!Z2(#6<7B6icWCC!CW9aYRve*XyBxt^P4mawFjc(~%X)T-*3S(8)e#C@tRk@sV`
z{48BM+hzY8ueC=PukCvkQk1Y#{^6Xs>dg~W?`R&Jrh7+upH2Fq^r^j%A2iOfsZn`X
zWo?)IZJ*)0ezR}g$F9E%*J*su9(S%rf!AQkgN&wKfBF1ZiLu9NzFXyaMZPGC=ggD(
zhf&bsFoO?1vWHLV<^5d#=(b_5t;Ztu?#I?;r)RFyl~#GzF<JRY(jEPW8?_%*bC|cy
zJvArm+p3B{pY&@^$KIV=f0fs}V&0LBzb(@K6&(^|zjdPb?(s^eriAtx-@3h@Yrii#
zEUj`<%I~6!)5qJblBIVtoSTbh_<vi~Hrea&js*>6=UX3zgoa%Hxw&8ZZit;-Xl0(;
z@yCzS_aAze9==Mnh$D#AYQ>?|i!W#SOZ#tp9b%zzc~kM@6^!npvhS`<_Ih1*X|3#x
zdnLOpUrAmlS-RzA#`U$OH$>&HU*vMKdbhVSd)M;6%}nA!`yb1e1q&YATmMtw$CILo
zXO-_QTgRhZv$^hw-ojM{5gV2<uRN2?H+Alm-?OiAm2Hc;eREkq*S|wwzkv2+%PT&(
zd^FQ=^(%+-a(qS7FXB%5n#OM9iktIr$+45?S|qlfJ==A&!0li`R^Pf6@A!iJ7Pkm<
z9D6OnZV_4hZ|AX-&wW??d%KG5Y}nN~f41jL3lLkL>Tx*nSl;<nvmPIbsgSx=(_9*}
zsARME$u03qk~t^;ke}HWbZp|D^C3Di=e`>z*mv#al~~e0mDRGg<#|VdU!}HK+M|c8
zNzt3iCxxZm$&>ru7bLZfmv<+pV(0H4g8JTz9CLf--TE`DL`?EY;f9Y7-Z)3~)t*Tc
zcj_(sZFhq^XTiqh%g-gX_g8J5AHWvcpryR~iuvw3|Jfe9zAs69b3VB_aE@K-Yct*K
zTs>~9_?#H84GCX;PR*!2|CaBx<a@c=?<-%wc_UMQI}4hAE?n~X&9nMS#C5A&=I$rC
z$M2^4=chljh`UjiROl&c@@&<u6rpRo*XkY?$QAs4{nh<>Tz((*H^2P!bFT1%lM7SS
z*^XFkoD*fT;;_I0>F3R-`J{zr2H*6PRS(JeqZjdZEAPp*(f>Z)HcYn6yRcG7R^&+B
z=8r`M85!cg9`8%uxcO$CVT0}0s{Ow2Qs*#Cx4HJN-15oqr;8l5@3&2U%A+wW*vGUr
z+1*;^TJa-|maSWFZdL2@4hoU}o_(;dH|Dl(`t6Lh6+W_y^WHDXRW01ealPlmyVVoE
zPu`bm>~?T{o9+TDbEi@*;hAf<Z|_if^(eV$>(r|~wvp-5O^*_7(|>d%X*J$D()Pq8
z+NJnvaJg^#g?lfq_e+8c<l@-v%#yM}h1T<9H`|m3Z);Wg{p~fc^Pj(>e{a+-xUuSq
z_M;6`%XYsyo%>^kotyQ%^S`1FJ`<Ydx$CXN!V1QJcgvoicD)>M!awM>?q<LExnH+U
z3fy*b#l@V?O!Ikf&)y7~Y8{Yxr>SSH{gv$+&$DbdTX6^P$cQQ~s6XD9r#VMs;TBhG
z>r2~tIj8&EXm4Ek)pe80+}~nrq^F8JPT}+qx0O1z^~DlDy|gVk&kAm-+n6zxPuY_1
zwmtW$Ri$jFMb<UVl<@VpRF?f)yGVFb#rx#sr!&?Vz1O*&zODCU<c7Y>+ulChW-lJM
zU*}!KvBgEU&-wjKUtXCar_ITI@1LgDlg8Bop-)sg(;sX)U%0(%QPk&6tlSsttGoZz
zcE1GGR`$*dgnPF4tS&OSF-dNr{ie-jMxPQ5?<c*LQCQQj6|TDDl9|uXN}c0+S_iKl
z+P+wjTk?jj$bm;k_kYk|Kf~g%T&tbdmgxpwO}}`Rs}5&GrCMja&D|Orw|8$e&!Q`x
z>vF{Y=!%$mF238zR<y%uy^m&fUP{Eq32WA@Zu<8uXV2MZPu_*}`K4@?j|fW2)T_F%
z#Z|~)>5ciXM3Q_x=DPI$QgFDo_`ofN)b=8_9k(;NSnk>RZL$AaBi^8Y&s#rRO;&qf
z;XOyIyU$**dg%phwrB}j=&P<;vF-d%cI!8ZJ+loiZoP21Yp%fZ!#!g2o|kR;eynOy
z!eRcv{_clq2ao)kEA!Iqw+Zjuq*EgCYmW$&KAodof7b|{o%#hP=zr3+^i8~R>2;^d
z9JjV3-xr$Pndx8SDd96~>WkMWLPX{77hL+^#aSF>5zY5t`;onq*K>uvFDx>)TyAkF
z`k?h4wY<4Cjd4a={NjEdZ!VRXafp|vyx3w=b$aJQ4VI!iWjnu>dOwYo-1Gh5jom`K
zR#`Q;{l1jnzHrT}2f<6P9saOe#@n)%X*ci3he=kOepwy)yy^0{EgriWLj<SKSU#0|
z!-oj7ZVQ#Tq>bCxY$>$2-RnJ7v56<vFZw}@;q$Fmzbf5czvdbH?5VB{y(hUM9=6`Q
z@xODb){O<TvyYyAX=8A&Db4C-C*QK|+XKD}9^AsTagM{oTC-W_it4NxcW%C^Fkk)l
zq}W^2w<<|*KKDEP;`a6KKiEHh`_Hd;6g-fc&QkQy`}tMdg0Cl%Gp7E$>NItY=$xE;
z-)0{uHTY-~%%*nw$tKSC9LIj2X6&2s<@T}%zmJsN{<Av3&Hahx?^AiZWVBxihe^*X
zoFIKKc3G0{a()BWvx&1eO)}+;X1>I<?(V~o1Fw^>>ggY!5F=|o!O7*X(xQKC)yI2R
z^{+Pn8S(3gw_0F&TXXIdgFoBMf(^wtb5HA>U6$Z;WA@==yW)+#9WORcvHh@pzxJ^S
zPyc9Jz4`iJo$s@6E7#5A`W*7x;xPYng9nNwTVDTIa(el3r{B6?{O(;+$XIpLxuCdv
zZ-dQ?r11Qof_e8<a&lUG7g+34_lVthSg-ulA%p2W?K~pkN!ja8tNob%JJHIy@l8zO
zrkC%um)huU`gX_c=HFKDSKw^rl<;l4&Xb-NiFMXD-)MCP%z08EU+l*{!!K;n(d_n=
zSM1k+TK)<)dtKLa+v?crRTgFwKlbcC`|`UExAv@k;n9!0ckJ5a@~f($tw-szM8%vt
zN9y>uFFmuQ!dXD%7k5;Pf!*)VR|CC|Z)SdJk-O=_$yh5J&DryL=Udo+wu~qdso(f;
zq1CKl>1!&sS9IbX|0&KsKkrwC_`;Y4;fxpZ*987p!{Kcyvo&;)!_RM#Zj)EKt!x(k
z_&#EGX&=`Gzb*YH)%Le8ZLyji<9K7gr@ZmoeZeV{CIv58o8njJRQ9<tZKu$it8d!Z
zJwEUBaM4+da~JmgzjLogZ~L+d)kP;(TeuX)ZF*%gaUHwiy_ehT{_c;MJh}PB{p}s#
z{--fZ%<<G4wHr>P>lG$EGttgHkoP2BYucrCY0nLd_FQXyydn7SPM%r*cg?bHaGyEp
zTc;b%lvv`o!(+m7FWWBfiJ$soG<dA}THAscQ-qj4c`sVKBS*l)Gh&&_Gp4OU9G}8h
zE&KOv#?1sb$D}{J_f<GF&9w>@0<7=#|32Nf<kZ8=AmMZS^1l_9<<u8`nzHTc<aay1
z6xh_BVwt?e=ljZoMPgrW&*Js0(63%;-JmJqD`31>N9o54%kqXkt+$Ok_kO6hU+VPs
z>dUkK=}Qh>DSz{Ft;{B~)jw7(+3pt+ee>xX<B}Z5u#9i_CZx@I#NcoF;)9mhG~-7+
z7PeR0f?^AnXxTrVxACRq?l-)4{jHqK<z}Y;@|F~O+xlO8-3oBF;%MAEXL_wt%ET#W
zI#;Y_==sFAEZ~s+^2n^?Rj*fn7LL38F5A^_-|hDPq^xb1-pDdvx0P0Tqr3J=z}b7x
ztb*@+-~KA=+c)jklP=5>is$WIH+^&3Z5!=yzU3l?TYpW^zBgCz_{=3UbQ?~u-4c0j
z^V5v8W|nW1=WLP-_WP_Uw=ynf#TtQ&As!cEWtPQro>qB$xg@Rl3y);=p@vsBkB+)L
zng1yH?&Mbi`|g-=3s0-?l|RenSALRru6w|}bthZ;Rnu>zN|(QjwO?^URX-+i-)^-F
z443ogOqf-jYoYa%A%xTW4a2J%)eo-9KZ<UgHsm~{W3eH_PEUN{x$BzOWUeS);`#q;
z_9AH+cM)sPoo;iw7M@y=YI$e2kgC$ZeNUk?T>?`+diAVIygFlx$F~U=3Y8{J-gucc
zuYaqN=be`FmKrXRO}Z{KJ>M;lw+Jb?bt2hhqxSJ*|6dlLK0U$VTT%N$Z@ywZ{!1#H
zj3rx}+xen$=Y}5n(Y9-&%skD=wf@%^Tz?qtC3;;x`pgu8sE&&hV>GU({N7}-LzUwM
z|C-Z9iu3h~H+1ss*5#jnUh_nWpu$UC&!(M)*O<P}`}OX{1p}SakLGXVxha^>T)zGI
zHf?XMaE^cWulg@cTWph)8UFrpF_$OPjoKAj{>KzM%T{Dvniu8k@%e)+pZ$q!X(KDs
zb1vUr9~0Rh;$=EFzhcKPwv!TUY6?7mryurwzuoPU<vTqM#jBA4bNa*0eyD!AKX20E
z2HRuR(|_#_I|m*r?P^fI(<Z$6R%h~_4c<DQ(YcoMw_Y@VzH-{uoU?zo-dm-5A?<gi
zbslTk?_HBr9!=eRDwwDDRJz4)AwS8M;5==wiS0A5<~?-SdTri}%r<_j*z8XsSySW_
z&DZ(y8^tV=biQ!5^R}nH)7ujdCm(!NapHA||8~XcbE?f2@^hXxv|RT@Og3?z)`X*G
za~B9bJ+><8@=mtZ&51g0Q!V}$$Gp&DztzoA+#CFWt>wSzmfL^JYnX1^1g*RKQ7nCi
z3R}Q2mz{kztv7^j?$Gglf8f&F{uF&5d6ipXhZm+lmQm4FTy7Tf)9aRKu&svC@x-0C
zaw|furUYy=@~<lvn8Y_v^se#s!lz$$FELR!wM><@U*}=pY6Kdls1WARx}vvdBe#2_
z%O{`TiE|zu3R$_e{HW{4FV7Slvy2{Oi-oK?>$SnJ`uSzHhY_pqMh2fg+4yQrP39S^
zO$x7PM9mF5c6j^TEz2C6at|(*y_1t0c4*x?u2kXvyt(I3b|l!&T{Qnz&nxwf99Qj^
zyBu1*GebYrp*q2sQT1a<`QA!_D7GhUj5F7EGA;CbF`ch*lcz?(`pW5Nb!Mz8nfLN#
z;^88*@3)z@ecM$0!s=di<F;eMFPs%GG%}|t-3yj5xpi_$kblfX?T;oOHZ3X*&^FN!
zk_zFR8oImQuVkf^mb+)~yk841KK|UiZe72p?ZpKPG6Dik{LeMLNWcAgW9HYisE?9|
zYEQB1|4ZKREPlfYaM`QO(p0pk?{=ieVczL?j%i+LyHS2HPWjw4=ItWBckeAIcpuiX
z-1g?436U2z@5|Ux<FIqT?up}9ub;ljA$^l$cCeH0xf{zCUU>SJ*?)iSR`nyT?nkz8
zcRide@iF(6<ogZ}`3-U`p7R-ZRJ8`zoLF-9qUP7;W~M{u{?*Pu;PzVf=*rZV`6f#b
zZFpv&a<p2^!+(+NS~ai!<sHAKl-;`^yg6`>>{sV&Z3=0H)=7G9)_!c7We=vWHhKD7
zvNj?2=dFF3(~ULmeVYBzSkJFOJO5Vm^8*{Jin^4&@?X0p%ZZ=g)ZAIOb<0bSJoO;v
zlv@`+uH7TK{PVsY@8*At=PEUuBe>Mb$Hr;4@;tqrw-2-wojz_~y5@TQw6Cwu>#qiv
zyaJ8V=QlhQUdQ=Q@OR-u{|mDJm8^e!6Tb1f;D+^vh=8}k=4P8FZ4+)OKJ=w`$?^qf
zbFUv!&bJN?oz7L4%e-UHxi=Q+{7>F___;~{nZr?cYJaZf*ICyDx|$0_cI|ic^n20y
z++6jl@b9{Vi__+-EUVpheD}I#3CB-v+1<~QFVC6mq2Tg~%l$zJ`#P4Lzb9>pJ!PO#
z-Lz}&oEs+tgjP9&E+?xIz0vA_B>B`Q)yc<in@ax|Qk|wH=M=hTC3{`@nY)+8R&5F1
zpHzN{{g(LqEQJr}&)>_N`e^sHy_X+ddv=B8>`Kl<%eH@U^L+X1fYkP*AMQq)glE-R
zn%Ann4_m!HnE&?Pk84D($#hzF#I7_=`9J%+r{jVB1)kvXP<M_K53MX$d09@&DVe(4
z>HZ2;<sHh=A_r!utLyfq6@_f9O_2(1UX*n%;dMgR?};n9^v+2Zt}@+y_)&%O>Zh)L
z<;-vYZ2Oq-%OT(DU-hrLlXYyjRx%v8m#bi>cFSwm^w|$)boz7r`u)i+Fjq*NPuO<n
z<Wdc*t5@}xb_#T@%h}Q_G~4;tX+z$e%&U@tho+v;Qk~-IAKmxHwvd-+rbO9Z_MZ=9
z<|%739A=-n{qL*h<a6=KuXDst*EL)&^enRoH?@q94Rd5a*|~P#gpErq1d87$O3Yp-
zC^381`coNK@BN$AdE9T_{$`EJmC@5TUCHBDj6SX9WTIzaF>zg!{d4R5rv9UQ%Y82F
z7OW~cWi-8Uakpevl;F&|uKfkB;Lu6u;L*RUpt$wWVrj`0Qk9`kRMXfuRrTBxn>6j|
ziFZzVXE|4|)J@zo*Y4Ej(mkfjlcKbug!N>DQ@6i(+93Y!n9<q^fd)BAJNN8+e_hi(
z?fcA^y$5I7<(bV`H$|y$R{0v0tEUR4&G_c+`CNPk+wVu89S=r_tBDo;4~)C)CUfva
zp>|2sfw^q8H&3kQd3JT5%vDCuvw|NE_szY<VSZxWv9jZC4|mL6-MsZ}!`}09ELm@6
z9$0(GAopX&r|d2L30qg#&edHo-&f$U6`$ed=<Ra~cJ20bKRh|C?(TaNmTi09e_{5j
zc<mlzbaw*lvDvAUo?rX9$?xB~MSDM_u-}k=<7B8@=&-SAa!TT2m-u`CcRu(lzGfyk
zFYz!w(uzL%<Kc{bD(~-Xx;mqItBT~sEwak~MoK~RZ=@^v#%z54M2(j{mnE^6Yhl=z
z?Yv@F+gqnI<y}&pF}G`%(x$h*hbDfn{8ZS|>@vwjF?`am!#`8}Pn6#%JtwC;`Bz5p
zeffyMY%R&RBHCOt<K?!#U%4Z$;MR@Kdd0~<pK|=NiJRWaaJ}hre7$w@udahK_h+Bp
zr+etl-nk2&&H3|S(;@Sj5;-#pbWcV9Yn;A1N<${(>$3j%lZFX5rHi#3eK+phRlrf5
z@+zTt9b4V<uH%~pkAD#UzU0FB3h}V$ZM*l_de8d4x0dO`_kC&h)2{W;-m^$&cJ~yk
z`F)Rt=Ts+t2y6e%vnDuma(8UN>g|S{LPx`NRMQ@x`SU(H8a((Z$kaFE+sY-^*UjnP
zzVq_VRWd$BfBsCEl%nW+%KK6M_NC=AUE61<#*6&6P~ofWRF8Z1@wvgJ(&JxFN4?du
zl898kKfCk#?~^xEzCY7B&Ku~lEMK$w<*lP-we!#YpJ?&+$!*``=M3#0Z%&LkRP|`X
z<otY@-?i;$og(>az8g!f6MVn+T@mXK$I1@b_;dH~3fsQj{a9<!Q<tsUg^vo~&wM>M
zbaoP__XG9E-D{p~f1G%zGI5>FJcr{ipKsHB+EVt2IYs=$sWijVlJ3{a?6YMGp55Mg
zJXmpF#FDy@fCjVgCvWb!BzoKLDo5+owe=Ba^4yY+wpbQM6bcGX?G4WCdK!}OZt~Bk
zJj~DKpDOZe>wPWj`yD$u$!gR8{OE9S*eEjnIIQ|E|NZ~G>=)aOGH<Zeo@DiRyQXsd
z^%myS$K<D{`z+qO_usPEJ=3RrSG8}Qxco-)1IOGu+x9vY2+K`X_EvYFV|A+}^<-~I
z+nc1)1wu|O66NgME*Bj-@G*OjtnscGo8>!8{?0jP{O*BFpSb>XJFnZx+l3W9*`EEH
zG(R@IBEPx2!as5&`$?sffd!l8@0_!IU?nVbBka;ii+$-g3;zUVRB1W?@Vu@!-6>tK
zaK{>64ZbxuE49A+nUy9?C_5ao?cQ7WU6)O71%!zFe8scV>e$q*)c8*?-)Jp&N_(zz
z_3=Bt=dyK&j(iQQ%shEHi9c-1s=W18bLG|hX8ewN^i}WDnoSpX9NhQ%y>Hxu52m>R
zT-9@5+06U2u58P%-(r`+t-O|o+IHC*i`pgMxp%JG`b#mk(&FDG$G1CjA5;g-o$+;B
z?Blss2C)fe=dnh0{@%9AX_i%R_O~j<rlS{^o?95y(Oj<BCL5#q_<cd|tTaRGn-(%X
zLDw}Uww<UeHRBBJRe6$bk?56F$-7*5#g7+f9&dkmCgQEdEQz)H)j`&OyPs4spWRgR
zTFKpg%YDNRF~5~Dk0n+vb5Z}a-|xA|9^G2=`NAjE>-^SO*_oEGF<xE!T0z5a?$O@|
z6l8m2&$K@bUU_tTa_Xb2&c4}^+BwbNKCZuCwMkc(E#BhC78U<z3^O0;-YgHFboz_q
z7o&8GN9tee%675%>B>$wk&QM9otBjU>D09sdk%<{U)t$;f7<rHLetClx3cm~Km2k2
zE}4I<-7ld1?jI92J@AU{tv~Dd)18TLx3dP<6@dk>4enTZY)ClEA}8uJKk8oy`-|gU
zHYK`!HS5Lp>B%p>*XQe!^0#!U)PxHQc9rFw7CuwI{&8d5^sO<9G8vLjCdJ-me7ep#
zEpy9P<u^%TS<}k=f0YO|dW77w6TaIPIp>+o#7*XQ^OdW_rCNRTmw%gMyP7j;d-T6B
zk6r!z?b<iPFHN^_%GO-y5p0pApRe!x_=j&zmd_r6wR&&9XM8{Gbz+k7jFa+3Cl<Q;
zbuJfhKe;_W;K!Sy=S_QT`R68telxkAdARn9>OvL9)ApSGvOgQE@-}^Bdv|^LiPFvg
zy#o(5PnO$mQQBOiGgm(QhPvC)IzOh<w<_P9)h*-Lr2hQN)A!Ln;QS@Y)F2%ZrmDKl
zE3!IW#rL^up_yH$kLtZw71MRr+>to@XlHW)NAdryr?UT^xR&{Pga10eZ&uMZR_ycn
zPUO3taXV01EZLTu^!4J)4<6eqcDsI=J=tRE<7xk0^R^45UaLC3VbZ7bWlFo3NNcaZ
z*Wj{1aQe<8o*N_Y7i7+TH8KC^zZrH-hcC?j<NtEQ*{sF;W3Dc??mk%Ll+ZqNmqo@!
zd#>F9srxg&NqjsXy1BVdSR{E-*-T#dvbW(Yy=M0Z@JmVZUD`K6WJj^as(Vw<OG&@;
zY*b%ZbIUq)*5lUPl!Pl6cFUi-rfL|ud-@uymo1hu#Uk7<dUcLmetFQXe6I4PO(M^(
z?enOfe?R@rw}!Vryv^d*@9Hov=vucw-xOMphj_>L3gqcXOj;(hcKOyF-Z!+kV<#T{
zEI#FWQs?=~s9T2RJlYSOEyI?dx?14oZufa@Lb&$*(B>IM-IEVwS#R2`AA4%jBZ~`f
zH??a=o5lp`9y_#@d2-%zpN;m51jGDfA3E8Y-TSpAB+IonGHOjlXGzQPqBH-z58UYE
z6}NZsT-}f|b?eNM+C#2?{w!O&{no5yX1mIL_T(Hp`}h7fcY&`a_wPMDe6?r$ftN~x
z_ri9)y~=I=<D1<>mE=a-Enm++X4<sc{q~}`f*XAwpDwAmJ=Y~8zq<SMt((7VEw-*N
zv)y!vW0m-`mACy&=1yVFbU)d9{qp3^G4tJ{lOp+;Hyx<=JSlCrj$?M*rpJQw_{8s-
zi7(yrWro*R&kyn2o4{QMW0r4`x3<-sp1FH+(A9VRr|zh?OlzpL?nyq~+vFqiV(k~R
zp7Vz}cFSyXw3Ryg?f)*J1Bvt7;u#de4@AHIk|Gg)bqoKbbK#G!J!3X}epZ)Hp+3CO
z+U>>L%Cw)7;ytc;2Q9w6$m$l*s{foCcE-LhV9|nQZ?-s1UFq;Yesh3KuEn02JYW7w
z&(1yEx$Xv!%gfoPuichbmp=WWPj&0emR(m3`}zGY3v>so2u<55(=@B-n4fyJ&+c_E
zEXoC@J$}4@vc>7P?CT6y_u0JRT^Zc(_1Am<{bL88oU2i_`t(WioZW_xw>s6H=I&tT
z__l0S@ddAgFW%kkOZ+b<88KmTsmZF#KGnx>TsZe~6IZ<T5${a-qp|nqZI(U0eB;dI
zB-Zd9`In8a&zE@&%3*ufANTW>=-Ir`teTJS+_`Nf6++wjB11wiURr(U<c7H}x2GCU
ze=T*V;{UyhMJF~hT|V`0@2)$~%oaZJTrc#!P(1#d<DS?$1&d?e?W|gFPj0Zh`?0S3
z&Q0^s`X+`ykDB-{dLFqH$PrX3ylK~yp5hfhxM!c(eSB4b2#?U&eZqIss^Zs~&Ea1<
zr{w1QT`ot>8r9a!>dTvz6YY0m-;+%B$f<Ac-&0JD@|fLv|EXrzoX7Qh7X;mFE6-Hl
zueSfbU#QbpuctaWRrRNrebL_Y-0bv`TN?SBR?Jr_P}e(Y$-8)`+0iFoeoZNkde73r
z70e~GBiQ{^*RAiFJN8c8@=`zj^vy$#(<>D8zu&gJIpyP}d9Fw2xjGhY(l(rN+SpJg
zq)L4D;(yiO5B(Rv{s>f&{TF5`yS3?g^QC^Sh(n(7`ct)5c!gK*j-0Chd7H&+8UOb7
zqP94`b&;p^Pe>+|7b$zj{bt}@ZdE;3m;J9!cjKvZs(0;5-EPc%w98X(-M1K@Twy-x
zk7|!zF`6&RSDo)&x5w0MRawr5mz%e|SU;KhdgC+3G>)a$UyE>DS|?_7(|>n>|Cw`q
z<}puZ1h!2`J~yN6tmd{26MPnk`e=n*DBxsKuh<m$JI?QusSW?bJ6)c;1b4nl+_!L5
z@y@q_JLX(DB5Le(Q_|maZ}6gbW{%EJRp#$3%l6(P;@bD`*A$CmokpIK!5XnyCR6vk
zlX8Dsn*DG0_DyByR5}k$F)Dmqeq+z^T;GM0=3J_p<YZ%RUG?VMtw56|-l`KqkJc1i
z$UQ#g+jZUL&}Otzi0<ykrgxf4k4({Tdh_w(KBm^2ViR^Wit|18*fnX15btcew!#Ne
z9Z^q@@jP<g?z$#nBa7G$t(=pktG{kP$251D@|=Xs=1H|bYqYFNLVbF;Hy59{@!WFX
z<jJRmKIL}$-CV0-s~K4l+4WxKg;Jk~=Hf2*_b)sWN|%&QwAD3lpZ`~6S@_Wl?+(Y+
zxLpd4wwt+X&c)V6{N`yLFLT{PCx+&o*to8aZ~BcXKO0|Ev0QEQZVz3h{r0=EtJBwI
zx~JaTf3KHY)g7Sk9=i6BWOng_UX_$NZ-rZ#c#k#4-DBA+`K<E@ukz+Ko|)gJL=z{8
zrL4b@<z7;~`}|xf$Kw|x???a4&EDf=`q8^rc+<-xlP<R{`BYT(Yk!_0w7kCI9&xkx
zRM_n0Sxo;A%~&>Pfy=Yv73l}F!fzR-eeZu5oI3quyT-Tb3!m03N&Pr!UP{2tZ{6=i
z0)1J}_St{Q?LVM&e)1*V{aPPaF8UjjC*j@5Qq{9%@`sO4Rie$ViA}oLmn<c(_d7HH
z;}^kMHIH@b)sCKO`c#sovbiSSUB7VCItA8+{3%mEWUq9K^@^QflNs*$O0qY+a0=hO
zZyxX0g-$n{TwL>d(uHd46Tg*LYR}v?NmWhHhJDj?+psyR2P9V(HP7KLd}qG;=C#P8
z?KkUBq(^(+ntn#%+wm<&R)1kQ`>%WR-tzLU>{lDx`?EriE_zeG{r>BavYGF!%3c~t
ztS{L$Ywens_g77AU+P;J+G%>na{{xZU5UiTzg2HGO?vuuzU#gJtgoL#T0{#Z|1R@0
zU;nOR@150oPa5w%l}g*LY&zx8v=3f$-SUDfj+Q<SV_<(i!OL6gdQ-CGjBi_3cjaE3
zsOZ<C!pSMrc)Q2@!UebQvxA*yE>ztS`mk=Q&yK5a*6HkYnKnOUx6n(STcXG9tZun!
zhF-dLU}cA~-<chP+jm`eRhoKk<(tClFZUkqX;rkBepH`xDB|rK!B5+Bl++4;R3zNs
z5B%L~z3u(8k6)G@daBx3vZu!50^7D<((Vg&`TJ|yEpspTZ{^!_XL?ZA&HmlD8s4ge
z{Qi5ZCGPF_;AN%kvbXP@+2J3qYgTRk((cq%vlkqfde?Q<iKRG5K7D*WSk__p-M^ym
zFYi9Tp;p8$=j%d`;&P#>U0<^|PkhYzZ(k{NJlUbj>EOn@Ov?fS1Fvs?BWCu^j60>w
zt@1jHcXdpUuwg_*!-);88k2cVRT@=4>7Q_XnlPzzztPMOOzlTjY6<M#VZ^6;`k_lv
z>qOnFj_rrn6^7_;wm<DQxw47h_QQrLc2-Jjdl~E$cIRZBEf;FY60Bfe5zKD&<(}vK
zWo8Rx_vP<B^xG#`;r#l-?UOGR@E`wwEc;X0(?_;4?O!w23+}LbQLuBtA-&j?i(3@S
z<+o2y6yC`rb8gX&_TZ2Ge)-PNpH%+HxmY*V$o5C=k=K*sw*B5Zk*AD*sqo{fdhf$O
zmD+d5Z{?4je5K>2AI~!W%<tJDaqAY9Z!}(7XmrbPQ}WsUZy#rP%SbN~nUcExf8OWw
z_qXRktNLw0S&V^VeoDcsc-Jge`fz6Pi*Res<e4&NLHvD3+iE{gxwW{Y#%ap_=;IT7
zcRTLCuh`BzQRw0&QKp#VmkzzKDEinI@0t>&TOLw-ob%kGbqg1NXP)epyu#tI*Mmhi
z(rb4g-#EX?wY5eq^w8=H`z=1XurZ{spS$gWzO3b`uq1Z@$J^^-#kLzCyR&UU`1OtJ
z+p5l(v)*m|q9k#3`O>URK1ZKk?WmI2=e{+*6I#5UAC-;v4Zd+H^sCzT_g_K|Iet@`
zyVlAq*zM8nPLWjBUA!OsmIln6a_r&JvIk|GnA(4{Gp>&gzSVzGcVqmeunmjeZi|h3
ze5>ek<{A;B0_#^H+m^U~-Es8fi*4c$BO*VTd<}GH?w!@|r~UZv?`yC{$#+fX+*uj<
z=@qZXoNW(+-qfyYyXGP`UH9-q)$OS^FE6V|-&%O9PrT$(-!q3vi(LGtc4WzDoa$xB
zC<*&=%<aw6oZz!eQ9(O&T2EHY<ZqvyBe}~$ajMfT(+3al)E%umD{hwfHqtllO7EJd
zPygQEVjm?Orrom9>c#qs=|Nnd^E7<gXQ+!Rw0$l;!E)t|$@kZS+qR#(GBYb-=gPAA
zpRYC4Z(XHyW5cAfbvve+PqQhtVP|}I^jyt1(eK_NC*|08@b7O~eY<3<%$AiAXZ7O0
zKl&4EG=KlSW6xeKesb&WcTN8J72<1e7fv{F^p|EVSFU`?4AU|T-||JLWUHp?&pxIT
zINxi**QXzD7xPWB+jT84C-jqW@jRo_(%JKJe6>!Q{1?}o1FpL`8clVXCM!9I-502s
zA2g-&zDeBi-!s;WH2Lf7<NN(PF0M0T`tQ?yvW1^&%a6B5G?b^Ud@0h(`=2RV|H->I
zFPtnz#Jm6TJhoV9;k<Ivy)gNLWxRs7Egt4(NNsxZe#MNVO47cUJ8zrL@i;n-L7m@H
zqO<pJ980hDh1iwHpLb`R>%10k>9(0;FUPw(2GZX)bGW<D`%~k!DkAq&oS*EYHM(D0
zw^zztx3M%UD%-xX>gHn?#<@qm_>Y(IDaBeWm2PeJnx{UshcP_R#_Ln|-(&A=x~CZL
z;P^GuQ&9Wq0_)D*KbHNNQ+#XB<FnZ(H|6{CgcWst%M|`S@pHMFdkz0y<u?D_qMPmG
zGHUI<?NAZh^7Com<IAb6W`9e$p-V0f7;3O+ynDT&d)Da{t(zBW>2lt<`EiP!sp4Dn
z2U!VzAAgy#-}lSTy3f&M%d>s0(iQ*NR?}oJPMcV#eKoWFX)xP^Z4y?VlN#^0Z?Y7f
zz0SG1?VRWRleRhUr<vSm72bZAm(^D+p<0>2U?XG5Tb;vZ@7D3=@!Yomb+T`H$M2;k
zHAbxEJjQ>I>=jukp1J#J(<Wc7w!aT+4GO03=-FQp{C^h9&%PZEH&2#jv>*4<$Sl5a
z-Yok~MgXh#Q&0JQcJ7NmSKQ6`@NM}{*@eYZxBD+%bAk2qQ497%CokruueXzHDm%Y_
z7W?zZvF9h-Ubt!JwI}v0=Tnv51x#<>?$Vvi756!|eZ@zmAk7o*llPRhCx`KT`y~0P
z;r~9!BCq+17uMXhI>Yk0ShV6~o8g<ByDK6ib#+c&eDlq!<m|eo-sTeFH+Ss5yRYVk
zn8oB)p-OER=Z%tDGB0n;d#UBHd*8d$UvF!1F1#h4psmKg+ne3~r1vFTkyTS;52u{i
zX=UNC<@oI)mlt0+SLt(!#mYF{GwWp88P+w~=q;nJwPt(KO0SSTCv)dLkMvY^*kXHb
z!coraq1TUbR<FL38Zz&}t>W6^<KG$r(<c9XwBf@Fj<XEf?7wcb&x=kA4XEA6_OP4p
zSs_Qgc@5wDC1)+u4}E)Hm-Eo>>)iFi?v<xb-uz^L_iG80NcEl56HhGP!Y7;+wovA2
zX8!b2p?L-GGXJhl_@!yFQ`}1-)Trb5>2=Te>RtbZsWt4+d-w;kD*41^$0~g_p4c}1
z%hOY@{#Ofmx9zUOk0<(E*H3LNJepR(zi#sDrURV&s-u<s;xswjYM;-vI>sJ<T#?~u
zLPzDIpO0dF4}^21{dM`>=`e2-GmoL-3;`VoWMyLQYE=~DfWR=0YsQ_TT~igqIAHLa
zaVKwAuc8?T3~qx6lySh|H{(v(uDOaJE*}R7_8E6FcQq@5m>}%Oan86?xNEXv(D{t_
z{QH~=`VK$aKJW9i@`+CM|3CNFe%E#X_uhK@!`Jt>f9ws)k!kx9^laYTr*}55&o8cz
z4!LtAYC`g~Q~#pc73xAm?_c|yd-*Kiv8-Ra_KJjVeDdq_>)Ze2-8Q~|rT_Hf$8Gzz
z#@AYln5_-Tu-iQESJcvfCI9y9>sT3c&-`xv+-0?Vi=0>29(epe_WSGo|L&jX{a;_$
zEOy~$p3KC%DJ$MRzrX*Vbo~E$@&ET-+uitSYE50|1OD9s^8;NT#s_|V_~^v$=(!qs
zzyEqK)Z8xg?B)yw!Mb_xWq-_EB{J>(@yQl1RhL$OIsd3{&Xm_na{_N4-_&_v-@^KL
zvh^3Of9B_(__>L1`}~;C8~1vg`nmJS|0w_bAO^8HL8neH-Fdp~^Uq)V_U_;BWp8rs
z&a}#Ziu-Pcyf(jl<;Oqe{r@%M>tnWF+GECX?cc9<{x8Lc|2v!SiGLk>)_K3h>O8fm
z*^R5555%8pJ5~GkL$>vd%PmZ0c6qZ4yYg+;T)uOhZ^4h_5*NNnY`aomde!+(t!LcL
zms_`My}Nzy{|!I(bN{DXeKOJ)XO{Y(^S=7`^V7zQ?EZh155F$8DENQ=oPQ<@g!k@j
zeitg0u$)<>sVw!D)b8V2%!ew@f4*VMrM-Ol?RoP5LZ4P{H`vI`;vfH6H|-4byZ>+h
z?0vr{ZR!5XdG-I|V@|#ITVKBZ^PAF5Q!+NM{^xYh@b`6DgX+(Rf3@knRjJzh`ftPP
zy88uYH<A>~|L6Y{2)$C~_-p;{>(}LfzpefM^<Mn!P+Q^Kr|$g>|I7b>lJS(~)!Fsz
zE4Q<~_^$Oo@U8Br2wrjKyGifYGIMdb?fI`-xjS3<s-r-T(8qVSw|>X(KE15)NJah1
zcXem0tUYD_>F@phpWD1;TKb0Un@_Il?AsF`u`fQt&Sm3??FQH0h<^Qhz2eVTfs==r
z8e8TR)lcmCbar*%r;Dqm9Pj`7ak{FS;j8NK{g3vpKVF^8|NhPL<Hz5v+dNZ$RX?wt
zoOR8W%lm6uAMZVXyxP6vSN}z;SH(@QBEyfSr{4K`&&uw8?EQGVw%g|lY(MO8xV~pQ
z>-igUcP>NFxhFdw^qHHa@^9eMd-S*7f6<@+z0ywBztX<XEl}Mhxark_?b5%y&R2^r
zk+M3nFaDEk=&~t)`BGMGJbx)~!4Ky5ulUYiKXxxx&T`An{Ma=oT#{dWX{<5St)5hs
zX|c-NM`4p)kmD`Rx1X#|O#JHhtZ&hd%jUQI=M_tbZ@KQ@8Duz_ahczxt&_!{Z=G3}
zx7_AlEdTF!EoRqiXBc(y|9LN~894WFpzpSmZogl9&RVZ)Hg#fbVQyN1<)iHBx_OIh
zx1D;ksCV1x!hc-3TbffQ-pbnUebJ6RvaNTeaMI(MdnLBq)_U3}_deGu@c!={Cq0Jm
zVYO4ro^1;W73fg$TKet!->Pq`uZl(2Fz9^`ocp1C`SI<o-hXwMot0Fb<B^=bCGno3
zS5<jfX<PB_3on;{R(p42V#DoScNTI?zqc(ocYDzS*;yQ|>|%2?@+{1@Mz3^Q&G+F?
zR7#NVz5Q7=((n6Be`MU<{W{(!b`IlV;p@|#yUq$PGO-QZWV!0~Qi*9g=K71Z_uKO<
zuenfHqMj_ay;Rz9tGnjK`r@XZT<^)P+a=y@+crNZ=yz?#^jqCt@13ouX5WtZ;k4H8
zZP2G58D8awyu!9rKa33ir?Xvbo0so4^ViF_t>8W2dQVC0y-Zxt%wG#C_SRneYI1hP
zr=Sz=Gn_+zy9-^r@W%9scJ7AXt5W>ie#ySuta&0gb-n3Nhy5YzGqUBX6>hJ5{B%h~
z>HVEgPwXv;D^k4AcJb}+8qKTi?i&P?trukXo)nEfTjI1vjzv(P_5OQKeWhh<;(ni=
zyX561jS8!!OcS}6S*_+;bLGKrwlz-^SBnKU*hxPu({6lvTF`UV^Uc4LmWGzTE`Qe-
z^z822^dGO!ePpsu)x5X%`_5BI5w4S4ch3BGxGL~Q-j?~tZ-kiSm##lw8_>N#slAv#
zTV7*7OG~QLf35C^q4rmWzWhwh|L*!GcI$<N-5*zc+bsMm%SP|Z<9*wV_T0a)!Fboc
zb@O(u`*F?oReWqt>WzCY`{$L;SN|Ot^Xt*mEd~{rA{6-?_v<|F+j+~WVcxyUTic&T
zYps`Gp8aUcPZsC*k6Pz`NUy#%{ri?Lr!01-=SrUt)%{*{QROIO#mq9>pweX<b7iMo
zTHpNEHz(UudB@Z1=biI4Z|0ihuB%+C`bKM;zVo{eJY~mi_kTGWkRO{<!fO#LIXOS=
zo+8_<<WxV4{v{T%2?6bITWma5_lRqjrL}vTo%N0^IkfQMXP-K)=)G}Kfo;1!C^jo;
z@433K-^AnUw@GUI&)<D3@bz_^?pMX{kMpL@Dw`^%qiJWAvuN)%BOR0dm3tHKtopL(
z!`2(`-Rt<?T~O~x<ZhXBW6z5F|BIa7Oy8ULzR)7ZQSr{MKkKI4v3&P=>x3oNS1j+x
ze=^%F+}8Y6+~i~7<DKQ@+e}y<b~=<=U2yp<b?s<UzG$cQ;%PMz$wK^ZgPz{=Jgk4z
zN^<*ab>E`PRTmznSojrNeq8a_X3`}du1=FHvm?LvJzjF(=8bgD8B6`;pT+!dh_3(1
z68ypZZQI+D4O11w=SEKbSjTushMS}A_>nU$+xGee8r@yD*jM`O)7Oihmjp%cT=OQ(
zZT=j-BoT&O&ZG10|5Kg2_4NCNQbFf;s^4~g9JP1Gj_Vprob@cFS%0rdSW&n~?S|gG
zl>h5K2N*sq`Z0O({P$b)EwcVCm4ES?h0o)WkN2V30j@Tem~NFz&pfukGyUMn&^Z6>
z%QJth77V?5J@W2T752}0b=G`dkIsIaHhssIYQccFJ`8<EWiFTBGO*+>y!+2T>NnSf
zfdA^Mjf&n|89P%2*q6!fj&ph46nScb{*|K2AaUi1K6}zx?=Thc#>BSI376Q_Wo*H<
zQ|4&elUG8ELnc+*C#ufh@LQ8lUEstq>D^Y3?lA_mwtA$@{}%rBwq^$Jv&l#AyuQeq
zWc@d)^cZulV|2_u{_Q6fOYX19x$s?3LaQ&@y-<9TZe@?A_}`oVMHesyYg;`I6<BxZ
zq}khbOIg=wq%Djnv?+~J*(8#=G9$mFD=grC=(^N%b$^_6gN=ga|A@I8W=z_)x;<g-
zd(NOtgOkB2pG7XOoz46vZ|$KKVIt;*nu}YP1uSS<d9z}h6(85zHC3rsg%|LuL_MDQ
z^4-4Wrt3`R><rk+wdAF&wbQ2)H+9UL=gq6};1iy9a`x?o2GdfxlOBGXA~4O8`S?7I
zhd1lil{6YG=C;!1JDR!O_Vhjd<^0?InjD^9n&~F7d{I`u!!8k(lZDr$uY5bXNOkq+
zwAW43<NYRTF>HFKS|K5HWoEwPZJr}n%3N}TV)kF!cHbsz0h{!t>K{f=@;*$m-S+Ca
zZuj0C<2}2i-`G{YS(*Rs<-Yoz44Ef`X8-eCv%@;$mRFp`J<b0Wq5D6@Ry;he{^ZK`
zSiT8DwsXEqt5#Tl%?a|7+)#f%Kk2vBTfU=?kDUG{z1+Je_1fBgj&<LB%mw`)=It{-
zmg^*wK1DbF{U@*4zGc^so-+|o?KjoZy*u~Og+<K<#ePquuWr2mlCSJ}_49~N+iX6_
zM{g-(aLnr|Ow38$@zFT{+uc`dEkEdq8GN5~_2g1#uDXwh_&hi9{9Jwit@)SN-{bqk
zSI_mTbF9nV^CHBom~*}_>*`C-9YXnU$K1(Ue`Wfnow2^>cm2)p;`*q%seM(R$EUlZ
zQ@b?ZW!^uxY}LOl)!A>)W_>E2c>KIRTgI&CTiXBWxy-$Fb5@#WKTCd}sCN0+9PhJs
zw<bN<Qu_VT?d>V+%;$LNUbA15J#W>tLr=wndkpnu&2pzNnVzh-ZXuufWnTS>wSH4=
zZp}N+B>cv)r|Q+Ge^Spg)(e*^NGe!9yz$Q~DyXl>o14Ma)9lj@5ue+UPu9;CI#I6L
zd}aEKw4M)}tNxbeZnEmVW;m(*>cd5C+j1?^E>tgT>bI4CowjsE&d&PxqW(+Xe%!1x
z{rz@Te+R}hMd@0*=XI!LOX!s}Oj$qqb-nKP<L5U1*3ye!$|vU6#5b9{nyV`C{#3_P
z(I1{hZ5997p(1oM|3aOJ)77_kbDlfaXa09=y>p}W!&ftDPwS2Hil_Fu{BhYXz}5OT
zFP8tAOa*h@e2!n7F{k$jxU2k-klNn8x2M>@Zu^w=^L8)h>YvY1YPfsu_b1gav>1ii
zU2-@1t^96xbnf%QPiq8HejDF!Qj(VV?BKlF^|@)z8x7Gng|h^0x4dCLw9xOEv*JXv
zLy3&{U)vT$)x6vgUOnZFQ{w%j(_^$h#$VYy!68+9<D`i!?wcI0md*1vs>|}0{9!Un
zQ+L;%dpqM;J?l=sm0)=LXotJ8?hExv8(a0B*vp1<O<84o{g!m%o0{+(+snb>&;M@Y
ze{wqS*V$Npd4H8m9-ICQM_c~sN*|;<Ro-w-Up6=7sgmYa>CXGdRQ?tz1Wro*zGi>H
ztR>QnQ=U{lFTQZYKWEJvX^X&BDwledRsGx_zuw6kc=ymohb$S+NQ2bN3mjMN>1$X0
zIqPZa$=x&Nm2l1AT0LpX%ZAX1Q`gU}m{Q25GgF`WKvJ+_%E=&k;h1$|pJHWaFL)R-
z!ELpMVvcQVXvl=mtXC)XOuIPq;`(NdEK>)Q?qAXU3-l&mJi2YdhAVwD6kq-@v)g@7
z{C3?{x5w*Zf^7|3Jx-Ooy9VvMzrs9)|BAt~J^9l^<NAaT<zHj6+g*C>rl^Q<qDpYm
z_AL&f`TrlrnJY+~wsL1aw5j}POX`W&$N8@w`}81}r~BKwc%Ax;z0dl~cZ4OEoVg;k
zz@>Vn)1=p1Yi~%*XYupn{Km;0wc)x(n``y^<$7y2?LRItKd4{jhH-nF;Un$MU%r2j
zx)9{7a!vih%xNoTs2*up6L;q7qdiX->ov|h^hW4K;+53j)whg2glhLa(!Er@*@tn)
zg{!4fxq@q^Jr~@4S!wV6r9Y<Ly|}Gh*7$62kwf&?FP5v!q-6DfERFE5t&Ce|>pi>n
zboZG$>y6vrZDB3g-K?SW{MGuV=I{4Zb5`BmI9K2AN5~h4PD$0r(dKtw-TqhS(R6*0
z<hkyEH%TsP&KJw3t)IS%`{uRq3F#s!OExdmeR6vJ`BKmL_Y<Nl9agJdR0}>3@wE4D
zzk+h%FU^}j%9fwHzM!UU#RO|V<F$`OIxDOKXMfwVj!&bswmoley}aDX23a8!)lSjU
zl}wGUvA@oB{XE@1>Dx0wh8$hb&f9bE*RcCV-qARH{_pGmp?_cO4cTjdPW9D0Z@KiR
zdpfJEPU)W2{+vG9Kz~*B-_Wz^lV5H27yjn4e8cIxtw$<P+~hwV?P_^k?(OH8wVyaX
z$=>?e6nEBt>%pzOb$1V0U#gFucTMZ!-aT{VH{D*4*l%LBDmpLx(RW_q>QnL0TLO&t
zE-m|?J^#I+^@jPIZv0(yc`|FEebB|UwI5ASvRFLiwGG-Rvt#MErpc!S$~pAg<a4@z
z9IpBCdbaL{<R5WrVXCVSTN$4#&X1HmzAKV%;meoBR=<^IF7b%aX9*2g7t{Uqf6n{z
z-J7GB*GI=}zp{hL?_JS_`)}5*y!}*ttAol~@#)ebdlD0iBin_=_r49DwN(A~qM*si
z%GYM?ZR-kNzM`h)hG)UV%U|~9<gYE7`7Lj+_3Vg{oMXqcC*3~X{fm9ockN>vH&6Gw
zpt?i2Y^K=V$a{A$?QVPZ<#uJjQm+>M^~!5Fl4^bk?J&`r$oYP<#j$q{{GK0lE*PA+
z@#vhjZi7~rCexnwh@3jx-Hidq*o+!`$}>6NKf5h(?8wBIr=m;jCu>}+y?<N#%)>`>
z9x=}U85MlT+j*H^!!t3v?y?e>-p(0;ZxmNa`1U=T>%3IPSAP}b)nl6;gnDFrx6uq-
zTyd)X>4I|`PK0fFe>H9A-WzfERkn*ua#<d~v;Lge{-)_Ok9<3ven_s5oyES{CB1jL
z_WgZnF&xh`ZyH3ZHyW;;B#=Ajdh6VK{Dsk3`c^vYnN4rplsKT=^QG#ek^Hwqo0MMe
zj_=_49p@`n(du(nY576V-|psNInx9zu8K$Yl^>6`$QR0~Xk2jFzVKJ2`7XP)NmIV>
zUnRfObDzHZ6ww8Rx8846?>(54_*IHy>(+TMUw+%8nRDsT)>S_e7-t(R$1CP!YuGex
z|Fme+m6<Gwvqj8re!aWVXL_;hj=rN$ZurRmb7r!?b3?YX@LDy;W4|Ra5>L1BsK4C1
z+jlGL{RoZKd$;j+PHVE7zge?hQ+Tq|uCz{@y_=p+S;YF@VfCK#-?R_L^+d`9>y^C{
zdL5}1&h<1(LHFd6<ap`#ZoVB0Pih_YwBuIV^uf2?hsnH^-LLbj>+gAgN^W#)|B|7&
z%KM53W6Z4u!s$2P-`d-!vby8ty<IA&yq8VnnLlmulk(+XLfbBg95^c-U6cOQ?BrKH
z7q!BiS+7fd4hnlu5<9bV!s7`E{MN5O1ly>|uC7?O^ts65U&|C$OfFk{;zRSi$QRZ2
zTP~i9ZrrFRp4E53&*b$>3)WJ<=Tkkxot<pFJ(vABCUxzn)}L9T4;`KsvJ`JqOm}tA
zICnWzWl~tp+|J;=o92F<&Hn67*ka$M50<PIZhhR6%H9!ZnCD?48eGBkwtoA`Ey-K9
zx1T=u=ylA8+aEHz|A{?SUF0!s{f*noe1_7}v&8SdTcozsaL>Y*FQRULOZygEu{LWv
z=T{Z&`z>KQAI={a+&(u?C1KNjQ?V!BW>v-W``jvy*&5pQOo;b&eYb!6i@fi<{+2xr
zUUJ|>Og^V<`scI#b=!0oX5L#>TsD=-R#yGi+%+cuABXV-UTupi3y+d~Y7-IpUVQW6
z4x7&3*FWs<&haUUx_UnDv2~TlljQulJKlZ$-2YK->A5(Qi@z_5O;)wuJJ)%Z?F^N9
z&wpFazIV<0>bGwfUW?t-o)`P}%Vm?|D&5>w2Pd3}>wNTPe&_2QAt!daEjo0EsoHhV
zt~Wm-pL$)Hef{L2Um+8M+jVY5-`Mi{`YVQAcTXIvb+b4k((!4I!`r!8GxVL7iwD`B
z61#bR_3pOa)t>89QhQ!nMz3p4xLQ;vm-A`fl#?swm9Ol}o1I%$;!$dF`pcrARGG88
zw1j^Iyq`7Maz1xiWV`&f4eKWQ9)BL@(zWz*d}fW;lP4u!tF|5%5{VAClz8hSu76!3
z;#HRSr0j$8>sae&OSyKhW(i5(vpQ(H?#kqpX}Oihqdg+m2wi(^?fXdV`rq*FJJywI
z&i<(T<HpkXT`u2j8K3N8VrWr(d3b(@&6;&zG^Sauey}sD#pmR!xIf`9a$oSe^)$=p
ztNeWbbaDCO*O#@D=GH%Lykzn8?3U2>)an~*2NrBdS^rey;J#NEb+5nApDvsxeeemF
z63?W+Ww$;qd-Xf*XIzG*?t|60{62o1EV0z`zR%V=L!F0*s{{FhU0%Mg-z)j__vyrf
z4HiKk%_dCWA^+}@Q|-?RvoGhI(q8@5pET#UsV1jwK&!rAZQ!4%%#5|^3=uObKD=Hd
z^i4KWlxa;;s7=CwzcuWeCw#G#H_y81>~1TzTjrSbu0=b&Bj4QD-#Jr!_mWS)_s&_&
zE+KsP@5$iyrRwe#YnyJfzdN?R)bIU`t0BirC3^f)uYT9l5V#!w*si(qzjxecshp{E
z;_u(HV={R;kJ;j7#hK<1MUTx()%U1$^(GyT>~^Y+|MqNhX<Xg=dBxc~jXqmCZcuoa
zQZxN_&EF@<mvRE1Bv)-O_$+X%b36CC&)fE%vWbsZ_nuU6o^Pd-9n1Ic(@yWp$*wXq
z7He)%_m}&8RsF%e3tQ8dFDZXkZuna4!}%@CC;GGdvDy7hd^%TPrN!xReUsLidkq4)
zzX^Q3yK-{%>F9Y=ccoabTw|LkzwYP5>oEn>%mrSss8{}%6l>1y5qTv#=k@t#ra3d#
zw=91ueJF?{Ot*pia2tz_dd-I9-VNWjC7dhTWch`Iai&e#)E9YnSG(<$9Mu*zY})lk
zZ+pr*6+L!yVMDLdXWYA=O<8p2_t`_<bH#4#wR$7E{%KCM&14JJjnxO27o2J<J}AO>
z^?M|H<V0cJHT&CR4cDF!iZ^_~wdBL>e^p1aekNy4{OrSUMg7=|{~0^{wwbG>SO=AD
zGoQbuUt(61!*kim#e0LF@7q>-x5<Bxg?Y)Y$e*voilknLT30$%TWUOrS--A6M|%aw
zi-2t(gQn%a`n^PceoZ=anUEl7!#9-x-K_clq`g-88fwXIloqR;`^tHp<HCC(+y<$)
zwm&~$m-nTzQH0^E=JH_P<`c7WZ{!=9=imMKwa;`-m4;^M`n2@Fxfk;%`7ITHqp9)4
z^`qIg*z!0#pSx_e=RE`CH*I_VGwZuDdt>g|{B4#~jXSG$9b9?u!jjcRC(T}%+eCI(
zZ<@T^A<$N0%J$q-jLt>!*Av?mR<FogTV44-Io$1u_Psc*N8IiGpG&qEy?_0E+2>n-
ze^kHSUf}QLn{$8lU$I&D9*BuWf92RdjbZnjzHd1`S0`q+C~vE*i~s+tf35C|hh<Y-
z_Dp!SWaakhe*dZxk4#?^d^GcIwe{yKQ#T!Z)h3^}|NrXU{+7yb>OF!j_81<j^Ovr3
zpBGuuRL3f=S-jo-$G_uc$DeCIEAF2CCwX7~rK}a(I9?Z;*|XU1|G&}qZ(qf%cJox>
z&HmS?{g^!`=INUAs|5D^ay^&(qJ7Ey^|w#eFH73FSn+%H{T*D!n{@x2-LOOI{SChR
zyK?8uynF5MGaiZTpOJYpzp<XVBr5e!#pv~X)ooK*le*p+SN`Ao=Wkv~ZPZEmcNX4v
zs#`A0_I%cmcK=hg-RmO9wT|C4lJ7ao+uqvnT+5kLa<b*|-2eN&`n0WDw|9a`t9>80
z!M}`lj>r6&zJBFi>^g>LPfdBbOn%Mn@aK1azSFT+{_(45&wtZM`Mc^Dp2sOp(ssBz
z$@<)CA-$`aXEdV@ergjpJ6a<2lZ|`-!Rxyk<X4)@Gnx3YeYv5(UNmjTq0GC-{wvS^
zu_82%ahlG(;*k5^;oJVZP4K(&z4_YV+y8B^bL_Hvc(G}{e?@`Kzf;HKpWfVHSC`Yj
z|JUR7A9F0^?d$m6)h@5!|LEcd`?kxcAFn^2QB`eL!?V2qczplGFVCO!94|PQTy!ki
z<gumYW5YQgXUzF1akj$mTm?_Mjas^$oALa~#`1?}_Me>DfAP%oCufcqJWDQmmTdCb
z((<+8oUb$He3dv`<#(=%C*4*p-PUdMyvdv84oBWUJTd+C_4xjiC#~h}ryY;~`Z2uP
z>`lM?@yF|b{&;O@DcS$f#pYjCkJ_ot50fn}%GKMt2|vF1!EjE6kDr~}<nt#_9xwQ0
zX!%pZ*G^qs{_x4;Po5;3{G2oAlZ0QTk6&faG8^^fHf_OjlY`|JYo0%;+5d3L@h4Y~
zXS^yddX;SP)zI>*;jCYC=KPZItM&1#?Rmy|;wQtC8ipb}2Fv*kbNC(Z{jdGEbWd~o
z-I~9@Pp!8<b$$Qq>-%q)H~;&3{Kv2Fe|~-cl_$`@Z@+c_`*VwJq-wv*-)HW4&v>N5
z<d>Xt4ZqU<;}gzv{*ZI212g+QPrP@0@?GImHP6o(9iME$sDit{Q{#R!7$s}K*emWQ
zujubwald&5f7}l_=UN|^zj6=+V*i$N0VC&XAD8NaC*KureV4eEtaJ;+Q26y-;#acL
zukQ+AB=M^_vW9<AJ^!Nm{)P5xNj3an)SqYrW7}MNmaKH_yTY;W3eUbvJWE!324Wzw
znHBdPH;@<JxQ`u-j&G1dWy@K7we48DwD$kk8C`2v*Zz-?tG7MyD(`$7|DIjPAFpRm
zxa)u8kNBM$hUdbT|5>f(-}uRLUcKrcca7c0&x{K`I$Wr5%CIpokyDW37wJ{+ImYb%
zm~qKRhf5VsnKm0t<PxNw?p$l5{=9MdM}{jE3RyOargAf+_yc;`R~-`$d)&DCBg3@{
zg>0LJrgAr=_zQa3cO4Usd)&DGBg2gfg&Z42vs=a2Ykkf!r#|M~{PDoe3Wi*p31;&G
zr1{tMvfntyocow_`^N*fD;V-@1kC46kmf(r%l_gRbLnHw-5(F!tzgLSSsw6d&K&k{
z$Czs#H}3n`a=+q1z72zg9EXfQM;|-eapsoCjfXzAJgj(7VAEhB=ON>-(Z?=!oVn+5
z<FSt|k0bnNzSZ1fyJVZX!gJ;cpBcdkj0u#$<UtASoVr4~P3wt@KVK!z3d5usitHFI
z<s0VscYf2n^l!D#Z;5lkt7^Np{#Uwg$9j8t%5RZeaOynAu8>~Yb>sZWN^pt=>2Ns5
z4zq^^l#s72Kk*-n$^2k-!V@;zhE90@P$T_Mq<Q@ReV<<UfBO3V*Vp&o?k=eRYq;;9
z?f!pz_emdF{Qmv9$Gbmk^t;{vd;B}YqiP02x!!%p1@1SWcyIWmm}wvTL<lok=u@@8
zPuq?^a?W;Yi)#2m$mZUmWTjKz6~IX16pa1qyTq$xrB~k-UNwvFJ02h}9Jrqyj6m#*
z``E$g_ysxPi*hnQZ9Bf%wtbt?@eRc2_-EVpZ$`&ITL^0V7bAAR`2`3izv%CjxZey$
z$r5mO|JI)~IzHNh&`;a8pEEjs+CmVF?a=Yw@XT?SHSZn4$Z!oBo87OL-$`=A{(n_I
zt*;tt|Bj~cp_aleF6+O3_WFK!-QU1{|6=$5ySq>N%X<4`*Y`iazW;i8bH$gxcPnek
z<(^)AvHSlIIfoj4g=32^ewTOzO3iIGeg{B>w!lx@wi+KNP+87C@qY4&<1SO)JA%;h
z=n5Z~pK=i7TnS_Ug_Ya29}dA2`*#IM!iSXA&~m$3c|ZHbec;3nN%;GZUyv8RxSt)2
zj$fF6`cbvOUl8K?Ybf}*S^$iA9>dui&E$m*_puv(Q<(A~_M__m6PvT`mSxoO=Yvpx
z9)xYT;c<1zXG7Q5)m_)VD_oOw-txchhrRh9Tk|#S*6Z`zBJcBT-uE-r_C}>`+3ftb
zK+g~B_siw%{`%>?Jg7Y|t$2I-;p6LyK@Ec3=W+7e{;ZBa{D1#lfx=U=FW%ayPF$|>
zNq^IcV2Mg^<-hUSQ~vt5N&3Zo)NB5F-RJuKqQjcUUbd)}6^B)P+~@dl=la<$ogotf
z^x9{u++HX6%%*bv?20|*JFTbxsuBOOXT@yS|L>jVotm-ej?*eNr^J|4{(vhp{wxn&
z=xVvhgS9kx(j`|1ktYk=75;TiYSDUCsCj?Jrz#64XG141^JQNIs|swpy56mLHK|K?
z<&jB`=1sY`aR2IKv0dr`r_7$tD4wXGnj93m?5;~{!#c-<%v!&A_Z*-4c*?C>t*6DB
zQ!an4zNh-@+3tJuxIX*r7kz3H$~o_DitN3xHOH0C9ZbA<Wli*Zr=E4b&u5jabA4`A
zGTr<9&R27^&shp<yFLr~srG<beW|(j5zXZ#!S;F_k_Foo(m6{H)lZKo-YIpxuJq>h
zk~NE*ZSR?${x^$tQ!;1uzWpJuzOCOsdCD{`k<wN9n$}hIr|M4azU%LG>S;%`YSMj9
z!>PIHg3-HE60Oa(mHyr>uuyWkx#4@S@_k!P?&XKB%#C#axqhn1tt0=sg(5$yed3?6
z?|FU24)(30XHM++|F+j_?fQ*N7Z?A3y>V^5&9+bbe%s4O*Y&@(-+wXr_wv|(i=}PS
z|JTL*zdyx%^*oQ*|J|K`rhhc~vF7rw|H~gYum0c7`BvT2$$rl6xcmSAeRq~<Z@w))
z{hzw?k3G{K9Q(08|Ic4bnKKW6%(eVuU*4?#cYnl&e64v${^aMqN@<rqDB316f3n=p
z_3R7nZCT$-Px+c({Mv8l#oY7Xwie6AsE03p{`8IFS<5M<zUx#z@6rZc5GdPz;0br|
zoyCS?=He4Bh%REhxWa^Qg<HM(zwPH&eB$%sJt{D1%JGvNmy`?k{XZ6VX%5e+_zSfU
zXL7M{gW>6!OSq0o%}-w=uoeRUnzU(gD*iKB{pbFJV|$H`?~l*_pVHoS{pOKc-58LS
z`^~>^_I$UxeBHq;xu0L<TiC$xm<7uM3jujG7Ii30Z$8lSNG3+nLUD%z82Ws;)D>ja
zK0CX>cU^1t(_1?~acFHanzP3DdP{(lhT;?frPUo`vp7^liVIp3oHjUs;T(630G(cK
z7H*c)|J(PLpDvO;epBOqvQC}AL1~Nsouys(+ca)&Qa-1;`qle)p|7VH>2NA&Htk?$
zIK$AB_1Klc_4~?y5A%N6e{8ss)pS@tH0F$Ul;zs{`%89hlApI@wu-pc@=ure+a7RF
zvS6{D9e3-go#W-7f6qGq->m)rG`~lygnhe`Ec3)nGt(PeXJo|~U;mUH@IU+V`P--4
zexJ@z>DzNN;4{ZRg>$hAs#z=LCgf(WvX`Fxgv<HejzzuCxXgt&{AY3X*iq2Z&7UWd
zYn6W4a`w5KB^q0;vb5ZN=Pq4X;(EQzPi*=AIfpxAjV?@&3ewp1c=4Lzk`>tw87J2B
zG90&eyCM--Y|#4a{`UIlRg+|9YCM^J;~!_Ck)og4lIR}=&HCF<wp-il6d&Dha&7<j
zhJ67mY}d;?tIyhyFMnYl7k6M%*wa+AM+-e$qt2IT7yk1)uEfsGdbj>t;L^}JiknY7
zKC)=3?_N3SAg{&p(LGx?F=|V1X`9*88};eqB#&QbH>$riSa^SqOV4gjPwijN7T*8e
zvuMWkoBuyV_tz{sJek2%V2#^e`5j;OxBUCQe%?*1<-$ToDvb?V7AgMO-r*kgXaA?d
z{qeI)^fUi0KmPGIe@C#zoI?)<_}UA2_|CNzo6RXU+2M7qb8dbNV*~>jzIt@;{Nalk
zY>Op((hO!T%&Pjeu<+=fd2?4EiRn!<m~}9tt?Sv#_CsIqS^U&ryFVw0dG?h9YNi>M
zmKPR$mlqOVaAK+7?}MI(7UgU_RR<2b7rZmfR)59iz`TI*0)vNeEayyX2IfVEE@1wQ
z^=;q&pUAs=qgQ%&VX}GNxv6Jo-00~%JEQ0BtQ)<ZcV{rbV1_gUPsg!uW(-mcybWv)
z%nMleN(g+emOm9Awbpqy-|ArQ*TM6y=PP=NWNKXDW!;_VH{bO<xA~)4l?qQ1{_)xf
zPR=%u_|0;lSy}GMdpZ5jP1eeuF}w_N&OO@7FHe70JRtpD`??szqxN%C#phaXKcM4q
zE2=lFOxWR0LWX9Tr5N{_PC+%d7luuO`bLwE+*#b~)Bn52;3!9=OYWs(YPmlSNiDf|
zV|DM|TVTRq)<yrw>jx6-Z)^PDE`Q<Op6TuN1!^->Ox9*?IpgLp^Wx#{W1_X$lgmCW
z{hO0>dw$20wmaG3pQn}yZ@Xj7Klz-jRqVW@s<|m^4w*Y&Sm*g>(GruE)r<dl8knS1
z-+sh;%<R{BwV(Cx7$>GQx$NH<`)Y|{--7uT%<+nAcvly{4DWIN|A~2)T-E)u{FJ}`
zy{<XT?8g#SkGVhI@<RId%Q&uGB9fW;kJ~#$w;LqI_xJ8Uwz2TpIqlr0TYLNds_IRB
zAlfNx?RbV$Z>oiu&~cZfPnuyG8=Y#lo;f_X`tSbbiGOc0OVwH&70j>_3p-PAaDM6A
z&syiLCaQcoebk<z3=F?<HSis|x07iD^99Bi3?&RzYz@p8B^N<?d<=ao4U9()3a~T4
zU^~O9^Y;vM%HJIRJZq=ex~bWZZ!tG~ZdN#@-4VJv`QR(*IL91K+db*Ce_6e3KPTG2
z2~Grz7b0ACvVS~$UK^4s64(;b1fR;?Dd1nVY4<vYGZ473gSDeoQH(*1@z3O1Sx5Of
z!m`$h$Nlb}f0R7``ugG-XK$T;`9=TFd4@L(Z$ueNqOR?G*7IVH;P0!%I(^f6;onvp
zQHi7b{NB7i0wIpAx(-$l{3iQIYb9#}6DTiaFqnu1g0xL!doBv{4cPHp*B5e>$S{58
zI=};lGTjjk5fVv^2@oiF;J@a#5Aqcca`s-I>Ne&0{tpUU-GBVO^rpFp>u>ZMtrONx
zUR@fcaf~;#8LR}VpK~4Hb6{Tp3U)~bx!3>+yfIMc`r0$o(`4za>u0tWtc{VrF2^uO
zo+a>9wPU#ci&w07_biw#9`ccC>9!Xh@e_V>6(u_A>6~a`Rb9EjL{wv|Xvy2AdtnzI
zbG?diwAG%`$~EnR(8L7R;EaPJQ398^rUl%pKc~%5tdqp3bT>JbOIIR|d)kF2)h!9v
z^>2Lbw)*{&y)A+f3RyqYck%sy!+7yRLmQik$%2K9yu2A14$jOUt6QdKIyf^+NnL<R
zHEuFG6pW+<BxMn`HZfBIqC}+Z!GFj0mDlb?R;Rz6yR{s~_#T1EFTZv#8Cf2zupg#=
z{wB^hNm8%&T<MpKsaje5;_LFGMLEmXDe%wrpZj?w^WM!7bC15#39GMp>GEby$6Vo!
z#Y$y8lY``xXGMnF{x5B@lSxwNFEJ>c5!3&i;f6KCv+{Q}&(D>=TV8)B{|^7&v+N9R
zXCm!ic)XQaDf>L=xyL)1lXI1KwmWXy%xLvz_2<9V_xD2ybq0g{gnI`zd<->veyX_h
zc~IB-1I(Fo+T0A1W*m9`zGkkCU+|`Y*VZu$omKv}?sAcyyWx28nZSMT^kn&6cxT4y
z%KzGZxY}w@*4{t%g?(?DZ(Yw{u6+7a^jq%yr&$l~eV?y-=GCP7!p}U@65sQMF$AbE
z$Fv##7M?Epz}WXy>$XGfzVo)JTbEC5yQnweTEUWa|Nd#6>72e>p-JoMU$IMm^98ph
zYAP9BV!fn&BzMh;TSxL;-<`W{_|C}L;m*mtdH(lQqATX=?>Aqv>}|sz(O2hVp47~p
zqIPSUM{x@0jrGf#-^}-kJ)in}!<S><%;nEK{bJ5uc)pl<@BDx;vBRv7{s_FBWxV^n
zp7%xH-20B(WH!F?UdvE&y7!~jbO&!!&yY#?RIb)KU*GWe=}*?@ET2#A&W<d5-m2=a
zlRblLo87T>s#%{u%RNb|<a{X_b-Hn(mfVYm7tQ+Y8+?EA{!Tj5v?ta}GGqQ1-`w-z
z_1B_AC%^yk=fb%e=hBbJJhLy};dEEU{K$3g&Oc@KFBp?F+orV5<vja7(MYZKtbdNx
z!n5n{#y^is`~Ft_cO6%3d4f{Wfy<XNKS@jNiS7Nl^XHsf+Tok}Czs@Hdz<OM`utDn
z{SRBV)js|)Prys%xQW@hObbg#-J)ObwU-Lpc`n)dYgX#N&o}>HWZu1E3;*L|tl`U<
zR-R8&=en1ETyj6RxMTS42En}T7L{=q&Obe=e(26gyOWR7Ha9-HmM(Dpbb;BMnQ>uZ
z&YnUK)vo5fwJ_m(@s8be{=R94oW7(!c{HylFJ|R~=h@OX-OKrX{v|k{+PU=d!S?i0
z-vG-v@h!hA&YfO*|HKNZNfRDEdiq0Sv*e4k4VMxab7h@B7Z-V)iOlbs*5SX0d*gBO
zcaJI)GR5Nx&!nE<Qhf3IV#ue=Jx>$v{Ej*I=iT@3>saKC_P>%gGv&Wu>9_Sgf8yDg
zS`+_|ElXZhMzDV_ycAO&8s;=7!0zZ~Kgl<H5^ZFwL*u=}<{CTS>_0P?P5<ERGOcFi
zd1C+9D$JLBV0LrD)yKmBF75LAvikVPbtgluQ>J@9(_}eSa`Q*<wLZhM3vZYo%$gi;
z%Q<P*i<jANj|aS2Vfp$&^x`;=u8pTyjkiZt{S$Z^k*u-ZbJgwRr}j-@@R@&Icaf|K
zQ=9R%0;l&MYbH!->F0~Su|(nMjH4Tblh=P{T)Xws?tOn9w)1;<{8y8#%saol{n^jm
zk|DcKR#mz`in7}sbX+uO@0<IV)#G>~+rRJ!6+An*{porB+7_eGDT@MD{;b@-ujSOv
zTTg8(Uo4!Wx9fo6wxYY6^QRro|CaXg^qRN;Eg@E}Ex!X=?nn3SoZKDcbuQ=Aku0gE
zE_ogI*~T5ZwTt=Rta|nB;3Ta}wsXZkuT|L_QuC>A>kTLCop&ya&t%{+3!Z=O$4<se
zn{Bd=^VP3!a_X8mxAOmkXX1X}gEJTEoLv)B-kD}&ahf|vHCFZ3$+HWRC%(xv68cfE
z;OA%io_AsWvx$$FNcdU#eh*mp!mM8Ke4N~@SI+wc|M{5h6V~@>-(WKR<;#fK3)L={
zn7y*PQT;|pc=`?V<0_o2E&hGGyM-pl&yKNYzg@gYSN3ep^T}J5^iO_L_2lMWhYID4
z=iOy>vD;SO)$>RS%T4r@_OZ9#w1-7x=82Lx@9!&=9xnN^Ibh+>uD3_GDLDt-33j@D
zMP;k*`pJ_;D(|wzTOEEM?wI~!^OoK9GlXqVcKj$w$jC|l^n=y+)^59ydD3syxBa?)
z`H92qyekhi-!0h?_~m>2ThT>n*5_A!Ny`z`_%AlgPQA%<)$y4JEO)uw$?Vp+5aaZ|
zCjI};*jdxhZ#}s}Z?30Gsbx+n-yhG^nJdq|QQH%k$8T`*_`&_=cE4o$%g)~H(EL|t
z$kE$vV!=OQ^WBsMEW26`zKq%+6LdN>boTzMB{@QT_g4R@`&o2^`)={XN5|9m#{a3j
z<I>u&+*I_e@c(rGSMP63I~k~cXRn81yXKa>&`z_@v)|j;P6}AYm{A?sIo<I5Ry}jK
z1x4GOuWp$CDDgwh3%~6q`;YCK=6&qQZ1%&fr3?JdC%NCtJKB5NIdHR!*dpm|A;&IF
ze8#^mcH7*4|J)oV%-WEZzW3JgZ>lT(zp}?vs>FZpmF&E%e{12j8*!}}lG-=c=d-k`
zfAN-$o*$vNcfCetagVlA$FbFSeO|O%e6v}sbRe<%uRwi*WWjE&%1vuFB)=-lVSAZz
z(Dc+JDWeGc&Y97>dp@h&xN`cCKXbg1k9lESs_tYS!Nkaqj~ymQGV<zZ-bt;`E42BO
zYPDVF;*<8&+I;yLHX>g(smwP!WfQ!8a<9IPw|JmRQpA+TFODvD(Hxid?kv`e4saC8
z^k&{&Xza-}ZE2lV{4$SOe#fTzFYEuAHqHHu!24T!Z`6oR&o|k`x#pjVUfnMDQ?|23
z+&NVZijKZXY1HKk-+AkI>i5YTrbc$n-SW2Tm1jzo(W=uilPxwLo_E<o-p^TklkYsU
z+W3?yV$oek=5;1^&Hi~dWqVzc@5*|co>YNX2d5m?-W4~&*x9u$N#0*KuX_2;v%$7=
z+lmg)IU4(~<(S$mX<LTmf*Z4c>AJcsdE8@I@$Gk3jntkad-+?LR5ky8Q@QlRM=I(_
z;lq;bWw$jA&F-9vwpjP*$IJCHzpPj;?wNM_#^R@0@}esH9d<d)dE^>au<p_Ek`um*
z?r3e>puV#{Z{^>P^@3B>l#kV%tmrJ(F8aH~VcLgit;JbgOEPcP&JWX!{WJM~y~yNs
zTK(51ODSz(o+tX|Maed`KWiu2M}58jyYCOz_0K6+@}JzCZ5!0Lbn=4Ez4w=Bd+e%z
ztTb`&uRYI_Blj;Wt0_*}=QZc&v8(n|*GaxCH##5YRk2l=d5+uqboMp9;s>99Ir_)q
z&ayXO^-L1q-n_rz*#AF_vO=rHj>bMYEOkERX`-m->GSR^*Z(R>TTb8oLh|C2bkoXN
zZh1$Z{49!^*A)I~-R;Ewj*bZ3qP@@m*KaHPyra^}f4`ExQeB6YeAnWP_ll=l3!2O3
zm6%(XUpjSjudVs~wYx-*H&@D>E~?t`DEnN#`0wx2o_=E#?ccrW83$iX@~Ye2hsw{C
z9G)IBf78Eb6U#qe`noVBzrR=d>;_%IBIPwHD|Q^NUS0a>=`NAvs{FG3?$*D$w|z)o
zZ9nyLjFX$U{+v?2TXqI#9y>-aI^0|H`3c)L`3jHbaGRAK;ZHN>v7G;?qyPHaYg4v#
z-d}MZ|GcAr8MR&GJOAe6gr3vp6IsfNxTZDfRZh{oHSNw{w?qFAYTDiHSaII^{;Tey
zWe=n!?#W)#7yh#G&f}u@3D@U^rL&k@Ydbyt?f*SJKBZ`m^Jks;d%fP<-wjwe<#v0@
znJ$HmtM5s1=K6oiJ|+_yS9y5<W}dE@K_9nwYkn^~yNsdO$lR^u&-v@!iw<S~y_LW2
zkG|~!kBgIM=)JT)wk51OYe&6^)vw?DK6-`65C50xjXAHr+H>FaZ+YL!s|_vw{Hs`I
z_UTO1|8tN2{$RYxZ0Eaky~f#}1+||aYzem6{U=ZS!}dozpWWV^^^G(2+v)PdS5m~c
zACAuG`Y=KA)0*d=y^lAaI9MeA;pb;gpUQ{5MxRW6=tMuR`&V-*uJ7Qop2NRPZ@5@J
ze_0W@CYfc))NX;DVml`*U0eT6#`2bh(Bca}cb`1b^ZVF^V#(x`IkxvFoO}43{cY>^
znmDOFx#k-Fh834D7Eiu*q10v0w|hI<xT@@B!U~_&dzL6Zc%#nuzn|ad;(|HG8abIA
zPp-$>->sN>=J~bm^UHV(gfBS$T(tf1Wc~NN+|%y=IGZ;2-L%b<*H3%jUpn)CUHV;-
z)H0(-t-3GORz0$>eWvx}-AT{)T&w?gIedTo@QR*P&zF1qCkn4L=vk>0S37;bx%18C
z^Gg<<)qnOc)0m?--{_ml&sQBM=CSQm-~8^5)$zj8ucD{-2>)cfaB!mJ^C_}2x6eD<
z)%|IZ=~%9>X!~}{f!x+<ulB7vrr&h(Z<yZ{@%sf|Z#ZaLEZ+9?kImX+Q$_MQ{#;M9
zG5xb5@<Ym@$&<C)n@X3~NbWgjsL-YIe^RUQrjj_xFE&fht$J5uY`rc$?#o+OpWnWQ
z^|$r*umpc8yLn>Mh4xl6*<<g*rK^7!e!g`5`m+z$1%7OQ9Q@#bwt1Cp|Msdi3w_wM
z{5Q(avR90JwB@)dduDBKVgAB^ir>q>tW9d!@~nH#a*HFAS7_V*)jW~smf-$1HB$SY
zcKkk(FvmORBAwnoS-Lyw&Az@be>UEUz23iI!K%!UJGljB3fOz@`uti#&_MlR;LKF}
ziCxbw8~@mx_(eUs^{uM;jr&>qwf_H)-RH@9+$)0h*2mTbl7F6?%1_!Evm}37y3nPw
zeEj}ACeDG3<x`f3hOQ6oj4eASru}zqx8<wzbpc21vdqe6Ua*lUyBW|~df{~Jv((i`
zE^-{6-t>oq!RGceHoN<|GKVKjekzn#uwLN2q5ol5nXu^Hr7@Y-^D?}CT3OsJK58t&
zn!pvf-aJgea$O+D<<9}zica5MVtX%bV%oF<8>ir^>^)ak{!wFoZ*@D{q;1Z}w#Ku`
z|F_+KTftrEcuVilhwzXoyKGXPeq0mrXTrP7n_|5~E!CU!nEz_apGyDwL{mgB>*I^G
zeX62AJ&dKzE;aM9e_So{w^YYOGv)P<Wp*y{iWB9-9X>j=OKshx^E8?>`}Ildh-v(v
z*rT4wH5M8bpLzdpOZOso{yQnDp`9~pg%iI2E6)47So)`S)*-oM?G$xC|Ff3b+ZDFo
z{_Cl}`QBu9r$F!byb|(jW`Av-eP~_aLGIYyZgH~uPoCz+Mjbf2=-;hJTwlIDnzXzA
zg=f*sRWD+fJeuG8{5PX|{cq28>7UoUDxH6?TzTr0>9HT@&k7a!9{eHo$A<V~$!Rgg
zLi<vW{tiF3^u?sz3KxrPZKf3eTEG7CX^U69-aQT%c6QjwO?3Np@p0|H`F~d1zrFeE
zF}ujMZ@XRl_SZV>I;wdwAxA&y3BTf3p6&Z*&3aO@$faz<p*|12tx{KXW!qbO+_|Pi
z#@Taf_<v$sI^oaNYo{+(h`qbC_T(Gyl}Zy8%a^9LPM$w|VXpR~IHN+1MSoLY-t#az
z-Ltx3iO-V*iQ?`{Ot>!h7g}x=ZU44M=;aia*P9ak%3_r@6P(NLN%5;Boe#<LI-_Je
zZ*_r<q{gBJUXHRsIqP$-zfQi}@i}#8?^}_ZLb(S0U*BB{7R~7{Req(~wr0`Pz`aY#
ztM+ZWnLg`W(rhlvLam4Z@8rozbw`5Tgd2CyIH@UZXT3ONyW*w9C*Q7V&M4HLzo&QI
zg#F*yIeEpN`&k&}W_!Fi?XMv?Va8RNng-XIHv?2RvxdoR=xJf;$YWd~KXIvh+wapG
z+JwBH^d>*zdheWa_t^8vUY8F1WX|>1TX1Y;z=Uk`zR5mOQL%l2m2Xw2uWooAaW8b~
z+xsV+UwTfM&h>*^kNNF<bwQo1hJDSOE^PglbYZ_!(X6w4l2`htspmz^KX&Oywb~z5
zyYFUt|5~=Lo6(dOIBjv6>n^K<x2=+=e7%0s#Wwxh64Bp<4*J&*@vPT+-P@;k*3xJ4
zTM?!=M`D*3PRz)+bJ<t(Xs3Ulz?5nG|F0=#|GWFCTZ+Ut_Y}uJlB+Zx{izJg+-dFd
zUGQG@V)HZ2KMVx)FP}3JbC-B@q5S`(THQ0%t0x5h@n<VPVm<K{f6e;(e+{pEYn4_R
z+P!15z4D&<%H#S?HMtK}D|XGCc4qP2>g0WaZ`p3tZ9Jaa*${oD#cHqNs_#$x?wy;q
zGqURWoZihs%u_XFu1+|-*5KA-zs^4oB~D8%?(5t?bMyanehu3j4>kQ8gKnL=|E6!h
zv7G0Q^y@|YPm~6B{EUyApONrY#pC<!8%_5^HZIGl<m)lD@07mIB)|QFi14@D7n}Fa
zk%?IS$??jPAl+8^mfpH~OJmn+dS`x~Dz{hep4@iFd9Ro5s`#tAaHdtV*K@1Qdp>q#
zEfQp3vZrKsTdtRr#m8U!Sdym1do2ufis{?FVs^<lojQ%T%M-&^yxA!=uYEy~-a#R+
zYxT(iQ|cd_yZ7{aB#Y66x!Dd6w$1%NH!P{FWZ&;Um9ImOR9KyTD-gGM@z1}_Wmh$o
z%M0yRl+2jjqaNS+UAXG3%VWFLiJJHO53x<GaaR5oem7Nz_3h&`s!{(oyqojN;mF1O
zD@ER14S_kYW}7VW{~yhMa9M!-OZkYCmnHXRc(YfwD(8JnZQIf&Bl+p8K|suor(e5&
ze(g?`RP%H8+8*yK)==)sTN65YV&b1Ut}~`($d#VdU%gP{pLtha;F`KEg?m2P#V0i!
zxsV_A|D*@g<~hCGD@0as?vZqg`*!KF(cbBYrV8y$&oA+u)a76K-QcZfpXb)>+@Q4w
zgsWU9Xy1Icv?%6M=L(^^MCltWnS4)=x&GZ1`01O`v-Q*2Pu+ZcqksCk`}Y0GJLhSB
zX^?7Nc-{M@E64tK7k9sE^mG?`{$i8wi62)uMI1Z0%L9U>W_hV!DR#J!ee}eO$3O4A
zo}witH&bnb-P`Ku4K=#w_4XGoyf*pimk{~2`=2TOap=ley{Ok~3Cot}O%w8E%(6YL
z+_>z|?}*)e?|$e#k(SkGqwdS|=ZPgAI(7Q-hTmt->Rg^*<g?*-4ENQStTSWoha7!T
zSt!EVGe^B#S9j%t8zO;?ZD*hST&=6?nY!<ssA10~m)T~gUH<E^Ypml}zTh_Nc`t+X
z*PbP8YCOf;`+gOjV0mIck>{5i&s90ef*BzKAD3{%PR~zWaQ|~#nN{m2O*VJ0+neXr
z_Riiv&!W_F=F~qw9`2YPIB~~;`BP?wW_Wy=Z4i_q-1gc3pI_wS`h#);Q^oD=wj7(P
z-+o+rbG5aHN7429X8WgUK9E^w^XatWvR^LwcKdv`w%#qiq%!5?y^m9DvW2vdIb~h)
z>6Je%Jn`Pt;z{)n^MC&Is&<(>XXX5!w}sLV3U-&>E_+y#r0f3W@YfyF_CB&amVZa2
zQ0BT*>vH9#tCj^#?~OFH4;1+~r)gJd-=vPTSi_RbH)mh)=9}qdwO32O-a!3dO>F!$
zt?$RZw{`q+O`qGBdMl6XZ`^{LUP~rF`?1AbRG?+;&&WJ2t==D^^Y$_C&0n2%b?Kie
z8V}UBU8<7VucXs@?hNZX?TXNg{ymZEHzqso%IjYocwu=~(65@W4}7&;eqCl#3tnF>
z%upD6eve#8+<DDS8mD<xKdqe-_b;LC&q1p+ofq2{9eEOBBgwKY)IGD(v(r9w@}`gf
zzD!;F{@CuIZA*Wg))YOuB~Rz5rqUOg<_oiZ7iGMW4(_#+HTrjXlIOgC&w^Je+-d)D
zWb3lodpn*gP2Bu^k8bp<uLo6Aw11gS<}I-5d|p|<uDNF2;wd4M!p&Y4EaX3BcwQsu
z)#TbsZ&mNLT$rWLrZ`iqYnA@KC89UD^ur4$7gsDf9kk#~gWJ~ClRYo1rqy_^v^riH
z|G9T@lF940+aH=o@m@Ysy*K`S=eF!aLRtRXR_x3X+4^f6$8wf8>I-*2Nu3+KP-(}B
zo0B8sjekAuT`PIpUU%>PDTgZVx<v2das4*+@ZGIU(W*@fd)zwq{Cq4lO`{`RyhU!$
zN>|PHub;JYuAlhy_~Avl^O~xs>W%&^`TbI+RJhxIk;vzMrVsz`F)#W%pUp^a^Wy)v
zSc7~em_<|9%vn^xwp8_oyZ))IZILOuUmxy$VzBUhP2aAdiz#j$R>r4d`q>_D`Yh9%
zrSb4qsHUUCyohB#4=(ci^|>~(t1?7(;T~1nx+yhlHx-?~G|M*ZXeXDo&h<?ho21(H
zH|^>BoVs(<wBWe;juZdi<$bl<;o|8fZccso_&Zj;VsJMJ+I(DVSBdW1PhXpU_2qi)
z_<6bI+IHi6VI4x7PHKzFHSf2XvHt4Tt&`6kv=Du{ZQrskPp#b>?N_QOi(U6upL3|f
z!^`@#`ZgZ@OENEymfv&qa{4WNdDFC%sXW!P@l|WDs`&ktd2hY$;8H7(xd{uT#hc!6
znP2N#!%=i+lH!zk)7kE(M1S3@wn5h_yJh;>j%%+17bl-QD>Pfm{esQRPzIl+?;m|W
z^-V$h%=%mB=G6yOuG`jkidS{bmDDX^PDwHa*R^<~s$GxN8*JI?ekH*A)R&Vz<@Iiz
zobInDyp=jtyI(!4DzUwOn~A*qg?}}V_TARp_}P7e$~&IDGyS_~-e7sZsUS~L%j0>>
z++QbGh8z7ZysbCuwbZ#_-81%4#+{kcsm%9W7HG;m+Nn{Or(gZ1;zvwPSc$<0ReRf*
zH&<UTPU)W=5ZtO}n$xy?vU$F~qEmSCzv5Nicc<A$9{Cqj{Gd-z_0tpYqpzxM7kxdt
zXwIG(_Zu!>nuJoOFa0uo-<9)n0W!BIZwtM&P*5ZPAN#~>lbWrYp3i1<_g=+qa+ICt
zlkEMU$;bHnx}X1--5Y&;)uO^=F`KhyzwZ3rAGK7iD)-F)dq<M<j%_umS3M))*<rry
zO2+J(?kLAA_B`3s*SwBUty^xt^i`Oh_@=6^rXRn%t12D7b$!2`YG?BOivOD@r<U)X
zu914}H)p*7|BEA^OMe`G?{>Fi|NixH)eaM*nO|=!l&U@THOS%Y*JXE%^QSOYwiGDu
z)n1|H%V=mE{zl{F4~C*j`H7EjKVG-s6?aJQS6`K>v(C+_o_msUfkM}cRnzA6=+sun
z2#0IN*uDL8KJNP#E}IoweiUms#-4GR_(T4Tf|0C?*};{!HkteI92C13kzC@l=cGn}
z_twjn-_JEHeY5l*%g-rdx;uZ~eD`7Z$p!NA0VmG-nx3xV?W_BG`tFM#%O$=3P3djP
zV|b~PTpPSE@ND30kIJIAi*`(Zxv+<iVX4hI$AzC#PX8+6^w{C>t*p{>>bLwIi_DrX
z&E-!{_#n9Raem6$MO|BGJ(&II<!8t2_0PAOuu0v#J=xT3?`c=hL~GsY`ju9BVUp37
zO>9bER$8r5-Q@B1tfoy~*6|afF%Rkmm+dY4w`Hxp_*9#?nn}6)pVWHGq~yOkdrQ+G
zZr<A9$0C#V=^r~?bStySXWs4OT8m<fl20W6d*u_l!kgvK+%MNvf;H!Q-9LV0^^CeB
zUSG-t8$=Ur3~uicS@o`8Jkv7%<@?MvD$&cct+roza{79-YLNbg?7Em&VS9he@AW%x
zS`~Is_vaNU*Y$m~;<ess9DML6a_i?O>6No)h`jz>w0D{3T#2?t5viF49>-Tcle!($
z^ojk^m4l0Nr5?J*Ze;B$=}hR~r?ZH6rLy*pr{0PIH-h=)mRZ?N66JVOC;qSC=c;ub
zwM$n{EttQjQ=s!i#pF#o^+$bAwHcKh64i`8Ha%(C3z-S$XV*$@d+Jun*&S-s7;?jW
zn@%WCf36p+n$N-QGQp3ou6+D;@`ANbe$3Cgo-8zR`-HGggR9&Vx2Qx<+>{pIe|uGy
zQ?UBFvWSJECnhD_6AQ{<>0Dp6_qfuB!nh=flw~T7_DeJ-tF4~;>f#%vS5xxs!|g-3
zo*jwaWgAxbJF=>7PlovMf9Ks-<XgC`pYtkWX=Zl-*WEuBMK$|QNqSA+6(arp=-Qk;
z6&iW!%LI-TM?Gq)67qV|JYC8s=f`B*FRv>MHd$ZZDm5)<ael<cP4>!~Zp-{qWm-Kq
z+{=EDzyF%yw@pre_Vay6S1!J|Q@rw6w(W7#U0=Ai?hU^ke@tzr;-T}mAD?*O5@oAD
zS>yJJI+L={#J@`Jw{CYjHU0Z?UyHH0bMFrE%TvD0?G(02Y74t6dGn&(nvILjJUY{D
zw3KV@?suDVZyptm+kG)1ZNk)VZqKe5yeaHcoFZ*Ex9PUpbSD$zrAtpe>U+9t&KWDO
z^%dV1%(Gr<&sw-@%2~@p%l`Cy)R^&GCidZN+mo|p{(fG4t@qmI%|d4)WL91|Zkn9*
zRA^tF#A^1nEA5XvwEXtuIQKT?RG0dtU;H8sy#<W|f;rLh={fCdbUsy`GduNf$F(V`
zZ&LTz{V`f{XZjw^nQukj{@3{-TBAJ4=iHZVCM-u^utsWrE_urwebPqgi-%jiUh4PL
z+0y?fyO(}r+bMPF>26E)4owN)gD1p#rSDI<FSXj_WUs2e{k+W*^$Fgx276s{8_#aw
zGwBQd_BxhTNcY{=bzS?u^9%d-v5WjF6k2roD(BkY$ISg@X1jX})W&ZNKQ(8n$I^X?
z!B&#<wM3uC^UD7C#<-J5y5`#(pO+T;Z$57J*!44vul%;Q=bi;&0#D{9%#pLtQFt|T
z!oR7ltB!`%%=0=pck#LGj+K`}Tq0QieqUFA_E~=KTP|sfEcar)-0r)pZ|{64_A@yA
zS-;$b#5e5fElghZ0dHd`TYdSox-#Qmnzrae_6xsOYlpfr%&~Ys+1k1{d|~z363g%_
z0Y3_sC(T@O-QmpRqcdkr{SbF|<<)JrdX{CITWh9X3>G`SLh8b_c%|&)-ar4`t+L^o
zDpz4MPcEeE`$}(@FY(*doo^aWtZ7xs&eD+=4tBq|;?JhL?d&Q&mcRP_mmXNU?~>E|
zdkYnKe;!@*ZJ!F;Ja)|&S#!TETQDuE`~FOh*GFY)_Ad_7*i|?6A>WsL8<qMKZPEL;
z@7?PVxbm31$e)UtH*>CwdoS$#Q*+@`WBr4<fp(lP_1tbfR=hu_=wGPm;>*)sueAEG
z&8uKVg>J8A|FW!^4i9&m6>nm;-&$a`N`6b~-o;NZ-)3UxdTF1!?OMPxZdvDhD!R45
z_tyVsJh}aSP=Q(N6pg^I>R%UZ6P>vJ>5quU@@TWx+p}NxZhl(vKuziO>ls%CYbU?o
z|C%+b|BQU+5263(&IVu3X53o;Q23;M*(Ia;OJ}~{W)C@G*R}iE(&ql)Ka1-s;;o*j
z+?#s7DT3v#+=}`87N1)Eqmy~>>32Fujy_jDUEKPA)q{)oUui5Ye)dMv%jMg4fwXW%
z$+d^R{fL+u!nZ2j-2Lu4zsr$fubuN|_-SrZ&c9>aweI!_n;k2Uu06Rmb$&p`4S(*g
z#})sqyK7`C?#KPu_hw?bsr*~LkCS}*r_@Wh1m6!44mfn#x;<C7|AlXLy4!!(pf}fl
z{Q4cbJ$u*V)6SDms=B_or+Ie2ncB>4c2hV+^UwBctoWb)qV8qmif><cO?Vquu(Qy7
z-z)LuGruM?ik|*|?4+LLpJnwizc#W?HG1A9(&5olUHn_0Bj_*tO?j*T=XKv~{PsOn
z?%xCc;*uzz>h;`r`kMVdS_SfN{C@lH&&M;*<;wNyUtzizv0vkY(TDqKKN#nKJo+d9
zX;?>nlv@3r^kz+`C*G$E^cp6`bwqA@);%#`UwW(jyjh8Jx28t&z1;innDBqc?{+g!
zJXx?azM^w?y_B}vqOK4gYvC(f8+y37+75@rKi^X6-_{$b|0YPQ)Fr0p`T}#w2U@3Y
z_g-~;`8;srpEvtL7F|D?p%NOhX+zo7hYlJw&zzo$<n9f=+p)3dsD0F~+w3CFo^Ox8
zJuCU;bo|Tb{AcF=Rb1qs_2{GNuaoyT@Nm={{QZ`7_SfE-ntwmb+&#(`s_<%W-BOM%
zE53^_<Zxx0mv!^M!OmRAn2DJpPj<hawRPM6BX^5sr_R!U9Gah?aDJ}Sq-E#!=l`0e
z+4oXJqG090YTfCMJPrb<gJiD-_*I_HUU>X!xYgojcD8lJi~O#7s)nY&sG9P6$$`M9
z3udh}k1p9>waV4-Z|ZrbZ`&3Hi9Gjk%5?d+w8N`V#z}o~kIXf<>x|0D5B)zEO(`w<
z_Fq4$+KT1V-W$s^ww$og$=>U&oVYnDW<^IUSE6Ob^C#BU?{mC6JB8%+A5Y_z?bOdY
zTv9K2Yx1IkrkJpf$xR#IK6@z5`{}iWcx&I1uJs}gsz>I>O{o0-hU@F`ttsUPw`&Qn
za*{i38F3)5zjc4ji&X{B7c97WchPzFcZx54x|<HvR>v6qI42pJ`P1kAjr3{Nib2eg
zC9bwEy-)jvFI}@%uW{M*?9nB!88g4`oSq(cG$MCzh_U<Iog16ui;w+Zs=s(&)Z>JA
zQMXsuzhRyA<-lg<+^Nf=<`;$)-RRR-u2hcV>U*78@$=kYsoTqv920A_-U{-r$~X5E
zzj>$r`<v4d*3R5qTap#u%~>4tM0<Ap@<%nNmtPGCa8-#|(<NubTpzsb(feAjxm|T>
zGp~Q*><a$9%;OI0TrHV-zI!L>hR#}``0ml|*}FWdc>i2b{F>8v?StRz?<>#bcmHDT
znjXmTQuv5x$@3KxKKh!?_j>5sIH8W`@tY$8p>d}&*W6G(`7Lpd&D#|@f`8fGYj(L^
zd&#4|vvBs`)6y=Aoq@tPUTeLcbo57j&}N^&&660v_goJ13sls~S@NSi;#6E^h0f2X
zC%cqGJ*H*teBUpzG&1brmO_z71zJS~BCjHo6oRU3rug0JwV3as$<tPT^68`&4>oV}
zi}_Ngu5Qwt_jR$H&+1LHjo#`dzF~=2Ql+u#l*km`Umv#a`#bA$)*qRvQ`hO4oZMD0
zH@mdo<B{BaQ{JnWfA!w|wwZm(ufNOWPV_iezZRI79{JYs$GfxZx3Sg!nY`lW_U=tp
zrV8g?%n#q);iY+Pf=qbKov+zX!i}4F*4)?W{T62;yD-9g@3IEnD}nb`p61j!Y18y|
ze%a1d2FJLIPG}nZW^YaUe~&rqQlYr*oL}9Od>!YRSFP0C<k;xt?)gch;LEIw@@5b7
zk86m&_;Kvzql`&EJ%SExU)riaIdEauiY+rc+BNm3T^7$UT56VMb+1NGQbDBomzh>X
z`tR!WZCShJ_AU7&F0*ddUCY%eyH^}srDHhRC2Z+g<x|G7v0Q?WH7vi(*)n&Vszv_o
z$GK9sQhnL8-fufK@#67^?PigMYXX+7tk~eHcSz{+^jj5nAEIl22EE!-5Z;;U!4<yn
z_+ph?-;!=mvwyJW!_(v2pKgqs=22qQ5m9+ca)0*CDw_$0PP>~fUzh8UxLCD+wZ(#W
z&aoxi+-Ldp|JXSB%uJ1fg5n+X_JvjNUijYJBYMq`PZMTbIcs4Z=^_<BN%h4R@ud^)
zJuAHOc=Jgy@4CB@y=%Dkr<mt%uiMlwd`Ubi)NNm1{;!yrjfQpgK{G-bK8ck6xs+!+
z|J>O(`sy2)CZ0R0aQRzAWtPzgkMB!r98+GaZQ?DTe(Z*ve&IQ;TXpktuGNK#eEMCJ
zS~~sLGOra|Zu5se5mEZQdmpo8(ELp=p3AKfwE67#Iy~s9@1>{L^w<vxM@!z1UcU7C
zJ=Uyes{VWAr{6x^xJoa5?yO7qMc4zoo2A!A?%T4wX=Ul|SC6Vb%+f8K+md_w`t*}r
z_2RG10#kMy1gfglJuojyYTGht<E*xpThs3C-se>-xG}ZZFEC^2>8GC?r~h}6I4M5s
zf?s9PLo?R0e+Ntg9-Y|D8Lq3X5xti8->eBMi!|PIN<Vz@RUycB4gZTL{(G~RPFcV6
z$R_S(*V*^W|8Y}$Rkrr*((2onFV6nGR@t}cNbrj%zt4w+M=p5&f8~rj>wO;F{hxA<
z^Yo#Tb`e3Z5^=xNeT!_x_9=Yd?h|!wS#kKqDf*KG<^HVX`%`@6_0!33A8j#z6H@x~
z!|Ad|rO$ny<hq7vn7wCTcqd;zAm|W#@`9EJ(&5YP64fsKTbC6d*S((AZtBvxS_kf~
zTgLzA?IHD}6?4D5_*Sgkut?-x%e+l{?{+V5uX#GhZoZ##>{ZLu(78dk^A^<0UiIJp
zMX4vVcDjpFd6~rWcHQ0UT#va{?+MAN`))5frCK@FCD3f<^cBy(nT1?iyy5b<ij_7W
z3T?$(n(7bs<u2QNyF=HYWwSY3sij(-RH^>uYy4ct)_lypzxt^B!V@zO*NeX`DEwK>
zwqm`B>c(Zp&MNBLmWMBIu{*yY);Uf>H-+=tQlr}Yron+PSyUseweC(9S<bV?{FsN~
z{NJTlpQQcyx9aN(DTawU$=<W%GjE<s>6x<j|GU+lg|a*U-|pV{<!r?L^+&FMJaGEW
z`jeYad2G>m{9*m?gLPjTcmneKCRI%iTWT?_xNHv3`T9hktlvV?m%?_G>{3gsZCPgV
zcY|@x{MYkWf4s1<)b9GeD5Wdy8HL|M|Gi=RRi~-OzMSXF&DVQ9w}&jd_+CW4CrWI;
ziJqwBPC-wVTc)SZe06_PRo|kKekS<V%r8Ak9+3-e_lwl3rp0nD{~1zW{NT~&eRrNr
zXRordNfPh<|NH#3@QQiI3a7_Txi;_jMfK|64LPeHmhSI=ey#cP<kGy?KSTchDG54t
zMY@nfZrZWx|6h7*cKuk#w0W1=v96*BqtGd=R&nRF<)6E)mOSnFpmhmft4)yfx4nKV
zI~L`y>g>ptTlU%dzvsUD0>1=;*4>*H+HEm^`~2W-2RKf@)b~51y~US(@ha=qXAX<9
zZpmp&Y^^n0?05TuWk|%o%gc}S`E?Xqv$|<NdM2a0DA|AMJm;$!EbViS*)0q?l{wex
zf=JPi!@<678|yCV%$seO{*&j)E!)NN+Na|@#V%SLFo@TB<m$Ml{pKg}8QYw;>^sfs
zah>JbW5u~CE}LANPFY?q?VfOH&lazp(*(a)SGp|u5SXu&(I@2a$;@n3+WgRqUE1bT
z-Brxx3O5O!Q{MaR&dk5-uEi);G(2)Figr1gqxxA>|D~_mzOa*drAN(IWm-ORTGBuL
z$CKEQjhbEVde2=lrw4P~(qrDfV#brNOB~m#@h+@cQF!>M(wwl8MAl!u6NAqEve%rv
zN@C4(hqLQy5=5uhojddCX+_W^&Q;El-x}l>#%<R9$?E$e;n(|9V$*IveqEu^x9pyr
zwr=Ir&JdYn9sjx&1^SNf{<K5H(#p~HIOq1&W@ioJCa)8@DropR|CNg@|MRD|_A5ER
zUR$y};;6M}@}6H)Hdr67KVH<O75nmJjJB=0h4sYsrGcv&bj*IoX?^<h*SYG#wAEi9
zvt7I#UfmtTD*xiIu1e+6`I}Ddu?bADOkXo$*2Fkb-_#VF>xc7BpI@n<b8ZdCu}L{q
zWy@DPbY_X#&RnMQpLdp_!t^{zy`Ld!TZD4Wj>{}v`?=csr+K_b`{7<&?f9|@i%R;~
zbh+nlNm;UecZSl-=Ycgwm&&HR*SS{uy5^4D^{)C4w{s;w_-*;pwpwEAkFEUqeAlD=
zY~B>?U1y{D_xo(GjQLr%FB^HLd-|8y?@n5Nnk7|pZA-(|%Xa7XulTE7{9^Vlze_Xk
zU+w>5?&j+2@M?*ZZ>Ph~BP(OQc5bx1WNPd9)J|jOblKBaFLy37waJ$MYaw4!x7W}p
zOQJmXvsv`9w~o)2=w7~76S13fs@@HsU2PnDoVI;D^uVsJ;WV>WsN~s5zoQn%gyzNt
z8vRH%(to};SKscve6Mf%)%7iBvUpZ!oIEZ5a*|f)!4)ndawnI#zrLy3tr7i<-Oy+M
zzXqL3^R+yEFRy<OyCbbyc9h*k@y_Yyq6C{xyHeHj|4Rx_r>dV&D4g7-*!Uu5LUpz1
zEY}AIf8DCSwlK>;T`$06Qa0yKjVFu0)HzLh>EFsOkZI3Wy(*SV+QC=*yzG;(o|nhh
zfBr5vq4rn!>XesvD(+@??Xf<dBlYmx?akJCf;;;<4bJ+^K5YJT;i~+fQw;V`ef(s9
zmj6_br)8T&Sd*70m@s|RVqI#b<;}j==T9=v)w1aeX8(QprrPg~*UOJNG5yc(F$JtU
z-lk<e&-ViNVaCvpOXhN{S@87wi4XC=zv&zMX|D?W_LrZx?K+E2{Wg}`rUe^b`T0Dt
zRD7p#ZA*<>wq;FZK|SZ%NlJC)YelC2RMUvI6i>{M@pr%DyE43Y`Q2T;brnyxT|aZ^
z=Wm7edOxe}pZ|_r_HWZ#|IK&ucyHe4i}`t6{>IWfJn{;fK`hKcRo`6Z25*{tU+?r~
zory)aP1krX{r<np@%LrlN~!tI&)q_vZ#!OW^!m{?X6cg1nCOQq{FlW{y;A?}Q6i)H
z#_*{QbwzpF%Rc&FN=Q-upSbb)_W-YY&f`BfUkzsv3Nl@{PkGAcKlYvW`AsK91f=dB
zh(G%7zD%_A^!|T8Z}@m+|B4NHd+2k0pjByVrEB6B(`6rLCkKT+_kA+$&gSov;tl>v
zXOu;FDouYgsc~6Vb$-(9ZU25mF1RcDv?B4S)}Nvak5;?>zkR;scLUd^z_^rUUf;tD
z9W~-#GkDhQ2%NoWd+(E;<qv+mUE%S4R`1u}!j-=pmh4W?US>b{((UTkZLjL~e=O0O
z@FMo^f5}t7q&M<({rao+!@ufGeDv~Ux1EVwKe?Ceubpl3UDsshwfQ_%Wm;EOzYsk9
zw8($9(Jwxht&uhl?QCB=tQXq<{?I)kKe^g{>UUMk*e|BAgt-Mi3C&T<-)`gCHf3|^
z(mUb70gT=Eqk{FfdoS&IeRz66{EUCw=LZD(cipp#dUj;t;-(4qYZiR-=6`WOt@V#g
z_~IL9)L(zu)+4Ge`RR(z<Jd}_Q0J;kOVtXkRh2Z_CNI0|<|lA-*Zz|y0{kjvIugIu
zCKpcZ@UZPG7D|^Cw_CJgqj`^O@_FqwUsIK*1lI%@9=Th5`MLj>&))Ncj!oSiH}$LO
zwO>Dy)DKIhs-54+_xM6jO4(n9*cDqYTmBAizkagEC4Ww`=elnOSN&v<NbD(GT)W$(
z>!Y>p5{Z4%(lsY5YF>Z+bMb-bUai2RD-tV;HlI6|khybn{DOS#c{P!m^FM3$m|Wag
zw0z%1{cXo@uedt>X<&}Prw_{uw5HZ^Tz@iqRyWt;`gd$wo%Ua`U24DZ{tDT`jZbs4
z!+CFh`*Nu4?6j$7UmtDqlJDPtTFR@h{Z-$p@~>47UU$^3|8vUxPjdRBfHxC@rp!0~
zvvjv+#iQ_=*IMCUnwND-H63BS+TP)G<I+6;JEsf(E{$%!knVc)laX?k<NR*nbxlS>
zi@7H~vwV2vT&H&U;^UcfUOr+C&-#$L{@zxfy>>N8K|)p`-LLzMJC@Xk23EMf)_(6X
z>#zBx<udC6H$I)Ued%W7RZ{0RbeQ*k`lZG9eV1mH>z!R}vi}Rhzh5_5Bd^;&>+rnI
zUS77HoSf_WmzuBNz;RTWXOr#r6MNTfxZTkf^-nwW`2WnR1yAn8U#SYQc@yINIF_x)
z?y;%(i{2&QMPxo?+7xd)zt{2Uvq~?6(zge%bE>ee*Id-2yrVaJMq^B>zpldkV3ws!
zf0BM|xTR#HxoTq1?qe0f9HFsK_Pzf3`mvzl*K{$p=NCPDZuZ-*_qGar@qxc=dr)cJ
zMydOyLjEpGjx_B4GI!4=_B#H%^0o^0FW0Y}vHs4l=#W>ltJa;L9+#_kd`8gTU+<>K
zUFmH3^&s-^Qq>9jRlGil``pmawcJ{~zg6R-(ZX$i{Y?JcW}lMtZ^^W!{~lHC*tj!Z
z_5JjAk)XfFm$JWJ&G@0hL$s<cfWi6cigUIheqZf{C0i#gu)lK8<L$>um-pqZJ7qpe
z%$}oKDCC^vzh!e380{B-+<#N@6z8sBH^0bP_Hikeo9j*elRh=%PvLzUrMm5Mo7}%;
zvJGL5p7QlSBZA((D4zfHsdFGxSi#1RS7vH4?akWZdLr}ITdCl0wx5zyEw7k$Z#(j`
zQ1gq%x=$vL@62ZjX1!PMYO+~rEsyoI$DjA)o>Z}o331PPv)pH0vgw|pe`_~5XU1^+
zuU^aH+4{U;S7^EOQl@!)s*%-8Zq)L;pU;22Wn-VXkYnh~mYz+YSnCccuFu;Yf6HN&
ze!<D`-XQt6TgzH?_xJGd2`1)Vm(cSqk`#Uz)GD#F>1xUDfcBzDzb86{pX-uV#r{^`
z=P4W2e5YUibHklJ`wzc0TmHXPe9Gv?c=k8bpY0c}eCV`$bH2B0i;R)d`fbx(GN#r|
zR8}!_t(f-xc;L3}bApu81T+|@hPK~H_dfL^Tz=t|o`c)}wF=0)R=L~_ly2X(c)|Xn
z+JfzkUoSJ(Y`Jk~nz$R+gR@er|H>^tX{o+4uCyZc-Tk9Q+p`0^rgy#+{-5WVYZ-U4
z;NQB5r&)9N`@PuJIC=iGw!8CIv|p@^3|h88>|@;KlLyRSOg&uwFZx||<cj1V)BRiB
zR#`lh-slvqvtX|Di*qy1p3vNpXTI}|)y&HMhy0&D=-A-X`bT5IC-zhOzFfG>xm-*0
z{<8@}TPCc`yXp7Sw$8R}-xkX{14-51`Nz%Tl6|f#?U$N=?_Aic+mW+8)}KpjWwTH3
zNp!wsUb{{^b46kPt(uzOFaP=5Ef6`g*zASTXAQL<?TKsd+b!@}zPurL=e>C|O1N&8
zJq`c+R8`>m#O5zOak9%6D>=qZoMI#~H%EHgq_riV+IdaaSC!ezEj|?Kw|tq;hl91h
zlCK6wI)3l!saKwG^2+CFSGI{iTX<!r`N#O8SGJKwPo$QJyz&p?2~l{>+q!ybDeKi@
z>GRoYXI>UmcD<B%To~Qyr#VUeql$aSi^=bc|J|6D>ruIF%Ksmpt6wJ<zX`K$l{Hc6
z&JRhsdb(fX`9~v5-ODBy|0^iPyEdH5ob%UJD(vbOADyp?(%&v$?2kMaK8JtVhqs#)
zMST7ViQJAa(dE^ZKYr@Y^4<R(<38Qzko^0NvG?m)L#5{@C$GP3x8nV#m2P(vmkQby
z&Q%q8m0x`~scYl^qmx4gy(h{(d%RJLr%yc6Uv+u?_M5M@mo)u|Tg0%pByeW&{L{Q&
z!|mS~e(squ$MV5tP2PXMQV;bCusgUOyEV-=_+(q4(>#|~AEkbLYW%MI{o>@4+WJ55
zulr@ZLRIOcev#P2(~p;iZR&Z`YW!QT+S0yH#VYiZ@?_ij*M6@%H7ji&lbn5B)XL+n
zzJGi*w`ELQd-B@xrMnL;|3CNG(lu638|KVRuZ;0h^1Ny7nH3#!-NSxzl|KK8-9=Wv
za;mqU@wY2J+_UJ=FK(}+TW)_nD7C1CkF_-N#BD_>jSt_;RW+`j?LEcyH)2|TFGEoK
zJ&k&=rtXir;&-CfO?H0z`{=oF`O6z)a$A4?S+%k9sk5wvl9x}YP)2k~-u$N%MN4!)
zeVnbcSd%~8Z1Qcxd+gh4G}{d1t!_te;fi{hVU%-gHb=<Jhkd^o?ibWuP3q`5ob7bh
zaHpc~oA4!1Qy)&BzkH*MdZ71ACG}O>>u;L(e6*{3|2903?XqU{f4vINo$~6v1qtq_
zCQej1tRM9EWXhjA75e%16B+LHeEqk&r|fl7ly_)ctZ3kAx5<&`Qd#}lj+AU$HaBK%
zQg*)7eQl}q`}1#i-zbPW@B2DEWTwQ+2|q$YpF5kZ6*(SU+<3EETmNJH%ZlZ8Rc$T>
z(eIbt)Xm)EmlIadVWqSD**(tlNlSivz5e6)Wa*1JUZ*+TkN78*m+C0(RT1U1-WBJy
z__sjrx|pT258cd_VH1^As9!2Fi|tPS`H$NJLYyqBlJooXgQZokbv)o>pZx05qo_*1
z)3c}g_o?nY-+U$LN^|lbrS%&_y&azl{Zf|7iIbb=`OA6Jl9<pr=F=~1pZZE;^*`z0
z2R2X5Dn0byYKWY!^y^8{J$xi@lhq2I>XyuHdt{RTMcr{=%jk|Xh?G}!Ix}g_aw})+
zgIx~;U(dh0D&PFf-%Y-n`#UcDxVEXXtYvb+vpHIt>pWwg7PT(4{8{S0aPM1J)2l6Q
zeg`yl^S&pVZBM>-dH;=%j)Flu*z^-aBJHvz4;z}e=g+lGC|&XC%4LCj`N#9c<N1#6
z4;OwX!!aYS`Bupv+55k>X8JWL=SHmgrBZlu<xI}6KMi<(>fYCH*ZL+i<<z@-3-A57
zGj*Xr>khXGCDrjC-krbxak|`u>kn^!Qr&&wa`RiWYp<5eGzxTFiaWHU#IH{Iu6Cs%
z7t8bOd!{}<xMkIsxqB4;ndQ!0yQDavaA}|UJOAht>kMO?Ln9_#e`MEnpt|mYM&`6>
zbA7hm=HJvjOLK1K;rquOSJj++|3fonJL{=w@yC`d6La%kn87-0g=Wfgy)^U5fsu!o
zOMNQq{j)c4sg~RhbJ=N6;@bShUtYiU>50{|`zzo6`OWXL^-A*DW3MaE{4AWmvCu{(
z{JC#!`@YzcHu+7LzwOicvppasuHsP>*Z+#iYh)|W{d%AMXR*_-JiF5=MIyYrU+An`
zveew`dg<bMeVt!E2eif4JEUDxZ`+ou?=HDpDL!KBrJ}ry5Y1inm!>?D=-#_6$3iZM
z?d(ktr@Lz+{ocu}cpYfHQARB7VzR`p>t*5-idFt?o15|?^sU!L_w!BOaVxvz)tB8f
z-G5{6zqibnI2!MSZFqWi<E*#~_g2_xPkHwJc**YurfW{G&HpUO{_u-QYn$ft%qs#p
zuSIO+-mFer6t;g2|N37n%U^8l*G!z9C=mVhNXW6*<;FT9L1Mp-tu)*8B-ZJ{r!sH#
z6NT4}D{2qDm9j1SnPaY}=5fhR?EO9`t?vitXEkql&RS`EX#P3%rb8BW=9~Y!^S_X>
yQ=PlRM{Iuozp77%JQi8X@~7_MXz(w3c52y=iw{CAb(IWb=4z*;p1JTUmKgv|7ogYx

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ad b/third_party/icu/data/icu_conversion_data.c.gz.ad
new file mode 100644
index 0000000000000000000000000000000000000000..d5abb06b8ca21e1e6116ef1732c661c815b1489a
GIT binary patch
literal 177696
zcmWf?*_p!m_du}ziqyvcA11gg$iH(vNNN6V<G_Fo2RL4Ti!%;e*v$VXX4``v-iOZ9
zLo79Tk`EPgxrK@9nnwTXy&UW|q1gDzA}{Gp7B;mL{X*}XdRGhDtFVb5Ial@1=)2v!
z{$!E=5;s`%g%<rk<MHD$`;~+*;i?<T|6e}%&pou_azvZr&Oh^tEV^qHUX^?~mc4A6
zv&V+Osn^*K2HrkZR&CvLR(|_lsk;2DPu+fB{x&n}Ve0JAygTgj8$ur2hl+*n6x#o#
z_`q7R*IDiVjZ60baS&12ziQ@Q`+L)R_wReFZ^rL;IP{OdUyJ%AegEf&nr_6Gz5QDl
zx$df6eZ8d0I=M<V@01H0WrgnhCi;1X+~4x{cUer=E6eK~cA{ZF-fAiQ@8)k2{rT3}
zh{Y`?^nc9Wj};H~ep&ug_b%MxlKu3XbDhkdtnF@xTASXp=I&{mzPEK*|4(ko;!Tl%
zbS(pl{_XUx+`m5W=j#<y7Jq%XA>i6s|HEZx?^VmxXTP$qb}+iU{uxL9W2L|o+lus-
z1sz+<ROAr6+eU6(+0RE_7C#fVPU8QdHtpm6NcEpp=Qek<S>MdJ|DZ1+oxdb|WsNzb
zYu4?8<xx+s-<iL4#@7TV{l}B)1a<zWF|C@iS(ZOGep<cI(HAi*%VmCTd!uuHrs3U;
ze+w3g6m7Y?WZSE^>aB0pGJQhD4_RHFHvh>cq0o;x@=H86`CW<-Ob^QET6A#snvB*P
zbFxn@S-Rr3>9WTqLO)OcT==gc#N=m+!-Vd4%XO{wHFoXgklG&b<nM1s@h{Wamq};p
z|GX66>uBG!WEY3|e6M{QSlX87s^)*VxhQ~t=ie6_<F=`O)4caI+5O3{iz?rZ1Xniy
z+|s-5(6(8EcVr~B-v7A#bmQAq5BJK>{`6>(q3E==Z*OrX+2(Bg^nKaV{^ZH0sul}r
znLQCd&AH;tKfxPiYcnJtrf<J6MZY?a*HBJ>rRe>2x8-dY%-Fhn8~@|4kLR;2{qE}T
zr*w2L<BQvdo=i5k&aKH`+7`3?M^Nxv_nl4i)fPTjknWvTId`qix90-Y(f2>QcdwSS
zx?h&Mtmxi1si(D@4?I1(@cXU<k5}p*+R?TCN%H>%+rIS61sr&I|HItXhBxLr&3rxO
z*6M6GN5l91r+!X&bb*sc{f5uF)zX@gSLZoBjJfT6@X*4tAGJAKcj?(bl{qUTb=C2A
z(2Tn~nNO_zHC>;5t=5D+BF1r%5=M6S>UO=>)ZF$rC2E#ipZDz(yPiz(^wFQ&t;F#0
zUitTMm4DZ@nkSh%W!D5Ba1Q<Ge=#hzwWIFjPT8-jC)~G(2E{Xc`YJNv;b+adMH<I^
zCQi8+vUb&~gDEFAx9GS%EqNs}W!r3igXEU_ja6@L)+(Le^0d17?3B4Tw{QO&@a0=l
zD3^`x_xzV%!`@tFkCbd&@m_Rs(S)8w`$R%>kDPGj?a-*6eLQhl(81R7+&Xu=C)G<;
zYPz0hU(X1!>_7BbHNW%Q$8HIMwdYQ`&JN%{s{VMD$n*@p9ZKpUd`stfKAXB)i|>V7
zYsh+$J*DNUDJ~1w%Y>zES8`lCPhD*8w49d=t4uxi`R>d4Q4+tjNN;8K+wT#ou1&|i
zwulS5ZDenEOiSnun6cQ-X-#Rpy!EGD%cmPY&_4Ac{z9Ji`w%m2HOb!gsojch*3o{y
zPQH%)lzP8|wPyN_WoHZZRxDrVwkf%O)qCSOW8?Mnf9{^N_4DdIojK7HUj6l4+g4zr
z_xsA(tg|M%5B&mrmS?G#HUD~7uO;y5SjD0*XO}PYuk3D`eE!$pP^RGD=gQYUUmM$3
zHCMxYE>E7iV&VU@&RMrapN1^{7qDwW%OdCbj}A_m^1b`$u7CGGH+(stKPfUKTVv1d
z#rNKo&ooZ>(xuS1sp@Nq%${pZwTEMGKlWAols)72(}fvHKi?LuSnvI4&5u}3uPJXY
z+&A}{9u&XtW#F>bo|xBPD$F$g-eV8n!v69~`^0;p#m-J_e>L*g^a{0oSGv+$@Hq16
z0%6$;mIr6=*`nU3nRs}y?^nwQ>(^C0<@g|@vs6=H>YQ)Ee?p5><W6RLz3#M`swKT|
zQ~L2_)%zN*v1}<z<zlU}_`5R}>^h_HZQrRycDv_Z7JA3)p#E-u&<xdUF8RwpMz9tK
zS%tLjWYx;aY0G*xdB*0VhwDEbJ$YX}`vm*{`R9)Nf6b4{-0|qD%e0zTnJbu%nW%Li
zx;8s!TgcIlNjm-)i+^UV+{smObsp1``LkmG7%BeQE+4SLp(H%g`uRzh8Sg|(kLu3x
zSmf<;QO+j)>~(oAk#hU4HTie4y~6)b)({jt$A4Ve_{&^l<=VWIkGmH9lzTgCX1s8c
z>Ftt$tJQ@SU(BDiyIo7k48Qf|+mElOPA&Ry;>Yf33P=3*&b=}}``-7D3x3VrWA5X6
zbH?)Z0r`>d;#K%0+444>G#722;PBDW^?!%o(e?FF0zYSz{T7g1TNGWM%lh`yCH)1j
zSM&5(HO^5Bn3BKp+1$-i`<Glh$@A|-pzgFE$M#JKE_i>QZBqQ~mrVORA~uHaPx>9Y
z>DFInt-#{GL$kGBT29(+6W8?r-@7;W>=ZngE_zwE;%EMeis|erOa3KVwCixH`1`%|
zN}ja+bFAY6|C_Gw|EXvG4o_dv|JT}m!jW5(AH4LRYn!vX-0H*uDXG-@JyW`-Pkq78
zoBQOBYs}KU-X$run(F)FCiy$2xfS-Ts1km<q_0k}{C}|cV<)8p3uPofu3cX?v2y>k
z@adoaTq{pYD?O9fuc5uT@Ok&Nb${kKw(g9s)HwTA&G^P?sp-dyO_Oi_=5{|e-$Y~D
zSBthp|BX9Rm-_tWxK;m|F;nzZ@xK>+;f!-9e~1ldoUCuj7ayo8XuoD!j=H4#${$h(
z*P9E9?l+V4G<<&Jr+Hb!bi+LD5~;_ryCvSIJpL2S*sNvn{mt}~WhyUZO0B&2&xwgQ
zvTPCkdn+Kp@xAu*1ig2~ZV9{QWfh&4FRjpKJL=|g>g0nx($ck|jsgGf>?{fJ>koBZ
zD|>31--Ev032NKpZ%$JRebN_MXDFHdLt$>h3e}Tx-5bhn<>ut6={Cx?8Trk+^~GxY
z>&!oY?`fR&-tf)M)4^`_+3U8SMfX~)lTT;$J@vfcoWQB5{H;m*HS_0PHs$bp;=FR1
zjMhFM!&`Fh?0GketfszscPb^Uugj*)JpD!Jwy+cy$573M^JP+PSsk_huk5<Jdy?np
zhtq39kFNCSIa~9!`1IV5A)im56OI15ao?34dSOpuPIbRJ_A})4#^(i{0euIrq!g|T
zaP_|3ZScEd(?5;Axk=T5S2gd>Yvz(VoxZ8&RpP_bjeEj0)&@;4|8LEim>Qb7f5+0d
z8+d9@o)L2Td^M<ECq49>X7#eu<=<4ey|^sp6>sxfb#uI}mAi4eWy!WbYh{j@e)zTH
zo2<urrZ23Mw94*IX?}UV<zuyX$v&~LJ=0Zcc5mOla?vw=k%oQoCrWy!%yoZSJUhSe
z{rxwS7rA^WH)U5(-g$9a$-h6@Cl5#6?D^QnH{n?2zNyD2em=@p!I?O9#gFBWCx82G
z=qG7j9l!BTR^jL7fZWhooo(S>fxj+seqq0AlN#BhB=fnXXLD|@>etjAvkO;G-~3QH
ziu1#MVWwRd5@)UdY4EM9JcM(NJNw3I>mQm(vtGDdr0}0-?~K#!W}2aEx@~tPiKo={
z|C()K$fYMTxv=UJN3P%8C;8Pj;!NFBlzzn|aj5Uj?ccuj#1YH&0iyGZgLsdws#D#Y
zWVQ8Q*sFWhCO@M(Ty5`~t&;TJerd11f8J95IParT|9fAp-OleZx&E>3M2=5Ye5X&E
z9pb1qUOnginO)zmUHJ6&K`L{Vk&p+At^F3IkN)Zhw6?6e<Q<s4V&aP5t97gD{xf=R
zU4G%-TNS36ca87$znG#SBN+bHsp`=U#ggy89@S56`?8*m`?UL)kn^R}+Ac7DF4Z|)
z-pg5+>C@F{FS|a~rrq%7+JryVFGMmvwRMGDdtJ_|zS7|QT#LU?w@as8v-|D({^HHw
zkE)ipf9m{vYxT8|B}QE<mukGU?Ff`&>2Y;@SXUrAbDfgH>2=m&)81`%k7Jbgdiv*e
z?y7g``=%ya`^*2!o0ju;!rrNx|4v5l&D9V3GC4APx2VhiBL_A=(!0-b^y}s`iaRxU
zY=4GY{rxy2CF|$9sdu-1nexK9v@+o0G<$(pQ_8+inY8ER(+cC$0a3>%&t09PzihJn
z`hRbk1^0fe4p=7iQMmZ;>qB}kU%rmGd-=J*<R8L}KmOS7l&!xSS;@0)k6Zd*R<5uU
zFMn$>=Px|5;QO)Tit4ZUH=R*il6#cRcv<#}FXykNOX~~&eE4tWwmAm&Cyz~F{FQ(8
z^6$^=DrT<lCx4T-{`>zlzs83@>Q}9v8dmGg`FHN!&a0{F{<phR6vazAvQ{SURy8}z
zWi;0|u109nuYXPN^k2)ZYj?Wd?roR2KkPuP@Sa^eEbe}KoV@YEXZcHagf+$XKZuzq
z`0L5S`A+Bjxm!-L8yXm#TKH$%-liw}Zr^GuTDZ#K5`Wt7{hu1TUfq>R+VuUO!bI2f
z=bQaAMLq3nr>#Ev^zDu>ob!&KbPrqNF0ml2tfA(zykGLs-3hP#XS_e}G+idq{>Ag+
zp5ow7mr@>yJezfI`MModH;<pHWaaYf<E?)_JIeI9)V^pv)~WNnCDwc}nrZm5&B-#{
z(!{M!P_IvXUw6XA9aEQ=X{Icn-Et!TwBkxBPq&>X%rC9qlfwGQ-h7+3%8H()6IR3-
zTkF+ZD260IvI<OXF*_ue+!Os@rfQK=M!wUmZ8gFVp54~Hsj<8D&c4qdm)*JHbYX$;
z*Qtpt?RERsth;67`DBV=PHIumXLjqU-#VV0^{zJj`EYT`ic_Cf+&ewnW6{&9SWn06
zC)0!u7e(uBk^ZLseRVtU*C~wOW9$k8j%=T@^_ka^r2H@8tDi@GH($mp*~u>Q>9T>>
z(z1DrR-eB%^;gWZ!rjkSo=~p!YW=q^<bO@(yXW#JWjE^=K2iU*How|;$yD*QrEMbR
z_xI%WhkpCs=DK?GF@C1sORDc~316x0@ch_;&40LWtyh1yNNSy5@kD;VLp#sDn<cjY
z^m36`*>81^PL0`h=iYVCovq(KGENOv;nrU2I`wJz?<0F^9gcr7U)7fr_nhg_w}`Eq
z?_UxB`PXpoZbhb@tD|$)yB~S&oO$QdXKBe7rloJ<Hy)bzqjcBCRcAJCDJuRUdMh&0
zwyXP$M@t%q{L2kWC)g!Sq~02D7j;_nCy>K(zt!_8T9f@71zBI+ikq&vD_1l0nfjIg
zr};g$?QpTpmA@-dwc^3QjZ1HKAN}X_WO>y4eVp7=u1-$Q{-zoE<=3iImEG2c;d2v?
zzO$)(n|AEK`tF5WUl)F?anV@z!g8{Cyza;P@5O(gvM*!%?Wi&RpU}ChANPGZ{l3@T
zuvgu7Uzgm!PgkBUb`RNqr6Ryj@sVG(Os(AgipKj9*-GL3UFu(6-?R90$Lb4fl%kg5
z^RM^62cG}VsZ_ISSJr$t&i=Po&#boMfBx*kKdtIB?4tLp_uu$5Q6|y<yJBQ`MzHmt
z*=o<_?G+-w@9aLh?TudX-EDq0=5rnA@h&#{!hdMq;gE9O&)Uu>|8p$;|Bbmz-{!UP
z<IvTUV}8%=%MNdhur%_2*S&H>;3l`H{l7!IroYt@Im6UnBYbG_rn>tTaiRzQOaE(o
z+^CtapxV;;@s`PtDDjTc8r`MJ#gQID=Bl~%Hv3L~jAJ_gt9Ium`LiKhc}KqYyq$Ja
zenQBvcf3n@KmSu>{l2{G;kRq&|Ga$=FBh=vSnk_{*XKDO_r4Zz|MKIdXD;Y^b%rm$
zrSbI&Pl9&;?J09V|G%>M*=z=RU$u-!OBZ#X@zSdO!Lm0vA=7qGY*|Oneg76)#fj>F
z>eW6z`o=i(PMxCl*KduQ>~-o!|L@+^P>Z;A(EFkE%gOr<wQsi7O%Ptm_2mD~1B*=G
zURN`<Z(1aB-+Si5e_=C@)?Ch;`e6HuWxP+<y`H+onoIO;X;5^DmEzkQ$5y+1dRO0-
zvFD^(u=LV$TfeunTz^)tuD70N5_fh>z5LnZ>{DLXO%y!!elyq2*Y%;Cfgk?8T;P9N
z_2>L3=}ON1nI8(RpYHLSc>ltqPv45})SP_yJwHO{-yg<{|5O(y-Q<7r-o|!M?)~m<
z_ZQ7M6!-V)i{n4jcBie{<EqO2QQytbCrs|5lh)r)vsbTOvv+@z%U^w4h3uF$-8O0&
z?O%&;ohp@mvUr-a$*wM4|G)84dzjY!pJuY=$*OqIE2nd3J{Ix)-|hN*f27O)wbMRa
z7h7L%Gv{7L$o0rN*|Tp>|BiaJ{lb#l`wEwN2mR=sb>RPYxd6|%zcaPYJa*H+lJ!5+
zVY`>i|Fs$pz22Gs{-3-WI&*{hq7AErPB!c6ANh7O^5=Z1lIp3>q8=Z2Cr<gFE*G$U
z*Ym1vKez6jS`%_HcftSMmr>g^WmC^bJ3T#rZ)@)Q(@%u&X59a9@l$c_vu})3FE5yU
z#M&e4U9Fz=?n|H7ojJ1YcEeV4tqsEUbLIVauKt_U5vckw{p{=u2VVziync5=L*ntr
z&r4)lCI2MZtnXU+@9Z@1EY9R7H}^~w6t%8r_PzgknevJ4$w8~$)Q9v2OgKJY<YLg7
zcWzQAG$mH9-}6@3w$9?X{leL8&uYGyR?7D(eS9OM@qG23RafT5C#>Z9@awhRuH>Az
zQm)IZlFS>cR=NK)6kWRC<L!UpJBj8ZZ*LvUn|y2iTDwd3@e8(A-V{&xckAow&)GBg
zpPS;c_s)(oTf1k)wU<?2ChdAs`SVt1#Cx9D-^ZphmOh@fQ*7O#pIiUueq84@?FHAJ
zbGG7l)^&bms}i5JZf5D(HQ5h#i)^@G;(Nf9@jA=C7~_J6<p)c}&PcA!-g58fOvm?E
zR{WM)%e|iay}-VO)7#$l3M!{1ojYP?JJ*ai|N6YIyFXSxi()W{xVg5d*)X4{=<(;g
zHBL;9UJM=%ehz*$O$V2IG05pSF*&|sWZ>cCVPoso{Lstm@=-tEkku^aS?ihRnS#92
z%#Aml=_soCb|%JH`DB7d&k}|mAb3PcOhN2f7()aIGDa{)=q@{8CtGult4*r=<ny%(
ztDLsXu6rhSOE$D>L2>id^*sxWnvQWuDQEN~m}&PW-0^IbbeMWy`g+}zZ5N9-vdIX2
zvfN(z<Mk}L)mIdbGvx)x&J|p;C!_w7f%DT9GiKHmvYojfxmZrQVV3ZdISP+iWY5od
zp(lK$P-4arMct$#Ig7bM%cdzE=M?y2vp!Yu(;TOW&>l<a;%5psuD)u%@VvX{jD_{p
z62>CFPo|<K)*V_Mni<J){emB6>X?Njq-Cu+pZEG=t=plUM&7^AiyH+$a4_VX5_{y*
z`(y7n`v2hCeP{uDlKqje)bwYY%mUs_|0%ld2K&do2M-)flF>QVXrwKB^tlP^x(|{E
zr##x-+}M27BBF;mwQlMUu31dgHpdq=USzr)(Jj-R&D^<a<>QkZZ!JqXmVEX3{WXDh
zx~nv=2{N2n`@z!kRET=?G=slqIlODG-C(zDlUn^Q`-PjVa+5TRg4?l;>rT{$dfBwe
zIL~32!Q_5l?~RP|JO&HqwMQP!F*we-B{-w|omQdJ%D+3@5)>l>@^lRT7T-NTyZ&0S
zz27(Q>qnonDlYL~_;~*Q?~7CRs}%OColLOz_!hrePc<%7_mkd_*bloOlplQ0^qT24
z)9<Hj74sQl3if``Ru7tf<1N$s-g4IatoK>#Irc4?-}mM6a*Yl5dv}(tz0shaw`Z&2
z?R8&gD;@Y_TinZ;KUH4p`nju&0sc)P(={fT*WFIG5AF#Tp1<kfzjCHzrsO0|A&%f~
z$qi<lXE@Jr`tX@CPi#EE!!Yk!;r#8F?d8t&s2EQ=<59%wkoIn!>O7@`f2=>7E1y!(
zP|&FPttov*{y@NifElYli_ST!qo8BE?7%BAgQ+vBpSy}(k6e9l))iicsXN2mf+SBS
z<gx`z`>ZrN6!@cO>O#IY3yHwg6;n-D_G~aZB&ICEQ)+NU?8~tZHs}1FHLRH*L{-kM
zI6fioi))~xbMfN29LnB`aZ3cM?Y8A@*!7a-N|c99(w2?I{c|O(^WrYw5}#6cE&Rmm
z&F8XnRQh{Gq&pTmU((E|VyHNB>q*3sA37gR891)0pRjX$$F|eE;B`k+WTWBn36C3f
zO5?6aOg?w@v37`9=dAt<yO-wfb$DF#mrv=$AMZ^I-!$!FbbE1Ep}6@BuaWtQ->+8$
zOxz-WhDmrn|E~FSthFl~uHW!rQ)8aoc=DK3QTV3ke`XgSvQQUr+bxklZKlul8C*V{
z#fL5Qb&e|)1xT>B76wQ>Zz=SH5=}qSrzr-9GdR6x+bR7dZ^CZJYw<$mnZGt{J|ycX
zwBLtgZ)}ZmXTDnP#GRc&@dvgYb)WnrLrC8D(R|N8#+~)=mb$(9GO?*&E#>|{hrr1v
zZXXSw{Nt6qe)+>f5PUp!*N-frc;82h|L=L(^t$PJ({m<smb@4DF6>=c9sg)LXWh&P
zlR52XKA7w&^=Z$dtUOx=3ww@5Hxiq8)ovs}iDd$}tevXc_0?`9I#s;iaFiw5hvW0+
zzE?Bt)mpyu&G-3m@<Ly;>axf8r!T46_WO^Dj%9DhQFCD<^V@a*&bf;#-(C2zvRG!F
zk7BV($~Vzt?jNs;D1H9N?5xAHF{49h;qH<Ta|9Mz|0$DqObq2#*3?N#WMS>nJ=k#b
z)6$I{?)*|=yA39t^2q98(F=>ba*9JGSox$JkC|uY`Ij%0HR@(PSk75z#mJJ*R?J-d
zIM~#{%q)o6vAZ|qbkd%?4;SyT_<t^jJKT-oXT9nR8^;%%UjBE*mND2YwRICZGJnCh
z!*a~E>KrkLzW!6{Sn!2qWxZ;Lt>aTpBmWDPP3MGL<}LX0#POWKZi$?xgK82vlXe)d
zXOLl#aXdGNzmxfpg?!KFgHR%;d;I~%L>Yl&KNIHg!w5|?A@-I+hX|{=H)jT0ssB)G
z?I~{ZRl7Rj5<kyHhK$nM_dg{!_LX!xB+b6k)3faRaZV9+zc=kKwJl^VK4%p!IMNt+
z?chY=KSr64OWZ9Bf22EIFL1pl|BEG#lQH4k9R4yp`A0DqL~7oeRQbJVyy0<XX7QoE
z;zt$}53u}=zF=RHe_^lVv&kI3f37;I78*tfH=BOp7PmLO!Yv+e`tfT0y}4J?lGdml
z%X+l9P3hsI#or!mcqbRJwJ!EUY=P~^RoAYt-ebMT`tLOBeb(z>n8$Ia!Lj0b^v-Ej
zFCHGeWA<g8?uN}b%^rV0GlQM!N7#;oKHHCp)<|dXXjI5$@JR4*@SCa4APk1VYofST
z8#<dYNHO&OZr3_l`>(ds{#|{8VY6X#VF3#t-y<0tGdF_;?L5m4a>gF8u&Cgk!F|J9
zCyB8b3@<m#V`DE!G;IGY=$+6OXZ9jF`Nh<UA#MSZj;Sn{R?3K;Osr@7ZG2<i!EZd<
z(hIDEGtK9EC1io1P7$+#$aKC*&DVkiq5~wKs2)8Qm?QHpt-zf7>&1lhwqF?q#?GG$
zd+zUZS=#)k>Vj#og~oZYz3hVLvjx5uX0hZRtrUz`{l>_^&fp&R+f{m{QOJtiBYZuz
z0{3e_#H`^>?=R77KP#DYE}^)M@7V@x=iZcDmIK$6j(R-nzRfg&cU$@fTj%I47xrqr
zy)J9!adBGZ+nAqoCTv`K!(+lm$75`g-Lgz8taC0VbhG_7F6F+!@6h&-_m}yL`wQxu
z|L7hUSy12nkGIMkTn6po-DGy+*Xae>&1*ocTMF6DJ-n|>SA@8Ih?w)PK)<<qW5nlj
z0mrqImlf?Wig}~c-JM#vQRm&%qgyxa2s`%jp@;n4i$7wrlH48_7jEq_Yd^E`e@x4c
zY=+p1i?+Ky`^DeU?cSYQxV`76Y4_UHjazlT8Z36&xg%Pv{Pd%|-?N*{S@K^zyKr`4
z^`kV&ylD@UA{Q=g*D%=`@vQOC)}q{)z~yEWq#tcO>bR-VkaKZ~k^9^;1v`1BpVqpz
zfc?M>2%MwMk;BbU({*NNMuO44n6`8Y{bw7NZ2FXx`twWw#FQ}m;<z-oDb7o6t|iR5
z5#RPU^@eO{z1nrAK*qqt>m6@XZ^VJb=5VEX-AbOw-JzAOux(mmw&b>H50fPI;6&Hl
z9R3r%nkEe9V90({)kD`Yn={EfAiC*}(3fcoR<k(Ps(NTRI&n%_iBzo6aJ1w6<>~0a
z$>lX6ps6iLAaIMaK8JEZaMPT9OT4Y`Xg^+8v?Ic8YuS!Sx2a`2qQd;1wOB|Tyuh&}
zs<^AveQMd19X;l!LR;MIr>MU3aX4dpqB84*#f9(B6|+4JEf0S?;eAH3%w2iToWycT
zz3C5&B>&Ac&e1#k%m_rRxG>c?M)&YlqqrAy64P5=rJuWA+56dqFNon*VF~j^iA6_W
zN*0^HZCtiD!IE#*53XLRG|LzAK0mjlW*V-zf7?pk<y^v<k9Qwev+F*noqvtt&*F_w
ze{+00es6i#EQTX~IoZuek9F~?M%}pZM=1R4%`5@I;`TSXC-!B(GD>mT60l>rRo(d;
z3IAi%zlv=<sbm)E^i$^ee$P{^bBjA3J)5fkFkoqO?WCH6kxOQ1dbPaVcrG;JS!2MF
z^lmTlGxJLY&Id2_sMXogEAnZ}|5c3Tz5dUxD;<kE648C-XcM3Dg~YZWIS1#O6vUJy
z3GQ-Q_-%XZ_15zs$gV8slarC;{DYnC*uqS{y(ST}bGxTUKG!X5{+P#8pdAz0*c-5*
zw_nva{lG+So*f4oxO=uQ;N+DkXk>h^?2!P5J_?K$j5io^I2qVn9ygw0o~bFjD)qy)
zAK{D>4$e5N#(j+E023Gr1#39y_$*}D0D(*!mPNR5<Roi{CttMiHhZzssFmwb6f;Bh
z0<*Z52WB2CjV7J)&{EEfW_Ym1qvqVPbZ?ojoq}p^HzJzFvM)#onaowldZ{RAZz6Y*
zTg&k+=PS2oas_Xgj_Vb?Y0^2DXi}f9s}Q(m`O`v|)vcYfzouO9Q}LG3pZV<8=Q}3P
zG71+QIeD<0KZ@^BtH<Qy;TEUDKkZy_o8_Tb)bkfJ_xG2s;b6!%IAHPUOUGlT9_4e&
z0zC`diro(UQTWYr*ZW)_!_o&20-m!>NLD|_cP3l?S(DA#zB8<S=h}oDZfsg(^}MlA
zE@BS*9Oh$|2{O!|_B>_EUOwNy=KqhLod<6Itl4?sX7{Pzui4vjO4ruu6?|h#*Dol$
z<`&GsclTP-QzmzPk1x#U;vT$V?TdTxfKfsGkJ%C7r%g(y-|sx|kXvToij>_J<yISi
zG%@(u1nT`LY&tA%QR6u6%$K~AtPhJ7YWV(`9T9)Z^jg2*htR({2_IR-;vYO<O`KA(
zsQ%x)kocn~U)$^Kc<p~Z^~`CN7g}x6M+;W}$Pt~t^#8dh8=9`OuIG*s-=S9kf<XzZ
zC9*~TuX!{-^v~w5`mm4Zwf_A)-|bw`mdAHjW|yX2WMaSQw}{04((?F29@|cBEqog@
z7hK^S7PDXT;c%B+RARg6!`cI4+Bq8!iG3GceK6u+Lc#_Xh9fG>6F<h>lW<JSYkz54
za-OMSk6mg1^wkk9@(=BQ_w(o)KR<K0RZvtwRA6evk+zHL7lw;oTfm~WWIxj)rbUk!
z7kpxn(1yU^)e0*_8AMO~X1m82YVbz7^55Ah-(+q7vjuj$=76f7O5WPIhw9dXXI5To
zEns?45gHXg<C)`;TP4@nlDq<fUu~SIu!6-^)kDkC>+^fg6>@@qIyC}n-B$PrTFv0G
z;Ce3F<Q)*x^k~xY^fx+uN2V{>$MRG4g}!40Czp@1SRq3tL#4q~#siE87#%XqdT+2#
zSnIfr?Pk;j-I%)y@y%7dua5qXNK6;i3wxNfrqSKRzjb!#j>w~bW@*1!vuH)wjz};c
zOsF!X{7Z;o%wRBKkYeC%5PQz}CHKO1$9rs*(ob?7?@ieJ^hW2pxfv5SJDwAJ^5)h=
zwn*ue+zDT=-w}`tv=V(6^3YH8@8&~uw0u?^YEfg@wkrE5Z_&0Yt4ohkW`!%xFuD5g
zp<QmAx~i$6V(NuA!6h1Zww_j(G&Nk>_IX36wrQcN>aH{4TleoiP~LdlNd2upr`~VZ
z4|ghrU&-IH)BB+Jp-9fmv(_(UwNUJ)Pm$lc5AZk2?^2(9o@I@yd{u?P^@CP<Ro*dc
z4jz^I*Z0$^+eqQB?&pMur#QAB`+9)?B){5@Ooqt}&+j+yzZoq3*jJiC$l2>yvrMt(
zXY*i>9u>n$OD1aE%d@+%vvH{jYud83HI0oIl6ZTLp0b%xsqp!&QT9Yh$268d!AxiV
zDCcZp(1AhKWfvl4#5Ol9G+B1wkr*hwFg6^?3QwG~{L&xkGooii&rDr;MT$Y;<&jt6
z2Fr_*+O@d%-$-ocJ-5a5*)1XdO|N>-&zg}`v@W^uk0lfLz7rP~Jgt=3Wi+|;Y0kz&
zTMzA3_{DNn{Y2c}HqNw?CP{ufuW60>Esd5Jd;Cn4b+7WDvo&QBo4VNP>2vcdY~gQv
znv>77bMv#sKmA<Pr@|++l9&ClYbZ~`XRc?DLl_<BD1UD0n{0XRyic@`dW{j^v&#8`
z6%&lxeLq<^Y%4WN6_pd;Y1}mNx?{17#%Feg;s8~?WhPBp+e`IQgp!Q@vhwXZuX2b*
zGTmr}5vPHp^vo8`+^07hH-7$f>(s{9<o09d&hWO~;kY+PIyu_fU~=CSX`Y=2C!Eah
z>|M5&$E91A!+}rYal=L#X^r*0OU`q+_(?o&Fa!x1@=0iaEKL%$OzaUp=Jlp|+rP3y
zsj*vY`&KhJESqBzc5D}$vz>%-r-7fv+a;`ia*v-Ff{2;*)*rY1kCDE0I7CxOX3y&%
zKc2t$ye#{-p(6TFmh~;hhg)r@G`8sY@6r9KCEGb~$89}6{~5XJt!4YQ{~SpWD!$Zu
zzV$fkHQjqWr@xr3*ZU##S~#i3sYm_aG@EIDe{#i7zfbxZtFvEuhj6?1KQHm+?<4lk
zX}H6;`u&DqeI@e~e#)M^_wbGFr*Nw^dk^2Tp0nrhL2H@ai+7Ysl)2|B#AL?_O9+Ww
zWoZJzk~0lcy%UTKBxl5EYxUg?Oek0Qc+i+XP0nJ@lEllrcc%!-{7+($>|i~^+5m$~
z85b~Y+>pWAAlh(Fub|A4)v?#L#HLf!N7Se9=f#_sMbDf|Uu($3?wzUvY6(B8kxkzy
ztt=fCBXXlpVAg{>vF?BW%$~^Wc1t3W`;uhp#8`n_dyFOtrbyndiWX5~$enm<kDv;R
zT8D|_9QT$I`}`$03^u;{%ynRaXa7-K-Dev+Hb#85x~6ni+rd35TR6LDjpLcpxZ72Q
zaml(YHx`67M-?t86S=}C{bqyXCa<D)-m)7BDKozaoKW^LyX0f_X2W4V<=smT^PMaf
zXkcV8a9B3Y&-ok2?A|w%PH1)VNcDe+tg<?*J)=icf`f-;!q#gCI0e74+)vT9Ih1Ps
z$9R*E)t}Am|9ien;`{&RgTL5iWvOWezwSI-9Q&643-g49)y?mCcbT8Kr%<hJ$ok^m
zg5u_Lyr0ZY>^>QBZHbMEtY`e>gpMNBU007D<zQFh>DVAI{=I08xZj(S0>k<VnhIZ=
zas&@G#N0Vk$Y2?%?6Py$jv3w2J9mgCI_RZ6U05qH@oh@~j<PwET_3FwD=-un|NZ>w
z!uEDKq3=aI#2s_kzZdP8et549h>&^rv_QYT+ValnhjD!0Z!xMUEITCHzzKz7ucujD
z<2i9GpqJfvhVggf@5Ou!iZ_j$4mU|@r53FE!T#V7$C0%%c~1+p+oLV-oO&46C-u@`
zwQ|MPhhco>MLQ-Ro+6{FcW{nO+LJ`v(i#1SYh<1|OWZwJBBOipp<&;dS_8@Uofc<~
zmfn!rbM&DuU;amry#+J64u{CZJ^CQzKG)}ii2GWbAbpPAizj$0y;YNCjOg=gn>TId
zwK+|!%h{BdOtvw&di3P;_X-v#PTXL>;NMgrB%-|Rv4L#+ON(987nl3IV`*b#%g}Yq
zzSJP>(-Dy>%NQVgB`sifu3_s&zLfqsyh=^#3`K!C)^japKG^e9N!=<W(;{3~_J~MH
z{_6`J#R{u4=Q9*ua9qo_UEDBLNRCfWEN~s4c(U`+x{lffnmxkbyEMY>;`>zkPrOYq
zOwMItIB{XO<9oKf(RRM7!e$b|0u2!+ZCrDuQ}QP4bbQ7(Gn!M6VXkyZ-i4i0g&ov?
z9#(c$Zi<op_I%=M!N7G1C6f-?6q(1w_^|eUI`G~7$~ggXWe)cLizl0l?UI}-`Ru!P
z&32<}H|C^xHWjtqkh61YDrB=t=c%rDf2?2-B$Y07TRN$FmGhEvQ&n*}IE)aFIC=JW
zc5tZb&Ey5Eq)x4TsZu=2A*4T}zMA9IgQdBTQ+sT9AC_h<Ws8>Ca9Gf5`9X2^n!<$j
zvKouos%17DXRN-%n3Ld_;FsXR#*lX1)OzFTMyc!;-*fB*JSuJzBpjKFnTng5mmcKc
zIm4F5yxDNmnZF=7hjGHeiVB&-EZoPM(?6|e*zn=)QFHd1;seNp*FC-kOdE}ka2<fb
zTg(gs**WuWuruUrGrPKnN#NAsAC?cgnfF;-*m{Hgqvdl`yL=rbm8n%ZG7N4(nJwbZ
zkj~F0V~KOlW~XZu^cg^%$Pms|zpRvgEzr;WTNX51(osNSipn;Nj{S50FW4ifoX}pi
zW1Ymygp|b$2?k3J-f{0)vR?3vwsLF5<&Ihdla%Hj-XhZzp{q(&S_8hSyX&0gc&~i5
zuIR{xfT!t98%#_3mk2C-Fmb<;e1^>d4u$mQ8eT2a6<304S}#^k6m(d}wo~d#R={e<
zwMo;N4K;+Brmp;Dyq>{?!Ra1bWpv+UiC<<fZY$(-f0Fyf#WoX^I4rj^uTY=Tx$vZ*
zv+zb^L1Sj`Q!PiHtk^qQ*{toBwdJCCWtFr8a>^@Aof6B~BAzQ=e!`TsN!!%(jW7EJ
zzkXNM(r?{oPha=zb=_L>t@}Vp<4Y#zuxN1!afvBqW%IOC3-{~pxO8IbrihTB^!*$*
zueJq=f1lray!Ci%I>&kNcv|z}lG{7f)*Ul#mTS7hwZFNtXz5SpW!rhzrb+MNc&a9#
zBA{|DyJ&G!aY3B+b+(2CF9i<;4`v3o6y{BaCERug|IP_BdN?t5<H4G~4`Opv+eHNr
z{!OsB{W8Am>T?q<2R)yaph12R{vfC(eye_k!#bX=OdBB3j3GoJtSyT11_KmwG2D3H
z$vgkX$4*`S?3PZ+vj@x=z9;k6a7un=lTc`3QC0C!bmX4+Nb{bco&J*OSjH3Q{B!;u
zx#usJePEq`&hI1p{B!<P7M<4y4STFP$nYqkV}e&x*rg8Ly}FZhejMJ)=BSgJd)I01
z%!ngR{S|SH9JwB)S|aJ1JBqlshH<5)K#^P|BZHDj;)Rx(x;+O==F}W$QvW8=I9b3V
zzv1LBshN|-BQ(@E#~1rBY8kIMQxq3mSR^yo<)D-0%%;Pk48LZro@`(t&mepyJz&0L
z3ERc9>(qWRo(P!hSju))(j{GB?$sNoAUt28z@Iae%EKl~XYcnA`}@jDOfA&ov1U=Q
z=+yR``b?KumzBqf%+a&ZGte{Gq&G97aZixzz2$#>#cjaQLVSl}f%EsacQ0S~%lz$J
zEN^{xPujZH?5A2j@m8ON-RmBm@R#}8x!A7oiuz00&!<kLb{qBmIWlp+#jBN{4Bj^@
zG#=Q-&>)hYU}O-vf~`lT*uL04ogw0aqeShD*Mcus2%bD5(znFms_GuUH!iV{<!5je
z>oMpt=xyv^?P#?WV+eymvA{hF2e~JSOsZr}U`=37U`{Y*Fi|jTJ7MRa^W})2f6mt<
zasIaZnjCM5WvNKq__t}6qz>cmyMJPDODz`vrQWI>9CBDO*HF-PE=RM6gq;NAr<5rg
ztwB1kra2mM&hotA^XUiI1mC7Sp_a)DWF(LB$W3~n$SEiEwD}oxf<%Hu&H@=O-VTLB
zEP`6B%L9^^1|%<i;Gj80aOQEv<4Pf!0ZUsZG~HbnP`or@x%Q3cTw?#6&Q5hYE9$#&
zt?;gd>D`x?2VBs&(zW266R#L!$9e{}6_<3rF$Qn^@vrnPGh>7G)CFD!Qxm)rbhg=*
z*Q7BrB=^0qkJ{suDWbmIMr?CJ*Bk~f16c#-=WH`iKRL>goMNcN$dJt0Q@S@|#l}Y|
zYpo;SsOF@$UC-PfIJ0=`S0|eYgH@tg8!T?V4z8WEZB37^_Tg(AR``B8`{$g8rH9&8
zX-kLycjjNoPb${Bw<-SE9-mX;8>jtE`k2PL`Oqx2o$ZV|$qc7m0@F7-i<aKJ|L%}`
zqqfo9-5*Y}Oxu{E`F8IDMajU+zt2l`aIfdy$Gwkx5BDDKKR(s^MlVD|OK<A$vOKUi
zH^%bn?rZP8Yc6rTJ@Mgmz(k(9&bxVW*5MPn<J+=QK6p3BCoR{W@z`sFXS46QMY;|?
z7JWhtybWv)%nJ(b^BGQn`WMZhQP7VaH7tpJnkfom26AikueIr3lx{cR@aptG{!Al5
zD?uyaRnDwKBHSF=J`0;RHf?NLXw>SVSGp~Lb)vf6<HWp2@u@6R1w#zN4a}FlQe`l3
zPw(p$G<AEj@qe?N#7)P-cinkS3C~2&ymet!{gJ&Q%5B$cz66$;4VeWZ4p~OFsXx{n
zOn<B`Q_2`&@`&fFQnvApR{?PvdvC0%&=5<HJ7{t%YH`Hm%CDD2o;BZ=s^Zys!2CH!
z#A?CdD_4%%&o%ib@v7#5NfU#~`h5&jH0~KQX0WjBl}^#)=E(f0Dq)~n%Cg|2^Yd3R
ztp{NfDKW0+nKn(9mzS7h^HYfDM#2FRBhA3COBK$Ft=c!`!Y7T8$0_fuD?h7FvE^2^
zJ3O(Ok^OP1mY-bhLMfkxQfF8WI3~%c90L)TtTlNbJUM$j>dg5WzoSH!=eaG6@!>q?
zy>o`s&KZ@q`ip{BE(|bR20}*1PtMjq7_KQF+{7pj8b&&=9sDR^<`wSQA*D0w^^D&v
zUSZ=CSUF?SCxhQdpSj+Wc_V1%adrBlRr;=>@m;I-y%BFL2E*TvB1#VyHo~AJ(`VC?
z;up^@_-@?Iv-@;L^TE4DcXZo7O-n46G@A}4E^-;i=pBB#=+t38g+|3jMbT@W!McqK
zjS3sD1>G}xqks4xjIi%JCR)kO@W$zJ%$X`jF5fzrN<&SvpBrDT+rXyH*tbx&u<cy8
zO!vc?Rz1xd*~DrlD$4zjV7A_T{4dv{LsA|3@9Y~EHZ3&zE)|@xG5cBmIp2>bdsd13
zF}#t`#+wEuMBdMwlBNEw{ZFLE+`<j}d48YXuwUa|F^J_Hr*op)*nQecM?cOgrL_5n
zpCr_aJhr$YpMLV-KLM|2N^YW|Cv+Y2Ih!_rXmqUPyu{t<w$tx3yMl8a=Pz%8geE3o
z6%9S{(0H#6TOY(pzMJ|mPVybA^qlJlrKTj*^GJR&J#k$jzPW~XmFbGGtLe?=BCB?Z
ztlH(1#bWRzThni|%oCXu?vxW(6~ddnz%7C|3tDfi+UVNM#*mSF(Iw~QjZ?SSKkk%B
zvU_{rZ`6YisukXEx^HwYowGwoTEon5;kT~M@?5U5T&}S!3_Yp_)4nDx%;+xGJ=n6t
z?^`&NaF4_$vlF)zvbi_$6|F1O61OP1S~tUG*(06m&Cgf$+i0%wSgD;9#=3j?;jLXe
zre&>qBQEPTc}j>^>i#VilSS7?9nkv`<Qf{C`cL!V25x!FL-BexAQ)Gn>OF0a*GjLI
zB2$AjRZaY^hG?!@859zny6^V0vx+v(=Xq?EAE!6o6}=JLcvp1FwWdo^3H7W`w@xs&
z6aG}_S#riZnl<3D-KBzbwdnLW(yKQaPCDWtr0m(F(rDRe5Xzuppb}=yXux=Z0X#l)
zzuGkOLE{-_$sWm`nVP8r&t@DFT)=Vc@N?D!aWkr8r5IHd87>N~Ibv@xHNs|}Q(_lu
zi|Vp~gAoTK4hGD0dj8opb0L?K(VUI*#I77<x~;uoFO*2mJ}>BE({kLem-h(Ao;9*I
z|I>8$7&4{zoeP+|&}kRvEnnRZtreQr5;ici2RGC$eY?^4`Td@PBjFo=aPgEHm~_N=
zMLT}idJ$K1Zs!iiwq`~4yX*&@UtT&J@iijHr$FaX=EkInT-_lnavrswxy%&5bVKL)
zQ|7JgTGzLUZQZb<*lVL%=Gu!k2e+N;K2vyP)|9Pg8`WA(HP?7{-m?CD^zxkK7p)7A
zPvG7f!KJJh_;-0g%Tj|$PH(3=y`8$l$~1z@G(wGmkL&RQ13s?Da}3ULLWv0$F&c-A
zK&?m6z!sZBx9NivLq4&`2?%1NOijyV&>8{oLV@O^79Ks#M=b0M;x}pd=}!IWu*UIO
zxj^B92pNqa`IQTWx?dEWbK;!p#5uK6^qJ=xQ>KHFIqe^VG!KNzt_?6a5LwkO7%pn7
zy=r4nNOEfZjTw^-)i2Aa{ap3!)R7#i%Ik)=Oc(IgT{hYEq5R=%r|+)2HgV7A){AO4
z5wY8Lc*e?*2OrPQ&6ksjIvUm0wN*qkTT|=Wr}L>fJu@DCbF+K^>WP}2(&yW&;uf2D
z#q>%W)A4l$FS?qd5|4<w#U$<#-Mgjj=hj0N+C9^wxX!IB(PFJMNb%A<GugV)5Ck{A
z5Zp2Cz??bSul4w1A6^gz5nng%QZKYGw*P!Ke6Nu~`m<RFD^mCvK5)1t3Lc&4yq)J+
zwEut5vJ!@f8BKx{6CcK|K9r!!A+RiXrW0#oZ?5$5y3}OR83Eur2s9~Y>vloRv$`OI
zE2yF%iz`@$XV;+x+I;;#nI#t96gd*LU6t$H{m3(Nzjv@bPVEawYcdg5nYrL3OQ70{
zK+|xIS5rJ*`?5r;+XQjmRX+ObO!geBsaMxka*BUu6ys&`2}o#SIm(v(V!r6M@Wg5)
zA}{=Kt>?yRP8+9HK9y+H;$3C3;?e@InR48&QQx#fG99Nn)(XsZESYvrJ%6Ev_rnvq
zbGEkrQYw5V)_2vZcbd@E`Z-Cutx3AAn~UqpAJ@%UsbIt>(wxJ)X{!vomyAs1Tbtz-
zr~gC?_47z>6rP)<_xb2Pzj+oB&!+XQm78C<$|m6co`WGxYnF#GPZaIiA=0&jqv3Od
z{vu8(4Uv3JFzP&dm(k>0&q}qGQ*NvY+V=9i-Tyf)OOo{d$EZoZ-5NgWpGj=Wn^zjl
zvde3aFTb$0CH6%}Li^p7t4`N&OTBzsSTc99?U9;W2iG(P%sFMsXP(gGK4rtr^?ibU
zIt%pE8(7qLvptb&H|D#Zarfus_qsbixK3@=)Z#Xte`!H`oK%%t&W6@o0)I9fWRuQ8
zB3l35ykM<WmEruKW7~~{Zr*R#FSw^)6*o`;K{W$2hDMFX14&$<j><;|w&G`X&+5{$
zb}}e5TY{k>)BpY(-VBd2{zR!=y<pk=^r$_97=swYwHb#J;=>fe+oBk4Zmbt&5oOuR
z%CM#lG=3%O)4R}Iz(ion4=&yl>`x6#GISktIJ3MbL^s`eVl&^~;7Rin28&Zli+)xM
zOg}EtbSjBCx7$UHVTHcND}{wWdIWr%vMxEA@zid(x^ObfLJ#@gNy!^;eV%pw!t<@`
zCbAr0SfIJ#`9h=SQ#_9*F-gbtGP$=hsjJTNuijv;HCF@~b^YOBIK!&7d5X9if0_H!
z$xdJ-G<P=l+}Yey{kc$)8Uynp!xUx(P(#>pDqE;jh-TxS3ys?Yel9^pDNGGI(A3uq
zONMJ0YFPQ#xu#D3p(S^xn@R2CRGSt7{xUJw*&&nP$S?6#KeS|xp58JhwWaQCrcB+7
zk4)VB{)FyFMcJo)SN_U=UE&n<Lw2p-RNrUoKfFEi<C25OO^F>#kMdtl+%Vysyil+5
z#fpsPg}2|HFkQS(vh-|6QfSa0r^n9TDwpqT-i&;qEcs`ny}_-@r_0)&OHD7I*7Hw)
zn*X0b)nKlr3^JvGs=-}e%ciX0f3WJ~&iC!Pk7w<Wx_)MB!P<{@)rwgjIcp-*I#1?r
z;xN5$#URn}LzR`CukPl<>9V{Wk~#O%T{MlG3`$hnrd`-9)T7>LXc@jNOIq3BS(byD
z+=(8Q;~o|ZjL)aKZ8#MTPB50umO{__?zFNrC<nEN>M_`a9DA_vBYy+yB0=T1AI^gY
z{Lb^;fAiri-~BfmPWxTE`=?RSi~$CnE(y(i&d(#b$xqiVzvGIcbrS=V+HKP<`}pLS
z_PNM7$wetVRk7J}uurgN-w6(d!z{91JDsobd2R`)KeYH-@|l|x1(h86&IfV5wOD4j
zUiTWq^i=NMCd!5J-A_B(e&1IXi}Z=kUH*-oO<6KQ4g!5DkEGA3Vh`$KPk*Pu=b`=i
z)D?y5<~K(hxO02Cw}x|lb$hhh;!<z$q%{HEOJ!7NG$jX5SsE}UmFYr^?$Moc>^lsZ
zUfc?rb2Fh~j_!$_<qStf=DPMIJr*f7esNcaFG=kBRx!4l-HX2$xUys~kMLd?;r)i0
zpU?l|v_pY$i#oF&?$8rS*Az+DoO9x{)bXl0Gmkg&9g#W0e4xlmnA<dx%QW&)i;yAX
zMCQX520hHlm3~R?pF*?dbe?-vv_OKr>uBSnbG3={jyH5|7YV=h``_}{M$b}%_AHiY
zja!_x`1g6|`OY$T1b+r5gx+1cVwZQ|=|hW>^nSZ$1Tf8#SifrLo9PYfHFqBn=6=bv
z`U!t%vF560o8nonv*vN!xv-b<+``$9X7k-I`LXOVch(30_Ij&7mzCe2f7s0zUvkC6
z{Z;-3*LGd29D}ZHXXG2^XoWMVgW+v9a87#M$a4%d&c$;mCh4Z-R-+ll5ZKT^Wv>WB
z&0IIZn-wfi0;U{x-pg}3bH${<HHVwsf4!6kO}z!LR9Ioj;P6TMcr<G;YjCO;sDvps
zXk}{yL9t+sZqqaO8SnIIZU|JE%koyGL!I?2%TAV=4)R8K&&W6M^vcL~Ee=(_GSTLQ
z;XbA*g)N-U(ww(E1l*gfgq}=Xz<<fynU~|Q&?&vZV$Cbw?W-+s9Sq!M>a{Is+a>4!
zEi;<bo=u&|V(4Yr5w!N;y%)cqFYLFv<@jQ$NaaNli6AROaemgRlQh|HFTQnp&o_n#
z*?yl}4{)g6zIFxDIQ5zm(4~0HgUdBKiG^`Rz?P)}TT+=eBxF07wdFCGF-SM?9uS*<
zaRDoM6$oTGh(h)0jZ>im-!JYhsBS)^dt%QKDTn)PwV;W(t&Z2gGjgWa7<QSRs1Tan
zwMpdD=jQ#py^_VRW@s8`$Ing`n%A{eq|{Ao$p`Cep;r?$R~_86|5odAZvMIZUkYAl
z?K|{GuRzbB%XX<j!)mdzz_n8sMQOWk4eQEU`)0b};;VeJP1av76g<e>&1b8lW||Ph
z?3h}l{@CqV)Q+%qz1mjC#j-_fx0oBKByM=&GAm-+nVqv7rnGQ8e%_GHyW?6zcHqYY
zhHX9!b6#maoBy!?*d3ihQ-%TwN#!#((<7NTZe@&??!P*F1yh6a1C0v@^_0?oWnbNq
zk|F8C>f>&v)X`@UTz0mI#pt{9W^TS?`$2=9r``!lOBbwVye@?xG%5qPr~f-3Ak84n
zptyjARn<d{^{h&Vs-rGvk>`cZi7FbYc0Eg&Q}u3bsf_Suzmt7nuSUSbMBUz7p1gTx
z+VdJDPkBuU<GOUnAZy_U$K)`l+=g3A@|C#SS!9$x?Kyr&@kmKP($hO@ejfS9_NIT5
z1p9C44f`3(tyihHmL0$T;ld}jH+cu%vEAw2=gKtU>!B<5E0;*lKkt0r#r=E0rx#mH
zmowazF3ETN&b#gYjQ=6(EB8dMIBVp7LD*a(Y7YC$E%G*7Y_m3qWNkPjSimRoyy2*&
zK*{}?N$FjhX7iSLXvC~&U-&g}$<lx&OAoL-o2nVzt7#B^>v3m%k(qecwj(#AG;RfH
z+zMKh;XS9autl_Mn~0R+(UT(VXZFn7zN!5h`$v(kZ6aOUSbSM@kBI39Zc+VKmh;l-
z;xstAv9NWKZm4VYs;+GzqPZ)zZUu#IS-L92JM`i-uZz=GVUs)kZxMGNcON%Tp#W%f
z4TMczyy(}M{-nv8C4AMco_V21qm;N3^Te*NlayH}ouAvey6|ia=b0N>3hA43d!=Pr
z=GMPaZO=1f+rZ19VxR_wd=07w%TH}(WiU`vP+yh=&J?}BXSU9-`H*p9pKX|Sv_YCd
z+OiprGafUiDDc_2Fn*p`!~Z~eb&~6LjsqRZe3$HGn*T1@!TS2P-+v(nEeSAmUeB-w
z1V4(pMY69?{xc`|(Puup=N-utR+?`6Yg6%@KcTHbNMwq_LKa@t4vnurxGn@XO%Q6C
zvS0~|rfP<!qtD9Y;SKZp&WQ_zG%XTRDLo?Q7;-XMxiaVOf{8!YY!m~n58UYNt)uk(
z>9(~;dtBE9R&*Ynv%i_+kuy8H!=3<5?qJQiAuG9pA!u<<k9%&)j^L#}K1(ya_gGZ2
zcPQAE{#$V+V&#<Xl~V%1WH*R3`7ncFfmQP>o>jBYEDMrbc4@`pAkk$(qHYWg95ba-
zvI15*E@hi46_ORO^2kH4{fjDD6+*HCmOIX6E0wyUneVmql{Y5pU8O%2j6y@>SFPF?
z5?Z@*)vut?uS<izE?xD?JM`-GRjc$-<W_>EA#$@mgsrl4geaPdrY;m_fY;UOwdbVO
zFHVEBC{%YP3af?I&a_XMev4rS3`*=gqa!<S!UImvRhRcLfmif&3QA1qWjEqaF-U)x
zZjiogk~V{pgt3INM6rgZbiBeEHwHbn2Ia$iC!WeMIi2lYaHU|GjM*NM!cz<H7C2ok
z`7@a{v%Yg>h(MTFpiR>M#+4xkA?yr3E1MP=E)`L7;r2Obec;N0D>I~i+dDFFa!oj$
z_K$(BL&1@i)5zn3W0OEo|9S>pl@3Km9?mR_S@N#ySveFPIpf`9GFcut@Ni~%OmJ?J
z6Jk-EHsQg8&+Kg$3?RtCshXkT=-gGcMkIG77|pzXXsh0&byqS38RDgv1ua<`v_$or
zW6ilAH6O2LUGZ9X#bcdF>PnH+l_5XYUrAaCMl%_BI*ti|#|RaUqzTM+%n6d#+8U;{
zHSC#a1BlDEF|yB<C#(4skCt)A83m)}6FMDhjwm^WvKdMqNntm9sbJc1QbCP7X;$=|
zk|V1hd4?XHysAq*RMdZ!*1V8Vo0V<1609DDo=n-#;Uhjn&)`dQS>wc84r=u>AGKnF
zLU$}#Rp1%=aMG$ns#le=qu!>j(h?09HCP$K#Ie<})n^sg%qtrF21_e#*Dk(q@v9e3
z==y9C48Ig8%@dZefvH(kV$FdB*0%HM{}|d>+OF(iG5`%<M3{eMaMm@5XgJ8kBbKm%
zp_x@e=KzPKjn&GV$)*L2$$Y;>7`Pd{eI9zOO5<`5ux9AjXe+qRupw__hevhTHP6tu
zKh%xi{R3?m$l_)2_%7OG`h!J6TS7WGLm@+q;m{F*EJwD_5ot5LO51;$2e&vpc{ks_
zK~TvvpmWXYAN|Wco8HbdEiE#)vN`lXFz^o7@lB5-zHyv7SY_nSuvWE0*D;r~$k^-O
zsT-YvOM=V}Ew=7>+_|j#2@6B2nn%#L-F%k~zX)`D1q#YvN_2|g)NY;YG<V+q&0%s^
zB=j^h!edSbgzzj&F;H#Rl=L~Nbai|32X0nXEqB*D9P8d+U{Y_8V0_+C2rAYK`A&Rj
zzxCn=llE7SE&M&wXJ&pr@W^IHne7~}sjuXxa%otdIdEcU!W?d~=Z8!<9+j5N@!lwW
zSJTO`{)Nqh*9vQ<D6I~0SqVlLMV<(VNVoiYW@^Cu&e-&`tM;wVOSZc6zx~^D>(=L~
zUK$&ZYA@|)ZMgbWWmC<qLsPrdi?cW1nByU)KB-Vi5=2DK@raFgT;?vbwr}y(xu4kk
zcCQK&QtdeTagWuz)jL-PY|kuNI7xHkXD!uWE!E(r8H-L#e0+~9X_j(9+Y_lvHU%Nk
zOSPhF_wP98@+SE6nVDO~C6;DSiDjGGcfrQwTIGcm&%8p<PW}1r0?)1wr4L^py572;
zyPsR<?Oua_54g*=-kSL$;_<<jHZ_;0yzJ|_=X1-q?={}v_%}BrPVY}L_mNr8E_gfZ
z^V+INw;g_HEMs^0VfxukXZTyQpWb2+SaR;mwsY@i`tKE$OK6f_e9wew<3FC)=QsT8
z5u0)N=DQ9CNkIi=CYM|#caakvOp<{L?mg?2g;TB^>hYK$$Z4eH=%#X_gJp9DclHF(
z0-v3>VXw`!;$7sYFXle5W+&4VrYB6#lnz|_^<i$cN#G}Um1U3LeLwEi!fZUTKj_jQ
z!G&^{E*@f^#yQ(2<j@4+Papk-@>`hSIB@N^W(n7}H#mNZBa1^WVcUi^8&~w^Y6Kco
zy|-|3b34%?WYw25aiXTtCYQdFQ1=xlUQCsosiYj7lG4?~Gf_~|DDpLEMD+-#mfwVp
zPc+?4KyEKTY7@{PW5To1>s(PpUEASwiyA)5Q`5JIiSV5C+E-*z0~S(yr|s8U$$Gd!
ztY<y*M)ei3O*b#DKA!mULj8oorN6u<<S+f@z4X`VIZk%P1uNDE{axzzao&RmId8p0
zXH<#2PwWprb?}H--&$$wO+mbBWl6SO-vwG99_hTIanE^=2dI_`3(nb)tbDY*?|hi1
zaPZ<+{1;{X?s(|r*?K)^VP9V3v$V!%>8@^J`yfr}U`^@8np;et9Ebp^eCM;}p5nYE
zL9%IeZhS(@p_jc9&b>akBVJP;qVDLyxeb%Oe$J43n!;hTa>~Tc-|Xz=N`YFXsamYD
zt4^H=N!=Wtx-mR;?RBA_>-Wgav78i`cp~N0?y3iG|KCiRF*9-U=lqn%YD}}P#7<xk
zT^STwzHHUIGkrU=6=IoR9(sMqz28{8Lfk^AcfZF;rKEGLZOxZh(>Tst*n2QWwR6?Y
z^LBf9x1T+>#w=~K;h(hyYd_W{&#Jh-?HKdipGqfl=G>C*xFp6<a917NI8ifLc1o0G
zCG&#EDxL`*2?ooO7~Vyk<6$vlm{4i(IpJC384j!XO~oQ<Zc~zZt14yAu3YT?>SEtI
zg#~@bFP1HD*r387VjyN9W+3wHPUMqwO5y^gy6>(}jJgnchUpoHo8_Lu#3Ei>8)K$)
z!Nt<q;88V$FoCNHhc<RxY+jbzEz`vF?CX~Q&qWVRP&k`*(_>?yN8z>iJz7G~6%JoX
z7Hs~|bV*?&ORp^B|A*}FHg;^B==9><$D2&bGv*wXUA}eJ{@a;chEF8267H>UJ}SG-
z&8Yr-%`t)8W1P4DR7W#ByLzqr;~$xW439Z_8WkCer+}7SNT=vsk6k*;8-)J(ES=#E
zMIGL~O{!CyRHrhjPF2#I`b!x^?oyhnq{qbI0vhV)-XvS|f8lw}eV&2hYaO4n%~-S6
zbcR0PyXKoSyymbUt4sh93aKr8UViMV>id?seVo_GBiQ|dWljiJ0SK)+bVSLzQQg$*
zxrOfZIRe`wN|%3~Bs?Jqly3YMx~8mvOa#dst4y3D9M*JnuKnW6i%)ijten!_dH&TA
zZp(?<qRVzL&v>%7=I1J#pI*7l>MKrgY8AQGyz}p^JgBm!<NT^KhyErVkm#Nt_A&4&
z=ezY~pi+u;JNLSjdZh_5f(t@_)(fw`c`IPugbS<w?EkA|qi3UMqGtjg{C6&B&*Qr*
zy-WRH)Wh|gGq|T+&@Fy?;jY&8V_S<$*Xc}?$ZV|-`;iVNo_O*7$Y|}q))BMv!+Ea%
zYaql~uKR0VEH+gN*Q(ik$XvT-(;;>3HyaPhy;k0}wN5to1(yUj0}OimJUp=?nM<&`
zl4U-_2Lo$n0Y3K?g)XAUPygP3?9oAEtq!e@tenWt!oOE%STUPLBsPme31y}kFSh<v
z>K0ux*}whlp7R_{8A4YkDNJYCtHR;hb+$n_>_$>6uiM5x0i$QU1}s0HJeaT0b?&{u
zsq2BcrmOBc=x}CvPVjDu>1&nSIFU_&-B}*g)Mb!iXgPAJqnBl>WJp@TOve<qiIOg9
z0yA?rvIWd^Ol6x2<}J18VVIGm>=?^d8re0Ozu{$xSZLR_kgjbR%1>R_O}+BWE35d;
z5wCAWIa|DX#cGf3GYiU@<K<WMrr9fW#g8>p_arhh7$FATG6R-Qy>V*7QpgZoW&o7o
zd7t;kLSeqhsaxYtZZ1z13`yk;+4N5RyY99bTqnP;o3=jmXymG*yQ?;_uL`=I)y?ya
zBg@S*J4w&iODgjBx;0II!xzpp4PTYDF63&=%2S1_1z*b;o_qBqYHF;)mRZZW`?>k$
z40|H=J~%&Ue|OezZD=OzZ_ll{F~Z@GXYEuvkuQ2gFn*h9-h<ZL8y4g;_<^7pgT-1Q
z2_Xrg^mL{I5EM!X<+)vNpdjsR25Q#6OMjPcH*<zh^GOphQSyUvb7ueItC=?$0vci(
zVrF=0GCVk7n^oS}@FsWHm$q|P<=hX{yVz_CUUKZ=Bc9Js?3$eSbF~TokJvJ|{0&3d
zJwwLUT4S+|a}ADie?Ao=#i6i(<*AB?yJ@%@n}Rbm2IGWSCMyWsna-Pb>474z9EiAg
z@qsMwy^BrdTaE{A5{dE<IQ&qa@3!8T32rsV7>^tjU}rd<`L4KYkBNgp)EdTh-Wl4C
z(V88rPkzW0nCY0ZjaAnH6w@JTjxoGf3*)aie%|AyDazOpdcDnX#W`@NCCRwstb$qQ
z0p&x=E0+d&&ReaL5bm`y#A{`U(E8^bJSW&vlDR;0;*0`XrYl0e-S*O*>ZLoiQaAf?
z569#sOT3mY>0)4zG%`JLrO9h|*{Z1)k&Xd<n@wi8%(-?@u1S_b;+4sYORox-Fbb}k
zW3o6%GjwUtDzBv>Q@wsZH(I%5^@HpGmcLYb?>znD5^ivlH#Wh=t~sJcRO_ZblX+&A
zz@+;}twZ`#>Zb;I>+JwRjy28?+TQUN@GZHS`<QFn*&UB;ZW{c`nVOw^Ci*%1bFKp{
zP<Z6TQDrNeMTen8l}Ed4NyHHy@k7ir!MztyCv}43jASm!9@$Ih6V8EFeSom#KIwB4
z_%g)}Tx;F$%efb-Yt^hd^VIK>%M$go;yJ8U@`4k7i=KISX^~~06We0F$cRop)z-;t
zzFoFo7QwEZ0a{v~kk#1vc@v{Vm4~jFp_83LFH7zcvEWaoZ_Xa=%axrk=;RsT)f6JM
zrD(>Tsv2E^FEg0FOlnG4Fy)CMD}#Ha5&r>MhUW7ommbF38@#`MFuZZnk?bF_3=hsQ
z_dwu<geH#59hyCxc3e7`Qc>u_VaMRs@+eqzN!gtnpW9DuEwoLX!@<Td=eIIL4tH>Z
z#SDQNY>xk%Z~PTB{B7$LZ*`}0lcM3Ig!0Bp7ae<L32C#{4HX53uX@gCJNp?LxlKvp
zTB&oQdEUmUR?1TksVYxmTwxr<86(WlmcesYqS25knMKCt*umtV5zO(<QWI4j-rZJ^
zRdh2@s&24SG3{~)eRNdoW75=$jS`;}>Jwee7!wnP8w|_)bRM7BH0k=3XSMtMR%Um7
z)-DK<be~!nBB_2Qrr9y+NXHiDYe`HGPR!~yl22ffi9N<?dOdAIUxRVczWJtGEBEb?
zSjROZB<K$N&isl;9}Za7opuzO)6KX#!BbVmjn#V<*R&8(otyXHFn@?*6^{{*5w`%t
z9s1yfL^k{d`}SQ@5byI4KkhGmKWl<W`{t4foI>-nCU9)BS9=iu=nmiWGK=kpcY5BE
zSLxvqa%1CE{uT6K{c`=S{e_oBo{P_}|J-><c%!MCf$8j)rbz+vvJ)F3I)CnN)46y+
zkk{r*UrWz}6C8a#4^~X<>v+(?(bw^yLjO63RKtNQ7E5`1f-C$_2=wthRCKm5YEcFg
zThv>=wdBm3plg%zu4TeQ4#~a=4?Xs9)?N`zn7rf8jGfXtVQDI@6GhxE&5TO-n!EA~
zBSU(|xeHv*`Z{TeDHq>el@!z!6ItW-@JZ;p%M-)?Y~)HbymTT#y_H`uL0-?<uBlw|
z;6nqR)0!Lt9#6ho%}Hidu?UO$pH*}3$C~4AUFTVkvAOw)7k*G-ll`sw?IerA4;|fY
zdNVANj?56|u{_vuli$?q`5F0~+V4LXEmQgDm3Fj-Pw95g`U8xrZnFjIIf8oM+cO$F
zom+Dxap@t`<~cn#_&?s&UK;r5_xbzD@mClG;v0{C{P8d0NrZKdz{jO+tPIa)ZQFWl
z>W+<$pN=d1Ouna|UAVO);aP{z;p<9aU7L9}xh4fZztY6K<QQ*sw`6xj=cm<zaz|!`
zXg!h%UNNP+bK!Lkb|nVgWdh2pj=by>VM{nv$F@`IW0H-;_v{wtK)bZQ12;5|Eh-H0
zj8b&Sy->0v$ZuJYpBsY_f5XKC83(5xI1!jGQ!3^@tA*Pr!%gKtL!yfkkKz$!p(LdT
z*PT1nomRTl1i1V%TI9%eu7gGLQ721->4O(WernpeytOXJd|KFD9o`Eb@OGRfmFTcx
z%Z$$o$J1S&@hmNIOS$l1r>u%MAE$D}w2c~%mxjd7k}3^gd+FHx#PiP7D-seafqa~*
ztAZDoCp#wno0TqpPj63bMQnxbhf66}G7djGlx@;p_#s&3e(U|#_@ExG+aaObR<6nk
zUYRD)ZF}NSNTc+_t37RTQdORZI7RuFaU6KLS2DcUJ$Lyt{))?HJhNv%kk&jeR+%@u
zxV~Y22y=JC5tTDSMr@m0XRKh~<od&r{oC!zFY2<Xoq{S?o@m~ca60aAOyY!C;M?T-
z0|zn^Oyth=XcXHQ+aH%Qak7%yxBiTParPw1z{C4yN=BA3q!@^Tq3M*X$1-Chx`UEf
z9v5!$>F~N6*jTdP@8fZ~)mNlAve!8BzT-K@)z%^SN+&Pf@Nng=t!44;j=S9}yd^Xb
zNc@)y&dIEuYuR+}QK{HU!zMB17uge&4U==Tu1I&f&N4qXtJ)_qE|=HVO@~2e(dLMN
z?JUM_X8+T86|y-~j&Z2D7&t%Tl$w4dO`@9T9AATF(~2zSO7(t0mb7aMLYc}!QbE&I
zm$V5SV^+44<d?f9(0uswpE-vZiuaf-=skFW-C4e;q^0sG^MMF{KL?(<pWTW?^v~Qs
z{;PJ<=Vc2e)cXn_frtkZ?0tm~ZiunpN@FO<>tRiBxx~M;L`%_Vs*sg1gL+TlAqn=b
zrTg3rJglX**v|X@*yHvVi({P0GCFD7+N6|aN(|*DvY05fSU4@<$X*e`DD^~}cb5qN
zEv4fkZYHvFPjt^__jD{Q+7fy1+7&ky?%oY86SkWuRcj<IWfPltLMf@Mxn`MQWUKO}
zZVi4DrL!H2Y(?9ml*_c7mM*Hgqwq0vf@V19@2-_RLi5>9HY#75ut+GZS3)Zyy;VL;
z=8fT2$EKL0C#@M%mK{jUl$mxcQZrF`Q@2u^)}6Z{|1&qWn1ue%T=nnrhXqHfc$R)O
zJLP-k>uDeT?5f|2*Fw}*oOIe0ze&$keBoiegI&B&|C>$a&8moN`_vE}p*e;B&Z3;7
z0ZAY1>vw<H`C;dWi&svxo{!$MQZn%SQP%l0W^HWr-7H&`_G0GC(>Hz|+E(Zr)4Saw
zW!stet*;G}`*yA^TwL7iS)`Gx;HBW@^u@Hb=O?p-kVMc^t_Lg+T7+gtY}534V=v)s
z&LAZLT99x2y*Q0)O3TDKEj)(u7dUi`42_jf_NY`dJXmt>&5{md-2^=l^w}!1?81c~
zQpf8O-v!&8>R<f(_>?lEOMZ1OQ;WCOEfcyI__N)Jze;fXyD4d(lAp3XmuR%_n-IaY
zaaw-O;Tg@H{|ppcY&JgRnfae-@mkF{*EQKvm&{zV^2~kH)SHW+@t@hrnr7g%m$S%s
zLh7rH6Ro<LTbIerU>5Ga#w^HDrIVCCalv_(L^Td0$BCSid@iJvNCkgNVdzNH``oyV
z;oSYm?iH6=4%W#g7F<}9lD(s>C*jDi93kFkqFHlv>i;pMoUnm(;qn}xMPFe00@~Jf
zjV&^Ig6Wf-37c6hr9qVAIW`!hY{F(|=5>`)plx3zPfSkK6zy({jc3r}pA%L;f#-@w
zh{^hdZ5IExGaN`)S$_G^gAWJqZ`&6Xag;An_wZh+sl8G>%X42}<}G`butY}Z*g++}
zD;w&}wd73C+!YoG)++RBHsBL^%wWjJr+u)|Z+X$1$&TGGZs-?t6&CP3JJXmbBXP{q
zurti7F4%R6*w%oyOC__tl>ZjLw36NR<*4V<RXtahTzQ{gcJMWLyP%J_1Q^cH-*xu#
zlk=>{Slx<rH4B_QxA}@rdbYaBnx*{3vx2y*>e4HGrPc3DOIlWQ>g<k*JH_rjzi?A)
zzxg_ysXtPh>=!kk&t_tnUzHrJ_PBiClpkuT$<LTs84`RFeCBF%Tu$Kfl=q&LJFRG+
zeL|0d<Ah@(y%V4HYNqHg%w#vpo*?O&rZRJ4irXYx28N9dJ3)I)4UUSu5je1vX#s<!
zRlH2-8Fu}mG?jgBtbx7fXDGel?3t=Lp-)+HZpcx`bvZ3J&I!sGrt~~8*l5g@JW=<?
z{bTRvNKT$`!MW|y-Py0sn_Qir$+7Ch;#KySb#nS5SyGiWt}e`6<$QU9b7=cA#rjz>
z<-C=bT>{jag8t52_RHA!Yx=BL=ilfDDW76J&YG5U#v$A0%b5$t&gV~j-qy_){U|HO
zIRCTawnOiyW^*&lm7k;C*2D4u1Y07eaxn-A1V08ha2=QzFkawcIELDkjjk~;o2;mo
z;x=KfX+m)-Xw%z|N0M<t2O<tc9FUOJ;ttkPSg>M$(%GiTuAKMX8oQe9cz$g-t|d7+
zvRT=d*T?-u3cJAZq}0H}cUn?3^8W@HWhU=lz00{-p6&OETi+sY+i8ca*?2ef=)#p<
z&dP7gma{g#Wx8#h&lLx5VL_T(7UB{nPMa#)yG%`fcpQ7OW)bi2^-TU(Rti-$%Y`gi
zc+lVIkBT$jK{ul(6AqjB{(A2yzJyV}rnTh3&vIslVzId?FI1e>C9OE?CM}$tATRmN
z@F%xGFsNA%_A^5UGbm#DQ{+x0oJlw{Rhy&ON3q$kc_S!hA^6C1ZAHDWldVKw32R<7
zDT+uv?AbZZWXs$aO><sqh+J>VO?jcbWkmwNrOw+YLGBf~%FBNRtbSn0e2Yu`eoXVJ
zJJOFHUtyW}@A><w`)}{@+Z8maOj~f_>YTSLD|Y@na@Tt5ZY3ver7caz!W1mNO5E4K
zbk{2R1<#J`1A9MhZtDyR3)nYh?cFQYdo&|=!%%hKPV=~{H)4Ot-rbnZwXSf-hpIE%
zv)w{A6g@0_sHl<;I`_k{;&R~XIi0y{+CIC^;eX(hbGg8EXX_@d&+K6t$)4N3b-deG
zH8FqVN5O5$EN846EgKaj!v$0XR0NjHs9`E%DmpYJAG|-b_je_;!!yR~r@E84MWPqn
zU^*(MvCUzlDu`h1(hbOIgouCY;5)%l@^s^^EsO?SzvNrH7$sO2Y3P62`e~`;Bf-=B
z2b3={HbjK0-F>sS`}Q8cs+Eg`RU?CHUwIhIUFa^-jqaV`@xt}4^z2Ex%8%4uxY%~h
zF4FrRzWYPT!@`4KzgX;vZhypKxnZi?6aDk{TWVt}Vk=^Q#Qe~B;%2J5^e^8FXU`eF
zFl4<e_p#LGyAsEbDQ+>mxV9)a#yb4D6~lyv88gzE5||RYrY2t%ZV2XP03F4|bCtp7
z%gilrwQuj+^fudI1y}Q`obRW~)+30GceQWt-GoHER{mAJSbzHQtb3=Y^yzss*u5%Z
z-XW!Iaq1<*&lCMD4+QIY4sadVw(<V=|9OwZPjP3zI(zqu%QDTst0BmA>fW4JXYWpV
zcXi6Ut5>RaYo1-ba&ctvwCm96)gJC1k+P!K?8|p=+8wo|`^d|6ChM1(tasx(`Cv-;
zuIqIdjg899w-qK!oBQPp`LkL6{ygE_pL+{8>AIcByQ5XV`Q$dX1Lv;p<U2O+%gUv>
z$2ITY_}BXuN_^vGzTNS7pW<}WZS~2&&9~Jj-!|V?%l&t0;T`jBwaK?9D;q!BlArVc
z*thwW$qDB`u%?kETL*LokYN*Om^?^+b$DsD%wLuMXGuLXioV3J<}mCIWvmD}%J5*~
zM%DwbHg?ENdwBMko83tVNo`*kTHC(1A$fnd+?1V{I(Eo;c%DkgJe82?KEXI>LQv$a
zpvYNKZ_QFSn51qn32nYQ`&RrW<$_(>Q<tvXnis6*4MwN`X|+0@FqYV5SyKFB-id|I
ze6mF^6wj3SeLYw9KIii}GYIkK)y$PlM^a)K;d72Q3>=Gi)D@4EY%5KVF8BL<Zr!^#
zul#bGb{1`1TdA1NusPaT`Baa_c7_9SCwX*@uk##ZI(Ddoi^GeLp;u7BPt-uvVCpds
zi(sw}uQiTaw5@I#G<0T7H<D=ia#^wYUx&(%^;0);JW;L_KJX***|%`^Gs}K_+-82L
z<gewk)VyleKJ%+VIbGuU5ag~Sb=4v1wsylF&4!x&9Ba7OaKCw@a#zIS9t2%^ut_s-
z4;a<dPYgM;aJI8~dtTOT>)Us3K0BxRzHk?WWmdjz=jO9>bKl9_lRM{=4v`nM5pUnV
z`R?4e?>63@oBJ-O7%G0wti0?iA9x*~@l>8;Jjc$27%`k?YhXIUbPRH80@DE|2PwNp
zyZIyU%CFAkXz0&6$Y7@>wlTMF%0*?>u7zO?k9g<o4&C|2m$5B(L$qk(4G|F95$`j7
zO}EK?J%i2vJcIM9f)6AFA4uq~U=9gk<ujXVnOU;$_J0x3&KnCWznupHdonrBT)%J@
z<f*OR$_+*)XAb5q{eH>9#P^Y<TgIH`yVf`M!U(4?ayj=8D%!$CVy$m%6=o0+2DRuI
zcpHxWJ@a=C!-Pr&@TLmGW<%pvo<lOA!ygiK7<4w8of9<iOMjmr-rsS9<6^Z<*#7*)
zNk8?<K3A-{W^UNJ_=Rckii8D+I&>;TB&>{99ak3myH6=CPR*8!Pj=JEik8dO_us`T
z>e?Q%{8r@r<;pZcZ?$>rBBskd6gPTX64W;#wkk#DZT<3}-~HrcH|j!>?1o)VkyTqf
zBtyPQ8$JEPQF|jifxV}&BH@N~Q118ZvCIVzOCNsi3E%(0Z*KT?i;NxXHdQ+B$_dwB
zyKaWK%BITT3Xgd*JXJ3iE}T92!ZXv4>zd17?8=NVYB_6V=(EarWiMksXf|v|*}8KV
zs-L;KO=y@W;h6qR(;)ocImQJanI(iF@bpB7lxT(vzkImnnjc89nIRdNu4{BYmDwTf
z+)Tj~w+Wybu$h7>4AUhX8Iu@`851CIOP&74j*E*wX4tVFSp3R{L7PFFVdbe0OL=a*
z-8d=anC2`OySIAt%clG9UVYl%KQVI6k%ALySDxOU=PI<-S<Uw3@w-nS&wIb?+gTNz
zt&Fcem6re9wY%nS?7nw;@!RjEJl<i}p?+bb{hVCK^IGC@;2FUA-0@c)*=vPl$LamC
zbMQ%fd%>YCPU@e<rk!m$&cEkNcW+RQyVoOcbp6m)yS^1^NeV7vZwn=+s2|MbDKa?E
zdqf)K-`5izW+WXwJ~K|}%cKR<WeuK8aG2sSA?Y6*8*_8xK?z6DI@V`(eS#?l=?3Wr
z&u2slNX;;A@UHm<D$Wiii81h-r7$`&ItnuNcuP#~o41VfmuLUg73?hTtxpee^j(T@
zunW4#C?R;yCE0ZDgvoVp<QvzVU!7yMX=&NdO}lGu#_oHwE>0^}?@gYdpU>Ns3#5N;
zQFnMdNzW!WW7qNBrG_#;ckHgY6KnTQPriJ5|L)Vr*P2U5{C;FCXsB~Lv;RQi3s<`j
zf1fJaIP>$~mD#0v*}v6~$I9?WPoU+82ik0VCB6t59(Pn{t2aE6WOcmNvn26CtD`tu
zti+Rq3C)gtZMs`8w<IP6&*f*h!H~mX#vuJ!+*aaC!i8o>ezw~NCk`sGH_Pxuzxu=2
z!`c9Yt>>5j&3{v0y~gb`gH4nOlff^|fPk2F&5N1l*oJ+b7a={Pgtb7$OyF!sp{MEg
zT{X93_r3M|_*#Ma=eFH7w`K2_P0!zbdi&j{x8F@UEqBFPde!Y4$|9}5dgoT`RGjY=
z{=7+Ve);tCyH7vA`}Fg>zyq}_;*zI--hKM#-KVeLfzkyiKz`45{s1}PV`c8Vo6lzE
z&bxWnQpb2InDch-&3iN7&b9n%yxTfCW|lpJW$L;$Gcv_b9|^wsZsyy0hqnkHuTR^Z
zzRg;(IXEgwi_Izd>6Z7#ISQNK8sDCG=yCI;;?Gmg|2gL_Y#VT_*6x6SarOks$g~v6
zmR8dVxl5hoC4Zf`;NE6u^krfqzvQPAEqwB8*L+Ui`i|wn#mVdq?sFe-H7s5w>TDZS
z5qU&LtAW9P(}e6@;y0ROF3Q@n{#_!_-(;cMA2mI9>vWBBhUf|2YfqoJ@-*??3e`^r
z<tyL+o~>5_h6dn8Qc{ajPl>xKv_IgvE3?b8#Q(P3*4jATwrKz5)~~9Lw_eaJte*3(
z^=<r(f32Td+P^itPKY}=rAM1Xh$D!X;Z|!<n|4V6M*^pj#*00Lzr;^0Y%yuk&W$^m
z(JcP+%FO3R=buJ2Px3EmxL6t&+03*w?V7@7wwb0E{vF?Lus%V9jX`W@!-9qd8`y6B
zpYnMwf73#P>n~o^m8@fRv|YyFDem#>g>@jqnMc=tbtvVpw)5{ynDjvY#ZUL1T`?D8
zpVsfTxgc8>=$yFxL-GpwEA@9g6Ca=7^uq0F{rBnW4;}PB@E=jX(R%7%*%GzN1>ePE
z3t|dl5<u`p^?N4gY{iS*x0-EE#mYVHyC!fcpf%3v^|~3`t2VTA&1I<#d7(U|Nuir7
z^2&)-88;LPHh#UZ?yy?p>b32Y0yf&41PQU}EotOm$9|V(FU#DJ7b-VgdpAh0naLa<
z!_UoZXZc3^mo$SVLrUBUn;DW>X)ESxmMxnhnV84$;*6p3l`{dxGj=`Gs^T&WTDZJx
zecnHYV@z#_q)M)>*zCsYvUhU_YtLNM4!`g2-WdjA%PPL)YZ|hOosL|xm`Pu`;MxO?
z2G&DX3=eepJQXG%cmhea(9%q9tEhUO?q!t~ZTIe8*ezYPea@p;)4=cB)fb-By!TY(
zeNortT<upI&WE?|=idJ&c~$GJrz-D@CY4*B{N8<F*MX|%t_K%-{1=$9a5iWkYUc;3
zmg`Qj_tTF!<<Dg2x_)Zv`Mriy&oVPi5S}1>dMd*l$kHh(1|fmT#jFb;8-w0G{}BhD
zbudU<HUT{E(81cn+Ms%^?W|bm4px?>3<(>Oxn_7>U~4gZ)#)cyqkbak*S3Q(2Wz&N
zo#{UD%>LZ8Q&-xY3!JUJs_HLzSFcf8oUkroozKiU8kZV0&s#nL`Q-bGG`71ZEbl#)
zc^~ML>)=q-S8jQH_bIC~%cHyJJc><~{hG$Q`q_2s7rz$u_lfb%JrTKY|C>XPL9jJ^
zi=1AKUd=tn*;~Byeu#WrmNrkM=!LuAXT9Z>?~iPHfB242_~TiX(XZ80tzL9|%6r`5
zl<}g)F+rB;wLnDs0V|nx&hhS7w<>JmTUg4m);UAg>9s&qdts@J#b<T}>0l!U&=GSp
ze_Jvte0E?<hQKr}$rekojSL$bHZp8DqGXlLBCesJG1b&yTP0&KV{qc}2&<2C`R>2$
z&oKYHwW`gib@4_|R)+Z(fAomFoEmoX(7Zb*Yd8+Q>R(*K<k{xyQ*deGp4WGkts0m%
zGP|kdSNB)xUzoh-U2I(WUhdB)e(ijrWD{Gl>qAN7+h%j)+i?Z4AoyTO)S>(*weP3o
zzAoBdzVrU>JMV4(-2Nb2`Ed8155H$S^LOUS>?(Y5Y@u|!pG;A~iGzn_-dt#Q=I67u
zNO>@!(V2};(qgB)t=2Qv$AvErKD6euw0QDhLUa2mNuCo6x!d(*b`|{ScNXJ|tk7sY
z*dg<);KkvE{Oxu!c26c8FFDV=lVRtE2zWuSzy9z0{;O%S4W|35tUrp@uc>4EDB9<=
z2(+i1^TVBIMW@SmKHdH5-<P}A_RmW5(sw@Hedkkb-cP;lHPfT_t=@P3Uy}G7J&Ray
zo(ARXn1l<nq~|5vXt?uv?k#rakDhtI^vrAR=N!&SzxwuN(e?73ufG@ONMAp4^-=1L
zjE$2mObb}ptvUAR&-vH<)-Gu`|F+)=yZN_Oi>P#6uHW#M{dS$pJ?9hA%w`5RzBSi$
z9Q!7p^Z(#C`GxQKzx_|R&A;t0g5dZp)+*bVYk%WgGn^>qDtyztuwh}!I-j)+I~aD9
zxPd2BC5u=SSRY*E6KdG<{zsJQ4arx54txzNW!`J2Z+TL5`G4`uXS&m^?}gTUo_{)L
zx7bx>%>}(>4?i9H9T?`YeD`VD_eIC6Ep9Ch-ErsBZ`X{pwE3$gmY+WT?$f5pHnUHh
zyS;wiPs{W6%ePMm>8XtV5mS-T|Mrlx_hdzr7pJ=nOQy~!sJzYjYi@?Qt?kRyINf{i
zYz{P-v26-n*PH$PRg61B0VCv0GaE(=5d64SO%}9jUSbzx!K$~v&2E&K3PkuQ#&jL)
zo+)n3U#9zW$>g8b%BNTvW-6vWn<W{T=3;By#H1iF^#_Zp!OLKus+s%^D-FJb&aAq`
z%rJv%f_Cr<hQ4LzMH}3wZ|7=Q9M0%$&!v0JPT+vr+1g8;d)~|w+57aAM0w@aSjoqF
z`MXZfes@Zu-16dg@!bzf4!&f%%<A?^FzMlhYY$E-e1CcA`o6-0<p;kX+MPVHan5tS
z=Rf~x&0YU8!<?o3#j-b|f4)!nW>9}kZpNhxf2`-fP;wOK+~qOBxk*Ooi9s^c4d!#p
z?@95q{8O+H$XD=*W@vb@Xl)KtF$5mEF)50nL2l7^=dPPhZ0=LFmd%jxlwvS=mXly+
zAnh#cQ)qDX%x6oxSdBw7#f_cMJ&h>8oBm>^>8TnA!7%W0-6Q#G4A;_ngl%1YFUg%M
za*uS&pUEKM2s%qLvA0WL@|g>YJ6H~?a_Bi0yxe$8n0d!BJBwLO3_FdMYSg<uoZwkd
zbb7bZ>AMORJ0HistC((m?`hxnf-T;)<*CyULyzF8zp}mDcX+0k?7SPBS00|Vb9K{1
z?ag<dhJqC8gN`Y)ED0`2zR>Gv&URO##CYyw$YH!Y!OI~H4D%v27U@_=eXCip!7pbU
z%OaLVSC|{-zfefxxUla#JMS5`v}7$ah6!6<r%dOl>Nvo_b)tWR!GdM)8H|DjY9+pa
z+kAmrbYAUUyzTsNIc1Kfq!X=*(^(koFLpZev03_jw~^rP4n7bNF?osU#3=cc#1$W9
z{~ld@N3!bAnrH0C7T1~w+x%K%x-3E9&c*i)@8g+;yWLmL_cSkhT<x|XIB(bK`uCsi
zzZaK%J~{8nzVe;#k&?nP=BEui?tIZR|NCzl_dE~`zjtMNQDbRiY2(r>4QHa(%_*6_
z``*)a<vU-OU$}itbdAyy9>r7+&BF~wB~unI=8;W(p&=Z%=l_HUMX>X7I2%?Te=&Ej
zfnUkwh0_aWmA_DPHkaIGIp?urf9eZWXK~40CpPr*M5dfjs#^43`NW%<cMin(eR2H%
z{-?R}nT>3$J*poQtS`EY=x}VCxNtJhZ3{2{k{7DZ`jS;AFSxhmrkqf8=9_utz~bF`
zZE{9mCdjY8%g(S^OHFiLTGoQo@$b3x%XYqweOEPo_dQY9H$|7LJ16b=z?<5$8|2Ns
zr+wcSUEY1?3u-G)&){BVU_nel+W}B3&b-9`wygQy={MfBzU`~NS9reK;-vGLL$>dC
zJbzxoJv*1#naSBJ#&Lqf1cwP)%HUJK&xwLh1CUg9UUbE*Kj3o$f9u;gkbVfUh$%7W
z^Pz8gIj;|G)0^|FH8%FfS=RiAe;7|JEn<cC(C#t>G)72fu|BvcE!E(D*q9+`>fhHI
z-@I0C*jfG5w_Hm1-Io}bHs42|@0>NQ`TZ$v$DWsZ`n%c>$DiW~kK2BF=Dz*6TG^i`
zh~Ig)htq%cZL6o5k1R~z_3jTm(R#iWbjHc?)??Dl>waAVRor?%zL+_=O6)Q(@jurs
zdkea*du@#LbvcGP@+^U%Q88t<g_|K`XWGYZ{A8XWyvv8F!1<4#f`LE;Qv>783Wd)O
zX<jpbOP-mj`IM`H=NTA+4{BsOFo`jsA!3FXr1Qy`z?#sj=_B2fTys#uT0Y^#Wsb-F
zSA}F<zWn@cW%RgT%{QS(=$NmBy5x!HZ$I6w><-~l&vsgy6ysMg{d{B0?!Rf%y)3ue
zcivxjB!7DU?Tv1xJgT2;chr>Y&A)rZ>a4og&*GmuV(s#rE<g8eEN?7toWGBK`qpbt
zHy8ih#d+!OeHkX}FJ~?oU)m<kCi46TcnIvc$7_jn*~M$v&Q5=!^W#{^cD{sZMa6ae
zEm2Wxc=+Zf1kb5AZaTstr0mh7aFiqFblHlfoq`tC8itcVf*W`l*wmOOe{@K5Id>~%
z-f5fZ#XSwCCwO$j*BJ_3Q!@sgv|aO)X&F-+Q`@0Ab6rbxXT4bM7~lJ1?N4S2E(tEs
zz9)qh3M-C_{NS?lYq=`tzWM1vC3#*G$B>)un{pp~?btQ1?`F7PQTzH6mhU!9zPYML
z-*n4`$p@^$_&9WWWQ#soUJ4F3=scbM?$fN@MkhLLBd2lM|10G`a$zysqeHI`sRy^s
zssDUln6b@L;<tDf+aI+axr-I9=MMbbtk!bO=lldk{e`D@znjr{_^>{hWjiI+-IK5D
zZ9tCtta}UZ@I5Wl*zRt!*ZGj*j#3S?H%W@?OF4|sBwU=9EAQB(C!{@vYrpaCBd^rm
zCP*}TY<FInD9>%H*2U_=KIK8l!t-Ai+~7OfCMVn!`{H}zzgGcS?GDv9COa?Su2uaq
zCE&63oORAK7S8)3@WN`!jfF=7+^t$}C>{y8ZryUjdBN*|q;?_emh1AXcUv;_ziZl9
zzW7I-unX4*AE(LUcN1fZW?7d!?OImO;c59wPkrZU&v!+Wtb4es7A~K8c)=5y@)ydx
zE?=lsd*Pz_>Ybpse*RAF@VifSzGpZv&voE4&^f+`RaPVYZu0HRhtn24<+&%cKRn~C
z6#p0DSc{T}7hESe8yK|9*lfw2bysH2nT7nb?#uA#NQ6&lZQmwa^58<_;e$<ytc#0I
zC~Vu?zD@RBWjo)Ztv$*=UA3*-b!0XbZ1T6UnBgqW7u%9=EivQp0hv!93dGpii(edB
zI61@n`Gsz0eZIXF7rYDf+T&!Z7Op?PVEuU(mPLoIO%{_}`$ndzh~w1597Rp5FCsl>
zl#kp}T=O7BQSn#u)}>4atSi)GdoM658vK0!lEYuUV*2lUPr;?Rxq4%3w3Jh~oQ1%g
zKR-(v%6EQ#zPjl(2(mHwoqW{&;K}{(hn;y2o?QRlYxnnpQ%(h+Uhls2e(sm*>Gt=Z
zo}bTq*F5g2bBMK|%v!c(4%-d)Nr}v44A^n)_(jFKIO}jVUDkAm1m;6h0@oTguzfLE
zu$<Ah*Fb`YBg-Lxxk>86`G!2UFGdTd3xWv7y*w{c9n_h_KQ_d%eKBHe0PQc{32ME8
z@Q1H1^31;^E*x%<W4m3y;V(OgaG$ZA?RWi#x5tx9t=J!2Y~^hb`7-aBZdDpzX_rLN
z^~>7DCt~fdFf@tH++6?R%1x1NMo;F+7oB%Me}CayJ1~>~&gU<uWQxx7?|j}q=|Q`c
zEsyG6{>uBXE^FI?2OZM8+&ABH$-lMUrT7*I1>bTBzO^_=>TIU)-gM!;=@&(B?)%Ip
z@xoSHcG6wzdUnCRY5Rq2(}Y1NRd{W(ux!H8{7u~}r)>PMoB!}1^V$UAwF$z1LxgKX
zgx>}W=LQ9Tai99gQqKLB&fg&6wSh|=H|>4?zGqFk+XZfZgDdA2U0iB%aT*x)n@mk+
z(Md|*m=U1$w4ftaAa}`rNzcyp4(gX&)GxUxUvg0n((Jr&{LkF;KFmR?pU=rR#LRqr
zajrpFY{SY(kI5JGXEk3Ep29cZ<cEs#oOSa|-Y-diE^ceRbdQsG(RKcvuiNi@b@qJk
z{O;#p->UpeRg=%n{&IK8UNv8n%vMkJGx}BupeBu{|4A4BlOndNFI8t|D6R~0_{`2G
zw^nhnXi7=VlEUP!W4$IhA({(*IzQt&AfkORZSni0MYc&@WgS7soB1aBeXdMBaV~Ai
z>cmB=35%E)G(CLIZ{u)$pUZN)m9OW#`dsezsQ0a({63fGdtLfr=>Hy<|9e~>+bHoQ
zMXPTsPO?@$!Q=P)r!_~G&%uSw8Hw)4cZtZpUc?y7>d4Bt_Snk!10Ut*?qoZ#=sU0M
zj&J)lw|H*fp+9T?#oxVt`4=zb7W{jiyKA$o?@czI1uxPM_OfL=Yx~EW?pl74`E9pc
z#umH9k0pE0EqVE#KPz?d+M5@g>Mq{vNlW{pvshS?_rwc(x0+)Y_O5>+D`tBrgLT@A
z7gJw6n3{0_gkEf4nsD@IvwdHk{qL#!B0C|dF0!*MGHBX!{xyQN8-?C(0HgInx$A?r
z?DZ;H3_>O|y-a3a+T<#2(lRIGyYX+k+=F5goPQSz^)5_#+j03mXuPiH!RnvP5}6;S
zWV}%^_AD{v-qewo#{Bu(pT_kvcc(8p8DtW?C^@jJ&$UG3*CgH432KH5gq!`89x-TU
z{yQg<&F)}zx!bm{t)yh(!jBVjw7EXXp30qHx@mHlhnR3C@8`_3)8(8t&GzV)D4O+N
z|1w9%H+z%MFB{i0Mtoiz>~H@eYuA^%3(xY!mMz)fy;{PnO1@~>`}LQ6+vXW==~~Fx
z6Y}T9hGj1vys$BLa}jcBHeD(199PD*kH7L}rh?Jy&G|exo$h({FK#@s=h^YV7q(n`
z+!Mc7rk?9$Q+WB}_r^ljge`A7mPvj&;PK@EsO#_|hEZa#Zr?P~oMRcWg1Xw5E-q&{
zQzGhdF;rm}r;9N|)<5-AzYoV7u}oE5!Q+_v!@F&s(U&O;m-l=VV7>cXw87o6oI&pE
zZN`XXE_d6qwg}LTC}4b}?>xuBnPm#1Ggltc3+|edacXUFcXrsjQ@0m2ub9MPKT&+H
z(<0|{#i!g}hjQpXcjaew=B!zopx&tSBv!uSdi$Q&S0*l7@S;QI%L+~Q`ia|XWh9?m
z+N62FdWvlQm564ZC$;jQ_Pt*pw9m53(EaG+6<?wiwhCEo{Aj&p+KQjrpN}=zaeWC~
z(9cvW@*|+phVOpa40X0IK?|mPaL8ZqZOr2;DORcD=U8!I1=pA01@oEyio96vV6GbV
z)HV6Oz#T5Pt_>DEI(#c$ZvL=>LGgkp#~OzOR;KQ#oP!6rTEY?%jKBU_vtNW`okPLX
zg?3I0_lt0>cc@@(fBuJ&CuE_w2nfx#b=VdV%*7({@9$6M37YcL7v;sU7Jy(W!<$&)
z6({9;y;}{J1w4O$rEhVVOz6XXA*&?+#@+m!a(1$={p*ku>P{2Qy|$fwQPh2@NBURY
z+>H-DpFcG}%JJ2fz}&amt5%lka+@B}P&DgK2{29CALQJVl(^aY{hvtYPOgMTf8j+8
z5m){T?b6Sxa^=;(yW~O0l-{ma0aMb0jy-9s%>H*^kLIB%8oQ@xY?z`k9fVd)Hu=1*
zH=&-9A$f{pm%*(SRU%h=gVt1t+*&x#Z&8WgA^|XB@>}%DchQ_)CYERXHUh3;bM8uR
z4W7d<Fx9K5r-4gZ)2rO2vv0+bhdEB`84_GXZp_##<HDZMcS<TyL-|jBpt$m@_<&<f
zog17IM446#gtcx+c`0gnWD48-jcvOQhA#+mS>)2fvCgSLv}v_K%+$lhf>E_D`wy<@
zHP|w3ipPQnj9>ZE%)acgmYaCgV;=X-PfND*unO_83cbmfn|UrOaTVv8OXAXPCaxkA
zj#!9zY}oI$%YNnWly~mO{!Ns)D}GmFfy3jgMw90F=hZzqyx3+!h3`Joo7cEjFz(v6
zyKVA`MfbLLySnpQ1x+($3IEx6*F&j9iP>u6uQ?~u{ZjPQCrwFzx0Qp-*6>qba6;1b
zNo8}C-p<f@6#3JKzd^GoPx+`>Q0B=F-zD?aCV4LZwUJ}BV*qbcmVs92`Rm0!Epwbz
z4zVeka;$W;*V=3^Zjr0*c6GsWwdL=XR<Jpma;!S*k$E@$-9)DFq~mUfBwxmK%B*$i
z3sch6o>;H?EAxVX3iITzy1Np!E>4{NulI@9L{Zmwm*2>(<-B%ug@~g=#J4;DZXHOG
zo)}`1y!rXvHS@dm4mor2NE)Q<nQ+_at9s(qPm^|TdTr0{9HjR3Pw<Zu(La@VTXjSx
z`FZHx3|-23<I^JTR%QP^%Wh7xciP0($g{i6BtPkGLzXb7xWNiO$5M_}hiCk{5-lcM
z`eKDYyC#cxN7LcPPL1Cpf!|!Nt1o&jeDSOC->P$Z`Ra#W353lT+FQq7oMyTyEP0vm
z>-H+8J$w&KIo3N@$R2tvaL=ykOylI{4?l`5IePr3AMZR|%CU*P{dMraIFXwRPuM*z
z{CiK{y`lKX|MP-V6g1kT0ynTMefQ~?(g{AtGO$W;6I5p1`7bvtj~<c|beXACWpwqx
z6+fAE&I@I^-Uei}A1>wC?7cy}?ZLt|cDzC5m45{^pGwGf+03Z>zDw6Dl!IxZZOa#d
zQ^q2V4ap(-S-Itv{{<9ly*{>cyu7@(qQIe5gU3mvW1%!xphC;@`KP~q*}wj|{J&QA
zKD*VuLGy$C{7<~(+-YOm_f4-iV6kS{`)Ri{!_`?{wW!RMbANk*?*ItiW4O?GW3D!X
zYy+PI`vT?*+zc`u=LD3cPBFd!ZR@xgb&bKG!9wB@!-I}#X4-qSdU_Sbd>DQyD+;T)
zH2#g+5w=dzzhs>pL&FTSbqqc*sI#cTqpCw$#lopcRHee<QG&nBo0khG+nijtR9<2c
zc#QfC!x@G%I*a&@f4OjThK2NHXZP4gE4*b2I~VH~HmSM>ykKtw-(ST5CX)oY1+<;P
zJGL3tY}_jRcS3I5{=Rc{zwa$qo_ulEu@`3(OKc5J9hJViigf|=M(J>dG6uKa?I0h%
zV`S)<sQHeegu#kIwn1>iTV4h}m-G(z1<p(6Enj@L*2J*q)_W!cC6+z13m*t-3Z`{f
z+)7d`O%;58Z@#_Ax~bWZZ|yuL$$8+2(z(6%^#(}3lw)Yf0NvjLUI@e6a15*<p@Z!J
z4-}qJI=44Ib?Rl&2FV6V@Z_2*XzbbG2oto2%XC1hJEAc{GKp2;f839TzC~v^Cog<|
zCNOm2XM-w}j@5_FV;qk$9kPF1<6RX0f<sy+vROE*z=_xO!AtfygE#}T&X0>q*PSb6
zP~e`_W+iZ)P5sgX7pXZe&B7T4OuZ92+0@TGP>_^SXl7RV_W#12m#25d@4w=ib(zs|
zGT-9xhQQBvRy1Xc>^FR5z>+X!@tu;89bFAK3qH2+ZLK(}_R3rOih2Et80qWhuI^-d
z@nbr}gdTZ@DGFy<4zMg>QFvUL%&TU4R{lWFM+FIE31jE;3?~>Bnhlv(d~kZ$dWgGK
zP;8#QC_}5Fn0=y2t=ldUsj1I;XRUTCx^8z@UstYVf>yADwdWg~U7}KVxt_oC%@VQ8
zy=GPVrsUNO7wO<zFZQIB^34x6{?QtG)c9QcH_Nx{y43t^%&*kFwa~PRoHqBz`ML5p
z8u$g`B(E-W(J1`-)%BTapXJ}R%tvngF8p`V@m_{S)|WR%t2Csa>VJRDGt=-?pXArH
z9veU9gJ(N__pAyNT&2a3tdM&n^YR8|W9!2GQv0Kyo#J?we(H>V{_=W;GYqGuW<R!K
z2$)dLr7t3<V-q3an3&~Tba9G!(TQUvj|{X{OD|p4JE`4m%kfuldly(T_H6Wf)T1y{
zP{Eiz$1UzmN2%mWUG;NEdNxd{%s4l{zV)ldpSXbX?sI`}o14!|-nTZMs$(|aSU+vq
zfwWUU@}Kd4X0#CMxs>6N{Uy)%7Q5tG&u^=9HXb=F-Pm)n&8AA%HKnlPT*;N!<&oQ;
z@z*_K{91CNA)#*DF%K!_jEGMwZ8AlU8FMvAPSxzW;;p>=7>AQU!$zYH<!O~CkKPb6
z-nRU2)}K6v3-j`7t{+R!sX2D+k(q2|@QWKWx>(-po0h-t&)8<CI`K@Qo4|@C{JGyP
zU8NqrjW4JYc)nc6L-9}Wy)2DKT7?T%uW~=j8z$pA`QH~Qx!m4ad^4l3mbTqGXLj?O
z#jT9<^-l}8oS%37SZHvF@4}ylFKo?^VY~r`9<_<Cf6mHVfpg|dRZv-D3@&S+B~JlE
zfeHhU0v}`t{jI=1FHbnDYzsYmdi&cY=brC8U2-;t`}(P^g>M-T{7jTF>RZRKgnjSD
z=6f$3^C9Gy1?E*3qQ5MdAMbeU<)687UyL1+nG6)9oFUg3n5;Fgx+ZP=`f0&@djpqQ
z*Q9-4_b&U|ZN@MoSuisy-(;;hOw8?a)nfO(QzKkvU5++cZvTDJ{BVX#&Hq#;rt6+-
zYu-L@eO17GJ;l~1^0#Vk>HSx~&FlADZFbAI8=6`FX1rVLUTiH_e>S!9&6l+uo|hl~
z$d6j}??3y$WPYEiuRrp?d~)2ry=U9iywJ0g+uz;(6!?wTIGX#1^Rt%P;Hy7h*xz2X
zFX_wQZ+UKF>i5jwCI4c$eWxsGdT#x;_Zxoy|0eX-S&ge?{%yOY`JWZiKF^nLpZDka
zxf|09_b}X?9`ft$kNY#O{QrM|U;f6<Q{QIz-Tf)LTx=`5?pv47{N;1}BJMhF_O3el
zO!tw|%19GUUEd(@mKCXYPgbow6Mf{=iM+4pQ}+iym!0$5<jl&dm1m=W=+4gD`ZH*L
z`G?5?{5xN(&fl)I(`S3lK9lVQT?ewA)Sr8si67b`uq|<YrO(#7dl#J~j-~0goNYBY
zdSlkY{&S7z7y^Q~EJ$7QEq2lG_Lpa(UAG&UpUu3<bINe8e%p%pua|6+ee_Zfx7&sv
zOLN<La(n8kz`R&}U;eY_c)mwVY^m92viWXVVe<ZFCELk|<SOSC?q>OPeyvXQ=_&4V
zm1^#9j84|MXU+a{xOQgD?qz5Et@6*do+wt_B&JmSB}?&BOvjqm6Hy&&-dl?+Wh-8&
z+$5%S9xTv0<I;I$D09N44-hTcipx5_z?hrF7M*F;i0W9>IwR@IVklE1s>4WpQHEld
zSkkGeekgNMhT^H1jzu8FniFhc%tfsl#fn{Gi!v0KUHJxMb|DO%bQj4`t#!*=S44Ga
z{<D1*)3FM~T-CawSaFruqAbN_JwIT~Rbq?IwXTTbSo1pAuja~50nuyR@2@&t*(ngp
z9k^9s^Xa#XLDCQdti%^(DXy9`ZTFk-rLV8<)Cv7Kqts*KDx*#Rihn+j3tqw<%RMXQ
zq}lbd>)*abd#*RFY`-O9GP_7%mdaVR%SY{}{|&pQdu5wuPX6zuKKpFmpL}^Kr8xhy
zu-VadTTf+&et)&+dWpZ+|2zo|<FHjH8t0@b9@GCd;Y@Yyz2du{+4p)rTf0-`OJ<&q
z=jBOTcB;gBJAI$DPW5H6=dP4@wm+snn;5%0RcjOfR*xm230n*=-g=|^wrkGylZ&={
zUcdEar`eZzi`oOl({D!0KAri)DI|Qh+`G!v@7#mu&uKj-96Kj9-+p4c;mL)30=ZWE
zp0BlwpEaj7>v8OyRDG{`>A!lM4xN)Jvr{wseBQPF$v(Czhe|JAx^l2weV6u)rDtBe
zs7znir>)&?)i!bB=4{PsP6t;1+46rQx0qGG-LmKqgXdMf+di-F95?!Z%Q-DZb6#p&
zgnx^Y=6sc}=l`GgRI$FbCxd+_pU_){boQMuvTw1<X~jFJ?Rq_D=Cmf;AJ6LD{{8&S
z9+SBtRjhXQWSQW!ne!`NX!F-T&6KD-Gv&}*$62>7q|e(n>G+J>%`I~m3Kx5@oT~D4
ze|BM_<y%eu($?i$e_ZIa)Ya*iY}{?j@ml8BzRNE@1zb@2n8|X)^UNYSiHSze^Gl-A
z<hZ^s&^dQOeeSkd$Itu;S`so-@!RKXF&!=z*Ox8zn0?L3_gT8Z)idljgtiNBpPaY4
z?X?@<WjE<7nGCZU!>eXa>Cr8=e!WfVU2f~{*4?h<+WNv@cZKGK-wlg+?#6Nd+79i=
zlEqoK+#D{trO(JLoHcRM=XJ%`cfQ_bT~S#uyJn>?_p~huovep%n*I;l@U+HqV?rmZ
zchrU@tll~iNe3s1>3{`vA}%$l>P93T6oM#+NoIlsB4Db)lAhetA~qa>s|HCL>(1Di
zaI9&j*qI1}<bwsz&+QlWvU<NMq2q(6QH_xpkD;C7@$>t?-;Od+`Prf|U*)uV@<D;;
z!k_;9souYq)p?PB#}`kdS|c%@jdqIH&+kvoD7d+=WIF5Qoc-<3U#5!j{H*)8wjfeH
zEPsD@$9$E??-i>*rylv!GROXZn8#Y-6Xx$XCEVoe6*`wL$g$1+#>@T73jbDZST}#!
zeCPk7?XSO-#httT)tmeI{!gD;U(Pr75u3g9@z?mvixdmkr?f9v@^k-xj>9+JuCA*~
z>Cc$*vS_+izv!#a!k<?kKP(pH%2#1|Y_Ik9AL|WvJ&Twn^KA8<zV<_MQy+hR(h%FX
zaVkSx(pT>T_i8o%gfrON<#XP6DpA4tz?3bHxy+U=j`@e+g+K_aqPOvS!`;5d?F{#l
z9<;0Ol4Z_k+;j9m^ntwN2cj9?8-9@6@Z0c%+<|JIJ?wAlc=oV=kYlz_aj|E6$9O0C
zf!v0_5<A!n`Wvecd`o^Hcfg&eX1$d``EPq}v%Zb}4faWIln?YLzfpc5A2?U}z<U{+
zZJ(y~$@71@|LJ3Vs%=ES{OynNe}Cx5^UGyiD*JQ2QB`*HjmYpn;g<hiADDEj>Vw1N
zzGh9?h#cQzd?wq@ZDQ_Td-6u(u~`}wZtre2W?am-;;@hvnZqaK*WGs6@yN>sk4h#K
zMwQIrbMj+XUEVz9@}j3D0>xG<EM=d}5wl(1yx@A{T)s(u-ODaJUU|9TRmp_XsFJyS
zhy2)6mp5;@yy$I-K)KZlYuP7zU+cyjo_^`6n|R8;Z~4o%TW=+Pi)X0%1pO*bI39d@
zP9}f1J3p6MX;F9hlgsyfJyY)F`Tw`(tNOU}WbeCw@3a3te<v7PbKz&>%Z94DFQ5Pa
z-T&d>-I&jtj_GIr{XdcY*)5y9KF5VW&zYTm<Iny|kF%H4=jnXD@?+MKpnJOw|9|~?
zM)-;Tqlia`KQGZUc=dVHU+>_x=lwQ3?)+92@+W(-#sBqxpR)e9=HC*Y{?Om?OF8$?
z`!kN;uU~M#ZoTKlegCJwFM0E?oc+@~haaUP?N%TDZ}^^fvhR(B*wg*TitOgF%>O(;
zf_<Zy>fKeRUY*HXeZpgNwwR(^{lVBN!YpP7djH>_d+~q0_m9(@nQecY|F5qvc~Ecj
z@TdI0x?2CB|6i9>&VML>Hs$HO5AoAI{`T9wu&+7s|LueLeMjxBKJKsW{{K;|{*dvX
z<k=7EE9Jk{+b;Zb-T2aEjj#2>pU&?u{eRo|pE>8#`b$~=|2Xfdom21W^2_=EEpy%f
z|J5#i>faRi&r<N|{`HD~!d+_a?5h9pe||7a{mcG(m52YIwfz5BUG#uE{B*_McWFsA
zkN-aRZD0NS_-D)N2|wrcoy&-?=ijRtaG*L&k@?@xTYqw{evkFBPgPfa5j<myr~JY9
zm%CPezrVfG-ubE8|3m*(+7b=j+dk~Cddsn=#G2LNSE=!zW2~IbO(m^6a+40X&TwFp
zG)il5&+OPTXMeR)?EXCu=V|hnzx>9m5zaG5;+X|g^)r_{NvixR8qb$~GM!*G(dC%x
z{9V2O^=2$r*;mLae`2xZZkha(l^+-v)_>}3UedGP{nU!XXSbeoSL}Vx@Hy^y@O?Xt
zHp!}GA}L`94?g^Rzis-z^Y`<Ad|}C+6{_*yar=GiZ6A03tc)`9mebIe;o{$8w4ZPL
zy!{@Q5|7sVXXHh-E%&$hay_wr+t)&^fBRSMtgkq<e*3>$rq4x{tt+m5@3VO+zl-bt
zTRU~<jmyN==dYYsbZEoR>3=kDN~;&uwf{{|wc+%xzqoe3%}pM~J^!nY{yzWh-|GGU
zrREDi|9$@JzwG*y`n=Ct^QEVrpL^=L-_qK-e|ARxytB6EUD&njC+ACFJJ0+2y7=#3
zX<v&E|Js@OHojiYq_bzso`ipkoxc{}|M6?l$DM^Wc{1~VAG<Ezd%pMaXV=HY=fT2>
zzwd0QeRqA|zKq|$c<s$|=KtQhe*VVVyc>2P_Sg0LH|)*d%$NRl{_n5r;$Q~v@8@-Y
zUvqyu&wKoN+vDQ>AHTAK8HYc9omE$R@Z-*hKkgLNyffdw@4=rt1$B8A_TOj6$AcN>
zbLRg(yI$YI-aKZ$^t$tZuU!`hGkC8*uY3KPd)@kpKXyL+btk{R_Tb;05C7dMsDHP8
z|GozxM&1tl@3-UQ3+nm*>8pOO>HYWowe>v<M)q6R@5f6$ki0KmquH2V`}c3@^Zl>p
zu8*$Io|k|1v-sW5YissxJ-_|e&+dOegB+fEGf(}s_S3%oU#1Ib#Ye5ES-CK1|GqEv
zp*Hn*zx3_=(zZWNviY3-Z=)+SSI@A>zgqHuTV+n~=Ypf2p^w*Zo{>>;@8p$J_R^jY
z*DQOyK6&Tc>U+(X^k<$psXW7C&hoOY{~sP-m+{{zOuK&S_xYY%4mIyN5&r1rOjaJR
z&+1?P&6%}D^-b*zJ}$?!-x}dhi(VC^SqPW#?*8yOUG|K?^E2&|PC;zD(@hWCrRaRn
zSwHzz2LIOY^LMT~a$L)4>;7}9z0>a1{I9%L)4Ko4#$$h0CK(hS$bC3}=d8x#H+v<Y
zteIBA*64oN-akRDVwP#gFSTU$V9xAGe=o1O_W5~9(8=;HgMyQy5&tTkf_l?u9&0)O
zJ*lX%>&d@JaluDPyOh7bf4~3ysZD9}p^Ep5o=tl9KJ@nQ_nUjx-gZAO@ig&P!YfS!
z;h*0N1$NorygZ}z)XT1S`eCnszkl3NBA3peaIdIt?Z-R8_v`1+5W9CeaK~Zsl}!xd
zlf?i0X1T(__*dug`E+&RkVeb1`~Bx+f6&=%6#Dm-zM2+y-|z2*v();Hq%LNPh5meg
z`j+#T^weKk&tLn;FaGw=xO=zn{M9CEn|N0E-d|kt|EJZnr(e{o&s6BBc^AL?eLg%b
z{)5f!le)4KpLU1buTPWxc=T~j@?F*7r|s*X|31IQX+wonSLx@=hPS82Fzm={H&uQZ
z$(8WCT2kQr*0z5?@0aGB|6if@|NZUt^<SIzv;NV2kk9zvkg1PPa>=c!=jVU@{9N0v
ze*5|Pe?LF}VO3f8FK)j5_v^m<@;ctv@YmekTyuBxmiOHI@}l>>1EFi*S>v~_i{HNP
ze*CxJx38^_z79gx*Wc&I?9Yn@p?5O(<E{4P$;NN*TW@{*`}Dr`(Z{c^ef;{|$G=M-
zzrOYsgwDPF?Y;f|{U31l`snQ-Wc~fU{@wj~VD#?J{{1EYca?%s@&A99{=UBU7=+F}
z*5Ci7UXt_i>%e2z7x%1Juk2m#E*al%v`;Rz#`a`(o$aYVRa1U`)iB<t|6|wJKOppZ
z&VBKkyH#~}t7`97*53WObzk0I5PG*~-~KPZc76RN_b|RZ?tXk#{apyEtgnChxAZj_
zJ^%Z6>G$nrKf9}bhGm@pe*Ktzz5K5*L;v6RUexb=5q;X`e%7<d`_lL0_t$9e{3NdW
zU0nD3+GlZ3qc2X~ud!=?#;*L(?^^dyt*g=A{b}u6ufN{zHQIYVt^IIn`!VgW`m!I_
zemS-MlJ@<m^;h<n9NGRz`+nE@Bk@=M##=p#{y8=OY52aa@tq(UCY}3S8)~u-T>5uO
z@V?iEE&F|}?w`K^;&g4O(LVHPEvNTyklBYnt>rnr-Av~%$ZUbr+nsdoOKtoDG9gIk
zzSqW@?4w3tO&cMaPREBN{=C-uG+J_6es09SMSQ2X+v(ir+E|l)<dgW+h<&9=Kf}6G
z{~hJ{;q6}Y`sk;%5~sJP>C^}IK8-e?mj5(j-`3=x-BmjGe{Ec|e}>Zi^B=@Tf5xZJ
zuif}7zP5((QqlLlbzdLe-8rkr;cnRtzuR9EdAhbV$d$cGiq2gce3R=A*B!1Y?<GE!
zaZib1dcyDhVv=m;Ux%ixDrHgj6B2cQo-aP08W6XQO9uiMHY_@FDOWT?Btk?ZT4+kL
z_q=6m4z51(N;!-b4y^^}tyA52dD4x{)N9&nnVcI9?PUBo0+N1o_%_1e+p?tKMeqLX
zzqRk*{<t6N+!_C3&YsIH%iCVOd+zPkb8kGp+_|uPim|&8*T>gS)J2>F8uwM1DcdXl
zeb(NxcvFSxbn$idqMls>fxD)-ns8`(Xik4MiF<W_U~PDYo~9;m(khlGr@1F8JI5)g
z{juucJi?O1WVqg;FF##jnIah8)cxk|?C0#~>;r=CKJGr-i>->5M7`~J35Akn=N!&C
z^tl8u8y=p|X~=AN_?+a4nI~p;z+u@r7Z5z-1cjT5#frp=roHD)5=$~mGE3Svck{&L
zr^|b1^h#Jh-~Rr7E#up{O8Lcl8fULf|L{?_riNGk{^8@|A0xvXgzmrJ!c@EZbnWW*
z*|qnx<KKkuT>q_hd+pBd{PLCMQ#3Ao-~FR$f7kkokFxsvo*do2=f^bh_|K23;~v?*
zoAP3JT+Pky{OXUn`upA;{l0h0?aqClj+XD+Vm(RpMRa`4+wS*&=dV$(-`%iCOk-8!
zA~_AMrbTKRSxuML&wKNaE30{tmBy>)MNS%7EsNYVR<(HZeb>rrHCp#7bV1j`6_Q=E
z7Os%$+VwE>3ZLuOfGhm2tbxx$pK*n_ij=w>lPhY^TC_sFt838;&8}ICR%mtYdh}H<
zXZ<dXSKW&aX=L>*I`%`Pd-vQG*LEz`I<?RE+}xs+;M&i7tzWJ%*QlLrJ*Of>=IhR_
z`dc#eVmDVUUS%HfQfiZTyUn(nS^-A`uS$pR4ZJF?6K7?8@A4w1_eUzyk6D}E+Zni5
z@${Q(Q+E4ozO;=^XVn(2ko=IZY+D_Q+WUWOd-P}hqZO}uSKV9jwro+?;#C%_x)#SB
zeHgK_Z|bXvm4C|?H7~V_uoHR8v8SZ|r^-*x;A5*Lzxb&McepOtdBR^VmhJhWr|J^3
zW_xIOXmn_t&;TW`4ok-H84EJ?O4qV1HtAT|vGOvvY1>5M4$dPd8l=JLE3R>lL!a4J
zdk;{0a`xGMw`oq>oVFQlGrT*0JzsV0zTSe9d@j@FRLsp^UiO<`7H_(1%jKoDai<jZ
z?o9XcxTYZ4q{2MuhkKv=&zet%ydQ4XQ|<p^eRGQUGUa?fvkps5pPesv_s{5@(K(|P
z46iSp<#_JZbQv}2;{r)y;Iy`R;%32Q*>j-u*3F!2oqqq~i-Ikm+@gwZS}Dl8KY2W%
z=FMW0r$z0DgC70)BzK={5A&bCg7%6u_Kk-F>)L<HKIi-L$U(pF%cFqvPvpyfbntzD
zRH4uJ<57ct-;YNV&Ku2nv%|`xvG4bz1NwY_9-YwV`x{>PLEOE*@Q3*E`podvngP54
zx8Ax;bqM4ZiU0d(cYMwFZvFa?Z}tC{En;3^!PMovz=EwSd4UB-*Wv{|w>$Y-9(iay
zUKPh`CDzrv&_e3ep1Cg?H2OLoP0;A;d^AJjvsN+dx!aw5J&#Uk@bx~rq0!g-=z&IG
z-=;kEb=LMjI`}3&lF;m%_((ysZ&LKSX+r8d54)GFm{OZAqaJ(w)N-%hxe>BDR}Tis
z>eR36`fJV+P<Z9Jm}c>#>UTB&^xh`C2d9ix7OOzXE^FuKU7)n`YOBCKwI-(Z6ARX|
zd_A^0&e`#3;QFNj>n|>K_<F2e|9<5@n69qJ&4)EV{EbguvC1NAcUccps7xp*#mYN<
zz1^wRx9ZUfF!49ajpNBZRr~l2|D1F0e%k!}`HAy0=QDIj%-8QO5Lu@m6aVCxo0Fr9
zqub25jv0#tv-uU4-Fw7b#_juEaANSp;8tTnP%?`+tRV22>j_H{lVPJoV4!bSeSg|=
zlP4y-Sq*ui@a(ylCtjWa7hh%iNmfa_=3bt7d7`lZ2rB>o;c#X{F<TK^5nB>lk{TE;
zn`=DLc%t#&y@JMq#)6v#HouwE3WDJD^|ZQqM(XpNCpk}Yj=*5Vp$}VY87DS>4cZdd
zb<O)?m1dN_>+7Wgv9;pizl^^=$ro=>4d3#lYg)|3N!n3GuHJgZbKl;zKK5Q=|KdA|
zaqkSu=ZLfaT76};;h&u{Z;oD!lX-XaVVunSqm2Kx_<lZ`q0jeAD4bpMLSnqk_oI#f
zwfO!%x}nea@6ii=zW<Lt=)cX{x{M*zA&^}JoW!m=1oDb}b$Gd5IFxm1yz5Gbt&V}R
zB3B&)<);){2Tl#R!a08t3ztmzSFM1(0mt6DggOVBi%fM6v=+JQ9B3=@)p_Q2;ZQ+W
z*}xDXSKYu6VOP_@E5fOF{X#-TN?ijZMYg&I#)@2Z4UC^sYaKW>@QS4C(!eWH+dpOA
zEI+i<xNmL*IKrXC*H^s9y-L>vtk^hpeNzZm$%YUR7fb}K*f=r4Z}kCd?S0E#S?i{@
zJ2f;fy|?qDUtf9I8gZ?l1<6mtel841e)mBTl3catzgWn0`>hMRcj<;JWzX|it-@9%
zFSQEG+CBG0!b-krufRma%D+*}95tV&>A#;dU*t)~&zql%pFIDwbL*FX0lI&FvkEY?
z*>CBITByxk8@VRWDSujC{^f<Oa=IRxhbDBa{HN5hvZM0qvwVl<8L#-09i$AW-QYU%
zB<pmT;Zv;zF-Iq@jwllKUiVMpx|m3<>60H>W=Blo1&^2<F*&m3W$TH|6B}8AK((g3
z&-c*H#a2aDMOH~xNuadFmb6CUv(yR6eVaQZ!Ek2BOju3na^}lj_cJbMwt#C&u_PFr
zHh1%c<i-`fGhi?mlJf5F|IPG^E$V})_qwc;q0{;{FTVXt&Td~vfBvJz2fl0Px=;Nb
zcK`aXeb@Khc)IV#_p|%fZ+ri3^^daYRqmZkwFkTPD?ak-?|XE#{7-1j>F)KFA8qyb
zy-<C(BY463s!r=KHMhIhSAVS4-}m9D_`V;<#N&Tn%2WS!O+5bhqw4sYkKOupAJ6J<
z{}sM}?u*;;HGjSLpK@&EYk0(=!Pj^xU)_qU>+k{#KD(t;)=m47{PkJBc)%4d*R2kb
z7w;s#UnSMGYvBr+u2TzF$acMIWGdVFSgV+Im2#KYq7^D#QHxfn#ksl$UJ-E(4ZI?n
zdiSMcV^;U3_jA^QtISgxuX+|;($MN%bWLLwua@FGao3|CSBkt{vlv>jz2aTP@Qm-d
zoapi8OI9#JYsvMnB1fyQO8qD}wOMamW$|jR<ohcX&yQJK-n+T+Uc!7(x;1_0v)Vz_
zJ+v_7t6OOPoj>cZTi2|3^dqA1`+kUNx8J&erA3eHN`ulls21LO-!G)Fp)%9@S^AUo
z)92^SSGJEAQBo=HU9tRVJoj|*b^06{9u=Y~A}u1TC6$&+*oanq))e@>D$)Fc!ZM|0
zO3P-swa;)kqdh^mP8QrCYn<WGbF)G}dFP`BsVo1a&gMD#IQwi)mnxDfl1c(4ClH2|
zVwVL$5LA*Gzmeg-&-;YO`g#A=CHvzKEbwNJwR&-BW9{UZmp$kE{VKV%bh6&ul9w($
zt`_mamco|Yh6uR6b<R@>P;<@g3<PE?rz@u`rzxjdi@i@MF|j*3@w)}6uDqNhJE=eU
zQ(=Hz->KtyTJJx;W!SI$hp+H>#m?kEd`0d*SKldo2dXQB)*lO4ki6I8M)%^sp`em1
zUgqOb#dw*|M-Ag2y;9h{-r`+1^L~p@-Ol?hzI7+>xA@_`d+v)L`fpd6vaVw4vRbf$
zxyx(8mfM}L8W$;PWHl|)(s<RhNKZqn`BJ|6D*i66g)0QRv=*)qwmWJdYLTZ7ttz_?
zFSL;9dc4p=;nbeFFJ@@)bs4QU3SV$|k%e9t^WrtPJE2wOqYoN<{hQu>S#Y4#`l4Zw
zjLOl#AQ`ozi9s^zM;n8_y**N!KF9j0#dC`Ya0vq@JaW&=`q_etnV%v~;R`?|4@gKW
z8&n>J9QWEg`O#caS?wj;w%5m<{?D%Uwm$ZNsnEKq1!r5b)nl9;_kLXU^;!1K-Qem^
zt8djkaARzNLe0Am&v$VqynpfV=)-lQ<^QdphC|XAC>@HrhvtWTb$feVFR1WJovt#A
z&{mPZr~TjE`t*6pw(ZlF#drs-TdI2f(xmK&B3|!xi6>2iZf{;D&8x&^#O1^l#FeC#
zWVH2fPS(MD>;0Q&bj|3Rv1^ZjW#b$NpC6?HpbC^F$!P1|K*{xiZ0iF*@u^M75qR8x
z@y+x$Xe+I4M&FFS8F|l3o|HT(IRb+`*69lA3db)Uh4cZg{gDH83S7>)oPohD#cD-r
zMQSJ|+2#qGYw`s)3v3qLEV%jYoL&%|0V>(v{w|MqsAm7Xl+(p@?et||FKvmNb<O*7
zRc4g_*(={_roXpd>u~>7(fwDyqwasdB|iJgw>uAazdapWv)g;`VwcA6fB!XuD%k*;
zx6o=6l-LC0Wj-A>{IAvb>(K^%aO12G(l~o`qh1uXaTdbpD(jF{{qBR{Rc2S$fGaGn
zz5&0kPL5^7=mJEan{(ycox;%dOrbokX9KSAxLyr-_SR)9q~Z*8oKiV;CsU|k>OH>@
zZ;`7mf&L<*u7SZKrmlgZQ%bD^a|1)fU26kFBwTv~uSmE~4gB`jWvg3Y?i6rxo3U;&
zBuyx6wJo>!xB}Ebhc?4jRW*Y=Z4q7=0!|yC;wj`S+se1;$E%}49)!x2r|k)TJXac&
z{GRj4Uj#MoOy9Yzb^z736K>73t~v7kkTdJP;(2*rJ|y{X7lzcR;8vja)r~>2+FS4Y
zm29~3Z>i2~uf?k0->wuq&HKst)90hM)pdP;=KS3GsknG%c(LWmx3UVWlJ}OZncnyH
z(#ANeYu?AJUM|~K-mn79KX=rme%-0R<{S&W+xEK85Bs&{(sIY?Y_U@Rwv=RESi0W9
zPeEPa@uZ09!85wk1EO8NT%GW^`li5T7!-8Qsy;sDbwiP?_qybhr8+zNRIhJ*vTNmA
z+2D1|r(k@mX>%+>%(w1MK2_SV0<6MnTFmiDuOo`?UisFid7W_)n7{bZw@l>58>EQa
zR%}*eRs`yYLh!b^o8M=YSlAuC_VEX}j05!tlzn!t)4a~QXxFru>yutb6~%h5%Rc#a
z_N{xSPj@+lAcRVFch&{p-t7=_z3OF@e(dX|*?UXYPG9%+Qhrvoc*iRgp{;Shd~@G3
z1tO{18n^43_x_b{V-;4xG=hb$SG_dfx|iFf6l@kuC`$kDvTfy!E2Q0CqZo|WuBYe2
zmhO*s2=o{E>Jk_zVtRD-3hh(7ikI3?f3;|Ze%G%>D-61%9*06(ajshfzs;=`i~v=g
zA6LOjDsYn>lr(?c`MKu~=iU49M+3p_JCL%#tJ2H1mHY10)40C%2~v37zw!-#5ZcSX
z;$Ypk`C7jEzQgU;`*w!i`Mcf$l(4UqSqB~s{4MLk6#^;q>@+8>*(|Q;eL-)H+dGR-
z#m}CfI)9e+=;@vN3Tho49O1o$ra4_RvOdk8c)5Gm#S(czOHoTvLs7%V84f-_as`~T
zelv7PMd&lXPEFX`BQM^#tLZ?<p6QJ-jt*-x>$^Z*(zY3Ch=OjnxS_1!U2xw4(q03T
zpmNUPoWmIhXh{bt=a%g^2lpJrlBUhw{AcaIh@(G$v+DdRxwy1jFSg`m5sz+lzunz0
zFPHh7$L<1^euA3?Ki@iWlh5r)!bx7;oeNdXw>)W^7Sk7X{K?}5HOtm6{uKD2x<dN`
zV_w$p3sa<<ik!XkGT<tY`R2;cDsm1$5=uGc+8`wR`KTkPc4f7C)BSj##RrHog)D^5
zwrO{EtbFSnoX6pUu=dV{Nz&$9Yco#qHm-oW1SAwy^my4ee&3yn0;V9xL4=&W^X^>v
zwoucYWl`HSxKPGPSw`dKr#@@XPctsGGig1>GFSTU$!DQ+li7Kbw34)vm_V@rYT}%L
zj5&RB^)!3nv~{}X{b$^gIYL_tkL7n{c4VIBo_Kj8s6_+ApI?EyZs4E?HJTB@4~YOs
z@I$(7YDvrNug+iXTED-+b-v!Ol8ej1fxpOGJa*TMOJ}~=&i?We(z*eYK54(^`GPxf
zON`EMyAc-cGV$eX>s@a*l!6*J?v1XEOEnf~Y;0M})Xmh*^j>5C;X8$X%iiC77@*(x
z^-+R;-?v8@`k<iz^~>hvKRWt;K3bsP_v_ILeZJq1c0k4cK6;?v_Yd4pyOgKCim^)z
z(ym#-+7-261)JCYxhvE(Rzcb|i_A2@)y?0sMN$h_2z6O4Tp`jGwQz-KSJuLw+nrf$
zi?TFcwJj>r$ZB6yrm?EsC|`Y*VwcvU6`;1Wg+Z6{VhfW~yXL-lp~2Vx=!eE@KYp%~
z7Lj(3o7;tDRgXejHZmGVCkB0+Tkz%xq_YI=F@Z;b`lg1LZU|xb)-4wb2Dfg&t)4jW
zK*806LGPDTq#ymhWrf4>Q>(2aUd@$yf9J=)_23F7<oK!8$L40%f_r8m$4|+xX{_JR
z@b%c!88v$~yY9Wd{decR%4662HWuGcXq;RZ3aZ3%%a?^N=vutWd{x!rZ`)H_j;Pmv
zJoJ9v{3l1Z{CV@!^3&&|r#C|CoXUs4ni`r~&V2EeijZm%St4>I<VeVTCc~L8XQx?!
zlGnR#P`k##C+*j@R?X>KUfons12=6>{Fpg&U+LcyrVkd`8t?NJ&*~7|ocYbi89clR
z>7+r1HzA!gD{wchZLaYI;|a!p=7Pruj0KF}&gqo#e(8I}2L`RvmC_Z{71Lnwrse7H
zY?j=H(olH&+|3g=Pu%Rdxv%~^>lfad8dmw=?4Mb)@}uGpAOHU0qiao#usrK8ul=WL
zSH6#~jnBUSy6FDv-%0n)Z-@U5tw9;u6u<xBk@i2WeNR9`o3q64gNHWztXYrAR_=Rq
zbo<^PyZ#@n`m~)}qYpMDpniF~p0DMMgPCt*N?}6+6JAD7UnbLXe+yTLxJapEprpuF
z#+my{Gfy3S?7P~b)G1J1WU5o3HbmK%J6pIy%tcBMuiYGZg)imTwWXTuug%snU4;w@
zfCd7ft(s8R!0;)h%h_4i-R^{qYJ$6H=QOl>H|49pvwnO>^SGruxUvBCltTA9F3W#c
zAg~hDvYBSxsTlwgP5JeVFWm^-tPynwH=?gfgT&-Np6e;k+Ee`acGvFzZ;v2)ajRgx
zz258XQn%e}Hh<Fiu-EF9-(9X8><ybY*QFdDO}@YGgY0MHPn(~npFbZgwo|RJ*!R)1
zSx3%_K6=JgSlqhfrwK=esEbI72&kt9No7a&XccJ{feJ9Kq;-{V^P6XcE2ty&(`Ll2
zm}4Jy@?RaJP~9mp)wfOA-)61ni2Y^s=Gu$PZu8A{ZBbmN41#LLwc0AzM1@{+b(Xds
z+2xcJtGIF9gotp@HS`sp6mi`<DOPpkx=A3rSF2nL6?(1JSvvK|t~eOO3q}4bhC+z^
z^dq|<3{?5ONG2l5zs6#t)$QAz`+glQ-~Z#2`2By6ZpZ(+I+^Ruc41kjqlN)8EJp(a
zWLS?T2FS3d-tm(O|N5(gukBHWMqk^b0*yXU;b*1NmAuG8<J6v0F^zoPk9KIhE?eZh
z*ut!<d9j66*W$$%Hm7!7KK$0D-J`Hagx#}niio>s;S7=EC*{{f)bDQq4I%VydjF<+
zdHnNuP^%;4E8EO{I}hIjmrmdj0Nrw@{l?(Y&sCryP!ItstyY<z_;S|py>guNLn+Sd
zUOUnbZ56&Ey}n=ac|9l*U)kZ-4jwE3rS%Vj_mBFD{CE}`@}Ro*_uC`o^EURc05_+g
zY4|T>C_ZcdG(o2Me}Y=qPn$n+zWQbJNk^=Zk-|nkk2TIhuN#h7ftkz*rnAuN;(B>z
zL(?<I7dra5_Aq2likR>F1T?er@VcXqf||f&m1~Thr4C856C&6V%ytBmUFF(xN9a_K
zRne}y-QZ3isN2W$gy#v*5iqP*I|AwUr7NYu;4RC`_oXd`4TYg&38&9Ro`^gV*%7(V
zyd$#XX2;F4b8aAb#_h~E_5a18PxHk$yu7sc&0cFM**^b`r8ahaCyopLv{0G<a7V4o
z)c(VuW>1aw2fh#1hh!h6{AyFVrr25Pkt91Qq8-MZe#A;;BcCTIG$tUJJ7CgCDknu8
zSH)C0;fR$Ux|v{2h3IC2jqmeDu^p^Zv9mNr=rvdx)gg~z(xf~2`O?BQ;9*gRq`B5}
z%4LwcW>+_US_K+*fDCa$>*`k#l~;>+dF4Oa>S}%P>jCxu7AHUb$|?V=<9bo$|L}x6
zoneizDjig>pWZ$1_-x*&#<yjUs(rWj|2Vfc6jsZByWI&j-#%RR|7@0&`4@Kn*`R9g
zzbs#+vlJ6~pd}?pssfA7|9<K^x8c@bn=UFWE0Y(rG(V<m*ClcP61WLIdFDQX2@$3M
zHF*xCC~91S6V*A{%h$|520r*bHS9s{%G%ZOPxoE_eso{{w)JH@*Kd1Y_VKCSzPsxm
z{f@0U*qvYTQ4~DZy64BF=~e7=-&cNg)h|3gm#Ow@_xq}ksrrTN^WRs0Y}McQ@o4zI
zAE&05xo-lEa^L^`s5`#qbNBkXk5~2g{XRN<|CigHfBRn9NBRE?V{}aoxWeXI8W6(n
zS{o3;LC>r*U)+B$V`u5Rf@^`Gh8(!41PxNLsa#8U-#K{(Y{CaTML8kjIiLDwYt4I+
z(1iMm*JDj_d;bqVhTRvPS@*5o@%z5t8sv_qzi%w-w%eWgSvm3_&z@R*;QK2L)>Y<N
zyXV5nO|atpy^RiME>6F%YOib`UH9#e%umbDlPz9x3#==seE8|mS5`(=PFBIx3gsmt
zM?^N-^d`+;bZetY$H^<-Hh0eHnjwAFwP)K_dmjb0;M>fPo;B>SRLJA&m<?wBtSSEO
zGOb8bZ>jsF^aHM0pn0UT`!c~D5y&jk`uJDpuddu5@37q8y7tz~%YM`Oc9mRM+N>vA
zk~s;vjD=(j2T%zDXCf8CWU56fWQI;*m|h*ap6U2o7kB4E3z6f_g$^R@E`=^4?JhI7
z3(JZgO$?L~KiU{5BXM+Mpo~=NUB8kNk#_gO8gTp4%BjnF$(!4qeUly;X!cEh<e<qn
z<&lTxZ$FE&Bv8@j4Iv=qXUK7`m2Y<*wp??^)2Fd-s(r-ee|3zndZXSgSqH6nwfd@f
zTe21R__C;fuUg9@>b@6Lg|A!q;oSeA1yzr~8a{qk&kC8&gw*$O?-%WQKPiW+#yC0N
z+O6^Z*SAqFq2N|6l=xk{@X_a_qpBPGR6xisUJ;%wD}!z;d#+LTT!U0)GlMG+L`DAV
zPI$!S<qkf}wmm--D0L*{$epF4&z>r}m6!zIM%4zX|IrO2TlAL+s8ih`i98bp8Nmg0
zti_68^HAV%jhg%k(6L+42nKBY7Cd%)|G@F^4;$+Z;#KnaI%hjSdNyH)C7Ag+blZGz
zH|K5kU-s80@9QnFICfh0arp%6Q;Hi=>rQ8IpbW;U8P)_Lqc!ol2+Jt$mBpZh81+tN
z8F;u0G(0=!B(%c+>ep2sR<Z^pR&&JK_}<Eo=X&3lwsV!N*<<(6S^i`0&O86PN+2m6
zqNrRkl6Bwf$~!Bg8{b0KaI69?<N$ZxS0yj~cDwU$6tjm4pH|DX`8B=M=G)r)XH}~d
z7DJH39T@?0_9JJPJ31`eCcm(L&8NSAnYjK&zBupHE>^W=hQIrCwa(II*MA0GyrZ|&
z@l34D?mYsQY&%%px2~ymWd;rF_>ftQQC<*|V}eJLETx5J&fBezwBu@yck5Sv)Yaej
z^yu|HKjw+YmjvBLO8H>s&n<7u7Ok)OI9Gq)m!sSF{eVuP-g@f-SqJj7J0G+TWZ%D|
z-$AuIU*jVQ4ZfyF8XA2~j|?>WnvL?*`&Q@GxUf4HnutJ3&cpI+96;R{&=Am~It{H3
zqxDAHi$U}CjO4Xyzy91CDg&;oL-#snRlh5EgjTJCO4G_Wx7fh-E_nUWE8d{n=li~I
zS#iMH@ZQXie!b;+A^CgkxcO_3lr!DvmAlvZEvoSQe$b36qL`dNYZs`0e*el1JJ{+Y
z$lwojRS0C(K9RTSnR?rc?);0Ni=R9{bAIal*{Ao+I_;U;4(|D^756|Cq@HWe|E=xe
zesb}~+>VtU6E8h36>m{e7fAL9w+T3WD_&K=Im`OUS;oTR4F>wK>izPaJjcE-(?E4P
z2*XBrUxHVLK;~7!<3EtmpFcANjNi;@oYOc5w7@7+{{K#`FWb3il)PNxJsIpCB-enM
z6NnQHzKy-~Q`v~EyHQxUg5pldMgdAwJ=y)H!eWm*GD7C<N6y~7=ND66_d!rr>8N3l
zjPg;(AQ{!v`+g-0MBKd!!3&gF!ax&NA;-H&ofBF2^%7#5B=+^v`&YjG`@GOQ;DP+=
zs{g!9Au{v-tj{j7IA4Ee$M5^|y}|W1Xz(d?@5f&vSNE3nFx@|M1H4?$E%&-c5U64M
zw)|Y*`SUY%ndaBHUAhr>#NnV&bloRWx4LtGVt(%2xuC-GI%IAiyc+H@w_xyuV8P(C
z%XTk}kUDbZowCyNO`8;#DIF^nReLIVQB+EGt1P0|4zJd!R+pm<4B#^Nyc)Dt2d&4P
zHh1$x(CRqwj4Y&9f6Lt=vql_IiJ#kc-~5{R$C=yqb@nseU%!Rvz4zPb+^pS4&MG1z
zR<W>nTUq4Czk2%~95vtbqf0#g<0I?1nv>ppo8PUI|5Xf`o&BP^j&D=3?6;$Z@iIRq
zAbRxlXwPaohz4E}bX5(!BIIfcTGV74c<rqVXarMas%v00XgyP`$XC~!+l50VUFQad
zNV%>J43SQ~&-whEMOhMTOb9f-69O45Y9n<T<>TjtI-v`?9&dgZ(AYQix_ZqE<$b!-
zf5$h1JM_@1+paZ5^vAmA3muMo<$}gCx5i&x=D_Y<y5>sVT1FPTpE)yc+0~!8qr`OU
zlknkgeS^=%g$K_q!_wsekC8u8n{ZK~?aDV!0n?3f&(5nVIrpq~d>_<gAAbE`9i!S)
z$-82bSJ!;|%d^Ym#ia$_@x6ArUoxG2K8rf`sHRCJ-TLD6!~`+xA8&eO!Yw<)rwJSQ
zCEhTz{cbpD2WFmcWwtzAE@sFKhM>WqCENUcEK$@jv;8i9=ycBM9B7dV1VdH@L*`&X
zBxqGII4yz~obVk#{_)2R#~DSNwzaQ$tC(GV;+7nY8FTI1gsrwIx%`p0oe|8(Fli)}
zCvM42$5iQj?b{A?Gr^jUr=pn&F<uYFcCgC!HE#=1Y=@{!$?ZRla3@TqVs<q#PCov}
zKBVJ+B~#a}MQ^OrR~vLmEnZ>NWwm&PNtf5+6{cNTk3+xBt(#j9o_U2=+3;D~sCO;v
z!0pK^Me3#CgbE_AWGlpQKg?AMVU5uSE!mt|@U|seV~?h{=!$z4wPJE{s~@X1XnyFm
zjn#hlFFw3M2x|J)SWri&D`&efR3P+d;P0*JA}j8#_?s)K`0-@BmQ7ThTiv!lZ+>=i
z{(AH?ye?1gbC#ZdOuRtA1cxl^HE$WSs}J0g+pul^Z>43wL^>3=)+V0%n<Wrz+Wyx2
zh=;S<{NtyuF3?luI3jUgl;_H-Z|a>-K?~?IK@)*x+(|k;*Wb*9EJFb;K>=apWhme=
zN$@h1^dHuWX$oL?%M!G*36umO`0}}#knzZlNC+%D_i3-=xlii<lTDw!2aQGgPS4v_
za(U@`-}!mJN-i&-t#`NYsI`P_uYchaTkzPVNp$;@$1`eVrt~lVbd#&1KcD@F?8EX7
zYgjkNH#dG(k<pEIo^OiHwI>qf!A#8qALC>`9EB`FiI@2dD)@ePfu>};mG@hG0j+oX
z*4?~+%k9p-|CiQ91up>43tBB$!Q!=h?h5ESbnv|3tENRp8h^_c{aUa>pi2rmFDTsQ
z1)36k)w(E212!+1r=isjni7=n;#zcO^%>C>eHyPi7EJ<;idQXKp)o1q!ycdGPM%w5
zT1)UvoN_ht?e7xK70s_-{mgmM?;$H6nf3P0#`!U4CcWJmug>_|b8E>r!$)ET$99!H
z`hQyCEr-qevIos&lboxcdCvDXS9g0C{Op;hdE|{#uaz`^e)a#A@BF-cQbP4I-tBW#
zq<`(7EO6VHfA`Z4tFsRlx_vdgHFLuGx_OBoc$OA<uG=YPbLfGfSK9k&Z%!Q)=9!mJ
zc%4V2=Ii<S%T4R|H~6m4ud2O#{kEMF<L=L|l4^47qi3n<s{i-+)a0SnF>hzoUg7hv
zXSzH|dUQCl>fWdJr{(@ky6F`!%@f_9YtKyFa;D2_+Rkaa`r^z^a#sCesL|kmcF9_3
z-Mi~0M|kCQ-+nU@IQQD%=d8))sh`&Q{7U=em%D9S^qk7L8!|h%bg#U(z5gMj|9Q{*
z#fv1pit6{K6x#|%Ot>SXX<_uzwy4Jap0dZpbLM5%0q+_LpV)oX&;O)<RBq<8c`rB4
z%r0wKcfzHrSkJe>Ci%bTwr|t?FMQ&x`d1rtM5$e>ZMv5IiC?^%+G9V7tm!zVYNqSF
zI6bIf=j7FMV!nv{Y_3mzF>mG8*%8xZJ^Bj{{eAdG?={=qy>Xtx8MpTYiZOd=FPN=#
z&sWgiTz99;=h9~j`7Zmnzfj-ZJXd+;;d6IxEKGWJ;ntJZ&kRSje?BPRRQT3s!Tp^I
zMe3ineouWFC97bQ+upw6&dj^aGt?Y1nI@<m$YAPFyZ`(}&a(&$f1v~Vj8ei4#*AUY
z4(2g4lDjTm{QPH{(!>x+HQm|mACJi`*>G^>gm=%59xwUjr0~x{_1kY&8SPIcy|rPd
z%K85u-o|IO_gwT`r*<0$A!Y@`1A;#div)b$ce!asr<S?%AIiMq`TbnXj^c)QGFz?h
z9)DYrc*2h7=JHD{hks?3`?$W#d|9}y#3K6F>^)w(d(XCQzIo|-&daQA{r)BeEAA92
zPLDjGU+lqijOD&r$o!g{-(p;+^TvO>I^pt_CQwlU!crz$kNf8I%~|KVxU>a4d9g0m
z#KOYD;@RJ#n+5$RZPm8VuzU4HZgMqz7Wasbe5oSJJnqjG+?LXoD|cPq6kflWL*mbi
zcZ=mu_ghJ`&YNcA9TK_V)Tb*RcV~(}x$6Gd!1B4ynV=A{MQ)4Mt#RZP2p3zhscVYq
zk~Ipce>GOSD=)kh<2dV6*r{mE{H1%Qim%^O@$~5I_ZvRHntMO-`B7{A+m$!Zoi2+j
zV`ID{H*v!Kg|<Jg%FZmC%sBgq?WPN!o`*h|gl?AbI~`^5+*rI@^!^6>u*AP*jH=Ze
zeRk<uZwfGCdsUiYt$iS<ZqqA0>rYG4=JGB|Ti{<=_~htBarZcx%8#BWvmzN+q%ta`
zZTUP?+&%8jr$=EsoKI@JH(b$B+4-FL>p#c8Kejj2w>Lbw93LsH!5KN3_pzzbyPd_4
zC9;KcQiB%dE?d9rfm^J>ymd>seS!@$e|XoeI5+uN=@P&D0*$twm;OK6SNZsemiN8I
zPiw6sOH4QabDvjvXq)lFEi<1ge%@GgJ@BNl;MI_omhDf9=d9lpP_=zq*__0$TWu6p
zCM$DG%4Vm$?fI1T=UM9U@|@0(Q)5g7Ub=ttIcWXtjNdWYh_rg%`<G3&NwuWv-jhpT
z@^tazogX)r&9bt7e*CZbo?qf`t$tQk3V(ikBL2UM^cj|fG^hK2YU}zR#Q0fH{>Sos
zb|?F%S-f#l@gLrbh3EL)ymGQ?$L+v5!qe-5!V1*BZJMN1>BzlD;dgAo-*rz|{5JfJ
zkUqXEb0)LZx$2oanH@gtjX9EiGHOa|+qv%RpG)Wd{GK9`n!9f9=^6dOTbo>d)f|_N
z{yR5n*0GnTqSCdw=g;+z|JbtA>s#cL*>}X>8!tK8TfWsP>~eACnMm7hmj%w7n_WMp
zGU4v!iSu{*r9GSH7UZ<6;(7mN)|-5DekVVy@v52Hvs8Ao%z@p3s*iSjbKjYh@Q&GL
z`QqO!xf2(!5BRrw!@6aMZ%oh5?Yn<T;mPYY?e9YV%u=0w^jg~MgyqZ6s%mw1C$Ii=
z+d0CQancoA+xUr=fxbCw%1gISRXmroM{e0dCZ^z;jj#4BEiGOjU3v6GdFBVMqe~+9
zL<m<MZPUKDvf+}2D636I)wPn-;g3?!71l*P)}5dE?6t|~1Lwl?p0D&2EQ#AP>D8}@
zMVblwwf`=<{qD7x`MfC$4ye}HboDQ*Y}L=$-x~F9(jSHC^$YDcSKeFt;><=)HlHW!
z_nlyfnR%i7;(7bviGS-Db%l@57QVgU<jFc!nW_adV}<7YdUj3c{U-C{c6W};-VET#
z;b17b<YlYymhqN?@iecn#>kg7>qEG-Ut07RZ|}^iIdxFwbS+<b!XY;G^uxyaI?Jt3
zC^YXgKH(Xqm9nsW*1;3<1u8wO8PvF+_sz38AAdyJFYDlPlQSn*oLLa=u=U;w@#2g;
z?ufdBJM4X*Cp?$PYPz%Z@>caLMYl@M+CALd+IGq);n|M)?;|!Dq-bsR<(>7}Kz-FT
zslR^&SLQM?YnC2LVoHj#n)g~I&4gFf*Xn204fQP(^1l7l3Gwwh_IGj5rd@G)DRt2Y
zm5(o2b!JBTny1sU_&;x`y1z%uKK)#s@$#KBPjRr8E|_{Hy!GYAL<2u@=Ot~YR&IDA
zuJiU>?Nrw?pGlmj)*QJXEPZpq`g7G`LK+sU7lpG$hp{D}ORsy99#l8w%q4BlC*Mwm
zoBY?QoS?lX-fzuHr@-jV&kisKhwT&8j-NcWbAtB9<5M#yXkS(no_E4?+p`s~cV3ii
zoLKehNRYUweDUhbPr}xPMoejsiOeldpV-X%@n2hcz4Ph)Go{X1*ll(Q6FZZ+$Ef$Z
z%++SaJsHQW-+fHiT6bT6!G<@}mz++Td~59zA?=k%ycQe|TKINN%&y?IYqMEuJ$`v<
zT{+J+d-nXu8IO$CU35@dnQOH{aktfkxqo?s<gR|V(ok!gz$Vc3?nb8nz9}~Ym`fK6
znR9)rFsZz-ch0VrL54SHsR+(_So$x2%5(4jr6JQ_nrO{i=4AUKTlv#{#YW9zzfUi^
z_p;kac;bBbqG_McEfX*IyR}1CG${V~V;!+gb)Qd0^cdXcyI(l*-HjQ}C)FBX3-t>#
zEZx1Sw!@xhN1EE{XfxN)pKQh_FY*W7_*i*-SKGaP|AStyy|dYPNsW`2@$38N<|L#m
z9Wj3W`rMXFb+vte-gI*ZeeavHG3eNd;Pkq8d(OLM%&Qcdy2WVooew@=4KClG6Uk7K
zCV6irYkmK<xi4q==p1+2eBreA*TN<5dcSx??NUsCde3X!ch-*|oA@go+xH|m#(7H2
z_u7%IaV|S_TS%AW>uAZRagwRgmhNj+)>)k@FS7Vr=+A2V`O3kr9e3{En1AZ$RZG+L
z?<#{%pPwhS=ecKNhe++D!{JS-+L9Z?j&X!vO7eWUDWWg&>d!4o%8t)wa@8)?lK=UA
z&ip0K_Cc+G%NUKT)0f?=?0r%Y-|w81`l~KUCsMu2;N{md_4*RaIw$CBG3#^v*W=>S
zf5rQJ<(YnF|Fyo)-PfOpEPk%CW3qbPbDk6X7u%UF%_s|-cJgzSlX>%bmSVR@=FO3l
z53}3c5Z#e*xc1+|E{#l)<rW#M&2&Q3{c>mS{eNnv@`p+DOJ<sLNlsb*z(M}|2NVB!
z^Ex-~^t^m(cITpIyRS9r<!xQ%&t|^y5lGB?C2`|#;=V^B=XqjZoW1tw(5owE89Vkp
z**tH)#&(fYx(4qiTVzi*-pUwdrt#2^=h>M;+2#AJKW{Bcw^RBheYx(@8~<m!zScNQ
z`QurX{etCzqR+F#Cs)pH?w)J;*YN8jeIK>62YGgU<n>D5bf)vM;?kb5{Yp0Xe!B|V
z96Y&g%eR?o@&y-j=2tUspKY9P#jd%tVxf8%>-66&;Tt{|Xme$}6S$Ee_W6iYA;Yaa
z@60oM=N#bNHt(L*_PXs46o2kfb?$!ooQ03?j8b~vp^H0@8s}S<gjGzD;SV{xP}j=H
z+kIJZTb85FB=gmKtZ#n1ohA7t@Aei(HT}<(!k-nTHvgEvTC9G5gUO60p@lcwrat5@
zb&a1Q`qp5bPga-SwomLCSF+B&=2n&SoAPOkXUL?;Dg37LpTx5So}{hT(^<*IVSTB{
zMm%cX_9c!liWXeYdH(p@2Gz4^RjX#Ht~`5Ae(tHIy;~3VN-hd?Tl7(E^1D4QRR^#8
z#rxb_owK1ReIeubqs_Lq6XgmouRWT&ZpX3nk7UY@Ez2}%<@SE+w2s5(wY%Ypo&Po*
z-MEbZp3U+s&UIUMg)O_j#A=DA(`luKuLmaxGy0s>yfOV-jK9pz<?(mLw#l+zIw@EA
zLxI2hmyq_^T~pgGas8LQllE|#jzah4kOv2}_}G`L#Z}fyx5!?XZr}ZQa=28^CgXLR
z53=gtXl0dNmvCpUUEBdb?!4nRSG9NSOMEBl<Iu4Dh-}oqph`>r%Nq8VFF(<CX^g9U
zy!a2hZ`GITO@W!0)}3rW@nlhQ%b6{~X}8pk&q|+}*}8Jm#&|L9e_lrmB7gjx>~oeq
zBX4r}{~HcrUk@n9JFp1dx<B*K?6c1lzTc?q=Qa9}z?k{DL}wx2p}X3=4;<NTUMz7-
zV@X&hSmF`!-M)|K|8LQI@5K!+Gw2-A_g`bUeS_%BwTpu<rOa_z_~EXDw{`=!Yqy5e
z%(A3|Q8tT{4ja$%UC6jObGGiP3ES_S>}s0ayu~;$!#FeV#4fYox5sSl!xleFyL#?T
zqi1L8&CQwu^HNJcCtp{6e(GlM`R9dP=iR2A>9G-bId$8e%NC{gxIOysNvht<{PX@u
zn)$hZodtHSZF8kPmNqP#qxsZ?`<CgNbIl9#Ldx@QR%~xy#mRr{{ef4pF^m?)ZP(Qf
zZ0dYhsCevk(402Y^Bfm~9UE4)eY{pNFJxv_+x-bQu9<Axx}kda(es?SsZ+GFO;fes
zbG2{iw(Y-jlc#L%;h0u4tsA?RX1q?adfxMY!lNUJ{;y}9=e%a4QJ`0_JKLn3HSQqm
zXY1T;A_DUw+RL-%Rj}^3`>rz2{=K?^XoSfBmD>~-wQh}@k*$5KVry;VQe8#kx<z-c
zy;ygBR{1);rUcG4%X#-sQ_srr?|B_{DC}`fW=r?%sik)tYTv(+<uwyqxMkJWxNo8*
z`<s|v{mtqR`}0M+_4jJl_ox4QZL41w_UA9tCszHaMXgH7H+SvR+`BudFioiA)C7IG
z^QT|vz0W<pQqI5Q@5}==w~TUk|1Mq9x9a>vlY%MU<<`w7H!#dAlaRG)){Z{r8KqGX
zEXwa0V8{3{=k7YUZY$-o2Qjh^ueUI>JbV!l)$D%UaqZW?<yrGz1<kr;^l1KJoy5@Q
z@UI_g8%{dSnsQ$;+IExW*CTBOZ@0+a<}Q8tEO+;cw_;0Kx7un)Ub3E9r5(F{)BV5O
zUdwH_RW@L}`;t|7kNJF)?YYw(-tS;yt@*yV;V5JAtRT;MM`8|~Ww)_V&VTvdBYsML
zq(kK7KHs*DQro@V({`;*E?)3t@`B$NmYq6#<-nI}<J0ebo;OF<a+o}p;pu<)^hx;p
z{#Waz&+A|J(s&|2r*F1~+UKQy{Q7ezomAWK<)WtWVPokGT^^1+eINZNp`AQoGShwP
zZ>s1f?Dx~Twnre}P2!KwqRI=e3%kNNxYOSAO5K^|xQa*c+{C&QKK!Yl%wHP4T+8A9
z-gnuV$IIsNTy(LS-r3A#apkskyn@f%ge&PYCRE&Ke)C7bAf|#p;bQ8mnK^Sh&pPJ(
zW$K$-o?pM|)t|Ra=c*^huBtZscu>vO<|DWG*#-KYY;&f6T~>Sg&bEN?ZqdMQ)qtA;
z)*<(KZ|oE3sC}5THfc(W%JHd;GauUuHm+51dvez7rNy<dGa;`h2m48wJ!YL<GTHaq
zsrBD~&6%KZ{m}LMBB$?NId<=fb!zi_Qw!DD)}473>i56S&Hq^_<#EdE@b)In^pw<K
z=9F}q<i}U%r9H6YIlOD<)O!qKb<<{BsMYaEeKIXj;_`E0O!HeX+otoZR!{Q9<xAyX
zGP>OQ(a@<oxAo+_ruhY1G>-l8`*dXPo~~O#(o6?)*nH;|PLs;vS&(z7`fTaNz59OV
z`E6_TUb<b9L;bqWchfcB)vV9`+gZ{c;Xiw(=w@4e3wh=h6%8TD{9?Sv7tKAvwIsRL
z{QBm^b?nn>9w#KMn0?@yh1iVmo?j0rnd!V+)!MuF#h;>E>;`i(47-^O<}4SV#UC8I
zBWV3ueXh9m`U-t<+)VK|jxEfTW_mnvmE=Jy?qJ8Ka`NE^YOg#$y=*?yt>6bzuf#8X
zl#9;!JN-bpluoAi(#Mm|T~vFSKKbD0_gZTXoeM8ZQ#-tQ&D>}Gp_QN|EzNAPRxh@F
zIa;Q7=-j`oXRf;r^?F}2J>$xBbHmk_55pauE+3u4bl5w)I;+)x?T5?<iAlBU*ZHk;
z*mI44ef*byEA`?BKAz7$D(7WGx$k-u|MGjjHY)gV^-lT2(%x}pZ)_ZXEc$A$6Fuqv
z>KS1%lfG!@eu^}?)b94{!-1Dq4)r*%6JIFcbD~u{mPcjkQ7!2;)y^ijg^l=%zeq?X
zxCPBh+;!)-mfYJaCfV~18QhiG?)9pohYxSwF!fm}>pgom*$;1y$j)#}4cjhT#rH&J
zZ_$xwe1|P(78Y*S4vD+j_WSJL6Z_+AXSZ`|+Iw~;cS-%c>ATM2@ZM7?CI2$4gqPL5
zaGD^0Xk$soeZHfI4xPDtWN#32*ujQXr(=D0tv=nlGUck$0=@DFr<U9gjr=!zipY^b
zCDS=3E7PVwIQHDLLiFt6cd9EV)v7I6B6Y?)zvh_tyTr@ka;_oUdn;tQ=bYyK+!15H
zr}4Qqi-J|Is-9tQ&ZT{yrcars8Qgi9;XonZo#%Nv#R3Ya9A^qH`26SjMRCPNGC405
z8&{-1D^pH2xn0#6f4i6Aq2=G5QD5#W9Qb<feHDA@-#bCCZrW8$-m%d4kLF$PM;Vh_
z`!>JK+2ko<;CrXX^rgMupOcpNe=^S95Oir)$N#;OuV+;xu<;mqD@XdD>#o1`d0FY7
zi(l^k-Osc3;bgVt$1i%O&t3e~`Vv>aielFp(UmG5TWe-(PgjZjw(+C!<ZRWW2Y#AW
zMm{uJlu>yua^i>8nf#wFP3A4)+%c;<nCo0c^e3f_d`iw8oBDmGdD_^Q$Y;+yWl(=L
zRIYhTwNla4C)ao$T<F*S>=ND<*w?-5(!{$jmp__6>0Vq!`;?PE7i>^_JlQ%vpXv2O
zpU9m7=Hds0_9>lQC3Hq=%|Z2jVe`vVAGvL_DSpuzvZr&d)H<JOK2>>6*8*==cJFw(
zC0+HUXS#2HR{G@BMPje@i}V+6`h0nFURTC-FaA!3h^RY1ndh7@wNswEY}W77|0XV}
zcxrU~v9Ig8C%Khj1z~b0f-g5JNN-zqbXKC?r*~Yju@QSpbNcTnv80`{UL9vtFj@1Z
zY4O9)vO84f_!Q4>wVd{D*Uyic7hX@>_^bPA?Az`iE0>0+Rb9|&sZg7(e0O`>`Kup}
z&srDmmVRxYh)n&eN59@OeLEI@Cg8GHe_?4^Wy{)0>QgQVzm<8TvQJ>eshV|-v#me0
zT<o#7{C9a{&YIW*|7-L!yl(%xp6<>5=}x`;l)U#oDJJ2@mmicrD}NWFWUc#ouHQMo
zuSI$pF%u@Gotkl^f7wm%yJh#U9eLR3Z`pa=U*>G$45{-aMW;SQ?Du<dh;#3{g>p~M
zTkn*Z@F&y8OZo7s)Yhdo7MmsKB-iYnc+daLQpI&2Sqdzls+wG_xajoG;ZOnh%iiey
zY@fQ4Y`1b>)a#Xc?ArY!P$JcEMv~vOEg8!yu9hzod+&E~_mO~G<!hy4+r_)?dtT{D
z_`A8cP%@!v+xbntmUUNxrMX`*ia)aYe)`g}qHVdSZnCX-=;C+a?#j-<)adEAb#e}0
zUQzMo`1@BZdM6HK*;nxU#xMRW;GgkJMpsby(qien;L=8(^Z(S^SIexM7k2ia+1_O7
zyElG#&K5P|j$gf_^Ruzz@ymCkwncA$9k;<|L&^Sa?{4hcDEw!ci+0z;ElC>FWo0rd
zf)ax_XIcd~Ca4Bqm%leFKKGZ#dH+)G<65!v>*o}FW7jxwUr*sfQ}p)lFIHT>Y#RKG
zyUgPm`@@Jl8NcHAQhVc-lS41>EZ?+LdzRg-zVO!Ej<*-mdHs3Ho+x<cpD>8nFinfy
z`)tAQol}kqAK9$EMc*^L!#_HsAeH&MSwXV4bv$eRwxX=k?81eY3Ocmo3e;aeOpSBn
zd-tm2gjjKa{MTQ5Ok&$b_ut^S!{Pr`<xq~^GzIo0yQVeu+%E1_fBD1AMdw@D-FMHQ
zZ(fu1vfy89jkM;v)e5_=eF&5G%QC)ljpyjvpO#)5+C=`GnYs97{J)%&TUOo44$iNN
z%`5a{uX?lV-kg~S178?zdU(aqzP#iUN0Ql1vjg4+^A-3GGWttYUHv&BDe|DuzNGvI
zaq3bAHJtCdPn<f$aL=R5Q0mS*8&%<+n^$<#>sB1w!uTrqz3N--gVDFb@2SRqyItkf
zCd2(BMtPFM>u<~g&K_U?SY*~c%6zuu0^jGPywbV9+Kl^!XMLy^YOn0vIsc@Pa?bSJ
zjjmsmHvgMi?(4VUOKf+pZeICz`R&X1<tk37x_pOoX5O4z_Tkb0>KJEUoBq1o(~X@c
zqU84Jl1(zRmp?w&f0?!T{CO*n{paQ{J$&)%+5N32`&4qL&3d#|{7%lF6*HSFm#ntC
zoMc}zCEfXR|G_?Y`_ILtWxi5+r}msVE*f&wc*nP!-^=z0asOkG5Wn>QoN?u6@!|uA
z`cAcZd_3^|*z<E~OPKA9Vr&Yol<()had}T#QuT*jm*2I0)UMq(^{G?%rqs1Z(!wr%
zS;LbuPdh^D*nYE#Q<bXa@*nS7JhOlQhq|y&GuExK-F@k)-F(Y<G1(8C6$L`yK9`@1
zpZzIOZSA$Azn1pQyH?C{MvXayo9h#+WcIQ1VQqqSe+!iFew2*Zy5oi9{?j6Da}&<`
z1!UU)n_Xj;bFi!5&nrt$SU=^DsL|gD(+vj$&&A(lbnz&T^tZb8&--}It|g{MLXw@T
zD(5O@HEdTZ_dIkc=$nso{~jGP@dKxVisgS!vn@6_x$#9y_`=&BPZl_~&7R$oQ0^9e
za>bhuiD$lFI^CClYvH0B2Xb!;G`*d3Ia0vb!i?kKw}Y)Ip=myq0yAXi7@sIsNq(C1
zQ0n=izm2wkTUGv4dcFG2eWE;n$)nTDCam|0*ZZ|8CE-_T$dQ}g3cq{S+OXG!9D2kW
z({wdBW$Qi<uCw#gnZG@#xVtiIGW&*;%m+5!dHI9&d4Yh7VrQ$_&!ZMIrEja9e-n4&
z-bRav_fFSrj=$cSRJ4XMN@PQakMzAvp?qP954+48=kKj&Q4|w6^Lmq=pjy0w@b2A@
zOutY3qqW~ViB)J4Q{IOC9hCyTVy$&ul5-2DF3UL3H~m(-=!Iq3wqO6&Z;tHU_gt&x
z*XMI_Yn`})MG`d@Hrj|B<k}R_``Y$dnQf**)q?ml$sF!G_THWM@L!rVSB95}=gCO3
z?X^aQLf6Vnn;uX0xN96a{nyHUCyds{D60i+bm8f_A$0iuj+>HP>ta0`Pvq?im-N=(
z=Ck@Zd)0?a^WXMw$`?9hb~0f$>+Z9P2D4_KfB5g4Y?U3;HtQA-BZJ>JKHg4g^|g4S
zsNMcn`jna9-^3l?yLYeK*ulOmvwn4u-R+V)Yj;=7-|}S9Z=b(sm(4Oe^=X?IQ~G7q
zoPGA}27&@JA1ScStzW{`l-#myx$NX)wVqbTinI=?t+_NieN!Z7!}V{Lf7-P!)HgS{
ztYDL!f990RoY@J#IV?FPMU}s<OFUEH8fL7~_AZt!elOpN6AMZbEg9b`FAHIqZIwBP
zSs?R4;?wgUa;-jk)7xG(C^eVq1-dAI@c(_F?qvGwe`Y1GFYHdx`jBvy!Lvc_d3dq6
zUdc9#rM!*3f1G_wZ%yG?5PjL`=~}jx8EnsQ&OY|nvNL=?V^+z8zJp<puRLsRQtI9C
z-=NgtM04nz%Lo3L2;?^f%(ZWO$Zz{I(^nyH5$^*j#s!K`ot@+!m9=vRM<`!iR(WKu
zMuz+Jdn%9FCQR@bvy|Prm`UVGkH8P#n_lN8Why@87jflKoIZIrgX;0pwzI6#f->9A
z2_<T)d_1+~*|dEwm()9Lavpu3`A^hO=ONG3+YS;2EUm#I7F*}$JZl#=zFzsG_4xDu
zI!a<^Wn-T#67?6lmC^L3^R7?9ys0-{FzPZTIC;H0cJG<rE>9zY!s#*BJGD7N6y@p+
zdn*-rHVZtn&F`Asev?B|LZ9LE*_dgQr%ssB{_tgruUZUWf+E92hg*tV&l5QO8=^!$
z@|Uswa1S~BKJFrmNyak6s)dv6!+ZagF@BTI<jX&B;>(@9DCgsLC!%zcY_C366cfpD
zHoCuS{gZQv!77h;Y2WC+V0=4GL*(4)B(pa+=Bxf%es|7{X>QvTb)1ZND?WEF{2R0F
z#h1p-4^CGWzvKG3{ai7xYdLSg`K`hSRD9<MD=fJ@p?yNZ8i%u!%{_URq(5s)v_836
z;iQ)90_I<9+^>0j;+wRgYfkUf{^~1?ENf<p@UN?pTEI2^#o;v<Jv|N>mwr8ZNY%Di
z!sGFo7jZjo9{IrjTth)&=F<zp0g9>m-z%Lj^qe{QKBbjePq*&%oz64oikXXLE-0<8
zUi+jvM)FDY$+P={<kzay$FJPTcSf)>=&1)Q-|eDDf*YCNX{f1fVZ3xv_@767-*%mu
zcNetie#o^I>CtRm%X;r%*Tpjm8A~eE&ukFtdBGX#^PqE{fy=sGXDeQw`|;x3j*K=B
zk2lv&-#j1~ZISfc(XqMxNRy-SP0Iy*iV^cxe`Mdo_43lD4Vxd`R5J`IIaO~{ep0>l
z-6h?3KhE?n_~pECf^>Acvh0i6e||Y;lTS>J&VQ~l;h*6>wZadpxOZ#+ynEzCaihkw
zziEMA{#k$guAjT`yO!iz{=XrOtXeaidw-srd3?gzoXY--4GuasuYK0uvGAPTdfflw
z@wCi!_NSYxmhF}MSJJK>*8Mt$vpIZj)`G~i!cD>oVJAyBes`JsV_oolOGBUN8z!>;
zGezf^XIhl=M9--W*pXGmB$xU7=adlbxO->JHBOnzF6`obz-?Zsu~FyTeC^rOUvr);
z%(vQd_MC<N?3V(ju`PaL&Q^b~wYMJlw(GBVzI?xv*36Ps*ZWyaUP{>pZP~!FGT_M~
zf!4h*f2nS~VG~h*@2ky;EvJ+UbMoqT_3XEwX&m^#Q{(*a`qyzg?;JIi-6`xix4E~P
zuWT9nUzNa+yk7}N(^qoLi`ivad1tr6%cD)cTW;-DYY_U-r7>-4^hDoZ8Hs7fET=D?
zTvL24@lnkl$)?|Hy)K<qJ@3MFXh-|2`Qlcb4(xoIN3*^3W}a`GyZ+qD>#a*UkJ`-V
zR=4)eIGNJHzIf5B{eALQOYhE9Yw!_vUM49StfQE%#{PIMS6Zpftu3$ovssqSI%7AJ
z?d1O(1^;%+&8+$UT2t_2;+bhVRdqsBpSZ58FLtkae?a)UjBj_*?MX8Xww^3`p#JaA
zCPwxNYN<`kJ~s80sbLpCzpwbU=0R?G>x_q%6WaSKuYO$4`rBx=<BsfZew!zDIiIEO
zy$QN2P(CZWbNUs|n0d$aB{{oUnpo^jjaS>~%x-T#`E17r;qZNrzxW3Uu<iSQWN-YB
ze-j?RyRu^@zyEBu>J-%}!K$lu@8+7V|GxL@rIY-Mv#lnxf4Js(R=;KE!&c9yA2dSl
z?<*=5{kb+xGvs3Hk*RrJ&VK_Bo6LWrX0ai1kIRgN$6LcM|0r(Q|9XbGe}4H-N9O&$
z<@c}c`Cs$FG01b9@Ampv-{Ma!T3l^C@3)#`;==>p8Ri~75^v5teXPi>yU(lPCd-u9
z6SwAXet4$!!V`I8Lze6NF0Xt2iknsEwrrmNWtk-pk{J&<pNrr2Zu*<`Yi94<*kY`g
z7I3otz-yf}DT!i_Hwimy&u7QaaAMQ9J-8`T#YV=|-O1Iekyl_{s>KJBH0LKQcN@M)
zPd@XpXaAr3ljqKjmMdoc`Fd95lkAd)wvWnjGZc>fP+azTncHc#$y4|mjjeM(E&P7o
zI{5LP-@VaO*Uz%6U%>VEFH_N?oZEfOQCf#$EX$V2o%R!9O|?EEeMI<^>l-OU*;=M|
z31w|RwU7EdNh<SZJHx1wvp3n4r?uMeU7NOv(0-d=n(uQhl%o^4T_P;)g6*U<<u9ko
zuq{2g)TZ^oxrP@nB4@IRInIvhc{qzl$mAQ#MuqFjvQ=%bk~H+)>gK)T;-0{5(6z)f
z=7DZx{<I^!X_J%Lb<@sFJ)QSHtMgq}Ctt9b!zHG8tKQBIl|YsIViLh|5xQk(?;V;J
ztWqhna})EN^u*iEksE#~I{($3Zk=-Fa($=I8<QpBa%VN(o;r1>?WLHi__RA0wKsNV
zzH-};m$doEpTw<4m@ORZToWr@AKI~AyYIpxXInj|@5~wBo<t`#)t^_A5AP~Gv1Hl|
z#+@9(k5)LAUHxgpUhsHj{``UiDcwH5&(xj_{&}XGz5aRU@o)Dd*M0Qx>R6Dnx6@|x
z_w}1<*Im2!aw^}P%RCPz7W{b7IVZVw?KkIHlk?u3+E(ORmF2=RkNw=6_MERVKHoE*
z3oX62bFO*R+V!uj3XG2UGS$eO^uHRhgY84>iup&lizglQ-gEOo{f0-s-!j#N7yhaa
zEATPcyt6v3+S<UPy6Vf$w_lRBP2X4Pb}MgA>&Jp%nUvGsQpq(RzC2KGDA*>GaPZ%e
z8`i(SCPaRDV|Dh6-*onEdHeYcOdR@GzR&t|neT1>-gJvqMjGob+jj5UBp$asu3ug)
zJbc2khL@bb?o?D(UA>?cf8~(j#B&8Nl)Yure6_<~wRiN`ZRQT@Qguvtq&w}xnd*YD
z;|t?{T$erX_-#py`(4gu%bquC@BP>z-uOAUS=;u8)Q{`aj4N~XMUu3vF1}o!*7@ks
ztp3g4pF3|c@m1GYFz4u^#WzniMe`~?j?jPesp(_%ncX`LLLVJJ(&YOx@o%dmSLXgt
z1$)EqKaSh>WnZu2bE%i?s#8Al?9C5X@#xX7m{j?)_gYWQjnbKknXk{MM}9Z$$extA
z{Mn{&E=M=6KeFiT;_tr>bk?>$vhvGmvt2#o?#m^;hj;mI*Wa-*;83gN@#lrh&%N4}
zm?i7~`PEOY9bGw(H$3bAR~EDUaLuKwFB@kao%|+baz)a%H}#&yTc_!*dg1?K)3n#w
zVfnqzUVr}SeOwvs|Kr&2|82|uGgVyqR6LXUg>uxD@7l^2b~WprwtRcQH2?PA&$cVY
zRyQntZ+kemZp+zF=E%9tvy0DmB(qr*e?HuI^5kq2UFLVUKUD56tZ<6{In%CMNT$Uv
z>e}b4spt1jobSxsxLRytNzJc?{Pthows$aCw&#f7?pq%>zxtr)0oFBc0=#=(KVa&O
zj@kV56w{res)WCLW`63qCq7+0=dYA(UB|^qQTI$a^`a)*?Awvlct7D|_3SB=8XWg-
z&wqdVUf#~A*CCd5>wfRMdu?xdtJ<CWJDm?CyuQz^vEhsZ&(g?(rJS#}9E$S0^X)kE
zudkuy|0bnHyH7IZtvI1Sr_f+QjNtm`Hj%Fl3h$`P{XX<~`srV)rgsi-M9kTBV^)ig
z+5-o#=5-5qW^58v%~(_ElV-W6UircKO|siEPV>xE4K~|x;Lz*$3XIHW9%LN69I}A9
zY@b^A0Y*DL*XZX<IF}d*niq6hN3TEV$h^$b(X_^|=KgD)*N5!xhN)Z1@BFQ`q~ALB
z%)@`x3%A^Nuk7A4T_<e*g|d@ZdcUM+)-a`?d3vPt_3hPmMVH<GC0~89-`Qc$@0Nde
z9x+e8wx(64V(-kU6K}m>R5w^ynl3zpGgOY<;qaHG@=jBBihHq#HHUC0^WEFJp*pqn
zgzmJ~kWK1OVlp1wFE5O2V&gqAqp0oqZkgqCwe*eco2?!-2G^S%`*}lQ+JtkFGHhWj
zI!s~()=xZ}7dP+}DLaV9scmE3S2uyrcbikvq7|oEt@i2gPt@exDY>&qY?Wn%V9`cS
zxzk36eor{Ji*diXi1AH_%?+lpud5C7!dR1K-B@${EI(Y!5qap=n78A+Sya*DSmT9?
zw#~dBnYJ=<vj&zO;yltF%fY<S)LYHSaM8zt<3~?fZD;%FARVSRX$I>A1;s{BjYTS6
z8-!hM{F%#M_DpPzYn!8I#lpzN+&l@4j5>{tiP|a&iPORbc_Un999AsYwR7*QDsQ$U
z!lfSfRGjX8t##;5m3Z~;5S#4sX@3u_b?Ghj?B>%wmu7zB&e>`2^@`tb;wpdnN~pVP
zPr}WrgFM^6PH6XkuxLum%?cKuJ-0lgDi6tqJk$CRHh<ac6M1G|vpIXjWOh2`NKSuL
z(|K;EOjf>S{hDXL|1!1kABr&F>n>it&FZiGH`Tj;vldiIo-|GWoO9{w)g6276Lw!L
ztKXD$$-&Tv&-7z2>#^xS)j2MFc+TEu6S?^PBEz@tN>7eR?n-Vssy?&D@XMKx7viUj
zFv{;fajH!Gi}c20!R7M2tHtl7re8arzk<pA>soEQ&`i0xSB^iG{(A9s%@;m-H%HO4
z=QnuW5c`vC^YFTErFm&tyWpJ}+pbHRz2E+l=lHePr8h0-)hrNSwIhM|y|l{)t*)ut
zv<}GF+I&qrc%jYCVy;2EB1d$$S5e%$sOSAz3ES^%_+WC#T6BwufsKFt@7BflYF$Kq
z)-{J;_}2Hit*euFvb*J9-q$7zFP$-4@M_g9GuEJa+m&)>Jh{kLf0+3i%aTp^r^y{C
z)lo0`!@OKdZ~41rE*F=0gzt;qwlDC~agn<oO7fxZDj|>6D}GlT60$G!TGKZDv~+l&
z!`07gsx~*zzw!R=M^od#S#KsQ{8lMqJ<@MEZRd2g=Gn4SURkv5Xzq>B*LySJtLlc5
zO}u|S?;fB1()QT&{B3`3&D^@mDdSi9;cYMT`Tp#-k2?J?jqzwh#m>Obb3QrCtCZX=
zicYV+F_S+ozpSkK-v9WYL8gHZE{pM}1?>2+?U^gv`rUB`b+1=HSReQH*$k;)yDZjl
z>wghDV4Sx%W4mMK`Gc45E<5+QTCL%Fz0ae%y<2}DRp|~)3;!j~eA8f2zdX~|>-9dL
zmQ^>g&r;0U8yUvA<HAx+lkb=Lzx;YRA+e`=#nkp=;Z}#L%ws+7s=ljQwye2(cf827
z8`^>Q7R7V1yB-VwsB3I8;h!*{=I5N}c`wBC_s(rv&KLUX<7cg<&a6xqB6Ex5HhqbV
z)ta{U-Swm^`>gt|AK~>bvf?^i9<xGCTCOg1-E@C*t;>ZAwrpM%a$u!j>IJ8n7m`{p
zd`Ujhb+Twd<bs_`y{}wufAuo$;d=f*E9dEj9B$?6E?MiMDHe9<)RCQ2e-=IOYFn-v
zb~|-ua`vv)$*vlvmPd4(Y7ehDseJajzrTig)c!d|Pt#utKYSSwx-%$r=dtArf_z`s
zC11#Y_IC27y~g2b%g=9Ccdhfdw(LsLFF)Ohdvgl9wN6I1MR8{zoX*>Qpj)--@$rO<
zAB)R8^f%1B`^|ox`>*%mE&Jmgp6&6S<K=MCCr$cCNAuZZmOo}4e{))Q*13(2mrL~0
z^P>1a?{r)DFKO%BirG5bMVI>i|HQUz`G;Ep4$D4u_1H=+Ig~%=zKB7o?eo2tMOSp+
zvdx<yV`P>bCwWoNyz1ixze%eEBbIW9huexOZFOAt^~c5!e*Y7V7q3^|`fqXU$JY_-
z3gzt0AAGGyz07>pcYVj}eN5Q~-KonZ7oGH2V=tdwt)6H<_5IcjUEYa*gpOLhKYB!3
z^BA}Dzq$LeR+vT0sx^hXs;i3URW$h(u%&iqd#vyIlqU0Lo=?tKR@H42Zf3S`y#Ct!
zYtGZP`%JF+Uw7oPT`ursO;v#8tX+cZekT8&(Y9X1M4xlT<<{sFhs}T1W#pwidVgW^
zBb{9ji|&j1^qjBBofc83Ss_|u|5nMO(>d{x|GNntR_aS_8qH#Ll}xv8Ix=O!`N9oF
zKd#M}4w-Xz+s#*UGyg~Bm0VPb{Pg+LfnKMogAZ857MxmDU9no*kLOLq-=Ns9^Is)h
zuZrCFq-53fciw&*w6=?fT$$<fgpu=T1mCq(`PUVXZpN)I-M2UF$A;zaF3PWt|If?B
z6mH0Ktt@$6Z2W~KiFHrUW?Axu%ssT_3L9JW(F<oZssog?noL(auDiJN?isZk(i|S<
zmoD+_U&nLqq~-I9a;XiDH~K%`uGcgFv(NG1s>!c2`TJx8okAQBbf%`3zU7z5F1T_@
zNcqL|vRWUeslrAZe-?*_beLV2Ub$P)?sHAN`BXvwFtJ}MQlAgGho295kaSJobgI|~
zTlH7xbnZ$AcdWKa6JGz!?bDj+XQJk+-K?DKYrB2=?Ze;Ne%;wtca^)yV0!Mp<VL4$
zzO6O48q23lJa9_t{^JRgQw7_8RGIZgD(38Xq$d}DKYWwY{|gHrhMLYgaP?G3;I&yM
z*PhGUzgf4mj_1`~o`uVA=6%0x7UO#TxSDh7YkTup^IzxFO>^8m_iJ9-C3q>3t8^3N
zLywEGhf}W~{wMb=hQD~@SB+VU&qWl<c~agVHhAG+I)C}BC#N6DKNR(CDzToO`(mBt
z=3`~j{-=1ZE>qsRPWh|osl%0L4MYN0q#1C@?})l1UH8c#bYHZ@(=CtYhGu@9mbTd?
z<f^uM_O^@9=Wbcb_usMH?@jr&^ArAm<oxv6KWyp#*anUCZ}*rw-fY_ZOz7q8pR<&&
zCd5@Osahb}zHUaPY5((6$(z3C|J*QZTFJVKyqV^EI3C%4p46jvnRDZ6(<reAUp6I8
zyyNz4ovzOAJ2UNG923?xxxGPFa^K9TJtxG~j32SQy!3j@tvuB&*W?yubZ(ju7ws{{
zm#LsCtC=--h3C0zt11*(_7w4cP$@2#6Wuud&4vD)i)`1=Z)@$=t^K(}<>H^N<9oV~
zZ|OSzq9#xLym}a0ua{iKJpubYS6p_!idEh<=h-K=tI~Ih4EV3zbM`uv$*}iYsq`1l
z8@F#(@tSSttyaHxBt+)j@iO&)ZhouZtL)V&`e8A5Zlg8tJf6AD*~j`{yUeiiRZE??
zY)3@GvJ-wWGroP=llWW1VVl#B{$@L^td)y|PUYTCQb=Ru+_S(i%zN&(q9j3Mg`dVV
z?7L3<|C#7BNk!OFSow=)%z=pecC2n0DamtZo~!=%s)9ZAdQ;Ps)9asin4Oyc*1<4r
zWrOa;M>cV~D<?6EyFFDYP-<S1I5l(2)2RR5Z>yTsQ}}|uUFwoFUGpg^$ccBd(5kN<
zt75D5&RjXr;~>BP6@S>={jm;=8jAUfr`<xYiHFwwvr;&E?y8%p_m)rI%Y-NIPhNJ_
z&cd*l*>dSTC406?qd#g|v)>wJIDdGy^-@=ynM=7^*{?%(=Wos5_jRFZ>6!11zIKnF
zZ(2E>+neR&DLrAI%O^XyC-(8#vzv*k=NNsFy0F~)DbwL-yEz<ZKdd}yn-}ubP$MaO
zU94%xrzJ0)-`!V|ZN7ZXdfKEiljR>X_bffN&H0A<O6zv<yEglOrm}P!$v)nZsKWoJ
zcDnK2m_242a;z&Nj=wRz?W>nCeS7Bm&UNhfZ~A2BgdgH`b(`a2Gi&j>m#3e1?fBLB
zM%v}}{&!bAnbQhoz1P&|K2vu1d-a@lT8{$nFCF<(Jwq3V<PI&1#Y)doB^3X*tW4b#
z_~H5TO`o#<&50FJUMwK7ku}-Oc)qvR*~=M+%XDINZok#^|9a}2Jn!sn!p~kvY@6%g
zxcFrociXl19>2FFu8Y#zp0uTHo6hnL3r=71&isAvK!8|rrW=QpvTkn7hjVEs77M0(
z1XnrnTzGln+zZ?OV_|j;-sgAE$*kQwSFb~^dfJ{JsY_jNWlfBXlKuSlR_@o{_xuf)
zjwZzJI`H47@@4&kV}GZ!&MDk>f;Dc6cKn+2-hB6;PrCo+49m~2o;r&se7$OI$m3Mg
z{ov@#`2LxjAN+_p@@-yEpwAmkvDT+m8&5KA?$q89STZ}x!MgOxxzq#E(*FgeYUDUA
z*IlnKGvPTYwf$?;vZs-E`Fw9zuxu8drgtei?DIOU$=+-Mxy;%EA$N8pHeC{5BIn0f
zTHzGv@YzD+TP(Lp>#~gtH`hc>%R2sQb(+&guO%LbAN+clw6E}_!{n7V3oTzZW(o0E
z>RmIbXg+szs^U@3Z4tZ+U%U}>T)XJ(<lozGNA(+;Xh(l$iPCO*Vl+>U<KExh#)0$o
zE!*9m?g&#mmJ+wUh)c7<Hq*4r^t9H&O-pi1jLT>2Gd8(U%z0^B^qdb`t@}2W`Ux(p
ztvhi}arN>U(~=oEFI)9lp4`*?(cS9%Mz-14o|p^e)Ia%_T;0#Ve*N{XD?DpmRHXC+
zmz};KzkgQW%PTjVxF>0@p43&l&0uL;Z@9v;Vn?wx>nkj#KJ^#1o49$a$yTR3JGeBC
z^MAc{|A^Yxzw<;neOAuu_~<HnbnVW)A14a=-n}Y&RCvR!RoRIQf7V~zU+;5l<C)!U
z7w?Du|F`If^XE{VSuKkMb}O&r;i?UM`Hqn>bCajf>5T#(+RZjayKFN4VpMzLsK(P}
z$NYQ~K0bStGrKNh+nqJc+WRlboK4-6$(j6^@8GjYR|Ui1W;P3>y&X&RPpO%ta%vPh
zKDrt5FZW1$(N*`v*{;&dd7~FTQggZEJ4I4LUik3TqdZ@aUlJE?32!cKIFq%FFDE8A
zG0xTM_@`xkCX-)(a`xi%yvAj^@!Wf}AAjQ_9IGBJ-^1FbJw4HN>&mq@l?F*$GUuIJ
z_wn?*?2C3cHtnu3oc77>+Ag!gbxRE0uB%j@yT<W3PJV)!xBZ<bVz*-SYaMR1cNSa-
zTf87sD11X-Uwgqn=1I@8=W6P>nWt~Oy2U4tF>ZyQ&i3DF2TTkMm9w{ss`~_Av2aTF
z4Q#i0rlYxHrt74{p3hAFo4%ghdtQs5Gxd%Cxx&=7Rr_7<ty5godg&(H^;KeL8#bQ~
z)$Clsn&x5mq;sl1+k-1B1gf>10u>6JgWf1!Fgv07py^-ZI-b?bwjTUeW_M0a{^JaW
z!@eI?<GbF)oo$NCo%`1JyQ=g8iMlhKbw=}dPctj-c`p4<g!Nak=&SeQ8yHS?9hrDk
z^kB%6d(-r9+i&N(z2xo;wa8t|!orSoO-Vcbu&r~V8{g9pS}t3TGjJNkI2pTSJv?(v
zcDl*uO|D7hGyYBLk710`X2_h&wn%;b8Eb*uAP3V8;oFtBtU3`n`>+qoks#Lu1tp;u
zXEbvxT3=ohloT=Hczu1or23r3S-~?T*9)kfpIux1IXZ9&WAc@mUNgL<Cin9=Dp;x*
zM15QENW<bmvAOWkMU5#OX6uZl@0{`|RBujNJ=vgli|>M-s_>_#BHv!m)wq52H}h5B
z7T@WMEIZRyAA29lcu3Jzrsa{Z^wPWA%J2N($bJ3WKI_7fgym6n)eF+3cUe0NB=4}j
znRYgIA-h$on)QPm_Kj~tj=gl+(%#y(<XGa9Tn0On0~|LBm^GHp{CuPCk*wX~q9c+q
zlO>|>?cA*;@Zb2hT%PsToId@)?z49`ah`lyeMNKg)k^<5)44Y;GQ_VC{l4>2$mR@f
zVV1vJt*5g1eUjUml&-hKWI1O=*$h8L?!;d!ES6ff+J4$=yY78$Tj7kvwD-!!3Wd+(
zeXA;T4%zcgb|@7(;CkAU;rg-(zbm%IEOuOUS$?f^{qBY$Z_}7p9a{RCQ>RXB>FvCv
z`PrI%^9;K{gWSn>iLWZ`s^;)-m&kb2*!8ZWkJ<HZ;Qh#D>&*0J9k%9c)M{ka9N-L2
z_;9ODM>+lYt8Kitde(7!Ew!(%na=P#O|a6T`Ut1i2eYr2Kk!&t^-U4JCUjxqOaD7c
zE>GliwwDNP_+!4TZqW~Em%78h<*u+T7pO3M6O+TKWM?9JTGoDBY|Xu!=2v&*hOV7`
zd!k$6pB+B4S-coOKT2t2x?Zzi^a^X+YSy^Oka-2yZ9h~z*cPte?U=nOxa+RG%G%Gn
z?mT3Cn8%VEes$LEONtg&$97~@#6O8#yocM^T*UB~#m47aXS{#jU)=HZtJl%{bun9`
zuPj<v#=CX(%g(DT>|QBlm(ubRBM)!d^5Ec;r;i;TFeP3TW_tTXEOn>pqvt90^RHV-
zH+{@r*njz%O>#>5TQ_m}T%)&>V{bRzetkBu_xzU$77aiBgr#2oZcjSSv+Bwu(OEZs
zDF@d$y=7I=D1F`>b>o&Ze`3H+z8IZr25;la&(2VtysG=yvcvK_JZ?>1mFvD>%}cHr
zBfpKy{_9%(e1CoE{&)vNmYKG*Cg+BE#2yNn8L@o(v|nd#$lu!Ib;|t8+nWU)r)Ix>
ze)jOk;?LVoE;L?n-N0V){Ea=|KknP;kmoR8E#CN@Zk0~XvRkJfL>umZ6Fc`v5W|lT
zvI+@*8#o&qeWqSHcT|vN!G`@}->w_IFh0?eB@xwo!9V@W{;G_&PP>aQ@}6t$H%^QF
zH!XSQ`m!YJ*%SV!=`Ra!H%KyGV}7{m@dW#To3=YM_8dA8`mo;G{#~Y7^Hs-dcYHEl
zr+n==ZJhXM2LI(Zky~O`e45H8kgK(6&Xd^ksJzq{P3yPGrHeQ2IQH80(5F8Wo4#KC
z;HZ`FXQ0of=U`xatTweGw8Avc@b!*mt@9RX9=rK&{`W<@&WP#k<K8DTyMo_mW#T8r
zt5s5~H{VtY;(nf`T)cShnO^nV&$=hy?^kl2dur$0(xaABOPJ*&K7R-*t>Cfx5#iok
z>b0$Wh3B~)H8*z8FqyPt$>eo67H)m_NLtd;q-n=or)jz6t5zIxoccXyw&=GzW}9Rs
z?cYrKtuj-acW2GzHCnva9X+d>mYneSe!#NP>QB>Q521i=ZC<fwUw^rs-kmu6;Q1*H
zoEQ7VYvTS_Jbb)AKJ{N6<CC(8m$Sc^E#Fksog7|y<-x_QK7kTtO`ge<n_v0&wwC((
z?|!NAoZ+DL^`CM)jYoH!cwlsNll2>xm70~;Pri9|>)kc;jhj5uXW7MT1iMK7J}&R-
z`ON4;ALm8(j%Na@SNFV52`XLQsb?^w@Z+(*UmRg+drS{6Ydm_XLHzPIQ?(x%vsL&P
z{)~P<iF3uqy`>uK&aO!=Z1!JcX#RfnA}%I%)dw1U@gK7#qhDDXn+jPs%I}(K_hVg(
z#|7iF{?Fz|ySipn+&yRVd_m0BjrOaU=Pj~Z@J+Hpwf5AJnb#KQ{n`95&olh)EaBg6
zI?UE<1u9ymFWwWG7HQ+x<S7u#-Lt!V;p$%5*`I$c{i^G-#YiWBrQ?_NIh`D%%KgW7
zt)FhZxyE_+cddZQ+%<=Pty9qT%l<jHRp%X((6XEUi{>`C6bC*$zxLluu^he4nHnn}
z=l?pKcl>HW*G;$CT;Es)TY1EdIG1$s?PXlHw<V@#lY7acJ1hs%mW#$s(aL;v>_viG
z<}C^5$%iKt2>(3uz*E$E^}funcJs@;cG)|(t@v8Ja^L6UmisOIJdFG1Hur7+ZDV^q
zW|GophW`nvzqU6(?=7t7{t<oPKGQ$34;VuItUXVb-J1D&Qg`Sn{+De#HFWDHZFm;?
z>4y%>tj(4yf}1YbU-gomy`}j`^$snMa8I@wW;-O<=FI)gS2yYGXTPnQI#p3Ed&Fmk
z?4RYaz%KmQ)+gM{#E$*svVQz$L05{p-^_&P{i(9-CzW-bKPkU2YqBezcyGq+&bjiI
zlUvSs-p~#It;!r>cTPoo@}pvd$dGwi|7IBJU3`?eDgEG&`1h}5ciJr7tEqT$>)xqd
zzx=~aG<^O!HB-Fa`Tc$Cs>kAU-^+gs_`&6v{`C4YuD-OmsAY#{EY#TPHK8S7(#(C&
z(m&_N-TfT&CgzpVd97n7jd~A1npU6g=bqM`YIVI*HjQmppTfDeZyiaCCokZBdNOfp
zb5>r8u;<EpvHsMHr(OTK@$HL0Ki73vTV?cP_D#R19d3MaHR0=vN_)rSv9_}=dc>WZ
zEg0!xzcTn$1CRXN>mQ$Nop65g`YhI`>#iJM$+K1J$_*n{wTqXlw9`xDKg`={ezV|B
z<54N2+z!$C&nLx1Z%fqwzqaOQ`Jt=7E3YTSU1KZDy}+6IG-`RmQyXtHf9D@-Ca<)K
zFr3gmJL25WC$kpRgsOEp+HTHNTYvG%l&3RoXNOOkrTega!Q;=XbR8qIuTGwGxqi*w
z;3prZifniKknnBu$#q*!e@kPS;BBA(Ax`RY(pJ|`Dkr-Znig-K_#@eNpMuG!O+4|B
zZcBRE<mc^MYrX&LnP02AwO0G@d%wJ0{jcTcxpO5Y=kIJ`^g8r;)ARf4MjIxdsY%;Y
zv+3G{n(IfWOtR%otPOZx%Ep{%=I%PpDL(DzM7<3s(xM_?RHTN~&y6U3X8SQeW#!db
z>{;_fdS)H_XCK!8w~kRc^Kn|)jT3y?o|5$$`kRA}zckRA8TR6TeB^$G>)EollT8`Y
zU(NsgS~dT#m(!$TldoqB^4#uzIVHyRDrK?Ir}J}7e4efUGqXI+H0871)m5_TOVgty
zc+Xzp>RPpMVbUEJ8)b*eKI`kdU-iEB)XLU06D=;5)!4IMV_R+TpEU=PLT}FPc_7*5
zt-O4aX3+fde<|u5_l;|&^Im1swrq(kSP;BmpYV-WQ`BBek$<6c=S5~<Zs)B}A>WtP
zevtV)qx9`&pK`Wur*y3+OI^Lipqp`Rt;mLpahHzYeN%HNsWS1dxnIUaJ#D4@M;_I8
zZ8qjzi!5H5W$F^y`|6tXwcI%tAu2B(91Rm%(b6^b!?(jCp9B8<R6Ox3p#GWR+kc6X
zzmsL&h`p_gnfFL@d&(u>1f@nJJFP#SOJnvdkUo@`cmIs~_wCY;zP0)9WwLn~7|`p#
z>RGart6@v5$2t4SNAzx~zkX-nk?DW<#3dDjoUn_^qWg9*&95ptn{#v7!pT#<Udx|v
zX*=!Z#g|Xl6<BP(l#rWg&bwr%)}rI*W50h*(_L5hcej1Gt=XRzbLBhMM^8td5lyV$
z5cKCQlZRbJ-Gb*QZtX0)rCNFV%7zUWj88h;e0*5*znIsnKSCe=K4+ZvCbE4+;^SRf
ziw(biFS1>`lk=c|&r#2P*PieeI6i%1I`hdy|4Vy59Xeof;L%)}!(1LwN=Z-EPM&B!
z#5=!SSHtP<WRG7e{~evSdFnakRX%JE4V+h<B_(b$J7lGaM%66|*>}%=uAAVztstuU
zhQ#yY59NpN=XQsdIa;2x%IE#=dhW}krSWIjxc?jZdlswsZfTW@k!Rkt<8sJb^WxX1
zS`HN7ebzHCJtwbxOY-lHOT1f^ckTMK{>iQPXEaZ#md{Q*v|h?&+Vh2F`LXYlY=fsW
ztlj8wD>}1txkySF-%}o2GvO~uUU8P2)6XvQow|rubK6e4*-uXRp8MQ3LHm2q#pMO(
zQs%|~P~Kf+TXx83V$u88{33T4f-acPoNRqLfUkg&`{b2U$upmsZIUVlyg2kc*q;14
zw^#aLt8j2qK|#^q8Tm?YW-05pR-O9Lx5NF($_FRa-OEmU&dI8txVh$mhA#JqM9T-Z
zQ?uC<loMjMs9af+^mp|$w|a%gzRC~!LyPvuJEYH5JQf)A*ZFWoPRraqe*5kEFHVVE
zyK{rG{pXC+=O;~_%m34N+1I7gvmR~wpDh*6?46xt7dz$I!JtdYb4|Rf1bo(X{%kou
z^?kCQ)QO4HQ!2W}vkv+?xy6e`UfOlyVdaU}zM{ttIBga25wbg}q-U=p8h<V$xM%nB
zmJCVulY39c{M(YVVadC$YTec1ZoFKRKkb|qv;9ft_X?Bf?WcDz`RvcVx3KDvZ*}h7
zT-LVd;kvG2X>DxtwFU2qE`R)dTKk$?4Odf~HXomK?uzL%&ecaZyxrElEyKQx&w+DC
zOx))aBGc7({4G$-o+q9botG{5Zt20SdnLKyM$@v_hA)`*M%SWp`n)UJ3$h}*B20R&
zR`K4bZ8X?Fvn~6oQk&kpjo1Eewu>y>JL|K&efeG6+iJ3Irk~f9>Q~q#+<xD=jM?t7
zw*B6TFYA5oy?P~V!8ALh-nZ{STJ7_+OR<}01#i|XaX!3xmbA2l-Bdm8#PZC9>HE&R
zu3ol!+1WXBW7h~2bL*b>`DxdC;H>V_J9p>Z%{hH+cKlQ2_9bi|=7?SI41GC!{sNER
zWfh4Hr~m&{+V#Kj)Q@=a3F~kFn`lsY$JK3}O-*>ITl$88MR6@_7oPW%dhs<nXZgR=
z1=SmKE0?QY{q<|7*bjjzxAX2l+8aAzUAXVJb@iKV5`FYPxOo0uaHl#Z_OrpB-~%5H
z>|okf9(G{E-S2MSIo}KKIi41HKW}^F)6Eet1J^!3Wi?yD+Vb^YySuVyzs)NUPk!+%
z^7z&W#%&pU59g`qpI9$$_4)k#BT@a+8`+g^-`N?y=<yG}=aOn}k6Yr?ot_zPlszBj
z`F&B>A@R9=t^5X#_dnY99h-e)s_^HMq=08)FaBKo%)Z@+o#Bq5@3~W#<}S&K-Q(C@
z^RtA->YZc%ZibqT7O4-r=cXksZ(bJqr!mHPW*pP0g2Hnw6Lgj|>HlGjX)v~Z(#rgO
zQ9;2&M;`yL5=kE{&-G6b*ZZ-+Dy-(Xs)WR6_9w}zXOaX@B^|8%#dKEtgLu@Ps|J;}
zaSq`=Tx+lRTYWVQU$iys<ry|@yG!?$C!}l3oK!QtfAdWYo7Id`SM$}1<$gWg38x>K
zpX2UKyOQ?!uT)Y7gHSeyLeOhA;k~<5PESxgQZUsb#%R~oJ(t)0pDy*zK7b)`K~&6z
zHtnr1R~%3Nmb&odT+c-lpFeQUUH{)@qE>cPwqKw1Pg(w5Qk)0eto&Q1H^g2x$h)#C
z(IH1kj`Jj+gui>9TEMMKi=!S-v%DYamG?ZVBl)4eOv9wJ=gv=@_Rnj6!m`H(pWpWi
z8(wylm@}(BKgRgm^5zW@7N0`)aKucUsP=cp4yK6Tj64-~lLHe~Kh-Si*vY-XtoqrC
zz-fC5K0Wd{Rr6`m=UeZ-_^x2vF_|eTIWK3;<j)L;HvACSFjaNA={l$5y#=yK>6(S<
zs~?53oYl4nJ$L?@XqB9W`b)M0?<@Uwa#wG8`&G83a<!f7-me+r$6s&#AZ}ss^zXUG
z?F_Z;OV=l`YUWgOPL{W1zFZ!0)cU6a+x7?6TzlnDvVK0a@<ADEjKHsXD-OIq>!ZIX
z??RdNyYPxH-D#f>JP!Zlx$E}%+DqCV-<Q9NaIE4Bf3^7OMBO~TyzcC;^4Fg&|E(<A
zykW<>29aGquSj07_O09Q`)10CxF`v6spBs-EL7(f8#-xSFF#ikdx^b4YyIsLa&aG?
zYaO_++t9b>OUg7(D|=Sk6UQD%+?D*7@@DgKQ?{3nlfLWk-F5DlX2q@5XFgo5={&OO
z%e`spJHkY=&djQM5)t*I$o%%B$f~qmdGbNNvAf)-uX}Z=_4dWP^|gmDT<6x`(><42
ze0|)n?|1K9J9>LpnZT*?nXh)AzyH_nu5}r+LF~H8dF>@X^^1;&Kk~SDQr#o#q9SXi
zlKXK^%RZ->0TJ`cYbDBOT1^y<7nJ%v*`;XC+!N>9!cPe8Ta<l+A^)S~((84`SG=7n
z?<|$?3Y_WhpwcrVW#{S>J=y{*&Yb_Cwe889Sru_7-Te2g+jMTd*O{tY?Ta_1L~`2j
z?^v-<Pu4U-@lyK#t9Co`?5C`sl(WUdF6xPxs;*Q2TBg0~>=G4U_**OM*C?EivWQGn
zNoZeeoXxgWJmUh_{PmMkF2>HPe%+JHBzIN%^N}S>3isSS?V9-Q3)AhQ?WtK47HmJ|
z!&U#{>A%%a&ZbI#XWIVx<hS$IQ`*{ReCj_QUh0%sQ!(l4!icA4+GTC_ArI<ZC-&N&
z-1u(mwnqD?vt}!gy?xbtyYaDu(Ztp*w)<`4?*F^K<KF)G$^WETW*iKD{Z{(-DGALR
z@dgJ9ucp2{!5xz0D0)R9(r8PmlJ|C=@`63PrtMLUuxRArHe8l<P))+*Qc=Xa3wQI6
z3Hct>*nTM2aPNs%6D=2Ay1U&+&%tT3{ECy8ud#}KsNQX8rFDDGm&IS^2b$guIj_FC
zv2R<{^H{rImf;5SZyqg_%hGe(HsQ7J%I3>&PWmiRH+G!A<<6zE`JMc8K3x0kT>5sh
z>90QyQ?D1-dr!R5V*52S%xYD+p{?Mulo*B3?9<;3@0;+f{8#k;+?DBb4c-1Vs=ZlK
z6aRCk_}41A%m<A^Mwcr7JmKSwKHdH<QgX%Qn#&GS;+vmkPWt{~Po{dI`q5&ex$~9=
z#YbB0mP)PJv0zVf2XEP;w%2z)75*x;F1mK+{FjfPn*$Q+6(oIK9xU+elnOYtO~NGR
z0Pktt2TYTi&Q(wKoj1wLguh8Ar=YFm&hm3LhuChe4q0bAmFJvZ)~%u&TPA##{F!<5
zHv97ZCzhHtv$g$@KM`e?c#Z4wT+Sm?1t;-o&eO?s;{W3P^5dkR$yOI4e#{f@FWFT7
zOxj_ZU$p(Kx7Nl}%IkW+@XY@a-c<g-bJd4<Q3DMbi;jm6L#As9Hb=8PUt+_v!~9ko
z!^Znv+1KRn=<GWm{K~p&R^<HIFTC6C8r_*UUxdltKHF_ifA-y*Q<pH$oHtcxj&){b
z`MwR-vP+6fcimY(k-f_5b++Vb^RBt`uI|+Fe;c#m^Q)tJ^VHrOeA*Pb_k7{SuQwLP
zR4n;Xq2hY#+UF1F%^U4b-&?ncKkoovy>acH5S`^GFN7Wbng65YgVvv{N$Y<yCfbHy
z$(a@19lUej&4PogCx#eUrdMcP4_~MMX3ZUoY2ur1)>s@V;bV(_nEc?1SyH`cN{Lw{
z`-RfZ`F{(izMdPoHh!|;p{3nx_<O(WpO2l#(Ru2r?W!Gd(dMr&`s-bHK9+g=-p+?@
zkt?d>^>6*3_Fi>MnYNXk7)OQ2^}5Hu`EPdiH(qFGoG|@$yKwE{GCdWgqOXgUS6;8E
zcFo_s=l!zt5%0qv*6c47lvH@JyZ3CK@SOa}<Hr}-FFobFR6jt4OE<KiYlg;Z%~-w*
zr{Bfwxp!r!maN7{M~-bDQkU;rJ+sGkk^oZ&LqErZrbMs1z1IJBW_|q5-C(%;Z%m!!
z-e-B?kxw;ZC*Coea(mtXd%jK2Hx)(iDLz$F;(RFf+lzJKvl}+6$Cdw|w0qj^IL51w
zzq5HWF5R81@nW9G67dCzKU~`kqdCs?{oYsjSeSXnBBjeIXEa}bz0y5@+VR;<oWa{J
z*<2LX@rz9jS>x8!S^CWQUC6OWJ(mCng@)o)|Lm-_!&xRYF0s`={U*v(CBw>Z-%?kp
zo2|S)3pg2Co(m+rH92|fd%*hUj$qZ#uhc%tPC7U1Et6<OUZLNwIiXI=wd@N@AG#&6
zE0zh!c^uyv(Yz!g>&S{{H>W?JJu_f+*D0@EG6C|>=dNC78>zd$?tR-Tr)1d~s@lua
zHeX3HnX@A8a{xo=f?3(0<IYx_Kbv@~?w-s#p=rA(EzS$%E_(Gz?C<kwkJr6<GKb;j
zhC^j9<4#!KeE4B|=%q`mJ-v)SFJ1j);gRT-8}7Ev=#c0P7flUK=yJS#RPC2?#6=gw
zGaTm{i_}`?`?Ky|^R8@PqMvTsuYDy&!j>BLVJ~GL3Fhebv2q+cG|Ri==bTfP4@FHl
z&8Pf#-ur#NdcuX=$$X3capYtz_kZc-uuwDT$ifG^60{AU70;di`LxfhgE{s)lRo~Q
zvGdO))w*BBm05r1%vx8#_o(pp*TYfE{Q8)dOgNkKF>Ra6@5;{)%lr5Lf3N-Eox#nV
zJKXQKiKj)mDX+Kf4`(%zUwL0x<VHo)p1wuS3r$7Bm-4UObD^i><3^3A6YDl_P<3(s
zzOg=r@$q6SKksAVo8H))XY>adiK|3!+qpN&`gV)%36lkD7dT}0W;U93XQ@bC@%hJa
z=w)8yhYdTocZu|_cbS+bpgD1VruQ<BtqXWUF7@8D>*tR)cK0yMG;(Ur7nv~U(5ur-
z517}vL@8XEXL+=^W^&Uu&+K-tUz@yyQ~kV5f@Fn6;#f3{4(F-7n;dX~jro+Vw&J3h
zH!krhC>bo8DSmEGQh`n3ikmU3b_*@G2=6&Et?ap<*&VLFed>+tXU<<-IsMM5Id5th
z!<2m8Uoh)@_1Sd&dv#{XSEEHAy6;z}Rvu-Uc_NElLQBD0S#e?T>x9C^XJ*IljGEY5
zrG59w)YCcV)03P#kHrSOuGwoJY51>>QT6vyd!39^c5}Eb^sK8c9ql~(_08UibFHnz
zpLkh)Fq~BK>a*SjhmWnS^}o*Uw_J0fn`>pvREt}S<}~m8m&0&-qgg|d<HjA!{Ejew
z3!PMbDEe#n>A+mm$&D*y1s_zpB<x`Cm$~Ehl{Ggt&MD9S!}U|4dsp92jkRvS&-me!
z!ID|)OQPf_-h6vfzD2)%<<Xtx_uj>rY`Uo^w|n!y*XI&<75`A1bhh!2&Ef}ljP2D^
zQ@EDfR-TV^X)9kh<LqpPM}=XHzc)Xz?dzZ5+tjNqc(`JP4QFSy+O8`)PLiUl-_~ri
zS@~qw_5~rS30G8eZiqeA`dsj?Q{;M(z)S=EP961SidJtE*6b8+Yn-{kQC<Al(_4Gv
zziK}InpS$rEiwO)@JxyGi<mbhXe{+)P@3&kRTH)3YRb<?V#{rm=1<<Ma!Y(ut`?_p
ztIuqwINwd_{JV-(&K4`_a!z@w+?1Y^^ZcyWq_c+>Tt1^<&30fN!*<aP+oM&MYxKTm
zWcYmj%&p!xx^eb;mnJ9OGug77arNul(v$i>AAhy#zW#y<bEdSeG`G+T{>}DMXszYm
z^rMDGB{Ba#ganv`G1==#sr~eM#&FGfMpW=Y-N+p)6Q}2_d)!+R=qy(;F=UR*r=10o
z3EHP5A68y{Tm5&7?Dt<!dG{`;Sj3=~5F^~fV(0i_F+<{&@~PE{H%;Q@n9^&ES^Ia}
ziW*GOn9!BIy7X^`lijWj{=z%5pVZC0)h;0^`(q1pwb;dwebXj>%Q+hA>sWR7>$>PE
z1zkdVj>juEo^!nK@927Zdh@@#K2y_Y_il}?nmE(>siAjHbV<hSJ6m}R=AZs<n^^B~
zB7OJ1q}f@nIulEUL%$SNDlAj~@!+*xd%^R4;#K#QE23k~e4K6{dr{|6mb&|YO;}bA
z<FnkR+trZ{zAtvnm=;zPDLH$={7?4buhl;8+o8sI;&H4%|BW!Ed3!>wXFq=)th`?8
z*8bM-msB+p6ld-)Sa|8jPMbqpg7^5_@a^FH@UE#gsYYt+qmC!P_q4n`^k%NW+U)IL
zzZ}ct+bn-d=F9oBZgn$-y>?rq{@vyOf2XM6yQORQ%BTHtcxSVI7Q5=aq~6PmG(P)Z
zTl1fd(f@|!*>7d{oK$z2v90?Z-ON<+(qZblPZ<Yh>h%;__MA?QQ8msu#I4Hqeu<B-
z!6{ecMnm<0=Pt~C!Hb*btWccubA{ae`QI;X%-O9w<Ehwlon@AHQ#HJwZ0e3MyS#W-
zK>FuBWfM=S8Qp9!5}wk^`Y`-*<b_Ee(xj?(9?hs`eUSP*<g}Ujx)lji&ay4!_4u56
z@Ht=I1;OA<r!Sw4`pShj{!%_wnO!-t$h1Kvm~D@@Xj0>?mlyseGu{k;e1$Q`@-EXJ
z8IRd<?^_v`oZK=0Kx3}Bt)W)Q-11GUR_ymXU3Ym?jqq!e`KKSw*4>fO5OHsg_JecR
zQ{E>O$DJ`=JLOO8nkKfi_{w5`$94vet#Nq_Eo$Gm)8ZSSYoA;vdd5!Mk)ywl-z#g^
zwa<@VDSd7`&@pd*bpOQo=XWAZwsu?-ooE}9S@<>}uwi{N*NV3HvvR`|X1)vGa3EfA
z{oeoambag`#wk>Je0;6s5PoE5*N&Kt2a3bCd~f(rAQ-K4XnJbQhBt8)`@0O(PsnRu
zjD0%g>w=@dS>*nw@c!D(x?#~8!6RO8in}ZCotwSo%8|S<y<D-Yc_{~;eO%U1>c6L{
zB{w!V@K50-vE2qA68t<qd=7iK<HM^DEj8V3tKYV5_#kjrRz-h#!#mbPTlO43e7*2T
z887>x4MpD%L_Fo(v*E6L^{0=kQln1noU<(W+?BKY)g~SI&hpD`I!p0w%|p93-#oH(
z+l5@VS~*Q?zAY22yRvRLggb3>+7%s9Q+e;(k7qSu=gk&BI-60otv@2aeD}>wxr*1r
zoesWzp0r)>Q*swqW3=|-kj9T&OoeykmbV|+m>Rt5did(&VfkUA8Me!>3+-fm9?P)%
zr|a(PC2#KJbmw;b-li&$bDO1uSE1nZIh8rL)zsLYPcExrX8bOi^V^s?Esy0LPxtJ@
z+a?^DvH$4WlUIvh)^5L-8FnymVVS`Fw90#|`)+UE>ssx;AvBNk{?Ya6Pgk@qKi1mZ
zZ1TiM&6QKY`04yf+^W?jn@S2NWLP{aH}7{ne9h!-etf`po}R?k1c%29t`r%YPvkkc
zYvQ4E)0P)FtNlN^dfxMhRg#%!xlijl?cpepGds*8Qq5lQCph%we&+)(Cwe@XqO^Eo
zwO`qkt=0S+W;^6Gyl-N9e&4pbR7I?M)6K9+8`gzHPU*3)-X+==wkc!Xlhhb-rpLVJ
z&YbpLn7hGW;fl*`gO$bV3YkpFFZT2<-!I9(ll9K&g%1LB)){vj=j=$Rh*!A(%DC|G
zJ6>t_+bQo=JhucoO>UayzsGy2pv3K&$6r5FsF{~JFJPP66)T3M`8jMZn<p!GUjOzd
z=IOQA(~27}XWc*PRH+xi`I#@^HcMl8U7t%q^{IIsr)}3yyzKtm_~lEs`7zTf^txrm
zzZBj4$n)m%8K&<t6|aQ+b02I}*xv8(PHy_mmVgExt(mePUT@yKKfY*6r_{M8SF4I6
zK1V3+Osovzl6}V`^~BUx)r5;j-9Dw-_;PN+Yu;TvOE362aHyC@{E|4(CVj(Ty4a+s
z{fvcX3j*S93LLRIed|^A!lj&*{M`YL-DVF2wzI6f6PMp?zr(S5`)gfkUW4LCw-el?
zm!xmbs(pNS*S!8~C1>0J&h-AwVYbikywZkcE9b7woWHC&%=3AiLC2rRcRK7HJOAb}
z9-Z|hf7QMg&X4aN9NBr>;nbS71rwvTIWc$gglv3pY}$vc#}ByqB-bvzY*v5f-O=*z
zUrYk7Yu(}gS9b3H0f7hSLnU@{UGn>8#}R#J*N=wOScBP{^_=$X7s|HIaLv1CJ$qwx
zzSX*)TU;HEoIVu&+f@GJE3Hd$ulN4?*!`>3_10gzr}ISP&iOXEw{2rH>SVTipToAR
zJY%wHaEbj_9(}d5FCBISzbNZE;`duf+fK8<r(tfwsmT&?7oMDQ`#XVYzH8&%MFO|Z
zb6h;Ntnl52momW)iJObE7QI#t=Kd6;B+xxqc4f?s9Uq_dT;~l-S@p1)P1Sn+glpkX
zm@g-6y`=tq$yCuLA5~t@dve`P=;Q{a<jVVp8n4=(d%3zwD%U;YyX0OoiNu4-s?XBP
zJQFP|824x?1hMk0Ik2iOYVCW5NqIS{i&H1qa4KeGn)ChsmGampdg3dAoqV68Vy|*W
zDV_UM^l|E)SwY!t(SN2nUA)w}yX%XpSguu8-BuOJ)5lmBrOlbu@awtO+4^RNU)nDe
zgA1h{m8Rchv(~nr?XYi;@83oL-{hq|7ZcPqe5rNFbyC4HyAuA6FX8?xnr>g^?#?{$
z)1x~;?yH&S>$wcqgOAMmbC)y8mSK(h!~-Wc#<&HVO`Wqn^5kvPqeqV1f3d}Ri_w?W
znSwo+Km7R76FvED_o;uvEG|>$y#D@a%fkY(FJ-*?)29nFdQF{omUYuj&Msc-oKnW@
zrThIjD;!?&>DV6FT<luH`{Boe8BS7LX4b!+X1V?Gp^L8%U3_$f)5YA++${Ub{wC!R
z@4zqfy~U4DspItJ+_~2{vo|yB+LO?qcDrq_uYAuVpS)1+i1fL%uvJp2rCIJi^Xr@*
z9p>{(*sIUVkP?=ftHimz_nOn0^^E6i8oNJ#{5IP`G@<f>&$%<_yt-r@GCr+|b&A{W
z$d!2_OV6DD+17+OYtP-CI>u5$jcmt!dH!yBbd>W{*v*aaE<JX7Ru#e)r8etcPW<_7
zao%r5{PsVp)@$8}*5}y6+CDLAeTckQi1ph&dt#>S@{*V_&1;95?UDuO{&L*1VZG04
zx7TG>@o&3FdqXDHZ{Yj;mg&@6;re}B?jFjyY_Q(G-tG6LcjCX+r<I9^f83a+QosJU
zRE5%h54SgedF}Ssf8O$}yT0s1%dzY$7rrk{ocG%{udGn&6Gy<Jn@6se-q>?LYSZ!q
z><J&G@)LA?+LA1{J@~w9?Ssu<zASj?thQY-YaWx!`sp*C#QxxA`8$O_Nb<7CGtFbR
zTMr+8sT=J#XPL-(a}Qqq>zQl}Wz*L5vGf$J=Y6gt7Una{=jL()w+Jrfl+Lud+0MT&
z=tWt+=<49@DePB~t!%2x3prTe_(9<!%bMLY_UFU}>|P&J++jX_f{AF@WH!a<?ehK8
zw=?jwyiB<2_4;SfcC8(6M5ezL@=f0_dF9ofoYu#)Kh#w0x%>ECzAs~EZ~CUZO)nZ3
zm?u;&D41CA@a;X;hqhm(S@Ul=Ja4+geSnXTPwO8O>pk<u?Z#&8+js0y6z7pOcph`^
zBHxx`J7ae1pB<k?9^7a7Z!G*cr{KlW$McRqI~H?eM~!lizg|b-`egm@#fz)vCy1CR
z&v?6$^}PS1chT(gRN1!FZ_xYumMK6s<K3gA`8no}6IwRia7=&3eJAywhfen28m)bg
zMbti4XDthF>)ZXlzMNri_x{B{)t@<qxo?`-+c@QylUc3e)_oIZ&adI&zw_WpqwYK_
z{W&h~9yk5Vn%KKOs-JCQU0Ej=%p;!0yLr;J*S}6_HY#os+I6QuOF-Q-tVM64qkF~k
zhgK%-vRR%A{dbzSE1p<#{{4-_AP32(UnkG~nAXBzc`EtIn{XyIPnC<on{}IZ?==3r
z-*AOI&r@|}7JcroDi7=}8}EFWy5qmk+x?#p+%mAXtE>^W(I{86zW3diBVJ+QADiEb
z#u=Bt&;ED!4~w+z{)_)BUGqXWZC`lcnfv)*LFcYgrrm#D%)Xy@SZ&u_*A#WnI*teb
zl6+Uk2#BA2df}n2%F$mcb7T(O-K78YxRdnGskR&MW^5PojN6*f<WbWelO?oTu|eT;
z)uh6`#mYDDnQ6p)I&?T_$LCv&jDJKn^p{VZ?rbU{t~j6F=0;@s^M;Q*m#ec~5sH}q
z^ZUskjgh743c2c&Kl)AEUOPERalv_oiw|YDA2{81yFuknmeQ7)g?pT=?S&<3JiMy@
zRkr?;Um^cLl~GosULc%f+bZAF#}Zge@(vX8FWqtJVtV(xJhKLd#np)$#4Q(<Oue6^
zarSq<ppBl*wUcV)FBv0kp0|A7zx-YWb7AMJZ>v4NcqkmY@H1I2TIkenUBi@#dO|bI
z44L)zsC?j0_<HQ@y9bAaHL5Q+oGyRK=4s@uGHbGR!pDT{jazM&cOCb?)!8Cv|NfR;
z|9-K~-1jn{^%Cy9Q!e*V{`LERhW-3GANV?5*@dbeD4kCK#w7UXrSa#A%lW>WYu)-(
zZ+&Pm@e=;BI#cQKqxa3t!WHEWCF)THX6|Ofd#X9|OQJheSFfm?D)>0`cI)ZoZ-2#v
z%Z0z3U98(%vd!t@dZj$x&_9Ru>!TctPq50{Ec)cJQeC~2D}rmsjLPpU8($V}d}VoO
z{epvLQ?e?X%EcPnj=Y~}|6n5DgNL$LH$CN?RZ;f2kLk1Pf&DuUERU#av$^i><eYqM
zPJ#XtKH(Qtjp_Q!<qh8RGR@QX*|;R>&aIfIbE|YOd@uOM_DwGCb&1Ws0|mwhH22LY
zcRgg(_~-o9h5O?VEcmT<cl*l~nH>u{JGUL*c3UXu!_rLqY2q7RIhe;dG#n6Oz1(KN
zxLx**?4PVDjJg|o0=v|`g(|pqe9vaQ?YB-o;qm3UF^Ovx54@cif4QcTzl2#ZB}d#b
z|B($-zInxrRQ{lYdrv%H_jWRme{|+r^|1U}LxCcJpOxa`K4GC#GTr`vN#uDbYrE^j
z$2Ff)X3h9=Ym0PE4dZnEi$xd8OWvjRX@>Y)O^<pz#Z`TsglC3J!Jg)>$=qwpRwoH|
zZadu&_(<+UQ{bI#d%tp~+-cbRsc>rbyB+LDZa$7v`B(Sf|3Znww=XT{=WIK2EAvCB
zp7F6Xb@5YGi9+R~6V@Gixhuue`(DG2^JOP%o}8}OyL|5MPs$VDNx$T}I{DSIMYHbR
zH;kE=*L3uzz3a#6wFbNw+B<4F_FY#kx}W#l$Dd>S(RZ_ROaJ?ZP78nh_2eFt1cC2N
z|F=tP8>HWf`IuB>>3BVhLv!Ju#Mi;RlTUY_eD?FII_HCzPK|9Rqo=)(U3w#A?=!B{
zMY=m8LTzV8t-F4)E^q2S|Ese%pAD_wQ1tgL6I<+-<r62&P0kUzyx~IIR};3A`@(K}
z6g*z=&(R=(^S|_kge_quhmUF;u9>Uw`cur^9a3&wYwxZ|;SCqK@vBboY<2Xa$84?|
zDld0EC=T5Bbz0;*nO7&g8E!1BT_+Q6*JORxVV{4@)epj+xmB9x`S)&3Oxil%?!Lj<
zOpDU1zju@!{r2u`_l~zuGQR)J-+s4m6^C@kw({-e8-%<6F7eV^xiemO_ME<^Jx@1l
z&i63=5dT#8<h=JjejDnZY@6J0{J=ls6%!LOOP=y5d!_U3ow7P;gOG_-(p%>@wKJ6L
z4TOzXIsDu4HHpXYxXFyv6_(3v_5{C+h<xsJbdtt?$yct+1m;D)6`8qt=8f77+uzLh
zpLuf4wS|kzJXbEN*r`74<L8BeQJJ?)Hihyf*xy!{_r3q+v6z+VzQUU2XLM3-viy-h
ztHt=oWaX?wE+;b2cGdr%|LE)46Ar-v#zqSov{;1=ss+o;Z%oPje7>lCB4gy@g`%2|
zeq39dyD8$SSyTLD_AZg-^Y0wWdhyfpbMfDpsHp~5<sSb}`1S16|BX>!zH?hV-tTBr
zX?0@$^`lIm_K7^=IJ)GY*XN9>ztwhrtyub7y|CH!GKWb?vN(VBZ}y)jejHYFIhdt!
z<E}JY_2CC)kw&_k)<piAwB}2!;(CR}#UJOr`&zgo!*qS==U>b2O^}XQT%uHevBd8v
zw|~i9g~YnUz0&VvFNtjB%h@QP^IG&$=f1>A$IV>+OkXR$=QLmbt*!|vCmd&8^ql2Z
zT-TbHutqxa^j}-wG_iB@qvx$JzxAYt&#QRuKI>x7_N>QytWVxD`JnNO;c?Y<)pw_l
zx5wF9R%-dKo3Y}I#>wtUAO5H5n_T+v{fgF1{%FS^jiyplze;ad?(lVQgZP<u28oX+
zd<=MI_j2ZCu@j||#@!ySR)(VQHm9iSg~Yb^EKf1>j@3DN;&99X%L`V|=N*%aoNq43
zCdPE9MbhYZ_k!9PuAB8+|2h2(zI1Z$JgzNQb8bZTh}qk>t^e#4Tq(sKeJ53h+a+@&
zpZ(-d;(yM~Uch*+eAz4eE0R^8rk`b7mtOSSxNt7t{eK0s<<-9nNi-;1ZLF_*mAlM*
zUQ*hh@Ix&1YyAG0b2QjY*yP80Zq}2yW4nuPKG;@p_!#d^i`&xG$;nr9&MdV#dW>7T
za9Y&8oYtRxZ=U?Qw=?YJ5sy;}A)7cCOgX1ky>?BsP3ff_#+(Igrxn{+=I!KJbl~_s
zj^!x_Qc`4Uws)_5+xb&>ZZ+E_o$dQRGV1=^;25>y=CtsQK1rLM{+wL4`-H;z$M>bo
zO^O+&e)$pgqcUeDr`>+Xx7*kKoV}=F21C}jYu1%t3Z5D$=s9R8&b`V$?XS<qLkXu>
zo^6WZUv}5o%;1U5rb&vDoyvPYFJ969_q4|0*kb|Tt~ZO$^yKpJbYp7szs@Q7WsY)m
z-MyfV!8@YA?B+UKoooL8&de8E9=&$mv!CPe<a-x*-AgX+@>w%q{<rb=uSdR%zG%Ju
zxLV_Zn*<N@mGWz~-%s9oCs)7cV(a(Qt)G^c?|Rt2PvphNI^C4Vyu0tJ@93|QpU7RT
zdE&{}nST}que^M2-_MTp<KJ3rc~}Z8kBFc9?7I9$$rmRkkBP4rJUO-Yj>z|G=NPBY
zIGr35X(<-Xc(X*2$Dfz+&vEVlF}go*vzA!(l$1_3IGy}(VSC9-rhpg^v-XdlOKSOd
zPoIBfNrs<n&fWWle=hcRl+?fb9sH=X+{}3Lt|hJE2~4^tO=rFLtT+?lV%(RQRX;0z
zr*6ZGR}7218U+tFN;4=%HJ$Xd+xXOY-QGx#KY~--lb>&Y-SFs1nMhJ$tH$CZ7hIRk
zX*qH5py}@X%AL|ilTy<DHB@v*&vyFvDErW|?AYZ~H@HlizhUY1KCSg8`melVU*t2#
zExRqZP3u}xbm?nuF$;yRof8GzxKsQSpPw#OvgJB?<vjn=nR1frR|RgjaZIVX!_7MN
zST5@U>k~ItpLoF||M{%Ck)?KYe|*#wy$ia=TSSiTs@)%z8LakV+Z|2blIV5mhFrmi
zweLAzJ-b5fRp&W}FFDc)m!4i)QgHsm!d6+mhgt8$6Q&ij3PfF3W9?`=`7+2Sll?CL
z&LpYJo{R2_v&!46ZT~Gbb3dPb@%FHdn_b`SKYmo;yU5dbe*%3={P)Jql`X2jp;ggU
zx7+VFLu+t)k>6}Bdl8fO!gv3=Ho7vVZBy?ET01kqb^EH+>#zUVUzzy7k|~Az+OalI
zSsno$zbo-|l8Zv0+3dTVz1=x)bB|uhGyY#{V!WwsV%IZWw=o*5=6omGZc^@}FS1j$
zduP;!U8m%dF1Fgd4|LcsK5O0Ij=9|O)4BcE?|#ecdfZojebVw!p?)pSUI)j#V#WtK
zqFE<qsf99r=j&H5i|(Jk`#;yd3%`%<XPjz1Iii78ea^~<B1cM_r^d1O@%6F2C=|8(
z7BcIYXmc|2kzb~JC!9SOmb|)hRrxW|_a^I)d0*W5_<?0p^Fzb(OCj=G3;lM^_~9cP
znYv6($Vi7peR0%=I9?8$mK{&y7%j>_hUQf}?yzq-Q^|jcM|aP&ziOMfYr7{FA2fMg
z_e-sdxAlX|&Xvcf89!!mcvpDvC3hHe++^;$$*#AOg?=bYOxyO_^USl&uN7KmEB^cQ
z#H`v=RDJ&L|1}+3PxGg5{Zk{V@c4AswVDSxdf}Wux_nfHW~yut@orI<{t)mouu1bx
z?C%Z#YAy*+@^$w3a%kB}YvUyX>K6P8FaFJrw$+=Z;S;5IP4mXT=UP|p>nl7z{>H_+
zluPM)k>btW&S%OQ9$fqV&8*rsPo4X6->L@bC1$#I$It!VYMwMzTx$WhWq&1WMdF?N
z|CtWjO|<-R_VC?A6Fs#%AGZiwn(Vr_b;~AM`+P3_Y{x18W_B4odq3yO+})ez-VD<$
zS(TR`I{&;uZ%vb2a{R=Y#BCn?*Tz@Be!ae1J^Fy%@;x>SckaEr%~tr@PuJDC4aWP~
ziZ2{s+jI5kiSE<0OD8|NI_<8JQEkSR(yYpPyPA(PE?kq)%AwJil*P&P{rT-_FKr!8
zRCai^>Iv`*#Y_3TUmM^*VXv{`!-<ZKGv`0QG?C}4<hF>&<9{BUF!o$?ebT9_)V_<S
zI8OYuwttu<rTOJ*aNOx$J{#YvOD%t;g=RWs{R~v#WIb$FtoAm?p_e^cWpeuc3prYo
z)E04nFK<1ps%hi*Key`Ln@9Pd>V!`FK3FR(!C=F^ifKYHJE!-jYX(+AoiYwHc((}I
zO=EiO)mXojM_+|8Mr^vz?HqIaIjcT$T0N6Gm@GP-*XQItufnRJ-IH5(Klk6sdE~T4
z<t)$G>7E@KArp?C*jv9&=+9rKcLq7%#G_MAiwA~3%89&ZXFXXi@2Rh%ZKBhBd%fza
zV7ZWm=ic=tR&QQ<_H&#2tONSTkJ>$6w8h-c?p>?6j_6*|H%qy0x0Z*lomoEj<YTMF
zo<AOOa;l3xO74I5dhy&jx4pNR8pis5acBN@QtHh9E$>d8pJJQh`EJ?2vPjD<Y?EKN
z{GGWc(J%2C2Ot0Krt<VhXBVISk<e$Sb#jl_j?abPOSXIDA6l)v)3+^0>G^T-Utb!3
zPdz%hpglY|$?V`|6^XTTe<djWzcuHbNaWfY!`InUW?Slh^f-6C|7HC0zm^_9J}Y-~
zs9(P$b0CvZBCN%Z@o?A7^XDEeo$)B`ZyaC4(#m@x&jo{T%j`)hQMn;HukEwTqdl`7
z;+J`EdbKiO>%u!BT}oQPG5;qm)4t-vv2E|_xAS+1YyWDhwR>^4Z~gbz8|v#nXzA6z
znf~f)aKHyOgYw&&R^}|6)xs(=O--vS97+mf7`bGcIu3t+wz#2Awff;6jbzbqm6O*`
zm4r>)e(TTaTOXn=3^NW$taaGm*{rjv@qa?;uivaWzqXusd2sjoz-sTrbiP;H_CAlD
z5#}ebAn5JeNm7eb&VFY}V>i%UklnFuna1n)x(D8}Za=YT#l5CfwzLPvdwvNle_P&u
zBk#Ta`(NIX7hgQudRZ)Do7bU6+bDU?PT#ImWw)D-huyS(mHPiic*7QjuRAN&RvvM-
zDCO8)wmT(c`)-45MWrjt16B&Wth#eAuQ{bjnyqnT)+HwPBSFhAym<9aTq(Bi4eyte
z=7p*<)=k^aZa+C?t>l5LF88v&?#|lx&eLgql2GNWwV8Xa);zrRu8=X{bF;6<yrv^J
zqP8&FNj{pUI8XBIw9ikj3CuRI;F><8u{UwgqWtK_T1)e|KWbKT`SBgIRyl6{q97ry
z<G9ymzWUE|HQZnJM7G}ba6Hduz0@*!OHSN8#u8b%$7dcyF!dW5-?cdt>~laQs86jb
z?Qdi8rMr6;%$l8$UsM?~(^yNm>`?uj^e&wR{(m>FNfvdfnQ0SIx$ju{-sc6L6<*AH
zv**mRd0*N5h_}Y>vQTlP^Nc!?l$-7i`&%E}kUVj(>-PB^vrOaq4M~6BGKrmNx?1c}
zzQ5G_aG|#AHDBdTUbP1ClSHP9=QEt_?Kpby>`IY)hbK(_{LLg-r$o8#`0)#;Cc9eY
zdtNvcdi!kq<+#s>UVkr=KeAfB-2U(9nv)(A64WG?-rOlwWWN2nNSt5lyv;o?z2h=F
zPt7rHYj~cqO<Z8^smFSsrg-Gm-}_#3MfGyWQ8B;YIonsibq#p>n?W)@Xp)Ym*R+#&
zyt}5VmYz1M+ThcALfE3P!TXG+5bLEV-jI;PnHvh86kU1M_mneYpUVALC7f$hW^b5o
ze6ump*JOIMGDA}8wH-&KTicQ+rKImy_d6~6WOM90{(#$u{4zIlACJ4b_tC+7Tsr0q
z7245Lb*FQu1m4>`{j8J}S9<>8yH}>D-B!*?-==T5=IK7O<Xw#4W*ydCTgoi1$8>nT
zzwKF7mpymu>hcyWzLT_WrkD1T9lI|)^i96oF!4{-;YE|2`$TJxq<KEI>3^MKHc?<X
zZ++_I$HJ_l-FM1=bBX<JS$j0+_cHw(ZC~`AUFs7Ap7M%>tZ&#+ekbmOdB@-T7p9b2
z`*T$MmRlpuxgsq*Me3{A(|ayln?-lnWv!mAcH!h~UmnR6@dOExiu(>rGYW-e`ncIP
z&-d_EKlr5KKnl<E&@~w^pD>q*O)qczeX#ejns~9^MP;Gculrx~#;#VM7|V2(@zca=
z+bpigsp(VlMPFQ5r#mNi=XG=c9qfnRYI{ih^09tyzTpC+i{?q@?)luy?i`sBtv0uO
zQRm&vm9HgY^LIX%iLu}3S#EXu+TZs<N&ULdmo`qTydqn&nrj{N&d$qUT>O6dXxvMk
zfA_)toe~y1o~Qr*IfcD2=+p<MYadT6Q&anAm3da(ThqQV@xZa{M-OZ#FI_EDui`ad
z=XVUluL}u1HkCP>&At?NW;EU0J<&CntyFCPmM^M2nyj}bbG`O#{I*2Lc(&qAJ4vB#
zPIK>!NltU-f9Tnu9&8bxB6Zq%(u&g4%%<O6-f@0o@jU4{(a%NcW4dGJwghqi3X28)
z?B^KHs!dL~#ZYrNAnw{{HG|BbpVOTWo?uCjn)UGJ4p*bxj)eE8GcRps-e#%Jxn>d%
zx1^`y>8(23Ew&{x*Pof9%Kq*A)ppPhzuXxykDT&~Om}Xd^(NYPudBM<B8HRaiyG7K
zUaZwmsTW<aE{#3-o_~klZ8z%!b6=lTxVL@n%}IBo6n`YL2UKn3Q~vsTRqLlSk1Gr=
z$#uoZc1(Nogi-&vL%;2sq=}JM7c>2CeOLJyE5PINOFzas!Lu>wg~y{i2bV6sf1A&>
zaN5!N>gxLMxLP><Vip`b-`#F5FeB~#qdA<DY+9$zKff_P|M#IO(T9$0DiJwpy`tCR
z?I+EjrCR6OMZdELxH<bbT+(9T-s|#klF#B;Iq`^fLA!E#PruzBJ@5YepP`jG$&>bV
z8JOjWwca<`Zd+>?r+035>&9^52eMa%Z4A0^hkwXrUAWxbK-6ATZz~sTUh0Jv)5V<C
zmMdkm$v)mQXR3hy-rXtEHme@qfAnY0ZKb21@(#R|*3tdxaQdvCvGQ&KuZ36V)wh_+
z7bJ8YldRn#S>$v5&@|30`Ny7jHgjg5s_|KNMmO4G<CXfYyb7BPkDvCOvs_POwRrxU
zn><q6W}SHI<RB{Yv~pW~$^GAX8_%!(xgk*SCRb_H_Vp3H@A<E7+#i1+Rxx~O>xxV6
z6NG<u=-NGVi`uYn)%I*X-Pdvg-+293gipxa!7OpZCE{~@yt#Z?=<T%E|19?(h&$~e
z%VRFD{AF%Y``Q%#tBaaLi`W-jm=Jb@v7_M`>)~%7nJzvm`mVjBK}h*Ps{Mkar?0(v
zF;%00gYEJY*T=_a6>|#A**kk#p_X{qxs%;&B2qmYXK7l?e5rc>u|O-a^?}n$d4>5i
zetAxBTI^R{@#xvVEaUo$zcE#p{{)@hTy#P5<AjepBa%AheYUTDl^1$<qs>2!gf3J4
zg{hKDF86N;3r^5!-+J>d*DJ}EgXbFf)f0HQ9=&q3a(B2{>&IAEd-uVQ7xELPsQl#!
zuz3{|_0_WJ%JF9J`v=s@XNp&u-iX_?dd9KGn|~a5|MV&UCA0SM;`gmPuRlL`%hHpf
z!23=q<0-Wh>F3l-AC`3FGd=b_*gM-ucdIJ<o)7=zxlEL|%=jC5=B3J<lFBWg8wwVC
z&R%kvNj6PLT`X?V?ch08lN7BE9#_x!_h+Jlg@fSIi=iP*Ug`Sm&Ic|{PLTa^eCqap
z3|oKwW(_g$XOdNK$+)^-YX{qVBW|mt(@!>}R75P=<~{9m@!l(34UbQAuvP}-bgzwT
zQ7QhM*YqIy&EB1Ut9J^l*crF_<b_*X{zyl1<;QLINs^6?m^w#7aR#H?t%GLlKb|xE
zfAee;pTVVvq6b#+=9IMjczIp*NFE>GEk`@+4+YEQ-Sm{Qf3R&&*x>DO@3hgTu!p)1
zGr}j&ynCWH@5<ubzR1WanY%q+%+6@~EL=9na*<!9wV9V0vnI#p00q?x(ofTGu>__$
z%q`wI@3eaQo_qd_l^$BXWYF5ae%)>UBck7&j$L<&KKH6luJOD?Tvys8iN(r|a%uB2
zTU3npcm?kfYMC<o=a#MUEEn>(T-wB(dUsQqwPNb(&RqwXHQR5@S-I(|i4Ru-^R3LL
zMe(eE5B!RIQZViOf3=_+a?#s9n}jizp2!t?&YZHjEjOXtY@5UErjHZXcFg&BJ)G}q
z9`Dz6Z!h!urD>Oxdq2_p`iX7gVun{YC$3!Po5y_r@*T0q*)s3j3|<MX?*8H9D4X3L
z7Vv(ZJ=exvYSYr6nXOtN{pi&{yZUWbf8R2lh%DoKA2RC|%gZ`b{pl-O8q|FEISL<`
z=@6IhWSg8@kjVP9CfreOPr<_FGwfd;n)BM~!(EA(cyaZ(ecIiLj=ufzygt7tD!f>j
zrIG$dUnyRF>W&GPJcgyri5WAB*WSrGvF@wq4dW*>B=VUjOlfO1ov9<jS*YfJtf|vV
z{_{qQC+D79&fL}+DRdzB<RXR|$&;dUb&jZZb{5K*>DWm7&D~slbav{_*o3E3Rex0R
zExnv0v~t>o2&c}eH8-<T4`wMIdUfzroYZB*)tR%eru1sv*5m9o7t!>VF1WEM_E6%o
z)N9diR&fe)Mg)5Qej|{n^?r-<_ww)k)eo!MHws?Zvmi8DpGRLJE+I~Z`?hb4C!=%5
z>T0KsS)JVnU6nJ^Hl3dA_}nMHrlo@IM1ZLb=L99kpp9D&RWIG<;@N33v&2JCdFJGz
z=o80fPP}<}M=k2$r%xZ$HI_}9b4~Qn)v9<7fs~E2ZpHHOUP!vTTuE0g;qn2Y6S<Ak
zST45JZ26bMx#h%xnGvT`7U~86Rz0Ay`so(t4=RltrdHnF$@$J{*TY9!t1m~s+H-g7
zPFH!|g<>mp{>Oa%`J460nptyATW7y8x!0b%Xzw4F&M8ZdMl4l2<!~)iTK8P!)T73_
z3*vRdmSxXBYOnq0!;!2>f7AZN2+iSAoEw^7{B&cz$oj<Eymc(6IhoHK3O(#3eeUIq
zn9RFX(~jj82D_+iXRu}Rc`{Go+oM^JAI33G+ubzt?o+k<te>R59c8#I;ls7=>BPk=
zH*aQadSV^7aFOlvf~?P6{;W|iAL!0Cp8RZ^)7H{;_a7;L-*4s9=5KW{tCv0CWbVUL
zi&IoDs&p32&fRg==nli)&iaPt3H1tl(*i6M=AF(B&QqGRGymuk^Jy#94k)M?u}D42
zt9}&AttZ!Xc-_^ROz)KHg3o+-a#(fe9&LHQn+y{#ww`+Zd+)~U_qXhO*SlN%%I}0z
z-Q8#Qbh5t}jz4g8{hhrF?yB!s2oP2^6kmPjHdoq=RYwoq<P({y9b%g5d#_4Vcwa=w
zowbQq&Z@s!XZKF;=i3!WZT>G(sjfJ*x7k1GQiNgqnl;)F)aHe<&QQAcvOZgt_khUf
zwQLtoB(3J2!n6F-OMUaBNy{Q**WN0!`SS07%+_D|0vB9dBy5alpL08@X0dojaq8zu
zUP)HZHa)!Hpdf9z&+PRq^UNAc=PUF1lg-7B#D;!cDe~n(x|-Lf*3eHZC$gn-)~<Eh
z$^JW|`OlVvSyc@D4m~j&imti~``Kx<Zo73<y^S+GHpl6Q)1!TP)1>2AZ=b(jXRJAE
z{iC&vg)%=_bqs7{8fC>!9NEXfFk5&+!d>s3Y1}n7(GQw7Kf9Z&-F{rCAi}0y@5Gm=
zhkZ(m4w{(X*WOlPQNgFaA<uZ*RgJJ~lQj4OTvQUn6+4R*+q->z6x)APK39?5Yq|A;
z`i17z{;^AE=|5`}Gwa=vpU)&L-XpP%No?nXEe>JRuSfpuV3|9qYKe2&ZAE=c-l}U^
z9w#r}J{e)-z<O&Bqq&JumsNr@`{ax{KR>l*D#>`aG<z5Mz4Ftw<!v|T5>%RY_sN^Q
z6>7J;>p6NVvW@D`9P?wmcRVt$JMYQEXD{C_|5Nj+Lu1l(0ax)w63>*s-u_kpLYepA
z#KP~}xhKUiCdoJ(q!cZ(KU?s?=1h-l`roHf&)Y8FFL<%r`efLz$)+#fiy6FFpm@yi
z#WqH_OD`WeUgtBM)b_JXG`uOdyLENI1gjRcefp(G#rjIcpOr2BlF5AC@^hNCu}k|I
z`_0GZ<wQU0-s&Tq|D>^`kuOYJJj{6h^!$&pF|6}Cc~0)KRk?M#<--~Nn-ZTpecwb^
z{Jtc3=;j{J<LA;C?KiA2S*d;U;yoROlPYp$Le8i2e=K?Zaf<xvb>8c%tW?W|cdsfB
z%-H?n@INQzRVO*sPpU<(((3xKB6D+)$0@_rpL6mZStf@U*c7)X926@xFkCic{^r@5
zT21V%R}zfQhs|1W^8CE{+qn#Xs{B05ANFAfr_qGhdt;;~9cVnMaz1*FUhpE8?EyP>
z^UB<_x)-)rK{w?^1^;E|wrx3bJJdTEbM(%zv$g5_%(|qzZ2EqWO%XYI@3N(insqC6
z7wUZW7RxA(G@tBx$0qh~-`W1#vlL%_N@na_Yrd+yE4XTPiBIm2lT)SdY!{Nv-_`bG
zuKwJ;8uIhkKW!;5t77d-e9+7`&Dl*X`>>#zuh*Fkd|Z}1e-gRc=d#sL3Of13=OlkX
zqRzU1GLpjge^nei_BqdUcPFnyPyKA359c2%{cl|LEnfV?gBy8FOfRi%ce%_oTpGQ-
z^!2Ab7fy2h`nQGc>Bp%}6Z;wdx-x&hI-y!Lcul$Fo}Uaybr|PtWY9i$G~~efr)-aX
zl@HA25n8cnjo!KFsWvY|mujpk|2g4ciCA~($xpi%US0S8D(}u`tG>SKb9emCcR4p-
z!8my0(UrUGUWUg0F4ik9t%$2Ts+9Ne(@R0WLvN=XQe~5nxN+e7&dJ(R^8z01V2Sfr
zOZk^29p`)e{>*nrIHfA4H@}v?uCDxiCzIvFuRB7DcC+|C?+V<r_}&zWji-wWJUG^f
zF3x|cSZcHK8*}r&^Qjh0df6f3IU;!$`zFm3j=p`u@XV{wZuyS2jmu1bNX+VynQFeI
zamU<jb;rjg1x!}uS+R3}82JUM2dvXxx|-Gc!R&u4q(5FO%Tbc{J+kN~uUi}^qyFE?
zb&K=cvh#10-H<BPZEM(Av+P05W$%gEt25d!$4!52RUV!Y-k>LV##h+d>hP9(nHm`d
zj(mq6Yw!H?@93lHZ=0Ai*YE0Hzvjm--@~t_dAweC((1IiPdDeD#nFXTVYgqKyj1a>
zyZqq4lm`b+{z|_3Dqp-oJ-DQFjoaO|L3h`t-PNt0E4SR_Z}-8<v#|$mg>0Ui(i5+L
zH9<l8_qm#F#&sX_Io-XNG<*-^@D;DII#;*x{De!_=9Zkad?#%uDK|s=vk~LM!)|vT
zM!glWvAJKFAANU!?f2@!sI0pSZ)|<B%W4nz&!?=vy1f~>51w3bWAf(TCc^G%k5=c@
zOzrI7^zFEEKw)`6FW;<->U``AJ%09DJ<Z(RY+E~LnSSum+G2~xi&9+b{(F>MZwQyR
zj_);=UhWq&ul4w0{yU!YUQe=pTAaG?;LkeY=WI^PnO<MM=+Ye#Zgz#EVVQmQ;q)Bw
zr2je}t&P;IgM|BeE~i}(jh@$$b8E)&sQaF)KP1dl>zI9(cjl!R*Y6zi*m`eUO>VU2
z-@>i;9v2-r?pk$HE_j*sA>sVrJLKb+Njat}Hl1b)ZphL56zx30xboaov7*F_>4|qE
z5^G<0*5udx-Egp$b6tjog_sSu%*kDQHq_ZDxm{kL_GtB!^yde5y#3L*&O*8E?3`C>
zfu6?mYi9_opFi!kY<+7+$~7s?(>B%3N=FksZ?Z+2Ppe<(bm#t;TlInoDN7z@hi=Qu
z+}Ekjw%=LRBuF!*bk5R7?yDc?O3e#7e!9tI(aaaVF+Vm<nvhlFIpb`so_X|7A@;i}
zDVL`&w34(t(s{aZkI$pDfTMms?=LTVpW5kqvb@{#nWv?9-B+{Cvuip_bkxqCTQ=)x
z6t8{g;fPzO66F?mK1fWe*zv&7XLr|W5ye*>4Ez$FhqcTDZT|MYWzCB}#;#nL7Bovi
zU_*S&u}gIgP3E?l(`Pizia%WXyV<~{C~>c(Z*`kg%(od*ZNBSnKe?N?=67%D{tfKg
zc^el#of+}3Q=#Qq{Ex>e)rUPEKQ?<~Qt02TJD;&lnAuRI$#?$^%Z>l0`&LGOdw!^O
z&QkG(jnR2aQZk<>-|shZGIZt7^tgQH^tX+se3qS41q>9<`I%m+Tl@DR-`ant7p%43
zu;Jl!vA=2D#{Z|9m1jJhRB-P>qR-^J8&0L{zPIIQt+9r@ZiX}O)f4R8cVai+QYp>6
z{Pe};uWy6%R@yxL*z$^V+d>6n$Jf0T60Qp;1ijlKuJwYgAnUq|`Q=MnO)Mw%oy>Ju
zD43oR(Oi|XZsxycQBX!o50(gdS)!wCJn8K9s_mAE$(H&}MzbAGDVaE1W#77bdm3|#
zZA8N5LO1Ky>BgUowrsx}XgE*1tUMtg{LTB_k7mzpl=V|eQf}Y+cL$3C&sH5N_Sbpx
zi|xc`C2~ruAM|@S<M`DVo~B2)%u0$nD`CnRCZfil5I<Suc5hMV#3z+<>-(8v68xfa
z%@1$<ZC3vG%)ifZNoirK6Ps2Z{NcCIr0GU4XPNq%cSo)Trp|cS&-AMI?~4V~wO`~_
zH$Ok7vB&Xc_yOPLAN-CM#P40(^yuCR^-S{{7kP4WZJ3)9)?Bpqoa>^`7A-tILGndJ
z-lYFG8>c^8Br&7Sj&oMXOR06D_t$*Ax6{LBv0!iOlN~uTU1ly>d~&MSq3mO_CZ{)^
z%1=1Bc!_h^XVuk}lP*~-&CLx;+!x-zME2y8i~|S%oRGTik^Z0I%5nwH%sZ`3%UZpb
zv2rbsKYKvsm5K5m0q5KEixXy+uX_>kj5)LOp>f294b#;^o{RfOujxL{U;KGmWYg(e
zA5Xg+FK>HNmix9$btUJgqpV@9;p@|qa?B#`oJ<Kl-M>^%XzBmw7bpC8Nd2SFopATo
zPu?@jtEavFw~(RCZ)VFS?Z&5<6GHuUJ{IQYEfPN(+V~;<d3|p7bq2A^^FK3w=`#8k
zkoSkt_3N%Rv-&fBGyjlV;*qH0_UNeY{=i#{U#t>-drE0b#PXPppEv*Cv-;Y+(_89y
zZ_|91;iw$;=0<dam~~0EuU)9NkjAWaT%RmuDo&cp<Ruxsdz-fL=?Cs<x|=nm?yrns
ze{k)q?_=NM+zUQy%|1kM|If|Y9=$sB&eua5wD%=kpS|qroRj&E=Z`Lua6Xd$(>wI>
zrs*FvO5a{S_2|WkV?6Vt|5)D;3fUWO#$zE``B)}@(;m6YXW28wukO8cG$wXw@T%pR
z{}n4Vws##rwVXL#$m6f!dzP^9?V{Wzl|FT+lW(j1VSQ33<vU|tXOiCuHldh=MzL!(
zo;7*3zOVE2ifm&xCVgsq9X4xOr;o{8-=7&VuP<fhExLR!N-M@G<dw{h6|%h@R*`0}
zpC8s|US`oz`z&tV2~CktGaHvq&WlcOC}G<1eAijYSm}pGM`v&O@UPzT!|r8{#_=s%
zrtIhR`nPD|@uRk;C(g`gy#0OeSwFq)!f$hh-8=Yve3s;$d9>wsTt)rb?29uSf{y+9
zCAnX?L#k|=%<Qk%YNqCNxN^##YJMl7Ed0YHFLCNg-ucbR@xgiOmuoGb9W%ehH~mvv
zc1GgigA4!9;#}e<w##Pcp{EjD(oa$rKQlkgK40Zf)v}qP?W<?$bhlmn^5oiuD+fQ!
zjIaq_P?RLb%d;Resdd_U1NTSVR<9mR3%+incz9}>_u;sWVbe3~x^I>mKAkjaF1O)a
zyT!My_O@Mq9qU`a`S8}c;<_hU^=6)3`m)kBS#Ha$N`sFrj!9+DvQ{6k<|;XVlS^t{
z`7y3#cMF8}ea)F;KQpA+^X;@1j2|=4Y~kG1V=ev4^v<W2h0_J~9$b0$__y>DQ>*B_
zd)cMmmPWsF4f=I{`J<hT*QNDJ3u}_+h9qyS-Rx)P8FiN91go3a!t<^>53rqg*euak
zdPkD)dC0M%<Xx}MOq`it@AtK8`JSqcoEy7XY&nAGht6CeXmwWaU1ji-@XU*=66ddK
zJ~D}G^Tu;$t`xSi8=p?mlj`|=ki~dLg3O=#4cq?wWpcB4e#te+Zg<(2x9^R2E^_W&
zn!01Lv4z#+1^j7$KPpQ)CdW<fHJ(u{Z1Ubsb(cZfHmx7EqBdQ!#SX_Vg?uodvMuxj
z+pVrZi$x9BRNhAIxGA+onL)Qa=$vbu#o@nxj-TxsnVz}IW|eA5`pMq?=BIilT=e@M
zqsWlC-;U)N&WyV>Y2R1bBAKj49g8gUdp4pClO42c&s$7BccyfT@U$I|_B8tUTTM9k
zC^bU&1jn8nG1adZtJc4mo%#A-&|B{GAEm-_GkwG^yFZY2H+;5buI2%k1#VJlR^lt|
zyM5+KGTqsACgo76@a1WNlX)^^G}a23ZDe_rCKmZ4>zebO^XJn!LyA8J70azND7jlQ
zw_Nk!Z{PVQ5&h0beJ*r+&F@@lELyT$Qe^knlRC-Es+CRtSA}bd-qKh4dv&Vv`;&Sh
z@7Dd+SeH_<cRE*S!z!(Sdn~7(dNCBQx1X;SVD7uDdx`g{4TYP`;;tTvpS44Td#PRE
z4M{$$*&6RwJ}U2y_FvX@vekLVmZ+Xi*CnBI)_BEwO8Kcf)$HZ;X-<BiKc}g4+O#W0
zpLo-McW?T~9Qr(8+~DQ5rPV5p3_Z(d<jC&2?`ITz{F)8#@nv;>Wril@%N-|X&e>g+
zxZogrc!{ibrCsVznK`#E{tWFgQx>~>@mlY~CyTy6Uv>Q{=e!>k3q7(#lh!(I__{Z1
z--79@e?AJ`usv=?Y3G7jQ5QF!v*!ACHE@IT->TEmtNT8h8*NSexJggqfib&M!WW&D
zjn7;jsvTcY89DjRDdQVONA^ZtOJmpQT{)|s#mo9bH`~Kci6*b*-~8@1x^rHAu7k|U
z8`@`Hv|cMZxWrJbapsvx4uwIk2@MlikItw+DWhf0d#~|Qwvw-jv#{jSbYFMw(<k?E
z?|igdto=iUgvGfU^U~PeyWamZzh%*(9j1GGU268!CA?WIdpW+m>dkzmvA#DgDq`90
z&i@U)392cV6f^%>wgi~!KNf4en*8v>QSW^cV%_DnU5Y;L0?bJ_6PSH@=4k#}kRI*(
zcXgXw$jq>=7gc#brEYiWhd2vgO>j3o^r7i)c+mH}mv{0)Htm+$wtL@(=)y(bZ2hcD
zx&LIW-(p=*FjZ3V>I$alz1Cel?gmwA&CkzYow+}rL2*UylZ?l|?{<B)ovx9V@+@Mi
z_WfzT27jh^PX5C4NySENQC)|Z%+(E&FQlvAEQ?ND!YXHdW^&k5txt3Q+rRl6yY0=6
zz9o$Nen<RqWYGL~ranowsa9d-UjIANkyEFZuHJn;Qt;M0y+9GyriOzIHaD3zZ8+^w
zdpdD>N9HCiqo89hVsDl*Z3&$&p3Ab-Gd1fGzezxlOQzteqzhM9B+csbyk4kQDwbVx
z%6I*hug(d^Pmi>0YnxhFtJq?u`|?S*-_C-$Gi+?;<gHB2(4MoPTKw7ZrKxsXqn2O$
z&LaMYi*wi32}{nt-)gw;()X=rIm~8NdL}hQRYzqw9Sre$5ov0ve51>!+cW4(q@P!?
z>`rr^oMY)>A&j<C>g?Y#eRvgZS?fdh&*0rL*C%iJ7w@<ooAflV%ZDC3X#6IlNPFT0
zp4_EoYnb9=q64ni334iThirE!40?F<sDml@=|^JbD^9%C*jIi?W>JF#=MkODO^q|V
zjf@i7cyza>*GcTCn;y1m%4&z(tkbiTRg;?TBz@4h(=+``tmiS-NA52q_e%<Wip#Hl
zwKM<0Z0Q}(roHxb|Lb?|$myHQ`Pm{YvfIq|w%OT#EJ>*g-y_-<fA9k@$GP=Bi`%?+
zDK<!5<|>Kin<&lhZ1eSNywKhM{9cd#7csYOwpH^93O>tex$4J}3mx-hL<0^KY!;MK
z)IaRHH>O~X`X}j%(<zFzU+#I7)q1bF&ni{JDB#fCz4(cw440_1MEr}xvBvA#nY4Iz
zoG^OtusT<vhx_u$nSOTLH}w8~u&QzOLeBWDtDHV6t_%7lt!<dy&U-D0V@^npwszB}
zK67*9ht~^pIJ_Jtopajt*VOYr^Yb(Rn|UYvH~;tk<A0(5>p%Xt{-+<gz9n-0!ab1}
z)|DEVF$>iPIk5Ik&@(pUnbf>WG}u^0v8-xgTFlv+n72D_CEhPIeVldds!yP?>)9Hv
z{>5covoy1ot&j|wDrOpd<;?O`b61tEnzfRrd)CUVwkX4?3s+_>Uo|)S?47J<VaD5M
z?UamTdmd)J{npN_XV#|8nw6Qg%+TV$rSOaYb<DoWw@jAxTCP^imAbSHM66S+Ez1!X
zRN>27wtD8eG^^tHvtJIjW~Hx~yD+V7mSk4iikS=3+>$@8n7gp7ZPv`Jv{f@#rY)N_
zGwa#KZ;uz>I{Bn@&C!^iHOIQ2R#<w!PjZdZIr;kEx&@sfDvwXB+LJNi>EDep4|X0Z
z?tCtlJ~3MFQ_NGH&;1)Ol<)dDMZ)AnlDXLKjkki&{*mCm7xD4fB=c)~HojQ9glCcC
zw7<tAejGbhpQ>4wxNcIE$%$9nXEjFzues>za$udfYcmI<GuMr|%^eB`DvgR<Z~PoR
zn>m=Br9eUs1}aU8LOe4Q)-We4N@yR9;#2BA7{Pn2$lzM@M!6o=#5EkREjx6$UTdDo
zoKTv$hC4Y?Li?~ApHlbX2-#zw46e2J-jU0gAev;%KG8L4jZpGTiPuLW)Q>qChIO5l
zIWjYvubQv>Xr#`$sKRSqo5SX86<SkJd+^@jq{7p_J#3NMAaO4j?-$(LjKsP(N7$Gj
z{U6i#+y1^o7RQO+jvE)md*1!(Kj>?#DB5~OWP<d;*B$x-GI2q#7dW!CE1S#ge0ZX_
zU)e(D;6n~R_X3Ni!xN?Z?i4(6n{o43lb({ro@c+8J3eW<zw_(zq?QK-EXPIt^{?+d
z{$DhBap2C!|K(FZ3Cvq@<)OrfnBM2H*Jn%pm@&&>YhnsB;}jLfowGnG>)GS1<5&Im
z$%7MF&uYmBE&hSlu5U%;+|P=b24C?BG+tV~@RnxQ@>Mfer9smf56{%st8T5#YK!QK
zKI`DU<La5S5AO7=zqRu!PsiJ}X?doXulOVv%(|7CwS48wWspR-a^|rIC0j3F^$RzS
zJzHXW`Kr(JQe%tQw<TMRubc@mc0QX@X<}+DF=t^J+h@14Jf_B1&x9BUpY>r&Up04S
z87PgV8T~I}GGf+mbANUE@6>Nai}&8i(EnbZzBExO?#z+y#7(Bljl*?#jn?WeFFp5q
z&W$r~`k&s}#<Fw9-Tym_KGpAG{5(a1M=$$s#GCoCWwS4)9+cLuQ|!>GOg<~!{cp1~
zb7ibfP4ZJ|@A~OZ3mnD%ACLHV?3X!9gCZl3;D+w-OA1a8irYe7&iJDu<sx!pZqXz*
zLALpC4=n8FU~yKGP&z2U+2*rhTJw#dFPB_dniVB^E+%xaIh#eCcXY9l_;gS};9BsE
z%L+<|CJ34p2?z?e%`sqURn+7OOzhy2-f8<pK}nCNGLeJFIY~n4uz+OSnGG|WJ8o>4
zY3-M1xyB27TF^B&lyb?dK_m`AnZ7$8?ubm3mQUYU@veKPq25{5|BDaak%`onPv2+(
zN~du}i<de=QtGEdP>P&$^pTEO`dW+ZquF_PEjx6?pbF1LeY8$h<-G?I+8-<((0pb2
zrOn6xi}q{BzFX{=`LW*8()~~UE2DL4o;%k4PdSxunwWLyO4~!tQ0--D8k4>j?)oUN
zUHV_&WP#yQp75+2KB2rT%ia8wUrAkFWT-23>Coy>W7o4Hrk^#lmamw*D6MPO%B*FO
zMD{aeYw%USP~*_EA*R8gVk<8>Y~|eOx6mT(;L^I)Go#PeASW=NNYl#_bC#9Oy7dxL
zcI}$=GHaQ!#oV3i%Ro$U$<;PXlIL82wewpZQ{xC{P-(U@O|9AZ>bVf>;IlrEBsY2M
zi3c82vsV=HDy&*~($r*Uo7(jYPo!qWm>k%&C*odqZMrCfRE+1}%n7FdHpV>Nd8T;o
z<_qPPdq4%)pMMg@cV;{~CS$I))9_?(WS&oP^1S+<nfoqtT~L^OcgCAzIp)iDPnB^=
z+hh2&*D~L9sT8l#>)y<KQ}&57^L=WQ-$_rtKjY7_H`}eBrtm3gCv5i0nBbbA!;~y4
zar!_MpOW^$sIEAv6hjF}5w>Ao^NB8lX)PPodR!B2TP=Av>HM4g^u_-Q=C%n2OIj4w
zcq|h+xSRtdCLIzGm6zTb{%Mzhc-tNWmNrELp3cOMo_<9Wo|TCmyt8-ye)g@pFy+51
zdTJ_o(sujSg-k)T^5@v`U}uMdid7st^`9Nvv0?d@M-ndX1r_i5prw>|-WHRNJ3lO%
z4hzckJt$ylSGE8ZWxVSxJ50al?dm`h+Fva2KBW7FQq$r0SJ<v7xE4M7&&BXxL*2Xg
zzxUeQkN@TV%YUqQaw_?%&V5Y4(Jx4RYeLqcE5~Zy{kXm&_Tk6J=k5Q#pBTQX{;$>7
z!ae{0x>|jF__s4kcUH_D>us?o6VL7`D02HYQ7C@;k6$NWe&CItIEUSI-&A)K2Z_i<
z<u$S|qH<om4Ojnn@c+NN99KR(bu)bN{M$q|H^c6vM_No<?RVWa_WJdG=f3~H?*ISa
z@_)XCqQJ8|p@$N7rzURu^Sf@s)4$&>{{1`ixH<MY`{U+A&o_&Q{Wv}Ex81L6hZ`#Y
zg*WZLeXX<h|Gdf9p8a}fpZMe-yWfXy^>v%#AMf7(B>7*=zf9-<|NL%VIeuPJQTX3U
z?Jrm5Cfxu2^>6xKf%$&|*Uhi5>9VR<p7Qi&$C4QTr@Q`lZqzw1J3rT?XxgXPbK5V~
z*L}Vj|Fox{Yf{bhdYdhiv_qz9=&aOVU4Q@kzu#B4hq?7`{1C8QtNF6B>zS_+yRRFw
ze!8+e-E@EbmkqBswqzE5eY5XMl8Qs(`W>bFiv|C>>o@$CoO;7fzW>bE(4SkaRooWX
zmEW#^#{K)w&ywwIk7wpCR6C-RBjnq$=b7aM-FpW<x2IhGoZD?}tMwt6cf(WdjPq+3
z9i6s4>-<`$>`EKUU)PJBzSNwJJ^y)5$mi8_dTgWr`%C=z|5v!mqu^WTKc367`xdRU
zUF@~|m;CE<9ozeozn-iqf0OZaHh0aH`djzv4{@&E?q4RluS)Hr@r}j*FNgh;{<YBc
z!<xzMUGnwX>s|_O5A<+W_Y3(Pro4pxrqyS|XMg{;m;d=cQP^qTih>)9bB=ZyRR*SY
z=v>^tY!g@g%A~53&LW<_*QdXbcThDd@#LvYC_8V`S@HX|<iC{>6VEr_i)E~hn|D%e
z)43yG@BCjsExICgy+QxfcWTp0LQV%MwQ$&JH@;cOI%%0wOM|<E!-}T*lQD~;4o!IF
z^n2X_y?j^kcqg8$K*^5aqf@?F>|dq3%I^)A)SaF7{-V1krx&@3XiQD(IXYjn$4Qs-
z#p9Jn^MdNbLi;v&YHnFDFUCuE>BR~4Dqj0G9dt2q`J%!#DRR|L+Z^A?S|JlAKGCRs
zV036&a?O<UAypA4=ZJ_uwR2kf?zQSy2btv0MTb5HEevnCr<(A9Q$29jzvahS#Eq*&
ze(d6KoG5$xtZ(CWDW%0?KV3~0O_KR|TO>2aYmw0MBPy~|lDkcPwHZZ)&iQVXP*jl<
ztIXz-*~v6j>8P^?Yl`?Dj|!c-yRI3Q`y!bpaQwY+&5tQ>#b*chiA!E3U*cZJ7j3JS
z{pR5{uYHGv?%nnlT<_$(SX_<asFSlmMbbhQmkYmz=Upk<v~7{hBJou2^gW+mdrSH{
zXI3N$?>_VUQ*TkXZThNU<B3^Ql_y;0e!k|Y7T3jvrDDGeQub9oI%p*pzTsoCcG#uh
z{&}k=N}u1*<kdE#ZsCd}SLA0c`_vKr#_{~?8H`$42L=2MO#(yz8=EaTaZz^0W97OK
zCr9Un7lONO4<DGTW7fB-VZ{w=rd_*Fve(yk?&;@sS;=VitDxti@m8mnte>SjXB!&l
zE{l_2C{a9n)2h@W&G`qFMQoS6P&RpAktucF!jdIZVP;|ESvB=_FDG2&Pm)_?k}dIb
z>C2Z*v8ulLo>x|!SfH>&Dngy>MYD%)rB!JByGi-VMY|(U*4eHQ{Z?8L<y&_{OCW&1
ze)IB*jw|Z7QZ>wtUkkf`pBi|e&p|gvj4AYRZR?+>yB4KPoW<|i#+|F8Vqy0%{Q%>#
z)E*0k2TQHRQ>R><(XS`I(`lugzSGW6M+9<@#ffy3M9zxs{dM=^bhEARL)Z>|Sfre#
zoV)vpyUK3<cB_ToT@&7ju3nlQs5nol`u<U)&eKjy*b?$KE?alNVzu1AoskvH&hy&r
zpRW6~H{L12X$GH-qUm?5OLJVK1w)^7Tw2x075HoVhpMzlj!T?><eN+`d^vGVG1^OE
z+5MdtQ$=;}YOBm$%jDzln0R%1)ftbabF1sBgQT84`*mP>_#t<<I-lw(EgrfT76qwq
z@BAe2s?^1#PWkzsNlkT26E$|3+D-WSc(u8l_bH{BN(X)@e-oV~(k;QT-K=qcZ1sa?
z70C}3E`Lsj7P-&p+GNhRG&X3`4A0g;+to)yH~w$iZ6tRml>6gM_fLHMstdUIkA;Xy
zENR$eTW5TJ$?K1*DM|reUpKzvcrocfRZ#0K^T`dXSQ;Nm=&N3v`29f0pLdM1^C~v%
zsO@Z8vh?D~=ToQ5WB&K%e&O%se0@QwnNs|rvnKqW|K-vo@A|uYk9|1KwO8rmk)Zu8
zv!l<<Tw%X+&Fbg80!%9}b#>L(KlAFb7j;nb<||yi@v79}Rf~EJf7E|{+V?!4Yike7
zMxkHMM|&0p#IKl|a`spD>^+N_10N)5RfoJ;>dLt?;<M@`d4ZFwcJ5llp!~+)vM5xS
z!`!3O!J({a*~{q`htBf%hkb9UWLq54c;-jF_mhVLzn4uCQ2cUN{ld>vj_b9q{yMX0
zWyS3smnUEO9w@koy}oqY^~bSaW><=?Y~opdH~#(7)2WMPpL06#DlY$}zH06zUi%Xw
zch$MJddmGbjcuD}D-_wk^_9ECDXI6%MNUjsez<zir^SXhCZ;V+e|33P3b$dr(&rkR
zUjb`g?TJV_zuEunzE2*jUh1rJckYrAxSx`LuHw)Bd(rx*PEGvx+Wm*Ec%Ky8<R=9s
zB^e)1m`+TU4vqeJEhk)3t2H^@V*Q^9jrra`7DU7tGKE&v-tb++p!)LihNCyb(;v*+
zS&+%a^4!VuiI|w^!P8M|jxhzbX+QcO_;B8z-)fE)MkgmF{SBFV@LGyOWcv(-8~)SY
zX{S7FJ#uY=k7QS!;V#z4o{B%b>VKD{Eb|Nrvaxw>uJWk3_ISmmu0?ta+$<;aYlMeu
z?7b+M+om{S+E&rRHMi{Z3q59<cx_m6_oLG4OXk7_Q%*@;clFnI@iFy0#&Kb;(Hxb$
zkPe$&fe~wu=f0lgW4<T&q@dT7nG3{jAG*jL=c<`L(==#u;)MU+QK4au)6UINwN+Kh
z^H{~D8lJ|#SA6NWX%qIGWO(xB-o}|K`%Y|Ga@Tp+x~fSlOQm?*LWL%63jMTsrLy(v
zR^7GcMY1)<+0_w#Qo*iY+{2x=PMDRPB+V-+J0n8vw&t=MBJ9)ZcIGU<lOcNZ$dfJ6
z$AqnSY*JWqPB!WCjk%96nyxVqiQF1-qtWT%q_<mGS9;cjZ52E=xlu5p>-_VRPIYWG
z7q+%-^FE{EFux=wYW<=sDSv-S9BJzjyrN+I!p$bDIQ!>g-D__jXU<H@z7cjf*(@{5
zCq)0HE~j~hOKXXT<C;x#GmfO%rhZ@Qd^?G=<q7B0$E%FyJ*qwVqHA8otOR4#xEY-n
z1)bg=t_jk&eVwW!b4)d4>-QZguaX0&DE>Y<W4Ta^;rsc|xhHBVvpKLme7b4=7mkVP
zk52{eNOhRWp&oeMu&StY|I(5&p_GYF^^SIA72Nr$DV2NY-X;anM@%UUzk02Xiius@
zX{Fq1*Q4=Q<5zpt;|s+%UmNRu*s?e8@0O0BU6t2PbV_$jo^<M(Yn|u<&gt8J<}B^6
zRVuwKazXK6mG7Gchhq4ORaI8I#h+?B<u~(#@unKznf_v@`X=A@i2Td2Swd3SO*AWF
zf2yry0%zvuuZt3O+OJGaUH;bZo1a8L##Pb3|5wjFoBeLjuAC>oOV?;T36by?Up<@Y
z$gDHl@BK8Nd~mhSXY;lHtMg5B9_C0gs<HHap3eES>Z9_%uC<>v_x`*-U-y-`jgN4$
ztZh$*W%%uI3ugiIFDd?Y51;3MdjBWY&hvEew;Aj8SO1fY^090`mbCG|z4_IEJC3^>
z&#YtjWj2T~Xnf{&ROn>C#O&wqTF-vpS-EF*L|^>>O`loMM)9nQW=R&bJN)lokk%u<
zWaUdyZK=PmF0j9{>Dc{>nV%1fZcOo-5_7nus_v)TI<MmaSE?`W{a?lW@K?#I0=sFG
zSI=FPV7%;uaqd11?!Xh8YH1T+J`rNon*VANQ*2?zE%jKr)T!NJRu5}F6ukSCmc7DN
zuIcf5zRsACBj>7f)ceEBw7)Whoc7ozV?V`Uj?JeHb_!m-?fVb;Rmgd0h4cUWEcMfR
z-o6Z8yENhRpPoEkbN}m}@;iSHX06*WJ7%S8^$cIVc{fGBD#Z$|=+Im1o8BTGa#%L2
zAXxm($A6~J-)po8%dWU(9p(2k{f^vePm4dsTXOf--}DRgdvMc1E-cevvApTt%x{@V
zU$~ua75`=CZ?!9QovCyGcTvAwh0n+KLaEixE1os|sD9j){<yA3J$_dP|JKdf^>x1%
zURT~0aZh>Lj*W-h_x#&d)3aM&S?zy#-R>7BLO8|6(y#CoPyCzb`mA=d-|b{8%h|uD
zOyE3f|L)Hm&D|%r9-CiobUymLZm!Dvt*?}i&*(n*VTLc?GrQxjLPXkbTxXM&N&PkH
z;S+}KpG*p4Lo#)i+@5rEiCwv*Z$s)6@w45lHYluY2<2P;IJV8@B9BnjgOz7nUwCI`
z3e}%4(v|R)6TbGu<iU3LE^mgUW$bh0v(A@9`t}8TXy(_fUbbv&$EPRZd^H`>orj`L
zPyLzGyHWg@%~_dOCNnpdZhYkVV#a>m+;`J9iXYJ2#<Xp6*cuz(@I3yBJMVmwj1geX
z-z3nbeo8CyqGVL0f1~C^UO%}_W+^rk9dv>=9)ER!M@M+#3`zO@lF4gV*j{3vXr<|3
z8qO3gE+aDS>gC?T<O?2dY?jwtyQW79RVvP($I&rmMvu$^zn1t@n++Pa%$XAPPJiWp
zResgmE)|g~n%b9@T%Kwvh#e9#3AcP=5^a63;l9u3v`Nz&ziFQhv`O3J#Cc{vo6ZL|
z&NI=IqJ%zo@fdC7Tw!oI<ivuIOCOfDo?z+W`?uHae2S&4x7Lznn>rtrIo0oYQsz`_
z80o)i<CPPlf|C;-RVIC2ylnHU;%wt9>!ux>eA*|R@7;@2R|Vc(idC}HxbZrBK2NA_
z&LZ`_E5E3ShRuJp`S4=BOIK1nmvB24RiA1qTAAHnX=UObawq1|muU;cn?9Ia>fvyk
zCVY>_C@#5SqFm1mx3}@GZXp_sQ*A_}jm}wz3)HOhD11B7#WZF08fNj(IOCNW!U<J7
zBR&TO=0yhNRP^&qF467zA6nEhuaNVn*V-ah_eG~qwgerY_uG44*vlD7R_R(_4HgzD
zn+U#By*xk6>BzAfZHXKA*K{sPICbFu{Kk;dX!YNno>wPKeRRpl^Rl>E$cHOTf27`|
z9y_<k)MJ0qk3)iwrpb3lFnH`yYrimUyJEw>J(o5tkXgboZMWREi#wn5-an+XMIl#c
zndh>K$$S1dId@F#*mqq2ToUg|fwGwe`&0h4Uu#e{zZSD+r)vECzd?p`7k;@XB%ZN;
z>Vk&+Yw>Q3xwAAB;}`D;xN0MG?3>MuC)pO?cgZN7;ZbLp5>ffh|C`%O<y-NwVO9rP
zPPPd>QfAiJb!jTkL%-07D9@7CSMFTmy(`T+GXqToHcVe8UbE-MFXh#D!you^T-dZL
zd|t7WY?Vf;h{0c*`}a4N$bYh1R@^^%@vE=l5`T=h+9&qhjcO>n7$Y*Rx^Bjy%6+P5
z&mZs6TA0J0{Y3j{Xy(U#IXsO~w&|gv&-kWCF6)(fU3b3bPg`|--|hT&p+!!HQi3x-
zihdGo+qX+}(Z`*&hnZzoI==gHR{6rUC()n&i7pUu{&r{YiN8N*1ZrfPc1ekA&2~*m
z7t9Hfb6(ZbeDP3^mUq8KVfoAFQ?7`Xg+BTgxX`yPBvSLBcSd8y^TT|6cVDlc8uayt
z)QaPiv^wsd_!uanC-lR1*L5E6RT=f)43=<72lUReTz6EX{pn{N+oBk8j}WaZ0_)#S
z{8qlFYPy#Am1oP}{oL$3eQB3{+mx(Xt^B@W*;hO?Z~QO6Q<-3Ue{Oov$pu1w$K}+f
zY&)`l+v{b8{GY!YE;u#)R^r*sR}G>9RJ<zY75%%uVSbWF){nVgoVojbf2I{SE$f(C
z&}@G#_-ESLC3R1OK3rJjm6K5)dnb0Y%xzBFnzoN2hmJoFoVTbZ^55T`m7J}w4d-6^
z__qILh=@^U;F;+&EAMixij4ZZ;qm2)JpJ+y8U-ge%$1nJZY5#(I?2BCm89$JIbBk(
z&z^g5Nu=}l@p7Yiay~!bE$4`hm76)^v-4U1=bX!SbY+}fwIFiKYc3<^HQgq8PrG|&
z{EgT<(a?@z%SVa8--`0eTJzn^MD`#3Vc~z_X|>jAwwfM|z%aSDs%JDjrzmNA#r#XK
zto<)|z99a~&G5({onNkBeyOIJs;Kn-*j{IkdoML7B<#C-dcx#+nvZ-ZUsDwG2xIGA
zb?jL4i`QP$_B`F)x@XE$-%FPFf<$cg_w5uExNvLknYp4;8LK_>!Zg?vE!nRf-+Qg$
z&lgvLG6N0gDmTFka%EiS<~qzt`(`}FcCE_OvfndQRy2fg3r4xBZk({Nd|IW>568t<
z-p*Ar&<^x0?=v|RJ6R*Lj@A6jai=c7io{Uq872wbnpqP*Cs`z{a(%WtLtp6Kb(vWf
zg=b%81RQ3Zks!2;ce=O7<ps7U#Ahwmzc5o$+B$*b$_Br#N&A@C+|MO(Zf6sTtUUP1
zGjpEL%<qb^n-8pSdHwdjk7x9fb#aCJ|K?q}uVJOR*yDA~?s=_K_6FPYTT1WRqO&mB
zYU}UaYYVsorRS_ZaiCr6=b7hbpKKQ7?A?64_Db>2(>teBzxljv*ORlo^_jUvjeYOy
zTC`uk^w@A`&HAH#M?@J?V&5PA+%@z4p_ZP-8ZMb!%RD4}cRoH5Yf}_|@$#Cu7S(Mk
z`70_enX_e^w7&1y8T&}1apEPvym=y-SFTR;=ruJ8o^)b&NW_AN&HRja!d~rv;bE2S
z%(g!1o?`3D1v$k}RQ7vHu86ct+Q{9)q3fAsv6i{>;mjZZr{*8i<m>O?{LPj7Md*Hi
zq`<8|R|PLNX2&YCKaxD)Z5t@XR-M}^_vk9e;)0XmVavSt-TRTM8vnj2P5tMs4?AbN
za3<aja&Od({q;hbPp{QQTu6S(hV7>&9n$-_i)ry<<Nkboi%E)cxvHmDb}T&O8q8#O
za*yJM538J3NNNbb=Km-<@#1Hhdz;$S&S;&TUZQI=cjmg-Z3WHjsTyA;UUh807N@&B
z+HQ`|hB^0&I@{MVb$__CWpCVi|6_-^*SDpAKgzC?t(KXsaeC2e@96zccO=Pkt@b*d
zzUFLq9-FPl<flw8&cD^qTW(?d=lDU{!;9BhKb@z=7~3>!*%RA~;!|q^9X3A}U+>-d
zc}1~;jAl((zvr)vxAB`6o@4i)9Q|Yc!<vren^f!smEtbGT6FHCXmCMY;t6esn7oq*
zW<T1uJ>-izpSL-8)ywcD#~&=Q*}m-7>G)6oqRoS!@3fw_)OLIS9M;6jtNCJCe<Dg2
zoT&b|BYTO6UC<xzbr)x>Gmx>DbDd?JpK0@5_{YB<4Syly<wqMj;y(t~)G%E5_=<I{
z?fd^bEe!%X4b68xo#)}%Th%YpsWt1X&bmp9KfNgtEUeRh{z~}#is!EaJ!9%Fx?9<l
z=g&B)yicF2Lu2ym`NpqVy??w+IeU<wf68;73br-t<SHWKvh>e&3odw--YX|!nLj_l
zRZlAM#+4o36Zq17Odr;4Kc=9ysIKXk%!(bY%HAIx%YR<0Uv_Stx83Vm$G0!DPg;|p
zb2ry_opsMvt+@#<4Bk`gtPZ=by&iEmM);k4mG9-4iC?nj$shT=KD%-Ito<w>^?zMB
z+MTV-eqzyz)m=umCq>w})&ES_-6atHAbIo4M5eEO_Mc{-|98sMOMq|ni=KArPlioP
zH?BSv$oiI_dmR@?9oNp6M>$WvYW^yD<fz@}QvYBsmb3pp1!PWLwU_O~!Jg_z7H)oW
zJr=buCVKBqn&bNLaF_oQ_qbf!VuSYMvpgP(g!sMvso^hmVn@omm#d4cqpT0gROUrB
zg-Y)FzDJ31`<JgD@=Yg2+U!1cWiPA7xrs&3AL^dnDYvwRQ~i1If%ns&u84ICdv$NJ
zyl(@y=Jq(ziN7QYM7_2DcRB5Gywo?T+CSgpclz`y<@F~;S8J@&bL(32diRaHX+QJy
zZ>EXLEN;m?manXmw@1LnQ|ypQaeli&@7w*SIGpaeH-vaxX`7l9b)ijVN8eBRt<7J<
z`4?;5JF|Iz+^!i9b)6gzYQ+8s<vqI7J@j0KtLPh@CE3jXigzis6z^Rm@ne(99bLJ^
z*mq{y^R{=-n#lT*^OE@EpfAm>b8-)ey9R&tT%VZAxzkVbiPk$0-u#H8HM)uOmhK7g
zl-Sn3z_4wHD${Y5+{^lckIbgL(3nvypZzd-(hgGrfyoDIS1sAP;e5pvrJIX&K7Y(9
z*SmAi=$6VI>m=cIy@%gTN?ve6Ywca{t*o1sAAZTS{Jxd>-PeDHotIC1P$}Kh)nh8!
zwpH0(N99!mU$W53BP|itx(b|EJ8ZdLuQ_zDQ%A>ACHT|i!mkGBx15r{7wr+6WMX$M
zc#6yCyAfVR{Y!K9Cvj~)etpBbSLR<;CJHRc<>5{HtEPWXB}jj9zhX$Mendxzp`vQQ
z<b&oq=1V{PX$`2A>rm*I%E-4k6|^jieR2Y~qfwOCbb%GWvTOc7O#Aiz;Nq+Qp8K13
zthJk#F1*~KRD6@p78c)%+WGFgT|+lT28rGOwB9p+!gJm4I_mG%IF-$M{qFVpe&5m*
zj{E85LhgnkSKMb#zS0)+Xwog&y&qnBTx9+^HCZO4X~NZ$6aH45c&*?oG3(>zBW~)(
z0cU<&ui3lSwN|EaZ*BGz{pnlTpIl3QyU_jgzUp0<j$SXz+;6_k&Ew4njm#85Wq*0q
z_09Hg6xZ#pRa6Vxb1b7|(~@12HZ_+!iX}f*IGV|-t-QUggH8PL217d~leV=Fyu^N%
zEWR(%?j3eLTJw(f9{b2`rL0;zQc8snuU(=3X>+}eOQ}(Ltk%bdWPwdlg-NMrdGsw^
zoi}syhj*TJRdvjJcvQ%3VwQJSs(_-C75niWr!zWkDcn-qnmkn@yu$aZUd=|uIfq5-
zx+knKG7f#bq*u>oiqJVj)s#)4KOaoX+Wns^e^$!;jW0ufN^-XKl}j$WyY^wn*=MKo
zB=!|rd|+gK@Tiyj<2l>vzkfwD-QEPvc*Ae_s=NDx=7L|WmsSYgU23Z@!EinNy?Kpp
z{L!E5?f>1n{(1GM(7gVRcS5nTq3Qc)H0u`UWJd2%mgi7>6nN=vThWIE+4zY)sZZU+
zy97@j5Z2(l`(2GYv8Cc}_wg?(hj-pwHNW%djh2g-9(8uT5m9xh=2+@)HOIr^{{+QF
z%$#~Q-{;>t;N7#&c447^k-jgh=;t&8<z27Ej>!D8(%q)P67lHcdydt*=jYDPYIO|U
zW3Kh#j@8!}yWHy8p1V%|IB{at=eN5jyZ)(NHEUA7_NQ4#!*8w?lI#koNNag>&q37x
zr%DEkYS8=pDhzM?&$8DpJaJj1FS|?NzJbb{jlc6ERE2NUc*)%{Km2upT6xKV1L0qt
zT9n-lkM|tkpT6(S^S@@kTx%z%y}o)o`Lto}Q@`Y`HjN&+Uk@s2#Wr7z`Zw`@&9`vd
z{}sDFY3!_j`#ATL(j(cg-CdK<|M>diWN?ASsePIL@_~OoOxu+-(UDE*X7bv*bH4VZ
z=6|uh^(ubvk974Kp^YzC&m^3Euu467ZcI?1&4ajvQ<JryPV1=RI3dL;c)>08&;P8<
z7c0B7PPsp2;g);+IcVR*=IZqCKYr&ZO<s3mVwvpxWpm~K{0WE?zxsLk?>`sL@&7Ho
zXi;xv{qaV{^W~ex11~=KvvH5z)$7ZD*Ctl~y%9hE*z3#L9%th28dld<eLZ&S_o49e
zH+3N9FEG>8;@_*@{p`o%?|!`g;Kyq7e{wSQrN`r&AFsdnqq^YKYU`(_7EiCv{v<c&
z(<{G9Ucbt(%l5D@-?J+?t~oe<@5=WFSFV3|<@$pw;rXws3tp|Z{%UIR_3G?ja&vxP
zo%`F=;`?gz@6`qG!pq-Xe{g60y*uv@?u@^?Grsw9+}_Oh2Q$~d+j9NEmhk*r)dja!
zTfa56czbpBH@P|gUisDY`qh73wx50Z{#}>jn=i-jefj?2%k}TRTz}9Lo`0;m;Mi*G
z$EFsKug?A`H|OK4a}~VjD!x9`ZF{TQ{#Lc)t!n36-LAK)-EUQU-m3P#)$Mz$+W%H{
z!e7-1e|5p+AZE}1|1JKM!;<}PfBb%YyuIj8Z2$iEpPR~RUL3#w_(%0T@f)x2JwN+)
z&(5jWU7!Db`=(*8?1q24Uc70ZJNLt8{(o`}n}zGj4xP9AkuC7&P0M_+pj1au*`hgf
zHP&zc0Ar@hpUPG^b+famY|*^h73&*Uq^rNm)|&%k-t4UUws0TYBRlKD`Ez%ix4W~x
zaYwxIr#DAxzdfwuJyKit$j<uWIlC>_9ky&1e*4D7`uBx<rysp>v4)C%*)06`jf?&7
z3-Qbs&-;ORTg_n1$8Qc-@E)<SHazFovzh;~slekmM=Cr!z|7BYj@Vcmp0_)5-Qmn;
zVG!@PK|Hfze9@~nN0Kkjoge4?F{0#o|I@E8RoAThUHi~GF81%&m#Q1qZNKqH(LBEC
zv#w=*$GPoqYPipD|MExC?*6CGst#{e9pCCYy;XI7tLpMr)%C5e+gnxlx2hg*RXyM8
zdc9Tkeyi%UcF*>y-yiIGvT`rWs=X?!_PVaxo3d(e%c{LgR_#5yYVVU(ds$ZRRaw2)
zb@kqq)q7i3?_IKb@6pwJpRC@?vSzQ!n!T=T?!J%xZfc>gnVoN#t#6sF@0hLcnVlb)
ztsj}KpO~$mnVnyltzVg~-<Yl6nVml|TYqM@{=#himD%|lv-Njo>mSViyLQSR_UEeA
zZ&hpFs@A^Mt$V9l|5ml(t!m?2-KMvy&2Lp(-m12~#g@u-mu&niJmdX;cfSocCMg~L
zQD0*JU)awe_r{L3O^a4b6>p!@lY6mTS(#za(*Nh?Rp$Jic2hepdjI}yb@9>r?^e{`
zE}!Zi5ncaxX4~)jrL$MP4LbclO!<F&jgv*${!<^MIh*!}ud!Sj@#A&9*RS8a)00(P
zce>mSpBA!FWy<Pr(wTluK|(jP^egqJJTSTO@5=E9|K`0n7n<Uhbm@a}@Md)<iN&*=
zE!>qFzZAQsi1hj_(ozuY6jw^j*z@#+z^Y69iq?-DLs%c}R5Pk`TXo*!<H{rRK0f%U
zF_FtsNWuNyV~@%;NvwhY9_&>4AvE73Rfe-?>UN7y=Un1GJ{L+=?Q8FJIWn={;u4?x
zu7@X#WKOm!FE4l^^v{Lwp@*!DX0vi=K?+CTkpo_S>$eL@HY`p%RD4pU@YUDeJ^LnJ
zFjtbXo4DV6(>J$$aZ9bjClzn{bnNyXkxoI*im8X^Jb8L*OSR_AoR~8%9qhq=MNgH&
z#fqNFhd(NODy@FL;!H<>;3U`28pa>AzTf>CdZ<8^>wkZgV*Pi0p*gmNbN_Ff>R87V
z%y4%8xk?SQ|0~x1->mL(=ViT<{hR&CHem*iH67anI&_+T+~{DK_4jMqzlASq7A4<0
zZ{_qqIr+<Z|BR|n%h=ybiA>sD{?te5!rzk>-*_UXPrT^5^F`IUb!od`OA6&y71gZC
zPHCIfRCjX1N4*MB`Foc?y8ZhWxh&75bpFcUK6jbhPF%5lwxICdcUQNgr!B0GAKF;7
z$8y!yj#pR2ll^COyBYf~zPRGz3Q;|?sI9;M_wM{u|D#1=f$dALsOq&lPDI_DQu_4%
zx+Ckdt<yf$ho-Q8eY2;0`~LmQzm~iIuL#nat<w`&XD(*@H}b}-`mzWA+5I|f=Jd8m
z^`6P=Em7&6QhwfWn)S+Qk5<@7c&(F+zw-a`<E0_bxwc){SiC2Gy?#ewufW?hE|G5i
zcdzr!KmVU${%N`X?P|Ss#Y&5x_p@{TN)WCL@sIzv=8#6|h5pF(Tehkd#VbEQZn-V>
z{jU9Qw@niN%iQ>(KI2ByuixhHf5`h7o<9D6wxRK|1%HEWKm9+$Iq~b8>$_J!-EX&7
zS@`n*JG;9QcARidEj}H;|6f?ypDbODb@qP$G_o81TYTK^5;X1IeD$dRwU5@{j{2>x
z{x|P0zxv;W@%#OM{dM@i@zKls^90^Jd6vPwB6?%B$zGq__cQl+>U@4~QLq23{#x_O
zx7nMjW8cJyf8W33?$NjFyz2F~IeRI2x|fu=xwotA4Gmr}tFP*S{T_F}lHZ#-{@-4H
z;av8cBmZS*howBv>)QBp`>VhhgVIH3>ot^@*h<_w!v9fXrJtW@o`~PS{OyaEPMmZ@
zZ<+A=jq(4FZvRmKj3qPnl;OwiyI=kfi2t|TzaztKUL>dIzi{){|KD3I%;ZXlUg~-E
z|DG429LhoNuZ)=g7#<LJHZ&BGbe?!|-ON^NsmuqLg_5QBS$zN1T=ent{KP|<p0dFf
zE57@7DwMjoC{5y>`Q}85yIQ-xQ~aNMzy7hub(#G*#4%~LX92^G#%5`Y9gUkOS|+qG
zOIswgs7qYTd0S!itUl(#js+K+-T0%vF^HUKXylPO!O&#%bRi?FWWa>4Zx}NsFtSTl
zOkm`Y?3lpFDLG+6BWI%8G|`E_>x&QmneI1X?l1mtyZ$c`-f;5x{TCwtx6NHR)pW~-
zW7noXKJ&5uPQd^5<v-*#9=yKuscsI(KK{j<_uVU*fB%2W+JF1YoIF}qoprd_r^tP8
z;pvnGe|YAamYluN>HK6;onnM)+Ah6HriuFATW8BY$*_G_&7Bu;aQ(D}v42-zdG|HE
z{E_O8bG;&L^|Jrtoaa2+@JFi8FzoO#4IW$PIa-pEOY4;v*u<4_*Y{-w{gXK1nDo3@
zrCsIi*4cl*CPkfn^1Jft9px`BPwW<(|21VR+p}odi9hSUWGb%fKiE-HZRl0oyM}G)
zTgQ`1;Wi&$-CR5G<<4bp8L!^{+uW(1@jU(H%l$Lw-r9P6zun8ygPXtq*?QmQ#W%$*
zk6PVT!ya9qpOA1p|8G?P3Ayr~S0PzD*L{2+B=Ss7L{r*i;oJ?Ywk+JJ7-n|r*Soz7
zbRXqAOWy3)T$*ykpxkHD&5UgS+D0{ViQ8)pZpUZ|dA!Y-eB{aWO@<{4ug4nB-S7Ty
z=B`@*mnWnirT_Q!l>SvMweQ}fvVUD_0y{HJ^k%KPbw4t9u2bmTjK>R24Lr)sdc42A
zU1Ih};@ao<=_W6P*kU*D%f93uy1y)Y@~%+U#-|c9E9Y0o&3fa!*y?wj+bPrKmv=ua
zPCxrIyKJ@7;iWYt_Sb)BuVInzej&DL>8GI8zhqAy)e|~+{QZiY^FI6PZmd1DEdSrV
z(=j`)>xTNQRp^_sN$UD+!|&nK?XUT7EN?z_PvK4_yOfL1$!)u)p7m8XezdrF`l;f(
zCim)l6uSL{f9YD}>{<GP(df_Lg8yt1R+Dw&_L#k%aKpc;IV%3N$9u!c5wCtvxNIB$
zIOcwK!EB@8lu3V%ad#@&Xnl{HY7=}mel_<eVHxciOZ=oBiFqzpQumCU{^V3>MS1t-
zSzaGAVxs-F96OQ~nDZp4cYgPr$sbqenO&W)Q+c4K>@7p=<+)zh&hMR~x-`CKHG9Y1
zOQCCwyA1E`T3qbj`(aOfzu|8=(T8`vxWxLm{gd|2bK|PpSHCYfapUxWr4tpqX1+4x
zQA<#~$}!XLQTNH%7@^>d-#0j?u<hO0nR4~LyzU&Ka~kP)U%Nc-o6DxR?|8Lk$zc!W
zmHM|XX&=cxk$(48w+-+4SWR`6<&BrJpB)YSq>%nDH0)T&(nQUq3-2adzMM7Z%Fi<m
zXLp|adw+voNXg#odyl>>Tvb#iqWYJgQ#-miWA&`m_=P1MlU%HB?aF$~B;@Axw%+q?
zWuNox*B9(8LbS|<EMqiJT@l@uJ&~V%<*hw$KgXD>+B$`M`%7*8apOAMYLn&Lxn$PN
zzM!{P#_ycy-lvrx4cE^peDmt}m6OeaFL_q3xBj=-e9@r`CC87P;;YYU|1TuG@VSY~
z=~I&*y}Ps|{AKZ}$QgCrFL&tedN<u@VcW4^dKGi*{V%jGbG1`1zZz?o{cL^S!o0KH
zi|t<D4KVw%{^QRjD+01pWj{yGJ3pn8|L%J6EUzUlFV`ejElz%X^Zoi&r{dS_`{--?
zL0EHverU-x*QxV<R&`wn+Rf6QcV*{i*ONDIUgx(m(n~X`<Jom~PsH6W8|~HA#s703
z|M-22`A?nj_4VdoI;ZGqZBA+^kDd9=xv9FwmYvz`3tL-Aapq%PuHMK4%)i3+bgocZ
z8zs8X<y`iDJLf5FA{%R8Pte#^`sRuL-}ze-l&vp+4b(0Dn)vFn^YWhj(sx&uuarCM
zcl^|@nQe!XKFD|ek6E?(+1_e%#zT64o(A{-*SYo8V3)<Vh~_^ZVvMYgpDKKJ@WO)o
z>(3qWOn?66)3SSBy?uO3)x)m;`uDYO{e82I`=ou=M_dnnzc#mXk*bwx$FF%gADg3G
zHwvYFJ7F6BY~y^pwMIR=QnFslf301;HzZ{L*<0U#eSFXVdAg>!<ee`}TNUyy{p|Hj
zGkzsxYb(ug)cJ}?t$*Xq^c8=Sl*_-xpT8$*e=6mo$ethPG6J3(RGrwo@$IDmpE*m-
zuhiD&-F)hE%x&Gq!p}R`xLpZYw?y)|f7x-jSscOA`KrHD<YN{b5LbP&=6TQSKLyA6
zvzDyi|F!S^uU}4{=ccFzmajBnD!P32Y`@U-M0viM4zKmf=iPas^!!dqU}Vv%us>D%
zs!D`eb-MO9m96N~-g@6H{`}_uVg-U$E;3&uQx@2~+?R1PgGXC^)5r3c!i68t9-U~s
z>$HKy+=ud)vL0E4i(g;CcFo|cjM1Ny%ff{@W^{O0`ds}hw!b#$dG!0$Kjan^-4IyT
z>$&=op~}48v|mz|XBRBIKW}Q_7pqU(ZC5@{dMbJ6^F*KSLPswaqbsSgn#sC;tkx?p
zteN2ucX7GTLjU=y(-*y(!?f~Ig=aPI{~s=aie@Z1YOAhp@6zA0?jLImqf3Kg-E%|T
zTf09T-O6*hk5T2j=l<!5KZLvvX$IF%pQ&`FXh~0;mfjh2&1H{wa}-1|eYn5i@Y!uk
z%Y`1b&3pUcs4T}M#(Q-sEq4=|U$HLeRqe1WT4JUnS)P^8ZK*f)tInGI1{1zhivJIM
zJ{0$JWn_!a4Exp}h0V-gBwqchILw@TIG(Az=5fYcqql84CyH{g-S&FdEgmpsLZ|&#
z=Ut*vZ)`&U?h-Y0_x<vz>VosEX<s)i{>x-(=i9c6=gR%RFN{9y4&Vq`^3Q15(uHCI
zhp*iFf2?J4uCc+CifZrR@6M55LUaPB>@D15nWt*p<Pvo8Wc}v-X73{w>HTm1T`e%B
zG*<MQ%DZQiBIFZ}{XE88|L-?@%JiED?!Q~QswF`}`)16Wd+bvB9q~_fZ|nX)_-?1b
zP22nS3iJOud^^7SGTWMb&Q;P1)oDI+qi?*>ZFb6A^FnzB%YR4Vkjn>J`OZ(ttyu9_
zZjJ^+xn_6z8_Vl4!Cu8@)?Js8W~%*Fb9#?rphZh4Z~fm@hPNdCyDtcxHJR0VX~Og+
zrkpbZC+ICziaq64(<1gv?5DXL%fhG=F*C1hysV!bvBUSnCb1_0sc+e5uI9dRRj}zi
ztB>I{gBwNjXZ2@iePLoSc*6hU>2uZ_rL2NM26En!?tkO>ckeK8Te^1l-=81XcMA12
z7ADS$m9P{KllZoG%Gcy|CvxWhxqI=+@sD>s?p9RuGj%(y^{tIKzOy8C>IuP=A0AEV
z4|i_ST<e}Gz0l<S%_Gl5EFKFt>rT8G!sl-FMnOAoO2cNud(1+=^qMTro-3Zk_QE<}
zD8OHc**@50YS&M1Cf`in`I-7BFZ|`zKHs}FJ9zRdgDn&JR*E`Lvg68+UBtd`+LQev
zdFxHX4|1%1lN!1{SvD*yFYZCo)(@5M3!{VXR9CVre14qM>E5yT>rSNQ2&qn6e{W`w
z(W$L^yH;)VvWU;_o~CPmJ*$^9<GNiRGc(7Jw15lc4|(_L%N-Bcwr`ull+E9NEIee+
zb^F>bldl_k)yy6!pOgE`_SW3w_lh?4v!<;6C(3J5+Lg6x>t3v=uQ?v<^}=S)6ImgD
zv9+cN8Mk+JX$eX*ykZc_UbRJ}Yw4NamT}f5a^}gGXSL6ctxkKRetV)?YiizHw+TB0
z)vhh+UbG|LDe?CHj=Ow;^;N35kL?qAc}`u~?c$Jk=i~3%y*V<ev(5?Z{~5QFNhics
z<D;?fzG;tHnzn>8UaT(NCy{bHT=RW4YnNcNLfq|Z;#&?XCDi@e@??voMHCy~zQ);U
zGR#G9x9(80)lQl7_vX6m{SxIlnKz|Z=ksood(trfefpgDGTWcD-gzAqSDo&6>a_8;
zX$I@RF1JZe4BNc*`d-KAWy<Tne?RggbmC5#H=S$MPN#k}*Zls};HTngp5=37->LUL
zzBYq(#<D9%RtGFKTV;`7qd39d;tt0Vfg8)TZrs}U{MO3&{tdf|4Mo@HwSL)jU*hiP
z_3>wZ|D5%G{^Rd=K0Z43mH*A;!$$<9|C$Fr-a9cy`hD%(mhAg8z8CF#dh3Mc4>i%h
zW-jm6+MYfk7xB=Z;njl0|8A8Wle?@VAbZL$wJ7Yya`Tu|(w|b(+_(P`YTsJgXYoqk
z=I*<ic`_5blCtg{@_RJn%KWzv#TvhTDT}VGRg6-<{9<{cME)8b*CdaN^0z<DZSnS#
z2zM59yq@^tx}u~_iX+dQ%fGG^{^nb$%{={MbY<mNO`}Tt$FFbe>Mp!6b<1C-buVu1
zvSPkHRqVm?AN&DIvkqtK?9F(l>Z_Zz>6=6EmOAO&Z1<TAsn@4jPwer2o;SlyX~O=u
zQLm=;{n+^Rne}CLeV@BJGdEwlm}Z;rp)S2|vG?9tyMre(KP`I8T_t?yQeX7l{cV|_
zcH~&^XYIW0J?DSn-oI7TH~DfsWBqvhitqlQn#MVCtl3K!giI+|5;42l)9UQcGKK7w
zkG>z9F1G8Si0=l+YKG#yD@?u!%!rzE^@{pL?h7ojv&(Pn4^Fsq<(;{ssn8$&0xtD=
zPFa-~cJBLqf9I_$U;7&t|K-^@ZP~V+oeRUaSTI^RU!BQN$#U`i#!MUEyW%J67OOul
z{q|ay)qa`hu~^6QPp{8P1@5YTbG5DBY8r>gB#U2veV0Ya@P>ca4PH>Xq@rrJ`uz8M
z_wU^Mq~NlqZ`EXtH=GmhuIS1A$PnV-WbPZPEm^S9ZSmxiTUWccZhcTa-B;?4vZ~0X
ztCBUzkFVP;oO-YF%HhZVm)9?h`t!l%@1N`a#gA%2g*yN4yS44Vwnp?+FYfLBT+bWy
zMXwj^`N(<SUD~|hz}|_j->ehPatVcRPyaQe(Eq-j@792;>MvDw*G8G%X4(Cc=O#Bd
z*U8)8y1g%C<TZ3^omIP66#6Xo@4KH@&ZmAn)N#voZvRf>WZvVgrt+fm9CeQe<Y_h9
z$?G)MtgG*A`SWuAgav{QMfSDvX4}s`+>sG|Jc8AsBr;HbYH*3n*Q86{%q)op3q;$)
z9{h<jwUbX=`Qgo_={yc%r%Jvp$?^W7?D;goCsXTm!bGi`A%%xMbT12A=DgjoykgtF
z-p>L3zNaS76OVFzar$C?PtpC@%MO;zi=T&;|6g!~E9ab^bGu38)!E$Fq__STlFzuU
zd4ucxR@*P_V(;ri!vB{kweZS1&)xU)@X<}tdmUw1YBLQKUVS~eV99(fzX|Cvi;ZTw
z319uoQCvRvzo5e<mWXdBOBOLznOJNP{=g7C!Ry<{N4^&ErxxnBu%_tl%G6K(zj(WK
z<a6QI*57Y@^7Z@hYuyW*yJvSsUkX}Nlv6W*{>#?Vd0%hwFzpSw${fZoao}~D&9}P;
z?Hm=o%kuLMpZxQ;W_@{t*^~YES6^xI_b}TZHeE*7xR0&oWt0A5Z)d*U4a#4XwM}L;
z*6#c`S$TsNckw#abf@z6%m>o@FU@72zq9e;hQCX)f`hHnxbL{s`b~PTCVNEN^NP>E
zi0KDk@9A^2J=0ht>9M`B=%3A|udN&Ay0m<E%C~=DxjkOJd+x_KZr9WFcoGje&Oh+l
z>%{RR{nn-5CP=?r+*Mcp=8W8w_j?Mad|lq5=s9mcm+sf;x^vI1<+M*ab>&@@i~L2~
zY3e_Iu48udwdFYKa+GoB)2EX5B78q@Hm{Vs`}L~(#E5^bbwcLXb{w&cRVWWv>wP}k
z?ArX;z`M>@uAg7QE%(Bd@yqi&eXloq1@Oi{uif>mHe5LN(vJ7SOs>4Lwskwi?c~)C
z)Ha^@{@$~A!taF##UmTGypVcS8L;^F_kZ*2zwfO6E;aRI)9=Smx%nHHfAUC}sC?-6
zk7+EvLaw%*uR>0~x>SGglv+XIqp<m1(@viLc0^@TZAgXG2h+Ab$E(wJsdp~)t9^8^
zTSB7nUdQKIS|)ta{o<Qzy_B^DO*nOnV(Lp)w%ae`2(Rxum1H=%<o=QGx(hg5eYhEK
zCq+qw-VOcp*8kY^mfn92`&swS>G0U-vuMwEM}e99W~ewXD6FwP()qKa(5Ugo)_k>z
zE<95?f2Qm|X5ajrx4w_#?<yA$pW8k{_Ip-T6}EI7s{d&9`nTA}PpiJXSg!K9=i00-
zzbh6VihL$7vY=CAmaJCymI`0n%UiYX+*B!7&tJ0aa?q830jycCO2y;$Y5cpzn%nYy
zZ*oUWg=Ey4W!D#+e9ijpf`i%Ni5vY@nv%3nx;&VfUz2_6)@;t7!uA#)!_JuRcAO#e
z>`SS}KJ`6{8Ecyl=6a}n`S_)Wv;Ftm(1`riO)O6iw>^#fpHUEP|KY2_p5^mm9bDFT
zt+x8}Q&w%2$)vVLirQgTRj(dTEq|`>`s73HhQHJIa;}g4ee3s>6IXU}PZc>CxbWOG
z8!koZ(z6dGW!^AUe&e-&EWc>l@(<1r!&|g<uS6VpIOmT`@bnGG_$R-KX-J5vm*!n5
zeyeWJgQ5pnehHrY>l$OaoKL-6tl7GLyUv_T8w#FQA1Qt;9c28-PyES@gk^er>hFGy
z=b9U@&XcgU+S#jkv2$^AfMd^26(>2D?yG^@<NZH4{(GAcDtcn|vS99W)~uIP*N7X;
zY?|o*=;%*p|9xBSmVMZ;jOFa>!c7(Ve6=m=cj{xj-hcPLaA<}8hb%AOe}$_Y<|xid
zeW%?vLEks^(7IWN7Q0lO`^=)p``1pUWL3U;lb}gb%W|HrhJUZ_t`}_Gwf9NAP3DW8
zZjl#Gr+ducS!h_hCcbUM#KO1vEVFXj&;Ge|Jud#xa&BkI?{@WnLq0X-n6^$jxc7h0
zwz}8S%8P#F-M9buQFHD41r0OIRb`h?jXuk><U|F7L3h@lx`5w_cOPeL3qNgE{=(tX
zv(MlAg*hH{m(5o%4EOlQWXF)t^DToxx`l(|gx5!&^JfF<c+9?E)MoW|nia?_-CwYy
zU{l!THR6*k<{9@c*L&mX(4@q0MJ;)%V`_2n>m$zF=LdZ-|LGUBA$NAXgFo~0yanAC
z{3qAzYZNni8x=?8#YQ~O-OR>v+@$rK`|G}!as1QM&R>{%&QN-a=zl>2OUGZ!4Duv+
zu2n=&Hv9Zk<=9n`jx}xX9~YLhG|#veE%Nu`{$Jh=oL{pA_Nmr9J{x-G{wr=F0j3Gl
z_PzTS$(?L)uuZ|ml#M6AY^MD)#{{lcYY{&^PCJ=tHH>B&-)G+vvJcbw;dNbxd;gc|
zg8vHZ>lYe*KPjdE`~9Qi##gs%+MhPxxybSMtka8FFHIJ;F?_7AY3}@OeTD3v4}wp-
z4(*fOq_(_J@Sp1LX`hrA@tbVah*Yt7a7lG>VXM0HuO&ZyG;WG39J=s)XQksKp1WP{
zzZ?a+Y+oENcV3n`{i}7#EPl!B^Yy$df0&)$zSJ&d|M|n8_)1+u-}A4wowegyX|S2F
z#f*|`+oJ#G3hjU9@!mU{CGfS88S~Z|S*vxJdRwyPUn|wF&ExxWU32kw=JJjA*KfWa
z%FFj(wfW=ZAh-CmCzZ2SMofRw?0acdT=P522@|&_Zts<K2{`(?TI8~((>WV~Y3}`V
zl@$JM-F0}2T}}3~FY5!lf_Q(LO?@y=!{cZAqP~CZr{BNbar^XRHr0LRQ{8ipwY+w*
z>adLcwmR-@T>jxz+jFJ=zHxr>D0Jb=uCu#Y80N}N2>4X=L#A3ZseNLM%bg>xTt|-W
zE?aRxXa5$bBQ;I`A4i;E>AqoB%;R=4ZPPJ5mb*tx;uV~hpLSw9z-z2$eZ9_D`&z`q
zg@0b;y`MgL;oYn!Cm0tz;o86A-9r8EOZEL%9-Y5Zw*1tCIoG|{cWvwBG-F!O`r?TA
z^&sQrS8iK=n3?CFJw5mH%jB1Bwd?HvR#|6#k;@Y*>^UNQO#ZG~*V3qjjYksQ7PYQs
z5wzfTxc2Ny9m}T^*-8hR<_bpzJX49zw%<ExqScE-EL-Qc9b9<Wy3+c{{R5|^Hal=X
ze4O%LAbjz`wV4~7rED(<{U|Q$lvnv)!pP<0zMJRPu^T#8%Gym|KR!-cb3HNUiu$yS
zzyFTt_Uz%l_IF0U^1_Gb3XiM3PT8TE{N3TX>h2}C*IU0;UC_E};<?zAV<zu}E=uj!
zEM9iXGj8ePxt>LOK5ntv3ldIEP&h5L<;0c_pSM@8`rTjc9y9AC*W68=T4^Ek4pqc<
zvoDg))-0QKy*Kiqu%zisnaQ4pjt1YZ&FN2j{J{Rq6G;aLh3JM!Z4)?@6)krcAAMHt
zc8cL++U@f8kGZF(9Pw#um^ycgJs)S5;B2XrF1LP*zF^-W9cFRpt-%YC@VbAA_1fW`
z4#8hV<{c|7W}JWWox?5x$Nkw;j*0o{Zxo*7^tmqYcloW;%jUi{s58H2_U`NRDRX8R
ztyu1J{d|D&Wk2IzKhu8f6RM8f_A&j-^|{Bz7JTOYq${%g|AT+0_sze3qE1=zy4bgQ
zS1xpEdd&HDT(|JhYt<Fa`b@dk{JXzT<5hXR)^qXk(DjeaSSRu(PG@nOIj14otYU)x
z4)>eyzD=L|dfK)(XE|Ti-=7=BspJ)H=Xm$SEe*$KfzGP@!N)jiO!mwv-1GTX?sGQx
z=02zM1*g*+D;;bY-Nfha;a@Dn)g1lvW6RO{+=<Wkb6D@>IPUj&(u~sfHQ6nve{aow
z+2Q}wQZ>!(X1?>+@86u~EI*s`<;#rEA{8g>V>6$8E^W41Iy?OA8#_tG%Go}($4v!Y
zCLeIU9G1<yW<}<4Oa09s-|atpCG=}e?z{AIk8{88Fi%#P+b~hWfknxGzI?OBw<{W-
z%%#?O&6%szxwEm^$|ZC1rH>}1kNNf+-p_CNCjI%ZT7dnZX<4>h?)M7MSF}WJzgwI9
z^m+>aoijh8cg5@oEmYYlaP>#kI`OaGL2DKou8GNi5w)q;zUXatk4$`&+~uCDr?~ps
zyLne6|2omT^m$M`Pf~{lgL{ipP58F$iYwICnK0~{U>)(agZ<8jx$)L>p0RgI3%h;$
zc*IUFQ@1_Vc>R`^>AltZ+GR4^a_^R1nBr@a>e62sx9i`NKbq3DdGSB4ESa01T=^<&
z{>6B<HxpcKwnbXI{Ysy{l4pgnYMa|lyP2BT-+g&&Hg8wKm9L>3Mu${V-+$AMc&xEK
z*FoTB(QUyWiXPmzas?HAjyL_^zO~lqBB$o;I;#oIrUv(S&%3kt$9MVc-^Ze>xbOC!
znc6NYo8jcm`P7tGcwtDF>>Bqy$Bh>jy$t&myY%5QZ=v4{7VVw;D?#*5i=$%5B?ES?
zWAk#^_6qLLYIz_jxW<gnY1`d=_mev=c3ZDn5_0<DKK*phQ=d`~d%mA=Zc5p|dta4r
zyjdU1ciMm3WbHN03qQ*K60}%tT&KNaqwkTwepdx1@bG6h>36)pYst5EY3TfAS8ph%
z=(g)7*E}<<$&{3p`91O3aef6C!BeNyk684s&3rx2`0q~P<?o-=3H#qWQv8QGDq^$x
z;WEx^+k1Yi=3MW%J$03hYt%o^(;mF@UVIJSc=!hM=YK2LEjs_|s#oEh=*@p3W>ro9
z8>H|#e)^{68+_)de0+EMQuQJI#V0RLtz4un7;o=(V{!$@{0-?syPiCdomuv4Z;V#q
zDjS`>pWfbEynB7`Yv)sI<KnAt?y7Y(*0>cSqVq^aPqEe`O!dO|Co?ruDm9n%Xdc@a
zo_Byv#{Q(4%EEY_ETLq*vwJl=R|nX95#l^B_jbx$`}|ae$gtaj@1I}S_jzG@|Bsc^
z{&m&9GpBd^X_cw4Rm4R^w=>qhc6xf$cgjgi_0y(BGaRlN2C?3d-d?c5Lbqpr=afI^
zecJU>pD&1d)8Y73eTqtLI>+4G%=w=`?D?;_Az1RF&qWW84fpEKN<ZEGv1i^&hujJ7
z(MKzMH+f9@IpLCvRN%3e19uiHGaDQ-{5LySWS!dP2OSBDyKYX}@A+c$WR?TQ!3~EN
zXs=}E`|$mhPxI{i+vUF+8?!vfZQ#*;l$gCXd*#{O&mjuipPkNJcys^M#hq-bitKW6
zYH!x6eC4_y_F`#_sNwp)AIY0#531kHVBo7?kniE*>~=`x)XZH;Cb{nY|Mt4q%H34&
ziCH>LGQ=}6Yl*<tp0-2M3)r|K!rkTy&P-L2Dm7X!C74#XS9#gLHU06@b7tLIWhXpi
zrD|PDXU<W^6K|YcPu$4<lwFth_|@#ymh*osxG7#Cw)Wz;xe{Xbhi7-`ZIiV!(GGv^
zJ4bfw-$=GURrBj*ntQe>iak$ud8lz_E#sX>KYNpJP7pun?Y82Adt(;oi%sI|FDz!(
zW}YAFkr?wr$J6Ya+v$68zqd1e{vi1O$+SNo4~kaQyqNIwyJ^JfB}+H*KG&FLxbM!X
zJ5hG44Mps{W0e;#c(Wii=5y)szu$dWUA;OUZLNFF?`*|%@_hZ0#vS4>rtCOhEE8mY
zXp--Vl#In(wUT|8wz6Efu+8-KzT;DNaLxXm`N<(e?8f?Mhr8oMzU&QB%(Bi2dcXYg
ze3#rMtY@3TO0xT<KdUN!mao)%wDrld&<U=sJ2=JX83wH7oIjzfH%;J$bLJlgzDKiP
zDlAybctur;dAdVYO|+dgV*u|Iotev`mrK<>P<FjtukJC?fcKB_uCs2@N-vD&Dem5p
z$!E2FuU#>BeQo8Lxl3=YeZR&1`^Gw{>;Ht;CVDXPoc-_Np0y}iHrgh$fAZRYe@x!=
zpISWeI^(x_BJS3oN+hQI4rNSQy69DeldeC5NapQA{W#~U*%!RDORAf<<rlo}4fw6g
z^#1yDW_|G&2`+ovR&?}M7N2?kaPIskjJvkIkI&npS)U%`6>_?C<HE_}5d!?tZU4od
zZ9dGt|A731HSx1WtUkVvuRM}n@5$U>vtBhz?e%NsT|2Jay7TV;j9ZQBU#cH+dX-MA
zcV>Q{|82X3kLPo<7uU;w&XeRm(Rk72*To02OWvL~Q@r)}N8FWl=cDF?UMjPE>+4x2
zZdZ}}Zu5e7zoHLBWG#xjP_~4xv$o7K;b6d}orgcS$EAk6FnCp_;rn>Ck$ZjSS~hpD
z*Rgx2R>o{>)4t~Uch}yR>?ddcTGRRa@DABcoT)lj-v2rI=G~7AD@zjgy=2{_T6gSD
zth-Xf&bJKvx8;AC{QCP9r!Dit-kI$Dobr5i<eKD7$DdjF{h9YNeCIS?wL|llwu?I5
zZ@sVBXK_x<>tA2`zZ08gJy5W8?)SZUPj9g~fASLFePsb}i-LcL88H6aP|vIIW9Rmb
zj`FYB{;_XMnyMmS`j&5fpU=rxnwl~OdxYZbo5H2fNaUZ^m+M)e8F)MIN3y0>beHFo
ztoiLRs*_GlXh>_%5We+mW6xs~pQGBvWp681?Qxite$wcq^utF?e<x>Z&3VleJ9Cpr
zSDp&bgLU3(PDHHqnJ>LKAW+O`zPXOZ!fzE5*I(z`?!Cps|J0RF%IXv3Ln`ktDYp-s
zDAxYwN`Jc4zU<9Umh#`P@%#LgW3s}+BlCU5CRyA#arDyt2TLbL9DI;;TfT_D=8j*z
zUX|US?B>AS4{K_VzVq&%wmxy~ypvs%W@XRhJSWJp&Gw*t^CsCX{ueL$sBC(!aD6Vr
zG(8uMzJvs~r!yb4{c9{P%WZ$vKJ`=2#VudIEt|4Qb7_Xx)x^y|N<Qi_&Cz}I>!F%K
zrFhW>v$Fv~`A38K!X!4fDqoIRYGm|t--5FO44EHPc;9XQqfz|3oyUM7)nJ>&S}PN-
z*V&nA%-qJt->S4_=1xt2!LjWsJMUtj7ti189gldD6Xl{iNpQy@!8WBi+h%OcXZy1!
zZNrJ(_7fDvm7cyS$#&mda%HVffYHytCn7SBFl&T;_3%>EkE(q;v(QnwNp(NRn=J~W
zUz$q4^!BH0T-fvAdeHgMH{Nr0<Ze*p{UQEIt(@uE?%cHAiC2%DFi{GW`fxsf@;%S`
z#K#_iikd95=G~r=zc|z*)L~LoTrT4@`{@^T*4wJjnt$78LPpZ1b88(>Gw#}?w@`6*
zNP?qLP^E~|R@+F!z`_W2-p|)(Z7b^$ss49W*n4{8mDiIV)c-1fVfgk!tbuJ`k6M$d
z(XYRgQgS8dpHJCw%xH>xA4g5v&e=~l#ZHNPV07PNb$Xb+5MOP}yImJ11$(M^c?g-Q
zW*&WOl&E*~j^l&d-*#s#TIS7?+i^!nUVUoM<esA^H$PHI^$dP#+T*1keDXunra#Q<
zUIZ1g>?l#VpMK-a^qhCi#lL=eS$OeVU-3%Y_A)`&e$&}7ogY;*&IIqeyeBxrd-A0-
zuO{u>wB6@I&BUwUnvQRekKEj7&hXv#+xCf_!SCKW@E1BvD)BA2vrErB&!u<Qrav}^
zw&d`yobG&Sk8a!fc@vk*KlqilY?A5Zy@{)J%DB3YPGm4}zBctshlnHZ--sx#4^_8s
zp7>_t^Zm}&W!I0VNquR5pqxBoGP8Hk3m!e?#Od>ve$<(*@>1n(u~4a{<>9gk2dgrC
z4yq>9Nb@|=U(6947qBzIxcY6!MdKT1mK^_Az3xJt`2}Ux;IITQ8HNA>*ZT9W5s_PM
z*DAf`@?uO<SSk9aYLTO#&pJ-2*}IoKkIj2y;KG>W(|TB1L;JnYPLqPEW`}ClHM`Yt
z{yFu>C-XsO0+*4}%OwW3pCftGwrpGcEB2nqi;L?|Jo0(HB6(g#!o8M%YfrSyH+%QG
zbV=3PX=-fQ%CAlZ?3I4k=5}S;!?K9-%H5r^j}}k+lIgT<`yJyMx6Oog>ksdEbb>L(
zyFiWqS5*0x-$!OL@A}WQJym4pmKkxJ$<CZFRl1`xejeF)isu6Btah&Z%5Ul%8K=rU
zEbnabeY!!QPvNZny0-nl)o)il{ds!EqqjFBOiy*x@HnL({Nfd|ck1CKFIE{&R$Qg<
zLHpY01zCLepKm>U@4e^QV?~M{=RJ~2C4y!c*anLKDonX!wCI?<QJ>L?d;GIs@V_ir
zy83Ue%kEwJn=_m$cCK4nWN|yoJNV?ZtwO)mw=nKx)i@S1L)*p4-u=3nbxFJBj$enY
zv=t}Jm-c)6`p|pn$s0aDQrOJSu*h89-66QLp=sHLqv}Q(XWf2m3Upkh!(ckU=*26;
zX)o$``ftvAza@3WarZZ6FSdT3!)(m!`Nn}s<sI+s^Sy6p<Uf5Z|6_*Ui=WHF1rN-v
z*|)IzMOb9Zg@zY;hreC9XTHH>-R$|M5_hk4b4c&>Zhs{(w{6CS6$fI?`Fz)$Hf`QD
z>w@sB37)OFU8f|k*WLWmG~o{4f&0I$oZHkp!gkG+;@F-v!Q%7f_jk*G3az-Ca>f76
z?<a-1;fJD{uS>Ic^Q~gNS{V~_a_*O~xDUC@xO>eVmIXiG{cBgjEh#0%Z(4?*Hrwar
zpFbEfzj@!S4^JPq*!xQ-@4flsr&IB+)_zuTg}U;(aN{YxvbD)VU9aRN-aWtkqg1Qv
zjk2R4>zoa5b$<$|%e{Zf^z!DOa{>kn|8UJZaKeFqa+26O#lSP}O`ku1SeG!7YsHOC
zrzgkVI%Te1_U`tyR7=h0$zKw6Tmz=5tAr^%+w~~H>0jE#hC7#SLysIjd4<m@@U}@H
ztLWOkhBZIftm9AH`UkyssY(&r$acl)no9QL8ZC<}3qQu(;(x3na7FoNkjV_2OV^i)
zOncaP_|~^u5+`TcWw-a5Z=AV_Q(+C;%54gZWJ<Lrr!8LhHMcv+TBAGmg|T7xBc<-+
z(>0TfS=634FIuR$CT@1W)cR`M=Iw7=Kb<X^(zc|02XnZrLiQ5fIJ3uBOOw86oD(cp
zYceYPq{gZ)k!{(-xnHvT-=Z`B-nuCW<sX_;yxjO#Y0}2`=dPJ>Ua*VEZoVNX{%NgM
z5>x8e3Z>n(hkVa!Xidz2x?^tsDTY?Vtz4zC-^)|~=^EU6#(7CtUgR@_(Iw#^>#18h
zRF(%nY<R0SZR>*g>-l0+xv$=5xwksu!KsKT-{ejnZW63flGiPn-rxK<^440Bm)G{>
z-an(Y`Ndl9yUR=ZCfnW0wSLf}VY1)IGBCRBc;YR?gPr^StX#|aY}bLnSW_Xnh=s~?
zZkx}ovU)f7*u#^`zN)23x{WzH<rQ5m!Cn)Yoe!(*f8D@ix^4Eg+R1zPq>~OW-*qrS
z;nI}m1+Q7|Y1TTk#U1joUun_uG=fb+dP}HdLdf?whh~bs6k0a%`(5W~&95Ks1Z+CH
zX_vL?op_^)o7<PxY~Hs+&9m$3S?|e_{DMw5H}rlqn5aI@caD?o^p%+>|7kbfRx+A$
zO7Khc%#|@$gkrA$>Zw@!wtA8JFTO>y_Z_(t(Osb<b6DZ_>*908-ZF<RA09~W`d2n7
z@1?b#PUozU3m%)4cgnBG{J7Zgz`UUTcVfFjbGEKZwktQX`!V-STu9w7bGD27e+BA`
zZCmd6f6GJx?$0|LQUw2H3H&N{UiKmEsjJ-27Z;qqMk{@rZS^DaB;Okb!w1V=#rnDx
z)#X0fX}^fisNDNh>(_G0lK-cscYn>gDH@=_@x|}O>bS#G&fYhA>LYw=DR*Vuil2O+
z@BKY+nd{BIOTwJH`0u9W#j;E7pS0socTB?nZ%?g0uHPUcvNzAgWtNiqT-gtT-rvps
z%iQ@V=-v{!d#N6ePKmyR%yu0?CLxLFmBD7r%CjzsJXjIJ=j5?KPwPqWxeA^w^AkFo
zBz*PC4jSH;f9CSm>WZiOs_UG`&ZcY@sZ?M7t@Djpfd7@&)e{4s-=9$Q|L5(EJh?Mh
zwl;28`J4XaJNq8zgm;N~4<D*p-01jl^`yI&u6o0<6A7s<@3g)KB&IIpK6cmq;OEJq
z&vVk06}(gKZkl+X`I10oj%3E3ZAaFGd35P1Z=HU2&w_R7*A2@Zl{)`&@b{(9xmnN@
zR@GDTzdmPLDZA|R)s6MP&rPY?xkhm5JFyR^w>qlNW{-Nh&Bn+otU=+%Y&PpFzxdSq
zxOH87%c{M*R!s^@-69<mvgiA$99{b-FYA{a+v=b@cSHQPIc*uqZyQS9EL&|5&O2@O
zq~PVIg}Saz=a2JF+P1jo+K)vu=XXoImj2qXVf~ytt|t|*Z2m0~G4IgF2#Mp|5~s!8
zjxTVMGFrs)B>m7DL!W8;OvArQIIVTuymew#_&S4q`zxH!eP=snp2g(%v*drP<u!4I
zoe6R?e^oXq%vCJ$DBkoqp}Ey~)7*cm=2h$^3*OXx+iP63U}|&LGudkUk3aU58sECN
z;qsF|A?jy$=H^{@``@uNsxxxko8xWwm+exSBmD0EuC4W(Rmz_z+}JVg;33{+^Gg%@
zOrAzIbjAyY%zJ5f%;^`$V}-Zoo+;^U<qqMI1^26rXPNJJ+)}Udzm-vrqs6CY+pRNp
z@7SIsJ4|!kWMtR8kh9BD=IuJJ*^3XX^wsKcR65)(bKQyk;^Kq=#{jWsGj_;|n;nbZ
z&^|$LrP<?vLLL@%&Bs679~obm_4Vw0vp2%(7j6X@^tR}-mEP)TY$~Y<P?yr#J7tA5
z_Zfw&t<NP5pV_xs&&$4gLrhQ2Nae~J+ulFdzwW=cWy6d!7g$a`oBd0D+n)PQ8w2Z<
z3Pfjb)qRn_^}eD_sP!zi4cZkYwt=#9FEffyegAHn{G;WU?e(wT;7_bOrTlhB!6uWa
zv(HLzPqa4v<GN{~cG%%^!<L@Lay8$!m+1j}En*6`|GKgJk?<3}Gfk(KWfdBqnmYOY
zzr>FHFP_(yE>)l8Q&z0ZyJKSHxf81CDO}OH6U_87Qn?*fJm<>peAv6E+D*<{+5hfw
zj;q_JzImC^AyOh7=xbJ{d@cT`q;P`Dk9!>t-!paf+I&3~@<Zvu)>?U^dy*?Z^~|aM
zIp1vdM0=)yNyhj3LYV69U;1XO4JwE~-NgAiC#mgx_F`kU$7PB$1%Eb9%uHQq=(VIw
z_CWSTnKKn{Yn6IdT-+!5T*E8QI`Mo}%m#h4pi9z~kL|O!oOruj<naRcdXKZ}Z%sHO
zg0*Vayts6&{@2-Sl@r$o+2<9QUdw(c>7l&;^%AR%%%2~*Gp;TyKEc0KWBM<_9No21
zQ#3w!Imte*l04h>N^q0q_NcXIK285($jX@;9d$iscj(RzC0&IFt~1t2%f1$}zVe3W
z{Lbw9>J1kf!ulpZv~oXEC~}%-o&V*BV)aF*?7DhWjvC)Nur&8)&$%mH@}9DV+e>_|
zaOj@zXy~b6KWM6*UE_CpO<|^Ftp%&G>Qg7qhnBkp{xwUq&ikT#eQ_XTWG{Qslm+J*
z^{-|>UdIyl%Eoa0Z>GM!6$dx{l<OAXeAcULkF&(fNBjPq^!k*tFmnO_E=k60KI^l(
zyi0mi4j2?<WY_&QsbLGV%Q9d1O+Rt-)T%YgucMn6Pn*1NfAG=S`uwl|?bZ=>|MZD{
zf4#%ibvr8+Z=R@0O7Sr?+EmjMqv7Tzs;t?4mSxWGe)GEux5aCOSmz%(b5&ncJmPGm
zan`=&m$ELry)JgXt$n6x!+EikdpFKopFU#xo=MQ#@5-G*8EeVHAlaIg_kw;c-BY{e
zN8Zi|P6bcF7==^4yAGsJeLq+7(3!JcA7&kB+i`x=+DIGawW7PX@AFEUUGjZR;cBP5
zPV<7Se7(OUs0kab`E_vBgGmztEO$7&WpS%-yV_ZK_kXO;qO+MlH^v(&+MNCTMJ1V4
zr>#GKws%-weD(8LD}%naZCSpSKlDJ^Z9fwWKga1kyVhI#_`IlH|GM+D>WkdFuXAf%
zK2QAgebqI;uUU(CY)mL+UT*FbssFsTeA6>)$#0xH_I9NF4W6*`mGxtHkBQtXU!U#V
zW6m9Tb&{~aDuz_ehpQScEK`~+dqTys?Bnmb=Dwb@<<^~g|H8U_(yw)U9P@bQ#2l>A
z4tO6Mre1$mptsIxnYCQ5!hJ=KwNqS6yMHZ=ye<4(^xRF>e67d#BM(-!&;9<*H^pMz
zjB0JkHHVmHMBlW_n)IK+<Mg|gWgWA`|46T!EqvtdlWNav&+nxLvB}naI>Y|T-C}3F
zHCMp=(`-I>CrvKj@AvDfta9El+r3jgJDPe}ulv|s?h{tvWMbJIdB0+&ewpMN{ricz
z^Lvi>cHeg7%wi8P6XMF`*vsRY(|NS#@e^ie#UQ=avL`3{T@pTjpeaLxQNmhkR<-c9
zu==+KPCY?il3q%`Q9GD-;S&44Hy3_wyp&kNz1uRuTiNVH$NlJX<$^;UW<p$%5B#RK
zoH=#r-nn&t_uEpAo|#@Y{nmrON1J{;-ZMku%@dwNr{1|NR=fL+=FHJZXMLx(ys9wY
zY`VhQHMy?@f3LaoU{lW5ZwiO|b1%KIabRD2^qBP@k3&cJZl_<^(6h{6jNRv`YVe!e
z9$m*POfypAtW`Ge{~?yGFm3JRpFh-H>YT03Zb}5R@J>w#+IeqQr!T|%&36}`SoyH~
zbN+R{w0}L^OLKFY51gLl5>&AM?f+9xGK&1?DSAmhy&k|}bjtNn&62}CAtFib@2@VD
zxPACdvzKsoz@y&YliQ+%nd_QMuJlS>bpFWS^XsvnMnOr)^DWhXSKc+5?7U3Up<|u!
zrq?O&J}A#_pS<L8a_#c17esdPdHmzKruj1@;wxYBi*pHke$KKJ&O53)@xw&(lIh06
zYiul;jL&zUx6a6E<yzOVk;iX&$L`+kU7s%6WxnY;_ANap<5<|~(^)T$Pv2O7^S!Or
zyQ;@q_cO}m@A>YSsh7d_!PNCzzL8n*+7%TavJV&6r@T9@?fL0Q^tn?m=hyq5l(w5A
zTDoQH$EP+On-{R&lbEu#XnpnZu0MD8=Zb!{-QTe?`}gW~_ESS*Wls3@i3Dv-w2BXn
z^71>O75dxjd-2h-so!g#hp#=gQi)SczI9Ujs!chcclL6)JF46|oxA(l<8`I+?e7)X
z1Y0Jo;Qqd6)tYJc%JS^GlMIe925&a7DCYNj%*m-yoD}VLcligC<4S5OPwVgAmf^d-
zF2!)&9*LFzTa_aWd(KM+2^dB%yK+Qro_gY;e`mecZojf~Mo0aEN5^BlS?A9Rip`w0
zYtsLn%QC|C4qs``(_H-dobclXc0P0Of4$w{=8?PU{De(b56{N`;@D@mExW#a<K+gw
znKA3rq+PaUUy#w-T2~u5cX915w?7Y6|4!6dviM#1vTr>r)89?w7wdgtXzgG9ImSls
z)j!_=K2C)tbzH}*f4T3j+Tx#`osk|}-}kR7{qeo1fB!GB{4Q<EnWVYTpJByqu5Ub-
z70*rVn02S(SYgugiM`G{Rc9_(bnLcumdB%Y1rG(4^bPAOO1RpZ^Yt23&ZlJ0@K~_L
zq=TV|>Eqmpgo8}Nl_fWd{O)OPU$V?Xy+|i#&p)$pJ5RrmoFG=M2@Wv@C22J;j&SJA
zVyoP8r|-EE({)b6t6~;{fx?2$Q|_G-+Y-lFpwP+w?|Rx1d#i)bp6N+Au67HVB)F~l
z>U6s%*_^$9*jn~#NcsL@TYAxJmBan=$Y70yZ~yd1$nI_GJL=M;bM#Ka->bg=1ir60
zEa<hmpj476C@tvx-6Lkpo%<KwclGKL{uH@$LBqOhyzM`yGCBpFzvmHrskuXG+O5;A
z=HeD>1PV797=18Ux%=-`-`1U_qGmqjca;<y*SzA;h?(qp=(G5FwoiQlJ{&eh_a!H=
zB<yXT_WyOU|CZY!aj8bn>r>9YXIgK(?2(ni<sZG3l@}E3+TyP!3Z=}GQQmW6GP~jD
z>L21Qo{ZAP?zcO(3u-(Qu=?XwVq@rkXmf7Po&)R8O5VDEvG&Ha`DbEZo@SkBz_6uV
zdDn^cJd7)Nrl^}PT>nm0Yhv>h#^Y`;CLVqzCO1FlE6;JIJvDP#7>_+O>-qP?&Ro;r
zXsf<SvvutH=Qj#3NHpx(rknjF?7~scj>ajImWuH9`CpSRvu5vh{1rc`yUs#APCfE}
z@2(7|*P%6+E?k_x>3~e!1ZUlkleOX>1X(Z!=3W1&EFRKV_s1<|x>k~f%)5+);@l^l
zX1+I=8-I2zT)*PLlU|Pa2fu^*tP8&vRir*yZIB^w$$j#pO$sX}HcdS!sIAb-wQ!&E
z(db6Eg!t1fY)e*s%<g@6Y?+Pn*<Xf_KJ@&#mHgr46Xou28%p|PgdRlP+ZU<o6vOOx
z<JmOz=Net_92*Q%SZmd4&fD(g&}{u&J1yLK|ME!hp1hVXw#)B#ul@2_-peOk#CML_
z&sS0Lt>tSSgSEMsa#sCZ5VfW6c@v8i^ZU1|7mZFgJze(px5?e~cfULCsdOI})s{Yc
z!u8V5iz2d1UaK*S{aiZRWLH(@|Jhb0Tfa2!S^rAZNzv_|%UZvR!`tKbOZeH`)!4m!
z%B-ftc3*g6#lLTKPdO!MDJgil<IB#dS;_q#XC_8l-AO3<@<wT5s?ACt^Kg;P=5G!$
z|D0o0{bu{GS%#m#f3Ket8dcU=>L>E<Y{#WQxl8@>oin#?*^v->L-EY}6Iz?*uv;ni
zb4JQ}WS`s}c`tW!c8PgH)Oi`}2RpQ5U+gkqR@iJkn_Z>T;^rx1fnue!4x<a=*Q>UD
zH~zl(w3YaE*Zbv)>Q}X9drk88^jN^BzQ1Lk#POr-uH9=Q{MgR*+zxo@#x8$``=s3Z
zbx(_*KKPPc@bAEcpPSDpZe8|o+0|QOerszNPmoyo%(I|t-IHemp%Je>T}VH&ROXF#
z(Wbq-#CAP*I&!mNRi4<sQ%i2@+%L6#;VgOG(_-cNJC!%m>QV&GmQM>&k~Noqpq%n&
z(oLg_;fp4|mRRsFmb0jvA-8nul4Pa%9)=Nj_w*@*ty&hb?3a1aG%scMv-(PodRbSr
z%UA9X{6GJHPTrjF@oc4bi@m;Zv$AtMKGZ5uqqD_BLqIiG%p#yl@$b{p7jK@2Zj0tP
zw5D>_M!|!MukRmG&GXy)q&KT3c41!Zr?59)(spdt4*tLD^ijct>z`vq`<)r<g};}1
zmI*SP3u$}K^l?ST_9u$WJ2msC<~wC-EXvTV+al!ui&<e#bMaCW-JL-b@*b5;e0y)M
zycBQuk)_>c_h0WVSbym5`o}4UlEi&QzTLRFS#ibQ?*FD{_p=X3N|y!;8_kcb-B}+u
zS8@CDrK0tny5?N*-45nE=B{9u*C}`P`M*LdYg^v6&EC0|v)Vk;AGd7}o8Fz-;ccj@
z`bFZa_qKBu3)k4k`zxOCES;-uz|ZH*%wYTa&x=_aYq%@Z1X<X;U;LVvVjlZpPxp*;
z2GMDm9LJQlt^N9P(fryi0lSZ7{`6#2dVO!xA$6-QuU;KE9%TB&>e}_6hUc8SIbKd*
z^MBVK(`4r7LcZ)S%WVGJpFG3R{gpvF+ge}`m;3X!>26cH@1(FVzbLS=QA$_X?rhDb
z*%5h_nNdpxtu8!XpUZV>{_z<)ZVTV)S=B8MxLr8+^v`q_?iC)(%qPt@d6At`%Nche
zruy@7UE!l~_9ik<{%EHjP+Z4+&hbOJ%ZA)Tt9~q>9Pw>cRMf^ZYP%;#^qpcXt3N8m
zWorM)=JY|M+}|-<o~#qM(l38HzlHCoP274}4buf{KBUC|-Q888Gd-!glCf1Y{HIIU
zi=thVSWju+o)?^@WZAs0YSpjO2WL8q^sii*Y&%KHfK6j#tVYS9Q{QegUTFCA)aAd$
zn_cJmR^4c5uvz17apdXg8IJ4Mdk0VFwGgbE9BFN^Lfx$1>%-;U{;xQ4m6jaIysxuJ
zGVZN+^*MG{mFK6GyO!0gk~*}>Mr-EXhoa|JSX?@z&%XZP`3>`X;u&La`Oi~R{x*Gi
z-<$g?4;S+-YMA=T_t)Z8Q@X!dY&V-w_e`zKH${4nk;ghFhZ7&f{+m4)c&EI3`(<7A
zi3yif4$9gr-DH^~zkAzjHuw4sH3sj3V(vP9^L+Gc;gRxx+{+FKs6H?;5MuwrZ1H7d
z$dCCE!NE)1Ua08to~*t9f99?7O{FcC8q5MJ&kVS<!kmuYS@F~ELMM07zimgoZ|>UY
zdHhwacB3nI&7&RlvsPC3u~&w!WXzu^d`oY~-IZK^3ofyq)F_V0_;0ZD*A$<hTAyw|
zt$r095pi04&b=dV&x**zAKVwpaO=ebFWK;^(h1yrnU4efL-rfpzr9zi`Ck0axc6Rf
zZnq`R=i8=yvZ+Wxj?rlCI{C++TPiQ6?Bz(xdvwnJxW|9PBVN|R8P}9~kHj4p(8{~J
z<Z8Z}cA%<jhw1jGd(DEKql?t!^>mJ}mF|$*v}4Ehe~E`|dUaXSLW5(ky1nte@?^H3
z{=4hP1d3f)dKJ^$rr-LvFyx@d&$pjfeplr?w)g*(2_0{O&#P|P{_VgI6S3*ayUriG
zTdqB!>G?DZ=Rf@yzyI`2@CtvfrryDO$=&b4k4fh~SO}&XG?je3-gx9?Y`@Kwg`SI5
z_g`t<{n%Oj^!Zh1L+*BeuKiy0>d3k>b{5gELLDkUnCe%Ed@BE6mtriu+3Dy1Akj5%
zOs^Ym+3vwtIa6ffhe_|Z{PFy_v}DSafK29<vqOvw;(k12{A^VgIA^|(#AM&3o2Cb5
zN;7Saox`8|L@c+3fycL?`7%SD%w)H_uTo3*zLq_-ab4BS?bGh^PJGyOEJRT+X368z
zcXKiu1(sgk|G4B*Y0%{Ww)*6$yPt|mJT6^%w^IJ_-k+vHXSBD<u8^6(USZ-~nUyuF
zJu}n;S4=WEkic@cYRk@-9&UM^*Vnv!YkIZrtWK(H6YK1*_)m^G`*`@9U7UVJ`Yg8C
zSlHt5XtqOA(Yw^80_)n(Jxa~wv)7erZHrf#Q8KCeV_nIU;@cwfPZ?hA$%s6%EBUtd
z)Y5AEY5BI-J-R;z)(S53wGnc6V&hpg;Z$FTp=^w}rs}1+d1``Y&NA%N-wK8l-^@A5
zSL}HEwBr5AQb{jnt>^w)dzP(2{iW5XZ`nQnud`f`_kZ!hI6wEhZ{O9=QxtpiPBAXG
zC=2q<vYq|tX-~>j5u3dqZ-=`cjt`76HF0Tu)77!y@BWufdRG@~+U@esvxt9{G-o|W
znCr*M5}%vi1#qtZ@w&gleY;U;Qn5s)oqmw8fSLXsWh={Tca2Q%EL}b;OYK_A9<eq}
zt*;*6LVdE{me$LroqpJveI?+v_vNXla+x}OuB};G8hCVrw@LE#D_gsDSpM(UuU{Ye
zJMHiElGY_>#aWzOEx&(sk_i5K+M@A<a`x++A=B1qyv(R8KfS4=Gw7>a`!~ZGil_7f
zON_rO8$GlOs*ibf_QKcT#xu1Aq1O)YYxG=UeAm*q_xKuzxqB1un+U43%O$<|W013I
zO63W4$pk5@4K~`V{%d|+$ba)qf|uG9fA_nYEBIF>gmY}%x=7MGMRUfBjck8~f19^0
zf5&Med1%qX*;21pe%pGVf91YqET!k0Gn2UYY*bqIc*09Iaghn{US|DIONgn-sg~oB
zi;VyJ`-Sz*bpJyUKM!8oXZ~~Xd(n^gRjxUCX_VjDb1h=E5yO;@>U7<UyWd(AD!DjH
zxcR$ttm)_468NDxe6F8H%m>f68xplQ+~$1~<~gn8ad=_rOJ>IPrSI4l1wC94a@Uq+
z<CJ7?j<c*^N|i*lcHBSP%dIQ&=9%=8MU&WFjTyp^G+v#sK7vtdPSB)BA$(^xFffVE
zTl4OL+|@%9E#%v8s!BTilbpKJ)c3(u(|`qUxLyRd#*{6VQMS!|{3@!_L3a0!uZfis
z-VzG}gFpY=`ug}fdG?a*rVzW3_}AB8)oz#1-@b-ZH2jDq(;1_^QNBHIdKWSttlRbM
z^oQ`n|G(ax_i^&tb?2V#*KFzA5qaa$DP{&OQ`R{pjO%J2tGR6wOf2RLGn_JYPVuZ+
z-_PuGlDhig_-Vy&F*i!PEgN?lRd!jiPCk6X)^?NUqLS+slbMcsY~E?Ues-Ftxp9TU
zR=XXU3YRp>yIU7eV>{ojacWkxb)3(19^QL<GA3+X(spV?>%+CHa{k43>Yj|g`#G|a
z{p5D@1wwHP?v<yB33Dcy)QPn+?Fcxku-NqO=I1{SUE3-2S-9?&S+d#{_WIzn%m4p+
zwLjy5(-Ol^Go?I3c>Z;tu<vY-n74;hw(D?q%89U+JT?V6$!)W@ymejrP_K3!+tVtA
zRl6N?B$+;hZ8JMntU0wz%w*l_TGtlo4<;(2ON(+<_^pz<6c<Q*Q99}&8+h*2+!lB3
zceh^L^;;;*ylTc0w_|JE&Mkj)Z`lP28CmI@N54LMf7SKwLV4yj$DY2LsWLx`b913%
zO3>RidQW)VPB)(2Ui2|6Y}tifH4;_F9$#H2GvlSisuJtur3revZ!UZel|R4dl4a*Z
z#b`}|J7>3SjEh_U?Ebmbr7uMnuF}x*SR;Nhjp_W<$6I1@Zr<A3weI~d*C!o1e@hs}
z-s@NiuHq<|pwqEnLkMTDqI<>Vx6`8kBwXI{PtMBukJ{y%MKk$ISE&c}-d15&UR`+n
zQki?{(jYfk9@f&bdp|Q~|5X;vT%nZv=*uN9FRQ9Fj>$VNo&I$C>#k!u+0V9~HR#g2
z7qF&B<ZCX!G2=4d@0pf%voC*<Ox|99+SSCw&iT3@uaEh<tE>N1vw7%B-|N4${ZQw8
z-*t%sB0Co_)gDWYT5`-^G+>_FyiTQ`GAVpJg&zfK?djh1e|1&5q5SGs&LWCFh3T%v
z4Z`I+4U;Q!|E@eWLzRc)YqoIQl(X!URrc>*c*|tzIi52&mngkXyxO&Qy+_hjKmMRE
z$FyV`4=H5c*EZsMB+b+HJAZ~1^S`d|rqy3!%J%OI&{KOm-!+om`#RUeXWoa(n<`_&
z4<|^eaDB+$YWYN0rbw$kt!#1b^G{vu(|4yIIP!5%Yw`IrE0cDW^*j}my8b=oY{sQI
zOVf=eENapZ3yk|R`{V4IJMzAlPEMFt>GGjf>|*p0!;_NLV!LL4oo(45a*b#C3WMmy
z>>DcIBz)}nP*lO@yvg^ia=mBY%s0Z7`$JTo|CC$4XyTUBMwe?QC6-BS6*_T0>D4uk
z+1<@=nmeQChseHao|h-~PBQqxv`uWHYI?b^C;c&Zt@$Z&bdj3gs_qpX-ufwBQO4yV
z=J{F^7VkK)Zzi9YyQxiXtfJAB6GFR|7Hjn$-nem#Lc-j&r^;t%J4e2Le!6<*o9^P^
zQ~7;oZ@qGExPO0gxUgBA-<+<BRl4_Mtr8cm-!VZtch`4wOOvjc_7q{Qn6o#G0$R1c
zUs$}g?R1ZkQ2)oOlO0pFzT|j?uDP&e|B;RF{(M{S9vo?O$oBi~xuurNpVj`T=6imv
zEW=kO;dbrUAi1^}hG4%|Me)NEA`M@CGg-6s&NlC|`sc5A`+LoO&Bi4r^3$aJ<Jv!(
zi{D>ke<yY5#qmFj*B|I`RK0nefBu0Rp3V1)lVyB_CT*68jO^K>=QdCD^c`(CkL7}w
zS9bs2<#FWh;oytHZ|y$qiOAo-E2Vbv{rYnr*$Z~h%WFTfa(!z``5B(wVri>%;>0Z%
z>N(cEJh|(~SuKWw=f(Aj>)amPZr4ee8#CqIZ?o{loK>0MfAwBDe1om6>|B5CpF<D-
zFEv!-PjKR@&gBs}vTa{0=RMX^u5<14e>POe$uG}MSoCOG^t+JQ?a_0t|Js}%P<rL-
zbM<zo87clHroR^c{=Vt`?1`_x)lK^S)$rZ2^(otT8_2RIb8VLQ($|`s!SQxxQ;buj
z$PZzOg$4^E#RZI6b-wnlsCo7$dPbX%qH&?@1b4-l&Hs0;`);nbjB(e$ZM9}GU$_sS
zD_3YwoBcY;di}dYCUchTy4bN=P+fecN&Ss|lJ?0<Y|qYIx_hpIi<a~C%8RbvE-$te
zm2^vJB&pdNzpZfcKDJqcp)<7arp83CX%jtE6i?b5^y8C1uW}^da>;jD>8r=(DsRr%
zE%VX;M{@5zpL@zTGUR?RC$|26!gJ+hzzk)P6CQEdN0=lPJ*IuswaV1F{LeqKZj;c3
z;~Xz9x>|ob@k=^nL*=%6!YPdWI}Jq7{_OFrNMsLB($&eHeq-yKHKvYSNt<LAZ9EXn
z<?&`!OwMhO)$c84bt>jY&RcrQZ|;J_+uk0(Fs09qr77;o&YA9~EG3nl%RHt}yvAKN
z^~-F%$tpT=Pmlh1J!j!{v1yBBW@N>PpYnKR)jsD$?6q6FHtQUC7;|a+OtWwEjh?pk
zY^hr>)--)>pS$*}M;CP6YHb(KxDhDmtaf5br1AEHh7TR4`TXG8x;{g%wd`;6vH3e=
zMXx@JFH{k>nNn+-T5|UHZqEGt>fWQx6AN!A9?N*=ZQpJu$i}9~_<ieIk;wHcWvs%I
z#Vn3<6u4iU-dW&rb?qzf{cX<*0>0WOwaRfnyLIZnzuBLO5nunUs<?XgLfX4UDLc%k
zWU%_nbgHk*XFE2(<+H_3HT9}x*L>C1X*m}JKXZ7{x5unw{x9$DOR<I?+5g|&@?J7Q
z@?Wgx$@Y-%rOMuoPIneWl*|{g^1L0Pb^eZ%=f;3bkB*k9T<n>cd*p<)(tVqqTh6a}
z$!@g1FkRlViL-REvFjFXA?M2W4F~2;FUo(n+RILJ%jZM&f>U;PcijKswC?Hi3)31m
z{Bpac(6VaEmrQTr?Hf(Qr6qSyn=?ascjv@gM?4IZRRS+E9B|CY*y^>;^V5-|V!T&9
ze@)X!NP05!%+-MFOXNBqlpb-J(EMsrmr`kIps-f_z1Q9=j}<7%rU`s)nwEdCz$C)E
z%%bw)`n`g`O4s)KOFv5S->iEx`L^x^9U-@=XXacNn`X6j(-ueJoc3!kI2aO69-Yeh
zZ(;!Bq*W6dvp0$UiO5fCKYeV^<mI2swwWGnVoiU^U+L(2$^KJ!Z*Eyv|BsFHcx*o!
zHghekT+x5-i%!kHFkz+^;~gGm*L6O;T4$^O+tJ))lkLm0?Yj!!pZlnuXDnoQw9YZ@
z)Qm5;|IEq1HtTHavx8kqofm{e#CMt$>7F@!XY<TcD%aG1PV0H&Zm)Xhwj^u+&0C7I
z5*AEm_-~xDomY-GZr4F^DJ9jwwTi+wJ!gcl-m%_uVt?`@pB;)q`R-5JLf2euJXrN4
zRmFRH-18$0lV2(dJuF=3`29uY{O)b$3!UbiWIP+Ph^P5|>im~6&+TqskrX|7@RYe{
zGmBo`ui2?8@+UgGrt_?M>R9%=u~b*?gTb5c3z;O_x}Q6=@&4cPF!<b&^jo3^TpYKT
z?{Qwl^|JHmMz?M0-{%@6bPLWoc5S=q1D70&jao^?Yl>JN&z}0^(*AO-W!cTU3iloP
zVbe8zahmnUg1BvWK3~~#am`;R<Fd!xTVAHE-|d^Jr1EvX#xCdV9Ix8BK`hHlZbeR)
znW}if)X`gQujh^3UjJtoYgS+K(|WkN^LJRpw$2A;&icy#_o`~mDzw?=5Iw<-eOAeY
zk8i^S@BeIiv;J-o!!C&@`nicydFR!%I)2dCzc9h)wEdIM4y%M%t4sSzWmRii#s5i8
z*!4^?+w;frX$(6b?_gb8#u~=B<jL6xHTQ*IoAu{BZC>Y?{8hCyzVNls9M7Ms0h2A}
zhjL8SshU64<>f7*YaX}KYPo~jSzjNSIANOob#D7x(_Z^k%sTwKG-6js^=a#kdvtfV
zUNbQd{PDMDgW6`FmA%ZB%?q+Fgoo<2N!b0Jpk7dT-h}Cc)BB}~t{Dd!k43&eUVG|e
z`RVFLi9;E4OB*+8U%o3E?|Z-er|INBR!i)!d35DuCaqj_|M>kA<;=T_S2RqkZpn2m
zjGdx5ciQjv3oeFhg3H2LCKOmX&b`6r#-Ceo`E0<S$BW;(+lZX|ntHj<@cX-@x6jux
zZT9Bc)MVzlY}NX$-o7ule{Q(_?19gY@R~2P@6L!_RakQC`Y(~U_t;J`UF1?*_o7%i
zciLs)|G!&5_5RB~-NWi6_PF1Xb<*j%S5mUIt^dif&rbLEF?ny-FKn)@aWlN&z$Ejj
zXO`|=ETyts$81U4p9T@`AG6=T@vzF12z<jHu*`fiH=C!}p@k*~e%<AZ>Z+LK;=0#C
zmoaYM9I^L33Y=SHw@o>0^69mKPol(LwH05#=VfiM;PG?25WTEn>i%noy#C!bTU45F
z)pStZ(1K@G;k;n;eF+(pP8Yo2qx#&w@JR9HrLWfgyVj?tvR-cg_l|tuDH-R!e!lVG
z@|P=IO^X-Y-OqHT@LNne_mj!@15GbI_4}v6-_?Cyh;bXQ?jpIl_Y5aG@$a6LoaVAl
zPeWfs>r!O!4EC*`4wlY4xN495Wbdu<zgMdnFYJ7yxxoFsmelLQj*?r`ms~w{@546^
z`9mvo%)k0i-+1Tg+wB#?eAk|stLL}bonG%1%6~T~(5*REYSTB>+ffhIp1d#=+uV3T
zIv{C+P^*a6n!nEu?q+dxT^-qK$dl|pWzXJM=U&{ee^|D;XMuu>%S1b7|EK&;y|Qa=
zo_U<Ofq$~W$CDgPcXw^d5w=_ZzLbBjr|P#ji!XDva(&)Z`^R05xiZ=Q{joWVpT&rK
zPBIcX)W3yWi!JtFd8+eyt}n&vuctbfH~h$7tysV`caDKy<=gIJru{M1ivwJ%bnd$S
znRPAn)jTetx%cNr3;cbz@pQ<E*hsZYKdxTfmmZO;ccw#vpJB^FhL&a4@gA)c-J$}$
z%s)n&_nfI0T>k8cZ|xs<pIMVv-&p^ap=^_i^~}!tvwLp-dm|cZb7HN7z|o-gTdLOe
z?`Ei-Hdw!R&&=I@{wD)xcX(}A*7@7|c*%6b)BO8=Dh{#Pm9_l(6X>=^NTBon`d7mF
zo{zhWIlTjBFIzMB6WfZ`1*t9xZv5P0)gJ?=J~m=K`?sq9n$Zk3$@QP-R`@zCF%UjC
zOM-27W6MLcvVx_{f+}n6wlr-CVmn?Psk{5p)^pYjiP6i>%n!&{?T@-w@$z9=?%jJv
z6FN2Y=RJIMr18<YU#by4c5kHTo1OgpXb<;CpA9q4zWsl{SRk@4_intP|MTCT*Ib!(
zv(9)gn!cnxOyotw;vb8iy4d9~IaT#MSQ2w_*5Ux;DFx>ZlAdgger3K;$m`tnogVCs
z$CW>?;hG+jd}H(D3IANIBkTp1-2e0_#7SLyaniKVNv_vawoR7Vye!ya;k-%*yLpyh
zWHi_}iJp5j*Y^HJRoNBCKL$*D@@?9Bi!C!#FW0TTJi~V0mE0hys-T&kVK3&LyMOk~
zw;r_uneXprPkg;(dWpvqzTDfn)s7bu7^gqjtM^vAugI-Sa?AaMit{JLe##u?Tv(KJ
z<ud24HCOnWKgpblIUsXY>#CWI%QyMRRqNi0Oy!FZesFwMZQqe=nfdEhZAi*F>;C7)
z9la}-eQWRT{8cr7pIXS&@Q{5i=C6b9pTELuWp(gJ;Uhi%%~9|F23WE-f8DrmQiEvd
zChvMyk;OVHoCizN*1zYTEx`X#Q>&srW5KqXp1*F*U#H&}x}3A2Jx6VU;oT{ZyB@U9
ze9F`Gbo#EnbEf`WyevL;iA}7-&8C#<7p@77(d&LSNj=)g?fPch^37^{#fvw->FJWx
zWLa`vCq}Dzsqh`<YV+TWXWqnCC%(wqt0z#!xpiX0mi=eH_`JER{aQg>bb+=>we;-|
zzm3Z#*ThHb|2X=QS2Ja{hgP1-2j4d*ZKvw||97Q&irdkE>Hb|Nf#yDwE0}Ia9dbBs
z`r~NXij`7R|DA18uR89~^khQeZRxkib*Jt~+3=U)R1fcUK7-u-YfPjUEcm)M>u=OC
zw-1X2wN~8_I_y;-b6|ppaQ27Oe2m&53z*Gg#FN-8VwUN>iO9JBeQQ`<Tj0&J8@8s-
z_iQW+shfQ|Ve%m!H}6FjOPL$yz2<C+di5#8j&<p>h9$qAWi8`>GGp!EA0Oik+WVvn
zN{pm5R{U!EE|*_XW1RX;K2)Rq|B5Fc8`KN$Z2VMLAby{l(<4xPOYOnZ7pY5%|If(1
zwb*&;ng2zn)-1^N-f;C2>xbJ`21+YbBc6+Kxv~73cd7Jcc;E3ekx6a|JO6Mii*B}I
zY~@jUyy_R1<;4!G|C^^Yv}Opf_uhz8KK0e%Q{f^<O_Ld2Y^x2ur)sWwv*Q1gZ4({7
zUn#oKR`WV)zG>|v_Ae`<gN&{;%WBO(`N{BZaO)(y2~YZ}tFPaBpuNQ`qFgYCN#f_9
z>v?{YyF}IeURehjPPu>9-0a8tf0`_N*3Em(Se$z$KWzEsbV+mhr>=R&>nHEN_LO;w
z+BVbmb{mdz96J?M(edxXif88CGuyeY{^WS5du~tNYX5zbiJ!haeX`U&DAXY8F^59N
ziZJns%fdY;_-D*7Ti$hf<0Jn&kq?7*bD6As@Y>GkHW#Cghv&nJCkD5J9<vGLB<T7r
zIWyPu^?@XtdUlJFru<Dy8`oWSNWFD7kYkA(`<(pP;sq^naR*{0oHMzOcR0#dJ2!Iv
z4_V;u_M7v1z@m$~8AsM+#k&TlEMw46S(sKPBzN^mA@9B3MNf||vfT9W%-J1*|8`Hl
zdpm$NxKJs6Y1N}Q8mgU<*Q=f9y{u7ZxtCPS;uT<^^f~tEhM1i4&*o)Uc;2ckUL~8O
zAwK1>Yhlg2V)eEv&%>e?(c7(7ekpb7p6jqgNJsY6%FXFk)fuVhb$0F)+qV7Cy@(sq
z*^$*x<e3%gIJ4K^j#E$WJ*8K&*Y0h|n_l-d4FwmaWtTk)-Muns(ml@0n_rx$o9W1<
zE6Mq0x#Z)7kd_F|3n%RsuhEfE{P^>RbLYf=Cyal5zi@Yb;|Ar4-LilFTU=c5Z@J$r
zmE5^!p3VBml6*Ph!_;S;*H>Sy|LG^QZ;#U|-cprlTaWK`xn|vG?-broshcvVvCqiB
zaDnZkM^|D!0w%t(n10u}a1mqJm;6nZ8jt@k3v1bF_)Ra`RI_c`@;j{Ulh$`QryG}S
zQ{Q-N%iO-Vt6u-z%KmMeUD=+-qkWs7|FV32LZqf~=A`Yn%=)zs^0{a|GWx?BA68Q$
zQXY_MGCkQPchmhzZV?|-EqwUq%g#2LZRBclh2!2d3#S{J`I9n^e0at&y;|2=`PJw2
zD?fJ5=J+Hq%gF3f<L9FN*I%<Q2|m!-r7`<BpMKAV6Gn<vzkk+Rm>)hB<C423K<Ti(
zhJe|2(Oqg42haaGKIO#&?Y9Tgq<%3Ao@A4Z`+4sXy9ig(#9oVgBCXNo5-T$l-bb4i
ztT5^>OnSY$Be-tkUWOAVH{5^by)&bBeqPGm{7ZekM}E59t5bI0JF%pmqyA!yxt`^n
z$=a@)_68_y{AiXTtpA^-@m7$VQq~Tk0Ig3cp*KQR|Nr{9O{#I#&kNn=PkBEa7hi80
z{Bd2^;jDw(o=v*2?7j@At7)a^60Xn)9?>s5xSEBnR~PPSobYbxOO5{@=1y~}Ug;3^
z<k9Z;oN@<_3(T|+dKss$d&r@vhvoXIh2P)!s2ni;bW^K2K`k!Ddi_d;Cn6d%wjZ4?
zsN2d=y#3j}`u(CEk8+kh*%xdi>%y{Ok?`k5zG4^VugSYspZNVkE!Vdq&t)r$8GkX%
z{E+bQenjb-Ni(ZcZ_Q+1I{Aj>M*iws{i4Pjlhm%2e(LU?-1+G7L&X{^m9tl@>hdE(
zo2q_n^vs=L!}@PROYF^qEA&5XZw&R_)iuj`)&1*YKT`w>z1Ou*VV`P#t8)3?$sZR^
zoUve{*P@9N7HP_tyn2}JuIZ8BG<V&0ovQ_cSx;OHo-Vn4N$IFmqGO-lUGoPrCzS(4
zI%lk8=~I8Oo5?8Rx$M1^WKMJD*>AHauJ!0u{L`H>v+P8H68E!jti2|y7rIBW9@x~x
zeq(?7lg~mQ)em0q=*V(^6rwuge9!Hk%e{TIx65v?j8S#FKQmyzpWnSShK~pBS$VSx
zQX9iAxF?8fa-=WFV`KP!`8|7<%Ll!WI#HKS^|#GSU6I}PG4VzEOPxQLk6hzQdwMQr
z?dJ<6liPgDcExKuzwGqT;|;yF(C>NuDWjC<u5~e6By%H;4Szmc-!0YV`j-34Ntuv?
zI-U;4w;!uoY>;JWnH|pBBVWA9tvgwJUT{+KqVs)gQ?mB%JL(*;cP2yPE!i~-KK^>K
zB#^7GJbazVr=$Nhi;Q+DrRK37TldE(X#X;S**m9h)s*}-^PluW%cO5lx(|wKJ<VMF
zPEhXh&GcU0n+gICO=jB9XnL)dyjsbPYv#NIXC`_daa*TpvOrn5)`or7`CE~<A6V>9
zm+RmAYH5?k{@@c!WX+~0xpMiq>L2uv-Fznh?2K7IFYy=^tvKx}eakH^b;C449flVw
zUfY!am-b}1=1gvMsGPcV;@qI4MqAcC6G|)JUwm7Aiq}oCnRnVOj8bkY`pl8wo+=Q1
z`sa!2S&VYV6O(Qz^)6|dmbv>)Zu7op6&FQRHfnS)Jt=<S$(`S-##O1`&NF*1t6T7D
z`R0;kJ<T`21sN=gp34xc-gMnqXNE<%@`bC%HVHo4$Y%C3Yd?cUSh-)M$(zFc%a)hz
zQCX8-{pg~sjG)(!il6^jmRx((Dfva8r(o&*JqB<8^E7ZQnvrzUuKrCZ<1xn@eL+Vh
z1y0Ks{EPS%*&NJUZezT}`1kagCpWEdeDPMQscC1U`^vA*miNvnismg)IOh88kFA#P
z`dv#UYb{>P`f~8Ygc+Zn?2FH|2=#gC)w?v^GA-)b`Sah`bDRE|_S*T>YW1_Vc@-xW
z=iIj0qi%M3<B2Uhty)UjmEVWITb(}XX=MMjxqn3r`krjhdl|KFUu4|$)O$M>{y#qV
z@`locQ|s3jf1dOCYwPc8yYDkUdVHOq?|h7RfM3u-f0y=3p=&3LUhn(cK2tbg{>S6i
zx9ug3Qc4`+CRg0rY4qkqRaW}HYir_umK&S-&ARn{CFhr#*6Yi)*RA%hDgA5r+)T3S
zuj-8g$Md{>wY$PXrq5sze=Edg``6n@hAsU53)ye(2Y-4u+5CSN=5*w!`unuHqqpAv
z-W@6t?sjUz4OyX6%GYfau1$zLJp1N)TTLgU3b&uj&U=M>_RsjeN^#{qb@hFs@~@V@
z<7oI=lE3xU^edNwCN4ELzxT9k$C_s~nl4GsOiXHu^XBl*cb)NJZQ<Vcr4uxl<%k$w
zI47MWKT~6d@S(_eiPKtM+&O&U!n!h>gM1f$GjEnu$jPbU=3J!DGU3CVhZf22)jyr{
z5KUY%X9JIzgoAG%bKeeYY0tV@TV~7Iintg>tnji}XJ<OW{D&{YRj*dR_u+ZlGg_Bq
zZ}_?FcDZ7X{CSN7J1<(!j%F@h#B5z4z4ObIuS*2BO#Slxl4RqR0LCcJpcii&K6+?9
zl(csU>QYSEP@dP`=iOz$=5yb&57pZ%ec%0LyRUZZ5$pEF&Sv|0Tz~HR!RY!ktL7ft
z?xm+H6VH@aYKm?N6lIE8&-&h@ecd7P@K3uI&+l@#UzT#)k?Xbfj%Qr=kFmuoD9qc!
z;NUEOf@9a6r1yI&)Ngfj$EK~6+Br>XeOuNo#sf#TUfO!)$^w>OYAZA4uD$+xblrN>
zdwy4<cm3@8$?5ucoeZ~owWQFe-)<LqTrKkonPN71?B16yaW!Ib#gcbv4>Uyg1XXyk
zoL;kr*(-cGPYHW()D}Niwv<)3Ke~C(3c696sr&JDzs{o-IkA3g_*GuS#if@C#{7<!
z=hjNfC{R(n9Th%3@A~oIO+PqY|683AoAKW7BWru&n=1kVwU71}p5t_3J?itX^Gqir
zpH++6@`G3IFMK?Ijpd(m?is#?m;bt7y%u$&BIMnblM@Rp_PppeyRy=@C@XkgLc4d7
zclL~1JOBFx*ge{EwJ>R8uvE$~^_^$SjJi)~%-CbOXvZHf&3(BV=N)v|wF_>BaK7P;
zeyd-9LSUbT%H;L~jg2L;`_9Sc_Z^$2?!>U<^}Kl#RD(}_H1Al_U9DyRDdgHwuFT%n
zyDmCcm2PLICUI&yJmb;Gn0?vdsDXW#q2{GE+Uu=7tsf~nIQagz-*eR3bzw<T=ryVP
zYk0CH56x=ap=rXxeMn6{DrdgP9Oq?cFHDk>@NfAp^-!}pLE7W;#50;=(Rs{2B-g!Z
zymofNUehZRFMd8}!Rz*m<KMk(efz={G3x*N83W8>9zJ>Cz3le!lTweT*)Iv*^7*L6
zLbgqeLXQg9C<d{3y*(t|6T9v8k&rps291aMP8z(N^la)H2X)1TlealJE%CZ^?76&1
zeBze;k19Ip4bS-wy*#XI`jgRRYs0FrHA^OToY+>rBB3bl)d}0=_Wu$y|F%f=9tktp
zcw+j(c6%u$_O`hD3E6({BTKaBCx3U)*m0rW^wy4wB`Z1hPn#m;w&L^p$i?XsjjX(-
zd<tK9A6RzR<m+;siA`C&C%W$zCEhssO)5E7`lMR@jd<Y=-Pu8lzUPWWu?feX>t{Ol
zj4f=6?uYzCqN3qfm+Z5gvm(W{`q|D7m+vp1ssvyDYIbf)P{7QqMb!a%ZbA`Db1OE_
zy|X2o`OEd4iHfYANB4%lHaaKrSm}(({@IgX=Q)TS)>)Inr=t>Rw8Ea1VZ);0@C@-)
zPaN0e%B;Vd{j~HA*Ww;!wWqOWUQ4G;K5($PMDAlonaP#<{X7%x>y<xgt*ts(C@=i|
zNXtUArRwWy0t5^v{fyGr+xKJ9kHSx8ze?2qRH*82N;&R)^h;&To{#mahxT<8%rH2`
zb#LK`V@}rv`?$JJbq5Q-xLq*K+bz0I&GpaAs2`?NH#59lKY7V-sX7;DbyJ^LyE02Q
zINcR`>Fa)Um1S0j)Ysi}?}^Ud%QYq1N;V?e(zT%NlgXr~=>>k?=R8*#MI80xnIOFD
zwWRbE$>-wRGo=@2C(Ln}d&lotSl*l&MIP2ki(PzE_iokvn(3AMsfTNyp5OE887qbR
z*9q(Dh~-Xfi?Q1K`jCQmI7@Jc$=0yXMUmTt)dc???c6PA^>zy1bTvCiPx*_k+{z_V
zR-eB{8Z(J6ow4SPzV-r1bM=*4U;au8)C*qi{%~TE{q0A0!%E)RUOX^YJc*D0#ICl;
zoP6W*^3nq)vMo3H*IX&oeH?Q4%zphjLAAG!7BwhO)VQX6SzO51fBCO7-N7vnk4kOZ
z@cGHn6SI8RuFF|JEo$}sP?Mc|eebE}D6O4n5;W6XTX%P@apsbwI+-0Qle4B&bFRF7
zD{OIZN1}4r>Lr($p4mQm^=m<z`JVjkKDRGiRJU2XpL6Ta-Sz+T=P%gsYt6jc)8<Lq
zi=N6RJWZKAX>viphPUn~R;gG1%Ubi``=u#L(^jV$h`N8;@oDb&TwTwv0h%XQu3>SP
za$GJ@@TMm{@7KR?Z+s$EgP$FlVmK*D?{UxaZ;daxr6=^w^sDsz$@ca4B&!t#pVL2k
zy`Op0DJwYNTh#aX+ZX>$9<DN!QWK6nxx#pEL~5{l&!P9sH<ucm>^qv!t>oeq#Ux-I
zaxPq;@t&i-%dxYkrb}chAFr!3)-zw3R%F{f=~+m?yF(=*MPC*?E?sC{oUG?}VP}rO
z)A-HvLr<Jstnzo?qT9<;4|;p#hwXT?Y)^i0+%}(8v%893JxkuJyVq{@o^8G@S68d=
zTP7FK_E;vYapIK(g(t1EHXfD|RbL=z)*&La#UOI(f#+^Y;YY6N=9WA0{Cj<nt@=Cf
zO#hQYm#6Jqy20*qncU;uvC}twzZ6*YTp_7QNo{UQxY0V7qMO}&(*lol@g6_p_bOIx
z)~wmC!v8M)OkOwB@Y}w%BAY}^A71xTxFY)Eie`oDo1;;}8ed9RFOz@jw(o)P8Rinl
zt80=Q4U*g!^;$+v57g91oNy&ow>Y~hDE;XA0RK6%_kv9&+6!hgJZTSJaLGczgrjKR
zyPYkCL0X}ouSM#226Bk)tW%q+cyL<LEECbYp;r%P*Koe!Z@et(A$DWs`f5G{<xNH!
zw(4hD8+AV0O<-5L_euBZYmeM(7b4#8N}rpvS-IHDT4(V#O&9saC6nFcBUB9!d`gpl
zy?6Rgw==v-4olv7d@NbLrywwQ(#EyF=kKdfUlGCS%e&xd9e>H>WgDMPQD_x#H1}y-
z!v4SS9&e+Dtm!HhyGL1GGh|m@{5I)kKoi$@mDj27dKYi>m}S;ycFlkP+kdy?*d||U
z`!A?a5iOonB)XJ)5|@8J@6>yzj7(jY`spjQ>?>eMox6D7UX|(~2Ig(Qe8Uo!%&+_U
z*K4EQb~|6SYtM_P?>Q<aG(+n_$hP@SPk&3Nyp`PI+TnfAVP#BTK~3`c&4mS<?>y)z
z`}?B6ZLSWhAWLk^hCgfOEj~K!wUm!#rS^5cfA_^_e3A9_VR2n{;LVX46V=@up6wF8
zVz_>_^sKky2VQ%ozWw^*n04sc?4M8NZ>&G~a9Kie;Jnlr*GtFjf@Nn0i9EY@u}QD5
zuO!e`@`}zn=Z>@Y&&fZXH<{n_x`On{GV}K(Ca%|2L~h15ztghM@;CA8{km)4tV!Lm
zXSbh>ttow)U$jjo@HXGvYn{j9CjOm%<<itx^TpfN%fHi9Uv}0*ah1!_M_y@34oA1F
ze3&4;)W=t`&Nt=OHQp=13%)Ds1gUcU);#);E%yKA>9rfQdz>9-e~jI(Dzmu%S8>M1
zGS3U^WK|w-nYTYYh`Hgj($nTCJcW<$h-;ow)He3&esgPOm1RpY`*VHi^Z$3(uTgwy
zc|qj*W3QUuUzJmj{gRE{ntR}e=cKFk8J?ExX^XDj5<GVFy3>K`_}lHNXE!8D?bD1+
zJQ3tK<-yZXo8M|HwlO?j_T}F0twxtyJeV`Sy=Fhub3)o7ra55t)9G72+D_dYe0*A^
z^9f&#$9->~+z9GB_TtFp*x!*$gvAtIPjboM&>nbEr|x&!?vm5~6AqQF{n{8<+;qDo
zNn)Yse7`fCjl!{~Rvp~1EyYB><xRr0zrDA&Xv(Pln_|q)G>c~@Q;p<L6Hj}WUB1fa
z(^_Uq-dx7~?aq(9BNdXNE_?^4uxey`o;7zq&*Je)<-x_GSMjBPdGACsPS<A3+V$td
zEs>b$n+{)jd$}t8J_RgP+%UVzS?PphUzK6-e~IYx1`-BkmpT5-V~SThT<3Lv+WH$3
zt#V%;-}I=P!u3a`F7Bb4gq-N)rzwli9M;^^rE@MjvFs$Pc$V4pXNy~=Z@*MoCh%~b
z!#wt7Qahe_1gAXSz`sS4|Kv}-UjFsPKi3&eEt?_Yw@XX@_RMOR%vH{v3Ne|L&-5gc
zqMCHFI{%l-Ofs*nGv?E-Oxr!X<w(pe=c5(fTU7Qbd=GWe{k?nTUFO_#EA&*p+ovd+
z?B`h;HS6w5qfGS!uC?AvPhCp)<e#W1G;w0Rz`aSeU6X{BLk<;sR2*rOHVxV(aoj7!
zF2wr%hKe`2IW<KaGrn#7JDsP!Du2f*lUfx6$v0iMG~AgPk6Z3JB%3nbrT6`h*xMg>
zeBZbFZTcBgQMGBEvL#j7$M0PzvX)r?ep~XU++A8TXPvM8uwL=qtp)ai)n2XYzlG*V
z2rlc~t188_C-kzZbMA^38I@`)qvgu!e79G`wInQA{c=S~))aqJt<CG$nf8QfZTIO7
z4^PPxTQN0$3wJ^R4`28jwnuC?pGarR22FT&P1)m~{6((o{Rie-?|T)%65FI?U>bgK
z-D{sG4<|f4y~!<ZrpEWq72<DC%({3frSYT7HTzVH{aiP-yS{z25Nv*trGK=B|IXB^
zr>#!S%1XW5&FlR+16R%!pV#vtV4{2TDQzJIp|$h;ZiX^FIM%oL(S)<L>Whmq#4@M5
z@v_w}jfhcF-Wu1^ka~yrTfi!%DQy=IwQ{ZNKFG4tY1LMZT*c42T61n+_Z2ajboXqL
zk$u<kKUMwp-~I3S-PE}}-BeUQ^_|AUI|o+DKK${z+h?2a=8Kn7qQnAkIZP^C<yp;}
zleSKYZAWBOUgQ2RN0yW<yt(Uuk!aIhnd9Zl-=6jIu3Y%~v`W7fn;v7prlahgM~j@~
zugwbj$v5p}!puF9I<J{hi$iz)))RmBnMZ2fLPfdKo9m^bd9M2y+0^H-uTz`-ZAty_
zu-*^Cf;-+hZ{Eh^aY1Y$&vMBP8|wc{eALWdalLgiH=}rrbf{9NS7gggoeaahPv^eR
zJkaOXSr)nHYLDvaJ>`){D|hpLJSfPy&N^IZXGla`z^Y~XN1xpZn!5DHrHQ4J_eJQ{
z@XGyK`L#9co!sk{Dq9zoOER_}GZLz4yzjQ4zvx0&qqemC{Z*&zC2p@v%KG~AMAhqU
z_b=E-UHo6G?xV>ce5$#I%iMNXWu~%x`=|99LgsHaN1vMAa7FrfV&D6{E&*B6OuBIw
zyB6M%T(&!v<<i_fd8VJdt>1&<)lbC;N;Et+KM|gJD?DU@SKoc_={|E8Nw%-pXq|rR
zz~mz1sY2~1@0IKAGh4NA<+h#a7v;Aa=Cd^~^YJxZJL5!qBB#my24nNfF9clt`qw;B
z*m_;z-{Z~&Ps_sA%oYrJ#PeV8hvCdSIjJA5b9QPk>}i@+QP|knn9>@y`{y3T+^;dI
zFLLE>Et*@iL{@B7*=FT)yk{9L`6QUGs7(GNrMdiz*z&UHC)`|uf}`&_m%6Sm`}9j)
zShBUkC}{3)A%(W?z~UmM1<w_O4DUSJH}8{3#PQ-C#XpWSZM^z|@qVi1yqV7ZUk;Zq
zXtiEixW?z|WKQG$w|#7L-(2C^Wwk(LGV7KZuVz&R|GMp!z#1{LZkeL^ta-<Zf(+K|
z`*>4z{eRwtr>p*cS3mpv{rQc}j4v0sE1Pc1on^7d^Jy|?vzjcIQJIW`PNs+3F%9<I
zO;2jHcAeR$t*$9@{;i5?!Rogewy$5-ynFmYsJtotl|n9?^c4@|M`eq$(|@T+tem$l
zkLm9ps~gMSeXp7~*Gl(zHgmmO%%aJUxX<Z{r%zzaT=zx#Zp+qLul5M-dZ>Gk*`R;F
zWZ<WFl7^E~F6zW9KcD8u_E`Diw28cD7@6;W+8Xn|qM$i#HxuW{lQUn|^w!Lqe?}<R
zHs1a3#ehiW&2FWCZ`@-@w|l)__`dpwL%Pdny<0KCeoLG1)Hz@O?o@9T`(41nXLLwg
z_p!B*o>-Kn(Dt35Uzsd_a9Hl(yt0(}0pdFPOut(GXSc+qm8=zPUODk}@#I~zT_Te_
zo^SoYB<;TBW$BWAGg{LRi>Ni+@3=eXFRLZb%os26159k+;x{hh5t-A%`0)Wxx#X^@
ztOLu6-Rv^?q<y4c9GPMxUMcuLyv9d+gFHV+qlD2n>4sg;vS0jvs<!`L|J?^C)kGp%
zTYREV#~-&8_;&qJ_geL2^Nkta0#U*htIn>hO`6tvMdSXSm0d|)T-zpVZb>iL&;7oq
za8KT^h1Kp7#oMah?mvHY`=%qugaqSEt~5GkuiNLl@9Oe}!JM;yEO`-lU`?&CR!B=%
zo?-EaS5h3QTT6Gn|8v1SywV^;+<kHTRL8G-u3EUoY1HRuv&UU|xu|2-jxU$woz3U}
zIkjyrE0>Yu@1q^n=UCMLPM;co_}BK&tG@rWYB1h4;j@M59scceL^(H~m?=>3ir0Pl
zsmqG*<9E6ioIMdITf0*(>D>LMw(8s8yWYjT-rT|6m0ulo?)OPuL3O@O22-BA5@L)9
zS9Y$K`nvCtopbG!+e=OvD!0G-yyTTYV&Rku>2NzyU)~var$`H%ugpp_IJj(2#rpQ$
z*PomINxtuX<nZ%_NjdG$_Bg4_&p)X<c@O7{t7%)B?@WpL#Hl~6d+BOk%SNk`yNgU0
zDV$d7YhLu!WX{Jgoh_nqdExF)w&^;*mr%O>xJXjr{1V@L)~Y{Zre-hu{i^&(o9NCQ
zZkx-O7Jd@%_;PAS0NXzOrv^RYajcEzawhHUIX~{**l_8M>H!(X%>hwrJo)+S?e>@b
z?BNbyaNF_!xkou7sm59xZ=OhQHF5j(=Yur+o%GY+m$0_q)-1ODzGuVM$9a!u|GZQZ
z6e+f9-}&yztK6pceNbJ#oO_P5b@lU!3r?Q5UTOVkTcwzwy_Ut2ut_an0#sKCpB9kp
z>3v)_wK7a`{>&cU7x$iM9eMihK2ONyi()bB|4WJI|A<TcXIji8AZNiiaj|&s<I*q7
zE~Ylu*l&J4TVeNn(dGB1Ec*~Bu+-RH`pTVSK8h?=O*TAA$By$d7M#g<v!1&|<EPn&
zglj4@weA&Pd~kHdDT!ap)^_~-Jvs5~Ny*QXXYPNJH+Ax-#ZtA0-fUuO|H1MoSF<tH
z>(L?0q<8fqR&~OA``(65zE<yDUaj_T?)_9|A!+w5);x*tuDmXvp3-<Yo}Y8o>@WHW
z$<s?ZQ&kv)r+!$O)+_${^#8?e>mPSo%bxb>*%eSZshaQNLLq*|u%A0Mwd}&|L-sF@
z=}8ms@|nBHPgr8hAI9ryiZ?wX!p!_GbLAFkoUXHXb6QqjCBA*;mR{2tQjZJnS}v^c
zvg$v#OyNPg@T}^|>h3u$dO{vMT6+&&p2q!SlKwHx7i*kOJY3>={lvG;5#AC1&fYV>
zWcv2PalN>`eG`t>sR;g9dgjf>^8%?K5*IODJLx+oM_F#c@<kkM{N^Ho%!1iDpGptw
zcotos8o}A|(PxfS+git-?K?ld@LRwq6FOUZ?|lPy%T*`5ML#EA_<VMG`HhEHs>}>;
z%&^*j)Zx5Hd_cA1kLz*CoEPrDdVJ-B&w(|+y+t2M3w&gI+&nGgr3b@r^B3<KcQqX|
z6ZvrLr><pomb#p&;fd%o2Un{5AOHHfH2&|q2WQx$bI$U&?tCWmz;a=9W|q1l+YFBQ
zo!>56a2_dExnZ$i^4gmTA#-QVY|Hzp<(5=%Aneobs@#}$`FH$o<~=&HN8zI6f)f*M
zU#)K3AIt1DNzc@O!+8dC&C-V|))#jstQ9Zq-ZFJ{_RK?ij+@V(n`--};EC{~O+E^C
zM(OVx^q#y)S$%=?O?CT?M+N16KCU_w++_~7+I`B|7q=s6bEm&bM)+OlAF<m6ShnX@
zX|J&NW7)Ffl=<Z^s}%+MyLI;!)J0~nI~+W^Td7Xv-IJTORjH466zl$}s5g$C=JzO4
zU!ir=&1|lJZ)d9hPp}l6zLvZ1$D1``wXrcC3ER_mxVj1bl;%3c6J0+i%k5OaUa>oF
ztdSC7nqtcivKih`G@JZl+5tzSF9!2HZC3rUZoSdlxLm=pyQzWY<pthLDf7QHEctM4
zbE;p?O#xf!%(?xkhgLEcEtD!~PCT|-)nUs0bpIEO4M(K^=jE1GPd|KcSqp>5_UGo=
zWpAw>zq&X7Ott0XOH6B5AJ=L<{Qdg2x2&CaiUdXHRNJ2ko;};pLRR_5p=D{=^JPBW
z-WWA=g?Q_4nP04($J8HvnkDm+>u68!uXE8;g6>5qOkbg8&zxi4&c%2?{Kn_cg{Lm5
zc7!l9{kK`0Rjk+2U3T)roEtmC<5pd^d-eAMN5<q+b9xKobFzfA`VU@hUj2L9!>_B3
z&ePu!JHP*B^1WkPA}bde%I)E~V<OyF^RszXihEEVufDWU=D}-=)Hh#0ev<py^-QOv
zmy@2I=Iq<{rR{vF;kLBXQw^uP&e7ada_3>FWS~3q;~m*?TdGwy|B8CfV){#`Q>4X#
z=X$7$y8p626V2A8_AgqVDtpTE#~$VNsTaCew|LH6wq%|=<Ars$86Ph^Gk1KRd%{XQ
zsLI3fl3t?h@oLFU>DM;-PuqG$+oDKAaM9ezN*8@E<KVOxjuVxytZ4lz%*HF(d*N;V
zT#ZAkc#L8#EKW)B^(<QXWbr1WWvtgk6K|yi^39Boy%xqG>Mr$X*1zvJu3E+WJW~_A
z|9z|8B^QY=Li^oQTKY51G<LauiuVmyKEBcMpsCYQUcGf3alBkNr)GwKy!fZ%)&t+1
z_`G&CA>+`G9T%^;KJmJrv%=Z-O7i!OUq6(vKK2YRo6@qy(<JM`XUWQ4Prjc2rS8TX
zIn}`X$p2i46xJ-&%$Es)CDwh{W0tLocbCyxvS5Xg^IWH+3uFwhc}|JED)KdcdgpAx
zBg;2FnlROB+BL0f1<8?)Gv02}t9rcU-|N3e40v>A?7JzaVqq3x#4MbY_vxrtlF0tw
z=Em%L`#1zO7uQVQr}LTP)4JX#u?c)<w^v@e%-a~b^md|=Aphnk_X_7a>V-;uvEgL;
zvSBOZU!&G77d(!Y{4Lv9ZN$4yrG80r{WGUK;y<U_@Sc%hA-aZ>L&#L~&U=oj_wVkS
zR68*vs4I2N^e+ExdGTwNrcKYUoz3i46szt%S>*en({neSE_GV8HFIx5&xEM+CwDG6
z(_3^U-{9%3jML}qvsRq8F5jiyT%sGcKsh>Z4Hs8J+v%f%W%IT?_WRW6C)B9Cdk%B@
z%d#DgyqA|IdEAMRH#eJ*7;xuy*^eh>Gg)SCzTc*Q>T<tZS&ZJUpMw9BxFr8{de8Ux
zwRd9Y-OK5oezE(5F6O<?`*rVn?8Ou6>PxnNd@I_gH@n1T=LeyZGq)zR9eWx5@@uHf
z_5}jpYZuBinZII)o&B=2<+Fr)NQ8gs&48ZV3k(za!uWfI%XuVud2^WBFMIl)h?p7j
zU26XAES~2&UaR7N>m^%f%;7Qpef0K1@re@0->!|dewkx;a?<BBTQ(P+jD63&Zno!y
z3)PNySFW&By()KQ4pUwH`lbGz9D2O(l>7T+{C6!VNNRc_-8a23;Ja~I*biNXz4vQG
zZpeKo6lUT5S*vF2@M7bJ|9o=ivZ6ZW%p=y!u3WQEPhaZI^$Zz(Bc&}G8(Ozd<N9^2
zMa-@={{E9+Yj#;mtv0q;u*Ye_W4A*m<J>ed|CxF*2m1Co>?r;8X3m+tx69b)6{*`k
z2@#mv_UUi)eYYnkCg^Bp__F@HqR3Y__1}l=-vzxf^P?7g*nT=8w)gX*x6f`pnw}XN
z-&LF1_2vCbxxk~>zNN}bRBfNHS@>D~<-c1y|6SyKD82N~)EBQAlsw&SJ+`p_s$T5Y
zIe+F$e;ZSsxCgu!b<IRiT?lz=B5IV7%C3<k>H1#7_%yfdn{8%G*YxvlJy<VbdgOKF
zo#rVUo6ife{0y4b|J*A-;{LytdZ&{OuHTONwX4Q`M{&dI@)>oLJ`{F;EidVsDBU6S
z>4?_VrHOp^>$%QJ2H7vwFk>|~nVFv!r8j5&Zr+zmonJj)et!BlEAu6M*N-FyJodVm
z_%*8IbCkG`^|WRAXAPvJkLL${NQ;brdDh|cynny;X(p%f{M`D_<x}=){#V^C(}F7V
zlGbx92u)?v+Os!3?B3U<*LJFhUM=3XE_!GFz2()`|J*)%I~iVyJ-c7Bv;GL{?Uu))
z!J%I->#kWVuTbl;;mP(xhJD`et}czS*?s1)Zo<Cges%s!W@`hPw@J=45`Vm8EpOUX
z(epFX4o$oJx^Uss{^jr2*UeQ4be>Uoy($0E|1#50Cmnl)a~n_Z`q$fCetUXJ)Wih}
zw@cjv1ZMc=OrAXLiLcnCZ>w&+^$W2GypVjlUiqm1`#E*_y;DOj*6KxmHVjHheUVkY
z`pm^e`K4bb`-vZ&mVQdXqMC2_`j3*^J5#?Y9*s<Yu^{SgY3Vc7y>pMu{yq2KhUp&H
zcKV&0a^G6g?Day4ZAWb8Fq{Z0%Ik8SeOo!x@ow<uy!$Cfp1S^uOIy}df9hLE)|%rN
z|85p*$&a|DYyNT<PuSgs>yKX)Y!*wB;XJT4^-bR5<LQm-r`?M$h}#h+Z@J*3gqTf3
zC$D_UW}O9d{bIZ}%dOO1v1(Tw`%iP@^IP9di`lAeT(_t&s?M#w{jFMX_>a0fSI=Kr
z7U?PWYF03JvS-^&y^h)3;iX<~hNg3#tle^?_hP}MTWRdcr%oKpRR7JbG;JcoT?Lnw
zeu_4$QYup?OtLfn_2G<(+{-T-M-H?1bvLuCT|4w7u}L!0<}Cl`!zc7tm2<Xs`($4F
zu5Y>H73-bODbrFnY6khm`SHf3mKcXR{*ubqQwg5#^E}x)s(w22%)JeH*Tpy-vyVz7
z->*2dQ>Iqh(@A~-zwP@?ozFL=EIn{V=i`%k6DsbiXf$4$VN+hY^3$u!XM^v1vB+Eg
z&bp&saGFW()eWV-TTgB%uY1#_ny&Of$fU*h8PlVEX47<nw>WNI|NLF~+Ve&t<wio1
z9Wx)6ExdHHjcKZ!=LL=9ca2utN!;;1bMNa{*_(@5beol!b(tMs$87Qq46i<A_dBK3
z_RGpGH`^XO(=4f!;V2VkTkN?cd$z!b3Ss5<Q|s&Q{e34Pap7%tm3!N~=ImV?X7yz3
zKK08paz`H%qv)Eo=l(Y_e)|80)9bw{-vUif|07&{n%T`u-xog)PM;`y`0w{M2hFZ|
z<+3PlJM)sMIQ<mgj7@Vd-gR>fo}{%vq^#|kNopyJ<JEWZYAYwrI<SLR?lRwk%D~Cb
zRz7BCmT~OTayAe@rf~oB){uRQxp^LsChS*oo9VFbVMZj2ZosUQwx)f)SIkZv;!5h`
zee`QtckrkBJeQRoE$#WsPfQP4a>w?(Rcg`GW1YOKm(S5(@oV!|wOpRPI#X*s8s}aL
zJf3)X759sqsm9?wHHBqME_X|J%x2)?2(Au|Q22A&?NL#+x?crX)q}17xbOQ%nokY(
z>{D$P5w+#@h&XvU$tB^D(8jkd<<^QDFWzary5hx;l^<%3xXf>P>3`^d|CUgre!H1$
zvFG(ZFF3u*D=v7VqnXJS=g+T<7ZfcLUOA2JXF|-*i7_*$Iy}nJnsO}F(|-T67l~?9
zeVM)HeKDH7g6EqjTl2m<$F6Ppx-Z%2%HbypS0_*GJ^9N_!Y%UE-yPpoPjRGFecR-j
z5P8YlTYBw|DOak@@2YNozFfy+ZT<2m<~|N9eq400H2AnlIQdjfbEKugLF0-89nL4F
zFFe}0X%=fSN6?MaFK(Hv-C<jKZ|<2l^LAb}USEATL;uvVqdoik_Xj@k$@{7|VXgQU
z9^ts#OBZ`6+uMe-M^>G%lvQQye-yUFL+VhL`ibo6%Hb)T0XNOQf4lvU<J#Z*_xAj&
zXj(P>*)zVq-%l+le4Do9=Jl77H`TJ;R<mqRJkx32y5((|=ZT!iUzdJ;ySHo3k|(PF
zbQYZFbKCgy<MzK(&O4<YJ2oX~(~;|cznW(I?!K1s+PvhErlj3gb)UE4UJZY0*i3dx
zOgrg*vR?W8?F?z33zy{^4l9Mm=yMo9E_pJkL-<X^jN4J$3Xj?KZh5^eewnx6`yj1d
zYZ~~r&&j#w?AO}MKKsNY!}+s6t<qpB&bwgRcl{dQoFfcL!b^1H`*WKz&KO%aPoApq
zKl*%q_T;$>vPGw_|1|NfZOe_doR`A3GWQ(eDqNxUye#zNv)GRETq^;~DL1MoB~_(8
zRXZW&pTFhf86L$8`;6Bw4$b{|rDS5hdGV&>a|}ML-sNcRIBV*fE1@-~=NiRWZ1y|x
zv+VuXa>kv>hctfJ`eoe_+&aO5<7x08=a{x#n+~`4&S_CuInU|z4Q9LIH?iLJes7b5
z)qQ-cW_W(S!QC!3>z~usXIt&E&S<<8ntg7qruc)VoMTGPn^h(r<=CL~p`YnA>*384
zzIcBqoMU=}*_f%Rc7xijP{$2g);X!nYu}s_jak*?es@m2bN`*q4?}YJ%0utoQxlX-
z+MLr~#;x__fE()tE;VIwDfP3Zi7`B{FHW-Y<Y{#*64739f+>_a`LN48!A^E*;Ysr+
z6&&B@bIWCufpz97UZ!ND3AuA_aIZS{N-gK`4XwaW^N;Tq?D)>jA{egk6FIq1d$whc
z+K=<)hjgSg0t7TtE^J)!e5pp+4rcQY-HtnVzTQ_Xk;wIN?M`>jKQD@FO#=+C{ci0y
zT2kiC>m{9;EG>95qWoL)4wbEwKVI_t_RsLi0---sJNbUkTJ0$$B>9wS-o|fdAFN++
z<Yah{d-j11lY=XB+G3Z;uB_czJ2~WftE>;F`XP<wAG$P49tnpXi`e~Aa+S%(Y{nUr
zHh=fjy8C6RL7;e^hhLc@&#IW>W2zw>S3WAoSS1xbd+#urndK(ezx%7)CUqUZwYToo
zrhN<de>`@xZojR~d&3s)pGVpqv^Iv%+q`Y<@eADx=7>r@`>by08+A#&HZ1#uzD;e4
z*rzj_w|x42!s4)CS@O%h%j1K;I9^WmEvVj;e*E{M&xf2{9~P{;(!c82gbxmp_Q&0$
zL}jGw7Iqg28h@LeJ~?;l`mMqKn#INMWPhZ+-*V-uk$JBtdqB~{uJFFK=PuspvAz7!
zYo&<SQU36c8eu{IpYg5f^^Z#M+V%R8_uF~XJiWrNo%fljD7r4&|MjhDQ)UU(zqt2o
zi+Iw_P@YZJ<{ClFU#erJre{mPnOB{*W?J>`M7wH9pLxZTT~Ditc<&VdRkT6+)wN&g
zMH&kGj!*o{`K%@+=G2>x)7?R3Uv{TNwjB%oP<O!aZ^v>*w>^{p-aq;8{!;sr%rk*o
zwlAvM=x|uVrhM0_ebw9rOM8-ZnPR(J7I!JOsjx3ByUnFMDeS?f-{#%^YVrnC-fUd*
zO5leb3vcs}pqsN~N`oglXcvV1Y5!%W^#3u>i5I^%cgIAl_dQ+jasK<xWMfuS-ff>g
zpIpOX|6VQkg_>ag@1%;=&&uxYSpDsU!kNQ$oKNbxU%Fko_&1<7dfnqDP3Dqo$A9Ho
zPB*^4@{q4E|2v;5R;G2_Q-1X>E!h~tk#^zx<oE0ho2CAJ*FUymPtuk5orcO$hQ<C_
z#}(_QE^*$qLw4hSzXRE~OQsh1l@}V6E%LgrzUJcYTV-xiZ8;&#0tbFvJjeX1=<ey+
z+cWFhd~fdC9kj@S$!UtTL^89cN{qJM`TT2g7b<5*e?C|AxZ$Si_rqrcE++d1)qhv1
zpW}M#peN6h%L}t_A6zcDGV2iEte#g#R_tSRRO31jlJsfE>h~FjM^|2INM(L|-S6-5
zn-{m%Y9G4bT67|_dgA@pw^*jwS<IbObKT9|<lSTUc?pY-%`&^O?nJ=SYv;LcGF`hB
zvpU$7lh5viqUMxO`@5DGuU5G{*Z<R>n_qpeC-L|F4_I<YVb1l{J6yL{xOPd(r7t+;
z@h2;|L{~U;;@*=dxKp2OeRQaK<;0b1j^*c_D0maN^>RwjQT^)4`9*2wnqRN9hFzP?
zB6el%yc@bE_nQhmBBED(o*tlF+b=k?bUsH;u;Sq*0xIcKU%Xm#a%HY)vHoG(n6Me?
zkE$1)x^qb&S0Zj-<$_<<Yp<9zwM4$iPY8~5yY$j;UdEHqvakz}PPqPhsko$~<DTsP
zEAO=xOHTy6V4vu1mE?J<sQ74i&V;G)OV4qCt~{}&;w`7jmESv$FfHAaXK^Bc%lK>V
zwIde|HQQB^)>(5mGAyv4vPbY`{1h(Pt@<0H%sR5JEV}4g>zT2*;jFs4Z(ii4_v_4k
zH@@vTz5mI{S-K%RBX<Vob(T-qb!DQ1*9nC+i}fa0oH06WtFQM*qfpoJmcy5ZXrAhS
z9V>ojL^Jtu6vpkFxw`c7q5N%|TGAdErfrN~95&NT_{JT97vhJ03Omb1SX}tko|0?y
zBC0FrrLFU_^X$hid3bhPDu2#=VUWtn8yWfek=F{_>t?k^@5}cECtlFl{~`D3haUYD
zd0uOmk2dS3Z#7ZTJ333!t62S!)ahH7elR-k`XHEkXgBwxxerb-p0GTi%c=C0(V^?4
z{pB>y#A~cyc_NKkdif)j4_}I^@ta#Ub8(Vu>_^?;YKEVJb_<o-AK5+awk_DI!gf;R
zrO?d1x|bEdUtZ->(tp+9;L5$SlYE5|xD4kUTB1JTkm~!3>c(j&1bJixeqFP<qs-g8
zVCA8?tG~^ioUC)%Bi*jMDnKynk>kI!H4;W&Uq1|5DZSGEzhIV>#RP5U&A}mZQ{6u7
zw)<5bE8%eX0-NtX@o)P~-#*wJb*X&%AH!!;3`0FqGKzou7u|5t>h7HR`OWz?CtqB@
zA~S22dT{W{?B;p1(t@i$TzPjl)jWOH77PE~<@wq6<xOQ@9_jPMEh?GwLyC1**7LuT
z=k}#^3TMpS+S1#R{ryYtA+eKxHe7c~zdUo671KoheT#QY?E5IDx8}*l{a($I3R^f7
z=0+Jvnwc7`R}V<td5vxUQ?JjCy#5E`gq*ad@;l^(+*x+fy6M$A?v#S+z@<(r{TxD)
zC;SPIe6!}%^o{WsB33D-20N{{G<SV*iih9rp@p*bHl{0cl6V%((e}J>#z6dLxbK-w
zKTfVU$x7L9q9lOjlhyHvv-x5F&i%XGT9lBMwfN4H;F<G`OhU@0Es3se>b^Tk;O%V1
z({5dJGJYD=ZE;T*Tg$C^@YuaF0Y8sfZb@sO&-q>cW0^|*zqP!)E7yl`D=M7Y<PcQ%
zV#faLryc%pt;2XG<ZnO!F8%ns^3ZoL6S(J<dEXbGbw-R;Vwcj*KTb#1JlL|0Xa46y
zf4|@RcfX@;>(yOS4+__9dh&SJ>GSn%vu_+LTUS4+XN$(H9iqIeY_=XP_By<>OSU?p
zE;b^yws>vQ(~~l@&PBfYT)dk3{q*WY-n;3~ugvV+%gUP@_T|!3k?ZaXVG)6!Do<`*
z_wT8&TGaJxb7iJKlgJl2dEr8OW$CRp*|{x~Q`OTM?f>n0S@G+2@S(W(y&2|rD%;Ao
zObz&Gqwvr&zIpDV#KVV;EB|NM`?&-^d#704C-{5Ir8&Y2Rv+o#$u9Ibt$^1wrPKKP
z<Lwjmbu%R_pWevmoHXI2T>uAv!RhtQOWiKXJ3gG@F;6Yjomp1$>>cp}rPXx|oA{g9
zJNPWiazss+Ma<|s$@^p`$Ki=zBpf&&#O^A-_0{Bw$(1jaKW^===X|Jnw=iR~`JA&!
z^Q<c)Z9eq;Om-~4`C#$iyqHXXC&%6}-?e8v7l&IhmiWJ0*q;(q=EOJAD1p)1k<UoC
zZ*E#(`|}6CtVLh$=-u9Gq#<MeRI{%{bxNVosXr&2GsSbo&Sl0tk+lA`FJp?SR@bT0
zrq4Fc6X)JBR{H-}!cn;__~2zdP4k-qbzgM<dn)d9%{9O9z;$80gRsut>fMS>nF{k$
zVx*=p&&hMWcX8><yVhD){%%k-@{JJIk~|ysZc3bR$>B*Gr7AoR`7(b9u2Hlo+>;r)
zEn<W0jf+p`2DaXK`C+%>OPz4<Nn9CcuixOv&rf0BIWdPdV)?gL>7#e^zezv-<a0eP
z^Z&QfJF2qpW%5)NX6DCFx+l6L?C7n1pZ_h-KM=U|{dVcFC$H)onpc@;e)GKAmdwEO
zY(XrmzewP|c}uT&&1&xsnmeQPdHdO_3372WWSD*O48A#sI&SIsZKd{IDK>idM&GZY
zbCl1Vtz}wvaO(b8@6Xqb=N~*-C(!O4>v8|&#UCg1l{<eLr%yg()RCh4^R4A9?Y;4S
zpIdtG-My>1==9$D4^O6Vi7)S+ygcV4<F#Nu{yiU>R>p6#n=k)9a-QkZ?Dgz7O?hRL
zO|%YVii+ygwPysSiudf(UB2-m!_)2SPcke0zIsZwY~}ln-65CehpL9GKNS)GG51Bf
z%!Ilx-yFiZC%l<Dd81EWbNS0d%e0@&+V!O;+1knd_TInY|5GkrKkDZZD5bV@>h#6A
zX&L^R++TSw&eQqA{dnUfl@nhoAGKWd*4h&l6uu_=VV2|!Z^fDwX-aKA7i${7#9ZGM
z!CYqCvC6?!_tuUwfA&2Elh1Ay3s}5ZAz0LAvHIHY-_?`d-&?Fc`p)l9;fhjr*5i{`
zN9a9`)=S-cvS-z*s5ha8U++YpnCg?Wafa5;>+zj)>sxes^VK5bmhJS2@a*-vCjCeI
zoAM>u=X|THMH3l%_vo7{GM>}#RlfIe*TP2sXLi@Qej97Edj|f0%4xMSYs#`aZ>099
z{as`Hm*cZOcl`nG`w^wSw<7*KRdj8BI&XgO=a$5ps{eWG_ZHcm)vLU;C2ZESZSOqq
zm4|LvWq&y~HCn)yXWmcMwyW;OD~)xnFYmeh(r-E6p{rMSe(=(9?oX{<Zt~!!P~OW$
zNq=un*0|dtl_`~L@7D4C)f29dze|PZ)~jAxJXt#9>eJO{6<eED{1<r5FYNJCqwU$X
zkSVtUq}WbQI5}y~iJp}LQ!Rt)_C8(`x+KzTY2T8dO($KwV>R!GM2kF6+b{L|_SV1u
zlhs$0-RsQaRSe}*W14BcrXl(Ij}GbO{0Hsh&$Kwlo$XC-&F`J_<#c?#K8wb~B~M!=
z{ZjTVZ|(eFXOZ~R@5$cDjt_Ff9{jRP-gEKkI^m5CQ+b1>-xl3|Gf#4Z(dDUcQy8V%
zrf#U-oWk>L^VZhD(*ZK4e<;qlb96z?{z-bu%a-Q`Z*#h`^m5tB=Xxe9yC*xVK3`{`
zV<C1#HD|96*D=B7Ft*vMUI{0o`kq|6GQGHV+ssuCf?wY_xNY5eaNhbh8IvcA3za#q
ztg+rONg!2w+DtB6^IL1Dn*<)%t<&%SD|z{&#$%-(kB{AzcKRwW;{SJlWm8t<vmzk_
z5%s<=LQ`h=KRotT|K=H{reButm7gljI&-IU&J+jHIzG)wWkw5~f?B+%*k&i6y}AEw
zeMGPdL(k*b4-3^<SLMfD)%|Zd)!?d^_i^=I-(%}{UjDvi#*a(Yn-+g)D~|{)-lufI
z{+FPuYU~T)GkMphbI(<cjKAb<m-W=xw8ihj^(VVt7b%?%_}C`RIBW9#Ilsj^-m5n~
z7R`L;$@jnJi>irr#lDG4_FaGL+VRA>!tun<Ozp<nWi5LW)V{t|K2a|l#(A=|W#`(7
z6~T3^tV+)x*1WKozJTpA`=or$lLtTEex@Qhv-!=5g@;Qv<v*Jxv|Dn|cc!X+W?k<j
zo0XNWoaI<{L2cbbkLT$vo7e1?d}vViA${|uZ-#E?lYX?^_jXz1zGYhPuPe;<4$*zq
z91@cEbxsSdRbRbs-(kt?tc+ZmXFJ_uCu`liGFww-vQlxU(2X2jOP)Q<Oxe9ZVwJ<L
zcWeoL%C^VSaNm?pfpEvuRy!Gg)l}F_yK_p#&S3iSsn$~-x`rn?&fLc2uKnk~=apd7
z2AxN1)wX=dYN&N_UiwtJib*y6n`^r3LT0hdopBM;i~mG%Ree=<XO$OMsd|08eCf&7
z;)5T&B=jWxgLXZPIH_TveSFK^x|uwem%hsW#bM^)HKotdXg<^8<rDHce?EWR8{fZI
zw!E=C_Q(ARiv(2GG@X1L&$R7#%w@}UZ(c9n;dowW-`jQ}#!sb-<wG7U-6hhoPwzrq
zUd9BrYY{i^`yMbmR&?X5VwV%+fv$B-yge-I6mtVycOUwFNV;~iV97g?>~=%txlb-I
zwOQmi+vP4c`EF%9mDBICi>;U5wv*FTKCganQ}0#EwB>t5TI_D|&XC|Q|Gm4Hd{t%p
z#+<W%M^dKW_kc$qx^~T<z0N~>zE4hV^P?|kole&;mRW!A;jR+HE5FJ-qpfZB<lKKH
zBB(Zro8vRj^9_etvywR_Sc85Y>`;1efg|OM&q>#s$4wTUcYdUVmCp9eJf!Vruvn1k
zrTZ+yUDo_3`LCv|yJ~W=q2nvZ^;iD#K|N8`ng^Y;ey4jdrLCRx+o2#oSH!nudx2TX
zxAj}%@-rWA><>F!8MxIuJjnm(_jabWUnjQCYSc5{cJ*q&YMrIMf%{?~zdd`1-*<sn
z9>>N_+{e1?-+z5RZ~3I(Z#GqnN_nsC)-p<|DNC0<%bR>;*}Rik*Tj~K{qYjdsS-?l
zc&j4+-XwL`#1+Bs=4w`No3uu|OvQ<>*W+ld<q4I}Nj*n=+>WVeaHM+jCQ7YrjCmjX
z;`x$DrDqXM@?M*_Y*cG0IaJ%V=axc(Gg~}!K+M9a(mz`guCadPWwf7oaOt80Yi-1X
zW9F=T;<&PJ(n06tPF))|_U0CvM_fI(Z+c_&>-`~Db!I+$A<b8K{eB9YdcbMVAL@}M
zX%ppks@zme%(#8S?~MPowVX*u6BGTToz5vJq%GqO-KSNx<Zb+2v0pc$pYDlx_Wj?h
zT;Be-pQM}rhD*QSbImmQ{kHwz8XY*oQdd1cuXjE{e1cx!JiD0%et~kur;OS~RGzLl
z*_C(8J34*6YQ{B=w_AHo@2Xq5!p%#r(pGwp?kD-gmdR>oJZ>w0WZ`|=w*T0bDXR7x
z9{4Uekl`K0lQ?nqOJM`84=-;>Jd2Y$l%%{XL2FjYDnG8kIadFUiT+pq*zo@s=cOF(
zm?^~xettsJEKOG>RlZ#EB<E4^&R5x2J*3jk2nJSvlnb7F!Ol_ATX|vk#DhmqWGxB0
zc)okuJ+o7@1iZFRU9?4a<0+OK4|dN|$azu!I`ov<4?#B5F9x^0mV3p|c*=4oC*rb^
z|3te4@BZ9MUDr$*5sOdLWdgY#3#^PjTYNu2bv@_f<UCJxk*{n#tw!^!7RBFwQ@Y}=
zm)R7{HGFG|3lAt>^%5!DmaOKqak@_5Zl9HN%qGV&r9M6Np!|tWO<eQIY~x6c>o>Bd
zt30|>WV-LF@#D+l>Rb1o|Ij<@+#WSQHLj^FpJI1w8>YveVqCfO*WM#n<L8IORcQTl
zpZL~w;#<|=wEX2ivWp_`?`jb|eoK|hT9R{%r~Jfcm1XyxN^cvvmCfj%DDiB?Su2$g
z_jvEjoTYCjr>Z+u=f*FM%D#VQX(Dr@m&WR~9pSCnw>)<4xwi7;^{H0Bs?twcPI}3*
z_Iv#5&-JxS|9zh)@cR9Fa|4#dRi8{Ej<3uSw0?NqZ)y<VLX%&!I2XTNyGv2v;MJ41
z9|P;w96oSHrFXT6`Y*A}tSv!@1H12C58B_PdDG|}C(D{W%3ls#$>5x|SD1a#{ae4P
z;{&!GPF`B_>)`qFj(-!>UJ1GKv7UXnM6$=Sz3sWda}jT@&3^7h^Bwj?tMc;9Qr~0P
zUbL)aQQ@IB`zPWontX;P{rBJG?)6<+q;%TUqW07OX|j)Zwe+5-te9w-==FbI(7Uwz
zo1On0Y&%<VV#RCSIm=3pNg8R{dGYa{oUyXfQg!39XE)wtnyzC?4cgqLl)h|UqmIzj
zBQ;CRFKv_wklJYIwxmIJp|RTzNA)R-=L^Uva%P(eha^7<%(VW$%E#>pe`ZL}8)?ta
zn<q}`ZLUmAKJBEuqh9^L<3@{3dn1;yc6^+AWR~W?Ew_DMN(W!c&ztBu%i5_v<n#`?
z!k61Cww&kuprx02@ZhyI<{4p!KiEy%DICPobLESnQM28A*-LX4GdNc*=(=#=_xpWc
z6=Ns)mcE<u^R%O@;DcA9Pwso&`&3%gy!OFOf%z8?wcGqrx}M4@>|i(l&9m#?g6AXl
z@4NFO`7P5Wwrzp>_4~{ClyiF~y1FgklPHw2O>DaV>-YbMcOUKl$ddg&Kq%DOx9Y}0
zE`8C{+n3y~lu5D<{qk{>IdkG{5BZbZ$~ZQ6y4-rPAZ1>h6NgoE`t?-9G_&KM1zpbV
ze3-e{;@>yhi3+}YEUJlzn}T-9<qIv}TXJaK-t<$`<!-hfn`Lln`=7&;(;mCeiMO;V
zau=z()_CLay$|fM)$+H4Bcl7&eSc4Swerkv@tnz9_ioa6>@1t{==a<Lb8C_CiHoFN
zKCtynWchG%-jbR%+>%!$rmszF=scWe@?&c9?YAe+&sD9fzAyPKDrMivwA)4Bwe~!o
zo&5J>R^h4O_ZOprmT3H3vG~)yKPH#k@0dEr%{ng-BImDiCf`qmDQ4cj<de)6mw5IY
zT71)6a&OMg)hTk{s&-{2@_dzDw<qkxmgFscb6t85`)0e(TFcXR?gvMbbHHMimhi5v
zDnG5Qy1eBJZ?CBfJR|h8*)?l|@8mMW6bl}I@6?%slU=X!y^DIN{US4WjpDBo&*Y{4
z@>Su9x35eWTd96VU}7l4A6ceH?m11u-yTd`HS7PgKiye1YlII~YOp`N(BQvbdX@?!
z&%8qJxO-J|7pXnaKdLl&N{EYx@++n7LA$^I{%&5g+@Noca==w5FSGi&Khizluhc)~
z75cz+@w<?FAKzM9q-aKb-J0uLyMJQhAs@A%6rr?952ps?@_Hs+FMPQ5?KCy^EMZyo
zeG?xqQJv`i{NGGg?);nU-<B^pw|-;b?SAfC{i<n=N!goc9?y5Uc|Fn1+1#|_zC2S8
zyV`HR+uvWu?%Cb&n8j~{bK9~jYvYd}`u;@d@X~WeJJ_vntY2IA>BWRM^B<SWtBchg
zUe38vs;*0dO|yf+f1%Uwt((QyrGHxe_t&RC>hFTD@YcLP{WE{d^@hJU9Sc>x4!9+}
zOxydb{4o2;P4^@kC;4nD3p#v{`?XUuoA#D}uNEHIZ?EFFNvHdEeMQmN$7+?5qQ8uU
zkKS3aDa=9a`D^x79#@kV*WIrPPAu&Yd9>1g&Fjt?t8bs2dgSg?{cF6&LUIdzZYD{e
z<TxmJ*Z6kB!mO|I($V%IKk9Nj*L`HHnk}+0fa&@wp7o)w2Uc(BNnz=i-DUghm+|I4
zz3p}G?w1qVzOU3u&iQ**FgR$d;_bo-&kUl??z+Bya>JEpzvacV@}FG)<w#F^e`m?P
zX3<G!g5Gc*K38ymf1rlH<QMU&l}hve@kIW0pTS@M>r&b-k9@PU`)00jb6duv7_A(8
z>$`c#6Hfle73&S68;^at{rT@#-{(;|A1tG{=%+tgeC+wb=X>XC^JsB%&53ew*3`1D
z4svsP^?Q=|GPAGN6`xI$uJymY#&=%6v1yb2XX_71%>mhaWyIyngb&uZ9a->o&j#67
zF8)vJWW*P-#N6)Z5P7ztxn9Cz2g6cRpDi7n=hyip85PInzfD-%{-xseqTi)Xv$L<8
z&v|lf+G4{Cs^05&UYzLWerub{lE}XrVsh^##WpzqE2>`2!lxZ^fT_{(<nJcSHxt^f
z#67iMwBXKXwh1Q`EZ2M~m0Z8ISE;>c$*wuyzq<>bzy4r%+9UOur@4AH@2`3P_LArd
z=PlQ-buE8nbR%oZ>JSf~i(ZQA$1e$S%>A_{>T2rm2xG~<YgzXD{97!41|6|ft*)`(
z-n6CUJI||KpOXGMK8$XE_r74`o%+|(3V|>G+Q(F${eFLfYfptr<}q#q?|V!?pKE2l
z%nAy5_ugw)>SVTAg-LM?Ozhe*za8%VwflBq&BL(E@iX7_eQ{8`S>s%>IseV+4#7`7
zHcDO@OqY7ED12Hm-`=Uwc-hPYcRfOR>}zvskDYj#BXFA``k(+K`|*XZddmLrepJ)^
z_G?o6BBQCPQ$#2FP4n#Aw5=iV?5P!rQ(MpG=FQ)ox-IjQtMsi)=}Qx4b~-=WA^%g-
zyCiS_Wxns{3fm54PRP(HK9iy%Bw7CE)~&9lOWAEYm(NbERO>h4aqORCD!+SMQPPBO
zI&YIUXq@i3)#5l&@^|tP$I`^m*cV4_zG{4pOG?=Au029Ga%${x8$)?x{T-HRTc*9+
z@#mX%<@aYIO|y>856ZG*SQXsxM&;cp;kec`%QER{jWNG_Cx7`g^F!SJ+3#oU&%Ji7
ze$Bow`|F!zmzG}l<kHG$ne}kS)qvi!Jf^2!))}p{*{AX`b((0*_m?Z<R6aPkOmKX+
zaS3yykmG-24Tpb|A5Smb7;;lu^6*aI8!xu%Ts*vDmtf}0D|;@^kiN4jMaaXdc8l+(
zU41EU+PvzfNKMl&e(L&1mMb;j-P-)89gpVb+tx1myKe8z6S?}Ef3w#mhWLlwZ2!{w
zrbWEvp<3pC-JPF|kM!KT62Eui5%F--l|Lsw>{BQ`%(Q9e=FM6L+OhN2#V$1Y(aXKz
zmPl;#s|&36N-SA+H_Ya{!u>axr&qp4FCuc9jLr#*NTa8Vr`&B%E6&}tIDV&il2?4f
zpSs-cD>|twUFID+ZpBpl=+CSQy)aI_L*fgTuYYqu)OX37Iks%I%#m3qPThaGg7>D?
zpB;jM3xjOx_Ba^b*K#_y`JBh4@_@&lp}%XU?~JT<e)s6!l_TG+t#S{a%hAwsx9ZqB
z{j2g#<t@UmUtX2vIapz<dHUSG-5i3&r%tX9*v_=sX2bVz&pWe)-el$aS4BoRF-R@Y
zs9U;y0`K&<UMo}liucXyEbokXbNb1GCe7-*O-C-NR#j#15KObmTh$r6aKqOVXZfG+
zPT&m*iL<ud+`j$zt%I@+pBK-TocO!-#ji;V|Gr#!uDqG8YN<W9A*ZUl<%j61BoU!a
z+KmeJ3%#uwIX>l<On9vP;J^iA+vF1$eS>ROGNe9DHMufvnWb6Z+g%I%*Br4Fa$C&M
ztzi82)Qp_4ms_WD*&g6$cozTn_h-MX{7K^TPXFE=bb8L0tS#~T?-krQ-oK}5_fvCA
zpS6cSP5SdH+Q;<w)dT-qiazdH9c|f@7kY2U)Q}HhBHwhmKCDfevEAw0JMW`Hc{{E1
z3!C!|KjsJS%Ree`-;1}dd&7?yi!0{>tewyDm>OR>6JYFow&z@HR{Dy$E7O+EQsfnU
z=n#C?$5i-ceBM)uokwTvoI2sjHNBO}ALFMNhXi{y6(kGXaX3@>Muy#8%0x<~cB8@-
z>3R>D%nKP1IN?#!#Ki{t3U<9Pfx#P_*n7F9#6Mi>-JYkbzkq#DMfOeRdwV~A>Ybjy
zr}ptz>G$_%FdX9vxOi1M^i`Vk&WRP~zu%l)=QdSVwzBnDjDcFSp=8g^1RwU<lE<EF
ze{<qBQ19_f^x;Yllem0nhWNQX1<zblmR%NaO@3NYoFij>{NO~#618LJvVEJ+aHq#v
zd_FW&{M?=Zw&l_iPc4n~=Tts~3kL<{HO!XY{nm|n-$tSQ4Hu)^8v1yWa(v7${boC~
zw{i9(kLFgk@Yr=SN4&bz<7P$Ap4n#-k-6bg!db^XnID&RoAcf?sZc(jHAhJJj><Kk
znK$#U7Vi%I^EPbb_Gs6HdCjjE#BN_bbH(Ih?WLb{CuOVcT&DB8e1hDg1!--wIunjR
zUwlh4D{aNhg=uZG7M93fES$fxY}u`uS#3q>phz&jdPXM3!{=COp-JDljRk9D^80!w
zK9R~4&by>$<G$GO(>Cull1XO;vfJ#f_hmriAoD`z1xe$RriYF_@@V#J?&X%U&6sn4
z6Z@XB>`F%42S=sz?^S#~wtM?L+XtYKulRqA#UQQ0khRA$!KbO;khiBY;S6)Kl7#WW
z8C;;iKdi1T`7Xm#D#b~{SV{YVWbvW2eQwKI4fT7F!tvam9~Q!gXG(zLqSaV`PUXhW
zKaD%i9Vt+21Mv_6E}xzRQYaod;WQ{xCPXx|FJ{Zwc;Tp1b1NG+ub5PfCyS)ZNz>o&
zJ|s)6G4Uwaab%Y4sVcstrANKQwRyv&)|l*YiTXFax#IGpUWe5mqkRgd#{~DDer`}Q
zqp)L-O1<pvsP2_-CuQsHd=_*)Ch4r|wHU^r?LxeFx|Z&rYW+WK$G5eUShhd%%3peY
zzi6bG{MB;-#v)v^v(i@01Z9BEGY_^mDd$a(5uX0ja>dM*X)4!bHa>Hhzg_wIgY`#0
zKD%L2*4M0jSY_w46Q=I#EQ`)g%<kLuti#lOjpdh1KbrJ{cKH?c9&s{%Gkdm;sQF2~
zMU2aj9%Jo=C;#*3<vu*?4bR_G_4ux|{{4#I$6jxrXPdw#5qw|<)6Bp-Ggyx47%Xcv
z<WJdt|A@}9iEJ~ukEKMUKAyp6_Sw`(xkoe6hcnqvV)CIGqQ|xvJZn{$pMKTA5IIRB
zqT<F=_lyaUNYeP~RCCetlEUYtsU|6q95H{5><b^B^f-@o50C9D%)h^5^Wzf=UJx(+
zyXzU2x#2>>!h@5VBQhuQa8Am$`0~i3c`BPW?=?_fn5w5Is+$`1&%N3F)0$)9^JhGC
zE%J4|Ts|>Ids6nQo8{WCFBb<y2Qq)uo0PrkK%w|a(^TFap;vzC31xHbuGEtOl{cV7
z4^92Y9yo0^zIraiIQXo9jsMyk;Fzcq+HMqX$8M^;Gi*XOYw22rYg|cd9nSaYOC3K~
zSg^!#u_M<5k2RtTxBpsH1mZ1r1VOHxI+g1udJ4s#NJ$z(a=_>4Zwm?;^Y2!?zV?Z!
z>Oi;oy**#wFoFaBwt3wCkN<i(6V5Osb7{Oh6gwfCrRNSy(H)nj0O=5SsT3^^?{57U
zNn*ECtRy}koFOpNPw_On_Lc{1HM<+r-Z#Z)pD_JVUTD(Cv&h(RPUpmx>ye{keyn8Y
z(-V6No*mV@>i{l`I()y@Oq?1HO8N!Qz!~M?neP54QI6|GLB!PedQ*Kt8KHScaLi?B
zo(PjtF>X8(#Td}9{~~GWK`(GDgh}lw7RZl^G1C1X+-!JwD%;nvW!Iul`u(@6NvaAz
zxm9atnCkUaPok#o3RAtd>PQstwUU|;>)5ja7m^{xjbzp{=fGPF%i3mv;y`lM5kFJo
zt7qDI9tOV+_*@0a3^TLRR!ORQ)G8g;d;CI%`}iqP`u~=b@S!zoO3dmLTf1VE+b_oo
zW<Sa5V(SrGTMUW?Ef()Jp+}-z*M#m+GSBpX<N+#y#7(LWcyFI(n}J*cnaAz>`i2o)
z_CrdbO)I($IeQ`%E?Dkc(7oEqkk?4GXJ>*>+ZxF=KK3(}{;7SL-LT*1|D;N>k|hUc
zuAf+$2ui=u;=SORYuXdx>lXW7fNCaa8b{;>XlQTLayQmLlC!I@@cs_d#~#gRxI4FY
zMXc`F+Ox2BpGL_>P?86gMB(55&*Qz2w6J?Bo97|roniZDeob0ldd_jpmWY?&IN)~l
z?lunHB6CE#@!R#;YZBXzEqr;QRKn_vh0VK@za*wE>tE-5(eZ4F>E)|F;l{NE3vWqg
zrNcrVT;ey)jt>=F+$Q-;X4x!IKtqb3IR`^6&t}b7SQua&e6}Tj@1*NXwVKS=@tS9=
zCagZib9E!M<~^p#u_!ccMW0Cq3@S@6a9!T**9?Q_cCa#TJH>N#B6r2hW6|buyFcDr
zo5~L=`pv<0@AmvXb&vl>pO-TL75!%zlZ7NUADF>*%*H^?$s+wLYix?1#N>lBc#oAB
zJZnB%(+e(liVw{YIkv_CRQ%7WT<Gvy%Q)otp^Zg*EY14nK0d*gF|iX-)Pw2**DI4Q
zi#IFtBGtM|PRqWq#+FEdppHpI#s-59LC?t4wpXC47E}Z1m_!r{XzPaU0#~$<1fLD6
z0A}mmGpV@zz_;1_(;G{Tzo}0ovVPqFhxw5^VK0(&g|fMpRO-FSTE2Sjy0l%hUS=&@
z4X#;#@@zG}at_qefj4<vRLXTHWxE!w^;jo*a;t&!;{|DH&p~BSA1E7&KACkUKvvtm
z&7)|o!gVeP;R_CJo~aN8QI6}T{^zg$zb$X~s@3U>pFfRy)p+z=!GH5@TEF6MuGnJc
zW@J41kMir6{_ocYZRqWB<apb-&XecsCWZ|LDT}g2cZDs45JJwOpZ;{n8N5EW<#q?#
zmxfOBw0kqEb}wc>v(NDFu`kk}^A#Ez8QHibEIt@+IN-p{%*$hAAaL*i3$rLs%nykh
z2@9GV`Pl3vI1&?D8ilqcNgd>3*5%n^@L^GdIO`sb2d<3zcC1G<6x<l~xhet=bTh??
zfTZR#C5Z?uhARO{amza0)hb!*bRA5nw5?)E*vZTwz#Y4W0S<W<y@+aE7YZR}9SNAq
zkYm`v+tI6M#sGn34BsS<@E!3Jw`R!|?^=GPQE{2E`JNeZH8Q(rzlai!&0dsU^d$S!
zyX;TzZnqxz5h%8{bKOgY@7G+emn0NiK6qel=Ni+ATbHdV1&M1MT9*79q$%n8tP@ef
zk=dt~UMVR{xO}k2*PZ#*A3lASG|rwAQHQs;Zw?FzJLPs$ll`f$B{PG-+eIu37%wnn
zFqkk%EmqxiZ9z6Klt{QPm9p0H+N?*iC5?J~Qan;TQU))Nt-0MHXYldZ8ENBupS_0P
zk3EszoL~NhfyKb!zyStkRvsCH2?q`^G4oDQS!enpsu@b$*vb}}Jz?vzI|;iS#1C>X
ztMbe-m~iL-SF~ZTf*YeDC`@6&&AR78X5(7N>ry2U;@~>d6H(1;c3N<`pv6MBKcgX6
z2RNcS)|tYrxS$Zbpj)&7220r(6wG$rV1UCU0cip0*c^s8FvzoL@eaNj*<N}#r-W{L
z`DDkkkjd7+Uo1Ve=YvOjYR)&OguKR`tk1+|L?`4m{`5IGv&on>jq8la3{8XJglE3a
z2WK{IW=Z2bBQ!(Rz&l~uftqO#bPh%`ZDw&RS$lY0>4~WJHKms8B_mBj_)g6(3Szpz
zkij$MMO6E`QWz2Ydcsz|$k#=2#!QQPI<%%T=zyWj#b(E5M`pFVGfJf|Hpm%#G+V@e
zX0PE-aFF}#H~i1WBq8y^u;73L6Ei1|j=_Zk2bh@!d3=6I>_|{(ZscJzlX#Kvpt(^1
zl+rnwHF=g8Y&hh=%?wW5+_I}*iEE`Uelw-{)3P~Jc#_~j2~P8%>~M5>g9PglaJGQQ
zkJHccXlyRO%(^Rr5dv>8+<*mf4g(Y#cF1<j)hlBtV<>xZQ*TOerTAfP`wM$`FK$)b
zwd`{FErHm}7hm|MJDbSXKS=N8Jf~~1{lFcjVs2<C+XWXad*IVJlL;E~ss?h`Z?(O+
zB@ipUDEm~VgH=l`$C+6sA~F|EovM{KC^t#$$(b^DlA%zn@`Ig%Pxg2-LzDP1uaaDc
z>ugZsz&hR&w;I;)TCSJec_oBt14F@ugbU3FKPy)qKrUdC-@WaVGx&S#4^jb>pm6ZG
z12Z!}QUPNy;bHQE=0<*a0mJ5Zqbrj|muC-h0mD`CQ4=%8{{R&*5}-hLg_k`qY<TxO
zh#!Py3#Dx4O=eH_JQLSKD`ajptmAzFC1RBoj-Qq|!gj<>T$%v{tr@Hts(F&wliC-&
zeIypU+x2?h#_a7UZs~3+d%dOW{-wg-V8N1IkMvlciOu-EY~h1D$mx68fr_Xvv7&Q8
zX?@xOokmc)ezut<o%0+lxp%C&J7H_|x$I+{J#I1U3!N4hIxPmJb0eQc+2$p0+t%H^
zuvL0j+2M6}Cv278<dzp3yP%u30R~$Wg=2RwL?|qK+XgjnmjnNds^oXQC(YCD&#2qI
zm_4DPkqK0`Cnz*Fva(4@RD3WrIOxC%D%%Yv96Z3rESq!Gq)Cv?OF|>@K}(|uTbM*f
z;sb1n_))M!4^t96or8lMTdZ8@oLASV*CY<{NbGLKZ1a@f4v+_6g^Or|DceTIjTeuy
z9e_cx<{J$+UL>(6Kw$g4w@JG`-_nykxctQ|Kdrk*KOHKJe*UH4Rfqi9;84*yph5*&
zx-2{J<E10Z6d`N(DRU=DS1E#umn93Kg-IH$fce89KPgFT*R2!13odDJr!Gxtak;Km
zlIsK|-rQ<g<9p(k%9k7ZCSD9u47?3&4$KLON3bcH@_ClvlNFQLl%*~(tKFMXxqC6Y
z&pyN7$KFU^&Ofuik%5g(LgRxWxH#wLF){dXz=4HXm?z|i#EFE2=0;vND~TTo3N4L-
z+mfUlxtO(i))-tk1g|wu6~7UeN?k59#r)|K7KMt1%>I9!7P2VduUEVKU)*Y0=L;iZ
zXHVG5896)UYOe4uvxV7U;>haloRPC9Y}I^j8oQvIw*dxi*)}q7ynK}JfL=r2L+>}Y
zgd(GrvQN*wzGv5-f+t}@9ot;4PrI>Ib<?tB-m=5d1$hU-p$<yd%MSQ7e%!pKEVKZ`
z0v9rzXM`mDtv9T0LMhob()-wDrBfHKnbNx{nD3M|gO47^q8GQC*6F^u)f5{&VJmB7
zbWz-955EVh47|`J-s&hE8+~CbE0oxf&An-xLbmt0Y*6~HNPgD4()`)I88x6r3@F5z
zBqS0RG&C}^2}wjOP*(HgUM$p*(A3DqrYCVC!FZWqje)=+2M%Uso+$<eha5oF=Y!Tp
zw1T6Ct0T~%2ikanmMMhVHLWLZHLcM-ack1&S@A1MHZpC1!F!Ah9aG^=7Bhxzh|&dA
zwt!2Q_62{%VvoC)@>hO;6lNHzqi>e?e#Pb9+p#l)ceU@SRy2SX=C+v{7Qu)><~k!H
zQ6Bf&YV&Gs{n+ml%0W#Oa4q_7&$UuL-|%M})&;Zd?wft1xg>Uqo2Exm+-GiZz=La1
z3H9aKbqfPP)hVdy!jQ|xpdhv@f&mUC%$H~XKD6NA0>;UCXLcFBJ+=qba`}4fj&yN8
zIGKYQjOAY#L<|ZJ9AHGMK|zJ=3ng$J3Q6PI%LGBeeE?L4!qWJO%Cq_+r@U@zSU+8`
zM>Cy)K?%|jftM<+bFTk61WNhf_6b-R<bjfn3>!cY6uw3se%JpnAr<Fn4Mb>Y`-wwB
z+WJe5xZ0+)qPW{vqmPDc+*5Gp+k|rcuC7&W+aKI%D&{_;3l3;d9ob{OzErjSz1ps{
zMcHb*o+-YM;{XTtZ{6t+AcZ)%JSn{#P%!O5$2!T9vkS7-%Fdp4J1Ni!;jzUUznIk-
z`&p>rSOBWf?Nr8u#1k&pd3L2;Fl`2@KC-UzB&gC{BYim!R+c2c0#%y7k9|QZOA;P5
z%(?!J^#Z7A&d=jw04~18A;Ar4sj#_8Xe4Sc69k1atfd030G-Zd*DVZ&G@u0bXgmOg
zuG3PK;-o0<wm3?%Z#^;V;htyWFOpajU@%p{72$4Q#rIl_Xl<1@3~!)q6^>haBH0&2
zp=iTnKiS=)ITyNL%u3?jg@j;b3A{zP?7)uCQQ;dL^%rG#nZ7_mH+ZL`mgP#>tj|sN
z2WV)+T49U~9c6*=1l@a4C|3KzRwgL1A)9qmm;zXfgE%`c8!sF0jFRMQxi0)OK*h*R
z^JVvDR40Gxy}8}GiUCxNfa^;>36F$?#zr=9(i8MC2sn6vm06M}XM^Zv4krl-a7T#E
zPr@THp>^#HH-W_sXl2QX%C*8d4^$c8%@Ec-6Fx_&-#FmS&;;u<aaAZj-|DxaaXJHw
zzRUFv(-%jiN|JbG99WZ<lH3k&;@x#_&0C~Ud(mZj!*O}vEiSVcRwhfduPs1A3FdQ}
zj9JrRZSiFX&dhED3BamKP=lrL`POv_&2g^RMY1=rKi;L$yD5Nc%AD(eoS-$+0>%px
z8W*>6#jd^pC3c4?W{YkLD~!7>&J_!3iGxU@TH^}OP4^#AXk=j%lgLPT(AdbqrUoh~
zF1SJ(C_JFzJIqJfB2l3g)<6NbKEMI!xc-7^60g-pwz(0_erp`nuN`2-(f~h~JvX5_
zZecb!kYS1&nHdaSu4|R_LUk?>&c4uTTEHOMF;y>&0Ro|=39MxZY8^&{T87KJWgBE0
zWUIDuAC)ROD)r_-;PHG`=`U)#mPItLLqc~yh|P&EfcAwT^(G{x=Mzln*_Q$eT+lmo
zrjP;>+A?R5O2*dtlscDP*Qh7PAjZHa#ZzTaQRl)xqYTm;I(9}nIRDJvyBC?4-JemP
z{ErvZ1$AI#X6I3v@Of7Jf^JCoORPwE(A3BSDjg26Gt2X|7#JLK;AB<@H5SF#;v|q7
zB+#_I!2E?2A`%t|XFp)TY%fA$EFUe#92goRUZRyZ9A+<gce#P$6A~*gj!0QFU~i#2
zZDfl@LVHhFuGPM=;Pr;D$nmf)f;nHqVmoqwNYmimo@=5x8@L;1GQnFcFri4WP!G5t
z3QG7(bEeEK3b;3A@}xwe*w+`f@<E9W*|M9iDQ3$)H{I_b&dmUaht+mnTbL~iB@(Y&
zrL1+nW(A5DIRj8_IyoOS{8Hx55AObhYEuU$W^jAa0i;N;3DS5;Xl~>KbwojZe^71e
z0BSLAG5FvFO3=*u+^JA^F(wo_3UFh_3ic8QlrwceV=g?4UPv8eivv}w5PVF*3|qe+
z-s``xIhMzAeJQK-8%Uc)W;?j7-ym(3rhOwp9Mt)Q)u+eiUf<QkyNi$MC2GO+p{)tj
z6#}<eKs|q*Mo4M1uDPd<`{Gu{Sm_O*PMbnD^K;Yv1{(RNJUJaeJy8iwu;c}hiVIsA
zcbh4IloiB%=7zNn7z#f8>5$`*GI$LdZ-EYVy?en34r@@Q3LEM=QJEXhz>U=THwbWq
zggT_@g0p#E6!)4%x&kr^q{+w;M_NITJjjIzcE$~i8;+s}d$NGFfOYH}hBr(vB;d*T
z{vmLX)^WS%feW1y82JX#mI+RPG{+yG%Z@$Bwp#!^-m(nb2ZakkMqEHG8s+C(@5@6v
zYTzCxv?D3~+;qP}2TXiJws{JugK4q;XX|XC2DSrCV0dUjCs@y}GUw}gFK+SfE_1$?
zcj8trs1#xM*<<+m*cEV_@$N<DXZIZ#m>GF^3<98SMo3Yk(A3DurUf2%IXlDc!$AiQ
zW<{PJgMdSzHY2E{5obf|Z-UE@#_5xV8bHI2pz#%M%k`yAynE2vG@w}GbN~%QDnW{T
z^m_FxAA`f1l8vCShhU@_V7vi>$pWAO#^4fPq@pEuBU|mGJ<AfBV?#vYNqsW7_C!>t
zK8+JU&uZc=Qv{D|fkJsRi`%(uF0&Uf!}hnHfb=(co^R!hoqYjH>^4)(7T#obxDwti
zEy+E!&KIQiY(X4cS?uf$AZ3c##&fP?c44V)q+s@ibc^+ZIS*hZ%@kt>20^s(EvDzD
z=y3qnqY5q$wHdS-w3{_{xq%uxuslGx(cWeHg8R8CtMr?sRS|I(d9W5tg83fS=VFKv
z08pL!1RN(E&~YzN2Ed4dJSnboLNXVjUCQ0t6hK|dLzTIX*TqU8#KCpCpzhq-f;h~c
zr}iAUGRNy;Zy>!suIEtGz`ax|8>xH^UhjPoJ(tZQU113+3PFh)8n{T!%LA3Y3C#M8
z9FVbh);$7|(GT~yZ8)e04t!7qAV&dwd=b~c`t3a;*#>Z=XStr2Rhq{>{(SeMMXLnB
zL%M~}w=PU*e)#<L+ayHN-|_iYz=ne#K7ZW<DtF+8J49h6EJGA-GJ6!_imwS@wCkC;
z8YJ<6E7Zl$O3vzEIO>QrvI`!1fe!>Ee|y^}*8m!NK@0>WENGkq>L{`Z^Mn`_9CTn~
zmIjq60&H#)5{V018im<{BqBiFCd|2uqBv<5=?X{z0v!r?vd4_W>;t0Q>1K*MQJE_M
zZsl_?&u)%rc3g8%;o1Sl5QQ-Kpl$?XL^=aQ(qg0m7hTxM7vBL0Jg1N+qW|c6SCVu0
zjjc*$Ydd1KldgB&*t+S-p0Fe9MUS?LZOnFk6D7Di?8rLNjoG^&ip<fpfR*ybtZDa5
zKmw4khBjKP*NcA4cHOwp^-WY`toDX%*0Qw+)=h2w9Mu^A8Qg{uYG89nUNC2!=nqqU
zM(`X418>8z2FqCO57}`{;GQ<a28IF`enfwXx$3|X&`1Di>H=J!f<|^Z5)vBbT<7|8
zOV1J7cmWS~J=qiX;QAgG1@P1nKbsS1Kv$h-33%#)S(gVgX9QNnwQ(WXaiTZ2GVKma
zxGuwhIS7MQA=mR+ubII83}}JEv*_|xru|_Zyt+{Tf{lE}AO)VafF^j*ToTfddVb*1
zb)g%xzfSrbCAvRMR5}XG4Sf?Ox;yOXy0?f*b^8IxF#55%*H``6y6VOSt+KUUv8$7=
zuez~St1Q<%2DL@A#?$#=;jU*(1I55Y9jeJsO4c4)UkPc(faZ*joq8G78vhv-4a^CQ
z35*GeXXdO61w}>#gGGkHi-Qk-3fd;z%5?#SI&{jY%$<KmUGl%)pXLJaDI<q-*`jr4
zy&0M~*wo-H_SK*`6d=@KZ;fAlq%G{mR<7M)iPyE>M774Qp0i{fcQ0BBUwdeMXp*xw
zSbjscXxZ9B>o^$%w2?+%)__8uYkycL?^>vWP>`v}ArEdnf(8g=o3?QuZ3`2Q4bI32
z1tU|-^`S@G!XOACzQSB&PBi+M*Xl0bwGgDXYuSYhtH9`P0p~fPIj|OuV5Bi*_yTE;
zX7Y<!j<Dg2nKwWQzMO$!(Z!>XRt$(dv;lRj=CjefemMhB`vufhGW-c1+AaUW0BXNL
zx=IG%iBf*ZAPl6d1fDn2UM2`?zkqs5@Oh&rdq81xq%ADTc`u|BfxWS=YPLm4I;xZR
zt}3h<?{qGE>WvFtIT@;ESG*a-Ks{2nz>;2s5U7iK4KcI`>YPe5MDrvuCoP6_Pa$2@
zZl&j-M7!$71rRD;vC7GLH5ipUAJ2gV{n56txz3@YQc+;EOWF@@!VYsp{Z8dtEwe3J
zW+nc&O9Q1q!@E5bUPLvn6NM46+7q@iMQVdujc)LyuhTOFCJ$DLP{b-_@Y-w<qiw>Y
z-kIC2s}6u>NI_ksV}GPU#Y`g;xG#!a%<zOjnvtMB5_I}#?F={2{3x?F()@Jmoa<T}
z7eWH^smv;`j0{ziEkaVSuy;?vp$&=%%l1`5Qc;||Yg^i^8kmkaiE~40Q4k4g%z$DA
zLmi@n3W^Kx6cT8lAeKS)FgRdA!Ml3S^;JpEt0Cyz!c|<nYr*I)Z~noZtj|F+rl1xK
zXha=U-aynPIj=@hiZ&?bo$zGOw&!n+PB}$tN`s~k${A+F2)MvTXyAi0pSjIZ2if7{
zH?gZXfW~hWvqeEYS;*)Z?m>3Yh>e4|BY0v1T({0P>;=u4vVlg&AayIK(AVcdYbYY6
z=q*Kjq2P#!;^Zx5z?|Hh@Hq;WBi7Dw4h2U<n3J<C+5j1`4S7hxVb#7$%VY}#nIRWF
zNQwUyFKas2ImkpcBqAQ4%MRVR5Q?4$gtoSYfzi7*?}BO2kzqH`q`y?kTF}@jd;({I
zq2Hox*usDX*}P?I9j`&uR7If<x?kMN7MpzmO6<O-kj=a4T0z`x@fp?d$-Lx$pf(N@
z8=r&)d@?UVp$XE)5$8!U*l^H+omn2##t~xklK_vFiLyn3hHAJi!KFAj7o4&TnR?*@
zI2VL4G;m;cu(0L;2GAhqqU<AWVdoaEYH15|a@JnBkZ%Fg#!E-J4!|IE5|23n2D=?|
z9Nv^{WZnRR&gaV(Ubt}o&|HGaA2eD98X1--k3*FCNKsdUQQ^)8jVZX7taZL_g;+iS
z8Y%O&)Py#0dKVHJw_&%Hu(4mn4x06V6-4IC?!9}#xH;cvzagx>)5yfe2OdA*MVj0*
zD1eS15LhUKR2OJJ4+sT^;?xTP><pM|WpGAW1p3f?2k%;N?Sf+B1<-UVEFRbs`W@eZ
zV#4`C*+Qi9|Mi5=QK2~*5OmctBy{0I2-^Kn*JAsFJFt}lpgtC?yx*c_w&j(~DoE|S
z>cR!o3OJHUd6OAvOr~{Hkl86eP@hGTp&>&YRO+xnhH-Rp4&xMFNVt%2p~2HU?XDYi
zol<4;tKOaFYWHW<?q19eUZ=z+p^@;Q0Ww$&DTWS$S1N&Lvq1}ES{en}KocU6*=$fT
z1Y7O`o=yU_=c!R8a2@dyM|7@U^DsE%U>#2YjU|Avq>FnYwz@VdA4fTaSqWpVn%RMf
zzZdiRUN15^<)j$^nzew8Re~Cr$qQV-!!xi21CR;%x5p~a-fe!71ZsSO@dMDn8EmN%
zXy|Oapl!mt-jm?4PyPjMc3Q6g$;!s50&V_)CZnL~p1`sh?1}jb_K|u>Rjq1<RztMC
zkU}rOuAua`2rr=#0k?0|&t*eugD&2+D5VP0zyhfFnW1S=6c>y=dkD=@weU^=O_hNA
zT)t9VeX!~Gb-p)1)9(|uN}kIG&%Hy4vRtQYYALyGpRr8UKimVGrx)I3mUvw)CD-X%
zn#Fow8W|}l9fj1oxh8oIRIVpEYj0dQwRnZr!iC`aVFfb-M!sRiu9ATPr5ZS;(6vqw
z)RF}iODNR^V$BDr?<?1Qqw$7f#_oVMj-f~>)?&TpjRl~#<+TM`2y{T2>zpof$Cs;T
z&UKNT08!~E(d+;uv_V?$U?gG*g3=%#=m8Bj!N-nz6PiKQ@3jR&*#VuVD~?EoB=NFf
z4()?RdoUHkI?o;J1QEJr(ArCcSKj;;d;91{0IRgg(KfTAQYJ{~O<UceSxvmNlALEJ
z@y<d*WrsCUmnh6J5t)OMK{$K*ctbZh2Istx`wA|mA~huy=dLgBTvu5c><3wLbIjvq
z&0g=4v))Jxa7y;wQ{D9pHWb8uzW>F+1&o{X&g|;VZ~!+_LG$IH4jQ<Q2CwZ)Sio?+
zEpAbE9FqWI_05AD7r-Vr!76BMJq6a|Z9+L0I(T!BNSP$@N^NB8jc9gT1DYj-u9t*Y
zAh=X&IOu>TBOr4rOdE_ktm69<n%#0Tl&>ui%)ZcJngKRi#sSnrgRQ~gZ9rLt(`A|g
zLB}>O<TA^!YG0*!Z2=NO8UO|L5kMuCgsk+cgyv<%E07~)RuXU7VO0z70(dFKc}7U0
zkJl959~R#I?CF9%%m<z{DRhBWBrE`}X7FZ^YF33WQh*AHvH5})JAjurI9~U8Giy4i
zrG!|u0Is9oy<h}Suz-f)V0Cms&I;7dw?(qlL3U<E_=F<Q640c2oCMlB5J&)9w68)6
zHK~nkd<>X_bu!Xk8yp|Yta4l9h!n~e4QLg#EO?R;)*@m~0JTk_H8!Yq%DmyYLYd>9
ze`o9G*Oz~;f7=*6^AY#)=Qign^*XlOsXFVf^*z5&;ggBW1UG>tjEqtO3m6Z66k?sc
z@sMpP)0Y$$TgeOkjDICAaO)Q@pT{3`i+$PwhNtWr_7OtOs)iyDw#HYiPTn4M|8=9?
z*E?5=g9Db`TW3DaZ~BW*pZoKrN-9n6`6R2(QaF?@kZ$bom+=*UKz+lnFQ;Pb+LYWO
zsIGhexi7Kvj~#pOxO&k(xzB61Xy0mPQz>*1Yb(3OSSr!=T>8j~A5VThZe5~?i3%T0
zjn8<s(CtgwtDcXHkyFy0cklFHDz$nuix>|RH(MiXv+;7aLe}OTO@Rv%n-4TOCLC1Q
zkRT9YU{RbU5N^<+UBJNF(K=6r0W_%$!qZq8n%cM>%UV}Vu{B-1)Y2?$-uc3Vr*}UP
zjVw^snsImPwdNhoj&~aO3!GfI;j-3R_Vgg$M|^CB>=0oFhKa)axYZwt^-a)`*|}_s
z+!0xxqq03(6H7EcBxrm{I1w!0k`lV#<YkrMVgrG{l_^h3KiI3a?!SA3yYipMjDm{)
z^Ag_m?Vr3M(Bs9w`X+<_pAY;m=l^lbcY4{CGxPpEx~!M-vtpH0{0r&x^{@Wlxn1}2
z{=HAP`wy{vyYwS7Z+Cv(&cE&bXY=P5i`W0Y|DW$k`GHS!_uhK);rf5$eb45z|64Bq
zA?5ku<pKMAAAjF9yE%Wq#`%V7`&Y-NybMnbzio0YbmD1!z5P4BEmm8lb;aO{)$iVA
zJEAZ1K2ch}Wtr@f7k-Mf&*eG2XwM7Lsk-NK@3L>Xl+(4mLl;GMHR`W=yi9ZFZo3;F
zp52<CW7PP1F>gJ`|D4H!p7MDSB3t+V;H};B^7UE!{Xd?I*VbKpKK<{{qxSn>eD1ab
zi3loTiv9ny=)C@~KbzoM#Q*<!G(Y~!PuKbJ-=2#@r60|Y{{u4Q#pl}c&||X~omc-G
z`0;%BxsFxm)f+wT&R@Uc^{VshfBktifBlZ5tIn%;Y<+h={DJ?~=fbREe?LF9j{o|T
zCFsiEpG9_Sf7RGbTwP!H_5Au@H8nq9{j0Hb2&uKZXK>X%UhT-L^T+4Ke1GoE8UOWX
z==}J<pO+rX`c+dhE4RMx^ZEQ=HB*?*+VB7Q+`P8#^7HL~f1b79|MK%}JCMl#Kg-VN
z|N65HE^Yq*&$IdWzx?!_fB)NabEx#Q`S<^TG`#$L`yX8T&*#!vHR;h8o`3)Q)2eUh
z>(7?6cmIA~{{PRq^Uq6e@3W70XDX|;d+uEJzeaY|)vrHeqp#oFX_>Lc+URA7*4(Wn
ze@iS={)_#dR{j2$q3U)1<FS9le)rAqT_DV`PTuGkNapv1ic4=^w=X$;_hYDJ)zs<V
zekta^%-{I&mae4pUFFaB1&;0B@HXk%K2^hy+iz^QDVu+1!U@^K(~o?3aXm0|$&3T{
zowPH0)w5TyIjzq6;lRU|l%deZf8;}B$CE`;9S@m5p5FCa%}^vQe2P|GVX{RUkGx<|
z5MS?!{EsW=Stq{yG5_H<jr~uRi+*?P5IW!d)$DVj_4BPu8#>pXI$!oBI@;)l(!(P)
zPZewHmamxDu5ka+{yf73J<B}X6Gu*5T_RE_uA2X((O{jzmy829gv;~uc`v+WF5ypq
z9Mk%Ov6(N$qQZX1;i+A6&M(UI_8VW#cr|U0LrmH|uFlR--{k>%{sIqmp2)pkIn_9|
zZ|0?A{_$PGucm4;rg|;>8eKC>X2qGwUO&R`ykmU-z4mCH;hiZtkIL3ph>OHp+}E1W
z$$Oqzx;#EM=6=`vmuY+dEA%~`&;GCS!F=|8@+%f9Pkfd9f4itz0P}h7nToR4r|5>{
zGR+feSXM4G-K~1w?g<l8uU<Xx>-?h0c=f*}+TN9&ExDin|6AV^T)gdb#gV7g&mVnH
zvrRhj&940T)t@$dcEmHs_13-bk@>^r>)3m6<;K--b}-i6E!0d~$@b>km$w_mx6Eh|
z+fct>S&r+0uGM{}vTCb)Ocnio-y81E@B7YhZ}DT>gzw8A+ctcEQNe%X@yj2JuT`6r
zDJS$Vf2`cl|KbM+NQ~n_dC4EX8{aQ~v~IY6@ngFD{`r0P8UOV4-Dj+}lbz34<90m#
zz&!us=?wEPew5ts{PIW11K}k#tZ%-S)UbXKm5pbtwv~-%tZ_Xae&C+jO#eSyo<DMC
zTKDTu*8AqaJ1yqLfB&4lkKb<n#@9c;?m0Sd{SC1vKd#P;KOdd52PFFQPu0Dkd;OI;
zp<!`V-}WrsBOg5P`k!^N@%yYy9#vl4UGX+@?>C+J>TB!wa^ILInsZ)gTRQLU-5qZu
zkABlhuD-T$FZa)_e{OD`@;37HH@&p#u+6roZ_fLg^S)_Yy6f%DOWsCa{-&2z9k$i>
z^sRZNx$h5cOHaMMdCS|#+fQ`fhon?r&A%!4<#*-v-DloLKL4guQXRHyulL<~Tl3B*
zZBO5Nd-t8Uk?+6hR8)uU+3S68o@xI1McdP_-roJ@ZRGcFIyKc{`}TU@pSLyt{G;vZ
zUvKaJ^EUGTHvxufjfTA{3D!(g#P-iWU-_b8Z%e{&mMLP(1U(u575|?7%PlIdDnJf~
zNM)be=xM*c@`WP^y#$lrKebCtNmj4#aIA!&P8g%Zv63+RpxOD#Wqk90w98bu#NTw+
zF8?`w_ma0$@2}hTJz(O_qgS5aOJP0w(P*`;!?z!`b^CX0dtM;4_tes(YgJwsZH;@o
zcH+E;E2|5>1Xqjo=x6^5IF)Q#p#8OKpQHHphriS=t!xiIWZTbe<h=jutL#5(&EsF>
z?>&9+`CfyXtNTljUfx@JbpD>wNr&yl^8%Z<>-8_#9yiBePwA!Tz2dWiJGb{;`nHmv
z!|e*^g=&3`MNikBX8yDCyn=Gz@uM&H2FzKor}XLRs@D}$Y`(7J*#EsrX~z9sO8@Td
z^7F5L?QQ3EzE$<t=W9#<R=(c!d;ix}0tdf08C}~`I``k-YobDbuUW=@+Ph%J?CtAX
zew?qgo9=Nv+We@*KJh1>AFfTk^m?r}>$&LnpA0JWU;MedUGMGV=x~Aa(dvi)6(4l@
zes9;ERrhwq2;Z}^JamuSYt`p#Q(GRb)z0|6R`}_UxPXWY(bJ{>2J0)$%$K!a`@Lz7
z<h@;Hr?%glGiBe`bp|KDHyt`~f7c<k`@8xURKND_i#yLcDQ$nLW!>IVOZPorS6Q6?
z-gN2To>JL`|E`IK|GQ?n`14+axjg&79+mqb<57C$8-v`{Pr?0Be>2^4qx0_H@-r_9
zIexzO%DeeLi{~Y5?VHsuxcBSYC-zoPr2Z{_^=#MnU77bUE}w4wHTe8IX`{=wrspd^
zhnXL(%Z}Fld+pAXZ&z$CJXv`!FS9yp_WauHJiF>FyOsG@&!+jW{Xb>9d9?bUZ)umG
zeJXoiCVV3}+Wza?vs1X8uGFb-+IM$m-npu*ugmxCotbyEDr<jp@cnmxf*$=Udp@1-
z`gZfnx<USL%L{JZp8Hg)eR-Pw+qb=IykR0?OP61(zjb?d{Hojc?&Zw7eea#lb^+V1
zUoYRkm!r6S`r=>KHhbm1J~}pUdDPtlpF{4>`5ac?zT&?7-Ln??U;n*)y~5A@ip|y9
z6;Cc2PWF7Za@Op`d8>4G`iJTT=HET5oqx5sc5kTpvOQOCPkyqe)O^aG(1isdr&VS8
zR>{1$x9j<;y4CaUd|i8L+sBaI)<OHey1(%>D*F?)K1U{}blS$p;kELAcK68cfA{_1
z{?j);o{qcu<GoSYpQ@W@>J^iZum7I>ueC>Z|Goa7;viA`XYnU)d^ddg?ni~sFO5x1
zu3Jxi`fT*z^NcUP8qW=19zJWGf3>)EPiX(z`qig{BkaE3HjU1|x>)w!s&u9L)mNK8
z+~4(lSIz2q((l$*=`Gz}Rn2#1`@6EE+h0w2ywWecYj5ahjepzY=2YzY`s&K#wdTv^
z^7m?YKC4c&V61$-J$S-)ySYt!zAiOkIN$pAZ@sK-OxN*>*GE^Lh+co{(9gJxBG2kq
zcQsE&%lmoW`?c1F>3!?VBTv?L&$7AwPBryTwE5DHHTojz8u@#zTq|Fn)jSpb-pl1f
zoP=M<y<cxbp04%Rd_QlO`!~7RBegHq-tL!`&X)Q0>DqS%x!tcPhyGdlUEbR4hDY>I
zu*jRq;xAXOm$x>%#j&UUb+thM-EW<5s$L1E3w(VR@_OxiN1L+PRu7rjF9Ds4+u!{t
z`EsFO*19adH2@^kwYXicMc($yk(0;u);tdC*UM|4l`MVj<w|#c+wwCTg<o4-UEF;;
zeA)at%T_#!zTaH&=Ggn4ErG|=|Ez3xF21tcN$o!O;ge6I_s{j)_w{ATr?vX2I@Nk5
znmYG)B~83nRT%a2+Ug{|``n!;ZOt;jTUPJZGW-7a)wAM$-zwu>$JK1rW(ls+7ExYx
zU%X0iRa;a`mZP@Q*0!LQDra@4t&4>gT3zS}*z2t5RJ(|OVb>4;(zp!g6;T?_E5u6_
zLkcUqvz&#Uwzey^{Bq!T`s&8(RJw?D;Vvi6MZb1kXbdn7Q3&Cd5?Ey_qHsm}7yl~R
zuk2SFzcs&-J;nb@_%3^>%U^|i*IQpHzY<&3zat=^)@ARaUmaD>Uj@I)t?J)l5K!y#
z*X^s|*UneUue3wtV>WPKb-%)U)%^<hs{R$%18R@7wylu=+P6afYvU{BSLG}09vyPI
z(qGl_>&j`kp#onQ#|eI2d=F%J`zvL=KZlk7&wnz}+txBX_|xU@PLAtEzG|7C3eI0K
zb<w@2Okd*y%nokXTYWk@eAle)=Puqq-x|H@_jWzr3(?{mw#SL)e2&Yoe|lr~+)v*p
zAC_A&by1!6-&fU%i^98hZJW1z-^u@)D*ij=itToOKI^{r-;~AACi42fE&p&UP}ctM
zio5oYSDc;ydqpni<F)Hvr^{|Vt15q0X=_l(-<Y1o_m+frz54dVdev-Q%T@2rRbIJ!
z^7mT3my<(EpH;|)_J6Q`^=`}Gu+=4Afw@6iu2(<4S@GV}X4UpDUV(FibX-dx-?(U4
z96x1ymi^Zk)2Gg_`jS2QMpem~JN>`begpB^cgg&{^<=A@tnv5my4A0j%l*}wKE)^S
zu3o46@vh&N`>$%*28YZR@eRDYqqO8rdBv@Z8}C2Nbl?AE!=JF}*3NccU)dStUtN;l
zl{N8@=BoY7*ZTWH%Wdmc$8s%S!KXQA(J$+Szb|gQ|L|pT^8Uk@`|C_CIqk2`z5C!r
zMDg<n+oP5ibY)FGq`B(uU$^_as(<af8qKwQg`eh}MZ0X)9}0f|>1ypNwbtN}ZV|u0
zwezlhS<Bv<FLd=zGS8LZ7k{!tD=yt+`<A=QF#gSKtGzsTE*oBa>u7g<`})@h_x-$~
zU|aH8EB~s0V0G5j&9m*UK3j7(`0J7{{%4CF?YBnnOqsV!U9u+Y(senf>tD_8D)N2K
zjpSClY+>fl7R+~k^4=-JU&^?yFTYu^@7>BOt?SF5$?fyc)qGoWW%h3!TT}k8by-Ju
z?p+>bYyUs^Yp~C}sPeE+E32yBEHA4rTCm*8Q26-8n(JM&ZgbD8zB}cU>8hhMkFN-Q
zZqJlapM2$C!B^qcPv>>MpMO3)_Ts{K6BKv<`~TqL{=b%TcP?$|Hb2*RbARQf?b>Un
zJZ|)T{^+8d{r~0n|Ly<kc-`Mq;ws<w&$b8t3)ZdsFK_dJ`RjVQ4I!ETxsKHzD*y9c
zzG6@5Ki!T`^$TY&{kz@q&wBd@Ez9O{A4%Qx|Ac0BmBs(t3jg?RKIj`S{^S1jQ~lq@
zYyY$rZhimstN&-;|6+cdSxx_6#Qn?vaqi!JwvY0E4?K4LBY*ew|6k0H;{#XxUlCUq
zUvNMB`|7g!ncv(0?by6|cj&Iko7fdACNw5-h$tw$|KDHp`{@q-^));1oSqps?^99p
z^#0%Oa&L<Kr@!ARb9(CWr=@KF`(-!%cjn)7@9qD;)A#)Q_<#DoC-us?b#vSQmzRCo
z-yd3U@J%}N|2F>r*W*5}SBtgy82@hR_DfoK`ZEeAy?j(96cOVe$2H?a|N1)~%8b|F
zytbHRci>E2O{9BY+iTCg1-6o+O~=2Q%5w_*J1f4W{;TBw&yRoPUw-yib7?L2n*ZEQ
z_sZAJ|F87_8~dJr`>*uw=Rfx2ee;idi~g&2{5Mp7RbscvD#m|P+x_J+u8;5Z-7c#U
zE@<|<?|UroQS_nX$t<=-9;-O+Zz`MYp?`qyTj8qbjE>*)7RlzCe>~B8Tlm@!w&w@!
z1ikzN<{$rB;c@<sZ_V%D6+Hdz@6X(Ix&1KjQJ3O*sb{+%A7?JMQA>ZTwlBKMVqW{B
zT-o?H^W<CaYktYoK6#J#YWVY|z1Oa%=WM(ebvkNh-i?yI8^g9u+qV8~f75dji}Zc_
zlGDE(+jnTK#^0Is%XlWSY|*+g_3drx-(J)221@^O|K@g6Dm6W&|2wPx<wa+@&aZpF
zcJ_;j=572Rj`{L5ndkk=I4&8<m8!~<=6J<tqej>B%kJMfE=?8u@Kj^Z#>f4hjiz5N
z_BnT%8Xi~5J$TDTG$O6fqf7m_#*7|=8PQ9wNLyQJ=ADsQD5Cc*aLtTIN6u{YdYH(?
z8yqpiGD%bF&Ef!qOxdZE&&li+`Mz|9elg4DrTuY>9xFY$?%z_Xxy1Wn%+W%hz!>?~
z0?nG$7Lzoyyal7Xu4(lpb6xtS;pBZW^5{e#$4IUdK?Y%gr<9Iw>}T2_QWLtRKSTMV
z-=eiHm5nmT?Yh<n<%F)h)wE~h;~7pXyB^O_T4BdEpDVmP|Bc783{6)r(c^}!9<?H}
zp)x1ClxH4V*!4J~P)55ixhwXUM{SJ2QIXnxzVp6*zkluAx4ED5)&K3T|DWr+R=e!4
zXZ;Oh=6CJenRi>Su1*vBToX7?>eOY2%46#1-)(7sd%X6{_E+WqZ~AWYpYA@Z+Dg@b
z?w8%`-<hrRTX%2i-IU#Nl}yU#L!WqkxX)fb`>pfSm92L!J(6s;v7VsL^l6Fx{_5*_
zn|1fj+kU+GONEu3|K7)uWj2fZf^%+*+_sTDx9r9<%RIBAX8tjoHMdzFEelV7`r*@+
zh3^(Vsq=i-`{ZTZluH-?_TD*tqIlyD=ZU)ocYdC*-1Fi|%}ew1mn*oM?Gml3{<~v+
z|M5*Pmxz3SRwz7CeO>A2{^#qyo>@OHJ6e5l1;@<jew+78(&v1s*=A+xuyd;=r)y>I
z)T?2hTc-v^|ID4L>bP@@<xiK+JZ1kSi<>!RTlk$gj&sZ{S#)IkrdG?NX3KA=9GDt<
zT60_Brq19Tqa`PMo{8L^lQcUxr?@M3(T!&!w^MwIyJVkn*ne=IDLE%;w#GKgqhWor
z#a+2ZXTkc5giQiLiv2d81^LlzIY<*&;mI}cL}rWZ>be|UGB4@VoFw;M`H??FvY(6G
z771IPVAXm-^S18Gy`n1a%l!PkS8G@t-KY7;_hp9Xb6xv{%KxXNG~MHOem}J`oO6%o
zecg7JpNbYo&DL1F%T^F)UVdZZy68HVBKaRb<&7feZToTG{!@P9j&{BMuf&(|-)x-x
zJ+MM`r|{c~y1I%J@{toQDw6NW&E0O)-^G`8{aLhr(7|KAsqT7(>wRBrlDhVKg;<~6
z!Z4k?+A9;i&F>z4R9iA<>$Tv+Ma$jOt4!`{r{|Yw-TEl2YJ011*2KD>FSES+mz}%%
zMe^;bSl{<6Ru=0vY8sc9E?Kwipyu;}6Y?)=7e8gR*!?*0h;ClwJU8$88JSYLU$0cX
zTlx3Ey$O;(*IW(BcK6*}ct>Vs+45Uwc&}F$3r}4<t*UtXl@ES*cSf&Tdhd?1xzF95
z>8qCBJELtnXQj0<?>V2nGb;<yUY+x4HV)2DlbbVh*XBzP)?ArAl|6mwyEP^j+F#S2
z9*kURt<8JRs#@86&GLnpPd`rc+&U{u*Zj@PXC7OvucqbA*p(c<YU#ZQDVg*3b$4=B
zPOC}=*|c$H)=G0#em~E<nwbS&uY6B72M3jVNzIv575wSJlq+**u`OSA?~09uY3cH1
z2Uo5%H|6*9y}L5AAnTRy>E_Ep<xx^|rd3`3^kB=CxzpH|FTJ<L#zMDr`SOD=SDM@M
z`}yB}nO;6ej!m&N>qkK7^0xD9f3G>SdDA(!@1<ArthJx6xpL*jn<KSlkL-RI++TU|
zX6NrWM{Ir@oVTlacC)jxENNctj7$Y2=CzxhrDaKTYa=oh(3yu+Coi7Xc1U&V<22u`
z)3R*MWnMm;n`M0U(u5r=KHoF(-g-rP_0GgseP2Vo_inT{_qn^##6tD!vL^>MS6Zv`
z`g!ivtSs<)b<X3yh0ukG`wiE+cza)TW1F?#aIKs7>5G?&wpiCEM@^hIZOKyA%<_N7
zwrJ_<dV8O~^sa{?Gc_t|*0gEM&e~XHmPSR*n>KAZh}jwywP@P36>DuQGN(pGEt@uN
zC5X8+Dr(iVX{(lQUA5VGewJtV)+Jk`E=7g%9eJGQzjb0(d7j1Pvzx6|*Iu1;?bY%n
zN24w!>1M8-wk538>UrU3zpYoKPd85%%ZBLllAE(7!aHh>>e^64HTT0;=1#&LS!Z!a
z)>1@dWfoLDa+Qk8%HJ*<Iseu-n|kf<kuz6bw95Kk8un(*l+71o&qt@LzuV0J-J|!M
z-yJaX^P9tEb1m}b&Yh7ipC;7-iZujt?%arU`7o&tP^_UbMQVO*`Q-2mcS05ApJAW3
z4HR7yrI)?C;j{J1>`Cm)m%N*T9C7D79NhmMo48=v)Qn8kTPN<%5L>+TYDT8%t-1RR
zbv?YbFJ0;a$J~@@VM~{WW|q4(Y%$PX>*K9`IZbZP77N|Ae%`CWOb6Yy0p8kI!sO;`
z@z7lx<h>fq4A5O0;;nrZBnk=XtCOy+n%`c1FK6Yo;M^x|<-06{b-${u4Yhapz9%vW
z#JuVcik+iTm!fnt*Z%6<ef%sal2$K$au6<#l;6%a2M3pDNzIv675w?Z6_ot8^kC*n
zb6tKv@4K0q1yLyZ?el{#SLV)RTfY3>mp!cam$NRG7nQy7eTL4vKkt>(+4`&Q{QXt=
zzxcTT>z>^wc3qyuXY<r8BBi9Nt^4e}Xj$op&%UphJ+|v8!)Ir+S({zYp51rXHouPD
zY5DY=%9Q<2gQk7ltNUxI;@5_wH<f?y{<CvkU*fYpcck`2=M`Sub3OMDXT;u+W3w6d
zY(IWK-QMb-lEUw^-cEm?+OD%&^Zdo6*Memg@ecc@yeK}+za@BmGS8Ru=fi%w_Lph%
z`u@2+XYJkppMJCL&9_^(yU^z2^u&JiZ6Ejj3Ce5M+P+I$?dE;czu)wo%+GsY`@L<t
zzP;e%@2C6@-}=)ayDazLm;Jvp{^oyR_+9&YvhcU#bHz$L(qam<pa1`o{VlVk=x@d6
zmgC09BN?2R)c?QZc_$_LOXm07w8N7Ne@L2ck2t>m)1N~>FZ**<)qQ<_y+VG^{P~}M
zZjHO~KHKiwz7zM%Z~j)xK7RQ9={L2r_vUY|wE44d!ybK|<L$EXVfVzReQ$kQ8*jO9
z-5!0N_uQv{@2jdwsEv=fFMjQN?bja%e(l?^Pk+t(->*J4eyvTcj=yp5`_{+Iw++kR
zbKm~GudF5kBqYA=d+pyJ2mbBbuwVbp``^DlHvX+mtdGBO|NGa+%)fsh`na!P&wJ_P
z?0w%`AJ;y#xbXSM0g#Zs%=_Q(9y7oH-unIbq2IOlYil0--S?p`zGDCT-TnM~?u(!M
z-uk)rzKvbMKK(iGxu5^uS6TDGChWoQLqGTB%gXP#C%*1`>+9P4R(1t@^<&<1U;n+Y
zwB~`8_+Q7Eb;(!bkDDLsWZ!kM@BZ;q`;Xmx5b@`_O`UxHxi8NuzIqn=O`4o>&a=dC
zRq>>`F6DfOlG&ebESj|I^J4x-$?RV@HvhTdS^q}G!Mgll*}aEHT<?7sKF*l4e*@>y
z^0N!x-RhVeop;~n!?D8qHqWEx=6>4z^H2MH*_X4A-qTtpx7+2)Uk{I+YP)8x`tTs^
zE9ZjOlUOfKO@3$m!Y+^T%Of4Pr5uxQD+MvUKlJx*Zvg9tcY2HF&-)P5|ILE?V(cTX
zgtZ+`+DF4#b7mGz(SNs7{rIuHNm>cFCtTiB{ov%}uv_tUlRXpu%P5^I>V5Rw-JSc_
z1M%+p)!&}h?%uyd^0EE;AGW?*ZqK>?{COEu_L<P#E=QkkKXRVc*4&+=u5O$E^gR{k
zGpDMr=T7@mA~x|}-m)&ef0NAX?)g5sr}VZ>F7#aQ-{0j|tYTxr|HLfQJ+NfwlkJ_F
z-_AXVU8j7e`pdVZ<Qsmo=gRd7+Sk7<TfTnI&E+58eDc!IJ2IjCquQjyi`d2XZ@)iB
zCg+dOF)#jgKVEhj{#5)gk}F@-^!WLKop)27JpZo7(0orXSF<Sd@$+k8d><BU`?2%x
z-zR7HTztuA^FwvdpEzIZzD?R6iml(@E%{a+k^d_z{LdYgcgr5T?w)Y6w(Y)o-?wL%
zgze|=J!#YRWLbCdk@J<=lZ{d*bJ+Yix_RZjZ43Wq^{4mFpWSu-#ac7HodTbV(;ZoP
zPl?*cGOFvpk$?JkM*+u^=SLpjf8u-K{aYQ&t2cLTIec<&e&*KQk2e&#{n$A%Krs8(
zgRY>3-zv<6ul>*ZE6?zs;RpMH|Noh8sH7I8eEq$7kG=iZzjyD=|NGanul@b^?=|)7
z9;+VbRy94|`*?2Ip3ApxKB!nL`+n==y=Ldxj;}ub@l{dXo1a^sR+L)C#rBy`|J)4{
z+Otb0f8BHU>yNK~tthp<7jwRU-SO40KfbEkvvZz&Y=8Ln$5-E06kFTxnp<46XP0b#
z-g9@bP^tC3JLmh~9bf(X<Ey$oJLk*a?GOL{`0C$^Vte~t^6TY}uRi|q*~GM8_IbMh
zm><{+9<T@~nWJIlCsncF1%$~bvX>u>{6+Tmhu=TG`u*dx<x2ZO^wsYlpH;3qUAd=9
zZvDLDv!DOG__^Y%O`NU&srtGvHgS9V!q-1O`})s|(mhqO>*J2kzW(##>x!>d<#V^Z
zT~hnp%6#7IJC*B-U+=thuXyh4^QG_XuJ8OCllgz=Ilnox=Py0`{?6YMWgqWkr9XXh
zcqen6&fCW4Y8lVfGE{k8Jgq-i3OskqP~|<cQ~cIW$y+-m-_@rt-YopQE#rAxhAHon
zo$9xCYTnwZ8Rg4d_^!Jwucs`p=h`KQIn|5j&R)FrNY9c+i|_8%<{sAO9@&`+=WH*X
z^97+QsgC0LTZ$Kh&^(LD|0^#2;%}&D_{03*zxT!bMRoP{+rB?f-?RVu@5_B{Wn2E-
zxOlB^TbYTT`(uC0sOgIfEv2R}{<vIj?dyv><a%FQNXx9feR0R!_?o&R>!|IE3+<)8
ze_s6i;x75#-xn+9$lLZ^dz@j@ckJ<>7r!%V<$K>R{#aZ0rFP3a_s5kc^Pan(FJ9Dt
z>~+STbNzAduS-qhIDOBp7n}a|j+Nn4tH7sL`|k*?n!6*7^SWP(vi`~D`J9pd9O{Su
zvadT=&+tc`X~Sfb$VI+oLh9BlC(nIyQmA&*WQUuc3*V>&PRp1w*X5L-R;v2UO)ksd
zsDzXSX<2(tRgFqj-?(Ywi{G4z7qzW9r_B{P?Kdg4Gwr75mN!eH$|mW0mz++!mFmZ#
z{&JJc?l&rNWkLGUnTx%wuX^q~1x8*<@lREtNOj`+NoA)e*G%0KoAzmH*%@Ko)16bF
z9y#T2d2!R6j7YyVI=Z|r-qqhYA09b%>5>CTXp4?|Se#Mk%9Fo$@3lYvyI79@_Qgwm
z)5=V?7MfW=gSV{gc}8sCv9B31{m154ZCU63xXL8&Is5j{-TlYjX52Z~|IYn&oyj}*
z$8~jI<hPv3+L3<t^~ZCY#Ki5b#2mtZKR&tiF=I*Tp7Zz4U;Wo0`&a!=?Jwqp)dwHF
z+H}0x{pXL;tw*l%cm6EhX~!R3xqjQ@#A?0#_rGkf-Clq1v?FK%*xaxM%KL3XPJzy5
zVtK&yfa$?5<^wfJYY%OWTpB3m<;S#3YAM4j2EF|Su+?I04yzdq6rz?x&*9Ty&|!ma
z(*SQ7WKa}+=lPD|(AwR|E7Ysl85F``$J#xzP=y`LbGxQ!%?pWa&`E2M;|1^4-xEn#
zCinCHtNPNK;P3lyKm4_Q$KUl<>x1=su4UzA<ug54?)Up)vlh!?mLqBttE$!?5NZ5*
zr1MEru!V<LLe~M2M$I0T^n#FsE1QCU$~3NI3T9o#<+C?a!y>d`)&Y~oOs2~cVvU(R
zKEiX<EPNBBo}cIN5uc%FF!$igp7jq(8ecM9W^J47+IE14EuG_>z#K&b=Y+NcJdKh}
z#uA_dB_IbefzQF3na2j&5R1HBDg|`%+XuG<$j0x@ENO-o@U035_B%Kjd`POWaXP0T
z;<e!e6R-Y<4@`pk1|PeMW8~(rtz=&9n9aR~Zx7!Qw#~6Nmsxjlltp)jGC6#>?NGpc
z_^zVA_zR_oyahWOpRvu0?kyGP=8rL-aQ8qN^IhH&^AC3&DzXhb9_(rS!gftMBY(l(
z#_w#8qPrbgzVkjY-*Dfd{<Wht!wIGha?K~CIv$tygeK@PCre7aR_L3(S+My;lz~|D
zMyZ~s2|Db_wi4Wj-buJzYuTvrhL4HOVOr}(y`I)YW44J~6Lt8KOC?Spj*v5JR$(eQ
zrm)Ga$2Dn<Q1VQP(?=rIk2!4=ZQjvzifv}=xlIvV*-JNsfr%R%#jU?39gll+qg%Fm
zW3qnPo1=Yu(fMFST?lzFAFN2bEJv1WsgapD!zPG%UOBw)^Um6fc_(fZx#pF*G4|-e
zcEb*xX@`<D!Ni)WhoUssOcjcDeYugDYf;m2AKQ=nul_gtfBWzIdmC>*RcDN|Pcptz
zx_IlX*|%mh{OEgooO!hXkFdb(B|jBQd|MCv=zIIvsm)ZBDVSv$2hXQ4!5RD$3qlLD
z<|`Y7CagLT(zw!`m$jWUlj$<cGESf52$p4r2EGNe9)J@V>oRU1@fqhHlpG5$*mVGu
zrVihXcTQkyHf8~(sTuGz#d8j_+#hxtl8<?FG50x?)wV_!!3Am+$_Bx(Mb8p*R6%J9
zbh5yMcm)NC3PT$!sehOH-l|4aG`5LXR5W&o&!{l;`S!rXLpFjpDtbZbgHi>v8Ea*4
z>tAr)aNBE*5JS^_wkBzbbr6v|4xc%Gn0&aCwPtrj#e-dqZ`ig;zsOs#yYW5SAyE2x
z&b!2X!##&;=C{0i%mwa8@hI)cSJ>b9m+e<H|1Aawv4*b_F4r7n8q(J=B}+=2J`llv
z%xMD+QdF-;;t5-Lin4g}3*_f7(~bt|aF{bhYO8K*fW^Mfv_nyv8wHQ}eaK^V=sBxU
z)_j96$-aYEv8?5Wke7@5<_NFev#bs9L~>X={PNM~@(hvMOOKYZ9=M^z)xC5>1cZQE
zcqA;a>P<t_it^c259(L{&;7Uk{|lQS{k;F$tqn7HpvfrXxY^qq%VckFNuFJnbGa|~
z)<$3JZ5u%x|7mYq4~Y1siOpy#b4svtSsi|GWlbL_8G*{DdnO_>5$ewtbNUF+aLfPp
zVW*z|)91er)o=a7e!!mLAL9pTr44P4)vGnA`Eq>K+sn4~RVLTUoXEoN){8~nZ`^CU
zW0yU?ZOgpu<P5>y_fw_v6LakM%4-S*^nE{ecj<09$y$?ty6>ka%YI&Sw~Xo6)bjpY
z>2U{4-#@O~sqMb^7<*3Tm+z<QQtV$<o&9t>@os74x*gk|F}!&-`R=tnZ}*;)+xhwE
z$G6dpAE)Q<O|jm4KJc^7p6Ek$%MP5AjQlm{?63d(O4VO%DXV!CFn`v*(DQn==XAGz
zu+X-<arLlmjO~|%u6F7F+AN=hLr!%D7@50@ry7=J-g_#?Z&SZvt;hF|39i*!ul~-Q
zyz$lCsQ2l?hps*}`?dV-`{&=2mzYUeHSvC$xAybPwYd?`MM}@s-!ThGKJR|-lC<rG
zR$k|*&-|LJ=iUCh`k%JUw~v2iqaOx}{Cj76ch{UpoMJUW+Fd_4GM`zbS^O$K#N*z*
z=Z2cDU7PC-BOmNPzol|n-oqW?`3d2@?<MYpY~QiF#rL-5vjtLhG5&wAzrTC<`n|k&
zbArAb&N@B!=DmxP#Z25*r^?)vGkJU9xc5zGKBlt11Lx{$6h0qM{k?wXs^eFBYznKB
zPT#(Kelnwp-I}G_?;4$R<DL58@@K8T-<WGWY^5d_%X_=N{Cs|2g_w)=2?fRf$0g*G
ztFj;3&(|$3z4OF3n)!iMXYn4}**8A=Zq$e^-!obAy-MWQH*Vf0nlGL)1bS7K)})A^
ztBiY}+>jskcFu$ErAzbsetWNOUX}goTvqqBX;Is^bR>%`3p;$xr>}Th*{!LIKVDnf
zn><6$tY7YO=-eB&n^QH8%G<1!yz`%V%|UyHf8UuW6m9!ga4r6$P3igf@AliT>Un(Z
z?S6LejRi>$_bF??Xpdf3tatr<@LT)BBZ-scj1{8(W}TR1RK~c`(Z>4R_1#W;VlMj5
zR=-jja;h{;XYcu2A9<h3^b0W;3DnM2{kd*w-JL1Q?{;sV|JEYN=vMFJc`=3lhw4o>
zUpeD{tU9Y~Te6j@ZC>|^`a4%9yve&J{Nm>8{<--9o0dI4cqO~feA&&CyWckU@8iE_
zl2ra{Pu0HLljZiEefQ;^%`B@^pO(&d-#slZJ@cEz-A@bVEZ_Iji`P4E$ECB=o+<X8
zzP@L=_TI&R3{#Vhw9C$MY1GEwS(AC{mF2hDoxc{}-8n74_tgJplULAuWzX=hn6c&1
z(Z3>fJKGH>+x&a}SSbEqdi?&YI_)8`Q-rNv{q|T`C%=7ljNQL<>-{Rrc5p0N_HO=X
zlkaPa0&}lN-u@@)Uh4H!ME7h^?e6X~lS^m3e77Mu;%o8jzN+cXAAg^8&oy>&|0;Z|
z=eetLW>3_l=eJkN{koIBHhyK;xm(j7`S+d;o4)3Abojb$OU|#HckEr}ou?1}+}Zoz
zp<m?*_uYBs@zeBjN_O=1otw$Bzjj$q?aD8A$~EWRKOQ&tYi-??$oonE1HW(PJjdPV
z;$C|~RV(g(aHx8xs@)gAQh%K||94fVlwUD!Sf}#;_exh&|0!F8!^47OCw=_s5H`)T
z{c0cIw>1fS3WKZPMPCZPvZVQE{$w@o_I1mDTgPWx{x~r0-S02cf3COQaQyq1?Kc)i
z9RIAmr9kcby#Im!|MI+AymyPs@!#v`g*>>_DYI(#BsqE2*IO^H$Z4IO&2w6N&(8SQ
zH_bYVw!14|dcFU3rFTkcjbhsQ*U$Q&dES4@{`aeRc8quF72BJhm*%}aysnQit~~7U
zZN*o!_-a1wQkwnlm`?1vr?(`%`U`JpI{AkN%zRn)cIB>5X_Mc-o07_Y@c-P|!Oa%o
zyCVHUKQ7<3>wD!(kE@r27nIEpwQt{%{;Bef%-<!aquuM@IadFCmhNv+F1LKWY{qiG
zv)n&Jgw_ks&UvDn5w)v&Id7(#%N@SP^vk#WLtm;reSY{iOV{G4YniD(-ZWZEzwcS=
zUK+P^*1ads@5+A7{c5rLtZhrO!LP{MmVM86PC9kdb7^nw?<<#$woGly`Ykih|Npw?
z$9WRVwN58IpZxdwmZA*aT%oXaD?|gE&tI98sZkgGYtQWOYd93XtU2aC|Dq&sZrQqU
z`y8c(#(7OO8%w%P60ObuEW92&!*tr4inf)XRx8KO|L{_EX84+UZMF`&wHLYe`?huE
zwOYEy?%sF&RQawX|I$;PpFi`=+fwvl`#aOfCrvd>pGzJcpWsqlZlL$HG4J^iiw4^{
ze-oB&kG3px*8HHp*z<P1;nVGN0__teiZ|zV3Th>u$vRmeEo}bf!*fR_yHEd=UhbbA
zcq{jfgzzNU{$$ZP6+E9LpPgU6*7NjF2|4{lR$Ix>5B~}uUT<^x|E#4qcCNpF-0;QD
z^Q9BnOP+oAUZl9^w_N7pou@RPTO1SP<F%^#cIR4(Z&N+bX{D=w&b~8#x6oeqYq#R;
zr$=TgHt*s4ZYl7tO<%&Pgs1x-@7MOdn>|B*7b|~%bN9UZ7G`PlburJE{OV+0UMMgn
zv;O%srpoo&?;pj=iyB|PysIN_zPHAuy!kcvY9~!SyH&pUpZaz4^_M4{-}0+S^Xbg(
zeftmlN*r!hTB|;#`j?u@&DiLhTS|h?8_b%?^QUggn}dzJ6e>Q{s6IZpO!aZR@s)F3
z-lD&Sq-xaUo?K<x-`xI7d`ZeO(-(}U{FVmhzviu|yVRucN_tjLUwG7CpEZAMn8K%D
zyP)4BA1^;eIEDAmE_I&u0$h(zJp1bJw81}o-@Q1;`@7s8$MZh8Ty@lB<EP};vFYp8
zzj&K}Oup&fIp^BE^nA_dOB%jx^iNDIz25eTo%7<H*v1uK*L<<wqVr8$>a`);@xE)H
z#IES%7RV)R+P@<8(OI1{#hUjDgi1cfJ+D4={_*?YlaH@|Gm&e3+Xd;15!GcnheOZ#
ziqCp>tYdNa##J|xet*6G_V2Vw&xOy4?u)$Y&U)lfN9iO@f5#V9ufBImTt2rptE}us
zapayH2Q}k_I-X7Uq%4kpuQ?zrG3kK8o%t);B>x|@=V|skaLaMxH%E5%jd$kpFF%)<
z_U@a%lyF-8+V`tsobx|VO0{(>-&)M;9@MqAZewskO#GH@X^nf%1W!I9mo-B_HoWlH
zk(WGMn``Y<dKUy0ALN*_CX!>``|ixwGh3t6uP#pfmQ$HN>B!>zY2S=Eb@{VzCT;rV
z$tW+*V{O@|aO)!P9&5QPdxV;%zK(HNqh}|dmcK?gU*6)E%r5)&#rHovFjeV1JXPCj
z$GPxpR~Y$?Zq5I7<><ZLO<(@WNi>(I%G^q^?P)%?@3Y`x5%~qSYCq0K9QyiP-y}0_
z-Z~EZ=AL5$_N(5loH_qtM~9lz=`G!J+M5@*CQE5N4|4nXwQAR|7|#d)-oDw{l~#W;
zUer7;E2~-~X8q*$-zJgv&ShuoyrZ|4X5@X#e0n_T+mfASp9&t|5X=6tus7j{PH^8e
zm4|B!ECP1NdS0F+dhwxDSJ=LumQ$A}iANQzoN79y)MB=|%_`-uE5A*imvi)})zSBb
zVdk0^vh40fhPR~FI_JJ!^rkeUF8h(&FP~6*-xpVc^ILB|wA;1(!;eMQ_y4kpDc#Hs
zD$|-9erfH`#y3xMVlO{F*dFgx87}irT2!z9{w204E^C#W@0suIbFJB1z5K0eUDk|s
z+CQ1z<*A%4=kPhm)2HwH<5W)R+M6#gdfgNG&auR|$U0E&YkGg{q3hZU7syLi@m)48
zx_kZi%j51^6P_14r(M3Dv+e%@)A(uW@2{UY6=%Qbi|pL%z9)s>%+WfNzVX0f9Yfzn
z?Y}OsWH-)ndt3eOx7F7fyM2yLe;~rsCEw@oI{QHC^4}X{B=6_9RoxTrJRer}Fgx?*
zwIZRN3b(`4Cr8y!beN@(pMAx;h~xCZ^4!uBnR1292Qw3HY<h6;z?6ACq4V~ADAC^A
zJfZc~V%ysr&iP8WuMrQKu(u=hTf!BeY5kv`&aYC}{b+S#di+1bmk+{<4%u$A5Nyyg
z_NXcFe|PZULFqaEr}fVFvuR~2Eor@Mcxbut$8U*NX*<-m7IJi@{%QGk%K4}Km6Fp}
z6<g1#FtomoY5u=qNyeApx99J^IDNtTdFON8Tn+i75>J*c7pR@-d;j7s;W=y4qkHTk
z7_K|Y{}Qo|5I%Wl`Q4Js{fpmae&X|4t-p3^|A(EndzA#v{k5I>(d?SMg`gp4|Gd@9
z@4nu>=hC~DTkTU{i0w&s%;`S*GD!Xv!^!?>YtPiZ`;w`(&E@mqxs@IKGZywaF)BU$
zwLqaXMJ?;=7Q<ij=H%?nP4iqC{V9>p-uvN=uvPQFyb<1hl<D_z8L!<ttQYpqeD_9R
z?&r|HWOe!A$?|m;m0|2+Qk?~9?YHG0iJAShDox+N+3;MyoO=ELc-^J5ZPteWK71~L
zZ}#)oZF7wd#826IGu!6E(uCs?)6`yUoKUoTjmYa{cKvL=l)1`%S#C1rHhWYW+n29-
zW|XzPtvY{2r*wUhnW^EObb%x9K6R#i_A84huhCamHomlM=Op<WootWtOI;^es@<k<
z6=C`IJmPwBvnyl7kssE=k^6tFynH-)-rgfS<=xK9+p^92_;K&m{r_g%7g^2f{$TMt
zE6*?W8^WihUEkXG{`kAij{Z**yn}Q$cLye{fAINnOzfV289n<llw*r}XL@oQ8Y};h
zNL>=~p~sW`YTw^2?2-SX{Q7ronJ2x<I9A&8+ylPp)qcyLY2V;130?lL$8ttefgIn1
zz-?zIySxdlQ1aUSU#PffvBf=+uP;Ikl(wfda6EY}yEUB8=Z@OKW?sIod#(?z*V@Ir
zlM#pv{xEN<$n)(d&lOHz?05fr+{p`qDQxdFk8W#S_Vn(OYmw#FHw^bbb@^+3wXElX
z)$=_|rrtY0@k7P#BkA8Z$6em=weZ?8aq*A0?=Sv$`))API!}dpCnrr5zwy3c?&-Jx
zJk561?W)MV`sZL#{_TPye(zjcix2;PU0VF{ech&W%O`JrTF&j4=&<vhzTM1yDcduD
zW|tSdS-98Z+WEJoQ+@1CGJV-O*{CY@?w7eqY<Fd*yqP%d*;cj30sp%kZ~r>?x%h5J
z#exelmUbJL-;U-laC^S<*RvN<(l7t%e-N24m36-1x!v#5%I7|L_T}d7w%NryHg(T$
zH=bAOt$*+K(VJEJUl#BC_avJ?DP)t}(W!jLYvmtoj>}$G|Mkwz$(N<`UN@d!`%wFy
z;T4_pzt=uve>L^hyUxOM<#jQ}Wo*oKd-I>|I$r+d+fL_ddF!iO>sodi-L!dK=X6t3
za>vVuQ$*HPdF@bL$!(_``g*#0dgRKueJ!aAZ~XjNUT3$Pclw)m&law5zdQY0V1I9!
zVpnZp=DfM~joW8F%>5b^w0oL9gY?!uZqbES9u=Nfe_L&D^Jkf`xpUf7xia~So9k4?
zE~&r0lVK)nk!!R!?bDoy-D_j*TEDFou*^FzQ*QbB@Vs-jTlFGYADlbjE_-gZkj(y%
zAGdtoo>Ernqit|F(ZsoRd;8M~rq^CZJ>FN{DR?yJ>8jtao_%O$FR;Do8SZqx@z*W4
zNy*!PeVgDs-=L^){(9>(f3E~{e7L6YXX)O%?0NI7W>nW~jkjDDQS^T0?wFsqC(6xM
z{1@MOz9htcw)k(M&HJ5Z)IODS`Qq1Cu6RUw_Op)nH-q;+y`A{{qRd%?Q>GFZZ9SLo
z_|IDNI@&+}wZydoCT`9RcQ`n2vTVMdZ+qP_Oz)3k+(p}(pWD3t$Cy17IX1z1i^!#V
z)fe}l-jkTr_22FY+x8=>uE)hZ`>QtWHjVgU`04Xfb-#pTCm!rBNiFD_YCreH>!}YT
z+umQSQ)jq7eGY%k)OT0wtRB`cHFw-KJJHLgwBY=%{7KJhlr6qL@ohTQxo+C4zui&y
z?n_$idoH`UeFMj9PH%6kyTR3){MKABkhyc_l+~V3Uteu7kE<6~JAX$^Y^x|=*zL3L
zcxp0~uSPH0qxWTAThpw=DmJGcO%SbG^{YcJ#wFkLso~)lYk%G@F`DW3E4+B#aj`w7
zyI<aQubcX52A{I$!S`<E34SW72kx(5#$)~1^y@D9yHnR@2%P2G`a|w_iums-GG-rQ
zcRl*#fBoNvhc-(xzXoV7QE1Z+4QKmS`N?E~hTptDHEv5}Ip-h!ZMZsP{$zLWzd>(%
z|1A69eC2Somem!N)FrEa_sGxsVEpM|>?x!9l8a^6wJaAda(!B6b@Awv2M_K{x@Iux
z^OlCG6;BlU#Lo!|Ha9w|XEWcLZuiOav63s(k_B^`rQ^7&L)DiGC*S>ePq)%N!Z<gs
z!R*_*7f%mlYDykRG6~2(qO#p!&)sl7){{n8l4n1!41Y0SYPJ5+uTx83sOIlFxVcgJ
zqUpDU?XLT8A4v9j{XE=O<jlsO6~QO#{On${d&^ohZdJH;#e?tZ-#HdLS6x#tzS>%U
z@*2myl<KA8xr*;fVoQ%HZhYK%#>G5gO<Bz?&Nr*Sq;P5`ty!WlHGTVKhVwbkf3DF^
z4|{bzUW)%mX2c_j4VJ=Ab}_na8mIqUeD_?_?alLZEqeFGR!oaJp7OBl`TKv}$3K40
za^U}`@$YlU<Dc(N|G2-zzi4v)tPO=r*Hu@onk@fMYQ~4XEfxRs9NQV6JiH?3|DIWL
zn@EKJyiHRtTL|C3`t)0fuz1R~{Zh-<p7T{_|M<<ZV!Bk;?XLY2p>r~y&Mw?>jrD)W
zN;Ai*&h^(aGa5clt3AP4RxcjLZ?xlVJNuUVPYZ-to>hvx_%%1Fdr|z}zb)Z`UZqbp
zs*0w*UC^@Rs-1iSYe81KwoKn{J#pEu{a;h~V?5`-khyYkrscjzF~3(P%WJEjI<?Qp
z;<V_ZNW1e{g1nV^*)8E_!CpL8nNFvQo-+zRe;mF1%hTn>uk36yuUFm=+q9=WS$<Xb
zt^JRpv-v9i%1`w(7iyTvc%Y@>#5u`1<))8Co#gD?EPiKZ><jz#QFr&5iz{`dCpEBJ
zg#Vm#^J3H^jpS=mUM*9O8O)PSC@Ne0H|OZRM2Y{~CVwygU4QmTxJAkHo}}+5u38@#
z@7K3}kUX(<!Y__FTBrX_UBBk}w}a0f?1~inboVuP=x&=^@*!833sj3eEE0Uema3_e
zb1SXwU;1q>@ykLl8m@XX*?xR;$NTRGv(+oV1uSj1KQ&h`@@eVvV-Kxm5~cR1JmS&*
zEBt@+{pcSj+iv_X_;uYnbVpLYik0Z|h5u%#&rEw`x36H*q?Ky<Y&(PNU(Pr;!{WN_
z{F%(vpUN|K{_@y#M7u@Qu;%}#%chlVwwHu<hZ;QmVZ8f5-M*c3KAnH;dZhewl1!ZZ
zp{m21&R?D)<|*~DtYmT}*U|jvB{uV0racO6zM1^Z_5I7^b^lz8HIJsfx4f8G@{=n|
z|KFpo>vQ;*?|A*a_+sa~B_j3JRgMde#mCDe-kcZvVaI>=S8?}eTHl)b#y>gr`tGk!
zyUL%3uucq^%<!XNS<|OzzZgyL^PcThyY?Sly;#a*vh3I2!SiC~X`ejX)v-%1xoHd6
z!YDrdkH*hV*6q7<=T7aizZ>UzmdDH!7vFSYm;chIsTD>&7ptCJ>n~6G#4G=~+T+Wr
zgR`!<3z>?STgz^o_d7TK-Hzh4s)u~O&+UwZ_SU_eedq7@lKZ(g>o%poda&uRq1ux(
z4x!V23A0={_cGg&{gtKG{7JDpW8c`=WPN`ha$n@n6G6ASf01W5GM>x1ulF)DWO4n>
zpwim+0nb0n{Vq1*QuZzM5ZKpjSIOO*$f?Wy&b;6Gq-<OFYlr!-J9~Gu_<n!rpW44%
zNUCewCinVhXMTrYi+r8_<JpGyE$5b>zwlBjenY@Kv88ph((ZVD&tLNTx$sAg**||;
zyC&N33x9uXS9AQb>GOxB&(Aks@BAD7YhL;uk7KX0US4M0`&_UjQ>FLjuTK&2mUlO&
zbK1OqnteguPH^eW<F&HqpR?>tJo+lp<d@u~Y3z?TR&kam&F-o#SH5WY)VBKacNyk=
zm8bHjuYS@zLsHg4`m&nc<~XH8OI=SWr*5tiHMD#bwm<Gup>-A)=Z+%%Yp1?fA9{7c
zt3WK}!_keR_dPpR=2mRl_;mWt^{;<^DpIZ5W2`^FQZvG=cY7q4BA56!Nj9(0gGFcG
z9DM$y;=sNlxyFf$S=0TD4+yY7-t6Y5EywbLuPdbQY2?9&%}+Ic=BA(D&bR;Raowk@
z^~1YbvTQ?>xdqo>KD~2_Z1n#tZoL5;*8LL=SGgh{I``+++mj|~PTIC9kjc02?_VF|
z^(L|_b6p}j$~80}xtKf+`CP}^=9+W->W$@I=Tbzzt3S#vcsOy-i=7TR2M+VurxjdP
zocI2%SM~hUk6XWLPYrrv&-MK0dX8DMM0L7WSmp#B>X~Z&-22%Z+iCqBzoTDf9&)VB
zovYP(e6Lac%2t<4Me(*P>tFrulk=Wa=qG*E+wRT7Us><Id^DKXvi3pfrf9zB{&s$;
zs@M0WEPP{`lN1|yq^<u@(?pr1XVcYhmkQ1Cm(5&VGv(2&+n?S~uB`v{m$gRq?Ctk%
zWg+h0RtC3i6j^z+;ArAr-S><0W?5a?JA0*H{{E_WxhE|@ZS}g`psv2vc18M<%h%t9
zPIB?Oaz%dLn#4PtTn#Erk?PZqUwp%HmE+o^36mZ?|B$?Gy}@C3wnOtoA36T+oWuG2
z-($7^#&g=Q-FQ>`+v@1nO+}wyzd!3dftSCP<IMSwlV>|xPt}@N^8do?-t{{su8%m&
zAQzbTaGBpe7WI(hB1inrZIn9qf4*Iy|C`mvzFJODa*_$!<74CVB;6p>SNhRog$47j
zJlXbFLS*K<1&I=Cy!{hjB`V8gzB`#O_fM6j@!G%X-%l)hn5lf8qrWo4Ozz<6_i_hs
zBpjS<$jJLLS$0a5wzOi>oeg1s9YwZjU(B4!X`L5V9_x8gc%n;>QO<+8?4N9QKD#8T
zy?m?f6s5_f0%n}LHxhn*X<cVzwKuESsd?{x`9-m3A5O8Hrn781OT@JD{d4Tv?fBl_
z;kVn*bJFwwqFcfL9gpwTJO5+*JdfNHlgn469J@bhf5!3k{uxIu+;F+G?}?Rv&rfrv
z14^$Sci&l}(y;%Cdiq&!NA_<*dUGqP-6qdJ_4#$Lwq{bq`ziVb?@iM8-I#as?NnpU
zly4fcJOOiM{-~-;U-~jx{-X8OV21L|`;Tt2S#7+`WyU`Z27~E4JhT7ZluG@dX5-hm
z?fH^+OWSuE>Fz;1Pd7`{`@NO)PJ5kye##!psYzmszS|Xs*hl{@oL`;)Y2%5&d8)=A
z|Nk{P_3f$fADKU!{EUPmGn%s4$|B10>p1qGIAiRlv()f&!uq;v%=QKXZrkSS&r!28
za7j@1KOS6LIX$H?;bCb>*ZIduhCy~W-fpf9nV*nkcFDK!pYOMb=8ts}%eAEsHW?L)
ze72C$lYctpX^OL|kl{}L96p(fEyDFL15ci`yKDGAplyfxYsbK2*BL`qKStI~`57Ov
zXJ(1O!@k#9i-Q-37hmd~*IbzV_}KokWml&9XSvI;t)HS~ysZ80_9e{yWwYj|oG(;-
z7BfRN_`#%~4Sy}&YlSZuOZpgn{=II}lZC&;*P88IBWI=_D!QTWgTSrysMF6UE&Xz1
zosYfUMb`_)W@YV4bJw1idS@_kmZ5}3&Sb?E_cOg)?|!iO^!>HHtzgqcVYBNdA`g9o
zn!BcaHUDdW?R&&34zs7)hvK;3OqgP@wEFhEy$Rc2J&Eaxtl`$<e^`~4R`>do|E|d5
zluUok1!1RD_c6cfo5-c)dF_Du{H7CeE?YLO&&l!c{ajdn{qshCg@~0`ZXUl;ZP@Wm
z+$`$P+qi2l*KJZ?^S)3i`qpG#hfg7I<R)5V&E2zU{-nBNAD>=fsPUGVlXU)gqS<1@
za!wCf&GWyPr{vH1pwsD+qCETc_ae(_{GHdPZ4>n|(mf-msZ+N8$m9cUf1imdR`Mxb
zZrHE1%VB-6!{-eXi*AK9y+~BafA3VwZ+Q6qNzJp3D{UEX<y`vMI8k%;=DDtV7lV~}
z-S@osw01Wai}j5D35Fl7YK_92wgugZPvN}MCoOc`#fM*1^W>yOlUa|Zp1)_Ep5;7g
ziR|59ssZUY9#386AGT<Yroa27qNFE(k~gsRY`L9${;l$Su7vv&%ho=XaQe22t=eVk
zlF9wgnQf(yGu_xcC&<iwS&so*8|#bSbCoTs=`wCzy&N|T($23In-lIQxFfXw`Olia
z37_U=<g0!2%n!V5q_;$T&haymmlh}Qw(My$jG4J>>B86g^X8Sc{g^jzAK&$TM{-Wc
ztFX>!+h7r~_2-JqdsSs(E}YvtQED?wPu`|^O2_a0x~-#;m{hrWeuCNh@V<0TyDbM3
z*9Ob=gsf1htmf03vp!+kouw_;o~ozLd&A53`gGQ{^&$s$TJ{D1Yv(BlSu*XhcU?B$
z>aF@`*G4n&&%ZU*PFqBHhF{YImk!7C3l|%^_f1=W;@Z5!PRYS3*}tMr+14Mm-PaVf
zcDbSE&#Uha)x3D?pL_YC>vhxVpM)($8E^7xrfRHO@9r{Z@f)+-j|;Ey>e(o|9C&ha
z@zX1p)32RvmVYY~yjPn0<n}2aSvV4SvNcoQXwRFvm0j9Mr*K*LG-uty%Xt!(KTc2d
zaQ%Ntee;p6{I*KslJ~q$Kb&iv&Q_FvUZj4n_|N$(n-8&0di9t!)$>?wl8no8=VQFT
z15a{p3Yy|vS>{<Zi|KV<`dS;a|I^EII3Jx6*;pD`zs#5`-))Qb#$`^A+?N^)eShF?
zGtcY!w8xj+(vH>Y_&4YMSJKhccAaqVM7#a|7>zg2ZQdp?4cYClS6k+=%IVhJ4XUnZ
zpIt7Ck+Nl;>GVjUOlAF-_x^Qa$+M<im|*Lr+~X8~QoyX*{$E7S62(H7|G|<n?#A|E
z!aCu$^PMgSD||e$>e%&_uk3!Z?{;$VP;VAEBy~?~;_<%E>36c~=k*`BCsTHK{hT!0
zt9K1wt~mb4DOc7yG}3<JgnwHEk8vyr`g1KIB`28W%>2ia%O#F{P`V%YbIpDA+g*0r
zp91eWU48Rk%qPHc+Q$=@g4I)Y#u=LyFP}bH@KRe%LBh=ytqsl#{tAo8U2;six6iwr
z#ag+=mg|#3Lfw}e^F_3RqUv2Ex67!!WSpJ8Xyyi^i&l5v#CP#5zs$bDxcmFY4O`Z>
zo|~>Q*?gCSzde`Onv<5gKku#bZ@qj^_EMl>xX|?5iYk#oKjWXoShS_2-EZfRY+1zq
z_-<t6!5w^N-}b+_cX4mt5`T|Li}RR*RvfMRJ!^#w|MsGs%|>zZU#(r4=RR_(eZ0e2
z^hfIBPltE*y)_nlyM2Lug2JZ5Yup8wtb6k7_-4NOG4of~^Rub4Cr0y4`B!o7QS!^I
zSm6$7Qx#cXhO(FaPBs0Zk{t{WcOSFNco=_}ll$TOUhSDhTV$Ty_?E|VUHX*xx6e~2
z1<mKong8Ie!Fr=RZ&Kw2-)vvo?jrv_biIYM+M=g&hx$4BN*&I{d~lhNdF}4i*Ei<d
zsyv>%ZK~`#uX%?rUpqd_>Wvpa+rljzEJEr}<jmC{^2KUp%~*fHoM-WhxlPQR!kd;q
zu&Rlu;^X%dEPJh8W2N3JuN*qxlKcBK{cy_*t)^S{1=++|HhuSrjG1Ef^Ut9V-if!|
z=5eqYmvzsYdRA4HtJzm!YU*msMh^{1L&hmT#n-PFaGfIgGv=6Jl6!_bmx)n%&Tr!x
zIz`#)m!5kv^`##__3Rwa%f(-M*jrb`Z%hgAJGN_o(oVV34!3&iXLB}2YcP85`V`_l
z?a8cf@7%xE&JR^y@TyYc)XSHa=hpt#p0Rc5i#@)!=gw5moA&a?EM3V!?`<pFoL=g2
zaB3>vp3eDb<56$L1ucH+-ieXA``soTXFB<T|4wvZQOIk{kdw^U{_%8iRhD0FGVa{H
zev7fqx~0Oda}zC93bpHB96H~?;D4kh=i98fuQlDh>6=3OO)Qflqxm-eExPeN#_#zn
zEq(sDx!Y|wy^KETv|Ig}E$jQ^f#PKguDx-$oP5guPfqggnU-?T`Q?mOtGOF(TJf<_
zyWV)$f&Xi!T=*C%lkxML`CQ+hDlwb+n-8egA1KzX5PP3o^1EEY{oLktd;iufc<#|x
z5zG1RllO&;`pMf9{Z=^Ln*U{?ugF2!w0{=_?LIojCrTdlcm8=e@X@I{;T2ZfJAO^i
zyQ<^gcFESFc2C`#+mr30fBXqJ&iq46d6M$q(t^r$626T`R4p&r`#(1P)?xBBG0p$a
z;}sk2AN4y;mgD04=zb^XL!#n|iB8vp7tgn=E$5r}LNlP`we`DV&-FUXW1ru9^(lj^
zZo@3SFB2n{74RnfxgM_c;KldaO_dW5Zh3a2*}OWuLTJnS^EGl4l1^Rs=gL0))1}h-
z&gsduOO6@UP325_Xq0ap&lF!5<ab&BXu98TrNj5QS3JMW^-e~YVS686LG=Hidza?@
zO+Oo?bMWucz3XK3KGZDinS4-vvF*gBKM~TV@+P}aH^<-nc<cDx<6>en&#w7vdhVg;
z8ne&S-!R9Y-+J-$^r>qm-tIc<d|paUU24m$cecMKm33WVy*~A0?w&^H%N(|kcHFA^
z^EXpO!;|;fM=|!5>rd{ij$2;&nRAx2L~YN6h>MptYqxTr`6X%_@Tqm)>VuuOwfl-6
zG^M-SZ+C6to$ytZyYyZj`}^{W_rIBM7jJoXq0ySP?#$&||25;!{rr9+a`}I$7KMk8
zYl?Y4CzkYHtyXP2Q0mgDI{Wo{q4tcx6CnZb1zKK<Y8yZ2e_4Ot<g(vSoru%!$vt8#
z>Qp^H+VIx12(|6h&GCM5)^F9;|Gi6Ut8C_NY;NheB@^*l(>_$=hCFwsmsWE8Jk{Gw
zWdeJxf2-K;n~=!<t)zMRj?G1%*E8%h+8?WZ`B;JKHN7{Rg?71J3$EMOXIk!kKX8BX
z@kc-O%x)PN#d#XaNw(D8o?~)hlKT_)PVe&@4xEv#pKhxD<Bs(4nP&`n>L0$Gyl0o$
zC3X9CvYmyyI{w}W7kc?P?>wiK?W(5|?<$P9?7s45frw`PmVbwr2y)KdXaA|tWP7mH
z9_1xlp0r$1?mlr-U1x6DhaII4R$Q9YyzgY=k8E|Ff<qgAvlpE|Gs`xMq4x31gApIq
z|8gsK?r~0Z`)et4$w^dpp3a6!!z+am*WXXFII<_*{*W(Ysn3&DvX+k%S1=yD^C7b<
zv#WOF{Epm(fwSE8n>Skhd}btceuwMk;1%<4E@Imux;ysAq}K8~@w#q5e7pCk@>ZXH
ze=o@2itTAW)8U41%O12`UwYBVS?c(8r=xBw>>r*yt|=tSqgKAH%dXsU<KDkF=EqC6
z{rNTZ@Zx8iV{IP%T(aKw?XKLvzEd>jSbw{9cFXeX`?=&;*FE=K#4zpaujx8+>-YV5
zJ%!Uv`sEwTvx2<e%Yx+UYg(!#cRiIl_8{QGYhiD@v$pSxPa02MJmdeQd%^DS?7}u%
zv<bSZL@k=ODdp)Cu4#8PgVkO$Z@=)eD&gb&yk`~-wsFr5&Ul_I+v)VFq~mP5#4$}L
zwxYW)*F8RU(&J{lP3Yr0Ya)8*1mr6ov@_W;f1=^T9cM25{He2?Cnnfx>RMedLDrdx
z`+`5cdA=sYM{Am4w*KeI_uQ@KvCW?q;-3GHrFLrjQrX*&&v2f-xX-ch&UKZgg0DPJ
zKi&DV->|ZT@4Key0f+jhYTM`3+$nms&*0L+h`Be*4;WX=T#nhqd^7Cv{8_=X<0n>_
zYTm#0<C?-e3*(?y^UjBdEx7Pt!R#G<sSm8CPZqwp*TzyXyZ+;&UXJzure3L$+wn6f
zooT94wvwMsqrJ?GKXcw2iQjz{S+)BUBMbkp*6qPRmMvJ-y)x52ecq`LZZnT3+3hTu
zC)a6Xt0VOL{&hBSA8VhB%F{j8NXB1civP#=-Qsn+z#Zj|BbpNr8pZ}J5U4a^+Q-IU
zZL7Ak{kYv{v$phNjjZ=2zdgV6?sKSBR#V%S@GMh4w<^kR-ToZ`_F;FXK4#hU*7A@+
zy&wOn2S1~_*4$BCay<HR#QF`zON-lVSE$@%+Got2YhpK7;qbARgEjiwf~QV;%;Tq<
z^z-MnQ#R+q7KIkxE8}kZn8U5Vn$!64%!&WMm@7BOovK^1_w|(28BQu;_VVw4?Au`9
zrO%bsRrpRYX2yN}`Oa00{0*~&k5#2edvD)gd!X3lz>E0IfX|j26IVOO3dDIke%^nM
z@3!@^MuGjC3+Jg%`O6y4CBR{paP>&yv=^z*e6q~m)NOlG?@{~xWQTp+L8sH|eTFH^
zKR&hox~cH`X@gDsHp@q`xNK;MkL7mj*(1Mcr`?_QrJJX}nQgy)&SASP8#c%<Jkt99
z=Ix`BVG*03-v4VfX@0Vpi_lEHqIFhtO5dFcVPEZSrImh>e|KS@>TS=<=Xk$XUiD9i
zGBU57oAztQ|A$7lXU|xA>CV3OH>IPv_&i_c?Q8Sbtaz#}uRPPwTr1u#)wDa^^!dyC
zd!D`2ySZ+T`bO2CS&N@VZsp0@>~-_u#}$1OC1keGnpawW_;**1H~$otyO)_{w;w$}
z@9lO+h1d7^#ZRpjwfyVvY*WqYb?Mh>-ftI7lHFRlS1D_UW?A*b@yR}4{=K9s(<<zn
zaZ%Bgc}{1-{Y34HF7I8tOyXFN$Gi8%2Civ0XUu%D$8W!7p~-H?yeEEZ?r(pl+kGH@
z`MbQoM}NNC8{uyo8}|CFe*O{d?5}p^X_F&fx&(iHvp9Tn<oSMii}&-=-z|L{7WKbK
z=8b9Wgt(V8X3v?IQg`##cawjN|E4$x?Xu2~JGXXm3j6G{kLLSLxgP$mcWsmH?f2|`
z&$F$Ll|NOEdh_~(*i6N9?4NdO+<sH=j`8H_H_589Q?w%3!}5&ku9>@9%|5T6zk6Cp
z^SQodx8vCl-3q^V<3dzS%a53!H-yT0J)cjCiToT^Whd@>uYP%zp?rbj`w5q4@4x->
zeBkH64_{wj;f&X4H!WInTX%lKL+7baTBrH1@wzn2+GcL_|26Svv-dqnH<`y`m7Ddp
z|HYe<+n);OYrp4vr&w8Kts=kucS-sD`W<VRPSbYY^yK{g$v?E*&wXBNT7T)^y~iJ?
z^*_ppe5+vpbn<S`#{2X1UhRqf)UZSIam<6Y&s_d1-{1D8;rVjkrODDd<{#YD8>;^-
zG_^eM{NaoD@yB&%?i|cbJr^c(ak{9w=O)dJ!{!$@*II>L*}mwC#eU!C-t&(Xrc}qS
zJExx=XZPhq|8}eMMH~L?XfI#>`~KVLBIzrFA?|<AalST>s(JqNJoDbVb0t5NU)E;X
zt@?J(@?~;{{2$qy{sy~w>q?vsFVxP{I8j(|=3jZd=Uvg3FQvcUN8X?P<E6;+&Nch@
z{Jq3%X}&|<)AGoMdA<qj*lf?_-qwG-X4xBKTlKSlcXSj<1V=@!m=;`;f5mUz59<%_
zcHEzQfA;aM^VXjJQB{6&?nhpoz3)yh%)j=YS2zAc>-<+YSD*iT?AA>?>#sSXSAH@+
zeXa0G|8;=xKUdvsv2?%MH9BXWr`0{@K2=(K#rFP3&9ya*1;10g=6{@CBBdGoE1cu_
z4<oi~Ik&><zBylrv$S_U#q?&cEq~Wvu4JE|ZeHI+b8JJmcTD#YyqQ{<ChwZgw!d-i
zzUHT=%M0$@)LgUl_dKz0ChkxA%j_NsIKA?SY%TtEJ2${;;@fjK^Z)uBU2He`><L@u
zzH7H9Kfhy|_W$I}1pnRZ*`F`jeEM_R{L8|1@_pMP7YFa35<8=+uIh>CmRHlWuTB3y
zd8vu%yUZy|wu$V2%h>(*hJDqyi$CoG9-R7aTY9q6aUak9!zI`LpO`Z1T>E;<g)i^v
z^)Hv-9qznwo^6W_<MW+Tn}aXx2~sjn-*9Q-wVM~CR?gsGw(+@@UCZ-nQ#=>nS?RZY
z$sB=@FE48no=sUK7dVk`%f3t8sn+)L8SyHMY8iJFg`Mu*k{)*2AgGwlGerHB`zmFf
z#mXNuZmE2JKCh}kBxRyomdbsrT;t~fPjXuSUU0ZIdEcrCjoWd|KW6)AJd%1p=hBHy
zW^Yf<zcT6Nk5}oT)2^*`-{<Bvw_Qo%^XbbLbFN&LT;+d4_@B;+wu|5Fd?gb%)*0P!
zmB~2Fk*U72Yj=f@)Wp)`dP^8L3UAwEcu(upn~EC&C*9^wo-fxJ_UiXdVZI-etghdf
z7UcKsQ{9wLPv3rxtmb2?d$+B4gF(98+$)E-&e^r}dv&1W?U{}oyB#X7oCz<j|5;UG
z_%i5K+poJ<KD_MJvXk*o4qAWD^7sWCftR*lj_PbZdT_$6$vb{neJ@Lw%8EO({^O^!
z$1_h(yD6<U`RmgIvW*KP9v}LxxOU0W74D{crhW^bmu;s0vA;((>VtpFR)hD&_jEO%
zJ@s5AveR#a#)RM}8x8Kf+-9)pg+cH3;6icd_s>^ucKUA-V5PcOBY(Y~iAZ|YZ0(~*
z+ooCsU7EtY@O82a-|5_xut|O9d&&$q2B^ySTIVm{Y!EFpcM;dOfLgoA?hpU+WS=){
z{dD`wY*D?&NcZ;l+)ESg#n*i^Za1~NIdzN6xw$tJTOT=WUo7?Pj{2WThL>_LE_{A7
zCb_<&N~JKwRDR-*l(VV&JDXNEEO{J#n5|&yW~sQlB?89V^IxR9t)CO=-u_eeTrE$@
z$!Y)6!xTGM80z|CyYBsujuV>o`K;sT`hQXmMa!<Q+80>!anp0j7R%$oD)$WevX1oF
zyuVTNHSU;I&O3WckG=DKWjF14;NpMt^SVl}zP=gkzyC=*t>;_syw$d;=ZK<BTZ*FL
z)_Hfd3hz}2y*%{gOtZ?Kmp*!bBH#Y<b~^tf>U=?+MPY0D_m;e+D$f4%7;h{+VYoz(
zO)$80{d{8?s~1l@zJC+3OL!j^TtDye0<LP_5Uq1_f6JbjXYlK;fW?vz7dCC0d2ask
z;&<yMei!eu*>rA^RAY?gW1l_SRE{ka$SaEMD&UzfQKPcRt0B^d%etXDY`&E6q-p`i
zeVmVHa&P^&aJT19-^hz9i5#;Hq!*-p2{XTRbH6KJ<nl=?_H%!-<+%`I>#y<ev_<~9
z7hiV1(?3|(H<M@n!w==xyMD*sYLC)>clN=TIH!#K8~SG))^P7-J+=4Vht4^hub<<y
z>-+e2{SnTHfU?uIHva3knHw^%o|n}dVD2O)6n^o`%iEqt>ld1At}l7jD{OU6_K#ww
z!=(n>nL!fAKis%<y(*=8lRnFx*B?*qYwbSmcb{Xzf(?ED6<(eXeEu|LzeQ|`d$iiy
z%pD>dX4>=~@7y&}=Kq=P3H!9(@84sw=cvE_-$_B{VLwmJ5qc~9Qe#<W{)2Y$`dCY|
z|5L91E1GrjUii5q&nnulY>pR<c|Gaw-{T!$*f&nPb|-e_zb~1Sk}qw&e(@knWy7h1
z^|r?sWq<!^Gt)5Q&2hP(YfAfU|0hqAZ$IhIaCX_%%kw`?^iOtPWGGjAf5YRPT$f4t
z-0$zt=QIC5i=QW|^X7fI7st>3NLVHq*OKz}lEY2;_%mCMe(0F--sRv)^~Vo5{)P5W
zeZ0SGuSLx4^E<SS;?@37nq#6Ur!IRbj8*k}>z-~WCaH6icl5vfeEH<*XVs@3D*Tc8
zp}Fj#<>g4}zFPKh8zoQ6`y$I%Zl5#P{$5OSe#blx`)#>DzBIm$fB51<ZAC+yg@1v|
z+4LjR)|ZG)lqo$_GUw4>>2F2OCTZtZiC_Gs^W?;0?J4(#XH2Q#{Q57#|Gv1J)i%GC
zvi_wS#+Aw&ZP?u#%;KNRu{WRg6KCi5KQX;_<?`1rj|eq4OtIFPYr?ROJ-D39Z~daC
zfXr}(B?@b`<>TewWwid-ut4eY`<6rJL##`7G^RFQ*!Vy4Lr>JyoEQC)0UW-)6V{z6
zX3>{xG*GVAz7c*zX|cko7wSp#f2w@C82WBTMcTs@7Sq7nM<({^T+iB{ct=-v@x4iW
ziBBcIbC_}bje4@`jr!L96MHVaVPR%o-eRxs8`zv9Rj`S@g|8v{W0;eRS5jG)S;?Hn
z5;?(1Kl;@7WN-K#uesB6;~M3kJAA&R7ti>tUacm#_q*pCJ6rF09|b==J1?^|YlA_5
zMe6fO4vRnW9G|;NWnCSU(=p3)k6R{-&$=p5xo(QZTgx5O<=V7DckKIqDtN)&mh!)f
zzKZwL*|*PpJIUXuMY5*GGB!OZ<l5@$e_1k04`#0Tf8s>*Glh0Rw?n;O!{4b~%4fM-
z>32`&e^Y!lhwmb5m36n~yz}7xk<dGD!{LI!a`#o2B`$w@88m+{=e)Qn6B(LBKV8$F
z+bJver%KPoqh4lhC*M=SB@68*ZdC{ieIpWHlOwiv{)FNewRglEtN4CZ7p-m&5mehT
zwdTU(f0H9j^^SBNQ9gR@e*DS!OP1BH%a1Hs(W+h)zq)F7)tvOZsrAp)>nl5#Jg%M7
z-(GRQRgH`Pf03sJ^N*MZRyX!l`CL&oes}W^=k1oRWxV^I<Wv^d<VxL6Dt~tC_teGb
zHkX>s|IV&>wCUrUk~vSWrt@i^tLk_@!C=erjs87bmdAaLS)v%|@{C_%k^6!RPbwI0
zZ}dL2J+<o3iUiZ(g%>2$J#X>+(===iW6Q63a!>M=uJ^}Tj*^`-Ga@-Rub+S1X3nD*
zt1l{EFHiaPbGl6okE+#)2TET$=N<er<I%KKx39vSdsH6BXH?&qqr2R)V!Fz#t(PV*
zwEkFi`~1@td2DaboRnB*So?39v!B$N7ZVhn<WkwQXZs)HDwVg8v~-<(%2H>!>TiKv
z#;%*{)t75bpYY7`)pE^(>a;m>A3i<sOAPAl^y1`x_eSIA&FF)f{5K2Z9_P%CkuUbx
zJE>H~{&LwQ;qQKSB|A93*nRtGQ~lg7tfVl1&E;O@?K#uVvE{`dYnTzw`}g}pBh}yQ
zx4oXtH1BGB*=G~$>aN-LIU5$Yw=D9XaA({0v!8c}7TBCi=Gy6#{mV|IMSqEcby<3P
z%hb!0+L}zNN~3hQt9-Ry)-LHE`JwOX)k&FmlPfpvzJJ1c%6rG<2c~BnJ9aEzsOa9l
zNx^Y54kTpXx&B;am-Acx+~0faBv{KPC-+yly=~LK!BHNt@AscGvzFJU`XAJ0pSWZ8
zwZ??RPZx52`b|@A+MHeS?dunZ|5f{}xxbj$hsBo7s&xqvynB{u-n#?3%Xv39KfE{R
z@XW?H+A$_}-9DMKSEW1s>f%s+a`MFM?<~QKr1*8MkM(ZQP7giJ^JUv4gI610+&{Q}
zzGbFg>f^6P4q|0XCh$yZR8}%J-F$piO^S5MZsV12Z<ZHK>MEI#tJ)>MQ_$wwE7rY5
zr<P|`7T26>=A33@S=<)+sxx`9`kXgk_Ep&!b=Xe&FMr6{ck*Ymf4lcp%e6HeHeGyt
z)hR80)kwQ-KecyPsfK<~-)Q+wlyz#+0WI;*JI}_N{?2=%eA!I8;={?M&z&E;U7KvQ
z=dG%C{fpJg2e&WRwYO!vG_x?8$+lNkG5qk()>#sN&&xZ<UfSt)Cb&W?QfFbl9mkzG
z_0{j+onU{Vu2y;d^w(tq&F8mHVO%4Zy+TeXa=xbq*XKW{3S;<JcbmNojLzI^AG3ME
zVX22*7u;^?W%a$PWLo`x^71yettM6T*9V-}mff3SR`evz@7SxAMgKl*=)OF4+v+!x
zlWqhp`SDt%RovrbV7{Z4;-TtrFTRhOJM!0mC@K3MzRX-&v-I&9TkfPA@vdRts(gAr
z$NT*`bNv0cDIfW^X??P@Tki1xu3*yAyN@kjSUl=>FH-G!@Mh`;-yhdbTlVYcHD24_
zz3l(bZ^ahE%Wd7etX@}MY?<%+#Y*JpizR8cb06ERFq3^)npf=q?aOg9(I~DjJnyQ%
zeN)-<X<hO0CtoIgU$SxHH;do9?Y&RtT#|O@6Q96zc(dQvj|_)nz5GtHcb*c9+^!+J
z`2XCa1xp>SeiZGgSjv3q+sWe2{5O`DpMDh$m>#dz`*r7{{cNdEexGZdyKurZOaJmf
zySVR9Xa9|@v&^3Na;?qT^Dm|Lcvd~Kt+p%LuzYdT-Sv8wO3Ra5uY0UkKHd@k)8N+X
zy4=v^>Ibg{&ylp7^x@IHbIZ>>U^=<-$DdEv&#|wrYPoo^CT04+{?~K+ep@?UG`##-
z#rf^&^sLK^C+qIm^tkF<mOtO8dG^muT&K%uZdd-^C%9|>p=<Jw7Q~+28z1z+;eUDh
z@49E<VRt4iJ{k5?Xw$yg(Yxe-C7#=LQ>#DtfA0FWOC1(AA^Fo-iql^Noj*0bBKY;r
zJmrV1adGA6qTlTKyHoGtfrQC*GelZu+W%C(yX^6|t#@lDz7PAiYyoF_xU0HcgvGw3
z4@H&BHL6>c<NsH=>Ez2^y|nAjspH0>p5G?ixOm|DeC1P){?=101Y4F^75}lH7UiFF
zKD~II=Dl~@|Cs-ND&~A{>d)z%mi&u9zx{e->P6v|!Jj@Cw;h{4`SsUO`I|4VURIx1
zC}SGztoYn=gC)bi^Y8auVxOXab5G6DhWhR6Rimwbt-c(>)&7Pd{Bp*hz$m-OKmW4g
z_U*H67r*v8rZv&7+xfQ5zV|8h`<p~{-IZ;P{7>jU)z&`fp|jT7cCH*xxL?=*la@A0
zmzNy5JWDyXWWkG<4-@ZmP5SrL;L@gwz~V`r1!;#0A7oFMs`5F)V4Cn_k2}`8zE2f+
zrmgM$_tG@mc{7C~Vs=;MwJ6X1WxX!{s(+X4|7I1{_fac$?YL&I+mO0M({k_8e~&6Q
zXnv5~_}tT_Z(hW+V>OZUs+*oBtC#LNV3}uG^;G};e>InKsT<S2MtwakdLX&_@};bw
zH`dm?do=e)!^~jCyECmhA9q_knz`qN-N|du)vZ+?O<`GKzpwJ^8>Z}+Yn~bkmCq^J
zQD2py8d&l82j?rlI%fZ0DLhHhN|Bro`R8}PEiTC8FkX76swj8cboO7y%8FKh?Vnc|
z%V|#U5f1irxXGBhyx!7oVgC`Gxewz%wfLMY`)Bk|)A8K*Q;~1i>u~pdf2!Sis@!gd
zws!sY*#B$pM;@H$Is5p8+nxeZ=RBTt|4ND69De-!#=zzKw+Z*8to>>A&TjUKKQpcv
zs|TLti+K2@llj`3nVvq&<3+{)T>iEHSomM-{uA%+d_3^C^wp`k{;vw(Z&+OU`&{Fc
zewmM!HvRQ0ET=yByIkeNSr^qF$(y=g*T38RGU3iz8M}S&JfHn_F1d1DZ*y??oUY$9
zLpS_h@=-YR*9p$u*I#mdalik~^UHrX%_q`N4hpww?)xA5_VKJO#mbtGOETAa&J3R9
z{7m?86Z^a5)mQn%uYLV^V|SVUg<zKpv#(4|bhy#L-`_v0NAT&dps+`(yC+S0UTC?z
za0mOJB^JszmL=SITruOdYQ}lPO%Ka9`QFiY3b8I_SlB3jaOQP!e_J=cV|@H<6C-av
z>~_0<b?@)<$15%t9epi(rDgi-MD=v0KfgKmpU*ALsa4{;plR4RJG|28^b6w$z4kfT
z-Elvx4|6}eCHn1M@00!O9hiP7JoR}Px#mo5=hdB8-0wHeTk*{Q#P5GkkM5Ve*Ah7C
z*6;Gtx7Jn1*^e0WPB%~LT<AD+UB!pjQ`KIzoV>gJmu_j}p0NGNZxSEXJzq9)S3?%-
MR>NZ6c^el30N55?$N&HU

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ae b/third_party/icu/data/icu_conversion_data.c.gz.ae
new file mode 100644
index 0000000000000000000000000000000000000000..0e54fdb9eaffd814477460f71bc194104c1b247d
GIT binary patch
literal 177696
zcmZqMxAf-;@pzr1%NOeW`*&#Vy@&S`SJu7Ay!&oipV%z#@@J*K$EQuHv;O$Iga7n<
zvyGZp^e5#w{j<Kbzg5_3|EtRn;tqS&?fmoa+u<8W-yf(bTzT=z*^_5?{Z^dzrhI9B
zig;Ns*Uy_aN868xw9HK0;lS4BaKP)Jg0qWVfDyANW7fZA+xF+*j=#GwCBOL4T>q;E
zQ@#pU7foeODLt^izJC6SzSF-??fqYuu;D?g(^l!Q-35jbIls5vHpzegZ0W1d+y4KW
zyY~8h*;{`mTk`(XomIY|lD&-U`z!T~H_eK*M|tNr=Gv@!<hc6C>s5Euw!UJSzv_xc
zD0{_0uJEUuR&*U%Yj=CMajfi{Swd~Dwk>D5=S|!=%Se#bQa|fvb4_>f4VQn<J09|u
z2A#Mz|LvYzOM~BD5-s~_@FU^Hqkwb!!$K;U&lg>u`sy09#j@aIFYleZd^dgh=aZND
zAJ1BLx9IZxyRzw)$0jSRIkmI(%<(u8(a;rp++TcKdxJZkZMXW9Zy%j_r4u47LQllk
zT4pAe)bF!&an!0>QTg#>d&A0gD;K$!hTq*;(fM+&P5Qlw^;>)GQwuBR@8!CCV0Xw0
z?PdPPdyiYm3V0v=X4IUK{gv$)$Br(k+uQH0X*(TQWH~SI(8FJUw0<t?(p~@Q*_urm
z)^_o_A6ERDY8&C_wtDi#S4(zVy<FEfYyPW}4VJPwV$#zVmj|(if8E4p&&{>@`OECr
zJ+T+Be-ZV*&0YIuIj3Lsjaq${Wq02%ySeq~&g)am-!xr5y?b(EYufuF$79@$F9X&e
z%({BE@c8k{h2?juj;{^BQ}XqibzVhU<4)FRVl$!>@)~EBlyaRDnWJeS{Ld+2*#RGj
zfRK!On3CV32&X#5Ju9Y`GzDl*)VZ$X5w9?x^Xa0AZ#QY(+Mf0MV%(G2eKEOH6L(p4
z>pjkzdToPMcwBAyiggStZheb+x#Gl|S7x`h_I58yT=n_YRa4WeS1m%V=ejQmEnm2H
z)2%Rmw$<kh!>$K~o_`m4*LQc;_bXwmT7F+mZmd6f$4%sU?EAmz@9e$39{l~jdFRBZ
zIme33k}Z=r&e<4Y|ETKh>{}ts`LTOS4GoX}yDmLl?Z}t?_kINLDgAiusC0OK%-)SN
zw!Z3JZGNviJ$z5;*JD?u!}IU#DZD(}T=wSuZ0j#Y?`I~xc)IA1y-Ixj)NB8>pZ<4d
z*e37(x7g#K{I90}MhxFK+&a$nZ<D3{KDGSxsrCQ!a#=OS%P;=9xR)pY`GMtsmhYJm
zA79w^;4$O-yj(+#8SZTS>dD@E=e&MDT7P$S_V<Gee_PMmzg;-@{W-_O@_V~~J#{I*
zsJnmL$_sBhC(iBiab4iE{mgl-H|6{D*N3%ogt~0~``7Bmf!vnU{)#5c51yW#`!eCQ
z-0Z|#@d2?xFXm+)=MO%xCZ^Qlt(g2Qrf!3`+tuqbPp%ZYm}Duv_s*=X-M6Q9mA#&9
z^2<)?-_f(jS7aBx^E(<_|7DHb?Ja+&_1{~&Sxk1;v2!!7EKR%|kawuU`@{|By3F8R
zDSLKn@v^VjxZFALiFm{Yk>I+OW_w?##clZMD3vF&|NH!vCo9(!Zan-}@yp6rcCOpQ
z^AgW}@+^@4@=RQ|@kw===(6zYl+yWso*Cb|Q*-am?Uk8PvFD8CH?8%Z+*kH;%}m?&
zu8pRLcy=vfwOrX48T9Xrt@quzs(IJ<->FN@K4DuHwoW}SYxf;zf2Zy#e$R9kOz-`}
zXgA4MFENyDZ!g<*-)*nX9{z2qv)v~oPEUWk<bzKqc+Q*Fc3S5at@@&{d2L1bsm`~`
z3BDEU7T#!^8-4y&L}t3&_Y!g6eVnX+*00~buJwgQ#p~}&4t6C=>{+mPQ}S#TZ{eh=
z(X70^tF}w@c{^K#<cqG#onlb_t>nn7Q=*qjf?d*Aym*)PP-Eem<j{ysF3fxUw=cf@
zwtU*+J5oWjW^pYKsCzcodBIwVTVe7gtJlSTYU*B)BEHBhs$qf1CA-j_p(gW8OmlX5
z=C8hY|Kz#_2iD~#$=QE;lAXu((}d$jz(beXUCSFir<;n@$GT3`dXgorYc9RPZ28ud
z%Z_*VaUZYEWYE$+aG7D|ZL8&{MK6mV`f@mMS@x4Vul5M;(Ylc%FZN7BZ{mJW&wDB*
zCX(B)F`eJNAwuw`{GPqLx)gKdjmnB<HOt!^eb_XQk>B&x;$~j9<wtgwzYOaPpWAog
ziQS<{&UM8?+~1=simE4f<d|nWmoZ)cdatPZ!-pHX^FoyzzAf4!%Kl$hHT16ER<1n-
z?d?jTn`Uc#TXc^tB0k}&_-(U@dv;r1?8)D`(SrB-G;^05$F;5B_y!8K++I`o_{04o
zhL0}-c9(j+`}f+jxi<9ij|EfjMZ8kky8Z*}@6CIzz7_djcB}3;)0I_sf|M&=?+Cip
z9JIZ_{$o|ir~E$6YfTTt#bvkX@7S?(dc(8}YAM0{&Ysh**rb=5_I>}l*CyZh{-_A)
z_<8@Cz3sLGe<nyA5BQ&2_x`c(hf4=`Pi8P!dicdTITL%SEjw@BR$d-6b;JK%%Ttuk
z+z{>D_hQ%ocTs)&{B^fC|7P5^Oe*qP=<4t%G4tm=xKqiuQF7{3>yQx9IpKQVn|HsR
zZ6sfF;W)qU!<9~^KX+?YP7gF$`&5m2s?gSqhwt|9_b#k7pZ2`=YTWbJI~JH}Yz=rM
z8=GHdGUc~#RqIpl>W<J49`n5)U49y;`$DE>Z~Q0bk4F}*v-rMs@Ak*OY|Bm@oDzRd
zNHRi+D_XYR_v!Z8R?nTAcSzq`^=S9IbN<f`3hiSRdKlF4JCpmrsO6=aJ6=m$)a7h<
zPXDP_d+nWk<oi{BuB4=xNdB?AyJM9z#~Qbt9c_C~6gw*Y@%r~iZMOWqKE2{ym4aJm
z`mHOty>Gf#Nxp<)?VnX&9-R|9HNUGstoER6)G58Px-&P<Phb7}`~Q7cmd8ich2@v?
zA7A>~&u-THf486h`1bZkS@n&#Q{H`Z-LT%G<E6r}rREhI|JZs5{JmJlZvHU6<L*9{
zvgW5>USC^$FT&+%8Q+(EkH75qOcFX=cqz0vdisMo>BVK6E%#<V63p||I2_s9AN{1e
zqUY1%*jufPk%8Z@y^g;$hvBMs$&r@<rA|fNr(P8Q{Z=5fq+;vt6^$}S|F~V|UG5Rv
zUzKb**>vL8cPXoXJ6}!YIrvT6H}+Hds^;G&`vW@;<aF)Pc_db{<=k779ILNV|Ekwd
zeCc_Ad2!P3wxH8-2?xI<E61cn%~tv`U3>9|kg#d7JZsOdOA5KFp%>`7@mbx0l}$=>
z`7Rymd|VW@{g1<bRm&X9|C_G&1Rq>F%l>+dko2$Dn{E~q=6t_kYZh&hr}vaK-fQZ9
z5$!j3_647OcIW(Y&h?+%bMMv9Pv2HPMYTC$dUo69Kitl_;{AR;2lezmdJEk1cD<}T
zb=LO(Zz`jT-j=S+KgqFg%FEl`+BF>0B6_w~F6ye;aK*`-`$~<=jOE)W@H<IoWjI{5
zK60c|PAm0v<Rr!EE&oiXiXWe%_+tB;fZuK}LM98QbvEx-J$^^&eek0*?3RU{-~0`m
zyn7z3)qQYnvG@11r3?9TiVnV<v@mD7vrBy^>zBnXdUO9jOqKo6`}G+U|ChcNlMIH>
zH*W1VjtNL(FE}c#m*?Wl5V~0Lyy&~zmzx{T+`F7{VMY6yu=U?9=M<O!6x_b@f@tz|
zv+cc=it8<_y*jJz-97uq%y73a%bL?`m-7F($-Qstj%SM|Z!X!?^)UK1&+3C4{A#kE
z%zS$M%KfTH!FLrmL+k$B=34x2>AA~=_I<HZ){)82SG`u3NiP3+vGw-a(%n}YC1Us0
zO2><+zS(>=Kh5`TXPim6-s&TPpD$Gk+*@g@w|$bf#OwRPleTL~x{IIK7vQ34^J+_>
zhC{Kw-c|OcN>;X|zw4$wU6QOKvitAv^(RADb!t`B|B?9?TT^nYtt3aSu#*4i+J#@7
z&Ya2By6~#Q*4$F<egCo7zr~gpWtZOJnzdu9`p-D0OVNK;Zr9MdvdAlI2gBl}cehU5
zEq(IS!<|np>bom1_1V{!&&^{m;kvNZRC?7SN%hG;_w9Xh^1*GhO}q1NU*fLab(+Vw
zzVz_ZcE#_{m#(e$+N<)idRNNX|3WYHmvoB$TN!-fT~g`f7r81472oUEE?W6=rq#;a
zR~OjNbrgME?6J=v_G(z}ts~zBia$+@4^h1MBw5H=X722oqtZrOyF)IWt|%^D);YQ0
z?ePgQ3%6QN-Fui@JnY(9uRul1W7QAKF1&OP_DTP~tT^H_f5oJEIq#2m+g59-HfNrn
zFO-;984z>HN+<Q)PK`ek`(MA9R&Dk>bnE`i^)(yS1T%hWi<Cb;nLkZ>^7JVg-ajp-
zeKORS+_-l_%$2L_cDTL&)?44P_e<}+C1u|hAL_n6Wtzl(SC#+k?gcIB(w+Kg@q~JJ
zOZPwTb=4kD>v}S&M%C9d{L~G-g{yV1mM9f{?yfG>SFI|U{7fh}rpq)^X{Glfca?>u
zx@>Y1UFCU?6n34{pFV%5dPi2B-jqFU_FiwVczpQMtXlEobo>6AAC0@-)ywe4+}|Fv
zbAQb4O)*zDsT_ZEd%Daz5%p(U4^?$L#H&gkO>|QeHxxIY7ZvxzDmqBFTG1}0qkm?W
z;=#+sW<H$j{vBJoIw?pk;Nkpt$JfVd9ao*YSn3mxzFf`1VsYlZZ|#boPilC(bjxP>
z>s!QjmQ8!hrY(JI=@q@}Wgk9u-HAGp|HUx&DwoMele7&cNfT=(?@sBtnj0Z+fAYN$
ztC3%S2=go1dn=d?7#`RC`gr!HbbuJcqQ+vi2b1Q8Uw^!ltwFo0VLD&b&eZ`*Tg44i
z-S+T!?LNf6l2h#!gYU%a57&AX<i7E5t66_yLAH@<up!g*4<}xSX+B)3w&xnxGw$4m
zwH3zsTXy*QH)%#QzWCYV$87hOD^+=Z$j&&udENe5UlwHAU)8+fx~8xD^M=aU&sUFZ
zH_cSpwPUBh#?$$l4o7#Mm{uTYoM@i<d{@Phe(MA8x}9rU`utS<E_84k`A>VY<3hgI
z!(+z<59gJv68={7h~M_;FKOc!^E3}#QrvgZRN&}amv<(06JLCB`u~$9)&0Wh>auR_
zI5n>6Wec|nNHZJ1R6Lw9b8pMdJ<Y#mGUD@2HC{_SwC-MD*%pJ>A4|*Lo<6qjZ125-
zrLxm@9xpv5`CT);y?SqEfp_iVU@pDA7vx<EjhPg3KI~#JC}sY<TDa}4*=CXVu02!q
z?;LW>ocdzvDuejv@v)+}zgVS5<i9f5&^_U}_tHa4=j@yu5ux&r=WPn}(TOt-9JDRm
zY!&e8l4I~S!^078dek;gl%KHdXh&vw|As5>^KC75Ynfh-bKfq(qyNq$|3&|W4IYzo
zpV)=Wc;_0%WLwPac4qw)U$wQ<+*kcN`@E*OJu0yB$BSNhChKy;Iq`RwGbKdot@Hb+
z%&|<~+RnE4!k1*bWiq;I=fvY$lczex+<B?%7SfcT-#_#C`4|D^ogq9c*H|ulKIvR_
zm-m#clSgWQ+gYlL{4!qUssA*8`hyn<C4rtMb7W^763JhYv|_JG)V#Mm?kWq{8c(xY
ze7W7jZbh61i}1dXRqB)f#ne0r<M+6%F3%!)-_z9Bg;~zG?xR?`=F_TBhS?S;wc0}3
z);DFv<^`zPeVgQ0+mR`gq}cxBl;Xak7Zc?pk5BI_jIi5(isjwDm)p5ACt7^E_j9k-
zzYSdPyF)yka~=tpYxZ5(X;)Y@`*MM5Q0d!=uYc{HV%1e0&2=SeQrYx3(HXsM(x%g&
zzl(U6_CjfOarrrsd%KccZ;9^Td^c-)^+A3AFGuF>e6sP~(SU{OM;u+V?9S_MeKJ`)
zjVpIb{3N|ol~>o+e9!icnf1lH`u>+koXeta`Ce#vd-HC6yNSJXm-^e^Y|RNDm_IS^
z?egh3`qu5;S&g=}FVjSw+t%)V8Q0qTSW?yegN}I0?u?0Fn;PbMpVF1I(YiTlwZ^Yd
z%WYpLiAJdENJp&|vbo=LMxQ~4yD!^JZqLqxvkfQzl6%Pe*sk<bafMOT$JALKnk!NS
zKM48D+Lj;YIw|u?=DaAEZhF96&bUoi^(5oMe;k^n_bWp0``d33Qia=M!`lz=*uJWH
zCe*w`q-Mwcn;+MPPG0xm@3LvWTlx0uFsLjKXWDkRmovQeo6|iW*4YJ$@+FGv%MKPr
ze>ka7aNyQQ{`DNq2UjH>n6vitro#yf6$<th^gr2?xIyju(S*+A*CiK}ZRQ-z`Wv`;
z2D`mfoc*(O39I*|hv)ECY`mCw&g~)7oSKPN{%a0RnUSF3?8`K5hDD}i>9f@bS>FfQ
z#5hae2r%tde2^xd6S?f$v6ZL)y?X2WLU8}|UrpzNvZh7b%+EgBrFnnh%Q;msNqbcG
ze3(>uW#X0B3n!L%M@k&!NdCB~D7I;Ho#vHb&!2XicAu)!-?V1Kn?Bz&pHuI?uzEjV
z%tuAcI>J>yAlYj3e}lfs97k3jc+r&d#(Z6cfZyeeg2Md*vm?}&T3qY@eBCvX{r121
zr|W;MdS`#CR9EeC;xA)^{@&X$)Bd-wOSJg1bienNwVwnh?r*t&>{SKhq5${o*M@Wd
zx!s*E_v-O-vF`Wbk(%ApzxS5S+tkvwz2fIYRcEH4sWYxzJkGCKQyXZ;G9%W{WS`f>
zhznc9EW=EH-t=hXZjX`v>iV}WdEH#;$<j$nmm0a<%j#V$BmLa5YR;Kwy!PeI6T@Ox
zy;<sDX#e{(_r_D-|IR42*>k90mi4mrvM=lY6rZy?9bk}R@ZqG|#^ZYST!rh-@9(UN
z__NKuYo>Jas+!Xme}|dR4sZQ^YR2s;E%#<u%$xe~xohbDJyWx#r>uGF!ns>5qTTD~
z{blzr+10PgacEjK-TbfU{J5`S`!9FJDP7M0JWa~(=agFu8u+KHOL#qPdf$1cziW=r
z^3RXFxe}jWoWA)%(q8*s9oD3MyGsszvQeqnw&2LuFN@b5|JU+*@24fF6k?W4Q1x7S
z=>58mx<cRF&{^Iu^QT@5QhOX^cr7l`>Rw2e(C_~3r@tlMzo&N2QeQKlcU_dtwx(<L
zc{7gIUfucR<(v&ucFf-V>Er$S=l@>&tg;r%nHijZKIHk<)$?tFY+DxmbiOM8YsUQ#
z?sEgbG+&9y)C!BzV_LS*tR~}o+w0Jkg&#d8t1r_|T3&qna^&mElQWlg9e&4Bcgix@
zZ+q)!gN+Z(em*!geeK7V-MuEy_oZGd{hC~ymf+bJnP)p)xaUXSj*rhyU3=;CH7cn&
zHri^g<h*qSp?uFDW=W>6|H~;Xyl-QsX7(wey`P-!?CR7t&rF`n*CRh=vG2W-q_y(f
zFR4av+wGyV^UGS@(tP<YZ5z#)>HF{ePCBWZY{|ZDPGj`GCDYnjb<PLwxYL`YEBpBP
zvl_`~A<nsJD^B{jU014JyX47f)`aJCEiIm0Gj5)p;e7W-WZk@)c7x+dk#obRU3#7M
z%6qbD#J+vulUaAIWY#^SG+(XX_$BY`2c};av|8T^2n?w7Yb-r`<!PNr97oON)$Io)
z11dMIUijYQdavKtYlWX?*Nf?Q+X;LwoO;h_&BfN!kIsrq#s4YVBT}#X_m$A^l7kJo
zb>*LGni<O@y7#?#{^d_sTEkweH${)#mc|;sTG+enUP*jR?ve$C|2LgvwUp>ysxv99
z?zEomlANP&-F=dpm*&j*^io_+m3L0++go4HY~Cffer?FsU9bCu3KiZZPK$n2KmEo-
z)%j;Nr`=j{YSwG<9gNd+Kim`JRlBbK=6e>yjD_5y^=~H4SSYlOb2Wc(da&G5@4RK+
zPAheH&emh;S~}sw>5XaXUaGVCO+TKOPcIZL+~M50;$_;@pIf>gw{_3kaLfD5mq%Kr
zzWi$c_7+dzJRiK!Z|fB2S)C7`*zjJh+z_PRzUp=R+=G+dS63Immv{an&sY18vrAhi
zcIWlo-eT)g#b?JI-4(v^g=@HS^CUUlD<2LC%<vc79)ILv`I^uQjV+wT{Tq`qugf1y
zJ3MW5gQxiEexb@0&(;(b{=4$`bHRhjZ`VDzr@pTCU&rpL-xaR)=N2x_bYIfJBHh8T
zgN^Ijq-6`&$jAI;+q2i=qZ-?umub$kZQg%h_NDg|SMj?)VK)t>;y3+N4hWcXTrlDK
zmyXjfuAZ+h6Ljk5yE;oQx2ojMd+Fwcu$QuXRlZL;y?N`zCC<n4uL?^1VrXcL*v2<e
z(Z-F<L!d+B|GU5E9v(fu!fVNgbdy)1#cvY7u1cKt(qzxxJ=|;^8oZe*Uh8&WyR)P!
zgVpuSG^Y*!EP8WTzgGX97!=3Ux=HKp9^QABkJ={amYZ;DO)`^7E;BLnRayK)?oxnf
z#p9$Hfj64R&Fi&0ovbDGQ)0tp1>Mw}PPs=I_55wgjws)Ekh?`ZSh2ey`c3Rh`@>Ik
zXE(@w`&Uw8{5!Z!rfJId#hOP8j@ut|e)QB#MO}hZMUB(R=8^8ilpPX9-KNDCAE(V0
zuk()B`S<;oXY!L@JPH(=WoZ+kpFUSV{pqrAUUBJ0PkmcoTjW1n)M~G3`r*nxk7b7X
zDycJ+ygKfGEfATm+qT)<s^av|?^;F$FFVY)$9lcnF~?Ny-Q7KR-e>LXF!=NSZMe-B
z=a<gba=U)*zuWXsVE6sJf8Uk+cAMTfS$o@c&cbDv&Yj+@(!#w|?f$$4c4zHw&S_9u
zxx~lv$Z5F;=`3Dyx<NmxAM|aSZV~2to$H_9or1vR%?rG(qV#R7?_OZG+kg70*x&m2
ztV>&7dOtdlm~vN3Wr|Lx{Ei8y*8Y8LRXaU)PGy9i%ZK`>o2)N~Tb>M`KKWJuoHwgw
zj{oIKtm9f#sBg%xv-(}e!=87Oem&eiA-m&_Yl>8G%gon8(|s(xuUYDEN!5?{FZws(
z^6@>YJ6e=&(pcVB#@^mH?>?7^{*%R)8%%g+3jV5?<JWdZ$$49=;^LzdyxVws_4atM
zeVVaQ(Y<%!N4b4B_Fm{rZgt-fcw>Tc*FDAG%;`Np!?l;3e6RU^;-Ao`rAqU8-(Q;a
zL2a3l$E8U>(@xfGev-OvNmzW)^fykICbehl?fl<SA9w5Up2YWp?N848+|_yL^!`Fd
zK2y|{0ME<5Nz<yo>O8NRHsgb8XOd_6&TIUirl#-jQ;6E>7XRpMis-B(pKtK*({vHo
z)%a(U@)WOF^Cy;)f&p5UH#NTgNxxdYbgDdSd_v^DQ=juqTP!@)TqR5FAI1a;OwaT5
z*ryox=5zSr2MwzKolM_Ox^k9viT=)wCO$H}r(F-1Dea8C9^H`?lOfE&Xz=O_k7R(}
zwzusui@ow|G?^lnzIwQ<>7w&Jb(ho_?#rjUN>aR|*LQo*Fz33yJ1|T8rO*YjLouB4
z3#?};-@VJY>_gl-|8>fL6LX?Eo^ww=H>vPp(gSg`Wk=KZ-JGcE7t5_Zy`)g<XThG+
zri@Y6yZ9M)p4MUdbExFON4s}3KQGS7*tGY==^ZDoFKLb6{N=VDx6+%t9Thix>pI**
zJ?5YCci;AP(Xaa-g=gOUvSgL3natd&#uj0j@e?M53%=Cz(GgctQCg6gaIpH8>@}O&
z^AZacUoE-dr=*ZGPxDXc)7Gm;RIJM?BY94?INNlT$3D|fVpZNT{c@VG;ErQAYbV{i
zt$noGNMhY~#^YL1${W2d9$o6?5v1@e|KXj&sds|wSIK1x-L-2v^6JZ$L#Izi-?_0v
z@R3TZX^z&eU6U(+Itx9U#j;ILGjGMUFE17-1|>}1ns!1g*`wHvC2Z%SDR14<7X4=b
z+{ta3`tQO0%M+HUYz{q{<r1;eqFrL&l<18+Bv;4T@Az4w`DynH_a6+VoSyeK>&i(6
zO+MOqPFZe&-;epxpS6TT1l?pD4u^RMsXSd^x<t;oA&pOKn`r&JWA834?74MbJm~7A
zn@4r8iY7%ZJQX*+L@nz{lGLM{x)pi+>sapc+g0rOpT`+{XmXcyz)`2i{EBN$OVWJ1
z8rp2sE@rD6RTi21a!#4|{(yeg4352zW6Fhm>%`m??PK3M{n_>5+x{)$6}Izshwi#A
zvOJ_J{(7saXRZE3lUa);1=jOK<|yQ>j4Iu%ROzW?oMIijX42bLI-c^2U7f>%3zO4=
z4sJH`Z~e1er_RV&@v!O1Jfq^$NS3{QIpK@rR3A&OTo=RZ*-^~WBe6>5DtD@lx!{+a
zy^MytW2!Egi|A`B&3{uDB)@a_8?&&JX<Jnu>=e{v3Yu5i7af0Gbo1%G=To=0-D+a#
z(Mqu|`pRYW;;H_}LvMZML(U&j(f)FSN72vmq&TO>6rN!9byjER-MhP&Wy-!If43}A
zyVZJRLr2HTx{Lkgvmfm$<e2klTX|!}^(*^c?TEfvXSZxmrkdnAmAg9&SMA=Q@t$e_
zvS^_zPt>kG-J*K%X5*?d?zHgz*(?VBHxJkpA1QJD`h`XL^7RK_auP3xzx>kEetB!E
zcjmH~AJHjN6Rt%rnv>|CrJQz?*PZ$4njU)@?NqMheN$>KPRTb`Dt|irecOCBejV|J
z%ebTN9^GKE>!$0PhyTLoKQy{rV43w`ss4WL9XC}yvpp|GeLC$fx9dr^X~>;j7Rwg>
zP?;ujvpTQGGWg$DSD~Kca^Xfk@eeEOPluVN9G9B0^2m)}zTETIMn79q`{=_hLI2r1
zi)ZAmE<W_UXu=Y%n?+oUPF<Q<@AdIn-Y@Ru<wuis3_HE;O1$MJddsZ2A3W`77kk)L
zhUBVcg~6>Yt2I|HJfd9pakGH8UZ<kB)9FJ8E(iok`QN=#ICc8I_gWh&SQ{Nzi3Mh7
zy6Bwi)&4l=$>j61MT7Xm&iGDZJ=$wN=lI8$omak2U+ytQL3!b>gixW)cM~SfUh%p$
z_welGNUey8T%Y=G{BiT*eyi~NdC0{t(@sBndP0wHowBouWx(BYqH)`MqPAbj6h9Iz
zbk=2(;B$`+i@13{w@bLEH>Tz&R5ryL9h#}b`OtBju*Zva()u1%FWeICTYR^kzjSBk
z-6uuS(>1kDu6rIIJlSgJA~mZo!TSWtI-gFJyZvW_Z+-WfC})4MUrTMzOkTy+G)qse
z@A{s{FUmt}9)!pG2z_6@WMkawI1Bx_#voO_7bU;Uk9N%PDZf7bLgU9(-X#Z}kJ@K>
zq~?1qT*<uT#Kvr^u2bBxZ>oemj~>klW(u0?{MBSnsi5_e>9^dycTD%Zx1;mTo{hI%
zi=O;eKX*&_iA$l{WSdt-g5A0ItDE23sQ1qNvn=)8uC1&gON2c&<tF_2(NNi@T)$BK
z?*By*?{%K9tFBsK%-0?CZOQDt;fGG@2Brpi90|PQ5bE^(Rh;<|9kEnZ&7!X#@?~yq
z`*`{9f~097OMNT5H4;}mTKDpu)0dqo8-MA%32c*|Rq7+0a#ZhxmXYmD9#0uhtys1<
zE{a$3SAOoAs<Y#h4rjQx{C*3eBfLMuII@m>*?s!u9QTDQCMm8e_o~#Lv-4XO=gB0+
z#)Rw9qLces1S`8G&t<yE?K<WVuz_jkUe@?Et2X#;)!VY~%i`0?O`%s`aF%Ia_3;-|
zQBb#-^X|=3&!j??FI5X;Y|JO7I9)j?de++R#{os}_1z{h{F!G|rOwwrSaeG2;Nyw%
z3F6LdOBw$!@AhsweP!?YwVS*@z5G=2x~hchg_G4v5AAo6WsW!gG9{f`5;N6`*E2_9
z{toYZoJ<FJgEVgNbl-LfnQZgK<8pwY*<pv26YYB9zH%Mxw-xmzAD+ull3FL9$Nzd$
zv*qUdOA<2HUAo5_w&t9xr^8;wRN10)W%Ifg_xk9ZJRMlFCdWv|{JLfJOQu^NmOXyn
zX%TL?XGdt@!q2j^LUzXGJ;~p*>S5zW7C+Tp)1;Jy1n*jD2pzhpx_!kFt7{hoT`t%)
z8qZtkze!Z0WATKw6LyMi5>wUJ$q2F+vP~&IoPFV|Agdv7@0{tAPOvbY*Pi)mmQShq
zyjBU}Ia5CtEfm<HwC=d;7EQso$9x0YwyMsa?Ng$+n!|SahU?PzgdU6e)lWL5Qg3NH
zO=D-bR`qO+73#NU-BMjv{imHdemZk}_>YY#Pc$!9CcT{#G|gw@_ux;P#e;QAE{0rQ
z_u!t&y$P$gOrM}Sd1jI2PtI+ZMZa6Gy!@s5a&E-smn<^3%WlZnXUtmu>5}jL;AQoX
zzQ{D!#jgyRb|_Bo=lj@6yHmtI`+V5F>idkMXFIkmj*NWNc_;ta(dE%X0{;~GLJS@A
z8o9!E=sCBiTzXn3ak`nMeB=8Hkw$I<f1iy!b<*W`Q(p4s>F4F!ez`qghHnw~1aJFi
zo<HsWrM!3N%VzD;5w~kg5xDNCJ^4`S$<1~5O=5HHH~D$T%+$JaNU})tN7>;fkHxE<
z`(uO>dw4XA0wvVC7v&yGozCU#kkV=K`4AiDVqvM}r#FjO$4)i)z?r{R)JOa3$HOOk
z#iX8jzq;Z*i#5Nd_;*)Oziy~{QoDAybEt;Q4h!D)6Dv+%4)0!`qa|K+>{4m#@jJJ!
ztXCIl{ClcEuw0nsy7(N6J$spa6_d`L)7Rtqp`*g=ylTS)^*m?32%B<7hC;W_fP%R1
zY#fRm!bu5&5?dy%c~r-%^ybq#d%-r}v~cBX5ldfv@3{8DSjF^?xc^G4uOZXd7^qs6
z=t#1>sN!&+KIQVn{v|dCKSy}BD$Tq-jk|4?^Co6bd2!x}^LT$A*jeg7Kj~rH4*mao
zHtsQZU*VaVs51BZS2xM92~DBQiHQrB?0Yv~;9#uS1j}SkrwQH<l$OlN$l5R8RPC_v
zrptx{JdZ9)?sMh&GTB3{;^awF_nuiz{!-ViRm((NCn%JOyy0d&wtU8(6<jCp`lz)k
zP2X&=_(cC!kBtlV^a{nPtlsg=u21$(@)iLfv3mt-m9EDwdmShe&JPbbE6%f`RW<sK
zyJY906IMwaYNz>rK1rLY%qtdFxU-Y(zBpUfwT03wLH?2cB5Mpyxj6Q5wQoDLTD9^7
z+fgeW?nZT=u-6h-nHs$9T{hn-U|8K<{6w#PN#C{|T9OYw3kAmR&<-svS$S1p&AQp$
z+alh(emm7~6KAX2VXxY4R5q#g>SE8b4{P|NZC}^t|F8agL@?0GNg(sc`mq1MCGBr<
zTnS?n=+tKZn4++ar$x4}a}6{5IxW-1OnVkPG6XPNo>08gZobKSUF&J-c{x(MZ|FWd
zKe?V^arZ;R?E(cniEbaC1>0or;F+AJV`5oh!QFFFZu>65^|}6=-5(_^eK%Ql-2@+%
zOBG&qu@Wb2OccI_cFuax^Ec*Q--;$r_cmdbYi>Ng%8T3M-BUC>v?dmHxjd5pd9(BP
z^f`(jw=J~%uqLo*|8CXEcRaL$FY&Bgz_COA){kX!b64HH)NpT7wH=?|#X#SEuYJAt
zt6eYdzdk)BSg;~Q)G=2!S83IeYg1Ld1+VXSoj%1_h3{?a{-ZMWjb*)NUvv#@1V8GP
zUfHLa`O?inYEtd4RnF!Ye_H$IP103)tgvaq8?VMF+1xmB@zp|-%RHZb-1_wEghCbO
zT8rsXZJN2sac%7jxvCDRDN4RbPC67L_15d#S}UV~`~wjkcH6yrd{1#);^h1)TH(od
z=zdCN;8t0YpEtxb8Eq`L$!jj>(0H`&$$|)x`CFe}*ZkL}pp{?o@<g`e-j0H-p!C-M
z_1rGLQVVALTsIQX{JWTM@;WPhmqiore^`9mX8srUwAM8fJsfujiF!UQFTEly*Yfa-
zy86NB>W(KfwDMIWXT}^=6f(S6=DYIrvPVZsGp>FKIQ~*S%12UTxkPaPcYUi1OI~qo
z*Wx@;axrSo>un`hMEm~F6I%6yFVNzwZ}hU|FRnj|_e$L@61dMr^2&rMyWa_~Fi=lD
zXSTFAMo)9CXxz+A=d|uOJ@1~RvNYs>kYjS7(%&~F+xMP-`{-gp;qz%(7h{;B^yGE_
zev>}!>oiX>B6;7P8T%GIpKkK;N%0Nu$^fNzH7DXTIjk-V6bdwRT|F9nP~>#f;x$^g
z+s|03rp;+gF}_h?_0J^sQ1`lWqepMJJLKB#C@ox+tpC!!DD5@Njgq2IB~APO9&&B=
z^laW8<Srofg~9w}fAx%btKFZ1u6$jpa{v02rnt!YI(&zBKWP`S^54DWj`_#eYM(nI
zj{g-*da(0q#*K+RT54Qd1#Ya1x^Aug_A8g{%uFNp2SFQGGsPQhzwlBlOsL0-HM&3X
zZa~uffb{Fh)7=kbP6};n5wR?_d8u-nKeHvKjj7v5uRqU5+Cz4dot<EFOKxMtcaHf7
zFDbk_yS25!si}0P?z7YKVkwV4t?4(9f6B_-xAdm|or2ZMUUE4f{+)C1KKOEveN3H7
z!@fvWW@VSXn#~uF&$^KCW%a4^$zCrL^A^0TbDR1-Of#{&qti=;_3&JWAP?o_n#)o*
znhW?%x~8OvUN5%zu2sjD>=I#?*CQfRzPjPJ-=`&hU#h3``SY7dab$Z*IEv-(Iee?J
za<M{=#<jo4zB4QD3GrtQYTeFu@w1zW*~;k~;zRN#C`MPV6L0j~5tjG*c)!xsy+0Jf
zCZu`F`MBPHrYhmOVVR<y$K=Qluh&di=T%oXopU1ZO*XFx54o9w46K}s6}IsB8s5LF
z8ni<`xIg*bt|#pLu3Y5;9}7a49l5(tk)h*lzEQYGh-Bh9$r({?f7)MNxOwD8%l8d^
zg&!VFK0j~5(c>}(t@lK4&%Z9Hy&+lR7~k?YrkD1HZCu*q)6Cew-L@~W`N91D#TNgM
zvgYkFx^LoqkE3nh;RlB&cZ>c%(#6)#mSeMLhs2u`Y`WX357p}DaM~!V)jd4R8hv-q
zf#X^nx1L`#lsLx7l+DB3Qk21aGJrq7N9(>WZ>uyfH_Mz|GS@%s&0+ZZy65nOR;Qr7
z@eM+}J|gFy96nkVz?=W}V2$~X!Y>&MLni(`@JQpkn%wS|UWRWw4*T6t+O~Cj$zI>=
zu5kTrbE9iE`?cg-6{q}{xb@1mIig+uc=C1c8*emy-ac9%d-G@Xt`$p!6Lv+sb?amN
zcz#{g_ZPC;7nR8ds4iQs#XUXf@a#`VqMK(QTPG0t=x;>Vvo-nUaXYQ8*M3=j_~x^B
zb3XNX&Xaq;_|4wSsvB0$E!@>xzbm)?7h~7C(#6#;kGxx57A$P6TilTTVAiQmrd#*>
zoj<hXe0g8^@(|C~H971_`H2~&Q(P_v3blS#)l>D@JB4|x_M&Z-fBqD4KYu1U{kNJf
z>om*Y)lXi?Om2C(;6sP;9{Dq@%M7M?F1#V~ctYE+=6B^ktLJVPWsiCKC&Z&nxAk1<
z>qWO&?kx3D+8nKP#O?jtT5%)K9saYX#Mb$*-@8|g^Wyr1v<I!+sp9Io?)6*rravi<
z4gb4%a^3}_IFDW8J?xLRNVz!gcy*iM!eh2K<<A*=q`3;+bAPY!3T%v1i6|)heN5@$
znZk>1^`foI`FA>PG=0<mZI+Qa@A4Ay-|grAmUQV_o~c-NFlM!?u2z)0rtET!@4Ba_
zJ?WpV(K>rYQ|?v4WbJt2vY^A(llu+$N+x!lJ>mH@=i?(oiLb}kavY1`ifESHT3+?=
zy?tJULCjC)Hrq;>fJ5as*{Aq(<u(*fdb;>_{-TvucTfF%Ed6iB-vx&hifjJ3tM_iO
zjL67*FTW|r?BkKCPE|GM#3s$3wr4@F?zO)~Y%-hWE*o@Qyl}1Lug71x>d>iiaWkg|
zEPGqzwO{Yp$EL-53ZgmvS?V2@ZoYD9&#NV+`X-Dj=}!9p0vDZK6#C%lN{01GC%ATr
ztv{@{{=sU?X}))IRrwnJ*szB0JERt5)nnyXBBw0Z_@sq*dWVMkt33j?>wNrFq%?N;
zSx4X5dC)LGudq_`(#gLb>T&l94u-~jSiUIb{fkr9HF|!wzcwwc)YaI~HEGq`<jQul
zXG(2{)+&ddOkUl0Vb!DhKNs@#SLs>ZyP_pFlcnta%+q>v!@i4sc&;blUiHR5ML2rl
zbv=U*HsuGmEYcQtufANdNN3_3seNXTJOg9=MZIfQRen@_x__aq*1}vj?_BL05m%2c
znY_*C;cb&W3CSO?815;Wk~>u=PIP%&?cBfdP7$in;d8m>?t3I~#zOT_;kt>kt0&f<
zo*Bva<Kx0S@w^{L9Jh0u|MmX$zwPs%#pm`3%$Ih_nC=|2A$xQ8pRleb-jbeOyYnj^
z&J3Kz6XN^a%KYZaZ^oZ9Rz@rc`<k(I=^mxo=Q6TC)#q^Wb~*0|Jk{cOJy~kqv1>ba
zEZpR&qTsWzE08aJ)~))!1uc4qzOc+I-|~l<e|e+(<%At<jPlCIsviiM7EEm_&Y9u+
zXWu$)>l+S}_pkiK%UpYUyL-)s9rthl*Zb@G^3ImoFD7vLdDZ*X=-c_uPMp_nCz-qD
zT1L6!{nJYJ((3MWO(yT1bZ_;fcinF$#;w;_%owheX?!x7`_goq8k^NqKHm^oWZzPF
z#P9jf4`%Pb3%A?ce-v+aGCHa4j`kY=xAjFng>PIG*B*&Jb@u80t+T}oA1s^vh%0>d
zjrYqselELw{~EvH%oOdye?Nk{ZhFTZy%c3V-*C<smk)0$%AaJs;Q2P`UR1kk)vf5-
z3nGReAMt%py>4vnepZs%^;M#uY!PG7J^ixB>#iTI=CcX=v&#DReXd9Qo=#tE;Hban
zn#IxdgGPFVHLSL43S*iVo}HR(5oV|_zUFb3Neg@H;_1#-^901LEv2kgOD8Rw)-&H#
zGGFhDQk>qp?yWM9#CEgo_n!CvOalMIza=|Yn}tM+?5gO_{B!4k@a&x>U&WGN1{|J$
zz_>0xpuo80T<;cUHG>;l87Ft7Po87fBV(_2w3m5?i9&PpYnMGO!Yb@7o7i906iiL}
zz2(E2NerE$JO7_O_w`_@Vb7=O>%Lrxl(bc-XE&*=PW$lOV9$Z0jagSU7nQQ`Yo;x(
zax^=qXY)RZiCbpLrp^BQ@(vzNSox`_LsKV3ROt8NYoRBsU;YYS>t*lf&%U)v@!tDs
z-%MugS((KdyF$e>F<S9>QscI%UU@!7&&4L0iMeevnUrhxk+oVth2_+}kdVT8F8}8K
zel%&ui~6Z2g71nan;o4oF*>fZHZSnpqGZR@F3mDg)0%EA6}Y|kY~?v!UoK6J{Ef-M
zH8a#d_IrFiwe4p#-^RG|zm=ErTaSc3_1YqRnsaCNsy!~1Ru2wd;MKDfHpw*l5b<q_
zRw(0JnV;L&-zf-F7HNL8CT998&%z_$Yq+`&oHkGWso4-Z^|IgVBZ5g)-vV0C>GAMS
zHF2Bz#69$R&#J&5Y<KMa!#1@syuA}FY;K}*`ax1R_eTNd%c`%+3(X~-*)O?t@U+FZ
zr-quPk-v7g?b3^BmD(q~c43*b-_3O=o?bnWtLA1MK7D#$&7Agg)j4vJ+qoyniiXsk
z+g_P;vsSQoeiwh%wN>t132bk_zSwi|^+}7oCHKD9UKOosTg1;guVPxqssmDtt1lhh
zb>yU#O>tn^RNpn;;-;^X60ExRFWa>EUvm;~y7R0ZLg&0R9{FZ{(5Z5>JEoHQcFC=G
zH(b8$@QaXmFL3W~bpDNiM6;K?b5nzU?OoR}W8Kx;su6$0?lnrkTrWKB{`1o@<ri;0
zdv)m58_%nnA>zxYA6`A>csSS6O=7R7tT1kk4qj&^r<IxaUhmUO-SC!hJ@W^D|2<6!
zzq!0Re9vQ5)2~-{E)RB^e9q;IiDqb&!TFLMDSti+H|b^WU$VmZW{-F4*MhUNgwB0?
z{zcky&!k_@j0GMhyPueB4O;x9uKAz_<AOu03eIg7DwpUq;!=C0)bz@X;Z&EHXOv#a
z{pk53R?8V3b((*(H5{weTx$1lewFuCmAIGRgs-X}^kZ2kQRfubmMiu;@1e&oA+Z}=
zw?D+N2Xy{6WnkF&DPi}D!)79d?5xGB{T8V)Zrm&^YPRr@*_LC>TAs6C^eyF`8}__B
zz4%sqCA+ZT(iv>|=5upqJ&AbEx5Vm~gkwz6)L4f|jlBvMaq9~>SM(iPx=Fw|PGL!F
zNt@BCQxe$`!Jd{Krg_<|N5qfl-M{$4WJg=@#={jAf174&d_1s#C-31hJ=SDlp`BBM
zA8)K=`qW(-+CGEB-0@U!*7YNHJY9ay2VZ6t1}|RNwX9nC2G1A!Q=gaL?EB!Ty!u(O
ziqgt}Dc&)E+6wJvOuF`i)x>3WK$oK7xk-8t=kC-q@u+&&Ecn&(&NHXy%+s5)Jhz?i
z($FhzkeD=^d#(`c<TVWu0+xRt_&OYXX|&E~*Vn}l<T4-bEIRyhfxXd+Rhw^!?w`QB
zz}x>xPCL6mh3}p-1`#qJ9oOA*tllQ_BeL{Q1N(`pz@wW)Ub<XmSo+pM_;J^C@8U~(
ze>n7ojAC*=i03fmW&6M5y!6Vf@Yvcv6BIs7JH~b_#p5DVwb6rAgG8xL#W$W#jTbGA
zd?&;>9dk=Po%`^`r35KyC$}d|doq{Z-E{YZ%XX`>$5-!_@HHA+R%uyG_trUWB-O1P
z)%4VHM(-I5UejV_*GbbQJS{4(d9>$!HuQSczS`es4a=v<@Ocd3r(=?=WB3*QE^O5*
z;G3bGFY;x3;33C5haTzm9-nZEx%6C!N~F4F{r~?PzvpI8T5PrX`%j(9#v?!f#^;y|
zD)wk^UbQ;CitqLP&tH8kbD57xw1`-pe7aT3qFz)dBJ*j~%T<xfV<!ICl+9Ewb@`$R
zUm>I7Qr{hSeU#2|+pQ3j4A^lr%X?Z>hmf`Hb<_Qm&Zq=G>GZmi<8;$_$Ddu>0y(d$
zN#35KGO4MSn`2?-|D2+e`GNs`UTTduIXMK&7N{pPh375iU9P9RjAMRCX{dB!scg5|
zJ_Dbi1?p|Ft8Nubt(;{l<@~kxv4^;k#@AU)0TC5S8schm-JcmN36;yKn=AZ!Z4|bh
z`|E*;+qun!a#DME-ts)!v`v;{7FWvZ0ESbqSFVr_nfU0ySywCXlAE3RN6uYbYWqS+
z%eh*?w~u?4+l3njYt=Wm249Hd>est?Y3j!r7s|!1%-vd<a!@4lQ9#(NSvq??rbQ`h
ze=_2*>0ImZ)xb{i<%tr{xFeIVY@TrE6zfzYtpX)B#pR}3UwimeO)}w=dvVlD`M)ch
za#9xGr*x|>(Kk(9)~zPfGfn1NY+UlQ`BGr`nQM9-5jDvdw>2)#h$%Hya6GT_z;Ky1
zbJ~PCymPzHIIa-f=3jJ8b+PZ=0-4P-ZmE4-^K#GUIc?{eVzivsg)8Y!{otMY<$~6M
z)XEyG6>-A4IU=jMmg=}&Jvl|NyT|yHasHp}Ze<VLRixjEY*vsywQEn<P4DdTxFaQ>
z{w=U*JJB?Cqr&PnhqM;I4?8zmzCFu&wsZ28xsH=>9}MZ|kcv9Y*Z%E_iS2?W`JGEd
z^p*yDIUf<Jde*5F^rhhG<CJ+zmwi_ATo@R1nDx?HpGR%J^Y7WawRReAU4Bt0?c#nn
zxjva+0`BU}URldoJ0eAwPV#1Nvewwed^LA2_t{^W3W=906@T|}NLs#P@7x^n^PK<x
z#nHVfD<`SusrXG*S)D858(lezUGLhJkljc0H1&Rp^+xN?=~Vi;eUiy8CB6y`sn}Jg
zd{nycX&?Gqw9L>(PejPj>XJex|B4Hrjq<qjXG?ALls{E(R$+CNYtr7?DK8pM`aa4o
zX^7QHNV%l8{d#5T2cHgyM+cV%8q2-BwemonO03l`kC`Gx4{ByS5U5jHl(@t(Nxxj|
znAWY}rQcqET0Q;l6RBAzUl(=1yd+*$`K<h+0_$H5E~khuj$YHJdxyu&E_Hk$Dm|+x
zcY@O<E%BYUORvsaQpLj)`}DfJrtDlZFFDVdzRP>7R>@81?^aa4DERcmB8^B9*D?{g
zsFgZ5(mr`ua~JA`@~rsdf8arw@M7<Mz3WX+B(I6Q&2jsJN%q-fnH9cL%_basSSK^C
z>*(IpHrJ$iwol@&L%tWgPDpY|o49p0^#-|33J+YeuzO0Oh@<ku8TQJjq&hiz8GQqJ
zW*nSxv7h6~>62d9PPe}4$(bR)Y{Mo!?Pu$r`a4G0wdvKhZ79F^#(7o3+I;<}?@yvH
zaCN-$D^Oh<Ag0{4HKrwQa??&GS(A&#GA?(7yqBowhP~a{yK2E3ah2E;X%~WXKSe*U
z*bp#t>zt`F6W8T`mXut$i7!?=#Hans1Hl6y6S-vFU%Hsh^RNzWTe5py>q&`;4govQ
zHcd`m_e*<L<+5wjwcdLzik5uLv|Crj;K|grN;_w)5Yk9#?)m9m{@d&SO|Pqug!FeD
z4XaA`^?Q19rHy7*0Pn;HYaV_1!1k))=cyk_lZ8WsSHBQ(+FkLLW%b`HtnVDu6qa4v
zDxT$NCKr41Q_Qyi%Tgrngy*Uk8ZFc=x&HLf;#;hj7frZY;wQt_nzCWZofZ2QaL&Fa
zv{%kF^w^at3*Y>^5#~4Pm$IFw*v)C3EH5`&Tv*egeNao>%|y3^TeEo=o7N4ktGTZy
z6lm=EuArEva9#eY7H0<Ur^xx%@#b$BJuKbDcbgf8N*#RXX12DoPAyLMY0}n=u#IV=
zCzhs|C)_`DVEKt>7dCpW;WZ2|dObzU{;1*O6EWMTPJF+zG5!oU-=$5}=`ZYlxGL_E
zDckSie}Cq`HtE%C%&#AjN?ClYvo<twL+^afLkU8hBAP8udXr`c2+FHy3pMSixhxtn
zV{_A!hN|U$i=(r)&MRTO+HyfI-osLjmH(t@Z@jGS^?5-v6qZ~P<#^(Fzr1U4_zW}F
z_!^Cz<mc0u{AE+GS?V05`Yd8+@Re^<nwqp%Y9*#GJ+-~dB#iUdg~pF}CQo43;E`ka
zXI#F_Y0H8g*J>Y!=Zlxg>1eI_CN%xslXH%%FEeusRe9YP(_ArkbN$-H)Xx(pGhK8n
zbU2l_@$E<VB_|ezSfm81bo*|QT(VOzbCSjRj*TL3l4EqgsC7Jba=+oVcw%a}oLhP5
z=>;B6o+Yg%RT^4u4|y)w77Ix&EPfV!|6taOEt?L=2(3BgeZX8b;L+9F#qO*^Q~J(s
z2@Y8HA$t2V7p27Kf3h}mztkwdb~5wp2_@bscSY{a4_cyR9rEz4QA&(^uI_Z#hL*KT
zr{AT!95OH5D^P2pt+ijfns@1@6R$+RWZVhv<mY;`^2@IFhe3hIm!9>06{RxG$kAeJ
zOmk?;756hdS3TXjzlGK%O?mHmS8)5=((@s5-D$JT(tk}f-Kk(aDSXNN`17jkBbRg>
zoTXhNvAQSr-{ZUrmeWazsVRAjnuWjZ5$0PL5xJ;0s@Y4s$NAyHOAI?cUWhb2({z#V
zlf%Th0l!pbZuD%HT)*mt;KKmPTw_T;nH@)^u3j*kcPfzeL56K}PnuiFk-jM&b2r_w
zGFiHE`OoPh%Q>vQC6XN`>cpQbm2lBJvF^hCYm!a{jSCm(eNh#dVjX00;8L{K&3&%s
zNBwk!#M!R?`@QF2SEAyY2d}x#igj%{+N@@{>($nG%8T9{2wwcz%;Nj0D>dBFk;mB*
zS8mkdGM|!{v8l22k0Jl{WUDQ&WY?B_S-e<UcB=LKmQ%GGn6mh<$gQvms9f;L;c4Jg
zucosr-T%Zr3DYdtIr)^NjpAP`t~usmt(~`ow{DM&ax~$X_{hmE(DIMRbV&o_sU1!#
zQ}YxjN9;({?v<HldtZLt>c^8Bic;eA?)>96pMU7={(|%UUstANx&{e6jNTcNrK^2p
zPnd<4)$Nahf%mwVuWXWkbRf-W_DLCWi+dAGyrm|l7CsbHOZ8XG3OEtEC@Imp{psri
zBBC-o`Fc&I<nI`1n_YY0W8|lrz3L!)v6<xZ$%R&mpN#!}CirPSo@3;v^>~eupZ4P|
zO$;F#jSmi_S#0l_eA;4n&*jr?)40`V6t+n&pHbK+sXnuCn&k4Cg|iO6<6ZuSZ~3gk
zb(YU(6@!UYihHvaMdC8{1guzmu(5l3@mb63bAow2r3+4_p5OUwPWCw%k>y~+_0_gD
zYk||TO$N6E6P=zrZP;`ncBbe}KA2%(6W3WTm)f(Lt9Ka}?;0-2ucZ%SiqDxokNLdl
z;6o7WdQ3A9d-LLtEnhfyN;|e)|0!H;eJM@pv?_ztheaJO@p?tO?mbuP{cI8WzHIJ|
zNY0Bk-#zX8PgMrxPrX&SdcJzP{DKvCr=Ggmne_bR&34l;_obVKm(Ej9mp_%OaO!4f
z5uc0g?|^*f!1I1riWQFCyuMXlcIr8~tDD`w%1qsw4PtNI?Ebdq(ju$yrDfk%%w2rx
zoSjZqNr;v8#dCp~>GD-p+Fv&d|CO0q_if>Qwng*jzIg8NWi$U@Gl9Qvjzm@de8B3q
zbAD3UyH#82i<Z}VYcCM&tGIW@vdoZk*UyA^4i&S19h{tyU}wU8SL!H}#I)KS+H>99
zcJrM7Gf}zo@#D?X_ig`OKTyy6Cy05Y(XGO^z3;5vrXO71k>VR3To;{fc|R=a>AS7|
zx8IiJI=xN!c>Uj{b+>ok@qhhyOYI7qyzSTdiss$=TXIkPdfD#S-*30voqy-o-lF~X
ztM1=lbpL+RkG-FElvm%oJ!|{!2lM+*B`i31`)gfQHTMS3^wizc%l6N|r~Uuc8K;u>
zwQT$U#+Kj9oBEi=QFDE?eagwlovEBPh1(P^K2Cg`lk&2rzn0hUxA3mgh`+ajzu8Fs
zjDK;p=eGB>*D3$3CA}h^*~Th*s)z17kf0$IzEtpN+R-h`T6z6qqI>eyrDayt9^2(0
zbkRh^>~W~EK=Ka>jULvfx}*b7ydBpUymFJ^T4^sjdw<|JeGX0kgS8)*&b=KW-?HE8
z=WRc+9h2>!z2^V1mDj~V)}wXjq}og0FW=Nod$3G9)FtD{iEEM%J!jrwbvvJWynUf*
zw2NZg$^*{>E*QUf!~Srq$&4*?^(RY6C<Xpt{IDt5lqd6E_mz}S2hN66NjZJ6{3rJJ
znYH@72WRE24qkV<#@x;)DJGd<Zsh#);Vb927v$%73WWv9&A6}MZ?H=Fr+7QNaE|Wf
zVii4!EsKw|L>J#nRbs5=Z~mp2&mZtWzW%mbpooFTK6Q)JiMb{$pLnCBFW%(aaQYX^
zOvx*Ak3IL>aMgJKq~a>WiJCkj8Ld0^1-CS<^LJfYdaSzPiK%B)S}#Xd&!$O1E@sLr
z4BG!y8)(1DnfPMrjl)w8DV;oFC0OuZQQAp(kAbpNdQdEb(-OfmRw{8zZXLC(wYtvq
zrQ*u8hg!=6(`PPRG`0QP9l?g_Pmf;-k#=I98s2q`;p@sByTlyVOcAP`!J{Fvw&U9Q
zBJ)EVUwLq!J#l=Qft^_7fsAe!U7P+*eL~+BFnsnpxopFyhDb?=!m}|x*E_erJ2BC4
zhNg1jK65D<)tKnVR~Alvb6}Um{FJp)f7W?O+zTi<x?%Pa$7`qgCIu>l_XN$Dxn}X}
zfGrwEPFwjRF9n2KXV`6gsF%lNviU*1cj~rl9OdjHr(#TZ+!b(IwAjaa=2J%_hHk!=
zhcT`*8xKl87f6&!e6x3&*rOfm&*<Cp$#!nGDpoFjx;FKSOH=0c@VVN)-J6USW*1KT
z$YEuY@wmHNW!l+@iR%=(&O~mQETVVZDSqyujw?G`*K%K5Ay#nl>PF{Vr;MhnWl!$3
zd*E_t!TTk9E88S{53SacW;-`ynMC)@eQE{WQ+tJccPiV8oqc~zv0ToB<&%Q1hEa8D
zw%s<toTKMD7v(H_p>$c)xH7Eq*m^CtwXQxlJ-_g>FFUUFX<e+DbIc|#HNmj?EQ+bI
z`l7u}ev22aR(dI#Yq(0s+Urtan|#oUohDgcXR-{MJ!f9L@^h*1=k=<``20h3)q<|7
zcpqQFbAv~3cFMV#FF2;HXb$+mdu-`*Hjl}AN1gZas9C6lKl$XCIx9&ns5)X%<AyC4
zdWBxO+8VA*y!>}bn@yEKTA(27h0HgLe9mq;ue+&9yQob!B<<jz)HKlx&umju15#vp
zdd1hys0c}K_^*GRVbV>hV{aDiTA_D3a`CZi4<Gn8SH7FW-?mQa<DJCaG3({8i!wDT
z8htWTb-ZEu%j?9E5WlH5w~E_$p8GO2@kw%)=8VaynUSkZ7W*!-Rr()hyZ0o&gJA3R
z-?|Sx-Ii@C&|UKJYEySe%tya@3offH`n=Y0g|WZ#{NUt^=bGIoE$>rto~yD}drI~4
zJin;Q%F|kJ79~lSXI-;0Esy<oyzAVBm4A*%YOC2@uh_`<PLWH|AhWNbcX5*bN{$;>
z%fCcREldnP-k0=XrPO9~v765(Pd(?e@`3FoUEW!bc%Mv5V+egGRG_APXWvdy?a$wK
z{9wsmGb4p*hqE}Z#9f2^KPOz3F}b&5PN;U_>7*%fmi!WXm-bk_GpT&Br(-RL)Lut6
z=DyHm_eXD2{_Y53<4tCh(7WQDbET4hcaU<n#g0VYH)$R2HZOOrm~!3u80%y<kIAd{
zoGhFlakY8bMdu#<JJW9bD7p6ho<+rsp5$dq`o-*2g)a%Ibhs_-xgGy}-nEhiD@;rL
z?WU}05J=cE*&*=V?78j7s^@-<PCt5T?xxUftBmdhOYclrbyRbS!L$5>$BusZC}|zd
zrfwq=FmLwK87X<SJYF%9Z4w+`Tqc<>>4>SHX+24C<tpRLD&~$yOi~nt+?So`dF`Vy
zb7q*)W~Myjxl0UrxpOzga;nW-FBF!XeLee5;MAm`pzufkG}88bHhALv>W;4UcF&^8
zKRB+4`mS1|_BCK~)KX8OGZ}|&xhdWJJ6SE`jz{M1wO{%lS4J+EOxkj|V;1|oP|=5F
zGMsx?Mzu$(1Z^qEziXGZu~qE)L}}yRzE8#JSu6JZUm%)%&1`n+JGI}6GG7_4|J|@m
zE0i~6>xw;7e3aCymvS7Rx*}))ro#qr{J1VJ&E2<FrOq?0_K3&<wI|CK-(YKfwOrrn
zk@=GPHDzz-g$CUIb)hoJwThE>=CP%U^@|pIC>4G4Kcz8AWRvE}WG&vKOQa_qJX~WQ
zD3|W|)#Z>xT#eSIlK~DwKIVI8Nr?D=JazH0MAt^$lngJeleyJvQYF8oh&CMMa#fAH
zzR}In($?@=;l#Vi3t2VKmD*0cns}%G(82>3Prll5rbA-Y&JCvxR_LT;&We~juk>EY
zl7f>Q`@^0t)t+rHap7dLmT*eul<OCBmVf;?%XuQxGB1y}DbHp;Nj(u>bz<_&2#Nlb
zi7$_5`TX>{&T`q%c6;%MQ@Q?!r!0GEbGSG$J>j8Kp=#Zh54YETO5~X4<DOc!Jbf|$
zK`Up=V3YP*o94?|lQu3tH?^dN&o=U2bEKZmYl|bROg9$9J{5VWylL*q$y$Ye=|8I;
zN$~pEM)I1-Zk0Dsl+Tm7tkWdZbi`-w<)o?$8P}$^JN^8zsp<Kt@2gKNT6N++({Y0w
zQT~;GgiD^ZDmTsWv#`Bbxy7jPn$?8uYo3^hRxxqPFqAijN5?W8b4}5a<WDh8PZQiQ
z@xsc-Q+)0{SSakH+`ludQvLshlWh&DbDlV}%k9ay-6VbB1*hALbIcc7(o+7-ad0w~
zJHdKQ#x9s4<m6OAbq2MKZ0!M!-R~cV<t@6YyjkM|>j%qL8vz%Is|vw4rX)<ad3eEf
zp6^}du7{6wt#=!r_!au8c<P#X<@4J<a7&3cPhOq!P0MoUBB>N^Z@$w>SJo%$bat_>
z-PNSrJXQJ!_nyK_s@;||>SSg_@kwf`HCGgFeS9Fx#cqOh(xsV6NyZXg9L7H;vhQtS
z^W|Yz48Nk0)5e~p($ULM!tzGZ(r|j^?tpx@GX+j3IZ7Aqc{};$9d%yL!pS+0p6p#x
zy()jziPbAVJNzqH^^dj1XuB$(Np$j!c|10%&4wyYS|Y8U-+D4n9;{_Ln%uVS*aUmt
zbOt*v%M)UnooYQyV&U%lt++%OG%kE_nbO3``TY2OzJD7g^Utwil)n)%Me|s8gt}7D
z!JJbs_%z*SUI~$?`28e~dq+!<?d7`gqQ(hXGpgt8;C4%!$QQ-G`NIR*wtK1Po}G+&
z@*|r0_53@bYUK-0K3Ub-XZhs)lUM317fy<uBHb+XKGf25LUoOXbWmF0(LI{06!|9G
ztPuLD*?rXc&f?Bc;nV6jjvckGdU8=>&-+JD<Wllqnzua8bFg_N;Ju>b#FLv(bfz2a
zRf-IdC<}6Dxo}C;ru%g05<wG%iaBSZ!zP%sYB(QcIC}N#x~`=1leb+vCM}t<S|j3$
z!zJyN>Y>jytE6f^UzvQ=-uQxSGpp*`ZC>wM9}9%9mTBp^H~ES8ED0U+iyo#ImA)VE
z>x^Tb9LK)rw%5cnvoEmuoDyEWIYsYjOhDu`>GhlKJ}fD{718Xp<CX8WmC<{`LVWw5
z>&`Q|mn?kBX<5X~niI~m?QWZHwcaEVFf-o2kSmsF&7Hy-(^6w4v=qK`wY3$lnY(rI
zBwt?DNUjIIIWI*+-QHvsZQS@MZ0e6Wf<;|Sw-sjWzHwts*F$-g-%0%j<(pr`+<E;b
zDP7QFy_V8^or=N~N4t$&&TEP~*Gf%T^)fXyVUb9k+I>Bf3rEC-=TA9l@X51`Ei?N5
zc227|kNW4<7##kpdskj((Z}r*gqLwQ%*}Z(U~O}R&wE)HM|HPuoz`r{rM8Q9PE}8A
z(VHW{&Xg6om1TY4;+rQEmEXuIrZ1n}puOv>mDcCwOdJ~7{n}?YcbBu-`*9a_D0`is
zCwPMWj>y;M@H*F>O*?A1E^sU~k(ehOS7lW6Dfs5R46pdoB}P4tSpxhLclH&8fBNNj
zcSeqb&7(AbrWxzlHy^(;?{i(}D<OmQa8_B}b5HNA;0RrJ!iHyoanhM5Zy!2O<W$x=
zJfZYdO`gS!NerKVtFoRraVlxs2ESX@9xjSDZ@7*#e{>RDQPkpil56c0(PnLJkAkoT
z6V7a5Z<!Xk-|JlB9^0dzPH`Fa{Cc%yO8=J|K3$iWCma<B);S_zS-hx8tT^pkn@=>G
zrGCu0%C5Vo=Cpp<uf=e5_mxjAp0R=*v6{NE8{6ge+8sF-yH+(T*&aG|CNc7P)6w)x
z6Q8Omi@MI9_+{@Oo3l(WVvQIJo}JEbY+^F@IU&S+v-Nn?!sJ@}CoAIWi#ad<F}-Fh
z6wvZ*z9QeH$!@x5{yt~fo9NB(PE7owlbc_T+qo4x;x42F>hRjMa#-A`S+u0!%;KoZ
zQ@UI7^KI7^JlWrHtcXeddHs|rYEHL#|ID4R{Mw$jqn#=04NoIG*GXSpI`R3eYf~N8
zZ9XD#!h)w+X|n7Rx8OuRtDw`NsS|{|&T(x^{J@mQd5!<nk{w<t%kHNw%I(nq;cT*7
z&FyC6mdz?pkCquSA3w}d!`%K(<fxTdr(+zm=5Ear-&3xh9zPdcG0}8()OaZrG9#N$
zPVC27nU2%@r1Y|Ug57iP{W^W5^wgD!9OdQ@@64U~%<3?c-OJ0?E42#P_D!hm51#o-
z#`=W(gb*Ry%ZfJxlC^d77DxJ~`|&C4X!Y0f&k8?X(Pcf`C%gUojF=0PPqtl;TpZ%i
zcrb>q+G@7oTV~guqLxYZ!9|@fZgM>dh}>AwaL?e>j1B3_v&7Z~G1tFTx$$87B?bME
zxW;U$XLnr`W_3pR2+L<4YAn6lW^u7#uM?jm(_YD05+)H5ysp~+{6ge@?A7tHFjm{K
zGxW;WIjVIznJyjQc%;{8e!sr5Xo{`c!K|gPB@S(DwVXV)uAyEx#s1{(zZMPq^`3;k
ztl$1FRIcKf%(IeRKCOrNH*P5nWWUyM$H94N=*E>AudQE8>s!gM=~>sB-del((XN0t
zt(H~XH}5!^O%DvbzVMSnsJzq5rFwmP<fj=t`lc$j!y$iqg~P#Rr*BJ{zsQl+d*~-t
z-8HvwO54Fdlg-_8I!#!A8K-qzk<E<z?|x6PQ`F^`<DSq1SA8~oXnHR+-{|OR?;{JO
zpQ>1;Zkd^o`_-^zhQw3`=EsS;tB&v9u;<<lm9GJXhD+lnu-PbmR!=PavT@0Z2d`$k
zH_Udqx%g7sNoB6Sq$*7|GeI%AZmt(LoIc^-<R+hOJvlW*!opG1>;RYVt@_00J-lJ_
z9#(3%-Jd#n!bT(R1m$_$P5MnwR<GgHV4JaW7i*kdsk6^niAmxijtBE?R3!VK&sGuc
zX8Tjg_#t%N>d?obOQlu_F)m?9@yOSj81u0sHuK1n9!t%IXS5Ewtm{*Awwb(dx{z0R
z&u2YH&!R1*H$zySvu+Dh>SCyQ^wn*9&|}xi`))##VwDvRe{D_(PL=nv+jWX>QNx0n
z?e0q%7VlVi>PDQ#dy#vmwUVZER8(mEu3q>+IryBXG2cwN15pzV)*H89pH$9vx?z6V
z&t1C@Iql5hESwTCrBv3a(X}QpSSC>M^Xpkbjz%_<iVL-vInH}FNtm99P2%1<v0z71
zlC+GLe}%-Pttyop7lawVH}p=KC=fc&x^;WfE{)2h%6A5CUBT*)51o+ne``N4VE(B&
zp92m_tKL2{<<=D5<16-<i5+avmJDBRxWs$s8J47VKP<0E2HeRO`mXYCrO@8vj>!&x
zr1?XO!l&*DRatN{f&DehsRhqPC(73C<8x3eUiR0*V@6{@iodG(!8rCvPWGii=cb%v
zi=5$hWdBBwwAS{|9D%1C)*9vJhGnmb*J%G3wYaC<*2q|?eA)W4w0Adh{38#Fho-0R
zTDg45tGT9Z)~bh<jvjvfYLVBbz-RX~&t!xz$<n$MD(b3m!RF+wfF0+4_sjnBw|uw$
z|JLhQEzW0IoJu#n5F+?$i&M0=NPvG=pXs~Rjz1qVy*k^oX!h3bXA6>DQYv(8*d8x<
zk=QRgRnh3V+XspKUF#OFZgl*4Wy1`)plt<;UhVCR&k1t&sBpP^q=X0<PH8scGFmph
z?A9Wc#{xO3AA_vcS_Pl^<$f_W{9&2K4-K)v6>qKhV&6}yi@mgd(UDzAcXuDHyjtsd
zd{bd?fXcEZ%jc~R;k+H1cx=mF(MgQAa*gU<PiVWavL%U2`qRv*tM?UOZ#QI^bkx^l
zXYeOoJ>HJ}bL0O1)Z~6$$|-7{wdIu5gpJjmS6*;e9elfT2IDE=wOdM7ICWY%u6Y0B
z_K!!?ly-k?_+g>oV$5#s`tFF!#LjJLo!ipbc1rDXsC$;$8@94J<OVa>UbmGy9q+u|
zVe(<NVUh%Mac(s8rwa}7FB<Nzxbw$3E`B|G(jKiVF1v%Syk66O%zAd~n%Ka_3-`LG
zPm0{5BGTbkyJ)#=-Nc2W?8--b{mdV4Y-;3{s$L*=<k7)|-c5FoIeD+l`IgmFKbw7a
za%h#ofm^$hv*miWxjt1fJ{)Y2u5-fY;~WL0`3LpVZN0hz-SQf>xOcgGZD#(*vbe4A
zt<V+qNyozPgvZQeS()z;G+(QR^9`p-=*$u|IrD{L^Jgq?30J!m=99t`WxC5)$LRfo
zeD~h2{c|0YdY1;J%7)1P>1OIRvh<c*d|zU-kfES$*+tI1Z(5exTDUYYUGTRO;?$_M
z{dXuYccRxFhnS8tdw09_a1_W2XFgF=z0{eoNLX>Tx{K=q&Mnf<t=z(XAL8+Bjq_No
zQaU3f<C>I;@}oJS6XH@|vz93Yn<)xjiBpng`gr7ILGp$;=Eo0NR1VzoKlbB@@73q7
zGY<xOz6vV#TPS*^T`4PySzXt9&q+^%o3341UJbUXcRDO`_uO15ttyu2BGj|NspY`Z
zM^m1?Tykv3sW8s?hL`hQJfBTV<;=2CQV=feQ1+3rOevYTw;)dEm1d)WWUBj(7cQ3)
zy;w|FexB2$wt_#kkCne&cDYc$S=Q5?ml>RP=om@z8%jv2Jz@=!7K?i1@@bux;BJk_
z&VeUhEn-=<!r!CuTA6E=SKi`1dY+LDEUB-RaukJjFIZ%y6q2HzD0HeR)@$X&pJ^AH
z#l){nUdCoC72>SgrgVxsdZ9wxiskS8S=m>&IBNy+2wNSxKB<Ytt=`q5^UBi1OncQY
z<(z+=C7k$QG<p4DXAgN8Iax`hy~)s5vu}A1<H9FfR9e4@i1cS^<^9^GG*hos!}MOT
zT77WT!;@>9x7ED*5YT$(YfWj5i+xMn83%0^p}_TPCY;Xn`tGN!vMO(J``1cIe!oJM
ziYBRBW_A9{?kIQ(uC(Gh$m9CAYKIe}_}!^d)AlSoy>CL?N{_{vQ{M3`ydL42;I4Rs
zQHE*F{J=BKw=^g4uMCyU>~<2`z~ji<qQv3D(x>>t=|#(icCA`H#SWdN*<wP0k5*i2
zJt`NK=A!)S>xQ<>izX$Z?`8WNqbH=@Va#w~UU9|9@$$6ic}nvB60SW=`6;s9eSwJ+
zK2GmcI{b@Q(TX|EFyQnnhZ3h}otwO-TwIhRD$U<9Cu6}Pi-6s?IW2kQtP(r@o-E=%
z|Dr|D=<0Ug7cN4c9_Mopb|+t*C~zd%Z?16FgjAKZ<=WT&TZt)(xSnu*_?O*RY3`y)
zlR^}4<{kU;^2*JySrJTYPY7yrPGb|4@5!6!x7{QCnE!beP9s5aH`7>AZbL5B^pzsL
zTC)q~HeKjZ3Rtwup;K#Vq091RWjh2D9i$(W7AoAku;Z}Zl%I<>WGHi8EOj$jv~laY
zUx9{4#eBaw9@i3aI*@ikHH0CXNh#dBOWN7pbOGB+iFcPo$``4A(P_Ik<Adw-tIIYY
zFxX)Hc(=wnbHyE^PxpJw_la??v@<H?b+8m(cfoJV+05=on-mHkl~@`wEe_z%oOU_*
z+q$6dLH}A`3nj7$KT2eB_wi`q+@kQ)VCLpq#+RastyUdhvm*Iw>FZtV+FCPbJP=pi
zwfMo=u2l*tEuC3DH<~?L%oF+4s;^CW9~j8XWoGDTe2s~{qpe%Ypmgo0LkeMrOMc92
zGHO|QVZ-(dY&xeuO=b;Q`1!;iQ&&rIuI*`a#P8~7CAb{S@G$7*VepkbG*5bkpR(+k
zOIr=MZwi>Oj7ehWvhP_<m9u=NO6PW8a0q2xsQ0JLcm9G*m)Ul8!PDk0^S_o7Hh*@4
zte@_(5<ROj+0UOjR!ln?o;){w-F%-f`Ahp}SokEZ4nEOcq!OL-;vuW>OTjR14L5<_
zI}slR3)cM(4(N5aN_8zxFDVtbP+IFganfRs@IZ;IiIvQrjjcydT$`0*$=x1!Cs_Q#
zQI;dFdj$13ID!H~!fV6tabG_i!>i?F7<4oEf6ft^o=<iOQndwki%(5k^h$E-#fzP<
zU#vKCM$h5G)CH2#{u6&K3W>}NiDYG6Cmt-Y#B9ckm5Uo5cm#J;tTjkC5`7?|&D6Jq
zbG4F^$)XJgZ41t_t`WXue5u#O$@Qkv&l3->2kSg>?3*Jho&U+<GhYRZkWrFD@(#Hq
z7we64+-7%NJD-@>%s#JF%KZk<1UJri9~?UvQ`I^(jTSF3Y!*JBxhF!3SGMEYfs(FZ
z+0-d1ZEHnCw|LAqI?=f{pq9ZqOLf|&e@2m-mz7guG&?KJxf-VPC8!FoQ(V=2UsTuB
zL0k03DWgu=h-E&RTV}25nrSW2pR91`@C29c*t81<O+O_vEE8Q^6l^A!9&$}Rr@7{a
z#a?F5#J_%D+O`|zr`k52;65W@VS8c4Ja3lcTT^~GG&vN=`!7kEJUQ)BR#uSTF~#={
z*N>E``xdD2yqR**&?#3b`I*bE>mGMzBz7=-e|J(PURd*=+9OqsDzjw=S<1}>&nF!^
zn`pH3G)uPI_tTwQ5``BUs7=!IF-R58Ti>y~%enP>{p0vBW;^HEpB;rKTzhf<e#iNe
z#qRcR&DtJJbgF3e@R?MesQhhF566+v3MQ9@pQ?DnUD$rQPt{@;IB4sTwn<3aAm%R%
zn}uXt*44x=?}D>MwmOI6G!nkJbr;EXG};JsHT>jGEQw=YmT06awN8DS;yxzr(hQr*
z%|Eu4r!SwtHratY*EOc?HA`Z`pN)RL?2Gg#KR7$(rnUUkFEwE;C#{#>@T{Ndsp*z>
z)#lVD70xAJ)OGzDs&9PV|48&hXz#DND(BdL@SlIVK0R@ULW|<g1-#;EN)t_YJ`oRh
zPjTXr_wKZvP!)5-@8Qe`lNQbHD)m|Y!D+VIBmGrpUh6rFieGrrlyCE4UgsnJr*Z`c
zo%GHu{hXDQ+<a-quk>4D3B9vdF`X5O{B+`-Ymkv&=@i@E(-F@e8kHMzR<D}&HDhnO
z*Q*&XcVxeA6EG1kTK3&A@`=X8Ei9+*e7!cgRl7GSLF7nJ*-GPIzUt3=Z6lo0pXPq*
z-LT{NMOMR|3hLrwp&1(Qjr?D>bWAw*_qgF_2Ik_Av)#TME;gPS=w$YbfBmf1)Y*Jf
z&963myb`%>RcTz!f!Pz=Cb)@ROpwhrQBO#j%xmdpu+F7gez9WXMW%y67P}1A^|PH8
zTI_M5f^l!EznjRV?9HmTR(ZUduDj#nvWXvNZ{AVj8l%%0;oNk?o7>UkB<nJjxgD<(
zE^*EJ{8wX!^wQkLHTg4Rm&W87$s1cmNH{A=ZZ?#5?vCJ$J+#zC^_+|P^W6RVA|ewl
zzogEyjw-G_B@^LjCC7DXVx`s!j|-C)6`q_Ce_;|&`h(=;56Slp7tHJvbu+Tr&~a$$
z=>WEe*Oi^DHXo3_ynnrsNYGQ>XyyY(SEqdJ@pKYB_GD5-Y~k#TnJU>P!S7TX!t^$r
z3cZlBtXYru&_%N=0?mFmms~%%g!7P<D#KL{htfqiW51v9)>7BcZBuh*on0a+oqK6Z
z$+>dJjdNy2KG*rT<5`Z#n)BQzo_$L6VVI`$<4Auh)8$k(wva9c#!9P)L3`W+x)z4U
zgyhXUeB%BB;gVG?Go?PRJSV$cIN%l2TB&Znd1f<@Xgrm?EGcqopM_Lkv&W_8%+hR)
zT(*ac9Iw1AI`n*}(vp^sGA8nRG52P2-VwE0lJ#UpZ|g+a9>>oCTLpu%T&DOqOytbm
zBzAm>^83q|9V~++mK@Dc@~f7}(vQgumP%ag_kn%yYz~?J)h#Jf{BvCmBZB*Qg2Mue
z;v8q46cWGT(b%98pJi#mtnny|VS!6Xn?_#PvqPy9f)38Q%xJwwuBT1xT<Nr_mC{#N
zpPK4W9uj-aRMS@fKgUkKl+Mow3T~`%-~6V9d+Du>YOGc}7a1fJAA7v^#mB$~Uf*wv
zR+a8`KCW5q!M=M<Lcs0C&JKFJR_r_Bayiv5#rW0hiB0QcuDO{leqW{DU4EhTTa)J*
z<HIp=?_)yz=0(2N_+P{~_l#T7j)K%ewa7VP(<(j0eoZ*qlrpnF<!8&&6WcdEQV{wh
znajOvC#Puks;ez+rVA6S8$E<~UCr3uvG~FoMT0DkB#!CAIZ<2Ec%CShM@n?wKDp>d
zq=>BhwAU>QGG@D8=zMqX^#u+O-q@6j%bFclJ0|Zx$Gql*gq?wLLc+68sXvU9!i^19
zALK52Ywo$Y<cR;FRg=ziX*?3Tt+ecGLY;B#t3_*0xc<7cNHhG&o6wZ29lvcI)~{j8
z(pc&6Byjewbuy9bws`n7@UpYq4Uo_YFVg+KLg9)Ix9@4E>t4akUaMZHa&fe2H{G1F
zX{`Wb@QTAq-0DuzPTN?*HMth8e!-b_QcR*iR!`VKd7bN&?gf4--JUuImHeE|st4BX
zzH_Hj@u9a#i2CFpfivl{a)Jxa6ed(|WP6jlR3~skqv*xwf&M$2B78rdC<?Zf`(ZP8
zhg|#WKc8&WWSQr$K0kNMN;Z=@TcTKI&B)O!oWt^I#w%07{<!ByORki<lo(b{?n-uF
zW3cX+n*;y+Jz3}8ym3(}lRcO?<7U|lhm?M|IJe(=i#JZc7{+tYzw^h>1atOk+Z{6s
z+)@fY^n6S1{&RU#cP~FvYja)HGTXb(>=q}c@f688n^a$5stMR7!&7hZqTEWNweo^S
zjqY>Ll5?5cTE!P+TAu4!8<?YE<2_gW<&P&tYm98W;)0L-d9Xcn<BW^5x(=9RO(^tw
z(h(H&=3HZrG4sxhgztP)@)?drf2(iaAk6Vd!NcIeHWPtEo_Z!CIt)r$${MlhJ#FFZ
z8rn>foftIgnmli@?eTRB*fOI?DqM2y>Yj-SzHQ-K8kqbR_uKt^TQGITyfu5c<er*L
zV`6iT+TO5>XU`HL(}k>8STwGQ%3Pk;<<?=$!}8{+M#JgP^-4k4mi)JIo@+4ebIatf
zOVZD@G@pxdo2=O@l<Z(2r*~|5!G#H*_!(lv8K#%(__!4-Jh(bN#zb&#Nn&=Z<rBv3
zhSEjP->*wpBOLL;MP>o#{hyLAuW~t0WBd|Ywd!D>(Vs$>;u-#$ALWl}ntT*r(EceR
z(9gd|cyb5l5|<53=T_((o;{Vt>~oUX&d}@{r-zJHcU)Q&0yF%!yk(nlDE-4v-?bm*
zJTz2_rk$*jzNfQh;^r57yo8LZCv401_YL~1Dn3oE*L~70+f9?yj2K%Eh*UOh@b-xO
z*RvwA^TFjMEP<S#OO%3dWD2?dF3qv>Vf5CC;rwSZkJr)Bxpj)zuGa^|vg+Qgn7{H<
zb|5>yh~~%k_NS9r4r|Tv`M>?s9-nP9L<P6@+OzTdin5$2{W^2+>#EnQu7|2u7H_zF
zs-6FycI^743m=?ySC6^OQX%;7P)Er=tIba8nLichG}Kn^NHAF6Sj774YN<@Pt@kJX
zOL`n0?jNRdMjIUr=+T|_<Kgu{2?=Mjj<wFrhYn^iP8JcHS}iJlrPk3`>_V`5oa=8E
zn}sVE%)9V4h0%+rcR|jViUm>z5B=3H^h{iRl6h0&!i+=jeax0GW0?D8?e--Wf-i4<
ztqys@Y&CI(V@=kHWm!UpXSr;N;<b{wST!}cwpl!a*SC~2d8=48|2e)KZ?-5N*FGbs
zqipgQB9?AG_2jJH6z=!&J8txvOxiG`=B$$8j-(BmX3?ULTzkA5?^yBkA1%yw*=l9D
z?i{NThaYcKP}kL)5l<FQSkr9alm6Fj#p>5e*&1A$2emZ9T3SNyt~hilBOsM&V@Igh
z1inAgdTFbzE-5O1ntf*Tg&Eg+Gw;s0#;ufL^D-`sb)n`3HlDw0PM=t_+PG<<_{#&g
zwId#6Ie1-J$^3Ag<Bx2PKT5OQtc*Rw#VyP_7WA$>ZF9*!kjFqNr{jp|!k8k*>I(<G
zVvLu>`#SvGD7b85gucI_nFQ~SC>HU#9$U{XkhIhamHk>T?&WsnjO#JZsjf?GJ}r6J
zZdMs6x_$CYgB@xK>)eDl8n*bk*G}I3;_#Ds+dOk6l$myLZ86LC%<8OM_HouFmrquK
zyELb)<DS~BbAcmF>Bh~~Y}<|<@cdEIl%S)~qIc(VSKqqVn^YD()9P_9Tj!q`5V!un
zw?n<r%SoEo4+lAvx*k`|Sod;8oaxQFnK=hPJd<@iutII|68-ZPXROmjqWWcL?KBjr
zyOVxkmB}Ze;1v_DhUZ);)${3}p59O<{CSn|^k54oSJ|8`M+1GFIP$o6o@(i9b1-Wx
znzQiSSAiMMWg8t$x+1oAH>{4CXDybM5>&EG_k-<^po{#9nPMwu8f-neOEx}gr|Og^
z%Qx9Ru$*;nbz1kgn*~o7Zmj)!!}Y3Fv{ut8DfzfB8E5%g8Q1Du`6{MkWwUGb!WmUn
zfvR@fKIeu0Pri|~U{0^8+1iV9tD4v@n)o$U^IXZ2eDdy0qKWpY&=ZT2Cahb0=exnD
zOW|p3KicOra?XDzc$(Rb^Nq6i)m;i!tc8m86K6lTsP<qpr*-zBOS5!tNSQ8n;LN=D
zb-}}k<yR}-9ZFuRdopNsO6!+XMTY02d>p=}IHVPwNuIw|$@q+S!_m~Ztv~l9t>gB(
z(RqZ~U-U+97qjTRT^Z_)1tpjGT^cyA?sS>Z{r7s=hC<6+VUE{vEFV-i3N;>|YSDG{
zR_2S;E!+N!+Z_*)Q+wgo5FE(Vq-6d~>5Rm6R)#dimLEzx6c+d!PJeME{m|1XI~Q<-
z>z~TmAeG@?rPB4}QIB=%(ffV-A4|sdL{2M6`Bb;%QpT#bD=&Ex11C&wPJCmjvdP6b
zDN)VZ__<=|?@KRU>a6=@w5VM4Q{cq(t&IgPH}*9(+*G`xn9><wc59B2T!dnB*olaf
z%1!RumM-|(RrTq{$3?xlAz|}k!>`JQN$onh`8db6X#OY6B1`stW^8U>waB1#&X(BC
z4rixS-TET(%w*Eixkii^TJ#EKgTE=}Gs*FA+`AAcTDGqB<YM;h_?8c$jBjj2<R&bN
zd5~AiSbgs9zAo++GZXmr6dnboUASjj)?F9&v~)V_BPI?;A0zA2R+GFVcHX*J%l3pR
z>ACa{?}l0IXR8Vdg<d5eQCr@=V)q@_%4wYHmvSy-2EFKc#lk+r@uH`frRFU*$5)Q;
zveeV#nm6jO8P5ncS7n!OsJkxut6ECf`r)LDArnL++CIpw6NytiqCVMv)=OKbiz^-0
z9eS{7%Y5Ueb-Y40cV@3zv1rPzZoWMkmnxPlH!--hXHn9_vy(C-H6kVo#$_ISzU4^s
z+fb%DOVfWAPtWRnwsP}Y^j7xGDT$dmpAK(av`*-Ga*Jw0<)16TZ4pyA)H0QMK1sAS
zFvk1FP4qkOw%$K7-{0uWWSMI$lC$@mT=c2?OmAvQ!cnIc7kWh{|0OWomOk)>&6q7h
z`Hp#ISY=x`$1x+xOF5U8R~u#hSR(s@g?H|WJyxDS8HA0R!WC6G&a5}|+@<s6#BAwx
zyb_NNHcm~GoBU@j(_>$+iKjvxJClk&SRGu*d0FX-ljT42r~};wI<b))iu(`mQ@@z0
zb}`Vj@Q3ixFu9}J`3!vu^Y=Uz`u=W7tU2eKfHs?q=ADNAVe@sqyjWxNW5Y?UqMv4r
z4ZgOXS(=<CGEBUOH=TQS(O7w2^Wu%czH?W*FoZjtXyK6W6tm=*ruD-sV;Xm+@xur`
zkN!BL*KHq+gzM6suCACSWb1ot>cxPFCf5Ce2@iYNLJMF0yTZ(*FS6?Uhg{L{tEO|W
zFDl;Kb-{`|WgYh<<Aa@BjF#9nJwM3FX+OuR@5ROo4#K}UUmrR5@-nAn+16X8w~XdS
z96$M+YrE<krPlc~znM+`pqy>k_S>K`QrBU1QRdD9owskN?EH1XR<1((vz*H}`Qut2
zeOxy)STdcSnwsIDD$KbcW2V2j`{jr=8+w=Riq!q2dn0$lx79rUi{D4;pAuhRb@avF
z*S?i}&gzH#@_ICrIHHr>l(kQBi%k^M^?tiXVP?-3MF$75Y!)|W1%=wDKSP+O7AS1k
zAjEm!tT~XWMR{V@lK8pp?T-$XIpv)?dsI<X>_qbQjW*?Vd`q&-CdpkARf;&YFy+zF
z>nR#itTnPlIVo(XzUHb<oKUlKg+##e!j9i9v*l#wuly)@EIat|a?a+@%ZjYtI~N3r
z_lT@({p7Y}V{npd41bvK_2^J$T`jMKyh4!$q0X=5t^DMkws;5!RvcN;ut(sejMP#)
z@#eRW-i1%A%v(KS$(#!f!F=)UEzT$6eUIl^1>Jc5YQmF~7M@mVpOn>>yVx18zC39K
zgO@AEbSc)>y4155rJjp?dFq&HWKk`!_H@x%9hHR+o0+a=*_fSGikxn^OHgBxUP)t+
zF&BsY%vBMUB{vr~PN@uV3D-%UF1xvV#qNx)-R0$P#VqDuWZtU5?m0Po>B+y7<a(yQ
z)=1jBbJ3fFqL-MT70HO^s8?t{Pnl-DxK)19A4X9YZ#7P%`b5S>NsOIrNlF==d|4Nk
za+(PRxjGzr+Pr*cQ}UeE3j?n@JkUHevm<9m%;T7Kh39Imc2D4$F09lL%4p|&e3?4e
z+eM|*xw#5=p4oXv#&(ae>DkTPIfb2pdd2E#vI#qv6jiZ`+dUOCH&JU^q4Pi|$z^Jb
zweYP*Hq&hOvsZi#oXxmCrm0ChacYj3WFg&~W3;tuX54n|%sE=0)_hvCLvH1wDx;%4
zr<%_MXwAQIh4)_bC9`1FS=xtF6c6fXZR%k?a@A2RCU1U;T!i?Oh8LUi3wPRGTG4k&
z{50pyRl6BOPwbk=*;uGl;_JClKJtynJEN}GMj5BNf_HM2OuVYqrJC*@;Kge7tTr)B
z_rNlS`5bN+L%gl_O-fm$D(cMG%~ZWg{q1HJlhnCgQfhNPDe3M^X$zen9-&peX~vcq
zmxe>{_pLbGQgz65!luP)3;(bgFO*zUR=4j^|IQDyd^YV>Zp{A0w&$@^HIrb|^z`6A
zYt=%H1K93Pcq{x_$!y)JQXUEQ4V%Rd^iGhfnY%aLG-v@wP&?O=WSe~vA02uZr5@Dt
zwdJ_m#ItqA$&Iq}ZQ>TPtWf4y`LfeU?y%1BuLnv?;=>Xn;&!atxwyvmjEDcsy<IEZ
z^HgPK3(Wa2p-hzLS2u6qlg(W*k8c!~SY`^ztm{c*pWrT?sV;G(c8Zmu=FcqgKfzyb
z1s9#^Sj^S8BfiA0)Q;Ky)q)#6hG8r`q5(_2vO7D>eWLeqm@O%|yv3?mR@<`cTB|0b
zZIilOa?b7;U&cQMo6P3AF;?q@^>qkL-Wq&<&WyE2!HJO)GHWCPm15pFR7e@@>)S7S
za&P#Or;HLs2kxaabjvuzXE>NHkdk9C31np7#caMM>SEl({6)T3w91vMoag&4ekoog
z!fQ2;Z-a=1085t5;<e%;r<#*ZBq!Gixg3$)IHSq$lJlf@^Z0*xp2`&VthqMdYfk?T
z24ka3>_VRyn*szI6SrvY=u9h^XYoN!R;Kq}pMb6U#v`5?1wlT`?q0jIvS`MRJriap
zEmPQ)<h!fK*Wv1h{fqAAslAz4K8MdiY3@UPYsLG^&TfBidHv>HKlU4b+3Wo3+!PGg
zsa;EdyybG2h0mtDK}~btCaUNxd9~-I#4#VE!$~4)kG`^m9#ZgP3yWSW7~#SD#yO*B
zgLIpUU*Z$xs!2!wv=>R}weI3CIiPZ<+q*F`Wb#}Qu3eA#JB7WOx`l=O|12=!t-n|%
zEWFG1<E!9{Au$u!W0`jHtKPhp$A4IA;-a0eRiYeKHnMfT^YScFJh|64@P9)(_p<^?
zPX@ir4=OoaL9tWkY_dyT@yKY7Qt*TYP2L9Yk2V*ac*C<n^`mm<nhRB{s<e(A`I*G!
zl#zHt#oRu}(Q{Uog5r7BT)_uLk@lA7q$W1`Sl(wz4Y;t~z~|C2)!9YCOziW7)dDu^
z35Hc1O#bxl+edMUO~L-`Cjv@mOR(u!_f$<hQLWv0S*2i>kI}(}7k*CXWtURQXuGs;
zNx-xJPn)k^G>H3L(Pl8G^2DQKJC7Evy5CyfQX6=6qxnNAUj`4+2NyXiE^KkNR#@lG
zoV9^<)1{CN?#@jnDK}hpYv*oh@X6>~GT|!!({q7+vZ_aBI(ZrKxA-%O_^q<gH(8pc
zWKx@Ns6X*PvzSuwajg*7e-+kCR=GPKo+x!gZ%*rno-Mojbp)2$l^eWr+>mu)azE$!
zUd?$q7a9vYdLxQgbM>xtcE9#$&D^HNN7i{4`-<=SP}v-#f9=-N{YML5c~l&Is{Q=a
z*2SmKWjXj&%wwvbII~}<SGdklQC#iE(IxWyKAML4JG;#eeNcZ{^J>nC7H!?PheggF
zcw%^bURK*K3+-?|z3VP6cQTbE`S*1tUD$DH#;!eUV>h0SU1cHuwfLw>`i_i$Cw{Ec
z`*TqF(iV%P>6ZP|ci1o5H9=r;SC{@F-KocYZYY+^w)#I&ic#WTal+HxR3NfpQo$*;
zKO6mI+ZS-|-0^XC$HhA~`pvJj?JJkrDP9&%aP<v(GC|K&GIKVMf`UStz&9SZAII-I
zY5km#yw*#g^VFgRIx{Qz7j)Isi*p2QI@ihgVV%%zBi9MR|HAxs`BZP5v$ZkxX2aUW
zZ~SYR1=gr#%W-Y_7t&L#YN(VkT~babor&YZpT*&8CWS6BDV>#CY;)iX!&iSXkquU7
zE=i@c&4}SYm$T<ZtZ)CLqvr~zY?~Z(S*7NK!u;U*iJzX!P3b)zP;<(v?e~3w3PVl<
zA2Y*>;!PTh1Bwra9g%2IY|>YoJ-;%4#WJ2Vg53959xa~XC)|DTma)@{RK-gs=gpt8
zm>3H0t}kF{+R3ZPE_puVFyCeA$Qv6pGIu|^VJ|Y(^WXhNQ+$88YFn+Ed*`a${6nHM
zM0s*+lS@JiD#9xEt@#rZ8Mitzb6dg^<?mKXvCe!A&U%ZTw}pBAX)T^Att-_rEpkKP
zlUFQW9MLK}mjrtIFtmP`+_q4Kb&FrupD8KQ%Fm*Ll>2;TCOS-Oe{<l;$!VNxPW1cE
zaAcYM$hLLHp&jD0D#ac>c<rF#p18b6T(smA)6<Kg2d%3_HqP`ppca(=(dSwXOOBEC
zgh+etG|eemM~nhJ)dOq;wLb;0Xt?%Gatu~1Q!W-}ZPhwb>0@#JahINb%Lkvi%hMUA
zFDv-B<(FWL(dq|6Gw&n>={Ou?X+QC#eZ!2D#zk|aG}Mo@iv@hL*Y%B$+?MT7(X#oB
zMSolCj|=8O#|m6+>NAVB<uJXFn#o|}z}p%>?|H?e5Z1yQ%32&3XI6dRAW-z`h{U1N
zz~g%Jwt7ciYjs|`G-~PHl_uGa@6wpej#PHMI>T_A^`A(iE!U@KpPm%Aa#v~uZ*wYI
zDAuc^UEbogf7&U{ILmJBWfrG}7?$3hXrKB`cXH-Yc7_jAHil${mCah^v)p3Uq?vuI
zrp@Xr+`EFYTl!*wiAn@#=KUwvW1hZ`VdcJZNF`*CLq(?H)2&AA+dMY%yxFc2pt*-B
zb5})RsKH9svI!5DD4$MLUYE2;>&*AfdLfRXvd8(RnLpIeUBi+PUT|wG^ZB(N+qO4r
zbO|u)s)%1HDEOtLd{T7MMaRv1^#orPJ&5v|`nGG1gY~kT8S7IzpZ(xjJ|X7kE|KHn
z+c%h>7fSoRdX}+}4fDav@)uvEyeKg;sgjghsGr=jiv82J!pV*<6$;)xr&t3tKX)Bk
zWUOU%_aB3k=$(R?lV`RZ)2@8?CYwn@{pohkm)Vml=S}%mpm6L8r^D8TT`hAYnomnS
zIPG@pT@&kkdG=$z=8J#wpPlz`p<Dc4&-1@d{^9+kuqNQGZ(W>%XK$Hx$zNlsCpRug
zzV|(L$;;%`!mWHw?;cJG&$N1Cw69R|@eZHlm}Q1~lO^(Yde$7XI=AA)YrO*}Q}+Zj
ztG+s~d|-;oLY3p6w-%(y$hu3-*ANYzRK{a4Y0)I(8Jo3pwtUI&oL!jo<93JdRH?lW
z4&@nN+z?bav%zfRl{34nSQ!O+X1cH>G;9cvxi;4(WLoZ;TSsKx{byCueCnG1Fi@|<
z@tgg!gsj!PTjN8fhO?h8VRD{p`}*as<8MB%EdE=XsTVAMK&>_8mDDG{5ZlHGl^?%!
zYB=gAP8KjzZd`66%d9c+^oG~EF?>^IUB9y>(C=yXl=7{|i#N(SEq!6#_C|Bnz0f`C
zq2hVQtlqpIb<URFzv5sX+cE92M!?F~jGLw;7d=|jHgVf*ahp9Co@nP31wB5Hp>K5Z
zrkXG7qBixdHp{rUE=)W;>*U$qhmBKQs+C<kX4!OnQQ_1(St@dW8p}<U%Nt*NOgh73
zag9aPv{KoOF*|9M4^N9{qNw7Y-8s3vCx7Q$aLKe0y}LqiSHQZ*ZZ&TIV)#?-^qf-J
zqzfL}zBzwowx4Y|*QZ&B+<clW`<e<xF5VVdZvAFTtY3s)y6#LJji+nXXX>fkH0cQ2
z8T7Jh(%I&q`<oiqNN<{y*;1OaLh1JugK)`p%D%C3%N{T`TrjU#{K9bEnKycs63?wp
z6?y%=e#gNvRHrcZdDpYITnFBMI;iyO&8(6gia$=U{93@sT2#oot=M&*^NDwiA?n<B
z9x_h76)ZTz`23-&l@Azx9+8f{D#4d5`t&Tr9PRfj>%xv6S=7Qgb%p1=urFVg3Q~(I
zl-;f{b1&-cy42#jWQxcCH8<Q%S_QxF>Fe6NieK&9t)pR&HBvY-VrHcJ)?U!uJYTae
zFfhh==Atz%CRWKeR_W}qyZqbaw7ThK9rIw`>BhY#M!h;q=AA5=%zH*zyE9VF>EMd@
zr+!Z~UTZQVtZeoR!+By38jij#5)zG0ed#N%&U<*_(CI9l)yG|@GjEf+K6&f*)0^l2
z+}pBuicW#sH_aXi-Gk~mx`(1Pn5JqZaIEK6H;mjSFg@V9sQS&6=?A*18hS!}#8g`P
zkA9lnW46uEM?GpO$JH~Dv#cMk)8iIhW)SPzIYH!+Zq_ORHO)QKeG@vlc3%2+@uBC+
zw*^;9Po8?mIL~JKw?~f0S1PuzQWSCXJYN|5cE_vJj9rcE+pc+p)%_AUc38ym;|j*Q
z7goITDW=?8mZ-#DxZ}~dg<CZ<)P0qaYOI-~+|&qH<#`u7(nID7e#%(o7_vm*RmBz^
zn<p!r{%lkz)8ligPSBh<|J1T?0l!RwtG1lB34Gz^@_1d%Z?6bv#!9)+1FakQwF~9`
z&+gOV7g?ArG?j1W?hmiU8}cSRs9?}Jut&eu(8^og;Y#b}l127^PrjP#T9&@0EW*Ff
zqkt*)d6DF)obx52$Fz7JUy<<Tv~slByizNC@t+G?(JN~F0(+N~C-}a6ZgXdT!7}kx
zjCn6w*~|)>)PkjYl(c#lM@&>)lg#y6qELjrA;HNrtuVp#1SiJ@g+9m4!n^<JA2Q|o
z$j`QG^_F$(OnlaUQC-{eL96knm&-n(L(2ub6?$dd7xG?GOjvqtjeF+q^|#hC`llQe
zny8Z4A?CY?_eNg98z)7U;42~@T0V3iVhWwGguCp-Po>>ct!?^!f4IyzlJ-zw;-Y-%
zFjv-xVh=N)h4N~cI;bf;Oo>i4o!RRDp&{_~k$$V*r`nv)XT|rLd#)*e)&8x=W{TSC
zgQYbW4rH8o;9+6Zx98kP$vK=nH9sC7xR9fL`Lx5MbB%vyGiw}h*mu@({qcjHt0v?y
zSaiAaam<{rW%)v(TIjcpXkfDsvjX4K4Z-Vn9hvcJnuuoYq2CFThZvd`@FhKUnDIet
z#)b3#OITv-MZ;p+Oph@q_bD3*eNX7@=1@O!C9C<L(`2jriYw($Fescnc{k&db@?UB
z@}S9PnIF21Ilnz}nc}OdDOWt{BWvx(L*M5r{eMvCacMdu=Vf)~n5jRc5AAqkq|@>u
zS8?I%tzI4ZfrU2CqRq*Htx6B~uSg2}X|{-w^MR{qs)Q3WSEr9>PQayq8ehbDeI~p3
zw&gwIHprdm`BOhU%J^hQ(ZSZ^9o}<z<wbTT?RcXs@hWuR&SfG;)b7WcOnA(8(Y<P$
zM}R>lgYS|PE?quNOh=-Zw(feQ>)jDj5)-^Z@YKa#5yt0J6j=pKTDM&^KfJTA_fG2b
z#z{+*o?NoLEwSaegu_L)$L_|N=d}1wcO1;z-rRCV;O)_#*)s|+?@LU&R%n03d)i}(
zhREp!%V)@5Ik9_AY~#^&i-r6re=Ir5ywIrR>0&i=#zw~ix0FR`ziiYuPMG~vh+9=o
z>u=-R#Y^0ODzfcdW}{Z$+J3}rr(s1zwpXdVP4CN)LvpT`UL`Znu4*{qwbR-p)>p{r
z<`3mHmS5hS`Ns8qqFdqGi2|qggv1?4__`-hO8Lt(o~+I1BDSn(^9stU3E`SN)m3_Z
znB1)?Z^Zv+-_#1@2wu4(X}a~#eDPl!IJyrNUbyFY^F=|d(HzaxMJG$IP0i;t5>Y<;
zRAk-~F{^6+iC(`dtGH7p$<6WZI90f==wYi;V1L<3%k1Nqr-Xfb*RkpV*T<ClxQm%a
z#$wtjYPUPtSMSpi$mrFZ<GKC>pGVpj8By2&kLvjogJjoe#eb7-Y0<tGrB(8}aLODt
zUhBK!t70r3T-2K9>Qz2@-n@OXMWH7{vmR*43;aB?fzfuMd*d>uNeoNZo4@>(d#t&P
zIniJN)1^1zefybCnVsqknr!Y>+rxHxN#D%I6PwI9WK`42G#_`ro}n5Zbms8!X)jMM
zS%0{tuih^8n3&~`N<jzTY0_tF%FY_gR4o5!=WnoFjPuD=zr7Y)_rI(>x_458I9t8-
zlouSH1#+w}<RT@$FSTHntax(&<(<CWo--D=T+=x6_UGd}NvpO!5!gGc<Z##2Q!7$;
zD5w@q<_S|>ytO1|I@6krAByXjlo(dd-7B;%BtvL#g`TI*q=Qw{T$Zm|W88dqhR~^q
zRr6y!>f4UXeD3@{m$SqF#VYN<b1G{lZuxGYROj<8?^Rw<X889CiCxyJs=i)<J$*gg
zj~-uKwe4(xs(#ePqaCGQp+!s;Cn6cDC)uXTiLPVa>hX-rQM*O|R2aY1KW(KlQS}hf
zwNH+36ug&tMdWn7!#fs!*52GHXN+1`IB|t(c!_E?slE?A6>Sg}VXn<T>0VokQy8P{
zI;G{JvsG2Jw?3?DSgjkb9p|)d@|orC!HLqxf<h}Dr@zrSspr|GZT0Hc=b-;$$Bb6a
z`uF;V%K57nul}pW?eKh;tvGG#f!m81O>d|?FBMxOvSpT--=jQ+1B-=Tt!mk*HF2HS
zhDAamt9f=scs>a`dTdeer})&(y=zTR*0<g+(Fr?o&ROngsDn_7^`@^q7Hi&chOU@8
z$K+bz$=3|iLxPS@E$d?6r*G7t$H^ZS)4J;It0`|@Xe5>_iD0(-p;zS4_>a?Y^{hkU
z2jX5ca(4SjA6X}P(@p6Fv$C9Tl=nl0%I<A{Ihq`WS9#7pc2cHta$m}+;ueLaLeoS(
zUlA3&S}^m&>eFWP8{K@ZZcNysWc24!?;*ikJAa60PBNOfD@1a^n$MbzURKB1Q@5){
z|5cls_}Fh(-=)>(l+G@CqO@3QqkXwjY`t5I+n=>Rq%$>Eu$An5U_Z%E^XmbY$ZK88
zOjcAJ$n%?HcqMP5tm|~vI`>aPM?IIAt@m7!ap{$5W6%ykR+W1`VJ_{Rt0c|4WRE1B
zm^LeM)l}<q5>E3B{WY$cbQ%Xg;ypjp<p4{wZXxTQx+<-+Y*%y)@*CzTu6*nn{pjNz
zMHb(-&1z3u4Ru?anIqopc40Ur&Tshjrry1IE8DKSURC%%Uo$)9-8BBE6?0QQy5Fkk
zFg`a?JA=oW%i4JV?vfWi@@YlBMaS%lYWZH2E{piN?y*9@zjjfi$f0VB&Y9}x?K(3X
z^?j1QZ+at?Xreo<luKpyqu$*X;V*+uu%+$%miBSt%w@s`PE(g1X%(!fc;L~SeCgeU
zi&O6Ls;zaItW%<K>EDFVb638m?cACg8Qd(ds=PXc-|Os*)VaM(2X{<jF3XHwVHzN$
zY4RZJ;v>%3=Sfqa{>lC%TsHH}t~G%Z)12!xBQ+I^r*3O9bDAk=ZS-@t&%BI}JLcAW
zzR-APi^LPBKJ&?rXP@M0?~L-<cIL~OnU%gqlWm`SrqxW6Oh5Ph%#&TKlTxjV_N`5s
z`_yY@<%%^IZ)Bu;9^ur9dfim$do53mMRL^yufClL|CJVfU0{|H^w#jClz-7dXBngO
zX@Wtj_h!tIeb~IHq`ibyyQtxFx=Wqpn%gVCJE*NY^u@9@uz-8By{J{3c-soyFZcBx
z>2_WE`>`qF#k$XWVNrkdSQ)bTJDzvz?qf*FlyUlep=8Rl151uR$h-e^qO|9UC+Bh>
zZLi?DW;NqSMgYIabMI~Jx~ozPUzI$38|0&Tzans7#R{bp+V&az8s}eWRVOU9pK)nk
zIII6UBjI&CF??b(V+xkYyj+)Ta6HJT#PFo=_1z|dPcHf7%n2$8S^8|nsfU46UoX~L
z>yvt~>8Lt;wDSknB};=k!zQu`ttl+npQ3kim;DOY3AcqTgBQCUJh6V0h5kfeCEm5C
zc!MXVE>>c8cQrWa_4<L=%$-J4&Oh?Ct_&+&Q1W7}%Gpq*+Nmym%Gt(#F;i{*MDMN=
zdbrx9X3pxygKg@S7ZtB<o1FOeiKlwyCBL|mGu@w+H?G>?>vD8qSxBJi307@y4d0A*
zj%JfB4{{A#Pj)#?b~?00;_?}0eK9c|os$z6={maJ`^Bhx^}!P9Ge<Z5GJP@ilBjK;
z-eI@nNl#X)#&t6<K4rK;+UWr=+p*?E)gLi0M5R{vFMfJp%4-SDt0{IDyx4E}ZMwST
z{c**F`oJyeF1@VYTI(;ipU%A^+H<4z?Cw2EwnqcktSp^7`SOxnJ}s5I3g=%1F<l4<
zxOUSs>_%b?`*iky4)#xL-BzbA*4%v2=f1@9(%2`AukU0eWNc|Pi+Ut&+UBNr`bCFu
z&-*CjBVksRL4Ve|L`TWY?sGl9W|r@twTmQXDs2_~lGpWqolGT*YR(BBzC-&B3XfhW
zvplwQhi6d_)Au<!OFq95Y+Pl%(`Q$(<we`{j_7oWlJsvUk7`fyRO!?HofY2ZuRc%S
zufe)#qIs|AZVt7=-^<!2PB_#3BJ#!+PPV{uz1JS>Y-WLWNn0)l9w?kXE6OavXK~n3
zH!k+j5X)Z-s<XCuT%6k4bGUzH_v*`mcAk3eOWWJsryk|q*m>Jg``znZ2NI0p6B+sU
z6|HJIcle;zoFgnulkJzRj}-2j_;A*C<+&~=MBJG3X3U<k#bV0}kFvZCNiQbc)Y0~t
zb=1eGaI%lg!RPztKH$vb6IE?DKf&{E&(|bT&7?HfZvJltMythwZ|i>RX;(~{yhd7g
z;tX}e(?7B%t)Ih^AlZ^Adqihaf2Ba?WS6idx&I4u63k8%`54K?R-N+O*4MRu(OVfV
zpIJ*IcD&Vy{&UTFmTmcsl|nZkaeaU7v?*>zUBgGO!V@earMEcRF9|I>^?Gs8C2gnG
z=HJxob>@hFv`w1wB3k&GLhQlj-8PB4VzPHB^`F=(^gWjM#JlEgj$A^mU5~b;nP`Oh
z1+3DxY_mzU+G>_?G(}YDvtPJ|_QU4Z4T*}(Uj&}ZuG5~Ippquk$0RN~_l)hiO8K2`
zTFeu}jjl}9`8n(B<s){!vX6J|SmS52Uif`>QgHO;Km)~@D(dx9Pg)!_N}u~mZGGCO
z=q`t;p)EY&eAfL3rDiSI9CJrHTs0zJ?gSsFVv5^-Ez3gLRHjFpUKGsi+M(kRvrKVI
zWM{;L&`<H98m+5j&V_zTnED`cwLsj?s7;K8Wr;7fxdoK!|5hHop>%j_?)<smX9h5^
zcJ*Z~cv02j>pjU<M@s0m+xppl6E(NQYAh(`*PS6{a=ElyZpBjrHJ?|Wl1V>W&iWZO
z94XAy+$*uu&1BQZEt9`WKYCZ<7ChDK<WBygqUIGT={5<+-sfI%F?u@1FkbY)Q73V+
zjn5|U`JA%kf#>ZxM<Q$;*iKI6RKMc;bbZ;n=SyVPdd@q<;j9>^-EG<JAZ@fE;8mWh
zx7VkYleG^!{XU^0`l#_!$EuV`GY(y!dHo2ZiL|hYgfWAJ@@MH>_Fl#B;YYpr)@-y)
zD2mZ}u%K;6+bMp&Z;QF&vJ>45QZ5=yF-$+9rrcw>%tmHe41dc0w8YQZDKF>V^qcGG
zc$Z_=&ESoKjJsx@Y4`oM-S6Mz<DNZ!*BTu@%<#R;y>qtH(aMEUEH<Ko7PpVx{P_P$
z;GSt~CU>YOh0UHZ|G<SU7QNbvoHpJ^E-9I=cyp@ZzsJOhGlPB|&wS+ivt{$KBk_+S
zzxa#E&3=01?mR*BrjP?C*SWMYZj$aeGtpA}q*+KIcarIjvzH^>6K5}v12qYSWFGHZ
zSFU8fg+<g|<$Yz&HRsz!j~9M+sBjJ3p#J>@$2EP4r-B<!onY??VvP#Pixa$Ccflt(
ziZ3E%*9sne^}D^fv&^OWRP!EvIN+yaoOClutJ&XChVMl|W#D;<H7~g2*UuA4)V#P$
zK}51|*6NT1i=3S+6R)Sml`L5m7<jF#V3OjGmgu#Mu7@q%7@L&#*v|LqgDdM|x9WO(
zp75P`>zNn#cUA4+=;aBYxt#1*EaF!7(!87E6}e;ksrSKIR+aUq{%qo6<YfIi=hocV
z=Xcf~)(p8Fe_iiKOY-xIxOa1ocuRSktrU}t^H?{pcXze3QQIlepp`8v47RPk<D;}s
z%2#Wv=hYi*$}KTh%OVyvlx-C=XfsP&`K!afRPxQ@1$~!<XML$<*<rQ)kk*Y{+4MJy
zb$6{$_sq{&D<5lLI=A)r%0;CYzR6zvx0L0KOB7eRzpBTKmcV7_cKoyx@#Qgg@?OXE
zJa&aw{k#W~%7SNC`Oc^|J-n{;fzsS<(|!K!Fl+H#STJdm<9YXlBlE+XET6R9&3r1*
z5W)GnXH|aijF`pg6APIQq)tjZ#x3UlcI&~ckZ324g1Gso+~o7Rea@TB?^?M+Vc#YX
z38BmE-zB4bL^q{fGWoDVS2f$4(_T(8-}XtJv#_>Sd+a=Jy=XDLl|H#a%a&#4azu9Z
zEv#)c-<hrvv8ZRd(d+3ePV80t{#uK<YAv(*Bp)ZaLL1F9Mh#q>YZP=M-e}+Pe<7rt
zXLPzJF4~$m_esaaEyXRJecA30H)xzSaxCGj6ZqA2;Ljo3W~cj0W~eWT?Wnn|Bl}h0
z7u&<%j_=v4EJUjwimI-ESft^TvX`Suv3K1a2`+83tKw6?3+z^DbDS65B|ItVW?_Nq
z={qItGm0(m?e>`8Ef{xu*$IP}>oeSxP2Vor-}`#c#1-4L7Ohzwxn%0}xkCMGu5H(F
zmh#&5Q9v%nMaffU?vkw&y3#J)cKWn(hg`tP&|SMi8M9X(W~%dz&^MPkc_UQGkmr{3
z9}bg>V>?f*o{*w+Q>b^fgk->R!7tZB9e7Ja-$Y(_Evx>bAob!^(voUsM)^s9+y00g
zxf$kmaaGK`&}|7@rZ!A*n#9$nsl=4@Ds#DHPnogmN&|KlAvyDo0_K7<i>7+*)Q<A3
zY}J0C_}<X*@XTY}53SQ8nv-vbuK((h-Q?(IpJVLdz^<95c6+|>o&O@={R{gA6@3<3
z`K;Kasn*dc;~=%*(mt<tyP$$M=RTS)TF}LA=V`$nS;Wdd|7B(5Q?Zk7ejMxzo{B_y
zw#|uK_;Fe1&2wB+=e9cesnu^<H|f{aqSz-b=X|W@WtQ%K5u1?fHs>tM)tM8lryD+2
z`w+?EJFUa(of>OJpWWpv1<%FBxiz<GCOzqT^0LY3nERHAo^e%^s^6t_TU#D?bUOAW
zy1D9%V@^-o!_*0OmooNT$f%JG+1S|{aw6*1r`-w+timg<%nO_JFhOdotX{9!%>90+
zmINry&A(CVuNo^l`NY%@v#KU4y^7UVPPy`H%B)qB*#twAJ~VhWKbL1WOf*`Ny(FsF
zR7Bv(!VX0)iAxb%CY{@<Aif|k)OZaSyUJOvNk$<Ktdq64BnuVQ^gNETZnRl3X-a6g
z=)d-^vH*We%`m;?UbX*}ET_6G?Of*K$3Icw)6@$dn%-9u&KMk7c(#W(DWY`m`!Dyn
zINP3X_0SKxWFXG_>5{;Y6R}HlC;Bq)di&r~xPF_Jv#+3YfWQ;8SC4yNn)@1U`oi9$
zoo1sI&my?R!<Xynv>j`d^0?|Yztr?!zKqpx-?NK5E_5%+ked~us8_;j5q{z&N3!4N
zoP&vx8R663F|Vpzm#@9?-sAelSG^Z1mONVXBtdD~n<Wl8UQ(tfi+DKaPhMR(fqACu
zCf%oY?|M2Ocxv0|DC(_mFF0{#Zd~FQcJ-eRHc8BR9s4QpI@hBn39Ewp@9$jRt)k6o
zpr_3$KFPy>*{T~&p)ysWlMZQ3bqU>cNbAOKE^&8eSMAkDyn<O9XT>GWJ<=5PQ|8dk
zDJv##ai1PxA=0_SDMmx_{Hg~>wJIjAUY`9i;@ZXe*1DxjmaW`UE8+FQ!oX>=RnN&a
zGd$xwFP!0%<Sh~{$TR-CDgUg2`D35V7DpYapQrZT-Sk|q-1OaQy`@T`E!K8Bn`|dp
zryKqBIaz)7t=EkeU6aK-R9-7ge#<{Ga<A>78|t|p3s=uiys6|@ot5t=)HL~7?ke%g
zdu04lH(a#nJv=AbFW1tBYe{)z=m{A{2N5Tu?z28~ZcR2WDf090{5gk7XtVs2TkC!r
z%zN_MBT7^7{PR?q+sbt!haGr5e>7*cp4^+xr9c1B>7)M|6jl0J!_61ldM;wDShPA#
zXw@CX?sY2_70tMd-+FF#ys*;9mwnqjmgCFx+&U*W=oze&jppK%<193iYM<hAPW&W`
z+vYXP6&JE4D<28?$gg`y@sxA(KW&z-6CX~zzw=~K-opwx$-Baq{n{3V*8;Ds_vT-^
zq(ktr-N(m8s$EytT-np~%|27-kbicQ!;(aSRhL?-Jsv8A#qQP=RTXkw?Q+>_>G2u<
zu8Ta@?R^<^CwV`ELZ;F2qMw`{+V@^=Y!6x5d^uIHd|~1oH{U4HkFzZ%Ik*QYoxN}(
zS|M6|avw{x$TVd!j|#zctLOID_^It~*>^go-y?F;y`Z9(oBAg2kh0Y7E$*H)#Ut|U
zrZcC6%C-bQK4B`eYCXgJ9|x`6B{$m(Up_g-YS)5EYEDy3oAj1URTOYd_#ktzw~Ehx
zlhDO-CuKEjS5;lv^Rl5895%OSu57trsp;dg$5-h-ONYtnr%~<SFFaXkSgB@jyR9tp
zYQh=zD}l{R_Gfa2Tb$s1WW;$-in(!~x{{!cKmwPZ&K6mLPfo8Jc#74I3ZE{Rbh4y+
zBC~-mcSc}?cjUjzl45a5uM*l19Qxwp|Az4uk5-n@tvPQNta>mrZsp@2Uo=i#%+R{M
zGHsb{Ugqu@dAk?=`r@givAx0XuZKw6qV-!JOQxtkmZ@^Mz1)7^gcy&QhT~bg4HrC>
zsWSLz9_jRMl}TI)t9Me;+Bxffzgd2Gwzch4<DC6h4orBje)w?Wo8nTJS>>fEZy8^7
zY->EG;nS~ae9CXDLvp61UU}b*ulrtvENl2A0ou2xapIe}ZHWI#o=sOYSG}0{>i(hN
zIj2nPgs=ShF?aF)bNw7lruXkGn>pXlI&|eap8V<sCeoW8vfA0b`c{}2^z`@@jqSTT
z8E(I0+mln)91>AGM{@P+A1upD&5e@gP4;RGa=-L@+3mN-Zr5<V>8^Y=N8NRDUeOZM
zEva|rb#K@%=07R1Ep17vu1c`UF<Y6bvisI+`_0ad4L!E!kXjkv^&P$QMC7Y?T*`gY
z_(1l#H;0}7PPuO@?`pr_IBj9#|A3#5Gx~~UpYroB6WQr4y!}Re*W_IxwHoK2A4!pF
zwK)CQCi|t$;>v@icE2jwKEAb({Tf$(&ue$*pZDLbUrGz9&!76)xa9amcQvs`ftv5w
z96#G`KW6xJ?fq#{J{=L$=VUi?KM;JPTO1f}ZGV`X^}-Z>E{DUHyYHBqEer5D=lk?k
z_`ZoBT2sz;AJSiSqwk^jgj~BVZx$t)Ox?PAIk!Y+xVu2|*B#m4Z<@AVFNt)j{QYTd
zvg^bg`$u!MZ%^I-X_o5W9G#51rHz7{66e^yF0<F&9-R7f&$KDmeoa!4=_s=H>b`pC
zx$*sbQo>)X-ha=(&iHNadpo^dA0j5da^C*><6Y$?CD*>yRd;9ibq2jsm6Bw9(!0xR
zdmX2{k9y<IU7ho1B+Q+D`--@JeQo-w_uC^sZaTqcy?4I%vc!j`?xzj1EOuQy^>>Q&
z-FtUTobz-a$4=e9=;rb7Z)5wf)=bo`dUGOl_x8JCR<a%6lD+M%pEiA7+^-#U{-aaw
zyygA<lj}CSs(kkP#CO)%BW-tEX=UVtWs7u_CcJh`dfa=$A@o_<{<0ZW+Oo6rT5_|W
zmd*B?dR+b8-yFZR#E*Ix^p@QI`7>wo^i#Q);*AaGY9IK#=ezZNIhpyqY)O@gcI@`6
z1YaEFx+8V%gdXQ?7u~zlmY)3eZ&vi%HG3_#KlV)edt#C5g=NK}?4qjOn>JsOcj3Ez
z=--{mzt1!2AL9;weJgMIeDjB)g>&_SqV034*{6PtPV5c-aF~0x4fpH6>q?9J4&Tb?
z;e0XY=Ohb(oYVaYIdlG&m1Q$NKVFlzVsUci`P<KB%EQ8xuWvDadhAQSR`p^fM{yB`
z{r6nu4B|i6IG&LbYWkb%Db{@a`|W8h`R}&uvyZ+MEBVJw-)j3ycN5)0PK~3Ermmb_
zPvp0zM#`@|^EohUweQB7GQMMTp5IJ9d#*U5IkrZ>wrBEMcefvlb(55jpW6Pr`7y60
z!@-_zcBg=V-#_-bHN{+hyVJ02#j6LN&mD^{&vo8XcrK!?&0AeP{@icHl^Zix*se=;
zVN|x$@D@A!wkErE-nZFyT?NHue1-)p?;H`hD5wAP-u+woP3+~Ga}V~vIXuJktj6r;
zzW(L=j$B^5O19P`nk&q2(!6}#wXz>2?$_)-UGUNUWzFlh90hxu>)lW4=63rpzV+so
zMV{&9%5OLH7ib#ZS}W{hRCYFP|3dE5N$(PhlX>5mn;z+iVqR>_qq6i+RrbnzKc2GQ
zxt<+4t<3Lod#mQBcGEInAN$kU_3HkU1THt3G#uJ}H+6gKuG$AG=arj2&;K(oY|4zK
zmC^o7@BEUjsaDf%lKdXH>z>%-M_XR+Oul~G+p)z)OL*5_wbPo8c1*Wt2u`$;xTiSl
zwYv459Y5;wZaz+$_aMK@M0QeK-t+LdmKLFF?%Q<a7qjZG3BDlv`PliKQ+ZvVbLAOl
z7yFBF2YYj9{k^ws{^uXZ=QgMGC4c^!?Ycyahbdf&T{}*Z_2;6lm#J&CEgec%PpH)U
ztY-YI|4za#PJZF8soB0~HeddF%<`bx;{=UG)0yfs?j$|GA$v&c!ndDCt6pu-JFa#9
z;F=$bcDYCLo^&(YZ8ux5euKU7(>CE{lm9(0JG}o^ZSjrIJGM?<{b7baS8e5*W5V|@
zzB6QbuaR*2AiG?(dxL<`wrOwp*(ST(pSkUmva#$lv93#tj<ScZUHs+N!v6PRd|M8G
zdnqC{?e$N^W%rM1-7SB<HC`qrQK2eHD$Ty}p6}5Zo*TSubuA}W`|pZ-kY)KJlku^N
z;O<qEuExnf3^Q+8{YX9AUEmY%PXkr+yNg55o4$Y9wj@t2YR&uu?BW_vzuq&n?3A0V
zecv-{m!PLc9G~0H`}cNzzrDR<!Tj%n+H;m!oo?K4qUBxHyFGKyS6TBm1}h0O-&;OC
zozvjjj9kBG_r555-MhI+X+reN*mcui$!Fa!Jk7ky-+%w<gRi!VA7uOe=ThMw!Bt;{
zeuu1a+28#@nsZ`h$S&bX!D{|jqMR0X%5}GRXC273+wCECt#6;l>5Il%(K>Rf5>xo@
z9q)ScRH8*?>)nIdcSDQ~XRh9GdWLDtB-QzEgmmV_e@u10wXpm&mvag8-+MN<f;Z%O
ziZ9t$xKwYa#Pyn{nLF=Th(2Sodvd-cc=OG*1w0vk@+r1_AI^Ng^zmu(p=-AdY^F?p
z&OBGmwBXgG{zpFbB_%Ouk6YYW`=$2CYhLB*I~UwcFDFfwdcM4Q<%g9MXaBvn;nyA4
zkE`ZK9gDKtlfU=1B!jffk?_y{@0to$SXuA=c6xHr`SQtG1*#n%tyz8*F9|h!wDybj
z_qzFe_?M~MFPki+J6Gq>R*`9cs<R5^Hf^_!X!DKT+0xIu<ymxOl+6-xy{>p0?PWI>
z&bq)NG-Y4p{)3F7KSe}-J$|<F$tT|a+7(M@_U(Q7;@D1Ip+CDycVFuduqe+inkfDL
z(JGD`5fO*>JT8^ra@^#Vq<po|2R(zI-=3MZUh3tYWW_fBh55FL+LfzUCciv4d3WrE
z-#>d@A1NpF`*6pbiD!zK%D4(y^~n59IPmUtjEY)u`YuQPZJZO_6LbP@G24H#PU63Q
zB{-(;IQxZ=KT_-F_~%_?S#cr8rvA<H&L-WL53X3$S?s&;r|9__*Km=IkCtw;^M5$a
z>fME!no_~cw#04GPm2W-VybkM>elLo?Yd*RXKAEOxt3*deZ>|V;f?R_ZuzqC*P}I;
z7|glrK3rhh(UcHd`d)Tsx!Ah?$p@m}Sna;<xY*?8Yu^&?)o1yob1pApy7jxtT5W@$
zyRCY!&))6pv_344m|)Y(-WeR8c7p%Ur;PN&kIl@JRga&`6lX7M_#^n^>8<i#?Z4%x
zFzHEe{A(x@@X6|b!TtB&<qM2=t!TaPCe3qvW%{1^`Ie3;d!F2WYxaKYdj`wBzwN^J
z9s2e;!S3q{5s!C!KUQvk+j%M2C+_Ek{M}}~zRy1I;S^mWt0q3Ld-VnOGwXWa{&VbU
z`*2*Y%FBs=`|Y><8GmIvPrVKMCdhdEk*MmykF4)gZ|@bCPQP#_x#e<^S*cjN@Wsg8
z^Cq;f@KauvTyD4NX8Tng@g?W&igre`&0&hYl^pAFb>Wwqg}Yh5d+l0%`n3ClpX>aV
z8@^-g$P5=bBk8e4M|X<%hXf%%DLI>c+s_^-Y`T$awQ+6D^ciBp0rzW`Pt3o)k7rkt
z?8HFH<bCs`uf~c_%J)hC()=ZEq2sxoN53kiozI*9=ds}iQN}C%H}lq>4L*BB<N5K}
zW%qZS6nngGy<GWZ`DZ12-c9G{QrvBS_jK&z@KbZykM-EZ)jC}NFmWSCi(+iX)q`y_
zQ*IncC@NUH+$t^aoNeyapfd)`*lZ;lg?Y?^CZ2z*|4^dn&isY5CL6}@`oVO3yG^O}
z@7EJl)3*F<PJXw2+3&sTJjW|z64p&=)?#Hpo4a{VnpXdgm2ajPDBItA)O@q)P5FBF
z{@;sV1eMf1do1$l7^n8x+m^w*=RAKL7aLM1J*_rl-cC_}xff=d+Z~rj+)r@+aqsQn
z&Cz@3YrB1FxBa(E;7_)yRnvoHeGT6o8*CrlS?Ve*eQxW|gDsl<*5@vrx_fBl{fYLb
zx|i2{Tv*iLYk2X%D;b`82|tbr9hZxg7jJAf&b`2S<xW3i`>h>~HfrC@?v~oUIGree
z$V7d)Mou}iP@Cf3bKbiRUolSLQmQj(ySH%B=jg&`_iVOY`6={1CeilvNsEgY{y#qF
zzH0HxETa{?qTB5XEb3kEeki`{{i3bPWy|4;wPl8z1mj=1X6vqzJ6HMKdRwLS^z$dy
z)#SLlT}`n5eze+sTY$p1+wUIlG*kFxq8Rb?xBZ1X#z$q>bZypCf5Mn`dS-@UX@`K%
zowXa{S+5Izp8YA$wXo>)*OMHTnMa$RW>4sjyPnB*GjL8_UTy3$^Q#k_ro_$hINJAG
z_^7&YpW#}wxr#zc4{AQPU7!D6Jms7I(RE7FN0-lB<f>UKmTY18>r(v1{{Qyem+r<s
zVGncUS-Mbly~OK|CBA`5e^Ml>79P0R+jr^w38kJ1-2QE`%<Clot?!j<yK}WiuDPi7
zEziL#Cv6P(ADz(rKOy7jLEHQTizC)ep4?J$X>0L1TL}ZfOZzUKFcbXXxggAG1#gnB
zNL+U0WOKFTMWJgxMur{xcfGFX>i7COxj*3__BZ_({j>f-J?nq%AMpqObN$0Aq^~h!
zUi|;x>ks^oXAL{@Z|#PI@4c>G`nX$uzx-p1naA_JpLfm;J@eJ%^hf4SrUey|b~Zb<
zuChsZFn#*B2FBNc6%V9LRCb-n;Mu`rU}`%v`Seefmb6RURR;yv1@4#{w#I1ctNU{5
z@)mv;EEmq+n5H!G|An9LO}BJEyL$cUyZ<h`mMzV3{Kpa#KTmpfJBQ`cPZw|6ltoMQ
zb9&D>lzqbO*nHKtzvVuYGgdEHKmAwb+%+mkmdah9u6k;p;9b3oZ;cjTJten1-S)Zn
z*Uu{Q>e?3`7Or-CE3xR;-}gV`8~-n4S}(ujSjwt7KG}jgsXCfY-KrOoZf*1B|Hu1&
zbM<9|`KM?7=iMxMWmO}`wxv3MD}xP{blsL@y4*7A{F&GPOKPKQ#Pm(Ue_yG1XU;tA
zvhl~&>^~j7O8UyPe(dDAQYv44q-+1SGhILOKYYHB;5a?W#qAA`!H%oR-`6)zEL_2H
zyz0)Wi)|Uai}ba9xx3GOO*b{$XnW@Bkvk%V|7XqAJro&sr2JFiYVrSFFF##oGQM`a
z=cYmO?6&!CO^xsSBfm?}+qy<5LD*sQ6&G7Ao<9aJ3vA!t?_c^S<!Wka;V165W!tLs
z<P<i3PGsHqtG_?{f#HR1ndkL`ZcJjja3IH6@_~_5{kxqh35EtqX|>|Azpvk$d*<gd
zlaCj2@9*jP?>6bvS7sj@pMU-J|9e?^{yg}U84?i5v*FWe^%YMh%BG&}VY{w2L-9e&
z&nMx=*|!uEzW-{Nv7}OX=4*B1qEFZ3_$+^1-2XK9pu1i+n^gGP@HJCCHz(~h{FNKR
z;9MD5F<)ku=>EBPD@?SXmI-sO(5lMWz4hk%ugBKj^WCAae8>46zh0^`Y&XrDyJu#;
z>tbV`ga6I_*8erR{iZwjUu)p(zq56Zx^JBJZN2UNTlp!U>+gwg)I0J1lEqX>XR{fF
z*>APZw6?G^{ZrX|{CDlIiyvOQzMiesq`fdD<$-t4F|%sJMn%R<^9?E`+uMp0rbL`~
zvu0bSa(U(@w;qkT3$I5T=vb~>bfik5>*fRIiQ3h+g3bxa^$#}6o9|e><q%8IF0OS)
zKJ=VSZ=CJ8ewy#6qKT(}`#uw!eDr_@=cI4d1{!9&(;Sn{#5OdYUY2_-ST=f_q8Ixi
zy@yI~oUgCvnykE`zFW}iQS+>%!=EirmubfKm%h0kQQoP)>(cCm)yG`DcfGi=@}i!O
zgzz552_lNwFL?f0arACICCKNIv+V8pcP=avi^6Yjy*SfI*D>UwnUL>s0YgK<1vji$
zCA;f(P1`W@*6Zcfd-9sH56?<_EpD&HcI3F8_2b{ak1@*JK46sYqIA|aO6h;!mYJ)s
zeq9tP%n?{|J8$9p_=KkUOKw$5oT|Gf_C@xIM653RyTBbh8Nbvdj-IZY!1CGYbwG)z
z-Go^+af`oCd6N7<AVTfs>}wTst_#f-y}Q}fciQ#h!}k|O%4hU#^r)QmY{AEat4%&T
zmc_i1T=_|(`Aq2i(~a+MCu};&r@UDuzS4BRS-i@`gO3g}gxbE@SRWbOost|`ZDX1m
zo}jKPX4_X*GfULpZEjqA{{5%F*GxQU&41+RanUEu%?^*8^WO*O>NbB-Ps-J)IPvJG
zN6m{9?yaFyZ~juSVd3JioVfnOVyAb8J*Sub?ff34F!7f5jLOv$ld3j8GF1B=e$sG}
z>OQ}l%HgxzR({!G{>Yt2Y|<_pM}I9f2D|js-xIewsy+@sn$Bdp=HmuYE|%mU5oR|Y
z9h~qyVfVq=Mm=V0SAM-XZ84);eyeB!@7+yep^R%RlFx|jn4VY_vL?S;@Ai|YliF?z
zn}`NFO$f@^B>f>LQNEfd<v#OHo23T9)5ST%bNj<z<(|JL$9yB1VYl6tBYSF%6wZFS
z^*CkOdAahYz6#4TWe+;uq@DNpZE0aTL9^CmtL>vq<$2$3?4K5YM_98+sLmnfnvK)4
zoGDSuw(pVLob)czaOW|>t8Us;ICV{)6lZO9syf1<abUk%RoC(Zm)l;mvfXs@6t)Xa
zJ9b1taHeJDk48mPH)%s*-;}0*m12=;nU7Y#R&w=PDe?73*29dgVL{0q6Err+vq>rb
z65h;|_u<!5sfmuAeEix5nR>6Rlwwwk=U(T|`Y3SlGWTyDqp4r>Jnkn{U-=|=%Gdk>
zpW?$OiyBpAcI=4fPHFD;D&DJ+ys#tn<`c0#pBIWUPd-Yv*R<WZu}muLTXtiPgy-DO
z5)DBi?v65^W!VAY-~Ctre)~#o*NH*_)$f-b-LBoxeBQRM%yyEbvV~1)mW;`)SyL2L
zM9(($Jy;v9!`u>o@lV_0o~jG$_NLlzeao1aY`_2I%GaO2MSN)1)abA(IXCy@UXGHb
z4;Cv#ew=K?-*fT$h4fAPs?Av%9c5&1?R=a)>F3h}Ei&A<uC^sHethbXX>_@-;gUh;
zGmVY=PBI+-b*F^QlCN1?E{m6E!@Kh$_hVd^1x~y0qVZHo?-7-yX?tzgY>r*}J^A^)
zz0c!jF0yj%YnRz563lro&_9P=Z;|nn(r-L++uudyUVpXQzvGPPbWdmAIqTOoK1tA+
z!N|kSq32L3uKD}=vxXT91H!j1Y<?V?yl>~E<6ni>dHs=iH+$W#-}CrC+;&sH`#pH?
zyLfT+BPkKPY}Rf2wz=riDM!9-J0$Li2OYItsdc(KHO_uw(rZnz9dDmGwy^CB+jmbi
zr9*z@3tmAvmJ$u-;Ivn|d#z`Rf7%{3zjd9w%Cfj7`Ojw$DzvpcklDXu#_Q$X8i_$6
zw}lUwn<mX#8oT7pb|q1z^KX+LHaGr%bHBN{+pkFC)8(TRKOXPD8XmfA;gwIjf+jAs
zO?{dWELnK(Lq=ES3nlX#oQq;Q<x9<**6wmDikWBRe=uG^{-@<5O}~rzNo@A9ELwTT
zjBVz9EU0)S%2t<KxchiNW3PtFHO)AV9;4m16TF(6=RA%*^{DK6^9uJ~^{1ycW(F*C
z3)a?7Gcj27q$)^w^_skC3TOH<Tvaqbnk_xDVf~3^yTslHJ&Ju$<1YCm&>|+mV(!%J
zn)gzRzw-7<EOXIty4d~oB2QG{ulwsucF&OYNWXE=)mk>#_|Zi}t4P;4ubPM>XBc_U
zCM-6sm#x|I_`Fwmue<eKrR&>+=S3A=UY8*4lYaPs607ApLB93POTWF<;p`7LnK|)j
z_YMy?uhn1wUeDWF<-_k`aU-qcVoc}W6}mGtA6SR8&Ny^gP|kPZj(HE6Z@cT%r@PG9
zx<)8@+JaNJ;skY>=QjFl>`GwqpYCjKcZ;iDMmC~H`;%zHKgDV0)d4Tockyx5%Ff__
z9n^TJD_$jLg`??QOV8Me?JumlR(IFW{lPN#<21GX|6DFU-P~WUuW~?Y&Y26v0{;co
zXWT#VVO8-fmgGr0j{M-`PcwLK60nW&-JRB#I$u*NvtQkFIl3xrbyBtAtJu&_7Nz@*
zV?;K~uA39a*<G9a+_%y;ApJu1T|pghPWG$%%+vcK8!eQ$ULTwpFvZ=IFW>Ft;hEhZ
z7>`%+b?mulczEO0hIKnXU8?F@I49(qMOn%o-i$kS<^onrr!lW(mTnRJIAgwq$^;v=
z-k=)IG~RbgW|En13yU5kgchxs<reu+>B487^a?+%9q*d?{C?^;i6pI$tBJk$S7OR>
z)36`+7IeLM@JdQUdGpplwF_DfW%nc2uHmqH`RF{?be-#Vdv@*QRDSc})tQEa7nlvZ
z#E!Z&+4gaWa&?^BkZ!MZ#6a`lFNV~2{0jV~Y?GE`IZT#Jub!3=kachMgr9d@_BeV(
z@^v=kbF&+=N|bKYD7_o}?!Ck|@9PYoPF6&8gk=9-#<<|?(kJmk3?3Tq6yj}uZEP#i
z_`V?CYLeLe^iJ;H%b_7z!a>Jl@~S1*yZxT0`QWFg*8`vICCe8qXu7;UPFmyDFRAX=
z?=OnhSWLJg=E-I5nYy*mdUjy`UdxxOl4d)cP`Jvl+3XQ_Be%5UjRVU&cN7&j?0L9^
zDLaUlk0<$m;=TCItSr+eel%Lz|01T5<x$TgrsZ+MUk^^`Iwcm<XmUt>-FCLQN0*%L
zl05wH5zpL{OO`i0`FhcaMMzzD1<!6a_4G697owJ5w{G}3V=n8X&f-7UxL8-V?%2Sl
z9(#M9@RH?^zf=WIi0TPV<xtSOWEv?{?bIWdu~$37tNo7Ct1jK!-|oaZ7YjzecX_uy
z*u0P1bN<y>|BL$Hw{f%F7yYxVNad%gNvo{EyhnNkyONIWc;D#R^YzfvIFHvg>9-1n
zVp^<X9zRm~<EeM`?F^-Nw#)oeOXTV}?D{{uhRikDeqo}_<r#-VB$W<2HnbVa?KeJi
z^w+7KERJU1KHR$W$u*&8XXp`T!LQ=mC-0mRYE-BhznbUf6aA=m{?@w*d!tUCzj0JM
z{_&dQybaC%tE4U0c$7}O9QUwPlxty-LXvUD%QxNEC;J|_pgAjWY0i;|FM@lM^xf1S
z>Mp!=GF<=dO!Ls9mX!fprlnug+d1jhvu-<wqq`d>%I|e&QJQc_wApEY_^%J@GM}!W
z_&cF*-Ro&3hfbvVKYbZj)3iER<y?i8(?taZ$9p#w)-IYOta9WtH}ev`otpV=2i*2N
zQHnP96JY=7d93p7UGJ7tDn&ciPCw=qQyWpuePQmJ9dkJ@cQ0Rcv`wqa<l6j2eA!Db
zGPjgREedVpJ<wv9eN0E^L~uoI*O`a$-wt~35%P?_tMNPb*Q8An!4Hlr7X|w(l&9F#
zPCAlqQ(OAjef#Gg{vVIGp8og8{+-@NZT9rA8;l!%v&y~@spGmA&z<0zp!90B>7M_$
zIJh1K#69);!ccmlSyl1x4Fl~e1~JK%55HVEb3pUQn|lh?il_dcvJ4Di=81aBpvM$q
z$kg`Zv-gKbC&G?coxgMQgw*GktK)g^-#7g${oH=%%QnV>gc{xKGhr9S;uHKl;$NNV
ze}DJZN7m@;3F`{eE{K=2ZkXx+_K@htJr)_+)o+$b+A1~`a_RirzFv-}%ScxC$HiAd
z$2KsjJMNQvSnc?zszl*%od53yis3&^)7g*3KTMg<aO~lm6Gp<D**=LYGo-69JT}_s
z_mg`@dFwM}F<Bqh^#>)CcX<{&-uoGts;0Zf^!1~ss@17UdrqG!dA2~btwv@h<K`L9
zA|G~5eDiHl>gPp{94RGh-&mUZNKU%$`G1n^)Cr!`XECKzI=3nr7<^PVn8L53&l|RM
z=3UdZ6F9<Ejw)1TPGGpJex}taTVu1xl(L}uNuoTfoPGrvE7pGeyy?X&zkY^irps0v
zD#eLuM`+ev)M<B8VLLWuwd@u{6;<ZK5?B95mSw50ecsQ#erDE_&#qc?S<KTC7qXO<
z-nQ8nn8tg|`sfr%rFkN`b0(`i3Obseq?E?J$zAfzpC*-QuA5n}Owv+d-gxD(htAOy
zXNKLk3?_P4xHLpHamSyX81qbEyQ4nyI)fme$u9ywy}kE}ooR*RA1k@As0p23uXXO8
z_6a&0_~`PF!lsk{j>iNa{kUaqq_QEo-|WJ<j%_TjW;%yBhaa#FV7aXDxN^aQ3p*U_
zq%WR7)biEv;L81pVT=)N+ibPGUBo0!HP*YjZ(Sf&F67lV;ov5fyKIs=VM>dCWVBrs
zQrx1oQn_V=k7(-Sd6Q0Ru65nYsWpRVQpnRAZ6eD8Uq47nVpY%4t#m(bB6RJJskwEv
zbK2Qo9Y6Oi5fTXJ@{vrEc=>X|D)+x^`Hycc6kBIk7H+g%YhT`C4l$K`4u_3W|9WkE
zt)Q!1btux;Dw?I>%#!biH6~tTW4!6j@wxSgS*BOht!)e4e)L*hzjEhyujaWO+hUG|
z?%p?X;uED!;jJ9gIv<8SU2^)#X2uV1bJqv3o%V8lGvQ$0)^{qq76@Ju@2u$6dHr4W
zj|x}ftFV(Ihmy=fITC+9s(O2?g)3rz^E0cAyk%0Yb5_i*&~$U`mHqD)cgOG3Be51e
z;TsS8ecrh35pWUdca{CI%I@8YTYJlv+dlKw6FF$KT+QRtJyC8~16?EWa_2PFU>56L
zoRdy3<)7rPv!3VZy2mEdSj4W_J16aUvF&+Pc9F0|>h?3I<~RxZypG-cX+?SAO4C4=
z-IHdB^ye~b>vc<>7C2y3xlZ7K-mS+fyKhC!O)(dq%6>Ihal>@)6vL1mVO{yFxEmNk
z$`flZ9J}KCf2xJ@Jp~mlyNaNLA(AEk5<3`guuPn?{Fw&RW=*9ZM*J^MsQ+~SrPa~=
z=kZ>_c@}~*B8{Xn;@k7ycg1!b+dAoE#L0%X<LkahT#-9(x@Z%>#5d0m+dn;NaPC=k
ze$iLn;Kfe4N;=zqPhMjt{&=#}i<TFzDRLj=dIg-~S{Yl)69xS_n{K#Xkx~{sesWRB
zzFz{GufxSV1RLgwzsr7Y%xikq>a*lWn~8lZbJ+BYb3V$(%DUb!x)=IDuVT)X<rfxq
zY20Gn9kXoF>k6JKh8m_%;ey|zBpHi@w`{7qkZAnic8byZ{K#)Mx~g__j;vo@m-$gL
z+<aSU+QZAH3%zaX_7>U7$3GT+aAdKrXZ)PZ(;+#f3c?RoJ%5~S{Yw9*_P#3-Og94a
z>Kj{1!&Vs0pRj+$%rhKZvJY5K1czvvOt_hR?VR407f#MzPiE-wtc{$NHOF{?dhI9I
zgS&0l=imNY<8wZyH+%h?aP|qAR?EC1nQmNtsoEg=a?h_x2e!n?6o1RAUQ&FgO*S}x
zja|sM>L<0#k=<OCz9}zP7+w=Ow8cxCY0dGj+1?!5lU5in*<c%19L&7xz!giiYuZyp
z#T^VBJH#GOp73^}y2t6VQbw;0N)D@?1r~o~kh*XD(7k%HNT=2NDP`P~jz|bd`Flk0
zcd75rm@}!wL+Ek$7dOW@zuLMrb)-|Gn<U;VW(z!CJ4v-sMvP^n6W`?@VOj@IJ=5V~
znKWf@Fkko1lNsN5<b77Ic_-=MsN=*f&AL&kZ`Q*W?r+TD87Y3=r=kq<#Tbnzczr24
z>9k;~w1DzPAD52aE0>(3pInr_kj9_IxXZUIa`E=n##8UuyBJ<fOW*UhVvnQ9;eQ*>
zd7Ap3KI32(nBn<+$!yO#${gxteOGht#V(Y}{krt!?1^P*o-Qi$W(MEjdo@$DFZOK2
z$B3J)(JJ?Oz1A=I+_huxb|cP|AZdrfZ7+AQY}#ZxZ^8+$3+l?cSJU#%ld{B3kG8M4
z-sY4)Gx=E4lDwTC7p+_xsPcHqM7I~WKKg`QUGX@#+A;Z*$>pYl%M~(pAH6rvmHl%s
z&|fK1u+36&Qf&5hhxnOE3+yEK&bYnxOA>1YOX%;_=BK~p2kD9}_C3uZs*=@xF5>#x
ztFygq_m_T4-KZ~cf^Sylf-N#8N#U(d4ol;FR0L-A-EPZXu+1S}O?2xewQ#e5GGVr^
z+YO<PD%xi(LOHZqCz?*M<yE{Q{`gv;Kx{_7kn1KVz8fb$iLSb6oO_KcV#ih+#mR+^
zleeuo$0DR~$$M+l#BY8K#sVj~3iZ`nc=%XV+Ky~bb3K1IM?*y{Al^@~-6L$?#3Lsh
zYne9|1l`}0x-<KiR9UCf))g-n`=~E8-d(}8bK{54LGKQo=($i9dBNhyH`n}Edqk33
z*?4zwzH7O~rv9^cUQ`dy60z6SRuehhOlMd${4n}gugb3K_{j0!>@?{;M<$x5Zq&am
zcG$5xUG>p>Q#<CUo<0N1`}1~RSo^kABx+;R-o${FVP+S<=X|{BIM<@owc3__XY-30
z{(UElYn0Bkh&{flF)8!+i8-oK3{ls`%^$1m6}qZ0N%ZNh1z|qYu@5~8#g&qr&ZSB^
z252ZV-@DP;ZO^#=^b>!Pw5{9;5l4>rt=0LGdb2g`U2dyH`{ZLQv_&MGW*DSy?iLqe
z{~OYkVUZEKG5oaAjBn=Nr&<JAKMLpB$SmKraOd%$TL$glCY@GTb?}nbsqmCp?7Zz>
zZ+7JroN8=*+4TK;hEj^Tgt$k_)L(zMmR)aN81CpHZnyftlBpcQuO}RTHE+W<IoU1s
zt8RvJ_pkdb`6%MVj@P%>wyCQ}U*mmq-d}2pAp3#r{}DGuyEb?|%Q>`XL)Ww^sy&)(
zjZG83O13#lCi9(P*Xs6S&1_QLHt9%!#Of=qg6}#Uo%H6PTD!zSV)bm*s7n!dxfS|9
zr@XqGqxEKAYg%@F!&bGAcjx|&JSd?ORVvEE^mW&XE&B}L9MTif6Z_(j^VjR)#_vB=
zCx&iJWu4h(+OIfK(6_BO@l4~<IeMLIv_hFbZ+hMP;FPd-na<6JRo7J~U;X>=yM~2`
zb?NDiy0XSaCsU5Rzc%&R`GjBt=2c95YxZ<97)`Tq)4Bg*U*Qg;ORl^>c@q3?UUNRY
zx&8XCqx}7>TW>U-(>(9^vNb#X?24rfDhE8i?a#~F-=aUs^@`-he6D!I%muX;Y#BG?
z+BbRRsY(Y<@coxnGv)K|-d(q6d;jX{&-Ad^lk0N4qd#&b^Xe@R&j{?F!F_sS(bA%Y
zVmTkHRt5iBC2}`htKf^`*&DA@-KI=bnZxZQ)-gwH!uB=}#uBdTkELGmrK^7^)_y$L
z%+~SMb2E>W!nL)9A(OZy)Vz3)=}zYQx6DnQS0Fh>=-caEm1-XGrP*=4DH~c76@OpK
z)PM22T9hk(*2M{DJSLp@IxF^mtL2I-#vlGH=v*zd;)B4=rRR8<Cp!sF>tk&B5PGsi
zKb-CU-omB{q177_mtEy9F_?H<YvuDB;%EF{v8eBUuFBr*rf1GjasBPhtdM5cy*?Sw
z9^@4F_$<{`nl5~Ft<>j?OP4&elTV3#oOOHJV`J45aRp6>U+FEm_IRi3`dw>X*W?5}
z-Qpr*TX-stdqGpseJ|VlF)Wj-7__(dO-RYJn`IK5S;XTa9Br{|@}_``#V=(hOW8W^
z=Ue3&8ZjqORP4^AkVK!ok7H`YHO(iTlR7jd%xFIQEy*c&J#50itz%o+Ep4JYxyXcb
z<Ld{Ux88LhS(ja<Va;=y-}#l*vAxT0)F;f&cz(M~{9((K3!mS{S<ZJ5(w=$OBYAb}
zj&13Bc}AhVIjp~J_RVNKS0p|0&eHEQOKMg{YbGksfA1ajTK5ds>JCPMsg6v+ok^4L
zaJ?|OaQkcGvPFy`22Up}Ok1<qA#0166yv_)b7$_(=G(bCZ*mFq)wzlFcfT~)X_m{z
zGnHKLn0%(v=9Xuf`oY|e*e!kfU#9M|dZc;(+#~6!eha<5D-4nxeAHeYT(@V>%amG$
z;!eBp4Npo}d*AW9^=Q#9J<SfC6>CK9EYjby+j{zzOK#5+1U*ZSy6r67cj3)J&ob3a
zi<5dwGET;BJjFA2hgq`v&9xT{ez=wz8ZFu7Te;%Iv^L!(#k)B2Km5Kh;Z0i9ul!Bk
zJol4gPfBg<>hYEgn?Lc~<<C3UWuD!-S?c+T6(<c<yJuz2diN-BvcN$}f1NKzD`w<R
zx#>_P_~r1?rCe|IkMli9{myb%W$~^XK8ug}JdRv(U%fSB%Z3G?U8frP@brEw{P*h5
zztYFmb%Jqw=5XXs6b!p)(GigKO`(UW_*v3ZUGvqmQqyYGX6Lx1aSFe){IKHuHp4e!
z=3<|hw`Fv%*mI$~ddB&($hajd7}FTbQ(8C6@iE`>dAZtFS@O%NsV<+G-Jab%G<Soo
zXv-3N(S6BB`Z+^oZMM&<+4N|7^q&tu4lX~Lb?4)*$`@xp?Al^jaHPP%%Jx7G!y-?2
zj<arB<_S5szfGGp*>B13F2_|IIjf_Y`RWaST&UaXd%dVPsXP6QuH6lnp0>BoraZno
z{mO~(y1gxT(k@&Lx4P4I_C(jj3-g3NdiB^jT;61#7JE@hO7n_WY-m)?t)@f)?`;PB
zXBZ+^PXAi+CY+T~#Dv)>Nh3yhMb(}!i@M%4i70J-<H;U<WE<;UsbGQMk=G^k)Yn^u
zu=m`*6I`KZbatLldJgx$2;r>{s|BXzRMe?HzPj$X@(jgHrd*z6js&m!{ns6zS^T&x
zV&f=Z_cQ2gHqQgzG99JsJN{Oroc}2Mef>TwqXPw{CCr|u4(iEiZz=Z;esQWTFY)x1
zbl;A{7nbNPa#<G5{AxqR-Ax@r_V@Zz4ZY4>bSjNdKH*h5Lu!G0#O>QBB%aTa|EMTt
zZnDwrn%kTO{Fgoc-MG5^_^)_|pGS`W_}IZG*?uIFN%C*D+8Lo}74_Ip5j_TLgl=Cm
zIxA@`{6ynu@uVjk%RZ)@Os<kv6<wP(XKKZ2pGuv5idz%kW)?PWoN#9UlZ_EKdOt;6
zNp@jhyTI|bn%-6pwre|r8<n@zEMDxW$F^1|QYdKk*2M}>Hg02d*}TzxWpkD5MaIT8
zo?8PnC&X^PVpDZlqyOZC$-5UV;}D9k+8df}*m6k7<i;i6xcjW1BIaDGkF<7J=1^wh
zrNwn;?x!1-`T8-1*Th3j1%oDr-d<vIB~kp~Mw|21sli{&G$(J(U66A0{PX4sTHE(b
zoRHT!d5Ou4J3{w8|0Q0(Wpe2fL(Cc$&4m{QtTsRMRw}>7@!L6V*Ds~gh~yJV%(hd7
zO-^VwNnH_RIqu-u`|r{nmnrkZZfkpJ{QbZ7=Dt_I#WknJ+*%wu!LL;%D|h<c-P=8$
zapW6bSnXnzq%zBIv)1j9$wm?<)ilLebT7Hhh?pMn$gkNsh=(KcEr-TJyOO<nSK_N5
zFUnY2)8ls1>&h`s24@MkS*`*;56vFmI<fS^B@u^LN0)6i@|`EU`drlE1v7VXEqqmR
zeWr`}3C;C&B2w%Maf_EYoeXhYe4^1j>Z3%;s_erXJolw_?U+z8>AI27d7TxUS2o|C
zwkoMN%`0@{F7+d;+}b@lLpRz6tjT(Ea-pup3iYXxuSyrjTy=0!|2fU=$&9b-Qaul9
zoqNJ_;ObTjaii&9>KkKJqquHz&h@-=C-zD(+va7qu7<ao>Scb5i??VmthIjgz{6m!
zSji)k*j;;XJePRtqNB=myfJRwPwR}JidvQOJe4HxlLg<L`?7Aoy-{q%F7o7(M3Iu%
zi^sn@OO>v4wY3^r%(0MQkkEU!O!HK&21i=MgM)i68cvVd{A8xc?52wL8_gMx#kq6+
znk3W0<G$U}y<(n`X0cRr9w%p~xOVboGkwPm8^Rlp6fnv}_l9qJ_$h4d#>NXX^ZofE
zm5-?FX-{yxzsjMy?4i!-{xy@?4)5CZ+WI#a$Mx+Y`<|#yZvWE7-rk|*eXQE+@2)+6
zW~5X+5<aUw-IV3$l3RDS*U#~Zs$FkU7`x)>t?lz-XIrW{+@B^{VU?h_B&#mSKwUHR
zy&!+OO{&MHj!QM){t8YvxvKB061Tr-lGUyc$-akgIlQ0$*1D48)aKXONBPUvD)hKs
z|Gbdt(?a)kfez=nX6`-HC;4lV?_oynZ+oh)t(bpWLi%iDkG^vD-=2e<_ut=n@^PMG
z?$_M1Y3@4?X|Fi^`INwfGH%hpJ?72<HRmO(4}DHFKDP8tQEF$TJGbxR#+L^V$m&l&
z)gixHd*;36-Z!|qm-3k}yF2;SEIx~qhOeikJL(#mur6zFG*x)!#$(d5cJ)rx+`@0n
zm4^!gre3)8ahmR~4H~~DZC@vL%Tg+rzwPkGm{?cUeZOvP3S-|8@LTxj4#ABZ&q^#g
z$7(gt#PZF;kC*!QoIbkcwc*<;p8WgD_mXR>x7Y{$>DU&2qCB?inEMvnS9&77{TVeb
zx0Aoux;-|Qnt$=z9ohL8fAkzn-(p*SJl(`@@kiA!N7mo{s`}>0(LME3|E`W>y}j?o
z_w^g=--a99?l-)>|68z5JnQ%5HSgkoYhGL5^xJS-{IT!Bf2t(!_ka9rG<(;Kc>Tw}
z4fjC=XYP@o{QdNkzf)@V`rPY3`8)Mfo!7oNwfD!L{549oUmBwy{4M!Z-ONq%gVs&I
z8FJ7pa)Yn<ni&1y@5!&~X6~9Fw0`=^--f08eeU|Nyng!1-x<5&m%UxS<#p<<I-lG6
zm%jzy`f8MGzwGVw%-@>VwYKk*-z>iKZD`(gqwRao<?5H**1o%LX<qf(+i`EUh3}3%
zefL|{_Pq~s*H`SGw&$&Be)YxMabL@Ht8c%)_wCU3y`SDj{n~bY-`k`4)n9LK{gb=C
z{x-`2bH)$d3_sKu_Hj1kGd&Pz{NT$_Bg}B0<$ylZgV|7tc(5|C#C{HtGO$FQAV?WV
z|K1%~M12|lC_zn~&G5q&YAQ%M)Kq-dy*@hWXtD&CrRKkfT1(`AN?zPlxh8PIOZV%A
zFDw=}yfs@K*zKnK-s$2x*TZ7G(f3;y<#r#lz4dMN;fp)mAIs(bez(Q_v0QKUmbbef
zTS}e3SaEOhM(cTtKg#qz&#?YyTV$hh+_}hR$r0meE2rokbv`Bcc@y_hXD^#2F1J(P
zI9;shJ(j&Cwzhp&ox|kKmdA?bUHQC3R{iv^mpe?)O|G1?r6eq??B1u(Pu@6dZ}&=9
zzjd>-?AyZL=%t&5|Gsgt|9#=Soesml9i?T9=GVU1-uNP2{nyRTx^Ibh*plYVl}J}V
zcC)=ux}&fR%#%Nst8naQXJuKEn|xW?%=ayCf3tI)iC{0*-zLeOc+Bd@*W*4t4}wlM
zO;nS~^zJp77e0$iEVjAh!}0s?zdqW3{JQJ9MYA(>*<zRIKK+}0`AEq6rrA2n_SNU^
znxVI<dcs}ba{WWq0r$GU#BKSmw2%Gk-V^Va?cc_^Ay(vu8Rs_N=3L=KVNt!53y&Yw
zNj`6{FgluKq8Vj#Vv+Xe1)XxrRUcd@@tR03*xs?}Q2Oy@#d|NQo!F+iDR)xN)2gMu
zQ*wh-Z>QWei@bOL)~VyM4}Kee`7UnoMyQAHSI~)X=Mt-*|B$xoeh_>~{Lkbs$G2Ec
z`#*I~=Klo7)ps^Koj!UX<jZD*O#+E4*yWnS*!FKuS1VJoIe1R?Lf)T_+_n}g*^Mfz
zc0>w2JM(KZbIOCPsJgUz-IOOq^F-RN$2dnVG<D#ex5M2th}T-`m>8@3vUbyp$IRoG
z6}~j%JX;mFz2ctU4O<P(kj(GL|9+fkVBn{=Hz{WIs}qM0*xubGx9)3a<D_{L%lQ9X
z-uon=ZCbJ%&%T=$Tx-R()&ExAc))FTEG6XEq(Ik-6+Jn->h_)z*lJU4kaT&M-r4Cv
zDO)DgKQHa_aa}Q=XXT%d3m+6oJiK|}xyIRbWgE8Znu%GnasS%8mB%%*{@c|9-)bs4
zS3b<-a`2zZ&BVL^?&Su(9)A9kkA{!W2HXEDF0it{$ra;sVP~Ap)))JlWe@2vS|4e6
z;~CktP@l;?B|q-r!-R=NFE(A=a#Y>QD^>62;^2gNdDH({Of0UhJ^VVZx^H>?)t>Iq
zYUjp(4&iPFe)liDX^{O>vu*Ed?H|YfeEr?9IK`y4U`qaS_BZ<Po-kEze5d^JulV<0
zKSN6X|8!?iKF#q->X5PI>h7hw=lew)9bdZI7%z0Ft8HAZ`f*?U)SoYJ@1Bvi?3dQ@
z{BtfZTi2hS8~^0jwRt<QOz`P?&(^MGc#(bo%cD)tEO%tGerC7#Wi^w?s<AM*GSPeu
z>&|71Jo0(lmEJ8mt{^6u`b+OkU-NNi9kXNZq0;=bHZPNY+jMYthnMKrSZB9fV<Qz`
zHnF>=6V~y{SFJ3{eRHSi<;f*kZ=My|Ub&KWzSHwU<k!#sTZ$|tb4{LQt=OM#IK%Lr
z1lQ)}MeB998Y+KP+UFo*@>Sb8nI}GgMegy{)#}%-zMjiD#Y1M+&SMF!A~kONY)lr`
za0y>n`1?3t)w9`iFK>KfW)R~4qT{itjO@nSWzUoO_iko>o57#F?cR&2$x$y)ElRsr
zd-~_D9lsvExOgh^*!n4^R>##(%sv0$`tj?hcwTO3GCZbqVYb+1?LCo8p3ZV@y&Dm!
zIPL!}WA|6>{?m#d9{IKB$j2Rr82_@a3asKezc}Xb-=M5_LGt;bets@$bIaGIix>8W
zK3%<DcSBOszSkF>Hcol;a<kv{ud{x<<K7f9IrMJ*;g9;_(|_A!YNo|)zPr4(`0I9U
z``LDR+TxNoo`(6q-d&rqa+l=xRY#eu=h-HS?%R`a*+Rd>IPBQ-`G3{tZ@cX}aTmMi
zDS6KW{<m+kR$cV|+QnbBw=!S9c}M<Q#){pmzb&+T%U$!EwP2aLshsv&rc2J2GJjm>
ziZ9sZwQk4q<jGS7Q&Mw`%q^_X^_v}&n|koMqUO;mg_fh?OC|}YC~RA&b4u+QtI&R5
z-yYMyujVNS8-JPm@ywAa6-M`2wL}(74Bip$B{6;LYL$Q+EvwhnDXYAys!P3RX<OfW
zdefbQ8*~*6*U8=Hkn724y}or}K3Dg%3=wNTwU>wE4$rxmwr*pC<mPE@Gj}!QPF_26
ztFIx?Y#!Th->>{H`8M<0Eaz{ZFPQJ;^4w?cbXZM9)7+Tz^-Yh;dmq>D*Pi*kM*s4E
zFXb8M<HcV}{g`jp#;J0@k!i!7d0xf8ezP4d>gvzCG5vV7RiWtj{@)xj2X?AP@JY2y
zh-#8Ax;$m$9s8VyX~#_kzc0(Zv$eLS`|!3T`6$+uGfxYroZ85f6kmV)nyU)W?XTsV
z^QWG-+nD*d&04hV)p^&&F2a+(=e#`4a3FO;=bC#ak3U{nUGeY2(Zx$YZQ_^nz52sV
z|9x1pa;>aq?%&5b84sBEOD>jrTy&st26H@r!;WVCh#%*V@7!B(bpO2*>E=9hJnK%o
z{F%7!o$r^b1KPJaPtVJrH1j&&ciZc{G6IU16~3|cJwK){At|xO=z~7T*-g@Keg=zi
zzFeHQ;^^V30`~BQ-zJ*LSEScVPN>^>yur}k=imIk7GBkVX&y#z7timze&LPcxk~v`
zX~nXCH4oMcD%6(*mrF&cE|17xT)*wwJrlD>#kV?^rWObPTCH$nxzLRl?T)pw6$N4W
zE03J!INPndT{2$$ruqF~2aDyl?~RW=`?2K9;RBQQsz=WIHv3Rz)9&Ww-lukZJ?qZj
zttVD0e(YMkcy!Z*Df2z#=9@iQ`F{GRE&F#hN2kpC5gX!g)R0-`&SO8bUDBKS=igjg
zQ~fEnbVvRc^9Aoi*;dO$^4(>cP`G;s6Z_$roi59s$Ep<e^D?w$#%w9ztyKCcu){03
zDVSrOx6Uu7hMZZv85`;=ZiV_BxS(0Ed;d9iCU>WRO>g^|<xageKKcJa+-nDp)2G>m
zWj^(-sk-->UF^~M?ZzjYA2$`=QBUDBy8gTJ{ZpoVyT5Df=SqiXN9+8^RE{&NEBo?n
z@tqIL4sUGRo8fI~Cg&o0X+dY;>@SV~3_gWdo6fi_%)R>C&0imS*YC;9%AYj9e81d*
zn7;<f@(%Kz`{s*vSsD1UC(Nj1<=MrsEqyBgqmQEU7v$5X&)uflwe4fR!V!kjWs@In
zUq9R9>5nhZSMmgV-Tl>;`(?%cLxMMMMt0mj(H6rn(@niz`R>wrQ#KV|7ryrG)cl0Y
z%kPQ0^~pTsvpzm|`OnAUpRRxWB6C#UXN7FVkK+e_|I@ChS#-U6UAQF&bCb~hU#}<6
zUwPobrF-j>Zp37%$uQ6V+I8jS$EEjm`eTxmo=NpDPZn+d6vw)L<N4i-BX#VZCb*w0
zD%pDDpU=_t`eB>)-Pp8Q)9`*d^K%!i-0<7qtH12*`TJD9gLB!IvjxBE_Q|hz-x{m=
zFZ8*?{G~-T-=n_GzHTe2wd<(jUzy0aAG7M&Z9bWcta!Zn=m+s{i_TqNS-<d@=CaD&
z%c4F{FU<e>Sf@JoRjSgzW7D%tpS?L=ulwip`FmxTZ2rhyH~VlaUDrWBf8NiJPW~J7
zb{}04`{UzQo9epFx8D>k`F@%6UR_1X{q;8=Zxu_Eea$=ZKZneh#m|)s!+)|>ndt2K
z7=OBF!K>qL7N1Nj*EL-~_iL)v#=KXyQ38)*xf;1HJ~mwUP%7AN<@>h2mnX#LvC7=J
zv{F6BV#o0bJSppU<g;~TR<Oq|<4E1jmbcCP{qFp-y%!HP9`i2V9Z@rDbvCz~`^Vti
zSw7ark6*gK_wV^L!}A1NtE~+8TDHG2j~Sz5OOFTb-!^}G|7L|U<)5h@Ek9g*{QvIB
z-OjHZ&b;MHQrVoY`+qj^t*cgjCuu3Y<b&;muVUN>XP7I@5ZR>WDr-^o;`q9hCw}jA
zX2fS-s1erTE@RmfSF4!o6S1J3Jw7pg^BvXv{O85}Z4<XIxc9fA^KDgD-CcRz8(nM?
zl8etQNuOqSxbZ~9osjlzKf8)^P9|P(N(%mO{-I~8#3ja<%6_?-z5h5f=EUFF)IRmo
zN749a_pUnpwK$=lx^C0kr_D9r7_@&swq4V6zHIU0^T(ekoi5(as<QmEaI&HD=VEjH
z9eHfa%j~AhOmyj=&v*UmHEYdhpFiI7&zAR8G<MPsJ3UWpze3Oz@y-{|<<%{C)0c>Q
zZxPHi$T<2fwP&;DsskPR;>y`6i~_pn6Zd6p&($%MNm<0rnh+~|^N{nEREbC5UMEX`
zFFIn$I_*{UUW08{<|*FVnaG`aX9aie&b9Gd&dj(|@io14zIEBcmzqZfcO0_KS+ZKB
zJXQ7NwuK*0tZT42s6N{)u1!tCP*&p7DGx>AWE~c7<{Xjt-$N{9l)q|qvHvMaoVMtT
zxy7g68Q(wi+kURwcz+@9jHka?HYWa;bl$RhzGLJwYwhJbS$$?dPPO0PHFr~S*U~S?
ztG@;v2rkIc|8Dd4{pYo9-_mML1MDWX=|6~i>$Cq`+`PD_=l$m&ZOJ$<Eom3^$joQ=
z?Y*ltEgARougNxzsynDR|GS>V4D&3*3gP-o2LhMvEAX|iJ*&U#a-7}7*y>xS3s^gh
z@AF*Gez>!`>Vbs{<IC516R%f^h6@#DRGVAor+!L5eXX`zs57wJZuaZCv&soqKVJO*
zXT|>_#_xr`7W{HeVXLEEzWm{utPmk&|6z96=VuZ1$@k~(N!=&?vijZ{mqmAV+dkiu
z?fY19KChxqv8%_--&wpR>z9H49+q?8*3Exnk#9b?=UKGcmD6nZ_QW(jGe4mq=@qs|
zCEI?1f&THJM1~_9F7B=Pb>4l?zd7$>Qu#mLdVTVp!H%=HUB4gIiFoop&;66ZvhF|U
zVj@+qo1gyUC@sFbzM<;pwfajw&NgvTGRN2J?>bZOeBMrF#ZLX&$4Bdf4p^APYsF2d
zj<(YiOE%npBg*&jm-I7A>(7Uj@lU+)?&q}AjhuHbpY+?;qZizKi0{RNfPgnO#s2#7
zqA%agd%mXhdd8mF+bpF%tiFDn$>D6de&>{z=eM6>NSf?bYw&3K`T6Iz{g{6FxMOw7
z?;M%<sFVLPkFPWOdHMXN?gEQ*Zbf(Jznpmc^pvzaTm6rGj?Jj~V|874t5;onnbE(g
zM>=lhcZocnZtu1;XVXURr`L~$dDWZ@dfc0+nD_1A#COZ@rX6@++;h6)=Z{NX7k@l4
z=dIlnVA>j%)!=KYe`xvlKf!SZrpFUPCx%rkhyK|!Q%d{E?bwxXqt0*5Rx>_5e{Zbf
z{Ou;+qkeAj@pkkUoaCH*^UqG^A2k&br{$MdXzO$D3iVynV<0QP?CyI0<m#E8_PL9D
zr+045_pO^EddA)O<@J!AwfrI9-@l(TU+l%tS1!jB?&*H1>Q#JSty5ipxBYj;%QJOp
z^XL1{pPtt%&SaaDpQiojZu;)Od6`fAPxllB-VFTyM`r!+vlpcGcK`c$^W7na_4bVa
z(so@txJ9D9vUb<cO^It(DBYJ`ANhFkne3N!zcwyk_Gz|mQlIa&XM1M!O!?;1rgmtb
z*{gqluK9C0Y)tgI$Q$4IHRRrfc^}d^o%bh(x2R3qe~i(q?{4)c?#uaNlQKW^sl9t2
ztKS=QU;S2~vV4VckjnMz_o~uoOzqA$`BeD#Uc&Bu4R80_8?T%y^K!-4AJ0q-3yZB^
zU8|M38yD9w=dQXI+s3l3`%+tHT=?R7=3;V_Rj7tcqOkJyjQ=lY$L;G8ow?h3Udqp?
z5VwE6=j1G_);@S+|M*SMo*Pw@-}gK^xU4qng;q`B!{eWuO4fy+tB&4UlRG{9;iPqy
zpDxvGoS(lzEa!UD{G$bXO^tOw9<_TB|H8Z`;^VY!Kc#1Uum6z~=H!z6%i_X?*?tN~
zwf<zyIU11|dCmLCkCHX(yMDg9_gvoO*!F4XZ#*)OwUW5>S}D`FZuKpd|5xr!?)i1J
z>}PM^`SaIKhb~*%zvq&_e9GB%H-FwrH%iOjo%7_(y^pip#ByuiU#+WWy#43FY$rFi
zkKQ*go?tlSdHL2>->BQa|68jEzrJPVKc)4|?BCbtuiJm=UVeY`T%NnL{T?R!yxNjA
zLH^Y;JHg!EH*r6s_TRjpU~K<(&)0_hb#2F9o8=t-wrStGw>N)mIQzM_eoxMn^UIHh
zzKHlLR`baE_`Hku=l(DL{@C~Z^L=dwYh?ahuRL*P=6UOx)44u=N!!}pyTs_t<e$k0
z4bJKBn|!wWzr!D~Nvj-HueZN9{i-&1W2D`kZz2^p#Sj0v-2e3Cde65no%s0n`v06_
zv3Fb8_n5z#^XL3gQMR8j|BqAm=Wm&hU;XyhRA=UDdho=dQ*ZL)C5H|-$+gHII_}8V
z_G#JHjn}vT`6pg*TjPrUp4apBA6$!<TzLQgjMs(9`qJAaS#JMLGPtYH_(e*t{@~`g
z_6adY4%Lbao=^VU-TH#D(dhWW(>eQ3^<KYNH2-)KXPBk&>rbzJzBrir%lg_y@7VD(
z|IF-~03Uvh&r^5Xg}AKe*!tXjc4s|fQq|6TMbq{5t$2@A-qc?6ru50|zpi5ay%l>d
zs<#%*v@PQDVhoA9a6RAu(Bf+Q=da7cnc7X3P3oK(wm$dc^7FsnunDWxFACybwsX(o
zw6Etc%z3TeGG%L=(he?x0PCA?M4h_l@80lZeT4PV{#ELR%ddZy+qK-?oQ-Qve5#u*
zi&@32dPS!{OTV06r)ah0b3gN^@)DKL3-e>6y?Z8P9xyHqkqUiX#rb4&Qm3j(#+386
zy35?!^t#x>1DtkFae8%Z!KITAkK_iJJ=~dj^!?6#vPLeQGCaN-Dpz8{*|il^M0W9o
z3R->t^lN79f`4wc70G<8tiQVs8nHF~*r3im>CvC)o&LpVtamlMc+R{tL%FJ7$Vc{^
z)H3nNK#|jO{3@;LKZ?KZoRZAaWfi#IZ<^TSzMN_PEj*P9>-#sIntC!V@n-B~Ha^R9
zy?g(BJiBVyL#~{jJcV<~Ps1(O8vXmRWcA0$`X2wQT>o0rvd-*|f9Wl2XtjIh&yT03
zbM@PP(KmlAy)Sb`qQU)^-bq0!JucBVjh9{Z<y^Pu_vLMMp})RW{Qt?M8+LPk))OV3
zSmv%h%cA7(u6KNQM=dOS?~zp<^(W%DJ$t@$>ep?Yk?-|Q`&e&2*OL(3`8ziIVYz16
zk99UhZ*=wbtxw4<4E+D&+_(BY2X^ecbR#~K$FH$K@FB<lw&lKgtGp(?n18}$|6|8}
z<@<M-ZRU6uxGE;Q{8|L}=D&<t-^|5k{I)*(abn2-zJ;^(*USH3aOBqpiTKkuOjq1@
zNSIvz(W1uaQI;o5<T(qz?`khr<ZV13XkzxSS;IN|DChK*Yq-=-A8Px{n;>rRKRT^v
z_Mso=%1XbUjpLXWd6WNV@BEKjcZYuY@Sc&Sr{?ASI}3Ys-v0g(fAG=2X^op7Ov?YU
zVE-B3+kZa2tbHPP`}ucy53%Tz6(>{fpFM51`R~!+H`5O9*D5YNRProni^%;oAH1jT
zNj+}<YsXo|h<z$;_I96sKG?lZ@$A#qmiNngbpFe;eA85}aayQzq|LlJZ;JAXCG9dB
zZycU<Z?EIdW1W+Zu!}Bh=~{p9px4ba%cq__CgIx6uXypa^{#L|w~t?IRqEHw|LXed
zU*E4YdTxp4UX@STe^2#KKRD}t$+~IRyYE-#^wvIHE`LAk_Ycc@J4e3CXBinaDN6hA
zKdyIXTwGDw{qav!N7L2lt7b>FCao4T(u_T^&r(k;=xY3~%YXevHiZ2@apnL1*aNo?
z8b`dIeP`=O$)zsae)h|Vw%$B_-=OT~>W2Mq*3>_IE!p~fw@aeP;(QNH;l}^nuKoYH
z{_N&z*_Qdxh~x7nt~EECKb-VG`pd<0W*_4x*3BP$N}l%yxOOs%@jiSLCaNyLQ?>V1
zz}BuENf%VsXkGC;*u|%Izw&jTfHsrZhRRSz{RxS4rZ=){^H>Rlw|KCh(7J!ONnDd-
z&AwL+7d1Qh_pS<Aaj0S2qg1VXwO1W2R_t08azrNlPDMf!=gif@%#95STS8g)No~+)
z3Ap>gsP^j13&)uIqe88GTEnOIDIPmr!jQ^QvTYU50rdq@tGODUHcppZeeOVc;E&81
zTa>o2or!1+5SYRsrD%V6=PI8X?>e8Fvo36Q51ZB8`i_bF{#eqfw!DX}cm1rCmnU{M
zoIG%|;bp=czB&B2ql82=E}oo`v*OJh&pW#VX8tprA=we`)A{)RW-mKkhUPhGB`kB_
z-&7Kd@wn)vW16ssb#~LYV|`-53VZi{zqI+F!Xxg|)A1TV#d4;0PCuA`A--$>p4Tap
zH{a`8B(X+i-(_nB?TUn-yXp^kY-*Z);Pd|~#qJ5r$EH^(A6h->t9h+ypUR|3s+?yg
z+<jB!wENg&shcmV{_T2gFWX|fKuAiw?aRjL+l#pB5?;P8y8kM$CFn=-+CM*;yaoFv
zEO*tvey`qp+TLpO|E{q{|7Dr~{O000$35%f`<>@M&YIOzc2;(Qt6^s2pEuI(wfV<&
zwqK9@$G`XW+5<+96J*+c|B7{!G33AJKV_=Vhs~W#W&f?Y#V!9|^5?oMZyorMe_e}w
z&dKSUt7lgK39A3cFWRv8W{`(Y(#nWzgBvkF4%~Wut?+G9VSds3zpicNxzpb(?ffg<
z_P{2y=JPS}_cD{uU5~%?y#Meizuw>`4exK!>~2qA)|@|=W;I(fCrx$sG0oeOIcZaD
zjhcOV&6X-wzes)&{lZu_{>kO?nKkF06syfQ{@HUf-9}B`=x5K$_j4!pEnED0Mrlg4
z%A6enhMl|m9?vrH?42cL8k`!udRdt7iYZtB_wM}kocqAN+<#4<9xi3*`BArpf!j`I
zb)WC?b&n5BY?Qjz+RFI&h1IHsr`n`EOO#*c+|ifoeX_rBcE*xZbI&-d@4kMrtYp%a
zb7^5m+~#bW3v%nroU{!y52mrrj=a}nk!sz1mdDKa#+d};=CeL#n?p`ooIS|;@W8Lh
zU)5jxUz>j``P!T3&ous-?(B;^z9{+X$6G#E4_si~meD1<InAy=>%x@<D;-y79k{Y6
zcK6#Y+droND-He?ADz%PN4d_z^Izs$b5H62<y;BOldj}^Pdcz)Jo@%6Uxn5m9j_lh
zf8&@Q+fw+qFz5c1-0zz=2+rTJvG$+O>+gO4vf2MevnH6V-}_O&isy#Zqt_o+{<PTe
zBcq<B{=2_(M(w4p+y7Z-v5T)_E$2B}z!LvFX;ZAvwr<AqwD^at*9~7s?31gk=+yPg
z(~!FV$e>WiDomDhw(SMcm<j2ZUwNHi$}iY@U#F2tMa$LQv1;kP>Dm(FcV<Y3KKf`I
z_kDX^)Md8!@6*5S&yuT)Jdjhf)9nAFIupH#;Ro;hd8&NbH}h=68>@SPu?3r~*vo%U
zY&w{k5&9+L#bGhV>2v(cR4N`WxMXg&Yk$Uz2Xns`M(n%G6|?WdulWleHe6*~UvI}#
ze{uHRdz<$BE>r#gYg0S_jr%{=KCt_Cp7+jLh8deyu60YY>%S#3%dAyww$tTh7q)1c
z<Z}HLH)q|Dy5dFF&es=X``=ZS<XgQu8SDS9>eGYC`6cC_HRi>Y9X-GM$v*D*wUu?5
zj4$409}K_7E;XT_HC`_Jt##|Ga<08EuV*j(^V!K>cJ*{7hD7d#EHazdrN)+D-ud`P
z1YgIViKUBIHm-Udd;My{#hw196GAuc$P)<-V0L}Ju5Ghi#3ZSL&@)*(S1dTLy`pgu
zyQMT+$9*N1a$&!#e}3&h9(etz>G^Jh|F=$LxB7i-^3>Y2<hXix{iJ!*yZVy!eE%AT
zZpcuOlDXKh<NTGlXGYKaJX&x6*8O>TyMlc7qwIS2WHvi7-FxP%&a3}3xw!lNrD~(c
zYR~@M%bYO(^Tr3u1Aok3_2~Wn%*!jxYuBwif8(-#UF-g345<>q8DI6|{^q|peLt3A
zU;jbBY2nje#$UZzpUqg1@LcTOw^d9fyT6nqI9>1LX8!8zRCqG>hwe@8-RIATwA|p|
zr`57n%P{VsdDQA~#W#*G_uTBalgkQylV{)TxM6!;5XbI2dAh&LqpGjwC%x|em%5L`
z?8RTFo%OmxNw@u`bnMfMxVN9@NAv!BuO8GdVsP4Y#y%<Nj>p8-<7_;)*ZWFrG{1I>
zxnjo@S-ra3Tjop7HjtE@$y4F;X})Jd$_$$sle3fkn3^Zc)opm{>3UgO@!ey4VWvM1
z>dKE@6Rb9FoqM*9!zd&svA)F6N3ZSVQG?2=r*Ho^t>@rK>dh&W-Q4GTe3k!%*?J|i
z?R_Gp8*V;W!PYIM6Ui&GHR6kIx0FuSh7|`Fv8|R`vy5$3)S6dGsm;Dw_N(}%x3>45
zI;?g#aq+fI!Ob^TI=uPAGxybDNU8Ab;!VGF>$!;HfXA#^Vd?8BZ=vPG<g-3z#y8I-
zZ<>3uOl|heoU~0dPo}BOW<0$&E9u#zrf1=&ti)s?B~u66l|`zuR~M{w3~pY==F2<F
zjr%p9cw1lAg{uozJEk|s3yWz=-%U#QcI18IR;a+XIJETiRo&~=X2IXT9SUakmA!mt
zqnPLysabZ*ULFqJr<cJtbJ>JUzL{zhGsR{uo0utO>mz*9WU^22Nt5Y5lTVt=_PKo0
zWUkyY&eU0f%QRDG1uyeVofWz)Gj&#2k3k#vt3}JEW#-LP3;*+eR_wZyUklFaU4K)A
zLOf|QV-+{q9kcngDT3H3x9;ZCS+Q_Jd|l<kS>pWHk2o|e^ISN~-E_v)gs|pF-ZWmN
zm5)VARqPHdmO7JVu=?O4HgBG!o%#*8_mm5|NY8q2mjC&0s&G)8uHOD-0h=wd*6rqr
zhzT>)i#d?G|52i-vZ`z#%iE384kwQPH>$|gy1$6s*LH&Gq?0lyQ~FbkJd7e$X6j5@
z<IbNlC1sPB(M;8uZzlP)d8(aOnVdT5q>1OEyl1Bl2J)UVd#QiV|35FQ&EE9v|E02j
z%NVsp{$CGy7&O1{d&O1%+mb0f?GM&G+@5yKKV;Sa1Ks`k45zL?xN|c)AV_2RI|JYO
z|G6Yy-Vb27vgeOiPgqXFSM}DihezjYuVdZz=+6IF+nG)W{5zBUqb)y3_Pt}YTK;i?
zgy6p))B2C<PhCGr|L;S+gNL7da$|D-pFWS}TUzPQ=(~*;Ur(C<{}d=Q^J7%U`TvKm
z2<OM#cV4|e^P5gq;Qf{xu{o|+S(skW-rvRczuo?&l2`oMP1D0(O!yp{a<G2hyg$pB
z1O)E+^xOZF*l;__QQ}vz>0H+8yCSbIA4}NuE3N1s%a^y`RoO2(9n?Q?=lQ=|QWcl1
zG;Y*<`nW%$Zppj-b}{qC5+*x^e5(F`;&;T)*Aa|$KPA61ngrLYik`bJyZ$;$gTn8H
zpB?HJ`3qd@V`BHb(fep}?G6vg=Z7z^oLlp`$@|l4MgcLOANS90)sI;1!8$|r-L?6N
zD&4tO%cMR}y%rzQTeAC)aoeA6^Yum2(U)$uAG>F^rRSFE#TTU!^+pH9x+3b$xEE^G
zRoWk~sd{N=m-$m`-uW<flViP+o3a~?d-vZ=Xf&L*-swn9c>v??Oqc5qrtLo(Ca~gU
zyX`mT$U1+we_i{Ygdh8weXruA+<OjzdtNQl6CC~?c*r2`aLd1XX)SlBmVB6%d})Q8
z-oBOlHqOZqZ{Jxk_396%*Avc6G3i<3%bXgi{d{tW+57Mbjiqcp;X5A59awFbc;v&1
zPOI<hcm3M@@A!i`X&?C}eTg`6YB%qf+xOWV{^_68eUW{C5u>c^y?{flZ*z<;-QN5E
zba2|&|I_DH{bT+SYJYPg<Aak@C%mqj3U&BL=RBFr{(aJG(>a$L@1C2ZS*Gl_zR36d
z`rCK*&0L|n^W5*=l)u6?$_3B)h5yWcB4BiHKIgfAYneX(wfk|&V}HuE{S`MFUhc0f
zOq%m_{e+S{)(^|-cOT{1@!}IF|HdsMbv&GhtN(wi2n$$Q`|*{$MbEEk6D_7C|J$;4
zeurYp<b8@QPb&it9i9Jg;_<(eSr43;=Ko|*d>zkY**`7I|6K1qUKCn;c;&yze_OP6
z)-eB!2#Cuul)s?7yY@z*;g<ia4(^XzQsduJ8O<Q|B|f&Ap}X$=42S>Uv^5s&omsPA
zVQtOyuzmZimLHBctG)boZQbK!g^BwZ=jXMG`3K%B){}QwrFL`wK7$YWX)h`or{CM6
z)>*TVbH3yBhIf2{ZJRFS_y1!0`gwnWb5quj#WoWEpUve~4F7brZ<ENa_ZL3T?>RH~
zXp+Ue=DCd5YhofCIgi|Mwy}I=myqx=+Ti)uO()E==WjZ)|HMhX>c*$ajFCUPS62U%
z{>JY1WjdF`2A%j7Qxio#uiHOG@4kiNflE%TiL(RhJ~{FP%+OwN(frcPe@9t6ivH&D
z8%X^AXT~XV=zaY5>_1VBqIyf@;-1$>nLTOKvRH6GU1Rb2TBo*#jQywIemx%FcjIjM
z%1XO+CH3~t?=wg|PMue4RC;Xwjl-OBHHY7{v;5O9p0MD=zsa@&$9(6X_`07<w!}ze
zaqauH3GNXSK3nYj-deFmde)cUZW)X`*+z%|y^An6xKX{iFRyp)2dn!(H|$@}uw(Os
z%bnBj?H9endSqYV-Vf2f8VQDfIV%tPeO>OM^v245vQx~ALr)eIr~f|v*7=$}gPpuV
z=!~xG){mHWewtQ%>3-Iye{FjIcQS2Q`RA|CkNs;II4;}Y@2z)cTs4#Bp5yFaLf;<#
zjajDoReje?)Bbx(M)%)cH=1tisna>3wy)sNM)!k9k{|9kFnj;bi<6oDs&d5F-;KCe
zzs)4R{rPEi!6$#76rXkO-I4R+{4zy_J&A|+eJ@JBC!yi9ef^ZxQ>uEn?w6lo?&Y!T
zGzz+O-du?z?rZw|id(XO+cw5WXiN*Q<6>2mIuWpZkNi7(&6<QCbxD)BGS}4vax_NQ
zXO=eE|Gm@mp;5Q$>BL5*+4H6E+`0LcC2huqgX+($KIcjN<UA^6-t^_H`GSM2HCFy_
zpG2Mds(!)#Ntf39ddZ&)AJklJez$$X{QKwCq;4?@{nTzV{{Nd*;NM3Do(-xe@}Ib`
z`TkyFLhYaHzf+%o;NNTdg@1lyO?|Tcza}P*;Qc(4r@d*L@v=NRy@_4^Z1{p(f1DiD
z|9m)loGqtn3&Zaf<y_GbcV!>xT+BUZe#+?cWA2L~v!pLb&(>YBSnF?1epbTrhum)(
z{nrWcTv|B0cdqW!>Fgdyb|u6IOe=7C=hgPD@z;Yit?kEF@XKw<TR;DY^8s(YV@$h$
z8;X4h|8?<DP3WTwQtnK@mef8!FJP^~V;88iujb$XolFgT?>}_@&=!6F;QP~$PG8%w
z`+pqI+s)E>Pe1?sIQv+d@AGnzzu)(tjSpxL+h8Bqao&D)^rG*FUI)j>9QTi2wC?M2
z9>wq1;-%%J;<ex12-n&lz4Y*;k6{`6u1Eaasn7Q2U3&ii=klMeOisV~eOK;dYPz__
zmv-yhzh5UYo%^*@@T2zKcAnxFqPzFs(%HQF>ttQ6cx!gSllyDd{C~+*;QKrDv_wFE
zfA!a&{2cF&zxNcrmA~R4$Bh4rCLe!2d+F|+e?c7^4(R)Au#a0HuW{v3aow`zxmP3_
zm}Bpxf788o_mtk<_tp#K1X=5r$^ETe|9vf!#5>X86LG)4t&b9_U+3VTZ_jz)_=SHB
z>whht|2*Jbp_D#nP|A6J!GHaGrPnC^las!aFB)O;$GYR;_qKKKj>Vnov3A;Zdd<Us
zF3#UyZr+f$pL_eiEe#>bcZBzswfx-ByM6oSH`|N3->ZL?uRinVd5QRzx4*QfY}_A~
zaK%6U&hOqw;ksIii|P|{6Xt!}eND3Gb^WCN1k0O$)|^?+zls0ylk=tG%$ABL?v?8-
zSS2oU=~s3BlK8qC`}Z*{W<PVDyYXPeK9iTW$)DCPXAC-3e<c6uckvA$xurJ!pUaf6
z>(qUwy@y2~hpb_|@%#C4miTYljPoTP#=qmp`j!>Fhr9N|<v*7uA7GeQCgYKIBz5bW
zd3MFAzvI@=<&oI<);?=@efGwix%2+)OkC!9+`mRp(CW3nrOv;6afdUl<!9c`ujiSo
zdRTP(-_8dSC08XrMJII6JI1~54>wQj{P^M^jwxUEO^kfEs_aIejPk2+7x}ibALrgI
zviYxm=ih^EJA5{#i8&N#1PRDI58zzty}EXJ`RS-X>}M}ubBOKQ_~-nV%D+YRKazzz
zPP7Qg?tLt?V}a);v92%1wLjl>c5M6J)+#@JE7O^p{YLUTY<Zu&*S^ef;CQ~B=3o21
zCX*vIpFN)+eRnx(C5uhzcZGvnx3>Se)Nv>Ka^n8@b$8#qYX9lo`7YU9|K#UuFCI80
zsx8x(JGXJav;)s?*27+39=My|{$V0=aXPQ#<+y*5y+8FX{93O6F;%%h`sdUP&;JgM
zAAdF02?iFlEs>gk!{hJ2@0sN{-Z+YWGH2?($nmGvan_Q|SjCjr>mPai7iR5{_@`I*
zu3z-Qx=Z)OohEGEZ`O8eJNKH}-~10Zf0_H2S!D5D@r(J%za^www<W(X)%gFVXQ^~g
zPRXy;n~&YUAR6^$U!=4^U%jf@*?heV_g{1FyXx`3^V9ol9hzS*{-`?qd-?NI(^(}R
zz1f}r=Qr1m{aW_>?w^`8``*oGe|(#_{p0=5+$Om{hGFCJ{gd`L`25|){PBmYc&zoG
zDgXaO|DDS!A^hh4={W&TKi~STjEVoBv+YK_mc!nU{*T1!ej9Fh82@_Nbbnsey1$+i
zKK?s#u5FHQ{V(PIZ#yGRVxI2r%J>rcV7-6!VcUP<hn&v+V{iU@(enN0(kv#~IM+>8
z!6}NmE>oKIayzyD+O~80JYD_%()txA&ZvE1J;Yl2Z~dtUf4PE=Wz;{d-8tXv%cm^c
zz3nUlx!Y>)2GnnT`tR-+POg{x?fe5bJzp*t5a9OzDdVGmHH^1<_P*CxaLnQAv2312
z^-sAK{r-qQWZajt^zRYV^tjKz&zF|}oA~vQxM+gJ5&MpH{{LAQKHDtK!TVw2#T13F
z`(3Xr4!<Y=HMPcC=dHQ#l=tEp^Vze0Y+c^-z2UdF^5@9`6Fx4kSC+bZy#B<@$@>i!
zRAe3v{y&NPceu{R`@sxyb4+X0EI;KRD&CX3=KjV#?3;J|zZn;3zF<RK+4)ZlX)k%^
zpI}f3y>b1w*vn5p&I`M-_(oYX>Q<$Q1pM1N<F0*d!^%zlPG^#Sx_WP}^IZ3DrFhoY
zYA=aZ(fkvIq<B5mYwOl#y`I1N(Z2hyKi&#nwWMBr(xK8{cekE+arwJ+*N?;Nwm$dX
zQnSmcZiVsxj|MAEcz)$+NVdHRKKrq<$NO#AXZ<g88y-I0e)&_)Zid1;jl6%uMW(F%
zFZ$y0a@J={A1|HxRD9dj(=#7!=J@pg#g*pcr|<L6{J)x8gYSm)>g)FZ7MMGph&wfb
z^9SpJ&WU&bO_K=OAI7lLko$7f<b0Q)GfKhhoj+yythS0M+Hfx+QH5XUwY7n<$>cmc
ztJP}`aCu4OdM%dacRtKpt+uK7pTc{M!1e0{<|R#E*D&Xc`?ZEcWjUf>cD=6K5VBbQ
zyx+dihaBDd#`-&%4#cKDdn|vRdH$@amF0WSU-&&UiNSQ=HlcGy(h1ISGoK1<I6muT
zuJTqvV=eQky^iuZx*s`bT(G_1vR^eMR@S?xcKNBvk3HEA6_gjP2=BXbA!4T8ntiIr
zq&WSw<*u*$+T62Q;#}SCe+hLTPWT%otXg3hAw1`1Is3iBP=Bw`e~YsxD_eYid%E5-
zH}T%r#WVROeeCAm^NISWr{J;GU(h^i_E$D;y(I-{yL2R`E7<2PeX?`k%3$VmJo~&v
z?r}C5&&dtvx_L<Pj^{!)t7|`JT3ue=eAf2Zp({eQIj1*0ITZ8SE8%v3f6phs!;VWB
z<Kz=3^mc_@_{}?WbM1p*1z{86iRYJ<ZQb_3S;PP7*N*}d*J=57t1*6#le?L+VO9I3
z1N>(e%l9AH@@l<`$hK*JV<hJOoFLA%-&ycqM2z;mwzkH^xMfKqvSpJV2wd;@ULn}`
zJfQ5Co{`x)&EqZC9B+FZakyl(nkQKFe&~#y+t&o{QQu+pzcgsR9@Dj*RaWBu>5m>N
z7HwQ~V~I!Yf^}-^rXGLbaX$O1qT-USa=rF*=kI>4{B}d_?DQqmGq(h%S8aE`cI*t#
z>l26c)*G_0YUhZ4^fOs*G2_iK=EgMviEozA4RiWX+jlbP*HzaS&Vd_Wdn#2Jo)s+K
zY3RwfuQlxCRvSO1=$BpBr(8<&_c!}x5a{@1hvUPQhkkK6TTfCuFv~jU&kLiy4l7%p
z`SOMBSHEJM%cXB}(q!#Dp>=vijR}w4eLk;S-@a0Da#W6RY_hgXs=;N({o7u<ZnfHA
zT=F(9R{6?~gjZKCY~Px_<5KTKGrQ|XEqO8LXH-pe@e?$c7FT3?>lQBc_@GUK<zA^%
zIr1#4l{wN)7XL8o=3B!%>Hda`WyN1D%1f8NI`Mn5NZPcu+n(KB!{<GB?$3EKoafdq
zFJ4&U`_`>)dGYbr*-oE*H?&%tZ(b+*GF$wqu;r)C@4^>GT>Ne+;+_3K{qfhB|NXwb
ziG^b34raRVXP>#HX_so+(mLVf^FJ2#1?$q@>^_qgUwUQoovRw#9rb3c5uLU)lyzem
zqt3=2-h-~DKU}W=UB?z=lCW!`deqg*WXHuP71W~kge6G|G5Sj_S*CaV+eEX&EOwe{
zrt_^rI$J%e7HwGlA!O#d&k|cK^VdkIzsc6x%l*)A`%zoFC5xh}-}G1B-72rX|B%w!
z9k!jn)%sc^D$}JZFMR8puw|pMQb>uC>f?(AJ-!9T4cYN8Rs1xb+wi@6-dwG>!|<?9
zgt@%o@(+x=obq%-->-@c+4geVzS*0Z`%msGGu!!NJ)c_LcbPqLtnFg9ZaX|o&aoXd
zmb}qfTB<hJ&9}<4WZ&WBO<x|oDCymFe+S2{X(u#Ztmet}(BHCqt6JuBqk|VY+!9Yc
zhzTj|t$5z_^uO$J_kz~!jY+2Z$1+8AzEnNnnD_EDo79sdZy%Ok&TDnhQ_9r2Hdjr1
z_W^ajPw!H!56^X!c{pYMg(p9o@8`X63O@YNt5W{))oVYMLha-F<a*~WQM)(g`M%$W
zlP~}6TAVSfrt8S+b>_(nZ=CJ2diGoQK*;*IW^vc0``G8LzxusY&eSL^YtKr?ifnU{
z1m-tVGfs)9bbMF)s`z?y!j9(7o}!QIMSUk^-}oL9{p88<6+G|O9h`ooK|gB?TggO&
zp1TLuI$mr_aygtHo4aG3T3qfgiRL#F`!{Th2uux7*)7MLbtTNiL+RWDsV^6|y-O`{
z>Ukk-`Rt&W;==zHe)ITVrCFW!o9Fv2r)%*6soYZ@TB<r}L5JCLJ@`7gs@}S>YwI1j
zdRv9}!jp<GWqaqOyi2JPV;6X(7$WN`zRvNv`wIDUrbg9!m;MNP;&9>evG)=y)2+M;
zlGo2UUS_T>wa3PU(UC)K(q?z<=7|M6C#koGmj4ir->W&tr(Zz%+_|&K*Z&pDNQH*Y
zTq4yUST_52>Bf0#&o4}Do49e`z0*1sFJ?`At=neoD?hzg<({GH%UylWqRZZy$G*=>
z=$(~4IqdZ7IkqNuCs(9iVQAj|VaZ2_)ny?<AEh7f+7@^JtK-ew>#fV!R92i{bgGp5
z&F+cEc;6r7lCcqcbKzTZV4py;n)&yqlb$jyWNYa;<&`a5(AnL)OIOGE&g$QCb69M9
z(yW}n>0A?9dn?tjBV%{PqYJ{^^E*2J>UA)z-K=o!$$}f(=7%lL>3+8=pp;ee%5NQ}
zjrT+M&h+u;QcKe@tNf@THz(`ax&2SNE`Hx#I5nsz+5h^xoinC|TBwI!D)Q^m;PFhV
z-E{QNgMCYkkGwrTxm!wL|Hd8v9go`exr&OLw>}mrQqeou7n2|HFSx>anvJn0`{$aj
z4@R4}UJ3U#KiTpwf97AQEj1mi1)6iB9KDZ3?!F#UUOeIH!Rl<$r*6CC&A&f!e|qlJ
zN*R;A4{Sp{LsAUq>AkDzbc)*DZMJ6W&SUw#SAR`<sL1gwV_Wx}wXU1?9f<lYpno!$
zxn}0A#megI8&*nMoMugrY&hIx$K<)nNqp6kPqXF=ey?4;P3XK4zsT`83)@F~XZ7hN
zo>I-^VUpXN^G>j0b+@_P1J~<sX2i%gc+HK;zg%`*;mn#o#*HQWF7=1A^t}uZ-_9%J
zsNvNW%JGU}ZuDA?*_#~dSZCYk#AxJwkYnFxrg8AQTH@9AxUH`}HnIn*_6uy!TR(?8
z<Cx92?w^Wv%vo#OJEuo8PkYgG>%>)gkAlSw`@a5EWb-e27cA7>z3@xI+|}O-&zS0|
zujFUalnjr{U#W96?V9e1v%x}|;`wiqg6F<@9oBkXarq&S-+KQlw|-tMy!dVSVVMcn
zb_!N6*pX{AdFh`n<wq~X9=ts*wM6*Piz9Ilw)Y2#yBvrOpOl~G&EF-N-!xk@L^*Yf
z;==sUf=x$w)@pmr@p$j?xtUok_`Oj0p0*_hp^ERt73YV!^a}s5dKRH_B9Z+am(_z1
z=N~JLA6!rBNPo-H-IDY_X3NPL4Gsls_wCv0?Pz%F>)!htb*_hOt4>iYN>V)IHPv{B
zwege&<=B@_FLUBfEq4ql5cnefm`P}Etl~G@18VHbsxg)mQh3Unp6`!3>Y;!0Zkof5
zrVmS56elIhFg32PUZ)`uWY4h8(_ph;Shmy7V>R<$D1}<~YHbeJW>UGy=5cKUx1EkJ
z=d3BRKUN)iaZb{jV@hdLCjW!&PXDiY^9j9jT$5kmz4!9enw6;>KX@~vHo0upG4%Po
zv5jG$TDI8H)2_#b4Oj14ws&XzL|r=x%^0bPo(CBJoMaW$I99f68&A&EPA*F)+n-G}
z1tyB(ES_(gyUh|8yjbD#I3}5wyISI9lJBxBmXS(%Vy34jE#ACBN^y!}LApTcVfHBs
z^KN&0RKH<5BklJk+3Qs%i%oCk-wK{W6VJRkkJS<dFU+_TcI|TT^fyL&hFt1Gk_Q%7
zYg~15n%?il^|E2n(QWr))Pr8H`}+C`@6IhJula4;6~D>wfQ`Ywk0O8B+$@i*;NN(b
z**i5y;ef+e2eXBttFD(xKKijCyQy(ws8GlzpVpiwvo3ka%``N)WO8@k5fyop-2AnP
zvTi-IUT$<L?kvnIzOqO;+T?^7&nBN{X@MWpmRcHozU@@EbW#4}&9O82R|a!D@P5Pc
z@zaS2rB4%EY^SaHoZZZ3zUyxii`l30-wp3FUp}upXuQ$jfMJ${X0*AG)<d=_3udpl
zRaLmgK(n%H@*n+(Gk=8k_9j_4zh~z0oL;hf#l<;HGqya9x;XLCXElYwCeIt+uW20n
z^6jUlNRVQ8X5_XU)}4i2&-9aa&e$ZWoz!VzZ+B|N`R+!gb*-_kzs`Ff)TphUm=%+D
z@U5$)-O2V~Pkp;NyF@>%Kgr6Q=2-9aangZkrkl<ci%S^iT8S?H_RK-mX<^naqvuB>
zi~b*zD|_j2vd-5i^O3`ft*=vpcIjxUg-STfMC>}m>G{(jKY?%C6b6-8hFP3KPwyxQ
zY_bqhw_P~l@CKbZ(}lM9f8>ve6y#{hFPYfj$mB7><nv|yBXd(Wcdjv;c=;PoZ^`r}
zb1W*ZtSYF@msGj3Vx80EJ7@RYkb3uXQDlI{{m_|LcJsbG)n?M?JZsxw5sn2vcUI`W
zS2aEKX-V?(#0;O4ofjTYe)Tc*ivHxZ^)YQy{96At&gU(bmz?otr%Gy(kmg~%-eaDL
z)|Ef*na*6fU;0Vvfs&A_cP$e<RbNc#3z2Y_*fXQ^OlV^YkF_taqo{LH!=KzYnLe#o
z)Z^qs+G_(uW_$8x%d2oq?mB&<L-5?aE0J1@l%`H<3anK5vB7(3)8E9u^Q#TSij4&r
zcfH?xK%cq%(RN>{$t#zK3Fk<!K6K){!^yKNK9;z*9%dB3BC_V`^?h7s>SfM{RQA{y
zTK|}K^4;bHnJ15w67`ec3Z42ecf;8j4WlQki_NFp?0D;XwQuTj*S4#-*NdKb@7#N2
zM$o?NwgpFQo$@!OdfqqMqr!KuC~!$lbY``(f`3Hc!Lkoh-EG}k7cRubi5hfoC|hH>
z)_Jl>Qmp&T78S{i9T%i0RNo0;?C;u@mw2db#(D{R_MJ<$ZdL~hpWgnS<@1SUoKlA@
zzQkW&-M>e5wxeW5%ry^}{8<57Z<%_7W}V1TUVdc<v#F7fiM&JLR!^~<6Q3Nm2}Z8k
z;d(%iZ$mD(%dwZmkq0{s!Vk?k!tC)-?WqdOZ05~kKC?HRQ1+CcHQknL_Opp)$#Zu^
zUr!OSyO9y-wCmy(lkbOL3Ox{78818~YtucWZC<Yz=zUrvCl*+9#r$&b%IC-GZf|>Y
zT=T-$sn_#AF4-KD>hm`@$l%nTlyj>)PIIKa64ccRep4|;$4sqIxvJ@U`h{mJ12f*w
zOf0^AeSf~Ox1-<d_s0c;)4NK~Ub1I5nVji#TzXFfr^Lj%BNwkJy-@SE)O)w!ovFWG
zOw|oPHx`|BC4Rmd)=x5S+9xp2Il`kBxV!S=p(7XaSI_M^tibY5;g-m+1wH|ZryLZx
z&!3*{vAFWt{K^tD$JRApB-r<v>ijw@IC<jn2cI+9H8M)MHk{p}GV2uomBJ{^J8RFe
zb#2-9?w0ERRmG315<5@ca{N;?xtLdxwJxGbP>KJs=A?*IX8aTO>N(FS-LkBh%Qiu^
zdPD9e(RV&Y9c)q8)0CpyR|bbMPkB8@DweIXvC~v6H%VA`K@Y>dBM#eh7dIuHdv4#f
zo5wgH*gp1e`M31<rJEQRuK3^J82dQyLh7lzo_0yqvqG%2k8vNabh5l_%3w4>^XajY
zdtnk0%dD%O<_F2z{R-s^-4T32$GJ_QVRL(g;>S(hXD6Q3j?9169pQgmYjVq}vK@u$
z8$Cj&2|k^4^Jm<pn3wlD_qTA~f0d$qxiPCXC!G64gRZ&K@+7z8S6hXa9gv;1b6$wA
zO|+_p%eu;~mw!wPT@au;OU3)C-#p2Zn=vmcuJg=R;F3^s^bZv?XyupOa)`aZW|d?I
zyT|eA`bQsdOR;vp3m2E+>3ntYW}e5>nH(=lR=I4C-0FEvU+qcsHKt9qOP0Q#_06wR
zWcG%gYdodQSqlyvs_==Je%gXbT40{QEY_Q8TdE`zInK2dBr<ZO-2dSf(fZl5y|gdz
zh_dwE6HW_mA9Gq8yR6UQL5}gCE%`e>@N;m#*tkb*1=FH$6V@+jeUdn3qJDF>c!iU*
zvZnQK?KYpQ&cR!5=SHhG?_bBC=es_cUFfo2Y{}`5vwnUP`B^3~cVgzj^p$T<q@13z
zwd<!qFGEV>#tj?PcAims!6q)J5;}oj(Yr5H_{E*&Y`(7!*j+8U+~9eZ=X4`y&V(tR
zZ+5uLZk@PXZrS4UyYf|RXGP@Z)LgRvXK_e4<-`_uqc>K|R_;4dbujYJqrf>Q#5j+O
z+!35zU)T2XFYC2ur<GcT_Y2k@4DRle>OJzA{rUo()G%|Q>icgu>A$$ObN7~q8ez#R
zF4t+C`L*iFMa=`%Sxgfp8)PT`{Ofx#EG5rsX(zjd-CM~tzKCTL()R4V5xYV2(VRjP
z#Y=iFQnIpF6!RwwtABEKzP0acz@ZDRsS(q<y|39PrX6My-*7_xw9)l146m~euq+CH
z`~BPr*MwtCG1EQu&mZoZmtftqX7#@39E-13%wVV#-}SO(pMm;^U&jh(9Gbf(Uu4SS
zg39d&jsD&dopXM%${kB_gD2CMo}X8*T_(0sbk)^E2B`s@438BQ+3oGhrysvQrIyFB
z>c;j@R=UwvhBqdfiCBA_);w4Ga7iNb<P3G^-OIdPcz@X+{*x}W?}cyA?i_`-g8FX@
zxozguEI+q#>dxS$SsyAm#O^OC^;hq*DHFY9C6yPt^p0|Jf5Ao1Kl&Ta{Mh%oXzjag
z_xA^XZq$yc*`$`x^VTo-QNK|0w`YrURF+M@ux8DtSgub_SwAF9ZoPhT+^YJ*O^@ek
z7izb&Ru)fsTTtWoGVf4s@2y!I6jn{&F}Eqio7wMMqnVdUfs~R%VZYQz$KC2Limz9<
z8lF<+yYQ(h!TMJq)50VCck%=#FS;pK+LwLp<kfeNQZmXK%ui2x9v;4#N5gxD!UWSZ
ziH7Rlla`vd-|l+wGH^Rn*Ks9d)tTD7au1y*N{Y<ypLBDN&^^aWv!5G{XC9PqPyF3{
zB6y;Jtku#wzHe7W<63^Noie5LI`7lP2fAHvru{f)lEt`LVU|y`^0dVV1$jA)Ef<zH
zb~ip^aaIyO+ctUj9`|Wl49|k~nlHJvavqsiDY;a{Oj0bYIf~`xI}a|2Tgw(!&osKv
zmEE+inx(NJr6q9noK2jzF*4jLd%Uh5ZTUK{;;NHoY5|8ud`f)8q&}l-m;P_aWZGn|
z7a#iLzvntF5tTZV=9LWQ(@WV}>V;lk+I+0_Nm9X*s(DAWeu{Qy==?eUWPM`n@s(U(
zE_2SyUu$R3B9~_7`ReJ>bB{D`Pu#jtbmfyYi$66SnO<jCiB$_X%+)N;*i^d5ZU66x
znd}yo4-W)y^D_S864K<8xnr4#&-B?cAA<_tK5P-q>-+6#*1A2`j6c}WRIyRuEpugI
zQu^ZkygCXuHFK{tDdnieX!d&cS(?qaR$X)QZqcNtEE5bZm0D`57P>!k-2dPVo2N?m
z_30j$kFI+@!L7?~b-d1_x7?j+3!A=aY+_=TZ{wR>?Z5Zi^?AwC5xy<{9djb(7raU3
zvN&3D@5VmQ<Edr8pG_~cI~b?bS(I_7im_?WWQFy6#8QrJTo)f^{H1VmeU9*~->zLo
z+2S8w23gs?VAQ{vEg8Hs;gnv9*;4Mht8*4yUUZp}fA=NE;x_3vZl)JbwYxW@P485w
zeZ%^+U41`8M%bR38z*FC)<_-Rd$;uP3(ubm0u`gqa63Op-`lV+Q@TufmmO#G-v?m~
z4kCv>PZszik|H#D{#%(2C$X*xuI8I2bU3JOdcYqk!d$596m|RE$$xgGTz5{g9DTZa
zU&y82HNx9WBMjs3C9BPU7MQOam&Wy2$dut(%koRBoPBs@@+|ax*K+MWnx{5r9Y=xh
zHJ;#OU#kUJ=de6cd-y;xKyX=l2}{IPHf~<0Ico7HY8vbOCnwK5IX8F1+LzajIakS5
z-#T-CddTjw9c_2RjxE~Ly-&jOx1vtrlinuYOKTis)^ML{mYT6n@3q>|tw#lPGfJkc
z{>EX#y=2`R=O;H`AKBbyV5ahL->=Sx8Z2Jvd%7+J?0np_?fqBo-G|e4AN&dSIRD_s
zy#RL0!w+u0zP(*7TQ@P-^~PnM`qc$uyAJ5y<4<4f^5odFr_E1Z7I8f~5Wmm0NK0tL
zr5%rA`<(X07MAdYFO_>b`$>)EOMxdjCa(`x?rF=GzL?$ex&O{o1^f80AD<=7Qgd8>
z#H@;%`SgO1dDgPS{2Dy@4>D8&4BBT;V34oOY&F)tbH*&hOut-N=)i(~;WKyN7C6K6
zwrRpHhjnJ#w@F*8us(LadV6`mQ%z=J$^4G!B!Q-djDDAnn;x|Hwdg-Bx-#Yi?|#3_
zcd~jC4>~pZbSX@6YFJhLueg7DwP{}Q`udjpa*7_cfihP)Pq=4U-!l;Iv)I8^x1e8|
zW80137AL;Wg23qq7f=2mtS+!E;LE~dA%;oJK{KT-7I3#dQ`{l2P}?=b`&_ubp?}-6
zs|ySqrlr1`qroB3>(yRw8(F~7^g||Yj_dXA4N~GglYV~<@!L|DXree<JV{&GChb$=
z?-_rqjwUs&N(xpqv1?6Kc_>(Hm;ddCTS_9AvS-RgHb$mfW}#2d#vLu0t@K0W+@wif
z@g6+95485atG_e*kb>}Po`s4^YM=8|_|AJwdG__-Wz8#0iTgSCIX>~NysnliCT_#B
zs^ndnNT}N#rfUv7jI-Jth5wgMtvLKacg^>W^?Xf=1+H?=vs-<n)GB8gJ`r<zJF7VV
zQ}boPZx3HR=-0e+`)kWa*4#_XD}--*m(2DvoYy3z<8e?kII7;@6VIC;W_AXihkoCf
z{_WrH1ucqYwL)eSObhojuo--8_qMTAG+=U7sth`^c+qRYM&*>6laG6oekkXh{dlkD
zOM&D`vy)r5q%C<qd*bovO?r2XG+wS3-{2k0*lab6>A!+lgH|46;*}21w_7+oPE6Re
zf6?Q^n*5qyj(l-WoqB?G-LI+V7QFOrvU_zUV}qkceAE*mi6Y^k87*<oq&-^re<uHG
zGj}+!OYeQ7-^1j4SC>BcmimX~<B|l<ru9nOFW(mXxxh*5NwV^-McxmlD+;GC@X6OJ
z^gL9!>(9irX(mzYG<pKUUfhnYT3aZ<{WDZ5_266mEw!qbuFTUg)_yEA`Ly$or8XMe
znPx8Pv(K#1kA1U8q<{KB!+VS0e&50NDXVMVrs!+CqVFtB<kFh(`t&!!xan$^jh>OU
z%g(CAWqU3Z@KIFo3FPycD1F5);=SVEJv!O0ya#gn{)Ya(V{a=Pbv!#s?B!CXSwAy&
z>pwbt(~hr{fiwTdS*Kz(CL7L2r%JWcHcaL>czM(Jb~)F<V7@v52CJ)+#ia5(?hD;K
z`|bNj5x2QrEGe2P-xhi*_Jwbpn2~xV#l`iiM1P`@%Juouy}1#Oujv;n*ZrQbxINy(
zy3gH2hPB_&vie}EuG*v+G09R_7Nx+r1#^S;F5R|an{2kq4t_;8=j>XRC-tmJ4-Tg$
ztFq5!f4t~Kk?o!kJB!CRCRRHf+Z5cqJ~BLY;v$1o*Tq(vGGbBDYfjW%s+e(Prqjh~
zH%_OlIiPeSH7H1H$`h`LsULnv%|7*D`<oWFBzBf=2bQVZk|s|PnJd_}K(B4t9g}&P
z(xw}QWA<!G44iOiL!XFa_NFaIe(p)w?qK-qM(t+7pv@QdN-i@nniyCj+Lo4;awdr7
z(7KK1zN@x|FI-q%-S*q?`_H$og2&t@JrP~VDiho~ql+^wr&&=<h*wU#+;Pb??^W|s
zN>U=-53+|Xs*4G*4%vI*@F@|gt_^0t^=>Z;3B6z9__0W0QfI+mGb^n}ckH6J=gm4D
zYOMLhIZw@p+ppsqBY)uEUJj25&R6QbMNcW3^vXs}u-!AWZo^K^>KUHN#~oD?f3(i^
zQ{-&nd2+vFZhX#ViKxd%8q}>n38YKv+?(;LPVLx>M<yJ6XV%!obsWk4x-7fAFf!j>
zBhgEk$+w1K=gT(E18+Ll9Z9_~wW9lAXq~hvr?6PsMqi6V6BaRZ)pY!EP%!3<eI`^T
zeYX9G_tVo{Db`^d{wqEX7Zq5rA-7<X%vCoAj<-SPvqj!j{is^pv0L*Y^QJk0@%qJ+
zB}_yov`0VZ3d*qBRUs)MK7AqM>hPIMPaN!E)R`qwDlUA(!?bIQOooHRkuBbzPQSVA
z@%vJ5V#j&YCFPscN-nY#a@;;v+9+VepV+YP(u<#EwLj%;U&U^5c+i+Mr$BoHx5?tQ
z#T;TxOldaHxH6oS11IY!%;w*rE!cnc-jhxHIB&BDTTEuRUBh&OE1>c9w!;6--`$1e
zu6#MUcoi#`>f#w%SF=R<g$sHEtCIxnkBaPDde+uSub*vF)I#&ieIElA#HHKYosX<7
z`PJ3`CuGM-wa0BrEiAiAqr!H%h)fr6@yyo!x*^-})b`oS#6vASRQVXbT0~E;UUHIC
z{2#~CoF@%c3F>lZw{5M{x+tjeacOSIsts>Hq_D)Se(-4S$%QeyRusO_(>dvNaCU@;
zWZ{lKrm<nFV#ghrRDN53WWI6zQ&W$HPxq~pJu`deW$Y4Kz_>m4>jR^e4K<H=4jwwh
zWTqoEeL-HNzyI|8a=s6p<NQBGuV|dCGs*0!qZNlT&m->13l4nY4PD!B`g+<wc0)$J
z-KRFV3-PF@9`Q6tI`g_sfcH<@PtP^UamOSlPe~HWXgCnq?;8}m<c)mBR|P@oP%Fpf
zY}q{qd>j)T3U2*Ret&*b?DfKBxnJ((er$CJ()o3N<L1^12QR&}y`QR&|2QA9Gx+j0
zk3gM37AEIoE%W@Ju^;66lXKRkFnps^YI(As_2IUsyUWh7H8E+lXcbqbT3Rmp=J0)X
zi^TmC#z}imiJfZWO*+V~G|%&F%pYCtGwz8$rze*&JiMP;v)l79>!H{qYu`vG&tKVi
zM=f}vb8h(!t9(b!N76m)oc&Kt?4P~d7?r!mx4|R$d4yp0(gWt}LJnGNSX<z<*+5fx
zt%%GmPWhE8&$|x4Yiay)cc)G-^VC$fLn{wVa;!S>n>R7?Q1N<?FWfs{9}i<wDQIfj
z-XvjRGROMOY{k|cY)S?7&qb{I(nDWt7BX0QAiQPH(uRiKrBkmR)ZDCp=cM8!(R|_c
z>MJAuOqV@*eG4z^UdD?7uiQ0PiKI=6JR;f`{5;I6c>b;FeKIAMM^EYK&#I7d(QI$K
zcyZ&Up1lWkCRFTfQ9phmL}-?`k{{FWt;f%%v|fqP@u)C<7`X9)3x^jow_mgGrcF+@
z75lgN*Dt-o;4oird*9|Avh~M~sYJ$oW_Vd)@<mLhb)kl{<)0v<ETNRUr^H^oPhOP5
zS#ZH*cemf>m8Rk0e;=Ev26HZCsA7D{rpkOH>aETRhTQq@O!kM~+Zy&mY-a*@&CDg~
znw+jP*n5s8CQ6I%<6u~3?bj8%Zw8C5=$(LBA;-S=zBiusC*}teqsDuQLW2zUY2tj5
z@1J!3zEv`h|Lv<o&k}4;G5yatAFEgy_vC=-&Z$3Da*p1cvQ2%q^y&1635Nm}a(%a)
zdq~=Ra*N4Jk+r_MZQkr>cPQR$e)x)UlZ?){!z(z%&Zcm0({xX6;QOuHc479l%gk5f
zw2I@8)V<b!Ruww=KI512C)ZQ*zg_#WZrgd=-=_+A`#1iJE6lt$J@?ZRdry|UTCa_3
z&)wYmYu7{Bj&il^hCNlRXYC*OJ<L)JyU-D4!nCZ{<7K5<!V^!?*FIZQS#%TPI9PAr
zj4#vSKid(q=KF#Lp-U{>W*KN_SZkbKbeFa2F30`5Th8A}X-bGuI;0+N@?hQ3`!|JZ
z7&lJdy3yoXQ|Iz!FIQbEn)t`8bG~q42=jKgX~jvGHmdGaSDN)lOyu!3zvJKQepR2D
z{ERbSrTXBpbN*W#T%wPQ^f{drt39;3o@@Iy1@}Hq_5~U@_CLI1X0cwENps<meZHP2
z>s=N}BtDsP)`&INLUA=`+B^0Y@kKI=->Y9{ZD!gcdZzd6^y?Q+S{S^!y5UCVIi9ob
zcfxM@aT)EcUzoHwsA~C_O-jGs<bF4Z_1d!VQ>~P6aLcVt{1LH<ZHtdMKh~Ucwe+oY
z*T*_%9{b%(9&N4KAaEcz{=Ko*Tmf6wOHDEi+)J|OE&qPDqs%0UQ}ln7=q1%`?;E|%
z9Wh1U#CDkRRBJBc-nhi!=OT;f2`smdF7#L(z3)%K&FHxW|Mw_Mxo3EtK5*J?L7?n}
zvof}MbME|Da-=k1!|A@)!BIQg7cJ)a-IOc(`;JMQ(V?36ljd`4)z18$;;U+t!Tjo*
zSJg|Qz!PdkvIRXssb`YUuXU46Oy~EW-FI%?<qNUe4)4rtm||mZlsfU9Vs4jjd!usN
z_rCw_IEh_da-q_*PHYk`lie3;b;`p;_tDNW&J^YQ%_?W+8|vk+4w;}G*qE*7*|xz#
z#8^67%|z?uo68q@I3@_ivu<?v4p=!uwUc#5+tMBE+7prw@qS*A5MgX|{Lq?kLw7ch
z*#g?G>K%{voHmbe=#H5B|GvSzuCNB?U(+sa@-Fr9ZQSh39ktk3>dDm$4X<)~FDEee
zz7jqA_QtM~yHUHIG+leHAKQMwMO4jEgkL(e_-lYrThx`yI_2HMJG3LdN^DU}4&Pom
z-AHoI&7_12g@R&xqL~D&4=S;qwq?kROinwa>b6MAJi%raw}O&|Z_Ck;ZL1c4YkJsm
zLt2^d*aAgPy_imKo1M3V&ukRX%dqb3J<i%(wCT(o9<#Z(#g4fAyP|mT*xZ%UA9HUX
zE1h5ZdX`^n?laZdPA3$_b8e(+$vcLKSIV;8Ytwz8EdA(++TFwxmy9l`Y?~aJ9^C6Z
zO<_C7+lUvN4<srrd-^A@a#pO1*{Oflmm-f%t+|)_<CE<*UF}LHk!R;V+;<CM4f?!&
zt#(w>=E<U}x@+TPoL;lLx61A=T;R}mHm#-9EMmuA`Q-Du`z+6eR>&Mseo<fY?B0^R
z&E^UhJ$9SVpP5{L@8ces$Zb7OqQk_EC;wU&dnEhf)-9egPlc8yvKYB^iLfo$$I?`L
z>AmjzSzG_Uyz@4w{9>`dYQK#lyk}Tfw(%^p+p4RZ(qW(NFC4m7;nX^hg@r+Jmo%65
z%oA95y^QPr(%XuYc1XVAw@TA&xE*DlB-(h`sao7Gr1pTLbO`t4bZ$j<qsFIeH-uWa
z#j*e1r@|Cn!DA>R!#4Yq&W(+t3AG)Mqo(oCOzZQm72>=xyS5>w^ALx(H8)?UuTro(
ztDcqQt}{ZPd2K{Z1SH@7b>6+p+Hnf!BMX6D=hp4(>$=JA;?FgACv)u_385Tem#8S|
z<+U>lypnI56qNBX=L)TriD<7pf1zT#Laxr`IqPR`7nI&}^xyKoo_ZW_-Y!_3%4X9(
zx3?qs^5m3tdc~h7HigdcvEG#%s=A4D4$n%*LrSw(y@+~Z9W(Q)$|6lxpZUdSrn_1l
zU9NIdCg$#;i<__AO8XF+>+x)^FZ=QPVnV;qJrCG_?xVMQ(%hfRmb2_;>pXk5Y3{PQ
zCUd4dKlyh5SEK3Yj(D6|%WzECS9*$2lemtPppuiKuawXufi-ilMQ~j6xbG-9Yn#B&
z-@9f;uphVJ*3CR*At!E`lp0wiw4`OLz{bkS32!W3wC6a^RyjI*jlgPV&TbjAr@I?%
zGz8z6sYDjuGuf$ru)fr@)p^m)2_LRLwFy|ZJi>1Gf?#c}-JAIgh1tZ!9<2Sqa76Jn
z`;obF3ek_8a_2w$QqbwC<|y^`@TL4Ce{*v69;`?>6q)P3Yr~l&pBO>=oSze~tlq@+
z<A~+Aqe^OTg*KQ~giK1Rv-u}FZF1+9CNGUolVoG7I(S$$UEP*x6@T4m6|lr||5M4i
z*QSdlJ&&54*)6f|?UtU7EuPzw;-2{3-N@VZazn=M`-$s|q7JS$-6$Am{WMBo!W)Mf
z?>n9}m85W)32gB{6Ek6--@>PRF7Xz3imp37W#LTTx0Amzv?L0?Je{^tto*p=1%HWU
zEwAQ?CAsa6?pSA?_T!Y*n#8lcPoI65tn$8`F?UDo_o;=pQ*|6(F<ewleDHVv@lF0$
zI&%GIG*6r&-FH;)L-VB<4Q?FwW@WOUPUmjQvG#FZ(#0F#Gtue#sVhsKmF@O^D);o@
zoyU5HEdmoW*`udCyU)q1m3{7}*2N=|M^4@>5LxuQT_EAXZ&tMlzvmwK_jt9({~3N0
zZirl2KVj!doq1QZg|&7Y|Na<o^T40f8+STB%s#S`<w@pj{jAq6YX1L)n<TD;a-Let
z5*pgnqU9JV8g(~yTSxoq1NET~w(p8co$^;BIXx>PPimR6hsMUhh}rRpMp<6dr)+hu
z+PM0O{e~uykjltPku}@Z8`+;|zTWZ9+{gG)x%ITiZL9pwT$MQ8EV6Lvwi3U+KaRQi
zwRFhOKe%|Ox76!RYCqzfCJE|x^3~X%<bR*|<mCDDO6Fk;y}9MytSa#R{$tX@FQ+u-
zu8VSQ@(8%_Zb7Us_azk#;YmHZt&Ruv)N1d#8cs@8;r_^O?t0eS<#W&Ehcoy%)qh0#
zY@BX($SraC@>aR#qUlR=V=C;{pOlxLxAcMIBmuR1VvF*_8u#SQ*<!rWb>WXSeV!J*
ziyRgnIJ8n)ar<SR_1|~4t%+auFSUp1Z1l|^!pE`|cCyV0-`Q~YWuu%}#4V36(+rr|
zD<me=FtPKz{CprnR^8Rcb*)E34eyl=6=pg;=O?B-e6r;m*Y7N?+0VDcR6lePcGT#8
za?0W<`=^hQM^oCpI(2nzf))Nf=klL%i@ST_ozypa8DEM+a%5*5x#1W1V$OncS?<Ku
z+kg8^*${e2miOcSN1IkwY^f>nb2onRxT7!ZsK~jWju+Qv_lqp6e{g!`kE&N!a{SKr
z+(_P2s`=sXZBK^@y0IeiJKWAa<Iv2RVHvGBPek!}=<5b|^*t9P_bPV0<LY>GD01TQ
zlMR6)FT~i_DC|(zO^u0GzMJvRH#I6Qex1t6K;Fp?yU&+ADBY>TJI$#%O8;MR_1aW1
zpEcb9g(_STEDIhlSGW@yIDfgts~zj-tr5{x;%EEoe`3v6_T?Nlg4Yh^OnF|ns!2L{
zZ<}7)=QWqN306fpZ?HR?R=BbJ$zk&={U6<@pE7eW%?;Xoq-VCpxtmXxp1K*bbdR$6
z5rZ|?Yho{Z&dBq*=4NJl(}MT$%FdLl8>aJ|>+Cpvv^HnK9Nr}-PpH>>hF+V~b4Bpp
zgjJvK6jgY8uyt~$@BFme>${ylSN$@-Ej`W*{QbVn&Gkp_G2E%nh&%CW_w%JU@?WgF
z<jnj}!K*m&&!M2y#D@DR7o~YO-TGJ*x#nz&vc*vohrFdze&1!4ww>8>RA^;Y#7$S(
zR4@I@#^=8CE}pPr&6>{iB;i}frtCZ=aCp~(T8{_5=gSKHU+j52bMb?d0dwLfl+S$f
zY}JOX>)LkAcXh0~`!(i+eNo#Mv*+DcQrkCoE|T5A%62Sz?dNOD)K3K6XH7o&Nh@*n
zi=du0b5AUv=6A87JvY^FjwMrbl+vH;)5P8RxMd${?<spNF+=c3aID(RwPz0<P7{9e
z*zeT0tLwuir+8}$@3_M;fmJoukb_IbQ1IZ37kkeqmcI<I6JnNGyXBzlb$6vJYB%Th
zeBG-n?LAF<OL7x~V)|zlo6}QeeEICy-b~D?oUx!);F(a_uPXoO8#ddwI(q(@c93<g
zI_pkGhK%o9G{X*QyZ+}D*?e&Bv*=~o8*j@z5z%<2#qGQ6i0AUjH<x7<i}6RySISUO
z`x<GyM6<}}T!Y8l;-L4jR=L>$*%6JKt}U2Y>3+9szMf=l6Q|})*U+85Hx&e0IsG$O
zoLE_tkFH?}o>O+;!0xP0E^DA1bJ)>EJdfwJM~58w|DjN|HJju9-IAIeM%faRw;uiR
z<c#H?nZG}sa+TUw@@w7P38jG!b4}k(WVAjIl9d^)Wg^M=bymRhlNT)+j`A1072;G*
z&`Imo;ODyWVr}y0j5$(?(+{tm^NK0`@uIJ3r_~k~hBbP*Cti4+7;=I;VJ?q$Leh?1
ziB0Ss*O_MSepx*I^abnlzgZP_ZtHcNEdHp(OM3ZvuZ7}EGTxo*+5f%se4KguZv9ES
z6jfhc`Io$5?hId9*-C%Ui!&?I53p74cHgF3xKDgt>A?eLCl|3CIX?Y2=w3R3{>M)^
zRkrzk>|++VRj{Zfd3Vdcs*ag!-?hw14`DpI|8AO6_-f&aTRCnxdABB9JobmrI(@JE
zvuH~xM}uvKx)a4Tg&sZ=&pxo^;;Zi;CP}HkGd;8`W6$dHbC*_@9JV_epv>?~$HGr$
zB@55obY@}C<lXDPf0*%Vm2=LrZ!^Wpk{oMRHYy~(pA>C<G|DEopy1hx<xd{JdvO1k
z-Q5S<Iiqe&XARW5<GcC!BOcxx-^FIFC~I+vSbZawr`lwrka&;i$4oZnhQ_LC8~d+3
z^iA91k@~1JsW3--ZHC5EhWjTUW~mfTE%(T4<&=nBudp*DThgP)f7Xqu;zogj9k-i$
zRyeGgv7xYk=Un#QCAaIQ1Ts!csXml-r0f}++EczMlj99VUtE(kk9-(%=FN$v=Nbb|
zCf3h<f7+}iY0AboDSrw%y)!4yY^+(!%YF60HvLY2f4%P~_G)PR&ta|jIQ5^whU=cM
z@611<?f83{ozL>L^fTd--R@ouQ!<jaMQ&Vok^N$3fr8o&?$Td&tMl}HY$uq{`!9JQ
z!#ChQ^SQ&u4z}Jx{FhR0rWbtPZlEcgy{WG%`FLy`mvUE<8}Bm)K>@>c#g1ZgQ`Fhj
zBhq*uE=YF&z#zg?pq`<hvHrt<xs}YiJxNg?dhR789gXw)qv7^Z@>ySh{P*ND!JBLq
z8fW_{i9Wf?$-}wP%r$}Ip2*S_AEheS?(vuOn*aD@joxVup@pYkvaX(4)=}e{9VaF~
z@x=l6BMylvwn5W4|J!Yz`qsPIx5Bh9Z_TCk!ik@^vrG#WJbmE%wu1A=**2$me4HdP
z@ub=})9@RTY6)w&7KzC)rGDD9`Iq)vM&*|gbHZwZl?v<_m;16@yD!xE!PwbWq&&~Y
z;`UbUWpPJ(dCL!O30P|MSEc{|zIoe^De+~uT;^O}?Xb0~W_|M!=7cGBKcC#l=6t~B
zSIYE7SLrqD+})A<!jlfDPh4^K!IjM&GyG@hELgvhWA65SiyD+7*SIdRQ~bpyz2NH)
zyTtWMU+!~!@fTbzva<6bpUDdL+#`Qg=k>Yj>=0Ed`7hG4-mbT+c*fIXwiCoxS)Nya
zVWlTpW6{#~F>ueXh4l<7e$g@0BhNPc=vu|Nzcspnr+bR)4uxrk6AoPqyRt7lrMbx^
zF`8dkN=%)<K2d+(O0(G|!gKa4SIcKOvpsX)W#8-%Py1aTtbQ4CV+W_nX2yE68jF1?
z<rd5n{43JuF`e;x$z2d-So(Nw^{ld)v$yV;AI{pAtDZ5R^^sMpZ~=F4^nzK3l|0#*
zKN&p=crwlTzv+~vnNO83oOFnkzLEV}@zT*pkALgBX@6{9DY~~sR`=e!w|k1c=4fwS
zUUBVkorKuBDFOdyB`s=nE{WXJ^Q7gIH(Sao_tf5bSrJ}GX61b3DvE!du9J39ernG}
z*WWpn54HCEs+G(wU(=<yG0LVn`F=)n_pU{KJI=2-@bmucH3uXrRaI7NW@cTDQi^&c
zJNxd&+&4k)GWQSvUS6Sb*;Mz<mV>toxU=TVnfWTUd%R&#{N7Qa^uXox221wUo6T++
z303inJ@x0Evu(4p&GIKZeZ(Yf4fUp%8mQJV8b5ZKbVC1f%<&{Cj%&Qe*;bJ%ldUpZ
zZmvK3JcI3~nQz6y31>DvxW&t|n`iB1C(najYYpGt=_%qcJuI?F=svsCy{0QnHM_UA
zt~bB3n8mqK@u*aeuQ_vczRR=7==E!K<PtN4zOCjCvDKX?xWZO?gYe&}Od2z`<hD0g
zAGWd$39FgLc>KslGxb|pr*8{;EL##VIrOIP3EtA?DNp+{r>uxDEY<p(RiAJ!JK?I=
z4FlE$Uok_bic<?4CoG$8@H0$PD|7ncXNDbWuH2>yOA5_ih^^5*FiXOE<Dn(7`<fD*
zRFeZaG=!$5s{5!rbxu*bQ9S*_vh>UU7D-3PtLvDZ(6fnVXFa7*nfdkQrcJLtm;GEb
z)hx`wWI}>z`ELKDZ!Hm1X7DbW`e98}p~`XF3CUHfv^0$0PWgOz0qb7LJ+s*ImpRrX
zZa921V%qr=ca6=qT@qU(^83xBOvBd0hY}8Dyg0B#e&X5(ci080&Nc5SUl0?!sKs1t
zpD$~OD5t@{SeJaImP2P0Pc7u=$TPcsHFeTVoy%VB8+>)Wj;J#S@NA3q&3W=x#4O;E
zG0)xWpEgx?%`q&?EI7OOZu2>Xy5K8^)s|WuDqVVJ?%SPJn^wPA;L`l>&<VXH(=$73
zvlz2v%aSvG#X7CZ{*)o0@<rQl=Awk791#ZnVGY-<nmqKH7A{R_I60%+=t5Jn<&#>m
zEA>Vzb+hjDR3`{jTZyEbIV?&kpUbx1JzyCNcS%Gd=N;LZ@i%yRn%_oj+MBaw(tnYK
zVu@@LHMz6zTVA|e|8&lh5~FCD`fFw9a^Bo8*>7QZm$Bl8iR4|UDJk!@xm(L#Jd-Pa
z&$)JP{E?a4yRC(~h3c$1JLhEGJCw`r5j}lzZU~>5z5gVmoj>NOo|(HYk;D1Km!EPA
zjeS4cPAxtl^<XQnfa(wNRHx&A9-K+8yrV4J|03o1sn^LT(hDx13S{_nc;oM0<qa)w
z_1;7zOCCD4P3LS2*Zq>kIi?FMo~PPR{m8S7eeRX*C0v^396CJ5y0*OKk6Q3RaB@f4
zvCuqj^9;dY&NBv#%6rykh%kse;J^HB#pKc);Rl(z5t{QhakR{9e3zEn+0V9fP2G*@
z8|pqKoUh4S%E)qjo<(uL&5@2HmeG%5QmSPHU$RY|=r}XwU?9uoiH@^Ul;@P?=GkPp
zYz%xb^<>M!-WwWQe79|wBsJUb@TZr{D=wDYdTJcU8PDP4vu(<Ot!F>VPn>g2oXz-!
z+Lv8ZAOD{*JMR;B^<ys4s4Z_BojrLxxYlw8XH4|CV9Du!<(v^en@x4wgM?qwr!QzK
zZ`k3rAxoDdJ87w4sP+Yy50;874_RhQcg_~+KC(|yVY87~sm|Juhi;g5uj$s^yue1+
z?Wo4y51C58Z)})VvhJh&wFO(Y{$703#kPt`R*+G*#WzEh@wC?gT_xG9Ri8zsx7-bT
z)S;KT`Mc(URg()HZ+x;~3fpaIcI_ogE5r3K(x0+T&dl8wGrjQFsUT6gt!f^t5B2*s
zY}xjCuT7r$P8lBgwoHx!fgfg@9|zv}SUSb<`g>vK<~ZGeB<@`j3VS=NzdjRuY}Wfw
zbx$bc&o|#43O9W|<q^vq+qTy}W@@#K#P9ltYdBu9Uo`CPwDXR(QQ(uz)4Q>Y+br*a
zpZbw^S0}Cs<Zv#GnBhA+?z$!8f$UGcqOy|bG(z7swBM5Z9PM^b?Q!1r$}1}ti+y%n
z9q5!E&AofG-=A|$XCpRqhrM&sE{f>*Y`P+Nxzg?r&KsUPA7Na@>*6i^hMP^Db!F2p
z-3s1Ke`Ge8sCI}3bycvQ^O%$-&{NblMaeIh(dmx9+uNz1($%;E=Pwb-d%&=paT~M0
z#`)B43D%NXJR0nE?F!YCq{KB23C?EyU36-L$~>LgiuHdZ^b=>#xe}N7i&K~9G2aJ6
z<_FuF^ry+XdGBxMT71^P(Q1lP73<{bT8HmFe$l%mxHPhK+ST-T(w=Hv7pJq>9qzch
zplRyC4Lf=!mntf*-5twvx~V*GhBQa?<~pOSw#beI+3p03n`s9pCj7f7tQuU!)xI@?
zEkosOhk*ZWPoKN*RlG}HDzDwKA(GXMb4Qhs;y2D^+WOPP1^Z-}6nk2@U-rx`+FBwc
zxzK`3>VaMVsUHu+q=W97Zp%oo%}hUdw{-J~a}V~3F@5{QJV)>LrMoc&a+4yNLzTUG
z&k1)(t!3w8*E+6yuk5moouKi#sng}lcm-d#F87jN)*&*vtXt>(!$R3ZXEiR~*E_aY
zD5S8c!KW~AW1a2U3HC=*+?oz5P3mI$Wxqv|pPk2M*M+Ec!5lw(Gu4@olt-vO`zW<>
zmFUV8w<WJXypXwY@c*QBP1_%xRF~9I68<>vmW?GVZ~QM8Ca$vemDbayy6cKsxwL8<
z1qe>vn0EHz@0GWXdx$#OzGJyN?VnRdl~Rc9!6}n-nvSwR>oEBq_nm)@MnbFCnyqik
zGE#r&L{wPpt957iz16+qrG=#2lKa||-5KwNEbm+Ya(#p%>(e)Tat=Hy(~(XuYpLTZ
z*O}_d!M7{@$RcNnoBk!bvl+h|oQsdX<;dA`TFglFk^jYmKQxqn{+jqS>_wmK=Qnyi
zuDQYd-9KGxJg3U_PEkIt=O7%*Y`#h9^@FU>_T^n~t-2TVJy`d6`Lg1&&SYPkzcU*=
z3^Xs^;AFU&no=Rs<@fwUr)}@kZK_3IJnpRSJy|(Np>p~Gme~0Qn|Sj|E-l;Go8#dx
z?diN-$&|5rd9pc!rON}p!0CH~xC>V5r8jp6_&jlu=#O5rq-PbIit7yJyiN^QCV7b`
zKQ<@%t1@UTjntZ;mF$`Q@VLH*rRx)~#7K?uMN9L7|J=Ud*41--BWJE*;h&}7LNx@{
zo~l1(PR`w~ZF**PeO(3X@0Oh%<|^r0HsPmE%{%(?mWc4JBe4^w%vpGH*Hst6y{3<z
zUA7fk@P3kY?J>JWU;BI?c?93y89J$CxvH=D$)mny!s&As%B|g{CACeB&wS_8zV3JV
z)<+#CO_Ja2y+f{d`c?NkqP2{MHqW=FpIT|0c4zg)xy@N2m8^SrtlTpxbIX!B!s^{;
zmn|=g>d4sZxoxjw<DJtwCM5#Adn-3r*vB2Yzy9Mjsjcg_zst~4>~LPFbTT6B+~dyP
zuCIBacPo5;Ic+*Aw%Yce$j5{At2ku5_6aU#ar-d)v1GZc#qXR`{BBdG3kP{!;rg?3
za#~GMZ`zkhd#2QH%y9g&O#b`iF2QHakMFSln;7c$Jo3Zme2v!@=UZRBoSy5y{1`{?
zZ<mSt=g!HJ?A7SKY%9&mJ*{_-)4Cv+jR!L(a!i|^nzCUhugL7ZeywIVgRUs~`LXUu
zJ6GHEagN8!IXwGLnyt5bRJ_M^8Q%mQ&VP#iwKq3J9yQ})Tkdkj;)9vf$F%t4{2#+6
zKhEZpU<<XdjZ)N)x?c7%@6(U4&HD{fV!pYWOK-n!E-9FN%fY;W&sISG_aDRDhf6b8
zJ5=OVX-u$WkY2H@kty?Qh~Q_PI~;$?y<6sHrp)@nG2?@XZJX^57AcnHnTlP$0UIwg
z8ngKpr2Z2TdD&65O*DMR55_$^=Sn}Ct2x0ia{6Rm!(747ZQ+-MS!05yt_+G>?y-u^
z&s|Z#q3Oj!^@sn1Cx@>;owd`wB%OKt^@7ihuH402PnIuJU;VSG_}6cj{$s(0LPCDi
z6E+zf(7cr_C3xX^xlPn3ho4tYl_p#N?=FpDdcNiNt)PT8TffRLnB`(JE8-PXR-5Rp
z`uhS`{6B~vww~wE%d&f_p!syBe@0BXPAPK)`7@SB9C5Lj8}5^k!gOl!hC2@g_<RD~
zA8##a)M2=}Wwy!%JMOjFpKG_Ow{WsdcE5eT)o>F-iKnK6dajv%kI_X>!T$B%S+^y+
zd_TLPqr<pmO5(lExtr_LOy=>IetKAv9_5y%=j8G>@pJC?Ek_kt`7{%3A9}vaNROV!
zdo?=$^p`7&PdSeFaV4cO%@m2O=e;w%_eEo^X-(sNU$J(rnQ<?Jx2%t{biGlt<v1Jj
z^UJ>1{4|zi2`}EirF+X3^IG+*MvOAv4&6a6c_mer>=(>F|K28JG}EY6yF!!Ey38~2
z;04`D+>3>x^{VWzSn<CM2{^q--E4xqU^0{XvICkwz6T$%Y<V}~_sW@jk~kwSaz=_4
z7wbQ`mdxzW;<4n$VxAAtZ)Qk-=(x-KZ$h++jE|sJ<W_658&kHi2F|(|VR9<oTCjih
zk{K>3Ul<PRFmGVxjB2^Pr0eK}iipOd9@%NUdJ46!T(9RH{4ZhFC{wWbz>Sk)G9Eg%
z25vQ9f2Llt5>Qr(%)Vi{J74OS4yXTC_EU14Y%eE$6u;>8=iif~SHg2=-`|nFMaX5A
zsNCu^_m5xC6BJ#mW$nHyDRHgK?(kO0<UFZehj|TJb(}&?>i%6xUdkn*o~xz#!{#c(
z#L%VkE0kDPJz5d|WT$=5`Bf%WmoL8FHd}AW!TNaNY@ua^4|gRlb-8IVYpwYn&FL5U
zobuEfd;b}35)n$jX#Oi?Y4m>f*C!2goco?Svd^?y(Hl9th->~#4b{XlE3a4unWQ6$
z;r@9~_}A}Y++gy&C?O$KX_Hs=nsXOd2CyIh?6%A1-_vC;6?-0T=4jX2mZTBC^qPuG
zWnDqcv^8~cYJq8KA%}O*le1_LTC!Pl#d@7H2Ws!BzYpx%YsRE8MJeO>#eJVYan4m~
zeR01t*Fzy~!?g1cV(lL^+_e50nz~w+LqzuH#j{Umyy(oks%CC+X4j@0X}k~T-SBKu
zP@i9OWzL#kw<{`cG;1tbTkgZ%XEvvC3e$7XfP->-FP^j#In46*J&!BbEzjt>R+lbb
zQ-kBpi+A68rMIDKBIlEch3kUSTlW67+bk2eO*Nu6Hb?h_px-a;gezB%*pzI0r52DC
z{a_Ent(3&0d1gvA{qku`KAtRn_#-hg!q#-#8RPAhU#3=M%r{t@DXn+lp^bC$<wX5I
zcgz&+(|t`?=NGE=90?9pKWjB1Yt?}zHZo$X4*mUitc3r}G0uO<rODl!_LP26-O|dW
z?-}#aT(|1u0-4r1(REwb%;1P+-Y>$Ro4qIP)16?C%1y!IxwliB3m4j~sWzK7x0dhI
z#aHZW7Cn18VX58@=l(*C-?oc1{#x98FUT(WVy2DhmZ$te%UHurUimrS(GWU0CHLIP
zBS)4CIy&|x=X^JK+`_W`<lcy{TPvP(E{nUln_(4miH1YHnP!@@;rp@#N!JMq3y(3_
zKRK`=QC!;SP3p}1dmr{jZpwUS(ei1{QLmeF3`s^m9FC@l%gQ?N#tPiI=-n-}v2x}H
z$(Jt#oGdplx7uOw(9gFf{^z~4yIou4k8HN%*&d_ADA2>SrG#<r$IFFMQ$H}jTJrW&
zuSuiO?lZhyOjizk-=v^-!jCWSuBDm0?1oz&)~D5Z<Yyn4WBYmQhq-yLbvLPqZ#=cT
zOQGTE(@@`kFE=cyUTO8WBR1lM-u=amCLh~MXZ~(0Q>>p|)ShJFGDSGP;e>G0oC4uZ
zOHQ9rY}eUQ)N}EmO&!ymoX@9Ojb5&C)Co8$cJ+$Q<Jz=S=bx^t5`XdQZ_oCaF9ocv
z<s8>H+<6kwa+>S)4o0i*rqyevF_o-PS@*Qae$HyHdozAIDl;teX^*kDj5q2yZECu`
z+cV}ir`jDQ?KdeC=doRCk}W^~{lMvy6Pg(|NoL;{+oXI(YF8U)%7T-}PfBw=4LNE3
z>*O{aG3(XK7qCn-E9G9u&trDn#U^8UX5ZQgCDu;+-0J!xcJmgR&&`z7ZRr<KJ9Z^f
zC`F1huRq$W&|>Wce#5ORi>0G?-Mr(G+Vz#!)$W&FL~XOP!G;|RD>?I@J>iRuc%kOu
zYQ6qQu9bh<_O%yt!Y_qbRGX+?`sCVr^A>Lt^K?&@5H53Xw=Vm{M-KL)l^Z9pD4)9g
zne$|&QqlP(?SAvymfrkwa`)ulC1yUKOSo8s*dDE{5tfr&EMC9fX^qqNNrHyc87kKa
zEH0W=Ty{0yH7`WbeANZBqlG;oZhl8to~JXtSao@d!(8rT%=^S;9lEQo`zJ8x?SFa2
z>D|ReVrI!FQr8_kfBLhih)#M_sN{jBO+sHBtR%a2Rvf6B!8h%W@-y|7*@>$Zg;ee;
zzjD#PG;z&^h)F$Hm>MU)@#IcSN?n~0`K~9~DJpWGZv1JpiCpH~%Y<$(-(9jxmG|Nz
zj(=GJM-3fSSHEa|HK%u;Y-Cc`;v?HKZ-#D+y`+ETa_EBy+p~w*3f@@p@mOBuSt_#a
z(d3V&tp9S(yia4i-ss7!<9X)0#LZVrj!6G1;R$WyGT@tMv363Qdu(!rQ_=H~8%y;z
zNFKd4QRn(CkzFG8i)41VzWtaYYxr?%py&go%0(`1ADEf5O{Q!zSM>HvdeeCPtlbyi
z51oMq@x0HuC)vnM2v=T{%bow>q`ItSc9%?i=&c0>=O$0y^kAE%XW_K>43nnFS(=$J
zr3z16?-h1MMd<t&38SqX=L5e)tNn~isredQED&=&W&4NR!n~D?@$y_R1$9?T992^9
z+7+%K^Ze6GC%btQ3?GF1KiM}gU29v&vvX>TMc+6paY-d?Wb!rtl-YAE>w(<ciE9q-
zV~jbqG}9n|tCPa?%#sI-8(zt_F6_LjC|(#mb>6P1g0?fK^^D|%UA<U1<Er}fYo#-b
zrC%zPau~k2l`u>0c4%t2rpcvyjvdQfC$eu_zS2n9+~UHwjb>~0IxehjHnLq7#5ilC
zb<+|p%P$`b65FEsE7woj;*<Y;q3PX3i~q4F)GuCPjD0KBW;5Y&wEm6s#k;xGD;V2W
zz3IK@^ZD39>DjN|KA3UEOXIDK=BDFDZ^ZTV=R2%V@D|rJUSVX;P|E4Kkiqx5WZSY`
zvv<{qKT*xuFFWD1+6I>oXT;~MWEH=r#u;(Qt;31c?*^|0+cN*--nlcG7?%0Wn&>LL
zG2rcikGYS_1RSFhpZe)c`o+=WJ>#+LQa#%>p0W13E~i=@Y_zf9;h2)<d%-C;VLsQ5
z54s6)2C}aX@FdDTsjs*vl<T`|`Ks{Pv){t+7#107xn0j)RrO|b+QXD>o2vp^1cO{;
z3^nVbrW$0q9$(nT(HoGEbo9fMPB(_cw^gnWmi&u|NRqoLxM=ZwwYg0*%snn0$jrYK
z#S`e%Zr^s|v5L-x!x1eTavL`|=YM8WG%aZ>TiBD+EI;9kkJfa_CL7mj?svKM+Ity2
z-n{DlqIE<iSB62iL-Jr)CX-5$%ZAT6j<3@`zc=&i+5gf{FFPjnz=lV?U(;eZr*3qZ
zG4o@>p-*i`6yGnHYuTps_`2W5s}m;$Wnb|=@9;IbRCktyRjA1Kvs^E-x((AZ4%JS7
zRUB@AZrbvjkGZwC_ve4Q`Mz_nPss-9D#H)ni_VrWnJ8etJLR+@f97-(&lRDMC#Nwj
zDLT-U9~gXnbK9$x&F)t<&xD9(oUjf3!|HqNL!a-RU>60ur2QW(Hyl1D_-F5J+X^if
z*1)!N+wM0r<zJ3%zPWoNTU#6}Yf{3*_WMRB)Xeuj3<%wqRd!;2;Jkx%mCN5%@$U?D
z?_FnQSNb@1_UZg{w&4%u1YK?@eB<R^!EscQZKlw^j2CmK>YuQa_!a)2Wuw{qlhgT{
z&c2)2tG&lh*!6KkXrbT&yB#(9&e<MIwNw|Mx^P|l-!Fzf@1FuMCE4eloS*5ce5cfA
z*R2?fK63-N3HLlscz@1+`y+P489%q5dKQHe<sW-z+>!kk@Gj`CUA$5m=lW<-z3s1E
zcfBi@UE^_|p^;^p_uSC0RnE7<f=|0WI=hHdf}=}riOPBphql%`DltE=SGi4ocA0DH
z$COS^508)MUhQ1?Pi|FQk<0auwNqqf_1m76eA}9((PgJ0U@o%Yzg7{qJ8R-H#VE6#
za{Z_4vaG+{JZif_yW|1O^v?!{PKVCC{dZiXHR)C6vZ@OgrdFh}_U%2fbot9!>#h`Y
z>REMY9ydGgU>VAA@XdkxD=8bCC6{WSR=a#?Da)tj39C*zES>VBPj*XoUxpp`MD>8x
zZd)>g)h4+f{cgaxGB?38`|(PNE~bT1SGew<+C4kJe|21B+gUdmzk}y?etzJ?yED1@
zmf=B>o?zDIjFaNhB1_(MIWfImV%pEG{q8KoH6yq7OV{&V^`EOd6)?P?;ZZJQ%%&ZB
zy=>)7yG*bC%E&`mX*;C9?w-HJy2R^;ztyrS5i@wpr<h(Wn_b3w&898v?9<1`m&r#j
zoqgK&PFLJ!JF}T<YA)T9VV}RjtEl6kT<N}-B3!$UZ8~2VU*I-<QcHtDX8JLyxT&uh
zY!glGr-<HZ6R_iUntC@vUTXhYh3AS7?&O~M<GiBzYwB{@P5h>k^Jl+t&axD7Hu6$?
z)H-GBL3^zo`<l*jOQ~7R&+l6~VY{I!W6&g?XM&0|4gYd-$%j_mT=BW<n&$+0$v^wm
z1+$mO9W1ocUwX@$zf{}SD0NSGh20(z{cEC!?%lFAI?pwwrG#6w?*5CYUnQ4CkDbc<
zZjgNL$?TVQH_H~jD_^nC-ueFF_cx709|_*r`iE!T(skFH&hNj+T<@r*+?=%9HU9EH
z-8b_#DO=~v*(hgYAhSyI%xkMJ?p@*4om;-uzH8j^sm0GABJ=X5KTJ!Ee$3!4TROY3
zMo1uMY5AqIzhCZl_`}1+q{9`mQSoMnNY9G1$>k<F=Nh`#9G%?TBsyhk&aBC@lfHC&
z87xrvJK=n$<$58@GYx8;lXe}ldRC#K-4U`+czFe*4dco)+rOCm2J+uncH)6!5M!#<
z-Mh^Cy|b%&#C>jm(TIrF&^nkr`CHyyiQv@vyw{o!WTx}zx^+KyVa!~4E$_RlP}qB8
z*_cJ;uim_qSf6XBYO(q-=aTEJo2PQGO695GogRHgf{nxA_LhyyW^k+#?R>MuVMT@R
zhLBwcs~4`}@rk`5wQrkscTY^hZXIbizUuWlosnI^X-`Y$_E}0Wma7^v`U|`}+%fBP
z!aTFN5_0T&SX0?E#QwaQ_(wm#$(iH$i>W#SQxEy?;;D#yZ@}Ut`zfH@bY1Jz`EQfk
zWW}SVyv|zV)z!dvzOqDN%7NtNRyDToB&0gBVmr?tzN`3o>HcSW&&7Fyl^zMrJ6by1
z=)R)R%0p*sW{0HhzLV*;s=sUgKJK3$F6CE0XDuwy`+T=5{=9;_UwD1uIr&!_={29s
z-)w&K*glXeHK(_+=a#MIM2+n&OIBqVM9#C@e(UijyFEvJ)jd?VEqt+#O)F!1_p&Te
zPJ_wM>o>hm&vg3#S1#^atozRL`6YW5`z9+M+4@*z_J)0%^*3ePX1O@*mU`Tk5qaqF
zTx-R&toF~hX8iV1yFOzLKQm9M0qe<3R-cSbtf{>zjJqbjG+4MmFt|wZ*lF9)(~hR+
zwI=i2KB%O6aJAkWrCE8QFP85SHZ@hdr`oOZ)k6KZ&_(4bc10ytTRgcps>^ND4X>NN
zd)w*L_KPe_7cNsPY>vsc6$sOucQ8|~#CzGM*trisSIn;OdblXz)j6)m{jAx7N+BEe
zh&`Nk<4un|qv=$Me!KinQ=cWCc>Yn!;)T1<>)?Ht6|H?$K3ub%+LX00XJ6f6&RMCc
z|8!+rZLj<;3*YsuNhM;#B6&^of_G9bjd4%)er79als$XXcld+Ga&dVkBi^^Wm~XK9
z*uLPaIxHw^Cb!`(^T8*sjQ6J&xkm9ePU~@LV7~sqt>pZ&gOX0JyrBX;6)J*(jJw$n
z&sn#s{Kacc&)>#u+b_)NJ=N$InPI(v*Y|`vqseK_mu>%d_*b|5Ii9+%YHNJJ<*jG9
z#6F9B@SUmhHG1}=CDlF5LaUd?OI|MMVYc_0v9fQYwxoBgc#D&YutRRtTrsD_r6r3m
z8$VJl+P>^djOPQ^gpcbO+J49wsO=Zq=<q{xdV0@lhl7WH%rICm^~5V-->yGtM(f1q
zAAT1)`H)S4o|wwl+Fg9?ejYd9vkJJ_$KR~DZ4#8X`_I(fHgk5{%$5l7_Y?fO@Q>`f
zQ<B0ut2jfR-Yw$Pb146``otbR<98n!YFtCt>D<?=3^*Rlrt2o}UnjX^RYB(rC5xUF
znJ;B`x4WNvlV8`rF_rb1?YABK3Y^)lPImm5a=|5Y@|;fA4QE~5UPSIV>*O@?@vRH}
zsdbqJH+G)k3!U;O#&zD#hKswJZcV#?@ZsZrMjdCrqRB6#Z?3=dJ2`Qa-94_8XW}oE
zw9ok|dF`OC#VfPOK*8Qqe2L%LDj&0*|IYSOLHjK0%$~<h441DRiht1Wn|bS?tLDCg
zobS>()xPG09OUgxzom8OSIy$}OWh`X7W>_uA+f=u?93J;k=?0}Dg>LPrQA|)IzQbn
zvgX8=!ZyS9MDFC(i<35=T9qchC->jYl!M<M%sUZS`KflYyOZBdJ$LVkjzN~Q_+P)-
z5V_*^(wh7F@~w+w@Bid6UFyu_T7F|$$}y=$nME-zrIpc7HcbC{=KVy`+$ovIq-N;f
zHge$&O=t<v2sK)J+EdxVOe#8p>o4afX+BAYtyNo}N~aY1*<Zd@bLW+H=#v&nb>_HS
z<41)V($n;22TUszsHzh^W4e=_NmOs`THo28O&emO^W8rC2?@z6FFc?0kEMK#bn4ts
z&JQ`x-g)#=)cTy!nL9#rqJ*ZKf2sVlY4TT_sFsgi57I1C=5}zOp7yb>?dXX|zgJ8-
zV9&8<iu95F3Wo&rHf~|v9x>b3JVtx+jJ;niO)ec0u4|Sx?g@D4Sn`PNNOE)7qFKKD
zmcLHCsyeDNV~Sv+`tHI9+=A>pj7OCI=rWx?b}~rt$U~N!cY7r4UpKm`_^B)X{nb@0
z?C7U(ZuUE6w=*5nUvGPq`+L*=GZX9$JyHdp$?bn`d->3&R>mynr`wLCn<UOxwcNw^
zN$lk@jSCBp-E!Q)qctJTR`~3yNp7mGhqg_*v*~`i!4*EcY?ss*J6w1DKX52NF+Jf}
z{_m<~7e2grRdjvlq4Z+s7eP;UmF#FvI#cRjn<Mmf))h;0wT|}Z`=fUqyCM3%Ni1~9
zS<7EX>#x`v+%VVNTipNR(vs?~_8-gNNV~rdzL|TeYj;Vky5r($af{x@1+9Hk<>fx}
zu1e>lMNSh>2i-A9>sh+Ix9gI!ufMU4#J2VWLeB%l`g0G@e7gN@pq}s7LJwovOX0`s
z;&=GGT$Qw2_-W>oo(WPcahYnngs;U&`JVXd9A;44$n|Wuew;V^O5uqcmUq8<{!v3L
z`tY@7d`lxP#&RbZom+T*X;0I(%Pi${!xPS_XTRDOvi$KY!`ZGMZ;OfCTDWfs?^f0u
z*>Q<#Mfz_K)N?DBq@C(>ed?%Jr1);#BFpS$;+o47j<h5hUXb>3jW=MJJHxBFYxPO9
zC3AgNOn2YtUbRPf!ZywlGmiCDyWFL?WPNMzCf!<@=XR(?@lQIpyUT(8Z>pyRd)D->
z%{j1?)x|Vo<C^v_-52(~yz&0@Q??6U?LqgSf3)3ewA|o<n6tb2F2{W*>fT-BzN*5}
zxnY*^FUD&YjBc-Ew`^OXBmHBk?_5)s`7))iLhl8H`<55fGio+p$iC6^`m5@WZn@bP
zvR`tj?@biaUl{E0bxD(Gi}%**hFuMZTLf%W1o=ECI%$-tXh*WG(FkWdsO_6@wbrX&
zh($G5^Y!M^i>8G-GTc?Sm+w*URp5Dk;PM^qR^jXuQ*w9A<gs7)wd0VQR@{vS5k=#P
zXAdiFJNB(oWT(o6n~x94tX_NLZb2j)_igb<@wb`3$#8%6zEN%VO`2Q6s^A5${Ab1g
zq5&Ta#pnL=KOo?6lWWF=U;CJ)q80fj@?3ma-!-Mi_1u~ZQ%#SyF>Km?W8FhJ;jSQq
zWb-*0eXo*}UUBWz7I?7pRL7agc~(Xi0bf2OD|Y$X&VN>7Yg-iV{q3ql$cDynQN7)a
zhqmYnJBnC6%-&jjWLbkBFYheoDK{gwnf3eXwr>=h+AGi+cuhG#kZXH4)0Znx8-A``
zxAsk1OG-)Rj98(N=T8r{F~_atmT?c7A1zRmby-vBoa5JBMlsKGKGde!vzGD9X^Nf|
zHS5c!rcL((pP#o}zEJ7Q*4?MSWU%(`Vy?H%bZ`<|9FoXx!uMXReNTcTTi}%mpS1XY
zdoI4RS7K}KnX=NYZgB?xbLYQ$te50|ziHO`OL}vro88@@#<9h&-`rU1)aRuoSLa<>
zs(0$i#n!V$)71hP%bzt{6hw2c%PZhK_;1Bx?bg-yZ`>L=R*3MK|C{u6ss4)?o2SAV
zmp3h5{>Hmi@r|b7XH^Zg%AA<(jT7xMl+R8({?ceZ>(iI7ww>AeAe_fsbNyQ5hZ-Su
zMItAAdNq<$Ps$iAU*0?SZ>ZA82_pBls<pekR{U7vaZfJQ>+r3nj@{FGR0_8@YUoVz
zJnryG#wv939<G+z?Tq_9&dV%+^CkIsa=^{AGyg6XIM)6!cG|-hOUs8ZBJ@lW|7;bN
zIMO3$^>Ev!K3z7Qhi(k1p$jHjBng%<oW7fIIy~i^X6Es0H&<(iZ~M0P$h_DaN6OFa
zOI<Z9c$4=D^ABsZSFK-QV^*?<m;G4V85yOMVjL1@)>R3qb;{~)ZPg5ZbcSWY5ov}u
zcfX0VOMaPV#`L(4dB%iYx6g3#oqv(ipnPJ(Y1ss2zde`V%{fq`?zDx+EKpacj@j?s
zSNC&|dmp%qvMuuevSPL7rFA@~=bchs?|4X4==R+st&NWtxO~&$aXPR@*J0L^^c}AM
zxS2QRGOXu~+4A+rTrP(Q_X|sA|Gllr(YIsA-B35~BwJ^XqL3mMfeM4ROk<;5KTgH_
z8&9T6U0OadlI`*BfbYU@ibTUx)?VBvo$jYtbYcI3aC4rO53O$hycu$D!|td>HB5h{
zj#O+*iJG3ey?ROE#oWR+-YwInJY#%jufiR@R_DT*g2!u2&fQ&7{N8A8ZTHEC=3Gp(
zTC#SsPjX=oitQ7axoLxPl=yzZcc&g?lzn;Jrk!+Hkf9-vY5#=h7vf)M&t2+#Y1fO{
zIxU9EMZK59+8^~jIB+x4_*3m8#=;7v`^`-z1--J(8_s>1SdkOa%lZGq*IT9+cK5Xz
z{Lam8O?Vrzd=t+ajWWg`w_nC=C@*?cvFiSYh*j5PJ#D8c$uql~8|J(2RBZUCxzWPD
zMQl;_gD*~}WrTkTsQ5-WiQY;$AobO3#bM5}hCRh=_k_M!A!%xzf7;>osaMDNSf#p3
z*xj<E-cP$5c5Ef*m8OGbPam&lk$K2EE9c$R-1EA#BAPw-OlaU&V<^kyUJ=Do%69FP
z<2m)`EGMQ-y#9~f<|KpEU76#iDaV7hxVn_It=YKQ-lfa_L(r17x3_R8t=<*?(!s%!
z$-2DN-Q-WN`cW^p<0od#T;`(`$a31DPv8OX?%6t3O&3ma&nx5-)m^=K`((ap(rud*
zg_otsmhCfOKlNe-qrXIA^O;hW6d{}Ic_xW9+q&+A_~|RImYo(oN3*p#oB8~{r>T}A
z$1Z$URG1{v@@7#$0e{}J9i|HzW+ezq32BCy%bYRVd@?I+$px{8OZL{pmszgXxYj4g
zSDAZO={rmF76FDuapqhH?PhEi-xbMuZIak)+k+FTyG;_4Y%fZ`=K63%xA9W3(!@#J
zg=(FD%p5ZWFHc#bEqURtV)wcn59`{CTONyaoC|Z3_*H!SX`B7W^f`;Zo{*{cdbM_j
zul~K$mqe9ix(xJFJ3n;9Bvo!(8gY(Qs!Zy^EpzMU8uh2HH;lQX9V@&V63bpb+_9cn
zc($HYlh~}Jj5f{88Hwz-EDsn5*6n<FGUJEbtlM9^lVmv7mt4HRedDBM%g^|35q0)s
z>ld8ZJ=M6rg?H6aF*SikE2T9VKNpGiHy+!S`}sjody>}*p$4;Mnum5SI^*uqCU|>g
z@zwO%)8#MkTCh5VU69vo?!sbE;jEW#ue&lH?_&HmcS`yrZe_)Tz6-moH_p1TV;jSR
zob!xV#8x_WF7nxCAnc_PzigwX<6r%{tnHJPm7Md|FW(}0WbuZYM1QB^DxTQjAJ^AS
zNcG&v^7ZHTP1|)IPw)(#Yqs~qr%6tG4*%qfZ@bjEo6C4z`?a>O#zkE&S~?FDPF*y9
z`mArQ?T+e8#r|Bbvp3c~J~Wea+J=&M>WAD6&R=3>G5Q)@xLRC&%Ir9<Ws^HE%vNFh
zeWl^=GZoh((GfQT7v6rh)goOvMxWtjz_czwwH|v3F<srqkMCTF&D6Q^DV$;Zvg9<Y
zZ&fe9?o9u^<YmnFl@jL!-nb-u(s5ancWd3f-m5B3VX?m_c1C?%7V)Yo*~G9_{gCtX
zmzT-}q%%799b6=t`@?s;!JFIziArl`DFjP%AAXd$Ms~Am`pId2n=6?s+;aAa#mti4
znP#|orh$8qg_zGjR!&=;ZdOivzN>{n_tM!ia!MMlZ*4dr$gUa{{`sV4_f-ei8y6Hg
zVm5i~6niPQJ%IUw;;)kynJt!)(R|Iv>p3qlde?5ay!eyYl$J?5%+^J`a4@<Pmc-J>
zrPQ^|b#cq-RrkZEO)v2`yC_+?&~VqD>67JFFc-EzQepXTDOlU~Gfm2)xv+l)Q|+-S
z?jmgmcX)q!+*GuLq5nuDk7t3>ALSQork;D|;+oAOA|df*WtxDKz{917CZx7jd|$X=
zhoZuSP34o)&Mo{W%~pBm8{gF9r><RWG*WC)|E9lYn~>{it1B@YW;QD(er#S4<H?#-
z!{KN-f5ua{*?!*z56yV_W{V5&?X4Zr`QJ}4>Tlm=<E~!c5aH1HZ->dH9p8RWwzfTR
zd!F=F|HK%f-3=U$dtTjM<1?MtCoL{laQz_x{`PJwp)IqPT;0d=KJ3~29WEQ4RDG72
zHk>kxy_D<D<Nt4}w0?Q3k+LLL(t#&D9XykRmNb~&;z+1ff3tw&%4+GEN8J>HZy6+t
zC~Q<dFRU=J{khHQ+Q|nmsCs25II|Z|U-C(7*^~>apQbc5MI1Z9zjWf9uq_7aOqY8u
z<a@9@5s6h_zT-gjjL<{5d*-})SUTfckK8q3@6X<EQ*Jp$C5j%MbY#&z<z4L>(@G8U
z!Ua_qUOu?$Ny?9>yRxHFb{V|QoweB7Yl5P(`z&Vf>EU6l8y`;J`uCJiWJ+WlUs=6$
z^d-;hY`P0CCqD4r6UJ|v<38s@z!VjZZC4Ixnzv1m{BZ8+xv;*iP3Bqr=dT`~z2!pF
z1`(md>ki63RjN5Dax~$`$txR}-<7bhaz3;~Rlaz`SG$D$d}W^m?wMVYeYE8G4C!ex
z+B5ha@3}B6duyS&#?SW3^wLAp*^lMkJV?55@C_gD@*O@(jM)?74^No1U{au!lG4hW
zzfW)e-Tt9zZR8#Wc22DssbZ7rHfy}7DdD!?#2_r?x?yRL6|1bOvbuTCpT7+4zDbcP
zCO!<kSCu+buW#9AT<p8hA(nN)iwPkc+S83=EEb9~8gHBM^n+<L!#>|l`wqxQ|7d)b
z*`#-TpW~MTZp9p~6oJ&1;DZdp(m(D#xVA%bj_Bw8a!HEEGC9<F{=E13&9@}n@mX;A
z#?$K_shwdkveM%AUwP0Zf#v4;rA%vXHaw8hh<~jpb*wVuYoff&X|q{Q=aeEWmp;w-
zx^ebxU#DFM|CY2kn%}oE^(|L9c!Dum@r0)YXNus3C5wAbtGfP))Sl}y=|%9eHKFBR
zLe;VF1-Gnb6W*%%>4)$Z?XR7Jv(r0&<qE_qWh?&K>HDYJF0L-&*e0#-oe@{XHBxS7
zxco8CaGzP1+}SpXrS<JQj|`8-7Q<hC$`9A_@~J;L$o8ViBV>Q^>`!y{$gH@NX@9af
z^wTWwxQU6jkKIqccr3ZyTkk~pQ@>Z?lkXW^ILywcqCa7$-s_b!=5DMH_qkOnX(7BL
zdTQ?FZuU%-w9i%i5esfG7}_r_>i^(T+h3{mvixVMk;C?igh_XoDd)sSXmPV$7HUl5
zw0_Pxt^e%R%icU&cc0)j5}vzLO5E<9NMqJZImI%e1qTeJ_T@>IO0qp~)eSy2`HlL=
z*o7gE)j}oqdfeQPmi$k-d}TtZ&#c`nuiJh+D>@MIYVNbrHSf<YO!H1_j*oEjcW+e^
zahsBHOV52zFGH0_@l>%zXLc$-W=oLS+ijL2<dc`l8J_(t@5llN?Pc28`i%Wv7H8cp
zPTV#+y3V&@!rV#AZaOWB4&L(NiO1>*NA(#l-*J)px76(ODeY~6;pa}A@tkAX={-;V
zlg+90wMXZj30ddqzAQ<6X~AuYGjW^Rvr0<UiiCu33D+(^#4_y$=M|&B?t=a)IsXq#
zOZjs|Xi|rM*AhQBrNuMjn3MD_2DeJid&m=cpove(Rp)+PWxn}eLyzChe*~oGc2~~e
z>Ob~Y<Id#JRdaZiywl=-cCfWFHH-4S5k5Xi(EN!uPuil6Un`@Gx8+}VtobvYBVCZW
z<?hMjVym;}-p!O={{HcX=V!ismGM++tW*n_$Jw*y>w&t1>CQ0|1f;JmEH`+vrtj{X
zn|q2yZy2#}YtQ?Vc_sgE$qDh^u<JtC-6u#qKcN_Ntkb!zxN82_^D~4_toU`SYIa|F
zYSXK(ndPjxGfQvHP*vI+wM@}cDXU92)PGHWtiaL_V)Jud_DH`A-z1z_y)Nn^=lhUD
z0=F4wtJy?m-s&>Yaec%w$#JF7pM*Nrm}&lp8b3IB?enp#So!UD%yzdUKb;Df+bv)A
z{-@;EB=*PdJbq$9LK`2peXcrtw=8ImnTX&?jaakgQy(AT&+pLZNP8JnQT3>?qRl<w
zW<qDjAL$duY=y0TkuG}cV>kKdFIP{uac!3n_;XtN*os|?oK?(34CYBLv{OFwylHmS
zv%*@nZ_W!0nh!B9mgsw8k(PC3Ba>&2(vps}K(py;O|3QJ^ZRBPmOb7$=Q8`<#$)ah
zOE+0ubf4$^YI`YP@Sfu-lMFvC?l|@#L21Qli&T-!V4bL*Uv7px4?pJX+@7|6-L^N;
zML7a;n$^cA^x353Dx^Be_nU3p^XF`?b*GQ0bmT7fAJO|>M5Q%q^1HAmYWn?n=Ap0g
z?Ez!2kE50DjmO+<T<r2;>#WVp=6ZSMK40ZGo9SlEirsx_XZH#)bJu0}dv2E8q|vl-
zU9&Dve%{Viea_)7s%@d}LX(Ugb5<NUI!mm3pa1D#38~7NN<L=<drT~SyPKZKmHZb<
zI(c&K+EUSb*V<$y85NaNQ&?Jh#NrNLW#U}!_~50LL*Scs!CC9M3}v1ih*-XYVTpLk
zGBKYeEbbn~2M=XF-FIom*3Gw5&d5KFythG6iB+IL@O#{X<cYJ7JkaxdutYfEnLy9H
z{!@#T7cQ?9cqwv#b={0}@8TYuON-kw^ZOz955DL0OZ}DvZhLvG;F?)yL={Uqm(t|8
zM0ab|rJYvIDvfb>P81w?CiU>kF10<PyZe?(aXT!(WO!}&nO(Qua;B<m3(Qu2V(>it
z*vgmYQ&Z1#6&P=9;d)&3T=ZaFXXg2rVZUrR7z5q@L^tXvd(24Nq|TtlyP>ev>G0V<
zIx~uvUC}bSud!%V@vWnUjsHxhU0xTSBRY})a>3Q8XS0@ZEoCd6q1JG}D|_O#L$h-~
zWdHWji1??-DYWvyF{Aqz)yfn$=dHPr)vKJeG~i%Hu$xHSuKus93v2yilP&IQg>U%M
z_EM{V@=Lx#D-Nz(Mn977-7>u8mT|T1O+VWtLya`Ch>aDm%;pQN{wyQD&fsyTS(<T2
z0GE^MqVlhI%~=1O783J}nP9-She6W%k>aF~&e{FR5)qeH-f7Bli+j@X_e0S~gZKjs
z*A3^ghA%ig^^ndqzp#I@#?g*S`a<EmRSM&mEb8IiS(BkGC3O1tr2eg!<W*w%*E+oQ
z{d``LQDc+1%+>S-uSAO*Zht&epC2LA)c0QE>7PS4%ubyQ+j>^^pztQo44F;UyhnVt
z>&oqP+nU&4A{sLHqd@58yHi3Bp8K@?;kR$4Iv>|-s@+JwE0xr6XZkL`^$ot|2Oiru
zr)BvS8ol7$u~YZus^d;B9@b9-1LHj`X17<rjX8ew#}z*YPw@{iURiJQD?I~p0w--a
zAhW|D<h<Q-j^1X$tj(4If_yigOq>2$?TmwI(}fS&2TtZMKg0b_$)iv4>cxieWd51S
zvu97*@Rqgv=eo4atIGLrPAJ;?u;vI`KXN{JYvcOZq{`>9(<aS$pc2-T$8FDctLNk!
zj(;3;0$I~VWV)9ttUR6bt#zWQKkK68-cZY&RW{rFcl$o}Y2^y$vYA;wdxif@W#?b#
zCgeRk#y{b_UW?!%`49$26Ni>P8)PJz+t)X2oFX!r>GF|d4-Y;%Sj{aN#iDf}Agy3l
zspqzH)?&xA6JMR(Vk3C}qUo{F6ZTK%9GPAE$<A%lfsCf}TZDNYJmHO5bgS=6Mc0$y
zEGJHluGKH!ZYjF6$Jjv5_1CuAnOAlt-VwSNrzXAV@|-Q5tWPo(r^u?A3GL~fka16{
zZ~K-#{p<e;X2+{<3b<_Uy-dxxW8&+hv!^((I9QXkgwtl<icKL`%&gD<4ia8^>Qlky
zeSXii4sPQ#efv)TDA$xF0dhXf?^KO9CM+(FE?NGKD|l5<5N~|Z$<@b`RyL+|Uzoh{
z&%VwT{1e5^T@4P48@*RlQWcb(#1ZRxr2H<E`ke)#O)~d%-4)M@`@J$+p0VkJ{Gt5$
z9SNQ;N7nqWsH-sKT-#Hy{xY|_&GT7TIs^Ki&z$2I=Gbq)(#o|wz{N%IR5QDJnNov^
z>jI7^uO4P>7E=_;F%Zbr-np`#k&DNwu=(Hd1xKz`F4~;;;>46=@{jzg_G-t)<$5S7
z?wr@Isa;|+F*^3-?4WBZ)@GMgS8ge~X}5b@;Q`L=iWk!}1kH30EtB2pc_`~vX4DVC
zz@A-M5}O!=3a#c((o>1`-*$JSifZcthevm&1)lo;$u7TpO8y1r^0%L2(;I$lnmx^T
ziE(~bh;y^u$LBt`ZohiC>EXmLlP0CNt!00$klFm(kHO4$t7z)u6v10syRW^ENP1BH
z+w9`%#mXxzf8ShpQ@&-3#J3n*PUX30+4b{gdfGI5PFA?SeC5;}$3nJg4|6-G6bqJ`
z{o3>RV95#hdFENKQ;(nTcz@xG&*BUn+y3j)MXRT-ev{L3U8=`WeMa_!=Nmi^l`rUs
zIII@+&T-P2ozlnrev}$1Zf#pBlFaxo@+x<Re<SZP$JheZ#~&rz-fm1tIht5tcIN0y
zVa;z%IT4D|*4LjNxw^5}((+KNdquxs*aC;E$ILFQnr*R+?P1i;i`Ff9ZXFy4nY`y5
z+cvY~XJ+Y4SH)wVf_2P0{idJYy7K9EPfHn|^(|7KNrH(NGJkptR!1-;-!xu0N4jHM
z^fA7R97k?+Y`AHb69444?vsC$_cHyA;M6@CD<QdN@(-o2rgE>R?R=DRuc2aX=mq^{
z(Gz{!KFswi68tEhW%{jsQ^L7RW^7IE%bH?VyKH;4tLFYWd7*{(L$$&@zk5V(yI>|5
z_%i&=rmg;-9>#k%G5JMb|8gWhH&G?M@3h_7t#YTgJr3fLy7XW(Z}D5d8yDAAs0aqN
z8*WVMD6cqhk~2qQqS}|3{r6K=A9^W$v%dWBj*tz>o)f>%*rF%wuXz9d3zvhNbKByX
zT{byord%<5uw;5b9)H~Xt}4Oui)wr!Gk&Y+pE7#Wc=EsP0ma!WF>k$G#2s0yHQ4pk
zy&S$wo?d3!*}`_+Z8Fb`{T}yngbeeQ+F~y`IoB!*PGU3gP4wDPFYrZk)5|5PJLew%
zy6NZycGk&-9-5X<)OmKKU$O1!IHmC5)55B_wwJ3j7JJ&vomX2`a7oiNDE!{1<N`Au
zeQWvDzR0GDE{d*K!&H9<So|(LXf|8@oX4co5!3G9tZUz|`>lzkkmX$2$}CL|C827g
zExMf7I$2(~`1d5M89gqQd$8f|qYoVEH~Fe8_#_$Zn-p)G%>SjaRrA>KqPIp*%H}d{
z*L&=Fz3txRJJ)~P{hy=M@F+vLnlI_s&4s4s`j)fWQml?nn`$8t>FT^??^0jpZTr1i
zBDq|jR4*;->I-F^knS30zj@Wf?yHA%pC5~xq?+}6Qtvj;#Nei=)Ply^h}-Y_=50T2
zr?YY2+dJ2nWk&4Yz}GH#d$P`Y^CZ1r53=9hjE#!E{_D+;rHUd3iw;a#(kIZgIAF~!
zi?$B}xfhvq_vy8Mx^{KaIW<Pr#;+e<>0H_HErju%{_Z_T4sAZpcBj(s^X}-TtWrhe
z+*O4O@4uYox7+tcP3A$nrJb1)VP*OOOfP0@Ge<YN^l6+7;5qzCXx6erHaicUUHn$#
z#p^%aTlF*VznXqNPqldQ>vL{lNt<6=NO1D#M65NQ_PS{od!+Qs%P(5?1SGt>`Ld{>
z-I^<j@nzkIt*7pYDM*$alQZEAw^_A@H{s#cs1KT|QR4Z<O^c^8y#BtSRCmkjka!($
z0XE4woS#=tudSYZ#HL82wl1@HdRgkF?J0lnvfE!>*&$T9(8e)xm7vYxH~(yNqH-&1
zeJ_1I;IU)jwqrjx<a`x-SQ-7uoTn?fwnX<>iOm%&U*qTXTuEz$^n#mT>OEqZG$F_9
zWs_UfhAa6QvojMnvUvL}+U&zOTjV)=*k|8Y3-e1=8y~Wsf3P!`XL+OQIU`%|jZqy{
zN17^FS<_P`y}e~K)-mjH>GM|L=9qQDWN~^x*uo6ag{!qh?nlqvnfF;iG&O(nl(&1g
z9S#1x<!)!JMD=d_!pCo4Sel<T)d(qR+8wuHiLl}<Z5v6Ya(ne<fmUTIpOY^L-ClPz
z@@(0Y7PYwv6BfK?NmBef^}MC7{%XNb7o3FN%Vbv0kJ>&@ZlUsn?#mXTo2I0{RVujl
zD$_*2`E}#O!%X?xTV)p<o9z6}!m>GdUQ|)R&11rI<F%NLDkfGK1l^1`V-(CTkg|?q
zPw@-j5>}kdQR>ZD=_jUEc4WiM8FvKo3ZmKNlv-EJT+GDR*v|2gGkL{Ku>@O%>aL~*
z?GNn^9+_!qnwB$t=bdK3V#ZfFGv(K;uhkF~6F+_>f+@7A@)4WLnFF&_7R_usvQ05i
zmzDd#w!-cE_Nx|LS1yd-$6h!i>v2ub45=9=Z7H_?tKMB@HFLJQabR6TbHxYkijR$V
zB+`0281F3N<^H+WPa=GEtD4EIB@27YO|q3wt$#fA^`st!*>@VU@?A<cT24&zU&1?k
zLh^0SpHG(0ye{{!PJ)-|R>q3I)6x{XFSz~>;Qj8u)4gDA;*l$*(bt<+JXvL>b7EqA
zkc(8KuUB@svhBQg6Q_R^>#N$BdV*mA(+;WKzWpu-R;xsN_lWTEvf4+iRsGm>e3iCP
zkzQQm-SXTd71vo3I_mYwrXo@13?b8GU!U%aZ<&?-bO*OT!@>V<IVsbYlyE<3xP3u{
zvq;8MwEU$W?>*5rMu`uHYA5}Xl{Z(vuPVR%{%7BBKYjI6=lpze$8O)tRNZ~+pFelA
zzY}7&|0Q?-t^W(A)z>u3Pm?$K^xwzXey?MUMyQbs%b5xh50`gK*?HF_SKe&g`<7Av
z{TqeHPx=oiYzQq&%iYNm(z;s8drfkS6N`_|`XdKr1lG@;u~A7~?aJQcMLClB|MZSL
zxwy<Gz~-a*^~tYpth2kI_*JZG#m400k9?Z%CVK3Y&Ahc?$y}Bkw_g*SrTJcFaG&No
z<+x+f*=7DpEPp3Fie;Rjvc+i2<PwJ?&bRJP?PvOFqWD}h`x;w_r<>OHO_S<!S!XHE
z68NopaO1-t%o_9Lo^9WrB*$4jN7_nVD=J0udx9g!m6aRhot9^rX|^OM3u(@Z7Cw30
zmF*<sqE7|WQrFB976|i@>bHFKB+t`;!7G}@(UN%<_fPd(SJM0YLoK?s)=w#T%4xa%
zMa#7nUo3leiM1J@zj$ei$M+L=9gNr`Ke32OW?i0=eem@m8?|tTj44|Z-@Wf`p294)
zV-LrpGQUTM1h;wzOR=cRn7oj6%$pt>e&z5&)%QVVXO0-i$arpI)XkVVcSB-Z?3@SF
zKWtxL^GsD~f?=xgtA!T=T<&a{^Gz(K!?Etg;>)!*dc_xd*7-j?E!wESQV|*_{k_<m
z)2>Qy;iaWTQAR6@Ha$&J=sj{x;OVN}hhK=lxN6tiV34n{RK39@#dl7JkBJ=jy;mX%
zHHq^#9P(NqEKr-Au(9|&qrQi5#Sxc-Vv9m|cqBb(7rS>c(z0ni$6*DX_>WZ+9+q6`
zTpKrSvLm-<mbQ7boc^1C9}j0MxI0W;ebVt>q}mp(nX8i5w>UmmW6F7T*h5legK*I%
zZ?pEz!3XRak_BAYSXa90t(sgO_p)}$T5s7k+`JhR9~*s|zJ#@WQk87SUR?(#t8*^K
zd$L)a;>`mr&YcpmzOydI#kX?RVc$(tw{9s|!?|jffo0Y6f?IQT2tA6Qs<7)^uE(Pq
z?%P|#nC5S1Fc!C5;CVdLY4z2EdDE9Cp6Q?4`C(J)ok@+2pEC-Q@~!%{x>vulDSDD^
zpB*|wc-Na>P5m2|F6g$loW3vSTgPYn-ZPfBE;rOk|9jtS-q}%qI=DGB;JDkKqYMwN
zSzbl&I>-F8a_WWp$$V>d{kT{8IPWXCns`Ot+n2kmgkR})T75^r;-nALGZU@YBMd5{
zInz{Y^yL)3trHIJ3SH#5X1%^}gY0t7lbdE(e0SNU&&M@aFIeGIVyD52T!X2M`P*y`
z&bBf<R5IyJij#uZ$q4psHIiNh&!fU;RjhdV;zmQ)GNX7$?WE-P8?IivnP>O!xpTX7
zb)ubzRLA1HZ{j=87|yET={(%hclv^wn$(8>ea(6Y3s&ShFFF-*X;J$+g|)Izqn;YG
z-jF)(y5~!6V9(0gn>MfRkz$E|edDn4>$8{RPFucH__^l9zc$y%sauWqOWw+xowZ&>
zQ1gf8sRO524;H-nsAzaz*?wW0bGM;pgxSfHlfT=&FgDw>;~n!1#;+E&58j5&Vq{#Q
z{xG+DZC&`cjBeJWnj3k|jxq=>d3wH`!KLzwL&?e6k`G-^wwo+%&G^!r`?l!sCDCF{
z{||N@MsL@1&EfgAd2gop-L1<%$+a8XoV~jv`{w;w-(H42)AzOhm9qR`?i5wggPVmv
z%-p#%-7~W9z>%I?QxgBIuwCyYbzt3A0lu}5%X0XW<9B-f{X9dinI&9p$1}^;(xi@K
zFEzF2Z&<*4cDde>IZo2nb#*n&CQ51-IM(;_EnYX-%i#Rtl<vu=mBO>l53;wp%+xdX
zyRmr-^YYgw)88{0)||J{+GKVrV@=D^8$~k%r%F!Ub~)c;gXf|Dz8MdiQY6iFzxTKN
zZ`ylvyOQb*&kRA%Ox=YtYEzE3oS5C=c2BIxk)`P$SD?h(4MCo<8`JI`*lfMf(?%^^
zF!+-7j05`izn^(2E<4j9nB7{f8sdB^!=)mk#qWynhl>}N@O!21RoeC0^V@kwo2hNn
zHlDS<T=FbxhnIPgprKQp@mA%Zm1ndWgC2S-cs(~0?h=0^x}?qc(#f4iCTyITUb9!R
zpC!IaOhW8u&(XauH4<XKn{LcJ_<3hHyZ_rY%&+$*nw;!pZ{FlRsqn<&HkSwCQ<~d$
zWjbi_Os-|yHeLCTxsz-$!`%rlgBTd6x5li^yH%&&R^On%T4NpO?$Rj6qr3c{y@>Sq
zb|baB&q1V->sam4g&voMc9$MzT~zYE{p8lgq1UPdR5dPz%-+`5<9qAW#`-TS`Coly
zi|aQ$s-Jt<sY5q~#qoLit))U@B4;=4J8LXaWR#lwe|^Svx$9e_O^-Z3X`Ac)<#J^A
zCWk9aR;&z|#gOuAr_kGL?+PEU3B2227aXbF5@Dt6;ru2?U3np&c!B)x6x*^{atwOT
zr{wFld+8ony_-Yp*~C>35<UM!Y^ypnt99dk!R;G7KfN^DU6{8?FT~<??L6ba#+V;d
zjVF1Wi}}9uo?4j8Po--!uUgiW%6#uK&p5u#^6|zcDPKQ`z5MY0x7EGUWnTr^r?Fhv
zcsA1X`~O>h^~Vn``utEVe%;=xhqr#0Uf*N-w$S(SQ^7xF>M64%-nHFbqv}w5D=<%c
za+SjIZJ9+EzqR~c`(UZ=%O!&0Il^C`)F>_A`{O&u9nB*Letl0|u=4kp%_ruEyqjox
zy}zy3ZR5G!Z+6YQ9+Pk^ctRwXcuH&;+sX99676m*n;G1F84ilEHNR^v`QjI-a9~;r
zpPx~qm1pYp>tE~quCLqhYYk^_k?GDUQv+`JnJZrkiSAv0=}<{YzT}}rJWu1>R&Zz@
zy=r=9dGuVTKSlnNj>Ud@U&jAS?6vyTvrEpM43k#NjA#*EGyR?57V*Lxg6VS_Y;;zs
zwA;8$k@r`=KF39RPREmjlN=H*Hl5<X^wY%iaEhQ&Z42L`+lrq%_)@w{<)Xi~ecEs`
zYP)O6<|7jya86sP-K>AFWmllx&jtPNo0M8({lj1WP})3a(S>KL=5#eLdf(q^x}>$&
z<f`+3jlaG}mdBQfUH^MQi_>p&uI8M*ZAupBXBaN~_u{%1ufJQ<x*u29>?xaBxuQis
zZ_i;H*TZL~&D5Ha!*lsg>)U<W8E1=w1I=FkJ}^tYYWfYonF?0L)qcO$ac(F!YZ0$`
zt+V>vrb%WcXS9xAct7I-<GoDH<pFc{wqH3K8+-HW|Gh0@KK!@0-Sd>2{~~Orh;??5
z-JP{{2fNc+PI9)r%$m13%7Hsjlt&{fzxmp25!IQU2e*j&nf`C%^7zem<II*q6}7p&
ziF2ps#{X5w)^#{@=f^y*9u0fjw@*&qQa&+_$!$v<i=X(El1Be>PUdC7%8QCaH~2Vv
z$TU4;zjjDdMP-&OvtyUr@zV;%(b3BuU42netfeXQNATx`!!oQ%Qx~`xt>rqj-++5}
zSZ}$gK0`|E!jlJsCTwsQSop0>&!+C--*B79xmHtm|18_QpZ`zH?XKs7yI%P^H=UmF
zK&k$mT<;yp({^WP$FV4;O>a~b(mrXlw>f6xi_cS8uPAN4<N501w?_+wb1&5M?6Fs>
zDOeM7_}k8vJCzfCPHw7X*z_W>ds^$W#DuxKKeTK+?YXr(U&*5G@5-Y)`j@NDIT+nv
z#F(@E&o&<A-<ro3tY~gqC276Z>!R<fby=RgR=Li1Z?N?`7Ok0_@p;+O-OAnX)*Si!
z_rQ7P<G+e;wKlEEYKeMb+P;j%)^c(SuNb5Hvj6ioPRdE#<aqg}M#5ve&i8ZA&R$@<
zLR(~W|KbDkbCz#CQP7|~^Jm60Q44*+vx^*agxv~`mKkl@BC4^%tR+#Su=rd~!<D=}
zKHbwFDoPhk_*gRkPSwGS6?%s29<trj`n#yEG}1~)LE20G;seL03(sHtXee+cd+xUU
z(<dY?8QpsrYa9FBURQY>`FQZqv!$9Pr&h~-op{V~;i5A0q^lOXOSbry`zL9!ypR8&
z;P%YKPh8U0`d_BB&$ol?bQgcS^Md`|YYWM{Z_muJmD@MJIoNB;ewEW#XNwpfd%*aW
zv3ustn&#xdtK61LdW%=s7QCIH#NAi3?U6`A)}!jRZT#-5q7E5IEO2Tr3HZ2F^Upqu
z12JFvm|n_F^w9DUc-6YwqT=p{=JR3AOojz7I9D%j6k`8;F6=em<HG(Y+ZzSHtFjl_
z=5EYlT)O=2mn~b)mq)*uTXE^s0@hEe7MYTYH#x;lHf9Jr`Uu_@xzqVBNW{vcm;H%c
zS4W5FRc6IkuQQT#tlVyQFq(HH^)`HL>Ak8@%CpgJ4*!j?#Cah*BsR50EsSb6(-eQh
z%k8@Fd_Y#jiu{9B`2i8``&hFLQ@nG(mkAuJn=QRu-l=Xy%f`E}PM)vY6K!`<>1^+Y
z?bDsyf7s-mOzK+2>b&i)$b;fNH@xgbczbP9_Z<;2SJA3qI-@+N>A(jL2X5^>I(xUv
ziRlV_3r=PU@Hg3`5qe|R`M}3+7FA4z`7Oc{8()ewz7t$hx2Jvivp0ba@=S&P8~#Sj
z3xECd*S^X7G(Pto(Q;Um)Uu7A=aC%iORhCB&atna9$@oZP}lPJ$42+#kt|JDA}{Z8
zUE2ODpG8RTo>dZWle!9%h*HP?#ScGiTKHkx=We(2db=dv8u*4;9g<jfVPR&`to?#b
z5n?C29Qz`*VmSYFo&2|?L};1tq4h={yk~x@^L?7Vwc-Eg#jO+LKi91|wa_{HNoK(w
zP4ga2L(`1eAD*YEemKh&{aLNx!;E#OI_503bC@e<s$I+bOKQ>AMBz1?aye9Gx4d!}
zPECKE<+uFal4X4RuBxB^Ht{LTXS3xy*LIdF_>{4PJG`*VGTC+JT;L1#{HZ_Yso440
z?yXp?V_@~8;OO1HH8zQcvm(~|aW)<hochvm&8{-rnh(lJu1h4d1WPx4T&HQJW+;@c
zGj$rTyql%va+ANGQm<#{PCfkK@tTx3Gv58V`BeMX)`qfe=M-Pf((&1L+iCTiy?I)^
z4SVi}p9)>LeU)1EQ~~7+UK7o~ABn#9Wb4F-_RPtF3GY90*#1;|F{7y9nce629E<r^
z`jthihL->3EO2!$ityfbOD8;R;(8w$>*KFiXU0yK?#%veX02*;%J29!gMfV3;C9Ih
zmsuuu!mCqUR1#CC3(S0eKydMjbT0AvQ!Rh)@75P;ot>=aD-ilD;rVLqCE@?3E>nE}
z>QG<(Izj#kR|V&KUg$h<vMSPXagl_uvGV;Z0+o;UuDPq8W7PfIUigy5G;#B7jG-#!
zGZ%~6EYQo02vJ_g+1vML>BO4{BGs;}n%x$EXSd{ylxgYO<)x~V^~}?*{M@6qI_JW#
zQ|FyKJfo(FhdVPp`StF(^rVepW)VzLX^(E`ZdiWXHYp(N-?p<$St5N~S4Amxx*jw*
z88E?T>pel%=k5m$omSnta5C{#4;O>`xs;e>JHt12xAa(4rcVp`pttKYx5?6nl{F{!
zEU7L#@PYO5*5d#C>yB;p;n*_&&AV3~B5I%4RXm?GPb!LS;*MRnzD#_vUrO}N>TkMp
zmM}=UJ@ZnHe##{fa&5`ktCtRVtZu4kZM_%w{O^IURug>8eB45tI?}`#nV5HNls(^2
z%vP4LT=sW@{CAGqQ~b;4?{1dyshG33;C8`}WlyE0ezZw_-g$G*^;J7`H;JYz-(q3Y
zX?`02e9s5d*Ka%?9`q7DEwrdw(W`?e<5XRks>39s+)FJ>3YLgEb(L|wmq=Dpxb^;~
zY7m?5`gYwZD;6a0y?JHU+#gFPJ-ZyWbN!Ya(J4hAb8C|3cfC89bbE{b&qvu_+i#cn
zu^#T&`f^_I-P<pZ&i;1oh2~yMxl;X<<y~hY^{#lIb}l^86#7JBUpYr+6_?Ys5SJ<~
zp*?pVPZO64=(KZw$!Wb_#B94+SKN+XgUmU_OP~FWe4eGz#bC-4l=}Mg#aSJ;VOA0?
zZXcCXB2|i2ezZvL-CpeynEl;pnMduaqTrwRJgT1mKG|lU-7?!NSi!Zxfmi%Q(H|Y7
z^PI}Z7H@v&dh3APp+`*#d*he<P+vMRV0yGmj+@_?9SgM<9q1^_+_C3D<Ai@DT*4gR
z+C8sr(mp<Ux8LhOQdtK%ZYL<S@aMHuOEjAA<KQs7!^C26Zk^!ahdtrUVqI#R_aD|~
z7D-A!<$uzyK=RznEo(mb6<LZotm4&a`fWQ$Lwb?*jBMsV*F_jAvp?xbac<}BSkU;u
zEa45C$Au_{&@!Gg4y>s=?_NE(f%lz6KzUF4?t2zzw(maebo@EjM5Z4f)c(%g>)Lbr
zc8uP25o1O+j^9cdHfO6s3{<b3==pshwdhXE)*bDQW;_#burVIJ^UQLN_#EHUYy58q
z&2rqf@W<BFS6dE5e9;ha(_GVH8>J$ANVo55`8nyOawomst6Ubj9<$6cV=3>%sqbaJ
zD_h$7SUO&uGo82a;o+sH)EC(r^ek-sVo;MPa5^#j$;$^UQ+ruNCePa2(EhpM1IJ>6
zfNj~u>C^Kiy8U?AvgE2Cr(K><#LcqYUa>Uq$cIf&f1l}Oo_S=6#v9!#u^H}J5eA0-
z6T^!`<<7DfP05K2KQdeYsmcn$P{sGV68TPkh<RaaICIM7V56|9ya_>ZiKW|JLMB$s
zmiqSKvHiSRk3Sn}1a64kx8vQuHM8a%ULbi)@XI%`@Ei@sDH4ojJcrwkeOY!V?S_ce
zx_KMAOKfx!BTl}V8j+l$kdf#ju~}71LiBQzl<37xa@RN~9k~&~`ei254be%hh1&b>
zFJ_8rnCkNLY-yX%&yCk^{+W9FT#FW`$%?!KZkt*!NuPatx{hJxuc=$~x7;_1UT>pU
zv)^;Nn0nOlG-dUiY4?q`JaTz6QNCT&&^+jFuS;p`>XycXs{7t<-getT>*D9*5x2}O
z&Do@1n(er@K_=mv#3sWvo*l9Y!O=o%T23*B$ld98UT7UqAz{<$?Dc)ZrdnmExVb9N
z>)V1>FkJa?C_%`4<zDAgn>+%I{Nnb%N?aVkBP#B8qA|<gjbZuCk3liF7gc<CcfaKR
z175SMRtEAnW-1FAolv-Y`&;FQm3Orc*V(^&Zevk)%4VV5^FK|$FLGbquvhF#M!-P>
z3#kPH$}9UEU(R^Yd0<w%lXuk46G0N2?Z13};BdvXJ+AMbZ1lrU=1XoOQEMknlE2o_
zW?il=zV^Op`jr<m->ma+E%sXFQ8?q1=W*A2-FvPt{=D;iwMdRb+6t4O-<=qrUHR6v
zG?ZV!CQtds&8~C{t5i=djcFZAl&`K|_$|9tKys>tK*-&Vi;iB={W0@_^W=ToTGwgU
zMlgCF=+J!gY{IK&8=3aUq<pz$nv*9XxVnX_Y~7`Eck3QSUS0B<ac)V*ln>qiYu5xN
z@19~CerDCCS>_&Z+*VAVbA0LA|2hhDSNZJH(-WPqB>QS-qw|UqwFLFAYI$O1f*L=0
z|7^RIyKt9xx735CgBd#yCT3VQ2G35J#5b2I&OlxNQX=!y$Dbutz6QQm^O<pHhBU|h
z&BvQp%(=FFPmaa&!0E?gZ9?WKE88^AF?7jnpQq|<vFxs9&cunq)2=Vk`&PWwtKX(2
zjX!Rs`V27_8>hUc_01iZpDq`B_qcIa$WkR$hIn3etr@RU)L(=%Ea*7B%e5m<JJ>So
z?;>^8f(+mGFS)4>Or~rPc6m<EncB|2DeK@%@y@KkUrd{1{Wn~!o3GN>l;i!E{dWkv
zk6CKm-)$xQyKZ(SDD}O%_Sx3pQ`5->Gg%opcWiHD2=Y3s<NUX9)wU1kul&^L&tcVc
zak)NglDg_?2CeA$Thkl2zc)L-Gg58tp<dM=O;(Icjrog}{?!I9Y3OC~Q)ZA-p7up+
z&bEfTElFYlGo$>Z%4U3RV4f#xR`Yz~YLy8x6F1f*RXl&!9~OGZJnoSC`n?KMg_|Dx
zG_t>~ZYu51>6NTz>#Dq-xKzEZtMwSm{j4VI_i9=RirUK@Gor;7f3{b=p)A|=dDqpI
zqDRZ-u}J!!U_W)@@)L&_x4MO`Gn2x<&i@!Bsh6-^X~hf~VKuA$g7uphri*Twcfw)&
z;thr?^Sz|6I3%WMrc@u)F;-BrWGwpdB9kq3aeC4+j{_c#Q|#`p|1Gk5s<+Af?{3Q$
z-tV{|cjdK(xp2(;fU;zsMGbiz;(5DG!XGDfT+XyS7nHKt?8(EH<7Ymk=UeU1D{^hB
zoFljP!!yA}#=()t_gWv?s`x`aDxu@uwUr4Mx{OlYY&RtL$UNNg?o!q<Nh_g)v5E~&
zt;?3rYp$F)=d#(;L&@KctXz`VXmRHJxs!1lmRw@Vj^x_5>DIJj`-5iF5>&kpwwh_W
z2!_^b^L^T%o1Uf+otm<~>C9ATHs@!jEX+=oWSr1d@(kAU5;(Kk!7YNjpYd4r3jNe&
zHmzUH%bH^4TDN@;JjcF$M!4&B5hZ<rxLrKYin>fr9Ei-loUqTEtGM5kJ?#_gttXW^
zM#9x!C;CpkYcAZ!I_d28=C%W!4rVJ?m9KFX>ie)yv}%!x31b+;gwMrImPb2vlNT_n
z&lGrm#awi+!?IoH7Pc)ZV3JuC$hhp1*{epsxjC|*{%=waQ~ZBfjC+6aRli3rcdc8t
z3q}Wh%E~y}WX@#MkZszkG`)ajc}b-5Db13*S4!VGXzvOUy5r-k^erU!j{oAt&yHQZ
zWXj9-`>Nv3XX(d6+f(XiYn|bbH;UGpz*H@8BhBdM+{-JQt@%WB83N_I1%4#izPIZ9
zyF$q@X6NDDi3{dknmNgnb-HIt>y1c{8JDImSbF~FJOQTK0K@ZV92pvvEj)5=%sqe0
zEonmDWyNi-4-S5MYpUrtcWL19n3YAV#GfA5+H^;MT@T0q1G{g?JzB}*WMJRcoS1lW
zzwKH3{8+WMdJ}h?wr48NRPtM(zu)QlWs&k=)eB*QRyvJVpOi~1jSn(RoKwC3@w)lt
z*=zf9>#S95?@bF3e;CiSv?SBSt5tl{!*v@L7aqIoRVDlK(;lrE&HHO_KfC|vv0K?@
z#UhEf)sGt4wD0xWT4r{CxLLes#x{vSX8oB)OApv)o9(`+A}8tbte2N@&el5TXDqAN
z6`yAlF76QIa+UW;oATA+7K`;pOT82;Lk7>+@3(cm*&KRiU&+mtbC)m_J0~RPcFvk}
z=Ff)HA61@|UEJWSuF8}fz##Cn-uq=u@SlorbHsAz%>Co#`m?XgX6tcx-MZ3wEo|-4
zY}Yp2x!n2tIa{ay=Ezln{of`n^If!Yx<hI~wqefKe;U!|A@`1y%~QE`>i@Aq#zz+x
ze9h`rTDU8;#j1059*3u3u4bpD(yrH&ZVD%=Wlx@G|DeG$wN5>*m95h`SY-L_dml|C
z&m<?QW$0|ySrU~0>Un$ejb`x$%dOLQ$JFazc&~f#ql|Y{SkW3?%`>kR^&cl4s4en-
znG!ZXa;jfW$iLW*?-Q;tCW~ybD3&UhX|LQ_yEVJzuKR=@mYH8B$v?mR=!Jgp{jk%w
zYE<WSIq_XEWGoRmJ1cQF*YZaXz8IWYpwjg%nXO`u&7U8!#&uT&e!dp^pKrWL^!$#w
zifxz92;IE>T!S~?(98YE0`^A+mkx$6iS0iBLimB$Q6G-STaL^xC{hXVZ20CNuQthL
zZS9eI9UD_E?b8<oi-H-t99~Rh-)Fo<;h=MJ<EEPpuQXp59J9P)c(B(;{nZS<1JMgU
zDQ)tb|9WNpiItk7?8^1L+~)U;6pAlTWGp%Vpjzy=ykE4W)2EQyHOKOA*KW$6vVpZ-
zUHba*kcn<@R!ZDodxzPfEarA!sC7HD_*o7q`^=xqH@}%vxm2J_CgqJcJ9B_J^M}Qk
zHk5>OEjO#5@Mgb4-0=&$G$n;??{%u}KB_CErDkz4<%RE>c?Z-~OJvq?3cs0~Jn8)x
z7Ojvcj)7*u;!_Q;y_b0Yy(wtB_wUsYdbg!6<bL9=(s5}Mr%%+xgYN_QmsmFIaysuh
ze)-LcMTQk_jwc$I92bkxbiF9|aFs&95#Mc$6B(C3yS&2NcS_?ssVy%rowm5R-BI#-
z%KZFIkIE(A@8Y?vXeT35ythqB@pr97&#K*zs-9=xG~ccB**Up9`TW;cDS?-4f@MSf
znB`?u?w(SZY;b7%rGSm!j6`FaugBW4PyJrtzhceoHU5J0b_&j$a#MPn;roySF_ISE
z9}M<im=dSu^#1{y{ptVwMdrI-d{!=d_rL#+!mA@2^z?Vv-C8F5@5c3A-{Mu@a!Nnb
zWM1%ahuydRT0TEN28+%LJQ<wr{@q|@#HpH}_iw$QG~N0a_s!rVYF!IdC08!I{K&p`
z!A#*$;f}A(yhpBX<mj|^{CMn>vntbc;g4rF?9q+xIeuB?=dKBxcC&n3C;Ol6bKifZ
zz*Xuj$qNjhO!U$8bu4@&#<OEDqu+M9OH67$mkcX*8y#O=Ewg#g91j`kITByx4)KU_
z?R^@NzkHU{hO3fyKZWetRnoulV_EIKxm=I<PN;{7<;-F0eyO?E>p*-NTmJs@bG%)u
zqzZZ6G@7~mXR9W${NKquiKAhWfK#N(lQ$vPzOZ$?`EgL;=Y&1RT(-=0My=tw(R-EG
zylLo3u26_fHqK+to51+7!=!F{RL!~tZzo-FxvSv1-qW2=AbwiUTffcgrrfjDWYBvO
z*}1~b;?n=wb-9;rsjA%!I1nx58!99{o#(LI3zr9X!`3)<Dl(Z)_rGv=&hz`uTusL_
zj16itW3C<A=(I)NrO9dcgq$7R3wd+BL`2_8@}1VP*?j1xW7LzCNeeHeY+1_S{w!dp
z(~SGe&TR|&AmQU~?9KPDtLI4c1*TmdXI_=XFKiH+lh3&9ikIuw4n?noo|lKFDBbc*
zQoEdK9U1XiW{Qj6bdi(cO7&bjqZzhse&J>Ncdx&qal-e9_8%;I)gEUbmdr5vvHg(y
zBfb8g9MO}jBU7KHy9?c0v^@2ezR~B|OD-lKe*Y`2p;LUqZbqT3lD-u-r%x3>GmEYC
z-t+qNkNw3$`+hAq`Jh;w+p%i)nl5K|<(XwA)koI7`+m}5MP*Bs*u(($@>X8ayt?i?
z53Vg+Ic3SiGJd;X8(7(Ix1LX4@*#8fmg8sR%66T*_h8%Ut+qe?bF`k9hwrm`exrLc
zqiXsKpB=n&jhxvRY?>+h^is%yJ@qm*R~nN9Qg$`e8Qo*qdF9SO=Vl)D6|+Q6OV0M$
zn96dpd)g(Aox3|1%C<CUe|TkFe)#l~up`dg>5+k-iZWA+qQ4kT<z9dL#)cCc5)SrE
zZ0LBAV|wn|<W{y@Q=Y0uEfh;~%TMG~t<yB(WE1b+c=F7dv>)<pY&V1gX06ulJaPKq
zOO|`yiqF;*s?T^~ICpvIjyVq=RWF#y7AI$JI6t&Tz{^aeY_5o%uT0;`b#jw+Ci;8c
zv^o_l@zY7K$;pairN`v}E$#}%z{avIQ+=HLxc<ro7tHsb%u|?}xitFOvQxr>_MFy6
z)}nQH#9JgfUG=q9UvkN8p6L=gcS@mhW}R&8jjx3-3S{mCYpzr-TI0NJ^}30A*{KRW
z+HSiqRzCbBccgLFBZpM+6PwhYIOIuMwE6Jl2PLRqYvmD6nzbwSt;{N`g8WxY8*+J%
z7=5U0P*u+Jy?;D#|MAP38`evk92c0>&pheDrI}rMlT{W)Tr}9-Y7^<N=hM@|X%WGf
zawwvJjrrbdBbFS_$qS@zZ0P0unfjSgEr-!uj&b_?(`BA=_Y|u4TGm@_Xg?G8-o2Kk
zkI7$YdMo2U)22lhHoMv?<%?H5?_&wLem?3-Pv9ZFEr}LMk7hNj&D&@oY4}ENVRmS-
zXJ*H#wr5K3cm0=pSetn8aNk6Soo^C5jTfX^D?MNMPjGpR$SwJ26ICvUUly4(VdnMo
zk~y3VtNWNHFp1n2iCEIcb^O8U9BW%gx2<wQ0rRhIm|eW(xMqyPdG6%bW^etON?Gdm
zx&>tJ`LXWpTn8_MWqF#Dgts<(zKvM+t#TfZ!eRztrnXxtMpCyfa`ct$sVYqC=CpeI
zIYcjgPoT!#8*<`jLXWMPxJ<V5&(_$gkjpm}zw}E_SCZqg?Y`lx+TswkGPdrr)~oYH
z)2uJOTod){^ORk40zS^kn(pwh$m-<W2P|t!?6^%04*!TQD46bD8otB!fZ+Svnb!A3
zKCzu#lpGP8Vr|r+=f3HS!Q`!ryd(`Li#|+Q;M3yOw*Jcg;+`9P(@bjC?|yTg?fAZo
z(-+h`_xl`K`BTl<vMRfUQSRuSS2Lzm+I8wFKKymrdiV5@#Pv_6mRwu4%6f_;hqC*#
zX{iC3A$rRD*}qgPu05(4xGCtXUSWbqmrSykfS2~O<(f`O0{JtS@ERT#w>cuSg7bDW
z#|`eBx2<1dTBodI<u6xc4}9{)_5GATl7jQB3{@8i{d5R=>?!khzN&TCBE`Fjzw$4I
z+2pw0i#-=8yRDGvk!I-XEmLB)U(u8?-uQA(t|s5p<kGue{-yumKYnB0v+wVuCowLV
z!tiZnQRTKznl~LHxlV8x2q<dmNeHJ+=Ey4Sl`6Tw@<;HUiFk;Tp-<DJ8mYgNH0A}V
zOi=VwooJyp(Za1v+fY-y@*}tM>lNCG$+H!vzShv?lKSi$Gf9NWefpAnsz>{KH#2T`
zKNKuz5pB8p$%}rK+rnzmA2>tavh2RGy!(UdyMhDfb}p=P7yIVBTs`v9?6a|vmK<}U
z>i%1NU-Wzb|IW968ectqYWq*^_{}Z4ET>9#e2I0s&MC}#bI1GL(#D3~-?E%~y$z}+
zYFeE9v0$F%gT_ST6{}f&UvAiC@bdpvJBv@gb(3a);uh&#w0qfR{heMV(NhCrw%)y3
zcxu~&om#hSCq1uR%oR50+BCO&<_$@S;W>Yqeh3%myD1$$!L!$OfttwjdF;<FuKOcW
z%K4@@b=O8N<*=2B>qF;MAJ@`<{U^&|AB$(K+w~L5X(}r|zf)_n{LxVQ?4;Ezj{PV3
z9c~%s9+?rGZS^-#L}};DycqK{VL7FiS7Y_{cKkPwOfG$LGA@=YOjPA==N!|JXR9V#
z?kNAte62-y&-bbCAJ38dexmfT=!qrLN_S^0_%&m;RsLfeKkh?IuS-W{F#Xsv`MBBv
z9f@|Ho|#p(4874W%fHB8d$Ps<PsD~JK^2ePvV65Cg!nIG>gIa2NwsE+uSp7ftGv2m
zZ_<G!Kef-8FWhv>s>mdLa*)39jlwhK$JU;yO}1-oxU}l<UonOoA<r%Ere>`9rJ8o_
z(l!3T=oY8f<#YdqelvBKeVb?eKYx4PuS=KRN;h(SiOu6ZD*N;C!L(P&!Q6LxWLeHs
z&!{@ww|&D$F_GE}S0sHZj=a?KviV<_9K_xq-=FbeQ-sZ4>7W%a+-pNW8vbJz3%Xx7
zc}n})*6ojPsd9bE?Gh7AxN+#0ELS>1CP(esj?a>7r$4y1;HamV1ZPc{7{^b+hM>p`
zL0y|=bFW=U3O;NpeoLodM#{o`=cb9jL>_!CfAMHyq5n3il-CB@#VKD(ZcdnMZNG8f
zt5>)F?|W^uuKZ}k5B{fh=j;kD_i<l4X1@GLddx4|$ETy$1qpULRIl31#F-Nv<*{LM
zTv=vnzQ7w*nRVO~x_?e9$=kK8bel=Qp2^qc^>5rcwnmTdj9!AFCYQ;}=Y6r$C*DlE
zx8>9l(KPY=12Ym<hg#gpx&6oQz1yZ!;nwHp*epri?R#sPY;~rjaGa3HOUM7~R=>Oc
zOfqn8>y7P`ei|0;c<@^L$FFZ^YCo)9GkMC~trF}EQ88@p?mcbi4lvwLy|N?bo$-#|
z1Mao*%TMk*v#<Z#*FHJ76~5Qc$s}CnYKbWHnRSfw{Tid<AM<*Ssrs&eZM0-{rH0F)
zs1>S@T&I<Zb;>RCc<d!^ab@OoL5<ywRqxAgO+21?obB|zt@EQMG_bKQ-*|@iL2$~N
z8*UQDH;a`gA7tye`e9kVLWub{zG#h~(suio@oql3>gv)=M~90`HA_qVgy$d1%`n(q
zVr7(Ku+`ek|Io}S|HB{H&*AnC-z3;!vC_jgI%9S4^p9m`C+nQW)-|5Bn`;p6X?<{l
zYl(n^@rJG(m$MFsmDgKO;}O4d>||?*puARO+~i3Inq)q&-E?|EMn~M!mj4Z_Ca3OF
zmPmTps^!jh`{}U&p?*(StrxrpUs$kOTD7U|zxl=YMRL?@mz0x<9p;tCmx_HkI^(|j
z#`Q~>nOWDbnI^dOK#cJ+0h9G?`&UP2eBZZAWKsN0d8>ad!dyCWhQ~@Xx_Hkn+URdv
z`@*+hzclH78{eO>XP);<Ui7D?6qNKZ#RwfN^LwVKVa;xGu(WfB1?#cb2l_mJs_?35
zNb6LnKFIUX+;f5Dl5f*H-JEL=xA9$U(%8D?hU!fh<;-dR^*z_rPABlU%D*m{cC#WQ
z_pZ{rji+r43~oHjJ;gThk9FL&4|8_RF%Z#KQBdb#vs`)J?xlf2$%5k89^uQLcmLdF
zyRh3=y)8eAb?4p_-m0lvn2+j~&O5w!9d8jsK}+JXC=GuVtAGocEE$gua$PeES|PR5
zfAyU`+Z!2mHs163HF1iFpoGNlYNI^K*GF=~9p(sUEM58T)r8V_!iC1+PU(tgKDmX&
zrEHk|CXZuF75^^g=5?>bL_H0j$6L;H``*hewo^gl#hq0ShhBA+Y$$Fhj<#I+X!f<1
z4Y&9v_iPS~KJ#F);^sWFa=F-z1+N+wUG8%}v$(+O<3`VYyDHSuSMr7^We1A~$x3(y
zIW5haF<)xQsZ?8MQJ$=nS#s8!-S=KwS#x%I^vwFn0zc1O$-h^<YvZ})D<<hq*>AHw
z+B93zv#Dp(dv!;l|4Y6aUt#>H&~V<O=x~RvgX51&?-ykkd~~>9ZgPm%ccZSrT#LrZ
zcaH9!V^X(zU9-@++?rI6s{#RMe${2IkuncpVU+1>^0Ar7b%}*})rI8rBgS!=*?S6a
z=v@__m%NHqxJvL&(sWhxKV7r^)@<a{Ydom^O=M9rPxgm9)9?N`TgY)I>CGN*iPO<r
z$~a4l@4jUV;S?3&Sj-SseL#5QVYlVIA44WyTB|bc%DUNM$1e4C?i22EdLyr4_)I{p
z!s)lLNrKU9$Dm0Lt5kk|y1R`hUh2x*Lg)0H-3qdQ<uoLj^V5!4pZs>g(a*UpS#?q1
z@sNkYI_X7P3vQ@hSd#V9@3i>lhI<`ss!Ul44aN?|-cK9Y@|Par*`tx~mNR$TGCuLh
zLkpgpwr4hV35kZ*zRTniFFG)1r(W4ijdqv2dJhjXuxog!Y;-+wNFX$Gvz*tD=g0Sk
z2l$^iI`DsYO~#|rqL;VilU5Zs?7qFNVABqbbr+^Rd7YNC;hy@V2{(^!$k9pL;IhVk
z_J&ukJj_<}zXdSwxz+MN^?H_9){)$Ksx#l-TAz6<{FR;Yx()?x*OZc`a&1Ayxg1`L
zVge5|hH%-3olIUY^ksM9-Kq@DME+ZxKe<oEDBp_~;XPS?>EKM3y2lF+t>h{x(5^Tl
znUy_Vp!KuOn%9D2#StABl5dI`Z127m+n2I`^~>bGLqgWO%Mx9!T|`2UsQtdVq(f=F
zlUa0?=8fzpy<C67`c~+~-i>WLsJ%RM=Olgw_rrbBQ+!G-3uM2Vs`$L^I8`}&(I$aZ
z>p4*iEJOE7#XYpWwY^&4ldF2$-ZQ7e4=6hJ37)j#D%)mzHv7o>NS==A?hmvt#8h01
zU1GB-;nZ~bh*`2b&g(TjY;|(fzj59B)o(-l^Rg>fJC;81d}-(R&MHvkRiFO3^V@xj
zb$hIMC*|n0aC$3P+-8~RpmbQH`IhR+YSz*)k#|Q9Z1=c1A>4n~$!#HGqFuTR&t@y+
zANp>w^XanGD~A$0r6y-4byl_{7kA$`pY={~h5P3(UrtT3ay$MrWySt9x2-;pSN!<%
z?d7(KmvxK-7@ZQsljcv@Bo*QE>)OmW+UHL%pLe8M@Sm`TbWG3mYV!mZL%&WhruHRy
z-={Q1oVQW-2}(}cAriG#!?D%>moT4m+4M#wiJke5Vitm~hmuzvk$d>;Y^G)t8(Z_6
zYX?s}-+#JF>GqaSS~B%{Rwt^TZ~Szcw^pQP=ffM%UPv7H*R>~bS5vLjo+HPuCkSuf
z<zrShYt~U;&)wCI8*B0nh1vaYrCDE1-ta58>h3)Ms?AbQYfhfddBRciVs`g_Ubl~@
z?kTWJeYE{nygl#6p`g1lCVHWFXJooRQGE0LL7&yLmblsT3a77q+08hmu)A3KvrK8f
z{I%URd72VhOK%HLiD25PdS|+o;$_bj6;IMRrZ<}>2(Ziyy|=Q&Vrim#h;>Hz$<KFN
z{^)G#VvVo)v*XH%V)vVzc4dDG=A>!*7MXr8t<Yyp@Z?}>zjh$x{qi?@OB&tmn2%`R
zitF-l%IHo=-5Kb&uK$OB=XB=x5>oBDaSKje3r+FJ5n^ol&m9qWK<D7U!h&7D_idRR
zpm2WIeRj_<&cob0I%9Tz(^zjmYsu6TvlcGp(tSMR<kP6#lNM@~{aBTDBKF+1kP}`{
zrL%55<PP?C^LVaiohkglZ}at=Qhx&vyiS>LXJ+<P)3vwVb~o?(Y$oJlrDdMvp7p);
zr@-HP6KX$4gdXbVk!-&6bSF2@d(#)oCX}ko)KrZQ(c%z%VEyI+>*1a8z3&C@nq6p`
z{h_gIb;pW*yeSXG%v9IR&i%NnWcKlDN9#-`j$Iei?RnWxsPA`GdeG~$<W{?%t@?>=
zZ`yl}_upN3eotbdQB!JaTkyoL&_eEvs~>%S+Vc4*v1r65giI<cIqSD+;*V#V?Yq};
zEf?)M!<<z*f8hz4tV?pc@}_yD6?z{!kUmpiq-J-Du}%~7@vF;Lr<MGh@3t+Y+9LAi
zw?87>fA;e&`Y5_*{+$KCr4ydr-{$JqX};iZ{fAR!m)8G#nEX%v{)3->r~l{KMem(&
zJnLWl>%R5(7sQ;kdYt4MrfFle<Y9R!cZ14m(dSbdC$w2FPz{K#Tr|T)hWDdD**7+>
z1-k1rC;m#9f8dC&a{=q8$s61!ONRE?aJtQ7dFa>Vp|N1^PZ4Ly+u75FHWjSM$hz(H
z?@o}*BY}Czi6#x3Ccjl;ReYW@SvQeUCL^3pJl|#BDTVSAT<tqN?{r^%*?9PzigfpP
zo`d?yk!K#oPi}mC?a;f8Urwk!-03Nn_35+8%96FgK7sohy-#1C!26PMqSWjY&qMqs
zM9q+HPN>lo`MIy-_(xOk<rYt+9gl7duqim4yCP@e>Ma6BS_}U;vT*x76Z!P^lGv1q
z9-046tqWjry4E)1_A-ay$|_eCOY_ec?Du&`UVQz(r2hX$-c#55p8lP{%3?H$=l<b|
zvuca16pry$)@2IM-J)~M&sTP%XXU~PY}aOmIV$s72g-0=U3nonrpCEo{=K;QC%Ve7
z@(Jaf@RIY{vX@m(v|;r<)3xPaxmuVC4lw-s@8h6zVKuvw&9|9H>?6OlhO?$!VOnm*
z)V<|?^GfTYS-v@!4!krt^~Z0T|27|?xU=nHKRs&MQ;!#K(D*s)R$0;VFq^du{=$t{
zO5dK~cM`}GKd^n?lohXDKRbE8zkkx6^0M5MFV^aquP#=3q}S$n*Ta0t+6xz_O50!A
z`Fgea5_8iDH39$bJ~8Z&OSqi>+r{j}^n(kTulcbmDsNGE<zb<r{Vqh}|C)6x5}R)a
z)OvBevQgyt;Ujq1^TG0fW!3`APh5{|N$z~$S1IxO)7=A{bN%KVJJmN;?$Fai8=F&a
zFx5;ijBnlX*g~<v)#<d{?5eo17n^dXZokm=^{}1klc}GDx+5~xUmuA$HFev?H8F`V
zelNL{B^YNIHX-z|r2LjQ8!94rc_L$(Uw;=lw$OE<Lr|N&^l|;2`&&dW>j_-Ysr1OT
zY<k`&p3>~H?Tap-r}^>Me>l1)BpNEs%MsBDZEK9xTDWh%VdpXN(*-Y@cr?F-Eu5;G
zrnT+(eyP`-F=j1a{crC)xG?dET_o@PKThX5cdgEn4_P7fF2cJw%<p?D`&r2iOA7u9
zCGIJh#>o)5pp#R!$hGuNC$H(DiFwX>TN}CicP!Z`Qxx0O=r6i*-FCI_F4NCkD(2Aq
z$2l?e%+0s8mu_WlORl-^|A@P<qL<N4uhV$OJMRwThQ0?9Q(F=}_O*GlOuu9JXy5h(
zvtsGULt!%vmQOwSUE&W1XZUK-@_UEwsE33z1z-ETCE^F$<nkp;xum0Kw%ExX)MS`=
z?fuI)4p$m^)*5!Y#_xH|s;yKYTXb7U#iVl2k?n2E7oPZG_e)l;Dt<xa@&}eLAAg-9
zIcc%X%3?3Y2WcODpYc~77CzUS`>x&1Bx?H%Mz=LPXJ<2ApYvo|M~%wvo<mk^<boc3
zyL@kkgQl76)!fF$99G>bm+}*wg)tThq2JcHe2DMraLiV}GXLvNXV&s7hx9+Fu{>G6
zK)ToA@5|cG7Y>i!_ip&(FkN>`)qeM1yG|Bgu`cp|@Xj$)z|rDZOt12vJ1(NTEdM#a
z{2(<wI>dhh)1|Ew-9GJ#^>V(y?D*ZkflPYV_k{a+*7};X+diKENa2>Tw_o)t`QNv4
zrn@b8Dk`lPH+eyh$Ep>lwCXQkvMJ<CKf6<ryJ_C+kKU|)TQ=3Kp0-uJuzh2idMbCw
z`vZ1QpSm&X%4imy;@t0NvGt2b@>jEkf2>yQpB6FYR}^2^iCDKaeHJ0A2UMKAL~UO-
z@VImoE4bY&Z1fH**9|#fd7U|{#FR_M@T;4~zc)K&**R4vh@25QP#gT{K}zJzZ;5YD
z?^(hBL)td`T~e<|Z}H>F-=dvgPqB@gl+AKZb<uwY-J@A2x*mMqw*9h+GUxgj6(=X|
zv(-5p9_Uz`^?BT2++X(EyP%Ab?U0W0k(<9Zd2<wtN2*)Yb{g$dRca09DhYH^Q~bH)
zj40=c>E)9=wb+-&9GdyI>FFf9<pnRhXCIzrF>kiu%gfR~b~Vhnafe0Z;)(`~Z2?6;
zmi}I{hV@F`ft%0$<lX-{+_8#Fk@r$jFDqFk(`MUz;MbzeE60tLN;tV=lMdcUySdM$
zae49yT}@-jS0B?aYwz4)<kWCFY8t~uSI4L2l6C5553YK<<J60S)mAUHXXI9>@Eqgr
z-tp*d-M81BRyE6F#b2LaA=17;U^nZXm26Apq!l8b#5$y@ZOD<A&AfZ1HP(IZq7z<$
z?dB3948}W!*`M=%zBkw6d4<74*I6MvhT1)P*SB4<i~V@&>Bsu{POpEPdl((Bn0Sp>
zbMC@ru}dyZ)GJ-ppt<4i9J||1F6WfpXRqFIrmI9(f%%;v_eZzqK}#~*BYZn;n^#xv
zn)=ZD_R|fMlB4fP3aM<Fl^fV7GUtZ%WT)L}H&^ycT$}lGX^wo}qx|ZKSAJnnK8j5}
zsiA73y>0JZ%g1{dpUf=#&Egk*bGmJS(vOKr`&+Hg{JLZmyXA52f|*VyEhAaY3e;*9
zXQY<1X)d#9Nj1Orf+<0kJ5l}hx4wy0;$OdB*|=@#A9ba!BT^D8v$m_AUdmys6yYqL
z>#Vkbwen?pm&UO^GZpLG96kyahjn+P+&IxGs<<HNBO~MAV8)YP&ed-(oO|obark)v
zdyft4(LjCa&kmEFrM|3JF+X(9fp6im$8!^2s(G^h@>tn*i*d8e`s1#%E7P;%tM{Gz
zA@eSMX6Eh(?!A@wn7kLS=Uvv`_Th%g$1OHnt;A0(=={KUG|{d+KeKejTZe!Jg{z~=
z@2w4;<QX_+?)q~&>6P}KQ-v&!o{31!lnk|4E~`8B!+pm|O!41-ykC3VQ9$%nABW|k
znj(p5ZB0H8I07FB{xCbp7PI42zR14+;lDSpzxUs5`Dwd5rn{N$?mZ@Z(^xUadh^Xo
znR`~>o?~u1N4e>=313a~<b1a|xn|n~zb@u6o%lRRuuxI%kje6+ekTML_6V^zUJGAh
zE1|BWddRAi&B8iKK)5kMvq?htLw&>P^M6$*1u;1`Z=aZ;%G!9NXtkR4`{2|}VTXyq
z4wD)Ey#>C?TRi8<Uawf;w$9byhOy?fmo`@(&J>n7&hm#p&|K+)jfKI+#?}<=sbb-4
zKR#BoJf6F9#a_k*>m2sBPPA<CY4UD~>~qw;Q+4a=5?_Yh0Y*=D+)g_>y*IyG_l<<v
z@lSuh7iMrv-`d^w%{)V;J?-NKKZ^^O*t~9EY7#!N>|C!$Jj-6mg}IZi|LvZ9=}yxk
z{x?d23+FzbreP_0t3zMVA&C7&Ss+jEioj-W-%TH*qI)yBS!X%+iF)KW$NEkwbG>uI
z^>oA~A2-KD>$xiR9=5he7Wm%k`l{;7pIfI~7Bn}{S?J;gw^y~@R~Gv)7CqqU^iQ7s
zJKbT9`GZ2{JGayf>q>Z77VMmqvbwJ$X@<<CoU-kWo$G`myQdqyo#NKV?&#Kb`rgeq
z!BJjiPHpQWJ_XP9+J4GaBvU!AZOs9#Dc8@6E(!1I`XO`5MO?(oA?x|X%>o9@X?Il)
zKFE(1l3Vyr_kmBXtf{n>>*XEJztk$1a%Y(b`%ZPp71~%C6D;BP=(im+*MEPR;3$>H
z$>)~t&u&*z_J6T@PmOYsdhCmBtXHn>Y}m3hbVZzV^2^f<6+J;>(|ivt+_<qx+rv6+
z^})Is%dS+Y{Jnh4x+CKt-<kt=PJ|0*m+zZ#U<%`8n?k<avcEB$e)e`eI!{6`#~9pI
zdsxTfxBaD$OjFj5!mJZ(UfJc^v@YNG<@gs-XL-go?jI91w5wJ#&X{{+<xjJ!>_hsO
zr*<m~I!#uZl5se4sSlI%iv2rgex4LCSLCA4Q<2%p2c&Xjle#XiJfa<{<1Dz$#6r1U
zqtasjGU+nsJ9(QLAH8RIndk7qHp#5vj<Cr2m$pg2(|@!5xvS!*`LI|~>FW)>?MwG9
z<77#<cZ*(kOZud7oa7fR^GAEORX^Fh_slvy{n+Wn>rVfEcvx%q>jZbk_bVM93w#uw
zbgV6i#iQP1Yk2jEUcIT!-w!X8`=vDDoLcz)r1cY2JHE|qsF4njzR0(3Ma$V0z8y`=
zqFMegO;~$DQ%@>dCZS@5a*9u4=7PyWiTP6hYEPYXd(tR4Uw~nOUhd0l!5P_2SF^+T
zrYS1Ex@X+5F#ADo_WZrqN=!67{jW2awS4}<AnvKzqhHB+YdPz*OI3nJOI4ZjCULcI
z*lGXb`_GoTjlnLfx%9kuhpE<yJALf9&$O!S(>e_vfjhBb4pZZ`UuYf`469uIt+(Rv
zs<q}UooC+kRm80hoN+Yg&1TbB5xM){n|c?8D>%+R#Bt}}4ngh+%LR;&E4#yEyW)7>
z&7LgodU@%l#XXm|TwJpEWyr;i>#jdeGhVlRXWGp7N!gon|L)Lzy7tuDxYr_+SKmqH
zYCc^3-uIM$P3b?bBggJE`TEB{ntN;7@_Nw~{oTQud4El$y`S|(EX>^*nkP6lfkn!Z
z@seT5l;1b2_AE<?Zn~Pqw$Y|#n#!Y`I}U78nQjw&s+4k%`aa50I^oi{{K4lrf$D*6
z7ISj#|4#d<o0QFQjgj-}22IJP+7=eqma}5>FV{Z%8WB4?Fw1}9*Ua5X?^<4TC(O?7
zm;G;LRWqlqOib#`y7}f?m?RyQlQP-$B+A6+T1_f>wdcI~HD2inIv)=@Z)TEsa{az&
zpRLxjtFbQZjvvy$zVuB|lA2Me?blM=*VAKQ(;LKi>gXb_zgIXi>ki853MFk96<fq>
z`Q}V$+kOL|Tb#b~T|VxcGv;i|y`yeEA-KHcO6NAS3fl;)(_L3Hx9nH^Vt&!o#g5-n
zPk`&TRr2-IX5SXEoh!7{-QX8B!CCI4;!XatSBV}?LacvG?z)zQigDI4ig}o<zP`-4
zYlro>yUxBslb9?nq<YMiy0Y_yL$t}c-Cqp~Ym;ATraNA8$T|?or2g&k6Lq^MW^X=i
zF4?14U+bPJJA;FL)%`iW)=|>OKQ5Es>cH->cX3mz)_?oOm3x1ur-bg4Jab9zjoHcL
zw`!mHSkC@kyjSYUpVJ~~H3hP)Kg^GL{Jr*Z=7V(G>gTHZEOCqPslVX=bL<IoKt#c$
zKgNQO6?422CVw<an$Q^bik+v#Qe`jenM-0tm#$wgvYPvK+n)n&JN9n*!je*bUm-7{
z{FG?HjY~FRtTXQQtf*$*n|I`^wnw|^?2WU}ADVgS_Mx_u4>)}E>|f+qG^d(>tQG21
zQY>9kG|6m&&nn&c6-(ASer9_#yXC!w{i_*C?TwpuM9$rp-niKL)WNv$O#h=|5h_)o
zwr{&uds!E9ObQe_akPX<Vb}Rb5)X83E}p$!?*4hx1gC}_jHRC!-Ad$nq*8F{pQMV)
zjaecq0+pt;FR@aMv^ddn@FFj#+1l7Mjnj{6J}h@-)L3GC|KxtjPOqv82a$Nig5)DN
z&b!FZ6nkc!`9$>O%&prJ-bE?ux^So;sB&VNr*uj7MfjfYE)S=g)-JIPOjh`>G|Alk
zMq%77`^1w%y+KC|mH!Bwy`W_j)M9&?L7@G*>C9>U9^dv(aN{zY;OWSo(#a6ronyqg
zhJkUbkwlx)-pxPX7nZ%7!(}aYO!lZ*0sq7G_bfvuZ25bFT~#!t&TT;(E02*<Uv7eI
z$AP|GDYGvwyJk1}qT3Oji))$N!wQx}bKg`=vfbe6l$`NFBC24`<n31z+yx^qi`0EM
zF=79_2e%ZS2V3^o|Go0gBhAD)x=;4()x5Y#I_r)3-*jAK-<_ntJd8<^VT~Y1^3Goy
z&P?9-*erNrkHf5a4>m}Yl=hzQ$XfE{!Pmw6nWk%59zIZbQ|vnH%Bh`_5i7r4VsKm@
z?AoweXC1?0-Zz?G%D3rio@iJc6~VOOx}>I8p4%0@oo+oV?y42XO%_advDnz+`<7ic
zn?WrnR&cG*rtn+u1YKp=3Tp2yZsIWZ+kEoY9)qZqBhChVGgGvhA4htbHy){*SX=X;
z)i?1~pB0DGHIv839g5C61$pR79PZjuoO}D@lY*!khNnO3byxoV$0m5)MrJwJsg*0X
zg_eEZ`(ndOqZrobA3VOFsDI^?rC7xHO*b)edSu8<-(8QSRBkKi3hqvCIISa)IpZS3
zHYov<n|;eJ`&~PbFj?Vl>5`Lr77u47rkgb0QeOP^7#sJQi%~g8r2excHop1k`(<h2
zrPhNr>wE)Ns2*awAlr85@uDA}-b@vA3V5lX)9p}L;~(m<s^ppgXJ_OVu>y@2(;2==
zEY&y6+-~$8G|vey+0{0IT|~h|cjM*on@>5eh#hsEcJ-|PGAr{9I~TV1Dc==vd2je4
z*K^|MgVLub*p%rB{$jqo$-_10q~-a$UwYZ+m|Of@X0ms}TeeNhUT(=O5;mWEWYKJf
z^r{o7nYX5{(G}@u)HRvVk`%7<yLRcB+j^5?Zi!tmKd9dLR$a!vce2$Q2G-A>?n_>j
zAMG>q6E0ht;5{MKCc%t_Ur^M@SnJ5F6NdusED2$MRJ&oO^lVduTkN;F@&aePYI>9S
ze1(&5#f-p~d2getIo1VhD^EJA@YQ~wbi}dJRSm%hl0??X3F>MnvX`?hpKKp8DdoVy
z+^{@_1#4IZ7JU9@@wxwK*i{2=(TmF^57ngIQf>LZsb%dXXQdmWw{4uvZU&lYO<uTh
zqREr5$KJi_OIKk$7vNpkq~AQ}&(DcF9eync;1KK+Yq9J&c~I?;WzK{-JMILRtvI;M
zLg27zz+4yQVw(+uArmi_crD0#`{p!zRqxFk9<>WpLVql0whU34zc5=cL*2UCWCD}p
z3${xVc9S%Z{$ffgHZv<ckmO~ntQir-*6wgbRzR#|a{Q{qgZx&WE}<#UM84>%8_pK!
z=&QLGsIBC;IQu!T(-zs897<xEsj=$89qozgtjW$L|5RHtg5FghIaR!E-uk&;wk7x9
zxW>5DSyv|0noD=L#-y62y=js27e;*Zv#WfU%3@vJGV7Lf=DMpVCmk{MuUMaQ`oQnx
zz-RpVAB5TO*~C6BzWy~J>Aqq4|NS{X?%xSGz2#j<<Gs0zo%wD5E%WVEHpQ;{7yFUr
z&JDkcpUJ0dKb$U^`daIHD4$T=d5bfDYel~{m^Jp6&CD+sY=18ubNkMAFaD`=$$?FI
zWwng)dFfnB*Urs*mK(cuc3xM>uh}0IZ|@9Sr^{8gFTl5M&b;~4I%fH-7F~bwVR}^X
zvBz&GMEzc}jbFWO?yoz?%GPQ!x#zk(xKdZUa3fdu{GTTkmp3U%9hkY5D<<75MKp5$
z=|#%+-yJqR*L-&W^8O%~_&WtJIWKNX5`6Z|^=zd7{jdGM!@o?iQktJ9lKXAhV}7pO
zSmSWhtJfE@Okr6*hyTxO;Rn}B7MM-V(t5I5!0Aw(X0^57Q}fcjbETZmOH>_;;4NPB
zz&ST6>w+em=DlOyFP)#ZmF-H5(Ylmny}WVb-=Ch#oT6d_tG%wzx}>GS{f%=~O3Al9
zGv}2#-}7S-UL*OMQ}<))vt3$-RrgxoR!r*n;qU)4<icmI8{e}pe^Kq9@3^<*?E)sY
zV{SXYT}uthJ7alF)ljeK<@{MwC8i(Qkr8UQWO|Q&j7)_D|Mp*O+1V!Qh8F&3lVW0I
zD|2>x-|yd=9#<6j>e9VEzizRfn;G`=%D2}-rqf;&Z=N#c=N;|erClLMzuB#we7I$!
zcJ7OHJxi)Dt@4SQ`eRK^l18!S)i1gKUdn&ob<_U(>+_HKe(sC^5^_FjllG&Q>h^h-
z>z1~AahFS^thaXfU>o53^vn0|>sL3<NZIfudza0VmHAr&gt@;?;Zd~Rv--8@y5JjG
zPG@Vj&r;Q&S$}iKx8-UXw)5sC^fdp=TfRT@UZ#qH>ZccR>)yUwe#mxLNshnohQe2;
zK3<r5EMdvpoe5UGjTRh#j(v@|Zfq-lqByZ~-6OZf8|HEu%e^#_Gg^0)_vNoc>lVGQ
zNG#v8nd{>-h11ywN_ysToiF$y)_N@=bD`ayKTFbV*GgY6jqlmeck%K5%O1Z&A~!_N
zS9zU$dExGsn{VdN<}y8ayhho6Y1&uIMK@+$z8>rP`)g%)g~Q4M<<j=G_MiU*+sWGI
zu4(HlTp6^vF8ZiW#Qbfiwidp9EoZ%VW!TGA7d>t%J)CNA^6$y*ZY337F~-*qP4$|j
z6{yuByNO|c&;t{ds68`}R#z=DG|J=umR<FVrPSqdYV`A6Rj1$b_1i9&fAw-%uzv6B
z8Gfk+cMG&E!W?z)@Z3Ex&oXL#zhAVi=!?LFt>&BVzs&zSP0V0|$^J`wI~^|xdM`Zj
z`)q{jF|DQTp5NZhe{#Ra@a>##rR&%K?hkwA+*nj~Q%FDhSS{zyiFyCdp0KOCSi%%P
z?G^LABhQ^Kc1Fcrchf7HzOl<M<l^N0k0O+r51yDlznzO$bXV_JletS<H*YDJ#&or=
zreRv9h|I2w*Sw`(-ZweB)@JYJsF|xT2<sfm_RZQDrn0W!lfikrDRrED%CiFEzbu)S
z^w6PqjqUH#&o<tw$XT9n{(XMk&!)vO@fTiHFMeNIw{JyYySmyXU;Sw>%5@KlUv}}<
z|8e;H<@+LA&XvY2PiU%$IsLX-{7U3chf`DK<AT~#9IxweY*`<-X-?dxE6-;w_sU$o
zH>ypcvD=;L*Zk7IH~;$cM?KaqU%0k;`^q`L*Z5uXc|Cpix&OPoG**46@{aG9EL(eF
ze{g5ud@d~s1x>BTSDTOi^_x?!ShIV{^Pg%9UU}}=T(viF=9;+uw@RN@uA89p!ISf4
zX?@*V?}cs)w{hKlb7l9t4>Rl6FYA~pI&Z3Xb^MplOZOHT&MBTWcb?gcb&lno7xWTi
z{BN9|{NSW&%f+IVA6A`~WPMw?z{YiX-I^Hb>*v1S`u^xdm4xfY3-degt1LYCPUP&R
zNq*Pw+t_~oa_3xY#Yd;#r&mrYnrp0->AC3D^|L0615WOX<$g2s{pruYEn55-Ke>7q
z^0{1D*8THo6~nt<D_<?T<hyjXmv-j;uY10x&J;cNOJnz{D?7wKd4zmCEHk}!Ys!|&
z#{q$B&rja6^J(_st#-z*jW^rLv&OkEk53KTe^VnzWLv0&_Jm)a45lvIL)J|V{T1}u
zEah_1ui48D6pnRBI5(X+HTl-cN#8!G97}1{o|1kseQ(M7>z8`k3}u@B&UHKERvDda
z)q7a_^93nucfOYS??Zb}UQ6-(@>lob&#2XhKG=%+ec3zXRb0X8Q>VWg{88dLz2@@f
zm{Y>W{yNg(ZXtKV72AV6D_1P}@_e44vY^)KmudHpih4XQx%Vh-Re>V!{vTX_T@Ezz
z%QU^&DU|r?Rb}1XuLst#m)Wk@|EFJ(y!Dq@dvxHXBa6N?yIKk+vgsW3ThyN2+NLt=
zhu-{sx5FjZUUFy4s{Z8{5Rz`YTEE(v+c?X&YEIa)jCUoWiEA$eFY&%wtF&tGp9`7?
zcF#TL#J#z}@1y+b)5~mmXBf)yh3K?4Z*6Fsusr7aKDK1Bw2&{~zjn?!^!L+JzH5gL
zMv0Wa50A4lROmGM&}ry7aYLQ7|9jVKSC!J2#jkvF_Q#!|UEeCb&hK67yv=3TTK`Mh
zFG6guMr><8Z?f@7_q4yMYpq;VFG-mxiQZV57|vVT`e)w%(ALj)9r{eGE3fC6cPw8P
z6x;giwUd=8YpuGupL#{XZV`#1!`XM^8qEuSoaNlYT9PZQbi_}u=5E0I(@XbrX{-KC
z-ZEva-NjpXEN#mVtz<6aQMh$LKYrc}zl~S2-(K7!^(I8Xh9x#^?eb*@*QE9wO}3A!
zu4vjS>Yu>VYA9AF#nZgTxpK$(+;_{letliJp7qm3yZxH3Hw=;&Y?Y{aeWX|InBv@|
zS3xaD=N@yev^;%oV%0g;A2t&f72D+bW&hIfeEZdnbMNjuvp*k;@vHxQ<<5?8zgsM-
z4m$nrE@WkntY~-g=9(M0V#Agd4N}~)o7R87_~>X=#0Dpk)Co2ltn8{iPF#65|G0hp
zzR=U*7pKfSulag$m-*DPi4m0xPbj+ztrGgeyK2emN%NlT*V^t|T@s%8d{z7WB^}(G
zLSzrF?o6IqW~8a`li}&<M<*=BFEQN=3F-cQWO)d~+IcT~d&}dw_b4fai&;LrxKv3g
z{946_yQ-#+NkyuD$ro4HnC9zN%?o;c**9PO>}B4l6H{Ab0#hq%p4a**J1WMtG#a=a
zlHjuLZRNYSilf2%x}i+A?fQ)j+M$+}+TD9M-C%yOQ|p4&di|EDV@if%CvUyfT;Aw$
z(y#vc<@@~l|BLib{j8cSR+gF=E@svk_TKFZw-^@~ieC9uTM?)m6ZYz2%+qbU>c=GS
zzezOM@HIWIe$}1J+Q0r}Sd|?8RJ(55>X7S~Yi641US4*2d*;-x_tvlEtJa+FSvK{S
zikg1F?SM=pBexlDyY4KWy36kV79Yi5oleWtvsaq+`1W3vTBxNvt>Ce@)~3SBrm)my
znVnLX&Xw+M>U(l*;vI`g(yO<ftX-vFk@R+T!R>kTs*FEHMcwu{-}d~OVMr2#+LB{U
z0j4j0E@EmsBVn9;L7@NTPDj_C*@gwjPDna0Prlt;yQ0_8XStzI&%&^Z<W}>ripi@(
z<4xABPTq3n@%zg$K0Ilg{+2L{-dNeRglo&K*)i*WH9a#})@kUw+;Hj{S^qQF4KIla
z^vo1Fx5q1Qz4-LL&+U!Z#0pmJywLsouyFaah12SzPw&tE8Lzy%*3VVw@RR!ApW?3=
z@w4i!eXDix^XuC+zciLjp6{7rd+FD*6^zdPQdx`6&s?`T@`@Qt(aWru(6hcw6)Xw1
z+at4Qm|s2he$5}zzo{-2FQayDtK@&CJ!?XasNc-@D^vnKSJ`a*bY#|!panq-_Pt2s
ze8e)>N5)okPTL8;<Sk1Y+#dQBTkF1f`Dw}Qu-YSnC1G{xPlbh_J@_81EwPth#zkw2
z#QNx4Z(g!)eYIZrlN#gR&5Nv=esxN<7kc#sC(X(0Y`nDU-Hbg-`+TQn1eS}HrU%Zi
zv=N%BqUE{VYg6p*NT%vP6}9|xpC9R|_{`<2vHHL265*@gE-FQ9N3+&k<5`;-aH}qg
zXXf51ri(QuN#EIZe)httz7v-{((B;oe8m0uUg#F#V|On9**g7n@8$-%N@*>RKc6F>
z9@8@aa^j-KrRVxl5Bd{5S3mJR^nK=z^fialmhbn^U*&2WaYehN?%&*PmxU+YTH{)K
zis{FrrSomWzJ_Su)i?gq;jp#CLptNigrFZ^uk5+@{-EKCLw|1>TkSnRKm6fV?jG^G
zl~Ns-du{T1ycy@7T0Z;oHv2mZnlg8bsfhZhth%H;*|vP~wU-v_9YuY<>=)A%U3#kg
zZPbg*rB5z<XB7&)iC}XwKj<!Jzqv5>vwPQ<2HiyhO%WYe^;i_e57l!oY@5z^!(Vp8
z`ezjy3uSgQeTtu<V{;&2(+|IIMV*iNJzn1KT=tyxWKg{7o$b3P|Lc$c{iitRwZH8f
zvyf>Aer#MX{pG|)ukMaN7q9PHxumJ>(eC!|Uk+EKpWWxs*nRhmZrTy9+b=hE9(oxS
z!Ri0ZF?Qk42=mVB1mz05eXEx{ef=VG?9i=Z;rQ9hf4%Z)-hbO!`1<tUDjl)+=gvEk
zp^<p|oa5{3uXzkKUgo+el*YfGAR%I3Tz+js{Q9XX63cB@uE>k<_3ZIas;RMjFLAPL
z+Ly;CmS2{Lt@741-J$tbKfmZ;CI9pfdT%B-{;dy9=xbhopI?dd$*;Q~wOx(N;tIB3
zF5h$K-=n8BOJ~jBa6I+L(d8YPJLffAl3SSlJ^0<vWtTv;{_fY`_h)afoPS;{_=eg2
zklM={i==jCN!;G^{)hReKNer!+z$A-@X_M<ci(qDxqMjRt<gu}7u|ESpWfP0wk|nr
zS*nai&FS<08_RahynfxP?~8|ta_YJ*N^AYD1ex7A^w~Rajmh?Fr!V&|jM95}w_~N5
z#%7+THKG~`)dH7hH0^ob&ok37E$aD4Y2BA!_I_<&W{_1TWu{x0+$pX1c&@>LO5xb^
zKYkj@?)cT~=NWBPyVC2@w)y+Ar(ZiUb&quRoruTkHGALI?2mKSOZ^|@>(R3?iH+B6
zbFgZUydZn;>~)_5yytm*`nlwcb;_)X+TmI!`KL~K(XlA%56^dPQBf_ekdPHCzFZD2
zzn3209rT-N!OmFC-T$ob)*e!w`^0lzdcXy{r(!PJ|IYkN56<)#Ds=Mwtzyk8ykd_3
zdX@jhEUGJF)=&Rkdp3X9&rPytW-tezX^g*e<>K|fR;PsDY!^(L;9vj#&$-uG)*bR+
z4_RG3->~+~_orV}R@dB|ch5n+%3-b0(%{N{8&@TlEK)CPtYC6Z;_3AJdL(Ru#ZBhz
zQ`gPiwyoia*axMWLlV=sZu;JSzGUsRKav-|1{AzmE-rVzfA;3t8!w#l`!VNlx2a%B
zle*{Q^|$7}xi$5r30u@f^=mhOI-k;?y)@K@^M2U;EJ=00cft#@*)M#(ao^u``lo+t
zYcH^S%$}BZ=~7Ma_QK~z`+qK&e~9nfY4`8?`&T~walLf+<)C@We8>L0_;j1ODEp0k
z*eR_u^7ogBeyeE?VzROMSmb<3J3G+U>D6l6tE)bnJP3RH&*=S@|2N{U?5LB`zy15p
zzJGU~^RMWA{_nEok7cpP?RERk-2DE`?c^5uFL(DCuFX7L`K{IPs@9Ju%a*%OZU1;h
z@$0^6{L*!QxWyzp67Jo7_Q|($-Kpn)IK{7C&rP0f-}@!x=ZfoG$18sSxqkU{*w?&`
zwO4m7pH(tpjn14!3-$En|C;u{SZ*#6tA24qftwA(=3mbFFKRS8T3q*u?qBvhW#(R|
zw~5OoC!C#7`SyqIL~$P_#a1EDjU@#lE>0V~4!jR%eH=1D#k$kae+jGShr8Z3EbG5W
z_)JW&^XqYLdTVYd*16Fm)85u_?yN~yMCNL<3K|wU2Y805n#<~KDEhUv@NLY>H96Op
zdAYDW*mC%T`NH2JK`WL7ocy=JMMbWu@K$T#t%p59a>rshU-xYh((8INN#JBx#=qt3
z&$kL^YUwVXaLP<HSL@Yg-xbRrFS?|s_U<%$b<NV}-_2IPe|_`twZF>oQU<SfdGpI@
zsb7D+Pg(0-#)Jh6{zUC~7o&Y$OtfTQKyTC|>!>HnR$YA)Zg7}n#b+LRcE9u5lB*Mo
zj@n&|$jn>1vuFC%3L)pTo|%Se#};_j+zC`GcR4Y)x!}0!>-Y<dy<CzW$tr8-&bsgK
zoY=G2<dp2kqV(T46famSF8{N#In8i|n^{6~LhqY}Zf(ME)!u#yVY4vYml=I{qQvbj
zYx$;nm$4^3y}pmjVA=HCpy=c0u5H|0+OTdGYo_ZW(U(^YZ`qY?<DU>#w{($<chGaM
z=NZyBZ@QJaToL}1bzd;>@oq8Jm715=IJvfVc|EF%wcEb6^}r7i)Au18=2w-cKg-~5
zT>L%bY*XahQ?)0}?O6G9rvAK|o-nmdbIDabC=zYWw|nnr`SjA)#ivTA&aS(2S=QNK
zcJ-YL%6B_=cklSSV@-PBSC6;SQ)?GL+B?_&-E{ZN^&E1R_oAd-UFBB1m|1xFT=~~8
zvGMPfRtKjZW=JfpON!EMShRLiW{6qusbzsTO&p?sZk@~hdTH;)Pq!qsvVZ;xX?vNr
zyfG??)nkpUN?6FXmu1OkdK@pa6|#lSjWSqt`}oT15*E_S<D}=l{~~ouT<c!Uf2G2`
z`?|lip4w-yF#5_N1+`-ahHagOSL6b3Dm!GnRZp30)*)<_{co<nzxVabEiPSM9)UNx
zwW=LMC#>D4vHIGX-A}Unv*bHUlvHi)*9L4bS-0GBg)f8CB-ukPb8oz!`{F|KQTgf1
z)-L#~ZWB_Qv#TVwExxAD)Kl13x~WL<eeTcd@2tX^T6aTsfm*)bzst#o?wKCB>H62@
zS}&4qlUDo)c)9SB^U~vAKW^WtX_DjL+q-b1p$SV@o5RH$(o%5={}&a$jahkN#f2QZ
zkZPCU7whWo_}g#0d_QE;wx4tN9Qvo~>-F@LmG_1jdrEAUzVkQF+I!iB&F5a?f2H-y
z4AO4NNHMuMH)L=K@V@`>k(D`vXMthUF@dc47KwX8cC6PwUY5Deu*2-y?|ajpT)3g)
z_hNBwX=T!%yiP_Ro`hSkf|sjUmu@#}4PM(b)vUSgzz2ibcE7%FmELl~)AZ|&eT&py
zE~sa`b;(ysVD0;gnR6#qKRdO`PA~jMyZ%f522=m_kA)jz<hOx}bA9>iW?`M*zrSFN
zTI6Wt^KZ7LXH5ii?NyBft22DduBD{=HtoCR|6=ve@}&y`Czq~!9a~yfe(v45O)QO1
zTQ8pz|9b1nsos)(tef+Dmxs+e6;QFz?egro`xY<y=BO@h>8*D-ji>uslFGJ6EB}7E
zveEYXn$jDV>mR#u^<RGD$ur|xa?0_uHQ_tn#Z>-0@?dGMsj^{SXQ6k`>~pV;<+>WZ
zd!jehRsK9OL-iQXdIN(sr+ag4ZPz*;H~z!qI_<hb_DeVQDOdkT?VtO#x+-erbEdho
zuO*kd9bWOvVn*>pIp4S!{PPWxPTN(O)Mm4Hgnl`_&im@qsHnP!!V=o?cKdP;tgeqY
zn4J4G_3HK6I<a|1n||8fzp?Fn_GhV@+e|O7y#0GSa{srUYeyW8_uAyXn%7=>Y2W>H
z&Q~AJ!ybLTHtSKn*MwD;e|Ei>Q(~M~9e??&Z*||?b^BJlnIQf1-;Cw_n$PB~%6j~B
zUUJf>xzCxV*Hr9Uz23?Gwf6i4j>&sgeoD2u8}_wsP0csHy2pyw=Kel>zOQOu;@;-&
zlF#oyefW`l(cb3L3$N+VUUjSO;P^hZEG}=uzdOG-AKSRzFRboVe#w^nuligz4Ao)N
ze{B0_F~7#Ye(Dl~t*==M?rLY9(t45=ev~h`P&90s%UzD(wQt^vmgJ^Zudn#G_>xm<
z>%|~Hk#9e{e3xyy8h*`B<lftbJ9lSUw%KtOR>!Q;^PRr#^oQ<h$;sRPJ)Tr@a@UGk
z%d%#z%Is}gwS48gby>Ssy<D|y^(+vh&^#q*+3m-RvUaVKT$Q$B*21i|RhAl6e;2=k
za|=J7U%$#@;`)EC&mcwgv8}~RXUE9c>@#@m)#?U?*A_8lnarB8!XPkl(IJ=CPA=iQ
z&tCueu?W<82x@RNce|T)M_xDKN$*1QwtEuQ$sc+znz!AL+R`AI;KP{AB~g4}2Fo!W
zgJq3|{33y`<1e@@Ycdq?Ihk;VE!pZuM`DVbMDU>*LdWJ9sI?ku_hcrX;Z_&1`{&`)
zb7YUlnQl12tuAp*e3hZ0zR+hSIN>C?>!0gmuU2Gru5W*dn@p^HSREb|#}vELAUJWE
z@YF?l8S>tXu3n$YWh|PebtYuSVqtDmu~`u-HUuOtJml2c%H>^>wD6ddaJkl<l?BTW
z&E+cpw)wx_|GmBLU(ymMZBDtV6Kmq_b7E4)W|dZ*^iP2wX55T)%AMA#sV!PG?Yh?K
zb5m69Qg%-7Z;Jg|a`kekZ@6jf)soQ5p}y}$LTsJC@`M_PoDDE_z6xUWvj0x|xoOjt
zTgfupdsi3w#u$fJo%p`Bhi6h?vGKE)JsOh?1JjQP?!OegZ+&XW-JY3Ni{GW`1V{##
z9^;xS7+zGd<u5lPLZ<%C-m)O5ArKT1A`4h~_85M;wuqHypW&}#FQgCW_v|-h>;XkZ
zdh|E_8LZHl=;=&2!;-9Z2NDlv5|<Co;6K)4kk-;G;RcR}$V4CR<T#aQu1Rq!&sz7q
zlx$cUU%qZ?_TyVSgA_kI#eL;jV8v{AQem@EmYds?Q!At-H+qW(C<`mY;|3f{PI2FO
zYP1(8rMqnAkuX-CmwIG#PsKCQl&mu*hQW!;4*9gUa&e1JdvV06w^dC{OD6;xBeqtB
z%a6@fdl#}}wbT5E&mP?>T)E3+{|<IL?t^n%ADnY4ke|N9`<m-jlhER9P|^qG0(jEj
zS$g$ys6Q+Qz`=f}?Um%JbVx3MrvE34?2i>1uj|Z)CVoi!<kt0`+Os#V`#hQzzw*Kh
zowwP?_NBdn5Nd^I*N4n_VS+ApCw?)jDc1!iwtEtl$uD|0nzMmY|3`DR25879^GQrT
zFoU&g%QyWDCE<fJIFE%GJZmzPTyxyONWO<N(T5`$oC$=FE!iM=_{1IqH8)VGgM%JS
zw8s3oc?06Go?2d>N;c<ty}4%#ex@hKGyA*?*Ghp$l+wn(+_O$VGR}o<Y};(4Z_j%8
z>-@2OY$xul519eZ4TtV5V#+#WvJsZ@QNn*m@I${=YteTXJFTT}6$*Z^;jXBQnWYyy
z{h|2$+nzrf=4@H7`Zn8XLHfrQIq}fTA-=CgLTqEdf|EKt#MiE#x2_7Q+>u;$Y$0=`
zz*n7)DlwP8J61^LW;a~<yJ}X8!Uuh>>(FjmNXD1R-?kpvH|@rCq2EzS@!lQTow>Rj
z*H*G#nURqMEqX-Nw0yqI#->d50^4!%p1p?Ojy;fGoX@l0@ZYsXtUip%e8>e*Pp5)N
zQ@x|`OzvYT25HTPvOPZ&l-nk1CZ6F;_LG==Xol#qEe4=8&oOhtLWi%gq;YJkJL{Db
zN05u7?jw5wWUAWKOr0ms5=M5e>ICk8tiC=W-~tnrz-Rhhf%r-4#BZLdtKva%(CMdh
zB_bmUQu1(%ifQRw>}-`Pp9Ks3RUclL1;leVg{3)aZ(cC<_|n%#e;#!^_!L#D-~6$T
zx8OS4j%?;TTN$-}my4Y;;!dp;d9!NyYUB!_YVm?sFA*ue?Md3dCg-U<p++IcB|f*Q
z3$HEPHFxei7fbQ_$wt1{5-&Gz-Mv@s#$~r{`@gIHG4I{;fAz2ZTAwtz=at2@#n&(#
zVV%YraGGgFD&rNK#6_<%S1n%&u6dw&?#{=8nh@LAt2J4#>;9O3St9!X%d$!Q+L!EX
z1>?{A8d@E>^_BCqVDX+-2fLq=8lS!$yqEp9s(;%%@eJYj8-z0)qf3&z^Kve#^PKoN
z_xv?;5BA)n`T`QSsuVw+6!@94L4G~god22Cyz}P${Z)~>vR8NO#-(-}p6q$8n>&5-
z67{vmpA@Zhy{#-U;m!@Y`(F-SUcB{d;OBM!E^iaRZ+PSQ(<isTt6tgKvTK?Ei_Nq8
zU%y^T+z~T9|Iz!ZtLPTH`tMUO8*bH%&k+mu7c+UX<IU@`&<oZ}ue5ADE8SP`ytLhm
zf2v$oqfXAEHzyg{0t-)O%T}&k+*WpEtK74#!c8j83LjU`S~Qo3!IRD5QETt5H8E1J
zr%Fz}@hT`_fr)CMh3Go#@V!%yZ0d`+cDTnaSlj%1)vrEvqmHO=jdK!ZdJCuMh;>W~
zR_#$%+uX-<ZQ?;IqlgCgj9-~1hEp3gKEB>j(wDN1VZsrEp1(WUk8914IDW(;iH|wZ
zD1TMW^&MsFR%saa%{b&RB}G6ru=8z=d#YP%&vP!Llj#pQ)+Xu*C(o4d_KE-SJLN}h
zUv`{mX8wy6?Q{PwY`@uEbH8rsCZE)qr%oN4dR_ZsWUJKam7BSyifU<xgssTwKX+?o
zZv5kAXI*L>BJZ8ld7m<^{L_gu?=A<vJE`NPaA`++rTXSQlfhC?0ynIjx2$T`s?1f-
zp5D9u{d9h;>aVRwx|p>;{|>x9|L?w{Y57@e%K8t!GrVETw)?mPi&Te0Gq1{ydNbSj
z-NzHYOT3X|4$Rn4$s@8?kZn!lGV>drY}TG1m;xggZJMGv=hRCztyOnocJBO?(sn+}
zW~UqDyfXq)$r(J047-jQM0GFU{<Y+-TF8}OUw`L`yvQ`&ZZGmq<lXaylD9A07`TPZ
z_DCr9b6$A%;m(9x*M#luQzuVdF100jMfny@maqO^(`Rp0e(<<@1&i~=>K~KD{#aK8
zoLv9>=bps+b$_`Of3NpsRny<J_XYd&iB`#Wo9oNIX=LnC;XU^EZ`J>w@!<>CImNc#
zNefPRz5CvKmN<jrOV1W=O13RnQ>66aR>F(w3U9;DbJNaT+gWLIzviF%qjlnc)x`F-
z&%Rv6b*6f`e0HT?fWK_omu(Td<?@!S?>$$yUieFJePsMw!>V5^8UE^Udn6kEoa)~9
zYtw_w#HY)pU8bJw?er7de2;CjoO#r<-G@q_S5*ZXzP2@YbcwZ?<8>rX#Kw<RAmHG`
zl2r~pRm;B%WQ#K_$cV~YzQ3~dP?tcm_`IculyoenarG@dq**=BKIwH;MStzr1?Pn%
z9BX1v)rK7TAN<5t>yw;lWq;Qmch`H1i+*&6boxtOW}5na#fj<=m}DPDODEh^>Hg&3
zFLpp{+24QXY^rNZl(F5JJN>@Ps?N<CRgaCb&ADzJySii9bEc@n{A(OCtAmT9&R!9!
zn`m(DRdH|BvM-D6p1k(n8hS#4b=u3rx4*YHec5yIE7$Cbk3Y69`yOoUR$aN7<6Bep
zvnv<h|C_tz;dS%wfcL(~689@!y*~F}N%AvRmj&g^USCw+!grp>rqVVl`~BC`8on*8
zo^td0m=2$t?NraPt@h@!#|2XJkHl46I@?>Z;4yz~q+h+TdU$Sa`I4`<J2<=aZTp`r
zTN`#AG}#@~&Q+~<()V9o>`DLGM&VjMAs*f~j%&0Q99tqKY|-8lZ;<HmJ!JCivgFuu
ztrJ&fWF~<M{Z_8iRo~b5cbm-re(Cyo_wAj6jyDVCBW|tQcImQf)2|7OZ!X-rVMXZ9
z>)O&w<BhK$+FE!tFJYBy+4ZjSJL{)vgQ~fZ86giZXYYKx_|f^bKk~KzMK7CUf6?^o
zD&qyOuB@~wUF9d6^?ujd*Jg~nN}q(x)f45fIm#UW_KSResPXzp?zJ7S&xNwK3LTpo
zUHCLCX6L=Ec?DbE+_<xP&*ad1C$-+E)IK)IxXAi}@AJ-}pn0Y*-KK`Tnyqqs)#Z(k
zUOf-j-*et-Z)50nk9#X`7cTs{weV=dm4NSkn~%=F_i|g`;`7!r;<qk(ea%T`HjF#D
zsNUy%ZqF*N)f+7~YQ6Cnl@B}G9x5DIDyklKv@^6YDRfT6)R20x@KaHhET3ajxr8Ae
zTw$<vMT6AA+iDD1oDS}%0u#UQj61!{_FI~C_9PCgzV)jgy<YWWm)5>@UP;-t2b;P1
z^=x*0cmQgxpY~&ImEx{tp1b|QW3Y;@_1S4rem_2}mRR*K+Q4q}rnJgyDqmGD?|Wx*
zXH9A3+KPwU)>bN&FFXHrqu8yFJ6Gk(&o2*{74v<1u)^Q_Is4-GD;DY06b1kPqgF9(
z|CKo>mi;_zmVCb=<lT9tgUh$F7G9A5`uMT<L;uO?S}fu+Mb~9R{I=b<|Mc73rQ}$D
z*1?0%zJK}iNjC1<;+~WDWqx1KoS(O8a=u#7zddW#-aNhh-yZSi^Z(Mfx?H&}zx8Nv
zcKxE*8{w0lvn+l5{?_MDZQ-x}pO-H_$MYfczD8%#zWryszMS5!B!1|!Z}NQMb=~`4
zedn*PcD^vbwj$WT^|t2jOZT@u7ua}Nx&8Bfn`3@E7wro<9a6sk>x#=|M%rDS!h6D!
z_sE{RU%v3Z!3ka0rp*`sC;Z!!bZqyX6#kH-_p7GlJ_`K#c;fe|%(BbgzAyiNy#C<4
zKL<bFfA;UUR7voA?eACAXFqc`zjga!*O}7#t*>%_SE$*s{dCvcaf0{zUw6B`f;XSl
zg`fUB+wta)-BEF7JLVd2rZ;`z&tCPu{DXbf>*PhXUpTIBu%C5N-tLvfpL1LPZ<yvA
zn3JDxRs7oauE>lf%bhk9)UEZMx_0CD_j41M-wF5s&cgd(Yv{K-tB&Tz&U<#Y%74@Q
zx}BN#cRM=&jH>(RrmOFf{NFWM{<F`vw=Zk{E_R=5KgUw=|1-noE)MzM4*qNZolvmy
zzvK2fmjY{7PcnRV^7roVmUC8Z{QBpk!Tg^U`{GYq-+1=g{ot<epV_|G@6xV$*(RLw
zz<X*$?1X3j^HY}FXxIz9KmNJ#|3{(5uAk+x%FA!f*D(p5{@=lGg}k*&x@wMPW76-N
zhwb)0JX8Aj)3V~Ad!P3kD}FnD{3iF_XJ4OrO`CatZ)8DH{QYxv3O9R&r+m<y{V?ob
zW0CBPlhe(4SxZ^2|2TcBNtyq|XG!bdv*qsE%RS65btqW;|7O&cH&&C?*Sl}EH~Pm`
z`TP0SIX5}w|Ih8zwZE7xzvR;KBi{GlFZa){n8>6g8WE!Shv``jx4?_v&zA4MvE|{B
z|Eqc5O}YN+W4tZn{QCb(9?OT6%nwsy@GA^euAQ>zar)U#g~kJ({v0MT^H;Cmx?k1e
z$J|whwXUiEzO;E{KG=InZ|@|Jb^j7mYh|m=9d>^eHaRSL;Qx>JJ!k*l;{Ng}=vU2c
z?uok}{*L{+R9Y_Bsb=Lg56O=u!7kEsG%fq<oz+f8KbbMZ;$uU_euke_Px+o!&-YUO
zd&{KE?Z^M1zfJw$zb-8In)k3k;XF6%={GK?{TEoTJ0<c_GhF7Y{ynqp`Fq}4xEd^-
z{#aONkALZ2g)_DDZW?nn|ClSx{ZicTnDEyZ{l~KJ83@nY<G96A>Sgr5PK7ppxjUkA
zA2~n$iPm>Gcj~;h<x|tzulhoJ?)YuwFjhJ??{LpOvomrhH0u}s6Sn^P(04h<iuyI&
z?sqQSG7bLsZ{1J#+?%J5UvhPsc$LF&<@=+ni>2o87y2{LHcviGsOI5hHmiuai3N4P
zL%wgVS@=@$XYuAQRl>jjS-UM;oPS+t<)nWb_ACGTm?~cSVz%+|JQu%PrSe@mzW1v)
zc}<JgY4KqD`mU{`r6xM~^?YRsy|Xi4F5|e>Z2b4>-Xj^V6H;21?p^sk(yeDo*!}lc
zkJsO8$^R)F|5w@JN9vQ?+qHk+<DGc*b@bLb!3}30oa&bgm0xt>t?!?kSz?LvzBO_!
zaJ{xqb8CIY;hE0=^R0dt6xZh~)>Iw7VSnX@oyD&GJM`o8CcN3|-}U{w`LplmuFKzA
ze9GT^|NQ#`UrtQ#D>n2Iuskl1Q$C6Pzo5mlIfnZGw*Eb-`(xu5$8URQPnl8wp6kv1
z57BN>FTK~#`;=dwG+B7wwa#oifrPsMNyeQQ-|u#hdz?P~sV48PU4`2A+xNSO{k*^b
z=k@%K1*fv-3vzEubXR%vTYihD#lieF-IdMmS8L2$-#l&mb<6P6X7Qk)E&E?h^A@jU
zxfwn;VdBEir~8ABJeVJCa9Mrc?D@9G{4_t?tvTv^;+|rbQuWLCw?CIOD<%2~PFlD0
zFYAWLY0u0TOq*|BVOkPe@3-N1$ztw~c80`)wC&F~KS+Q3Ngz~WSJZE|IbMdB<a1cN
z-oI-<HOJ%g@A8g#*W~bPFZr&q<WGG3-j4I^j~B-m3+n}S#ytAZB)ewbExThn|K1qd
zxlTH>{{PoD^~zs-9kFQ*`*Q;SD7y!yyxgg|K%Qx~vx{5aWeb6X&=#)%mcRePWB<SM
zZk@kBN#t`!x1DjkR*v(<p4}b(-fFvVoqziEqt1HwWzY9l`los}D{Y^^<W(|PT-RIa
zclFCJ*1wNPciCU!x~J4Rzsezc((IYGQhrA74(dH_n#jCBKCS49?B5M@-Ak67*F5~@
zZK8ns@q7P-)mK@_eLGp_%(UqF^6HQyo2B%2eDG(nU0V11o#umQ8!o?m!M>S!f%(_X
z)pN@@JpNr_vOP9=*XJ4AK7Q?bWA)C)>g$cKwf1$+yg&b@)mAizraH<piGHa&bildt
z%lD=`%gsAD6V+4nx1IP_v~=h5<nJ1>Paj^W+5FF{=_7mlMqB-l)3;ibdI*<tSjO#d
zHook0_T98^r7@RE|Nc7jH{tKRPhs|VFV7d9JY%ywr?9<EQVj1c`}L|uet-W2>gQWH
zXukilL%!$#zZ|vymC18!o-uy^d?2jE=i&6OV~MY7(zpNI<N9}c(c|0FQMK#E&-&?v
zU%zC3&vlhR!h3bOv<~}<OaK0PPFi5M(eIGspWol@9D?ktmwReX|5vnZ|K5X)Ir-m$
z#plI$2is3j@Mzst^+H&5!#ahg^)hqcrf<Ab^r$BN#Q8Uuj4J<!ird=1V!b;1_UC;b
z41d10DQMMIU*7M%|JN(;!u5Ymwr<`0f6s=K|B}AuW#`v#ZjNJr*8l71&+dB>)8}4(
zaobVz63f0suHv>O$9ugsuhv_ieO+8~Yr0nEw~s$R{+(7+aoAbuk+=7y{aGg~p1J=J
zZcU20l9f64|CF~(aliA!FU85+_1pJ#Id}EfGt9crr)O=vJn_rkZF4f`)n9v1ZhzyX
zz`wsyzbYo3tzVV8<os41^}yd66I?!jW&T_*(EESge?OV7uEgKmo$p_^B|Q2UtK0wo
zmh?&wu9u~^KfO3_&-`ltZ}pOMGn3cVpE3BX<n^<B?MK<)(x-#k7Q8<u`sHoEW{%w+
z$Jf7nSyn7!m{<BLKjW!f>BlI>zYaQ4o{{S&pMRqn^zo?g%c5W_)s&yH>fip!-84M)
z{MQHPMIWzvu3PkHfusDr0tx&75=mda1%5rxAoQyLi#PMs$Ex*7-wPNj0@Eh(tA0M8
zed`#1Y41noXKS{bpV;>&_UpenoD26IpSpK`^=yNKi<ZCJ;J<HSQ}z9C!nb#XZ0Da-
ztMEgAs^yYj*8ld2^9as=^WitY((xy%f85IJ{<b|}{U4~yB)fFx^@s8ArK0Qn)kHL%
z{%y$LSJK=d&N=;}Isdxp=3l-9@%+3#H@oS#ZRghi0owv%{^ajj{ps%?)oJ=cXZ|b>
z|92o^<GY*l-wED$di1bL;oqyz-uthwUt8gFYw`W>_m-Qx@GY6Hf9>BXk0QVM*XlED
ze%<1I%*q^i_y2(lzcx!B`}`*N`j4*q3qtl&>i-^^82zRwxK?&b)lT;Jn~Ec9TfY7(
z>OLX&&3vx#f4ADTb=RNE1g7Zx`nUPX<3H)f`FqsTfBg)0dHd6Ck=T}J!q@*-$X0&6
zWBjdVvDg`NU#{nOsy{DFoBsaX&%68Xd%oLe{b$YX&(}UVZl0GqDPCjN@;eRh^S$o(
z?hbcbu)j_7%(n@GAA~u-3QzxZdGV6MIc(<6DdnefzJ*VE^^NuB<MRHO=Xamot|j>A
zoA$mX%_~0NE-82V`}e9z-7d$StN%H%cGOR_`FMEtuD`E^e>~;QJ5p=MZ~O87^t+Gk
z_c^jP^UFPcu_1o4^G)XD2j`{l*nPai#r|J5&Gz~(`-t=Vv)A8?J5l7{_J#e)q`%3=
zch|U{JUMUfVY!&UZ%-VK{MB%()-Cd(dEQI$xc~Y01y0J;hj%N7#@}$8QvPmziy3RY
z%JcT0`!3h-`}iYWR%Zr-r_aA5DQR16e+ukd(PQ<tk|}V?yQYi(j#+>D_;}l|vPw?z
zPggC~wlPG8+5}H^e$4pqcEq2|eu<;=_d9sSf2h`e`KnYy#`2%ENmRX`)6bIrj*YVG
zew!~0VB?8+nAFNruAfnMsoO>8{sfPz$A<flT6+8aT{PG8rS5-DmjK7h=iBF<nO^^E
zMfm?C=^xWC^=?kCEo;%{dC}gU|HSy)l7RlryWEbQd)`qk!`!m?#_z}f>s9lrqRu*A
zc@cl=#{K;d8>{mFY_;`kI^}P9_x?SFZ)^D-!n$;SbSK)W_x{}=#=rLu=egq_o9#o>
zrp9ft+xAn+>e0s8`)>uNi2l+Jt`hcn*Cu-@c%_}n-)-NVQ&ZQR{T!Y9>lg1|t|J03
zeyun8xJ^Mb+vxWGuI~C>2NUJ;kL=obKYl6Wt~-nOl+|CIHQl`8&o6F;Gv?Wa#Yfb)
zUg>{%P~`Z}$!E{HF04<k6MK1*zxd7%-nU<)`F|_#`cv6_+jak{uiZl5+4r?I^JO|s
zKXT(Q@3HOE&1)viJpBISozDA?H5uanx&7Bp>Hj~|TX*gsA^kNX=L`15cL_%-P5$)j
ztxCw>%PY#%Z~R%q{CDH}IV$#bn=6|?q}1o$vO9CMeoxi@EXLg?hvVOEH+E_<-~U!@
z>9k)>2cGq8tC#OETJ3*pz6;l-4_UPnA9<(w=A?2d{#<+AV`u)xPiz`5^mT0~m0SO8
z=R4Hl8@;<xOtAge{}97Vi|ezK_TN#kzM9W__TG*!ItuafPe0X)D;+;^S0sc*^04>y
zjiFO={&fU-$|qiUv&!Lb;{NPK|3l2B&vQoGtvT{)^Wr!C@h^71cW970wLT~O=-J2j
zK09BZ7I#XJf6>9$A=B<}DCeEAyR&opUB{W#r~Ug`ss*oD<xeVqBCL73{A&E=i5sT#
z6#n2ZoPVO`j;-*)kB6qso4P9e|CZTx;+M1*KH*jjdD*(Ce&@^Hw9osE*i2O?Y+fj0
zziDacfiLzOb25Z0?W~`;TJUx({bl-PNk#L#{<d5Hx+*?ho!-uSqI~Vb)gqsiTdO8&
zuGeI9h?}(kLE?v%pPc&tJr?(IIbb3<T~e^=hx?_p_xv5pt&Yz9beJ`F?f!ZZPK&ku
zFPp2(DkJ|*=Ue%6hIwb}dbcUpza+gs-`#BeBlOjKZ+^=eU#)ZNCNh4_Enc&C+r9r&
z0}q+(QA(;ku_gK2*1I?JGghyTD_eSULG9#l^WP17XNf=hVEgmF>ho9J(<T4ib<26%
zc=68pR(%5_F13q+lACt?u-jnYx#T0auW<EOd;3hj+gH59f3NVj(f_|evsQDV{pur)
zhxX@o?*6~eNy;Su!iV*;*Ji%06S;Ij^*P7F>5312=FJI?XUVXC-MI131}l$xX;<@Y
z`9T4H-?6#dvwNNt{~*x0W7ft02RJ^JG{1PfFH=1K@jvq;i$D55`E1YW`{MeSqw;nq
z1=rN~pYi7R{&YBMitizX`$hpbc7MM5r`hJ?bobr!K3x^z_WJPo*V--2f70}O>YWVj
zboTu_ll}X*y6xQug6sLFNcopddY-6#zOKcTvFO(?mKpu;>rEvUx9qv8*?R1L$;5km
z+OHJctGkzB^X=^6m)_<l4DTMX{wiK6_~e~M<*w87pG^}u_3LMg>~#$T$%?w>c+q>m
zcIVa|p7g8zw722I{YL{e_`l{qZjp)kCCvZewDcVQpVoaw7uWCcocVoe)Ekk6@9C?K
z{J5j}f0M4d{lob>Q_>wiZO?l7SbCaywV?Npzx!S<S~;!p$NFVc_i{}B75V?$C(G%N
zj{9y*zt?ELU4E}$R7uq48zLKfEMsncJRELt^0jluZYC9f_Y((FW^6oL>wIj(7pA1o
z-*S0_?`Le@yJ`7<<41NIZDUryXEOcD&yhbrdY>l$`Uj7#MHY0vQJMXt+0pOcsn1tU
z6pYONxPIW%S+wcVdxKefB$AFY+k~vxv3hCe%vuH~0h^77L>Aq7xKu;nmPF?}#pZMB
z3w$?MtKA5H_G(+Lvrhf@!wa9rKiI!$($&jTk~rdYg*I1nZ2$bmeu7KwkBr+X?>|kc
zG2Gsxv%tFcw<}|S+^6?He3z`Ln|SzK?_~bnXM8IZ<~I22JjrGJm1^>HLHGmLaQ%AK
zH3uYpR-IYR-0*`h-^TjLH&(ej{P90uvQ<yaR9Ce<_FH{N(zf{qmR+WLjm{@j*u1_A
zW^NSLE>!;QE}8TE)gQLwd!BdyYf}6?`~5!K_4TULBZ?0x<W{Ff)fj1eOnJS)XL5}A
zz6a~0&QG89;@;2mYy6x9)kAhaw$Y3~uFJXd|Fe$ZHqLGF?o+cjPpbJH9`em9*S_oL
zYOWR6S0-Dy_KUxh_g*1yckQ}_=XCx)-NWanze%or9i;fq=a2g0`kFf@*WA2o6t?lK
z{kxYhn?EGyuG<?MSM^5zu<h2{|4e32I=+AYyl2NY@}E(7xw~cS{oM!I-<mUoEL~S&
zpIYMduQH%9N%enc%=4DV|9u%wdc?nxe!FXbY38iGZMpTXUuy4J?GNd0nNmM#$)>-m
z_tV2ul~(=8UG(|z!yiWb?k0!Yox1+(@jTBX%QtiGS$65iZ1<nMcGdMx4D0_z-kT>r
zd)?jnG7Xu(mj>qEd+_Y%<;>OR_Xl3O`gK2JjyA)e?rEkcCH}MI|9KT}<*H`4ed)dj
zr9aDaKYq*kI`hKY(_626bT+fS-|(N4&s5Rfd(D5|vmV;--yYLzo_pU$HRb2D{b{Bm
ztClsq{du-T{(RJr`z22=^X5(dWjN*NTig0gy|=jct+KCqx-otmf9H}TJD#nse01LW
z2YbgY7qNZ%c{g19BGh{;4~KjcXP@=g`<7s&*2S4IHb34imRa_(vHYI+kHd{?7HrX9
zzRP#L_~~~2bd`%)pPY8{=iTqvT({!PPk+DrCljADtj+(ncZS)om!HdRCA?IAAN|4c
zwDsq*NlS`<^`<?Vwk^C-R_WJ|%vr1Z>vC@EpF4Z=o!--@yQ_|iA9x#|*ZpVl{;3z!
zbJzYma9isCmZ^U=y{CS?Uia+%)u8*!pZ!@}_gLirhh=()KiZ3NJ`Ji%TsJlFi@v;R
zegN0q`cqcrf8s+ul0zB%s{O8ioW4Hf)a|wIeT&4EU#8#O-)*?HUi#fTzV*U8w(sq8
z{`v6Ug!)s_AHK0I_4ubN;bDDd|B1GWKiS$5Z^S=c+7@(u&)!TOxvu-~QlI`yY!cFZ
zQ9sQfT;R2=(++(tiM#vKCzkB~d_O4n+&hi?9Rd^n+&j7VulNV2uZeo{-|n_=-SOf3
zgKyj4{_U$<nX#wu>7J8~e{?m=eoueU_}P9Z)BfgyTU&beYroi=cK**#>CX9YcW>Qc
z{GOpa>fd4Bzt!eFtvdwUK5rN6|7N%J==oW<FUz00=x^R3{&0Jv@w%6PXMd1BURiVa
z&axJ-`4anD{;}8^ap?ZKd}Y5^@89*8_%wq4TC2~^KVr&rv_4g_bV1eMT-~b0|K#=>
zF@67@^Xtbm8{c>i;mK>in%|zd!RUTTRppwK?=|)p{TGk3`K^$CpndC?Z46%b{j8tp
zhs^uyd0*zkKR2Eqv)9e+zu96dP+`QmGl6Tyf9{*-oE7-kTm@ZXjyrujT<@myZ<DRO
zVBnrLt^ptP-&o)3-aqA=!?h~?4@QA^E9CF}*?m3#!h%1mCT%yjm>(#XpcMD!;g-Aq
z=b!X``&cGbNyz@DespbhjeY-%e>|6dR!u(288fAaWnGWIQg_{b`%_+@R?D8c^!<!$
z>C(RFy*hm+r3XuszI^iKb$s<seEM1LfEVAb3vHY)?T<TIp)X$-and86rN8cFTUqbx
zz-hPb=hr?8{K4I{$Ui?I&-m^2$C4**HGKbj_VBs?DQ|vjMkxOE{Brs6{<!(m@7(s1
zS@mmfOY&dSzmMg=zpY^VwQsG9iAOBQQhUJ!yFCZbZw_a_yFIHg`v3IeZ#y!62B%!P
zo-gq>b$$KjN!J~l^|dp#jn8gSSAFtyvxw2p=aw<YvdvG(+qp)3Tl)US3;kH-`j0E#
z)_ysBUT$&R{}jhv_5T)Uti8Y8HQ<SSie2!OOaA8?{`FMX*s*O(-}mAE-mNEo|IdDZ
zB40KmPx9|4zyEihr6+!8uUg`smg|4~hJL8n%&7PkH{y9LO+Iy2E1KW<x5e}EsmC=|
zyMI5Qqvp$f=z+3#K&5uqpUvSF32%RH_*wcVz;4UM`5{Z}Z#J&#PpJ02@XX)u_HWUf
z`!6`^N2}Oeh;R9~fA7M&ngg*<XWVZJNzv<^`Tvnyyw-cc|Mg$)aRr)uyRYJ_HSx=h
zcYfD?giAMM?5>x7R<YBuAxD|zaizH|f4lSUw4VWYuK6?eFa8^NqkVO*bJ!terPC~v
zH+YmkV0*r#;qY9UHqR3g2VL!bP6Wq0x6~;<y2T~^=UmJAgpS0CZ@cPqZUsCk3CTHH
z=oAsaeJeZs+nW2mFYTvx{%QHW@yz;drS6vdRZfcS_%=JYD$u(&(tEx3eBWP7#N+1h
zpP%~gOj?mdtGO(@SMZ#UwSPZ5te$TE<@ukNRe!n#+@4GQ@?=WBA1}98$*Xa%O9yl5
z@5dqPO}%vwc*S?E<nD<w-*J6X{zcWh3v<d!ygC^gAOF72*c6|iQtj*YGgGzby}47#
zKCjsy%*{2zmFM*Qt*rkOzxAjym)+?X-!gTlCOrOldkX8RO+T1*rW?(#RoG;9^I6o+
z`cnd5_o}TI`=jl!-+E(zZpbE=<K};zCpV^h)hvDZ<@pxF`Wv>7Ypb@rw_O@HQz`a|
zMuBKe=CSvm+<LzAP3M<>%J|jUa${ZgjjxQl`&~EIS2cfNu+Lo+;C`R=+8z(4pPxLF
zqU-0KZu|YLbn>llAG7?w9a@(D&09_V>ZQ2f3+>h{wLe_<@8FDXnJ4Qz1og}-#O9Z}
zbm~1@E+&(`{`jBrx1~Og4LPx8|3e+#Z{Kvwe(TcM$A9);G_SSkT_`mBpFfMp_HA+p
zbE4|w8|vj|pB6O#bfvS>)8hPV<Jw7&|N4G-aXj}uSJqz3)gHVHAFv1Vep>stxc=Nz
zwxjp<Oq0Je<2uWsKYlvBhi?8;`g!ep{)_iJ8IQa&-s1J|-1a}80w1flWl6{1SfFC{
z-`=xmmGO&ErKIm~{~DXLcQf5rT{$7++w$GhB{tP9P~E@RQ6@aU;#RHvPvgA4{JRRL
zPpyCIv%%Q@g?Ke%j9~k<eBJ$iAAT~uTsFrnzhui7^$9h<^X6<%@Az|a^1j+;pVjS^
zg}3kBa(#2%=Kpl*BROAJ={`+1{QY+E5%%BFrT!NyB)NV$RZW?HYWvs7l)pi$d6B38
z+q@~h_h8v6@l}&w?5+1=(5+kcU-HLSgKvAe(z5?6{aZNo|6c~%7ZbUXPfYm!_h#bs
zH|^_~H_z8n=>MPfbD5sXx?lP-=d6nV_a4ou&wc!>NZ0Hi*G1K+BO*@!uI~N*_qy25
zBj!C4=a&8wwNt+@Z+{|0*=u6G(E7g)Gi~P2NPRYO?}PIad&9r3Us#u)9mw&;|D5yB
zOaE^@`nXh@Er&^7;HtfOd9zK$Z>OFcOWVtSH}3lSS^7xwSMdU4#w+z1!S8>zH(IZs
zJ!xMfN6VZq>HfdIwEZf~`1{*A`HsHk3A@?}-cRH=wALoC%stH$@jCdk{rU_W^#lJZ
zv*rHp?EDr!O*LGg&gIv)O}3_c6{R)L^Ay%^yu6)X{mS=wN{^1#?^=EB`^5--ZNW3o
zr7M<Pv;WSZDS9k_bN$YMFN-tQ{93cZ{_f>t$1fe;KD&D7LfhlNy>)*(Xb42SG(UJs
ziQ~U7-^PzQp}!u_tNdeb!nY*#-%+>HL#u0@-^;)ERKCZ}oB8oh{olaHQtW?UPM*T}
zbM?vRtN#Wy{@9`W^;G?zzflGZ+J|4bfA3ZAdG>qd<a(n=e<O2${K~#5UH0Mq)lW;M
zW%#TA#+_TYzk~Vx+ho3<_xJZ7p7dWU<?h^f?{(CgT>lF9Wt;O_aQ)qC^`+AHt$2+8
zsk46#yqGS>r!UxHS-vsu52steo-g+mE@$5M?tWvw?YPz4;JHhlf4ls7gDtmX>D(Um
z!Z)Sce!RDRbKCsN(WiB<!{xk=_q;zCwb-Vl@0M+-=byCy+D<9T3vS(<;N!E%m+8!W
z)%Tz8={<FBIqVzAxBtVXC2v`Sa`yP8|F8M*W%=805_8KXHuy`c#2=_*PpSwO>s#f#
zx|4f>%9RTH?@HR=&wDnelzXj9u6MbYloBd*;r4_r^^+@JU6y{rc<<)ELvcS7DmH$*
zcdMe<>AJZ4`U=%q9P!Dqn&p?)+t#tLTyg7?eQD?9CpIIX`^8e_`vTKip7Y&!Q`z))
z$Na62|6jYK9dgBeWy-efN5As@j?2w_cJ1E3MBh`jE@DfaqkM87evGcFJamSux^B1P
zt<Lg)Y8Te`ZTl#GkmY&3Gk0c3Tclra?};PVHwPB~^Upon{`-Hd!joX#Zv_!tlg(E=
zxy@6;&+u<MZ+Y*dE$*cqtLyJD?~U2LQO_&uqaE)e>H0@OA#<iqoT1I_v)aD)^+8d^
zynp%~S9g6}y^NRlWT{QR`@X5`e{Yuj`1<{?SM&9cJpLP8dp|Gpfx2^iwXfW2p8EBV
zTt15@PLsN@Y1TJwsl8>sd=EtT8TD=d)*hhxG5w$Lk+^E!l;6*s<bI~kIU3K|{Bx(V
zzB#AWMzzkp6Ef{;=iFKKSfTZY+RX>IrCFGMaqs^Bc)v4W{p!iHMT1<H#mk(Se{aEt
z-Scn#m-+B<vOw!&d6T62y<+POKD-Z4;$PsYs`o=hWo;ZY<Nt+vl{*y^<tDk<ylZLx
z_voql?^|tu)=CFGQN8o(xY>O@<9!+ob#Hxde4ctB$7c46`u}<J?-^W@_pJ6x{CJsF
zx%5p-q>Ois)ekNCWZVDZw!!rRS2Su)@uy{Y{WqE9`Riu<O_i-1=SgWVeW-tPgNx6n
zyKLG&9&^8Qd;Pw@Ep^-Ndva^9`^FaT_&d8qX06`NyghB7{;lKJH_u3O6MCfo##GPl
zf0nb<`rHln)tVnS);o6!@O;_r=%M%guVl#GZRyGf^i?(qKmKBV??uF#|9UrXe5|ki
zeZk`Yzt&@i<ZH^!{siAR_cZXUrRkNey5DB)dAL$-<@5NA{@s-Udb=Vv{|h&oU7OG(
zwEvTj)~=tYEBG$UdY_)(-@$(Tug5i^oG<zx&GRPKiwEqEEwntp>vjKSzE!{UCrQ=+
zX0zwAYrfa}=cwh2ElKiol9PWQO<5N%wX}YZQp=Ga!p{N?vucfhPuu(8{QsjD1H*%E
zz4zFB-Bh<}lKJL8UsqROn|c4y)hQ?b#XEBSJ^SmMZ}obSkMb8J>nfVrH`?`0Uc~nQ
zQ{Z3yOHun@PWztMf8Bcd)ckWRuZFL0{8|6Q?HlKt2mVh!Jx`eN!@Q13@%INd*W-K3
z73+h!R<Acq;eT|I|BRj7#{YS?S=n`gled0<;%^@Q(=Nb0&9}CG(j~jx?MvSl8{c`p
zwc<*6@CnbKd0LwnFScL0^-obXPxE$;{r^mI@173d&VOl>&DV>1PCxb^6svv7HnDu~
zJx&qdUuzfsQjmYn|9kzl{WBWAHA?*6Hmxo-ZH;%O;ID>G9l!q9_l(zNpUtmqUG_gx
zy>rXg?N=@w%41+%9bc^e{~a5@<exSzRrja!pDNdUUAjVTpX#rl(|*)!I9dL8*|IHr
z&u#qo*KK?KB#DO~xxfC2w{hdSW6$e-@zCFN7gavFYURdLe?CT+pZHx@*!iPXamUT*
zANS%<ZGK!EbM<!AEv1AX++QyqJD*q|ziVOrqF1L1zFroeH06iuHAZVjzWdvc&t<;<
z_3V{LU*)@>Oi=inx$o+)>0;m3|DU&CuIPV0vv<;!2+^u4>++RLs{g&OZ(sdFe{#&f
z|M@?jo69KAtKAS^`0Jnfr(eJJUu;kMvR_i+fV-vBiPQDd|GU-M>)iiT_&)Atd&D}`
z_CMw?SD7!A`>_AX{w<GhKf6DVM{nt!|28?lPn*lEddaA^`}^YDez(s5Qn&st?Uz4R
zc>IID{Iz$DjB=Z!<^8<QGwbYci@Wya%HzxRwkhu-CnSI2xMEqiG5PP`?+X7to1Sm9
z>YqPJ{M5g0nJLr0b|-IGtKmOyTD4;C?TbfZ0}uR4bl?9)DfC9_-=MGFt--(E?YWRu
zD|~I|#4|g8=Ev?YHTrgv^LBsd=lgwiCP5csq$3#HuD^XZWyw2_iv52JPwrOE-0ih&
z<>bDzT9!wC-LUMBpJZ+<D;;*~pYt{TnMya=57!5#tCYG~St-Bm@@;VIxSMyqY`c8#
zfByi>TZ$W8R%IoA6*K$)G-1DWUDmIY%L6`HmV9~i(Z*Nie%ECGx&QzDf3~0J{fQ-K
zME=MAdwqP3WKqn&Y_Dm?zxSWs_;ptCu2kQNYcwZqt^fc1!kf)1+0#<aPWrERoaL0A
z@1|v**Gw$!{(krV82?1y-r`4Jd>2<{t-O8q)&J!m=a=ih_&e#U{{PsTfX(GT-}+8=
zT|M<``kTL><{v5lqx9fgV*B^`|LWiDjokIvLTJhFbz5IF&S?Cb|Ay_Y`SS%2T~e-H
zxVioH>Qf8)T$&VKe7JemHOJK@NMX`m-~0c%GnQoW^-T!3p6D}KY0{~t5HXhz+t05$
zx<cnz9M9_cF6TqLA1<g%HEj9%cTE&ea`jum`oBwb1CRdI;tq`D_{lvllsho8<5la5
zsE$>wD|B2^6{l?2wf}`?+v@l|0<Vt#S`*cA$|*$5N%G+NWvv-HE}xw^4FBKUB5<rd
zSEuEdjNNRpMH>aqaxd)TfXF8+&JtUcsVK#LagRV2$IqM&#?y=-fmslB7q<xba$k(-
zIOb%c>vGx2L~KzdTth_1vep+-9p{`%^j+E;GZn44FGhB(g9){KS};ErBy!H_OH!T1
z#(>ok9O9XZyTlIdJU$nsXkF_I9hdJ;CAu!xol3+OZ5D{-zPMFjUFM58upu1N%}T_a
z^4e|FljRm1JIrmkMZlTcFrwq2(}|87U%z@f{r}f3o|K{3CU)rO-sV<`sE&oL1zKM|
zIPvHnx!}Yjmb3w+ev3eJt3+JKLXiAHCmu1U*#7Sa9@}u;ZHMaJ`WUQt=g<G0wrjeV
zi6vzzs&N}ecC2ij5ykPlc$s)omf|w8q)h^oTW3Udto*3XZMat;nA>oxz`BCDI!CTJ
z`J|n2^3iX(CHrRUg5qwbHz%EZz6jVJOi_es-BmHWRiNdqT=ASfn`?g;bWamY%2X8N
z-Wbubv^64%<M;jP;!F?sZxLAcHCE@yB_|!-7Vm4*#gj4>r->zP6zFaZFy3{yY{oh>
zk!5D$nTpd&4z&jDke_zoU+AvA{}+6&`>QRu_k#X3l{-<tINk5u+mrJ@)l(?%PR?zC
zOTXDo^g`D0&duI@X=~)`TSEI6mE~IMZF{zLVO8#nI6aa3{ePl<Z2gqo+r7Hp?eHxr
z``smp+5fhF%zl1dPQ3j8ou8E-K7TlRct2lV>!<CK;{N}8g0?U0%?&%g?P<KYm&R_5
z+zgGb)9<EC*uZU|IJx)4-xczcd)fZZczo$i{O8FYGwXEQ?bbe+>8IhZl$^yQtd;#@
zP4&CdXTSgU{tF8IZa%B$=ImMZf0yol`TX4Z{}=fx|64z-SO4?ArT$@y&_7j~kl+6w
z&34yXU;X{hvz~c2eS-JqynAlm|G0$pPM`HGcG<u=?CbjeJ}x|Zm3`Vj@6(rGpI!cm
zeX8(oi;qjr*URbGzn;H0<?Y`yd+)q@d)xp2o<7a|hw^j2A7_4d?Pz_H@Bj5S54W<t
zkheVf{}AVT!x{gDiz|BD{;)rvaB9EJk^9z)wO)z8+=D=@|1GW``e#1f|Gs5UzD(z}
zN&mko*w6c*>+t=0{zobAU!Mi{UA^(YuBEopE|TN7*^ThG8}qAX+xS=B+iQ8aS?_aU
zxXd?gA;)dYWp3Vnxa42BnCkzs&_lI<!{1H%Zt4A~-8f3_8L#>#kEL(S>!#Ee{H$ME
zrD^Sa@mF-tk9?LzKfVY2nq>2^NGjuEkLAJi0%fcD$IEJ_O?`1b>(8V9zke+B)XzWu
zoZ54DrKtS*i&qw#?X%0bJO8@uc;K8hdcOaD$t2G+zO{M9i_*>W@3!nZbeLsLrJDJv
z`A@~ZygVs;s>JWr^XHLz^KU<Gs*Ib<_hr)?F^-QvWnN6YY@`3;xeu%D!}D4q{oGFs
zeV)Jm!TsO%!q)H6r?%EFeXRa9;iT8E^_q6OZhH5>C`l|Vt9MR{eJXb&L(bcsNl78}
z-qrZacbKQOm|R~MEoSBZuKcFiAtmWNwcm~37-!${{Jl!8#&}l5q1Z34Wgcnu&#jo0
z7W@16fipt<0X%zltAD$2VEue|d$()-KMcN|uWo6txS8s;sQK{DiBD=S9WRVIz9yOX
zhyMHf*FJ{iFSEV!?EkkNE>iLH*H?ervH4r~;~xIga^-9O?)hmKeaKJRR&&q!@6``e
zR{7?uyq(CnA*!S7jRRxpvGa%j|5sl3>AL@o<??Hamm8Kb^gK`MdNzOIqWncGvX<}V
zKKA$M!FQ|v-2J`0k)OZMLPX4a!`8~j(|%@cdG`47<1Y)ZJ`T3F`}+AvVa=T5%S-!e
z?>(D%PI~EQzwrFny>n{r>C7$net!4To;fx54Cb4kna;l3Ceki{#(C*ypX+{pD*3r5
z5ybmF^)&Oe?w|GdHq801J)K?WIq&Jp!$tSj$H;A%W4`7&@9WC_Rh0*-D$C)_dmHAN
zuX+CW)>FpYm50mjZJ7H#dpdj0b6zlSeXQJux#n-4^Zu^fUsrjc?%sy^=5NG5?XgL)
z&%beA`rGHa$DbM=?|EpEZ*lH-?`h_~&ut*ydviU0i*wTFKDT||bAOM`1Dkw{^U~)&
z*M0uf@Ocw6nECos!|Oc{t@3xA`@Qxw^SaM%Am01!di*=q&Hi^NX4~m?_w~;2_<a0x
zd)3dHn?IxX#}(ZCb9+zCzB@n9<?C(VS2QQO;lZimbw1Bem9EQqeym)tM?KcU|5SKb
zdF>1C&z|D%ChNUXkNxSt%V0stLx-1#nl4RcU&<})_4HJle)+yn3riARULNYa)GE4E
zTi;7OZHnF<jo2H47FSyWkGQ?BU(OxxC4Owmx;dJ$5y87YEGtR$dwFW^rB>Ue-1T1K
zY*TOf9*fDVeHi%i(2`42y_a&&z9pajpMUGq4Q743F0V7}-kr4hwLm0qpV!s-a=i8i
z4mCAF>-y|?V@`IOt*`O9vDEYTvH91&EI+=!{Kk)UIsdk_um5r7Wbf_v-(L^Zh%hX&
z^9_1BiOb-}mlHEqzeueK(ukO^wKC&(!8uoNrhShtpD-)XT=rp8i1eOI*Y@ae9eVkA
z-jpe}OD3({6J>0-EzfwSQl`G2nz=8(pl3$PgrCV<J$r-Roi9my;A!@yXUW}%{rz*&
zYu?GeZC1&wXE{BgIDFewy$F~35TzN@^N$^G-*x}v&YO$;C(b)2bneFw30YUQlczqq
zXPdluWqB=|J>p^U${FANa<!K=$;iv!y;ggv@5AxE%G2$+ZWx}Z=sth)l;Ejd_U|<8
z|NT(#><yjs(eQm|QjJU*x5o1g$Da2%O>Gf)d^IY%rY0pj(sR!h`Eu*a^3g|kDrn95
zSeY;Wu>W#jyUK#SMH9qVMNXRW@%UY>ug8;WbLSsC9ru5A{r~)i|GXdeGyXF_@Sp9+
zeTM(i59*o!<Tv~`XTS0F>bpCKKQI2&^JnKA`PV-;#`?W`{=Bem&hz7?mX@*aD|V*t
zx??o;HS6@<%TDKBe!49)Mt{d0%h<fR`sL@gx9`1U6%S@!XFoSR`uXX#pP!!l`Rl^d
zqSr+`OLwmO{B_~!r`N!|;+=LSJMZj@$&1x5Uw6HI-Sp_|udVmp*%>FlD=sfqfBU-W
z*4JM<Z$G{E_S0)`i=LO&z9?I|9>hEM_Sc2CMX$?tmhQi^GhTjIeBNFC?eC^rf0xdL
zGhaLRpI&?X>9xm2&kJi`6z=@G<4#pf`PHur!D6o~cYfYu_hrwWs<=E`{qlL&+viP>
zKL5IJ-<`@h`Ks8wy?Wc@rdzLn?YzHu;a;u|r#pY=AA9a=xXN~Wt@%08&iniGe+4c)
z+$ya9+_-NBXZWtZ-t}Tr6ZhrmJKy>r_%!m;_G*<CU7VpC<*#bIjL$o^#Y;rn_o>4Q
zvw%w0BL}u<&*C~&mwRy0rnO4>Jz1w`?K|}={GUMNXW#hrt4rSA>~cMt{F~KvxtiDc
zy4)#?cCB4hw0hC3#zns-EvlOKx2tcq>-V^x88e(N>33Zfny)+M`rln06KA`<=sHyz
zx>7YTVe;Zdg3Iq#UTGHfzhxxC^X~Qe-)mS>w+3&TBPsRa;=Cex(N~Yy*r%R(n;*_D
zep;39)V_*02cG|)?`&_@C7rc@%9pq6ZaF!({9R!F%Rpq^>g}PPKbFiBU;p@D{`LLN
z|LxR{3dcNKz0;<!R?F@B(*pvhZr;uQ|Lr^LhGxF*Uk=kb)%a#Q2VPvz&2Cz0^+lv%
zDtGQIE?IuA8I!G4^m2MGbFsH&@;FyMl9Rm9n0V>Yp)(dUU#7I3<e4m~X6%zT!=^^@
zamun_iDd>pi8Brvv?g}71)f}XD!S`cc$ZfwSEy#INBzQEO9Da{?auAlXeu6>wX@QD
ziP@iB;lg*#mcDxPwZYZ2$+}S^cDZK!a?MY7LezFzr&b>AEMId{)OTG*k?Uk_pLGU1
z4?W?UT)eOKU(1sZ7v3r=Or7SGtLc${U*y;Kz&*~>`rb@FCnt2;P5trpxj(*T3n{C#
zdGJi!^ZAUG?XeF5`z90`J$@o`Cy@R8k?3iOMu*=7rvA<3otVgVSfovpCs<-xh`ron
zkz-({flum;QwCRG2T3kB@-t;loMF{8_r-&ypC6eV%j~%<aqObZ>8ZyIdJ|`fFE{i{
zpL5P&ZlcxUlD3z1zHxKTS<Ef8I?U51$zv>$W^m@iBc5ZDJz!?SjFSeF6PJmn8=Xs;
z17fNj_Gz2Rvw4q2ccIwhH46I#`$I*$)?G?l^<~D~<TAr^ih&j?ob4+2CLEET)Z#48
zzvTXftIpy5rDEUwmaNvBx2q!PrFH4oC2SGm{;@WtA1@vC7T1s6vt55x*Wa+vi=nJF
ztCRMICf@rS?_JRSfmd>$;t}&DU)v<^OR;PHGH)@zdi<2tYL_hSud!1<<fs}&bDny2
zz+fSB1gA%cp>my2mD{On8WP8LIj5PH#YK0nes#b=<8o)$kGWT09WZDOy;+&78ncQS
z7J3FgUmt}WTj{fWnz!UPy>_jIezI+$avM+eyf|%ofw82)INhSSZRa!5(BHc@EJ?VX
z)H(Ne`MdjTy$b7J7?rNqS~7E2_Tfst4r9l;oS%Fyq)WwxdYJUet(wWl!#q{(`h4f*
znN^{RS6SAoWtcjy$&`B0n7H8jmOZN`<TRxTo6J_Y#d1~cgqh>^%$ZLV?()2RcJ`NE
zmfDor3-9neP16V$c>k-oC}Bp*)oF<f?(jT)_9Un}d((uxwjyJYk{n*iQ)dI-DLe0y
zyma;^m*L@*%Up*Yc1oW5>uT8(yvD|u2c!yQkh$~Dex8?U9_G$FL0Vu2xHpDzZ;4uP
zm1(V5wxEq?W7z7<&{+~)5>*me5>XOf5>_)7{n)LyAv-}M;YEDcHRkRpk%jt)c(h+^
zvJQOxYTfBwdark_dA$kD_?_B+>f6ek`B}Wv)K}X?K0W!$BL3Vz9p_|CFMZZ1mw1=Y
zt}5|U{G(mo|Gu(J&B%1(%I@W$r10|5R6d=#$NiqWe({~ZSLxNV@I=SO>RFnH?#*w#
znzyE`_{)6tY@5i<Mdv1)&U<sQV#~3`rfn~IE=w*m_DP>{&Y(X}cYYx_5ku0g!EWgu
zV~KQwGaI%ZFlbI>JM1(0Ldu+z7LyafsrFj>?_jUosjI!bRvvm2BwzkC&_JJSdiVP<
z$<0Z-l6EHTNZOvXjmxzAeAruti~Co^Z_z&`ez*IT>*vrXs*i*+J2$w5DQQg<N&Q)6
zG_QQukvY#}ZQea96z|_%apx+ZZghR$im38VtF&ov*XXi-<@&z(RKALmU}uLzOM-%w
zU~7ln0=|oF8qOt(s|0*Ilw9}&nll`yC|C)IcCfiHR{h^oySI98<=*nWrF)C_eth+%
z&1;*4f5Pg5Y`b5k`G?(a%gvr|!+M!h+PLS;2BX7grf_PXKNQEf{=M1CAD(wYTzQ`u
zF}zW-dlgi_@mlYlIz!*bOx^u?A!qGheq4Q}KYiN!caB06f}VL6P7GPV{ZQ@QUfc7Z
zKFj$(a_2QZT^6<}a>^mSO+gdags5bT^1uAS)F|oHbn|~;Q3;c<+ROWEN?yjU(XC9M
zw#WGN2Sd;4Y6;0=5=OfeUcXwidb8d+hcgM0FCMMw{+M95X`SA#T`wLj>h9Ll+qDD4
zkarCikBu!URNa01^S7eNY_G7{49=It<MdYhom>^nb@ybEZsdlgMTHI@TCd-~q#L<M
z_4M|X#7R27la6u<`29K*yv4(Fn&wF-))?iJ0hVc9Gf(NyuKw>nPo}c;$%=!S$>Mx`
zva!>+K6w1v&T4UKLUYcEi>5C3KJmSb&s*7Td`_9gvTd2(J>Bbn`D5pn7G6B+DlQ%y
z(=|71)}D2GyKeoze$-VwJT|7Z@M)%YKezhTqoELP?88QhO#$ESOG+;v?LB(wsIRzr
z?461i%T|A^ddT#8*N)eZ)^%HZpLH&ey<1v({iw4zf2>W(#|uZB+vV15S1sJ6;WzE*
z$wL;AQp?5Wty^H?{UoJxj?SdXsYROt{HC2+*?voYO4`XeIx~a$6JIdu&Y8TlcvFSn
zwDYG-pK$wMRI-Y6TP}D{i9f?|uFi5#soVQEEb$Y&dUBIhWQ4l+YmrA*PZBmM_)Kd#
zxyT?=OpW_+O6Ck5zTgvwQyORJ2wvt|FT$L5`bf^q8M;}URL?3$UrFhnl@mBwH*1rt
z@3s{uHSYw)9f9%9+|NIIag)b(_QK}{bDkEI%v_va{6!}1(Z~7hJl=*CaeT=aZS-e9
zPyM_nXWsO)pEp&;@j+>`^QWIx-h@iZKxvqwZ*eDe|J>QwaZl&wY}M@>dS}mk`tQq$
zcWdgq|J82LsW`p+pX@L3)meoz*FXE@p44!Jqf=1GS>>1q&m;*|!!ydpe&0+a)Qo-h
z$p3yTsm33F!eVA2xYnza0oQ!b0wtCk_<<_!sfk*LL)unu)aeP9TyEr(I%9){<T67a
zXuY>*56@+ZWrjY<GoBrN*>hQPIi%#cyS_7Vm-upHzw|i=!6gN_?%QK=`2Flcw!=Ja
zGkJ_9(~Qo1dE|3!W{<IC+N?Z@G@~;~GoBue?Aa`lZg?hX#!-Xr#Az?jcQ>cqTQk#K
zdk^mlby?rm*(VE(W%HET3^eo9UN$eYO%NyzD4nqN<*$iNo0_*YZ}Yu#^2l7DyC<K_
z^|{}l!sV~_F-6K>?Nf@9|FTahM*gbXVmoV}YtFhhxzl#$jmax*XWj|!bTv7uKDox|
z>~bmVo6NIbO+IONHNqvh&giB3<vOE}>caI#UzdyRQ53sqtx(CW^1ktj?}R;^lgb@G
znR(cXU9?vGnqV$>VYXtWvC8|lCvPXX=`1RD`83Vrp76=pO1%jQau;qZ^+ve7?|HIz
z;vVx!=I)=KdEApdd0Y8xgv<L0Pjn~kai3)F`AN<5p6bclDr+NL-cNb5c2ZbDB-@#L
z+Bd6JtJkNz7xjLV)_GTF<80Nk%}4U4-8d_>J2Gke1PE=8BB{9X`($U4iUR^{?OXc(
z+dNPda8jz+oRUB7&Dlxb9OC>h_D*s(fN=LlZZh}&7IzYCO?$%$a9XMctKB<kZ}_I~
zUVM%d!HV(}86^uY2Vc&zdwqXVy}X8jju)s2`G_GF)Pj`hygTuREQsLCC{rnOPbya_
z+kWJoM^32kivtD=J%4^R@R0*Iw!qD*KNg^-l>v{?Zsm>EsKjnnh;H|!?}shVz*<wu
zbDlj-odIfGNgX!P2Q{uN;EgMu%X>kMtH<C325MX-%y@WIvhF^ld6k0Nyt-U+*kE#^
z*kK+0b4hcKT6BY$ZIL{ied^<6z)qfa<F3%|=%jM*H+h|RbvIUfK<M47ApbxmzfV9_
z1d&wS2%({hASO7&O+u1vKk^PmkuFFlKgc>HMPN6`A-e@8X*&Q8mFA3f=bzD(UoE)V
z-1}SFNl-#b)sVZusr~lzydo=I#UtFA<(xgqj2nV3*0ayfTL1aB<{p7Ozk6I<@0aDI
zYyOkr-CuuKL-kIFfc~9*$>kf~eZNyLw(a$i+4r}d2p8FNR3TaaaEMv{+q>?wr)X##
z>HGWSZ>DLd;{K=m?q8b0XrjF(&cK=NhE%_biPS--KeLYOJ4LcEeZO$!mb~mYg&Wsd
zl6Y?XZjlxe&)LGU*jd-K?)P=wBEIE+a)0~xyDr&#aDSQK;`E!g4}WW<MO&Ah=Hr>f
z(xyB=(y9B>pLr(w8!JN&C>tJI7ZBCx7k>7|qV&FLPp->u@%r&!{NdfDDG|AH3XyF}
z?!S+*&#*MsZO#h!WEE{*rK=<}_sL7uCLc8qhKg3LJtw*M#H$#*So)lk=l9%_%`-J;
z&e&x0eaDolE4f$Jy{et2u50zZc=EMn9MdDuDs<MYbNNvycsc&aZP(bWwPGwgwS#qB
zliq5(-A!ITgSFzT-;*ew=Vw<6t5jZ<a!Gi8d98xnD}^cFTK3Jo#HFUE_Uzo<Z;on;
zYh+{hGnDUde(|RGRx^{Q%htnJ(lyN{rBt0wnc^X1QG7_?&F<@Adn)_q=X~<=wlH|4
zcTw!xq$Tl(=I{5f_E)>Tvsv#}VP?r1zEe6ff4KRlJWD+PR>?Tme_{L$74GW1`B%;f
zs-I+Ew8w_&V^mLqz=67kd0UFrF8Oir{Ycf_d!0w<N0Zpy8_ykQo9jPcpWHA*`r~rr
z>1WP<U(dD1-?zO?bGN#K$Wpe{ZL1${)Ru8dnaz>jxL@Z=-b4MDPY<!Rtx(p@HvbV+
zCbwjsWW|O=!<*I)2ka9{&KDkA_pMH-xo7|1Y28aEICtbPG4+3RCu`mj8?nHNBA&L+
z-e%uEN=yHrxR)zcu5$9L%1H~CwQRL}T6A+~jo|gUQ$?0>yjrnZ@94K~i<f@#vZqev
zReXFMc1T#iVC|cS;=hkM+<kN6(CRd<?r(xKeMElDyz4Cea;|I1tPA0jIuq4TIMrm%
zKa;%tOzrZiJbg@agRTV4_%JiV+e_glf6_whsoC5)IxDv|{9PFoDEi$vO425qec5)4
z7f}T(v-jV-e&Fnt)@OWG5n`HeAJqv<uiW$I*u#mg^Hp5mNN-j@zKC1lqnE?&r?1cP
zf4lwu{4SMHjhFYG*j8MrR`1m0G|PCf`t$4^(~BBzruVN|CbNcV!@pb8?rySjzxn6i
zuUSD)Hg!C{nD9$dY(mEVMOFWOCf%@kvG}mwsxR&Cg_C8tJuX)?m+a->t@wAND)QR#
zd4hJGtZM!~()Nx07j)n3nw0jvd{aS(=c@GYf)ibemy3U#eM-8BVXxY{CKa|@r<7TX
z?=SngPtzr4)~<h=V);HlmS(GaayhfQ8tHcZDpg(FuGtqGvB!1&%q*omK2Ht9sFzZr
zw#uDND!k$4Jo`VWB<<$c^$mW-C{%Hvt9#86H}%G|LjIE~JTwj+RF_$$KSOG-<QBuS
zX}dh6Z@v6@y^}j`<+4`~=4dWhHRltL?j;9>jm~9Jb2zl6U4GZy=N4m=nzp6Yb)BZI
zQqae)OYavx*sw;)a_O9ftD6{>FVET%eIsM_l4q`yuetunWbNKI(^xaYCa`!(yZ5au
zS0kmu;DSUBX5Q9qoRijF6i)uJ)a2g#rn%y>Di=8)zW2I(ucY<<m$pkidy0z}dv5qC
zE?9Ph*MIl#7pa@yPucf*V&m-{3MXe}$XgxRbJ4FkTuaf*sZM#zDZR<Zw$D}+eag3O
zN5p>LjrVI02}>2~?TmBY78QC!v-tY-D}BuFM+%w+I+jO%5dE>knp?kBJVAPSd5l%^
zIhFMn#GLY*o@vau9=5F9*0D+L4wJ#c<1VSQy{_Nw^szOqsN1W~>B+~u<3Lkkv+?dW
znY&wDHn;uTdcSm1-~O^BNwLnG*3D(6zWMvl1PjY4t#-7Zm({j5sY&VD6FJw874PKu
zyyq-<xyy6Q(VrXmC7nLha4}u<*NRCvQ8KBfW5e-HLVI0mxZkfA$-6uMQPa}iZTXic
zH8|d3ZvFi`e$!z#X{$uTi3fXI-QS(<I+FEqcF)p1RVLYqw|ymVb!#;Kc)c<!vm{xH
z>x<_xzeb(1#p&f&-{`BHT>4N*^v0`yM?_CQoyh%DnfbPV^$Zh-%rBGpxYIKEKmA*l
zzNqu$x}#b*vp1cedi_m{!rzb?7Vn)_eVHe`se5<Z!bRc9PkcPj@kaU9+{=IByYV}d
z)TFfEajy9pe-l)?AADY9dz$HE`u(5WP1P5cR$u<~o@J83&F|ZH>kCc0_#joupgrX^
z^GW@*4dL^7W<TjZsrG4G;Hz>c%S$CGl?RuIZ_6~aPfO^zGdI8_D%JU|lSf;4I|sL9
z*NMZ5AFkDQz4CBhKGSdi{1oGjp0DrAS3c~VnIM;&v0Osn^tZ2Ug6}U?B}6@2;IRAu
zoycs)y`833XC|EWpS0*j&egV?N>k3dUSHDkW=Uz+{0Uw0FP-|6EJJ=LUQAuQ?Vr>l
zMZO@9D~(sDoZ0`vV9J*XHn-O0uTs7c<jeHlXriv_Q+J{0PqojNY%$2%BeYO=sU+{l
zlpP1H%VM2A-(DS3{%KK>GVit|ZL@?2|9Im))c-}BZ~5gik$sMz*R~M0&grVQ&Yy31
zCMiXiRoz*dK5HR^@9ag!RvtHN7e04n@+z$ldardWk1cbNb<iW9%%=xFTP=-S`+Uiq
z_scHXJ&Fjs^xZ$1(R_vJv^K|kyL=38i}P=}t($S2ZwasbR<ZJhonc~@)=cx(MBTVE
zqoO4w!Tqd@lLXJB)3biAo2tGb>ieeKndu>h>^Bc=F3SC|=w!_1w9Si3+%5l4G88s1
zZFgGoK>W=2Wxht7+g^Sx`D0Rjb?ceEhi)G_YOed~_93sevy!Cin10NgwnM8uNnKYY
zomHNtZ_Z2mrE4r+@+BTQbacvvRqYyq^;>3ff2qjGs`=J({_d*@Q}<t1-?M6#P{0<m
zIXBm=Ew4Fs(JwGnV)y^+ua5iuc6~O{<UppF?Gyvcsr7r_3+;Vzw$Dz};gfpjq{3{S
zmL?kmMeQ#gsZl#Q-?!e=nih3y&AFdDito(5F7f$S<IkY(lXG^9WdBNiI4Ak3VZ!7p
z$#)$0teYqH>+foMx&F%A30oRgB;8jo5uPV{Tj%n3-}Upt6q15?q9>V4*v;WP%5fuF
zp?J@^I*Tf`9*?`%I;=Ln%bn3EdZjeC>U!&v51&={q<^x^RArmT{8OadT=Hs%WZh)l
zu2m5)q^?ah;5R?wyY;dzr>^7Mrv?qPm*07_Q^qWK7RP!c$=&60JJ>2pIFGp8X}P}R
zO{?UF=`r)mnmv}rJxcq%q+R&q(@--m!?67s+1ELxr>Z`1k4P5SmGEuR!mu}dHhWeZ
zNqwK5ZEwBab>q9g{qetJG#_uiwfADB;MKGgtExS_j^1+K627*a-$t+Od&`Tbl65a%
zpN(F6vLpEKtxtzcRL<^Wc3N8UeE&gZoy--xgLZAsjp#j7ShYB8&B0C^9zpg0*R9Sb
zRPLM<`TAbcCBOfBi&nBG`g~(Mce3<XQC5q#hF;7Dj_=E(r)i#gsP@TpW9<!}LyQyd
z=3ZZD`s<x)r^9SpY3}~gw-#5H)|Y;_pOUvrnTI1G;XeE1zdHisGp<KHZhar*y(=Ki
zKFLCVoB3vk*Zw9FM_&hjNn@Kgh09%DN8ir*eO%7fKDo7Xk_|uio!i0g{^n&UulAF*
zkvC=UD(PnizSf<)WOmw~>2Dcw_B_q|*RoVgyz@%6<<l##qh5aU$xr`ysp?JH+8Gf)
z_%y%GG@I+v8r@vJ*hnN=Vei7pi}FP8EGV;owbx6+(sI><4cj_3Qu%U&7<Hy?I8jku
zaXMixf2jW3B_;)CAzQ+JPOfBpq^I;*cBRG6+v>4d%eT&%J4sg3Gh@Z>7w=YRWgXvr
zKK_`x#ms6O&s!_DZ)J*XIQDqj<y#)w4%b#rHvE?G*!QGb*@Bc)j(fKMYia-BHSw8K
z($UrLdyUsV%(*7HD>i$FlGL84Q?iZTA9tsig;#~VS)du(r2axJ*6W~i4NLCA41HZ|
zcI|EVd6*9@oEOwtaBS&?P8&8CSDBkyQ4{_yeSb{y-TAzvtSNyPjf5Ddmr4lBx-F{S
zxb*v+1(_btl&k%lRxLi1v&~+lXVt@RI!W!!asL)HX1=;+YW2kH%@#o=$-^!eZ6?S5
ze(MvXJv-yx?2rozdd_JaUXL%n+obX1<Bfv({<;_cZ99J1k9W&zHqpw`tv3p$nP&dg
z{GOtCcZtGo)AEO#yaeBrWqj?MIfL`<g0+12UKVY0Jglx)w(d!+jrR|+E8l9u-(=a(
zhze})^5vDAmbLWCQ;};r^QMK$DgVoFF3&pqGDmCa-U;q@oUfga{0hxmwbYAE{bly+
zXNPOUAF>urnlLx`f~)-DxF?NbEoFUW9a%lE{Q~Fi5x%&nd}G)9vpHpTd)dV&nDCyP
z5FK&4YG3Eh#^4}3%ff<>>ym6WZdcz;DOJ)_UGRi?$#i$MBUcYM?mfeq6A=3Qx9Zok
z4X(TvrE*3FToPivyKcR_dPA3a32!vVvuSr%J1<=?c)r+|DUR==?nhZewQLu`L$kUb
zsO);P=;0K(tF=*%f6nx+NxL(%>y*FO`~n}17l$T&I&;^pH}2bvZzpF|Oy8O~JHeWF
zn#bYIo9-A)H2;~rg<}OP|C>3rrfQO!T@435ZIu01GF9N664##HtlPUqA}+Xm+ZwT=
zwq#D2jaqjKTj|rfwPC00bPpanwJ2Zr)tg%$Jhx5nw+TA`N#T5^8M&G_Rp<8d*gf8%
zeKSILs6>@Du&S1r*R=^V2xYmb{o8n>@20uVjJLBxRW9kOo$uRuC+BL3(xY`*!7<Fw
zLyqqC^J<qnl&|Ny|GUyT!?#UK?(SN{_q#|=B3#3~`w`Dmp_hiDcQ><cmziw7x%$Do
z-V2i?e*beiZlm+&kW<*w`|n!}-&A`%JF2w%P}Te3<JViHbCu6oC2;P#C}%8HVVasB
zn-sDxN1dU`vMV_Dkw@No7aNx3RUXey`e!wIh$KhNzWY*Vjw#on=_#j04qn)H`0Lgk
zOC@xt$qFuDF@N6_K3UI4p&;V_?F!3tnw~e)J|9}&>HT(M^08}&CdnE)pBJj#;kozb
ziGl{@T(2b}Y7;XO%jfZhI2uVOL?7Pi?)6skh?d-sNgLTc)=UTtJ?6+R@|JT)#vyI-
zjT5u3rx?xlnNh%;=U?Z-6Z2;B7X8ZTLl<v-zchhCKBu)+)K$cC%IzmlCP-HDDYIW?
zxn05;RWviKsO`arsYW@IeGfE!R64Br((Ur9&&PVSRCb)Om>%}>$+9<BR5&yz+ll?t
zw_fO?`R$C~xmU9*7w?$0_mc0z6v<~EAD2!ydu#dT$m77(t)CWbe{zTa-mWdFqNnuA
zdORYhaa}z+X_{h?8dvc1H?i+bv$I^+ZWnJ#zNK?RbJKHWgBM}%cVF|(ntD6rWa6op
zUhYR0e){l6i}7o$-O1ydJuDAdN&LLQHCgYYx9N1IvI%FjxvrSJ|GGu#?{|)!c~?>%
zS2=2C&zSYe|HSctz0n0Oew#lywS>HHm1uL~{B2dv&cq{O!oVSvn^A5tIj5grH^Z{a
zcMI<u_xq(WLVjXlr!pEC^-^C?|8Q4Za<Re2-U|nxvCQk#PkW{7voSH4JKRcBaqWtK
z)A_AjB^)bRB+FM#n#1CixZ6{5cE>K(((GFH+m+$_yepPwZ7zDV+nnc1PN>sIohb)A
z7gQW7EbnZdE+l@+PV(l|zQue!?R}+}rj|aCd$YUDB9vVxw=36l>2ZH;^|WJ4em-q%
zo@233RL)Q-<$ceE0JSR-I_p(4pDHOhF4S<2Tf}?4lvirHSj>bM)nC#g9$05)TPXG!
z&6rvcv0AO_R<L7J|EK3>s|A+jR6hu8o&4rQC&#UubC}=kcCTrjY|YHRVe?AGjT`E-
z)8|KXaoME08khueEo(B~q<UjtsNNi&x;fP=-!4iQl*sw;^smv&g)LXr9^|-+RwR9q
zO@C*%!N>Q0>EVNGWv*p?3&?qzcY~ErK&V~W?^N#Oiq%5fn037*(p&%bwul=`ov+*R
z`NX2*E;TKFM<;hy-m<#-xYXKl!ZB&XAE762$WGMLewKMSK`tlfrl(_|z#aLSns1rg
z%x+{?%{eQ$*<=2Ff!0sO*_K~7OpV<j^GJU=*UXsZ((7Cdx%W6^{>n;PQFJf2&hRAv
zt(wp>fjFUqKXp8g&()h0>XqJf?b(x+eeZJ3qJ(bBZ+fHlA@nz|wPwcAKYm_e+mEOR
z#joW*X|1ugC}CayicX1h#V1&ngsjO+Ex9Gn+$5cK>)jL$)s|};zNYqX_c?ZA^(ns9
zDrMFScN>^*ox<Ie8>aS@QCw-Z(zXw$j?eo%an9EHF>Z2(TuX~?AG%#>V$t`l+GFdK
zq9EHtevHms1r?FvdRwzBUz@O&Okl5k$ojp+X<qf|!tJNl-gv=Xbect`V7vCrDE)+q
z1(WACuie7$bkB3@{#$0ctqcVncCUB01sS~Ea%KCHZ;Q8iO)2%6!TzwJ_SI>9)xhRu
z(}U+O2!Hc4Y}u+^CpCX>NUVIj<VB3a9@%?=`*nJrKJaf|?`?ZvI@|3V)xs|$mpwL`
z-D}GE>eCj#DNk1!E_lA!`u-BO<N~gMiz>QXg??7sT2vc_>wT?QQh7y?m)*eVw`h$#
z&#WmrF0<}E%IIq=+k5iqm6k_SShm%Cbm9>g&#C&9xLC#Ag~69={V}&WSB=(Oy0$`2
zvD~P3($3F$=RBtM>dO2+?YA#B)#pv+%BtWk#s*vo--UvA|D4Uae?m}9OZk(8mAu?F
z`70C^v`;<I>}pO^uMOCx+PGKoYfkp5LpRUmEob3aGgHfrH~OX+%f;U(-C{(QbuX|z
zKhml>&*DVBz&>HK^6lL#qI1`2aL*MzeYj0~UL`AY4Bt|ZCtp@)*lgSz%DT-Xe$}b_
zz6ZHHQ^G2>Ur6Qb;!~f0F66PL`1_cdOYFAQ-k$c;#%tBo*S!xiE53aQw0OGZ5&!kA
zyDcN%7If<rxi7uM!RvmpK4;&tmEZgxoY^3?^S`?L%PX;J3*RmI?c?e`ZSRcToA*0T
zI_&0N<tDVfEK$>J-g1i-Yqy_OWZOIAuAAkxgj|D#PajQooO5oT%+;GdcVagu@@&%W
zW_6vR_w2#Vm{W4w8ikr&Gg7=RDVt`xw@y<sxw<RKdC@w-V=6L@)}pQ}4jzqE6Habb
zSbUh9;h>GttQnJ*ZJj;+?VPojj+8%i|5SZ*^A7gLq*K1Dcy#v2Z4(#CzGM17SLUS8
z$$KTMG%qmD+8%NEkAbvoo$Kj6oV()uJc}nC{E#NQPFC}Ui%a^+H@OOh)}3JneSQg3
z)&;&kc=5}p&7BDy*LWr8=p2@>nr&HOFDJqHGsroG&3+Q^eo3|INm?i0z4a(KXB6L-
zDrTK#x;i4mmy`R+#+>5YhvJ`XYhkz*yDTxxJ3h{3jr!g_;+E^9Ll)fZoH%W-n}onI
z>1mH*PM&-A^H<r2RL%^Z?;GFvUQ)KSxAXlWd~b*1@)IJ>xo^MhZ(BU8?aH1k({S~W
zhS@bccetIG*ZXj6-&}{4YtI(4tj)DObl!8@yS7{VR{rs0?Kt|dJ3jb*>z=(nZcJU(
zMHMT>vadvUtQ1Y$ch>RjSw%^+qe52Qn^(4Yc@{ZMcWMlIdsp~u-=;h+(MRTThvOD?
z{IUoX4O5+}5$O_TK7~hl@6GiN&2rjmNd{ju_bKi8^+h&{eQti@rHsS5J3e#@ncZ8W
z!@GUz_Or8DHmVwjZ3`+p;#s)j&~@vdJMJur2=h!Xx^(;>-|dNiHd!*3X07*4_BfH;
zTRYoO{^`?+x!zIR4%c?G90@Uu2wVK$O84f~%b|L?E^HtDmT$524e`2l<hJV^lgCoC
zG+3{t#heJ;y!GWP@mCVcvzKk^I2;$Le&xYto)ocnI+;5C^Sq7O6Ir_MX{=~4<$3)>
z`eEO-HQ&xSoqTDd(q{bYdWq<)V$UG1=h<cqHs$vWEVuAowY1*7R^8fv$@1R^yQV3d
zulnI_v0`VcGlQ$%#SM|`r%pNEV-$6%*!*JccKdLj<chshY|WeRdFsrWw96~+HV2=|
z_4RMI-Bk09n7Q@Z;z`Dy^U_842)^2AHuw0w!YL_2m$&Y+&PlaoQmIy%U;g<(?6eh0
zC-poE7VW>|EjoR|+kR#Fl$DXjyH+Q)zX=Y~_%J=kU){NUie=?BQKJ=aR<3ka;kR}_
zb!k>R?+c%GljhEzx#HLraZfSl!-`pJ`|qBqcQ6gpaxP3=EFpGp@4ep*hqZOf5|?r&
zgavi@Y<$<-+HtUDi%!ABgUbTWY}s?O?df?xzNwoeD=xb&^HEw`oM3sve(ImTRaXs{
zAJesroAQ6h;hIzW<~xH=WL|n@mOs<gW?o5k{;@Y(e2$n^{fc^c^39vM$0a>YPp0hW
znbOrKxp5D}Crx1o*>j~OqFxmbzFn0q<Iz4I&YPT8XJxTx#)-v`11D@%+nTa!O%40!
zqskV0miYaPjtEvXRNmdzlDk#)!t7G3_S}=bF6NPHoj-hwN*9DWSZ=&r^s7b4hvSmo
z>`7<mwi-HpoUeIk>eBzb$K}2sv0K@#@l(1{{?S!c%RPzWw^feb?NmN`DyeN@%Bup8
zg=RjhPW^i0a_&g=8m2QjVn=^(oVw+>@|-VQS8GJ{&i>3JBscX{#JQxtAcIY{60E|x
ziQClw@6isLeOWp}H~j7)F8OWJ#+CC=&B-z=Kl^Z1Nz%p1zIy6g?LD@fS`~Q8f7On)
z^Mlv!v1eb^!S(de<XHw%<(-Od3s!ij@h(vaT6H*By2-Kn`tFkNF&&#1x0@vW_0<m&
zEV%qpe^*||br;i5^EUK;C=UzWB($|)Z)&>HL^~UScdYw+*BE>b^<Mnu$^xJ0#W`N3
zb6@TYo>Qn*Z7P{_pz^Ag#0x!(6DrX@ZzC$_&vCgS<L4P+aMpE$i-ecTea~2>8^#Ac
z?s*uyKMgrjIMGyF>w8(0S(t9emK0X@rW(P?SB|A*&p$KSCoiv0VSUpsm099KL7`@@
z3xp>-ZG4!z>+lM>$xbDUCJEPsD{fa)VlJEeR)38J&y}MyUKdTNxyciIF6bVMpq}{b
z?Ppw8-C%qkBed5iJ$mudcULm3-K@fHu2HVeVrhS3tQF##7q&AxMXOJFZw<F-SkLN~
z>&2G#tGhf;TD}RX=Z*=<&X7o7X1=P?mx*cd#QfPBCfjxwpS~v}cC+<M3$Od7pw8tx
zf)_~UT(G=3M?0x9=a=$f!P#8SmPRV(hA-}JEAeRRv3wPJ{8@?fPRm`9EQw61@h6_H
z_}eNV|75=93!VCMjg``#o3&Clt`~bI`;?FKb^c-Y23D(8mEI}_E_qFNkMXSgaIo*D
zZ^`X>>`CAHWM(IxOx|w$cA?*oHBw6%oeWO!h3c5zii$h*CM)|JQ{?KGE1Yl4<6}`_
zpYvTu_NH#Z+1S+Z1`Ekc7Amz)xte}P_sb=t9#3>hc;Mc4^rrjw$*z2tw?qofRIHp4
zIN9OiuY*UwUpZZNM_W*DLDgNSiit}m=;kX=-n((R*dpcKA*#yVa(eG2Ma-7Jl(3Z(
z`)Kv?y-VIHJDZ#}rpB{HpB}$d*|sCz&Gcri!Q=^lj%oQk_IM{HekE!9s>5Gr7Yk0>
zKIf}GtBBR7t=B&n9*R&|u=C{1w=U*3j7z4U%s#mK{_C)lt5mM*-jTf`7g`}bd!fK2
z<}XRN`Y-h#Uo%DRZBh2EPS$A8byk%sLeYOJgv?H|86Euj#4kN+u85a@)4PqfC-aY*
zKCL*qD$sXfNp8F7vCHf1)nWy*_>!g1r9OM!<)R?8_rZK`hg_HK^UfXPSRiA%pz)P7
zo8F&IN*9;y?V3HME6>_nYN_tbRC|@+l{eNZvp-GMG1Q-*{axMq&ds+X%hTJh2|J}d
z|H8X<?w&KEr$kIy{fao(y?PMFT;VdYx<O?5hZkk({ZF{ll%#VaUGJ+l{58thT{QE#
z`X0Z+R+km)Zv5RTpO{`eD?3elukSO9n-6EpvwO*WIdIydE$azih{<ZtX?II*oo}0L
ze1+{b&&_MKt}{-x*_%XN6|mr3<Z5v0pMG=9L!E7bDHFOpFLf4nUEma49%*(ZDD?dy
z$qBLHOQiXar+;VX`=Q};zbScYxTn`Qh2z!gk?eutUC}qrOq(3O<7juo4ZC2z7R}9T
zm(?b^_pSQ(GIrC$1uK~}?^#M`p5Qc+HQFEhBu>y%RKL1R>G7et6AeF4HPY4GuyLZ+
z#}Mg4>4_G*w{sMhaKG>Eiu=i0xQ#dX$1xjq-7P*w=NZ(OFWIj6@9aSt{$(D%ocuT6
zRb_uOwahQ9SH1H3vCO7zJG^H#=|z5hWVPl<-JL6kH<|jsy|8nmm67#4lW7K-)0wYx
zH2t;rndEzJ{mUY!J6}b^|3{e#Y!+d>nBcl!^P->b1ur#*oYk|1b#7i|ni|Pvc;=~8
z%5~oz`-7$Y#pS!i41Sa^J)^ba5r5Gl_8mz__donp85#4l;9rBfONNO1M(?j;tM4jj
zC(TIpUvzxa*CjUQw%b45<o@OU)K=@!lUf^@m{TIVZtmV}xmY{dOU?S#FQW%hKSXcb
zJpJla*QcGa=^Z&P_ll32eAwqFXMQ+I^>f4iAkmB|6TFQT1JxT3h01>BoZ&e$#zt*n
zVpF15>45_i5AFSKHu2+YuG~p|V*l(7TbJr?%wU`p8m;$yC-ePI4K;(nubJ*Eik_=S
zuB|prSXN-(z_zGqeZs8wPkppDs=a&N&{uL)&TrjItFomRT!P#-*-o4_^{1M!`KJP*
zxS0p^7-n$X;b01oKKC^CaZpu<$<&2AS-M#Dy6U3-cYRk)I>S1t^y$n`9~@8dim;2b
zN|-U-d){Q=*mC2rzouK^3A^M^UxOd`D;_-O#!#easlMJ?aS!itk9jX%CWf4SJufy;
z^yLc+H8#VKUCT-tZUxjO&x$?E_oZlt;g)Eb1u7Ot_N>>xrR~uoy|R+^?LHwdPgR*k
z8bXf*4+%eTHJj+ty`;<6Nz3|w#L|~%tB!5B)%VRYGvZ<I0bS{!%c|D4w#u4ke_Ts^
znx8UNf8#ny(+Q!U1ejZb3q<?HVrQ-p+o+e+r|Uk|O8acq^i^Lyr{Cof<!kb2a^;od
zm;1BOb-CG9i#9)Pcb!!0^sJK2AwO<EUHob0gQ$~^*KX{5$#Ct^qS-6N`Mpld*56>M
zGdkH^^8NmUoGkAtZ{M@D-h8QXONZTf=9FUj2LVSTq;E=SNLScM^ZicwbY+>;4bD)h
zb>6{j(pyS}ZM<j9%@;Xgv-ZZN16K`hGG1*Ao>&qZEw#Sfd-<o(HDA*O5_dgJKeW!~
zlY0p7gB?qS&x-L)Iknm+DVfXZ;Fc5X%{5-Hxwy7M$Rew(Ev$%d(&`^E+m<;=w;%m;
zYU($a9qWrGgg$TC`DQ{&cS}dld`qF1(yq62i}wYUYdu)9M?#i+eNnQ6jgIRq2gNMk
zL#aECZr)tCskr2uMwdyMQ{5(m4<|Jr&UrL9^oHMqy?%F|h&IM=U!LvD^W9}b#)U7u
z2F)(LPoM3SveA^BnznF(-|}NGr|Ru&R%$)+V)ZKa;BBks6(;nYeYJF&u;0D1doLKR
zr;BLZI_Ys}@j{l4Evecq%naOmic5lu`<^M7x_s|biZ^(-*sUPl`0Psd6zekwx^Gk*
zWZqTiw>UfHBbPYG5t~qsY2PLW1*@;mR!^Jq_uT<aACFb7i3>KdMK8K!cwMk1U}fLa
zja`#A2u|ld<6$kfsVl;9zF_w3lTVGRvww6>?bvMhiDCKQ6UravD=nD0^44Ttt%K|f
zmo1)>czC^6TAjFhh(p1A^WC+%-_FWwv`-9L&av(7Murl}^C#ZUT{0nAK`eX1i*>Vk
z#h*+L%FMX$CO3Qerk{IV7Fn-5`S8^DUW2cKb8Q|+PMg-ftMItn!{`d_Czmfgdv|_?
z_08}_hhslITjRI+|3+EG)T>ic)DK9O9SmDOW5M5R(k~Qln<U4n?QQj$7JSk!`_ez*
zSqrm%F5ALn;w<2C#O=A&pSx^&s-C_UhfZ&k3=1^kn6!Z_W{pg1y>;(R+5HX0$6Ain
zoHldcq&&O2bdBs~$>ZBgT3RoMzVnzI|N1=hU(G!ie%pPvY6zNq;hm_flJt^<!z$kd
z&y}SMw|+?7FFaAa_(o=!$i`Ij__iPFPrpBtzWS@AQJC@glRJK^^khFy-p(1qnmS2u
z%iIlrpPat3DB{!9)YTme<gUK|_DHRBnXj+h*8nfS&BwK(r`%XQYZrH;<`=Wzl~;Qd
zC%fzvbckM%y7A6Ap96`T4*e)-PE#_O%rnVH$7Iq?*3<_M)~!!Ziuo>Hbx^C`-e$}0
z&+Bit&2)deXp?W-XUk<G^PNs@QREF_OVPQa{3`HG!_D08XHzskNIMAxU5#))Bk@uq
znO7=0;e(%Iua0V9-ZovX8DY`|%ieT7+)%DGL%A~RyVur3$&2^gG&o~+hwtI;IaQMm
zrmqg$wTbzq*ZjB3SmyM0#<*&K<Ya8P>0`pWmyx-6hAz+NKn-#6D}8@9#?(Ko$?KC%
zwKr1axiaDGBK4}x_v9;|-+H8bvrBM-#>RAJ*8@g#)-64B-orWW?0cU0S0;VCnv)!o
z11Ad|{O6T&;A2JW+RK+;TBP3CxXtkEM4claLv(7+q+H*;divDu9$GiQzI${+BvE+j
zSMT{Ue1}q}@8o#Oa7W>FLqYsz%j;qprZVEIZ(KMVmQ_}3>0tIy=XBftwqOZmMb@Oy
zIw|qlXG|35TDK-`5}y1x)^Ce^#rca2o8GSOvQD#_mmt3Sqq1@3>&{o)2J>Z2%#Im&
zBs`ItsCAcVl2Q5fzY5-LKPE(MxyR@s-t|~4CF{S7X8SRNjOJHI;v?*$-Ig>vo_Za7
zY>~LmO{+5+MK`*+zvL9^&yF@)aF*{m2Lq?1WK@>qH2%o4=GL3xWjBnpp4~0`d$Mwp
zOzOr$)i=L$W!}fEu;mo`cDZb4(=Ne(Cv{GJ*ywI_e)GA+Bh5TM=jW7g7`RN54N<sJ
z!M%H0Q|&&@=1Fecwodjp?IKtgi%!wzJDB0GHfi^qu*IzhD@`tb+m&G8^KEC8P@_lQ
z`#)kIWG~ff+`6Kt(C9qtQj?W#uBcRoOH9i`Gua4+c}dE3i-c^WSuX9{RFXZ@FYM(i
zrU#-1{f=`L6@R<Cgoqnne>bDhV#2cAxG#HUSmVyFs9xoxcT7~%Vqfd+G}pD)p9j84
zydm+a*nfM%rp$7YBcduBCl_Ab@A%?~0q;^?^`JewUL3Em`TZ@^Z^@x5Sq(|&hYOot
zKM#~`D)Ez4-MFrQdH8xS*~&m!#`!NCH-7h@7T4Z1G4^L#i{cT^*NGJyVt;Jy?9o|q
zz<9~xUd6XJGtJXgr^)PiAry0Z{YuOEPEqftY|WS};{I-vkfQ{D`lZ84Z6#^Z!F~+q
z%4&D^Y1wM3`oFU~w79u%qp|1XDeHfI*>)nOUS)c+`#$Y1&8y$Iuiom_p7P9Gab=2D
zaT>GLQGu=(EuD?qy0w){emvvgeWzu<iQ&+?6&*&8^H!Fv7ZqdjalG<|t=b?h>R?`F
z$k`PFnU1zm3eDDE*4vbRV%3_&{L<*o^yKebGM9(7varP+D){PCBJsdsu98;8XVzIt
z^Oj!tlY1kGReY1#g@5Hi8Vb&OPvY(=<>Z|{zToDR%4y5HOMADbvB;a<GWs&fYFEm#
zZ7U}Q=!=zK>2OHn{aLoF;*M@=X1LaL>5FRp3!0Vt8rPqVmXcYX^!2ul_GFU@PqRbz
z?)YoG;Mj~tl__(0E}2)V99w+)S|Hc8U&_gua=vakc{_FbeIE4Q)ew7gnfc1(v(tP1
zmoAV9oV!@+qHp5~qk|tV+~2y8zx%zm)9f_?(&YiGHgV@_K8@F#7WTfxWV6T70yeKB
z2hIx5lJMcLbY*zVm?r*EFx6^d@ys~wG;7wxbEX&PK0I>s++~@6p~;I>xli!z4T)KI
zy7k|Zn&WQoj!cYs9r5ts&9{bX8}B_7D!FpH-T6a><u`U&w%dm@V{(j6tT~{t^}v>T
zt~)=o@}tZ$^WJPv-MoK>?=4qmfxkU}*bdZ|CVn|$=04M4m+DcbOVu}Su>MaDSlM{`
zUgh74vn;zV&A-vZWc$-qJho7Bt6r4B?<t+~!eUczHESK7JGE(1lZ^4f*1D|@4C+qv
zV~eIMDzZL(yEE>j-@K)<vsM0;|6gQdyx>Ea)AgU3A*b1|^~Y^`)j9d@p@kuW=VY|C
zTEiYxtEcY^uwJ{%KBE6l#L4$e9?zb&7Y5Baz5GTP?}znG>DkqCuJ02V+k;mpeR;Do
zQ%fT7Q1jJnL)Ai+S1HfLZm#TR*jin@vvd0I%c<X&pT0PM-7Sq!i6bYx-Q_L^y$LC^
z=qve<k+OXEFY8n3W?x^`J~-eJ*OzyNC9t<`Mb*u-e9N~4dZoB$t98v?TWCH*N^;G&
z`>IFcg5!j@9zC5D%_x3d`oP^@-_Q*{CwKyt8h0IK+J8-%+w&uf(Z!VoUMf>nx39Rx
z5wd9SDOHtFm7=A~E;)S9k-vDf$VDb#qb!dz>vZdjJ9brfiN23Kut;^)zTNxon+r#J
zPUhlZWm{fcJBKe>XK`o5<l-Gl>aQ&7wj8o?X%wn(|F7`(sMNDZe=JsidUyTI^d0pd
zCB$}5b!}l%mTXeL<$7Sz;adT}nVr(g9Hm?9#9wnC=T$d({QvF#*T4IgpV``ZK<ad|
zae)W#p=Rd8Cqhfy9)zvwOIiA6($|Ue17#gjr|#UuXUesd-7Z%7<cpo3%0#p`f85Gs
z)gG4A+&9Cg=?#mZ$d1*wFGs{(IVa8LGLz3$aoVmz)^$tQy<es$$r$=@#+-v*A<wSv
zSg~9FT-TjxFW-FKJNM`joqPM4t~;`(D)M|gu(Rtsvs^|4t506<Ye)8q>rsve#n&!p
zR4mM`zTM=!gZXamwu0G<XIJjr7J6s1l7XPi48}KenhHBE^Lcw8w9{HO*=LsFTjdwq
zHYHqpXv-!idi$nsF2m}rzH(daCs>QzJ>@>P!KnDxTEYKEe=|%xRCM*TcjF7YTaOK7
zVuRPlD&9GI@z={Pj<(D9=J0zpNpt&D=;elmzG?HkdHk2n1?kg8PHYQf`i?U15ae3v
zcFn4*QK0*CM)&0l-Nt1lJ(3~Sth4oVd~Vt~|Mh-wx{i0BkQ<Y$sg=)eiTR92A`h+*
z@Zw=VH>GUff6pqXhgS}*WW8TwncROhGksn2dTA^12whQ?R<CQttjVjGbo6$<yjSO@
z;;Lq?m%M$l^q<f_zOqg)SkgjslfLYER`vAl2e~}6nr)f>X%!~BW5bU&Zh18!f@SC8
zjGZ?{uO5=Ms?jNv;j!T``L{Zx@!j#n<cwe6GsQox`Dgp(?pft^7DfzwGrP)ya#KTB
zhl=tjuoZj?E$a&I+p^*bch~>2r{V^>Up;p`479jq_->i=r!3RB_{BwCi!O5=X+6Yw
z!pSRpx9WWUV?K(v_nA(zx??M}a@OjD>*nrgzAV03!dd%)*ujF^I~__-h%MEVHQ4<`
zOYh~LOY1lcN&;paIk53ZDBt1*dzezV#QjR8=V<-ByPEUvTvyGh3EZKzaevk<?A^1K
zu`ck<$vX@?wNe)D2$jq8yBV`2+RsN_x%}m!gO+z0qM}Wn2pqY^(Hi)4#h+t6M>^Fl
zpI8O#Klz~RaML%Fk8QjElu7-Mt}n{^)|g^pRLX5@C$Y1@{uGB<-Oc|?|LjkZ@n_e0
zpps!U_x{%ZH~X?K|Gi)+Yb>_5<VJymrNJ7H@NNGc?p(O&_LS#mgwldywz43VRTtKr
z7Ps;i{C@md5>I|n>b`tCqkwO{@)z&0cW!)mQ-8K<YxT>P&sJK+C3P&XcfGf%IWBhZ
z-HzfVj+F`a)>>1tKbW2F_1$+Nc59W;myc`i-VoJ%dduZn|K~k2mPcdeG1_lBEMXVB
ztl~=4(h1>t{XsA8M+ltv%hO{y{Y`&<qyHjrYh}~AH8va%%fAb--a9On)Y2qm(U!NF
zWzV8CmmMdLPM&&S>br^9Q)S7Cg435L?{0l_aMByI(tvw)?tO8oQ>LdBc|?3ym0V)^
zGq-Kg#6L!tjelBxFcJGb-?U-zdAH4NI&WC?znO|!OM7uyZEj18;r07+;BvkG{Rf=W
zDkgZoTE=~yN6Yxoa}{C7kJ8_6J@Bo!xP0$K#h;1I1vYzIEQ5b+Jh$NNQ}ZY7oOf(4
zyUw(`zvOB9_4Sel+2R(KYcIc)dGpGms><^Bdk*Eab+5mDnJJmGPscJ$ptWl1Y2M&p
z77E4F=byZudR(H&K-^#Zcv9ZH8m{%fzH}X3^!>*@;f?A-eIoff)-T=o(-ut>$!>|;
zwEXS_T{WGFZH<iFd(B#WpRC+oqo*=6a^mjPEB;MZ|8BRMvMiq3?Ds3{Pt2dq|KII1
zf4_8w^6liXuAfPTd0v+PXWQMM5cA_y*Iw4&{HO1I+!HIe+$OgC&X@Hu_k&92O?(#<
zkX&{<x<F#KpzLz(H!tRGe=$|t(`DC>kkl_hHdQN+-hQ`Z>O(t*dSAA<Z0!pS(;AC4
zGp;RU{q<<l*52+NyG!L+<-AvGd2lXX@gUGI&wtk3a6hN-CR*Yzg$_^M@@@O%LhBCm
zb1!f2WJ_k-_H&a$Y=u{$z3v)`=T}!Bx)S$iskM~7|GRI!IpR4Dyt(y<r`*ujvC#4L
zWxun?|CYpy6FW*z3P^I!-h8yAX=ULf6W=`dS$ES_ch4^@;|(&?_;)x_s9j)=fkx)U
zm)3LV8ZEl=Emfhy#w{qBgY(wo_4<>$??2%wzq5_$fR<?KsijX&F-*`}^@qXhlj~w3
zp++Vt*XcKz3UwV+e_q<T-YvCjnaWzW9g`LtHHtYuDM-p`c@}eVmG;BGoeK>!4=!Z!
zcUpPZR{z9T54kN?wzKE3pKrdY{y^kYnF;HqM7N`C;-44}Z!h|DK)kO*&1b?5Db|yr
zFQtCRKlnG%a@NV+8dn<wu5wy04Y}L+G;Kw~qPzq9L->L(K0hcw``l~YQyt&!yY3nb
zw_TW$t-`)YImPPC;;Sb^4iswj_ZTeOebu+)l%$SZI_uW<otJJnS?Ou5nfxL4s)^yE
zFR7MUd|!Cx?2_0c^?8TOv}Umm)soaQ&x_KjL7lcI-RJDM{Z*Wsn`NPb_bk(zMcGm&
z^4<3<u1`sORrUXx&1K1A;l7pwc`xs`n#TkdCq1{AZ0viuG$}A-*|rm{0<VrOu`a3)
zYTI~xo%+|Sn`D<4r7wF_{bu7!1J3Artwa%)mRuLDS4U2#W@<d1b>D6A2`ytwn-fv8
z+g~h>ZD3&jJN=o0ap`*H!_y|8RDaAK-Fszz+LZfq|MFgUe7|+?lx<~y6Z-c!H5(@c
zh4=n%o;&Bxu0A#Hi_t=rD*steomYxaeYhpqtvI(ab{+F`A7$C%S22N$7CA3*nmms=
zC45nq@=Z=Z(-6O7f{SndI3{a+bV`@lsp%bSeYHPv1#th@`u6d{^EGq6t}M1{bo+Zd
zc!9};gne9+y{muxKea&g#=n314=nY*dj8F{_6~n|<4)z1t`KRKbN`-O?Tf7tnZ~=i
zO)PoFKM&&!uI;*J$p<V=PVJXcin)D1HetE-Q57rEUm`*icLnc#lQXNPpW)7agMYHE
z2hW}J@eA?37}U9KmQ3d%n@yryWDHL>@U7O{RS+_DbH%ru>EbsHIiI9Wb$IchOZBcG
z|Aji<{i(+$S8F_E@;ajW{lL2YZ~hz=o&1`8xs6}Orzb`~QjUa8kotLVTA5|^qBk!i
z^Mn_toKKy*@JGh|=q)@VO%f^2qQ}i-Q@-<`WKM}sP`M>M`_ldCFBz4*>MMTMILV&R
z2%5pvc9D6PS-A9VOTjPPfv>y0HYKpSY}^!G_(fNy^>^!K!PPE{1q-%L-OXgEb70N%
z*v^|(amHsXCZz|?+ncW9#=czjOH9RmChIWqtzKep`xuVyj5)V%LXzgvEX}(m{(H)c
zGq3CITzDa<^Y6d))|av!;tzzMQ(?TTms-#HE#$cp-yPoxHSeTXEMr&gTKa}{g7XyZ
zS7jlxWm0E$gm2K3u_<4)Sa`o{y}APf;|cLg#{}G^y=rAoO*k08?b{~FpJ~1Sb}g6Z
zyT$j$DB(Z`r>y?*GWMPf557!=UiBL1LT;H`J6-w&csh7qt+-L@SXV0J5V6r~3;&9@
zU$<1t)LZ<^fBoQm!=0ROESlAq?Q+$bEp5A;zx7zS{8DHRb5+<aIpI~{TxRuD(Ko%e
z@$LKNR37@?p7$*|e@XnDm)lahmuA?CZPex!xO<TK?e@ux9Wx(zt<u<3u+Yf8u3YQH
z7F)x)d*rshXq$e6$MU(8Tc4bRrrzz6nCe|p_exu(U%1>}q>_-RlIZ?kbxO1S&+aW;
z+f&TzmR{;x*?dFn*_lPEKbFPRY>%Bh>vY6q?!`uXsw~7cY)+_0{#fxqc8+<?ll1RG
z<=@y|-12UGzhmPh>pubA-6xY1-m&FB$c|t8DJT5V>+1jW9u?_z$GLu7zJA>~jgR$$
zK6l#uAHTmUlWg-z&_nm3e|X^T)STy&a(>KyGU?qDKIxkW1E0prNp#It6gFo*n49;1
zlV#wgZ8-_I8`6&E7N0SdORZce#q3gf@>rwH@%9au3wN$!m|3iPM%W-RVYYWd(YlFx
z@_Wk5KdUaZvR%AeU-#XcqdpzA|DOHmsJfE#YhkMYr(*Tz!ndk-%JY`+X_XoIY<pO1
z`HRJzx9HjWM(L22M&s=E!h(Od6?SSX#D3~=xys5Ovm;zQFwSB5YersMnbYeSwm#4c
z=$pt_6nLqkc>kpxUK88r%}=<mB43z((llJQvujC}se;8Wz0YkA&-%%%VY*suu+~m?
zmbP(Hb4|3&{KxM9Hnj<EwCF#?`f6E>=jYBG!2_a4+;^rFo%^KtbK1G<etyfnFQy;A
zcxbVRdO?(o$eb0&f)CYS+7d105S{39=9P)sY;lIRmCZiaY*YPxSU+ojjT9=|D$9`O
z8hHD!+dK9-bKIW(Q(#T;PB3^<s?f83|DDCNlEe;b>pycfn|p4hf?7`C`X$Ws4#<?A
z_!T(OdFuL<$6G!~+KTK+*tx^;R*cp~@x_dnLKrS)?Qr1<+HYrdpyf#o?_a~6lh;KX
zKbqQog`e4avh@v_vrEdXrT=6<j;WJAW0dQlU|^Wbb7Pi#qJ2Y{-duredpdYt9@_J>
zQzRksOJ@l8!KGH)T&uMU)n_(L`jFk-CeN)}r)usy|ETYjFNYPlzc>_xhtKKqzrd?y
zu`PW|-J=(upL(nloV`Bm?B2sAU%p>uHod>#C)?V}hG)Xjb6!}N#9yd&)VHg0vz1#o
zaj#2e#d8<g+=Cq6I~LDNDGBXM5wT-Pd410Jy<XwF4<V}W)a1MSKiV<}M?Tx~gqdH}
z{gF+S?b6M5>yO+wboj`6wS3h)&&o@4X3V^lwAK3Qp2g231oG^C{}-LSu>Z~HBj*FJ
z-}CVGJ$y4K=C$?5<ZsjG1nt;Y(A9HR=V>&%k#4vDQzzSV6}t=_zqU^0zB_qCQS0mt
z;p;9i^9dNW-B`aYOJq(@#KgU!D;}z_PT%CyzHEu1NwX#QqNDaE(r0gPsEwVU{`_v*
z)q_?0>z?mo-?V*amhPs9i=Bg<GrW)4E-vJs(6jBF;K8#8-km+i_%r1DXYHl~tmoHt
zPuh1O_UbjwMQ{C?t-U)+TgvKpHSIFKm1Lq*Z5?{@A4ft&c<j;_Y0?%ZLT>why{Ma0
zo5cC>!T#Cx?+as=n_OnNJNfgX4{zG$<+$}++W2~fOGc6F_7}SOZ<u}yPBD6y{IT${
zR*m>k{e?f?Nq@A`n`3wIc*DsO;~k4R8yCK7ym#wLv&t{^*L8m)YqqNE?mpyUziama
z=ey#u&ke1t$}5e$jHbvria&OIS+ttJdv}U;nACJrr{_j|cf$G)?c2L1yRgSCa`yWt
z1-q*)J%1|HES&sv@}l#9^Yi2b=Ke`tV*35Lmzz6liuEgjZLdmAE?>8;iK$atyTCi`
z`<`;9|NQN(o24(Wo^pSyo-~*A+OWB&H~*j3o_}^{pxBikl^=7TCwzY*SbNWcQ)_zv
zDf=Bu+y1jWT{H7ehsKB6?_X0Z9P|nlCS3a3)^WT4?()ia5|M8+8Fr^R&*W(9n^*8{
zhozS2t55f@GGBS}^@(3pnX}1?|4Q{?@25Pmmki~4vwr{AWxIPX`R&qqld-s?(m46X
zF8Q^qPrs~NvoGye_<hf|^*axyz1y(yVrePYmi&mXCiWtYR&zbv_iH=%Fn`eA@$I9{
zt*iNA(o9+@(rwOCY|8z`8*i^<+i2@(=x(v>tUk{v_J%5np40W}`~rWkZ(=u?5xuFg
zD9z=kjq<C-C)=5W-Maic{?GcK$1^AS<H39S4I8!Y`QGnUdfjx&{X^i(%8VaA$pQ-!
zS6tnj>Tc|;{$g6r23=W~|0naNJ?Z|g|7F+rt)eHNZ%9f=Xw{$0H|L{A-pYB*^GaVn
zi1WQ#f3Qc2IpSd)&pV6k=j#u$av4dsar;bOJhA89$rp{2X52}g$F=><EYItDl19^#
z4E&}3&6|JDB6r5yD#e$XM-HExK3Cs5-i|Gtr^%H${o{ixjC-|m$|C|!?tL38_Cl}m
zqQFWCp9MV2t@+dU@8~(dPUO3cBFn>$qaTwF?VKR{=>(sPMbSj1qKS(<e%}y_S>VF*
zKs`it&kF4lw%c-Fj$15fU!2JMZSmi~TOv!3cjelAeDH8v^#c!|GX7ib;&%3YbCSfD
z{%z$ytR4QR`y1ab(FI3lfB&Dh!r{#4zlQ4mPs1iQX3u9%<uPU#5}AJT^0QAYa~+s&
zKj;0g$5FQ1vh?4x<d3I5?A~oA`ObKn&xA#C1vk${-MwGT%*C#LtuR5q)o$Wjv9lLB
zigz(wn!C*C_3C48Gx9B?j=Nk;_>}xIT-9vy_qU6#{k<eL{ndpZWe<cUR&;Kjb8C?;
z>&!dzo*2DNQMfCUbm^S0Z1Z1zdE4iXX|GJqO<ezNi$hW*+v1x!KOgosyxtj`;V5|e
z;(qG|Ow-Q4?}?tEKTWB>?a2!n9dEUiHGi*n94qp%J9PM^|5|-DpW1@N<!&XhW!@Xv
zPi^?DyFkbMxd>OOcHgwe+*<Cgo?)Jg{(34kN{g)N{JgGIe*5!_S4-`8amO2NQFIH~
z)*)uXpf2!==fF>%3we%7B6p69@V}3+zp8Jua0ctH9VM0w@{@P^1pTp)m0jSxtxA%y
z;?vJn+cz4mfBdZ4nBnTKqq)M998U;c<?8%o&n_b1Id95)qnWRp%T%_+yb)TV@RzSM
z;v>h8x?|CM#Etkun3=hx_nh`f=q?T^$f~(7ve2xcDTz7Dy7=wP?Z;Dh#J%}<x4*g4
zW|Op5?!=_+BK7f~6m@?bnN+wVh4o$ImZN4eClik@6nNMefBogUJ0gm~bq7DxmL*JL
zHTEeJl<6sAp22a=S3_xc>0{;HQdO6|<9?ZcK6dxJ=N!F5{7VG54;2;(O*iq&s=4{&
z(#ivKH*zmOu->djMEb^s+ZU(QJwD}mrMSrRp}V3*nv~Vkw=eyIHMZ~ge}U8X{)ag~
z-7Gd=`FCpH-@HS8)zTF*_phXFa4c82D4c)t^yR&Fa~eV?{*CF}9=||o`;tnN%YKKi
zD}KsnyPI97dr`mAJ<&N;>to-veX_qMe}AfV@zBgJ9SLF4X|rVbyhBu_{!aJX=_K&@
z!93e<v)krsB%3ab2zqEU?_y}@{@ML@=L4iq3U4^J`3(EC%~sQX9x2M;P&jO&u|%$M
z)mI6Peaz}XJ6wd-8?9_N%*iZMc_0%LcFEWH`mWvXoAld~KVIY5KU-e!^Y3PzNtyOS
zzq*T7a4R~9{yy{k;v84A!%yN?)rk2VOj$aI>DWc_WA(aq-x>1$@*FAOx~%w1$eblY
z;n^luzuXS*_uk06A-DD+W7~U++Jk58emOmlR8wDTtFdKMp6Fl3tH)23$NllszmeIa
z%3KoaEcw~>>&BlF;(qdy5ltJVwy%?7pQm55&}V;VRP@>@p;_gYzuIngd$?XN;W?GR
znXTr0$%~Uwaa*s2uU*^t_9^$@y3O&)-?CNKA4vSnHEVXC?zID*dN<c^e&IX0+Chuy
z-bGf{9RJ5%c8e`nO0cj8b1qNwJL~e(NI58XhlX8@h^*9CmMdLc%WpUq?l|;QkpH=N
z$o~_qz1(xP9{O}1%b6vtY4yCsUg&UXT}hFD`lGpy?ehyiJpQ0n(04IHZLzQ4%sUmo
z`X;x}jxpS+v}o=esmzR+{W~1aoQYMoP?P`sxuId@Lz&J0q&ae{v}fDioXzefrw|vu
zM}Pl@W0{r;nJW5|4xX6R-Lz@q-YXqvjyI{i3^HZgS?gM#@;90zJC)CE=Cm89FW&Op
zw!-S9mPqp<kF8GikE~AG^`$FJXz&X;`{PZTTj^%!>I?NOB{yW#iXS(8S@kEZ@R8R6
zb7gggoA<6H#IpanEY$p@;Z6I=*13gN_PaioJU@OgquHA8oXoz*54sMD=AE##Ed4b5
zccOk<<ezU5J)39TiTdXB{Ic|29hayb%5VKHOI2_5oVod#bfwC#+u|NKn}TK5KAHTQ
z|K}k)xo#W7+P}S*+g&;ncdTOMzQ&-e_eFMEty{P7=2o?&T#1W?H<Bb8%ltW&?SCA}
zC}RoS!O62WpxrivnLl6X)>}PoMU@w)eOC$FPkmT8ui@_ErzxkS>fYE*EskkTS+!=)
zPlK|O^n%HG-@nFOl(@}c?e?j9hsgomzGdH?6pB^Prz>kr+k0_R`K`wx|CgHw+nP_U
z{BUu4fIGjzCiWNm3Pj{<4{noPmtB_eb-};rmvirSXkA>EZEsaM?a+dEOy<X(xmRfX
zIeU>c_^xOOU%Hr5$kF_57Ccch`>tf(`5yS@ch-#~6+4*v3N9~vsJ8j$oq|cWyVh?I
z?!B}k@%gm`yPH{Fn`SB5ZHb(EWI-Xj?tYeN%d%TH!jq?aZS-DYut8Se&28<%Ew3KD
zxqPm*IauzStndE(>WWFXl>?Vcn6;g^JjK6Rng3#saa5bkx@7Hm_a(Ns)eKHwWZ?bZ
z$|EYfx!*2ujv+IXkEVCZ&#6*ZXI!i1ce>ke7nr|bmj<u;%Zt)0TwM1_>}7vZ_@(u*
z`HGLW;z4UyGR=OcFCnS4W*z_9b-z~s&@aic>;J6z@_OeT&Kd1li%Pl5R~|cN!j)Ng
z(%!hDsrT9EtgXfupD%s)d8VZ!!y>7OBMBchPaJFvme?jx-gWQeoR1nY@tr5q_-?J^
zEpB-bs(63*1+mQi8w_L0H4Z&?UETFYVLH3dJF|^HCX2cKmQ@mv`TSJwa=L!}8%Oi=
zmA`)L-ds|clKx}aT-OtZuRblBvZMOrlJ@?y%%bwl%glbe#dV})I3z4<Nsg{jkGFVS
z9lvAK|8E!be@xu#XZhem*K(PfPlwL^G3z`h@6&s1+kU>e@^cd}Kint#dDCsbH|H1g
zE9n+Jcpm(I>5KTX2#1BaHOKv%gio^aw|%rfSbSRKbNa5FckM?f{P||^T<6b=b#Zg$
zxxdfwN%|Nbzb|=<)Pqw;UFNvYv)TJYf5P>k-tNEArng`0TQ_^6f4%(-PF~A5Y*!cp
zd8}&|i!uLTTJK`SG$HQ$zwOU6^$UMomfe2-oY@&att%IePx?Ff{$;Vabo%m2g_-l`
zo_*<6{C)PMJ1h1~>AG{O<xW52bgtvA;=xl);|@u;Y%RVZ@<W?dKm6uAwlD*!#*|mz
zN^~yu#>lyu8Qf@S6yC-ESyI;JJM&^*y$Xq*gREAX3ZiQmRfH@5u)lx(to5$y*=vqd
zn?;tq+wcDQ;;R!J@0GNbo6i}XPPzTVTwi6c$g23tuV1e&oSjjtx#!9w<@)sR^VE+7
z^<IrQx$0b7W?=gsq5P?>3X4^w&lo!FzpG``e#&spFHx@zU+m^T2wC~&jfR!SmCIkR
zu3h-{-N*Rj?b3fAHNDB{PK#B0!%$eA&D`GH@jmb|bM?i$-u)kz6t2tM!~JC1#g0}b
z<2Pluf85&pD)qP78vE~uz5K2}bN%?e`9Z~I-rEIJLf>3?vaIYt)0A|Dy~npmE~t9)
z&boAd#+2D6kMiRs?ygDadGP$;TbGN=r5A2)%h0&#aL89|&6;{f!4Kw(Re!ILnSF8N
z@9MsaX_IWdyN(tpty5voE|Iaj7<Zl_qTF0o#{T3jr>%nVbKBz<?+w|+tz|pYAW!ju
z)0LeQ)<0cz(m(xF=p`<${$<h^U3Omz&)dJ@=j}t=_szT5w*Sema~YF^<%%{ogi6+`
z@%1-spSbyn@<W~v?T=XPKPlBXrhGS@e<MjDX{t)}o8v!O0!}{&Z*PlP{y(p8(!suf
zyAF>Z=J)XIx+faF>S(N3y{xrdJaZ%GGl3hK0tYQ8={<~^zoq!I8iUx*hi|7`J*Y8@
z?QPwS<l8^Rrq(ZfD&gLg{B`5c8?r_^&sHB5f7KY<dd0w9>fgV=0(<AT{@y&(*{u1l
z3G-DK@1;+iCT;uKTJqj>@$GAU$wz;>{}%Lrdq8QWPV838H*?M}-aA9eOZJ3F+UqL2
z?kU$epQe<YYv%i993^bqH*LvQd;a~~WINNlHvU|0>moCq{lyuU?h|1f`sPJ^Z@p{&
zqOQp<nc;qBMZ-hx_V+e54?nbC^v^!Rvn<`f_P`gOW$d%$5~><rGN$ots5$hB^R&8-
z-v#STzmz<`lsq@JyjQm_|2_I)B4bBh@P@M&nU_3E{?2ex{MlLe_>8?idd<!m*5_{O
zu+2)iwKrR-&TvmE+wYF;6Zce{e579)xcZ>?xAL{Fi<uT%DqYMGGs|W>%hlL-^ZNRS
z`mc<4g%tPfpWat>^T67-x?4GRd-`Y2NHP)hQMk|TaJ`myOWC`Uc&2EFbBmu%o0S*7
zB>84djNPx_)BbFqKRaW-j#r=syIN+!@<;m2H*eIPa({AOJX`3f+|fs|K{a(x^PkL~
zy3_3N^am3>H%Gg=v&*er_%Y|Q%YzP&*?c$5+XH4ftDT)Vv4BI^AtO&{%ZJx`I=8nS
zkX8x15HLkLG|%BnntXTvSI%Yc`#R?QerlimCk>3IoptAr*u<_Ow=qQCcER7|ofQWj
zF?O@Bna#dJZevx~Rpv1M2-}3O4Of}3NitU)d8TQ`f5Rr>bHiC?GqxpMPaDG~e$BtQ
zKlJ~kq|;pwF3*kGFsC}(e!VicOjoD~-?>X~_f51t%rjX+&A5kIWM-n#;gq(@lw+OS
z4;z3u5`61Gr+2z1&N*b!ng~8040Lo;mG7FY&|@p(#M)N!1WPV6^7(oz1bhf4<RGx8
zsWYA(1)Yhx+z@<>Ao#2%eZTM3N#Ij4cNJDhA|HbJqt>`*x3T28@~T3$$3DlPM}wuz
z$lUeuk<YQtPwJo(nk3RdXM#P291q6IIE%@A@`E&~J3cM5!(W^cdv7;Oz&!ZTGe-VN
z-x}0jxpQse+r@XtEOoviKYzw8Kg(@51nWPG_&zu(GSBDXNtJm%k59U+yT|`#eS__n
z2GwtK*-cieU#`;H_}%B-$uIMKKAaS(mlBuTaLljI)#Rr7<yxbM>cMqJPt_;a8NFRD
z)}8UmUw1|``xicsdyFTe6)O3c$aTNSov??0lDX3-KM&g*z3gA&Kxc#9R;;vF$?td~
zJ8_Tlr0p)B=6O6$NRTt!HLWsP<$ceSxfA!8PWtXPYh%Yfxs$(@&n9%rU9eWE>{fX{
zaZ8fmUf)UIJwLg5-qSp3t-3a0)4|le9XDsILeA$p3pt+)e0Y}^^JQ^OiS6uP(jX^!
zRj+?CfeEbgi2#ctCtu(LMrR3`OA1F0Dz1HO4>|x0a<<pX3E;E6Am=$%gU=b8DsxE@
zY(b23T&2VNsc%5%fE9UkzBxMyu2vg-CRu>v-vjs989Rhttg3J1bu3S`_3*2dxOA%`
zw@kdCwo59Dk7wep{3jFE7RP|jnfs;#BGV*PzSXqJJ)I6pUOm&JB$gTQ2vsX>REHDK
zFI(Kbnw_NXkt!4377k5cJ;LP^ZU}eUPDCYik@Ue#h)VKT`R0O3Y&kN|<4q@w$YK8A
zuksBlW>$2^CF%SNQ9+BtadS>uyw<sU{_}x(pu^`HBOk*O8(Y!>&jMrKyo<8uwOfwN
zn|9-}P<2$2zxSKWPFvlL%T>SeK<O<<=1oswfuy*IzSu{avkpwww3&HmvZu|=W0O1Y
z2vytp$0YDaB;Bucm|SJFcDYdX`6u&y-m@F*)%;iTyYc`tucFw+l=(j2Pqx%ATT|&k
zo%D9`w-RQ0)2v!~hRpPa830?3%u@sf#Fit+PCb`cbbz_dz+#DGNAuxze3`QUp$W-Z
z3)`u4B_}|KiK$+n5H5ECmIi~Az-bVY=)QY_6Zvd8KcQ+{M=`ZE5nuHUv-j|1=&fl}
z`R0_Qukx+RZU12h$A=F#lz;D)so1353M$9WHzms4JHyj_B<}o&iiqA%yFrH;wkGyW
z?l}_Y@#brZ$URL5NSU@WZt^LMsf9gngFs~&=#*c-jjkYb!G<G<O-JJVf6tTx9qP*i
zJ|p$}qmupV#~=r$o=KQ-(BN>lu|&GTxr7-Hzn?W|PHemXqwmDy89OGj@Tx6ve$F7`
zyhtn0|ET_?m|X?>>>Ty}))^PAo~#dDm7y+LmnUDTA{?#}G4sK1>30g9uURHYEwPxm
zclZ8ZfB*a4e)8RI^6Qn;_ZOcuI*@jIDbwD@*2q6U=g&=TNUpan{(h!zf19wV?!nI8
z{R_gsmn?TWnDoyo`h@<IaM9Y`;g7U`_+^RvJ*Z{#T|Q}&Yt)IpIk&zC7sh_y&iUt{
z{eH{Bc-=i^e?PCRKAiV;WnJFC`^nY+{_Ouh_rw1qoVQH5xZcc+`^D_jQ1{bDZ~pVA
zpOa(%dEEZmU-DP^cmKL6vtR4Ks#?G9f+z2fkQY~zFLkDMcs;oLU(=sScy{T{Si9U?
z3#(7P>+99nTy*((j7^V4v)LiN)qi69H=on}ID6_PlWn!i?-L6RW`;T({qObb{oXCj
z`KcR&>LVXJ|Gr#USHD;yOMkW<=TZlO>h|?U>+i3B{9Cs6n1~p6aB+<Qr_IW()}0IE
zPOgu8xjv4wfWNG`GFEQG-`~@|)vNz0_Opn4z4)!k77?+`Nw2L-O%6Vs^8C;GGp@c@
zO~pA&PF7uf5&Y<C`q6gx?#QhVCVHA~s}b|tcK*$+Gg2noiq@aEpA(keyeet0vFz7s
zfyYZGCSNaGxH~%ixXQ2i&-*u?U$T&~_r{{*?XiMa*OZ4|nUns<u-Wq5o$5@b95<bd
zGwQ8Q?XNpjc=>qYb;0+pwX%_l-)HUrFiUb#_dTT;tN%Z~o+*=lbL#Rl-xYC7U+oDv
zGtcJa|1g0wr`F%MIk*0s=DYv8oMA8Da3Ay%{&}41;jSdlur()cy<dG_-kSI0{=1LG
z=70K_x!?8ik*b=Teg1|0{}0Xnt3PMMZvk@~=HJnGVmtTD_kHIwe@frwwZ9{ld$oUS
zjw=bS58C^;bn!G5TMy4AR~+Ux2VE|S+n#rQp@{6R>#@Sl#R`XJEKxdn^HA&dn@?u=
zPF(+K|MojSul(nG%cE+Lw)pwq_h+8{?-CAr82_nF-g8%mkzVqqPLFusmEEtnoa_`{
zo{0PzeCgXX!Gfh9-tuv$>c125IKAz!?H8%=H~EX)uYIuNRGpW$$-~^p&GV13x9~b6
zu7o4iOIf!4G|@WkmK1dS|DOFX)})mOF}+N6|4>mp(>QgrNsD4{(cxaD3}xM;?_vri
zFF*4t3R<zPq1jjH!>Rv%Av-6}j_`E5J*Cj)+%3C)q1MpfCR5a|d!~I7nVPw#GbBq?
z^?mu9SE|dm-Mb#}Z2q-FGc$z#Fy<xPe7DYi9gm<~`F9bc{Qj3EqE>U)*m)j$oGGZ%
zo$0zL_Vn&2Y~c$Xrm4s5PZxigUzYDtBrl;qX-TNT7onxU`6pXRKB;z^^(xrP&Bxfj
zKWMW1B#ouoZ8p!M9|Wp+`yAM$V$f|<+Ha!VHPbVG$@w?)MA{GNh}3&0xGkG2we#)f
zm1jc4)XwuwOH%d>ePlH2g#9GfWlD9N@**eYJ%fTv6Pk;Z3o}BGde-=53P(Pg9OZH9
z*p?H!4+l7Hu@^RAv0l_!b5&@ur&!Uw1-AQ7RD8}{{EkOxQhr$Z`?@=CJ;f@fDF{h)
ze>(p4&;E+agZcV%*F4^xzI4~0?-#tI*n;kT=y>0k{BA;{!>Jih-gV82jlMnK>*!<|
zt(zAmbq_u>NNt+a5x9|Mx3Pk9<D$PsZrPq?4%;f$rmSQvjA1{zXxfRgFZnL!Z!S&U
zrS~Q|_u$;w6E$}{%u`=-{@SHu&kCg{*3SNR>cZvK9#3)-ncn}-36#+{Z&LD&IGA^S
zkxJgC`J$o?`ciTx;T6V>OSnA4!gD+IxtvtPIS#!POzPgUM(xT5#{9Kxi|THF+Megu
z@z(3h^KI7c!oezAcVEzT(-U>w8dV`2Asc+i?biKmt0$cD5%-Q^cUD+hb$-d6|K?ZT
zI^PRx?JSF7crP-?I>=S}?$R>FOJ8?CSY*;7rSLxHDW}2txKmrI&#v~m^5V_L_nZFa
zUDp1glC?|ys2%g|HFh_1v{IFqtlX5c_r>z))YD(%BN)HNELZy6sh{=S)8D=5+wGf|
z|G8A9ZB4nRd$mm1L__t<`?z1r)F-Ii>}=kUFZ%A$zvIP^7N2Wezq2A~LWRhRK>dAx
z_h~IX>bfqneAefa2l?XKeq4;(x?%a<clS%SzPq*m|5ldWBDGyRPCwc7YSZJt2kYNn
z-1SiQ{DRH19>3pyJ6WqF?}Ae^=g#>7Aw>-LZ@Oy4shj3KOa8i6ZR?re&zN~9Zh6yZ
zIDa0a#<dEoO)<0cR_tA7<gRqadPDBzUGGFylRi}}aH$DCDc9>7df#1DwR>q<m)d+k
z@B3fpFE7X}Kj-uEj{fFI^V}|mxA8MQcFo`1@MVGSmh08!dr$bTI;H3FZRzAh-pul;
zneW5PU#<Jby6)7gX=-e8?~J1w_@<RID>Hf8?8>Y-&v;*aU$F1i_cJ{mMeqI!$y%DV
z+qd)4q+Q03B(f}0R+>LH+x*kFA#rJbYYdZr{FaKos3W$wtCzPbc^J%B-oHL>VYa`h
zrJ>i4rj$7gG{3F7_mV;0mUrj2_s`~PShXg^{a<?bh16SBBi}E9H+KAFt4eHY_-NVv
zRA+BoLPvR)K-u~i>B_8pJPFf#-?wsACul7QKC)p^{fB=7j1w7^@2pfaWX!bU)Y@d{
zvLH90j^A5||I`6R`;h4E`&T^7QvGnvL*=5MVAwGe?#Z80<5FH21|}x#P<(&1YH6<R
zl5G2x&(D0E7HTKnSG;jy{1Uh5y9X<xOYdmAZ_n;>m(HD>;I`~jyS0$|^@jAxSBpNX
zeO<6*!mR8>pA~_}3l@GBc1_#AKl;YXTPHNUrH`?cW`=O)+`9Ny(zSEa&ABeYAGN}(
z-&QAEbtr7ud~46Clf6qSc59`GEM`(*P;RPLQD#<ky>+x~kGaO-TT5#<Zd<vmJtRgY
zWz%V=Da=7Rwkln9G1rv0$8Z(i+|E()i?@G8%LJyd4YHSv4}5fKI)6a&#m2jHUM$L7
z(8Q_oTrzUUtOLec?njO^7_mH#c^Dp)7Gu05O(*TvFXuHnt6yp^==(7DOubd^u9l#S
z64L{N_T{cvpU$Am!y+6tD=Y8H<>^IpJnWZ0h@0H^?bIbp&mErP*-8h`rg*<lv)VuF
zY?7x;+4YvCeLb=hWtR9Ee~V4n_p-lM^yvFlBJR4Zg{q5=Y`PTkxA;b5QmIil_cv#w
z8R1P_^YhracW1xT(Cur#8Mi@U3A@f4^DiA~cQ`#SwXiDIE|5OiQ}lrAbYOYEt#Y#U
zw0*(T(|<fm{nDi_YL+qOO0U*d*BbwyR#&@rek+x~A*7Ld^l8k4EY+-C^F*bY_jw=U
z-mv+W6?d>w=KoWE7lSyqPt)?6EyC-~l#+5&kW<Jz&sKD+zt%yO;El;%FQk^nWe9ct
znV7cgt;lwdxQ5MIc{8F_bi26b@=Y>W5$l?IwK;2XZ2aqWC#$6HyIL17H{hC(!+N^G
zV^UC@S(~rkz9hf8=5yr>SVFQd_9piqwv2Y)+@-%vX)>c|^m_5-4YJBArl+25;P5b-
zcZ8XhtNZK%KCYnP(&RU677K)C-if_*G$zz4Kj}7~4_oHy3t37Z1Fs3^T#!qD&X>6_
z@?_3#9#O9|e%|~qJE!=0?I;Ud?s;LA@x(XXz2VC@Ezaot$I@JRT;cBidn+CW>9o{L
zxtnuhk!jSX&6%x#7DorR<V=5*XQ{DbvA43kOWy{!Z&g8gC*H~2ILkIi>C+<1T9=iz
zoO{FB@@<xVxbWS67Vl2=)GOAXFE_udmU)sDekyO1%pRG}TdKtgjhEAc=Q+r;2wA4u
z25g?|v29Xa&{Rg3-Rtyf6|22PZt@@a{I1IBdUc?@;B^P}JM5FJs=wcfeIa;b)h>O;
zdd))>s;jP5e6C6HeGsr_*V{!QW#_Ld+3Z>U$A4?<JzvhVj*@QsEn0IX9d%08%@m8j
zEp1_<YqZmB=Jt?~l6i(RTKN6r&gD7okg1t`+^_V7vq7n3Yx^HP!_7|Yt9_y-8||IP
zmi_AF`!<!g%L|^BD>qe7%WDeU;mg0`>%H&xZJKsh=iapt_4zKY)HN^XWXp+f-S5m;
zvX<4%l87`~RrXQ!`y=6&OLv=2_}A=bj{YO=_@?ChA7%AVh0o-aA72(IeeLS+o4GwA
zCHuxv3sJ?%#o;F(KIznSlzD$bBr&y-?afF1HPVdl)7PFAiS@k___bxL-zN8cdyb2~
zI;G~eG$mB<!zEc`?mdrNQg+X~de&UI+d)SxRK%+6)o#5zuHV*GEqoQ2`Etc_##Kp|
zUwI2oYg}l4FCf-juHx}`-|gk4Z?E}SA8NSJwyI&?d&_NJ-X-p5aw3kpWm!!zbv*g0
z$$!(8uTu_Pyv+3_y&*PzQ*`&Mvqypq-W@r<<dm<C(}e#G6IUGH{)zRw=a$Sfwi~Ck
zI4+&@EuCR`_hL=iGV|;sS9Ur4o%&Gk{PQCPDxddEe#I9Q$9_sVIlJ^r_|J;oEVcBq
z!yA7daj9WQReoY2zHZ@RO<}t&hK5@irdF!_kaBC-F{|W}L$9!Bh_Xe(hK4wH(MQLp
z`|*EPIsfA2(a8(#Yn3-zxUvQHp4E?Ou4Q6QkTbMMWy_ng{Ir<+#_Iu$q1qQ4-<-|h
z*qwI!jr;O7w(c|B*Z7@rsQe~0k)Jc|6VKG#_cpGncanwW=F3e=zNM4$;BkQP{>!IM
z1?Hwpto*T~ZA$O|BgyM~Y6A9&S6$k-x2=aerEL1`<2C+OZHqk>7IuGryRTUN_3eY2
zQ<;{Q?7E;=qVx3K?GMs>N*fn-vX@D3NorbR-@z&Nqu8i?c0}}@#Bhd$!j2O!%jc$O
zofoNB-Sj7L_tY2tcY@dbFHU;ypI^C#H)VB6W8%LRN9uFf71Jy%gMa?_kF(D?$J(Yc
zUrJ;`7DM3u1Icr4UVmkLcaQJ%)yy1kQv^6S6^Dq*eAu-j%whey1^TDnbZFTh2|WJr
zzm6HtmPfsF@@C8}n9t%l_o3X)k1VB8>qJBr9G6JHcitzpv3a7T-|^pG>>E?RY|*N}
z>%5dL=rY$wuFOs@50m%4$F-ISe9;iE`J|AMBdZ)`<0n;mkMCaK%ey^Z`khl%IaAVK
zTx4D7zvYkTDMhQ<mkt<LUbcyIOO00L+P=x7I^{y*p)AdXp(3xumdK@i-gb5?_xV#h
zf5%QgtC5{uu|KsZam|ZaAGb{FiDdjgcb)5%e-e5gQ|x9e(h5{F+}0XnJ@vZY^OA#m
zPCPtQx#AV$&s)9v3p9L<PB3t+=+)www6j;3KmEq`{KINDmb4tcFLo~8(Bh)Z&ZlAZ
z+zlo_IZkw(-}>;w^>u%_<>&Os<vvrIa%A6gO*_u-p{yDz;_}r-jAxRcPkS_%L2^Nl
zP50|_A)yan=oc(vxUZ>piZ$zoLgmHIf?s`23u~Cl)qhM>mka1vY16sDz9og*YEItn
ze9p>#j|(rjx=t*dcSSx${-S8|`9~GEgE{I&_rK{A|87$h=N(ZL`Q7AI_PzX74wcni
zT<nc<^F-I5<r7q&mD6>qVd~Z=Q?#qJFW>wVcr$RvMuyjqj(G-h*59*!DZ=<&jO$?6
zycr2!>s~9#J6~1G{`#$|zlk^C<Q{*gFpWubCp}kvB;fUY(#Z$qS^?IzQAy`h#rd{K
z8oti9k$Jai-Hm|4Uv;<huYb^ZdzXu2+I9`r8}pX*zpQ<0;k-LB{mW{mxcNC<P0y1x
zBm=%x-SX{ns{XL5FT-`sy3;FeXhz;_Sd#eWlgm@~6MgGar)q6zuRfyv#i!Br=67qo
z%2>JB{+-e0&KZ1{8dk6J?K+&^Y`WBIiogk>%ep(x?shsB^)3Fp^l#nXxNXW8SM7Wd
zAbn6N_lx*ssVc9J=T4q_bnxD4&(rIEg@4Vx9j*PIxAyCt-xrKzR`A?BEM9eP;iCyF
zx9<!Qu>NW%yHnfaz`Ca3yIWSg2#J^&+qzswY;Lb_^KSOUqleQ@u6o4P_tgEYku;Ze
zt-e?4!tZjKH5U#s9p5prSUuMMXW|S0_3bV>m!GZ?lDkoISZ}q9>FNSjpE}#u##Xl`
zsw{8|yV3v3tKUoAuY^IPOZ;PPP0+W<8qU9E%LGJpZrh!Edwgf}QNul@Dt7<$cwbt4
zop{Oc<R01ilQeY?ZsS`1tY}U^zyI4GTl`Ng>`PpB%Q?sFdH#XTU5;-PqulLeYO*a^
zJ?3-=JGylTpB7#b`>)6@<lxGs_n!VW%F?>1E1tYz!t^x?Zj94TI}}+jFOxR+<a&Jf
zWXh@*1FtD(SJzsEEZ9HmTZfylJ+p6QV2{)H+h3PdAGVq$Eh74taYxv^w!%<}#Sg@e
z{CxSzXr=(`^3>yd)pf4z6FjD2c3Nu5#fLYpMNg`{U6Gb<f3Yld?ylIiir3Zt_*#@l
z2)6b;ocW{GcWTe!tq-C~(ktfPyb<jBZqg%$wz`)QLPxK83GJ7bUiFtvF!=5c|Am72
z95=VFVAfkzb|-%GjJV6IgFh@<5U*mmSMh9+PW~hV)tM`mJX>{rHf-2)w|T1I!r!-N
zu;_kBx_s2-^*jOI&9f|h9(=vf!M0*@N88*}B6||JMYgP3&Mf$-Zq?a4DS<EcJ`^(G
zUUNav$H1~b#3!Klr1-l)?F7Cgku}1~znpy+i1jbH?3*()SBZVOQiMe06{XqF-*uVX
z5e#*`=jJ4Rs`lmr=P94oR&j2*zb!yz$?LbmkwrJPCMmAX6gsXjNxsatjUjPTUhrZC
zw}U)KKI){$AAkLw<(21~!o)3FO^UWMU$@;*oqyS4|Ge`8=Oou3JlD8EUu7~+M%a-%
zoG}6(v7HB8*GydEkfYCPfAS;qW|yS(K1||4W{VQ+Tuh^8{E>Ij6S&Qyx28z6$Moel
z7QG$S?YwWU*mlGyFmc38IkC|3a)4Klsd;r}@y1<IE6pyj?*1ED`P!mv$H$59Qd&cM
z7|k@)M04y8e>2#A<f}%}f@I#j#iw$G7idhp_-6r+=EEe>!-sEj>@U{Zx?}DP*_B62
z54Ws8;v>EAU#wf((VZ@h{0B4iBaeDb3Mn>@6z5WU`)snszqpgXkCblAcJ|j?w)^&*
z&Mni$OE<UnS-*Q}vUBdb*M6Uh65ehu-?C%TXRUd6Yqnj@5i*!9lCtuGm*m30&371g
zy$xQ-b(LLPIAHopm&*O8C;w5~A`;1a^=O05s+j6eC&Pc8Y`g4Tx*^tOtx;)%)~n4C
znfKIVzRmusAr!VQG&D;1-|oNJcUwAc%1vM4Z?3QLJ62R&pJ|qGL&U{N?}GL2TvB}J
z<#eR(yqBl@S)aOKn<kER3c1?R+s*drc;&2`vir?Nj!Q|YnJ*51YF8_cZGGmtuw&Qh
zpDTZcRf}F>*?DUJwH2b%c3zL_x%im*`%Ulmoq<6NP4d6R<SRl~t-JP;%f?{eq|(wL
zo`2D`Ys-c2EnCr|KYydr-yQWeH@e$?m<Xl6-B#20{!?L8=lXA+o$cbH8-*3_mw$S2
zFJ$`rHW%Af(#LwbXK~GVwXpQ;+aHTh%;GRywrAn#%A@rfpE;Y|HgdfU2;A9ma$<+p
zwC(9@EMvAy3yZdGf7BWN>vF2?xtcW}j5Wgw?H;b0eo~0jRx2w~+{i|(qwA5k_v%Ft
z-rP!8Z>iTR{ZjCS$#WUoR*z|S0_UGfsQYv{f1}FHxv{T|3oe@9-8=cm9^NqSp8L6*
zR8%%))xKf$x|i4L;94uSF5;NGYK}JBw#%mvusUe}l-eWu;cF*v(UNVd8I#iHZM?YD
z!STITY3G`+B{f_|fA;Pziuq!(zNow9?s+whr>|Fg?K~QN$Ff>TS>>ffvV+8`BWG7z
zH>g!S3^Dn@sJDN?)X6)dRJC<f)7ejLa9~ZXf8(|Ckd$cr^62Og2IT~!tFl&C@3p;A
z)%@_NkE{LUMU{;k8{Qd*99A<_J+_l0D@5~Lp03{RxRr9E-$LgdEB?MZWQN-xBg_AB
zGQsm+?0d7o)3$1EsMa>oTN^G2_H6x<8U5@>IBT)?`ze=xJe<ecKj(qENUf-p**>pN
z?JQ~wpEJdl?f<KAK~QCO<-1>B?o68b;M^(s^k0o3Zj+B3jIQ?&I_}#4c~!}VCoHE~
zuRconnYCX;UFOPMzO>nHTr(%|sm<n7n|-Ws@nqKee>>+kfB$@@@Xt9D1JOTsOCsCL
zp4*h&w0UOOFMamaX;I#dBA%P&_sg5ldVW_wUas5VysO2J^*oaQ^Y(d3y4Nh1ExNL)
zbG6nPrQdtw%MEkB{>VP&$E+S05;dojqdYdTqvgm+H_iP9c_9ZUluD{eM`)#OI;L`m
zuWPMY(t}TT3TGXCu`X##*0Hbbq30}RyQc>0g!zd%{azDv#!xcjqf&Emm#5aPsS_s5
z_0gT@%DBEoA*f#QruEb|$JlRtZ;s0bu6ivog<(NLdc1@Ui+eAxyqAYqwd^+k&w|Y>
z&Rv~UaqL0hqKS&ltKufV5qZgeZ0eM=%3_NcxK=L^OZxqmb^ly%TQQz_w={3u%8xy<
zPB2(q)Z@(-fwN~>RhFsmyz}+tgk?K4MQ41y(X*3nX4;PzMh<-)Uo;l}{rU7-9iMsM
z!)G5eV_(>BzcA(7Tg~Vbjq79oHl36Tc{n3-HM_mgv!)=PkSi{2y_5D>gdShCM0eAZ
z*O$L0on3z9<3pEuw#Mr=I_<9emZP@i?sm_sXU=Vq+r3sMbl&f~bCSOA5ufv4d+Soe
z^&x*W4a+0HzP-t}-uac`Nj9cZ<#{(#T5rE!u|#pvY)=2k$dofHg*ob;*k*f9^1HWM
zz4Y)WS^Gf#ZvwuX&F#K_eq0e{Zahuk(No1Q`X2(!>|H$nZ&S58RwL6qEn@4Vo8d33
zdGjw{OIdDeu-U9=yT>{4ux$@hru4k*c<`=Xe@{ga)9Y_8CRgk4H~)F9E;G3@B+Q{|
zZib!U+VyLg4it-iYMgGLC4NvlD|#!Jr)mBaN5OmRt0wQ3e=&XQaXaU7&la~3Ipv@`
z;X9`;ueI5~#BH^}LA}iA%ly7^`>vT@Rh=yqwYq6>#<{AS+OhLA?*F}Ud%w%1NxDmC
zy_q2LBcbZeq_}?87iabJ{7c{86<N51dxqou`d@s%jY<<(@}}>1+x2ha`uZuiUBCVc
z-u>_5`ud*RuAhHhE*1K0)m}OG@VpmgKVMpPR?a<~v%P!W<QdmJ&%Bm-maBNJROq!;
zXX)I-yL6B2@=J<U-?*+gVtZ@Eb<Z`gRi>y`{(QYnWOr%h-;H<vt=hi-*55;QzYFjD
zo3(v^p7*3nxmKEY|Lr=fHuZPw_Lob_e!n)GZSqe1-m=Mg-IMb-pZtFG<o1_OZa;c*
zJLgkv;ip`ypW7^cn$50>pHnp3Z>O%`&a=zzh%djhEI6+_IB&D&_oJHIUrxFG=*sP!
zSG9$&a;?5@v-oN@yDEN8)oj1Lx_*1lF1shb{O+>LdEJ-uHfL@>`qN&c;-9>wqUn-T
z?_7SUpZ{F1{^z#;jaZA{cAF<&%A39Wu37o)?{8}!o7$Rv=M??C{><dA$#MUdI{vG#
zVVZi><>r!kcIHRhzHP04Bi7l#U$QKbZ_2&xk}m{LXgto$xna;C{&spp$dZ^Fe<!s+
z?3gE=(VJj*!hb=@{=Y(^))F5>e&%iZm^zos%Edx+(-e_%K_TZ-lM@y#R}aqTJ>eL6
z)Pd*Jb-f2H2URr=I{)Bka?PlcP;KBAWK)q)&3ik2#k&9p{YA^z1x43RZ!9&NaI@v+
zqBrJ_TWna)D@BO8E?_yLbc9uLJx8%?gf@TZ49zX7IiilMd=`hVU`Sw2extZXBykCc
zv}=dPDz13;pcxGQ2iSxUbvF2l=?CW*FV2d1AG|}=?H|)u?j!mIn;MQYNoda$S+S&5
zD#&4f4%;51bD|=N4uX@E8U;=U1U)!DLCvL0;HBei^Z&|R^Bf;uJgT7B*bt(mTwKjm
z`elLI$scZugg+TJ8hNo7EqyV+N=Bf|LCr`-(DVMHX(@Afr}H(xS$^BhtU+&CmO}`0
zgMg}8esqB4RomEq%ksK@-JLgYz4)DP+OjzYChOIUKgusF`8(Y&fbaCbl>M@<Pi43i
zo~z$*kM}Rh-E6DB>fHy{Qw_Z_zy2NEylM&C+<^P<oDMB~&ZWVa;BoxL=Do2;PTKyp
zdA@&3+mqG%`o9%j`tjGZzF_s&e2J&`%bQL$+V9=||B}t32cPF}lA9H5f4wE?Z}W}Y
zPe1RPkhv?y;Pcb=qMv{EG5+&6?!G_oz_fnZS6klIze@Svt9~GUS7oxf=Y-Ic8-Ggt
zzxer0;ir>F3o8WM*=6JkDh@nkbpOn7Seei5N=_TQtXxUeg_n-$-;Q2hEay>fZ~OSu
z$<x6Rb<3Vgu6ell{k|Q)oo9w=WqH}9$<N)ecjt`MY`ZdfcOHYkN3E8v%=+MWSIC`z
zmdy(7lmm_z#Q%Oj|HFU5(t38&OP=rF_cPh@U;b<WA^N#>{pF|k@xQq7w4`=MU;E1&
ztg}J%(SHV#*$n*_5`t24%N{05$)(LatYWi<X+{2g_c@2%`7LZ8d}=t&tjE7Y;sfJW
z$?^sK!M6;S@`4HB8w#1zjV!|B)c8wOrc^C>)3}NKl$?gUZ0xs=N}Ka`-tWW~E39JI
zk=sylK&U^!Cdologg?S2;bX&5(5fEEfR;mSzrt6%oL|s%=;qpee+w9VdxOL49p;|A
zQW^Nb{rmTF?FWzF7C$KJDeC+4yK>s+X?ya`g?=e`<S*uQezIMC>f7(?f0wehP5Jlb
zWzwSg_m1qhoBsb9XO7^fRhDA%tNYI$fA{!<Aa~cb`*#i4?po&G-N9{JzazPfDQ4U6
zcNzS$zUqsoPF-!<ZP+s<aPgHS_4w)MqaWViX?2#vTKD&kpArAge)-;S#d!AF?ql6v
zBCkHqvY%IThwtx;M?D^Qa~>Yr6fo_cyYoyVi@RL<ZH`$NPk3x&63DFH8T@PCPQmbV
z2c~RaT)9EshKpnJ{epA9XD$2nEurJy@AChB*Qf8%d3p5r<h?eQe{VjS_~+!L|Ne8d
z|IZT+i2S$cXzQ)N;c+MTzi-)7d!PA#hwR_m!kZrURXn}9{?5~{R<qWopYF`DoVfa_
z!?pz{{wm-9ZzXEB@t{V@m$}=|D(CO}`Ybws-`6z3tv}DrcD?e<{ic&kuAJe|$Vu!z
zat4(Lo-j^k_u={B;D5@@WAoIXIjMHhGtY-TgD@wbJte+v?Q-@xas^cfo-$5nkGaXL
z)4yZRt9>>Pe(kySpJUdcQ%vpu3V3d8;%ZlL|8M?AL*{^oT|nCO(DSL;*^%>OuO(0a
zdgJN$&422?F)nkTa}u0>zBF8B_T#s3=XoHweEANW1cg{V{v9?CJ~gbaWO%;D>FMh;
z+3`FaQ}w$yq$<U$6(#D1)nBZ6W5*}IydUgj*OpV$SJuotHGSosz1$^V8cs5+=_~0R
zxX@`;b>I!-X7)331_oz8cQD!{eG}=1q_#(lvEDtr8~Foll0JgOcz*OR`B(p!rR3cU
z){pNC92RX=&;D4yV87c$<;eYCe|J8YUE_O||CVssPglj2VXL#N?0^2hvg&`jUqi^{
zz_b7Q=e^nglSRVm2h+aGpX`%1{cja!nklmWN!^yG7yWNJsBQYbl~FrvlWWu_4{x!|
zlS;agOH{kJq)eQqb1B87vSBJ;oWO0F<r6+GI{vBkU-iH8-iG?WEZ44ZP3}8-{KtCR
z^C#RYTC!b?ohny``K%J1oTVPU<H)M7ld|?MaWh#QWmvd0dVSpGtfL!$sfO)8@@o2u
z?QaB^FSoFL014S0HV?ey5-N{8)r5vCB)eX?*q3#5Nt7YT%&qUU_AZIiUwO?t%f<Q?
z=lkh437<gW$Unm-;giy{1O9%8-9hYc4L6z7_-E|x&``T-vmB!L@1(0$o?GLB)=s~1
z`H1+uBv9@ZgJfe>v5i$nUTKE$M@W8f-*t{LJj|I-UH-FTw|+@pv%~SCg>TD#>MK>a
z*!etPILG{QrSV(@v3<ekG^_k&sD$J5vnBU@kchVo<%UR_VA{DcbT0cWhN=^6knW2;
zGtbR_i9$qQKye4yNCJ*oBWjkt>Fm8H-~2vL(9ux5urG7pz6*a||F778`F;7XzvrN|
z-~RhAzgz#UTYmrjm)~<gLg~8wdU<8`I(>2mru-9Zl8jE2e`<KAGVN|lFH~*M_iTkK
zS=$6i;78abc*-rPI8t=J>ZiL`-aMt>nw#by`mw<3R&?;2wVL0qiBxAVxRj&yw^zL@
zYg4^yheE~~5iiN@msdxBZx!RrKg+SQX^lzUgE)~L!Oyu~W?rBCAX`0SuBKC!P{Uo;
z{=8$^S(kOfT&k@FSfUde!|H6R(tZ9sxjFa9k<aO67YlZ9{i+pIQTWr7bkcg+8?hfV
z)s<P}u7_uD_Ae1Rc>m20pXv?i$(v*D*-UD*UZ|Sc;9@a*_qpOIlYQL0nv8#UOFEid
z{roOV_F=@&zW1}^Ua@{TkY}uLa^s`Qc13+Pdku}L+mgGOCixyT<yh8z(pP=*ii&G%
zf6TBq>L@p@h|m6M`FZlU*4fs@Jx|L^{6gO!Kj0B{G3)xvgn*MNdxgz+)u=|ao^8^S
zFqHTHRp}lk(|=}XoZC<C->$n_<iG6x?(lwL%)NCBUKGR`hMNRRYHzFAB75n}&J-t^
zi7O}UGg;oG@m0uNU2*N{J2yQX*~I7H`Znpn%W3oZy&h>E7FXnAxFdE)X8A3n-ruu3
zmamTQzwxU?D)`He8&B_vtvht#z}kC94#}*ZdLdaw<+8!f>yJPGeRB5BvMJlQpM8~^
zvO1%3`<uz07D818ui`Gg)6HG}WYN{rerq;dxl|+Mwpikujb{w=`$NTX%TBytv}icW
z(ws5RYGJBfvgw3ki;aPCo^#WU0yZ8DzB*@;<~;qV_sO4RMJDA-^SM0H4qbA<si5da
zm1bD0)T72k*GK=@oxd<`&8pj0Qj_TU_eR1co@YCmkAzCRj6JvKty`zX*{yfvdUSaX
zW=Zv)II3(?axgM~zI%G5wqeBEJx#Y|)a-nwsLz!xyyciCBf0G0!`H4cISs!uMYnFf
z_$u+$U#*_DoqwzL1#Q{3ZC29S$;-J4?Y?<3R`i{}J@xg3hsUG()qTUx`W*W9cgsPE
z&V&n=y(NGCM&{0~Yt!vn@?}Y5plOu9SNn(aY<~Qo7VOSFvW0D4m0*lnhpX}7)lZ&m
zYAMs`WY7=%@nqYIgIzP?HYCJP(4Vtx>E~LH%;@K<F4(-ivUdx&`Qh3lyNVxens~i`
z;$exZXw!$^cl#bH{FLBz!1-Bt@u9-rnfa^O@~zxe^5f)I$$S=@eRI)eBkk%Zs&SqS
zcYeOUf9GSa7Q5m)ak1uQWs-?ko25@RiAt+#ha@Gpq;8f`3*5Z5C~|6<V(A{^q=L`h
zOH)d$etKNXS5G&RdB0`;9fpM#s|)kyn|oQvzgPa0_RyVai$gk-!L#CPaVOf?CrB++
zn5UvO%ir&}%m+0U|J{4en)2zf&%Ja1iDPF*gyL*x>)jV>Sc5{(Z*sk};TW6b@-WNm
z$J2w>J^#t}URw3UqsPa7eA&<6_^mF7ZC63NS!LzqOaAF4oZ9ApJr*ZzyfJm@wSv!A
zI>L9a_fXt&)~dfGLg)FEh<Fc+FvY6fzW-;eKlL+{ui#95vg-r+@G8ULvyV6Tb)EXO
zDq%v;Z;Rbg@p3tF6{nu<T%~&a&!+#|zEo~kVAE&H`OTcs^Wl2EEvIPKpSmR#5;LWL
z2j#vk&3yM>hF9}W<8i;b{vRH9&aVD>{gZFn9BqSq>xcK>v^Y%fZGSoaI@?cMrOV7-
z`vboFP5*rQgh9u7x2vw|{2!<A&3*jd{>1NH3q}7=xVLq0zTBHrpFTFpyRBNDaL?{Y
zhsusg)f3gT73Y7;HB(!+?9q|#{_X0=|LTVXEcE1kJw?9dy8E=6ITNnCSARbrKlx6L
z{E6S|dp~XW519APByQKW1Z8HgZp)lmZ{HVgb}!V~zwAQK;c!(MbLPsE3H)ET+|N<G
zUAp|ZM|sqaOB)XyvZ-Era3h~0%j_SQ8ugswk2}9x_WY^t$rC&38pDs4dhQkrczfj3
z^JD5S(=G36%bAp}o{_gfW7(6aw$${ux8}Stx?6R%CR*&qvm?`;oli{9^xKl~)Jpw#
zo}OB7%rWu3bC%4@tU2`h^V4rUwyJUOPG7ngKS?iQ;(Ie*eTjLB#kO;ADA~SU_p;jR
zxtC-7wdG6d-RHdb-l^KwlAW!WALZ5fCw;1jKI^_ouieUCP07|jal7r!0qN7#l~)h4
zS2O%hzhSqFBX*CAzVz|hvd9S^USEE6ZS@ttzRs{I7i#okMCL!}vHhrLVK>QqyZ`eS
zYww(75GwfiVb7g)Yi{3vzj#{rvYp#LGv3`H`Tg-8sgmWX-RqLxZIJyszi*=NlC5za
zag$7Up7mV%&hx<LhYL0<ACGhnJMkd!v)?_==j_j(F4B7U;cQP?u*n7ayTPwsojkUo
zIQLsk;GM7n!99-rR&gTM1#@()WDnWj5r4bvmR&*V{uvgx|7<B_^D6LOem8Qvv%Njb
zwX#n!#ogIezh9nxB&o#Zx@nD7;E5=i%lkIH+FQFSjq9T98NENVCC^RwW?|2HG^MIS
z$?lTGhh>i!B-j)%vP<&sKWLgVFZPMe8^b>`KNZfEuT{GK_+arl{d~*%f|6BzntSXx
z`_`(yE;9@>yYhOj!OPcA8B=}g`=o2bK796U%~|)sHSKrE=Ze|ycZNLLcfb3(h30+P
zM32ee1>?W0sACIvv%j;?-SJ&c!@*a3y_asWSF$|$rFoOb<8L$HCKes*>9u8Y50L!!
zyt_;N=BqR5WkJ7JSk{_HOshOQOTNabxwl|HL;KC6A=?e4Ulu&QCtq+i`B4(5{^!FY
zaT}!`FXk+nW5e6>*UcpJ-nKLS{<Sls_v^koK3QQxx7oi#>mKBvoo7AWK0dT7xz9TD
z6myx8OO$$Df$a05!h`Qtg`B&>RTs2R@~Z2j%dT4=N1d~r=DTgT^T%aB6~4^Xsp&i$
z^EY=!nf^h)vX^srNdA3#ME<(PBtL~L|EI57{nIx7bYi{{c<t=(ms2{J3a-i8|7>=U
ze{k|kX3qP0iyt;j`Q2=vRO{S+^z=Nvb!};#J?cS@E44e__Sv3}>^rZK-WT({$xL>q
zU`B?ZZ_m22$3i~~xzD*+wts)T(&mxIUaJ*rXYmK!&M=!jNwRAHgil*eJXW1o=wo8H
zQlWj_?AyMvw)aBkrSrV3Fc<r?qE5ul>u|>Mz3JP`=3O^1*jS*>x7PagKP9J6y~Pu?
z3Nv_LURb^0%H*GqCHH-B%(Gr|vNfRS*1@df_bL*e?TAUNopC9wZohniS#81UiFzN3
zie4?h?$q^*jk)US&F5Mw?$%sAu_?B>g>RP_%(-Z={dV2AZ)eKSTCP8RZnNa{NZFj-
z8Ff-yHtoIb?6K=Z?cR*6Tg;~!xdlT45=-{Y)>}VI&uh+OgENOq*PPv4$?aWx%D>2C
z(%~D@H<$4wJXL$hnt1D+;LLU1mwKvJtUKED-Q~q}tI)%8?t6EwRSER%pZCf2r@i9u
z$4aMcmM`8Eq~o`J^CFww-3tPHIs-49?b?+3{@R?t!?GLp`>)gcn;5!&)AD_<=KkBT
z{LxY2#N!qjeN!`6oePh7I%B^|&c0dYCs+G4OrFyJQfRaC(xY;p#T4Jv%v$r?Zq?T>
z{PHtXvSrE_^{)OHu<jX~&T{$pZJJSL*WM=OZ*A7R<oIjRt&Xkt=Dj>~V4;$Q#N0g+
zGyJY|_q}}J#B(qE{)w3OewB_{pM9R3UUB_mXm8H!4;Q~TJIC$(-@EX@ZoNG^VY$-c
zyH3AYd_Ur<=h^5@tb(8IS^1OpM8B|Fn>2s7#1~Vps0*tjPUXp;I_lhZrT6Rn>6Y{P
z<xkql?^w><bo=O{dvmz$9@h4EEd2QEe2cd3^84-yy)9)a*JkgP-}8NY)y#V}_y0ey
zPDm-d^jLBa_vHIVP3<C~e%oHJbeXm4uEonICV}POE#6I13XpDG^T%xh|F_Rqv*+ub
zpV+VK!2T{NW6#CC&*z4fyg5*M(`Q+ii(Jpl+RaJDACptpCvfU0d@+8RtK+t(TOjJ`
zhB(V(N_%yV$%ylMs>hsA`~B#V<dg|ow=-Yu`Nwa4e`RR*H!&Z>xsqF+D9xR%8|J@$
z!MWf{i~1vOlg@t^K5U^;b*)=2b6vb&nD>?US?2WyXQ!-AV6Lve;r?c7{zD1LtyQ17
zcWUk|dfM`I?!~<omrp%sZ=Unl>6Dd9V8M~u?Q4xr9BmKMU;n%0xMRV@qjwfYY+~Bx
z?WZA`Uf$+-czL<;=GbrBbhfuiNnf5QS%34-3xo88Nxo%I<xD?Kzx>@`2Ft_tRO?@f
zJNNy`-CH^1=eLGucgjusOPVWJc`(0k`ZD!()nj(vuLn=Mo5tPpd>^m0?%bIdUn`$I
z4xhAg-86$;lN0@%_Demk36L>r_%=c5#iHhn>sz0ljX3`0<buB$PlR@fYB~iju3rAL
z&d#PPX|K=5efn!{4nGv$G0$e1Z)fm%f%4xj8Q-_>nY-+T|NqZ2{!t1E0%_-6?X@zl
z?OQNM{7T`9Wf|AYl<QU3J*_!gQ(yV*`+ZIR(1VpLr@xnX4B2ST|Lex2ttU5UU%S3C
z;a`iL$f_@?%EA+^il_hFJ$K8b+=ItuC%gz>x+jWvbr_R=etGZj*)I7TLd8z+zIwXh
z@$c2g*XW=3NKUdknKFCZbM32lPcW=KJT*;RUiZ=GoJX$fetK_u@IBCb@7|<4F4wnb
zKeM|2yLn;ZjX!%Pxn#v#<nq<K)h`zfvR(DpXkkwLCHrSH`!ZBtsK`u8j{jVhd*yGL
z=)LDR_b~d1KRzS-x1gxLvP%Ez#O@iV|F-;TJUGWbuB__cfhQjq`=m3MJ(o_;wwlQM
zM=`0#?H8kU?o?;@x{0sv-Bhig|Nd{&qxRhfU-G;C?ydcL;O*2+r+y3lwYYU$SY@Bx
z6^1`&W}Fs(wAWuw?n7hgvuK0o>wTrdkK|uUJJ-4ILTu$j+lVXNW&2tu%s9@c;wrCI
zCGddT_N-1ycZ0;Q!~EQ}MhthZh4XJolss9wS0}We<(c(!V`GEto9A}S*_Us2<Jp$^
zi)x;ETg=s;bF<gNR@J>~6>Ds+{K>cL_wFq=J(g?o=EU?P8#T7oNcR?~8Q9FId@Piu
zxTSI7rJSe+-=^cbU(Ne(FON8Jw4-F^?zySgCqMl=$34uWa$aZC*T*ZCOunNtt59Px
zi{!MqzxBA{*FCxKYbo?~?xJ6+F-!J*ox1LgQPR89O*hmsi`LaT)}|zOF@HU`a=%)@
zMDOBKfB9Zs*>~sHdCt{iIXV4%(5Ig}`+|9`pV!74GC8tch+J=>sCw^JRrIsdmv-%|
zm}BAj!cuKw&Ay+%Ur2k&mdeJvslQdSe)C_tL(*uG-NYi@I?377&#C@Be6Op>?Nxz_
zdb&&N#NBI8OXag?Du3o#xjBxD=fv8s=@akCPwMKl=MZobXH@%Wx%gu*f818}!uuOz
z^f#|t^V8_~mbrIcN%0(8x@XOPeQghMrL5@+IcL9Boc$ib*lAa<Foh|+q5D$px#t<T
z+f8$S2Gpony}G;SxWk?pfu(nKF7LeGw>#(9>Zk?l_ZplpiOZR>?RInL$~`W9nn$WU
zxwu{Kf1mo|_;-FEkIx;2y|eE<mM%Xe`Bz)I`aw{Qx}(Ss{?6?C|GpnJ{4jg6fgSgy
z&3m5dJbN1;Zn5*;zcf!7?H?w~GyeSk%D91T{~7<=<*Utp)PGjkAKRCFrNoFysHrI8
zs^N0uBgdKd1f9~5*L?c++K$k(rgj>|Y60$|?zPfWYx8>l^~bP0xh8G)ms$Pa@7L{W
zpBhU&)MXw&ePTWT=jPjj2Wr;#SlrdD&wrFL$9|5>((-M4pS+**W%cBZa(eTYZOxOG
z`ugILi1L;H?8Qgk3S7-j|7W}Z>htKri$5Qiv58z)4XUZ;KFYb2?Y??mnd5ASPwlbw
zftBk@uT2(s`+Jw4a?Y6|8)n5*$zST;-CrD}melZhOQbn7Z(D1hZNbT@^S&4HbJP^e
zR0YoIG+#CM{6>eA`snhIZGU>Yf;IB*>@b)*VO8eA`<}C+T}0mXg(|shn-FZh_<;G^
zBCC{ri8m*Ei0@o{^{TNYljY0YcgKx2G<C1cpA)CC+3tT`Cx_e>jSHPmmQ-2Q2+p}Y
z@%C<ofKOjuyB*EllXENj4#)XlHH%rScr%Teb2TmnZJWO59b3(>+kZ};Hr04_RrA_}
z^$ja!rdZBjfAy4r+ynP~ZDET$jZ%lBcXxQls2{z^WWPX9<9Nr%Lk*8+J}=>$rN3Ty
z*Th`wXFgM2I>jXawt0X1@LA)`Uq!cT(jM2FI2EgCu~`Iu=d$=+F=t^3o627KAFV?1
z-<p?Cc)4N1qUr2)3y+lj>98%Bc<>3&o_#w%)|?Ih8{GbD=Z6!we|!GOOb*{(^~<TK
zA+O~9ZjZemC+TIK?{F76J>|4;bLOW^FXQ>%=C9p%RXmtq`>{B$eEq})iPQY|ee8_X
z<KMeCdse}<@3KYqp4(PmPU!o&>~yxh>U;K&8}H4JGkSVjPr>`qBt~DMb*tah-?DnY
zecpUe{(s*rcmEKR`(0ukGsFDhzj=JWOXJv;?j-%~xM%h8x!J2%^7;HX4$W_y`0qu_
zjqm#-`||F6X$|=Qr)>3~f3`s{-sVboe%*SnSN!B9`4caWX*-K^e0jKj`*HhHLEV?K
zxsR3lw{f$3Pe1?sweqCTM(0=b@9Nc3OIcSPHvf*e%$nJ+o0m>@_IUa8TiFWd-_7P<
z*3WGS{+ZuXG{Zsj?)O9W6^HKpKBXDY6(9Wf@K2BU9}h47&+(hFj^Tl9@u`hwZv$HU
zXB2F5>T0?F!7kVDnO50$i|KC|s_j2np8U7Jf~i}7lbLj6tLB6?{<62SYC?|9?>uYq
zC(h`)0n;XDp^0rJH|@V3IlOECoK1JP9<DOq(x_GNe~PmDx=3%CGHJf9nGzFhmb;6z
zxyk8jJAG;Kc&REB_^P^O;)N~60nf_p!e`Z~H(m{_2rRjF=G@BCAGdrj^3B=Ad}NJF
z+)BeFv;BX?L<%l;Y<_s-`SOw_y03We-`vxEO0O*9{d9-BpS+~L=E{A|pZKY4y+!KM
zIWHoVAKq(?JT7}}<*~-^hHq21+}!$RedX6Fo6n?Adh5x!+U~s4G=>Rh&-;tY%y*t?
zdE5T7PIcjGxxAOJFA08)4?ChK!p(Q`c~+6m^$#9v-=;mN`E|c}1%Hp=^%cQ;-Rk8W
zjJ7}EXWRNTcK_`4KhG>)DDz`i`}MN6XP4BwHRndLZ{K-m+fwB<9s#@JU!>%w7t~uX
zceuHJ=kXsIFRykllu6nld*8LayS%*PQi%Q0!`qF2PuXw3gYU$#vdHGmh8JI~erf9v
zrny{Me#PUj5fi!gZcATK%p_MFuAZln^Fwr6t%~TW*&8>n+baJoboGtQzyoE6S5&v=
z>=u6XlKq!MiN-=ft=Mhl*OvM%T{HQ#?i+<gJN7g0Se<r8y6wwDuD{C~=N9`i8h7r0
zo}1D-Ny&O!$nNczKMQ*t`k5B*X7&5q+S~d3HrLKZW`-*k?DLhmHs#!j#f`^z+&unz
z&GnoadtRRTK3y{A5WCvCNPD{vE2CL|JUO%Pz-5iM=Xxww$}A34li=WZzUk5296j%U
z0-u~VxaR&>)t0>fKs){q;|A7k-=98|c=-9XoqWdJi0SroO2Tj6^qV&KX6x<O9pC5O
za|zkcoqhY(=eWbO?`xmkZ`b3NQGTNByLwz*@9I0(i!51UIj+^;PkzdBv^u@b(*C?q
zXVvvP=a%1z%d6ej!e95TaN&w)JDoT8S1h>hz_=mIV%fevpGEiN5Byb-TH{kE{q~>G
zgGBLC`)%*aDklH0dZ_vDHh1xjbIX;h*Z*z3I3s*t&j!Avz4ue+bk|m&`qKXT`$v{f
zLiLYXf4tS$U%BgB8MhJh|5sLj-wJyu{@GAbZ{oG2^-qP+u1ohz&n16x4dxLqd?vj4
z^f{j_MuTWQQ{TLW4svdr(=%O{`mH!N@kwd<%Tv*EH_uLFZ+OvV#k-j+{m#7t-qXKi
z<~*9o^eyMkRX@%ji5XAM{+l^rU2Vio+k+1B7u$dByZ7|in~Ro8SCk&SoSqZvs(wfN
zP=bj2H~yzPk1FymH`DUGTJV3hJmZ<w@BS|467ltEQTR16t&Kb6bZ1k#|Ceg5XT29x
zwpZ_*$Qao9s(|g`QS&8>6t*4CdbL4L$a0^E^~*)Mdn*0ye^{>NI%|FCS*-Mn@@c{g
z{;AI`*A7Uw*{^uw4=dkx*RA~vg-*TH?I<hX#o)BPUS-q!X|@;K4@KFVur{gp8lCT$
zcrPyZO79enH{Rc8=r@=h{2agMuV_B++um@6FxSv&tJUuW$G)4}V0HiTf!3a>&i|*M
z$V+E3kWoB!k(1+78}|n9^_M$^JB^}w*S_YNw`cMpZ-KXVmn0?KnM!|4JDgZ}pq$Mh
z=cm!!vYP+PFDh)aXZypyuiBz+`yGymr`{WHmmTWd^Wo#$$1(pMR-btJsmgwTL%(vO
zYWsgp+kU$V*F!>5PGx^$O*3`aayR$E^tW$CBG$~#{<A#%VJqu`+pXT0W|z&G8g~5d
zGpoHKGh8O>1o%C?a*m~DgJf35D}94ReSz67Z-W_3>sM^^y2=;0*~q-RXwAvEwX^c4
z8Fe0gxM|MwNm{Azx&7|7-n!*@hW$tF+x%@inpfUm`11dRhEqrRBO@*s=zd~acG!Y1
zNxWtT|D~VyKiIcbuRD=hy!*hf1CwR1?|2!RpZndr&aJ+k`Tfc?_q5}^3*`1F?%u^`
z_B+pJQ{=7G&1@IX9Prl6zt&+~yXeK<irKBEIU@7Q7EN>AsJT~>`Odu?#Zy);Im}TN
ze9&_O=ZguY<yy~!|GFOOD4BkC&2CHI$`kCY^8bD}yQ|uS?^#l<rSwE7uu*=8Mu|}I
z-V;-Qdxj}rvOV_e5u3ra$i6>MwT(`HJ^1P5&4r50#<^c_l&uL`t`M1hx9jfqbkkS0
zdk<cmY}U4+^TWUH@{GWZwR*{$rf9viTU(TVS@E&HU{79Q&9jb2z0KTrmWZ=`$<}+V
zt&}uVf4hzM_lAnMEbof93ol?X6IjSF&4BCP^5x}OyfXG2VXI{I_~UDfN@Ihbt`mQ{
zOlG#UzWopO3JLuxasH}xY%w<)JQ$B|i@p0ckEdhPzg6Xe$qPbM4a(9ogaZX$3z<mz
z{>>FIS)_kX+-5<1Nwk(J&)kT(wUd>Z*V*=8^Os!y@A#KFx0}y$J3TUcznkY}S*Uk>
zw9p>LLnS-b8_ihv^Y+hg-_QCx>^8b#cJ6U??9qR=X3GnFdJgIqr_M-EdtMsy{7X#F
zgxnuz<7AAUoQW?s^q+Bsoh^3z!gMx)u9SD}&pCzFwdS4v%qz}N{$WG^?`CTatABIF
zb@YoX>wMyS_l4RV{<&#Ge_Ub0;>i0q&Q+dbv0ADs`r-AvWhw$CtV`{btN5pE)0W&J
zwq&E?qEe~yjC2!;@^2enN*la={&~*rg1dn%AGZiI|Cyy%e_N+0QP29!{R@R}&GpJJ
zR!UFoxibCDlcGmjM&}FdG#{y5yJ2>I#q{?FCRbkQdBUw!&-s1#>jecFKD*ry`OEJz
z_)>3`$HO$Md+qU0d>qotZhUOoHl0I%hj~W${aMp(Iy8B9SJ;Ip?zs77=j@aRXU&&3
z@f=|&-t^~}NXz4cmt$+>_@{k#FJ;XOn;3juo3peheS!7Ixt4Q&AKU-o#6P!X@hji=
z`yc3@`#(X1=kbnLPMNR2%x#dYVx4&6eW78E=>9^rNt1qLE{v{>J3YDY{p5=w8=CiP
z_et@3e+Z6v@r3oF*W3GV_8pZkn>tVZ+~@WaN4}k#ZMQ!B=bkuu|G0pU%p9}c9XWsf
z!6o@`VhJuQe!liRz`XzKqTS(Kck|rCS9q*AVCrr6YPzk%v$q@gO@dyYj<22|@4sN*
zv-f826YRbk|Nl{AeQH774FCNfb`*WA5<T2g%FH0Dem!K0;f5DxH$JSJ-LOjN--mxJ
zOn+I=mp>2FYAHB>Du~xnOw;40f$rQpO}|47*Z*B+6latB>2l`r@~34VT|HkPpHgu(
z(0S9`#`C|Uq_tMwvt6r^e*e!eRh`+T5oYX7mSwK?zJF^Y<7=cj*0sF2?6FivedD{E
zZAuKjRi`KUhc7tU%kW~M<CPc3KF^!wqW5g;iIlo$4f=7Xs#Pysw5-Zo?k~S*U5<R-
z15wr2x)--sSuA*MS5ciHKK=F6yq|pcS}e<t9?U5JE`7?s;-0|<k6ZC>-|IbRPIoo4
zd$C<5Am>_O-3MzSri~wEbz3gKbW}{do#{RK#Bb*RJ_<IHCk(clhbu8ZZ{YS9@BYYh
zdEVQD?@Ke^ynX%3ggL<L^zVi&My6Y@Mcz%FXZO+M(8UQ#p9>cridz_%Z#ngAkl6ZF
z_p>^*U(G-K*PkhH<5r`Dvlb8kmT^l;NocHlVHLG=Y4?AVXgRg4gAO5GEbh*6OJAkl
zTV{FqTaTE;#jBm?%R{tfG|UTPl%=k{U0}LMAW(GXTmw^{A0|07?qA~LZQd($)4yWr
z$s<-9X8R>n$V4t&75`vQ(YAYE4685by?Ix(L@+Y+ZH*dJUBM53&B+3J75!ZoKia>U
zt|729XTvYImOCF)yr*B8kfK+SDYmn9%b^_ypIY@s?%S2AX}KxM++E8oaQXd{SFheW
zpJb#o=dfjle(;)?HT*eiKB?`Bo6%xf{{DQm+S|YHyJzdDyegYu^YLN9h1lOa&TTH6
zY~Wkcmin$ld~reFwK&_ZbN8Ne8~?ks@$ojU*@26-nx3c3-gc0g>G9PMN4NA@I9tkp
zxD$GA%Gva$1#g=^l^GRpJ9BQGgO$6Hp4(3L`19fW{BAgPIoV4jOTNq1Ig%4{KBYc&
zMbGC&7c2JX1-`nWH`nFb-Gk@<Y`=V)`~076uL^^#9*TWfBK<FGPT=~@U$e@;6i8aC
zS@tg6(w$r{mwm2luIS8f>hTNj&-ppyJ?Aommst;HFP?da_4dXXiwax&`ht1=r#oNP
zB^{V>H(n@7e0`>yoery(UkmG|`|;KcG1vZSubo~Q+571FY`q`Ze~V-8t+7&yyZ!d;
zp%DAjKhq|k%ZwDid-QXu)Ys2x5!3sWmPT*5`e?%B*)zTvFQ3|d>iNnKOn(lP-i!av
z5F&W2vhtU+K*!y`5566`oa1ZA5^lHomi&$zn~g*5wsu6ncX7#b*z@T?%-rpobpq}3
zhbBCqeD5Jc=N;E7zf1R*hwMHhyw<={WP<F9-earl4qRUPgOBZ#pY&HH!C&e%T^nn8
zzQ1}jZQuKyw>xLH{61{_FRb$ByEQ-fZ5(?Gj9>Lnf56P~;&0~m;I)T4=N7ig)ZRL&
z`~PrCJ8PEx(q{elb=xmR?bYY6d-2^e=<I)8{gS$knitmSwQ|g!yLWG_+4GN5mp=zH
zGCg~`^uPXvqv9Q(>ezmB6z>g@f9TPFwk&^d$ElyQQ}n9JTwO!26n@c;`}y;5ZtAa|
zpU<9eZTM*SOLET6o%_~voCr^N5Pjv%c9Re5=QcF`nO$b%^TWFIex7T^o7v{4XIj=*
zUSg5k$5?uOw~W%Bz#PsRk28y_ZXNh`H0@&GtgD}l6>jv$th$%?jOkun{)W}hYGz+6
z(?8v`WShh8-Acv#LVjPVikhinR6DEwu-GnD(Y&vZWM`?b_B<#0CVj?(M?sr5FL{1D
zu5%q%{%c>)-Hxa4-FmgHST|AN+O4Qv8w+Ek<EOpb{-t0aN6)g~f7*;sy=vzX^s0Lx
z(OzhB-!|gKgTSJ#z1+9AoqGS}+T+GgR+){y_2(BIom-={XwrWvyJeMPhCz|_X8C&+
zcdy{RwaYzizn=7kJK`KKJI(nH{}MG)n7gOp(6M*nwPGF4mGVKJ6>kr`IKp}5>+S9e
zU!^5=|4aA&jr02N%-OL>bYA~u`-7Y3$hBm6y?$r$+2_lSUm9$xB1hIOm%sCqc}K<h
zIj6$~k1!lQ7P9-dy2FcKlj`)jm1A}=mhRoQyY$leb&1neWsZAtv9-De&E<^xw7Xhd
zJm81chDG=GoVh6W{#(K8SDA5s+a=}~Pd=g3&)t1?+ARC-|8E~z#$5k%(DdEv4SW7&
z|E%!)+`O~&+11H%MV#R-oCfoMe-7sP(Yo!h#@^NcuY8F$T&jMHO+$HEdq&j!90&3L
zWqSjjJ!F?Uc)I36^7q?a4C~95O`gXv|MgzSpQXt%`=*FGJ`CUSwCGtzLZsr7cl@hE
zR)kww?rx1UX)0cwxGes@md?AsyvyG9a>+jrt~U^F*Op#xeZ8jc*B`$R_y4g+Ihr1c
z|C@8XLGhRB_P%RL@{jT^hCR7_`={5H_5XjH&vg)a%w!R;%9j6_SX})h8}Gk%g{z$E
zRy<RFdT)E~?8SOd*j5*m#jSj@>0-n0-ApdA7nZFOEI*y*A39V1=DAv})pDLG7K>Kw
zJhENlW$srs&yx%41AlU=+%nP&eDhVYyYbMJg0lINs#~u9a{TddTHm*o+19?xGt;h3
zPv#Kw^xDpMJlOiD_RRa?Yl;s|J7Qxmaj4*6_Wzl?g`5sAUd%Da)_=ttqyKN40{Rvh
zPq@u{C*wu+vX^S&Cre*F<Xj$StT*F0>z$`1k~JP`JF-H!<>OjcTJYX5xTL?1`^sJI
zzR<#r(~hiI{X_rqfx?=yb-{@o<q^|wUw2M0-upRnLWumOpb0kOYj&Q^NPYZbcSZXy
z#^fJa2V&gR;-9Kqe{DR6Kl=CVIS*Lp-u&@ILoA%<2=m#^;UW!8Y>LHSY|rhs-nreu
zwKMeObkQaoxfM*y^L!HDiY~XByLw7&QB%Us3tm>&<-W~MG%Sw#Ij=9jE@J9!uEN<h
zlD+0Ddp~<})%=eCBB8dr{Oe(9zlw8nzs218t;l!o(}4wZo(cr7a+CJITT;(fF5eRD
zJij7xY0$j<K#`W8Uzb?7NjKhgd2X3kzKUmyU}@_AZvNk!-;}*y-dtmTe4_nci3>ZJ
zgpXOw>6nw`dg_neqji^Uo;m4%SXQ+u-^}ZW)_cv<KRT9u{ja|=CB<<Mqxf|BwU+m9
z<!`=XzV>zQ&2wS=Z*xmeJc#f&cYCkLy@#WzE0QZt=iSNV{;sF8nzA<wAFkNR=<9j<
z8n0K#XGt~H1(!E&toiuU&HuN<#@RcRmt}p{a-G{f``gme4;Qwl{(V``U-V$^+2WAX
zJB)IJR&aaEpJzU_SfHZ$dr#hXZ`<dw&u<si&*|G1YxAQ(X8*VId;UGnSTWb}zW6<h
zjoVMy{%AgAx!`)-)ycp1tv-?UL;LI13rDiwSA0(`{VhFVt=*NEj6Ef3A)d1jPujmL
z=GMwx*Z8Afr%P$5&+08ZD$Z7K@J)Dcv22tfr^0-tdtdvWcpYw+7VxyokH1&Fex7<1
z)13UKl2z-u|NpYTq?sgqD)cSSI{v-;*53>`=V@f@xtvS!@7L6w=NJy3zO$ij|C!0J
zTC|sKNWQ2z)#D!b@f)_QzPtOMueqsIp?+V(dhgkv-Crw?^D<XF6FqUIv7g_2^SwF-
z@2m5lTR4aO`C+%-zUo%Ly2AQ(S7m$`$A9@<^;v4dycbT51@;pURFrCLo4)`2_y3P3
z?-gb{AiMp-mEXH!mpdJ)ci;(RF0Gw$y(FWHsj2AN()#xno_!gv3Jb(G&q&gFaDDsM
zcljThGX*>S9@i-e-hORV$aL!bbh{1m;YohyH{VZOuwm(=#LK^JF8+wO`n&G+luw`6
z@(UgM%DwvFouADU1fHFhJZQbW*sim6)w=87F8_Q!m-BnAZ^MhYMPKJGs*B0#>Gv;a
zy`6sUckPlAwTy(<f2KCDcRs9dW=t>_Q`Y^zf#r&qD_flX$EEi7gjPR1!};q8+wo_+
zdVai8bX)o-QMgb@q`B?i(Fw;d?95|4biQ8U=jCtJrw(i>ka#$$?Mtcs1dDHza(8XL
zZ~wM^;<0e{ca=d^4_CZwesECt%dD2Td(VD0C%;dTHkn(te{ufncGf@YDRL*b?vL-A
zqgN9CC+wZ<ivKl*{|~D#Xw-KXd-HF`Mf2RKhdZ18KTP?<T~+Em|FPcOh`HP47;S{^
z#LwcG`g70Ib@P<(1ae*2d*GObgVM&1^`Xzo)8EctudMW+y{&lN#Nt`e8yV$wW2V<O
zwq@zOd}qfyW$pg|>l)X6HTl9dH~ItTYPr>$Zmi;2aP!3ifyhVe^iF&}q;b58&B-Uk
z?M@Moz1LzPzE<mCCB5)?f4%2Z_7!coRCj0M^o#!j1Qs=y?tb1@e)MS4@35qbd!2VW
zlvg-R{k=nwFTgK;nZv25ij(>W9JFoocm8>x^!&2pwwtZ{e)iqo<h^&E^)z$QpLs8P
z)nzX)^gcE}!~5<{(baoABZPLpdc96H=HT;>HQCHZ?W&I+fAd~_-#`AzM`Q1mw7A*L
zw^`@N|M;rh&sSf&+^@b_JcG}5(pe_&?dO@Q4;su>Es{Im^8L1S-iC|Cb9?g&ju;%@
zwI*Qe?bQ$27dKsZWKG|?XYI5%CS84HTd&`I_VB3x^B0TdbRLC06f^8G|5X-$;`5O_
zlMK^UYx$h^Iz0?w%J~^|*!9NU3=w&)l)Y=-Px#K*)^>%9U;5SEZqtPM`DMKYg@5l9
zWKV3Y^n34BxUjTa@_<A9?gw&t7RKkI_J7VPnZDuiwWEziVIrTtt!MoIg>P~9=4zqB
z-BS~uPPl*LTi~*T`Im$DhHMVr-EUZWI7|DxQs%S@wv^LL&y-Af=lt>j`(B=x0ymUy
z&tA^4{AuICYNtbi6#{?b+qkcC)s*aXdEIbG&u-`L`EM-FGEORKe>{2m%83oubAoJh
zyz4c56qd799Ay5M&V6LB28;2Mi(Br^Tydak`)2i*r%g83pZw$&*%-wyIqTl;g%TN0
zqx{5LUKj*@pYqy*^YhkS(__oOy3FwmIdbL;-=cuC4n1#k{yQ(cF8XO-yzN64VGr*8
zeplZw-kAPIb^W>1eUrYtjmYw9Z=ZR)fcKEWS2+nmwypCTemvltBX;HH%Olr&r=Bly
zmXljD|8ss;Mtn|LK-Y<ny>mX7*6gbYcz<>B^GW&X=QuQX2p^j*E4ELJDS~x>Z_WI3
zH=oV?^SJW*ERWN--tn7#s@<31WjWvas<ZafCq=t&eD*s&{e{Khx_4<(QO~rle_fX@
zrL^)+VS9$-i(eN1pI@63xWZXXPO~F?LH7QaFVg~kCUCe_%B*X7H)R6ryvw=9vTxGv
zAKxqUja5jKi~W47f<$@jrJZ^8!ruFIHiodOZ`*IVuA0Fi?>$So&iAz{6UDv7{%dF5
zn=@a2Lgtd$6-;KoyY64PWE{2gPW0}j+|Q%WNBc?bkl5&RHB#d32BEwRb@NYm&plw^
zoYQD4WEsc$BR^+}aWYSuw2ZaaY^Cg?8FgYeD-U?7Y7|@9)YnGNUcM*J!fa{P^rf=;
z1y&^vKk}G`bcEFYMK+b*oA$wU2lr*q73*~B9$QyhKRPDAy0wx0hd`HCda%%?x}Lyq
zUdm78E!Nat`1$Fb<o8pK2_KrR=d*l2q|NI#;e3&f#jUB@Q@sDTZ?%noy=}vOzG~L3
z)8&sDOWoi1$g_s?;QI2))wjP@dVVy16QOlJ<9zKpxBPaqBIN~}V$IqWv$W2a8ehm0
ze_GA7%Dd={#o<#qEgO$2$3FXXuJ^j@$=iB1#c@G8cfb7-x^E-TbNlI1Bc8u%U$&Kt
z?CQ_%?KBLi+h@4Y`l_qqj~`Ahn)CHlp51=6@omG@JxTFvt5?}I*F0RazjDUUL+Zbt
zTmCCsa(l%)>&9i@xdg80?@+(XYaX=gzS!Rn-sc}DJrm@Os+*H+lCi&kL*7DBo!PUa
zCg#NmW`DfGp0ECS%KL9Jw+;p73&d?Gm3;ZD()hI0``|{`ucuGSZ|f+z_ON8<boRdP
zdj-lbzx-f&6R9_)<(QC0)`zG$>{nlkOrJ90_MX?A;sy84<{f&{zT{So)&46>FCXvY
zebQO?E3N;Wpy`CY&t|XrG128?e(ef@qbq*+PU6=tdT(95=I!3%q~BIjk_+_hly3B&
zUih?p{`>Vmmj9OwcxCGUXSQ0z?#LZmKZW%lU8-cgNnLM?!?l_v%wM1Hje3#FUjIve
z<(vK4F~Lr&e`U;`!_u_+$@&F<orCnZ{ge=Qzu5V$;AX?uT;9c!7p~tHT2tzKJ=!*W
zw_ZVIy>mprBF8$Xhqc-D%RXGs3dmWK8vOIn``1>kJH`LLcXnTBQS<8mlak~8-d9h)
z`<1<2`ScGdPRqna=Px|t&)pZ}EcYz;ch#q(Qh#C(tiPHZy!YAUy4cO18K3=Rm~s2`
zx1afWzbm&KWJvxk-TnJ7=jDpWwes!<^Gf+2@$-KWYCf^w{$;;<Z|9$9#jEP`75C3;
z$bLF$gYB>VJ1)(5yZ>RIyu0E_;Uifa&+eLg?$6U7JfTbEuS^M=Q(ZKRqkl!fj<fl5
zmr8jQ+B{P*TJtt}#e|9R+Be)pw4aqHYiuo-Y+U_8XSS=Y$(fg@&PN|ru9aPXRP$ub
zTe%y5SMwatdzx`!=cKLb*0E~yt{)aHYFN3oJWn8(PyR;gxrba6emtFYZsvSOo{HC(
z-FNk6+iy=YUvRfxeu`q;-^S=aKVARc?7Hw=WcAAfe^&g|eJsiSXL9-t@8h$3x|A--
z$Lc4{_z+-aF2}_lr?9SS!@ha)-Iw!%H7&OKPrs@CEOMv&ntu~S^YT6$_&dL?-MOv$
zu2HA=!~2e#)x<S)_Bv?IZ%g%m{Byc-Pr=3ijLMz!+RY4l&-~96KY#i1@9z>aR&U$S
zx*t22c~kzq?fip__aB_Q-I&2Q<gv3EtNpyR|ALlYe!G-*zx=%W=<|Q8wAIYSck<0&
zr1MgR!Ki%Q!R=rE2y!*^T+DyF;qx~!=gY3j_4hiD{JZxoT-Wi|=U~H$dOA-W82rEP
zo+p`-x!2|7^>4*tsxMPF`uv?+<P`q5>h^EzdHMdo3LDS=ocphlzv;gBgx88I>SW$-
z5_$c2Noti`hj&i9{4=xp7qg!6TZW#i-xsz$x_9&32m3u6YvRxB*P5(3m(yaFajN~L
zs(kgyz24Cm!kd_nBpWj?KP$oEsktiQoLw@@>BT`4bQV7@{mY_Jvqm{L`Tom!Q=YsC
z`Kx%pIpo!Kt82#HPXztzOVr;^Uv;CZQNMuSRBuL^cJBL7wva5B>Dw66bY3gFFK}D2
z%kSx{+T8LjLd(8xitkVeU#+)Z|Agtesh3jCIm~)qZxs%y)lc2m+Wexj;E-T|kI)UC
z+pqUrzMWC;GB3opHq-d~V-1z1Ew7aq#GKSETmLXz{O*o95`T*9UQP*~!?@{czohPW
z@qm{)&j0*39d4id;h8#f#;aydMRm8$<<C}x*)Y5Bm}$4LGA?V{iYtFNtb04DjKy!p
zYMuQdvkH=O=Uv`$S8D6&SzpsSG7dJC=*MsMx))Or5g9&5|E%6*@0yLd8{d7s(kac~
zbALhfqMA=VTS|J`RzF`Zw)gM7$?oDy_19_0C>+#K=X>JzM0)<|?kNi&uhZR}IOkgJ
zyS?(vHzgl=U9Wrpx9Q@4t(E4<RpJ}waRix5|5bc+JA2F4du~i~pU?95n6%`#o%^5v
z!Y{mwI%EWYm&bDb{g?1VLh1R7Vn4NKfA}Z;F}n4~;-~y`SBG0G$|e?=Znxj_bh~_q
zPrashi)VbZ>(7#u=0NGhSJh$q>tA2XUnm=YCr`igb7J3{wc^sjRu4o^sQ33Tp0{21
zwxK@D18?gMlg><P3w_AE@@nj&xo4~AN7c=Adz$%@_t(0K_Drw-&A7DP@^QXG<;0zP
zAKZST&GsfqN<qu@+Q;`d>SxOPJF$8g_I^rxQ2cS{Jq@XsPTcEHPkFC(BXy_U<wMfE
zf4;F~&0Nc}^RTGHo^YM(S<6HI?)i`*t@%OZ-v+DBMZV=xzg8Za@n)a+%Z>Y2?$YDF
zcPTYdzx$3}%jI?XE1upy?6OJj->s*4Du>_v?RGmKRGYK+OWfq2-|t`AnIg{7<+pch
z{_{!iU-@je+B<)p1!qNL$KHQ;pQ$hLk7sm#Qu2dI_~wT19@qXkPMOeBdnI{O*%I&L
zm*+($J$HRu8&b4ty+p=OqfdA0YP99&&vN<5z3033w&TayXT`KE&`O{2Q(yPnap8mz
z`zfXWmoESP&}_bAuw5V13Xdh-%}oco`iuTJz7kVooqxVh+$bwv@ag^cLTef0792Yl
zUMi*YW9RbEX?5Rs9s6aY6v<m3<gP1f{)t)to8T0IQ^#aiy_OJ+@V$M(>6w*c+|BL!
zya)HcjTic=$7eWe_0FfC#4@MdEnofY?eg7oUG}C<_UL=^#irVH^S<;)-&D`%ZC=@v
zZ&AbW)l+(X_2JD9`sTd}oqP9+DE7wA{ZX~|de7BDkH^A_CIw3ly8QH6wcd~an&mE?
z-G_24mM+-mJhLY5cgZ2uyGGJ8!vF0JmpatRa_d`7w`a4diBHJ<<}Qodan-C{pVoPo
zFYCA`+T@dX`mOcDQ!*PCo{ZG}eB<V)OIJ_7&9=x?n!0>P{i(A4iH#9;`+TP=oOBFX
zE94p#zwWr(1JV1zG4b;bPRRaws%8<FT6_P2Xl>*28r^Nt**Ywa2c<PEGe2KrUjI_&
zq_kS=_HQ|<#aVgmE<Qham%Xo3Vw<QFe^*^Qa(~=I*WLT3SJ~YEt5-TldwXcR-1W~H
zo2xcFuYA-d%Tn>y{K{|kpm!%{)mn=y-K+cf=*NEdBVnZvxy<kEI{SNu{ZkLovp<TM
zUhbN2#jy19-BZoef46R}lG-X?H*4`+C*S9LUjNuMC3p86ll#wlFF!ea*yE;+Ud%kV
z&#&{J&z}76_ol~9{F;xI>ZbTbI%`;1+-~YSXLJAXyalddMQ&$r`g?3z5ZCG=zg=J8
z;mI`{7JRv@C-J^fU}COM=?9;nKUqbW{=Mb9ecSo|wZAf=EEXc2^ObBD$L7`Boo^TZ
z{q)GMpDqQ9>u0wZ-oJk*VdXy+y_)bjjNLpd9K+d{eXMD=Iiy?iY|32mr|DZ(FON^!
zFKpVdD*VFS$@Ant9&MDr`+oM6o;R!i*W}1^{%<#*a!>C^p5T}1L7YO1m7_JP*w$|s
z{~%qzd0}<G{w3SHQZMdJc-Uxbt@kz7>(#z9mopUtL!1gT3KwlL3SYfAQ@ubbbkW3J
z4;p@G`B~k3Cwoxg==6fa-#*D~UmE>Y?R9-ex9fCv^YafGtsm{WZmgKrW1i0zxb*Cu
zKevpltUo_wklmyB=+0}w5A89M7WZvGw7c}3c)88~_r&wJTk9pRPoAH7<Lf8Qjp?6n
zpMKg?czWaKNeqF%Kg!OPjZyw`%B0|`OWd(P|E?88@vL4rv2OFtopQ=oCU-B+-=n6h
zb^6|k?1#^?-t%|we&R16^<fiFb?m<K&>bFUl9;N_w_CkTVEEnlxqZhG#)MlJLplB3
z1QuVlUMQ1$`QB~TzmmEOZtT4lW9q+L`&^rJ`Y!q7-G_X|Boyb&p8KiFdXMqtdlKS~
zcWb`aMs@Eh|MT;4`yp*(=F_jDILb>~=WLL?*m!C8A<JjCGtW&l`gU#Gv%Iniel>=_
z_ZP~hL@rRBTHIA_vGrf<d+Vt&^)vh&O7Gm>bLZgtJ*$<)SD!hW_;O9bf=2%qH^wP8
zi#EGIp6wbW$iwow;acv?*j2_^y9?ddY<pnzc`k2ZHS^{Zhcu@Zo_yx4_^P+FzPxq6
zt<J|U+QwxU3hqdE{Yqh9@yh1rUo(Bhuc`AH=S+HZ;z9Dx`ReORcFcKPKIw1V#DCl0
z7B#)!_ci@N_`F`NZPjnZEsNS}_MhS@`;a~FVNKlU(vz*+6Jp)@B-j4BQM1P;-ZptU
zKbPhG>oUP62UMS>*ZBxV=)7;05!gFj>AAMrL6a4y)3|QjD$na*|8l)oP=+m!U07?7
ziL$wP`Q)FIMJz%d^0!7vJgDbP+cD4o)P``LcTai)<~7EZzfN*F=N<B{X2Psig}co;
zrvr9(hkknU__SWuou5AUI<`++C!Mr=Pg?Ttf8Sg;IO@iz9pGPR;;P)<e`w;CRsO%u
zFVzXr-gK|}S)j+4X|ZCf6LLO(nz#JSobCKCmsmzP)Tf(Vl>YQu>rdHChV;slj3>V4
zUhxk3vS`awabDA{m)9Q^TKjyt%Ew~f=hc>G3oq~A{BlQCUCgBS^F9|%y8c`3^!p<p
zuCp3#ysW>N;a%>M=Ry}Of9^3T^5wjJtM&K7ETvs9e>^GVYO+6j?eL);&pdOk&G1~k
z+v4ISL(#q^N{il_TYF66P5vrqfBDS?&UNLMA75^e-4$46XQT4dEbz^hdG>1Nub6hX
zByvpTjoH7R%T~Jl-{niaW-A3ctd90Ce14vD=8Euo)ydq=kMdKlChz_ex>Qj6g>#+Z
z`r`{2o%X$Z)*L?N?}dAPb5<pG*Ue3wdh@-e%xzAme;t~0c%Pl$RD5UidRF=DDATO(
zOSlD(xW06K{O2B@%Nn(<*JBe7+s@4`l{j%NZi)TnvPYLXm$+R2P*<0C?mtKB$K%tP
zZe4ACGG(rG@mlk^tS2in!jm49?fuWKG122wuuk%Bv)iV(4!<a$S-*T@mAobMlKM+)
zn&kB@P1nxZF7CaHNo{57ZShsV3unIL-?VpW)uX(<yF&BgzO|^#V|%tUl;h6^@5NO=
z&Oh%6|Mi*kh4=p~`#tv7jLW~BzWk-z_R+I_9@A18CeF$+W{tY6&f#@{cjIy`#vO~d
zdYmeAoO1cV+~w-C#IogO?zSk#d%v@KsQmKV^`bSFeHjs7+`mk@_u2mA&YMLR4*U3K
zf4G~xN_bXy#<|mXt}j1vq|D>}`hP!OG=7|ZPwDHG`l%5IM1OG~_}FrOz23>Kds}}*
zmXyl~op~&#J2ftG<DuB5)CIlYbw3sC>CE`GH};m{zi*uj1I@3k4DWbQ_vfrg+(r@J
za@Bivb7uI9xA=5CZhxW?RrUG*iFK>l`HJe#=g3^KiCvzqS+w`?T`lgX-s+c6z877l
zY`pZz%B>|Wm+p$|@u-J2-FcG4SFuc8ro1y#Cb%g1?Do>-ovta|^LE@(X`jp6l&8FZ
z?<W&|{`m8%OH@98NoN)>jc4#$w>0)v=%oO$f(xzfRo07V{k)VbKI7C3_4f?rVJrcc
z{-z2}uz$G0e(kC2?7g?|U$-ode)LlR;inzJg~77_EH2w|{EiQQd}Ws0f;qLD1aDch
z?)%4|R=!eJaov|$iD}E-R*MT-*_}4AQS_^*z0F{GC;NT5Vpr&!79GE`(pra*ZE|)l
z+gTEo?-*9sF0Vc+J@JRQyWUIodZysm&&P!)mLIrUlk<*^|K|h?*DJO{6a8u?eHY$b
z{q4%{*DA~Y@7nkB&787%FH%=O_M7zQ$HUU|n>QR^@`LYm86U%hyX>l;Y8*DP*FE4p
z?z(b+_41QK8X7yUi)Q>xQIdDhetJHy``(Yqb{cQ>oS3ICcm4ZEdLh?bhp5BlJhM*B
z-xwCodFD^S!RNMiuYUjgQ5g6CsqhEYoiEh>K4@eNxU7+TmEq3yW7hR0(Pk&}AIuV5
zboFO|_@djF9<Eui@T}O|qWG;W$xm_~&&gi;>i;o|#o6oQt{Tat|4ZPK*EWd%o?UzI
zRmF$6mSq95_kMm`u=9QdTbA2aqkzs!cM1|OTVGT<s-(c(8}vjvew(n2KxU54@?Q@X
z-><*pIjhWefrQIbk6Gr&^Gbf$=&QLNIq~<twp2;=mInE<$9L>D@A^K`YiF;E)Ajg*
zkbV2VzwL|o^4;yH-NTYy%pB|X=kp65uF+&a?ozF3=;KoRudwEs{#*x}7yJP%i(0<Q
z|25fPf2YW6n!Mku{q`^Ce=zyKP&nYx{x``6GxnGG6}p|af7ki%usXx*|LaX&{&*~G
zpc!{$Qp_T0fxkrsCvNg7o>o<rU*-F2?$ndpU#+Ms+PQw$5i@DEqTaUl#9O{kYmT2a
zovIpiE2hX%tzS6ssg-Q|&&smYrP~?~Ec|#oedFp=4JV&W*}G_x@E;X-^_?v#5j#UZ
zPWEQ_z0}80OK5WObaQFRSM82cXP6BpzuF_)u2w!hx_{NGr`EqVa&>H5a5{ftYqO+-
zCwu4Z6nXvWBCfSv^65LICR~4Tq$`&3h2A2m^v0x?45f9)7BpPkan_}%cFO0~)83xZ
zUBxZ4?7ocVy@08Auj)!_)J19dU5|44wDGcJ`}JfO*{4qWDi{A<D&6vPu6E<z=((l4
zygJR8&;7D?SiMX>`Iq4&VK1?jaw^j%7_6HVHECPJ;fIGdo3xY`s4WwypDQf<yZPnv
z$A9Gv%C+T#Q#QQWeJM!hw@s_|MUm}Ik2sD`Tr*AP@=CooJyRx%=(=uNJoBWrh5(;T
zy_O%>p3){i6^=)|r=C}TJyh_8DQ{!K;|iAP`>m=MnhCeQpXQ@+a=qbE-aqkL^Bh0T
z4ceo1U$gfy)3#!pdq;JiO=Z1zEWP57^yS04vFEM_?_FR2{PWJ<W7e5!&$?f&@mZ9&
zQp`q+y)(Ax`mgHiVlTwZ&p%d4;&P2k^yypau;Fr{!|%$t7WUE@abA{H^PVML)chW&
zP&)6;f#bJhuFu@^-QwCe@5GC_u`dOLtV`|AD?QrWesg!W%KY`eYL9%nw5O2unpO_i
z_pgZza}z%MEWFRbuKy+f3t!FMsh@TqPPlz4JegxkVC&h}Zyg(06chiQ5IHaYRYLCf
ztfr#%AFMd<luEBTWoq@Sd6n11Ya8Z&O^P`rzn?u-@H;2}i;G8Jhc2|JYF+bNq#&wJ
ztHeI{6#LSEC*O|f9q9ad^n3Z5BOU9OKK^-Y%Cwvi_tShOZ#=KaEM2fuWX+6;brHLi
zuLYhx|9htJv!~XR+bdR77_OU`nRfN!uabp{mZH|t#ygfZ%`sJf9_zKcr-olrZm;jD
zFBetS&ioG2nSSrGWd7+cbMvl;CxxF>yPLUBS+D!i#_U+O^N$JVZRIa=+r_`u$|_BG
z75BZ_|C_3v+pX!$)~B^5&(Zz9X^HsrT$g*>wlgTi-sDfen%*<VJm^Q_&4_|kbLy=&
z=H{>e)}P;QX1Yl6gXvY-GhV+o9GT>}tmHvoMUdT01;P7o^pxzM9Cuf}|FPU&(EC7`
z&%;H^uJYkq|4uwAdQ)-#+zm?mbGKA1>b*A8qS1b7L}6{|ljuU_Pp>AO$#8kL>?_ZW
z`4`0wPwMtb|E1}6A%C~6wASURKHG!mJ5A0GyBFiMv9&NrmdB@$BlmJuNVQv{_OFnV
z6l>@H{QSAwcn|*yJsfbe?s=2P6xM}3u^;9)opsfAa_gB=y7}O@NrmU$?Js|`#QN~g
zrx(<vB_8Z~`Kw1Hg6B?zVUPP2+jf=i+tHI9Qr@pvmi5}=Ld50Ht8M#a9_}q+o8fqI
zmqd@;i)%}s#oXh6uxh7V^z~|~J4Frl>*rm0nch8hZADOL;4P~-z022wqWjk`=eJMz
zwzqwqh|RqD!H)IJwI8N7PncJAG~`@5+kr;^S=ZmH=-C|nESK`5zU<+IXCJI*{EG4u
zI(AVYRq@D`imPG^^@A$|c0D+fvsuWX>CW1dGr70BeVvoirSmrT%Hty|8Y^aR-n2^T
ztWkVf=*;)0Lw!rXuBll%=eovPbGzVEznz_qr-}deyV+yDPiec!sg<*nRk!`0QV@5(
zhBxT_KHHq!>09@^bDj&2*Eb2WUw<z3=dZQLr0jxY6>gl}yKjln<BnZk)44yHlrm4e
zv%R){g|y|XZ}v~bE><kLo&1$iiZ3pA!Ojfz9QkK!#Z2<s_t?7UKVN;c^PBwJ>pskz
zr-#KIv){pT=~JS9P)fAy!Cyhru03mR-T&r1J2@s*>}ufEE9YNKJ-B0T=JFK}Gvxym
zdfqkte;IS1zvfJU@=n`$2A^{U_IxL}TfQxy7XJLwQ{`jjEfrm|0i5?2{k<;up*!`r
zOg3A+id-Jka)-U;s@54L;nU;yNbY9ssM^E#_Th}U3@6d&T1OM}ZtvpVZT!59&A7iV
z@_UDDrBJ*H#~zvMoAN)2pPi`pdG6!42R}0Y>tvP>)(KL$eq8m)$It4EA3phc>8!@_
zOOi<~Nt3g?C#TGR?WDJVi{Xdt@@=77=C_4}Z%i`L+Vy!V>k?u2$#rG<#qMh#wiG=n
zk<EG;n0>RK>*Zs<AKcDw>JI&%GskDQ;rg$sTtB?eOm}{I?^q?fNz-Z8lY7?Xw3q+N
zc*!I;;jU8U=HJ)yFYy@N+N=Ef`p@6X^w^J{cb-^d!z|{o_bkhwsTIF1bGD0r+dhY5
z_ou>*^R@5K&z}ALqI*Nun$O#o#79VS)YokbPnlp>6X2Pmub^p^qtbrj<{aUd4xRZ8
z(W_k_FMC}qFl+hd$^LoQX586yySCJP%fIAPckj=QWc1lDxAC92-w!>b%8TxYFV;-2
z$=ny<u~YcP%@<$yJ^9XCR`S@?<JFnE*Lt_#ojxsp{_w}b8JsVlwmv(pRI;fkv$lLm
z$(8Bc$3DN9F6tfg{B+s5w>JuIh`oF}vrd>jzi;lEYBx?z6FyFj-VaV<UpGB(l;;yV
zaj*BYWQ3Q-xu3JH-<;?Edh^*?CmahN*Zy<(756OWy-mZ)S<}uv^^yPT&Su~lP{eW~
zc1B7%=cVd@cYL!SIu%a~(qH{JHlkT&)y-3P=A}P;YQd~hEa7lND{7mH$4OU}n|=r1
z97x#qUgzf1iLx433V(RSyx;DZ@~8H%OGEPfl=TbTXWxGNy??Qgu1eVFOR~EAbydaZ
z{JL~>YMz?ytGCmRubt=>zCGr|o^LjRZ_afe6#Q>+>GX=T{(EY^RQ2y(S{J9#^YjT*
zvG}pmj1ottpL^L;xyPidH*HVos=kTK48IrdIk{TE)$?Q@?>-y;HFj(f`g~_g4!sG|
zDcmb?<6ZgEGV5^O!rHZKDt6nRQq(b&z5jOp%RQz&oBkFlF8Z-Gt86x({(k!xsitQ>
z6>Yb$l0LqF{vOuMKK|er?}Hr-JJ~-@yM1lpR^=@FldD5(KYg>=WugE5a9MuBTCT-8
z_Cd=;CtUw|&Usdh@PzxN&*e;urXAF@RgU$$W%=;%{B1`~13rDWxo+*7vfgE${rS&(
z7SzUFe*f9)?dv@UO6{HU_LLtn31qL4eD8BC|5=F0hc`8hKV2C+?l2v`qN#fA<Fo!F
zvS0aLsuVDByp;U=vXt@E&87Dq-p;d_XSKIH^;vb?1XIh|671DR*WCB{sIRxLmh^bs
zyuf}z&VHqh1y|48{CpmrRhz%GSKW#y>OgkKs}sGt$A9lXIiaYsJ2h5r#gpm6&p#_K
zP?NhKP`gh2PPqZA(5ypC*Lv*StUBxEzYCqMZ+bbTgKj%}yj7Dp-fH6e<7vCC+orSL
zTIR2A<}Z1E`_Gc8cW&?2*7$x|WB&7y`>6*Omdj~`>)PIb_~^D|Y|>o2^v8>eWLf_7
zS$!<6JHq+K@BY3OUGnkwl@y+wn}4!v^W@&5O?LAy+KK4y`fV7`boS@A8^=9gv`iFm
zU-&WKIOJY@s+HUJgIDeD@yUMT-e+>_(rKZEtN$r3G3AQfVxqp`W0}&e=xsu267NpS
zGI;#AazE^QuGluPq277Nit_rYy9{?suQl1fP;ScY>tBnj=K0Nd-D|jTYqNtAPt24%
z@45SK$?SNu`A@~CqmMt&p1k7Dzd1)8cU|g{xO()q-Fp7(`8_Y2XMbR>+<X5O%Yxki
D5Bj5d

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.af b/third_party/icu/data/icu_conversion_data.c.gz.af
new file mode 100644
index 0000000000000000000000000000000000000000..cfbeb165ad3428555276a463a90a1ed2e34740f0
GIT binary patch
literal 177696
zcmcCNa%11BK6&?FA6X8_#c#Oz?3bUS$KOAdTi$Em_ix$xJLBnZvp?ScZFzE6HrN@@
zw4ZOH*{2_Bu=?M>y?-yREoewjFXo9>p6ctQ-%>hRu;SGIo8Oo2d8_v2`1LRSKjr5B
zvGA{I)ct)!|IE_3$k!F(D*4KCCue@Tm;U?m$Mx>}b5?Dho2qcWew%S!@v^Db;{U`p
z`isrotm1L-<+o2$Z-3e=;duLFvd_cUx0Pj&%=&y?@OjCc&3DYi^Zv_pa2$CX*;~5q
z^Pc@#PMai`i;JJV&H3btrQ@}8-M71wrhIgBIDTiT>0X!RCqDOZT-daGrwzYDjnMy1
zzBkJ%KIFe&)Wo{zXYjgda|=pDi_??O7JIFfnYVlKk6l}Cn`maAj?n+gygw<^{zE_z
zfBeL_-I>qsREgxM{yzWJ;A23WneWTm_@L6X`M2BU(wDgZW}Wy>gu&w5wdLRM`gum1
zY}?{^NaR|r<>#{BzthTP{>y%uckb!kw!PnK=YH&|xx{POl;Ixr-lgrmrS)>YZ`Cm~
zZvUR*@w$4s$0e=&tp~rGstJDjRbE;DH{^JHxazi)zdqXLx8(OkHZd*gEKBCD5w9_{
zwGS(tIPtqzN<pK0&g%T?OItpET4L&Ph5z!tJ+HQ#u6>?mG4o?;9Ishz-M@s=iF`8+
zulBq?U3l-lgu3+lt@ATju2{r>y09X*Zmk7>*6!cca_c|W*0O0$uV>D7(mgJo^5FNQ
z?bfV<z5E^Dnyyxa%&&jopu)i-^7`AK8K(Ttu7+ONf8~yH(|%Q+x#c@%$jy8HgGZy_
zO;`i3j`1<^n7-f4d&55&AKzK~r!l6L-@n)FLsasghl%_4$*!xoUj5@Aw_k@@%&&zW
zVmE)YJIFBpKczkA-u0iKb1#bS+uQl_`c}Kc7XN-0Wo&WvJi?z{R<GJPtJ-?vvC#E9
z4g6Rm%I9BV53l)g=zVjI#m4)WX09lWUwCl)cFjGuJf7-p;=COFDXx3n?B1lVyE0wU
z)9li$=^syp*?+2-u$uivh4p3KYvL1DPVxW$_Vxb@dznt2^p=>*J;C&5);&vAoyO|c
zmf(_gvR!_Q9IfgNtU0~vGyH$7{W61Rq5jRV6}1nSYv=!dsVi6a#aimCZ8h(Ox%wy7
zqwS`@{O4R*&B=bzMeE?&1E~S)+q9}C=By~4Z*bU7<$Uv(x^FQm3o5UFc*zucvh44(
zy-gkKGE<`)Vs`yg;8%}7bNdU2^4|&H-c5dGY;fir&*QH>OV*b!&omQBwN9M0aD}&^
zo0fc-%H4&vcEUctcv{NNe7$15NFlLi&x0>9*0atO{wpb}{ymZH<NRkkpBh(u%XGP;
zv|jV*^t%&2{}!sP^^Pf2PkH6`KeXnCy&dDemNhlau{-Qj|91O-JFf0h{rq<M)wT{b
z%WcY;JZs-4CG}Mn^#6Zg_4l!#gY*0ie-4=Hhs94i8hvP;%I@86jk|JZrf&9(y!$b}
zTCVoE_xJMVZqF<0{N^t^^Hz87zQ7w*)qGoCaMWBm{rpeern(d9sZ76gIm@=sYWgAn
z?XY%CdgZ=Ss|8P<zhBVIl>W;(FkbQGA9-h)|1*U*l>HM~C$jWSb;7IvUoQXIDy$^?
zBwm`;!ZAHMcYk5Zt8KeyeK@%PSAXR6{k2T*e#*0)t51}x;Qq(;@16Zu7MuKdKc%mq
z-<Q1o$v@xW(w57;-?l8?H?Of($U<?`s@<0-mDyjat6ToHF?xaY($dYF6R)#HTTV_~
zcJ)Ef9D82ldWHQ;$!ZLacVpHx&u@6vn9Hx@@o6r%hiu8-6WjLa{k(sDos8-l@8-1^
zDo^m3hTg6>$?KiQ$LQCXZX+LaVBXCn_nHUdB01VipNl--lPtc-&;O2XW}E%hZKn%_
zI}K}1@?1_GXQ-L4q%eEVr1^Ud914uN#Uqxy^f*w_#Pwp)+rRH#s4J)^%r@UIC!X)`
zU%s<q^$b3)waX9Wb8LBAl6haMV%==*J(pP77R#M{Q8)kOj?$TBLQC5Gg}oKFKA704
z<oZdzN5(ObUtC~~_Ic)oN4}<96csnfS-;PsYR~JltxGsxOn1=bERkbl{nhi$`?4w1
z#4F*iK1vCS-uC>z?Dq-x=Z`l2$@%!}k>RzOZ>Ar3@!-b!o0>27u6?|Bvw_~98_zaw
zuz6j!Z?4E}wWp_F`TOVYsyJl0OvBy$>c=~AHe#6*e{49hGU-6!&F^iyI^8`rPR7-G
z%dK4@8*(l)*^O`8)0&$;4{!e#o^H54b&u=j1lN0~zXm;XW_j@F{7Ll(Kl|5+xIbl&
z@Y=QZPr{_lvI~C6&tCb$th**U?oxh;;3YBpsf^cNah#HL{Won}$;WMX=1*8~I6L%}
zVCUQ8{S$Bgek`@=gTDI5tbO7cb+yv}!{;>ENIa8vxzm3u#He%D<oe#y$!`P-J%de3
z4v4y(5xc|n`QHjz`(5jIo-Ahg(~&Wo`G8~z<M%gO>n4Anc2#70bE(_YDU+2}-CCXd
zKv!$k?99eTzfJ`3KXAFUUiPmyqu<jwsVmbb-(tL8x;IU)T}P$-x{#*1{PNz%IoX!|
z$qEiT7CqD`brWfCkLW(?>t?r;afwu{n)~&Y98&Yo*4)`_)o|Qt;s>t$gO$5)?p%2B
zk|49nTlRy36)Gp2HrpLM%au`Z*!YV0cGcj$3)Wo|-x3=3VORUf=WdQ4vOS-<bKS~0
z-!E}po>M0O{Z{MFyRRR6PxmlkdNh50LDd=g_?S0eWtYBq&awFR!Mzq*uiVQ&$e)^0
zb!{q>#qBi_J%^VT$2d4>1_<1G`uSz8<Hb#C!SOEARg*H6@0i}M{P_28{^3Zc>wF9r
zO$86!SJvOVXLJ5e?hRGb_rZz^NprSTrO5`of8m;W`{dzsnub9PYwu1oHn`uEz%;eV
z`mQVA)oJS&U$0!1@O6t4hos4N?loaKu|nO)_RKv$@zm$VKlZIX6Z2l(aev!AYb(|7
z(g&S$<eLIr*#0@}Jy3X0Mvr6t@2wu{?UTOhxu4&e_t4r>u%__c@$~i8_tuqMy6<-6
z|DRn#J8hKb{$Hl?MEokp{+eR04nr%xrd`L6oydzhW;#o+K;TmI^}W-dtH19pwV#|?
zxqQ#<&LwmHGO&Jp%ic8QdfT?wb=n2~F~^M#IdNUmnZ9<)xn+A)idy7<p7wiwTzu)7
ztj~{kd~)?vF;%HbXPEob_`A;O;<fLyJ?zfphwlF0zqkH&S<cmqYcE#Yd+*Qa`22hS
z4ZaZVZOX@Hw4bSedMYM0eP;2D7iL0h8-EBm34Oep^7pv@XMgLtw)+H2@24lt%{_VD
zXNG;fc>ZT;_d9F;ZMOXL|04gA=)eEFANOCWIrnet{>dyW{kOdjS#7LqE0I^wa^TZq
zr*K2fnf340npd8f8u@gc{=u(7d#3l?*;(wRukd6}V^N)@Vvbm6)}rN2{)%zC9v5G7
zQ<|1lG<h{^t}#pa;~$Y9JZ>Eep711p(&u;UzP-3+Bf4o}n8l$@tJZ8=y6E4|ODy>e
z!euR$mR)%(&!4>>lz%AR^v<64z11=;H(4IvRd1J2Q>kJ2XVL$&#q5t7LxtSzuy3`^
z_n({Xzpu4_^2cjGqTh18|Fr)1s~wx3_iG2vpI!W|cq8WpE1%Yx%|Ap<wy$sf^gC#u
zpWA}^CyRP2-?N>veWz1Zb8c7B@r|Z4GCh7;<<>1+xx;XW__^)pTi;nGIRAhC^6kmg
ze|$TaMD5mma_zjeZjeCE=XbhHS?h1V7j2f-ta_ie>txw;@x<SMuQ07|w)f-WmNbuh
zvS~(I#b0B&C*RL9@BOx7nQdt2Rrc#=PT6FYu0C4z({#agt+4ui!et+4y#AZdw*7T|
z<Q7M%q#u4qW@Np%+P6LVxZC^1hxyKCezetJ*1zt9n9ebMBbAjJyF&jMUuc&6Wo~Xz
zx9i&U_h)Zj|MxNQ^wy?Lw)>bRD#G_@yIxh;X=iL?7ksF*RM@QEN#mw*{rmF@zka)~
zoIkVk|C!QKzVw6}p@}TN<qH?z2%0Z#{OkACN%dS0^6$Dgyu4OW;y>?o{@f2rDqq*f
zyy)!D{G89;_8=)Z`rqo)DI7ZcqVq2Z@c(_DZ7*le@GU>CXThEKazB2jmpdEY%lA?^
z_BCy#{d|)r_m8u`J^k(8vFrPv|GqE(?(feTJ-)hsWZ9>vGk)$5w#f0A{`%jR2Py9A
z(!2Oy$9>uNXsflv-kLM}*_*3sjg|aecP@(yKUdWI{{GDO)z^OS@z1uuulIM3Tdsfc
zx;GbIO~3na^060xAMr1fVab*LH)Bpf#DgvOW9;V!*FQfu-K3wl<MN94OC}$ZkIc3>
z{%i8<&&9zEg>%pTvsLEQI{3Bk&yKkXoBm7JJY;y$pA#^1&6j+peJ>T)o9fgbkbZTx
zbX$CV<+uONS?ag`-Ishff4`mZfn1A}pXv`jmH$sp-QBlNPT}9>^urO~^tEble7~=i
z{N|_FqvKDO{hhUYm+)rpTP|Ub7hnGJFmqAB?+pI>>2IcQxT`NMr*`!H>$M?z&p(H`
zPC38(sNS@!mw%-SZ=Wx(_!qsRaryJL0ZF{2Mt)IGcG$l?d%fy!#hfoU7e2bx-_{uQ
z^kY#%e)UA-^Rma}r30B{cNH?lCO@;^VI!UYD!b=*uJGSn?T?1bWsUwDG(Xt--tv{H
z<=<ZCKlw^c#W~MY|IhRk+8p=&#54ID6)S%H-pP6IuZ?x;&(%{u+6vyX75_cE^y59R
zANr~P^n-pd#LLf))YR$=cRQ0Ewx#9VLY|G%0&nN<XZUVk%M!Nif5XNf|L=0;*#28@
zIW0l%-~00GiRJ(6-&<}d`Tw_6?7w*2_y3(--wR&t|84p2v8aXOPyOA+@Bat0_SO75
z`0K0vOUM8G;Sct_w6{>Z&hf+kUG6{ium^MA{y+Nj)aCyDQ~UofdajVUldID7YE102
zcRE)Ew9fxot#;byPl!(E1=arL&o-t^_!+#O|B3IU1gU%OpMM(PFD+QN`?~Jl-UX+-
zwnf=|(cD#h?pa!=2($h-Ugz$AE<dBH<U{uIzrLjHZ~VFP&AHq#m+SV<@40xt#{F@z
z)cJkA+P}yB=e_I9g;me9Z~n_=pDAn-EOjrYqx7Z2rgeu;TvAMVDNuAb%!+N+pSpeX
zIr=ACm-zq9IWX__#)7pQuUGBdQ_;`;@?N;@L!L#=8CSXA&nww;*rXtO$)6vOW-qL-
z{k#3X{WFQ})oJRV%kJ+yx%FzlgGqHo(hSv&iJwDKDz!G1tU2}FaJJKK+rGq^lLd0k
zz9z@C?YPcg_?1KBf8Y1aM6Q7Aswo+#<y%%)gx?EW^v-h0y!Ob#2tP@sV=+!UYMYWy
zeeJToq<(i^b!X%~bq<wlJ>GBHcE1d@n$5E7{kPqTzdm)Y{pF<Ad_d~h$`fW&Tt9D{
z&iDCuojqgfL`Q9V^X!kMXBQo^{qmu6n(qC{8#Ctp(y?CV_kHoVFONJo$)CC5nlf>2
z?0xmv>yl>o=15Mh?p~O-<J$e-x|>Yw*o?3KwOoAD?%hA0XrmWtcRZG_d>(iF>67)*
zPxdU=E!U{s`aDW6Z@KNqyti%7f*$Tlxo&RUaOE?L%DcG872JJ!Cqv49%zJH7%fI)a
zRr#82*Ph*u;WE(DsQy#)ZhdxmK+BV5-hMB`cKBWV{8{Aaa*f|Yza;|grat2866b8=
zSzO2R#P(R>(zH4&t&L}<CG9^t?bElCuX|U;+)qr~7Iyr%sh!xoO#k!MmtV{k`FDNZ
z_v}6M-~BZIVti@;zwd3!{`?j#U<f{PSo`K)yMKSq1{7~9;;)D_m)B!`XmLJpx4G`!
zlIsf>|6MHYBgg5dTmN^~{r5kA{iwVzQTpEfRP&C`poy0k?4AGW=$5BBf8!#Zg;wNc
zAHHUjQ2DHV#$_vI-8`?^27w_NcfX{SdG%#m%6_(K-NA4B;?(ntA8hv9&HwzoHahqr
zXWV;Ek;}m?vvcl;cdief7-Z+(WGQ~H<^L(a@cH{goG$-bbz<d=K;`ds-LEUAWzYS$
z>nV3+{qq;y*QDPB+*_Mgwkh7>)2?I29Cpt&uDf|{>2SaJJ1*jKOx5z@T{U4o`}Ujj
zPQTxi^CSDOAZzXVSDqjK@7wZZyVu8XZoBqX@#QT-$t^yqtvy!%ow}+XchBo*s%m{-
z+wK=zFFAXC<8P}?S>oCGH#+KymzDoJpLk%e*0g|**{fDfy+7}zT<d3vkCiI=i|Yzs
zd}V+B?Rn|!9y4+G%J@V1p4Ojq9_AN4`?vW2!<Ntbo4)*Tdb?fxkpIKiRZA}3Z!dl4
zc`D!9I3_t?iuZGPTzkX!rTaOR;~&RwQ=fWSxk_}dd;8bhzV}aabN{o=+PU40Kfn36
z9oL<|+uc>XPww{!{U5#lf&OoOo~ASApI@gleEs)xw?Z19&A*<ha<9Lq@8RwaciPXN
z{qNnDo`-v{`z?*{6HM8^_iMW8GmTV>;8mO-ZWpNjyv(@a@0}AyMZf?0t}(Cs6Tf!q
z$K7-OeB1Q7^1M*nPy4JR4?jHhe)a7AH^#h~Io~JE`q8fEr0;jM&CjjASjh6fcg};1
z$M2f=#k<_!T%IbUqIdD%Putt~>*jwdtay{h{q^t9T#b)1{~lNT%5(j<R{Rl1)$0#m
zF0T3i$os?Z6@R2xno6v#pSI`6PotS<?9bnL-aGwvzMf3;-<icvJLm83cKosU(x10I
zQ$_#Hf3kj2&>roHp+D^0PQL$d5wqL=TmAa#fAd#12=Dpz<g?to>d>0s54Be3S!$I3
zUVHfegq`QBPHuYs@!|aGO@I8~^!<OtHRDIj<mr0LpYZhe8@@coy+7Y_@9b@d>aD)*
zy*y(Hf5dft^K0jM^}N|$iUxey!<n1%w%ul<=+z?q2h$r~&({C^IJ@I?W1NNg{Kp4A
zelK;7%umVtx8wVC>x1D*cZwDlp4xgNMqF<4w!WWve{Xs1WUc;bt=1r(n0c@H;-Rgd
zU#+^q8L5<i&2Fas|K_90^84!kID{zPxRdN8J?lb@ditN5=h>BsYp!0KG4=NO`JAyk
z6jo>@|JORcT-x+;=|4MxZ~J%O=Sck()w3h};eOqXT+yfhZ%Wov+IHvI|HY<1;){0q
zUHbX{V`==tV~)a=ytz?FUIZUIzjr~qiHqCh^3a)yO-|QD@2qW{em;!b{@p^$Pi_&b
z*pKF4Z>X8$eWNAoO~?7Cn>!mM%1?BETYEj}i6O_m)lC7jw!gQud$=g4v@Pnh?FG&F
zwTT{=KmL}==XrDc`tD!1(%ZhxoE`o?&f4b8%+)KOoX`EUUxdHl>uRCxY=7p9ZTZhB
zdTG|YW0PJx`nVWu+tqv8&`|&DTCGh+1y*fKO!=AC)$--uo{_TV*a^$TBRbRXmYztk
z(D(T@Cvb27%q^x@s<%|s{$F>{zAn!;uGJx*>Flf8^Uo%)P`mQ$cHETQnvV{f?>%m=
zE-3BYdtdgr@}@`6Jzvy?Cur(OHOQCp>nfPLM(s)pzkgXeapn&z;jr&ZzDk{#)gEc*
zxOwu8uYbCG7+YRkFs|QzL4fDPR*RJjgoX4Q-hK+aRQ{*6QPc76!s?h0v7Ft9?QfmW
zk<VSfzi8PZ_76AD{`ZxN4w+WJr+ByFbt%99b^OJ@f9Rk0c=K{=>iYHd-|v4n{$KL!
zp<12h|GK097e~%X{QoTX|E{^edks#-&+nGFQh)JDyyeyXkFWd_o-@yW&asc*|C!H;
z*)i?qyuW{aU#PxF*>Ka>{PH_qu^FrH%l_Bk_#u|L|NrCE4}3ds2Z>2VRsEar=Hu_G
zANth~|IgRH6Fl+z{quh!&Uq|cXncJ8+;<0UI_%7@oIiHf`FCzow!HB5J$<tAadtkB
z9$wj9+4}C_vb7bwFXQhuN~W@LA3nV<?VyaLU-!c9GU@!(=W-5$e|zVCd^W#2xn|n;
zyQ|h->6e#qy!idTS-|n@zeA%}ZVUQ;eAUdSXXh(^ewJ)5KRdzs{`Q-Pe%sI7e!u<s
z_s(0PpJwTnN59*4=dtkvfgc>P(XR>~@foM^h)=orZqt|FUWcxg?b<pwFZFBCtY?AV
zhh8$7pMKA?OxIUAgM0a(i1Piqp`X`ZTlxRu=C|7$GoOgne-!%?J#X4sZTb1xrW<zp
zP5e^jw8Zn1l=%AHCrowXPT$)azkBhDIpr1aLe~n|9sDz=e(Lefyr0+W%<D?Y^}YJ3
z=zq{z%eSfSE!)19nrmB~4UL~S$LIZ@=(B(S8F_5onSCnkyL!>1zj^1chTQq`LqtP;
z9@m@I@iyo4`%jiX{VSV&;j!L_y#|sj|4dH*b2Yg4|KDN33GR<_LJKO+%>D8qBc}QG
zyKCQGIfe7?I#IcGj{ZcYFKaj!Z;N;qfA8)5Q}WG~{!jNC_Z^QHwcA&Ij;TBTdCk^8
zx(BbXTt2zdw!Zq%M}9sd`%^qSe))f~+_vKX%@X$TfV$NzKJ~KY*ZhTbRPy%zpLn*p
zdH?>vIhL>fJ8#Zp(+M{+Y+jSkTKc!9{msVRKaK11{uOQsjg@^><NSWN{q5rRPu>@9
zzdwB9*XNzie-<y>|Gx5P^^3i)FYRAh`@HT$-{zSTMZ3$t?)m@bq}b*k@1|~<#dDxl
z@Am%eT{~p{2=3bY;ZuO*tp8UYeNbw?w^#DDyfpuRH<s)2bNBx*mETl%Kk()J$j;;y
zwYOx~{`0?cA@arl#pyOF`=3<&es7fV|M#4Kv$gNMK7X?L=WWa7k7QW>Z<_P!)djn{
zi#vF>%q~BoQ>>Z)^?BTzFV8Rj*>7oosr>7*KNqBz|9g4z&|0Ci(BJRPBbLuhI&!`K
z>ZZuD(}|Y4h4;?pZ2KV?Q_m#n2ZrbVee?8UdpUE$^z(Ih8Uy2PZ@s*F$wg!BUS>D{
zpGjYCO`o@0=d1j?s586RePge@3uW6|Yxu(Gg^__{<g}xgRAlEq(KIS(l{+Ew^p8eY
za>Wf>o0CTC-$*R`Dj(H;-*QsEj^%ExQ=dNXOEN#r)ok<cy#B7I8>S?lFj%*G3(so*
z38!x+94-kK3Ou#p)P_?5=fAzK{=3V|@Ob6-Jw3icOL?N}tu<@R<PXm=b3Wc__@?x3
ze%rPmI;l5i&+Rz-srA0aGnP}CnGy%?y?5T0ow&qh&E>`O^=B<@JaB$P$^0)f4*sfD
ziAcSmQsXYrzrOcs$cAO>EozUx)b`_x`gutG<#jph)~TtUXN;Scs8+7g_ITaweA7}b
zS&l82C-&yCqKfH<v!inMMc7|2pJ)5<tEQIy{Kw~>?$&QFpK0&@eDUs?OQs}FPMn@N
zJ&|9s#Ij7@R?usL<dd+3@T%DPr_S3}*gkz-ynpt+e`hUTzIML)Zd+Mo-l=~l7M$K#
zz2tW4joJPI`yRY|*tyKzJ9z7S{kErDVy4ah^mjf_`iX>7g=VK`SA@3(gQ3;8{qG(c
zU$pL5UzO*3wyn&#AmCKN+S3kwx83g;-7&iJ^)#mlx9QUf2d6#$CiwlDzxUy#l{bsN
zz1&~AnNx+iFYv~$KtJ!zyX=j0t_x{7Ul!W)VP;x++2+pJ$(l=f?q@zql_*m>b?o!!
zo%f>K6c0~GJYkR`@!DuY@>jvtJga%~=WW(Hd}}}7rL7(xTl*tYZ=~GVSIcuJ*4O;<
z$pbxS=SakH^F`?_<=M@%o9Fl2BYMZ*S@eI}dPF>RzsB!1{ZoE0Jy5di-t<1C?sWCf
zU;b^uZJSRfoGR>_pFM5Kl!sFmdWs4BUGzep&AackdyEkb-s$H#v%TpJ*pN7GgYHf1
z`A_O})qQNq%wj%yiAAq}oudENoN0G2`MG?{@0h^3VHx|aVEadrQ)(Y=)_CiZvR~u3
z>rM02T`N;I@kqAu9QJeh*1aiyN^Sq0^uTZH+s~gp8Y1#H>D;wHhuqf~fnbgG#1k9V
zop$Iu>z-p&^ZWXV4JS98+;G>gCR%^J@oKQ=z#g1>WZ{f`S8r!!eu+P|Z`Y@g@OgSC
zKEFDZ^vC_)<Jb0*=cN5K4qK)eJYDf$V3Yi*eVuof*>M}U{0;bW_)oF%l)nMrO8?AX
z_(uB6l%+i7Ck|||NU;YAI^X0^wQs3S4oQ13@w|9q=A5~TQ%?NrE#gf4ze_>AdA5@0
z<Kl^l6B8#s{2j+9DP}ErlRx!-jQ4Vo-<;bN4{s2jSbNuJ*+-MhvS)Zw?=OC(lN#~X
zB_*DHw@zvV6yC^FH?uU8|5zKkhG#Yy-hO*T?f5(8GD#5prhQ9kv-K|VP5BQ#tvOW}
z`fc^5^+HywH`H&P^+jNRME%o$|I`m_g5gd*W=p-dIo@sFZSVig0L7UzB>K!fzDvDJ
z*|9!(5l{JKaBAGnQ(Cw?aQ}&Yj<K7yeqH4~zUI`vkJ~RBoX#}5m>G61vi0w(7vV`&
zKb-6jDIJDF=lA*-ZI^2O4!yY^;*R*%THDo=98Vb59N7`L5gc7>jMkJXZL0svxO!9m
zmJJWL&e4TLS58rmUDD)-)>|IP`GG|4`@Jc8W0y2NasQ&VW}mhtX)M+J^>)?ocW%qA
zx4nFqx+}dR`_n6){EqzayZQ0|PZpelLR+JVqCE$HfxHO9@4IWSr79eMtDGm9Ct0<q
z{f+b_>)DI$uiI6B*SgnrhVRXM`}e;&=WY1X@nNIz#J|hln5XX7{Ox<w{K$oFK8?3W
z&Wn1T4Qg|R!bEkY-@Z5VPt^r~o4x7&l)q|k%w6}#md3}g1IJeWlN~&(c}iPD@0uI^
zlTCUz@!pltDznu6oWFf9=7-<ipJ-~{6*2k76%{knPG8@o;K>n2FN)su6`%UI^+svx
ze(m3}H?F@j{T=$<)a=Z<H}~(pzWQW+RI%c}uIl}-_7!FYUq5qm>fd#5?$61Jz45-Q
zZvU%&$7cmEzVqf(UHG@%o9?&%UH9Vq>UZU-_j@=tYyXbD`TkU$_&3uHhZ*`#)P;Zh
zx?zs_&zSIUXG6^MFWs-Z`gh%%@AIV(wY@1%{h#*0xcS5zkdbxa-%5{s|FvN@i_vLN
zf%3*BW&i&@!k+CspyWCs@x+E(ohSA+N=AZ<eNfSVK}K%X?|I7UnupcEu$*h}?M`2C
zc6?(5h1-R#&6sW4cYAv7_Vn2O)9p9Eow(Hc%_$ozYkMwUUuLt<r{TIxS#LnObDg>R
zYqyJcQ(yZ%{Q5P@XpYewBNNf1-N6%|W}f)9vpl>|s>qnPBIe?>cE!`q6H*k8mllhi
zC^!v;<&tk+*S=BClf1|By5Bv^e`C-5*{RBs1+(_soQ<FN<h9P+r<?w${9Uj`*m8~e
zR-b7`IxtwKr02VALgumwCxbvm=|aBi7P`7cRv$nGCoCgIymj&Gn67?Atv2)6U6Z+@
zvpC~Hs50)B@1I9UBd#;PluHS^_dn%E>WzH$GRrdgjqw-D<mYbMKehIJ?7NSiyJr@z
zzxjJs!}^C`r53(x`;+IlZrL=Rd#A6@Jq7mY98ivQh2%)1FHB4+_1eF0KYp|JMLz$+
z*^z;praYVmhW}g|PdW5S_vQPqUmnqx`uFFZ8L3{A%ACb!ar*BuZVYNuJLdT9`}yD3
zm@a><Sp6sL+gYtkC11@wjhFH`w}H}d8h>UQKg6SnCk)=Wq}(sqxa|ASy%7s%rYx9g
zv1F#jl3dsS_wL`wYnt%dddt+mYu~(<es{L!`yvYo6U%KsCw|{%|GyMc`rPjPdurRZ
z#SiVOuB_jDtU9HFf8#sP^Ubp#*yj9@h+8lCD(}SIcY92?KHpm<=zsBR(F+;oJ;4H{
z{D(JBc{pid_?Er*R!*DNzas?--+wHdc(Ct#rFm-g^Snh6AGaxL8t#sZjqP8zPI$WH
zR*l_D?@dUXrBv;4-E(y@kAo(({7F(@_NM&r#=N%A=L{XDLF&UdN2dH;*B|&?q5&MO
ziBl7|3I~6?|GoW1$Yo2m1yd5ICQeP<JNL#q^UnSoDIoYlH(GxBrQ4nPYrYHK|9^7B
z=?$kh#9urX<MnMu+?2o7`|A2nzCXD3`I`5W_f_u+E77i%hIsP*yC+#$bC?3oop$IG
z?mJNz2&%X0@^~Z`#f$uXWcqwBsAAJU1r|TA$&fk~TFh?e+0NtJ(EdW)b-%>gtKhoM
zIkOt1`@H6Xz1|FLu6;`*PHnie!(4LT)=0@|m69hm{^)@CE=9tK$6D|v)YSMXe;2(e
z=K8SDNL{i*T4fWMKYhM`+RpAXMmgJsXUoh6`|(HF;%N_~_wJngW7_ZE^FfurXl_48
z+p|lJFJFB9sJl~7>3U}3zdM)To}ajhz3lzwwt&0!(hKiA+F!1|@J;(B`9pOYzZUL#
zU#n+cGBK#lyY1e?&KK>A<X8Q^mp=I^B!z;=cSbjgOg`Iel0Q|a@q6K$*Pb_>FY;gg
zD+l4WgTy(0FMQGds*XR^e#+n7#oQoD;2VajKX)twUF<`?SA&cJ>(ltH_@;f)`>hc*
z50%3tudVKdRr;V><dJoWLz{0~_})tAzu;Dl@;B{Ua@~9XZZes^@9xSQa{KN2tgg>Y
z{abTK<K4thw(+~ud9S~V3jKcH`qQd6;i>zzet&%~8@)4SM;FgDndyk6nX+RW&o-Iu
z{X0@$=*FD@RkR!8MgOkam48G1q;Raqdsnr7**_tt(=yeSkgC9`f3@C(yY650ZvR?Q
zaK-p{r+J?8@6a3TyXr#kz6T3|>g;&9!tYR5fTcvgtzHx#`n?*e1T6KhX5FqGb8kHP
z@omlC-4T-euFmHE8vZ6<{hRh}IoAEPg0@N$v%K54E#96l^)GsTzwu-}m;Cas>TNaW
zZ@!y(c<cT$Ngqo;+e6EE_ImS!YBS+);41Nc-Mha<-<|kUz%^k0p}OENw?n^!D>_)c
zcq6~7Zucul^;dV#(_voP%GkGt%U;f$-sslO-gg2b1F9IeZ(rzF3NZ*;UAFvP_Tv9p
z+cp2Et=D~YTjccpcb*r^pS>;Co^|a}zSQmZbtg+}mb~0`pjPJmI^lDr9y{7Me_!${
z>aF&rBjRy4%~CB=p+&?hShU!c9|$~EaJoQTvLaQ;uS;%4->+if|BsfusC@OornF+7
zmDZ(}{oAbj-A&(Y6qaB9{Ic%)`tN&qOfK8aTGPeT4TE!Mw0&iq6m;rC+&>W`P{Zu7
z-;u=auQo)N-dJn(CQP<$HQ(;g_Pg5hU%|ZB=bzV|_$MaeB^f4p4N}w9EWa=@aq`2x
z2H&ktZol1`AHBW*$F{~he;<VQtyfXHWyxOg=25b^cvJV1a>ox7O0U%0ovxeR_V~;u
zA$99xXJ-f2@lF5p>E7?}uP^?4@^$CC4?2SOPZhczoZj^L{;L)L_ly4X5B*_USa-hu
z19x5>=VSds_oKUhzCY&w`)6yb?$cv)B#Zz2IeXNh{Z+?HenHRu9aWzmi%d3N^Y8O7
z)iC9wuj(rp&Q)eGNgkicWjudo$(4Gw1HW67{{J=9`5*o8e@JV5aPr)j1r2|v%Pimb
zKilcZdAs}X{)jF9f9{m3y1(AM^Zv_n^OjrepYmtv2X3*&#kntjYJOV&K;7)h{;tp7
zXXCZ@Z@6o5)8)LsZhh9DCBbgqi{|Ywsrz_hm$S*I<p=$n=er4={ge9X^p`o(i{3Ok
ziHrPNkfJ{2O{<f+$ghPL1SiT0mbiean}Uw+Ez69plr||h^>OHO--O8Rc)v(l$y(_}
zN5;GZDuHD#7X&ZfUE??fMH>r<m*UPMD>$z>&D{m0=%J>7Y+z%s)uKy+rq+wn+(SUD
zseLPM3Qp}?QNj{`{Y{4#M9|cF(It@Jg;WS1rf7b_`#;Z~Pha(gdw2h#60zpac{6@p
zKCl1w<)1A-I+LGYSy5N_GWhxQv-;7CE}PryUT<2Q{`c3)oqkJH{$1L6PQNvt%c{`&
z)Jc8aH`6y2m2BMnDxv7~na2HBHhp$}%6|1lnb_)hopbtnAr`r*HedA5zWsCI=R)ry
zv6b<=?)*&sz#Vzn=*7qBMdfuf{uroQO4Ns~FSGkB-uLhJ#Z7f9>z}uu<&T~u{N<0g
z;-9bH?y|2}#QrU*^zU7>cv5Nn{`uG6|NJ%g`t?UQ&cFU%^Y`5Aa=6(0J$1)3S3PGg
z*N^>a`bl=-3y!1fA5Hi9GNrT6r$o)&zwnK)Qkr{u*&nTQ{+5<jlNTFXR{8bqQ%ou=
zNjtv&(RF`i_2UUeJZkJh|Gk;*T$*nh&XC>T^*PwD@8A;&pT3EcA8R~(uAX*0q3n-V
z*0+b}6wk%akWp4Y-WhIVe^IcxPvWMavUSo02&=hIqU6W|Wj5{qKP}adJD9L7GBk?)
z?Qv7$#IYUQEBoXhZ?6+|KUi_WAjGyOf2L+p`uT#o;Qarp*V-JPq&<56!{~q3uQ!$F
z{cp4vv;X;={lB`-ar5O)v+qajFW>!>ct4?k{ZhXE&u8Y&EKWBn|B!Q9Kltg%ce8HB
zPJTD5WctdM^Xp4)E;+3hYZhFVcIuo<nc61f;Ih1FA3qy^GM+ep=BLea|LfykZjXLf
z{^9$n&aE>dx(~0iH=g`_(Y!@lgSK3YH#b|K({S(1+yCcJ-HZA1Y?1k8g%6wmZ9XaU
zZ~LMDVL$G3{onqj>R)T@lRxkOp0$=eq8@zt_v%mQf{R!GcWSd2zFxO)XQyA>eC^Ku
zo0Fpc8r?1b|Myk?GkK0*`@;Y1-;mAs>VMhn1M%;T|8w_8)*iS2UwyZJO8t!J$Mr1|
zf8y7DC_kxEzq;V~{B^%i&W)&g_U8$k+IO-4#&J*oSGQK>`fdHcyz)<eYkf}Z-+uGJ
ze?Q)r{{Q&*arx8h^F`{zens<reLw4m`~Sb<p%wpR`RXgX>mwfieB8a?=J5T$AAG+^
zx7Xh|^7FpEm&fn<((mqT?O0l|zC(4}ao$dQd-FSgPbN3q37DTeZ;#mf?(>2tiVaHF
zBplBw2@1ID`u@$WPx5aTdL^0%_gTKUvFY5;D?8m*&SBQp3D*4{uKV@#yy_|O(J$vk
zZ!eZ9i>jP-Uw_fj@Z(JE&SeXp$LPH-m0P27a^>Yo=U?YEd)bvr*QcHdmRt7r?2ajQ
zulC*e7IW_B<H`2(>g9iPwQXAOuJyd*?XAb30u8>(-kx1z^YtzBztgg1vu7yuojm)$
zZ|$Np^*`zkFVDZZzWv9wAG7oJ>u&p)`N!t%<nzTUPu3nwiPo;qvvpr0BYkjv;Z0jz
z%bgQ##s0<HTjy{qpZL#t^RIX7r=I!!-$LqoE_^M~t$P>JU!OPkf9|5fh=1WNN44gZ
z&YM5`u~Gl&{JOfw9p_%&$}TwZ-}JrR^1S06KmJ``z3>0cd;ed(ukZCg`t#rOs{h&b
z74@?!r}{fTy&Uv&OUSM8r~b)LFGu~{5_T>8sekjwOoM%KZ(g5$bUC3$s^{O=^U1T0
z7R+V&EdN(ze%b5m-PimNf4yArcT4pCxGi;3>*kBEoo@yfZhm_?;rEv7`{K6L_C8#G
z@LT5kTHfT}CO7POxA|B9ea-rN`N4mg8|rVpj@Q3o&-=~4`S0bp|5XS7Wfs&($;|(D
z>^gU!fAizZ4}J(;UN0^)pY8bagC8^V>-IkQV`5>?d(OZ5^J~`U%Mbp{EU3FRJ6_+y
zo_C#p^Xtp^epMa(npsdQ6*K?awd>sL*3bNrSx|fHcD()$d){~c&A%_-`&V`FZ)QQg
z)Sda?zFp^j_x|L6R-^OR-_-BBx6vk^;lq~o_4SGejQ4Twb9fLL|Nmds&+<Dzvwlfe
z?pyfW{LN3c`T1Xdw#~o);W>B1DRHK$Kic-ZpZ<?&;ip!or`&B`HG99zKk)hY{B1vq
zFGtJI{%hHCs{Y&4Nt4f&UWs1#?Dr|vX;nEtyVxH^zMnpW|FtZ0?!V0&A3RVyx4%#R
z%D%614*Q?H<nMp*Gb!xtqW>FLwo9M3`SVEgQu*@8uRk?yPH69)RZ|w$n8ctLzb(P>
zN2sa3?bF-sPaW3R{npUWTk@(P)xx_ZWcP>9=Xs`YT2_5ey7J-TCyufIKF@Leq_ckB
z^5QQ~e$Rio@=@~C_NR}+?@l|O^X1v^`NzB0SOxiSD44T($H7B^MY-SS&s#LN{@<)k
zS1<0;_HfkPHo0oUj4931R~ft8mo8Z{f9n4)4_^N%-nC+hPvMf#OUqB!%dPf$9Qo|<
ztD<||N0%=7edBMs4&Ulijvt>D$hxciudTT{t5$?x+s`uFvwTnA^~wM3xO}(wccuHT
zOe%AKvm@f?`_17m_-<Z~Xg&3^>s^QIp8vHm|E&5CTLy;JM)Y#=z4*8L%dt}-d=L0o
zHT~;WMBV$ZC-XA(YP;yGnv+bS1#7<Fe|o)`cU~%+_to|Pk``+H`0)FEki6Jho3I&O
zf%X55nc0@retEr9?ffP7)#8ET|BM$4FFPM<qorBNu~_EncY9-zB&!sExwe$M?V`OE
zlS&Iag!|w4i<|qIi!7=5|GBhhLih2Xt;w&O1iNJljb1OhxSl=Q=B)m-*NX&J%Nkm~
zF1xY*`7N6>>&x%<iLaJ>X#el?dWqYg?akEx|9XGDe#(P-$v^td|5HEw=V@+MWVUbn
z^fSHY-^}V~A3u70ng93m`H!KMb^rF-{l636TVs0S-p_v@uBQFW+Po#dX?@!1@ZC>W
z-`%v2bKk7K^~)arPThas48mOf^he#}A6Zr)-n7@>y|1rd_B#CX*VRvd{k`;c^~+yZ
zOa5l<tlMAmH)~g|>D_%%dH412?mPZt^~-Npi+<O={C2hEch;`jt#|gv@2WSwyMNZZ
zc-`;sr@xE${l0$L@9@)qe=q$Ve);k0mmjZw`jNZj{|wH@uLF-=U)-}^y|Q<`yJUR7
z(LTA<8rzfEb+)JeR89H$Rbz9VaE+<${#o<lb<h9w{c!cmpRB6dt$X%G)vaB>?0Wd+
z*Q=j?{d?*4>X%<_5ATnPyRW}@|EzoQzVFvh`~Kbg{rY9!!!Q3{{q*nOOTVRK>zb$M
zJ5=wBpZ)je@8$U&mAAgj|21EfKc#YC;<L#6^)>tJezNWSBz@v<ThV=k?^^dA!t0oK
z-JkGI`c>Rh={Kj$Z%obKqWv-2y@omNC)=J+y%nl;uRHQDXy0EDzAte7k$l;Yyfsgy
ze|Y`neOLdwBmaZ;{RiRuF0Q|F-|CSx!)bGdY56QVZ@2gTU^{rJF1lpj{Dpt73Ep>z
z*w?tBMmaHaf9Vl(iD~%;I`=aoeu2#Dcq%P$+FWVczi9WG=A@r&M?dv;rT)1lc>e%Y
z%THde)8=AOEp^R@KlQdlwcJ<OSd$FWVpJc@1af4I4oKI;jeppBpGr%gHn*6TU!t@B
zVDisn$2ZjlyY8F+(7fYM_o8WPou4nBXWv)#^ZcY`*A~x^Y2RCP_ghe+#r(*XX-nG@
zl{?Q%*%rIbd->j~c;nocv#g3c=j=RpHs+yP_U$uMbC%t#SY})C?2%!y>kQ8S#<}Z+
zi(mT~zw{B9ci=rsg}Q=u<Mqy+#aS=sJb2DxslMziXN2FIX~GX?JKPg_qh?^vxPx<x
z@0$t2CbJ*hU^zI+Aj5D0{|oIeUosYYbJs?#YAMOCDqgqcj_<PfDeL@}JxUQuJD4!n
zGG$&(F^6AEsnRlG%`;7H;jIN1uN1w=tp9kVq}oQTx9J?`F5?%+HU4ca{PySDPn)!D
z!5{w3G}ZsVMr1P6FD~06(UQZ4JJb{xiGDd{SRtuANBF(P#AVzse|n@}3b&u188P#@
z^46)w_l^gf+y6{Uf7;-BP}<b~^Yat)vp20hxiR%@_~});v$Ot8$la>);%B!1rhAoh
zYP#mx_va)&Ij<baGtVrtC+DZ>rdK|G#++$FXH;f*8ca@j7`FRU`_%ZiIk}&^_}~9F
z;j+n}zvkm9&Vx7W#HKc_N(~BVwbLpI(tG_gefa{-U)rs;%J-~$k49b))O>0!eo807
z*<Z}+<C@~g18XloO@DjJ{^6~KF9J$tT-u+$bLNs9Z>L{7E=+ID)A<tX$6vJCH)yr5
z(Q04eRlc8A`d(V;yJ@BGq?Nu!D}94j`WmhD6<+E4X}RyE<-VJir!IVudUZ~miib0w
zcD-3wUxe1&4)sUUJfQ|5Ge1>Ny7uOAz}(_~k<`gNsgw7lO#YJW8L{$k0n1@Q9(RSC
z7uPrmpVe%hz`j_h$4lYuj)M99UvBxGe-*Br^WjF`<$T+_YtOs4O&7_S^dfXq-pMLe
z?P+U7KKLw~7^7TY%c`h#lG*&zES;=v;q8n%+RZ+n5+|+Gjg=0IIu(8_cTLa52+0W>
zjSf8$`10h+<Y50t^HP|zOTuJ*Pk&LAI6R^8^UToai}(Ngd1jUIYQKAq_rE{c?v(!J
z^UTojP0?R&H7V?Tr*b2*G(A+ttW4;Q$_;^>>RIdRnid6Jc<R4=!E5<>ZyH?q`jSI5
z7U*BFPGPNTdlxFRr(wcF?VkIQrkfj@1a){GzFsr=r+TR5nL`<3O*6O{y;~D?eGjp*
z9?lUt7`SrtgS8UB4?PffvoC%2S)D0hLPHaWNXh~QW>!UwyXRhhaFj7x^}&<n6qA{A
zW!Qq+E1t3}ixdP|CtAFi!rE{3aIcDs8<$high@>^WJ*powL7`+$*3xBO44z4OyhD2
zo-nKF#+0{fqJ@n9PME_S7HPmQxWJ)-iIIy%<7>;G%P+ZYnM61O6dW3#v`iBA@UwU?
zxM0U;c_%>zORgsxpG$OuuIL(F(G|KfcTty&fKq6ns>=i>N3KWLnF_g{Xb6Cf@;aL`
zF*I*tYR1HZ)QS1Z-8stX<(Bg6d7@T6SMD#XI`{kY#?5@Dn=h?S+3Iw`JItmwL`cG$
z>9OUSlm3<Ju_}@i4JM|5U__=;XT0gk?46Frm2Yb7Ca+P8^xd>e!DH976y1|ilT7bv
zbVdtpSMl3%cJhYLGmqGIRttSsdFOd&vc*HQ{+s90!91G|X0Jz%`n_wcKChl>^K3_T
zXUqzgFuvVhzgp+mPx^k!OI!PG%Er&v=k}D@o$f5Ps#iH!HbY{$x`xDxcEhTQ&o18Q
zn8n>Yx`RLJNH|sPyYRJjU%hD6{tHiA_4L0KXP#p5dHKpoy?EVm&##XfOu1*h`p0Ho
z^&v-o(gZu3*{_c>R*8ko-ScWmV%3fd3wQROu&?Z6oz2VoH}b^=r+%^BTQ1CYv}gM(
zQIc?>u~T8rYW+|Db^BYzU;dxAe3r`p=n8wCp#Qq(-iiOKKEul~SJsd7Im>0m8GHv<
zuq>YR(nd^<t@}xBH}hTBu6?b`4=L&2SK__@$))6}Nx$(~|0O?9z4$n_@sal9-Mo&c
z{}i<>eVQKDW%Ft6Pv5io=N6|_d{A8BS-UK~N^Q>hl#7$h1M_t2bASF_t-gGo{r4v(
zpJq?I-;-i^BlOAl;}c*1nP`7rWyZ~CF?qIOcm6+LzkaIy`j<OD{XFAk{qt?g_RX)(
z&C*Z$VsZQ4m-C+kE;#!yo*~k`uIi1~s#`Oc23=VBCF_;qL_Rf74|VaXoh`|XwGQ%I
zBrdeKe5qI5w84O{wNZn|B~gXTIZ&e8#=*Yc?1kj)IY$qE;oUOXL6`H&qyP|+;K!cj
zX|R>KRwcklX>FRP#h+=%XZ|^HU}>44&wY>CA!3Y*%`=LvW?Vivr|-_&l!L}bt5*6|
z6bAj{R7{>_Y!jg3(YND7=L}Y%Il}W?I=XE-y1%q{uW65TIQW3AxqFgB(_z=T!@(Bi
zb8LFD4Cbs<Y4Kv><Q3245s%~%_v8_`<Pq285trndJi+nYOaEVoT)!Q1{dCCn+#%Of
zhg^>xN<DPQb>AV^U58w^9dg}t$TjbfYuF*zwry{&_PE;0f2~Q$cx5ARy*Rit;;h&4
ztr~~JKO4Umaf&eixaOy4PWHcFk6e~L<2S9m@@Hq7NzPd>)6VE?mSO#$+5IPlp8vf>
z`qxjNGF?t97F*6Ly9obH^HOY2R&6uh^y-1E+drEhia*c(sZ=|7=Kb?OEU)EOAOESm
zdR<PH`tsP9^PL=*+cftXZTY^C|E;05_5zL<FW28$64IwC#`r7J!T<WxKkF;ACa{J^
zIe5&MjEZ@&b<W4BrMrzjW=C0XD)4!^)u~;yN+-N9>5;hSGBHUl@t+!#C)AzxWwX4z
zNB7ieXE8~2@yttNk(Z`z_tTxUM7L;(ZqQ=gARoO?i*zq7(%rOBRq^ip*LEjW#3j#*
zah@0Zc~0!*IkB7PrbQ}<?>uc+c)c-$ebZdw`6?;hcal9%M7;bj!V_fRc5p?@!8>2F
z6lyd1;*Z~{o5O$J-Cq84&75yWn?2Re+s|KrB3@Z`wZ)D#Kb_C!XHD9ga{1HsDZkEK
z-(bXi_|%4}Q~UC?KZ~#N@z`)F{;dE{-Myb8k(p^rXWF07NlcqNvouay`)$psxuNCT
zwoQKRw`p#QbkiITUfHb+7n?nEPL9mgS=}3XF(P#Kuk7T~+%?O4JugKVNq46>&(1y5
z^(^XUgo^a$#fH{W-6_eFOu~*HTVv7EEC2Fn%%@xZb-As5K7Y3wGD-<AC|ozC{?e5k
z5rIp!t$nAru2Z}^(^ja<^{Q)V)%FWZTkoBj+P$PtQ&ROdN0q9$f>B^f;|&e-5<dm|
zHOjrEOEwlwIIqm@vf*G`+)2^Jh0nXgXI1QOzPZo0Vdir;b<Sr$S3G}w<L)oP5C0|~
z*Ltd0qh-&z^i-AV>%Cl4%ZyHXO`NKv?Hbngq%3aD=8i9SJio<wRq3hPPQ5a3^Wx=k
zw~7T{>L}TYPn>t!y-j%T`JCm?zt3%7);iCo>eGvx=RCxewR_i?eEL1}{{FP0StqNX
zoiDy8ymX8Iroceow{KWC&pw|X78_Zb8Y-h%E|l}DB>n1K-8VUj>Id(Y6fS-tGI`$C
z^KVP4=jQpBZZ+O(b$PY%UOvUYb3d;(UT77Se*E3lnND+`ww=lGypeESA;(5$@7&9)
z7dw|-n||(#k?@H}_vY(7u*#T`cRupk#>`T)Zch(s;i_Yc4dizID0!Be&Z|@#BV=T@
z^Hs^S^z@xC&zwHD=S_}=^=t3&*KacB&b@gnGQo=5$Nlx2Gk4G3dArVPZs_vOuS%}X
z>}$F?^ZMMmDd`_iu3rA&)t&+`18Jt4{KqsV{97Sp6m~FzwOOPm(C@{$mUlYgb1Ux_
zzK@f6ux`#R^||*?y{T3A$eWu|_T1u8%^t5?UEaTzPhS)E>G~x5={@C|m3|_TpIH{K
z_qnvyNHIL^+}ZOg-6|a_D<YqVZVozoExTxK<Vw~3)80gd<wkzKIw|<{D&5uJ)?E4g
zeBRoxYc7AjF08r1DEz{yJr3Hk`@^4oiJZAVJ$2Q*&zg+v@t$UrbJT905la&~|L)-p
z#fXS2n_rz;rM-IHnj`<quicD%zV-9-xpT_GZiSpWe&eL6_|1dMN(H8F?$~igurOgo
zaMPl|k}k%D$wv)DkF7NDYi-o(2~3<IB*mw=I8<Vt$uj;gmk%y0-5|g(_UNFKjm4DH
z_KGPg+|DyIj3VZ-d%bpuwr%0<;$wY0EBjl*Y^#PDj7NCHJ~`j0ba%56FZsli$|&8o
z&){3@!SrP73&rdQj5h@F7Nj>=Gv9gGbw<U7N2ri3>b--$M8pmD55^aS*fdN`@_D!1
zW0d0EcBt9*gF~c5g7D5`lLQuOdiXl#a1{j$%yC)87{oi_m5(Qjroe(b>iln#nsYxf
zY-ihHdPA50-ARjjrIdqzuD`UOu+Cuz(@HK?b%B`<DNGYPwKNX2Ny>ZpDHzT;%YOUr
z!Lo9OtmePE-wSl#EsI$4(!Ql~##FW&CZbOdS%2RBXVa7I&so);HyXc9nYl{uz4@8*
zC4!chQ`Bai4T)4!`!F-ZEI55;XQr`X?z3;@YRk@sM4n))_~+3yi^X*HwKY+j=gwm3
zb<g_g?*HnX^b>yD&gyN~CLdkUI?>;3)1?#h-9O2ltXA1QGc<jZwc(rR{dK&VZ>oa-
z2D_iUXFLC-(s|}|EhS^dOePQEwe^c02uJSosGL>nEieB2#=EWcFRuvkaV9&3Z<3ss
zz<=}0rUegAMO^G`jngT4tX-J8XpPUB1QnGtOioF`Tzg|)JPSW`nk7!{gHhvj&UHR-
z?g&TBJa9@;ri<_XrL(PddS40-9byqvt1z(5nyqj!WxoMq6Q_VrL&BoDJ|7YuF-};-
z`M0gkdB5TvfxK3ULoE#_C2zZ=&r~?T!l+hUXY^dXxOSKL%oTc*_nt_Z=3g@P<g}DU
zt9_RJD|zL=>|Y9;c(Nj=iDB|=?xH%!6D!yjOP$@x?z;GBT6a&fLEpu>3%<OX_{`+R
zmd_Js{m`CU|NW;%<*(RF>tZ&!oz!yAxm=n1ux<Y(#`>kZ|Nivpo5(UTRCzgX-qDwK
z6Yk|{&wUvFYv<Z8mGLj%ZZpz8=EeTq`NF2G$Ci^9s5*Mz(>)m_RKigu^rdFTx!J0Y
z{!Q~(Y;V8xoTe49YqGG|_Djh%OZo45$?uxF@TRux;mHaw(<Yr&nGMNOKATQ@pI@uo
zsP?OrH}g$h+lC^$$|Z9{%Rk-o+%P#o{h(jdT$b5l+nkThu;sZGtSPal{bIGmzV^1R
z#W7s0Ev!!NcN5F+oSstfQJuH_5R<gZLIp0?NDrOM>G=<rZZ9~0y2<K<Q_@YQBL%;F
zomdtr2ykXveAkotDaN;#yZ&zL@^_cq%OBp`{^4}&j=Oq${)olZar@tD<$rsry6mC(
z?hi9!f1FBVR8$HSXfslEk>g?toY2waq4MjY&J5OmtJ;v1B?SRd6Q(w8Sd!Pmo+#$$
z+|GPZlf%!kh-=YguZvEaQ4WuoxTpVP`yrPWs4?RdxJA#v$~5IWmwwT8y*t928{{mx
zSTqC{Xb34e2C6Li%dL2UTQ!N3=?j;NhKpz-s{`i|O@?VJoB~?~UUC_zO<3;G#{|*A
z5cVmosZ~8iEI`4bGOHo}iIPmV?DrE}KPx=^;r{QJ$L?b%B5Uow{nUtDDYEKW<K36b
za~)p1oLe$8=A}_(UZ7{##8SVIOJ3<GFS``2I=Luq>8z<)X3>+6u1;M#&3?<DRnN_B
z<rjRO@iW_h*-G1`kDquQKRLB;@^j;8xyVel?vu)AYu?SYUw@q0L+48N@Beds%IoG!
z@0lyTXKu$dzbVfz|6ABBx9i!;k}J;gwsV{IRlQi#ey?i5e%T!U+eeoM9XJ-ZtXX)y
zS;f|atJhxI8n~-qcZ@|;+QI3fRlh70Gpm^z7pIEZ7}n08v4v65Mtk!MfwHJK2I)3J
zlMA+h6gv7IW7%`?e$m7HogcQxSVZZ`=x%N}`zcUzaf_IYuDIMZZvN9#95p0%Z<6J6
zTW?U)sVQyBU4OcD|I<VFiyo%W{B)W@>XQslf`P@s2QDAIZx~oqcK<kZzo^jt&7rqN
z56fl>w~A@&8ikyx+hZVJzCTToHP@hlld~dCEwT8K<9Ul^b*=5s9__n+v~)T*`&pHg
zFxJJhKi;WSivKiU_3=^vs$c5H7ndhT|C?AgVN%==#?Y`{l|Ge@H9x!7RDD{T9(Q;y
z-%M*^=65A0?!4^UX?*&J;)=-58=t2A0o6=fRj*HD(VaQ(O|W&XNqJTJJ0H&tmvxJm
zM?X6ka@BgSG5^66tCTDIRU|wOCMBpG^k`x{=+iWl#d!6(HCD?LIV!$A3Ml6Y+nL8(
zdFaW+GsSP+J-1EDxp)Yak*qn(gx=)js^9e6#Cx!Do{W0ii^3~Xtj%A1x?B|>%0_W;
zxAYk{$H=6(Z8O+1Gcxt@!kr&4oMH0hxRubrtz6^d;~A0JEU3@(^WywX4f{T==x|tR
zr{z>|Wra?{WfAuq3jC~r8XZoz=AWA{TjlbY;pVaG;>YWEe!PCh$w@`!ef5F(;>YV}
zvbHq0wF@UYbX*ZRvP{98b*CoBxyC)c-p`LMHRhbSq)9{c$#Mlt){B~R%%(hD;{1yH
zQ?S5-CI(TJ6$)k`#eQlj7jlaqZ`=9t`kf!yc`~NveX}ABvIQ4}D>ybjVNnu3af<Pf
z@1g^}&CiY*8*{d@7CO3sswU19)m02kH&`wSdblgdHkPm)67+CY5WT44qQf+YV~O&F
zJ_ie?ha4@+0)0mg_qIPf=3Fe#f3~>w`cDu0r6p=7TEy4eFnDEu@Y$=V9c24@x{8Kp
z;hRwFxu&0;HvI8f#T1z(H}S~CBkMhyz>rBPN|`fDD1<}n{PeR|Rn~dNOx~eZ==*5a
z5%<TpUar}BIkI^9=4XECXXl=qWj)vQ{PPxtmC-@pRbG2un7knU>b%h3i`qB6I#K2(
z)RU90a&rOm<eUqKJa$b?F+K??KHYXroO#53xplqDSD!~a*IP$k4BJ0B;ragf*>Uw+
ztcySWc$)S>#+hwm<LMT^*)1-S0aFFFA66WmGEti+>YcKAWUAS^Inv_if-n4Ry{BJN
zv+()n6?*@dWVO%J)BO^@a6gx9*o&p%3+HpmhQ3(oBrj^U@_|V3iv=#C!nbpZOmqJj
zoA=F>Za<?lgUdfa;ZCQ_kMy+01()U?G~syWcwz2AmlN7g&H83aA3vj`P-0~vIEVR5
z+P#fCKb|<l^sFRL;d|o)_DS+wwHKZ`PG=XA|59_|v*U60Ao;uHZ<g(}oVn)2^V)ZJ
zEaZ*Nd7eK$;qxS~Qzhp4=BLk3_-j37Q~q?_gDrjgmYwcx*2z2M+Eyp*xUFhyS-#B6
zmPJ>4>zXT@f=+ws-p)L^PSvq}fx6@Kg`X!@3H7Y8+Vtwos>$UuPG)|}yLry*(bOZ_
z^|7C$YL#*wInMR{{;zW5CLd>GP$B2C_zjV#?5$bne(~YbG?Vq7erfhhue`^ncYb_&
z=SNW*qw58czjG!VSy<e+D9oE<QM`bw^)Q?3)0;2;gllPnh0Nn-ORqoMyE{$nN{;lQ
zbMs{n@jZO-kd2o=wIyAe*{N-rfm!Rplb*fTpB>9K=3b@Kw#y)9rR9U=J3n5&^CNQ^
z$D!UrtJzCbQdR_|cFaDkY0CD;WV42dBV(dL!$Af%W(l4IgNB1wIF1?4kj`9quOdH<
z(e|;#IpJf5S2^dHPrmT-@CLTSl0J(QESd{>*bFWzYzWA=F*ct!Q~Law-t;svgH$)y
z$rBQJ(?Sh)9em@(%EW&5EbppF1<RF}(|<Tvr9Cv5RIFV8;Q1fh!o_NNS3*QTt(!P^
z-E8Yv^YUGJ^SCn=JJv{@w3wtYso{&*#L(=pwI_ecTzUWHrOkZLfXU^jk7q=#Or1Ne
z)NjTyKl^iY>*iShwK$(%^fq$q=cJSGu6|~$n)FrWwWryn97yry(KL}onbQq4mip7f
z_+;8O%b#u>^Gyz|z4xN$!qMRG+gFvJ^OQei*v%%pE#j+FNZ8%T()6<-LRn5>XLZ9_
zcNEB-ZECENSlPbL>(8O~lG!=Cm|~{diuEmxG}zITeQU*Me-37+wjVcY%+4*Betx<4
z^UG^LP5!&sNqjxDLHyX>kW{W;uNE79+sU{(=RiTgS*9u%rBIPQVcX+YORvA$d;Qh1
z?9#_ucYTaH%VgH!EqCSVd>cNc;0ZIE&WIdYs-U0hVWA)+pya^Hsdd<s$<`=DU{Q(9
zshx+beQqfDvo?B}9Q|4P_|L4C6OoSRxSGNyY-w5{nzB|Q&DBzyBP!A_{Kdo_O(&j)
z@rK{f(P}x7;K;!p6tV5|T+V(5HYN#<1UB~kcmM9j{M`L<*4>J;dFy`PwK$(QCw;QO
zMxG-^0;e5xm@jZHF%~%Ipv4%MclXwvJG*!8C@+?M|4i)OS=BwWPoJJF5XqUMDlo~x
zgGrGyMP<T7fq+X=lkTlCe*WsG#?MsGyZfGq=tVyMp*7EbQ_vHB)74?0D%|Qm*|MZ7
zExG^BZ~xu7<=5W&T;6%Pz5My#V(a&_tXX=!Zu*^HWxM*@np>N*&abk~-d7bS`}6aO
z`R@O0Ha(p<HO*+&$rnGg-j#f}oxi6dY;K5eS@Hc7aWVCs8*aQjy~aIAZeh`qMV!)$
zgHH2oTom_<{rwmIdoLyD%=Lb6SMpx?%Nu8V``$&tJYAlS@9j4vPPlZk@Uh+;)*LR8
zo{%KjxWm%r4^>iLc$LmjOj*H|94fKqP{!0}i_KmcsBjwbCiBlb82Le(IjHTDaYf_7
zU_-6eL`@#=L>DgSNfR!%@WgcaAKW40w$Okl^{4cno711Q*8NmJzcKOL3=uZfs~g#5
zOdcFK{nR|~`0UkB%*{{c>rJ|UM|r-h5ZmAXTx<%@t?EwA=6+=U{c!%e*86w3_U{z2
z<DBVl`T1k&y&p#TH8aikor$#gFVK*{(9Fn_7GuzN&|+Cw+x5@KUR!cXUQ38*&YWS<
zwEOe1yOodM&1vm_Wnyz^?!HRnqkn3q+?g;X=7YD_|D9ic{F%7?+QhKyC%h(pnYe$F
zn9ii5oPBp+o~^N)?0$BY>FRBNwmx0oD<);H(m&7sdynE<P<+I!zA!mZEAZvbJ(oBC
ztVz2%F_c|@TB+N<C@H^7Pk-*o*=7Lh!}@HRsp}Ip<BNCDd6jv$H-ss(?K&Q3eRJu=
zUHAXWY&iIT^2KGv^PK`;-dO%IecPmp3maEIc)F)B|J)%hZl|_|;6cJ37cFbqkOR*v
zAHSK?!rS%ihtrJ(3;uI62fTQn)UIyz;O!jN^~Ni_?u%R5Z)8eOZ)sniB*Uc~@Zyit
zd+}X+E<A0u^Z)+rk8qc3FgMq){SN}>SzJ0eu}}Wuv6q#PzszCfKYl`_P1%r1|M}fe
zF{>9BF0xo{Xqf)_*g{M07m{o~J7k6R>dr>`aIG>@ztqybxmd_ZxhSf9S!ude`7&d>
z^V0L5>r@<{bIrmwy-}ptyEyLD8Qaq_EBCM$TgcAZxn@Rp%TD7N-VHk!Og~-0-)R1_
zZ2hsjGjo5}JTvdj-?VYs8^4}6e)>;8&+PrC^5X97kewz|xr6u*YTQdawRDNK?PO2m
zy_XFumnVPnJO0Gk;B#mC=QiJUr=NuRFrU4wdNW|&!F_6bSCnj#ynary+*ZE4?b-k0
z_-6l4$J8qyx6EPv9#H*!%U{p-w30W+w%*V=Cq4a9uXmx?y<}cn+22|vx$ks7sM@(M
ze#qwfd0MuSi+S{t>)d~1Uwp9HdoaG@vHG5mJohTtlsPs?Z;;+&SR41^i<9{Dm1hk0
zwep>lv@n?P=)vy74|glR<=Ir5&)e(iQU09g$ma6LH+O%0bN9!myqe0*4F=-5b1KaJ
z_DHMW>lJ>lDKSSQZpB$^wq}t#OmaLy8sEM*<<@-`%E{YvuI0_o<;E;#yw7^P4EPRO
zummgI{cP5^E7D;542FX|9mOt@=iQIZoO^lV<Gt1EBCB@fXxeYSHE~_Zjk_;5?7nRM
zNn~QEvbL*H_Y<*`x|14w<{Wp-U%vjGpZeXoe!Hyi7N0kM@?H0K&dGOoI1B!H`02_o
z2sHh<HRbx`HRAUaURHeqO(E=1FZ6#j?}%+@fAhWdNh)7G9!*S9KIt~8S4DcyVFlfA
zP(SkKttFqI`Q9=OzC3@X<<tsK<)6=b&1YOcoA+wjCdC^kOVw}sHO<{qKI!NGvl@;Q
zUl=&Hch0lF_*q@sGVXBXo9gZ`A*(}6gG{>W-oHM@<kV)l_1_n%Wtwbz3?8*A`ky$s
zxZ-i(o{t)SPHy(0jh#!AnOuYw4!y2;T)O9@RhxjFho_^DQ2z(3dlj<jjER#43^h-9
zItFkx1x=XJv_d3hse;Z-p^V41dv?lNa=&Pw%v6=KOu^)ZxP?YKBWuzmhqm3Hj>T3!
zUN@&@ve?A=O$?%!riwgSq2R<>WTCJAuqe){-$ImSrGjT?sqloum6n`>%kQjsuy@bL
zsCzrK^Y2XEEFof(rQqMlzzUr%Vql#i*yPqBC2qDr-0lcJqYI1UoH)fP3jU2YtVJTB
zKN{sMxt?(JrftfdnG>z{i#P7P^!n$$kkUzTwZlB7O0Es+6IMB_Vp_?yq-!sa-o@W3
z-)zh`&F!}d@>EW>mj7F`CrwIsvtOOtwf;XJ_hhHB-k-_;CuF_B$@$s;Vm?2gX1_gW
z`9-!r(PkGmWt|4s5mzcFO`IQT+$;a`z#ikr&;9tH&h@k_S6g-O<$^tzWh);y`Lt;+
zR=@oA1_N7-@MoF%n>(F17+z8P3hJ@U-WL&*=)T|fr`sFv0{MG8`1Aj?nb#ebu7B9O
z|HHAsit7Crj=5G$K3@D-anH%eJ3SxlEEKtC!KI$qYI!&M#eFCC<yI~2AC5U!JeHc%
z@+D_sF!$c57gt$$Y--Hq_C@A!yu9kd6;o}#Zm;z9d%gbek1a2s%vUnIvF!1g-G$HY
zR(#C6Q)d2eS7bnITza*m-31*1hhG-U)&6agcE72k5@Pgn(}L4Yc3f8Kv*Lq<FIKyT
z@0t9wSZvdl>&q3jzb|RlmOlUd<k$Vim!JPUQ>FS^L~HGzJI{?3I#fXIC*w&9o3fr7
zb4FeB;Z-iKFFN_e&;8n5Nr!T)P4mv@>{q`W{@io&0(HmptCZyvQm#%6b-z}_drxcU
zxhLkEFXcR&uN2A`DPkt+(5b9EjYU1{kK9L=Ta`NJB<~-4E}B~U_~EOm)1}?7`(C=x
zoBsOPW>f3$k>BG^>8yxak!64TVyY2epO=MB$kO)WtH+m?PIfXm-ZN_|``J!6t&6?w
zuQeq)4@o>()$9HG*iutY$&O1?8DB`Y9MX7Vqul!9*g;cH_Ld%3KkJ~5G{(Oz6KamM
zGA4RWR4bV*U-Ed{uEOZMJJ#mi2{W&BoGftKOK;7O)p>tHH*;`3nH+dBa{8j~H%5Go
zJJ=r!^f(*vH9l~kI3bZUHU43`1=B(PHs$g^78}(R4^P@$o_zOkb?M{1yFSL<tyrJd
z*us`9+7oKvVY%7dSi1gd@BCMG!GBIX<+lEM!hcJ6;lJa%FKt>-wD8mFfQ<ph9@oWo
zsIr7MuezxDpx#WnGEhfksuc)|-kUn>?MKap83y?wxgohWZq19FFDh1)YFPP}&U$sB
z{DpN3^Pz*;b4o0Ks)w?Ns^3w|S+@6CzRj`hSC{y9&pyp}i1lGp!P)-LzbE&}v%X)j
zz54iPJLA@Z{@wd4^pC2@r>=Qdf7D24_R|d|KjdD%|GW47?Sk%>@*A?{@9xyr?RfLe
zlVj4sPu%le6g9jY19`fN3K%*+*)=u=PU%^cA=0`?Lp1%cW9DCNc~{1hv%ks7#Av22
zStHWAR6{=fJR`H}5)CC+Nlh-#K(DT(N#W1zer!FIaO2U0#!0~<bBom`J)LrJO5>p^
z>GCy&+P4->D!I^qiS63l)>Hn04P8PZQ|2!EAhLA1hPmrb&Zlb0i+>(k`Nv|q=z@TP
zM-u}oIiISYclqJO#Hw=7Lov~2nz$9q;vE|b3!NP~pDIu3Um>286Hs~bjed=xPU?aO
zElt8)ej4T%-)V2@j<@riyl{pvSCEE>;v1L!b62FaP7+lL_|o0c=XUP$34a--xH+Op
z|Bmmm_*`tcR8v7GU?0~O)+qg>`@%Dv;{t4%el`3PHQ;!`e5k=dfk%q(h4z>AT+Oc*
zJI814KeX)7GN<_M>)D&xYMb{t-cZO1FwxkhTc%-EYE{Z+&sQhCL9$Hl?S8iRh6`o{
z_-Lq!2zPn<uD7YsU8OhgW14mIBewe8{EMEIS-gALdo7Xu0q?2r_Uv!tCjBd6|6KTB
z-rvvr6#i;56$?%{eSYtsJ$D54?8*x#c*kgRdUGBYlYRfFLEQaMY5r|ZzN*42-HYR7
zzCGF@&c4s$iT;FL9anZd3-Y?L?!c|0-Cw6ndbFbX%v1IRJ*Gm>R3XXj_cq)s5EFIZ
zZ}H}6q5X1?DMlq{PVf2pPT1mC>dzNlj)8n#O##z**IJ*S63!sOp5Vp#ZEmNZ>r=)(
zs<zt2p9`O9vOoOMzS(X{bHSqaB7tzeN9TT8znt2>WAi`1-`4Nn_w=7WP{^BJVo~$(
zL+fX=TYvAW?>j8vcdJ06@t_iqSmK6?cAa;Ew)XK7zTtJ7VzsVlRrH9Si{bjpC3<i(
z!+EB!#s@%@4R?c%!@3i~YyM2WrY$mc)!F0C<;{y6KV*gEhP*9!8&I;NbXCc%_jXy&
z*RqAO$FRq+i!`rlp7nWS`hpoY=UNJ!EHaawb}TSx{gnN4?QZ{lcQ@Sp_3-7QM~&0^
zU)6~Q9C|ooQ?!*>fAdEj?-y}T`yw-^3EjKM{q@QQ#^dWP8uTY<xvU6C<zdufStc;S
z$$^VeQ%|9_L1a~*qCx8c5vJf3hHVT}Inz=fI5dYSI=JRFXsVXH@!roP^Zn6|=9q<C
zl1dRB2|EmftrWYQ0!7*v*@!${s3G9m>9F+3T1hr8_du(z^64T^nOnF8^;~ucJY<SC
zi$CGusnrz}IIZhY$P~4Z^Cg|%M7Q;bY*^IL`g>}?g@z*mth{_be=Lf&+B$Vlz$d4Z
z-Ac|IAG4OV-H?2>;)hT0sQ`ygZjHkE50bisB*kO&b{$du!a2pK<m!$YQb}1W7PnbJ
ziP>!uv)5e~(&$ut*0Ml2T&Q@SQFM0QvaHDZ0!_)EmNU<s%dFE6(fIT2)Vvih`ltS_
zWsTJ4N@h*aJ`v@3ZPvFBe<mN(77^{@a$VZX^**e|r!Hs1>kY5}+^A*0QIb_Azx8ms
zgJDMfg?}HmCR(kq%q=`r8kBx0&#``U)u-EMzcJ6AzVX#;v3wm7-Mm*hs}}r=eXwQ+
z!>9SpA8uSc+I95)=?!1LZ>V=Xy{anQ_3Wys@cMU0JB~6ks;pL+S$CRm$6`i{u<w^c
z3)VCxi^^n6u_Qk5VC)MmSYv<GkXdAPLd4O1THjvwoe{S?Ibq>r=4GK<x31b2*1Dl;
z>y}q#JGw$cbJuW%hOStZl^%Lx)eq?pt-3j!-$kpR-h0Sx`hCY?2|ncuA+5`_OhP9v
z>oN&lvhrQiSH`ropUyts^Au$E75q3<DcU!0_g$@~5Wyv@)`m`8ysL6;UfgPippX1X
zX16C5^R8XHYHeu#vi?77e*QevTKXjY;qFKA-M1p<IYqtN|0z^@$Hjm%QXeih^tY96
z*my8!0XyHZpXx25hqM%*ePOG8oH?t7_YhlR^9IKo3U5rVD!vgadsXs^r*4kTotBp^
zv&$QFZ|o8C)vb%!5IO5l<uPd!&ZFD>*T);230zmezO04S;Nl%ov$LgdwW57fSFg-5
zow`=7&SdIZwHlL$j|~c@bS(~%Stc~S(f^T$=<yX67VCr7J9q@>H}=01Xw>(u$<&H{
zS(I~{FT8L;mon=<ld0>~&Y4VIe|W*^1*aGMR_RMCv|wDcOz2w3KB+$c8pXwm2d6l2
zGAx+IaFu06+hK;w4opWE&Cr#7x{5Q#_dym@xN^-D{hY{|nq0ntF{h?Z)@`np%#o}(
zBO=x4xK1s>v~kr+XVcOZZWFAEub*7q(%K}-6{ZoP5LLYUric7FaUre5!sY+x{^t_0
z3KstOrSuuMWQZgO`@$owOyXQI8X4;YHaVGasOpHU5#VrQWd1qJDn!|Ds(!{xn~493
zuY*5>#^uh>oWFkR(SM2cE)Vlo1_x{mP+jqfE62g{YD4V-dj+0boG+3+Ip%ZC=Spnd
z;1scNML-D0hNGol)}8yejd_mjgEtIm>;~2zH!s(Io+5jQ&$WHg<)rovmy@O!I9Oz4
z-KY%Uxg`<6qp@tRab}j2MP{Ls{lfWtOgYQu8b93pp!hw1)7hr8zN)-`CtcX%EZrp?
z;eFLty4vy>-=;8|(B0kl|GwUzSjj0mY0*!A!^+0Hv2WhJYKY}||EekW&AX&>@lDnn
zb7~YDjhgswTxHyS;ZEx_J(JhJ&x(hI$Td9EGbs<<ucuPpA7ZsntVS_$zi4jYM2#*M
zfjf8EzgJJWbd`5^z@4jtyC>XfeWE8)zR;rf(EFUp4ih!HTmmJ!E(Q49iNE&NMRk#e
zkn2Q-uH6B5?tU*TJ)x)aefRyhNk!o+-#8VCPulI3T0Z~wRYmKU*M2xLJya2qm~vWA
zrCf36?%uacVjpuy@f}pVm!NzrreP<?;yZhk^cMOnoN8%eF)3fTllyFJ#<lFZ`YB76
zv`!LR<iCZx!YE^U<0LUJsffF#MrB*C>6Mjly|(V!E&mhI=XU=&5)t}Y_-<FE<)`9j
z>5sm9{E?H8W&dFtP{=O*lyMfn$i0V^KO8>FHvDF|!~W|%!)^8qJBQB`e{y~D=U_g>
zmH4$YbE8v)>KcJC4(*oSWwZ1-6Pq`9-dONr%f*E+Os>8a*RT?~yY8jXr7I0}7p^oE
z+*mBY$H@+=m4k&3aXxG*uo4rxw|d#a{o?x_>(|d|taIF7bDXu=x7%EqNyPjHUl;Gu
zWlN|3vzULl==i_F5BI%(vVQqJ-=AmKeEoGCCtsYpB)Ux7ExhkuQ;F)!u1T9hM3ya*
zX#C}TMPBRG%w;ibr-A}!IQ&=q#ea5*^V!-A4vFhARxxsIrw<oYN@Tq|d^K^06i;|_
zkHv%yo-0;}w)R|2++oF|%i`R6_si27MP=8%)TA9gJf`0sY8l3;9y6WJ=YOxA-SP0#
zqzpA#J#pvamjzqFMw+#qJ{GW8UQhn;)8rL5`KM3pX`HZFDDoOBhvckF8#ET_s2_fM
z<jb>~hss*(T^my)CH1rqr>5>$(^EagKICWn>(c>W<<}V>);&<bZE%OFXE&?FI_txw
z={rOv*4ZEan!e*7&-M9**XKX{Wqe1O=la5jRwgm+ZP6_sn`f3z_%~U<<_PyRab3}m
z0Y{t#`5Qlro~qJ#y>LzYCb_OEjaLg-xbL33V$J615_%zrTeEfq@O~FLT$;7RuQxkR
z5+-CWeV8?S$90LF*Dd-UM7q=^>E`^r`C0hR%%3|IZrD2rvOZz`wP>dxAFEjE4S^W0
z9IhNDvxmV4)Dna@9Gqnwc}{Ki<da!uk!RIb-#oIy_-5~%B_FP2{<-mE<3+d6v6+i6
zx_l0l=9%?n%f*H7FO<L7Jy$hx`Ktv-8ILY_wcyo;<b@JtuNn#t)dZCWm3}gryejVB
zKAC;4(%gq!gTDu9yb$-*b^pkx%#-J)`@;N-xtGi@-oM+zBNtq;nE&(ZeC6L7iur<p
zZX!au9+z3Ob~*TpuXw4Kvdhl*`tp~*Qh2!HJyv9hq%PKwS>t@UG=0~eGrND!Sf_va
z?(vfEhNc|8)2F{&mkbqE2Z>tF)PvKXr5cvKK3Q_zFxGF`>60bmMy4FT<%*WGYnW1(
zYUr%d5B9!TQaMvk`SR1`dry;h)%Z;BfBDHU)^6G9<0Us|tP}eD&TiT1!zGn7^!P8w
z9xo}Kv5xEV)r2_qmY466yIutw%y0T3a%8!J9qY$R{?p4I*s{LYtnfcr&2=yM!@Q>N
zdmh-bzSfNJKUl?e>9X$;k;Thz*t~z1$aN|>U|!Nkk?!RYHb)<5viY0wYn$_H8&>=+
z2)g{K`04YqtkjQMA$nKzLUgY1bR6EVutq5CXR`bXpBrh#8#4b~Q+TuG%d0;(er|NW
zk-0dy^un<Pos7XYi^U|iZ`h%ukk^>Yom<0L$lVbw6)JY?M^(He_jT5zi&k~c+WvmW
z+l{wVzg@kZx!_+-KvwPF?fLhu<hD=lW^rBYHF3iYhX{Kowmt^4cdAOdf|;48<Fbx;
zvsSERSpPa{*Oo|q!OWG6AmW4G#LP~g*SltbY44OMrAtB+r#Wn9nO1sM%3C4w)nf13
z5>6HG+LBIr@0y74wK1!vdLM{*bVNvfn#IN|QqyledX%QWZ^kqs3#$i5E*b1O(ybb|
zCR-wo>8KF@G>Z*K8kYtpTK=7WxjuIHyP|pexs0bXcL$Z;vW)*&aNl0ycHEm&6IQH^
z2@F`9F{SwXv810`A-p%mxNa<0$X)g&%`5-QoVQc%%}6ny{o^dl{hUYUv%mN9ZMoRU
zQFqdQ<*kZ7x2t@Y{iga&<uf;XF<Ww9V5ZrwGimek%kIvex%Xml**(o#{YzDQZXdg?
z`ptT-dduutx6j=yue&ud|GaDUm)f*a)BfoC+dEsoYnF3#v6Sn`ux{ynyYp(9-+Sxj
z+ix8UyM5!_+qZY_y=&XP`@!3o``=}1tY2=QTxb1q`|i7I%{E<i+g4l`pMUgi%6-3Y
zZ}rM|pT2wU=i58?w8WPcxj#^u5zbKC`f{>G)R~WOW$uSZ-thajcGHGXv2CXgg+<<&
zCzoq@;DJO+jAFTjv;De?ckd4uMO`|eeQK_(kky)p8`=&_@+IH#e7fe7OwDSYoT;sA
zCv9MDU0bw)wf*fez2-EPf*Xe{StD`|ChbnVop)oK)j>5Tz5N9@4kdCwztMbZ&isk<
zP3_NXR-Ak>bJ9g))y3&vZs%o~G|v5=-!xw+m@Cn^)nTU70-uv2?;``YUdmgVR;4w~
z=i@xq9H~;N{;Ml4RV+&D?p-BoH#hx#$Xk;;nLC1u4}5uY?}UA#or26hla}UXY%Rgy
zn#0LLagKmrHSZzShfM{JJ67y^6SXRDRo<!`_eW``nqIdhSxfO9|6H>(WWJHi#O<Gb
zH@}JBwtIDjMSW3BYJgs|QnOOC^%GaNqrTIP0;V)x37XV+X~KnROL;{57uATUDhs}y
z^3`aCP?&aDbXZi^qo66%UAF1G*EZptI@jbyswQ)*hM@jw?K^x9feX5XLPb`F<om4*
z5N%zh;pbX;C~tav3{!N-jIK)|BFh)qh^lJ7zn-?p^FiwA)VrJ-$*L=6EZz0yHv6)0
zEw8}9o~|Mht^f^(g$%7llR0C4CPjEfsD#)#?c{i>G^M>q`i#DwRg8lGvl9<TyQR!C
zmjfveD%u~*_LXnpHn%HUJ<X`-hOxbN{D~7E`HowE_ddE~T2hR+;ku^=1yxVd&z_$)
z-`3vHD#q5X`;PhC|4mGbm_o}!JZs;G2(X#PJ6>Iob-~F&FrY<b_VbNLUp;n=U%384
zIEQwN=pnVl#aWAc-R7R_pS%2(!P|n@S87UklxaS&QjoDZ+fZ=fNW(wV1cL?g7x-VO
z>-ur1w_Fxe4^<Bp_tl*hC7{P4-f~(@oM)Od>)*D2Y9;L-mMLyN%G>(!^!yH|_V0xW
z_t$y<eO_!Z>7xF1rKJ^n{s$YZOA+B*AD|(sAFy?SdYWeAq(G70ML8m^i#24|FRBrF
zx>!TOl~c3JBhaGj(!^JZMgevVmz8FS9o)jP%+)}*DO1qL_}9_$&@P6nN-M-1w{omv
z%?|UMY(2e=XUa08cQ%t6KZQ(LvPk3ULl2`F`*t4+ow9CG#Zxtt)7Hkdiz?k61GP^@
z1)l0k3Y)Ti)4Ms}Jg>>Jrmoe<an023iVnQj^@+_&ylnl_370?qVXjcyYq!*4;gi-$
z;#_eW6_VfP#%okAeA2o=#4Dl0Z=s4?rSXHnhT_NhU4`bYI)A1J#m(3?Nx~~hqjTji
zF{3ZRzf`<bUON=Zp7`^}C<6q=jZ+saX`3YV_Ur-opb+OkURTp8R|0|-$BT%$h4g<i
z{<Qg7`swrY<}2G9I?O-(eUhO4&iVxf-<;zU0v-e;eAF_~oFM*!`PYWZ7tFsJ?D+M$
z0v(q(Y;cNDTw`a^5_-t1d+PIb7s6jif8qVbR@*%9s`7e?6HI2jU?^)QYqp+me%^h@
z`wQO-lyR`PI3Hqs_<)h^%i(kFbr=3MoNYLpeJK4<`k^!kbh4kmm;;>W6k<T}{C2(d
zAA<;+18rxau+I7XCg&5E&Q2G!pKnp-`XaDRuGQoC&Fja1ds$X>&l6q0Lo@&Bso9g3
zy^>q??%^uobyg2w39Wkd@R5*&;-to+qpyTky#tB7S$Ilromr!ylB7m}YoJz_Z(xqs
ze&b_RT|O_ri%yxf=!VGAWg6zL2Q|I?0!z9!eb(~w53K2WbXZAHkFQZw*;_hlJHJR9
zc!cK7@4f4{y*JtaImVIyxT(zbf{4S3y!|^5R6K6!Ds(#Ou!iw-udvoJPSM9LT^}7)
zTfPb0YJM}XM)Ba$=^r=`sRzg%O5|sBIbrs0dCLS@r!tP4$C#%IY~p42Vq4^9@$cb}
z_Q&#l|C<Y*t2g~y6}jclo1dWe^zvtAp$48@CajB?LK_!22yPL%!c$k>YM=CD^_7(d
z>kZZigj8IA=lsb>)J_x>FZ&$t*T}yxW_j?3v$lPa^F`2}pRBU1MO(uawlQsFk(|r)
zi}7zm-Hk^ZgdZ9w7$+DnoPWXph5DEEoDW-)E<I{Wa@@EeB0wiPMnEqrhEGPfO8yxr
zMz$U0eRwvO>!I)K(%#K{e;;euf8*EsZQHps?9S?C@eFI@|MPmME}WtrEp$xOh|}pK
z7pLf=MFJBopVTar5oKMe;p8f*#T6LX;kDO)0a(D(H4rS&6~uBQO=Akrl+}w88h3?W
zSv#X)^B2)ywXCWuHN0FSwYY)<XMjpT^#f-5Q5#ZSS84dHcoRBxZ`Uypamsv}*7CLf
z3vxtTS8If(-hKIY#oFI2U15RSHmnE<{ob|F(pyq9E%m;7z@<Z5j29kRcqlxyB6)4s
z#;~PaudfQf4m$c(Yh7_5^U@ZkP>s$)RYScr&CbGu-kmJ>R%`8XjE=Mtce>cqo&R;w
z{1&eijfo3g-1hQ_oD5KM?&MK5nsP$muG2{#)yc~5lV*e)o>?t%H?-%SmRotMrn2f+
z(XLf(Xa2Y_uljlO)8^;T&zujrb^E;S?u88>UinQB%iunu@wWHrqKz>oCFM-Vxf+v~
zH~1cSCi;Q%2TKl<8RKlO+SYwe|5q8TxDftA`-|u=E!+N!3+hep%XYE3HZQsyvOH_~
zO1{}W6&k&)i#8gWl)OK7=t6{q6t}bU_q<p4dQ}%L%oQ#Slgcs^OkcXuz(lOgSRz12
zLrkRGY??Cb(Ps<IOykXyzskC^e=1bDZ<yM5>$dX=^IG|Pd%vc#-+ic8zT>@v^`)Z=
zpDF0?-=KPYyT!Dl2X*?A3MIVvetO~|)-Gg{zo<g^=xmL9F7KB)E{hd`aIH!%D_nN@
zs<z5JaPP})v*LgB@!7w)=ki@`m-)rLo$ubx+CTfly)73^68EhPHV@p_`Oep5e`EU3
zR+cW-7k6FWt5um_+`Y2c{PVOY;&!j5CEWj6slFok@)5p<Vq=f}5&_eevN$<u$S<s6
zIOS`QC*H!VvH9td;LFdBevS}3y@93tGv|**(R`;fcj}}@q^uBByUg;zb?X$+!E<1I
z(}=^etzW_jbh-)X9IqME8)Xhd56}Z0qPHfNEwt|{SRls;3=zi#feuBw0YA~|^oGM1
zTH^L3)m{5jBgpA-hEs0ej;@Q+-(7C#il~&kv$iMcEKJm{;(VRYHl1m9_yP4Ao*R&O
zZSeKsl$HJwawFtMNIbilsF|qQdan6g@_p&&*^kXHy>?e5=g8dHua=p4>bixeb&lNG
z<6=M0zK%E7w>#|Ug3!zJLNCt=z06bgVw316-p5`=cIP*q{@HxZbC*McN$=C|2fs5j
z{mFaoIltx^^UVvNRCYew6lu)1``NBIVd2lKx6IjUlOFke@$>7BIj2w9Yq5wbORp=i
zHE-xWUwHY__TmGDcAH;WF0|XT=`HW^Pka1JB2MQOZ3^M#6P#0BBq8Z)9P#)0$Gz`v
zp0y30^EB*Bs>q8B&7Ir-S5MR3{kx%T*?~3b&tv+t@BOsd5#o6vws8HLisZ04#{PyD
zMN%_=8J#ycKi|0ijb&H*>A;(vA!bTb{~p&Ya)}q6x#mQQvzoNg%?T&le9TfdX-?*y
znR6ng*|*uZ9SqA?l|9$0vC4r!wwyNMvIFS{(hsCrJWm9dVskbK+X>qVn?Ydkn}S1r
zKYV`Z+<-&t^n>Y~P8%YZ<$b-Cdn4yYj`j1ndr$W2#fc`il^lGt=;qnHijQyRI=Q)2
zx4YjD*dzU`U~fjNscc%@Q{D;RZ64m5lTu;$sp*NUXQI*fCp8b1yQa9V&`C-a)alNM
zY&p&2_>@~d@Q996nxJv_g2-1#KUB?ExTmIY_1moT(T0j4IvkNYT}SSmsSn?%G=u-u
zWb4HjroMl1^wZ}~se_D`TTd+ZkTa30JT;wFuDVxbYwwq~-jGCH;ag93#{avinQ$2b
z6EAbO$lA?4VgagdS<OH&)mh>Lr<{6%@CHtB`Z82)@&8}}N>mVRdUK8~ljrY~2mR6>
z&E8VputE9vpJhKJT~s94qB@Ir>V+;poGn*!#pLAGJHA)<+<3iaitWb@%XbI+=P#dk
z&hPnU<?pY>4jM+<DTvQl^0#ZxH8qaCnOcHfvuCXM)ey0bNBq*qJI3=b%$u_Gb>&X(
zszO(Ny_c>MdXEYOJGwn$S{_|mx1jr&tA*a7Lc#v-h8UM8N-ERqE;@zi9V&Vm&Nyk|
zff$!(N+uwo61_vkg7dn6#IU?DTIZ#DxbULW6um<wg3G!sVp(1(1&J@*b>?>G=d4V-
zMFp`(UMnSt3-0E)nR?exWNY_^*p|1O*4<Ngp17OiZt8tMk-gm?Vq4xT)rm9i>3EPD
z@B3-3*D7zWM~ZFj5^_lu9S;@Tju(e|Iohy%)Cjknx)3I~-${Vq(dNhpfy0lpcITId
zw<Z11UU#ax^xo<(T@UBWcFye(nlgX=Dz}!yDpi6PbB+Wr_!(dBmGLfc<8h%48jqaA
zA1*S`0F6pE_)l5QoBnX}ibV(25{jGFe)j*axM3v>p1t1e-|FA$3xc2|zByj1Myh5O
zmvYXnJ+G3*%I40!{P6O_%ZbVrWj9Jd(0ch{aAH4@27(qoW#<~sH=J)c+X97Kiq&eC
zJ?5RwIh%7f>ulE9sn2zO=s*(xQ#Lbhv)kuxe%Kk_9RIMm<f^XIU8PrdZ^Sx^A8{7o
zHw63EnFq`W=XiASU0h%rB!t=}<lZi7Rgh}}hnIW)`<*4H*n^Ae_9=0%IsN4H!?~T!
zstdSgA8cFT!B^Y(Z`Sg&JCcjolQWmMxTPLYPY^yTx2(bUfZDEpj`<Dew<a7|2AX&K
zWwycc-`St)HCj0kD3vp7?xhEp--z8XfkEr#he2@J;bph?Zg{!jWtm^QfBSh*3lf4i
z6tmT^)v)Cp+gBPD_ompcc)e}W<vOQ;=zH&r?TWA46_<n2b-T$Gn}X+i+D@)g`{?zk
zRaK!gTxh+D-ST!_<{eRSf>(^XR;LPtpO#qndFB!8&hvlO?#P_ZT@d&2glNI*xV$a*
z%Z;DpIzD}N%JlCY-$UDi{u*Wca{DGE&hl{Iyf?p8V&f(I%uODqG0x6je<2}Ij5TLN
z=adV-7^UCJIOfZowfz625LAmd@E$4=d9L$f>SD|1^KJY<Q4YbIi=}F$`Y)RtKd|iJ
zvV+Uc{yz35vTOFtM`_C>d?S07X|V;gyefJTDyG;Ru$$MyuccDyjBw&Sj?XSLmLIZl
zm4)!*HbjEpxi75qS?9CPR;@aib};R5n)PM#8nc>rPfy&N%{7~Mwl3Gc!~mHZmP76;
zHP0jW+&rGQZ1(KP2ZtV7d*yPk5RXW$SopQ}i@Nj0uc3PTxHddxTEh|DazjXbRjS6R
zeO(sOEVq;{h)vwavDI~rZd0ycuCZH#evJ#`_KqE{Cv=nY1WUa4&eeD)*0`Nx&(gbJ
zUD}fJ1#7%_&(-)R#<+vyfGdlhQ-NT!_kQgwZ(Ry^a2#<}&{HZD?COq)VR>wnr`~lu
z^-KBDEd@p&+rIg!$u-2Z6)>rZJM3gQ#Tue_ponog_koyY1x#k*2X^|LU9o|29`}zJ
zh8LS!gbj8vTw<M|x1fY^8Mj3&!>diL!W(umTw~p!x1f}99d|`6!<$V<qYSJMykXeH
ze1cEGy5UX57SGF;jA`sMWF5*x-k#sU2$uLQI&1bpu%f<2o<e30WehhZO|Mz_XUZ%$
z@e7t}*X(ufTIPAVwSHpshQhu}7H*ky7Mbt`ZffA2AnWi=`@kE9%~10}7WJ9kd(A1J
z?ba^wcB5ZlV(Kxm^qhkRVS6t#WQM*8mFb%0EV!k6MQqDkqxCns@3_9u`>a)MB(kUb
zMQqFaO?qV=4_rC)ohk$mcPqrXd{pY0Ub@j)Mc=7X@Kkq19Lr~=S>g}(yy=$xt+gav
zIaF2jqO%LXpsmZ7igj^XOWG&MDOGj62wc~v?F<s!qd!CcPI|`+#YOEMa^dOwyozI<
z{(W^eyl~&F;yb<FJnP^6vHQ1K*x7?$(e_p7kyH(kI*<vHa!17N7IlD4ySOe(|8SXg
zAV|@i2(VEgB^UG6zwM50SS)m2`>FaXj#D1t|Dx6I=FB{>Hq~s-w4V}DZk6AAj=z!2
zFX360!g!FCn_cJ5`htCjzOH&UE6}pyASd^AZJG52yAQqOn!bA0X1?vKyH<TS<4Sue
zv;N%2R}dn<|GmlN=RWg{<x`(luc?_d_1`D$`RQA83ZF8stywVjh<0bV(0i48o_k)l
z%)MfLvTV{@mD`@v$`8zqY}p;^%KcQ*w8rfI#-Dv>%X9W6KbEz&bE#DO@_N6YS?{@n
z?;jjLnBVNt*ckpwAu|45SJke=DU6DzoB|_HELi#HYvb-EvX_qaYlx_pm+Vb+$Xq4v
z`gPHYIFTf=pNGU>?~5q^u&e(<B}?7cEe-Q!u4Z>H+c5KX^`{NKvp29xUr2g*rOc{D
zSk~!l@pYl;LgABM%P)wPHCv~oA1=rmRv8*IHLNu>GBhN<YgN#~01e@;7NLa>8vOe1
z@A&Sd`(*tr<dI=wKk%>q=Kr$&d+V*`&6v%Y%?_WF{2=*3asvd;+%U83T+8{E^CkQ&
zvtL#mPCJx#%aYrU+m71|24A1ztr4p+%Yngdb2mRMULO(mZykeHW53djNIm~!8<J0W
z-rx0Qjraatk8e+(SNbx%e3HWd3A1edpI)A?xA*0z1Y7^Jm*?v3<tbQtTKIp-;?tk!
z+8j8q{N)R?hk3)a?PkwD*3WyC?)L4=;?s}+d|8;S^@f$<*}=IseL00G-OkZ6w;w$a
zbN?PHbN>;8xcd%^Lq`?$`U<W2Z=X9XXB=SlQ8gh&?#Gvvr$at;n!LSHAN8wQeO+qU
zkCsI)TaIjY*>z;+w14+sMopM8tz+7|THRiY%)_3-=kz~Xo21X5z&2;geVIyc_x&ZG
zvtQ*Lojj?AIX<>irz<5;rQ0Y{<kTj;JDSs759)ZO1)2o6IyE}2v`m^jPu0TtjLk-G
z{l6xYiuW87jMLk{ZhtwW-MuX@%MPYTT@lxl^WXEHHFHgr?(!?^&B6{{JoPQ?m;ah6
z>!)t}KGjY{TUP5_h*kZP(?|Rs%{^k>Sq@HsbJAO8>aTtF_LQl<_ulY6zjGYw{^pfd
zmsj02xi#6?smN>Qlw)s%+Y;{oc=*zC+eNNjGyKkNRA=8Xnfb;^#y3wG$`-{unI90M
zp~V|6waz3a@p6fL=7&s^I|-K;WCWOKyx8sZd~e^Y$di|s-k$O8!)4c((wDdGvt+;X
z)vEHHn22T&PwN(?WulI2JF-&aFHb!X#<ERmljuU-tA{-77eB6?^=i{k<GQ|Mv(3_y
zly^#fd;Vgl#<vz1yG$<CSAwkbJv`o6v6wD;A^c*+r1%dWD%aGnRy>jo++disli$+&
z{7RF_t?nkzZSPFCSh;D<w0(8+e)W8w*vYe#a~@B83*)o&HkNq_v!~t5)t!4g>esfd
zCArxzZ(h5!@vY7d&ziKHdFC6~o-g+bs#7aeDb(SZv`fv@yBxG<#k4b9=(>uXW=FJ_
zoA>_3)AHPP_x8<lpZjy$E6wZ2mCk$;`NiO})2jdO(oZ5;{tJJJgx7P4t_W!63SF_o
z>PqOMdr?z(?pIw2U2<iK0mFv>>A%mvn}2qHb^W$~Z~kn(%IDN6l{l-nW07Cb(|Mkn
zpI@#2{M_U1^AnZJy{%*~RQ^Brx{QZwuc7Sv?J7@qoIlm@Ww-a4rT^Xv&03#sf2TI-
zILDF1$CU?imfYJ`CmN+`wQRw;jf`2pU2^AV+!ju)UbNg_zHCvozR2gJHW4fCasOid
z+f?WHKT05v!@NcMkneBlU%Y?Y{<YLRn{U%l9#x+gsXHy}zyI%@ng=KUnp{+BmHNK&
z<Nok{{}!`spEF}($Xw^UmLc<;-&$Uo=ls?3m#jzVwSZQ>(ibnv(-{uCd)R!KXE2W=
z{^~`)e>bWw9lti&_M6W9q(6~IW41*sh=`9c{xmtYBeX*`)v+O1yS>yzGDq{%=EAgj
z^P8`>g|@k>bw;v9vWA+jid*>egXdYM7yl>ExY~cTe`){J_eW1%5Ad11L)FlG;pXai
z@x?RU-<bxVxOPS%+PLw}Pap6~S!?~flcl!%)>ya;^{dEv+Puh$t5W&u@oM4`<IZ%U
z^D6T^Z7N@S)<4g1s}u6Q@@eyP^`Cv7bABFLadKg~BUj!#h8Ou3LNZ)BK313O9?2Fh
zuKvfne&<J1wi`?SAOBtP_};Zi@h{HjN4RZ?K6OH&+hB$g&oYA#8v2oz{6`(*>c0Jv
z`PupIh~a&1_G3IcFYL|jr1cLn1vaj72wTE^yP-C~bVby|u!mtk*M5p#rlWUfL6BAA
z^0_M%^gMT__sc#vdFNr-GI#l_eG^I(j8|B#uv%f1;Ppy8&t3CQU(gQOr3KbrFIgrQ
zS$aJzn&NEn(D<S8TiX}vU$_sk9&Kt+2zuwNd8a+d>f++LClvEsHScr<SvfAb`zAVO
zQqT^Sr3Id+G-Xyj|M@c}=1%>)(gkVvU)j&sUM;i#IOCqqj{BWk^Gx>jCVRNrtNhzl
zq%&c@t-Ys?-RI9oR~~Hh$vTn$$Bc(dhD+yCk#KK8lbeO3@*IWj*CKz2EX)-@)Zh<l
zL~q!ssJP*xMoi-5X|pq0giQ`{N<C!R`Ic{?Y0>MstT~s8gn9Wtw6RwlX5RCV@!ki9
z^J^=_%FfybHu^*Q&F#M1tKZJyo6k3&Z#Lg-Mlei#&hvxkI48JLVKx&syL@iuhnXK{
zZkV}YW}kVETF$b$n;&d`usHz^*WEdgelY!T8VG7zzJFo<yl7`e$wLntC6RT~T-SLU
z-la``!RA!%SO3##o#NsNfhSg+@-Wc;YP7;g;6hhJSc?(U{M|2AcPSR%ZFs&%;)K|3
z)h(`#eE~c3R>YWxtdmx{p5GB;?gC=)PJG>Vtkn7Ou7!nn4?I7|sS|&~K}6?KydVQ-
z=j&y&U$`AA_1pO4kQmDzr7NNv*Kw4&ZqXKh-o@drP>e9(Na^FLyE8R*oisWxzAi>_
zMr(o?m!F2k!aGN*|7u+?jd$bn(a<QIZgPD~fS;0HbJbj-oXDLV*LMflIsN1a-xJ`s
zLrii123J<q7#`<F&ZnuyOSd^}oEbdZa_iAY+{YGuRF<)S^ierNfOnyu{M+Zn$t?^1
zbe!tvx&J{gqsFMDMlCBQRl!>_ZpwZG{d4=fJmMd%lQ0r_)146S@>}VS_{V>tE`OAs
zh#T(b__g$&sYlzPx|QXULO;7N#2@*m^h5mM{*HgHKlI<0Ez))<<kVQlc+_>l4i1s<
z#z#g0zuvlpH$4gp_!X;>`XVVHhPUgq<IC;Bv4UORPKCZ2dR#|c7w*s!ncixYr@qTb
zWP019TLCdLUER)wr5bwtsrUTmlpB5*j8*RHb}5{wu}<h{>Y_EbJJ*REUAkyTn230{
zfU5Hpo~?&=WF>M;SB!DnucdkUg=gfIjoS7T&s{&mWWN2-dvkrkhlb{lW_7jBS~6GG
z<vRbO*XM<U9V(vg5B87%uiy|pBt3!oC~s8bgR3iUt+>46a>XjQxqO$;<;zxGTE`QT
zx^B*|P@WyUQgQ~eHXkPxIDqy#`Q4fU9)<q=O_Sf?p>ZMzLeih}30v_%(Yys#tSz(e
zo5knN$g3_rEV{O0g(&~lM;|}${D1Rc&9C!&)w&+m#Gfuyb#x0?S!b~4@q5tpoVL7|
zOQ*KCUE?jkz`L5StW)Df0-kDyKNoz_?Gf+tOX<q=TGO;Z1()9fZ=JXBU$kTSE%3&9
z1OLH$8#>c&xESXwk~F)>GdrWr*W|F;tVH3rFvi&poxzc9`FrEq^3TR?>0f<2{^`H0
z@9#VECHL*Mdvjz(`RWJCUEdUMwI|3o)pC3*H2n*;zy)f7-MVA)i)uSSN;1GoF22t@
z=69stwK}Wr+Mgxi7d+hR&i$!*xYl)z=R~Pbv(32KSoyR}bM|wva~^W^W1M}V^B8!z
zxb(RDQ;!YWi(&=e1*~Z2`eI;z`+-55zKX!J6Q+hCOWM1>1Wc6Ey7c(?pB!i3su?v&
zD?Io`UnaimT<ByQvVy%mv*w|2$8^^PI!-EFUjr7(J)3hv`(lbh+@eT+)7J}?Y|rQ#
zhOB7k`g)+gM_*;m^b6_&+T96}E~k}ZR#ox`bMu;hSjcAQ<7ar~Nc*fG0mAa;mPZXN
zj|Qm9FB6kWW0%c(=6tqh#ms*BU>@FJ9^TCnJewnULcc9MW#?mP5^}dayy%(u>7X{H
zyd7NaC#03~JNB#>sB(V7JN?-X@6XkN(+<XV#5Qi-%p3asqp(++#>|ISGF}-P3m2xW
z+QlEt$?GF18|?CbXQbwgpkGTX1DZ6OF04>6(Xv{`p|p6*d$x&zjwvA}zJe`otP2A#
z8oCNdin_S5f<=X1UF8H9iJ)Rp?+S;AtevbKXjBc<(q=YuULf&7GD$3l4Gcj=qA{pu
z042+#b~k)(_*kbO1i>^=e;b5jd0Njmod->nK=8I=vl_FSZ4osGBTVXA9=qGry;*Qx
z_FmG<#Z&B>J}GP28`T`%E&J88N8?C*PyPFt^*2s`IJC#&e^*&(sCM^4#kMvHsicgK
zg^Ddj>&s7Ce>-QgWa%H36?3+^*PZ)wMkf4oJI7BY*%sDQhw3&ugs$jcTDd6mc_+ub
z{~{asW`t#ht=f}UVI8CGU!r_4F-|cjz@$#uSI7MR)RR;B?o@>D__(g%aqPoYyAOqy
zr>-(Dei}Pd<BV`?<s!}EkFgJr?LH)1e$@IMSNR>$cRRFW^1^rASy!+#_F>WPL#Acz
z1&8y3OF=OD>$e@T|2(?9Z0hZ<C!L>v{;U7z=e4D0s@KTqxWE2-(xKvQ;dgsU5lM@*
zqmItWGkz3*ZdH~OvYL04QMWIn(4{+hrQpJBmlslkyWCpVA((Z&9I?ty*E=qlDwQCZ
zU$CfLC%7;kQ)Njn$6s_aA;zPd2{C>@itP}UQi8uwY=@}4-f>|)!kzEr*6e0~{qEqd
z>J4?@6YknZl=EM^+gyrZeuYURseC85=03X0tMiuMD*k0<SvhO+GRxT8tl`V6i>H~X
zNAAD8(^`7>oZUR_JT5zxPE9SWzi%Y)p(`M~W%H(Y_r-6dZm2y}JKrHhTPd6)R556w
zli%k>zNRNuvRUSTdbo0x#Z`gjQjcC{UZ3<mDT!HpqFwf~&KQ?D8@3<5$7gLF<8a8Y
zEOzd?)xpZ|u5?((rZ0c}x}<jQy49c8tzNVAWkFQfx<#+q6IA(%xi>{laH{xw=S+jn
zckvEIjesY=^~EkuRXp}ny|l?{s+GAm&mFD(rgh59zmpEe+*tHst`>Lnm87pv{Ei-)
z+tt3|a>(*m4_Mhs+3vLYFZL2Hzc2O5L_%_xTid#cjNW!n@9woqzW4k3_Qr?*)DN%T
zzW?&clOJS*l)PN_N++prUG2C-s-@d;heS)a<CX2#RNcdy3rsn}n+ps%#G4BYJ64}=
z2P**+AaySLtIMn>KA3T6_w)3#{yutNPM$bFbH0AkL9>(~=epve!w-I09$->FeBgwE
zfe}v+4>O24VOj8=O$vLC$+xqcG0vX-jNgpySICVG8>MpAut~2zcr0tf#!H69V165v
zk0d3Pv!+d2+lC=|!^T6(x?5)Ig&b(*I&pMCx{%MZYbk9$OX`)r34=Fzc<xY$3D6N_
zJjD92t-yJQqJrgDC;ONe?c9G0R!ZpkF899tGAJ-6JTOM9%lkHnY5#sVYj;l<&#BZ8
z=OVS5m72G2s`x)S@P~@{@(U8vw=G?|Xh(?1^!6>@tc5!@^nAU#JE9dXb-i_3*5hjA
z<9YgMVWv}*eVoQ-6Ioqf_tQs<GM%FIW_br+x-?}=-5I~VF}@(Z!k3ClW7n-aeYJGw
zu64V2>E(KR-@a6|HFnmNX=+QCa%N^)MO`o164N(*+2W-yGj`hOPk#;)b&9f2TOPOW
zy7%Ramr|yD+H;9x0^5$cv)@cx_*l|-N8jobnc2n7u9Lm4)b}h8<6+`fx_-26-X7lE
zrNUOVZ0h+MDF@8;>ONQee|P3($b_d$jh{>|nk?>GG->KB``yK^zaG5$^lsVm8^@<!
zx%kj+Nz$^@-71{+KaW1}(Olp0=+UcL=AYl5>DkWN7&|kkURHh9Bi8e)GrZ^gFqZqe
z*`Xlq{?u!q)k+^OpXtBX*tGWev6<o>Z1wvl|I^#D<nYnZe<%94SI&^xC_3lbr=K%&
zeOI^geXD%9n9uvp!kR73#;>d5ZaQ**UZ&l4b@K6xwj0h>uuYk#m|nJKPpVSI8s#s#
zbJNY#w^*9oEuNLo)*e2o$T4l+44=N}b1%X_nJtelzjr=$(oP?fINirf{>4cCVYje}
zxu>d-bUEgh(c3?pLTsilJK3}88~a3Umvt4|VMYQwmEK(}pMEfOr^}`tyQW8lccjcv
zo6hy(lF7`2m;2W3eB-nNwAWx;L6z(7No(1}UZpE;Sy-ue-uv^j`<wkh$^4Al^Jn+>
z6khr1d9d?e>4A-1(e-IY?8~ID7AGIOnXy3H_}sGtCCO)Ru2_7V`Esw_uKO=JGhZ$H
z(7Y&U^T8=0k7s9Vn|ZBT>EFSvA+1{aIQm5Ef%XlD*9E74c>Lfv<7LL}jN2W3gL?DT
zn7%VlKl^K6<o%txJ92pJXJ4%==C8P^^FjLX`DY>`Yjk2XVs-@VC|LMGF-9QoxMJhV
zq<ybjyw1PbTz&L_<kNWyGX*!#p88ybGv(l-+j^V&<PSdasB;QDuQ~nZ#&Ws)X(l0d
zn<Qn5x3T>0Xxm=5Zh7U$w(Xs=_Yd~uzqDBRe~*65ik(hR@@H=`(2chH75Qwc;+*w~
zdR6)B4W>Qgmbtoi<F@%%muxN3^RqrKc>PPz@pDYa7l-D&k;wGWvgn<2eBITZ|8{hq
z*(CXH(?_M7*M+awCVp;uzNcB=GGC|oxn`eFUfbgd^X<6Ip1Oy=ej?#va{jBuSO1`A
z4;mjO<T9z(pWPBKyPxsuLF@D<aodbL7ti=Gzv0z-QN8)98GmX*ZDU?mT1}rB8hx+(
z(mJ*oo}EEj^B%<tSUFYyf3EiBvy?c$kU!sd{w;Ek*|+^-UgLh?OIhODo-_4Rl6sEh
ztJKA>5X_#JrFq82Ii+$2%ly>|r_b;8e^o1d%D1sjmc>@3>ELaz%BlYUySs~wgqC+X
zgtn;VTiwxGaC`IFwXH?jOK)l|iEb_VKcz~tBh+=nA*;=yF5A0aeDf(@Jt1$c+3$Io
zd7_usChZFK+1~Xs%<%5K%)GDXebvPGO>oF?K7Z%c^xHN+qiu_;Q+C#$nZDn4^_^L@
z+p3<0oxk&HX0&bb>iBo7!}GqLS{t|1b`nVO)v9Whou<NXt5rY(dXH}_pL|=iFZSH_
zTlcQ@ZF#>|Jxgl2PWi@36PZN0LNo#jSv5tL1Wb6e_l#$S_^#cTT({TkKXklo-)yTL
z|K!>33e7shn!7qfLeA1N-n!Un>AATdIV?;0=0BVFjNygPixVE_`qaL2U0krPT<vzX
z{QJjsy(<>a_4a)xRm10(F16{kW4YY6e<^HjdwBP%`1NJ!tx@>sc+aH$_LEBCoB7xJ
zV_JkZl^vT<n7qfk%rhr_hVTd5w(5@WYu@~3JE5durk)&^YPVA~t@ugxkr#1F-x5CO
z>sY4mSzf)to!4r5Q@b9ozy6N<G0V5KbN|)ydhzJp3gdgq);8bgX(vDAkNZ0-)%{HW
zg1KK;O^AAZ^63`C8*`PEF9`kid+ze==;M@$yO-ZmI#YYs;Cs#Wxwl)wcT_F9VsHQI
zpZ0c@pF8FRPB_%KBmCNIpEEOzis#k;jcM<C`0O<A<<hFsMIjG0SPlq02=LxM{e2Ga
z(&+T-YiI2W^*7dC(!vzSQE~CUR%YA%`Til38j}tw?K|+!ouhoVgX7oNt5{F%Z?Zbr
z&p4f{_FKoy4W9M&n_cZ!sm4gix141YKjxn1xu^QtiICHg7lMv$Wchx~dd0!H9}cF1
z(8Qa(n<Mrq)~M<FX{hC>wFq~49zHE59x5Ix{zk0S%*Q;`e2=;5<s6Y1Hu)H~^Ld8c
z(@!4XSNG<PdWgvUSK?*=lmFij)J)9WYwhziYgYbptMclj2W$VUgN6h&^qMC$E<ROv
z>2G#ZdyiI6)vPN@6W>|br|4G%2+g_5xATho-`9J#J(t%t{Z{;H&xTZ$HqINeGG%Vx
z+1JL^YF#@1k;Cnl>#_5{m-*(;^v%7y_QwOOoy<(%MOIkcm*y+}Sr<Haj&)ZMW6zvc
zPu6|DIhry5${4TzdEIH!8|U}h_to2#RZBlV^-(_easG*?|JzNf@6?y;-#mNPZ0>f;
z@C+rRjXNWLeOmi?o}Xarti9G>npf=ce06DkwczuME1kO&+%L|mOg<DQZTo(*koCrm
zOQYVcU6H(aWm1Cg6#J@Ak=<;?iBDOXZz{f(Ei)4SEA{=R+{$Dx5tsbu>-5SMGWYEI
zbyejt=VPImsTbx~b1T~Jy5U}EcPHq{hsj>vQLZ89qD#|q^rE}=eh4|Tyoj;O^=~1E
zT#Kemi_ToRvsH&e1=k1Ye&&Aezx`3cJ)ZdqCjxG0iEk)!F4`xsGid3VASc5$97(Ps
z+TuHcrk>GXqtUXIYvxjaHPC?Yo6`;s+DcJ^7f<b5-O#cqvLj6~W8<gxnZlb&HvjN=
z5G1@Psw1f|P{(DXz}`cx>Mk31a8J+>TNII1ck+RrFIfGo)(kNaV`0n&mFMC{tBfGS
zBwH>#l6q@Y@LrzFN5f#@0hj&RE7mW#<FbGKg+zr}7O%M_OBOxhs=MSk*Wch_%hGv$
zf=}N+n|Ley)+ILS_-)INU&))PBsPCtr+?Tjz6kak59}r9S$$q5`sR$Se#pC-^;=&o
zoxFRk2g`Ze|4LH@Zg29apPOtPb+%XbcG~;mfaNitmpMsZ?d%qPG*!;-=IQ?1TJJxt
ztnArh`s(RV$>m3Sk9IYG(q49V=FLk2yWdA{o;Uq!?U%`n*H6!08ol$ovC6-St4US6
zxA<(C@~u_*3;S-NgK_Rp!Z-RK`8e(35v$ebvoHU!c=$xm#3}cIj^@mgZwv>Vif`NB
z+Hw9}fcx9>D}gVhzAkx~B7AxONyblXee9|47}Lw{U6`ECXM23X-Ma!DGdLdEZtvV>
z&A+{#^K?nFupXzf_@{Y`<F6lg@GYBu@?u%=+jUkJ?@RWl%yrHX{3t!CV}<y1-qzSu
zHG$6#XF#hj7x2GG_Tp$x*cS3)wJ^^w&{{CBh}bH{l9~Az%wNnl7Wu{YclAdGUOUk8
zOHjX6_7~sZ)rS{2-(M&nA{!$6$L!D9bCL%0L*{OnT`*e-v{{DJ@l>VAl!P^Z110{Q
z(U`_6!p$ZAnMbp8QChU!v5UtIYh;cb53HG{;aVFqrJ&MW{D8XL);~3iTv7`9ZT|LW
z%dFq3<IO4i@6nIN2gF2gEaI9V{=wtHRkNjWOz~$0_4z6@um7u-w77TlW7_pOhqoGJ
zFY`WrX7h_?k<4_1Ioa8>c2-WAe<pi6*YeD@*Ux`lUbSUn$eQ=k{VQ#*oIH8tt+Dmh
zrN(wCS=PUD)DGJvZ{4_ie)xlsb>Fu~+MKw(DSzL$eXWaIMRzYUxnG=l%Z6in=!SQO
z<<3jLE~`5KUR%Qaz1Z~COQqAU&-&<X?EmFr-utMUhFPDF${b&nxO(N|Y~4$msf+pE
z+8nu;))lW4HB(R4ru$E)O8LC(-Jc(Z{W_Nz!Pm=nxvq5ezkA(ln@o2tS+#AMr}#7e
zWj))IBPy5koZql!>DzZY$+J5@YK5PEF{793_@VR?fAyy|Yb=&;GCrFZUD;YyoSMzx
zI%~!Hz_N$cnPKaa-e$~s*Xiw=prsTnIIHW%k=^1vZjoI1t~FXrB3k(y|1NqJy<t(B
zrm`YWc2->>V|?48NWpUfo!wf6fvWr3kN<4#Y1w7;{?fzAon@{wv|l&1d$ga6xiYic
zswmK0Z`Bdk&>wlzy3)287tY)LyV$WRrt8Rlf&Wf_xUX*6^Lw(siokcLJ>5l|IWN}L
zi~oP5*I52>_v-tx|6)EYy0`RF)uFrEG54KnxIgONiVaZ}c;+;tyG;CA*}=z^%^Xi-
z`(&rKOrADhHpB6F%+HschqeX=1gu{m@lx*Zu6ud`Ex$80nEo*?^vcoUyczpWp0D|?
z$GLv>)0IVP0nUnAPdciWzA3u>#7F)>XYfg;B>(&Gdi4E$t%JDFU;chh>i2{EeJ4zQ
zJo$3Jv8!XMhOEJXx9k3D1ux{<RpnOpZ=czpPdk+3?VlWAl#btddPQa7qp430t<2KS
z`IVi=TXQ&Og_~3%2TQ`?>ht24qdr=`J^LhI@8x9UJ&XR$G}sr^zhciV!M?L0aW92l
zx?bGrvE=2R7t#;^u1(*5OLEeM`WFYBKTAw}T9oQGdFy5OqQVKq1$CQ#EjcRrx%yF(
zuZ=v*?k()Ms~)bBH@Rw3x_qs2V2*I(?aDPSf2)M;p2fCC<xOK>ykpVa#!ny0nWp<6
zmz!bxM~d^g>`?^+1+lDUxi!xx`0K>0&la6D*IDSvx;^FZCaP<HGP<U-AmhwDwZvpU
zlNZuodRJbAEar;&WL;zRMfTUbE3F%x4ObZbcx#$qx5D_2bCl?_$?dZR&S?o9NN}=W
z2w9%nogRH5dHqpOEv8sgi%^af`+!L1*SQIMd-q&Z{@+sPyno^S3;DP1GhMmbE@${k
zb6V3?Tc?UyOJ98Dh(CXHy*oqokv-g1v)C@SI>%bn+3k$fYku9Ra!d3f-(2%6rOw6S
z8@l@fw)SSf<0`!qa6HtICwHabs~wGp1m#Lqm{S*Nh-k1b<P%u;^Wm)btj4Okj_%)Y
ztcVuAopbW0*=3auPu}vTrI)Szj#TbwS@?6;w(_gLpFf`)r{CY`ZujDVM}gRp&^$h&
zzKvq*l4q_q`TAs8-TU6}mPXIFsGFbGK5G1t)!3EkP_gLb(r+o!@$c5HP5ob?Drvan
z>f}2L(+@n>^4|3{L~vt%3V$2d#Z#Pfj(3RuaXd5Kt)ox-z|U6)Gxw~NmX-F+duYs3
zlzMM@aJvYTW%8wiiN2dAe01HXp)|>|_mk?Yz2Uj#>mNN?y7PbL-zP<%yw2(b|5$eD
z<`FX=`}8g6ywCoNJj_)vZTA9CokeUDjU%(=cI)=lT-=<#>~GA>fUKqFhtfFOPKVFT
zw~xEyW;Uy~YDY|~ef-9vBhkOU@|_G!{l#}Q%0^o$LbCjwzj~KO*panL1yd_SMLCbI
zQ_9*KVB6gMf5&3Mta<B9BTr?V%(y#KeU|9MH66*W3ffAMf~UG7!oJP@{d<+a-rGZw
z9O;U4S`EYwMQ|kFxb?X^wNlhj@~$x7)ExrnoFp!vk$$ypVIG*M=1s0T=bmwtH+j<G
zFHM58@}2LRZMoNGm6x%X*S96nr>L%QtD9s;`9meib(0kT9X-1L!VO8iiHdx79bt8D
zbxF6A-~SSN|6}P6)jV;Q%>foGC9l-5h93T_9ooFg@v7J2sk5feTJQCFs#UL*Z*_9z
zmSs`fcF&%5<AsHpvB&ZWzo%^8HLELe*BU0{3dW4rw<MG<e>P>bZms`cBP6<GY3@#?
zv)NK9NyTb2K2OS+eeUObQy$l6Txq5U6>}K+XNvq}C}7?FB_`vo`}4QPOV6xlJMm@Z
zzerm(udh`vH*NajQnl%ANy6!QC*G~<`1^5Z_xD@o_4m$w(=~ly{M?MIV%~&}(HRo6
zivpP@v3qBKpJ@Gozk)}MBV_9##$vPT#V+sbo~dP@NW8P|>gAW$>%OQ)%Eaw;*>&@t
z_yltu-RCie+VkAya$e7re_a)jm>#jpLe}}jN4A=}j%#r|>+U`0Ijy~C#xB;*XW_-u
zttC31eO!9%?$Ml@pxYTbr7N!WieC?1Zv1@LW!|6fPu06$u@;$|X)uGWc@^jsgXW!6
zFP?nve{04T!^~tS!<9xWH$9m4Qb*bL+~yBoBoFaETy|<%7yFe}Q{OG$B%&r)_()Jg
zZuxuVB%9Ah0@iYg%TBy_<h%28+@WsMU(9>D3WAT!EaF#N_tJNT<K8v1f>tkIU1{Ij
za>(yvUBQ&DRj;QgtybrKntDM`?bL#k2Nd=B$_lr1AC7%jF}?15&^x92P#;d&FGhN=
zqUz(=KG!9l=F8u?>(!^Z)BPXp5I_Dmbe;QTp#z6vZS&TyyB&HtsPyl;b+;><`}{W?
z|J}Z{{l->dgJg4|zX?Urjgy~?v02@I^!d*9C%+5Y>}PJ6?QNI!D>!G%iIOUXd4`Qr
z%ujB-|68%^%-jCWjO8IKP5(D-+_G(xc-G%T50X?5tS_&>v;P)v<U_xUzUoW6jjrq4
zNmpz3o!n@Vk^J&Rz^g-RZ8SZi?zG1Wt(d#;|B1})%P(K7S}yMwZuNMve{to%nU{7&
z)XJ?By(+z)IX_)a_st?{?dO-AX3SjmeoltwT<^o5BK5bQ+}H4Fn)m^oZ3fb^w!Zg2
zem<u;IdW;R|8?txqqjYGz5agH;A6c1JL!{kVn?!B?G}8S5ZSd`{q}9WOP>-n!;|}5
zKc3pN{O9?Tsi#vtZ0=pWciH8z)|74gv%Z_HD~o!+V)Ck@$WT4mJvvu2ST+lsdDIli
zk)|;<@4D0NifON$nxZ(e7Ct+|F<+m>^c-j2K9|P^o4<Ijw^CiP*C~WMbd|srrvPq8
zU8v42tGC`<v_PyWYDYt&{nePMPivVj>av)Ai&eG1`KS4+uDzv@-9+`{2j<V0W_rm|
zmL>k`<>F4yt$$x^lMSd?^{h)kpw0dKh5bxyjI|E&3)kB%`NjU_>hm=Q1-ozF3BHu{
zc`s<$NOVrng`~wt`Su=vJGad`$13LntJE%s!*xruL(;g<fwqB2Y%i9Y@M}hwRC4Us
z9oZ9}S0%l$d4K#bgM~A%lZU$5nGe^b_^)5?>5?s<9R0s0`16tvOSd)0_=-(Zzbm%g
zZc%zrZVfMgUcbz%qA3F3ehX`RU6$po-P-zY_l-8`YnzP!-%37lCHYD{AHyk=)#q=r
z>xK$^=194}<m;o-xxWippZ)ozaCsm5icVhb>nb@r*bXL@T0F~JY#qZoZQ2a?+s7_Q
z^Q-su&6=AQYMd+^`c*Az<>O`bM;ATz<+~B!CZ&JL;a9@d6*~=bKFpk&^`WQk>CBG~
zI~znUc*k=uO5SoXROr%~n$Y4rzKI(rf8V*dGkM3EB)uHnl?t<(-Xt77wIS)+RfA;{
zWeQ@MBZb3nf1M(8;zvXhlhe9~X-ZFoyH2Xiv^kd2=c8JGD8Xs}g7^#i(QCf{++0wm
zpV;v{VngJHn^5R!d*obCo9wh=(KBx%V<LpF&wFIDl|Nv{Y_8(q#A`V-mgZUXFZiA#
z8M8#^Xi%HDm44a2&!0{#h+$)Teq8?Ht&H5rEjou%1@*fdemX7A%evyroOd~L;mQbe
z(bvnI?uM+Gz47+NlCX@dYvsW==3Me^-L=y*N9)>Ksj{QHb#en+bevKQlNNGJO1(4H
zWuZ1#Y~Y=)PhlcE7OfCXUAyKt%c=0d16@L4A{#g9G3s8slB^zmZgS4!6Av!@b9cIW
zXXA4IOFxVJ7&lnI?sKg&@Z`*@?8ut)Qe~oh_e>jh=RCiiU&Kx~te-x;Vf~cFGdH&H
zvJ=&Jj>x$u<T-2G+?f+kxKDAfdN23mrQ158!ion1qBg&O+1#*PboptjF=O_xuS;eo
zGSpiwStfVxtYCPa(SoTzx|p@YrPp?f`OEEk;}gICvY(Sw!+~Ds_sy?8H*7j@zD|?1
z_K!x?%8cErxj(b5HoradB30~h%(6$%t$uD@#Ghi7v-CK3^tJ0p;<_*1&FILI37oZB
zzD&rr@@>gC+e;OBr+0F{7W^#iaY$_8W-hn=>6?Dmb<fpdw|%*D`l|AEmqUxs+Lper
z+;-AVy<F_2=8SJMUf0>L%ME(oTK2uRYimQe%PyrWd%F_CTXqUuce3DqsNJ%@BG2}D
z-IUeaJT+=scg1mp7jE6N`hI6n)%jVoRo2z-`8FqGmypTc^x3O6dHEU%EL&9fQ{62%
zH1Ag8Y%Sl*PZz8-+P>Q8^+P$|sXJF1nZMo8Hpwmc*6g$!WuecO*=}2D^;}i|`O{Cv
zpM*cNah+S8ZuqJA+4EE9c_m9&4>3J`oyOm=%^_xi-URU%y_+vszJ1y_mu*^uFL=W|
z=M9#eI8i&$rZ30=7>YRpDM4>uSZGgLuCT*R?ZW|{brU)DD!KC}pR8-#;Cv%<<L^hk
zhxOm^=Oxc#>-BkSU}NT<Y;|bEoO?4>)xYo6;LBL1x#Fei#AVF7Ip;e3R>)2_p0{$-
z{Ozirj@@5-$Ku)FWoz6!S1+sc^Ek2X(1#b>Uv?R6c$6qyWfOJ&u!`KgIjU8@3;vzC
z?(%r8=&j;?%&WdWW4PpTR@7=|^!)Ea*Cws34^p?@TezS&@$Ku+_v$CBP3)i9X#A!m
zYhA_aB@ZR?FP&j++gX~H*ZwfmOW7p%=uu5xm;JjE;zeIQV?8~`VkXBsjkLdc0n%oV
zd*TA`U)GtjE91|d4-2MFZ)ZEHw71=`T_pVJtAjT_2j5%wcjv9#Ytz5W&Ds3WJ(>5U
zz1aJ6Gjsm$yxHq)J=s&2W0Hz}>b)e3_j)xk_D_@#d(@`yDrwoT{KxsBJjZ{BAMA(h
z1^y`}ffm~9aKa{iK@-1;#)*@^y9qJJ#Vo#~R?sq+HAgIG+K;UdHb2<hVsJy}Mh+OJ
zcvP0~T#i{PvTIGj;-Cdatn<uYwq(6uP*#5SO6=O*t4qtbzFT+g?$uqpqsq+{Y%hko
ziN~|v@UIe+jO`Fjj(@u7jdkDKM_YRCspUxS{l9s4w?XtHqfebIx0Eh13B~5_S>5~n
zwbh-_<Gah=t&6%BdRjlE>de%ocl)F6-M4<Lq#+(CX}Wjyvht~&>vrAysQ<Xzn#<DP
zXtGOI6}N%k)_HpqFCOlmoh!GdWA@fE!R*cFWEbvtInsRa?TVFi_7yxzPn!Q<EN<<}
zrpHSIB#%zh>^OfoO=PWRs+`y(r?{Kh{Lw47EWc>F$zDw>#%zkY-J^T6k5`;XTqRgp
zn8@zW=KkyP^SO(*#O(DrKm9WQ&CuYj?ZVqzCkM10I^WM7uGZy|Z2D|}slP%&$8D3D
zU52|4UYK51+4Sn|2VVI_D;IuEEV#2!tiZC^GG=XwaPwUMHLV@2%AuRQ%#MFP^2GAg
zha!ICKF7L8^1;Dkeg7<`@7%05H`Msyspr>DSMuw-6<<75Q+h0EpT4x2RM5h2b@!5A
z`UanRJk3@2fcLCtt{h^QE~GRqPckT<KYNzquHWb0?b|FZa@jWKt=yXu{<2-aFPUty
z6ozgq=~=zvq{kM$MO(cK%MahWyP?c{&0XH?^24`cPwrB8IeEZcXk+(;*duS0GQ<OS
zcic*iwTTL^{&KP9ol=?j!QCDAT)*i3el(FG&gFwr%k<i>7gtYMHL>fVVjE<83Cl+T
z(9R1-n<E9=<nAt6bn$L=$=R>5#kJRC?%$nU!r!pFdUEyEkQo~kcFh)vUA!p!*!6Yd
z=D~&K=O^9TDzpBR&WiR0a!plNR=1u0&{W0oN)f`>_zz}?Us)V`_MW@Q73TncM_ZP!
z0#_E-i}V{GGVbLUUYlNAa{Y(1a>*jiD$UBwv>2AXQbkb_>q6zH{y1|>GxZmri2qG(
z^|Z>U@J$PscK&|YDOIp%>KvziZ|B9m5RK@0-P=<9xg>Ufo>A!i)Yu7Zx8AS5)7|yy
z*!3Qv^O3ErN$L%ABd^R@&u#rPxP-N8-Q~(>_fOq;Y4X{u$Zn?Uv>SbsHchwwySq9<
zqMB{1ag$?r)>)2IXXO%iUbcN<^GNpBj|uLdB95PY8FnT(uuxBTcF6A^YxdOavSCco
zo|I=YTO=^zWBQ)ytyB5dtGm2^aBU}>*>-c&#pQXwW-*m!NSw@Yy`=NtZj<B6;(asq
z6uui>iLp%UIuR7TJ}t%au~UNgBFDI2Hea6|NX{?1T$HF-DE7sgo8`-k^5)Yz2}fej
zewzGOA?%X)D@pyP9gmndD^9-k_-;Vesv_ToZRzP6pQ|n%P!rj`qg#tLXUlcAoHv&n
ze04Y<20wh9w(ge|bYo9tX!+6)D>oPxYz6J;xT*WiyV<|_yzd*<*=(~x>rC>Vm;5OC
z0p8KGH$UNVicH_$bV+8=cH9eRzfPF&?e3|xuGJ5=fl%;;4<#uJukX)v@)uCz-1w!N
z>(IHT)9aE7_8piTZg}HGkI$Nb4t9^gLT}#LytCgvu$%AGeY`(^aov%t2mRMe3v3M7
z94==+XZehs_fMsyu2;DBF71|8_Mt<Dk9BtTR~{6Li_|`t<g5Di=gyVeeYyj`?VWj{
zSn#o<(PY;Lu5)I_J$hcuvpwI=Wy8HQ`sv%AY0g>r>ZQiDUEk89-&^Rn9P8~ldidO~
zjvH|rC(|c46zjcHeV(|-K-QZ*dilY`>j^eq@8tU#%bx1qxW3+$!6;~Ulj=(Q$!x}I
zbq`Y}U0^ZZ{f=v%QQmcdFX=IRZz{i?t+aSq&hxyM%gf6aTJW9~l|DSn+<8Hi{=S~6
zvL8-eyX<eB&k|o%_+oLeg-7+haF6%$vgbFNO%M4aw!3Y!)P0`)vt6$UbY9=PCUKdX
z{kl`L5A2)xt#?cQl3MMz(Jp%?XsE@_pFUgfOPX<Q%hBcC)6Q?5&)wU9BjeeHPi&Er
zT$``^&fz)wv(=+|;kTr>tkD;@acno&`u>$O^n>7i_muMDy8n|GZisG)?LXNjT^zng
zCspOuw5o6y$<Ezx5{gfLxOn8A(vN9n(?uk?_PhQutti@ID4%zx{kHS-WFFD^3m4C7
z+n%DpB5t^Y<B;nfou<Uda>mEsH+(+J@p!NI!&H-pLY9&Lc05&8+i?D^&Oy^p%Vqvp
z6gW!dbPK;%uQ)$f*YM!)%rMow%+5~F?$c3Cr=yti?*(7@x2NUxbJ-6Euj-zu5MsS-
zuM(<a{a|v@UZW*C=a&VnjQUkHH~gN2klD{Pb31L>*h=5C2e{)Ca-N2+mUhpxJjBoF
zd^yZmJo@Q}T`M<*+Oo)<o7wXu_8nW+D{)!V(z`QmIBlp`-hJHc`n{W5r<Ys{*knGt
zew}|s&{s{bD~pw$p8WUi%kqaklV2*%*sy!zzRG)dPFh;dG(2CmdRjC)zj@5ciJO=E
zzvl3IX7Vw&{!7k{_a{ZwRuw%D($>G-@aKzc&6cv8x|O$@%X(IG$?f64ov=&mo}{Hb
z`?4R4VpqLZ4SiRb|FEddsVz45V_I---nZSWdRFDCPGnSXUH#DK|C06oQEOuE1Z(9_
zl5=`B``fG%VV^TIWd8e|`Pw+wuEWNXwJ7(ONO00#pM~33a>eZqI3lusd4+B3Q%#-o
z+3Is2ZCw0)Uq?09?$6IBf4=+c{fuuD9=ttuyUuRe`JZ3s^RVy+m~9o4ykec08t^-v
zDSO7EFPyhhFR0`Q?c>c6GUGfe$JXT+SfZr;e!;2j?SHOVJ-Mdm{kk*B^-tBqWd>V+
z`K-0OoUm-=#q_K%Go@;m70mg*^+$(!;pts*kwRUWPn%{jHQjhtt1G0T@-)ro_P$eR
zw`4SZTC}I-(<d%gH4%{<<G2q(aUX=_o*cQNW^5|^tX2Q4zDh`e(?L#FbrG4e#_IuA
zXOGJUzrA@|>)lmjN!b@sb|w<bH-9)>XBe|>uhWIIU%oIDWZLO%xwa}VVo9~n*S-a}
zt6Sz5Y+`)dmo=d(^R%_nHtw{v1p(b!yDU=wo%r<G?@cwoq`&!9pU4&Zp0nyVm%o2i
zzwDKd@>a9IUrf!`D{!8h{;O1NW&MR&m%?`^q!-*=xYaFKr%d_;Z<4P;M_O6sr1jh9
zq$LLZ^4TG5A9v^9zTKN+dgs5K&-Q-O&8Om%ZrHqgf6sR2vZr@;^Yp35xirmjzt87&
z@mOf@-}9kn5?yC+y5y(4f8%xIFyq76>~~6*oh=^MGemy2KF-Z6wA%IjUuFEWnR#tW
zk?+oMtT+*K?fF;V*Yg68pEt7=$=_=8`Nb0v|MNem?2_gvINacTh5Mo&%Wi=uP7}Bt
z^;mWaJn>+x+9z<wset>SF3a{EA1#_<zF2Q$-MVA1`i{g>rs?5zyY?mJ9%4*Pd><iU
zB_`JQb@4TwRqXL$oZ4JYsm75D*9S<vxcuPmkF(5DG5O+Y?2lrT6EhV4nG~Gj$y;H;
zx_>XvC6hI!5$(%o*_{1Snpq#D8oY0_>Zg)@0xSjQFVuJv{HL?BtoIG|Etl>(o16pM
z9N;wh!=87BUp_y*D-h+KywfLsYO3b^S)Roq^QyJ<nw6M0Gfc70DZkStEyMod>(0$j
zR|OjxX>~pQvnulS^Tz=<QUh*RPtpshx_tJp+;{oo)91e1cHZ~+^n=Dfw+J70*PE%a
zyZf^BrQ`)CLYII0@^6Cr^G{zi&Rp0ZbNzSGY}cae@67g0JiKb}WqSh~*SXe4@2@-)
znY)8m|Ellf9LLGKUuLuU=3mu1bJbeZNPHvfJu?w)qaEdcAKB<woSd_@f9a(8FP+U+
zeKxFKa`s{3<9D3t^Hww7{r;w4N;dPk7eyr>(wPgBYs8M-xx)Kqtz5C*wsos`W@a<D
z|13$7n4$e?!Rs%MhwmA_*f>LOx>?=c*?D{NETUiC$P4~bcP}zF^m2FQy2(|W&M%rS
zf9NLH=@Sn2@o9S>+nOi8`zjduEh}$kHE&AV!F20FmY1Be_GfPYy7}_G_HnL#4xud?
zWgor27VXT8&$~4}>e;1Ujr|ig#5$!<TWN7YYkKvErQa%!Yws_Z)E(ox;gLqcN)f%O
z2d(#=KGFE+kb0!}(tORBZZpMOEg6p+=iJ<SxP>FqE8^Y7=9Wb~+xC?vujReQr60+;
zd0~6`>XwtoAHB=oE2JvP@hfrp(X;W3W}0k?vbBGv&hyD{{%Y4cn~6WaT7JHKG4%JN
zBgbaMJ?1tRPAu5#S+JLLW<vg*{2ZRQKc*i2Cz#Jssx~{N;HIa*V!=C=pH^qsq%`|-
z%M06qPgdCw{8dgbaC5M6zC-8PN$y#5;*T!qWqi2ez1Zgl8IQASt34ICy1yNEwUkmY
zR%c!QV(zLNGc4q5L$2k={Xb^3+2cUKrnyraV`pBQH~D8cd){*Q@Tbq0tvPSH_S*dK
zcT;t3<%4I}2tBumt>582@#kL819$gGDqKwwS-8b|vRp#7tGk@$rY9j+e^oOq+b8>K
z%C0BAXY}T1i-q((TWWq|7I$6g>=~b(n14R`Ea$oD)<vO*pVW(Yi!*$m-=Ck8=oHq!
z>l~BdjXopSnsp(!FK=q!v21qV-NMM35r)dUcIvKtYAD+^=h>e-<)N1z-CGjqT3_@o
z;9%CpSUaob%vyW%_RoL2i#siU^WB-hl6DFB=J%{jh;^<#dF6aXPuXT2^SmQhloJJu
zpJvsv)f?<&6SAKn&Ajt`#qS4k^NkNr{Ly*D|CP@wpH({Pcbd)%iLBVX@cxyX8$SM;
z{<i0KK#pcisocirzh=f{Us`<f$H&Tx7i~UXyW?`R{@!(s<F@m{uH^swoT9)d+PI>_
z#Z{x~wOvx6U~AWp<cecuFK_wHd^_psp_xlt4hd*q`?kPl&YZ|wMf;<svn&b}?CYv<
zQQG|H(JiOi5caJWvyMJYS$SK^<<b?6hf@lEo#?A5JaO9hSHbsN<sUXq$hn)aE&K7c
zLwmng-Q1<0Q=S#gd~)wU<^2niEdJ@tyO%I&sds0QKscY1WrCn-Z&Zx>YF>qut~)Xp
zSM3ZnZf;yyUfrb}etd5F=0-33>yKvs+;zM`(k!eeYe(jN<^C(nZf8yFJfbf9hhN+8
zXwVPiQ?oL+dK#B}KhgX8T>9>`x4w3*XOiZum|-q_H=6nG#V1)o_uHdpo@7~FmHTAP
z^S__^G)xyydpIw>#Ab)B^d$B3jC-dWi~4=(p7~Ej+w@s=`lmX(cXujt_8l}|CQ(?G
zyML`>=JU9G_mAh2N@WDHstnxszq*z4zTl(J+1+P;{C(19vVQG4x3y<tW*hS_o7KMD
z*6qU9RS$pf{N2aw8pq&x=UK7}pY}n&@PFA)vUj{JFLHi5^Rt-!<;!;*E~zh8l-9Gn
zWv_pVUwHo}yJE|}YPscCZcf=VRU<XdgIlrpYi{;y@4t>W#cw<1bgyn}<#{{Rz(@DW
zX%nU@Y5$zRtG972Nncy|_s{lklh-Eg>})x&f8>w=x3dn9=tdCLQzpsLrYP1XA@M5J
zG(7UnjTzU?qQZBwAOFR%^X|e6?|iQ=IQ6pIr+~p*`Rk!nnOMCAD+<@PM;_1(_ML0L
z^74aKLN_XB`At0-->fxr?$S+vChauf=3qa=?xWsf{`PuHQqa`^^R+$QtGdlfTw}L<
zi4xG86?I{@b@|0BZ3WIdRPPAnt!4cz#hb?!dv^BGH^-(Fx-a>+XY%`yPapI1TWa>q
z_|D7Fnd{E+<#UDRq0^j~gI}taZF#|RAiHX2KKl$S{tb72Nxi!Mq3@CCp&dI#HlDi9
z&bwD(<*w;|(n}@F_*A}@FI#y0&k^nEeLQ_;Q)>>lDp+jssb60rufV$3vHP~RfV8{(
zY=LIs8`3}5l{|HN%b(AY7Wp}=F<I<v`%HTmGj_iu?!~Rw#MVE^F27u1e$LHn6Pt-u
zp;_)HSMM+1ri3KdUvLe4$0n3NrR|~BW-)Q`AePIwR~lVRyRNfYYx})*+poQU%X(XO
z#q_QRTQb-tPwbM=y+7mro}z~g?dm-Tw|zbskaFhUzE|E+r3JT_ovh=Zf9mU<dlCv_
zFAdHYh=#?<XlX5S*r_x1-=YceJ(~aG^z>rAdn`V4a&B8BG5MFvw6DoKJ=sD9gv)ol
z{+j4``cs|0^ZICwqzkW;e@~paBBVw2(tW#Sx2i6P37bymVVPPadxv#G(dK-8$u31#
z0nOk04sCl{+bGc_V06fjH_`gdnGMDpS;N-<6Y$bF93-&)@}hfHpPTmR-l!}TR^8J5
zFKNmT*|M}demtMs+JB}kzW<fUiT#4pR{>{-Ebr&%T@Q*W-#E*@{Q5%0hWjGzTb_o$
zGu&}~cVWNdQ@Q5g1GhJS@DkSdFY$4{x#IGVK>IGf+T}+BYp$q=ckb@@n0mhX(xl@I
zud}pMUc~IIdprC4mbjyH5ASdPeQ$PL=DQQ;UtT!B_uWe$vv;BA72RH{&1qbH&P=V_
zx%Aj6g$=3q8kt*W*zB08x$Z-Dmd1)xrS}fJ@c$<JZqCsua~StOe6nogOvU5N+T?G9
z1>ZfnL(n-SJc%vx<>O_}w_n$u$e6YxOYHEQhktrmy3`L?2(2+Ww5!Tahh?Q&wV~gy
zg5>3E9bymNaj`!?MORw*iIrE-JH-=D<}vO!6bjr{s=fcK=zr|7PC0kxhs2iinjvwl
zM}z+SDQL|(UwYK$t@+;S%XdPymDNw_JZtzuuzlLRy3^5WcTy*AdSBRj;=z~8(&pD5
zUhMdpab;D`a`Wke;dy5)?pg^vZ(PCeQmb%h@~X*<Gq23KQu9*&3FFp>kGie8t-7^N
zYZ;t<vFp{Pz=++O)=b;AbjrWt{$rZUiwYhpuy8Xn?%r3&F88d~N2t74LvBa7f5@uI
z%Qw{pdsQqp-D|vji}iKW5V8E|Gm2~EcFj1-eO`*QS2|&G*t<hP1?H!MnhLicx0~5K
zB`SE@U#7+VRw)lWrdIbZaSoF)7F+b_Q1F-C9zI{&bq`KUxZSW=WSMBfL(N$aK65kv
zkeoK%VT;&J=ZyaDw-0}`J(l9j-jf(NryO)LSo_uNSzX?<R{q&MW#eH<r#*HDnMGfI
z?VKd0<t%XMpSi`^TVEf4&Dgajb5qP*mG%0s9j9{UMltlAcCGT4{IPdtgCyJU1nc~r
z2d%<umZ+CmU43XVBh%=>+_MYs-F<$4i~Exe=eKGIp0%--%e1T5_?$6vpYo-7o@yJe
zEqgL0cU}LPKh6x_zPT8`pE&Qk&X?j-^>0}D?YftBpWR+!TGvsfEcVy$+`|9&lwR+b
ztX)-h*CC&0-88Mdt&ifupP#j=WSl<vr{bHVEeV~~x0bu#xl*KEd`4`2tar2RI<1!;
zCoJxtZm6zU5NsHh_eb$v?3bBmwto`&GVOZO;*a}m_VE<0O!Hd(Zslt2YVXgbr-WZe
z&v-unNSE&Ceaq)uiQ(QOYt6DqpyN@k!=*V)O#UlYd=R>48R&V&@#U2l;h942a$#5g
zy!m-@(P^*Mn$x|G{&jh{{ZPLm`xEU4DQt}m$#;4=)@X}7%5PodG*P>L!NQ5__s?3t
zf6<YOXkXR%dJxK6VYKbJZ-&xmlaNne!_73N*8E-Gu+RC1>XW5LCl*iYSUTzGe~|#5
zC>a4hj`o(xV(MRne{ufZ(!6l~h1HjrxVx8r_Z6#WZabS7_wnRfmB}krKCZu47w37!
zpXb)x=b{m-7tU3@r_nI?;k^}Q%F{|;>Ylo{w{h~NVvY6N_46Y&uHVW(TK=wMS;Ug#
z8tE~2ZtdJUO^omU`EzDhTY4TZiasA)nkJa;os%&A`-vs|k*nW0RIc!g@$pyjir()K
zvt!1>gw-}>XN@H?m#ml({?2y4q+q#uz~SbGPQgW+be?t2=i0dY*@aNmi`h@-sVK{y
zV6(J&e8M_}uaGllb7a6Lp$QLFm6klUTkI7RzjLSA?un)=1k>Hr<F=>7=J%Z{J+IcG
z(rB>$j`sl@D^In@`@6TiJ{VdM(zNNc*6pibUPh!a?A^%!<R_<)*>T>B>wa0>E&bEJ
z`b~*XSGGd*ytax4>yP$EeMLKkXU~xFQ=R*&=$nL~iBRjTuuF?mRljBB+3r4jY-NP-
zXYt#O&y}xLI$f&I<H}o;sk^>=PRYMs_d55$7Om73LH91NU2WU8@4(Ob!VW6VQ};Yu
z_~7a0FXf4ELRmH`O`2N9nfvl>T!!<Zh?mnFJ;GWxDNRdU%@KMkU@4EPMu_I)RZ|vv
zN#1DRH09!yi4x%~JC#lqvP_FV;nAU;ByE+E!NY0n(`vQSeAUJ+H!f{k^dRhAZ`M1W
z(?`wwd*c-^wC)gl6xESkc$szD?yQ}0J70#X^M0ADqkr&j)vi}}Q{U@VoZ>#taCOd~
z<slA>?oGX@9&C4LDW9zAx}8UDB27ySO;ziWQSEEPXHU)txp4*T&~V$|ZPK%0{$4LJ
zj-pzd1%*vQy3?!ooN_(d|J-)(OGCr^*L`fRKZtqP+0><P`OstXR7RFtQCq|d)-g|d
z{ATsO)GOyNow#kdQCx4+b=e;u=d~?5x^Pm`<Ab`n%XdxmN@I9_L#=&f{de9xm-6a#
z0o{3*cXOmi%~b#TX07mxBimK(x~gAq__kW{{K94K8lG)&MqSOGKbQE;H0oJbX?|3$
zC#CJ_Q-SAut4=x{P1wlG`}l^%zd2_`zo|X!n7=*O-P<#HeLd5ny+z*f{dZ*UTEC3h
zwznpn|6LP%jJnI$1CGG~{F$n<lM^eyx9wQfeP84KZx0LYUF!2+UfX4n>hLqS^86m_
z|1rvUH;As$yZYz1vi_gns_S(`-gLOtJ(ArN;lS`*`?>3yWer;#q80Qmu!=A`F4J6n
z$8w@xwTQ`*@OkU_XDx{pjf37y06Nh9hC<G^8wnQ_as<pcq+8@BlyU5Ck#+KKvwq9J
zPo~E5&n3U+@~RD2x`Mo?{SQ7+8dQ4gpv|!s2HkJ9m4BO(E-t#asas4uN9>E_sxuo)
z#UT6gyL=BvJH5Ymqr61++VQ17-X3~-!jA8)a^=O-9nPJ<KA&V!S*)~JTO-`%RO*5$
zBDEIXk3M+@YaO%>IPr_?B;%b&J2yu*T-*Ceb?ycJCr_u&oz0VZ`t$nPGt@WKZ7O>D
zcUHN4<)%;HzME*eG^Mn)FOy7|EdIE*xMAD%zMw5P_q1)-mE1Ke?_`YER9@XPm-*jC
ztXxy|Fm)N5)tigXpAA;4dFyZXUw!(eSj@-sUE4ki9iA}Da=KOX79(9J<rwSto0Xl9
zEanT1(cMy$w&p}ayZx=_rSDd%96afN%fxqadtu*`(wA=;v)+EGjd_w0=@YtnlTM$m
zuwg)}XW8T^cC%-Tn!lB9)6RMIPLWSL>7(9EpR3)4E!k`Vb3dzW%R04MPGHq+o4iwg
z2aX8Z6kjf}$=tzp+B!(Cc;ozI&-&|{Y)VqYFV($!n~`p%XEwF9Hd)RnPI{Huv8wmm
zR;)|Ade&jushu;zShgyy5)FJCnkT;Pca@RQ!!CjEWlVxkPPy(>Iupx3<C6Xi&}9Ut
z6HX+=usJ0O3ZLF_y20NlMMX|*q5!J}d#uyN4IJ&R7CMg-1qHf0B91ue9Y3L;q9F6=
zQ&eo*>SMQ7CB=mHyniYIS{ix0XuX<{eBgxlse<>+1B~aL{_3-Ur_t!hQH^rZDHlYI
zk2uS$`piB3(C(_Z`(@Y9?|7Xa;vKK}E~qLw`~57ffVN5EUU56tx$WhNiFEelQ8k=$
zBtQtvSf_HVG%rZW8FId3fKqtX<bBOQgzSy0Z$94S_-<0XtdpAKgog$9?>z{4@#E-S
z>lO|3zH@R)1)qF0mZu($RN$4n{o?95*>#n#x@;2`uBcq>8nyni?zU{^8yaus9##(e
zwah>B<n^ePd+zLxm6Q2fkY3Ml?19_9NelPc_8wE-&M6bN>Bfvu717Gt-H)mcJt-;u
zpZ({4Y<k4O9-~hwbGmEf`}(J^+_vt#5A)RJ``%kjH=P@v_vnu6q5Y}XRQYay-`beL
zu#?f1_jH4G*4xvXGi~%wdO0e_>|xus;LD~o?#Xr;2XmPl_io6{fB&XS>HMLxdt3H?
zJkVi%-O0vs;f?35ykdGX2WItlFI1WRT}AZY%!zVGE3H4@lKa+nLDDI`W5&hzZ^Vth
z&OPCuBe6nI*WSHONmxIGwOstt)cN1z9tQ+CM89z2`y$P)5WHy8;sr(P-BhRSFIw;R
zG_Kb<LyG(GnyVpSbgw;M?6m&t(nD-pXBRBI6|j3}++&dqnHS7CX1DM{uh39g@|Ul6
zdF_?FbvIYUY!2MnyS(VM+tX>ir*x&KzBc81*if)y_s!P=e78U+2eW^fS-)tK)TW|l
z$ER$~r+C?>O7m!$OI#^iKl5~SlJ?4n+S^q^J{=C(!=g2Bu_*7VZPLrTu1fMQnzVb*
zq{qj7(n4j{e1EfY<>c+zrFpMBeV^`GImbL?Q9M)g;yKTiTbmv{l)gB5o!N{KX~)gI
zpFi}xKXK;V^N*f8;vBBdwBw%qJUgc)_<z(gL+=H<89gO)CQ9C4em-=^0nJ(s>FSqD
z)93t(Tji8#R1u(eNmQ(TkEVjX&GS?L%?0^#XZC$OwJ!M~OM&#f@)+}pD=oI<+eTfT
ztv+A!&fObAOV6!cYIc6|hR0R&8pR^&?}cX;>#VY}X?U7y#xAy^B0v5^j=~%n*?)hI
zS6-XUS9JQckMh^EZXU<)&C@F?NoxDNWz~}B+_zbHmZo=Vz4@_+qiRWHN5#u&9LJ|l
zzSp+tL$9@*nccPK?FQ>Sf<NaUo4MrvkHz~g7I0~@d7JyG=HGfUZMH<|+TykccRO#_
z9`DuP!%!jkA!+`Da2ESu8B^PPv#O7OD=y9vnfNBzqwLU+`F**(o~MJ_oFX#CYgC1f
zc<uJT@Gk7U*wJ-LWugaHf8oAwB(NjS^@Y>SJgv*V+Qq@`d9N<`&fS@*lXpGh&XuXN
zcNJaMxO1gfI#D3<SJ!L1#se*Rcdqo#?p!x}*T%(DmU_2uSaNFh(=}DE{l3m?$co#b
zvo<`>cD};9U8UmdcBkx2I~u$B-J+*OqF?PC_iD#QoI3E3x82-k8ehBNJA<~<ffhXv
z4cZP%@Fm}Q_}g;Q<xiiVTwOWyj=f{ElE#X_6%VIcX>Oj$ZF+jas(X7JH*Gx2_$;N}
z?(>CZ&*wZ(5b2I*K5VqgZ>sj!<IP5QcCEC&ksP%10nfe{C#){1y>)Fayezzo(RDi8
zxwgKQkDt9|c`W4~v?{@MU!&Qa=+ZpZ6ThdIhThE#xHw(2v>;~Vva;z4Z|^-Y<$TK*
zvQ43DbJ5WQ>n*vm=PbLddc>GdNg!x;bj;fJgF81@%lcjEsY{l%SlV}}a^V3bpY-&+
z#yv*X6U*LzFI^-$zpdEuy5Z)^_um8V%C6a0aa8xtnrJn%U71c@Vt+1~&i`%`wfE-4
zOL4vWHLK3P++43zVq*UMv-Hl{oZgoX7=4a3a(LEeZTtAK*+j$MR*8>XG57ZvJ<)#f
z?x52Yb92=Sw<Ej$3s^+&v|3b@?vgscfU9MP(!-rCh301jzkI7{>w8_NJ>4ZluhW9n
zN4n+S#_sEk*PTMvwXR6Hwm#+B`iK)+nqU1H9hWt)P`S8X<>LB^9pRH+gunA%;qrg>
zj@JTy8fu~HrDCOGXT;ZUTo56!CojeJjJZhM^=%7p>^atU;R(aeifeVjoeOWp{*DXZ
z{5q>F_UyXdi{D&-b1T+rxA*dNqsUVak|w-3>~(W}{*KQU-t!;Ktv|^o^Wjmz=BRrz
zUw_VL>8mTe;k`TJg|OwPqYJ+u*p@y2)ve_=_LqdG1+o2I^eT0MhX`n8v+GKYE>`cX
z>o!+ycf7shQt6g1b^Bnb)6S#&I!iY`%r1?XsZd&asNMYbch(6ngG<zY@%lOb_vlV-
zn=#+VOxJlE+deNfliV40`THE#_Y3u&)%;}Cc5Ff0OtB|r+aD{iKYOAOtWl}2t?ruX
zS}C(gJJQO0Zk+k=O#cm^**;BQx&3g>oAkhQ0$r<=XV*#3{xHqIe{$*EwY$Upo?ib^
zy+m=({=aH+o8G(+@wmSC+3Kgu<{Z{p)pKO?P0nq_{#Q&pR<tjF9~F0L$DX?{ehF9S
zUO6M}eSTWU@3?@h{->J#4>w)jDKfp~bmFX9GwFjlIvLEFn?AK&UMJ3VPOFe{w~_Sc
zO)snK68jDve`B-rS<#zYh8$0sD%Wm3_H1gQXy?Kw-I=Z?+J_<q&lf1XxD~wHD(QOP
zEJtU*&Zh@Bvt47f<35K%4w`x!>ata7&D7anQf_bhaV*8axaO*C=g#YY4n1{k;TF_m
z*%7dmt5NsV+&xOscci&pEw*?I?+u3`_2%6hX5KJ<$EE&hVd|_Kdm-pv%e${^Tr*Sl
z%sR1m)``9MRKnGY&VMoz5OVh55j2bub57wAG>nmOPH|~lWw{~y*#W!rljh&@|GfFR
zde*6K=HFB2OEdLPSJa&6y2H9jZ1?3KJI}>VVCQ8MVRUs!?3*#2GpuQumdLNJ@TfHb
zHe7q2we9%X5OljTySe+urCQmyTfCP~6cg3|Y&-k>leY!sN3uW7(G^eDkC~MbQQPra
z!Br}MyQSxyvrjq2?@7LXKKIJkXNzoiciz%plXB+d=g8l3@0<E!<}AA|_dM{Ab)$bA
zQ_`G`FJsN-h4UUcblH0Gn+sMu!<6q@1+6?6XsoTOxif!9U|aEqi{Hu)?y)<y!+xT!
z@ygzq1}WEhYGG}v?_OR!dn<9;;p%h;b-pe3Y}fdWbJoSwi?b>HS-CsiH?G9qBHYK>
zV^>d1`-c#(Zuf;h0v~Mm@OhjRZeywA8(nu=W#-PTo|Z{>e4iegv`(Y#%p`~ZPh`(7
zoRyhnsU6j}Wcl4EA0xHH<`#b{Iqx9v?0izV{K<6Bz=>bFmMs2P)D#%`MEmA-?VHyp
z9??m-ysYKJN47V7we9zmul+MUk`jA9I#J<`03Qc?3-cjG5eLJZ!cQxNyEt1nr(U|!
zQgG?W7g_NZ?nAS$h#X=)x@e{91BVa0(hmpZXq<7~p?Ie}T2;q3RHf=2zxdvk_1rN|
zVLK<38hT#{`YJ2Gebxrn!Z*c&uYUb>*%IS?amT9nKQ#Gj3ZHa4$H}}Ynma}2-6g#l
zR#%qBI>jG7sMq(+Hc<a>)5p{csShTIAD6jWzTy{O=)Hw(hF27hT~%mvJGD^F@QRAi
zxAJq7d_UY@tmZO}@zBnhdnZ4ByHs-5^7t&bfQHbU;j=gInrXcJ*}iSb^G~pSK9cz&
z=%8i7M(f?4wU;@0xbJ`RbGX`7?fEj9Z{=i`#fb@eE$pxA5C8oCIhpCg?zykt1wO0L
zJ}TXswp!Jv=;Dq%TScvjbt^a6<+JF$SNmvV?7RQBY{aB;f6Yn0i5)&C=jri3St%D&
zS}e4Ey|np`E=w-IUB`|_Dl<wPK6~lc!L3!BwyY`FDVosw`=LYi)31{&w+D61^mE@-
zs^~6xr()jhQlHlfg6&VMc5S(1G_hSrSwFJRSX5)zjZX|e6Qx@VxhH>mT*)Neb0#I_
zPtJ=owVzL!&tVsvz5UEx`I4Yj-(6)Fy!JA<($Jo4d!?ma>ca&uE1uc@zQ<-NRfvY4
zSE>>HxO&a%pD}DskPQkg>x({SJ2$Y|e399)enR{Tb6YLH>q|fF^qR(VWM_bJFtc3N
zo~RFgl~;{E>K@s?gKL)5tdH@|yx;#Hzqfy>@1@6KvtI1&KdR58P%-QOkur&E+{MWr
znoGS_t$K6!Z^}RU!*_cXdfHh9SQ;DCx{cGPnOtB~ozqy(wX?9E@qs17a}N{t=W{Bb
zJ(%5`{V6j`GdKU%&+Sj+SJoNnb7XF<{pAsPW7PtwXAT>=&itIAw&Gb<(}TO~q(HE*
zesV_f+gS01%R@p_6ZUZny0e*t=B$zk;R)rr$^n9fqN^^3E?(`F7M9tycD;G8`p!U!
zb9osHMCVM8&UyKN3fIgfvx=Pzv)TmjOrQI5&pZqEnV(K<J?+P0a`LKPw#`oc%eImq
zJijJvzOd~K>p@M&7u!|c=hz2KigXqI7W(YQ)Z<51{t{wInabY6vUd5GNY$wuN^*BH
z-0@yibT6FilC<sDD64JvcSyY0bHVa|(L4G3oW&pZpW9)5|CV3=PR{o}Z*O~D|28Ld
z@0_*oIo<7f?#fK`w0*xv^O4`>cPDsipLp%fD=b%i%QPv)Sth4UqO&yoo^w~z*Qrru
zOQ&-5-0(FrGg9JY4BVUdHO|Ot$y|fxxEGHSs!kT%coAhky-Bca<+160O_%Rqu-N)l
z<Y46*iIsj~Pj2#99=f)`xM>qts?ULCM+AIVu9?l$Z??2OL&(SXrQ)nh&UYr)XYAy;
zD!<A;q;S=vkfSTRCU*skE)P4)VjMER;!kkOv=chhZVSy-wekirCq~Z5yRo9f=h=_U
zD1lv9XSSZm2|c1LbKcWvZhg-d-mu6`ulO#omc%ApH=UQdyhyb1kUfW}fS8b&h?tm|
z#H|K~4H|PRlEmi8EI9Xm@)L#yCJI6GY$SinUt>5MKV`z9B#R22BTMy~k6&A*q{}>;
zEk{b-?$D%8KHG~6(`WXp=k*E4RxR@OFPYuxXM1s_y4BY;+D|kZBAq69W}k^lz9sef
z+Ke5il;@vLTP(z;esrDmjD-eDmAR*<KK5ig&ULAA#l$$@y1tbzON<ywc=}XsdhEXI
z^>@pm<Purlw?6)}(mfx4jM!I_YQ%s1+GH=&!+(63Uar|<eM+`a*jU?mqc3Y*z21f^
zX3L%*^>2+kt$Z*}x<oay_m;|%!dab#obKD#%~;=aXooSA=E5&RAE)v?zNXspXEFoN
zvfG(E&mT`$H8}F%<ps;a5<eFC#SU7ZyxkkO@)!Hw;uPs)ymfV&w=`GS+$-gCkH1Rl
zc_V4Hb56&;PBER5X@|D$_sWQfOf3Fv#3^;c&2I6$Gwb!99@h{3e&vVek-Dypc``~m
ze(T>e%)VEbG;u@z%=T@Mg7f`iFG;rEjJL>USX<UMdG5m_;pLkQ=j8CHOCRo(KlCK6
zy6zwoYeVj|N34zpHy9FUw>CZ4z<7jBK-we6VPoS_Hi_F>9E|<E4rUBD9a5Ml_5Rt!
zx5g^t<c#U=)|@(<WTs3G;{CDM_EKluEaeR^7A%^s)%oeg+AqFRPQ4-DKDmAPe{AKc
z*rql<DXWYZ3l@lTS!ujju)tlgAvNHFqa!=(UJ#`(^5<D@)1TX)@}JzFSD*U&$BEF)
z`ln0QJBj+5`iA<J`c6%Cj=LY^;BwUEnCr33_>`CwJ3T8sE0sG%6BP@#AMSm}UUByi
z7vB-zWy`sY1$PM+d2%Z~@wzN0rr9=quIJqc4Ncyc9WD4Lw{0_=7bY?Pcg1qcu=u4a
zAHOWS(G}r+C34~gm-{;|mK@@AyKBv3S;X40@54;tc@t-*W>2q-mOXX(oM?^5w${Rz
zHClp>VcQ>67{1?m_rlD~i0Qu$eURE!a`$Eld%(fj`akz5C%rR1cGOnr@-_SFIQ{9{
zxLUv7uj-GHnBH@7<)7!QoPMhMd?!T%J#O#2qP6^x`>O{cORoDla`J6|Ai>`Myvv!X
z`i;8PmMh=&H4ncNKUeI}_geeRNwowwO%E%>DF3RWlCRZ@&n$J5>QAnf=086%cJA}(
z^KBebH1FGbb8%_fd(FF8;=TUmCC^~4-s(r@zBPT<4Nb#beg7=$lvff~TCAPVC3SmB
z>Eg`aS4$p#(sNOBQFD3LY`e1CPkg4Hk*tQmOg*F8pumz{FL)+ODjA2=ulUE{%`jP!
z$@y#hEB~wWSJ{WwulmR0w~WIF!ddw*=<lva{_FKFMlRaylGb@fXr`)Bbc)&DU9Rcf
z=fq~}8bzmUJCUO>DQ@bq{TF_SyS;6FwtwG#Y5%!GEB2%bGU%>9w6iI;x?ugA1}()O
z3Hgor3-b@^|Hx+8z_^#~o74^M=}X)0F8tQN_SvRwwNnG@!lx)HX0^3nRNS-Sb=cnG
z%bFi2&I^1gY_>&LdTmlg^4n>BU9u|tn?3H&Warkl{kygONU-jC#*^;Pmi#Pyq}Jzl
z<&o{HSF;{><UCn;^_$Aw@ZG0+JolHxy<pkip|5Q5X$#Yjbz3wqmiqcLD4HDUv77ST
z;^Ll-+S#8t`{TqKrhJTj>1AYd|B+D#)3&%>pC`R^+PHtY>xtmct9ss9JiaIXR;08v
zt9$jzA1hCv;XHry@6SydPc5Iz9N@EIEvz??X@7kyoUP%Qnze!4!-oGXD*M?AXV(8~
zJmypLN%U1+zCep*>c2<p_Ab(W9kngy+3suEOU`>|8Kr3S-(k$@y&jbDtJ-bJ@%cMU
zFByBpCVsNtc-pHsQE*EXOaJ2FxxXA3ewco^ejvU(sHKZRBBx;k<9@b(cHe3kZa5?`
zH{IN-SGM-X@>@yQcijLnyJKG`UElR;)4e414H8R_x+QEBjL2aTlWsifmT<L@%X)>5
zB8Qi>LGFVsXM=uooo>itUd_9~?82>vZ07a6d$xr}u%+DA@)c@Yu#NFHTb}fe+}&1!
zFU&UFX2@sWn>#%(>Xha!%@2PIKioc;&%R&wPZ^l8Pxi;O!nG6QmtM`~S<~FRdr72|
zz>l&A(T<s)xg&H~e_dRl$kA8*^1=a&YjN9dd_Da3mxEW&1l`~;d)3QRmVP(e<9GAQ
zG@b^(N$091wBpsjpD=efnHShvRDVBZQQX$e&n3>q3;%modOB}{ZHCYOmNn~`mwK%J
za;n%X=h&ph(###}o}6m?nppj2$!Xu)n(9h=&F|Fi#vZe|yXoE%=T8%^#2$*#l*&{{
z+q>K8qlD%`Essq+$HVz9Onc|b_VjGe-U-JJa>lk@zEF9iR;mB~%1O^(gzU|EGgsUC
zQ_16Rp-x9+6}P-Pk-NTje)Lks=3lNQ|Ib`n6qjoizQg$S3_hbjhqi1Ec0Stexbau%
zKlxj{8OIH>eN)`#%{v*qt*D$O?}j+HWo~lxPWzrG;?nt=Z>DZKQ@mtVO6%78UC&Qt
zlqMXMVmKx%QJ!!oI#)T8`M{3tQOpN+EDp;4z4ya|L$ju;vNV?Vrzdl{Cfsc-T_64P
zz;5REe0!?(6&UZ!)_jkPjulKXtxfe0I<RVs<;h1&L{}zDdoKQ~>m5?%x-#G_zxa3F
zY*F^lt8%w~4{T&zJtr^0p(G{+1Yf=JNT~@7EGf9BsFd;8)n!>HPaczRie1jpf|Wd1
z)3YU4omUk8p0M-xFHS)PvBZvk0m-Iy<r7~jU49ub>3sRcmr9pimv#Gy&r~#WPAPdA
zAUX5aKcTQ310yA4mp1N17Sr9$`<`Emh%R(LJ!!%%jxXL%msY59DNnd~;jwo7QC;47
zEge=pioUN{rk5?^(>!XDT0EmZE`Iu~8!Hn89rK^<zGMEk<YxET6Z<Zlb5j2ku`kd*
z_M>)8<dg7w3omc|;IpMcFM=~=y3^I?aeH3-EloDdEGj+`z54szT!j_Iv)n)Vw>Ixn
z@zL9&b^b!zCOwa`I|ln1cdyXpSO0YGVD+(A7k7r$_2~tdtol&t7$v5<?%B?B>Ykap
zANMbHuH0~&t41}(v0Le;_i|I0DF2@pS+50m>Q4&1Sdy>w*!3h2%bm7AvU6s>T@y07
zbheDxk(-Y$Ejskx^BvFG=@T{=N?p~im@!>OYM1m8->Va>bhkEil(cLS+;=~?lIicP
zT{8|_O3sU74$e;KdVf+XB7<+yVas=qeng~*WwC3tpPg&{-n*n@(-zmXVi_G!fy>~?
z@J`pNCF3UR?EJT^Kc}TOEO_B?YkvN>=d*JygRQ+e81}M#(VCW1$GEWR{-OI!52V5s
zZZssc>YmU~;FT~dxWVu!eqRm;)6rXHs_PCZ75wQoWja`NhiTI54evhR{{BJ!Mb3fE
zjpv`T22A8yqj2j$#_EQRXHuD$@y;=8xaE+gy<%IO5RaH+HuE~(J!TBIXUsEuEjq(>
zoA<emvntB>ig5BT<C)-l>@NR7xhI^WKZT;rC-{D6Ej2JrJ)*W}htl@t?UBdIre`L0
zo@(v8yM_P!%uc7%yJMCJDy;JHQ<C(t{7_P|Uv+B9a!VJD-v@IOp5K~#N_$DQ<dU}%
zH*4;$|Ep!PlTXGy>SSJu(Xv^ZQm>b4&O9b1nQ*13>WfZM>eIU-k*#G~e>FLtT%I2m
zurT#q<A>Y#+LUkjF{Wm2*-<M{yrH<fMDv0}T&8&wujj^VwmhnAhnrW&h={!D*b}z<
zlEBjBg=z0D?L4TeIeT5+u?UY3<#BBqRVh0rv%HL($5g&N_J%~8*u$?D`&#>C^rpYQ
zdhUXh#P;BS%RYpLEiVlXxL)noGpBxgbNo5ykm$2s(<^sp&)K~uN9Fk2StsW&Tv@YP
zM{)Uk59<iAg-&1FU-i4X3eH$%`1Qq=cNwoP?#!CG#JfA{myYSykiv)W7ya_~)x8?O
zYJWoM>PKNm*LE$Ly(ahOin}v;x=dCoS1i%=JQ>24x;)0-J0+pM$7(u%U+1H$J0)B0
z>8<UzO5|J<{${f4ft3$J3ZiuG%Pv>4RGwe)<j9F|kt?^(+ls#|nep*(<Ok&vzZB8Q
zTAU&CT$6N8T>c_-E$qbgFABRf6t8qUv^%sjd_FZJ!v9w4vT24}bFOF`?CyD)dPYAo
zKtE|_{3iWW&Vzw-Ef!iN_MV*nP4!;vOxxu#C-Zo!l$N~Rb8n6u?=G#0{afcOn|ob}
z`ATi8L)|YI?sId5tX`e3=HC?~z?mWG>{IsC=iRc)+ij$zp6DFTDsTVbd3;r;Ynb)2
zd4iXxOsGqCIWar=iNM57a+_n`c=oi;j&v*ZuIugAUbtHFRCm%T%XF^r=|5j>I9hhe
z)x~OpE^BI#$hN5)1;X7-Vz|U+XeHF``aCuL-gRk-tv$vYVrC^*KYHqse`w<U`%KOs
z&rRDj>k031pZ8amCRNlMHD^wr<abGNtNpXj%FA-*e)hllVwq3=PtV#4<$F^uW@S0s
zr+%JQX<#GTp0rH+$CM=V?T7ZBw{e`kbLXD5KPxA1D*VnL{^sP*RZFK|_lcck-88G~
zOln@GbU;SJ!n3nZA6UrP&L$_-kfE?B<*I9pkeP`RgX@F9-|M}rr|Kwj1VpJmW7RsD
zUG#9erme$L4yzY08<sQA<EwbJD6X#2{j#Zs`qM=%36%#KmVMfELSIUiWo^t71&&o#
zD_*s(vf8gG={rq;C0BM!>4j^K(d@aY>RgX}Sp#F5Li_@EHhVr5*|8|yQ{C#rYl~Hu
zYW3`WqGBHo)vkIMGx3ooC&Otz9qSEm4s0q5pZ><h@!j$REaFp{wG0y2EEhD%1WdGC
z@NV1IbD!5=`{T}TZ58w|!R5xp`ir+#u6;FMy=a}>$>03i;XdcL8a(m#ib|0FB+?!C
zZ)))D?v=U>J8hP4)7javMCXL1;@=Lf+wDS+7UufDeEia2q3Bw%*w@y}kA`fzs@a;S
zHS@8L_r+5wt=`QBm;I(3ob)-a{)zs?|NC|qEjy_t9Dk=i+N$%|QmG;z>7RE`X&KL-
zwW=^MFfiIZT-A17b*Ap?r8Db;gZeJkDCq8<HF5QnKi2aOpL!UvtIe)-!ny+YB}wy6
z+0VAMQ0Y@&<!e>1yviqf(z362H?E(2BPereur{CBea&zCtlm#-ba$Ux^z-u0__)L~
z#h>@QRZ2|L(A+UAeVNdo)t)V@SBJm$J|(32we<0!r;*cGzJBu#YutKe*|UvTGuLqS
zJfC$srM}2FE!}9h+Gb@Y*JIV3M|50{xh`ATBocA=J>OA1PMcRglT3NeUb>)Dx@>`J
z$-&E;x2!m@Xj7H^4$~W^H(V#1n4GjalbReX`1a5R6+_>Nv7IszAx|tqrIysn%inhI
zUh_ZV>l}&2p7-Jd3|A>FxTttRF@SM?mEa50v;v2D6`7OH9Fy_k;N|Fj)y&u!xR|fe
zYMGoXL!Vkp+Sh|EOFh$tu0}t)EpWf>hS`$MigP^;*Xw63yBPG+D<kaK&z+%rKJT4b
zE53Om=VKPWa<!iL(zcMz)1I&Sm5?o2GMAk<eBlZeiAB8Hn-kN_dWtI|^7z8c*;mc>
z{yqD#zuD_vmutO8uH|0WROMVFop?d&W`lGj7<tx9?|!Wk(9ROaHqFXufoqQExqHVJ
zpA+k1&T;DFJz@A$LGH8u>>%IF+OHdfE`&_!TD0g2hp5MsOQI82&uI9f@r&^lqZh*_
z`N02<SCnr|eBDvF#A4I#vWReX|1H91Ci;mdoLSVjU6`Kv`og!p+DqTR%`hrCVDa@u
z(9ut>dn}5)uV-)DFjqIn>d@AYQHOF^Ub9uyp4)t^NNL`bx7y!(HSPIyZ(q>-7whL!
z*L{Dw&t<+ZmJj|GKDf=jk$VpJT<&?Y3G9#dD4gqTd*d+cdZW&TXAZNObNLokim*FO
z`!2+tHM6$#-_)fmpWZ*QKQrL8iT}jRd6&K{U&Odr>&u*%2haQx(bHU7;=d4#UVdD;
zXm!BfGrvT-f~CE_i+Ft(+0`Id6uFmME>QR-x73n#A-k3bG=Wfoug}wRF4eMK+GV!4
zT=#U}3Fuwu-rm7yYLij;YNjXu%i}A>md}f<N?G}D(b1)1ou_uZhz`B8YU9yYVZWol
z8aKNZ#Pz@KXL@jC^5neaN&6(OotBu;{cxAW`fkSEYRz7)QrA+<CP=+3J38q`XcjBS
z73XJ%9<=ChC}w;3dJ@CIJ98#v#wA|s;cD?MTJvJoa!KEBcJq|<6u&EStQ9fZ((W@c
z_Gp0N*C(}ndyLlP-HS<{?YB_6Df*{TRk((%>a^mS3`Rj3^Ukugn>~t~&G&Pg(p%l7
zzPqn(-!t!;u<*Ny=Qr$~-uIy-z_|U)T&o9?YVFbCcNkcIo{>INv@r7Ji(Q|Jo}Lo9
zt-ap=-qfnN>s?F4f4*NkfAW_A>+tnQIB%Xaoj+r3a9wRM*WW9Z_ouLT^u68mVwJ_$
z9zL)90Dtk_HOCaKLK4o4y03K7kF|-|l|P|XYu2Pgr@qZyX#3>H{$+W0^mhK#ziS+^
zYdS;ttPQ4(Gk?|@K6aF4F5|glDDa#!i|Lxj%91Ikye@rdIq~Ja{tZT5yLI_nWKtLD
zFs1Hf?_Cmeiebs5t_PDD1K6B2%RK5B*%PK7nVdYWWQv!&m8Dnev7Dv@8dKU1Gzcr4
z)!LzTBW<%)p;LlWg4nrW!3D+>{FceP%Pmu=c-)!%yKsun^!$Kn0!$3K;p$C|Y#dSw
z84DILHnZ_XsXk}nyyzvnc1n-E<F(}qT6xnvlmr3~Ok?rta^zrqkrG(0a9ObR+?f*X
zna4y*mn@ndk+SNSvb<u1kdWZY2fja#E@)X`FJ$T8A#AuSEb{WL;Gj9)`|s4dAM&1j
zU}s2R_1ilt!qy-E-I#yne7Rvs%ll6{9*;jiS@7+VjoDfDAB7X%{r+e!JLT=(3oRGI
z3|l5KWbW_fSnz4lA(0ywZb(hr)5f~~>enqQ3{OK_n%JdN1)qE`3E27Jv`;ddu7A`U
zn+>a<I=XUvez%EPFy(8Ka6zqSwDta7-`TD&nR0y^^Xdh^-v9PK(iyqYb94T!L)*9H
zXqLa4Y=1uMbK}F0df|2cO7WMz<xPxln8We3GV=1<-OH}X_V&JsRsHg|=1fef_lX^6
z6`T*Aoi2Cw?SxRb@Swi3t)5L6``57~L}x7hW_63nb8VALMq{^6!EyWPERN!`0c8$1
z8B^J(fEfo<e!gUxd69|zi$ZRg#W63T#jN5}nvP_6GHfZkaEmef{n8s<3f+qTtN#6c
z@x$uV`a9R2Iv9Um&v&6vGvF=1CEwY|NG+FxA8L+sy1T6Dd+54ckM%&?i3vQ{UD?>H
z9z}Z#hWD+yP^T;MR(qCNmse=1xYFULYxx5WIysIkeWqq);ni{?p@FkX^@Z3)&WW58
z6&3wDc*>PJEWaIe-`34{j&bfg(V2Qiu_;z_FT199oD%?dsM=2Ov~`}7Tx2ZPeB{Xd
zGcW(GE$DB1zt&6P(22s_C2r>H&+F_x`aJM{@FtE<)~x&LTl?5WBSW7gi|a@$WbL>Z
z(Wnx(vaL{4zQsWEM(>^1-$JJyI%H9_=Db(Ztu5L)CpM++sE|4)d{g?pbM2<o2^xX&
zdjj~somh2a-~J;lVaEh*)TEv*Z>xN@@4}7>xtJ$+J-L1=&Q{r}bb0crLZ4mpR|tLc
zxRP=HtA~-rB-zbH(QzTGRgNG0zVOwPAeAOYD<9{p87m}suViaD^71ZTxJsq*laBiA
z_2J9=f62KOq;L5rXQ=)paG%6vwXe@)db|pqi;~yATjpAH|FYzqoQ#P+x|3GA8g2Dh
zCU)|Q(o7wpN!$H&h4?(CozR)6Xm|S1e0CPlkW_kOm(pgJwC*!vGkKdP)Xsf99W)>%
zurjrI9qanm^(=Wt@AgXVNOzdp=*-q6F(a{ot&f4%clW|fbCb7i%kExSBmMjjZ+7nE
zsM<P5*I#E}SS)_tv(DJ+?5gWM|2BmRF))=(y6Ek;Un2J5o?S7wnsxjml_WZRRgcbi
z;^!E&Tw!Ifki&}#&nZHhs@@Z%`*I47G`bfyXcVM0D6`DDS1xehv4;JV?1u1`r4tm+
zN(;SgS<O`K7O5Wc;xs?!R*#=gRZf;pdOj(v`%kRfJ8fmlYwbS#Q+a$Dg$tVVDt?-8
zn9ey#^|kkn!wHW#*D&PSoY1QMr!2l`qGI5@GG6<hGxwIt?+FSN(&sEcKCk|Bfl5h|
z%bM?MvroUhTCjfZ#Pn1yvH2%mFX=x%w{YX<>03Ld?hpF*-sH?%pB?)ypV)TUY};o$
z^>Y5FO{*LVzn7f$*7kN~+qvWF_PKRDpO&7hs-1K*HGTcPZ1MB^CcNI7>8RaRo45P2
zVzgiIbeXv2*Z*Gm6XZB^`MZd9uEi?xkwvqDCrnu9{{6!AswYxYLllH>6-6hfcRR~T
zyqsRQG-gjq;r^89{V6;4Yn$z~Hk$pe^UIUC{Y&&?u5WKLvi@f1`<nlmk<(NG6Smut
zCyt2Cdb&_l;KnBA-Lqb_WR+Y!z~^_tdd)+L!=En|Yy1z;cn?OG12m3akp1a@#b8$b
zGP$R}{)&Y1bO*Z_nm;<)A^LZ-3t#YKF-yHA`BxR*uwQfUnY^p1X8(($ul7r+vi;-E
zPK`7@-PL(D=x6;WZY{AooAA8VOzaF1rf0hwoDV26T$4x(SS>T{j({l3k?leoYw8l8
zByPN@)STV3&XrMyWf2F9uHmI)65$@shYw_l>MXlnEX~x+)w3}rO5rrq>c9k3u0AE^
znPPWp&jbrbM1P&hHJfYqudbQ<rMOoMJt_TQ_I_#Xs)C1o#SdePH@@6g(6y8EMD2rj
z2Y;J&&Y7cQzGl0~=?>FP3Te-})*7#hmeYM|9lh(t>8Jgw7Z*>NQ>ou{`^H31(J4nV
zmRd`7+81f9kCE#t(PMso=}z3tjGU*h4xig-66bB4oyl~GTa-nQf4|4M1;MP!Qy18M
z{VB1@R3fr)^`l)=9y|MXuID-7v*}#`PyN}7qbtH)_<jg{m(;cla#ct^5wuELXw!;C
zaS4}1!aY}5yiPiK$|To&;qmuXUbE-!oqzJ)w5E>rUv6Fd`pjpKZ^X6RkpbfBhe~J1
zyliqlddm7~u>883yf3Ft*Vbq3*s0UGr?1s8U<#9KXNbc=-jdpj8qJ&hOkTAd)~w{?
zx%f}>#1^3^99;((PTc*h;2JPNNW)|KRF*yGd@2Mk6f9|M7Uc`8n6i&q$vWYo%jBXB
zPGS8AZaJ=6pztAdf|drSg0#WIr}wATml_#=TJM(>_V4eDHOq>Atnm64aQ?4H(IO`m
z9}_z@Ej1%Gqm=lPiG^Tz@^#3<7`^g^zt;WYoff)rrBYzG{f=$sS09+UuQ(dudgj-{
zF30YJ8>hTzc~u_zX4Q?*jh7=sz;tjRFF$x1t$szFn*fh$Bd2QPMc1}9^Mz|PEw+Yv
zDILALe44f+qj*x`<{f(SoU)uEQ&#(&J`+~Y6kC3?JC#puDcj8F(`KC1zdhMeMSQW`
zU8Qwpy*_iJ?yQ@mEcs+%y3M7f3pV`p2zT^8#JQBqGv?*WWvR`3+9FC6o7pWLuU=v`
zt$SUtt82SS#M1x!4lR3e#H`s$V@tEmDr5O8wfsq`CSRsDp5we~n0GhxY321(7B9+M
zXUcD@IkoAK-=$&&9j%RvQf{1jaqt#j{N_hO`aAb-EcC4P+<$hRONHLECvJ21E{@pY
z-IzU{>E5&DPZay@--~crDW6$<#Kz--<N_s~m+MWgdd~W2UAoQ4y`Q`NOU$oX>w+(r
zzPx;CP3)OpvovOH_1;{z@?Ti*+F#NB@0wb;TFMz^#7`79ii#%$2QWEEa4ekQb#cGw
z?jsGY_h+*FxWcudVxPhxhW7hE;~XVqT>kPU1{`Q$ocUnU<D^R$4_S$|^_g9}c+5(y
zz3=G4>4ziSwO>a_S>1|sw|<?GF8g*(a^;IR(u<3yUEgR?qQf`Wcu(KAYa2~U`aW2R
zpZ>(HWmIBy`1Oi%*{|!0Tf&*lKCe1p^>~%>p1vch{a(`;ejokLF|l9SO~&#OhmiXu
zi^uOz2)R$QIP~PPs-K)A(?TKDGr}{CJmw26@$ZbgckPa^*t2(g<kd>E)3vV1e*GU~
zGC#WY{wAYG(c#h3)1xQrO#JED|7F>OcWXM%Ev|B#c)!=iJ3D;WQXMU=JxL9`lBcIM
zp4h>@a8BvE-O=B<bLaNuO<z`6zo$I*Wt295+qA-kN|Pl`{+@m9J^#=4^pqP@&-28}
zp4(r(V8b$L&CK1meXma_d;DLr$db3ydTrOwsZ2)Fox)doPM-fBr}Oi%t<^2_JM!N@
z=ces?WVPlJbJ~y1^LDS=m9R8)*H*FZ(;i)8+SPrt<KMdGbyM#@3%$j2=a_O=nOVra
zZC1(mlMmV*3((wrg}-uIMn+%Tvy3GbCN+sVAO0DMSOhoeP5tbxR-k;nFSRd>>1*)J
zRF)N$+NGVId^;@TO!-b!uJ+`+VYzC;Vs6V*6PTOimNqyVIB>Zu7hSmM=*})HTT*i2
zBBMLIjOY%-6Muy(Uw=@WS#SIQl$-R=?a%L*#7eJhPGeheg2!PeXUIXp6|ZJ<ocLbl
z!7h8rv7PT~TgU}P(GbD)@10VfRImNwun>mI6^-<a^o-`KS<N|N7U!OF;?OOZwiK@1
zpumz@FIrAKIMTD5bJ=TwnG2GgN}NiZQk+tDohShhT+i|Q-vyo!i%r>ef@h*+$vJ_!
zidN1kZ6J=4u}j(oOH1V<)r_g{bxz1kU+rp_sCM?=7F7e8V?KU~%T^y&bG^GHefFK>
zC9aS3mdX8m5VM6nS7)l|ojYG`|FP*Rr#wHk>+!y=AumO~mM3q^nBv%ZX8-g$@h6&R
zFR1c!Tdp;Woc8G*ljxlLv+L_4nQm`qN~^8=_u}Ivt#?I1P72?g1w7|Cb-rMXeRVgW
z#_MIt_7mrw{xL}N2Aq=r_gK?jsr0Mv`hB_zw{Oi)`>W`i&40-#v5RwQkgq}LvBlFg
zW|!^dIl0~^Ra)aj!KvSVpSDgtkjTAu+XC0JxpP04WiNaaQ!DA_JCF1GeGh%sa~+?S
zy<O0>Pf4f$^vr`ljcoNrS@N@{`KSH7qBDQi)8NfDS=0P4cz~(2pGYeMl$@X4WM$C-
zO$xe#+g`dxE>CVa91xqUYb9NwQ^TU5FYt*;Y5k!S+!OXVoaCOdX38_sxlfuNH18;x
zBKo3fSJ;Z^4bctJ9o$c(W^@<dci1Se@r_yg^?{T53AwU&6eM5#_GIW<@cz8wvLlQv
zjtR_7+6#M)&O8&}z`-MJ@n^R$lcK>5t`=SqvxFN55}7-ARm>LLIFO`muq|#8*R=!5
z%oBKB%oJ`ONYM^33%J>k$~--H%7^9$D-uKJX{}w`Kapd_>0fryPuCs(ynNY=UrRRq
z;EOdlbAh$0#Bc4VH4`l>+npwA-}JUysbZzOcW(4C*}d}Tuc$=-TlD+f>PeR^AICmx
zd_PxBaq%M!^MgksqYAy-_r>^L5eYS%bnkelx?QRALC-)<)|E`h!@qs+op;!^@5a;Y
z{gyM%7=*?Cy#9{6u(Gi+S@xY~fv@AM{ib@VD=sWnF(_(R7OhgzjV))coMRLqYP{cg
zp~_{eJtDi}7pAUnt$wl9;@lCl`A@5k@<vRGjhtp9BVZtxV>2N=-i2Wg$CvP*!5)9s
z2JIJaYd=x{Q$f`B$Gl0mWx1IA-)wr4;B-;p%7X>s?mA2&T+Ex+8r){cW!}oWXItbd
z`N{Gp%nWXaISQ`GJ+Q6uHd}%8iCl;6YlGCdyBzYF_woKQW4Pl`z<h|8XM1b_OV@3^
zzgt8?L{dJ0=Kc1`{`hC{<L!g(&G-50c28Z!?fBrfcP4l$&^^Db;LgE9{i{1mp3ncp
zb2MnacfB6#E|V8Q3%%R*RBs(DWIw)lsuiCjNEQ3>y;CyTkITxGgLv+XziF__H?=<J
zn`d3|?!nGx7(;ws^?KW8jW{9RpVvRHFWGrwzmaLJiv5SJ=l4h3pYGnj$h1<-Ieyu^
zqSQBs?CT$|elp)9yY~7Q27ix9>8qA`AADcJYk5A{C3|Jk_W5t$>Yd#AIWS?quA~ZY
z;CWxKzk6JJ46HQIyuT&$I6SV>PAk`yb&BBopPN6LZhdc=ai{tG`=57L?ONqqy-UkJ
z$a?#g>!0KA{@bh8|FZbq>!;3+7kG~TxOjZsF5l9&RT`0(T^qJ|f4aNlLHxt&-r0$g
zCbP`g&Zo02>b|i^W|ma?!mx)DJvUf?yjjuwVuc>lRKu4+iII1#XXeDp7YohK`}Ox{
z#rZ40Z{8And2@?zgxr@q^Fx9QpT3Vg^E|-%n@SIt-ReIfQ8D$;`j>TzHD#(w+?HC)
znrtX>Ys$3dOxqdDW_I^%p67DR)$O~&^0L>#yT7iyyGv+k>!Ww=LQC(qo=ADr{Cn{W
zlZly1mtES{*!re)uq=C>_4#yYvR*3-cTCC#wZ%%umMnU~aykZFVkVw<-?Pr|`InCj
z&KjPf2=wShEQ^3Lc;x`NH3V8YAR=A#d&Vtx&G1kS5E6YD@8l@rWXSTa%ezD?tm}l%
z#7L!JbvcFmix$1exp-4)GpzK8PMM+~zH+sboSWfIcHeJ~lWQ-e$eUP}o_H+8^Itsp
zS<*WGXaABG>iFiab*KqBD`5ANwRrx~2QJr@uT`>r`!;8WTDfoW!A&nuDP}G)_Px2P
z;7IhTtgmgE`T3i?@^*epFIqZvSIzh1Q<GlLdinRvudrp)cBk=`+~YdieXZi*!zJ%_
zEVuT1eD9QQfNFT8^tpda^0^*mx5YdveXqao$K!%|e>Z+={rSo!#{QM?vi@CW9HKH!
z1)-5j$8+!QI&%7_?LAGI@9&GAC_3#e%*g+dC@C&oZ<$}1c91>$<h?yb>(<_>JY909
zJn~t%02A-*y-P#?h?Q${Y<#}{&fd07PoC~;VK&t3<uRR-c$?*K<u->M9>0rv=O3w^
zDdEj{!fn@Op?!(}SUxrH@cDh{4%1zyyX<A$<=hqAm5p)C)n~a6D@J_Cez2bL02`CE
zKu*F2#zSme(}QiozpCy2c5+?qrtJAW7ytYVTH(0i!Holt<KM7zG#%Z)c<-$Z*Xvhv
z9FXeBaoE&&(%w0yQ-_1;v{#Xg&S4!z4m0ToYkYsFT(-NpW%-9@C5K#vt&CUM!lV;&
z7i<-ck*>&P*w%Q%%zKTKzy-5_+g#gt@0c~*cF1Gi#rwwW!0j;3SFc<?+-4|X-ps>c
zZg7X8kooA^ggXZcZ>#?H{-ONWv55VkEX()R%Qgk|-?d&AqY)OUS-dx;=Tu4Wja@%>
zmEPNDnK7OF#asQ^H=h}c_p5g<tv?-+;~eU=+E8CLCFJ_Nheo^eG?z$Z@3NY+|K2`1
zNp-us_tWblvNT`64ZYs8-RZ4{>D>^XbLUi3TKh}4Zq&D3eSg`GEq{!ZBk$f{C-6RS
z{RYurnc7FrPfOcd+L<eS&?|pR`N4PlHCw7q_08Lq-g0T#iylK>;om_&%r9+xmVC;2
ze@TQ!V_KB6{LJId4<;=Q-gM%Unb3^0&Nn7543TEJ_2b_sZ-rdB$x&-B9S{+YIvH8U
zQ6l%|tAZ|%7LQhs8>jo?xQFW=NIa3To|9aEx+w6KW$tD5+h2H``t!Joc2Avj+Jt?l
z?2|Hq+l)DR%ly7Qe^S@&rR2=FL*Yxw1a3WVhvVL<dUJP9d3I@*V-EX9*&}5P(y<LH
zg^^AIN6G|lgT)R^W7YJK5`QAZGwC?%$;X>o6i&Qty~9^zed4Xc_U1c$Mb+zln>ga)
zU0u%}`n>*Jgwp?Sf3!ERd(!eIYyPdbs{%@cYadT=6Y){Wxgc<f?em-yJ5E@{bo6xh
zbe~xs6x-3)4PyET&r~z=O|kPWnf1bCqN2VNh=0VC@0W^;x{LY?v55!Gm+{=2tT~_Y
zw)Vvx$$yG^;`=_H-k52*P3QQur!&^n2sHFRc%v0*bFTT^_ul7GwR%iPrj{`UJUw!1
z_V!L)%YK>SzT|T!jVhRJr`3nLcJ*AbxP1N@L#UYLHc6JC(o9c#tN2$Fm;LhXf1<cN
z`}oUSg}ch;tdm_n`My)M&vmiS>m*izpx}yGt0wBN*t;}4c;=d2-z4;E_0Rt-$uv@5
zQ4$~Pb$q9~@U`}HVWpPKlB-|bP_wdLbZ*)H<TD4D=I>uUuO^@Oy29!?3>Du}%A2l)
zzdC--v+$AMwQ`9ar2>ab6(vtScdoLFJUZLwZstLWCDK<af5!5NE_8a`xZ~r(uNNZu
zV_z@Kwk;63eUNi)`?{|eT-(<4$|wEczX(}vWw&xy+bjO7*UQ|0K6L`4cTPDz6K5(J
zySgphFf~nY%3oJQHsJ%Xl~(hMV#MdF8nyJrK?b?ZJ_M9(dyz8{RB|r!%XLZ1XZXnE
ztncuJ$yGn$3zNHk!xyGvi-$jKrX6SRQ4A@FsfcNa>G<PNlvvbgv?AxgrpklXERJc~
z8D<-9GNd!l<*hLLaFZc}dEr`xUxlVAt0arhl$~UMFfY6~?(9Q}-IkV7UV0kYo*D-`
zPu3oNvOdu&eBboLm-@Z$FEcG#!R^tv{6^2J*{6b7J2!Fl&EWr#uv%7kzV-fot?&8V
z^H|u<Y`n5{(Z(<3!W+^pH=Q~0ephx?VfOn=OZRcF@9|-Gys4pob&|RVbClHD=loNe
zJFjc1uanPgJU&}Fb=s}J8l3U>KA+zszNPqO@?tZe<$ve>X1e)nFWY;SlCZU%yK-*7
z)z;c+duoa6!S&6rZrE&#D!dc2Yt>TT`1D=V&-v@eUf#9&eeTrer==Hzf@QDYUUF1r
zi^BW%uYrau^kOA<bN{van)zzGfa2Z<d%dJeec!B*zj$sl@2Ruasr-)|%h-Qy|MTPF
zwlb}1@$XMcVg$YNgstR$2K$`-yV5w;QZxIyh}EnEV(T{9DgW_lX=T_gBWfnFp^@R7
z*`_yB1d<rDo`$n^cbz=gS~OK4OW;vX)Uv+6Yu|jBAAiU;f}6wcp|0YI#BEQ^(=7re
zbx!SRObI?&FWdBGS>cb^eQp+73G+?~HOYb@b5jp@AD8s(<Hw$DEDX6Z!^TXlzxQNv
z@~0c|i`L}l$VtyW{_H||@+Y{=b>D5hXOojZ-?($^9+K)(>+0msH*VON*<F^NeEisx
zjfRyuK626Dw|`;Px#n|6;EsSw<_aIVsjDikf6aRo-!{oV_D4~i(UyA(Kh_o`ipQPX
z`RhxLYVPY_m9K9vWQxD5u6Jl@JIj}sUykvw6Fqf2<iX~+<t^*81NQrJtYW%eUYw{J
z`HEMrxqR;P(lb%wc8`8C^B>sYd|lhxGsljd<xqpj)+r|rpZvJg+kLu>-Q?8Ai&rdD
z`}VxJ>F$F+#TrvKeL3U(ZmPe{zJsgp#!c30wpw9uHE7;OB>{bwC#sRtw_aubbj{@G
z;`*Ywek(;HjlwMRA}rQ#$?)4}FjaQT+YpHwdqtNDhRjgk>0vzmDO2Z8V&@c3h)bW`
zAP|_f!>RiW&$q_NPY;AMx}I~`2LG(+c;?4+K;cTZz%SDiEyB{jIVa9;=Gd?^t;aOE
zSEz~2@xlGZ`jw$7tGZ$@-mxs<JGZKN#>Rt+r4uBKyg7d9Fa@~CSv)!*$j*1i`i2$j
z@!!8wp1<8yb8qUS<*)gw{4XboIkWbAO#L%qhDJ#9ij|W8mG<^*3aHJw_hLci#JB#Q
zRwvfU@2-={s(3MR;r(SRYM0GB_;}SKhSkS{4tu*y*V2oMS+((K*4+gGy`R!UkM^$F
z#Z$ie^nQOgi%GL3)4yu<PI`XIko&xjYwO8eBmPHoQ+K3G{>}f9DOneFtbs%NGj~*4
zhY0KQL)X6_y8eAq!1+nj|FdpP<GfPua?C&}x#D2A<|(JP)DJcva>4|~jf8*u-9G2%
zeBN*MS+(O!@_UrqzIQ)dJ?Yl3>DgBzJruoK)Pr)<Hu%KOn6P!a-#2~NhO?hdGgtb?
zM~PKETI?Xj@^n}I-N=%;lUV03DJ&B_@Mo{<oZA<j=P%QG(rfjTC9do6teiJl!B?KL
zJm-*@eywP_^tY80y#FfA=-(vQDg1~}*mBN`-LYH}fd@MO`ewZ6U$~d^g|AKiqZ)rj
zqm8@f3)~I*;o*IX(fx6Q@@Yp)P0I^=FNo}{P@28v<+I+S`3a|jmA4nC{^Rt$pnf{q
zKvDj{@mYD2K`O>tIhx^9&%WrasoXfdc23-DiM1uCUw@64d|J8hdKOdq8lwn7>suVE
zxeOC{K25)JLMPi+I3?;JpMK|+-E#zH3Ct2ulG-ZTvGm`J_ybleroBp<Cctp9$Ko2>
zmX}_!Ocwl232sZW)#W;vWBs0FtAE<s6k4Xlo_4ai<w)6v)119?C#{~`uEMEcEtB*4
z_(u_^r&V0bEN47x&Ex}73A36r`KDP;I)2bZ-=|ouO}T6i=OYfe^m!IW$7hQB1>{%W
zFT2!Ua;fbL&w_hXV|0sKN)!X)3{E=z+dfU~`|cT9VPfmtWBYfl*x|#Qne@Ks@{2C!
z>%NBPrsnR@zU*Y4`g!T^_Y9&|3%q6@J^JwI(V2&jK6ITHbnBge<(z`u`})?$<nRAf
zA1bKzWx;E~J=KL<eb2A+zV_4G>=d_s;=xm`apjyjBJ7sp-!DZSQ<)b2K!oM+)@Mae
zj=$bf_i*2YW82o}2-=wc{2Lb($Ij@jt$Nz+{EP4XZ@w-uU#C|b)O26eEWNh+!(OR?
zeV3OC->PwNn=t2kpZmtiUoB2c2}w-ZH`8pg&ucE}sjsh8to<{!##P4LjPsj+$W<MI
z@4<Fb!D1V(*0mYfn8!?<_GN3`y&0$Vu2g7fZGAB9-y~HV|4KLA;2YLihr?p4%Yyh{
zeJSLA6Lc+pPg{-;r{y`mYf~@&n0GyjN#gc~t*I=n>t@uHFPth6<+S7?$5sBT&leqL
zzs`AvGq%5M?c8&hLVMSV?VdH^clW2$WhX<!qNi8v<-RJBm9=8&odu;Q@9F8E&`Z3n
z+PGWgLu|(xk(nk_%hyiy`(?`N@pfjvI`_oLRXbMf2uWUf?AqdJ-(1tJUtb(sRTRr`
zh1Dsx*J$6?`@x)PvE4#UVJr<++AEAXBwM(?^mHAV_mg>I2n!?c3*!yv8TP9?%4kG5
z*4eYk{W~Ob$;kM`45wovM$aa=3UX)}HTFn;`Mhi4QNGvb>v@@c6q4Cq%KX<;+z`p(
z%)Knr`OLQOm$zP()w=OV@8RYp`U@9XH_BKmE~;*Ade_plZ~51dqYE9veyg%+y}5kU
z<knVI^Tt=P@3ahUat?0K(Q4yuZP8xYIJ<7b$Gcx&oqxx6_nV%0>aAaMA6pz>R#0#F
zwLwoyc;%;mbs1iqn?J4bJsAC#?*rGQnL3^odrtniQp70bwODi6l0=yeMr;>)BmL&Z
zpLuk!@$8O_XqG9q)_eKBE;!ugo!raqTE6jcgJx9Zqp$l8?E1`-_gwMzx#TT#muGy}
zojq6k$uUXswmUb&mgh)DpH}DJB5HGL>#I`d;~nYo+j+95PD}B$T)J!jhlV-2rKU5Y
zC-0u~NO|jlwH2J5Dmsb=mmJ(*9&VG%d2ukT%AjO(3xh)Df+dapY(`QORzzFMwgt5>
zI_lbb^cJfmZ^jh4oTco>mp8WvWN7}qVd9v5Hm%bwO(8?CS0}sBSs<fy!BtQBmkMu=
zH%7201^2&Lvr^~cr*)V36(a&(JFI2i%6G*oK-+hYh+(9YK&!CtoCOc>UN}-H^seRf
zTMO>;J)PF#Y?B-`7bR?LKFasS>cXoN>Fi0eoNFuQ%#AUMbTU}Mam5N`(Dzp^2G<*7
z*o#{G5|aO2J#_8nU-x5=Cf2u3ck*t0T^#Pz`l_Bw$|Y&Z#q{S>kLWUAm@4l!Yi;<e
zpqCleBFf6|h0gC0pR?{!rE@^%<?1+*GYt}xmzwV1WgI;-p!VOCdshyzXYE+daqj#y
zy?QH^eL~mE{*^YyyebOMvzjya+0qv;>y~!M*e%hD<u9$h`Bk-XNqdI5t<v4#V|LRO
z&zE){-B`Z!O_tv7S*KZEobsQ0J}5PybyM;Q{vW#jHd2#{=WPD|?ULS2!TV2}4g_d!
zm;JW*=dtM*ZJYmHWZv}0`h6<X9<>KK4EgMO#oKb1X{O(qoA`3YpJ{y2GaL3D%Xq%s
z*gk8|iJaSCQmWp*myw=am2;=hC^g9RsiR4%@$S!yS4%AFluwgQPy4k_@8(RcMlKN+
zk*-VXAqLN5i{++^i%W`2x?dEK6rZVPB!5WM<l4Lwkim>)oj%baB69^~g&ww^NGMY|
zl+wW?KHYwY)8j=4G!sD^AVA~2Uk)){e7WedYulB++UrT@ol@B58oQ)3a3plWMt?6^
z2A3>*0cn+|HTfzzKYMy!1hFd$)DpjF$vr`Mt}1x*Lx_&T#G57OKtrF=DP{?g8Jsb3
z51xiL{M=!D<h$kU*K-_7`I>iao?h@x=-=AiSueQ~HoZuDIaztmuREHRdNaPQk@o48
z?#&gpO<#O5S8vB#nVid|Dwlf|XG`WA+gFJ_mCUKq2)Fk%Hu^Nzz{+@<PwDK{d50%5
z`ioEHbxF5=yl2kq@BNFu>eQT1aXNY6<MxGRg-ZJ^`l4jqb@gP|?8@12CHSCX#I%q#
z|EvD;F~%ITYr8z-p=sKV7D>Crw&l|w{z&~%CAltsQ}8o!fddy$7ko6DvzzC=-r?P+
z3*JAnkpAp9Z9_-+^oKj2{>ZSji|Xs2c4hS+B@OG;(3mMoZ$M&^F_zDsRV2JWDa7<t
zG1_nAQ_<f$j!tvV*?H(u_R}8`mUdxrha<U;FD|s%EhrZ7u!7Mt;ivb@Bi|RyFS@Jp
zA^fBl^8_9jBY~3*D$J8bjP3R>cp9nHveY_$_n9f00xYxb*sLT!q(Ii)+|*jgaxio~
zgO{X1>VqkaA#6dC9;psf8$;!9t3LO-p!l*un|+<kmZA%%9JQY=ef=cph05d?Dao%=
zl3&SOF6qm(>eQO#(0P8+d{qbUrfD3mZB|ZKqTjw0oYVKD#AUg%wd})}XV~4Vszlg*
z1o|gE7jS8_5<V`ts88hUERQ~gQk9h=GC`M0!03xj=<fQH`{mCs0xbfw{onV;`ty3`
zT)riR(enzp%Q;&t1)lv-wW{05*K%@Y-A3aR`d<`|Km1vK#k1_qp0oS*&pBOh(e_+q
zo^!>yJC;T3nad@cEb{(eF|<DZrE1FJ62D6y7ax-5N?*b67xcsB;m0EjMY1K|tMNVf
znEUI&;g;t@ecnGbA4;@N7xMGkp;24ld5D>Fzw({F4@QSha>y&&blX*kJ&b6H4~+eH
zWwGchzpo!<eoc8UAm`k6e8NM66bl~7dm{Ggl2T@OG<nLo4%Z&a+L5(K!r;!{lnFPx
z*e6$+vVZRU=(p<fZWoEuA@hH(|JU<6<j;X=@;3jM{nJmqxi#JA|0$c>0SD)p`&M19
zG54*w9Akb}`Pa17i>@aXUTfyMs_rm7`>2JtTS`od(TV8rys7KXc*f40yvs=aZYwCi
zPVQQC*+t!@%_CmX=-?w!$whUwJ4z&0$*;N}Dqoxt{&(H4*sn_qwNrz_Pc2!OV-PE8
zw981XG}Y%-(ZBuLd>17b88-+{lvHvS+o@<2oU)|wwD@-Il|IuMH)yZ)2@jgNTVUqS
z6&hCFC7|UZkxH9G3RgZ_^eE`)+OB9(?F-Aa^0vk&D!Quaca<-l`10-VZr}4Qt5zP_
zV)(4rFwyRqis7?nL&ctl=jQI!zTdV@z|ZZ_z8#NFmgX;&mW*E=vitnf=m_I4j?V0x
z8rQ7$<_^9bAYQX8?|q4^aG5NBS?s-?v(@Y!o=fJgu#V>w-DFZ`I9oVP^HcnF)7Ev_
zT(y}}FXEi8?`>IE%^CY$=v`N^>9Q=Js}WTtFSjpUw(UgD-)Z7pyYhre?uvQL&-rh+
z@22chd+#@kuP^9-nUh<+K{Nczq(45bTVDEvIQzZr%~}47DP3Zl#>%HppRX<Ycqi%n
zCU%yz&?T+C#b>WIpHp+cE%5G+WcjO{N0;+s4jI3@B02rNPQGenr2UTPX?L=X7iIkU
z`S;R2H(}4ur)P%!TIu^D<zt~_@Ej?Hsf~fxxU**-I?&yyZ;^i|zTrkf52F#=DG80_
z13ioepH7_DS76vB@gfPNfbD<;$Kr|;KUrHGB$-orW*8P6V~~p8x<g!~Gb=Blf9)h4
z0hU^x9~)znI9ZGoP8?8RZs8Fz3OK>l)?vCbX8GJXeshg)`*|&gpuXq`^{taR6t?Ld
znLqVUv%;j{V>v|*r9}>^jElmpE?sF<)~}3W5>5FsX+`UVIH#>kvO2F$TCEx}aSh+3
zDs92LUT%k#9$lTdzOUhH*e+d`=u)NN>k~Jf6ZxjGlHXBft?-odi>(A_zUGL%?j)%r
zy;l6vvBnDvLq9(>x_H5f`561x(8bwveQS3{)}}DttV&Z9*L%{q(d;wF{M$S9s$a#{
z)wTO@9~81@{Cp>-pwA_?`|{0+YK@LZZ)&V8i0yaZz9iIP%6Un*(t@)O|0P|}Vw^Vh
zX;#KtlXvxIOiiZmT($>FzU$z)wqxJTlJb>}MZMYcdtcA$HHi0BwEsFM<>R&GEB(X8
zJe|TdW*>TOa_I3B<6ohb`_$wL?=+P8=(KgJrhaOb%&C34e34W0frCtT`a5<#n0V<+
z^sS`WPq$R}#!j+KwC@hFHEE1_^le?o>$6FF3MWjwneC~XnR@N{%$l#a)-7_eidwSR
zEo-}{**3p>8|O`*_%8ob{yQ^chtl&L`nI1Zem!wF?Ma^T=WTZlRYMQ1Sodq$uQmtH
zCtX|@E0|ak41{IRX;_%3hBb8ebn`GWi})DsJ0hr1!*G##7AR})={@lz^^V-7$5Usl
z);0_eXPCXN+C8@YT<r9@@46JNe!LcHS8!>|U|~@{z$H<_cW**$$*vO$xz=~JjgDw3
zgdAL+^tXeLk-P1Lz_x}Jtj;MNu^%KCIde3Fb{H)?0j`%{&e60AE>U}|YZRQa3^c1y
zat>*B@{IJvNYmmC`z5}K9lrPJbAMWr{Xy9cfrYyim`iUxUUA}LQ?J#ApR&v*8^a0>
zqT?cWGhF(~bY=IGgDzbYZtAb)UpC35mdkhMWG<$~URr)q9lzp4r?Qz9+wtxbNNr5@
z*nH;wYh#vo7f(L_vZp8e=YlA&nYT`;U*T|{f8XEk^Y6>-)@L7_Tdn=xl(SFjx?J>w
zwmCa47-a{%?zlLuU_yCF$P2-VXV0ISd|8BT&&0jUH961U-O?qZ?d5){<ogQEf91hm
z^WB8zExTgn8TqBi#Y$-PYY_!+i;M29Ci|ivJa3QRy0M}vSaOlLjsCv<Z+>@2UXh$|
zPDJq|>&2*DP9b-V_Xas-bmr(zT&o%+Shdx~?8dJK?H5&y>%~t$WJ=H%_{Nl^Z}3fP
zf-VP}Any+21!o(!-T!#>-@k24f$P-H+&wqvoDkE2jz(p+J_(0KvnzkHvNZ5B+w-h2
za5&V!zL<Y%wE6xAj#X#MI44X}=n(jI>ZP_UdudoW|CCHINA;D?>ar0KB;?Rwx#;v-
ztxH9m>)opyPjowUu9+06x@AJtxfd1MJK5RSawx2kF?`CQkuEVe@ry=h=ZT(4Gdnch
z{VX59_0V8XpIi7n&PbeThscw7$3oDy;r6<_4vwo$uScw!yy*{z^iz`u?N5(>5|?{+
z*Wp8__I<B~w@f{y9%{ThfBUBo=em>M?(fj4nBLnO74_!(@|>nbFXoRc=A4>;VT$3y
ze=3cJOQe@atu^Q_UpsMy3h$YX?@wQPnR)2D$-~`mrs{28e%j%MZy(z_nR9x~^G|oY
zFiP6>Y3=Sat$Nl^V+Hqad+C11zP&CdE4j0Ct$*_6MJu0$oYi&fSeCyb{Qujd*8`{i
z)D@W4T*Yi!{qb*!PEL)z7>m~XcS{z=v8-_LKk;%-?E20BzwL5(@l@wr-nE-GR<r%n
z`)B4)ys$39pm#RM%?<mmpLemIYq@g!GPBR|+z#I|Id|KH1sv20*n5QI>A}}tRTr(l
zW#09C>(z37>f=q`LS?G!UU;=%pK9`W-3^!JE9-*)F8k#xI$O&uGzWBIf}y~g*sJ_k
z`M3J7_!SW%!YEUCasLK^QnCG7Wub4f;&-k7^stoko8H1&@l_T_uRNK2;zZ(Z@7p2Q
z`qyP|jQzB8azv2i$zAIl?~96?9w~GU4*nheb^ELQtvfO|Hr+G6qO>{tgyd&#|E1lI
z`=us7NV+pW`kYV0bD?*OS{@j8W{F)jSydS#d{QC8c-1nIVy|=e_r5d#_WrTY(Jw~N
z!Y{k<`M$n0J!|>Hg|!>A|9{ZwtW~Z#J=;R}@x$#!A1-CyXzrSRMbC1bX{=f9uAEyX
zx?5k`WnGTFEG-($yJw%p?DDOK8ew9=rps3NhMtHPJH@xu`(oD34F(bhvHv~ORYQKB
z{h+ti{hsPBPGiTFn!ZO3pQma1Tz|{EN%{Fi>8ix%=2k}AmCG$1`cFPz8Wbq}USDWy
zK*XFaOZ<v7-5XEdzuM636|<JJ<XU%xn%?du#g!L0m7k|xn$dDetGW2r)QW}+!G2pb
z*UvD2DrT~Q;V$=;-FyuV%XP%p?@hUR{oq5}b#ETYJ(sN2pImt_KWobG-*;wS+0edD
zs$Hy(gYDgwps8>BVv5-^q*nZV-0jL#5T&@Bac)rK?ipt$g|c^fIXYIql<2KmD)c5t
zROp=|$1<q}S%HrjnjXA*KAnX{tTBQ)seZbu;*Xp%?#u(X@7D{k6!D(e+GS|1V8NB?
zkjUJYJLN>vg9b2M;m5LZ>9@rzOHFp4^EoNhw7Dg*RB6S!T!-{m3tpviIyT(%Sd_Ec
zO7P6~&`+XG3-&SoW_u@nAb+=&;2ZM;_scTZ3b6d+{Zszo{`yIGIT+j-EPlLyu)p~~
z-@o6XS`2evgq>ojGJy$M|9Jo4|8eh?4h$38Hf}Ns+jOsc=l3TcuBOZ8u@$LJ&~N;~
zp;orRsN;N$cLd7`@pP4)rkn1Z>hHZNaHcK6jHP)FOT_IkP1npT>=ldVuDlcRNhssQ
zeWOc?leSK~!V_<H`1o0ctRLGRo{+cr)PAwHvAM1wRAGnK1pYjZ4>6Cw&k*QPa%c@;
zWqp2V;(xxqvuaMYx|s*(|BH=ZZry#S!1>>H#pU;YUp}+w^@)lX6Zc#cDcZxk^I}^}
ziSxVlir;Te_?`12=GVFz?=(L3o)ylhpX;I=bb>P=asQ^AUcP!J@qmR&>}7IqTwh!=
z*rrvwPvh^ujqhRweyt3(bn5-EwAWyH?}ufTvHPy1i^%Bz-6`_o(gN>RJ=t5?N2f^j
zniRA&D)a1_>cr;=V%(^loV{^#<C`T>Z(5$eToU!>i1wQ$Q9eu_%<fVahYkp`^%+1k
zh`&+bmC1atw~SY*BGcy6`eSa=EBix4Ji8~>r~ixnS-x6oYv|wL@N+l17CuoZa5?O9
z#O0{Vv&V}bZ%{OP^WwtPn3R~5nxMdvm=Z|!+y|<pl_qarvOCGX%(dx_*VY@N6P9aV
zO$tAAVqHl0P2EMpi-eVgm734zb@+%&u2Xonwsl>CLu^^Yx(BwgE5&yEu!6`S-DlQT
z%R|=&zbbyU&q_b5Nb2-yyYfoIM!iKvB~O=bl1V?UxJ6%l%8vC%Vi#PPF8^opYp%#|
zpWQfbY1q$Lyz%@hnfTb$oQzYwT{oAfoOxw+;MDw3oh>uCekHloPW0KcVPou}9OZR>
ztkX}d@%<R;wpjbLziVK<lh)^%FAg$?DAg=7oZG{x^5kX2d26-%=hPY_rz|*ZR1u)N
z;{2B%Pji#^8Jn)TeoG^_=YIB?rA@pc8xGz);=nAO5yyNsdqE|y$$AwBZ(*j%U6ZG!
zuXWe*vzT%8U{f<|BP%<%6F1Z9FcX$A!5cOkDif~Hy3Ty5o#X56RE5`GnztkmuA8zb
zR?~I2Q%KoTk+)u&x26Vd(O&u@wy=xuy{&88_UBKRpSZyK<v_dK0_KX^d-JA0zdQYe
z&W4tG3zT25?@9zMR6q5KXIjq1m2-|LIhS9I-rX?m;tsB?m$r+)r}#=l2(w*lX*5kt
z05gw=e;04~{(heU3-><m{qr^EZ{~0i6fk$V<50+atbUp$2jj!}>hoACa@QVlUwbUd
zM4)ey_EAQT-@BjkHXdhO`ONIRp=8g;@OS-lCfOV?kbIoNGsQ6AoJK`&)8*NDss|>x
zf9ah5LF;9y=)yRb3F$nO<6Rj7Ppq(5{HUkRSn7_2`qL=&r%}giE*3615hs2|x=?ma
z!@udxLMt8qFe|$?38ozEXh=9>w;?j!Mdn+S?8AHKbw4N_;Cv$~YwR{{&U%UT@M8z3
z6<a(FuYD#G&BYh(V>U_NY{DD$n|kZEmpzN0GxNjagVnE_&8JVF&hU@1kH2BNRm{WR
zaqT*LwijOutxgwB68gHqhe>6ofsLg7REY(N|1;hkl%BNUW6g`r_LUo!z4G=c;9T}N
zXVTI@w$#Nf0hiNbr@fEqdKuI7^2hJ#r&rv0^xkRHp4C4RG$n){c3w7mU&?g5ROm<P
zozi!kR90`kv3KgE<2J6BtR`KXxP?!`BB8;qIo<!b;0oc18z&!FFMMdt<OBMrqYMwU
zD^Hf4`DQ_9vl`zli;PDn-t;SP(dYH}_w~<eW3HP&QsM+TYE8<t|L27+-nJ?{hmFtD
z#Zlb7RHsDThau1qwAy%wz)b#j)d=a>6i{vi;aMkMJksbp+~stC>bEu89JS#Bzt$;y
ziFJIjdC`lO7cD1RPO#inGEy{hJa4epxx^`@<Cp7&K0Tv~$z@jTzeNr1BX-?`2e3NM
zCCqU%EnP4AM@U#S$bEk|f5xq;e9fs>MXwZU1YNuE^$)1|{O*X}&%hG+I>)XvA~UTg
zCfA9~)dV-B!EI~TXZ!BdPKWoLUD7(w%nAXGe}OhuK&GD4E?Y)J=Am8o?CDT06WsgY
zk(8L@>dzW3@f+Cp3&e=8bk{R@)3m^G!k^Ackwy9+G)y_9Sa_S*9Ge$hOsr(RyL8Wr
zmIcli6kiCGfalOkB60-au;jt{*MefwEYYpJtqBeW3KCa48IQg&Jy?{M?a8Ctxl+||
ztBRU;QkJI=lmqD%Y(CJfis<pXq%k%=I4U$(*9zh8wmX;Scimu^CN`sKnRvmkBZVvh
z>z%G}d+a@Mg*#xc!&UAkej8_jhG#)9T&(A192b6Du`iF!L12zqL)5w*KUsC`*l(Ab
zusA-rwWn`hm|%m_jNF86TPG<B)YzO>;3%k{cUWh`?E`ttdw9Q?J-EGSNl>Yhz&~#{
zK1YW;4j<z`xN&@Yb>w*xgN|Lg;L^?iOJt`C$XxAo*u`6K@pGw$RQ%oVviH@Wyc0gL
z;lrby98q#FJ{tX7SUKf@&Qs0PhqxK;JJc|L;yq(-aG#-;`D?C79K#>BPtq6i6ZSX$
zW&0uhB7gG-CdLF024@xtqXQEd+XM`jG^{<u&C;mI$rG4xg2`D-;?RK!Y}-}~3v#r%
z7`QYk3iHfN;9zyul1NfqZ+z!}!ILIsahaVDI#}IxERqfia#zlEjPF>4DdywFp(HM|
z5vovez3~g~h?;-R3Lrgw5d}w_<j%d~j&RYlC^{&}4H9VDp02iOMnFc+lxruFocJV~
z4oqM%%QjTx;mJ(!V0M;~_;euc<wAiutsY?K1RWIMZi_KsX;zfX5mj0nl$$8{GNFUr
z*-nD#&;)^P+!F<a+NKyRX?Yto%{$sH*3IunD3@{5AwiM8IR#Hzl+|S{AD-ZJ_Y>gl
znvyej^Otj}>-@5|`DJa>QS6wbASIG6KKDw_+|9iZ!E+D)H2ZB{<XEV^rnu|=PM%9U
zWv}gYyJmUjeM*`)69<=jfJM=viNa8$ZB_{{-X^=D1IeQIS39Aqs&i9ZCd6(I__>-_
z_twP4vX)@t)WpRymO_UYFJPJYu!F}v#X{-uM2Ws51uAXV-!%yAjQaUAeEO^>+g2K@
zK5Sa|r8Z0~XXeV-=?g;(v{%X4hn{s=qQ}<N)!Ea{Q@#pRdVmi`Y7JC+tn^svvdgm0
zGs{0(Ednv1i*sh3sBtQ>_{cMnbC+uhXw~xxgCCq16$9)pw(mY6u&Yr!x#DP1+H&1n
zSHyC&y3O{_P=yu;FXkv(H5Up&_nrAnoLO>CWUi(asDtfuZ1%a6GqtQwyR*q_iUpOV
zm>&KRTxX~?<8f*5RAG@Ew;4Sbrft}pc=h0&FA=RF3BrdSDH|<)+|i@2mBcF~yC!%>
z#|Gb4xu<cgWh;C7E|mUW`&utLyl&&VYR}ltqwk#hbpn4UXx{!c@j+X(ecr^Idmg*K
zzYuxEE_TJU*S}92x!w<Xe!<%9yTo1RH<w$z_Ql<cEHS;CRMMU3viTd2fau0$o|oe?
zZk3kxZg-TeOYe+4x1&v$w|ZfqzDd^H-jyu}isvrnxbpGV<Ezf{SL%6sL|(Kq?KSxH
zOtgffXI9*vLmEs4LFK!*nyG2mO=J!4PR`8LS=+mG>&7i+)2<!UG60o*Q#->z=Ny3+
z-b8)`mwgUf14JYC9=OV#us7oCChIfr^(FZpR5h*>|4`-l;b3Oie$f!|F!6}yYhf9G
z?>uf(Xz0D0SXW<YpumwO4O?uRTt4H5tXf%n=iS641ale+Z=Xy!r~q`>+4z)g#{W-8
zoi==$XLIq%YT@}AHl@5fRN^X=90VfF3--P_n{!NU_QsQGIh!PBBQdukctv8H0w#)U
zeR6VL*AV^F^`PPDwjFFi6UA2*%J%7mJPuV4<epoxkV`-0e(S5<0kz_*-XD_Ay{dW2
z>1phWJuWX^-Qhha^h4dDDZcgVgB8q+`*svQxFTr5H^-{rRh(8t=cM$G3Wkpl*FNDt
zztw8W3;_l;*%>7Xml{3$C&d)7cv&stIQ`G*V2s8zZre!-F1F6{UrIvXNltoUcX@(*
zQ=hfPy%YOqrCQy-v1-|^Po+Ee2Q6gp4HlRqo8jcYYeoI3o&zk)+B(cbkDO*w=dRJV
z`w{javN72;*!7OKlc=+9+pBsf7N@VyqRrPG8x2h)oLe_W{xI-<nDUkJC)+dW8Tk$S
z8h>r$l4q!6{>l4dd#IKU3u8U=-&}D!1qL3@gboI0HVLBx0!-VyHQD54EFYX;bmy}u
zIv~j0CsA;uL78oXvCkVGVe7+3&KYb(Fi*pI&1JP(-lgpi7Vnw<E$-Ft<m46q6*@p#
zrwhGbTgJShL#E(KgEE_p;e!rF{nUFCnft&3>|g<Ro(j#cFagU49gOZg73+Dq=T>kA
zeOdQ2{Gn2b;=1SYkug`~<r6PlJ0GPG%e^GKQJ{ax=674|N*GgLfALtj_np&uo&%2k
ze7Z*$E>$@%?ArYH6homn^hmE*HPNe&L}V}J>V(Fu+5sC5U)mM@Hh=4W$SPdkX*m~f
zF51kv!^wi-(;?5-Ot17lxIFvPyZ+>{S^G9F%U|<SI{TNK>-~t!<#%Il{+N{$^6l^6
zrG0G)2eo!+U9G)-c53fCkh+lXy|4BK-nGm<zloz^3tN)Zgv^9xg<o<LmNU*f7Ww0N
zL!h=yt&p(ehUqm69wi@-6L|3Fj)jA{+APD%JuA!fS6`TI_3@>R^F7nrhfh7ud^*rO
z<3Jlr$V%(QQR2sb+8k4I*?O(ByJzzm&=P007FFRxzou0Re-V@fO$hp?q&P(wDeBA=
znC8Rb4VIn6nRv2ggMs22{c{%;jSlV-pD8d?U}nNw;h)CyuFh{gA@Fr#hH=b_w<Ub%
zxFarINCs~Sf$RctE@5*}F@kIinW<z98Fd6VHW+4s1|m^L9YOPgq6S^(gyupIMFSNU
zFX!l5MVFW{XdB(i)h&zezI!cc`_$jrnJK=mcjdwwB@4WDG+s}BJ@K~oL%rFr%&+FJ
zx{<Of8@!6~L*Y7w-A0j5e9!ePpX>Vf2mh(%WpPQr&(HsV{Ofi5Q2QgX9lOuh|N3$6
z`LF%w@6<>AeZT2_^SSz+|7QH(`|JN(@fAfE>sSBR@4Z#O`)>@}+0#2V80hKraC1LC
zbO^!JLgD@TmvqokK;t+<9;Z^6#A*~1Uzh%7Z3>*hctTnu|G=Kc&-*#f3$VP@)<5=K
z{L%KL?x{L04<4@xb64M5yw;E93-1;4g!>0-nZNTsFrRSWtyJkj{(}9C|JnXY|H$WH
zaAvx(#&3@5`b0s)gcFR;kXFOCRl<U7Z6*ds8rNQ8Q0Q<lP-%Mm%B7`AQIuz6!U<Mq
zy$H^>lD^-U3&XqqwY;nUe}3=!okhZb>OO_V{x6iZozZpGy#I92caMWhAD+oKHg`Jx
zKr@En<oerxOYiU(_bc!3{e62Lf3c$S{&(N6zx<&xz2Rg2jvwm({w%jGxPNxX(ke6i
z|Ns8R8Qq$9pFRH9{pY{_9smFPzub@dqs%+Yyb`{y4@$ge{VVYQzS>3c|Krxz*MEE6
z9Q&UC82jC1eZ4gw^}GKFXFBcvf79sc`3ku+%2EE#xBo7$eJWYo_<eg_x&8$o_g~N7
z*Zz2#rS@G)Y4VAL%KF!S2To33EcbhD+y5n#C-uL+ci{f>$naH1W_r8*_rCXE)9L)=
zsEa$DqW>!Y%hZ~CZl`gc*<QbAi4UKDJumd@e6&{3_VRc6i(1-#otq-{?ccrm@9hJd
z1-`}wDho7+?~5(3FIKx3tbH$d_5S78)Hgfdy;^;C^}T)npK$-J=l^5>qA@+f-d225
z`mXbP3TC}>w7Sr)@Uq_C_<P9Sjp}QwIoW1Lcn23=Nj`6HxA@wx3#(3W%Ps!+UX7{B
zsPOx)^WWVSAN*K;F!RM%W5w4yemXYK`}x*m_JcF3!YlcG)=EvjyiVnSk70rIy0SC_
zp{LK~%VT76=j9tSMIBvJ_P_Jp{N3{+#C_Mjn)kHc_Rs!8f%$Is{;OY`D`~%2I;FVZ
zV2@$?REsS=Kb^1EOJ!V_KKm}iXutpMd-8w&F8q6I&-<Ty>ncC1`C5v%UJyy#m;bJ)
z^UWO5@4?gEbodL_P1zC68T!d+LB-x)-7}A?|3}7edQ}u|H8219=k;Izyv}=DXZ4=_
z)6$!t*Z*%m{I~uAqxV#`ev7K`Er(aP9bIFrpj&<`^s$5*Yta9fzU^L5uYY`L8+z%w
z3VWR4tlQlyDrLU@ip-a>KleB|D`xw-X^p2QN$UIEKIp5R7nS$u|M{;K)hj3W+ueNC
z8$FA!<<x(cweyeteBnPW%D>Fzc75oN`#<9xcN%|a74vaDbK!TypD*blx1-g!URg6K
zyE}c>QF~p-fAOChWVxOl=M%lmCmSs3Q9u9BiW|2-KHhk(%HFPb^P`Q|er^2l`{3Cf
zeO-?e(tmt>`5^5`<i~_`8SQ$RJ3X6=j~zT`W0fv<=gj8fXPC@2n~Sd<JZEK<E_-Lq
z=3+j<{MRq$loo70ch2(2PT5Cy_$u>$znN3Jq4-3Jb<3`|n~T34JZEQ>zJKS(*DqxD
zzxi>o-~a6ok@TZ=e=Pd^KmXWru8ZBeJ@mla2gmx(6<XieXkjON=Z1y0+#8usSNJPm
z)s>x747l5W?qjpGz4h}6E9y8x>{(X*zG+#vp_n5ipWXcJn>l40F87_wL}3<|`N_OF
za?!G|Ksf#Rk%N9RbBl9+{AEe8XSm7l@Qrx_jJfas?zsP(t}m*p{cwH%?br7|uDcK~
zf9&=B&tKnvUAG}#{>|NOdvBZB@2E1*fB)_6-F>}_FU@AH{+y66_ot%X-s-{Gmi?6<
z7W4l(+p>S=2Xp?qFU|XBX5_2gn%!Pj;!<sV@t)5XbN;t4I=-bX+B-8NU+&S1jtXPJ
ze6>e2xZT~4%<cqnAc7a~`+PCy|NEljU)rMmGhgJ({kkal_pD33@x}W#J&To(&F=h|
zmb9l*!ri?mIdacTiF~<Z7X=@mJyK(Ac;DvCV&!MEJAbAn?W>&O?taEx`19E#HJ>;1
z^+n{XU7Owc^;x2<?8bXOYs`gTJ9I2ozBaq_Z(7p+${X(PZ_I^%pFL9dc|(8SjeNCl
zvpZ`aUE!Q`oiTK}{CvJOyB@8nIB?ap&!0Um_Vt>I4WGsH<nH`woughd??z?Q^Y|B)
zhn{!NXK%RGx!{?4pZwh7h81%)ve<iWH7+h=SUJ}ti+#$i#-(KqE4S`Ha_Zo@#^q%U
ztLAoOu`jvRxU#Ha)!Y?X>}zf{t`=K9#oGFLLzwl3t9%7pg=60~gj;{O%Ga<}IPq;m
z#Ovi(Y<E}&UgMjwl{x(_LzMNzYkVuVGH1VKh_=3Xjc><R=KQw|G1h|D`A%$QE`G}p
zYaMu<@5WZ<^0y3ev+Ffg*{#{X-D2EVwrIWVhivwLw-^tUDQ%Dy$YE!@ZB@Etb1|R9
zHs<!X3`y1#Z}4esWA1*-kZgVN2A{>Y+E+QtFO_BGvMc;%JW<E6X@0{SBAHoR>Q`>5
zf4Rl|`JAY?@mX(IUZ2<XcIC}~OK-`+%wXCikDFONN9+GhOFWkCbedOxx9#Hn(S%Fm
zggZAQi+#?m(S%E_g!}XU=k<FMKI*Kx&R+j__uhZ9@45bdP2BU(HvWIyd#>MK58lhY
zy*u}F^Xl@?-}3I?dwbhLYj5NRbN+A74)%ZlzL@!*&yU&c_P-9?_xUlKv;Ok~cl#f+
zJ3*yuWrn-^7IWdZXI-j4U+C)t7s8$2o+<YAb(jl(OjGLHSy1A#XC{aP6$F*8pD*<H
zy~tPlHM{fQvxV}q;6hmV@7co^R!N{TR_)mA_QH}Q7SoQ-?)>;HQAXDAo)5TmJ#1r@
zv~T8&e6?q@+bc`Jg|N>VbN<&aI=-eQ?VTBsFL&*t;OnzTs*E@CwOCjs?VSlOUE6C*
zj?@`%yzleIod5TW4p1RH^M<<LT!l*6fYP5IUmjSceXff=Cid}<k_T6}f~v;)htCe4
zjXzQ8^t^Mfdd0uidFnOuepE87n9Gq>_vpt(OZJXior}s2tdO<HVxMrUb4i)QO4*1k
z_8C#Xz1PURk*UaHUvR5)MVZ4Y*%?{vD{ghJDm$=Bc1KpB*Urtwd>LDrqu(-wTVK4&
zSFx2j{w+g<wcs_rj;(cGcbjJmr@n27v@W>Dw_vMq=G%rS>kZfVHf$Bn&7Bi<;G!k_
zgIk?D${g0o&d6qeajSDz*@1PkJF?k7-1_6KX?eqP!*#w7TZL=iHpE+hxX#D0O}O!G
zLxQ!zjcY5F=guuYutC-$hh5-y=aDjpjj|Cr>=L&-kCh$R_%=A*dg`{|Y-_Yy8mYp@
z!3<vf*L&$-@8z~u&#z?t56br6`z<8L|LiyYtKS%)mA>$Vzj&CBLw=q=XNlY1egA*g
zo;v;U{Ahto;{um<Ernw%S!L!HGpw82k<I?(R^!gHhIMmSWV64y)wsKiVg1?tQzkXf
zWv+h95O00)I^U12%=K>>608Mp@NsNoZqA)+b?K%hJJ0RL!(|K`=X&I@i`;HJTGp^}
zZblBf%x$~UZNZt=TemSo)0jWFjR{U}1em6`_J`csUvg`4vESNn`dhzU$@V|`?Mlx7
zpj=2hl_ABR5!z(M%Cz;H`v1)w-1a{Cz5o95p6qWrkOH^%erUYtuh)kw{#Nh%xA&gt
zx7W>kZ*MEV{fs^Q{j;}s_wBvCy)dL|b3(q{o3qXG@87$#?}4;y8$s<`;eXE#_Sbzd
z$0%hN%F2S9y25YII?Kp{+P7z2WQrfc8oN+I^itN@-U{5*Rr>`lWh;yg@A>qY^FMyk
zQK-{xF8ml;3}ckCpI>x<nz}P*AeFNBY}SC=wO`YcWH|~;Km{*qDVs0%?V=#4scXE^
ze~y*oC#waoK0UtN7#coDlz+#rV;^50Se-prl)voXp|j1i_fPoTa!%OV|HB_{TmK(6
zA3rk$SPNeHbL8XVVm^Vb!me){0<8<K@F{E+_I%qAWWC`EpTX8|(<3ZzSbn&|=de{c
z@NGkgwZT=sfUUxzZyQ3b3$8wzQk4E&=0X<xwp)zr$`-Ab{gB1J?-t{RGNm=L0@;73
z>|!@pKXr?7bJ>G6vJTnmmu@j`ElXG{n~<%3>z2jVhQ-Bv2et|qzHNxHHn`4rVXJWI
z+lE;0J7;g`PFwN&eCvjp^((d67qe^KW;|W?Xp`)L9CoAIjAzS|Hp@Q9VYj+%wRQRC
z+k6e%m>0fn$gnoN#W!Ia^U}8snbw83_!exd%_?4QRsO1&J>)mz)jEf*^8?-z$$Yh|
zUUPSS=5F`rbAG*x=X$^L<2<eRD?k5R`b7?AhSOGg+|24Zvl}-T^R4*Cyy|a5mi@w8
z{AkSg|Lgvrx^amv#jdT-mNPuxq^>PiesmVpQb^YOQ_*qf_1?N&GXs7c^x)&zBk6AL
zvZ<gvFYk-dgS)&s(>7}gZ|v#0TAu&DEs5j%?u+X89_{z~vGf1?#{VnXcP*NH`-gf)
z%89?Xz5lMX(PsN!f6s5(_0qHJ=Ct4E>6yjy{^^A5gp@gF7Bc)d7y9S@{qFul%+4GO
zz3-G?T0C)k+xyhn41%Zbm~knr`Ok0i%f^7E+$gQj{6>%YowLmEo=Lw;J2?06Y=KYz
zwPnujws!lSfAPov$Lw0Ss*kRk=2ny%)O3EYnX$Hm-AuvWsUIIjTe69CT6|P0=1Q4z
zg~y~kJdou;k6L;7Pl4zC-{wC&yP;Nct$y>DiT#h)O`Q0{h#~&i-P0M>k}3MhuWOB#
z#kpNN%}^z2u7CJx>V!Q#z0(yx+G*}EVu(Ao_jEzI<Ws%m+o>MAuN<|L3^HNvO#5?z
zM|##pCu6saHny`nznt_~Ao<_;a9!G#^E{tsF0?gXl+O0)PYT0%p4&4oS{Zw!AA5GD
zV~%8p@#B20vU0cEXB1{jMjJb4r74{4nZMISOVZ){y2%?2jaAaz4xRZiUGkgJ;@CQ)
zQ>n+cotZE#+IeR3$`4m0Hv|c;wC{3T>v5&aaqW-GUAAj?Tt4P<LOgg)rfWu6LYiwz
zm`>W!#IPT!t+D?!?rnOcBOGVBy!DZ$cHgF-!bU&hQ(O1^pC0Ho@51NlP3?bv{yF);
zVZHpn*sQ5cmW%KFX>K=Gxaj-w|EJgKFXq;F@fWVy<8Ahj?^d17j*5a6F^^YH2%Okf
zq`$Y*=4|?x#S5Q!*KeQH*w}Twr_5wZ%gcrPe(hhmr2YNHAN>{o^uH#$_5Uhg@%QV6
zKj-Ctsn5FgpE=~;N_L~F%lGF_cbxNY?l<E(SNEIO{V)FW=lEZv|LRrW>%TSjSg`re
z=VANi^XuQQ?{&W6)u9ppel6Vl?{B{Nq5ruTD&$TSB>aC{sygS-{HVGAKl5{(;3_Zr
zcw5}$Px^uf@@5vt_J66i=jX6vzhFP(C9|U0i^(5<<){7p+1a?Z=+bQACJ5Z@oZwXG
z$Fx3P;a`F#ugbv$OWxL9r%!RO(w=wf{|{Mn{``m=doH&%+iGUkuKwziHSeRE-r|x|
zZjw{x76qJBa@cez+de0)X2);q<`s@t6tWysn^*Pz|0DZf^x38VM{?%G|2Cgl`%gQ>
zWWM_6N+SsIJX=ry)ZC};pDT>|)e%J0*%T>tMh(BfD>C!WJUl5g&!g$2$h-^-^G4RA
zrkaX76m}?rp=-cLj`s&N$^xa92AiE)ATwo;6Pw5xeT63iMI4qbGERJo&qa;w9s~sa
z^18HAsEHR0*TyY$+*s<xVFre5EY2*&i!vDhmZx3be~8&~*M1Xi)AZ#Vc8h%adAq;H
z#I;xR*X<=4tE7Uj{!5m-;Ai^w?ffU9hnFs|=ZyXS`L6vfsVAQ|?*D&p+N+usvGz&d
z@;lG|cP(}P{Azyf+28(p$M$c&|0n#|*Z7Bwy5iw$PU}dg29%w0wA9=h8NzV-ceiy(
z>NJ5jN<Z?WMC^XQ-tQ(k<?rvyu3U$2M&8rqS}~XPLw@v+`ETadNIp*OnZD=r{aHKy
zA8+f-ecP^k@K1dQ+YkBjgJ1jK%6$L7L~H85J=gbF{#bwZ&gDfR%cn~oW6M~`cJo*L
z60g0#er-QKX`8FmirBC@Yo9OOG_#5M{?gd`t?B<O4tmd%{B$-zf_LwG`G^pe+<(F$
zw$|4l-TyK@<@3+G&Y6FITYszHW?FUZpHYLQ?UPr6W$WHV{x?jL`jGcq{`<-L-oy3p
zPxs{g`_3|{>2rVj)BW~#*Xv7b{zda|IR9Th$8~mR{r|Q9RS!I}uXPB%Fw^?nox)`c
zj9Fy`3a-uHEH_u<%Fm-e);I1?IM}GczFnrGU{Tw1ag*AMUs_f6Wc+jziraH4bkc{W
zMIZG*D?}VGC}aq1X-t^C!6LCx>BTkI&L=S*=USh{l=PVi^aWnKX>akHJ^lZKH}Z4-
zHgB#ksGRhYJLO~BgMZ(vf5#{Jc^2N#;G4mF<IL2~P}TpwKI>Q7z1w|%uD+Bii#ilO
zZ#&TR$jFOB4+_^svMlKETgiBf=R#V8ne?8sKXQ&eSKF{~M`EFKLHB~RpKcm?YCqE^
zy|4bn7GDC2SaDSr-X=HotZDgaKhvPZwi(haHxv%|9$5Hd188B36^m>WpW}R2hAzHt
zNTLLhy?$53iZ_ei$T2jJK2yK?W&MVGMFs~wIl?Zher4TlJ?)$B$zKQ6cBv|EI{{s7
z16niYzighQm+xw!qd&5y%y}F$care*Y%#`d-QwJb6*2@&z^iMnDJBG%Y+K0Y`1`#~
zTFSK<v%3zrLEz^&yM*JJ_bTVJErh{izh#UTea!1E&wgF8ygtpkRY-dIs*n5ms&8dT
zO0WN)QoLsR0`*7sH?sEsU%qzU#;?D<`zJ>}U8>Dyy>LVQ?+^9weT}94jLuD9br$;=
zFVp(pmY;)-C!!)(#z^}7|HtN4|6ejQ-7$Lm=kd?K`Ul_qZ)8t7p^<znc2l^_{-26-
z4<C5Fw#VCk>cId>9#motUp}ws!71Nf_SM}z{bjqx!-S8Of7y3$JtH>N`yKl;t22{U
zxEyvm=(OKyuhVX)oxc{uoV{H3bARTXD{oWnx!RQC{(pV{-e&bV#zp5gc-8Zi9Ck40
zS+lA!OJYS^W0pi|o9K%+$L;KQ<zBoyXu<wirsuP7f@9&qM@<hPu<d~vM~+5b#2o=R
zR5~FUzCfC7SK<O`Ht7WpraU2s9ZY+y9&>@sE;*dQECM!rMVmQe^CDj_4k>WFw=cNd
zd5M=f0-{XkBGd^L!WjWM8*V7T;VNO~F2-)gg8~Qt7sM^zGW}d%Ufzpm^=ChA>MKjD
zJp1O~ul0VprMHgitiKzWKCL{h5<#qwyf^ddTvTE&vI3~m@3YWV&5gX5jcOKHDGd#j
z4c)Z(Uzs28ul@Z`aw2aol(N23&++S6--#PSk#9Ta*iPK6Ui8i7+`JPvg)OU<(*2)o
zMdQB#DY}l3f73bVZtv#Zg=OaF-nC7P?Rk@wBCfn!;^xs4dTwQgn^uE}|BqI!xOb?M
z`?>BM-d&%C&V4Caw%piu?w`D$t#6|KD{B8XU;i()`pNz;2Xg+u<Nka4W90`OQSRnj
z^>g{eEO-iT+@GiZeE&VqH@~)TekU(<Fzbg+^Vfp8k9&AO{(t8E_qzNE7L%}O+4Hq6
z$(L^zZ76=L9`n2N_wflEV<Y3=m1vbG^;j#OnY-aB+t1ij>c>7uckVELYvP%F#!-Id
z1m{bcS?xg#SsWi?`;43;^ZuRai?-agdijRkn=h5fs7>P&43@d`*!t$Sw^rg)rhfkS
zYhAr|X#J_`omOGb?fw0~I%rvy%gwD(U)v*ed-_YQ8X@T~ob&Rc)8l6{J@dO*<SFA@
zdiTlM;J!aI?I)YhyVaqyH?nJ`eXf4<)TzF@%Zom{U-xg#d$q(=U-oZi_EvfC^68R%
zp5;+XejmDdNm_63#Ye2~_<kvWoG#(J^{xNX)~9({99wkn1tq@Q7q?&hM2J<`qi8#y
zmFYS)KmL7K%6!${=WTcRWVICw=SjV@|DU^9*+%~6*Jsn!y4)<^C*8T9-}Xi>{cvPK
zGS}^*p#AgxbS;m6-nZ)O@;k2@*UH_f`n=ZsoR3Al+U<L1u5$O87yn8s&|97&6>Csz
zn7A%*bLeris;d)@e$RXn#PxBF!*nTLsmrFi`+u_PO6p4L-dvEvq28VmcKYmsHvwgF
z2NmB4e5>&7RGa<I_~W}fUIz`C#M!sri(SV1MrG%^+>d)=9;WboJ3HaJn9F2aZ3}kk
z$+cS!7w+aV+#+(R>dh?e^PR_AO)a%QXuaJ&tN34<-XiOX!4HJ1{LfWyGqox_dtdIk
z-_@s58%uO%F6Gqw6Z3xN<p}$7rX{QS%Cioit9$Wpy>HyYi|o0^$G38xa`E_nfmK1^
z&GzPSUNOg6v7fu|8>s#=D@}^B{#@>+J|p9s4)e<gzx=;vZ~7j+p-yar&6Ps&Zj&dD
z|2?&*<bSBi*uK$X$4+bgS64G0%sXUW@I~ra+-vFS_uth_Yjj(2v(ogZ&dd8oK_}wt
zPpVAXd++7sa~0qJRi#_XrzHm~J)Z1&A=cgFr}Og3d1CL6<bT(Ay5`BA9mZL;*P}Gn
zpLuOA?CX}XH*D@d!<y!qmQfEKEoF~0|KeO2sD5sb(}k0^6Idp3$UAFEC++l$3=-K8
zIa%j;mSuNv<n|J8rRa<YDmr#T6VK_~Y+tiEGFY;38t1l8b7xzvJ`#0ubBl%8t-oA%
zA2JBLq<QR~{$Bax1D3$F)zdy!M>y|u=``^54lgR*G@~snOE>#@#RIo@n&Aas51xG%
z<sZM|QJt=yaGc7LlUo!ctqnda)yR3yJhbB(<NBsGn}ouGgJat!w=QhqRGMfz@7(Em
z_f_=^rr%%cZF1+nVCYTtghR~RQngG>OrCfAoVe<zD<jw5J5Me>2w%ye8qxE<yLt7s
z>X-WZAK$8KODtU7ApTz8d$a3;kCOkVO6`jL{CDQ9&Qq$d+OEvDnSHLjIMOs&*_1Wq
z1n0EFO9l9%eDv$h1Oxha|2?oNZIO$F=-r5wdy33ixSg(Mvk0u~D>!@Zmc<#1<xedi
zE-|Q^^vG6s;<k5JUrWEf{XjHZtK81s>ghjUf#(sjwhHmtM*mcm)tS;y-dI-h?%Yo6
z59_7{cuucL{-gSd%lYAgnk0w2LWSm5_ut1Ydg~OganVfQO_57QeM9W399^kj3S83h
z9a}El+`3I;>zwbOxqi<3yY^`8Ei09F)tK!v?J|vzR0Fp0d^e9UpCcLk=w+Bk-!q-)
z>&xEfdCZ79YkpztrGx2tE8eZ?@xAuz%|ZLmzxLgzuRm}3bHcOY$7Pd^zs4RfF)3&+
zoxJg(|Hqk0IjiPs{tAAw?1b;cXr<|&Uz}$yZ#euX>Py(-shu)Utd3kN_dcs*`Z+Qy
z-S^zr(mPwP+zYDSeXIC^`=O7MV|Pz{TwizNQF3>`uHCj<w`JEY5l(qAzdAO5>-xF(
zeMI+de02HB^_Js(q7^H@+e_QGtmVDsb~r|7inq-2uKCT!j&^&m@oe05>1&p^(N^{o
zU5~7mzn^5e@lWx)?PnW0&P&|+TYR-zL^1A3(?xE+w@uBgA(0z9D$JfcT5O!PUSsmH
z15eWw<rk<fR@Kg(ZC4;wz+wGa*PQbwtMP5c7gx7GX6y};_+r~JTmF-MR0K!sv;F5z
z?>xNPL@H-blMdgUCwKl9ZtTw~*kt(BJZ__|*j$;nb}Jjr->{Ukv&pc^wEkb#S$O&B
zGmTaJW}g#2He~b9omCl@qWD(mrO$>HfgBOM5j<A<57rzGY+uahzSrW#mMh%x7Dr#d
zZxsy6pU=5{>A5Zj-QL&RTi&zlN62rnkYHs9Sikp4T^zewRl_v*wMXaQjBR<@z5DX_
zN821W)%|r-_KsqEn36L|LR%x%v@d;T)}sZ>&VJo6PwM!Z)H>VR!%3gm%|edlyQNvy
z)ou<@t2E`fa%bN>%XO1h)*LnF&oaHnld5!eO6w=)fZvk6ubP)X?LF!}ZAaEhTisPI
zo*D-<7u@`9p7ohA*({jX|5=-S`D3q?(p!&S>`2T$ef7rNpRr6!)Vv(cmEO;L)KOUL
zuWohXaondSF$X#JN|^_R`aH4%A7(w?!1u_);W0z6_yGnTz6~$3LX_{hIDa!-IL}sb
z?vEAbyqnXVr1l+fQd+l1mGSr9Lx&7P1+Eu-@Zw`&(~r4v>96gx^`E80H%f(dwn=^N
zajFaaydcE<ZMDpv6QMbmynJqTEUZ0YQ!Q@Yt=VxdH1m>`PgMKTwO`g*xTbfV6PXEG
z!4BH&ee&<kQf>J~+g<WI?+NWxE%JVn@_BZUp3Y*``%m4Pa&*`0{)$f7ej;z;U8UbQ
z7wvW}@4hEyd9v<L<E#Up1Ls|SS8d5}a%#bwkc=FGfN~zql8YxNy=}`ePuZQ9`Q($S
zg+_U@u)5IOWjBufYMQn-IFZZZlwF!hib;wzlOH!<>hq~GhxiiLhp)0dpD?dF^1*^P
zU%5U<UKe`(Ri1YX8+)&ql<t#d3tYbcwOwN_mC@rDBfz1-K85)j^C8FiEG*1T8@m{E
ztPSoa?sVP(fySLGiM%JXJVGWnYPc_M6KrB`TKucQG-GjF{-3swg?*`VCeC(JpDupr
z%5Q5hVOgvwk@`4*(@`v?>G-MYs+>vs?;Os=d(YaZsIj}fGH+Rtb3;^SLa@M}75OsT
zmaIMg@`+;PaU=6<JKs5P*tYd_L+Z7fRR!V4y~5M>bUxWSX;0e|*WTzh7PBLw;u@bU
zEp8lTT#&H4O8)d`J-!!PZeOXK>hkQ;EVqLezgLxikC(rrY;d!@<r`Odyob_tn+PMG
zCm(eF#6-<pe~N9Fz??VV&G&KsKD4&(VCzGL$&YpVIrVkVoPJ{cpkSX>;mJ(LBHquo
z98dY3bg%q+eUSAm^JP81V~>Kkcw!Hzth#A@^~k07dpUa89p<gv{qX>!w7gSHrKj|Q
z(`Qs)d-s`N63>_%R#f}?RcrOx6OVMdn5*t*o0>(fsBE<4es8WR-L@$)BED$R1<SP-
z8e6Y!J;=f>tK+j&+5Z^R<3#s@C9#Y9K6?2Y+DY#Yx-$3e(tQ%Ix)<)dGOHjbUih|z
z_!sGI!DpMT%pZ#`YT5eVO8Y!xmgMyA-e<>-bJtEkXKVAT|A4LY`||Ve9}0wA(@Ydv
z@lnsU(~ouT-u-P`3(sz!8kH<*@J`H(bEV$iHtX#AS-uISYXTpie%jm;ew;Z^_gdoQ
zdyg+w%P<#D{`dG&eMwl&^|@L3tM+<Y`)bz~>1<tk-GA%!6?=JKzn(uYKC550;LwTr
z_4?11xt!L$SFP>jPTjO@QRsc!*^}F1Umg|RxAO5C(Jvk6?pwG&%sQx5+P?Qo>)hi@
zJ2&QqAD*^T^uLP#GWGXY>c7cP|9kwVZ2pyY#oo96w|CzQ335w+5WH%(cWLp;ymL%1
zw%oZerC7^y!led<+z;C%&L3aaSCOAJIqx>p8(lZ^o;Y!tjk|XR#Of_ya((Oc*|Yl0
z_lw_^dgG)YrN8v0lU>~6HeSQhi-j_=uVRuH2wUx1P^{Fx`RY`6^Nz(kEtt7C@E=cJ
zT+DntHCZllZu72fDV)LHr-O_b>XWhy4lqP4UlVQ+VIIMM&OX<^+fO}jPOIhmM{eqI
zPIp2&K5c1@`^Lt&uB_@$n9ID%5{nOxt>1U*NB?{_hjoe*r{fLr#xUK&XO}CkAKPuV
z>~F8$&TZL=-VZMX=KVTS=p*Ui(LDLV+~!V^JxQ}xH!r^Pr?K3}x?sURt5>TRgNS7L
zUGWPqJ1uYZ<CZ<NSInkeZSKrU)0~9829u2ApGR!IoV5JtIc;7k8Pguk#M1#!UTp7M
zcp@-g_UpC4{J+uV`|aYMA6VQl@uBu&Mvk1#R{d;THN0O;A6$R-XX^~<7hnIx9lz_J
zvU+OW{u7%XtjW$5>#}_o%P9FiF6`7(eZFbMt3)c|Hi&!KFY?yd6PVhNAN`eU^#iWh
z3DyU!D)sl?xY#|{Q_c7pa|RPv1hXrz#8!6q-=B9|Owwi$(`XYqQt6ib=*;aMy}b@v
zhsx~}8;f*$cI2uvYEFogh`hle_Umgx(FS%sohKJKbV@c{I6E)=qK55agO`cZOfNhQ
z%ZLfeu}k<}@=PwZ@zFYKfv^J~LvC0cXEe~N_I>*CHM7{%Q-6J4vO7e*Z2EE8%uVtB
zN>RB3Rf-pz9R%1iWEKQ&k<hTe*1kH#!o8GvWtP|TcP?pb{a3!1GgZ!%WGNERnEp3K
z|3gOBiDOsf9Msrn%A^!c@VuubeATz3r2LD!pK7q$Dz(sco>QG$535bP(eVGm6P@nv
zB}V@raTPB2x$*PsmCUA|q!nc=N;HGb)2pIZb{g}VW_exDwn|8LZ~J{ewz6xZ=kMu#
z%a`4X%__gSx^m~|yP?x>MK0R!Qs4Pb=%(r>?<W=_6Pd2=E1m81;Gnexd!}z!$l)5Z
z8+v~vU&@v&J(pD8K7)_r+XH2***_*Me!lnC>;*>CPd&<b;vb)5xa;T&5iJ}4q8V2T
z<uk*!I?nT$f9})UjF>&QF1`HyIHby0KXlE)%B}B<U;lq(Dz>BkrR^TR&gEvk@^52j
z9SL~1jO+9TRw17cm$IL-F9~QUlG!Pd_1x;pQS1B-XDiAdhlY1<_IU9t<N6W5&#M+&
zuhIVApzoKc=QqE%F-Q1q^1@Y72RNnE7W4NQnuzs^9?y6mbX97VWb*uV);?DfPT#tC
z$$Fd7deO%lL?+8i^e&CII2pL9zoXpZb0EvL@6T6-Kd*VC$LPATr*?_q8(-ec>wCRZ
zvnv+ey;Ax8*oW9sCdP?3Z!nlf^}W8{xx?`E%ynYAOew{WHC`~!O#RZg|M0;!uj>I%
zrB=l}{*%4nlIfh6pBOD(esNsRZ}oZ6<(uN)-&xfv%w2cxkJPH@!=XQ&TC-Oa@QSYU
zsh?cEY3HtHN%MXF9h27_(C)3^5Vu&t&$E7Z+ol5j*R5Yq^2jb&W+wc0=llDwcsA6|
zU!b|U;qbhtv$>Ax^cj@j-94iuwy-Dp;Ki$J&P}>|JZ)j@e@6dxVQ+m)|7|@Lacka9
zwVC;IwiUK!7n^-~(N>}Is)qgbrCI%qTl>BW#}r*RyYX#-$I|?qyDhUaBvz;0J>9tJ
z;e>hH97`5G$+@y@+d_*^RtbrDg^T*$zfw6h|9&NFgG${yq1Iaa#Z2$kM;6+wUsYMq
zDf;m5p>@eCR{PHL*lxK!`4(?Si;d$F=D4}iY+1f#erD5Ovu`R&ezQOPc3@S3GygM3
zmgzhOXS!8RC7ohR_4$2b#-aY70>bH^*Qu%tPGFMpJX#&`^ua!Z73<ra?r!+c>!MMf
zce<xM+3CcNZ-yx^iYMkyPd?aI%EKl6`$6dELlUihEUPx^CGHCBaB{y@7aIS<XlXpF
zM)u-O)7Kf#J2&O%R)?vkLa8-3O!;RRoN!q0{MGbk&h7l9)JQ9@!VT)vo(Vm_vHN=6
zn|tTa++OjwXWEh9D$`!QTmN=%TD0e4t7Em(DyMzriu<%6LiP;f|C7w=eOFeM$Q<l)
zSXZAiuR2rdvwEwy@k+Z>o7y&}+)v$esA}5Unq#+&*GQfFc;oP9{z+G>KPl+!O-Wg8
z>u8|-Qaxqa_Na5$4qBzDEa&SxbEY+T_4}8~N_)J_K5+$BIdKZ?c~)@X`-;wE4^L0J
z^TXy{qN+`7_94M}=kszNuXQ{V>%Q#Nig~Z^N4`&ZxU}=pUNQdl*WP}w(RMdZc^tcp
zBW3X`4z={@P5i6P+4f!K)x67a?eo>vQf>+5eVG^7!Ze(`U%J@MtJV8hSR#-b<n}(w
zdcWU7$y?W2l-!;kJNk9&m1SE_|6OTZInmqt%}MSZn;W$9J|*4h&1!yb@>1>3)jcK)
z*&c0K?6>=y>%zYQHr(#>PCso`N>;fW_BhJRrFz=NQ?L9=Z||zRyY19c!zHt=Im+#y
z)$oeO810$*V%n0ceL7o|x!;~}vfgx|ujc@Z_J+Ax6FW<P$CgZ+y{#d6CHu9yUpv*Z
z?zvpz&hQq$wM%o2)4r<9Yi>Tu?q09E@7sb^61TI|D`)R8yT-u#b;tE{I$P9ouJ`#g
zN(dd--?*SRY{Nw^vk1ldjGm72{k2;zb7{7I*V(Bin(#tM{gycEZ>6Kj7kX}Ast@{h
zUq?ztLU_9(JFnlG;7g3Rp0oU(b3kbA1i??cmZh3|NqsohlzHcA`8BuO{QIVUx#-Fi
zRVL<a5_d53=J)lJ_iQt5JQu!>H}%~l-H_R*x~|L(+isM;I@o(t#{1y^2TxvjdB^?t
zyx;7<799+1Ud-oSc<`W$^6FCg4c984CuqejWmlfs9Jup|_~N&Iv$kY6UHMwsSIm5l
z=^WF!rpk+vOe+;C-_|$%_22NH^PBC$=BwpjPTqUC;BwINrGDOXr+u1Nw}GF_=jOpO
z^(W#V54>4b>%Zas!b5w{dDYI}bZgl?oeyiaGO#}l2>5u`Y`W(nwHXG77>ZnX?3ZUb
z&C6r%a3`&><(T24-`sYByCN3X3g|T7zVtVHCj0HV2RMXts++=Z6scXDYOa)b=DOvF
zqYsT*<j)07o&O+zwe7(lVG*u^Pp&egGM8MgzC8b~$J_R%yB=@tn)W(wSkUC_w!xj_
z?uFVnv(IUyyboWlx9yjB)k6E)rG6~=ya!ALt~bQoHr->oX6uP9oL5sDjxjmjSoKF&
zZt5CIgDVI9=1w&HcXMX=)`k!4KE=O3X<YMIZu{{}!lM&GSL;MyE#h6g;M|($5k94Q
z&%$P%s9NxP<|(-xrH&;Rnf6*<aFU!i>3EgGynPo}|8(})->>u`+NHJk{G9zgbE6gn
z7_aH;Qi|ALH+Rxf`=W)@cJhf^8*lr?67V(b@Z}48vj3d6UjF}D3;S+?W&7F$#khNS
zMy$&9-msBN<h8>>zHN1ZX|*wTXJ)=Knf6(_*EV2FGjI5m<V?*=OC8v|iW2tmpPBtO
zx;*NSV`}rJi%I&Hj_FP3_^kLjm%I2ReeOt%Y!g!0edcGXc29n5?VT6;v#rZGyJLUl
zPOrPY^jg`<t+B6irc2#kx-`v3)^pv;vSlw$ALuu`wbUl*>*s`iNsWoIUdhvsyShI;
zcIWAfj##TC_pAKxj+SV4$L@-s{_c8U6mzQClHXtTr%$ktFn)TJujsxU&zAct_2!4_
zPgZSs`P}w_^WD8K@9Sm%6_byy@nAn<ZOZF0{UbBm)zyKQZd}!5f4W5EZhy(1zP{@h
zKc4!%U~&6(IhK%J^S81wE>;D>$F6f)lh24O_U<*Te#dYv?9Rg@yi5xHpS#X!R~=})
zyy@~}J_D=b<B^Zgtu~X&;F+KE(0jsMJ~1DunCnm19(<`|{mH%8n@_5Er5>l*uLT!x
z_m<}uOqeI#=ylz@_4SIprK?KI!Vi|I2i&w0d)3!AjU_`fLo;LL=XG<><+ScuGv}Sa
zi#@@h2Ik!GmnIjo4@&PAuH00hy>#)S(vU6Z^j3$M*0^^tq#rpU#Lm#uXt2LylK`7H
zkAdNZBWY({7>+uKFb4&t?zSjA^Inl>#*I~h40b$!3>c1tt&QPe7G(>o&=%#pvLW<w
z&0i~R@Bh8^3)ND8xkxQYXlQC=+jhPGz@sON3d1f~-8xmga!KE()5cq6d@iy(`b}LU
znDF3n7{}ocOLxwyiQ-Lvq@$R_y14cJ)_0Nb^XxflFI@1xeIe`Wch&C^_s{*BHnp_7
zd-cNH2MIC%oFti?R)1Iizo~n#g3-%YE7S|Ny*e@fGh^8LxP_vtO<TF|XPa(IZ{k#c
zG?U%u>Cf6f3SoEsUS^2TdX#$YucoBO&M?D=Qf{U0uVZA5W8GIi$jhlJ)X#aec1ruV
z*L!rbXFO3~l$)n1E*pOTlVpPFvQ<Jn;SEO^dYBFP4pcDuFm^F(@Gamun6&PjVrr7d
z%mZsDEC07=p1~b!SpV)&=oXzNX6;kj6`v+1sP7R@>wYJ4S96!>tb(q8LO)eMc|S>I
zG1@urMA`D>Wv;u!^5lNB{chzj_FL@p?6>N%u;sHB*%$2FdE~lKeE0Xe;^$AbuT(md
zRleYWRejw(KlU|q-bL5cmK2I{2eaDWyrg$*dWL%R+jp#-bF1sNNo?44Y~tz_4_zX+
z8=L=Qdt)N}-Pg>Tbs~e#J9)kKgvBmmaXgn*%jP_m)4RrVMNse3`lE`R+s+ofp5>7y
z<08_V#OOHN_U=*bt4l8_hQ?-{{b6_SSJ>gc)w}#2neX<p?0%k}{73p;xY&7{GjC^o
znb&yF_(HXo(P_8$k}qtT)us9ld@tUbzTiF2mpP5bvW%ieC5&^UxZ29o8ZK{I|6Eqo
zn|ry@B>xi&3%uo5?Qrs3aL}>2nT=gs`iR;5;A!)>t_qnGsJ7;}{k^-ques)4d$7jP
z%CR|7fJ4pig430pYYSI?nY+p3s^=evyOM{jeMRnk6T8-aa^38&s(U*$^fJ%vHCXoT
z^-o9luitu{&V7$td})5*NzMD&vSn6FH-D1eduQh7M&-ozuYU8t#I6rGy?>eBV#61+
zFV+Q1g#=qZU@+Obv5w_b^!7`+CVf@69VSX?1Tnw%u5;vGknJ@$R%~UX($e6aF)MQ~
zce3tKSo<MyapJ?uuuJOu*M~-~HhgjSz<(PFvw#~8;di@lWTZ5APW!Idmat03%OQrj
zsB~h|;)T2>CU4SivL9H}xSnm@Ej>@;d4U&PO_tTT^GPUV?@o*I)lm!CU+4BD%zans
z6l)9aZPQ-~&9+s3UnR;ev#enuUt68lyS3tC@6LTJ?PZ!1QMh4$>BCoXJLUDyXs^-k
zkp8Bt{$<uo(W9>CuV%5lE?Jws_;(}sTECQu%uib`o{F>U*g9R;=i5CUCH)4?%8m0)
zC!B9gk)NrvKEGA)d)Vol^?Uce@;dL(D!rn>S+MF0t8hlvB#VG&YnT1_8m)ZBCC+i3
zj@;56Qy#3)w5}7*nfk1?M&D)yXRxu)Qv2YY*AB(1=?H9nQnp$|MqTZUKBt8c<FoCL
zo~sG?Xhz9c{aZOxqv7+DDQ%|ve_T54x9ai3-RqZ~IPV&I@%o#`J^5ajXZ_GBTw=a7
zLU3-%siNxsQ$f|<Z*?!mFWRqg|Ebp&*EscpJ5G*<vo5uZ=zpktta*O1?xyOX;1ct@
zMV$#w<*%$`%=Y%&pS$VcXVDr*ZE>Ryv!--UVtcd5ghN7PZJ-lqW?x0E?altBAwGIj
zf3$A>>2&&ly20U?*!6nFQ+hfl8J?CGus`3^&ce>e&c}XkwFR$<)&KJE;UcA0y-zf!
zUwNLf`i-KFmf(WTS@G76*W9{$#hSxAZ<Qu)nl5H&IL~KFN2icspq$#N!!DIGH28zR
zJ8nImn<2gBhU7Xw=DBQEykBPRo-1ahTN3qRkJI{6_JfQE0~Z@CIH6<~wsB+ER>PY&
z++KxbDP7XuUu3tVVAfJrv8@|dl<wFs_)%DJK1;$}(OjL(WKZern-OP@X>AM5Zfw;#
zen{x7u&2$$Bf|UpC!OHg&Cnngaqd`Eos!|EMQf*-WZ%p(lwN%^;#}P6pT7ia`1qn?
zpU?HU*Qp$-d?)a5EpJ~|RQG}P`A^oK*FJIS$lZ)-uMZ2JlQ-p>b?Q>cov!An@7cAd
ze@BaE{nbAue_<6%z?{;zTi0Cw5t)$fw6^QWLDLU4enpcOORv*0U$^C6s&>wrkJIaC
zY?o5C37-4)-J0_Bz1FYqzCLFldH1Pw__m2E4yV^DeNj;0W}mhu=-&g!!X@_$5@&hb
z>wGfTBaY+8g9mF=49~CUt~SuIFM7}!bw%K!@M6!mUWFzR{V(R-sFV>r_taU;SN7rS
z-``%ltp5Gb^0nCVf=ymk>rEF_U*r_$`gzA<VPu7Z?zWBp)^bcyxj*TDgOd6GpBAlK
z{5WR!C)M)4+iI%GE+cd;^_FTjPoScu;OP|&!8|WlJ)iVx!IHhX8#m9FY23LX?Wz<P
zld$FX!kwxwE0hYk%RN^{1ZhnP*sA+wt80i*tlIw7`X4uMIJmhp@=^NAndgQ6s{V@p
zvi(K=#rsOTUGuu<^yN(nTc^49&fODFW>mg%D9?{;3x4rhtFLg<n@@$4r)|7oBo=i{
z=-yhRWja%OL*H3*PjBhf{C9{y#d*i#-V0xUus;iRTOXY!Zv6O=>DQ#wd1rdp703rg
ziyfPMCYrtb*oNo4FLtfk5s)mF<+FE{PRu;VDLoU{eK+~Fp@(fLyM=D}JV|{%8NNB^
zKKgmjQJcK!G3TZSKCbZsCl=3WGv}Qfrf9o)sZo5)`6EV`IA4iMtubxO6^S@>^YNwN
z=MDuIX4L*V@cX@$O47@F#ab_#7c4w!r@o56X&MxM<d=%tvEyi)!}8X+y=V4rKRTf-
zZtdlO<qOaG_$-|GO1fQT@tN=n8Gk=HE16OYrNv7Y)!f%Di;$7v{#AF+QhG+usfNdi
zdk(F*mw(<R@aVs_Q*=#_rV2}+&v+@C)%<6PM$Oh3ezm%*8W$8iPQ1MAv^Yw_`@*qJ
zIu#8D7HO;F5`P9CoPQyWyY~B`ZyN*FoI0}6;`jN-tAEb^Et;?(RAAz{r%^{FFW%bY
zH2>+*M<-v+5^ZD&Yjk<ayuhiB|M>1-3F6(ZJ0n^R8Zv*YFARBi=!n(x#I22;d}^n<
zxAmwsWL*}mU=7R6NC@vO@06=?Kj>t6rC4(HGuNVk8PB68@;$Sh@!Vma_!Gv$(%{a|
z>s#lFm;Y1GxOM1N!QVHFo_Md*J%4<!ik@y&=ayG@&Ym>UWzM{GN6hS_`ifglt;sJo
zEopRG^W|9l4X0@_23I^wo?PlFQ8?3Rkgxyc+$Jl*6GaNA8@2jpUHYo^=#-;Q|EnV_
z5~KXnwcJeBU#g$vsmlAMid8`?(INWb?uCjKvPQ1_xq8J%^ZhpK$AnKQ>E}@T?sC4=
zpCep#!4B?;^OaU#R!<Gd^pYtQSNg7!f4irCivAS0*PE~0GqMmf_kOFpbvtOfjLUsr
zLF~;->EEVjuz9})%}an2u-=rSh$E~)s-YQPt1e9OdiCu2B5OTCr4_f_d~e$S+5h4b
zcVJoK_x(3c#joaYxVi3muPQSm_hR*)ww^W}g@i3&$g|>N!{r9&3$jdnuInR%jN}jT
zCAKec-ViGQT7$BK@m%EW<HsD1ZnwE4CqH3ZV8#5)(>JY|a%^M$f7h>qn-sG|oOGwp
z`~PGyUz~1+Va3t~3um}4NZa4(a3LwYJF{@tDYi`)S<QAGSXbQNYvf?CQsim#d5_t4
zKmUk*mA2m;wy93}w)v*H*}j^GGAp;FJ320~y>)D|&icZJ8KRZX17$9BE>2U@jyqmd
z>adCNDEG|$6Mst2jCY&zS9Q+OH2G@9-{za@l_A8-FDeI&wug3i`lT+F*1i+*$p46t
znB}p_y%}!D6TMci(Y#mR-OI_^=qhhev!*)z?(fjW2V0VF{{GtI^wqJq`s=r<wvAE&
zot*vNE4W@>)Ur6@b}3Kgt=pv{?VW<k#;0d_K+DO!*Em=r-<>lmpJutOzxTnDp7n;m
zrPBXLI&GL%zS=G!wp#kU_7CByCB<@KiF-Jr4`{FAk9dCW*SDy9ADM-w+H)=_3X@m(
z9eM3s(YL5`I?3sh)~9bo9+m9XR?&E$v|_^7sVp~~D@yby1$2B)`0>Ph<;OkOy)x#<
zOK;ee@#O8IW%GFtH-7P*Q>M4>+(M^+dH+hUES);Z{KE5htA7UboOfKU@bjA9vz--L
zt*V<EefHi+-*ooW?la7*KAH7)JEvIowFGsysuT+}g<0r@W`9hRaew%1dG*#V(ZX=C
z`SVrTrrny}wo2p8!q2^{ex>it+qye5-0w2a-M_|f%f8*8%xNP!J@v5r<)a(?T_>(N
zeSi1mg%yzxAMZQ*V%`<L$Q?JP=UKfFT3)o@&69)8#NJpk$MDwF!wUp!(>G*xaIa_H
z<HNrEZ1>~t)uGFG-!r)7fB$~VJLMHWl6zOgFqEgqJHC&Vy>>i*haS%}?!}(}&RyA2
z5xP5T-(%m@J*zE?GivJlCeHpiBj@3?D>I%XT#a>^nH#j;)2Se>AWZCq#GRiP7*1DA
z;y?5#tvhSOE_SwMyVjg2`t<r~<)ojo6|B!iUv0a1$ly)Evw~-5;wtZ!XnO1nFtHVR
zeQ~LQt>CTqliH%rKD;)s+V?5X?_$1*7q<ksq&v%=jX3_tXx@ar$AZ%e|Ne_MO3Z$I
zV~GXpG2yjJHB!ueOkmh_E-q(9QmDMa@#bmPy5&)?O0t$W`Z4dl9+EaW<D8+*cHcDF
zv&kwi3NF1Ao?Q5I@~n`UeqU?Rl?Ob8Zn3|rT5MaetS0g*JM%TO*Y*dpTN)L4c?uIw
zFt?UObTxlSKAB>3a@vCj@47?JT|KIKY|1N_*gkfRb-PwwscZ3OpOX=H(_wvLi>big
zImdIZ{c*Uw<Aa56hWsbd&`CB5XAJmO<k-jfsLr0oSfs78JyiGm{2M3PzbUUY>CNP@
z*e*ZMx}9~$dMkx{OmiMMn6BRxd#Cl#<AiXD=H9q&?%6TlvMK}_*%%e>AE<bADEFMP
z)9M^iCtgOMu!!@!UyJ!YezPFP=u$yPZ2P_NO#kO6%_0uRZm63&WmW3$$zE4GP3={r
zcDAxuP3B&feEGl<@u%E9%87eF^Vt*xTCu*iKYAofmPNhr%n_l~M+vrbUP+ZMe4q6G
z@m8+4?5Tm~raQ$&XB1Cj{=fB0ito8-iPqMwcV6BJTW>Mp&4M-8|1K}Nf9T%i``e$F
zPyMU-VRq-ai7_7OEBw}cUBBkk4_`M;t*vv^-y83AV7gTma$07d=ncQ4oYT_HzB9Ns
z+)kaElKC(sL^8=ClYOb|l(3HZe-=&XVBNCg`?dHJB9~@`T=RKj`cFuYvvonlrN-nv
zKZO-Vr`mTg1_ZkOvoYOpFTeAsi~Hkiv)_3LGQa${;_QPd|BE$h>NU1RYSh%<C~mX9
zy3VG`S8T<G{8ra_kCHFh-MqH!R(w|Z%9%W`<*gjJW#_aOJX*Ku$~mE*s++O{gTIvR
zZI8?DekT^0@?PwwZc$}}n3bBheA>mki*~<U#=UELVAmq1E>^Lvt4pjWKV5lA+Nx?{
z@xRH(V|ylDkL}OC)P1RU>1Ro+c+Ke%tKPlu)f2vET4L|K-8jV3d8%=U<>Xc^?<&7l
zvrf89U8>=|T)+3sg%do!XD%G!xqtu9Bh#ZX#hW%w`djpP#)G>%ZyB6!`Fg}^uXj||
zmMayl%i88FP`Y$xjfue;zW%b*^YucrmKI9BI{P=L+2KOag`f+ULKk=}cipS>{NYB$
z3z-`)XDweE_nCReE7RrHwIVt@?#(~v&ARCF%DL03*UINT&bzQeoSm7EO@_xpqBlIz
zgv+&oTYJw2rXHs*b|nQKF^{#0`)BTm$Y!kAaEQI(AeVpxlePu}b5XtmcgWhfs7*~z
zUVpR;xFJ=)VC^p#nKe<Rtd6aXS1W%zPKny;zx;X9t~WDxnFfA2_r+|%!s?YToOsq-
z+%R>_x*0d+YURHkx6)^u_Z($3K0l$pmD@kY=7{8v2mE#!O)o!moYQvFx*fbKI8(A_
z#fsATUUIJ)TAKO!>?$5I?|AT_shPWI&RP9ep^ui@NStEc{bj@P8E@7q@X8bv95~3x
z&i>~hBgdvcbD!!9U;FF2w*K1Q&G+U$eHXZhyGf_mrCf6Ij?5z=p^<Z~1)iE5C~_~E
z?Xr#ObUV}O`$t1IhP=^P_Tc5@)n`--4Ffe3{(vuz(KIR!3@mBbscPh%0$QvPskHd^
zzF2qj@7KW#7Ve1c)GdmBvhBpvi@q9cFJe2+h|S!a;qtkxdk)LW;!KPE1)z2KN~`|f
zbx4>t$A_s~e?iWJ-HtaDas<pcqT3rBFRH1^SO;HJun}9;w`|js6gHvdOHbZ>`y}OV
z=FUe8czEhB?h}m&PyCRrZT7W6Mf>`Gqe`vX2am42{_|Vu*8-Jw^{1x?io4&)6iR$>
za$j`h%R`0*(;nxzimSg0Yt8$4m-(8h>)ZOmO&QI02cCz%tX7|$cCPW%ZH3<q@y9dw
zGq<T29NWxVeWduu`~8j&b~J2W(C@VTqROOf3y!<?mogofmwq$f>8PXs<oC{59+I23
zK0I_}>ytf>_iyg%dw+i6mXe0uja99DuVt@UuPKjsSCaOg|F!0;Hnk~A=btd2IpW8?
zH%)=}%(;UXpBeRrs$0E3CHnT`tjfr1NmA!CY`V3#_Ajj6*DJM-_h8ra1FWw!gSVf3
zw8Da6J<owj-9hf+1?w^ymPoD??t50z>ojHKk{)IIt-gse>dFmMx#B$p8Se0u7zdmU
z+pGQS{H*@ap#9ekm`d5M-Tby(xZbL9vG9uEdMn4HPRpk+m3X*_OG#=%M#94QuURFF
znp^qAtRh|{ENpHR^|;pPz|J)BZ{Hhz`5zBr_QZtjC_TL5MB|pf94AU*J&tr<KF0ib
zXU59feYg4LpL~zHx20iq^h0Of?Elx^hACdWP<LQWP5t`&EUSJxOz^N2yzCw)y!OL^
zjm*nlCofoJDD?Beg$sS+uTKZ)?X2CrCP%*S>~8O6>m6zfFU&e%c=e|Lf{V)@SZ3|K
zw=rOPAY;zbdyg`vu=DRex3I!I;0AZ=?e5L98kg+UinwmTw2W^?RY-S567#Z4XV{)I
z|63~(5tFH#I<ec?bA<wr-=endpYLe#aXCC-$@voJ!OEVH%p|JTv#i_Mygzt{?rF=e
z;K`4cd}v+x;*FyG!`)d-g5rj?HS<$^f>oBCc@zBh<f0WX-b6;e_1kKE)i3tntgQT%
zdzXHjHtY0_`h^*94sy<3w6Prtxldkw)wfo<H}g}@#k)$oUCX-Ph~4D5TV5}6_iWv_
zT@|K(c7HfB;b~=Y#m&NbC%NX>@cb<GWX{e=;61x~V`}QPRIcBZiQiW^`&x2p-TS`i
zw7ORG;UyO@+?d5+tSO}<y}+{bdrDI5su<^~*)vRoTbr*&8vaYLbgn4Pk?WP7F1{*m
zvQ3QWqL*^2&8D{tf>zD3n7rzGz}p>5w-oI^?#tC~+OFI#+|JD1H_=dm|E*u(!yRXy
z&CcMKp1OA7CetR>H~Fc$R~o&J)iy5ueJhLgocg-p)`Fk0nR)l`zg==v!z60fU&aQb
zS=|MWlC=UoZ2uoxR0l6Vy5Q)7qZ(g({rIHb=rC&f@g7~d(85$?dE28a8#+zCENPo}
zrKDiKPFdS}!+$cjy&AL58}6AL<Ge6;n?ShS^#Y>@DOYZ3>`{>7bVy|oJto|@X8wT_
z+^t?7YX8G6O7;Y%eP#R+di>$3o&%8ueNB(2T5ny!V8ndVZdR<!&B(}juDv(dH-v?V
z?V8GGk^Szm!sb<%cllfo|6Q6<=P%D#d&lNg%p5(9*E7;rY}lB;-fEwcQ}!&qIiKfr
zJ)gE|{^LEdZY*IrLUZ;fUbR_p{a^$mYkDGQ;?{~&a~|_=**M31*{+|Leo9%zAAP7h
zcYWgj%oP=D-`MA<tq~Iv(z&($&m)-+r3bDv-u0im>F3vGqr0699_wah@Vx$TKZQH#
zL6!t}2={+)gG));kE8y?8Ge;_xWaA2=*zA=_ljvz>0`Nw#3Y4_UtAkgtF2yhz6i5=
zH2LUazlBkf6AXea1tmUx{!{vB((|o4TVDVCTW7XbKhFFur&2)T^wpnN@z;G$%qWX0
z`2V*sT>Y0HBlqb)>cTq{#7w5GPxv7^@z|AU-rKu{cOIJid1=~<XAHbszJ6%mq{U-n
z*fsg5#%z1vwMKVUUH{wtRG6$V=g6<Zl75~QZ(2|F#cyvucdx$Nb!OEqW%(yLL9#1j
zB|8gQ5~WtPiAo)MyS#FB%8tg`pN*|k4k^6qJM`_&eZPH6i%y6hlRW41ve<r`>hxt|
znTK*8^c`AqJ~?V(UQ@vW7dG{6UB87JbL1)-Or9y6O7ZACBeB9L@$`ZPG4|ijCspW4
zOj_KNtP#p`^wS*m$&>0dUSIr?xlmEpWJ&8WW>YH%?wh|wQVmMx+KO~qz1k63*zhj@
zXg}Wtk<br%3tm5MRm^@Z_A~bH!p14G20LCHQLUDLAok7Cbq*ij4UvLhdF93q*Iidm
zTiR*zrkQb#|CXx`-o2kJ46cN3%UsRV*7!|QLdct&cguyI|30UiQ?0hFT2jk)E4@HD
zb5ScB+tCB;Hv1Pe^V+XJ$a*R1{(~bA5;U$Yf72j&zRt^IStQ%>&Zl=8Io|y**%D$=
zWwbG5(~;lo>oXhs+dp%~F{fvpSS|ef>zdN3Z?_wM`8dboWRG!R{>?czi`gXRFtWYM
zF7W9;s(QEb;f9Ml&%Qizt!{3R;eq)c%?B;_m)-DBnEJe^_KIXejeP&-#_ybbp;p(|
zC+|5Rcf`#~Yh}`r_xlf~c)2(IKKQ6bLLu<Ar_hh!kMEYgzvuEGBlL8^wU^Dk-{ZF5
z*GO?MZCkzl{^}>Y4m~rr@OjC|e#yhaMdViJ*MC+%Jbu5`xk?r|uvZm7@R_dnXRF$a
zH79q3KUw?GUp*|ov39@Ts*mf#S0svT^160%Wzr&c_m>A7?@8w<omStz_&Zybe%LJ?
zcR`JV4NX57^!D->En`_%V*a!CZ1c@}W8Nj#o>#<q#?4|iTQ50l=Z86y-5IVsL`9p*
zKS&W*e|mc3{#_1fPa{>&-Y(l1QO)%5@cRn~_wWDXIG1^gSNu7nwbM@dEPa^56kD*<
zyQT5!slP=J?{B$K%p9Dy;2zs6-RYf^7}$)TZ(C?@aIa36t$NX`JzUiv6edZYQamTh
zx+L)U_gV7)VpixsPA*MSklNhB{NQ$DeyPm%NzDO$f~9<11ujP$SAPz8)Xd_j^7F6y
zN~d4`svj;eUtRQn-*M~ZX=#0@E}Tg0)H$6TDbs8#R4CZQttPl)^AYD0;TfjyKF-b*
zJfC-$(VyK~_DzX_u<n!<%v)C|T`+Q-r^C&iy=wJgOAp>HryX|lOgZJSl4r_Ehws5M
zFFdL@nJwbfzOmr{p2|hu*Gpo!3`BBl{+~H;WY#+Gl9jF}x%Cv?3KuvRE|~nt@66my
z(LT383+F&9mP?kOc*V6^sNnz49WLLTm+Vlj@=|D3YvT@{uC~nJe4yh0w3+KZW(M7P
zZhutlukNqtC)-ct-<T@N$6h7#Y`4i;J>S*dw@Z#Pbrdc05LYu*IJ<grVX4X5zGYVf
zH&}Irx^H|{Ds=ne3ae*(`k1Z;F0kqfb}xJt@w#g6kus4beN9&^pU-+>E&FvD4^Q8$
z$rq<*9?V>Lj!|}+x%TPI6Ay2gvq<TfN8|~B4m_UoqK)@Zt_Wj9en8%aKRXJYEfiwP
zHtBx3_;)>z%=7b`t~8r|{_g#H$A0bJmXmsK7cGu0+0xZ#bTLrP%FFrqifbRQ%_`-(
z8OUdK%F*5O#S0eMxec5xi}zYcY%y51dBUo#71>ue#q^eDEp|(6oVO-KR)Np+mYHSl
zn-^?Lxi16=CER><?C+B^+x;Z1<uZk(CD@iZoakbXid$cPX3{jRliyYaHhx&Obwk$1
zx9QKWT7>6|tNWkctx`B$(w~2if}S_W=9;;ouh&GSU#**y`dTMC{ifCFYwMLdTGGB+
ztrkCc(lUO*BDcE%8qI=={)?3jtWP}+Z4&mswPLf~yBZhwgrXS&QjgTV{RQg&dtRPs
zv0!)jt$@Ukog7w0g*~p9M34SgJlwJ7ZObcd|1%1o_+(P#uULM(Ah|N=W@N5F!cV!z
zz<nR~NHIKWl@HcC`@_C`TCswTmr40DlUY@&Q$3tie>h06CyL~jHS?Mt^)4~KSl0Q0
z>;9Ys&-~-r&vfrh3IFL_?qan%xUuvh!;y~~nx}Uke}DJ%%YAEtt=M_jU9>E-o1odg
zzKPS#(qB<~=B*U&YctcMrfHp0Fy3Ht=~USC&Ff#Yc|SUz%k(NzYv<EnZ_Nv|Cq$iO
zbw1k2ceJtZ;M529+3E)~Jt`%b8mv!lei<FU@Nl2!?%=nR_GWbDsTR#&WbYcbvC;fb
zOWuR$-+xD6++F&@O=HonqqDwpv`*D5N_l#@@ybzFN7re`3Z^)=A9*3Pc+;K2D9aDb
z$~PW|-8mev;KhS86%WOaIo=nJN^P$3UfA9A*r(61eZs69k?-5H>My@r7Hea?b#35*
z7Xb%e>|StK&+0`GvrO`|?sKfmt{SGUG<>DVIw>K4OUo(wx#cEb<{V_YFf~9nzAQ>+
z%IE9Mey={MH`YjreEhiMT7Hk?^eHyt>yyNT*G^)Z`H^}3m5(~IEFH#iM;}~y;h-6?
zVIG%oSJT19x6Av3wph-6akcZD1+QfDQjdokf!9`c-#%8U7iD~6qm9MfC#KVK7VJ&q
zw_fLynszJk!L5be&QoPdy;4qh9tynJ)?oa2eqa96?H7-JdgbzX#VW~Qxx<Zux8Gh8
znvmswXFIpS$;2~%b3<S4S6-9-F@IWAvPjgQYhh~Q`ClAYU%XILu<niRM|tskJGpBU
zGG<JzO-d40yLMVX!l}0}xlw<8_pY^5BX3AtcWJp~#oq2AuGTgCZ<O&J_UPGr4;1H|
z=HZT8`7c`LEx*~B&3W;qzZ9P3FALSW9-8&aw<Y{#!L7FkuiiiDt;#k#m0NnndgiaC
z3-$>eQ%p|aUsg1KzhwE;-s}a(_^#Y&l<IEwko#)E;`Uqj_Jb{|s^>0zsypJ9sVXsP
z*_P}6mL8dP(<d8<)jmE@lBn$yQ?WL#u)&r)Qbph8koY?8*2)Fy{9hg&+>`mV_jpp5
z6}$H0n+GzE*>pDA`8c0EcxV5)!kAk^Hr_m!lBDeeTxWGKC>^_&KVwN}EdPR#)BBsv
zwipOLxZ8O)z5kA*!m?)0zZ)G^br%~lcG;ia*%qj}=H2|50QTzz;wzM<@mQvCC^-vR
zPC7Bsg=^jOJM!yGj;UTh#TN6DH|#Fsf}{%*4Go^3+r=Hgp8E8Ac=}%Ese5xApO#b>
zezBg>`I8|atm!D%y-Z0*rE@#fcW5nSaZq%+ad4i)#MhEj*-JL-9NbhGURKXxv+UF_
znWsy`-S?Ju%s3Rvcl6(kSKk(IyS{>B?sexCO}(1)lBe&yB&c#LH<V*`<3%1u<|CR6
zoWYSs>(*)U^<+H#6y&~}*>_K03A@`Csn)YAy?tMLN6XATb8W?n>$X`XY%BJNFX9r<
z7h<andA>INkEzu+hCYiIw{k1?EGb_5|J11m6>9EtS@~HCAKcsY>L2s+Gm~$Gm&AE4
zy2Slu=U<B*|0m8^#qh9EvftV0_}=^?l~aE^Iwk#u{>K$_R&hzlaY@L<9NBWNQ`c_V
z?$gsBynEx~<g;w@1z9c~IWC=fJ55Z!1lMRN9iOtQZ?1pik@rq-^-c(B=XUC7CKmea
zpVFzSwn}xPdB6tI2pf@zKkIg1?Q<%>)tngpHt(MKQ!TEoOugx`TQ{$c-}>fz%c7o5
z)fYvj7Vqs&&$%Ja6C;xv>v!{os&L06m-j4}!}raPY`kRg`{eGvJ%2;*>YSI+yd-RE
z={&PrRP)$Ue*eBhr=Q=y*suP}Aa?3R3AV0gR{oNRC%+kEq~}P;L=+|zJScc@l#%<1
zQ{$&uWplVI7YnaWwk|sNaEYFvm9BNO!0)Bc&TVq>yAn~l;F0uSwOu@F8s~fMG6GpL
zR$dHxEIT{u%EN>I>;!krJ0Qf?xS;%WY5uib<2cn-!M_(9WE8z_6SGoye(z@B7MZOR
zo-41~>LKv-u*ELUZH9lV3I#N}h36J5680-O)amzQhLQTRGo7cK7jygZ&6Ku!=%PO7
zMN5m3zoKPE=9cA)PDNc<v*nx}Cx^1sQJ2u!E3#4#&6#1KE_KVrc(%vA)jl(~Z14`%
zzWj8_b9MEveVoOM7N+WyL@}-8Sz%;wifb~1GIKAFg;PP&d(~%$C!dqrQu9H<`@in{
zCk!D*0iIkzmS+p+yxp-jeAVNst$R<+%v)6GYohmuu{pP{VB-oY4v|w8A}cx@J?sKD
z=^5&;yFP1&Zm8|NR6V9v7iYPXrF-6o8Tg!LwJ6?LuEXab#hxj%<RjCprMz32E#exO
z4`>L-mMUpx%gd}N-~T)Pi}(+rdnz6}3wo7gL_RfG@IA2TOx@Fv{$kQm_dX$B*33t>
z0s9JGMf|LvYV*Rr>EO07(P!`P=AC77`Q)_d?m<=yEtevJla4CvoqHx|#LfM4q+L~M
zrp1KPn~U%5m0Q5iAjO`rhaqjvE9Se03IvWj&S_yjYR7-nuHR~1UQoK&ybSNQBdyk)
z*TptySYH>PcJ27u`5b1({9X&zX-ugEA?vrX9e+fAs&10kTcvw3QfaeG+ut8TGgXbU
z0|Oz~J1A{-O#|&d*EPEJF6XH!r>oL_?t8x;mFnO7?4%c?w9-WFrtN|=trs6iO_^L2
z=yanZQInhdse`*<SMSvy^VoO}u_Y$YnAff5#_h(<_u7wH^^(jXIe`KV>nXBMqN`Rq
zW)#(~blhNgW75;tzfTwIH~GaZiQi`UE9p^;wOUry;-hZej@^#l+|~z@tn5GLKX90)
zqT;tR_qonmV||O&laK3i1naHY;&3qOz@yLdGNpw}ZoA1&+$Zp|dui9pyYliYT6@%&
z_`5A?YCgB<@ItQN=Pw1^n|tlyg2M-Q#ML$?KHTXOnYbr;qwDPx|Gw59lGykn_0g;u
zrH%=L3-}ot)C;~naFc6DP-yBalaWoF%74YlGeU#Y+ijDVSoHU{&*i_8t_8g*o!Ag~
zLq?`a)JvpI(>y?>`Nj#a%n0FQ^UrDVg<MjSNNVU1?UFD^deHIqkU0mFD%-Rh>o`>x
zOI;IBi_lr4|DgHhzMj0r@9r0{3+}$sdB@|FK-dgT=gG&lpGWc*Cor!G_dU&gVzEQd
z)5`Oj!SfDW^;sJ1sM1=LwbW*cO+vjq_tVCJtKB!|Pj=>r3*0`LYm=Du$tJyXjCbQ?
z*6qq!c`>kL-!68}B|AmC3)q%So%YUQmiUsL|D)D~s8%KXvp9Vxea;c4O3}G~il(pU
zzTUGcLS^Q>@T}b>c28EUH@JAvg<W{B!N%fmTQ)dtyYExur(8O7QlfUwnTq?V%NS#l
z7v_dMcQRn^V(jA+vyylb_#%eG#o1w=``5-|*T)C#=CWJdc%Cr-g2d-{N@*wdrEu5k
z2}eYRibb6`^l8sWVZAvVR+jyn1J4^BFbbQYyVZ5#3FnC?l8x6I%H6%TT=9$FhFdD}
z$_ZDd-CES~WFZ($OrN;vssC2x@6$d?TwZo-?iTA3DWj`%4n^j?G@hM!^#0rl8->5J
zxvpQAc_;n4caM(YrYrkIK6Bk@OWglZeEo^1<%dNZsxmL>e%yHUc-ZeOoh9PAFDHdJ
z_6cTYc?Sk%d=AL?{Ho2>;_}(Pjwz1j-Jqk<wU1^lws`p=eEHP(%Z%mR9&^8u4G)-j
z@BW!a-wh9T@z?RXy>43A&9~#hfg_D+@d;A{ckFVJE9Qu3cHP3mZO6~a(JZ`JxrZ%R
zlrgI0-p%YS`aR2L<j*Ppo@FrO`8n(GjAcjr7O_9Coi)KJx#vjleHVsnH6PEtQPbX$
z*1Y+cO@_~b(+U@EeLlx^lE=N*^2d|r0Ow7`)2F7Lp8VePd|=@P?FN~^46RQsZVUgO
zbGlTyBYcI^Gv@=2BJAH}5;N{*YOfb~5%_F&kiyKwFc&keJ8q@7=XjWf?C3e>cY5x}
z@;>A0|Gp1*$6LvKiQ>F9Q`o0u@uu96ZM|J>(f-_PjF&FVHQ5{+_u}g9zL{<1XD6{S
zn8v=CdFXg#`0UT@_XHb^1<p0>Q2$)zcxTPgkQHomBn#5`=Q-{O`r8=M<>JRyb$s%+
zWuKbm7?KWN`TSQdB3(<W`rE7Q8C?0V?ml;6zMYZqJ36w`j?KYV{Q0+hy9|l->F=)l
zTuDlj`|&k;o>umsnZFtOm_dbR!{LvitjBa%Uz$tb<+`EBv7my{D>b@|;V5?igMdE6
z$5|@{SlC2*ZPhPD@4xBs_`wOr<sY9<$V?8o`ILRz#q@#&9M9?trU+a3ebg#?J<B!0
z_Wd&X3`U!R4+psUBfiOP6`OV@H175O)87`(dFwJarqnFNZL;p+;&Uzq4w`n}>l*{6
zt!u9?+_Z21of!vRowV)sTgs=cP@FiUX#GvS{59(e-p~DTuGHL*Ju@~sJnt^wTg}PW
zf_ysG88506>{@*C`EG5wKd~l_H&*@buu-u~IbN`{u(Y-Ky_Lu-@rTV%xa%&TkT&z$
zaH;b|m3FT;kIvTJSKbBf_I|sD&vENMx4j!|lEl8fXPv`8E$!X!4ZmkCD&45)X_|15
zW98)SKW?y9A2XTyYTwc8V)2!|@Bc;g#C94*_vGHYzjf)2tn&OF^Jb*3f1Rm0{jJ0O
zy+<5Z={|Yyo3Z=W`>jj&ZEWA&#5K2W#;?CpdmcQhxD>GW!;S!(Gu~{A`<BfWmE71Z
zx69z8bg}B!S-%;Jy_R(zJb7$J-qDHEwkDr@@T<x|Owm@I^FZg?bPwTP-o#g?(es}q
zd|SJxBGPKkPW$y!D&7}ZY!z>7)}4IiaMGWW<o2&!M_xac?ESrYQDBPqhHH9WY8P1D
z*S1}ZaFRZFvrDx`PHN7US0+53z56o1oYmut^35<=A9gsjvG{?;i^&W8IW#^N#uV3?
zzx{RkHb-&Y{u>U{3q9{mHy7{nVv6Zb{WJU5SA_$U6Zp<QuGuFoyz+2R{(4nrg=bGz
zaURwFDmB$pK8M9ke!t)CAOF=m&cwer@7rpXCtZ}OGp(XEaP_2011|noYF4kGT>9Bx
zqx9J^F4bU7!dAv>Y*Eq$xd%3`GcI7$y1w_3gR%a>r9~{y_y0AY%Bt~^`G!vB%5$vp
z4C?|{vAL}*cvJad%Y@nYoKA;)V0nAwt@#-bR^KJwhZQB8rdLkhaE0fp&4d$r-}RQ2
z3%!t;!}#{pvBrJzr*}NgG&0=uWm8n#8&<20>t`(q(w#Qzkbm05Kl{{z4EWdaWo!5*
z>`ZR_+;wbMw&<mmtgqt=>O|G`f3Wd?$?{p^^!n}rm&L-|+t$z8^}2HFtO%pRiwTnp
zMZYfHxX_7TBmM&GKl7<u+2#eF@-8{BvpC@e>ucxf-(@Sx6JpbyXOz1zTx%@c&eZdF
z^#Ow{Ls4}<bul0QnQQK>zP#$_EP2(WY|HB!4Tt<b7-Y`5^ZoVKD>FT=Rz_a)$x$_l
zciMBP!o=^<F1?$p7G?!R?F{+Zntb8VRL;J-Y)0R+B}-O!hIe<y)TiG5tj<)=DH@mj
zAi`6@j{WG<0E>@>Ee>mUvHSGi<|<iKr|(*{fmyOQ_}HXlA8QZIP|(jl%kj?Js`)+N
zp6Y|v1$Ivs7?iA-cpxJvnNw@IT)M4=x0P-Ym&GboyZ!#Zn$_H`Pqv)AmzQ;YqQmO#
zJL=_c^{bzHRpu0a<=gI$M?zY2_j4Osu(MpbAhLMV6ye-QdF7xT?cQIeztF!Jue9G~
zp7Y#2J{NjxS0*heZCkf++pGiM!u*^#_FwV&<?vT?`>mT<>8t0iE!%eM=2t1Htv#LB
zMEC90uBi@>iGSI!NIUn+1vlRHU-|^sPXCsfF~99vrpDAZqfCv-Z9<t#U8KBJF0x*J
z%yEb<Q8@9YqP5kbgtg0yJi6Vw-MYipyL?axKYglS&}x@wNvTDlg<{I~DQ+A~_&D6p
z9q`mNxp`pI6ItoG*A-%{{2yP9GPr(u-Tn(dJDMkXDYr<kzml<Fs^Lm2L!N0D_biHA
zxJKYB*VAJ``y9V=^{4-=NpqL&vrSvwmtMB=^WAy>pE&HbnD)kH+PRfScb7%IIn(dg
zJooFr47s^t*6(LHx=BgRT-YhuC1z~M$)L_QPog17p^K5>*s1B=`xj<J?$jwrnOv@r
z$r)l0;Ha*zHaV;=er^QEMaCnSavQkPEu{q#6I#|jO1Qeaa6j)J1BSy6Jdw$P#oR_N
z4dt~LlI-U$`K?s)=!r?0;I-oG27M=+*H05S+feiU$5%)9=+mDh-J<jq7#(C73K@<Z
z?EMsUdD%V7&RP-WYt`y=7$47H%(CD|sPSaK5{9Gl4hvNqj&OVU92AkcQ?NSx#GUM=
z+XVJ0h+kg)S?YRp*2U5}29uhZIyP2L5Ie}ZQuoZB&{y4_XJe;3&B_!E?cQVT5Hwqz
z-9tv9@B{PLqXF?erLCbi3nfkoHMNQdu4izl5K)?X<%h0XtcLE&DB-p0;az7A%~@|c
zG2OP6zm<El*xWx4W#&w9emGygC{$anrkwA^tSPKN?Wdcr(>(KXe*eq=cSIHTX|sR$
zd*O=n&RL?@wmdGoC8ED?Iw!aGnZFxu&CE*g`X}<U)G@qkx{ej^*7chAGQ%ByE?M#U
z*3z=+x1zKB6hme`nqM|$mVLj@z5J}7P3w!U2<D}vd&_6dVbxB^3Rn5H&f|-J-wL(6
z4uUVO1z+w~@+);+^8Qo(IjtY^IeTx0{jL_j?C{yuKW5984@=gCh)wPN_vp;yMVpuE
zOzAuoD0b@86AkXZO%JR*UeDO`=_tE+OOIHOnAxtqtn*(gEOwMGNcg&pVadf!D^u@o
z&{?{W$9rAxmC{1~4bvwDoN8PwVLqkBNXF8tX|b}!yVI!xc|R<^8cA55YFeygVdeCa
zO*CMh&n=}Z`=>q*nY_H`#r6KLr^GL7PpCNhr;e%1%Bv~4H1mFZ4oe*C@ki&+%XW(W
zc71+!zRIhAzt+2D8uUy(zUOihcqdT!o23aCF6?-aaN*F3qi);_+8O1Iv`kBw-OUU3
z^nd0)_BSc(eAE+GrGINqFilNoe9f(KK5Ex%$ySvdnZic7O|4Uz1rIOYWP2m*#PTAW
z?lsq%EE2BX-dDHgY2LzX`|X4Gs>PfY7rdeWVdYi+0+H2PZ(sUM<vp{X=_1dzP3Qiu
zx+gzna*?gf$^Xl*Ro`QFJ^nIv)dRCI{q7d!OMzRgrnNt5_!_&>e{SkKo&5Dr-|i2;
zq<!U=Tv2S<wC$Vs+<pA;iTW+si|^*tm!8|ZD|u3Z%TDD(H!Bw(n<DGN67l|ARr$5u
z+vA_FWa8Z#ude0pru|}7s7JR>etM<#>D!Ul<i2v<d0KRB`{q4!pT6C?C-U04O=Z(e
z)pTNt)sBbl)$#8rY1Mn*QS|fQH?`+8@9hw_*y=m|eg5?G`MV6i&ezBndRneibaCz?
zYuECwcOo}6i@cvqJCVw==p48IQ>J%aKKDwm?^m9AbE<Nubh|t6?MY5k{{P&Oc=?+D
zBQ2xK@RU#UxZ~R9=>CfSl6~6bsonOg=faGIwv^vJ{e#J5uN3GwJ4W@6?cH(HQn%Ko
zZp-ty({^O7@w~Ppx{T54^}N@o#tCgx$#&jkR`yUeK$>YQi)MB#(<HGu;e{O5EwYFB
z5|_VLb(H>g`dHp>oA-s!&m4Jaav@`ZVXR7^w9r<?K<V9Y+Z=QqqnNg^q(8oQUbu31
zZM`6e;*UFr+7cWL%vk0b^GDRQB>!bU$aj$Mpj|hEh_wzM2Lx{V(z8|l{OvzXVv%*n
zb{UJ!+xV$RHGS)|9@q3WR>?PSuIc1)44b#{%bIKRHvZ@dP1mtbzAMRnKBDs2rOlyp
zPb@MHo4fH&&(|~^i{yop+H2Pc=R`^u@!fbO!EPnIa(YB|$JVxp>>qo}{=`=2`btM%
zHo90WzRdhcgmaJSoiOWntM(oVcg`_A5(XyTI!J#EnE6{v@x-G-vHm5(<@MIfC+wYi
za8KC`)jo#azVfK!|96`&3G`G<l7Gz|k+RXE`{-f=v8FYm{j)l~l@;!r`%M>{eTe5_
z=X~w%Q`v#Kyi51!ePXvrD75T89T{`(QIS}<UT46Y+4iY=9S0st7nfwDue@=hzAHwg
zab}tZb7Y;`r%5MYP0Mx_&wTIn_JxGp;v$Vkx9~NO*F_xn^nAA^`0d)Peru=T<BpZ7
zOCNvyvi$Y5ThYC3OKkME?!Iy_sCed~ecICsr=Lzc{@rn5&+Zu0-A>~7S?_MK-g82;
zqWd0Xg^J3$rrL+OnS1BxAK2z<A}H3(=c{+-+oxR<)bfgiCf0C2iLRNy<Nfsqzdb$e
z;|h|*Wh7$5mY-r%XG?tim8Xf#G1}EZr1;*wHCFF;1h&X_6iF3HePUxwm-|t{BY)|6
zf#<jH$0k}#SlyQ%6*+t3vXH*{U1nA1<>Sx)=oYBGu!>)5jp_4)R$bbTUh}v7FfIA+
z|Js#-?^cV$@4Zgi2PHYq{rD%e;<`;o*37k87m_X{8F2ak5OPz@5;+yQDPBOT-|)@$
zPMwF6>;JzJwVGl*#aie5r%luA7hhOWHizr^$}^7_UrSkA8zuAhVB4XVsEg-6#ZR>{
z(P!NFd!Ej_KmmvDb9cWo&5fMPu}9)Z5~~5<ZH1l-i3&5Fi;Uj?ZLkbmu$jBma?yUh
zn%fWG-LeR36}aA@es)i1PP?m+O9!(>?Azmk`<NC?30a#`tMHmRQvO7h#hP#rri3YP
z<JX55rO#b#$kiIPTS{WiUxt`%tNFhf=XCx3-4@rxUKV3kU2yLAgT%YL_@*%5c~-4x
zb&zHCd6x!1mPIlxTKmQITcm#2`Muc_eEg~gzsvns6=BzooZ7T(o7v<zj<)LW&pUSr
zA3J&L>zBqpsi%w7wN=uSW=~L^)%oSun<;y=7go#glo(v#W-65XX;WFJzITqgLB&)t
zCeyqp*B(VN@84J?H*d%0&^7&Q7{r<y?`5@}>U9mjw<ziUgV%F~kL)X56w|%>!CrxA
zmTax*>9Pth*IuhmRGAY|xlC+<h!+1&&ouq$2kSa_XW6~rX9`u<lA8K~dGcN>H%0D_
zVqsN}2OI{a2d*3px?8<($L-sN{%0@M>sl>;8$R;`Gyhhu$V;MKKlm@GeKHiA+I{`w
z$-N(fqpBXpznC+{^b(V>;ck-+eGRU{!4nRy^ERFP<8Hr7sgh>Qj*VI+A7eALO^@Ce
zsrQPme_3|+Zos{B*HRz$ZT=+RRs1n9-|*_bJ7L-v&ipDmUj$m9^5oi)rJGt4AFk78
zD~c@8e0s8}sNF&G&z=g?-siLS6fM=hb1PQYaEDpVt2vq7`*SbM3R)r};&Cu=SCjJR
zSD)rj$UiH#IJze9qjEc=+`a-Cg$2*UHaz)#{q>?JXUy-KX>~@`8M2-Zua3AJ^Xx$d
zOPgcm9qo!+i<!#|XC2(n&NOB3UDxf$T0czTo?UX}+*aAPrD7d&PgK4}Z8*1DVtta^
z`FJV!xpGpcQl3Yc?VHLT5Y!=>z`HQ&d)*`@t<H@hJ(@eLmS-Pw%v>m|yJfmqmw>{9
z?Iru~mz+6QT6O<rN&eQj9g|)ZJSd4;U)u8d$%D}3f7e2f*_B0<TAhBVadG(?!_?ZB
zm-^o&9<2?ITkiL#&DVSK?%J8F_B~@w+I=u{b*i<&pKVd<4K4?&7GIVU7HKz3m+z?;
zwu%jVAZm6z`=Iule=E#S`7NqkEb7sfch<tHRIpG-mQ~Dav9Qi!z9^%2r(erzA9r+}
zdCqV33*nNo)*MN$#G@y@k1r|-Px{0B`tXXqb-jn`bS^)7uu*U03at!}_GZ7+bEo~T
zYOK0vv*>B#<eFy{KmKnlPkTM-w#DYX=hIJeGqNoIUcWKeNyp=s>CRQT+wcCr8DAbb
zbLY|r3IZ&R3LGnKivKg3y{QV&T(_F7S;`<g!dH4lz4QO-xi+U+7UW2;?$|8LJd^ck
z($qPoi&E4WRfQvNoZTA}v1HYW$!r}uXR?G$XO%oN4LrjX5iVWo{PMxlh!w_L`S+{&
zyH?EHn^nCp+(_aXziMEl#5o_|o!WEvJ(&M!L8bEC9XI#9Y|aQaZIoJ8x^CG@;p|PB
z8P7C1gPDBa)W=6WvzZhA%rSY<M5UvTXK43Uy=QnA^uOfUy6qb@62pamg*MeKzdp0>
z$ho6Np}bi?)|D?^KJlW%*%_UC)D|!<Y&&YYtY+p13$cLL*I%#IoikU&|7Uge4g2og
z$i2yzjNLC}TFq_#*_3%;CR@pb7=16V`upjvVY35M-&hqdWBhK^tZ*trePZIGa-9N(
zwJWrY=1tAnko7=S<9F=xzLl?3JO6%iS@NzV>zD~cDQoe9a-OVb$8K>?^i@>a`9>~8
zx_-kSyQjY$ziqEMFl%Y|=b%;ZCd@mral_5;W*ht4{v>RbK0DK4&$C`-Jx1Zu?NgtI
z8?EzLVr*Y=mV5I?u9X*~gJ)aTFPOe~Rn)YTJKC*3yLIkSG0c2*bk)m<ClZr9mSsen
ztekWHHiN_dYJF~#Jjce&dB;kYw8;m3Sy^~q=YjEhFRArwnx4m(#|53{le|{IxcySq
z_k-Uy9#?D*cI;kRq$Ygo-t`;jXKRFadzpNm{;Zhqa*W8*>GtLdQa;{aC+dg1+`ao!
z*>}cX`9(jHlp}vmUtY6l$;Wr61j@b6%$lNbIBVsfrA6Nl?%iY+pK>Ym+Zo;FC{4ZB
zYnIrs`F*{qRUiB=Y-jY-+^L&0?>jp7owM(`|1wMRT=R_iGVYp{?#tbHbDa|sm7lwp
zet$82>stE@@0$7YZ#W)j3%K-Dh4-Rm!E@%c@OIS+j(0bdnL3sknVQ~uUlCJcvo!fX
z-z8(`Su0o)IemC@FS2MS9=QDI_+PU<dp0=FGXH<>wdbp<s^ph0{Qn%(D@zw;bA3Ko
z+x>bO=hS6OY@a;pvf_Jlx6v{>?ahH6*{43TGgdPfi@v`dc52d^z{M*am$x$tOaa55
zZ3`M=gQ}LBFgKX7s+P=3s^h+p>cUsn*QK+-E@P|jQl}|b76k2yZ%g^>d{Dx8{hCvs
z`L85gIOeF%H_JjJVakbc^+mgQuBI-w5)|fpn!5SPViVs|C9X385pBmd1x3DC@jAz1
z%A|$8C4639QJyjl8A`iD{P!O}5~yvV&)u_BN2z*iW%B!{-xZY)pPFLrRXt73>fYAr
z?)eL2lU=S~I<)lbnjD9++ix$`UrBf`En!>G%PZ3R<bZ+H1&_MSMOvBr*Y9!IBm4K>
zDW==g6JkP*w+INWJhwY^_C9uou){mzW~*HG-WcH-cUDv6MUqOky@vS;%~bZAe{Xrn
zt$DxIJTHiO|LPFm3wzA&U5`BR^ZTba0mkVjR#M0GU(LJEt+nM7>z$hwoD2Mv-7be5
z;>iChmi4`MZ||&U@~`5qx$}y-CfN49UaNAzFw*?-r@mLSSsA2GE>254xp(nQnX^wa
zmc0&O+-)_nLOlBOl#1ysj;e*y`yGz9m+v$&W;Q*&dFKkPx#6qc2BlsRIr}tYosGwK
z?bk)&*PgC(zFc**r6eP&ec_VgqTecFF?=FP2ZGBkOp#rpaw^ty%K6hzJRkZ#)I7}g
zt#^_}xoY6MC!6k^3aYT%)uQb`?ULv`bMAW1dd@mV`z?0Yk1f<>SNwOtEB9p&_ob#C
zmn6&nG^zN%ds7}Rb(k-e)BeGco|!ZBg+D*F+Lim@|Lhrt!k<$g-P0~kO`7YoX7Z`p
zJ5x^w9cp~8*xPj=_xLO8UiN!FSJot(Y<pzs=3A6{)gWB$nb*%+v9MD*pFSLsnOQJL
zE$qaRUo#?PC;J3Q>)%-u@!`m=84;4g+Q~_&K0V!Cy%T!1XJ61*XnEZ&^kbn3*NmJc
z86sC!arUa2c;!ryFj3S{&`@}>*n!1F<oRzq_FlCRqlfOtcK(SAw7hiVK%-Uux<+As
z@lZ)o(WP@EQc^=gUOlpt|F5xf`jX%SO%rETB__xCSPGRY&%IuGezBYAH~w1(0@&N^
z9_S>@QQd3PlUnLCXBGGB<fKv`o1kDpj=urxyC+=`;Ci07roZ#6qp?uJOSbhTTUXzT
zG1{MhS(b6vCS#5UsRf=qqDe0VoI7u<y6*9Ok=+J6wJ`R)HG7WzWl-OEMd5O(7{lD-
zY(>tE7F<UH75bPyUXpfJxjkK8RpIZuEq*_C`LE$UbYt#<s^beMtl#*BvFbqfA7PvC
zmzcxe+eh4*A~Qu!qSroP(*0Q;*DpO1X^~Qpu9oBs4$N3@q4BWqoc1|w)&=rYY}Dni
z+_TSLdj7WQ4zGzXxC&OQZ4r97W6G*Uo0jHedQD0AY3jxz@I7RSw(0_Y8QTX_k7b?M
z@-eQ-<VfSh{%Ye;|7EXEhVXJP)LvK+?)0#38K2xQE!zpqL7gr8V~T#fJn<{ooyV#m
z>v4r#;{C!%jmCm-riXpqO*0k<ag?hHyqmhK?Z$kgH>JzcR0ZChIpL*nXR3pRyvR8r
zzLnwI6Sje&+1fjYZXUYXbhGJV&+Zw8;>Pci7VTxq=ss~P>Dbzt+x{?g&n(;cFWzUx
zREr(gt?D0!-m^GiF8nfyDb{C!Y2mtA7UIIslODzFcxEn~p?hCk_(jsA7}LUQMzw~8
zdTQs69@#m=LVWedqc)M9&DFjWx;qOE4fVnvKUKJYrhhBf`4ikHgv%TQoC2;cXaK<#
zb{t+D6PmTk8>}0y?-OT8)ZY89fG2d{GrPaR0WbHkKL|9vb|C5UnWWOaGyZ4YTm3W7
z@V@S|-LsypJK=pcd+V&YGuhooza&LPvk019zFoCBvCMG2?z62&_k>-(kt7xE6W;Aw
zsu$^)RwK#mb5bZpz1yTp;PZk7dn0sa=9#oDG89{HbE3!XZD+1g%jwd)+MSo|X1o{s
zd^c%c^pU^TpJgLGdv$ds&z)PF<#RlA>z-9=afd6#-6u#HuHlT<5#w3t7d16A#Zdmy
z|5p~5Km2BHI3Uv>w|?@83hg4TGv%_WO0!loHps1bqHylVcd7q7f9G9jndKoa{PI$p
z;9m#d3+#Pz1z#NIdOUo==*}y!ZN*iSz#BS}_gp3k^D$`^EHYbfTkJ6H(T&vD;O6eB
zF1}GGnJ4Xd`l<TlZ!d+N+me;|KZ*YSZRayxC3x>5K~46fHayq1|M*^98q~7rXvp4(
zZTe}?S~cD$#l4>D`E8}qtdkdg`qGL_E}ve^#;>|yrp+pS1~+3%kg(Fk;!QX9P7ha|
zziC31t7pmXNiWKjKBV7BJ(1%7bkUQY@p>yGzkPidbTCK5>ZqhylAUkdv|yj*_8Xc{
ztP?VL*687D;JnyDx_#TTUu$>9MV6iKeq|gwf9WrcKUX`g9M0U<Ie*h(Ny@G}G3{;h
z_H)JY<YZV!n>9>&R;GIPEsJnTN?>^2ozDmFX3EUIKmE?e=pE4;KyVZ1C66=>xi3!w
zZIAM8wtCz;`RSJJ`5Ub6Po0)3kU0NBxU}1mV;wgn1f`Xel3m_#N##s?EUmn#r>*zm
z216+`mRY?YH=M4xA;B;Gcp+3UYwO<ijxD>R-d)e>3SJbv=rYUlhND_XR&(}Fl2Q$Q
zw82eSs)6?aTf%HshN8u<7!w#IzJ_U?S3PhrdWXMcx`5<*Q#F30>l2T~YyLZR;>Olj
zcKZV!ZCyQWl9lg^REH_$`(|CpbowjJz~^=(uq3TPTsq5*nISh}1+%lfM3qBdQ{Pra
zuYR@u?9CdlBSKoIuG=w%Mb-G$(L<bTCdNjWnXk@aYL%I@HdpKli)g|NhG^3nJ&wUq
z2X3u>ow=D!_@{mVN9xpvpYP95-Lq&<*z!q<dGg{HOs#|y!Z}w+sRv3}9SY!T3)`@F
z-9ABKUe<>i9QLoiDmI2ieU@{X-msEC;lBIPAKq)HtE}^$vU>7T5mAQ=g{{*~H-7vm
zeCYifn-|iH6O&D@tg61Aclp!H>#JtJSv_}WGG9+eui!>QrLv{3Z>-ssy>+jC`pzEC
zsWJ1mX9uV<{CZGycIzxr4bHs})~dDCePv{r(h&FI5_fl|%-;tu*!BMkTxAhyUM#-n
zi_^>IOUzT1f7QI*{mtX8*VWLMmS)9Oxp!SRNKKsPESb@6bfIqNlU3CktZyqXCMA1l
z&pG^Q&kVhF-j|zi)oV2!___JRvIol^T$-A9YM-H2{M~?kzxD4={{VsFEL(JE7X{u`
z{iFIv^^WQeT`=@5Ros#(zUI-7=kcuDLGWT{`-d&5=Jf^ZdbLY!til&A{dH>d`gFz(
zZPpIknzawGZD7n{kY0OE(xm=?W)_1~1K$GX7YtTw&vzQtFIdI5fceSncaHpK46_+8
zIIOAh1Emw@4S|Q*3T}6FCv&i$Xuj$akUDAV6P8()7c@1Rq;!1V1wSy_FRIF}%5F5h
z{jq?@S(mCs#hQmK*l%xoQ!6{cIM8=$ZSAk!MRsvPv)5eq3tj(5G-RXw$w^$nvFlz>
zYL5*o)19+E^81eeZ=-HIJeL-+m}4g~<%B}mzFF&L&hWcBb&@!r^Rcbh{(jkJ#v1za
z2$RI?Q~M-VRhc<*tNq+9JoCYeg$0urvhmm^XfUP5b(ha=UH(^YsnjLQy8Wz-^5u-q
zTp`DIW}R*;lYP#+sxv&U`;x?J`&lifPVq5KQ~0GOUcL2t^Q!z;xgoqayabx1&YXIh
z(4lMUbX|j4e&L$N-%9J7r55D#v`$vlRoBr@>TUS;`&O|bpOIYaHKw1;QYTNUJk3yZ
zhm<gJQYSBZ^tlz8Oj<lqruRvPio3F=a_WKdCBmgow?>ubX-=sYnEKCEzTwT%8H_q-
zO%^WN`fuG|^{U>F*Ul|2Yk%}rL3PnCk=B-D9SJA(CQSdg)Iwj!$!PT}$E>!l;*dFu
zAMFWMU-VccR%O<{UISH?ORlf<Tz0;57n&WG^unfJXu+$ub^J>2kFX^wrtwx?6wSZ8
zTzXa7Z=pGpCTG7{!{>1FpxXK83g@2(%sa1{oG2&o=cw5ncFl0HlF3VhOPBR@XeuY2
z>X>1qw5+G2#-w7c%=;OUd;3k))FwObjOBCUZ5K9pcgt{|S@lyj&b=Y;uR2$44;G#O
z!PCR#_a%kkq79V-Micj~<;^LzIsbEu)TS#FN@RSF&B;D|!KOnd>+yvtKaHk-uY1Mw
zq$cF4uFQI8llQ8BbpP<~S^0A|-|a(p58ahzQRTSNy#CYT%3NMqul&is>z2Klt@%4`
zk<2TNRhsG>-Q@4hXl9FE8YOd(H{w?Nt7+c?_NDE5|3KGiQq|H#lZU4AMJ=9-H{6@K
z<e_DSd_d@>JRyeQt4r9nuD<qSP1`*4td!md6SkRVFP;(H&9b1&&0Wei<8xNmi7!S8
z@tK9bI_o-*nr_^oCU*VEqf^?f6<-;bM!Qb#j9<FcbmNw12dmOg7#kYyP-}nkir>-r
zhQYzUAmzOdQ3=A_I*ebV;xoIRM(R(Im?AMnB*b<_6@$#;tcKa&tR7DnS~TqgLwM5>
zzJm2?SN@&NKF=Qd?&@ofYb)MN5;-8be*Ni4gOH$A>Pr>pc{$rm{hrwMxH#@fSjESn
zU|!AW50#m_IMT}Q?LJb$H&5gq2s+)l$-jqXH;eWn=NB34R>t%$pB8m6{NsV?r=puD
z3R{NjicS|imU&%dTL`!_7`p1R$+|7qEamTNrE}Le_q#St-OjrI`|h%7bC&KgdEpy(
z?4;iAU=#i>E2--byq3#;nRaZ0P=eO}!@H_NXS03$d}FP_riM9_)bcuQvQDq#TOm_l
zxj0aGv6|a@v*ny_Q+wLn6Ie`MS{^=Unw56tM5wYXU%lOfGIQ1a!JC3NU1s-dwBur6
zN;&)|B0#@;`>Xyq``H&)>6`gV?6@m*B!I6i$Y4s-LT(<}gcl6WGA2`PLl~yq$;kdN
zt67b0u5_I5fu~|qL_knA^aT6!+U$o30h7JIy=K|?nt`P{{$$|Xr+22V;#b^ftoEh)
ziiqj&MGpmH_sF#0Fjqft;Ztfxo#)d9TNVdPD=#zZ^iP$Zd&<lzHm7N&$>QLbuIiRI
zwkh#WUYg~0xnyp0x_Y^MxXbf3UqUK_pXyHC-czlrJLlQOC9W~kS}yhaEtR&_*GgYG
zL-od)RI{n3CgM5&)=kzw_d3iHRE0-x&|h}nHe*_uOwPmnhuL=%*SH5rotw8bgpXZU
zGu82QmCcm)OJ`nIYcpJWC97k~-@~}=LD%yJf9>j+gHoB&`niuYB;9vg+*%rU-fgDE
zDds~7OcA<8C+DQ>iC^s8!vunV7!Ej=P1sw=dV{fqK~`b8u<4JvhVOa`(|fFw@1#yk
zO)F$*FOtcScKXM_x|*+&|Jc8T_hGwJFK>J+(v%&t^5V7yZjQZ8Qmm$2St4`QSWG#r
zUR#~tUh`9R#?wgF8C&MustSudyzYs{zkTNQK@3-{eRo&f+{~}{j!E@VnWyZ6n%3Z-
zT#*@1ESZe=pR&mLl=$TR2l=)g1<s8fAws8;XGnah)8bY&ve7>n|8?E6Q*%St9ecW_
zX!<L|(0d`4-cNO=Zf4ofvY+L@()U9`de$$*ucqn69PwYaB#GP1+}3xRs8^1~)Egi6
zaFlVEaffL~9RG7V?^3kN`V-6MJbvKCE_-h4#aqq$6ocE26&!w%&-3t@KpxM-V;6EK
z)O~JuJDPQZDThJ&0NaMOObj<2%T~Q&us#5))R-G?J!DJva^7xOA`K;czJxh%H58dW
z$IjA(F^_@$0Aqm#17lHxOo$=N8s}as<&CkA_xfF%w&wcBSst%St+MLWwsJieUh2Gq
zug#-DyH(G0=O>B4$=TZHmbm2{o-6$RrmOxUHfL4dN7cboB`c~XZ{76agW&ofpQ`&}
zmx|2vdS^EE&f7gR^eR*r<~%v65Pb5^vU4xzo{}*#^`1NVoX8uIHzGNkSeBoAY^#4L
zwkf))*e@m~r265?=;rrE2d+O^eP^}z(}F9U%jXo^9rQka?D6Jr$2Ku8whwsh^~r4N
zlUz2J(&RY)-qfBt)ymmkpUj#c|IPZv*Oupy;pwj+ZPr)xfiKJD+icyg-eWuNyjA+L
zEvaBbre%L{#QSd6(j!;j?liCniDq?Rw@wVx)qa)6k}eo^Qfsv<*K3JW$`QKr`6Y6V
zAA1G&Xz?#PEE=~hjzOTbM@m9<nT+vfwVjE3wN)0bVSKyR`L1v1#BH2gd>7<KJf8FZ
z=*rGu=Jiq*n<N{Rdt&`3$I3eH^C~j;&}NNVx#{Rr{w+HLgBGhS(kwDfR7v^Q7s#+>
zXQ*`S>n(1cj+dL7nwl0d3QQ3>pSUp~d}>wJ;&qQ=*=w$E+{~4??oq1lm7oQEk80IF
zEz6wx;Uxbw#krH#&uu&}$WhD_<;ZzUl*1x>LYAlG#B--BrprHzxv@m-(odJ*&Rzp<
zBfEL8Rv2?Uk+OI9UA27OkD3R9SO4YpY6`N&PoHS0IITD4vH#WdvU!v0=C7K*Eyt%a
zE!eSThFfvb@3rPC8?xSpFfWr*sJ*`QeL+FlN(E&XiLT2*-9{-NP781O*n51pXLI%Q
z@9Has{WZ@`Sf_I*$}}@~&y`K4@BOSjc24_Mv@}Uku}#b=cGeVg!L4P9$x&e^Kb>65
zrX}m!F?p2=D{slu1vg$b#^$@s-4XiH_vm)5T`My(b{~p6;c#I6o%dELN`@~APwrUU
zWxZ;~JYW5;&(?eGKJ1cbJaBOJiZ6|aKg{~F+xq-6`OiYyoE`u6>aBabP)fVyRNjt1
zv7u9)-fCRwb5a%VGJBI)BoVsx!BtZeT~pJnEMbPl>xBw8U3sMuaLLtWXN+}MNVV|Q
z7K!jTuAFw2i3c}o%6;G6`RQ!go858WV%;ZeRXX3hv5KuEX)#;x_DfQSq@T=Kk-W)s
zt$OdRjlZYdar}NUh0pR@lSPNe6W-rdbFKc)xW}I*a3$gTnqs|gpN<KM2>o>1TI(Rc
zQn7KjkwEgww5G`JgwR#|mz0j|Q3*KkC31>J|E*-F(ARU8zFeXF?L}|W(F-g=f%~V-
zeNz!0<iGO6zkOz<{ik~TeLomIO)|Jt9cCT9GHSW-bJ=R|T3yu{tL82c(=<7^-dlX)
znSwLY@u{wleZ#nVKL@_aT+^s|$Fjm@>aI(&tENAD6v(JQ_3*_vr+0?O?lri+JkD_0
zLb2vkmgc2Ximrt){!{&<TJa+!`ontNKe~6$awV*i-kxyNaqH?^RhjE`@A%fsd^w-^
z{`HL+dxBD*NoTG+==L=9)hZ=>mwT7q3GWL!zVWy9`{2e$i{{zS@Cxx*DP!Zj@mpwI
z(zp5gHwv?-&5&EMyfI3OD`0-?>6@#<RQZqXTf6GuzJC+K_oqGy3uZP|eQXfA)12Yd
zXUqO}E2$NW8)MBcxH0RRT(D#I^ZVzZe_g82WuBCTsH9ZJ{KjiLm7JK@8ePz0UTYMf
z#QfCop4Xf00b+}%u<qWw=G)w<<}>c=s~QFfJ&tX^(-l-)S%3WZIp_Ge#92kEMGO|f
zIp3mu`^q;uPF=G3*pCYm*F$RNX71i<ZQ~y^H|k|i)PgQ9!zJ%Oojq6nR&M2+s8hSV
zzEv!h<lv~+PXB&IC}+lPs}-BRUDCVx>DIxbzApW#GcRp8DkW_s|MBjICkJ+{e#@#K
z!qHW^YSA0x*wA^`e7rw+f6%vJ_uet5kX?07%CXhAatp$2dW9pyx9TP<@kK<R5jgfM
z^!LT(m!7}a@vgyS^XJ5AeY<a&Zqhb;uXsM=T4LDq!?U?~eZ!)^Za7tR=LTD)L8z?X
z|L!#(gLslQr7u~tLQ^^MdeN3^rNXA6dCwyAo^g5jU%7BZVpYUdm)|~H&bjP7)8*-#
z6!_+oo%g1Ly&?%B30JSOFM3w>@siP-96QE2ljcs6ogV+8Z%>xNt=*e;XR2H0mSjEX
zW{)hkFm0IJGrww4o!YVUAhXV^a2P~xyY_7%^SNW4cYf7AzWG^TTk7rnD_2~6eRCFQ
z-U^YeUVZt@%Wl8)X)A@U=)B<WyPRxkbuoT&>I;`xUy!o1-)q^nD;`mN`1N&7qSvmz
zvkMxuk1MclnES6Zf9B%y*9vWhdG~H!xprga+Kta$H*fNp92H>l$a>MeIg49O=5YPj
zS&^Bsc<B`1^vK}T%T}6BcxGL3G&oX!*|tl5xl3*5&t1jw>|luPvQx^Jg)4ogCrnRR
zk}FZ#oiOd&6~DJD&vHFpeUjIsIl|{%(3LGaL+%AtmL77r+;QT-oh$!lM_ty<k<L9d
z_g0JbR?h7+^gQn!TD|3n`OPn%m(4$(bXO&HSE!w=>f6m1thm=so?VjpSujR(>y1A4
z4^zMXXDG;-W!vBWepLoT-?Ye#nP0>FPPT4~+`(7Ix^;`a&%=^CIVz8?ZaeU4O1MkS
z!rD!29$~VT&-!OA;dS;^TlDP8326qu+jE7qXFpAOVU;`IXRV)O^v=`^X56`(VMJ<>
z-Rjm)5u#V3GWuILMQC)kP6}v$y2^%u&n3=lQB5ef>c#*eZq-1&m0#C=cAXmhNyGnV
z)&2CBN3LexGCEPTB>2MAIjd$Byj{I?iC1eznV=bOUsUd;*|T@1zK)pk$gEycHd(<i
zc3Q!U6XH*IeXiM_X?S*zuK(v*x6Ybxoz4F}KuUdCyy=%GKEbQz>^z!%=KZP*A+`~A
zXFPSg|KEOU?Av{=;E+kh={25yjBXE>zdG!8eQA}>Rp(o2hvy$VcJW^0$#Y$?huQ9(
ztvmcl=ziMcWqOZ&J<FGw9bRJJUMG>09of7xHS^l-RTKQaT`+U#(q889ROHP3%Bt0i
zqDAkn+05r_R2`=ryOT5Wt;x}o@g|<XnWyf3zGtrRoh|!|y6#p=dP<jWE!_IP=u!Ta
zYi_v<6U$bv@BOu^cS_C?y={xUxYk7o?LM=)bnnkOv-^5?@a&XY@?I-q|HWeIziuxo
zrn+awhd8ZS-D`LI#pX|Eucz(Vv?}ZL6PLA5@16R&VcC|K&%CTEm?}BN+>gqfcyV!M
z<K;|0-#N2F{1TTxX}EbKBg^<#!+}*_ZoBQz1rbvVw;1-Bz0&<I+9Y9eu<!IGleB{&
zGVJ-86KkfvoP6Pt8)IJ2lHPyX@_*g_=YEm5o_8%NIVo?;<FB#l>7RaGdKv9@sV+Wq
zqUW*iyFNb(aC&<ux%xl=L*j|u(>JbY4!t|2a7!1L*01|QoR-0DQm0tT8Pa2&^v+y~
zclnv6>E^ZfQnvTH?@Q<Um3kb#xOKf*@w~8Bse2qZ0@WlF_Zo)k%PvSP`lDl}Y_j(5
zv=?7~imK0fooe@)`AQU@_%ETG6`eVIwzzhQtniLrGR@)`clgR<PbVGQT)K9G${h(G
zt@_n>&Ys!U`9#9oa_a6-y_*}q@2YwkQe17U8FuQo*CO8-^Q(bz(s}M;|J*-R>oe@S
zxq7yE)>a0tio(0wi%u?6e>>AO`d=e^@1(sYUoSkmu%a=PcTHA6wuP68mx))&tXCpW
z_!`XX7=C^V=U>NQ>A<_djVI>#ZXQY3pX?0M`o|u}W$MZ}RL*5E@yhU+zjzh%$Iq4P
zSBQBv{T255Dg&CixUa9h`p;aU=6#%8J(HUKW{Z8?=I1$A?Q50v)OQn_s`j?}an2Q5
zw?w%_D%vh6vqNdmljoN1v0B_yS}YE&_Sg`mplew^eZ?W~kcTqs56%87_qphm-b@W|
zk%ysM0^7D^-ajDZUU72f?k7iLkN%lrop#bBauJjMiUZyv4^+862(eal@)pFgY-c^r
z>8C$uL9|7R-_20lCx0AluPHqgSai*h`@P4j^UX6if8&+@T(O^#Kk1uS`boVzVPZ#W
z;?Lc6-C#Fem7U?uJvILwX4j|PoLv9m`_n72rY4cspKP78uVhZ!5<`_<Mb9V7pV=>@
zeg8dGa^bQwypN_l*|U7ZYR_Gp#4AkXOn3~}Zip-T!E>zQV{+jqwGUO{mnNF<%-JqH
zb#97Z)2BVl7WtU`S~NpQlS8K`>|$!Ms_XMBW#vXwq>s;(6kc|0pW(Cj8+JeIvP!X;
zSSjsz@?bOv7Z+C-x5G&_ku%4ePDp=?y|ix4l8W}?WV<4Tr3vq?YD4lQ_8jRJ4T;m-
z`RV!7pfwimKiR!fnjc=clzzf;;wi^14tq40UoJlJX7(qKQ=b>Byn4R=Q;*Zrrf>t+
z>2|kNcik5{o!`l{$AvL&5yO0O_BnqqdRKT?aNqBI&>^CK@ZqnAf3}NDbKbN&S}Af)
z@=A)tA<@mO?pg<9TsQO-?=afrS=Dvz*pIWy@mUuw0_0@o>3;v-X0&8h$1gtyYvwyw
zR$iLF;a3R5>{D_3t}mUsX3~n)i@FxEd$okRFwI!C=xbMCG1JxV69IXf-Y{)F_Wp{)
z>nD9?g<Coo`_B1!H&Ct5=;FDh#c#T~=F~4(>le+uy?kx9%)idw?8;?R`26H$-s~v(
zrQjj<WX}?bn&)c^PP}^^b<gbAqQJ#&y;8Y(Yko@ms(wA{{e0iXGbilVtPZ~X^nw>3
z(_9C~#67Jg7pHI1xOuz7EN@y(gB?ehz`+F*zZGn0U#z3`U;T97QkThoJ3ii8JC|Xu
zS^2FqHw{YXEWEQo$9`SNeJks>86v9RwQjBp5&Qluxv)F@aO>(j9*_2Ch)%xc(!Mx+
z-{OFX%Ff_D5p^BGHFjKkx$<6%*!`QRxkGb@<_yglDsOZO+ooF_Ui0pgfziQ@`DL7b
zjwM1`IyEN`u44S=-L^AAzNFQ6`YF+PYZ1XHE1TMvAHJyOeH7YW(HT9bV__@9-7TT-
z!YZS8L>FArTI%vZ{#odRM?(5dl7Xw6+b(5jbhk~)(CD;c-Z*trhDLi^QHDlqTTsRe
zCYz;|bFVZoKU~5ib!tIkph<~?GrP&Ln!|i6HYbSByRe7p3d1Vfzz?f>LPDf=&H2Cf
zg<fb5!=@eA=SAAEt`<Cdc=MdNN38uxoo5#Kil+()#GTYO<;*fV)fQjiJIzK>>h{~(
zrwwyk<}_{7uaFF^4RGJU63V|yF2u4T*K_Kfo{w#b0Vz+`Owvg7lHiv$$X_$*xofNO
z4gr(I{jdKlw!U9~xG950X{-INO&?x<KOJ59yliC;i{$}jb`Pyq1?gRx8my-$sN6cW
z#79-p?6c5p%g0BKrzWe1x~yIrBG$O+g5Xu}HEBtwE7zZztCh|rC%!3$&uUgx=hZz&
zvf`3Vi$8t2bn>$HiPLuj#F7m4H%^~^<Fxmir%PYFzr=aP`^l5dCv^Ad?$MQyUDe7L
zP?6d>XHMHeNBPja_C?O}Z)>CZ4&LBDR%f_CR(1lz>v#UzsRBCIOtJzm7~agv@eO|v
zlrAx?h+~TL)&0}t8;ke6`4iRm<mBba>?=9bXLw|?cqyxdrk5XbEbLL8rr0RPxy3`0
zqfv-+ipK&MCRL#w;*#sE>b$#UGN(GMX8Ab1Er@egFuVKnUX!Jv%O_1Mn0$G~+w!BD
zdRZqQ?YOr`G2`{Yb%!VT2i;N)P-?ssAV253rrhMcb}5-`rd>ZZ0)O5M5isR>rj^Mb
z!4VkM((AI*VgXyM>5A5bxz#%E=IqXQ+nbK;-ju)GbJEIF8B$qZUd<2dT<afx={g$A
zzsW~3G`KWRYx25=b4sik8P=Cy%@B)_Qc~8RIB~jP<XIQ~cB#p?nscPQFWx(v7%Y9d
zc}8MFV~K0Mzs^+i&~qV{qIrg)rLrM`97R<nm9tL&47?au6x_Pay!L11N}I!*s!k@K
z(R<yMxv8+~;1-?s<s7FZ<>#5)ys|qx&^gZf(DoHOUZ)y9*#2?D>!~?fZ(iB;vEE(v
zBFuK%DfPnsu%OtgHH(j1ZS&D>d9Wbh^_pvr>E82SifVpjcm5<{W0A?P{BA+*&i&SZ
z&9?MhX!sH{asTOOFJJ6l_p|1DQ^(`X%Tp8f@V|~(*%;V-jA7%^_p{Y3a?8AmygnTh
zeGpr}oI(ET?+np%Yd7DLzroNp??LIKirq(}zQwk7NvWE7<&@-UbMfwtXFqNpw@6gC
zVcC^7ZtX`&0h3ou(q5$1c}=b8)3y+`o7zvLZtYZ4y?EuOV{h|z1=baR%%TD>dS0KX
zv-G*z{>$M9qXk~?4Re#Ny?(J+*7o}S>y8)ZGFiNS5xShS<6>;pn*HltI#Qw^dF0Qm
zetz1rC~Cqkp^asl>-I!2TogU%<niZLYj|qH?Gt7`wpT(NmT@1vCb++>Grl@bt-mQg
zIBUW8kaJcH*G@-j@!PlF(aYJia>J<!+dxX@E;!}gB=%Hq>h5p9Z&gP}xwhQ?yr?K8
z;lVsf3HL6~r{}fuUmw%0@jG_s_p2Fqo-4jJx}e7lCT5jqPG?-aH_@b*YdO!BQx3Cv
zdX_YP(wLvKGvVP&4XHz?4VNrf^hl;cFROa(0=84m&%bBHnR}&Vw@h*Wnbvvm+16P{
zCVXAMd2eT<kc<QSggeLF)Yx9O9h_vID?RH%h7<FKNl(pwvGq*~o9nuI=7)@MN$cjL
zr3+-(l6TuIW4@hcu%z34wtYZQKoG+n#&-;pzI!_VSoHV!Z!6b%hxJXT9sg}7!16%b
z_w`4y>nRx_9oF*iPwAO2NhsEMZL)Of<tfu%Y)t>$o@8&?b~H`3|Fcqwm1solqsh7(
zBhRZZ|F8@MA544jfnWa*f79vU$rh|doqKm3iPM#D?U43fzC!<_(Wf)+4-1rDit)UB
z3Yp>O$T7V1!2ikBotF+ZuUzrzyIjTD{z!&yd5)<Q_s1EU`tYpY@g!x5@3N<ytF`MF
zump2jrOiHUt^C01x{2+~dFnl1S>GleEIXmCYU!)_d|t4?%AH&7(|koze(EF`$6Wr9
z(&C$T|Kx#LqTLOLCk3hPzHBimSUEQ<CH2B$E}kuCRrcnru25Eu&+BlNKeJokDe`fu
z1WPFQ9wxV4TNj*-{?~0X&FDX8t-w`w$Mwf{Rsa7K>eKcrWyPk5Q!)Gb7|z|=ZL(7F
z;vv4a4=Ehyzt?U`;0#(nX=&4XlOt0<T%2QO-thFt!xkyu(=zEFo}RZ{R{G}DrH$Iz
zi7}759z1B|2>RDn7N!5``1L(&gBe3MRn?#Bo3p7PZYgufzmT1iTREOyYnmp}EXMJ4
z^Me_TCzg4qTvm-cZP_yIR5b4z&v$xz;=P~hPu(B#FX*TDQ+?6Rtmn<V=15uHNbL;X
zbwo4v(Uf`O2Sv9(JSxZ<y(4-@kP^4*rT_@h{lz-)9Pf$nmd2@)xu1SN@?vy5yVO-I
zDrKSTsb%pR6I+EMHM(0jMJ&}3JLM$(!L^g=24h*nO=iWV8&4hFIipYf)FP&t+^HJ_
z#JE!f8P9~CN?Ir*c5302Lr&~EB?p|?r>O3YU|5#T*vMfPz!J*K#lh8aviAA?$wIE4
zj-GpeF>Sr8x$!bf&d)3T*2#iXO6KnN^SoXeBN8Baq@7D5q#?P^(tJwtruJ*+oY$R^
z>5NtmXgTxwpr*Uf?3v3GmO<gPge`I63yWTwv@E?WFs)*W_i_cDk80D+W;|^MuNQgv
z;j-=wU5OvdqYb1qZfWc;d3BJpR3?T~QvRGM|INVlFJAmH*x|RXTf2d60pp1}gE!aD
z9=O5N10^)r?jMS|yY<Fho}Qw6xomE(zs;DFjb~4~VS3ho`6|Cq>4vF^3m9)Oq+H&1
z?6}_j7}mF^9`Bb_n9aBeL<DWz5$k=H>rgJ+ku-;#XS>c?XxRU8`&a(+p*eGcaeMBQ
zI_2-bYv<duC#?6Aagr&z+*fBfZF^aA?o!daNnb#u!S`)t7xPus(<lEC3(n6j{?oQW
z{8QWW$%keiiayl*d=rbiicfZV<<ax%g*?gj`VVyv3I^P0Qhv6H_4_KD+Cw*+9!f0I
zouNC!H|EiTnU(^d*w?fyF63P|SKF<2u5G~@CWeh==cGYC&tQ;h*l_&mk#q85;S914
z>=zhI80N;EW}ILjaMX(FCgZBfby5zz3$8I_gsf#ea!#rF-khTO;S9_N3>cV>EXbOw
zWy8S3z<hwgz>Gomh@WWn&78N}?rP1M63$?m|FKj)z*Y35x$TqP(w{n)-qF_D;u>WA
zw@UK|7jxmfb7I!H-E+51JyE3nYqP80ZIumol9uOgyu3{)VxIUxor5+!OFuitlz3Zs
z8-QT60SHbi-gt4_jg06@jqN8gqEoi$8b!T3Ak0^kcVegF^R_wCFvj%n#?40T;S6h=
zOc|sa*cM!jo>c67j@Q!s$(;!>h8LJWCuG+06ie_jipAemKD2OYJ(+FJaEBr9%9cQe
z-S3|CTl{bMf5*c|_ow-j`xEM!HqExkxu)Qd#=e@Z!SmyD(F1%5>>I-6O9U*v<CqWJ
z`+hKoVbb5449?k~-g|x8a`ecpq@-Nc>^n+Hxo;hADkbC!ZaBxjykJ=YA{{53+dHjb
zTEVo0X$b|_br0$s)bZJ)J4aV$)4_b3%BF`F27Y_g3RBuR6~a_&cb;Bi)3UtqK@W4f
zjK$*xKZ1T9eqgn8-S@4#AMKH9nDZu8rcwQ{dOgQFkgpY3vJbDgYO&%>`%mW3FK)pG
z3qN{i*jPMXs9`S|KJ}Z(az3{-OI_pIHBSE@2_#k>el)4^e&G)FrR_eN*^aChH(oa`
zUw_rZCB=1#TJ9Ak%dMe0A9JMij#&BJwo3V&@Y-MbPkxKzXWyRhM;}>uuTo|TTBIH=
z-Mb}gj`V7k-l;49P2}6Y>7LZRI;DqkmN7a<ZUpJQtqiWxj(23r^IFpBTkV{4ZH<+-
zqhED6Q%ldnX?v|+?pZ6GyJXp)wR`Ks(vNRU)?(grB;>hrW3KE+zV;jIazkf+XttX%
zmwm^@t;df?af;mz=QGp1k`-xf<-^#Ny>Pn--?=y2D^4`o{+hgR=G6S@vb?u87JM@*
zdDf?sqO_oKweS15vsw9OY`6EuvcGE%56~(;{c!Sa74z5>j&l!X0ul?}SZY7^GF)aC
z7(KZ%)bn=spB0>Cd*5eY?Xg<5%BsX+>K>B`Wef-YndIJ{^e6mN($3j`bQc*WH_Ux{
z%WYDKMDs7_tL2^Ab9b%QG<7qPQk}Yt_Yv1eyL+Z{muSze40(QbwdNAvwA;QVEUiMk
zg646nLmB^UcH><W#CR=xvC8c0%u=7O9(mlymG?ZNM@81)pQ1`c;nU`W!W#D<n6eqq
zUzTv3W$LP3H+a5UG2a!~A$$IQb>>q)&$*MVR7<^Hd1NeK@cPMnS7DB8SsMi2dW)9r
z$zI$Wev3(orP6BIbd_xH{Buuv`L><lidJd;yy@Q5sHqdG-qoDjIak2;=YFHsz9ZBB
z91&jEbV~K7ptM0@wp3x5aAsn+`Oc%W&zy|BRlMrY6aPm`R=7ki7tnM2amnt((oYx9
z{3uOTxo#DhcXaKB2V&nJ?785Vc6{xlNo#vo&avmN2g5q<GXHrZd2?CC1XlGwUbVsE
za@o|?d&K?HWq;h|pL3bL*Md!7*w9XG|K)sk*|a$kTB{!|<hjY7rBWbft1;t!i1y4C
zZ8uUbu{TN0Hc)K5C}ZKq>tiagTm6Ioy!wKyM^_H4Yz$^z-Z|CnpTaBkOTG!yOgh{e
zCpr`)Uk|xA<COI0`;rgezq#BjpR%?|(nR--SJQO4S8lv}Wb8{fymPGD?-ng>n(@0%
zTG;;F&uWSMh2P2+39PG$oocx0;H|zoM$H?BTn$3aLd|C1nHl`7iXNHHwR>@uy}jlu
zTjPl;SL=$obFbgjT+!#8rrz)OK6dIX{R?JqW1nmCp6!paX8o%f_jF}W(CUOgt(y+z
zebm_ABp$tyOKoS#e~ButWes6bUXNUb&g@ycw>x{oRNb;}|4KXWvbigZmYv#JETp|=
zONgGefB4Gdnc|^eqY59Wo?{i%No_Z-)0p_?kYB{A6G5))dW}7#V=A9kIR5Tfuyb*l
zofh|vtBPyxu71dtt-Ih@0(*K-rEs*;*K1+jWseIta&On_>fH3w;OD)FKlg7RH&Fdn
zcYRr5?uMusPqSs)Y>lFpmCbyy+u=OxFWm?o)>pb&8$)(-o~~xlFTeVz^MmF|pQ%Yj
z&yT337EV#(5n1>sSp2wF|L;Q{M%6XvpR|RXy%jp?#I%A%x<9I~icD+2*c{Z-sC4D<
z)D!+`FBF!z|FU(~`pa-&rI7Ge4Xvm9IWnx58Zfs;{+jq^M&_SN-KJBF9-h5+2kuQ@
zq_bDz>3(jJ#Dqm`Rr(jI8}Df=Yh5T0{H6C`iU4Qy)AQV4;ugI3I?sA6MBu{1z21RA
zLCcr;U9}E<&+uJn311QW&fu@F|5PxvGn|~Tl-2#2^yOo>7S5<&r9Y#xJ!gfN{kH5~
zS(}{B`|!(5U-Cq#;?P6eWsS>Ho%bErv%P*_MfzmNg`c6Kv;OKWOjPN-IWzrgXO-7I
zIa!(5YeDtVZ&^2YU468#p<ypy-etKpOQwZ1aZUVw#4&Ewg3Z<kZa&!<@^I-dwabgc
zRCe6hYIZGi!s__aq8+uj1Xb#TjjmamD80TRS`eR=DQmF($|cu!xhplRRcCi?+PS1>
zrSHE-wj8=VV!~UE?6qFLOgiZmG&$*?To%)}6&ochS09{tpt?I|m#dzwK6mA<s3-~R
zH|-kRqj?1)Re9~I@3+Of8<vSAAAk3Hhk;hQW4qY4g;LumZS=AWQhF3|D>iDyCyP~V
zC$bEu^R*mbanott-YcP{n_6BjJu2^$6r_8?GwiP3@(I(@WKVD2zjx1yTQ0}z?p-^%
z@IqvrU75L5?Ix`U$GHQ3%m}ay4RbpowY7|KxtUp8zhT~zCTp!7r@JTI{#BuUE?RN>
z?D(GyD;h%zRC9Sf%iAtZ%vwG<_oDgNZ7y!l=UDGan{D#x=dXzE>~j`>(wcD7d)1zf
z9nYE1t}84#s&f9l%EL)n?xs)w&Q&v6<>e&t@<(6a`xhWWXM(lX-i!@Dne!DHc-l66
z{>S`a#j+FJ&S%&H?Gj!+GdwrthUUiW(J%h&y?01w@eHYwtP7bz%b)n2bhevd^~*T8
zBme2lryq{!TD=jIohFmJ+l)D3)3wU)ys>sFwNGQU=DRr`=byR8%YOCbu*f4<#TH%)
z?US1I{?^vQZ`_{2%U^w6l@inY@VNaZG2PIDm2uur*N1Jrrk!YiGH9pw5APqh=4jmW
zHJg*T|3ybzUWjG%)0`z15)}(oJa?aZ+jnS^fL)nPUeG=hkyRC2qxanUd~<El+A^ED
zQL5s0q4I8F3tZO3O!#EgZ5%(dz%yyu-k5n$4(~KuH>2Rj6EE4h{105!{AZiII`B!y
z)hb}g4wbDZIp)XOh(2=I&->+;!`!J-+FMPhF#b8^IJ17i65n}lPa-Ctby<6m!>h5_
ztHkTerb5X=zmt9^7Z)k6dBr@vZDrhhpZwlQ1>Ro2dA6K)_#LvY`A?(JIvc$LE@|Iv
z6J3VBET;yUw_!8ptkBeTmS<sQWpyombg=Hb<01~B%B+2>J8n7n@}Bj2zWNPM<;tav
zQBCjec-e*)EV**oL7R6=P;UG=d5x8q!~7KFX1!{kVp+)fj5%^f@GFm=xKEe!46C+F
z?2X=eciW6T(rWL;CYN`g<Yrt~v@I7bv1IZ!LCr(45fVEOxm2F{UfLiZxpLa6Xe~Yd
zkh_ciOj?~QzFO#bV`lmzG1dxg{am3hA@{;8r}svj&f9sdyfybp-Hno9ws0e6`M7Sy
z?Jjfg+w$L7{)07EzyAIAv$xKF)s8Jx_|9JY;7xzA<YKo)3Cg7xSBw31yA{K_KTFoT
z>gMN#&0MlydSn?SRHap~cg?C{<LWS-!glgmpV#_1x7d;s#7=IxCn&b}VzJrIAmP<^
zGelLF=-;qh=#%*A$DNbsPiM?}e`?Y1OP8wR)wXTn_g!J?dcu6kpEl;;8lkG_<Gy=h
zT_a|4{fqf{y7QKjL4_qxH(%|_)t$3utytM9zE|v!aA~Va>VwsROwmD0_NiUm5~bjO
z<%Ub`+46}ZoL~IE+;HjlU9+N7YW9k?owsJKSbK8Xu_Z3&UhAH4_V9FMoXcdz`lje`
znb#`eU*N`o$--a<_8rq@a;mCVumtriuL^p0Tes*_+?=wYbDQp@EcGZ@wN^E?P(smX
zN0r{HCuxamj%<JaZAs@$vzzTR8bdTvZmkTkJY(_t{i(li+rPQwZJc4gCuPUfH~+2Q
zD`{#+icCM1Zf|()a@BRa&$q0qmfl_=tM>R*`&`w{ze0W99k}C?*mZXgr*p?$9#+fw
zQ_C#xZT(ravMKxW{@#kp{i5p1&uXf|MIzSTUX`))=h{zKWW(=R%&m4@Enf1j`I+37
zZ29jxCsYF58Dr}0H{HI`e}3QOyX&`|tXKH6c3OVazV*|Dnf+N+_PrD{v<?oIeXm`3
zCeZOeGw)o_+DdKrzm3}qHXO1N+aJ6-St0h-rRz_1ZB`}ec=kW;J8M1DWO<NzOwvqi
z@xIb-Z@)=*9$eLJK58UX{UIoF*WQxP2URvi6ootVD5U?JyKK^u`!}}CUT(6=>sarX
zgQ>^wwA7kv-!s3Ob#=+c88Tv%nHSZ3+j7z6@mdShSffQB67~Gv7?%8VDxG}nLN@E(
zfV*5OUR*UXs`vK!m~>Cq-z;{3J9J;b!__+?`NcdcFRfdmEu_DDPVrUqUHkOHTLP85
zOJ4p{UGMbI&-dY>Wzx#C13WLvG_HD+wU+Ipi{1-vzNudI`AjUiT=zDA_nvrSxg&4W
zGB(p5Klu=)y9<PyrC8=(f4RrcajyrX?7m~uS{HgI8)oWkV}AH%_M`)AJeTJ)iLQur
z^_*D3Tzg4YcFVoe1=GY%NrlX+o%ZO9P@oW7$*nut7p4@21{Z||_02af^;)-4rfE~h
z&)v(UJ5HUkn(bz&RrGAdjwUI~qdg~XW@LX8ES!J1ExYsG)NRI#FV^jzdrr6b@viqu
zdpaNP-?-1*F>k5QxjEPQL|*M!S#`S6N!Imb?uDyMjAM9bR0`WnYg+g-#$DU@qTUpV
zQ=A2sE$Zs)CNC@bU_HCsU-{%i{hN)-GguqU6mxQ&Zro}qhzodIVlX*<-m+S!+^sdM
zTf?n3dpbV(cPHycSfT!$)6q}gR{G1`T62KY!pc3<Ms0FzN?7WXeJ5*XpG-Ec%u(z8
zyYBKMlS-8XPuN6fXnUO#K4Ud~n_`&i)O~>ur*{6{b5Y}BlhoA6%Rj4ser4RY>fn;;
z7M+n=Rf`vyYgHxB%QyMtrku{JkvQ!{@Z!jO%)bt-Y~)|Z-LO%QAuvnpLFotUrl()c
zuBk@c(%<g&>4s+5?ZeWc@t-GI>{QG2S}dG;QzC3i;_>{D@`?LAjOW%b=Ug(wAvHh5
z)O%X@ihZ8^A*-sAk6ll6ogSUEc#^qR<>E=68MV`7rtHZK^06ws%zp4Sx7F(F5s8AW
z@zJZQlh@3w=<-rL`}*~j6~T-Bdd+IDR?gkavU0JCVRDLrq*mqPN^>dm@}`s?@2X!*
zUtTJ*(@o;svS^QQRK^;vp1IR2KfDQi9K3hZ>Xti3lf+qXrXEpgwX0uxhm$w;<<+pd
znaod1GiE=E`LiZYt@3yFN+$286L0vpE%+$eUKiCRDt769d^nf+R}CMlNh=-tBUhad
z`q^ESs;O>i|97!gPWVrLW#^B-|Ezux?{c)@V%|gXuf-uk&OHvB%X255QQc~hyt7*M
z{qkRB7w$A`$$YVt3$cEYYyE-8e94OjyYhfW<_AhYqz*j0`l<G%_1{LJbwB5wQ*X;H
zyWGQ)FePP5OK0BKV+qG+eJ-2B`1s3w%bR%|mu4})U@&8lZs0v|jUglAJX;a-2F4pN
zCWqC;^l^2xX19I5n|j_k>4{Ci&ute!I`TG2xm;cF=6lpC7E^_@D_KoDmCS=CYy5nz
zy6y2LwN;PYet(<Y+-Y>Nvp%#r&NSlvsb6c<re2bmezPoh?TP!9y-^~byRR@@D!+Fq
zfVchH4d>7A0$9GAZjq6EQ>8ll!vs%5({qKUXHM~J?o6K8%6-W9*3m6K9<rPAbeV0>
zygj?;|Ek4$A^$>twwznILVs#>rRHM$DXc7^mdu~@nr^P^-QfLU&9)m?en&}b?4R1^
znR!FHPVdN0-H4<o>ve7BypFC&tz5{zHmfw?am{yUnIlDct6pstS$-+d?A!x8u}=1{
z`@Zkk?!E7{#-*%H67M}0K9tScqrF*4etNHS?}t4aou8d5B~Bd*5DPoiBv^K3)8PQQ
zwoe8*InJFDUPnBHo_zUx@q*T|Pm8y7?d5gfaQ$FP-x9Cg61R?C&}%C*+|r$S<5?Zc
ztP|VH16K!~4hUL(Z~k=M#qTvHhDELXwIHzBjW=uYA!XsRoho0gO%)nXd3)<SpGN53
z+bevo_wwoZOLA@BHf$Ho+|N8io7+8e%A^%jJF<i4{d3@SRy|NCk)*n$s31~G@9u&(
z7b@lY{=8o)664kQmTf6>O!B?^a~3S#827Vx_pv>TFK*EIb2d@c`-7DAsa4bLSIelf
zp7@ddZ&lH>Q;{N<4WEcTEwH#6Wa<5MgLN`@9d{k~x2>(*p2^`~<<B%7UdsKacRA1H
zFRG~zG??8bE!r9uGtb%hzNfM^A*HGOO4f0nH(R6D9NEjhM?PuBah8lwUW4Vmvm{Js
zGDw)Nde9%J{J1nT_LW7|lr0f$OrfS;TaHd;`pa-Y^P*;_++~UKV>^tuxjsL|w0G76
zg=eR3O1?f9QGTqdw{ZH_Y2R5HdWED^O;+7m@hQT_!STWa(<yWRiazIc^n9XsC8DgY
zK$7E}Kv`SCyF7-SMJ0Y_d?mY=7|9BQD)%p0PRm<%uK&L7bunn2uxa_`-HZ*dT&^Z)
zc$sfWbYX7}m*_FPUm$4F=6Lx*z7fZ)V|ABr<V*ebO_qB%Jv%w+-<qNmk+!VIGjoCq
zT<%SK5a(UN?WXT|bj=Mlse=*eb@{XIM=w)3z5dYMLwDVds<=ddFyP((Abu-<&ShQi
z*V}ae=>FOJV{u%gvhSK7l7D!e7Z`q%PTmhD9#$o!FP{9;^8U+%(H0)}3j;nlGKwC4
zRHFRs>$iPnx(}VM&&ca}dF(<i&)s7ea+UHjpPp8lx1qJIAerNO$z6#<y}5>~E5hd*
z-j%q_`*{1YlID85nfs)jtv-tf1ua%Nr+q0RZ->&Hm%9v?NIUPky5)|<>E4ZZBu?}0
zk2RPg?Yz@)isyDMM@1dssXj|5tq5=ka9G|Dm%p(2j^n$Kz07wQ-!POh{ANqwFl^7*
zv(Vzzio~|G3@g6g69<m0%d|8xadtKa6A|VHy6tXh2D<HSYL^Q#GTPeGGBS?MUh<`7
zUYxfD*YRF+&w%L<rach5nzH|j(3@=Ta_(~O>Ai~48>E|-KS+48XWIvlX-g)G^=wI4
z!m6w*k=fY)Oe)i$Vm*@3L62^!%!3@<ZV?8DvMObJHYBvLPJX=g?MA~xO@d-Q1{yMP
zlT!^YH8qNIy?DVsVeTq3hV03y%o`Z5FfmvxlX5)B!KHRIfkE`(!Nf1lldrV#9s95^
z^0idtlsP>evrL*nHm#T_)?;86C({k$BrMsOu)(OsdCI~)3xC^kE#P@sZnZWzVEM{g
zF$UF=TULGLtwOG*V$S;RN$pLIiFb;H`|7Tx@a|632(US_YfjOu!n%w;mx-6MgP))M
zdtkv@<>f_@Rja+9f4TCxJkWygn?~v!cbA(hyC(Hluc{0Fs?`3W`|fdX(cPT)wtBPd
zO5NlUle{ncO!>6iI-7SNvr61`;K{pi-|44fr|b>^?P%C>+fkuu$CZcCM{`7#@5f9R
zdJ)7VYt-Q!d+5u%OPsP=o@G;o+gI#cy1p<yY*F~?)MtUAw-_8QuW^{O#mjy5fjj%m
zGBS3q{4F4?ops0YsG^qV>5?rz?jYj0$@?`TYq$8i-!9q0ySJQ4N}oY}`{q@3p1cpe
z7HMuxcGz9EqVsLE#v&WtoxC0U%x=91xf*)))<NB`R|BJEw_d%u)?XyJ{>O=&=~k~-
zY}Q+Nj%n`p2O(=O$!^e(`qdl0!|QzMmzd)L4HMo*L{u6dkMO!*`eiO#GuxrywWURm
z7QVk~dDZj7IUYvttYfxp=YK4H)8Hb%)`oHJRcCkpvM=%V)v{rS+fsh&6h#L1yi0Pu
z@XzmlN!EpxOL^lKhw4aP?=}>(ntM$!)7{U?^mVh}o06#Krn0~PP3I3$4{9!q(Q?e0
zpeT3u!1MR~y<C|gx0|(IJN%p49=w@v1^c-)tFsR(Z$H!g>Xvm+@$Bj=r*0itID6r8
zf$AAr^<g`i=1sA_{`*#y{13G@krxwA?-O}CVNpv-ZQ0Kt%beQCEthYvH8{RosJTjW
z%Y(?NyF>1URYpI}nYx*EfAhSF5?vc7Io-KG=l|U8z3ml`H%n|dK22nW#>1bV3kyu=
zX}P}q!<YGL#gDU&cRotFF3tStufTGtnt8AJ1rRaiU)-;rNoyD*N|!ILy6gUVt#Ybc
zCj?&DaAtMp3zqMz7?XE)B>HjsUDlG{*nhm-xwbbboVnCAAiR0)Ts`yT4Haj!Cv(?a
zp8n;ALqD&_k9DF3?4cc-uHBn@V(+&g<4bewFMK=pfg_|sVZZ1PPNtuMrW{!UTWsHL
z<8BaY_FK2P*?4l1_VVKny1Z8|Iq33kIp^@2XUo}&>MM4O{5a<dS*eyZv^TUbD!A~u
zJx5jbq~=@0)M=uNcOHz%E1JY+*Xvo9=fCOH&swc2tEJCePaQiMSn^y=)o7`Un)XV~
zw~fX-Wo8GTN=^!W{AEV|rrD?d3;0H*?6YL*xMJ@8BbA%)YU;sPs<zLyYGzrgvO)uu
zRiFL&k%asdgX5l}zNeDCDt$D2f40eKrWyu{38w~5%d)gpJC(FZNA1`}SGPQk&Y}dr
zDMws9eI%Yg{W3$NJJNe&=cE}oE`d0$orNJf&x`8LaHaW7IqYKTGsXF)iEymJMONWh
zgKSN+g$H$~I9QsiStTrD?(&bd>x)dTHM*vrT<z63uWyeVWADC9kqc%Fj-1!;CG7P)
zY;!SS9?#R00rPkgCt1H{pOAdtx2vegM@*%lXz?7m&2!`?8`?@<KKeoL`SBx@X7Ic`
zIbq(@`8r%XUc{8n$>qC$V&7TAefMNqCp%XsO6c@HT-sT`cjnci7ka!mmoD685+d$g
znz_P>Df9K&eW89)@lDZ9(M{D?cgG2N^R12iTPw>Z`9n35L$K(?8ejECb<>Lje?<R~
z+F-uDZ1tYCk4!>lp8x2r$9uB$fAK@{L(6V-Y0FP3vABM0u_;sY{m_od3e!q??oNpe
ze_?j&YCtT{#iIhTK94<{m#XmT1XOIue(Ivc_&jD}iLr#KVchhh8sBT`qBCONpIWx$
zZ}RMovEI{8{+WOC-)ZKkmN|+F%TM`Dox4x5w!7l^%d&-D=VpKVDDk=5aDT*O$!^EE
z^pqPPcHG~5caHPwD<LPpNUeE!s(y3Cb&*cHQxi7_h;>iq>ek!1VX1ef&Xi-9?s4vn
z>`D)KCzd5JMdItWj3^d{shk!W)v2=GllDhl4+vPkAdjKM>xF7%`>zGZclXtux_(V_
zMR34;R)?upsy(|km)FE}-jHiMSSM^9V31sVOGxjE?$eF;d(N)hQKQA5k@o!d?4n?w
zhyD-G9rV@BpVicU%8vE-Bb~k4noOLn(@w2yPBfnG%DU(+@A_Xy6Jw()59Qssq!)cF
z)qD4p&h#KXY4O)}&hP!XZy7L{6<%!)o>RCxXRX>Ko9n7s1?wfgM?Za6nLYDw8`}y-
z(WSX&x7tD#FHcT-AFMainc4YP#^PSHBil7*9dVP*<!QZSC;fKwMG$c^Q|#)yMUGdh
zu3oSa{<_6b!7M8Mbw<4E*$?Hi>`SF99;Pw)?=#=0|Lgn;A+DRR8MGuyj+Mz~&g@Oy
z7^t^fr@Oc8=9=r>UpH*jF*}vKSjX&Clidp8IH3nAx_&1cw($98i5%(N6C}2K%F$!3
z2|rSOR<~(fJM31OGv&Bj<(o$#Z_8zc^qy`El#_nCaqEoz7NOElH+Bbay(rgnbdjQ4
z;R>@;DT`gqPAPZph-O^t*E!d$%<8K5?+YRBAK03XNX~XnetJd8D|FKIrKhint#TIs
z8oWzpl2dn5#;@!Tt9j;W-q4IFie7U1ppm<|<kU|}cT{hvT5LPAo-aDaV$KIgzYD4}
zKy$DlEMELU?3~ycF`nIXzNO8Y_uk5b>#)NC%WJ&57CV-kyy#=QYdHIyP5p`~OR8VD
z9i2LfCxLm#CWDQsy6ss^5?yE5!WnoSm^XZjwro(Dcd}XSD|6`ARa!}}8LS)jNpf$!
zlc%7}9QIP&I_cJms>KR?s%Iz8G|ezs7$|ge%bKE@q5p!ml|{Uj?*I+%nJ7XhY#H9L
zGsqtC3$<>L1<zqJ-w0=HuxOdV?5^T0eDcB@-&I}~F9ovW1+s6NG)il;_gFNmty?gY
ztt;(jO@Ku(bXxX}Lwr+Y(?thC^$SNhP0r3z(^}BUm9dCv){2Jb&CwR;Bqh#0tdHJt
z&a>7pNTfY~@f+bvv9wkWjyWk+Z54keKQHJ1z-I4nyPikTF|BO}$M<O7n^`CBg{y8}
z;H+Df*<p0|8E>z&)cxl7SG}^TzA#BJ_^i{qV0rgo);#a;Eqfd@r$t?5__yVl(G%z9
zizRjm2j9=tW?i;8Ti}ervdHF=8BgB0hOtgf5!EZ<N;2M+Q_&ogVY;pGp5*4$2Q^K6
zidQS<%uq8uD<$J3B>RdhVeztFyHmd{%zXtJyg0lRdJ^7rTn$LOWTMgJHt*A^RSUEm
zWDmqG3}W#7zht4$yrRDj;J}MpAK2jhXNk<IQ^E7%m+M6-@Cu3)X><!^EMl2;=PjG;
zi6g8g*CVFQziZ@Q8OO4j#X0Zp66TrvZ!(`+xY^sl+dziVb<-Qw872+BSL}q}vO8Y~
zt5XhNu;<CoB_BMLL!yuOHLosAl`S_}cPOM~|HdA(Jr649#)yYE@FpxiDy6WYf=7S*
z!868-YS|diD?E^7*~79TcB@~gtONUm98Wgw*vAZ$`a;i~Rk$PBVtv>%SKuAbqPdQz
z3@gewFRX}KoW*;9!GJ;KlKRf}@aYVUNda3K3>bJAK<VNgLy5r6LyH+5QcgcOE4jzA
zSwEd+lTZSOC40k<qGfl@uc`cBbaZCS3ALy@Cqw79a7z~PT~=mqi8-NmePgFw`l91c
z7av-BuJnuanaiqwRDY=MP~8!uIWNAI{bUMf9%tVbLC~DO{-@&nhdBo!aKqXeZ(}6t
zFWlkjU+-9KoRL@gHirFrB*Px5X#r9k7jAvxTEOVBD7m-UZOicwFQiJ$4xbE`k#0Vk
zw&4bkSTvJG$AunK0V(E%hKdJ*PHQ`yWmKr{Vq-cesjQK_;c`y|&+;EnO(%%1elpMT
zDwD&Ce^TrdF7g=7a&+6=wN;`o>agMLaE9vY6ZZcintP@($R3GfW<8*IU`qYOi#jU*
zl@`v8+o-6oxbdzGgVKD-PkNtbb}mR#nk%`)Soq}`5v9OgM|~!1%uoeG@!OT4IeiO*
zLe4f}F3?#fD$kW>&h$uAn=~`As52$)qnc=|MN;%czUG{?31@jW&2%&~K9N@0db5CI
zsw>|Uqvn{j38#6ABI6iM7<BK2zmoiGu!AXsVbbDb9-a!7?ZIl#RXa+LEw(=)&Nltb
zgGo;>a>c4eS~FxQn60|Oki#H0dokmU6^cJk8rCR;O4zwK_huFx;pvNrW9)f#gkhp@
z{G{Iw@0fSC3r!1Ws4h)n-xz!pyprKI>x`oNs^0_lom(F~eQEa{Y0dA8s%EZ;d+z#$
zN761SP1CYEj>VYOm~}JDW)|hc%l9v=)H$egu%L{oeWl((or8N$%-ztsnRPR(vrxOP
z#N$01Y%78)qAQ{+j$N~n-Wq%I$K9Qmf_NLEj7r!TzAWFPw~j&jfLpgIgB3HwuE`JU
zPeaz@L<&pT{kZvw3w)$O1K)!6ObfDj75dyJ+$nz|H)*1y(J5_Pjf3qmDHFQSWiv=H
zur)9yu%M^iB}c-ZO*#5e!ta{I&!bxvC(Pa99Ks;0y0Kus+{bk#8*jPpd*XD*;{D`r
zt?}z*q<?IRkD8&^CMPJnjp@~mYiosy?+CDZsP715RubPCoYA><NzCC_((U5&0~emL
zj|*Hd@4j(#E1$IU>?bo~xN@$)v`gs<4F0p$SLCad;Y*o)yRYw=ySVb@_ge;=Z*)#(
zUGaHlsI|#<=fujsqN|~`54Lir%l^?@&zRn=bIZTL%s=1JS9jK-&P!JLw{E=Obz{ne
z%%JX1*DGT!D#b$2`5k60EXxUYW1Oph@ZpNex39B;QiFHR(0%z}ndb%fIIl-atDV|5
z1o3NpJ?GS*CA8#3zG~T3xfNl?g}dcel$2lh)N|d?mY+Ix_YIHTMSH*NBnBP*xo&do
zs}|)(qZ{AW`K9QT$lSDe+p}chb+NlPY(oAPyM0+5bRwgRublImTl%)Pz2)VEsq3Vk
z9qUSte;+?Lc;d>^(sd7aUp1H(cOma<>-k4j+@fEjm+Y~clwGhi$eoGzs(yo&aB;}_
zAlKRadoSD%_tIKDD^$HGX~tw8&yx*eTk^hH=l^+s{lTQT6FEYbUG2*rDGHZlsL9Rh
zY)^Z%aAw36`{rNMJ^NktPh8Z0HNW#$`l5fSarsvqg3dH8z0q*Gt<%aRl>egm!yxVT
zk#Qk+x_!<siHXxnUpdEg{Ug8cw}jpZtyz{A_;-^?CF}Nr?FlEB9-DomS$*}c3kKV^
z?a2de?$&C*IXyfpXY2Im0y{h;ri$}=75|%i&nn~CMy>{-9-$t!C(~*)&ay2&f9fWC
z@p-oD*^{?a#FSJAGHP7ZxF{CRrE+yjne;RMJzoO&v){F!oW9?`Zn>u&m%|Bj+1dhS
z)rq<;+ZfO6+Y_d4=9RT&ZJ?7hf8Z;L<IK{GP4j*wylh+0vY=(b=O!i)Tr^8Sf&GNz
zRZxiw%7fr~d3`y!1Xettxc}LN+tF^Hu3QlLcy3wSvnv-w)}K4F+Su*M5f3#Lv-?Lh
zA&cTb>xWWQy7ki^ou2ulNNwMpO2IQiXM}p(W>pln9QEB}F<UAzSlQ=`x$s|Ic@e&u
z3wpK)hh++9ajRLMGg+`;?^5<35`PSK>VcOIEfAgDy6)16#(QZOZhzuZSQW|45;pr_
zmNx_Y#g&Ud$BZ=aO<;C;S1Hykd~m+RQE=hA@j8!E*TJ~mdmji}w(m`QGs8S0n=Sin
z!dl5SrrOtL9gH%L$UZ5}xPkcw<C~+}n)WrYGKblM#lmE5V$^<vnHy(p?BAGWn6)w0
z&}O4`{EYsHPKn+bZK;xaA_+UEO#Aj?Pg>;JML#v0&#g3gvV6~tZ*kIxpZ@S{E!wns
z?;p|1W0oZi&2!mOj;>ZdS5%dxx_0r#OB!arpGr0M2u|3<TUvXX;lzi6`gqT4MhRtm
zPHOY>Hh69-3cmXAfmiht2KL`}(!K>wX{ArDuHr3z8#`^wrV|P0t<+}gCuR5roPEx^
zBu7<xgQE6<@3Pai?>Ropvk<v@DCW4wbscjN?$vLO@HW?I=N?S;TPC@ZO?n@bx@^z&
zHIdcke+2(TELmcF{ENvnWtW;F^TL$iu(d%Yv(t>`E}gP<+MUGv(<fY&=4Ji4@!7_Q
zSHC_D6|fdm>oDTlJdtPm+p}e%zm|%N*|1rEv%F^MX1UE`+4jUp;T<WjY~zKeZAtX(
z4v~Md`fkP@uKjB+?cnb1emPfW@y~T3#lNgor>n+_ZJEj%8@coJq+?0f)R?|CM9tLU
z?%ksx^_+i717l7qL-G%)w)G}P-K=kavfOkmV>l_PCb~D^BKPH#g$u)Hs&R|*mK&^R
zF#ENP@eV^z6}R};<*U~QZmU{5BP-MGhJ}W~x~Cdx;Zv%aKI>#jcKsFqI<4Sx?#d0*
zOa3}tch)<@-e2wGV(B6_YlV#RY9*(q(;_N_vr=vES$VHfTKu{;RoZHfR0*4p>W6n*
z4VJk%n@R<Bmfq=*Sn%9v+4J|7l^gum_r2w@>5g`uVUqUq`8t`fqOIL)Gm^BOQcIJL
zJU303-6)=$eeGoFmc5obe{cObdQj_}*p`}zud}c87vI_<Ahh<POSTWs*AI*26sMmz
zTD7$NqtHwK<IFo(@=68k)rVZ1wS1b^o)G_)+81SgOWcL;UOcjJ8}GxY+1|c6$3LHp
z-BEY8UVL(Lw~m2pxs7~~;jwMMeLkgp&~hm@xhPY(`TicRh`z9>%<EpOt_ZwwZQh%)
zLHuL*D))ru4JnFRVKrU7+^g=zRQj5~%1l-`lypW)!d#yH-Lg;9%Dmk4a`$BS#&yPo
zGHVB>xNMv;{a4e~julyrF1~wy9k?A^`l<A#+pi-lJFg4%?)%t!VAtmdyN*1I>&l&W
z|4pTN#)LUh^A%^$Ic*%u@#(ri%f)-wpH*lp9e2~u`;v4?{^guw207m%b|0L4M!Qz%
zipm6s?oZKj{H;3zJEUznWG^jnepV{Va6<NppS1M>*#mK}+FTs=&bn~?uI5S(&4uf|
z?N|1Tt!(;kYGqO(eeJP@-b7pW{Iuy-Hx4f2@Qkcls5>Y9*B!;+5|4PkpWMckXAjOZ
zTd5r$cWRc9zqMiUlZK2_;Saq6UOw^;JM=g}<>=||t*+d~AJt>iW_8=I^x0o2Sd*Hb
zKl@qRFV53bRlcnL(0@Aq@uSdP6;qdVs@vAYwfO(~e5*I|=6UD2UTh!5U%o8t&V9J7
zOi^<8x_|3r-t6Lvw$}UH{&8K*>+8EcnFIAqlFK_p-%ib0Ub$wi(QM}si<RfQWoM+=
zTsPejv>@<*h#23)O`G<I9^!7VcqyPS`)KK8KO;u1)#a*B)#`VIn&lU5h)%6spK>>)
zWm@iJdxjA8DWR_(oXYX-%8|A3pV!T@;>v-P2D5Fer9D^l+FWa!WMbqZZMiXZmc<dT
z_K2C`44#*Arz*{wa?vtm?wfm4uSSP6ST{H~r1YpwG^t!1|D;zf+hFDVlcGMW5{jfI
zcg~18r5~g?`OM5U7gU7RDh)Lkd20ADy%e{c<~i4AU$R)y6EU@7gUFPV6Mrjdz8CXT
z%@yV8tT^fAZ5JQ3eV^88&6&EU1_{Q`wifcP`&+v-=**e+t%{!`-fO2Osp+v?U@&3e
zby&P!<;Mmg*?0S$-PZ7QoNHTjs^-N60pSHgC6gvxU1zh3VUot>6B9D03Qn7{P=EE3
z-l@?OL%PaTHCR>zX@=z;)Bc^jw0jLthYE|qCC{j*^HWZl87WB36r5P>(a!hsf}*U|
z66f}k7ZdnC-(q~BP`2s@;~NG(#!34pY{`GJuzF&5@0{ExRm!qar*?X@>s?ap>@yOb
zS>W-}HOu(Zl!;-zQ{tw^`WWbcDxA2yTPHqci_%rG%#|L8Ln_-&Y;l>UI7xou&2;WG
zZnbckbkzsTUj}UI;XASCzLvYm1GfuXvj1s|xyh=oaoM0VPio;!n*(bzSDxFq`_!|A
z_iyh``}JANzviF8j&sM`Y<=ou?iy{-d?6tCBXqfU3D-N38jZdw=lV9QGAr4%AAWsj
z{h6cNI;Q{n9c8x6*-KP8WC`=}Cg-b7%}bo~N-TFwG4t_yTqAnq;R%Dp34v?3`H4^0
z3uk&N>!lUaUbr@4snbhY-6sYc3vHjRlx>gOb4BRR1)p2XMK@g$z7o(b^(w&m^OvV#
zeh$jJl^<?o>saX-6m1&oEP2)?JX=_ATDoTy58ukeilr7yDlQilrd@IP^~NPvC1I({
za+l-&(+j`5t=adibbaOBud_PsW1nPCztOQ+poqij-#Web4^O}LNsga#qB{S|lEBGs
zw{My%{Cy#_qN>d2y7jix#S&Ssjly>7_(<1Yb}Lo#KE8<6>{!x??v#Sby*g?yo0Qrn
z99t1$@a1}q!1U}3Eizg;^OR%D>iAcyS(v{me=BXYY4!C^wO1zl7d>WLVYfpl_|mr*
z*RwD8o^VuPKls46|I|*2CoG$6t|ULKE9}_L_tEUD#F`1`ZofFc-N8+zR?mJd&r$88
zvM<^9_SwD3$UiIB^Zk>|^<#%_-8;H1wsyJZlT|@sg3WgWHosnV^LXL*k~GCbulE(U
zh>PBc{NYh}H?ksv!BKC;*&dhGMr-FE?%HQoZ)*9lI8J!_jiah+FFDjJ=EfJ-_h?^V
zrhR=`xAU7v0?w(vcRsr?otV6nYu>6GA(p{UeWn^m&O3a1!tL5@xtyt=JEl%PaqM13
znm~2HjkV{xQx>27`Qg=jZP}kcRaQz}ub3`w*&S5-;eGYnf{0ZT?HqYi_lN#lwnuGl
z$h`jn0?v_rdz8NHX_T>*l8yeVRQZ11_w3D!g=U_9`1p(wuk4PQE(zAwIqB_h-|&8$
z?OeX>>raLDOA;^Zw>;f8EsJ+<ec4XMvUV@CG6ApS&h7iEeyf$v2<7dOl3>c5k};WG
zGV`cob9Um6ESLGkKQ8gb-+Ji%d@5hL+r02#(`e23n@dlx%<8jfTNb?N@)5VjvtDm1
zl6xmbdrH(^&PbYj;ciCKY?Fne^(lMSpDmucrDWsBW8OWd#E$)%xOjHlzB^~%D$cjq
zBD-2+fAD09D+({(#=SB7VJ0o_dx72Ytdz>Sa~CGNwH|ycy1co3y6SZ1IkOh{Y`wOg
z#bIk=#b!mb?3N2t+HK~x925NbX4`owo;y3@9%?=ke3_S@5ux$8uXf73r9w*zW0W_W
z3GTFbvixrA>e~fV_|F*H+}ZSbtL=T}WbRgtzMb94aVsC4zi4~vXXL%FikB@!(;O_P
zxZiPgE1!7rGT)PjE4zHA+H1<M-+8v^MP~ZfklJalCUc(JydXe_#s3dzPfMJlRZLZm
zZq-Z9RKdcHxAcFDxJv%GQRv7zXKn8xk-K+Y%gdLBoIN*LbyMiSu%C1G%$qy4D)4C`
zpSJqiw|`&M9aqq|l{oTLMyjT=Dg5T1#@CYWJDj`AGd{*`sc08_f8mjA)E+S3@5b4M
zN2@Qw`Ehz086Wv{?|}uqyVQI*+U4F~c)VBV!mJqcSK5(p70f53yQ+S;YPRa$imJs8
zVfhOJofg09og@{?58CTewO;i-W6{NwW}zlE7FG~+F<ltU<S3pgU-5;n|K7uk)-o~e
zZu$%ptWWG^S$;@MD?wzkLz1<~FApu&pHsHH{ZbJhvVk}Dz=o}UjBmTXW&O@Fc-m67
zq~P1b%=5e-CEJ>KR+<!;m#^*NpW8mm{`#Vb(~T>;&iNgUPzg&*y=uNVDZO$r6Su`p
z_SNs>O4=&Ij}?nw{$EoiYa@HcCdp&nDuu+FtLHsbSI0_v9ywq*U;T1!P>N>Fv$;I!
zt9DJ(oUk*t@YeM3N$)y?qjlqUwLZ?8lD}@U`C8#q@>c!_1D$V`Jm}iL^9EyNa?L%H
zgoiioe=$+{$?JSL<J1!V^gD9JHc!)6tFN8+w@-6gv&(F;nAu`;e$2TpwD<a?Id{31
z-1P8bbWiv6Exi)r?pu0hOV?@3-Qt^iznouFAf+l*n;??G+wka;gzA~n;}1WTyJWq4
z{q|Ls==8;GE$5b8@7Qn2az{fd+x`5(hBHzYrUh9Wv>M!b=P=D-l49NR>IhrX&gKU<
zJm#04F<g{V8T<aV$NNalj#oNMVi>-CFa5yee0jm{t{wf7+n=Ti@E@FbEO=Wp_$aNA
zg;y821+F*DNWO5)PrABu-J`dc>Sh04zgWL_+G1H&j|koM4CM{&4Z-g__q<`QRsXiB
zd+(bh{oOZC>wWuFSs0t*re1pHXjIw8Y3IAa`;$POu{fE6eJ!9-lFcm5{mGnZoNAhr
z^o%xXzhBYa@<vH<x5o)Rp-+`FZ8YZS$~=zIc^sqG>vE~XYR=<5mt5V{!gy^xj``gR
zXit7q(DRvV!NpLi%n99}xfD!vo264(cA8E&!8MD4Pbpi1S7uX#)trZWn4U0fWo~f0
zD&@etfGvP|1>=<!k<qLSu<<<R0}KXipdpf@4B##pLm5Lg!v)7TAsay*G6p`zWplc@
z-UKwph9roK-f(3t3q&wi9MIF+(9QK`5z}r2(>f&KxVj>`IFbP|JTUdnAk$>EHaN4E
ztvJ9ZdSfAzbx49flplBQUgvB<r&-RNWf})L6BMLRT-(W%!{B_yXoaxbg77&%|1_?$
zf}U>a;CJ2gfx0ovl!SJnHvvtuUIl4aqw;fgWLX_VrBC!~u^pIdsKdb5pv=HAp;-$w
zD#QS4pL~0DgfD6ND%e0bqfAK>*S;MG*Rf31%i7Q_^=1)U@vhf5e@*$n)P+&6Z$r28
z8?TU;i#T>`9MbFB(5*CwpDUk}o$t8N87S<@F;`t;qu&HNmKcI7JNw!Wi5Fis+<W9s
ziJ|%4L%MH-zlJUo?>W1NE&NSYP=a)&TkoA_DfZPKTn0v;xfU>HFkEvw$gm}nK|w0C
zghAF}k@KW_jv1A287w;Z1pIVQ+P8SO7t9QAP>|O?IoWwf3CDE4hnEy%tXfX*VLFj0
zp!4O_|B$-nt#|aF)G2;{y1ZR&CTKWc6gG$e9nQCS;w$rsmG92APxpC${%+e-R+OZ@
zeNp$^7eA7w1YMjObkWt$Mt?GUt8ptRr84_gbEbh(>SUX#{gHDV#nUfre-azxct5}F
zr||zOL1`rwmzcH{To-4>)<rd$x=%c(6%ov?DR^@OSIC1~=8puz7`Y$up0K=P*M94H
z!0%673UXf_l}Z20J7FPuw0YxmgKwX>3M{u4u9cdXDA>DKX2;VFy8Xx9?kV1ht&6HC
z;k>XSY4r>#hqW@h8O#_SOMcY4cSrP~Ry$kGPVEHN_;U|WKiz7zvH8ZzBzABTKK~)G
zwO44tBM;Hy;0Jz1>x*a2-o#{<%9?YE>jvM(lG!ivd`q@3+tT1`vVGa(>9aR6nr*x#
zbzIMUDWi>k)6AwwC_LFB;hV`YJ0jb6&DLc)4Zb;B)wV?iiyioUP<M{*9MJTQuEaK@
zoP!;KXDu0dr$sd{y_T@jG$N~4wKUE;eBr6j)9rOYqb5`R&N7%V=rR}x<QA}AU})Jn
zv*^CUeA5%@Yd8IS${%tOeBvMb3dSoh_K5uX!wAo$%o}pSo9P%6pd*=Qm>3LKK>f&2
z#$YWpz3INf+(Mq{Q<dyHm~B|*6zPA8G?x6~8~JzEiFDs5o7Fx=3jbW=_>V*KS8vQN
zU!5(>-qiW3Y*7=666Rdn*>cUr$yz?<Q1l^~k0YBRyV+YegU4w>W3?bW`OO(Mv8dp@
zy=<Io9i6f#T#!=P!p6B)^354GZr$|;R((fB9hfgLm{fT+@pJ302W>C|C1NXEwS(=g
z(_oti8)}#@tcU^?J`7n5rVUaKxnc|*y{lLoK%Ntwz$|d-t2abH8+6{uoWZJNu2va%
zw2SY7K*Fir*|*(<xz}obj)gfsTWO)`#0%%xgU=mx%05+d-NgwavaY!2$pOpxAfH7a
zQoeXp=ne?ZveDS13W>HE;K-YC=ktXU1E1h$yO*6Q?}Z}||6-^6yivi;*RCuuHO=tt
z?J4i=Dd+tbw7RF9_g9ee<Jji4R}!>MZ+Jmu)*9V2z2V7QCZxV<iC=MpYy$fYcgBVn
zAsZQQFyvg>z<7lrt7HMoeZd-|`=%@GdHK_oKgU846H7-<_=G!$&tw`Dzj~9{H{)9A
z)2~mbR_6%tJ*s&ZQ(Ba^wfIBsKf4E?WqBKFdUa$cWte@?<o9rwW$=1;{O-xM(<IKy
zZ&`Pg`N{#lb+czAwVrviR>oIyujB-!<uM1A<d+6<uJZi0Zf)DmmL;3z=WHqE`Bq%k
z8h>K%vz+baL7(r`8XOflX=e~>xjL-6?vwT0qt}{^wUPps%1)ck!0|GhTkGBXQ+K-T
zFBO)B^!9wU$X|GO;)ccFOp_*U@(Sc_Q%afqMOt?2;<L*yW%h~0<y_Qme^a@3mSVtO
zQHIdn{{oZTop!5)oVmohxn%Ea3uTEV+*z95US$tf7~kIAxBl`z`(G-rbXoEhJ2?X6
zJ{{D3`FhHoUM3Sa7u{GH#t??|uhxs^#$4=L-%@rx_ZzFJ2B&VZMC&8&Cugqxo5U(w
zvbi%s+iF^a(0iVvTYY(Xo;l84u`|-o{88u3EO*lop()EQM6PB&(0|g;I*8wNZ@%eE
zg%B3W(xA<j3<VX+#rw2Z_9xeMearY$8*h5yD#M8@@4D_?b1S^-kgl7xMPp~g@s!Xj
z^UmuO3O!i7eR|d+NpYq;x4IPvm$A+2ioJNxIn#3$%ie_w99)5_A|KCeuLxVfr+G^>
zYSFXVv)=#KRdXw6d+acO0=xIAvc)-1zhu0SJNIbi<BO$h^=$rVZQ%P;a8BBUZ<pVm
zg%2x_d`&+h@a?GGz0!pAd-nHBGggH!s=L$@&)z#XKf&NlZ&LB;&5wMS6zD!QEvgNx
z({$l*mCKoEy6E|_*=Ad<QzKQB@|T*JPqEr2|D;xjZ4ZlQ)^Dy;)w@|=)Yu(&a!!9F
zvi{y-$CnmDUYlgCgP)p9-C)>vDLptrI=tHN=Bb5O?*%nPE2f|H;g$W@$~9}!=9}-c
z<hmkFChat9{WqIqy6^KkQ`N%UN1<k_g}G<$Ois3+HEE~WN441?QWZpARGFPIsW)Yv
zug%G2DeHVs#>~#>_uX=QS<1SCm(i-lxfi6RzZ_%RyK&-~tq1dRREl#|igTx)*$U#u
z<!lg_esh#98YJl$mNWIt)`MXnB^!jLPsT_;Il^YS(a{eiJQ1v6;+d_ERyh;+q?a6a
zGrAsjG6tkd<uKdH4UR@2Z4kpn=j1^c5VC+3r0Qf0$Sn|&07i%iganBje3k+m2J(;K
zN01K#)1|D>fVppHb(Hzq>;n1i&rY>}G0HQuh4a^SnqQxCW^1QBh}4{!EiAik;+d@=
z#@n@;GqZz@t}inY@qM#Rt!#GD%k$Ct=`kx0vZ|;oFPgF`KW?{OirBm=TNysvsHX?k
zsJ5veZ9n^lL14d-Zk@;BtlwuVv^C{KPwmO;(pviVOyD!ourq8t-)6fUJ|tRsnUn9D
z<VrWcB6YVNSCg%#o4?O~|2k*(nWGNO9imnnojy)5iz!^O%j=BCCAQtlb3N@hl|Rk7
z{JA;%s2#hlk1dC8-d>xhGXhn17FCC=`!jjv-it?{Y!qNxxcKwO!#k|5ADj9tdhVp^
zfL^bf!hnApcTBq!pVMr~uwu<0PvQ80BeN}{r>;?WyYE<an|Ep)L&(Xd?Byy}y;r2;
zR$BO&p1pkK`OG)mzq-H7J+?V-$t#2HQ)DOcs_&Lq*4TMUWA~=+EeB31se9`#6LV}}
z;|)4lpc3x3OICSzU+A|vUP6<OJ5Alp5tZ_dF|%|@<jubL-9qYt50B;jX5%$#WKYQs
zdUK+$_(bE@ReXNBN?jKgcs?_pG<~}Kcc%?)9p~(-*A#2b_iXmR`5-t=*rF_amDKA6
z8TWQB{#y|~Pg&XhSE<V1z>m+_E^cz%k|Uri?Q;6_EzO;m)%UWjy!?9Zz89b-@7A05
zO82!*TBsMNyfSjhYv$Pg4EYy-`g~VdnEGYC^?NvBx}xf|$&ErX&z-9*oy~&Jan5y|
zcvIIpj>$8~PwZ4fjC$m(vek3;uaXOXntkcQzrV|TPWb*gv2071@5~9aC$*&XE`3)N
zo+g~Vv2*K<BT+f0I_|jm<tV9}O%#q!kIm{TnH**q+}jsz+CAll)jY<Y<$e2Js{{*N
zUD;VOW51BntzO%>sOX%j{!)P^pB-OpIOwPH^hjw%LY{om#3P1ZxpY6-T`IDZVz^SK
zFXrUz6twzP-W!+4CrYhX$$ql%N}Qyiz@3nPW#KCC#BV_}E=WmlIkqRL`4msKNCLxc
zrUYYYsS{5cCs;MDy^)~(Of=y+Z;bE>hK}anOZpP@&2Dt_eoNA}%9*(76aSPe8k#RR
zPuMJ#bF6zq)4TiA``0f%?3r_Yf#tKQlO9)i=seLiJs4Reyn{DOsQ2D=E~i_|oiE5{
zEW9Jty+C)bQ>J`IAMdQkrg`ciw-lE<esc`Cx?t7ZRl7F7?z4NvF^6|sjY(Yc@~pX&
ztX02PhX2&y-f~3ZmJbupR0|XSC00Q}*VUp@gmpJ<^bw6+#(Ob!R>j1b2iWIVUS5B2
zebEoUgMJ71xm*&B-_%>K%Kz-af|a19-&$?G+4>Xrv;Ph~s-OKR(Cnke?20M54lhOL
zf2t|ttmg!6N1k8&X8VI}54Js6l6%3(If^ZpY5t~z%X2<xO4ppqdE*!_<1~3Y%ly#3
z%B<>;{1zFW<5~^1rt1F_*d5l~w4CR)i^2TKs;zeu?78O|D=(SzDC^3Z6_HD4n`Why
z%-O<rZsyUjKC4G74_-N=ZTjJuDZ__z{0i0$J3G@hr-~;zYr9MGoGa7hmYBoKmaeTD
z*u=JZ?ZJ#}154HoKN?rDhw?Wpep11up`oyfld-5lW{)19*|mT$-%s%Y0qfODsy&__
ziRSpKd;j;ra<6wLYn59wtMW92c)2oGa~O(TU^8CS$#lzwF*o?+{v!Qj7tVjKRn6fK
zbegTP(09s(Hb%q3KOMqVaZT#6g()W5(yyLBSU;_P>TTDUi(!!*M<sJl&8e)5)yiMF
zXX&Gq$PVjUM`Uv!^_dmA&)%`v_s&KA>AEw{e7_J;mgKo1EZJ4{T}E=_l09d7gRe;6
zxyV+&<)CbCfjgLKvfioLixDiM!jVznCOzlk9;OV&hQ^4>O}wDyqsr<DZgIL&X@Qy#
zddw^q^PbCSHoo<sXWn(6&BwgHTWB4z+4@A~=E^;XSPpd*o@$Sh;n}fltzTgN1m9@h
zlPCYpew5_%Ch&UHw-pj!*DU1vxy8{gcY?q4lgnYwTb4aP`Nmq?jTa%v_H)Zsjx`g_
zv%^|zZYj(+JF%SiQ|9VOkx!Yyakq@`8Hz+mb}i;<4Qy-DaNan1<%E+VM^c0|W)|i7
zJ@h#UgE|L0x)m>F3cuXaS#rz8DpzT)*;2{eM{}MvgU=bc*>v#gF4n^f-yE@yKVr>U
zE_7uJSCP=2EnIm*S4@&-NgSBq8(qnzFx%|JGTu$4&Dpme%-Uvkc#or5?gZax-8YJ}
zw?*y0UL><FlSx3!#HlrGc73Oa)I=v`rwkXC-UfMQ)=insIk&+2D_K`CRUi30;d2zP
zGf2C1%H=&w5~2d{7z9=rGnIxF?&>-(IDNCW9=pOrizhOhUW!VtVTf}*o~6rueM@I{
zmDQ8ZeZB4?E-9DQHf6T$G@EGQq~m+-ymvIaHru(96IQRdmN*ozOcA}_IB#a0!grZj
zbDCBB{TlAPPJfeLrnKqIQpra$I~}Su|DHaW=6L9O_rjewy-%b)Yw&aOF)Er~aNJ+R
z{rQ?3m+Er(RXR3y&evAu*!?f$X_xXc(-%6+PqjL2*G~_9qBB2cVlL}5{=bQ<M32ZC
zmra{lopChB{9hoCdtmkKOPywh)w5^p^*wV>EiGSo^Y%~^mn-*{<t%^;bl$vu<V|i0
zWA37FW~sk(lD;8pSY6yJedXSsOIKT7zgdxLQRsT8s4V2c3cvM=2I(x%dK~5mR&MUv
zVH$KX<jL7JkItlaCEK1e+N-%S%-Q8pkH8&o7Ph3fKNDu&yV7xCPNl2LO2<B~bpDWL
z<vR~Jo==@6IK{ZBz)fJ?oqbR0=FYfM&?Y_SAzQk|dIsSs#&7E0@@7Isb{v#Er`^Dq
z5Tef3;TQ@UcVTz~>c$yXywtTg%zLgtyIJB~LEL=Cx4pL>v3wK=w@Be=-*IT6?y1U-
zU28rs+>pe#uG!^r@aNJFYnexU?K=)DzAMmBPV2CenWG>F6M)DWCA6(Le30{z;=2Nf
z07UV{R1C#lIVU?mY6`QTJtHVq(9Zm>;D9XOj>C-Q7AIy2tqEmScp{U?SZ?uvkNwWW
z2J1QD47m&m3b!6{DMZ>AH?Z7!sBkdULDsL~R^O|Ti;P7LQVzTdD~vV&b2iMkWOqnd
z&vqw~kE`t5hieUs7?dtP5@L^e)Zi^s(8;V<ctDkJ#}UTq7F|&OyA5;SXK#M+vsR${
zg$P6VqtrK&rr&y`X1n#qE<Uy`F!f5-5(Uc#t5gG%G%guhMP4$?Q<-(3E!fZ`;h=1V
z_?Fm-X5adjY;@f5*zWZc!{qf@e0L{Q`+Cm1v3J$y-H}h&OfoLr2)8(`J^P7PSj4i}
zl`$&ELz$NbTobA};Z?BhAfIKlF+;_>oU(04t!^i|p5j=<D=I1D!uBvpX35S|my>Ja
zAMV`Tv@+M_0-xX(XO_1Dj@B+0_%0V&PmwcmIm0Ko+4;yDle}MRQ(Z3D%{RI9(eI%i
z2p-e}L$BK%D`k_i+Ap=Xb=*>1RMxRVRw;}9_=ffww-lF^X~bqg1YF)WteCsic8<JM
zxXyOw^WPnl`%jh@-er*JNQ+?v&EYX$0FMlX)yaWR<dJn?zwpv5Q*y>F#^B$3%s`>P
zfGvQV;neB-3|U)5)#h*E0vRbQuvJ*WfoGK%19;?1g5eu;1D_Jx&cyKOn-?L2U(+v?
z3bYFtPS{w#G21YEW76r4qh-36LzihydnIz_NRJqgdthc*=&lO^yDmU{4h@9gH3gjM
zoavy^A?`FVbSu(p1N&%!tWrih$1O%jYlV#GUJnnj*>5=5|KQfnq~G$BwYU9qH1|B0
znDJcHfq4SUqDjRdh0&lfFBRc!Ph^_P;y?!u<xQSrUBqW-t&qlk<W0j2S%J;WNo7m|
zpQp@mKEu~>lMya67c_vm6}&!?`2fQPd4>`J>rim!0*@xdvobsqNIJE%Pl<miKMw@S
zy2>r#<yi_wtVzzCc}{D#29zBLC_4aO#`?0-i|f#ViO-@c{aSV7PfgxxDHOoHb4BRl
zjN5K}S{nnpBNq!W9{(&8<+@onK6--F=h$_D)-C&+Eh`*ZY`T8eZ0I{KxzW|)ROh#b
zoLBF6h4RjNnHF;DcUx*qPlNL^E|z_p7kbBTzf#I2D$VmJ>g}7ZgcHHxyFWSc?9&K(
z*tlDP^H0{*TL=8gDlVy&bydC<NnXEtb287ZQptpL?iGF^rVD;<Xw7lXPgItCzD=vs
zU*maFhquHdp|(R074;H3yd_+}@P2(*c{l6fGR5Nuudds^VE5Kf4-+(%A{mV>Kh!-h
ze_qXSWhdXM`&m;Ty_cx`dEm)0)_~{bS!{>hFTQP9D&ex}6Qk5N&V>@D23ZNA%!`+t
zWm<lY^;Czx&asB05*s9##geytTIX<xvq+aM=UBs8i4Bs>X33un=J@6CZwqqR$N8ey
zQR`^K-4mI&!x_LA@m-vf^dZ`H*D9A^N~;d|@t&H)>&4^6GfT)sRYOBVL(H^8Yavr;
zC{zBzV29-mevGyZFQ#13TzGzWms{Vw1~JJCO3orPebk;!I<t_YDl6invhz<4=a3l!
zwv%dJc3I7=D~nB-WGSfq(ElK)Dd_dMOJDMlciWFiiu+O)?3-fuVPc)p=QV8=lNIl&
z+`MmeNcD#54c!~MH@@`Djc?uEx|<bzzY^<Xne54ot0%`4a9pq`3{93k7I)wVw?uWN
z=#EMAp72lBX5BD<cT<VfqJsjwIujGtOGrN8v3pVaBv1C`R@d*BJ)XqM@QCCli%owa
zciG@aQ%TcLhEs=Dyu9>?`*xc~n1B-ZCmy!VC)GS;pGfye-C_*3JYg&$@T$qCnQ^|}
zgUhThS()WxnM|s-Glop*XA`)n&zpFOlkLlP77l^tRjduAM{bs$5pR&;N-woqSE#dY
z;`;L*htr=tH~#clvhst8QQ}tDrMx^DIgYmuH0%}g5<l_KNJi_hWMt)}YnQIi72kAY
zqHW)mVwG;?sWO_6<DTRwr%H3)II-Prm+7Xp6P<gfWa%3%y}EL5lcKrp@eEtPMO<nZ
z&w0Gg_cKfmd-lfIPv`L*gFep35e7dOF!x2o&CPx+6=Qw<V5!)FN9>AH>3Ndrt&y^Q
zq3tf$E{5eS5mdHcP{bOtZV%H72CD`>g`Xu8i#1}Vrt2kSvHGlSSUEMk;f8kGobo3!
zX<>1USy$FD-eBlqZeTE65d|LcEWMU5%Xh|e&k0MLm3x>#1NNZdz*^|+gayltqCRsE
zlNAEIuTrudeVPxjN%Fjt@G<C!{jleO%ib<l*4Lf}!3LnU)ZPX;pQp^}z9zEB)G?iF
zX*>7EnNB&&1ZHa%J8`fxseEzvXp1%wVDMyg)l+%0CvAce(?a8?raQiUKN_d?Y!SnP
z_p(k?=I}p}nHB_EcH4Vy(i~yulb|K3%r6)woxJbC-3yYv6zE`eO4~u=@ucD%u7T?A
z;>?EYL4%#9*$ie3g$YW}Cv<({N&tI}F#&Yg8f0xPc%J1OqXT%|+XI0I7nWF?Eay8r
z$GL3olQ{w%I~)}C1R|B4LKbXV;=<^)+S?*v#RE&n;-<SSx*YE=XkF%1<}{m-5OH*a
zPGnQ!#zT{y91Ut|WqZvxO>~V_gz<)y1jmLs&L257?5-;<7c&p2Vd^@+wQZ*A^Q{$)
zkA%V8)DKf)jVFGNEQ(|ET^o3+4l3>+_tNC>mMIX;hU<IPCzUJJGk$!Q^x%-o<ow-F
zW==XG@`xujUBlY>=Tp;Zw~Id-MV`6R(C9sd@q?5{$s*%xb2q*-y_WskF!^5g8S_~}
z$A51;(I_l3ne{-${o~aZB5|{Vc0Rr+=(SJ!F&FO@Jw^9OjcX5d-X2e8jjwIEw!*k-
z&6?-mWL`fn=h9LYYPNHjZE@?!MKw{r!YQ4LePmJ}X-uBpSF-ZQ#AoL2n=O{U@OtjH
zRw7%cL}l&enRCy*Y4N?_)@N3<W#ads8M2b^x5$2dvUDEX0p^2JclxEOB)dGScn*c$
z3kW~PvC4DzdG(wQVFumQ&9efsLvD55GZLBpY47IV=-P&qUM43~w{Owc{Cn03EY?2q
zqHFTIt)FLwD!Axad1&+|bX*ioDruF7ecl{w5p3~B4z%$BGN67jD0s>`fu-6mFS(Wq
z@)+)N0vllQ!_ws?SKuzEDeD4$9RQoiB@r9QZqGhtUBe~OfL)(%?G(~U-n{3_lT{ZC
zU}9XLL3OFUs;-<%wHK7IPFc5yDI%SzFs}t!)wyiQ2s<xyy3g$_=psM{1D*y3LHGGn
zz`7(D*cvv%SLlE;l<WaJZiXX%qU{X9=eAC^JMw}H?9>}4c5a*AUe@1!iM{O-d)u}I
z2xA_<-$S2=KA`jHpm>i(MJ}WXzzJ^xObYBs5mAzC^>9-JGv6?F#TN0M)KHLQRaxAS
zA~K<eX;NTZIAcRZr529^U%IhThQ~ScqJ0xTdr3=7X#d2OAiL%j!?lgEbDWaqcs0&B
z%*kcp$f*7T+Q@;NI~Wr37s-dPQiCypK|;ELFKKxfq-g`nUYi*l-h>o#e3<fCk1If&
zbwxMR6<5Yr?N9b}|Gqp`Z$tN|%dS6PF8WgvShq{V9ukqTg2UV3iH)b_doS@DqBCAB
z`craLe%B%9*eUb%PWh|1)`#yppm)k&{ONL-5pyiP_#S$kvksZhcXhe**OvjcRxqY3
z3J)eBj;;(W5eL%*=E01r3WS?>QoDio061nL{!TKLycTk*vi;^{wCTQd#)ca!9|<I!
z+Ifk&Y_9KZSG$)=^Glr19kqLzG`|EcFwuY433XJTPw1YOpz_>{`-ZF8OQqQ*6Mc7`
zP>W5OtfcnljuBg=7XQh{kI`A_+23F6+-+p;?;CSLX?9ul=ErXmKJz|2yu?Q~W?{(F
ze-^$nZ?2yVc=hhU;rgbX%QsJUVQ+9=>bvM*?y4uJ6~0R5*fpQqewDd8Fe?51R;}>c
zx^ew;qN4gkBqm8{u^gz@zuUZY>e0ijCMO!c&uk50k6JLLS<vk8)II5%+{vZ^Ud+$@
z+IL=l%QeBPQX=jjvs;wz;?Ml67CGyD>5j1YX>DpdAv9Ck`L@H)lhRjNQdth{ow9%Z
zci+M@F)7D?FMBWR@`{hCggyIO^N*;N`W7yc%{F1DBaTe7GrqQN>2(G1r=Z6770>&5
zF)v+~tMIQ=&8cvld`)bEfmMp~C->}w{`XC|GDI%C;Fe+NI)44ozQayRfseP=y`M1A
zDrE_uq~`1k8k&4^DUn+{?y#$}`*L%2y*jdhW!l7`o(+?&-l%+glKnS~b?Wsc!A{Ft
z{5a>Hdo6Myd+x0p^X*qUE??po+~(!cs(CF)JbPvL)+<M&UiBRcsXM6$65wP?J1VvN
zhIj0nrSB$ia+sA}oT1gpBOPz~z{QX2PllG>XaBN`vv<Aljs3DL?&RMsp6j}wU76Z$
zG=Iq)pEru(QQ~)3cE4j0Kc?NDBBXXvD*9=vQS(ySO|dmA;&y8VOj+bw`sIP=)R~W3
zid?QNtKHEo^W9%{=I!4Xn>Sl=M-}A?YsTM={1zA+XSeIv_fI{th30QRbv!upT40@3
z%WA%dtEN|{PTN+vh4;v6zK^S%_q-BVZv|p%Ss!?3Ui0?8^6{3<{(o<s56?VzYUf-A
zzs}M*I_hj*o(ImH()Rs$RCM7)qlTGR6VnR6tc%H1b=Yx@ZL&KzgW1L>Zl1l9%#J*C
zdv5BrE>?Z{g_{lRceiSroL|C|C3Hn)h2w=K3lw6!*VoKvzq0O=C+qK@RwC>9j%u$L
zxiD9)#0%V*oc@`M^;V{rzU2;^<S7OrhXNK@Pi@hNkDan!@09o{jrp>(-~Tw@U;c4J
z(d;!JuNQ5781Vhi3~9#QI}&5>Y|zV#uxq{^+wVJ-`@HulCz*}wdeUMg&z|dkFbxC~
z;?5rt-RT>*!+S^c1`w1Ho_FVjo>`E%^vdquD^qW->8`n=C|`QQeec2EHz{VXQ_NmZ
z=(1X}$S-X(U-^p3H*^?zT#P|e#%!R{{+14dK(Cfn17E`2rnd}M4fhxe1Y{LJ6U7Wl
zrHkC+l=c1d_*N-$7A;<Ngsn8f^RS0(#QevxHxs5_P+_~hl6~6Y3G=^dY!<GsX%D{g
zP$X4YDpW74F5u~cQ;xU0UkY9^U}I1{JoDh@!Vhunx5Ayb8K!uy3|m?DsiIJPqerqt
zP+)t=6-Cj~qBXTU<^H@c-BbT>|NaMUDo1?3ML*eEsJi#^ix~FnCrh7xE#0;9bNg%g
z2H^vo4_FGAEZ7-@kL=vR6p?9jlVgR+KKAJ2d~4OzxF=Xm3=3xvZ{R+_`+%*$nqjwX
zP-N!v_H_(-4CW2hlWSZW^kuEeJr;(C3HcT7=?gidefxFa$x=~<j<N#Q4Smff3~MU_
z{yuZ|-6Z(B{p-GSHD@;mFo2dwuum{fiogD{-%k|OA!f{BFk|pE_<hOq=JDO1nqnoR
z*6Fr_4}=MvD{-E8e*2o}9r>j-DXA-WIxFqyVP$~V6w@<iFDcDvf9bs`|HMAU-)(QC
zL5#hszf|At6k;#53TIFU`*F>Whgu8*mWzWx(u%)#Fn!{BAjO~~ARZdSkjG$Y5V0x$
zNGH!lbKT`%nM3a|<S}e#Q2iYx`r1p~>l^P)^OXA&YMsBm_46xX$S7F^;X#Gh2A0OM
zGAs&qMRf07P9mJ5_<Pw>zMI;t7aaC%yT}CE&IRhwgU6FZnGZ00;9_TzV1O)Y>5@GV
zx2Si@vNzUQ_XTUGyF1wPtegD$(XRX3yYvgczn(92>)G@NT%hPW;g;9APx+Vn0j9#n
zT?{b{df+J9%EfTRSJb>g+My`H`<v|6^GXSu{Qd0%j-J(f@%TYj%-rn9An(ZDRF8_c
zECp@9%V3b2t8ueD<^IH4_iwT{(`BQVw>@7N4)P&H<ooy1mUYZ87=CFr-1_M&3Ob;N
zErEFhqsOlW(?PoG<D*Z4N5yy;7F!uQ@T{t6-J{LS0PexB_j{vIcJH%;G5@q*n!kB(
z=CAnM_TstZ=gIP7*Z=N2J1gj;>btie^%-U;oI5BAPKJy-7-IMiFck(KWqrU@0OB#c
zV{YJmr1zbzK^mL_nNRHd7yk7B^f%@y^G$Q&q_0=Rdw%~6@|E<a`xE}Qy)jSOADgMG
z@{9MPJtL2cG-L<{6f;(Va0QAG$CSODQ|b8)q_38zHpu^;r$9m9gl}vN0-)7K;X>Tt
zvyfK_>(62j1#h8r*b^c2x9!DtqwmZ&`BUsC{B`)oe6u}a-u0>BXZ{*JJ-tR#RGxq5
z`g(?Qj0ac>*OkvbHSs#@1EvjrZzCAiF(fqZSfO;1UG;b48}?1|EB-osS$qa;0)NVU
ziPKXrD+T;+eBnOxcjKGmzpacPa;%&C8gv{m6T?M5c<BO)EV#}5DfVzJ?3?6I)G2_B
z&axNyt5FPg+~Znx2F4Ab{@JS|42CO`pq&^7iL0HANcktZfnQR2#&6l1?@!dZf3x0n
zf8yW1TkEf8>lg1o1B#>fe?Rgc;Cv9n%J3*Db`LbM$?Q|S_&)VrdCGoexX!Ho%D?uW
z`P=uVJVicTlKo;z#V3m$QsJP0p9mI(sC|FRQX8LMh^iBH?r+Oo_D|mla?Tr=JuCk9
zz4&hQ{rB%vqPG}8Wd<nitrBKFvY?AGfuUmJHwGTW4oLlrX=@EO%w{V-q47&X#i5xs
z;{pS(+=7G5d&KW-{ZYWME>ctO>Y9jc61}^aD(0#eGcaggNMM!P;?OLf@qvjqX~99B
zEuk_tCmPwTFDzh}`s2_npCQ1)+qU5CqX&*cCJ{|+z84m9TIo2ptG?L4B0KHD>4ltD
zCI^4}%sndV5YfcveIb!^pUkWU&cY=bO?<u=65(p&`M^3&9NX1iY+%uT?LPPK+YPK!
zZ$AFRxNCv4c*%<<zSs+i+*Ub`?YbO~%LV^O8Ps)#KNmf)LV&d)a4tvehqSLu6@Qrp
zwk<d;wq^$V&dXv8TUy`br_?L|mcDs^iS*5mf75oZmtPQhkk{ly<M*cBV)CgSqV8{{
z771*Bz#*mL*sPjyfrWS80!NV=$q>h8?Ti9e-c1V*ikZA<VvE0^z_X9%)PjSOCKk<X
znHLgxrM5UWn`i9qs0(1%?=m^#*lzj4fL-?4g2OT;Gn$t?xpm^7+qdt#{dN@1Xyz;a
zU{%fi?7``Ud{$Rb74j7a#C`wMsC5XWYtrA|H~BMIq%Yn#`u>$Uw8iO#0f#JGqU!Hb
zi59-*2@&}hea$xQp9oTw@_!GL#MRCKj{Kd^<$0Xi9V?E0O%SNk_muh3!uI;{!6?R6
z3l55!oM>W;zR;zY@qsmef0pIUtb=#u*lv8BF|~_>J^z9NpVS%0W~+<>cHU154$7@j
zzU-p^>eIW|TnZvi&CVGUIC#YtiuPV8Yt37s<m1%rl~KUSYqap7+Lt6@s~o3x-xnJ=
zWz7~IRx7#Da(!drQ4q~H`-4?6KU9!!_JwX_MSQa_B!YD6L*+Gp$KI=rWo!w3v4KlA
zY~iNRlFIL;tO8*R4{Mb~wBGabS}nX#)M|}WKbJf+$LbIFx((&E<lgT)tnnr3oPf;t
zt{tV7-&GGy$ZU&o(nqOWKE1oI$g=16_TrUdx3<2}U_LXUw(|SyH_{9aEq)mvIC-5C
zmueepv~H72S=m<}Zy#{<tHSwG=Wo0>r<ccHU!);=oj+*4<0a#Ws?tMCou_^7mUdUS
zT4J%^qtLZY`(1S5of$J1ZnL~%HrMqsw7P@o4P{U`WKk@U&iD9C;vCB!<MzU50%zhT
z)q>=q)#cN>`!k~V7OgJ<iEaO0>ywh1#-P|&8KUkMsJTMv$j0jbf(_FvH_JcjE5EPU
zV&Rw?>%HaUAO1TFn5SRR_{SY$(9E4S;UJIMjDu{u52uTIX?#2RG-VZQxvm{U3v1ei
z{W)Td&lnhvU9S~WYM8K>XWMkg??x};*v@-Dt1GP8rTn|^jrFGYC-yD=CVO$cPR)ih
zC8<ifX<|J(`fg!{8x;~x-gysI0agmCboc!J#posOqurkQ)OCwilW2>UzS30ZB}E+}
zd=pP823fkS@=DjZSneYKe&34d5J9J_wTCnt1eVL}o3ee8;nqh@qU>)DmcIs@x}w(o
z%g1-G+ZXAI%D-|9-xX5RxjX)UocQGx6#>l|;iu}HL3K~SZ{CZ?cdwUUQ0S1i%lVk7
z9rK4ztJxT?H%YT`JyC7mTe5Uz)4r4s(|MRCpZR+aTEjsru(lVE?_TE*5&SgCG0y0Q
zH(Tw=2l_Yl`glat+7CSmm?6V9`EWv!gb!cy$wY&bci!{f%s&CKbi&`ZTlpP!_iFz#
z2=Z_tsj}ozoB4Y$*m$T?nEO^F1<p*+vpl2Pp7BJWcTb1gy;Drfy99b=7EN?ESvAFo
z@8XH>jFQPuS0v5+GQBc*mE3{X@t%(q4q11U@y7gTHh*zI)r9T($<WiOXKH_lwRCCc
zgc-JcT0Sk`^E#-u)LRE_!>~T!Vi4=-)`GNXAdOCGNUH|i0N5O7#nQ1;{zToL^?P2s
z7l2L_05><=OtWCi7)h%oVRiXi_C@jPM<>=?I63pF%8Npw8CUpTJ*t^@n2kX|fAYRF
zvxB~>e*b0u<bnLB(tT}aY4hd&Oq|{HY13z>uk2zhj1PHU&%0gU(tN&zD-8lU(>R}P
zZP^T6H4a*lyP4&)lJ|cx(Agnp#K3SKXu<fL$+fGr-}E(me80Fuh{^KBqRF+Z<khZ&
zmS}G9V=`grU1!96(F%NO$OVQHhPisyH<$lv1obgMJL)YR-ZCr*+zMGF4qB&~buEms
zK_WyKb~H%J<N6}!ugsx543Uje*LEb%j9}jY-jys0US9k_;K774`L1NobFoG5CKXSg
z^IJMhoyYuy-3Ph2BMqJHzR<~Y(7^y6Fg3fM+);h5XIXx7r!s`{Mv|FWJbjLJ(Yq5n
zm7n)P80dWE=W!_VAcfT+v!LoheCwijsxbXf{)wHW8>DBMKk0MZ9P^WvlR-rF;V3`L
z{QMI+AFARCWR@Y_+yELj@h!Ti@*IVj@L3IH019yeWYm*AiqCm1^G{SdpOY@S=LI4r
zfb4;bYr`es;%niOU`-(L?O?~Dij!rS%5z0a{wFqI>SX1f#HY>yZEHC5-?Fx|TOAfT
z1A#(kgnHm=JkB{6$xZq^VGcWps@S|UkGt*gV$elC-<~)=%z5Z>ju~X)gwGD=m@WG!
zfhqncHV{7l6B{pNd9Z{ALOq!8aBeY@dWc5S4U+l2$NQXh(LL4Yu^{5_*>%DkKQy2J
z<<<Yp>dw3C9K>rPXTYA@<9%-T6C`54<*s0mF0fVvA#3@5(&xT8<xln~KbN(9f1=X;
zT=mIHcaSv`k;GMDlGa7{pyJa}G=atA;F6RXw);tq$2s;QJC*0oAmZ`aby{XK4s2~N
z|I^gWDVqx^Rk%P)`rSbCU_}Um*;0PeXGIY4QBC+isCb!YqaoAxcj_GeCp8MsnJwi{
z{B$_ye)6XS$c_o08{y)rFiG|zJE%Bl-UOiuEFK7#q|C7PCpBK@)Qjv?p9h18&u7;;
zngwi}uJW`->G@~gyG5wpgsK871rx%S^Cx}Q1Q9=<HMD`2gxgjISVUV)`rJ9k|4EI~
zb79N*Cw}UH2p5nw6Jg?;;gWFi^QfA@;@&8l$TzI!zBJ3$4YhZz-6OQ(rzGj-oR8gl
z&gq|B*5uYdvNO2;DHNaDcwXlB8$X3e8Q-3Fv-c#w?|OD`;@VH2#2pH)x*uHQUND1S
zly%*Nh*tJpD~`=HW2`-}<fG9t-sF1*&v=s`8Km*K|F*hy{o=mLPmcc@ewA)nzqnRb
z^`(ZV`}a_#_Um(PHvd>P;k%xa@PY6K=?3rXS1omVijx;eE8mrPc`Ts3ZI|(t(x}RR
z7k02+KUZ2XH>NxAHN%yK+K^KFex>7U$E>Cq`|}QQnL3`+UX?sy2ai*?;2m?Ox%$d)
z7Ju9z!QTxh7Ts1mp!xiI-VfdzY&qOIl9!xy0%O1KHft@+*iyJ+U7^>~!n!-PT5p`#
z#Qw5+UpWyO*BP9@uIQtgVo7YvJSForQk}24>U9g}1b1v-*1Yy*Lblb7{}21Bo4ryl
zX!Qtejys^&yQPwOw)qAq@sw9D|7Myzb9$B3@^@DQcOBF8Jr=2S%#qD)(RY@B=LhHM
zZORt>-^Ci*Goh<5e2D?0-lbW#9z3EK*1iZk|4)Iz<Ij87Kkut!J(s*+e%H_7{p{+V
z2XkfbKk_c=ymoVAeMDn_2g~ESI$_4`_VZU9<d`DaxXHfJr0;$C!Tg1P3Jqgtefu6b
zW#O{Ff0pHUN6X9oD%K6soAYh8sNC`Q;XlsbJ#gh5%h{B+edlK%*PYvQ=I7k&eir}T
zj|x4l7ksLIVt#$#!Ox2;gx}N)zL{??=N0+u75`q7Pv+^4k99KUES<jj%c?CMxAdmf
zAFwLuVVD+h-SY98lY6fI)!BIGHQSn|uxD(lo7{eX%ay-WWc1{FblIO{efMvCl(w!*
zKAyfVT4d|OpKD`3cjWx+)!o1Gsrj})%TkzcI+Ud`u-{-TTPCtz^QdEK0CYq$tL}V(
z(JOPezsFYGm-zbI-I;0sj|2MW?ALSXHp=KKZajH8-cRQ^$Fn<@84P<S2{TmXMy=hz
zZYZm~BH5+1XG5Rhbweg=w}j(it+pL+B@W6e?@Ded@A=Uu_`s0KW}EgK$DZRHpCkh1
zl+Pp|sqAs+7kpu;WUH@!<#9{ZoPd7eSA|Ztek+c5ezBN1Z#s;}`PE{focfi=E>&|p
z{O5%(kXH+5Ko&cH8p>OocuS%!Db`c3uf4`Z)^vfqR{?{{lgBMJb0+i)fBG0U*W<)-
z&Yu>F^6C)5^HZI*Rtc}KEe!d1OP}$9>k?hhDH+ct3TCkLeNM<+W0AP9b(O-ZpNTJO
zUM>vgpZDK#Yk&E_2<5jnH~*ecT;t1YdFRsqMTKkS{Ull5zv=k%@yVr_)VW`lm!FHN
zivIWS(&PVEE#G`@+yAyr-|F4t)i!63PYWvi*Vp*RHBDqn*O#yt*MnpBrOWr1w*~xS
zU2ePQ`Gd;kmo4}9#rI73Q~75{-v73LKZBl^D4g27g!QMT%Kg`mYtt)&dH-GVmj7S&
z?di|c|H`+#_}mu1=V6b@^VfHNa`~^#%JiRoZ_`4X^Rv%>s&n2v>tB&pu&^>~U*Bu~
z<waYUxzGBy$T0o+0%KNNPUiN16IShcEy$D}TW`re>tXougu~Oz{w2N=tl2!7_wV!g
zPZiz8yX^jN_`>chTE+6(f9>|oKi55Nne{$v-~6-XruVPbgn#~<vrpT01-el21|@Zq
z4WeJ?3dh)8^3>ui{XX~e{h#b0EoVEk_O1T8?pgM|D=+0hOv~V1AnmJv)|u^3{XBct
z{VRXU%=WMR>3cRmtIm4%{Y^i8&#F%?u%3PY%AaM=%uU|i{4)(C75w@3O^|@;{*^!9
z=G2{jW)5Wq!wtb=oczr{0iP`yw9Y+ipCzAFx8i4^S^V?HVy7w)%l=BuIj(0vo}7?W
zpMBnO@xo`v!K!B&Uio7&d;R3wPG|W|?Lj)vzl;Y<o_N+i>wMUMi`ihKa$5Q}|FoTb
z|7y*)&TpH4#-4qjWjFi9ubVYbkG;*Zv+ntI^Uu0x<)-uhioX;WP~5rt=Z4G0sduh|
z1^3TC`yL_<VrA8Zf8IUo{`qI$b6~7;)BUSKMy7vuH>+Rq)A_7?`semp{I|<i{LH-D
z)x&OExZ<bebBU~f0iWAv&A+zxhnzt|+pPIvZvs9)<l!-`pK~_cW1@m~VoOXfzv=u9
zR{}mCzxik44d=7+S^q3%+h3^>+vj}t{M6THcKMpl7g_K*=U>R@<+J>oj#Qe}r(In>
z%m3SsO0)V^KZDPzZ@*?U+kW%g;IrzVFWAhszxrq8v&qv_=bfE@^Uum>{-*Ww&d$#Q
z>$X4t>^#IA`|}_PknZ{EpTo`S6YnKlo^v)n>tE34>9h8qd$v9Y#?m*fU-{GfY&=*Q
zlr??U{wsf$K3l)}r}k_Z%NwHm`prL6;jE?4z`9@O{5$t7-*o<F)o=Sxg#6s|@N?UF
z>3w?_P8E0doA9}Q`L3(}&uzX7KL7W!MDJCw;*#AP>Kpdge_VZpCnGjt*`jX=6X#y(
zXT4`wdE#^Z^7ltq+`fO$`FYz${_RhWt=sbNe1n_rsmpouzP{d-r^g?xy(jGJze|tr
zTsgJ=!YR)StA6jhv8*)stK;hLm$|Ih?Q(5&-`k_{rsMT8rv6nmB4sh6{Y>veKgQko
zyXuGN{GCcddpG<J{TTOPuhu>8>KIXZR=ZV?rgP_mSaq!eZi)S?A5H&W6LvH`@%-vX
zn-8Ri)`)%Eqh-fk6Q{gr!}ZXQp$}GT?c=JB6^&=zx9ZVqh|q4WecaXYqW4+%t$y_T
zQ)(Q9wR^+)#ILLR%l=mI{#zV7|IxmU@%DYMFYmEi`}^2x3F~$K;hJSz553Vk_UKp1
z)TGDzK58Ai_VuOK(f7xGg!bH@^#;U>*>}H3`S~IH+#fR&Cp_C~w)fqBuib4MVv}#Y
zbIiG}u`OHTwrSs6xf5l*DZAS?#U|f;cQEIA#<pyW+otp0%3Uby&Dh<xB{uoiJICDX
zE4F1v+%{eJR_;bwZ_e(vZL!I>-yO`o{$g8p#ck7lZ{;2o-MRjJwdHk<?bS1Go1S|g
zk@zVu%2)hk9_PN8)>!6y>jd+^PuQM)=C<jJw^Cor_^Wpx+Z)S#f8E6V?-#abzqxJt
z<E_-+GXDDA$M(l^9nj-V*v_@VoBhU8mN%D}%2qS({?ZWJbwJNIVY}7_Z}A&TdEQ)N
ztIKBIzojw$)&c$82iwg)cuW6S%KK;Dg8bYE(_y@8Y;~DXMM%PT<b%J2o>|KK2WG~7
z<DKzKZJvGf=KZmh?N1PM-D<}DR~q884(OXc*e>;fB;ge<oWE)~qcTsKl=|&Y{5vnv
z_T}a1tlO5$*>$#?iMsMcEbz72@_O|Q?ysg-F1^lv_O<i(?)2H~*K=9K#UH(}XJ1*X
z{QCDwrg8DEe|GQdI#*V^S4qe_*r{Xf-c`Tnr8(TaSyZ*cS~-2ky13|_UzhDuv8;_L
zKli5nSJ<YFN5Wp;y0^`rxoXMn)!l*O*Tv8GJ^pDNS7+9eztdDZchjG$Nnz{tRgI&U
zocdL%`QE)&bpE2xcKp1R|9)ItvSUrt9oAckH|G|Xd0+OApK9{$m(>)VwqJE7#;-2>
z#l~;HAvt?;u>JmLVgGKKI&FIX`d(So3RlN%0sA(*UT<sq^K0o6tNr`zd?)%GDt!1x
zM)2#^rFUOjhW}ghdi~5P9(G&x;!cXy@9&N^I{x=<=IR)>lQz@YCl{)jfBMSHv2R=P
z$1gvMgQsm=73ux`cFN%ts~dm+Nj=&TbARje7r(58-u=>A^y}-?!{^q&ulo7bG{rY2
zUf259uURvfznfKKQ2A;y2U~GWv*r1y-TS&@>S|-(pE5a^VRt{WHk8k8|Ni;wc9t+d
z+s<<*=Dy#)V}Er&H{6LoZ*t`O_TYne?)&{Ms`uus`tvpVl>guNQ}b+(_SOBGKfT9(
zCS%=shjar+YxOHdfl=o|zeczFFT1g0)4`hPKmD)1*8geyvgH2@Hddp@zx>$_pW9b`
z^Qd9qj@Q*bd+)FIiLRG=r2k5AQK0?q)PI5}{JjHJUS#h(6MtjMztX?|j=eke$tY-2
z;hf@teaHGw&6iJj_I=5-{B^JD%bfKlh^#sx;OxfmXM18@)!~1$zHj=TpZw?k&WGPF
zytw_RKVaW?`^X6UZ_Qm@6GJ-dH@WA1Kkq*E&HZz+uA3)KU;dZl<LpImq__UfzPk6W
z?fyH9Zh!pzIJ$0KasAH6|8&hy`25@~VL$ci{++7V{-$p~k@{_`{Tq9otUqV<|G&)n
z{qDdo%~NkfemBev{5pMY{IQ>V?IO=V{5i?rm(O^&n*8syTR)aR{kQAk%X;re*FSE(
z^`oKq_r<!XDYvh4eTcvOVE?nW_b<QyU(=zj^6h$<(al!-AK!mnl>hgk?Q!Mdk3Sx^
zbJ*{CX<GN`%lhz7(*^1x+Slh>?ECf4S9YJ<uYmUTf^`u``0qdauJb8B?)!D`4e{bL
z<k!BgKmOqNVQYo+|J*dEAOG9``s3z#e<~&a|NO8?Ud!dTAm_j1R~rugmcPoZ_5R~N
zCa&#PRb@Y9qb;QWep>%Jp69mwl&AiTpYxYJ<e!#3F|~eq{_XE+FXw-|!_NKVYj5oB
zebzg-r89Rc{l04ZF<M_?-?V+N`)*4IPPzU#X;a<KwGyk2yV}_5*}arE`MX^7SK8~R
z->?7u+yCwd+oJy!<;T}M{JF6CqqM;NVw1mnjQ3s3v)4)a6PNzyOWQ~J>?Qo>B7492
zC)VilKK>s6r~UZ<J!gNV9Qm_c;{Kfvb7fcl_q2U3fB)m@&)+`Rf8KrT!*1`%_h0wC
zw|ca%afZF={radAf9|dD|93DxW%4<B-d8{8ulGFvf35wV$J6spY<RieagF@V|MzuM
zY~$<JiSOUb8f0>R)8*^=uMgCGn;n_=Uqm_i@$c=oCNHSJCpsZzB9Fk+Pd}!<necbB
z+y5xb=;(cYfAVK7_Eeg<`4i7ixeX`&F>T!Eneo*=V%2of(qo(B+ICJoxVEr9=KJxg
zDE0E?ht3G7+ds}QF^jOSudDpa$9eSBpUp;GpH<fETKV8r+V1YP%U|gmWZnPzW_6v$
zpB2`(TLbOaF3Olw8uRgwk$ikyw(Y*R-~N26Inl@OKhyT%*<0=Te-Cf}`0=26-%i{8
zg(6P1(LWY^Nx0p=-tFGpUup9DezvnOuKkf*^W&O*jkLV{9Ql7c>}q}o%>K6L^VWH~
zap%w7I3IWW{Iz}GHmiRVPONu`kgxju<HMDI9}d~=S*UDh_x+)$?Bm1Szd!h$vER3k
z-|3<J(g)5z|9y1)ub}jAVa<PzUmH9BF>bBZ*jOtv|J%cZ-yrOa!rvOVb8CM*ZvXVy
zxwEjQ!oY5ep`AzZ|A3D_B9dw}4DF`eu%Gs$(Bx}>YK_MJ`i%Pb^74N!F#bKf;EjCN
z<?qK=rMcWLYF>QbD*WFL-G9DnftHn3M-?o#Z+P0S7IkLlS#$Gmvo>v-<7w7+X1UzY
zgwuZ~s=q&e?!4{H7xBB6KJU9=yu97>P_&%chA)=`k~~iTt@Kq_c~a}eQ6pCMPU%OH
zzsk9p#n1O7sy}@$op80NV-d@TmACzhJcVspp8Q!Xa{uSfr)?KpcC4Oavb0#bNKyXL
znYq&4m3~{-*<ILwCsyOuE$vR%Kh~jB&&qGh52#LkY$d<Wdf(-@p-DZvKWyH1d{Xkm
zr88qTSK5bO`1xFa)#vU#ALD|&*Ty8x-*f0h?y~3dYXASA&+VM=|8FPr*GUJ8#XIWX
zUi<Sk>0Au&+v<0RJ}-6UG%$X%v-HPf-+7!0a%CsxY8}zm^$0q7`>WwP|G&RX-bp|I
z`q6LVhWt0XXH37#dNX#?UDZgNynWo8cbn|?-u!jPPrVE8nf_kf;C}iKn5B1Ke(mSo
z4*R*k?B4S&BVjjpPI=>7z4f+IH{LPbj!oEoI=i23+wKE9xO;Z*sW35+d;d<);N7Q3
z7r`P2cAS3Q&-QKifjaRU??3%o-1t{-!+q9oyZ8LdNZ8LUQ{MPkFW*MW;vLiR*asCI
z&lfj7)+@Nf+P8bp&x{9qxaX8Ne%8ykld^cv^gQ;#p3}4Y+0N}gP%6IT-KT4d8(-@c
z++|(2d(YR52UR@>+^0?2z2|SngMHk0${T;{<=ac`c+d1b_QAf>xBJ=NJzr3-G5xXO
ztA2OsZqY^dM;?Dpej@+)=Yix8@~=Ps4E*F?`&r}H%EEbC&zsLxYRz4Kt#Z{|{;KB-
zE8Z-uJTrgQyPGfCKW%LPwz2)!#^wJuwli&NXWP`ywP`uurgouC-$FL`PnutKLaytC
z+^iFFt4_?@bwcjc3AtM*<X)Ya_v^%?m5=0m>sF{#zf!5zP_52TtzMy8{Ytf3L#;YP
zt$KypYx%V4?icyrrtp7F@&B8`&zj25yR`UO$uZ}DFK@nUpSP)f(WZ}cZqIvL_d=^$
zL%TXdyZ*$B({jH~$#I>QlR7<5>$IHJ>1V}pwNw6tM1Q*w{VgEo+l82KA+g^s#C{8i
z`*tDjU&Vg?=<}9;FTQAJ+q|6X_o~zNuU|hu{NI`1JTma-cIRu$?cV?1EPMUAjq{&>
z&GvTNk3X-lyC7dL(SQE)$4XoI3X!+RAAeq)^8GmTKaE?DKmH8d)c>5hf6?EI8TC`{
z%<IkTUvzu%#kYY))n>nz@69VL|26&IPP_WQH-5f%SY7+xe*N#o&)4oRdVc=@=lTzB
z`^(yE-q-)v(z;y#u<gyHImwr9wd~b)Jl$87`RwWu)q8W~Z=G0h&*uLt%dPX@b;V4`
zbiT8HQEmOdHe<t(cRi<M%ZqycXlKpJn^kU=n-Vyy=JEfoxZmo(9^d)A;?BDFHoI?4
zIT}8*Wa`;(@u3g@Pn0f-m@jwgOixALw(9g_M~YuR`F)4GeA)Xq@8|3;yO&X=F@ICd
zsZWN%%Uu83&)>3={lwmif7d@Jo^Q96|5kJ3bK?6|UUqMPZb|b!-+tL}?n=pBWpe3C
z2QKfP<-fgV*VDtg@_Dn0K0myZH{<5FeKrC=OIGasS@7iWPR$_cx~<Wc!MAEF7d*dg
zJO7;Q?pw{P-)>v!TJ>zw>szd|U1ojuwwwDq%J^!u|MH6SGI>^qdSX{u-RO*6c}r~8
z&nqP!^47M-+EUhPr3)sXdw%(2O^(hPo>Z&Z{_dY&rr7a4yS%f;YMTG@PcH-P=AK-x
z{If(!p8eBH8#~$K%P;>Zk(ht((d4g-CjYSL@Xvdxw|+~^oyv(zWQ3+VZz?KL@p@F`
z6RV=%p5S$drTKv8qmqE&>Dqfwx88lK_@2wXWal}5N6{V?e|D3(@u9Nno+}bm>ePN7
z&M|iXq1vpbxTWn6w_8%iB<INrQON=xVp}<<on6nBSawuz_qmi;Tw!Idn!C+Po`r|H
z2W1L*PmY-N<dVvaYcJ=VYe)&;a(k31<ezJO_UZPCw>{liwBMJ72QB+|?PqnqtpB%b
z8_n}&qm-xKS(@aNlr<@M@`70`S5;P;1@e}3FS)9+POZ|EbB$+$uTZv7<m5A}EKaWR
z%<v7${+)H^{tAcOf9764A7^E}`S&wOhINhN11YARWvA8$9g<>R2jbkj@F?i96qEjS
ziGp6n>k<XLjMoKV#N6vwraJG_ZmkJt`?-$g)0{0w9iyL5DUA8_;d=SLY@MBLm;4K3
zc0J)gZFRKrucUKE*(0a*WlCF(Czn~*XQxkls~<bJv5)6!TFzr0&pAgXTU^dM{?lVl
z{}Cv$X3xi(W%JK}`8=`C?$UYpUq3zU<zGHmR=;X;n0MwSMP(kz=o2fPmT81uI-!60
z*3UB*^DX_^Y|^*!h^9<X_*|bBlaLx=*339_o$=rQFMs;auUEV7WB&i;T>JTv&y8<R
zeqEC?w|~v&iKTX%HW|LINt=KE&F6_e+WXGCfBShta+gi|`l&1{zU*0F{A$wX*t{~!
z8{vM@pH}K$)Og|%Z<%_sTyOo(KToxnYwtX9dHG!b=qL4`ANq9E`GpkKub;E*$wxo&
zQoFa2Tb>`D$6syt=6uep`5S+pzwxL2M$P#tMz(cwv;W&&{G)$i(eo>Xo)7Cy&FuHt
z$viAwx_n8+|DH3C7w)mIb9-j5{CWPwpXX2f`TO&};LQ2|IA+_FOz3y#cq%z7|M=hb
zZF2MF9hM)hS!@>)Xd`D~D^>b`x?s)!x|sdX*Zj-qyX(8*H$Q*tlYWH>^&CvLT>mbe
z+I{W(ii>r2A2Oa98{Bs~CBAZR^oN|Ai;Ipw-rS$g{pa6dXWO~IudeU9wZm>c+rQfX
z&aC!zf6q4j>2oiwvzB}RR_`bK#~;QoDz5*}`!k>G`2D*Z<}-b~5b^lkC*wVlrV{_&
z{N4Tb^~F~HrK$dFzgC3*KD;N`-v0iZ;Ox8nznn8|z3P8Dr`xg|k=HtHzhcdE>HVko
zuQqzi_%9&*UqJY4x$u8i!u<P=MU^{#+w;G9Q<cewOf}mnSL6DB*_2#;)$cCfTf1u2
zw;fOQw%zaO7knZA!})tNfAq_hPJa@0e%9RiH{m}2lZD5BRPXuj9xC_g=|>X|`O9ye
zP3DVTo*Z7Z>-xI7`aAy{&A#M3aJK$=IPcbPP4>^V*+1iZv$WU0*l*79*Z<i5`Wc!J
z{@mYhBldOr{SW2G_6N_NANPi*E2H+`w10l@_ic;T@O^h*aI5e0?S6Wqy~j#gp2r7-
zR7Ko>E@aui{*U5`ox7hsKYHY!{bb*wIkAQB)<14IU)+7gGI9RhDVc5mL%05w-+8dZ
z?*75&*Uv9LfBwPe*WrthsMGa|zy9#p?XOcMld}Ew$MDCWUteE*KK|hI>+6fp%Qv52
zKi_qJagWEJPm7nlzip84`>}yS;?w!}FaPa-zq9%1b=#KCug~R=cYJ<7dE@WB5*za>
z*gD=T{j)Hsxf*cjOYT`!`)%(xso1ao-M4yP-HpE=``+wqIo6+VW%GW@oE=Y|ES<k3
zT1EAH?`x;&IeYHBXL=v-=y=1Lx=8s~J~CnQ@1IBQfBi@C&oAG(_AW7@&-p)Idwr~K
zZ^e6;?CReaKF&V%>7D8Ysb|X`{#2^^<Ev{Dr#<UT!kU1Ojz_-l?vD$4#jX`=yDsqE
zegl(z-QTX?Ntn&?>)(Yy1F<EKkEAvf7-kw87XALf#uM0?-}cg6wBYg9Z~gfmu}|dL
zdi<8|S9<O0_fo!!?-+AWbNBg64|zGnPjPOU_OFoR|B7{aN50NnlGs)HJ5u>-ra{_*
zjRy>F%6pjpo%Bn7_NBxBKFo`+`Z^&!?%w*i_6Li$@7teu(LQ&Yqe9iq?Q?nmuXy49
zeM4=nt83PDgF4rSuD`zPqq6O*R^Fd^<(YnIc;KpccK`R<O;rE7IxqA7T*tbV`TLVn
z_tbfu{?Rq{<1a<s$J$}XUbk=QSsx*}UjCBNK7x{dr+-YF`tg^g?&IjNW3QiY>B*0j
zT>t*EQBd%|C->D;A5Y%&(e{+v|Kl3cpXC?j%=z{`d`|D@<<G_DT{(Asx&I}$hckXZ
z^S^X#apU|Yz5bVusb@a^d7yss7R9aeRz3XnrE&enclYb(ultzNQu81?W`_I=sj`^;
zw_^=A|9*FwXJ_i~RUhi7uG%<nYrWa4b$|E0-X_2Ge)(Jb4<?WH{r>kOs(-e8zUiZn
zzxy+!`lIEd{$1?ffBNDK^GEy=QjY^a|NgvFWv|_TF8k!WrT>+llrOi}{vT+4^*#T^
z7yY|zvi_~O_W1f&hkfyD@8@5(4|%!k=IvmH_5I~H|Gf|bHJ=^!pUy6?d_JfC?;nW;
z{S$w@=q(cmH=^UJO*Y(RJr<jA=jqYKjfLVr1a=+g?saE=tY`3!>9O8=d#M}mnSRG6
z?8j(KLmJB;Gal?fYfSGr4QVV_id($@^lWkCXT5@ZtmmNZYMcMEEeHNLv@fXmv41-A
zNB(;muk$;<{?EVn;ra&se^Y&be>S)AQ04bK>G^EN&ZD-TEsr8U-dAZ}dOVu%eSP}w
ze}8zD&ZKm>IL?uOoD%X@y>`wuZ#VCMcYXgomG--q5%lcbn(ax~=di@s>(2FOI#plL
zTq_h{$Eor2zCY8ceh!WL&kBD|d#e9>>Tm4#$^ZW1*)=O}^zTp;h@ZpF_{!N-G1&2`
zokRKrmq*S#`;46L?2&J}D*9_$Lfw*W2M%kLam%nsNEF0$6)67HdDZen_|=i{6mbtp
z*JBL}6Ee2&ADONsU-Mv}$jOF`{ZBSG{gwALa8-A(tkX9?c>D^>`uOv{9}G<ozP?^8
zT;n?B=hr(d29o=f&K{Ur5yX0;!0{h{P>;xcr&lY4QV$eH_zNAI`lwkx|Kyq_a(|_n
z4&Ap{S6_A5mG4(s%>U`xpTxI3brze(zy0q7mygbTM_;$AbgkYo+xg@1uYdSDAMMWl
zInCu`bS)F(NA;Ai_Z8A3{wsPu;nyyntRH88zO#@VYX1b`$DaEt{(AZKsQ-NQRol{V
z-jnX@@=uTaP>HD#(yMDNFs$?2@#A)y*Y~gQPfjeXd3It){XXtLOQLHo>Ds*%n_trX
zoBzMnuA}<TPIOe8|9d}K@7u0#>Hod@dcJyie6ZQ`$hogaUMBA2cL9CB^M^h++Mk-Z
zQSPPkJooULGZPDI?A?z!+yDLhJnTkr;NSjv(TV;?zutZxcEh>)rgQCIgV;7(lN-i~
zZMG#d-txB{xje~2Zod4sBf;IZCuV)TkYw*O`ys#XGYIn6{kHgu<i86^_GjioM87Fk
zw%MOuF0XR_1Hagn%Pm#60_?r!KlC>~CEr`8wQ;%X*E0(r@<+dN7V2GY`c<*}*x`A<
z5~X<SXN3IQu=3BsV}CM3|7uKav*%eY|LoF-_S~oZk6$?(@z&1>xA$81(4YJC{PYh$
z(#?A9YageZ^*-mV7oC52ih@Qhmt@+XWhaa`Esk+mzr|$Ax#!VrE^U7QnwHs}7q0mG
z#-Z`PzSd*r_m`KJ$o2kCp1||s|CSju{+AbJ&WSbjzx>H2&uZ<LGfO9Snm&46_iK6f
zZ~LO=+8{~0l{`Q1`)eK6e{$)^*Tl5%dVl)eP5%7r_?zWzv!#veWxYk!!>_N?^{-s}
zGha}4U-h3u$1jW5sUCMrEuOGM^!LNv>INY_ztnkJ#YI~0r)lib+I{%Fkw{#CNZss(
zADR|@xXsggJ%9c@2Iq%gZ!cFr5K!^=+e=OZu!0i-KeC@~=>MtukNMDr_AA2m`|RxX
zUPSL>;d=P`icr1p^BZ$dJ<ZR$c)#Xw`6q$DtxId#HNNG)=*@pAZ$9<s^%wH%<4-Ou
zy!uex(q!Mt=l(C`<M-~6zi40g@#+tA=ZD&2$KQW_%kOVi`PZiX*x5%Fq*0TPe*866
zM#^98`0MbZ44X(9kYyiJ3is66wjWbJ`groAkJ=*rKl`6vx$$4%=;PA~Kh&1G{q6nq
zck<)L&wu{?IQj9%lZu)m8#_z6@6Z3|Tgcg4upgaw>V<P!Z@bm61D`6imi*T{{n-7b
z>6_-FDzkSB=h=O*na6cXPU;o&`Nwbey)KTK|G(FUe|tLD)ca2>=Qst=ukU>R(RJFp
zc{BUZzrN`0KkvAB<Sy@uDKh^m6y*NOe*AGqe*W>#jqVpW%FX}&cc*+m`@S<p-{s|c
z-}O)W{n_`s`^CQ(U;NY9X)k?O-nHL-@$uMCFAJ>q<}MH3a^&T{_ZQ!N)Tpt!Z7a95
z-+lM-#UFm=e6QJ3ZFkF7UiEzX?&phN{LJ`XGsR~9+j;$qu76Io_li3|eXpGBdiTZG
z7eD!$^QB5;uUv0j|D^AmFa8evQfE?S?{)9I+k5wmzc0S{t5In$z4!ehfAepDe}3H5
z@4nc7`?tUC6*cer&-c64t7<*pAD>$NN8|AE{mn%m`#0Iv$$a=PZWaCine;!cZO7kF
z({>A<r&jvQ`~D2G|9|;TT%L38!A!rp(>I<)A77lx@@TE_wWm#|)Qv)xeYhIFQgff?
z)!Le^GyZ>@X7TUEwEY{u*1zSIbUu}E>E-bcF-^Z-dVjtA_h;>{<NLQ6{_m51<N3W}
z>H5FnFXZ$Oz2EQiy<(&D?*N04g+8uIq8=eV2LDe_%VL%K{4#!F^&XMeKT3BU_bxma
zzr1>n$n2jW&cyv-&h-Bv&c`2MPCJD29mFv!QhN9+b=Pt3eaG`J+s1vct(*PsW7GSO
z(k1=!EHUdVR2@?v?be!{%%jDA`1=ZZ`wotK5BJo`uP%IhKHNBa_P$v^4G$h&y~**9
z9{Uqk&L`rNV;vYO4>FvT{-^QuDEGDYyno%sTz}rTA9#47X2F*i{3rgfv-SSlw66dD
z;&XfIrrmv<er4wM^P8&w^jLjvvMvsLnVi)>|6^L5z4zOX#`n&=PG4zTcWid$U!AST
z&sP|)pTG3m56S#zUpHToldm+kpCaG?Jnd`!Qjz+<``4f2Z1gGn`}OmR3IEqr+xK&M
z79IY--@SDSOKp&pohI*ougr|+w|};4@V@<VJ!P$h`jqoqM6aKpKc&sz>#+LLhm$V_
z@c*gz&-~GSDeO;iOWS^}+@tJPhtqF$xi7ZA9QG$PQ~Q=~*#ARoL~{(%gLbf;|G#yP
zXk~{(=92o@Q(ZT{d%x(P{$b{ZN5+Xq@0TBrcTN0ZmUwiBP>9XvV3kb8yYo}`FZmxY
zUA#3i%&+3-vHwkK%Zp?9Uh985Ff--RdQ0|2b2BA>Wj6JE+`LBgGk<Z=oA;qF;#F1m
z*Bs6~HeYV8vHej=y)QlQ=EiND$v5$9Y53pEqW|P~9-QaXcK(Ry{7&sp`{#FR->VON
z)HPp7qK^43bG~BzPoMaY=1bOFRDXMaGB$AiXZh1n6I=i5YHF`r_V?;Zt-AkhUiL!A
z<AWzB`Td#s_sjjg50mBk?yulaVORPg^o#xc%l9vzyBqcG*P4ICeci?Q@T>nX@z$5T
zI5uDV>FONm|0T)mrR!>LopP5t^*Y=y`lp9;<=;PBrrNK0-rV+yUB8d>_3`Cv|NOc7
z<JYsRe-19xv|sCgDIo5?_x6vrQm0;5zckzD?p#@`fA`q;42|{gm)`y%$NTi_-V*8i
zix=+MH$CrhwaJw0-!J9X^tXNb6}PMR{l$Q|`~KTMDodSu?f%lN&c(U%ug2YD{TUkT
z<(J<6QMj$w|J6zLuTPY}9yxFL+ns&cmiyth`^ygB?2h|qbpHR`hx^JrpS+)1T;wkj
z|6|v&pSM^lv}{CwcAUNEA$n<-`_bdCU%S`;`2UD)H`9lx*q6yqFVx3R^w3Qzi4VUQ
zews68zMpi|{4DR4)7pP!%$Ydv(V69EUp{-;^77YO*%~Wxn_pjCgLxn9<Jf99kDI&p
zSJ1<i?)%U1{rck7TJ_-RRdsu@`TIgDn7=PQ_Nzj#`^(Fi^Q)MPM18i;Q4X-(`YL#x
z>DP~2Vr5Pg&vdK5v8sRhML+X7Y=UzqpVp|go4&u~)XKv@PTab$C*Lmh=<CNAA)Z~j
zuNI3n|Ek&6aLzLP|KxSuzh=ZqRlSNyy&U>+oA39vzgB!*V^_W-b=p5pyLsL2cJtn;
zE}!m~zuN9l$D~<bU#-_KHk|KoZZ>zyM&l(1UO#>*R$TS0<SE<K*smW;UWV1LU2G@)
zMC)aw-MsD!yLta?mQOyXcEK~ggj@c+m`=j}2=kgU!A(ZX4lKVc{_tw_hAmq-C-y(z
zG{66^(iK(PzVpwmX7{iD!FuabZKwE~rjn+QhUU7jlbhd{-mCJf%#_&k|HI?wXXl&O
z-}_Vf`T5!ZC(bUHH>q8?Qy`-L$KS8J`D>W|_%qE}c1UtY;l@Qjeth~}yxacR-;MTp
za^Ig<f+lnR?zlNm)vjxc`^0PW_-(wlx^MiZaqZdXM)#B7G;YbO{%${J*Y(Z);y;a>
z^LfEENO1GTw}B<qB6sKg{gVTt<y^PBpL~1s#W#&zwq1ShCqL$t*mU(hx2~HabKdRv
z;ujSnvi*}D-+b{;W0!pwsG8b*@vmll|DxZEpZv=ysTToPR3{(beDP7E%BD-Mf6{UG
zi*+Kla<2XEAkm*1Rd!wT`X@c#eDQPOml_e<`HNrOd;u1X>z{Oe^TpSJ;A%;Aefr|-
zi(h<|xd;{oRZk*&<yGIOFaH0x>Z9-jQs8~Qf{f|kPCh>MzogdAukY$k+t)o)(>>HL
zuY79fJ$Le^(p85}t}nW=_tg7OL0RJ0{bz1H%0Ex|@n4?mc}t(IPoK2mghF~s#pgfy
zSIW+nKAQjk>U(GYGk+#6lrIdPTJtca^r+;c-=FTE=)D>L?$4+D#+z^a40ijYyfV-8
z%B!EBXL_lZ_<77}Jp7q)B6CjW^sLj*zuezb`8?Lx{`{V=rbiZ>-n3o7H*~`vZ=)4Q
zXW4b1)Yn)1?qYFX?PNyu>Kt8L%Z%7JFOAuL1@&w1J>z*H<p1kht@8d&r{=t0vRC-S
z)Z1B~&-BdP{p^!b(ebyHpN?J0lnHKM?K5|~(Nykdf2=<D+ul!_Cw;kOjnCyyBi--a
zbHp#sdA+9h|Ce9S89znLo%q}P)25P}m6l7Bp2VHsvqSIg9+iIk{aSmgIMi5#n>Y`a
z94xuq9BZ-nrP2&N3E>{jV=T$h{JRQbt`;8Lkq~2`Cn4VBFDKHs+RE1e26YqTtP{%?
zy*YBD$%3Vj<B`A-$Kx`GEQB7v=F8tH`OrJlHucw^{JaBxL4~I-{k-p^_5G>p7nSb$
z|3&-1zTUn)_)uBbm2(IFM;=_pWfr<&Ws|RHO213pREMrvmG6Xp7X94Ca@l<9H3>)S
z^-}K6+jTeIJb(VzYS|Oh1$+e5IGAOwavfvoVsCwXs8IXuO!2Is(yOTkDlNgMVhnr@
z&c0H8;%d-)F6U6@q0A-_<UH7t&{W6yu%&>BRe8pYrFoy6u3xl_wTQKdHTYul*-zX@
z+-G_5u7b9Lwgd=dTi<z3@{r^q$)=ewSaMDT1i|b0LFd*lDxOvVx{?xfHvtHn&D{(-
z0^IvYnyI!?ccRK{rAh9ESyv`@l|BoxEM7LpQu*8*!Sr)ejOFLel1;a>{rqw!|N5GW
z%x`Dg!|nEbdU*ZR@^HJopB^q<oB8v@x%P6qJ)a+54@<kMus`d`v~?$R-6u*iDVvwJ
zmX$v3sm<uKH#sRk>ny+T=k`hFGS1?@?~Xrme0U<nz=t)5wTE>L>rC!2GbXnt&DB$v
zsZW+o>N~0^k;5IHKS#=Z%R_DbCQ!`IN|==}tLo{M9X~^_f`SQxIjz)Ngje|`FmZws
zN9W4M@8Q1F?q?idxQZt<HKBB+mQbK{$%Z9YRF<h3nx0(g8Q?o9ODJ@9$p+B)x|!iK
z7Uf1$&o!PYxfc&kikf_5){$!}+tm)5dam;<$-R7VQrzSpvyNO>*{{ZE=DESMC0Dt2
zQqpvh-;J3%-62kIgp#HU>~66L-uLwS((VKv?Fa*y$j+zN7j`G89MMxO>&fKY=-pxL
z9$gn&Dswv$s^o_1k(pJx`;*eS-C*W`gaY6G(b6mfX^4dgS6{HKzjfsFW}ecng?;*0
z4z^lt3TPL7HN|PJ|Bp)-Tdbx8EEjww<22V{#b)->7Yq8NYhEsKoU6IwGD~US0zZK(
z7aOfi0*<r(W6eF~%vdV3;2h%>&IYUB{>%Tp++i>K<@+VcdaJws$A7(au$O(h-1$#Q
z)%<gBt6!e5n|prw<ew#K^Zg!QHmtE~@OOXzmpkX@f7^!QJM-G&Rj!+r7d;hQd*bN(
zo#)Q6=B`WkFV-*GI;Z=>9rmfxbBbhVb|1O3OytzN+O*uWta)?9%4R*;8FyARS+BbI
z>hGt=iZ$kKi_wdEZ1PyieEvMOm?zh6)^BKkwC2rDO|!@ih3oQ*Ib~*Uuw6Ula?+Wr
zm2<8orTn{j=b+4UClfzyLqBOlKW;<6-v)lKKg6`oKR-t}<W@kY(_FK}C4Z+p3y$*N
zdEW9;h1YFO-{qItdbZ5Il=Dl!EO1%7zgTJ7cE!bUa%ygDKJ#<Btc`fZGWCqBxnAn~
zZ$5wTRMgCs1s>@$GiTlNvYA=U;py@7Qr;z>*2Q}#S=ByU8#2Xf+t2I$%VP|CKHCR(
z^6&eTo_=lJ9nD(5Uh7!L-zF<Qc~7|V`cJ-H=Fb(H!D)7HetxK|d;e2){<oSN&e~J{
z<lDXZxk2A{0(>IvXTj<AXuCU~9)_P@9&LB`)58mV`BVR_4|M)-=%9Dwvfz`KY*`sI
z^jH7*Z2!`H)|1^c@^^L=NSv&l(jRG;7Fc=LZ~LS@o<Hv|W#3j4_%mK~Cjai3(1ono
z|IFp>t{;3pz2?^Ehtv4C*X($Dc<sYOr%ye<tULeNe~_ih@_jWHS8fFqFAe+r@I*V9
zTlUi=ZC&|!{huZ0+jm=O%fI_saK3%F)ZNdSKW{uKkpB7MM7z74&6TTJUrsD{xBL75
zM)i6V&%Bh(FSgew@n1;#FTMM1+B1%p6RWZ&86W<9|6kXekO0Pq1)28(b}lY*f7*9S
zcB-}fF7HeKx4++F&-*9yfj@J-)Q`&t=Cl1X`LMhZyU=WDgEJ9NT{$*r@@iaJrWwk+
z<m!h%JULfB#B*P4Q??O#vs6$^qT>0oCw^ZNn-p!-G@H5urZzA5*|}Xj!+U;6W_hq&
z^n3>kvlP4677xvhZ!-IC5D>qAC%vOq+=A<Jb3ca%yTYWX#Ny`%6|Bzua_D_muz<nn
zZOg~&27%Tao#P$u{Wtx_=f7&=ncoMed`??A``e3>7fdO+Et(f)gd1!#_!K6J@)U02
zc)wnV<NUsL5?dEOFH2k9;c#Z_#hdQ47ai$jmD#s-(vO_E6V^V{P_*C2lyT_5$Ae!P
zyM;2m8$P=P%08IMBxAT|rnjP@_QJ0Xk&B8SFZo){wCzYtnvi08ZvKT;R%axyOgq|^
zn$f{(rs2)Q5P!nS;p+kRt@{|B<@fnJ6q;(!I=f9$E$6H88)t`kdPiO?_Lon%wmEKc
z<x0K?kB#9QiacM>WbKttuzw%$m4DYH0a2-H?rS$Zr?-2Sv%JxB?$q6<$jd!BSkUsj
z+5D5+3e}GlZV8l_RPfBrX#J`U0tFX3Qkxr`J8aKzXA0D9y|MAK(WcUbMq^>S6d9dA
zW-nWUID~SNjhfw$EHh%rH!cf#W4eTs<I7${@yROk!qzQI9h7sI<=<#yGI{q+p;LEO
zaL?2Hj2M<1J#}Xn^WK(uKOy#@mgGCVxNaW5h@>TQ2WxIyY~WTen#7WjwdiU5>24Om
z$(H77(+{t@EpxMP(_*D3=lpit<=mff&V_w$)6FYMzH3%XZmaYC^CT{(=9&1W3=!Yw
z>#v{twp*ZA`uL<9J8M(-6;HhVB{k3bnf#=OE>kOFm^W_`4dTeT?GY^7(4oG`YOg~F
z)3Zc@lq+#+wK~Fo&-Jj~WOH8DsUtMGgoz>N%$l0!<D!z9JX7~yw!XA$&X<{;MSnLO
zWbB@uR<HVVLt%vV4fdk?S2H%{y$zT96MjkO$;LlEXIU?m1uob=^=B{-gRb<{^9yz@
zzHpOUIQ_>>1Ga;!6EFR~`OW?DxkEZLCns*2+qdLqg^;OzWnRIL4I#pDw=FCj?P3M1
zrTVOvq<fSwd#5d1z2dL`^E)ffCP-hpwb5CmN8)ztC5y%{8@E=v?-QKG<`|{aP+BO|
z#G%*c@L<X#jsx|R`&QiewqK3=J9FkV#pSQ9GG-JgJop!-fBO2T?stEcCa`g2i(ELT
znYbf2dd7WW@!g-EwuUG-+9{P8NH?xK%b8L1_pR8O9X1m>j?6246sGtt_k%S<#fojJ
zY<UJe+c(v)er-B$a{eZFrw3O$<K%<MH%r3Rk5AkBbajE6&kdRNUW?{?Ful3TcCW(w
z_Lsh`_EINZ8UH8P@i*ULnigBVr|5N-P|oeb@<%WJ-ukhB-mRL~2P*}P=3LQVd}N8Y
z^t@dM58k+T#II1z!(usy^ybe3B@^Et64j~+SiC*7oM$HQ=juMcJBJ1CUgu++s`le8
zL#Ml%szp-y2G8}3`_Fn@iQ4?f|5x$jol3Ioxu4X8JH>JzWyPs<*7a6o_1HJ@9sXr|
zW76B`H5tbu+l`BzN|pC_&SCZVxIXd!t@))N3QHe9RC{+Oh=t3}W8p28z~*kv6Z>0T
z8!nhnu-CTm)Xp`U@ssm;tKE;zYQY{+4TZZCdo)f73rj2bwbYdqMLZBw+gnuFRp6WC
zHuKT-ZH99alzjerNQiiZwq)eb*KG_;$kSJr;yokxiN*DYbcAJ)n#3XYn=-SnwcHjn
zo>ZaAW|35$uxIkZwGDU7{igow_bEI2b*{>DN&OucU(QSLnY^^>L4L$Si#z`DS28Y4
z_?IngHc8rU?M~Cf$~Vh)T@+lb)0J;nUh}5*-Qi=im6~$deGi_wp~}Ba=f#x9$Ft|g
zv|4Us4{Ue1FH~6BT<uc#J)%uzo7U$gJddZpnfBxBY+XgMPpN<Ib<7gZecq%})^IQ_
z$z($Bktg1r`>M?v^_9Nf?${_))^TWdqF~apUjGBG&VTCe%yc`!7P*b_c-rlh;G1f>
z{3RFJY*W&$T6`WZOXBUiznC#NCEN8Mcc-evE`!|7C9$2J>Wdrag|4ZSV^_bkajn&X
zzaB3uxE_Yu%zW9WDOD%?#H}aqgI~kn*FMK49=&t$=bDBMzQ6Q7XQhg@Fnm1l?l+H}
z=;f}Ho-eMM&A+?QOjgqT*&41RGb}R7e2P<Mv40lY^mYD(!&dwyOOAv*eC@~CU*~!`
z_JU)+(8qV_kJrYOykYwzbFiaxpWGU;sPJ=aDPip!Chyesd9hJ}J9@$i8#A7lijiB@
z^`^wcoj9CYJ$sY<IfK1Q2QNM@o1rr8+5Gu4V)GyTQTWU|x1IgGrOknztU`6CWPTJh
zl)bS`YgoLXX=z}Zz@0gb)ux~JmM(ElJQ-z`XKcOtiF8(U3(FE_%jjGw^~8^FpZMqP
z5YJOTGG~H&x5&-h(%eZJlBIfT`e*ufe?2-|{>Xu((p@f}P9Ait+A(?M_Fg5+H36TL
zCeEAUyZMEo^s))F?`5t1;BZrw<>2Z<;k1fJs`uK<Htf8gqwRC#eq)gOov8xC+B@p1
zQ=*SB+DXrzcSn=8L-VIqf$;6zyP8pI_Z#O#6%@9*pV$^MML9NY2jdIRmiKD?Sxav;
z=_m>BK2_1oqICIGmD8MCY;iB9mhw;CreYQIOV&3ej{Rc(6~C35U3H26vl}F5#v3=>
z-(tG`^bE-kuJAP-KM(fy{<b|RER*?nhsv@mjd9Dmmap)$?wNkz$Aq$sM?U;+tWk$e
zy&Dw2uD{Z1a_rO;hjn&FGHp}%e9Ds6`BdFiR#a?nRd#s3CUbkb%sB?$&P5hSBQ!d1
z&z)17kmlXG!|cu5RX%KuAswm*r~2#cRN~Y5qS;_+9(Xe9O|Z|dy{{iQpU@7tvoyN?
z%aS@ZmI=Fg_mwYk;QG-u+4^Lsm|v07Rnxb^8k>*wI`#@M2}#BZRZ8A4eaBhcquuZ0
z+I+@0$SpN+!ht3H-=6S(58zF@)P7vKwAJN7pCpf7&K<>s55`BocR45BVVQJ8)u_&F
zzw4v8XIX(q%}btM_te~)(6RM3r^k1LqpTm(wrxFOVJ`E;@vUfRXF#&&T+y~*LyOms
z6gRnNon6)7W@oZ^;hCAGO4kk6|8$88-6DJ<L$N>Q(w$dF+}r*>VtUK-N9cEAwY{?q
z6T5}WBI`Mh>@sWLPDxnUEK-}$u)zP2-?FIWPeBcA$KG?<B*`V^#Z)(jE?pwle8Y0D
z)KSx@<X7E1Cnas1u9>ClN6x967jmxfndSjEm9u}p7CdNK@c6Kt+@&8!)moS6bufLp
zJJ<Z-<{-1f0gkJ3XNLyPJ(5;@g>f<8^1|Z=iFpTB@mM_DwJ3dp0Iy2o#u>YgRIZO@
z(U}~3d|~(vpAy^nCEs|g)X%sqf5u>Sbb{osb1|D@S0Da7zf#sGsK7&OPhQuQyfVL?
z-qNfORw;&O>g_%<rK>mGLSkikTnO)ni-n4AZk)xE%eNTJeWE&b`xS+RmFg>I^6pA5
z%W-m3ynpHblwTXSPY9pZXl}7MP=-0^gNm=zO|IYvfxEdDUK9(jD%0b>JZaYax<%G|
zzDlnQSraR&IYIs64X4f@t$V|dl$U44EP3~F_4M-}y5rYdX^A??DE(cgw(QfgjlvIf
zQWuom{K+9Kv&wL(+cwWEg}(mntGSdU=a^j=W&eBK=14-u&P5)pmtL5B=|0o$Rrx&!
zH#<IkVpC#y^sFe);W;XT7d9JBte)}Y_=;HH=1u(km%BY{xp)3`uq|MUF<YiG(M&bd
zOwIJ#(!hm$|G29*Ixgg>e9U+?;d#Ob8#cEjudQJ_ADn0rzxX31OzDMU<sM6}qQBc}
z+a$slE0hHWPY&|9XuW*8mGj)VQ>%JwO#ZBNJR@D{X58m-xjQ31FTcm@RmNt0)lCfg
z&yAul&$}2ojcG&C)Y&hDRYO;J9JuGKna;V!;gxZFx{6#(|CM#EN@At$J+)Fob2o4^
zZ_aez#1kpxk$mc%Uzv6=uhv)PYogNyEj82r_U3nXBs@Fed#K^4<{5!|aaKlC&RI9@
z%+W5@lIGeO_G0o*h6Q>zo~>7871=pi>8?;`+{9FsNzPkW@17W{Y4I}jkosg##hVY8
zv}QOwlfR_Tvg)aU`2((s>ydvd7uZDX44TlHsv#L5VBDKBvFJ{I@e=LDVe#zg9fvs|
zNzMw%4si{+lx0xv>p5T2j?-wmwCMg_^-c>vw(Rj_4*tL-tGIna;k0E_np-#BFJN?7
zHZLaQ;g?l+)GtU}&{8@bcC6*bqS*y*U(+wAt!mvPbkfq4wc=&74bQrFCzW#rCou<}
zS8?@JbCA(WPHcL@FR{35wXN}qM0MTmEm6%1iVsYswrl4upQ<+L-d4?3Zb^6YA0IH>
zykBZ<%m)jP)DI$^KFV#3wgNr!>m9!SJRov2>-+X8Qcu>+IA)hoW_WXnj&PN(Mk1q6
z;+eE(cY8Oo#~5ro;>)EgJWt8>zh~FK?B}KHH1qluALlDrvR_~{^}eFJN2v6j*;T(Q
zMn(l}DjVi>FwahKaG2$IS;(p0;lJt!zB50Uu&GX3BfXfXN>?}H;QD}IUzf}TKJNa!
zs8_q4q8L;?7bmstd9m8N`_4($iPq}n9@l29vW_U7l)EVYziCqco;f!p_L@Cr%Ae(R
zYQie-(-r$13!A+>Up#R*DCweYr{P&0t62JJ*AZ4$S%u=MYO{9C(%!IKqmgBmQmuVn
ztzg0%Hm%Y)0Ry9tc3Ow~+S3lJFyAw?aZ<_i-02}G_~x6924DLYnH$@;yl*h*R4x*}
z*1)R%=#OcU#SxcAHRYy5YLntOFHu=#oH$j!C2?UAlbg7o=KP(WD-I@oSDf2p_FlwX
zNAS$I1#7b=8EG}_+M>D4=UT!hR{c|!X3{4lJQx>FdJ<f=AtG`AosF6+?w$?V-O$MT
zjq6#`q_?j(EITYWc|)qz;dfitB%QAFJ-B2_tF`ikxpQwYa~9Pu6>$E2(#5GyW8Jlx
z8&&3(UDMne;QioM$IeNOtv)L_?PK{ECg!*-`kVa8LalU$<~H_?9wpl)YRbA!hX(mB
zcAX<wy(K%qGIT}Ua&LAyCv9=o1&l|nCHqdu%A}S}(tGauu<H!(5rL$R4~B_%C$pHR
zP8Q!-9eDTkGF5@H=nHn!TIO0#WDQW=X1J?_KX8l0ZN1`sxi7Am2KI_&q`y_+d~qVj
zZElqBiRjtAUXA}v8UkdN2FYF*yb?F@n*dj^N2RyzN<Eh-!?qofx6{|HX%w_`_guC|
zA}eCm`jdK&8JagQGKqMel@@x~Aihj-L-&8~7@40_j;L&`3|j9w^_KR}j-#!vnn_mk
zSS7vQez~x|RgQ0o2cxEG<HMG{38yamg-w{ps}dc%&%NO7%PpaMR(LP7aX8M!5xR4I
z?STdt23y`sr#x?aOw2Os^3j@Ht-f`6eT(^=rIV-VS^6kUx*7Ru#{uj9LrX+lwKo}W
z4?L%M_^(XWr`X3Sm-}j-yK<H3e_T^?wIxAXl_`x&Uto@sl~mGPX7`XCtcl4FzDX42
zc@_CSa`P&YFaA4yO27@xOK13|WjSsU4DpS6$rWtGQhe``aJShK%bYF8C$DAWU<>oy
zCm&N<aGHZ@o{iy+QxifYEPHQXT0h0HZRN%-cdJ9^+**H3TK9No&xUB@=%}`X1|HFz
z!bX}WZn#b{ELx(~|LC&vrnG}QJezmj^}q1ZXO~Q%)r(BoR>sRSBscUpg{It^)$r6P
zd&;@0l(#C+oK{4u?tazk6S;H+=U<i*9@piqT_+TU=4C1#-F2ajF}Ue$@nppXL5Ht#
zE^XP{cv;ckXfwa>yHo+W%(F55c{X~oerijn#Le7x(&Ca!b*RWwbH9{;p9TwOEM-2w
zKrbypl<mZZGVbhCnu>?swr_L3bH_;eK)*uZTrPF>gc6UuSx2G|9((-m*q<`F^yGb;
zvzK~_7O%Od&H6<pjOm+>QDbx4Bgd|6bH1%pLKfagSSG~Nynexl3mS7xDr67rUf*af
z(^1G~el*$iz;S*<b){n^pS|aD72W!H^h1Nd$9J-MQeGyeX?L&h$Xuwjpt(tMVd>Vu
z(%72*X6|!)E}pHrT>9{I=IYam>n45f3yDy^SM3q($S(imJnKe*TQ@Hqm@L{i;V}F9
z1%5JeYEf#XSyu%2X@nMKPpF-?i9d$bw(EsJo3HNlCdS7+DV%cmnMD1R?#(MYdi$zz
zro@^hr^9$_G|px<JCsS*ZBzKtdnzVdV$rfH-jIUB?OQu+QapDy7B%v;Y&m?<?^K?0
zhW4Z?>vdum9vGcB&#w?z{4dXYE_<}=qK+R$^Sb=m9%;T5>A${&<zcQ&M)Ukux!rf~
zraIcFO7yrMTQ)^x>Z;=!YjbBjvGcfKbzVwN{X~-5C09Yqfb+dCj~o`8*Q6<2Zx`Y5
zi&JKmivV|4<D-DwTLBCAa7WEp!M=9Y)RdOvy$bbW_xiT!FN*N0Y?`L^e5KpH=oAz8
zs(Ob_(hpiMCNAcV65Go>`D)4I%R%u+_#Le|)@=%oc|S?NCpL0le6M)5u(8#W{*_a?
zB6D81-S(OuHvNlgZ+Ti%fswjr+SC)5{Vvb#o;~B#N#5wAK2FShZ>%D6JhkJWeQq>;
z<NAh;f3<bWu9kF@Sv#g*s&3fIQXF)p>1_ia+l{SjCTne7R2`Pb^~5pm3-|dDfA@RO
zt%}~P-JmV^<>cFG83hY=$)39s(csxWTSY!+N9~0tx$_F=gs(bW$aunK{)brAV1p%F
zj%@E;&9+T6JaRF|=S7wW+U&M!9Q!ZwDOP|vOW?5mLx1Mit_!~>E`5?39k|coO4$pS
zzmp7>_1I4>k$uBhnyDiFP&fJFgjh9?h{LXP??37k3NdAJs5SjLTST3wi1C!flDxGZ
z|MxzdV%w$le3Kk!kow){DF-}k*0WCN7RgAuq8hZZYwszkTN@??uj7o=<Fxq{DSS~h
zM&QGfSSQb-ZU#%uEK{bpYjz#x5@Co?nAQ2>rLgN<-#G_kuQh9bTRx@cn33DO$_-V_
z)yso56m}jrO?{NV=x3|mQON^-M$^1DEO^(O!(phQ_#x$n<K#JZ3nniRwfb;v@)}Og
zcImrIcZkh3Ztb0wqUw{^agEQ2>ATTNQ<Va~RR)g2)_<iLQe0>5_H?~+#7x|XyVrQy
zE|&1DXNf!vbC>5co%2$4=hXC6nV2HjJSp5btNh-HIZr2?%>Hn}H0YC}zp35xC$kPu
z4q1P-FIVo;uTZ~}PmTt!yt%l~NpE@UuYxP4g_}GcK8t&@$?NDl8NN0Dr~K6F_Of<N
zZM)&|TfXa3V;r0BUBSOR5$?;j%vknj{hdj%PVP0kh0XmE&k4<A*I2P+yMi#Y*s+u;
zrtHQOIDA>>m)v@}<fP-AvkP`*hj2C9Op6q6?bciLz%#^4YMY;0o6tf7o0oE(j;FOu
z5*KPd64lPXax->@ife+uq*zwcKcC#mD!nG}GOjZf9Nr^jX=vg5*u?*&C1YI06ZO}V
zTF!Rwz8r9-U@{Y9cNG6g?hU3=&*gs=rFC6eW<0CO#^I{qY7u)Ex82h+y37jW{dX=o
zd2+$)4P{6BQa-tCId^IEcA?8Lep?)NPGMHA=nWEcJ^V;Lx2bQhUd7WD)_+>B`uP1`
z*|^c!Lc&>v>HcDt{STGIbF>0q<(3!t3B>Pur!#Lx<k6&w*<Di{;)0E~tNii2FSftI
zC-qg5>5r$L-KH|38y&l%g^w@%*AX10%q8HIHjm|SOS#$n^{uYc3mt<C-Ur`Ov6=Me
z@nePBMP}=qHH{n+vTeVtU$-K!;Qf;z^%)MCN3_^iGqP3bh*mt^DcjOH+p&!0Om5_}
z=Nysd6LR@E_8b42zv+zl{<eEYf4>RY3Z2;DK6AR4n7izsS+nw<aT{f^%}lA#@Yx)E
z@6$4-2=%$bQW9MER;|rk;2Cy}Az1P2vAy2jPr~FaS4|3ORPEZObX58g-{R`c(-?Z!
zNghrUFw!Y?UGmD=_KMq+gPs%F*!2Qh{k0RHdO2P4n8K*~+3i5PV^8avDJg|bvRa?B
zZ`*O1pWL<jfZ@rPhp%nWdp=!({Yv9zw&2xwR&zhS?ebADO76tAlGD?6=3ino6ywrK
z^IBl<wV2nq!G$MIa!dKCrM`KK5-zcAjomU?ZB-lFOr}C(t|wZacS<fcojP{HIoP5z
zHY(EcN<g~g)2cJ`3Z<PFFTZfc&zPzCo<xA-r3*(Nv(K{561{C~<m`8Rm*JC(_U(%<
zJn<G<tQ|j@;q%Sn1t%ISH9p-mjN~tp+a~u*S##E=I|`5VgB;K5Y3nO|SA8Y)+HmrP
z8K!Zp1?PEheE0noBhb6<XVdpPAO89sKhzj4elEB1SmTNxqLa@uStM3m>+0|Nq*7U(
z(ew4+qsgag850dcrl?pgOD^M`WUSBJ8To0anf|KNN{b^VF$q0+!T&XAp_j*}<Db|K
zi!X@0=en(UUNcr>g+6D#rL(zF@r5;qQvxcMv8he0yKyph+Or6w1vWd2CgrE4n5gaR
zm>{7gy6DlbidjGH6E(P%omZYxOP;awTf}!iF8=K|JT-YMcJF9?p?R%jL&ed6mMgIf
z6q*@=&s?Y#-t|uYHeYL^@e<$0c}5fS=FXkO{3SRrJLE{;Nz**1k4*pFn+;my@;0dN
zU;Z*oyd*{K?Z=Ys*Asn;PCwb|Ygg-cJ;iL^-0O*+SEqSK@feBrCLOuL;;`I@Icxg*
z#ft)BnA11kUMkVn|GpylpyJ~XPaNWUrC2?(6TI0Do#MScL+ao<&8eU7FEM#)zO4HC
zg~Zz)?#E7@n(MqNWx--;mB}8zeM3qXL^x<{d_3hhi;PdklDR^vg3E5jaxDGFBess^
z-}=+vGWu#0#7{n9m#WKHe=hJD@5&o&v3H(YOggHgCN$-Uh0Lx1ZYHtB+8`zMTh&LH
zdQ^mtew{ca<r{}(tz@^;O2ZT96@nHArJSBAqb$**BI~xS!%TCV``?_NkO|ey>E40c
zRa*5=_IB3I|1f8jP`T?vu8O2L>YXt~S@jn$YAL#D?v*^M`)|(0Q{QA#Tm>|q&${BV
z@M0d108@mk&|L?k^;48Shdemiuz7WG)-3LTvQLUrxBcsCZDEc1{p0$+b$jBJ7f(E!
zID7j3J*yVYSmAj!dXl>E-F9pJ?>>8Zml#|Oz3b+=&N1Q-<NELw+h0CQQ+JhHJ|TSZ
z0k$XMy$y}a8k$|wyBmeM-%g&xvHFbWL?NG00UceQn{(eE(7SlUnCtEG5@Yq#oZXL?
zT-Pbv*>`!$hE?4h!P8bcH0UK=*lOr?Yx#4}MfrI}%KAOKN=(j|c(~pW;hig@nIJvy
zR=LQk-w6s<YbQ0|*xIsjX{y62<LC`K#sSQUdck3zG!08mdEY!zr4`rJcjm!bCDu7L
zo*QO-+NL%=Yj<D~PsJLUBg)wsCVY<RQzS(tj}(P}4|#KXuZ;ya_o1n=uPvX>4|Q0w
zKxW$N{dzH*uH88tlJ;tW)6~o2EN)Bp=cX*!cxA5Q%YqlD&XxFcgsAwoR5V#x&T@QU
zx7_@BNwggQ-%kZgd|VxG+Nv;fY&kj2$n!#S-^W$9C+-Sl^{ELt8Z++EUr==9=Ofj7
zFYDD78-KZbZV8*!S)~cCyP|~h`O7<|{9`I*nBq_^m|Mus_T6{Ftr@+CYZ5A+Y{~Qf
z>@a(;&T*xP3pZM~n_QV_wffWr1_cR~nObT=(|mmsgSMESXWAR~xXYO%Uy^AN&$(wy
znRm8^T#hJ{=ve$<*576QDbbteoOwHCVs?xl3(p_VoN1@*o^{Urk|-tQeaBd(R4~gb
z;L5fYJ9frhyBfY^XT!zUyIR!eicESJ{XJ#9P4MJ!_n;MCk7B<BRrseT6w7S77<`y1
ziv5@D%h^e1C-+>{N;0<nBVpsu{?$9h*N5FWV-o)y){Z>J>rzusduD~seBE3lxTtT=
z4}A#{srT2z`i>VSPGmdFkh)Fg?h4KY)skDpVye2Mmxim>-i^6?@Wx`NTgMJh$eg77
z;^+jqrE8Dt@c3@3&i3eZ-6SMh`l+iV=i=jsn>vLiWX|b7CE0$n^W2wDi>h1A|3<Hz
zHEC~>SpTwy-P_J^{nfoH&px+u<;fmhi|G^ZU6yTp@xt|2Ce!BV?e{(%D_9`2<kiD<
z>qQDY^||x6D6UOn2)?M2-ne**it*C+Cjnefug$A@uIiOE%kJnq{&@wJ+l?)3Pi~Q`
zJSnkseah{Z(=F%DO!8@*6EkPksU4x5HcY2$monVXDW1KF{X6$j7r70ZVt4ZxUZ$RS
z>y$e0-jdP{+AgzRZMrgFGUC^Sr1lkQLYJcz`z~ACyeQluY`vLNO{=ZzH=jsRSaV#N
zbDn*pmtVT{5n~n4@2%4{Q+^!1sMO}*Cvdel@KQZvw1!ut&O#3b(PlsOLz+tdCtjO4
zw$18zH8+G!<*4Yw$9lil*lsxN9{A;K$eJR}my_<ke4HB{=5|GLFL&q7fKN)tZ!d8?
z<914XmgeLoYO`<5vsqRDVhW2^2A9w-$<)QtMNiD$3hWns5?7%rnssxsjHLQi$<8FZ
zm}Q0BEOJYns#mq_aPXdUCEBFWB=%mT;NA<|K{u~$J<NREaz>6%beqcdJ))PQEB|&k
zos>JHU}|BbvhwxOo6|M+yG9EgdmU`4l6Nt0mQUQKgojd|onLlr+?k?1>(-9*tyd>j
zJbHQTk<!~K4`+!t{`H@z;_%Z{#LuI&cHydeKh6%$9)@oTiuIFiKh9Y^^~TKpBkuwp
zWHC3poz6KpEs<x_Beg02w7zX`;@>E)@j*IRO-$q)yT-+4z2i=Q+YUWfEL^(ukK#N5
z-^C#!+9CNx+h$k$mP~x(_}VbV*5JJ8dEWYOKK~XM$R-wEvV3Qxr^>d=bGPu5i&Hhc
z+%#X^5_vG|wP?kr<u|6jd{}(Uc!#;t^_>AG7k2h#nNK|O=$!H&(M1lnEi!XBWlNO;
zcXZ|0-3U|rVe(3f@%81SQvxRI`$|jXoxaWVFkJ9fiPfwt{~I-YWI9iNQRCeBG+^H8
z53-p@ReVA!y#=}M|8!g**3IU_bEk9xWB-HGB4>5ya4KHg^83^R=D3YJCuM8&$?BQB
zn<A*{e?%*7pVN_2`#BzKdy>TkW_0WjG;v_nn%?`kshY!LYTrhSF4j<yw-f#)zVF^*
zn4TBuI`1I={vbgO5q4Hq4zs2${)~wY9248-FO-N>`|UDuy{z~?)v3u-gwLq#v1Ffa
z{pr)TmotLura2$(m=XSYoyLX|6D>8X+()f*t}$&%{L%aE&6=PE&sC3wR8*HQQaN!X
z=d<^_sU9tj>ndm7sa>e4nik$MX(B@}!#oD@O-bG%<*PV$ESdDDUvO63(e#pyZ=cvn
zNKM<Rv}D@U?aQ3XqH1;rOZ|AaYsnvn^`AH|ncA>zdSJJN;gC{@X!)~~B~#r*^_9AK
zT}3y!^(swW^~1CEZV3yY(8LxY(~lJ^4!)mJa%>sb@qoOid!|jAV%)NeW%~+;!&-AW
zL$yzCH+_0%N5fW)i@ck}_1mVM{wKgtXY_)7rOhe7n`f%0zAs5uFLytf7<nYcU`}<T
zb82F)O5y^RM;}?V-Ynjx&KJ6TPQZbOS|&?BH(XoBdtTG$lX+3fyjiN50apzwn7VkD
z-?+r8S#EOQRPIc;*!B4x9G6+_9~V9SB=2a_@Mc<($?T8?B6_JTlFfo$F5UmWUI=VZ
z)874g!NV0^l2I>;1w@Z7-(bpe>x9lS)%M%MXKP#aC%k%MFg<0`ns)*dSEQG@|2nnc
zt<;nCSDtWLNW}Tbbw@otm9`@B<O3F?wbx2^lx(e-blrbae_v$afsL1+uyZzBSz637
zu1HUlTh8{bI>0xs;^FiUVQV@c%*)Vu^&m{>AdgbAPV+IX4@)#cW_hxeo-pO}wpz$n
zBrddYy^<Wyy;T**{1S6Sd`+hmT-a9ZD*h&>W#Nf!f_nX4Y7EiRo0_y4G!DAP1#LX#
zzWK5#OKtUp{G}gvaTgU9gtH&y6v|iE+@`gAi`2bOnISzoTMx~Yb}<p<n09HR(K5cM
z>u$3HFWnMcZt!W@ED;`oYYLtsHqT<*I9gu4uyYF#N<FfY!FZP}b0q7M1-#q0_O2+H
z`El7U8_#;%yZi+n{dZ*lpX8m@8CX5RvG8Nz;$xzx1XhGEO|N5CZE?JC?CfQ`=916c
zPcOv2N;o*p=2?T@-?#hrHtRhR5!h_u@oAl76Zd=d9hVGDW&hqVnX;g?(7k+>+HR%$
z%xABhwyM0bxg^@}Snp{wuU)RWtR+$wciz4&R5uguy`sumS=#mI`v#v=9h#m+4Z?4l
zmiP%czFf<E(eaQ#;=Ef`lgo{!cwLIT{MLSwv5U@qhH!^-6F5{97ANb~Zg22@xr9yT
zq2|q=pg<80p$%LjYu+^0Z#%xEZ+=VhG7jI5x}Gwgn{uK`)`(p0?8^VZG1)78h4R+F
zS=IVq)UP|;w!Ca3uvBf*?sn16dkQm+e4bb?+7x@wrS9jaJ(rdEA0<?aSu3e5UHR;p
zJkNgdyBd#{8*2(aoblxL$rav_e-`8_tzxqARNpB({qKU)N0z*Jb~-pnu(ztGZn~KH
zqbx3ugwFP+;#nqPp5dKaR&CT2n9+DoNo#3`fr?cio74rhi<>O&e)3sxC}r1&g(26r
zT;QpipmNLU?o8PLHlr=Z8*IE5t?bDXIDLOw+qEh6p+;vSHg<n7d->_PL_&>G>@wEO
z1_On}g?g*qr!gH%>DHRA`N;f->xmV|9$oL5(5Jbku-`Z71D`^#k3}HcNwJwT!Ws*f
z*@(ubz4u7wdl$Siyh}ok^J!>P;-m-L^zLh3pX^ezO=+3-$q9VlZ!Vkbkea+?{h}E^
zPH;CQ8L!oRBCU~7HnCOD>eO6s(J6}cK1btE%%8o!_sqU(Cyo`0oi6;J&PLe(n)|rp
zx`rm3!=f#=+!{?P&hMf){*;(MiQJlYWhvjo6{j`aGIWkD*|Bl|1tW!On={uw>Xy&B
zvowNRrmBC}k(DcFTrs@7d*1S@uFB%&LXUTE)(JM5ePFHppOoo>dYWfgHQ0~v3h{fq
z(l%OtgXv7$o1|8wlBrW>iNyOUNQX~TJs3YZh(lS?Dk-#zTYUDsOb4s!+(#dH_G~IY
zd2wHGwI82?=RU2F<2*Nomn4bSr7XH%mGXtvaDJ+Z*fE2N2VP{mgui?>VaiI)ckSh#
z>UJusQF9sY>GAZ<zO(F-|C2oyZRbSR7yVH^`F!HRYYQDtJ)L`K5xdUKm?V+f8=INF
zs1*8lMhh%(<#R7*;N*~;k;O4h-=l1<aI*1jRms$gc`N@du<6{i$zuo8x7rukXB%$`
z?`-|Lg?H5gwU+uSG1W^O!$r;tpI1E6>@s7X;S5Cw0neZS!Gmt@5gSEpo-@i#@!4|W
z&X>tcnp+K}+;-{+^PEdrQat@mmVsf29B)xIuYts^Z!2G@DJkvT^7-IH>jS?HGK7~m
z@X0)wqE@~<FM0->mag7`0BIu~fq6<7>Y4;>Jy{;?6RODW@Jo8h#q*#wL2hngo5{`n
zcNSS?7Uyn~+|88f@l199bejXKxk~vbo{Zd~G_x!J!M4u8?5BI1Y+0n8|KD-el~BF4
zwqeJa*%9Y<oVd?Wk|e~tgV%jmh3OmlhaY{?vU|C<>+J|syPka^`gWPiM9v_)RG!Yz
z8k^4o%2y_GU#&Rk{%pa9I~N|jK9Qi}x{oul<2vVa=MM>GKd&CNHk!;TS7+@dacHJ&
zP_fvpNwVUy;@7Vjh5ig~cr80Y?C9j6iPlG5F7&vkUh)p-xjR$U_avi*=hvkctA3<u
zREB&Lw6nTs*yF*qyk^?=KG#Lx-9s$4{oXyvFm=mcmF%@gL<>Gk-w6ub?K6+}iKbP6
z3wx);#j=&B3gq^ka&j)6DikSjNKd)LDDZC3lDN}zIFh&~on5EGf5_+bl;}8C<x^X>
zOt78h!Z!V-eBG-r;u?D_D|VMWeQMw<?U&W)xggm4#j1IZa%}|*iUe8r{8X8xWa@qH
zr}u_bkvaFb_I=BbRuq;j`gG6x;Mr%BYFK8SH8}q#Y{R+@*IFJNZZ>1x+tQ=g>CD}s
za^z5u?A5yyBLCD`huEB&W>l#4@buZ$yi$&m?kUXk9o=VH_(i_io)Na_vP084!H<tF
zofj~5V^NuO($nSFGWpIAFD6w?@yYx<RU>nqin7plk!kM!4-Q=B5L)wy@72k}(_8#m
z7fY<Q(v~vhJR}{bCSvp9X2d_2X$|L&SavNGd#4bQ%6iLaO1)OZEsjOZ)+a@s<uvk`
z#eG*O7@U2tEoBgQ<jtZ=mU(_pc5aF1K3c}sa`HNh?PMW!P5;TS!&^JWN*&GKO?R65
zI%I);?7qJW=C>w_ypHf_o_r~p=T_Bjrwb?J_N6o}pLSSo3wKVx*JGWDj7hAlL9sI~
z`3sis^O!wtiz)N@+f#1LcU4=uNKxQ#!GE@<--{L*tyT$~;^<_4Frwh~UX?dO{}&qv
zHEUJ>{2H9ic46wbWoI}pOpgD(=%8Nn71zl}?9^`cXNs}jb`EQ^J<Q6=XsOgR`Oj4q
zCJWs!o{Ht2(Hw4-nd{Q0OiF)r#VzLlviO#?HjO>4%U*vGTlKX262F+_$0LDRiRr5{
z&b_&;+`8;>wR`10gEFbYdKKG%WY@K<iZfTsx%m}x&409U70drqS8`gvb5*H}ZP4Dz
z)hbu)>wJJiM(#*lvqqzmp<f!ycR$lZA1~jDNm8`f?|Z`gbbscdGA_$GT_uk$I{1Zu
z+1`GBM$=Bm1rxnB{+w@})ezgP#wE7f>!Ap<bLWx_-z`b%g?Fa9^vbQdbl)|uTC~o$
zM|a+?ZHWbedl*;l-mub0Rk*6{z{CR~VpBh4pWQnl!GD3emGT^=xu+Hdg|JF#o!slB
z$`W>@b}8E(hqJfYe<z(%@(VCzKJ0K{X|9xOIsb&D&rHYSlCG2={$4iY#Z(gsM#jSr
zCamwjlh~4KQnA+2GN!_R%Dy_0!*@0^7d~A3$MMGzOP|Ppg-K6;I@Oz59aPQY*?i^H
zjTQ#wmksKhv=fXDN4fuEo^@tPlNi_D*w>T0q{UVo`oH=9$J)iqd^UErn6@2T7OZ$R
zQ|^sOkBF-}OWuuFFV;*E*>vyn?ZwOzk*ceDrax-DG}kIt@#KS@K@kT}+5DKnI3-bQ
zBj4PBM+z_IHt@RMx%xqDnt1YaR%Ryub{=7G&NYr#gASK7hwgF?J)x$)L8zrNG%V{>
zyYb!asVA!MUW(-DI{HntP*xz|+4M%qbAo%`e{h%^ap23^K${Hpd!}guA6!4{I(V$w
zR+`i4oZ(RJD3mv~)Y$jvwx%kvxEXsaG}t~oN#SY_xfOV+X=A_ORY4_R`AdQ(o{=io
z*4e7rXr-nv-J%$=Hl^tP@44Ukn)4Pe?rSlc{Clmyf5+b(O0FkG&-1e_dKoI!uJ>9h
z>8;V(*c#D#tGi8#+s%JAUpm+;?DIK>Cn<DETc_vxr+Y6*f1cvZ=DV}zY?ec3z>6%E
zb;{fvhNt_@PV?HHPG(<k*QB&IA@t}eMep>E`ph4b;+o?bl^Y#RD<p#q_sMo_7Ff)r
zVOW#sS@8VZ#CM!CU5&fbUOoGqSgliJ9V}DB`DACu)u4_g0WZGBbQp$m==4<w@BG{1
zamqTfOmgMiDG$~M+`qCzP>OeRhnZkiT<D&nh+Bsh4yyBH8pSTYxn$QVqa~s<kK8%&
z*70=csuizZU08Qr=cwayw#Lr?T#FsvZuORs`c>=EYSUF$UT41I=*#sv4O}yw-YxMs
zrIoar%dp`_!<_Y#V&+WR?dlqO;@6rJ`iHhmF+a<@{!F;$v8g2-QEN{<UEMP$=|ZOB
z5!N%Dd$=W|ZkK$1={iI6%eM7X{bWzcos2dWZHzr#*zOhIbyQ@FOx{F^JU7-ZpKU3x
z9yQ%uy8aQ%dG2>zY;A5LY61mbe>mD31LfF^W^l7_;^k}k6J@e+)8+$9C*5#jeXm~c
zcG>A37t8(=mr~f8Vp6LXYJ|#NoikzQi#$sWE6$zVqSmKP7&BP?PcU|DeaEAGDDbDW
zJBydi3+F!ZWBHp3ZzOhpGg-kK{^PB{KXbcH8JD={K76rq_RPo4y6NAJ8#En=e-q#K
z|GCT2%T25@CbM{cmft+SYmx8UrGX_rQ!)-7-o@nAmKd>X(iipxZJW#vrF*Qu$!pb~
zsJ?44*W}Wbi}r}D{Kvs{?1EzZ^Xd!Q(q6BAJUv-kk<60yt@wiP!WgS{d%m5JnNpq{
z8+<<2Htc#?_4#*fOWVG_Nm(e;EMAh&^1`WiYt`i!ivzVcYUgVnRV(@z;dgoBqW-6E
zKPa;${|KA$sB3FiUA(5PSK7a0XI@91J3I53(eF=hf2gdubtC9$e=EP!=iNmeLie0y
z7R_!_D&(1(*7>1JLh!6B*F2p+Yd85P2JTxVAi46>lw9-6)dub>eN(4Qzc+FHtW+cY
zz}t#JyUqtHoDE$4vd8U8(y{U@#Q~RREUkBA-ekvpcGhhtuMax0!P7P=y5CQKbfIPS
z!KSQ=7oS7e^OHJ%n|1pBb~0@K&=W9oh3|x|hh|A<U5@{>MchcvUgf@pRNNAa$v=}N
z_n9vb^WcBHqw<z`<%&Jqrp=SxuhtZ)eZVtsZqf4xE5C*F>Z{h487C#}Rr8(S;TW)0
zBf63Om1vl_&&zDbXMVPmU#Z@dm?SLDtNk;Djn{5hSZ&6!noG0m*Ef0ydv-n1p7x}q
znf>yzgSugvRi091Pwy`3ZC$IFaNtag^Z~U?7qYF?Ce4d{w_Ej!zf|d$$VO8Wx5Mmn
z->v>veC_<NJ6FuFt9C1SCr{h_B57K_m)$>kEqxEYSEn{@Vf}V^N|{aT+rm%Lg~vtz
zJALJKd;a#_xtTdVC+{^J=5Q*J>DrL|ZO_S=ojW>T=-CNK+9vkT^Z1z770vN(?t}G#
z2eM18qz;G_=rPaw7$jR;=yYpY*AC56d1f26G|>q{PJs*EEVXPiC&gRJ?rIO%`9kla
zsHb+VwQZjD+ur0A)<+X~j+@Qj<)Lo6G{oN5%ka}`_m-7gehAxg+4H~p=8!LbilOmN
z-2#n7QDf0*w|u$t0$Z7+qvW=8NiK4Cny_7hE5n>m%XjgmiE2+HJx`_25ow;n()EAR
zOcym4OP)vDJkD6pePYCP{hOvjMOM#&V;593dhPf1aMd*%#gyH4VhG!?f>}$|l}%vf
zA_sm3_4uL?2c-r_<;Am91dcL^SjC_E9=gJ3-jpR`?BSC_k6ld4u#^!on^Ua!WEJ<>
z-_1?O61ey*DlSdAHRW^DKDoeNv7PCcpF2K&@Iv{aNKk}l;|C*~q-N2d46(5%>lSp~
zUCY_*ku#O?B<nk=Z=E~pzsUH0*?q#bE#L^}>7^n!8yBW&9C6`osn@KL++bH;$Dwh3
zRlDHrcAc+h6dYH4Fl}NA6f@e>Gxb%<fd>{3+}(_qZMw>vr&YOuWA4-ywldnn(GAB{
zxwX}6Tupho0)H5-pTTUy-0f!;*HSs>?rBEhBbp&8WqtFcQw@ZwFUS`L-0kz96X7Jl
zv}sA#;pL^gy~|90PR%}h!RN~L{7_|CJF^EK3~#3W+`WRg{py@;#afevC$}w5XbW8y
z^1YSe)v<)KC*RT}+HS66f4o}Jn(xOso*UmQj<uapvH8W!x7fvj<8Evq`_`CUyoxcO
zx5r7QOtrAcnqX+{b@6x!BLmMB^&ZBgRYxZVPHR5F{;++MRBZjKfV$p4rAHm3bM&2~
zrRyfR$t^qh&|u}}AN5E4`n+O~t8mYYp7dSti+D<%+TT#MsbwEB^sa6^{qc~Iv&V5A
zt7Yvg9vqEb_;Q}}iFGGd^lGSDEM6ES(kC&wm;Kue2_MdE#gfh`TW5Cd5m?ds|IoLO
zhkmon|JrEO+vfJ?_M*im7Y^2>RhgVy^Z$V(_lDZv(;rxJeeHa!YstNJ#SIfF%ez<G
z#QSRJyuR((duU2R=*goT%kJOh(K?emx&7ip8Lh+nwbmEN)TcJeo)QtBxiY}@MaPvV
zuT)bnKlr|f;Vt*W`-TP)_htt6WnB!`GF+<`pV?ZeYM5~{cM_j$m`I7@YAx@NHz$Y0
zed+6T$#dgii4)n_BDa!plhq&HjkYV=l|mgRmU$Y#d3E}j{~E)H{hLp`ZcyvzYVLQ+
zQ4LbwKYhvE{wW&I<}EcbEciC(k>sYhrPZIOG&%@8-j*ohE@8Ckb^8joNm?A8-wrIh
z^1X5e<MvDXy7RfF{QIWTdNEPr>Vl-=!VOP_57sDp)CRuLiQrfiQ4zKB#q1D~sH2MA
zp(%?5A7vdbZ=Ibebf(H{>Wm}LLl5?Rem0Zmyjj1?`3EMA^Sp1Fcgk9)7PW}4Rdm>>
zv-)#L!$y{1(}#Ikec!$G*4|K66`94caeMvLiE_NFnD$Rn=e+nf>C*Hg^Rm_)5&y>K
zVfU2R<x=yTgF%;=928GUsjgXZ_!x`vDbb#;x#wkPUhLdoS*PB_AfzDgAuTJi_VCxO
zhn?GgZOTkEzHpZ7@6|i})|dHi$X@Ww$h4Zb>1~<NqP}Vl-i5Xgq?@7w7S7%Dn`0uw
z#pYYeWug<}Q`i~8bY$MjW>xQ7D!1Hj&aeEB5AxgZthvN;usYZ(*OK|9ZI`~M@}utE
z{F^Q^ai<6~I@zqMd2nEjQ1#(Ir9WP}s_#*}8C><o;d}StyVLF^T@3Eu`Id{fqO8FE
zckUO4B{LRyXz_0fSYQ<Yb9anm+mlYk=cWCw;(J^Kp3GsA{JDzl!6TuzH@BQGe(2KX
zaZ%@*tjIXoN$6<?_rf=SPWYFs4)7{_7=GyOj6#9LpgA*rIOQI6uiuiLP<-=Rf@?pI
zewH62m;SHGKCf1ISVydTJ=I=&`i?#39qVipe|+nEcxPeztTjINY<;m0c>hFg6`Z%$
zqo!v4{l($wrsqSS$cOGevDoGRZbQQo(Z|mtQ<G!<ag^vjbaxHZPR&W4s`6v@iV5#l
z@JM?e3;YqU$8%KS#1RFB9SgqbJN8YQlq!3>EhFrW+0Tf-fy>lZZDQot=d4-S|Csy6
z%aEN)3^P*;XSyG15xgxUu6CO1<)@>i;b!qxA;-6;e^2eQ&NNhBUt9O=_36iflfUr?
ze{-=dpJKmo%k*zr3yK#dxgSj7zBhfn)`BIDpZ<szFgF>jIAbKi{&Rn+Rk`I7Z>e`H
ztUMXjudbdP7T4rAot5`)>S;ZetcAN6?A$6EyC!v-uU%F!^$x%7-p*R#uU@5MA;qrK
zG|E@GGc`>uovH0}OXv7OrT|;X%cr83?X<aLb|QcCo8mtc=O0Pjq3gGH1Jfo=@m)W6
zZ09&A>fj!ky5Y<f?~v)AuJ;u!@1B?Le(jKG*JBG^BQCy8$C#)8Pq>s<z)-qNz_9Di
z43#(m10Kl~p<gXFh6!i%d(Iz94S&UCaC*b7DECwBGEd!>{!y5DVp_|=hHH&!H8nwf
zS!|mb;wp<`zAST|>2##!!AGx(g=z+sGDdYRPX!)ods^t$_}fjXI2ceUoLQ{oBHz+B
zVHrb*+ac9k3Ljsu7MwA`E?!oit=eg?<()Xom%kJf(tq)VMH)-)p3Cx2G-BHR(mo?L
zCxwCp3kj9%Dwb-+D9Z|`NzXh_3urA{px)G&YUHmH#Nl^jqwliqOr=d69ny7d{yz*)
z9yz~a$xCJJPNBLNUzlP!-ks<QULhRP#I^6*_T_btR-ba77<0!g{>ZG?Ee}KQi<^dA
zbqr<Sx~g$i(^c+O?aR1)MWe2*cDwRNP;2jW38{M#0_!IwG^pM=F;C^ws+y(l54#(3
zeD}TdRXo>gBHFh*aDC+Zf40(TDKqa(5|{HTI$l}ym1l0~&NuV-Hm>1vwH57E**|m6
z4CU0Wt7_9fyP3H+6om2o-{f_Czs>WB_SG@fOS7}yd`e4>{dz=x^1hUbXMW9eRmf2@
zxL>hwd1_c-*pbIywhIZ1gzBtWF{h=~bV|$K4Qz57gcTPz|4CZmbkK88m_mN)v4Tsi
z(-UVJ`8Fo6U@B&F>8@KLG|PE<P3%4QXLq%(Io8<MrSBDge{a<a)9-EHc|501H+#2Z
zyZSptmBmYn;|_)%co%w#@kPkmnuo3n8q$xtHpXXkWo7d$%FSoAek_!|XUpwq!7IIM
zeYd%tjPnYus<$v$vuNF-^v$I*QZa6;_HL3_7d=yF`J(vh`?K-S3uiBETy`)s=*OF;
zd25_{4rL$KmRa~xTgqnpo(<ef+#62sh}#^;oE=y`aVtmUUiMqBBMMhE#L21!pL`G|
z#N>EaxJp}rz4N)hTm7FIq0!B4F^Ws%zwmtsc<SnIxy^5$_5Pb)B2(mg^$s;Vm2Tvp
zyjMjtOlx)MtaFMo8f)d4f9tHt{&ZD1mG!Gi@$Mqa#-9s*GPVAk=s5q|%)8bh>h_$O
zzCU*^x)8{@>1`p$!VEdrFBjY6me1ofUNIr^bM^9v|5Y~^1SM6qeA2$AIw3K6Rp`%8
zH}Y4m&Yt#l!-7-u)%WMtOsp!3oX^Z%INjNOsr-~6XZg;eu&sA90v~+${aPH}yhg!U
z!TGxOdW-5@tz{d7Pprykyb;QIYD>a{P-ed@^U5VoGk?Wgap;`o|E(oc+=cm*{-@-N
zo}Ir=gdIHa?^n;%JRvXXyQ!XiOwBWzIh-@9wzTOi{}vUJ{v!M<%fav~3LRD~awS*#
z`o8%F*}Cv=bbRX9625k-!ZFSd+&jMgi+lO1`374%7iVmNM{L5AxM@Y!iyc+`4Q3n`
zy}U@A>8WzVF7BS-`HQ7K{K@v@<v643<zQ*ED8#Sh*OMo)jhu?EtbhKmIzN?HeSS^S
z`}C%bhwh!-BQ1G}Raw5X|J0oJrt>0fjFvo0c3r<4=z1Zuq+95QTHJ>Ji?8SePJPR*
zv;C~@wwfZ%HWr=Fr#r-_Rk$t<@#i~l^>I;*uj_n|9xZ8s!?rhfyjjyb-%9(y2^9&w
zq*-fRj9yG$YweX3GOf~Whw4!utuoIE8+q6+FYvs7Ia+S{)Ff*$m3^%hH-4l(p57b3
z&nmk_s@rPn(YdbH-?h(_c{g>P%3Z&~A(it>i$`Pv)6Ua(cb@+3nphLGRO2~^ufV2=
zHLZnDohF@BJ}IDh+O_1g>lcN$r8*IeeH<<G9j1IpiCJiK<ID^HhYgl8tl0_cFMT&Y
zyUHl6<=-;4{;b{NqMd=S9AAEjd4616Y+_f?uS3%xOy&OCeTh#?RAHU@-2VzI9J-!=
zo6+pQG-P65_I#Pto+Q5Q3M+i1gr>7!<Sa;B9drGCL&Ee~!cUvyHm9}i*Up~&z^?Wb
zgXSu><xke!{9<dJ@ZiCMk~cTJ&-tsYbe*hzNQSE+WZB9iKV6fQyYf#4AFsHOa3%R+
zxP{r}_tB1P!?sq0?g^>8y;Ar^im<!g4%bXBq1x(2hqf$kD485TIVkVZ)sz3PtUh+<
z<xH!V;w87F#3n{8JCM<&VxOh#@pX!x$z(zAdHZiKWLkF0eB12G{V&8SL+{D->hJPq
z+irL+eyQoS^?~b`JUrvytIyV1m45V?#KoO+r|IxcNqD~6Ig;(C2vh0l_|ks=cSV~t
z0(Cl<+jo{cP~*C`;L|#@y=~u5y~<jD>X0~Vu7{$=%Puw(2JQ=2UN(I0bl5%9+-=#f
z83zxyeGT67U}DeA#of)*l^-OFOzqqte|#$At@mNR%8c_u<UX1uJ^Lti#audJOJ3Nm
z77c+-Co)=3e6W`ksZh-fc$HePMt_-d;;Ex;+*3XF%ZBS@u)M5WwCx6yq;(`SBNz7`
zZ;yn@sR9LBHTPdEGke7HlkrGl+TFwX{a061P5gB|&ieRAFRv9McaG*A`e(P$dDaqP
z;X@BwL^9g8oJ_v4_)M3q+M(9?>oa0^9&%#)-?XldXGY2-3AdFTVhed#)RbN>(^S$d
zeXOl1WtA{d{hx14;nP~CrNy6_V>=ihT10e&YH^3wMgCkG(<puWoQ_kK=!(T0N80$b
zHER>jI<9`6F!hXl<AWJ9E@-xHH}ajhbLp>sn}x?-&oVA`Nfo;k@45Z{mW_M!8T*6H
zMOn;N@TgQg5-_#({d|~diq@og+C^WD%O?h1+Oz7L-_mu=xA$|Etj~(HiQlwv!^G56
z``m=@m(@I}()2RftfIqx$VJt&*QnjqO_l5K^qEDC3dh8szL|LM%2OAP)f0PEmFN7v
zy;<vH@zvF*IJ1<q7#>wDdvRb&_;HcBGmLlzPZ*T31(myBttvIzH`hxgywgog=~c<{
zCudUJ8O|*{#kD6hDK_<c$EBkmOkZd0xaZ41{m1X_I2GqAl~4K~7M?R<RJxJAC%j1P
z1W$9V;wMQ1)@EZ>af5xU7X6R^TW-4G%K4TbdXGxN{Ej+?KT_&rTmNds>vDr_eA{~U
zuQ-JFJBzwqTCH{JywR!5w?`ecKZebd7pPow<@1yGXIQOUHmWqNQ_@JT3;YuG^_6IQ
z+)V?$V@h{+DalmSE$kBb%XagY(y}!MWz*NcGRPE?yl9%zd?`z;{k^1-ZRct2N58su
zm?l-PTy)JmQ*!dOiFK?-v&9$fe7N<O%!!MtnU=<3v-Yu<xGpFwDPnKgUf_Cv-W(B@
zCjDEgI<q27E}oixLuoOudNaRzGhb_!)0HkKufzowYZU)?O#h?WRhM$~R9Df|3Cmrc
zW~`FYlsau<D>OU%PvAM*UgrFqEw3f5>)*^`ohB%CIw8B+mCai2RpEz+Y}q{53bNI^
zcS%dV_77t)48F#(&u#Cr?@bMw>tuxgzvjq|n5y7g7T_Gn`=jV*RpvphZq@cB2Wtgc
z-NHmgrnJ0ibxEl@yT?;1qSx>Kz8-$g|7o+f3dOe=^=9i=W~rZ8bV7wYm`BphO?!)Z
zsJiF+zi&BIT-3FcYJ!X>mri!tZTz3(Vj^#v@rTy!;Scj$y<%cMgr4AR$yx8Qp-=qP
z@<QHA>__6*w9U3%{;^od-~8W=n^Ecgb5(a&y;*ZH(wWney_@gfvYH!{&-(oPHY+E+
zsG!)3doEvY$yL>$SXs?dsb>+BeWzO%?tACnbbb{>TKANsrIY%*?4O8V`uO4$^Tp%o
zTD31UF0zJS^=!K6@V)C*qF1^}q`}KePqtobeQ`t8u*7|LI)`$7fTMre5#NO>20pE~
zmM(mC<%8RTySHLb_8$4WRORYK`={&5?y>AvpFX+d`lsW|7U?Y3P<eam&?NmtcXP9K
zYj20lH`Tf=cA#&<pExewfUd<0H$ONL6Zn|Zq)SzH%Zkb0tZw{YcsQibSygJ!C!t57
zykC~E&C>d|COniQapk&7r*5gM={w5J6jxQu(yN~OS%<lK7So*+{;lCpo0j*T3herL
z_{r+$bEopH-gI}#0>c^lfjxisZ7PYMYGh~{!{;vXA!w!A<Qp+7RjN&|P7vx{7#MqM
z?YWOC8#M!SxLyPZXnQugNp9J~F{x<DmWk`!w5IAz4M;m2w1-1-tHtFm@yhvTVQ+Lk
zO?sD+v3KdiPOttW7Ov9`L(*F1)gt9=B$qGeiCmRz7=QZGJQd-NCkq<A{<5-&?or#j
zN<kx8iNnA-^3G{}@&02X_jjAE+Pg|2Y=MSGuh*ri(i+t+b{u=9GXAaVij@`lKTlM0
zZ70L*W%@Ua%o77^G6Ju}FZ{=}KJ;nCJjK5nC+2Z{4%m|U^w8Yy4Th~wvqV3g7u?g?
zyR2xb$IKN)8&}%6`zbT5m73Dgd6Zq5JwBr4Y}&24Hsv<ndyHO-N2LhI7sO~E6_i?U
z^mniEWu?fYr=HF75zIOt9I7I@^GZU<w$79NK8Y!|I}~pmKW*3M<sTZh*fCk|mz_(e
z$)-0YrME0NPe#w3HRa9TblzRBx2{^aCcJ&6?<)^sK~`Rm8JZ7nq<vk|b$EaH(V1T&
z*GD}JzhtA@e)zQ1rpqUL=VY2@&W(5J?aZy;Yw}Nnf41AJ+Zu6ZCp+Kn{iI;z%J=f;
zfn6zoP76N^$Y{L(QT;&Y5+B~S5B8nUr%9ZdKO^;WoZ9Rs`TBw9&d5)_y87esGj?q3
zhCf&Oetof$Euy$c;gfQ|)V%-iPh@YbNc|h;Vm9lQso<xD@2%Yrwa5f9FJk0xd;4ld
z74xbEGdr9Z4i()_?+uPhy1x8$-ux%IGoFNWg!iiK<;+^8zCg)#|5L6>KQ`~N6wO>$
z6THKmOMK~vdtN3x*R!cJ9d8t${Lpk>th7sJ`diIM>r=9J7FBRZIA*Zy2tFLW%|o@Q
zlJllR0yC3eMwMj#d4(+foVA+|c_;92UFhUnslca@HTn1dCAGd=HfS!1w9%-UP_z1V
z@%k!J>r1h_Pt0s|P@j=nb^C?6uS&6|h;C)zU!}{v%lEOwWnI&@SARNhdFN7&)f+FY
zw&GmID*0*CdM))PIaSYuKE7XTLnfY#?9;ohpWbxv`^tuxrIYq7p5eGiX4QH%zOz>z
z?~jZ3`r<8`@_6HlN2xJ<3|qy`H;B0WnU}LCSzoLArIA0^^I4tcoI6!bk`|mOS$UUf
zf7rUJ6Z@uKn*S-$b!T?*!XH0fA6HD}iC(FpJ;izPq+5bwD*I($-fch2))kfaYURoW
z0eMN^&#!uMb4|PXg69iwbvn-XYB^`A(mshrB}Va}(=T=bM*GcMHDsse7+2)}nz#On
zj%(($gziMgRg+TI9Lx@P%(RtFh`m{|;z#Erv96XMO7>qKy<EB|B;ey&#$a8S9RewO
zPp$`CT(aHfDc4;tBY9p~!_KnAILp-$Uvr;rUv>W4u6-$gk6mNy-m@S{F>LAcnGsTq
zvdWbwcORBKD8EUq=BMk5rt0sZqHFF6<W4Qww&D6mwFT1-tQTvKX|oHDJn7MQVwYfV
z`+Ut5gV(Xsmdu)(`tf@9;$z_s2@RGiZ%hOa{`4y6Jo?LJ@iS2doq)sMn%&|HGq|?=
zR_)DwyglWo`c9dL9v`;8*!pVg$&?wc;hVJ{HHx-|X!eD?`u+B3u|4y5^$w@7#=9Ym
zS1Y`hu5g$nwMD@<-S@?|rGXoj{wr56UA8S)NpsFFrNRii`PzFWguHZ9xF(j`%wIR<
zR>mpMCzjXkGV=X>zeH(zdROc3-6qo6-o5d~aub)DxmgPeRi<>Pus#lD&R%unisNPB
zEf0BDEMLdAORMbbozRV|7d^lH%C2dlAdjhmNz=l%HYtg$gcXesd3{7@Xc>eiY+|^!
z<kZBSrzY;?nz=(M+yAn7?E0LBYq@MWpF??%th&C8ZIRZp1>vD0@oZBM?%I<2e9bO4
zt%O^pS$FC#mZn&p7YtX<`m|`>{uQ3OO!^zQ9$R9_wD^=#psH9{AzLU9W8CCd=hx|<
zy1hB%^HqEQJxLomx2I`t%_w7@AEeo3*qNa6$(EsH>UD<V<v!nUZe(3xv&w7b$`aR#
zvySV7*Z<n*R(q}Z@-6mGA?rE%GD{B3-L$VHCHbXh{5$*kM?F(AMK=3+hW$Kre$V|Q
zXRbcn#WK@p)fKIZsSg8Xq~_;l2%k_2YUlYDsy0Pyvgg#)Bcg{)6hc%vXI**i=e4a=
zD1XAXCNH<hE$_8Y+{-tt{g3<5qD*W@8`WQ9Xn237<lvFh3@vs4j&IrEBeY?<$=2`B
zt#mkoMNFSuYr4t)RIyz@P-=r!N!;ll*An<1+;rG)nRO$VMb!1>d$;-0%}aB`rj#|7
zNgv?<_miE=t-+i9@7_bdU+=eP{Z}8jprUb=`99ZOaqZ$Ws)YAu^<2-7J05<c&||4=
zBgY|=z|}HFl4=#U(e0A=-gs!Q%nA4={LMJ4&wNj~<-8|zm$ZmKF*@-t_G8dLhx>kx
zkCe?f8{HIO$o}j1??L9jEerR@E3lvSW_nfj;rZp+e=e8pdw*iK(T>F%PUNo3Py5q4
z`_8x8`1{gKt8abi)V!Ca$heikbwY~3r!2py+wGg`SKMw7&s-H(^LgsWe-D}eYJR9+
zaUd$Jdx@J&KI;c*KE*rV&jdcT-IxFJz4(f457(%3fBnK}v2c~2%PU9q>)|#E25#Z^
zH@rT(x@kw`!&lFj{H+M+O_z84$Lr2{E|Yi3l%4v97ml=4^d@i0nDp$+?$UEh3Kh<#
z7wxSy<>SiCc1fzL`Cm}6dP(D%Pkb-^0;+yyYS-^S@aoIfs<(cD57wQQ+|<8!zq!}t
z<JD_pzdx0<e^-8F@5YUncR^5Kza8&#XWzBeWik6JW#exOtbKSg_9Gidl0tRY5tTYu
zpZODC3LCAB-f(gK%ej8<ldAYHS8bT2e>?q;akx?a^SA%S#lG$jXwXP)b&Sye{O##m
zu?J6HrrxSLzBhQ!j(j<tv>CtEH#i+%e%Y=wrR%`2BYR9GlP#~VdLH%mOR{|}(?mrf
zC5fd|YNot2ne;??;rhSz`OCu7*YBv<GUHFB@t%zTw$hL92Pibj_a2(z!sOI%@Zy{?
zznTJTcGnlplH}$0%bFHT-Z-wgBH_Y!HV)r{y1<=pb7Xz)r1EmCDq64hJG?Jpv%~eM
zT?^Z`-i|-RFng7eSd+NPeYIU5*s2OltG3*#ojgHQKu2-%ztR(&jvpVCq@+EY!!uuJ
ziIv}nBKG)<bww}s7HDbNZa?73VQ$_uy<u^UN|gRwi>JokXR58f-?Z4|zTn}z{0}8;
z9Sv<7*S+uLetCMxf@$s(52v&G{PU80a=F9pe%}jJC|S&J^-;Tp?Z8uCXNDZNiv=(G
zD+7LXRNtL6iSu7z^-|uy-F9ofH~r{+SDh*FO3fp*@po0@!w3I#4!l0jnq|-NuVH)D
zn{$(n-wW^F*#Eujq~njk3gcRivb2H+eJiRbtw`FTW_;ms{KV@iN(^G_POds(7dLrk
z;HMpGcU3=%7MiN*D8F;F`O<J)-qU<fWud3q!(>I@xQ5j$a-W?{UpQY+&zt4UOmVfm
zGaU?@!;h@BoZ%O3rNR2;;X553|D#`T-aEycdqH2bc<+Uv<Wt!k`(ixee@bwCJfNE*
z;J>5U<5eN+#aKI5|34R$oPW)jXDrOf^u|w_i$#7`$CAc{%qK1$v(_+<b?ge6DyXd$
zXcGSGMoaXGb+ff}^m31O8H)<Y-kfp8bkEdeYga$V%@&IH9Of;zD%|~HgIb)!wlzmS
zsf(2~8dqKO_l)G3dHc#Y-$QQ$IGK5Ob$pO*dvxoX)1>(<i!$7~^`_O_IVF+O`MEc4
z2BUaJV$V8*dup+F7P4<po}wYB-G5d)b>3l)J@Z|5X~<li;yHWc+u&!@bhz@9({c{a
z@@$oEnrRW#x#&IT0;gHr(uymT!lWNgYdYTPBX%^8y|DeoZ@-+1prefLuU!^(aC$0q
zvUIBKFjCRq@u6A8uWj-&FWrns0**ox{aOs`k5#HUyz~BYao({)-GI49%wi(TW;|j!
z!}_XzLiHnGmBsPP&m?IsU{Up$!dP9N{eV~NOWGa7q$4sMe7Q<$;l4&uZ+LGM+;Puu
zkPwpo-^+4fZemmZ%LP(V$8yu=3K}shyD#%*-970>cu<myeMv@>lHV4k`x0iY9hx;2
z#!B%L%gmB%qS6#9+znrpnJUSx6|9wF&TOgM^fA?`w_#622m7bhQS&@Ax|Nq-{L{H|
z=gHQqJZeYp%zT?!H#c~;M2sqH;~kr2V)}_KR)%w=gUuz+^lsgHd)}7iDrKv9Ue!cj
zxvJ3AAnVX}{X+X*)@4WPtba+ByQu#vcWk#XKd`C7nC0>Hb*@i3a`WB31RYK(XP9Ie
z?lPe%AYzZ#(<Mn(Z=xzrx8JW>a-ZR&>Hpkgl9%q!*1ID4aYeq`{QRnOJ~NnQIJo=r
zBRoFIUTgB@P_EywcA~t-t>?$S956e0M1d{dHT>sP{jJ4O%*?kcq`xn*%{g*f?MryF
zm`UPthZE91xg8wAEoy8Q*B?LIy0<HvKQU|9p|*14MPW=c&;0E0c<i$4+>x$}%;vq3
zoAw^KQ>hg^Rl{3xc}ug<_D+qB+m+Y&yvd%o;(AZWJEqU~cda||(DI`Bv-(FSS_^8E
zW*=MR;>VS%U~}7A=;`qpOU*fw)9oZ~b23;SyT=*3(rROQg>3YR<LfI`j`fu0E;;pC
z|2DJ53l7eO1+^C!FYe!y_@2}GG4q8(Dv`-HD%;}(=gd7@E6(hiV8tlCXX@R~$_pJE
zK6EW|cKDxhr*YeMWp$Y+LhN5;r>u22ZvB(1d2-P)`;+sI{y9AJt>%fzDibvRRXjNL
zCrHKR)<d5RKE=&X&whD&U@Fg?)Sk5(Vy&DhCXO#G(`^iP94-EvXvpF&s^MeMpl>O|
zE-CJ|#5Lx?0hb5w@<db?y)9$iytk1nd{NrWhwT+LZmdRqejZ%0$J(AOT)H5%^MT*l
z^Q?!g3bL)mBsV#WxdndoTlPgjFk!N)t&5vGXV9ayx)D2{sZ>twmOCIL+{nW-qrGyk
ztgAqxYCe0)in=R7pFD)q4<1PLT5;;m&V2rjjEu_9lt0JZT(@;43uCg+6gJ*P&HNF}
zDebBaHkp2pHo6-s^Djwrv1c<pd`=+Y*=<2hD}B|X%nlakPswb}jovAIZL1t_|5wQK
zcz3E`neVKQCzm-=6XsUj%gWvEVZ^S;wa_nrj^hczJ@&$jV{R^b^#8*`O%1Mv&!#9U
z>AH7JTqGxIU2(Eep;(~X_-VVw731wf;Yw{2u7obVH_tV?Bk=cva>;@T3gr(JS2?XO
z>~`}Kc)%*<%F!4kF6J&ZCH^&o;8WcT{46^;pI32d_$~AhQI<%U&9JW5S>R-&+NHfc
zUk<jgGa645Wp`&Vn)}fF2OH1Fx0<IEyc-xB4TN{HPn{wa%-9e=ucA|A>Liyds+?RW
zOs1YX`_wV1Bhd8HtQmzVoBGx=X(YNBmD{MdW*n7P7H4P3bmSEdVoS7rW4O_5YP8BC
zT{)&?YmVp1duDlBsI;&ga-1{mt-+@$LRpf>TBVnBK55t3Xmv_va?wTR)q4V~9>heb
zD>R32b=xn#ao~4P<JOevjtr{aOdFs27Tofg$Pg`hNH{~`LNRMyo()SxNC}Ui!bBz}
ztC^pg%vQaWKf0cG&NmmUAM38XaEY;)GS7oeed@#;O`VoZ!Y>tqY?7L#UnlKmW@4_`
z`ogS7%=LuOvs?$oFJi|&oVM^jub5xHNw&oMR#rSa%igWq59R1K>;Ex!&}x%0>-p<o
zo+5ScitVf?3GW#eZTMro#<(j*K>AH5*U>hYPggGR9zHtp&`aJEB@C{6h2@1N?XO)S
z@@$QF!ajvoWmBCWjQx=+Kho!OWedlaiymE^WE}ZRI>xQD;RjE__H%2B@06ci!v4%G
z+dx)bwwvc?<1e;PTopOiJGmtK4;=G+^35ux!^Jw(ZGXVKNPW*LHm-8V$0ui`b{t~8
z>vv4Fxwx$&yYN^3lM8i@UkrRC9@o?da%U{kQrB{v;;|ri+S^45lU@ck%Q4P!zjC}U
zH)J8hrqH!B5*WM>ceOA0+gI_=<s0jc?vP>*We%onr%Ms<B%GaJL`gTih_r4#IpwCh
zV{XYaDLH}9cV9?`C^)=$<e?G~w0YYp_2jOLDyGl7vm#a`uyZ?bw?r{qaN2(?udHb2
zt~WyEtWVa<YfcLH+}!su_V~MFPUSqELf+4v*t?V;pS^r*`MJe>S|%#ZF{0Pb>cn@h
zu<%Ih-zj+`yg4tBIoP0J#+heJKl3VwUrIBn%Dt@-ywmZ~-Hwx!R@CX8;XgJzRZF;}
z)u<%t6Yn;84IP`^^DM4yWB9yYiB&3ib(iQnhOH{|CpSjV@c7JN*tc$_1p6YUZ@dSL
z3TB-8a>tfo`T<wz4WIbTMOVGo>X6Ld#3?x8L(7Tv#wvc=`%@47O}x(RaAMM=`N~Tb
zBX=HL-4MOs`OATyQVi0Y977gvZko2#Vye)pjWQ2k8U8<Tt3vsrrk?3z2Z!@*3np+}
zcGlJ9h+1a%%GJU5L=<l}%O}SUr8CTma}65~wd`7u(d;4O^W4=^t8l;T(g)t$wa-;&
zC>VWSknte@sDF_)!_*%2c}2&xCvD<=^|3Lzb-7ZAzfvIkt5#>Gy(@OF$}4|(@V!Yn
z!*l0K1EIZt6g)rtU=~u@ywY<Ao6VjD@j{j@rw;sgG39=sd?$|C==fz`qx8}VH5M1s
z1vq3rS+cD5oLR8fnX9O2_9wgg%kLNLbN#{St>AitTU_p{6Ndx)>(d*GmvMxByO!d<
z+Bx7s?A-<X1kZXO3VC*KL+SIWSu^J_&YER1QLu#n&Qk@cpdvZ$HZ9J&-qn0xcIff@
zoVS_y;)H;q>&z{dU7Kb$T<EUKICAu$ONx_j;kV77&b=0%IoD;Iv{vxl#07=TMgJ~L
zT)yRC^;W?W$2mMFVisSq`*CFE?!K;6o5Dpu(k5LlkYk<2C9unE$+G97RwwM6FIZZs
za>#DjpE!kKXJxe>?<^y!2>l`t$!Ix~!wzpBmvqM`S8tnmz_967;#Kv?Gv>k9nC8cx
z6vz>8zV-BA%7aJ|-oLGn&hzYfuW~p^Y3buH?OSgiU{2&Na}f5sBpoGCvG}sWL2KK6
zsjC@`846oN_{24LZgo?XGU?;`y)DCCF^HY*kccR!iEI&@oa|0Eg{T5=Wlgh{n_}~R
z%<rFmVqK)uKBv!C+sfKcG_sekbgh=o_PeIgxou(X8<Q!lk+X#sA6Xgrac)J5wEh8?
zLvF=N>sa3zGJRKhQ-1q~Lbkwmh9gVY&P)B~7*}#De8G%!BKj+)v8>6AEYD8<r6W-6
zqmdc?EH=m^qWxsddY4aPCnq-S5{&1lDtj08Ta1P8&<kk=xun`$iv>k_6By^NR&fkC
z^#75FEUV4UExeXKdpPehuD)b_BDwjc;jY-m{3UlhmdskP@aJ6FFCu<-PF&5tAeA54
z$ouPs?}7uFmQ{gw-W@Ray^xkRGk?~a-upGe(gL49En^e8QMDskEyJl!`1FMx9X!jJ
zI@I%T*)9xMb5JbRzNW-)_N&F^*RiFcE2jIFFdVAv{^YXU@A^XKT>ZVqy-!8e8p@|{
zeD3>N?!>IG*|*QA$=^8Q)9ZcwTie;32~jQVXBMbd6eRsk-G2JXllQuB1CAX|%i7Mt
z=C$AGwUzVv35JKu`k6WnC$Bc&qB+4Z!l6LcY4y~*e2KA58scl!4gP<~V|uH7n|0zu
z4{Ps}Ol@78IaV&oUh$Pz<fqr#ORIe<Kh6$OXU$HnKk+<Qw(OLF=xI;B{bD}phm|k2
zvnI8acCwVSUPzLBvd(eE2AA#+%pyGx8hj>})|>cf-)KB`fT`lwQ`S43x;6_d4Lw{H
z&B`8RE7f|=oWmkC@tCD-!6#?+vgi^Ul}ld_HukS;Is1)!=H5dt`UY1@1YYqxD@qbG
z+EJFU<k|M;>r7|d&)H+M@rzH$|Etl*i=!?plqX!y-N!y}v$Fat1%GCVSB+m~LV8o~
zHL1m{xXiOAlOfwIVM6gHHWl`_j+f<l>Tj@gd|K1dv(nvHK-#GPXLg$J7EOU4Y{J}c
zPTj2VsTX}+C9=fmnDC^lkJFZ}vtO{8t@BChrKX~94tI_h@3Va_{ATwJr8|`j_mlst
zDL?t{G>fOU_Wk<@mCxdf-ZS0$ePgfTu~4&LVO<Mcdwp`g9RGWdH&e8g_wenk*Rj|2
zx>X`0U6+ds99_hC`2NiWCrp;=G@V*8!)Gb;1GSw;TmShtZQb+GWAn0@hKl=(H2!Mr
z(O`cv{|W!pr43Ji+DG-O#B<)f?XXwNncra_$NJN$hFZK9ad!oK%ku98w5s*^9BXpp
ziIWff(R$+YJ=X9aYHv)|NX*P^QulCk6SA6RP;f9q|LOJ*d5OnPT={U;+&Ecc<2wPx
zmdCd^%GvdAh-Dl*A#_@6I~(J!X`h@**6yFYXa0=&VGpu9E_nDo%sd`Csp-s>S59}Q
zt(bJ;aLOt%^{;grk(CifSL7qvc>Y|{@O-j;_FF@?7wO6#a|;tDU+d0!GUY|dp6xxc
zd2i;O=4m>9wDqIX<oz8A6|!?wPuyK_Smd4e6Sl^iPSVq!oZAw9MW?}qM@o0a(X@RM
zXLgI+QSPv~afLU}I;zz3tHLHeDg6o-qbHolPF~<?UK6MAc|onFhpNJ-@E!dg42$=8
zI5JE;SnIUn!Lw+$9v@HTvZL!=8?U=vaMDy?7@#`e^=aBwA+7xRFV;qedq|%wbGhJ_
z)XzT4C`W$MK|aUehR<SiI#;j+p1IV2aMHxoBEJO_(qtO)T6Yw#IQzPN4pWcBmy54_
zWI0{qC+oDx+~a&w7?jb<%ItK7W&ez&8SQNs_Uq;EpCSB0j)U_l_Z|1CCrgYp%y`8m
zSyJ7k^lJJ(e5u<Lbm?cah|e}&kJ$z>54x6qaOB%MscXXShA_p6pWL1*3hZWOY-Q27
zSa9&3^h}o9cjr#yj$<^5N!fK|iI3jlckfizoi#ldxkl@5Zp6VOC(nc~U(_Y5U#@Pr
zy2oq5Jk_Z?PShyfT)cyI)%6!CuO?3AH1YkDptY;@Zd&8yToYy%{^bS}@12||b%pOp
z-$XCfJyZQOE@yS=cKC|jjc`;uz3r;^zZ0K*kJ}h1PG0uYXY#YzQ3{jmHqFs&Hx22z
zpL3x=|I7t7$GcmK{_3Xh++MhSs;AIf&J@FK=eRcc&tB>Dtmwt$mo1sgb-VTR41PDM
zG(A{!+B~Ajg2QoTLl$paSCd_Q$>B|JZfv{9!{RHLeB5no`K0Y1GOi>qRL^<S(zhu^
z_Vwzb1JiE*(+xh^Ai-aHN@SDy(+iKnj5m8v*=}Tep;GO_k_aCu$sM9U)6F<%<R}ZJ
ziCekO$XagjPl=z;?tt9GcdK0!cYfHRajUZJ_2~m5o81hI4ynz_U$wt+PE5zM%w-)(
z<$oQ_t}2~nvhn3A$(g!uo51(15NXq&(%&vy1e)<lD11r&Y#yVvA}3I$jeXmF8;i}A
zkMDoVIeXQaF^n@*-%k1L@A!P37?1eUX-1uuKgyFuXIEWccx^?A<@<uUE2sRh{vGDy
zdfqCbW3Ogh=l6ujIpO)Qo!d+T#iVZ>kIQ?-{G(3tHoHsLp<0s-GgCS>6sorsZAe$;
z5Prx0!r$hX|MJqQTRP@^itC$kT=?7u-m+tF4`&y}y?AP9sQV>)&J0nu1EF3E%a$x$
z8+d5i&8^~U9PDr36nFh_<d?6pN{Hcmv1^a5xXITwA8by}(QHy|KlJj0&+GQPb}v?>
zN@=ZV=;Nx^G&#_=a_=QgO>w3^@e}f`_cXuM9^$aNtX5=i^m?Uyk<cTB=UVYIC4N^u
zkN&{^=~UFx#bGLUjcub<H`<1&EqkGzA!6Iaap~oZGlD0wT1tYZXqisp<NLvWXi4A#
zt<^F2oaFUWk7-U3*6Lt6-?nk4{xjQ#XGX!wyiSMTNDD+xJY~|Pbx4wv$3{chCozEQ
z!KoQ00Y(pt7b}Z&JX;djoKvvy)TRgX`nT*7wMj~!(l$*fP3poajsMjFr!}rwiM^F#
zd@SCUa(K%_alRF4Q(PE#y<FDIbXMff4^K;D)4~Rhc2m&}4<lL@=|n3m`WDmjaO*p3
zUfF%A!i;m~x?R^}%1HUsdL{GH^04Izg%_{B-+pD)7pttJu6@cUbr&;>?W{i<7S_0R
z_hY3k8%y`|KT_IIC>XJ3`aWs*_x$$XcSya@ocgBn@3};g6`g;r`<xOLUr+gJ+PBY2
z%JFI<ujV`Tsn_1GUR?NPU+qq{M48DOGW1y_|IeLk+~a(I;=FYcW&t4*zF8Ba(&r{k
zy7G1956|ZRl5>ML1#wQ;vg=*KuUzK#89U?61rqwI^(LNJwoOIi{^Fj+7nUn2wu#71
zQ4yIa@rBE&drhi+M~+{jZy1+dq~;5zLn~@}e9j2oyZv|eB7PyZH79Ow{K)oj^MMx8
zN=`*V;U{wso$)$iqyEz~gj=NFlc97j&$NaE!5Irv&$JsWa6Z}UZKqnOTN2-NBKV&6
zltl;RS32F(d@_%7sh_6Ex)#nSTh*naEx5gore$dN?u)WZT(Yv4G5VV6TKUML=DRft
z%V%D@!&1dnzKNyi)Q&|gS*ILM+4LQ&ovn5$kbA?jCC+*u*{7&ZEls=h$m4R7r|`<`
zPk&#=Oq#2I!l|`PXG5Gbx9@vh(}2T^+&Njc38>9~CsXp<xAd!yoayPz)hEo?zPhtd
zG=AOn@I_j&T+`g1=SH*TS*$+sLQ>gX-{a&J^}R0{bJm$H|M}zf#Yy)rW&cvJ%;EU<
zF?RYRMK$^Ow;RrfNNEVB{+RY^qFIiFh2qJu9GMw&?oaJp;q&&>vT&=z@3gF2vwEX!
zHeU$Qe5|{B#s0Gs1SIW)UZ!>J(EQSAeEEw&fa^bF;Vrjn*1AphkGk~2mudS$=Z7L%
zN49X)dLLT$At3#S?`P(->!*IRwRm56_NA~#`FeAqY5#-yKYiz3b5NqKQ~83#a_4Ee
zN<l4AegOh`^OY(mY|D_y<4#jldb2h`_oZaahDpn~m}YRA$y&7go#uGF{y;y=k`0q*
z9Y3<~fN#R}RM#aTi;Z>(Ib=C4c@k&LEL_#Pa(kGJdGwK6Pi)w)hNOQ!vS>lv(qEdH
ziM|fqQ5K)e@;$AY!s6TX7|y-Azp1UB>%;oQvqvZGnRPir=Ip+DiQnxzYb(A!V7LBy
zU;e;ahwZf?>jYf?zpnfh{J_7lzLrs<rGGNBb$*+>%fDSqJC%61hA*_~eEju&Wrp#S
zr|)l{@U1)WwA+dGe=5s{eKXu-;}?f{vP;}FTh^4s(qPfM`sWTU*2Y~2&SceRX+GCK
z{;hiR6#3m2;xq4y{oFQN=zpE`{}{%}IseKRYy3Y@^KUcjr&;}z{>tX?*F;}`boFbo
z?&^ftt*>*7|Lf0x<<9!)!{y@*XX@WN{NKqW8JD8}>bh}(&6!<RBKubUUv?tet)?qf
z<HHo~6|n(37Zf=@Z92s|m0K%WKvXwmOTa}3S7uS(5R-u0TK`_IU$|i749OqzGv(3_
z?>O1hAo^Q;Z^@TSKPSlEExGb;#hpO#tsRcv*X>|4yl}{Yo0+%gm0Uu}KdFM>ztYb9
zZ`=Bhz3g<$+MoK{73CIPNaBrkQ@i4FUG4GTr2mK3<lfFnnmv=DyNdNH*DBEv?KN#1
zL*2h@<PzPgofRIkZbeMs&P7E_51eJsUU4^I_rfCQr){Trr?S_{{<vDe`-Ztr?vG^d
zW!1_BS3BL*+f4%V7u|B%+L<MMRjth=u(HUb;Mr`0B@b0Us(xPjVd<x(AD6Cv^uT=!
z^YT5O^)|nLJW{nzd-b9|KlT4sRxX)~r<2mTE!Gq*d(Yak|IA#?KR04mUQdcY@ayYQ
z!(>Hc@lT#=cV{)#KPpdk`nBy${#hlnQ~dK^-w$Az9nQDJRd1n5`J<(uTNm0ES~a;Y
zdnv8oDJp#1vsLci{~s2@e|j(0EejQw&)@yxzR|||{SNx^r#frjzLTpvmiBd<|Fr{|
zb5~FPr!(!_(X*y|vNf)CMqJ;>^|++X?Oz3h8k@n!G~V=G_0tsG{|m}S9=)CW<MHEX
zKVEkmefaTMXXg1O6RytZsYu$I_UFfg|8E)$U;Ox8Q}KWA%m3`$J?t)D{{PeXSJcR;
zW$CEG>!Zo+cJd!<>l}p%o3)y34&OZdkG1!2<#yh?B_)?Hdb{h(?k$=0xi8OZ*UJ~n
z7SHZG)qL>g3LB4O?Z4NjTl~MVe}zpz<$@x|r>EH0Y!P2r=K8ejl;~7#E#tBd&WUIF
z^$y!!_@^o+rDYzHx8lyK<Rzj<uBPu$`se;`>i+lxTcQty7;N|KEDOFhQ|Pb#19g##
zOP5ryh^PrZThbJ;j7dT5xA;_Tt?;PT0Xr9c>i#!#tCm@(1N%39NB)4F3ym73&wL4H
zU<wv^9o0}7Ca^p7!T+d+h=7LEO$mxu-mSO`NruqNV-}aWzim9lJe6IGKSXYY&5sMr
zTiLU~%)rV;pIn}vY@fHnHn1AR>^vnr^`X)64RsZF+K%*piQ{Y4pQ_)hKUcq5KX~0P
zr>|WtdwzU>yJLSmgG{Vi{r7&2i~}M@1&`*ZE>BM0U2eyC>H4ymewLt~pZhB|{?q4j
z_>uDB<$qPj$M3~wbPK!wx%WR&?BRQHg`^TS=~=6JB=qLYuQ=V6>=Yup+tx@a&&cWA
zn|Q1GggouDek(3~QsV8n>ij3eZAvcNzh!@a-%m;UU)RXPE|!rgy=BYU|F74r%MDsK
zwRF9CR_5#nrre9g5Scf=W%8;Ov7D1!?n^@QO<9=3+pb%@Q>7u%5im1jgFaiF`IWpC
zcVHLo%;mouxO;J#JLJwCP*~Nxc=*8ho4!`RR!!h<aZvc#eQ|l)c}sYzx|V;)ycITq
zm7f->XZc@gi)!zC7x{2&@2{P1e`R8`8<>S}-B!-z+|n#=CVaE!L$=cF&N)pdI)v}2
zzFq%*!8utL{({^&@BZ_KA3SuydENyr5w;x*%3$S=c!=KQ6Fa_SIlYA>C~(%d)?QHN
z`WBSmVP255UXjdR`es3&?3InpR%HT7_HW|tO5ES}-IASZtyLcK4wODWEM(8(zaj@>
zF8Ji|wDD9kzinW(Pzpc93!x9Hzv;_YE&4X+NEvTUXN_W%&xsnrKicxL)*TTaXFiyF
zsBP=?6KNci#aOL`Em`@LrM2U4_@o#Y3cUHTX=>5$SF5bvZev?p+rD;}bKMJr`0}Xb
z)^D0+mmTC2y1MbKRgahKRQAIEfj7k#ILHS#EQ(tos2MO-#BIL=cZkB~sMgL~U#2IY
zxf-LW+mqJFx16DkO*Ei-k=c*;4*zC%h3*kJZ7s6zO8td%|K@8p@E$o}_QKS;Ol0?&
z8MDr0NbL!*m0BUk8`6L5Lv_=ieDM{*QrlPM)#>`|w{dCNbD!~Grq?CE887>8_;~(r
zs8|)Uo?kM@@p5i`bk5$|!@ZwY@0UxRn0d6=@#f17D}J&p-QSqWbL4b5!{Lw<Uqr-@
zTiDOCj9sU&qs?N2V`TT2Wg+XE7R>LOUErYcXX{P2Z5zzZb}#?jl&@A;dF;!R$lRp~
zn@pc94S)Il=FG#+xsmR?ALDr+<Uf9N=KK_%451X!&CQZ4TWt6yR9I&3P*>l&d_$(>
z)D_Nbw=TXgm1@wqlqi?mvHP|5+zz9@Qk|n7t222RPRQ{p9}POJVZ)GnHS_u_{xl7T
zLRqgVZMGj;dpJbTBs4r<Rqt!`<!GDk(^=JT!j7n2{kxlc!%yKYEE*mw1RvX-+{VYy
zz|%D|_vJqJiRoI??rh6Xn3DK8^i%B`*{jp*9@$J2+xc^0?}ob<D%Y=CZdJNesIFRI
z_Hv`MvTNSw{;=`QbUv}aG-lV^M>*C7r*B;~TBf$==k4HwahzTe>yKw7ofTihz2V5A
z2O<v?z8_F;GA=mox7j$F)j`fGkVn~~^7VrgX_o~HBKtYF@Gh@4Xpk~@khD^KFYH&v
zJ~?9H_t)QTEtu2A3K~^^J~QZgDJ>Z6rE$E?Ci}5Oqwni|ZJpVy>aAt_D&J0Y4dAg)
z6F7JMrGtt{9n*T-$Mg30tJnT8n}0LYJNw_c70LhG>SkLVkvlmrv7X_;OOs{#9!Gy#
zG`#NT_w_sTef{RXT7KU7o%^SjJ<oN&pcVX}Bk<a#gNv32ZF8Dg%sijFyR_r8%L}gJ
zj3v@1^&&oRdohV~?!J8IuV?x>*uQ_avFV@RDp_84yE$dT^4@nT3*`O;o&3E#^wfW?
Lw%fap+;#*2xAJ6r

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ag b/third_party/icu/data/icu_conversion_data.c.gz.ag
new file mode 100644
index 0000000000000000000000000000000000000000..bde20b6da6253d866f87fcadc7e6c3571bd64d44
GIT binary patch
literal 177696
zcmZ3M^T1A8Z*SX)DUA^kOG}kBUb1K$6*;D6xUA4YOMO}L<b?GqH&3(0aFqxy`0KUn
zXo%Eji?f>Y;`}A6+hesXem<{zbXBZYsPfnb-Lu+lXD52^^L0r)@M+fhrhjjPG^aN$
z$lChGTzy;kMEzSc>X=sd_ceyCn0w9QnO3L#F}pQJ{VlWH_un#nTQZ|leQVd&Z((hx
z%{+P@D4J}1ZFRGU!SWcx!9R(=H?rQ2c0B#nnde@{cb6IS`(*AU__^l1lTeI3c;6;B
zUdvZ+vh@yU&WbAzlE>Nk1%rZT*FR5xb<0<jf39GR(Z;>6tA3b?e_Zi&lT4hg$Kkw(
zXHDO=PB3h8=G~;3!8gmIPi(&2_H|R)y$;Qc+H#xmKkwmd7yPbDZP~Lul_$7##j0KR
zKYY8+87-5TYvjDBEXI1S++xFBFTLXCyn7|Tx$Ul6L8_j;`9GT-2UPTmeLnW?|GsCb
zVW~!p!|_>P)%#Z7H=Fadv${T~ontZAztH0+yzZqoZ*DcW&r80;Z?h@#XXyS2U6Ipv
zbNgL?DQ~Txv!w0i`pWmyj0>U;$SpNmVHM6_$8GvM?AWB`n<C!r6<NT%?&OO9hChWi
zb(>wZ(yKk?62n=)Gq5?byzljunAQG|ZA2wiKK}HW|2zD6?O$WR#plnb9?Px2HTC|y
zW>LHCmrAC8WuN=$K%U%<508K9&AGUfN6KE_+N5IgeM#YSCoJw3)hX88xT3+d?D6Im
zU)~1!Gzk1(|3%^0|CuLP+g1pl=ljg{vCbiwr=(7z(DkU*gs18Z2c{j+;W<{Adci&F
z7WeP$05vCp2@M62_oe2(y1wVL#B!&fDjy|T?@wEh^8ZP*_LIrha@!f*@8yRS?Mk#g
zbs$*MRrKVHihq5|w>g|*E*^Yw&P~wejzrR(b-N$%-aA((<(0qH@OJU(XTGisJ&d)&
z|I@Fx+PxD_G&H;&<G-rnVZYJzFMg*M7an44zV`nzACL8Fd(+OqZ4EyY)f->W<(PNZ
z;tqo%`vDzc+fO?m-e-6j^5VJRp}@vE?#he$XHt%CWAV2;sF8bniC_M+8Oxug@7-xV
z^H{9n#BMj|2MqoN0y>igRmJ$0@%_E%IIFVLe&xH@5sKMi21V-^Pp(K=R6pxX!J89L
z`h!+o$tvCb^S96}tC-Vn<xky?o=h^*esyU^-Vqh`h85G^=@?Jgvaw*QSb6Yd-VRfr
z-fd>8DGv`;S7@Hn*_HaZUVF~g<2SySu|I8?^={A2f);~kZlRIM-*S{~m@{p&KCKp<
zwPi>6+wlGS3+|pdrf1kVIXE;y!uNlnLspq%>cv^p`40W#aFh#lb8tw|QoB1xTYJa(
zGKW>G9M2t2nbzX4H+S_C#<QuHkH;oxFYPXG@|v>IxP$ri?1kGc%W5MZ<?Xyz`upMq
z=d(qN?7l7&W|5loe!`n}9q0GV1<T9cG0!e}cSbxW>(%q)HoZKHC1#&mv~ydR)>O;&
zt2&bx9KHV6`|ZCQM{)!np05#&Y<t)qHS<#w`_mAQBwwCO3{O6{9KX!`KG>==d=qa<
zpoEA?l!!?D;gh!Z0u$|CC}02W$G?A${qJmxyS8^s8E<S_zUb`N|H2KIb*9BOmuXtX
zt`A%PIP-43&gY5h@vQ#kcQ$Gsp4{>8X_Zu3@t<8B(`S~SdlCL@hmeh#TISzlizfuM
z<km)rA3J<t!J^nh*OxTD&{~jKwP8Q^mWotOX|@-$Dlg|+7ajRKZGA${+j|+8@~-YI
zex@8#zB#jBzx>RO+sXT8S>I}T`0(n*zsrtFF3QZd<qmP{zC6u0-B!>ia3&j9z!5ID
z1@Wn~o8rB`X6l_<FKFPctIxRg^SUgf?6<qS{HDxU{OWzeoR*8(P5&;*ajg~Vl~t%u
z-DQ;3V8OTJY*KsE=Y?S@wJUQ?GKv^x8!|-PwoTx-NpMYT6Wp5{tY*65;J)c;^^aB`
z$dEh!=;#5RKSJD|UWuEQm4s=ZE4>+d<B9sex7)Aof7=jQ#>M#dsndhgyPMYPKZ<;{
z&-bloMsA5wUuDkS@2hf`b#2_jer?r#9og?nZ@qNl_l2Jix-Q{9zc=kd;nMomIxiN)
za)$Pp@N+jLTHQEsI&jhA&j&BGzIbBhEH&eV5zFZvPiFb<txjw_pnvd2X_}o>lAe~W
zjMUT#yW+A0KYUwLux43`TkTzj?eR7*i`t7O&0{v*#I)|i#5q68rHs4ROcbrDY_GYT
z`g7$~!-Lv7JfCf3H>~L}=-m|I`|i8do(lKJs)7e>^jhN@YsF%u=IZEdmJhDo;>IW#
z^*B7YN%Z=qO%8AEc5M477xQbW<{X0~rKaqji=C}9D@^3xZnT&9Rx>rXH8b`2#y72&
ztAu;wT|afdEPOo4?nP0emY}Ki$-HB8`;ThM%?daAI??Lw6ZgC_bxU2n&(r4pH+hiC
z_WtPZ2Ky4$$F}eHcv|~N^%je$-t^;;;{E=U#pYUzP?dmH@aBN_7hd(xmhvQQ)eS1h
zVV>XrgzJKS{-Sfeeu3Nce%*ff=UzL@ozB(e=`0I5ADA|7)!N-+Gi}~?*FrhpvU+#x
z-*Yb<u(>$>&6R~!8BhLp<yXc<oOM4bnR?)jnr5PdsQK3;W}ns_z8ti3)#+_j9jkTN
zYy!4Fu9`C=wkJX8vG>9CUsPwbFY{RwAConYZ^DYeb$7p)ZMu18&$87&%I3IE+`-z%
zcg&(Elt*a)HY2eDH@S*AcLT38gfKmpE7H}TGWU(<Vw*TO7ICkcsZHNEMg2bPd2UXx
zb-M0rnH<}XOFh0?*;Q<I$~t^8zeiwRK<5R+)7RGZ&iy{4)pN(;bF&v-JQ?%$V_|0a
z%3FWBW~nqK1}+Kv`{?k!uJ6Lkhnj_TlpJ^(`@3S#Ej^?5xn#EOrLMOTSC`+<UHvj!
zB~08`E%4@(I^_@1r5%%g&6!@7Z1m=cY^IvQ{<}$&A9if~KKJv#vmaGnT7*oF&dbe9
zp8aPfd;aaz|1Z>Ix2#=0^~}%7TKCp5)_?i@{b$FIU1ycIW$pOa^6zrz-V+<1elS|^
zC3M-p_{OB*)o%6^mi)BOJ%9U0QR(?RWj`X`3HhZe{S=7V8nCS2JUIRMmJ?e~|87{d
zJ@*sWx{sG`@JTQ{-XO8Hq_RYM-Mm$gj+;iPeY|;knwH?Z5FYmR>$^Ai9M3I`TeRgj
zb8RL6)J;{*`#v9764#>l+iFVHl-|pnKCawb>+bOTSX=%+>#jLRZcFj9{NJk*_ntZ4
z`})?xi2)0%ZfCn!pXs_8ezA1kFU4~vRewu#jV{;79R2dn)qLeK8-?Y9+bW~_`b^(t
zZrvEQ`t##mycrLUy~;DOOnrN~_^<R8&gPBwTUxXKn^rvCzS;fE@@)^2woYvfsN>m|
z{6hDhT!M(@oE-a}e?R6Jd-{Jbuv_3P)vkSgQDo`QTMIgq_aAV7?Za~H&kMbeU$#pt
zsNMOk?xg7UE8vaECrQ?ZbEhlfmseU}TfKH?8K1?>*DeQUg?DZGc<8*$bgRGT-@P*0
z;KOgd<m&3Cpzs*AgS+N#IPQ74w_`$^KYPyC>6PoB$NpWdXYY6Hy!go%b^j09{QRQ&
z?rr`GmCvW{c5V5TlK=N+f%~pFU-{b%(pzqRIXu&*;iIEYcDcW8cuDv!JHyorW!TkS
zjVh(ie7Lyg#IkFRy2;_C0@u=xOzze2pMED#dRn-9{5HmQVtE_HG^SirpLOx%4Ox|#
zWCbhXyfbczzq-$U-Yja#Rz2%}Sbm)3PW!nVJ@-mZIrywrv#!)3l-p10p{(%5y<2)?
zLgYSI8vABkkLPikU8>L4dg-CoQtga;TU>Oc%dIzR`n^&tbbVp)@-4@6=g(Vw9Q|6~
zI&aaLlWzVnG09TfhCh8`-~xm1Y@&OXO?mU_(4D)UpFi=gzuU=tLO`VA+aJ-sy~T1(
z4w3nf?;O3_yx_=X4|S6&-FR-bbpe02Fcj=_D)T$~QhD;Fls8`|+pRNp_n09co$@bs
zPIrFO+UFHk3oZ06cqV5Rtn+$iyCH{J+30%it$i~-R%DfHaNkvb@aP0P_qAD%*`3eM
z+VsAEdXnD$FV7=Q(w}Rj=-IF3yYlH@KxSZP?oQ?Tnn|1qjCQiK%WTB{M7+M#mv<@n
z?~a<^?8p9YJeJwDZQ7;US6vp(Z7I6yf`;Mwg7bIP8~Jo}-z+&`x_~kD@s;(REt8s0
z-<$P!=4rP*@eilxPJ5_P6Me&GkL4m!gFl}WmfpTvT*<q-=k>o&_F3k6#<y=>zV_cz
zHQnO+bl05B1zVS_dtsvSr$}ho@5ndH-M$w3Jg~C=&@K^GB{l1t{o^~Iwm1JTWBQZ7
zrZ#fdC!XJzu9a#goOu3H?`t9VonNn~oc*mnZP~$RiF^Xn-I=`o?|oGd%2+sU(Y!e;
z>TI~>xqIBTYz}88FMgox`eW1Li()5ZH%~14`1pJK?NjRV)~8lR9NYIze(zO-+nV#{
zxK-J_PWF#`qB|#Py<Oj_C$X&JJ$HLIZ<}rY^Oxq%D;JL~ZdR>1ysW_V`3*?}m%~0E
zEJ{Ce&sGV0elAIFmEe=ztF9jBGVcCX%dlYi(o4naS(=FwZXyZxYrNCW6yK~0e04;A
z+r~LN1a4nCAyRooK0$hY@QsC*wOb#%&EC!5B-_X5n=pCx{uP&OZa!V$$G-jDYT4hN
zhqG>+UzXp{UwL?2!kT8b)XaBs$DU7qa_5!J&Rg5QUAOJzUblbK7Pq~Ytc=N;hb#BW
z%&)Vx^Kches^yw$bI)$EnfhX%NE@xAQHB!FZ1&7meKLWUUwzNvM|WPo`@-yQlQfB?
z{36%s{*?d08UNpYoVe`;#~r!9TPADGW$%{Lz4&3`>y>LcyH*@{e09eLJNvV>ud`=-
zexjW^^XfddUH`9EU0$h`x1-g4W5+(bjvMugyz)7W%;guC&CvWJrJKxq(Cc-IxWr7a
z+8Uj_rTgln3(GlPoRz-$^a}g)wa1s={m+#iI{nLzSM@&n&r8J0u9+TWkvulF&;3e?
zVvR%O)ohzv$8_3{yRUt!5%zv}mvZ-V3tgVKOL7cOt_YOeaGz~!p0s7GrT&C1?R#tW
zf)m--A2(Zd{N;)5*{>CRKYr8r8gYKN*vltp_|&)XReoFTTc5qs*59z`#`W`FDpNAQ
zH#a%<f3{p>9>1|)S?Ptx;}XZE&gBhzcbeI)`SD{T-~V*hTe_RR#Ie-rzOCf{=jMOj
z{a47cZX=!j>?up{?D!hD?~b;56(`TW@XmPG?khFdrabreIvQ^O&C$NB%Klnr%kK-v
zYPWxXRkG3R@}={}xApr4!>^Rq-#hkW<^0Z5@$t*n*U7xp|MsY--tPUU|8Ft^6LOa5
zwceduw`D$WT8-npI-7Zp{jD`;?(3Z6oKQPuTd~Re4PAMc<yo$KOlR)epmnok{%_8@
zlP~TUc%Dx0*ljSs!uL_AR7#t~l1c0BG6S9pzGv>?Y^#-F{$XmnBfp{huEp94A+c<M
z!`Jvs8bex`7~da#^wF$i+R3g9D-&=2`29dLD?M}JT_4-84?bo^<);VKOx2T<m{sx6
z?T6P_EtCI;5{x75oB!<9+WXMjR$)iQ2CXoog1689<ogNEHB>H-vi@~a)trC(Y@W0a
zZwoR6#e<&;PxgOkBy(>Gm%!n9sn-(}!g}*>D{S@^mwdBwW?olNiRpT_#n)<<D;`MX
z@c8Q9p2(!RI6>Rt(1pXtMH`pciH2|6IeoflL#g`D#I;%0Vn4XM)oRX5FeUv6=$V+L
z@@v^?smtPXCp}SHt2epcvbDvmB3wED%MSg*b9E8yHE!m<!DX9!S1vmqwUjk6QE}g^
zLXK<u-}segZ(7-Q{3FA{w|DKI2S@xmsM+Vvxplrw*l*t+-PQt$j4LaARc~o*OD*Tf
zW$V2lbv5C1uIB}RvHKf#F7MkjTWiBZNzVJy2koS`|8{e9;E#G*>tQlQEboI*Y3}Ug
zug@G$_}L$6WxjCW$o5KyLq{s!Y;<(#OkQpKuGEfWds_UYxbEdk>wigGywYFff7kqd
z!E>$&`@GVh@^75yv$0l6>a&|bWSOY<559`z+j{ZZDXTnx<b>ZisLpVAO^##Vcdk%I
z<!|%d5(9gME%M~KdbZf?(Q=EqsF-y5e0TV3spROC<?CjylJU26^jWj(WcNX%X|?rd
zj9>ozBJjev_LrpVt;dso#y3CyG~N6Dxdfx$cFxHce?JV^|Ha#&JLg=@-uHT6J~;nz
zFqW6+zy4ADPVM1GTXxx}_Q$N+b9wP^M-KBh>q67SuAkXj)_e4!Z~d&Vb+ex&@4NQO
zR3|&ydfn97dE1qE7WMY8{rc-^T*T1}p8E2)EzdeD*>C6K)p_S}Yx{m_@hhKSHhlkT
z^0-jD&bR!O6<5!q7k^FOFJD~9VQxQr>2apV%C0{urQ_``{w#RB`gYa-zj=0IGaeS6
zs=IQm=RTjweVMs+_s*u2Oxmlf(p%YnX62WiYZL9S9w}tKAS844?F-{oyJQ8QsLRT4
zxqing>A=g$W(g<XdGeK|t{1d-GVoEHc4qE#sQ{4%N3AZ~V-XMK!o=2v%GGU`H&47<
z-P9<_!?1wm^T}E+w%CSCT6?|~@aOXU*mg;nWuHRU+&AZX{EuC9H*5TG^l@`?kfVp-
zhFjZrdOf{((z`nO(62c2JPEOPZrK)VO!>X!E3`CT@vty%d&U3n=eO(6)!*K<yWH}r
z%0PVImd)GF&hsx(=iemtSvc*)@yxxVFK!01EfI-Th}7#?qU_OYXT$q=TJV`0kKD7T
zuYGl;^rg4DjD+?i|FDgx1a`d;y&mNDNa*145{E|*3|l$pYCdKAv51kAv1r}qMxHd*
z1(UX=elAzmu34e<>VZ{^=*)t3t9;5#>;l%*Yzmvx)sreD^I;0(|Gp2urit3vysBCj
z{QR$#aD4U?X=RT~;aS%@Z@T{AS-jrt_JJJ-g%`Q)>U-p_Jgd_8g--m|f~=+T_bekl
zJ<IAh3$3$jJ3FzlQ|*SvRo6y^OTVsW8trVkrF-)nQxmsAx?UqsF6*bMI}b5?mGZLs
z#9ew;E~PNdu+Lb0UDAt|*@3hBFa2tGm!P7WxN6<KwVNuB7EM~Y;=Ao3hhJ$g{<twl
z7DUA9%D?D-thQb>OL1S!ll*eVFW=V{H|Q?(PHo}jyE60h!Jb7Qw%WbctlA$n^TD-i
zl6q?c^?St^{pojKRM%5@%x-gQ*qX_^W14+pj&&u@G2fV`x35SpW4&nO?Y-|rdUva=
zGdO%|vqB-y9Y&7JU0!nnOhdo_W_K+)aPQ{&#|Ol&GYQ--?kzAZT6_B7O2*9}bPn5b
zU({TeULSYqXbofA`<&VH=Ow&&{H;NLF<W_2|1GvQPpSTC>Rq#(^Zu}O3J9IwW|?Lm
zc!f{eExzw`@gyN(t9jR+Fn*ZC%{fEi9<Nk@z`J!WE7KoKD!u)4IpO65f9C_vxq+8I
zilnpIyUOh-yAv~cdC}@?r+8QKm3=b~-+4bJx3!<)N9uzLUxEExY%82zXRTTix6*{~
zO~u`%lk4QK@8)>dQhi)U@@0ek>wk;iPI;zy!{Dt+h4q%=DV06h=N|BK)vzpDlCvu+
zZ4*Os<xbPnB?^6<;aW+Hd5(JUa)eGbcsqT$<WG_K`gsqIOJC?@Hf+t=8P>XGn!}7w
z`UO)17TZ~{&WT#XZJyFn`@f(0Vct9?vl$Ng`RsM9XSjCrdw+PcJ<dJ&T$slHq}ykg
zpIT)g`9S8OWB$3-ApwWv80zfx-IZ0vqA&lvH1S65>PJoQo8A~`1o3RjOq%#?0^1>p
z+}6!kqjS>ARIah#WeARvxuag>a`o4XJU52wH7{nBCZExF<65>#<7B<Tx8FN_tPV~y
z*KaC|I<)PKL__8czRZcwOhr@uBi5EpH|KI|e)Ev~+-IXb+Y%~nXsE5bd!Tr}`OcRM
z6dYpb?_Ba!{^OdyhVM7_n1z=;zq4uH2^Z1%HcTnz|6>GR*@-{@yg>bR-ukSQf$M)O
zb)@KWnLSVVv_&bV<<uAcQsJV7_Q&gHoW7eCz1nZf*0^;C%m0NN=AHP!|Eg%I(C4jf
zq2_niT~#u<xsLngNw&s)pGrdmQl|a2`EmcijjR~sKC!N(gLh~1M_Wvv`#=5QZMGjf
zQtM(CY!lWyv3ysp<LepUyXOC_G&uBGq(l1Mu_iHlu8Y~HJ{t?NSRJv=Nc&V*<0fuf
zy7!9jbHTOFQSUd3?YEt{o#{g5n~PV>Ee<&Uoo%Q4rn2tU=RY+SX=!&?H|NEOzx~7i
zF)nb=o1^)zSv!MrXN1pRf2rz7irI?uUez^fHtXI!-&QmER`09-_a@D8D`Kwrxz{YJ
z+U9#df9<Dt`!6ppys-F!Sw`2w1y#4yjW_BZT-vkjLUGd%14dSz9S>Dn&pSCq6a+1?
z?(#k~cf)?o`;Rg`GHXL;_&#ktymO0*$AyUeuNPiPaPQf<UPw2#|AV(Sm;1!YeLC|+
zil^_^)?K_(qKGlPtUx_ad*Pm{d(&($eqFhOl~e57<puvtIA$z)m3HHce#@x`x;;Oa
zO#RiAx<#e-((7-UJt?9y)sL5@v90FLY2|eEeR+n(ey$|j42Cs^{T_~2M9V+-&9^!p
zX>EH>@5W1(3Vt(3Ih%K1q7tXwn<>V#(r)H^LC&cLm-L&Kxh=jZ`;N0xQ%2imyH~<#
zy&1Om7fI&HEWXd6Jo&)w=xIli-0Y4UXWx2n{g?M`kLBbUTGe(Bt)Er)uh`MbJ#|A&
zgPWMN-3q0fANH3$S9?=j&X=@on^%p=2XE7hds)_$u8jHp^mCtxZ^7F1ihbTn-p^C>
ztauyZ=W6gf?Q`UKo8dGeU*}5anWpt@pQ3ZP&HXb()K;I!KXA8V6T`dck3EkIIvo#n
z>pogw@JYh$7-Oo!ju)+0Uu>HmWZVDt5W~!lh`N$Djh76Zx5u_U5V&>ek<@YXOOuxe
z<*w6YnCaWUK>Jg<ompA*ji}%EdBXc{o0K)*VAY=9T4bp#_>kk$k?Rh3!a9CVIC1c&
z>7RYaudOX$4K{tarGK?5qwqfGElbj8?cu%6WbsKgAk{|e#6ymDiCsHQ)?V5;XTc=d
zz)CmEm*PE}oqVr%hIH4)<}n<Im^OLV!)<r|U6+0wa9PmC@A>tv+l#Kf{@7CU?7G&x
zhgp+nbGY64!`9(bnl8HPP0#JKT^CY6h4Sa*#vgFv_`frrN%%_MC1nevhoV<JkG{^1
zOrM@yxBTbo-+8mcyCNo^)=;p%zoVv4bQjmxl*uuV?ALaDU=v$-xHHY=RYtf)&;^ks
zr7#!1(DLXViIQ?1HRp0&ueQu6Khu<dqWQ(qP1mh#Cry#xC%!GoMcY!pJpI$m69>gC
zqrM!?Ir>^J=jE!4`-HnC_+It2O0jc!|I7HU%@eGy8pl0JPVwVH!7Cr57u~v}*ZS1F
z{F&co`JfE_m7iCKs9tHy-a2;zv)#du&X(aT%&gUR7~Fi*w6XeUgqDoIeZ`v9jy>r*
zN%qqht4*2a@}SnLEkG;on^W|MGm|+|3~kQ-Dy+PdtGU|8vhioa;?KGv9eD>cmv21#
zQY7z@h4?cCCGY5GSwFaWGWCjMLd@<jD);{N!jbFgo%YWg|DT<juhoBat4!~eY-M@r
zq{{Zdt<#GBacS}Lel1(DPW;_5(KFG9kN;7>zMm~2wB<y8Px(^o(yZUc1`FTLoz%X%
z{`E#<Imz2)4=&t}U+=Z&mAj{7Joo!=N^5$~2Ic=qV*eHKPP*xuYu!2S*wa@$ey3JD
zyYKsIxy(JqQs1Wk*rmx@(>k4}>V7WdJ?7iZ`(Rb;!*YfDu^un%mv~nGdVSxyGdp$O
zw}WeE&$Qdww7p7bb)C(PBIooN*8`sUo%*X3B<I&;Xi4tUTJvhEyikwHws6tvQz3`5
zZ%l3~wEoyWUGmPU=bA6CYTVH>_p5u_P$l{yZWkkSIFs;}$s9LsH|<-W*r`+SP}7{-
z=0{o5_xy8j%tHjHuRYoo^T$rmYqj)7-St5%R&O}gs9$)ye;e;@PT#3>_&)DEHhbyL
zpjgA#LRPP09RsHe+<QLpA5-7-fcf+Dma8iD?}^~ElQKB)yh%=5`+ShD`?GV-?DxX`
zEYbop({F6?uh*UD#I3be>uH*cbZVxQR<UJxRGH+I@YlIgar=6dEw-s;onQL%rfqH1
z4$}o+N+x9Qi*)`y`?MP0|2erY^rji53MWrWv9s&k`Qcxy&az%lZ#CztCm%!PTh1Dt
z{6GIw(VholpL5P-K7Fu8Tjft##;Z5__lpD!ZiVoKSv6dHoN^^?=gV6&{3Is5Pnh#u
zwXbc>y{o6*9j%)3EyHHT)7L_Yz0Xw=zFfX)dF|V~Y<JF^Ujveh^Q^B<|J1WPSH&hL
zg0E}wn|+hJ4m%yR+sCymHhtqkp>2XI?)k|wCm9EaJ^p0$^`SypSw_!<JBb$$)kM9o
zJ5wY7T%G;HsZ)GG;Zpm4J(%%W+VV%9=gTx+MyZ2I;!#NkFC-a1O`5i8>TKs*F&?uY
zuL;b1dC|CR$?dMS?bQkYy~AJaG<uhtnJNGEqW@`2i=C4V9`ql5mF{s(X#E#iZ^iTt
z?Kl7LFt-2ucQ2Cx$MmJQ*o4%+-;?kcH2<#He|f_3J@bED4$-|V)+xS@&Em&YTaNXz
z>ZwA9>Uk}67jUz*8SD^IP=7el?oIvLlXH*w?C8B{e!)^UlXctfIvaysTl0Q8yy>np
z-*>1o>e;`k_cQ8r_XJ$ms9$w>^@2w%ViGRVNvGCMoyTa`)c#^0`@J3?t?wNz9W(E9
z+_3rCH6?Za)&sqc*@wio?6oN<oPR*R*8IUEzK<URE1fUQc*wZ1Z{3YUTAgi|&y~$p
zTdBK``MYT;k8Fj1S^KfN;62y8|Mj#=i%anBEWab>5|HS3ZE0MkX^{HeI*F%-FXH^q
z%1pX{vF5zaR*AbseTUYkyY0yfHW$C;roX(duU)EtTUG<7?z^43ty<qNZ!%I)5Y~!{
z;#eQMM?#}v!$VPn7p+p_JCCnTHMme5sheVd_+OJywoI+X`v>#e3vP36H=26tcgU~H
z^FvCmXtC$^eQ_zaH#-^<Gc{xJg}i-g8S&m+r+3|WXEncW#p{dPQWxu6q{~SCPTuy{
z=ltgsXUhV~KpVMJ(@T=gq9-?-%r{#%xnCuC>f(xd>oza?d?SCY7USE;OP;Gr9hz~P
zW!Ka_ZjLcsI)10-*RHXiajdneGxqIIb^m=0!4>U)-F&`h?s|7I#(r5;{MAKaoZ+|3
zSDCi6zuP{~;PeIOFr|&xh5M&Il`-yGl)=wbAu*A=w7lh;kVsqY?BiYXVSCaR*BQxA
zy=FC4`P-ufF06jBJN2B_JW)7fAbRU<q1`;zHLUE`%l&2KS6qrU*4Y2(%ZpUI2VZm@
ze$2NmeHQ$DyW@>Z>yJxb(d!Amsy}^apEDO9``lHNWG-F3_UDE~?8o)fE9dzCe#hDW
z_OWwcN$9b!iYa9WSJ^x>?km;Yx7k3Wys%nsd;OmIz54EK3!_pwOuo6U@Qm!cRB-Ln
z!L#azHFJz_$MkRXey~(#@3Gmzz5gfjtpAvPYp1>6yRY3rwVM{MXjy;U^nc##@N({`
zMO=$KUO8@_uf}$6i@0y%e#Laz$Lq8IT&-ocd3rO(o-ypDan`%zS5@vuJo!`C`|q^;
z@^?CXb3eGAatgHgG3CRJfX?qTHBSFsKW|Oc7p`AtO)klEPbnzMf46+u!)1FldAh$H
z{F2Nzf9YADpz~j|clS;SzxK0Q=~M2sbp`W#b2-kawcf6q!~LwTv_k86!<m}biuuR=
zW_{3HRnvEB=Gw1`@yCvP+E;{jJ^viCs^|2V59zxn*zI^@qit&`|C1-{Qs2(}@3I&5
z&NH0;yUj>_$5v<CpL^EcI_y|0Q)|6!x7^vQo9<1k{G68e^z~-ewKr33=3JLMclr0z
z3x3@{?>t>y@2i;q_Dt%5cP~Re|9&{-n~3`Fjmu4o^s@8g?JHOB3oY6^Q||lzjc+eh
zHlK6pKYs7j*Jm4T?an+|$zy(W$K-vFzsx!ByIQ{4-;a0s`|I@^p4O>&NX<W~ZWF6#
zm$A40f6Mo>XN$%EWF|~EHhq8j@RgIx@14@*`FonNS9qfEp})tIZBCyz`_N%;U2H60
z>zjM)lZey5T<`x6nYL;zUyvEBWwN_BP(5ASbFKC4ueJ3X>hAI$-?=u-*w=Wy<oaKZ
zFRJwtey=d}b3V1@+!WFFxZ9k&VvQc>pAT1z|NYnan%{f70O@B;xf5;rL|&+GSa`jr
zhwJ~x9Sw5Nt@Ia6JzKR`>P>!^?u=b|%(<=)Z|oAVne4K^G=Ap#t1_Rw?)_OLciz7B
zoR($Y?#TSC-{Re&oD<|d{9oAbNxnPr*X>LlqrZ;r_osV`PmsCaAaQ7Ex?@MALGaAw
zYt5g#o7MVVUv`ys)`d?e)u&6=t-SxT?{1{PQkJ7{K1s>3pQ<`Je}((Z(?@<ios%+a
z{cYVxYy0kocc}Mj<?fmLUi(M=!>YNV){n#9nC)$Cs#};ZJmWe4rTm*;9tD5kXkNuL
zUD4R-;erFA?%amoPFVKZ{gBa5jg;KJ=lX>N{d+vA&%5N-e4N30D>V7Kl4$0_-3+z*
z!Yw=g&9j+vH(CDm!>ggQ_&$hTym)FcWAQQB7Gvk+@Cg^5v0nVwq<Nqws{7)aj!#>c
z{}MPJB-ZpgJ3HA)|FZap>%1C9oIf%y1tkBL5Xx`-W4VB1?#CzU7fxN(%zfD|ah8Mm
zLwju9lF12E7*?<Ekg#yhTh?K{OLmPW&x<=>T4(H^UL<F3zw3kX(~0+)trVTFTox_n
zIy!r@&rj7GeSQ2NIQHo~+CI9r;J{9c!j$VuS1KP&@@MCp-*L)e+GV!Xn@YDelH#tP
znfdO-N5SM74|^15nEsw|XyW`I4+L4Bz1Jy_Kh<kHL49Uq!wl_O8-w*zBkLdOnrwHl
z%v*lq^6WP4rOoGGYxX%Ay3Kg*WWT`T=Uq*=S5MUwy2aw-VhXtDZqV|1G;`C-eS8Pg
zdd{|AKG0Gmxo%<g1oJepx?YZ^X4Q$erK;;yC*`*6Nm!%4r$bWjsXxd5@Rkz(UH4y}
z(<-sBnR)xxHgV%X{%gkze`!QcTJfj!Z^)ySBJ2XsuLd<{#;AQg#QaIiSLcZ4>BPMU
z7sqdzsBIXd_gv9ydElj&uZ#3p+b12$xc=;n*r}_x7pEM%xvxezbWNGM8tb{cWp@-Z
zUMro=-FkG&m)4DnKlHzURo){q!`8Vup`Ul<zEgn$eaXS%f{R0zJr0TaucrTH&!+aP
z$(pY}PWZZ_R>Pn#%<XpNdAW|*n!_>YeUj$9o9(*)UFF-9tr@4+z1+X$!w(gW%bzqq
zDbH;xa-8BRUH&*f`p{tu$2nIkbTXbwUETNN277<HdT8y=M_Zh1#pecVZ8h#${5fV>
zN4&gtfZTMQ0}E!ZI{qP{Kr$k)UF-ETv#T=U)|U@f1-+RjENm8L@p`k_;rSmrVj^k{
zqh;6`%w3$M;;$E}MLv^T#u0R?`SWQDr#W34g09_J|8J+<+Z$7TrT;KwyK`%w&fWih
z%Jlc$j6$!X?oZBUv?=+dwDkInB^zW`R>gl<QBaf>^HyGiH(^Qzho=U2W=w|0VlCJC
z_8#wc_4t2Q{!tckCThWk+xasz9@;ecrnFg$cT7<)X=2u6(qDbWYkKdcYjW?7uYX%`
zr)#&3Pxa<&8)I1C?fY;$?$ex_SqCn=d|P6(Q{MAn=7t{0J(CYi-!^AiRp5m!7fw0q
zn$<0My88I>9}egD$^5s-KU-Hn;ePmLjte`M+;zWt)>7%H<gs&wcH3;{`W$|47P&>Q
zzpMZ8tmAVxtUUC19oO=deYQ+C;{J?T|32qDpQLJ=`S@AU-T9A%j~Q}hDTYkQ6a0Ly
zR`}P%Q+La)_UsXm3X?0BPMqlf`R;8~zGf|+efp`1(|FXRXYtf?dnYOL-gB2aezoM}
zit7j6XI<K$a-wm4CC^m7^SK^ZpXFtg>|FQI@XzO8<#!k#Yu@UA$7~;T^!w_6x%2id
zbp7k>{q)GM&NqBdV_Moj8K-|Jj7TlKE3H`jY;wkp>1$^g->otEY~fo{ec|_&Yq7^u
zD&3QRUOV~MWce44T{T+5e|FWS9O(BAsN<iw?RxYV&zauAhIs+(SGL98+EIPsoxB!v
z`*U57Cb69PE3D`HazE4BBmDOH=gL>R`m@_h4f2nCb@MH}yV&k%y=1`RquSbMu4=2>
z|Ng5#pHaO2TBvoR(vz|*rTT*Vce{4aXKv{`U-oy-{=Cl1*ZV$yI=J!e<HJ)o{L&9m
zd;YO@dcSY#GZBrK`aaK>?>F>+`NwWspoRa^I?LHtc1(MJ`cK84r~7J_{yqBhcSOQG
zqmJnx&7W^Sb9+YpHI6-()!yeWv-vKtP9fx;x`%#4K*+Bfr#y4-B%S*G>8xDIl;-v%
zWvA0?PnMN8>p!@Y_;MG+$}iHH>$R7As5+a8KG`#ysmj*U$NX5t*6N;%=bx{yydgMQ
z=wN#Gqu1L+%4>~-ZXVn>C;e%Urpv7v-V)hc_Fq*>Sls(zo2XW5r_rn_F>gInWVCPd
zOs@GWaQ6GJ_Bp;2jPI=H+GzRwUsd%L^=+*)17H4%Z{4K#!sq$3DrI^0wI%VH(@v#7
z`BH1I_vS|3p>-eU?{E55_v`gckB@IX53gcc-g~Ei@3jL9za__W%5M>B%s6wuY{iPd
zuQb0|{J3{5Pv*$FTBE*ev(J6_&T+2eJbRjq@ulB~>bO5<{rJOM_tW6%)vB+{ZN6Of
zel}G$A*4~H*2S@{?CYUcUb%idx%qiAr%!Lmmfn>8&*<H*mi|70&r|fOLN)r2XB?AW
zFlCzcx5;mxeObU5_UPx)m^Ig)?e*DRd2wEw>918r?=C;9xzDeeS#c@le%itRRew&%
z`|CZ~lN$T;)5r9hc}fmhx#!->A3ANYtiX5m-=~Yuf0Dku`Tx|Sz0Us}&#@@~KC4|Q
zw@%5UZ1<$gF%xfl&iCHksI6EbQ#GY4MA+n`sO7!g-PK=I>Xzo@U5Y=*UzK<-;rbWP
zxrJ`+tbf03`I@J`KApQ-+q-#Yf5)Z@7pZHkA7VJ$FCGt_vT(w)K<hthcJtD`j{DlZ
zuI|XNI(p>!?1hyE#ugDR*-dTh%p@;|b9Ls~{qzss^ZUX4%SOREEuk%k=I{5ve(+v$
zTm6=c8sF+YR6eC@yGngzJscOF^3$|eV$DXaDIJ_HYr}e&SblLO+-=W{cw5k*P$6xs
z7;9o*x{x7;&ndN@J6Dd&Bsb=UkNk$yVFiae%%8C4JMYduly>LvNrlS#j&p%?3ZF*{
zTHi`M?OO6H!AMlALHI_^Em!N_{b#lqm2VDOv}a>dOUFju6B1P=HlDvV*DpL!^xJjc
zrW$7(`SZ7??3r-RyCCAHm&G*hneXoka(2u#VmMrMNlb#P%+hh+^cq=rcMJBUUA=oQ
z9@<m2Sg-i@*3V6MHf&rpnJ;Romh_2Wu}Mrvc1V>Uevp6a(DHVj?6~;PH?MCwbNh_#
z->b$K{`R*88*Z(9s<G;&du**vKz{dT&Gmj4H~#Ig>#LkwUl1F8kB!NGP5&9$^4C{a
z{9%6Id${21cFyHuFO`2t8i-62lbulY$zgU${+caQw=q|*+x`CU*4S6C$}e=DIDUHj
z4p;fT;?uW3{A{5)Cq*;Pc-;+QgA$7-zL}v*T!LQxv6;L4SaOBl5$S8i#yZFTD=Eos
zKFxjSzWo=Q+rr_09>xBuY|Wcnr^CRnmilzgm83IsHJto|ZGAqRT(qFF@9@th@2$`E
zXq`GQ7_d5G_3py9>($q$uDA5Ayv<iQ`^J@R-)}Ekl6*)rJfzZV<|Kv2%~|F>*Dufg
zJH@5pV&H#+>^c1$M{8R2)A;A~Os*5Z(^bhY+~%}+&LO7y$rAIMzX^o@XZ*t;q7x+i
zhxM(#yGiiFoT+6#QS+{QZ~jtU{ZwXC<Tj5p-e11=%WaXlz5dQ(1Nqx4yDXm_zV^dk
zx4O1(!=Jrw``$mbyz+nJ#C=Q#+z0(WZd<qg*=Cg`vz-dgO`ZRu%f0%o^MV87A7+{!
zWiQr0alVG@OLfe&A9uuhuGx14u-w+z`%uw2go|;{vw4cKRe=^?*QlJjn^}3wbjbmw
zp9NwMUzhy3_P~3lqfe#Df*Rc>+c>SeUe=5(JJ{0nm3M{A@UZLK@Mr3=E`j3mg-#)p
zi%u;t;r|t(G3kz5j`&kY=4^}UTE(&beEalF&Qv*TUUsYQxY9ZyPVvY&kInN#GX1+N
zuS!jOnpCS9dTRTxfCUGBe|s4&tx}+?ws4bu&aujU8rQ#d-sainF8g|k{_mCh1Mavl
zNpG6gUdra%|0G}d`<<q3&G8Y7#S)5d-#BOVtTrfzWxJ+guG#&4txdnwU9NRb=&ZcZ
za%rwW^Sq_o(|>FW{~Ys1s9@%~cCiI-a}Gb9dU^kGJ)fM96Q}V0UG+#+tgV7M^6Ohu
z{TW?8%L6#Bh{kNpZSiIG*m}hI1zS<-hmX~tOO`af>35g;RQCCba7xs^g))*}naMFt
z_N%wMoWEc4N8ea-m8rz62m3!7T$uGT^32Q#V{^HMF2^k=8&_Og>uc~p;!wfVrsLv`
zu3s-6`<Zwy$ck%efi}xM(>R?9otW)iH31*8D{_x7mb_nKsm^=k=nRFw4ST{at~B(`
z_q3X{XLl@PoN{1i(1Tj-LvznBH$1;MH>Ygr)+XKsM}luk<p<2&xkjk<u>PCd5xawT
zzx-CPeR6w|!(Yv>8@~j;%Gi4*`r_gnAJ;6k6+ivd^LVY6(u2R3kJ<R25szA6+<r6s
zR(#Xz{>sBQ8q$(yw(Knlbn;WxzWx2C4eP(DpU>$eYIdFe*>mt@iSdHvpCy0SE&TNR
z<s2J7ce|bGhrez%oPO%oX~nub^717!zlx-p`M+DDypPpjiT92#XAgg7vC-46Nu3ca
z=5pTQQCiB9IEQcsu1WpcbG)NPTW{4)ExOsS$Ti=6*Z(WGUNZgtaYR>Jdd~5$tp3H@
z1(Hwix%D{h^OXzB-?-HN6g(UJ!O5=LEqLDjxs&=8_#Cw>JN#mwWp47Y_qkoX<&2E$
zsZYNSn6k3mK4aRiuce}Q((UoY=jw;+X1?C_x8JE&RO;d5T>;P1T6(Q5RXCT*?GzIC
zocqzs`PPzW)oY~Z3;i;+eRVVXx3Q#W*D9}<d#26*_hfq89>upmCx|s%+-DSN$NhAw
zg5SJ#x~1l&e7E`1UrnfE{__7~()N8ed;aQAR$T9qKC4pZ^A6*eA7(C0`v35k{W_JR
zS2NvL&oe!AXHmqS_MY_ncCL!Y<M`5U%~@j>^JUkg-PIOCJck3tJddgNwo0?q?(2|d
zmq_~Qqjc)iE1lA*Mui&xLigtNhcmx9ZdSCHbHmaL8=rlS{h!2fG9vDJs%o;*>~fW3
zA0Fge<V~2Jtkj#d`7?`e=r6yj*M$$VY~LsRcaGD361mo8YTisCjovBGtV8Y6q8$$Z
zt&v-(cSLW8i^LWMq3MUj-rG#;fA}KhfI9Dzb@6eDrn<Y?F4yNg_Oq^vx?h_4_1Nvi
zV>0i}<JU1ynl{bv?87y&X=~a)>&B_>zG!kJq-f5bW0{%97b^&Pv7~9*>HT`#I^FJ&
zaiom&if@L3wX+Ra=H^)N85EWCt@{2vY-irzlSiNY-fa9QNT=z!Jk$D?_DlYAS_Bq;
z2w*FE@}9v+E<`Hkd7oZajLA|fw~KDoUeQjMD|Rek(lIIixUl5T%%1xXWXn(e-JP=~
zS!Q>R47bU4-m0R<znhxbb~kBj2KYKWbKY=)zx=`;1DoHMb1qio>M<7FXA?hUv~~0I
zrcX5<^_PCX-QXj8qi`w9W}E(Z?%g@{JMTBKJ$tUwzr{G~+j*J0FLsyRT4cTd!1Hx)
z)85T{yz#ocRK_9W<v%uxMjf@2GI;b{^2bM?%O2krPMs!`Gl#23modQZ%cOU#GIF+C
zWKHM4UoQM|{%?bw|5D~paWr{0IbL+;7sa|74-STMQw^(fBbmKjf8*W>KD~61Nl~eu
zm$&?W+mrg-gCdiT|7&9|%zfOII%V$bo8f$1t34fV<cfzC6sDY>f2Q;0&GSargEJm%
zH}>E5TmQ%3NeBPO#`P>Z)ZVG^(Ny*Og5uEVd8KpTOq>%k;oPF5OAFR7S1EZcBAaQu
z?##SG*S0?CAKC2cO)9CgU$|Voe$cG%so$?Po^}VL_tn%sRNG<j?RdetBcfAZw%mCA
zH*oHAPmcwk)tSFNt?94odiCz$&5M;@Epz#|2xPrkv?cfW=Agyj7kqu^zR&!}w5f&O
zpNh9!^vgX{bm+nRpSyTMG>>gn{&w88Zp!)Uuj$bnmam>2Z5Q&!QhwE<HtG6F2Ypk&
zNB(=&u~qoak?oI9ecfEdWpnDjuj%K-OqG|Tf0wJ~$W$$kxS`Ce>@!!|c*nAKm#wTj
zLZ)_|ta6ynze8^d>+5j-)O1^sS?L?!rtLp}hV{08)IzVXXFE<Qx$vydzH!orY0pQG
zte$;GpRW`A$k|Z#f9KQclhgj3`m^GC?s=>G`qNn*C+y$-<L9$e-%{;26y+WG{!*}U
zl~dAI4dxlf<sZ)Nd32Wh#>=YfpXdKv<vOS8u4u`9jf)Pd4W}f{OO9@|i#k7DS#!%&
zxqheaFAXoh?U=5^_-ERRHP=J6`h~dyk55e!%G?u@c=*n%!tz%#$8Q`Dd@t~SMrvV(
zf#&XulNC0~XwKih`32AHnHBQKJ8nz7%I%o7&a`P~fagQ+&7W9eUeBHW@x>;2*XpQ6
zGr#10wsD;~Z>!*Pt~d=|&MaFibv=iUoiQhmZ9F9=-TWr{bcEgKm9<Y-Jt_Wr-2ML#
zk5^x>%n2**`+WWQw5GLJuO2Q`^g8?b%hZY$r}+-NzRbseez9%w!peTGC9MxP^>PW9
zE-Sr!{%1vFz`E(x_m|%9T>P)-_20@Fm%>`Vl$|fxsj^+o#a?r>Oyiu}KiTp*5{^zT
zdRaFkL-t=|v((mX{+87aDjF{hFLJq8%`cAfxf;bezhtHGE0bMcbJt!Fsh)Z9heGe~
zkh9Sr#VZeRx4qIdVbEH`pXalqTW$;glXo)apX~0QpEbQOxMkAf)sOhfzq~H`^{3$G
z9rZtvzh~W_7x{9d&$5K;{+jC_E|mD+bWd7$+xO!!e&5{9*$g~;t^VF;EZeaE(mVmX
z|IIRY--}AbdiEX{?`pf*@~55U%SW!mA9M3R&-wQL<@(~T#+jvWrtEtV&aSX7%f?Xg
zN2B|Ce|ecEMZaww(~U0Q{LLn_?dJ?emdx$?%ikaA3Vqbgn|C?LZsx*g7IS|e<h9yS
zeOj~nN5*$s)5lWpUi`Dkp6#Syv*@+bMK3wN1EK3(3u3!n?3N|B1$(V6{3g(Tai5;u
z|Hn+{{(RM3^>U&??EebQ+_zW$Xu5qmT^82(kK@9}>fD-Ni86LqHq5fStFB(JW3>Ep
zRmC&Sbx-4+#p_lx^fcOM965aD-&Ow4e=oPyS?8aRy7->!uk^fU_8|;9U)Xot`uWkU
z;CoB`-N%30xfyQ#$Y<#OzlUMx@72$A_f4(x*lG9nviA*^E8o>5WS4*0;gfZGgZ;;H
z$G-Zo&4;&Fs-)_^Xnv^dJ^9mXyAxkx*SYk{1aG=4*qgCn&OyV%P7j`TZSKb0bejl?
z%SO!ZYHl+{r!T2kw8v?#$W4<M*KQm<6@5i1aK8Y{6v>%P^S2sF=uDJMo!D667W;Dj
za_tZPx|y6h&!(Dn2OXVh*5!XUoTplI)A0+=CXELT>X$3JNQX`MH~-p9wvU_2*%DLS
zqEk*~_S)@UwD$#f-t{AAGPn*m{7<^<YAWLR(9p~=;^T}3FYaD<`TjuXME28VXFFKj
z1ZrQ)HC<ixTYJ~3ZAD9_D$flRnC{G(|00fU?Y|dacu$-<=`%HEfqDJn2PZkabI%?4
z5V@{HJs{^OdsE)e%mX3!`fe;*)|q{acf%#0hR(7#TW&td)D8c$kmKW-jI>s6?!_%`
z6KszrK5IJFH^1@fs`jJYyT7kty3F3$>-D;L$?r=Eznk~+ZnN!@erol()!5cB?n^?m
zZ~seGU%|zZwnz4D3`tx6i9@(%k>uM6veHNQeABV5Pvd*Y#Hik;n<jBidc*FWxyvik
z^2;Sp^?%U$+EKFNOPi7J7j}y-FJ<fVW=6fSD{@}%R=Dtjvz5W>yu9o^|9M`Qy6)#P
zq#s}Q&GpT}%ufunh4;Gif7<<2+VaXx^}Ww`t}O7Ha>et#`pZL#k!LC{9llcN^>)_2
z{cYAW|J8ZiUmurWemW|j`Cf8XOZFj`eya)kuWc%h+{^1=|19>bao+2Pb6Z*c<9;af
zw`KI6e9*SoBcJ=#|CgG_n&iGckv6~C%fmB2)7Y);%NpwkS2rdoYQ<>Z**&vr%lTEj
zS*up<Zi*F(5}7c;Zu+4WiOy+mCTCWKeC^L%7HiDc71kiTF?Z$R$mYZ4?v0DAZVE4V
zRAtp$<Ty>{CCf+0$_G-en=Nnd4nOp9aZTys)vp)D{jGn;EV10qyYGthu47h9l{a%7
z4^Buo+V1$n_Wj09#fn`|A65N`@$*zr{4QAk+~ivYzvQOf9vAs|U3gwK?EY+j+qd0v
z)sodam+O_66ifeKFvDW)$Ky9W=Y(wDZ2N;%_UI+U&i!{IpC(N^d!G05?I|JA)jJa(
z_dXS}a$FW6GV8*e`B$=6+qS;{QC{i#;rvFXeVMmpoL^=pR?BK{H9dUkbHFV2%M+&8
z>?nV9>ecoDQ-OyE0u-KB3OC3ZPYU^wBKTVRu0|<mkl;z#|8u4;d4Ad5m}}=!kIclj
zb9{5G-NW^cJM!;qc<ORpVbx=aAkJ`sNsq%ER!uoL@kyQQq}^-ZINUg3(Ps5y$NI>r
zhuL{A#)eG{efi{j*X11(JG!pUKR)rr(Y22&9&X-mb+D>!nbXZx{1I|4z6u|vJ~$vF
zGT(bf<>LciR~{BMPhR7)y0lPo@_F;iSMT)~J!NY!zg^e&;Lr65)mgW9PJV4@8NYer
z{lhcmK19{~+7+s`OHDsEt>JLZiR3`t$S0~21@{`e>*~s$OFmk$=7)^ZDNXYau3JkL
zG_JOuE)9Qm-`Re>Yr@|vo!rMa8rVN;YwQT!m-_Kwo_^^@8J7(WXB2;bP-U34u3)av
zPvJkp+_IAkrLyA><d#?#zgC<6G>Kj4o@>;{^oElQi?6S=SZE>lp!P$Fnsxc38nf@6
z!F!S#ID8-cKPS<=Ddy3<&9;RZ#~06@`0b?h)}Nm)A6#31&-2r|=;EsaiIWOlpPlDB
zb;UDh%I^aCX5G?CgS*A%w%gM0);(-DT+RH~X6v<wY11@6-Tdk$dQEJ$>E)@vgSMQS
z=kUXRmi2~K?c_BR9)xDCVX;u{zM3iD>?c*C_%?Z2iLAQO#M1(k_`k+7ipL#Mn%*p{
z_oT&oiuz}h?&ZtpUNApX*R{hwXAMVXbmV&Z&q-^ZWli*4w)^Y6sGg}W*M;y-o*eY9
zh<Vn{H3qtRJ=sgXXf`*_%<BI<Vd36~yl<EVPdg|w&QUj>SSs^))}1X+*p^gn>%3Mb
zeXB5hUi;kT!P~9)9xlGu>-u($(4U-75yC&RJ{Gp!;(y{c<@cck6H9mBGq)d_2QIig
zLuJxON#EUvz8<;p|E-0TS_qSkQq8~TY1ii+%s+YSyZn}$E!r97`)#6ZR`RaB;QX+y
z_~`WL_dgB&Gm?eGMHaT7*?3h_aBqoVZB_T)`CNTRGMZK1E{YM+%BhlVX5nFe^xkvw
zjBlF{e4cWj|IC{s4@)N8bvx6(?Y!gj2E#<-c|yfmyzDvQ)l-Y)@@~}YRGixO%V4`+
z>V$@{Ql(eRnwHM+l(hU<vEioF-MI@S_WwKEye9tBu>%E*_wKCxXL>{L_$<HOdZz;3
zxusM#-#U2wjzMX1q(^SX+gDFtEPqincgu_GF?#AI2P$W^1a3e3=-rpkCY{%9Z%lmH
zXxJutasD$;&b=?5G#rm!qqF?OmaW(7&KEqE&Ypd~OwYph?$KGRFCRF&KFcZJe&vKe
z29Nm5WDR!b&RMjisB~`5(@PcSH_GJqPiDH~@aithLc^AXg)85la!V3YYGCAF({?(c
zF=0n9%Yioci7|60{fv1!dFG!xQ!)-O*f;C){V)!OkY(o`?8=n2O@A$sOS3KgX7*Oq
zb?5qBlkd)0*R?KW<wBA3CL+h!W^(wRKXb%(N!63<HIhn-2H9#`&Iwn2dB17pq$^Wd
z_iWY+{h{GEqwtyd3AS(H7gFsTa~dXBe@N5%TeWKe=ceP?{<2rAq$C@ZC!D?1*c7%l
z$WL|tKdn`vHg8rhc=}1><<tdD-#A*OWX?FBT>bb#@c9c9ID^k99GN?PW>ojva}pWL
z7k<lJki6EhTYc%}2#%%?v33)mWIyw-wVnRz;Qgn!pOk#i%RQTNye6e(MQ@dK-0ru{
z2Nx_9oOg|v)nI}phoXpG;=hu-`IYaBVxsd3MPJ26<{XjzR-@~)tN-uC$(#O_CLMm%
zHPc(N`0bA?se8}QxqOe^{7~o$1*;_wgrsi%&9L%q4v(0$-2P5OJkOfrw-2O!zGD^N
zn=|!Yzwx)1`Cs}EH?9n{V%B>s(U%#*d87Eg;@=%Ar%&epF*&x;idD~KLwc(I?tJ(3
zHP3HN3fP~PDs<HUc+t6N;q98zpI6yCO9#HTKe+i9gZFon{(F@j_OlXW0&n%W^QGGU
zTl_unXHk01&6{t|{rbcy&cN{QNcW+8v9~?=T5fHw<9%K8NYCDf_4k8>%{%#<Z*EO|
zX=fUFUVi7o<jBOt9UuFJ_SSRu#C6ACD6Kki{HN8-<H|it|Nh>PW^nqxqT#;XrFm~}
zxYl~zvHagPU1G+L-_KlXzBT!t3->bk&380ij)6a~B*9{f;%)bDFU+(&N*Er#mfx{x
zxqh_L3-xbe5(jl^UhquYA|a^joY30uJ8hHy!&x^NnN3+rB|7#Scifk<a7>k0|Mkd)
zBiZ+liwdl>fBN!%js4=(x83*3Y|{U)esSZ|AKwKPd9l5fPrdv1vhQ2Bj#K;d?YrTJ
z-{)Vj5z+p<LGQWctLsXsYb>)qdRb+CYWTwE_<r@xHQ9bywK2PbAO6f0e35AHu%iEA
z$INP#%-H^*{F4{_4(Dy3TcmCDY_i;%CHZ&P+n+nXmf=tDxx*PBAK%)3{OI3J%YBNE
ze|;ThFSqc1)q-hz=7#NyJ*&Ta<>O~P_ik<X{<1iai;>AV$zS`H?hP5)n^NDK6Q9c6
zZZ7)v=k6Kv-0a=6bLHIGVpEs&eLnE;?k~NshU_2zsjpuxzGc~$B^|9={%VqM3ItwH
zo)z}z%EP6GPx^ky?g=cutgHLy!bzu_zY_VkZK-?x`Q(MdH98{tr8O6)Mc2Nnv(rlF
z?7Hu?H+|x-koJFjTITLIDZ6s>ncmii*Pku=Txzo3%>Et6?icG7p1++T6R}81@7ZC7
z+_~QsioYIv%C%|L=SsGA%UhLO7RMhncPo9`_q*w2)^4s1=L_pL_3qU2uk+tK&-z&T
z9LIz&YkC$fw)*zWn{DH{pHe@~a-BIW9vW_EPl&qnYlDLp<Nxrd@l~09k7srpAJ<5;
z5uNk0@M{m_^ZM-VdKzo_HdjS1<56J=SScCKwWiE&&4!7m_bv-~x=!o+#LMfSKg>D*
zIdg&j1<qXkSgtF-H=V1}e0+R`ZGPk5^#u=|u2jBPH=A<5GveuS2G(CMVkYp07dcxU
zs&TyXA}{3Ix0>rKe}74Ie6O3!ZuqQtwT!o&g7oLbhuP*XJT;ro<>^}qzmjRYE^tb9
zay#D^d0kWX=1jDY?ri(86uY#iQIC#n-Cn~t_0I~qA1yHzdg13cPnPtE%s79(y>&ui
zXL!u7!}GV^aaIh>nfGL25$8+U=aH7SY5PonXi3FZzTws_DvIM-ud~IqPWV*(;pCaW
z1QPo6=5>Eo+P7n(t<B-(3m4SbvYuGK-u}qM+Y^5YWS$WGYf!OUZZgX;#gA>ZFE;k|
z>xQ*fZg>B)THc?LYs+EXpD))Jvi4PfxBHx$y3Fo|^(?2~i;GgWZ@PHlTWH*FyRz`^
zjY^lJ_8;tw`>!TranRrBUbRY*!iTooGt?}<nEtYzq^Z^QYK6PZo@@5s56`Ok-g895
ze_lxX&L+OZT;B^tJt^Ow*D+L^+^D^M({yL-qdLaVn<m_vzhpy|@Y)H}pB*kaU-nfe
z%JtW~RWbW}cCWp>RdQ*pZR)3+6RZBePIs6TJM+KE+1~5VESCD|bew&W9@2mBUccP@
zzXvyqT5LR|<gjRS&7*GVTG6cde{;4c&b8fqa)V)kl2hbu3A>G@Hht{>cPwU~pLpbP
zpxcz?CsXe(pRi+ri$&jUL-ogY3&mt&_D@{?EIs1I#trKp*t?j#?QgYw%aeaDMc01H
zoA0g8d>7ii@2IRxHQb}?6_Y1^JkH+i%O2;4HNH`=Dr2sFTfSyR{MOIcxlDe(I$qU%
z*_QVc^Qm9=<<Gp?K26Yj&68q-f_$H=ft%V^OMH=^F|R(FQDaqQ&8q6JOq)`CmL|P)
zm7B*SzS!e}(0jdIWxxK{{ycW^{w2mD$)F?QX{$Ewe)7ok^wS+XE=@8lt>3?Gum1N#
z-_FOx9h<qJacBHb0izuusc-H-H<r8gh;#pe&%b>HPWQgIu+7f+E&Fx<J(m9qnRecN
z_CtH`{p59X>rZd~6V@mtH*tU6YF-Ab)E}4YKOC|#y?jjl^7;LVPXDSHU1Vzt`~Qfy
zP5ga+`^Wve9A5Tw8h#AkbBz0s^gNZz*XI>pQ`It>ck$`|9cy@w3mKbT+UL$6B+=CK
zUzdl$^Q!#XNzS|<lGnQW)xYD6ZnhSj(bAw0zxjh917qL9rYUk)xI(j}Zhz9Vy|CvY
z*Ie6K)=b8>!t>WUL`ODQTo2Ru(jdQxub}f(;zip%+YX&N(Gtw@dF6)Feo`MT>R!|c
zmna{ZKhq>cNL46QZ8NI~V__H%D`TdMnry+Mc@u(j_-2W&d9kys=uqRvsJX(O9h<ru
zx19TMWUA+qOPk!eZ*M9useD=H7TP1~C;Revmt)_lGshS$F6(m7^$+~<;ZJaie|pYG
zh6NM81~p#0x@yrC$(09>WcTh{Q<AqRbE(r-`;!VrNpnAN^!|*#c<@PI@qz^mfqNM?
z>{h>@cqghP;pq|nUCpd4A1*0Jo%8>)VA1!fyO!}w)lOkHuHrR0crzm+O83i&sE~hV
znr{4ara2yWJ7$x$b8&$D27cwF-&S{9UKk%?e-QWdgPLq^!pY2ze1r0ZGoRcwp816P
zn`fSv+@xR3uFB;HWArZlj(@AYv+~ZQ+3y+m71v#1h>6-|)UIJXUpesF0i^=H2Ja7%
zbK9m#$uJbx7x7#0W}aF8<>r(L49mi&oO|ZnHRY<r)>qb3KJBqp>CNnMzT|Iev-9ly
zw4Tj1oHxA^Yrn7GbNXLg<1CxMY7dQHbr~3by0(<N?SA?DE*S-#&kJmV9<n($W#%!Y
zYCPmH{v+79&`9^U!=bNT5$e17gRF!^OxV=Tq_{NAj5C_z(-^k|F3zy!)!<97vwnAZ
z!Ai{sb(t)`=bUk4IJ|RbLzs)~*;Ts}O4OJ#=E=-+?CZVQ(_7ftyFt6)*xC-A)lZJb
z+*|C|?6&zXb1efyf|0cIgUMG|o8o6XXD{e(x_HB+)G9|wE9XFMvW!_sTXxyQnTH?6
zHR>vfzK{u79eX8O;+#|2<B67a^R)MF$}wzOko-ieDf9P|Ykwube0sV5D{IUtx0SxS
z=T~+_Evb8JxZ3D-{DwCfSAHq!z5aFmwVp%3kKFjmlv>|CkK3+W6uNJFGjp@KQV{<u
z7gM|Y+nrn1JAeNc*SpE*Qq^z$y>~f8xXO+^VDL+{EM$8BRDyeP^SrGgbFJRl{+m5z
zgM9O*fT=F4Ytv=EmR4`<EDVq12xC`ud^mH?w4HDKi$lwA&nO8_WPQHu=;AUz&+rvS
z?A~@OFPvJ>@UcqzOm5=sxi9$rq;GJRYs&uKcJ}SIsR8fz+4RTq*?mhnbH>8|ugdpA
z`KkZEy7c~H`YnB2Y<AI8+hqxxeg^;lY<KSD%&N1WKb`%|<m<<C_J!Q^x3Y76BzmWP
z=ISzEE}s53#=i5D$*c1Ae?QJF+EP-#CjYstsK4DD-TAXKiyb%oXlU6g*thn}qA7>w
z_wRge`$dsup4i7+wdQ!y9UJuW7)(~}ZhJo^^=D+SnCKRX{>a-LslP0yihp{e`1E`7
z=gpmM;VMpiWgai5lrPHDQ<0B(8$NMf{gT%|*E0Q-+3@GO)KB}s1wz@noX;HpgvTr^
zd9_H;-(;7`CAkT`qA5$_zBc^#+F#v%XX47Re-V9O-?AS`4Et%d=hv64f-PbF%*;M-
z7VEc~9)72N_{|i*<10#%@>>t=?7U(2W~b3?bsH%~#Rn5DxpeGyuJQS2;2l4|;OUfI
z3+^SnE$7*F$+lDH@yD0{9OEu+nKe@<x!L4iQ~poBh<AqhJ9nKrt2!%It^G{&&lIQ1
zPsZvY@*!IMZoVzX`J92TG`AYFx$*3lD7SVv*w8!k+iX2Y=CIkX-zr|6T(Onk@#@ot
zsVieHZMri>`r#@qMxpbCS4BS^z7t%>_VntXt6pupQyCxIJ?ms+YIFOgcI#_{zwpk|
zshmwdRm*(^Z^^nOR<7}E`nCLQC}VHlI(ysw>m463Z@kW%J-a`5^StHnI0|Hc$j@C-
zy;q}C`Jk}jCN0fJBLAn)tdCu|<9JQ)hI`lCj&UA5;`Dq&cl4h4=+5Kp^Y``z`}G|1
z;Z1yd!%dR!cD6^YIm?RHZHhZ$f`WQ(KbaFLV;jR{bv9%E)a#!%O1IZluM3&3pZk<e
zpJ#5TrQZ#ShLo1giSYt0)#(MI6(7PMc&@&u^@nwde|O94P9I)Df#%AOo!P~$r}Lk!
zjm+CzG;5*ex{SsTPlcx(sM%2eTX^@Z*M-NHCm(v6fAR8jL+i;?)Ssp<n(Z8IWMg-H
z&hw8O-+cZo<tjMy)z$QTp~y*}Ycp~#FO%#|u9aQBKlb(1NvpFu&3DMkq|DqKp?-=f
z!fA%_{ZBGt8acYV+CM*IVBz?$c-iUe{E3YrdyCi}y=d8RPC#*$m-+c)`<^$qCMA7#
zH27NM@?)yd%(lH-4o-<%b5NFxx%)!bg%9p;Bu~zLI<0!!mEh_hTysTjGPA4q=&`YI
zFg<y)cgvF<Jk0Z+mgX1QD$NK!xUVPr{I~Guq4&<a=biccn$5Ll=Jmo4R@2X^#MJNo
z`u*km(@eJa^Sn!@=QXdK@v&<5+Wg(CvJ<t!9%b*&{JrhOs}=LUMcb}*N`GZ-#T9Jz
z^2)-GOc_lv+wM>Lv2Dx3<zGIud}VJwYSuOJ`|X9zxmxPkxzilxYR(fen(kS(yR6sX
ztjwyNa%+1gdLIrg*!%R|B$aLK3f^%~C(QCHVRc@7t?!YUV!hVhV*N+S%a%-E?y~ao
z)*7uyzQ~oiMKUZ!ubd2)ELv{c-P-D`7kj>f+hF(FPX{JV)nCjW-?~`8JoU%Q2wUx{
ztS>(gpN@QHcWy)4&v#je%<Na&$}P`-JH<7W&C$2Y-e7vcOTHDyxYkYD){wr@#NJl!
zmeKqVIrVpr{qJOYF#mt{;TL<>{lC7MG4^SFu7vH{d;dbc|L1Eda3A?!C9yw#!Tq@p
z|LHz{lP<rh{%FZxajq}xi#OYuIbFRhK7;4~t>u5^MPB@$K6h6F`>)+ItLtMHTr}M*
z_-((pul%l%=4EX!wCC%_O);DkSR{F-Je{w}FFfJP!^9&Kx~)t%)vmk$`=OxM(L322
zUMnt_UpUP%KkR^d|C$;#-rQL^#sU`&EfyUV-1_EPY+7dCh3K*wyUzTxw>WW7k-yJb
z>oJe*jRyu#g+d$h&vURm%5gm&a=Q3n=*b?H*^}1<I<7QL<$t*)n)BnlpBtOs_+2qd
zOPjg$=^igmjitdX#^-wuye)W^_S$QfL-Z73?(O$>EIh_*?>Ke2kCEfl<QV%!4k53(
zZSOHFU(HBS4=UWy&h$57^%hp`&ov?bowiTU+!5;hd)4EI+Sh5H438|43;x<==5h3|
zCzJ5z`!A{#Uup*3Y05hgB%OZju=<LXMwh3g&tfvNz16T)<;&F$xn9rZukWnZUs7=*
z!Y9~yZgKDurG;TK?W?#XrdnowNlw)dEt<4|QOP8ciM>N8ZG~g1+)lwet?M^^u3GRz
zIPapF&r^j}Qy#wAvq@gkR!uhP%O1uHE8B`MrR)l;kp7+ZX0>eQxrf=74`i2Klj@6K
zx_jxCmzRQ*eWiETdph6WqLO`Y^_Ev9Z~Q;K%bC?t*R!(Nr^xP&%&bjZUmx|BKe+fJ
z`d5tH&x_Tc-+y%CQMUQLT7Ko@r++P<7|pyVr>LX)^n<L#a_&oq!{u8tHynR>P@wB|
zi}vB{4WAXipV+x;?whhrw@>CR|K-&9Nc+Cf+v^jjs7<=0V=sAS#^w2<M-H2Z#7K9q
zzvbr0FR*G}u4_@rUOxG<+q_4XxG0$zS}}Gi$;`f)7Na~xZvBco`%F3iv%f!hU+cm$
zvsZI&-1XUI`EvsEtMaHjStT3#{Fa~U?Cz60cfrQEn)eRpnZ+Uxa@~%06=iQdc;fS|
zseA`y{M0kDnP=>;I(u2Urf{Z{V9vZ1D~vZrvtQosbN{7JMNZqy#)-T<nJK9o=bVtV
zf3e<Ys@2^(UiQQ_ecscgS59>gPcAjO$eEDw)Ai_wXF=`#H5Re~@pk#YKHgr%aPa*f
zS-lIoi4F}(GJSKdFr1p}_VWmPPL%l4FGj&m{Sh|&Q(m7C6t))YU}>)2D(g^Str5wr
zc}k&g?Zq88r>QPbVPD3`zqvO-%wDDak#dF8c6OPvNv$<oS2Ef3<!kTUZp9bAd)Ise
zhkc3fb@MG!EXpqj{@5a5@#6G6NBzv3J*ycMe#i6H2mM;Nhk;dO*-qVnm>Ex4*dK0Z
zC~m7hz{)ISwn2BTz~_V0^+F!oE}on@ZT{nX^72f1toIX4esFwq&p6!AY!-H(SGPB+
z?P}QK5BE-Z^{2`tWtm#%%syA(^!C<=R7Jh?_{RE;ve!A~#Juix*}gp!bxK|6&bcI!
zL)Z5oTyyvTt(yAPp-jnhPPXv2G}^R0nZDVmNB!lR<+D4u6E4}D$*MG|OE~#NU#j6y
zu%0kq^G=OuA#U!Jw}+;#_}ikgv+TwaubK%Dck`4@i+yW%%vAW|x+_=vvL3omf40)W
za(%cZ_oUQlKDGcGQzNdudXE>Ur~d8Fa96L`UL&V>H*s5Ew3WtzZM@Ynk+qv7{$8u}
z%-Q#G@7oV^Vz&Me<a=ONbH?4p<3N(g`m&psxBOR7+HLc^`sq8pJcq<-Ew`n794Eer
z^4-3y=l>5SH&NGR=M8f6%2_Yx#%x_<RQ&U^gWOGyeHL+luGE`}|4-+7vh0f5ym=u%
za(KTinN;j>%5TmFcIU-Mo|XodE~}I_|MRZ?@#FvP+#x4oW`^y&u;Nf)m(8pGoa@>X
zuNLfd(@aiePR+4q@h*RJSh9&DK6M_e%y))70h^r;{kJDr_qBfJoR%H3-6oYu`6;uf
zCU+p8kLuymEB1Atp0YdOKik|FjJMuyOq;nQymD8K=Go~lcb~W|(lt+bo|@~0qNfMC
znzm2p+0NwXRQ*>v=hy$QVgX5s|06V#_DvA2Fks&BK&Fq!?T1JE%2kiv*7DwapRxUz
zZN#s_iYbo292@?Zel|JOS@5myQ|M3q{*oioqW{^eCHL4J`^)#c()!DeE#;M|`!oB*
z9iLv3>G(VQLXv<@ebe04XVgym@N+$BS-i|}LanECFJnmOh2zdQz863KQe`!#c6F1-
za&~pc8yO`$bJcfTlT#58UbR2ya1YCaq_nk9J>PlkUhsTzszzd!;EJ8LY9)aI+<sb&
z4WDc}{X@c@?bGnwYj02$ujjl>y|Z5D@f67?^KXj&Z(~x^onhd=|DsaJ7sq@}t<JlL
z*Ev~U+B7LiJ9g(<ujA|%ibkI_10BUBO!=I<W0I!MI?&6cnET(D)hc*~_W86VkLt*!
zQb8NTe=WZ)ULoc7#c=aPrYGAM%zyXODpvnSbAaB(7cp-OHlN%+o6~BkuZw<VkmBl(
z$NrS*csrg8_`hPycdw_WGp|q7`!;DClkt^`sOtqwt~E~BAM@ExFIhEmeahkdqf#$@
zlpD`-D^IH2?b#5u^WvgoA5Dx-9*Mv0lsE5ZOxJH2fBt^gDw{S=^B=pJlCDjy`}4B$
zxZ#b!`SUBcq%+y|EuV4x!?B-<`)u@!rT<lkWt_A9rk;MU^1{F4_tN(DXWYB~;@a)Q
zKKE|#Jb!lk+b^4)vpmBee%dJdE`4U4+@WK_XKYJ+u4g(u$(Y{E$9_{Z@KoaWcG368
ze_eg}s;|zHukxK%Xa4*j|If2NyHor2_d>UoEn@QNOHVA=mNj7>@2ly@9=04VE~wZi
zqEUE!;cm%2ui8tJK4;WzdO7o6xGs;>yf@jK*6%YpHCb!rNjvpwA?J^i3>(vwvUana
z-h9F^PP9Sx)NSPj)9&?%DTl8v6_vPM{z`che`ME<nd(2|91cwUuvqRL@7!BOH${UE
z-S`)N`{lY>QJZ-$=Lm@%ny|L=#5Eh0<h(gDt?u8S$xdNcZ$03tSP;Z8C#B8o)>O-7
zK1mULpUY=Xd7+lN)AQ=%G8OLG`Z;xnzQmuq(y6sC|AF>D-}LnV!CAkZ-Wl>U{m&3z
z!n!ARqTp`7^xy^R-h8(lwuUb{r5*U4<pgX08?M9mofb|_k@DZBWYoe_5v~1Ye-Xpf
z#jD=Dxq9Qal-1@2b4l~Vrv$D|I%WMnxOMd*|0lmJ`;Ps)Rdo2-gBh;fcfT#Z{;urG
zGqIWL)_v0yJ>9zf&1Tt6Qh(0~t?zpBFjT|Z&wqQ^nqrA(FLi^CF!J)HX|yX?*YbX|
zxN$l`|M8s?w-0|_9KUdU&)2_C&#nBzxc;w*n}1QMJ;##hre!z!RXJ~nf49+;o};wy
z#LNQ2FxlmkOWt#@<7~PW&wKUj+Jd|fCp1>O$JqMtrQHj?sQ&x><v)q7df$2<Sxgk$
zdimO~vjtu@@vlUmFVflF&Zbc%`)pBH_P?Y{<+s_M?KRjwF-KVL^|yK1%Wpq?xbe}C
zs?u}C4r|sw&I|rlRK4~6t@y1?TMkdSTEY5M^w<qC;g@cQxceiPi)(zI(*EV&2JK*x
z<|*}6i68jxF0(%RKD~GPio^exKlXa9y;uAFb<MeN+W(%|{mnd=wSjZ-UuBz=13wF`
z|G#EB(-qnO)^Tam=eUlet5d>Oy_rz7PNg~Mq{v>anM#KjU5NcAUQuIlDbz*o<bsFN
z|7Vmv2sqz*^zZ(8H+rtDo#yuII`@C$ru_KB$_B}PYf>C*vK)^r<jniVIF<XA*cDxl
z<@zh40;X=f_-xMfg^Anh19vVea(&u$N_1*=I_s5na*=ZzXP-&gDfuXF9an44ia%Mt
zo^{4PLXtaI7=O`~{4zCr_QR4Xk}Y!^%iI|z@#F@jRpnTerX2Z{zM^(bQ^V2C4~`lK
zGCatclIV2a!SF;x%xZ(?_AH}`C0~7Z@;r2T+Iea+zinXkq+q@y_jo7jXRhU)C}O;k
zZ_O<y!v(_W5;cN{Wv8X9|Lpbo&8M|1wT)YTcKZd+FV`i%adEYa`?bFP;`3EM?uO5*
zecCeW%k|c@PBq#5v+dODyZvR_A-fE3Zme<5SQWf!{ipSt`8a!HZ;2M?o$1OwXZ^N!
z%ai?Ce{b!0Ik)cYHSzF6^)Ds<X?{2_vcpA4y72(#;iS**(I0YdFF0`atTNB#-;BMB
zZ@kxEy)bG23iCT%nxFP8UU+-&<crlmmi^tmoIQx|*q`hLW*HV&0uSv-D48jI_J7Tu
zdagggtUWAci>EN#IWu$HeE7Oyy}FU<<J+D2>L;=U%rsW6($LggcW3!e?<e|#`&Gic
z{%C$&-ZH;H%-3jRyixthA4@j`J@jw27yZ+seaS`a_tH<(yW*v^P5)i_5q#L+rRdF{
zrJu!p9G>vF_&Gk2aq)3FqT}M{{KUq^$K{cWiXVvZaXAEMQCsYpONyV%GnW(}mt$}i
zRP4iVpI2u(Pt57)0TE!<nW}RpuAf>v@3^-`3QT>%5Uo($d@p(S6Fr-RKbL-XSMIqO
z<jI@jIM3*hNkFH<p~?Gw{uu?XyfEAMJ4^nwrnwOh{Ik4Hx71W#JP?}jLFYo@Ou5T-
zo_~%`Vfw;mel~59_HmY!1KqnCm}JwL7=-FJ{$A0*Y_>Li(f21UjUQRt``cdV|GD&|
z`z+Vwk294F1LtShAJE>+ld6A6zj?pJX4eH;4u=_Z^o2B@xn6g2i&jmWJ-0LMn^%vF
zUER~i!hbhE`lv0x|I^3D84fB}P90xg^QY^0`l&akj&B!NI`!z%RA)h^I<<h#15#mm
zU0uG44k2=Vrw#{hl-1^Qf4l0)<-kHKtttPewlDrv5F*ppb(k?>BimJ`x<fBBjvRgR
zFvCXX&d1(B`-xMf{M`?~%<xQ@^`o_Y@xuZW89s?^QaVq5a4tVAW4Pf6RMf}ivWtnI
z%VC!b2ku!2_6lD#5aboUXd-w??Qy!YjGC`0Q@`4e`?qg7Sf}=_TPxG|KIC|rM%=-8
zg(~-5k=2P75*7V@y4R-%uUMJO9i#tN;-<PqR{w!5Z&(`@IxAb0WPY=6-NBl%=Z7m#
z9B0cB*>$(x-m#gR&MqbQXlePGFm=t1H;iMZ%vgI*M4@f|t-{`<V^!g|bk9UQzxMKm
zRZI8lJ63a}-IpDiBcbguN6$*@ar*Bice14awD;Z&_?2_vN0k6u^Yd4a-pQ^!d0d$1
z&6>jNqWs6&#TqIb*UQd0+jvuE&e?-E_oTi5YyR21p6ie1hyRC}HqQUgKGVdkWJ2)G
zE-Bp^c@_(5e_Xw{M4;^F17AP6%2(5lw!W0yBwgk3-%Efa@YvM&1Ck+k9JC9h-yT%X
z5|X(+`S~X$S-H%$8}=|~tXnX5`lF}nOfGr6y9L{$!p^&K^FHZ*ayvSV$0ByciHVc$
z-*0m{?ByEz{oE<1?P0ckx1KlrZw}Pp@Q!0+j#za2_Tydi=lLftdSSkJ)AWiNW$!m`
zSzJ6L)Icus%NBX3rTb-a9=b%YV<}lKDm&-y>JZ0Y*Z+x3`_i9#`=OS2#_c!yJ7+1r
z`al1B(ySS8uKKBNTAg=oc8Xr@bElIrA>S7#H=VDVer<PnT>j~*@awzFMBco98}a_x
z#cvnSYRkzUUA(;Fi^%D)A1`#|_8wil-D2Az_x6`B408Fm*+yDf`G7e3xw^~O9)4SE
zWp(DZ@Uz9oYrf3r=llGv<>w1SdH)&xexJ<xYDyyd_)cGpzd3J1PIq}pMBl$t+v=Xa
z*dQZ2ZSnqHwh6XYYuwwvzF42fAJNbE`eLH(uhqAiuPr`YR&t~7->q*AZ(nSXmED$G
zpT~crkMH)yMC)I#Z!>>ee7LUUM*qKG-x~hD*dQ<aZSnqnwh8uDGVbk<U*zZTTlDcA
zzxdGNm-KDszPB85vVDv9@3ehjV>QRU{qu|bJbsIQzULPo+WeY*oB7<|EBRMcbB^2P
zoZn&KFTD3q-}l=W`E%EAkUeIw?R)h3^rp?FdneDC`)p3fIlt1fO`9*z6@GYA!~c$l
zzmN6W+wxV5?FyCc3m4m`Nn3GTFzd=~yIl6?l3POENn7{UErCWctM}<%-yMIXz&bIv
zcD~u86g`{l)koK!xK?4cufN#7+Wlj$^p#|W(wyX*9ig{8Lae34yN~@X>&ebLeCwKV
zuC`PA{9ONaYOlk#=RN)W@o$}F4vW>)+DGPAk$umtfB77*X1cH@tZ)162%qiFrF~gT
z)~<Yfayy^dt*>3ufgNd+-@SIeZU5!*VQty>pW`x=`3jfw-8Me+Sn`?8s=ghj_S^QQ
z-Mi&qS-okK?<KPik-qPx(i)~S>=qQ(+&H!U?cZ(neR0pj5^{JC<?W7Q*WV+TAbp#o
zdg{jTn|aGCa(-3##N3%@{+8uq#Pxl-?|M!CY3-Z)WncdC+^p{h=ZKlU=P#CV+VJYQ
z-d#Dh+#@D$Z9?woKTDQ*o3SPT_t%ONhiKW&bux$E^4;z~v+eceV}(n4ugv;UFsGqc
z=6mtaEx%^H{(Jhw>yOUS?~`x)|M~bYod5T$yppP9<4KXReeM?D?pQ2KKa-o@a{1fM
ze31;P)9v+f$8UGMp7-SE>_zeaKmSi~yc>MJ{(qM2@%jY}X)McG!p$sI-Wh8JY+>BF
zUHQgQmVMoF>YS52m}c`=rGDaXa8`PsQ!L^3f>*UP@O#EKMd1%UB1h)^Q`S!UK7Zwf
zwV$WUe&(3>Z2k)^(Jf)0owNU6cr_#NhpyGvg&utJb${90Kc+S&iCp3_P19(2y;8Yu
zw)I5A!*T_Q$BxHKACWtru~KsW$GS`JSLE^+y_)7?F>`5%@cY_7tNSj-YklYzZ&G7A
zVcKEkzKOMB*Z%_#7~-D9e>V8v$Fyu)PeE@(a?4+4@y@LO2TB*+wtuiTPFSS7gnOz|
zmTO4&isk>Mq}}d%9OIcPtfl^~;}qxAm(xFbt(hC(|9{cSmRpiri%aDDPMI)>POGa@
zG}<g=ae;$9<hS@#ZuxBs${gPwdhYtP=@jc!ZY^=U634e)w?wCEYlVlbo1+`DuI5$O
zEzzyoT9<z=Sy<%!wCz-{|6PllEA9sD1abRL$xgMF-?g~R{cYc=x!$cHt`@)D7l*fv
zx0t80Yw?H3t+3f~q2@*7E#a-|TK-j+7kqMg8p;1(YQxei^1;cvEG`RIX#I<@%G|Xe
zshLHuV!>p;2X&lx3>we=s7p4!Q<%cVxmWOIlN1Drdb){U@esfA;=}T96AwRp*L|n%
z;&%D%GOPYe`(-}*W;MC;+|^$v-p<|;8GiDF?IEFuF^P9>KXFX_9OR=uuXXLwUFMtj
zJ>&a*^sf4U%V}Gt_Z+Wrn`iOsdQt4pZ4)f6B#V6BYx(SSwRT~x$KD-vrEg;Q3aw=F
z-ryH^dFrLC__)OVdbdqGW}IJq&OI?mexuaijArRqHhZ(rOnh!TZ81~2-PAIv;+*3%
zJl_V{P3;ISnc^+uX`|HXuA<q!GsgYkF;5GnlYw1}4yhb)obuDAeY>cdZ<mVK#1NIv
z3;)-?{d=AJ$fEXm;iHz%H9JamJym@B=3HDdTXk-~gOaajrro)RE0g1N4Ci;L-SvFA
zhpY3u>!-b^{svvHJfm`A;u71!^-I@uPtt!Caz)|VH<NR>XH52p4LhwOVj8D?KYUJj
z>XWO&m8wTpDqnsQesjMR3wyw)?WuKJzI2u-%f6n)CpFpP)_>ErcA_ud@0WTl|9olI
zrfm-jvlC`7`)rarZEDwJXZ?O#$tUOT*Te+xb4YyKzWV(9^EF+McS%07dFv|IK6BmC
zpf~(S?e4feEZH&d-y@y=)JHXU)=oFt9y`Cu@2SPyQzCxn#oX;nXE|B?w>dZUbY{}y
z70>7UE7vZaaL)ez3umR>#rwWoe*4Jl<8Rj9yOA|P*Xp9L^=~eJwaRY&q3OL#N^Tyu
ze3NtNal!2AKU{;}A6I>+{`K04uR3<$nw&qrwea)3;UVO@n%hII=IzIKnOp9kTlLp8
z|IoiZ%+`ON{#tR-(@E!g<<@weJ^h854t`C_CNc9K-sQ1R+o2;o;b@AMghYZ>`$ZW)
zy$zEOJJ|`GXTKg7p_;(i&MWh9szLSPTSpI^YUGG<;GMz3nthGok;Wa#M*Hp`whG10
z&3x9TI#)I@J51#({^D>)j$wB~P1{XLw)6v+4r(y|KJBna?v6yK6YCNOhGJ%K_M1n1
zSbe*taxx8g4|_15=3C<vaQE;xYk~Rh3-a2kosAZimOJPza-7aK<=~Dzj59mfgl6Sr
z<}65^^er+fjs5XQwiLx%iJ6T{-%c&OVY2<wCymz<x}IAUFWzwre#Ku_b7YFF_~q9-
zW(tK}K7aP=<v0DuZ_4)@zv-TRTr%eC|8Ti<G5g)y?$=*_d`6-9T<RnF`px$L((3=5
z{r6+`-;d^(fB(1sm-hB;1oJfK>-YP<UVE`%zyHGu>Hh66HU4C)ynbD@XYKLZ+eGc4
z6~FvnNy(UadD&-+_s48I<lg@Cg@OFvr*9cQUre;L(s8d(tv*l$tNG=?mHpwOFX7wS
z!Ik|6xxZK6GJ-05t2OTRrPT+jO2CzS{B5}nva<ht=Wn{{f6smP<Qwn8x0Zgq9hX}C
zJ-<tR`K<bTt8?BxeQx*W|BUp=8@4YFt$Z20^m5GG$vd{1ZF}c#_w~~EZQo}8uGKf*
zteqqOO|$#A%=Tpc?aA8P-@e%bVu0A+-@f^htp6Q^v_WiGQ=+Ww*TshykKf2R99Sar
zzJJ>>w~+0dK3D9op1Ra4ug|2S+$Q%pzvjf>%0KU&_-kI5TJrAav4dKTH;po1+<$rJ
ztG$lJKgsKh{+o0CF|JGff6D#$Z&tJNooA!Z)to%@t>Lz^Ui#nve+&L|v%Zkv_x>24
z!}{%KewV@pg>By!DsX@O|JAUKu~SaaBwb-gbMm754@<-?n(YkzM1Jrd*y>)!a6qG`
zU`6+bt;Rncgq;4>A76OxkGjvx%n95E`l=CYCfRJrm6HFEwZ6W+`{DkFGylo5K9b+R
z%f7C({?Ds_?|)vsIyumG=ijfd+vMiFyq15!%Q_}(U#7+!DX(;SE6W2lH9O8O*D0Fy
z)a%=%&$DFiK6#=xUBR;V<W*n0pic$Wb8Y87dHQUgqwBuhMZr1CP6#$7PkcOk&b@Q<
zBGbb)etz2fWTU#ddZpz~=J2W9BC{X<oX&plbnIIXX1#xf>g7u}>)+O%B(h`i<eH}*
zJWm`ow;zAncp`m@`lFlkb#{K2^V}4(V^fFG{6M*+cRi}xP5gG=*;AH&raSxc(S=_7
z=EuM3n!Wwh|4&c8&Y!d5IFtGE6{k<`-N{q?=Gi0PuZu4J^3GrNysRvLRlc9u_gl@6
zW$X6+2-*82<m=*}KW{C6B_O%)%gS3_wNHb;np#_)PM*m=e^tBD_gJmvT0cW})-Su$
zQu_a7{-o{e|E!LCck9$?-?*@%*b?_+ir-#73Dusiz0>&g?8v(CcN*I7Bk%3_bhz$h
zZJ1rer|zlRpPSXMd9RCDeso85(%vUsrFv!McT42`H)+TJPN}VbXSRGz{`Y%^t6txX
z@r#td9pM)_zjl)yU)jHwE}mZ(Ys~wC*S}|<uXAeWzm|TxNc+WUyW`JZRe${`;g4K-
zy^cs@@U`o9k?Rj@zRsS%=K8}aU$^>SdtGqq*Ddwer535RZzq4XnzL!&x06+}ev$FN
zQ}*&LU$ee;Q(SxSweNP3>+fp5R!5<#=da1XV|4M+<n13e`(KlnUt@0<kuI<E=l4?n
zu%DcYf3sO96ePU0U-9*h;4Ax{U#S&arf=jaUv`P%lDxi-`Kq<scMBhRR=G9N@=xKR
zUyob99_M`BFMO^34rg}xUz<kp+7CN_Jh-&UI&;b0>Xh#nZ^#;!$<Nq){6O5_H&+Wk
zeb-p-RejmMR6=RP-a1P~^W1;@yH7V4-fvHK;rQLYqxJ^JneLnOz8vnmvw7~_&HYE+
z49b_!xhtCaeNpA#`Zu-VvmRSrdSiWk5qsL9_w~u!1q2lsCkCzTe?9O2weqxCs+~TE
zmX-%goRA4LKF)CVp>Ot`%Uk5UXPgt+xB2Wu*W#MXLf=h}e>!96<=t_qd&8Qya}LTZ
zG-XfHUFddOD0@qb$zHy&Thkm?s2+Nu`7J|twduu=nicQN`toKsHNI<p{z-=4+;yMt
z#)|EQCAUR9zURu^HDkTxTUZjzb5Z2`mb>hBzJ>nY2k!qoWSNo4#q85B8Qo{SY{Rll
zm&-GjUs$18_R&tSWcDMq_sx#W3r<@Wm2mCd)6ac(*?|lBkDaz_8kBT#&po8OT;Sqc
zJGT-p!MV&4%THXl+*iz%ZTtVuvx1A7OeImhnP-AUGXMYomuMq)!|Qtb|1%3#+^<Of
zbGK2Z{u1whADd~;fh$%lzk0f2Vb}Hf27C9{JAXgF@r=@{wuA2+ve{xgWA$Cdrk;}i
z_fPOYyZM3Gike*a7UezC`+q2HuU{4+B{S)Z(6<?K6)GuS@8XXb<y6_;SZ*%*a?-i~
zGdY&utL#~tEX$YOE_`!iFyH(O>VkgNzd8%5ex3h!fJ6P~<CpHso!_Y+;QarO#o})|
zqaWAi-@+S0Hk!z{UD#LlQ;W;p@!)pV`ejlv`>U`0|HRv6_e<=A{?(&9fBPr8bs60f
zIbJN%8nvj`qh2Ka+2U)HGai3WbmOuK-ZPV_XIo%_=h3HbF2e5xr-s&=3R$?G&+S?h
ze4+c$V>elyqXC+2JuQhYeHuwl4<)!d0y9no%KJ?e>v|h-cCBFF!Uq#9KR6jmDZ3<x
z*c<N+6qJ@~Z<t(WlG~8?Feiq;>;DY4<pBn^7SqEHC!V-|$-4g5!GHgr=s&EM519Hp
zVfpgvXD{~mx0^iTy8Qdb_V*8eIBq**e>j_O_K6J*T+?sn`9Ec;ylpD+%JuHPmA@XG
z4!jwq`r(SpA>EmEyPiu~d{bKX>2iYC(V#qemf(AC+PPiE)qMwoDyA>m{A|Mp!Ox~l
zyQW!{zV?d!{(V`_;-={NdAsu-3w_-c>n&JU^2aaZKf`w$<>qbtr%V-%GJY?J5mq9a
z(|_93&?w{go)}@}=56O8oFy@r-)b7}=bgXxhs?)c!RubChrDPGSh(oM0{x;33;vaU
z4|MiO^hv($pj_A4QCsKi({Ir5fB8nI_?9=CAuCe<ngqO656bwlxX->`skIJ<&h7u-
z;h{KNV}h^Bi8QGcVZluXlW)E}Yb?8|Q{bjcXO5Dq*=42K9TR=8oKP!DF}B@gG5JU4
zC1KA^Gu3XoOvzc~Wj0ZD_RC4WEhp7Zr3hzk+EOsx@_WIoFD~<Pmie0ruCTmvLfO;y
zpWpNQ8k6d60{;A2x_Zj<_L)Ivdrv%*OD?wXvG;vgk)RZAc5(Gm*;~Q$9k*9%7^lyT
zvlG_S{_JUe{~rI9pXGDhLZ45bwtV)fFXeOPXKb&w&tLV~arSfB<>ALI{?1*t|MQ;%
zUtUEl-+%jO`K{&blYH%cCi`8Uec9%v$zCg81k-==x#zR>ZIwQM`DSz0+F$?t=RGi{
z{^vcXP2L-w-R+aF_j%9hFRzf8R{7t)JhRF#n;RFq-1GX&v!(M&V9eZkC9~u1F7w>}
z@+>OTI{q&2w&(XRr@g<tWOwh6KZ<_!%U{0FyyWk`^#A|O8MCiX^r&}Raz200#qZ@e
zE9$rW{2jmJ;rvdcU&_nB{gLNhxUe*v-+al!rnNnSCe7!L6uhvzu}%D1=2hkef?Ne_
z?oF-SbyB@<-`oDUx$*lGuUK=fC_nP9%l63f-*Y8yes<^mUjKy6dfEj&N8Q`OA*XLI
znD$oc>L<Gusk_~_?%~h6xh_!meRI^d?2yyHNt3$Ix;1BepsqX%I+Y#5?fS00X6gG=
zr*huEI<-ezd-LAa+Mn00IrDAZx-+{s8Qso}o!PAN<Y?mVy~Xv9{#jptE}~fV_-n|D
zj;l)>d-lvTUOG#P<K)Gk?HAA2|7EkBk^jE(rS-b+>ym9l`c*?)CoH=lVz=LAg@#H>
zv~%z^Eu}8GNaYZb1)FMx78bSj-f>^KX0A|wk}{WzBG1=?Acs>Kit2L8YMVohqq-$?
z=ZAwS`Bq86Gq*F8T}?D!WSaP@NVHn2>Ylrl@ZR^Ukc?7)`MXQ@pVrJzw_D4S$$9R_
z({jyOk6mWhG}On(r0nKa+|jLY<DC=7J0*^7y9Jtdb2PoxYk8p8lz>ahmF3_!t(FI&
z=#*Dd{lowEjg0n<jM)heuXlgm{mX*)vFZ8y^^wj`vcJz`Vz#>f@8y${n!m!5A7IEp
z?nfujjDP<+d1lD(=;ui&%l*GPd#`(pLwG{pZgrv5<JUjv@jTFXVv4fM@4x)^_fMTA
zOFov*<PV=E_rc(<$;C&RA9AJs{EV8NG4)7t&oQ+JZ)>ioWnDd0`P*h%t<;A33cDR2
zFg&i5`(ydF>R+bt*-bMuewLg*RCerorR?vWNzWtHW|{7l{deo}Z=1VzC;7OK&T6Z^
zH}CwyDVeghi^A*zSMT!QZF0U?DQxRJ{Y^jrwT7MTuFYHRzh%|cOdH{qt1f%aWL-Kl
zsP*Nv&s$%%M5x{1o@gaMC*w*@U9D1J+_Ky14|5g<2!HeaZO405{XyY0cJ;jKcfIG2
zWNIp1Synp#U3|u6n}eNZKV4==%|7oqQ{Z7Sdwu)S&>zZo-|WA9xc>Qk+5dkhe%o6A
zv912tLHn=vx0bPa{9XUfy+OI_)z9WH(sEP#W)>`;o^Sp4&(f2JBpnZL(+a)J@rQp;
zS8jE}#lPFjjx+tbV>7irB9HU@QN@3)iF@`cmPx9AUnIJ8_mbC%MmGZY96J8vg2RP}
zKZ<tj_+wc0Yr!Me?)9g_Efjt-o;+-D{%>H$m+5&0I_}$>Th1Ln#4OmyAsEtd*Sz7D
z-QBOd(?6%*Nw5#KuUl33;Qsto^5voSD^7l6-_JW?M{oZPI}S`L;Ma>6Z3ab09$!>X
z<eb3JUFB!O%lt~itA3iKW!|HiZ|Z+_^i44SU;CAp_4$eX^XsQQ$*FwJd6Tc9=*s8w
zH$Bhix80rlebvDZ*}fZBP3r}&u5>;9YT|SA=hD~DIsI96Zo!{5Mh3e$KfW)iv#q|T
zza{qFx0TbcyvnjaUSqa4%J$83CanSoN0Z1x{bG-n_}3}|JCamHJQw_6cjeUJNjiIW
z|C?_dYXW{WKGHh$mE*5;&4;Pku}i(?d3p96;{1AJ_r;~FGLCWeZ%F;HIOOEkUk6ed
zl#No}vMv>v<~vdD`Po|qNn0PyT(v$qEL<lvGN6)u>mfmXv6C8I?kk?=&ABeT!b6m`
zxZ`qscI%xZjv9;FABRbHF5A9($C0fc4zoV<O}H9=yUI1;M`UYC)~?U$Y#CFJ*S<K-
z9(VQmE5`JG!5XLJso!<C<i&5<a^Y9@=F%fxGNn8`Z(Hm_ndVo&iP-by%chE*ns@G$
zY~MEbasB0OM<3;;i+8_dyLVale037XBFD@7tZFU!uN}H%b@I}k9VZ{{v)R$s+Op}-
zjuMYeBGcvdT~#lo2J6_Crx~%x?oEGv?uOXBE!%CE#ic97e0!oghx4+$epP5^#6Ia0
zpUv*>Yh|?G-!$KQ{i@Q;{OZuZQfYU;uDa#6dDYX@#}zl1_Fg?*u<328>#XIZQr;Ff
zzwJ{xYdR;VI)36aDZg!d*LxK6s^5O6eJN%l+m4dM?!V-7=PZryb^G1v^}EyS@Xnpx
zUYmEX*s?gxx281X|HKtrzEm%|^vye7`)ibi*s0h4KC@~T9G`D*m<dPE@7o(@erC2e
z%shVoAx}wU)1_%irj=>N-l3fa>>hhNe7c`&KRaQv<f8nRA9{LkgWf!G&4`+zlpS|%
z<;;$0z9HH^6WzRL&g#rND*R7ouGl$+c_zF$N-@_t&lqN2HrwChU%l#~Vc<s1lGT2}
z8Jkr$`q%q!I^uJ5n!(o2uDw0MpF6v3dC#jmJu_CU7F{Q{Ok;ES!z?@b3oox4OMZ^5
zn|1Pro$OxAo@}v&X_Bu(_uTsIlI~>Int6|X?N67Z(^hPq*`=G9Ei76sx&7J0r<R{P
zC;ap|5!brDRrA%G&>yq5znu5;>SsIulpMbJ<br-NO`p|YrXRZg?*qrC`xTEBy1xJa
z8@TPy{|9aLYE^3|N3uLQw7?;)On&#ZwcI7sf|(3{o&49j^3YYE{}JjZYCYb4jIX@<
ze#gUo3OrG^3(sDbH@)$HqV$^xpWm9ipM#~IEjKyipE*PD=*0dHT&>k-Ci<UOdFtva
z@o(jcw#70%vVqFQPnwt(=uMcf#FwW%<3sx%z8}92sb=4ul&c$aDX~Xl+S0w}L?pMy
zIxo)io~B{<O5*nJB-KBA*L|;#U!0;FxM{7a7sKOq4ktwnx5hdz%k!SBk@qUvZ*js(
z--|)lH5TnZeq-LF+K~AxvOc<hooZT?!O0cnynEmLK8{U`TF*7xwZH%K_2r+h((@iq
zYl)vePj|l4Wwrf1YQGNXZecO}d;0SG|FO=~{w)>%`&YWI>!Qv@X8WTVb$9;R=l$RR
z<lFcA5j|@EHS_-edi?Ew;qU)D{?>z}>i_)(^Zo1pKMgX;{Pf`Z${D9!7Eb6}zgu1Y
zIs4R%oFd=$li&B=xBhe8$wYYWuLT0Wx|gGucg}lvbC3Ul{rWD+mqVOq9jmi4IBl|Y
zCy%dX{(bcd{W*WkF0MX$*gjBawrAKJu^-~x7td_pU-030X8tlKljm7eGM8np%QV@!
zQ$6VGt<C!<tdhLozV7c&3t#0~wZDU>PMtpQZ)DIcj;a1Fr_!B7mkWRDzMApuc6`Ox
z$1)F#*8QmLIwi7xl3K`&7nA+JyxeMCzs%o^@z<I8_Ky$8tbTrc;s3`M{$J1gkuLK~
z|J22jBd_Oiw^eL(y8OSl$fW)d?+2ZdwYPT9dKLS=`Q=e-{=Xkz{GTfQ=J?|N`g)sR
z_Huva`~P{0{o7<+U-8+m|6k$t$YW1Ra_>}syk7loSN|O|c}vNobzvNpp{Jir^SWmy
zdGnf{@}B;dmygyTKe7I*&$V+(+k0J?9oYK$Ww@&EJNEkPwkgrK+aK%;=lfN>d-kj)
zI=eqF<gQtpU36#LG?Sg>Hrj_CWh~e<#r)^At83%_p3co?owW7dsmg0tjGyTk7T5b6
zIn=T#TYOTs>BQ3k*R#}52Z$b2D*vH>#s1;y=d-l*^R4sed(VnlsC$uX_4@i~X3n_1
ziP73~_w327eY@I7?Ydyx=^1}YKW1=o+*qchwx8#D{Jw{pY6;a`TosbS8*Euw_w;by
znAgM<cZ?;+-;q&2nQ7bd1r5^;8E*#%95_9r;jN~|gH)db-$X<{7^x-fW@Fvc&v|2=
zBjfvIrft~)2Y%0Jcq=91@TcPcfBFCa`~Uxc^q+tK#sBej{r~rGe)>Gkq&jxhyi5D6
z|NZJ@I56K{kO7XK-?tZJ_{?lC$ngArLjPZV`+vV4&3$~n;?LB+{hun6E&r`K9{={o
z>2II?tm9Lj&lX!>Kj)p@{j|zmzyECZGoPRT<#Vsy*_>Ik<`;kd+*-Hq{PLGSe>&O6
zzj*%pkjdeKeFk&%c|gpAsLXl#J<oqXGC63Gd++C=ihcTi!adKqAA^~`!amQrFaJDL
zvQHnK`T6Ientl5I!e^dyfBt#MX72gSrk^#5cJVXLf1jG!^t2|?GCtxQ8Z*^ukzXU1
zNAbQ5e#UE_OTYektP0E&=9`zl<~jFmlfz~EHuy$ZbV}sTN}9dl!oxV$(|?aGJln-m
zzs~q?`LB=v&s!Az|Hu2kNBn>PoLl$zCvy4i==Qy_>_Llt;*S3ZdB1H^zASHgra!MC
zed0Ykt0ym97vEnhedqJ++P})_U()X+d|xOa{rB_V`T5R@+rn#O<650^*8keIhpY8Q
z+}F}dQLc@9U%mdMC6ZKq_4?CLjU(Sytv|gwpyO?5{PggJ0=HN1(_8O^O`{~&#=Wm!
zf6^04s=j{xX{^SPZ|l~d-W|~KHa32G`9gu)yZ7n6cT(I|UK@L_)hXxwuU&h%TW{R^
zT3YF;c~8|!o9oehWzWO+=S^*}6OWo;TrIutnW|m#j?$;e)1&vy`S;^tVr-JB+x1E1
ztP*ZUe}bAF-#xylwz%t>V9zVV6En5ff6EPRyM3b1ccE<kcALW;b``-3A8Igf5XqEi
zWxjVM^MmpN-$3E@t4;5)HT2A4|6g{KjkV1tG%q0G#P*}oPhQkdH2rUV<6Cg~ob3+p
zgkEmvpY!a)%zSx^vk#5$+ZD_@oW8I2fl2%Mn%@U9`R4!p)_7UQ?(<vb<(4(iZnOIp
z|2nh%{M=_>X6DbgKKs)6zFqmO%jx_6uDbl^c;&yxHpeY1%jOkl*UkR?r+eKWvFDYm
z&hPvDW3tWv$5#If=l<Vuu0H1ZpXz0n-yU1d|M#zO?s@(_1#|sXPNhkGU6rx#>CBC1
zC)Z7M$`@X_X=d0>mo+(y!n*BimxZt2^deU&a(|4;{j^<QS7q(<U2;0@R%&V5i_1&Y
zKcyP~-PGAo8)Ei<y78<3w>SL^e%>6fH}n7LnP2zEnEZF$-4?%o=Ks|*zy8lX`)6hK
zVg2i;|3BsV9lvhI|3yz*&a0fUKf7K%>F>^aAMaN@K5+f_|9=kQ*Y{T)wtqTnd-l;k
zexAD=XI)KP@G|)Pa_gq8&w9i@+b8Mn+spXu&wabD`z6hWGK_8YQjaqgmf!s)m=gLU
zxb3%n5%V53gR9@`=N0{$|L4Z-+jIV{mand!f4Tl^a{l|`U+e4Mysh>7f3<(_-t#Z_
z|Ju0yyXD_-`)}Xumj8cs{NBCiU*iAXxcwW8rrmS?McaS-=IHl-t$g)wh2{UR_3y2n
zaJl~V@q2bBUhe;zoG<_6%l%&)xAPbM%Kv-gHoN8D?R9V7GK+s&%U)Ui+-ch4-`6_#
z)IM<CSE`gRUokO%?upyazcPlYfBM!a{xj-v{qF;<HLo4-?bH5x|9^Dt@!QX@GKby&
z_^t8tk5|_B><Z-W|LWWRe$Me3x2DAYon7%Hs_DV|{Vxjs?C<0HIX%6k&*k?&JL&)S
z5+@6rr(Qh%CspwGTju-LG4F4)?<?PNZ##d@`+~js@;~l9th#UaVejFu`)VIlx4*9a
zec(Ia_21t*-_Kq5{cZ7m>)7|VpYJQ*b#MFm+V>@U^OwJ$d;H<%|Nnp7umAI2{D1ZD
z-hbaeRR4K@{m$2@`akcVU$_7FUH<>?{{MfE|F@I-@o~R@{r?~LpR>RF`#%1*K%ds-
z`9JO-wzL0m=>NVc%JV(0hKT&v-*I0(=Wpuw-Pbd3%-?Z;`l5TrPiJ|R{NMDw_S?mI
z{~oMl`&-8}pX>7eg5Bq*zFhu}LI2f_{wMZ)Pxhq0QZ)!&FkkC>e(>U^mGX{u;z?Q=
z`z>xSfAvlN=dI_bzTVX_kF#2PDrSD&%0&CnmiQ?7GyA@Me!cV4pUV814*Lr>`A=BB
zj7@*l_1jy{+%`F|lJ!tXdEKh;{l0%imzi<>yz^vb&)-K;^RsT%KZ%_G`s(xI%-LtX
z?GDcAyj=2c+BDvIYB8VtZC@w6tzTFDdwbQjor&N7|E-I^{eMyV-|1f056|VBbYjuL
zEx-N?uKxIz_k+}*Z-IZ_Nk6!-V1A<<gVZzY&0q9ne#|$$^M9grgWb1d$M@Ann8&Lw
z^WA4*b@s*QMW?&|sqgA}@qJ=@`#PSgs1NU><$r&xyjwGM#=kvz`?P%P<IL?fH_Q3g
z2bs_Rclzz;T|d7>=gq%<cKQCI-+S`x{!Y35y!7+S`FHA;=IyI_b6fxV+3EeYzn|{<
z`DtVR{uuLkyKir&zkYW5_`bhIcWZv$*dD)bw!ZxDZ>LK?KTW>B?&PEGd}aR@q@H~G
z=hcTD`#!He-TBSRRC>R@+}B*Kc3V@q`sd#JFYH!7UKu%eTX5pv{FTdN9{>4#On=|q
z65$=??eT0ocO6)M?x9Rd#p}LUcFFTezsu#`y<O&faB{M=foQFQ|KFW6ie+nC)lT%*
z&%LrV@1M~v!95$omdOWlJ9>LXEn3i!AD+Ns-QvBuZTW2xi-7AVHNzC|`aQ3SILLlI
z^8WP)$5&t9^f7zqpWm~7_^10T@=QOwTlV=@X-57@PNtXMW@*hzsGi)mZ%>I!&64RF
zOKz9^(>qb9ckYsS;@Z>o@zeg-esG8?FK~I?b7NsM%bdPHZ`aBGovU2GJKMef`HB1Q
zZe6@@`Dy#^vM=TMrChClh4=k=8;VXb``69+X7Bf#f6lj)_pIN>e^vS8KIhxZ?-p<O
z^Ia%D_)|aSho(h+&ELmc|9`yozv6BEo^OvE<=6lH^1c6hUEPECDQCs!$A0Awn|p5W
z&YZ7LME*KYdZu3gI^Fa0Wcg2vm;9`%%6R?8@m{+9`~PdFRL);<Lw<(w`)&qP|6l(j
zuI=#;dBg8B`+H~CPoJazvL5|ez3F>>TmN74dlnf7|JK(}*!|vq=fnAh@86quem=kc
zjJwibv(!ufIkP{!<^7=l_gmnfZgYcMNqpNHR=2rT%qsuChVg&@orL?9Z(hz{pf6Vb
zdFJ-WkGG%8Rqxzn_x;A@i<@iTZCoy$`uk3@zWDX$a@CexAYu7kAYtc{-*=Mz<6nH+
zd3;~i{EuIrM}EAmU-{<ghWm9lll9yGemgB!ZTV(<{I?sY6ZZ$QR{tpq`-39(chm7W
zzTdubzi+Sk;isk+KV9bAmqY)vjQ)JfuB+bp_UF3YCO>9QIeB|;ZJcxM*}D67>lf}z
z+n+!GdO&^Am-;y8THS^FKvKTVV5wyX`P1L&ryt6fpAsl9cWdc{U-nY(Pt>jxFaMQZ
z(B~!a_1(|<36l3q%AWt|j88vubp76JuHBOV4<_%u`*hy@$~D_}-}!9a|Fvv-b=~ug
z_udtqe{XZ`cI>;KbB|Z;K5e`Ib3}gm&gbQFuixr@|8p*R@7?F~_J4}VUte@SzH-g>
z@SV@s+g!W7`p)O;a<Aq6zxFS$`Wo{4(aL{^o}7{MxMjTH=YDt2)2~*RedtQ;`SjSV
zemdL6J*#%F&YW0tuIu=*xnb)b?q2o#Wpd*U*0aLqr<ZS)KmVs<UG3MH^K9pC7MesV
z)yG`7U-$Zd>Ge|a$lZHxUFG^;7%$KNhx6U*<$G_c*<ZNqU-fY22JLLde{XWw>q~e!
zragVR{_U@&7f+wsU$*t{-$izkTka=JdKMq6+<tJ^{GEqxD;#;aZSSk~d0H7iR%>6A
zicf#`XKVKEUz@dCWt^^mZB{6I>T~$zB;PA1mYkLlirn;0cFiG)!{;5pnr8%jO!VDe
zR@hu`uk0u}v&81+^Y*%J=l%bdoS&cb^U3Gpzdh&ek39{JvU~dV;HEzj)BR6p<iyKu
zNws)(dU>je*md^l4<q92bQf>jWw(Rp`1GS6Q%&~i_30n}m|9pP*}J~`@#%t^NSSyy
z;cG`ZPZu28WRWQ88zHW)bC~n=qn1>Y#Mn6-c+~Z}g{NOUQd;=PXwD8DzZf^+>qk3J
z7d+ZzQ7GvfC9bY}*z@$GC8;Kvv2(WYsOxr5o_29bX<<<HJmr-~#CKVHHXkv+B699=
z{u>KT=X;tbBxR2LDt@iBp*yUUM<Mma2TzGzhMx6#caJdyz3R{Zzh|%H*442hu0{)Y
zeS7tOA19Z^+Qm;?)GHDLcGb$)e>ax2=+^(?IA#0x9<SuAiAu_*2Ls+|+-z;O`LB~K
zYO?;leg6Jm2lyN08SEMUr8AUhUDaLdSKaXM%V+-xmM`Tt|9zg%uiUkBPW|=I=gsz}
zR4M8*KW=|H*KF_QM}H1{iJE;c`*QW#bNoK*zrDO_=U+B&tv{Hx_vUlS-(Rxk$GyEg
z`>@SRgS}Qh>-n%S{nqzgp8eS7rN#S?UtU@4wLCYu@AB%$VCLD$ewSBY{_@IXuO&M3
z^OsjPdoA^?|F&>eN_ov|o3wml4wSL*Nv2Bl%(kFq3#VjisGi~XS+93__31Cqf}XK2
z3!nb-YH3yR+<SYjf*7T9@5Ma#y#DfP>9G&sDHG5<NjPGj1jbx>+9@urY1{eB%NQU1
z$W$||o@2k{>B5<(-Wbok`T9@dynSa~&WR~)57;^LOs<)*RW!$4r<Z9#+i&{3$x$ma
z3%358s+iW4ZhUa_%ttfzxebMFXLrs!8*;uTvSa3ln?65ube_~jcFg>7)8|jlvO2M8
ztur_moMpL?#+6YR*)fwNDrUwxo3<S~M|?PP)mqGgow`puojGzLEh)p;bjuc<BR&GT
zY8__5F5ah|&bVAmTa;lOx@C{f5g!SN>S&LJXSYCAx60h|;fJV>_E>b*1f)7ukZD@$
zjF*efwp>g*mBF34Wv1CJADP@`ecL9u&wjc1EX$|*0J%SmADADoADGYlU~ca7=-~6`
z58LzqxA{{o|IhKo{Pudef0aL;KLj&m`TuWnXNjD4cWKZ57u?|oT+TDTINC2VWkCW<
ztJsVCOE#^PoSvBT!ajFXkHC`GFJ)M)9xRPN+`r|OOVwHBIR<~#x64JbePD?D(a`_=
z!>=<2DSle-n`EbFKjx4Sm+|_4U9#oOqUU|`H$ob{-SYP)7s_NSu>YH8`F0t%dUN37
zkST?>zS)wAYYw+ttU7&Yy70rJ57&MU$h;aDXSex~=(CSo584JbKMvl&{%D5snceZz
z*Opgy+<o}rYyP>~eeu?UZFe;eD$bWsy8BmuHusjW)5oWFpRBh$xaN8xuZXXbc0@h1
z*!JfSzi>?4`+;eG?fE;Ltyg7#A7B6VaO&};CUHNapDyq6Us;&09j>q3yWw~r`wWdj
z%^h2QI`q8$_UGvOIDUqlZ3#cVzy80&VD7}%cXjv1Pwz;5FIy`Z@Yy_GN-E(Uw}JBq
z7yj2xoICm1-v2qi>{m@{hnsoJH||dXAAApHOY4QY8+>khFC%X>ebL$C7w5LyZTt})
z_EIAK$j{!=6fTxuDg`H*&u{dyapL&@`<*J|^7r?x-S5YWwl8Baah;c2HbF^_ap8)#
zGp8Fqe`i~_!l1P|w~INLVTRIvkxxql^p93(?(hzboqcGh$nm8NdaikzN!|;ix-JG4
zTwAt6r!`rV&%4=sktfUI0PUj@e?R$b-oe`V^+INi%9b;ZqI|0gd|H#WWP&dWi5}8y
z3FcTP;G;Oh%V284s%a~_ayt$P_^3j-9smE`^fBJ4`oo*8{^XMtk}jJXtVIi!GrbFG
zaAQ5Q^1;KV_U%o}x##Is?0EFBt6f`e9f+~^(aWyo+H&iBtPdaN<u|vvb3$BtUS7qn
zhY#D@rRDOTtV~*EZTZq~->%@0n<pd6i>}RGQ>vbS_hiiXRqUMU1?f$S^=2Q_4KoTe
zHoZ8@Hd#$vX|>@g)@P4C3C-a+wo~fv+>U;kKH(nW9$_Da84d=C9^5U$7IPd78WWff
zvNUnB2yeBRSfJD{tXeAI<I~*jw(Fx>-_P^GJI*~(d{&?_)A^)~*EEhLtj{z}<$@m@
zn$8P;>}WbK<Z<BEMXb*<w<<+;B~--qvR|u6>UF=iN^HjRYZW=Y$FHqA65jOiq^VqN
zc2vT=xW^w&<>DXzG?hzu%xNZ<m~B;SbM##B8=E8NoDoFNFaIAAm5LIRL%a;2G6M1+
z4L>^>*c{QT^gN`?zwxuc;wWLeA`7TARCkY(&yN!iKL}WZ6hH~XEbXuTHb<c<AeIDn
ze=hgeUE<+2O+;04FIRk-!-;~1&sGPr?+UA2^xyh><>BVksNL6u7wlnqp~q<V!r|k$
zM!s*1&+Q-lVDAdD_`~?9_`peiiysf?&j@Bb!!(1{dF2fEqaM4~u({_S`1#&>jYPoF
z1}#Xeg5xfxrER&iTz18pha20D@7mHoS+3y1|4z%l^&ATx>wa;5Wgk$upw~(A?W-zr
z#$<7ZWbywCm~9lwkFQ&Le4V<}m%i9g7r_T7g7!zd`*u8to#ozA`sR|X$k7|B)+=PM
zq%634L@&=dr9XC7ckHxokM6&{+9JObs!P+JY(Bp3n$zdr*u0(zLAzc^?0z7z`^Js@
zH_q?G+SWxM%hPG-6WZOXRJPmY&96__d*`}3b~N#@3a;vI75pT8MBT&xL}H)7JOyjV
zl2=IyEq7QpzO;`~%DA#2U*do3zq{Sv9Hfu$zpzGmALkFthhdqIci8vrvWYD}?i}*`
zW9@<OoaJscmz4$g>#3gHAT&Ym?tj)ZE$@wanYXaG@HGARmT);&KI{5>_rsuyxsKrv
z!;kL_C5)S-c>?s6OLQ%Rzr48=X3j6N;;_c#i>&wcSFQ<fas0Y6uCeLZ>vc}wf;jdv
z1Uj-F)7yFZL)Vg%b$5Fm53Xpxy83C$Wwo|Lq1HR<m-bGZYRt5{_28XeA+NnQ{_FFv
z^t!Cjc(P;hb&FS}g}+2o+k@r@C3~cWciz0aQe6L+l$So!*P{#en=C(Om7;T^@yg8;
zLH>J%1W!pw-k$aK$tC@7fe8u!e?96xxky1&Z|}A<d~+Xq_xNs@sqkG?S0wuUnYVlm
z>l4f6J(Cs+UX#wOC~W%3(AfL8o$p2k|7J(tRd!verP9&sx3zGpYkb)|&3(Jkx5V;f
z$uFPc{w`VXe}5uJ<>H`?8dn-NK3}MBrew~z{Ng^}A9s6$R`7>rU8+^CK3Bkge6QQq
zJ#i+h>@ObObEqcu+N9Uq4wn1oD=*)=^7_n^fA2`?cx%WVatb^r6m0wOTIxy<j?iOz
z?_&D4^joD^UtjwFKEuv%Ij#-iCD%`^`+B%ibIk|Q-9`d27uMYGGfHS>JzP++(XLi*
zU0A5!>UD1gQg%(_D~@b#U4H$_FDXT?C8;tK{_Q(+^qqU{d-a9OJHM~enrMBYTQoJo
zh07-{;`52buWR`|BHrw}*A}|?mW#2YaL@+vME{1qn&2ZB-`?I_epTeY67y32g@%oS
zTTk<tZMD0(B=o&`ZEoEDm#=>;2#H@Xt!GtH>#|&tyw}S&`k!8PYH{GyH~kL;6!c^&
znp1OJj>Tmx%P3cnyq&V(^nuJJ6Tj8|u1aqHb3gH!+a}@0rM;3b&z)sXc`dg^*87t9
zzUMLDrLyAxzYkd>eyx#_=kePsY5A+prr$6>v(?!=?VFff+gx4F-~Y=W=x03?@!LOr
z!>gTN6<Sw3yiySH+wz~3*YT69rhJ=wwPKDP<2(r;5mO#!Z6?Oo`|ibT-ZY!t@kxY(
zp_WEcG21p}U3rrWD>9}W`BFVm<kokC=j)XjlRmrt&sv-~BYn+u0gg4#Cl`A8+<)yN
zxR#5@=<m5a9ZiqmpS!d-Utg<t%xKbHr|HdQSN3gK9G7yK<+s$ghmt>=3e~NAxjjSW
zH)lC~T~rrezhIxx-{KkX4?OQXUVC*>hEj)J(PYO74C~kXu9ua#V#dRzap6=)+=7C?
zx6TJDe0=XeNrX+z<n)@S*J648o~T*twb1gjg@ehYZ@OB&Q<gTim~7HyzIk-v%ikCO
z$KO8q^~Z|GtkM%R(|-obKB~L>G3YpVz|Qsa1=wf5oO&i<VR6)}gXgSRa|>D?q#Ryw
z_rv<tK4N`p7MadP;r0t>`jrUjq_c8_G-a)dcJ+B3ADwuJBfCcQ)53o>^Fxn+-}i6f
z+#UTvb*-mwgkOy~uufj|({ZM67xXP;HRM($-xinJ(wvj{c-iXQ`0rE1i%JX*oLV$_
zZpmVnxGOs|wN74NsmU^pqk6_186By@MKS6XvC|$_Dr|kfjrZ}VZ-=XIt^K`F+Qod+
zttWmtn*&=em0z2lpn35G@0-xwO?iDAa!UI}U+A(Qzq&f9Ttoe&^DWNMMw7<Y`=MvA
zMxL8m<llUIm-0b2y$L57esA?Uu<-7N$y!r6SEn>j4U}AK{Gws<<O&v*qi=OST=OxK
zSvE2Ab$JJO^db(1K*NSb?{tc9-{-%l-6avIe_qXV%LR7EvYV0h;(AIgTlewXxI5PL
z9a$gv;UJ?6<8C|txiSj5@{61gy#3B^lzO9ytx!aLWBBRfDf?UM&qN#)>#-BsyRYHv
zVX3bXO1j1yuXQAC?=!yieSNg*TKR|T_cD0i`^~zR`<M9>?TE8;a|DliRm|G8;A;0j
z);KOntAM-?>*tFYL#@;M7a6<G_$e{f$wPFv&D(;=#b4YPDBL`s+~4^!+qz<Yc;Ml=
zHR@kpI%&<U2u?gG`6gY(eR_Diy7|Xj%Y~%(rU~3IVSbZ(U;(qssYAOa3a+^A`n1j}
zVV_vNBKP|rX;aqETH@+6<IV!ld_||6siz!8lM+5FONw3JbA6^yM9~l1@)L(x|IE34
zD|30E-HRVbY}+cDKB^iVUH|uvP;BEmr@9qeZuy=+w%)yWubb7yi+%5=in;zQkJx_V
z`s8&Y=DWNuoexcpQxE3+?Hm1ERW{`K$9?9D8e?~LL@j?O_Flh_LzijNB90t;L6wOU
zmaKFY*(dWb_{N1pzs^Opa4c)nI^J71<(;7KqtiF0pQwGGES7yvWUu0~=AB_i;U6ZS
zQ{Ja`{iwB!xv21~9p4+08GH`@5=+!B-(VdeXw%0vMdQE~bD7g~Dw24oESI-?(IYu$
zyS_I+$HWbH(`T>bHjP_wym2Pmlo!WdYf2`~-F<DwMuWMDMd`*%bH##NR)lP+yb{L5
zIU)3=SjcgWJ&7?_4zHZ|PVZ^odV$!NK2Nh<9v^A8ZK}BF$`)5Q;m<$Y=bWcJ?o0VK
z#;jM7yRi4}1hbyI=kNSY+S4&xCP`L|%{$`6fxtc6&#y=pO8v$qyQb+#2<H|?ja&Aj
zEA%xuLtb9_J<GRgMNosl#MRy__iO8@EZ|Mbj_-3?!Z_)DXi|d8Ws{X_<~}-keC~#q
z(o5?C{vU`sJkh5{<lW2XYlHd^Cb<YWckMMRR`<vcdST^$dP#7g(Bz5d8&}WfT2+3g
zSW4b%Qn*>%za)dB8nq3_STCNPckl+U%7k`qe~F&QPZB3h(_Y`|wC?qp1+PjSvZGJ#
z^gFqt(4+Gfn*^VQX=cfRc^8`|2=WF-w3jbelfGaSR>I4YeEiJ;yI)D=a)Kf4!iVBk
z3q`v==ncOz&sF;5IU&_w)6N)QJU=mb>6#S5c=!Di-fc>;Vq)k1draf*)pUv9TF&ZP
zBD*JFo^m<mvw+s33bP7Bja17oEnSaKe|kNKJ#w*2Jp0=9I_KFu{+=kwT(Ejh>IC)V
zAM!{2H(s8*`e3nGp#$IT;I{nsS=>7Vov%-dTX}sU_hFqt!EIOkJr@{sJ_?&FHo-nL
zZ*6cJ^QSMYIV!DxCn;7stoRir7Lb+Mw|m8b>l-$ygkODo)w6BGgDn@om};to-tbJ<
zXPjPk?Yv}v%tO}6?-y38eD(F-Jo(Y$HuFVW+K$Mk)OB7u|2b!=r^q4MEepJYZ-<^e
zXR%=Y-qTMW{q+bD&5{-R-uGa^;wkyd_od9QUBBzUW@62g`wOyLw=OTAvSS%je5Hw&
z*0WHV&3ft24l(!CChZ7${?0n3v+fl4={v5m87GWx7@2iMzA@gIUHRLotSo;bYmKmu
z?BQDvuN0sB^>;#Z!`{S>^q436EMxD!ik9jWJsj0-b;xfk&(cqGwbGw(3w38mUs!Re
zkv~gLVKqyXWk|V%fhtSip*P3wiJVIl*EuPAVDS(4rBByQczDB6M00wW*TGns+tO>D
zcCT>~ImxZ^hM{fFg83;|<Ml$GCOF((dA~`&!O5-5RhDtH)-20bx#^6XJ*Hl-eDL#k
zmFBsOu#f-$9y}%zWxOh*^@=b<0-MHxZ%mA8hg<H=lwP{&*a?*_k{MeHF1|Ut&oo|r
z;alN@@=7~0xIMxqSDpQQPidwo%bS+l3dbjI5n*L4ntAX5vuD&&rFj*TRzG4|%HGs(
zZMkFOrNsPSlB-pAZ@3uDST0ei<+3#IrD1|W$xpUS)d$rk63r{SPYABiQC;OH%6>+X
zJ49oF>f}1r1$EaiNSXfeJG<j$yHs2XXRmNKUx#r5yP#0vsfV4P%9jdxrS(0qnfvO<
z>uD;H--H>xrH<}eH>X^BW^j)8l$%O=x?Hy0(~nkOW16_-L`3|Glu%FhxlG?QI9J6)
zFV%@Tp)GZD$LXjUnYvApyW-2|8%XE+OZb*pRn9rSeg3=~8QM4G5B*k=7Pd%fk=}cL
z-^E{nE4Ysri5z_Iw$ks(3ho5fGZ7n8=e)I>>1DX*0Q&*vu$Ef~&dVHa><!}JnPb$Y
zGNajAbmr-)<}XSLS`K{L`_*ET+UJ=0t4vRx+Tt?dvS_RC*HEn^U*1(XyscgJMAUI%
z@1iK>X{(>RKkxlv&ihB_53^HBdEkLFF$W?X_h>pTU}ssI$$Eva?Vzm9+*ivaCnh~#
zYSehxB}(P4(}P!T5>i>7){YIk1RjZ~X{k?A3@V+HU62*Zk&;qkvg7Tgeua0YuL7su
zie1>{z2=4BcHd>s+&Dv~>6j@R{JfsfHg(qfC7WkYc4gJHKfH@uC3u3?+s6rydW|1^
zyuTyK`rFz?U+Q(A?4A5m>-3X=g>%cqy&R8o%4ApwX=JEes!E6weS1h~a+K?(P@^Sv
zw;twdFR5ABnW21VI@84LuS$|<*vj0wb#%0j{PehYD<xi==X|rRs?CeTjVB@x2CbUY
zSX3Ues%>{p*x$P~k~$q9BumW1Rdy~vSiRcn=mEP+<vl_9g8B_?k)A=jUKZTo`2ES`
zLPf=-C1GhnUA6MMyxZJ1c9ofEN+0_dxnbjsIRe&49aqc=vUgg%k=xqrOy$hyyLVW8
zGuVB4a+*o$?8`k7FZ&vz=4oXLZk_moz257<Up@U6m23+cN5AbomqoqL2Oa$O!(kf7
zR<<iFb0m6HFUj3Y$yjQ6d`Xj&8pGnHQ}&w9w4bxgN;zIqZMA~t{P6IUZIhD^7;#_T
zIq8YRvzRjtT%94Rvag)G+!mN>GVOZbqLp(e)$;Duc8?w<&xmQKSL*AP{_>OM3Tx+)
z<xGvyvYJ$|%!BQuH+N;ma^_9mY2r_FO`K09=O%qw(C5~8^&&UZWZvfQp)*@Nx4$mP
zo!V~YZ@N|LwPE8Gk(hbu;R`C~yU154aB{YvjuLV&H{MpLVzu)8mX{aiBu`2;p1xw@
z^Lz<~o+*dlTr1>{vAYnn&->PdUqT7@bL~?7dDl!(H8oHOU|Do>hJbIMezKskg<j^z
z;+YvclYF`MoPMh*bfKeo#V3~YBC4-gtY+Pqs8%q;Z>d=!Q|rI<vdvv@3dNr;=UT$g
z`>nL%d80z-Kj&_xv<EV*|0Z}ecBgMSVJD!fv>|pL|I#(wUq2KCoas>dTN^WVTIH;s
zZ#I*HpG<FC7+Tlkw442HYphhmsR`|xdxKtF=J~bm!g?0<v}%df{fDcj9a%FgtX6$j
z+N%`lmaRSym#qHQ7|iiy#$}g>8T-nAEPQgb(OAZ>UcH<nVwuFtkJG1ak9V(dKK09o
z>&DF>XLV(Z7h2^UL1L;&zdSc9Tx>mUr=OE~<M`g!v5tIsZ7DncY$~fr+FkBDMe9<k
z(`EzRdm9#>-aD^sriQ|dW3j(}y$dmQuwiB`%Gz8Mv+GgI6th_p>v?sTyfIn!{G{1L
zK7L7ONf(i;^U`-X9JpUIy`V8Ag6nY5Q>EF#R)5W}eOwV8nPL7^%|d$h+Z8dsm!3W0
zbG{|%CnWgb&;pSF@mZqnUDsZCFS@64-#n7HhQniXQ?{>|`8T^e3;&ihGxPTG9pSj5
z)Ejoir%G&rpV*3bf{J#2Q@78roH;3K`^&?3`|jTT^x~w)d6Q)~|MS`1@K#=CCe*?9
zn~C+{1Dni}NLSN}CVp>gpNmf1I|BoYX0~TEy9bD`mGOKOGu28o<@M8(Yo{0(uY6!|
z{_T_?hnK(XKK^~8r}k{q><ArS6_qP9^j?_q2u(aRV@0obNmH@*h3*@YW&CDR>a(vq
z*w;_{nUwL+NZ7=3+gvWmuR4~O*yUV*r9C)1b*<q=-dlV7-`KWwZDn;d6u9)JV}8*k
z(}*U`2M%@o3(j+%Tag!&7xGJC-_*lGmrgRrKHym`(U;l0q1kj*Ae+o2oti!#y{?qG
ze-D(&%X_L#^gP~NF{>=_uTwiiCfgP3^=md){zz`VGX2Q67+H7j&Sa*geqALE50hS}
z@0je^qGsijtykvn?7Wz}QL=i5NVPj(dt<Q6Elt(rEtmXaCI)so`d)~#ejPrc<IYZr
z&(}?7diYGV@_H<GBX2^e<ir<Btgfs2edbKM@v=8@bxd;s|9|0>X}S%vWse;HOI>jM
zp|w~?lG#>Jh$Z#fw8~ckc1y|~o-~&R2TrmTHDh~qaONcw=YK+HZ5UTgS2^!<dLwt_
z%k76J%DZWZ_m>}iR>Nz3`S7yjl*>QgXZb8SD7HjRd)vzAMdp!dOb<(1&glsSDQXAI
z72wY3*pc#rtz5KCh|AaD{LhVY20L%R*!Aqn+c^_U`dE9FYM#gjPU$_juh(p2UJz?;
z_!e>7rcDnH@<<-pt#YP!hSF&x--!ahRQ{FDHcYkKwW-PTM~mrVtxcLi7pMGPbg4;B
zW#_7%Mqc@#32ZEP6C^k_LqjIXRFwGcxvDVF%13s>X|1=;TeOvC=r+fOwEa*>b3CH!
zrM;}Uy=c<m15d6jILOqZyQt4vSLjRk1um)BQ@Q<el5c|FOW!(f*Oxy<O61A>1sBw&
z8hNZgAUH9w&CFEb8_S8-y$ZaN*F_7Z3Pgj&(<kWtH~slcYSL6iJrUkziTsnxO13+{
zex0Iv!pfz&PewFa(7f5kP>JcuGUnxrdheQkP!gK8lINuRwG8$<@|P?moOEk-e7>qH
zU7y}nE;G$+rfP|t+XCS$-;}hS7K*ZLu?$qdrI6~mX4>x452qgI;NHksCtcO0ReNx%
z2wM<)@^Rs{n=KCPzWnA`Y~q^(0(U)F1s6Q%ImGNg-AZ?Q$O5JbTjMS?P1^F<#p$93
zyVL%hp0^9D!X@K_BzZ6TH|C^St7=J1=9+RWhdn*#i>+x((20KxiF%JDOn*hb*mBP3
zm}r~1%FVJJFFPjPo_mqWw~#x-Uu27tNd8==a`&r2OM^XmOg+{Mt%^0Y{XHRn#V(Us
zGYfx69+dT&a>rexP1{Vlqd{xiqWQ85ckuZo*>r@T)ZMiEoyld@L;V}19=x!Qa+LaY
zTxxd3Clxa`(FnPVEL$`k6WsErH!aQHQh)OBkspgsG|cii?914nr#H9a`JcqO3tiV4
zO=?~yQsur{rH$>hzOT}b0EJ_Ep_6lue_geL;S86w?Hid-+FnK_vlf;-iPUd=czUYb
zez&lx8QQDwywwT1njxBeyHe)Vx-SziT@IQke>nHT0#CDDN2OOk_@a6^ZL-U*Ll%nM
z@6W7Cdou5UM^j?Do<~aW>)^~!3MV@vtwchDKgC%-m{|Jg>4uf;PjXChrUh?opP+nH
z`_gKLo6Fy)KHv&#>3rg4DDY)^;N`!zy(gbOV2*jy%P6PQc$wEXrKr<!UBsh>La(wt
zn3ZE%@9jM&ukh)?T~@aZQm#^)W7zM!To&rpSjL<wQk1vTX|>|Lw}(7aQj~r--r|#V
zYp~wAA=#jYVG75CCXM7S--9B%D_q5w&#hZ|td#k!+4L!Qr!0QRmBC?h!g95<;q>N4
z0}=K`XExh>o7mViRj$9wm}l$y4qes4N{f}ox7WTuv1CbM|AH4y3UjkOR;}^4nj<25
zYs0S#JN6iN%bDc3@LybU=vlEreN*7`gKK9sd1hqf{ge3keZ&6G(OXtY9b8e^Vm9lF
zzI#V<z!l5WEz(uj4`oY<zEmk!b^a_7H!<~Zf7e59q2}*O-LnPrg|07}bIA0K&Tgi}
zibX2c?n2ffS`$|EP863@u2{(ZR>b=6mNSoE^2jdC3Aicwf>(0O<b)lEn8oL2$A6y~
ze3(Jt!=f{D3nE|jn2Bh%cFbC=f7)a2!kwy_)_RtFo+3p$En4}0s{b7(%-mof_-4wT
zi*hDTo3GZS>4*q8tduQDIngwu_1#=eB}=F5;1x~}b{~m(IQPSWsuPywapr9^S$db8
za6Hj0F1RQ#vFN^}#D!Tl1#6B3dz`kiHN0u4b#;wl<k1NsPUoN8mOlKX^UqhFX{lA7
zQ|5cD`gSg6#l!7ui)^MZxWI9&g1<4<J>;0dl=l-7m3O&@6+L3R9Hu<W+^eeh+$pxR
zPN#3Mib=FxIL&&9Gr5GT)|SKRvYz!~_Zzyin6{t37+X=-e&uZPgwzS*-?}y&Q*qwR
zyL0j*2|lH}WwB;OC$EL6C?7p8866~8;A+n8yh`F=*b4>Y)O)TOA9XdS<<+0qw83!G
zylny#ST|O_xFzTnxU|v7Q{-fdNu#}*^Sj203r^i}-kf*5`N<|(hV5R#(}E@*uekCv
zP(;T$UTa5{>%xXpZJ(wZPU4mH?7gP!(s{#~Pc!6(@UuhQfloQiQu*F&X*sq+WA%xj
z;!(`Yd`@pNuF-s*qriG$#yOY$RyNmrSG&}%e5`D9vZ9FP={2i=?)gPI?f#Sd_MV%r
zo#-~<s;*kDNvl%pewkI>hu%vRMzl64izV3K)mFARz`DAL=~3C_#zpO(L2Fk{@p-_}
zI!*K`-$FOB)lIt!Zhce9Kd+DzqP2oW<k$6{S=kc`*0?U$NGg4Cp<{EOvP+|PL+Kq!
zFUHTA>dZ=swP%Y0uW)}<I$<3+i|6H&+LRd@ryZ?73TjIlZ`?Aw%OPk}(p^s`F4xyf
zLPM1+B1;3~I!=XMb>cleuSU8xNc{VXi{i}dbz*8{q8~|#9X+15hxeW3kEsfU&iqUJ
z*UX;MFd@WT+L>FbRb`fj#!0Ovt>UeU9ahd0pRnK7Jp4UE=hVxeG1Ixe=P&GRnCSoT
z&ONte>A%13Zky6zp39}~;MwoTb>a$7pR1I>fl7rRR+?3}<dwZ<Zah+%{z@+3m&Gdq
z*YckyoF`V~^<U%_Rg>8H=wZORAG^KE7EIG*R)2L(vuo|8+n&!ge-sAni9SBlsdiWB
zP4nE8DT{uI{}o#j(-^{-(~@#EHKS)!^Mx6VLN7Ezf+A$LwsVQ4O1yVp5s~JTb*=fE
zz^}$A-sc|*xn>)A9L`+t#38eJpESRclKP76hDGZ$>dR8CPB`CvyXT6u!L_}QH`eS8
z=->)9ZaHgUs5)Uvw)l=XLAw)Lrom!ze#>^h?ogiK_{;B9j`&JLk)o>Lc@_s<T7ECQ
zcS|y}_>%kd^<tf`1AY7&!yUM)gw5nxGA>*w&R2XR^?lmUSJo~2wq0Jh#dvP+^3x2f
zzMOD;S)voRvSiCwHOtM`Gh5hoF6`mGoA*7CE0{YlMRC({!B2~G9{Sx?Slqb3V{@~{
z`zg{k9xN-*)JuN)-XWjP&@y>)-;4Ar?$;Z`&oteXSKcwltw-g3-F)w5x`!$`B>cIW
z!xrq_d^wwCLGaWUVV`g9U=7hw6!`K(*}~pgZeyOwD}|zYH#eNg6uWn%MCyOpuixca
z0@lyDgS^ixr;0CLb8|1lmfc4m1__FuR#NVD^i+$s4`#QX7iyX3VJ7+D=;mXksh=cn
zPS?J&@${jD)>Fl)cX;RSKP>LnT@>&pd5TnIyXtbeBg-=4m7ZEBTQv0F_!X4;RXfnV
zUr6J}vxn3DAH8Y){rJ<C12u~MGOnds(<XdfoWwFqDbe7rOR?%9qX0RnFEdwaobd>Y
zNx5FdtifKXqWb7iOFe6fLXpjcpBKMNFUecbBY1XR!fw~Bbpc9&N8dfP=(X#aCZQZB
z7xen`zDH#X!Yx&U`gE+#-$rv;zjz|GLa*+yq_K~~w+sRI#g8Ry(w8rGS?$4El|Di0
zZOw`O=_1eEu3I>pPKe%X?^e{(`S9>Ii51%Jo-(|XZm^~4t<CG$)*I#@&b8i%b^p4~
z+iItd9#LL0*GSBDsrQ4qJvxjImyMJ@@``;q_wHrO^4(Ydo|raaA9wj>h1W*45~}*M
z_D3GNCAsiPQlsBzmZW-(w}-@wmvp~Z{&zUT{O-bfl~<cO&MkADsSzRPwJg(B?3OL}
z?<aYdO_h_F;x*UHR9=`Ha4}o!sGakT(;*yBz4|xxSyZm6@!+`Ktk`Cz&aWJ`aFW9N
z6d_lkiyl+nZM=S>-P2@=<k{#}rqHmrjB2Y>g_M%zau+P-XtdlAce#pnbE=T}{nQsP
zoQ^uPI86<FaIo>;MGISHRw1Rp?B$Varw_<V{Ml)GbGy$&U6r2BdyQ_bJ;$3HnKzhT
znV52?zgx_DLZD$2pV$SZe8+6{^Pd8*scy>orpV)x(dD|a>)y>0aZin9!LJ<oq80{o
zUYU2g@6Oj*enN3_Me{BwO<u}jWFM&4<keo+dQf!HZ^`mkpB!uEHt3`~El87GvDW3o
zQI*x1u2qSx`HYrQJW}l!XWJ|*v|7G7yz!_t%ZWEaS*s-$`sBDRyg8MT+c~~J;`T;v
zXVG#&-)Ab{@4Mu<lvI>ynb}-&3RE#TY3}>0)!?Us^wt?RYsz$91V`R$de_bMcZqzH
zkmmWY%66&H-EQt-&iRZAy=U6HL|DD=w!0n8ee>1e=oPM|lD~dmxMSC`aI;y<7KItI
zMq({fzi78|6_j;JD+Kzk>-YKXwNP*7lYslGIgO_RmzIj3U3745U(d(bB)(f4!bH;Y
zKD<4BXTNsI-OP1qx$G|wv|sNQDqq??Q7>)QQtiWfRWto7kDSn9-Yd$bvPS*;>dXm+
zZvCgG-Ap&|N>u#RnpSaj;)~sJ5msU+j_p!kk~{JA#)%@~L46jYEe(_AWL|zLc#^|(
z<>BpY-7ZUBGd_D@;Ij3^)^jOW3k3NLXT8fhA>Hf9v1Pkx)SifiZHdoMiK@+CS<+}2
zKI7o_M(Y<}GTCytk2w@?=sm-sSD>!Z>((fk_}s(i(}@ESV!4xcx?DTkZZp-$LUY2!
zDdC5hVrE8}uC-lIzvNeI$t<~|N!Dr>ryV2v4J2k*&k<y5R4cC6u;AWR<GuI}Qy;5q
z(h2Eh%zOv#Om#^uOJ1UMq{qR&IOxQdivR1RUaZ)ERr~d!ysI-MS;9|tw61bV^jxd;
z(y=N0>AFo%cz1>@J;)=XbkT6ikz-}I9Gnw3F3{QBonx`4Rn%2zMRKo^@kB<~2<^yQ
z-&bzZH92zc|Ki)%BA>OoY$@v0=sFoMcEN8?(r)g(v)ueA2<xx>m$6%bQ{}YbQQ_q$
zJ+ii4?Ob!h#_UUO8e`dX_ngC%d=CaL`)4BfLgRNq2!r#K;u85Vo~p=~WttIjQ%hD%
zN|I>1e7$YK!wW&5(iZO*HS;^yyVJl!HH)Vv|I^iXS~s^k&tB|r=lx01cf}-u#1GsS
z$$x7Uk7lcgFHZgZu&MmlMkZ#<Ulq+Ftg~*O*V0gSn;52^e91&;smR(h0n3$l?mV3>
z@OU@F^d)D!R;BG|d-J50@gZkwk5Hz1W9qRR2Ftcu*Sw6&oN{Ut+q%cuQ}*>Xt`ohs
zPPZ%J@|MS&79DybsOec+=W}q%mex)7cjq?l$#-vL3r_JW|E%?+cVdo?R>$f~wJT3}
zI4yW^gHM)^bIX;}A(3qCkxcg7l7_FNrr2Gax!^}j*$t5wT!NMXJ%aun+EGd$eH^To
za@;?)@a~Fi?}a;9k1OlGX^i4g&W<^EQ$>D_@tKZ;n|dtXcT7>8eMt7s0h>8fvwb)#
z*Ho!qpX2`1yKVW&EeA!UeNK8jC_cfo!iIB`<ou`Jw@$27^=;ywmu|VMpzl%KQcYIx
zNg2P))%J#8OjC{S+IGM=BD9jhNcQ8bWo#9J6Bco`y<_RPeEgHQT~|fmftR~On#(kv
z2~YO<zuw)~BX!#h-pFRJ;Do0Cez#3t`8+t_cum_Uyyrqr$u(}vys}J_PhSl06g+uh
z@vd*mb){Fl{d>OXDcs<Y<ak=6YvNh-cZuee*ymdfe{YVSI^(bk*PA8b3O?4R^>VpU
z*H892sOtEcw9h${xk=}Ue9`KT=b`*<H&1-xRAN|Y6lf-PG+5cgUo~S@jLxf0W}mF2
zWhP&LtrPJ2n9*`ZY>v>=RgQeq3;cx6-#Mta<?WWY?<b{p?BNkep3%Ggda8eY{llyO
znHgs2DQNR-QFRO1a4Ms#s7}NnO=aH1;&j#{4Lb5)BSn4~O}ee%eNs=}ZG%Q1YrXfS
zOFwF!<+mDMz4AcdOPofz=(cSdBA*HsCDJbJJ{f=Tj&%5ix!Uu4n(X&{+Ff;VLtJd*
zmD_9erdBQXQo1xjCL?j0xnr5mEuII;**sDQY`J<Ke{NhdVXIVs>hraJ4AN;=Ggz`G
z^oA~HpRZ;A@zE|5yN(@mJN6`{iMzK*v7AW1^5dsNO5$OLZ4R~;Mn8ME+;Uwlxyay=
zNLkoLr~7TPOMPCg`){Z_Q`A38j<ddFU&y@8tKD2ut{F=Pg?5UBbtJm-P00$%d2k}*
z?#y||T6zMHwjFK%*eO13;f)8aHx+X(H6-(^Juu*I-<k2HHn?9>tNQ#yTcb)vYt4&y
zALMS8bG<0PKq;W_mdAqqrAwMF`9_KAIh|JiyW;DIkA_T3C#8k$I;bV<HT7Ii;EL7u
zCk`!f+j`uh<J{ANg%i%_7-<*1T`iNGd04oa`~LYt<%M@54qunpm3)V>^+`CJUuYW(
z`>Z~#WFf6yryWZIeOB6^$jKIyNbq2sy~B|;v2!j*wYSkMaYg1SW?oM$R&g&-P@TM0
zD@Spvk=7aUH80v#!#XroAA8M?S^A)jd+QPlxg|0V>E}A;gm*eeEnj(Xu^ZF-wNv}U
zm~tA07iXk;eNWTlcW=lskvuW^QI^)*fE`Pe@6KeLQhQ75uz<u2-iK3XNu`?3s{f$B
zFY%#nSbj%|lj`NiwtR+s$#QE%d{{J1ULJJd{J!Ahm1BwJ>x>>ODw)q!7|yk5s@W;Q
zqV9<=C$+nE7AdiWeG0rb!!E?eqw#8ckJjR_pck`(4mIrGvMwy_?2(ACK`I|U-`DsZ
z(aODO(T!Nwoa}OEm462=T{MXAFJerzxcz}seYNT?y%br)nY$%RTpHXLbZYUPbUfy<
zcA}8*BaZMF)88z9qG%VWo0KDd$zSPdz@Lczs)~tQ^n{KoZ~nJJBP2@sqr&r+$zp|e
z0e&kk-S3O9sMm16IoG3hm60rm;VBd2@RrYEDW8mjRvp^dV4zUr+HIn(rFNl7EFz&p
zJbgj_MJu*CpDimZB!04Jc`gxBs!;m0y+Uk)_mbo(LE3v2ISxJ(?k#0x*<u)R(zN{g
zTDxn?n~!j=IvTEavDegV&F86|O`emuEyZ<{7$YksO)n=*f2;Rsi9|fxjdwqcUNP5l
zTfgUU&*T5Jk*WSnPiyC&BPqIbde0m^ab%Xyx*PI2zoz+{*j@bL@_5a+zb7Vy<t{xl
zYun$&8$Zh5fA912cE4J~iiT*pj))rqFH?2?J2UI;Th4q=^y--zp?7X+yM?mwg_lGn
z)+M+8|ML2N&-1P(Eg|KLw*s$kSCUv|<gxftOMv8MHf5WszIOX08ymNa3e7l_s&U3d
z?@n3Rtd%|?2bz1>OwTUt(LD4vWTHptO7pLKye&Gap0=M7GUt(bHQmiT?&$I}Q#it(
zvCePWI(t&~($~9;0+s}Qk|~XRQL=iAc*xS9p~sGdmnE(iiPVl>9s0{jyLwUR(YqmO
zQsJV%)~`8h`<!!Ypye*30{5S(6_d_|umveD>QJm(B_ZBb!ytO`%3WTU*HMSg-i*8x
z@Af9JwORFq_L3L>Jl1NZg)d|en<cW^$yrk+oJ~RdZQhUBT6gL`EfQm^4n4-BC^*Y8
zz(mekzFl<VsTe=OnRn}?gVyinob&9%sy<%p6g#)x?aCZ$7X+!ic2)axL1NA}?OE;d
zQr+B=+&rbyUW+Gm-Ipqk{Ast?SYEYJ;_^j}6MaENZ%P_U9Ne;dy>7OJy7Srpj(%i)
zjCu2)%(iSj=kUa*-My6$O`6PX&i?ORwAlLW;>WQ(w(e7SO|z#aFPPY6Zs{A&rFQz~
zhV@dxby9Cn_by6K{&QTy>@lzO<Xq|1&y8-~?bk}u`CvGwhs$uO>>mFa5Bu)UikNz;
z*}`dxQp0*x1*wUAr&zS@z5kbN4srg=_UziF;$tVW!`PgzF;4FN?zNrwbL$d2srxTX
zzJBC2xE8c2XVXSgtC;~y_sbmprD^?Wk>cm=+vFQ3oLlv@V2#l|{g+Dl*_Lr<Y(#lx
zuwD<z<O;nx;r=PF93N|A)%g#r!=lS>-MA4f6E9j5{IS4s<*G?;1<J3CXHP2AcK-CE
zGc>DLb=`z}LJ8M1Kh?4xy~UEmblBvhri|d>Grt3>WU}t{c@{2P^(sZnaJk{;Q|?z5
zB;0R5T{L;4;Kk&=`pAmD!fBGO0udhC?#J{Wt&V!~A!h=kG?%-{u|m@?iP<8ef>YA|
z^kz!+@Gh25_z|FVqj)h(`+-?=7vFv@n0|Q9=3{Rf1Zy?yS!Jt!1YLf#XU<WX0w*Pz
zoQ_*hnzhPA%Hu3-)4ygu*E+DrNniSZNNgaR@`VE(>t}2|T@WBP;ry8c$JeX&=lo$2
z4{pBX?%C3_W9h@Q!LtO`eqHq?S?#fHyo|)zg*}TFOj_R49qy8Kp}Esui(l=e&eE#x
z(jy@~*JL`q=R`c*5~!2&rSHKa{qr+2zq#hU2)KJgiS>RjOTm^44Y|VgpE<))4}6!N
z?BXoT+wiG*+efzZQECh3_J*waDEng*gHp7Xs6$FcN!pdi=bjt?Ew+C6-u2Fwi^2CK
zTCYS{Zojy9(VgGR-uyULAXj-u>Ti!0*D|ZA6&qyt&Yj>T^mC$kT7lGRi``yKSrWc8
zS!4K~FPiF9pY-2+vxk=Ww5JPR=Jv=eDUQ&{lV;NXQQ#L<HEnNe>OO`?(`vV@)AG2c
zAK`oCh<S-j&I^|nPfj-7*~xah#yPy%&3?0%TmW146x#zk1y}ADzvo~+cjKiG=eUnA
zdEB=@`$6!r<wERhbDkb3U-?Vy^s%J=jGxDv1HXS@ozwq*mjUB+{o_(%lV&KH`FQTQ
zD1Rg3EVrHA(SkQSjz4{t`doiUpcHq0@7o1g7eXd%bSRTiox1D#X7wtA0H;!?2ma|R
z4F9=W1RS{k=KU4zpI;k#Y<A@xzJ60c^xo!=lYZ!)%&%{#6`8eiujk_58!on7$gs%!
z!gG4|`C0aRLLdIEX)n%Mrgb=wb46Eg+C8otW{V$8NxPllZ0BY8HYS_vzVFe7s|%K1
zywKxqZ8a<Oo#^#t7nx7IO8vX@z&-a1`+pWcK3_2@Guua2_o<F1Pxr=CjV8xdn^|Q^
zwLW$GT(bDgoheVRyqFxum>W7NWG2_qnFnpprR-e5KK08z3%l@L)^a7*R>vMrI}`mP
z)_>2WI>i;uQ<cj*u0(iMWzKz9A=-b#=itZPN&kEWr!1TKxkzlfTFNx0`=ST0??3U&
zF_$MNW#P>o@1!CxZsl~)-Id4_ySP`TW3_{+@@0V){ZF<nInG^pfK^rCO-E;#t42Tb
z3csmZ$$34~Q)e<%6_mU2zx|e=GWSRktG`>-q@B0EO-Q!<c>Kum&>Fsr&vMK*Zhq!e
zAMmlVl_j_2@TAvqd0E|;SS+lhBhM@|WAhP6l@TfKep;{pYcJP?O>KhqYz&Widlx6J
z*pXT6W-wu8)$~<%Z&&FZib$N@cBAoNk3@!#&trGC>eU*pO<T7o7OmMK(zZOc_R-pz
z2S58hy!)rfnz^cZUHXshFBodhwbXB$cdDx9i?D>~l`BuqUfS`hdT)QCU6(Fn!?N$;
zdoxplLRqVB?f6u_ZkBQT)s-!mZmcSm3Jq{yyCqQ0-SE?#Oa;HZ#*H$KRW*wQ1SNZC
zzvK<-iSs+I``6JlNvq{@MgO6SRMCvCZ_+>HMWQlSHM=j7(<@+`X!xYkX2Jc<`@;G8
zm)>WYdqiOiPuD}9Ro>z=TDG(6`d*mCHnsS0(vEdpa{@$7u5T*X>{gO|A&~zR`~CA9
z4j)jsc*r~DhfGn_%*#7B|9>PWYAbBI&11^)%(^L(U0sFgEdt(&AB`_ZyYGy0xxx`8
zWv&*-++H9Mw)O1IK9Q!2YL2`ceN11vIZa-6V3tG2PPb{>J~S`n(u-&m<k713%IWL)
z<l46Vp3gLC*TTh3?UI?>jsKOq>&Pm6R=oaayF<1z_ePh84p)x37fCP6`o3G@r>KSK
zHAmlDyFD&uh9~^++xmZn(Xkm>5*sZhovk@C@ngBEv`OpqRHn&Ky$xl0zvuKCglKht
zbW}d1ta-{@bCI=SCp+`itl4s31pioaec)gy@d$SFzN8p-KyUu_w6r?Gl^YCZYi7Ql
z>}1OJ+4s<*B_8#A*~4wt9ZRuWwb^jr*{>V2`CAlMv~_)&FXOb}>e=IUef`2~eAZvO
zXxSz!|3Ps28hO#XY_Dcib{WmR^tazhKlkz3t(jMU20T<Pv^`W|;<lE5{?lJij3?;z
zul5Kv>iu`{m(bRXImvsP!b?Nj`d@yHxwj^3UZmmf9ge3Ll@|0~`06AVls-o)wRdC2
zpDK6B1Ag2aIG-*|m}4?;3BxoA?d^v0S9hkha^7q833lC^>2dF6ytHaX*DK#(eJRKC
z-BTuC%t{G=@HKAB1ZA;J8XhJ3kEXh+2RM4a2s(Pb(6LKJz%}>XVqU#{7iM=ctPqv1
z%o2RJTI|!hF83L`xEv=ZYe;=skr3sQz!j*!@wLZO&k)JWd+n|`d9@la9j@5;vn2m*
zSFD_4=+wfUKIUN&8IRsBQ{rCsL_gT;&BcpuqN~ltl-F{7Uo}Z;)stJRB*Q`vty{Dr
zlxfo}Rl|ON2Cs{}QZx6&uRB?!x~gUSsyS!u;@NJ9Sv3ePJ|Dth_Ssx@{c4+Y%Q_C3
zC>2*G{?L|4n9QUwvp#>zwer75e+I~I*UT_8+w1s7?}Mn_zVr>p+N890&xoHq&4BBt
zNn+OJ!b~es-x;?ePEEGo@!-|LurL2&@4w#6J)yVZi~+-kc8z0;{=2&!>3zqdVeB#6
zM}F@w%l3mk5}V~kkM_zntyz$0-Rv)T=kPU&UlXSr#5sGMlZZHa=~2d_nv3gPy=&I@
zr&u2q+0lF6mp>!Hx8aLfQC%`8>#>B_vyL+dHa4(K5$*DPd`9xKS)1(Yw)4j?nBABX
zy`lNj%wI?4VjY+C+&K5?Ih)8aXV=ML8OvX%fBlfy%W|}JQxog|`#$H{UvAl@>v3Fm
zt=THM?j~u|?aRKu{S`McW!XuGD@T8C&Rg@?kx^&XwCr!ow8cDM?uxLzEFP%L7i#{Y
zw|5oqUH5fIgVaCw-d7U2y7k_-fR_O-`C1a!d)iOPuZldkXw5k<t#hvFaWj5uOu79f
zE9JG8fP}toRs5CL-B<Kk3vPHY>i;>TS|GZse4<(K)EEC29iKVdpZ8)!+m$xA8m>>$
z+UGsPC(E9>ar?%VH5wfv+QkQ_1f?#%%2M~hTd*y~u~?$^=8}j79-r2|uS|W_tNHQ5
zlFAugH)n@Pb8Cq%P!nG^L7Jf>c~_OQe4>lTx}ZP3pEenuy)N<Kj_~zRVcn1J3d*;A
zzi}T-xfk{0>4}Qzo9})RV*BQ%=&TZaIQ`!FxqV)*-b}ju!gGS9%{o63*3MNo1WJD_
z*6_LVsOO7Ax0l_7x(03eCyQJbS~q8xapb+YyzTL;l=bHx824CioGm}QRie6Hviqyj
zrup_-+hTjS-kjr}C_ag0>A4vOo%vt?8hTk9Z#8|fit){_+5@r;EHk!-7$)2mIL7kB
zHeB@4qJk~)%)0m2Y`Ctz<MXmE@26*P<d#Z@#ks0Qo?ElILgo4#mj$M^a|>;E8+<Z0
zeq(HF|3j%*WnD*|ij54f=B5kwnfk`1jy*LgvsxBpUuAxLu$F1UQT<J3_R1fRUdTET
zabdfS8$Zv#MRtx0l%6hpWGKy7oS3-dQ$XyVGy2B+S>EfO;a<XUS3ajDxZ<@&(x%sE
z-8t8Ou$UCCVr!6A@Wos9?(V}9Pb{n==O2I1wBTdjvaB!x!@{e)kqn#e>Fi&jnK;`c
zTtek!RMaoWjb#Gsd=G73*||tGcf~E|jXxUlJ``T%m72KPOEELT@o2@@%EOOey?(mN
z`1Rs<I`54wH-F53P~JG-?5Yy?9#`87p;0Cu%|#o$Y8HL?x%R_xr=6=l28VuNFn@ev
zR&T#@wODA}_ci*59%neGg;p48q}u-1&+Soaw)$3-l-$I$BJ!rkBAL?3i~ZcWm=>6E
zFOxeH@S^bQchi@v?o03gZpeCFAhu}fYPs&!r+QXzS&-ebZfdX1yf;pn7pC>}p19rY
zrn?~Jv6xBN<ShQH$M0OxIsen}qRZrp?1#<0{&UQXY;V0OywWQ1)ve%ce_-UB6|6>H
z3s=vS6t;4o|0*zG<BNl<*jMqmJ$4QKepQVjG(RL&Z+dGd&-{(vYuLBh_pwQ<xgMDH
ziJ`YjpzD>*Gp+)!lXkbBZ#^FQCO>3tLFdL3MVC!qTFtUr7||f2FumvfV%as?(|7*A
z@a5lJ{vQ%y!V6Y4-(K<UveAuvp2+aa&B;nkB9=;#oUfYyD*O=qxl?P8M|y|!?DH$l
zG!IKWeJcNUf_rYZwC2t=dwtn8KhOIi6;^B*+G!B7r|DcEN9)~rsy(|?@?w7G$;WE<
zyjN5&;=HTp&bF?`@oB)JX3+-~itYEWOxEdnSk$p<%BsE{Grz0YH3eE<3Y(zdxuJLd
ztb1F%8yx(mJ$aIB8MAYh-hbA-D-2(rEq3diWw!Cs?>TRxz9d}jm1^jHw9%I#^ykeh
z+nU&qgsO|ge{ht4t^Aey&Y`F83&hHvhiX69ay#wE-*vLw(oW{9%7u`I>9W&%>dj3Z
zf}=H_>o#tG&|qYE<K-8LtI4}!-nX2}?`Mkm<MJRvqfl*n(!<0D%VY$CLVY%#DBCh&
zy4xGa<t7>G0(10FT>R3^kk#Y!n)_7w)SyEWMz-heYj_s)o{ws~DBwF;_1$UBM+Ids
z4%WrjbnM%4tyh%KpSNrOr)#_{y@I|m75ltcFX+Aq=acz*_>pt2)&<QAH&YHq?zs6<
z@8gNO<5|Tr@>$}|5&t^Oe11%mch^oRy5VLZu-k2U+lsS6OO=ftEfVya-p`W8bn4Up
zz_SmObSE<`n7GlP<BpA9C|fwA9@CP5%59welU`k|pA&xX#HOe~!Ce!3jx7CTan&@1
zqfbvh%I^A!*qayL9BW&X+wOUP<xdN(eUDa|d0AX|CA+pSAw=9}<MOPk6??yg{m{KC
z-nMZy!-b1|nHH@npAvUV*xacwO_kVu;X>e<18%qXe*C{=b=;v}$uCYXmtHL=Q?Pnn
z>;2^Bl(Wx2v%JXiO4VBCxW#+bevQ57o~C^a+>*b;Ge9)Z-lg_h$5Q6EJ6P(N+)ie^
zpP4d$ZnsS7X@%_{tY2MD6&HEl+-bh@4uglAn%sth8Jl191aFo;JJ)7)y2NEkiAfnc
zQ}cV~q<L+;mT|uS@c-6_t9DDQaNM7DB=56Y>eae;_eJkcXH|I7z0#s@<+0BhdylMk
zn|rKw)^^^hMjstxu3X;w;e^;E)>X-Cs`|aS6<B;`BrAIb`&Xnh>|Z4IH^<l_%cSR&
zc8s%^&9ZZHd)#7GOJwfGO=6IF`u4{>!Bnp({kvzDb}!tsG|u8!UTMuj>8aA`QueyG
zMlOqIvB;}jZ{>44vF=u<%^cT|7+!|hQ&anAz5OKmwPm}V@|5+hWrw_LH=I4|EV{qo
z;dF-Zn<@t*pB`SBYT4axu~j==OMbFkPvf784T5vB9^GDXG4rLN)QwMCH$9ULO$+Yz
zMd{yIw@T*6s<o@W2rT&0^R9cBe`|b1<o%Zkfq|*t6dsG5`1*wJVES6m$4<F+VeZOr
zxA)626kWE6b@$u4;=6{=j+}}2#p+UY&YgRjq^lR+U%sqDDf6Cb&g1W9RzGE0HDm6l
zGhY3kShKFiTzu`z9P=ti_qS>_(;W3byZSa?nQ>LhLn>r{R?8KcC&`7Gem~-vgIjB+
z#eG)GjGi^ce5Kl_wH3m)X|)=4*LZHkExlOq#CKK49OHF|K5CgXnA^LhetD;xP_=u}
zq6*GEDXkq3g?@(cRQ1@a)y>ZQy(Y$WiSS$Zn!j}|p344p_s{P*KKFTyUs0UplokES
zI@iAJ$nIGuZK5a5zv7+sjL7|AzH2k~>6|}wMl9|p+w~{4S9GuXZ0x9DJb72Fpzy3r
zit>uRaSnSFqAW8e{ZV->@~0tw{^Zwcb+-yT_m~H>c-KTe&5wT8FXePapk(5OWknUs
z@`T=ePn)#tw|JygS6oNZk)kUPDr@a_g}d7JFUgyw=F5A4BmX+{<Kn9oCJk&-N4WO<
z{T-XVIF#eZ&3-SdB@wwkUaMm(rn5ahp77Ch)wK=F^8eRAbqT$-S@?s3e{0OP6^3VK
z8&;H6aIvH%&hVO}c==8BLV5Rv{{y2Xjy7Kjy(<4|W7aIzz1te+<}`;Ke6ToPmHVJq
zamai1b*>jDaqe3Ao6SJB;MEo{zh&YghV8c;tM+HTyJWm3>WzlOiS^2QtE|^;e!S<v
zQTZovoaa8TxN+M+h&A-b>f6DGpLhmc4=g=xIit^^;lrXiJL}}<Y8kD+9mu2gs(s6$
z=bTfmIJlR;`>w+<OK^1rYv>dyTehdohn}y}%(m^@S$$bNs%3@wi3?XMHnKks__FQD
z?818z(qa=XH}Be%a$s&L3)`_)i%%t$%0KMpJaN2GIyLv=7oFRy-5-|LPn&-=!#c?6
z5HHh>P3wf1xa^BdpPjODjhA5j!LcQfHE`P6o<&p6ynpyTnMJgE@<;vu_dezy*ni}}
zj|sf{AKpLv<#EQzbLs`JR^+mVrM(un6Z-MTjBD-3T3_+G{OWTK<#W3`m-{SI)$}lw
z4rJ8|Ka<45aJ3}(+K1y-seI>5<SsQWVGUPYuke@KHY4VKvB(yq<p&o}D)Zo5ufWO|
z&KrIBsZ%*apup8u4w0RQ545ry1%5mly1AEMEM}`=+&#b966fr;u5&Qe4X_Q)@vb-e
z$hq~Q*PaxqSH@H42KG1YU#TRz>%8`b7gKgGVQ5_w<$HLpbMEA+n=CcHri!K?Xa3W8
zCoW5TpLp#_qc2Ap6<)V3Vs}ehv8deVXVe|8pN|U8CSN?3_0jSAr7yRi9N^S4k_^e$
zIbkgRqOzIk#OZV1hh4uuyHaTOA^6GM6*nX!ylgMr7K^yBJDx)%WWuU=dCvOVQubzh
ze{@{Fw)*=r?R@3KT22!#&P-Ubrg<a#+NoR;n|ut9pMKThI%m0u)G_g@1uxo^oR6<)
ziTL_^#g;p44a=5>*2Y{)GEILS-rM25F0GmIp6?d1(mun_hy2eqdcTq{Ra4fBc;waY
zwIEMy#Y)z&UGl20L=GIuuI;(Ur}3h0!{*rQhFz;xKUsBA)}%jMjDff9NSs&bgl4T1
ziC68V8RxzHn0#x}ortgTuf!N^0`g*o8{X=y@W`4lpZomn3mpaZ4}&B5)$jgEU@ffk
z{wtmK_rQ&9|B}^n<Lk8-D}7pardo-;`J{Q-;()?s2dp9+ZHt|g-*TEw<1T6H-Erh!
ziuW&@nwF}cA9n4oEC0JrJorj(mnzSthf8ukX)ShAst?uH{t}sRBXoV~ny`(FpJ~09
z{Q9IKdv*Us;b$LQ7W#b3IJEZF>t*XaM5;?F7HyG=DXm|_?C?aq)4sx5s>h{r`>(B8
zm!xE>gWnm+K4P=o_qx%IA-$fjkNdgqshO?sd^4|$%Oo5Ln0&~pQ1j1%_Zdg;hor=r
zUVP9I!s^%4Bp;iQ^7H=ed7=W+276zM{@UjnyS2cS+os*=w9EfPi|Q_Xm0)MwSYi43
zhowUAywiy)-v#s5b!cddxL!T`M`BCnmIt{(8;tCe4)h3izi|?Y+-GVeW_ju4Re7D7
zMM5zT*M_E7Jm22?-R0j3AFC7>-u9mvPt`74tY;Q1iRaiQbfzq+i($cnO_M7gu6l1O
zEoY@E&#>U)tY-)O<o)J#pRZc*adWoS9X4@8skvPnj;zzZ|3J!$MQ+V;X4y0HZ<eI(
zNcvS$`PIL7{*TlLx(xDit19^OYf@eYe5*R%`aPxYklm)fv?cOu>?2+@Y00!jF|Cx1
zf4sIg-mK{%+o51rfo0ooYVgGz3q8F4&zt11dnS?U405rmY`#|y9#q|G*2UEw6#3!b
zq4zI$9`Rf-c~$+bpR+%E-@R1ec)C+UKI_pm7PX?1R`XD%6*ZfL_^Q;t@5`?-`h44~
zxnk+pjVJ$ZdA+JU-{w-0spgb(DeE={UJ9wT@_l=}Pi!NDfJK1LO9qjpMy@|sU68n7
z@_o%}1`W%gzs#+|PhMA;-*uf>X*iMfvNCtTLLce<M>RdRg!EmRqLDsHNkmiggB@=)
z7rWK9*JoWBy5w|z#kPKbaKFdBm_1WvM-Xp$Z^I0AUsW~1Qx}bNPG+>p&r<VtSa+>+
zUEf`!BNNq(ZZ9pbK0GP->I-wR7^(dslQk70>U!SY-#w2-^W}!0#m~;q(Dyd(i_=@&
zmp8>*mG3&Ibi~Gon)}-&6GR^Na;-l6x#9|I+G&$X+>2z&E}eP0lk<<!N52H;g{!qz
zJ>#s^4qSIuux4#k^xtI3@O!?-HB+9myj3>Uxjg&00keF;%|$o&`bU2GbFkv>>UF#K
zt$!|}p}5G_P*3`-ewFQehIN|-9seKceq%DD(zSQiVvmKla#r=Px6;0*oICfzjPj6`
zXSD@>aojmPmD%X4PL9;F*M+_F9=<w!Qbe`M{pH11>Sr!5&c5{C;-K$ArogoNm{R3e
z8fFt89b-_4`C9ACbm51>>>A^_a~fa2G`ywxNvbk*i;4AGlbNiGl$I`f@^*z^*A5}Z
zgkMREDi<~G37-&V`sGZ&YJb<79V!<+>s%JjubndUzT11@yIX#HhD=VmuF|RH`N&3$
zKT_z|!UBf~gBrPT#tSoUEDqSD&JeP9Wf9A|w88~$o-f6BeEJ^b*J`lwk>2gJmu0*<
zkD_mJY3Z`{ivMF;HeF+(n%CUUi4mPQU)svgh<CG_|AXOHn0m<CL#_5n-;V9Obu2ew
z@7t8cv9Ff}sV>Ocv1?tRaYEz8y%`$H6K<G15D?iWQkt7CaoJ!UFRRZZ?)AyXyo$|w
zZ+`l*M!H*WjnspT*Q@@C>zK|{(NzCwK55r3wJ`J7YmaToTNjqRCND4RmdUj{iiJy!
zk9ft38F;hRZ~QNPXG5!dzKFV5!$jxwWvgFHm8hOLdrX@37+Zr9Ln!~YHLL56pNU=Q
z;AxcZlvTH0-Rb3ayBmLIPw()4AUiv15z9Xf>Cgo?WGb_tOrN>R=UM)WdH1`!h5ghd
zzJ5RQA~M2!Whc+k3;z~)MHbE7%J(8|#d}skD+Rap4}VK+G_7@H=y@H^d_iQ(%~PVs
zTT16oVRSqAi2I&GddQ*H|C_{_j($4wV429w=>Mf>L-uZD;z{@FDfOBn68U3OJlFn|
z<vL=aK0T#gI*E&J3A!s^m{`)gS}x5m`^u>kPWRToir_8O-1tm%y^K85_e&=}_-&e@
z9(kfaQfs!gW|7Xxq`qvSh6CmsJ$<iC(#mDNaqzrs%p=al4|m^Q{P48*1P`If`-OX-
z{Jm7bxIRJ5J!xA-eL(I-rF`E#>`N32w@R+MqW|m2{cT<rQ{_8TM9f~SHmrRX;4`68
zVY1D{zujx(zxYj(?Oa~lwEJ~6W02vjUg?Qj1Uv0+C$96z3ZE>pa!J;cKXc^Vf2*!?
z{pG0rI7(yAAD<q#2%##bN1GVK>H_Yq`Qs^mPwuCFe!zCWCm-u$nfFiquJvi>q3x=6
zaTO<Pc$sR2Uq`KEVy&)!F;z<GP2G-HTMMs0=AW|Q$bOqV$t|lBm`qI6)ykJ_=Ks@X
z%`Ni8Wp{B>oX7LVKT|5b#Y_$<&P*#63%%&Q#YsJKzA*DdA>lQVa>D*W4<w$Y?9{oC
z5x3qnwqGOu?)=3JPnNM;I9&SVq|5$T?7=Gb*FWZde;pwzHOb)qBJKM=Rm(0cG^t9{
z3YTIKy8omkQ{x_Ix%VR1ACq6sD#_iEw_Lp?{=&ZSrIUF!hF*KT#8NGCBBR1<HlGb$
zH&5!!2wCKvwtk72zo%~L#b;uR=4H7o)q1D*x*%wUe^B`SB>@UwRu(Z{IN^D0)sKB=
z&Uog23KF$UmJN@yJn-)6d)9RMoFL&T-HrW?*90AwsqQoneV|_A9UkvKwZ-d|+_|p2
z&cBTTK1WJbZP@lb;=bpuxYP4r?x8Kqu8M1VToMlYy}+0?`_un}YYo9ierPj<nDPA2
z<2jeOFYn0yFaDdKBpuwZB3t78^+|A=g;t#JInB659X$FiiLU%zOoDyj<7;|Aq&n}!
zf;k0q5=8WuOJy`Bh-8Uyq;a<>_g90?wrNaw64T8hyj5dj!J=uc2}%cDnmSp8In}ZX
zg&wLL@?@FJp_W<6sV3-i=~0J``k|97lR4Fdebi?77({kY=2RE<12J0@xDJXmX-3U)
zvS=;n`Zhb^%Eui7KB^L1AKuV<7;<POOEBj$A)l)|10Sd8`YF$Gv*=BbI%v|A$zse^
zmsL26Z-LCh-z%!kwKnxiC0n()?_Fx*w}$UgT*+0>JFFipE-_i~ZT0(FmX`54p?~t)
z#kr3&ZgV^mahUt=Y{@=f_B$nqeBJMq9Pv%RQ*y#R<VD$)UzZ*2n_sa6xc}|Z6Fgur
z!T<T+4{EQE%<kLs@<g=E44sBToki#NzI-s-Z{N!ov-|eFd^4MG|I&>enF8#*ff5N$
zjloWW4vLLWH+CFi7FL>JaPX6YIg>_=fnqaAh+WuehQXm0A+;$64_l@>9yq4NrSH>`
zc!)<hYevA~7HPF78?3|ve<V5y2)E4$IMSk|CS$1Bxpia5Az|S~GYpQlXkBBQC>s)%
zQYQZ-?!+S}G4V}1CLC?m){7~8*u@<#(AuKkqNA5msMyUN9<$@oAyN691FbrC5(gi!
zuxkq(ILOoYpTErD$H4@SW@U~@unPX>of(3$=w|VMd*CG2-yF}m0B$_L*+ZCpI`i1r
zRUAZr9N8~z%+37o>81eZw6iH&J&*0*;yO85z37cg-`o=$FDGA_Ghx!&Z_{1*wVG!Y
z%t@GHwu;4Fea1$6PiGaGrMVUdEg&Z%nMgM&9`}24<K@D<IiO>bG#04}`CM7#a;P&(
z43eURd{iVPU!B}2Y+0rRQrRhEZJW~Q!7_P^MQ4G~!;~g?+FEn;WD6*5`KZlsG61J7
zT|dP+P8P1eeZkz2rj=21yey^`XdMh`TFDYzn#Zv$t5E8p$)QY^%N)xDeUxVeZjA1}
zT*B~_*HY_B->sEa&Aa&?t_xm~)${0vr;N4jxp{AyDr7PlW!j^bS6OGTd;v~awPNf7
z4zlkiitL@sxkqY|-;bH<>J9E$lNf$ESj{cI?6}?FgC9fb72ou~IkwiIRK@Y|Qm46W
z<>iUnEes+V9!I#$t-gHFeD0sa?n@WvTm4>kI6qS-N$QBBvk2RafCDW|YBB}`m8_0z
zoNQ4hbleZ@|ITFf!Z#{?bJ;-rziNy&5seIdX$1wZ5Aw0uOYkHXus{=EqJg;XBJRi-
zXwoVGC)1Ay<zh}eD*T|$D6(k>Oh^r;X2Q|S4;ol*yg$cZHjDB2IsUNMy+6z!#+%L8
z`!W6C+%IC2lhcdd`1H*^x$$#yNUu`I#RI!uI3Dc~!ct((fgHCri}&CJgeGUf)s3KH
zO4mze&WcpU86E}`LENT^+H+;BVTQ{W-8(UT0*6{AxD<mYEc}HQs0>@l61)|ZupWXF
z7U!~8cLaQ{J~BBZY1_xKOz=#`D<^~Agjr(CUovRLC0^&eQ_504_u%4ZTl^l*JJxyC
z@lN+TtIpjFa_&(K#}{5(dGB0ShSiPY?6UKZZbZshcPFP8zOk_=6FYvcfAh5lhF2Z4
z&+T|QVK)Dg%a-P{CEU#KSpwYemR$2~zkA4GpR!XmOD)F&%tGs*pz9)d@_O`e&lQ1I
zMl}(Gg$<nSJ{}2`I*UFWXyH)v*<f>`mw`*%=LA^jV2glSkHNwgPE{X^#8iO|8#@kh
z3n$GmINTzocE#Xf8>i)(K39>t83IRI6xBow7j|&E`*<Yf>MZ(jq(wu`$8cfSR>u_f
zsiHIQvhRcxSfDZrRA7Nipu#sgeRGd)+zj>{)U(eIg82Uf8Ti8#8k!|}@(MOED|Yj@
z8CWzy3Z#z)GrGktwt@^&?B<3PW#a38B=GB{e6;v6m4OQ(Bn&AKK`Ak_z=0{m%kbY_
zi9IqO^mTcs#O+X8!xR2dUpskXvUt%Or@q(|8@VmZ6p!om2TE>u;n)<|s;e$BHKnMj
zut`#_e~DsNImdC%W1Mb3FE%DLr8yfsxzQ@Kdjgu?NB9Ll1<Yt&#2|D~Mb}Shj*G?2
z1fzq{5-hvW=wS-Dq6D9M=Vaj89n7&@!0%$6u#e&lCxgYew~M8h#Hq~zorkA&u;<=<
zP?}rpt^5#NBSviz^ih`h>U-<TGuPb06C1lN$^?(=J-YF<uuR?K@sU>w?_9&@9;p?n
zmWC!WF^KH7Jt7ARx43_`kbM_e;3KoUBU!xgjZ$Ci5x5Dd1*m*ind%nBX$Rr*r{X>+
z^{v&Y)PHmXMd`Hv_qr{%J=(zVfW!C_ebQQQpGR&$<iwk`6RN=}16=;BgBCX6;BZCr
zp(Kd^U7z(s0xOfnjDv{`@^2nIxc2xzlST@-TD9SMw#c;v-7vcw&D@bTe08b~PGaH^
z^X+c5zze(zDVYfj<vy1_=v$Pr9$&}*izOp{!J9*06Y5eQGcP;-bI-?0Wd>yyWe#N?
z?W%yxSuKxQ9xoGp-kQ)>td!X?Y4H|Imd7l@9BKkS7jih-WOh$V4ljD+)fX%8CB+MF
zWCbccbkXGox49PgG<g^Vc2DL|7xYt_;bP!<+S{NrVUie#JIBWWp1`(R%q%bh*Ot&@
z6&D=^ov^q(V?QT2y?Gggb_a7V7xGh`aq+<1gjMHw4~BpmT^B=VfE!(sy!M>SG7D$%
zDLkBz9A5ZFt1tHG#?=;OqQ}=YL;3RVkJ}!5a6hoPVs-V=jZm50T~G-Oh3S=+j8`Ph
zWs5#HmRLPs)|{94U@pU}6|>LndbuH5;zYTMXLg6x$7Ra-R-eS!)wmO6x>pp}FT2FC
zpbymYI=AoThiIwp8El0Y=##7n6k$j~0rmFvgP@@K9}O+SWY)bp$j7(0s^Im<dPWgY
zdO4h+7v+Lc9yZ(Vm~iy*gK${li`nt0*%#EvgA{ek4>EEV9Dl;|zbV<g;0;^fUAZqD
z7fv~xY+ZdYMyu$JTi@Li8>KDFl#l22UyN-qSo6ZM$?#BNQ{iQ=&rSx)Jqno}n<_kd
zF5MFVpHAtxSy2K-(~}#$GP@@xn-{+M7SLCq0<Iyam`HHCb*90jP}Dzys;7bA=9Rlw
zU#(O=9`^jDTi@Ly8*i#?mtPR8UT|ITPAOZt?4j7@1^)0dEta8QS-0VF!0V9i<n70A
znmIF+ZaM$x8`FxzxmI@{-FRClW+&z#TX<Q}eD251Ay3fSWOqw$=<+<dTxl*_aT(Gm
zv;X{v)u7My%1-;usjLN0B3`|ijh^Bxk{_`K3|yV51`3ne7G=W6^B&#smD$~yy#3(D
z*%oF0_cO??abRFKwrD!VW^JMKz<Isgov8od5{mp3mwt`s-&>jAuZ2G3AKB0O|B}3e
zH1kujb*vU<tLj(f=<Bm21aeyL5oi;&^)Ogwsw!c=)<B=dnCai7HjVo&JaUs3feJE(
z84GR-@K}^Nthb7R@Er^$CM*(D74o~XNx%n^+LR8uK&r*g1fhc(FAW5Jlq8lKG$w!V
zZ-I9oXC`bC19cz4WgDpJrO6V!#iF%<>!HY7b5L&*)M?BrMCm^&&RBIcv*j|!a!?cB
z%>dks)b+c}s3P;t;r_{u%$uK7URAtP%Jm)If0T(8hb5}X$&a~Z-xO{t-I6PlJ2&_8
zL$kT>-o6o8sQ9X7cHfSd9rKTFV7<V8>E)c+e7jm3%b8Xz&a-;G>~LP9!u)f)Umlq4
zx98=F*?oJCHlAdXIIi^S&TPJYFQ0&VkMLwwp>j8?<4}Wfw?9KGvzp2V8?nHO1Sb|@
zwFtQ<ZVav5YAGP09|=wz!frDHKwZc=8;m$^M7TU`<uvq}k?6!LoHe81aEnaXaV0K$
zAC9C$0>WLOE~K)W%0@G>z=|X%5#ePs0*>BN*wlQg>&(06JMgLsS~nIx<gzUL?*y(d
z5sh=Um=lkh1$q1mHZU>DoO!_UU+&F=525_rKQ63h{qSQyL)*caEggs!)*(@FN@K4s
zQ0zW_a6L=FiAOM@*#+<_)W4ZoNI^aMhq%n`0~>ik#9PPnwtE@37zAV~9ycq>Q9i!y
zc*cDX4mSZeiLfsU2WKpi*VR0@BfGIuQR3+DT;(o@b(}Y?Iozy;In*-DSe*NEpWJZk
z%jFN9!2_x^F}sg5Rvh(g0acJbN;8%`Rh3v;rVdGF@XC=>jZo`rYQicpP~`}1eO*~(
zd}x+@+ak$?njbUdUKMwETh6+<@sTl?-TOTW`v3p_aj)#LtDX^R^4ZxfeQK$cYj)(>
zD~`&iFS?!$EZ81=I##LmRqW1HM^kQyN91n0ye#OZ{o0br5ii{~Fg5!$$}awpC~|F$
z&QpP(+dSfr^*Q|SIkV@$pFJ(>nOHU)Dw@{zsrfHQ=%xo=uYx@@e)q0<x9z=qmi=mj
zuq|B4CW|)uaB8!p$L+K5wO5PZ{j;>Or2Rm<u0qmA`Ib{k5r2c8BqeU&yzj)5=PTy(
z?&lVIpglc4bgOoefYxD2t+|)}-grAf-1_*cf_cJGVgcWtXk^uJa4h_P`$PHR`+KYX
zmvY_L7hO16RB&=_TyN;FUcIOdVKG(aXOD@#n11TUsg5Nb4mWZVLJg+%EbyM<@;?6B
z3GT`2Wfk33p=By_Ow^oA+!mUfzR29M-SAJq)0Blpi}O;RFW}v{WA4n&(^B+%t_XNu
z7HGUIk#|8L=E{rk43l?e%dWo{*eYVZ>2tSJ`xKW{x%-XMHcuu#T<}8wS1IqGjx62X
z#pfelZK{vnX*zAnLHQp#n?C#&Z>`gx!hdY`7u(C0)<?y%6hH1an#WqoTqas^GjK(k
z`#bG|eEna67p8FTdaOLVSzBOzyt}Tk%~35mcjg$j-gyhxa`Zg@tMO~==AL8c+Gbnd
z4bPdMc}3=m-{hUkq)$)Ue*9uX^xf-$HhK9{R$I&BzScexZQaw$e(_toR9kwM>YFbf
zh4XlK^hR}ARpk3j7Tl{;n;==2zP9h8g|zSEo~8QL&Ci_PvTbMC@Wn;5c+0}H{Vxt(
z;?Ag$ylEip9?9Nd=Cf_S&l=vgdxw=*hZVYH<(|n<x;A&ymw>}DUr&e5zH>t1;j#OD
zMmIXcg0?WFf4OuscG^9wxYg@?*ql6dPZ-#!xhDThT5y&9vQ|mwCh^<ny43jg%H5LA
zy!~v6m)<IqI$fn8FC(X7iO%r9QC|(Z9&FyjX5ug}$>cNRM#I$AGlcSdBxPQ_X+7tn
z<Fi;|VI8l{FSgzd7k^fj)MuHz&0Ax2-YFv^*3qqf*009Whs+yWY}IR~w7u4`DVToM
z*Re5dqNlG`W$lKr$D&U!uRisjb=S#&yR%oHco?#8f94L84oBHupTv71Qa4I?qkhlb
z7~NPcu;p9zgwn{ZKFs2GBVB`^TGfiMY@O)6G-~FmM=!Nl_RY;pwe#RSSaP#4P;y4x
zD>bF34WWiVb5-ZBU$dY?X-#DFnN{&8D$31dFCU(AWb?7<k#E_a8giPN_<CJ6w0&h}
zw{7LGYK8VR-#E4OC4UX4d{eoxX_tM(%u7GqRarznbh>_R>=s+KqDN<ueW1@I@o7v?
z1>CnD@w>gaFDPlloR!ZKliu!e+5B&sq*Hjw!|hYQhb-F=y2(0>*Euv*ValnetE?(y
zKC#RGT45m)YM>Fc%lpjh^rPCz`>eD-sZO2it>xKbCaKw7WHd==>&#1z91+Xi_1f(B
z8JvJzm{7;?=ReDWuV?!8JPs<C{7kq0AEMZ%=P@_FtoCcs^UL-E-kO1d1wSVp{p%Px
zZPK*MIzJK`gPd3H=kkwUf3xp)WNDkJyV5@QTE%7Cj5k_7eBE+eQ~LSsk9W?!do2BX
zw$Z&3{=KuWPcXX3T=M?^?YO@miVy#PDcxXy$a;wfGym=<HUIWbed#>+U*6;VlD?Vq
z4KAy1E4qGq-;?;i7vsC*m^MfpjALCqu}<&TPWiWnN!iDz*B?6c_x_H9{q{Q&m5wY}
zySJ?TdhRp%M<1DmUVVIWW9wP=%EX9=%yIud2V4*|^7Gj6Q+>WYi>>UV2AN~0|FxL@
z-uZC;u7@|SG4$OxJ#*oF!0Y47W-~b$#U=l-x193t)W2mG`#RgWgU!BN{dim<<)g)-
z!T^5}#p~-AB>!!%n6UB2vHr*XSN=09sY#bF{kV+fki>3dgO9OyuS{L=+{!7o`hLQN
zbN6IQ6A})E$aQ!u+<mj^_cg=qJWp=_=B?bL`Q1D)=0tMXeZAX->3&s-b|)O`baVea
zzWx2LL*a~wN$SbILRNyJYs5DH-<I@?v5n(+&xET^5>IcunKW;|)&7^C`FC{2ta)!c
z$E$<w#>1FLYp05Txw9_$^}~IQEHTHcUw^!3buZ{QZ^E49iQ4`V+Zl38_dN8E__@E{
z`St$;v*!o=@Jip>JNfqQycg#p=VWfSxh&SW?5vP)w2Rcd-oq<unr^>nmNYM8Kk-LX
zXa1u<+umKik*WWcf7$Qi!}Vp(<<f8a7e9R1mwa*eMA07|4rebo7Qa8f=tMX}ptvDd
z&h&)};>+dRQv`Is{o+o@ShnYPX`R$u^&T~oU9L~A6?}R6T3Msy`-2FjP0ec0|DJgN
z#L#Q1o?Ay=heFqf;5X?7>K<0R9_^gLZd9wVJJiKZ;<QfBueCl8dH&RK+<qm*f96Ky
zlrsuvN`D*f^8O|mGKI@1Yi>h_p0+|-jx^_z3B?n6dy@ql&ipJ9>bw_IBxF{l!m+qu
z`3JRN>l3ofXJ3Ty9uknx$=Sa8qI&dp1F=H(uk4eX&AD`{Tbvs**}n-Z-j}nTz2UY;
z@L{eir_7}OzMjy3O#el|OvzX09+>9v{V=a-H%NZ}A&u90>&8XBjm_oNOfma^e_X0M
zN$qFbfzuCHOe$rO?rtfTJJjQr@ZiOUoyv9$Eibe0GM(O>bu2@{k|X7@$%_V`FD!|=
z!lzFLG<k4wH`K7r=a`Yc{fhXrRqqSr79Y1h7Vuz?>;JW}d)oz;ZvV^^`s}cw!=b|8
z=j8Tiyy-Gzsk(B)fw?P6QewsG(>yaW?45bnEw}49a?gHhvcZ`cuCyl)*R8pq=@4RY
z^#s#ewqvIvw3_`JCuTQtmRkS5_suz~lkbajhx5h6s~WBfg;x(OsS0mw3)(2r`s-8F
z<d0paclXPEV2isN;Fe^(VO7&U;e?Q5w&n&}7Bjd$PH#N6Vfn<sw<{(%nN2;}TPDQP
z5*GApadvU>h81gyPAuq}xzlM$Cr{?xPM!+a%RSW*q8v(><jO3z{?ffWfh}vs6{Vw|
z0<n78^O!Owxve}VaQ|RJbK9K;rlhV|Wz#!T_j6@MRK9RMY5nv;pVZ-%uAJ@lh8<^C
zzSC~~dv@uuRSFmNuHQYp^mLzzb&wxh)0Xxlv2EWz-muD8aBH{a*T@fy{E0Sq>pPk~
zEHxCE{2d<hJH1w@pL^`!dQQnH5_hI|Xg2d~aQxI5H(Oxl0)>WmzkAzVk9vLyn85n3
z`j}FxMTz2@FBi-%?KxnR8?bj7`-evde#E@eTJ?`(Rd|R_cgp&mkB(bpKH+S7)7$)Z
zK@H=ty^V%OdJfrNO05q3{-wCo>RZB-hkXg05py;eSwDJps#*46701-Nx4BVri%+ZF
z+bHcXyY|pr35%Tl3${CErMuOfZeHtsxjFX|my?#E(u9rdUoQKwTba5Ze$7<7*TAJ&
zYGG<td+U=0|I*$40_v5IDe*ixSmAag^vm=`?-#zFdN;O<b5m9TtB9DIySC1SlV|uQ
z&L~{2)M5O#MElUHqzCJ54OXwOuTM@doT0fR{6O@>S#P}=UPZr{P`BV&je5-j?l0Gk
z-V1Hi>S;*Yez3jjkoz9NJv{51qa~k}Zuz`y)){q49VQ>w3h{4Nk8&eUtkdWZp15OO
ziKG37#<{B=tmci~cx2bF*+uaOT69@<z72j=HOn}x-DHKa<)x$V&MlL={nKar<M%n<
z*Eti?ChOc{G72|PNLuC};l&hfYIi}%?C{zNGp}4->al74v3q3#hpmEAbLJcWS={ht
z<IhOLx!)OBuQV*Sy|&h1Ntdga_Z9uQT2qevObv;0s950m)8uHl*a9}=S-WQM>z~rt
z)ie9<%DP3D<rZ<c25H4{dP#rHPV_b@m?{@P-PEy-p((Ir8J}42g7!y0zOikwSkf0H
z$UE8PesY`1#H5Y2N@iObjjh(S^}hK~kfoAT!g_ASLi07*$CXNy1*R#>bxaYilD@CI
z)z^sW@Rcs7%Vlda9gNvBa(Z?@$+vKl<haeu^(#t2Z^^6Nok8vjiB2ry&!UA7+Zi?}
zSxP)%U(;N>W2PU|QKojSjnf&DHm``3`L}94t5XZFhagK?+s%fG(ik?&SLsU)__~zB
z7f(y@+MHZ5**2u|yu%cQ)UrM7Az}h0uegpe_$?0lkf5W{A|TUZ;`1}s;i2&ozphD&
zwOJO|-F~Tva<ga|)Cuq~Dal$LcNe;GVwW%v%h9d|pUCO$H;#y_=osr=KhpFeOW@zd
z-!Bcl+#LN4IBvfvzPmAbi@w|P?PruGbg;f}Zky#$Sn#~Y>cSd{l&vy%CEc6vp1&fp
zW9?$wRVjSRjK!rlzHIaf5$O>OpXi<-V$*!;q}opFtxW5Z0vBl9e4kP9)xv5|(}u6>
zjlDx8h11Fw?>iv-K1GM0=ZsE6Vvn5JB#TFSQ&<Hyt~p@C5j?X)?;o!a`;wIZI%?B<
z^iEl<KC|6at!tx*s&$P~gqcl=SHxp4<GYbBxi%lX`EI@YCF?CLlhl+`4}8`M%@nZB
znml{EckDzhrfJ3-SFo*CQK;!>(O3L<?}h8JLr$&=g$AxtlYdIRw2_#5snKUw-oh>i
z!Kqix**-S?@&DVjRL@mHSd`^((wAgbtrJh8A__KsYuvhp$D7HNZ;Qdk#kMOHm6*$4
z83%ZZvhts_2(~vkHixOPl&9<k`<)wlvQ0{9a}r;$pYUcVUe<6)yhPl6X$D_sztbDN
zZL@Y53!FC)Oo%o&Z{X3oB7TZdC{amST~cyw$1Xnx;p%j)9)lojhhNN%jY7`H%-8*T
z@sE4a+$Ay+z8W6oWtZLss?Ija|FW4;`|K6|p2FQrIu9HQH0?dUd3KS<f3DY?9NZry
z^{laCouIX2%9Q-&jG0f?Z_w8Y(UmhztDPAVrV(YW-E||<S5|RC|HW;xjS~bon6rC2
zwyv7BJV|!Ju?C*7DL)rnnRa8N#wV6duELy(t6U8}e1GU%@-!;th^NTg(v$^?UpTY)
zj@_|{a4MgdcBEM@y3rt1o@HL@-i<teucRDn>2T`sY|^`Qd(DJfeOLcF|6jO1;J>x=
zj^?$h%+|XYZg~1Ps6Djtn;@q6!bIe6+kw5#rx*Do252bg8%=!jIzOv+!n(4TXZT&p
z1Lil(Ie+hjkEDjj`KFY6{2cdwEU<fcWcINYH67;##O00h0z}HB4sCK=TfnwcxtaSC
zt5BDmvYiD(`-!jx!s-Ur9!L2q*C(?r$#YRmUgfrQ-~R1E;hZH0H}zh37Q8j#QbgUl
z*TQ~UnHe)~g)3}Ukn*#QZe`&7#m4G0&0FllIa!`f(O=~?-U%9BTp#Yz9K<LV`=BKG
zib3}>!;f2@+g!NPeq`=B#)aw|mAbEFwVZtCaB1Ikk3)@d4^`GPGQXX(Wn<aTW;Z`K
zCRVMvjJJ<A>GnjwXqQw^Oh0WfY5BUC9}8m+ygqy{@yQGyuIY@MUyFCWymxWw4aMs<
zg^PPWAKsBOgHvj9zS)A1<v#HY8q1Sc1<aQ^wJ3c}RvJ&)F_o(ywyd6Q(Pk~(@AxcD
zNz|=p-UY|wLbHTQzc+QN{W#F^f`8Z6m{)3xN`H5r$niOr693e=R61ZgYsk5&mcAAa
zD>AQVYP95k)2?0+P<>r*$F|PgfDO)PjG4?_!ou6X@II)}UbEhM`TI9%%cdBdxte*U
zweI1RTjgfn3*T?$R(^2#fkg%5J2_!qgS!dB+NPSTSf}oJ?zr3N3(u;HXSOTmCo+`O
z7q0lC?UEJgTh-=#rE$*WuVN{?7+lLwIWEcSEWCNaB*scyLx{cUjL@xRGo6gqlm(eo
z2r0U|=lO>$V?WcdRO6R!%1jH#1qwMSzmyl0S??9p`_DY-X}_vMZI_OAn+n&8?L}@D
ze=n(VxmNP9XuW2XF}@R2GV6DY!;&jExNosexaN7b(8?(H?_op7+Dpd1HjPWJ&P?<F
zuWj^pWt@gN)3g&lYY%OdoS7-)C%AKSSb(<5PO)b;1-9H*yY?-MY?HqA*<nLWlF2vC
z6CW8Kx=VKct@sw&wd7Qi?S?xi-!`~Qw)%G+TTopkr4n&Ls;Q=y?~=!+C5;oaQg5C8
z?Vb3co$H{pb9r-rh?DZ$M~3H><u+MuWp5Re$`qd{*!lO&o|PuIoE|KF_n=0(r(@Fv
zrHk`7iOQ{DPCm2JAt6!WR%6pskLyg^cItP$2>*U(F&jgK`4Ub6!<4(L1$iqtOFtCy
zEU<2<Fi+F?^}$Ot@NI14f*CPW-ZiW}rB!u*rFU4dW$NP%Sq|Hucor&%l{GT3@T&P<
z+N{PFbyR;>0`J;HA#R6jkA5UHxtVUC&!x%Bm$)+crWkwWvgm`EcbeXc8VQTA6&&!3
zb-4LrzuQH|6%`Zt98*f)T8pyDtn_Jb3eH=s;A?HULTZv|c^0>yH`9s_2eLv`7hHHN
zboJK+=Ou?8m{|UHysVMccXQGrLnqd!Tq*}<`$_$~rrG|zagSx-b3wz!QBj^<4l0hP
zmrSx=x^}LKh*q<3cW4&}llUv6z(10`4(;=#R~lLT<*3l=?vP{c{++y7C*gjxS8JnB
z?v&UAFT<E@Pv|YNx%6dU=Zdxu5~|Y+7Fc#qaB*0vwPViY0FgOu!s>y|{!<G@6-rx$
zEH5etxE8$X|DtD+<1TyI;rhOR8G(V9EamR8wmUL?5k0bY!*?H<Gc3QEn63*dg==v+
z965AEK|L}>ZOZBV$6e?7X4tNfiFM?;Fkz$kEViWv7Q4RP+n>If{cqmiegDGaqh7GZ
z?ygmMzdiVt-0rJyxkUqSG2GYO{O?QL8Vi}b#@ps8@fGfyeN0|ufm^=yJ^3dtudEy=
z%;wh@?yqZcbWYS;Dv{Ruz3i-L{_9WjEmzt9uHC?PDcNt2VUtUtp_i70K!w<;xozkF
z>^M@+y1eX)rStI}|H2ZykFL1DzwshFvu>kZor+}Jag8#`4P77RmFGIFU)ZhMV|COq
zzo*2UVe;>J-=Bv}T9q!5w0_qQxd(*>B8faI|3qIg$T|sL{BcI;m~%!F-(sf)Lbpm8
zFHNaCQT6HWi+931C(r%uzDD=1m7ug|xAP(I#K(_)jJH-?XIvG)!E}`OhjdEcFDnt{
zFYArlSgsly^zO)Jt5PXlC7j^je_{!<+``brhL*Sm(kz~xlGd|c`n-6**7bOM-bb(d
z%wpC`VXZYv&X-aSIl1rY(YjO1yemoI#XhZ=gsgu@j}_{iDQB29b%6)p3f7oC4UFum
zhn<W$yp&t>6jBmhTlXF9V3NpwqoKfA@_JL)+x5K`%~yA-3jV4Puy?-Q=HPsH-U9gt
z>)NU=Sky_HE@ajX)-XQ#C!+1DsowYN2c-HoUX*S$)hk$2I{j($R@Plj0uo1?z8!B|
zv})m9=4~6Cp0(baEa~nRU4LLgQ_T+FEsB8#{8LJ`Lt7U7ataY*E4m~r_+q}nDo%&Q
zWtHzbOB$aUX{<X{#^0}T+2!euU!AX|oC27>o@4wb+p?!$aE2C3gWtjI=#?rAyLW}2
zwWy7H;ILMTrEJ-u6V-uo%Aeltc3Ay*<!8kR5w<0EaWlAkzvy0G+}@-jFmKMun>H;g
z_B?z%qhNN&x(_}b&kQtexR**tdAwSpaA@VEIIFF99d|muTd<<^Qm#sHNGH?Q86BUO
zD&H!I(CA$-i$S)0=B7G@?u`q+_jN4u%i}ebVSfGg)E<vj;)#2lKAFtf%xT5ysjO0P
z!EU+m;{AKV#6&!67HVF#<hg$)O{K0hg<*1&_>Xk=2)2&+&@JIgqD@P<KTW^5_Kw59
zUA1>+F6aK$qRjrQ{i(+V1E$p74S%8&ER`HTysLYiwxE9T6Ah1qyprcHo-(j1-MGql
z<BB7j?WPazHhh)R><dqCb2Il_pj>Et!Rh(GMiH%Py4mk)H#_oL3*UVhdVnEnX_C(?
z{#cnWjaK2E%@)d0nuRAO@+fK;cC#3+e8ZxVBz`U6!|s;}ZmQE1C+RvXayhRznQ-**
z!iHe?gA2S08s(W%3^_}inFJg~3k5`TT(`WMrFJ4|A`izQj;D#n0>>O&R~H^sh?dYu
zo|xl)=|{juh6!rkA?h36cX;34D9Ll*T)<srS`OPQi3i~grUHkO7d{txzRKz3f<jKu
z+e><O&gPUU5u9?tQKUblI;xtvX{L&+<4e}qH51m!rU)K%ykPWeTUds$TGz$a$O|rP
zKPD)Iz1q$<X~xAr1;rDND2R$~o|am*rt09l??2s5gfnN!a$QZ@uYB9(@!eRPG|ia|
zA$}`zy{gW1F@}6vp~OG&s^eV+pZs^tKUYoJ!#9z6|B2(utn1kC+DdazoH>P6t~g}L
zVMF`U?v1({t&XnJN5nQIvFda4v=?45TP~R5d}HCXNmFv$Ii3f;=1rYs6njj5cJr2v
z>r_3W6JKjjGVAyq_@Z^U;qwzOg!@x8V(q(gW$LEhYRH%$n0Ix{u?w2Nk`8bR$|(n&
zcD&9}ob2VEy3Bn^2jh3~w67ge`*`_FqHesbog#4MeHV{``=X2mtWyuo`s&ORU)8cT
zDu|)3L!fAayZB3X#}YlBpMs2nHPU;ohH!bbB)$-+e7F8g$mCr&m=~`8AC{To<Q1E;
zc2=Kyg{CZD#p<4pbF4;h)bG~S+FS8_WAyzaWK~tKn)0%vWP?Vuf(f^!v~tWAktl&Z
zA}gPk`0#hVGrU!$WK;Qfh4E%H7u(}g3|_^=t~r{aeQVO>mga5W!-L-6V7ixa;Zaot
zBXg<JH$zPZt{|mFiptZ>6PGR$su9*Y;beE#V6Xd%<##P_ac7^n$hq47*_?yf%H112
z7Y3Ys_rh=4(G$JvG(}D~sCKLCEXvh7nN+T;qOvGui+}5?e2*wWZ?`KF-xs`|u#$7p
z9=_WB`#dVl&I$B}O7iY|`k`X}tKw(pd(!*d6>bQ4U21d}aD4VEY)f$eoxL$Hn^pxl
zh_LbInqAl&QgLoKU&sc9#3l2RJm+jQIiTCYo59ZhckRJL4Ku&?>RFUz_=Il|zi=+#
z3@__`hYzOjvVOh(@#;_8zCxL4Ym<1Cg|t3Nlx<;UU19NAc5c;`yUc<@TPJa-{L0*5
zYWpGc`3!5moV{x#E7Dp_gj1*FSyUhReoXY;I*vJe+&<};v>uXhPl#o%E1h*^^~!_G
zl{hV&-ZeO0=-9${Q{F^jp}$Us!s0)_v=y(IXx&t5o~y%Xyzq!fpV-n9Eh!z_WS%Su
zdjDXL;2B=4Fv)}j<zl(G>`6v190czlQ8v@cG=8?<C|uTGoVU++uj940519(Noy)`~
znf>uOb?qzD1n<rZJ-69PlP4xh>32Rnv*bW#@`F2R3dI^L#U8Be*pkhjbwu#K=k2A(
z`*bflymku~uTXvXjniZ@pLUu+)Z&KZ3zf!abWR0{@8D|9vx%xJFpry;8OC(-u$`--
zRA;_S+uE}ZUuJX&Zdp2|v7&B+(L`B&xfKdBq8po|oMcS*iU!Itu+<n{Wy)$<WEIKZ
zv6215zsYYN1}8`=6uey4x6`J@;2qDmO_yg%%5z0~S+&0S@4U%dFG-<c;>3vy>W^{F
zW)nKGqGXD4pGkx24m~9a?Fj!R`h8wUx5Qb#p6r}yG3kb}Ca>xiv%V>}m?kb%b+g;3
zGVgL_qpF)6t8g99rN?Os#f^pzt~q*_DjgSHNo5qWVzX$v#L8vb`ekcA>*=5k*J8JB
zbo#I^WZ&Jy?gy34UI%SodTG6oPBIJlCSf2M@h4$dU+Ba+Oj(Zq#j@HRA0@up?+~H9
zNO29D$U0|t%O@Jk4fl#VHBDnbwB2aM*2W`$oh#$b^(`JcyLJ|B3CMA~l92Laj*oMP
z;{q+w6^kF-xFmka)$5D>iI(UEOOCiFJUeI?XY)Vw_%lbYhREo(JnTN^a!J2t`uy&Q
z5Df})Gq65-=z!TKg#{DdWWG>}e<Non7<g@hb76Xy=BsK6Nr4tUo?8)Lb)IeHThk+%
zd1FC*i_)1F*8~DDP08{WDRkt`@S5@Jo<{4uG|r1*T6z=n-s#mCNv2HEQ`?>}wcue;
z<Etitdl@<+vZ0$9u58|%prJQ$S~Y*n&o-&N8pD&To2xGM9eQzUg+!n_=h`bwk2f4F
znVvPJqUY{{2N5?a?9>F)R*6iJo0(JbrgqKoc&Eg)Adje2xrmf`>(x1VI%jczx7swN
zkEu=akL?U&r56I}$shiRhfQJq+J5O=h3tjv?4S2X2rM~W{wq1=baSWg&I62+FQ(m;
zdi?!`pyi^Ozh5)`UiamK#Jt1rgdI!meq5C2P-xmTD}iwdKfiOcw@R;p((Cu#J`vgc
zq0ufb_nAwhd*uZAKL0s<T%NH$Ay)aVs=x-}H5`4L=gKWUU0$!h%H+E)v*YWF>b0W6
zwW}I6Z>-wq;T@<hyVuCy$#LGlRVFnpjO&kd6=tq9Pveu7w0arXZfa(D?$SkfA?LSO
z-5&c%wEbe~jAPYM?U=sujFZEWhi!+PJr=%NpS&wQTxXXW!(x}D65sDLL*A4z-uB`0
z@~sdr(4P6NVJWA?vk$g2eI2>yKl-7Zd3_hV?$?6~{d$I>yorn}Kj}0ze7h)eY37-$
z$6{+OI_{pWHE67KF-S6GEN<46oW$%e@|Iy{J6BccCC9r)*LUq-ygHdfv(#jVPY1Kk
zvY%z$o(~22Ud&O=d2okIsiM{Jsl(KdSAY6i=1HBH9GU&A@r1reuhRwo!@UJJn5&bM
z{`KuT=Ch&uf``EW>qoEOx-XV-|BlwVPvW-T1rrKHUmWr<xpw8-gr$4Fe+ytsGS)Tr
zFbY2wC~JAK{=O;CoRnpnMNNzwgJ<xamQkP6&)amsU1w(^Yah$6+Mg+xm#;N$|9N~v
zP1ot=TkK{`mp}b!rQZBopBCsIHjD<{cfJ91-}$9anUcLzzaC7r*rwB`zxZQmVU1*;
ze)r?k88xvo@okS!Kl))L6)%o<-MLP;@U)99rG<x#=4?Q}?mR|ZUGH$`=|@LWO$uX?
zuRFhFG-r#BUzFSAYezj#7hKw8Q7P#gC$6r4_~hwFPf|@PW9RJQQP=ODJpJO6(!x_l
zb2jPtMY;uFJF0oQ;M69IR7u}R@#Q*)HBUdfl4?>KJ7*Wqa=q^0=@+k*7QQl?vrES>
z)-Cw@(ah5Yw>DYiO8Q2NFV{VsdHT_oRFmA;Ioo*B6;JFFNQvj%q|b6wyy*?M<F}~_
zznv!3;uNkEc(Q*{&_8zb9f4&Z_A@?y%A1#R)m?DkXM6VFH{JXGZu-Lh{p59z2K%g!
z|Lk`kRAt}rrSW$cYq!D!5R!S&E;E7o^lEO6x@e6fN^C!r*lG%XDQFxizo=6G#OS}f
zxxv<`(xVfaKivpiB)n8f?zi(%kFsY@Ic~w9lIvHTJLcSxyDDe1@y2TB2&HU$r3*n9
z>rdQI6#3`peDv5WiNA_7o(H~b+FB=I!B#8rj$ML1@Gw)Gp2{AFyADFOT_G;#n9uQQ
zr2G{WS>W)H)52=P%!3!MF41^DSM`I#f<PnoqeT;fCd>%CJl}DVUe<#KmDH)0fvHvz
zW@aqq=~qjf=C;r6T)iTsnafneXY;~s>{=Il_Bm+XZhNG>_^RJ`!G%jWva%PHNt-RR
zXZkDlH8DV(?e=YvS(`Sy-CXf~)8*hZuiAC&%D&YmHr)7^kZU+qB>Cs6xrxhGoIB(d
zx@@xnm+~9_sf!;K9;m$G_EuAda~|i_h@*$sEIp*9Wg#xz?Qt{m=jU}skLBB)d)zYL
zh+MkIoD|RPW^>*=UsZno#CJbfUTi;m_Ve3_f4>W3#J|VOu<y}4m=YMOXX+ibb05bp
zx7f++BEz>vHJYEzn9I_j)RCHUlY7CbIR_1dXVjeCl)gJ^-Hb^W-Djz<lS&f}*!frB
ztkj#?mS%VFm^6ufnYK*P<m}2!zuYbT7jIv6V>`I8@#c{n?ajMk2464Y5&o%B6A|DW
zaJED=KsccL{`rsPayRSWvB^mOmv%6UUCE_&zQ;7QwntZc&7y}vGcK&{bFi4wS7g9*
z`k3p>>Q)A>`T8|=TLhG5co|Gh&^ox_khen&vmR(6l#q||3^#+`1hc90vRXiU9C)Im
zF9sjTu8|4#n~`gg7m_IOc(u>lzY)5}4^FHQRY;ZG7{s|xK!J712DQe+3};p|$o}VN
zQgF9P;wTVZut1QNk%viw)nFEbi(`*el*oa_dP<US?_2BEAL7-mIpAIHn)`m=r=T5s
zK03$LIj~KYTBgr0Ahmc)^zS)87aeQNJC?SvTxfr_b^3<R{QRzee|`U7`t|=3@4A)$
z-xdDxHcxTuTM)+5QWJ9{ZcVU%O~`4pwkEM>f#Ta12VJ_k-@@sao%HDk3agg9`plmG
zu|M|v$NBHo4R8G6T>gK{!F_ue51pUzVeW4y_BWUREhy+&?Qp>)_zK6b=-DSv?R&e2
zvr=$fPnV(DZ@1n;K`;3y+k02<<F_i2`}2G4rh@eeF>&wpO&a#^PwlD6s~6<*<~N!4
zqcTlvkK-K;n?mK44e#VXH0?KczS4VWV%F=`0v<25jJ`@=wYd`VE8Oh+ZRwbsk^i5T
z{J$x^V$H5s;k&1VP49Xs@35q*`c#jZR8w00uR4!Smo(1Ft6Y5c;mh9%bC*`nX!-ZJ
z?bn}Yfzl~c-mm`pvhlh3m5&FEQ_ed6I{Nc~#)dib6}@Ln<0`VdaE#gC{>qlRyyf%0
z#ch~8>5KckW&7)yqc0deem70&->20DYZyKTcS=0x_1_~d?&me*2-oGQXE)Tp_c05;
zc(%T+^vq@3<$v4bnzJU}3(<Oh-oWRdyLp5~KXdo=fBXepAy=cH?5W6`bG-i58jh_k
znU^wNtDNz8HQ|EdM}cpWF{LYfW@ITRMz;F;G_AO4Fq6As{>fV|g?Hm!WCG3!zxuyl
z)$reG^9c*zCp}SG;W6vnCZnedvaguFTovV$Fz@%oc^>Ds^1uG9cQL`>*TEa?`{n-B
zY<PUor2N?S58S*Pxe`PVXf=c~tY)~T!VoI3x-EkB!wOBU249BR4BHeKd<AB=-C)_k
znIL??tznDb`%QlD4_oPOHF~qj&)e~^>FcPH(7xw(Uj4HaX1!{j$uJ8HS2;~vt!k^J
z_@lM$1(O9=-%f^IASfoN<vhVje2Fyc%i_OMECw74)=akRTC(_0dcqZ}^-K~h1{?_j
z2Q(T2yMMg%5mzdEpUC><`SD#=d!Cq2yzLS<**8wu()Wq+#N#e}!au#A=uO<7GI_g8
zpU_UvC#n-CyVwcs^mw9p6KU0<>I^SXzN_?qVl(l*%RFIAKZHq>uOE40YG~=}<0m}%
z@)5V`GixmxpO_j}`uOk(Pd<F)iGiUdgqgC~_0AR9V<`gWk$t{xg_9HS$Sv-%;eCAU
zki;J+=MqK`EMnwh5@{2EyI~S{fWQI;2gimcH!&6i9{nhhBN{6L85T3RG4_Bm?KDOQ
zfyHezI6fSDyu{%2vA~l_ua5XJZ%$KCUa2qNkytb7{mnf09a4f5dTZpT@~X$`2<_-8
zoOdE>8o$Bii!l+a9zKwtV;Out;$mnXqlz_4?z%OeAKfI_&(3(1<S^lKZ#=I6E92GX
zl*=+-8T)5%oL$)aiRt*OohlPu`NB_l=^QKQnR?CJI-#J&xzlIzp{2JJpFcm;bE9KM
zr0&z7ho1lcHL3r%L-g;LFJ{QxyssWBf3xo<zp=KL-upYwrRhq=I<K9VT<u+A+GXaS
z|3GO;X=<HsPRH{?N3Il|{S2K(tju~@HrIHB4sY7+%D#5l<6Mhur`snIqV7y8o0{1n
z!#h9ToOjCR3Ex+msyQ)ibUT-_GP$QZZ0F_A3JxZn$FlSvr`2@MUGcn6#Y$=Nl4*vT
zAGTS{NLO1ckuH69k(zYQBQ+(CS(V#uUOU`xTy^zYa%88%<g2<m<^@?k0sHRW^%7xS
z`hIrIYs;E_k=cQ!xe6Ce-uMK0ZnNv~@x66I<5Zv+SK%Cif)##gWqImx@{;~%wi{|s
zuc%532x8Y*Uby4_ZNH?(TU(+sCLKI7No9o%*S88@(OAKay_+k7Tu-$-CwsGtT87&T
z9d3N-zk9;X3A~*$|62Z?Q1Nl@Sap2eLEe(?{$6hHlf_c3Pj*{WxQo6|P&6(JzcKx7
z_1YJ$lP_#gt65}k^DnseM*lfs-FUHXcmKby8#tUA7M*!D*U!+y?4ySK>cdyQwYuqD
zz7<*YT*A{&>)_^iwnDvEe(hGAm2Gp{qC7?2?Wa#Rdv|8Om{<X4;ByPl+zZxH{Yoai
zM|>j1e(|rF%NnX|I7#Vly5ik++upxUvfCLx%g;A*Wk*)l{2kIyn|DSEiZ_Zpo%?OG
z_w<jQUsB`_2diu-p1W}kb7C)_Z`j;zi85B@;#yZfF7Q}?v(d{V+$!K+&Mr&8{Du3c
zeA&}`mseZ;gv-Q>uih=b%v&}$&C8*)ZEjOsPF&za4uP+~ryME$pCh_Oi`%k!@1z4y
zZ#Uofh`o`aIHw@cO5Jm|%;~2w<r<SdJ4(gI_eqNFyZY9qaGi+Tip9;k4^Q7XC;8%8
z5Z@%emmF6&&5?foZhQOtO*%7`^roCm-*%n<;Dm$67V=&G*t|t0E6#sX;k>H~?^3@y
zZS0p;bax1-KBW2h(}VYZYad6HY&j{|``&u-yJZJzO08vG%A{Ry6gu~u4lFX1-n3Qy
zKEtXx>XIC6T&x>Sr0=xFR4#5fz_tI#&XPSRuQst%)g}GBF}bbhoctr+`EMjx*LyZp
z%(`~fFH8Nfi&6chd%8va?|(U}8+d8nc-(Zy|BJzVk6&jRcO0?m`&hMSgKYbP=RX%s
zzq5{kWtVhS!JQU{Mb9U{QJ>3v+`=MGZTG^C$(G9uT7TX1U9~Lr??SV?es*^^9}$h%
z^TJ4Mo|#Cg`ux0ADib3jLS!<o8~^!sG(Z1=(nY-@=ZJ@wJ$k}iR_2J7D$et|<d=Fe
zR$qVp*G&@>t_jyJy_qjsef5#T@~>e*!JRWT=Pyyd9U8~HPt8#Fs`A(12TNDvy)G!W
zemD1jaHP+s#eZ4YOCM;Jf3Q1Tt9r-j%CyOn(@S<PKk`|(SLDa4DFLQhG3Sf?Cg*O;
zi-<XSy8GS_eYM7oMM_gY>s=RJweQ`fAN*Rg_}qEt@3p-CdRdIyay!oHvtRA%o0=W5
z(=IwzJ$`50)XxWd`0s9=C}`Ss-#2Zo%g$+b4237-EIn%&*6Lmo$Vyq@<LLCt&-!s;
zg}>&8x$Ifl|2}P*@kaU1gByIJFMjS9Th{y_<<x<vp6V-8cK32$+hMf9WWzJglj}o{
zg-vq|jH-3p=B1XX736)iW7~7}SdOV#^E;ia{qwV57Zz=-s-ARv!PPHoy~{2ruR0a+
zrABDwT<xZ$pK*PU{_AGPU+TPTQm`S5cmBr5JinK?%@zK>Z9;BUlgf@8TehmWJC?W~
zdAz{esAZO0{^7C{6Z6;H+PY4er+bIy%T)`bMGlr-Dvx*&G@)RVx5ZAO8qeKD#;aF6
zyIX(zqSM9FDSu;@Cfg<Fb^6Rd`)N|o<@qm;Ut3o)@q%k#pf5*hY8W5WL&3vVoJYAN
ze=XiJ=b-bP{Lh&y17)Sme|p(ONLyYwA9!6{rteoxUzhvx_tU)$Z|+P8dL(uAkHJRq
z8O6((^HnUQ-#*>N>2X`-P@j;CRsBwzz6sw74_W@-;j?_3dZbDQWApr5NqesEPbu6t
zo!PHrYMZRykrWv#zmq8kYtqzm+KlsJ{Q1wCf4*a6sy4|+%lu1NMsz|=07w0n??pnj
ztQjwoUo$qiK1um|MJOfm;LV<g52inEIr{gT&F23r>Q=m-XuJI7>x|6oeU5z^&s>Gq
z$!2f3rxGQSA2xfnjftau<$S&tPvt8n?1H!C%NJd!^19?Bwyyi)tt_^KwTsK@q<pe$
z+m>@a;goO*4w17F@3M`EH;Bl2taq4em81Xf2ah7Jg`Mp3nz(kw)(_1Zmp5|#-F#U<
z=%$PH=Peo++*4IPZ}Qv1qy4dzOD^1Sb|Kd;PCjkH7KfkLKdHpcTEq1tz@Jxl`{TcQ
z<u`V{_&LvJ@uq7st?e_~6n}WkHYmNfXnulDtX=L-l@3R}r3$yVDZM&=VY833&_{DO
zzK&f_9NXTzE}dQxzvc4g3;F&@dpzyTJKq^RJ)(a0Oly~k*u<Wn8RECr7fyQd!9k+<
ziV~|=e%PthrKW2<+a}D(``Q(8E83n_;L9?L>GOInJdv3BbJF^Xvqd?#rfs|=wB_Q<
z2UE`+5M6bB>2}Yo?VfJi6}Q_v8+pF_G|k}3qLX{4&u`>@t@6y=eyL^UN_Bg;3DaFX
zBqKu~mV~lY#x0AyaP&h<@3Gk#KmE2u9yL))QD5w|U<KpM$X)HL)(UkiDrKp?jgbm}
zcJpoFy4A4<=Pmi}xoZ-~<O&1jx4U^fCbpY!CK^@*$Hmy(?Xz3*c9&G+OZAFd){Fj_
z{I0zuojI|pb&|jH{@?ufw%!zc^1E_MCYOBQ&A$-~mTx{JU$Z}NC6DMGru)l<R~PM?
z{QtF$^tz>YPha*pwDG!x@6zrBsb6~~n-;R=FUlAF_MmL@9~OmBk)V?tI+<HCCAux-
zK|4{mAK$$B(F~=%UXNX}tUg}Jepw@D#<MTx+TO|Acg@{>WmQV@t+1b;4*Qw@IeeLM
z64R8IjNcL`pDl_u5OUUC7Bqp^?u28c${kf9w;kb`tmoc-`CPE)lMtKKz6A!yZceLL
zc9d=Y;<@1dVz+I#Th%XyS{>@2_2*+{35)xG!AV9lx0=jQk)LV8v+hv4NARo44|mSb
zG&r}`)3ol^_u`+ol_GXs=VVYVuAJP=^W@UHIF5wLAFC&Qk4;Jcl<UmqaQ>5Xx^Zm!
z<UgKqpE94!-SN5Zgo(|*$YpGAD{73Ke=eCd_w)Y?pG$WhJiPZ$UE`z8)%wR)&nR?~
zQS0usNwsdZo-Xym%1mxu^}|GtZ@p#;Mdx0&Nvo!Pu2{GAzJ{91`G-D6C0DIveGXeK
z4m{eK*M4%v&$pi@W!(AyV)L@elDf|}u9Mc%ZOS<EUO~gX)N1~_+z_2}sf!ndeqYUY
z`1*a1f}L3sCHtNfHKb?+h%HTQ*<N?+k7r?#X8%03;^aTpsgep7`zPpK^HlBGbfviD
zuYAE|59S{x^Cgy?uy548(>qBoUw8Sc|6x(_adPLk9}|~5S@o(Q<rhz2?W7fY>sy@`
z+<2ZoJB{;l%J=Dw#{V0xr~L{1dCaEO_Mn=gG1K=CuZ$nWO)c$KU-;q6@oRspcru=O
zYzcCCHBtYJUG!H`R~!3zm7di?HCp{ny3@^tmA-F_WiMmc@H_MW=HC}Ad?aUEPAco$
zxmI}j++VN1RjmpNh?%6qba(dC-pA%&);!&BaOs$Co{Y3G@2ea8bUfQmr9L*y2*0x1
z!L0MF2<M`nSAh?OJOeM~rdaf7TrFN5bLjP3>046ke`ZI!nreD2l=b*=aL&7hGxyuX
zec|g7W7=l_ws@~ihoWN9wKu9)&L7{_u&wjbO<udp8-$;V@2LOUS@~^-?dL5k{&(d#
zhabIpw)IJ4+P7D$A0K;l|3FaJH2pTW&pY|%eBqb+c6`}S6QB8E7uI)wDXMup=lu8L
zfHi9IoBZ|n$aQ?@%Io^R>1e6;pHl~Bh8wMENJ`22pizIwyj}R)hUG6pTqf|Ij+?6^
zKK)?(kE(57%v@AcuDo2B`8X^i+~={xVcX{wyH6-rF8E(-_`c<J(7p4D`_jC81T9)7
zUC-Woam9ZZr?%S~Tq_raX+NvhSv2W?*4oEbinAV?ZYeozn;CWe8PCR=ik*UjrbopM
ztyjKyzo)}-?O!E(>0{M@=KdAPlGLB{RG{9`?Szx^+4#*{+XJJOt6VdcGgHNQ_}wP_
z{AzR1ptt>%$)3M4mak$iPdazaVtTLg?_F1>sYgvYT*|Dwo~29gy7uXqzX<{ivkn#c
zoBu2-j^<@sy!nl808`k3FO%mbJZ=4u`qbv%yXEn9SKGgD>R|smMeMw$=Vk-X+Z%4V
zsjaVF{_((rf3boJ+OO;KLt<VOzVdHA=9?3Ai*4@upQaC{cmI6L8y1#h<~sL|lhMl{
zvy<P8L!PewzvkWd=<jcDtew|xdztlChQqxzas3~E-l)7|zHLY4zC>5g)&IiXBjzdE
zUi6*3tFd=V(cX#=B4&n?O@~}x8*0X=Jx$qcHL?9$)RH@|UWzLg6skl|cJvWzuYXW+
zaf6lm9ACHS^jBiSp?q5vV?5eVt~{zb&q?h=YtICQtNw9`(p{Y*bI+?CS#UVr<Mp{$
z$5tz(=N?^nwmCNAPG#JmIX|De=YN`ZW0Uon=dNu5J5*0^G4(Gw|NTda&eL<7F2DU%
zX1&s1^k-Dun&hSXk9{{-7#(5B&aBks#M9N+Y-Jf&R35ckuC**^_Vg{~9jsHs-mGrT
zxwLextz_wo=@pYQzR$k<yyh0G?aqy5RsJ>gYen|Pmf6T?c-V4!JuPzDdGBos-(NKe
zr~Hzmx334CoRj`H<woIdLCMP>q}yLx{1iCf?44Y;a#LP;(#b7%@BCZtz_7hoea51)
zq-n=5Uu(N}soeT;*}LtxUFPY|nR9ok>8}2bb}#ps{NUGEHhcZ*#k03EnP=Uonh>Mb
z_^{ewlF9V2lO3HJ`!mJF_A1Cr%<`WsD>cJA^|NPvtp6O_dy6EDw~1_EPP?|QvT)6n
zfRv*B7X_qL=N7s7L`Vk)Z+gJjCDfK^({fz<xNhq7pG$mJi7{KHWU2gLFx}vy1H=Em
zFP8F_Rhg#V2Fq_f-MS-Ej;$r7QaA2}%fB#H?$tXR8`p+fIR&ox5*zjX!u1}$lIeNZ
z{lrCgEm6&Q({-k2@+sjbogJ4eC!TAso8`X5C+?{F{1BUd=P>@_jI)v+uP^=2S;FCY
z_w?5#f9I)Pig{NY;%mgZ{L>MJ<wstn&iwRh<$-X{?P;Ehex+rfHN&3%-2Z3E=IO6*
zAFJQG=3lYaR^dwR=ZVWUEle@hi2rV|kX1Epra^t-)XG<-Po)1^egEdS_gm4FenYN@
zK2HM<{-1v0KurG1H%x-NZi!q=UcPuf*8|@|lW9{<elqBtezEY7?5>YhB6%5>;uH2w
z;;%Z^Yq3;_uZYFBXi4`)^Vs}~kGEGEb3I?i?ze|qVUy*~Pd;tw$4+t=yTyus|2FT^
zz5B_NnPSdI9_}=-4WAJh>A$p~F=Ack#&szlZ<`0()^C-+bNH9q&1uIUYTCBkj=lC|
z11Ebp>xyOX#ow33>ub;O_db7mcGljc*^#?1?6|c|x578!->nsIT`#=9KV$du??E2_
z*MHOq{q4%~OFTd0$ed%w7iv_4dRZ18&DTh=eOw$RQ0n)vXv0hWT@StgOk92Wo&Rs{
zOEt6FqwIFF+g)e%`_1;(KF;cw!sI<Y^G%X={hsu6shel(3e(&6DpzW>_NsBOpI7!Y
zvBvl?r@LhI&yZVb_O2#XFXjnW<=^k`oi(kZd#&;E?|C9R2KBpj%&lbJYJa)mJA0mJ
zYkG`TaB6YX_gJ~9hDztU{cKG>s~p$+og%Sb+{maTPAhMBO};0qms))HT<?@qe7iT+
zy^(rx^1+MaQr?gKC0-})GK?*Mw#U%k)*_TE;mU-n*Y7SRxCwS^=AN&(eN3t<G_bF3
zjdRV+Jw?ZsYcFVg`u$b%l*NUKW~*PPR(-9S5XZy5I-zpe@;hZuIN7>9LPT#qWUdUD
zKgWx|-?%elu8XN;l>0)xIjpV>yfqxRHvLu0eRPEXtkwKR|0yiLa<=aO@-}4NCOxgx
zryE;6)ws3q<?zVYYdw=X*X$6vMr+E7mCna6FN&!1+Nk~WOG{W(+lv*GA1PRdIkJ8c
zd-dSr&E+k=nUg2@J<9z2>h2WvR{Jc*HFM-xW>r3Q?ecH8JI1Lj^zy35qK7qp*<!~(
z$;{fpKlS6EiIpn7UEeqS-`&-D!F+~@gyp9y5&c#7X3eOaDu1y=g)>rxaaHU>#*!m%
zr(M3{d;Ot{SjM?ZBipIQN*SMS`^@~AskHg!5@n<NHWOP*RJU%gN$RTEwYPnri{_II
z7O(TE6S*fw)l4i^54zYlwP1B$kZ_0kwzCJexowTIcj~pf5GEb?NHYA)hM9J|l_ss)
zJmH7fi#y6)YQ9G{Oq)LI@#R_b*^1fB#5MCK^Z0p|>ztll_WnzF-(%@5ZOfO%=Nlzg
zzTcJbZHdQ!W~Ie<MgD(Wtae|kG|>L1_NLuaukx6z{hV1l@n;unnu^a<N83-Pe|ncc
z*0f9E`)IK8l$!3Wub!_zO84%udvV#*>aE+y=(fDcPI*i3-TQOh?D)RxIqFuQ*PReE
zuIkz|*)!{sierRrr&eZaaK{(l8IkdFYr=Qjit~{&yIB9E#&+e${B(b*|1R9GC-Xgc
z*pqaC*PrWLT2Q!;RoPX;Y16WT+<Z>8dirlld)>`ZvsL7V_STj1>x@r+?=5`rSoy<6
z-oIBWe=&5uXKq!KJ@vh__lPE2SXZe1ef`&mbRTd274GSzb9&n5b%yt9Pidcd8>zY4
zd;P2*f3v#}W#7%b6gTsSh2sb5+%r>hx83YzJp6Xm>)LJG|2gga^!M17PePOYZ%x0Q
zeI$BqXwV9)Mv2Z4oqn@W!NSb&f2*^k?|(S5K0<QWsZ`U%@VQ1$mwNBKy1e$=-@27)
zvRzh&%)8UoYiBGm61}#x(?9dw)a#2Hrmj3D82>RvZo#B-IiFb*%dejF;_`T<Hd8D3
zPi2PJo;jJH+&8XsnzLl@7tbH&U0ckY*7V8WGWDIy?C<>F;_QLfy#6--8fM%|(pvD?
z_l>2hs*UVwIl1*pOnfUAZ}@nZcbQPq?VU4<4zBRH5W>s!Jzek85t#~$7_RB}r-UqE
z*^%}0-x9-}B?e{nO&*oIa%}D`7CX!N_)nYmM~l1?O&>GWoT?q={cBEGtxV+MS@-F-
z>e8!bQ=9&3ACU+M?QOO@756{JJ)CQnzg0xwrn!?I+wyrGIv{$5!+%ZbJ?H&X7+(}l
zn)ON2Iq1c*u)cd!W<P$tFm1t^+`X*(bG{W$UOmS+Tfl^E<vgaIQ}45+{i}}{cKkSZ
z=4DdKS$zxN>cfT|Ul^NADmy~rotN5Y&fddmTl4Sy(ym=0zE$z>KCN8hx@*$J<&%!R
zUO1WQWv1=-HDB!**|tWfv+r&CIbG8~&*5gdW?oPx-<MmdR?NnGXPBQ;)L@>uJZsf9
zcFp?Q%7?R0S*}r<u~a6$_u5iD?a-Ca{W}xuZFaWm#+M3lv!DFLQpD_kF|BldT*=wL
zYv(Z6e%1MK<^8fpuYy9|QZsfwSB|}DI`{GCiCnW|+h<?fcB<3(!xGz5Qh6@+sddYA
zo?6VByEgWaO>AcRlgiy)7c#6;?{e3auS>QzZ+Oa_{&uI=;uGrc7i^xS8$NHF2&+xs
zr?=S*QMcDGe(~&krPkSVSGCp4)6W|8mf3pbM!k2jvHvR=^SsP&|J72SsRvfYr(Lqx
zS@w&|fB&X8E~S?=%O*_=RP=dyYxc@@Dz>`C0lI&$`%IEctKFR>a<a=z{T+Ask_Bdy
z@6GK#$n@xNgw)fwCr+iG^6`C}o}!iE{NHifVGr(I{mounTRv8-+2OQ4cmlUG!@jbG
zt-)t)-sLVjaxC)V^Mc^Xll!_7KR;V>cRtqvn`GTueyO7P5_`!$o*>VV#M0yQ4jX6k
zSEjPWo|cV{ooBS@<yHmB>Z?o%n_Q-qB*t+wUG$1^OwDhr*c{~Q?Rh3ttGL=qOGh>S
z$=(we_VNGCU8H+?%eou3|Bn~mW3iki!LW4g^1ZsR*SyG?dF`f}o69G?sn^%<dHs!N
zPu%haiL7g<&;PvG#qjtywz(?tyt*Yfd5)>1Eb7Vqbau{`%kB#~{jaxf2&`3jv1i)C
z^BT<-Z>&~H741me{^dcBQ*vFu*Yv-f&t>9X{P~^yMNm=e>es7Vmf8JOPqsN;9rCuX
z@@|}ksUM#WyI#+h`K%dS{TF2tC5xsn?Y}HtI(fdn^5UtSd~>Y3cHImvYQCi9^Rl|6
zCE$Y4>FeUDt5~N<vSzPZQ*W7H{3c+}rfJu;=bsDqTl`b8B*o*;bsi?A+U2X)RcOvT
zsW9DVSDxtdf})GPj*g4Vraqf}z3{T#{M5tsVt#Y_t{*azeO|U*J$U^ksoY5}p{=cr
zr|+js-@%o3`RS~w?cH}TzWi(~aJFctPpvhd>(2YrQmc1#IEuaU`F(MB$&%F8=aaq(
z&0K2#WAgI+X$i{;<W{LEN5zE6Pwv(3e`~>SyZ7mNv7nEKHUB<*x<T@Yt<Gl2YawE7
zOTOoB_Vn9i)c4f$(8-0dX0LS~f9UN?5ZbU@!%o`i+fF_8xr?++4%y9LmLQbs`$@xk
zS>_boSl3kT>ysO|_i%M-{Ht2IN%hjrCzDo~hy0E4)OFY`xB2X?uV1g7V%?d_Jx_Sr
z`oB&ki)VDSgk?L|M;?DMS>)8?tq%77r(ed2#jT9FsOG)ivvl?H`DKwFv%g(BJj?Bo
zkX!7bRqvk*K4UJsq_#cX$YhI*wP*MFLyLTBN==vSTypEAiN@;V_s&VO$<F&&bn@P8
z?=7V#`@4)k{#|uh&;8)CGS$cXi|W3nJdZWMefN6m!!Q2sQoeF($<ImzYK2zaQOn$S
zb+e#D(55HWjtREOPd!CuaZSHbm)5XLW$RgW#!G+Ql04EU{Al@-{lS_)U`J%<EAe~P
zbGB`nA*1?1tN46-f@)x&j`HavGeQr@$FDiWopJv1g@5y2pKz{~du^Lo#oeiN=HRL2
zkF{5xbmxCq@@kV@x>v!8yhp`_D?IH^mgH|r<qUkSkvXx>VdeXIZ6?}le@*!+(J6S9
z$LX!-FDnz(@`_nGOj~0Q&01NtTI}7)6y`FiCbz&u-Fx#hb@%zSh)tV1<7v&E>Zprc
z`&Rc%V~`Jv2t7Q%HKZ#xXkO7$iPJ{4Wyf~vufCbF)XY;pV0qk>rUflRE6;v4I(I6^
zhVNhIn&s+CelWeUa9%rS5%bjhJDZZD)q-|3EVopVG4PH%Fn!0p%9?n+#SxkZe9~9x
zM%xKqU!Lgq@W76ZbE1QHw((5kezWeRI=A`Sq6gC6s<F9Orp~_Q_w{Ssi{SH%PNlAk
zkzBH0?dIFvofA$grHg&Ge<q~8Y~vK;m(8tr_7#ekeXCQKH#oDF{i5yMrH|Vp<i-0f
zw*QShZT)opvf^`mUdd(QkFMV8a-Q0D<k;4QiEA$OgsYf}-+K8a;PR`Z1v>7JZ6q9A
zVtky{v%O+`x3K$PUi<aN)P3*D9$DMoc=$TjXkKK*IU(z6v#Yu_t6KlP+kZAB%jEb;
zq41Pk={uig)Rq^{SpCCwnzi8^zX`YhhQ~R5Rxi+7e9zZZoXvgZDcAiwkBZ*TFxs<F
zqH?R`-afes(Z5dHubAr|%52nGWo*yA`p%K3&#HK@R`2$7S>^ia&dT^%P1kJhCf)s;
zQXHFi?}@~MpFzJ9t)7*+y7VP{$_?DPUr;}D-7i-5Mg2QY1&ZDC|FY<2$~u-=e>U|@
zv7WDRxgz{Y;S8CkRkGnbXPBqm+T}CPZQJaC9JW_GuKd0EBx<<@qj2Sisap+yeCQ0j
z9A@q6`2MxVy4IVm>sAS$V@*+hZs9X0Rc2n*n&_vw%cPE(h=@*W3XZYzc)sucB8E8I
zpZR8O;b|YI|32Xn_HyoG`I<$BiM9nQ6`3KzQs%b0i5=@*``x6iGv99LSf)Ky^W`F@
ziOXgby^Ra0PI_`DC#&{i`m|}6tQ+;79iQ8EKv`r(Mv3kWrSyGkbUMG9pA4OT{Zh@Z
zW#^=W%iSWbhl(uXEjo50!2e$vbLHa4k4q1{x%@-zpTPRf_g<)}7uHF)d2fExQ1VpR
z%yakT8MV6W4rp!8H`@PRebT$7Gov+rT20s9dvm>MR9KNf`mUXAD{SuGcvt>P=69t>
zhQP(|mH%^8Bb1(>>zICz<NEpkR|OZ?a=tRz6@NVLyg|S7(hrNzee2F$@wg-2UoZ61
z%P&11nzIVDnh(4Ez4N(YLV2CB;kr}_p{%`gC$1E1d#b+6Rr!ZO$Nkklw+lG}s#W50
zKmWY*Xl<<El}|rqbN}eCdOK6ZdvaRjmSvZoiXGjOpSDkqq3@trTJq$3X+b5qm(G~Y
z-eRe3E^VP1p_C(|P@cWwd$`Yqv)_;B?0k8*Uccwm`6E87B>o2LTP**=!rHq}ZnA7v
zQe=Kg%SGSw3DT?LWj}qLFmucDcD2Xtg)5hZZb?<$wajL!_$OV}$(bwO?%K1dr+{zW
zz8fK>Duq@1PQ5k@F`k>~W*)|x<-KyBoyoIuSKBq8#Jz*rUR(NiujlNvJ<H$lNVD+g
zzVqtSZ-rTQGXz>m?d`Ew>DSA>@?H98vs1?8o)ryWEkka7(^R|Y7jM?Av8rT-_^Bmt
zCs%HrTbZkuI`QI#&D$&IZ`k^7wV3<%z{!#sBF|%{)-gStXXJ21wT17y&|A;1vsgB0
zN&nvYbj=~txxAB)`H4PT+52jV$j&d1&$erYX&Udk>m43a_$b5AW?|aN?oCf7ulce>
z<*IYc7f=5J@9Sru^T+5+w|gAC-|5kkM-x}Ie191CX<_TyGSio(juZX;!~CLOzK|~d
z;CJv^{E0LFu9V*U+u47^<L27sQ?Ff3y?a;9WOcgA3e5*c@?OnN@Mij>am4%n`g`l1
z+WcIyU{_T2>`yl&uKj2^xNh0Qo&Y()CsS6hcy`|<%_!XTfsgFHV%fLbtZqyGykZ_H
zHRE?^k)8RclO=3yc9E%yi!O#nWnR8%v&Z84<VRsAkF8tx{pHT0V+(9{PJcG<g}Gng
z^i9t%ZM9i5<+gg_eAjgaUo@Fl$uIHl{A6H|%*z`W8r*X9BKs46tI4bNO;aZv_9*8S
zQ~zEie&&*&@D%=|r{;@jhKtPeHvRO&>El;*-Q#MW>2nvnI<@iGmx6^8!eXZX^W-d<
zU6lPy$@O(yjBcY>?VNYJJEwfR>~UtpDN(kRhZox_qnLJut8L;_u_%tSUH3e-xgvV{
zce#l>GWW;d5C1>cC@-#IQ$OeJl%<OohD^)a!RCG_xLA12<^Rnowg1?Re@biXtK>i2
z>|?QoM||p`w63l^&5wqEw)rf+yV7_?<4(`lcVcXPug+U=@#(w8-)f@Q?su8Fo6But
z+u}+73l)7kHDtV*n=Pb_?{De*d@+3Eq#2@mwk`jr7H&V;>H2Kq%SVB&`^tM?a+NQg
z;56xd-{kX$E&r|G^lVk@(=RX2n7W2{I+?!vxlrrz+~5_@dnA9Eh##I;b?JKX?Cd@E
zf?IX&8;gBg7neL~)y)?NQl6~qWbK`@;akhTi(;=XT?;-nk3rdT=Iawr{l9dCY}}{t
z(y?x`)}HXE>@5N@vrlc|T>7Eua*nfFw4`}lYm4j5cvnFNcfKEW$Iiahc9Rp%R1Y)u
zR$C$Vs=L`)|Kg2<m%Xp~Ey?yZ-kW6GyV6`phd2N6vZ+TdZL<t;R^s@}9Bj(-f>Tdi
z?&6!IvkQ;EmI?Cy?*3Zl)h?zlPeVi_tWQih<Q96n@!wLd_U3mV#a}L3$9BtSZhTOw
z|Hrvy-%3^0-9Jw5jLmyl=&>Z3|D{zC^R{hiLb9*zjQUO`8}1icyjsV>f9}yYcdAR*
z&fwlZXL-ZCb*$aXE~i^v`rY~7tg`dOvG0NJuY^i|+_$3PB)36ioacdCnbXTkuPsZN
zGc_c9$@_gT#EPmz^k%OTi=Ui4cXrUBrygsiU-&;SR&a~ce44sz-Q9_DuHO&Oj^isy
z>-lv0<E*ldxsJWFLmQr-be}nG%O3~6z?b?RObRdMSHB8AoNtxy#(JUV;*KMHB?>n7
z52KP*?5|d`u@r5w2{4b7wbEKP`M3$o&la}jQCq9*(%*}JTKxGyYq$C|gGD<}y#Czg
zdU?j$P+PG#XUY~IPX1yWvgY7z!%Ka2+g|TgC|1m$RGwwkech$lO7Yn3!#ov}cJsX}
zI`w+;lsowWvA2}J{q>9Ixvbc+rRPbZ%lU~HjSIds%CNsM;rsBrQCI!U`|tf}hc?y=
z-wk@gel~CSllZUOWaP|d9c-7%;y#`)T)x<F*C{T$U2D{B?N4si577@+o8$TRX%(Nv
z&#Gyen)Ss@t8bi~JmcJ+Bju($RX=Z*{;;~}xmv8EZFl&k1ck=)8`(2mC8B!2U-Wcl
zEnTGY?abC0@6SK{JIz>B>6FuYVV+Zy&KYbIjX(D8#Rku(6I}GNoo9VWxKxmHDJ?JO
z)!R*`Mvu?Dn^<X>cSLgbk(5`)`zM@qu6)j{zviT$-t2l&<612NMaN35t@~$g;g?P-
zv<^S+eyE`(Kb-OB?TbII1l(;aiBW%bc2%mc`?6CaKT<9gd<vKrbl7B>k!R|hjEBXM
z=dXoLS-48cD(vtsTcL^BHRsP5O53~s^R$Y1DpW7DHDdQw(Id92(^q!|@)y6I`uj(3
zO?h%s-SJIoy-mS_O<N+LY^dOVT60=>?I*=+{$8zHPi`|;`FvgAr`ywo#s}mBxh{QG
z-+S!VuFd=2e?O4>>h~U<7H|84-9-`n%LRX&oG0<+!e=hOoN31{>^2h3_u$N*VUV4E
zXH&#$$xAM&rBz#PS1x+@UHZAuhM0*>IjldD+GB&JdizW`Tlwt5@~3Oh&71S9L`y}=
zXnX05(<l97@7z_j+WR1AlJ%z#y-!Saj(xvbS2J1T&}XOQMt`LV#}m&?wpO=0`QfSE
zp*IX~CKo)*s(D|y=F=|j605&crFJjd<{=jzwc*Z;^IxLW+*UfC{9O21B2K37*S?5J
z-q%V@TlX(~XTIk{&DuR8dT-PBZ?bGXxPO@iyUX;3Gk0cwY}39J8>Ue!?C<GtxzLKm
zwcdZ1LZZjj_75MdS|x*SowaPRsIX)6{BX3e@TJOx`%-6j*Ke&dZdp;*_uyDc)tys{
z7blwU<rfNCtXR%v(VxCUK<j&qvcs|}k*6{H_nY+oi+_4(`mcvYGlE*YcHcFaZ}Rv0
z`!BB#WLH!&WN6!@wwx+H+*D#Ckse*STZrexWb?9q^RSlS=;*vvF&|Iau8m=ua%pbb
z;VtXkHhwtfclXb(D{pQ7<;HS&9ZP<e9_1I!&$l^Z)th<OOLNrMZJj;&{;QwH>8XqR
z)>_$k2THv9Icvw$Q<?cLZKro#%=r0a*8Q1kYj3%pe>msX^}jI^pVh-f@=NaS(39q!
z>Fc(h@!+)(|65VbT&$MChBY^u--{Pn+Dw@&b@-H<^8BL8yiJ$yCG+MjYW?W?WL=?b
z#=082`(n#9xh!>FX-ztD?S<>Ri-D}kve|Ntzg>4Ny!!OJ+uoyBXZ$oex~7M1_k7j0
z4T~p+JeatU@yScKy&ccCs9C=9h@QFU+((~z4`)7NTYvg-o%NjWUGv@7&bF7;**}jv
zO7{QMForD^`z-v9^D1xMHb+f7itTnrkag~-S@s93^Zl--swA%QsPJ87KYvB*)wN5#
z(o7ETJM}m#r7o6j>5Ub8<d4N%ko&fpXJXp-lX->HFP#gvdNuX^>YKS;n{V$bsq@%U
z!15_WWBU;`Z&u!k`fc9ak2MYlzR!Q7B<ovJmSOc~_mvcun$64Cn46t7?Emma{hW|%
z;^73IE4=E4rCPt4TAxO`xR%6Q&iKmtbB_2?uE0x`<z2eZ4~m@h`0hCQ|Kp=3p8MMp
zzcx>rwWUzc?waWB%)Z&Lg71f|KH0@BVj^WfTkHPMj<)IX`4MiDGahQp3cP+HYN~W*
z|HY*_wj5quoy$v3NEzjyc{XRwG(G>1k1~_gZ}NYf`p`$1Q@(V2`huAv9UHZ#*0m}2
zKQ;a{A=++f<&)2I{WnRozs!<1zb3r@?aw;qdH;VO;n)}N@-|hX=&qL6ch<>z`f3Mn
zomwn-!-Ub(?s=i~#<%ak=mi~o9wt|^$3&I0cKO|&KK4|lKWegPKDoS}C!#a&!eOp8
z$-CFrm1-|nZeP86Q`?g3^&2anoR-emp0g=A>!;-FU$Z<TB|kk};_BFD$Dg6Ra%CCU
z*4$51SIk^`zvSUlftu3q>0Y6}k3LUk`g-$D<`dDsf%&P~+3RHT^q%itd++tNFg3o!
zOD8Y%<aTj<3N;gd`Aga^<<5l5pQfGc@85XClg)HTDBsqyxf8P*?5#ij`E)na<bAr^
zsTo&S&u}$1@mZxk*Lk<(+PCW%)_gm1x;DPmFX-9D(r;<E-+C^r_PP?1In|TN@}J$t
zt4mpWpBS$_`J=GbadEND>BY~dENr<E{Ab@QmGq@<{7<8^l2UhEn09;9;(Wdjb(5S@
zD>kz)*R)eVvGwwbyq0{6Z+7bD*H`Df3%cmj=-;!zV$HtGzdR(ZE?vp7_uIw4_4Lw-
z8)8z^v}b;D&~I+ceQ2>K&pAbJchAQd+v$^Te0r>R>2X%vWV4`_uald$E}VNX<7-v_
z%e?y$<sQ$LPv3pT?fZ4Bmnq(Ms-D$d`&XJ&<iAKUx;RyQ*E^#_MWs$VY&)y<OlvJ%
z&%a-;YBhf!m(Fza;G<=(y(g>9zxeuxuUP-@c8AsG{a)&8;*>Y2rAOHM{;Ny&HVAc@
zon-#kb3<Eyl3vZTh4<upJZHwLPJ7@l@c65naN`$s+0efQ%eK!yzRBj}bc5W<-Wyd6
zYvo=p(N4)y?ULWu-8J))UYp#TOGj*K5^gLkRa*J-v3{9^McxPX&{LN$+xc=`2n#!Q
zYi5u8&gi%o|K-@H&3n!8GPbouzp$j#^zg=?Z@Qm_UFF|+{4wYL&3XM?ix2*Pd|Gr@
ztm?7NeC;Z+r(!sMw_Q<}EWh#V>AGuC!oN5F4(`ponY!b+t!36(byfX!DqrO<m_`&Q
z|J}49@W4mIf{A}uT60}pDXnh0)*&(H4$o_&9fxF1N~JaS%;maP9JzCEOhT#O`t_eI
zU&Vd>_4C}JM~<z3RzLZi7L>ziDX;TV$z{&JN!t#r|NW`(s-(K+-`i_fzGMj5`8WEk
zzW9cDi+8kM^Ap{k#c%kdW@d!$di8>w(kb0~i*9G^-+wWuXqWG+-)HN--@j&YB#paA
z?0WyxImbWFb(yr_NQ=0pW%>V`4qwaUE}LcCeYqk3`L-z*x=$h(EZ@9>^~1jS@5@#%
z`<yaC;^Uq-OXYS4`x$PVt9|=cDQ7_Ho%XZ$*9Ay=+E=UG`ofTY=W081QFvr$rxDYQ
zy4~BKKT&!rw@t#q_%ZjdfGPD|ja!R%+%5m`&E@|7oBI|T$~#6L7rFnKd2OV;+D)OP
zUGHwOzM9L~87S4tcG3J?;T>te_xB4YYnRUMdF5NZ<DdW5)9VhpJV{!6UR`R_+2C)R
zr+%M(!}Qhm)xR&w_5Rn|Dr>UxO~BMMmu*!ou9PkHyvts^Tk`HvW+|1gH)h<vwvTg7
zg{5Cx)|rR5&6htox$IBbrC7QBkG9Q!ebd%8|KgYF*LM6qx9{%C<<lxY1>6ble!NU7
z-}H-~<(`^E-tFfNB)_JyTAKxIJF)yr=D|F7H|`r}t0H!*rT+W1%Fnl9{p0+fPo>v2
zoY<3X`Ss0?sBdN`Hq2UNztwzqYv+X}Gxd+wMqXGO=eqFoP0rhA+mAY(5I+)mYl)j(
zY1jQ3Ndn8|*Qcu4#plfTQoFOhj3>NFK#gtx`o0y#f$#Xwx#e5D4bMOShw1FM+qU~l
z_u4w&J&>Ls7dmg+x3a^JcsU>P`#ZhWtPP2Oy65La!KPw<<GI~$%fu=h<lju$-@5$c
zk`|xszuWpQPM7=m|Fl$ZZL7|r7Y9oA=SWL;x{A%1_-*371%Eu4V%R3Db!v4TRQQy0
zBA3hU^nK2tmv)99YO3SHcke!$xBUOB{d#S++C6uiHrA{&dCGOO)JKfz^vleg=UzA0
zEEHslo$xKhL3;nnZH=CXlOLxUlw9RJ{iScVh_!Zp*uf=-cgYy4yxKh3Bgr;8=8nEy
za6;UYN3%sDJ=%^d&im7A();?^uKD%iT>+*x?-ozhne+YG)&4KHTB6RU7*CPdky@#%
z<MgI*hr@q$(OFgv8&%b&{mN86f4#1F+xugcp_BJ*mOS%+Yr!(UE8@4BuZlYfp1-h1
zDR|Y|lSe14cUURymn-|a>Q%q=?05<F*Hz|WCdVeuzEbhqrFq-8GX}@Mn|F84KJ;>L
zesJc#C6{E%*_(5$U%mR?a9(^(=Ao|N;eoTozsj$_t#WqvidjC@Q6Ux5vFpy|3w?O^
zZ}Yd_y-&otyytE!yDMeIm%wtzHU8YxMf`8w;&P`X*Y01m)SESZ8NavCnV#ufCU?`f
ztKU+0c3A4W_10g5bLCo#ZTE5bJWcT7uGgPq>p10YLjL{9%-h9_uI|1&(WWwVN6s|!
zdvl`Zo77Z1pT5a$a?`Czb7C@?Y<!f>kFB`fnkd-vnVZeDrz(8;xpVtu_6s>bGMD=A
zuKnW^TZN+iN(SF}w$s)1`L||-zS}0HaCf<H@QV}OzK<)jLh7CfS?8rbki2advt;vM
z6Zg4ScJ2$wKF2z5N;ZG?yQ0M{I+4%hly3*O1_Za3exJIla@R)IO+}~PEZB7}Cuw%L
zx7|Mehf|#HxC<wEy)_WyTexQp%WL@$dV6g63QzexIiqs-?dnPVvsI2suK&$`_~~YU
zGai1XX59!q-<h@-e7~0@%FD%h@Y!j$$;D~-)#q34T2P(H9JFd?x21;2rQhkQ&RdTy
zNx8h<PRX%DNiIHO*A7mrZ6B|`N$vfpes%hjH`44kna<{KiE!}BkI6M~ly~d<BOW@<
zJXz@e?c2hZRf+q*ROrvSd&2+Y$F*y}iE;h>&93w9r+vSEiu8Pwx2%(vSMS?>=gPNk
z4+h?VJi#c21)t8Q?liWI3*f9w*E>G{G)J-RK7F1H4V`PuYu{_SO?!B%_umPZ@Q3Vs
z-#y$ZAt%bawLnNU-a+R9S9`3*jG*beUs_FTJaSHT#qX+U>&GW8nBN9jXP1ZhwX&LD
z*IijG=E3D0DwA&zoqQ_B);S?zuKUEy!+9n*R+xt@c3<#utG}FT+ING#!gnc?PyW2x
z_v7xb#93cs9;cmtJN0zb&Na){@!3{z9}PaQ+`OYbP}6oj*RN;4_gW>aI(=`)w%+W;
zLQ^7nbJCrUO3t^r-7S56V&kS|tCoJ^$O^l|8$SK@wDXHEoZ1qo=D5k`Srqq^mW9>Z
z&wswN!zg4m*WCJizpA`$9uNC_#-H9;$^72Q`NdtWlK-;vwbo#}MN5~?4OG{)yJqm=
zuPNKbSbN@Py`$phx9jwz=T2Uqr?cKQ#j1J|*W8dwHujKeC(g$=ZU?&>|5`sOrpz;v
z#s6Kb@m=9*ckLdD+?Hf)JipI=ZR=hCiE=9fw<&sSYpX9k=-auZxlZR{M^)f_>31JL
zv9Yb4F(=uaSF<2kqkH|MW2Y<p+U%yObk`qWKl?=Q=Yy3+b@SMM9i3zo`FM9}Yg|uh
zmBNwC&4;TmeaS9jnJeSjHz`n5^Q5|gnn-w?i}@M$y`e8GDh}oE(v0(OsB09D?vReM
zvN)XbSbSq|<n^LjQ(JTWF0Lh;pFLRe{{8cl{=e_0#LX$2`fM@#12xq)vEM58O{*jx
zT20&$xV5adz$*5o-;5KN70(I#>YNfa485|bv{uAoUCug_D_^GyUf#1lIKuC7*2Bb%
zuHR>0U)(b1`rIv^QB_`y#wT4b@AY@;Jffl9?4Y{y_}h>*4xIagYaHeFU+p_AnUh@3
zG;`{Q7LnFve~o^uGN>0iI!!nrtbM^$$2q4br=C&&{LAj<gSG3}Shc@rWR~u?R83X*
z6DjbbrlBW!_9sT4rUhF!Us!atIM!$C$1B$BPo=GupP#y|>hSTq=NKM*-u*du_SI|~
zCFjLyC!W5PXiW5z?fdribJ|<aRZ~BtO}ZDsUaR${spoD@{=^-&UgiHItKE1xqjRtH
z?^^acPp$KO+ly=SKD_nh`Rf{{F!P++!ZXjD5}yeBKfUL)<ITyI46mw;?=GM6BO+Bk
z|NJ-Y?);-pGd~;LvgLhw;arZ*{p(vl-;H<vSJty{e*kxE($e?eWB%-2Sy;bl;y;!0
zU2p3@GVJhWntn4iCu)1P@2jV3m)0rtKbX_@!9{!Z+}#JIA6<62WFm9+mg>G;S6?2l
zHvD_?R`Jt2&zJnMx@|ro%<qAa`OM#{oR39K(%4@-I45;*=1QBT_uH@Tcd@LQUZZ?-
z;W{tw?;7!;_ik5Sl=|W%zB2vVVf6`dtL^PLcMJV?y|$nGM)Y*QJIBAbmU=%9*vk@}
z_-t?Nw5M-fie}C@^KT!=vvlo4hWn=%tFzVf`u;f@{x*B-%hSFWmfw5QJ5#G~^V+*>
zCOkE5Z}a;y@#h~m(MBb+mJ<H&?q?52T_~Hh&qeKVcUQAst!rzvt<A;$f@LgI%T`@I
zV=lSrk>vCP_s<3Yna6Y7bFuxtJLf(4kNt4JxguZT7q@6fa?f{`&f~vU?5tii^|-pH
zrTxz}g_jj<KJH4ld)0BgX0C+#kpr^zm!6yLzqsjGeE^$XZs0x}iH{1OtLA^VwH16I
zAb0+1ePOcR@!R+JnLIyT`^KyGvLw^@>(M(lKhMqF;bSj7Wwv=X<J@Pn`_|}Rf6b)B
zaD#2_Q_q-htLwf*ou9XBN3M^abl=^}_K`ax{qr8MrY){y+?5pUF=c;UZLD&)cDTTH
zQ9b1=zBhNA|2cKnIz!FMgcuJOufx~RKed_q)Mvu1BePeg{WmS^{vP>bPW8sAM_0y&
z?zZQgI@|SdYS_bDoqx@jbzc3gvq36sh2D0l(_8zmoRyx&5q-t_n82GaEZ2XCE%$4i
zX7OXzwBOk~{r1kA{cXz8$lE*LpZ{+<uk+iMX+2r_K3=Z!Rks?uf+jWZx36S)`??|S
ziR<Qaj@hy2U)JmD1gkYyKi1#8@?5U<`Qpbxn(b@*=Kr?Ld#+}nQK=-Ydskp{_RL$~
z@0<-e@cU<b(d5R$$h}jojMn>YkTRdb*rm^OFhctAA1Becj7R5<KBkv$lwh2-R)?k8
z?Ee~>CE}}IuKusa`Q)U{=Hgt%Bkj^G%lIGcjN#ZG;(MoS$=`>L4_e%6r)@rdN8{p`
z7REJ}W{%sX*B#ujbHy`RS1YH5v!Ac?`Nx@CfBxpR*{<t)vfpM+`26VE%M(&p=d50N
zV_i?hVy7k2hr9$|o;lGIF=ut)#n45|_%rSKq)+bFV%s6p+*4lDmAvKK&w~qVJy~zM
zai{;Xa$Ds$ZL-SgwIA6&P2Tdij^Bpw=A&EptyM(BIf}aPta-=zqN?b0cY>;w)jO%#
z?S@DHAHO$!sb$~mGyAf2>jVx|{+HgJ<`S?Z$Flv^tI0RNf4?5<d0OqDBMV2;oNo84
zCxvQlJ2szAz2z1#-%#qw)6Md2ay9|4T-aCX%DFZw85HKF9A!6uX}m@A$<-IiD?F~G
z^;Jc0yYV2fSod|M%Xiz^A5RoXZMIk@@_foIhj;yP4)dZVmQ8iMcy#llC-IX0+c#gA
z`6}Cf_E!v(m&~(Ya$=Q%r>Fk?xnQZ8>*=lY=T^)MG|&8dspf8~;P#E@s&Z^k3-WJC
zJXadOMNgdjdE&fT(^pK)xv8yr*X`8)&kwKU>F@ot{-3hRUB6R{gwF1fy_l2KqVmuy
z|H_h-%%m+0uIF4<a;39PpT@@^Jy(@Q_t<UizRx$7&fk=_?M~^NtG35ZS^33FW$d2v
z*>K~`yr^mOXY%X)S1+Hxys+r<#aqpdY4VfB_1PEeZf97X{wSq%Da*t)Q=V3KdHv7b
zb!5hkjOg`3%etFiMzKm=)jU-+IU#F<e#A7h{d525*u*Y5ck*{<`kgg#1!6xf<*$F<
zdUxgVjqGkUpN|GzIFj-Ek^9-2jmd$FD=yBG5BvVB>#wU#*7I5y#=p)U9>4g-G_5v?
z$Y(FkzcafwK5x<=r}DLbkIww|qjl?ZNqfPk(<jvMpI;w%o%hwwCKJ^zjZ&pB@iT9N
zzwcWgQf<F&Nq2eeiaGLC0`{rDi=R$kU%2g>Ue|fOxLvagC&p^#WiVb*IS}8b*!PJ$
zDE;GtDQ-N>WrZ)xJ}#?y=zLea?!dhJw<?Roc@!19)_>;S_io{%&%5p4Gsl%InfT+8
zZS{}fGt(W!cK*n{QMf0@Ynh02qrdoE#@wai3%Flr_-Edo##6WFw8|_&ot~3*4=*0L
zv94M6^0^2LcTIJb9TwXRw|bkj#OdEW-B)V3Sx)X-sKj@3#o8YU^M5i|Yu?=G_uA_1
zYnwBM)iomSIz2Bqf7#`m1`8CK`6m5lU&$Ay_q0`a#}2=W7>l(|v(C9>c}B{#-~Gt_
zQz!Xbt&r~7$I8ZAWoF7v54GGl|1#T+%R5fJ-(IY8D}Y~atIs}(oa=&r!WrYs<L=qm
zu4mpcsVLcIwc;Gh<NhlrpSF4C@iJvWM8(PUzJj}lCze-<T)f+R#$=K1FXq*oE?dV<
z+*aZH&&y6_=iFQ$9j$p)*QaQ7E}iChZGYQ*m8hFr%K8@<E>sHmlO^%s`?QuTQ`#@R
zW7=iN*2nlhLgSb2aqsAS*95U{n-g2Fa^LrO8c-QuxACx7os@N4gwPd()@k2nxEq{G
z-#aOE^_)wa_AV>huw{9kn%%qQ_g@4?Xr$;(QR2?p_&Bf1Z1-M)&g(^%8<*en-`uy{
zEPdt2&rVkh=6%SFp1DV_t4TIg_l3}oncb_Vi!4vuRQTd-T(kUt)3tocViu1zzGW!K
zTs{`3IbVCy<=KCtOgvu|86Vwv_<6*iPj>OXPo;Ab1&xb#&M^0jv|NAa*gUqHW1>+;
z=9%2`#`lg&-aO;J@72-lu;+`OiB0dA`uk05<lOe1HJ7c9B-{2?1y;WJ_BrgXMqiCX
z{^hlG_U`(@K9`p+nxEPz63h2!*1Ek~=}$i$Wb1xd5_@|`i}Hy}p%!zMEYl`Fc-Oqw
z)$tLx_^#liQ_ku4g~jV>KHe~&`&HQMeEubm4qmGKGkv;ns?Ld%O1FMwX@o_;{almB
z!1d%<`reA`4mFY<$+mJ%-NASFMJ%nf;-B;AnEG?E0tT^Xh0*gunIEglt@`crY?qc!
zg~fb<DKA7hKVI9}=J;%W|H9<rGavg-72i==={jkjXpz<D<*NJa-2FClo$k{9uI(=*
z9$opT!foxc0KSi`PZwwSt7*)Y&|TT8;VyjV$kr^8Yk_*^yRH~a`nA9#cgyZLr&C)4
ze*W6=r^NH$ZqXM}Yu?U3wJD6>;=86v*Yac8JGRM9v)Ss=5w2n^8)@R|_UW)??jyyw
zjrv>uoMD;cFYz&an*9U+fWSYqI*m;>Hct87qa&fc=w?q@uD6G5_hg}h&~I0TdXHQ=
z5@>YbwRnr38r#Jq=Q<5MloiEJ9SMw#ytl}={O(yXz24)`x*3eZ=C8bGI7RB*??Xq#
zXWh8CIB5>!LZ3O;!`^ltNx5{`?CHzpOAQLc!sPG$J+wi0>U3QVxs!4hu9sdvH(;BP
z*4AF7w(jID<EI<8%~`1C{(ZHow()^4X;!UkY>Q>}%hm61+cNQOl9h~~t$slCy=~L8
zV-+`7%>SIdFGZ$ie*O&=gTPJm+_%c#^%A%f(^)4HQBeFbEoEcOrajMFKh$1$@<--f
z_owcYmh;S?>^m|uEI~$m_0!*#6V2|=*JPG{cm1@D_Ko1f{gx~LOyKFcQNQubwma6h
zw%jWIe0rf@L)p=jb&VH`cOO}r`T4Dx`P|z_4<>x7D2zVepum?<Z_D!Cf7--zE+x+M
zZbyD8-cY`$jlWVQT+!d`ysiCyA0~}66RtnJ^7?A&$4Pp2XAWB&TvpOIo!cyLOY4<=
ze-$?7RqTkHWT-k<<Fd`2qY_mS6_KaZ<%QWM%WhU>YCV#BzHP^yUa!UGX<BR-#l@zi
zi5)&Ox!kOcOG;?fhWl}vS0>oyZso4tDdyKP`}oe|`jd{y=(BR(KKAJL<^%6DwF9SZ
zkMTLWy-LRYV^MOE;fkL*W-rU^_EsExeWqZCn$mS?t9klM)BTpuJDi&+nq6fk&g8sU
z=9GBd+3u6Qe3>^xP2!wpEWI%GM_Pvgd+5KsAB&g2OD%f-adNiCHHF%eCr7^Z?A-ix
ziEJg;_oxanQyFFL=f0&OOzl4%ZEUZ2PA|2w<vNz*xlW?vWP;00v*2l3o%1SAe>qyg
zm#`#`g-d@)Qi9(4;Q#Z^$#2-v{PFqPHE)(L<7xchV9{0c&saaY(yGorqbg%x^s^(}
z-*X-s+3lTg6z`~c{J8P#ynik^ub9@|dU{3o%#-=j!4W6#q@0W^o>%o$Devlh1+$>m
zs(Xh1msdXJneh5Z;<d~Exn*}BRX*w!UUjI9>*6h$VD9tWt>z{+(&dNpURG{+TX}l>
zrku{BT(1*<HJq>v*jM>=RatoJ=Q0t-O9!{Ge7<8@x;=AC&d#|fBPwNCt{?lh?Ciy_
zI~^nArzULJf3H^Zz|YpeDdDOHF=-x*cQ~#cd>QO^Yf3_n$AP{rWrrtB@OJE+{3B_0
z{+wTj*tu0t9eC9{(PVay*K-47e*3u{T?YkM3hG@C?2PBwvQs7P@iA>9SLuHf9p6aG
z-q_)*Vj?+pcF<g-_TzdBRW3)?OIRKFzT8jF@4{>IKYy9KKONNC7&-M%LG5+Ba{bok
z(`+9*k4Z0`aM|O5z{($YY(-9mdVUGp|74NK#N3mOVmFF+T1?2i8LDEuW|HK-2t%*e
zc9yr+{VV=BNyu!<YjgQml}G1kFA<)5Rcz)TPSuX}{U!=!GY`g|JFxZr3e^=!;jYrt
zxf{;@uDf~O>G@BqcK5qiW(5UBvTxnhS6eYJa^Jb~PY=6-<ziI+uQ`>P;+$?-xKy{z
zHB-;#@O0TvryUP$wS06(M&z8I=hZp0gS}rF@h;PPcJbfb>2t3C+Mv87S2q9Y(Pf?w
z&2l!c4EvTnt>MBcL9WYfC%HK$b8-u{N>6#TE;8@@-uWJ{w5NZL)a+L2P~Y_FOmvRL
z0=;if8C~w2|Gd?Fj&xMsdC}ZNDNX6Mlcw`6nryo53g4lknC8VR3iet2I=|P(xboJi
z^ZD;RZf5bXtmv2Zx_4NtZq}yD;;X9UMB+Oxl;*5CUZA@E-|q;C;OS@cO?6jHnK$8%
zite2^jnXoj#rnIq8EM}>6gx}8dA0bQ$y2{EC4W1v{P@w+DcpGzembpMawN3q^1*^9
z&)JGQr`=uN&ET6Br<i3?-Jg-WK3{pc*FO9C@z?Y=Ouu!)V)~OOYmAsBCKvE2-CH+P
ztiDI+srR1RY;kw3;?=sbQo6Tl|F<MtR&f|-%`mNYs|;5PoL=d;FtK-i%(knoOFg<A
z?!LOrQ4`<MzxDe2gC(zTWb}X4aH^}(Pn!EQ{>kqNbANsny!-8W=Ds_e&p-Mfdh+||
zHM>$_5u?(?bLQX0zaK1lwy>0C-pd8=&(~aj)_lMJd}frs`eSb6?&Nj*bxxn}XyW>L
zEACC^wLR<Yt#e~e{rH%CX8Xfs27W!8!yP@BUw>{EzxOrQCPA-Bomo>OX1uQp(R+Sk
z<!!_DiHmo{RK(mdUH;i~lGpau%O$^OtO*Dyf9M*THfdsV#~JPKIt_)^uaBGGTA+JV
zK>XRg!;;MVx81+HZf5a^_`}Y!Z!RTs+_u=0bIOQie$Dl&6~BstKeq2abj)Vg;f60Z
zG8vZb-n*6WJ=%EGX3oUZ$5UN+zFl_uaai<FsqCi>56Vu<r<wl0b!uAXzbWyjubK5<
zfAVnQrJs9ut<(1D`H+xWA98EG<o)S~m*>Bnq$462aKx@U=H0YiFMb?4dHv)jn>&-m
z_o;1p{npc`QZcRkuHb@}pAF~l_7&Do3!i*?o0`be9Y&jz3S3^!I{9JYvW=3V3YTyF
zT%xzHE28Q4g%4WH$-7nLMWx+?Op<o$*L-uAJ|b_qd9l0j+;e*le~4c!eNDo1=H-7S
z>#zLX^e%Aut&i4veEr{cbmiLa(^wO%7Gkhw{iF$>wkN3{ke+eMH2sH&EuXWT!?kFp
z^50dv7-pu=waMR~kSqC<f$hkiE)lP3W>seIo);>tzg7I@+v)9*mItKb*E2j&a{sJe
z_uh2Q^$zA)ljfc>UH$Z@?8E2J%ga|?{=xO?xc5%OJw6Kyej08Ftgu^bU!%U<v?F-c
zjUC!j%bmpATvP6RI&FB)YL1@oA#qQ3@r_H3tK$D&f3%Y?C`#ec7tOhU_tpQ&V_a1m
z_xw+s>VyO7u7=Az{s@2n*FU}D`{~x-k7ksun|qswGo@^y^AeLiC#O9=RaF?bsKVTW
zY3=7#u7<zVA1{g7YwMu){}<EASH?`X|DG2<Te~#*U-{oPeivexEFNFV6*;}%O<#+3
z%gbAq634dRlbXs|?(p_jR0CI=TC3CKBfl7xHFoC<9o~PVGDPK1rkKDg=909it8BCT
z+C*-h{#eBs>S(>QpP_n(%rzmo`9@#9PLQ|U@@H1qE2Zj<v+ClQJKxU!6F+Oy^6<`A
z`x_cdtYt6OEf>C#xrb-1$(q9F=d?`x7X7H38~=e-;(B}2H?@ghq?Rb!>@V?pJ==s=
zpJSFTpP|T2zL4|hA3E!cH+_2Le#LI_Z}DwmHo?|h%j6fv$*}DCUo+9?)3tR=+(Xr@
zG9#at@b+v9{Th=dQQ$Q@Jn~PsWpK&$lD~1wW`1vv-Q==xsY3l7cD*>I*4z3UW-hAV
zs@J_R>({Z9E(brB^Bp!je`tEC9jnpqe*V*<`^;pt!$pkZ-&_%zuRTwzMT7aak@hX-
z{M?^S&!RX_a|?88p8s;aXs2A0Zq82KIgjL5Sm>|gy<a2zH1XKeCU2vS=fa-roZh0F
z8`7_Tb^XS-J3rURUJN{xD%+KGKX)fn<lY0G!sa>h1?R8ZbXvq_>vOEy`=B)5VfPND
z5AE%~_r;Vh$Gx4p!{09dX~NpGsv6hjEY_%hcdt&EdAwfl+RZmgJ>hc~g~Ut5OKW-l
zNho>siGANS760!CH&*RPdvdG!PHuhUP7|h?D^vW}pAz3UY1>Nu!`uaewest({65IO
zyCQ9SzuxKeze>L5mm<2e=N*0f=$+^K$10N_D9W9@KKr=7L~i_zZD;jw91ec^K|uem
z__T|)C#=uEFDPbxR21zIw%NP#Vx-N}Z9ZOllMG)?5mK&O;@z9>`=fn=rsHk}DZPLB
zJ5Lv=Ja)DJp!daP>W1GlZr5ov7M%R2Aw8=--Rk=<F{Nu0YD!;h+q%f=r%bV!nW)w2
z^N&UMA75~IVpm`L0*!qewsl$Wo|l%s+JgV}KhE$s_v#+(KP0=u;_7*SNtK;4N?X(}
zzG4r#`PlgXq4%*2hs3Mr=|2BsGtcmHavkqkakZ;Q8}sGj<2OFD7LxBctX1>y7RUGJ
z;$`Z^N=d1ip0-bB@=NO*zxMsVzBv3LkM{J(+FqWY*Tnj7eO)J`CU9NXe(GP&cbRiy
z8zfYg$<FyQY4W<yc@Is>gX$(vPJ6zZWnTf?vIV)zw};GIDywSiz_&#=XU~VJmXSA}
zw(+~{UpnWR#q(=@HPesR>?n}gw`Rwy%Fb1D<##fjDg4=ZYfIbb+p9jehWTxMwmkgf
zb0MKy-)ofk*PrHFHK#In`tODR-G6T?x+TqV`sYcpD$8$a{=Bp6_pzzGNcPzyyuIL_
zZn5MvgD$hf`A+$!-~8tlW<^KsTPC*t-SfY<emsh4GkE%j{rV{xkJ%9(vL|QGUu^w%
zMfKiysk>w*F8bwqWzCii)`lm}X2%yfz26<6Zx@kpLG->zR?Igc@us79*BNrS9Z25$
z?MrKU_{*@iBL#txr;dr0JPoJ_u$=4u$)n&=U36~o^3n|xZp)qEb)D&5a(Rcu_jk@0
zHt<^=xE}c})o|s8tTj=pFC&@N?Rc%0U*2VW_!hTLGehATjwKzALC2G|eYd|?l@)od
zy)60W{3DqH>V4r?HMZW|k{o_+m*0d5((6pP@3!Bx)ooSoUBsVtruF^{o73LR&jYq_
zi@KgW^tmqa+$)XQPgLbjZ&6sb@&dbD(Y_~>9F}gA=y<kBpe%iIN=IgY=yzk+#m$eB
ze;Cd7<|uw*pg8xisL#E!OB22<dD<3ZIwAbXuaA0$huC`$2WO@>^*vn>E9ZAO{zYF=
zpL5=?E|2t1-$#~wvPX6tUnAH5SSd2=!iL8tS03E5c{e*EU)z!CYSH8u#(wOkzh16w
z6g&0x!poDYFZQhvfAo3hA%(TcMuA@p&V?@a<*a<XF}lBG$CaJyO_Y5e6=h0@On<C8
z&(oS^U9IM86U80B`ZF8iz1+_$uADya{2F=5KYw>VTDiL9nVxyuN@@G7O)r1uv`u_F
z>y>!WwPeq9Beo8EMwRSx{b{9Z3ThejtXe8x%|9N;YG|L3`%ETkvijURj^d>npAYEf
zX1xENB$B|<w^MiKLoo&;qsrRUJ2RI>?_GcA*d4>tWhNVKI?`%Ww#b)Fm8jG{{qbX#
zh<sk#|FYJj8)uw;bp3Q<s^uoH9mfRQwu^>peoobuJY^Q0uXNh?uvhJ^3wPX}RxO|W
z^Vf@*$;T(?icjKCdFbU65u@W<yR2)z-qx38mpxYH>^*XW^|#&aqn`!dJ}OGy_5DT_
zuOVxW(Ovg9&D@;AH*B41OItN6FP`j8-s%yv<VRUp{`6_@%^rF8GG~Y1Nxmx{!<l6^
zPh?`#L%Tm_?`BMiH#+}RVr4<}_m;I&Tyr!_!t~re{muR)J}Y0rlI_qDz1;=V`wKdx
zbi$XMXpNXyG4q4Q+U2uqyu|Y6o75DFmsa1qwk21F(MMy#uLh=NJD1knSfORK=>6nR
zx<8dCNGp2v)rfJO37A*ReCBYJ)?4<rMK-h3m6Q&y&QVww-SGbL3d5JB_75I2UM=PR
zDEwb<o}K>1D@VRu)=ZJe(TmyI9a&~Qf4%Z=6U)5WJ0AV(ky0=D@nU~`U}IRba%1DR
zW%ie4vh?`&uQ2|0xkE3|u2#oa#^%JMd7%~Bn%AFo=+D^jxi@@DUi~he2i4#E*Xzc=
z(6cZ9{>q-;yYJ^^i)niV;+tHa{8o@%qW`_+<h$Rjb3c5F`B?QK^N!+q_08?CWB1<i
zo4<1Z>rGD*7s{F3ko)wY%4th*$8BxCJL2so*L9CLSFT%g`1_VQr6*s;o;#>Mf7#QI
zHu3%E%b%UTT5`8BCS0cAeE7VGGc&?8Z1!83+5hr)`ll(7{PWSiZ(LjvTg3g6=FN+@
zI8?D=`Sm>q+iiapc_sx+U`;%DQQGxIn}m>KVVOJkhF8xQD@|X%{zuc};;Uu9Uad(o
z?y~Qi`>Mfy@BK}u{;|A$z1oI-@72uy*h}6MqATZ#%)0WKMT<T9`GZrHT~iOQuQ`8w
zS*1zjs$z|)r?*{wKK0o7r(cESO6?+^1>2r$V@v&bE@1BU8&mz36;9Hv;e8d{`fZP(
zU#XP(eKox;mgRD-eh)Vt)RFc)R^#>P)Xjohr~bS>?6}P;{?wdeTSK4TKf&4OL*@T3
zuK70EYkKn{m%Ou9M@8>z>@{2JaZ}Z(F3!uVzWw)|e|y%)-mz`HrhIH(th4fjMSB;{
zyzCNJwuQe}!$__vcrDY@ZC4a|IG5Z^ZJ)e+>*uhCdsgq6t@k3u_W3*Io|3;(MoV{3
zIofydS#fNa{o8L7PR?k$yD?{?m3dc(@1@hF7wX(Zx;)-4`*M5($6pS4wFTKsuHF~6
z9^WBfZhrLCB84j|C$=cE{BdUbWxvYd@UIgNwa2Zsx{gTwseIh};m^8<`ZHbxe9X}D
zW8th}6q|0a?eCt&D|QzM<XW-MJJ|mD(eAw3FDKrXpEk+A^ml*Jw|bq+`^4PlCV0KQ
z`R!rR)fvVa{~WZJTwQf&-r?KlC!U}Cf8mEGA)P^qlW%-Ic-yASPW6%I*4@END_jyb
zr`z1s@Qf=fyqu(xdv`|2vv$F)#Y`^CN`=e#4XdB+ox{5*ZpYKp(tJnvDEEe6no|^@
zS+BADR*O-NtY78{VIRizi;vBVd9nQlpT;7s<||K1?Jl{ht(_K;td&01AhqYozTZZs
z8vL{W&03)|eNw@r?b>h4R%pj2Z42<qos^b6t5G(1(%yh=rn5c#_%jx!a=%^3Jb$`N
zzjvCE;ryvfwjXt$y(h6aT*bhAmg?dwhf>#8sV47#I<cvj@9?&Q^<_u5-I~;Y_*2me
z*IoB-3!T1vN+|eu1#hU`yGJFbeco<eV-T;M{!}P_{U7g&8}A(@C67pn?~<ST)5*O}
zYf?ij$IcJiH^e)b#LK=;>h$&5rMa^!C@ii*xnup-R_3}HKj*ma<yiOOV!eEPhwUA|
zclVe27$(hq&Z568-174)8;dn@1{=f9@7V0I&t68c`Av<g%BJ(5f366*efjSX@7IZ6
z8*`8Txpw8`o$1@|XwEqL`s0V5^Gq(Byz`B}+J>&WEWowuLwJ+2)SI{?hU?!}Dc5E@
z=Iq>bru?VGqwP08d;c#gk2oG4cK4@^W%N6bYj!(s*H<>lhb^$oo_OSL#_8L+hI7l+
zUOisTH?f_=v-=~*W1ha`RZCO(&fk949BcPR{8Z7bZNK(>ESbIN`s*Fj_w!AEzKd^X
zp0kzPn**Bj_P9-Z+dBKV@r55v6Ivxj?@s?cZT<D#2j|ONI8-9L`1TLim>{=*dpqR9
z<}=%#Nz{w*@jbD+E9vsLc5UAJJ!0uO>%3>ZPoLPRug1IY&MmzojWMFz#V;{F6I^?1
zwOY};Ngw<#o%QxgTOstzr@Z4Q$DKn@+#~nAaGv+YQf6-c)NOi|7tdYM@eY2gH0SU_
zEn#1kL|64v$(u3a-w(doDtIETccN1B<mn|F%uH{^YekD0?mKfTK34edd6l#Av7-HZ
zug?>;Kk(&i*xA2FzdSoUf8)`^cHh6$-95g#!tKLp!5`u4_a6Vy<NoH-)317V>wP%X
zpB<k5BHkf!PqU3WAJ<m13%Z;sZvsP?O*sCqK)m#>g00RMC7XGdbKM>k-QDDHD{iCL
zzB^5hzbAP<I&=G)rNWh?f9Hii7i?2I(!N&4<G!8uuQG?v#SXXGxhD6WDA!*s;c#Kb
zu`-MMM>EQ5mc-Aixc&dI+uuV1r+Gejzx?mGZfoRYrSOczhsvF|)@iSt!he6lv<DWI
zTPJlZ{VaBP{ncn*|NZ@E+;-^QuwYpxXyGwaSJA`a@$qAKUw_)U_uihIilw=J>BR|i
zeA_qwn_%j5y{}bu?*pasFqxCv^h4_OzSqzGC+*nv*v|2D@uXB4e)W23d*PFhTk9Ij
zzZVwwpI>(RymPMz^8>jzC7BV*k9WVUh&ti1B$DUcQI+1_VCBb;pYk4wS#;)MMeH8;
z7&gvIwSxto1{n_wt}nJd^7p9g_exD8<>qO%eNPV8zI+*J(z7@{@=QXv$MJI_k0clP
zuHShm^&sb$4YJa$R?(GZ1&w*{y6-#*TzKa4vLlOa&wbKfl_6fa=G=!voF}~ZpYUgA
z>wLjJ-RhfJyxQB@zSpd|m+$ufcvd`a^?ZH*w^6$sN@oY}efDU<*LR<;-*c)>5VMmC
zJNrELB+t=Fu{Q6Yt4D;+{=J9w`Pq5ZD~tYLDdF94&O}oEZK$>J#E?^#SDS==c<VP<
z%I{Cuu|_}c{fA$VjIFjm{~3EI^SSs<i^jFK$&x1n1Mcyw&F2wsnwxy`TmGKkVe!E#
z20Q!NqaL?qq$(xN$zcmEmFIhBbFf#-qS9~fk8|noKfVq;^}PF(!}Hj;T3%laSsrUI
zN-CIX{k81ye`Wpor;krgTKa95kZG;>@5!$YKHU6iPv=gTV;cixUj*IUd8J@+;oo20
z|My2VYDVo7sFj<xuvxNf(d11>?N+Efuily{`MPFl>P5%?|F?W*J=hd_cv|Qct1HX@
zm9#wTJe{5!eO`Wl=I5QFF|!I*RWZaeGQX<3)4GDYRMUNZS>}#!<)-md%EMRtUtM%e
z-sRDXoig!plTU9ndVV$UsY=4KSA`RA1~S#2ZIl&Ww7NYgk}dYnv4Sr9je*ivQ^k{e
zsuVvq+5|X%Zx3AhFI-gP@t%NxNBucCujV(i&;EH!P13hY_~oAL1?T+q7tE`=@tJj1
zYL@tnM(5?epGui;JZ*Wl!R6h`dn!t@N|#kG-TKqX__tzBFW1T5Qx4OO&Mn<ms`>Pp
zR*LIB^H+@K$DZ<a#l<b%Q*9u^6CAWFP_TO4h9~w*E}eJnD`9oH;qda|jIvO5uX$}M
zTi(Yl<6%6W?$}uK!sPTLPP^KDCw4E1j!WM5+AZAVp~cg~&t<c`_%_7+^(nW!eD&O+
zns~k7H@%XAzqID=e9N9*6Ta$jd+xI9+h-_6txIRIS77<6|0*wK*ZrGCwMh;OoA>dg
zrMIYGG7;N$@74{;uQL5Je^2Q<t-R*O#)z$)_jd@d+VoRT@lMX(voF#X797?qUFxl9
zWgc!>y<7K}<JltKT@6>B8r_^OtoZod{spZC?+-7KSBjF~>#rWS@A8V`*QaLe^!t)8
z9#|UNXztd<vu);b{#FL@<^=%{ABJq%wZ|Yfb_MVB2Q>mG40M-osZg@pb^OMm54R89
zoUIp^-uczb_x8^arzaa$GW2hsUa0=&VHZ>8eW}K+PhY=1C;yyH(@|u{9NYiL>N}oA
zF8wU^Ixs!<a_k{_J)IR5G29jGH*0oA{wV7FaPZsv=aHROZ@UXlOgr!VN%s6M&G2ja
zJHE`nrZTtV%Cpmd84OL`-454&cqSO}?sv?cv(vfT^uM!C&$vI0C2-yBgU{6~_wikR
zu}k+yk@1Z?>mr4>f7&oRX`=b=7T2<~VLSDebeCwz9DH<CYInV3-jM`bezw4O=h<at
zW;t1#+N_=Q`wrW?bdilaD)cV1e3=z1VQFpty~^@6zj|@omxg0<&*HwbZ@YC^JZ>ky
z-@nCPG5ZT@=l}Wipr1>QPx8&<#wU*s`c6C(8s8ziZo0mG%KHPSGcR*DTP~AYd{~Z?
zo8NbPPWfSxIggb-X#{-xv0=|HMHALz{JYXLPMez_dAves>cWz1f4fW`J>0aR^3S2@
z4HFeszLD=PlzfnOw)Iy{z0pmZyfbT3F3qlaX!mGiOjyarKho>1J>`5t#Sdt?9&LTm
zbS&f)QyELvL_X6Wo$Z+)_x$_E_<yJDWO<R=&UZx&9^#L_UEKWo9Yb0P+mikDDQlKy
zAN<t*{@?X*g>W|+W@XuVub*07Up)6(;;QOz(_epnIOBjSyQX+w%E@|`SC_dswB@HV
z7y3F!pEI<4wZ!VW)|H*MT-#ekM6D}gkND<pdKO{qdqyVl$z%Hy2M%~mwBuiTvh2`&
zK9?TF^M(%EKNhg*)vP{n`e50%Pe-`7z08Q7a#Z<4k;~lUJ0Ji0vrlu=^WSSe%ZBL%
zZvA|0<;O2?3L~aXQUA*Rw}xfypTlPZEw}%&S-Ne0dHv_-TE#-WkMH05+#scX-{Rvm
z`2&6_O(_pozgx0@Ta&g+(Bnx#R|OO-_iSnX(C!oO@37KfM)s|FSBs=$->oiTcbxRE
z`$mXfy;sPu2NJ4RY~!3gpBKD4wP=>crbg2<CRZ0|{!n*RsyulvNRVZNtNJ2mjqbZo
z;{2T#8zoJcSNqOsmBF;GYv-gtT%PW=dXWO}Q7+Hy+=r$vSwYv_3j2!pO|AWuInAHf
zq^0Vo(!6V%Pn8`!m*K(PwWQ$YdX<a&?7#nc`D)F7!>?{p3b#I|e01KJuK1}^=lqpt
zAt!4VO3#~iKJdl1ckK1GZRbq+=GdKh*Z$*0F?(RZew~uV<-M)lGlTc+58)HuD>7fx
z<IevF3&cv&ST9ZqSbzQNW%Do6S|1J-q}~!ZFFoN^dv|c@-}N_HO?%GoZohO+y?xQ<
zGmkER48CaiIpF;HTX|PXZ1&lmRh@hGow<v|6a(!~Q-ex3B(3Z4ueuQ+qHSBU<V%Uj
z)zw8|6%VJdWUOOm{ltAN_{H9D|9r|d_WnQjY;n#1{EZu?{OfP<FBX(bxx&9<nRU<8
zsm^CB8&yPvi}N47S^MU^?Tj16+MBCcV)KfQ{yg;Y(2w0S*%pcW`1hS&ThMW^*=c=3
z!^6(4wNd8MXVyhTXrJzpzY+1H+RkZ9>&ByspDwySIDbETo%(sPBQ`R-7z<BzaQ$4K
znQ^4PKKXQ5*GmD0!j9RGHyUeHeq`?v$<8;8&-?zFP4s!*SINVAEZ<LeE>1~M^;fjd
zeJ|OUYFSrot2>j)A$CJ??z@}&-pbpuMm}eb%6a?nn~;Rtm0Q|ha=2e!={+!Qb+gOO
z<E%f7>X*i^aOjt3m$%tb5Nm99fByA`sB`>rVv0pG^y)5&-}~|VM!>h_cXbYa{;=S~
z)8c^sN4?_HCTT_=6ne9~SYXz)#2K3NE+0MNbwW<1v;LuMlKcmWcl_7G#oxd9Rx`0C
ziX+&{Lwx$FmgvZ$0^eKqk#R3vOdrmY*SY;yirN1_Ow8hLOC96D7pJeAS=TRI{=NOl
zc9F^ZXSDrR-|@a~@2vYfdfIm0ZQp%xzT>Z#Tp?fkT-FsY4p?b<sX;9F!-q3(DlLRp
zp1PdZ+?2gT;?4Tp7JIw1@|H#NYpT|6suTQKk?>wzZKd7G1vYkX<oef}n|;1%%kkXS
z?*0Q?yE$$9KHRi<;%;{B?*l2jp36U%sv1RZozE)o@PFrN+j)Uap6ti^Pfk03{>}T~
z?+>0TJ>BuK)HEdbxmBx*YOpKky1;XNR@sRf+trxZjDP;p3e@m_|FQmBTC1#x^||Iv
z@&$40(mm&2-#I7m<6^~%8{(^-QZJgv6`Z^;5uiC;JM@)I^nv5o@1EVh_2P-YK0C54
z>ek0Rc(q<JY@N&8!%1Syj2`;;Uvoy)*8SgFdm;00O>o?|t8X5E-14(4+9Gsm=}qp{
zt!A1k`rBUSY`gZT{r2y8D>knD6TR;jJpEj??RD=L{l4ewT3bH*CB9IPoN1q<z%|=-
zh1kBCN>BL9q@Nyp9VW;Y#PVm)F_DdKo#Dzy3?AQeh<ww*Rnhn6gTWn3)9w7)MVol0
zY)&h9UZdF+>)aOc<6ha)g7C)Nsav{NDtW9opSAnM3Za!-G=q2UYR=!Gyw5;y)>NI$
z4@NsTtx4BiTUhEfX<fFKfSK1u`5^h3H_~Q5)XNZ_<~HS@#=)#98DZ8(=Zkw!I`{eX
z@&#X+mz#LSuICZmt9SQhYFp~&wp5Ev|88sO_fDEz_c1l7PAc{1pQJTAV>VUpTeg3}
zgs;}o8R2(Y3L0n1zi-)5f8*-%03-d7$uoQA%Ls-456qI9sasRnQzC6>E#LF#(H*l*
z4F9L8gzmP@+J1Z$^9s4dnSMv+O}aR-CHv--%!9REp6lBkSH57L?6fgsO2n6>=XJk6
z?F(Tvn>k;SEBM#1gMEwlm}%YonDli1yzQ_2W#Z>X_Ze(;n;0n~5}kfbS@Nxip7|us
z-V|@;oBK{U+fO>#n{xD{hec}w*TImcmEcQ0cuHPP<cU$K7h7rApR!-ca*M|^^+{V_
z@k&25Ih4r)y7mKl4Twao;j2BpR$=ZU$2|CYQiEoOPGel~ireW3qj;%}>r0u%=eBE<
zDtYOO@d?%hhHx$5T)+*6*%uu?FV3hH@sxkaxOheRlwCSqMec#Q3>;EX0*EUII1=u#
zD%=vxZd(ewa)2vg4ReEn8m}Q|g3tk#22X~`49j#FJOw7V&0sO$P7phw(-6tf%5WyM
zQQyQxv5}M0CnCX#Sy(PYvRfm>;9(P|q|b?jLu|rUGd>)A72&e5g;U+fBhiUVIBdp+
zLoMQA$CbEDeRd=|@d+2r*l@T-PVLVIGqFI8q(g$jlXeswX;s#fDSX($?QSMMX~%>k
zttwD%jK!lvf)*v8yH^A;K-JtnacPBXiDD<WXN<?ALxSRcI|^DPgH>)QGdS`%&r)h|
z5))syqu^+(wqDH5m=lkj#Kbq9sb|uNDSX&<d}9ZQyJN!9RvkT%0C#xI4kp>ulsOF<
zOJ2IoI=ABd-V6<{_FxH<<ilZRmr9g<%`cTS?^qzt63nqofal%4iyoIH*>{SA<I+cU
zM%0xF1zHbHnvkN-BmcTQ=W;<GWr;~8!k*?6LB!n&A6$cGhKfxMUCJ<vVU}2z$z7Mr
zz92$UrvoC?1LFCb#3rT%a)BfAI-`R#H`9^>>Pud_9cG=vB*LPR!pI>Nb>Tu2Lss{s
zC9^$tbuQ7Kr^Nuh-$kP#kYO@|8l#UMEY@H#`B{%4Qebo24AuaSghLF%Tr&y|e2Q>c
z*vQH6(~;oBBCIwe;9v`P*fAw8S)U&XP8`BcGYk&32&v84U?moqk$8w(IBAB!;TB1?
zD+UkSwr=cj;uo%)A#kKcL5;_7VF#ysjD^!y*?k359aBSQhNMk(O_gO}3G{g6Bm|0@
zBam1-@ytmG60@*~j<GoOL->c2a|xr!rX2-G4-3rUUQm!KEu|~I2|0e)ESCq`{mX>J
z8zfdgT8ITg;&sOaR_j}ymn2QqjF@<sc7@oPiL|<VKg`%Ld9KP`&r5Sv?jBucooV&B
z=t9_banOA`pls-DHaAee$`O43P6#;W+H!VH2B~vx^{IK?#dx2)DU(In?5^wOSQ8MD
zv+I!jf-_pD83MktDo$FqJ7ZV3Z~4TMomLC4Bu!kV?dMo>vE-<>gNz#^=#q~TU8&4n
z-OEltm0ilPlR<MWtLLtyBxVOtw6TKj|1jC4(BQ~$G`GXI{9=h^r?!HiH)9Xz(vNG5
z4v^dii??9029*v^XCI~+tk4|j#IW(D-mG_vKt$5zr(2|+o|Am~lZ(NLRakGvgo7<S
zYB>fAn>ppz^tp<7%_un3BBHj$AhDHG*QX+}RA*7Z;TCDNCkBe`oYp=s5)bibCkwVJ
zsmT~Bc5=IeGa4wuTyl1GgZ;Q{_l**9&+>^S;-23RLPbDcpFTZq%KTXUR^_D2Pu<&L
z*^<L^SND?N377wx6(8VOFaezX3LkcHhaXvXdN%h`2Dvpq6`nty&u|P|7IZS38y2)T
zA;Bu=xr3SQgeB~gzBo*hJF<-Z(iewWavqnNZ-V$9mzX`P1ZMlWx5Wwgs6ua0y0WOP
zB^aFFz_%w!9W-fDoDI^FS*74ReWtBf@JlUDube6YU;afe9ZhUIeEF4LI;zBm-DgN>
z%5Vf{K2V-ym?hR@V%xEd{o)seSs<$?mdrF*yoR}<O^ii@19n{tL#qyhqcgWulz_us
z8Q(i?ncb6?uuuGAaN|pX^hAEo%gq_P)`TYucuv2XHHF`VLB-KWDhXDcsxh9~&5+`F
zMk-0nL9v08*+(Pc5aa1&!B#dk8w15APC=g;3807sm8c~K51Tm^eK-;iaR`GV5>%qL
za_akZBs%d3r_BIG<h5f;T(&+x5}gEun`RgsX;D%G6`(gju6MbddRZ*>ahuU<9j(_4
z3LW6Ahg8NMS;l^`<Yw>JAW-27it{6&$ZXDl22xIyfG0SRYG-g9RhXrBSY`^O_~Z_6
zo02+v&Cdm(h~x^dap4qQS0U4_ky5DG%^ewYqDj{G_T;z}F;>=_Up!{Xd0x(Ry~Hi2
z)0(Ww63o$-Q>EhDzvSga6Wd9?{VFd_X1-uOH~}OQBIJ6@=Sx$9)<IA~>TG5^X_@<_
zFCMe}dh7H#mVrvrjV}XDY`d1ZU;LtB7HgF;B{Ef#VbwkbmrGwXX6bD;^kOt&G|?5y
z+!eAZP%Cl@(+dcUG6BiUxn5qJQ6(~4FIK~gkxLI0eW0?GBjCb`CI&XfDFOjGRU*Fq
zi@;X+_A9--blmmjV-8SWTruO&l9!j}EEGAQ(xA!hAarDjq8j5FXm)f^Y`F17V;0PK
zIoC^zJ*q@J`<r2AEpy-aGEl{~Yl-{BFBTX|cRiXT(4fF&?sEcECzj2)aJWSt6s4{{
z7D*^2X9tE>5^N!;RJE$#d6$x$J)e<We|^&B-_321*bI-^QFP?K*t!Y6{!3nVn%GWU
zroQQAr;6>wCF+yDsF;EHm%g~nns;KE`lK%|ljeC`ntTJq_qZ&aS*0|4p2sC&&nhKP
z|0FOUY)B4>uL3fp6U+x2y78r_if!i-^@(3hZhTRhHP7Yp<cunz+4Ed3P4=h~^7L=^
z^<VVT)5NxOnL0vgqGmG#pVT5qg9k){>dN3*{+B=|99gD5@r%l&c`lbOCtRMKv5Utm
zNyyhf>GIUa=2{b`+uY<(;7Yo3VG+39GlS&=$MeTb6Icv0J3Jw^C({fTgZE4mSPio~
zK<ynh9)pDqoa{RNt|C%1J{)M_P;)UzY~mF4*^!W{v*^OX7JjuB1H~3jWgm^iL!7IV
z1zW|`_82I(aT@x}NOa;A&Wn)l*7#!Zu$|M<ha>5bfN<9gP)1AwdlT%p9FRAaK*7Lk
z2J+@bES{(m!l!ikZJ$o<hC`wop#gOf6hYuHjZ4>gq-c0ufd5(yLqDhu1g$!!3I0CS
zsx$MrqZ-GX^k2QD=G8gReP+$`ylk9Vr8aw>=Ots$DmBmZNnpP3`6VxB=79JrFK4RQ
zc7ypQwv(5oZvycrFG-*DMaK-pzx2gt*1VI;(kFfKnKaMy(&vQBn=`7!e9td>8JSZh
z=6PP}Wu%HNzgg_L|9ee9Sr&qI#WJCaOhAT%6?HFpesEd(#V<Ou=DA+peB+DGq<OBF
zHhWab<=|3Ua&&i3*P10SF0PxxYFEVNrg}<|!O=%*(ITmd%hET3Ebd;CKJkmq4XBwJ
zRbsOtW{P>9Z}ts8XETLKL&YGmk(1Y_0u;BPHj!G2fnqbKw9ku#L+qfq6;hjHpxDZ(
z=@XIY#4Q{bA=|A1Y7=o<`kY8S#3x)e<HO-w5iX!Mkq@kC3U2=h&IWk{92QBJH+w+B
zqH@E_NE6#`us6+Y!LHMpG>;v(()(_H9O9t15~yig_^=D=mEZk``%*RR+Yi)uL>^O;
z`%?2q*g>%y)Uth~%rv8*HK$6^SAWUNR+GIGeDzgcwyNx%uta<#n7>T?(ibN)5P#Aa
zr%7=~!2DTp9+$aqg7_YnxIL=`XM^~eR*&z0Z%P1FubC{)W_u?r6TeiFw)Wk^h;Ok=
zSC&j=Q8nA!u}u8p7o}MsV<vu4niS`9>2$*7?u;ryU;RZdU301gJ@u7dx~lBufh%1%
z=cESf1=b6%85S%AH;!Octs*z5!PF+k5|9?Ac6X2G{A3}fS$cg*5aYtVZbTeeCO)yG
z^+`9vypp4OZkphZ%d=R9O$uqehTIDT4zw_;i5NU=c)GCz)LWS$aIl3#&BtJ26Q}r^
z9#;`iZ>2>*t;Ha*g;UkX0@O$X^;Xo5fU>Z$&x*uDyr4#soZ24)#STtKABiL<LE*j~
z1ui*IUzzOf@P+uQW6AV`&;W3`+<gNQ0Nox{k`yVeHy8V8kRo~NyrQ9e(6ocI_>1iR
z^}32co5(3Ar(8-Y`ZlxOQH^2K%c&-NCoK!V^u^0;@1!N+lfHOOiaQGC&x-TBtbG&2
z_q?R-StU9f#Lui!^<AHGS=+Np)pNbd%g`JUe+k$S-}Ndlr>g9owB-81W#Jb~TDiKt
zxSEVs`DkrEWyBCt>)5g6WvI#CF5mS^FGE%Kc0rVV(V7+KdU^GYFIto0TraKmsFKa8
z67^lb2%pj_X+~>LY-)@C_I1tkVrZISazq6jrD}{mOfxpY8dRXZ=w^mAMiHhC-|#aP
zG|jmnPi267l5}~sN0q4OdZm}88^E4g7Jl)Imf7B}CE*i6o@Sp-veN%L4eh><HY7`)
zI=``_eMahrXU+4#<uthEd}5jTr7v!??s;66zWK#%(mjt$(w<eqvqAjKDrMjNl*`hd
zRmz_EDldI=K>Q^ydrkIE^vzd!*{ia5;u7<XU_RIoGZ23g$dDsoKFHA7H%gXjxj<US
z!oK;7UizBs?exu8dg-gOw{yw%gUie>eo>os&*k#d%~GNT0!zUy<h`BCw7>FuL3%oj
zU#u81G8`M47+4vDSU4wrQJZwn<<jkh%d<18z$SVbu3=$wFFSGrMbtNESJNdo)tMc>
zDtmd&KqiVvdx7GY#eg&69;*U((lvq23|x#p5(!R>!h8{O-5Mqa4;wiJeL4~ju?TC;
zC^&d3!UfdN@!?2x;t+P55pbwQ7&O8F>gPD|2q(=jINTzob_X=V0P5!mH-WmVifST;
z4?DQskH9@P`v%0tvpuS$b099p6$WG|{k?x<htsso$r=sqpMn|qHM>#zKJ~}Fr&e13
zU$tI)+B;_cd+$q=gM^wzRCda#{rsvpJI^j@v-Kf<7PEk3g->soJzZj|de&s=Qj?{b
zCaFAk9At%=A1|JAg?owmNm;d%vTl(RW^7x|;c!trRCE>3G%KMVkrze#<~`a{$SUo}
z+4(Plf8zP71zgVqXTCUgQ~qJlxfLf}Q<H6Vug?4N)$gaxB2x>0u`TCw*p8<(<*-Cl
z%nGsE7^WAzbgj){L!&v#4et%Oy8dY|b)9P7bLd}{Ml_H4j9=IHJ81uTBqdj$$8wcp
zmEe_Y7Ah$>IKRlJz0?wYnA@*4*~{s+z&l+HHpd134ZMUMAh4S$p(!I0(&yU6u<LB!
zuP?LjwY@zqWi6p<Gc`5x+tc|gzb{CXo%qJ&{DMH%UagD}$HiQ}42u-pc(u6_L_oKG
zGAw2|X2ig;D1!IczXk6Nyg>=)QB1^{{aOF&-t1z&x%uu39iQ+-3l4XiCDL<SUMl&p
zC+=duY&m1y!2%Aq*Ft$ET+6%-WSczAUhw#YC;WK7sMPhkXs@<ZxXHSVn2S5a%p6{K
zt&^5AH_6Mmb79BALhp<V7j`Xt-F9rP`8)M@ecD(04woGd<Zi#$(cZFOvYz{2*dNzq
z=j6jnexKa+IOcBQs&^K_p0edJ6UzkV$-2KhXV7AO?xO7S7k$Pd>+haC)Vn=+XP{`M
zh0Wfr{G8VD^RFk@zI$qZqQ3w7K@B6Gs{8D}c5Sw|PrE1j@42Ye;^K9U)*}94Mrz;B
zZ@=(eUiUS#z=MuY-#vMz{=T;Grv|TvvctW`AG%*Rr@Gbuox1vRyU*tdSK4__H~q10
z|F!PJcC9bd{r(od)8G4LI^W%kbK4JHUs})m(e&wzDM?rV=}Ig+c4`mjR%s#5rin^9
zNB8a$_Yi20@A0)Nzg2Jahh=+!vYyn$k8us(72=n<Io@Ra7@VRdk$U5i{sD95Ly<ws
zdD(~endRgJN9morYfyRkjqhdibN4qew22z%H5T&nbSI=pZ{;~wYM|3%ti&sRsPywe
zCkwGdTLtpEc-!W`kk?+MvZ|%EKs6(9@dArIKRmDf67qf#))l`&U|nm$%=$#dIoq5!
zDNPh)6g#vLB&4}`gMgoJTgok=%p}DV_PfOnm6{%NV$pGV;KZWmasb4-0B3>3SoZTp
zbS!A)i0No(O;9*=0K_^_1{dOp=l}~ftYa2a%1~qyS4vQ15>qOGv!G&1Von#cMAoPz
zl?ieSZWSo1>W}E?Xr1tWO^mv-m{OABA~7dD3qfwdjRKpVOjj&F;^d;^a>eO{f<cm^
zl9*GOtC4mmNJB>y$4ZEXB~K?zum(vgWhtuYbVhWvw0c;T>=5AOp1484lY1hR)d^>z
ziFKwMoqt@Hr9O*8J*wkT`|qXu|2tT3b}?RjCGz&~wKBdL>n5yN>o@nVuc+^e&t4&w
zOU|_|JEkTwSIz5OP<p2EkF!2?rjrxT)`on~_RODjvMcS=q8+PKRBBI`sGt9$cI>Mv
zyV=6<q-DWU>rDPe|LK>1ag#ysXWel}VL$ugUps!J_}CZ!ntAzntOkfWKc`MEGVh~v
zy4(g$BXid_H|2W;^}FKd8`UhnFvq)5JM^Fh_w+>%_?jiyIb6gRetLT+Dlw!_a88f?
z#nt-{ytbAXJiWnuH^X58aYlRAe<B}*_x5;&Y(8;+Lv8gozT`>IC%Ny~Tz#?r)Mt&i
z%Fm=k*Inf3VC#7ip6z^tm7!5h=HMlzFBgCG?%FA6x%QEITqt+yo~0`zU+)i9@Kb#h
zqF2fK=uZCRsX~)E%ne;8z3(@0UOQ!x%dWn+7b>N7f`6N6F>Q8S6@2njS)lj3Nz=S4
zz5loRSt=JvPg{9v)&8QnZWh~1ehYp6xb6c8DvS0-EEJitfbYnQKcZ`Xto--!tl0OD
zg&~b0KknTBqt);I`LBl3(<N%%Ul0A*beT7G=8`$xvKCf1*LWxYGuOT%YPNiiee6F!
zOKJNRY0`}EZqG=NPFXK|{h1ll&21Y`<u<(g^dz@t%j46J_r7`fLnQrZNzKfD|EE7f
z(vL2dwtklAu3vWM(3=;>9({cI;^?O}6(z}Ef0TTARBTjN^0;`i?A{-9dM-P5b7#As
z39-I3_uF31zCM2U^oKKoEPl=EEPCr<%MiG=VzS;kzZ=f63429r_k7;^>&n^EFB|3e
z{5g@(`739~w7VJy!}u?3e0lm^%Z#s<5Bo|Mo9w<{lP8kF{`I1-%AprazVf#BbLQCC
zM?F9Fv+$g5WAVf7$A5br|GletMRZVp`8<=%J)L{jPKzr2T$B1U;#kcY9r@KWo@<|W
z_y1WD`?Dgp^4<QLlHk7Sp?z_u<^)*o(mY;s>Z13|-|J=WPD%bhb@6m7_2B-zSB5jI
zz63o_;WKO9R90tIwX)cV-FDHNx%2zt)I2vdtKD!Ed9#4a>|~Q{$w9tD3GAsKJSG1u
z;He8>v)5?$KlOk;^+U7K4@b$GAdrCkl!N@I9!v%aKordf3qTYtf#`C+&Zv_2;@qjc
zn_ndRcKoS0^IU;vu3P!xfPZ|+iZwiE_O|x>>?~b8Pi&>_2T8_)A0KCgTBKd|be0a~
zexmxL;>>FcNr8sq8k=XO0&VMR<mW&0?s9kkI3vVHP4v09lp5FH&kMEB3kmiXb9ALy
zRRnnnzBs`D_vuIBH5Ci0r|G|_RnGmJr<4D`dB!$_BMO}5TdZe%oUp)I+P^?wn!m(D
z!rvrO)_+6Mr;ia8D|*!rA2gdMaUs`6QSE}^v_l&NA9wa0d)&D$^D*Z=?&Aeqya~sJ
z*N0ZEviW^XIH^|d@Jb$b^LcH1IQe5FY%>zS9BIASEav7PC1Km!`Y`$Op%<Rq!pEwQ
z9eVNd1kV)#XP0Y?ckiUx%PIVq;%C*2Jr=w_RgT}H^h?9SbM3#gmRQIa?zEXFp!=$#
zN#|R|uJy-d!~5Jh?({9cbbi}o&hPIZzuc#k*>|jBifq40n}wWY*t?1&XRkgtX3mu9
z|MJRK?xk+I&9svbVm6yTp6t8t>8!Pq^B<(y{kH#T+ULE;Q%EGo?qEyCp7a}SZ;D?s
zZ9Df-`S24}LqCz0Wk1hk$Q)O;-O=xE^;fq4`_KEGXFqnt_btE18vU5F=6{UG39Z_`
zjocc~-@VFT;3OxlF)`z5#SZ-dp5wwSeM)WtKd1i;QJfHV^jPMriI)z2Oc0P-9htOZ
z?_)!rDshQsvs)i0D0p5y{Bcp5o8U5DTfS_L5aGF{YWJD*^L|a|`xGtd_t0VDu?SWf
z{?GeniLGMIx_`3KEc4D)*+Z5aZdqCF>MMFIqwW_T_s&j3@JK`RR=X2yO}4jhczo7#
zik_H!+Aqy3^ZGZ(&Hd}&=da2-Al$C<H)Olp=7MErm-swAV?U;9U;oLWCic8yPF!sA
zdAZ{DoxKWjZ{ib@rq*6r|3KS&w)BB%mFJbOXPjMp!2I*GMH?Co!o7^7+jMj<F8L6_
z8ddx7^~`C971rE;d!v_GW0Us#$kmrkR((ABMCrhN8R;cP{}=Wu%sPI0$&IsYp$Qx+
z0y{1zv0X@!V4ZvJ_c<lk__`0N-+tSf>y&yt`<STi6uZuMzVO=z$E3vLN=ur)*-cx%
z=bOl-jbBf$x{|!#*YElc$wzlQk4qIW9^1`1;p%e9f-f&LE(RtaaJ73DBKOMi_(Jnr
z|8r&C`)a37obpJ*^?B&`zY!ll-{qWk^!RDZ7KOihjW4aP#7W-T?AOZtXh-?$@)%*k
zbF$oGkrU;T-R4FvlRi@0%&t5;R5Oc@<Eva{20L5Ld*?fjkJ?-f9n18%pEa%GTKQ*g
z0BcLrauX{9xjoB2UI;Y%S@CL0SM!VduN$4*7oE5LQZ2{NW3igO&FTuXbI41@i%-1$
zCDj+oJ<twYsJUnV(t?_I%hRQ{N=@ncvoz81%gih$Tb3oPSD2HtUN$mnd~tZ6V_TBP
z)uNyvQd|GKD&vMv&4FbZzdyDdxcsxC<=1vAd5O3}n~zWA{5!2CeDtXQ@bQZ0jauH1
zKlg<lfBxI6#b%zBeCo#vqc>|m9=WjnvF3%`R2!#>_fAUBlZajRX^u&omxTW(6WM-I
z@!pRvyFxau{IxUSpu}UwEqD6d-QLOiGu|$<F<f%3!ii^2a*p~puW36S*<v{VRV7L^
zPJ1kPtg+Uh;=r2Y%uW&g?Q%Ct{<XX)f86-7=$P<hv7Y5kzW0CZV-{_Eyt0(*ayYLI
zul}W9x><{R+_ldA+$U;QY-2d*W`$Qu#4+V5=X>=hy|1<ji7?G*wQ`>N`3G;<Zg=Ad
z!Q+oR5B43Oyh)_r{Z6Qd`LB=XrZgn2_{ApgU%)Z1zocKvKR3p{S4}L@?x4#qx&Cuy
zRa^^xKIW{tdu;2`PLXuqh_ashUenuq*){GSW3F-HQ}}zXt6Er&Bh5zhoY?)!M4p&C
zi+MKb$ml&boY*hbAMoR$Q$~XR_S%3%i^3Z<S=&nlKI=V+l6&|1Qs~#mg1etqJUUxu
z<GM-t@kVz6?azYJ-z&5><X@CqSt4j#H|6-5ioD%DHD6AAl<$9XxZiz_xp(dbi90pC
zGMoO0e>ieb-d;?$@bS!F-t)377Tpe%e17-G+WC_NM0(xZL*7*IY`F0-<zUO>_(#e=
ze*IkiqwAx#?WTVB9Hl!IyhdiTYG2>Xm}oc8CFuC#<}TU(1xHSAu2}SO#s^UrCAL=+
zB^4z7S^Z8NSKgc1?|#?n%L|bkvi-I8fisdE3@_~XC~+mE|D%FLg^x*--Mr0Vk^Mz4
zWco|2ZRDgQyFSVsuyg22eh^rg^P2OT>G5FuL*IX%xX^oiv4o$oUD|>(A0tXmN}vBa
zjg9?kYYn^g)u%OcWo7-{{GUFaIq`Y_@w$1z8&V!9J*x4jF@G$mxV&$<({DE0Mz)?_
zbDK5C*58d$^WE`@rSZ!tvH!eE*|(4D&%C62xbCfArd`#GB^Pgf3|=VwzCvZS<-8Y8
zOP4#%d{g1n)&IBRi?nkqi}XUl<N6z>NZFTWHpu&LXq4h--R4o%aQRQgg!c>NUhgRQ
zbTZG(FM{LKGvP-$!B>Qjls<dhsrdO=@Uz%Z|AeBnt6ZOVn^c^4ni<IO`evu?%q_pu
z-uyay?t6js^eP*cqRzSv^JO2_e3&0Hf70fb*!jP*L_A+JUYhvv!V~p}?A)(Duee`n
zB5!Zy5%S)60_R<;MY892Op(czyz%yjVe!nEKWWDon`vK`I9|Ex{gn7uCs(vv%C$?r
zs3^-@`;p`4#C4vv+=r*PFW;ou_3$H)=<S1Rr%CVJ<xnS5XH(U5Im>e1+^p%3CstJ2
z99rTLuE!BLL958d{a)e@zP^k($1k6na*EMo|CPqWjX9b^J6^lra_N(2-hTcgM-5j@
zzp~F#xy0UEJC``zD+ylpL*dz#TsxzXGao&||32QRnZ13U@>vb`(-vYL_Eysy(#_ro
z>^+k%#l_g7yRql1NAWF<IPs?6Pj?=2DK1)N;ja{KchM`@zAX5l!S*U8WroMKMyoAO
zKT`b5U7W4n5_seS)6c7QyN)!+AN?qyzW<hIwT;BJ-iDuZRvK8zRW6a~7kb%q+(G>R
z<DJp*ufy*B42zzZ;b}XM>#4-!iPvf#uRPck;LrN~`{R}O58tjSk?MM>`QV7Ge<1e?
z_XoE=ZRYF#7}4SSc;ZH`KKF`i%O39ximtGkRA?nvxJ!ObdV)&Z<Lim{*B*Cq%d&A|
zRJ-VzF6vl$JlR%8n|bFAi<7KNSr=z5zj<NG%Dm>p&mXhRCs@j5*j3nct(NPTeE+o~
zWx}Gz8?&Nc37XX%KR>s1s&KUe|Lp1NK}$cfJau0t$XI#f;U<|Ahuy3<Z|I2>J#i>c
zr1@o}m1vQR^?JD~BegU+f3~fcK6)e{{V1ZLbv5|O$yMsQK}x%?nE%TAyz$N>$NyHJ
zjvCD0mC$_Xj>9ukxssD}Ija7gSZp>?x#+2h#k`#^$CDpToOt3y`MKTl&W{B*9ynB!
z#xwQt#*fFAxSt4fbU(2xZ_2GtOAnbYI&-|Bo4s>!ww{{ciQR{m{;java@74%(FF14
zV;U1u?;a5BY8JV@T>WEymC*L3>P!4Jk1H=d@$f<9!5cC<+7VNlQ|}%K@R=7H^(^7?
z=h%MEcPft;KE6?7C2>v3G@GIO#<3Zv7N+nf2;7zkunbvqu#ekRep!?01lv2uFNH|I
zy6W()K%P6e^y8%iSvF}3vJ78S8vop9+|zjF!@`3<=KNY>t$Ok5-!)fv79DEjmDAnS
zZ-4u+@m;CJGN<%EOS7IQ$*;EC#uDwq*Y@4=M~1?=&a9Hx6;|El{2P8V?D7$}5%bWP
zd&+tFMH{7z7}3VrTWY=TZkhMvh{m~>MLN@ie@Ba2YpLwnmahFIKC|nm{fZASws&x~
zZ+-OXaN_6B+RHzl*S=x$<yW%Am)LXtp@+6N9QeVJ=Gy-5<x>wn-+S?K6FBaR{C79_
z7c3g|<G8}VIMy9Jd)G{iKcuj<`Pth2RyVxzE*z14*2kgTFoAJHw4KPd=<2s_|E<NI
z#;-8oUHRNxUhnVsT{EX$tIyXf+qCR|yv~fsf5#2xr9FJU{KNO>N&hTzZnf1X9(q_9
z{^v92|B%*9`R3&(_J6!x@b9vyj-BGx*tV8Ola}_!*)vuCdl<1y*Vg)!eA$=xYfqY7
z*q_j}tngpuhNrg*-|tRd82iuU$bTadwgZ7XX4j|3&SSqVXStw$;pEFI4+RZ+-`lk~
zZT{aYFMt1>$%p@&FRm&3?{Bkgzl~6x)YX2jFGe%YvHV|LSTgbA`L)NsH2pA^h}HW#
zHSdt9?vG|$F}a_M?c@71mv8rF?+R}HqH6I!sMWM=-e1L@{P2$ZTzOyLhiKfFbKd>o
z*&pXDDHg7K)z?E`J$^kwUVh!j`!XH%`I};Q{omELhq3(E|5C%ZAAbH;_x@zR=7Q^?
zH50#y3;(TC&)jn<q^>~s)79C(H`^6enD+A=e0!kHXxr2H8D8O!dHHwzKUwuRq;*|o
zlhm(biP8sQvfB6lJ?i>j*gEZhdviwr5^=5_^%uB(p4qM5^v?Okk$pD=Lkgciy!CkA
zzv%55|Eopb*asf`^Xcf!dKZ&tMXdj`PnRCuU(bK&s_O6a(_TAo<Wy3w-O7DJS)t|c
zJBfK(H=djB6O}#uZ{~&j65GGBE%Wvg4PJ0dT4$Nyd%lTpm+1f8VBEp>_dHA3Kd+RH
zwkO&26JzczQ|sBU-r?|r%R{}8v3a{_%Y*>d+#Pw26WB%VL!5V5InH6fZ*Zs1%wn4J
zF82JQJIxBeo2=C9JXd`PVf?<#Ii~DRvRTK;ON)PfU{R}8F>dmIYPM+cTlWi_^L3s?
z&inST=aaT0!vmX>^FLqRvrf{Y)g^k>f#1gVdHsJLewzOE;flbxh3|xUv}*jk)_k&P
zR~NkZ=K1YUG8X%swr+7;6du3+UAVbSO4HYV7PF3r8w0m*_1k23?sawa{yNV&pHwFP
z`S&vDdhvq4l09GTXEgoc?fam-=I1Yi+uK?0zP+UK@U!2)zJ^~HI<_63{-1xxkIAkF
z+h<+V|FHMBe19_A<XvKIZ|c`Y*Mu#tD>aj6Kle|nr$6p5WAd@1zZRToceh(zQ}V99
z_F|lNyZP@4%YSc}EqUTKziZlmhrKqBTWbqf#BYzCnRtBPUZpo$|37xwgwD8lZXfI6
zxY?p-7bwNcpNwq&_gD0zy}ja@{|Dths&{i6e6N3a_l9@bf7PCJ>Bjbh|8`m(wc8`N
zC*u0I$f?is^n6dJz1jV`@6cuLKR2E+R(9+&n57@O!TYTsujs#HW*qfv=GR9|RK9j<
z@~l_Y!fZW1mQFn5AV0q>d3`UluHC*f_AX!S4MaBnP&GTFS6Xua<+J{d56nI97iatt
zaeb@*Lh}4I&H6qO;{#{R%U>_$JoDMEde`6olNT;Ku4Vk{()5KJZrU3^-2CPFxBWT)
zi~4VTKO&v;Y5KaNZwpzT2p!wpc*%9D{e33mq!;GlPd+V~VYq#t{UpxEFSYD>7wzMD
zx~XYuxV?7pjrL%@o|2e<Ck!s0TAuPJ=k8Z_?(ZrNzh15SV<?%v;`;t0mxUjk=Dr#D
zbm^4Wj~8A$f4|_V^t$@<U#74X*RvgtYkB$qoy8kZhVs5ezx!X9{QuQ+^flM9v;ECG
z<M;2&H9aYG;${84bMo$e?zQ`69rBddKV0X(&qnd}lZqP?YCiqYT+g3cyJERpZe><>
z_}OdiTW`xXWJm_ZFgi6ak2K(XloLAbq|xL@Kh7TB_itL*kGuU@c27$uRnAS(TkiIH
z!6&PeoBTGdne4J;^Uupx9h>?VE(u;-Dv&Dc5_vq}nqf|<tV=9=puhf)Edsjdmwx@c
zX#IhiZ$D)mT{8W1?%eH{U;4IH<jwH2e|Gt@Rp2xExxC-ECPv@hdbv#dd06^{<?p^-
zUh(}i%lFcgGY)-u@yMk0?-#yXnVjD*d%bz>r66+dQoK$6|KRr*{gN&9SIl)we!l9t
zXG`(n%a?Ba^qq5{<O<{5MCT)GA5C)%kT-aC+@Vk~u+I2Sv-MNYp5L7jeWlkteIKse
zs6RQ+NzXnlW_80!4i>9d2cO!=RP^pi_xyX{>z%E$H|rJfO-^bm-rH7nX4ztAt;yRS
zd|ps?@tD9jt*Gv>0=>R7GjC2b@DmC-T4H@aN$2&hvp0>zp492o2X9W6y)2WY!!6Tt
z!tz-Dr-|<MrX7jto1YzTs*0&s`j&O>lw28)(Yoe0jIw>|{n??x5*&Jau9kM(eOsm%
zE3xcI&8MfFh9{0G%zXcChUBCjy}Atde{DQ35TCKfFJ<Cscjd_s4L->RRy-(?Tz-Qo
za>kvlNz?k*8_rKj{(tkL#Kho5^4$T`zF!LtmUlnXmf`&Rq_ui8$MnaYhDM9p+7?JI
z7ELPZ<IOqE&!Mm}C;G_|>$b!y(KV9O=1a(X$uKZamRYZP``5a=CjT89GaS$UeRsNQ
zip6%Bq$C;XCGE-lcE?{F_D&awFVyz9EB9<tnyulD-y1Up=P9qetW;#QT;ka>-CO5b
zo}ccY_WI82rE=@5l{ap>87%cX;X+lsME~c=lZI<*t{v<W)I9&+Y;TV9v8j_cNY{0D
zv8-FMEqdC*0J&-Ij5Ci*Jn(GH>HT<H;4+6u?@C$a2(f3Ey$z<mIQK_6Mxp8^WB%*s
zvB@c#bGEe2IOe#qtlS}AdBUwet23*&D_)jxJ3FT`{eVnrSc_%vmpZq*uk#K{9-lYi
z<KuM#h09`^5BfVFdU>lkljnqWacY$1t->qGyg&GVzIeC0<6+sY+&|7U)_y9gi(EYY
zcKt%phux|2w#RI&?@NBL{qV2&h1o@M=EHZl8-Jdv^5p*R%|fEbbZpg~>Z3FFY^$<9
zd2e5J<j)OG?r$Wsm^IteRMh|et}^%dYdvdj))!t&GXwjQ%Hu!9SgrSbZ8CfN>6f~_
zUGBpl68-)!e@sa~T2!;<_;Rzv^Lb{oWpBPYndUDx{nTdT*?VukIr}Q8vh2lX=XkYk
zXU?Rz+gX3PIbVPC=bH=9^VNP6pT^mCGjWD&+s*h0HHozL#4?Gr`ox(DGh~n5Oq}!g
zXlheoW%=gA7QeesvpjxdaE|Zs8+~12pUsC&tbI1`-{NC%mhbW#gR}oG867mSKC}6-
z&F|BvSw6opIM4TalkIfQXEzh)$v(RoAE_3R)}C4xkyf8NGhwFew3~@D|4vP9N-c{>
zZ!i58KArR0&BVE~*KWqgszr$N%mJ~Y)o!G<=a$__tIwU8Fk5!p&BWP%x287by8de2
z@rL`_`_nS*H`(huPCsY;{C$sE#Dx0wr=Kr<ZoD^gSDfZKuOdH_bX6<utC1(qb-vSS
zIWBf8dEca6rI)xrZfyN^!>KN3sr@#A1$Dm_q}!vo&z}BVeeCTn*&{E1C|-H`nJFW=
zFWE8IhQ)>N*>`P2uhgH3itgJkC`WTnIy&{3zKOY?!x=f-74I&YPPUc*QY`uER>DjT
zrK!5ShcXJ3{9Zp1`;@|6yJ@P!O{ax9ih<iMXhe5T(G6L(dFkfed*pIo?~e1Bs=MWs
zSklweVs~>C!?#_~*3Q2Enl0XA+SIQRvE?_dUbp7lblRSy7_%)%FWO_e?v~SHkDi{M
zxj8bn|4Zs6ZpO`1n<6(~mbqlqzU7SK+G3B>Xv~j0kDq#baY59)%O0mqE*bT1@ljrT
zd4~OuPn9qC9Nl{GQuUqb=eO0wExWi>;B)(rtQY;pKYu*WH@o-TOX17?jos6yzS#c%
zXS4qN=^vwGq;<3tD}TmJJ71c+q2%r9OFu7dUhn<f`}#hex%#%%and@mzl}2EtN30o
zIee)!LQQtsyu(&6`Ni*r9jaKgg#W9I5$}S3>4N7SZvM{_*!n+StL~)khyNa|Ivjp%
z8#<nB-f~vyao&S1Pv#{@@U-bJDY(qe-1uXQh;2{GlKCaQKUzNIy)D|qKK)?*0@;>w
z1r_h+*_)g#zUZ%46A|XIJ{RM}V|U)+DA$G^mVb8HE{)&BEBIjJ=NQ|%qWu!TvK4F}
zUpRjE%ZI)8W>fb^FXTQg*)+#w_9D-$Y#E8d+=$QF>R+t26x5hM&2S34`TyI6UA7h(
zuN2n@^UF@mv*47@T70(cO1k-r3wg(;aAhq2e^JP=TJSbw<)jO|QP#U%l2r<v+nycy
zv8&_91O;8AXBmDRALa<KL~YzVGiB;ev6*fHP75XqicDN~xmR+r;A9C&C4sD)69O;H
zc(lQOf$1hrC#6{v{;3<jS8lS=E>w0gYx`pPS6@n9PfSX=R8mPvMdhaVgd<;4U0(Mb
zlk7E?N;f$dljSERIA`J#Q%B$V3+DR=Ojfz@oW-}MsZYZ(?~?qFWfvq0r%Jw^nJF&u
zT5{GGmckBIwXg+QU7aEcA`@2}Y5ZK<^wT+MQ>?0`OHfNo3WorLbfw~xmxf$j!m8($
zcP?sj*FK@Q^nB-X??pCNo_m#g&4Z8MI^pQiXg;|?;&T0s6DyW$TnJK8Kef?)MaC<Y
zNmnMDr`hZY3@%B&XX7RC;l#s&-;&?w7yf5--|^SZ)i_mSdZ@PbleXj}*Lfbwp7*W?
zT;Fk+W3qJ*<Dq<><+aIjHky-Xe$@Ne<GaVa-!mmqEwNy+WcjS8mX43ZjNh3)PD|dV
zlKLdM<>wNO+abQHalF5kzt1VWELkr5b+=FPlo{2<CE>0r76R{V9$z!g+gm@APs;F3
zy;H8e6x+oQ@4qIgOxgeA&CexFlLP-n@4b-p=(^9h`Jo1r(v@q?gff&{@Aa76@>tRv
za%<MPWVsW+8#6;Rv>)a!`rBEwSas96PN}~~cuYB&ID*%;+uzJzcQ}5z{mJ!{|K#bh
zHK^P;$z-v88+)NZisA`2i<=BfULIC<_qp`u3G19AH$CUfJrX%{VMgWziAa0HC3TbD
zvP@1<UG`$htv^?HI($|-5$p5wfY9a13X4w)udC1IuecNtT+@;)BYb)7yoHkKDRTtR
zF22z=c`@UKvpZJh95PLv6@P=vw(R)U|BEEEG~|7&<L>;JHskKkJF(~Qe9^uAExA$U
ztYzrp${X|dw*Njd^PWhL`j@^77EYE7*_<oH7e=&rO+4kYLU5r&%PWChbt~8xsJFCD
z*yWhPx{$v`Z9|!w@G1Khe;0gMyo%q;ZpHTnH&)Nx*`miEW%J_wik;2p_-<Lfc$t;1
zwQyE*5Z@~ctw#%*nm_SonLnv)UB+u=mT`SWc(WVtE|ZLlS?Nj(Et*g9oHAN<a)Cl~
z7LV6PuGfv{teu$I?uu`XyHXx_*;O{2^|HrSG0~`yRL@S=ucD&iA(5d5-B*GF?Oo4K
zDG9pb7Rc|)8^Su-W2x{}-w+eeqb^T{u6l)Ncy5SW_kXW%+C4Sna5ZDe+57ex&D{CT
z;)k}H@jquZ<GB3v{gci%PX5d-@bvNJ+evKl_r;9!yJv1cdggXgTHU{1G2<<5A_sqM
z*G+pTEVXy8PTDH@xnEbGT5<XPH}}7KZWeVKnS8(F4QDv^9bbAd(LngJQR`CnqpnX)
z!j%`62EI0Gy&89H`ug~pr*%wT$w!_pS$HSr%KsHxeIl2Ah)tepV4f(xMJC^Jjs717
z`Def5%o1iWbCj;MG54LbS|+`{@aBcWGKuu|)Bn0ESK3(jfNE3k)6u@053@F30@bE5
zYnNYoV{`Ugux0MgkCy%lmD^6AUhp~R1^3KTr9Z1|lb`3Dn&UfjvgYYmKM$$>Jf!~f
zkjBqbnm-R|P2DMydhSqo+)44Y%!OvDe(QBk2Tjw|y%RLkc-86Ok-d9Y>FQ3uTs!yG
zA%mZX4ELH9Y`huyfBXKm(f{@&Er0!AJGlM7OY8pRyN?py?6zPKxw&9%?W-->hZX7;
z9H09r+WBxr^R<2_w^I`T52Y-OU3_WvefGqJq9?XLUbeP-YaM+e5_`WY>f*a49ajz~
ztlV?iY{!nEJx=VF?Lx;C{mL|W=d$b8a>utD{XF)djE8M;A<yw?5f{ZbI?5?|+F#}9
z-?*$^ROruVE?aBsO?STe%Kfqw`6a9UV3$zc^@#QVUYk$SJO0L;X~lMC-(52mWzKar
zUJ9Ob_>_L_@yF6a2FLfMCqLIsy1!{-yy@fhGw;-|Y+Ad_USGF<oBQSY{krwr(y2wA
z*R9{?eR;lq#J`;B%k1Mf{JE*;TfZ;i=V$E=A!)XMH|JEPTv9&!Y-#BEZ#maKv#y@^
z_R^Z?++nu2H%Fd#53~LM>5}L1f0_c3vN?ZhR`%P^mdu%6#<uxgR>FrXyXX7vOYv*S
zvfKK~&**XeZIe`q#S?x0cJ=;!%XJ~*^s<-X{9g`p@tgU&&z<I9wQ;`ui~Fk;CLgh9
zFG`i%K7%bz$~N|vP409z|8vY<W&YFC{BMiDnWx8F*S56(b4t-XpFICEljBEXYc3^B
z;RtcNbl9ouzzK!i{$+vco0s?V&OImQCv$sC=C6ON>kX23Zc4f-cP8S34qx(&_xVCM
z@7TSS)H}s4^6`fK>@`L9Z_mwJyY^hQQQV}r@AV&ZUv}xUuCrMG{Hy1Av*^nYN+&;j
zF#Y(Og)stJ@9v}*uYK?Tw_iKqb*}vTy*s~dD>3c$x3-Rs+rRe1zMuP4^6jqvR9CBz
zJ-PZ{-s{8O9&ESwJl`MxUg%oQpWSQu=j*HY{67{yzkZ|XPjSPeEB4j=;1>G%=R3DJ
zM+C>G_piM+oZJ1wd)e!k@1HrS{%xK5_{Z(*i*8SSxFK<R!Mt_bKbq^-fBv*xuHKxT
z>+|}985a^vE&eTgs;|o!D3QZ?{Qj>;`sLhQ?Q0)ijgS6!NNB>*pZOIZ%{MvP$Q`@?
z<xzfGbB#^U{?wn*)A#rC8nDl>>D&M9k^b|BM}N%s{5Wm7z~cW|{eNO+8pjX+h>rjB
zRA!AV|1a*}r&sTv!`$`x{)b2TNzElTJ^P;?jnA_adA|ScuY3ig1qBie{rjJOdf&d}
zX=joT_hW75$8S#^`f=(&#qLc8|G2W*(1i9G8Qfc^)3ctZXS+6Bs<rXsw^N6HOhu<`
z%;F*>_pj~0f9?4FjdptWVjJ}TO!!(TS9Zu<mUV~av{joIA78%xqoLit-v0Z?j^{sq
zy#3?HKYyfq`{P?*N3XoFy|-UH=*!=N+SM1==p<V#xhnKEp8t9}cjm9U9s5{U{*KK#
z_t166zT*`U@rI6dS)UJa-MntQBVdiy+0*|Ia8x?ps=v>1^Kn+l9)S&;d>-z0JF!n@
z*T;s<5_jJV|M+uBOe&3A@beD7R8tpu`P!w4b9HUmj9#~Js_G`s5*1*7S{88mqv841
z6S<9hF03=k{_V7@Mm4rhDYlrw@~q$V(8jX|8;X@;R;$)UrSFpYR9sd3;q<C0tX~cp
zuVu+@%qhNRW*x6PF*ovAu4KZDfJ;WpA`?#+pURDV{&UjlO=|l-_@>T0e8}SGDa*BO
zCw9p2ekjhIz1Dl*ff(M2)BmzR?*8d@IydO|z6tS$CsIvL$7k~Vxh(qkaLlp&Vda1H
zLw}g-PTXC|m$M;3?Wp!%GjoxazZR?~HL_dw?l+H=xE)Zx_KA21+yB;e3#8v&k9wbT
zoxSntqvRVqmrH5ZhCP12?$rK>g$kwuYnGI|EZ@wf{Ze46#IyVe&;RpRC-@&*c{}{k
zCEnzN`p1_}p7`MU<4e6eOfvpX-BHrz{`u>U5~K4!s$#6NW^P}9eCcA}1K+;xDA}a_
zrz*y3rbwNw-drEf{c_X$*c@(Oe|*VwW^<jn-rSzx?fJ)-PA+`)?d^^dBfq=*1gF0X
z-LU@Xy_}6xZl(xStdg9+Mq=JV-@{XmN~BGWeYj?;+mgfP>X)zP_MR%{b6=iO>0h?`
z_ST21&KTNlh`wvb<G#G01|;Nt`0C{>lcTy*CPrFHx15_0HUE<1%n1w6r5@3c{4(!W
z!SgeRZ|d`E%B?vf=k)62lCRm#bJBwRmnAE$@VEAuwkBQp#Il0;=f0OyQeLl?oPYjo
z9h0}ry_A{1|0Jadnukwao3dY<+cD_Fhc9{h;R!cb{c5TLeV>(o4e>SWGJ3E(YlUFZ
zwew<DhZziQW^d)a?Q~RSa_%CXe<EwI#?M~HyZh*Rsc9#x-W=2@jxd|Imi_s&DG#3*
z-`pAFDe3X6_2-qHx3YFtMCMAApGu2nueo&l!nEHv*7a}x7rm`k;)==2Nt!q3AN)C8
zJ4Slr!|9WGZ-1L9C|zf9b3^X>-W&T&&K+%gdwA!Gw%pxDN!3P-e;%#+aap>0L*nmy
zpZ0J0bR+KA#Oe*CQky>Au<KL)p7V5Lt&ID-o2Qb0TlC%8JoWguN|~M8TaVv9^!T)x
z-0I499h(C8H}=yPR{U!b`WxALE|S0g^}eGrJsZ3qvK*9J^>ekMO{wU9>6824o7so|
zJf!sWOVY$cFYeXI2t?m`-k%<L>mzI4kK;=WC%SXqFRPi{e!A$c`(oQH*-F=|XWI65
zo;NL9yW$I9{l8cHVh)N<c{Zc?uq^NL>enWbrne<hq%UVbj@WbgMPTNdV(Hb|UoAJ*
zNz1?e@u@<-+QRN=q@irQT$^jPrQOxYk1w7HwDFr-rXE<lNcL`ojh*b?h<T#@d28h6
z_r=xt9qZ4l$y>KreE#H*i)Ux5mL~9pC_heKEXb^P`(bnW_t>wTTmAP3b$)CX)%~x1
z=-2<W){2kGQQ9VdnUxql8yIffkuF*I(CW@h{W))zizaMiE8g;>B5&?grJu?g|7&Cl
z1?DXe*yKL7elq`$7w#IDxBc-FoOGE>WjnvZ-&N-SziWNbJe<MWreUCW^VRW3ZTWNW
z|Cc!Qc$$?@d2Z#7U*8j37js7+uq!KGB_JPVe>UmQ1Uc=PiSMU3-^q`kbM#+e>)*v(
z`%`)J!#|xradrC6Kga*SN*C2rjJeolf7t)Gu|V4Vt2^GDv0qjAVg7vyHi5@oNj1#J
zu1f#6;@DmF^?#4woL@gOe*fnd-M3XB;s3#(`2r_?a2=d~gDLi1-)mKo%EX^{L=y}c
z$}>-YVUV0)cbe(7*_(oe+_C~~A0<uG7MEUkdLZ&H!TtB*XBElLNw-yff0^u)6x-SU
z{Xp$LBW~$F{`{X;qwBu<KMmJ+HLiD?X}9J>{)zRAg1YSuAN{>sVC_6n!}{quS?ivN
znWu!<_U_%a-bh7icAtNKziLfP@_Rkmb@#mUr!U=p`cnPf)RzYfAHChSgl+%1+>_zQ
zr(V@e?KG{cn;vVlNk!m<_g<G|#yLCApTC_h`mdHD>fp;IMb<sPtdBnUxwBN5x9*Kt
zMB{v`-3O}ntXq8VM~3n~D>J`?>aHS&QvW_A$FEyfWzBEU{BY^YW%BbAvkwNABrEqV
zidJ+D-THVlueF%-9v@y8^{Dl`<mV?!9@<z^me<qjF1<=_exlHsjVW`B_SM{)xBtzX
zj_jScJ_?>ne(tgP$SStm(zS1<Ma&D`K3lnXh3#w$_gilpzX-l@lP)cvHz6h8GKA&f
zRjy|bt=#5X{XPAwQNm(Y-j${OcmL?W*m_Sh<Uzr)nVC=iZ*4hQStE7jf8*khbK936
z?DCZVeOK|-Rk68?Yfs-@6!*NY=FPJH!1dqiGF<vpKUtX0SlHXo_fNv~{R)Y#ZZCh_
z=$*Uh-kaJ8&-LH>j(TQ`^!MNWXOI~&_v<z7-Q`oaxIO%FW4*KI`Zs?5lj9;U?9AS9
z@A7d!Z=v>Lb!pFh9giJZ8`!TL&zn8da@MNF5;F=;N2yrbl?lsfSxm@qc>lVOHS&G7
zy<OSwi)S{9Jb!rSdUyKfgHlB`Jl6f0uQ!}NaHOQ>-Lt8wd#s)}=GOj7U65pBSI&HH
zk?F4fZaK-nN2fG)Km2hgGv}iA?)lEGhdV!g-EiMQ|31S#hI<Tk+6{hcyLOzdJb%r7
zzWjCr`{O?i&-PcYjCuCl%6{9@$xEH@?KsZwJ%8Jm%);CaCE4Elw|&V5kN$XH-{)60
zcddTx?@L?tZGT%O$M5>KG#ftnvy0FBeB#`ah+TZ%*Arh_u9?i|eZBCdm7eYK(o2Q4
z$3Yy+b(8x*w3VLi^3qG0FRgTAmoI(3#Scus-tuM6HP7WsU+*cIJAK~gm%jE1PH}CM
zrZ0SwsuB%lXjDp`44bxaN~%Vxq!(v)^U~LwN@AzY^Ny6(iJiXm^{y{5)8}p9RT4XW
z-Rn!mvFaeY>!J7g(zzwkyYjrR=f2#zb~2dWrE9yr^iu9pJ@rqUgtpYqHPiog++Vx%
z=Hp)z{CdN*Hm^@fTmSm7Rc&<VoVgu$I!~=l$=f{j!p+VLb2aXIy)ujLoV)hOxv;GC
zx~Pt+4{myW$XQzPY<X)$=e)Hp=fhTkg+JW%`k^!D#NSNCut(|IA2(0^6=|V+#ET(U
zwP9P3LpMnEf}@ugZthc@CzhnzvQ5Y-T65vi%T8;KTnJ0b&^Fyt64fzP;g**`?$VBF
ze%u>{T%$V|tqr;ODN`}*P=>bRmZ?IwIz?`Iap$VGZ3}X?p6DF?a^YH*i(y?E>XS0G
zeYZ^2h_W<fT;3Y-a?#qBi(#iS)GuXd=Wdy5am!2gt&3dl(*7|2H7BgvgggJ#&Uqkx
z00iGN76{F#|FffZ^|#$0L2&>5AH5geA49<X8$Nzz{Cl<NcV4r3gL%V$jt0FQO|pTx
zI<qgen+3nrVm`I@_OuV(OF(<BSvLqO<g^3`DQx2X*u4O<aca?~RXL#D56glWe4%hH
z4?|N1d>iY7kNTVq+>5e1^cwUU92uG!*cf@Dj^vde(Kk5)Ixzqw(8DBgT*=L`M=A-n
z6~c|NCs}YZ;@l3+-UACd-rfG6cP3N*!}QaWKA1nwk2Df8H8U_xuTHL3X6W9o{4;r%
zWMV|1bCcnr!luH@s}mdz9{*M0{dhg`7AP$E85R623r}?h`Ay++S?Mr^>rl?3D`FkN
zjTWLDY63nAGaL*iCM@D`QJJG5<Y+K4LFu4NQzwhCtA@}+l|!B^lR4BnuL$@k%}{Zg
z$e}Lir#8dKU}l2R!IY+xN+L50j2@;mon#5-SSH}3IAhUK&6Z$}<sIJyeH3Rn8B9%B
zCAM71Pj!x$0r;?nivdk5qvp6-^cFz&9bN|Q2EY8sL@@c##@H(r;+FSZ`^yEN#EMk<
zH&=CQymwu?;*(wjd!_gR#|N$t7k+#uuJoz-RQH+N3ZI+L{dbb7Vy<BR!O5Vu?D&zd
z2Uax4bHpC~IU_<|c&GWrYT<+C@@Hmb?CX3HE%9dtL)V+xFZOkQh?ek~nX$j~$LtmR
zJO9i+vH$47hnxvAEUxJdF9i(dGA&C!P{K3g9OGrjgiHaExva|$A1INU(Z_tbFu{u7
zhrQ|P#ttiCpZ2DgAqKMC%aRY4=*{3`JDn_8YBqz9{qoBMD><Lz%`a;<7)jjT*kPsY
z)86uOih-=~vgAW0elz&ERwoOVhRxvPzMPq8rRQ_J_2nJ|S&3`MlwO@Nkd;~%e7Gbh
zLi)`22m6lJy?-pVp!slV!45vY>jxiNna4bDdwr*1uJo>xs&}7E$}93T-+8~IH=iM&
z!JfszW=eN^(Z>M0b*J>yPO%G{=AAxX8hOH0?@YUkGt1HAwuvz!$9x^x+rcKt9b=7Q
zISMz~0%o$^I^K27hac~_%)Nk*@A^)No&B$yUq2|2m08z(__9+#fKbBvxleDiu5@b)
zQFdN6QSs`D30Wx~rW5|(FkT`R8Pw~u(rsCYa`38&npaP_8x$Pcu&9kCx#ckn9C|vd
z$f(PxOQ_FqFmRlgHH-J)2es*ylTYtCnR@SuQGU@(^POjA$2{}ZdyP+czu<gu4Zz}{
z<fXbOL};ZVm|Ey@WTnfTQ!XogrkrA#%&8{qBew8F%VbV<VIQ>_P6n+BTn9x2yPPaq
z3%WAo4~jHtf)0J@3J5v0vL%>vnUIg_3@s@ygV2^mf_}<#+zfi5n`19;vG6@Iqd?>8
ziAh;0o~E0?KGj+2CKuGJv(jx^h_d&piK+<6N)Ln*h-B&H)4NVSEd-INcb{y^D~dGV
zxn}m9lGz%^8C7QGEBz8XCaKb(^jmFLzS1ADV~#4f@|FIny~<bmul6awa84GJ#PP;2
zuQ$BreZc!5ongigMPcWZM;$&o6FXInKDsPxK5$7v!;0Idz44`qfh_y7#|KK}X7I6`
zZeH-h$3T{AS@6LUof&<smv<&uN$Bu5E!upr#A1dY`{j?I)cU-6>cJa)O^b{Vm3YkX
z<GgH`Xr<=!yyfMB4OS~Qo#VQkm}sR7N~k*w=87+425p9(E4j@1a7osTKHlu+pK^D$
z-*{ge!(0MOsPN>puGkqVe852f@qFbSP!RleXS7+z!DOMw`FL7EjBrW8j~yC4vzwqP
z&!)$_UF;%S8vW6F;|JRZwhzV(N1T7|aN0dXqyMp={u3eLr_k*2p>S%MtV~_ZGS&cS
zTDKOs|3N&cmuaP2W5{BMRe^a+_*m<;|CzxHPvw@0dwUGltWZ2C@zCPnj!v$P3NsR|
zCY!hfYJrN<CQcS%4z&z37Uz|23quwMtP0e)sxc)?#LJXxsZ?uFuM$4t4{G5RlePDp
z6utL^E5E4KeCMLsF%Ny$kt4icFdLfS6}?0kg{XRk3U+zTS#@Nk*OXN*AY#!%mS9&6
zDbzEkd|MWEN(l2Um2w5UbYjS2k5z#xS2di$<fJSSPt&d?2uYrmZg3??l14~MAo{_Q
zrzW85he+Zuyk~aIMc;KV)WS<9YwtQKdiP0JUXiQ$P9re62wn2j4ule9$;Tl5zvn#q
z&o1~KlpFdPW>}~SuTDvt>a%AO&V<a-g(V>?XjmEhw70#yVjwGhjqS7Tj{J@HfA3&>
z0WIxH3wE62g_VN%Lg3eQrivIrmVy`|KIUrOM^iF->X}wFA1=+?agLY&@uh{XCE(H%
zl9pkmYIgJM2Olin8NyXw-}&(IuLo8ZHoU8J8mu6h=!T)^u~SEU*F91TFPyx3$H}YG
zPpxD6yY~C?*2zBG%eaT}4-bQ|mf-ZB9Vsqq%4U)mPj>jbwJZu*9JDG>^XiHzSzqKM
z1kMz&cpb7k8SA=&;}|EnvR62xCgEkOwNz?qP_NfYx1}M?CtdeIv`oqhL6L!)L~Y?y
zvtyq6uKT1GUO9R7u9Knno@nJ4O*P+1tnjM5qEPdlt7gYs^?i3!^Q-uSGoVIJKf{8M
z#a^od+rgyj)fJPnLOe}ZEs+Wh>eX85CJQE~qDz)~Ae10UUQtBX50RvU;icd(x%&j}
zOIfI2*S%5;N0KbPgHUqS7cP0V5TSoQyZR6QMV~>TBhR?OY{vV8wcUU39iLiZ5Lmt>
z=-dvDzE^QB@-n6lrF&TSOIGh*F-NPlF39$G^pfxCDt_fr*C)!Q{PJUXU-IDIj>e9+
zCk|Zj^t6wEr?Py$(p;Cu1y;suG`7bq=l&aTlG9Ud+l`7{iqf*O(|=72%2-m)?*8g_
z-l^O}NA#>GMdp-C6)lgRn|AmA`zgzWXaDZJHeoW8b2$&&XD$(+QlovPPB+Cj>igU4
zzdQVbQ&?6#&E7N0->P|{1cPf|?rWB9{;%s^DEvJ8_sgp#1}sJ!YL|Wg{A}N!?~SEO
zGAAFJDX*5R3`z<LDm9U@nEU5zzH|^%c_ZgD`IP}Tb_uNbes%i(^p!6w^|<1>R(xOA
zVtgjR!sz?SsX5kh4T%}G8?~4Ii1_SvY~cr$)i!^B|GdNhdaeK4x^LSG9oY7~bx7ZA
zz~il4DUj?oVV~mf27#(sGugQ7f0SB$h15h21=brG+DiCzA8qP?|KfO--{YriAJ;CQ
z-uld7?!I!rTyfqEpLLgAPtL7k2~rR&57s*GwNz!(5w;!on+tmzRg4bb*mL>$w#O|m
zr&jf;7~OEQvs@H7QOiH;ykDG?{{^0R|Nfnw`y*Fbr;2UuH^Z)FElVB;Z7N<gGi5`F
ztj7Y0xtG3ncO6|R`u4Kvtrjs^!;=LWrDnU=-#yl|<VoFc@lyMo4y*Td@sExjeP{2*
zIa9{{^tFl)E*}0`oO$+T+)otLBDbbV3tVbs_j)oztH0arYtGe7L1nhSWu1Ti)y~bn
z!2N3O`_P^7(k$QqzF5h~eQU+cw{wgezARpzFFWzZs^p^;TUhrkTsQAUY{8<)vVYd6
z9db=>urA41T9~(N)|}r$$9(^~IQc!2E%ULMTYjRwPSFlpYHc`UePr3hq{|)~Gg}OH
ze>Lgo-Da?WjdzZO%F07`tUI%o$Oudb`xNiwkYeEDb8F5??t7lj#f8(S%$@Z2US*L4
z%chB!Ci&g3s!uslmE7jeD7`y4N!X*(McmrGKjm^lpXP-ZFP}@boQ$1Nq@MFa@j(ca
zr`MFbUj)^w1Sf_$e{3@p{U=p^hhvtGwRJ3G@XC@&b0w@Y|K~+4ag$%((X%W^`LjgY
zzt?v*G-@?fv(DY4pWiZJmgXwy<F7XSKWy;(`dppxltfd{c}jbEwrM$v%)IvRkX3AI
zlSf>!k^e=W&r9p>aw&9iL{8ptV8P-YMH63rf2_${R=vZ`@Q#oDu5BSxqW1Lb_Gs{*
zC=#!@P*y3(lDqrdVOjkO=U=%Udu67tF?(kyRbF4Z@U>w<=daoypFs&tnPH7+pg_^8
z4J#XFEDB`eeRYfV^xP1QKgmnAy_eUkowPWiq4V}chkM)_=9B))t>>29pR3{KC9kCE
z*!q64i_F{%hF;C6{*q>v=J_5v!TkCho!k<p?n$PT;*J$B-IKPwD|f=@#HAXyPjYO!
z&ZS}ys}(deJLKlvs`)2Xn*&&8yBu_Sbalyc*~91iR9}3brgw#-m9yM5jw{PxgG-}`
z<vqJwe4B;W{cbAx=zUGmRp=Pcew!mDj-4XBihubQ?DU&^Zo#*r=L^;T-8peU{BqP%
z#aB)}8}v-FCKNRO-7kFJ?PpZ;G7(>?_uLBC_GVt~`TbC|a`yqzHqQWIcZp*msqZfB
zeDS+H`l%M%zBCT+1Kce0&OhLkF|)F;I->Pu>VY}Fdz>bfEPH?aM1uLA{eR4^fIEef
z3_HFaKD6Z0q%{ZQmt8))_I~#J68**-j$M;ZimOI2G(Azf$h51ndtF(li=y`(?r_Nr
z+s@ZT;=JE)xo+7Mzz}Vvy=(EC6MSkBg&TX$-+jlYn#t9iD*4u++ROWi`~Ap(RTmpY
z<aG9~s_U6hT>hw7f1A_xD_(EoA8${n?q)f?SLe^oFV>$HeZRItP{iv2_oHQB7F|3l
zdS&?qO;N3*3*Jkf&u0GGD(|PiOJebZZ<o0ZmhF90viU-ibn3#}Z>G%%pM5_#kV7}s
zUTeYH9u4b*d*a=MYO^iZyPdVzcY%L`hp}ae`s*HTwte%3zMPu8d%CpT0?!9O8%$qZ
zeYyVj{miF7%k{1P$CvJ1p~e}TT2=5z;ce#i8H-A^0?W06dj-!Ns0e8ZxVJKf_sZIT
zR%@WawP%_7s(B64$2K{2b9y?b&Jj}9UMtUabxy$A&|R&cEic3z3}O1(RoA|oJK}}k
zmTc91!K(sRf0%nrLFeUU{<|5iT+>8%F-%*!GNt)IN6Q02?Wxl@Oifr(nQ{AF>Jq!n
z204mWnavh&A8ZO-C6(cq-}`h|JqHiRh2uT551$J<Kf~gUVNdFV@6zwhD~c{;a|lU<
zFtGb73noXhOjsyl?(uDRK-P2q#%I6g`rYgH6RbX{U-;*R>6fBrj^z{E7PCA#F*{WM
zX8%Htkm+txy*V@U7+mvry`Q9}=%3u<Gv)7u9HIRsmf=OF{5G$DUtN{T+Z%2(txxDI
z_j2a<JK}PD%TJtW_<xs=Jyrcj?{}_gIoI-ac*R!k{n5JP8z}P`GyFQbt<yYflkto5
zA(Cop7Zi%jjoSC>goNgO`fxMygXYE;F}f_O6H}9Vge4xowf1%V;ChjNs^~+#&67`T
zGMRFB&W6*gev9tqUXi%vx~taTUv>J0x#f3k6aGBV*<G+?`ipCYHeZSkZ(w@5)1Wp_
z=VEq)XN@>}xX8plue&iH;tpihERST**Dw5-+|?wSn7vPX-P>~sAF7vWF-SS`_y*>e
zxi;Js^<z}cw0frh{(`LE+}?N*!N04XFHkVBp7HHifWZ~}pr5zZ`&UbZD{g!f{I>lW
z-!18ilCVu5nia(Bb~j$WcF!SK@YSLZMzL?|^o7q`9i8>?^0oV~_wxMN_VKn_9fO-r
zqj%P>Gl`Wyt>dN##`kXeBw4y(SsSB}S^Hlvn`+P)j39%x#5U=*)+yTr7v9>XoYJsp
z%FD~lT1F|a@BPo)%jcrd;9fqr<NnM=&3ewhx)+!e7OUm@-mu<hT9W@p?OsOXLc8Dh
z5AFY8W^&|+>$>08oLnKVE^atmCHX;RGxw4de~C}Ed9`7&c}|j-%qFv|PUTH?J8)BG
zD~qh3pGHCWjSr&F?tj?6%1=J%&=f`nP7kv=5+zZ*UlMt3G?-(Pm}6ft+rDA8t!TEX
zV)m?9V4b(%h-}lZ6G|I3bRI0&dU8to^+yREeeFH0ukuyP?yt;i?AkxK<y++f##be-
z++62oXWt6t*cA0PcgHOTv5&t6A3VRF`vo*)D9==&Ui_%$9qWY_jq9xJ)-1Aq(>mY1
zn809hWrpAJ5(AUi2^V&nT;S^BD=R9A72)!7H#QDZGRaUVUGhkG!o2{tfS(~hubfo3
zVYc3x#o^5w6qUJ*@AY(t?r#qNYz<zyFW|3hcwc9ILfE;H#f^!R=Z~O9D4$6dPY;um
zHRps#=8YyrS3fB7N`%TQuP8t8i*?4=t+RWg0zBk5@NZ?%@sw)!Utzucn&e4`=7aCt
zLOxFa`n$C4iH^gzi#PL~Q$FbZefj?V_q+`tpO`n;Gbhv>y5F$>kKIH4oilqk>b+R(
z%y#j_H>ugux1||p>@qrJz+}i|Souq!r@Q)^MohBGoP)E4&OBHnQr?_!WF1@6OEvY1
zD<!b;#Kr`tQ0GIPEyA2?LOxeE3HYeYSTPmERTuJ6n&D#5nILqq=lcZEa3YB7*}XV3
z_0Y+vxjqgv8#cv>JWM%svSl*o@{Cfhha!hShfRV<7o7}TR|j)|1{9%Zg@!b(jGDWu
z*V$q!Y^>(XsjpS_ii`GuyGISyp*#QAt7{3`ME!sLL;8bwTNdXLb;Zrz+i$+*H*ojJ
zm~+u$aiQDeo@0`|+k37_t_818%!s+LV?m+g<FyM)9bY#cW9{XZx-770L80s8wTnt!
zUw0jYpDm*;y`a?jb=xuCUTG=wsOPFTUwmcbT`L`BZt^b2mMi(8&LQ7-a|NeKcJ|M7
zO?33`o14%Bg4!2SH&*oMa2$idokDlL*!yMrgnM8x+*D=Gfrl)R@dwZl?SZK;uidD4
zQosZmo1NQ!ef{k};Nd@}0~+gJRPaq)^f4=Ein&zl1h<WgL&f$?v(|r<C$h0_<9@ad
zUIyu-GkQOi9lBk4`+n*E=S%CKulTkl?&5y?>w2zBUo>eMbOt3d>Re<x$~#3r;2DGR
z%X<n7OHY~4;$@d$VgJ$3a;il0!O{~gr%pQD<Y-l#(i7n0wDUxKMLny+ZjPPalMJVH
zEezpVnj|Osd4+`k(F!f`g^Nm$HfVv5+~Iky*)b*K_eIxpGu)jwx25rZev#ilX>t69
zm7sZ?e~TFuikI$~R&RWC-9)E|Kdn<tAI)x<FTBcYb<3|Qzk{^6f36PJZhg<Z&wqvF
zN8fS{&JX7&dkI*wGZl%lED_`|*w1CK_!aLG+av!un<wm;u9WcY*p|LdV}_9BOg%x#
zd~CKXHS#}vpNQOG-qkIm&CXUa#oqnTtg>^AYlWlK`FIpol$?%H{^M2n)y>kh_1csm
zaR;r8fC~#4G9FH-`~R(yxdK$YIWxGObbihpDA8s1#kbRKC1+Xs!(|_MG+z9iXngzQ
z_9OR&>b<jOobqkd2-tA=44<*}nHQ_1SEyPwE>66X{Ggvj_=fUI%Qx?MeAK#5wY8RV
z?bm&?W5eT9VpC<q%L19D!~7N$F`a9>*VZr3WyX`W!ZNHZ^yZ>XthV)5$(@-3Wh!r0
zv`yrF=`K-mJ0q$>FKa@#VxrrDJG(xHFWQ)&)FUcVxZ|<I$>mei0?XF&x(Dxglo#XX
zHs5q_-M0HbXRl67o*Tcp6dc&=nO>amknWV7;alDG`1?DnGMf@{CXS|+-|s)H<2|)i
zx#{q^2?8II=h&94;%|T3Vj=TaU*&3ST4NJm*4(7a#a3@kDh?>OZ%7wwbxn0Na5G5D
z$!JM_xa>xu1}rH{Nn1U%Xn)M|n1T5>Zxr+4A7$x`ukZ4@X;(%)a}x_umMY#NJ$aVo
z!H8sMb(!U&8z2dOx$^PdlNK#pxTv$(qA`K#AWIV`!{rO>Hby$Hz9#XzZ=c!jUXO~#
zkIYSrDm3m*x}?osrOoOV$n_=i73=gx^1;>9Z?Yc=Z_!zk<i2&^VyPuTyiP0IymaGc
zcK=~20Y{)DLn1rtJCSX>1Kt#fHQ!^MBl4&9H}4$*VO~SgcOk2-=QgAnIH_`KBz-c>
z5`0_bDdl+4;k<*0qmZM3BcJC?*BVa=PsWeauS_{IWy_Q$TQ)FiW-6~e5<mHF#D8xo
zsm#v@7A`RLJ)<C!miOeql@pCKCo&5wvp;j?^XitFA!=!$S(M=S<iL~@jgk|Yg_YQ=
z+E|u4^POpvnZXk|?{(wK3Cwf#p7P4QxF9=)&GLexwE<_*0)NvE`PGwlNO$h;1Q8Qa
z38+|fK>y2Q2NZr6a4nx}8D%M2-}upnY4MB&1-Ewo*>(0GsP&@EVDx5#(H~V1s?puX
zQ{KRo-wj2NA28*2ANnvWW6#t@5wit1K2=)u-o@qJ%leBu78NdC?dJNr>l&o6gPxql
zFC;s0rizlGXUkVZ=MtkuF)bGj_ei|3N&51rS(5qfBxTwCq6U$C{2DKGHiespBm`H^
zzQIu}C27pZdho>K7CrUZCtn`X;%-`~-@S8B7{@WrV^4QHYI!WvC(IL@l5li^L(|kS
zo~dCxT?a%YA9JyS(gA1yR>0@VBbP&+GJaE}c_HQBOuZlbH?IY!>BS6I`Rw9mi&Ccv
z%U(1(z1R0ho5*dCO}#3&JvR5c-1gYon{vCWgI9C=R2!Qo0RlXo+b32@pWGfQC>>Ow
z%xONUK$+KEs8Ctdd{SYylG&0cZb5k}Pu-&GS{f82CKYEdI<n-2U)R1JsqVWX-G2s$
z?~q^o;E{#quE~q6Id@H703%MfUhup)SLCkeg}E#ULiS;d@?)um3KBvk<*6~@T}Dlu
zp9)o6%Y#bP9m|Cf#Fw)enr*WR>+Wu=2bDhkOfOoP&K<iOx3GQti+2jKD=wULo*sEe
zz+=nqj3ae#-=F<8|K6I(tQJn?!5kHe?mJQrXi7ObR$Ni)G0;1?LAK$rwDYNh-DYL8
z*>>DGWNuzo-FG8bI>|h@qhM!;{SK#%{o7`zc;C3he*5jl#J*l>CZA0Y9||T;yql}1
zbY^2>FK=?rjikdO(tNTU?Sh5P2U*#^i^s$#H23c2ma$M;r?+G0gGWu(-0O5V>^(M@
zl~3A!rwgcJ`tQzYa7fMcxoy$Sy$L!}o7VT*uNGT6^%&PNDUY6}8>U>J4;>OcS<Lp~
zi(g6LhQDS;Z#UL{(BvvTGVKkkw3gnAh^&;|ue$2OeE%43{=dd@L*m7lAF7Woil+!A
zh+GO}n>uTLTFR_RnMORF?fM~#JwKK*E!nY@qp~+hTl82Et7%Wun#@^gX|pC}`kgo$
z6tmFNx?zpnW~r1=4W^1e!7f*o8j@KB64q<HiwXR&_fU(e#_VMmzN{<Tdo7kB&0_w<
zk9MnM&R4CDlIBlV=V7Qiyl!j3TGpmL3q#nezIWX1wOz9P0<+dvelE7|{at^4!6#=L
zwzbagjX0^p)pfSh)T>Y`!t~Mutxwxeh;{B)ZMrEWs4M*4=yo$3>oH9}o>e_Lff^jW
zQ^fBnY20YKxm4me<5P(*66-eZK5J5NBuAg;**T+CA<W^n;s-WxH<;usIpN4+(ZRqj
zcu(Pwv;@!o3wN6sX0=_rQY?J(X8F`m)#|hvTMw*Q;o8`pW*511s^dzwsLVaPh4Ok_
zeIIY{J*3>W!K|)bT)OhrG3Px;v^opjwjYtW(V2Dhh}BHP@5c3~d-~_z+#ql(EMobF
z#KgIWzD+IFS{)J=m~we-VMy<R*>x$^OO`pVT=9RmQ;GKR8^^B8m%cj|-w^_eYkTH~
zg`XB{%-^(Pfrt?6Lq86uWg1*TtGy=nFMHpga3r!tWtGZ-UPH~r8qp%D-CeozGi|v#
zH@2>oYHVjzSm7j+BB&`|ne^gMmGz^?%WrecmY0l^PoL28bjxRr&Lg@XIh<B-isbMY
zA82|Yq;aG5$HPNbQ+cKL=VVnZd3#rM!+m#4HjQOb$0Wtv*&0+B7VP!f8!a+TE;UHg
z;nj@7@X34={vEq&d}ziZA1%%ajh2(IS!eE@JV{kW>eYj84F{=3*?%QwU;Pl?$zI`j
zqi34hx)*D&_&(q`%dxOl-)R4tKSAc)^Mq$UcX`G*apnqz&kZNceeCZ~d9+~DUws#F
z>v=xog{BFG3aLS!B}z|ECPf<G@rYnc%nxZ^(giw$dfCLVmJ{xYOmA<r{L9i%F;BES
zw1J<|Nw8UG-i0aRtqgrfl(QnPI*T?~3C&&6Jdt6kC}YOkGL}BKOfA>$vcGjK>S9MP
z&C1owa;<&kG4W86ibvu>sbgAuI1X+UWG!A{@!H}wOJ&+=p8n7oUCTaf3aL;Ny*KGW
za%YpSP;|<*C!C=k{@k3I4`+r5+}&@Go~$3SdDF9N5nD6m_N|uL+@odcsM>GXTW#Ds
zM`^{P+$A*$8eeypTv}!sDaBxtV#MY8e%Xt;8vmKSF3<XB<pb{QNiz7Ia+tUF@?3*`
z3S~QAa4uy$)fYRXoV(;1Te<z8yBWHREZ!a1=XGyw(62bnviD}m8ceHmI#a`f?3*(G
z2&F82l^0iPzJAhyD~gRP7a4~x54u{r*lR=3QStDPM?;^aq&$BYuVU)4YQ@73CX3GU
zdNZC#W^onmW)S7wnf2<o##XjjTrWj@ypKtxPIZ0i(Yds3-Tl3*a+nsBbgbL?d0FsR
zp-lNhLA=cj#}yc&WUfD|2w~P0x?fkH|H>n8-n=i*GV^_x`=?bdJUO9v^K{nYD$DHr
zbHC253VkcYZMNpm+Kp=4T=)ZyuIde~bk5qPXxkq5i!}l~4xr5N;Caxj6KaPyEi`$k
z;5yrP)iJeQw~ib$-I(j!^jhs~lJLx2udh+F@)&(fwsr0^n|M*p=+=?b{7(~-cgP3t
zQw?g5O%0F?O`FQ{Kx(H*y>HikwNE!)>}H=x7xusBVmEt5D*J>#3Spvp78)K0Cw}Ya
z++DS>r?ID@rFpfw)RX?H_F<{=DZkwo^jdOA8#f(fk$rx6U9r>NnRn6^-n~!l(>h-H
zh-tyO1?&zpr`Vkd%`*DEZW6-`k+r|YcGeXHuguhrUX<-(5govl5MbsiqcpX$IAYbo
zy5w(SVrQmtEI4NRws6B!*(3Rk9hHx(54K%gK2=k8Pu1Zg`NFsV+tonheP+kBse7l)
znA&MO^G5KG+iHv+iF3WStj@XLk<feau2i&%*$t`ah9%)_!ig)Vta!!8W9SvFqN`Tu
zbJemyz##hH%kNqap<hk3=7~<#+#I1cb&-B`=uMZF5c?j%6p6Lc!eR{}mYGxfb=W5Q
zFEHV=^eF9oSbaM7W!z*1)l}v)zieNBPtsy<SY&th>dO~4tCraA^|1P<_NUcjc@S^h
zO1Af`!TJkA6&!g4UVd4$G4WZygj0jTo}iC+xta3+OuEcHD^4K(!hVC7vvgOci0{Y?
z)!Y%dNq^Z%SxNcd;!<WynnJsBr({W(HZA$Z#s8T*@C|5&F`dCcTDQ4%lgOnM#a;th
zrlJ))TS`(B9vetnM!1Cvm^RGWt@SxE^i<Il*BcroWeeA5yQ~xZn)t0q-Q$n$kKGpy
zSA1lT{1f=)o!18UrAs#N+pJdU!uE0X#50L19~aAdw$8QS+UB3LL3P`SHM4HkKAm9l
z^w>9F|E!Ete4fi}v>q3vcok2HGI;vQtarYOo|jPV3Xvl_uUq_I86SN7Ot|#aAhA`s
zYdhQ?MClv74Ah*YoGUfaH)emk0z=}OH|v5Lk}d~vPnfo@eEJVLAtUu~MxU)FO3u*!
zeB9^wIw$>VnFpP#XJxpAPMo$tyf;B=^(PO{Jxy^-?EbU908eiyGt9Vc(|M`HDc?!z
zGXJ{OdjqbtESj)u!4ySz4JLt(t|gwFb$2dri8|I{RBMsKb}HA8!6`{a#Qh|jh>FTw
z0Z*NZMG<UE-5hRBJ<_bM>gE6FsPIu^jjy}87wl%(!O^;U5%+@Cjn@u{emv~{@5c-l
zg;fjZ1bY7YwW?U<Z+@7^x~~1_*dr!><7j4eO>-4Ez;k)77x&~i7O_rV?r4#w)qgf_
zN=%)5>b}(N&udOP3r*2n=`0z#NmBCTn#1bB4%fZq8JZYaLU~q6uqb5h+qo)7r&}#|
zmeBSi7h-#ab8;rfiA`m?5K&)z_Nb}JzK8R691Hztwg;NbtuJT@&1C9sNn*YxVwAk)
z>x$M#&kyacUwCI`nC#Rr*}aKL-7X?WpYESfA!=0E;nUmXV7xk~)n4q7=9~Q=j-UEA
zb%VCFS=ddf*AoJ#>ALGbUKb>MW^st^&+j>EFZMKv7#?#~S^8qbL``@7vs=t+?s;6=
zEp;ojNU(Zy)#C2~NAI`ocUyQVtw@JUr|qk9^h(3kR@qkBhvb&quhL4iZe%y($&twL
zTVk?1I$`b!dF5&uM!_G^N#UKm#??)>eru;Z3Ej8zlzI5iW3POtdAEjYwKHyBtL0iT
zRY_*)n>i**`_^djIdT=Qnb?-fwc+>g2`?bYJb*!2d7>5w2~K%jy>LHg;qs}8fto79
z*Zekq|8bpV0jF)W*i<hq_Nzj=w`QLR518q?lP$JeDRGfwLHwFc9~owCR*rh2{^u@N
z*Rh8azB2DHdo<H6?7YsC^*bBwOfM~WJLD4G7sqjH-|8vc4WHAGyf`fGcy7MpN|Dzq
zZJzCU>G7~9D7o#R!ajioA|b1igcjJ$;b-$d{k%75`m&`tQK#IT55=+7%70xWxP$3V
zNw&n&9WOsBKfLik=SOV&^%a`AAtu|`1?1)&b9grMep+4RjeQ#uALdW>6OK-mx3Di?
z^L}pz%RP>@8xxa7*Vyj*Q_ArT)HgC_aOh}|IU*`;VLjnm%PF%uaU;w6`wEK>iZef7
zJt3~)(Yi|^Tm>R>%}rnRCrK@F)LZr9s7B|bb%MLxw*6Glk>XU*@ZBb(an&G$Yr?0H
zKX;|Lw!E1ps>8({Rv312&7??HuVdAmT&}rA`YcgJOHNpstW6hV-O?(s(zE~elKpR&
z3QB3@CBKN9wq|<fi%p*9J&z||6t0?BIsb-g!TqPVMZU`X+%r?Dzi{W$L#~>=#pXX;
zEaxl^S<k{~v1i#8<ux<f{_=)i)43Y(_(&o%gRf{`{i}^1y;`ybyn-b|g(k%92!7Nw
zX~NDtznf9PZ<@c!JpLxW;4Ww~_&h_wL$3nGU;(4ffE^sHM^0+oOkkfE)E)j&JWbX1
z{kDw{wl~Nt$i85$=RWeu;h@!&Ie#bWGDW)_>RZ!%@35f!#88FLnOlBmZ26thQ+I*;
z-?9aoUAzHTk`ucRi3PcNor}MDe1XgFb(c&|$0o9stu!cK$?n6ug-5{myU~ieJFE(`
z_60DXy<%c{qrqh5_CT?zxob8vq<o!o)nM}>9uZ}R(EH|g!6A3=_bz)NWz~PdA#DHj
zb@z|&zEZtdZcqKfl?Ph(dCK2g!k>Sk*?dK~^wfK6SGP0E3wJjiiah>PbyxQtor$wm
zFEOl8oF&cG)S311w*YwIlskjVB9RQ0$p$93x=K{EbhP)cJ+jp6&1x6xfKH7jb!!2I
zcYYmeK57dmGjez>+#)`8mHejzHGc)JIyAqUc{}3O_cix7?>nE&B4u0YA34orX6&}a
z!^*OS%enTge)7KOQ<J_+nM>vatrnxzSKcq|wBcn46ct%`ee0<W8jdFgztsuaJp8x)
zp?=G|-i0Dn2lMBO8t!w^n&I(p#=C{)M<RrB>Z=Z}n)$Yp<K!9^)!VbGoM&a$ES%>P
zJG1}&RfC>~8^eRkr|e1Wh*WKVocuyAB<sVvDkhJ3tD1#vfuhMlk^M(ciEtSnKf2dp
zNvGhiAH|}t>IEXfy<-jr4&JrKRf!sIDF<TL>09rg7NXv5bE1Z6>6YF+t*^TU`gAhV
zx=txpv#YUfnw-M2nIlE4C|$Aic*MVbmKil{i8|(-_9e&9Ys|4vWNE#!&id2ltR<BT
zoEE5WJ5cBtn!mVjd*II7(rI<kH`w`jQpDX3ertWp9UT&OGo`abc}c(=pR@V;99uSK
zC7*T6_`v<*f7#i>>7I{gEMuMOnHqMoY|<Rji77|9UZe^%EO@`wY}wX@ENgjAMlL+D
zf4bMzlf_{hnKnFNT44IfFzSh)s<zKGrWK(Ey3Lug3bxs|oEQwQEm)b9UA4Vx=kApy
zQv;VW{IP?~M<_B>ixzQox_HzG_*~=`->VQDt#V1zc}nn+N>QtQ=B~yLnhb$o7rp9?
z%h_|xIE=&l+ZN%2AI%@;AB$J2GcSE_Z_WL$Wuws($Hk1Eiy1kEeHX<(*go-ra<t2~
zjx~#)h<ny<?&dwbW<$e!)`Jfs7ASh^oA_<h&x*Jis=C9=V5+WGv`JX*pS77QY&^7k
z9|f}cMrA+#75CZ9_v*12-^s1AUiyorGTN3r4BfK3tIT^VAHyuck~bPx7pzFS_<h#m
z%&29Rxh8Q_boO~RT`8%xc(I+y)MVwI*&%P&dO6-%mB++&;liqKsaN-f*tC7iar`VD
z0B)<eGqfn&5wI-?v=#ri`@?<L+e#HHTlP<_nY#E!Lv6283!C@exD9Cua}VfW3|Fqr
z^{{C??sg$(euU}9+O(@@?i;Z?#-=M9`+lAlH{rH+OW4gto9cIlP28GRF!|U6wY>?4
zy%zN9O4Td#-(4G7W;l77ywCYrwl8yp-`|;_#g}Tpcp+%UwWc#$-ek@KqbkWz`#wPz
z(+isdz3v}LF&E6bdHqXJc6w!x!s_QXS{w=2IJy`_u3kPgslhzW^O<-|@+;q^Il0b9
z)?X0R*s6L=Y0~MiHyvu-^Oju@@Ksct<>a*B$@`x98zhB&AFhA&^TYJ`jyQ1Nlj(rl
z6qkjHf-NgHTlPxMx$*dq;fK=)+9v)eRX5Y%SeAa}MQYMqpDm}?tkMzSDqE+~>&mPB
zx<r!2yVXVhaLG}(izj@XjeB|XOn)e?NEA3QA*Ab`3uD)+^;=4!r@wsMy`(ErbW_!;
z#WF43%F_Q6WDc8Nd6dZ&8-8+<Y{|OGOZQz5F}oi1Et=Px>4u|%@;|jdp$XnmvvSnz
zA1s-Wsxjy5;#nKBR5e^;4A$<vtCc+0)M@I6<Oxq>8!uduyx$(k)}r6UCahx8x!ddS
zx^=p}fohB%&Zmrc(n1^`ORzIvzpl`kcv3L;@3v0;p36ULHNf?jGlPq8m6^}9MO<!A
z3dMtVdNgv+-5u1kREte<`h`2K&*WNaoW7r3tfjSoxqGO<zxNIaYJYQnWFL9SJLPD;
zgkjR21sxhnuOF-_m?!F|^-L+=K-uNsw}4Qq&YAbySjy8wKQFOsw(`}=4qdsCa|LG~
zw}ZGYgM<0==i)gJX7$dNno)P%Wk;P!*rNye95bc`CR}Qq%x9B+n3p~B#is>vB6T}O
z>!NRL*p#Tc<&;KFZ)nc*ezg~~AEsW8VNXr1x7pBiukwO5|BhdR4A0INs}-G@nB12l
zs1(_9!ch6TB3Io`*RWn57LlLHDiW!6zBOJ`wy2x_Y3+cvNR1|H2zMG48VOe$eRO?!
zm8)>2(0Mb<|KW?mQoB!y^@e(CRcS7M8k_68BVSE4;!?4YuE4!nZs}4JGF_s-N6Bm6
zk64u$+N^995n}mdM<fH^(`ezFhEtVnq!#i!iMY&jDKU#$9jtZd!*!42j>m;M^WxLR
zSdTtiBNf5EK}a)PX`+`+xzKcjOip3ecYmDr&V2LK(QfjILZQcVN*FySEEaaS^kb6N
zlNClQb+oQJ2HN(zh)-Oi>NR1~1{vdfp|8rn-tlebP$*m>>vQZ<s$P`Z)$=iH&rH^u
z{3<=-xKh6P>mHqZJgi(deokw;_224G6=?8JkYQ>?P^WsdvY%;Egt8a!>1NwiK3d1W
zXPtRnDYo~VlG)E=x2>A{Ht=Y%YKSm)tdjWJykUlmX7`13Pj{m^c@b`mC$b{0ELh<r
z{H!|kq~dP3CsHS$EmB%HafjQZjVnYL1gF`}%Ugc=f1Y6Lo?PSZLyHy`gq4Ij-gj+T
zJz<}C*A_0<sMeG<6M95Ht!z!tzIZb#@6-ESmg9>T99Yqr8gTNKO~`J+u9xkGA>q!v
z{rM|BmIZBj;ksPMDPTon^2yt76Ejw3Z0r`>+3{(*w0?HSX@M`TQYv9fK1=yWCdaTp
z4}P+9lj<&0)ifO+pN*4lzWe_02Bd{@VvF;MkTlU#`ct152QSEX6$mM5SXSb?TSubo
z<w2jmmSgu-8`lbDg>)WT?NKPYKf+aal}6Ft-Vcv%_Z(QgWpOAk-*ng7^@4#y+8_Pc
zA4o0A?ht7T`S>I>XoXg0@YF3UyL=R{H8QY#6>!OJVsJC<$&xS?edc~w={H+$o=B1T
zrA5~^Px;2weNn?TbGwi7Qr*|?W+6!{4#{iVmT60yG3CD6D0t4G?ABs0){+8S_VATc
zf@d0-2Ypype&tKvt=wnJgdQB4c=o}GNCvkT9KGG2mMy!yDYNxmv7^lPV;OmpuI7R7
zQ<p#3yF%mFB1u!zy#`{9tl4?3rBdDZ+mpeC8Z*P1d7iSFLGDt?92->L%`D(QCi%xn
zbg~u)qoc%xH_i-RfiA}wA8=e;Avb$vNb|9-brS<$>HpmH)}-yQxDvO*IW^NIhA$XS
zt$4qfp+|Fv#_@9o@jp)6T<yFcAeDD2RyHu_#adxwu}L#tpUGfRiuj|up?3S$H>oLy
zPgqS1P`<5x$ZyR|4qm3s)iZA@ZO+P*N)E9+7Gs)mz2NHnm51|ooeSb!ETwkV>}7dx
zs&Q%ls~c|TT^Zc--mj_(ZO9gFaPu%(GF7f*g;js^wJ&m#*^R2NKP#mM-JB6(s&R92
z_KS53ljC3a{>pdR2AX}I&!{kMnrDy0!nrK1x7-)++j!gZ)IVmQ>f=ft^}of}$=BGp
zPc@oe-t_vVS!5a`$0~s-PMTZ}A<bvrFV~4_zPkVH)#CDc7s)S+s}v#=oVcf)4B1pS
zePu-doQ4lkb;`l#4@Izu%#2T~+nKuUbWTUiYL4WFrh~h*R%>z1QmSxdNNdzSmBFU7
z_|%e3D?%RVC)cV-JPDX;d-AHmj#CSQ8k(%QTp1QgJt>)OX5&0xZp)>_dj&ab+T0#U
zIrhJ6Dq1snb>r*m-V>%w;m1u_4v8~~++6ToIRDBk_djd@s4c#6A-`n>cU8UECZ>eY
z63KeyKJXeijs}M}jOSdBh`di#(o0neRNIxB6s9QD*xYR!#4RPZZN-u8S}afdr{1`J
zLx3r9N*32mzyF86G#f~{^=f>r72)joW#`7&v4};=tVhC8GQ{w8Iq%$^>m}<O7+x)q
z{k2e3P%v<!?JAMNiyet~ZZPiX;7_sExVf0WYw^O}*Jj>H*SFl-+cv3%BZ5=ud7O%v
zDC>hj)oOjSH7>hckF|8JnH8b_(R<IuBjKKnG5HZUHf->F+CMcutSsi{h9Z+tUhQCU
zvwt^!BnR6az8j)8HRp)<oFxl3E(z9r$uy~?a7vhj>cd&jPMX_pIXtc8p~sfxE-Ouc
zm;L}3;KmFhkrPcg>m@a_lxKSVnEi43F5|9cQ>#MO-joUJ`Wv`$8OK46Lz^Z|)zV%r
z!+rBY>c_Q-CsZaR#C(~QFehQpiiIpsTZBQ23@$*H8c1Xq%g&q;0Gc890PkEnvQei+
zm{VQIM`Z>`K<J=~uAkBz7YonR-rz+CDovhIb9^jj7J&CJu}uCti$g8D(CA?b$|{5I
z;4k|)mkac4I;*ud3v@AQXm>Ctc-_IOT4#f)2_hFaXBJ95)bP6b{_rG75-xDBUb4Zh
zZj;mZDJOnBE<bePqy7O^;qs~n@>hzAPVjWhtX+R~FZb@rPu5*nySwOl>Fo3)M;Gli
zu{B<NXx4>ghNl;=VBg7gH+4htol_Gw9(?@ipT4U_wBLJ0HxoydwSJC=^*$Cbgim|m
z%jLIT{ZGquJKf4%B^fO0H&drw>Ys3iBV}rtg|sMpD`T#5tn)<fIdQ9eLf>}C-245`
zZ`$8H!OVwuGqhJu*mCz`>GRgb=h?2^>H2Z0z25$ARe<E{TMz!-ka)9-`T3zGEk*%-
zO*_)>eiu>>S@X2v$)!%dm4S}2Wp1lIIsBUoc&&@KI2~7f=`;Va?2l_273SymZ-4z@
z*XO)NHM`Zz*8bh=!ot6AzYojfErJSv<L<cMSS%M3dH2KGa)G-cD;C5*x4u8aJT_6o
zr;&q4D!JmR*rwZ48z*R|=FSMs+Ar99kKrCe9oqp}->F-__+<$tuW6oUb$F(W6kpk$
zmlu!j{(s)1Fz{tTU&B71H<DH_cRfz<o}2ER;V{9FmHUp>$_De=3AVem7cE|Qu|xS0
z7sm^q)+aog0&ZKrG1N=NtmAflB@yQq(D0k3OZJ}OT+^NfZgWk-w4Agx3SC*Ix0(2M
z&0nez|Dk750{hQ}3x0T?-k{oMc+ny}=Z-@7+zW?AzZ(}gZ<%t!vFV!y^WKy><x`i7
zH0w_>`^@``qfb;(NiuSw)AK2!Hai1T?s^<k+_A!9*0y&)l1sjrzFe^T!J^6O%hyNP
ze=oT6)0c7D+#^DPXBHh=`Stp{nc62*sz2HKrZm6jzWeJax8s=%@y^LD$E)L499k^;
zT5P7f_QQ_X2b7MO`c&>UuKC2buHo_On#CUP-Y)sSpT8I!Vk``sSDP9{8(x+<V)Z<@
z!BQ}gY478>mdlDW3Z?ekU70qyI%T=o-12udDFsg-6kOV^P-b>nQ|3(YtIpC@KeVM8
zixwVZ>a+GW6xgwS^Wm8+{73XPm>&M!d}xkBzl-|zT9=fN-n1!hL3a}yXKQaiIO9{?
ze$fe8pEk}FV$=zGkSZrL_leks*>}Yr`9&S}%eowp#@55RV{XZQrBwymi&s5<#?W3h
zQG1S+#VM^{x^})l%h`6`IobC4o0Fa7!*dUo-Ks2Ev+<hR26>i^N4kR-NX{|*Vz9Jj
zXUEmMt0QuMGJRoQ{`srI#l8o77CdmBlp%MQ@#wYl9^ni#7bvo^edz3&$KNS<n_cE>
zr~1FP5{-$BQ9|#dS<82JnFkA=TJxy)TZ)Q--krWD_x`R;+hyHg`TkUUe7h&Of@M+A
zi1pilOZsw;<X!iIsq*ve=Q3IAS{+b0w6*1&!`?MZdKO%E`Nd&(Q`AT+c;%B0^BZ#y
zzIR<{b9h46BI6<>A2+4#vOiA?B`1q^C|Pl@ZD_Y%^WNjVlDCYavhn$eYn3!F#+0i_
z+AOF@d+uvHx$9xZOBwx_qG^{-d&CJf%{*YCrLp+MjH1{hYL@;>7phr)tXh1K^^wFq
z#fJ$R#*?nerF@z0_|R-|%!2uSwW%h-a~4?pU;MYqYI2o);iIoE-rW&dUV6PHtLIvu
z%L(sVbNaI5zhA5Scse&fJi%A8JZZk5wwUF&Gr}wW2kn^h_r{`=XVazxgm$h-yc1`+
zV((oM<w!>9>K4;|TYB5Mo;v=xe&1>Tl@dMcAhv7#YqkWhnO6|FQ?V&|(Hl-iiMO(H
ze?cRJ8V5h0(y;KDa`$DCVlCUo{wbHs?rooN_tlbkwmlyIA8fJo5GiulBDL^QPl>(a
zqsK<~XC7?h@b8%7Z6+6<$|hqT!+gR$ZKCvqYYz9KCT?z?wD9Y?E+%_R-P_it9tv|~
zh3;jXxn8~2_ImLWo4u$1I8Lb8RPu?*R_f4HW5#2CZAIZ54k^!dpCY1K=-~CfSiRLN
zZu{41bJhJfoOy6C+-jw6PeFBl*ph9{E|!OyHmld?eQ}U_nEXC_R%Xh@le2~1$sSVU
zSpADXdi!nt(jWY#cKa{gVsn!0dHqjFs(JO7Jpq1;v>tjq&}>XLn7p*DvOhTG%6i9G
ziDUaT%Qd$czGs&$^HM05-!)0P++40@hmcFn%o!G1nmWun?gp=T1bTC$7C&&?CD{@B
zD{gK6><^#{f!SeNrbFVv8yhdaH0M}%ZpIYF-yJfy1=i@_eG<M>$A)Xlx)=AXV;y}y
zPJdzTw@$+GKsoE_ws7;Qvm##_30;lZ!*;@pXIq1{$5S(-eV!NDBb|ifeGT1~>rL6y
z@VV^B$JP%qbGA&L99Hw8ZoNgyixNkslR}5%A|Ayt{OnTM{H40dW0nz9>yk;L(lcB+
z%zo|qS}J#NN7Fi<&O8&Jiv7+ts+ILpYcf?@S<Y9nL=|#sS6y4&b>o$n`mGa(7iV_X
z_`fxqB=9o&o<+tY{*PDx79R+lRB+C6YO8ccn)!_P#}o|%F345#z5E+EYw{$O!V8NW
zgz_Fe`Ix8Go^yq%ysbXmYuVi8CQsdXSkK&9^Zi%rwnm-_3_q;z@?R*O%J=*G!^@yH
zem?_)tLI&&4X^Lcu{z7K`F>u>ffvs@Lh_z$WR5()X2HFwi$a!7Y21<eLUpR3IveLk
zVTqqhTxQL5W0lQOmu7vkZnA0eGP&?2YZRtRD;c^Tlix4gFiGRWG9If196Po*XNO<V
z^9a7=aWzuooVf9Gle~<8)TWeoZn|pXY`JsI8<bL0eODiBe>u(RY5IioRVhN3tJnJm
zM$I{<FyY6gJ*^YlH5C{BQwS{KZ7g2CD>QXkCLbG9X7A)B=C2BSe_cwfW?Fak<X`r6
zi&;-8^au%Y97xf<a=PW<$1h837F}dn;vHtS<*ZU<qTGeLD{EA)eg8gxfpddJk73BK
zgRZLO2bcW5v0<{&dJT)qJ-04&a68VIe`jl5djFjDi4B_Nn$zoLzd_e8Iwb0d1aFX?
zxu<B!?=76utx6S7%Vc;vF$H)=Oi6ACViR9|#zH&LMDtD1=FZk{u}t3=ZSp*QtmyO;
z8E#(nE`K{Augz>xUZ$%3{VH2CpO-iO4rI~L+IB$E=w-;Zj{EPXXa!XWNFF_vu+~>L
zW@)%p9)m%sd!xXe!zS*I`LcbQ4Ci;KH?kD6H05mA+0ZuQmUVNa&6$u4wo5Co@-<qk
zI4k`Y`1?J))x>eqD*n%6H`i@g;iarp7#i@Spwe#BheC(X6F(H(DASaDxa(2VR70(A
znx@Udbx#)xloauN*uuc|W?8_^H#?+rOcrtm@wmjDj+*w=>(Ey10*B6Pc8;HUWCApf
zXVi-efg8*`3=%b}JC>KQ2);Ueb7C7$>}HO1t|R^Lf>M*RRNCB<l2vY{8oZtuaebrk
zo{fjyth}<-Pg&(1>zsBuVYi1=`9$XyWB<b+0xIPaGK2~QU*wqm{bOYuUM6{N!Ujnv
z%d%y92PZW)1}HBMDP>YrF8?;y#a&o^s>RBZ3qm5wHxd`uT2_07>T;S1=cw)RDVX_S
z)3uHVMh9~vw`@0b%T?R6Mx1f+ez$*iUEXI4w!F;k_Tex0>g?)Blh}K2=j}s&+}66U
zwrsc@#6ETEL3^&iM+~R`ox9Dx&Qj-R@AH(@@Tsw(WkHt}-e|CGoGR(PHu>n*fDb1Y
zMwvK&Z*bmmU**F)A^q6>0_ot&p3PxVJe#cHyO0INi(2m$6ngfARqC&IRAYXyK6UEF
z)@c(8jOTAW$;h%OV^h?QXYMba9GHAbCF)Gpbo2KKE``l{o2K1<$iU>eVsd5sk2_Q3
zEg#j@$)_ze*k?3tozvC{@25vpv^U%nNLf8OiX}Pd&pVGc`_q3}H+8xNe^OZc_RJ}^
z$J-gsP5NihDkkwz&0g(-3D=U_0&HIm^Del4`p@>4;jOgr+s!M5rmCLl_{w-c<PP_w
zHE&lq>xx_qU-xA%hfOh4Z?V9dGH-*RH^mtRv*dRgD9(K-$@zDK`OKuWb$V}FnBH0?
zzMsFM@y+XS#x>^4HRIn-JoJk(QnmC|vuDSzpqcBW4o%RLTo{|Qcxqz&%sHtpD|3%-
zS+>E;Fmkccc~`lh)$#2X;Od=O!N^WiM@=B&c7_+@-ARjlG~dZ*#032jj1+VVN%Wn@
z$L6+qaztcJ`XU8h-!hF|2}_ms-q>;Aa!hppq#E&yF2al-WBcl^1O{#oUCncJ^Xa$=
zLEqJuUVAu2NzBkQ%y!K&<8zC1m%O+W`zd&ykA-nrXW+XnmOVYiTP9`i&Y0ukWSy%s
zNBe?oBG==WVX4yt6js;el``?apR31dXtaK+nOVU6cM~URTfWtJB7elgXvdA3k}Ljf
z>s89nZEj=P<n?>!;byMwvrjlYt~u*|DNwfa?=dH}-5!i5!UZR%#JMrQdA8|CpFq`9
zmCN3ful7B0+jB?dhW4S5&H4JNZx7!#IAYPpb-2>RMXpms)pe$N*;?28w*;3LNl)jx
zbvk)hT%B?jI8SjjBo;+a-K99`bhC$%)-{%=^Lx6Nr!Xrq_ln8Qe0R<Fkxy#11Dm<V
zTc!Q{*0qPZ@7~C?bKvsb@8Ge&`o;v8&BjqlB0KM0<XU^_1@HFUCp=5dxD!^myjj%l
z6gA_8`H43cDQ^yhhtD<r9g<(w{Qvs|(St0LN(z=7_OLO3A;HXLrt)*4|Fw2SrK#0l
z9N%tgED8Q|@7&3-n{Rn{msa#&PU~~A?M;~#`blWU(_#_Vst1Q&Uff%B;m?AK{fyhR
z#oU**AMqAzu@~rYc(U=XWg^Ej&9$3qE9QsYO48bTzo6`m?GeS3ua3X2HEk~8X?>>R
zt5R`3h=0Zx#w*-kK8qaF*<e~EIcbVqFWa|;j0dD!ooA*qU72ND?h?M8O-t`<TfMv*
zIE;82`Wno9oHotaqax_m`f2Ut`3`rOlw4a37hPyBx@6FyQ|YJHd1HBJ34{Bw3<YP7
zV3#A(oG#xomo+(rZN0perD})goLiS=9qf;1SGqpny>7Z^QS6jd4=J^BYtfmlvrZqd
zZn*R2Vfl;+Tv9srzTX~9yqWr(Q7Li9zS(SVHht_C(tdd8LG3NyraunsQkgZPzSZIj
zQ{<05P~vdUNYOLg!Sids<0AphwpVOc5kI!bD+VQK_NKqAzI@@#yG`zek!nh!>~DX3
z-B%Nmelnv^G`z9SzOgCFFpDYk^$CtE%a)6Y2`vwNVtp?q<i+7_>jSl#ofV8CtWPU_
z_noyoSZ=f9!P6%u1k`DqdFOXDL;X2Nw-fV4q2;a_71E9S8TT;CFf=R>y!iEnauV0x
zS=MFu*PrLT6QFMJWXs8=rBUu(^D`pU{=arnbW=D!ZOZhso^}a0b{i{Igl=N5km~v<
zYCr3#*1Q#$f<6RhNpE+{w$qUJ?yt#CdTA}#xo7eA3JzPVgD1DlYo5^dpKHqcj^~d3
z(@w}~^KY{7kxO*EtNc~5;(%{rkjLi5DaU&<MN>-N{r3Iou>03G!wFrd?e=!=bNlHT
zDaa$SPUC}n5gU&)Px!I>vQCqDiVr9mXSK8~a#ct*46#$*?fQR0Y5aq|3qvJ4J+^HM
zznG@ny58zf!3y@&Ozuxl0^NdwUztofS1i!(dOx5yN%?tbYvsJA6yv$epZ7H!&cE`u
z>snj;)|yqyaR)XuWf@#Be|DqUS;s%tKqg-r+zwoP=Vc(D3+EM!i>Yz9HrRGYPIs=7
zEi^q+-?)(XFtg0VEh{^dcPzWIGHb$dw@FirCHdyhl(9?l743@FSidzku9DZR<nOtB
z5w{Bsk~We$w}pBySlu-ZlPru~GjYRb4z3RW4!z8iZ3ldNqBzuL+-9zNkv~VLcEMeN
zw!@*j0**a?C88EABbSkzyvJ06sk~B<<yMSbicye=@KT$#xwCe%N9wxm5qXi5x$?6A
zqI0|7J&0{y<1h5q&R1^Ana!F<H?(Fhixj=bP*xXcbynhK%apuH@7On;Z@pr9+JHlD
zf#}KmyKXg3Q(l*S`NPF2H*TL$J*?&N>;ArozgsVTDL&CX=ZHMJ^WKZ)`}ePT(A~5+
z|Ea^_>0wVlxH$42v5P-(>|4CcF;G+4o{^zl+U?5g2&W>SO`9V{_Aa>YDIm!DfA49{
zNqZ!9gPz>?JJmTslIPHri02d1J#u74oR$R4kujd@a!Ao-?S`}24?JckZ7q>{s#eV1
z*YEedk?-MzhbpIHRqJ!Vyvz~VtNkW8DMaMT#Rq$h!c*iWzSeK??9`E1-~7;}Z03R$
z=|u+(8s;yG=O|wC)pyZ~-HXynziKAMw`+OtJ>t>Eam@9sic*r{(f;7W8;rMgELtM*
zcz@2ybICfIhD|n+%<kP&EcP&5oP15<68AADf!1IRKhE-vM~m9#Sy)*~Hkbzp3TjMh
z-g1QFs>;DjxjFlP7X4ynwQp>ixsbnrd5iLnh%~G4$tH9A!awZxO8LsYM$9+q)BSdD
zNEOylvfPQ6eW%6E`5L$Tyk0h5uAaoW@3*hjML{Do(`lb~t1xm-dm<@!pH<?XG2?*%
zUZoov+FVEY`<suS>sde5b85=mM6dfz-4=5Mm#$m#gT+N<l9=jc&cICZc8T%{G1A&1
z0X>&oSbUaxC-}P=7kJsUSVgvI83pcOXfIpG`X^*!$mEZk+a+UDZR}Di#pG8Sx0khc
z7P~JIJt`0)mXUl+(#qNJR@j99K`hKX%Y8nb^PI&dxyoU?G_xt^7dzGT`3vsq6!t7P
z3XEF1J$rxY1(Ev6*Ob&E-+cIT^yHD0o6DX|dn{YtDYjPGT*|bUA*t&r)690+MSA8z
zKaTvK=*Owi?yIz>t9f2@&%8~cFaK%fv)V@JJ<XZAswuw19~wTdj(XXx?#|`bl}Rw0
z(0Pi}GWM6Kjlry77nbXCE4Fs^GhM&+((LNYo--34UY+Zzsru}$gu{hAAwG|d$37H$
z&hoc9RIVlPqA4XPG|wVrPoR#X@Qb)p32Z*a4%cMZo=sY+c+`CMlE)907RD}!_nx4?
z@brK7Ns8?SM>4e-Pqy)#tG0PvxN$jy^TlLE$Ffx)UI<!k*%PSrOeL4mw0@#v)YeBP
zpKt7*w=lqrMKxb?jm0_{qfI5U2mRhUEHm`bF4_2WYf8(TWr8c$<n<(p@09xbZRZP}
z1r2S}=JCbPa|+d8qGsiun0upO&E1unqC2K4O*&)T`Eb*!Jm;x(zfMJ6cl>zm*p>xf
zmlbm}ij^(<?69Y~V8gTBC3|YW%#6A|tF+*cRStCi$bonH*#d`*eLOC{PYs;jS=>Kx
z>7t1Rk9LmKMuVkZ6E1tW<Qd&qA8;$kW07|0a?WX77TjwB1U*>glpnB0<*9S3ODT8H
z`eZQU#BGCX*;{TfoVF>FusCY}{UeL#+d`g$YowojG&s}n*FtCVOA!;F6N=w8f}}3B
z)XjK5;q{*N1xpwXK9zA1w&}X#EXk*5x8$z7)f3q%6UA9M{iKu^nEX2Z&Sce+TNByZ
zCvqM44XYC|f84fYh0erHe<f6oi$%{qna*DMFZ{+NEtTZmnuhGfZXdq1p01vJYv=nW
zl}mFgSzL~}b?~Gztt|b%l-*r+g{pfr%iMmeDKpa!&(L<dX3wY{=(&b>(x(8`!^v@>
zyXLswoM#=Px}OhnoC=%6ZO_dw0yiqpi#=|(<I>wxw+|`&HhFD2=|`==F11s89tY*0
zP~_aI@hDIEpz@)WLEUCL^FQj&IeXAV`-PcoE6+uC!Q7f3^LrdhMCJz_NZ69|zVBb-
zx0^j0x7B^q7uw9><F0D3j1P}K)GMvB_`dw3gr^UrA{}kDRR6}Z%zbftzN-83BN2X1
zex2W*MZVe2^03kE&KKWab5`$`d*&WH1Vxi%+^nqEzLsVVx~P6hbAP@_vB8_kJZ3%5
zH1-%-%1YH9E1hyY@X=hRq^}*ui=^JhGHupW;Nf=iaqB3TdTJbd?1)3l;SB3n`$Q(4
zNtDp)>DySOdv|J6h5n;ysV_Dauj#Q9Sk~Be(a|BFN2dAXhD9g$UAbHP^_#-eWy??g
z{%w5w@8=U=Aj_mS)qIqCsl~%<WvqE!{zT+vrB9(tt6tXRPIXr^ovM3k=OiPa8I>l#
z^py6ACe3}vm14_t^r>h_#~<HsPbVt*{b;>ZJa=ZziZ#iHjAq8)KdrJbv*nG#Qs=Dh
z9aSApu}_xyy+7r0Y`#q*lkqIcj04=pU)a2Kg)VQM%%Jp2%cDNY;GRoiQ5(ywzAqkr
zff5V3)MQRLT=U&>@ZO>C>TQW!cU6rGr0%w<F+5VfTqV(a!}{X1t+HVW)1&{#rTE`H
zFOb^6|MG&o!Bao+dy~&x_EZm+x$B{-dGu9^@rm}JBCW|KX)o+c4)yhJG}l}$t*axs
zc6y)J!b_qH?manuzPYsG;H*DQzXF5GDqmlkBNQOuI7!WyHOfauY~pl@%2-3I>+78t
zgKDw)j10+7WP7C6pEz@z+nGg(ZTeeh52JKFp=bwR(|0H8K7<6CCAf7-svcUCa!yM2
zhxw*Ew!g9#7H&^6He9x2#&(A|`@kb_dzd!e+z{Qzv+1Fb_v+0`r^Tz<!ipAK=P})t
z-t4qEOf&F(i$#gmoSErrciogqR7ww|x`b%HKNBVMvbyU+mCzrvUqNA63A+{c&9}7l
zdgCqce=8?kD0BL}gQ|1O+#Wt#v&ctWmv`TcOl^%j<$C$fX&k{VdTzQ2Yi7J#)AwC$
zX2MSINspGC?NC}EEv%p5$md)w{bb_%sZMGh%d{WQn()i%32%9rMere8Id#Fa9}1f@
z1YXIn3VfcLk<L)$wpp<L-D5w;$={A;3#x`aoDj29H+aW~?f+S4KvU72d75tCR%%+o
zF<fffb=ZShGh0n&TkEzd#=EfuKXZ5HP&0ZO@Gtd~|6v9{y_d=fCqBH*+hDUyUpe+u
z&Jnr%ExVUBnL7T^V%~Y^GV7|4W89oRZpl+Ll_P=|njAQJ|B}qP87Cy?^#(aM<)lP5
zvQ%C%ue|bp(caYSq4SsBiw<>|=CkSOi|xKQyH?t8SmzrDK1!O~bRvzxIQQ~N_vbVB
zZSvBHI+7}W>$WnFd5hxRBg;<Z?Jq2mJ>|DlPh;MZ;7qB*;uaep357nHq|dIJ)$jP`
zw`APcX(j6xwVi&tPr7f$mphHi?)m;(HEAQOEQ{kgTdvr9cl8AI?_?~9i23%mw`i&z
zZ$SOh2RSk>O9U^NHAVh9@l<i4oNL*Bfu-P{J&QwVg~sKtmD3+sy?6RI<LR8^ENzz@
zJcYFStptpg^7ko~rYQ?oPb-j)+SAdK9BL!?GB4?e?S+;1oSpv_O(;w*OWp9LP(aF6
zWTjCj4^Q5q;uY+3&0aou@q~{xnm;$z({pLN%*rdNHyh7<;W+Cu({)LR{P&(CJ5Q?h
z70Xo}?J(GS<?TI@H_TghYT1aCOgnt?zQX61-u}GmOMR5$9nY*@61XXnQFm(5l%P(B
z6Fa7xw7qxw<ae)Hsy6plfar^|rec+gzXT)S&j`Jty0x%t^MlIJ%zeTRW{Dymnlqc`
zYwW7Ls^xz7h|t>$A7cX^E!vkav2d?_*~~r0_oZsyy;N*?BJki(;q`?<5{fM=ALPwA
zEmko#xoqG4Ht<#=d*+797%!=t$EBG59B#1enh{*LSCut^E9J6K?{SgGre>0S(z-XC
zE_7~~Jb#g-#NydcX8%c8pq{hDqR++lgZIIw@imG}8;>OhdPLn?ax_@)m&RmX%b-)b
z!ZOB8*(?|Dn9S)kVYhsyt+??GFTX&M=6#Q8O9VT%{5w`2Ip@_vYt?-3+uOCe_qa^e
zUGly5pz+^Jz0)-2X|7)&Hua~DwBHuqq~~(h<;z!axV<V#FycM-#F&}&zCq_gLA#Qv
zA`+#G&)FEYURpJ|UA|<=f|ZBAi}+b<daPSiWRbFL1$Sr6^pYEaH!hxUle0ckx$tz1
z`}NfsLUT@@ceoWSqcM@wUCFrQi*D%BCvBz`vO;Q-Uma{F=XXr-(EbzO5eR7pEqGkK
zIFKdg$*H#-^RAc9-XVKgn~(8-*@W{Y9{E2T8M(CAUT5m5+07f@y0l{Hgk|$3%%9B&
zH#3g%HoyCR=Cf;#u|X+j)wiamKAY#!Q4@M#mr<74(ai==6FdadR-4aw^tR`!`(>9?
z{4qwdlI**RZm35kUl5xc#5<3vqwa+jE5{^t$y5;@O(*le8rywbYZi$dJnq_Tq%06}
zOpu?2X(!v&UCLJ$Di-KGJyhhq=E|9-ImZ^Vg_P}P5!|O@dEsQ0)U!LH=ACxsYBSEC
z;S6!PT>f*X+SgerD_wXq7Hyv1^tNfq?k^`i%-ogTrwgm)bp2X+j@3yhYkzOb*=05^
z5f6WFo|G3L+j!wiuao2&f#&Jm{Lde0x*pMYnQz`;-XPDI;Pv>a(5k#+t3P&hhfKQC
z`Dl)Z)yxOiru;rFx+8OPDN7I6iQLj;TaMK))#Xz*KXlHxG<m0ipRjY=8zs)JI6w1}
z4~acb7aDjjV8}f3`CV7w(RFDi3=?);l6WYvCn~KdZTY9Om7?V=^5q5#9qp_ha0_O3
zEz_ABkh&swBHIm>IJY>KWtI*9`qECf{<5@~Y4+3oo1;?44B4MO!pc()aIXvwn3UQP
zFlUmY8edb9(dCrcD^+{7J(X+SLxiR0IUi$qSRsEb;ALp$dr3{fX+5kmxsKPC?AsDC
z!DP?s44(BnwzcgJTI9E6|GTz;7X`E4tZolTY`tl|sPBoA<OP8*yOlIm<{z7PDJH<}
zYDH_NKx3#`+HR+&*i(gGfA!9Qwl@FJWcX5`-E-PYxvyw)%%as|yZ6d&Dv(^c{`y3n
z)q%6Bk8k(jGYPD_XM8+h+B|+)F_8%+p^}-hQ?F0_#(H6?Q)kwv<!_RvEQpD5T($W`
z^lopT;1w^9EEaM!DlBCE?7KR>EwrV7=?tIDJ8l&dCt7a`FqU{CuyU5j6Rjt5t}j^6
zYzkee-McB_mBG=_wuRT9bk1n+wcxkZYEw0ADmixfin!aPwO1amI(+)7Tb*=Zh4UV<
z6R$S=ET7yWq`Y-g$qehg%LEJsmnpY*>+(;sJMUIzbvQu%64&G|ms43wKKnTHKiYDm
za8;V*N5jiU=gzbIa_VT};&9icEQh8Re$8CQv9~x(E_Y_pr&ul1<v~Sx0{#5aOUf6W
z0VSKjSw>gPB9qxyGCe&JoY}tc$)VYr-nwVCr`XtD^bD|+Ovz3=74h1n`ks$vwr<aX
zTZgWmS+O$lh(l_Ko%-3Vq^E97{!d?-sBT(5TWRsES&y=tV}cqroC{puobm{Cb3elM
zSEhwy&ob?o>U@hUj8*@)xGymB_?8jmb8EHEf}1XFE~?fNCr-sa(c(L7Q?yWzahp)f
zu9(ZhYqhjJm&#V!-Tu(P|LdVgW+t;xPWZ7%vyyeLR>@Bpd!1bmlxl9;U$XK0Def#U
z9<GZu@Anlhm~b&WR)RC^y`QI(Z={pr)pPA`;tNxL%;Iui|7GE{+`9frKMkk)U-=YK
zzEq_2r+%c|@p;J?mH01f%vPM~Z}{-&Kk12YK=Xg;3^P7`yp+FuqO_mUN#9FOrE1?!
zT=#Ce{L`VLTBC5iW^w+)7u$TSd|pi9n6b&YOOY>W!sCP~uEs|-)T1s09u4#4nkmY&
zux~>9B?(SZ$q5s8^r~s7%r6prKWSc(g^QBV&Z&-$bDm%KYZG4-um1myiKYB9)Ak1|
zCkwoMR`HQTA!5efSy~4Tu1%2hjbVMN|7@a|Wz<tyw>q!oS`Ef!A2%HikY5)xr#D#f
zj>#<}B@ao~E9PmLpDd5cty<cf&D<L-;aJ#ym*@G7B?o$|F3Z$R>oO}UeZYE^SxSLj
zTgXa;!^4P0Z;H&pS1OABENl8Rocyo01+RHF_4#_C(pL&M3tMlj@^K2j)SG2*rg-L4
z>ino%3s?M1&G-sw9+yS7iDvFt#j&8qKE;2+%U<@)-FfO4K5n>RWiTP|ZqZa(X<q3|
z1&1W_el1y|yzs2wWtWe)6++6Trc|=$^BlaDVd2jG#K+&S_u3cZ!^NlP#NIk|O4@0G
zFkj4f>%b<#U8g@qC&}Cho1rwpqW0okv%61EtmfQwIO5h!t%I_U*&FYq-pkt)Bf&gX
zA~L0I;lhc$N<2>U4&<#4DeF8oWz*4+LyD7QDk^NOY~mC@Es(cKPB5&VBa=Dv&I`Xd
z%MZ4v1-1k^JvDmO{<$xB;o9)fBOk6;ZrR=UZRYE`<d{1ii)S6Vz~Xjs_U8p5&#ESy
ztlRu>gUZGuA!nDI4lLXrw5PY`#v9+_(73ZsS6L_3e9D<p`G4K_l`ReMMh(N64-St#
zV!2mpuMstJn&QM^XZy9$!DFgk)|Iy*8JA)_Lwqf-zL+Nd(syn~@^#s#nR61ABaN+j
zip;f4QX4MDUi=zURul3<@qlbI58I_bv-rLyvpl|Cvh{xZ@p|Q##;5)VzWR5s!q27a
z$aaZ*<>(lBn>oS0oaY~ic2%=1x+rEDDt0GcWp$&`j!A{7YMWMwoa*wEW_ur-d6UtZ
zw@QcgS^M)3htK&qiF`M&YQ1D^c5oTLd5@U8`Ht%fYMmz|HXj#1(ix*UC9SI3Rp;e0
z)!ttzleetAa6@R*qByplDkozur*3#$b??p-weuBBjdHUmUfQIgHjBq;l3*tH;cI_6
z<^;T8YGZD`BE9^!-tOwj!Gb)od0jux&GvX|QYY^Q9blgH#M2{>``txP!4;DptvcY<
zBUzIq6O@0fQAvL0%;Tzt30s41O}H0!<G!MLx?=tA#gbeXHgTlylVSTfX{X_gzb=U;
ze~&-?obfw1MS1a0AKgh)Py9Hlxz?Ree%IO0Efd?VdR4XF2F>}%s=>M-a%a(+QtPKi
zN&hwXs<m1yP%^l05FT+ruS)Uhr^El3Pd53btXg$I{8s5Bg*PtSR0Ef<nB3H7J=0~?
zQpYzds}?+KOjo-vIm!CV^pF(p<R!fm*OaAhX|UmQJbAU<(dX&S-q&oq3#|@ouc^7+
zufZ~9Px)f`8-FgJ-tWk`PQk!NC$=z8Qa}61`$GqF+)Ef(qu)PY^T@VE&6nA-VCjqE
zqg=mREx@OpIWvd^o4Rt@cYIm5qqx^8S@T-V!49pb5+05|iOXD`xe7JA1U4>OHR1YX
zbN7IdOpE#KUOGx!TMKp+rX{3lywaGqcD`UqYRS>ZH$3)m8}SwJY+90d;iaQlhLwBw
z#8rPXrk;7XK)Y*kY~<EMsrw4AtPx7B+URXE_bJ0;7B}`P`LO9<&bF+vG4j1_a7^Xx
z#I=3=kEO3)$Y@$0eXfI_PqU_}!||7c+TuHj3MbZx6|lSR%e<NXTOeVo;=31|@0p%n
zb-Wk(Xr*s^vC7@_6%~?Zrwn8?4cYA9nF*~nJr&!Q%6T}<utCBp@nB!7vyIQJlj1oR
zPc%|(=DN(f+hvg}Y2x|h{0A$)ncclpN*x~_aC)?IUglZhd(R$&8heYE9N)6W*+Dlm
z_RiC}2G3LH*k5#re5=ag5&KfL_Jh~Q4H|6z*DtSNH!27`>-wYSMsdfbRMy^4WfQ!k
zGJX`!5V${ev*N{FHxDl;6_J0Oee0n?xuNfU?ZfpuddmJ7Twa)a#!Xe8_3;#yjydwz
zER23H3psIE;r^sW8KNsEOi9{wK1crG>@Pb1&wJgLO%uG!)bGLOC-cm9tE}VJsX7ZA
ze<(WI1X(!m@MW9N)lhX}X9ZK>G@}^ncFus13k>tbM0edz)Uh};<KnwiiSGrP8&q$d
zTfoS-o?*_rJL){)K9-@ETG==Kb277@C?r1TaY!{A*Hzx-8Gc)`WKXaCI+f4)n7uU9
zMCp7zuI`(~K?g2={{HYPXc+-N;{@9|r~kbDvHfuy-}=*C2ZD<vmruAV`dHLcL15nP
zne#4szBQlqH#A^j+wSV-xlWB&4z};9dZG|;XELu+^y3XL>hwg_LuUKv+|(5HEPGI@
zaH%yTIKl4JW>GCcKOdv)J1>oI&Cy%z6C1z!QO{-T=NI&i6u6IUyxS4r$d(oTBmJ4=
zF~b{A-g6q7sqXSM<SLuXzB?&H@qCNOHbcYZF$<gh7eBSIxhyMq$bC!M3=yLu&bz6a
z4mV#!UVa=BmZ7PA*+Aot&+`j{>n7&8FwR>q?fk^4|F#Gh_raEl&K_dVBUgp=Szn%8
zu03br=`R~kO;Zavd%D16lVPAlNmV1;%c;$G`*h9*P5SOA`}6y%d+~`cMKmPW`k8Ay
znR@@~z6H<!=qEh}1=4(m1#b=hK6Gy?+QAUEEAM+m<EQ;gf_J|$^I5gVXST=*)4mpq
zb1d0gbQ<PJJ(ClhxM8=S?!{NGayno1QaMDWLK-qvvbT$J)N?E8sBXHsuDSV~#>U>3
zMBj_+XWVI9rmEKzB4Kb;wX)T0vgBQ#Qw}$B4GhGbdJbGV!gpRIC2GR`AeDugYUk%{
zDfo7Gay(bNRI2Y`)4VxmO;eq-x!y&;VzOF(U27xX)i%9UhA9FH3zC*FpKn}LU}9o;
zpX;T1Qupa8G4@{%I!8H5npj5fE4{w4C17&k#@r(lj2Csj)3&z!ea5Hg-4$o^H2qtd
zc~2@Fm-4Jt{X0qEkL<=$={=s>i%pFT3?^GU=1$uEfY)=*&f~LG5{xW$wwEtD3u^7L
zIxL)I<e}yFYx;}H_7(|?O<DFhcrrQYXh`!H2~A0#a6&oe#oR5k_CHZ{)zjF1Kw06`
zf7Vr3=I-lEihRFx6Jw*4%ku1`BGt47KTq!CFpf#w_N2j6y~bk7`zK{5`_wE&Lp!HV
z+VRfmW)o9mL+Cp*VatozH4<lMzA;$o{zNmRiD_@oo#HKP`o4Tyx#e`e$MpsQgEJ}e
zqI+B9qIf^L2;B*ssPCCJi)VxO|B%(YR6lIHa9?>Xzvxagg9gdBirhL;Zf?)!8ZAwf
zPPr2MEntH6b;Zl`OO0i-&RkYIAKRw1A#uJ+x9p9^qNj6rUE+26t$eby>94oLfo4Hj
zbAux0934sd)fd##oVU#vJScQ=!pamr6OKiy=~K^Ne($^t6gu*Z42lhRYj#vO`1)*7
z?mm<EzNh<8MvYH<=JDJ;?OHdr^Ph4aY4NitD&C)5*)w^2nOFPDokI0}JWP$2^@*p{
zH}8yf^cQ}6&{#K(D|C4)bL+LOn_pPx3ve9g{mjjNxG&=Q;r39$|5B-^w|J_rJ>#ho
zxFk@mSWo%rk>6~_uX}W+c*Gp*oE4*Aezd-5SFYQsKZ&dIW^UPV@$}J`>0M72Fs5G6
z6iK~5FT?QAd}Xor!1|L3lc&7;@_O?LE>~qG%XRFYf_!uGCe1dwW60pOr0(Os8=lWR
z)lNIj*pi-_xlOvVdZl1ngVY|=8!2<*ro1;!=-VRW$Ws+0k)&3m)v>+9JZOrUrPf@R
z=BG8!)iM~nSEQ|*y-4{Iv+^q=m+S8jgF|ROqrl66nNJSsa<a|P<O-;WewS9-C}?aH
zaYAB-Tt{Klr3qyYyvs~J{T6((LGxwhQ&wxH(%mYG4?8q^?`d{6rL@Ky9od@oCAY^Y
zwOeKSDKY0~n~dhxZa0$u*5SO(r_;J+tJ?ct^YWA?yG=52o7tIS$N!P@$GH=8xTjpb
zv?oa2_rc2M2@O5h&3g<=f8FJt<0I-c|FF88(PNigH`WWxHMu0RLGIeq)|)-nhdEoD
zjM&qK7wcO8Ffp04E9QdJZ1rv*hbf<*$0_qP=V}Y}cBH7UQe4ToOmh3>AF>xuO_5J=
zX=Pk{t@?hZ^JLLY2Qt-fo-BR(NJGscNiV@|YEhoW9M7wcieXo;t)6m6{NB%$Z;=)&
z1okgF|A&9k15m*u&-h~17QLTshUI-*YL|UwR{9>;k?^IXPw0G8$}6V3@Ar6~YiSkP
zb}`lYWyXYlhRV!EQ!07Sy!fB|{`NH)&%(U-e|7e8ocr)kf7|4()0jG%->zBpZk=s?
zovH0Zo@7QRj)lrGjOK5GPaE{D%50uk*3b5H>qVt!AujAbH&^YLDYT1w$&~qv-^8E2
zvE$3diBgKOKZWH@z5ZT);u2A~IQ*@Gn+_9q<HfetcF&8Y^^?z;UF>}4Se`HD^!mQ2
zs^6U#=TH8*7a7%^Q@VPt)nxIErA3QRhTM>7diAj9?aVh9FPE}Di^+N#>7pNS!+jaU
z%)+JfmS6U0I9=7Vs{Z7v@)L(%R9x4r<~nrf4RgU<$3>nM(u*!+JvMAj%I=!UwC(py
z!5s_yicEYK*YYy&O<s{cz0}gma{i&W6Zh%OJHGjP=Jl(-TMdK`O#EJ+dFRogf4g|U
zGCyEHP|tEe<Z4%QW2S+lP`2x~&3dv?kFGKGX6;u!{O6>I+SIrgz3&uyOL@&K|L1*j
zX-seRX?tO}>0a8D`EgrTc<tz}FP!0{YN8~XdhB(0&YMS{?p$s>oN!}`b@k$~<Cd%s
z*u_c;eCk$oxPO$8x+S4B<!NkQUhB+cSB)+ap-yw*BRAMrl{$SlTkrGx?v<riRk^Nk
zs(diW5S0@CQ>8L#EyvuIy_;5Ts;~ZNWVvyL4eMz?mWt!~vRju`9Dn<S`+H^bDk0JJ
zF6R!#wX6tyzAw<_{*0yEKRth_Z}svN3qGn^`pj!?#e`hNA`373{GaTCOP@SB@Yd0M
z0oTmKFZ){7962-Xa%(fsM~1j}OH9)COi2%1kUsJD=E;G7+LISOo*$i1yK1w=Kl7ZW
z@(lJ2|AgB=D(SO^vD6)K@3lLvyE(|wWmnX_T@ORqS}U7m{Y@+6>^AqY)Hp|omoGkd
zU`^582DRl@RdY?*{vMEJYImP#C1!T$jL3D)q{1c77acb|xIs`>Nk?Acu2YDWxcwee
ztMIdz4p!f~H0RgS=}}AGb9P-)%~-P5PJqvpegA1M@mE?~i>L2ey<?J5(+sP{dAseF
zX;0BGQIpI!-)NZ@9p0v{#-DfQV`vB4@jG2{2Oh3+|H_;Ej`>OHli7(u!eM6$PMph7
z{kq2G?y?n9pH8i6c_!`cXDr!pRrj8^%<0Zc+Nr!X=Au`kM9;jgV87Y$@62MInVvy`
zGj{kS91A=y*)A}3PSAA~iRsDvQ$v1TE01t2dl=z6BZEib_3KGXy?(#mcA39Hp5Z?)
zL+XmGFUPz;9c%7fIs4F*JeN&d^v+yquvq$@!|0gi^RDy{cbgsOTyg%g#(&b5Lu#fR
zy?JZB9xP&bb|!S8_qtZcSyM}kW?Y^ccgQQAb#cbQ%@enlXl^-mYR2j_<pn{R(!n>h
zc2CvvcM{8+@<{%s=V7P0ei~96OITcUwjSU;df35XzBh~RDkIw)Jh^*6Nq%(6m6*9|
z#>Li`;U^U){%d*ew<M_f+@;XV39Y^v@*=VMRj*X0_nn*;STWIIZ>E;-ulaxXJ7_C<
zCD}+FUKKULsk_l^!i1;U?@Je+Shd6Ltqs59ytcTBJNo8ChE6o{-5=>G_(D|bWyiVC
zPrSKKoY>kb-QxGjSINP{|L4+ALepmE?6|*J%ha<`<JW65Sn&MfP~ItM$|87Bd1wA#
z{cFch{ArK;su3ajEBs2jT$lfp*lG4I3+^*-OW*#$HS>b(p{;>pE0nC=Q&ZxD-5j!1
zCKw3zsrOIFlwGT{J31scqKk>+OUR`r_CpK%Jh!coVt*XYv%)BL;T!e6M{RvGKKL#9
z^D=ZzkKMDl{fExXy6dfQD9=xmLs~@p#P_QKi~0J>gfw1ipR96wDk-s4IlbER{hH9{
z#;2uD*PWhq!Aj6K$X%~g#ruwNd}pe;)f$@_stuYe15Sx(UkDew(WYw|{5t-@=3be%
zy;%ZQDNc`Es~TMXAD#8>a_m-9!?#VD_6AepJ}Djg)%(V1X6YN7(+M`Vm#>x__|YSD
zb6HmNAN^d5$qLpdKkyyvcz4}-y6E}c2jX|s*D?HI_>s?`vS~lp=d`pv^WN`XSCPE8
z*Yu2;Th@X09UA+Do_S~~M05)=%A~cOSurQNJ#E1z37_c)vO^{X=OjPnDD-0S(_Q!H
zn^pP!kmbe>ys?r#aS!KQ@^-3dY1p*R=uQ9A%&paOK5xB(7l!wqcsI3u3UknmFOQc+
zoUxG13t8`*_ac({Qb%Q(LGHrl`Mr<oR~&lrN>NtmZ-OTSTj32GgBR@`pPOWzTk0!{
zzlL>sb1Zr4sqA<u@`3qJ&eF;q8u!<zesl7!+`0Wg&}EZ%PCpOIgw)n9bgEalFYz;J
z@oMYhV_HlnPD-BmY;e-#=_84dt;^5FJ>B$f3d7Qthu1G3Wy(6j^r*Hd&C0|-x?<t2
z<F88&2}&;g>-4Lvs_TF4?`P}>_#6JSG+YvHl0B`vph>$a*fEzeRi7!(XtVy#=W)Vk
zE<bz}sFN~5@I{<aPQVn-SEfM)ArIyUb7{#j6qL11Vx0M5j*O7@Du*wRPJL3(<WX<Z
z_L<)K@>{@)<Z0!ui|oDA)vh-2TNb!$?tSs@UB$<BTP6fvef3?1VWLvO^Z7oqCr<ok
z-)t8;$01<Fl)rO4Ii9?nxOm#74gAah|975uYhLELiEkz(+FE2Eh><M0p|DD675mzS
z6Yjq%sMZXb=zQ>?PFJl-;J=IWj$b(JbKdpX(G?1%uVxf{a<ObpSvRZbpptfK*o4<+
zo`*mE`Ru>Wq<3B1O2OJkx{tXpnp`rA(t8ld^e5V@QAc(6mY>lfC)Sm3I$*-L^x|v#
z;J=^0znu*(qoo=C26-_GO8K!abJ2<jtQHdTzv^m#$unZsNhj6PiIHnAnDfuH-NAd(
z!#Meba9QEKZxaqLd2mRpIBk>05BAz;3hcAW8MEJfiaNL8myWtc*RD>x=a$m*a|#ya
zO$>F|Ya?A*UcSs;RG-W9r;pN-@E6(Ul21iW9%50Rrx|DV%FE2MC4_J7YCEZ$0wU&X
zlI2gnTDp%@=&J7Cjw!E9J*79Fob2{pY{u6ar7D`Cj*nhmwM?0|Nlh_NGf8##`x4hf
zj<T+Q10`p3%6@D+(#h3xv*e;j+KmbJJG;uYA{h_=y{op?#;C(J&RS>wnorT+?x$7s
zK0fjFRFbIAymr66uVyHH)tFj6Eje|<VohJh&O%oH4vu}FPHp_#`uhbq;jl66HJx!s
zbE5Bam1z+x4=%kbZ=PLceqhbZ2#0wG)z-eSTN~qS8#3GR6@S`Wk3H<|n|3=H9QK%c
z*T_p&Mq=+p%QEL}ni+)x*H5ludvNlTkv@<4OQVwM>fFk=x!Gf8+~MzQeyKUv;M^J(
z{Z~ttJWc$+p6%IIud_+q`*(D%toKUg`s@?-qWOT)ygdf-?4b^we#M;SQ@LiZn*2=d
zQf;4m^Pkm$q6=>~KU{fI<&I6no~s{qxXL-dJYBrr;`#=A`H6z>T<2+Bi7{o%RSGda
zeBIWdUVO{*^9ytOOjO-p-Vyehsugv;|Etf=8S$0VoSNA=Go4d!SaG)6iLN@P5g4s{
z;YZ4kw}*KY|DCjw3TblT<dB@X%P7?UiW+OmtpD8R=OFc0!!eI%X@}PTshwY~>7~Cv
zvr@KPJ6>V`qE$b$<Hariq*bzt<P`oYwK)BKuB5!v&)AtuwSq;|eu)QEGnS{ji62^M
zVW8jE#+cr9V9)Z_uQJNKbLLfbtef+|Fy!Q>2k%~-5L>v!(_p8lf2aSic7u-|FK=vL
zwxE!8m-6?D+P+_Qet~oRBo_V$ol~%3r9-OjB4u62`8E!_zTBE&bwvCA!aZmA?tFG|
zqaau9N}E}aGBiG!S>-b&T=0|>VUskktP{Hu;nlUC>3FWsr_;ZA?^ezVI(Nb~pg62A
z;trpYzTe!uhzZ9-Po1^dBW3b?Pp56q6q&!~p?XqDb?2XG+;^7i=;I6&O5P~9>)XXQ
zj2jiMo|J7rmMSsh|GwTY;C3QYgP}&B-f@k@_!o<gS!MOHZJ9UKg|8-3?ogQUq&ns`
zY~PONFPfz1D62Dj@pNuS{%;DqeFL3ppRbS96P<VN*v!r&FE`|*GMIHO4D~d+R4nq)
zMN0RT|H;jF=iP4pmFMwUGAC*Ac0~(e)*J4AT?|j|%B%Tve3IZkw>@aZ{}a<>*E?*!
zKcSs}UqR5Bl<sQIgt$|;c*Uoz3sk(lJkTMo%KHCQ^RME353CREF`o8zLaD~0)X#0~
zRZVfqV#mBHz2)`9Kd>!~jx#^uEBrEGv!P47R|e0*?1jx&#E<eEid%L!Cw=Qqj!X9T
zo*8WHYV&5~x*m5g*uC1aZr&3AMc>+XN-PMPc)g}g@T#ro$|qC$-17d!J)9A^eVSYK
zr*D&gJYhe;&rq#)FHbT}$+e8-b5nY}c*8`?*;Y3^B02wQtTB`k$mQ$b(PET2QB-MP
zglF0A38(k?@%^^abYb2osUdUn#=KQp4tKw{tkKg_47pSpw8`nylJb*}=AZFyO0o*=
zsY&kBK0Za|%+kQ6s)k{w8{Av&nS|9WDc%+{mAm?D7^kCUsC4ZV*`k$p3fg&IdFw)$
zP9?t<$<<FkaiHb(itNW3a(xBU9c!CJmgcPZ(%_gh`PY<<7uIo{^j2+UQNQ~tGV|B#
zpJzq)Wqb=)QB?V^W8=A7&1>h`YESEx<}$tezGfCq3G?flRo|jk8e+)IWfD}NvS{kG
zM5oSh4YtZlH(Ql<hKsCP-nZ&I>s9{=ldV6$Xkaj&oA@H@U&5c&EFZeNZ2tc}_LK1g
z^Mn124F2sFD-Awq)P&ec$?<<s^KZAXd-{-f|3;0YCarOHOO0-HF|Me(xWxNL8=DiW
z@DC9O@$WXlEE}{HyXG;yV7b7(puF0Y(XN@H!tsmLqV1cQnO?A7;7$-Y@PI{uBk6(w
z=n6z0CJ7b;jsyc0H?(UI+m2{-1UmaL&0sO$Ob|Mt(r|LZ3>L%84$zs3X^dx>X0RGa
z3vE_N<2B??5IdmL5XsQZAm*GTcw~{H7^4nT1j`1F1i=GI9&4IboRAA=-N3crmE_mY
zjsA>loF0kK&7YyMs4o7Gyvc8Ao1e?K$9^o$s{A1>E}h2k!rnb#?ef3BJ0Gark3Z~o
zAUeR|@rC<`ID|f$Z7O`Texl+ONeN-8<?4&Hq+;s#-FtRfqiXdZ7dd&k*MGltZ;y=*
zo!PYaQ^B$9DZiTk?mRo$d8ds@#`KCq>)P(Wx@KCo{{LT-AN#K|Sf5zO;qUsY{_phV
z=Tm?ES#eqZtetV<$sdPy{!~A=+Fbk%^R8CwlIx%IwrDJJ`Ye1<GxqqOB}R-!HFx99
z_n!NEU`wXMyeIRVq-%KIbUe-cmH$xkoiM`}w#qHK^2%qvFHHF1yS}Mb{LcL7>Eaid
zFG^0_)v|r(+TXWJ^1hYwwWPfLG++80pY{X^4}-=dACBhlj>_6oe<b(6(ya}P107AY
zUaZvEnNz5?*Wz+qOo#rtEA3e&f3KfxoB8hn(;t%`)lzR>&)TrpJO0I$l`BexJQQkM
z48L&oWM#6<&dFZxXWf@~qW?;L$J(iDr{~N~QNHq>`E;#7Yy(5S*5CG>{S5zV?ygCF
z`zDypGFa-I+nI&zPPdQ8b#(vlx*abj9dz^ixqmAYgda}!-S<XTylt()q|FPQUubdN
z$Z)%)d*yEL1=TMHD*tUtinbGcusCRc!B??6UA(&&HkO7}$0zu0t?N0sSM!d))t<Eb
zTkDI>=T|>HIN`;H8y&3sUTF36E>BWECR%1G+w<33z<qlocie2LFQyxgzAZQFT@-&i
zK_~S<NJO3B^!;gloUD7EZaX@+@58U9=j^s#Y_Bb`*u1g2B%!?RZG}Z%fkv&xoSS@h
zYkGoS6dA-$<vHb-%hz6Z{ge5#*9Q+7X?XU2SN`p&D&YS(*5SdzzpkF06PN#&WBkL-
zE0fB<R^dqhx~WTFFIQa9ckT1US51OE|BFt?|FaZ!j(E{FiLr_=vvJ1t_^D6BOMhlB
zynA9|S*((dpVY6q1G_Dr?1|jx;`%|-FYaH|uXw*ImTSFdUwmV}|5m-(dwy~5bTJmy
z+Ytvf^#vLEHtnl;dCBNb%3;@K=l3_=s5oNTr=@>oviPf$+Fw(g`E>eiZcOk$@Kl!f
zCx6$p|I)lNe&6;5K52J}Keg`7xs<v_PL>z{kL+4-Tln1)IX=@J4=P&ozLgbhdHAsJ
zeZu0#8v!%Ri(bcUvUd42rQApAL6@HQDi_D{UmlnH`8M9;?yI`6dG#LCLg&v1mS5cd
z?#;fUw~1DVI{&ls%nP&l`ghq3ZJFmqPfDf=r?=$)4|#F!bAd(5?nc&pgYP|S%zJ~6
z|7;8ntNb#Nhhx_~y{JIfE54d{m3j`Fy)fS~x73$sQCp*K$}gAiyNVy}c^l_8lP~D6
zb?U>L8M$13hj=%AIG~VpN@UI!gN!fn3tk^co3f56$11{Jt@B31)O~ew%XJ>OOB_%9
z@?OePZqv-aJ2kd^HEb*_zbZLrj=<-h1DOeXSI;XrWfUEd?`OA?#m*w<aqvnt5eME<
zt@o!+Fn{@AQDW?MJz(jv*!P7o3p2ED34LVL+N8wwqHCqrsTXVCcs<jMo?fT#k{j{M
zGDYl!lJ6J8$1A!bLeDpI&Ex)S?%91Ybe^L~yU4{kR-cW-6-~aMY+p8?g)1O-FWb_5
zmHmgm#xEDS#{4|w{*)E(``!h<Ra11~f3of1#%9~^b8gysooeNpwmc*}VztGt;Ad)c
z1I{$HI<JyfioJM7+b8LWr_i&tUfg$YSW3%SO<H>=SXHuW*=MFRwz4bD9-NMQzE<1V
ze3o}`IIBUR!rg_|Ta`N9(<Z%ASK0e+|EG{h`9JR?UuIway->a6Kd;)SC0nkEYTZcq
zV4tqreyUnDs$}*HVafRdW<Oa4#BSg7J-@gwLdj=k@v6S^8^#x8yJT3;F8X|kV+NDj
zv&)&={`0)P{(F+Gbku{U?iuc#o$9OCB{J^ca7FQJ>|w9;l({Y|-0$}09adF$e`xt~
zzVYw(mpmAL9T3)<Rputvu<u5B();d5ekH#%cGoNA`+lfVitdn#d$Pu6_0NKp6VqmF
zKN<0sy;th(sgRu#0comz>Sno*v})&a=;VA%*|uK&uEOuR63YLj#J}9r%DL-b^6q`!
z?PD{3ZM|_PyTG_C;h*IFL;af0yX6cT-tFlW;r+1t_!Of8@e}7;&+~M>$ZVJp#J1$k
z#NXY{x<_KJ&ssc5=kSRMzYi5I7CWP5^fzMrx9`8}T^8~vSZ#_hpOwNUUXt)7jrEz;
zlY8;^k1m*@etJp>U!$eQ@6-Ag{ax>jGFEoondoZC`SZ-GmdCrz^?w(ID+o<9nsc$M
z`9a|Kkb868i<PGGi9F+KulOj&^V4hP9r3mA<wPtNyksovKDRibeR&7N`Hku#j2d1Z
z=9)SSUo#~viWEo*73FT%z58{ov=wXr%?ql$KkoYSu9@!_7x7T1{NcWBXI?+7P{_T(
zDpI*kR&vb|hi|NQO>?TOtYrNI^Q_!9H_z5^G`rcUe&XWN&AJD9);wq~QkhbA!&+Ko
zMwfH6f#|E=#T#-X7wl9uJEd)5KIf6;K_(rUe$Az8k66lnP`Pp9?vDRGK5B{O2~Q<Y
zbm~Mr$ToamS|(uJnH?_l^wMIZ_mf!<D_XN2Pu!Ppwc>Jkf4JcW=coRO`y!(BK5~bt
zKe~B7U*lZk_b+uK{GGuD=579+^_`3tKTezzaj&)GP{>iE!hk&PWi2AS6ZM1rF0WGE
zxJN@+=KbMY4z)6wYYH~;9$`t&liB^Zd`oT9Ro1wuZ)_87*70-xIibz08X?3rHSE;u
zl-8Mjt`i(=C(kdjm^g`Xd-Quzq1}t)y_Y>XnN@P?hSOrU8Wl_aumaz0Hx-{<e#bG9
zXWo_x6&FLVn%`ab@Wh&32bs>Sx{xpH(!F?xp*QpKXPR5ApJ{&R@eclb;Lyh=4%_>-
zk{7v8);{Na64Tq$yk_-ozQ7iR+QrM(D1Sa>&}wD0{(wn;LOf4t_MNny<*X-Vp9no=
z?7GjYmdtj!b@#L2zC~xgB`>$FxN$wYMJ@We!H4GV#zMo|H23<1E7M=Mo383=IAwlo
zp1E+K!nWkMyDH^!*5rNqbo9(hxq>@)HW)jUn6EtGt<|*kK5MR~%C|&6zpl6MA}i)Q
zmoRa!$vd-2|61EE2Af|KR62u#TDYZpFL!wAe_64(cE-IuOQbE{HSK)5{N`8dT#<W6
z9m;%*?D*b$U7UN|P{urG_reRjl`GW4vw0S`ebC$2-le(s^P%^)JEeZHd9U8n>R3^}
zhll;z>#9lZMa$E@-rXo#ptWJ2`VwP*N3Fw4bUnQ+9j#jxL<D4bnL{F!vezoCyCcE#
zwCtl_PiK$rfgbsj0p@IfUtVjd@wPgmuthrjlV{L*w|9O&TkEW>9xqlYn`FFWSxfeL
z(IdQmOwWQf!+v+Z)BI;&*xIoB-sN|b;+d>n-unnMYlvo4YF(3$Pd;sNu<Q+s&!OUr
z0$aMabLek>UwC9gT-y6I|9wB+I3Luj(Guj^BX{w?@bSele;;r+J9n0}@}=JWFh#pi
ze-U%Dd;MHa#sAswh3;8({u8g=99YyYAw5y^O!M#3I*Ep3=L$E!vkSQD>Z!3x@X}AI
zyXQ5JZWfc799OC7x%)N0-???Kj=c;y_-B{Odd9j$)}$o+`dzEpf9qK=@6u|#YNBvq
zb4{7UPK|q2$<tric5KRjKBKhIf5IEdiVv=Qg0bc?j=R=hxc<HM4Tn}A$IE%%?{-)4
zN18lb!+dbTvbJ!eHWtDC)~#P!KF-K4zr?v>?!(Xx(x%H4IlpIzm8&=TTz_!CTPScx
z>CU-Nj6U>yHz|3!f_u;GhYNh9cxINWy?kVMb7@`vwQc9*6>M@R7(TGO%=~)8k$G*q
zlT>0HLS|p-`!FxFgXztC;nR=5&io$7RrSUtd`Gy%!DA~+1-C3P@7t5N>RvbR(|KaE
z8&>yDzxsqR)ceTUwMDZ^G*){anfg-t_nVWCcD?D((&%-q&osz(5<R{6#f=w1_Gas9
zdiOPmFX=xPW2K(^X5HuIdLM<Qb&D;E{yaW);WxkLdxejmue8nP&tnsK*6H7RH_=!7
zXl+1)4bxG%r{Wpk8_wMAoFYFvP<L`yMTdOE={C{CGmFeu&Y9i1QRkkA+r+thBn}k{
z=*%_RzucJZ4*PsDn~<LQmTS|Np3u*gnd-Z%s6o}TSmWC%E{>J+yY_oMsW8m@A@1~G
zdcUG&MSzpd{qUfdoYm)-%{^$Ma4O$TY(syex#nZ@5c^L@nkFbknjevMe&T&>_UFD5
z1GQD3R(<4bU0inGpyYU;Y=7=2)fK8Ire=h+?9qR#yZKVVck5%@tKJ#R$?^}I&~Sp^
z?w#u81ih(oJ+YfpJc~S|_^%zDCgPvB$9AfryNK8MW6M+9TOa5fu{7VhAbunGVXD>5
zHI-And>0hxvUGPXN!tEd+QL5Q_i^b4t)R!eKV8K`jCQlJs~6u{ka%RHj}~(?lgC|#
z6Vnwun!Q7RZGFV`piuWYZ^JJ&_qluJm9iCY%@4Y@T6)83KmWf{41J<H^S1FX$XT|D
z{rvkT^9nIp@mD5=Mr(K!1vc5&^>B;Km{vD&$MZ8MgDmGM{>V1e+Bi=$v8gyTOR}fb
zlJl>HX7t8xmMe?>-kg5_t16)QoXD?(H<eUfh3jQ^Mg%|Nne*3w@~@r!PE*^I0~}wO
zf6D5+mw!*9aLE(@O$R1kTrb+FdgY*Zz**63KgGJqcP6gxyT;J(y<yY2%q1Js^BaTq
zZ@-(lDOBA=e9N*8>IG{Ae(lr|P2X34>dA%b)}v=8&t1Aa<g3+;Pm}GITW(w>o|*Gl
z{>{{=?+QPPa=!VCMlE2oa+>lv=H;fqYtOqxKJZMlZId>QSUKlZbI7+7l`Lg#f>oI_
zwf)p()Z~Qv!uO;uUGTp0tG2)w%Oy^c_q#v;Jg`~NgUR{!Ud@uH3dX?+%ldb(cysd1
zisW9k`;{kt&ik(<I9JBCX4l#TRYl`eu{4#xjz^}e^jb4o%=9_?Bt`AX=MODUXN4+g
z-2Z&MHO=tA!K0lSyh|JePEDU8|LbM6Vd31HAHIF~F#Gf`-|Mble#j|pvz9&lF<fwl
zk5b1nzm5KTH@CFxd%?~3@W!*olcI%;y#A3q7n7G1*$cnrc=BdF_mSp%sq+)u_7(Gm
zK2y=y6!7eEXOqhY|ExP3wtp#X{#(|^ktrRjHLqG?-EouM5nK!HJ5Gk}3a`HMKSh^4
z#Hq>VtiWQs>km?{y{@)Cu%*rYy?E8a3%h%{oOoZDZ%%zL?idol8gPW;#b(E!f)`eP
zu`&7WSa|-Il7S){XT-*G_S*O(Uf)(HyTs2G{{C*`h1f!gjg$7AcCnan_}`5j+eu%K
zI^8-UG3%EF^V)`h2gWI@w)btB{w2C%6W_uOJT^yz1^2(sxN@#c;GOCAGyfE>vvZVb
z<a_kb<5(GYf3bF_wL@Nu&HeNX{ZsVL-QM+l@A?}{e;#PQ(EPMTU4LD{pEp0W<>o7#
zKK)<*P|bqlQR2^<GcR(fixq?=_i#S4$z3e)jQRa$A9<lCOM?t84d!cI3sOAMtimnq
z>^wj0^18okZg#wAT(fA3k@m~^oNGB(2)6(2$$E3*?L@}Ef4o=R>nmL#uz!8Q{(HP;
z-qS91c~~XCX20`@S@e5E71zhct6Y&OW-m4*C7AWp%{5<O&tj8YHSwXEhOu*~U;KNA
zmvWWMs+Pv5wz;<11#if|@ijs4f$7v=;vS#xJUC#{#T>vd;^K75B}up}#k}KQ%M*bz
z%U;Kx+ASv@#`d0fIrWuAP~p1stj-$&39KDcuIvh0^E`i|lS5>p`KHj!PkXM3B}?U~
z2dv^3SS{2qv{7VL>w>=Og2uux!M*)=`yaLz=JPnu<M-c{6`0X6C&Do5sqE=<cReQV
zdq4Ta>Asx>)&=uD>J2u?=cuxOOFmRMfph-K&>!K?a;3I3r*_}5-LqiAJaezlGMZJ)
z+N%<j1zs?pyK>@4RwG*z%S1byrH5GTyJBy0RN9?z?Eh%E#i~*H;-m)$CUYJ3RlLx?
zJmlES9;c;Ct-tpy5o$SJb?3-|SrxI$!RoHT^Z2i94q9Yx_txmSZ>?Re5!2Diyv@B*
zwd=U=1aO&)&+WQs8_J#%<{w%!DNA;e&5fuOi7As99j8pc-L>PYJD;=o{-UKf)jw>n
z<FE<*=wWlA@!&<7;GNvut^XEu^PS?Jv|06y;DY7r*>B8QT-?pPy!-ru1sMgW>P|9l
z_uj?Rcwe!-HRRjc{Lc3580kET0-MdLDvuw05MlHzx!cZsTU#J-cE%?Dm0RcjvE4Dx
z^`7Ep=LH=zHah+AT_UcvQ~8X9g=fL}PYE?9XDSQ!@jQEf@Y{p_hbO<jnDpS_H=9H2
z_iS#K{4IAxGGd;?wj7tF*EhaCsO+2<y7zJ$gYJjjA5s{_H|%y%ZFu~tj8S+EUsR%j
z)s7@bO@1HqS1zRv%4-A#jQ8-{%$S=lWcNPzx4IbH4ONc!m2)b0J!3ifk>B64dtH6w
zXWkRJ5*<cff9~w_5<2%NO;BBg{|nE1iDd~dldK}-yq(MZ7bTdyJ^6VSquRR7N9t}(
z4HRb*Uany8;w^u*JmKNI1;x)EF5VceV}C=g@tth*b3<1NBP+JMhbEbceEBxXU#ahX
z!uQoG6Bpj!dEiZ#o{DFYbHbZ`ev=&Amx~{*;A=U|v3I4mNbLn4JE7**c>(5M_q3gV
zw<>sr^~yb>Uq8nS+&NY)le%|>wfzIjT`z<BI(M9Jc<g>}v-!`5F2w?cM-!ATGU?s<
z%k)1blj{v{pZGsFzFEdnYYQHgzIvhTuy@fr`$gWZ$Jg4aeLmiJb3vKynsfVH(pi6G
zl)Rs&{`9b3qW2b_J~N&bpBJCo&&%i=B4Odb$AS6&%_(y}F4(1VyrLz^najcAb3^Ik
z$tN`O-biPvi>eAXdhfW(vQAzrOfay2#+BvG`Rsv`XZC7ke+jfvT(sKZ+o2A#s%1HM
z-u<e)zMVaKK}%$6oJEy@aBZ4^FIV(?WzjWd4I%lHeb2@{<Lwp>Ty5YbJh_*_Sis|}
zV%4JKKK{ufFAs1uO*}G}yPWZ^N}w1^<{Lwes;5VnUuDvGvrgUS*p;T9T2>=}aUQQL
zuS(BpPinDr-{-ruT>Qnq;8klPY%j`XvK*3L_hznr`1bY5U)!Ebxj6mZa7aWtDNsq7
z<@JKTRieqQ(v4LiU$z7uyqvwraMQBF<%*wWzemMyaf>n9ZnkhM%M<6C2d@@oFt~*)
zM!NsxR<IS-;JxX+AtT{bzuu9I`22Y#-0gGkNqb9{Op{(%QLDtObd*<gF8i4?8vPx?
z7LC&v9q0|NVNbQcw%~p2^Isu4^>;2CT5Xux$Md7MVPi#C<O7|*4Oi|;iG?joeslTx
z(**+Qy`T1+xZk)ph10W6xHU+qoW;e}R_!!rhxgA~>3!n!^y*lH_u8c<YdKh03WrFV
z?CN^iVH)UiRByqT^EZ><&)_za3=UtXGS4>Xci7GkE;R=iaBT=&dZsZvTtK*T6PHvT
zLsamt4;?(QCeI!&%6)PCs<*hWvZHg(l=DnXIyqGnXCHJ9IL6E7(#X1oXXOL0(n5cp
zHHW>eI5r%dEPhZqqp7h>F5UBrs<GhfK*5g%$Le;3xHPv`o_I6qsnGIu!Rkgo-aI^U
zdefDk_sl%f=I?NMcjcjx<|gC70^_x%dP$agljXQ=nz!(-{IPtea7}x)W7vU1m4~d}
z9=JHccKIWwg!CoM!O0zkCw~`w>t9#EqN5nFb4|ffhIZw&6OSv-EA%hkbg$U-!E*T{
z@iR;0g<bUivNd&oseW-|jn!X9pF2~R-f+w-n7B;2?vAiS<6Fhm?A^y)?rDBBUTT){
z=-q>D+fF|&_CL*{)a<{JS$Rpqe);2lH3D^u9yIOhuHN7%cckdY*93={0}k&QwmAr{
zTXFWyL6)D+dATPto-};mt*+SM@bgYnsjcwU>18JA#~eRsefV&$f?;)&;M2Y!7Jt7p
zXWp`_cX3AeFzkrGuQgNehYG`tKWBgY?^I;cpMS4!vy-5`R^GYp$&OFh^cQconi1K#
z?2>p*fl-uuz&r+yCUrMsmd2@xOY9~TA2@FBA#-!V+4i0bzJ?D{rmES=_cYq=d9tSN
zA(xfs%t`$64-YJEb&dR$(WDacGsSnpi`L^@6YWm@<&x<xvNM~q)c;D{hrmfHx31OR
znP_C>eQ{%c$EOs%SiP^muL!paJ)7KBoz@(CtG4>kpH4QenHMj=_RKY`(E4=YQO=8#
z;t$GMTMd*XxN4h{{C7?$DKJ&_cQahDoaw{duODvMTb8kTu}DlQS$sQX!E?KsN})Wi
zFZ}HXZ@7HFw{*GJ=4e(SVU2JrMpusCf1mraNmnef<1E?Q?sBLhkw<=iZpZ!v2giok
zhMi9yWVPl^wcb%4qoKW3HZV5i-R7mr^^2UgbDt21e0lg^%l5>MX=QzO4nex1J5DAW
zI=x>zU!j0O=#kT6cE7jku8Z=tq>L8dDQ>X5Vf5g6!wfC9KL-^W6uLU9nfKh!;8(~m
z+hMM<GD#_I+DgL|(`NO>cWx#c1$jDt+_?6>v`}wHf6B_Q0{f<xJW%<<s_PgO{$X`n
zM9{-!TO02f34KfK_<K)*=gB?M_c9l(HXicKKf`AEUiWSAwKq&py#MU-o_JTk<6j*A
zW&_1(vJdq&W<8ppw5s)pXZYM7CecZOT0*nj3pzE<FL|)R{bjA|4f9id9@jQIny?hT
z&0O<)d6w{6C(X$YlP#~glw1&-z2eh5!xI8q3!6Q59lD}s;k)oOW2dKYPj=n?Exab0
zU+Y!mCY<ipj(Yjucg5NVYb7~XF3)?j>O$|KvLDY{X4h4El=QwVXV?2NxmEA#l-jSU
zH>={e+{|zFUdnrxn^%u_*9H5ji(~J7xArcyUb{#9ynfx&n2K52?5A}?PE9={;(2t>
zW>)u&X2+&lgnt!U#`N&aMcIkwkpkC)T@_9|mER+I@#vWXCi5N7)sF7@uwMEQ-}b(f
z)zP^z*Q`Hl?NPiD?dMlCaj%Sa!N*IjTk22UGBMc8_UX6pamLeuU-wJRl<1suz36<l
z<NfQs*O=U*7nYrPAgMP?K%~<2!4V_p^}=_14=g@&|NaM=+jfQ@)ID!J2>9?>;QYqs
zGo8UN^CpNWJ-K9ebIa%c?^l?^+HNqpg=s5K+-@DVu$S`>&$$mf>{UKb|8|Rq^*j4l
z|7G7yZyw@q+0wAbNoCbX<97lLH?7%&i(~VS={9NTrJ0|f@?-9SN8%51ZX~WcfBMXs
z*0L6r^V28stYdC|e3HA_VbbIY%##<dwwNh%_qq1T3nv$`f19Q&b6`<q){V}c$1;0Q
zJj<|l*dMbv>Pl!^q(Y}gg7}Ldjze;{&YhIKDQnJAJn!hC4Kv*?s%gZR&RNF4@A}XB
z=BA9}+7|IkbXVLgd~@KJgt3-ccgl(fwNbVF-`cN!mkzfy%D65jFUBZ0rSaW=w#j$Q
zE^%ro#%(xWTlwh#$BZY99~Q70FFw9w$4sNmS*<pz+LhiCM=!rN7n&~jriblr6L)j(
zCF|rpj+u`)I2mw0Fb?9X`qp~h%DD1U@$m|MPj<_%1{Y7wp1u9<+uE&>heZqoB)83d
zVjB^C@LI!vgFnV8JRU|>m8T*$BsI(|T>pE+`vUon#EQQYXJ3qaa`23hI$xgKgbmCY
z@(*tvC^^cTv_$r-omby<Sp~tDa%;|9?w@<6hcQ}Yhk@KiV|V%ghNYe{eTh306gDvj
zoLI^{A=hh}NAez)3au$`{M}wu*(82*+`_(2UgeGZqvQ8fjO~~w^2PGCeP;ct*#GU!
zBLUeuMpsJ_KHZX>7WVjX9oK7#XK$U|tF0>JDb<&(z_@7I8_}wwJ$t@A@}3daXzydT
zs@PiN{rpD@FL2qtd?d=cMw-!JCD-hPWl2ouZba^0B%~~9#?bwoo6AG*E=xm#hKq4J
zr;{Q_-E>V?en#8IV+wb9-`X=*F<CD<Ty$+)*X(ERPxn6)ulE1@E86;i&Us%6hL*J%
z9B&pbS=1Q86W(XW{o&U;Pf6*EQx%S-FJ8S)tl_FfrE@r!+3^$&pJTfoZpkm$^{bv?
z7266)qnEMVDXEM1zR7K})Honn{$;PCdTWLB-$fk39NdotjKevvZSwI8yc1k4y6~z?
z;6_ueYW|}e!ZzvFhktu4^?HJ6bbPjyEmLgE1BE5GH--AIarP5mBDM41+~TVy6M}cn
zKA^7E+4%Lsj`s?yXUlJY^kCu3>vv|!FW9v~?{J07&ENMd7D@PC`*>$lhk`@Q^?A8d
z`-}x2{o><q<g+-Yxv|?WklRW9v)C53*d+5gr4#RF>^=QP_<+$2=dBMNv&<V3q*7}8
za-5$pd&bRMZ??SPiJG8;0k3=6<nNr$hv#42eSBh`!R&)2b&IMren`H5q|kAAyIFt|
zgYNe>JE;JbHXo$|b~UF{rFH3%$6R);V9;~=$TjhbmAj0jr{cXs`$QFLpWDV5-`M87
zV-cVFlIaf&nHX)fpEqn2@wh*$^iEmbvqi@DJl5{HcFWbM-hHXePTkeZj-*^ZSF+>U
zBx@t_RauYriN{{?OWdT)tJvGyq;)=s$>G<w%?c;Q)L-7NoB6$${iBulztRT=8=ib;
zR=c@UK;wZFcUz4wpDUyEmmtOX?yk>uMmuc!*rw;+E?GGtroFJ@fi(-q%c==hH~jhj
z+>V%|eeXl0MwE1Qx$3RFk_XN<Gh$EvZs_XkV%s_E_-5_02!m5Tz9+j2d<&+^I61zw
zW-c$iwV~9$;Pnlwb(0!wjrYIFm7dO4xZ(G)g+lZ8i1sx~aYjtqaO0b=u%cJ;Mq%Hz
zWqCWyWM$U9e6nCN?{$vMmJ*d8x#BS=rf5_(YO8Y1vW{<R=duvqd|O~+r|20LfrH7D
zcuq`FkZ;mt)qM0_?&uVmr5V*b=d0CCPqO(Tm}>Ft9mknf%xeA{?^>VqJ+)k?@y6Q=
zdmdi-t&kB{#M2_Y|NFDeA6@({-T0iWFV$>bmN<Q;%@6-6n}vSxe+%jrneFMcLpg8-
z<Fi8+ku^8IbGW7pFL<Gv*D#w=k7v@^4`&WNZF<8|pqv?YX~Ww|3=eN|Tx^zfjCTM3
zGlz4j&5pnayc)0OOk~KNb9}PA;VZM9wOdj=`ow!u+<BMe$ePNsCVBeQg=TnHTg*5%
z;ZTdkTfKIn%LR;2e>TnLR1JT!&tZ~)N!Eu|iEmCXww>zVpeYsT`#v&am3~nB<=lf8
zo;aMGw?JCIY~F`++Q%)MUOAO4ELzxo(dQbw<GktZXCIU>?z*w7O3m-xn*Icv+=d0a
zQr;WgFMT4?u<&<}<I6+F8->mOE)kzJwcr1Pa9x$D^R^x-r8OKa%^u={ks55A=kmYg
zWd8D1xm9)j;!W%GWd*YWpRHVR^p#jsoO|1ku1Mxn{stF)!y`WYd8c%3uWSat*4*#e
zS8alJWqsX!-CQP4>&Y{x_{DmBi~Z$zJ2_HbzF9Qc%%SGl-A$`aU-50aS<97G5VyN}
zwVsQ3YV7Nxl8GB+TP)h<X>>FF&0sNlUOQ#xk0UeVdp&mB#A<|1&5dt+^et86MX%IK
zA&Y$-Hzw73c7M3FD`D;K0P!{5d1_1fGS0GvnV*amm@#)j=!*l9cP=c{ifC`&wBxtW
z?uxUlFI%$}_BnnJ>@H#AJUFrWQP$xf`x;Dz)VDr#U9+h2qW|>u-66fapFi*Tvc;rf
z>z?FwqS5VPoIE#!x4zxhYwSB^t>twW&C>R^MW^>{E@PDDzNnFK`^AIHDNaYicxq$!
z>G>~Q&wV5HiVBnW$4MJ>C+k~q@pC+H>b-PSR>6|%e69I6c9xvC0?$_oa-UM>m)$P#
z;y}cbHDB)qc>JHq^gOReK*!B0@Z*eg_YS0Ly!o+6mM3}3joCANmS~^3QK%8+%Q$yU
z+dP{GCNfD6iXPn(>{+RJdvdJxiQFwZ4=g@BEouDC-1OcoK;o^5!`7_HlbQ`B#2@!f
z-Z7s;@7-_bL()fMKfQ8jj7s*NU6^>K@@bLk`Zz_O+oyLpdzEQbOwCld7j*OEv}L^y
zg~}ImO-;+=W4qn-Q01QZiGq@AU-D)?V3lbP5?+$YtNwg$%%^?M0i9L*H%)vl!d<MZ
zCcfBz?t?mc+Z}x+mkO8fH_Dwj$yG;SQR{)Aa+XQ2GcH{%V?UCit2Cpp^Jrqvx?{Bx
z53DBLTK(`^Xl@^q;;tLL+qW6+Qrq-VU}5g=#@8SCcr|bNWy|i1y^#B4^@>kZ%k#Jd
ztK=0T`SrCE?Dnkrk$ZpgygScNs!VddRC42CrIB3Ras|iijymJ7ajiUi7hjnC-NEB_
z&%;XdKK}Q|em=abr|;OOx4XpWijj;@bc>?Qw#L1;zjxe!D*67Ka*L@)i}U-rf|J;v
zFI?k$giXq#eaoZcZXfT>kmb8^$XeO<-O4j<F1if|KfF4u;>^0gx!I(bF^D~P`Fj4#
znvV`vb{q`&rr7pZ=Hu=Kj~`oj*zD+gaP+0j$77W<GSbBFFUl3-xWHCm?_y)&&*O7y
z=V!V6J?wiN_IurPaLm&Siu@_^Jzn*(kwNdP52Za^Ra&_Yby~~X{v9;>rSfd?%cQES
zO=1U6beJrS`S(J|Bv)d#QgK1bRrSwjtnv$XU5N1wztVl;t(shfa^^$Fm7!7<T*`CJ
zes$!hm{0wDYX+O(%Kl?VYz`+_A2U?GxAxbMil5&-4?E>Z@y%0zee2?~V0D?FYAUSX
zj;iE21uHyTQMPW$2Z8b&wZp<O6=%b4wWcrBx*{8N`Oc$dJ4<gyHY}=tw^KrC@vdDy
zKK*Ar<`#?Yb@$;|$(Xd!@aV3`oaVB@eEGjWsXn=RgqOvUwewRom-VusD+hw!_f&10
zlKXEZPjtc7mx?E596mLhp_1c_ThI%x({o!7M7*5nFg4aK{=wz~-RB#M1S^#|gf^_G
zUFRhf)^p1EkJ}8b_Is8g64N*IuStmWnV-PjHo@QUu%52;&gr`jXSqeh>X_YY73w`R
zBk*DLu>ggO)lL5E&u;2I#T53LZw<%7tx{?CrFLs;Tsg&-^I1Oc-(mmylESKATIcGQ
z{OsTJXT_QLt3UfIc~)GoT*`d^&ZnjE|F}-a7yeC*+IQx3eCp5D>+ECN9&dZPKGh`e
zU{~?!f|!_pQErbv{W_d#u}-gEckxH<qCZA5`xTDsV=(7zP*B%d%&A>;$Owbksa^EQ
zXwD7=b-l$H%q<G)x{E!vi!K?>L1&)SF8X9NXODuq{^FC`MVok+-!Y!LJ?8oHgZr-?
zc=*EENodV8Cxe?DX(0?ho#*Op37<AMK<twCYd;O{w^LKqr=+S+DOI16nv{BzGgZ<z
zw%jSUEqMCefaw>nl&XhFpNs2yrmx!gKbbei^mz5UTTT+CPj7Vpcr5nw_*<p`zWoMW
zkL`~q&AO3ZFS_gBG`29Oi93~Flz(q1KR$W7`ZLit|Bt6s|NSpgdd>dbQ?3=omsAsy
z7m52T-;0|*accC-6<lYT{_%^K#IOh0DQPVIeC*EZ75t5x1T@$itpxbP8XVqk_$SPA
zULjZ}%i4u?;}>qpykoy@rLIZ|aJ9VRS#j!sgw>QpkH6*`eN9&O9xK{9>UY#Cb^L4G
zWf{4$HslAd+%FB*<|i#0e-Cx#ZK+UHmvFiyxIj%k`pNnw*}J_qE9&rxMg`1fHH{2#
zXDE%3TXUePwJ~VHe-_a&x35NG^3H#hxN@92_2!F;DklpXO_SezDS7tMKl%RGB$6`?
z(~hUu8j1;2TU>1VIWxz9yL!c)5cxAF&081dFX0URaHRjc)rZ@`OZU$RVwv(Wp;ISc
zbmPA+-yn%)7OzSaoy+fCwl<m^Wq(VSuX_2WCWGiy?WvdgHca$ZzkG4pRM%hrnXRsO
z;{6tjFIjeOTF3;xCnw78NI%N|#$0UZRBt?wKgGiB)P2UcMW<x`Y9BeDeE#3IkIj>d
z=fvE6C#}?Ob?n$2-TY%^_a7K+E4Uqz{Q2^YJgK4_`LdsZ%#YsieC$6_H?iZ$XI7=_
zA5|_!UhDt-=`rWRr@T(ZbN^-k%CFfc?_|j3Cp<BZC-KjLl?$uyeh@E+XgU0eSLWYL
zwgWX?`#GQf_h8oftFPEy@~4TlPw!Li;mmM@Mp4g;-Lt=>UkE$@dz$->^Y`0)mV9_)
zvtW7g0ppi5zv@)jRRpM7-TM|}b$i+Jptoi5fi_boW_4EyJzy@C_^{|j<X4pgc2aT$
zp4ToP>EiyeQ8+|>h2G1V-{tS06P);Crl3$w7E7G+2j9IcrSey5FU)DG;(uk!ksxp&
zfTux$O;M98p(MF4&|z`Iu@&;}j=fD%re`AVpZ}P@V5z-BZ*B3k&+~pClQr2LYx2nW
zh2&SCs!J;WX1Ma(bf>IVa`;`(6vDDXg;66?ziS?7aV1xR=mD(;Qw|5wBdZpMGOl5Y
zVBNr#Abj8%%L2}%V83&{yc=hC{n^CLd%R}V+~ngP1qZ}k&#zj}P|5g-=?UwJY_1dU
zcRt_qVU@v)r}0mBUj86*K%v1gctg~T8=0*y+kT2ToZc-9S{RzaB*1F8Yi`p4mAdX(
z|JvAWtnPeL?UyXi;Fx*Tv#u!k@tW^@roORPm@XPC<7aiMwJ+;T^7rq_Yng@ky8_)_
zK0R~t_;NR$l0Vzm9GstjKs)L;o5;O~Ke%^%NOxQRQSwXA{gz+*Z#Foc<UQLc7|&5Q
zeL}^|StVb5Qfu_>=P3m467aYABfw{ATv2vX=jW6a_j2agH)vng{pK~J@P5F?iz>Uc
z7B!bCzT2;|*)z$^gyVx|tea^5O24Ais|#aPJHj{@neLrdwSvz>Xnm&b<6wqw|2I{g
zk_x&u!DnyKw)&`jiyU661YCOAd47R<r1{Yp>xrBDUaXlcc|M!<T>MVqolap{$#Zz>
zpC;r>Pgv)(Xo8dWdY=ZKTbYH)G2e8b{yJ>;tj-`b?r%<D<=z>J`BqDATi8{ux0*k1
z&mt>pACqe;Og>5%QnEZ|d*5iZ-LZUqr2P9%KXcjdQv3JF?alm>efk9N<2pgE{KMP$
z)dg31Zpqpm5;7xV>*q<H{0F5MYTf(0dC%-Ww>&4!*!Ae|#gLQVldjnB>8x1)-(Bpd
zZQh-4(jgzrO&)XxmL`SF6$w>$oyxVS>w(tZxSxw&Wtuhk`ZM-0Nw6BYF?M9yANBvi
zGs%2v_}_yMO?a1QTKFE$T=ak0MU&oTnI^q`+iMg#&A1Xo4(!ls{Ga$I*!yaPj>+m2
zf%puj*=i5WY)$Kyf3)ts@5)%v^rA&jy}^%RE<+XL7p51yN8<$c8a!gXz`a1+LBAo6
zp*i4Be$Z>RtUH_LDIN1YvZGVtu3A#@A~*G{l0|OnpUOb1k~tTwW^)kOd*U`s(_V(E
z+xl?=d)r>H8pPaQ^>2mCK_}_ma;HR&YK8p~Q*xcOX0_=d*_4BgTnn79Y`$syqIz=s
z*Z9?J3syAFQ?oG7Ua`OXZ^YYG3?Gh7EzLYB^nCM`gORTiT34Jp=5k;*qes>XxmAB-
zzg*#8?W1ZGHgU@)N%zfq0yAH2nzJ={vxlztr&B_|H+%3iylFQ}|Gm)dQBKOXns2p+
z3}!C8H#1q{l(XaNU$@`;w@vWJ3h`GL(`{KVM6>@hcKZ2$H{;67|CL0PE_O1XkhotG
zpR;Rn{+^i|CY5|siq~gt62DoeG^_gL{(mb@th)8!>#t{S`xf8o+q}cYYTCRXSsW_T
z>PAJk-UKnf7oWDc;Dzdo>W>STJ=~go^WS=vo_x~>>q?eoS1vxOvioC_v-}KB|GT-(
z8S*K9_XEWK9_F>M-mN}Qboc*Pdv~2GiCR9pVDi&tVu4G)sqf!ndvdPJt!br)UfiAS
zwYyLL-@=b+yMJfze)_?reCFxZa{gM&KOgGMw$9C1&in3&<eZIM%WqmIp5Szj+NvPF
zc+vY)v!lgYyP8)%h>y6}FU7+yV?3AL_VK!Dug@R1*l_xtZ^}IDEjMn~=WTkB`Hdlf
zWuwQ#<vqq*Zh5i?FPW|xwMQw{kRilzjnty7BbTzap9CTGPscWMItU+Grf9~P!*qjX
z180KpfiRv11v95DyTUA1a|!SIAo-=IimUU+N7tAI9igIkuXN{~b&Xo5)G9X5Nr|^D
zEn@Ojx2b+VMI6){;u!W$@HqTX?XcqKjUO~YTu@SUQsQmjH9_oI?89IN=&JTEMu8&>
z6x|@1K<L5y3tM-#bcBn%<PDv@!(QIGPv`vN=*tUI&HtuPd#S+SD&T!77_@>Nv~-<y
zgB9b7rU=s`S`DELs~N&TrR!`46UQ8>B;f;U4ZaMs8O#`Sjxum0DZ5<SDjaKl5l$@5
zR^L^&2#Lu2$#8&kk#`l>f}p^O>*l_QIwU;LN$KUS&ULEAkC5CC5=qt2FcJ`v<mdt$
z`K5<Np~KM`oQXl1Q={SH1PRWMC3R0KYWMurO+WG8e1_?-8aCUv_EA&i9%seQbI(6r
zm~vChynXVCyIc&e0^M&yT*>iM+`>h9K51KfcKvX_yI|3hZ5ee5ftyxjg*;o)T{ERm
z<JCR6()|8i%iVV`cfWgj@w=z1PA}dhQ^N%+15`d85-EM(w@cCOuEgSZ1%c(eR;-S^
z5~@3Ee#De06E9t<rAvA*b-8h7uR1zu@s>pw{~TF#Wl><(MGe*5DPbmF+EPnJLGqVB
zy1)FfxQ8+DSc8VO-;`MOQ}={Z-*={M?wy}MY5zXP?*W(PvTiy1R<Fq9H|>0=u|fEY
zll6{MYB5Xat%B5qlbfc3CEqSwW_x8z`>byu9hb6Af^To#TO@(d-<j~=Nm;8fzV)}(
zo$!bIxW0$(Sl@=kK2$6EZgoLCFGw89hANc;i9^|3-%IBjTXwAM?5Yd-vUriasNvos
zhp?&twp{w&{AXo{y(pu;Q<!0jsN(7lWwF8|N=qvRokK3hwJs2F5IdsdFq60a$;$WL
z`}D0IdkHZtJ@MR1d~NYX5Ha)L<?lNs8BACFa@=hBIlt(NQeV`O!0rg6Pp4D%2Jrhj
z=Sx*xzA(G(p2^W^rcG(~S1!z|x_9OCGCs4lr^3Eo5vkA<o$Kv-+~=ZUvhw8Bi&h>l
zzWA~!$R~H!<Udx1;k|2i&5aPtdi+eoD$I>pPQN=p=tg|Sel7pK+wZ+OxvgGQ{$=y4
zAFqF(VE?)6t5(#<1#jG*@w<y!rzXs-nWZ;<>dMQE3PEfZ6{(?fmi=CK=FHE_;is$b
zp4nqQJ$L3duBktlf4n{S>-3EcoyU)w8!Iz2+NsAq(qH)1c2{+=#mb$kKkk>^?Yvc!
zzxVQPkI&D0--q8&{I>RbrO2010fVZqjX#tk*t~Z{I>pB|H@uj@w|B$c#oeaW3XT8M
zB>KMhxIJ*ubhu~E<IGl75OYPUQeLkviI<sK_Sy6Z4skwS<z%sr2fKWhZPxo#C1hK_
z!SD_P@0RD*Uz2x-r6u3tRQq6br~K5G))hA-_p*H1#DDV7Mt0qRmUpLj=S#}Z+dZex
z=WOe=z;!}@TfWJy%6gg0w9Sx#$x+(9XHI};?%S&ODwWT+KD;Lzp^<yCSUJVNtL@mi
zIfDN0%1+%@EI)Ve&b|h<nfnd4hZ)U`V>n(d=N2L2e^w~$=H$TY{ajb~mIo-L{BZfa
zu)p2SPmcAf?f2KdJ8pJQk2ts~JJi#d{p`Mtp#gUrOV%#>&iRTjw>nwizHz6vRvU{6
zue!x;(=Tiaub9&JeN6haseSt9|9jbD^FF;hlX^gYj!n{TlWp&x*jeZQWWBs^|E>**
z*LIw1T({HD-G2s)*PD8G-kAT3ycO)`37%_uHIY%dnxRFhdqeg1J)il0UvKT%XV89i
z(e94$xfks8J?8tCUvYbHmb*Crj*xEL%T0pkxbJ^!uKsi4J&$znLC0;SjvLM&;+iRP
z_37HMZ>|qG^CBg83fRPzFOIo)Qz$Idr}^tzBNgZ9mwGqK5`?UdDRPJjoSRX}(6#wd
zu+K5uX9C}wWwh^pcCK?-q%R#H5wzjPQ9GydKGEV6yYCsM`rXy_=2YA6y4T-IMgPyy
zof>|zayJB6H+Lqix}7;~wX1Kxtii1Xb<>aj*uKU7uh_&jWg!wlaz-1f+Sngve0mvV
zSKS|dswFf1`wP3Dis!aCp0;s({iZ44c)yc%r%`%;dypT~v*yinSDd&yca=f5^>L}2
zH+D&MK9ig<+gjHx#{ZMvXSJ?-YO@0kwtd<#d#6i}V#U<#q~yEpSu@T(+R(YYQpfh#
z{+s=&r!#wA9{FyV{ju(9o%hYZz6ll{Uv6;hsn~xecy;B@vlk`KJie7YsrqA4=kbR=
zZGRni@Xd(V5BzIrRq)VnHOJ9C%Su)*YS=b)<IjLCUw?5K-}MSB%&M0^{j;mo&2#U9
z^AY}!PFPKRQ*uvjPT+Rl`-`5Kyzo0SapiI=v2)v1=3Mbx6SZrxi0hKX3e`RPw7=|r
zvu%=qbIr2b>5F%4S#<2<yHm-Zm9EVd=ZM}{5hcmspmeK1Y=Z;GJBF$0Pybe)k8b~&
zta0G8)b`Q^FCJet{LOegTwKx7I8P}=);Mo!N#Tx6fBPNv>i7RViJN?@A;>S{M~;$B
z%NC`Da|CMc-VO6lx;-=Z%j%<ne{Qnxp2L?neQtwV{R4f0%w1J$mmYkQYIXeNG#44c
zi;<x=w=b6kEWPl`ew*@!Q`Va|P2lMBQJ;QJ*_-kAXYtPw;t5L*9k^F+Zk2zpL0wX{
zwJ_f)+)q7!h0tfsEdf*AzMakH{M=W$d`el_`855XOX?RM%G@7rD8={3CU1G`q7=#N
zmJ^oWS2?QaZ0m8Lwool|(y3!rrAIE@t@)q$gg3rD%p&mi(<Fvk@jdqrT|Cz<m}z<Z
z`Pw}ZTm2kfE_h**YY_Iv&CZnlx43z!J*T$b+A{~w7TG5YZYc7*e#zSQqH0xGOLdmC
znbyO~^<KN?FZsz7pqD@EzQy(5zROqtj@px!CNj@=!L>*1w=^~yt1nfLe7VRbt9uGZ
znvpi!3Z{*_wGPYI7-XjHJyvLOa_yEGhm?x+gqOU(@bzQ!T&egqwvqi$r7iAx&Uwo7
zartE_v%<wCM%I<9mvl=v&&*!=^n!2t;)-qT8(E?(rg^`8I>R}-#PP|C=s1yk4hxqD
ztNhHl@jh(&@A~DUt=|h2AMNN~(|c;du|i|JF2zLaPWk1WkzyiQlh*aEyfN+F!v8Z%
zj{W)Fd*Fby%UlJyuNf0rS58}3*p?g8e4%;y8}l!X$@O9zbep*&f6r!~b$7kFX`k^7
zSp!`b$M!`l=GGMUv$48<)GF?}@#wYU9@)p27#6+XBE7QW;=$S5Vt*ffeY>vdNMUwI
z=JTJ-9EWZ7Smtg~kgIZ7{wUr0h=WO>zoKmPN^Aecw-$QrDgH8Xre)8}8z*+Ja+&IX
zc~kt}Bb8Y?SB)bwlV-}jc-XZxv3FKqSw-}lje+mfj^(Xpf7`e_teYwKw_bso*6!19
zeLHV#lwj>K6$nf{x$@oCs3qnq#;2X{oL((at84e0`*@LqxBRawhvyU}hA1sjS-($b
zy*+DDVbB!DhGOH@0!dRc^sS2@dwP1!)=^u^TbSvyic4BZX3^v+dt^&O3$MmjUH$tk
z>1XXkIWb9>LxsDQ9+)J~(+iW>7-RJK;@-u|BK&Eam-NYB`0#Kgw}I8Mt$Dk5o@`tl
zed9-%UwNco)r0@nze()+`1Q_-)jO=%?r$&Km2l*+>n1(-=MwVA|1G#vo*%Ge%a_xW
z0|I|t@7=agKRTUj_4L+*r$qM7TzLA1zvcW3uLX8IXT15T`4Ur|#m7IVx>}BJ%I;N2
zV)^y;()*qsKIvlbps7<IuX`HGIa}Y#G|A8Xmu<_z-QOw>dY$Yo({@XBG%0m9uQ@O!
zI%n2)Zsz6ev-_`$ZsRp9ee|f%yy5o7lp>ydl}(p7-I}(p+V9c7W8M0EZ>(>A+Y?ZD
zzgWa-a-pZ{jO^=a3Jyzxst;95-!i;2<8k!c^xLf8MTPHO;^Y<8TcX?@f9+p&fOeU|
zonK|!8v8?6TTgE0DSLj&;nt2Y_kWJF6=t?p`b^opHc67dujKNy9ks7#e7|lwX>XGG
zZkDWhGd^FH)SBG9gC~5&mivi!bp;#PCOv(2LfCR?cI1QaAK0%6f4+4hN&DEQi&y8^
zN&J*k^5qr3+w?lNAmh#Sn<{+2J}a{ytGLe=b?mgt?-C2eKi@AD-8>L<W3BNz?WF-1
z%L->$MR7iSQaxq*fnSfL@2czwk5QGnQT(3GU%CHWzSsSEZd-mh#WX9ryBqVbdi~(k
z@4V=n`TX@6tOe1$%dARHdwO4)kaf>_ar=$PX$(^=&K$ecS(6aB)i}9z%5`tqh(o+e
z?u#e+MTMNR*qi?MzWb|>@-jk`lP*7As3>}2vy+u!sNdS%FTA9>1E>3VH*evswE36y
zihq_|=~}-v_p@emFm3!=aD0n@z{~q7+k-i0S>!XPHm~J;q*r$IyZ4W1?Pg((nP1+;
zU%ub{_WC#8x`k)&wx#L!{C*Q~MYjB!*pg*k&x;?;cVzy%Zo$U<<&S@}bPAoV3EMh(
z-zU?3Q}$LYSok#l@SpEe7v7d^6WdjLJ5V;WCH%c^QGJDv`kgO_AHA~q|2MB-LH@M3
z4Nkh<ua!bh7Md^Fx8U<0?&I|dF*{c62}u6)@MZk@6MOe93-V4osyS=-p6Lub3o9J=
zxFuM$87i<w&9vLU@s(lx_ba6<nmgb2u8Q{&*?!@Ls#EFHNB2)v-#fujQ=;<sAD2Vl
zjdRIXjtkS~J*=L-T{`Cd<%?eOOAelXK6SzI8!2`-UkQ9Rk19LYp~vN@z53mANA9^-
zRXvwGa|Lo{KKgv+cU?e1AK&Wq#v@FB<ch`1ou2mk)kc@DWWMy_<>JB@S|y9NJiR`x
zGJA~#vsKEmtua$SwUwps{r}b?U)mvM#xb^EB84_OZTSZyt#^wm=6CCgcRr{o+)(2A
zJ$#46yZ*FUbMn_cF*tvw?V!rdwb~s)d)iJIPK~z_f3ej}^PkUmk$)>VpG=p!F!TKi
zo7E+E&C`DwPTX_DLvD{ke9RxG=smgiqW-0?J)L*{dA)f_+;5GWS7X;my!xJDcRp0F
z;jn14*Pdo836(9I*Ihm5QyP$RFZworwMbm(uEX2*mdpP9cP6uPj_H%a(6vt=Zq!=Y
z&-v!~rm36n&W_*E!t(F4@AS5p?f3W>uiSm&;UR+=_jNg$a~qa=SgiV!A!~g3)+xpn
z3bU`<UJ9No71+&YC4b{d*yMyB`8KV|TP5djTj&#We70ZvliM>dUp*+Vxc0h8)_32t
z5AUqgj91*6$jrMZO4je^=GlMW{&-RG!Lirh%KK%+g1Sac!*?p%S*2JxT375ios#%A
zXtv||pWzFWCd$d}c<VRI?x(3AW802X>#vtpmCxRvP`CD`gX@K((P^`lCw)qJxVSBU
zs@<HBXX1L^C0&i1-<G{hVCq=EBw<b8boR43At@5K+f^%H2Fyy;NHFWa8*5<X`E>p+
z<3p7XoO-3sDjeQuET`ae`bA2$f8z9Oy7tdZ8+TkiobGhC<ISQV2d@PycHe#{d0x=Q
zBdz#|<4$w`=ufw!buNfM`Q<(Dz~z<$l4ty$rkvQ(^JU8^*Qx2MwX9D)cAQ<_@#bi>
zblrn3XU<4<R7pMBvb6oUddBs5Swlk|o4JaPMmO43k|(Lo?c}N6U$l7H#b|Tya+y<N
z{^~O7FR%6P_|I^9wSw~9d3mCHpPVwkSeJeH;mL!ld_5w|#m=oM{pr&8UEuGB!)*B-
zlN%2jMXt5)cs=pQtNRaiq;}=S#ZEhLd2`tOj{NR;hR7#x-Yk-F_z`n>Q>@z56^Hh%
zmNhO4ztelhc~#tT8PT)iug)od+ascCJH6v+w9bW<8Naq(n`I_>`OLX#M<w(sudC)g
zJZk8jtY`1Pu_w%`!$Njqw9S=!7BBDWzYO!+T_h{*w?}e*Ps@_l^dx(pJ-44&6|2Q%
zG4O|RTiMkH`#CYMHY)qNbZ(KlM_Ae0J^K<(uQq*c&3_S^Fa06vvzn{w$FiO>-;a~(
z|KF?7lU6u=taZmP>C0u)epu;7ZRK|DzxpbwS@6ARW$6*Of6e9TGZ;l)y$N`?d#!o2
zceuy;fX%C>OY2>+ys*S*(Y+7W$u52U;)!e6-D;DSj=hWJIs5wEzuG;uR~jCEe|KW#
z@zkzEYijPs-Fz@zFZ1o+4WHg0=Y3IJKKYQO__Wz`KUaqDIp~upm|MM%_uuZekfu4z
z*Ee{~+COEI^_znS0(;x58a8G<3rv!`D9ZHc0*B1q)Wt$Jp(j1-tfgPStXwQ#u|4Lp
zdd;lN&2!$Vl+?@@?#cLF7;!kZv9Ch@sLHa1TNmsPkvO>RHpl0MuT`|$Z|z(lzx8ry
zZ}Ej0+NR4zI3^v{=h!}bY0v~aC4+u3f936~`SPzQPSa9*^~jYyrS1Cks_gp<)#plX
z{2OrYGGkrY&Ii3S^bTHHdLiUxS!Tx!O`!{UCR{w{&VRDpup#m^L%{s%wh3#!(r1-g
zmWS0mbXsuYFn7hG`QKz1ZA=bLmS&UOnSMKc?eshO^49$7RpxIV`<01Hgn4t#{Iuq%
zslm2J(aZu5#>3GwwLEujl<GfRd+|?v{+fdV;(|A?seQj4Rd$zK{g={9o2R`q=WR>O
zdb}#_ZH?!-*%4j=I<Zn=QNIK%WcB(gqrMAo%v0~RQGa#qHIx2q{pO7GTBesSI9}MS
zvwXVtE};ujON!oYT6H0Bu~fbOyP&;#in%c+2P@}(TeveMW`b_%3CDl^x1T)Qe0KY*
zgXt3MBc6mAdtQ6<R>JmB^`)(fi`Pgsex0{Z`l?#`Q!W0&H><?EWH;^UjBboLqp23Z
zTq(-)Qv37JB?t073ZkuVm#*krm-KD_>)DwrSNZo&@X6!qV`F-&<7xBzMNiFAn{S7|
zw=KMRNwUmztEX?m-*vBcm@a*E;)+Szr9XB5pKbelfa}k1-WdU}uRhCqd+lS&t<$T$
z|Mo4aD_WMc#VtPYmIfd1-3iSvdMXy~O*P|^R9n2?ud2Y(?emdcSqD~hF4(Zu@{o9E
z>obW(f|q_I?AfeZzio{~vUgp*Yem0F`;V%XR=?Yy{=LoClfTX?nJ+I!<@rR3Cnx^?
zH#z#ubW#V?RTH(-wtt@oFg)M4Ix^6?v}Cgm_t$^VM0t#!zd7~#b+OBYUz2}5c6wgv
zm-BsYx7VCA8Or_5srQnSI{fFv#Gku#{dp>%n3m$yaMQDDiu+@W=6rOXwtM;FTa&go
zy0Nn@@!f5GVPf0Ghee@_jQ%`G;X0To#U=W1^GpBE`%`S@Ub8>Vv-Hhe>)@Wly<&On
z>~|U4XNxcwo?(vg?dv?<I;%H7MO#XH-IlzsYJ5#qyLNs`k-QaRe(20vwg=L4opzYp
zT~tc3+f@>~<KnXQ-_ITAIGpmzn(f=1nGD6N-->7JKfWv_>>fUO4TIa#+#AlFA#4A!
z?Cjlr$D1dB>95ISx!~A{n2DQiSXRBbu=-o;wRb{~W-Ki_lhf6k^@VTorgBN9qls@;
zE)l-G<qb!~*IlJ9mP#(dv-1ym*3|0x)rs&f^W8Pq`0k3toiSm2li5XN*8E~yyss`Q
z%v`1N^rd|Xtq(cOH}$pdi)uZ+^|hi_KuT?}$i%LxtLAnE@zs5D*eUH~k>0de|BJ7T
z$ldbUIlX~$Cgr`M>r?+*ueTCTd?1h&G*wpNnyaQ_EmNdw@5JIo9BCF&3*HNNrF=ah
zV0-#m;_c&gXN3=Ko3*P#-o*5^Vvc);b?b-zo9lmGJTs%{rtMw!bwz?Za>c_|=xjZ0
z@pt#jOG=IBo%$Y}csMuj1AF)DllPVcEieB+W44N?>_j#HS>?Y{qMG{LPMUq0l>J!K
zIz}>3^nIkzw<)RDPAm8FPd+4ZB)IBKwKj)z<gYhD%OjtgZu_stui5_S9Gjxd!#K;j
zZ+ln@U-Tx4N?)Dfo4;k*tH=B+w(S-CRO#^|W^LX^>q*RK^;rFqx{Q@eYvhvmNLJ-p
zh3voAZm?uYLB#TwZg<JXc^y`|=Byj#KK~I^e$sR%PxI)#sUIh;5dPA;rt{Au*LzY{
zTA7pLlI`!jmFuzGZ(n;L!ARVf{q&CFPm9VQ6tA6Qd!oD}bH-uGdq1201{-SnAL|!-
ze`&JoS6e09H!I%hOqWi)x+tb%hkkF^l{L2-CYyblvAiws#rEyTa?Q1j<g)tT-q%~d
zUH^DlmV@mf#lK$~S85tx4*d}ouwsU0QKs#zysf?Obf3O4a(cH{y**Ja&BH0}X2J88
zut(Q|uT72%PVCP*vHtCP{Xmzm>zhRO%-Of@j<MeLn%iwxJ1%TfJafd<p)ad{@${t4
zw}YbBo%CTSbbKh0E|lHkoh_|-KW)DR8>@+IzvoGA-}|fPWKZm>D|A}E`(WA1Wq<7Q
z{FC`x_USo93FbY$rz3vB*7$>RN`U!!=Z<ap{@hz_PKRII<hCx7<>mWTXJQM(zW;g~
z<Xk$-a{W@X<XfL-1fDr*GcDr1k|^un9iG`$AC#BvUN%*_N>KWYsE>0^+PZ7}mv<!G
zz0B@q>{c@Kt;w5Hm4%{9-)K)e@@$Ul2i=KlcJN#}_xs^{x5ZDV<TD99QR$AFS^6*F
z@Xn`aGF9#>s7EgOmOFiM#*a;Fl8y`a$;t8mk`Bw9zOJ+N-&64{dA<4XSdMRC<+_(Y
z?ee9VJ;z@^o4<TU>HM(iO3Q?Ioyd8juB<$_viXtpv)U{3ypLs=zgIne=4gr9%uPn6
z%+;44?(UU7_O(&E{aIW3OO@!$-;T|hA|P>l)|m;3kC%Dey0vJp;+8cP?Q8DDZ=SU!
zPw7MG#*~$9Y>b)*&8k<bAClxveAl;W=KYvUr`7hxc%F6@UlH83e!HfZX1jdrXNk5-
zg~Br?r_RoMe0Ywfp>=9{m)Gq{pFbNNd0oW1ec>(%_X~MJ((87K_^Uk0+a0^#&cE>2
z?OB?>yyue)-#${;$uZ~K^0SgFFS%m+DO06mT5nJ3c}M*X^f)M=bKvj)h2GEXcH8Lu
zXAUa5q;ab7R<DZ!PwST53zmN45;#<{jcpN=eT|~$0(NV$?I#ajk31%O>`nB^vb@O&
zpF#zz)7Nn@SuXSX_O)a~Wa8PSidw-3CG%R8f^2>raGkXC+bzLG_jk&1wp7YZzrktG
zBX=gCu#jWDFxL^yvtDZ^vj=DM3Y*_KCzv(GWU8=Y;O*Vjj;Z_aGrzPeO`f{o_PM3c
zUfv7;>M!u^S-9%*`0^(XRp+t~Z+NDx?y74kbV=b8vx`@@c$N0ERa35g6WzY}^@Kz3
zo_T1`U-no%C@4MU(VAk}mvMEn{UJqnEpA<8e?Eb2=Qd-;fZvwcwx{y^nirqj82w7g
z&q`tW9)^$|7ye{!detL$H`*-CURvtn+LyoVQn_z0Vd@J^pWfQnx<Gj5yT@0AFY_0O
zUl9H(dgWfts`A~xn><f{I%hdmWJ6}P&$I7yZ}!HBTBj6Dp6yU(qBK!_;Rzm(!@mDb
z!seZdlbo<Qr}%vrt7^HGQa!(|zVrk?{%!GE?e=?*xXhZT!Djj|Vwe5Bip`dqb;&g)
z0<%6>@^37jQ#O02>AZy(zdkBIyJzukfp5EC<d)g~ikErR>})4_r`@f1TcF!9+Y4#6
zIWp(V<l45x?AUclSIb*q#?DXQY*JU<*nj8Tx!hAr(&w%ES9xw*L8*piDtm`#)Z?kK
z#Yb=L={dzGr~77^r(lA;$NI-F_V1s=QoZl?Ma!^Q!_))parZ?giF%YJ-+Zw2NrB~3
zeWugz@7*rF#h&x{$9b*CmrDP-t#>iqdv;rn)M@*f+mElB+9Um-_1x@<61(qD`n37<
zo7#CVFD#1>_-eWM*x#cX1)3{&{+Rk|{+V}`XBNbT=XcxpUs=ldRnmN=Vfm7+f1=Kx
zvnUS!daQb_ZD@o198nu?pI;hQ>UCF7w?F^e9kYLNqC`}o{<<$(t9tot*gLj-j)@n_
zh@52kO7E|ZzQC?{*3N}C>1UO<*|oF(F1saueR+9oYwKabUB99({#Z20a@mz90(?~u
zR70j`g^Jy>TIG4yBm1Ds8~5)mCVZl7EPDbb2W)<zGQ;2N=c=4DUwZhA(qb}BB-Q9m
z<mGs0>YO|A=aQb}>4hBIp9brdXkCeqj#y^BRw(mf|M3YI*FINDu)PrYd)AW!2Jugo
z`0u!1S*mgM!<+REo8NZtJu9?jW_8rT&1V=aifn_|%{(G9>)7m^{nxXEr=6XtXL062
zogCLo#TON(YZpG9-`ZI(8tHTSj7L|(J;U$I_i)XM6Jd4L+|nF2M{cY6N5gX^YXw(V
z9;|YE6fZTQ<ULnu(zX>X^Zdo0WO6*`vD%{NKDRyX$BktVR*U#&?fu*SZ-Z;b&f=sj
z^GTd@&+4*<-ri}tQDN&#At@pDtrvG#Mti-C^^Y|2ds4aM&86mJGCb;0&(6*`nfdM#
z1K*13wJ-K?9Q{(1cj?ufOV!h#YpU24@W&OuUnWp&&UEv5YVtbfNH@KkqQ^D&ObJ*n
z%6<CNre{r(yF|s0{)t~UJL#7<7uV$$8_oW*nFTi4;Y-=efAlhoaK<d%w(H8(P4k}a
z{LLRNwe~h=REOQml2siSr<UlxpJl6-aV<b%t!`@v*Zhmuo-Y5~v&Ctr<GCm0??g^d
zc(Gi&s%_bpInNl5UXIhv_s{ctE1Y~p|67u;bJ;UJ^Lm}Du`|{P97}9nT9uR-l6dRv
zH@EvoZC<;r^4XD+_4xGU$&apu#M{WZ=ssip(U9`pa|YA#-jvKUGaiLF*|2I#JePQt
zSh46<XzI*K=5M!5Z||9W_kc=paY%Yoa$5Vn$&%A+C;EkQbI;Yvu)f=?y62Tp>eEbD
zqaV$0Kd<JT{OF8(@kPdvmrtGL7q4zu7*zf$`52Gq>hQwgkT8Q|{C@6l-*oohD~{u>
z4>|klNz=2hle-+=O)%}8bnoCcm3+pyiBIR-X*W%4yLWDeskub{LG$BAi-T2nURto6
zA;Ns-Grg34evuc2r9T!vKbx~tg?~%L3(p7E?*IJ)K5x4AKvG=rPyfz?dRuq>xwF_u
z^`PF<?>o+B79{L(5$0a;+w(x{EmqAuw@;f^<l2h|MZ0d#oYeKVC-IZBlxD?>?CR1)
z|B}nf|2O+Yt(|)&{z^vlhj+)Md0#&^X+3>+_WO)~vt&iL7fye?F#hW5b8hQ@XY#bo
z)y$8PwaWA>x;VH0!u{5?H1+2~_BI_aZ0#yDuDngVU0ubM5IOmX&yypWzi+IZ6Rl*n
zj#+l+lf&F?E0=CpoB578e4ckl;V17$Oaa#t47MMyul?h9()Sx{*6j)Ncb@O#N$b6u
z=9yNUP&NJj)OUZ~)(7Y0Mi#H+n;g7+$E4|(yXLJ64vYJ&vG|TnzN+~P3sz6Nca>i6
zzUTgV$D7gV8+72D;qTRIy@@9`96A27QaR%Dsp7w?UvJ!(RQ|fwSNxii>ZYDshTUts
z($;TUCa^s_gK4^T@XOj&d~*+<ocu-M&-&d{R!vkrrf=(>tGV&Q-UD3IXC7ZGbfB?v
zM^$F7!gQNwrTcbz*XRG8o|lk!#;W}JUU^ZajX$5e->Ym8JL9uGulMx2ml2hpL?$Zo
zM*W(2;px+d?i>A|U!U`K$A$O(J3po6e@{tTW8uR!soMR^mrKl_b?=BObA4VLwkJzE
ziQ)8_>Y2fH*F}B>GGD$@Q_-|1p|LuA&Q+NN^Cqz}%R84BKD}&fSEBfP($i(dE8iVo
zBDH<8gI${UMVId<A2zzPZh5{dF0?ms$_yt({S|Ze{V6{9KEwRAhU(InWuNOl1unm=
zGUL9*pZ&3JRjKBYUVYNKd@p+}k1aP@czfz~hX~=OK<3@?H}*W)pSyzRPg$Aq11Fcc
z$6m)-n%}87zEq$rx#NNFuN9Tn%Rfh07Cv*G=ejbFX?Nu5+SD~BmrwIZ-cylIZQ&^S
z@ioOW@#<1>);;^SUh`&l<E))jvTw)f)wgf2ix4TC9&v4^f~RMdv2omF`wedl=D8}G
z$1R-U{7o|OjL)yYDGL8*_*Y6h2^lMycrtYTFueaF+0p*bfq0i&Z}pUU)EDpGp_O%a
zWvrx?f%_e`;0{&Ss6VT33T{Z9EK}XzwQJ+GPXQ}$e^$1a`IDgXX4ko&8~S_ioiyLs
z_Uc>Ap2_;sKI!Kj^|$iYI&)mlwhnMK7Wk|<^Tdh?-1jbCI#KfK)FHDf|NG~y{|7SP
zy0t9&rF!a=`(_E24BoxcY|6|19xe!;U;4(hSK~ix{iVSBWx0Nbs=vQ`F!fM~;j?=?
ze_emetC@2`vflr;)t=s&CtmtreXsO>rmj!lF`ioyPi&2UwgxTzt+ZUbRNi}?w$c=@
zH<s?nJ3h|gtn={RX!rerZ~es?X6CH6F{!4{bv|4C+n#;AFZ-NA^SAkq->g1tnw!GB
z%6R>!=wGW31h~!8DE_I=XZ6yGxoK*fg+kesw+l?%zg}6i>W^xP!OIJ~H+_#2Oi45f
zo2QVfXH|Ur@4Fo<<f;!ouK3rse5z)|MCtRM4)@A_?q%!o+Ubx{wmy=nc(tEp$ooSd
zC9jzOOuzH<WMGc{&L4B)@146ObmFq`K^wcUuYbQ69y@X5_8P9kT?aoWPO4Z^kT7=#
z_gv%LP+|Awe^oprc0^2_#IC%g{=yQSy}^RDCdn86zKeM!I(emogui><t&Zk%+%X3)
z-(2(H@k^$Hji+CECTmVw?#;E);JEqD&uq(UszR(T?epD!e5c6GHyV;2Ix@$~gF^!E
zT1}m@^WxQ0f{$iZG~O>)eNyIeCs9n=NWNmVJCm^HyNQ2qKN9zJzU6UltK7;=4k4bE
z7Wep!-{<Z58>`~UbNGnMbX9Kl4-YM^cV3J=>s_ao`Zh3_N2_el;tKUovp>ac$&LIJ
zZ=)x*IAe$5|BpwmyxFwINI~PzRCC4mGfV$}l?#*p)O0>xH0|}FHFdRRJpOb0f|o7&
zRMfOLFx0^C$PzowO<T&Bmxu8DFnE#IWD)a7gFoh~)?={=UadM6u@gJ5ZF=|GXY$h{
zb!XrI@-?;G<va7?H-+U^Qf|wWQ=Z&jH1onb$;0RS(zZ6ob-T58GNv56XTL(zdWzrc
zWj{8Z3^HnS>F4pjvgF8xFE?$MM=!N~_hS1NUVVvQ5n2zf?%|uby>}MVGd`oo;R(mY
zZymL*4B2n~{Jv+pY1`Jd(XE*a^!_Z0u83n3y~uq|N$K%{%I^sd4w;jhlupjkTyA6S
zUC37&H~;w3<3UsB9P_=qN9EAntM4W}d}?yi!ZGaG^@A1sW_<TD*Qw3gmiRE3Nj>iA
za*m@%uFhro6`HkF`}`ISk^HF-PewY;u{64=AN@`~>5xy%ttso@a(&DwVD9BP=W_S$
zP8YFrc?;I;Dt&)7bGn!84rP~yOTBNtzSr`f@aNfs$+j~}WNu%nO1YKMlXrW`qV{I7
zJ=XmjA1QD9`(3d9cEi;(x9YA5Z$JF4Z1dd1hulIBq=oG<**Q1&_o;wCHzG5t?`d9p
z;k&JZiFIPl#5~nFuc!TU4f5v1&U8<@9-FhJ@9Hk@Yuy~zGn$g;K71Fllq<>FWP01#
zWbNk5F2CIW9&S2wuVGDL(LvAbS?s%x?e(e<(qC0!lze)P*|HrcO}HY33PRRST~hJ2
zc`|4C+XY(5MYC@*J_}v-IA^EWi`mZiuP&~Wv3?(*_&c;kak8wJOJ=J4F`cEtuO|IV
z-eDDZ&-~~!pU{Kdi(hbbeQ8l^zt$=AUoiFcX|^w|Ce3St{w{bUklJvw+wJz^OmB~C
zrzUUSoSvli>;CJImowcfrtaAAtU9iL#m8mU2jX_m_$j|^(H`A*(i6=+cH3T46c2Kn
zv*TOc<A*QZH?Go(_^$M0ZqP)g%XeD0%eHUQ)tUGFT6*aEI|`q2eJ7k`s8e5M%pTUF
z7gYau)t(<qa_wSx?q;2sl)bEP-qVzg44=RKdUkM1!hWY*dDDc5U$OUuEcV`&49i%X
zc2eZD^z7?vWS@M!oV$HOz&g?14@Oa6Rd1Q)KKhtzt~%GqKX`%d8jg}p`4^mjZFbV}
z(wqF~W`6_c&NJV=wmqFxmgl6u#nLpD`!avV{l)P+oFe){)9Ugq<INJjPU_|=U-#yF
z!%LyZ`rCD+K88(uBAL4Krmo>-%hv}EPdV|qO5f6F`>uI)d*pA<;up?!H(TcsU>aCa
zF*~YqrbKb}jTv)m@*jOp<q_^|xi|Mrbfx0VK=+BIZ|=PEI6tkebgI+5H$flyB}y!J
zM;hD-)8+_H{E)R-JM7HqD;HH7wg1aqQL*>tS~<CR_HDKO(-#(08wCE@`)QtK!yMN8
zoKb(;<fbp58M|xh(!}DRv&TaFo~M@b{kBgz{)pw$)3SyCu6XiWx?9Rdo;Ry#EYW%R
z)9mc^%A`*=_u9fzItt7;u5_uI@N5^Kf8~XynI4zDV%xnRY|ZZ7bjHu~ky#u|S>+X1
z-2<nuo>D(lx7P5C%`*K6^R4baAB#;m#HE&5y16@_Tf=hhu=2M9&oA#QxpLp=-+8~o
zo@tl9ZqHNydg$vJ^YXHfPmaxAt(UKUUF<}iZJ@?uceTGW<##I0nEY^oX3{a^V8gkG
z1z!F)QQuY4yh%e_J1(B{yw0W>4>zrnK6+;sJO9#6aY9EQ9J<_<zPD#>ny23Pl(#+M
zflqdD?LX1uzJE@9Pw#T3ZwZR5wrZcRS1wdcxbwWVHDUIree-tbyx)3jkFfmOC5@gs
z8io5Wp4!BBMX@k9ds0>gQ{_(WNTx?lO$#mGDa@PbE+;Lw$l}pOqX*_Q4QH=e{_fuW
zK8c9hDUvQf4{dW<yh88xjucIO=Eu9ZcOEf38{yj>VVB1eC^h}x<7s;5f4%3>PQ0nS
zWLEP6r*+>BZcOr=^1`M#@lsc{rek^3%J|s~(?87F{q{=7th>%Lsw;$NI^TX1%DGBF
zoXx9tk+pe^{SV)Xn{Us`=HabmIWKl1d%I`o@4ep>;-_ogO^)~cw9l~dHd_eeOzWwO
z9j@QGFl*Pss)aj`PBGm&kMF(Vz74Z3RmA!H=y`nYOwJQ?&x99z(?oTjh+fG5$DpS4
zYfHul&5tMT9QGgJzjNySKf{1WM_GHFKDC~l;w|yyYyHD7XA2r5ndSz)-}YlqPx`T$
z)#1r~6Q2hQZJ+R7lwD_69gm&lxl`?VC%6C3v*!BP-d5Uja^9!QCf{S$Hz>!=R6DS`
zFlyTMv$0ab70YI<5Nc<(eLU^#gN%y`KPyiK)Ow4YnNmKz+jZXSE54epcjunYo64<l
z_UCt|-<<Px-lZ_5P1$#rf0DBD_fO#;cGcf?SXTJ>um{7cV|E@j#%VkE?swYX;?z9B
z&b^cA>4bF?l2&vs%exuVvqfXo`>HcH0;Vu)`^)4lo^1d0aKfx7Wv3_0h2_qe_%Kv_
zeltsGo9r_iPy0m<5kkKeW*R)Z`Z_DO{Npc~Mvpp<*H<>qzQ3xqf7|6#o%j3iNSE~e
zQMf5SS0R4cWy^N`OWJDZZtpnBApKvnTB>zxevY2f<~cXI)|K5>UB0yc%o?lChm}u1
z{ro#~qf(#XGI_n*TKYc}rT%aTE#CUqwS;H2v9IK=^<L}F*E_7)_2I6yN72KrmM?9!
z3RY>q{Tjnk^Z4&%$AJBlN-kb1tjUk@nNl>7Q|rXc#hR(<bCM3c`*(3!F-P~~zsf>E
z3yxoKpBHPRw1y#I8DIZCtMx@aW;407cV77y9DG(e_ae`p_+17u43<5s?~2}CoAiFO
z=uP%0_d~4HdiH<Zx+E}1Z(<ckM7i+pBqK5F@~1mC$F4A1cfEYhK^|S_H)S*TsI|>W
zec4;KqQ`5;>9DJ6&d$AEMhl}=V{ga5nEG<n7d2(B{(#`!^J~K;U1}rygJ#*=Px?LQ
zLdrhp?WaBOF5LNR%H&O73XGMDQ>JMd^J~nwmU~_^)Y;B4YJpR|$wt##N&TzV==^_b
z<GOImLZKEuor~79zD23+d#J0Top{Zpn!)Q7kJ<F=yk`QwJk-_WnJhh*kMHDXrnkH%
z&(7Z6K2f&8QSRB1*Gq3S-<b16)7o?4e7Of-{#1##KX_H7J3}NrZWGVlNtsJ#h-I$6
z)IRNl%*N%)jy%S!9@!~7jo#EH@9fFgC-UXztXU~iWgjy)PTvqS>A3X1!*(jKob?w?
zKQV2}>XrFd%qrR4*rbfyv`+0Roh)?4r%nH&Wtcf%y>Q%(v+q~heE;-rMUmrj<7LZl
zo;97~`XV<`v-?fj)FTBE{zvsx*F<m3nD}xdPi<5X|2wB*cm0O@o~+*N>zl)NpYVF2
zuYB~C!atio4=S$y746UBYI=Qc!)nRnFD7MJzEM)iwN8_rm$afrhRH<MW(sefzSzxq
z#rw9TYWQ3;as2vut@YgZ2TbnDc~@QxPn{#HpSNyTblj9VmeX<$h<|9_W3-01`ii5w
z?z_uviBlDW+fNI>-(PkrJoe(X0`}IM_vG1gLe6hG{q)x5%+Jn|ZGykd!ZN47{<f#b
zGrC~s&&}l$>6Ly7j;lWg?Z}a7@KIfP_}2^OqX83~nYP>HEdBD4>(ds4%4KU69es+`
z{fa}91J!c0ToeOKxvfn3rTg!7_a^f_n^$;kW&NrPkuzQ5lMZXPZ+ud6_uJR!^ESRX
zrm^>Mhogo5M#<ZAT6qlZW*@(~RIs>uoh0vDo_pt{?2Wdpo_x&sxv=Lar9)aLn>@l&
z)b0o_KYCt#dTrc`_&Y^66<*w!xi~V@Iep&WlV>|ug+BdnXe(>8X8%grge#Y3+l21%
zzyI|38uL@`ehuNiRa;K&PSxLXoa0ev_|e!xvts9ET7hZ~KMo&#dql7E(9f`-wvWwc
z-kcNezIL9wb=Li<4^!rzT5h%ctCH!<Roi8}_5B>ZesTs(fAx92SYqaZk~z~@rWo4z
zG_6Q#GmLlM;`wH>)m+9p>zn87rR0BJoBivGx`xQ&{ElfCCxyMbX7f+vLDu&<R!@X?
zO{p-mtKzINnYizG{^3f?C2w{3)(V`Pbzgk?-G})fHY_RiE;ne9iEX{y@umISAGiE+
zhcC0X^zG{I4iGryUF`JfZ6%k%!)50G=DqJ1N!u0~JUKx>U43Wgr^%ac`<A5#e3sd=
zlh;-*e~w6Xyw$8tZ9Sd#iLzbN@77-T=4AbRlIwl^q-&?Y-jfbE`S+|(!V_)r#&uu5
zS+ne5J6FbeA+d<@_Fv_NYNew2N#Qdue~ze5>0WNk-#dBF($5zRY`;ym=lRd0rKQVg
zTDjwHbmWW+Pb!}KOTRXl(#`P5AjoTyzFhO!JE5Nz<#MQ-wEBO4CL$I&@969Pce<@|
zb|v48w2!p5|Jr^{enNKS-z^eh=IS5zb@BQuJUM)^Mt1H0mG@dMuKQ9vyXot<EQ7=U
zj0$_RT|eB;i*)}JUg$Rcg0y<!v4$B{X)Ve|r`<2jyO|y#-j|Sm_Itth^0~J^U(z}~
zYu2v!`%bN{wTuoId-%fO_S-G)cG4EjOW)5ulCiC#^jfQlNZ`ASySLH`^UpHX7<+pj
zi~Px$ExD$tl=Gg<{}jFbGhUQ9to(RnvgRDkJB7P>Pfv)l?h~FTw{cn7+Ra~oy*cH5
ze_5UEN1N`MV!s|t3E18m{kQ4l@%1+Y;+zh8E!ll5v`{7S@7IrOGy-^@#GO31b@F|g
zU4G8*s|*iske;(L@Q<+LAD(&tB6#1v^4`A0Jof4H)VT-y7hL$VV9K)Y+-qCorG=FP
z^!FUi__w$4h4H}$JZInE5@X3QT$C-Q>E2|##M%BCV{*dd^1sJwt>v$uNlC7-j^@6n
zcQuvy*}~v1k(To5n>`NdUH{`&b6nv~_uFl@k|`6WIZr=VGR;R}-erb6{l?oTzxds&
zoxN^3bIZ3j%e<v|+h++qbJ@Rf*`>;t>+GELroU^JpHua7j`^oW7Zq2O6)&$o{JmhS
zGuNpE;n$b7xbL(5+S}^3_V2~=d;K~FU%~=6_3p0q>Mff5-Ffoa53R55Hp|_wWU*9t
zzO?Gn!`R~*E^#?UZ<d%x1n_mOIy5cOdTy}wg2;oerdM2VwSBr9u;TA@(|H$Edmk0p
zFZ=QT?XL#E!rKeqZC@74{O<h2s(i(|YYj3nAKw@AHoradSwhV!_KZfd-%;0hLG^~a
zCr8fvJmrqz=NBpSxKqF1oV!cJ<D*_wi{Zy(3cj;z7Vlgrv2u@6x4muhANH?H-l`l~
zll|_{$t_#fPs@Cou{HF_jGfZ1->c+nvn^LH*m){$#vf1Bed}k%8|kj&V=3{Sxq8u_
z2^KTfW}SN++%~~AbV?T2%)i%jSGOEf+2rf~-#l%f$%5vGd$?AM<(Tw1K6_dc#j18l
z=;y*YH>NC{d2)S8_-^lsIX`{MJKoOWZugqK>QRb*UM;h0sr60S{EN=<DxD|R$LyG~
z>(H6`=08<mo~vK8$n?(H8QJ+KcP?AA&QShK;}V^RMb&)o&j~KOD&DYd-w&Ikl9#sr
zb6(2yv^sZ>?dtd_N&CpJ-3L5xEwDF@GW(@%xa(KJ#6=g@PC02ldB+@%VEtH;`6}zq
zm3X`j59s1vz2#x$)ijGAQ_h&4xcvS0%7Y$B%XA}E{hjhBM;_C*4Jql4@!onWWZmJ|
zT|PTQ4ciNz*`EJhn^vq8ckq1j*{SM!Ya}_29l4VgzbjU8O0U80jJ+LUPZC~>tT&i*
zJMa1&wz$(y#+P0%RFqs*$@@!MR)PNpyY0^Rm#(RJCg)zgxM!|u{o8x~PlUL--}|xM
znV}}w#B#cUT~6C0&}B<RMc^ITDQhgQyi41{8!efuK7ZQJr80jWzRi%YmRe~q_a$c6
z`#zulYegywJSr>;uD!IgQtXK_tah%t=G#{skate;LFl~PWAi@0I^(x_&dyJksz&F}
z2YV>5=eoZ~U}Mjbhe_FuM;^cLR@t!T%=dNMibC@ww#yZo1VnO#JbZbiC}5gd`iY4?
zC1EAT`?@0wp4PnGrP8LW`~6s{)A1V*%$%woeB1S;nd9HyhCf2K`O;qu@7_Fk?Aji|
zJ9@j7#DC`%m0nvp_wuo=-gdfb^R68&^AU4oUn?GVH#_6@Ud;(<Gcz_`UX@X!)$G+(
zcUyO9&~okj>nq;Ho4uH&kiyLQG9_xUWMj<N6zeFdw&o{wQZ=$(-RF;&y;582=&2oj
z<J_vJ-X1<jO*wh{>O2<lxw!=~)K5)5veu8=COUluPfFV3AIA>&3Y|N0*=A+a^cID}
zFKu2D=SwOV7H*%s!<GN-%lm!r4<r^U2W}}`pDrq<x{GO1;le7#U7J5jZ<N^m`Vi})
z_{(L<EY14*tF}Jd+8}sz@!nl;K4?D8I&+fs>6B?@!aYn|s<zb1a&FL=5IS)lgY&j|
zt;Zae|4ubL)ZroOp8BPsGWoLh8E^MxleEI4Tl8c+tpn}$u3vPi?9COGe-?LCDrat;
z%_4K>?=I;>9-PNYIKJ+P)>Pm0?UeLop}+pyKQf1xW`?|Dk-H&(EY)c5tL=}Bu1_lB
zNt~nIpW>Zg<hS|X1J>hj{ui2mnb;p+F>TEz-zom_Q^Lxke=;-|sZF0*WozkK{k9}M
z)$PIiAM+>9^!!;}QpuM+uSr~J(|y%uPj8z&vm%dtaW*^hC@Ru1{N)L`nZLUvr|UR{
z-b<NSvZpKZ*|PZ4#s0Z_;#E$Zl&^lYd&}M}o!%=qKKt|_bm>XwXVx#XUREWQAJ}$(
z#)}mecT<IBV`s=8IQ970g3mP(HES;Yy2rfsMOyK>q7y5-?=|t|c}H%ac&y-Ht=z5m
zulHWfOA3#BR4m`V_-<pxo(nB^gnU+hzpiN|@ALI?LR^{GpTK1fkA%K?vvnKqc{#7J
z^K%i~F1`R>JN>(7&&=HUG3uy?*6T7sd!vxIEt{UZ*-wzkZE?+edOyJaVn@<1?ka_g
zj9-M8GU{)+?7C^+$pxowo=#NK_?$j3Y3i!@{JAQ9{A!0Ae!P5t=bYU}=9B5I*KDU3
zJFnfG{j5(&GIrKnp<iz^P1?I9uYa9Ue);vU@?f>6Z$JJ0f3u-*=``!ojGh<A|13QH
z_BI>OEh~#-myVQMyr1*Zb6JF8-9^374g1aX<~f+0(EGX~;!|{bl*GJwAr5E1bF%fl
zG&9mX*0lSHy8dyGa4Gx6X)NX+<kB+p{^+eUxbw};WAUjMFWUC`st4ZoyB-|2aKB;Q
zg@%YefyHO+-`e>sntY>Xdcvd)vt`duQS`hkyVytKYVpAwzGF+>D*v2G+MalDaaM4;
z-u8c8d~T96-|2ZB_>#Tm(B1vV{Zvn^JLNp{(6Laa!<%%Slwb2ioND>DhbMBWW<uYE
z%<HT49&NRAKWgh85f#>~l)g55y7VMAiD%Q6ObFRpl481Y^=HA$6>`rCUoD*Sd9Thi
z!Ie4fd>uDLSr7ed>VAEjP3FCC4QsENak0k3lm*ds40qRtUgi9F#KCf*XV`<2#Q{P~
z3+~O#PxsW<vfg>SI3?bu`h8&7kE=|3yqTT4gsiXDKD#?r>v3|@j%`(!^o;pMmdlr1
z+_+*J_wH!Z$*eP(oQ2+q$@AVUzQ=I1cl*PZ&nItMyb1dvV=8z4g!^gj*77=Q>!6Jv
z%9L);lPwIYnOEc&yii_rW2xj;*<(kqpWeRc=fjt?@{P_5G9M0q@Kortm#piV{7aW3
z1B(}}Uiv&V>f}=IUxl(slcpLs73vo6<&&T8__@yF<x+ERy+?LaCSA{SmpQ$!W%(gX
zz4?6Fd&0N<+ME%1c-OUwRy?<Cl`LD9&P_40Y@IRnlg+ble`}o1n69zCf2wcR_EXGp
z3-kPV+5UJYX*{g#I*@GebT*IN;?;?_qofM|+I8gE@!Iu9=$j~oZ)gcR(|@3%_;iP0
zX;b8e$-mnqKfIF<zH`g<;#}zqPF0M(d3}ldU*9eBj-RzidP2wd7p84(eJ5jQ${%F9
zb?(-kmlksDqF0}V%L(MhJhXG-&!4vJ)>0dHle3*6hqda0C+rg_eZl{ecd=pfrZtl?
zca*J3;yE27x5Vb<UwNG+PMOh%o|sQQpXN|yw`VHPzRmMBwJx}SzV?3V$JxhUtWn%*
zRNZ*hxa*2_anfYBJf;J}?~fU{`YZaM{hq6%vqS2v*B5u`%BdOb7IU5l82br7Ia;iD
zPoU&WWZVTanbvPCc@NbzMZ#adh>=`(cFmczgFF7+^>E0~n`!K89lW0Ti=N2aJ<^6u
z@yB|@_Z>S?u&G<JBqRAC-*J_!WkQeFerR28lf2<vIqxopH!LOdZ@nz*F~7^o`QhoT
z<M$mO1ZEstv%z+~lZB*{&D@kJ^*e;!6*bqFf6eJk^pcQ%ZTYO&q~~R2NNt^o<g1H1
z;$4S6tiAN|(uK8t3F;jtt(HOuM7Fs0ciK+ToRhZm`-YF7e6%0w7##dxBDO-s%d2$#
zEd#Hf<gL1&IXKc%ew=*idg|1YC6c!dz6VZRT;k-n(zRym+gQIpuay_=bd%0woj6-B
zVCBBu=M@T{m+*a9V%Jui)BU#R<uaw@H20uGZ&T79Et#eI*E40J`8}iMg_}RBx0T&-
zoKau({dJpqcu?7lNNIr#{d56$!PkMYi{GVOoU!C}j_V$SeKi%5I`3L8$4C3c?Nxv7
zy4~slujg@p`v@8F({g&fE+QctR~BymQ*!&WmPEGDu5Hn^Gc@WYG$me7;P`s#-uVFT
z;Qcqho%_zoR4?CuKg(hDF+;1>ONHH+$-N7o>%Ssq(ez)1pDzpjd8c+@fe&Z-Lz#MG
zsa+GV6}-3WbN*6PbLWiTz7111-Oiu>`R?xWM2$XnuWLu#W!UnUXFIR|lArhD<_h6D
z#o!GSu9`54-PvJj_2`FR&!*bA$Mb$A{Fa%vIqgOFq;r{-i=7Q8A1-$BIr?1l%v^!H
zovV#jJ#$N1`a?CpYxOLx7OR-njNPXf{aUrf#eC}tyQuf?%{S<5{{3ldb8baVp={}<
z6DjV6yXE6!{gNFnF7JBt_xew(ihl-;MVYG<Epz|vX*+0}zu6+f#@v3r=5@{cF*^KP
zkNK^Wn|!fpcGUKya=(njM|#E2HQ0FlPg(GFSIAeTyLY$Qt~dU<d&8W~JjS~M{uuK7
zEbprA3ALDhNaA>kQr)kq$rUfPk4tR&aQ4rUtG1diCe>YIk~ySM|7*kfIEmY4CvPYl
zOYEKBcgSbO^mI;{JU{OH=iL9UU$37(A?t|B<m)yqd@J;elm+)6?tRX0ENyl6?9E%H
zfnS4@xvp`EZd$YL=NXAig+k9xWu5si6d0bR&e(G!u5EX&iSms4Ccnj@lmD%~X{?i0
z?ju{dOp|+dTY>fcg0hmSj&r6QdY){Wr(XSSUyZvphvEhExvI(c3PbI+{4P2M-sI|^
zxpl>&iA(|SBD|Fr`<^<LFL)}IxvH7(`|ibyy8cvIom@GkX8H$9-ow007o2=2cl)!*
z><wj-^Y43~J7`jJY-@yW%dOK3YHI}394b|dYQ)n&q+Y-BVSm))+_TF{Z+3`ZQ2xH@
zNl&19nw7bf#;SDVUzZd%hzXYp=&!nUiR01IZi}|5ENb$}8&}6aTytfS&#kT#J<3-i
zWWty1Xf{1Q-$m4F?oJKM(n%Y+a^GCxsZ&pDZ1<VGs4(u-uRT}J@1OhiuDw{x-*rdI
zZa2*Nzuib?x0%uHyw%?r-z(+n=$Moy&rLq6?zy+@SYScIyT!Y+1b<7+Ect!7^NaGE
zTq(|3Z|b%m(VS@WL-z6ixEu3MUVOS};-A+X6Q?I9xdqHT((iZh$jr-sf6saO;eLSq
zm+S;_ffMn6O;SHT44SuH^Y`@57D17M2a}X%D)EXPU)6FkcK)@1g!%_c4s&0Y`?h<m
zFLZ4XDeVnWGgQti$}Tt`Y?fql&MWlk`sOM7i#a9tNuD`rqdMouQyxuu^A|!7a~Em%
zJr*!;n4Y-X<puj)-=C{}b8D~9*yUFIworMdrR#>-Px8+=9Cdzut#ab4mu8874}UuJ
z`^KLWJAZBn>7VoPrs+E|De<4u6PrSM{|H(NHR~StUb{PN@x8yVZI}<<xZYfQI6xw>
zBwS(l67HMV)`tbSIseUjx4T~|;N7o0p08#?&%Bu;{%)=a-pzISd+MVz%8IOiKRw$s
zQ*mC)ywc8Pv0qNFaFDrwJYAXBzwCdAYLs@5$g9tbBQBn1<FVSAuwzeD+|B9N9ZqhX
z_fY1!iK6LT6MyR?tE%$t?Dd`R%`C{cExmTd%6~gm`0i)#-ugGo=dNGU(|1Sx;wv&!
z*G*G;e82UD4@Ys|?=2E~>Z|6(NPNCIF)fBKUWGqdYC&(+dfR6L;mb5OK2pn?qvu$h
zCucr|X`ikB>gtWVel&V-ys$Xv@SdPBgDGzh3$4lCo}Je&KD+M8(%;L{Rv0{MjkI6p
z6LRzJ^=3=`a0ySP>4gXNo?kiLn0eXDHe2UXo#>6pw}bVh+CO+Nub#Ep-iFhXMd;-M
z6|VQfM+z!ylYX7)&)-oOo!7fPIay}G*|1q#boK4$eer7h5jyp2(XnKvw^7y(|EIoB
z(@N20DLxV1*?4QObS&@Vmggp!Z!5VJE4#Pf&vtO%-MytOSLa`=LaCovv0~mGjcND#
zI+c=S0(8pwwqAK=VRC-YtS#R1T!L4xTK?Ure?D>B7r7Zz=Ug~dryOdXEw^{=mbE!8
zw_CScP23e1Qu%FPVfwP)U!T}tezeL_v3Ku=_w_cXYgzBL#_=Dyw)s_=>q#x)KUSW`
zn-3Wt4nO<dF5tKR;S(NdeT7>V#ClvhGtX3e#aV`cXRkeN4JX!G6bn|iEn}Zq@btw#
zmsviNSF>d7T=#Iv_s&Z!l1TV+^600vcP6R-aLi>rdNSyr@!P#C|2^FJXo{uprMeRW
zv%0<Y=2q9Kf7N}uJo$T1?NimyY9~3hMbE@tDg3eH(hIwz*458m&L~~IM)u)pu8#*)
zD^f&1GtGVUYEnsy$93(yTW4$8?0lP~_~6+e-jdRIV>VmgN|%V_muDCh!n@n_U*Gs}
z{PB-PkKe?UFJChGZS8{7{RJMMKCArQnk#R${G#=p)ARKsMOI6-nr#vI_lDQ1?UGv@
zV_U?YPqt?&6_upcpRv7c{jufh+_{Syr>^c^?K;`iFkEnE`K*cObN$l~mKbjCC=lDM
zVHPi%t^G5z)9BlxCEnYkttP4}x>-GNsG7Zfd+3QzkMj=1^quV5cGksSXt$B;X&Gy_
zJz;tdTX^=j_5QJ`>bHL(lGrszjhAh&(N4a$_eJk-7tPX;ov6FvLr%})&X6Fnsc~Ji
zip-{LlHXT*=giB5>)BQx*SH_reZcn8@i{9dh^}0rUq6+3<yukBbKbR6rDmLTSG(+Q
zSN&k_!lj$Pw+C#QSiki{O=NFQ`;)c@?)l!v3#MFs@vlg7roq#wi-$H`?NoU2M8Es|
z$>LIvbfL=ixflQFt^ek5>Y#kw>Iz{4otP;tNq0@XW3ww~-PvTiBHHrV{+#ssZS}9a
zKA4+4VEx+rY@+3iWmPM~P5kG!yG}fMu;J4OrqrxS{1SZcmroD8@z2HV`Qg8Zw*Ph7
zEBeGIMN|G#w(z0a={FvR{{C?GXwO3%@mZHxlO*b@@4T#E!s2Fl{)mOps*7b$R7zGH
zdr@X+=XGjT+k`34S=W7RJ%7r@CqeWx=cxmgnayd3E4?qp@$py6^#rO;n(H#dlSx56
zV1b)`X|zm`*xxI<wqezM2}*7+f?u8~SatN%@qmb*duOaVGJkW}y3_C6v$C|*;;)=|
zm|7uL{7ouGxl?_y^%{c}lb>xpvDSoVOVH!03EhuoY0j3KEN<rZ=|uR?1}*2_%DmI7
zpN8goJ?joLwB%-8tnMbXH=;|*s_dhi@zOIpxQ{$NbvASVomq;XZqI(WKhMFQRXa;;
zeQA$Q=koI#*DmA!<?9o@I3Pz+*YD!>kj?*-mWXZt-kRB?_-lXi|L1JK3T#56<0T#L
zlp0Qq_SoaFC*w$nn)=Z_)hj22FD_cU=mZx>ZzcPu8@qq0bbmE!v}k&o$!77!v(0vv
z<FQz6ugJ2rC&oL!Jl#-Z_1bI6^V?Mk`RX4xF^aN_2T4w5vv|7fuiL!?OBpBk*3T%Z
zdhohc+}wC#Y|88tmn1gKJ7#`5Bxru#y`I~z`VT)TbTm|pyVmRweWkqTzD{ISQNzA#
zA3gn_l`NVssyTO2y>Cy{Zu!}>wkpIe`EKu+^yV((qS-nAXYLzUvTMEkJHw@B-dv%;
zx-a2bUu$<Wvd&x*U3gLNzp&AROATJ{_+#BxlyU@dhI2(oU;g^e@w)cDOP^D>T*=)o
za&o_=v)-@u8&tdYPELrr?xo$6v-iT^0L$#{=RVmieX%_3WwVmAiIDf4(4DfYH&lDw
zx2toR&2n{;jmLz9u63)z9z1saBkCRM?rOlX>hq_)vQ|?{OM`6YwnV&MJX^-#!lWzv
z4j-Rd^u(sesXR07$?jPz7XP@sV)rEe2XC6C=O2$KnZ><mV}G^&`t3)YmVRFI<VM>G
zuDt~^i3wW@|94&e6Q6THn&a_KabbJb#%9(f>+Z$&wBJ7+z2*5-(OIQuZP%Ll1a7v?
zm4Db5u>Ilk>ZfZzuJw#u#I85p`BOdLL*aa8t9@s8Hhx;fuw>qseoI3m=lhAwC!6PZ
za;{+#4K$Sqm3sF<r>aD8li``T(v(R;n^@U$^wR>u!w=4zw`kh^h|<^Awo?|WF6&j*
zsJ=Z<IB}{)#O9+SM@@cayw_+AT(DhY?YzzR<=W5Q(F;qr;_2nzd~T)PGnMpU=X;XQ
zkLOOClJ=@5_L*oK`{6RZ?wH?<y+^iu2ujl}YqE*o?P-1arIA<u+um0nZr5DsFns=6
zZQsJh`?%^B{4iafEvkLWd8@y6h-kO_HvLt;D?g;HnrCb5ocP7eIVXdYWq$0jm}_yJ
zah3&&pMB39Hn_~un8iFfA-roY$9XTdb=zF()aOf_+V6I1bK<`V@@Kc7bi01(veupV
z&$d}Dw+(*u;=ys<uAOqToDyE%V&nB$XfN}c+3@Ez$8Hz(nmmcqHBbHiOzi#qVAY*P
z?(4jjzdG)m>Gw}@`jgX6iC(YmV*gGxz3|5HcJYp``OVKuZ~0rw3rKI6o8soBZLEFM
zZpDv7dDnkzyUiK-vU_^(Jts@k1+v-aKGvN(wa+0bY;*S&i`JxruV;sSxc)DYd&#cY
z*KcRc{36}Cb-m!rF#p(sGs&9e^Iq+bTHkgxF#nTD*WdVf{l0q>?$5h>nr(+9`>O-G
zr^Iq<yU*L^oi$y(q99;>-u)Ae^|u?$pQ)_>VEuiol5m}5-BTUQFKRc$d5a|?)7GYG
z?7Z+k#c^p{(Ya^o^Vuyo{yTc+#$}FO3yvR_-hKR^v%O7eeJ@AE0sqWm-4~P7Jr=oS
zU%gip5nFDi_3EX^uelQ}<&4&!y0Rp9yXmHqEe~@~xjah|S$L_#F2v_<qUWCz7vH}w
zb3bZ#ylj`S;niN@r}tC$eaK$-t#I4fNB$P+E9%ewPEh%stk|*rZp*eJ|1^WZH1!!P
z+iEA2Pxf5v6nvFCTSIp5f0u8!SJ&_Pdp{yYqc3L8g)MhXHB0m_^|lo(Rr+>#lfX5r
zNArCfPAzu+_i#hpgPFpA6I2#FN#;B?^HKlLX`8r?)NS{<l*_#9(w!BZl1EYnUu0Ww
z9salMeVpVaz2ip}GCqGeb$Ut1w7IjFJvZFYV7{XMO5)6K%a8K+Z~R{}Z5vPG?U!7i
zs*@MV|1&Y(u<~f`sd(-)`>)8#M&*@>T%PlHmBZ23XE)Bux_f`G#hp)o_4dC%x8K9Y
zLaOzpiCy~Xhq7nALpdHR<=_6&o>deu`|j2=^H2L%{`}*$@yfLGl49moQw_wvmOnaT
z?|xpayr`|;;d<7yP4^`w82_J=+LwF&^qIiqX}3Rb-jO7-@J~oZqUZYG%I!?AR`+n<
zdwb~y^QAMs2d>|443_=eX0TFQxHDRDL#fS2mlfAJbhF*^<{r4N`Xb`wSx<|ZrAaGn
zzb*gsfzj~E=0BeeMPdx>0)OhSSaBu&vSd}~>O0e%r?nrIzpQt8*-u~R7ZTA`amUX8
zzZLOKGws%+1)e|G1zVTan|~9M%CzsOVP4WJHZwEk-KA~+rQC0y_)|UY#irY<uf#ds
z+I)9uam!EBmwDL%r?z;z6$ynM{}V239x%t~Z`5ADr~mYOpWe@L;QztDY1SnH^GF$|
zx_fQgFWyLdCwu#l{!thAqN1w9r`T<bdS)-WXJE4Ca78G?mmmGN9A@e*(#qD^*w-4i
zOwVUdmO0C;vwTtqKh(^ODE)Ul+U~D%yxhI#Q$J>}O5<>tuNJ)Y?;=mlNS*IrzFGyV
z?KyMM<F(FbN8by!!eReSLuzbSZ?l@&8@ZQrU82Q2$&Gg+nbJikCWkHy^)9wHTX$rk
zpzWhod%WJ<GF>E^@vkm$eT$vhCgBzDjwwyKIOChs_hPB%#};VGeDLyd;|^Om*<LjJ
z@=1R8@YLmPf4`{N^S2AXzv$|7=G(G|H{C=tg1jV`9$5O)cFrEgZ}Kv>spq<#bpJPZ
zI`zdktowD_AkWBl%JlRDz13E_xq&_jdS()$`j?N*ZPi@oTKJH4c|u)}vmTS%x=H5m
zlJA`QytJBo@`1b0jAYMk|J|bXR%XqsBCYAmgn06zCBv7swfc1%{HXl-GROMW3#}|a
z?%R_uwuERH#c#X5_O0uq!b>NYh0fW+Wb)R(+&NTh#+P+3RSJ2G-*tpWsmq)xo7*n=
zPRp-%TTEo%-z8UVm(IvJ)gaS8tJg7U(Zo$zKH+!duPChXNpF>#^yzQwZ;{Ee_Y+>9
z(XH2z6;fOO-c@_@wc6v0Yd3BEog1FMis>QyDPy6Z>P4Jcp0mrVY$v`s^{co>F4SD-
zPvOx;m(1SGFSObhnRrg9HYk*d`N{1o*^2S=ZKfPrckz($363Q>-8Z`}3s#?+x5vP)
z=?mYU2Ci-5VvZ{`pZyk|8}aPa5#{IHO8S>}@2*_Zo@C#gzxj^A+a*fJWn?AV)ZR_D
zvJDWq`(u-z`Xh~dMR9`W3IY9Qk{{;Q_x*WdzB4#)_unw)XRdRX<m4^ro^zotRsH&f
zJG*|zo$yos@>c%pyK^t6-iZuu-uR?qo@7d|O4_<3nisNC^&5}AjpDo|y^ve$@=c4F
za0&M8KEB`+zwNFiIMv(=-Bh9#tIWdwyo>AOPS2h_MoS-k_cpVgdw<E}=YE?lUT(<Z
z)s??=I`5^J)Y6&8UUA8b?Ju6~sXm-}Rm)i-f@i}flY6mGCFgY?pB*P{Ipd{$$po>P
zmmajnT{6C?8h&)A@aqN9mlLfwNK7tVWgj}}`}({F>#wU<Gp-K`R-4w{t&*}Xe5puW
zUnk#eVO9gN*N3Hy<;@pNZ+yp7ye(+f9Nh#}`|n)yZ?l{4suhxswW_?y_pG#fYGS&@
zoxK)XPqQUEq@ITE`1H~5y!TY8-CoMfYgMgpZY%!o^U!ejo2jY${g}e{gw!t(^tCEW
zx3!)&cj3c%ztpEbmN#Fpp>~bqre3L_1GmGT>|M7@&;6>duaIws;q}gMF-;vSckhfy
zHW$+T6RPuY8{=Ko=S^kLt<_Xtm7C}J2i7cFT^yGbBRlm0dwIp3CDW_|N@JwAuQ>eu
znBd8hZ~EzHC$vVMmpm7Kcl(cK17*o0A`7x#W~2pMO}I3xrsMARhwRR8U+8qqJHO((
zO8j>2@Lff7-@J7;O_gv=zY?^!;8gOv1JW5+u9#KE)JX7n@n6iB+MFkC;P%w0`f-$P
zmh6u6mo$1r`t^4&xnLH%_>xD`u`kN`NlXzLFM3xCsc$}RAz!@TAX8+ryWBC4vyWqT
zmPBi`-1=FzQK)Odzpzzz_pf8TJFzbOOLIx(lb)q>-k&g0{^wU7tGywf<BrCEr}gQs
zCNjZnAH{ypUGXG0#(Krxsz95>_9H4EukW6@vi$!c50zESzQ&b$aVl%$VtN(_L@rbD
ze)jQD*N<10UoTto{a&zYm)^gU<luyRr7ybHty=nV+bWiz*xl3D3iTb@S8ILR>3BWQ
z9N(}cjZ>;uY))KzB&2h&G~E6A@yW`s<fH7uO!oy$$axsQ)KpD=hU!#}lPWdu&&*2`
zn^$iBB3gr|V@7byuB&;!`F-mpoh^=M*0Hrd+t3}8QL39)R^Q2W_=th*v+TMH^#!l)
z=Q-@#z_N<xE{E@-*?X?3v!5tG^zC+@V%0Kpxg|DHO56wKr`}yt_Peoimel9>VRKqa
zHb(w<650LVQl-u7pZ3f*|7WsYad=lDsivCN`rF{o{KnsD7T5l^U0Y*sSg~xQ|0?H+
zPp|x2{-osC##(I)qbHfta}Q6c%eT?l{j2l&()wW5C)>2!{?uQ-+0(V#^2+6IpGTJA
ze`YuK*Z4huEHmlo`X|-@j(s`LdgEVX-aWH~Q(vTyT=ajo=)i9?p);|U{eOM1Nfvu;
zuXXf-@_YZOn*UcYUfz@X^JlEj?7sV4>ReV|;v~!duWd{}wKleN$E>HXb{;sj-(X(-
zPDX7xxnj}w;zE;KZ_a#q)UMc*s`}rhO)BfkiF<y9pU$L~2Qym*dM<7IdMopU_0$ht
z@87%Xs~`BHzDl8py>si+=`LR$Zh9lVbU%M@$r+c7pRUs+9&K_+U-V*eorB)Y-}fC@
z7t3WDE`1jz7$VQ+a?$nit-tY1XG=K^)y~vD6?<CAnELJN`wbq`xIU$Lm*4U{rh8}Y
z`|>U=zBSu#&%R;5W#UVn&(@t6Ur6b$ziGMJD)xH+&uN!GZ+dX>w%$Tx{;>Pc-dt^N
zGr5>{vUsv`l5Av2W$G`#Ia0PS{%v_$C?7rT_oIJ{=E^D>_K2JpYZBeLxbIATt$_}&
zfXRi)CsWU-bMBtE#()3*+w}~pQc~)+o9DUO$4$)%)(_>KEtOZb>bud`IIG!OuU0Op
z<@huEr@;@A3rY*7=BgcCaJ$~--p^9M>96CO6^=0qX1VRTVvuF&7_S~{y6@JswiVU&
zGE70!;~aL&s=v!>n<;i}=6vbntMx8R+Uo9iKd&DUd3n;-DhJo--OfQa7LRsq{MY%j
zw=a3_7QMuZ_hOs%)%KpQDq`_bNS%_>XA^z->aBG*-dTQ9kjs6qx2CvZrh{VB`+)XO
z-K|zrYmd&D(<mriv&}myLa#03g{hzYG0v$^n}X!8zSwg|d_t>Uyjq9UH1TPN66Y@r
zjCeWS{fl6!;+(bvNg471GC@LbHMoD48MW)R-VWe@Y#kIQ;<q?3F7(!V&g~yNZfo3e
z)6G5;{Ft+KW7n@om6-?kU;2<_aQjzNa+r$c0zKcu2iJaKQ%?Eh>a(xscmDt8`H%no
z6zYBQqt(Xj?7K&m{~uo25#tms9qP9FV}`+cv-Rb^F?G_9_s)s9c%&;{`jdiXh?@RK
zn`_2LOk@q(RvD=s34SiR%FDVuc00F8=yy3^>!>o3wZhefZLVu0tQIVL#>#Yl<D$o_
zUhL2+xw%_PzJJvs;T12w7=~<ZOp!BiZ9f&e_VBWIx4XjSc$cqeeXV#mscF`VRiBr<
zkg0SGwDJD8V!^TGm&|i{&+9DRJHPJ0=GQG<*;Y0tK|K4UD;_^gyW-OL_Gsl({at0x
z&mO(}{_e9?na}0sD^E{fcIn#rj6(J;AO7fA*?0GyR0@9Ko1k<4!Xo1<bGQzFvGltV
zdwN1&cC79xyT6&I5}%x7-!3h8+x7Ujm;=k?0-oq<9(EB(RO;9^H@qXg^5RF^J8yR8
zPL(z8I2&<nSMAx^;mt;WBe+G6WWAZxrT=)Q?k54W-R?G%?`N#Iw5sq?mCe-9FsAcX
z64xIIUkH14;PQ?aCi5=LT36<}Oa9swkBq~eIoEa4Za-^`DV5lNV~S1BYQFo^c~9Rv
zEkE_yW8KJq&E?@<Txzl*$F9He&|zNqo4>2xzSfH?^pfLN=6H#F>t{Y?sde?dc*84N
z>eGY&@3;btW-RRK3}C3(`oeVH?j<4xYxc~vcks8~kkiE-`sh&W?vmWsKQ!F_{JwIj
z!`Vyobj2ZuwC=1U!c9V_qo;IeM9tJ#c9l~r&>{0^OF#Ekj-?Z|7AE(}>5Ioybie<2
zB4Dj#(BwapuUwX%<`}uyYtM|{iMPMCo7uKb5})09>6D}O%R7~+Yy1wL*3(>j<L@24
zKgywON4uvRU10zH<a)rtjX%UMbzUyoQMx_w$V_JgGtOQA&gL!N6Ri6F#L<%%9;u5|
zE^OU&)K_Q1ev1T&=%RDGf7{9!<!$y#S~)2qF3|VFkwac0Q=<Ho&a%($H0tf*))TO|
z2>ae779HoB!)1H?)-ib*VUZpm<-q>gt77ubU(DrspZ=%FwQl9XxKl=QWs<F%CeEy>
znWxbzbhGk;$eC3vo3$FxF!^70x%hI=xhZ>Z`E2;Qe`31LQ`Qr;-j3fp#Iql7*<Z~V
zc<Vz9n@g8(SM|-Q+WKA_gV$sUt(<j4qm^^&2EOK`;K@f)Zq)|gzhlwM=b6dVIw@%K
zo69PnO+rr|>9#QSyPW<tqI=%ghI708W;9BKHl4JdV#d4tC+AXM=d4*Z-pfy}6V8)L
zj^60Ba-pnry#9qt9h;6${eG%(hd=L}->ud^-kUF2b6MqeL{9XT-OPoTD=!`q&0eFU
z79E>b7f@!>$seY8$@Y86m!+N(FDz%q?zr=LiqNbjP2aZoylof$86bTmR_C$grncwX
z87D3;yqtNbL~8#66}gMP$3I6Y{b@L~;`3*ld)zA?3GVY<ahXxZB5vM#W3?uE4X@+O
z-y3-4cHBRaQJi(4NqD}7bJH|Ejg$MN-^_LYo|%*Uv%>2B(p{ZSt|gDzFZ$2tpHuTE
zV6~X(mKpkOg8CZTbq<m2CZ3P_=NAb6U*7U7-}J%9+BtPa&tIzlulr^5e)hRvf3Fl<
z1l{Z7U0yKNGUXQ|-@~e`Ie8299v}9Md0RZwF1)6$u*F=q_4X}+<sWVKEh?N|AorXz
zt21j3Tgc{_pNn_Sxlxr;Z}hoe`a-sM(}CNZSKihxzAt_L>z3CudlY8Q(OBH}{YN85
zu<E`&hrU(*D#~B+l=tVAUz|p+$+_0f%?CbD5}L)aUe?TK_bmh6l^fMAoGaO;rN4?r
z!L(`4=K5zBCPYhJ+*r5Q%6Z!j>3E*@$d>Ex6qEl5Wg1$&xH>ocaZaRL>AO!&PZYO>
z%>B^WSH68kQl+e>dCoCCzxkJUZ8_tcEY)vgdPQKagz!23-<R@V@7OC{vEGTH?#be~
zm<QL>YnRXabA(B;Kq0<EP5z3rgzDb3hChJ;`HC-RUQcW*YP|P>k@@h;^`~Z73oU9(
z2|bonv6p>j@!N2YSIk}3(VMPu{-{=bDZlq=wrY}?;}Ijp4-S6=J^Y0IkH3sBeH5=F
z%=dE@_usmvhZD*_sjam-J(tNmC2(c%5_bO5(=Vr;xK|i*UiIMVf8R?!Twi)oHLRk{
zdhXp<OD*<(t}61#?|YT}U3KoR=O6znomROZw(WCnfBW7mkHsg?lG;|hy|nuN+vujs
z+YRjNFDF_*>}G!(6+M-gy-UtUe=E~g9eajuPosnf?YUnLC|?v(+3|Aod^`DR6V&fB
z`s&8&B%hwn@!vOOD|Zts^Iu)J12Iye%RgCP%1`%tc(GRI`m=k^kAJ=^nHRlBW+KnC
z|9<rci~q^1U+Qj5=l$bQ936Z7aM1kFNe8S;ER-e*H7>pN`SNpfo7PDO<z3mCxmL0C
zZWsQc{$k#Q)4Nw*=T<5azw*ldK{n6b^+GM*Cd|y#tl2T=<*NNFSC!X(k+bZ*T;E`R
zD)hPAl+AGqkD2qPMoR~A%(h&%Em>l6bpS`Q-!^a8D34!u{wuhW*NB{mTc}mD!TjC7
zzkNE3?~2`J)V;VqqtlVs{zmMS33F6boZWS*BEMegTJqMUm^ENVq;r%1r{p!i%M)e!
zgdIXU{q#SatZ55PnmXNL>8yDIOZQw+io0~#_XxAAmVV$V@2aR(8CzvP{acswd$;(d
ziEUwj8a!s6@-F!_X<A&0*`cn#?m?gG7gQ}@qLF^q>iV>`={>XGvd&+fX?^F@VNZ$A
zzw~|VFKJy_l3wSi87dOAdfr<H&0OnXuVVQCuge#rW@s$U%<)am{%N{2VB0QF)vu?u
zT_(Od%Oa8gTRy=1u~0zG>`&8k#Uqw4`NsWYVa9i(B(1uJ#kza*ztoBZ^!&cO`rkIj
zrN@NZUhg*)lX3me=zYf}ColSB<ED2DSM2<}K74l5hUt5%FHX4h>h$D?_Jynq8vgVz
zE7&)0o%LpOuQTpiCC{0E^PDXf)RdK9Q1w4;|M740H&}bUiEH{9^W*n{D2<-#lehB&
z1Fu#*=q>uj{&CNg{r%bk=jKdf`e`jEX<^9q<o-%)&x?O=JQeNdlK-I5^*Qd-)&QZq
zUw-PgW-Lvz`OsBy@9LDz_vJRa3aI~_;<RJBz&q9K>XWzH=Xi>2+nbWDf6>yTB(C|}
z+xMPaecp?`w*+3`o3wIjFlSNegcw;Bzx^i+Ja1Zg)Xb9#>iaj>>zv3w_jSv)Gm@8H
zs!z&))_NeRYWmCRmm4%i_iOI*)U@<)x~DyHXROdO<2#F+lZqzHkWo=vuQ_*m;)ZkW
zFLmz+`d<BU%Peg9&X=3(oeg~-=YMS2H0_*5_D|+1rh>Vvg-=Nx_e?bQI8o#3czb4o
zruV*=|0a28q(psvR+^-rGHuPqX`-egSNoE)-ye5-sqXP&W76xqGgF$*YH!)``P||S
zpS3eA`}OxP6Z>$?zP=*ZrqcP<l<>ab{Rga$PA<P-Gri$c*V3gNm!JP&6aB~ZdWtMx
z@|UN~r#zlrv(1&_5)ET|TouHg8aLsO!P2Rm;<MW}9kLc_Rq3dda+_&)-CTo9{!M4_
zc716B_3)_$AM=<#&N}($;Kvw0>mpW-<7aA=<8to(IQVSo1mpRFg;8x4wSGQ#yS^_I
zO&6)?n=;W(KtzL=)zevRwZ)v$KW{$%{MXYkLqt_l_iE5(zSKRZfA1C-=vKJKIPue@
zrH8IY#@xBYth)HbtioibzdZkB^^e78IqcN(>YI5n=#!-G(wc^8LGzyl*j@<QZg}~X
z-vsdvks_bAnV)|ixoF!uTjQac*NV<bi6OJr-CNSN_>GY2_Kb+L{_~r|R&d(R-81Rx
zW{+U!ulKH0uVy#d_SryV;-0l}XBP%)3A%JC2Oe^rk|3yRzQN(SkIUpg0arHcJ7(M3
z74PBZ>iGC6ThH$s|6*NR1YZ6zGUl>dp7Nskh{lsU0Rm5_X;>br3s`=BQC$~Xc<U|`
zo%dgPwEt=F$!Hv3&aZJiJ7b^El*g;%iehZ;^nP99cxUeCrAI}hAM>z(Y1RJH`gw-`
z3g-KBI;OPOU0En!`r!7C!=?v6EvRqqd@p?ZHro`YMSN9jORoqWWA-XATCyzohSdH=
zt29K8bgp{j61s7%i_!nX7cW<Pi@#}AfB#->=Ho!wQmOq(4_CN+SU*2)3Ljta@)-ph
zKd1iv_b#<uM0(QN^gH#g8y~tF6`v9~o_Bg@^eKLMk%R>~6WN+xv%h51tWn{VJ=wz7
zEF`Wmg-J(p=F_L1XI_<?yUBeiI(9cOl>O4#l<j|3oU2$HC~(f=>CZZsNL~3wYp!KV
zymg+k_3fRfclX=hXt;DD@z>j?0L^)SUd}3+QtB5QtK+#lVv_L|=9R%FzZC9Uxh``#
zHTQhy+oVVhwY`q4sVjDxyxqh4r|Q96wV*?hd5Kns>gVhaJm)>>^R-*MxV1FytDY=h
zoN!Xmbn+jklpw2FUdJB9UNFA(Zh3=4=LxCxZ+e>oCJ9VAbKBa}e3HtCGgJMq%nr|w
z+Z%Yob5ioUqED;T<Jq|v#wee=+Ag*2a=+%OGp3?VZ35T+x)`P{yBoyvb=J%ED|jOV
zj`J;$n5?o*rBg6dmGzX!WrIa+;)fIO819ol_ECQGIn97KH@<p%oKgDzUjA8NMbA@<
zzf}h&Pne~~=GgdL%3UOR(h-Tp3sTwzT*bS1z2c*$@^0S69XDZV()q&t6W=rcM?Y!~
zs(H4_WRIKG5~io7Ti<n`JG^M2DBs(HCqGlSR@~EWesyO?`(MMvgt+51U;AgC7wO0q
z&{T}rv*oDB%cxmfEGO^t{*)Hq7W}u;VpH6o!&A(@a2)BFvnTiL`PHr)AL-6AxxLLd
zBP-7#@44i9lM9D-z80$&Ry|{~?82w*lQZN6j(iVqxO8y0?gf{iwYt5U95GjPYq=*j
zIZR{t(5oe6)8QXG&-}voz>nJ)`KzDKjJ;5x;3W6slxg(T>}^`t-z3hNtRQeD<IIHT
zduq=Kf4#QrvzXiuFWueddGA-YG94`U?0R&#P<orO^|F|c>%%84m6)u0?99v1zcalw
zZv8p6ZqbL&MveJ<U%X3-)KJ`e+aU3%m8zO<8OQU>=`YO$M5oM3o6=Gvm#p#J&?x<Z
zVu5vtV=lASQ_bYn*Jpj<T=hTyiDd5a3X83p_s_0eeRzr4CY3!O=iLkl3flRzaZ+Zy
zTVb2Zxe3=&Z8a)3sVv@V^s=&4K{0Yx{Qi17Wf`HMi7hGi9#zY0o(pv4U0NO_p(WHd
zW$$)9&&s<itG(RwCx6PyQ042G(lcdc`T4}(MuzK;El)8_R;_n^p8N2|{Y83dmw$i1
z`g+RaGgqt5{gn=T!>{aHWpB-u(C%|6bY{zHBhLGQ+!q(hZ*8`l&G~^(azfN!wRbcA
z<XthZ_mw@Ht@E(-@b?pWhZ3bNmb8Sa1Zm#>sC~uY?!9^k&E9OckGba@-^^QH>*e`2
zm+#SsEmaxEkBOX%TlDae-go^4jh;XM{k@!iI$h_#_T|0L=H71=(wpce`Qxcp_lr9Z
zk1QytIb-sc-FELK&Y%pvQz{vp)9);B^17hE&8qnOU-poIf69y2|36uB*WT^Ro_~9y
z62o1USAXj%U!uJGv%JjDJxrSOf0UmST(<N#f7spZ**~V*fA%VIa{R}_>MHR1Ms}UT
zl<Bj=|0r;@H(JyfWX4|3UH0OV(R6F?s`+21%sEx<5$Lot`I?#M2gf_=eup2~E_fjq
zpz=@pz66u<(fZ#Ly(fR$=J8~Ho8;-Ppk;p9yV&+PXJ4_ntl)f4uxlNQYj{)m3SS|=
z0^J#xmD1fFp6#tvt(eYdd3rv3IjeVj4|nrZbA9y_+%9o4eX-k%ukQ)m<J}l@Csp~L
INu26A0Jvd?cmMzZ

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ah b/third_party/icu/data/icu_conversion_data.c.gz.ah
new file mode 100644
index 0000000000000000000000000000000000000000..ae31dffbe2afc8ad59ae1dc323447d8cf9d61032
GIT binary patch
literal 177696
zcmdO!qnj@oDmHa*$K*{7{uA7#a{8u9zM8P<(w2U6=bOrMhg&w?)%V(at)|yZc;ipy
zVvhXR7jA7^)bIIW){_U@BbS)&GrKlFXl+nR_|h|Wizc6Z{_M}RUA$YYq>Nw2OkPo`
z<5Vv3{i^+`MOs{2o$ofqx!GRMUbT2l>6?9Xb}#?&iQAD|Bs(r`+C3MGTRl^9b-aTk
zcBSsyqQ-el=*h0vkwOk~DLWURyt=ZzoAtwOtF(FPr^NU2Ja01p!1I%B)r^DQYpyIe
zfAQ((s?)~h4-bCs>-y7p_(Q}htJ-_d+Uz!a&+`kPx$M5P_p7Qi6Vh+EUi)#`#p!B4
z|C3|PZ)arQs%@Bc;^fB*#-}=ym24iwrOg(5d$d%@=0(~P_mAhkFlIl`TwZeCY>LRT
z^M!UzGj6xf7mey;K6~%z-_P^+I6Ri9QFi%U5whsnN<;rclV5JCW!RWAy|+SY-<LT8
zSzZy+pCoKvBzRTzZw~ytfv3YpTetJe*A~8ArpET?JoDD?v6*7_z_cnWLNNCDlZprX
zoC2@5WLYj)am~0(OZ~X;4>qmhKQ}Yiyt!0*PA@k(`?+g`+Oa)#|M$69c}rE;Em$I}
zvCGpUGx=;}(1V~rhdCb`@~s}ceYTHx%ZCa39k~5#v!}2w^?ASQp6}tCyUUaL4JY1<
zl}lLu(bE2}!#rud7rTEX{f!M$=DjGZ(S7{O$9+YH&xBL-MSf-8H<q8b$tOH@W}Ro~
z!<<V$Z5Jh8Gj+boePzSO|NZL@MTyPluUQ@w(7pchqxqJ%y3VX(x%I6%q1=D12gmyW
zp{pMjd|mXSV9UOz-=`FH$~{x9_LThJ>!lO3;)sUg-_QSQ9c`4?bo+eF^4lqQRP50a
zu}77weS;>6nZ~V2Jd>}XB=&cQT!p)hu=3^J+UZIfQ}&)#oOj251DD80yOV!vZ%u#c
zl=pH|LiJ_UmLqCw-EZr&e$OlAHtCrBcJ}5!f7t}gXRDmr*0X%hG3Jk-Y^7B8Ov!a}
zmSJadIn_PWZ_?ubhgT%j8FQ}8R}Xmf+iidU&+g3S#>c$>EkAzfOP=q|+y5pyPd+LX
z@3+@(?VMA0E^O*q-}`!jyxPNE4OWxHkKDcb|EWOV|1&G!U-)`-*^2o&UAO8IWeq>5
zJ^tJ??V0|uYlnWvHve{*{3q8UXO8CPTc%wPk{`#(znE}(_M5%N_ZxWbElD|C+#{+~
zG5N<`pF4S0Yx-V&^*wy6YhJ0;wPSgTMdqGY+D|&~a?|&_d(6lr!8fR6c74IdCjx8l
zFBE-LHn}5r@>i)@;!1ns4NL#9uiWLiCpO^K`7PXn#d6vyXSxfW&g2Lg9$&d+qid|y
zA=Y0_^XnIEZ2hdsI`e(Ywh8wa_wpX!G4t?CrA=Q=Rxa>l_3i3R^hw!tawq4e>(<Jw
ziw{(s75|}pTjHqH>f0KxykBW%c3yUQooSV&5PYMoS?8gO-kHk7hyQ(d)T)?mZo`>$
zPx|wZZRbVj-<P@RWphvZQ^z)*Co<}vk8>P7>HmDC>RG+6t$Htu&e)v3bNSRiqkB2)
z`=1_pJ9YbShK8PdE9XpC+NC?~)g6oY1)HYv2Sl1Kxy8m*#`mVB<Ad6a6_uuZ+V@V@
zF6Y^B(qf(1hE(58zN@dl@>1`0R$g41v}KxLnY4>*-fNG7n>*f3Ia?=xY#NK^`op(R
zN%p23&KEAo?#YurT<A1k*mfs#S>OK?%jVwrGRNck)9=m}&elF{Hn)#m(^K@*i8%31
zJ9E;?`Iq$C`2(JqTHC*v?sL~}y7`Nbp$xYeZ4<3a{n`GTS#J$f^U-$h%h#N~zWiI`
zYiq-=mdquwyV*aVK6YSk#Wo>9`KlRu%cSCEeyA#MKmU^V)S@<V{vE2x_6u~j@dr%X
zc2I^_$)a-GV+Wqcs<(n<thf4dmKD0l2)}tWr(&{B|MRx%7YjV<yOSTeF0z>?>$Uq%
zU*`1bA6<Q>JU^zsa{BBb{q2@tH`Sf|{5aI;?j#n)^@lZ{GWo8$`$x(*Z05#3&JSNU
zo|n@8Fyn1hVcV(89-iIeQ?Ay;HB=ei`<Oa+*Yyv#%oeWkeViTlpVij1N6p7n*OXsm
z{oy081uU1?+|di0xcv0)D?PL0r9M9SsS^D3*vC&Svx1HC+Ew3~%zo?f$*Q$xzvCQr
zCWgMr4_7`eeHGE=CjKSp?(QYeD>us7SWHR26lxk%{q&>4m5)6itfcq!F*ANhm$E<c
zGwyi9UkBlDuWp+A+GPCvopx6E?-$?I`%iw|EAhMMUV&P&(XZ_%9ZZiHd=x0E*4KM~
z^h$i%$*Yf!Ox<zf0dH@iQ}muRUiKB~amT;s|B&r(R(^Qg`2T~6AD&-!RbQwqdtoBq
z8~H!xx9>Ww@YL7nV(+@tY398`^v62mw>j>v0{z*~MPd`~+&tI2;Blhg%Neg<w%cgk
zOFKB_)}kGZ6Q4J$vhYrKo+VKs!m|He)1-5EKi{dC{&Ulb_Wf5%C2sa{r@qZuzbQWK
zd)>GD>QlD!x*hr@nr}0&@5KsccbjF0WJ9>KX9NTt|MjWjmB^O(sy{OoW}R4<c{Iu@
zAh7ea&?onwwsIS<)t26JnQrrf>AKDBwJ%eXF1)pQW>oQ7E8@xPg=?CQE)?v)wBYim
z(7MCAbJEW){xn18OVg?I7dGvhd*Sfs%R<xlIrl1ORZR2i{5QwG{Cw1k_ZNbX2CcGR
ze6rR2MPcA!%K(A?_sMGK7af~pU>tVGfKl$T<Muta(a&b6_J0ar@#%S}mFceE<u@CC
z&h)jY*naI_^WRU^k9W@8w&Bh7$qkoIX?nh>Ts~KkEq9thqWzj1^@>Y7j)!cT!ujfF
z$6}4od>a@3VM@{zVC8fXj(SwqIPHa<gSU#6hD_w5a~8FJ>&^LR{F}IR@7xpb%@+uW
zsbrsBu|iu*BUy9xinCgC1C2fXojhj+OzL4c@>h}lYe@gh=}Tl~Bh`gk7H9^`-AnzK
zVWJnj*s3<~cY3G6?MtWm78FQ`6zXKBT@>)TVjTPLhr%%zPp#r(7kBYB>)%%0vNdpO
z-lkoKD_6aKGVh4>x*I!|tO@>Zy8rz@<yVUYJ{c{{sI31`bLHxGsQ`^2txK&@w`Ou^
zD4d-Vw7#{oa3;U-Qj<Ur;l^#*PyC#B9J|SwpC0&Bc}u{NDp$d^b2T4*4{KP*rM7nO
z?8xH(tgZp)f`hJ0+E01imzngqS|;#ZFx!o7%R6Pey$jEU)Tv$SoyWH*W{SplE!Q=l
z-wW1AE$5c^IaX`%`2BjOr3cNKm$g(xFovIdyl}@4F{zC{8$Yrd_5ZKv{`Zc@KFi_a
z*Q<xB);q30;&Jk82it$npjxpb#@iP<{Ah9yaQgH4ih#@WhrCu57yfgesb9@_^wxic
z&AAqv7u2Wjs!uu}Ty_0tdXnwY@{I>S__<~L+#mT?E6e5t#~u!uZ9XOP1{YVv%836=
z@Nn28;u6cjH?yO#C|&r})J}=>Q(p&)T@TQh;6Crw&6ZE)dlWP0WL}VM{j9Oj#?$hq
zaX{Lt)`A`Wt6P{u>w^RQg)S|xX`FIR-{|AEa!a4hO+v-e0-6W4Z8L+D?w0wPSe(51
z@J?;#;_9xi)B3*HT;uL=y<zd%L*vzckM|sbkL`Cn`?zc2pRUV1m76V8f9!R8wQyod
zYpKHI@ZKf2TQB|C<8++ECYOCx=IzRxw~j{m3zTa|&sed{H#z4e@6j#XO}X*{Z%&6c
z+RXH|2oJjXr%37T9o3_+?I%{xIP`(rB*4p0sPFlk?H{vZu6x*;oNBU|;K^9&HS>4T
z=N)esWtY$6XnhhNspoY{|Kg|nb3)#6y?)nI>UAl9ncu~zmrcBnr_9<a@^98TwVR)2
z)$B9(KDFn5ixXp{W_Nng!hjlw_WvxhaW6g?Tm5P2`@kpr(0@U)<mIp@9iJIBgR5+U
z+&k(ke9VLQAE?@Kz^e7Y_LsAd${lX`yTf<>U1>Q%{SU60Zw!uAY+QWq!lhDWuZ;B?
zN?ps6Vx@vgto!t94_wo{uG*RB<j!^Z`42YJ=)d_X{W|=oe2!25X!oZ6^Ne$s<~p{W
z5t5Bx`r_;<!|AW@O?dQZ%BG!rJtlb?2JM`4d(u2ho!G}Yr_`+WdEbsU`8SC@q^s9#
z-LJHyqfQ0eMN_Y?z5Ce1r1>uE5z#E(^MS@Eg><slnA*D=?PfkzaO_;zA6tKq>fZbB
zg}D!Ge`{J*u<t*+*3-M9KRk4Emrh&A;wK;AE&S|uOJ!H*seN}gvFvxg6ei9Ww&m=~
zk6t_?5B+{leq7zPFlM^yF3q_w!<L=@+7<Qa;j9~ayUN3bi{4*!y{YFNw7F!p-m9g@
zUf(O=^K$rKH!msNbl>4)3lEg~$o%27+H>JyddZKC+op8IJ<~bYv|&-pr3-~OYkoIm
z9s2rj>69-<=^N!w?+DoTJ9ecs_rBftTg^2%pYLs4YCUhBNL^iHj|0Q{b@d_(|9yP&
zTRd1@G(cc>v44E-ql&#3+T9*`6&9#+rF^s8x3H_?psQws>@<yO@kb<=ZV49=v|sQ$
z{*Ybj{_xBGhvKc~Sc&}O)9{#oLS5iQ7mJbiZl26Dmz<CP{ruELW1*~(-Sa*Btvxc{
zf8uqn`JH#_>6(rcn|eJTbU!+mto_esoA41K7Q=M{Kc7|}<2EVy@ZUjAbCQgg<)Vi@
zAzf47UkF(GepTM0J}Uta!%w26Z%i%2=L8<J2|6*)_U9~)Wo^nVW%6Fas^<Sq%nx6_
zq&qoVD$1L^WNVFExVHR))uE<2%t}YMDs`^$P^z9e@tf50{q8a?HI8S!ncx4LoUNF8
zTIpudIgi@LQm$F2dUtLWUdLCII#DOH(su7XR@?gvk7Q=(t>@J=llv@o{d<U0<9d&C
z##$d2@#(*3_}}NXNA1C3t;F9y>NxVlJ0?Ba-T9;2*5~5K`3vqjn}=SG=WO`&tkCCS
z4bL1!wm>(D+x>^;*ZN(b;`H&1RIFgmk5=Qi;sHYEsvpX~cxLjqqV?ks#$)kiu7ZIP
zz4?BUr+WV%ymD0f(M0nj-1h|RAMyt7Q9Qdx(f5z`-pAZ8=Udonc<;0Rr{KNsr1j6e
z8yjZd^$6k?Iki4?(d#KH;*VY%9yf5gR?b<ZcsQ(ct(V=)_H;eftKmD_6sN{5HMcAA
z)%+UE_3Y7l>A;ZMsZ~1#SVf|rUpZFU>ala9S4@c4gG+uF)D3-%{AGm~lw`a(mCrNV
zG-knyW5!)szZhIYwU<4TUbLWk`eo<s(+*e6HB3G8SYr0?hNf=`=M25M=6<t`4Hn=t
zSKd5LednfAhAHg@ak8goPqVJrEw9dcQLD1pf1}J-qn9)Ft1bi-{FtSby872aD;2K9
z<1IViF@FgRSuJ!`XYR(f=O#AO<MW-oGD9cb>~35B&OVEw@Nv_wu#1oPZxAdN)ku~L
zww1qr`TmA!TLT4-Smf*JHlMGWcDmHIBR4eZwu|b+F0VuCAwn*yiHjx)EI(g9TZiW`
zqpJmDWOLv-;YAkT`C=Wp1(vfO*s;meJ=EyjX_3>{_x!!^;*7pNXUn6m)bhCRU%5M7
zq&4Ks1YD;wXI(mKG*3b8tldve)6A%SCAJG+W|Sq==BNL8%O;{6_w2=={G?)y>oUf2
z$Nt{6Um4c4rNk>@uKuxLJ86Dbb???Wx8EkMixrctNR6%aatq!*rDGP0ym!V7zr8*a
zQe(AKe>u*z+kQR7(ecInlc&SiXVxYK{_hkQdb%fg-aBrswekU)Pj^@HzMi>H@3?E?
z`ia5w-&|V$@$ThYD%*0kYqxG>nl}A$E0;lrMX9al8Qt>SnvI(aieipM%sy9qUePYC
zBl+1<8KLt<9DVca=S?qs;M??a&XF?f$!4vW$}JPk6)U@$l~k{3>i#(GySX!K+b)wO
zUv@qD9h$NJ@q@w}^A;_<o0!x$dBKyc(5l4@DvUnqUWZoNXn&3oTTt|7;k>}-x7b&F
zp4C48m=9xs<>8MumtS?+)jwOd-(RC;&V88!AC>O?pT6r?9COz;U!ga1%>TF;7Tw`*
z?hTl<_pGemoA%Bqr#W`<0jHWKO<?DaS)mdW;Q6D)Yf<2DAJxJfU#?~AHB|hb*e%w6
zR{VX6;r*uz`a%VcvuNa-bx!eBda50=wBTaxu7tBQZ~nfwz;j;0&BY(TAF*r;II>rI
zpa1i`-D^zErQV;ul{<Yk$JV#@FM=-rcysRYn@5+8S{5ywTG0@9DN`c0sH%He>bKZq
zQjb?o7oNTA%ErL$r9EGpuUa}rv&9`>v-`@YqMy4s4ks>Q_Ig#Bx@Oa>+Z#7FUDbQ<
z<9%(Lc}8HC=Y;<{pH1o=ZFOUuqNNg<Q@u2kC0<wUw|H<?^Q}k!+dcc#IOixNIo<xV
z-S?9J#FX1J5~nl8En3;lew;1V%cAuD(bn%8w@#%l)^pmkWS(Adf>K_zblJ{_CU)f;
zThu#Vsuiu^mRTS8Xwk7v>vF|S&Sex{H?}ib)nsvPE4Sq1&#dPokG-*ctXH$<#Ygee
z&-JIh=lN(bv#zZF<DQ3>e;tlj9BPz1e5Cl-%(HHe{tr#(R6gOdO);I*Ci3db&bb$_
z_CHS5xVY_@VY%0r^eIO_FrHd;O#X7{_v^QlE%*O?T_y4SXxw|B*UEJ%v+Lb`xh_Ai
zkbCak@!yI06p!DdhRcfYOHNqJzVmxt>908FsaR9-?U#4TtTxX*m%Uv1*2|1Ct68$l
zVn3B$Yc{wtbc;p_9nfmH%IzR}WR+qV;~J(2)(u<>Li2*Y{WyLiZ;hfEV-C{|mJOT_
z-f;%=I!LdnE52MGcYZ<o72}thDwFiBKPIY|y*`pRcgGsx^wLKY{nlLAxKAlQ#g_E~
z=K^5|{f0P(y%RjnJw38v&WtI-@<mDZert|*9=0f)!Kc8HbU~n*fsK)eNrJ_IBVh%b
zgTUf8304EH1d#(84S@`c6%DzPt_Un<P-FCAn!#egnPB8~#_!FE*OFg)s?=;Z&vQNB
z^_+Rj{dkd&Ta`e5VGv`~VT#zqAmXSK>JRlwg6IJ$)&*QiXRA45SDR(9rt5g_NoC%4
zrjO;qAvud&L5pWCnQSGOb8pW*EFsCj(^F;JJTLJ4lJskvUqtdYC|GUuP;c;Kn9E?r
zSi<z8#pQxsT!<~}1?~mnP``29I?l~|5yE$5Xl7tj6x`A;E|ny3K%oK5XY6S^qR|oP
z+`|O-+QkVHtcIB#o(z*gUURz@|3*ivF`cJt;_>bk>6eUOM}ApTWoEa#Nt&TcL2UhU
zh__faa3ri`a}apFY&iogo)XS71+Z?+>Ih|+%>a!kSH_H{8zx888hja~86VwKcz0y^
z1P)pGI2YT=^CHhLO1~P+Fn0pSTA8>CXcU3G7i-GU^up8y99s-kj9-{uw7RV5S!DF?
zp(fN5JWLWTE*W(noSdPt1&SiDKe9Rk85T3RF@pUkDKuF@jn|Mf!RXSv`|cO*ZraTk
z^jBNc8@NT@^*5C8-*$evc!N@h>ML*pg!s;!aYa*v=@G4lP=?hI@2zGOII;|${1Su@
zOk-WZnRHn|nsL%MRdrX@#a$`CYq&a>)IOco1vOdT_4g?m(ZwG2t9JSIEc&Z)s~<wt
zZmPA}JHg{t|DxJdHJIY{HqM4!Ulzx>CW*Y&e(4WYU%RQ+uGQsIoyc2le{YyrYHVlV
z68TV9HMfiVgjTtzh8zrrL|xkvl@8C>Jh#I0L~J0*=GbS)TmFk`Q;lExU)U!!(?!+j
zU@(KX2*|sT{08ydUWO3IHBw2U2Vfq~nayD0m?M=Wd;k=4;Mly%AfS{PU*5ceM=MZM
z?E@&ugIp%`R{dffjM(^B<&wYF@;>oMe^B~lfh0_oN#CZHF!+Pw5u8*HD5y;OHaWw-
z6G6OHpIEnON542`x&*~1I9-CWoTO^gH__w0=@*P&&HQqvO3YS!9{;5}vA5|L_bpog
zQRDz9+j)yMD0M7-rHP2T#mK$``z4NHmVz1YM$QC?|D+iwec3$c^P;4Zk33aw(3A;E
zgJwvHa_$6=Tjv+m9$oZpIW)5|R4Go}GT-$#m(0;%hP{xIqs0Z{9*Coo1U`x!$nfZC
zk}y32E<zv)leaTaU~(I{2oXAvvgGdm@FRDw2S2J4d@FttPHe1oUCrnqc;wMnh`+i)
zDf21=N7CIdBJg-x6UnejA&hq;7t|-S8MbLK_zKK!1C@~p$_|(Ig*XHr43K5W+~4u7
zJ0o7uu|mvt^*m6~aj8!5t@Z(Rtpn`PxC&)f=!kRv!t??hNuct?L*=db#eGh}zl=E&
zpasYTk6ZeS{wg7eZSgL@jvfJv)E-rTpb07{8PuGUgpMpxRAcmUyY=_C)~)H=ZRZL4
zcm59DqW`?oX7#*Y{|j}Zx7REE&3aTP`u23DtUsuv1QjWFSr%|5T@`?4I@S$b32WIL
zVE%!`R<u}yT8FQ54vWXF>2jj}>(eg;d#JoU{ap%LoPf$kC#B5&UEiMS%0AWYT@Dfe
z6(N`EMBj#A+&8EGqX@`%47H{VO%f(YU<D(%Oo6KTllc);q97Iepb`ZfV@4`!jy_T8
z7wUxHnqLGF8~>_Z%2#^_;auGJM*^G>Rk<7lk4%Eb9oRc@u&9D%vu(bp$#6Eqwk!sY
ziCgq<GtNWQY2Ti%XoY%#1yqR%dH(j@q93kh1FF)%DG}s*O)j+jH9I4|&aQ$B8gI@Z
zuVSP@aP`KSAap>b!INQfVAArxF7D!Xkdgyb;e1@OWB!ZYLr5k5Nu~gnjUl$6c!BsL
zTckm&Bh(pEPlFTTS#Aem<dRa^VM5)eh&e9uY)B^aHYn`!g{C`D#jLK<^i4STTMZYi
zB4emh6x=dDBk?&XXH`M#<jw_uU8*hTL+WQxg2XKGrGyqkN_>!~PI~OFfA8|k?@?XR
zqFVJ^<xoPZ^drm5=!f^7xcv6p@*YZ<y!r&HV6XWi)zq^<H8m(PLMt=Qgl?t_EE_XB
zd>Ljlm@%omJ?%7Sj>mn^JN{sEwDP;tFK&LN+2H4)@^*DlZqU+OPh5U$dCZ=ZG2boL
zZZ)HT%X&p{bq390DwDp=&4@RRn=Jg+`r<y}x2uz~i<D~B3+*j0?&I2|26pk)Sl?B<
zXCc|*w`IN1s*|1HGOrvpMycPbFYXh+{avY6duOLOxN6<}k@ME~q~CE%-mAWQ?0O}b
z0bGPIMX+qhgVodEgb8Y6g)u9D@)Wpj1&&5tXf%R+sm%n-Wgt7=uC7y?$Mgc07`3L(
z4AC-P83Gd%7xMhQs%>TnuP{7Cu4i))Xl?_ysz9;k$k05&<5qa(sU@GL{FQ_?uu$R>
z+`t0+Zmn^KeaE+EFma<!;3a$K?~1^$IW<jaViU>BYOC`mg^I)5kKa<S`fhrq7Fjy^
z)NNJC(#bu$Jl$d$3YuPQf_fO9-&V~|0V#c?32IR?uqjU5q7O34{N$?c<&c_S6{EnB
z6^fu@1KQAn*8|Xo7SoI#AxBs%MBerH$Gx*<GGmWL{k&!Ixumctps>iN@RNDrCu2$5
z(<#$ZxV$|BKX{fcQZWvg6sUGmVsc7}r<c;46e&iAGrrThRZhE3TFP|(nEJN0Z+JYv
zwXpnlyvA+z#A8X1f6%eV?Xn9>R(kHb%+jhHKOtmkhEI@&+Dq2SS86<O?NM19Ym^(i
z$;Bf?<>G4H6VX%igbJj&Ry~N>aa-=*k~M24FFVzL;@{_@mBluVFU+n#QN6C=@UV4y
z_vY#E=U(UEQ}<x;yvXxP4EtWSGO#joF$u6Ja5#8WNi{5xcl&Gbsy)zFb?<e>S4RV0
z7+v*XJ~K<3Z}QTBb3yA;HKn@rVqJwY5`VMToR!h<U7REQzHQpoD?$oy1XGHb*&80N
zc=447+Ol4iF3jNW@!?C+9_~+?4u1C%{21mk?9^h2v;Xj`G2D6myuDRlq^bhi4{|yP
zG$=4Q+Ojcr{c^m*vG~^M$mb8A?tHNLuY>cHuF1BaR-|o<k1XXr`j0(^N%p+$rFyB~
z2UL9<k7Wd`oqo#r@~Phs*mt&7?ewgww0<`)Z;D;x@;+sMvHSDCSiYXK{KaX9-wo!)
z1=nm9z5PR8aI(*Lz3<{NPs++J)qce^ZN=>QH#YZ)e!S89EPDCUOT8_@2I~8cO=QW+
z`>~{vX_7^ph~ZhK)v*&<7kexXy}FLeVJ*{zFU@LO^Ed0hRbM3N$`+Wk;1eT5M_^!r
zz*AOV^_<J~OcE>_DvTVF`d^wg<ZrK#Kf&=nb;1{;a}}AdUmTIDnEHJ4$H*n8G=y@y
z=WOq~W%{pAZo#RjM8kS(b-toFzLY&RVYw=d3VctrWK8}%FbP@y|IZJ7F8PV?U!IWO
z^UTHadX4tXk4|mNA0172S^YS5!X*(GAIH4Oj(L+D<0kCRPZLp8W%Mh0!t3d=IG=OA
ztAFi%6>d|;j8KMIAFC?s^IXqc&1K9?GJ2(OImc)FuL-72I+M;tt<#zGY+_&fRn<Li
zj4QHkl(=RJU7uBV<HzBLlh2ktkXQWd)G8hq|6|GWI*Bi}7rt-$RXNik;6!KR-%H$I
zCUg07^?az~O-hd~SY!2XiPU2AEj7M-?=0JMXW^bR598hmDAzRa6xut}c`uJ%+%tZC
z(GAnP`s*j3j5*aPpt4yn(9-9>=a$nYv#TD6O|^`%ON}{Wv_(X7F1YM-Wav_uwgg_E
zaV3;u6oh7o4hy)c$aJHNQQ(MMN6(Whs#>pwrj~Y|+U1fOyU0k-KT_XCBl_xQU!^|1
zz{u$<)_99BSU8q=LaVi#t14u3os>3*vuO!zf+`5P1~r4JVSyqWuOX~i<$UjZ3j;^e
zg*mXEOMu^MJ1L>3XOCDE$1XzEZR&B;TZ`Z2g4ldzr$h6WNq0m!eW-W(uW?V~yY!6D
zwt;@?W;r|lU3$25vyNc=<J<MzlPb#EEDw~j^~DFgyt=<K(DrXW$Ch829}=7OZ~ax;
zv_99bE#vF)-%RznYt~o)5a(ZHtgT-^S8M*upGT!b72bAh>1o~jGQ&A;YjFGI`SA%x
z_YP^Uebk@+vVd#B{uALp%@?fX-}(Mq`djC95=Qw4Yv0@NKk|D0buWg`Uqy94iJ#=R
zN{e>i_1pT{*UJwMv(<K6M9xmlIr8i!d+L9$V;09%|5bf^Z?k{$MyFj_{p)wUoO5Kt
zg88g9;y>q3NqTmmZ@tj|NzLV#&I|23{YTGx)5U{r>$&z%YEHO#o@?LgKYn~6pXRUp
zvFfAt&*>(AdA_{v6k4CXK<}eNr@GgkJbR~3b}yU2N1a0JUvmaL>J*Y!X?>jzlCueX
zBzl^4O2wih!m4o^5ale@i;gg>&eQnl(kbp`15)lkrNVOuRC%iq{}h%*F6NIF%)G?B
zr@U~IXqm(?g=LY7fa<P|tP4~+OuNmwRT(unT|5=Mm;!|wCOLp)Ilw}l@2na%I2|A|
zaDAvkE{jxzR3~b5I(VvhDFq6(PU4t?EF^ME3?k&R(7x+W2S}BOq##si-Q!O(eITJH
zk<t($6(QAK>>q1Yyq?Su1&OFEa+#-H6yxzDRmm&mhQE_1lb2AS(&6QA;yCZBFLEhc
zs4_)mfr_AN&<1S|BSzIu4Nn(OAupkTNgb11rl>4ZNt}iuq$&9JdN<q7h1#3<g{cO*
zPu+Fl&mSS4efv(?*czXYUUIgw?&O}o(#Pdfvs?-?Yw9{L{M9}$AL!Ye{p7Q^-MjlN
ze>=owW5jlUiB^<&EX`%QBiqjS$gDYElvfEC{H>|ueDinp8TpI*<(&Q)JGY&5wO{nx
zaNUaYJduupIWsRY>8(A_<C3DNBrLe&xE;5^!hr2c2bnuG99#sIxKzLOJ$GVnV6xI&
zGTF&W)5WV%@n2+vsOxD_CD!ea(;0(A?87^zG*w-RpZ-+;%@^&@;><ccuQg*IXgjE$
z+uG&+pXqDbzIT`Z&OVg?LMU!mO?bo0y0h=%FT`(C*!}$a)ZO)$fAH?*JRMlGYENef
z*W9U-*WYmvm0nTA9GbVlYfp`>++S;@WkQBirB@_@gii5>RI2`|ve~!e!*$vJ-X%wU
z>tFZlU3#=Wopq;%$%aW$T3e#ecNJ<zZI~3Z>Z<-B@x9@BOSs<uxikL<pY+P0GgIbh
zItnidcC-pz68!$M{J&D6E4SHZsF*n(Zqk~u>7(=uzj<c^FO~i2y2J2lN;<Q7z>!qt
zht3B-<uuwWJv0}xE73d8RuvSHru@*E`}3QvhQGsO?jQTwn#9Jk(N=W#%^iCJuG&=F
z{re~S-lX>q!>!rrhbL+6+g#`=t<S&9)}+ogkAe66^d3$hi`K?nzTb;q)b5iz(iU8@
zBDjzH&)xms{%pLy-RN<yh|u;0^6a&`r2-;C@%-;hSZD0=-MAzmxW;x;-JCC554~Nm
z^hZ_7Ka1DVt}2edzkJpBRe13_*R6)2SFaDAU3(%_JzM$TyZdH;cf8E5{Fmo4L-~-%
zvsWnshwuN5R`j{J{v2oGxtTv=R9|^dd$8SQhVUT~&DV1To^=?ww`kT~vs64InCQn5
z9I>|8X@>G48Np=<(a&29+*>RaeGGS;<4o-1D0Z2lR6JMs(42QHPBWOw<_aH@5q#EZ
z(E8Czzok;?40j^eyHcZ{kKX??6<jtU{&}l`e@msJ&%^`gxD)4be17!QX@>YA8^LF-
z2Ccl-U>(AVTxw-;9G_igh#wM3ezQm5S*L-le3wmIQugDOE%~hmN0qH5FMlZCzgjS{
zE2zwogWDxS;E>3kHz|h>-v67e<fC|2k0Y^>gWD-$ft1<|u0E+f|McI*7@zU7-ukRt
z@Al^2zx&F!Zn+z8v*~9>iDS%MUegJ(`urWGf2z{2`<K0#_vh*U!jO{ft9O3*cwLt{
z%Xj*hh5ui_iJPBORK4}dUWR)sj&52mc>lBYF5B-S7HewngeP<VUcRH=YjNVe_-TKG
z@7llJseZTb)VcaMm+y6RFMfEh*ZcSKz2f0+hxbZv_doW2?+-&sdynk-J@>0WCL7hg
zKlXm|&6R@DtQGH{$j0f))G;Sj|E>RE@O#Zu_S}EVrK&cx?)OUhtuK9~d%AIc#TVHN
z|BGdF*9EVCzvOAP^8Nik;t!?oSogeyK``#F_`fBg2k&p~xE>n#e`4StU0$n8!T%ZB
zPlVKSp2%&s4yosRm~p!Jay@@}Md6?K+Ya7qoqa%Vccj#j?s)yp8}u)2Jor&uKfUnJ
zr!ukItJ#j{`NjU>P2O?z$z8oYsxv;VJLt{vHMZHCqx{UP!q&a-1>edp)DieMlWlv2
z!N=b^vAMfyE?BP5uiSO~h4Gi>KYO-Mzuz6TMWLN}>%4!>a-H|)6-=AUcBTHpSAo7s
zsp6LM&%`Huzunfp>++ifgMR^v47|)DY<y+C9JVj7+bUOR?%DrNJYMJH>+|cBUhd6l
z_+D^nmdEjnjA{~IySDMZ6=Hw3g6n6NFyr%!>TeBR%{bp+>7`}PQy{*wdQv~b*QJif
zrcc@a`#Hm9c7+~A$EO$Ge`Tv&EXsb(>|ftvGZ~AQ5<)!O9$k#W(*L_7?f1@$zxHC~
z%9r-r#KbeU%O%Ur{eCfT>hx2`YmRzeaIUSD4Zi-ke(j0tx~m;`-}DvR5M{G#<&uZl
zf8Ni&^Y}B{-RCFIE%r-Np1El6!DpZ3y9Enk4s6Z5AHMtA7skV4_cV37=g-V((^>I9
zbi-DoD|?c;FSj1E`}y~>@*DZb|1vAQpIqGVM*j7`mAp3=&u*8N-Fb1cwCzp{Q@Oc2
zFV2?UyVJ^4PWIQuh4a<S&YtP#`}^WT|Gz&b*7jCk7CYPj;=ass_=Q0a-{BYfKEgbU
z4;xtVEZ%S6YtX~@_=Q2=zegqqE!N(Jv8;L)AGY}AeVOI*3xi(1%P;hOg?%O$pM7D_
z`|py;K@+Ppix1oUI(?bt^9zH1zRxc6V5}GGeS~!uA2zbmS-juKcS8@~=@%P%{+%*8
zXk@i!@nNf9t1q*>ezBpC@AZrIe!^=m^U26wyBI%P?S@->ZpjU|`rKIw(z4qwCQAR^
zn%R_l?~8Ep&1JFseTz+aMfP`GexCU}zw+O~EkCzg)y}Jm`+4@t*E5A_(>8yYGxOz{
znDjJj?)x((`e$48e>Pq{Tiw=B-n>xWe51ViXL)l&1@l4$^NkATpB2mv6~)uctiRfP
zRx~$MGA~pz->78%S;^c`*}PC$y#J8=Bj0K(+sZ!`uT~xBUp6m&-V?QDPt?{uQQP(;
zZQm2MV^7r1JyE;%<WkWtpIHApCw<?XJp1RQFV8Cr^$LcPiU|b_yln5z+I?@1kndJ!
zPx;bnXQx!T@U^tp{9Ruo<eC5Q?h^SRAs=7=OFrvH;LF8(@7fz3Z?s!>CV267-`Ta_
zN)Eq}-#;V!+1;hHSAPAN-C425;)8JQn}5%LNj$7OxJ>cZ^LJ<0_FJB|o*B3B&yN>P
z^X%$%lhjg6j#t%wnaCc({(4pIyS!JwW)z$}u>M8A<A#N|o=>WrW|-zKU%H}nMJ<<a
z%9OXs9X7A-SbUYKz4_HL|I3L4pH+5c#|vs}uAB+JsJJfS|B25No%Yxr@{FJVvbT|K
z;jQEEKCbKU-*i#>@I$v1{qxdSy;k=(%-MR@?zZFW1My!sT7P=|SlLx>Kc9??{O;&I
zcGr5(N%lF4{_=UOsi}0^-JjVk<J*?+`+t7tZ;)rOXQ*TN!|;Rg1M>s+1N;sDXEW@w
znE7(s(_`)y)-Q`C<RAZ-vFuo7jo#yAS^3*nZk)CCi`})-Tl|^4&)!`tqs>cSpZe3h
zGTprN?eckljc%^Ywg<Cp5<&F3zgITTSC?J3?(mh;86R!@t$bqJSH3p*D)Y<}M3>4g
zpZD12rA4XibJKZMeV>;+zEV18S6}e?!g(dLcKHQg&wOQhcCw#wt#9n|mCsH5tw8i^
zo3C@8dp=+J+NN~w@_C<~ZUtYjd}ZZt`@Hf}>Z@HovC~&RH}VJ5uZ`ZXf(l+=`P|BX
zmtXAlm9MS7#ys}~(eGb?1$|?;uY7Ix_0BWT|9@*ff`R}oOzv|uEDJ9#eD&~hQ48n!
z>+|cDC%$Vk*xTQJ($qk>+wQ#l|6-NDhyX;>4GQ9VbB4Ir-<L><wTt-2|1tTI`N96q
zgZKAWq6Ixj=RZFN!TRkrU!Onzdfnds-_MWd`|Ioee12U2<aqu6n;#)@1X2i(wD-&o
zlPBKZy!>lML~6g&({1XLPv`s)S?=Gzdb&Mtz0^OGKjzZ^{>nVV>az#Y-w({&VDjqn
z;&A6@+YMg}uPS;RoM@%-MC<FX7cW-5m|zn<KkDjo_Y?k0LwIM~eO<Nw&y>HHjaGd-
zv-6Kx?yr!Ueyh^UzWSfGJ$;H<=fZ-RRnb?MuV;)_ySgGP)T8o@|K2Wr!3)N6uOC0o
zzBi|0^|zJjHrI~FwO`;)(O;@jvm;BsIxv0VLh*&$mTcB=KjB#sCR`9!KW{~<n)dNm
zGQaK!|9hp}eO5U4qMcpUq4?T&#v8tzon8B7h0ob-wS0#{^yg-0rq2{uy?Fht<L>@H
zR-eyIo~EPr=j!o!@5ANg*EpVkne%PqrZtS+Wfh0RPk-BaJy?8_)yk-EN%`{fmC5UL
z!&dM6|D}G*!~5s+;K7L$OuY<sdad@;5B>kI*SdyjasOVf9=p1;2c-ACN}Iob*@M<4
z>m^?#>oYCx@3&)BT<P!V!FA8>n`B7i*^747&x+kdCGtgc&3^At=j=Xzeg41Bwszlr
z;Zx4MT=^;YY@BV;=JLWReqp6M_VV3ZylrpNow&rE`SbSn+_|@D^2EKo-^|+g&WVxH
zHQ&D9*ZRWRH~P2lZQ0DU_iOckyJdfu-kzNow)Win<X56w-#mTs@3r}?9rvsMFXE^U
z|JKHwUFUw{`nHR^H}rDW)^6Kg{d~oR#nTgK1>C&<&iPfUNvN&<8|PP3r9<7;fA-k=
z=T}^f)s<8?9agcjE48(kc3bdnDPOYP`q@6S9<2{YGv{o4m;d;{a?uyvmkl2OE0TXE
zetmO!>Z_XD_xqf><F_oDo;iENl{Nq3i;{2N{Z}+0_Ic(vGoBR}{kDZ%TUKSw`(Wjy
zrpyU<#aUma${o3qzJ7XQmgT*76}P$to;5DJw>|D%hE`_1x!ccH!`1#Xz6*YBs`$3e
z=k?4x6?>JrOnctG-Q%b((fqAOI{)&Pf4AeLHa;;lTs<LT8DswAw|!wD&sD!pYYe|~
zSl6wzTG!hAUf<t0UefEA$BG`i-5GQD#d^QrSxghv*1X*+_4-ES?F!3h*H&M@b=%P>
znz8%nZNr@}=7;Uv_;&JZ^>uq?ioJ99++9BHaD`=B*qsV%j+|eAXN1q5_3rblq8tCp
zZuMxa@Hv#5oVKuMyRrDYC+oXfZa+AjSoc=#R^_{^n%iru4*KRFcC!8by0)oMt6-<E
zw)m|tb*rw;npe$xCim7c)xPhSe_p-wrAn%NhF50a<g=^$wrpJ8IMrsayo>nZFI7nl
z!JK~P)0Q85`E`q3?(BR1cUG}?u9)$p#_wD1M)9w6{7#weo%b#5K2!C|^M;o#!Wm?Z
z-@5F%{zt~;#j@?6rLMmGYjV+S?>x0#@$Pfxi<MMauYDepUm~+VlSy!yWnHP!z2B`z
zzF&WRaF4jb+`eaeaqris|C3nuy}7%jZR7rVX)oPfuM`%1*ZaObdDg++>yJOU{r&X!
z>z-+aRrb$bK7S@`uetcjtCCImpI;lEt(z3__iEvu@7<^VmHBMC?t4GpKYOP8-D?)Q
zY4=ZmtDE!pUhd`Bk$Z}RJqx3MnXKBoEVJ+Sm#Pb|?^G-bx8E-J<(H~O&-Ke*dAC<+
zS$7N8RoQM{V0Q2M#u;<hM%TWv*|?zY=<WOZO=~s_-P-)6QG4-T`AsE0i&wlAFqc#9
z`L)gY(TnLHKUPiBs&4%K_phDm{nvMIzm61pnRBV@q2Hm7`B_i=E_KX5^^`A@#pu+(
zs#|$~cig(qUwOp-yV35+E|(sCsp@!FT4lZV!n9+*6{j{Vez*R)^CF-3>enpI=Z9rn
z`6pZCdHhp_?b%I>3+nA>bn#z)-6?X}Y!#>fBTuf&uV;$<t*}=S;d47;bW&q)!OM#G
z*AL!!Z5X(~HN=o*s>S}f=MLpo?LBXNK7aYuIr04WZaxiLJ;~;z)?b0r-%@@~&;C?B
znrG?z{&?H!;?JRX<li6H+<12Va@M!isrB|^Cg;{a=k}U$ub;KoU;FEu?N29d+AnvA
z*Q{%Atm?VHU+vBsInGm=ZFN{8MEdTRDzW9CztkCA*z<P1aR@VSYo7eBeV<RQU0wY0
zzC}&^0@pWlYi&!6x9wj1E+PK^{n;!B3N=<q8`rx!=x$ox>tJ@r=a<;4zu(x8{<>Fx
zRP{f1cD%OSu7tZ=onpfdif+pe-q&%bI&SV8Kc;IP>Z?tHi|So8yp5`y4Ii%iaVfsB
zUe(UTQiSo{p@Y{OqW;_WJBluN_S)iIve)1Lwr$@`ciSD%ODGZiwz{27H~jwG-XABe
zXZ;c0e)n8`^pd_st{3X|Zz{`+I4EqZ?OQIH@0_ST`{H%xl*^(MTCYpYF`r<rWo7ts
zs*$`%V1n0**gNkT<NG`Qzwho_`ar&Kl~T>INumE1JrrHn^Y=ef-Rlow2^%)`owYvy
zUWDOFt*`9=e)r?CFT(r!&N?|UXUQ+U&;Q%1WNxB+_PZ^gTOV-<dHnc)Mtj=&L*|-F
zEc>o1f0nv7Z&J>Fx%dN7;s49tl=t1Ze#qU<M!u0hDEz;+!@Kv|SFdl*{`iaOD_iI5
znH$VMJ_wI*xxV25ulH@OrIY@JAJp9bQTxyOq?P+;<ovIfYm%OFv)^Ubw}mfHuHW?h
z@0t%2e`b_rzV2r~q1=%8eLEk&KbO`W;af?){Z3Y|M9+O#I_s0tz0Os_Gr0v8&&pVv
zSM@3CcfrIdz3z(c<)QQU#=85wYqhfT_{TBf^Q5?dSJM=79%=fW^*Gq*zg|6crMr0M
z@AiMQS?0>fP4Q+rI&B?Ou52W0#Cwxps>PxiQH@L!YQFtxzW<<~t?2xZ|L#ZimH+*a
z^8L@`yFX0)l1j;{gqRhUakgDtWvlodsxK;ct!=5BzWzzzU%`ckYZwAvgum6Bx#Zvf
zzq42TFa2BK@6+(#eYW34(O2L9ZGHVe{({Em;{khX9xh!JQ2pO=`O1CgnL@wFS*tWJ
zF8#ndXWbX2iogFq?vuCs``T|u<eVASYxB>xC9~E$KJVQ5MR(TEr4{8*#Ll+%@6B9%
zb=AR!OW!k|tn2CH&pYR!CUqicYwzN?rX})yQRnOKJ-E1I?!vB;ge_Mbm^@S_Xlxc<
zACkfJTe{DLU+Tq?+bd2Vs5>4%L21MLpJpO<CwQ3R-HJH;6ZU;-pXYn|*fJL3uR&es
z^N#*-l3r^3VUj~~xXSj9dHo02Z+%(3z3}+AcK$yL(}Rz<=jY7&IjO$#ufeCr_e*k0
zB-rOW`ma}y4868aasSFcuNk+dDTs4!c{Fit-h;nCx_ee_NOe!K-zz`w&++d6FHFj#
zFHBBX3`wncWzb_Kb8y3f9dqg&D*Vzj!(BhNSIK`ZKgG0xbCby=0YfvTDfS!QZ_crr
zcO&;v$E6>ZFPRnRK8pOx^DFLnnRd;$#+t95e+%E+h}oLXnBiN_x->*_`qZ#KdwmfX
ztF>wFZztW)udXQTdv(&}=b5VYF6uv&0&E-8FBn(Z9^IIH@q4Jh`ZT#K0pUON?l44L
zHlC<j!gnBD$bbE#m$7#aieG7SiStrEDiITx@VcSy&`g0y6=9K6EPo#g%iFwU`oCz(
zCGqtER~zNoit4#cTklUiz1z25aLaS=Q!W9)B7Vz1JZs*6!}XE!D)p^b_|4^{^ENND
zu>Fw{vh(o)k1LxGtK1FMi0}6}x$2e0bHz*l?LQYU;gQ#ja#UEpYC}+V*h<G0G0(lu
zg=<<b<~%o>Jw-F`uN4c+^6B#ue#||*_~G;ZV`7}TQoCRGC_Y+pG%&Q_gNuyKkHUq=
z*4%%;?U?()!>o2T@*3q_oD+`p%U5mnta<PuD`%77vghFzy+x02hE)Cwkx7oNI^5hI
zwenB*^DyP`)mB%so<>W&{pHKE|G~mA&2qEz_df@$k*>cn>tyW~{zQ8byMpo)|NHma
zdzY!UFVsC)-|H&xC-ml-{l;B=TNemNhTaSbn?Cc~nMYj@c@^#Tni$Lu_%1R$bB{0Q
z)}dv(PHP_@J?R;<vv;q_t%&A6>D@bb?0C7~pm%bb;<Jj=pQH<$*(Z9|o%?QlDWP|Z
zm$L6I5BDF%pF)lq3F}uS1*cu(PMI90H_1P9$@AamKI@;Fux(xHy?^&7?2LT;F>3$B
z^b^n5zcKi9C|fk#&Q+{%&-_ZuPoGUQB2zw=WoYW3|MKVDyko_Qp5|$nDqio5@ZS5;
zx7bR2<6P<A9nv>X2%E9petz`PC!MyhE%&EfOU`uVi#+0e^5~Jix`5?w9R+*b&L33G
zGLdpOwr^B5`t|hDqT~x^IoW>q>`z;6iG7)tG~;skd6QQ2b9u)%eBDu5zeVVQc}c#3
zgy-iE4W)_;D(64n&BxSMJ0l}yU)Al#Ps@Ezm8IUc-gzYN%%o?}ONEbZx|(~U;U({a
zufjZ)CtN>mUfy75^15C4hx`GNIoBC??qUo-n5?bfWhCeIUNxZqsJiEwUzYz|kMCDG
zF7;p7LUpgq)aZwM>u2(L)xXkTw|{}n%maH*-`6Xaa(+@>Ewe3-zbmjo%eP5kLEaLJ
z=ccvqx6CO`%m2;OZ#h4}Z{CZ3@l|iP9q&_hnEv8y!jjdSOG@v>Kh3?q>Fu`u>$dZJ
zzWiKwTd3*Zcl+*!+KEqM*cLA{tTeO^{U-ZCh5563&b*48*K9S9LoRObeO<V@cGKT4
zHAl_4cmK1!RAjMVE_Qp+`eyw-xn=J~HQKHIJbQOaee>O4X{R4Q{+Jo}{LgZmwW%ew
zw)b`<{jBCIGTyQ)Z+XntrTp9XetdD-_jWit8;4E&lb>IO3;w3D>fDaLAL9RGexTV0
ziF@BB->aBcv8i^y!5Q<T2}kzkX3P2aE8jlPeW;*`JNNZ1xtimDPHw*X^2{BdC7*5X
zhe)Y5yk40TE?4;R?<SUAmqM%M6CX>fXncE;$gW>e+p<Uf?g_mG*FJpYJA3T&>mM`L
zPs{()-YdPSQ09}y{vUI1#m!(eaQyS2Y4W`m!LVJmoh=hzoM6*0;JdT`reny*Qw$tJ
zJqJG5{Wbi%Z?*r7U;itgAKhcx*!rcINxfsjQ-O;zf4&?(GqLXR$2Hrl1Py;od%wPb
zVPead%M)kssN-I)E2me*|L?(($=r9W3mgrE4=j3f-Zt1!^9yfMae=x5$C5b}kDt!W
zlj>V@XqSlTQ_)#JFWlfg@p$W&O8$LG6+-qxe>OJze%p}zqqfmP>cGV7t5mcm-BX+w
z!M^hJ9DliG>NgrE-fx`l-Xt@3@nO>q=gNDYE);)zKf~f@j?|sQug&f1Ju=SUG?=aP
zU+SfJA4`ziuU4fchty}MHEz+FaM;pP%A|5i!*TUX(KbJNeoPZSZcw|UdH(fHJAeMX
z=5ddw@L%n_5`pr6|DXQ9%pX(v`cUDX9dlzm+&{?J_{#X%oSQoD_My3_zfXFsdidA(
z&wqak<y7*2s$nlUDPFv(>E+&?#h)KuS~TO0TEf4$6aW1`JWuI;aCA+~nhU>K4rxVy
z=Mz~bY2vfz==lqhH5nGKD*6w%MXTRSF7OW9V<T(7O5N&%nZ?AXHm_E_{M32t#fKgl
zoyvy=&ielE8eW&!?))bl;P?EZ`ks3hEp@wOPjT8NUGL{tGU(oUHvhwH_tHCD*RpL}
zHLupLW%;q<cHh%?asBP_A2c%BpUbmu$v%Eu`gNf9g!^m$`=s9S`0$8z`z4OFNcML+
z?_M5TkQ)6?@A#sar=OP2IK6&a!k61m<jW8K)%RTFZSm1A+id=qBllW_zO%`x|6Ts?
z`6711im$slzDk*2s{Zvh=3zqN%jTYk$wia$C1n)0X@2V|^3c~ibT#xw)rRO5xi4qP
zt(Wf?|9kJzgYtDZeyp4-U(Iswi`V6eIlp9HPA$88bKP#MlKA6S_FjE2yKCd&zSt*u
z5B3Jl`RKKO%Sz2kok`#OPUn=S_rH5zDPpkaVb#O9+d?7N|4)8(T7Q*##o>k9F7!+b
z{P;RI@6?}d`<@F~-T8G~*4$$Cf8i9*<<q)$o;W{q{@aVPY~2TC&z0)_I`90y;(@t-
zOu}dThMf;y_IgKtz3I7Zzj1OMpI_mIlc{+!Z?4aNt?TRX>$dvr5~lm#W^TLv`QgV)
z^7b$5)wfLL`x!j7;^m_MGNoa?GuEBzOO(<s-1xY2w(dq<jjG50`OEkJnyKq)aGL%9
zU!JPpOzUr!OiH@ESWT<?p#J73^KP$O>$xTRIj@2Kq1~lrl5ENkJ|275`f%fG>v^vp
z2>0x&udC0Ve$W0VJG(H~6Y)K3H@dZX>0g@}mtyYpvw{El^dRLES6BAD%C_dP-gVMZ
z=X^ph;{~I?>7TcJKb~CwG-TSw*=1)6t=4d#lJ2dKa#cM2CAp#}c+%%n3uYu{T-g-6
zE#{oY)A=V(+^B!9KDFWE57$GhS}&A6HC*U(Y*FH^2f6yk+bo`OGWE}zCfzx)Nl}pP
zzG3gohUfPdC={k~xoEp=xOT*;C&u>UwWz)F`CfKYKRlV$G5P9#_0`J!_AaXbjAeg)
zn{m^yN&4FQz6WUwUDjQ<m^s^4r=?=|lU9$J6PHJou`~Lc%TM^S;lisOE-wXAp5JU&
z^WXQp>f5`2?hC%`YIt8*caLYqy~eXf>z!-QN6#v_7j1Rm`}5jY<y~is7c<84ADY2a
zc<*<_?ayja_y7N|FrIkj+^=so$^ri_Ec~1BAtBKHl*%D5%bMcT=Xv5kF`N8ed)Z3k
zL-i&>wYe>OXYcr~ILS)x;`)h4YT3igSls%*3z)<@t?2PAlX#!vF=f5!r~mQG&BA}C
z2uA$de{;`w<HB2K&crlqC|A%gcy~Q8pW&MRj-PCs8_u=gNcnU1d&#C1b4AtW^1QST
z-9OcSNtykUEZ_U<oc}mJ*l4-4m}NpJ@2B-MtIjYjnq7a~bz%Oq1Hyi4S_*L)3dcET
zJX2&azUZlzD*oZG(y2(vjnB4v9``)%u`(sfar$o0J5wVk{psP+vwJ^((#)>yEqeZY
z-}fhcewMUR-1bC4@@?hAT_*~Xp0>^`JYl&k!N+dN<sW4?Dqo*DKYK2Jdc~|iZBy@-
zEoWZ2Al^c0N$uK$r?%htav=Jgw!hPV$&#F;3Ca_6pY>?ZeV6^9?$NHjsx|&*^G_aS
zncf^?*l_ap;&fp-*>_9+t7kOJIqGE;pZ^s0tNfR_U!B!`8w;MkL+#FIp4@tnQL<mK
z-}b=z&)w$y+kdWF>%Q-QFjMpP|1aBEW3TL%mJ_(~X3ivw^Q-0rPB4-y6RWhJvMD9{
zp8dI>UuQFy|1S7=iQ!=C@@Ki1!^-O`_G&MS`LW_(UH7>?ifjx<dO?fBn(o;xGUU--
z{fF~YRqgCYRev?_ypO-bQ)Q>3x9%rv&;y&jhJNZH9@#fae%{+wd`79)jpI)3W!@RB
z*Z!sbyd1M#@5tQ0v9~IFLv^+tD1D#wKvvy^wcx*ybAHnEm(3~O0!@ee(`9$K<-fkP
zw4~A~jWc<2=2gbKQCnN5$M=_Jom*LPP<pqy)Ts}*mev0&*?IQMc|C?YsfcH-nY%?o
z%$Y6-JL~?Rpn7E=gOSv=g}JYCwlUt{7M8@U`=oHL`lO9F4q5EV?ba&TYot`^@`Lq5
z<7?^c7Q1zdS;wr|Ebm<Aezknz@dxF<dVasYc=_qc9j$3X`&k1012_baSsy(5(Dn(3
z+S%7nzI|!FcTDM=x9IWXx0-c=MZTuatjZU;`O+`ld&lM8xCp1I;yvAMg@MOdbY3lU
z70@gAze!7-dv#AY+s7cauunHEHTLh6-Zr7<&-ZHgx@XaK*L4NsF3Rxn?AbB#_N|TA
z4fsxLX#H9DVR!S+&FvSbI{uLEv$YggDf;-YV#)u{+G$SW`IjwE>Nc-fD|kq7(zzZ(
ztDhgYCv@L@D8#-@dBf&8D~(Hx_sqU#$Tt15Sz7j&6>I!hwq>NTW?XL2=DDDA!_&1=
z{oohdU-_}2ZSz~q!yZYTNeQ@eiGkPY3D49`mNC*U>GPYpH#htd|ISl=Aw`3=_~3$T
z&)(h;6gZMNC+0}S?Gydp&2`O-*W}gyzBzaKO_T1(+RKkW_=t%uEtEa-|9xcs+xqRx
zX7j&w+In2%vdY#e%&8U&HpNYmF8c6n<MsOgd7LYRpB)V^U6E{^W~QsL_0;QF#)ryh
zcR4m56i(TH)PLz!Y17`+nR~^fdVg7&@w~nL*v&fUw-)1T|5&$AUMUT?>O7}uxSF(1
z*F9dd*En>W<RsI8-8t5bc~P;tZ*QM7a2I#m)3SWstWy(ZEIpsx%oEx3Z}J0f=kGmP
z+jTkZ4rOlKnbX^Ld(&P~JO6wGn{7Yd2ArDq)9F)<qi^}P_p7^&@(-Fg98BX|?!0FX
zzv1uvpS|4KLXS^guur}#u4b|G-1n4#BBN&(lFjpD?$;WvvRo%^v(w>Eis6?=HKpsC
z^LehX47Zf5mz7lhbnw&coF7_Duhg4=&7H6PXPV8aNh@U-)SF)%W}4pIU-@B6R;b%{
z>CPpG-~S0)?8}tVb*`gQ+VS(Lg72|+Dt;`QIrUGZj^DyBCktNK&QE-QX2*Bo;~syf
z76vE$`=7G&!Q{VYf6d|>Z}e<U{}!QH;9r%ZTHLmKiSXo!&jeoP2US|Tdo1O;zqGGC
zssD}Xil%eLYVy+TE)S=y&iy5`Byirf$U_rSuUY37%bo4o%%2@!rZ<(r>1=|m$i(*y
z%n~(IwXE-X-<+Sdng8OOldP=iVWn@+DwZeKdpv*gsA2y^E05D(pY3n3ZGV>f=FXp8
zvm-xlKQH@8`@LqEe#-H==|*RKo7!eS{{Gf*^Ua@sBQ;Jt>^<4XaH)8+*Z*g;mi!TV
zS5=(5UV7U7kH4C3s#FvRF-q`zo8CLm|3xH!V}zcp*XuuZZ04yyKXPlP^j?%zd9&%t
zzh$>S#+@)$e(K7l)M)8hxbJ7j^%;wg>!(eUXg=WF<}z7C?dKUwN13c=3S5i+t!-7!
zF<}x+(T~?R^M10T`?biXRKFrdrZsZ}K6gaR{m!=*>Yu3c_L-&U1hY#E^Y2V@-+StV
z(KK%X1^I~)Zyzbgc1hn)E6SZL#oB#h)>9jA;c3p=z5g1de`d#>dakF<9PIaq&(c6&
zfp6jC+YQb>`~O|)cX^dP`Lmy1+x;WoCv}}&viM}<@{i_fW{&4RbNNo--}}TlcKv0~
zM}dpPu3So6FzaAygt1&bV-B;PLav0!Bb$(|Ht!uWWR~aFHY~ST_wRPtP9`y5ljf&;
zDi2BTivJRF{8y5^OGwM2iteh)qW?mTVu#)>n4-}yx<f=mCuz@G@vkqmY<4~OmuP3N
zeC4~ZaI$vILhpngHLFASQ`=Oc3xzH(4b7}S%2hG{y3p?Iq#vL5-7DYE%6ZJ-$AAAO
z!^<<B6k-b=oy?29vE=(9o;UaYWag(j`?gQ|xNq8@SWVw`-m2?vCWN#-zxnmA-H(;e
z1@`W^T6kOQ1}pcHd-}#qD{ogF`T18{ZV!{&^_ZH_RbO42qEatJhkf67+~u)t^IV%J
z?>?NcI+Pm!n!Edq7yp;E&o^8*DEr(n+g-GpdE@KTv%=0g-szbr)exh^p}@1u@2L6_
z{gSvN9%r66Iq(#(+;QY``b@jqB9$}Zukx5AeM*(~dGs+Q(crKBedm-J@1NK=u$|wO
zqEz6Q9epLed+`H}3w*aXJpJeY?~(6S5BIw<*~}+qED*jX%l4w>X{Y>Yp|w$4A__0^
zRQcr1RVg>0;Ji+^Zt~g#Leeji7Jrj|ku+gn>>}X<Qh#=a9-RGl3v2U^@6kR^l^L6U
z=1lC=WD-;oQ7jR>C&9$8ctJoxxN{><*r9zFm`tZ}hA(kEuk3b%p?gM3`R*Am7ZzW(
z&zT_QAkw(SGs8si^UE#HCeC@&c)sqsoLSzq{?NITL32)ixqkWP)Y+=~Nr}yq53c1F
zNV;9f=%#Zd_xJ6OKO<lCbmlzQ3Q4MNI?%$hLD}H8Reg@=f&=_BALcHrV4l8V)ed$0
zc)bk|pL|-;o#_(w>`!dNB`b-T^cQ7a;Z~+f?*(_bH8sidytD0Yd~Z^;e@6Md*Zf~5
zoKIH~jWk<Q-C2Kj*1mX+2Nzk~XE_{M-WfIFd+xN&W~>%6wMl)s>7u^d-m<E4XPJDQ
zet&<(n-_nXl;@k=sSGatGMhF1mY(B0@wb_W)CAt7v@<cSO8%>9bCYqgh1#Bc;Z*`Y
z3m2F(By1MlwSsT9neX!h;%*1T{#ZQZ<+3VG<`k;k_-_4Nr|5T|*`Mn?+rP2q>goGA
zoS$`U=Ei)JZjv?JdvLqSTZ^{}w+`iRtND7`d}l;qbBcA#eSxnkUsL?;WUq4H_Ugzz
z^iP+WAx-kf<;MOeU6z$OMVc~8r#t3c4M|s+@pJy;XFbYpvrlxN=s6vxIQg{jy@Hnv
z8c9ORK6NR-Ic}cGagFu~`jgFG=%r-!QJ3SGQ1-INhDRdX7<w4Oe+eZ%Q;PmLUtq(=
z`bF0`1a<~KVr;TxGB{w$UZeK^9OtFE2EGp_2Dupi70Rviah%V&VwUXQ`ZGcsY!g2o
zS-|XeTrt+!;NAU{`7777v(~OzF0rlm+buhujRFQn%w{uf?0O${hPeuKen<&DG`aX7
z;{@w7e3hq}rwAwbOs?u>PsrALw{YPaWpSn^5l{Kq-h7{@Ivv}vqBY|8{`HlzvusX<
zrm`=0E?^5iBhT_CU8ir&yo@ieA6Rs6K4{xu5@~eQ<NV~9*9z7@-PBrN_jo>B@ORd!
z$)drX3-`=d{@8DB{aNnGN`~W|N21FfX)@1XGf@Bk-`9Bllj{~IcNR|C^<>Gcee-|*
z?{R%~BL4VemCc6@r#y8(-PCa;Oo~;QS4E{cEpzQBodgg5z8CQvTz}11oLzGN!L1M4
z4OwlxveWK!7hQfUQKR_6f1P;!r|G+TW_oYEqH1?eEpVEQym-L(pz7k4n@;Aq3(a&W
zoYZkrD4=pvY-L`tYU1>n^VJKd&h<Eba`pS}_H{3weQMjz{J=ZOQr$kB$#C-e8z-1&
zH%^{-KIZBEA4{Y~v_)pmU-S3G_j3t#v+qV6<ku-`an&$-uVZy)e?{uOi(i{(@9H(L
zJKfA8^z8RUhwa${=AsX@o7;0*DupIU-f>X>^jX_K&~cJ@RMlxcOCCe<gVXkEs4i#x
z?eU6n)7QuxJMMXvv0qY@b{NczWx9J>sLflTs?%`wnY*(XJC=UZUMcueYz_CmnBTV_
zyo-03UH#W>->Zk#az|KrML*9i$ZT>9@|$4MtE$@nAmzJ3&R=z3#`-MfwUhs@jd;24
zVfx&8Cl5tE{x868vRWxJ_vFX+151rdqm}2Zz39hg9qXZPp*2Hgk5xmo20Q<o%g!r)
z&FHJwV0~<MKv-*{{N+FI|4u)_C9{&7y>#QWvp3lj=O5l{mtfuM`O}5T_IiGW$WQJ1
zk9%58_O43NDb74rQ6^pYb=AFo|7$s`w?Ah+x2s`hz5nJGwmWM3*d{f*r#Y%*Z;R7;
znYOv1Ps~Hm#UQh}*DT%9`wpK7bB$kiYp=i+b&Uz{uPx14u#oqUO~$5mG5H%dX06%T
zu|`nJ{2K4}CAsG&n@!H<%8*gJAE#WEIg|5LHKP!}x_Zz{iJHQqm!j7qRATryv$3-D
zo@)Mfg-N`k=2T!NzXOviw{KSXN|unx(?Vxe-kAUDw?+?ldez+9{@cypy4>KIKXV^5
zue43jBMCKTY3=~2j}^)%>Mkxm=*#@WLp_^8m(~4br{RkC4M9(I`wZJZOt-najmgiM
zAuFOPInb8zf%~2lRtYyPpFES0`xxD&adQ9HjUDkmC%b0;xm@(K%52*6Xc6W`Qyv`J
zUsiC9YbL|}EgR|!HWeFn_}Z^)n#&qfXr(jjr2UDmjcJ0BNkKC88lJb-W#16{o|9d+
z#_Kd+*z+gKvH>sF{Lc@$ozr$qtYXocG-ey!hUGDnC-&rYM&#C<^sVGQ>a?aNXPH4!
zQxT6{%EEm84==6E+OJ>FUt{)q-z?!6#v8nD&2Q@ax9hfVxH0uK+lzmBx(;(=%^n2G
zSH?Rmeh~8V+L`iu*Y}q(o8>BPPfa|1>`6|;lCw9Du05{oofx*zYk|*vo;u;ULp`zr
zOa3%+Gby<>dA-fQ`1i^@F;%XNkLwS(TI>rJx|_-~bKlPG>u%-?OcUD4<Qc`_>07gc
z)4)`}-e~y*^&O3>Y!i;0JnNKc5xr=_))!mTPciA;T449H;=B7nCXvk<X64UL@46OJ
zCRUaveJCLCI7cVXgczq-?%BeMPSQdiOue2OZ*q^;3PrzMw|m`HKFRjyKlN^1t`>b*
z5ihbLxm@4=$CousQhJgSZJ+bIufzq_Y8+e{ClMr`^XiiRl2SRnc1a=gr<E+R^P2WQ
zz0A#gyE3LfB3AUJR-#yliBC!0_N_DT?phdj==Fv5V*6f+Nt<@Bw2!q}y}gR{-iga8
zi&A%_pL(CqpU++Y<$DYFnab8_y*#oT7d9Jx+0k}YadAM0BG*LOy@@3sbq+8bT)4T8
zF+a&&?s7I~x9-IaPdyaxp04aaugSP5$bHSiX>%A1ZQt@YN<{m3FIQi<{9SJvUq$?d
z{C%SKzuE0J-ujs<cHqkMifPf)dB4kK{WbFqn>aV*<o0Bz#v&cTG}c+})furj&%c^_
zy!FRcMyGoL8#2V72;GtTZ7&hFdC!TOyr#Icn&0^g&pzh8y3xU)|7jQ77ca&*iIjB*
zo2;6xcP*HBxw>SdqT?Kwvq@Es5xE?jTQ}VPP<5sByt{^i;`%j9J&TwoY-Gq|ekAlj
zXq(or!|&Gqcip~*wa&U`VaqwD<$pxpGTq%dwc}~uY1hi-`ti37J|%d*>^LgD@0nh3
zH}hYHfVnJRi<IYoZ}8TUJ$?LH_e+VA{Tz=c^CgGcpZioRSF%5S0jnaHe7~-R?8;wf
zj_KSATx)SxVEW`wR_Qx3^(y{(v_527P|Na(@7J}K_fic#w{|dmn{jw~bK0S@t1;iV
z8S;D1%wj&|KJohUzz0`Xe9IE)e)?%SyP4+qjSpAypK@_mIK=$U{qK!zHa?S_N?${r
z%UKH=PyTG(TvFiAx6ZxDcS8&334L)z$Lb2TtDh@xL|U?Xm0K>#(w;y6)F&Gzfj4dZ
z7hmjuX;{a-p`(w#JENF0^t*Je%!OmGZ6lAiF@Jp^VcYrVEz7G-Y;RSTKJ_er8IUwB
zWIy{CP4y$pVIRtNCN!<Rc0nL9WbZY`hYz0b-1C&>&WdC&CXKqL(|5{#&eXI2{8ar+
zc8I6xiiFuy!$TZY;|}-N{d}vpN2M}EAbx`kx5WCBWwT39Dko0ntZ-tzviCv8rlf7_
zOIj+Y<UgtsIC{NevgYeub#kkOPt3{J+q7?|(X(g2=6J7rG3n>YMVcIipIKS+b%j>U
zR#-kcO3BhdvG|6~gF5CVv$rhKKQS#m?byzV6*r&#H<$gi%;QGR)3t}4<kWmmNV$Eo
za;Tqa{J-_*ju$d|3a1ZOEr^-)XGbC5mrds%uTrT<k~`4&oN-IV3Q46sbvL*lDxRvd
z=35?p^rL8^^McJgR3CDO3ED`$QkcUY(IH)DfBwdwe+T80k1sebaqjcIs2h?3(d&~n
zm1q6tsdex_vs;~Y=J6mOlTLMqc&%Hn;v)|~x%9-nPHoT87JmL|&lHc;-%e$|qWZ_Y
za-uLBQ<vI&&#lvL&J16`D`Zz(Q@>_Dzv6Fap?gvW0_*oT7`%V=b+659uKDNgeciC7
zJxx-H^UPB1FB4|gtDU~8mfYar?Y5A;?+ZJ_!?zVXB$VPFWp%HXK9R*)!`S?k`)14J
z_X`#0Obv^bSaZH<C)dAQ+YY48tT-j5!6{hfX!9uCH)mz1v!m9x58di&DqL*ghn0^X
zd~Bt$a?YncOkB*fxn4B+u061PZ;cS^TB~o|-vR>`23@y5cwv9I=?sPtVbkVSQqTDh
z2G%#J>zL&l6$f>H_7M2^XXbC+KWaOUmjzr<HT6jsJnj&1gN<Rqkw=R&sv>3LUbwzu
zERQc(v-Q-oCl~7vtaU7{Y;Ly6cRKWtTP)TrR`0r*!S!}=ffbd>WjSJZE;E&R?^9F1
z&GPA#P<LCMNmiiLs_PdfO^+0vkiA&az2tR+m*=IOd)O6TGaSzOrN3d_*^_r<-S1sw
zIo)y~ZR@=GO_?D<rj9yUYOD1Y3w%gv_0C;(W>xo8MW4N%IX+Q$<4krt#5v^LezeKA
zHC~Q;>Dh<d{a^3k+bSNrj(75k>uh^oGt|rWHR#WM8KX0MdgFyBJD2~v_~<Cl_hm0`
zloW9%>Rq3=af4U6W&i6v-`6i|IQak1iZ?Mg*%;gI706w$EZpoUf8uVoZzOxri;$~o
z(sG}_3tBJEe2~4B>21-pzWrBI#aeYI?u=Ag_TY~G>c@v|A7!rd{_VRlwB?6Mza@Wr
z$@X6x7x(SAzjS|dbcM}~3A<GFI4`UEb_&+d-*@w6_w2Uq4GA&wa^(wm2~2q%w?ati
z`ue}3tr`Va{%&Rw`H^lo-8AN{<{G`6{tI_D&27CI`Q@s>fuMtD4;_6xd+$~yyW_F#
zb=%@|SNu-&u(;QHBURv{<djvrJRD!%artUd@A@{cZ#A!;I{(6)=I*39{V6Sy8~0wg
zdg2*JBIhYXC+k-M5r<z~Yvo_1bno85r-|B<SrWV7?w|gzY~PEkg736a`~%)zDu1s3
zcA3!SdBTmIPb}u$&)5HXVy+4I$FE23M&>GOxxD%D?!#(X*-!O;?|z1GoZ5R_KY!n>
z*W26Y{<!t_y5E0>{JI)P*+}k_`|iFLZ+XAt=e#*>QGVeQ=UcM!zv^n!p5Ojtap#$x
zdHV5tB~Pz8b<$bgPG7fvyUib;bF(;KZQA}NDR<&g)(6F%Gj$EOsICsary*wEW^m|?
z*=3bv?h~{0B%^=Liv41|PFG&*vcdhNT78pO3Rzb^q*i(|<ycPW-)yZsKg@m4HS7Mg
z^7q*c#q*<9&8~U1M6&tQy;F0gtEb%O=M!ftS?m`u_uBvHukuz`uf<+`mE<ZO&wct@
z-T!TCIpQ@mTz-gLh`pPg8X;^iY|3Z+yHe+n`c0wL-)B|COt#mEQIyTg{IEVYK1_+}
z{8s(l+nye)`T6tOJa@gYvY(HBc)3({bc!v_dOw4Ihjv`ZJNKOP){I}W5)#f@ZT9>8
zqNefm-<pk9XMgfOE6re<kkhr>cFx%uR|@oYdNo|vFl%MkDLBJ){M1x|o8n>*LrcDv
z*Vg~J@bbPw@}-}7=YJh_T>dz;|J2)W+!OV!|BBtW-g3Emv&5&MoiQ1a+a{a2e7M8d
z;B|gwwB60q+P8n+;G4KR`So8<l{;T}(k;z@9nthjnX8sjwAjq?uFH=@Qv1T^FUxb-
z)@?R(8e7iZoyV(Bn{hKuu2OVhE7nT1pLVPD;osZY_c!|ej7vAnOh20@BrnfZ&9{4@
zoY&5KZA%yK-R(DJF{9fJO_?_$qGdnl>?(M$YR#O^ZRK;-rEjP#s=UOIb~NGAslJl1
zYUP_hQqTW#zIr)P_pfKf_q~(kOCHD-?6sNvbRMUOe8HQ;cPb0k@^MW&oyT~;?WX^O
zy>7Dc$-JqH84MiHPA=Z>x`h3K-FusoB`F5Q78a$I{FxI3-U<7++?vX@?@7pxoQm?B
z%h!6nl*@=n6yN=ZuTJ-1QS+_S()<4uOqW;pXZql{c8bJ3=fv<blL?EJ)-!)UXkTzw
zw4H16xvpb=ow4f-jwJ8f5i#B3QBs)Rwbx0apQpLHm!C^)e;>22@jFLU)nvI>X=h)g
zJ}k|f!fNVst&G()YsvLbOSUr!nE7woQIz^F<0RvxrInl&Y40ApeiM2VJ#+hxE9d??
zJ&imQ&&jlV;i;qhrwYCoa@lwH#_C0ZyUHV{eo8UctvUVs$NLNFlNOaOF+RP{+Oc6H
z<1M`xw@qIOxTJjilRP6fXF~tI-QKMl1<XP|?sw*X-1Pr|QT(p>cehIHYHlv=&fXiX
zSby5RvZks1v-_3}TCZ>YJ>P69&S2&^@lM2n<+B~?jo&&wNzWD9(7M)TV?)WNI9`U;
z-A_#)u2b{7#`11ffz-<tnOFW~Obfp7@3Zlz8ykL1b7<UgTcqG`n-f>-=U?l!@_wbr
z`LDN+W_(n#R$=E`bI~4)ex|*4r-i$!m>7=)_s8^m9AD#ihiCp-{asVF#LMPw*vC6>
zt?v107Dt0$pWc#t_QvzpC2p@SEcmsY`<k!G>h9#s+xc~QLdx>5J*}=E3w1D?y!hDK
z1iAO0FBTUGtA>bq?&&xAQ6xL-Xl!FoXl=XFPh*{^N7s%9+W!8)|CIl~&wrQsnM_xG
z`8Ho#xl}cOLg$tx+Lil@c-A-0XRLd9)4T8F#|@&3xceeGZ?6zqz}z4|uSFo{ideq=
zf2BGFnax*{{@pSWI+UkiUjFz|ZxidL6B}~<k9^;5?7nc`&hOPp0Y5qUuIhYDs$R~<
z+3T}j-Fe-yWs~1e6g+v@a~6NdX3>-J8#;PC+SX3?yvkmtJWWRE`qJaq@(*lQ>Dkgc
zF(&v%OsmJn%gzZiE3W8;pXJWJ73R+TEOc$dHI-N8!e73%9$WEgzuJV|jGpEWCtp7b
z6Vl>b{o?Nb?AW<G7+1$EmY5-?IVr_sQ_1SbtYvpk<{36lRR80#jk$b@YhsXM#M7oD
z`&k|3{68v9H)GOzc}Q&GCBxLoM^jHpe>-nF^PtGAjIf1Kud_eSJbdDpQAY2wf;q-1
z^VYw9q+HJJ<@;}WbjP&f$a1$glO)QyXYQU{Rle@sCyfH98>dAlEC|0`YrKW;?#^BQ
z;d;-0RX<qEBe45R)!h?&cb`1dU9{F-y-RmmQM$=>!);NGKcYD~ZXMMRO4Yxfcuecq
z9i7DjlbfbZJN`)9cL|rnqUe&R0iIt{u5((;9h}<~d}OAB*{<xzyRL?79xz&U%fa7b
zh3K7<R}l^#Te?53iwp1&|Mg;Bz6sl@WUGd)C5Kn!%?&O$e0gED#5x=4OD}m}eyeMZ
zyIb0J*WnlIErw1t;qMH$JWHIIEj@gBwDQ<rAB-2tXkaV7sF1&uvr1>3uE2^|S<5*x
zZ0--&))^-rTf%wFMQW?#zmx!3n-wy0OPr3k$ZMNVJKJ~XQzmQS+d1O9xa}Ct#J9da
z%=+pu&xr|NtOd2NTQRjn3d}iPCNWRmA)%-FZHLbH<QM%txu$ovHUF9N{;Td2lersX
z-kk8!JZxy*5V@9VHkUI~_ae8`GVwM?Z>97pUUIMGUiM>-RnuW-^~37?_H&lDbX%?d
z`S<bStoS_R)2vKiR<eG7$ejN8mcw*DD~?zT3zwXryFdRn#edG7v8`U@0pES6@~VZl
ztej7c&Z~YF`>A?>?|k^#%R(B0j7b|eERZzkxYKdv$@<U6k=)f`Pf9cs7*BlBdc?Iq
zb4o(u^rw~Do1#K9Hnu1Ky|QTC&5Z`Kic3^8QZ7ck;$h;t@saKI!9)5*?(?46+5WtI
zxSpr!^4C4ayjLc~doNmgDtjGoeh|Z-Wr-{zEL%3e)w}#=*O$ANIoTfO2?A+foc(e>
zr=QFEyLtVF3oEa^Zhy_fQTgcB!EbYyEMK}h-}*qW*yVSJuGDuGNXSHOQ|!7rM{$|X
z?91Qh&k%YMxbA3k?C#CiOd91Swl=;O`c@HZ$$jL;(z!Oq-N9^}ryE3cq!I*YtE(US
z92u#w`*U&Mtu%|o6O&u*)Hj}({NVI$M!6MJr>8TyY1>Eg?FoA2Q+Uwa=YiU$Q_`1o
z#B$BN_7xQ`I(+Ez<T9;9`LL#EtF9e>6co>QwzcOUlhQN6(5Rg$?_zt4D~sQ&{+pk)
z;m+RYe@`5pa?;@Hi88|!y~p~}0umX^isruXIB&dA{F=R4n8mi2wq-2S5-+BNO>0Sw
zmN!4?>7bgu)w;EM`qGH$rpe|DU+L_bBvjo{-6QXO{B}-;zWIdrO@^6o?oYVQVpz$~
zJn_<6wJxPFpNV01HXmxrrZk*fv^H|}vIeCiPRov;zNVxmvvN=3%xwa%R(NS}udd7N
zZE|`RdO%r9xq?~pWAcs1C7)jW2`{oRe{9*L+{tIsTzz+nd%Z?Kf3A~BRZ@T5rRa#S
zDkrvlGH%^0`2Bv+o}W`)mvUEbPzfl1vhmNu?MDJm9b#YoIe_)!3n`ll+4HMxYhv#I
zo6vAV|K_}Br-gJQccz{$`gm`P#ec=cA9b?3V_g2HaJ@UZ>7K}pvkwv{1%JILUs=$4
zFSpcI^UI<8s=pI9FW{8^YQT``=y31$B=)bC`?`B7<jn$3G0b<^&zUOl!kl~ZmT0lb
z=H+=0B76@0-e$O_;^5O)`MvuSPUN4?bZooBuv_x;;VCy59^L&gZT%|uhd(}v%IEF6
z_vOYluleo|l%~X3u$}+;Gj#3OUqX6!_D9GV{`-0N-|vp>hy<6^89#fDr5vgJ*r$50
zaMttfOktb?j*(XnN|$qL9aPf$t)<{N=fiA=`_kT%9j0!cX`-a6JiGt*GZ77*b(Qbr
zXBo+|pS~Y`Uo~**Eyg29OV?iSU8Jpdkoo6@wOK1w^*3!|jS}0yt#C@fW#)Z(?yroi
zt#qDRulr?v|5kI*m(*7r=d8q5&HVg3ImI_Zm8<^dwgWnQ#rCGgtk1sddq2(l=lSKk
zv)1j7T6aBb_iBdlhhLXJ5I<dWC*jW5*r|K69_jnKIEhx?*s#aiylnBc7PoI-Sx&gD
z|C_PYIAh(6$Bykg8s>!-8HMgWUuY4(fajl(WyUV+r6tp&vn3yPZF5*2oqqXg!@Fyj
zd;Uc3{@^@4Y1eDP&Al1hmu#`#nN~h=Ld)50^&Dzb%W4)zNIep8u1RWHYjLz9M>Jsf
ze;enFq)z*D+1dfmrOfMu{+^zHntS#?Zr>e_KN<OoF8i%?-W1}}-!Jr{{+9vczoI1h
znI`|gOP-rFvG;<4$CVF4=4$o7_M365#{7*oRerbDtVsLGkqs7-NfLp_StTkoHa(oQ
zC-1N4e@20t!tHa~8@7FBY7;bBaJNA$hlioX=TG`|&*=|s8I1(*SpNxj3*OiNyi4sx
zVOw`6n`=R`Ll19i<T)cdUUl>M8sBmuGtU*B^Nsh1e!5<LHfM{PUGeAnjpAPa%=!<A
z>nZVF&(nzb(sZ{sE%0~_OWcwBAJhHsR_^q8@bb=!(%sj0=8H##-K`a<wqLxj>Yr`G
zx<8(*KR2<RTiK^v$jOys7k0;Br+D+;8*JMx0`=DGT03#3$v&T;mawAH`QE<o=I7jP
z^<GS#mVYJf%;D12hyQN<bnNZNkcNLR-)@U$Ozm3I!OCm1j-^C>J!5XogfgDk`z$^z
za=RxKl>dL6_m$7(TK^UQ{;h^q*CUr2vkFbqo2b!UnxK~`{Xk1ELD=Yr$XpR|)tOmx
zH$PtBc+zpz<lHH%t9xE}K9t?hm667A<I1u~nW%Hq&s+TX=U9KIDa>TL;5G5Kncn8P
zKXtdyj-R<$CxdIlY?p;<dZ`UM$4WNMzHcA)Q~6M_ZQVZBn5G{hyi>oL{fti6)+ss|
z`QziXE3dpGaxyay-8K1iOtm5S_~qC4)WdV+LthrZcJW;IZKG~u>6!y(HBHOsIXE^2
z*IqG?vwo2{e^>kwb@vNy6F=?p2sAgJcGNHY&8wyzDcmxwwzo}A8*ApD+&la73dZjY
z?@ml%`Ez5<Dp5|>#~<@Do$qKyXRUkVy7p=GTj3ay%hSHshOi2JZ)y&zJMv?GGjFTf
z;vyrh`^zdXb)VmIH!iO%udI67TG{9u*M3flZgQHUxxJQKZPUx9=bBaqOH*eXUH+3D
zf4`=w+u!_jme{TTOii8wzl&X$8}Hn+q5jSx%?BS@%$GUNvkJbv>Tjlnb+1eMv9|wz
zj=J24*=uxkV|vw{;^+GFX21RGAu_jLefGy?##=VM)QLLhyRND!ey)e`k=2_w%S5M^
zYl^H$-?ipr9zX9EF8}iV>n0y94tuk_b6?=o-(q%e^<6&9{o2v%+P2btXU7CKjx`5E
zrvBcRm_13*j!}(4^;9j}@+Ge6&8Djw@0~yK+4lUNjDu$v)<!;vxxK#8IPTC6Z-KJH
zg~AQD9Q^cz?>q?LK9PBwA^qUzt2Yim2%dUZ=b!hCc7v~*mi=SOh_h(a<9@w4Z^iv6
zCyzFL=~azt8kZM!wTm|NewyQNEm!pM(Auv7kDNdHZ9aB<mpoUehoQ~7nyQ$$pCvSG
zk6G^&*bty9``+9BJ5!nYN%zUW4?O$)_)x}~oc_d%kAHoRi797oRgqmT*1C3Y=5~ci
z3oflpZd16qgZ=iO15ME(q34rK?;JW~8xrYtdCG({kJd$(#|knon0fo(ff+|Hxm;Vf
zFIN4#-mY~kTYepn&8ykC^ZSpr_D>!zyuS4EC+?QH7L_|vyA^XC7e9ZmwEKenY`NP;
z3w7_LRcU<>U(K|_JJR<_jY&iIjM-)e9{MGzokzDE@{O^)bJTa&YYo|ZZ+x!CJiFF&
zefGgjt|u+?Iut~`PtN)sws7_q|5EXzI~Ro<=Ue?-V&46;kA-x&uIapCJeK(`*zAN}
zy0+HiS6kz}FU1)B$y^ut_$-gjooCaopWWknxy<ylr0?Y^Ofmbf#e8LpzM$~uAEWz2
z?af)MGLxeu7cS#nX|>hN^_WcEvTueVOeUQXs_Hrd{1fI^uHD1PqMo?X*W>PuL+fR^
zrtz_!>%Eei%>LF*_iwdHh6R`5tj(@vE%g-(UdLDcy{CPmc<Y?DmpA*E%5}C)T)+F|
z%e%3YuEy?mWo)*-|LwR_8xOZj?HxZ!m&Y5K4#^hmPIO+b^L2gMveeMDX*)`0sRdom
zJ$&tR^?&x0nnp9<vAw<f!J<DmHsh-5qj&4pFujlvi!5i4Esy{DW<!qrdH?sTKRYw}
z@@TN0cs?oO!p5yrUaA~3ZmG&xP#G29IU{@O>7|@`OOu~Z^Xr@ZHJfLRk7v=ve|5*3
z0}PzMXaCyqu4`)ifrz!!_J4gVeb3^{Tv?%iwv9EKBA>oL&b@w0V(*LXwr=XnEB8gc
z`^^v-dvK@LrCt9PO_$uZTd{G@I_CQ?j!rbX@l`--wPKZhM2}U(LMsmu-bqFJCpI2%
z{uVz=T~#0@HZRJAeM*Y?L&1HijZ0!0)e4u#9_T8XeZMOvZQ4Pl*y-CpeC2xliu1f!
zN9n)MXQj%+e{ANT;c-f266Zy}?l6T(>q4tSn*v>3jz%lhb3A_=@jvXL%C#*fufu1B
zYE~R!n^zgvaCDV0SJN_8-%#bvj2l(frEy>0=y=eqR68f*7n2(E1D7Vb(@snuRCbuP
zWb9eKyYqf~YQVda>1!jp6irmzLge~h7p;11wED4{)M48TpPPzy{f!l8RlMdH-j-da
z={NI!d~1`*)y~d}c|{!Ena;1KU$}T$HPD~&L;sAETT~BQ9&{FSIKgr-qCsY-*GJpa
z51BY4zcJ`=3QS;SI@I?-Zx46ENe{ofISh@P>XK^i#+<!-W!CJawg0-rj@ooBss41`
z{{r97%4G#@9UM#B4gY^$f3v8*iQAg};%UaSj;Eh(7FfK6h1)Ej<=wd(2i$rVewX`I
z|6>2nr{7Jq`2WVtzw-0fJF7)^@{M?;4tIQtO*>NlS#y5%_9MqaGb4Te*vZEWe5<Ib
zn3d-Lfq&oY$MHtoZ$9{2zRY4+6ti>voYu{{K94!7VjTN}*9z2a;+nb0Grsxck_O%n
zJgfX~-QO>8zvK9{$9sCMu8Q9jST{lF{#_n6JNGFe3ws0K3e7WEKEcZBu0vYZ`{mE9
zq91nN(s^jWxv~EBZ>|2$rE4Zl{847*@Hs+4d57KX4^HCs5|8uu&7DyZRoK01g6tu|
zK!)S%FRZfSNVsxpbDXf{cix81zT3Yxb`*64Uw9IIo53^T@_St-(eA#vk-xUHANlp~
z+u}K<k@By5d;FHI_~f%?2KS!%hXWcoKgKh=-}WeYd{*X(-5hq42T%6RZg-u~-l%1M
zn~|}sn|*bl@FR(JAEHuU@^`LmzC9uPz>U-AbYII=T%G(>)Ar@;^Pk0zsfajwDHZVw
zuQ+m|_tE{mKKHpl^+cy%&bYDWb&q^_V8|5?`>!qEs+yA))Qj)C-2Hdwm0xjUFV(mV
ztlCY&CY01p$lJ7bcHzr)we7q=H`cw$OnW@bJ3*@NO@GNQ<J4Vtf=YheMp3Qjr+Jw)
zXB!whPdR=u(M|2PflgXyk72J*f612StukxY8h)Jdd^*!IbDo6^i)*JfaIwx<Q)ATl
zDJ;e&v0`UO`R=P6F74}%=5G4<SFC%7n)z+6g59!<tg4z*rE9NLUT_cFA))l-UQnXV
ztEOH(RRg9T8P%9|TQ#?`OnDJ-P0;DZ(U1wp``>Ob-4pX@;iuV8{@nI3G3AN4*gauN
z&23kY84v$9>OJ+4DPA+df@imUWXHQo-KiN>4{w$|KD<VKr`;{llpXt;vw3HDC*C)!
zJ^5}{_^X!PsZ;Y`i(T0<C$?#V-o%c~|2&dWYcmSGvqCm+X1P*2eQxRhyp{<$a#KHU
z_b-l3I(ywEMK8f1ulUouu>86^25Y99U$!xsa7Ni_SL2mGe@e<*U+L9Hlo{8oIVz&2
z^ygjU4*vuXF~QqOYFqAHNj*4uvSY{gUU~7v2hXl;Xx}bm;kW4I$+_K=PjY>fi(9<^
z*YbZ}a)-PpoZ0iZB3+__=l_%E3m6X7zWh`ZyWQr%(k*vur*G;>J78%(FDmMmR%iaL
zeKXgme?5@Ec4|eIq}$AA>TwVE7Uwvau3oFLmgP#_J?E3F_wB6z|00X+_kvd{%Vkb;
z*X=e9cRlo_d%0|t?F?CW+YefdUyq3T{c5?)7w8|SxXx~?X6g6BS!ZPPvro<a%5&LN
z_OhysaIbu@(v8S$+nC)(%Q7FnTqGcVGKGDA!u8U940jCkKX{(`yFL1T2-B*D1FzM7
zMeaSo*T)?HH}}MB{YQHbPqdNX*8cV||DN5YW1W(HmlU7=JCIrRc(=is(`{S%&5Qqi
z&yJ0Ld+hB=XZCfuyAAK`*s`OsStew<s+fWM?$Zl*N4Kl<{9n!V!CfnA<AcBFOQ(77
zGrXyGx>lmKJtI%}V{WsR(`M0nDdDZ6&!wv~XKTNTx|`gr+idhgaM{nFQGQd^=lzc{
z5Ea{3_mjEa@7}Y8+pNT{c<lEJcQu#EQ(BN5{_9Ta_2*K3h1dTTyVrR?PuhI&<G)<q
zBjPD5C$9N_HkN00^xII=!VRbMHgqR>$(22O*|&WC!;5c^MYj2Ff3!AN!n9}eww;gH
zs&DID9>e=!QKO0L+w7TBw0R09$E8MyCWc-q%`yMJn`O&se($8MPu)!3f7y6KboYb<
zrL3Ed={aq3{2<~JGhd;N%jh6m)no_lJyQ#X+;UlxuLSx%zyH$2>axc_CvEN3%Tz;d
z#x?I=v7j$w{qL(wa&%{%nzp{sX}?R3yx#7~Cw!d$oc@1kzOC%n{M|i|pBd`SUUlG-
zc-+}-=WZ8HVPIY7&Ny58Z|1ZWH=0FfxFoN$;cVKakZ?LzO5NZ_OIe}OZr?vUPTE+^
z{F!#CH|==p1MwNpHB{c!WGjj;QR|!48_oZ;Aa}6=>#CTwAGf_rGM~ZniNB3iPxy}i
zw)|<2c;9ZCqkZCz?;S%iACqgrd-VT&Se}w{d{d84+5I~KD{JdhrfSdj-X;;%&X(>R
zrX9PYUoSNHra*q+<M}BPAwjWT(kFKniLvZB-Otr=OV#$R(V}fO+#kBu|FTd_PbvK`
zoVhc};I``#)rro=y+0Itvs6!2Ka^6KnWs_`eeI2YaVt0TET%In;)?p7v@^E+b9MRs
z`?kk+b*HapKYxDmH_e)N=~c~tvx{f1W>mUdn7s5vm~L7@N@7KI-KUby*<H7S-p|Uq
zRuXEItroA;xhFMXqsoJrPwBtP)3YmzUruk;UT?bZW{lGGD78Y5F4_Jc_fA#EUXDBY
zxZ(_lT*BhAKWrb@pIMe*tk>8&rCBC)?$@9HBv@v*Za&)-wZh5hu8W^~4`+vZ$s>04
z!wQE59=>?+_yd1Zx8uE|dt>$W{#MrgaBqM1;_GGpx?fur{Vaa`3_rrM|F3}1gvy(t
zUv-qTrzdQZcG@SMVDsad%7!~!yLLD4iDuX<cjaQKsk>9`!wc~;zq*?r{n~C?-DLHI
z_bM0T49Azt+;&}XSTAa!mKWn=Q=-%EU9NfYcP@ifNkIF$j|#g_`*B+=y34xjuIq-q
zT2W`q0ybGM4d1-V$~NY+mI}v{ZxVTjOz+InX3gpU%XIx;5TntP4;B00o>q$7pFhRo
zh|Hd_xlvxrKGw2Xd_54)RCapu7Y>`-jq<Z(E;Qer`QNm0vQ=UHcJ;z6alTKNjQLo2
zWK)jpcv1F9ad%VE1)u$eilwh)xIacHbk%dR-Erx<{BW+tvH2d&tSzp)9#6WyW5)sU
z^zSb?il-blK5!_baDsljxc$#d&!rA|{+MKXdi`&oG-cUqjGR($wjbQ)f3WFw!z{Bq
zYL+Exb==v@GbNLk<b0j+HT&b|?Fj`;->sJ0FR6GPbz3jmm?d|5L**Vv$A!zIejh(4
z6ce}pdpMKQ)Kh!7CwVVe+5dFoJWK70yYV(z%|E`he(fz^%D3$8&uew;*KK}IF|M?G
zr|@alksW)Pmv8?jar5rBn#9`sYx}#yuT>~kHp^bSqPcF{oYdOfV2K<KyGV7@-{-p-
zlTT`<PEwt{^KWw1A^Uvs{7EfLFPWxmKkP6IV(0xoWw-sD=w0ioj;$*?D0+1Nk(pcc
ztdbvdwx4fVo9guaFW-#pXAfRox@Q${*`2NT=*1<uBLTZJme}z=T`qCZB<TFo{TB1t
z!^<Ae&#Sr6&AqG8_PnmY{@=@MA0O|q>)N>3wxH^++>}W-eP%vdY!r8Hx()NX^faFt
z)Bl+=Y6-<|pJLq{Z9ebK!O2hA=cRmEe!AoHPN5q<vn=O?P2)1ZCj6Xj>w(QT52&dx
zj4rw_%kj}jx5!BQ)BCM!=G2O*GF?|qzWavJy3z03os?@)YTq_%PPxy?J6+^g@GJMs
z+x2~S`JZhJ`?XQ(_eM9X@Z@c8J0JF5wVQX`^=;F?5SFN^bD3gHG_^%e2biCKxypFe
z6zQd>dtd#zuF3oFPhxYp)eAw3bw&#OkDB}PXPFxEnr7|%{^WJdv*%9^ttz{DO#Ikl
zQ=5hI(Q90THaF(!i!3WU%3PYUyy6k7;p_)#yVY3k$jzI(sOmS@y|0V?PcA%jWy)@o
zHIt%4)cAV)cYI)be$DcT_MGs5h!+v*8zU{hME?n&;(CqmpiyLYfb|E9<B8q#PdFc#
zb?@TyFT(76$;G}BQ!R{o%7rh6Rc<W2btPZq^jg=I{R!`u?Y#LQg|p%Lucd2^mruVQ
z+jGUevhw)0bFbz8T}!?CHI(U#^6p7T^CAtBcd4<4zOLLDciJ@~Ic$#lPL=oW1wlTV
z8yMEh{M$KGjPLFy<#%^Jdh7^U`LnX~qGEBHmh72n>#F2i96tq#Wh!0HQJTLsaqqq)
z>(+{O_TN8j>J^K=wfCRI+2<Vzk9NzR%iFUj`Es4yoG08j1M>sjmx%^cetG`kahR3l
z4yo^_C(X}IDze|^^h~-(cuxNmfoi5R@8frEydE^`-`|gZFTcvxY+d}k%<|A&@9SAJ
zc4<Y3M0J;(b#XR7zSm3Bg(;GAbGBl)_1ved(YEX2MCJVNhOe6~Dt)&|H0(gmqV^kN
z|4;BveO`Jqk5fnFOW5(Eqsy4SXjyBh_*Zmh-`n=MvzCR8d5^(|mM;g~EyEfnST5M~
z_Lt|S$)%nCyN;-d`@d{ZkGpV`c})lNCEmZ6=C9(eh?bVgXa1~v_v+Rh%X3dpO}4F4
z-}p&oUS0p5V<#K$t1^e`{i@;rxTmdEw*6RR)#LM>&eqHKoS4aeMD(!iE2a9ollt>c
zx^Hh<94&Q7-<H>4`RS77A0L%$eLuxBQOe%$@F9^4>($%LZ#>-ogVlU><VVH_-?vpn
z%$GkN6lz_bzae2sXyJ(fhl+bzf@W{$&N`{-VbD6~j^gv_cTRnl3!m$?*{StXN#GRD
zPajr``8e8nc0?Y#!nmiR>A`jpcYn3@+aG2;W|G@$(<h`nQ*y&HcXr7S7dI}hVEJ|L
z=qHiyv$EcVOkI$)p^R1B_O;C&!>*{lg|}6MI>f_%B*e;ErRD|Ry8CYSRq3MSrT(U0
zE_G!bKGc=R=`R0tN?~2&*|Q7dkFI|dwW4#o=#B*L?FpB3s(k)x)t_5%{E*`8l9K&9
zTQ9ggxiB+q_n~FRhCd%()}DDN`?OY5vh88PAJ)@SWUf!#^7>L+w(9=U?6XPt+uHBn
z{JO=N_t&nlMt@cJq@?__yB>b8_Wf&r{9D~k<`m|es*~KU7O?yIPtfPzbpK&d)#blp
z5z%wkrl<0jZPx0X#mHW1FSdX4MVnUhg!FaBY1Qi#H~h+);LZ^7D~?}1R`}n-<7fRh
z3*627Wi1@f@_un6;}ZE<zv~~By(oWR7f`zAEN3?lpUeZFLc2fbj|M(izvGLU-N|iw
z)phOH>NrmBsMuasVKM8u>*mac$drSA{V$F`;!n?Xm_FU$)DQkSMn`qjj&UzCel~eF
zuj{1W(SMpIZ|)aA9rHYK+BKFt5{?d&|Eanssl;)=`MNE+;F`YWx8otd6&jbuPYqd+
zF8<}ESkJb1cbzP14hHR+=^noR(OWi$JK1NC`kvb&`fWqd=Gnn#9QJR%^xyP$)ccjC
za@!Wjv)^#uR(bNU-K2A;UF%O=h_uN(byn!qfg`)#eNLWs?&<t#v3|3>yzj4`_)ac)
zi~UxO5MzymZ(d2gX}Zr@>>}eUeb4AP89#|SKQ-{HPk!wthU86dNof<`o=KVdY~zb-
zXLkSnCpz)Y6)m?|edAqR*Dm~<^^$qrr|n(`w=Ltmt>wIL)#A6GJo8G|sD~_wJe+px
z!{=~c>s;o|TAA+ppZ-Ukw>l8ZB>k&!ehc@iBHKe8-dfX?4@EW3T#>&n_wIjTgSCIl
zzQyQxs>+van)mXlLAH*0>i6T<6=Ir>oaQOlyj>?G@$aqi^ytISs&*f6v%7z5<GP7I
z^HTOnpG#9_yywxd->qOa|K&rv!O{OYr#-(O`!ai{((Wy>tGC<=H;U&L<P&?Ekft$(
z%f{q0?+l^q_Yd66VmqAcVx)0B)N0Gj)rCu)-kJ7Ua`-ev@8o%N;(*PU#gz|Krae|p
zPnstEP?Nu;@PQ(~%^NGvjnz^@Jf{-u^zHncZXY<2QnS;9<yVK~(rw3uZ%mK7{pjC{
ziD#`d9M+s!SoPXUeS7};1%-A0w^bXq3+VjjJ+N&-lDT;ZW2N|lc)rWV|C0_d&#I4T
zJ#cARc~sn%gFfqo*o!|O(POclCcFM~<@z0`Ep(WzgY@!E_e{LDw&B#us3?ns<#X!#
z*D6l?tp9o4<fBPqcc(e%h}0O))=LZCTF3i5<lL$0DKmdP7e8%KZIsx--CnG`IVs(b
zH|>1?>YAp`V5<`cmsquJ-Y|I@LxJ#T(HUkp=CvKWyukXr_3FD;W*HWyyXU5{FW(z%
ze#Iu5Npy?o7T(~hR{7oY92j>qEqk|R=N!4^x|h{_yoANpdgk6-^6fS2!kLoh#hF$<
zMoiD{{JP!p>_Ay@XUGj^C3CAd$%AuSQtx?AQvJ5q(o<-?!M9j_H7A=Dkt>cS^+taB
z!?mjXmr4I^rf>0{Y{JhISIPYPy_j=x+;Z1Pw%2ZZUA?Fxm|4_h^1LynX#1~<SuXZX
zi#Q|7ny#Fi9{$UE`!5fJkg^ZIYyPh)NbP^2U}+f}XXq_cIIS`6@s0ZW<0*?~`&3W<
zd9Kv+-EY&!@1H20l6Yfgr?FXf&0Lv38kUWnH%uA#<h_0L+<Y_tawBj4Xi>4Sjalk!
z?;pkI{W<q5QFfWUd)_v=-N7}*53e7czOU-M^!|#+#&UnooYmPnbC=oVt7(&>{bFao
z%r(}}v(l?|m&}foxzNgx!apM|P+hoSXZMYR+Yi3H?c)>Fl4E=5t!w_%HLr^w_#EuH
z8shTr>)fq;{}%jQbyj9dQX=;Z5sUxXCHvMaUToGE@XPk}L?1<=R-L@LXNqG_w0_-t
zS<&n$^YZS)%XT~{IltAWEKM_I3HQtr#i@EnoU$$@t<&V2A$z+%?R-?9Mc{JJxc|8`
zLZ$_uI$U#QPE&z~=h@x9R~k*;Joqi^^lS1AuiV$VsyC9>zy6u(J?l=ln*9OWV`W*@
zM{J*o@8Z<`_c2o^=}`Ud7yoPjZE`$vVv5jG`)|AWd%tZ@Y*{JM`SnJ{<baMT#}qG=
zWnK<C{4qnUOOEZLiMKe1?f(-6de2KHemA-B-CoR{`Pm#}Gd+(gt-xP#|F50jd{T)0
zO3kaC9;(%x9S?OU%rcuXi}~5)1y?j9nx-t&{U|q0knQ}ceaY{1jtIZ{z&XF-X32^8
zI9H2TJ&l4<n=ZB|H{ZQ~<4dgK)P>&LV;1&HJuiJ@(07SFI$2G~Huz8VDNBpBx8@lw
z^8M~NgXzs>Qw~jq5P`t+cBi}zj#TpGp3y9>ce}J+tt8NiZ~4LQ4@$Q-ht4z030<`R
z@5+mN?|kghd3n;j<Ya10+0mMq-YFtlTPIptRY~3!-yJRa=<9T8-bn3fbx+UV`}8Ds
z#Xjz<ryt(d(ag(Z@JxDtP(XJ}qScYc-er1w6efi_ofT4e?X#=cYdhPXMR9_EH>5ns
zUBB(8X2p}j2X=?_U;5g`zcv#)d}9;epM%eQdmqh{^y}=)pTRkKVx;-!#Ef{i7grtk
zvi;~`WX_qhZU4uDeLqUitdUPjR8DFBsmF2Pfzqb-{I9Z_PcNPP&l1=7|1gK=(=(!Z
zy-Yprhu-(3@+NVn70cb;aNLRen_utmLKBXeW(vUzOUn=NZvObF?d}t|>q-BwEt{42
zavE2P%T3S6d1@znySMC~xU$bLQ07$qjU~=3hozJh*F8OZjrG~tqc7G@n{mDGUH`n}
z^41?W%(V|sc~vjwVtPhO=ZKHxMvH^%m0uk6zujVY#-L-_ny_aj|LStN9xYRRX~H)5
z*pkn)j_r7VzAi;HU(aEUOI_3Y_RhMO+hQBNFCM+6I{WUeti|uk%f#6}muH=}_cD2X
zJaNz5X36HWjZweFf4o@c^(i(~(NkN+hU?x{rbq92?kv0iC?P)UlG&mkcYp0VZ<e!m
zueNk@PeYevE+6;SHp6~4>xp~VL)VE!EcCyb6Ib@?0K+B8BXXS9YEN4~Uo4O*jZfKM
zf7z#QzHxixQr!}}({|Rcx_p0F6x2lrb)NkCPI~|STOGaSr$S<X-_}=Wf4u+ggPFH_
z?GFC=|9#tk=L7#0Sx+QQnQv?FUguUfN#mcv)lFJ|HkjS9(v{jS|L)SX3oBbA)1;PX
zuk@X8NkDmOko%eC${@H*`B>zB<z>ptl%E+-FrHxir&z!k43#gQN>@l%P#1W->{R27
z4_g(~Y7C32*@~D9mszMKf#5Rz^j~!ojRhfav%uyWOKuP}<c7l2(>G7tJaMxi6#m^=
zYbk9hZMpToZW13D_US)Ycy5s=P%IEUd)F>=?WA@3&y_%MnFps#y(xoiWB>a0l+q>f
zuCFy}WZL#~IkP`{>`_zW`__D!>=n~hve(wOmG-`ty3zN>q<hW>!Frd(rK(&dHRkLU
z#Vz&!Y^3Ec2Hul-b~f>z%=6qS_ohCwyf(XeujQ@(r?p+8cUrKz?R7a`Q+&bPy|(zq
z|G-;X7aHSF?!8m3=Wp?2_Tv4Pe{QeYxlk)m#NxH;DrR5Rj4a1s-dR}}t~4)G+7hZC
z7@MKx94tNS)rFAuWp*Z_i{4gszH$lHytR(gr*V~Qu>P#Bi++1uN*Aw`@qL@I%01Zj
z)=JI5#^rH~tS*I2S~kt(>XMaezP6cKUc0{v<)7VTde-XvDxKKam#@6S!*}hPG+Vad
z(95vYms;NWriI!8u`g4ZcAdHuI&IyilBr8qYwzW8jBK(GUU0;SZGSzNPIUm=LLH}>
zEnwMxBR#H=@LjtuU7Z@YY~gC1Sdb~<k-K(XT6|_I(`grw0Z>7hszv8FHF!*!KB0WV
z{0kpGg=y^gwV^2gxq3?cwM#c|-{Ji$&9vc$aJTJ7II(|{a@moDjp}06b+5lW`nW?N
zk63jQNDP@+jz_Fc*JMtI;YL$I<66th-@E5@&Vazy8Bb3~o`^gV*#U?9=JbN#j9xH&
zd;aa|n<s9bu<QUq%~xqhWa7^`o-4eg@J!*E5*Yp}%381UB<4uWk(lH0ybkUU9=|@A
z<gfef3W)ggZ>>|Qf?eN{<JYg$CGV=p7kRNE&vZ?8%=H6hDcKv>-t$_k@#_2~Ay%t>
zvzhmp*<MTv|CW}uak*GEf|z!6-u?Z1HAP+&{8xSxynuP1D{^|8cuM<1W4!9Ns5d|V
zo_?iu;o<$Xv%H}fS00Ge50uT&at!94m31Mcd6}Gv=)$#Cov)mNC1>3Nr6ntquM1ZS
z<X-m<QJ>{?vCKDtD@1oz)y0sWvRA6Bq<x=eWVr|1&U$sx&$pAy=Bv=viOVLLYz3ty
z)67?%!GX8dPOr$Gwl?!xWmqaSEnT`Q3iUSF?@;3WrW?WCwp(Gu-+PM>K+=<acI<zV
z2x_D!#~J%=?G^7Fs((0l$<es@AU1V-{kv~;Pm2FdeRC*DU*{Ww_~n&+N3i*5T>Qz#
z)000vckGF_FF#UpBw?etSal+VxPJZVNoK?3+k%$NmW?wUd=%7bbUQfXeH7Fu2zPKE
z@qxhjb6@@{r7Im<a>nJ1%NeILpPuSG(RreC1P<fRxq;vrH!%G6{M*x<Cpk}Kj=&*i
zLeeYJE7D8SOIr7MJ=+514<ElKY@9Auow@P)_xxnn;}0J@)W~%9uV4B1Sc&0v%@6J_
z%MVChH{Bq6Gb(2?=Sor6OR-Ul4#@x9$|U<R>dn``x9`U^@_l}0kl**&%<H|^N1504
z*&jt;6y)=Le^!yt_e1QfknHEPj`wB0o=v<j^X+WmeVHG+uT-tR&2HXr`D^y!{by%M
zi>zP@i3;l8yXA&ADzRGU+odFZT@+&dCUjNa-Fr`=tJ2r~riE%2*l2PMtoCiJU*^!6
z%2QWZ8u|Ly@&lVVQiZ@JnBA!z8Uc`Uu4-l*xWodNQLnuEjl#KfgfGGhIap~}HM7g^
zG?P-GQSiTKe^PwaJecpDZVz|5df=r0Yt^^8Hz%&n5NJ4hFXBxvDzWzHz4-6!8fTo{
zPsOtv${NZVLY4e2?S1LcV<V3>H61T)Y?SEsR8LxSa#qJg7Rm0t8*fN=@7;eb6_mns
zj_Aana{|FLPEhy>oPctUKw*6zB!xMifx(YYuW;!d=S|{E0wp3ncs2J->6z8PECw;j
z_3NMKYfks>mUn;n*q}zHy`TAGp^DwUW#6PP^SzR~%J(|#@UCN}ygBaM#HzD5Zif=}
z|G(`ISfCu!y?5)4Xk=pl)>%>Ad-vBbG2<%PVcuRX9r|_6ffdtV%*f~a^=w5x-|uG|
z{zvn#1_|Bxuk8`?pr3iqGd9*$452fZ9NiZ2J@;zUGAWa*3s$oGx@N47TeRy!NZT?q
zldB6?^7+<gXt~_}Ds;7TS<seP;TcS!-OJ)kL>JGk>U`xEY&q-J#gP7GRa=6C7cu$o
zbtzr4QpHy{bCqYX@2sjzAydj;tKOTHnmwy@UMQ$Gx;k~;x{^{?WT!)YvRmidwWNGq
z7_tAk$d7ccDG|pWv^Rvu#9rI=PjiEhQ+VXAQ<vN&#aUNr$AXfR{^bkEDJSrmS7WLY
zIK@HI7*tj8#q>xA6)*F1_h-#lwm-?)U|drWSD~mQU$nG&@5_5J@4Fauz8y@GM<aB<
z9ZZ^^pQ_$s^JVd$LV?SI%}4Db-oS|XnS0ohj%`yey2Gv_04j}I*0UNu?B;yJQp9Aa
z$2_wG1Pi5^b-w-F)l{Q&tjjYz!xvO0k&&2Wp2%QHOv_$;k%hLIp66fux}j9Wj_>gC
z!XFj_^6rlwXJ_p<e7j&>;9uqX@C0y^DM$C)#Uy!DV*aKZ?%lE32l~W*XKd_;5>t-a
z-M7DG#cFkD_Tj(V-}j={bLQ=}#dplv>vS6~K}?(f{=S}^skue~>xx+q&)?T_l@HX-
zSOsc2Rb2>a+4f3x6}N9{MwWB1?5tN8uCy<6+7hcDI5$Jf^|ooFX28~rSFXW^w^mMH
zk!Lb>@k&|WuNkY{txZ+6K3wq*ai8UN>B^*Kvp`KJbzj%a?QxIRMs8ac`8*QTA_Av@
z@bE1++`DVhh*J=!!F>DsKi3~)Rxh3852r)Lnc~3Z({t}PTHfK2zcd~%awuK88kAO2
zrH~Vk?Wq{e4L&a5L<df0FjZQ=bhtZ?sQ=$}?EZ}T%J$0kCpl^wBnx6IWT)qSTUNce
z-kCw{chW|EIPw1aT?Xr<4L82Oma7hRPbhLa;%t8EKAU00o9?6gpv3Q~Q;HSRa;%j=
zO>oroq~pbZ#OFvviycy<N#c5#KXp^nw}XX$t}>Lc*zJ?k`F1raUKfS9zv)JJckR|2
z;oWbg-q&rtp||`+cz5k-kg~u3-j+s2Ejn;tZW32XOZLX~V!yLciD^go-M43p0w<`w
zmiK03Nl^d8l3n(?9RGKpi-*+;)DCxS)N+{pRjB?Clii}I3n9(Rq)fgpSjmxl)jLFd
z)~O3u+Ll>?$|ru`){Nb8i>xk&bZyIK3SN+P@k;lyq%Fbvfp;^s+=H!Wy}B6EUzX(+
z@?hok6@4aCm#kFvH3gMV{<FF+<?THZy*bBs^SRAh-k>J<B~ba4f$Da!SEe1^7xiW}
zocKPU>Az2-&}oT3t4?W!$8uC0y_&kVwiDdymQRflS+TAJlveioI3p(>_nF$P(>h;(
z5*;L&!Bn}<Pi36=W5YRJ`|7&BKRtiu{QMd9rvB`?^ONU;6V&S>iAyV6y<cQZxF~RG
z!X<&rf|ms+i&@W<%mgz*T#&%;zs%6?obugo4rgwZxtw$GQ&0mHPVwAHhs1?Xa6Vy4
zVlos4!^_i+Yxkd+*-?0L;Tm;eLvF*<(>G7pJYll{9M*t4Z_<VkczgbBaPO{I01j&`
z*)5M3HqU6D(G7;LPwzaj^Tf`MogF*-_DePN9XNiS`SqLTqvx&#{;=SYM{YK4Y2M|w
z_3*B%8`c%ZGT)xqd-R;J-J-?2zrxzzmWMXps66CeGR3^TvUrC1@k;5X<x4C2zS`$i
z;!8#5>~-B$oqd0wJ;?9-_v}OdW^g69_06}>$NxN5Ud8Hb3M!cRXLVf&+v`%ga3zm#
zZpJE5!Sw4wNXNF<s;h*3muBo%GhkgM=DRmT%gy?$(AC~$MJ8Jpuaxusnz4MD3+ue9
z&Q~76p0lDZg-l*H%jD~ll^SoadG9ejea+;x&FW3yhzAu+Tw>P#7I#;HdR=SP@>o}C
z@AYv&P7Tu)HM~E7tGESg9A`FfVowz~r8LD8oIJ=alU)AII5vL@xJXK`t2=RLGA~2o
zf$U6JnY24`qqG<bF<b2S-+HMdA$#Y$vZSfCSPOjqI9>9D<Uh*}NeH~%cS7<6=Mhk{
zd6@0!^COAL^3oMHcp}QX)qONC;!PWf_%?NYj#8Rp+Rq{}5KIz-!fE>H${?7g42Hjp
ze;1h*nH50>1VAKWP@q^43~MdLLC|pP=QXhA_n(hP_@fE8m3C)toGpezL`S|^b~G;v
zl}Oz<TP%9s=8Icy_;%lg5^HtJE+?7mLWtl0E*fhaz+2$|zpYuf6ILM2sik{?gyyH&
zV!Nke_$lg57|7?hzvnNU##HZ`I(3yE!=33r($~I@C$quysO-n`C(lou2OTk*x$*b(
zJ+?a;KD4Ig?=t_j`>(Os?&OWusKjou-+$ekXRJEo?sHXO#teru?((PP(|$ZxNK-hr
zb}^*21R75Qb>)6U2?+0-;ozsBE+7nUD$VSexo^(P->&Cedj8meiYSswq~JXfpmBkH
zbDHNo-T`WbyPg5TuTNv1#2}3bERcB=^=94CcOYV#ag9f})h@B!*&C}P-~9V~`)*9*
z{@Amk5s!DVZBvf!zPt5CES&iJd;9dQ>(^Qyo87#}^3?3Zdo0h)e!Qpm((WLe<G=3P
zWkQ<W#e2-zYd{6i&u0tr`+hy!0BU*f$mjb5>frr8d+@%@-?QLR0j?0nu7zH3(-HRU
z)hWB4RIZCceBXrR5SY@_=&H(f%k*ht;NRcdmqa!0|CG=AFX-yNBl~ykT2uNrcmZgv
zY2CUnM!eu!4_vZcohp~QP9#EY5x8>)uIFH?L~Ez9x+pO;`nf}ExS3+rOODFLZ~B(P
zG;8n6doS(_G#r(Sc+-eVEIeA*dO=~?wF`>NR{hmcbU5R_#aaB+{ZxfC2*lc4Vlr%m
z4l2DoJ#%0D6Hq4(gmuX3$l+*vmkE_2T1yv__;tTsNZS4P_=LB)JRoNOrW?-Pwp(C?
z?%#J3QEvjpsxvmW!-=O_=dWGACiD2L<2{+Dpdrg=XAAGSy^PhIm%P{V#_Yv=E$_@e
zjH9`Py`sy#<{iR5%j?1w`u64!{)Dj>>wJTT2{>TD1SClI)jv_^`lEX#=)w9GXQuxE
z_wJ4@{ig{Y9w4bd$Mlq&|A~6N+mq*K&TqMKveInQXPxis6Spf%?K1ym&XTb4p^bFG
zJYI|QY;)3?&u#u*eO6t8t^04;GDn}aW$r##pDG{YYU;1#JjiFOQ9rFwVtcDxha{Hz
zu5pfok3yOf82&0+_Fmi)R62n$NhK7t)kH*_srl%%&p9V5&7Rmu7tQ0fJl{4a-TB<+
zKrpHJ{N@A{Nw-3T5@gAv==vd&BIx>umf>$?g^`LC&`5%8)SC-Q?PAtjZY(^?r(2e>
zvHP=*zeP2;1bXGA?;J4g!FdsljXekZm-#gASG~&hXKMX#q5nl!TN|bqYJ$c+SCCkU
zd6qZ+`E%sqgZ>#mXNp-{o-cZQYo7HTrx*Ut-`)PVG3c0qlN;mnn+>0H4pe@tTR*e@
zNW#YZaVGoxj$BD9KN1pYmt+?vFk_m)>kEya!odwKB9q;x$GaEOXo&NItlE|vNXZS7
zTuxM)b)h6TNODnpepBPzX1Efhgu2)Qnp{yOdmo{sWw_)}F??^bBX~vyGI_FYT?XpB
z0BD9_x5fESEt|K4Cv4VzF=9#O{0HeGa=42AP__?zkiT%6{>SyJ))ZY$jnP`C2TFPo
zq|DXSd{Xq?`CR=1!#lQmr>HkiY^1?uUe19^vj;ZG?Z0va8jiMo?$J%Qjy{&wJf&?$
z-;6#9op06Fy?y=p!iHO~$r^?|tzYwXbHjl(EdlNSCflFF+DR<T{)SoL<vD;R{fM9P
zV>g7s<I{^L6w8=*Sc1BRAiQr*Hwez?p7Hv}YQ`5FH8r*oIj~X=o-&@;NDt2<jKq@t
zp5X$>^dG2C=>;0>0*}!n6R~sBD-Yd_h9uA`$)-&#&|&y1_qqODoTB+*`;uGJe}w;k
ztNagC?1h7eQ{M_LaGAOkT1?^}h5vZ6J=-R!j_DCI&x4$rAKYbi^Zx89*=&AS8azpe
zRL3$tzxjJ_$IFg`N&CNF<OqE6{fCiYu=icT&l|Pw2id=7_wni|d13c+c2BPR6CUtb
z1Z<+YaSnK>0+h&%;iFs<<t?*2W`ZUJ_swYq!5OVk_!K-9c(Vfv|J{Mi6@rE$Ao%fV
z%M+G;(nn-KlYt-{f6f&I&$yl`1-ILFCC(L##!`ljtojsJV*I`x+Movw*DqbI9rb4I
z(R-+b-nq>-58teXBu?bHX<sXmAIc)35BmQcnf_yY&}sKSp;JTG?GpIVwfc0_dlHh}
zUR%ra_ta0HpE!Twd}I5|N$SsUdYs!l5v6(sHIlF=K=TU<phXT4%+}2dt&hzfAX<ga
z=QhKt>7gPFE+>a+B@1ur!YcuCng{Rmk>`Fu1*SXt>;SF;^OV#SOAV^5tEl-DDdi6u
zh&#Ep`F=eEYk^6{4{!78%X=^GS7bm-i+k^PLn`XlV{7)WWckQBSMI(_lAT^j`PVtB
zKd=3av`KJtn&Eu^<zsy#@U%Be(xK~3Gd>(;DPjW6ii2mpUCw-Y>T_SeNG<8k_gYY2
zg_`4V;)d5hhBIFf{PCk|!)O~3)ER@PP;k>1(z@Nde=DT%2OR>w2$~x~okuAvocEM7
zJs(`}#=cDDI3@IFCX?zk!yj*}S01=8C(HHcW9KiS|0n!wj@)<BW&PK6N-H#$;fEe$
z>e|1{Atf!LNvZh@KdSt+{AB!Tvd?e!A95ZH=FRT;kO5$as5cFtbspGoTbyS)x4Gf-
zo4;{KN{%c!dhWYZ^Nd?B&iFKaXtCo@vJ1Mj<J{e&#bHmPpQXQT;N%SK>)$uo-h<#w
z0%*dMMzyuwzIj`2C_axoQ7Kk5uXRp3jOkj4%!5lDMN<Zm5W&>nIww64VH!jk*p5MO
z@Xg=S(8e2N;!0GfY+k{xP|yT<wC^3RknpOR4UmEmGy{Rs6V7kXV*OWkIp{(E&)d^~
zxCcK|{x>Z|`$PD@wFlx?RZVZ`T3vhOzCG(IZTxA?W#8_PEa0S8{OozVi}eHB1_nPj
zq-G)OoOH(Lkq0Wp3g+#-|C-x@2Q>F^KiH&WW$ig1l@33JXUdgxdQ9|+%iqpn{TcQ%
z(kj93k@Bg9ub;dZ?BG0tJfBYc0ljCCl^Iq{^6~tjhT(+II!|o4Ezdih+YDxQKSJig
zCA!d*K_oPwnjk!|es0V2i!9)#AxR8=gTHSFCo9n0D7a`tn@j<5aWw{0dF!l0v_EX0
zJ&ozK${)e5Umf<JgjxTIWodp0583AJp!mS<(fO$B_N;Dr=SDlGKQMm|pNW|KS!WU$
zA&N3kTtQ1UNC9{6{nb!M9?<l}{r^iHecpOxow{$slw@~l&na1T#opC-Vy)6`_qo=E
z@ztDh=O}t%`Lh^*wF+AQs+J^{G)+HU5d_l|q3|boHp?st0=L<32TylBS9k`4KZ^K@
z`W~|xvKxwn;q~do6N@K;C%RzEH`u#%G0UbwUj)&tG`{(JGI-t;vMv_1mJww>1Ux+=
z{ycJC!M89_d3SYcT-ySjgYz{5k|diqv8D3C(wzRs{$ID<p<Q6m6vq2lCav(d%Ksj{
z+H~#fdb}%RRpwoHKY`Tsh*WtFLg3O2Qm~x>6>JbDSak1w?G+6YI^PZ??f;)45M14^
zysXU9`HZ_$S!CSOzRCT^T;Hvmvsvc8)c*dd%*&LISd`59V~VdOH=5+&^*qh%>6^cU
z(UKhI1O~n&cS`P0*^01)nJxQ$t!j?QFI>R{o#(iv>d-lL^*_Z2qT$oGuB*qh*irDw
z-3&YXpVuJMt;7{_qW|1^#H=^mSkEJMq~yqzYY#gLCvWbLUVh@Z-pSQx3NKyUaqMnu
zaoFSfH6J$@9LQ-AXs?@W?*>}gk9EKWI&DVcT5QmIY)e8jui&-Vr!m`QL$vutg|hG4
zzZNp*G8`kDjo4H9PbvH<Tk%cxU+(g!%KujE7yYqVGx$Ni=T|*OuQjVS#Y2Z&Vy^D5
z>I7Sa)H~Vh1736F(HC>a*8U^^k8?BUXKo}e$(aWBOXz$%lC*y=Wc5+7b-VJj$7<>C
z<|xVTmRT=!{G{7sq4MizYu6mxe&qZv#>s+8GP`{Li9LiajT~*tK^lJp(nkEgJQz}A
z!pHSi9-SBU2ARO!lshFe#a5#^Ky0DUq5H?C+dayk%*Xl<GP`k8)uA)=|KHVGt7A$P
zAEd536?NS^1Y{GIb(7z}J+Y}~CAB%{>f_EMR-Fi%N9%YAg$I)kqt(Hz;LaSlI!3CQ
z2dgN!D?4%vK-8{0tct(+d-6(1mlfKvMcvr|-sph8CZ6J`(H)=$*#)(@r<)1dahaY8
zURwuUdih18c<BKnLFC0N;5;#X<%`Wa3PSTmt_sYU@9?hW;|@0YefiIypFEE|eh2B*
zf|D3*#v>m*)fU%PZ<PdscJHL$r7d|^@NAoA(MInNTG?|#OytFP?9MtDRP*|`j+BA0
z;+}=)zkHm|@dUKE4z!#=*bufRlcpmnayr|<xdc4U0Uz=h0YfG@>g@OaOCdvR@ENvf
z^!*7$4jBJ=8xVWUwQ;|#mB^3B!gp02I=3$QwLQSQasQ%CM&Yat*S-IM_Qg2B*RJbB
zb|%<qE>hI6_v~ZdA$+9%1J93b&~?Ayh2+wpp*l#XmI<_qMG(3V;k%vGN0zxF|BJwT
zrXIpa_25b;AdmHtDq69SDQZQ{>es1zE7jL<7EOAuwj=fOlgUqdCYV3_zIS;LQ{T}u
zr@l?|IP%+7_)aa;`@P)9iYxD&Qq9*rnfOU|Ta5bmTP==@N+v{{-P`0AC*v6|tkamU
z(~%#!dH(WE92=hMe>^Gl@5{0{Zz~V|exF<W=L$}V7malK@i4u1?aPMcIS$EYpZ=W4
zKWW>x`-+49+53<B&+o5$I)DEs-nD-=m#tfTf2Udg^XQ*%YF1mvzdCnrcKGzU<z@Hw
zOn&pn^5*Z9_mk)Pm)YI?neu${-17^azwNvEJMH=Gx%p-LZvISre)@df=I>|knOpBa
z=XK!QG+z4`#u-fqKN*zGb3Oc{LPDNDNeTq>AAI#^YTDj#Uw1({_l>)o9C?}_fA!v!
z?C$rx5R}6q_}50|BDQ|W4q7l7($P5Q;A0-}l3DOV$08=ngO6E~#B8Via_m1UZam?V
zG{=i8OlDgfd~_cKaj&?-q_(xeL-$D_cg7Vav8@d*x(@=lHLftRZEbMSed5pk;xf~>
zEe$rlw)`tDGd<hVV4?fOmpkJ!)3q%PCb|!NxHT>_9oy1ip!>v|`^BYuFPk=AlgyqA
zBeF7|ex6X-2IGPyvNG>}p72H~d9Cp^D;RO*(yHf*{f8r7_iQ}}C3v=OdG=lrq~KS<
z^_1;NCH3m&y*J9H+@ExubEZYiz1Oo8xhJm8`}EY}S{%pkzo+k28m!*=Xlvv9lAm&Q
zE1y;sKhJEt-|rKw{%P*Pl3UNW-_HKLsYv_vwoO^xKR?Zl-yEmCdh3d-r*>K{f41h*
zDXFz{qKc=TeWjDNd6SX$e4EvqHmPZAZ`vfMy?SF)<on;V=3VbeH4K}Ul&TmuEiqN_
zT6Fc-dq1+SNN1#d44oF2`Y?1_Z0g0U)sJsKzq(*ff3a0v^C_Jzr;JwXY^us$ea~+F
zu_rcv&X(Sa^tfkq^{#^Z=?Q=Cub&$J`Oq0_<@0CuWIq2n@%6DLzKJfn#`-#^E`Pq}
z(cbt!jp<8vVuHad?%De`y!7@@eZurPJpFFgcC*g8GrwB?wEva-Q*y%k#M7VlmoKYY
z{`>mDSs%{rF|J;m^HFqe;j=B*9@a1Y`zL?z$9j$*{}(VkORRgyZ_Jveex8TpzYH5s
z-$Hx0=?BhUzIalw`p4P(0);_A7Mio0B-8h4onqdz?!l5p1wrYS#m6MqWXuVX^y|_~
z7km8l#Jm%CN)lE)(%Dz{#^-JQ?AOk5uaf_6e!lAc$(_Id&apZ8<n}Bvo6zh}s@=~u
z*FX6iV{_$|-O=0X>GzI0>&Pv7sa<pRpl9OqYS-p|`4h@Iat}YO{r>Z}wZ`f>Q$qbt
zYh8{%f4`$h`cU`l=hntgn9rBk+Z)I0>Nm?9e>K*QW#hgPcTc-lqve6++C9(e8Q4{%
zEjFL|z54w6H%l$|K7D%r^%|ZBJ&r5vJN~do9IN5n`tPiKck5=p^>fzUoBx9MjPT~!
zA)EZN!qd;Kx%olt`zPCi3d=orZpkb?l@rV(@od&~zm`90&FLScetlh9GUs0pXWqRG
zox)G2&SpILWb)n0D67Tfb{zBbw-S<8EKT9Z)Rp-;{Nnsi?06#k+TSRqPDx(j6Jy%H
z|7Yy9Mf0BWe>fD_9CU`KEXl|;f0pZE9ea7HW&cj9*cH3|=lQ3W*5tOS`e#(dk@owC
zejYCWdXe>2>8yD*d`q5wS4q!cnzHglzt)Ej#e45vYw=4wv}^)z;;X|ar%qAnyzQ$Y
zr*?-?cv7><RV&Z4AM2j+loT6Yl3q4v$vqD_#}2!;)5UV1I(d_4IVx{R=1Wc6dj0Zw
zwTunVl&>yG+;;!C)Pe4shEo=Yu-Usw?VWMs+mwza{?GMzthVILao}KI@<86|gAM<6
z>G+qmy7Et+?0j8Yb5>RT!o(}Khs-|;O4SE1mcMuUKj+%p|3z6%((}^J9%H$-t~gy&
z=ah%>pJzf-&Yqq5@R>=WaNn*~8k3Jqp6s$ricR|&8~3x2<d_2<UsQ$6I*+W6>s}T-
z%b58f$HtC@9rY)hrtJH-?qKpR=?&35yYeT_Ry?smCwFm7fV`wy%(hoi65Kg!ZyAbe
z?2|a&_lxJ218CcAEZY&Ebkn<!JLh!Hd75(a=+@W$r`y)wuzU5)+U)+upM7V`p_?P+
zOO>B1AP<MGV7FwqWH%Hy%+%f>D8aLAUiqgg&efM=5_C5R&FDFky}_z~YvoQQ{p$0d
zQ$yvlKSfMDZe<hpa3kA!(TX+ulfTMlzmIS@{>f78{KgVJ@iR-8OztlXy60J`F8_GR
zj>&WR>kps2XD<I3R^Y!)+aAu@AQr)AE)*frsl8|Qod0`!*94qelKyDL{3na^P8Qyr
zWWHKe-2L>|y}GqGr^!Zt^NBa&+{Jw3)3V={m;dfrzep`#Qy{nOMTlXrr0X_O7IW+V
zH#VC-pE$Sv*8@(MPlfK63X?6AKA!N<dgr0`ZoBh`tGdRI)?ap7)#AeSa?*c$pL%od
zvw5?Q%oaX%cK?&fpM^rbZ<fq*6gII;Oqt+NdU3;!jDTnp4PB`aF<!22wo+GS^Mrs<
z7x!iRj$OGv<CoOG(8C+Ec0|6t5cBNJIgwir3x3<r@BAsx^l!h$gZaPy&c2uCX8d36
z*xz)4c>Ut}`{%uv`8EIa|Khi=H_e-N`K#x@^<f*o|9@0`Z0(Id^`CAO|GlpDf8XPo
z|84)CJN)1O-LLt7XWtGMeEa{AcD3X8-}@h4{Qv&;G@bANSKHrQ{dRh5#a0!=f6wRt
zRr&mGsng^CSLLg}#Vh>S|KrKrhL8Ub9RB*hoGH%z$=rbZH%(r9|2zL({SQCm|9+;p
z^SaqN|L)f|m0!yITrV5&&;Eh+4|%44?G69Cxem-d=U|(-reyBrem{My*pDSg3ZH$v
zTzKN+rjG{-4}9GJ@loNU&8y{X8X5R({#n1Pp1z55i-LXRIwyt;D_5VqrwodK7vJjd
zh5xr_`SV_2g5FpA^pgEoQ#*~{`0RVLrs(Fov)fl*y>Z3x!AkiXv-oAstna=noP6%B
zyq%}!-zKH?!E=vX(vE%Bz5LIUcLFuJb??=+%cAPspNOCTvq;?0-m`A+<`3!VN6)66
zKI(h%#4?4H^gQ#%iOyW#RFt1H?y~TlIJuqK=ryCjl$dME<-JvhZ%+R0-#Kglo9jmT
zOZD#+<r>UvoVmL1Ud0bz;gXE?yZ4oEd30q;;|Y_iswLMh@0&SuCuhXHnVg?zUoHQs
z{xp4G_l(f<Q%>%j%j*B$dq$-DU#Is!_0)LZo7$Y59lK}wufX-~Uv}In-u!rW;jJtE
zIYIHW4^7_w_|*I=-@0;lM>V0iJ%1gJ)F$+-6OHq<sgD!8ZSMEzc&2EsXx8_AhrVy0
z`IEc0OfJXGFYjOJ<c_Cj9iH%WO}zE^;*;mHJ3VwhiT6J7?|x#-TX)Szc3VxVa{Zq3
zHh<dQ+N<d5|0%rKw>|N=VExh<mVYOO*3Z;jKUe#JxoiBDCHsF)$WfR2q*N<;b<cyC
zHpjbXkKBuuxVOKd$zYqhq`{Inu7hzGp5)rUm!ETUp;~iRa{b0*lRk3J&zhHca?bC*
zO`Ho)r*B$%qwVXTf)dNB?)jT!-<colzt6c>+F8Lq_`X`#35h5F`DNGH2HZaOAo0gO
zSLfM_7W&RSzwAtUu=SR;{I??CmQVWh??7?g0!w>?>gOFwxv3TNx31gSc(8Hx7l+uY
zIJT$dZ#Lw|)Wtn-s984maOU%=vy<OF+xtG^gj)Ivap%;Ojf}DO_W$-x{OY!~|IG=W
zD6=HSxqXFoIV{bZ&vS3cq#x<uU!Ia0^mfMN{>u8>r}#zEo~*Bn_)>D|&i-WUa2L08
z@sWZP_UHS(Qs?Bo_vG%`zwAZD`e{oJ{xOkTw%VuDP}bLhQBD0?@&<1?sR;}RmOq_l
ztzN9EF8WZ#@q=g5fAOnn+iku*;9PLhe}XHYM^m$nmqTB{UH$is+zU7QEZ(^9Q)}YA
zGQ(TqXWULZ?q9Afk@q02@13qg^?moS2^&+GTZAV4Pbi<4QNMK3f=jy7Kkhjj9XDI?
z$Ad52t!Wnfn$1dIJmEO8ZKrBHd%|Cp#WAPviUm*Bm;Sehf3mkz+|s(a+d?)>JXLmK
z&*3$vL;k2qnX&fWG}yR4-tNiVy13k$#7}b_{`l$Cs<Z4DF1x=||Cygc+@m|WYoom!
zFFS{dcS!$Y<$V4A&Z&=~Rtj6^N_B+ooA5cYxb-j_OS?`<+Ki7JMZfZYif>!NDX*0H
z#);{)`tk$D40HG!ihptCGlo7ZjrtV3ea-yLooN=^*ZkO$Gik-@r-znIGnsAa^!a~O
zT=bnY&(6=?H2ZsCgh_=_W1eI|-=xXj`_G;GSiR}rn<qO?rUWTft}HSBU%#kf@#6l!
z4-I^S9)D{)J<&lmT5sN+gP)!~lQJ)_3n&&pyCCuHm6YZ+Pu|tJF8w|+>hJk8ac<9A
zcfEe5UA<NO&bpgR+_qo2ZT@J-%k(9W_RTjhKd@)zspUDczif^?Z)=#R*<tr>d&hzQ
z5w7v;euZxpUw7!uIi@*t|9dXZ<9aR^=JC{~A#eY)A1CFNmNy<wn>9W2fsv-nx7Swl
zo?Poz-}H8e=8C%ZE5F|xCBNv|8ghN|vZtZP=eWdN_^|E0QG>W@EpOnq_Fm(gmy(~P
zH|y>R)Uw_A<>c;rmlsEEyHzvyWz3_iv5!o<7;-Q8c^tnluRgslvh(0F#r3(1maej&
zU;pFZiuOwPbLpE4es8i?uRpl%)bvj4{eCw;zph_j_Fks{zg_;csb`-o*gt2!*wy!6
zCg0~%*I)BzPG#b=n$1B+JO#D$@0RTN=_<amGW*RZwY+CcL8@iHF8%bl)73L4q^xRz
zH2cDfGRqbnRxvHwC;76{ZP)CHpM4)F=O|4nwKU4QS;-w{)yz04s)|3dx_Z;Emnlnc
zFHHLWWKYeeg6}DrZ%#JdoV>J3-8;qq+RnYQRdF18PkKytnp&UVqxq@$*_y9>`zL=5
zRDZqP@82Qy;MwZj&HmSX=dR&DmoIT9>;3#wUC$2xdNSFCS3Umv$*D`!uh;CGeEVFm
z_0-&__ZXiSip<eD{P%Wpx#iBYF}H4haJaYdi`nEf-oBg}uRffyU$1DL^S0`S*0&97
zU!9Eq^K4F8aq+_2fAqaO^k(0UQMfYuw#D5$-4CCI@tAK<-?4MSGqt&Oah1%Axg-Cz
zme|`a?A$(obIOCiJzwhV-YZ{ix^jKdoi(5LT={;l<fnd*m$=eW>%!XmH@_Z?FM3q_
zEdT%FljVL*Cuc_0uas1NwR_FdirJmYrkBs3auLuwmTGje^Y>1fqHXMF7cHzmonBkN
z&?c*7P7S-?9cjI|u&|h<6Q`B>W^a6Ez1IJc=$;MJ>U{!(p6T?nZ|_VtyZpQI-=o9R
zm(4o){{d&5%)wsH8+WR9mRZaUt_!<kB71o5+oVhTcQoWCDMTGNdgAik;-Ot<T*cuz
zZXWj*G-e9cPkUIHcJ8+m<7MGMSy7W&vZ5FE-h0LA<#731i|nDtT{qj}FVu9E1n)kV
zcDy|!>gBYn({4JbT&mOE7-yGvx@rEGgl5hhCk``T&7~g`GF`T<&&+u9t9yP|`bSUp
z?T?=<J8X6H`KtqL)8em5THidEwfkGXpFhjnG=u%hhVn{VVjfrB`Byxr+w<mvNpCgP
z?<CAVtI6;FbLqil^Kb6oIRDsl-5Do^^Y6S~#+^T#U)8&Gvh=KPH-e7-I9aaU_-X6D
z__A|qC)JLho_6Wd*|qbRS?v`*bbFuv>)3@GGiHc3tN495-P`=RzuAAspMBkD8P2h-
zNHdp8klFKj&V;{<%`g5r7JM_~c$)c}v)k_6WRY#Rwc@p93p~77BBa_*M%Rih{c!az
zvHwd-Z6l**1vB{We)2@xv6|h`%I%9~w8ZY^H`&BzNgC~0`gQ%+XHLiXwe|NqUQIOa
zpM352gRlKOiwfBI-n~$is-Epvai4cJ=aJcl)xS6YUvcr_^OHB4*>3;tdz#b6$dPV%
zl~?)SzQ@ikbJa6Be5C?CZf*P1w<2Q}(_?16$#*MfB&?r5bDC}a_U3nTTv3}(W<1T~
zv+}+5sfI%<oB!e+-=4S2+Kcwj_|ROs@5Y<SZoek$x6eI#KP`C5e}>uj4o~EDylDL9
zL|P;N-|c@lCodOzbalt%Z6<fWZTY-@as}f>J?XQt-{$YW;u>GJaCeT~cQzM=Dr?>U
zhxYn7-*1>3-|=IU<e8%%ckgI?sgctVawq6O|GbaFu0pojD^^a)kd}Yw%J%c@rc4u-
z*h0&yeMeoUn!Z}URsLuDBmO0c?VcZvmc}{tU$;K6_JvGs+*b{zQ=AqvTVLkSt$)`h
zb8?p9O=hL<dB0eqEw--8^R3%7dryk_&l7oRt#26al?&h5|MJP*mMFcz^Qoy1Q*|G*
z3i*{cd3>DfcK_Ld&d%bATQz*br>nM#7Z~h+o7mmOZlCh-xJ;Em#Xjd(OhR!CSyqph
z$Q*rX|FP%DhPf}cy=J(3cyZpgjaAd?6@RSR@z!#O>E*eL9n*y~-nJzDkLr5+<A2-#
z$F*ywcpUh)*GJ!E!^6uaa~57NJY9A}N^twG+^*@V3cn08FCKinGgmaCNjfg<)-w@<
zJsb7+Ywp>|QXx}1|M>a$oH0KXwZ4}`FaFjSd~Jii?DEyOxrAjNGSn@yxwiAw)2*5F
zXML=Fl=9Mg?&DsoP-*#9+Y?v&mzErAVVRY1VENr&Y5N2B?BNvtQgkYhbCta#<KBN;
zmLG8YzvSW1cX<!j{!^K+vhH4VN$ez7>ypoE`@<h>XYDw0WVZ0+{~w}V{a#j7#+(S1
z%D=ts?Uy}#FAx1u=XdN|bh5LikvHzr_E@Gl?;B2C+h|xSe=wx~o$Bkf{~8bevTnXL
zFK}0ZOgzh@ukzv-FIwkc=&Ij$uKfM}|NgzN*Y#ZXD+}*sh<<SFg^!%X+fVr?!rr95
zu3hI8U-r3W#p#Q&cdz;d&d`3v#PykP-NzRWe70u}+C9CQcRToc?)fjj^cWAA|6I}>
zdw6HygS4p*cOPhPD6VPBm6_--(&<>@G@0?GRNdOVtiDYX-6a2;F1+#JwMzN(2=;Gr
zvLUzTq%9IyH9PN6_93xpIh-;NS@hOFJo{|^KQU$AKR5lCB}PRabG!ATwop3NWb5yJ
zi4PWkzq`bp^URl57mpo0!KSBW_%_>pn;Xj=we`*OC#LUA*JgVzzF}o!*4{E@YrBBp
zM|<|Ymph?+|K6g5wXq8Kek|7hdv@-;7tj8mELs+>wWnmIR;|>BOIMS9k4VSNW(ihy
zoyxZ$bDrZdZI5eeS9nwYI!ykl^62xwZS4{&iTQJmR=+*rTC<ByZ|Z%&2Q!i^b{6Vy
z_xN?}xvfd}KZU6G{2|@i65H5$ZFgRqc-WPJzwgRd@5XsjKP>B>FbW90w^q9!e_r6h
zp2_E5CA^-wZ~u}T_oV*E>}0H&H$i-Vrrg8uO<S0^{GEBggL`@5LnpmIr&{B8esI{I
za-3Dey7K3r)cEHUSc<GH>P6D7yQZ2iwD{v}l*w&g_|W%}pp$eH_xA^!eXL?8CpJIS
zF`ILgEoaIB(VuH)d(X|g_Hoa=dmjGzOZ3?-1YU~do!i2<Yf^Y#S<TLec}9D)j5mMG
zxLCmcFQ&d=ZxQo_(?<StG#_Ye`^47m760|=hMm`gIu3Uq%8Oeem32_#nhVRD*qo2w
zqEe*9P6(Inf2m;dmvP!vySXAs>ZSKCA3nD^M#duL*{$a6bu(YwyR6=Kb@^-0hvF8?
z_q}D_aWyt|V(^RG?|%MWTz1~&;|I4`v%s}+cJp^c<a50#wG&xTeRcQ!oLMHhE@f+8
z%YCo9<UalF!*-Shx0nKFAG(}g{rH-wlI*Vsx4XlaHqY|=pc-E7x#wO~>`jZA?~TIm
z?>W2t@lC&TdPj8feC<C!QaI)Rk3&DMe#055B@tC@?!onvj1Mp7RfW5+_xjS5)t2Ws
z!`F4QZBy3mvWs<2?JvKdU)8a!`DxJflGmHA3JS~rSQ39D!T0#VLk~~cl+CWnxy(N2
zJ|C0#gTnbv(H667C%;jeyJ2ndwnNuGzqj^%p7U~T=>waXR_7TPSe&I9#r)=IJbwC1
zeeZ+sEOs#+Y;6I?e2wDE`R*TK{L^4HA-R64nI^ydyw=5RXEuHOX_j$Xdj9i;2OiDb
z=FLCxO9tO18?lLJTq4^WR_vTKx$K44tj(wY&;I%IiF)RB{+rbeB4Vb##Yyi4>Tkck
z{vx)oNa^GY#&1hz|1Xy8wEdjd{4?ZxUd!d;eOHd>lvV6mSbjUXd;P3F(<xCe8Ow_6
zl)qL!OE;IDVRXOjfmvJl`q-6!9xTy398$GC$lw36Ug_4`UjvKG|7*nk{uNQ9d|iCz
z+xCUu|8eeov5()sMBtdNUjA*y@}JwL$C#be{QTLNW5P0zKTNNDOL?59Z?{d@yE66N
z{M$D^eLcu7w=MR5Ma%K)k8`bU`M-IecvV;L+4z&=UEJd5+#Gu%^>;b*S9Se-_I;V~
zWDm`_kH<Xk{hoedX2PPAQ!^F*pG^Oi%k*=XV&`K04`2Rly#K9sr?9>?v(W$Az5@pH
z%ASXxWPPtQh4rKCanlda7g}UKZ>gSi>e-(CFMfY~;ydS@vF`3v-s6+~&g{>t3_gGU
z!T*DE*-y?t7QVyT`)%v4LLoEL%FcT437a%;R^Pk2L~=!7;HI1}2UapN`@IvJ8&E!_
z;L=H^zR3~~JlFqRl3u>a?vRWDgXgpvJ~FvoeAWL~Kj>PcQ&}17x32tL>5(0e=Y|xp
zceSpOE7)Q0y7Yq_&xu0DJp1Y1J3crZnrrh(VP)T(z*If`pc$oVFOB0f5@-E0x%ZXz
zd9CrLD{JcxWcmL(aA^CbchAy`Zu0JVWb;3>GG1)c(@QsJAAY)2`SfnLdsjdGS$g>M
z>fo0zC#Sfo2I#Ke6=?fA>XwaBb+)dQ=?u<;^S*5Mx4L?Kiq5*PJ_o*iu6kE#RyQ@<
zcJeLPZ}<1FExUSJdQ;pJ;g{vJFWfz{VD@5HJMraKV!WnvXKJZ2&s7x`=l}fd!~UPz
z@ivoAU%E9rJLz*@+wZ@9%W_VfYuuD}sq*1&eXq(T+xPFB>h$*FZzJ=#>x+W}W}Uly
z_EyJrWxJgxFPv(xt}TD1EOc~#WO<=xjNkRwZw0=bJMGv$Ws4x=kDVVkpIp3+>C7yK
zNaee?cJ}`GadOeCGujXG^Yeb{=yx@z&Dz|&s4u^A<$ujzjBl%$=0D*4bRguXU1PC?
zjYEi&iiRn(0Z;9@2ZD>ZE{gezzp?3v^@xzE6?@dYKJG;J^?7sVB<Z@}X1(##^ZEOm
z4U0TqPDtrxXk<Qe$)@E0(#ew6^Os%jaoppyYma42_MS|I_)Xg9MHkfmUcTY=;XU6A
z|F6yKsuhu*H@)Mtv*X9Vlf=z;os!<kSidd(!Q5PXx!}k@2WP%{^7#K&VU|fcg)ijW
z&ZThhM0sB=I&5LRWV6-ABf59DY<DSsVfTGkR(9mI<1KfNe%@PgXnN7vX5+gyM|R&m
z5c5+1b-elBcVV)9&-?klU48iN^sLNJ{>Ku04R*-Rt(m*4Y=!Dxkr_^kSM^GlT|e=m
zzpeN~wEwjS<^3P#T;ATg_gVaBzYJcf4LKfj{AzOlK5yP#^Kp@`*UuLr@pW>Of6rvg
ziT4itX#Z8yy2Rn8`QM&A-u&&~^ld*cU9R4yo$tUmVcyS)b|tgf4{vc6y|B4s#wIo?
znJ<m!?LM5fpSzh?eQ$C0_Y=Rkmp*(}9`pOWiGOOYlg!<!%Oc{N`+u($kN4ZPZ~Ho{
zdHXd!N!@#8t8nz%``QzCOHcnUxOcw1a`L)bg+~QtVF@2={(JrP(fG1>4fmxB@55ar
zmgs&t&unh<<+tl({zOLsZJ9s+okKQcZfaQ`Q<nDMe9eZ1$JJ71+%eiN+pD*uv{2-Z
z)SX@5?wecXrde$0<##ejsyw`F-uGWdIVaLh{MsuY{ZlVk(KYdDz>0?+wcmd)?QUD#
zZ#jL>^CvT;{~SAc<=Dhb$%5@yel&-22?=jGJ7Y=IdG)M&4RTBg6C##WJ~BLU=ZH+h
zrxwwpJN#8WpE=%OpIwkzQT62FmhMz7&$UV_57#Z~jdh(^t5I`0-&bkwv%d7RvS(*m
zFElN;%Fda1>@TN$q?Fm^>J!HVmR+16c;eJ-mcRZ>_nhP`D7HH<GQ0Bs6j#RpyHhPe
zuVZITWAxp$CeX}9kSEbox$i*m9m_g#!!ySmPcx)szmdFuM913x{1cJdqNyzVWmMNl
zt}E=?cYf{q$GRJ?9AWj7On5T)&#m^G?-NWnGd_rClU(!Q#8VB)#5E;FVhc~q-5RGd
zcV*2rmXCaWR~n_X=H8mj!=s)#)z59d`p#Uw-O)j_tC#&h>gLupr_wUa^3LR<ZzWp(
zmu%-g$+Oygp;F76$Iwjd;aZln6^~C^U5K%F_w@{$c_`<<+54&IrO!6}k=yZ}zh{Q!
zD!)Zd|Lb;4Wjro-Ok6fz&HP8yoeBHjpMBB#-)z3)bhkT+2T$BJX;GW5(IRyzEwb88
z{K4<Y%vlRt{`30p_ii_OmfpytQ6DES()V0)`cGS(_or9i)Lm_J>F(YJf&Wro&Kly1
zrpLH8AA7XT|Cn3K^Gyc}?q5;wve)=!@jIMhzVcBMrgZgd^MA)ns7hy<h+n_ve0*n}
ztGc%O{T_cCrKrd!5<2Hp1Wr9Qsr1-V{IW1d>Z!^-r~lht+?e3Z^hnmPl0RnZ5@V@}
z;;w5%=GLsZdBS~HOTmt5rmC@*%%^|hUH5sf%6a_-CvOWFKP@`z{gXfL%h9%{b1(PJ
ziEZ!R>-FNZ-0#cD#kyj$oZB|7dcN#`v9FC_c+7^=)%)Ed+Z8?sXJ>vq&cE>bT&<FM
zi_MOTwn*}t);S+CKWDz!Y7$41xBun`Oz%#J_*#75Ic=TI)i&M#c@n1t=cmNR{QuYG
zz02{knO~F_htujb3%7|K7rrD*{W=+YX0L(vs%W+tx!Ie&1pZ&lJW-~hFePKnjA^R<
zwMNR0UOCNkj<MyObDcVEYstlvTET`d9lv(2Hkz0H>S@!*^ka+to7FB$-{sqUGs#OW
z@z&mZwmnD7O3hE)R$cac?>zf?{r{Kwulw=0*!N}3B<_Cwf3uQzHO%2Z8Pa$7x%t&g
z?ss3v*YO&;+FWX`{^I$$Cb(he{JgE)zrNZz8J(R}z3|kI;A|l_hKD~Yot>Tc<t|CT
z_vCd}$h0pP&+!OdF3md`dOgmol}9GMLcM*ij9*^d_uFeZzct4&XQX_ekyOc}!pkDv
ze)iSNcZKH?T-v_8{-!K&<!ttGzL0}|b$M!Zl|se3FBCj%ezJJ-zoNAXy6@h;e3l}x
zqyI)j=j7uvb_=Op{=D%^#LR4?9a1N!&718nI4yOKk<e-nE1$j{svQpmzWnpKR--I9
zWtsoH>`&F-J=fncw9zj*pZ>9!J@K~6v}l>u?YAb%C2mWb*=@Kid)<Lo{#&2qc?iE<
z?>QycCcV~Q!c%ZFZ<1+vM5;x)fu8U^)znWlx}Rq*@lyNjewBgMG05(wz@qxc%0_Vx
zS7JWgcv?U6tn@#J!wD+=yc=fO?4J2q!0E)b3EK6|7kk!9rFKvFE|g|_<8yXMspr0w
zWmOh4`O=@-sM(ypnGx}MqWHHjOy+62mp<{w)&Dnh6tC}}pr*dZpy!&?3!Mh$Llr$2
zvIUB3Q&{=Vv8C7g>g_dNcIMNsFaPg7n;iJWdY5+0ZP9r%or<<P?9;n`W3$NO`bf0`
z`C>1d>Q`AaGCajXh3t<d9m|!eWfNy#7_P3Aeedu|#~kxDJ7<;gG@H%J-Q@Xjp6qY?
zkMoZj=QaP`{`IHHP1Az~ccORgOfM<U&ve=~H_E58A$v~tK@Z`8M>qZit^VDot0%7U
zr20|W>3xUFBtA|2%j|NhC{i-G)GCf`66k=?Df8z=Uo+P|I%mrFneR^DIQ!+&62<N*
zAGprX<F!pooBQMB<!iD!0dDzhg*sk!DhpnixRpij{%ZGbHPeOZDRadPF5VVsyV^D5
z;K`kw57Yif3Cu|6e6j6y*p5=4S>Bglc<@~BcxU8lY8<g-m%{nNP6yr7N0zxpvz%*}
zx7Igu_L*0Gh3VNHL+#`(IV$H8o-aP5z?)?LZ@=*Irw7%Y_k6k5f6navFC+2gR%_Sq
z|K$3)<FW^%DMzT(=^64r{zN}MF?F}XZpr1WOaDkZ&E?sW)|NQ?LUvlm)=mGi0xP!~
z^(kGyyynsM*?XTf>P@_MHd~b6c1f$)*=5(VHU{-?@z-OC`l+4r#A#}6*o*aN)}{JA
z5Mqt4JpZn6`rSzV-MiD+UNp|xzsY6)-IeiSD}Jx5sG4h?vMDEjx6AAmO+9|*Av@=O
zT7A&7NqAaOux|h4ij@m@hqyeOcwOP0y38q;Td6bs@|0h=N-bLLeC5LHX*QMGI+YyG
zdXA?}&kC?<3bJX-MQh(*v*G`QBQMx^9CUrID=4kG`#OHrt=;b?Jw5EN#<#O@M#(Od
z$r=rh3bU)^3RfIgdS$+5hNWUjv3JGog_C8@tm~KTd$TMe*mHi~p62=TK~V{g_x{_O
z9Ncc>Wo+^5gV{1emjH*UzYjlqb-XRvVt1~q$#Nwr8+UQ-v;z}vKfD^K{8XWm>G!_9
zzKimwP1yVPTh{-76<>@^=S1HqDEawP=&Ls0=9p<yivEa~NFF!-GP9%hwbthes}Gjz
z`ISwI?vCDl{%Xw>fA<f|UVnY|Yu1lz)n``QnZJ9bepqGC{aX&XB3lc8J*vAZeYl`d
z-+9`_-oz7Wk|i?^&1%uRw(^Pm)&-9ugE}`<%T<f*Ija&ibIzPQSt~B}Mzj^p+ut`k
zU0v3*TlVd=>4Nz=$CGX^d%rDnZ?0XgwEE1|C!?1A=-+z&?R7JCmyq3C(`K)n|3<c=
zO<Qq=o3iKE%p3IucWiEMw$K(oTXKi#R@&*G{_Cyux%Y3|HvK`0@G2SUBpLBXPZw0Z
zS6o&2^3lbK^H>bm$R6PO63OwoeFfLzJL(&o4pxi)`#R@=XZ*w+yN;jR@4EZ)6_;C2
zdswZdmn`ddR(d(%o2Sm>-blWOp8o`=x3`C_j=oULxlXTskGHO#4*y-(UbTk%-<CYT
zn9F8(<j1za+x2F3bMF)#>v$y9IOFc_!&CShIZTuFe%I|z-?uV-hrPxb1;2*@3d?85
zykhf{H2QB8_*c8o|A)%6r50IbhD^IyRF<f<ua0l4E0QaZc3RMq+_pOD(~Q?PKR)rt
zJgGKb^0DTa?74NyJNY-s&E9j#J^Z~;pN^21_y7LM(+@l?$h};nHu3zD2f0PB9bcJR
zNxErA#(y)|=)TtVr0}-#iX8V&-W5I78UbgwX+Mw>pL=SDth4Fkg?>`q4;IZ-5OS({
zyktv2VrN>1=D$ZvpMA=l$gw?HwN8nbEkcrkXX1picON~@nAm!<nDqAbl*d%mcBbul
z@o4HNofDgUlU95c<x+HrjyJe<zpGfP>H675U%M><c}jBcAMo$j(mB%c`m9UW*%hY8
zySaDS-wX_CZcN#9CO|X(=9Mttkd6si8V6#c+1}j@J5$=bHRta!y-8g6{wqFDd%M7a
zHE~u)%(j-4js^d;tks2n{&Tp^<UE=4<*AfvR+f2D*NYFxt`tgnF*{=S?6w;@!n0+o
z3v&If;=KQNoZ>i8WT!TzX13$LUdb<8%a@;iJwx<==rME6$Fn6PWs;t3S6qK>cc$5-
z_?<m7e^}0T=Jzw}+roAI!Mmz!|5zfIoOw`m>g+8)rtg2IAFZ5u*viuX#iqG;wuLSE
zdBD3oC9WzwaPAX{Gp_?y#!TC5Gv%Z0k=P|iZ*BDGf1g?KEM?1QfvpdOQ@{GO`B(p4
z<$SO6YvRn$&V1iP9AyljsjBUb?48*B`R|!8JceA-NxPTLXt{4)W}N!7?u~Q%o4poO
z%h!n*3QoD`?;W&n&&Q)a|Gzygd|JpR-z}kHn31I{Cv&deSE+;FE_K;*&Tu|n+e?b<
z531!Pmdkmc+O*~8#f2wXx(&~)Yw1?()DF;iTxEKEs%pivSmu?Bm7h#m*y%9kV1sJ(
z{Aj<`m5W);1n1qoe?RV^0=MV61EF)KFFIfCq3hH$GhJAR?bd3;fSD`5ckkTQt8Zj=
zGh%9v<@rw%-!#vEu6n-2XjbbZ-_4ihyjRY+vU%?B-=|x*9G6<#^6|@N9^P4juB$7O
z%pWsYhwVu+VUBd)+-7TC(fp8wZw({!tPsY?)1k%|f2wQhPj5&p%Ux%>>}~zWhSo{>
z@4c^VkoPaH*pV%yE#uVQ*&M-LQp|Z=<<!h$?hWeK<ejxbQu1`KC#S}2=#b^JyT4Ug
zktN0`I^O1G@T{X9ksROuMxW+pnYgwiVY<$;(~6h1xi4{^{J38#&M#Gq=b*8e!rVl)
zX@?GEcLa&{@P2Du78Y54Gle^2*%Fte&+BiT?np8#^wLjBJAA+(`}m>Hm$#jp?eR0Z
zVoz@6i36;;LSIUgrukVNO)~Gg%pmq4S0zI+%=5{<-|2hbzYv;nXH{qR5$!cw<n<!>
zUJ35LP-EVBEA*z_-{3Uex|t3yWli3S&(FDfCu76vqdONpt&8;E+Y%`*9^+Blx3Ht}
zt$|6VXplta2dTd&dCq4pfByY_?lmjfbJ`7-+QRI+c3tx@GuxPdICrtn{;PL>=3Vn-
z)oWa^g=yx@<=36AJ$dvrZ1Q*cWwyby7eDTAW)F(Gr|{^MgrZjMq~c%CQ#OD8eDeGk
z`{HP`LxO7mC(iHo>$T?R6gV_BwPF)b^t6L~PX%QrTPUub!@Xy<Gv@+k)}q^`4vQF?
zBiklzUzJk5Ywhu)_6gUPMjDoXjqgphTVrl+VQL;@lHB?v;(3q6<;bwBwPCfne|G&<
z;xqkvckMgF6v1<6PCZ(&(&j_Nj)Ms=q;^T@1pZ8``LXTr6u)2FA9~H2a8WtmAmwG7
zmaC=eM2pn2`M=llTCHGlxUaT+r+9^86RU&D$$MM+LIU)5#7pcuytaKYFVn8h>L97;
z)aSM_QxdagR#~~2r4`Og_7>}SDC^W|n0PYj=-2&eLQi!bEe!v7f0?js?76(E0}Hlh
zJ^cM-neUb@g4;Q2?Jhg(b^UxkN9Ksw5ercf$yGK-oHj)I?}>;?o}uu}^X{p~U4<Qu
z_c*o9ZFB$d2ZWvu(9`D0tYPXsaxqNcR*Hp{;_q{1Qp{%qj+uVBxkYVu(T1}d`m9Yq
zD6(z3_ax2Ofc@^#;06XswLSG^$+9A+*ITi6%?*vp|5NoLKgMgSdv~A8-iQ}d-duJt
zoO%0fgGLIE#8$zp(n}rw??|jq;dgObyl2C`er^WF*;Cd`baQ-vFu^wT-j*F+466Mi
zPl9rjoja1*u6m`+6nd1<W#Oaoe1qA)4SsWKC-|?cu$kFuZ^6CproEb`9QS#K&3fLF
zn-g<|<AUGszshiO;%qi0@xW(_E`he(tie)~kEOc)OikNyM&rYWG+%}8xe4d;F1X$6
znR22sVm7C)h-=Nv^IUT8N|zR2W@^k(S~)qj+IsQg>(h7deD|Ged79>_vpGCHx6S{r
z^XB+D>(jJ}!5;U#D~dVt`wY)-GQBf#pAh?tjnlp!l&(DR<nxDH-IrUK&PJ_bvyQqp
z>91bK(&=pJQbN8>;^(^OrOZisagis6Lu=X3Nk5m)$Yn~j4fdO-$?5PZ)vnB9TiK1n
z$;=C%3dgM1UsrE-@$|A++Rm01)zWMH&)Tjwc6fN<$5Y`uTTgiL&f0ThpVfyruBQ*?
zCr*w!HZ^bJ&-F89eD@XKn7^K*>5D(phiM^umJ8nZ-Tvc3V5E@TX7|?6G;@V-nikQk
z_5R%E;?Hj0uWiHAZ536sRPdBUmX+R$_A^gz8oe_3RiJq3_?@QiyRoj9&zKlgSNAPD
zy*}^Oqdz~}@6Ooq|I6e4^`9;>nqPa}I_+6!)?ULee=j$(J@C0NuYZ4*W|+<r=X{~0
zoZCfT&Z@Qa{o7zVIcb8?h1R?|_uKDVGbCD1=FI(WliA51=y5_q{>6dcLf?b}w@eM4
z^Lv@1$48+jym2><hdH`4p8l;h&Ejl?^QD6!t71A9FL-VK?o`=~doQM?l{N{83(Ra`
zuX^Pd8kW7IZQo9{-`n>nC-**NmE81j=e~9Ro3?!NZF?emeZ6#O&63DPO)EcdPkr4{
zbTZ92hT-I!J&OZ9zD}5LpX`+EeAZz-&q@6>n|t|3F7`%=*>9}>uCnZe)v`$I<^GS_
z<&H^D-fYr&-v7^yJ3(4^|0%>euGr-BTt|cH^AUqQ1D_R6YqtJ-+${h9&Es9O&wPD(
zT>rMgx`-bi&$_r7-<)Xt`@&wQ2Jthw0gK8?LceZ({PXYW?-ET`A3eU$a*c{^xbwv2
z#RY$xKkNMawAVS+&w8A(Fe5x(hX41c;@CUyoQpqRw!dEe;nDi5F}to#>@^MB&$;&6
zX@;PNlYe3(t}PZ!m|P<9|Kr5gFUcpT30ynbdNFpH`kNbb`<?Y`f7*pD)taW+^|WV;
z$+nkoUZ3?_Xe55~k1EfrwddDNSrv6YGbzKYHb1Ox>HRgcZL~uhF8B7%tdZXJG(7(7
z`D2aeG<n>Ao%|m;<4=mv>|amA-leVHT`RFs*#7tC!rj@YH9z^iv^;ImAU{{2M(_T~
z9IZW!pU&~!*s-7^|HIMk7go1DSuGx2ljkq^lIi@Hnd!T(9#PT$`+TO{=hupJVtz1p
zzdz~mCPC5v*)RWRKm7Cf=GSePI8&l&?|xf(PSz!+lSfJueotcjqarO_Ustv7O_*u+
z!+*b@FP~jq()zLZX3(O3`%_ov-Me$Narccivlsk+eeK?tLjCBYQNc&;gzvtcw8W#-
zJhJGu{AIS@ry9rGN;S@&l{ywK@Z;I{efM-<&$(y$hiT$dSqAI#&r~csjs~*C8UFi{
z{p^0*6_;zv7^|L|?kjs9=#ltqO7LR;JGts|rlS8B>Gv0WQ`Y(^QuDXN%+763KVL_p
zsqCu7_rB(F%_wy=uc>PG<O|y)eDjavGN*-+#alJ&oRX$ztlE8R(dLNSK%u3b75`t@
z{7O+%EazTw(EnVj*z4#AYrnVnnc2QIJmsohlFW8LWk-kewkb@vud+t7+~!=aleO9F
zvVD_o?2fG853JP75@&i&6-`~;%p-8S#OwC`^Q-6k-wa}o`8`!Fu;=-_oU?n*S0BIT
zf39n(;*=<fdV}SgPKW+F8EqfA+n{6$yT;#kS$7k@wHI#votkU>p|_p4VqY~+ZC3MF
ztt%UCy~StUuHdzhnX%$I_n$bIe&!mvHdV*1+ie-%T1@Dl_cZxu*GbpB1-{p%ShkhV
zuxBi;{_l0kq`uDdxMkSpzxo1pDgHvmCzJ1{g$D~-?Nb*s-yr9(ZC;*d{Pm76D-Y)x
z#+LW9>D{>Yt4Mrdx%lM6uT|bVY;Uo;`{*U>_U2P3v~PC>ebe_kS@xJuIU)Gm)IGD3
zo?Whz?XLcDT5o^0_zAE5=DbDAO#g3{ZCBc?!Yp+2U|7`q?=iFHo_&zgH`}mas_LfS
z*NY2++g2)vzkgoa&%B%=H}dGaUt!CH-S7RD+jISJg>}WY+vnUbv`B2r%KPBUx`F@f
z(fHh(HLGs<i>wrX6_fE*w|rXioYI%4FG=4mt0_redm}UK`GH+p%dc<P@zA#Z)ZDF?
zmhbkdd3;n%>usK(<uj*sYbNw+&y6(ybuwJG|LKo)?S03K{vKa?UF_ojgtzRU`^0m0
ze`piF`tHj55;55)^QK&xmonMHb;61&>8dNgrhcA3Zw+VH!L^6yg}Ad-Z)Gi=a8PLB
z_J!;#&c!!P=P8+W>^18xhK*%A4;@Hcw`i7O>5|yB7u;eQ&au6X53|zMo;_`SHs>>a
z_4Y4&)|(eKE=Zc0o)9hb?()HV&hAygx10ZWU*G-t^C!(0GEox?T^Fmq*)&V7WXk#a
z4=EQjN{WSco{5>Nu)}A~<-3<yCaqU|IsMJV%9!W!zr`~*KiPQqMK{;{C)2#{-#S*g
zyzs}tL#el3ef{|<_<~ZkOxl0V!zWdqY&KAJU)o@j6Jt>F^ndcv-O}c}LKmkqZ?}#x
zlz;P&jqmS_$t9_8j@Z^Q{BvA;d4GiE(;ZdUOnx6Mc<%Y<6?fj*v`)|J)AA)VTsAQ-
zNt0$Oc-UXfQ}Q6OT=o>hq*v3o>0UqV87qE$veG`CZTHvk-njB~gO9;%8!n@y^4C9O
zZT9N%-M+JKVdt|T$$vW}E|=U)Hq+q_J5_mg@^Xc{ULEe-dP(eCg&lO4KEGS}PEh>h
zwK;DOE{&<^oIKsW^SSHGlc(*%40I!AhxM6weB1o+$+5^gR&#~Ff2!K{zjJL>Q_7#Z
zIXmxf+;z~&UXppg`YGS&dkLw+uJXRi=Bsh1PkyNW@`TOCX=~*c$Osg0Y%Da;eHXV~
zFGfnmbW`V@r@f~?TXbFV=AR!T-rh9%fcu<XP1lM|Rv-9ib9GnD#VEbU;c4G;mOh@-
z^SSf$^{+CC>GuChvr4QQg`fAYk{170c0B5OO6uz7y%$dL^8K1P!+AzY^So)r<rZ&3
z!~9s-+5cS7DSN&xFKI2mt;$O2-TB+vr_7$|Qh7IRM(efdHNnpnY6V|e|8DZp%(k=c
zb1PdtslVo%ZfpNl{g(?0<I>~>o@lewtS&pW^=`hM{@x|Y%nmyX5A*#CpS1tFzOBjh
z|F>FCZ$19q{ZQz{YX#z)8ozp9_r4;q#qWSwu9|sF7bg?T;jX8R@m~v1K3%}L|29)&
z^t}tr(`GHP_+|a5VZM5<Oet62s)u>YA4hN$?u$5dZutbolxfbN7Uz_1OS{VU=6(Mh
z9iBPy&(#Z>?7Ah62}Ngji1(dVIWoV(RP9;nrJoV0Hm*OfZ+vR<BbMWO=*j?rBTM-&
ze@%>;HS@krJ;$8`MZT*_Rusn7+*At}UZYpPp6lz<{ojl%pLtHSlM1|6a*p}BdBony
ze|LV(`DiogvG1$De=HBaXZgYM?n7i&DC<7Y!{H}6mtQ?*wNpDf)aY60tplYq&$tP*
zr!oC#-Em*!Tk)}|mu)P0oz<!(HIo;f-&7KL?Aq~%F8Xt>^G;y+wPyXyu<!q~A79Mm
z{BWpbrozM7drowI^JD6&_qA*2j(aw7qGs?ntE-bM&4kosUe4nwTgVi2B=n@M&8jf3
z0LfI_7K`?cr5aZ)-*`{*R$ck>t-tR2Ju?sd{{HgZI+Ydbw}tHv#<{fD`tu(Ys#a3-
zJ;m_XYu{n-PtSwCPn6?eh*eI%)4tAKDeJ<ybNAnwTgb$+FREPhIIyHV`1Rtb_3eCa
z-z%nYKejtBP;Sy`v`h2Rxt3X$47VFMvfqhPi;0t2xnbGUmK&Dm6+)9gTO~|+y0R%g
zI$l$=b^hZdb@6p3uUj^mnr9b@PrlONoLMSe_gwVK>t8?k%ciVYWq#~2?=D{1_X}p{
zrK)FTcFZ<8a(3?P&n_+I7f#Ka(YjP|n(5#ForPXDnbU;>b^V;bSzG3sJ-YsducasE
zu<NeRwc?eBMc))1I{A3vbcK`@&(!jh7A2{7RXN|ca5c=j8S&@8{Ko}1cU+JCG3n)%
zQ~nd@T*&^Rzu@uj6OWHidp%+9*Mg-xvzvD1WZ0f~6ZGLqrTl-Xmq&}vCdUUv8u*rL
zExo6Fr#;oV|K?kd{<9CaDEk)VPMyv8j`{5UZQIU0ZT2lXdw*I-(82>Mefuxok>}a|
zYSFWp7u(KCe+a1xO~2><q~mu%;F`CWt9U-oZg$V#e`#4Fho2OKiKmRF@(IReliz-7
z(~^kcnh?>nWaaKNJlD5B=RB<v9l9aMW|msrl{vAOugYC(-@~M45x7f!=LCiR51C<?
zuO5{5%5eYE^Y`NC0^vQ6@AJfp8w6T$+|rvkgC$O`Zg#<gbnBefI;SJ)v!2J?n5ARE
zY4c?J!I?J|%&auH7r2_IKD=DxlWNAPbgRGT(34_AKMy0vO@i<490{3b|HNing`u>4
zYhJ|J8^7FUvwJh}m8ba6m3^nh|LkD-&a>w#+{7b2uSgj^X)!4Fm^x*xr%?HV+)0bK
zfA;qJWY4+SB!=tp7S4EwL&p5?+#d;6w5;!6_}swPJ&*C%-5mMb6W_A@jBt=&cgJdt
z#_XW4{?qsQt^Boo!-Ev*ohK)qt(-ZLul|j7gSl(zloP7<32xcf`Wr2tYfTTcUYL-z
z@$B6#0u!sJ=S<xCaN5r$jOT(hJOA8IIq&#)+p{YVc1dQNn|G@mh(D)!-pF=q%&h%g
z5q&C?6TVIM2#i{!-*Z5?c-N;5(Pu(Tye@B+I&w4D`EWR^myn&~ev_@X-8a{6oVWh1
z{JdnnCjx)XIyIu!dKCEkh)ukc|4}&qR<=lWXVUvidZDME9$2u%vsrzY^Rr7&Yc->y
zH5s$+ZwghBIcnE9(J*~x|DA)Dr<!s$v)_KOX%l-|(XI<6LL#A^e?KohlxqE4$feBL
zCu|Q_{F5hp=j<tsY20>qS=tPK$z{fEe@<H-Z2zpeS!Y?DK`PHQmHm^|Hm4b!-NiCh
z;;X=cTD{!Wz7_HtUYwp%kr8p!YvTXt<Xn&IZobuL!@>)=1@oJ>#4Wv8xNrZXsdG*)
zGrp5FiAm{RPsfFrl-GC6Z0DJZN<BRIk}Hwz@{Nx-Ug|bW8UEcqHL<;xWqD*1XY?VF
zOY<KENbWB5d1%~lS~{0gDSy#g-<q<v$@1yqq2k?^1%jcqO&Z)`a~*pm#8Qux|M_tL
z=WSVu2^QLy?i*+)JFh<dd`n!^>OX#qroV14cw^NWXYf?UUHIGGlNLX(pZRwCGGn}H
z`O5Zwweb8+kCTheozXhJaLue!%%6VmTsmcq=Iqo%o^uW{*{ZuJO?bC$PR{lxQQhg1
z25WBp@Vv(QNv+p+u~c`1)Xz!B_~!RI%{b<C`-{;gORKU__nW0sU02UtjrwqGtyYQ4
z_gSZJZr<{Jz1^ajpHBH!%sF1L=3w0JnScJ(C%-?t$<%$~nv3PVuKbIH4gPoDonl%o
z?Q#6BvGFFYckw&VbrsmP9B@gF$lt~MkMY{th?SbDHQISuT^rwJrF>5{|CyS!Y0jH+
z<?E{V?|Mk3iRNg_zP3D|8=1WA0Q1eSJvU9Jy0S*SI+8Rq%xfbD|51&)Ku4{g*S}P(
zo)wUL=g!Mzl^vH3Us}NX*RoeA<W-)Qt@@&!E0fH=HrHx6_^()dB<}jHh5LOB3SQdZ
z6v^`ZZ>o|a-j`tT+g#esxOR?!-K^&UcU-$3Z%Ev6+QY2-^c<Vk9?$AcuD`>VPs=-}
ze^=$)rvv{>r=(1ob}@BU_c5_0WtaO)jqlWH`Lr#S_5EC6P@A%O%9EESZ?A=9URo$|
z_ReqH&B<ZHOW*9AIr;CN(Ad;_kJ}&En6AjK3YoR?&TLn0;Yz=)^`iWlEN$s)zs8$x
zHaN6Y@^5CIhSU9RDJAPB<^GCv_T=#tSodkx!cB7)-dd7)!aak_m#w0F>tFNy##L;=
z)0av~cD!FAvQeGOFUxkajYgWCF7q5O|0%4s0bQGF<WkRWXg|R@>+btm2O}2CuT;1{
zXU&<aom!HQ&CZC0emjwN^v>Hm+R8b0Z8u*?h*pQ+E_1cyn%W<3{laU#hkldq#Am0r
zI348(u-tw51($2=O(xkpZ6}_{AI_S-ZO50G<-yHtaqFI#Pb>+wa<;v?Ot0u;qp#Lu
zPr>EYtmiI&b`t%){PsnYMTfuK=Xj7Y@$nV@lesp>SKe7yRGM}&#?)<zN5I!vvQ=lj
z&cwMG>#zD68d@HlbKB%t>z!@K!cN<->Y5;%)zflQ;q<Yq`Pr}RZ(q6ccc;<Qw%V54
z&R1g=UAnT?X6n%Z(<7?qv?6WyZ2onHyYt_h>no+NnuTqhe`t+#@4x=utVLVOB=Zx}
zV(#iWT6b>$H(NI{uI=sqdoedZ9h04QN^@q3>)q0?4_4lC-R2q^Wc>P=W)-`~gd;_(
ze>wSHf1|VV`r_QFhPkKjESENYdCws4{^Q9ahp(qeE$q0V5imLbO42bwU(V&L@)Xb8
z?=olEzhc+hpU)@NuX-!}bI(etj%722XIYh-6tjG+xtqklK%;nt$)B*=y-Sv6R{5HL
zWv&r;zI5y2Nmqi3yOs$qObIWY+M<2$^(vFxGDq{n3(pGqnw%}W_+?v8(IlUx`X4-V
zZn^W>g>GFxFD3k1eaPu~qW)fH<yU+Bc7C(5t-Y>dQMGv1)n`jHOXYWEy4(ymKD;%0
zy621orvn}=*m5a&&I(7ZfN9HFC)!j!{c_;YE^QIn757$VM+>sP^fR~~_*AOrh574?
ztF-p4DiaRPm0VRgH&nRAq?A?YOTugG-bLFDaut6h8QY&vb9_82???NyurmRo{vQ3`
zGr71#OMm>m8}jrvkEvPm2UqrG`#+}&u?I?IT)Va?JUCR-@od?<{G+SQQ=j|Ip4C0$
z+?IPXS-Dav4r;cc_r-z((s`3(rf}3I>CT+o<i+0mW#b#M4aeo@X?X8ER9F>KusPu5
zE^lrn-&Hl~KVoL>Q2&{^akGkaylmoZsa~7QFOQX-?Q=Bcdphft`bxPjvBd?uF6@1w
z=5uw!>{n-XBYLaW&TzSO<nXKq(-;eu3u(!Ta&&kde(Cu~<!(-UwpWkpn_c_!_P4%?
z|C~N|*ZQs`wzCHoax%^eJb$*JGT7+=uf+WNT(L8yUp=0knzcHzWWF4yz2r`9oBq7+
zwAWT&V^(^$ZTO&8<WseLRm_DJ8{fU}jy8E3#ssF`TX>FR8*|G^ql><)=d8&(R->gI
zJUKJA`l`ay+wX2>Cf57TO8zKZv;5V5zI$K1e{Ql2HhG!2NoQ(6n9}B1N^a-ou&oz*
z@m2Oahgo-Pj?szxf2;pw9{ykULA>k(|CHiRo1c@u+!T8K>34m<fTMHbv1vlLUK}sq
zvan-yD96W;?lp$qYu)ZnKA_~fMAdaw=;ctq&JOcdvEB2Jtn*)cWJ2Dh9iCgeULH|%
zd~|G8+?P7XkL>?{H%m^a3*B@%RblG83^(ykwwhJ3XHx5rm;U?pen0O&f0GJ6e!tAg
z@?MJf7T$5LZ24q(xWDA6yym|X-yVwvde?m7(GR(QSY_4uhs6f6ub$YvTU@>Alu6+C
z@)-x}_dB|E&)K6@z3_%s>im^j`<DFru=B^7*#EB&d|cPJCOdaV<n>>i!BOi1;)Py`
z`rFnotyoy)IPLq(tw-jDKDwq=m_K#L_IW&#zpB1!uKM}sF#oUo=Ku1h2R6ygs_<8G
z2ss$E$uRr!|F3_3@B7D;F>mj$8$2Cf7c8;aEvGr@$2vQfWu~7GUHo(Q8ApK9qs0*y
z0_<L%4F2uW|3=pK>{CI%3S&ucIia_A?M~?4_{15cpE$Qg?!+vy&sMsNXEevC@~kVZ
zK3zUn=-h83?P=|u8}pvNh&}&VJ-WN>cSM`?!Ng+u!Vmq%H$EQSduztsgD<{c`(eLh
zlC9D#_lp8~=0ERzzEmR;t|N2!;@JmEd^-=mT6+1<dnPlMjjOr77Oo9_?Ngd??TDH0
zC#Q_sOABnyyVz~wyD2=a!~bO1#-u$W+WxIK^~xujpSU;I$#A|<$+O5~!Igi!e;jq6
zQ{r>K;qtfHslJy_we#-z!fWXK@m5ce<4eg^nTOlFV#~{I|E``kqt;Wxc|FS&&;Ppr
zm1ihxvWA#s?r4#p&YLX9wo7l-v@haOF>9whGF_Rd^fOy$|FqdgoZAGBE{OWJF}U}Q
z->Nlh-d}y17+rVrox#`DS#mATm&C<0HaRbzmQ#PV=3m_*!y6O1zNv^=b9zqtzh?QV
zc^fiYa|E5X{3(t8#-DJpcg~S-J6^<vtvs80f3kzn(p5Qcn+5oMW3<;V@A3`1{j**9
zzkKF|Ctdy)%R?V;Gz)&q^=tL=_7l7A@EukvD&f7XE2dF2KlQWzm7|_T_xX(v2XO71
zRoZapSGk!mALAkBO{c!zKYBm>{eK-kkw>|U%==FqY~jgRB(Zz$ErHhQ6PSEu*Bm_+
zV9>u~`3qT{X(mp*Jf|ylSseFWdFo$e-=?hF(s)8n_st}?4-?Xz<TErgJJ_4gEfP3s
zVmn{IrJ?F}T3|t3lHjzfk`XiQ-+yL2_GZbOdt!@JUT)RZdo1RDYQpTDOJ`5~tlT)Q
zJKASL$m*KTzdP1_3^;Yk$N4GKzO%QoG;U4y+31;<P}(eRY{!3f^)yi<&1sQ|46NPD
zw>W&V*xdE=x6e1u(6_m{-;UHcC`?qD?pxwywj;$Z+srkqvs1Qc<((_@A|J)19(3{3
zx&Fmi+Q@kNyI|uh(@sZ~FlPO!oSP@2b1SU&d7j%J(?Z>I)0q|pJ+9GI__@U+jH$C@
zO6k!ln@$wHwTllk**S4nfLDamUdi=&i&Ac`5vnkIeN-bXU#Yol{`Tmv+S6A#%vx4d
z*OK|EQ%gl+7mwx7w7uO@Wi$0#e|p8vTI15&GjG0gu7`eLRRt6G-IX_U-kSXLx+st_
zUBckm8aty|HeR7&n}StWE)Ytp;=UicMJi1!=;~PqYr%sn?u4pcKIav;`fI7)F5ghL
z`Q1i5n}k)&U#S*PU!CT$_sNYA-@yDpwx<QQKbA**T6j8#|LWN&Q<qKIPq_k@FVmTR
z!njv!$JA_>Uir<R65mZ$FZbN0kQw=5g`B<N_eri3cfI=3XeBZ+>f##z=~_8*OJ=b4
z+C530oO<o&mt(tHznCr$otLXAxs=71yUJTEL4WQiburdQjOvSRxyz>ne--J!5mst_
ze)-nm<Fk@<SI2+V@c*^aPWk+u&LyutDp#DznaU>Adt;?qiQBHZt14t(X%?>f?IJAP
zeA&pzE{fx~*hgiXv)e*~HvP1A)h$gud$i$+WL=5#rOQ<n1~Yz^X6$Lvk<Sw}lbfO&
ztd;J+Z%@bU^(Xe4{yX;hex{zqEXM<Bu}Z2OEZZY2JC^Vj+K4c0oufJ3)w-rHR*>O{
z?VGTRf0|eCmf~K)wqr$<PuadbpBN0TifuUYW%b9%i8saT;`d7C$Np1qFBdxaq`Cg;
zjA-XgoKJk`>iJ*J?p(jrZ;Ip*yFH)RF5{i0=NCMmqwchj+(*e9mlvEkk+gqJmg@90
zvz+Z~!XE8Ph^$o-4YuLyJQ`Q2w<J9=_q*4z$l~;C6_XET-_6>!`o}lJkcpA{vjY5N
zbo@T|it@I-YG1x$&(kZ>JxQe*v&4?a@&A5ow(#=0U2mfqF5LaP!eoBsxvM_1OQu}q
zbjl0aDZ9*PwS+>6<iCrv>?L1Teew_6yW*y=*0Io@pBX->pRQgBNMf8fjcNVwb=7%C
z@9`#WEW7kRTu0&Bwpr4#&T+G9r#(wHigV{}JaX*iLXkFxgp=)ZS2R{WG>u-oOet`(
zEaRuh=@)B{O}!tMe|Xu#mwx3hX72h`wBF;0@==pzg46X)4s*LF&HC;raBuU=^kS7o
zTKcw*o^$yoEt<CORmcy|j$`+2ZcG*RZ<}+q$KjE6%K3#^dY6*=qSh?E@gyeHeD?D#
z7rNb;;_pVdZGR}_<#&gjy{{(lb;#0)qe|1$7tYFUShgoroq0CX=bhedPhF*})r$6>
zmCM|-l6}gwyZaSx>gK(w+v}@p*73mU<m+jEp9`hmaj6F<vHg!+s_N;>(f!V)_f_y=
zB~{n<r9aiKSUWYza9Y)Dm=g7y-|~!9gon7=%=J?oJ|<i~uH~BMF>$jiYtWtCch9zc
zY`083nRF&AZ7n}%>Fu=(k9|1HV{ldH^qlRb(VQpscNp>(zY$|u`Hwm6jDqsa=56!#
zw@Y0M(9CiQ6#4k_==HN12Pzt(dsXN8+O68;Blcw5Y^nb<L;Wu<IXM4nMX1IVokmX4
zaJ%%v8Ls<%R!rvHGxNxV0~raAXTLeab2O=nJ#F*)pffpc%d+HO)%}{%yy8)~#^oi7
z(o?rq_B~kJ5zRZ{io5sQhw~x@?);Sb{#@YR4VBnf7899mAN@Tu9xP9IVSM}Py5gkH
z*S}MXV|uE+n!Gn#S9$$Am!=+U=w)2>Z%f7ZtmtLmz6D2G&rB=indqxDeX&`Y$jZ7p
zfp;8V*KAqx=J1o>?hmhNUyYqP@3iO4rCB!|;;R+}Sg&7lB19)~S&?7cf_a@HHLENf
zE@-yRJ&@YO&9c~QhO}+{$s-GnEO%Oe<#OiMl`)gw$-ay(%rd;6>blA^bxzXi#TrW@
zH?2%LwnK`?t@3haoz<mNXWoeX%JB*QB;u2#TAbDLXW8;sU+y$Lx~4136|Lm)HO#+%
zjmOm{zwTO*lW~i6LQgHP6Ih-eCpcM4*;~)_p?E<rmkG~Q8?7BFjagZhbyWcgJKf*p
zU0X15o8{4amOob-J~v%{r}w}W?pVJ*ztsjjH}ii#IwpFG`N<oWZQm3VLf$&Z>c*(N
z^tgKHVD_EUIf{lF8_#qF%X>R)y%c(zw{V7*S?Bh9zpZ>De=bN{5!4XA#7cPQ_u$Pb
znr@{`vEOzVe)p`b+pDa)#_f}2sL78tYh;WpAKmf{%j&w*Bf~zGC1vru*0_l(TbqKq
z<KK9dSo5=O+P_*xMKen1z1zW6H=SxzmO5_cs@t3ty0^c?_i~6>VDP7?KU#t@o<7Vs
zC68U3^87YW?$h?hQ%s#lDz+Z2<hUBV*(L4dwn=BDeq7u>=Uw#=34w}R2amtKc`8XO
zb8+YS8mr#zxmu|cwY~3#`E6Zr?S!<^uiWW5o82RmZ={-j6t}7L+Ved64M$PJo@I6w
z7t;3C_<ZcxI8Wa5;LP4XlL9`Tm#&IQDcNYDbI9uSwjegI$G)c%uh#}nDXnySUv<^@
zw^pZK{Y~jjedkZ@?%(y|-Vw#Xrc#aeDW+|E+y4JJZdY+b{b}+1-%DS7&}ymPaP#Jr
z+4_flXU{WMFHC&0<4W|@Ii7k=TE+Rx*QCYoT9XrE_A|;-m3@iV_bhAM$m56Ie)eS?
za(jAtL*&<`=d<Rfiyz<IA7k?WL%_7m5{;M3Z?0X=u~=@~vuP#=?3{$;v{qj{l&qj(
z#LVe5Ih5u46TXjIx}1M3&k=I=WSV?%=Y3D}=2NM<ZOeXD-fjuGxcrvmkK>DX-<%vh
z=WwW$M$D<Zxy$l8&#(7f)n#%2hST5V-*!qTF6(V-uf3%6ebMA8w&j{Jdw=jvswtY4
z|5xmtr!2F{3y))WZ(7%RxVRX;5$U;7dhPt0kJC3-uIA~9Rb88G##t<W&(r+C=3uw2
zx{EWV{dfOv=gBj)l6$7P`sxm=s+Sulm`p0V6~bke`mEhb%jV8x-$Jp>*=zH!O8e@q
zUA3|MYXQHhQT?%r{Kl7UT?K;~H(YtcvGhlQUzMKDy7jvRy*ch#e1DYJI3f6u#m(;#
zQU$qTSLQpNbnE3W*M4>8)%^G08||2HX}-Fi-pQ@=aqhW2K|8vB>nxc5z^>mx!|0lD
zWz6LH&FNCD<)=>Zc<-~%;dnfcr90(w<f(ooXU&gGtc~W%8LF(_92vfFPLyH%t79&E
z6y8kcYCO9*aI;&y*N)&B>vsfj99i=wSna0fivWG2)YYoKD}FI)&R?9&>EzSKsWM~#
zg!`BLH5qlz{`{)I{BW*_{Yt|v@(<VPv2WV>MKoS2Hjj7u(l<4sdzPH5U^-kNu41cl
z_SzafhY+O|%k+G@R5lg(MfT2UDGXxJ-<o8`uzbOh3ylfxu^+u7)pvifv7RJ2CrfU^
z)mxD>Uj@HSYsuZTdhyCH$K!+k3m@&si<6zc(^6yQ?A$ezieLIM9#uO!!?oCf<F-m*
z`{wQgzO#Z#@A??6T*Yp=desA#Yg0BC^0#~6yL|a}&ySro|5_ggJ^5*HB{Z{8uPwo!
zyI>wmZPs*OOZAO)6FkF3vO;o&WL7QQrI$E&lOgvp8_BiiU(aokc8!@bslNG+;ek}w
zo6B`l*37G&6Q4TYh41Q>)0(Ge?rJac-*r!_+;yjs?OLrH|J<9~SI^;UD>67__w;fz
zSJxk>Kbb03y^pTst+~qF*6J$2<~5gXj`BJ6CUdT&xBYi)G?S(}*hlVEU$Czx*6ct3
zlcp$FquKf~I*ZZ+_e7OOryN`@)gS6A)Wx;xli$AR-A4YAs(QY)f7V`JvM}PQG3!Uo
zI9;JA?+W$3zf>|d#C>?gt#M_x+tW)o7;nlbEGzOWy8p^}W^a_wse6Yf3bl4xrp$hH
zGNaF1)c>Z1PU?J<vUxAsYMOm6+ukg_>2qq~Ka*8oLpXY>Pid72KUY4w>XB$3&w{Rd
zq2G_p4&647qf2#S|Dvl&T}_LkkKAx{UHxdz(z-`(Gkj7PPMHwg=KcLd$;#%771Pv1
ztas{uoPV)Q?2%#ggf&Y}MC9*YWu<Yn!pVWVsJ7*6sOqHyJ(t4#0}NwSs{KB#ec`GU
z;~cik{bQWr-_?2gi*u)}$UNnAvou36@J(d?Iku@OpM$mv_o}bu*rwI~N5fIIJzcOa
zXjN)ex7W&ffAXK|EccE+BJ$#rWZtxw0XMD(%RZa8x6I?m{pfQ$AADaU7xGz?`~H6c
zqjW<vr<odp>D}cYFMHjwlIL61u+7Kli-sGI$^WH^({wy;9X2av4tEs0th)O*3&V7U
zn17+$KLiMUbovp`=(8tRUhLGOg&fYS9k@a->3rR^rin*i>$zS`h@tuN-wzj==*Ca1
zSmZjRQ+>*_r=N>fig`({OPeQibCRgo%m&Rz6BgaCKmEviRY12*P|CaAn{r+E$2DtC
zdFcJiCvxI>e<|g~r=HI?^z>cDRnL@~CLL%oP3!UfF2Pj{7g~Q;I7QrI>0VeKxNE^G
zt5tKRP4}C-aQCKdrMKz=OQ%en#Xou7wp&`F6J-qeB0j$tPo2WnR&#A>XXmOphmMzi
zHBpOv<R^Zxz}G%D^cTPS(#gLT)PL+3Y;!*}TZHq5j!bmnx6g@7RK7jmoVUIH`{uiO
zkw2E@eAP|%_IuXetF?UjhrR5_ywCTl|EdxUIK5?d`?td$Kf|UL#A%(ba7q<reZ6za
zd{Mjn;1931oZh@{w_hp4-rJlNY3uA4bFa%e<bJdBYRrVANw+)MD-4hCe$Jg@B=^R=
z=8|e#SI@d9KLUa+r(W*6&0i4acj`&AuTsd=%f*v;@0M1n`u(k$yP|!n(aTFyPE38e
zC8B-v54CGPhDYy|wNBP5o4;c7<znZ|RdK(k>8-rm9qH!#aA}MD*Eh09S-!c>(7f53
zwK~WlgIis-DnqpRrQvjqD>uF$auGF-*F5F=a{J8STuWQAN2&sUl0$XcD=c-_FD?jh
zJ+W)q?weNO%A#}CdetHewN@Dg7`ck+yU2S!NuSxCq_{xj_e!Uj8_ScH?R5SjZ>=!x
zMAQ)#sT{-C&&r>zTyY^^>sZj&%}XatYbfr@lQS^czVw5iTcYO=mb?6}Ps^IvHZM2T
z>}xw{$NEHfN5Iu;&%lMtopi1)e<i3N@hoLx!s_#DE+)Cw{wn>uTuSqnsDhQSW2X@3
zi%F+9uGr&U5_nQyVrTWH+6yMlM;D3j+^HmP`orUt^Uc_h)^@Gs3q?9FcVAvtsB~n8
zx3hig`Me#omKg<p^XXWd6Ms4MZ^!}uD4As|rQ#nh7GW1nU3}$IWb)_#GY=}pS^bz+
zlrue2k5hayZ?X7B?g<-N*Y4u^5!Q20#OvT1zo|>2zn7Zzg>B3>+8^b({^sKx&Ze*3
zigzEE9Q?FdsVO;itK2T<4LaYv{QJ(&dfDzjJxt!yFn6o3+Kw`f-wStKYcAB>w)YB;
zZ;*|M;5nP$YqL-AfA$b>Ssk<3%lFjFa}V8v=B)m7O~qsTESC5U>pYfSzA(FPMn#&5
z`>ab3bFCvXWDoA%ZZBlv60JK^-q&_P^!^u&6CC83T5di+Su$heEET~C|5R2weqA}S
z?Z$Oi|G9#0UJ;r{pHK5UEa!Js@YkEw6X(B;@a$IOh^WZiDEV>qePOPfzS)vhC5@rl
zOMj<mZPn@6F{6LiqK9cLTOU1Ks5AFS;J2-sPLdg13EeE2J_Vt&fq`!yu5H{H>Y0)w
zzJ0RBCjb8J(^<v4a~CYRV&G+*`9wO~JS1~f_BB(vB@^|h-CgAHX2p>eC-aJqZPmDy
zdr0xr8g_w=N6y?lpKE5bX3~N=rLmj#8`kQ&$i325me_DZZ0A&20~VD(NgrpsZw+ct
zdabDcq*8=mJ@k>sfjq}psb7B&b}QLw^!<*Bf2!e|B&fPur^Rt~pXZl9V%7^*SqnO)
zp4j8TvwcxnkMCBtqd!9KUaZ<~e%F56$xvRi-07=tI4u>>%hU|9*OHpI%<9D1@MG0R
zBFPf5Hzl@aoD<vg)Z^3Po-F_G)fEY=H?{EptO@&4Xk*{nSM{Vg{b=NyYb&2{P2Y6x
zqM~C){?^xCNj`J)gAaAZ97&mCv`y`4WlY@aE|afJ#w%3#c9ky3x%T;qSK4}I!<V+V
zuTI!^_rRx+V1=`5qx}2#|M}5>E2N|D(7iym`}^I)&ZRpn;-4#eDQatuh^oiCYoC>0
zOC_rH>pb?Zxl;5n`C0Ll)w#Ep*nK*`n!h;4YsM#`=lf5*5R!TNa0gF<A;+6-Qk!^L
z7?PhZ(?5PVIo8a(|H?)#hC?|XGNR169&fx&FAJ&pp0PU{ve9h*CiSO_9VT4e6F!OQ
z#aRu<@2@jo9_Ksh^kvRrElai4Gq-TBTB&eh)|ve$n`bcunB?UOGsvZdWOz(n*2(Zx
zGX7?Vx{qfl<MHOhZSNNxS1kHv_m|y4s$oG#@b$^{XZHO}Jh6%Yrpg2Ft-UMD`lSk=
zR=TSl@X61Mtl#_ZJ$u7Sp;ghxw$F(_zV`lltM0nU&GkO3JJ~`{TzdLje#za`|Ivq@
z#O3yxYOP=Uaq-&;C-bxPKejyDcvD|!yT{R8zF)+?nyfsx<Ifkng^_LN>!u%J*z~t0
z{L|g^uxqQhghel9e2b0s@yJ!@kQ4TH7fkx~QJlZs@NKR4%S<KPvo|t-TXi38ddtGu
z!NC!7F8#N;x~uz5qa_|5oLfz<ov7N^sCqFybaR{H!Q-n|iMs4tG5Nw&_Z^eYZvDjc
zGvxYwcJpiYDghra9o#Nw^IG+2q*~*?yf>Scy@?8K=I}ILdHa}8#kx%IsHusowPuQK
z4Jf*^aADi#|C@OFZ9XJ5a&PB&cym?PS5@s_yfs&z+Vi$1bDsWc(-~&WDRed4Z-?1$
zi-QZUOTO-l&^%qmyVzG`wMxHR%v6i%6{cs)Dvk8ko%&X|&guQGoWN(=EtA|&T{b@w
zqU>a^pryRR$slq<;q7VLKV%6WReKQOS+#hB#MadjWg^$hN@uhCOFXXDn=e&RJL_La
zv%sBiQBTvtlT~C7Ub^CGF(uk}xxeZZRpXAt=c^u0TP3x%!*sIK!>B0_Pk3!|)ZCS;
z78}r_Z6<KF=SQlZZc+4(mBEMSL>$^$HEX-;XI59wB$1Wcd15*tuI%|6XXrSsKK5Z%
z)(owO-$nke%Dz6g=B)nH^Q$`TLfSIUA4v%^*syx?qy-7vrwNJZ918H%y770FfW79y
z!`q9sy4WV2T)w`-%POx)uV<=Gz;=)KveO*{FZRV(oni=oxyJh$U*yuPeQr&Y^Mz&0
zm|gcRNbXE|#Qs6)$?bK;-paoV`WsJmlyolH8vb=^`@w69F80fQhdoN-U2yE*nPmwt
zuJEiqy=KN1JI=PSJ&PhvtS>qFR&BuwWxrW|N-Lz>e7aa1^bRfk7$M{NvE(Yp_O4~R
zUb7ZuF1$I_^YV#4r?YW&({{VP+<bc3YG1{5vui}>H|H$eK4sC~slS6Ao=n<N`~Een
z+=^RkW_+L7x+XAt^{rigqIbg_)lRLmoxrns>B+9jyQL~enWu<;?&6-L(|M~<qji3b
z*O5aTe(y<J825F7zpC-->N1ZDQAagTS*uxlty2E1ec+D#RHF$S7I^IoQ$4u((t6A6
zcLhha`evp0txGZBT4t&^>(U0MwT~h`OP^8w$*T6s{@|ub$D<aX4>i0kw238U&D;a4
zjulVRbu07|d$MY2;lGDpf8S<%^Z$pGLiVg4b*`3QU0zcnWADBa_XrXSjrD$$W>9ix
z%9P+N=d1a3(>WP+JAQ{9Wpi`)eDu@K_4KE_?^C0+xN@^6G0yb}Jfn5(T^YA+gH72y
zXU_2b=X&qh-}U>p<4sbaY1;#jz$vS1j)X`)h+I`UWo6UamEzY9e>AmS_4nZqpM@G`
zKIy^lo(d+<e)j(0&hsl=Eq@+Qb{A~35S}B{86@<=?tPY<tfbj(*$+L{;+M~P&oDTD
zNNc4?@9v5%I%-1I2G>>wt8-|3hiqJx@HG9$EYl~)jTdDXzTNzF4F|*YU5-!K8NU0U
zkXpW`lW}U_{I?N&jAtv}R?gDgnzmwVg}v(BR~~g9S2iyX=@Qvg73j63>fxPvj`Lf0
zs($MK;V;(Ny~^#fl1z=8Ox0^q$qyHEMLT~b%NKWAtzD_;Hkm(v)q%DAUzdKruWYjV
zYJ6ouZ_P)|!Wa=R5xLf=;Ja=@y-~rc-#4c(*PBvQuC;91&-|0yr#v#ddf7AUcv7(T
zZ*RwdO+O~^E_Y;C^<M6qJ}qa?*$S48fjP^>!{zmFm-y$rI$f3~q`L4>{~_TYM^}}w
z%I$g}w&`@*LotqJx1?<r#^-DfEu50^?3?Ypx3|x9I0R|GoHrqQMSNX=ocr1o`HwN7
zvC9^|zIroN^^DO=&L2rZeaBs-XRr7A^fGNz_(4B6H?J76$ZPc`FPI#Hj=rlnHor=m
zKeDyn=7rFktuLQy+br~5{-eV5YNtoR^<_)09uDRG`A2Qtrt?~-j&#pVGG=xt?8;vr
zd_s7$J?qvNX*DJpyBOb1eyC7)<mTodS!c7BzDy8Rl<ldyIAu%Kscsd=K&DrYUcqTk
zj0AtE7I<<LU)^cS8>m~jdvk5?r`SzeM|Z6B)+{>8#BaQOvRYX5^aF`I!i@K<5}ceh
zH8(tc>ekzvx~J*A{AZx+<aLA1bL&aBDcZ)*%Od$Xr!n{jy)KwLXG-3bJ?}cE$}VMJ
zlT@eGrKf-Rr{@P%A=UJw@}IYyF{&!&-^(yNNVVx?>H8gfw#=<NDsH<~L1*%#vnw}M
zb~Z11o$Pt=%7Ps$t<Kr~x->Oo!Va008X9`l7k(OUHa@sS>C%kl%i<^eFm)FS(b}w`
zG&B9G-(I6@4D$Oqbc|#JS4q9)vJdt>y?)Oht=HY9l{covfB&?4y?)=*L>Bd}th1&~
z6Ti*mr+s=o>#S?n#KjAu;_GxTY$+{z$2zBa=A7!rXL0J!;|}bcesZUH#ygnsK8MTK
zPrl@~sGT_nohiv}kvwzGMhD|-Coz~i9gMG^oXKrbJaY~zbNb1X;VkO$nLnpak58@L
zsK<Z%)9bjpuV0UDwCQ>9WQ)V(X(vx+GOAB~dY#EP>vcD`#Y+ACVi}jLEGrq+^(Rjj
z&-jGN3>M2cWo4PlxLjv4Dl@EQjyMd3!x-W%b8xW#{1K14!lCy|xwVMzsn98RUi&DG
z`wQ<p=2reQqi{>=3yW`uPqw-j{t#*R&#lySnxMp?*)&0khu>O!U**gO4(GUK3hoL;
zEK2SYpNx1dnI<X=s2&QK)~RvY$VI`cWucC6ppt+pQ=k%0>dX$Q#ub7cQi&^G$8|Vd
z;c)3SxRR=r#j?mvA*yAeuJA=A0bQnx(|C9KxO6*g;c)3T*pjOBj^l`I;|^t>??!^<
zERW*)Tw5N-sXgj*xFgshm-yne%NGupeupm{&hcuC^xKV@E-LfX&g>9ulo0F?Oq4k7
z(!+72!=Z=caD<@?OOlI1T1%pyu%WVmF_WP(Pw~tS$;KIi9Xl7SOjR=3yW)S%jXgiI
z9Yez-<6|~ly*e$Pd)KaCzaBMhT*sQ5S5fRbNBdb!;PWnP^Cvr#&WqNr`%&z=AoJjY
zOvi<p2N!0#F3LQ(DARFq=E238u1hixF3EIUnt5<(rt7kmYp$O%vgEe&)pqjbcJt--
z^5ypP)eiFI4)f)X^5u^6)lTx|PV?o?^5xF+)h_boF7xHC^5w4c)o$|TZu8~t^3B~_
zdi`YNhMcGiIZ+>Sq6BlJ4059aa-#}zqbBA?ZODzfkQ?<OH%c%s${;T)ATO#QFKS|5
z)P}sM3wco=@}dOuUug$!@;TR3eYvaVa#QW)rn<{b^_ROEE;lt^Zfd&R)O@+C<#JQ&
z<)*gFP3@PvIxaVLUT*5T+|+%!tLJi4@8zbx%Q2bTdNZO|Y-3e?&Z_ua6ilMAw_KWV
z`pFZBAex%qMX{c@dY8p6xVhri)HSg$Z%y5BSL627EwKVcaRNngf<=1Ty?cGmHBI>3
zHQ{p;3?Z?L?|S@WodYo&tL58@6iz?MSmI%Av2c#_&X!Mshx^4&m1}(eafrh{O(s{;
zs_$<>$-_VyyTpo<?CQ#Y&psCY|C@JGmX4hD-~Glj>f6~eZ&tm$S?#)()8+4z#?^9X
zr%P7<of?pO_h{Z3f&Sn-M=a+pJU+*K&y(XX^!^|EQGGNyV*j@<+vE29-_2{XVI$j>
zkAEDeNBP*;oBVFd(ly<+r){dej-%71?~_^Io9$OIpK-ZS$2Is;+$8pu*Bg0dxHshN
z?9rJVXx1%w`DG-N@@dNyPS;J}k{SPcUsBYavVW=ACW*gd^W&Km{x6=|`fU4xpJICZ
zv+Tvx{%_zhzMAItd!+&Yg!)~4kz3`x-zV*io%K!mVQ2s8;9VwfmuK2fJ9+fejtntz
z?}trabhm#i*0$(h+WTkq--uPyqJQt)D&KjyE~0MQ>o>Rjuh;2Z+x0a1VvhEStw}S(
zc73}nI_LP*_-_s0l2`h*eKX!NKhZip!`}I~<hIH!^Am5UXW0k;o_Xv1mAYk3#<{<J
z-X6cy<8!-p+T916{ytvmYO}FpxoF^g{;e+CYO{A=6l$pW#()0r#0?vBwN>xm-nX;1
zTz%DiQI`7;_8rdcRzH1nYEHkrX0@NpYs1|~H+;>Ijup8awLva-bM-H-^RWwOYWJNM
z3XeJA^-xpOMDb{Ai3od;YsEB<+uzUdFRZuP^S0WbV`9$p?0?OtjpS_(-+t!5?f2x|
zdjFycQgb9TgI52K>M!G%B+ImMb@zvwzq-6CmwcugYi+dp|5Do{=IzsFm0iyxq${@9
zR+NV&8@|ckTJu-6WB=c_{m#rkf7tyy8~2Zo_lx(l{Wnk7OaECeb%W9B(%-nn_31L5
zvZ2qu?T&tVp@cms^0D{LPqY7s&R_0)@4xWoAKSAPK6Yg}zjRx<Sj8ix`?9ul+_gV^
z+h?t<UK#PSz@F=_ZQLmh(J439X*T6=I})X|?b?0jr7It-Jabjc`q(Rp)yX=_Oa9G_
z3<_K`ogwVK25*z$-l7Lj85`Ibr1&l@<jme8w3w&I$l%n0^#0DsSrVy}bS~VN@Xjz}
z%}-j>xl!%dEQ!+|JN92K`&+-zG51;Hr+wT0FDf)}dX~_7^i)#v$GR>lnZF<FKfZeD
zT3GfmDM>Kd<BNGnxR`XedH39-j1!KoJ=)dzSha7uQ8%kXw{mxQ@tz|eg^pL>372_t
z<fGB?(=z(Sd(ND%w%Hu`tYYTl)V}HL-IEo>`@$_e9xAln50P@zlUY(Q;gBQO{*^^N
z#|6Yv`@${bo>ok~AF|W%i29M$AWrY&tu7O!k7vuQzgZFbj!ne4JJs5F=aTx59)ILB
z8IGl&=Uh?$$!Fc%{&g+(mt?|&X8oQK6dQ4?B49(r+&kC!HT)94^lYnhUtYZY<-e8v
z=fr1vux^+rsM1)e!maTC_RMQ-);m{kN$=#nZ#;9lAYX0z`8-+a?e>4J*=Id?p1v>g
zPW`gk7cTCv3r#HCxBs91pI86?vKfWneVKUwnrQT1sh2BNeHUsUm7XfQv~iN~UA?D2
zj05rn=9+h$lXh8t?Dn@wjZJ?#f?jFxPFXhn+5MbHrhTiXb9wPD-SvINRasSWwM;L^
zwbc&~{6D+?kGk}O4{b@ms+(SCRPEEc^xwk7F8p!6R`;PR7EZ3L>v#X0c~r(fU*z~S
zTgk8aPan;9UeW$yHq*uZ%a2Yx=H>8U*$1({icJPGb!XPwOIWXs^)g}9_j_Mk=3rzQ
zpe&QwXddr3`%e<vmWK=~>qB1HpIh2=Wa`Y%%cuQ+wdv1f-V)Yt&L5^P%-$bgWbt*r
zZRX>)OJ0gjI+oVU-_4Htd4Go6n_UG@%<czW2^YJc{#?JoJAX?)o7eoO`-@i8vj1n_
z_u_um1f9U||8K=^FIM<}TlkZCmV)GikT{#wg?>?58lh9BRL#A5!(F#@3Y*tuZ!L{!
z99v}<m-JjR^g6Dg^>?n<TI&u7|J05EFNVT|DUE?_ZjutIcVO(95~+`Z({5ag`dB;v
zzwfl)?=Lv)m?h0~nYZmF+u@St#9b#1?k3F0Gw?B&SSHOA{QrC6)jFZJcMThvkMT+P
z#uhg3-D4`XKAh<opTu!)k%^4`Zk=<=;wu*aWc>5o-hX%G`>c}6;x9k%E~*wUel4!6
z7PUl_X^RBI;tB8WGkp8yyQD3v&dba(^X`jR7k1P$Cuv8X^PhF2fREwNLQf}odmj(^
zKPNshD;%|cdH?dg=f`uOX}sb8{;~SX{MN#K+|Qy7ew^?9`t|v#zO@sz?f$hcbV>QE
z<`66WSoO@OoIg(XQ9qMbE?km+<K^-F0_vw8*jz7ne(B~pIp0&zz3q6ndqeexYU>Ac
zm|Q{{=5PswMSi=dztZULV^M`(`)3>Lauoi=6>FB{{dbOH+uD?+vDYP}{>tR0n806$
z7krIouALt6SK;r4_T4(++4)mW#oLG18_Z<g)SPr@_cWicv2~g)|ISX(6Y;1QZgPDt
zd1}_L_g=sL-+sU^rEyv!cj~4$9s5-`{6ER2SGv$AXXW|u+uE=GTb=n|F72@9OV(1W
z@S{`mT34&asGFV$^39*UBJ8ZruiaG&Tb|g;FNqIPS=DdoaY%MX!=+0{&el9k+qdB6
z$;tOy`DdtA)}HpSw{Bnc=BUHdr4Ll!`f|<RV$zwA%pj$;@Q!rAJ6?_N$L{+bZkesO
zaYm$$PS_jH8+TN0LD3RnHS?2)jz*P2OBL<z-l+d)zy(gf^--TZKJ$17)8pDQ>kDk6
zgf9G1y9h;T8w-~I^j(C6xSpAJq9V;1_f=+r&=cWj=2=S{k_;zoeiJGJDe@8(vh_UH
zzZ374Szo*-Zcl{=OZNI_6>}e#_C2pzD>Z@n$-$3o$E9WRMZ~ln<z(^;_JH}575w|m
zE$=<4nEcqc?p_W{Wzn9KAJvX?N4)Kqd~@WhZ_NE8s@}^tm0Am(*z)r<7+n^4Ygf48
zCwJBspV|9;PAgoU`18oYXZn|)=?ni)c{JbW&tj>+%&C3TE#e+kWNxtFEP{rFW!yRz
z%i=wdu+abf`IPGfZg4>CIrK8*&th=d8~3oH_3_nig>YzL`#7{NZZBui(~7B&L;J2<
z%<ZZdd2;Nd)bZ>s&U=Nsx2R9$_#G!JbvfhfzQ`qKU-?!CJ(%zGC%B`Y>!W|+pW+=6
zOjkjnaqgqlTTj)jeWCn)<`(xJR5Z^w&+zB^Biyp4^~Z97|02ij+fO_0t*YZTV7m07
z`Tok%J(>0e{~K4`mRhlLNAw<R=6eSk%qQgYKAGKm?!&3JOT9tvod4v-b?(hQlA~Za
zJ5zsaNWdz-MU!XnmO1erv*%4oH=ZodYM4BwC(y|4q=f3@V51oEqGKA8i&J_g878uB
zUcojs>W~KOVv(Lt*QL1sZ9iD_@~V95*?MJ<|8=}`dj8yRd;I?50sTPzyY+7a>b6&F
zE}ZiHSl#CTd!IdDpZUP#<o?}$@@wka#MJc5o?g4~a?SqtcmDXBc?hri{F1A9_KBBS
zpTqN)vIRNcnrX}OW}0gJ#+hy_IF0YU4AbPaS-Z(xth;$>T1V6^{zjFhQ?h?uNODfS
zD^)t>)cQ16VWX5O|6+KR9RC|%ZJQMP>s+<-+zv6eJ*-wW6$@^Exp~0&Y(-V9lf6jj
zZTHqMg6{XG72J`?am)B+b*L^sD}MG1@$M76!f)71c5K`(+n^$6b;eWg_a;Am%|O+q
z+fSVk%BXmmdVIm@LyD%d9N%8<l{~|Er$_9@UE%rG|K}zzZ}?}u@6@dLNBnKSe3?C4
z|D??Qs*|$b+pj39WbU+;c2yd)7=8PLMQUqO1LJ?kCq6&+@SgGQb=Egr<+IaSmo{iL
zcyv8%{Ox{SVeZd;%$u`CY-^9VemT%C&yw@VGve2^{ahR7C#9#}Xt>TA{`LBvkSS7a
z=QaLbO7=AW*0y3}#Os}U(@y35I`lWvhv9Eryqaguo7E45lTCW3<Tb4l<y4JhX-;2P
zeZGB?^{o=;sbBj`A9oo}shy&v*Iu-^CHtUR_;h`P4!*3}jS3&n9+N#S9VPW~pTla|
z1&x<fGG2=vN!TOyY`(T+%Hdunj+}#PYb2L*X#e>*`O>mo4tp0nCN*zYd6eMZ5>vnZ
z>-WiXL}fQ#J$tYA_@X3%dAp=?7IMCO{%+~vw>Hm{d>8dvL^&#NZMvkiZ&z_?w8Xm4
ze+zc(Puv`9<~#RWlu^mA?%E!+-?@LDcgQ`sv3>X5{(qiU*?X_D%GRFUe49P9xu#9J
z^y{Z7Y;U9L<hQMj>3gx3%SuK@b@7e$Pk&75FO2)MGAvwwDyIN5%k3bGw<aGNH+}wU
zDiql)T*Cap?}irpfge&0KC|1NH*9Ike>@}3>dUzc(^&hBy_o%le`j(nUsZ27y>s#B
zzaOqWo8DTL=d-i3Mp4x!|I?<TkXs6CH?PW>elKaeQ<&YiO=dG<xP32Ih@LA{UX<k&
zR8=_X&s?tdM4s2%3OlPM8qXUy?B}sJ-K6e2#rlWTo>0qIFWQ#ON({{vXn6WX;EiC1
zUG28BrIN4Tt9Oe#><W}_xLy85hQUW)zgBsp<BVAw_tj`TTson3=L?4Z*G#(C>U-=(
zCoSjGpZc5G-7kq>eft{!T6UKybC(td9cbO2r+-Sesc!#E#kbpUZ}R!H=3Gq%+s2kg
z$GtD-P3d^7a`}SyJ^Rxq&p)=bToHRu+F^z5v{y^p*Ujc!&}8$SQK(zt3Im_@YmL^O
zk__uwU-H|2(mm6%(8F8&bwXk7|5!%x0}tN**AV$~!Bcb7#$PXP&3YB}?a_;`7d{<*
zb)o6+I%T;h54Z1~yvX`qqNu#%n?vfFET_bm@WidL?iQ%(n|IJEgsp{hY0vjk$Dsc+
zBo^jw39*#3+aW!vvf|hoN44k6nAsfGNd!J$zTxtRAQyL8oroarX6CbMRX#agjEBFL
zX0TrG?7seE<qN;^tqc}yYNl&!?U&|H*v2rw&~-11bzF&H-_CV8-qn-ikFkVJ*dNl|
zZv62nll}ZdrIX$juor*jl6u~m)@yrb#hf_JC{q)Sj0=%B_Xg?Tx>9~_#rc9qAEhcg
zo$u>KcW?4Pclpf+?XX`D!b8H^l&q_n7#3P_II2Z&F#G)4<ybnyzH_Tp)aO6kdn|J|
zM;c#BZV)HeKEs8Z1P`o!Kl!Mu)|@9U7qqT5$nH~-wW^K3D$mokMDK0k)hEf?T^IGk
zq?hmgxI9|tUcSyL&ND|U9>*0fcbs+g1;^j%xeia8`E}Fx2nM8|Q!BYTX}jQZ_Aiqx
zoYD+=g3C`hy>F8^Uh8)Kd3~E+kYS9w=fRmrnPNSj8ps%Mr@XrCI{#OgV`q@f#5Vb;
z*@3IAy|mc1AKy8*C{k|E&6xR;Toc?|RC_y^(++N&rRpN>;Hs3-Tl=8eZ3b&yfG6X=
z53ax8JbUy{DQML}O)vRYXW0n#=Zx|vx^BNuKfSE)`R>mXP6+TiuglTV+IG1u@P3nZ
za?r;#RfDMtcN(-e7d?926)v~)>E7aVT-BO2G0YXLw)^&J)SPv;mdMvS5*O9>W_i?;
z?9~D{0=2(wdzm&dcGa7x`w#xE*e@M@a+^-?Wj4tzWfE<k7608<{(k)U=EIDr$Uj{(
zKQesdSGrVjarw@bMN2-VrMfM>XXsyeWUK#G%ec&|*WKA|FI}uyE~>Ksd&s^|2j!04
zzxHTGjFHUwf_Yn~3i19b*l^ct&!W<8cKoM4IdeZv3vBr=7nb=Uk+~#Fs_&nQ`Obyq
zx|5qL%;w+Q7<Fl>R@T%MSL2N7OU`ww-qlXkaSw9&H+R-0&yQasq$1B;eW|m&t6X!v
ziT``f_wPBkUTV2?<_WW0sINEY(s-@|@}4KO4y`O};nQvN6e_bgk$+Yw-~`u+;HDp5
z9vm#uM)A9OUN*&aHhM3AqAR{@`_2t#7F?gnn;7*hx5DeW+GFXaqf_=hk+Nz|v9evJ
z`rc^8gl9h&3-3Lbb2LGAsr1s0E9H}F-l>Gxd^>nMe2Ygd`y=P&a(^;QxkI&OcW=11
zKSz)^M{dThE22vub?*@=ov!j?=exRHA%DMVH{X-8&pE)*BF*+I=EMc-%e(%ERjEAv
z;2K}A^t6D#udXVm{hwmSmWQR&7Fhi0ZC{bs(H8Vz&Wo47rfEp~?bKRfH^b?lMfks}
z^>v}LQEZQ`j^A6eUbI~|ujqG?ZlmTr^Z3@87B|}0Yjd+#Z9dCr8|ulQBlj_4(W8sP
zUqnRWyk_?{yT6Wz?2z|6@%{dMlM8Ftet9gl@}FXb_{jxU3U>oOZTo9+$bymM{||-V
z^J*`|7ranEthH>t)VT^3KE9X7#LgYd?2Ovi&7m}DUCGW*>(WH-Ju(onX*SYXVHI81
zY~uTCO>Vv57h!icrUTtpQ|?`RwxWJD_XHlN?%!?`Zyn6O^wr_`JCp6}{BCqbF)IEq
zE-GlMoDuwEUGulIzl`4X#s!=@-Qm6?&5qUb_M6PZE5s%I+E?um{%OdN@F`7y<tE=#
z0ft+)Csp$q1peBhT%IGe@4?J<?H`!`y%OCTAJ1{+>d!Jq&4nSB5ov`})dcT;tJ<gh
z{9lI*3;&FSP1+g<x*V09cls`r@DT0wy!laoTm75;z4J>?uCaf0;hO*SO~2YX#iLLB
z5!-Z{bx-+{>Zj#=e{L$w3KKi+x+Kcz-Ug}Pdu!Srygw)OLvg9sY2Mb(BD=<JCiCyY
z8&{b-F~rp6cksK^DgU&Klv#3x`TPovuh9(p9J^DG9{k(P60l}kT*8jccekCI-X3XU
z6CB97>#j@Gv%IE9&sKl^e1E>FfQlg#*A0n=eM&XvtQVdAzg|*M`=KZ|!A|G)`|~k8
z7QOq=d|b$`@i>GvSy<K8`stj|_!;}eCtqG3#U(ORZj(atS-FM)?@yEFO#PPm(X3*J
zMjqFc31KzeX%ebjrtul~&U^Vyh%Py$S;XMccg*)*l;#sD<yQ}uaGZ=-9)0MZtegGz
zPf>mr2mYmReelw4VfIV^oO@pt4pw9-PFWVuA=h`}+w+G)l@(u)+`Y21)Jkl7bHnDp
zT1PZlSFf+IJr-$I`i$*(AoCm9rgN5NtgG7{FI=B?I%MMWhw+gg_qLvQ?EbU!O86E3
zC$kQD&Ptzh`83-cN52dUOMRJr`Me8v-f#JH`A+R5tDfz1|Nc(M3vv*>V97J@Xs7F;
zW!+1<gSLNIF#qKD^I8lqmP`9t?OgZyz4phgUgqh4mrJ_`sCei*mx^-ixn*9vV7GXr
z{;S5Lr{35pTQ9sX&=c{Mc?WZOUeO+gu!guf|9tm+Hj0#+cuM~C72bCt=KhE0Of?QT
z+s*deE1l=vGKI$dvI|PXH(q_<Isdse*G8ez_+v78)uPWowA--Xo^AQA^ZI3{({f*L
z)bPLUYw<qn717LhQK?svEwga;9KlLHnYc^anmO_oZ2BK%@u7d+)E+aYqBrO33@&Xk
zn<AstmvLV=x6?~yVZnA5mPNk{Y{hT1aT}VQo6Gp`=z)FCzlyYLO@aa$y)Nfw@t<le
z<GhvsVghSJdhqcHExS&u%l$RDzUljMbN}VtJgUMQHsv&QMg0A2_AVyfPhgS=`-I~~
z@%%A`hxY{-E_qng?k*pGh4;$Cs$exY`TH3i$C>rnU!8ZTUVkp)LgF2sX8G@;k18i#
zb8BY!Ya6gCM@Q81&S{6jlc|!=dj8ztoc;J?$!*qJo51qLExWp37^~VG)2NL9c>P+@
z9b;xGu5WkFKdkxZY`p%9?eoAdOB0*6KDW-hYBXcE#<UOHW_zo3InC`0(O`J^@Wz?N
ze==Skd);F5DqrHJ|84iM#NOAnFHSSx_3`s_R#oh-I(S;sSXRYH)?Pnm|E;dc_BZVn
zxqtjmhzYJewz_Cjk>32WJ+(ZK%fr8Y=(dQO(Xd~`Vaff}67gd3$-7Q$$+(fp7TIpS
zd-1Gk`;Q)tef`h6qBM<f`g111Um6LncYU^=nzTr$<k%9CbvO2{R6kpn?49&tTVMGM
zqYq4{rkY=LpYSEs&vjWyL0i;0m$;j6YFa}ta($a^al2bu`A7Nm+nVeh{>Pp?(2%{?
zWfSsDWZ#5|ht8jm&v;(u7OpbU|KF64nfJpcTX<icxuVuk;Zf`pkBkQ?yleh4oRe5C
z9b<p}M$652Ejx-XJSv%FHrqVv8-v2VD8JVhi-jLKJ>Getn#udeQ|Tz#kKB5777Nd=
z$;pUZz$%@?x%AGHl(zGEiT@6XJhoQzp3ykh@y5rC5gs>xF8}!X;~{&|6+#O7+g*-3
z`be!gt{uMXW~D+P$E&HU^#kU!ZVo!?_Unc>`<wT37Hziln3v?@J^QQJy@dOT2jpuP
zR@*LKrh8`cp-CP;{A;|LrpT-iQf8g!aQoTT*8F|zd?E@@{ytN&cyT$OotoCdzqWt6
zUWq=hig|u2t6bscl)IAQ?VdH~lBN`?`yc&cm@a*ir{mamk?-cx?~3n#dv&UR%2~-8
z=>oT-GmR7r<#vVtVp;M1Nn>S`$Pr!^{lzX@)o<P0wf}B#o{3NHoov=v-ZEA>X@QF;
zwyc|UZo)IRUSoxor_H69JpZ=%ck{4IxZPp+l%XekK+~h`CU4nUjfxnxe-{*+ALbg|
z%}u=i_L+zl_dMQMk(%@R!fP`of88Ek-fz09-EPsx{Kr!qubDm;xm)p4z4(86nVrzV
zKf$vls#R|ohu=8)?qc-Cg<hNsUN5`4DmpDKz}A21M5~!%?=Nk6P-{M6>&b@9Yg&4z
zB`rMfxvN%vvbIf^TYr11`=T3JjH?c@X)2}v{Pe*y(eB5|vU5uoPZy0TyDiLm-kbaD
z;$ADS(%%=hHBaH3#97I*>(nFP8S4Uk7g(w`&fU-{&9Hru$TQ~n;@s;qxBF%^=oYjx
zuTEI{_r>20JCg*Srfz?w6ldbr7so#D#?kpRe;+UT&TshLl>N}1kAW6dXA~CqU%dSA
zd9s*!*wv&**>Ad~LcedA{ci3;-a}<RD|q%w$<6q3%z4u6T%%RVU3ad|m#K|;F+=Rq
z!EN61OLuha@t6?f9_?$h&!W>@&e)hCp0nXkWA@H(J6`T&Ji*_o>~dwrrl*hGzI~f_
zdfM6wo$EZlwqI<&l-{oi7j;sdZ*VH~Gr!1byIXr5#dwXmY#P1n_V?`0U|puG<<;A-
zo*=b8oOwdP)fb|5EYX`E@g@FBwRib&y*=b@|A(o$zgK*054psacc7FtPbBL{j9bo|
zylD$J);3t>>?kr0y8l$|j9itY_GaIX<DV|xy51mhKxdVxUX<*fxCJwv8V$dDY_ZUj
z<TW~b`m@$W#aGK-maE*_s`SN#Bj#Q8?%FnU=M``NOns5^DRqS*$F>g7kX)HJSM5LN
ziBEN2e`n^kmVd9GUSDaqYDLIoF-<c&{oAW^-k18ZaZF(DXaCJ{SJmcy!})@^y)v~~
zmjlksn7Lxd`su+Mx4M!KfBP5yaZk7A!(a`r|7(8zIC$$&lGDXA;Y%{a{;_`WSo`M9
ztCRY57Rxj4M(putZ;E9(5tn39tNljfs?(9llcf{0|Ll6UeZrAbDho2qZ|c?8ALyUq
z@%Hyuj#Q6|Eh}y(tcf}H`Nf9YK37AU^F^+`%KQ06@$)>dGRKu-rE<&XKX%T}Og#8}
z|Cx$)z8Tw7ZZF*8kZpTfrH_kg*_5{Fr;dIT6wYLjH*R{hddEW5<)5RAK5bI%P}s8f
zTH1m~M{lP5?pZFoG9y3t#`ZY5Z%H!)wkF?Ccl#PF9JNwX-uG~K6xT8lSt*Id8}q-v
z>8;m#Ud~u%<KlYq*ml)LUGF9zzu(bj{&27AoyNXT+KMmj_|LA`yY$e~z3ykS|NaZU
znAp3<ZF$O0=bpJnG47i?d(9169GAO1a^nnYYkn5u7sb!Nd%4b$^|?EX%6n$*R<>3v
zh*@!{z;#N7K<DdKLY~VT);}(do|u!gs_f3$nUyo6SXExFEBw_mzgnQmIlTIbsnfJ=
zO-&cRs=RYqw?6L9=JhM8Ca*iVee#a_tXI?b?U7tBawKY>XYs>h|C3%<_?^0T*{FWr
zQJ?c(erxtdww%)6%<?pLUJEDJ{y9It=>`4kexvlY`PHIYPO+@4A5xDVWVqUxcxbM?
zyW`!7yZ^sv<lkDRyDU`roY1ZvQgM4Cbyy~NEl#_4I*9LXAoKUiuMbKY-WALK)m&?7
z6>F-p@5$F5v8wuA?(F96+P9|XF2DMDw|%8;VC$rq$+pa%eeF|SU*sk58N6hzId2fV
zV!<)P)>}_z&Dvr7^}9>w;-xYSe{L@OFD|@d{+~~gzyBSM5YM@1d341C)`<y$l1HK|
zW<>1ymcBk)-7!75;&slo-W}7KO6)l@zy3WI9`@d)WBU{>`wMsXGFGi!CHD9`U+uSS
z-9t%k=J!-4#J6wAHMtabYt|FDys%TR)K!k3d?0a-wYBlu4d3P~Ke&~I!#N8QbKZ#U
zxql-x<M<<mp2LrB_uQTt|F?_tf^+l)u9Nce6L!XJIl;=lJb!HnZ-}6>@Tr!*_qBRg
zj@p$@S6c4Z_|;YO>(gsWoGQDU+*1#|cw^cl+wji1Xa3CPA9n6HILC0ZMPYGVaGsdr
z%Cl1#m5V++JbxpLJ1dc=?p)vV9Ld*wN1jMe(M~_L>++3zD_WKZeTy@UdBwti>4}U-
zAlD=At2~pvJr`{f*l}j%$|(!3Z}ERvbY4mDyOH7lIa{);XD+(>j_F17N3}C2ls+5l
zoVVy&X!h$~8jJ9WtoQt@Bx2QrKS$?%`Wou7JNLJ;^G-&OpPxJ4@CD6!z1zNK-yxA@
zCN=&_0gr7{Jr3>rvZnd%e1mV_#jbLGTCUH$<LT9<bGFaCvuo<DZ=$NZ-}xOqS-ioJ
zb&GJW`jSh^HhELRf+pA;-eItySM2Yt#D<v*jh;Q(T{1&fFT<{h<B#?frngLYFYtZ)
zXB_hW^M#)pC-&H@eHWc`vwElE`DC8mj~<q8nWAa4cFMYIu`*KU-v~PWntC<e`6#ET
zztXQS7DXE0c!Rv`R-SkDy(#rSvFTpY6MoO>kM3loUrcQ)+nABcvn))1O~Nmw>-@Qa
zH<KQ0J#^~AVg1F|SKsL0=cM^+dv>HsTW8UsQ(jxQt!y>g_qvCPYvRJ08hnZqd^p1Q
z{wed}I%uusm4AOVSG|eowwEbB+y6vQU#{?Lu^8)$X}p1lU)<<@e3j$R>FgUyHUD=A
zPFc79x&F4ExR!M)@kjr5-CuCeFmkEn{$F;v^XHy^AkXJ=^yjmD%_%2Nn|a%_Jh-@C
z@$B4;3!CrGi`b(PxyR?slE!PM`d4_=@?XYWdhOTmzwFp`RW7Ml-iMp~MY0sF4o>xD
zjE||$e^K6hbyL-z@0TA>F8_Y!!l{#-ZSy7GK2@LlI{8#a?Kk#|T3>G~o{Kr8qhs=K
z@3AkJbvZ8@T>G`WVaZpM#aErfE2rl-^tbb_xjc7oRo9yJb;n;myZ5-|>d%5Bic58S
zGGz|hd_8{X?`yNeZL6<;dJ^Hx+5ftomrrPlnV6`~sr73=PQ7!prT?<%I`(&yO2uEF
zeaN+3;L4I6^PB_Y19@!>gbumKe3;kaW*p^seXHL_WiL_r#xTbm4tBQw#d8k!-d*;`
z)pD`-v<GJcG`5)tPqjYw>-Rkc&-||PQjQ|tpFPJ~4(Ii$JbK!A^XuK|&-=}0ylGE5
z%d)*=(@&jsd&Q@SUAB1HlypMmSVK31QuFJQmA9RE<3;z0ZaVw^@V~+$?wkv+lOC4b
zkMV2?oR~J{Nw4;tU&pTB`>CTf<InGF;(N?`!Xiqp+pXHpZzG+vEhGG0>V6q6ojJT*
z4?kaFT77<rSE%+w-p3m@6}$b2+3I|&A--*)_OcxbV%#nJBvMM9uVqi%&;LZTNqN_n
zMFJZF&UnPh<p+4pH1ACEd*jq97sOrsBfPpVSa@OdV-3Yu7P$uRC8rzf!~_%@+&F5x
z^XfA1RiUpUJl#ajot)-%dKLfu2h5K;s*B8h#II%^H<)@p@fFwpnjB+6n~QZvf;2Cm
z*;aMt)Q>m1v)@&PuiJM0;!mr}OGeesti6rPxRfUHaKDL)*x8zuX7Dn!{K8+3SBu%2
zU+)rT{-w+KJmctg!})gH1~oPH4By&sEZbPgDe9E8bEBP3lAO@#!*}|)zaI+L(Vypb
zKYB^9UwM>w?!}8@PJa8V=OsOpu#{?C;@5eDb&b+lh5u*PaoT!xKWt_A-ENi|Go>Wf
zBJh&!etWIUxyP7XCcpiWnmKpD;f>o(9xF`!o8r|H{34xY=Epl~wR!hv6*o-SENoSG
z_8c$6XT57@b>8ht+4E1k`Q`SVg#ty}eEq6+oXaVR$#+)HTv)dG<=<y47iS&)y<M*A
z+NTe*vkVH}zj(OsV0n9CPr$6NJ>TAPDf$QVe_3By-&40vPx^4O#v9>s7R{oEcXsH^
ze{*q%`0O`U@ji>pDlH2oOlr0s>|5|g;ak5+j&j;YXI9lSQ#Z5RzVqr(=bOl>TUgxg
zEHTu&I%nq9DQkCq%x|5R7^eR5X!*gtEK6FWZyH@NI;^C;XXD-uquGZp?R>K-de+*9
zrr**yr!3jFa5J00U9R|BLUU%%|9o0wk$m;f1@`Mtq^;`-a^jri-+XjUw$y6PZC?U*
zpZr*25Lr;%#{BekNm}ouyC#MOwTxWKw^gU;o)zMX5#{kXG(p4RtjIN$O>O&wt}ODM
z@HQjiY*BGrTe6~mZ{w{C3NAKRr%x9+UF%vguUzGn{c+C8nZ5rzlj5Fb&*9<jRFgey
zHf5dcqMl3bf8vrS99U$sJ$y1tWQ&;KJ7!ap7e2-Hh8~&5y6(*k4m(TB7`0jrF0L|l
z7hdwld`p+p)g?Zu3r!xLnKC<#D_eQ*KNp9%kaxE%7SG5qoTa(#s>~CslKb)dww?d|
zP$}r+o8>M0cU(Eta<HKK>Z31<W`xA5HPjxTIPbZW<jmTXn5CEIoV%P9xlmKj$Ww={
za^1ciEEjBJ<~2>(;JADJe$5lfA9%jsD^so)t#hadT72XT)7JQ7pPCo1H%RNUlR6Z?
z?R#&9^4*3l4EDTsrzStG)46l_)|84%L5ycFy)Fyx(SBhuM^CR{qn3(Awp;3k=Ycj7
zr7|X$R$W(WxO`VeXa4lLcIRDp+!o1{*Xn6|U*s}Rk0E)+#T0Kdan=5RZaaFrIAlw?
zr@xK7c4<|+&_$oL#r*oyc2D5%Z~GRry7p{h<JExcHGBEb?+%$bZRJ0K{EF~~6Stbb
z>T{~{{!()-IN2JLCSa!%m?7S@Dtv>U?4spQI$vyPye3n?7NNELxWu<LHSJ>TTDNcQ
z`Fzy=qk-2fKCT)E)$YK=sZx{HzRG7lvNvE0i`mjWd8(=>Z7+5#=Cl=Ca+&>Q$%?p`
zS26l_%8uOG?CE!H7DO$1(6Kh)V&oskhRpcrJL|)}R<hmdNV#yJ_>wN`diA?5N9TUu
zGHGx5mC6<VH`tsD7i|?@8Y;d!W?Pm^_<G}AwoeKkzRl^3{C{|z-I{rizDaoH{WAQ0
zHQs<p*6~}n@P!TGmv{Q^eKViSdmUG>YQ*6gyPV8fcWyeE`9+3buGnLFU)ZU2Q@U<O
z-QPAztK0qEHQSd0_8$-0tXEk2`rY*2Z$`UCP6YkAd-vWUw)*N9+^5cDAKmEY;%U#Z
zfX_$NCMx^MXE_CClh>O<rvK8uprPn#;rIRR#O^P%IzQd1sq*vfz5L?XN#3faZ^vY|
zKYu@wq4%ES;<&4|y<0wsD#w3YknQ&D<@6&BF;xyX4Pqo$&lavI7k-;8KYL?L<BY}E
zjc=uDi7opWUF*tmaV@XS?xzw3hmtlfbBN%%zEGNP!qVU}bzK>5C7<vWu~Yb5^(G`c
zYP|i_?Yq)Jt@mBPi~bn}NpAmBa~RY9`|9_9{VHO>(J1lf_sW)!6UDw;EZG0;&h2Fr
z+tU>v=c(mRdh}qT)<W0g^LTS#-|JtQCHV5^DRriOVy7NXeE!lY&*m#zMQY^lA6?fk
z-l)mF(!>#^%pH7L%=PLLuJoX*rg!Sw?f!kQRWiF$tMr7AXVQ}R6>WU{ELX*JC$>M>
zb&`L@Vd*U62c_k0+c?fYDd)excFRMbmYxS2ntwmv`TOmYe@XmbU%Nbb!u|P{bIJtn
z)Bl#NJ2)@#&cy0@-rv+~cW;omV&L8Myit06{ER2>4myenNUoBTbWyVrW?OQVOR;_5
zeRIG44TpC{b^q1gyfUIlZ-2k3Xx{o|e%H2zyuNZ_y^VCu!>OvV(k%?jq8#n?&Lqev
zAKU5uH0JZnV`n%ROe2nkU-<Gk`m1oLg6ONs{c|+)%Bvlx{}7RWKjGNvL-Vqa^(-)Z
zU6aWBZdLy)w*FMfPa&rNST-3I1}5t*ntD>&%Vg0F%RKd9!5Id!MjLPTUW|D<u_ANX
zW#>Nz^?fBB4kp!oA8UW}`QO)F8xTCVpyh0x{lo7EZt%_A)O?UbY;kbzkr$@Q7GJ*#
zPkR@8^bn(x)k|LfV&Qp?DYi{7YdH$8G)`$hF1Tqse{Cvn!ln$Luek>H>{o(2-cBx$
zebHgJF5wsxgSquTNAZtbLi-#(OCFt<+oJW?=gDLL;*1?CRdc__<StDsbzCH-xNy0a
z$o#1{-aLIjYwB^^ZSxLm@AqhBJe8XxYI@?G^7$qFnhW$rW`=*7{Z%bH$|;%aYV)~}
zl_yrbT=28UZt;%&LFEC>5B}v=JP@<8uPI;PrYZPdPV7+GqlX(`Y&ZWDH*rse&a>}w
z?6<AYu8`3w^FEv&c<iFmmk+bQdSC6;Tp}?ijr+)}<62XE1;k%!>xgk(%wE{D=fZ=X
z0cT`=zix5*>6*L!#{W3ImKC!fPUjMN9h9iGEGlmH^lN3^4z+U*a#_mNzFEPvvR7<z
zQ$yQKy$Sb%ggAtSzUsI#KV-M{c093Y8dLC_{kkW-U;4>!I}s=@UcO)F*u3u7s`t~*
zB$W$?=CjP$EUIdA`sGB{<JY!-Szl{e>|-2fKK0sxY0?g-H1_DMnC$-Nt7}k-hn(!8
zD(-b}98XMXtbVCKMJMKur2Yik{NMZL_&!@H>pUyxVEUu^b>WVjo(4=@pKktr)W}t_
zNrUg)^F_9oLn~5!gPuK{YUFa0ZS(J}bFaNue)!pY>OeNnY8xBhl<Me>x~qB_pXeA{
z9ePqG=6Pl|qeD4w&#C#Zy?d8Wd44u2B<9q+8!Cd2RbwmuzdWXzDjECaYovSTqA%WY
z-rKrnw^X>PWUoB$;H9gfkbY_7-M+I6f4@s-RMD8tbC|JQ;DvgT)zuYS%cJEr>)cY0
zK4truyUfR2%Z~fcr!BVn_gyoCj6`OMS%@7qwV8OM{;{OUq0a4TM>pK&dQkVkH0W@z
zXj{wS&Y3YURWDR_MkvTWYdrO~=c3}P4c><qd;M<wCQ)7WCbY50GCEPoZ$^whBh#&u
zdu~^)YMozja51;x*5ruO;VYO;jL)qVF}d_sm#=X9Y@-E-gl%#%CZ4?TzQ|2FyZeN$
z=V{5g2YXW`)86a7ep{Z|v+@4si$?s@RF^nzzIo2{)c*4~1SfVJEy{mhE@fA)wSt?u
z&hgPTeTm&AKkVhB+SY{hJ8L~?sJeTj*Kcjir5~^Etr67b-M3rqBv0JwPPdqooF2Mn
zV!9JowQF4pa^uv{`SvAy|GFb4(I5Rz_Ph)~CvnrUUVM}M7MG%*_t)rcIr}%o%5v4F
zK7m^XQ2|fq#vh2S%=zjjeSAvshOdo2p9&QBm@n8Gpv;yn9`F3ivVi?D&%7yWGd0Y8
zTm(Ku2Z}abVsurSQ0CZhMd)x_(}rA?2I;w<qyhuDr+oE(z4F)EqfSh_ly#@IuFG+#
zUuu0XYu|;gkVQ3~e3B~r<ntJkdkba$v$&_}ew|ttbjMA&Yhz~6E_Th20<%sVx-Iy!
zEzUnpX4T_=6O5k*b>3g6a(><8tJ5d)h6ZoTC^R+tz#^ZVDi?Cb-twAI%#t;~`n30_
z_W5-$4Nz&F<|LpND<e>@?6FyK%|Fe&-G1^*TX_=MRF<7MTHMaEMtu3*fFOqufy$jX
zpU;ah&AazTjp4U^#=lQN{;a93t0gT|Q+bcBb?^I@fBEr0*^7)}{N=h2g-YF4Pnh{C
zuhMDa+2^}-rGC{}N_m`dHor4JcAM<ojdtcarBjbw@_6{cx~lNQ{!JI(ctz`ro!P6G
z)4{a#(t#%rS1c*|@bQ3?u9_{!!ezzp<ZjG<r{TD;V0ZVQMH`L&_tdn7Ondievj6mj
z$8|ryc_q2*ciP01#-nXnfz@7H&sbaC`Y0nQ;S#$qa9yR*(Ve-%D|fFg*>UPx?B^|C
zQuE$@`pa4P>j~%Y5`!ZzAFgtUu{qGeKf`kI-vgU9p0iu(^tL%Z*(tl_i}S5K;eA2>
zr2QK5c=IOR_G(|9$=0+d#p3Ski){;wmR>BWd>-`qi}>$JqDOuz99}au`#{uoF2hWp
zEWM`dN<K!-l4*`J+&woo?%wh#>h7getUr!8d=T%PA|X_8^LpNdg-c_V-!y&R?O<7?
zoV7*l{a?}jE}P$PeZA@I%48Lp&7I5bJ=T1^(OR9+$$kCYg!ZhX7Uf5G?h?EiWx6qX
zkF!anI;-bmo0G4a>`t`on6@`hYO!`3o6F-f$3&*Ax%XBy-Cpa-+lvpQWHQB6Ro?c@
zd;IJ6)@!B0>NO8K<jc?2vi)~eQQRqc>EFKd56--(sz});>X5f;v3JGki;)*9cKaq6
z^LCz@@cf#iK)CGxdk>b}eEe|ZPpOl8rH^TT<~jA{=){2ezI&t+des>J8u3MbEaz8<
zsSuCYEnM;6jN5ee)IFYFJL1){9kb4shcL@d-NrgIes!YUm!mtK{I@iA#T(@&u72>_
z_n_}$#g$LP`NPVJcD-7l$E&g8_QB{ipQl&lt($(i{a%Lr%K*I@=^laUs+CMz!jmVv
zEn~9oR^R2mV@deu32`r{R63Sk&z)PIW9GQhsC@SF1*Qv}*e3^{;V*Z#KR?GLHg>UX
zp-cydTFZR72A{_#f3>|Yoxd)AMb<w}o}~G6r>wO7U}|okBRFkGV3$*eXCs$I<FpsN
zO5OMVlz9d$@NoH(@p}2)k||Qvk;Z41ycK$r8z}PP_OgBN&Tfp+I$iWN_KDsyNxh}h
z#b&x5+N84N;sH}X-LqSmU4wGG8NM0*wm<6r>!_l<&riEabEOrxCe`$tR4lq<6et~d
z!es6_?pM{vukP|uU{SwxGe*Hu{pU_c$*VFAm*(g#xqVcxOd)3~-%XD6;})W-?>2gV
zZk5?4d@oAsf%8PYW%CQJ-z_osT6K>1m?Ja))TvrF?`@a7(sg(lT%vHOuxQDo8Rv4>
zJlyzY;;pMsA0_RLDd_E$2xF0Lsa<URVRH8C`3tyY`L_IC;H2d2kXw`<p{A)Hpr9z@
zxWJuzQWHn<)$NZGCCrWUe|LQQvLZoWYQIxE<9?oB0{Roqudr-b!M%4?!4`I={*8P)
ztn<xQ-kdu37t?<2h)qWp7|-1F;9{)qy5dz+y!5&i62A78ZPZ-2wL-&6ZQ82K$J8aX
zGZz1@e*1N={G)fT(u4YLY0h4nsDG_;!pp7;=XY%=`s%#Nbn&eF?X^EmLhP!$qxiTK
z;{L4K$83ISl7>f7?NuoW&39bMzG<yR$zkU8+LF?%4!dNmifj_)Kd<;`<E5?FnKmd$
z<R|vs>x<Jn;5bod-AW(sMfq8|Hw-umkKJc~lk!dB<$@B;J6~H^j&t$J3UEy+Qr+Ov
zzB%3X=GO=3CU(YM`Zclc;S^~$b;oDS0^<7?+}R@7G{@pZf6NlD1M4rB&TLiqm@Sqc
zu5#Su#V51y-*;S_T5C#j1WOqA^9prLlN0HRJGbMdOVY)|CeF_`-k3P;_E!B+PKhn~
zI)`@bJCt$B{f9_G>@tOwGizOL1V=Bu`(B4x?yuDSEo)17XSG{&&;1)-Bck@OKz^C^
zx<X%>jSGxw>Xv<~WSfx3bKtzsL*}WiL5hO4g}==By{#r*3F$mFtETnt<6545b005J
zUGDDcF?Hro)q8W=&36Ahc0jsAW2H}F>+41D49j9U84vzSt!n!&|1B_YHCL2&oSaVl
z@&`Y6c*XNwE_!YL{@FAw`Q_J3_8wyTcv+%iQ^3#om4D7Rd@SG>k*qo&``EG~N<isw
z$C<ZWckXzX6zhv_-L*thF)+4awW_GA62~MiDO+ov4;9~9PGotA%~?3}f$P^QDfWj4
zIdrWnm!InS5$l?&8+S+8U1aaot#PSyOLR}?-7Y<{H8e2EJ#&GGt-QYZyXU&cP9Jj<
zWAwV3x%-yq*=usg)>{Za-*YfA_p<+{i2YOD76^V>_vr>}l#=EJkA+)&k{0HEzqQEV
zvc6ME+>DvK%v2V+{qneG;a^m;xG~XC&g*-fb8N?%)vv!Sjno!MsOyNgb=!8PAc?am
zj=M=wh|g=bP^yvQb>*oEyk4D57uoljUq0sa;zJZ;{gjROkMX{oZ2Dy0Y>!nB_AX>e
zK5}IFdTX_u$W^Q6Oy8?opYF(fCGY2h&)my+<UYD}2(Eov@G!N1+scXV0e>eO-e_--
zyXmgqAGc_!&%@2^w^j?3*|@&=u6SjwT9$p%{nqnl)$-quhzI&S7npF{Ka6ijyRS!E
zaAWz+B_Xy_rYl~|*^#vFfMTH=Yr0CZpws-m<WA)^`Z3k4rkvgvzH;-O6?eK4BY8<A
zGvDs397E&tIUSDk^9r@={(jfDRJAphUUGrg=*wYde-m}Sb!<oPobCAe-Iul7ExGB_
zf}I*QqF+udZt%)6yVW_N=6jk;X?c2&-X(7L7rY-*6aM+kb<C~U`hN0)tI5A*jmzTZ
zxp+H1379c2<nWTj{b_w&L7FcF!$fb~$PLsC@sZmqBDwzgF2@Liyn`ZB`X>3`bL{PH
zohhjDH#6pA_#5*Y<q}KbES24-)yw`Y5qeX(oME5jk`sFB-@DljQ}|>%n11rSJ+@}e
zliDeaa!0I;bQ&Zt)CF9$SDfP4x*+Wp+aG6H-*4GB<7cNn<>C5Qk$rDHXHJWCj%;1T
ztDFfMdzQC7T)ORK;;I#&KPsEl2YmVeJRnfZ!F6wI<wl9czvk@faGY6_A~<<h)t(eD
z<%g@@Uhydlf1#IhV$t!pA-dn5?G8J*$iOmK+hAeR2~mcl2hQF~u2{I><^zL=&Ue3v
zw8eBPHd}v+%i3<v<{wwNW7<;l`}0@+uv-4;&~%scyifn~>5G;97i3#7cb<;2Xr8m_
z?W5HpOV50Gm{INWeP(^8x9I2DF-E6U`Il=iJrrc(>Th^O_nI&lKg*RX3-|((r}Rt@
zyrt8|5M1<f+uiS-j5Y2)^UEdugBT+oKG<0xzyC&VZq4S8DakxDBWJPX^_IOl$hF1U
zPryXw_#|y@afzCq5~XdLOy5*{rCTDsStib%9=juc@y0l*19gWNTWI>cN$AY7ZcuaU
z(o(e8uEf}6w!^TIQ~Tsq-vIYB8*CT2MEA_O)Nz0ALb=m%<vP;ci(8jIJ;eH@>A*_a
zV>XX(SO1DXxKHACRoQv5`_VUfFI}*RC}s=KPPt&(C%#K7oQtj6f0l^YgO>J_ZO2V*
zgtv11y~H!0f8ob7d@m+VQByYERJO_Gy41UOYuA62c57I$!7Tdn;fl*LCU*}mx%c+X
z<CweqYA#qmF}=9S$*afijh)=?%|CM{@|wvv#WWofe|2={v8IcP++HEPmM85VFs?cs
z&RHLkeeQ<rvx%#J^*xw6H{v%pC%fRTJ;CNL-NW^jCY@T6`n*J$>21?`-xmo2tk-=l
zr`@~B(@^5PeDl$$+lE_K%*ksI_+`C7D%X!aefx6W`B_HoD}&;kuK&=BF8Oxj=7W8l
zOWX@{1C0FMSgoCV`iJ|Kg+4Xn&kcEA);;!G#CYYOn2N9YH37l2-Apx{!N>0{+7Y9@
z?Z)2MJ*HC@urQys%jC3Q+$^x*SL&*J*ZaP_6aSyk<*boC@f!o%JavbZcX4I96EEG9
z;{Sc_!I45k&!T|ZbEn>|__^qp7E4+4_nK#}Ki5Vr)#GQ(p1xs^XkD#ap~$DtiJOEL
zF5r(}?4+i6WOem*UCY=Hh9So1Pd@Th(ARQQf9T5*8P1t9;XqORv(h7>^Gh?&?AHI9
z)3e}gr1bMgoqYQX7!S;?aIlT;V?D<u+r4PNxm#${^{v(acQme6a=9!%?Ohtz()ky2
zy(jN7W-GWbCvoCjziG>|JVL8{{CmYSlrJs_&1U={b<H>`vE04n%%%roR~aU@|I954
zmT38taMa=cn#1KfyV{D5PdFjB{$_GdwtW4BUx}FunkqFIji=@__peme37it4KcA)2
z#r6JzO@6u)mPm+P{8Zh!m*HY(i(SHut1;KNbMf3Y7F{*%ocrn8>V-M`<EtK>^M4oO
z>{HiU|5xz%A4QL~`d2Tzd$_IJKQnn<TawR#0NZafm^E}Iq$W8|2q=DRd!)U>b!NbK
z%i1<homFjaH}wDNgzXDbvAZhQVX-|oH;;eO;<ZgdlYY6}*6Ao%HdTzT_`nyX-6cZv
zFKjT}++EGjx5p(d?%wPjpBH^xf33ZI#l(<HJ6DNw#x9G$$T>%8htK8A!VQPN22IFz
z>*l%Fd8gK!OZ#f=DZz~Cr`>LEU2$XS)-A3R6m;+DWbx;zEWUL)?Sk)v^_R}`-(9@T
z<J#dw>-{^=H$4*wk%@U&th0Po;L5A<(|d#7nf#S_r<><CW!oh2<TZB#zdo20vfNUp
zFjj8K)%Kl#&joO9t2UZhxM};V>s3d#C@t`<RquFY_A)lH(`b#v{EvUtV@i097mLo0
zUBWIWw{7c`#`OZ+p$1Pxl_qeh*&TC_R+GE7@~Y|7EBWtkwecuM>xpD5Ff|ngGe24+
z^22{qa`GAl8Sg1C{*+y0%KX<Av;28aNS4z&OYVZ0UrAn5DrYwJ6kOiiXzkT0HmNbK
zZCX^r+;%0+*Nm@w#Xe}|7;{Z(6Laag-n+YIqTIytEo-l@I=i=Z>!r-{%;wXp)1pIn
zcBy@RCn{r9v8OV~{PPE)8=`kxZshkKYL@IuQD83Y>k;u=r6T^ii0jKT1M$r-zTWgp
zVYxkdPKVr*9p{d?ux5Vem1Ul@iE9l%dtrBk=$Tv7F853BnX_h<kH6xIb&r%4Oc)LX
zbX{e0Ft6vBXueL^pRr8omZ!-3*LIh-O$hlv<-3q}$CdS*KNCbxhlCXIX{YLRGv92m
zT$r~>_x`Fye&%6c7jVD+IdO-o?gZX$+p8ryi(Gg8$+oE4sTT9L!{=)Fo|lXh^tgIe
zlInJxHat}7cww7`UX_Zf?!lGzKfc>v*Xa>p%U9R>%`W;wS%SMFt9jR)=I`^h9CYI@
zoZaGOF5Gh0&{1gOq0NsMxo`R7SJQm-wy%lJ@yXk+Zg<kU!#8`Ow9U;@_I}6nzDst`
z^|dZJaHZ<^yIt1Hr)f^jGjO~l>iukE`Tnx2`Ww^t->LKK7Tl}hXII0{x>RfHWW)J$
z_ayY?Uw)~R`&262{MoGZEmBb>i@n!qUYPx>L%;p}1+J_nZpO81);elNy_gajcVyKz
zH~HgoD_@BVYgsIJV(!`T;bKdm;N?jRx8-kq6x8_5y<@5B=UhhBn%w*^t$D?V*=xUb
zP5l2?)uGBuy>qkmR8z;=)LZ61PO0Azes#jz;N(QH+-sB8l?P3+6xe9!TI~9Nug2Fl
zujXm9R1cQur3YMj%9N$B*mh1ySi5|#Xole0h6VP{>$-L<HCTJDHE&n4&a18h@eK3T
zuWwt_8~wR-p=I^{mg*N(H=OhXFIeq;`0&V12A`j=Ted!a=2x=7F!8U*+1b||Q};Zp
z-FC@RX3?jm@^fOp+TA(id!l8L&&?H5yQi!dQJz2PYSE#Ktg}LwW^=1#)-E^RI_=S>
zs=y1O4*LBL%3t5cJ}#cv%NqYIDqC@)))RZyq$!6h|6Mt>&y6+Jz(Awb<GHBmL4{S*
zqp!AcDId|_&U#tyM+Hm6iAjEwR^49a`nA2k`$C7k&h!sVrJ1~+*L0mo&RQMo#5`$^
zR;Q$&w??gqQ|!yl@-D)yM>D!YpLea)bPTgg>@*C#GdtXsVf*9*ddpYJ9X)d|@R#-^
z#p$_Dx;vjf7K?rMF2mIS`n&2L#e-A+mI}x{$}40(wL$o#)?VQ^GZs%OGv#T}Um<5N
zJTc~HtMs4UtLl$4+-R@AT)Xa}aMHHQ{SCKWtJi$nrDNT_p=@P=UVKotD(~dR)6eJn
zD_72GFtAJw-a4&AytjKvd*2!6b15NF>^o<7W@rU3uQt$|zkQ({kK2U9XY7v1@Q9sH
zmOYqW-C5Q6#PE9QuKB-9Yga|(YjwSQ^U2P%)o;rLzfRY62HN_wY?gl#o+#KBYBXPM
z(@eJGlf`zL6rS^bAbq)Ifrs`3&dGOO&OI*cPM*x+kpJZK^l-;jjO^Yo&xGx^HT0d)
zf3)VUQ-V)!*<yB%ms3K1*?Gx#d|GawYJQ9<qWtRWIUA}oe{T?O7q^yr*uZy%wag=T
z!&gg5X>Rv2=k>D2TT3TIHZi7EtW8{V?7G1lg(b>DlgmXo7dGv=D3Nm0Z3ai3|4p@|
ziIeV~`;xi;&o{|mT%R0TO-&Zt9Lsr=@9{ftdXvO#GcyMZ-sbzpCw=;4i>AG@Q`~P`
z`+wEXHidg7Y?mXH*R9{VDrifn>qcjZr{A*fHpgV;x?R5QTRL&JUZP<&7o+1Pc1DQ@
z?$SkzdS$C(k}TeFi_G0n=`^Wi9lQ6hhwVFUi-Wv1L(2D^?W;=Eo$=_9q=}00-@1J<
zpBG-YRa0Xv@A&@c&X0MI@)ei^{BK;ho!gUlh{IN3cIK2z(|iA_^v`uLDq#t~#wL^4
z_I86O_nix7D<`uhJi3=;-u(HdA@9#;TKp>vgj5-Lryh&CCj4;zvzOB**gEdJB-yfc
z`CJ3fu0z{w|9$chm#8Z^`Tx8BgMQvQYDVw&&QtX2UYEC}``ec9qLa=_ElEAv%3t~3
zcJf*M+UjX)xr?-y@PGBb8GMH0mh9{5Xi0{(9t;<~-8t@Wa&5~xB7RVP@m$3NGImKD
zPLv%q+huUZ=Zep5rsYhrLdxAIOG3}>kn9O}zw${^e%rpQ=6g(zt})f)@?GD3G;pP!
zl~44SEk`&q@9uTdh-+5%UbSJ(>)s-xs<$PfO3V&9D|Ky`d;SfMj9mVIqwRlZX2tK{
zJANszUBhrR()&<$<;CO|Z$9d|mAyT%F!cqG%|6ae3p@*37JlZNdwW@$fUI(}jFe>d
z8pXYQSBiXW<_jEVee~}_SEKu%N1?N&EcW{-A8Ygw;ksURndwu4Tl~*Q9KsdMaZ`>p
z^|c+>znY%U={CcJ?R44Eb-!hPt_v}j@e6m?T|PPJteeB{*Fm8b2KNtI><c{?&fo3R
z{l4v@+l`V49$uz3=WiOF(LT37d-0tV@9A6rd$s<1FRRSTdX3GlB5|VIBS-&xiyqeT
z)qF2VnIXBasHfpQKTqI~567g}?75K~+M26(Iy&UI_#XMoIUOzAawk1b_}MG6Ca-XT
zEzgO5@2BRuPR(A0A0~BsSU+Y_IhXNc&da0BJ9n^!Tuc6%alP(F<uXyjB&mZhjKt;6
zFiy(ak=Ww<@JaMS=DNVd_$_nORo|W#JfrDlTfh0~2EMrrt+|Jio_V>4#V%hf@ZrO^
zs|*$@alOvF1V1)dE)%=Bt!-xF#)xz6hUavxrv6s@doa_u*zXtLpNY{|EcQ=s$lv^N
z@uAsFf4ExzuPUF<n_g}f@KRWBZF|`Czf%`vH2h24ntZKH$4pFo;i;UMnV!4y%yi!7
zJQr$qogb+D@!`IuhgHO4g9{y&xi5cl^PK9#sox~+lFm+db1K_q6!z@u9`9qBilQ4`
zGJpOsvCwezTo+=wBL1VXW#X2p50|SPE_k+n!=B9llUl;X?PqaHyS!Ceb>*1kp1?VW
zg++t2G7j(?PU*kUwDW*+tyEy&WwCuf-%hcZyWFy*;{y9}UhOk0xfGsl2(X{uGrjeC
z!?gTeeD#?tAJ?poiMuO0S=3m8!}Rc~C@sA;7uvsnOL{y-Ynjf`qaweWR-V`wlG(#i
zwu9BgNnn~#gz%RDu?t4LhqkFGvMMrueXSFB>FGzYt&CgCS*M!o{;;hG4*b~HEUvbA
zk#^6m!*ket6FHN9?(<i)+u?3u*`6lyoOAQ^((XlvnwG4yQtO`Tz+T@cA?0shS7c_r
zuXn+9j|R`p|Mt%P#G~zfBO%qqVs42?qJibcxUX#asu_iBA&ph4BDWqSmbKSMRXpU>
zy2fRC)WU{;W{AJu-y?6<TIs!bs&h*3SNq5RDe2|0j)!i{_2s(fy+S6?^toZfM>bte
zap6fE;=5nHsBdw6sdcI8+s;SQA>ZPX45~^zCZG587q~xRvihWj3nU&##5kI*_S#a>
zE3$aaG}DfLr<3pAi-mnw(}__y&+|&B@7o3|=iIU$#Zc$Qw1q2{%$v;S`(McB;#8Gu
zR|T0ev{Pb~(gZrxEdQT!^M4zY`#!iiYe7}tgQpT*U2+pYXgy}F658@?N~4Hz(ygn9
z7_DaS;?#PyG5f2w_Tn1_kEHj+HlA#H8t9qFy_}g<nqO*<zHCd9(iP4$o_A7H?L3)s
z7bZ#Q`iJ%BUdzZ}6n*<pF7Tx1!c;{IMJ?sETI%XKA3kKx>DW8n<a^}q9rb#A^?`+F
zGHTj2x3Rw2ci8;<%pbwI{0YAo&z0KlX!$T=F;m6&2_GA(UdHUn-YQv>vnuuQ*_SbY
zjKA*P8NR3Y^H*OsCzI@#bNs9xT6yR?6nuVqCaNx2EoLt742328-bSnB_T60~DY-v;
zi}e9zzngbC;%691912%a+2pP6EV%H&&7Y@U##}m+?fRs2(L?vWg>Q@{+Fp4s-SXw|
zvy0^~-9wcBFS=%aW?sZcH?H#!7nJPSd3o>Kdv9F+^hXC+FP-cE<BM~Adc`rJeP7Nh
z-hMh!eS&w`RH-a+cY{#>gR3>!H^1&kOse=1ALMp(YuTZ)-4jKZq*!r0cDnRXi=9K5
zkxNnATse$kv2Cv`$6htBGlx?}*~@O0{PTSidg|7hmt~Lb_@)@Ie*brByw9X9jsIGw
zH8_2lv1euMgV|i7(zn?kXWp&mb1nN`WN}dY?o4;xSK9tNruaPHIDgaM6{<6mPE2z=
zFWND8s|3%HdwEQ}mEw=@W-Vdty4iia+|9IdRyfbfH1`d6`}esetr9ptJxZs>h12W#
zt7)5OO)>9yQ}tw9;Km!pU!Jjj%{)|@lyK|I&q>ZX2ln`NH@lvCt`p;V;+8Sn(kYj}
z-rH%Fs=xHe@ou+_dlL?IU+H?bX5yy6m!8~jHBxro<lnI|Q775)Q0UW72NqT-#TFIk
zDJ4%{C)(IkSJ|c6`PA}ir`~1hk5&seTb*^<7<c;EoTqKk-k(<shVxnLd)dF7c3H^g
zp#J?I>_1EHHg0gNsw`b8d${e^M$c=tzf@0FPTa22rB}Tw_`$?5+YK6j?a%t0_<U?#
z!ksNOd!9yF1nmBirn-IlKD8ywVkbuku*OI~y4t_Q@22S{c2mbXKDDa_lW*zz&sq3T
z)4cZi?CY146E2=&GkS2+GBYP2Q1V<DceIE3U+?l2yLWB6p~3XJTO;e2B!A+cuBb-Q
z^c!v#%qgt*&dGX38}G7snt$WXg2PgsORe9$eZYR>K>W(>p=Ty)o(|u+WkX8Eb1O0X
z?xXe#4y9$fTxY+3Ot|Q7xt;0U1LqcT`E~C$SY>Qn=**j;bfI|13ck0F=MOyDen`ml
z*Vfs8&ivgs?Srq8;)Lr;?rY)}oA`K^J<j+&d#a&PqsGSvku&abJnY-HmHk&lPi0=t
zle)`NkG8FT7;#HgEMF|%iDB>I75R0Zmp2_MOUM-e&0=+t=T>3<2_E%Tn=K+@rt1D#
z_tNdU)$fPf7Ura-aj`Z{$o?dAiTwbp=gG`m;g|_CdpSe3^uleE1?^wdDf7lKaURnv
zS-3gKK<=!ww>0ZxwPhR5Serzc<j>VD)&JCaO;~Zk>fhp-)4M~rUiL4H|Gwd*)c2f>
zZ!&)mPSRZU>#fa#D^5@C8}z2NL?q=JWPUZM^>5v~s5)nX$<EvNJOjKeZBzeFV!M@Q
zAeT4yjqYmwTW`%*Z_!(L@a)pXmb;JFmoMIAt^3+D;7&&M|IO_kN6Oiii?*+r<~qM%
z&5X(YXIriZU)DYRcDKsqI=fTm`scq%*SkA+yl&Ec!8`xYYo>*h{rQ7d?_^{YG<)+W
z>3x^v3ijVc-@7OD_9Q%zySZ~75AVb91!i|Y#pngj-14ZYw<`55>kn;f=ene&za7s!
zIotbqowir_f(L>=o1NeDmi;l!I%A^8rXJL<bLnQV^kU{o_jF#YZ+(AciD<-M@f&&a
z9BX6G3yADVQc06ITDE3+cH+VxUGk!D4qc4bPR#vz^>s|8NuEZLNb1s_sJg9-x9>aQ
zwmtjl@Av1WY(BqY^l;L;(8pO?d3jS7>$m?G^MiLOElRCpE}0p*=i{*h7UI*jeXo>l
z(a}(yVsz^+qn(Za!)7)|<vj)64j(@3^KrfzGi!Bo^_t%@3+ukD{d-Mw?uDO$?+xzm
zwEiI5Qg>?$!yazqW79I_j~QF5Xnb4$n`iRNAEC;Z7`is7Zjp>X`ikY5MwDjU_nmB8
z#0+-WAJTYmsGy)}v1i1b4c;64EDz5s6|g9q_i%$Cv$>7zktY*wKRtY;eZReqpZJBV
ziCdFiNgcS}ZK4!C`@%N6h*yV`ZB~{Y-Y)gyj2lyUE9XnA=8M)jC#-DuS|_W2FInfM
zU$e1%)sYULu!pzb*z+DrKUin$f9BI))_x0bO`m&Gy(emqiEsE)z4*G#=ZY^+PMF2&
zWeaX>k~A`&=jyMgdvW@Q+PTJ--VPEou1e37?M+yA;51K;&gQDU4j0OoFZ*H`{)+4L
z>6OoZsUN7{eBR)%rH`UPAh&hlbcKKUGm4Vi4|&yE`PS69GCh)3*6DgbO}gpOy1sY~
zHnqG>*Us&}>|#|X(RAlgiDggaN$#3t<?rgZ%P;(RCzjHe`{MWO+kZsAPj||Yt(CQD
zV!gh8dHd@18y_;dtF_F!tf-!CyuT;-F=Kr7p2dL^f0V6|o#UkE@gV!b3xfxN36@#f
z&Y~qxEWAotBcG?Rojk;-!Boec&D9tgvFUbW%xAx^vxA=h{*(Us+rQboE~}VdC#hYK
z{27zvtzK1oUjOObbAB({@;LWde2x)Ov2t;po?Lt642QuXFX7cjFCJQL%J!PHi|G(k
z{mK2tVgBK{F&vRjs=vHsUQ4}sqSleQK=<CRyr#M33{ne<b6$UB65*;VdUjpW!))i7
ziQ*r)7NoDA9{t_egrk*Dxj93oGV<n!KS~>}yKH~+T1jW=gG%)coLdduTo=us_hk~N
z`ZDAD87Ds)#q+DHw`x8$Dfo8d<<XnIbG7Bml$R&2`>>2p;c&=}We!K&=61dPv+(ow
z2X#A&J4~-G{I{9?pE$2ef$`tZe2Y8hopR)z%5|ui_YsFr)6yC1)~QVpdh$6>C|-7<
zm*-)15ATkt#~xTuKKfRfK`<n1GDEqaK?tj@qgre5uCC215+r^!WIFXTuH7qk+$w(`
z^8~}dDGkQO5@OXiEF@}#76oe9%zo(ndRvHH-`kw4(|a~3v>x51e~nkcT3x%v?y{m#
z+l_7f54`70I;MF*@1?4b{PpE)AL}rx%=_{(uVr03do!!Y&almlC&a>LRF@U)W4G5}
zn8%^azcu~s3WYmtLH?=JOOMX|A?OqJz}@@|cfYw&)#IqZ(|cAtdsN-ITetOA{ZXcG
z{Za|nKYZz5Kh4#AQ~dwQi&|ImEXbZPEyYs!Vei(+WJ9&U(_c2soS>EB(s?EC-YkwA
zPsRCpp5A$<!e~D=!#aTPvBTVX#@g)NMhp5T+|X}ZTRGe3>)#bG${wp+)Zb%Sq?n+5
zMtFac+CH}VJrbYde?L6(FS-99?}`oX6?Hm?7W;_rJgFcvIjVo<5u0><6E=&P$2dGD
zC;odF^<)-r)0y5Yho3xh?kiaBW1il=`Bbe_dvt?j`h%-#OPpuc6@F%$w}5-$Z(g}o
zN?fylZT)V(`t{qF^+vw;KF^%Da`zk$o!%bvUw3rBT%U2;%C$o5&kUDX<FhFrwbXki
z3%2i2IJMR;Uq&bQ!NGGfmHW1Eyq;XixQDyTaW~(oTShv$TiP%3{hT`U!h!FMiFYRm
zh4ChtrOB_2^qDiMTqJkyYiZq#lEuGc9#jf8{SxaHO5<X9$~&dx{f5@7GYVGDh-p3}
zc~~cIp?Gx2FVzosJ?3Tc-@E>BrTe44=D>i$EsJ84-Q$FxiIo*_8*xpI*dEoASsf*Q
z=H$NoJ<*o@CWW7Jo?o|84|I>3@M}fJ3YjK}XG`9_{t*57^nICl2UgeZPc|IyQ+H`N
zwD?1mfN=f0O0Q=-qox{%M8AAK%PKCVf7*_t%9r%lUzCttr#+8#L5W<)P7(d4<%jk?
z5-Q!_=2b3ssrkp6$Itve+*zAb)UP~6JL&Q6i%X`fIrx<RU)I2}ZL8z+X88klHzTGf
z7K(Z?nlMJ~+Ht|lZ{1bzUZG!Go44^L3THCr?z~WPk+VFuq0V*N<@YhQ9zXQf`CjR-
z%$UV$GRIU(H-3V!jN*n#-|k%e99{kAv8G2}Qrg-h%MKg$s!X41XS-`-P0(%L{FP63
zo9uWfu}CMw^@DP)PjdC{b|KZNR!*tOx3aGW+`e&*XUiqo_U^d{x2<PhC%DW>Ubjzd
z!iKP4f*L*F&5})JE^aCpe714nb{+xF@QB8vm;N!`U(#Ht7bUTb<;RPF$DP5GJ{#Sd
zQRg&OyXyS>i|SSvPj9|fTxxkib4_{)ck6%u*2n<27n-`4E*><x6@E3jT5#$?)8tD_
zlQ#XA$a9>3(LUu;q0{=iQU3i*;tnzuA_`|#s5DHz@z^a@;_*H8+~S#2ewnf@JO6;!
zTP$~CWX$H{GtPwOET0v9?@)m6YWu$LZzrsJ&7%Bmw{y$yb;l>}VlLdV`gCu$X7TUz
zyybTmeNFk|cd2W;efY6$;r(2@Y^HQAvYg42$8?A9RnWx`HlI~@gt%~@UUkqq@#;%I
zmCKd<x@-m(N_W`|UO&v`b3br)Ly(Nl8Ap?9mrd&zr`c`abXKj`kb8r;kMiR48+I0-
zel=~=-P~t_JxcfAF<N}C=J|K)=-eH(j}orEvb*x(pFx^RMKinj%O(q<s4dZ}FNkm0
zQr!7`>07q$%iYH(C2x;S?~(jpczUJkuCF35^?OP!<~FJI9+i%Dx#i=N%znz`b*+}x
znccIptOGJXhw4w5{go--=l17k=PZL>rM-+XZCGo%H0O@?nQ7b8%JkTpr991olICyz
zIzicBU+2L9!>(IqM_YobbsB8HG~7~sx@oJ1#$P2TpTINk4*q!Wm%R4fa;MsmJIB`k
z-ypi~@|xT_j*WWP6l+@<WiO?*6@=V6FvGw}>cS;0tK2NsGo4&o{qe1fG_Fqh+P!Ae
z@f)!X_6KJ^x?X!VVdbJQR_67GYZ4AHB_EiO!1B8B!G?g#Y|av=bYpD&u7C2C*`{YJ
zlmGb#>%a50pKiuoeVSL&BpuwfZ2v0m_Zd<dFY3j^e5xO8o^#PTZd&`Y@V{kxf!X=j
z^0Zs4N~(BE`j4~M-n)>VY*?egQE_EMteVU1?Xx9suUG$O&g+sNCRzRVba;A-%>0LD
z6EAM(emPlSvrG2H{q2>fs_I21?__V)d30!PrCD24v(OIR4fm%>YtCPNe)Y9IHcu|w
z%;bCQ5TbXhnQzuvzNoZ5nMmC&)$F=b7oUAy+uUK8qIWsLVR4JfgM<n5pJ!hFsGu^#
zi+jP2lP?Tcm>FF)x)>i;YPOTP?d(QHw~+rc8L#@SSYgt=Hzqb|#n!}&^&UBk0_uNu
z+?vgv%T%}da<}Jl;V;4+ibp@I9L@MJXQpeGTTIbRmg<eka|}+LX*QgcH~sDtK{K``
zPj^3JyVzF|eD*ibJ5#>UyZ4+WbX+!6O*kvWSZ@$9)%s{=eBz1O3ziyaKHNInl`m89
zzQ6(HY95JN8$*$;n&u0AZtvN~QF`}?7FWRS<!>)*7Th<w{5HaQb^C!`+fo>hem>VS
z?VeunsW^qnu`;K)mYmSaJlmWfkT$KHGo$R{>m>g_A0=-Ev#iW*UY|I3pHINn{#{#}
zcf2ded#<FPYIa^Dh#^cd`fQ0^QQB#n19A5mroE`J^kkcTo#}VVTAoMOPCODb|5M8!
z6ve>#X^W3yL89R?29pi<cHX`8wrlRHqV=h}E4w{4Z?-U;nz7o&Sz0$(#!Q26bIcLV
zvIkj<S$<o{DwZ%<-fviPYKOx_ZQicS$Mk})9Lq7#{MHrtdbM?gp7A1$1?!wvZS=mR
zrk*FX;^g{y6P)+Sw!20-Z+zbnb0$`?xa7K1>#ZXvq)+aae6wNmQ~qsDzuX)<KdIP%
zW#7eNpekmrzw4v(gjCTr4k3?iejiN#aAlkGM9l{*uNSR%`(Lwmq0Ev^u~YPAeg!O@
z`MAfj?R9szNO1AJjEsmkZ?7JBu}J30+V$LdUyhar*nf7=N?s9X)1xND>2!Ox;n#ew
zMa@5x0$YD?e;oPpPwP$nb2p+7w7>au-|cMX+2z$&%D(T~P%7FtanIw^Cu4W6h(F4-
zjkjjL9?OpH_wHI<Kij19cDDI_9X9QfmUkSB9$lO3$$REfUHsG)9YPOYG;zdFbh=!2
z{@C;>$!wmSn^&)om~R!myEKQd@urQ(hoUUe<LehWU45=mBk}LT`3bj<c0Vr??dJ;2
z3yIP@`sh%a+rQ*H?o&5hkzKVT=J&IP3lDD?sV>zu$xRQ)n(E)UG~Zg@UwsN2e~aYy
zEqXZ-$(y!!7|5Sl(6z|%$fE`x4aa#argk3Xx@CWm<LlJ!!1?Bz6ja~zi}M$4z9D|W
zLE*(d#g*|)hJ1{ISsoX~UR*HVdf%{ajZg_I*AE}jiiKwRol48{H06FI^QO*BSh@75
zuI!Y@SDrn}NLe($$KG`AD)Tq8E_Y6>kvn?EXW_iYYkd-P9nK$m^las%@bB#3IwS&T
z?2^A@J^#d^qTM$?e4jsedb0V~qn5kAcd>on@_-@x-{KDSRa0$dJgfV7<<iZfcJEck
zC4(-@yem>|I(<mY`X6t$x~N!V#H}*1({@50j66*4$E36#zI@)-bok@%OSj!;%<l=F
zp{IUe(rz{VwN~BUYv#>6?;5{K^o>TPs!;E@ntw*i&wtl@VYBM)&u6TmqFis(yryho
ztgc*ET+6Aa+-cEkv0fmubl0f^&G|>K+Sjmbh)s{rtUdOJ>A~D3H{xQt_w7@$mgLA=
zz5cL)y~n)2&%7O&e%(5&<nuPV!Zp5OzBBJz>4PiZ_w&tnj`ZKr%y;w^%kEb5B!ekk
ztG^%qQ@SQzb8SHUjh0JGq|~^s9nJhyl(|7@+mcV&C1UcCF2YkcO|01J5;>Rs=Y={o
zeX)pDk4$f!UMaT4;&%Ou)Al=$dFGosY&AIDab|_<EpgW=Az>F?AKbgWE<1AN)J6M#
zX5`kqbv@nV{hmKWUor6Ey%%;<HL5iHe#~M^KdxVXH>qd+Y*$r_wV4xriW2soOtH$I
z_8@iHQQbRBZ|kI%im|SjT+A9)u>R)XfZEg7r8iG~xHvjLnrH7}Wn+!R*DLw>X5Bui
zbai>Fl<#J_?9N@+J#~$gnZ?UXFPy#|BfV?+RlyRE@C9NU5)XSP2)>yVCiJlO`W?#&
zEcv<#w#hEdL6If83AWB(+RtV0+Vg%Af6T`_4YkT^RzFXa->`UEchRTbJ*9=K+vo17
z-5hz#%8iF<8Ox`xW81a5_qW972bD{<nX66u{$f&T-1=w2m2*A9r`P`cTDhgybNcOb
zCwsRody&+%c%EW{=k=XcpHFZ8;Ks9k4ZpGcrWbK@D(5;nN==S8{=sYhXOa2qhu7O)
z&3WzT6qhyqjPBf&%fUHoa=+B;DsUc{SN5*Kk4MK<Ztm<0nMY@LzrCHhAnI)E#+OSZ
zSl6A2+To{sao%$0PdnEv|NZK!0^8rIvNL2pJS<o05#Caf|MuF664zkw+}MdB2Ep~a
z{_%^xl+h3mJO0;R^5(`F8NRVU-t0T}b>aF=pHD_E_?*nW_`~fpci&Y7F0Wr^IF<WH
z)lKUHy=PBDc3u!qt=@J#&nh7PhIxiq@Iv2Xyff;wwQ{FgURY*WC@#k$FJPOL_Ewh7
zM<?jw4TWirQA&C*kGx&)yg~QM<Vpdbx{KRePA~XvDcd(Wa_*6-u1{Tf!cM*^bYE0`
z@A<4+$JsTSJf0DOVjFJMotQ16WYQ%T#b>b5b*@g|UghFTUuqZyA8f1tWDz?fo!e9Y
z_)fV6Jv;tOcAlK(A99FaPEGP2_s_%IK1Ll*<35?__TuT;)zR*bd#`c?Gcmq;sU@NR
zGs)vqcK6=3o2Jdb<hSaf_k@1^ug$JAs;4`c-80D6SMjU9YiGXbjX=iH<h}(GY-*Bs
z9AEvqG_`IW*B9Qy7LGH`mU5oG$@pKzbk7crCn;;E3hcZ7y-!s*Ahh%SFX^=Fo1N`j
z1AeXh+^es{J<axB?}Zb>N7m=btZWubWlB2o@w_HaL)W(O0A?5C6qZYV&)ro{I(L3t
zU}QLHmX7DX>z7w7o5J^`)@`zy>SO18S@~BB_h$Z@C%Dq3D=(*lH6$da;(OB3X~JPM
zAA3dDb4N^>{G(NN2RG|kIl*1QYk5B%JdhL6zesDvw3Xg#^umJD6a)4wnJabF>a5El
zo*HEj!CR+ZKD>JN??T0hOADPvdA4@%Pf?U}d0V{u2luy{OTkj2jdj2OO`dkw{M5qK
zNh-Wc{l$6jOzb}D^!)G5!na-@mqpc!Dl5CL6<fGp^J$G@zA>xIN8ZVD&ytGs^mMK|
z-#Pa9nbzMqem7RDbMzZGdB1$OZ0+w26^qQb7CiaVef;ay(jLX~makn;&TRD)=wG;x
zd%feD;Dgz_ZCqap-U^tiYml~Zg7imKe*P^j3-0wi(VY{OP;E7R_CzD?UF$hjWu_;v
z?aJ&-(5O4e5M7ygB-|!RL7@I^+qHL=g32;|f))?w>xSKT>*YCr@3UmG6wA9yXRa>U
zBGp_aA@FxXyyDU)B9Hoemv}k~WwPiMF7n#lkYS-OGfipcbghYwN7?=qZCxR4VSRS3
zZ#{#L#tkN?>pD`)rJt`ewwfALpmSrt$E94uDYv{W*kqrq-`!u%@O@g3#wmrR#vc79
zai8axJ+*jp@%LUn<F?C}v>i67L~7i5yhrGm-~Sgp6}Prm8Ex7(&u{v*f^Y9$9S>%8
z`^vhZ&&9Nh-PWe8Tq^&=>4sTSVe?eYH}Ww)Fw=hg-s6pI*q>jg&fogS{JXT*HSO2Q
zknpM7{T}wl%6*%>#r0n)hvU3G33485-5nDWS1z7%vsfv|Rz`bEEUS0lI@2i;6CS>N
zKmC2=KBpfdJM6Osd#Wdd_<Z`YIq<-PgPRg&E<C!6XUUGM&+>N8zRMNQSLE6-x$8JP
z^Sd=pb2xWbS2phXv_Ndnq>pJIrZ9C!M=yL?_PhH=BFnme+fQc7YU*r~Iw<7YSh#b;
zv;~|S^XD&jU+s8hj`QziCEv+UJeH>%a+f$I8M<6_tK)%)h0|<Hn2b5I9iRGnvCe!y
z{pRvTqC!50wfUFjwEWulD>P_PBB$G@8EUdD)AnrKwcq(-@Z2-jTkb^soyhyPNpy-q
zZ=M1>?`mnzg&uh#k}S3-OFwoj3tHrNf_Y#3@^(RoH#e1k^oo>sUOb=K^5l^B-|Rz$
z)9+Nf?#M8*{K)c}{kh?iYrPV-B9qN3o?pJ5`+Cyq;71oM9X@9J=-a63{<tS*bF1l2
z*0PTYeDC)<oqSatFL>af-mx8%zaCqXzJ2YT2Mw!Yq7@9o6|Y-fIPAEJmp}fUd#JFi
zyuwxOtkA97Bu`|Sr^W1CRX4rv_R~iO#@D;^>O1PZyLH#Li0lrzoVe0^ZuOSTm#N`8
zoy*Sn<OT0~@zyu1)@^I=g0v1p?T~n}jem;PE?%+Y)TD~1$4ob-*c#qyitLwZY6^1Y
zd{lho<P>cwleg{{7IW{Kns_jF+sX^t$q!~X*?hi`zN7Tsx0_uL@5KCaU^>Okex%1U
zfQQ$vm$gi1;@uS)g@<Y-|Nc50B=Em?W5S0EysJyjuGk{YS~YX}{<;1str{-&zm!va
zV}JO~7JM~9{PKyOAAvm|7A1TY-?#H!!Yp?4?K%0HnVkj)SMxqEJ#_DAzQfbbxOJ0%
zeO(~)nxBdPW}tGRyW<f<yQuwQCpeT1yf~CO0xRtQek{3}aIK^F%SYWjDZ%>BpM<(R
zHvT(%v;5@6Sv{t^D`%}}n0{gVF1JK|N87l>OD}h^-E>WKR(w*ifmPbzT-Gyv&8^-c
zw`z>GZF{{^Z~9!0{D#Hv8((cXs3pB{{qc=q@xK$Uh@M;EFz>e9sVzTeIk-zkrF1`7
zvUPuf^)Wk{8<U!InzN_>-f0oBTH|f0;6rVztB3f_z8pB~r*N#SHhIdc2Rr9i&$lsK
zEmVEPz$Js}-m;C~=X=Q8TI8;eIJ3(A<K5Lq&m}oLS8iaboHp^<6C(%ySKe`zEK`?P
zXEyG*;x+Gwz2mi_*ctCTr>&edXW`M?{v~X-|3kRhMOHe@N$HXh*i;*v`8VdRn0jzW
zre#R|jLGE%?VFXaxuiv0-f7YJLim)Nt@xcUK6kt3OU0#@AG{hY-&j5KNpafo*&;3e
z&-e2kW}0bess5t*<e&KSZ=Y!1SmiBO|M&am2Hk3w>%ObiY@LlBKTEq>xiu&ubMe+E
zB~>pJ_OJ<O`QNL)8d?-+HAOJ3yD7=%QTVR;XPOk4^WHt0`ebrak7ez*X2#7$GO5>(
zxb`-OHE=nEU0lMYALBpSwO=@8-9ytGGnNX^+?!atDppm@(zc5Gm__*3OExxhxu>ob
zS4jP?`jB;6+&6)!v!}NN91eTHc+|lz@5s`pAEb6JJ=?X~qS)x+)gvs2U-iVk548?>
zukkT{SLBB#BLn@&O-06)`PcVU>@mLQ_e<3`>bbAfmEDWB9I<JU*bv+CAw;12+Ls7N
zg{ujA^2u?b{;U5ANQ;Fyo)ifZ+Wm*qEV^&yA|uzhYnPkWH0_(e?rz<p=PT}fV0a<g
zb8k`f?CHX~H={LneR|-<acsJ<iveqz3eWNt3UzGEB}Lh6r?NMQ@d~K_lJo1zPM*rV
zzt>?O%Rb4n?9*+QC3;rNpB}!W-P~ufEsp!h{#N7JhiZ0P^EW^FUZdo~vgrEj>IW*P
zkB9I1AIB@V&v;qMwfn1POi<EXD%sU&(Iu^6s!`%@>0Ht0=J}cDsNbbLzp%-tHk%0Z
zPJ6Mp`O}LkR_lq^9dfK*-dT|N_HT*Sww<h#C(SGVUy}Fn*XqjOZ;!lkEGU%Rc1}ZI
zQH;aLFNUeGA}3Skingq4$B~b+>}vM!-|xNj`p-xAhZW8?mlWFX#eQBN+PKE|)ICAY
zlYeg=wC;2jnk0AlCzJZgyQ*_d#T371m>w{@`Jg@L_5H~YcbGV)wMeO758FJqeO6<M
ztmH|%Mdk~wE|i+Th-5jI*0<(nyh#>A_FBK!?^kbbU%+{FTWR(7jk&xZHgvnpDtWZ_
zgJS@n%zKY_r%e=}tWkY%C919>Dca3AZ^}OZ{;dJxfh)@_E%;0rcx9D-OgB-uXr*yL
ze*JEz{gpZMFDCkhA3p8)>-**TU6WsgC%Qy--b?ytE!g#AV)ur>iw<5Zt%}<Hs4w;P
zpV>;6x-}0l_d0p!NHy@tnuv+2P8Zy3XZOxy7LUxWAK_XBqOa9gu`klyDs*hg9_?)H
z$DK`+cO2ZK5>b+E*u}Kv@<xY*N4IAES(ByQW*aNyeC>(4Lp6UKrxQ;vzf@RqPNUnb
zj6aUYukmMwm!7tc>QuZiU5vZ?q3NU#rV_ocBvPavv1A@q*&DxqLuTQjn$x)rzbrH_
zYBO31K977Sd{N}Zm&p^Kdv(h@`kfN(`8DTHdpt{2OVWmrS!?d5sH})jxgc<_{lm?6
z_qhE&^^<NYh;VnGoh6s{=&GK1=L&-f;c>~G$tyVL^sjyP+C=ompS`Sir#`CfPFv0L
z^JAlR$oqwEVq5yeCf)v1oH(_(bV9**i|aajZFfJ2_I%AEKEvsg*fv+zZ7!^Q%6WFH
z*0X>5A^T%Pwob<K{OT2Nb?)hyiCvZQIN@?(X%@fQ(r}%r?hiXvS~xP+9__j=<+bKC
z%Y;WiB(?=<Yy1elEu+d>e(1bXP~Zu(2iw-Eq%G4idc#qh%>9~UQjTHoeuLMamF^Zu
z9{R)Un0oP~itE|5g7<#A#5OceID0v#A(^#v@fPEU_BUA!oH$oBdUA%QocAtl?+A)X
z4!*+m*&)61y={QT5~nF|1Cx#}o*6#JN^??8#3s>>)Raf5bMnOOm6t!#%zry2@{Vhy
zX$9kpzfUhtn#*)LdIIOpETP;D(gK?oZ+Rm(t?PmE{6!L1n*S>0tX3Be&@I^D&?m?y
z8dxgk>&SdFY?t5m1S6rt-zy4YCiG5DoP0Ckj-|w7XWd%8a*tn5%epSdJ^$a7c*WIQ
z@0Rzq*ejEqH*T15$Zh)irWen1zo$t|nba-8@#diV<H>7Fw@lIMKGMjkk#*shwwIjq
z|G6^^^VBLW7jM~Xq9AM`kTrRZzxwmbzd8TzP1pQkbASEh2l18yTYS4ycAaiB*Daa1
z=wSZ;`qw`lmt`xk9jpz=Q>eICTJ`Pc9lMwlTg=w~RVWPa){B1jc6#{BUU?nICe;NI
zMbma}deP@vCaF=q{m88u9~cYj<3$9MI2H>3?Z}!gT|aB#ixcJ+D^#nW_ACtiRvKKg
z%P?j&<5R&`@zZqMq%S<+i`c#JpgH#@%ftD{R;=6p{H9)B$)eo+DaWpPD=+`DxO=kn
zdcBC5s~X+xQVh@R)Y^M$^8Jk+rfZi<FV#)@WUg^jx$Z)g^3S;%<}PWQx9lr#+41Y#
zvu9sU_N9G2YyW3oL)?N-d_R5S&+2jA-E!*bE6tq=N^H}(Jig71*9$nbZe}ar{+2rz
z|DB%b{a)bu|LR>q%Njq;t>X{>6PqXhd&-3S-{rc0U5I~eP$P6_zNY%Q3%BcArsU~e
zvk*yV^YOZ5(%d7<_@4Ryp~5Z7856|zyUuJ~wxe52EB^VTiOYC(ejR1a%_y#Za_0}z
z*(t8v-?-<!m9Gx{I6qnAyzu1YIgwkGytL<QwLkT#TsMbH_~NOvwG(|NCH1Jia%9`G
zWkV)spKIQm$4AfHl360utHhnG7F@n^+qHM!Cd{wfYO%)V;5-jSXXR<hcjU{s_DAWm
zeT~^|_hH4)u9ZdU`_`@L-`q3zZ@W=cj>m_9>X38m?T;rMXT7q;T%q4W-@2}>I(WjL
z#gRS&Leo!pU&-Kj(|L8(uDZWm_dYmUU7RB2V!PpnmeuXv70%x|Ryn$h=l+gm4qDju
zTG=HmJcuzv)p@Vb=bn>`g}NFq{QlDHdblA^E%Tg@*OZJk6Eb+e9DXh)si#%TCt}nR
z=DY3j*7f3Sa!03l{p5;}%DtG{QQIK#i+NklrD&Cji|&cBWT-aZ*W2UP>5v?6@}{u-
z3*%`n-Jskf2RJxpg~Z6MaG3VJi9fwz`&rwhnmzq*U$s9m_A&RHaQl7Ca`)z(bdJsk
zvkT)E7-~ujD&17#F_Bo{8E$j9W`9G`JK-zU#}_oeKQd*P-UG{|qi(C#U*oMWnq+&m
zC%IGg;Gq`N?xYC|gjcD&;4U#pUB2w}A0`%m$BFvWuHCZV+2ZwTAKQ*YTUXiz6tOrn
zf0NUC|9SrG`K(!v(*hk*m+zBUyE{dFV!FV(JtoEN%RjCb`0U01@E6}!hYNGr{7&2r
zbBu_){ee}un17pWg<qXYxLESO*tLnfLq2|rRu64^A1C=qSu@SH{&DQjSydP0t)d^t
z*giR1XYyjvv7ZwI*BlW3vtZ+)OYG@0W-a+MQ=qcC@Zbp#<<NaHF<Z+{&V9l6IO^6G
zTgg}2{Aw?jZM3_Y+5XJm=45Ns>vvZ!*@tT#WL>pPIM48qQbT=l^yYJ(uhpBDJIbx+
z6);dVXh_SQa%ClF{>022Mz@I7KYbh)td}jg#@6_ju_@olPwh0PFL%>CW7%+{zH~Pk
z9o>B@r+<~sjaA{>x1{h)rR&{WR*Sn0Jd_G-xA6Wg*mHDUq$2+Tk&^iB6LkezH=I;1
zaliQ5p>uOm@NCl#>E!}Gwf};4{P&VrzvjEUPW8%m<%zqQ?ylZY?v(m`T5pQ~rfRY5
zr;F0gn%kdl?$!u4jF410Id8|gAid0B?udgE*e%<9I(O^*TKbXu^s6O*%_Ppun{#e`
z>!BH3j=dHutlxa-J@-O2P$=<v;hbt7+vRVjFVIq*^<HH84^2J)5EZo-^89gC_cC2G
z=Qck!GCo?qiaAipD2ZA4Ws6$$f;kc)?mV16=K1HIY3!S^cA4AicJcV_S*O!pE>UHS
z?N+?-Fu~BO#$f$R=i@RRH~LKG>89CyzQ5gEXz?#unTR{_*H`t32cDDrcW+(p2j^FA
zX-E6#q@9|0_xEkbcfl)EHNM)4J&TUq(l6t8ccsh2l^NH>W4>JIIrPK#Z9xw2$#~sG
z^RkZVwA)Fq$*owp`P24i<wra70{$fzo|-M*FfA$he)sb|*=lxG@Afw<ZV5M@8LAqo
zVdc&w!y;Y2`s?A~kh;>fCW3dT3dXN~q*mAE_BG^&!d2rLrk$~$&p-G*Z)xSLp!eHW
zvAo>4UN%HoY}U^f<F$qrV)Ex$<}2v3y-E;VQC@B_xjB8}VgV+P`RlDbzIS##%iqeB
zZymW-_0E@_O<JdZckiEI_}liFv_+TAm5v(EzMwm??4n1HzJJ@aGnaeW1&jQylIiE>
zlrC1AA~`d=$<fWG{s+TU*L=nkXK%exTDZi#X@Pu}&_W+quZ|kFY@XA<X6cE~SfZVx
z=`iDL4O=wd=jB-i3<5eXtO>=vi6(PbY&mg(d3iqT7njNa<)~Ex0V`)t5Yu$s^yB(j
zec`ox7#p}x@Xr#z+@<{Ejn&~hCsoUB|1*d>>2R}NvYT@J{-f1i|Icu_d54>sS2bCF
ze!1d>)l24QoOhRUgv|YD;k-;_ts$T5oQ3~o^;k|6^1j=!W+KC!4I%rEW-n+z$L^?d
z>)j4@quoiF0_XbH7q;*E^~izakv}I-nw4{kkD^uWyobNVa{Hz8`Plv$yw9)g<?4~x
z7yW)#xzExwKb~FopSoUS>N6uZlUt|H_NEw{f8I0gx4qZKv>z^B2Qv;k2ft_Te7df<
zbk!Vj6}#TH-P0#U+4cQalVYDJowMV1QugP^p`t|w-JViS$9c8d*}h%5!))!p)u?sC
z>Vp>58;{KJxo*&Yxu-oWSZv#p8FO}LzRj{cpKy6{r|L6?r^!3brk`I^v)y@O=%ML}
z68oRj`#4JLyWad75+3T|ol*NFB6QX)ma}JhmK055xq12Bg6<0`llScW_3BpB^vvrE
z;xZ(jimyFzn^VfgZF0QXcKxLpMo}_aF#=0l{nzWw`E20JDiQ9Oc)g<5VS+`oZUpBM
z$=3Sr&kJ}mD)+6gQraB*wK*eO_Flnn0oK9*rpUwQcYaG0T)5FUIrpx7IroZPFL$Rn
zM3~)O^N>SPLCk$GQ~heDeqD8bg>^hzum5@#{!90ALGqTyV>P-QZ>QZgH8av>OcgDC
z)xJ??uIt&2+YKiF-Ri?MdB?+62kz}}n`dYr%-CXduAFV(`KdR;pY)!Z-n#bW_wuEZ
zl}g_yJUF-XhzV=s<V$-!v!ZL$Y)yBTdw;48sIhQ7JbU7{Q~c`E=MR3rw@6~5wBI2=
zLFG?uySi6+abLL3;ls8+sfABKb<rJ`NliCisO@B$YMkbFq(perPS)M2{k<!6T5dnG
zkG{rQa`5i94U>-q9+)#DYH8T@Z!ca?P@cvhW%8rd!6}7Z-Y>U$q3*p?0tVN70w1yb
zX$?B*`N4R%dzI_rw@ZV&a@BwHJLC&ax^VM(pvKjf@+n+VN-n1s8vk9LzxytKtz1h}
ziF(2^mvrH~9>MEos`^jfC;fM2X5mEBIie0*5>#aW>Cc?F_S4=Cv*xF%Jw4RL_hD^6
zV_n-11Fc<8zMsoImyodAc=^8LY$;z^mpyFX=lEFj{K<-k0irFuljNEnn>gQ_v1Iek
z=9Xzp@4L5{#7#bYxYNAsqDH{lf}@j<&OWN~d2&K<ndzy+iLutcA485+y!L&0_l1k2
z?W%1bF4#mFiF(eg`xJdlHbio>L&TQ{p^WzL4~OaptyO9)GTL`H|6TKcvmfPBD<-6H
zw_l#|=&(|qnd`D_@l)+m_7f7`Djq#Av3*BD!@;v!E*tONKBkq$R(Y(#<f~AYc#Hj}
zncrtb9BbJ6W_c{HaQvq4rx&-$X(|8umKp!X=?ue?!<YSC^;)+sd@+g7eDBq$Q|?g?
ziCTIQG4E32w3qLgD5&(feZfSY6NhT}4*M?<V^()HGtipcdaL`j?ZS**cLMK3X;-@Q
z7M89mf4M;I&Xbjw9rX|H4eNe*ZJrLFSiIqpLu{Q-{w(B}6;U0uBO<o*(fQY@2fQRK
zgCEA;O@IDs!?DV3jY5r@d+!Er3{;zL$Cj3*_axl$EQj@z+ehNQ=zR!d@?Pb7Y9d>z
zbA(IIv~#t_5j+X4JhL{b+SIL#609gxSi;)+^XuV_7gc6!tXov<Dc_ym$<1F{&%5Q&
zs^ER+C%CL>32ooB@j;%xqhN-ZN|VB;m)~P-Wm$txWH&YmtE%38d$;uCp<R4pkJn93
z>yC}FHDA8vRrwOH2TeDM4qVe~FKx>(+`QEAb`;-Vr<$4Ol{ZbSAB*{MnjKEpiPfn|
zE1Z~iz$UUz^iglisWa229yLAAnKQfekvQ{g7VqR|b-tdwVjH`)Zch@==Xe`0U4BzR
z{rK|@y9=Jy9rc#H=*c(hxMj)A!oE+nRRN0T;*Cpfd+scazmxFpsJ82+K1*lWOB-Fw
zKVNJ4t+KM_YZAwiX9BPL>Tb&hC9Ip%Su$he$CqAF%%?X0S-H<4$|>W5>Y>&x`4&8#
z9_yI7)edo~9-o$WbE)yJ+2W4_I8JQ2q4aC+<JZ5g#frslIyJAuAn?n<3EWkox(kz&
zZRVs#H74J?a+G!D|5(MZK~II%=k@JgeQkEro$F_pe=gfB`~9f@_a%Net+QVlpKg8C
z#d-FK7hBA8_g9Nc8)hb0ElIu9Ha9xo-LfH>cgg85=lA8AURu0wi|v|C+=1fNPvs9j
z^}FX<|5~^5ZEfQ6J<Uep{dbqU&ugD=BzS`5-ZzJ`PQ~Mnu4^ifif$?Ud6M7dt-G3+
z&9mrV>^IF{PfL}OJ6OSU@VR98E2nkY+dtUZ-g)`FNH4-`zU8u~rw+`#Q5+MlvQF;d
zL{%Gyun7k=U%fxbH!D<5?d#47AN}VmUwRqQdf<vn%(|}dKUN%-9oyHPuiFs6V^{a9
zqvDATb9#<VzLol7?%G>yzV3bRC3d}RvTRhGTCNwra^LjuzW7MhIvJVce!Lo|KUn&7
zU-?`1FqLJlVc+!E2WL+I$$3ovZ0wfZJ!y$O$I7kOHb~{K)ZNLtq3@@BiFx6r&3A9g
zGOk&8ZU5!7e!{;lJ>S7$IxV?Kvh9qF)sxkyRU;3Gs@=A0Z!-Pyhd(6BF7xf3+PS(O
z+wGiEm2P%5aI2+Gp7eH+U7S>M{>2-NT|UbU4W!?i+!n3up2NP3@k^<C=v`}u+E{^G
zf_Xb)%n$tf^!xClH`_1&xMf$A)I81h)b6i&K03!XXhm%{{Q7RU?)gK1tK5INW~s$D
z-r5l|WyS0GCcPKj+0Qhtd&;iKkXgR@#<uH5IRbSX&i*<p^Xh5g?@jXa!_r<ZIn23p
znLNjIPZ@)|u3tKuCtkeUBgNpn|Kg4XCY;_*smbxpJ-)|_xst84gdH1a|K1*!I@7);
zKp|oJf@y|HXByQ+-uM{3G}PE*7VI?VC6B~=_xK&(#hb44G<WD}pPsPrR%_<cyI&1e
zM6WzwbL5M93#+qsS%%cH(}8PTPww&I_@NvX^5Dyy#FNLqt(cH@N2q_{JfH0K75X;i
z-^3Q$>+OihoS4RZv2)dnlmC>1UG~Sw>fMn)wcRIdm)6y#VXQ1Jg-)*CjYproHlK1)
zMe0In(&n=p1-9sane|LONchEFk)zHP3pekOw7q2dz~StJ!)GQ6{YuaOAaL!5Z1#CW
zi6z?#o<4qfsd>&vQ_J0A%c^E@>E6{?xn}XP3ekVkLM>ud+>7#3*juX|;y6;?vR-OW
z{yS?9<DY5IS4AZ)esqZ=i*=^j)5Cth0+uBOFWfn&?!nujnC;&;n5{dfDVM<c-t5?m
zS88<vYD;eIy7pws7O|CPlU|CJDHbgEy8TG%qeR=+lIdMyvc5eRcAhySHCg$Hhhrzt
z%kmH9Z=^XAj6ApQ+~(%L$53g9o_ult?pb$w+;;JDR~!^^d>I#%zqHA2k!WI1-7@Ey
zTm5-XKMQ8vE>SvjEr(%%b!ow-Hv%tTRPMX?(}6+sWyh1YV(+q;*+(Ba*`4>Cb8dV8
zou=55ns1SR(hSl<su=xeOMeP+usi){ieKWI>bU2ho>on|#&ONdO-$f8>ygaL%i^)F
zPxSjdpQ&FcHt(LM@TaJ^X-dNGNY5y>j_)rM_O^3-C*;jf;@Q1glyeF5(YO^Ws$1Dt
z#8$@k3+yoEn>epDH*WJ|b#~Y7=3b|_Nh~Z(H~jR!N>B6RQt1Q^z55B;&MGwrI2@b3
z7VfxhxnpO$+tr?3AF_9=EKS-VA-m-ER;G0`KBWoHOLLgomOe4pZ)*_e@0gE^m%UgL
zvcTzyYVV_Y72Evyq>f1Pr3ZVLSX%7<e<o>PWxC8)>9h|<As3ccC54*j$M+xTU2!?8
z#M9Nb-fO1ly)QRdu1`Do!6~Eat;XgG){R0>ZF};bJ-n=a_3pe|-!8xVBiHvxPoj<U
z>5~;N<<fb!tvLH5FD+Gdo!3IfOU+fQ?($s=nzy>*!6%D*+zE5fa`4JJUEpiflB{kI
zd?Y)674Myg3LoZ%matgPs`?-*ZnIONV8)TS>XjOWN>b_*b!vGZD24phsi>VOb7$Uk
z-E(ey41XS6{PFesb6ytZmpsd!{My#C<gVu2GiD(RyxzVmU&O1#_V#Ivtz=M9ZC=;Q
zr+>G1&uTgGkx$Jv*|0m#>w-|Zz1^u89*;FwQq5G&zl1g%<9f)reF{@;g1$mk6{qy&
zgWilbcaCn9zGOA|s9;KI^P&y?N38bFIq@?pL3+{J`{_HTALwD+aL#r047Mx0lLV%m
zPn3K(CHT)aAt{A7&HOpHnt7_`t^cv8ehY7|O1SLS*hgNQZzU`;Vs1&z-P^k1y|v?o
zc){$cZ3>F6GZ$I&TRPPkI2p7!PPuvTNEGi9Kl$`4Vtv<Vl-snMt8D2ldfLdEb*y96
z4S{$52OW-oE;7F^x}>ezrEE(2Y`)*$*tdsGy(hyb);M$FiJu%_Y7U#^X--)w^g?d8
zuVbmw72yvrKL_y0wp_gEsLUC^(o$wYZgzxIQd&%3#l!yA%k}+i3LPE|&lTKt71s0y
zoCqj=p1pN-$gAkehdj)!KD%{Z_8*>*W^TTN!|&!p?b`obZ)g2AHDVJxb5eI=i0$n|
z6(SbmTyq2kZ<bF8@spTxq#^UtnnumcnuS*kstv4Hrj%-J`9Af)<nML@wOi(8-+B`m
zTRn6AWXDFn|5r>4rmUXsRvP8fy=8OP{F@ewMe5SeN$k9(d}LLXJKKeG7BRVV_UX?4
zwSRZHXp_aaEn6ZF_z5_sFVMOFLO72l%4FH*M(w$pUAqO>b7-n<-F({3;t<y^=G5D7
zjMJDRt-fa2UTa@}BmGkQYE!jCD>D3bu3fNiWm4S~bc^rM-$VW$iCI5n3^zQo=)4^G
z^V(6iaHSQR@3n<@<n=#(!BMrWlx^ka8$r3%OvZEX@@N=q{}g<9`o|77fx}+YOD_qh
zhHe%-a>m@N!ZP`U!Of|kzeT;=vUa1VQAmx5DZ|O7-8<)Ie_^eB(lt}yWl!}b=bvWN
z*QIftUB)J0^?`HYEUP(^cIB73{ybZeBvyCs{zvZ8u!Plh5<IhO7QRsFQsF5#p8a;+
zoJ|M!&WULkTGM9e-*I=<R-+kJ)-7x{C38L<I@Pu8j^n&DJ&6nw-))$xzB0V{x1o*y
z)Evgej!SnX_81gQaeJqj{blPxLksoJ*ek8+69RvFJ-VX9(qz(X!V%x0dHVPFr!s1N
zo40!3DoWm1{6ePTu}MzS?yFsCRdNFS|C;?@{b6DfI;7P4!A`93gm-g3+XJ@grQPSc
z{U7Q?$+34w_jDf4uzeGdx4We9+dW2`Esr-C*#Bx?w=>8|ndi&oy=Sh8-b&9q8o<q-
z++v*hyuEiyNPP6aC6Bs}o>Via_-s)n7RbcA;OLo<Ais!rvrN8T@-Sd$SR_@V@@*^g
zrsv1p{x`ob5sPk^u<C|k#Gdbrhi;cM=+}51Uj9b^{EoI?aWREUs|_!y6f_?=sQvqJ
z{t}I?Jzsg0gH3JBR15ejRu_HTA->MHd(z$#i+Rl77rkZPka%?mbI_q9J6{`VROB|~
zabKVH(%V4bxX<qxhZ(h7<L6AN?_<zg#Z>Tr#}tKK{t73aJf3j)w5Lp9rn}uThaC&<
ziexA-edCQPdh(s=qQlSIf{EUVPxc({6}M$Cnb7o7yH~bV$$_t6U0~@X^(Vh(Sk)RY
z2z~WWy?ZhH1Ew=qj{9gOx?Q-`qBrfz*W|f9(o;1Yz4@i0x<5#GG4lDje|vs>*6wR9
zD`)ft@EDxiaH&P}n3sgQ5APkW7XcN{{a50Ao#tgI8%E7IcR^yCMCZ~Cm+o*q<!UIB
zi@#a(?~erImb2`~7rsj``1MKZ*PQj74<tlw6(!cKe5`hVfz>%9v5H$Px;vGAMKo&$
zKaFF`h>l~sviz@XwCav4ni0LaMzSZgcs;#XW4v3=Y-v@xHcwdUeC?^I7n8*dTQ3JW
zugyFnY{RmZ`_}=}zL0>T09)^-Z=wetZ@-|y*3?nUR__tE{Bzp8qdVsm$hL0^SUzFv
zmU)&<<<(#M;x82(UvaU6bHdB4$Igd?BBT#oI((*QE92YjkecYs<O!dX;*tf0r8dVt
zJ9G1OcKk!@$l1N0PHgzSHcw9M|CVinCw6?Cc49f}IWNhu<5s<`@wWrHK5TS1|ERg?
z&m4chB+k#0b~bCW0;*1VDEnBqhi7Z_C$BcXc|SU1J_oO4sI|#&%Vp^wxaL2TR#I4-
z_JiSms^SWV<5mX_#pOgE>;8BBoK*bc#E`ESHX43c(t6SULR#)%nCKya`&ZR}J!9Kd
zI7#y1dyc!uV`j>-@!t%-nUQ*!r%X!fi2?g=72msYr}LIgcu_XZS(j;>Ty(?L(@(ui
z6XtuSHYA<hR`F0z`?}<xJlh8k8s2Q=J2Ty8e|d!9--Q*Ojv?!XIkWU8NC){?|Cu-K
zA5+)Fvcv`FZ7M%)X*npn@uYUoxd#VD_>50w&C5xD*>bYv#l%;e%GNHcy!$M?HLHKq
zgO@2OD;iG-t-sOrR4t>U^7X;W`!9K?X&<tG61{t~fb98i-=r7a$=-PTv$ERD4bhHL
zI^A{5ho{co;j`*yI>&;CyFWb3e7EQK(!$j43!iVev$`CgRO!Q8WMyZ>V0V4xgkLWY
z1=s3Z%rgG6zTH}*<Kc><wmaO?ryFo26-?yh+UN4sP|NC<;T5&5?`?vUqK;(c{PEIW
z!!?;fH;?h&l6i&4w3~`niwdl@U-XOZ&CK5AQO6cM3(<X&+qA}6)O6YXO7&{(b?rBm
z9B0|acFuhG*WBBC!DMbO!KFuqHCG=?T5`8(u4}GE{LVGiLV{EHFMGvY-<jo|$F(`{
zRC2y(>7h$YD<)hrT~pvL7pt4se&yBmBjzVnolaTYUy<|X`DvZk{&!M;73}ir+PV1$
zm*S)+SMKH*)_k;n%V_g4Y|oX1Wd{tny?qvxT<+z1aXTTqMw!P+!Am3Q#nI^=ic4%A
z9(owcI^TKNy7nuRr#VkZ{`TYM4Aa-lxWg}7mf~5csP-ZJo@G(puFeE@;qINcx%dJ-
zw!SV|E4PpD>9(Iwc-4|*a#y~6@w%Z%^vMRXe6h+EXa8oen7SallG(z%+Te00<0}jG
z_#S6z2jzPU7pN!5Et{(6#J*gU;rmpZ1EvMf6fHjQRNY(B&Ymlpnyyq{cC+-@pR-Ta
zcrUx#y5hc+p=9X9-c_sFTw^3Y9{y2rXk~1-aBuiqms_F|lEJp0B^Ujc$*VO;j9IPw
zz;Y|+&;Kl$C*GL&OqqSxCx=P%kmiY;LOE;K@0~l(ZFk<uUi<Ks=#w*73-?T&JvDyA
zjERiQB^RP+a&4(lv*&khnI02bSW%@LXz63N;8)g$8}<@kxzl`uc*1Y#Cm&7Q-TT+J
ztCK%{y?*$O^u<os9A_Ron(g)2c?GZ63!R3}8H+hybjGeZI(vQU0q?B`q~|G2)SF`e
ze4F#lLluAi2_1X<XNJtfDDyRTi$nJ0E!g4NQDf&ADReQuz*N&MGO40W=8Ik84u}8x
zLEk@l=X0{#z9?FrY`pV&?$uzm%(+4*t{mae(45QD$!)oyU1)<V@268e%N7dxWw36#
zth_IEw-V!8kKDx`TZ>|hL$#H?mMR|P-n4GZp@b!6O&n`^yfio8Z%UnTE_sn`>vXH-
zXX>JVbj^M*_+{FDKa~To7@qPm1X;-0vE@}e9=YDU>+sRmk3zo+>lZM8$#RsMd0qXY
zSm5dI9#OknQEO_R8Xuii9{b%cCZ%hEQ<P4QVAn#&J!`%^JgcPnwe`2(q-BW@9-ied
zINE-DhSnB=pzmAi10LjkS*uVk8-IVle!ZUGO3n2Kd#)Wk-{G`i>y$UWE+>=k|N2s}
ztRrGw{*LX53m<G)qkE%yc}&ysDJQ-;?H6IYz3cg=nubD)NZ<2!b&Do?K5;sz)3fwz
zuTMb!j+j2*6dOzTzt3v;P9&;VuJ=E9^p&o$LV~Gp*t;9|Zzfmfc%3-YBD*u#>(;_4
z);HF*KT5FnJHU|~+U3!?k^AAUo7XJQ)G5u5c^C8Ooa~iG$zuEKHV!+pr|^AY5q!EJ
z#87R;`r6RXN3%ZdoO|@^7CV#Rg)s&(sajFZuNOX_y`K3~SY=|Ke!!$~Vb&T|o!|^X
zHoilAZ#6#O6jL(e`lIpNW?HyLg^qc`ZIAfOz#o>gCYtbs_V@=d&h@g1>TrnuvG-}r
zB()6IvhtN`=U-{`1xO2vtbVG^n!~)++>2MW>*AM&6@RoWer$`)y4G}rFKHdKV0ymH
zyhoA^AHKC*+Pi{LR5gN2^>Kjr(<j;90;+;}rqzeHw}pg>anE!Ael)D0M5~wANo1#h
z!8xTiS+y&L8{5;58Wn`FI7S|x=hVZS?qilaxz)#+$=GH|_iovP3m46?Z`rMN?DDMV
z1z*~ALl0j+yhL~Y8S{M}YTWAYJJ%+BYIoZ0^6#JKv>?}@{l~6+xP35Y;k5jq`X_aV
zBo`gYk2@0Heg0C!`R<S%&zYYbzNO>Q_3kfk)lQfEZm#e?iTjFoSBE6NT(O9?J$p*-
zRnL`StpXuib2-_g@Bb?O@O#dI$UZiwzN9|KlV@63QUksj-e<~t+tI7p+<C7<t3EI^
zW|{M!3vbjkHXY~G=x1#x7b~(_&a3KKc0<D_YEE05bbGFKw!!<;H#$U_**7hA47{%G
zd2!c4<Gt-^XWDmeIdb`!(o;{{J@fWRFwJ}L@T^jQUjH(gC}!b8t|@VH+xHvUoKxqu
z;h$mbr}Hj&j>yc42hB5T1eUEVHa*sR%==)im4?t(d9Q_5v;C$hoU;_LXFqu2to`ld
z-2H)-M;Wrar9K|_)H)_%By(@xZ?~v`<WkqIf?5faZ`!OhT5|1HZE7@!tX9tVS4r(<
zcNLDTfA+}n<)1^`m!sv*Rm7X??LHl<_bV(ZH8P#yc9ZP>rn8#Kw@*fuf4c2`L0n~?
z=w&@|p_ENCs#o25B7I5Wk{$c}Si68!%>#+gj;j0*Qp@g6Ug=u&W3Ho3+(uhQNhS;1
zqwYRC;%XaH=cL#t%lPe8Jh{QoXj`$l+omZ(l8S4ye)Kus-Zi08O{HCOxix>bspZ>~
z=T0nfEl*9os<x=Z=TnRDo3~9GU!Seyo$w^dB2N3TMgH5)Ym*Cm{f?D7zB{_&|J!d4
zUnhmx7zca5dp^C+WcA|zjYcyUbMXD#@_>)CSV}<rRLF$Xj3@e01_wL#@Sc^`Ix8z>
zsnW7%+nj*XHTtcEGc~p=y?hiAq;YC<xm2FwwW;==CN`6!K5<Q++IXa#PbuZwq;!KN
z*&o&hG40zFA$H-}q7~m>b*+xP?7_2aqJ(*Opnb7&(eI@;nIBaB)RW%r?A;wVt<fmJ
z_rQ{i7xR)gKVZB2E&OL=PQj~3?F9+H3KZ9=O-McVcEQwzI~*HUUsvA0^XW_1%U#{q
zgr;kmE#_-76VeP0I?nS{bG{<)l*haB<28<kd%p_$#rcG(O;225!y2o%{tpkIKW2SU
z=V{d3$EC~JPtV;eeDktx&b32}SfA+~?0fS1z5e&OYq26fY+4#TRQAN)HMytA9?1Jj
z=*zYosR;oW>Yl!R;bU5$sOesNO{V7eAw6YTi)ssN^V`Rw|1Ns+NXcK6Q}f<Y<(O~L
zV)<7d*Oeac;L+yP{lud@L)?t->=Lb<y>3gQrf9#vBi^TI&Um(^us(F#>kmusD5WZV
zvuctTQ7*KXcy#3YAx9zahK(N=B^!527+6RM_iUK--T&m}_`{#Q<@8tb7RY;WK9ge3
zc_oz9ZIkYCdf|RybJZKCD)*b!E#rPGwa+70_}Q`TwM$oe>Ya_db<iVWUA{~P<L36A
z+V|z&_ICO9M|^yEyR9a<|LcbIR94&hbDwOB)5yIdeCgzI-+jOO)|t%g@H}mvP-QBe
zeRSiFVp~&gt{FOfW(LB%H`ZpHUnn7dJC~7PSFGWfclM>Ogtl9*d4=^4V%GLrW`FVx
zZslYb`Fq~$AlFNd&vMDELCdnk*0mh=@!K2dro3TxYNDUsiyitI=hjcI+fiULclq&-
zD@BjoMZ}J?sk~LVQ}2CKME=9vBN^vTd`fy@;KtBlJoC$$iwqIBi<Zo<Jt(7T6Mg&e
z;XF-;ErBUBxP0&FKUneTrJD?^$ffJo+unI`Jbo2q>Dm4*_=)3N+Yh@;e6P8GzqVyj
z3L8H!6IWXISB4EyUf$yElg~}}ez^ZXi`em#|9<_5>|5L`epOpYe41zRF&<&jTn5t;
z*7ve5``X%0FW95>w6*?fdh7c`RtJ2a1-I4MGn$Hh(#`xhF-!4j>$a}sv&G&blg)Q7
z;A4DRue#I9^ZdC4jRe0XlhZi29P@SPiOLZ)`TjNRlK3C%rY~&u`lXz97n=T5JQ9hK
z-n?G?uT%VC-HaAPrT=9ca(5(Z*=>I28Z}X^(d&)PffoKb3nyLM;5Iqz*;$vkb4Js4
z{5i(*;k2>i?LB8T7z9)IIUoJEa3jCZ|IL@49j(;Z$o}upUe+ZMkukiL{jC`>M*F8+
zZr-u2HPW#Comcqtm}yB$Z)YCr3S885;V^6Q-GT_CFA7^-&%OAtlR0;NEN_U}KKb`2
zZ=ExJkgvzr>t(T^*ldT(#h#gOvXlFjcTb31mA-9ngF>AtSM&w(DHkV+NIyOD%C~WE
zfG(S~wdcd5XOE{ny7*9CuRO19LjAhO7oKGl)i(GxT-rXdyy9}htDb~wA{`M6YsEgv
zd@#veVIvbbU4m;)JhRqbtz$|lZ?5l^7P9!$w8K5BWUk)gT5*Xx*}2Ral8Y~DFVB7T
z$IZ9AXYa);S%O?2%jfZ}U3dCS=!q`9gttNBV!0nxC&%RX<|{ZZvG%`W{7m<XyVS|u
zvCrj9CY-(G_3n7quht#~_U1V~b<*Dt?R#<Bam$5(H&rgtR^jajg+8BIc<4jS!rI`%
z*(zIRzqZV9`}X)|!pD2}Z`%|ktGi`vd(*OF)rz-%6FxaKS1b*!njGE|opbc>ikcSP
zkN!R1^Ng15kGb&aKHK^Er!=P3Wb>XmE||r%x?nOF{}pBZO$yJhbQMe$kX_q2uiIFT
z-$5>9bv1*yXrzX+ER%)rQg?<KpJF&X7Z+@tw55^t<BGq{kIfH67q&J#9r>~6U8s>b
zOY-`sTRz`h>?my+CGbQfedFRBtNRH`^Zsu$aPE0;Bc5l}ZdV$s@k939RhOp|SuJO7
zIDGoG>FUW>61qDwZp~s~a1DQBaE|NU3z2nG1GfA)ny@M*v3n|?i^$y_Qge5PUZ_c6
zl77A^N$7p;E3W0|np4jhAA7Q0wLx~)&6TcpdmMMa)YG&}^c23{wBcO95yiHX1wS2B
zn<tuDSZEwRYHL=$XU@bgPU_{8ie602a+Tw|yJd>0Ti1m{{XgCm%N{zmFVZCQa>U^S
zHWLE3xCzcWYH+>yLO^6dm2X8@qrvs!$u%K*uf9xT=bLx)qT8{!^8ubut3}fsbDmCf
zk!-re^LhtQu~qHL#haYpnJC<9I@pr^THjp4XtIm=QkLG?w~ebZO{ZF1`!HjZXZTLl
zzHc{{XiV7~&>^(__`>yH_s;kA?e9>GY@S!#<2kpK?Kj_f*|kdq7C&Bk@Ou7~h8brU
zd!H9D5H<~bu;O~koLAbn%^e-p1D+fX|2|RV%=_bI&xB%xv&-C^XWsiG#2fBXb!Bt+
z4q-8`?MhdIzHVNux=G)`{y|I97K^njs$26`dA%)JD4+jvf|=Kz_UC>V<=@*(_Y(a6
z_Pg)yi`%D$apxTKz3Jr`b&0Kf`?PaMulc?Ib(F8TS7y`yzi&I1?cdd?^jcz$^6t*X
z(^hObePxy5$`<)wXLNg#O)7p*DcJ5G9*}!+|LK>73nmMPoJ`(-XxUnSbJ?lc%6CK!
z&K5g*D=1F4=s&F7*UDKmWp=|vd!~c_{&&`RwfWoey_NBm=kB|FQ~Bo+g~@r>w2!ig
zW?Z?#GO^-eRDjj`Ig`3N*iGN+TWobaqqO2j?xr7pN0-&Lut(j=vDub(ahhXzeetJ{
zXEqzwNq2NPayHFgewl5t!M~{j#f-drjL)q~Ej_09J%-&)L2g&R+O`SR&A<O{%#ry1
z+*Yh9c>da`;~5VpmVXZW&L%&9qQulyCl)BIc<Z$1%nZ-Abc5zulZ0-o2}m1nW{&c%
zl3Vcmj?tH=U$01{bn<)Nd2%XBY;$?dj24$;5286mLZ_^nr6{!BJ|tr5?){&9G+M)K
z3@l58FKo!x7Vzj@zGm0@w~X4;ck%>(YxX|0@~E(vNThRg#>1$F<lt@fNhz%v!A7jx
zvKGBq@@MJfiUn2XRWm{Y&z{@Ptzj&FCFVv+luWg|#D^PidoQTWy6vYfy452|Yw|Sh
zD^EqtH2nS>6oj8rnYZcuZZS=s1>c_v@|X$Eydcc3X`>PO%EPylMP>R&!H{g;zFD4C
z&m7#-g8Ex;Y+aHYHSNpv&woy-&bLvUc|WkpM%!}d*^d|hG_GFV$EO<<y2V0aX(E$x
zuIRciOIg2im|h85AH?mb9dheR%e=++l#6dKeR1^R6-UL3R)Iy$=T~%ih^!P5I@G#H
zcg`F|PYwNzJHsZcnf-sfYZdE@&s$rbulUQ;Hj{nPl2WH5dlDrDe{OkkufSl!S;Z=a
z$I@xbq@A6Xa-Hq$vYv6Qw(Yz4eh=P_9ol9AEKaN0C;rLGS)pd4zk0{cj(4FOqL$CL
zJ>Iu|?RLXyd3`IaW{CW$7ku)>e71XHg;%%tG$Gb3cBS)9xz6t=Pu&na_uag*9iIhU
z+`Qrx+$P`bm|Bsnwf)h`HB;X%co?RxZxoTae8C0QojiP==5LIcUuT|=d&cVYHs-;5
z7cINUY=LX1J>Q6Ju9IH*WV=i2!z&A)I`|xo4b8MapR-0Q{`!&l3x;a*56EA?BAEP3
zAm`t=eRHH9|7^8&56E4hZCi4>#*KUZeV$kH*)vVM*;w}ZeU<s#<0I#wKG*xl^(EhP
z6gYm{I9JFi_wAamvi8r0^U}O`ZhZJ4#{IylO)Y5F&W#a~Eh%SuShsB3d~o_!kHVx4
z6Bd-*V-4RZrYXDD@ZCx~mEK*KB0Hp4hCG?Hxu&Y;Y^hzEonTF7)1nKH&aR7@BbIai
zK)2_;V~uR$UV$$Tf3e^<6wf^1rgnzst$x~6ztt|0tOpyPe5p`6k+$||&-B?-`6@2P
zWKZ`x#Cj^PcTLe@7MHFKBAb?MJX=`tal?ya&XZ~uMS6F5Y-Bxg)+^uQ=#w<Pymj86
z3%)P@s!_aP`@O>7TW4LFdqSOG>h<N!a(7O-^14?w>uuNFb9^^J^NQ!Ldo?Lidbid|
zT9`TVu3h4@G-7?wpL9jD=9I}|ZU*x@tAc+wb!{*WJ+HrV#=fnKT-_U$bXLz;WF+O|
zrjx2WY0pYO`yJLF1!uW7i!X^-BT_g`<Vb1Hlh_H*T)UsVJln%uxGLoTz2;|gt`|T5
zdbc^MuHs|IMVaZYdMTM*pAD<3`wJq^Uw)tZYD?@4PH#`$x@T|h{QoDPwP3rs;3I3r
z##{L{)l)^zcWmHUd0Zi?Xvcy#?l(#nH+b!?Jvwh|v&co|yOv>|(z(Ht;urfA{n9_{
zU-<Xo?WR9PlYjn*Uivh=aOb0^O6U6Z;wwYMdEdO%>vhUVP`GqNoM+A!-OH1wH9nu_
z^D1pYgLKYZvHN_c`|s-=ZQaiEdd7s^Mwc%)ykn2_jTA02T)HK|u;=Lc`|YmJ{W2pR
z`?vKwmgOe<%@258XE=Z2xg7<yY~if@@s;!S)j7^bURyZ*QuxjC3EE*|yRBVzSbVY*
z3>S`$)nYcxc&YmS@?N8Tekl<K%kO(>OuEL+Zjk+|<8b%Fzt+W4;(ErxVnwnk;#b?B
zon$ZF{jsNq*~-K1bm#Fk(kwl8?yR@sRNP8#zW7j3{9lKUh4-bBcgp*qSILHx7W^yg
zWig!FaB;@naOc$<CVrOfnQP{8hmVit-WRX)g2iiBxX*WFxTRdImb?9j)4jYmZ;!qF
zlFH8Z_I5P0KU;N>N{r2^<=MY46gqngY!>@eY<T#g1Meb>@?E`A%Q&^J@~pEpYMJKn
zrONeI72D3=vNtcRbpD(=Ut#L=ofF+IWr@DnDb7*r(;7FoaaZiwzKNVFyEp9EwI}Xr
z*})mRQ@@;@w#0mP_g_Ah4UQTUxB{zLi<ci!(iZeP{WPfKc7$!zf}r{(`nGj-jVYQJ
zgRU$#%+|cLTItiv<?(O2|0dWj4skZw{EYd^ru%QF&vski{{7stkES{^zo*Z1J=PaD
z^OE)cBdm3n-wXU_Z#{8#!WQ#GcjNM&u3eh(Z^2r%dAw!+4sB5it62HKY{QhZ4mX~0
zX}V<P9LP97m8H?G=jt|(O1Z%ASI>t!{%X{-K67X33&Xzd#ksG!9!fucfAo6fy?fQ)
z63#sB*~ah1*7Ct!zg&TT<zLQkzZe%=3NA5Fxp%$!`^P0-1ubklF7T(Q%}+1MnWJ~*
zq06Jx+m5mi<CaQ#Hwq~B8rnKca2JiVvsU*oVsonXYLR@xp7!AW!;KzKw{G(H%ACR~
zx~?F}UbRQ$p7E^&)oIr}=g$<hP=C~%VcIQv#G#PgZAYJ`T<gAh?_SRHjqFresiV%M
zkpBEU|60CgQ?q3eCl1{Zm@czNZ*lGXuz-xqn)6@Id-RS;xw`U^{q^MhzdDX;>$t^S
zHE#r2O?KYi>n{0x&nxBoo{oGbk<lNb(o&bTXar7jF^--d6|=x&Ygwa-VtDV&OJdXI
zJ}7;$SR^!mQq^X$Ywh2kr)*!?{-(;Me=9S)Tkfn%UPs<(u@1&Jr>Czk`LNl2$K~66
zw-k<?Jn`PgZAIIT$C1;SpB60DW#gOLBrNc+eH!;_Ux`N#16^GhJB?dU&s%cd>7vy?
z360W4{S8a=9?f5|+V^|WiR>M-uQF|JOXk0{n`MsBTXvT3SK{v{1~0CX)8G#a7gOBk
zyKs}>%{?a@BV{`S?3Zp+y%>^uy)4!vVAWCG6t-`6=R=+awp=Oktckf39^-AZ*i!Cp
zJZq)EoyXC8+V9#$lr(Dx^c*!jw~#lY_VI+=Ydt6Uk27piV|sjD-TYRcXvbFRocqg#
zT})RTzFYfodb#mA<&a(ME>q=Z2K8&kp38Dq6+LL`;iDaTbe{1^<&;<1-3NpEP6qv%
zv%TQS@n3HW`j@`zSXuU90~7NTE5+zDvZt7H8IGp&Z}JzL_^q8|ah>4#?pI=$xG!cN
z3+O3a-QhOjpq64!@v7)ncV20)XhWye1?Tummoh5OndqT%%Y7=#$p;6|YgKR>En-^f
z^zdk(P5iqJ7yaermxnZ&XqX52UOZv)@WZm-DXVxqR*Nldv}g7dpYtO=T5^V{*c88p
zN}INf^vDi(>*$ZCmxs%Fv=+`+G;h7Ed2**g?Z0#BD|(fqO{DVfKfL%WWK*!%`}W^F
znTB)jNqv8r*|X0@^J$FPoRBY`F7GFDaG!avcE~fk`OGb!O}Q<{7Vs!HPiTxfDwoCW
z6Bx5^TCL@D_p`ZgVhyW)vt4?gEUT_~hb8ABS4G21_r#f#<rK7kAKq4bOn;|@teMa8
zvyq=V`Z|`Z)wOe0f3xGeQ<e5k=EPlGUehkgO3aU4B5S*AgRqOVrFHCuC*MNWRbKxl
zP%E3C5!?G@sdWC`368mjVcYtzUYmMAXP)=GEq4Wt6lTiQez?p&ODyc(o83vrY^qpS
z9(vuB`K5Gj!0os)!6TBz^IJ?cc+-!p<v%6$JMHHx3Hjj0#T|tgXT}~l!rGVD{_UW?
zbRyIJkH`9Izj&T}`l!El<yCw2m$w*iUY~S%)7+m<`9GcqGEOWtSmXAfj;TCE@TW4%
zB9Zc__hsD6I(?L{A8OIQEB<5oDMP85Yj<hB;a{ThGUU(wbPm%;)6%ck2Bq%(ad*wJ
z>m^(hJ&h#qmd?-H>BTSbQfNx%gHKLB{8IRL#@>j#C8@Og?aI=FUe64U=foa9ku#xL
zFpE!@Bk=L@iTb&!I>wrxob<j=-^pHk?9OyY@0*?phh5vJd6t$79ee*S@swx&h4l@l
z6X&&8SQhh`8>9xchG>{A7tSeS=X%><vhVoq1Q&0`SGFz3mn5=JHeJ%RdDf}U4vAK-
z&n?-J(G4$8OnuN2yk^buNfrj8D~y5`R(Hgoo-ZE6l(^t;VUgR66OzlG^r|nPnA3eA
zsV$%LvS629vHc>!>um`Wy~5&mDlR%<ZvSq(re1BV+EcOKsSoDQu$48MFzdYP$@z9%
z`sD@bCt~b(aux41S-O4Gom-jncx?N+4lSHkuv}BaULeSeohSMJn-ed7?)j8(t0(8N
z&zqn;kJ`s+xm!-PmTYdS7d-2c*q%K9M}e8u?8<XI<vTCPB^u6M`CX#(K-DSPWB<9?
zUVZ-JB9R+7$47);_`}sbr_xGgc;XhYpZd!#w2^J|_evu+L7z=oCi?6Zj4R4}Dsw-b
zzM^jbSGPKALU8<{v}au2cb~mmv1^Is`NJ&tR(ugrJjE{mqFdJ{&~wVk%IA5$z8_=s
zb|e?I&dpw@&|h4-aE((}f`uth<Tiij13^p{KMdZk&AfRv=l=F$(_NF(@`E45*zzU(
z@$$R4cin>Q|5-B@<u9Dl^uTk4cFU@rZMltHbN(|Y&eLA1@HHl>_o|qO%G~*ddX@ZI
zCPG%~`)|}#*e<Tv?e;0+>`LR$Jt=85r;_g)PMmqItoiMu@C)bMW;-g+vN4+IDW^5}
z<>!LE4Gq&RPwi@Edp^4{;!}Z7Ygvb8l=B5ewF9RNii4M((3@?e@bqM}dpg73sUp?Z
zNlRR0SLW~6%*pPkVm!_(wLAUBqpJdZoI7`is9cXT{-}RT>!rH>$C@kWueO$K2sP0@
zRT?h2{8jJmUoR>wO=jH;ZmkQbKKJGg-z0|}K|E#=0VRz)yVu@rHBX4JR4ka2_)y|w
zCTDTLL-x>0fAN;k6<)eB+0_-5{9Ts93G=z7qlK<*IS{pC-KqAslQa!q>1Z9h_vQEc
z9Sv75M_m`**sirUNc+a0CWhluRrVVBZZ8h?3d}t}{rTMUz7Jwv{S-X4b*+xB`toBN
z#l?NznzHWDI_{fu@uyjCih8y?Lypfr_n)757NwOL=J`G4_-wvyy;6wblvnTlAO3#!
z>ulmN1II&WIU>r=MLk;O@^-hq7dKBxwV0@Wwehqhod;qv6Eh3q7sTcsw7z-ufm;3g
zn%B>yTK2_VJFR=*hwhD_y$fQ^7U?djzQ|+cvP*>bmgR4qK8CvrKOWy%yn6Aik5>zR
zw@$m`v~p?j`Ita~q|8A7GoQOx|4Lx!z4<MP&-bd{ig&%GZ#PZ8&wFn|ytLOB$+@ol
zs$~`vmA6LM7;H~p&l$07)t|rxO3PyuSLKA<51n-~{La4iY3!?a?>7_Dj_1^`QCEm`
z(^hw$JM+sX(G|5<vrij|8~k-k^%G+G_vfL+zi)H?Z9n+mgE_QrZTbp_Xb}@3fz1qG
z*Qi~&w)u<r<6z^dIZ>|b+$FNkN}5hE`WL8j^+HUk6yvHxN^A1Cy><pix9ng`n>^#a
zu>_mww>}5O*YksnC(d7J$p2*N^_@kmL5?@;xZ*Zwb@gnlf3P}umhY6Vmiyl_ms*|-
zyW`z{NtkV-@6(ho793JZb0WlRXDV$uV=Q$>-`Xki9}6ds#+zN7PlFAZXJiFGxzu)&
zZQ<Jl!_5_wci+{yYVe9Fw(OFl#2&Y{B8QFg79Ss$t(>=K->z>#-i#Bja-IFzFY{yJ
zfk!P-GuEX&-eqexX<FT>Gb&3;<!q8vI>hxBTD8x}<>jC9RcLR`M``5<v8Z_>88TOk
zUWiXy^xr1z=YL0L*AT;L4`tVST~)dEcC9IU|0>7)3jM6YO+PkGalf9s)l2O5of>`7
zcb61$6aQuRZQFh<ZTUrmTv;iv`ggT!#P%HSmTUj^N=9H~)7D&vkm-5x3cvT*F6!Md
zsd#$+pLjk0z`*|JS6$`G&mVfHI*Vg^!aeH;D}!fL7<X;?+M96i<Ib=$Z{0j?_EpAn
zGJU?xESSAy_J<Y8uQq;}QOSHLQT?#@@;?Wkg|bWh{_{}N=1be0H8DNLdAF2Rrr*ib
z)jM3ZQ2g8Sw2Yt@dEd7h2Q`-dJ5v2xVNYy++~teQf2>@%t3BU;Mcbxq*{d##4s@+&
zQEIp*I$2He@?O6FgkQmCpF5m_9w^Vg?Yb(itJhV6uVP(i?6rbM-UTkpwp?EP_@MMr
z(RSSlPnXUz`#$$f!0VOT(JOkN^fhX?hTnZ0=^fIn!auiaP3OGgyLz)@{EwL(*t$j}
zbZ_$uq3QBZ5{xGVzYF#_afiX9(1nM6dI)bf^Vdg8rQLUL+}*Nuox$(MZo4fSH%(vc
zny;n(Fe5a7-{XI7a+;IuGFbaEk2=fjn8SDa|D~<B)5S}B4o(UbT*l=VySzz~>r<GQ
zt@;^OV=eyW**3>|rfJB###Fm34pg(x+3@JvpI>@Ua~eNhYq+U@E;?NB#zM<?7WZ!2
zhn-n=jPD3XueO8BSH5Dk&2tiePFC~UU{cs6_3+jand6=5{Q?sFZ>73+oGyJmO=h)P
zuJ^G3fm;8oWgL20=24Cd>}q+|)<(`flw^@&zd7J?sKDh9oW;Kx*5<xpVVrm9itNVg
zsv-HUzYM|^o_gqCt-g3!qi&tSo6Dybm_#p`Q0b61i#?#d%|L72(-|o`+$oPIZ+hr7
zv#Rjs$A<ypGbUyi3Z4z$kP&3Ud3visiIe*N$EI4PdnB{1LV4YbyoCF=uiwD6#pa*Q
zq`xl}?@e@e77?;cS@ksYT^QHI^)}o`d(s?jM4A_5ADB7+{6oIuTaT@b+Rk6XcG@xW
z(1k<4eZEOCN4T^fU)^r})QtbADVtx4rQ|18kF(Eb?K-I3`Z9Y%nXTCFG*{j4ag0|P
zvf@rOUU6u1jJxvQWoyqCrL!C2-=7Hhbtd+B=p!|6*;&q-erBfUB2MMJ(c3SZ<g8g#
zX{E~bRCluPZ05PT3Nb0C`d%h{d|VNpEb;GDwBN-`YVnS?Jwj1C_#5;4Or54bSaB#R
zxzJPgu*>C7JXJAjp)ITVJ_*Q2sde2dZR_6qxcIG}Y<rn(Y=E9Y{lW&OPq%Dj0^|ek
zDzUHM(bX65`RGOIX;t4eEJY4_8D8~hYh*fmif?;fvCCl-Da(D)3ZLyO>MDENdu6uO
zKUX_5#cQu&W-8x@nAq~7jGxb>O2nkQn~zRkIzvfD%Zhvb+*7P8v{#nQpUqXlVkKE}
zN&DS5buI1%OH#gMWV8qi+SPfViG07;-}cRH-zn`s-&q{l)f#(lORMSXkU1gtuYGkp
z`EvwVUK!{a?iMaT<`+~o_ukwk*P|6K9qe~qvQBAZ^~!=(;Rdf=HnPOn-dSROp@T8I
ze2SgPs!f^)w@mE{ikYTR=(;1TWqZn^PLWjOJGWid>bu{0Wfykwnt?>Rl9I*}w<w<7
zQPuOJ=g*!wjis}6&F`vjj$BJa9oEbhn<C^j@$}lKm(L{KUVY>0?7#duuCGptq+Ff3
zUMe`(%XrC)D`5tkt)@ND-a0?;>UIDAil+;$w_ZuKV@bclI5S&LFMX}7R@Yk-=1dD+
zZr$zP(FV>Y+jUM#M(9qGSS0Akcj(^JbJMuETJPE%`Kq_7YTj(4>U{<hQ?|DYzABRc
zGbPP!@xEP3p^SMZ*6Thn1#p>c{!y<hbuIp`gP|TvqTGc{!=|0<T9?SU<e%7(xR2{v
z`ohW>t%>DpGuvJL@4NBn{xmxxu=Z2f+PqkisT~J@?`(^nep6cJ_)SkYLypfa)15EK
z?fG7|UGKYOO2Lg!=H*YX+x_pJ<}d!{_JrtpTg~OCJ?LH3v_bXF(GPQsu36mge%g{^
zvq#WTwvg`{L(;8PdHe4(Ru{*wbn)n_$c~%8r#0lVp@mxjpO)UHug6Z^K9aa!ud&n0
z=e2D{i|O^Qo!g&=#(aqSy0_MaEm;2VhPJAit&bTM{Zm(dC@f%D*lOQ%&X#ZSKC#>o
zZli+hMRlqnEtePHyV{)_Yx%vvAmjP6314F#eR_SkYu?I9x)0Xy-G~YExKY&FViVpm
z=a=M~7FUK2qxYZAcw4ofo;N#EX7;`L`|74lSeH53r1IRXzTd^2{*kAhiYuhz*K1@h
zwBvUTug!N{lD4V0a$V$e7Wb(IhWgtgzb^U6qF)@rzN{tag$_rbj#X*fkF^fsSDmi-
zWUSp(o85ALXW-t$i&c31<~IpV&p4cPv$+2B#DB}aKiPlHLD1-Hu89TzL$>5w>GKs1
zntwSdEpoDH{r!x4GUegldK>;;kJVgwd+ml5N3N9~v)~c=@pSS@_Pd^Uo|Z%{Dp|!6
zf0^lR*uq%v##@UUW*2=?J6}J^?1W6i)%NcbL*8dp75m4ooaD9f8jpx?y*a<E-kE&H
zX}oKqRpas&nu%1FYy4mTLaJkS<2|XPzfDDS+i#^@Q93rkgEhoo%X|GqM~&jl&=a|N
zNjraaEnRh;@$O13-{`CdudYQL`)K9F`pZJ%g1B{cf`-@oQ%w&hU7m33&0Imf>vQLw
z5REnv{_i<)yE~^&+`Ki<cta}kFJ3=hJ3k~-F7Lc_($$zo+o!A*UJgpr*x#G@9%*~D
z$o}e*r#m(jC4RkeYo?~%Vy4uy)Aq$i{`5_o`ew6!URK$+*RO6qi}#dXGfC3NGj8|I
z=(OC_w^@JZnDfPGSw*+s5^K=@t^NCJL_xl6+eDU&6M6Nv99*ALwYYurKF7X=Esyk`
zXjOULW_`3QVX{Z6MDH3G$B!LB(Iq^}huzF}tWZ4saI48#tvy@4k4A6dFP9K5P5u5)
zvpK2F(ac+=CO7~2rkFL?6((pY+Xu+~K0Wtn!0K&Mx^EW04B9^J{8@SCr24a;Y%ad4
z?e0*xdRs})zx3U~ZKc}M^8;#EN5;%Nbt_=W$xTO3Ph7;l)>2{riPNQ<|NrgHnEwAE
zn^!_k*i!|WZ!ZL*ouU>W-x^U_oNsqu@zMM)!7%0NMssCWu`YYrX`XUn-R$7m){lNm
z%v862`Iqmfe9?1-uU>ZF?`Q^o3HYeGEjP(nVQaq)>(jX9%O$_|K0d1KbWq0Q2g_Zb
zP1y_L611*ahljs0ZZWX?HCJ)*Y98YWsezHLQ?sj=@7fj;P<Q;#0jJ6Jc`1#h7xg@5
znr?U9Au7B#E4X!<t?T^<8UgcFzI)`ha{W5JJGNlqZZ&a>@ChM1*%WWOJ$x@Ipq0tF
zUGn6-M*&g?bVW~Zywe}KIWbw^OiVG!d(Y+H-B;H<v|3#9R(5{CS$VGfbzTdLcCc$l
z1RJa7JB9uc`KWNjY{LV|E2>A|_BZtPw(m*{?|Y%v7uRuOS`fFS*?AF3$@~2bzlvr~
z*>KTW>qPpWgwK*ZHy7VJa;;!_j+uM;1#UHUtpHV(?Z;#MgnJpDP1C#C9DC`xRp{g=
zFGC`}ZR=KRmVfs|d#V1HOhwOVk4s`6p;A+x^F&==bTT{n$6}%6Yd@5$W;fd0RyoeQ
zF-){`XWT{S87F-|X)fJ-(@Etlcjvh+zwTOHHCB;5Y{M%ieS7aF72Umif_1W4zB20y
zRoAvhd2yH(WV{aWW%(iK^5xQs*I{jEnZxbk6V7hR=T&HVlVSShJLA2DUyqda*4}?p
zJ+tNhWZ??m=iTeFel9BCb$oif!b^K?QQkw!M*?rKzF#57t&;oKiFxT;;gG-cf)ca1
z*Y9EL;CpKxbogrZOs&7Qi$g35e%{q#`W|v?;hW{RF8azfyr}rP$iIBrmRT>VIb}Au
zPDtoG{-$$6-&4h2{hp6<2d+L&Ir05*G~?zsGvudLPk41ouP7+3*r|J#Q}bJ=*2l%)
z)-3tOw*H#xwdhX1z6UMrmd{R#AK$w_g1Mme>IISIO+j(@Z6`!)TyD0Wf4_1G=k@h_
zs>|k<vSx<7de-^dENta_1+Mx{(cJvunGW(xFWPKs^3vk<(C@gGti+OdfnEB!zQ+g0
z5B)}0T|b_&{Oz*Ef9v<AuiVUAr~Y<z$UWx7JvB0eH}uYfrx#v-l<iu2CuXv=+Z6W~
zor{`ZWv=@6MsM}OaP^bZq`OrnUP)sR(2#F=lxF&1*6ilLllK%lw=m|Za7_2v(|T3W
zcEWv=sKSXinZj?WN_Q@9c;NCz&Dh9w<K~Enm^{{q!sH2O1(qpZKKAh0#JJmmN(ZBF
zc>m;yIyO=6)$$83@(kFYJ7t7DJ?5aFt9Cy_Xo;rDsU448>hEw(Rn*GnVoN%HD}}Y=
z+YM98oLBSXlq@-}-#zxA(9}Kf>_@-Q505AQso&Fh>oTjZoo?^zbu-&neiHOGpJ%`4
zQ|Gp)ne2b-D&6cK8cVottcvk?@>#p1ap{)VivkluCWM^at>)(L*)vP^=)S$Z98HGD
zMEC88<KAe)#G5<))z*M7%3D<TZ*kT7T`1W#af$Mz{M6mgKbKaWTkAFZ#)lr0U1xp1
z9oT-cD34Qb{`yv{vce-rE2MtjIvO>d{l<#f1y?#=w59*PZc_GPr(?I^3T1(l%||`E
zZZT(UE%tpj=eqQ_h*#G4Igf77UN+%$?dz*7J9p~3eC1GJ^^m?DCU$sfQN~&Slb4uY
z-|=X)E`PF8C2no_yq$bbMv*P@>#nV9`Kq${&=rxHom)S%*iK!}m2x37fnDubyxE3h
zkJw+nx2v_^*_D4KI4^KIi<AQUqi+qyyVCWi{`*{=*PXV@Dj+Stqne+~z<rgh+v2B7
z>rRBMOUd#rT9O*DIq<`5I~8s2zIK(D=?yF2*Segr_~m+|I4So~dQ#BBnHn=!&68OZ
z8W*wpNN!KpmT;ZjIw~J^Z6aHb@A~k-KC}M7)DJeIa*r-~Offy?nH;2(d+~;n$+L=P
z-giHnrYW2Y@Rmz1aoU~cut44S<#VC6dzWemyL9(0a=z|=e(QvqP`=#R7Y@vn;jdql
z%f4GT&c>j<<hiEp2a6RC{`6I|e|GCw+RflTb+uE)`{Kg(jqktSWIVl?(QAFlyMDiK
zQ|vrl-}*1OpphcvmEaU}){)V(urg8g_5#1Nx+nH)wrWhV?D({qB`<vg>&#w--qKew
zN9Px+sn4{1#lf{!E8%###?kW~!QOfkybPlf`>rYTIJqqm-FN7gRF&V8z+2}QZ!t(n
z%H{oh@|73cQnrW%d)DsLVXwG&x9CZtj+Sww$TRkpt#iCT<ga-W=~dA`>He$6wR&rE
z_ior0w5OawEVI5>##HN8+{KlF`!aZISk*cMw{l7E;0!tIn54yDzMCU4tK@i>-%aJ>
zwSBo)RfB)M2%hk9_UZ5YdFyUERIZqx^e}eA6+sbmv#Bo4)_ElXr)EaH-4#~r{^7Rw
z*P5BDZNL6pl>Wc<ws!oE<wp7Ib;Q?9vN1n-$8)i2+^R~pBI`wl{WsQJ5##8N`SIiF
z?1j41yq6m~BwSvjxPmWSasBq#I3s4a2c5r+{+&oWncls9$J1}d$Bu;feK2lW;{Qs8
znRTJ&ji!%>kFCf%wn)v%(I+g^<I~4a2YDZ9riqKckoqd2pdkL*@bdkYVmhl*<Emxl
zKdxq|pH-{yJ9oXnftjZsKXk5Np&P%)=FQ(W_r3czy!px1HvL}C8Ur~w<tJ?Hb@LOG
z`JJZhm}$IoqW(mIZS%b!Nj2$2J&O}b447u#CiJ^(qGs8p9$S`gHD=lwGk-;GneFPq
zxcQ=ysiSkepwjx>lYAj3n6KV?7(bcW^W2X&*S9iGoGNB#P|v7e^Lf*8CVL(4TY5LW
zC%lo(|L+?8?Sb2DEk@7i7iM?$Ht6;3E7^64@#x-bM|kfgz7f#r;pXjcE>WMrsd#W|
z_4jqf=9=4P99ehSmFe@H$^O<0nq8A#FdMbMT)x35qU_|9^~#>5!FL$9WlVBT$Osc?
z_IPkxVD9A%n>-hO%~LIlzgmUsa~oUU3#xr-<T%AhjI}6Hsd<uOlpZsaqr%MZX2l<}
z4~4ClJRN6j<{qqU$Nu11&SY=Bm|v3nw!GUBwEFvIBku5Nvur-_&8+cLdwt0vPEI^q
zFXzmeQy<t$0t%kF1cj}<TXWm!r@{FH0eAMqwkN)nHM1&yKlP#09kI7f2ElV(S4kyQ
zF;^xQm)}oSRQD`3lAD;e$lsI8)n%h?%r?cV`P?68F<1TJztB|_d^TL!c3E?`JEP;7
z>PrdstCp{;ogwk`?2orM*#8%;VJzHq!)4k=*$b}}KmQX;`IavoVtT3ipR?iSh=?^h
z+m=^k{7frNFWg`e_=ml2^P(9i7tDHG+IQVDE??&!%g*9G=RR5;mlp5tRN(J3x48G9
zqWN*}Y#4j;W8b=a!iyMF`^+uxJ*k*{+0xo^o=kqy9`?NpXG9!%F3Z2pxiod-#){21
zB4rqUJ`~?G!y@ich34i6hEL5=0cSr79ru>e=WUu}5%;WO=Ht}9=|3y`X9}K@(J$U}
z=HsV?LZK&zK5`w`mRVo0Cy2A?VMXg>(Y|ntxQEX}<@U2@FF5g0>-g$9=T1HTe&^td
zXBAT)hxT2s_#?NbBdhPaMcn3(MuE*%n?a{1w;$_0I@|UB>6tqeh0RkQv6*%{7*>B|
z$UAX_cjgYqOkEM{?q-E@hzFH7M{s-s`SWl6o4vPhz0q2yQ!)SiByqQfiTC~=tr8Fv
zuv~p@;kUTo8+KLKg%;1apDEnn$e}5axyK}KS@)b{+K;DxJXP`Z)zyNDrz=wT7{%?h
zT%9*@eGbRzwhcK8%Acs^f3*uYYe{vEFuU-yX!7=|h{n5HKF<0$>*KA0qFpD>RzTp#
z9VZg^7{p2F_lU2H+oM+mVjO#&ckW)yle|3;_;1&VyA>dKxZ?0tX}LPEsWLYU7AfCf
z(tdv`Z{kgXX}l)u?m{^ibfqO?o~iAgyj;KhbX(rd3;V-!Ux@Mk?O^-Rvp(m=G~T<L
z7Dj_a=JK4czFl#;;&g?($qR!+v(4h9^n0i0b<fc&0x^zxKfd}{Y5K=g5Z_gO-E{)w
z&DcHKQojUw<(D1azI*cFlP{mFz4!bS$QoVgU02)kE;?K~^L$zF^`7fJ*W=Wuci-u`
z-XngD`>`vsKbLgN9D5CddFSrM!u>f<ugLtK0T>p1Z3FoX=F-0%Y+{vC`C$0IgYBid
zSNu%_9qAwUef`w^<K?FD9^RB#v;LNWj?^)b%X+u>fMB`H;e?7)U@vCwDfwy?^TY(~
z!}XI-yZSv|3icS6_|BeiNL*fBT{7|Yqu84UYs`4M+X`I{x4(RN`|`=_C!a#Rm<Y16
z?P^=%MTf~xVP3p+vS{Mr3P=pznZ7<p0_>KETnRB==0y+1c1o`Y!QGa>OD69AxC;!A
zJyzm>%nO0x`4vB6;pQ6jWh6{o{&CsIz8r<!f7BkU*oT|>oNmj!>7aJmS5MtPe%>_R
z%bOCTa%YI~e(q>HZz=xhp4mO4dq(#Jc^|Bm(C?jo%=_`u)zc4yy?3c%rr)&g7>MU8
zOk<uH#YwL32|u<v+O{Aff7j14i<@(0{LV!`(UxA8dU#9Dj4<hCu7^`D9H_nxPMf(_
zKVixAqs7gbCy+#Xtoynv-{aPghd?Rv>g$I}<&R52&^$d}ugE+O5`iB+?B0|Z1$O(^
z#3+*q%{ARNJ?6dUz2<l4Tt58%RNM7U54Ywd{P;V2&-uD_ITgCnai`hZmhL&Z>tyM#
zPuVv=M49aglaA|d``*#UfB9H67+NoSsH*?|9we2ni`xfHrK^vhd<IXYd-aMyeiVPd
zig&&1VeX3`cD{Uf8<a?2KiTEF4`eCGuGJvP(-HaKw?Xn_kx}fEG;k(7bn=~2`^Q7z
z2!$uoW3Si6$%8!?XC7Bx{H<tW`^Pp21SQh-sfR%>&aKdu+R^?Bnoi$$w3UKV(1*A0
zo}52?`Q&#{T<V-|OTD>ajhRk2n~}P`dG4OnJ*j)XTJ4N^0%Cv?CoJZ`={5F=2{^{`
z&b@+o>zMZA&|N1=Kd#G(n8tfL>+q(G2V&FC`}*Inn;YY|Y;tLLP3qxIaJd7HkB((S
z;JP>)Xvxz3@JR_Q!LGN5dCEMl{OhmR(5U1!e-4hnNHdUow<boK=(Jtjy}j2Q6oavQ
zUp_1c*>d82sPv^piv1Zs4E}!pQ1Yp?$a1Fo&z+kTb)_$DQe11c#I^Zc;r`6Wx{q}q
zuPxaG%X`O8KU7-&@lpk-6x~(zbqBaekz9Xgj%h43eTu*DYI|}}=sP5N`l;UsB~Yl5
z&T<z&B;Eq2&a)M7ODE0-$KRofhp*tp3#iPIImQi2l8?U@z%$*h)z>e(zB?ujf_dj2
zJ}ggljxt-4>Kv8p(cPBv;CgX2xNyneRrQ%?|E3&`(`~=vwl36>dId^=>+|*1FQ43X
zQd#O%`<;Mkys0+>!lX|vT3B}i#GJ;P3RVe|Lgx=9rQx{|=S6wF8_(qJ`uz=*Zm)ZO
zefl-#=7xFN($|Vp4_B?bwPBvN)V1WI!&U2YBF<0ceZ4d5aMikui2jqjuXk=rv<){~
z^L!QW>z#`dZPm=zJU`V2*L}6^Ys|$4rRCsq*C=jhRr?*NvCNZ>X`4S+H2>o%f3qQ_
z*l5lhzk9D@i<tBF`{}7)J{i5&JI?_U>$w(|psbp<2LvnNWfFJz`w1ZTF;4=;Jjf`8
zTpX3ByBWk$TN7==P1YQLf^Z+)cL7f__b5xB>5O^iqF1aWZaH!O?0NhCTJ4<UqwXJX
zrYYnJNz~EXyz?T?ZaO&UM5^BtBlRNF$<kq)a}3mrW(w~-(-|{e_#`}~+v<UfB~Vek
zy8Hz1>PWL0U^UNN)}8MBV<5Zl?pzr^KlS{tdOH(MPqP&(O}CtQdd`WfJ}4H%`l(>8
zPVXt2uSvU^0CvPo;hlQwlU(0{DpwGGsZ<WBX&&=J>KYVf^N!61L+eF~K6w#|2s5_W
ze_S>_fBEFn$?hgGeoxvV`B?X{>_q`J^EDd~zDz7S%!P1pspUi+P}>1iqlUurc2D>^
zkPDGCO&9)|SoPx@xX`FvU4D{RTL&K8;WO@RID`mPke@->8C2ms+*bo?O~HfCU}s`$
zYBM;bQvIInn&SbhZl&{W^gclv5(f96O$l&ZPPd%+x=1PVN#-7S46n;8I{a!HFHE|7
z&WZEB*Z0<@K+G+*oG8Bx-kfkY-<ep7@a<LKyokG~PQ_N+?)64f_N4vg_3hIqzg+V1
z*4lfj=4<j0;lAr6@9Jpe#E}9kto7r>Ex`?`(#NHs)E>4S5ecBA2C9a^jSBw#+S0oa
z!SYHC#R5>`u-lXvVa9X1EpcmNgbB}N>4M*&!rSkfuUrxHC2&<6t#@0|Tme!)rS4Id
zYCFlhBX<v|f{NYi49bh3;@j5G4BmKpRSIqd@2UvTHBc|&oh*F~ExN60A&rW+VecoQ
zNFILDW{QaY>rY)a!P~<xmD<4#rd=O*fpSfD1Zu#bRKkZ5@qo?tn%&kvLA8|aN$<Rf
z%Lug-g@5jZr0uoWPd-)Bf2{j>>BmbIlF+sVs5Mo7f;T$~)n-V0Uvm8=*LB_Hh@hUO
zX1*p95tn-ELazHj%^-M!%`7@R3z5{aXWiMb451s8ejzd3`tjFeXk84A>}y+4BOBg+
z1VuKI85l13bodLn6hHYiW!@82^EKD8dL38w+<s?693r`cGdHMpb!>Hax$EKVqQh0X
z@N9b>jr-K4ZiZQ$F&I|Eifn_p9iT!V-AoDnp6k8Wd#@ix6kFgD1Ie+VV!)=y90G0r
zfSZi40vcp>6e3t(7b)5M?BBO{j*t3lwf>9<HS;y8h)6lnxyLmQoW@Us+KsTL2&^%w
z{dg)ksi73mDHBgufErzpq8-x6g0RnY#+-iAwiPA)BK++wzEe^koM&5Ci@!S^j+%&I
z-GN`ojTBH~L$e!}%$I@Mnx_%*_mqF9B2xH+0^INZy4f~R6>1tWHzy!MrF+ha=qIV*
zEPVLnJ9x#k45=d<TjY$A&Rr(1&+$M+|5W=YZAi)Rr*j^XBsgtAV%s<l*4E^H+zKk(
zQL+S5)wI6*4k*?^?aG+FpcWaV$b}Z=l@j?9;0|Bak62`ntOhr_kZLiZbIsb)r=Y$z
zQZKs7zjGlXLc;$}K$ZlhqdA~X(#J~`@Wv;of>Je~f}9mk@Lzs-`|inyPf9+8oqy7{
z36cH2I_GI10tN1GP`CXdyz8)|YC5Q$2&+`Ubq=I2pSK6Imy4ndR5fMp$+YQgvu)|i
zIr02dn~A#7*-oEgrO$>t74<bIoh;du7qm!GU;WeJEgz?Wp^8)hxCa4>-(6MhcP301
zIl1KHrHY@Ru7ydQ6u8$18>Wz4AG6mv?}T^Pos-Jam*$)>Q8xma_2t9cmrq_lDf$E|
zpg-;0qzJb2@|+VjJ|M*@mnzOy=+4!FRzj^G57quW(N?oA$D>&3GT0FL$<m)DDa|wi
z_mSj^oM(x2E&&(c=>1*Un-iY7EVG=L>6a4cr_ycHYu;N9s%|s=Qp_exUuuM!cee9P
zT9LC=K*}uPnPBymsCBXNPDNP_NXT5Cb3#S^lb9vA|C?$R32$w|$7Uqg=Y7>f(XeOE
z3I5CAhP~(Gt)LdB2YZ(f%rCP~^1gHcIS_33GvrEXeK({z0<DzLl|hGTDh?lhe~Jyi
ziSA|=2+v8XnJ-~>g@np1;h9fd>hf-Zhh|QvT0dzs(KJH%)k6g0(WM_RUA2DFR<ZyT
zh2Yrw3~JlJdimE~^&U_C`07#Y)`N3S@Sp}PsL?rhUoN;6T?WddNZv@Bac2Ub2qeg%
zQ3Ubvp^AxmMH1lg4C(c|LCFkF8A<_NfN++vPabmMf>K+_kFxSl&mX>gQax!piglBS
zs*sVK3yvJnxD=>ndZ~2%=x$I^0M0XK^3c<d#;2*gNRBQ;4HXRKI*`#g?#Ew`qgGAG
zQ3Yyzq14ARh*bFo7OVAcM?<CCTn`)UWPCZlefi||lTV+tIq98uV|zb%$SCXZh75`O
z;BF3RJWd;w(ZH#B>BpiSCqS(&(1;wUR#eE{1B&50bK37TYyo$rc2$IfT5-r3ClTG_
z^3V~vp6hYqJ0;gkt_M|2Amh+|0vh}PrD0pd@Z-BzvgpR?z+2a#k%EIrSr&`U@%Q6E
zgA<e8O?*(o+own=f8um-0rshI(@wZA*LOooXwcv)rp-l?>-FwU<pq}ryHA88=OIg9
zjC2MWaGmur3N(@h8KsNaTM)kwkw#8;roaaY(5qtfWOld$0zpGq$eBAH#b0M2xhvMZ
z5>l!?iMtVh^8unbS;ptMKNHf-^eYpG<z*w-XwNZiQ1c!+Hcxkgs$!%J_1P4qD62vD
z_axW6V~~-b?&HwGy2L#NU-?jU?Lq1<t=|tF3A%^WUxE}y;368-?gEY8+5TCVgXE6l
znJ9TT&QC?{uC_E%`j3I7|4ctnG5*Bm9<=3A5-kI&A!Z^nqe>U7@AMQiy&(@atD+Q?
zr!fma(9nQEoP_=%;=I1R`wplH0`elf83J)GC};aW_60-PixcdhAjkRV_B%*%`V7?k
zf(*ri24jCZ=OG2tWKi1+)OqTij%(<7H;UPy5)x4#pGT3yuoux^0yQy_s+5_i4%SB+
z7Le`-jmLt#++&Wz)3P@cKykLS-+SLC<P82x(HzMRv^x3k1mxH!qE7Zf4yVtEmV58@
zyss#7o9j*>m)xLK39FSp?t=6f*Mn-M>1eUC2|RTI&bDYZ@@w=6+;n0R?^B0~seUOl
zg(IK2TuUx;whVx<r&&(?I^#|NpGf30muZ$DC9z0@BAzA-;Dr~cOr65)3Q;suI1-eG
zU{fgY!RZyeseUQ8YUZd4e<q@Il20`%ot|^TNPW|pPS7w_(E<otr&tNxoCKxt9`R${
zpwV1Vfql2)E~qttt}Jm61U`INo<hJoFA-za`a2iOXly#usZ$JcLl(MAPvYt*BL^R-
z(S%ViL%frQ60)V16Cw37JaA1XOC$RpLpjcR88tvpBIa2@{bYPT`kD9()EasHq{zts
z(<Btz4N$@Y+)Kw^GoxD$iFQ!!+;S2*VTEB!Nv4lMaO-tw$qnv0oj}%g4HTZB8q7S-
zJdR%+I<W*w^6h8|Qw5Q?K|SRHM8eVb$<tu(f<&s3`X)W~Plvy}zI__hqxjU(mWL86
zSAEgcPJ&J<W$vi~RaH{@z2T6m3OTAkwJEH#3`)2J%mj5|Kr^D)D`-e!0U3#_uZ*7F
z!1=u5a0Rr2hWONe@0YH&#3JTPcemHe&GXymH*cSfj&xh<;SD(wVbDg|cJLTqxeKU)
zMzC(4U<RF*an942taKVFpC+ONp(%QB<E)pFa~mjy!DAO&y$mT^Ks{M-gBz({1_zi<
zu~POd^dzD45!_M&7i`lkCz_&0E@)5$G$DGa;^j7rCv8TWn-F=$Lj*BFYW<`QwS2Mn
zY0ocLZ=Vbqqeo6#%TSU$IPrq!yFm%K1=@oGP20$#1Q@tC4NBi2>@U7kIv+F$Ad$bL
z3f!Fn7e?T8JINI>a{bU0C4*L>dJUAuVcq1uj0#jqgun6hl#zV}G7V8Nzeg=7uA_%e
zBy7r*`}<Q+vj|?GVz?18DuT6UhEz$Qssz*=0oTkkg(FX+*q1#MEzXe#?KhyNM1;R_
z%@FVPM$7%MlHyVYcmx+Y&!=L>A!sz@8296&uc0F%d3(T}2&C#17S-U{erP8GUfp4u
z_-P_9Qsw)Uf9FA@T=xguB!Q)Ev?2R{pe6}C<WI>LH500xSMfe`2}9Mk3p7~p@;0PV
zy}e2e)C7XrHx=B>f)q8-#0i-nPu$}nZV4IL?7bel*BjK-f#pze6%MJH6Ze3}zK~o6
zX+0t-g=8~u;iaC5;(@Q=h9;8INbo=qsIiGWm8^#nUZ4^cV+b3P#xTrdtWHG^FjRTa
zSR7KcfQlqoqwH@XyjQggG{AtS3|w-7+Mj4O6x77&!aH}*NMWA*Q02GLWqW>q^?3E^
zlPf2+-)Xo#h1W+aU$TEuqvy9&=*$dg4aQ~gjIxw|JGAq7sp93U&N~MnJ>;U|E*NuH
zg%GdCrWEG<_Q{apy}ho}auST@n#5Jl$=QBl60eUGsCT@6uU-*kI8#DD#&)9g3^VWw
z9Z>Tf6uF><9928OO-pdw>#Fe1#!SdO8)));a!$e}NLTT&|0c%mpcNg<J}z5re*Q!D
zG~TnD9u_Ihl<}Ll?~ac2tW@XCIXmWfsPt!KOuuvBHh5&(deOqzB4)_42b`nS;SZsU
z9*T;u6+s86cg*pCtU9{mFww^k5@_8uP%G39pR9v$GN$oP7e1MS-X&fbX%^$BGWli7
zPPoTvk^P7$n(oX2mB(#qH!tW&>!}N!1T6`0-kg&$jrVMpbA-teRda*8wZ9<^s+Xt1
z%aams3WP}?TBNAAvk_L;I>&%K>HT==N257!;1e#XyG~?+M>u-I*TwOQLo1)zn_r3j
zJk=(xG3jK%ChN-8-*;|$$kvroWI5CC-YZ#<IpF^8;gc_w%)u=w)5k|2Lkl0!tOLA{
zu-me_7@jf_OCDgeqXN@-6SperNP(uw<c>*$Cr4+?K~`LC&v_9hz01{k{}Z-l%kv>q
z)0e03PIcZ6UR0IkyglOuava8koCWgUrQC^^;f<#%_&OZz@bz(E|AD7dKpjZF`|tet
z)R#}*UNv7^I&Xo8TJQCq>$4^of}#y-`(aQs5L->^3hD)c#+ctddvg~tT7@zHdjYyk
ztH5%i&L@+55Gcs|c#?G4uTyP_u+>i|d42Tn`+;XrB&LHh4`igdF9XB_B~#GU*mT%h
zj+(hK;0Q#Y_l;ZH9V7n!RNKeB-~ldolbRWF`)udd%#HE8_o{Xsc<5_WB6zjb<4K@Q
z2Fi2Z>ZcJs_xOFtla)Q*c?q{b3y&)9p1ld`xP#}7dd=m;!P5nx)&iuhfZ`G9e5v;*
z+A={ak<NJMJ$MTnfV(_>Ibz)tDEtu}_|KE13#=i{e8S7A_B!V!pe&~XMcrlau=4q-
z;00S|KC5{zFG`F9r3g?$?FP*~fri>Z?IHN0zhkf0AzJpkEq@n*6DoM65oqoQwDJqt
zpxL0*nRgD9`M?X1U`wVZNmsxobk|*;9)B9V*y`4YD71<E4{=4z`O@GuW8wPo&~d1%
zk5h`lGr6EKMtJJH+IG4##R#;x5mdr}MlWAOa-+~m&|pg$I6H0yFXI9)t*SoB>mv=C
z`RXp%v>Vbn*Y6QWp3c4A4xRkDes|8^m?y|JA(~%in1R!%ATMaqfE8jM_xx1e)0+|_
z&2(1to?es)FE67(MKoxF50vrXEB=mohwFo~Ab7>*vo-IaMW+!geS*9<=Z#+<XsMB&
z`lORjQ>-Glp9lvpfPyU8YAZd-y9Y90{_GXJXp!*qgHMQ{%-JFa)C!9EK%LKNpfytw
zVbarF5AS-ycF*jd!9A0EC10VFdf@d^he1j3UuA{WPph4C&P+A|P4B_hJ^h;iT2-Yh
zeJRTs6pt$KTFw%_u*y69{UkI+kP)?4k77ZCSKz|)-IIbMtDpCj;%^pMPV|vlZ|k=W
z<mXG^Wm_P#pMYl-aas$i-a*U#P*zrzpXAL1P0?j;Rn&!-oZyvKppp}`$jWTV)wV4e
zC*+HqBh99yI!ESebhm*<N&o)(R%A4H&1@T7YpFnMryhc4H9#$z-s#6qUxyZ?se6jP
z>VYQuPHkGa#%zi!xLi!#v#TN;T#13&XlRvSq}i0KZJRQ7fSPiUhTO4iP;CKPSppgz
zJyiSiRNJN;jn%xV7eR$1Xfb0Ps3Rf|9UBErY+>4o+~ovm2h9!6ww(CZaOc9<AeG6G
z#>>^|vppxiW$SVY+O%+{@Xa%wIsGSj-?DY3T&g%-A;b$+JP9<;-!}O!)Qg#We8eq5
zoovuTW6;=6nI|a9dcyP0Ip<}lxo(5khUE~$Ahls^k@M~)kgB6?@@43FLiRDVkzlan
zV2h<pCM(_M1x*?5`iNd7%HF(irZeZ6i&?SKZBW$%^6uq|%b*_iiMDSMW;4LPdFHY$
zt;jhWG~H}gtOW9Yky7HdeVZ17=CVQF#Za(z=A8?(Kr^mq=bT9MOW8fg13G=T^y8~S
zcnJiWsmuT`u}bqxF;?FM8g&B?Qxz0hffo*~MOa`~qy$=02`Z4jf#y9y4A3AM>M-y^
z@baqJmJ`p;IZ*~$tpti;5XRQsL?}b5DHDsFxuKDKcFqYgHS;Z?r7oKw(F!W0p!0SW
z?j|iKd38Zk(U9<qo^j{GA<)DF%rB5!3@VdE^-$J_A!)h|DijcPCMa?nL6fY;ajzl8
z6-w-YsuNJ(3p5hkf(Tvi$<k3sQ3Y!M!$wrlCud_p!@kDqAn$<MK%j26HPp?o)y%hm
zmMVcFVy5uTT_<_hf|e>lqZgE&5etUUB=d@#U!x?BNa$kEW2<4!SfjYRuVT@a!4`6W
zhJ*EXF1+gi$^eq<ebwV(vsiNKv*E)D@J1eFRO|wH6%-<Ufd*B;0}7xe*B3xbC!q0S
zzH=dXRTolpPvkue^|i73Cf>=?w~*ra@RK$(M67}%2+?prlQh`5kQb3yzzGgCPkBuH
z@zvMhN~rYXI`Fdf?~|k_yxV6xXO7=KzjdG@89Y99<G<v3aIi>&mz6`8xP#Q1f%_v5
zQ_A7%!!Eh<ffv0(mkdoX0~cQ9#d1Z5{ZQs7!6w5;zA~ZH8wH@*b<mQ2P>FSa@}4Jc
zW{4oVZL)KrFRDL28Qn7iucn8!0ZrmSOAju9S0f<>I&^itR_qf{*$gW#z?s#y7TQkH
z{WO)=JV0eKs94@!HT}*7Py-6o_66sE@Nz9go&i-spfVe^hk1fG7qoZ<o)}Y5*JvY|
z;X7IS7HAy{D2box%mL+Cl(FAi*@!F<H(45)3(66Yak9)kGT;U;WWjy;3EsC`a}2<_
z25jU>aNL5cBa=O9=3BrknIK*OtvGx6^fqYm-{X|spf%~>Y5;9Hvuu%bHB#_{N?uUj
z1{G9jW4{-`tErHZE^OT+Xi()LlB?FA<jqBtmfJx26;b^r&b)JB8lu36Rx{rMS~LaC
z>d5{^8~nWmS{efnN|3)_J%W`Xx|5}mrB6R;+lHQR!0W?|<FKrhLW)UPS%p45i|QRj
zj6)k$2f;OqvHB+1N-0Qo2WMFLN~tuzl+*H%g-=i$|0tR-L4*Wy#e_0N3u<e@ZHV<#
z>8b&(D2Ju*ZK$RwtA8rK3`*U{vT=4%r{4iD*Mirdk??&nnEQX=)_{s~&;qe#AD4Xu
zdmFUa3L0yWwd&xi8&V@1%EsIbKorAkp)09yDUXD1mpQijICv=uXde)?(S=m$)XhNj
zTGb$ns35r?$<LrRIcP?j`*G_>CFDihlOSueAUOjxd<0n$cD)y|SpS&xx_h9cj2zbB
zHAA57B<dFh`cLvcbeNdvmoh_G@|nxA<RWK_00_Iya$+H<Py-L`Z|_mR*Z`ik0<Z7{
zHKz&|oG?%~I@8Hhtn?V##Kf{tii5q2r&#Im9FSU!;on1zP?<BGJc#DUG{0%^I_crQ
zniOOgeVlPefKNp7nM<1`$k8#NCEegvaMv+=u_xM4+y`1223nTRQ>?UjCW<L0lckYd
z(gVuch^lxNN&vYT?^Kl0fcX0GoD(}hivXZSbQWU4HmDFrbtc-H<rZdFhzDi}OM+(g
zA%iYQAEy+<>m#)F%BT*6ths=9ZIMC*!!Icl`$20CU_qhmlZPB7hn}>d#-qKDbiO1A
zzVB!=K=CUFdMJU`bwXw+5j|bDVkI;iK#^YZ3%XMXv|bz2@1W{P5VV#9ve@&ZQtT50
zaP14)SHp+m2vC5-$0|S#kq3yx>xP~-p0tA-yPy@k-&OQc%z2?^j_eN5+yIWn%gDI{
zRF6P>+!GF8D+LbOo;>u71DTVBr_>UZ(BlEE-8l^2CdH$^9G=?I>tuAhLA?hshRnso
zGuAGYG<pow=0$Xb_n<^^GI}6ETG60Y+bDhE320dY7TK|Tw86{SW&O&)6Ib6sBN?z^
zuaA45uq^>KJixmbVQtJFanP*(kqw~Xs>ju!4S)vwrh`^rf%m{wpe>bRz69E1bW9qu
z%Lp{dIF0u(c=$`r{LO7}->Kp<X88ka>VnoR&p?gb%P0W=E2}^wHz?g<%!IbHCtUpf
zDK>ENMaWEhA8GKwx8(Yz-FKkr_|R4G%oVtZgcTN`?TjFd)(k-{Es&<5p-YCqi40WI
znxZ6QaBuwO?b6BaW{~nb8zmWjoP|=(fl5or+OSI%Gxdt3K*cq5#Wre6TLf7c1~#(I
zaw3{B6ToW>O&^;+-U`|c2iZh6>*J$Bc-aMNcOsWEpau|pOAWSl%aHm9lnX&kZSayI
z(5R0`S54hN^A&ap9KnG%7C$KlFF5_IU-{>y-TCkK|L*@k$sXd>vuJWpea)1qdGSk)
z^0Mw+sSe(~ui(c!$Mv)8kNxqy%l@AycbS<*@P$9;&lU!MUiN9(t0iXT4^)2d{dw<-
zYv=Q2x9YFBMSsh;T=&B^y65$>Sq;Y?eonYst^Qw^H-GhAv8d0>?#b)yn&8*Hc2}i^
zx%r>m4h!9G)*m}7qjml~$nA*OzF_Xr?&v)`jxL_~V@aOw$G>(y_rkOOEj`O8_O>@`
zuP(#6v-L6@|J};}9Df+PUCG=#?8#xB$N%0ihHpRlIxTWJue}c2>YCk;s<n4r+iR}B
z;8u5A>*TK4ugp#JvNrW>(VR2;HQ3R4-*?)s`&)9@_{AeWt6!Ywic9TYik&~Tan3H*
z;ATEv7qN|b2VaS;@jH0#h0J|pi;YhG)tm2M*|D-R_`UDi%U$Ub{nkC~&#q3+G8bLm
zk)FYSW~F**;4Rn8*J6d|m%AtI{>*bf#{B0!wZESq)>%D15V9uZPIaNTgkItup<}ub
zVn<8iZACq={ThE4eYt((Z`o1X@AIy>z7$`&e)g&8BXxntTqVU*W2Rip7i?$0ydyVb
zW8T8Tlv{7x!gX>?AMyBhpM5wxeA@=cf7+L&>{V`WwdmhkZ$G=YZ~5Od@3g-3|C8*f
zH|el<y}ME3AGhnT-;Mw6__uuj*`Ck&`NI6VA8+s58|VGsW-lw}9`N+v<9?^@`N7<G
z?Ei+!A4&c_P5qF4Y}O+0i*lRlD=zBB-}yW9=l_x|-`DT_Tb{A-?&tqktoC!u&flH0
z!#?rv*OT^6or}MozwrO{hP!KhmX-edzT(P#{y*LPe||38x<CD)^d!OP`}XD1cEyjk
zhxQyeCjQUo`tRk(=C5e`xARRr(|(rvhfj8I{rWffxc^-Hi;~}-Z>pF4k@`XW+_Kqk
zw%5=4k@-n|)}enfR>$7|d-_yAh9iI9`G1)Ymmi(K!}7)X)c=!ysQ%=(epB}I{rn4D
zy=N8D3Y?u;q?B$5D6z3gD_syc*e0N_aY3N5jl--Zg7IJ*hl$Gu#V4XXCN2vUn|KxH
zJ8V#7;uXx{xXHrIE11!d;Cw<!Ag9B@nMGRZ21pl3c@wW7lF*Bcj&p4<azF%>)z?OC
zF^P<hgKZLvx;81YNhNJiY@3mk(b3o@k<+2LcM7lJ1%YN>!zb!)&O9bZ5}bPq&)8OQ
z@*3U{NS3oL|K9{wCM_tPv_O$fI_ZTDqsb8mXCAX77uHFdL0AHra=eBY1t#+v-UJa)
zmN403mgf_{-8(sR#+AvNf89H|wf?5@J)J*6e_D5&JPX?x|AJB9^y2r2+V9VO?Z33A
ze&IDuVa{t)>bKng>-?epf_8NH&&_-8>@WG}RK_-K&TqYc{qk*bH(#BY?|SB6Yu5>H
zUl#34_SqNzHkV8cU!P<8<h)by|4l!*%U30R{HgSQ`qlc-TAhISkpHP)<HO`n-)!Ba
zdseePs%)db&bQ9ih5Ik4ibW@FR85G#`DE5#tJz5xc^IotJ9+DDJEF8TZQ|4ozEaVd
zl2@W$M8(Iv(D`!L^rvuac6VH->CaoI?`K5CZ}HMw<#?%i35Ta}RE_RW>Fv^b+Kw6@
z{}nInyms%*X89-g79Y)(zU?wUdiNcBRnbE#xz>g+HhtXsgvo5D&P}sF_WKKCCv8bm
zIqvIpAkBTY`f=Y$(#`w#apsz_hunPPWMdedXZ-A6<|p-6n}1Y3N*B?M|L1kPLh0J!
z@@T8#|J|Q9%$#&~$^08$M$P}W2NeISd+?t9U)_WG@-NbJ@2pdNcJJkT`?SZ=xi<el
z{8-*wZ}Rkq-<;4d>D9?|X1+1~ob)WN`|-ij&)+}&y}se|{*9&EL^u7n`*fadY5ghY
zLwr~NZ2$Mu;%3@!@%exH|37U0a_G<c{HFiqb_V}8v-e$P3k#fKQ*OSvxZ~u{_bDIm
zuUq&3C9{FS|NDAJ{>8ul&&&03|HsCE?@M?8_ig>*e)-FP8<)%V^S^yQZvU^m_{Vp-
z7yAYF=l^w$`R^|9KU4DGYyLmiPk&hd^uz5v|HJnGkpCU=f11g!>qS4dFJrxybpA_v
z9JgEcg6zrfT&-+gezyF+@odFpy$=`SZvW-~^F8x}M(*=F+ZO#v_Frf3dL!&u?$q0C
zdlrklX*Cp*J$-50+1}e5cC5|49kzpiTHUv}=Gq@M)v42OrmRuBd?c;;$~K`K=_^X7
zn*aRUGiA?K-8pAIoc~p)zjWEW**X!2wR=q#|9uwMVm75}&;L1w-aCzD%^z>8bl76`
z=IOGcWBvSfcQR)!n$X;R<y&T$wc3M|KRN9-y{p#wn#uf+J^JXfqi<G(-d?ow|2EeZ
zYwNeyt@pCOu<rNM&qwxOe8XP2hwc8-+JJd~FV4UH+AQqRbHnY8seTc&)x`F;{{R1S
z;{Old>!!>%Nc=PN_vJJDOFsVBoLDY@<KG)&!?)|)oQ^#FAG6l%pYq>V`X0AKUD&_c
zbG~?Ou~3zf;r4<%DHTl*Go%$?uD|70#DDAcnma2$2)r}5%*(oW<u&_t-)Fv4KbxId
zF1bQ-rR2)q`R%;ddam_c`}91KxsCDh@+5mxZss<|!xbNccN-)~81!7P*Sf)bgZD-+
zD#L=uqQ|^W%c5@IT*mf~$NX8_8QU4#KR)wkUCy|iaXA_zo_XJ~>UymoQXfno{Jv}p
zWo7UA8hkMIK<a^1G=@onlzzR|3^pI;WgkB+Z~AG>#5?2I>iJyL7*98(?)kMcL2?66
z1W4f8`vw06_0FGPQD67!#=QBB|9j?tIe()5-hJ^pw*QmuU&=q(e`>b;i}=pZ+C~4A
zitAd>t-tb9)VhA6wSCj_zaI0%_Ah$3{gVBm<$pcqe^aizwf;}0J=2Q69t*xHH{Dv#
zlV$I;;%mI&@8&<+?YH_Bvg{|V_}j7Qn|AlDeuFIgODq0%EVjMK8mlg{xc=|%zbBS{
zQ=V~aeM*-7qLqJ7Ec>QB=hpg?Ec-_*|DIU>O?knsKc{6rGyhdye5-##*8ZSXe>+yj
z&iyO@b$W7FRyjki(C)a_|011tJ!d$v)Z3Y7wq&ia3%-5p>=myUf*ou^US?YVc^|v2
z^jd0HzwhCbz5g^=uZslT)?M=E+3F`|vka$l2C-@G^!a-+#b9HwV=~vZswLZu18(x(
zv~^HP&SHIDe=IEU6LY>@l$L+iZ}Ta4_#OmID|x?p>nD}x+PAsv_Dy;)RdcSRYg1g`
z<d*dLH{S`*yQ=3uN9E7jtZjvp60T_e-FL<AR^gw`FOPZFX|0$%Vb2<szkhP}*?7uX
z9Q{(WVv`9Em#X=3zmw;yR-Tw}eAN;YDXq7Kx4P#rY*Cu??@eLlq&qXtr{7Q3KhhCU
zWPE4pndwr3yFM0Q@;xQEwM^AqHP|!Y%~g-xTv;lUST88ctqSyIaWWMWU8R=AP<^C$
z()<b2t}1v>y(8kQs(ypvRcqy?*>x*Y?5-U5nE!5jb)aeXjI=3to`k6G{<1Z;sb1ps
z|HbwH=N`N-`3HmeZ*P3#*Qy_-XHWWDfcCSWo8KP4eEQ^<DSvJ5)Ctxr+~!Yty<h*&
zf^G6QzA@jv_MY4R#9MZ&x(Sc<r~Ym1lYjBUvF|?DasHA&4#)Y6{)FxN@9<sz#lPF<
z_D_9hf8j6tuKx<(?|XrSm~HDO9M4~Rod3&@zsKxPJl=lkF?-b?hkp4dAERIXWVWqe
z@SOkCpS559yp6ZN@R_~xUs%;Yhx76;zO(=QyY|a(=D+_O_S>I$AHVAxKbWVmfB)1t
z`wOqxtNtpizwfnP{>4}3y>;$a^Os)d|MIKxocxnNYfJt)Jm)X@r|>*~>1TGUdiO8<
zMgPKf)h&3<zN>D+>+P5R>$I8)N}J^TQNDd=>6-7mJ$L^9_wTpNi4*(HkM6#{jPZ$J
zdi~)BmK$e$E1os<?Y^ySqR_*3<U#{?hQp@@?Y?^P|3}qgD-Reb9eG$U7nktm#gF8s
z$LD=#RVPd^*r&by_}>@l)(l*7!PgIcWsR>FZo4UDz~&pHAXC7u8}P%t@bTf^t-_&h
ze8M&VmnS<P*j({YuW?oF&z#Az&mVf+v)*8yQ$Ek~O65QQ=ba9}jS`kM9It+#eWT`2
zZR8$@6{e=^Z?U=Z{P|d!DW<h0;QoKUf8BTa4a!VxHt{B`2u|O7bH(iqcQ&qEP<)y3
zyLLjpll?<pfws2Wy0gw`|F$%{|8D#8{cqe7=9~$b5O(8-gud|>bxFtju96S>UguWw
zc(~Uj{5kF9v_fQ`{@iO<Z{9d~%<tdKAHs8=y*Yd1{=_=27^`=pRR`bR<+ne#`uxm2
zSN>|*99s4Hht`9=zjkzs+a5NTD_Pg3cWFZ3lYIgr-O-zO?p~VBFPycF_s))zIJvU+
z&g0vpmt3yBbd~vq`pxO~pP#k7<h`x&<JNpDzdE<Jq&X(51Q&m=m-_zv*U#^DKf>4l
ze{P)dr(3@~<_Gg}yXtp`_J`ck;Fe0gBNqH)vZP7cg(+K8v_EH@S+X=vr&K;JgeOh-
zFNfc>DbtNRW}FXHoavixHuq*p+U1$P;ipTUUB0ipsxZ-P?oFOF;~8fIR`Zy0f0jIx
zW-$9;4%=qQGij>|)jrQWlWruvb+_8QlR0XeXJ*Y=c`R^i+_TR{^QZsZbLRP6ceBT5
zr%UcV^CH!*`=(CX<{4*`%)Hlbo^k#Xd%77&WbzrcnZBz^C;e3U?3p(CjM~-08NNod
zgI<e#)_fNH>)Ee8OBVZl2r7Fs<JQ_A&+nh_lVHxf_)q-2Z_dPDmz#da8})xM`tFl^
zbKn01xAo)dK7Q(+9>3?$qo?LSneN3^R6l&$I$b<2uA&meu;<R#-?R6F(NB}k>B9Qz
zaejOLc<z~7Vg0e_(bFTRTc>mDi)+VL?EX;Z`<`2SPfW$m51+*D|K=6w=CTurvC?mz
zev<q9G~4d!SGvzEx7}kf`&f?D=FBt83}+t;yD=}}&&)jrvlGk?<}}@8+59kX+xfB&
zkNm~dXYfw_@%HrH>F08+Hb1<!<NUYVkNm}!8_x#uU!GZJJbU3W$usFkR^FNRxi{E-
z@s<253R2%aw;k<V7&tj&PVu9MOaHnTAF}sM%iF8-B;v&S8M1%1XXILIrq7LO@Vh0h
z_c{LHukv(e4YxVdSIP!jKGHdrIr-cpKJ$%gAr^<e?XReO^t5xjId|W__*}K6=iEvg
zgzcZ#p8GuK%TlI-bN{Em{`CHO=0c4RY|id2c1Ks7<9V?r;#}-*nX(*tw*0g^X>B=a
z8`6$tzOxbRl4g*cIkRzwWb@32(<Li+24~(q_u%inzT2Pd1J(2TTG@`&FBB=<ob$n5
z!<V^yoBZ$dsgD9Rk4IE1n6A9@P<2Jb>}y-=kMaxq^WB+Qk#YAc?~Lx>t&KL8-d-CI
zFJ^9i@L&Cu!`s~1Rtsi54xBbOSf_}0)yFGL*4Yd#=aR~G=U81i`&a*BL`c?3=AVHJ
zWqwp{ny`j5Zk~?BrXK}wKWys#VEom8)!ZwND@*iJws89W=HIhn!t12p|F>Vh`R}!t
zeEL<}d+CoG?$>)w=KHtM_sHDo2TwnFdbRpb;iuy#<<;ZobWeY>v~HV<^1o{-Ix9R^
z9sHO5;n{`wx%&-g>$!Y;V0%r?+3cZHkL!b@v%Sr`_0{fsB&irqdg?#TKC&*V^V=C`
zmwh*WCSC7t@p4kV?{RHo$nl`k89x)AHS6>3v-ps7{gcTTIqs`7Gt-wfSMQWI)()NM
zzwV&;^(V$zS#e_B`y*x?W$Ql{#=_!PIkU&Wr1IFA6prnlpU&j`7vHUSI7PEZv?prG
z>v{K|Kb+-Zns=jRtwHGQ_lxF6$+rgG+9H+{wNJlLxz=%k$h1b*v%)^F=Kj50!5L?z
z-z+}u!Nasa=`W8ynSUaEb<hrYqV?T(V}^e;I1z)=D`%SUi8jL$37gHv1!?JL&YY{5
zId`Lx_2ZPXV<+BNX#9CPzgs-*@(f>%In0|m(}d^h8=W}3O|bOX-ZwmzHznU~GQN_U
zKI8N`f$0?!1(nMx)=FC}`VjSS?IG`gHjfkc^&O?FGMf}W?wx$GpS#)L1{Bsmlx8nF
z=rbc=>7-d9Mg>>j1+w<8UQr+_r)9I`M3nayW0%DXUUh8zm~Oi1W;RFe*L_KxjLS6L
zBqQHGc$?gLb;I<3r}aBam6jRmlzg*)!}fyZ1=E|pH$HE;|GbL-wEIH(jpiHtH*0TH
z->CF5S;g5T@{8+U+VR929S<gn>R*;&QhmR4+V<$eO@ZluTYfeyJN{ws+=<oJi@%k*
zy*b*wo~il6?LB9nrGg76P?7bWNxE}lg|#zliQ9v>J^OA;R%|zZk(YiTwywbL6yMz&
z3#F|~lH2p%T<DeEwbI$V?8PyN;1(OXUfErxibfwEEXrU~?!KKUmAuzvR^`Wv3my_P
z9Jg8iXubHLRrY&p+SfyDe-3%=Okj!Iz^H$t;rl;vx7dUR*0!L8D=f#{C0<&v|8AAu
zn{eQcT|$Pu!AAZIOC-G;5}7|R-P79>Td`AO9*h3<3(qD>Z+H@1u{&e;w9=_!U)ry|
zZh!W={oCvIM&Av8mPqo6OKqJ0r1j<nj(aIA%SHcOPGO&}7&EP4=Al&X{4;Lb?oH6O
zP4P)DY`ST1jw@__iWq+sv%6gRCHB+)n>q>;&0^guK{Z3V*<9VrD|C{7&Wx#>BxvKo
zcC7U#kL35O7R^6-#M0N)pFaCxzaM{1<dksUKbv&)SARKmb-@dhHG<Wd$>zD<yl3X!
zx)Cb9J1aRp_syEzy~iUPyQOB>Zrga;tnB77yW2N1rK@w3x96^8OFN(YW>;_Q_Kok&
z%I+Pzcl$<d^vcGY&9jr<ZQwTl_VAdW%$p+zK?G0r{GRN&JmvFw(&zG|&gDs-%ab^_
zCue@h`VZ={UQk6~l?VOKtoYiQDIuPA7^IO)Oavr*#`uHKWdpr4ZF4PdbT&hX8II2<
z2QHo57&rOglF5xrCOa;f%(!H7;nVNkZ;l>}1Ka#3KAf>U<G@{$YCq;oiNL^(ue1N}
z72f#&<!|{nY|Vndj~4&74PYxX&T%%>*!baOVqd#+`T?6BPKIo|3b_YI8->~9WGyP#
z&K&%}qxedg$wqO?<sbW3sPxC)$Tga(w2^!F&d(pxcqNn)jarY2um#`nPSk7ND(UCV
z;_Z~6a>T)Dk8+Edlf+`i$3};Q6@>(wxOvY6&RCQnfAkNFZ+b%2p*JljIg2m<KdSc2
zx_Q6x4>aO`0vo?r!j?~be|CR(->$v+(fO(_vFXRurRHbUsQftLzI@y3hBVn@t2SRP
z=?nVs_rdRj-zB*$`a;t;=rrs1Jxx`*yPo@_)5k|o`t`X#tP-jz{J8ze*^s#%EjM~s
zIKJMss%}+K@Z4EimqQ+vb?v$SZO6h*@2}dfcMjhga=klNlHW8e*mQQ*wUFh@ta5Kl
zO8={2VBEKoC(d@mmji$EU#;S~%2e%mX7#?J;Ms4E8th-TD&SYx`Ni7C99yr(HGisZ
zud}%Sqb6*t@0ZKMU*^~qtk4y>AY<MZC*gEq`+aVwZ;GiVKYue%K6vl%uETwg;%m<L
zHOTxns*K#hBF_^gb>p^`)BjH<`Y}I_J;|SZ)92Y=uJ@^I8Q=SMtoVJ%ee!vgVsE?1
zBkmQ`{@$6q>(%?<7r}Ry?OJi?#<N}b^R3S8SgGc_Tx8pY=7l`J7q2#YrNvM!tNdbf
z?Uw@~Z0p}GZ}3uGy^g0#e}lh4p5}U&duH4*&im(^+4stA`h4U9ch*bY_FG8}+ZV<z
zHDML~_t)g!;dWUO)5lTWKX^7BiV_u<JG;&D+V8Geh7aB)3EVw;U<WsMP|S}Lm%6*p
z>+G9fqxRG1(@dqLjR8mZ*e{j;X_v(O(5htaE~DhSzlwpz8KqBp*R77b;y6Xb)Ok^w
z$CMuZ<A#2dQ@3u5><-_%psCeMNbBS_r;n?6ewebK@!s=_QPeP`DfipM(Ea@jdA=k%
z%}@DnvwzFWhunUCDw7ZA72Pnrpnqo~zy6dY{r{I3ef96x|2%d&$4*~S<-6E?&wUYl
zBAM#sVrS03RqeISf9v}zFJ?;VJUQ?oYL8!H@FqXcYb*H;uf9`xrW7)9g7b;nMjbpq
z6xG%=?J(PS;@iBe_nfPvgCkPnLR$j_o!@S+uiF|uf6d!voeg#qA282fP-`mjrQ*Z8
z)3etrDSkdGyl&?EdJ%t1$p@A{l~?`0$8fl2|E>=|&n$oW;EGz#wr{KB{v0}Wc6FRg
zRb89Wj9-bX(mr3^8seI|#JTmHVUX|FOD&%4m*&b$XIaQ*zuqY<;pMU$7ap)YXj^1w
zes%u+zgI1)6IGsVz4B`L&H2luv;MpDztBGV(<f`o1@=Xfe}Z*{G`TBMKOWZ8O6lgh
z`KV?4KC5}(nj@16|3&|Kk?=j{yF};J+h4xblxC^M%{k(5SpU>@*PmSP&g|P$^W|y&
zdaqqSl8knzbWZM9llQfYs@XK-W&5FmT1&I(yUqu#kDT>z+OZdxc(yNz(+&Dsko=-?
zA^Q{Wj8(zGrn9<yZti#?d7Rg@V1>&<A=eG27G~3rJ)2atxi4t5$>q1}7l>c&k!E+l
zu;$ODjTcYY{bVq-^w@ZwSv=@!$-aW;%9E#0_ph7uQ{=sv?YueVL7)A+c6(%suUyvp
zzVm%yoafd|uYXJ3l2&mB`_72x^Jpo$*sOKl(t@9nTU^$8&ng!0^J@dxyzQ5)`Zs6w
zEr-yD+ZQfx3E%gf^Qxu+Ly&woA79+pq!-`U@q~QkPv3GS+~;i0m(&ld#M|xKZ>e1|
zn=eu2*n4FC{{6EjxEL<lS8Q|OiL~QH-IBIP4fo!E3fjRU&u=GR^J#Kb{-;CYlkLUi
zU8ge$ONiV3d>FFgMq77DeBAPNPWe^qTGz!&&IwwT;On|b$?K8s`l^lG2{xw^c<x$k
z4Rf=~n0WY;PJ!2=ttwI7Z#=j8TrYj`>(#8RO&3DXn{IVn(fZirW9FRzw^czaxI!|2
z)J}4oD|ohbubBHag)9ZVUU4ID^?L7r+wQ*co%m1q;)?G}XS`k)Yji1XlE)Ek`Cldr
zO)p9;*!TXmb%||q#{}PqfY%cbzSI8g_LA}8BGbuRG-IUW8$KM~y7JocTN_KRa>w$o
zN|CVNu}RC&kL}YvHgEAIHx#e4y*{w-Nn-kgL$l?d9C_gM`;ho|P2J<1K8wz(z4`d;
z$?y5oXXYzz(LeL<skHw6h(Er?zl9b>8T_m>ysi|u<A>YF-bd0lYdmUP?@8=hwuP;B
z%GJ6p9A_g|uA9F|{?qHZ|GzkBEq>DMyEIFSqs`H5<up$_mGvvLS3dmO6eU@IY2ReV
z{FnP4?LKnzt?3W(S>ng*9`ruHICa00gjk->zN2oOTU}SY+Z4Ja?Q!o_ryGwO_+wu_
z3b#MDU-&=YJ*hq44}Rw8s9MPobND@XeqQXy^G~1VYdE^ctf=+fH~-9@ipgJJubMCR
zc1O|99*^=pjvQxJuXMFq$I`5GCw%Izxk=8CeRa<jUhQ^Pol)h;6LMz%j(I{W`f^^b
zyXUsgCj3iP;Mcv{wViGJ)gmw7?GTGU@@Mz@zgMS*@`NgeJn~=Ru}~>-+GEZYouRc!
zx-%kft6%$Ox#Gizp2@Y2yLmILejj??++SvQsq(Y&jE@hbC*Sl5OZfEnq`BJtik~Vz
z7M$$ndNqrG)V<W#{MOI)e+k#`4H_I<x>rl-d|7bn{Cle}qUX30=NQ`lnO`uwA=)=K
z^s75>k(WT0zSm>NP?2!c$m{Q8zyHp>;V#)B$NZGVcWuv=66yR;)*oJMd!ke`-(j(F
zU;MuxlM*(c$ZEbaE9dG-P2SzxKO`B>f3#S}ZSpF%b>882d}o=$?>+BIxF4`_)+yGh
zOyQ|lrUt6p*dIQZ@aL9Ys>S6=U(dC#ylTBp%X2PImwv&dD@JnRTd#k9wX4K8_9|0A
z_;=o`x*_H(<bF$?GH|-Sa`ml-&2N_cnzf1h!785PKZ~jgIUH<O@Z0}bVs>iss?VZV
zXUeP;@cjEU_2H+=p9*%rxMzD`GMZp<mvj0_?)wpE-cJ5w_%rF#%qRUvzgr%N%e2&=
z8Xyz%h3D>r>?u3ev)x&(_bKba-Vn23G1im|UlYRFocZ^y%>KCR%c86!avvuxI(eX3
zt5fSh<d0DCi%s@h5BRUW|9NWBOq<<t$JeX>^tf@M>GYG-heCgN+h$(j?C2K0y-04S
z#<Lw#{%;Zvwl4jZkQ6PmmRoJ}ku<TG881(}i`7S;VY_zjuX?@DzWHZtekOgK_()l#
zqk84tt4fbAhKkmlH{E~y=7DLKi#0R24lGTJ$vU=_aZ5&s+v6AS4+^zJJvyAp_E6ya
zw7@^Je{BDdKl#v+_zNE*)_ppEbo&=0d0soI8k39|Ub~b1ljlqla2FKccO#R{N|5cx
zi3j=f9giHgf6_lWV6y(cma3mmw2w^B-&2#kF=Ug4uH9rEx6-SIAt&qdPS`C~`k4H5
zdQf0@Rm|7j-<Dsv(<Z&iV1L!e(8c?n|GN|_xw2yZGNz#AR}Vc|wf^Vl`E~kJS4?3H
zy)h-`>MQ%;BGauN6Gat*yG7TYo!EFwHgCz5TQb$%ohA*wo%=uJKfLe%SirrmbQ?2|
z-M>#t`z7r>KTe(!#uL+=UdX1yx$DxE<wuLMHqMDsVcwSgO)x3=2A5yvQpT_#f$nX(
zG5$N%yAS7wnH0TU&2{(M#MnQxxmv}JN8Y*HD80(cv~v<$VAic5?`dW(C4YX;EU{tH
z-?Q_x_PTFf-KlKKN3-wzZt;ub_t&hcSfSwe<8bSK)wxoh934IEe?Hghrp29E_ao_}
z<QCg?Mcp=kX7W9#eJrwN<qs{xFZ?0y``e8=R`P^M*6gj7b&~Q8-)r&Zk-!0-R0DtE
zm7(*BQ@1*)@=Vq7i(aLY@b*(%T*z)G?HiKLt)73Pe~9NN>5Kf2+4EuI17rR9l|N6M
zzW*U`i_*PHt!z!tMFE$eoV}ZN;)CQazk_y8S!ek6uG#UmHN5eX!@{R)U6{X|c(n59
z(x+<cK5gguYkThgB#p`IKkjI&bylx7lf8cP-s26O^4p&mJ~-jrz>`pCRW^C^n`=_v
zPU-P!#CY$&uIZ?mzNP=q>m6pX?d#5T$j?cPnZWqvXsz;>q!&joa4)=H{bCi*6*G~J
z9Senc_HEewI)0st(0`^~$zf~G&CLHFxueAH{?Esp0>+gOyXWd3$OtakBtG}eQPy9I
zH;di>-8*!`UF|;m=EX0n)}Qk``L*ie(|!79azFMS6}O8DP_6e#5{X}8@*>2taBBDO
zhu-qv)O0@;ralWW{l4{e>%#r*Mp6rTLM|^mmH4)AwNcRC2V0!1wVTVen!ndD*cNn3
zo>C~?|D`o#sYNJ<=-i~mtHn<$$A3EUe)YcS6<UVpw#%(iPCI#NmYm~;4XRB4nUVue
z@UPgkReT5c^#{fWcQ;7C(P^H`DpDC!@!5FB`3KU?Hy6YgtdA<}jkl@X_F~F?eKlhb
zCAEbYjGW~oxhrC3CI&yuJff$2r*Q5rz1Xi#*NbDn9}i&Dx^nNtnbmR?Qy;GWz20lV
zrQjnn-3}*RB94DsRGN44eO%Os$E%iwa_%(Hv3%mK{clpT-1ST9fo#tEvzxia+f=<B
zP8RlW_<dVy;*aeg^rL#?{v=t<tL3%p_;j&ogMJNvYG6U*N&kJee`~&kuIqmzCs7eB
zpZ!YN_sw0UU2*wMS%q~My8lk;U*TIJvLmYc`jty-o+KaN`g+wNc6pvu4pENlOL^Ml
zWP-2n>HQj1P^7)!9J9T2d)%+!|1t4W_u^;e@;`nx(emF!)9;eIUOip5X64L1B@^d4
zD|`#p5HXF~<2BulZOzHmg8x?q@0leR7y4xB#04TPob6NJNWNa&8T<KqxAzD0DGm1?
zKU%f*g^Qltv7W!rYnYu=KDy+s3-0E=@`!1xm~_fVhV@PXtT8|SJ^J09uOdJ3BG2Nq
zha%#CR;JI${&?}x;-h!E{5QRw?!#}P;d3=7_WH`#t5?VI*vdEMWpR}qcNKQY?Cn`S
zt5D-{VV!>Jg15h>=za=bd~j-`ROcdRKSoKOnT=5kSBlP>wQ!~E0?BE+F08cw=baj`
zG=FvfZ<AQ(>kD7MWL~*?l|a@Wi@7enB3UY4>s3o#=M~3HH+pd@uvhGr*OWN#SKV(I
zRL@r#eqb}6;ZhZ${m1&_`|kS4+fUXva$l=Cs#<^XZd}aI@J;o_z3UyCr@ndXx!ze-
zU4IUHsX6;I-VO&I5n(~6kB?I9B)Ci%ruen^_i@bKaL%+&RrJx4!nGAv6(2=DEc(1Z
zgI%MDp<v2vt(#x8tV_J+Eo<>urM$eQ^9<XLRq0=X*qrrOw7uD}^53ji>y7vySe3>u
zpZoRDrDkuARVl$+XT5T@o*j~%vOp_n*Yme~-b`QlwCxP*)TrDOHUTFEI`=FTKX?3H
z)rRs||LGMazNrhW+U;JPV=HTE60Yx`e}J{4<aW5?8oqx|CeJ-za*ri!WAN3x(NQ&p
zM$ISP)$iB;d~y1IoY<qM-QD%IE>2Fu%GV^iPMA;nQS@EqzQ?n_p8ICh7|zW8*!w7d
z%2Xxp2MkX4{s&le$uz8%3ppCC(R{CjH+Hwti>eLt*{3FkzvkO(Fy-Tg&sp2}x4v6s
z$Q{1d;A_*Zbt^ya5SucWMRL!nkD;8?j3TN&Y}g$nxw2IwrTbuvhR4^Wm$&U-3He_J
zwN+PXwL}?Tsa)~0v#&wBe){`))e}CxcmE^#x9CNB(=pW_H3hs<H4laVZ2X+}yt{cO
z-!6N$&5|`mp9EzZJ}bDr+t*>Kw`s$R{BC_U`J0!u@7;OSVIFcOU}fl*Wt&W==9Wvw
z?KXOGYU649%U>S^xH;=*%>EX3u~Ikn!>h({{wF)<^&g$|bN@oM5K*Vl8?SZarb~tM
zx4m1fwQ$*;$>PQ{&OC5jF7ac!-?Hg<UBsAnCjY#=yfBH+&i;#(yl>jnp0ktx=z1>y
zB~pC#NZqbS%A2k-X0umw{1MHW?qnbr`?X=gZJVQ0=I6+<Dot^G$vE{f58F1~Wb2Q!
zcD?$3?AD6ClD9g-x4K;BUf8qf<*MgSp%2<KZY(|Udi83fS56_K$uBH(Keqok&U^Kz
z<HDUE_kYO0To`fC`}kAg8iA*rM|f(R9xGeeHO1@wIJqH`RbR~bKoK{aLClhgw$2g3
z2~2aVX39pD-E(hSsQ!h~Nwz9}`8x0GK^sG*StRO2LM7Z&UmjieJNZuAiz(KzQ{SyA
zS)P;Va>su9^pF=vXLp-VkMr3#<BZ<PFIsk&<^J7x6eGPpSpRbFJyWmUZ1LC4-_LmT
zQ|Z<H6#L4;#F^|*l_WS0CUot%pnt!z_`^xZWc9u`RT~cfcdM0{GD$({!6EVZeLr3u
zo&L1q=dsi2TfU}!tQBN>QttQco9DiXHH&{_OrDeVYtI{H=f%|tUz@hy^lWo`8}OU=
zO#dFQwN?BF{la2J-yYUgc<k*a&MUc0_eAxw)-~68AN$qH^A#4!6od*sHmQ5wy>Qwx
z)0tlPmKS8c+5MwpvTUyPuFA+gA5NZgw>#1Ou~NS6QmsVTx0yMIR;tYr>RF=x;O@1Q
z3vTmgzG?~<D#}_k^FnvwT&;T|7U!~@E0?eH4&S?B%0#b2*JfrdJXsO3<D!4Bd~ERb
zAYqf($V+q2dcS>;x;W9EK~Kf7<erOvdd0fusp9n;d-&NJ?oL0s`IVZy@4sn(*3R%X
z4ED2l|9y?b{DrGKgwOBPS6#7p*QbBWSN6>5isRf7u61Db%>RGQTw9;_d~woDVbikQ
ze`NK!zjc<M_Xqr|xZ3^x<^;i02`?KTO|6~$rY0tIQGlnDuDI4o^^hqN-r=bWj@@}0
z8DI2uhjzQ)dH!2TftArA_r#`dcYV3#$5{!Nw~=oy@zizN|9r%0arnEsyszR%&z+I>
zN8jA|E8Y{hr!3aBIJ0Wq%Y9y})^A<1L$`LRYHi}mi_=w32j$1VdfUws9HPH$U5!D=
z#Q?WCQ^h%M?(s`C_^BY9m0m2|&i`LZ#%J=pX(!L-%s-hcseiwEXI;(8iTbaUZ+%yr
zw`NA9eC0Cl+g5UGZR9mS`|{0Qc0ZKk+;uL=R_XP16-O1S`0vL`##FsiEME0sdQnNN
z`>lD)g#tHQ-3s~l$34OKgXT@|{Z{K1iaYOL{o;?x$~wMJdv0x?ek)R+Z|{l#_uk*E
z{zn!cVs+nUcQmiF;m+@-V;Wf;9NT-l>n?ZSa@hPM`BL$$nvF9u1vk#<?Fo@Twcyfr
zt=h?xGBi(qnzTft^wuSHEvw9T_he&I|3&G{>fHA0?*_4uHT<P3Cv|9Sy^yAuvo4~r
znfbBNJ9R1XgnB*sqn)>Uzdhf?y+glu`Z;d-*qS93jL+q7y1mkxA-79tJ?92aL19Uw
z8C4DoGI-zeJ)gSt_?>#m{HQ?1<|M<}T`cv@(^l>=i;d*}a@_5>xsgMAPL!9a)t*Ax
zUG`HNR|#aTVPD<OFD`gzwUwtHSKOBR(hr}qHXhvGbe=)TuP9~Fo*&WAf6wh%x=@^b
zzT}zG4}VXdQJ0^4=Iv8y{rDR-9DgFU7I%p*KeN(l^@l^*hnVNiF#q!CLU*vf&);>M
zOb(hqJmhrz@1D2S-@i6_EoBduT<#TgYQ?l!Zz|taojtVlrSqlp=e6S-6f&b46`8MU
zhO`D;bar(YXzuN>Q#$Etb^7^Ip`V}fCWgn|N?zV--ST;#*+jvn?+;@i^)6g(*3|l@
zBUZ8`^~Tp~ajX4IFWGnO`*U#DTCeJV!k@l5DKeia_FcWIR<_DP(rI<}!etI3o3HH`
zxi$Zo@2`WrGAq1~Ka^Ozd2(r)k;Rq5i{0H1;|!aAhu=<})Zdk<Ix9eMQJkV(4TqBG
z^_8zLg}zzz?bWK^T94ax?_QnzGVyt+Wa-`;UmsogtY06VZf(}MZrZzfCvRo;hF@Qr
zt=W9pZ=v|2D|uPIQx#fxe3`E=<zb7zVc~pyBLAP+dk!n?Iv{)D(JZl<6D;<82z2NZ
zE)#c%aWK<U_};A_U&Hy)X!lWZ!=U5kACDe2pCm9pWZ{owp^ZP3L~LSyYF`nHsr-N0
zPq*rFs6J0<^^1QJyH<T#eq+Oqg{EH~tY!}>c~>tx%k0>6mH=JZiH;xH?0;NnzxL;*
z$)19_yI%d2$SO1~S~1nxm4B=8(lZw>JX+`(VYbz?I{NXaO@);=)%HvE_h;JhEc<9M
zqgIiXc~!;ZnR>nHyG-Uix}<hog+D&iCU5D-IaB{8eLDW+u1dkDqbZq_^wqX+6|Mg*
zQm>O+Ry}K7aQMC>`ZlIU%U$k0am&3Iz!uA2^6Ykqe_iI<g2K>)iN6o??o&7=WzX_Y
zRl0S4+r01m=C+q3Jwk({KYlTb&#&2U*xbJ_Z}RtlIdh&pwWwIKeU<!&qZ591{cN$~
z1kHb)QCsreXm%|Rqt=0%U8@e~SZ#g@n#$exMd|a)GwIOz@hq`4<0}TuKY5-NJ5?CX
z2G5U!=FFGfn*P%#&3NY76tl@U_ndha`gqTo=c#u3n=5CYJ)IMm_StB5c<<&J=acNZ
zZ)!-KNHXibnbH3{=&{bT#doKl%$amkWiw=^+~Q<S(BHP3Dw_pEH_h-hlonk0El=!C
z`q6t+T_;<fe|YoNg=23@dQI0&Vb<>EFTa`Rbd&ks{coRg(yuB@Jy_fuYG5d~uG_4S
zf2F$;i>cAqh0$V%S)Df}RUfGSfBT)*oVjJTQYiv0eYOYIonFXUoK${W&(*ns&E#6T
zaNJQn%ajK9XDzllaT_P3+8?w^xF~;2xcG)&Pfz@vb-&Ve8l`$ptG#H@zUr~|fEM?e
ze}`sz<aAB<-F)|@fw9?&xEnq55?;Pfw94?keQ|loy70$8vkDaaq(nEU?(E*a+x>A~
z-JHMMLY}n0nc#W**uK97=asJM1lbl8bnT08INKy_J86D#bzo`B?RzgyY`C7V$Y_IW
zetY~L?bFGX4|1<>djF8`aPFhX1HB7#zq|5sY(IbOdt0df<K%!fYMxs)rCygYzxR0c
zd9`oN?9(eZ{C{C@!y)pUwcNE*DkSf0#FigVl*D8%oZHL95os^DAX+g;YTsSO#J6(=
zcAuE*`uyDe0Qs;K&Y10|txg=|+_ic4jdQYUbz9!7E4Z%q>fl``>y+BvYSlS2pHIJ>
z(Zc-tu<YCVUrveE-W%CFJy`P3Uf5PTNyaQ8x`gMQ+A^WV@1>NY%69S=bm{K-IeV@_
zgxfbIIo(~!9TVQqF}Tos$*jEa%a%k*#~m9FOBXH^HeBR#M0eGKqwBgoEq4mse3$<#
zT6)@X4*O$vl3(KAZtOeOv;M@cgM6DTMVx!qGAh43E%t;@=O=^u)4O(p56)}vzBozE
zhG$*?OCw(=@BFG{?VP+zM>d3gX<u?$?C-31E&Q1*i#JYAnxpXGl=7jQ@22T<@4W9~
zpl0(&G9mu0!-eRQwF@pREnTKxrLr^Rwvy02$!DkkI`!W3=ve!9-*HFBlZ!Jo<{go8
zDQ?=(^-mzishM@x(R-_VC1U?DonV;LRp$2KUE*#BgG;P#t=sC=zB49VF`DGHJXOCi
zgd-!G`z~+DZ$G7@a`UP;Xe_$>Xi<f{*MSENt*5*5_}>}5{c|#P#WuO*wg&U?`5E={
zJ@=M8iA@ateR$!YfICIg7TK^Y<;pv>%q;D4!F_>iv%j!4uecM(t5fxBg;5vl_a!Q?
zykGL#J)L%c_pbVN?_X~9R_j@R&fuEll^+u(ud9{0eQ@iRXQo**tM-3hqv&Osvvh&c
zeo?<O>$!`|9-UVC5!9)_tK7h*eD?cy3xBLyA2H24uT$>V?}HsNkD~IVysh(uC2p7P
zwVTnhaG~Be4Gxbt%kS)Eteq?>D|5(q&#AlS8zelfl)k%_6wQc!zOmzcc$@QEIjtF8
zJ2)9XzY&|7G|84HsUhE{;^>Zxd~K%--1lhRw0SQ1dQ$Jf$QMtiA1>fBDPN&9@zyfA
zIX@H~m#01W*m3Ic)!pZ=>@L)b<LmEF+W+?acI8F;-v}*T|Gz|6C2nJh$Hf=jQ9W5)
zD?bJlz432Np2_^yV8_?=ORfw4F1?Y;@$UI*wa8Bn6RUWRB_!YBQ<(gdZ4<AR=$_>T
z$_djRthjBjYxZ=Su*p;tt$5Df&znV$IfWcpl<!>=ZTOHY-OuczxW!Vo-J&5Pa}G(r
zvXot1l4N{Ej_byi;uU56N$<9XKfLYawp;IN2xHDSQQ4Y+_fi**epkP-xAxTe_;eAU
zSMjmc|Jbi(HmiMHqPsC;$L+04uZded<G9Sgxx4qH_3UlOwuRsA{3w)M7<&GmbGFpC
zCrOnqmp|t<cD|c5!@K*R>4gtB8m;>bBY$wn`0n;y_@m+7=}?uYY$|tT%K06#H-(w_
z6)mh4u;G(oPBfOXva9sm-D2|0<@Dv#)2GQ@5MW=^8+(t9OUkN6VmDvy*7~i|ueE{#
zj(k`f@3Xrov!>2@#-Z!6G7nsh4&SpEe&I3ed+8VDb9_zxL7NqqmG8<}@IuhKq(RNa
z?Sapwzh^a%6y7p<_F{=i-Hk^v*>5iS1Tk%R;J)&E<O*(;E2>ILdmXx0Sa6DY?+Vwp
zI+HD-dotGK!~KLGW~;RGWyBVVMHl5Ken0di<@yE1vi&s~^YR{5T*&wQ*zREUXx5>?
z6~+tV3g4)QmUjJ4mX@62$E$AHH0k(Ju9IJ)y-Z)8I{33Uid7}2LHL&A(dGp_Txa@!
zY_(e3Yh`#>>4%Td8Sn5lf8*BaA77opG&#}3kg@B#QaAS!CbhSB^WIzND4kpX+xm5d
z`MWQ*3SXA&_dlg@K_T&Hoc$g)mYSu%I!hNUdg-_3OG8J*&Sq)Di*ucWRToFBs+y~}
zc<wt1JK<#OEe|s199G=5y;!{KGsCf;4I!b=rcCQh4fN`<Gx2@&B=p7EUhkW_H|MXL
zt~qms@We2#ypsuU9^6dqyR>l23^S(-Crd4&1eeF;JM&ymWe(f9x5Z~u^T98<Q<lzE
zdM2uAwo<C_xkG_ilkMuv=LcQ{d8xB}TXI+W^ZA5r)v;ard-gvGE7&OPk~3#r>ftcA
zrsb^fz4Rl5#Y0c^onD^o*Hj{A%)#AKxow@H$^mU_9{WJwRkQTJ_O7%ny>!AOx2aKa
zLx0^roh!G$yUbwzyZ3aw)sB_#`LjAj1!InwOPR~_F~3^Z^J3#o^$^#Ej!FObt<(3_
z(&#w%<ks~|QCkYnmiMqt@N+c0ztech-8}Zvmq$-5E0Qv)SU2U!!za=Ar9DJ8<pn%a
zE&aK}UyI>X--bY&H@|*rEnt+0-Mx*w^xvCJ3)T0|>5^F1roHQ+K}bDU*p_DTgND~-
zX6F|)-dP!wxUti8f2vaNYv1?pMXxQ%d2^g?LH2bgrsjZ>S8baw6wVT}Kir-sDU|gi
zR!;Lml$J~8$^);xIXCXO=X%D2lP9M1gYonz8Kcj2{(03>pI0pVlpSB1@O_PNZgu_K
zgEI>a9;o)LjB!igRy;O6#5&F))>chaP@{SE8d-OKub20hEy|T>4`|3}c)P|y>ddtj
z?A4}x-oJBLrK0+GS%v(xt=cfxDQeRe-I+24H`|{Ty>6_VV%~Gtl0&p9=-C;*i4PQ?
zx$p~r&zPGeIQev2!Z)rROx@q*bk(b)@~jfY#LqNZHcack7kcC1%k6P9Wz?2kKPTin
zE7R)q<;Zz;uWnwvG56`KReE#$VjA9E73A5|+$ybeCw)Umv#Uf)Rbr@A)<yQJN7t5#
zTxBQ;EM2brRdUaY#EOm9Hm{g?)4Xn;*s${5!d&$NE%OkgU(p*B<;pHEJfX_Vv$OF=
z;qU08WCgENT7HrXf4)gss`lFC>YC5n<n`KJ!nXVu$~3xuGA91nF5i1UCjZ=f=6$HN
z;qLFYy2t9w4BeY17@XLB@a?gMHCGNb8dP)}c-#J4$n+)hmB!x%|0X7NF95-=8LKYL
zy!}qS*+%bGt?ab&Ib3JW@|Q@xe>chHm&!tsXIjr+p53wHxc|CbWB=yOttX~&>b<@9
zFH}T1qTu64HTCPB&Z@bGZ@+Eym|Ei9GvC+vq$j&v<FVGyhp)1}jl2-CbNR-T8hhEx
zpHDa>VzyzOLbMO};e(mHVh;}XnQgiM-eq^jq{SSi#(m<<ZMrVWw*u=$POmtUaN(GU
z2gA*i3troa{CrTo!am`S$()|Q(x;z1ZJMa<|Ne&NyHhW8)Sj8HnX|r__5MVuG+9@6
z-%hn7|F@so^VT}%+o?J!XDjbhLVC=mNA{b2@SB^reMOOav{hk}h0^Lf3`agrwaxv$
zku&PnP3f{lOU+m>bWOVXA@%hk1}9;aGarLLmJ2wnnZIK77VD>nliOClWqbCQbJ9_F
z#_0wZ#WS}wU)*HX!&z+{^zrxN@IVGrwzNr4r`-7wE+xtK=z-Vwi&B3U_R7_)a=UDN
zMt1E<sUjY(lbI2UOAC7peyi$kx^v?C)?DGH9hY}0-2La{_oDQdch{TeENOXJTPvOv
zES%^2*Jf$cUd7NJFB8Au40k@$X?8nYmdpBu-Mub-J>~DgcfBVsp2#o=5|nEzwyMsN
zlC6oD5_r2(VWwS3BGa)-Rs+AlORm!nuH$$4|M=Y_H_6r$nh&-He|#ksW@xhIM*D2B
zQ-uZJ5^hMHyQ#*m$~ISNYg4tv%r6VeoaNeirq7xsxv=QGkI=;;wNKGKnzemDLry4J
z?+B}1ex}9un%GO1o(8ML=;p|ee~XmEpC8Y#X%zFGv>-GwfG@yT%4q4^)DtQ0@g`kU
z4i%W}W6kxS<ZpQQ!M7J%l)Vh6SzVqdd{f+q(aYiO`MQ^{wQnl&yVeB!UtL(j&nBDI
zZ5N{V`-b+*`Hg0$%U=JPP`yWgr$dIJr&LDMv)hJCCMrwov`8^uOuD#T{LRA9vgZtC
zPt<cSUlP*Qa9_%HkY$nOhkzIA?A*Oye`FQp*yQZ6Ff-N(*tUpu=cj<hKj#T7<w}h{
zzHZMxD}l#LeZBt9tv+)hHo+`i^W90&HS%|2me2m~{;k{Z_J4~vA5PvB>GBOac`HoG
zMWAIb|LVujD;Otdr|155dOo{F;?kvUE-!n$E+2i^9v^b|-Nc!d{R=9;v0r%6Evm;`
zAM;;7r?*9Sa&&K3*ZI?h4RSkvKC8~QxE}ZGkOae9ou~1=&sM&)l#JQK9==HD^@B{6
z85)zT{Ox7CH?<!5bX?6RH-7cP(x}OMS(gc!os#~=^63}*x%zo2F#*=ADz8bHJ3M}O
zuutuNMTGkWbB@m!4sX5j=+W!Tny;TeYW<&XH1GP7jgzlg&FHc!SKj~S)+q;->MNY}
z-JOaF-=`+sdwNA^A!Auyj$xvybKBkNWq0p>EPL<pGOnb`XIZn<wG56(GpqCZ)BAI>
z`sPP%e&=VHBHE++o9X%zn{$lmhcE3em3ux_XJ^X^_YFH`xm&dbT3mNUYj|9k@SQ`w
zJ(-2CE?H8^DNW;(hr>ZH-`&5eb$(vVuwS;W(Nv4GyeC3A^vlCndF6?+Mq7UWlG+wm
z_H`!XC852F&3Swulmu2SxbWIA-{Y`HB*%qWlieot*VNBiVBP*q<hja?lnr|VKP~<7
z@J@~<V>{;*wJ+{@DiTgNJ+@2kJkjLpbA0FJ6o1V-nLn@f?34cacU}g^<o}9t^~?Xy
z3H$bO@51`;mIsg9unRw*{Uy5orO5R|EBQ5>pPVg`Y<>JQt3IY^k6q8yX3MCJPYvWt
zIq#l4zh^_1k5sgw(TcOqlkK?<a9_SUX{PWO;aTffq!%#u&Me;S)_=6^v-481??EzK
z{9B*RSi0)6^f}oZCMzz=t&y$ry`5FS+xK_3@QjbFO~yjo50~WGx5UhOBUrG9@9d&2
zJ+ax}d{$Z?uUNW)qxbjv2Q!u|X`CQ>FnRr=Ca3I-p3LI=%jcd=YyUCf_(a`meHQWN
zChapCZl;mW^JOp2c(}%8>GMfHEbhg1efn!FX4>g?+rRpWtmchs31)U--OLn;>i!r0
zz3CiV)Ru<3bRA2WGWYO0<5iZA7z^Si88$3_qO9i4^uA|V>;?A3AnVi&VP&q~Kil40
zN~l=g5p?8yvLNDA<F|vCP1&~gTzr~wVw2>q(}&DH1a5e@q&2nQE>HB=<t2Ml?#|s3
zXts0Hj`u#@(Sav!YVWpb^;X)oV@HMNx*%@Tu9HXQex6#u>@Q>QwD^P9q$vrNlh^xH
z<xDn-DXHb-W0{-n{Fr4K(+5+R<qgjTCOS$f*WO`0EfU)FY~Rfkwi!X0QfKeX^;!Ct
zdw0SdL;q`SIuB!I*_*yiEbmI5FpcGB*wtvY@M9IJKD-AMkEY1kPPDr7=zHVCUGhsc
z1oK%YUSAS^+DP8_bc)ynzVm+hPi_WkZ<5+3d1-rC$Wm9`nJJ176h-3%%gcA}X!&_z
zhC#)``Z@1@yDwu_3}*H?%kYOmK10iq{a}cG)s@QcLIQU#uH#YTOnUIU<mThaN((Qj
z9o-kX;i6!3OZQB<UBO~MC%A9iyrub`setcuTlc)g28Ye6Z&$3?xb9o>$BdYKU){{<
zPaZznH1D_24!>s+k^Sj+pG(^-?+d=9w2Eu*yBRN~)r?FocuU`0=ypMpCH_K-$Mu)S
zLDery4zo{~ce>7Qi<<wr9!34I<!3Ed6xSzwn7(+A?0lQ{*L{54`O5yq(eEvb_!*yW
z`E^xNWYSW7OB;RrRvorQJCpkue=f*A>3Wqx*Zc=#+T|4<w+fnS?u)A}R#oCVoMx}|
zL`#K*|Ego?2}ZwOf7c@p({HS0PS?84^v@~BNcsPl88t#S%b&2l3wo0!+qY%?<Wlyn
zj(2q4vbe^o%bKWsDKRzN`L?7l&@F3iJr7UBf<EW#l^$<y`d#c4y(P8W;bg3d=laL<
zVl%X+&n%u3x9~@uM4$hqb(d4#Ju$zxu0e6(X~Wa4Zyx%+sgx|yUR8MU#_GzR1r|T=
z7^=M0d3h&Me(~ub_ntX3LU;<9vloQ?58#x$wNc_g)d}6tn|@4o_xSg6)7+k>i+s#3
zhnEV(FL*Fd_tgJG4_mEQO|Np_l<KE($@pVtsYBfPE3e*8oc%*lxGKoI>q`AmO>v)z
zkJ6_sY)<ohqo?rGJ<M>G+NCl@BXwU!PieVYE5-RNSLBY#bv`$7J*lu~y3zWc>ysyM
zUeR%ZqxrNz)w3-@vX>;Db{}n*wlT~!=y*M$y-w-Q1i$%F2Jf$2Wcw!MCFWT9bVZC&
zfz-`I0UJ^!5+7!~f8+jrv6tfa<8}XIU;PiU%b0qks;F>hm*JA!#uqLDvkZMNo6Mf`
zFTU^a(SLI8`y2WrqGy%G&6&H?S8u=6nu2%5Qpc8f-!|~w9<^M_gE72srRlldOZF7i
z?v_rjZkT4$)84jqk?OR0ACgs<er&u_e3p&*cbLMI_$(nS<7tcD`1gM2n!F`ph2v3E
zj!sttmYhRV)9xwxuI=1i%P;c#o<<kfLuT`W6YO_Zo+-<GC{bmw`%v9kgI`^5<9d%R
zOn&_IV`|io3rA|t=!xi0F8+LJdET9<nvfU!`Yzh`eXH8;Vi%vcZp98S$BBgtZmi(v
zZHTeDXkm7473bcrTgpFQ&Ncbwm|9!%a_OXa9{FDZoL4UvRhRC{C{#T-$EMgzW=AxC
ztkbo!8_rkWvj6?me1&nv!rT4aCvWb&^1Sc;^kX#{E%%qcTORzOWL9+3lBt*aJ)TZ%
z&zxgx8Q~th!_cSt<7{oSio%B>L6YpftUW;*ryqMO-~Z~gH%8>(HS>L+7@M|P?c3ya
z@RD4q?&KY3e^mUP81}tG`O>~uAN0ST;5WaLvB`2tz??@1v?ZrFzj0Z2Pkn38&l8_)
zPM%>_=hkyEx%|K~Nov!bs}q`i`j7FNPVRdeUS+s+t&O-Nuj#$_y$?G6D#yBA<M4ko
zIXm3-<+5ClOaC6OiIZ@<!DVzY<LJ$01u3jw-0W_rxN)po8UKoR{^9A9#Z!vjHO#Xy
zFc8WVxVrG5(!JW{S}TOLti|?s`^-)bSz6~^yXp03f6G;JJAcUKI@e5-i82bX(OEs`
z>GgFCpZ5FM+)6ycd%{<(RQG1`i4=pEU#yktqRrwhT}4EsxR2N_dfL};?@QhLm(Kkz
ztIxR|lxsM6>hMX|xh$_gFXHD^nRRJa%_;S(Cd*$JJczwG>2G;JYGZw0T=1KU2Tjeh
zBjt>$%9Fe}rDljf-xDSA!>hVQr_$r;(^(=PWc~;TJzqN6TCDtK;p884LZmc!4nA-$
zpPy3Zx2nCTXR^xaH(bdV@2huS5>9gTllh^{mgyzD*Z9fa#Yy~US7fRkU82Ud#mCtu
zw)VoK&jkkec-Qm4{e1iLNy!&e!_MvVJ$}&p>!)MkVg4^mQV-rO(%5RacizMhOIhcG
zE-#p-U(c65GihD_q|_O01zeLK_vWlQV)4x2o%b>`>8NRbw-$+oCA~dX^GnWOz_i@w
zg0e;b<;%MLew$34vai{t=GmCNKgKp^j)GnK8=jjFq&g2dYk3w4h|N7dWzFmp?d!g*
zIs9KiJ-m6#=_>E3qPKSMi%6fNGGVQA)v^QAVy1q%pt|&2vukF%c9yM4=cipu_0O%l
z_4(``Ylnj@*To$QzO^s=C{@DLv`NKv-HXd!>y)-9ot^vr%A*%qk5^_*m@@10!=ICO
zN%909$tpax+*<5W|6!9AYuIOpH@-Q_wCMDlJ3XIoy44<)X%(yarEGI!mZasyvf2yo
zzxMnwkF|RG#7=3h+>*rUA47Y7H)V@WJesQdJU#MtdaZa0r@hLDNw!LsGE!Y8mg4F5
zT3OG3w}1O}UvcY`Uw$<u0j*nI`WIU`&VBUYpwF7CHxGYZcP2(rn=?Xg%H9hf1%zK7
zk5I0E;e7R-Wu5D);sVW!#!lM~bIw|0&?X`LSE^XdJ><~ysWB}Jt<7F_D75IDD{*vm
zyUc4=9^w5q{?^t)1HZ>I$?q&h8LM*Te3OlicK!dbY>o87OmFtjO$EOTu1N*`H~Cb3
zs^^wTU{dj4m(y9zcPE*|Zfdu87OtK6e!+y&h!<~i<ktBFYOjpDeC(C$|D9TmPc>`R
z<kvM^JiNWYyLPFn`?)2i+DBwFVpb=!-GApTyHoIz{BEoI<;NFx{_0LB?J=D^r?H#;
zjMgvP$3MjRzb8F8du+now+6<yrz|Yuwv3uCw`%dvdv^ZIri)zse({ml5#~w9zXj=P
zdU)_TG-euHkvlXoZeIiMFS*pIzxQmERyCb=^SWB5@07|9Pxdv>Dh)pq-tn$bDqQZ5
z%_;7aEHUdpEl^|GUimWNsM5lZ6Q&>g%Dw)}k~yL~mn0Ne9edSmDExpsX50M@=h6#Z
z_=}z|v3e2u_+<E^1urtL2=J$tuRa+da%)QHucX+yKPE0yjb{GUFIF+9aCcGN4zrm7
z&JNC31#K1jWUj_<7C3fp$<n^4kiAkjVx9g9#wur)y$sl%a9|Oe$EL;`2P>m0PcWKh
zs^^*C6S#Up;jm<*y%iU?D|e3bg>VOr_uoxK^l}{C_lnFvG;?3H%H<iHx-}QCTur>s
zDJijZBD?4%%|7?%8Alq$ZUmh87-#fmqSH}bqZmttONXwzU)bN#vCwz5?{lf##A$PG
zDDaiYsO`UPShw-_g5Z`jmkt`MH>q043kOE3tl7QmW=e9w{k=M}znQaw7v&_IbMG|%
zsk-Dr^6srZ*Dq`-^tH^^Jd&||A@8KRr@}?q{?7Wfi+cCnN%-Lt*te5QV^fFxxdf$>
zx(wH4!B6K+(z3SO^KFy*#r)o<VryG(d;Qy&{#r(Ng3O1A=Z0$zd}Oq<NfgpzUT>4Z
z^W^x#iN4Y2FBV%ACN|qz`z>G0+a_Aeb+lanC*u~TBMQGRADSEcH?><e`i*mTuH0MC
zV-hRm-ix0x7F3qMvSXPE&k3>4Ey8YIUyfNFd|%O!79TYC+Vnl^WzPz>@x5Enzx!gF
zl*hw29qnG<|CMY>Yicmc<UO`4!`_~~{Np9IPf03%6SBozuC%WSUKJhDbh-6>?5?+~
zPl&G9VfgB?!6Sa6qV|^Z@?~N=McKWJm-(!7aantTFS{~w7stA}dXE&p&xla<Oxy5G
z_~<@YbtV4ky#BkHrY~!ETDBp22JcI`UoSYkN){fhNWbx(TSzRVC{wdrEp%q|cf0O$
zXYbvK{9W|^NRYi<V4r&6avq<UBaI3U$*MmYUx^3%ZE)?r)2VhlwZfsg<`!qy0^Zra
zzfI4*)Kz+wU3oF{&ZB>dIeaafzFmFkY<g9BrTB#%eyS@2-*i_UoT@v`MN09F{+%af
zTlxhoT~<e}VST9QqMg<eKgptI<%!G*P7~BKxNX1ihkH+umJoQnnpxwiwYX@r`$d~4
zhu<BQHQP9!`K*L^+Q*iTBQt6ySLcgPd$sy=e8CpR7T@Xj86L|ob3U5ecyRAY$vaDQ
z4;?wZc-O;u5hmJt>kJ>fS^eI&fY)Ye**24)f_LYCT=KqtLv-Q6<KO4ht+HL)b?4yR
zun%WX=UtXqD|@fk*i#{jWwGkKsZ&n0_%5H4ZU5`gJ0Y2qJEc=L{U|xASEwQ_vTCI%
z-?^s?vSN9DxW0S!>zZLp>l>$1(eM0stIp4@6gkP}E%++_=t{2k2$lF{UM;$Z9aq{p
zt4$RTy~oVn{r>CKInffK!PQf4K2|#K*R`l@S+kFv+o=qri>J;u=6WhU<XgM_oSWp^
zu8D?zdAdjUJP|x~(D!i3y{rZEY6I_FyuL+$)!p8}4V8P}c%O7K%ey>f(&vn;1$Tbl
z^4_{**UK;7wdd9aYNb6g3RH~~=+fQ#<dSkk+uYc|xxc(FpIh^{)+#hn<lB)2mg<qm
zzD#OAlhI?pQ~LHzjs<adza_GKVLw@C`q{&HR-e~v1Iwk7FH}yRwpr&_IioA6*!ssd
z*-kNMy|xwY84u02uCv^@O~EDM`r?}c93R9k*memn^KW%McGz-GUA2YVhKjc1imzwL
z{?y$X>RWwfR#HG_zf_p{!gF6foLKMv`9Yo4&N@43o~^;Xs~`27_4-+5A9*}2cF9-e
z%}*U>Oz+o<nfhLpdA+=y>`lLe?@NyDIc4X!aBK7QcZ<Sh=O#<Vwj7hl>Q`V@+{3_c
z;l@4j)_&=o79tU0Z^bTdS$emibK(Ae?&}9F8!|4jUolSUd}bH@^_08dt_1>F7Fma7
zI)gr5{WdY>dq(>GXxaDL>Va%2@}+E=weR^^HIKcIv{-lhzan4XXGiJB(|TGJ4cGM+
z<;p!fWU-I$U4+2FnWx?U9e*C;^LAGAv|m-Bt71zGUM>9co>TJbALey;^q2iD^_6}z
ztL(<nkJGn*=PA|u%f@<Q!dmyV^%jZewy5^+YngOqw$V)A%uBnY@A_8fJhi<ic++R=
zrDuC}pG*!D7mJ&E;^WosqpsKX-Bx86Eon8JB9PrJRCr3$^vMCW*oRYBGTq)JwcXp`
zjj-9*3nd5il>a{oOTPBir*L9Ll<q#RT&II<p?Wuzo7EmB3-9y_t?z5@{OqkhmATe!
zeb#ivTN?~>%%dDj7<wKF&bo8M<<|P6uI5>54i@sh<TcKm{60gP*L!Jf`%<NqpP%MU
zE0Ep!VwucR-^PV*1@E?1O_HeewU%BeYpUh#C@edrsN$R4xkl}N%T4+0?n+@QPMuDV
z^|`(t5fYiUUFCdJ^&y4gEroA?NEn1JSu?%s)aS${D<>T8Gi!I>oFE)ualI(e+rPH>
z=3@IO_I+DUI<LR`dUx;MjFYcKJn~OHQEIzw)7SV|Deb-Gk^k*!Ta?2++bi3da@H1G
z7&LG6)Sal~w*T_^g^cSyJ0&djIma@$uW^U<eQ)_W+M2gEEnQz8>GUm2BcnLBe7@wY
zSt(bhpPtozI?8uaQRu@PSKX2vs-Lc`d@^rVN?zhRmYl4sjH(Mhn(R2cRdMGs&-2#v
z)Y}#$3d~Qoni|XSWa>F30~s0ax^wRxj1SCO#u2P~?AEiFAEGrsC3>GW_LJRTUE{e?
zW^&#~&+Qe9qa;Q6Zf&~#DW@sCW$($mv*L?(x9_mMwfpXt&$@kY<E77rd0f5dF(=q4
z&VAC>&5=Aku`6#2@<bkZX<A^XH6<$Qw!Lrk$pf~N16kPE&iy?(@5(Ee`wK<29^bIp
zAne!us;n$WXv1apLz>G!A7MCa!>@j<-r{!u`Q)C>#i}xf?|<8-_c=Y4S`=<$bYuCo
zPqSusB|Gn~yZHUdQ7*YT7fYsyDSs_I#IU(A^YqOmiFDs;?u~*Q7<+ezv~3IQdUd0}
z$9LvNiA!0<N8*odNck!0u6RUBqw4vKg>$}K+N*T&BgfecBaY<dg6o$g6?1Pcd9T;z
zu-rMPKUy<r{?#QOzHtX<&Ah%%V$+r*KX;#WKCOE+!6xrCpXouxM9s^sJoD1IzZBPg
zi@8<2XGzr>rqVlWXV#u{ILJ43PQj$DKQz{42ro1i>EZL<qbbMCyxrGMQ*~PNLJj?U
zrHAK~`nBv%^mzWfsO8qXIZLWVPW<f^sdh`R{{1uI`Htgj%{o?IU19fu<LkGY^I@X1
zY$r80&im@#{3=z!M=R$;WTtwz*iO%CrwiNXzIiY=TjuSOxYtFiw%qOE;+pkt#sQD*
zw@e-_y2Q44r;`R>;iMO}u_jJM{QsAH>Q-EL&@ukd^U(jZmNyscHu7s)F4SWymq;&u
zb1>_(%hNmCMZUhWWsdLK#3Sih$olxC_Y4X3Z{ml(+FfV$^R3#w&v9OeMEnZXq}Tw5
zBcCI3cgiRpy+1u&s=YXqY1-vo#Y)_Ud$-hTyLfQ@+`ho+UFY0=Qw8QPU%{GHSirZ_
z>)b^<&zCLyk+XPR{|fa!oTGN(NnFyCrz-qAKcBcK`-ID{;enY?kY#(@GY6CGypl=(
zcV8@!EjnHAw63g3xPSM?i)Ri`t;>4fbM8n$@T1(fKPm)%rmri$!8G6U_s^|KRjHH3
zg?~7|&z0d$oW6MDX@{?IhZWZcHKZE<yr9-tx2t`^vdxFrw}$z$8K~cV?lS4_H9w<=
zl24N^2HXwaxp&SQi>dk+AsZKmbDvbUp3O2TNp@0}uJ#<B?Z?H9J(z80=VfV{1>C)Q
z=U=|!3EpO1lNXbXl3$m+D1P&$xmWc=tiET6<f0{qgO@J0ax8xN;6m$%H&-q={IgP;
ze0ch^WT~ry+dqCt%y?jDrrFDO?8Ei4)zZ74Oa1v)(8SxAeR;x14L^(d!sp%7gjjtV
zzMZ_w7BnmB4&xCkxj2rq{TKUB^2GC(Z{_XzEuJ~c@7L|eEiAj9iAtI7@LRZud!dZc
zmLppiUC_I|;**Pg>^Ixqi|0<xyu%<6k|i(bvg}{o%?n&dPsEn~tq5`Suuhw~=CqgS
z!CfB?E{yn4qQQDmm7jUx+&;eJf$p{scic$Xy>Mq<MuWkXg=b2l=44J3yRLC~nZtaI
zz*qNn^IC7+$NqJy)r~3MLc4x1vY7j(_CxT*IahXmWQgu^NY}sA+mTr}<IyL%*W5lg
z-YkC5kuPU`_|mB|VWX>e{>z=S<MH;HJ1JfEb%M+%*2!({#h=#9Xfm-oc(i-L!WlCp
z=00Z8wcD~GS>i#=9T7Wy5$`G1&m835NBlowd1mFbb;)l9v*QKlt(aF<7bUOqcSE%A
zyBiv8DU1CTwr;4|Av&8)KuAF%Sp9_YRPCpyJl~lfnRw;X?w1lGlRi9=uQy?n`?dR|
z@AXB+9s3;TJ73)JJCiGbtBN7_r9u5fO^5BOQ<<fHrx%2t-(Yl6lx>1p=!FL&CJb^O
zn{+pd92N13&YyI)psZ(a6#u3>`gd1#oAS(jd7w~4f6-iJOQ&s&A3I+5zTWfg+g@o?
zN#<E<w|ARs@Bc31_~z7tO-okf@u*r_UNZRdta`449b;v|v8WmBVt*4#R%Gp5Yx-E>
zr2j@%Jyq}H0(-cM{uE8M+psam<KVm$*K>>9DlE8T?II>^XJ<0gv)T3aoM)lbcZ-85
zrdwt%IkLf$nR)k$Cv6cb8Z9O(-NL_3$W=XGHT%u%N*<Os3vCYlld%w#i7bk0nqpe-
zQc)xNZFA>C|NUoEu1q*t$<8h8b<=aNyxD7?Or4mY7pi-0eb-m@@=TapTi^N5QFwz>
zN5ib&^DaF4!&xY9wf3cX7ytgM42@}A_qSFoJ~Hik)tru6j^wQw+atGZ(hT#zw4*7Q
z<630p#qddA$_-d9i{&I7oS3}I*C2vj@sRYQsRc)-Z?T?v{MvW9=7V$fdfC^$dp4_t
zL*8?FGS7qF<gJCF9dnlnlyF@-tH<~~x5mHV?7_K0>L0$PeY^PD%Ho8?g&Mmrf}SS6
zZHlvJ&)QmUSnyvaIk_`IGy5dxB#A|zP6)cO8_bz`XR*^t<NL~2mTj^3+UjgS?Xj#<
z;ri11l}x;HCO0mMoSvneUuvj*Zl6k&-BX99H*5TD^rt(lzVGpBjzL${gR=97bboxy
z4otaU`$XL)x+~SUYjfgX$)d#*R9SD|ebmVrap-cw8HU?iUYe}iZ+84$qV=94sVtdd
z{(hgicdVE_*>_LU(cKWaDL3gAw@rM0$=of`8K2jyo$$NG@TT#}1D#t&8#EsYFbn4Y
zc<8nB@&xDA$)6VSuX=B;z$@`UvTfQEcVC&ka~`}AJ8?37Rpri~`=6YUlX`sos8rW8
z1<5M^RWFsSU$XT0o&Ip)XSL2=j*`i2x2I3*JNoo~lIM1b=#KWIS6i6m6H_abKE%f_
zS`qSf?{VkyZ~QGMoXi8tCseAZ1Xw>T6rDTc<M9dsCu`~JPh_&J>iGB+wLaSPJou~i
z&iT^%=uK&XN@kO^n%U+FA5q-$zTvsa+{!hXm$^);*Cp=qd2iGF;0>EQ`^0kFjO~}7
zsqbQGuSl$ysy?;mO3K%!vp@gNV{S2hukU-~$2aaCg%0PQ8y1DPyU)C9i?H-mirsqH
zlA&<@mirIp7+APC>PKwbKDXi8t?+GoPrgo{@`qswBX8d(&F>4-(tD(;d>i@=uFTbY
zCB7)Bq%Y@_(zBVT#MP&>?u)a#b7BR<KJ`b7osO#U%X!QwP?)hoeA2^@jM|O=cI3uQ
zx|zD6yCc+J(JB3;a~D%?`kPSS<*i37H@Lgx`^Z1`URkgxaIcWi-xVKQ&l&XlUE90i
zsula&Yx@>C8ZMImdTohW+rt?=%!Pk8USL%JD1UD2rz20c{?mJx@ucln*XHUEOaDb0
z1)tyTHF^6Su@E=w%a>~3D=clhlNt~jQz~^uZWiN(!z;ZDP7ANS+--6(U`~%%`FiV6
z{kbiFPsPfayfU3<)nqZvdvnva47ICa7pkv+XM3K^!cnq!iP}Dk?U!A4Ml7@2alIsP
zvhUH->yi!C7L9^WV{h{Np6*@VIcdN4?Z;D$cHUSm!X3Y-)-CR#*}rAZIs*Gs5;sLo
zF-<a=!@;(`huP?0z)E%Y&6g`H%YVo$owWOR;;Jbg0_m$|{|2#7&ak@dnPA=`uvhI?
z!m(n(ibsdrrQ5Cc+I2H$Wa?EfX`XBFbgow4HjaYr_CAxg*iMqy_T9fX{e9NyWb2t4
zN$OvH-fdi@ZIGbDJL$!vsg17Ijk_fxr@XYe=j$D~@^;J{W4)UaxA>2}UwxQEV8yG`
zd+*NWzJ9S{9?$vx)e>(D=Q+<=P}7wZ5Yk`K;@BbAc(F1pWlL5-;We&GjRa<$lBM@t
zCyIQQSR)W(Ha%=}Z;;!*Rn-r;eLtsMGrZxq_I>9Tq4SE2dKn%k6@omEImiiVl~0<=
zbmoLrxzoY~SDmenDid54{i82g+|!*=-2H7+`;@24mU>*UEk3l1cgDN8u-c1xC+-Sz
zz3%SKKi*n%EAs5At%shsEs5XbIQ4sKWTM>l6ziSOYFCzh-n96sdgkU*CC=>sZ09c>
zeAM?bvMc6dMvoSEb=^JTr~~tsh8)-;8S+_g>E6Q?e;-8bUhI)9=EV|fxm@cKLy*Od
zFD%gxYpQw7PTI}mP!s!fQ=z0r^-0%OPrK?Y*1Bzbuc(&2ybviB!F>ABlWA!zQOl~C
zdBQgZZ)lfN5)6CMzGwZ6F3yt<AAZ{lF0fN%l?XFAHjhQr$$PD^TikA@$%#Kb75+5b
z>j`$6JTWKXcyfyj+c9pb*^()1O^>~hk2=0bB_N^2RF-x63(i~Lt#(XJHIL9c8{JrX
z&b#YH%OB}2n^?{Br|-GXA1SP;`p~(4-4D&E)b3=L(_Gh?-~N@eQ=L1<_if9(lOGN)
zU|OCtJIl=Lg<akb!$PYE6YYby_W8ZOBKPFPyx3Pw5#N{iPIhrp{krnKrK>{bjQW+2
zjDB5NpwRrzOLm+6mO4>SmcrzXSMKJOZqv6tGvTiCQTDnW*)<mg*PiXZ_e`TXzC-EK
z)~Wrr97Qv$9SeS~owJ<l)i)3SdKW<p^*c!{n>pqyUXp+AJa0j}bH@3^0-I=qm214}
zK7C=gvX8IK&seQHt8dwv@_@;bWg&tIj<VNM4&2;pn-!$1;ZoG`Jo~@J@*C?MmVbY8
z{CRc8PSyFnN&n1r4Y#)(nkgM|{(|m3k4HD;o66MJy%Afd-n8^i@}#XXO$IW@4@W2@
zUC7WDJHL6!mO59aN%_oMrU<X8*|TVi?VaBL33u6>Rv-G9;jCA7VQ<WathMqvxAM$*
zBt?|E>}MrScILHK-py2>6fI$D`|i`*_{jR{ESIhX%?W<4p0WPnL6NRyW;@-vi#g&W
zHZ_^Z)f_8*wB)1wq<~`|E*x1nv+LaQWruA}DA{&reRIq3a@@>ewt9{PchBO=ex={a
zKjK`Eb{8JK?))}N-6Bv<=DH+jb=QrHfz$YND-r@8Bs3q{ci_mv9lxKjSG8_6c*t<e
zTIhy<ytdJes+FZV&#OdTFY!G+nYBcMyE?DV(|P8NH$`uz@~;h-6>I(%<k#rJzMgx#
zOqN*7gK3wh7RA1ju+WNE7rAJ)YF$$6>ot;k%n8RA&f&~W$-K4eh{K`uaHHq)LQS()
z#Kwd)g&Yx1WxCAtgzaqAt?wKAe@5RD`TAgnjYsCYWUjfk%HD4FAsZ@mCOW#Yaq6a2
zS>D}}k-4CK)*A6SvrqJ9S}d@u+#=w!$CGj5mTxaB_-^rO-h41?+FIS`Q!{Smv}Nh*
znIEiq|NHja_p^Eq9kTVan&0?$+HKPpyE?vSP5kh+WlOYB*6fth<T*yH+BGj@y{cYT
zJWx4j<bO`|cf~S`yK8ssf4cCiM2W}Fof{T0@q2r}YgcbA`5HHG-J?i-Q7NWxWjfU<
zAFPh=?($eYgJ+9z*-OSRt90ayxpcc6qaD8M%G?h5lqGjdtxd1JU^#!697BLaQ_%l6
zG7-n)PsX|J7vP^(v4U^Q;X_pguT1;bH9yP^cems8J8zS*Ds%gz_BSqhyRV&^X=dCb
z$aMK!h0exXA6q3FYmdKhd$C#d?(d0>?$rnSU#G4%ofDhVs@KNCvo64J<K!2UI@U$p
z(F=dWRazEp(;qcQ+oj9BZ^87_fjqTUpQQXZAF{b~#3JJEw;7J^&&-bRS2x;qO^u)L
zVyX&PuSZ9#;#t0lE(Ry(8hgw?df?Foh3l)Ec@5QrD<6yfSR(lFop@l)-%=a%*aee9
zW2fc2Y+2MWf&JOLnEZt=SZ3ItG{`cVr#j<tb@NglMptq7jKdWg94&oEFZYSvQY_<s
zR>-vXxz3JBw`CT+UY)^X(>3KePaM06@-o8&8I8!2-)6Z_>z`UZ`?K)vy#l>A4PV=}
zwsjx=vSay<cXzuja$6*97EZaH^Q5IE{_#|fS(DZukJ-55Q_`n8ktdov1)H8sEbE)Z
zrKkPdurWF6>GzEIU{O(jmm`N_r-bi2_B{O6(QuiJx>u^t*OjH7d(8dVskr`F@w2~2
zYHJJUGP{dSjJfxB>NL%T4V^18rq(aL6*66+;h0kP&P8==7WU0e7m_x4<as%uf_LtN
zhWL%L-o8R_Z1nF=j|lm`zR&)_&F2eO++29st&e?M@QXFd|NYLKIjPw$cQ>3(y354u
z`tNz`0<CnjYD6`c&UH>vmtNI5>0^e8=<X7u@RqByd48Fc+8CIA3OeE-x1Gt4$Ga`#
za>2@A<!7G0CaS^bj9%$-rp|i4LgjTy@w~6f*7lJf^w<47@$9}!$-|BFE;*~kXm<Q}
z+g{joV(G{BuZp!PJu@P>jxN0_m=X3_CnRP1g*-FMQ!lgfifoe3|9tR;Ni6QM)N28Q
zxOY3-{G3<u`26CTwccR$t;mq5yS6sF|4Gi`vV7xmf^p%L)An;Sem*ps%`)kv;5Wb0
z<|S^+eAjjU71f;CT{%nb{qvm%1>^*!w#}YUCvA07GNN00=lPFJ>W5nT;uq9i&kQ_P
zz_d8ENTJPaE6cBiYYucp_08Z=*?VB_sp9mLGhc17u3Pf>Vi}+IbCK8E@Bg2)ZF1*7
z&hSeMKD)XLHZ%SC^qKwcY{hFo7S+qVI?Ks$KH+j_>B2=z3i=;iDVXqA*e6Si_n_;?
z<rXs@y~&O)T304DFXOarA+Orw-37^!kB=P}UiQ1iydtst9oOsquM?gY7FhQD5RedE
zcw&YPx1og4!@r-FKQNoQw;=KQ4YhyMSq^kY#D@IVusnaxLgtOojm;WoCpe0U9I0AT
zC3nL?|LUsJa|#}3EV#b9Up}Q=edl}LS+1L=siJ!lZBh>0E>y_u|1A)EOUbGE(G$I&
z9Pzi-Xs$RJ=NYtr*|pX=GZXukOlN1??<crEN#-%*%{+wwzPWqOcy23oJFjp(I?zO7
zN(Z}%!L!45ZR;{34&S*||Esm++43hFu3T*AzrO#=s-<aL7A|FQO+9&XS?ph{r%PXK
zczSsKnOVNa4ya9I;pgL$z0i|<EuQb8`{l}uK@7*Eo<CW(lzncc>SDv{xXb6ZIp0lC
zZ%$`y2|Sl<a(Ie{oKgD5bBm;3z3`i`K>a2A(HBNDmRKfqAOF;|`-yMhoZyQ)RL`3`
z`;{4aH69bS{&V!kg>?e|IG@(dFPd+kCbDkA%k7g3z3TEc+$@5(T@ja_`ab;c)2XVT
z-%QN9l#ubnGSFqgJ=-NG?kd+cAJllkyZD2j-(%Oso*T=R&oT3MwOjoOxU*!=rKt6<
z9Sm12;(g4Lz4x`x@_$?_*B)Rn>3vun5U@A6Z^nf`W(KpwO0P}7{qg$UGXHCWd-fg;
z{<SUZyVuK#e%*cb{yR>pC2vgXGdst6e8G{)PA}dmJ>FD&_WW7hc2~E@cP4n}UlMHG
zyiwvT!))yr+}0PrSFo^MKeu#8y|;4-`}W=6%rmCO27k8dzIEZ#m#H;Y*K}r`x99qN
zP*v;p%VYBdEx*iYdKx5b%k398E6lkt*LvB(Bo1?4gOB!3^XKMmRAiOeCMQ-ARCM`^
zZfTLvu}Fyt#a)ujTix5A9@O|GF#ES9*YxSK1rF)8nTsAYEj;<*qC-pT%^ejQYP^p8
zxl4G`E%x2&(!F%@&&wqrdOem`Tund1ToEbqHsz;In)1JSW)9kYZWoQG$FP24T0ZyN
z>BFYpS1i6{e*E0!c7!eNp5@z*3pa$uK0P*Vd3f7yze7ep{;Sk~V&D##Wxqk@WoPbA
zhRF*1qdv@8nXoCil}{>>MarV&@$M&Xi<%qXJ)Rz$7Fgv|WFz(U<o6fbVs<|J{$umG
zFy*XIo-b4$Hi#8UH0-$4Bh5SY=<ni%Qy;PTh3&|dd$(odf@>v`3wLkp;xQ}`R8ra!
zcjZ}*PFFWirAw83`fSES+Y0<WGW6MI|C!?|SdeRHbkMHr`!}B0<?Ckg&tIDTD(ch&
z3&(dl)jPCLZRkCnYAKxk!o@gsfz0eKiQ}s;=`sIwS@gK{#_J}YM*&kqj|8+W<aw{j
zd}@RC;d@gZ!~0@g-+<Wbcy=4foGO^(CUdHAMTfH_U#i8FhXHNQl6{vuf-U_-3$+ff
z<XLVs=Tw2#;Yz7Ot>Y_|_MQ3=+P2=PX6nP$JohhmtS_nwJrXXt?^J>2;iWv&jbfGy
zPA`hldKA>=Evc8fV@jgdVVm9=rwl_~-u^zoRu(fgaoy#!@>f1LhH~iLsQ*3xuSxQj
ze|9^k?yCPUYuCZqvrcf%*0U?(GivYj)^1OBW=XgH6)5nCyIZPG%T2a@ZT+J}tBx1f
z-871F`f3Zm|6LyRX8Aq+3S*Jf8_E0M>+M+Yxc$+tX!ofPzu3FQ6s6tp`a1i)z1iis
z>jo}2zF+(gxz*%8|DXSl7<m@xwtRC7t2&nHRZ)NGwfbE9m;QhMT%32`{?>c>h4yi7
zAO8GgH56q_`j-1)QfOiG{@>eu&n|w&@l^GE?XD%22TJdFB`G~UaLCx>Q2H9r<o#U!
z$NK*NTamDSe(e9=eP`-_IXszu>vi+tfbAO69Wzh-|N3n|&&xE`jZ6Ps`l<Q7llSj-
z`LC7l-!sckJ)U`co_gcG&Be0&W7GeX%@y2V|EFy3d5Nl8pHz!`@ApS6+%Ny5{vz}D
z&2?<QFW3bXl`py_n=xU|C)W2-=g(e_PoKlGgdx_wxO~nzcK_=$*^~dBR!VrMlyfza
z{aw@Zmp`2{Z&r(clWOU|dt{r+q!Q^3FE?nHpE6c`Ui<Cq+2BRy{|z@XFR`6c|5_wl
z)<)z-fR33oN42k<OwZRj4yD=3d~4=h%YC+0P*H30THj*5`lJ8eIvC9Hms_)Em)p1c
zt$*K5wl3iR`00xMfBwEF+`o3Ns8qAj5j1?<`#fhxt$}P_*y+XcufmfOl@2~%8+}k}
zKmVM_ZNZoC{@0G;oOrwM(udDq;+Nleu3s-3<DyvQ7k^)S-(;!(Q<~bF&+eVx;2!Wk
zRbl_?zLQJVYDjoxi8Cp$m>ZTH<GgG^$^IG6_B>7=?2)@)<lei|Hb=uf`n%eLNzWIU
zXs`T#d;j16jqFlw>6e!r-TH6Iw<7z9JN#nx7mr8%$x7+TezN~s{J&q_ao1MxI~;#`
z-hbNK|E`9wKThBNE52m2-M>%bqG}~qtXJ0GntSc`-0RZn-<p3+KPlU`BmS@czfafW
zOAj}npDg^&<o>&s3&-~BtiAmso4MC`ofYrms#oV^_I>}k_viFS=YnIz{_J;*e|+Uy
z$nl@k?-~EURQ)Wz>by<wVc9uzE;xLAY$Y>u|Nqw_r~c>LTrgPt_|5IRj`3a|^W1(0
zeJhpvesERRhlmGT&M;`Q7+b$FEjjg8++oRsHGX`uYGtxXHw|{aXO*=4ATsyMzLICI
z-woQ?G%a7SoS3#PMbnyhvyILTgNFy69o(=b{MPbUogYG_trmK2<q<V_V0O^wp63qV
zYqNj%n0;APyehHiZrGEnFYi{GJxwkUW{eA5kabQqq453f#<lSZsvDb|0~;c@eU?45
znR%PI&lY_N28%}`Re>)mj<efs&SlW+>TR)Z_|&7PeP-JOro~m)>kdimyS+|iMw<1u
zw&dzpdE50`IaY-v)ZLw*ULWPNaqd07A1x2hRqo{PPt$uabyn{d|4p%#Q4+uUGHRuJ
z<gT?}^Sky=P<rd<gycOR58YR<i~nFEQ+ZTeis{Zez5bhv<)mk<6}ZJXkGoFnMyT_B
z`<-_m22MV~_B}P7VfDVr%TBdbmqsxDW-0PA6XEG^aO)|MIQ!EuE=MJ#{ji!&*j(!c
z7jDNzIT^Yqx<6FuE|tEL@a(Nf`F*!vM;9+_i_GHJs^LEqa>v5wsrTwv2Xt7!%WZeg
zRB4`fGxU1H=TASiILb1btR_B^mngN4X%2aRVzOk`FJZY(7m+J%$3xzPZ#>LoD<Zvm
zVWMJ8qR-3l842G@HEJ#$ju5qG|Do|cf_dY^?z0c7pKj#dWLpux>Fe=2p|;t7c`6id
zYwhCXmH$u}*`#Y|wwfiG=MUE&w(AlC3QGGDx|L)EjV=qGyB)Nd{m&n>lV|jQ?0O)<
zkkcUAerP|3`0=FA0UDn3bsz5I5!rQDC4T4I1I2%`mhc|<G+S6B;ql$R61VS5-aKKf
zJ{a+1X;P`lw48H?@?*qJXZ4oVYRtB}mHYn2-yHs_BBC2SBG$b1{Ou(0E##ktsN7NM
zHKC4QJ|^e?-*YB*@$1&Nn_VwRywh;5+|fNNqT}xGqpNb4*d0o^yjMZ}T@e4=?FVd+
z*$B@KeJFQZf9LI@*+sT@6n-VjsaNJ`Gp?{TchCE5K5fsMbKPx+pIi?4B3_%g`IB|?
ztCf-W78dQxG2QVtZbJSBW!Wcke3!*TrS33o?A#pj`S9ZH#<y25x%%bA=@|>R{9L;(
zE!lyQC$X*RXV4S9Z4bOJy%IMUyAW5h_<WH6@o9VAb^Y%N+?wI;ywXb{(u?QNX$?8y
zu&Hmg<!bgl?^)=(h4Jd%iPvW@|Ja;v^QQlr<ngKV*e)!!xEi;=?c0Q!jgyX7I)148
zUn%A+yL`nPO_2*vm>hLJwg}wHv=0^Go*XY1A7NKgqhUGo`}fA?=?x{yW>E?UA|5Q*
zr}^&0pGje#)?7Y(@XMveNAp|nw!XF8uIL%}Ddh3JE7x5*KAN4=$eg71#d2P{dBj4F
z3F-o?JikfY+FGc4ZQtBRxkZ^LjN6o-oSK(mm44}1{~^<va>xBMo^*aZuvpf^LiZih
z;%4hD@=|yFx~j^qIf!I>NQZsnlKwVleP3eZ7RyBGhe7v~)K3~ey0=I1nUp_w+;?V&
zX9126%*6!PCEltQ{m9t=IY!%WZL`qAmpt}AUKQ>+Z}aGr>g8_fiypa;^bf8z{lnL+
z^wq-N)Yd1Koqh7Wr6NtsxplgiRf;8LoI1|7@$StNe`NgvOAaY5pSU~!`;Fxmv;I7_
zUG!emm`^G_K=_^{?@D(Ghu@iA%eH<wwVIJ#Pa@0vvgEJpBB7tw8t?jj^Wfo|e-w8L
zzB;?G*)jH(hCs;dW4^mgGQ0RC%Vs#m3)vfdJIKKD@8RU&6vYo+C#rg8__&=8-;#V%
zC+A|@m83n=ZVU~6d!k*7yqTsqy>4Cp`Lf0?tL@Vqizd$UI{DS9*C}Ro?BRuH=N@_a
z<h0|7ZQ8rzR#v+=G=1loYU}Cu==#ej(+%^d+>#4z7r0%$gze?48BLQM10v%$)PFmC
zW7cn(*2IpDMPlzJq_D2yF>{{(Q=Z4CH0uKY{@7EYkM$frg%~tA9&ZzQt<uIGs+2YH
zN|(UwDem9R(xPs4{kpn|XSdGFzi*ekTsW~n>eR90hO!-wOtoIOc`m+uHBtQc(^XAD
zyo#Zv4`gQvIJquMJGHs$XwZ!GE6J~=S#C{o^R7HrW4`U&e$Fmqx8IL`-nk&`SZ!Ce
ztxdXBE^FU{cE@RryWH2Qm)_LdD$qFV`ouOH0qIEgG_wiZ`;CqVEqiIRY(aptgX+7d
z&t|!BUA283I$6+cinqeYADoq&*9BB>y*pLa@}TO)>08T%nL4!fJJ?$frc^s`(d>~a
zR||N)eCku_Eq{+ivMt)c`&#a9&I6g6#D3Wym9LZHZ}Hgc$Nx|=GQ4?y$L82Sf1WBU
zo@MwOFPGJrD}R4)$i1c~+Q$SBZ)Tp}7a=%9!Lsl>>u2s&?B<Etf_rA#S_Jdos`}uq
z!0<7Awc8h`KTdD=#?LivJ90JeW#<{@BJ+1gEDEc3d09TLPM-C_BCu`p%{fs=FFLI>
zDEV&ZvWjoQ%$b++OM0%@EuOx-Ry-rk{g3jjoqegll-_*4`S)(*^Sia*KTWg!@G`{U
z`-IlVlm1+^uT3n@Tj^XTGjH$gKmK2jDQteye?mT3-fF{EnN!bQrp#C%pRr)mWWm&L
z9)FX5Kjib5P<dG5@`v-K9LH~lv*qoEFV7U3|4zH9_~ew^?9XMU6Ji~HS3dZD$z^-C
zb>rWpqXt(OGgW6cbyt0md0cU2TQ+xBu)EF23kRwsuX0pBj4O}*w$f2o=E)2{$<M8u
zGe1w-r*flCjG=saw2YGM^X`pVjfbUfa0=vB&RLNDJK;-Z-<OJo%q=!Dv$wP;x=NJ?
zFut&{{%gQBsZ2~j)S0o)e)drp{m}5>!o%HaeJQN$cRc2ViZ52bxXm%h=DE)sjfCtg
zizgnMJnO@6YZrH?2l=m_{?2aX<6sPCN%o0csj^i=?}0<td(J%pN57{&zY#HaV-iC^
z64xa5dmiFX7Sy$Iah`fGz43tEEg!b29ZwuDX()u6ubs@}bWf#I;7`^kj{VnjTCQ@Q
zY>9f~e3XB!ZUvjo+dO@(vZSz0`<ILSov?%Lq=24~Ag|_j>z7kGcK-@G&Hl;z)}Mu)
z8VuLH+2lDocy3NeQcRlkRPMs-qi2KXS}42h`kR^Gp(G{oxTDRp`&{|O8yDMC_`iEE
zykrx^q~sV~qS$F5sHgQq=DqKkqwhb5I!tMZT`}>%57zxbDsER?tq<<Am_7TKbHZ5`
z*{2zAn?HSjxURp{N;P!)`@0i(Cm!jy>1Ui^<K9xSWRd*+>WbHIPIa~~bI{<ubp6>>
zgOmrQQRf4)WxqNqJ(>RJiRTN~?=BhnLbn4~FEIDhpL)pOlO<KWNb_U}tH03cyv#o)
zY7eI?3oVS?m-BV~jLA)n4NZ3EeWNsf-~7Y)x^2dkj+37|d^OKq{I$~D^@{gX38ySs
zUdg1cWsY2$^E4S1U3c@XnD$HTq+9qBlM|}~bl5qy=LUDr6#QMQxglX+j!?v`5RFX<
zE04d_+dY+iYE7Hmm31>1tZVsnolL%#^e)n~m0u(^D=^S%^_Nl=u6t3FGD5z7w{mK1
z)QHXC&b)GKY1$qxPq~OdN5+R+><umobTMi>FB8c4$&w_KTo>qfLPz7MgxTA=9Z#lO
zFZ_5q=-~An*|%r76uN^aC5Gn(ER^E^ub_3#F5;B@vWyd(q)#o?_;k<NZRy56FSB)4
zedC$J5oBVwS@V%Z@7t*}rC3|C`j-lPsq&FGIo4iw(m<(h1$WR!cg<-`r!W6I_uy0g
zyVDZ#5ps?j!=;0IGL3niPQ12bGnuB?^Gxre<dqdyyn>how$ByTQrhZlUnRMAZnv&m
z@O1u;Z{HORIizZ9Zil|Kj8rPJTB_Z=_@dH^$%0GE*h8!8yuXB>?c1{V@YyZND-Z2o
zt9<97``=ev0)8xQQf%Ab^KYB{{muW(y6d;<R=eLnFnJ3@o)yP8<(hl>;-{<63z=Mf
z*J~4B`9q+Wb@!`<kBf76P0#=RuZgYSaq|}Oi6Y9m&hGc_to!4-LjBgSU0)>MtZT^6
z(Ej<i`q2yK$-kE$XmV><G1vG1yu*{W&APum;darityhk0cigBbIQ!z(mYRxJ+w9VE
z?#MkcH)cC0XWCG^^WFV($`Tt2O={ZJt}%9Ul-g*|@aLPk|HTxQ+3ZXS>n91swmN!z
z>l0ihy*J{${eSrb2g@aY)P3@P6R~=Rw!3|Nl8l0kMe_z;;Zrt6cOU03uvMRCcYF4S
z)HkmRA4pxPOm&yzTi#W7NAB|1S<ayh<r(=*|NO3HabMs0<#pJ-rGL*SZRT6?&PGXM
zqV?{OkK8TBwE}ZfyF2ebx&K(v#XEoP+?8+dUuJ8HJ?;1FrnZXKqcuUtQ+hx7zF>Lu
zKv(Xyk8MHr+gO#Ut@ju!H%4r9_)_@g@K>kA=?{0$xM}%=CszIT%J0{%x~@50XXIrj
zpM0+5g^KK{s_T5a4z3W8{eOPX{j&DNi4}KWS#3R*_3EU4#IN1E7r!j~<IE$F@!F=h
zuW-$7wdq#28(X&RHEEsoP=N1)RO;HB$t#NU4IW)Fo5bSHsQGR~t--0pTh$-lW+XX^
z=<K?_VEgy=O(KdX)U5bU#(h}eTzI{1^MY6F9{vrgb`O-)SL+Lr40!nayN0%aVaYy|
zK&jH6WgHy>hWh2T4$apYaz%``?07x9j5Ac%XX$~>9dn;uH<ordb-{s6@o=QnDeK?4
z%{`g!%U_3=+V^r?t+oHnSIXzs?jpb7{B-T3wk8R$rYxvbm_M<+dTG{@D?NT~8fME+
zik|m9^momdTMHbPZd~&+*da6BTJaIfFOj)VCs|n~FHF8CJ-MoS`QAFG{PzYTr}}r=
z%j(v}>KLzhyL`Umw-dGuA2*BaO?dUV>9)D1xRkk4zM*B%?&N2kMH|jNpEP&g)QS~b
z%PmeHzhpEcI$yOvWy$gO_Mac#N`-X=9pPs>Z09a48t1XhbL!+noS%<-M>aN3=e{4R
zm}T}M?3}uYbghHu^7_^<#qqy>tljp@mp81^Y5#losjUu^x6e=J*Jdl@)?2UYl({zi
z^PZG@CBBy?WHU=EE8dW}#<qMzaS=;!dfnBc-b=m4(spO!b{t&ipD;sjbKj1<rN<V|
z-1m`R<Ik&Jee<L_AMYtVK3BHWO>=vxMaFym`;uE81dAQY>d$0erRpO4H;3cMqBc&C
zU7o!*7MmQs4mLc`TU~MP3-?hu+2H4=7g-zL*x7bUUEyik<ZVlWuQD0+afJmm|N7)I
z>zKdw&bK1L>N@RPez2Y|eL3U%fy)<N)-=1GT>nDl)`BLh)~3UC?EO#6HYu9U4Hgfn
zd|`6oVRMy8MR~`02SF);f}bbE{%1Wg;^S0S;gd6HjpH|O&EoNuO6m(Vtvs`Lm+_qW
zFOnAq{QCQ$!fNTSbBv4lR%hpQ+Fml=wZZzl+}n6x)+wcdQO_K1DV{HEU|!VsQOoJp
z)hda$^`|3*le2gZ?_c##%J~#y(hbKm7u$L!Pug#Cm+eT*2g?(dxnX~g*gQ7xo1wgx
zZL*iRr?gzF^C2b1Bwk7L1+x|fZ{R$+|Mtx@$y?M9v$mHWkloMQzvV@KsGJs8$xp@V
z>TGpmDQPwKyKOrXS6D7QSlXLeUH(m0uqa_$rbE+I{i)&FC+&AB%T3B#kdU=+w$z09
z8!vADejpVnEBHg&OQ*urx$6Js(DF;Kzg=ft&aqp0@wzs>O_dLp>~xI%-duKBvtqBd
z{SIMI_1BX-*`qty)UNDXXkhefWyJd5iQn{;%pU#QYIrv>S?C4BIl0Hxp9MC#PpAz1
zes6Nb_9-9sE?8K8)vK4)+V-U4WFEB@&kJ8$e%!i`gLzlC-L_M9N8Ze0Pha%(LuA^c
zR@SMt8QRZZw^h5YWZ22J$|<k4)?@Fl2|p|yRIAjFd!GDtW$ALgh%MVUdA<tYEA(Q?
z#I?_-6f6w(m^z(3>wHM}-$e(j8<)D@*ZTdl<hRD-gI}LTrKh|x-n}z}^<k^`>5`*C
z_c;@_kNup+P?fRm=5A*;^UW&kqE42Jt_p_r`R(;zcFV^5?oz+{1BuIbr=0ZN<*em;
z@OQVU{Gu|0GPWk?ERIFdp93E4JsQ6}M0CM6waBIa4kkR|FAipT$;diYG4@#by^488
zMu)fU`5v9H(PF`xerfiM)vy1&Ub?HVkaPA&=8p>(o61dju=;?S5W@sT$LIX#Hm8TX
z$8fxw!~Cc+pq1yvE15s?GS?33cT8^8v|p$4^ZeJvzb_rn-x{Va>ON(gaI;8o>EgWg
zd6!k1rA(`N+%>*2hc1uX|6WYFKCr2gi%C-Cc?d&?;+zKOzjBw1Rz6U+Nm>4zQ#RG;
z!m$Uxk^&Eh-_QPA(sk|r>|L@?zdu`4u)xabh#7<ToT-Ozo|{s{Thx`Z-NEbPtsgT5
zeEk=B#mxB_s#0`h+9NAxCT}&lk5=qcYTvEN5{Q`V{c#!Jj@RWW(@biA9@pG@;sRrm
z`qK@u7q6-*CT`p>U%5$t--T22t>@gnt&)7p+s2^O<cmYgagB->1<U`edc61Ynl^^Z
z-G^SZ)a=`=&?50K>b&L|-$v8^LwqlOKHgNTaACu@7$Nob5}zl9@GA9An|Dm{Vx~wm
z&-F_Ue-&r*U#t9Z@8+|Zr-#lQUlA`*s9qwQ@%eCL#)YV^3p3{E{-|o+;(Y(zqH~wO
zaee;tW8MdY3Dqh~oIZIL?hjt~EH&z~wRQ6H2?>uI4eehrK5yx>v|g}xb*=YWKi2;y
z)d8OlK6E+AE*ECA(5LOhr)Fl}+h_I}1v<a+ysECD{H;-U{lusZb9s7gPuK;1mbNbz
zkDS-^Aho8f?tex|b7oJ=mDftw=by~JXkfp^R(Bl-x5oxYRvDul$AigN*oCqcvgNg&
zN%=Xqr>&f}FpGc1yEn^!TyR(In4r~gK~K6U`&q}6q9<t{y-pYYuZuL^7;x=-@4Eh3
zAvY|oPIMhFSf(4-JLAF)%LmLwU2Vl3nu4q58}lW3_%GR1*KL~;)u>u`qJdjc_1+hM
zQTDI>M-;;6sw~)Z{6+oivbi(DRgTz1m7TfmzwGLT29A^isuNyIO26B^_H{_jt?;_T
z7X+7cXs_0&-F>TPfx-SeA_eQ_i+AR-y0R6V@~fS)$hY>9g>u0s$#;DdT^2A*@@!D}
z`Q=~m6rNd00?ftxZydBbYQKQZbb(?;>47c&&kmHCUy5&FWiW8Gay8ttIZgUop^z-o
z)cW^vha0^T1HaByd7a<=ucfT3^k81ox(6C_j+vD{_fCIh_F}WI)rN%J{^IP%4GJ69
z{+w*KyiQ7RYxJjAx+Ydm8}gd}iA>%q+^OQv;i?dzk$+pnoTt%_FP+UL%)Ov;(VZiF
zPuE8Os<JWjXz35$&CY20@psGqvhe!;zc)KtZd5;sd%ND?BkT4}7VO?T!Z}~*Nw0Xk
zXOgTyd6xB~GoP#fCMT^I3)tEBIB4nB#HD=}cPx|}RR0~Yx+=N))7<5A<BqI-SkyfA
z_Wl2#g@jXD<{t2R_@(#iiRxRA%8zEcy4^kUOsQpA3TJ-O<&UM_Z+BkxRsCewa?v0|
z(5*C1=Ksn0`AM^X?R>21f4{}IXU@8{AJ4DvJA7-~zm-SZJ}@nB*7Uyie(BWzCa!`R
zjdkbQSFd*6)><E=bM1<`+C%C7wU;;D-eoVzu(+$Va-YpJnN#;KhZMC2Sv%+Gi)~r{
z!m{`K{MYxtFYUk2B>!%L>b%%PDhI4W4<0Q2v4^97`B|yAmgcu}3)&|1Tt2AQJGrO1
z>c-FS7cWb2%=|lb8RyT#wTx5jXDCWN?O!xY#^Oj_%dBHZMLzR}`J`*iQ`e8!!D4TI
z-FNA+<KFyVD#e647p{?iA^SYoQLlH&nU@j=6An9@na<m#_40~N%}%j|srNk94@^mC
zxcaYX&5G9cs$1TV|6lZdk;SUrFzaaU;==ZK?TRHkJeT)IUo8Cft^QuN!ma`lBUxrH
zCKtE!ErH(brI%W}zaCO)mF#0It7l6Tvsoh1eOcq^Hqmzr;~%J>PntK`#j4;xx0IEX
z_?+e6Ii|B~v!`|MWi#2DA38Ccy=_lS(&vsH*Ay-$?8p?zFvxVMod4)v%@gbA?_?6}
zYkHVBU1ViH*j6bVBDcli(3SdqKdb(BDqKHsm900uNZ25ZYjVo1X>S;8Jmg~ebk0vJ
zEIy;nrf}oRB7xkVQ@<u1DPG+dz!kwQBF4GgQ}IznpQMG&sjcr4CigT8F&i$=XZdyR
z=ZB;GHXcg*@9*`CKd!a()0YpA7^IiHD|p}Q^Jl?^(xUP!lRlnU>(XSmG&rC}cZGyN
zd~K0z;jSu=8q;H$bFZJ~e0FY1uBpn=ifi5bd>lI7F6pp#klOO0*yH%rWJXJ~bBe|*
zQi?3E^(iF2F~4R}$Hs4A{bB+K+m1l-ooZK>p4iy8QtzhcP0OV%Ka>`#eo_)NK5}hl
z0C#1bwaC@EiY?;CJ$0+R_VIaKsz`g4?OLC2?Ns4g_~)Q)I_J_T(=`Gi0-X9Lil*Av
zEZz41<mD9)DqWUy>cOiyjuLNf#m{1Tmo6RdW<14OJV3-J{$0QAU#n%m`ZcVArPZEE
z>MqgF_)shPJNt5RT-em(Z}&vao?z-O_-EC_Z9&XGw^+{62>o~XYw-PWLGOh=#oGKa
zkJQEIJU$Rp=CUPtUayDyr;xa4XV?1X0|%SA`x>5jD?BflZ{G1MOYPkg*ZL%`^r?G9
zG-kgoob_2mqrtiC!b<T&;$|gcNf&1Q&pJ}xuxAm|DwE$GPElVD&a0}Nz&)k>()k4<
zOndaha(8e3WOC`6cj%H_&(@~jk9yX9|FSaoaa(*^NouL^{K8dcerFG;S^fO5CWqJk
zhmqCFmaK!7Q6JR4{fnHM&gN`kcH#BaX_FgxtJA-}Tdj6WtnO}<xpmp3#*W)E3k+%}
zs(jz}`HnREL))p<$1|UZYin=#b?@>VeZ>Nqih23}uKkUgJ@s?YhIN-`C_Fo?%k+BR
z>@>!#_v=>WKB~8@*ZJ!D;`RHB5!YXSTkUuCQ(+wEhQ7L8Q%iIBpT6{;)mr@G@nMJc
z3pY&t8XjEmug3Jl4#VZo-%gH>PZ#u3j&d~LsJi8T-`=mK_nv-KGAYRo^=n;KDpkAs
z%_&abAAjp_RBtv351N<z{Z8^n#|x|m|2yBhe0O#z*=8Ibw)gAb>e#C;x70FPCbUGc
zl&xLw`KCl<YG!!s${6vRXC7~JT78x+ecO@)-uF#Thu!(DlRS0W-@9{ELwonk<*C?v
zwEVL6(S$cwZiJqna5JW0O4b!#qes{LUeB2**s*Zsi||6O)u~H+H^`*U4cY0({<LAS
z-d>)aH%q=On$I_B(=8vHW;Wq{zoVDRn{jV@aXazS_OffN6ZKv4GcwDkh{%NZ|GC?_
z;G+2U5`Gz8u@a3ZX6t)-9*6AcNVm?~{WfL>%f3^l`{vuLKWch0lOgG($Nf!fqfMvX
zIoPpi@8OFld^SB$72X*B=U2?D_Iq<af0^FbKJo7revUfxZ2rBzN;#+2+aA2M!l!|I
z`7P_C-S<6}%3fVjjw=`2(zWI0{=#ZerG%vSrz#y4*Y7RMzBw!Gtbuay?U$RMTw6C^
z&$};a&e5m6CA)28XRKjaQhU?m=`jPDW7}-xr|((2QRjBNpF)Yp<!tkL){AZL*z|ro
zARVJ_$Faw){>=YH9GyN>-p$S5b8qUtIFa|-k&}H^g+1n~HP|@gY1EUOY<G6e4e}8D
z&Z_K|a(C739b4!9Gul7%+|B8BE8_QDxGU{@s4d^mSo-NFZWDpC)%E$ZfAg9S1Z~^j
zw&kH(;o+~FUn_d4C4FBjTR*ert}36hpJ2XS{+E+qcQ;<X<!j=6#c<~}jS!t!*874t
zeis()6<ncUud+CBANOLnWl<`|h3j8kJ@V$_+p;R(3t`*uWlUXZW3%boE&b?wE2jQ5
zow;Fc@>^$n56Q&p(3rjanXj*EOf8O;wZCwl{o~G#1AAYEz3PptOe(e7Y&>zdvnk)t
z)5k4#eK9Q6`4T8|e&&H(@BWmB_Z6hrtCse@uT&15SNym)YOR`n70aUKqBl*8mY!~(
zxVmGvs%J&i8LfZ3araqr3esX9dR@|UYr3<_?dQ#6vyWS<to>si6sw<lck}(;wfU~^
zKW)}LIhAv9`F^D%#kY$qvu_;VFuVCe(nJ65_zNeV&0jC=bwPWdw21VSO$;}#KagzV
z2xz(dVaqAU4)w%DGvDiPtpb{Qlfox2c)2B5FvoPt2QJUMrVp*Um0SL__)6NbN*8Wb
zmTH)rdF<td+0_|!s~XBL_pUyik<8}&+oOQj)nRdL+**}IJNHEQ|IAFW7rLHqmbUp&
zbe7cD8#4EP%Ls2USvyUA+D^N(p<Nd~iEQWHpmXKHGF8v*dp&2~blr3BcbHn^7PEu5
z9(~u1*;;-z<@LSmQor_uF=uSsyhAP5#qrR&G=ZY9?MG8isU_vVj#<4;$$Ijp&0lzK
zdS-V%Gx$DvMzinwtNTwpQ0X&~eg3)a^URZ~!kdb(7d^H5rFA;E@Fr`-HsOet8PEQ%
zOBSvUKWe5oqiO-?O{c`$6^l-C|LFg+QTE8L9Ea6?=fpN$I~<iYK_cr%*kY^IGR*0!
z1tzO+@fT0t#QHDGd7*6dF-rwwf%1~oewSKb@7ti3_I*E3%Q8m;LFtCbPQi0dQ*&ai
zsv=(c7&4}}up}O{lnr@qHL*xZCD8BPZ9WTS^=B5ZZtQ57Uj42r_o8q*d-&d;OTRx;
z(1?B0utLGOSimQ!Z?CobE~SrF-X=zuF6}M)EwNQzQ+Qpez?xMt;r^R<IWTN_GE?Ap
z-_8q}Gg6C_coqAnrG80iP?6KlTVl71aq|oIm8PqbzE*QoC2(4-&tHDUF?2_tMC`^A
zwH@mooECa*HhoKpP2nOf&hT&d<n?C6KJdMD@TkhGURj5zH@9sX*fS0Hct3WDm|_{R
zX;-e?>iY}Wa|NfZ`RfqD>6E)*;<R;L=Wc&l6DD&x;<cHj@UpL^*H)fT^ZI^LGVx+z
zX{5$u-x=35A8*Q#d=n!lJ+pOTZ+lwDa#6eZV_y{Xo8oT%Xg20rZ0&KgPs!!jFTK~_
zmR2;M<WGH*E4k;O<kW}T-n}X@keTw#Gii$5d#fMKSJ%J)u==%`{bExIHy_Qrmb=fj
zy<GBnb9(Q~^lSQm@BB`EEO5`Gw<N{;6-SEH%X^XUL=Gr%n?GF0_n`9n%Gnz`b+&Go
zK7Lqw<;U2VYc_W*r0%VauD{pm$)fUfpX5E?n5Hfzv6yGE^B*=dIDa`Y`_ChDw}-u#
zF3j#c5nOI1Q?Z)k{@(XHx*FE-HQ#=a{zPq3v{6gC_+qPrhU>1}f7NsH$)h`^zII<G
zKXiV(<>>L$S<!Di1AVmK6?ZRuzqVi6<8hgukIK2bs{}r8{w$w=#jR+^MCr{BX9V0i
zKg)lT?gzD(t9>WGNO3B>m&fB)wBz1{#K4zdME3mBnbbX1=-bMcnJ**F+%`rztjH~z
z=;k}oQkT`o@mb!X!iV#6XJ#KMa5L}wp|a$$(!HrB>ntU7C(KleDL-Y9xxM$&$pxXy
z8tW?Box3CTX9V6}vs~zmTj|T-_sctvr|zEkZ26&p-Rt*eGZn?(a6YHAETl-*XWwD<
z7ETApeH+x|#Fr*+lwjryJn>3vMoA3M>L^#gJoUnt7t&?ij~1=fZCg8EV!mySz1k&a
z%gG(xUw#K(aP|p`-<VgN#k5gWKO<;e?&OD6pGwj$DlJ`p-K)^yZr8>enp=K3EYr`v
zI@y}pSEa^a;?DeOSJz+uT2Oj>+q!+s*}Ssz&-`s$7ZyFE@cVJEJ3X#Sw-UtWI2UnR
zvu3}owf%I-`gHB~qT}zjtG&$GaZ|V~ddKUqezmjm*M2(R+Z*z1{;Mwwbw0D3?+^L&
z=dw-brRNVW{eJmB_rv|nfJ=GJ=`EVZho(rzpX$+n`^(tws$}A;9{tkxEv0U^to#?9
zD_&u}WbJ2_S2mp5;+C~V%(DM4KluMYnpxZ8_Z=<;oy6_gh2JyVXTH5=_5a4lBwMl2
zpC?V$uKF9Y{jojsy8m9yaW*|1Ie&O`<W`w(XjpXZM3mOk(%nDo&gSetAS`E;zvX}D
zW&H}7fLS@(9^V2M_CI&hQ|PLkF(F0s#cl0r46&?#bsk)lxqG($D2u}6$5Zzv1q$kZ
znl_su`SgU-JGwL?<vz80&J^qF30}F=G`dZsyJG9o+evHu8brG`{1ONgJruJ+N2&BK
z*Ik=4td};OiHMSUB^B|HLtEuy+{RO^{cB{-YB^1L`XOzV-c;#LubBC}?G`>c5Ei;3
zN406qG>61^9p2e57v0hhQ$5hdJfl8qYW}mNg)yr(2XDU>c;)iKKEAKTE9QP)5W_dM
zXvHMwZx*7*A~kCGwHCCpNyq3jZd~!Kck*myNxl;~Zj+Ku9m-i6zj3)<*V%F{|Mp|-
zC!_l&PT9R_I`cK34-*uW7gahOy$~xfUHxdJ#u+C^u~!>%e%#kxuwzlF_VZ6ma=5Mr
zeQ;bRb*ec?qbt`-hM{JD^}BVhsje&34smAN8LYnFwl!tz<@EI`abZ%6uS;*N+3-3p
zL+10kBjS;_ZeCqkwLpt&`-z=m-OB}a{^i{@Du4JtbEkdu;Wyg@&va!wR%-k2<c%u2
zcXhuO$D?h2b_v3(_s7oo8a8*q#Ob%&rg>E8&Z@69-~Q|4okQ1tRpr&qyIt!1?Ayx;
z@}>-F-#`DFoTMk{aO*Rx0{>^vLz_eM*H8Z}?6#lb#Vu#2>wP=FTK=!KZ@pt^;Tut9
z*t${IkN58Dw)^dgVmzPA6MmdGWq6^__vV`Scg@PX8KHVI%rVtmioEBi@)WJz-S_Xh
z!VRg2zQt{8br0N_=XpN#{F?NO!moZRgv|;nu-QA$D|VBM?c{l@=j|*H_`+(t?WRMw
z=dOdnbFE5WEs869b#?9S)7!JRq+FZE;T-%S(*NuMW9CiL>Ox_+H5?DK^U7~owt%nM
ziszNVio+KgMXn~7B=-i+bt{#+ni;aZ@U8yp@SPj}uMd2B|G+WP%SX>wD>j8Y%FHZr
zv{^jQ^L%OYySbj*)aPiwHc7YtyS#6c)u~q+s~69yW`AYqU8C`8!-~U;_`0ohUTsh*
z4Y-ngvTD`gMSQER*1X!V>abTkzjdJ4o?S0qe)&?xX1jRa>UmL3OQ)P)vTWgp-BzDo
zomj2BzHb}T$0(++OZM0>o=`s-EX!6ZaV0UJeWAYMisKW$gtR+CM9#03dO4x^{U!ON
z%l|vsHiWA4**__tA<kFy`8)rL)gK-nxH}<cin^bEtwF0%Rg?1yr=u$^9xOC}@qM=2
z|Hu>dyBWhfepTw<F;p>%<a%D#?HPA|LHeJ#_5b_L97J|HswwZ+ci9>1?-%)a+q}O!
z7S@@mr*P@dOt#PQSFYX2@IQYJ8%wv+!AGrkSzEN7b||%IJ6XhLC>ndFEVEVBOuM_e
zdxckFRTsDTllkm+Pg>UPol?9l%4T&{gzD~fuM&Ry8%LQg-kPzb?(6x}_PT+J&Q?ox
zN+#*fb@5aWyyUq_lt1q1<nYG_Bjb5OkJ_7Dzn^`8@2mFtnY;fiSF66wu;<k+#pR(#
zFK0cN-#4Y?^G??(_va_~HNT(!`P0p1MT=)ti%ocaWZq8CxhL7gFU3_ZJ8#O#I9b4?
zRCK0*Y1>=XzAwBtAE&)2**^0~_iENc<*Lgs(r=&Wv|Ru1@ZNQaR_&8Ngs6sGEwL?p
zSLav0zkW)Z^YXb%+9f4TT2F}l3vX|i{$sys=ljIR>z1CWua>jVXIQTBNW@iUQHQgL
zrgWFUI*x~39?puE{BLS52FtpYN?c70X*cA3S38Gq^U7JiS94aS1)r2YBEEWF(AxGz
z>PqXvPiCbBo6h#CQrOo~6{oHCU-8G{{#BRULtidl)s-E3ciwGrKkfi4{S~#MOIIwi
z4&~X^)qMTj&ef@%k5=;Swfgev!fNNK+%6}!91#Bc!qjy=Sj^zf)HnaRw&@)@=6YiX
zhxGJC%O_mql$<-^VrAlLVKb}5#lh`<vR0)pt}G00uM1?4v#naVJ-j_mwyN~S)rF^4
zFx$R%ux+dUeNx{sd$(tK@8w&j{trE`emA#O>RPkQXZ^02^+8Lc&AL~}|1w_TIKBC6
z{8Z6HJHBaM%adJ^_?Gvp<@f1doit~4UcS7#`O@+m^JjU=mGN7}eW|`w{8sjh{-$|V
z8#B&k%g^j{SR1-=x%S4gKMPjw<a^4e)BDE6dAZ~?yXIT*SA6DNyC?kY`M=$1FMjc`
zx=bh$owROK^IEG{s|!j+*R4BrHFU==uf%1iCffYw_-E|$bN^k2H+!bF+b{5xljL3T
z?a_i=>S1>ePj~gdyIm|#o&Up4?p%AplY5`U?GKM_^u6>rE^_(;>s2pJyS)`QnH&-i
zX2_ni;l#s-`^+cEvpDPDyr=({_3YYb=f!^3uVZw-^s-~$#b~F@Jsua-+YS`-Gbb+e
zz8}c<_b_k7ymz&yw>EvfKZ`rZ`B=BFv(9gylZQ^F{yO`$_@p7z_SEJV0^egB_o_?p
zKejRT#Mb3pN^7o{zhRF#)fsLPn6v!wnlAz0rX=n4(2g(3WOnisxU`Pd^xQAq|L^l6
z>aR82EN|YyGE0Yd^_7CT_MgvuJF-eoUnr;OuYKu^tBGOlk+RR0@NKF(6X_4)2e!M)
zij_t*S!ul5n9&7hUr7vX50wpD&bO*+&8(zQ5W6(uYGP>nRKATH(>1qk{&INc3*~u%
z=a)QRmHtjvq}H)Nf@SH|Q<oB+x{5iz3@X|_M>k}{gYGrUtx7;i4V=E1l$JT(eDD^Q
z{thqTYi{VsESQ*C@K9-1smrD>2H!v>RU6-t6>^K0NnA}_a@hXh>vta%ttKv2KXWPg
zd1TF*Z#8H1%$TR&{Sv?3cJ<*ke8oZRt28;|y4_MPa%dk~2Ps2YKxODf5pa15Dp*%r
zt?`o$`5Iw*bQP>nUB#y@%e8*)hg+PeMCqQgL)q#OVp^EK^~0^gal8Aj3yHs3$u}30
zZBn%u*T){uR^KztWX9YNQBHAXhq7P(4>g>~*HpzbD=7e0*aoyOWHwxNe1#@7I|s_T
zS^e?f*j~Tz;F`V9CBMA+U&?*y*PAl?bjkbb+xH(__v_=kcPc9NPga6_)%D@&I$6t=
z-$Edfb0v}Ul|yRgMA=0t?HV@~U-`+YTDfor1v*XPGO-kmEnW5dY_Mz?ERN*&o)Fuv
zGWGt3^b5wXW`2qJdS-9)BK;Q=Pgh@wwSBctuGGAL*KGILql^7s-O7INcCse0C~-=R
zgsQMx?4(6{zJcncy}PpA@7`YYt}yWW#ucw4r$lLZ2}|u3kC>um;w37jy2LfpDfGpe
zd9Msg6x_b9u&P|)v{+<w;H(Q8z8O>E?7h~@uS`55J}W7x-BVVrbjFp$MTh@ivHY`{
zZO+0~#v!F`tIh>nm0V@z9Gbqu>(9E_)2KqPctT^tvbKwa&P7#Ss;#B$>kQNQy77NA
zTW~@=%WtI{;s^I}d~@BP-&8C3#=E^%@J%<WxGYQ(Dn313|DrBPmpGqh)89CD9@d*7
z&XM|0*WTjLPZiu-!w@#{+?EGV7;8^0@{in`&Zx~D5qV(KnJrRhGfkFXjJdQ~J5@Dw
z-K&hHf0K)AZyfP6Jt3vpee&r&KiOw%_;TiN{$`my|MQpmb@5-T_ivxNiZ3hPINl*>
zvq#pnLd|ZK^oc2|e2bWU_h!8c4PHO1>gtu%%i>HoM^&*)RbD*N?G~{`+xueh+c~!^
zua`UW#>`!&=6+<I(DwIRtrz-vfBG9W|ETCDK2N*3dgt0EHNLIsJ0!PTf#0@r%KdD+
zA9IdYZ>qRj@zdVXeT`1VfuIM>W!kG3$bIL^)6RXrB+UKa)S1h<cv`n`UvqR}yWx9S
z%kbF42gzDrS1@(W_S>h<-)EGrsd_B4y(IRprAW@hkEc2n3M==&SvmLoyKN6+zn>`B
zJ@1^YUTMWut6Fix^0;#sS4}^@&g8o9@--*KRBxXC-W9Sv$JYMW*W(Q`-@20)s$9IG
z#PX-y*>O$Y-%DH9arA%Zb)I9_E$}>nqh$XA2bRO@+;4Mi=ijz<)2DUUdpFLxcUZui
z)o;%E+83HaoyrU$WiAuHyjdaksWs<>&<>XG!G|;$_kNa8y`hmj%OWbjnB7Ii{$^h{
zzq5zGRrbRE2_M*RRq2FUFFz-iBrRoN<&fvt+4Qa8K8x-7Z)+Gu``=dNU6Hu#qntA<
zKKsV^T?*!3?!FfKsakizVV}rUZ|7vEZSr!dmkw{c;k<C~{p&l8W5m9hp1QZ<f*kW)
z8}=^g@>vDNZ2AimZ?ApJ68`P@zThC|ZBH96hm`EyC>@o5TKe$!Y1_jJ^%TyT%zaQ2
zW$??~!||c4(pvxLAHJLq|L@K_YsHmig^5kxWsNDf|CBV{epPdxY3jD(t1LZvWv{0^
zH`&vu?!#Wpd+T??{n?JkjvF7)Tp@jO_DyA}Mb0gogLkiyzIdZbKh^r|<-CVtht4HD
zGII<6zWhP)jc+FwuVy<YU-Pu+-CdU-DUq_=D*Lj{*M#dnu;mc-4V^gQ=+Vd9G{e5-
zd`wz8mvv1u&&{%9^H{@Ybwq8~&2xIQ?Xcv$<$>2^*Wb7nbkXTTwX(0F&*xt0$9dT^
zf8IS`D1OjAF#ctmujEHltF8Og53QNgl{ndc;i1o7*ZJCdFTL5QCBSoh0(<>6r=$yw
z21j#uoXT5xb*7}3^31n9@8+2Qonp>D@7n)kCy&eA>EZpkCHLX>2VF0wn|(U>Xwv-@
zaqDO6xToZ`|4_^RZh150k94}}<8XGzNv(dnubv1vt`S}OJlgba-;`%voW}yr%9<<X
z8C*NL^j_=P(27omHC<)i-^&u%X8Flpe))Z#&^G1P4-=1yiRpRNo-@c<zEizO$6aGN
zzx~$p--Mp(OG{SCNlE_Q+VgAI8fmLInZg6lzb{X|AAIZK`z=SC6`hyZbL=-fw7une
zZ`?O0E$f)2In~o`TD+Re_Snmr6y=qR-*_hW-J+Vguc&eM#y4T5QbPVBa~<m6c6~4|
zkWh1X@=7za+&C+=$?IkW--(<2v#X2cGs3SvZB?7P*gBBA`gEawtAE@^G5a=Cz49A^
zuMc#eUBcXJ&QjoBc_m+>uAcR3;G2mvuk%jd^tfcc7kA;EGmqrFBML=z`;)Zu^e;-8
zU$%U9!mr!$!GDK2TdMD!b*w+D{JO+&OYy69)i?RKPnld%mX&h7y+H9#Wl4>&e#z_&
zF4KN7`x*)!<9ff$@ka4+A^zzdtNU2^9v&;4%*8T$)2d|$zjf`=?aq7F>#x<Dxm=}s
z((`<wb#uhU%eH^Ez44~Zx~eI1ZL-Mrj+gC9k#dZzqLN=`?RI~}VSVWQFVU#oJLZ*W
zYfSo;&$8vFuh5n|`h1$70`AL5)$y`sya>FND{VYieM$#=)4@BdKK{5Uy}jJ5ODlH%
zi&y8bq)1<jjcND2QyA+$@qPK*JIbp~qn_qHyC(ZIzdG5vI`Zk3i!2u}Dn6Q}r}XNK
z<z4IdU+kwZu*|*Q6g1ab*geZVXCZU_q|jsQjD1_fAFDDt+r9f!eR0|Cngg>UJ2~s`
zYqq@dlQLm4dhNSPuaPzJf)BS{$;#CF(9Q2B-JammQm()MeVN}sr|U5;rE2dQ^PenB
zXSX;o#aREpef^Gq*HTx=EG)RL)Ec}oYl`@){@m(&6;=FBhO8HgnvP_iV|!h(kfm$0
zoL1@Il{!4CIpxbw8{0p#I?!tLaZ#eVW3Kb=#Ww_}c^4*MU-4Yq(YfT{v?SSbmEVW-
zt+zb>EX~-MedxsO(rf*u6P7r?Yjl!1dH9}z|C~L$Wto!e#TEsIam44dv>yKO*vf31
z#N>O6mv6rL?5b_YtH}IgA4L0dO;%Z3=sfTJ!F<-?<8N=dSyST{)!dxWXdU%js<41p
z<QltSab)@DIV`icFz?a)F=zU#WtUeo%m{EPIqP+kLn@2)?g5#1dTZ}KsMs{!aeJ4`
zWd6?#@n!~cs~%od@$X17DV<qTb9d7gVb@*!+=3qZE>9wzAFR7NH^lYgF^fJk^9|g_
z!a@eKs*Lx|@OY*6^py71_PvrDI`SUft<p7nfAnd$^vu#}c3->AKYQ#+n_MuZj`PjN
zq`FH7-0p;a`&IX)jw9M}L5BP7_bcU;E<3L5+jXonsl4`_)-MBJ-9@wKOt9-ue!>?0
zs?pZ0sdM(jw2j{LA|{{K?=<9EcxPtO_O3p0gR}_=D;{sW`)Jzk*|P&n&3&IXSssr4
z_oU~hf%ta@&F0<vG~?%WynC5p$L79U|8w%Ej%xi$EJ~VxD+Jf=vy)=j_|D>P#7iUA
zV`ZB^UE&R&`{#*q`#t|gWBwg`3QXR8UHm1a%=T@S$w#FJbGrXaxSZpAI%x^F^@+bC
zQGPOeUR>#rX|ynC>CO8ZVXrq~<89r}6-E=U@a}!j=#`f~zh=MYyxm%2YY&^YB(AOq
zOLrH}E#0*J+mDM!?(-_k-~E!)^UH1C-UAbNH1auao*w>eT|-HIvXW!G?cqIqZ>Qhw
z)7N+w`G`k+S>5iEnal4lndj?uf2$wQg(-3F%xl)GA56csag(6=<h}1=gxSvhbCCXD
z$=q?|glpl`p9as8*X+r5y|dx-$>!HmC*lmJ`+j+IZ^~!Wn<2{LNt^b*X<PcBX3ynK
zFO_$>>p1vb|MUNRLi&?u4*v4CdW-5Wey(}ozVY7o4$g;35sOR2XJn_Rn&nG>`Z;lX
zQnth^wrML1C#|)-!}L2nVP}rRqaC){@6I(Q`(;|j{@6BEZ<Fd$=PUCk?&+TXt#{6T
zRcUea!=)V)y6<>tos@mI_1Zd~%c?yu<KEWU=A2!A`_O#H`+Cb2%OuavwOH+YYk$Iw
zXlHwucJ80g64<`UM|fDTw}=t^K3#b+i+lYv;X6gQen05(jORYNzva?y%QdglX5N)r
z7_;w#%7@8;25xs%x0|@0p0GP#<q!Xg9Ww<>YyGs|teTwMlrR6V&ttw=#4CgT<a-n2
zZf`j!bmvcvVAuQY@6MMUu{c-uxo7=aukX>}o>Pi$a-@cp3*EjSUF~q&v5q&${it#N
zkMsi#=S5E5FI#oTd&W!#<CW68%l|z85ajfbasSu$3ID^GZCuq2Wnb<-pXqQrZ%6$T
zt6!hlj;XGz_%WZq`DNv$SuL+`-(#&?uikq2{PX1P6P3CD$v&vP!%%x*wd(VSOEY^f
z`#N6^mhr8d&hTfV{q()LlAD(+I^TRdJH)pxy==!@UVi>76F%GaXV2Nj%6xk3;pfk}
z4Jsc#Gb)-<<8b}Pi}R~*pEipWPuXB+J-gufoD<yp`tB;~zH2$b_+*X1#y!`cCu9hF
zzVSTwUaVQ_yW7Rhvy=NRtp6RDyZ8+6-_GrO*m~FGiaYz+RiwSx$&r0xLI>yfSNB3U
zJUN?iHS-_yMOVJ``4WHP%?vIooD&qOyCWbPzy1EMs%d9<zy8|R$RN%q>lI&OZJ$$V
zt|PJgBk!hJJ5$X*E&VZX&EmS>oOeDL&7M*bJH6y#=*RraF>~JYx&3O2{=3+ESAOr^
zUAqq!y#L!f!{d&@o30((WB=u}zF5v*zHsTAD}R?6zc9B;e#7@O>+^}ee>ql$S8krT
zdM~V}YWZ>YoYXE+$?z#<8}#PyuBo}h!2D>7@SV$Q4C@bXopS%Aggu|niRnd3VKw@}
z{p)v$oN@l0_-D?7jrLvQ?0>4Yw@KHXk1PmynD~V2Z_`%$pBj78<OOU_%4b+qPdPW!
zQF`jE^}D{MX&BkQKDuenkuCCFKTmIZtQzF1Z|&mdeBX3SUU1p`(l>YeB#+hBD!5zj
zK5+jW|M5m+bF0gbRdWqoQXQ(g+)V5(H+{Oi?zhB}jnZGEoExiuul}(1#nVetWkqsz
z9g|m_w*C7cC0*`@?Qf%7UB6F<ONxh8#XQqkSi0%bThp?BU7P)U#n<bE3sxERo}b#>
z@%4w-rLA{AerlN6;;wYsZerJ(t(Nucf_`a7xxQ$Xe6#)4)};UYyUaW9Zn(6S>Gd{e
zE8|V`mv8WVHp3(TKr;JFro+2c-O}AWS-mc*W>#Efp4yjMyd~nycC}~o>c1Tgee?9}
zvG-yrEelV!v0Qt(C&o2yooAg%&xZpi{rMU;=dC;SZ1<I1DT&CkcWl?ilV)$$)U(<c
zb<O**=nIRvnvY6TO87meEu6=ocIx$4C+?<)QQ}VJb505A@08wt{Itd;1%Yen`!(-s
z<u5#CE?DvV>bt#$2FI#3Z+IQQ|5Zp<zgost*E;dXMPskGQa^v##Oc`Ce9wEaZs)2q
zjSHL|>z_F%?wcnQ^RP-!u8wEAJF9f2z@cv+7MyjcnYKt?dV#2u*@^9+oS!@t`YFWp
zgE{y53c=LQ+@;yk>aP=I%uev<%{<GT98`Q`pZtau*PjcTO9qA-Y&PdB`+j@plj&(s
z+|P6FKC7!`by3H(I3#b%op<XsWwZ7+voft;TFrWl`)yiP;<69b>mL~2+4X(TtvHUu
zx2*CQ#U1|26}+snocUy(w5*Ew?yk5>t$#Ob?)l%CdRzDA<+RhUN@eerD^5Kk9QG%A
z^{bPA3--M+dsD1)ctW<?LuuW6_D1@9yw}Zfc|5N|p!iq+;g=@;B?pCeE3aVOcqN@*
zxi<6Ty#ELH{mR^XtzY`VwQrYIj^|4kY@flz`>IGt+5U|td*ScBt2h@=4oWExo1r?X
zc<r?dt8aI(o>G=cZM(|P_*5ZZLo~?pb(O1B`onXvtIo)^bbaC2`8V^}+#0=>Z9M%|
zDUa#|&#GUz>tW>F+wB}Q;rW4O;?q4|PMxvun(WnM{BJop%~=nMmCQJ{E|{MyLd9Bt
zwUourc@y)v(jLw`t|P<F>gvBjs8sgQcJaGkZ>-@iwBy^nalTD-73Ue=do!!G&TYGt
z_ob#YHjL5eiTAGE&xOzCC0aG-J}L^Da?oj~iJ0EA+4iaWN(E1iGdDfFDB*gqO^&bm
zrnHtsO4z(Ly{yysS350w)YZlry(IUQ(60@`N=&EHO-t+)7DSkY@HjO-Wbc_2z_-Zz
zy+;1LrO_?FGoz*+5L#0hW$~_7D02F*SwV`tXQa2t_&BUy;WQ;?cgGx-)}>1mC5~qG
z$M`Xww_3Bh<!$^4=XVMxidp}(on-i9*^yw&{vuXCc}J<O=9llbqN20bYFybqN&Tqz
z?~voY_Ut?Ne|a1C>fAho2U?*MR8wyjh40uaSR-rFwqpMpk1Lj&j`QB{U;BNB|Cxe)
zo8s<V+-X^U^@-I^2JxS<yLU~_&^J?hv+c%*lw!xo!@qM)d48o|?p+|EcV3S5W4iNK
zm2>OEpSJC6F1~-aTK^rdTZcL8$AGspHtpE0820UNW8K=--`$iJ|KsN>?<|+O%Ad2o
z{$4{(@Or+`wfY+y-G1J3-21=zS@z4Yro;VtCu|blxBhCo!<+Q`qP>1FOAMcG_;Tr`
zRdUZ>q+UA{`D(7ER5}Zrkj9<OishS~V&{iXKf(W8;<${W@P5h7|I>Ln`1KxXvb$>r
z{NyeBbFOAWLHgyQdnZe61wY&K%#S<pO2Ol~_x&dMxQ#yB=Ex@3&OFjv+th#h-UrU#
z8^0&?O=iqav3p^;^2E{)jkjOC4*j*)EH&oVDUN=*-s#VmPJUjY$s4zyJ0onK;h~*r
z)57Q5&d&JGtfY7M{av$(0%~#d+a1=NULUkjv(=()`#kf^Z94lr-X1rZ?03^pI__nY
z*Bwi5fi?Au1CH1gEPbzaJ&;A?#4{NIquaOc=dq-HKUKbdyZKw!9%j?;vp?KgJu}|?
z%Wvz{{@5*TEr(-VY^DlU<nLXskblQnMDk;uSI&jz{E(KvHRUTGH+?I5+A&T3*<59Q
z7xlk!JMYNw8T?pwuSIe1uXKSX)^PcE@|Py<uB<pG)?(ACtXG?^wY}|v@{`NU7ThR2
J{kL!GUjV2A$iM&q

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.ai b/third_party/icu/data/icu_conversion_data.c.gz.ai
new file mode 100644
index 0000000000000000000000000000000000000000..981b869561a615f21639482929b89d2b2e5ca360
GIT binary patch
literal 177696
zcmeBU6%oEX<<8S}3mJEKX9c*KNs2r!7Fj60sPeq`zGk+#Kjz|hJJ#3fKi=`YHnoGX
zV`0A6|Ai`oo3|WdJtcC2DQ}rT+rAQ}%X>xqJW{&vo$#3SWBDD^u*ylffv@e%H3ckq
z`q?cHnk_$5$k*k;wSS#-#EX;5iV_#uDSJiu71zy?I&$;TjP29E{g!{lw(c?K-Kw{p
z3y*uhQ)zi$_(iLFL9<F{AKzV;Q_FSS1vv}v$~@t4RxWj#CEIjPQn_08eD&(d-j_Zs
zm^I<c%logJHg1*itt%~$Qa}50ioEXpWI?+LO4lY{Y!lJkyFJ<Igmm*+2G41e7M;<K
z`yTkde5U!!#Pyrg{-++~mGRm5>BpTfc5P=?{Od5BcrShaiazmxo%d3Xo=7YZk#T!=
z^lh;BBazp;!WT?G_hd)*<7qCl)>|sKg=Z~dP1*Rczlfo(Hi$`=>+>FgglaqGb&Y;q
zJH4H2@1!stwbzvR;>N@3zRIGfTj=8sj<*NRl`gEWJZu%X(BS8Tf=ugso4?n6uI`%L
zHgV>I<Kj=`mM;3kWbbL*o$ayS{Gq7Q>m?TTsqbB+|MJ#4oZs_f)5RI#Hw62oxBlKG
ztN3J{y>4BvRzg|$x`!@DPWYE>{<$J+@rLwWo&qQ5rD|H<Q)%mGy{^LDyZ*4y!*`b~
z6XSpATF1xCxApJ4>;LN0o|1wcEq6D)v5;Mp`nm4L<m@k^?(+MqCLY=NPFU1WMRj7Z
z=HldI&zGyW&UX%&t`l@=Z)7|FT*>OW?^I{|9GyGw%xV2Ao%0WWWmz?0{_+gLH?E)O
z`Xxl^3H^On^;vmB-|WYX=7k)S*8S8lcMVdu+qE=bdY_JR`c=KZBI*x>V>54^&}Zkq
zF3DUVb#8Tqedf7<4b4tgy64vhTv(kGloNleLb%hz-ZvrX`z~qr4qKbU86Gj`pG1ln
zI2YDT4$|Kr;9SRWT!D9&d|Z-*sCw%GqfZ*V+AII~YX*hAU#Y)qANOmom}5uf7l-&S
zD_y8rcGr2mhhj;yN=0(~imV$}Cx5U{Q1st_(@W^k-VgpV2C;uUb)I*enfAH#MbLNN
z_6fb;vja>HxODd3J6}2Bxs6xflszw2?B?97CA+zDqL%&5)w^vU`HDv!_ET)hQk+!#
zQ6%g^d$r|B-?Z;1nwB-3Xb8QzPHk$M<C%(rm9zft`Tp^kZSs}}v3hs&8`{zzTE-r}
z-F8+-Vqw`OSIHYwuDr?>@?&|^d_nf`g>w^Ec*LY^OIcRzVt?(1$dcdTp6_ReNTjPo
z-O1B>5pGs|y&%VxW#Wfhu}^9%-`x><S(x9n;GwIY7tb|!4V~)qpVk(}&Obgu=+XQ=
zZ<h30ec#i$(WYAKS-;GO`t^+I|AXC+<(976zwNcAr6%)7Bei8Kq?_*uys6eb#d!F@
z#%I3@^erRai=Ch7rXiKpGB3UD#Mih48Qb$=E8H|=jAshHcyhp~_GZMDlKOpq-ib@H
zil6qr`x*GbH1lcSrCIfTJ;I)E4BlN{w9xe3wi6Zeo^!@979@RL>GkRUnFXmT8dkci
ztWI9+bx_&FoO91(b$#Cw@84JS`Y&Gy&k=t3%%3?bKYS05xavu@=fOLC>#iHT*__?%
zx!KrnQpxOd3zIxj#eb;>8F@Ya)ph9MvBe9vMB06H{&?tY;MN~$JW3pI{X<SSq%7gF
zK7W30vFX8;{r~orzK`Kj)GcD$WuUkqe8+__7Ml~P)806}<dM4TJ8k0q(^k<{sl9or
zpFdqYW$S)?!fe6WVG|V=pHX2@{u+Bfu7p1$ao=id>pI7(b)^$dxlZ3OqwUhHh9iCt
zzC}tk*vfs>7s-9S`FX(YOJ5cq+r2E%;9;Cq%FW$H@sC}K)XRRPFQ~l9@W-a$ek#|S
zKP%La%sn+bk!hK*&>h!@G2c2SpPMuHSHhKU#~(X%cF%r2<D8=SGx>u&hPi>a8l?G6
z?kM(oemq=tXye+Cb84m)2c&x2ne_e3`Koy1@u90C*O!D^9q#<p_VkI;mhflt&asz_
z-kg;AmVB;J+wF;ny8PrV`>KV`_I)k+G^1~c@IzLnl=fv`qOHG1GXJ^3w|(u4ZPJW|
zpXDo}l-bic=a~z?Q(alM_n~}%==+NF{+03*x*yv|zrAL)DI;bD>+-ALtiC^$KO1y8
zJNL$YZkuOGagU$*ALt1ec+{%3b-M3987cKw{C96nEbZPRWWUni^WLdsrPa;iN8U4+
zUT3I1YP%~!`S42rFHeqKj`N<f;;Y?5{tpd0_4{=GeF$DJ{o{A-w_9t)&q!U$s}5Wo
zzA?_?%A5KlHpRt~2b5HdFU2~A=5YxbJ5Q23nk(IKvih@3{Je0^Q2)I~*QNcny3U>1
zdd`61fy>3st9CX|Qq@0S`(TyO^x5<NXOxyJA7oy6w0|8-rh%}{$M_YF0kU5*nlx`0
zMJ`Qxk@L_^ntA4qPK^aiZn->vClcYJ|J3<>aD++G=fL+LE+<@=&Rei^=ZQ4QZ|5$i
z@4j<XwmwSnle(2qD09K)p08@P%0~mk`!*H6?YQ3^sZq_^ATfV)u*S5UE&A5&EwSfQ
z@9uS3_LqAdhmyd%J(|qc@&{$=6|-H9u5CIn`^n2|7uKg&o$YmOee>5*`t`e{z?kcE
zkKOh2dy=+f<57VSRVV+OiF<=ZR_<ST=&MpW&$0&^|E}NN!v1V_-rpG^Mb#e_Zrg3x
zXDPU8cFpFxqMqZMOhx?yFMRi_lvMXQY+yOP|L9`bof}UlKh5zudUdJwAK&lc?W;nj
zdcCiG<hksAt?H*;!VMn{)&<?3BNY1lxn~sL@6Y8kRyVD^Yd2?h_NTn>Ze3Sm-)q{G
zSFV2dFL3Q;OZW30J6e5nLocn?eJ*}dR*>geF6-wFn`Oj~9j#)!Zt0ViBvG+(QKp8K
zmFvwI>nJ6S{T6cs#4c>vEuLD#{h{RLIl0OVmvB#6#^=-6^#U#}&$H>Vu{*(4^kiRg
z{f)O-2b1R;Y`t+--CR!TuJOI%$YXJZf}0#${LEb!ovwIbyz~@fqT9EHH;(lb6?Lo5
zXwZJstJmH0P}X;9qvxsz-hG`FrlEW6IBu~VuWwp+TdC?!leFfc6XH!<TGn-}xM{r8
zH$`}pb>XMkVp8s6>d)Tp7P&R|i2J^Kyd1aYOy+0K@?!l^u6e3>GHcLI@!c;fPfb6+
zUVcf_%nI*s=O^B|7j*vm<+n+P|8Lm&tE~COyx`mbqgWQjl5Lg}cIS)O9tQ2sd7N6H
z9q%vJb-MLolSR8~U!3|v7HOO72|wH9zbx!uo7n%Z<abWQ3%C3T!;AB6HP}Mc1>Iv+
zXROPAuzmN|v$lNV|Bqk$yC$XY!d|&_n+K8K?=Afucl~$v@65atJa_-r)aD7!nG}|8
zP`GLDBKf@=w7BYQ-FB_H%=z+p|9(%#&7b$~5WCFri|>>0otiecYm#3w^b>lfM=r0l
z_0?QgTCEe^7AN!lmMd%8szsObQv8-ZoWGf0F|ESfa=Ya8Ng2;Ly<H#coh_Yn#i}J-
zMt4Q=(l5=z>ML{7_-Fr+e0*?b^|y5zr@v18F7Ycj{PLnpo2PLu<LeW-Tr6*Bn8bNr
z`tjwmYb(wNsvfF+)Kt84hQ*)GhWe|l2@`iahztHPGFz@Ud(D}TR&LU*pYA-La`jHK
zt+}1A<`czyrOm$g{<$C2KV{aGVl;nyq4NRv_43+E{_Ju;1kW@tSf21DS!;Lx;cfq$
zU9V;}zY%?TBkvi5;hc*FZr}D6Jof#y)+%UnOva}}pCyj|Xw=)<_0saGSj3(!%&SkV
z{P0zane)Ki=b?%F?RiXYN<H{x$ayg@zvWr^dfRfP+0mW?-^9PIGD+Xq9u&VpQ{%B>
zaN&=2uMga2o52(Q@#f7LlT!D5IDg*e<(st^&gMQ>E|ii|KcRi_hONDhLR3@CFLx2G
z1IO(9Lkc!uuY8rHWiHAWcIEH)Rdr{B@}|Ds!JD&lvFr3{2j8tMdzUKqGdTE}J8wWw
zt%G8L#$Sco*4?+>J+1yaq4v4E#v%5-Pd95$Rbg<gyEobL_SMZ7Y{RU2v_(YjomqLw
zXUSgWXWtgFrddX=U$^dRUFEx^H)~pIoL+Vw+nCN9(H^VR@G-`&epWww_)D$s;|moP
z?*&ioh^xOTFOquO>Cis$Eh}Yfq9jCJtDjAOddv2J=gh<M>w_ALYePBrW;ZjLKbh?I
z)=srehd<_m=G>CW0o|7#g=-z!63iZ-Jmt0c-A&UUZhM%oqf&pO?Rw4HYU{5~tB&yt
z)y)Y@d-}^tBjUvK$<O%yG?uni2(|96XtmAzbXMfE|Fxbqn_}AE960T9(D750=(0Oq
zt5_|$;-}7Z=C9FsxFyv-qWznn>ij%mnFp8bUAHZ(o7uNYqDXklm$Lo!_ZWVD(_=|L
zxU|LLs{bcP-NVZA1$)jM*v&Kd(9`~R$8Kyg-SOc>RnF>Ld-4x`-S@8QwCyK$LHUpQ
zdh=eL<Gk{(QTLFY*<B-h&1b*2F?PPGF1l{;?m_9N`X}ALuP9Eg5Rd0hx|>-p{cA)2
zl5kC7_FdiYdOmgYM^AZP@L%=e#r)7IP3O)`Q&d0yZ!7m^<%v825%ctoCnqhinq_%u
zlliQusI}r769Y6|-Q#AYe^?hSQduyud-l<_&i0MjH<oiAO|AGRtMFcZ-gM#n?(&sW
zKVOVzjnlexxRrly#wA}fMy7f1c(!TURp;K)HvMJZpLFl{^?2vI?+?rR_x7B$v?*Dy
zclQ2l;Z<wJHJu)=eRxy&)&H9X_p>f+uAR%cGmcwN{MoV3f0Lpj?=f`f|L>hBEBR+p
z-JIVM&8tkFS8$d1_hgs;HfLG+$MFmMmwLTK{oH!FR}aO%ZCGTzHl#Y8VZHRi!|}_X
zJD>R}WNja}a$%)@jU#7;{*SLLx}o20E7UE>wORSUc8|y-{sgV_uQLp6T;v)O{Td$m
zwM#~RUZQg3ThH%@Aw1dpbQfEOy!|Te{il2P_a#b)9g2A?j@-5XD*8C!YXFbE{EAv8
zNg=MXX3gEkitm@)Sdn>LZ)Z?>e|J?`l{4e?lFHo|tOV-r8BaU^%$sSi+ET&rz?8%d
zS;h)3xy2#RL;~_pNe8xX`k{Ow*6cug&`$a3(o$dE#{SyVnD~vm-)>#!?Yt)^?-_`2
z)NlCnb<NpdH&1c=pR;#&%L0o<arf7MaCrUm-!5$@mjlYv|F$2@U3~oE%w;V#;h~$^
zeks2@R~Zs?<LvZX2VUJYaCMw@VZV9l*$>IHWSGk=E55%pdcu5x|MU*4o|Qg@r<pI5
z=4+e2f1t&BU1r7P1tBq|TIY3RmkY_L3TkohjjSqN?X1VOdAcm?o79S>51y}Rxb*Jl
z^IhRy6CE>xP9<#kw#w>(=*ADr3oTvdp858_@EV_sLQHpU#p16Whb<n+hO_3VX^Kxe
z;uI5Mb}zGL`NwmmF-cWj{g?M#T<HDi_0Gg1OW(HzfitT54j<)A&#Axo;}P4>8_$-y
zxW^oI=TKn$c$s@ed3tb0<-6cA_nTpX3m<)wNKa?1_Y|v}`C`(J%~}g@ie6ZIu+;y;
zmU#je`sZU=?#l!|Z9DtL&EEJ7x2;Kl!|~S(pGVZ~R0zoD`|;x7tY&XvgXEt(+f(PI
ztf|{GrOD%M%#DR_l8^mgICtT?u)8y?H{88_+;49o{~X>7Wz(xS+)h8)kYnAv=b|^)
zHCxj=Z;S8!-ZoKZh4%bMzuvH}(x0eOoU`t`?UaRUOs_Ys*!+0+4&&;b3EG?QdK&F(
z<&?4r{ld^a@glS7huJ2g`?pUGIBK-{Ozkh0KUeOlbl&E5Z#x`$wL8Z%t10x+x}54V
z@xt`u{2f;Una_Sv`?1&S^X_kdSbv;;;86AABH!INqCxjvK2PLYug-Duwn>^Y+w!96
ze#t-N6mOS5ebN$mu>8IG$9Wy*Mpq)|JUNu}jbE8VuhUk&z*Bjxcf-f|PMdAwHlJwF
zeN@srIh>=j_d~t7cjnCY*AwFTqGZ-NWvnv1pKs?iEiBPVCPrqN3VZaTd5UaaUhjMF
zl<aQ&tmx&LSRoqIr?Mo$p#0(AV<F#W`Bq$WH=W^@X0G(MEbmRxhJwu9D}*#V_^LK<
zz4vZeW^aF7WM00L^ET1zH`BkraJ=*Nt8a5nl>LEi=cNRG)lTL=^5;SxuTsP`P5wo-
z@u%aiA9&`!C@rpR`qjNm7x&bks}c;-T&8Eqw|~dqyq3*UjI6JvFEmu|ebF&#>ylcI
z8xp&gm!FbooOovU=6{T*Pae7ve?Xx<`5Sv@QvG^HSF3e<+GFivH%*=J{8`+qk2wbd
zGrc|U3Cqk?{<Yih$6J}5Ou5x>%G%VfxO;Hs2Xns{b6(UnFZ1$Ad9`^a%a`8^Si0`i
z%ZiP$8VVmA_kQJS+jd`i^U1fnZccinFA)=YS0;4Rn{@|MrfpD|I5D&M^QV&P$xh!3
zUHIlUZ@X*6@;tTjUfaUA2~nkQW8JKbt``0XQ0G4WYI<g>O+=N>A%V-sw-&1X$hLYb
zGF5z?(Z-M^1wyuYnir={&XcY_Q1g)c{Pvd{{S+R}38`EolXPso*S0*~rYRR#pMPm&
zzUyuhn>>}{%GTA~TlTh}eDcoT?BpCjPCmVk*IS}r>pI0NpRkJfxK7aLRnol3K$kP;
z7jn)jn8^JyRmT0+Hs07Zk6u5^PC4{9=*E_qTPNf6I%K6H-Q;f1lbfu)ZdScz%nVNk
z%Sze#r`EQ2=Cu_qI$)u6_N2nyPcnsB8WlULwQ3#vua$P0<*4=UvhlAwXOSx}E`1<;
z&xH4nr>@lds^+b+{kC}IlY)sqSH5L%eWloSufU-6`AqTcbxm6OO?uNy-39n^mhQXm
zSA2eN(G?!&c{}EZe^hVsn!0(-oJ;wWngbp^cyz5t_w57&Q{CUYEqK3Q6#DJ@BlX6w
zrbUO|*WAtNcu?*$<-if~KWxfr)5Ds!EQ@uDJ-)a6jh9vDjTdub{kKdq{JH1B)A=0A
ziREesrZjVwAKmzw`KrSkmG_5YP5GARcuD`6HNn$r%I{{~hcA3OPPBhbvUsyHeCd=!
zcGD#GF0eMQvH$5ham8snbMgJU?3biJEmh>TeDuJoa=&lp5>~5!&PF@m>pXZTf5kF(
zbHcL;Cmdg1ZM}P8aYOQ5v5-@`lihBcgo~CYZkKrUV7bi)_x|g5r<oS+T5~CQY2xAj
z(??!E^LUb45%OU6_eSmSxy(&MaT?5mP5M4_a}p;k{~LO3pFsnE!}2X#%2;<C%DURP
zpJmVC<Jr61*U#5x{1CK5sOnKscX#k&qa?rnD=T;XE$v&eN^{HGMc<E4`&&||p2+iq
zual*?EZi^L{>!d??lEi52OoR3yWT^iTsig2lAIOOr<{FzEp*$LBYvkpPpN#MYU9oy
z_>Fb{Ldh++Pj5fqZLvFWMe>XvS6<6qy~fU&eC*G(@7Etq-<xpe;(bA$_S5GD7o@gm
z9-NidwqnMOH|N|qw=G{4<1C^4OV67_zw^sp?PX4{SeVvyuPb5K?Bm!XovL!?xzswH
zbAP_OuX8VA_>v%LzvSnuwskzID!cxzj%_Z?-T42sXJnP}^=c2BZEVZ0zHd(N^!Q@i
zd)PjY{r3t+t`*Wjlk^U)`j9+x@#6rS|A*#i{Jtc3iN8E^JO7!go6>c^V$W~bbFSX+
z;X>Zhxaa$%l&)~+7d{NF$YrbI=X(6Wi`h1MulT!ZE>eG0EECVCeq38QG3AMqisP2(
zscCxl)oZ?=mNPsibLiZTbeAfw9p7qB|K>`Y`^4|F*8jAro$H?@s@*p|k+-?yZQh^v
z+jV)q+dL~2jeWv)u69f6cHRqX%U1v1YH!Yac9QAzuQn<6^EFgg-Fw#_uxr`JTXwaI
z^ZE95Cx@{awoN}gVP#d+g11i%9u?~4nU+d$c!dP8>-C@Ft*&2wWbq4MGr=9tmRarJ
zG`}UgB|&*d%-v<7uj}l#-|p1gu>0wg$%4iAZ!;yT><-><sS$8h_x!`2UHifo&&tav
z6FpW}#qs7uQDL0=@eG^1w0yx6ax3Tk4o#j~e8QyswW{XaYfqZmPCsl9;XN8LFHh|F
zqVtTi8JFFy{9(1|4d<p8t;~C_Ot0AU;K;6(y891qDZE%E&i&vU<C<t?jhQw#s~S$p
zyxdhX{gOf6)EU!M-#!2FX5OLbeVujBrZ912KVG%w=I2n8lT%(l+RSdASGVcz-1h3T
zF*E;aU5}aaxnUA};<gu(n>n8;>%CMHlGpfodiSHVS55Nj6z?2QzW?dW%7_n-4nBD^
z<yLBuW46(UTgGZrP3|smo7h(Ob3wsjKjk-V<%vP>#l5_GzSd7t6O~`HcyXGV+>9re
zj-Oa##N*F0^TjKHcDo0Q7aq?_w&1<|KH<;YmP0da<)WMYW_|i4lXz0uSEWv3>-G3I
zOo_GkmL3nCw0Pz9oBkDd?=Lg$ExsTcCwn*jubTgrYZg4Of0p=qOxLol5nZp+TeVHo
z^S#c$==Q)V1xuTEO{>m({^nDS@8czxP9|*r9u&y=Ui#t%{@~Z%EDOKJ-EUg?;ArLk
zx16hgD6U#_<S=JX@sTbk<2yeWdAb}kl>cV+W{yQ-XIg2xPRiw*bNrfwibQ2Yf=|?k
z9+gm;uICu@X@Z)`$=he1-3g9vueSX;Teis}=Edeie9a#&-Q?QR^Kgf%ch&AsYLDvf
zP4@q`LH5zy#MtS_jz8Ub{)c+qZXVWqCDZbDbcO5HefZF{W0}Rr4QIbtJzDbPx0>*+
z)iM4i9FL6N6}(%kwe7a9fN$m`?uc0*ENVV~^M7(-_8z}GGv9x5pC-LjiTi<7=jxR<
z0cCYryF>Dv__p)=E8J}lde*9R_G;rIhQmL<O*H?-@OM|Bkx}pJRh8ZHr{1nQ;b|Wd
zeN7_g($saECAXIgT6}-=y7S=PI}NeSL80Fs=>!|~S-C9#JYm+KZ8L=Tem{Nk+24zI
z7KGj}oSDj}oOv?S-6%*^vM1^Lp<siTVO3=(lw3~NJUIO2FI&bZJuk_e)BfM1?sR*<
z`N038v!I;UFhRy;zDd(_d)B-=iA-Fb-2pr-i!2qt7m3ZXd@%Rx+V5+ca}0#VJGdrA
z&wa8;Pk!fbR>jNyo$M-?_LY2F64NEnuxM&A>&}PCzlzH1)-(Qk`9AGrQ{J3H-t)U}
zre8jNx6X3b<70~LJ|$OsUdr4%(mv__Cjo=~M|wZK&-eG7ck1;L%~w6KoQqWj`a=Sq
z1WJ})xO*u6SAcFf<F17=8TSo0oZk67aOb`5>dTIICi5CupVN9_c<7q;za^1!5r1{&
z+aBJ({rhs!-4CyJ)ctpDe7KQ8J}$c{?rTGI=`{Z*G6LIGv!%=?Gxiyo?6@yAZ9&KU
zLtOjMNH5Px(%ZFYqnyVb*Hz)qHpk8CwNu&o+H%u{ee1hc7eBpsu5!QEbpH(7wM9Ea
zQufca_;_6ZhH6@yxa~ttCDuvSQH!2U$O+dwSQN2z$(_>o*IZuxjStM&|4Y#O@x{`0
z-`?E$SILtmduNvyyDa<lW3pctoKju-?_!ra?~eAplbC)c+&Q-VqyM3V3lr;Sy!p9r
z`V9Yf7WHhkG80rnPhZ)-v*_G^g@1P}CO;IowYqWs(RcFaa@X$mt@kyPP4u;w&AR*X
zPrldkR*m)ZBx=vB{8!-TQW{s0mpE_lv_Dpt_rExsA2mtNdt0}I^8I_g`IjfnpKp2V
zZTlMaFrP=$-|EaW{guD{<t9eXn^WSBI4%7g8Iy3LSWe+cdv|v5`rPI@g#~@_+j>f#
zXRMj8|708ErMrQ%Bl$wAc3#h$@cs9(OP$IWWAD`$r~Yl!XO7#SKj;6B<A0>)Y!ht_
zjavQb&DH$;=4bc5wPpQXaN%?CqPRWodwle+rwcF5pSmFJr0}}Ko3}mm{8;_`+q0A>
ziNE>Er}xQzQk$Hd>fdEE>1@l>N$LwdW^UE~Xf>OOQ>TvQvXH)`>yfW%-X7}9{+Dro
ztU6R(tv&N{Cm*Mx{h0;d*0KrYv`#o*XZj;eZp-D<0bxEyPeODa>)DFkoWNJEX?@34
zQ)vp%@tOD7lsR_2GZ&ORGNruf#@2o2QsOQyPan@{wn)DFQE<<(JPGb%{=;j(wr~`F
z;_r%k^e*Ph&#OOwUJF|kvvXDG%yav$-R|@4GG01!POa>#GGnK=znpd{Y`rY=m-FMS
zD;}$QH#erN=T|Pzn%aM`SMsfH#)pryr5+wGU)bF5D0Y9#a(f@S_n|JL>t)&|&%Daj
zyfg8f-Pe6Q^`90lwK*1C(b{=U(E97Mx{WsZd&B%695na%^g+8^E6Bd&@)O^4u6JJa
zUtQ4lTKsD8r^FwwJC)0J?MY1tb=`iP`?OuuwA)usraX<Cv|pBI;^Wd++t!q4`hQTo
z_<eeO&S80JVeNg+EgJI5e|GMEX!B9_w$;lnJB?W<L<Gc7@Lv>8EUuonBYfh^^RZej
zaq%HF5?L+`RX%<eLZ=km+&yO%A6Pk`IsBVff9AZrbXIkfibGddPg?o=nDAt$7YgFn
zMbjU6)PG-8Ah)e&hjQMx{3ZT8>ui%x+vRcJy17|uR`e0yg?nr!9iBT;Nxd_=dii7i
zjaKsyZ?I!xKbrV_#iSW(h6g)edta7ri)Q)95qfc+cmH0$r0sX5vu><E(f2>ILZRrL
zdA_X*|LmfSJFW*d^!|=mAnp17fy{?2yKT3Ua|6!Uv@=|0;+rS_pdjw>Vjq=?wbka#
zKB?_O6OIcwu4A}gBEL-M(8EQynhTH1=nE|ja(x$j_I}X(yUXj+5+(Z843|yZJE^Ut
z>GcQp8@?;fnQ#91R=m){g@rAq^?i$=?T5_||E*vwyYxQR@cf-$Zv(FQ&i(v(=jL;%
zvnP93-<Eyo7&$LMUuX4wCdO}*=1sEqYOU2e&G~HKTfvW)Uf%j9od4rC8_R6@b*Xyu
zefY(#zVCffo7Lj|RerggPde*^YaKkc@+t=uWUt0Jr=I%%QY5g0<*(BBw&?lGXMFqk
z_(V<fmLE<xoC9Vm|FNCX&>&y(c>h73q^5Ox%E42f9d|f3<L_My&c}@TdeTRtEEis0
zRQ<!(l3{_vh2#2@Rg2uumriTvZDm+<?&zz9Mxo_jyzc5Onz3xzb%UGb-Tkh`9J9V|
zI`VF(_ZN<E%QY5HniIMn|6N>d{5bRJ(&Jj2D&wZ5Jd!)Na&rEg)gJ43rBplr@osm{
zixK)4y^dqap_>cyp4?RkJ<zDJ@a@B-@0a%8mr_?xof}tBw2F)CM2B+R17W_M=C^CP
z8&BwO=nj9o((YREJ1+Y-MjA4^+PB{SU9j=QLLZUoU7sd)Uz(-2`?90X>5@5u=Ih1X
z-rcg_=<Bp)Rgyj5E5<W@@;Ck#JhV4a;@hBpVUOQ|`kT_M^F!MIMy%i{Sl}X;s{3@I
z=x4t^hn!dMB313TZ`oVO)oGv}U>Ty;aVOi?>uj~O;`N?Yd;hg{^UssBeZOM;?NavT
z`*-BuC`qVPIuPA?HZ>w~2SeO}8K0^bsTKPK-Mas<^UnTHPvgFOYWi^}ch}DfkUqRx
zY0iDoqMN+4zjNPG-k>d<(el>sr$LeAED_-=yY^2Pm8>wfF|7HVe&&0|ir<TZc3gFv
zH0Oic8iu1)`NfQ}pKrb~WUVeaoHAW>=6Bh;uExc?Qy+b5c7Dn}z4qNg;Z|kuQ@168
zEOl8@x5q61F>jN)HUEUi%|``H?9Q*9SL?$*;ryxPx33n}&R)t{Zf5pas-{O`GGC{y
z?X#(0b{)Gr<!8n2(*Nqx0!=%gvdL!2)il|cB-h?<@L1k_SaaES=E-HT+hjk6?&o3J
z!N+!~vO>$}dk;%uMbILHStZhSEYXfXck`L{)Rf7e7c8IFnYF0t{T+3Cce|+%OZwCQ
z^g6Vydj4&NuKF{_7iW6^i@WV@R%5@sAZTHoP?AH_n%TmBp0AfI>`S%zKG8t9(DFyg
zm0x?#xVU#bQJL5BPN`}`?XBsrFYpScul^!Gf6A_V6BcK_EwGMIjoh<SX|n!)Ys>4G
zA2Cc``NICg33bj3GRJ26Uowci+*GsD{HB5blK8BdMt^FSaK)E#G#dXrscAO<U6hY*
zPRh{_TW-p4nDqZZ>X%&RXI0K_2J>Wc=KeeSRd#OHd21huhmEg!7EU<-uP5opjrz%(
z9&Nm`_^?-PA5ZgFmDdxCel0&5ciz^u^ZiEk1<&=mtiI34)0qEu!|hYeN1iA!y<cYV
z<L|i=V<&Y6Porx_UuLpMe-rt{wPaoT@#*4w@@Mq(X{|SSQonz{eRhV6${VH49P;y*
z$DGu-xj5rn=$E33JjarhPr)kHGal4VP3@h^-+p|0^y#k_T%GrqB)vYmDKe+(-74)p
zztU%k2bLabICI9>u3U5CjyJxOZe7}(z2p3nd$G)Gw1r=OJ{@VCU-PU<@+;5l<%%CO
zzyEGq*)1f%x=N=dRhYNn&)=0SuJ`WDi&V@{s9eqRz^bJ9r!vR7^Z7F#u<cN5K2@F2
zzPC4%`MTAentiKt<UR)6=D6$Kqr<5cZ8H75maSIC{3nmwb&^&2#OI!TDzjcTTJDEH
z+uL(x);fOLk>-+$Z(G^5vUcd&ACNxhuh%A5xPyPiO|kHut+nn^a#D3~Y*ss%bVj*!
z`7GC)w>aPAc=rFjw__EXGP|zrFN#z8mb^<a?BztKeh-I_+AHgVB;~o^w>}l-e2}*8
z`tG3X-olp|LTzSiRq08K8TXvF@GI-gytd4DS=fzRz7MPS6iU_Usc%eddd~D`&nBOc
z?VdF$S1TX-o<3TrrI)aE`yqk<oX2OsdQs`gGW};>Sbe3*IquCoe75|jL+;!@x#`B?
z+MT6!4=PVuhA&H1cRr%NZ}$XOPyIfX#OSNuc_sRL&US|G@|d&tn(sm;#-CF|GJGuJ
zFMYfn=b?BfJ6Zqy_JzG2rZPLu9&m|^ni<xYSUK%y3iHWQ8I=Vd(#L<Ot^cSzo#~~}
zr_JIw?wGo%{#A+)3ZDAn?76SGEmn)J?hj;@ni{=f!n;#}4^CUMyKzm6J^JpV)r7Xa
zyEE4%N*p_7xs#DA<9_x*7PE!BXIO4oe&K>n`q8|=)9cT!+RjnTYc~0lGP`ee?>S>{
zi<igdXik0<aU@RfIPY4GeR<wnw$E9&OlO4?qi<Bqy1O}g&wXC!ht(W#_F8suu5{>u
zv|gi_u4i%4%PrT08TrdDE9mdbZ|ryc_`T@R*}jH%Z?3TYy#1|Of99>m;2Q>OswV9)
z$YR_1YtIa}veVTC6E>z>uRQeb+UwS{b2(b)b*%oTy!W2^p5Mh5b*mY}+800L-g{x;
z!>X6rUp3t)aJ$6#=fzw&X@1G}V7IMffyME)IlOmITr!VzJe7N6Qvjz`M||c)(|7*f
zC;BYk&JAUmR9qx)^P!J-Z_uor3)P*=e>;^em~pAb?9^U)m6zUiNy{DbSI12&m?iA0
zyT^;`YtRBe%ho?JN8cnFE-|%vGU0pm`TO0gLng7lTXsn3S?b4^C;m!J315@^!!z&s
zeDfDKgUn}KHMSM|yQA;M4XMOedrrKcF#Yb_ht6fY?k#Vvzs>!sm0A4A-^XeZyN_Fb
z{(kSj;PidnxrS=Xmn81C_EyYz`RvkG_bZ(~JI=>S?z_)m`nqwU?&;1coZ5%_LWQRl
z7rJp?{5yG($nBUvJ5vr{>)@Cb>A7f6<TSs$vMai_<w<u;&dfQ!?uFb7KlRP(|2Fv;
zeK}Fq`u$>H0jF}v$ESx@#C|^e>g#si9fGT#ms&)}O$aTQ`u6CRaZF~N-SX3H7Hhsn
z@1HJP;xkWH;ilTL%k!KIWJ3zFXSknpSs;J>*yQ;MpX90~g|<~_9&xiWyj1bm<Pg*J
zO?CHY6e%meKgFu2GyS<s(Vcm0{kk#V?j`Te-ECm_(&N|kle@p$9?gF4e@bD?js=s<
zzAvAt^x)XX%P0QzOF6VJUog4p)FacE{*y)7I@6T+H_l@@&^*U-k*3`&o0*q)sk<#K
zVvRbsU~^UI0ajbVSr^1@O3h8Aeu(!9G~S!Bey5AvYGciX2_niqr~jUv;li$e!*bUH
z1=&O0S}P?q_DtbCF3{s3rrV|ax1nkg%gk;?cca_|g%)o-!ou!v^r{!K7ieC5<ebU-
zGh2IqKb3hXA8K`QZcBt+uu$Ed2ByXLryh9suW`fwK(V;+_9TwU_Z8cWlb9D?y4ZWQ
zXOi=zgzB)k_@^>kdos4)&eVBY8M*pldcg6XZ_A$=H-5NqR!I8$A+|GDfB5Tnr>vV2
z`|Cvk_l9p(KkTL7-R7-w`t;Y-`1$kPTR;BZe!AqdfT_&M{{~7jeKj^0S}!KsCB8M?
z+4DPRzCAnhqgP_-1}<zT%BKI&6Dn2;+^c!}ds$q?*E5z)%a3r`pJVzIpL1dBiS4}s
zEM@yw{jsWk{O!=x@5TSweGBzFw;em(TVq*SwEbHH_nh}H*Ur0pI<Gp}c)y@w`?*Kk
zpEx}Ev(u|`-?_Y>e;0OKta8-p*!A;D?)Bf*-H-dH{flHyS@!-IhkmJg;DYFxlUE+8
zs${%bwK!&;l=}YLg-m{N8EZE%#{Dq*XIyaTn(_K|;x{;-7B%l*n6qcXx}Be5Ekpvh
zJ$#iVdTygV^Nh@7r{$G}`(Nn%Phw0t6}ipr_w<eauaZpCU#-1eD{Xt)&`7OXv|d4z
zwQXDXg|K^eF*o#@j-A+bVXN`$4U^v}O#F~ut>~)t>d4iHg~z_Cd713!>^gQd?VRT0
zW3G4K*IKE0h4_7G{?%|Q<C9loL+888ljiPL++@!wHGOZw-`@+keywbrIWg+g^qMfX
z+AG49f_mGyqGw#@VBc~`$|hUnlG^rs(`%27j@bQ)jSF+}(+@3L-dZ5bBP{i{HMdJJ
zW&+P8>pCvE{crLswihewtrK51DJ}bEc#}kLZTp|CSuda7*|8}uhVvuu&onE>OK<Y~
za)0+syZiUeCi6$B&o2plt>ju;!M)u+tKs~u(3|a5_x^YW?DDTN>J+)owP1U(6Vshj
z=X&-oQn>yk{?*C99hyHjaLj4$n{}lkPFL*r&7=4F|Ni0Fapgi~^wh|ipQmk}{++eq
z@yP|{Yz4McTvt?o4E=aR{>ij|D;W9O_a|B}H@GqXTT(&M>zgLc!Dnn&ZgKmu@U&sf
zxwIMU^p>P-vi-{a+<|{i>+h|LV}6GPm;MwePBAoj;#mD4%{-i8=BE0oi3?PjPioHm
z*m20=XT{FaYAw#&b}Od&oZP4_J+p02Qv1u0ohz&r;%>jIbA4o=F6nab#r9*fS>#fV
z&fl~4QGsA>!ku?NRYm&WxqH=Gaa3zHJo@~=Uht#g;mMoCL%mZ>4cc-vQ<jF51us?D
z9+T7XlcV0adTropxzt^4*{!`N&mFpbx=Le@ZG>pw^5VbK=I9-bGfQz{dG%}Fl8|@q
zZ0q&+tlC@iq>8U&-Z!%=zP_%9T6Y`mH_iEEtH`<j&(CXn&9#m#&ii>Nib-z%k%qWe
zI`8s?rv%t&gzwz(<x#`o;-(UfOgp3WbzSloj*3o-(%f_{cIkwNE%PjOvbJ|DnD(so
ztCx4(9mW%;D=KDhaIPqyyY!MnR736O&_lv|oRe%j4SvYwaLv@pWy$NyacVE!_g&WV
z@s|KLn<}Y)cQ>5wuy##f>iX@_;Uh^0GS)sxnYrek;hPU^VjuK2@IIL#{3`G&hwZo3
zf9$@>%-nhZS-EC*Zg`%}Z8!gyvx0&J{8~@Pg>IJ8E_L2<xV!LF*(3e7eZSgS6wOZw
zA2R26oqy<(u%F!f@70_=pXQb?eg1cs^4EYV-Ck_dyB+TA&vd!CZl2%P=ki6nl^e@n
z9F*@_uqR*b^{@KnjNi|jAJlmw_3GN&S$ZF3OWP!D<&M~{jhl7ipr!Z=-tyS_<`Yk(
zn{SU@9CzfCuEmPik6g6tv|I}}b)UOG+gp60ety;z#uIYcZ|AwUUOI0klh5B8obUC^
z=^Sqh&(+0yE|@9tY*)Fm`cALzX~${h4q~kF8!!3E2q!N7Vsv%$;drTkr|ury%r!;M
zJ>=*X|9|1x6VEj7af|Tp5B7ZVp>y8a^MW<b9adZ^PW@$~TUWoo=a9L=@lWkE*KK#t
zCvslrTG08sUm)$sthn1N?c5K^Jm0puX`zFVv~l=ZgHB6ju{!;DpKo?s6!qWKUpOBA
zGoLGQ$A_%SX?s>>dN3u6s=e-9D_*kgRM*AeO)`6)ABqd!EdF7Vy~PRnuGpI23V+s1
zKhU|bdNt1(%M~1Vzq-oqZ++>J;v!N|&^OCpo=YS=zG#j9ne3L;|2wC1us=Wib3xla
zqqH5@TXoJc@LoUMH+5>QgU;&G+beg8w`cDQv|rOTtLOE<oM~Tw{ylSQ&Ih@V+x;Ci
zY`+@C)_*I!AMN~V?j7xARkvmw)?4B|?RR%O^U(vYCO@5wI9R&G!@o>FJn_%3%Pl8v
zZ@Ml%@8Xi%yR@`ll<U_f@36UaAb>L^%0cDcq(^h7^~$fjI^&?c<f_C@^IxqGtiIQ+
zRrq<&%l-RffydQfW$)@A%(~9&5olcf@aEK#Z%_KpPyT%W{knJC7$<pM@-=^L^1rQT
zdv?R5U(V%xCY)UJOkQ?bCcfR{&GAKR*Nb=ZOWwX;H{t9*PEMyab6L(8t$6crdiG*w
zu4{FR&zuilRk6iIUS!L`9omJVPh+cGuLWCu{3v9`dZnkJXWrLk%O~G@<r?$XS@iJ=
zhlaOBPoCbdy4HQ;z2-dMQ`@eGXQgSStv^36tgW*BuI}6)b5`*bPyaN*xWt*atf{a@
zZnNa#_x(ze^+5|H-WJYpv;Xnvn4r<u!2JKG+3sj{6xU_;sW+ZesaZbpz>~&@A8bS#
zf5c{MmN;3i>REsH=hs7vA60idcFnAPp8thmxuxFW(x_9rOPB9)KK?VBuUTb=^{v#V
z`%ER<Y~~ui*SO9+)%p6O_uO1P_da)68a$9wX1Y6XRpPY!pFXH*iiB4#@!@*DEJsI~
zo9Sb5@v8+7<|%jI+7-KP!GXwUOx$_*H|yGj8XV`@EpbXJ*JGOTl?f^43R_kk5501>
z*ju3OrO`SA)6G`vtF!*dnLpYf#B-;^&-VQFXSU~Ts&xDQKI2*LnZsAy62Ne8O45Fh
ze7QUKIQt$i4)I+W|1t1rgIqLg-o{k(rEku;7zr<KI{L1u?OkfNjq;O_r*qz^M@RWr
zJve2bx^cJn%#>dLkE?#HR5-@=_E${Na=D_qpO(qmv%fAVh;2VH=al!&-osN)8@A^O
z1zNHBPjqb#xTL#QHBs;Dg1-m*V^4mo3R5rlWG|HB{=CuV$_9<Y4K@l-+*X}lU9YC7
z)2H*+dh?e1CeHj<bYe5^Wxp~nP)qV_HI}@6(^1TYYo_OkPmZaAZF~Eet#v!HPv;+1
z=<8hmeBZkUYwioX=bgAwI+0K9J@c{krWX_EM|_N$q;Y!dzTTc$_dOVDc|Ep&pJEzR
zsq>ll`@h7wB1{M07x76Y@7$s5SpRS1gG2@CC3^x{J@l*g9eFPHS(U|N=LRLMe=q;+
z_`B~*@U)q>{PB5ScWNH2@?ADF?n9TrQg)FS{F^ElMZWR0=&>rhWfsu6izi$=?sA{m
z9vLylf9iS*k}Xy~3%)UB?}xsY1G_>6q7UDjn-<7aV$O47J}-Z9WuThs!CfuQhszni
zR-fOmsq8j|^XTF^W^#;0PABuTZQL!z1eL0M3{{l(#?RZn^TdNWzvDihpSiAW!>ac!
z;WGkXZ+H6suATF@i_Vpj=qYDUobkH8&E|}k{+ek&qW-&myLOuG&XfE#>KzK{Ry`Bf
zmZ`ouy2@!s(Cc5(uBUIGT+D2x!71SQ-0iS%{UQeCi&`tESIt`(+PLe!?#0ia**vyR
z&VFyy>2NRJ_}=uxf*N;h*Im1Nu{pVXq4n~~f2T@cvVW!1)B9q=`Kv+B4heG768A*a
zeq5XLp>h6+>^1J@YW12D|8p#4eDnMSGxyeMSroQfhCR|*{$6j!&0n|ls%+=%_g`mj
z$|d~#>+|T~C#N`qi*1*dPwfwydN{&gV~+W{rdO-1Q-0bWTl8|C?k9^Hfm1(Ur><HT
zzV8U{+_P=>uT09mmy)k^G(K|m^rtGW@>ilxFID)Z&Nc60d8M+*^6N(IMS<a8S~i#b
z`Zy<G>mKdY?LF0A>z%h|cWi&k?j)O7-ag?8>*U8rwSF(%D)%L(&~l}pUE9CZ=68EP
zZjO5+^s#f-Kb{+kwzg(3{BAQ&{vpg*@sEA;<MQ6Aw^JV9w0<q_XP~$Idn8+4wd2C_
z<i`g(emPE=`E?7={e-ib@3sC))=aC4I??`P@*jgkHA20OOux2&dil4K`N*=k-zk5u
zulWCe2V>v!`6;iDB$fHhsayU%=F)}9yPv8=CN#ZUG%I@P?y32+`X{~o^EqX*!|UI@
zyX|k;&XilU%I?i0&d+wt?;;-DJEHnZY@Yt))^qYZc3QVa)EwT?BviHO!t4p3Gn)c(
z1VXvocusA9Dmvk}UNLu)ld9n*_SJvan<%Wj$13Q2|F@$8_m<BY!BU==mgnBgj>&s|
zR7~diO5bCb7i%wYD`D97Qv8ZUz4*&tbFX~ge4$nGO>>pqZ5#2X={=p=hSMYG8`XV%
zar_ZaqlWTvrRwLt>t^w`l|MMNvcxaZ;LhoQ<W*8#mv$yySn*ILCP#nI=Y+(2M^?TS
zy7_T&%#-Pf6I9$Mh3`5c{Zd>z^2)>c+_zm>S6nGnRE*zjptom%z>^s_^rv)O2+ixS
z?engCTjeCQMSA)&)fkH%Q=fjiA#iumjeRrs2ue#oHHmogRrCnQw%<Q5>}s;<dn~f#
z23MCSul|>i2OOWCt=l6}_G6yalr1(-s&{@=ky>_LP-IJdPC!CLL_6o(roV}Gy0Pt6
zt_Nf<tB0w6>@7Sn$3f?zn_ado`<?s{X^qa+ekYdaO=&6fyLWYSnX2q%8@=BaqB@s?
zHNNZ)`TK3s!_V$T?N?Rbee8UCX`8LsrengJ7bzUQI&b+?%S->_He_gJ*F3m!{O;>T
zd%UFhti-wICB`$vtG{#NyT0g=l73H1TUxkl_xnehl2!Aox%W?soqGGVjpp>ddvhGD
zTwN0_&HYmU<jtLNn{C2N^KjXcxMHTn|1vu_=YQ({Q{z`|U*NFrlh|+LP7Cv*uaoA-
zpKy47VPm}uxAn*D@`a%rcdq%^^uu-P=1-a4dP(y*R!{%%&`g)f>_DAc_0Kb@J0=L&
zuWFV#cg)f$_0SUu!O~sRjAvAB)pbg>v;2EMuOVmS=c=f<S%w)RssBDcPUG+X+xA?2
zX?gw~ri)cwIgxWG6~>+4&hvb|#<LT`J@3yQ%t{yhzy3>G-QCIE%AE(6zmPa7{cYBd
z`*+vH=Ovim5O=ctK0PmH?e~@Ob!PH4Qp>Ar9@+f)bZO&7?~waXD-@OmJUaL5&MTXH
z5smp<Ol`yb)#KL1UqAIQ`Q?S+{J^+Q#h(4^_s(0-?6M^^^NHPCO`B<(XHB=^FD`sh
zG|hI6;=jGu4;*vbw9__rcgox9&9@)subN|@*RZB~P0~i5z1;_MPkwqHmO5c^l89^l
z)Z4}3SKnU#vq<A-|C}%K<_Go{>^pzu^IE>0-|os@f6tnfl*0Fnd-n#-RNj~UpX0jR
z%dhFr2(<ZhthIehUB1qGix2+~+}mJkQ0W}x6mIygo9+E)CeFU+Ayz9+?@}q8c<I~Y
zpOVL|E$2ObU6Lxlb_#di_xCGLt8TturS$c#uYdQI-`wJN{Po*Tubunz?Gu0Ajw?IP
z*jB8w+_Lvcc<lYVpQog~DDpV6TGQNXRu`kr$4bdf#{ZprXPNWa=`5Sy8SEx}#cPS3
z&6jyo6EaQW$|RcJ{Vv$BR{bdF|M{GIcH}gidMkZ#<#b*jz8ZC-|L?Lsa6T(5@Txgn
z`-y$Elld1%9~H?H|80*i+W4k;;;jBE#Utg@(=_gLZ4SD>X8QC8w;%en7lnT9x;<A&
zbgRwE?DZv!HYCTN7dm;@zQQEVWpbd~%XP<A-ggiD$Mb9Y=fbaRmi_&k7-yz%tyW$6
z$o$P*U%uYo+wx$Ky<nBO&!qoiIU9ubX?`f&`NUww3I4poM9Yb@+1d8jJ0*X7&TH{|
zcSF}g`vdnZP2;A0U-b62qVEmw`(Nv}m|w45a_VhBM#uh_M-$fUspg*e>uLK1wXFBy
zQ&#+6%$D)kDrUos7iBZz+Sk5$*w8E0oUvZZOy}!<lS-L&_1;oyJ6>5gRwc^5ve@%-
z|ICtqyaK0Fx3$DG$JKex$;%7*9cs5N>Dx!U=|58Q@8st9y!7C*Kf(E@el?@!*7egn
zXCFVhS0Lm6N@sJ;zWUSL6DkFt{m*xrZjxxe$RM(FLqc+EM4Ytt|Fxd$(=A2ZyXNl-
z{5L<-Ve77ByW0hNd9A&x!lh5_*Nt5~yJ?DSVAGfM^3>ax1AczV+G1W~Qdc*BXO4pv
zZ|I~QuR9;zE53D8aNhQGwYr@p5xaLXiT{jmf38?&`d-8^Mab^no~Kqj*jWS1qg@==
zo<HAQRABsLnZ5qWkowI}gj5svJj%&SajAK(UzKdHuhq0?uJFQ5|6iFhsrTD^E^0mh
zZjr%+laiYKJKnyu*|*^F`Zi<N;v27K-Kk!zFz;+%QR<TLr@KD$PU~vEzW(-$Z9CUi
z>sx*5jJ0?*vHw%5hL@e^+y9*JE{M3kExS+>^wv_p_P^z=bZ+M-t)b`lWLLb{yCU%E
zOmSP&tZ!9pwu|TSESkN*?Np?H>8z`nk!8jPqAv=)dN<1N->ncj%dIo|aoX?nhpv}3
zzH?vuFe`p#wZOHfHmzmi^;d6a&U<UL<LugZ<}ddwkXqm7lm6k#SKErXbFZaDLZib@
zGm2DSFxlJ2xHB^G7yny_yGheR@4S&)dYjwx{@SW%y5`>b%9Z<HdEHrm?44!N6WO(w
z^yci_60oKGfzk0hJ#s0t7oCn;q5W&)zGKVY{(5KXZvX7YyF6=yD%l{TU;7VgdFM<D
z|Ed37&Qw>QqxJC3O>T+(n|sCQ9~9F&AZEuP&3pWh?$zpjD<&1h1zf0WFPd?m@pjVj
zZ5D~qa~??X7xc=m3yhXpuG=g8W6=Te;~nXHJZ@dw!v2_9{1x+wOEbJ48*Sg~bYe5d
z&OHmlURgW4UElGyfYVOn_rhDBcFo&Y?eNK-V`0;u^Omz$t@yF&_rFJBYmTPu&*xZd
zamH}Ic-h6|SrQ!wCVys`7jv&c{rcw%sXH0`3+0=Z`$UJH<K7+Tk+}84Qc084{h=Q;
z=Nzl#`cZo3=YdF;FR#AJ1UF4A2?#y7Y2L(SpXFIUstz<(u{~z~_5bYxn`hkfD;j-2
zbln%(_rPlYi--rFUd4828Sj-a9nO{sQ9fF-o6%U}2Kx=>c5A-Z=RPzZ{=4e{mw!mr
zCrKGSc{}D#_CJQ+-G3id9FpJsBl&!V%(DHDrFA~0fBK&9@nFM49WxtoWzF{Up$eRr
zkF{4veQwZyuD)i+eu3rtw@N1F9NWRwW@D;zex6%{%6(zZ#!fkIUEkRPhj)Iz81Pg-
zc~)G+)(dMx4*ZNbkhuJ9%j&ektDHG6xgK9O-D+zhwAA$6svDE1>YOULrnKzWA!oT|
zQAa%D&Ccv*+usuVznQJzuToz8k+;>lAKv(#ZaKX_(rV8Bj@#co6dHa@Tt8$iP`>xl
ziofCIyopclx7y9AdzY%i()KUwu(u6AXSjXC@-_EoOV8Q=W%dh6nGUAQcP4PwUC#Ix
z$gI+*)R}BPpV7vp|KE2>8}Yqdl3DZOScE3UN7x0qPgE#;nBJ2TsN%54LHkK#e)ulG
zSATgk9l|tr9sgHl!N+uT_0^rmWeSB;k`I4&-1mYvK`BwD+Be_f;l4_-x|NKdJSWB&
zv2yM(ep8aVpPhM;mm;G8>v8rv#pOKT`Rn9Y@0$>9mmtF-e7{v%$$g^O1JO2?<!#k0
z4xcQV)N2YNCisTD$i4npeKoK6YS#)&HUD1?Z<HU1aGKt?tn0A*Zxwkri}TpwOs7p!
zlFN?YPyMw!?y(e8*u;iVm58qftp+Ewi!L{|&*b^Kf8`v5XAd6!R@?oSEB*LgaTCAg
z@Aayth*h19PF^xUZqo&q1>#TFP3%2UeExF(yiaxWJnTd_Utjj;T<p@K#{rg0<{FAF
zk>z^vNhHPVg0Ww=MrKu@^0YPSmJOTci1!qH<eL&2yi%AU>q1lZmHy|xH}$t=)s^xK
z-l(@VtJ`aNjYIO`p+}qlychhHw26Vga;bZiruWgJ#tR*Cdlz1c&`FP2Y52iO=Ec;X
zcZ$8$=bhv(<`%Q46P>>P{JhNSo(JbTc#hXL+irNMrR#V;|M-<7sjGLq;eI2@Y5Q$K
z(_cBq=kNLNURkl?xb3Qg0jt-&H)d(FcUGA5TvzQ^FOQmiHiPuKq{E+?&aJLLP*%OI
z<@e#Q4R6B=uf<)t?O%UCL2!=mRF<Nts~%YNRy4YPWY*a=<2k#?qw|e_-m-0Y^mx-`
zYkljv7hf(iU%-3+UCbHD=|7G%_!#m(UX}VM!Ds6nspXfaPhV|%E~?7E^4IP+Ryz_-
zhQ7(YX>P)2eO^ENp5>h9m9I?QP31nnUzQ%p|LJt=!8ZHIpWCGlG~YMXI{n+N-fsHm
zN!`}H*^`cOYM-j))~mWKHs!iT$;O(U98><Vm7c2X)|2Ag|ElQS*_Ij8BLZ4Wz7|Bh
zx|+Dsvv+GlRey5W=0-PWWuLN}|I)J5l%^bh<P&#8D$b|%%KG2>zGXjZw-^~Nc`(_!
zJpa=1%(%R$g}d49W_31wzFwi|SHC>_i9&6}|LEXi*Yk#|dVfFH>}7aox{1Ha&Gz!l
zLqXXt<-ZnXSIgfumiqNy>2CeRpRrG;_{|ncs@Gh3vF^Huc4<r4kA?`z_VA+zbNd2=
zUy063+QIhPtY*)FZT9Yq?&muMGD@m<tvi4F-f`x2ZjEZb3p|;8KV6?QRmyvpUH+SM
zH&uOYF5YJU$9MaFP2cVG!bo*BEAvyk&u7K>#m`&77+(5l)hws7U*}%$oPUW|*p_ep
z<(J+Tugf>DJ=C-`pl;&=_x(TLwdtJZZ~Ifm9I;4PdXD$4>4)p4t!QJmG1#>4_|rYp
z4Gh%fuD({%6=n-H|7(yKY?~wd=S9S|Z+n7ccHB=q|NUTyzQEV><;-$F`L4c}v`$#k
z{_{ZK{V#00OCN3cv^f6UfxR&dG2i9gPj!2)pY^3qI^m?c!JLPs?=CVrRFs@gda``~
z{R?~*U)MJ?ZTnyL=|4-~=2d_4ZiWk#i%Yz+UlQM;YP;Yt|E-3aPfa;WJ7n7T9tdce
zQSR{dxw2pXQ(h07)$Uxgq?Gv1s&bZoIaKoH&=EC%+x8{m*Iu^TG-~X5sMF!{pJmye
zoCcO>Dz#@7I;+i^EuN{A?mZ}`BB$Qz{-A@2U8vt{gVy6CcTZjP5mCRGKHu?kkL}A@
zR_~+~#Vb2qR%K-<O`NXr<l=+r((`Lvl=UZ`@Ls=RwPr|p{0;9)%i40qrx{t5v!c?n
z_+1qAZ4dExult)WrLg>DWI+_K&<uMsIR@8TpJr>6yxUn}%5mAdDb}-Uo&LA&8S&m1
zH_Lt7KJUBG#}{}1h54@a3tk=gJKA*nu6_67Rp$I=v%js#GUvYj;{ZmvqzhBDN_IAg
z3!Uc=YwP%8)TXvCUe@UH`eufSFSdVwoy)AULAd4o?mzlxo;IK4^1UPHE9X($p0ns(
zj~aV&U_ie4?t7o-{a^0z;9SP1^*fi(QK)g`nIL=r0pFn|)7MnIxcdA0@-OlccYi;v
ze%N*YG1sG)e`da&_~oY<kLzB(8H%NK0{xvPRV{TE%C?VQTCWm|kWhM6zM$WrTIN&N
zsuungTukexAGvvW{oOC$rQbL(eVyb`UONBWa><jQWiyh(`cr4QUA0qBEGnvs%W=3g
zB`E!oS_~WK^xq1ve{S|=&;0CrQ`F_6!pdMyE8kB%lWdxK!u5+@E7r1c&Ioo^IcvY-
zS67AA)4P>dZ~s41tj=9mX{%>DU#<Jg&t(bspS=y5wo#e&fL+^@%6lKyHVaiJpI!Ur
zc^gaBk!eNqZfAZyvC47P?l}7czV*M&X9Sskm*Ri0>Lt&*U3CoWv{Kfl&SJZK#Pyig
z@?F!rCudaXRFo91uyYbP;>g;k|GYK!TZ@diO2k>w#A%<;DO}j2{rAO|D)X@Zg3H?b
zc30~pWt7_$a+upQp02EWot^V@#q{^44l<R^&rYZxPmOnwDSQ%}FZV%t{+AM_#9!9?
z*qApR%6#$pp+nh^gzO3OZk_KM1s{HGT5^gjDAAWQ;PBBzN3#m{c@wTYju1$4_`iD(
zqyLWwI&<1;Z!<VPy8GmD_MuPS#g@93OtmS~W`-T<W|p?M*uY_Ox$c-vqHE1r=?`vN
zj=Q$4oL<7lzwt;u<Lss?8E&UvFIm<8eEa^iNuYC4N#1U)?#%D3z1J_7i`%AsjaZiX
zV3O8Pmow(OWAi;60}q|qKl^pj_68Z#{t^aHJAL_|Ph~xdD$Dxs@T#QFbO^e$a(Zk(
zhe=@3DVEJb%jG898T&>`+!mMGcjb2A62I<A!B>*MUcT=%Me6(RW{*qKnqS?np4ckm
z5OF@(;a5fU(`lKCjV`svr*PcU40Ljt(o?WGfTdj{`Udw5Wt%?=kpfIdyPdxW>hHh9
z@KVid)67>+y9?Uvn0gnyIoFbqwPV$Fo4o<c_BSdD9Y`%y@I75^`~C97i?13C|Jf|%
zE_4X~eECX^;IWFc+H(r?_a9jBvz7VAA<-L$D%a_kb1joQdS$W1u4huG6a;=xKUeW{
zeL?_Bm_f0xQP%B8q85)@TtAA6x$4^co$ePFJNmav-zHzrgCpjv|K!?(ZHuSy-1qtP
zli$4Iwfr>R-}lqqoq2vJxT`*DFgiWcW5Jr54rcvHj>^(9?Ps4SPUN=pQ18F6VCR#k
zpO#8Lj-6yMQN?G4j_@2?-`>lc)7xL&w5a;??%SoxmS1o5*4+N`<b`rt9h+IrZga7i
zmrrke;Bhnj<nIu6%EafdK+0F1+x16(MCc1_jZ@HCbbS7_Pz!zQINs8Ys$Y}5&+UD?
zo#W*5$$zePxE%krTs*po;|m+l3HRW)yPisw{pX#TlE^*b_A}4*?dKW7t>!J!U;lCT
zJXr>pR~Na@z23bbeNLTs@@|2wXTRzr^+jwYB^RodUOoDy>AFXpa^>2~>bBRjpE!hh
zP6{?LjbFa9)i_CE(V3smo7?z*?{&Aa6cwoPy>T@ow4eF(ME^-gCaQEFGU3e@;h)es
zD~hMx^7!XFc1{{B-CuTHydC%NAsdH<C)2sSn@<_!8I5@pl#bsiP(CtQ<>Qk58<Owv
zE!*^{pwKe@#y_oT_Ro8K+Rv}9RcpNAn;*R1b?>#chCMMqmAgN=Pjjx?^;2GAhmiL<
z-o$(DDjk)!wdK0cpV;`W%n00R=f38o;qN8ZJn`o~ya{Wucb>ML_n4~vOUp3b$cGl|
zKh?#Z;9h;Cao)U1p|2O!_dD{$o<H4w`k$sp!s+*70rq$Pyf<ywyx;k&R>=C&>k@8t
zfvO*K`wc#Y^8E2VyYr{~j8~Z_j2ZUxcWviQxO3|EPG)Dz8aAItmrvb~oU;FOpVz@1
z;fC+#^%?utzZY~kWAl2ck?ETEhOF=Y#g;gURLtJ~`K#xzrFSjXdwlrJ9@G_6<PrU{
zzG{|0Qk<HL?PTR8bAs6}eHNQj&0XE0{P)Y5#SeP~cPBD0aNwHdD0J?$`>)D&vxO^;
zZ&Ywu>@T$7AHS5sqP0S2(*%wfKXvqZy;{m(_r!`YPn+Ppt9)!Kb3-Tf6|IbWvsJ`k
zwaT2@w}&t1DZKr1XJy^*-&eGS{HGnc9PjbV&vKFPiJhJn9R-tB?`SB$x+B5)e8!6}
zHcO>D)x96R?DU&?EBVR#bKAQwf0#9YQtcm^#Htr>PrKz;|4fj0(3)VH?#SDIK5&7?
z%fEXj^zvDTb{DM@dm6I$c5?N?dEx)PmkYdnbyY6Rp)ujBdv(O(Dk%f!xKlx~8s}eY
z2mHyIT7Qk9QOkS6tj+7^C92PSnfvYPhc}|L`+5Z@|4UZ*QO3Mv!+%G)D^E>*fBiH#
z(XTJe!tHyfkxMy3&iHYiL#v?K^O|i+)-@HrRsSYue2mX%==j&aK&|QD>OY6p#Vj~9
z)B05GOIgW}3Syi7*mrTAcpF*tdyfB)p4Os@D(jzX<k&qX9H`#);OW^)?x@-im%V+0
z(=P1H2@HB1>fbAO?1g!lU#iwSWtZiGi|W5^GG4CR`}^Y48FN%8`CaZxZ<*-yPWH#5
zXVaGLRCQFz%+lP$m?-3JHnG%i?yg^V#ox*wxmunX9R9OvhRp5Dvg^NXFKPUF<@7rD
zmRpZ6e6=yT^ltW^4vz!kC$m4-HJS@;Fk-!JZ0qhj?b6=1?@K3CsqM~vx>S4LorVK>
zQ*?Lf_~^U#2AHHouD>2|jOU3+jEaxn<yGk#C*OSKD+w)LefiF+1%akpckN?Lo;oM`
z`);j;uO@gD@0{^^qx+I|O8Rr+%I9gGniZ_#viHo^>Nd?yhYcS-FLFB`oF5*q#JA^+
z)9EZh{w|lMxP~ip`K8~)YnLTAn#-)~ko_ut%E2+>+`{tx?E7|j3zpBEwEP#h6+`?F
z&KWgT%xd-bG!6S7Go7<8y?DFsX~E1(p%(d{PtH53r1M<M`t9#~5mq*yw?)>SeB9f&
zyu<zNrOosB1$|!Uwnd4tJHM`sdE8?9@$SN|L%Vmxv+(5pi(&opg^{y;wPt9}{P0=N
zX0WYEOo_f1()vc~@hi=jkG_4^Rk_FX{|ZCdceUg4H36M*%a0Z8NsbFlu6$p0=5(s^
zj(gpE7H!zCrDwi8?ri#x1HbMqjk}vQwfE>jq2GB=u5(Z7bJuK&Gyfvru<T9vwYmO3
zt@kqeUrh3{Ygj&KdskcNoNuA)LnBLH&9FbSwRy$7#{WVu{yQ;S)QT>j?-<A*5u;jj
z<+I)Us@ButhM%<D?p;3a_n={BeCk=*+v;v{NB30gyWWrWVcqsy@9C<`ckaktO*kF8
z=7*VcZjiF?S=kL7aW!WGezWCVJHcJ@uVCN9i>Lg4zfTsp^MC)e<2SOU|Hu_jl=bVZ
zI6h55`A+Iw-djx@?lZYGYZX3V)ezwI^UOc5BDV3t^~kIB3mJpX?VnH{QKIn1TtqPQ
zpUz4~!3;0?nr_yKzAh?OX`FHQ6CBP|?UeB={L}D!e|+h$mIF(#`+Q5cZs_>tx$MDZ
zMi22Kcj0HRW-I!hSbfRwia4+TWb?oc`!c`e3hc>ESd=t1EM?w?&epFtcGNI2amrU(
zHXm>~D85?d!|&?{6Pljh?tK24?~gXy+&SkaoO}I0MQ{J*1=CY%Ha^)jHDHGHH=QHD
zkEK4~>FhXX`|U<ikk;wi-%mx(*~U1?C_mWvP`!P@75O*wefM~o%s&`(e)X~KZ|-^;
zO78Q@duOob^_Oxx`Ac`5R+X)KC+^ZxGnMJgyyQML^YG3yp-M%2B@P~WsudePb;X|2
zLmeF_-ny4BlT{N8_dHa~BHX@O<-V2qz8gux?e`lmx;)X^xa0i!?5H?~5C2Rxe5KY2
z`Ij&Fk{)AfSU7KY`YiFiSy^?nq9WXqg?an;wr6f`>g(%lTHf}2<wMRdoV{iTUs<<b
za9N@ie4NiE@IYU(p_*K1r(5W)g<3~1h;I8h$v}_m@glb#+1r!cyl!1HY1C?c`JPdC
zPR^}`rr+$-)%kXx&yw+vpQpeaAGYuDEn$`CFW+x^{iKaod+$cuyWg62yfJkCI755Q
zg@xHG-?{DIboE+j)1$R_dME#KSy}0?Ec*Y=N&BjTwQ*kJz1O(~ru|{$=zbRRD&mG)
z?D@;P7Szs4zx#gCidU{$iP<_Qt{A$`Nlf3f!iQ7om5WQYR;+yjpX%JAH*q>r;bE^+
z`L$NO(%ha~$#0dh{;<Y{_Hw`bmA3o=mfx5p%ajfmbNQFwyL4IP+}^k1%%1NOioT}y
zOKi9u^QZ2<;;rQ=>z@Z#UtApa{P-({TYumGVYAt{qPdE}Kf>>@^O5_h=DfFN8Yud9
z2+uh5v*xY1XxP^RhB6Cf-p<WYk2P)_`z+)Bc*V}6nR$kO^~=p4bj^ygIUvea#}IDw
z`0&xQ{-6EcT}ZwZb2;1Q!K@6ITap`J*dJi)JL%LbzQ@yzEk9t<b?4-WgWWDEAJ|pv
z=1L#p))U$<_q^hj;bwy$N2mPw`a0k9*T<)eN_Kusk|?&jTEn;~#*QIy{fe+-Ob?9y
zzRz5jI_q^_qHBTPONLdO<~fGyJ+rFQxm++SYVNYIyDaNoeT}~me0fDvQ0&($eqmO}
z?P_N*oq0UP<MLj1MV&Ois;b^u;@9$?aUY)<d{;Mp!@;^eEmiuB_vTl(ovtX9(C1sc
zrQFY&!~EkTwYWon3~is>&^Y-fZcn1$42uJ7)*q6X_f7uy@3C#3ciMNQpMP0(Z)B&O
zKOyj?yY;rWV=%`*hD!zaL{|K~eWYtv)Qn|fo5Uncwimdq_{+4|O!b{0m(#ssp*GHY
zA5Ap_78}PdUT(gHcT>H~w~A%r6aL-zpEjGNuY|#Um%>*AzJnTj4=72rUoyHjZ;gge
zz5CyJ+urM)__xZfGJDCr)9<*YK&|_f_+<y)91AfD7P-m4JnkF!>5n!ge$N*^<bJ=2
zJ*~3l`QN4l)>9P`@hQKW?Eh3PTfd#xE$FT7o&@{lE?vxa8y047uxVqinHeYXo9p+*
zyIGG-67A#b0-Ri{mzf)^d9E4j#PICu@*uS}E7R}X{>tY5`|Xm94M!GqUtL{Oa9VQp
z$A?xmoWHB5OXNnHzF}Zrvq9ZG`d;H<M!p}9rpTQAdA{sXF7uAp+7ouH(3IK0{H^*-
z$&9NzURa7%<l5x*{)v#u+0lG_|3bqL&*P7a{Xco{7ytP~`sNOYI3^jW^?VQyk=^(>
zSIAYi&!B+o<m1U_7C&t2QLK|OP_ALnPnq(bNwiVTW5LYN0`tQCuN~d4(AmKB(oLbW
z@YiqNr8V5!>lZLGg-n=Z()_SNQ_#S3!Nsmiwh<-Le7ktRi?dlwNL-XM!;<%&*1Ju|
z-m<gI)R>t%J;u<#_tcm2yQVHCJ5MmStr1dP{^IEw9+f_g#>Co*o45*9UR7D~NBil&
zQv0PgS8S8#6pl-ug<iedu4>X(oiRr&{M_ejp<n$sS6-~G<DJ%D_46(phq>3q9fzmv
znxode<g?!MCuI-$lGU?B+Cmi5YI#Fs&epNMdj4~&c|^eZAK#>xBwI3nDG&RS&O71u
zlm|s^+(owxQ`2hrrJtCk9$sUcnkL7>`g6_yg4$iG(_A}K*Bhx?{jWWsG-+>i$M1x#
z_wO?<*<&v3_Q~4e{#^c65rh0{=2IQb|Eq;sEPiDL@4epg{dbE=6-(oPmIZa8{pkuy
zFZQ1L{NlglEP<BC4|y%>*k9exYFP2h{=_%QbLyYtBj4rEDP8i(GWxaE#JwNxKmLE>
zYhck|pTCc11}R+MuRJSI-MsDdOs50=LhsHmaM+huS(F@h`0q)V4Pvi7lDmGG?OV#|
zS?rzo%GGH#=M7J$vPm(u)4MOXyj5|mKDD~-NbSVlFCXr@_H%78_N?);i<-<NwR%$b
zwcjbF{4O64f3Wnk?fdD)RX_O`pWxp+9D#rBr~T_&-4wLMZ}Q`?0<XMZQZuqt=Elx!
zh`6*_o$Ym{Zd_9mi~lv|ZC9S|iT#?iZn>O!_O%=NPhQQdUBAqHMv%V1*1Coj`Q@yP
zzuz*w+9{~?DgQ~d>x$ZqHB0WQq&}JR(KznxYf}SoSEqwxSMaS1;So&-s%37?6>mE5
zdvdPRt6D~Al-^vPI$yER<xkqpjuT!pZz;ZiWg+2lV7<ZN{cdy5hjsq>bv90*Z0fdB
zH&K>5%ocytAADeHpMT(CdO=J11CITSeV6wfV%O+Vsqm7ncAIWL|NPNS`U-M-3Vi|J
z&3bhoxSzUYb|m%J?Y#?mKQhI6M6_ET>%Ls_?(H6n&h+)FOr5EoNjDX1-(=RkWG?Ev
zGcD!e+sc%~U&?EEX9f5FcVbR)eY$OJ@vK_~KYCVq*DYHuvT1$L%vo!8c*%qqUD?{V
z|MRBT58NDA|JhdT>~c8y)~$+P?-seAwOxPNWX7^HSDv0-RK%}(*I#bqn;D+#e@x%I
zGO(g=mj0~Zi#Df!$!H#nYs@;G8W?Z>$F3>jdub)_wEL3JqIuh1zY(;4#aL{vn){2v
zd6hyO`;m!E*WU$iJ=OmzwCvo`l@IIkIx?2;*_3`}?+e3~#~XKcr2l>Ysp|2>n|;o^
zKRXFrzL)x<M2AOnO?r!HtHYy6)u(%yCZB#E{_oZC`KAsFqEvm}7VU^@I9$)fr(f`c
zQ^t;OSJRcxM)O~<lKZCMc*`<xw*C93JC7IiF7a^G|GabF%4_*Ay$(zM=b!xViB8M)
z1AlI|2zLJac;vD8>cr!MYo5vM31>Ikx6CF><(2SpH>HHH`yWg5bcjz8YJ4^0)VG=j
z`y(ws+Ab=!{T1HA82Rq%r^yed^UYyRe8<rArSr~4tv$&{gQvf5`m1}o{7&7D7>=Op
zio)-9&D%Hm%2&O`t2<NfpE2ruvrK32<<mhgV^dz*o_l*GneSEM{;JNT=c4=Hyp7Lj
zxU=HS!D@jyj9n{!@Ugh=T4j{bapQ*J;uiB68?SQRIm~_YV~1??x?9&d9Db&@R^@%Z
z)ih<-{z{Ad+j)hd!K+^~tgD(Q&3nIbwV0{Q23M)mv9C@|+7*|&F(&Ttm!cQ_QVOU4
z9-nky$vW+1U-7^D+#SjuWhdqtR+yAD$W{jGTdaTIG%x=^<BA!oTJMU^&3oQ<=bPDc
zonp6XuQM(0S_GWPTEF4&{Ef_s@BE9`DY5R0ZrE?Jy`%Mi#lM0jXLa`8np0b{b4IYm
z<iExJjuL`w0fL`?-G3?NEO5MPv9=4Z{ewh%2G1S#%=<0QeB@@?8=Q2NU*X>?ca=kR
z0*|E6HO={9AF(UL$>vf^g`@BzbrZ!dwI!WuOJ=GqDK}C4D=fo&^P}u(<C`CS-yOD}
ze|jb3rmTr`tW;A>1HaCgaP`RLa?VKKI9($pdE+P3?v|ThdLGJkH`)IubA{r(I1?w8
z+Nt-<H`g4wY+3uR>c<%+RVl5#$#(ilUt(15Pua-ea*+R0rjnZ6!QgYztuub`*|=JV
zetI3dGs5Nk)ZOu#MNdMd?r)FS>}uL4sub~U|7GbjD_^UaPkU?sweFrE%c*%Y%Y8mN
zEkAZ;g8FZ6AK#$xiC0P|t}LHt927qBh-s=+sd$3(qpInpS2FYRX3g&Y<riH1I{SiR
ztEgw({e#W_C-Y8_?M^wKto87*+%IXNP5B~M9y>%VIdG}8&edvvcxC3^4Tt47D90qa
z|6s9h+Mw)nLvWd8n|V9e$2TTlDiu32-yaZR`BA}AQ}XNKEIvQp_{+JbR+kS4E6?_q
zIqlB#Q)O}CmBh}cJ36?fChKi`cP07KtIH9#lRVC6%!$+YxVE=mDNymz!FrukwN0lC
zx#MkWzx%If^I@`5b&S)0HrakQzfz$8Z=a-F{0<YIPMq@Hy8noAJzI!SM%AImpOu!)
z3!QEF{AFuq-_Cu<I9TS?r>IYu`7e29d-whW9Yu``pEC2f=FR@>xPE?;i>0zm(yh$A
zueKXsth~Ez-m3<iMYm4#TzY<0+x${u-PcDZbE}1&u5l`6{}j;q-^}K-&FjD%;pKYi
zUtD+8Oz$q?d{uZr;9&7-$J%8MyX6F3SM!_*_PhDChv(AI*pw}0?=~nLe=^a|RNwsU
z#iiG0G-a&1bNTnY^E0~deDi$wa@C#7wNmpky`G1wF5k;EegE<a+nwvLFl2tPKNcY9
zyZX$9V!l_}Z#bB(CphI72XI>-YLQJm)c1i86yP!&m17b>fxl>jvfmBCza<kM%`AB<
zF1onthQHj;j~uHimYo#075i!-)^I06RVu8rZN0-(JrBL2$ClBB%0BsmOFZj}51&6`
zw74?+&fc3T%PvpwO|z==e7r|~+e!B?QQPHbeKG!5_Po-}FvGlc&B~`0FKp*OzoxRF
z-@zm<;hfa27`+^+pfwA1ep^oCFMqbpQP|aN!;}wIzh_w-GP%U4@^sIixbRPJ@2tqW
z@_(Ih#H&wiI$!tHY;v9})i&KWw^(`6=I<r<o@Z~!>5F%_S@XR1ZOp$b$-j&KEiC-^
zH{IUh@{Zr$7fiO*=Ulk)Kl6ne+a6Y5Vb*n0rWYTm9y5tNDE91Ix3TZl%({0;uj9_I
zX+FG{xxIVVaXA*>85Ye)V%LAWuwupWjq`mcNYuw)m~l7Dc6Bb(F*%ceNw5DLy*9V{
zj{dTZFB%R1@%Co++<qp^S0QoCNve7oZ?V~Woqap(v=@Khuqtt0mG<TPUwPIYt11uf
z6o0br<0Ii6k0j;wBz8ZOyL|P+`u^6BnfLVCE#v0AKQ{Y*a!68XZ()9dXW}pGdCz}t
zI$pn!aZ{u4>vDgIUBb@gNl%{0J)8Me=UdHz#dCjb7S1nAI_{Tqqmy^C+l=R{_%>CY
zaXlZ`;4Ujx8gV5tuw8Xk^CG$KWnlK+Mdzn2+@pT;>#GH;j<3=TD~-6C7}_2xdu>->
z`RfLLKb3o(ySAraF@Cx8)y%Itdzrt!IAT7>s^ry$mCnog=3137DXE;dU6|z;a<+4k
zWuUS8qNgrf?g&Rc-SPKB>#`SD7lyaT$+DG7TuBrVP}F?K=~}o*;~nS20wozgm&2V2
z78g0%oo48p99_Y;*sAB1!K%Y6_>$R^Li#;}eO?)?JiLT2*gf~3*%kl5Lpe&GyyurZ
zUzL7s^NYyJXRfj-UuW!{_sU@P;Wd1lt<JnMn0~LZO6L-TIK+>xAkT)C27C%DT_>*h
z^5)6vwp;UBH>kgRx$$U8z~yH@ZuH)W6~FslZ0>|ohfg8`yx-q;uSmaCeAW1M<d=|k
zKL?XL+x7R%gZuN<g{k)%tG>)y1oBW->5Ho%pMe#bxUT1Gw&HnZu;TCnzUGFE9r^JY
zU5*e>hqMRE9^*T*LT>RgiK~eqo?EHJ)x=j0KCcWjyF6uI?Kl1$Wl-jn*mSO3V5OJU
zbI&I=S0A2wB+c4wGWUhO{;y{zzHxD`R*DQ2S=Hqfs#93AP<&6^2PdKR;=M+pfy=Z`
zg#-rO5O<ViDHXWV7`Q~sE5x;Nl}5nZNgkrDE0t72S(bD)l>Ax0=I+Dyx92@ucv{@`
z{3+wp_hwIZvt_rvx=!!af@8Xewz(ISI2T^OP#ReCf$OgBokf?QmxLB*a8{<-cF%j2
z@%#Kr=I38qt%|H>`N>+9KDjb6yjox4MrM&v+T15$?QtjPIV!XH+OBB6V9g<AxMJ}N
z@kMbRRf<_{fp3$8*aL%mQl&0#?(M6&wQTPhubBCYMr{Z6lJ4I>*XSMPRGu#K%=Bo=
z>l5BLA0A!0*Ex~p>PO+LA3Mx=19;u$xm17aDM&ipz5f20J6XNAX4!;p@D1GVl{kH&
z*D;4Tn!-ho+4ptvuUL@NulaDnmvl+%M=PZ`W<EA%Il}$j`fs-E+7k@Nl$%S;9W`V(
z7;jyEbIOlJ<}0)^SGT#!Ml6k3xNFV5l7Q7qc<mK?4U9O1oAq)XB_v9&a&d1f-520~
zyZ)8Iip3MY#C%=zO2O-1XO-fr#E`=)BwciKUG<DA6-#bfc$o1`>TFJVq#zk_^N;%{
zSI$}29M?7rv2#vxWJ(ce<eenmd}NoxN?nVKM-&n@R|I}|ykDcjrB;6PybFOJ5*`=F
z{g&M_g<&?oqduFi;|Y;P=bQvib3O7%nzBsN%*E@~j4Mjt&b;B-wQ_;dS1q?2pS`~M
znB85uaP@S>uQmOj4)4<t=Jybuc#&aS?&`EnS!}CC+7hppt#x`iH8_2#z}$I({#%0|
z<?2a%s(SlQ)LAdC_0f?#H!r`gUe9;nW$5o+S<>OTkF_4Gn_wesC8o=I@OI2UpQZOA
zLY4J=g*cVc_lhm_lwYu0*Km^iuD1(T-p|!uyhPw{diK3_mwxnoUedKdF+yiohT6*+
zll_u}`hCoQK0A9dG1M=o`rH3S^Sl}_ulcz8Qe3?)XY)q=kQ7V(s#UOjs$M$bYVx_N
zRfq4aHd}po5ns2Jj-#x|*9eoNVD_sG(;M$;v09Y|TutT;{^g@u706NEZ}Mv1oV(tK
ztQ?x}t^a(gGUxmPXqi;a?izo^_t?Xgpuz~0Th$e2tvtU^;F)FVi(l(1HkyI5s<=R}
z<~jq{d9N<4c3#i7H%-Fiu!4fqCZ;8syo)T9WLEnvJ;&l-Dq$tCxT<GX5<HuRv~OfK
zTy=bfrdsKYD~UnvoqXz@HVkuGxA@<&_`Ui01V5$+n`*zuJ>P8h^Zw6gF<U~HRcsSZ
zjlXv#*0AJ`on*wkw>9;OS(Dc)^!*PGy>_2%|J4KQV$QE$EIcpXW!ds?vGWdAWH_Hr
z&*8ZIg*9HH*=nJpsHDNZ3y+j?-81cXuYb&>p4ag^=Jxj03uWJ2YFbtN`uBcOxr(d{
z%Y634%+AO(*4xInDqzdzrA80p_g`T<w%Kz><CNcT>#wYSn%_K|t#`FfZm{g`jO{CK
z$h{R$-)~WO@$9*eleW*dv9GM#H2K+^6_U!a^UC^r-X`k1{?_~?T(x?=`)}2o`knDp
zCTV`Wlx4s6&z4V@v-DGctA6&rc69ReuD>BaJolYF8fPS8w(V|2<DSens`(8c7!K)Q
zx^D9}Dt|u%hkmMY!w0sGWAEOsTJw8_uZhX53oHz~Ej+UFh3<-9-Z)>$-t%tC>0d(p
zKFO!_g}w^wOZH|KIseXls@{BNI?u(rlYb`tRQ(*ZaXy#5=fco8OSATe)LnGlb<6u;
zM$BY~eRW(fqI~2k>pYd;iXYdHxxGW)%s_O5+O27~{MkAZ8aNn~I=(PL7z`Z;{z)k?
zHC&jh&HxqRm|QpSAcG)-3xhz66pW>&WZTTB#L&T@z|_#e*1@5m)L>xvN|IMV(7}a4
zkkQ2iOtqYhI-k+d!Jx#{!J)8-5ls0gRJ$>|us{?-)F%erHkMLQf+&Qj=a~Fv-bIi>
zAcZPB<QhB}4jtK_ZsefCFp<H9N#G=eQd72@%&5fJ!J;74;KA0xqOc^^-16g09s$oa
z;d{g#RG3|u1UNwys>W9b{lC=;b4_kkt#y3dy>96#SFMF9f2(T#rny|a{rL5A(a0jx
z8{2Nj{XY5eR#$0;spICm47c+=W@)eLUhn=}^TXtIxoSE3Zv}hxZ|6(IhLyZqotwDZ
zdfWU-ao&HXz1cqX{*Eu^M;(5jys#kF@V0;C`qaA9-=ZIGKa|~&Vme_>LyD+Jn8PM1
z4<m(>4Jyo&auiOklKRE^nP*$dhNF6$&gYz%)ac7*CV8XAR&FhG!)3X*<_VK|%_MK6
zSa`Q>uCbMBJa;V0XpQROsC{y8#UJZPo}2VgC!W9hWQ2Fy)|80T*BAdynxlF+(rC`p
z*NOU)*(cqnZ+$v5(Kw#JBc<8#p^oUeNp}y<KYA{R(TerOjkSM%ma1=`ZsEO(HR6`;
z)7R7PscdUF!Is0d;b)b}4$+2S20uoNZ!aU?c!>6eJOmLNl|!e*uG-7bCTg(4!Q;RJ
zB_6Fg`j@$Fy4S8M*uzzF%!KuP(0}Vs=jWSebho64c`RG$IPdDV9h**TG39klOkW*+
zt1fA__U)WqlhRGUta!7jizm!w!2$0HXShz9)SlWbB^IiY)|xb{?di=;e%t2XxPR*3
ztV>H3vmD=j?(n+$WXAgSMoo3k#Pit1{zvz+h3c`39$HYb;uPbG82*N~KU;m<KdbsT
zedPX7z|D|(zWwi;A2(m7FP|@~UtRb0SLju7@%7W0?b-jzd?|kL^u+qW{ceAyz7{=t
zzGVL7I=?-8KD}NQAO3gOtEXqqN7`$@KXK{6Q?ZP1OsZSc7n^!dH1qg$G3e#uRs1`5
z>K{1EXs*7)QqtmW_c!B$;-=ZccVr5R51i%9JNJFMtvxeH^59uUbM`xP9<*7m{NVaq
za);%?vz&Q+J1ieOV=>?UZTiF~vLKD8t1Lfjf4Ly~H|p2v*7=Mxgb&PNc*Zb;Jt2<a
zvuuJ~LnXsAR)d*kZ;OA6Hh9j;_utENV8V;JE>-F?@7ZmSXPmL**S9OD8F)_K^W0KC
z=`BRrE{L)Zb6slGCGXo6w{TY()V*bJzEJM*%Wu)X$!qfG-*)9~@Vr&NkeR3PzT!vg
z&Od%e`)l*|Gi`XV{q~X_^WHPouUma<R@WDY;N#(MQ$Ae3wqoVn?@P*RHXmCXdON%>
zwp+yJ?Ub)+t7rAT+spg#&HVb-;0407X8m}+!ccW%@!QjVE%95u3vSElto$3iIj<;s
zT^f7DPOF>C4XVufXDlhdwbm-<Y}_=bEw;uN`u^MU&A+%K>gS7Po%c+8FYTLmtNzmS
z|JuBlf7x8W_RpU~>wh!bap9V*3tLzFDwXig-S_0g@|6)joJqer|KANdw0c(6eR<^)
z-+~qEe|0<zRo}n;%8q+mJ@%WQ_%5~NKI@P7CLXW*cSrp9e*J@8YEAg>t!5_MZ~s%?
zt~@8cbwX{ulKbU5@&;*tzZbOY_to1^>y|%s=Wy)zxAQA<3ZC)LI()VMZ|iNg-IjY^
zJ>7P&K9z6#gqoeRkGFj9Pqqs?6Scq9VE*?H=^5|qOuV1p+iFqzFiv#)|Acw#U!U9e
z^3Ytq<J)%?)PBC7{LKD#g8Av`%O8K2j&brTUsTBZIY!~u+~kMT+Fu9$?|2+|nAP6&
z%O~?k?~ng@@ML|*`nKu6FJ6n}ebqB@TYs_he;MNsmw)ULl1?)*m)?BB$2-S&R?x~!
z2N|ta_wpMNyVKo2WNZ&SeEaLYg?BTv7Pe<LZ;14Kocn!MO4qB1*#{M8Ts!R+xn{q6
z@QoYcFIGpjpA50^t<QQZBwYMr=e}DBoYga)-Aa?aUVQwFkZ<Vw9WMhHg<O1nZS&t9
z3sNF;jxow!GEohdDZRYXmvwc{<+U&NsvH$ID7Yk|x>;c6#1$)pzQo6Tef9NM*jc%)
z(bYS*?_j8wH(Zze!7N%N(e1O@m)!MRUe=Y01`8jY6xkb;_v>`Vwzf2u?CPm+@7FK=
z;JiDm`m)RqyL;a&oPIB_oZWKzu=JihpU3@AE7x3~xa-#f`8P(gW&8hr<+we+@V9nu
z{o2Ke_x|bJjxYUU+g7>5aDue*PQyvv!H*4RNH5-LI7>VDvEc%F=3kkKmrwnXxX{n|
zSK_??>d(mz_H190_Za9J^e`TlFzESsXv+ZuHlE}I2FJKBGaZ&N=wo~=q3_4tbNOB`
zl$CtIf~_ZckBP29FXLqigWivqwj3~F^GQBna?JZO(`AVQS>|=gdrG4oSh2+%Z+P7@
z?=sVMLolbJGAcoi`I%vY+~cR2jg>qzjyF{HOkd9W%rHTcd75F|OtFY#4XHd4$0|~z
z5@z~6kl4`k@zs_CR%~mM4_F;reVOUC#D+e`*AnafxYvAMIKShx(cE9DQOU|LAGcfl
zwVnGXb#D9&zW+hX_Aj6K{Y#ou)iMj?lQ%EUnfc;ONV=M}^)6#UIrBid=jlP_`c+DH
zd5i4wl<f1C+2<)a<SlZ@Q*z8(=9s7Cq<1EzOsaC3bDok*-XfPgCD*)Vu6assd5hfg
zl-%={x$B)%sf+sJGU<-b<U1}??zl|7<1+1z&-6PkGw!&|yyG$}=2qeK%om+jJ3DQ5
z&a~Uv>9DiY$@NJ_{+|{9e{wv2{o?ri)8Epicy%K3lsikmY|pE|R+_c9hwtpK%m2@P
zE}Rv2;<AXkk?G#Z<(>WOXU`A(f4R4Fd8U@|%&6&^Qw%@a_}lmh%L^Y-yQprrcf#?2
z;OD|Ci;UM632!VC-dZHQv&eXF*K3oxhO;|ker8Vo<FfY8of;wOw4|r%Nl&NC-hcW2
z8Ax>19~Us{3(+lXdf>C_+56>Be?_{@YF#pSib=Pp@k*)Mb48c@0%oZ$?F{(Y8Sryv
zz|oCY^Ag(TpSaR;SIpC9*UVY|i_f{7@&_R;{*Y6`>wa2rw@wbRn5eN|Iq>w1IrBZI
z%-%QObIN=V5c2=rl>RaE*m>^Dx0dp9UcU98)%p7WWe@M#|CgNlWB-){4T(i7CVDMV
z4vcdC&+fz5=UBj#a%T3GLlZw8bLD+}`|k0X$5&J;nyuYy9Iv?Uvwv~2LE*xmCQ^CP
z7cSQ`N##pNthx2SMyy`?Pq>uCDeK^8OT^fw%oEGMFh^~kSo?<)Z~q@<5?*8a{)YFt
zdqv-7Df=B<y4+^|q)z{nOHQUNirh4zDdi#a$rS$X?(O{Fwr!ujcki3Gslst*&FyDC
zu6<OY*6PJleY3{ujI-U$-_=IPzo*2#J!!(18~Lql%k;&2Wg^$@m3jO0MAzcIvTxsJ
z=&M&--aLIW&hJ}M(#2=VNqqb#ZPrY-3|61qczLoz-_^Ld$S+y>>%V>DTJ!el)wnni
zaZbnT_U)VZ&V4J|HhuHnId7ll=%<U#efu^kU%YzfjnhZt^u85soX*{?_h$3n*f&qJ
z^uw!n3e8N?4;PEQ%n`QoTM?^wi`s*eOpA2Z+*Wzsa>;n%ZJ!fAI6EyL%FVdY?lpJL
z4hHKvyWcvRKbRwYj`L+p=>l1;J8dg{j&tX~WV=(WFrWFi%!~Bvp}$X*I5@KNSruGh
zeQ1A|WiErj#l}v)kdg<Bn8jsJyg1+{&N^pB^N|t<Pj)}6f=i84_|}wo9$++-o$=Da
zmp#v_;Bw;}zBeEtN71YjZ%vtu%CEDTCC!^2IUN1H&a2|<?-%SL#SdmNU!SA2_<$Mv
zuH5w%0loHze+bWF>Z>f^*}rdPm&J{z2le>NDmR?|^*t|o8vDjQ9J4f{)H6#M;taJf
znh2T(nocyGaH>6Z{{(|He!Y+C>zh2}T$H9&>b2`-aappq*2ElldbF*8|69d`<IOg5
zPd;X@xf`mhbxzsx#mv*zS1f*OII!8MrykM}Sl<!du%mLnrU2)8%c=^G4*uPlcQ)Cw
zca~l_$UP-3#ba@^(C$sl%U0i#+y3_1Q^AIk-AZ-SF3I`LxluIh+lHkMuWCw;tMPsE
zdnvn=-|NWg_~r-cNsY}XcNl$0`ETsd<Q$>a;9^jtbh7E}2UX6Ge$Vyd@{WJyJM=b2
zbGmhg@2S)8-icR#ydUrUI^OwpwsW?r^Xjr3W1YYE4EN-GST7wVJ=b1v)&8<;Z(r{(
zy9P)94#iu){_;KE`t_efTjyRqxaHRTx<5ZT9>2Hv=kxe{Y1W+M_f!8w9-l9!W?7K;
zD^cpQZ}KjS{RX_RmlfVMS*QJ(`|^c4saxl}=Z3#tzVNTfmicbqK6ioWwr`hDB*vA@
zXydzlAyLZmOgEUe@=<QDyqG9u<>!1n^NEGN)wyaLUfJanAMbf#!7ICb;bW7cS>kh-
zExc?}GRw{H^CA%4=6Cr-W}L~*wqX7l|E{?QSLeO3@lBq4v4-t{m2dLgWgGn%P8c(M
zVrj5pRPa}RF3C_SW##YuJoAYSvx2|#^voAVy)n;X^)FvYmD=T_JiYQ_s@+`>XXm-@
zb)PkVUu%=G+y1}oVXxb^f7#Q{|L5j?|L-S%WZl21Z2Obv{`J>;dtTbd?YZ{s&&fts
z|HYU6{y&@TU90bx#+7UTzTfNnMUnaHakGo<JRI`3UF2`@*=_H%+aO<)CtRCg`{Rz<
zuLETtcKUv4{Pn+h_6z3Pug}dcvQKM0pSy@Z<l+3=F7gtq_9Y9~p3wM}JNw1q(wg7P
ze7`6zez84w5x=DC{#=-hv)je|+b;4(U3ItBeoYAcve)-ZeChRxGkwL3B)6ulo5Y(v
z_28`LOT2!YHk{m|EZpa#6}ipuW}bzC(cM#v)RX7DN#E2{T0GMy*67)(ZC86MU7r~U
zTFu=5qV4hJw~?E4>hfOKmQ0?rrb}i?p5zmm(#;Rgz56}kVR+g7yu!)0Z(F~9zj!uQ
zM&;(l6lS}(vv~F!O|8rOReR!?hVe?C14gB%*mSI4+~fH(K{g^SVWvQ;LQN`9-HAH&
z>J4v-swaQ=-^aM$TEtv!&;Dh5_b=O<efgf%nd*B%9h)qJ*F4#;``!G`C;wG$l?xZ$
zZa3TicKOV&dt|TV%|ETovuE|gY4dho{MU2(!UXA+TYmJhpW3_T=HEuPhwo3<$N&5N
z>HhTp`~G}?x?lglw7&O|jkCq-b5|<NeJa&{_4v~C>N87xH>jw^ot!<3C6(*{GHtIt
ziuMYuJ7-jFTj{jPr9J(kylcJftb)XsuYJ@H$IMDN#_@LD^Iw0WKKZKccy!j;H{$P0
zGiRQnP04akW~b*Z{lBt{{o1ywn3_A2gV?@J|NPhA`@O!vQ_*<&@Aq$>*suS;_UHT4
z^?!;E7N1{fr&ho1`T|}4)qV;}fxqU9q(3_!c9`|nrv)7yTXRcoHfi`B+1MYXaba5f
z^!WMp^ZqYdQU2nJrPv(9zn|aE|6ld^eO}0>|2yXI|NrO5JMMbxAGZ(Q=lgfQBq1vJ
z)4JUsm&@1N9r|J2XMg!e@m#6rAIl}{uVqfibf`J{oB#5RJgLh^f3{{unD@S3Rv0@o
zen#83%NOe6-t398=Y0#xqnqZ3zX8!(=DYEIUUb+5oJTS8YN3>spYrj_i-q?dXTGrL
zm3i)ZJQJKpCtlw3!h{!`N73`@7c1Vqe#+~2&$XM=QabCDm2c|Yv`-PsSFX7naxGKq
zwaH}ndCMj(U-=|c%kqrAkMr})7dE|fKL0snQZlbi@A8RMsh@$%7M?aKndzqYc@c>A
z|6x)zQ#^M0!q+AxbKTZ`UIe1s)?GeP8doxR<>|gXXIOG)PVbHX_3g4x(lm>DkKNPL
zv^JkfNz?q?UimXmttij^Nm)pqS~0)5u+?_u5^FE>$-CNTcg_W&m2*E^JDg!Tf0pI^
zS&{QUWfjsEJqMvx&;RfV7(XmFepsC7{;s@XM(6z5o%3g}oX-tXe&H<3g|i|Tj>;;e
zDZO~q_wr0}hqc3*mJ4TFE}WeL7EXGRmh>Vm>%}EL0b|D!5OOWaWbc@%u?vK>c9pVs
z%rw|FQ()Ijp<S|?CC0iX#)c)vrX|)zFVd=Bq&2-r>v~b9a`CLm#j`FKpUxH7z<k6<
z{H~A0T_4H2eI~nRn(dltv1_K)F4@Ww<JuDA#y|EN{QurQ$Y-y&{_(!#Lw(S~@8CR9
z`}2JxN+zlOS^xc$dFs16-W#vw+@C!;?`pa4#~b@~?sjeu?S6NE=e-N3T)yXTU$t_r
zMah4oo@I44(%Y9M*35LfY_TtP`?AcMneLZMYofO;OAcJxw=6ld^yaL-)T<kH%%_EJ
zPu;n3jn%ZL$2Ps%2I3#zbSEP0E|?!NP5Shz+aSK|>7wMwXW8qwmHsNzIir91`t4JD
zX77{r{#^H~Dtb2m<?YFZc7E~st3PM{D$_Z?{_^$PziQ^~ljZ($nd$B}2YcNMy{v7S
zjd{@tGsIpTV>-OWVY{w@4|hp&V`9{T+hP*SSa)qaV4!;;mz(D@)7=P=s_h^Rbvg!q
z++S`pJ>KGAFD9{^^;d3VVblV7?w-p`f43d5(7n*ldMvZCE;?b3*o)&#kGCAypljgA
z&6C`i8I^EDY{oLywv7i&bPaO2eJ(Q{j!2j#Cb5lmS!QFRj)5;X&uym5TMih2R2|C&
zX?S<vF3RA_wDWe~OSj)x@h`W#IOwrk$l`r(He{qMs^9LY)b=k`|JChn0gYn6j&1V(
z7VchWT06OF^_J4DuU3m$9Esm)-gUd0H-E~?<NnWYF-6AwzJK`k{5MlF5~7n1bKQFV
z;PsSG_1-zchRjDi=jZBP*n4^L_1oIB|7AFI7M$pKs4$6fd8P18{rKgY=8x-g(*9<!
z)m9&Twvy@I>P6GmrW*u>?rv_}Z&`XveRKZy_qYDt{IC1q=I!rqd48y$_{bprXHnpy
zt4bUHMHXIdF1=ABALf3R(OccX?X#@H`!1&WX9E>xIXd|A`h00Q+0Y=9zfI=ixh#oK
z0=1$~O|}U<WZqc&^XvbV__f*5Z?B87NUylbJV`M?$!oRb_r1HW>HYr4GJ#*tC;Mta
zjp~~Nhc3ApT!;#s;ASfQ{)Cw6%||@a-%pFx+?0!du(|zeexBQe2^)L1^tUcMYNvXe
zZ}}2aZOd{ce(r$nHNJ0?-)gXIs{2+`b87;Rec6hhEZ3s&o9<f2*ndxSKKAnWV@K(o
zU1viVl`d=kp0RLQ?ro{nIa7{gyy(8T&-PWO*u1Asol@C<ey#nRxt!}#SWBZe|2gR=
zIjmyaL|%xr=pS1hS#NVX<Ak(#e~1ZBzLH0xluS|cM8ye9nJc+xCG7a#pLp>$pNflD
z&XhSX%#2d_mou+2x19C&#xncaJ6JS1{U%LOk@OUq^y2e`4ayJQF7+%C@6;CZSLyfo
zSK;w!LW|-=C#9AYwv!w}T)$F;|8|`1Ucda)pNn4MhFn(BM<=|<N%fm)xz#7&+eFV)
zwW%-FF8eN<?PDIPaOz9&i{%%57tU6ir})Xq+*LsfB+&6qfBxE%R$r;|$ljf6-0Z>^
zPq=9jY<6yn?Vc;=0?n>X0k{6EEpMN`8K?K{)8ZSqkKO|}|GW3bym^|WFJ6(far^4M
zv2UM(y8*XOUyWO5a5O7Fyn1)n;zL2=6|b0rV;`*YZkeiQaE<k6a^q$_gZ|b$tMp~k
z6`v1xT6f!a+aF#Kn|`5$d4Ayu7XIf)AE_E#>|HMHZ$2;2=I)P0;xE+=<~o==E@HaN
zvWv4sSVfQJ7pIT<1>YsJJ<KQOEpp*wR9|+^MOjYi{)6fFmlgCLo!z6%u>MSHs^@AJ
zg=I{WtTtR}T&Bl;AYm1AsO*YY2SV95S#7x5xDF(=hB;MM<Mn|^_Dfb9t~G93$8jKG
z9doJdiq{8X**{rrxZb#L9oK<`4a}{w8gC9HvI|vDxY2w>PQ|SAq@A1G1vd!qo5Du+
z&b1D+YNtr}n1k519&a3z_=TeHZvGZHNf*R<uUd4O7sSxcNRyw*2V(KV4BHlz`fcLN
zh9hz+V1sNu-Yjll7OUO>w&`E;8dFK3otOMv6dD%FPRn{ZOYHal&Ht-^*Khdm{@Y%2
zZdkSA&D*-IjdzupFX=8?_fwDc@V|r;@&&<MZNCo&*h@TXkB<o6rso@*v1seuMzckl
zjB_>r&dK~aC-du^l^^G<d^_jmOP_Cf`ZsPbt1MoYRlF>!c$s(cGPUAmV#UkYt7la7
zugPB_$8)(p(f5D-n+39W)Hr6HkbPU1Y#z9~qwe6YzgI(;-tI8_xPI@|f7j!(mv6b7
zK3m}4y=?X^cdsg+ym|YgiQpd3ij61B<9TmS%sCXBbDeK@=?&Xs>rB)(&pz?*>IT_^
zYG+(epZ;jy5|A@Fcjp&fcK6A$Z|`Ma@9CIwDdqSLp^l?!X2;Y_qROV+(3`e#CFedV
zv#_fn!W%VJl6LDISChN-)p8a`s?e42!{7Xpebnq{{<w1FV8*7&oqlSTwVokw=N{|u
z$@?Ylm?EAPoM-VbfBXAoPcy;`fA5c7y}0%>$G755%Xan5PFfPaH_N=N`sBM$Z`WO{
z+bTb4YtO>?OU2RlUQ-iZM&07R_{-<p;jX9yv)pF0{8yWOc$d+k`ganrdK28hD*Ld#
zIsV;(Sh=uZ#r0_>@n^mEC5grxHm|o(x^et~@+Sq^x0^nlusN|~e@I-(m&Y>vy9!Gl
zzc?0Ye$|Kjst@PcnTu*yq=iR_$@R5A7pzF=jM(7vCA7A)<>aSXD#5k&f9+@QdGhwX
z{>xi>Q{w+;|IFPl7vGfid;iwLZ_dWccBo~T&b+qo<V10;%%X6)J(JdyUjB9C@U89D
zk=1LieV_k6COCJi>J8sm@mglr;Bb{@-v@2Ch3wZYyRg-IUF{hgogbT)8pizE|L4KX
z)!XNt(O2Agd!<Vb&(Eb_8IpTqqt37W$1gi!zw94qL*e#8`x}nQ%WwXend$V}iK$X_
zO|ICkJ1Wf`?oY-4Yfi1Qzu4o<cYk8qVa<w)NQ=x^-%7pX?8`Up=AN0ayS?)Cw!1MF
zI&+JZIDZPA-4s<YNue@N;j8sKHSVj?QB^OF-O^Fsyq#^Ixc}Bu?sDduby=M!qLnsJ
zOWl3aCf+!+a`S$ceyRCe&rUpbKHhk9OZLRBtzj}JW-{$jeIHjKcJkT1PhXEt51FjJ
z>isv(84EvWKiFh8)qitT9cN7LCf+4mXa1R$yKvn!{li7PlZ$rTI?#D!7pvvk-sc%j
z!b^>(?`l_-$q#!sZS5}IEl+=Zdlqpy;a1w+Ggs3jHgB@HShmD=_J<VFl88fHbH8P~
z@!pWUJV#X^KP~(9y*taf{>8H<hu!vAo!7G9v2NgI)iwF~CbrpTHBo2pW$#;-dq!Ww
z#O9Mk&HinNboTNom|j-z+-9q}YyXn#?W)T@hm>#~VvtdJn`!y<qxI&w#n(Pge-q9$
zPqR&%X@R1Tnr@ZxS@zOP_qaslPdLauS?Uv5_o(_+A<vPrQ&$#CuF=lhz1sIf;>ma6
zhbNfb<Jus-&b&RgpL?0=!~eG$^sj7pa^1!r;@8#GxNw@|mB(T)G*<`gd2>26-D&5N
z*^`S>mSn7$y@Tz8s_B+}X#t*Txj*m8zSO*xHTl53DAr<)vO8>Nj$Y;}Tbtm1n&WQK
z{oTu!<(w{ZwS3wv`}AAi1_tjndsE#{G~e3!-fzn+!O6WfPXokfJ1jQ1D)M3N>8X!z
z@>My%GdsA=qU`*xyu2!lliTcOu}7Y^{{J<8aniI`_f&Fb=Lht@XsBP7D0U-f>in=*
zO)0bU4U;w0zH6<&e<t%!ROm*X+pTZzw<V+<%9oB6)}Q5Xb?yvX)77k2lO;Mo#H6=W
z-TJqFBKMM)i7sco!f!4<yCF2>hf+uAjNKv=4=ii{KkJk@Td4n>nxkjWczIdWF!x+L
zVs`q>6r<(43(B)aG$$RpQzzh5xaNpNlce_MpB6QyQL;<=)1#fM*E|clpW$vD<{$cR
z#kPHH2hDbuNq)APcQs>H=#~wKm}3gpsvWv+T)cYgE>7-uJh|)T%X-%Ou$l<YdcQ`%
zOSyFIZnoKBwF}>F7fv+FnCP;1XD*wqv{dW4sX`IbZ*FY)x!yuNqO?(e!HtcbX#q_2
z1;H87SLVb@v3`4-7v`9eUMAM(bG!dre4<tF&rhq=r|*zm+UjL?=;61V290Ol*Vr;%
z++4$PZ^xPIOLru=nfpsVPU~$5D|Rt*6F$3i$*uo;g`PIg?6wuMsTAdoQ_y&(?Wy-;
z)tRp{TfS}G_=lrt>wanRtoPE|>)BkigLlsTyM<$}_+sN$-Rv1m)n|^$++u#S!O6hs
z_J)qK17bHmw{nS|^*23yq4)31`L1n6(TcqqM^<V*6<lF;QEk7}ZR3~AH-4RV?cSU}
zN-XP78a-gXVkb2(BqHKy`l3v?utz*{o)5A<M_S~r3;*!J>P?ua+NOHG^SQtOWqXEg
zGU~WzWW7LrUAm~nfs1a-H*pHH{ZO5`b?wChZR4gJ!5^h^mtAx?_pi<3ZKTS*gdN*=
zMQDixBs8~n%x=@uSSlo;Aep!DP_yW%!!LKNO+I@2*z6q>CNTH}@VEttYl-P<GxFqW
zPOLQfdC9ZmqgAzBUglDn!VBxm#N-8SxRQzk_OGf`nfyURLgUCr=i8?OOg_z@7+^iK
zXdC-6UmoH8vm{>@{aC)Ut@&A9{2Lw7JvM9d6xOTW?sALK`j8!RQo3)(cK_fNJKGM$
zS0}2>JfNEXX3e2co>|@veCE-mOmk<=`PwS^t<+2={z?7qrLyT?XU`G(v(2=srEsx8
znbGzJspcHvWwutUo@q%=P?EO#<u*rYp-`Dmt)O8{f;D50@t#<VbTtl_MY29ywoK|3
zn#EHne16-c6<m_OF_)PZDan11t1I}QaVUDr<l8Dw+hl&UNR&6$uXkBEBj7>ds*Rb}
z9L`T~J+k0$UF~>rV?|Kjr5n?jpT$4EWjW0vD=_Nj4$IxQG}KS__3o_tc4LX~?0BPP
zKeo>247XYDBp0?kZJ|i4cJ`y{iM?DiGhXyn2fyCn#9kv1?O&yoZF8%2$-_|9RPo3~
zYeHwIJ1seKA~!mi<3+<oSFhlR5Z|XgQM|JK1%fN(rd2h>zTOrQ$#eUH&q}>!r!z5|
z667ugs&9NcHACg|#_Y-G9-YY!?U=Horg6KJd8Ulg?cSqHJRhIc&e<xIm+o_LtFD&J
zPbR^KrK=q-cf3Bl<APAg3XbDPx9!z-FEQO0&9OaFHB&k`b|+`3Tv@kqn$#?w+{Mb`
zQgd$4Eo8kceK;jMJSBJ4N)f;NP1PzJV?qiw|E3+x<#~8k?U?Qzz2)84&-Mwrgh_d4
z1{rJQ3%NCvJzSjTl&1dU+#{B=g+i8p9@?fKKK)hjQDS(0f5dDKzms!b^{h|SbJ^Sx
zBbC6@cJs&fsdiZ@OJ!B;#CA7soyo>G{k%u-oj9YX6-}`!y`8_xJl8$*i22y{k15~r
zMgDiu7wK=6%ATFMAsJs_WirKh-pi8YToHf2Wu39+Op`xqZguh%l)E?KcjKqSfpho-
zx2|+~)O01=yO%RAdi(A<AMc22iB6n;-~fw%xyZ5X<LA8Je<(dNJMVyWb=JQttm?m?
zd8n<r85FFW_BliS)P>VF!90yMSKsdc;O=(htCQqpw>MeBrNu5ftIlW?UapYXZD_{Z
z%C{{wga4Y@^qiA&xy#fJsTb!jE0}(fo3-BT-9-mkjy~guYaADOorrR|C+n~5dekrJ
zar>ld<0IP>Cq6feOc03gn7?orf7bn(E0?_O;9RV&t}?4>;jBZ;iaNJ;rG>idzEX+W
zYM6Y)O`~|fiimUEsTY}&$0QyeVUd*%3;x#`YxK5n>3Xr;BA0s+<=N@WxF0Z_wYV|+
z(Mc2KJ5fF2GgWeQRwo|I(p20pCEhKS5h0}FxkyC2VELhIRqJM*>+@?)XcgFYyz9n9
z_WDa(3k5kdlKyFJ3{BnLa#BsZ;^mw(OS>Pnp5SS#PMdN@LCRu!<&8(TS??zVvCd+x
zdcU?}+xFW$HX0sZg<AK&Ub4XR#Ql3O8lFsW_3~6voRHkwD(x1qW3~5f?$kxqvg_N#
zS?}Haw#u?zb@u!n<uR)Jw&iJf?w#Rx*{A5JmyU?@X0?=ck)nFDA5N-{P;pCi<T$fU
zzE|!hv+Oiu)3(ltDcrFvciD~|dnN7LJtOt6lG~xV$2Tl9{W@D9CEeV^%-iZ|*qhpi
zUtIlMuU|P<#~V3g_w!wnFYQIOh4ZFzUW`kX^UcoOH_`V|;6=mNK|%{APulr+W0A5?
zfQnL}?>bQz%TqU1u54QHT-#%*7<bag&&P6B<!;w(@sY`YzIIl5+TmX*(<g^+N_{4m
zek%BsoV35eFNukrw?0ODCge+Ta9^7C$>_C|evQGO-%?kmT+-f^Abhs4xz0jx){3TQ
zyJBaE-8ffe_Au2{a*Lsvt7G^hrZqaruDhqU?tan!%IM;6y}H|5!w${8D3zPB<;S_0
zkUfTytVW7_rzZ8*+6%m#b!Fp?Jpl{8`hGiR!}anNw|U6opBwV0hi2?8DAaW;^|e0#
z^!Dtg<um4d$;gt^`H*{kpV&p0sR^^MSFF$6DJId7n<{;#&&!bIvl^$!?wV&)SFJDm
za{KKXA&rY2lGXnjzDlv!wl(xR=HKJIGfVkbld58+csJ|2HFubjTds7bM=g2k5wuY8
z;@0TO#fo85E@kbSu}AD}VC9T!D{HxzvG!%K-Ie>p6*|c#uhHPnHm;<CHvf={9pay#
zX{pwhRO{7QUFXdE*4mjZk@dQ5H*fL27RM<pr-PT<dG1cRBd8ksXx`h2JNLegINZ|o
zO<AJzu~-5_fl}O|*AgMyt|YDC+9wr%H)U?N^_<mf1o;?`h|Ip!rgE7{T63nhXxQ$v
zGg*&>+|zAZ?4^G_#WY-1b#Y_GfoGeT#H1z|9p6)stQm8(+ci1#h)M92m$Ul#a<=|g
zmpymp@{!}qPMd!%GF$eln)l)Qx0@erd~tEpMXSlPdgm&?vhZ`-kRrps+lp<+n?q#^
z8&pKTW@ISNajTv6=ix+UDe2A58IM9t;^(HFHS#><x6AqBwiih$lj6Le-+2(S<9ce|
z_BmZ=zUG-ute$%1v4ETW47-HjxMhYj=Sod&wPlU1-W7JeFJko{#u+}#g-bm1U+BJ@
z?-IrPcd_njgV)EG?TA{Mt5K$(yKVBHDM^Mi|1u~lJHFVv>5S73o+B4lMU<LYyu7+5
zM{<*RPe<h6H)4xh44$y=<0v<bYkzjNM03s$j`*#UJlk~FoHEhO%}KFeyo1y0_2IN<
z0b;S<UcwB{@@qW)-i?TvaABU+W8GD&X0Q1dblYe~wV3*b)5@ndEHi$y>mb`au10g=
zNqSSXmn212nO^SlXl3&K&g0JeW!WTyRaq-KK2LHCGkl=--11Axy4N4WE?9G`=FFPJ
z>HGB|)2F?2uZ0*m->9__&6t>{BpBRc?sugv^N`boAE|5#{g%g)8;>r!(h(n>ZLf3D
z#Pq69Mep^94%My?U7bvghZe4l*NAEs@)p`AA}=R8w|T8cWNK;Z-hjKW-{mr%Qs61t
zBKSqESz_wvBvz5Te7n>*SDOp&@_6m`Vn*${2C1o;lb27^IIS^V>4k}p%4{~>#p0*f
zWOwX*d3W+btz<`|IZZrl*D`drOk{ki9L~9#ZIRoClvQO(pWWm(CRNFNv3XcfU}v&-
z3Ri!lvHn`|tB19pK8iNr+s>ljyg9t%`Qo*UWS21TGPW-gj7g5$nd#Uf)O%&UhbEtI
z-(;2U>LrpB=e&!a=%exCwvMaImbKezJkp-@FS@BQOZ$jQ)Z@249?e-w*AHJ|J(jb?
zqr;C)pC><Uz02k2OJ`4A(RGA<_jl)S>#T|@z0-pfa*ukknYX4M47{xJV0~_e`h~)|
z*@7>*(=YQ}nyGb*@zaF{5$&)_YX#233%mcyMQ^l-O61f~e*UmAOTo*@Al)@DX=bkZ
zVYVPQk*E2Wjb3F`wKeT^N_==%CidY`ZIzZ!L1$erMQ-`J_^$L)XZ;g3S9O@56*eBy
zaR~8{ON-j<F0^?S)9*u>u{$TsT(V7Rwuxw30xP5S&ZTl$9CN0hWKezW`@kVW!YHe~
z<HWqq9ZQ7bqwBwCd91dRX3U9MX_gdp#vuHL;_g?A-xo`GglBdgT)_3w&}>f;e}Bdu
zslxgS8^b+3oANdv(EF-sl{#Bh@z~;*X3`atB5idq`OS!t64#vE(>v>A%Y@Z!D<u`r
zuM;dwORe_5q#Tl`Q0drt?|h?ll*wVW&2gRYR~ndGo;|`W-YGV%iOn-&`wq6WDLiT0
z4bR`zUlYySz&7L9{keMlt2Jg9cC_Dbku|Y$E654Wd?EN&Q)*9Gjz{DiQHjHuXQ~o7
z`OYY=%2{(wEVSnnEBn7{Ysp4!l@gr<7w)pDXUcN>pWS@yyI|XqV%MxCw^r=Fc{8^>
z{7ceZrQFjd)@~o>?hf(GYIO)%qbYhRCU8e+Mc+=3#66e9l%qH%#Z101Q_?YFVOgX1
zA?1@iktMSy^;=afZ>rhp#JW2z?_lR)mEzk`hcyhvZ;FelvvrrTSmmB>jhStvq}1Jc
zr=rtu_UTFa+mnh}UUuk8S|{9(U%LB+dBKmuskXOISM5K>yZX(JgG?Jv%@7T*I`*Vg
zeXDhos_B#SI&-bR6}HbSGz=71o_3r;sK;*k*4cfxo*q4ScvH#O_(cA&nfthVW_7oo
zv|?K<bhL4<M_3}~vfmA*>yEu#vig}0r|p|LN~?90`uLiA=Pun9%V%btbK$M(Nr4F^
z(s}QGxviXRBV%ACA{^jzCVR(@8CLg>Olg?3uqb!q+>SL`&TgCd$~A%*Px#!+SC5km
z<V;uFz16I|N-;u2m~H8k>H1D#uFPUT@+LT07D(RVS$rvO+tPz`Q`<cKL!^ASnI8^T
zSe@!vq$P3fjYY!!nB8__#~MVPisU!k4hg<m#doNtxYl{I=KB-JJ|(}f({&5Z$>QYo
z-!S{SbJO##Yd2nRNZM$uuk>>Mo`^l?HyvB6F<DtA!l?du{mXlSJyO|gUdS4yeRKU1
zZS?Jxt25U!ZF36;Q+w~Cf~;^~|E4P`zkKvRntMIg6TDHp`Lo4^N}rg(7XD{~t~Xb`
z{Clz8G<Vg-GRw+^H9VK@&EzubsPr|9pP4%?PciGlrsIo?9(xM?GDu=I5KMKO=p6iH
zp61(DRhOxPj(!e)Q%*jR?y5=RYP-iVzms$6yv;HuJ9G*!7EbWel;*yr7}E9Z@HVO4
z8c+5figdi9`zokMP~-3e<0)ltbsVP8)cLey!Ndb9hyHG2Ond4!&-%BCXiKEAp|6f?
z?czK~N9#<Z*?ChOuX8z1TsPTqaoQBy$m=&R{Wa-4wWDNe=Zw`B%g-tsSj~L8YAI({
z@TpneTWx)AU1arrmF(K_$?=xr7fYj{ltSbAC6mRrdFV~D&o>CXcEoSug{?WcZ>mnX
zI)CT2S?(NeD=O*t#7kFPQ-t+UtBhIAi+{e$L`|ML8)aVKqTn`1I9y41lS=66OHXCZ
zc2E6dWM&+E>BiPY39Nb-`1>ouExV?9ncw)Fm}Ope)X(9~rrCNE0v>GYTK?KdH+#y@
zJyqVJsUJ2)2QcvpwJcI%pC!2R(Pd@*#-?P?gS<@Yt{IgFr!SuIB{rb(=3BD|T$>`I
zV$Uq<6%S!)%~G0r_vj8Ic9o@*3UfSy-fnpPOzyhE!`=@ci-i&{NLs6hJGO_r>ltnL
z;weqt{D@&@$L!6W=67PEK52Y@&f$Oaw&7gGYY&?hU(buw-<)R<<0bFs9y4Qd0ZZna
zI~L+u*I9LA7yZsSbMVqflPMl2f&=m^IZW&3?i4@S%`jDnY4g?pXW4dl{aQVH`jRyq
zXHGrNh$=ks&~Fa|L-4r;6SZ$d2k)OX^<DBcopM&$1CyTqx%$E6hThb|f?SOp?Prgr
z&jt#v-q53d&_rQbE7Q}DKaOYbT%fpWgVy9aX0`3Cw-;MhEfVeaHs7_yTYlc0pb&@u
zvbR=i%$>0E;Kf_1+{z73RXg(^o!&a(97pnuT@zUzyGl>23qM(FtGnu^T=JbQ=M3U+
z-qT&6lrQ!5LQWyCRjjp;>XR3bL^o>9cypO4@OAJ*hNCT)59O{}E0X@wtnaSXf?e7N
zj%{qNIJPXU>EzAco0I%n3stXtSK~e5&w4oSF~8cU8#iv2B=ifpN*h|uzP0dH=-uR1
zW_ug+L^fqtR)*O7?29f@(-rM<wLPJhq$Xpmn!Z?b27}1SY_*O%(<7d`Z#<wq%jB)0
zS7>IN(V~eqv8$4d_tZ!-ow>bydCzRWyr>T5z*9GyPNyAl-jHxbr|t0DBdHsgs;K3&
zy}Ej&V0Y(omAi7aty`-*HRNM9%4mvuE-YK>zxb<#M`h%yn(xw+!c^Td?R)*#&y5cK
z<JNPu>SNHIcfSAi-WE@-P@k`7sG&ahvhqpggVC=Jh~{}5ah-h1=$zKn?BuETu^Tr%
z^T@lv9J2q?ceamL61)t>QyAQ~xt-s1f|IRh(i@qDWpS^Ri=(GGEtCyk<ZvNa=SD{7
z{Y)O&!mUvO1*~z;8agCx7)l%Br^$II?~^|^BdqL@ckuIog*GMHCgqmv^!>85`?>SC
zJASulQrOHCmnvk-{L3axpG&5cIcKJ>BFozDG6U^6y`;X`iq2XtiC)QFV#10yQtm3q
zi2rauvFmL?aMy-&s(fKt<-GTtmax5-Dn0x~-P?C!MbouR(`x4<n=;!jx~8wbDf&C<
zseRCky4J$SvgdE+t1I4$)$^LHWjrzOA-7<F(Je8_xy2L0S=Qa1y=T*m9j`8Sn(sHc
z%=~f^$7}{!sVyI^CTiZ@;W}YOZ+NWXu321*IPY0XdY?ah=lqt_a>oU&eTrBoO|)*_
z7yE*ZQ%+mSVY;bnxN?+5^OC>4CTSHN7xN;Ytlf0tUY^n<iF${XPj*jajn|5~zUj~t
zvB;K1v5&GRiQoM+C5f@V%0$sk^vILk>ZlXm8=b>;Dn@R~Q9r4^z*RR$P$aYZ!YjKc
z4DX)kyw_eAvuVqgi`8|$mir%l4N7=*)Tl&s@vODab1tziT2lE)^<_uL_s9<`ThiDT
zEV>~tw!=pCUGv|GPt&I`p4}TY?PgE+!ChRYQhC-%GYnSEzPd~A%;eia)_rO2+svPK
zM|=Gj_bv)~;+De6Xd1ila>Q@eyhyXNao?*ola|NzmG6)eo4fxV#|Dp1V;SZ>zqF1V
zj4W3F>+-q(Nl>VOPuF!(?NefhYIPp<Mg}r1s^D9Z(Yr>nXtDHZiR&`g!y5h+IGt>1
zJ3qCqwQNc1aeGD%2?_B#8U?EPAr35S4(6QQAzz@=WMU=B=KW@ZI@h@-rJbu-JF_;(
zB`4nt@o}5!!&Y{K*QlvfD8zRAY_IQaJIbzVF@=P`Ub~AWwMFmSili>X*=vO7D@-Uj
zIQ#lz`8$ui&zPmBsZQShRw}FV_Q!VzbeVIfS0*elIq0EdxsB<!=}WmrF2U<p^L&^Y
zB7=TeJqXhI?&0Pf@Wl8=`SwLwhgaNRmwT*l;+b8>jY}`xo-*s)nywZ6J$Gjao%Obz
zF)jVO-?7$;r&Sx1o-`OIwnlEaHA`3M;T_xRoqU%MglJ9`3Q~+pR0_DZ-~s!(l-_+B
zX=@uYLcJOqmVXsfdo*$N-ps(*e&wx8b~FB)b@8#;ssx{weS5C-TnIb;Icu$s(KR8K
zv@F38<HA&-)n)1%MdPn*-Tz{qRlVEVU70(Q6GLxgNSxC8T-(jX$SUZod3Y`Fd|TPS
zt{-`g{LNMMx3{@7-8EtpG8Q;c7Wk|xWt)V;lUpq9OgghzH@%po`IoO-(`A_ef0DH3
z(%2{a67s^A?Kza^V=m~w|J~weJH*R=y+~bQy4^xgQi5TR&*jrvqUss%K8AdDQr36f
z)>^{0+dOT<${9*~7gVfp{hqe7{L2O5@D3@HFyX+jYBx7Y&*b^>McAupU9eV|Ab0;m
zk@p#vjv6jA7Ac?D9@%rZ<5Sa>2co-kZ*dFeDxaPGxZ+4taKMbPz`sZ5gvp2I*qw^f
z*j^MO;puf~*<Cl`hm5j)OZbD+S(#2s9W*;%oF{a&B1Q8NZ&LKxpC^3g>vk`Zxyz7S
zz2nr<^-<RX)RTWN`;~1I`~7y0r_a9qY7-(Yn3eJ$-rKUMv^$3-t?ab9iu>$cue`4G
z&bC_QnkKzM?OrVZ1l}D&#%#Yo-8~=p%$IfhqvGbL-!JMWt~lKMM{cM6(@R$^QkBhi
z&R^J3@w|ZT!`+p3-iJ;F?$UkY|9JN7Ks(J@Cu<HqshBo*i})p;={j#5eugi$VTt(Z
ze5ukYGBZvxtncWRDL(cKDxJQZ*x|jVLOG1{h@|M^JBDI;mG^8|&+4o@-ImJVd*@s4
zTd&VTr7A+^Z$!_Z%qri`rBJebyLZNSqsWd|x)}*)Cbw>0wW92&j7d@M#awP4^^Hk$
zSS%K~$>i3)h~Q*BYHg@7*Sbk>dq=DGlE}Iz%Ws+;J$OZQZl)UF=1MiaC)@rV;61Re
z)+5RL<@AZZo7TRtspY>D+VbGK@y7f`F>3x1Yc%^bPK76JI~nG_LV-QRDM3nEaS{`^
z&9R{O+ci3sYK)SD)n8w#*^&C8x5J$Eo@bnnfV!?k`;yqrM{)umaR=vKPdahr)Z&LT
z-}<}FKO=a?-}Pe1OO|P&%l*58ck`Duzm96L4Z2nwBwF|W!m4#@uh+(KFKv9{q`Z2k
zx%57>&I^u9vgLQJc4}16XAMr8=5pom;^5V-mt}f8?v$l$KEV;Py0ar~_6pYfTzUF#
zh8u)u7F}{*SI)^?CuE#|D86-e)e(88&pyA3d?yCKdAG`h;i}$MwWncvZ!=QEUT%Ne
z)nexR(c6ttp~rJu<EdJnKL>IT1bvsX5f*XSz-OR!=egW6hO-@))%wcjr6xwMp3-)2
zUgST`sZTiNrpXlg-M&)Ln^oy<#h7=SaYgizUyD^@P6YfqQPO0W_EgRzWDySwYj)l4
z_hOUI>0C&?nR4a4R<P59ILRGrT`s%ZOq#On=pnA|NHMNmce_5jwjK`&6aK)HA+5aq
z(NUEXq2KN*xoP~J<(&47tEP6wto1)v>u%%ztMzW_v@HQozD>^P%{V7Ky(vjr?qWcS
zC(kAyVZqHx2}|caWh&gLyX+*xG)eYGE2+FGy*txablq&z3b%eY&uxGBjkt-QHh71*
zX2&ep%Hq{_?uyYPo@@J7apk(~)xTt^KjEi`n&YZ)Q~y*|yHAXThcl*V9PO%7EuQYP
zX_<cPDif{abFWoj)cKOz_V#Xw2(O_}{IuQ%v(}wk{&xSlW_v5kc>N2mJu~-|>_5Ek
z$ALpS`muBLqQo@s-k<a<CM?#5*^ymxQ+Uww$3ObKrp~;`U9mUz!a?mym)l>~bn!mx
z@ZTle5iMe~Nh~^*>BNcuWv<(}yEl7EGp@1tJ7Wz?*sR+cE{vYKUeTLRg$WgEbqVQ;
zu8jET;Ba$VVEM!;$8w_1%PIB<?MYxTo;5XhTjSr)H;kKexww={4z6o=Wr%0BkILY_
z{a9D&r2C^uC$3!CTfXELOXu!wnZ*wpE>12vo%h|WVyZ-bQj_L=pF)+7iyk>9vxlBE
z);XQ}(5@l8uc&9z&aP_ii*HYDOo(W7$<eZRHf3E?y>8-%6lLjF9_H>(wJJNkx3BJ9
zG4Z&leEyMytko^+XHRdLvC|{vvcRN>bXQp|#w~AGeeus#VlAC1*wGSrLPua<^z%*y
z|49d5Ejy_AUv$Iq`3oa?cJ50Kd@M9~Nt*edsb$JDUY%45=knR^;L^QixgvjdWJb6n
z(+(pw-*(XxlNT;1_<gZ%rnErS;{{hlml<5ry)DWqc6XV8`p(V|_UY@of|8ZZz0T?v
z|5ZNCB)-(MJNEctR&iA}v9KJyn+LL;L>A^n-Pc;X|JS)mH%fRNK51|+&gu-vzw&_d
z_{}b(gQribo}Tp}?4<3sZS3nxy1&%U$b0)~(!A(^jptYC&vJCPW`CCcT(G`Lk1J@C
z{-*8a|6NM_`XVy9C%syC>W5{n<5}Oh=pFwiFV=Z1wz)&4GV^4!>`W!*?kVCk&s}rv
z5qLZG_?Dn{+ly*?UrIgh^O;{yI<-!tsyFGlcY79BzsACS8<SarpBCQOb8}b!^`uP6
zId_6|W_H@#FGyCNsPKvRhLaHY6Xqw!cEvvIanZSRdW!!sft}e`W(0j^o#thk=yl*%
zYF^MK%>)iX+X?Lx#T$>^3VnNLcFwMXe_8kP%(bS@n7Ga8YI|4UE}e2ecP7E*(npne
zZ!D8Me*aG7Of!$ThHv5@^L`##5E^`DePZI2oi4A`m-D)16mYp1*-hW!*vNTF@^DY{
z1fCTCH@rP_XSj0j2wr!hN$Y6BqziwnFEMnPXrBpMqFU)x?`fG}B<nV1)5>-ov#URL
zF1L5ycjWGk?`L8;EOMtm`P}#}QvKx1{g*B~MEN~?xF9_<+~*CagUhU!Qmy%ZUcQqu
z1K$azoG-W&Dn7HWO*X7ziuye%_s;B#r)s`*d{I;QBH<PMEhA#L`=KwFFI7HOYFW!t
zXSP<z>*;f`hZ|4uZ7g`-o?xiRI!}7$i(N~<Y4x00yz$H&`59ABc5Y2?ROE{kbNp<l
zab9lXjkhW)dpp!V+FsB;-M-*qf~dJG|E3eQK{mJkq;5{+w7Vd4C-jh|%Df$pC(FWq
z2uHRZ?k>NPv`Fcl^#7|VtN-(tEC^tnVo(;|RkVEVUmM|!v>wZ7-D5f1v-c$SXibj1
z;>dHdneDAnqSvoy?=-UZN&RurnY5J4Naf-29_Ouk1+6RYw@T`9WK8|-AiV$WVvkL4
zME5ARTdxl=dQvyp{M90%2`MX2t9?xiex4r8ka9<56U)@DbE^EN)2`oDDt)_o=eFEg
z($#@nN=7D=iVt1B^m*6Xm5Uga&jqcl^jUb|M{mMy>8cCIY|0k7iU^0x6dH4=%`3eB
zyfvei{W0gAM&-bKfmiL~OOM$u{H^ieO8U}EVyEMS#Qy$PP@3tYp89rS9qYHX9e0)V
zEzT${o2+@N$0<p|s4${x8EfceC!N<$2~ry+Ql@N>uMM~)E4O660XI+Grd%(#2OWD2
zBR<#192Z-<=Hb43GKaQH#HHJ;s8w*&F4;Tfq~%ekA5-~HcC6<WczBbqdDqUa1!X#U
zn=>^+Jb!J`@UBr%cW*JC^yAnRt8;mZtY^cj4cd=ADlm08ratM+!}upB9;66}%r9k-
z)LytZA|p5>csqCOL$k0CGqoz1*}u4L^H}z3=FWbtUs=ALo7J{5ykq&+n_6wgzVciB
zmac0WGg&5Z-rgkb>$k*TiMxAm?2FRKmvfF!GJ7BNB+2Jp&6|tucP1{1bKy8J;qu&P
zQ?CT9%6#|Ubvt)H(=6K;>%Hn6HYB}hPgHrg&$Th*QlENF7MGVs^%C6+Hy1N!zc%xX
z<~%77yK|Z1iiN)vjc)Bzx+&Hvt)w|8#y?$*$2suH!o><4DvQq>e@!{)y=cZ2<)y_F
zKV3VtzWUwUCYh)U0bZXr&3w7>+v_EkKa!-4zi`D&HC!O(6B4&;Udz*k0-O5HS06g~
z=8u*D-|=~iI*zCv7t!7vyQXaJl;opt_>UUh5nX*f{LF;NHLKz;>&<*O%ZAZyb!wLJ
z-}c0?sXu+~J+Jrf7yN5$rZ1c-HL27mcZ!*6S$yj^1<jsV=eNqq53Z+9R2Pg5b?Fzk
z+#nd0UanQ@K6}v@nLQ_F@myT=uw7M9tMh^2@6K<E`8Ugy_I+?+nptk7^~@=+G0AaH
zg{P>owupCEl+H)Ni%dtkrf(6?QrIZK-#<Cu&$jAZowDIhe%mPrzG!oCdsiucO1?Ac
znCb}$qgAz9YfgW3Q&n!LTB|+(!|QY@F$QiMo!Dh@3SX5u_s%r?w{f0<_mgRhrrQ|R
zD;?kM`eMQc4abes_dJ!1d@Hez<@c@WCY;eR{s%pGC~!(2d>$N<@qCuS>JM&?yhlQK
zq{Ftwi*1`KoSNkwwpC;LrCm!yqBv8c76xd)Sg>{7v~Pi}+jU>&)=etraX0Mot<ayX
z%Jg~mbZ6sttFD>d_%A$jV_2)%HuFh)cer`_vVP5TEL<3}@2Jc|E<UNZs>$^hTV`_2
zTfHo9xqfo4<gZDL&Tf~og-hnKufFW`fw?rw>7BRE6Yot5s}8MQ8DXembA|h4?gy!?
z{w~h-m6yegEk%9qv#J}N-nK~FKsK#TyCqDz`9MXi!A{XFtk=v18+}f1<FC9L7qVl4
za>9Yrwuvv}xLjxcEu62v<bjsaI*rJZgPF@us4u?RakkIvS5>jvsiGB!l4Lf0`G54!
zif&P(tpD4OtyZ6H|7)WA%b(^atnO{u*?mXj!{%d8H!kqsk{EI;@!*xW)8z`)qk|16
z&-a{n>hbFDFYaAj?q2%lYgI^Z<C(*=Sd5BTdGa+5cW0UCh<!_Y_o(!<{po4apB%eR
zr0froxjbca_j{ShR+*Wtv;G|sc@t`XY4?=(eY<NGWOCG43Y{>uG&6K8vfKTRJuizf
z^w0{mW8vLbv!Yat=B-Pc9{qNa(>ZRYLl+yus%O{B+g^OpbZ>kAQvV|-eyr7D;L=L<
z_!y!zLHVYa(L$l^4`QaA7CpGJXriJ1MOVQYHX2qJA|}}WnP$~<MDFf^8+W|}o3@KS
zetm<ZJ*nc%wj@uEj_2IZx;jdY7CzekO?$mwkaLpB>L+X}>H)4QA!{4EE^UvR;d`ua
z!5l#`zw;b^oleU3lj@G$iQh9fzdC_I%k|_h)AYJEr=PK|Oz_}O60l6~cb!+iM{v5_
zqNHFIL!J1o|HHX+RR7*;?l4}y!}xdO9S@BgLYBAQ1ZCeh>2j+m`d$C3uU9lW{HQ^S
zmZaUp>E9lD<fS(1GIDR^iI31dqc71tanDb$2S=UE)*StM!Xmo&_NKc^Gs9w5Yx!k_
zYBp+F&dLc}cIWs*X5Yn@eU-oWUpo6Rd1urwW}}R?fh)gg<(zj}lO>c;y>QZsO2gS^
zksPvrr9_n47yWKcWKrO^2>Z|_?{)N(gLxUxlO*RYk*azNv|{#1Yi@WJdSuJIM!QKn
zI%iL}HMkhH>c|Pn&N(tAlfNx6Uy!g%bkR)h`o&i#Dl<*f+U%1!-zfY-%gJihZ};lX
zW%nQ69uhHi%NjMO1MRoDH$ByK;)u1?teZ3W*}THTn^G5U&|>l`>GI!mCs4B>!7A3}
zY}e_nFPA*w+%)Z-e`2Cx%DR#lJ|8>3t-3Dw^+4dQ>b3W0OjdU`banc1eYxDvy;3PY
zx;gR{%_jnWH-5O#@a|Tev;Msj&#NJiMK^9p5?HhCR933&t^aF7)ebZ7yQr<#&;9=M
zBd7n~V&-clPR^X6+;6WKS+(Rx$b*ohlijav>*}2-etq8DOS7)6opG6~bNXh>M=7pa
zTU|uF*j7|t*>rU7njhX97UT$=IzM$opxft^hGW+xUe3+@x@*eR1TMBqvo=iH`agZw
zm%SmO+k6(@T=3KMRF-e}Z${g?P0O^`-wX-!`4f_^bhooJQuN_ri|w1Xh9{|OEtI?a
zYI0!h>BZ+e;*ZA5@LNqi9k{Z;>+!n1OSeR;bZ@nFdV9CGK{BpsdCJkoO{;|ZYgM)e
zvg|s=dPrrCjrgfk`Hg~m|CozqKS_PE^vd*HPRZ@R@(K($C{EsXDF5b^S!XwQ#;b{G
zI(JB3K3ny&ki|HBK|+1-1x=w_`Tw?0%u?Zg=68A4T2CJFPto04>LJUk)I@JD-1&JT
z=iQ5YgJ%gDN$gnR{w{RM6z7w1Aw>^YTnM&w+uSi}_v|f9Il3#v*_~`6#S?np#7<?N
zF-b>Mb+hfOu!CzJd{=!C@}_=??&@h5_tmCf&#zNextPqZme9N4;+*X+hnDGvDNX2F
zy=v#rs@4iiCfgq_Emi)qLPh6it>u~YgZ=rt<g{&thn}q9crEhoT%N=(je}9Dg_FFF
zM9!XSc9{3%rnO&vxR&(f2(otR1YBr7E_VO2+X=(EiM#ht{=DS!?oUOxd|rO>`SqUl
zxo!2UYEiv{=sz=meYD*ha<w~Df6AKszt5One{rSpQqbe6X?DM!c`PV2K6o>3FH6-_
z^DPtYs$TEqS~*2XI6U$bbHd)(4>DQ$k6o{yX8Ej{<m%_(Z1(g>zGt)6>zHFYYv%N{
z>OY+|ePR0EA5-Jj7zOTLF`MCX@k0B#^H{YP9xJs?)PEPo=l%AOX=D1u&GqL~3WUmU
zNqY;Z^SnIRB&#&9@5U6n!`EhN3O0vyuKeG%c>AS54eM5sKCQY*g+9?gQ#fVCD|r-N
z>aGfUGXL<@?e*8rn;d(5BKdykLhgB+oWBb(*_O`A+O=~Xqw3Qo;w5|Ts6UO^R8aV!
zn)gcc3a=}>c3V#`DybBm9r3dNMe%wSQMvd#M^9_sSge)o=FgP#M$lq1qkZ22Q#(ax
zmp=va+Q$@rK3|)eeOa#NR7r5A68pXOEli<(b$|b`@6nq3>3-pg-9M%O8&+C>7r&Aa
z`*q6iZm$UIRF5xvW6#^X4$Zy&byNRs?xK({MasekdzxnKy)=K}wzwv<)zxNf3#}qI
zo>b`MkodrV)<&l!ePecK?d-P;A2~>_JAOq;U2gB6Y4IGnTPK$bcy9O-dA4(2*C&aN
z8~4|KXngA}GRw`9YkibQP2NT3y+%!12@}_a1x|d{wc!5VTlSonpVd5A_TXyxf$E0~
zOP0@hmLa$A8=rjQj=%-opQP0+GAF)%pVU`!ZEM`6U(BoAgf?iDPyMR8!SxPj$jS?k
zzllb4Y0a_jls@8nLVXW!OeJSghxOK%8=g#lxSL<ne)Wz8le2FoFE!WP_vAoeX2{D$
zSxozW*S-$De)rSv-M1E<zx2Xl?~C1!)BX2cw%%a9<nlU|#6qF--iywA8Vad=ezPyZ
zMLFZBr+3e+go_qN<yEcMlU4+)8g;F|QN)uHr~i8&d*jKBy)!jiT=l>6KHbb89KZVJ
zlx<OlA(b2Y_w0TZzKnT&gyZ7S-8*J^|5p+Wj`CXienWiam&2>Wlj~2dPQRKXa@CQm
z{I}y4Ez7fV=^H1Rp0~SS_hr$&^IvzWI>tT`+P}u@@y<u<v-fjrH8o#nIm&ol-Q(5A
zK<8@$7k-p&{8Ha5azicUB#-2xwFS(}A2mwGFMsyb;qlb`-a{!`c{)cr^&_rVy_>sd
z>AD$z-o9BGxG=70S6o$%&-dmtO(kOM&p0{!)Q)7=KFl~#r#-(i=T@+6I`5RLjb7_l
z{!731FkI+~>T3_prHTAg-_4ugJ#Bg3vB)Mqu{F{DGUw*+?Rx3d*SyqbDueQagV~A(
z+{bjiOeeLpxSfb864>^EHOhKvxPa{R^6N^cu3kQ^AJbLw>bGtnx9p@dEB^+VT>qDp
zx=o;4=}|DtAtj~YClB_1S!H|baq|4Djt`DD6z3lMll*$cvq?ARY1?b%#da1x@x4)O
z`at7lL2WBzSS*9eVSUlmTG6bPJ3OX-xa!PRbv5y`_LR?y>jJoxz2|M}_!$~2RK+yE
zD|EViSKWmpwvQDqh6TDrdatRy_T$u3jtQ$+c(2NA5U*yhkv|&oMSf~B2k*U4+>0g^
zo(_IG+d18B$7}aDzxh|OO|m|$YcWrV`<2Ht-*Z8~*Y-R4Pg@*Sx-gWx@rY}c+Oe;;
zXZ++?<95nUajXe^^O{L-mn+|V?m71-+A@C6whiy~H{8Igd3>Vwfu?&O*F^ayeW|?`
zxK}OCNOP;DzV`02Bl6u_f>*IF+U?Q4a5Hnx%3D=WC&bOY`1<?WJpa2fPA6jz)?fYl
zy?Nc1Eb(8}U&6mcg=`PhUj1iHg4}W+LB4BA{=HkX;+F1T!S{I)XT{okFEayz7UcQ2
zJTBQ<<ku*|bt*&l!Rc$4i#Ve87FquH|JA!vBUR(1@6-0zI=#JH4;@wdkagtJuA<tY
zzwg_Ho`mtvEm7`xWstpR#ly{d@t^KB^xopHVdU5&bf@~Mc5vAXvvb8RPq$Z|R#dh1
zOndYu^2@~o%e!YyN$3kYraQgx<^E5m9cv=AmL@%@K4v1WBEF?%t<dasqHi-#@`%Jg
zYwDY!#5Z@HtRLr_*!?#w?%L1sinjQ9`^NKqxAx6jE-3g!rTcC9#2xI9s%1aEecQOx
z=Hu+SE~mV=iHHA+%$)TzY~j02o2_}9yz{RFc5uzvpr3Kx&AqkwQPAPZHyP(>d=jj_
z*QxQOf02IRg}|5v-CbuczCGZodcMO}Y2q5Mo*wUq!g*Qz{P|xk=G~kkoT)Bj>Ll}Z
zUrd$OminjWm0MT$n2A<)sh(W)J@ovE%OAy9*hKl}n@amWXw<G2N}X`wy~>^FI^DA&
z!Cx2ZJIFGG35%7w+%{hDy{$v7th)BwqS;$gw{N_uv0q>z!?&e<HT#)6Swf@)b1r44
zMa|i5_kGDPsjjneJ9poheEvmhZkyk0#;qDXs~<If<Eawc`u&vy<DHTV(|2Wko}Jv2
z+P&>R_rHIhks@_Ll?wf=v6;y{GmT$9_!NBeL)gA8(w@Ob6ID+9neo!C?T63hd3#bk
zrHZ`giq4C^(Uro!Y}x(ljqffzzhV7i=47Y*1$-un%XPxk5^e?WU%$6GRQa^VE0L9F
z7oI-h_3g^{qRqx9Z&_W~wD;PR(AEjTGhN%NQy=TFg!&a-UtMA4>bUiof0NkPU0s!x
zsnfsO-qtxQ=eI29cm0FY3;gG?>xO)ak#x1Z`E=zO|4wtG_vv+~?@ZE~&!wwtt?kJ%
zXOd@L;pV^hRz29Y_S+0?EA?qR9Q42ZFA;Z6(wJn@m9+BI=H(8hQCALs{cv)gz9pCK
zB)5=4*5$KKm(AXu9>VS#{Py9eP0iK12eaF968v3m6$?(CcYoUpjaO<H5@wbvK7GB7
z?^WM*bt&D;-CH+=@hodw^mXMEd#(QGKc@s-P>HfL(hs{QA!Q^L{{7A;$1gU@->%$#
za{5x~!sQO(yvAOJtM1QpGyJUdv1x1bnng1=%3s;AV(}uusS{6o?`PRRJ==cf0WsUk
zgWp5d&z@-(c+l$FWVzJmnA@Y3GW(YPXufvyvXHcn!la{DKi1?-of8su&Tq@7lOJ1T
z1^u>ZFuq)LN4MK!)87T+T^F{cnLa6ZY7uYUs{2BJO|y$XOQfLNlL(bfY29qz-#Rp2
zOnG<cl2lcy%F&PhY;~gc?>N`S@*h=wxNY*ggQsUD{#^dG^65>nyMDECCfu29#aDm7
zy8hMX+TO6%WCbzJ#@nac68(SWyI%Kk{=DRwAiu<(R};Rbyzu(7<jo5HsRv>+moR-W
z%buFE<WcMKM{hQUMCpBU{nX5w?CBw9Q}L>N_NMR)Q^Ol}AMf6NU3}>)8~YhLNs`%L
zlLD=M4`?RqFn*77n-ur-M0H}vrO+Fb#cSF7*Tp>(KPkTEWcI7L;O|}z-j(c@kE<nZ
z*D$MSa)xx<hW*~3HE*TJl;y4qT2m{&v9MJ0$k}H52)sC<>@FlQ`D%sX<(-UPJttp>
zJ#0R=EQf#BG7C$^$-1XQY<BV8K62IZ6?fqAKv}iY6`FP&&$+vbuTEw8Z+PqDms+N+
z_1Avy3wM0oz1D~0!0Do-ll=3ZXg$C9&@IiG$3Fi4XKml!Sr1RK+}UhZBlJn?i}B}e
zYZ5=Myd~mxYsKmdm!^M=zA{_;ep*{dr$q6i<GZgE`+wZXK2dzf>&2J)PsZ%EalR5)
z`gL}Tv_5y&qk@?mt5l!vf2aE`lK1+*5cgR<!M@?4{8N38y<U{;Z@T8yhBA&zeJN4V
ziHlb#q{UY3IaRNwI<2q6E7~W!y>Lp|spu`+%uDw@zjyid+#<<ODn;*>E)tH?JfC_{
zz3A+cbba-+)9=I<RD@rXzf|Y?b@g(#6FdQx7OY~EV>7%H7p!s5{4@Q_6%7WX&dsWY
z+f+79UNUo2+n)sMEm_YeS<KIhzV-W+{@3rJ*0;5jr;E?1X`i)XRrpLv1%2U&*9x}V
zi==KC-ctUcdg^Y+9HAMK@;bL?9yfRx?y$$Mx^|h1a;l_3)5ap6FrnJ&qO%q6O7|`j
zp0`MB!@;d<E>&Ng5xThIV8+F#Yi}P23FPV8@v35t(!)*$T_JY2z=dntUTM7eJVofZ
z=6d7qzP*iUbE};H%aooCyXf$=DOmEl&!uV8wA=FEu9*1aly)QQ#G)zlO0*d7$LVZ&
zS=YVCzGwZ*o5ue(D(zBgo!Dx=BC()qRr3Pz?MupdF8}pj_$90-Sayw+;%nElcc=b~
zd@CaI{MVn0pFKEpBULH{m(AYl%o6ma$9~F@%R+lD`Tn>uKa@A{-!|6Bb5XlzeR#0w
z(}$YXbF}8=uigDAWZm(p{%u>AlzdY0-Yy|^Wd5P<sruS`HTn0wo@pr?-@WXG*VTM)
zTlUp_&opKPi2RIGQP9qq*<+j~aW&v)oWh4z27~r*=S~Ye`u+F#X`x4{UK`K+UA<5L
z{93=6+jp;gpZTAAU;nq_ZSR$%(wxPYMyv>kiw(KN#JXb1oG+_h##Midn{Ora<JQWm
z^B(R8qx;vIR9@g;Sg&WZcb=o{nyOXHnk*7TS3UaiI(7BN^_lEf*C-ti{oa-r_i>lm
zP5!v^|7`WE&fYnlb?)qvvX2|O<ia_Zg$k&jnw2hc#=66VdHyePsgE~5FO1H7n@~Qv
zVPOYH*(%p{pZGSrKUsUCzw^FO{5lOg?f{cpO4F;HTz9o^>0-_2>*ub}`)$#zcSd#H
zrDrU*zxguz;`V1p>3#b?_3hmqk=0^rb*tB^y`1?qg<<{C$!mAjE}SYQlFM=;cEkVb
zwRcvy1$gfLW@U3Id)9)v)3dZb=6%XKy!@zKzsGG+Ngcz8hsUShH)&p>`Jtj@rElbi
zIa(e8kI!-n@O{gl|MQ-afOTfro4-raC4&;GqdLs9JeSXk|FXPu^0^a=jBKihx6ADE
zFS`42-pZopKTNFd>s4M{(^XlWRT!1MjQ{M>{Nksn-eHH&Ug~1m7^M0ohF2wLit~1f
z6`2>~w%+`$ySVVZ)8u<^zU{niF1GdI$+erqgTDG-4K1w|ows~ab!Fv=tBa<4UrM@a
z!x`dtI`p}psQH@(x4-XYW?gUb)x%D>-#|{IP9}vv(>}Xu-ui<Qt3+Erh`g(?72A7i
zO8BiA*K;qN<nveklXdILwQrf03O^S7vRJ3SB%Gc9)GgL6{$~5{Ih>!zsP(Ei=y~F6
z2Yt2DE1PfpyjOUIsYI+x;J`Mck`>x}69c%8FYumkyLujz;}wRCnQ~@vEP<RJi~(EX
z-bSbTf2s+RmEWwxV)Mx9&BvWvOjbz<Zdp@oyEN1#@b10{)2eCbv|kiodH(3@ZmTuf
z^S7onoDpwOFZvUnsxobUR6y6|3&mM(H}Ab^U(goHYtneFcGJ`4>h=2H{=08rf39!0
zbl-aaZxioVv6kI4nBvVOe)Gny=5VI>-=AdlgnwP2y48qz=^Uo(8x@*;6_U(f1YO(9
zIc33)y)hD7v)Z`zIlZ)li&vhXUq1H||HNIZYZu<@InHjmopn*giDlYOt7lFYzq0AT
zH*wR<vtkCX`$SVDHqA8Hn82U4GHP?EzpmM>_xHa3{B-ekbR3@v*B4R2i6NavAKO;t
zsmf_{Rjn>t@kTiD?ag;nT2^^=?*DSyar&N$cA-}bXBU=i-g@Q6JIk!#?vv5KY-{DK
z*&}(Y8c&8=?OV~dy6*h0-me7#Ki?!IY?TjdT-?00{@sN~uKFjogryw_{H!6-{%>Y_
zP0{zq8$Ybrvzn)Pm4WmRk<6?+bML%VWX=+**e))-S50r)%=Vx!p?0CXdn(h7PcSph
zp08Xw{qpAeYw!PFlloE}an&(F@4~X4ZRfmXS3F$B7gjpqYqj621<&ntvMvN`hnEIa
z9@LiCYl&ETgXhUCA%*bVg9&NAtEE0JS!c>`H*?B$?n~3t&+ACOO*9B|FxbAS;jeZA
zFL&3)!cWnUBIoU$;hdi6xM|I%<%ZphzK3^)?3H+NYyPa|&zKfoFV#AsmnysBm1%(H
zq~%>tf^RN%^8c(KzoTJxoWXO0z3;CU>hIC2aJ;~3P?Qt?V)foRY&oxk)pE~Ywf?YF
z_6%2M_R*S+w`S_cDAazsu52V7UBg=VdYVJOxsm>i-JP$(*?QvdN^gAbz2;V{mdd*G
zc{%R#i>;qen|5Pflw-`-CHv>K3u*-|oxzdf{FXIFET}X}QM-D<`I9DrY?j`S#HPnp
z?GX<-n4#+J^d_8hVzb7}$&0kA4sKemcjm^ty~>5Zk9^;k$Ljj#YuwxA85d{CZq*Ri
zTp`BN>cJxMbJkVH(0iqZU7M1(rgK&4RsE}fQyV$$&5nt0>er~+J?N|aFI~-P%*@C)
zac1bMX;=Qf+Zkl2+{D7ze)xIgXYU!Ab({aHJ&(2gt)p~?+jG;Z1qc7Hm^HgXTYk<~
z&E|QWd(J+SDE7Gi-SXAmxtAL@&3&@{(2qG>;)zf8domVowk|d@eD~4y3cq2t)31A`
z0nd1(zqS5wk*`Y<x$$Dd65qui`OYc0^PX|>RWLNkvFW@RvC-6!#q#2QQ@xr+g<DQp
z&yG83VwiSNEz8(v&SBr5wLea#KR1g!`g6ni<^|HpSFUbnR$^*fxO!bAbH=}APX%5s
zDSxn;?UK}un}<&p8#-KPo*Tbo+Tq-Sx82-t^fs)1ukEmIrSk(vnK#n1r|;`T_H$43
zPrv@-N}(f5%%W|^b!+}y{Ha=h`j77C&NI;oNuia$j+~m=xv}uE)BWJ5I#-x8%I?43
zv-NZIQ~e_5rN>l$?h0Uy$kXBdnKb+8i368!9x?Xx^*nXUGibu3OO2C`Tv&3>S=g-e
zlJ}&2lV03%Dy@v+``g*iG+EeR)pVYVerVD&FRy&l+|yEOg+&=2hYwX>?wb+uhG+7@
zrJqh7(wEMCctPhXBkRGtyI$RO$*i22#xj@3bj|O|FQTt@y=t^rA6ndzy+EZ_)7M;7
zigDJpxRR&6UY#Mk9p-Xw`YL|U*FP}H%qOY$i_7(ZB*VEbZ<ETdNctNwy7sBKss#mI
z=$)}mgzwAp>$6|WEO>Qrm2^lB|Gcv*%L6AlHED+y%r|~;a`LiCOI9wqsp8`&TXSCW
zqf6S<z2{D`g;sAl@`_o8U!JjOmDJ;XUEa^9>z7W|N;$pxcKL(J3aZH&|Dt|qPm7*0
ze_>g!nfLQY?|#kOYnOI-)e;NqklR-u+k|v2Km51iimA`S${6NT;p$su6c2Q=mI})n
zcGi{)J^ix8N`1<S$$Msbo^|bE-S(k&ZurDoQd=1!pU+N9{d~4~)<pgtXD+<C9xl0e
znu<YxcE&C?UWMKJ|M=>?T=dxKc6Z!)>4Iz0sXIST6pzf@P{eg{vtFX|6OEvLcI)Fa
zcfOM8d+GMdEbPU_8I09CitMZ|zgl{o<=D5ldz=2*{qgoxs+c6zYxdFN_byG(Ylp9v
z_U;j@jW3b+QeWgYIWOba!qC{CY(>UjL%m{o>wSJVq-jQ)tHg<_oxP_OYrUF5Gp2O^
zAATLpc>&i&a;2E{T`I2H>;DNUeQ0%2)Zld9IWC*cyE~=+rOtYi{{D$)NO3^TF`d-;
zszq`KxUU>dJo})A!JuZH)PgJq-!5NH)tX0I#U77&YRuZX=SWm*Jm$U^@cI&m;>P_q
z)w{Rvem3>P+wNmc$1=;}eh51`YA#6Qnmf}l;<2Hmq|kof8oSS(PIs4m*na4~mg2`%
zzowmEEqyNMyw+(~fh$d~SM6MMVD_7{#`EOvNliYQTz8FW&w2(mk(+(L6wkH3YFzSu
z>Ybcfo2UM?%xYN0K0D3g_l{LhTeF|tk*YM?G;2rFS6QPT?!0w36va%^Ew3^^<7?Em
z4qg{)uzatT@vCm$`x9okADMl$<Le%w=Mgq?ADh=@U)lO%6}w5#S;ki~3tk>v#~w1F
zojW|Oa@v0-W%>Fk0tf5kyKGo>Bw|giiav;Kb?E%#E4IpG;Y{8FCI5maUYfyaUNdGI
zrKS~xhIRg4@mVI_{K7GHt@!~Ty_zBu96ldfb2H21MZ@D&ufDB|(EK)gvZ~GMnK5C9
z{kNDLPg}iHPHc_zQtpa*^MCH%6CgQVT>nj-!$uq9b(+SfUcPQRs^-+t==0cdf~Z&5
zno>)>sqtm|7QYm(f4yb-o`?gAFK?(GGky?N`m~!vLh|_Wt!Ejn?!T;RnBc*4^R&_Q
z^qHqOuX%i2DUjQn|JZ`nY#mK)KK5_CCbI;pW+yb9(bqW5a5&>zSl#O#AG~`!n18Nl
zUMD4%Ai8CG`m&DnXHMyAWhtjtFIJCw+}wIU%r5?Gr@a~9<sY5tJ)78mt7N)OFxb4q
zYaV-i+2=ytUvckME}j&==e6}Xzm$h)#t#0--xD$~$c10$iCIwSYEjUvvafGz-}=-!
zXCoh+%X-+>wdRlQgG={w7*BLRc1oOlSVG^RI%P)H@%qa4UBzy(KbHOp^E@BAPxtf3
zQ{E4&cP&$RST-%%B5tMTp*QWT<7QvGqP=Y8o_in5YMA1e%(OIKu6){Jw&vNs?uS8N
zZ0slAx~Oti?d*;M8J~Hqks;>#m)Yi~FLF=g2ugDh@=h{RY%aezO{bo9nMr*1pJdnX
zuNsZkORdP#-TOl7`5x1v%gnj=7JOy>yYA<%AlDfEw4a~XELdn$of0<pnb+-zWj1p&
zbI$cnkyo_-<CAvp#iJ8KG0T4|pZ3$NR$lMQyyR!I+!?72FN!a|kBApuKRM7qJ6rkm
ztEpEnX|8U~3wwR#PD#Y9irqekcW&Guv;P9aEK89IYu>yl^!*iUT)y>w0$)^Kl7P-#
ztKTm-xl4MT*>`h_niey&{6!g^&Bf8nWGhPoA1puS-#b6eP(w&cLv|)_V}Qev(y1F&
zUz9d(4|ICxbbQ^EM|V`;cCn?b<di(<rBQPG{$`eAZ>FABb9k6>vv<mo|J&>&>e+r6
zAGj~o;J4_IT*Hzdp6?DOdN%IwWSw_`kL9EzUwP4lh2}oSPYi?uC6jK4TLsTiyz*g9
zUF^csi|Ren9_E%ENL$Rm<Ap}h#_Y~n9S(Jx1`cjt{~rh!tefSP-@EatTVi<s5;^Ht
zryBLObEhlHtm(F6+t#hI`5~L7*)6>fmu^}ebXXYhZ2u-k(`pV0YpxAf8MPdOUmTmB
zKi$UQ^f~1}&09Yf+jV`~>{QyyyYXVhm5fI-Mb8Sxoxa$`W?Pl5)XJ^AfTz@at5c4S
zQ=`(YHK&*Sd$?x8oxmb?%W@O*(q|04T*qH&#LQ5=T~K&Uv`@x*RoD8<QH`f|HhwLc
z{p#hWBE9UdZddF>bFVB&S`j$a?_+NIJJ%~qH*Q<LIYFI!#-=mMwyOGK8_#G2nskR6
zP3-X75)^SxfGtW?JCxH{Ql05;ZM?zqy)M2_j>i9pm(ZVk?Q`IPUU^^9^HQ~+vVP_0
z<egK=T$#OMc34XP!P_yR6GIj=q^UKeSj;Wi6tQ9TzR0FCnNcr($<4pAmM=!$KFj2l
ztKW6G%L@Yjuh`SS$gfo^c)^6Kl+x+nd8U=eW=6UElv@?KLhOjirNxHXi%WG*%#;;~
z>*Z4E*fG)C{KN^9bt@#Vn11#9Yjom<fC!5XTZ4sU+Qn?qgAR4c0l$(ImO6!UR&Ta&
zIac~&`Qli?lcFb?na|AG^kNNLV~WPqH*%?*6Evk~+HpS@K5$F?ak?mr=dQVI+dL%p
z)m)qVq9yb3)zDY{FINN=_;8)~*fD*I;{3)-2d1*FyOY0F_I+;m+ZmhAS@3MxGHV^j
zx~WmgQ`jrd?pf72|8nUO*YX#<y=~IZF9x`W%wEgp{z@%1UOltgvU5_m#@YDPGWqw;
z#}ez;>oscRo@f`;dMC6XF;()uve#VGGjmg#RvJD$%QLM-`&{d0yJ^}<M+NJf!}q7#
zUR9obP~-m5$hSo<i%$99I`LdU@0i>64#73~M}vNCsqUL`_(W{j9EF{#-}HhyWj1O|
z-aPA(!>gX(99(-7=CpUk6<YPae?Kd4`LF3ya=I!QtoHd{*SNv{D^&Tr0@t@jKf~#V
z^RH|<eD{msN%5y<wl1GO?67hC6j(Xuox%0qbL>wSU2K&TTWQ%kZ^0q`KrvIpNj<j9
z3s={eJFc6x==`zWY5Tq1*q1K)saeN;PQFiSx0vene*fzt^3|GKG$%P;ejIbJt0sWK
zVsG?QTUq`5pQS(ArRp5)eS?fA7yX}nXsXcaDZJsGcVt4}@mub`lX=0<X{k-_Ym?P0
zEK`<Uo#rMz&F<_Q{fReOyVvFxC~jZu%cIC_5%7ss`<tro$7Z%YTCb;MY5X{|W?SY5
z!&~z*^3H8qm1eZ4N8p0a`My(Y4}O`z@8s~cSgT#!@v=umk6P&B>RSPq1zyVhi0Bt$
z7u=x`)YAR&NM=O3<P-7j=Rd|@c_Wv?wDx5F|H`%YvXb+dmu|n$zIW%{%AfoS4i_4`
z6crkT78g%Hemy~b%SVGR61~pCdDCJ!<ClLl<&e#_lDz-yWA4GTCd^Zhue8y7Gk;fM
zN3+NacAxqG_uBpa@VxE+WpkC-&Fh4n<3j6AVt21A<4G+0n^oKVxK-KGXw|}W-d{HL
z=YC0YU65HP7yr+k;o1LVyk-Y8lmGKIN#ElUc6q!0FS~u+hZX9z?`}Qd`jwR<mACf2
z!Z`+~?RGsC_r7Yj{cV1k8KB-H^ZfKb>4b03+v;DnB}XSeTFzrBe)SJ?j$VhzkD4d{
zPwwOik<IUVY|kpp)U{9gmC5J(zmg|yI-)%Bc%<jYAJ%7sCu(@_75uWWe3_@sb$JWv
zAK5~e!sop|wIQ-6jQ8*|rj3UMPYW;L;N9%yC!=zRef{xczmI*FoFW{#KEB*9BW;ad
zkMyaUV9ESN0Ra_1_9^dRe{<@NdjqfXIZqzxyY^dM?ax1c5cGLzs=CAC2Q$AtlCKDV
z9>Jl@vcyQqZ?)i~FF$wJ1w<TR;G1;P(#>{3$my@Ezn+b7s@eRUB|_bIDWjXx4BqE+
zq$Uczyp=2Dzah-ybnd?GvhQ<0>*nrBt-1TG&Rpryg;uwTt@XFn{z>P(cwi8qt~kNi
zc7ehgukGJBSKBoh{ggjoadGARg4OTutbKpRSJw1Xb=kUz#|tOCs!O-2^;+&<S$}uK
zmDcY%Z$I2;xUx|B+<v~5N-|f@D6mf3{y%Ans`@PXdPbAO9!^icZr^;UFg(z9RZsq{
zvmSq>?zSq#_Qc5eA645Rr6T;<a^du!OS&1ubpEVmvM4^r=6Is=w@m_D<cU+!sj|<u
zb31&zza#R$cAkMpleRa9d#cNa%OUJW@}Z|&j;bYeF|YS^x02^&SR-bzE#Xf0i69NN
z){{;)Gdh0B<hMO-;Z)l+ZI{s7i&G~DzQ|eL`GL2f+P*0N7|)fGvj@KNuH(7Uw}4eq
zhS5eow$|5YeyZJp?>9f)J@CtR#bN%P`(J&G5c{?;gNcQ~&2uM@#bKS$%oz#E32x~J
z92#XxI+(QnC5o^=+1%hz;9}yeB~hljrN1S?!Q91FMCtg76wxf<GXm`monj`&@{=_>
zcB}ilmB}2Iy7^XHL5k@Sd(WMeRhxtwSG&1AoVZfy%K{;m4pX*p*^lw9o{Ve6?RxGf
zES>vp*QwwYXBpjFy_qt@G}d`@7fyI-d+4@-UvbhA#opsRib`9jos_&I`YpfW@Qo_2
z);y_+6E1#E*plP^WQLxlpwPzOF&i&d$*5i4=;!ifCvStQ(i{a=zWnGUqfm=m*C!OO
zJfM9xb!v;ioa~0BY|o~z_Nr>TXd}>m;jj1#j*ub)y(eqczSUdrSk|1Yc-q2yQAt~x
z)r@Ng6icSDo<Ei2E^oo!9BC79HN9kRbEZXx-l6Q9JF>ofF*3?y_S<!F7rT1bWcw@I
zvw}WY?3r0Uxim+~d&7zOvlwO4I2LuP{y1<ZwKVI<4VFLQs=^lQ<8D^++({_e&^}fE
zlBBl)BS)j4Eni(gOqc%F#}}_LJ#qRcGRyM#Lzx>dA8@zIO`hItx}b|UJHz?UnzrH|
zD}%!}nz>xOVUxn<++4!)<w)pjA=PBAHUC_K&S-h<NST?J#Q3FVrOle7axZjcz1a<Y
z?wAHShb&6gnV1sjKgGn+a?TO~r=8tGr*aj)vZr5k<dHUu*P3AE&R)s#!1Ty56}N(k
zMP;ov9{dGS8&%zJpXo4I!?L)4f8>TtizvrAm!f6+*n89pWrYqe$ZCGHM=<3;Vt^9o
zf;T-XHLLT!{>T>lQBcS0dW~(j)Tb{7`;<!>JGCF%t9qMA77B_kd33kZ_@0>O>0{EN
zl6+;EDGO38su)s3*dLzyHM?`)lBA5%pU3)=7-l@>o}9?y@cdWL-g_HZSD3bRxqj(7
zd`6t{aD&IsjV@`YFP!+l?3vxYB`jMyrY|%R()d)NvdQ#PbZX#*CeBm#+@gCXUT2Tw
z-^t8&T%+b}{He{{eHo=$At#07BwlKJ-PtV_$yCr28o7w;W%#G;PZ?6&GnhWI-JWPJ
zn0je*+^ieRc=y`gyxS+IB(m;t_vIV==WNYNJ|D%r)jww+OImB*mtF~pnaeG5;wSQL
zaZD0g{QP4@>eDsmv+_kBu1FE9h`lN0C+it_=Hc2|oUHjWOBOyT6)@zA6YsoY`ohyH
zv{1wGaltC>du=R>uXo%KZVPdmwo^-EEnCdd@DG1Gy?M>AP53G~Us=y|PL%Wa(yC(5
zyf*PEDg`aaFEkk3i8~zY^jGBQWG?2TnPr(R2`tBRESfK@iQxIr+jFvyIdI0O%at6S
zcdWnlUp2e<)@@#NSN#I@V{DroYRnq8Zok#Ac)c8}?k9$SM%Fq>jje@h9R^LuvpYhH
zq77Ft_v(tCF?<#+^6=j|75g>r_5m@R!uglF8m0%wyxo<!Y^mZGsg2yru6ju;wk!N>
z{Nnn2zSSiu&Izk)ET;ALB<^~lQNFuL%hSuTC1k#d&n*`XuM-WkSZbC{OHtUnB9E;h
znDf-+IZnQhR0LcOus(mO6rgM+W+%O8?W(8SSHAUMq+z*1Xsghi1)f3=G=j{Y-fAd%
zD3b6{ce9(gT7W7~vqj@eBX-`;X&kZ_bYJT#1g>M+oO!9X;=SXtIj4PF4otjxT4u4=
z``u6UR&n{{%zs(mw&Tf6TlYX=zdQyfQOAV_63K^On6>#_UBbnnyJ=0pgfl;RUQV<5
zuq0+~=Z7URmo_`>Qc9luAo|z}iS?6zDBYY;{qTsIfWhhuYaI_9+jKlo$L7rrK^LCC
zj-q_JcRf!vO_Xfzt6iI;_Ge#urO<^<hdeizG<^|a`P1Xb9m@K3iX&s|io5fVrf*o=
zxzRvv;aQJcAy0N*<5YWkM04%5P)Ro1BWoO=rWvYVuoNo@o+K1$Qa3yO;fmW_X6Dm4
zSEWzda`RZ1-A$L}HU@bI7@qJqcDg+>-?Xmzk?%3t$IJZWd?n|8;h)g|Ibzc%riSJU
zF#-F>57|DxzLe|uSD{Kobn&dOSptcl=fz$WQ0Q_9nEIh=aTG@gYmioO-GhpK2Fqlp
zZCEy`Q{bS`SC)oiw_V1XOICgo571w|XOT+CHHkE>+VI4@V{0_SgO3VrTzOnFy+6p~
z$14-HUr$AjaxJx<<haRx+L7?HK2n_bLVP6D7Zln1HLf`{o7I#tlzH{7e`lP!m+-H!
zKj|XA;rpT-iDR1u4>z#28i)GDC)S+%!8&u9LPz|Jh>)OXykhLe98bTW*!uZufY676
zWuA;qx@CWCZ!<OXX!T6k>w5g`qcWadA@@$rUdYoK5q^JFyg}Sui5kv2-+q3lIrj=m
z8NSYt2^CrCYV8&9@@=_F${djkH*ZU_Y;dZm)MVgU&L<+Ye8Wnnd(~kfC71S}O5whC
zo5{c^lr^|umWb9A0hU`l5lR<7_deiX&LS%@+ib~9pOwWd%Xq2`(|$8OHk*~xmaz1Z
zldp5KZ2r}+uMM8>6O8nFDJ|@l`Q&4H_}cB}O>4DRRU~kE6*rc&i}(E(5v!2U*Oly%
z*3UI5lfG5seYS2h!`{XJ9^6hn9+IKVEOuO=qVS*%+j8cM$(zl?4dR-l9v&!9Pj{4>
zD3SBd!t=0<!M$~#=IM(Vq|VoOx$7vB@?iaDbH(Nc4J(PQT<X3K=9cR1ZjSl^RvMxm
zivw9?He6cpZU4<Fbvoj|yj{Qj+9JTW)ak0E`;~JQ*^js)w3LDyJR*wH|2)+8effqv
zhC@|4W`WYXGB-)L2Nsei0ve5ZJz0z7{!M;xSvr2x9*xKGFE17wd|{d{H#OHqPV=0x
zbHZ^&S*zC7!CbH2@+$DS8N{*wky~@A@m=y}uCUjW?RPv9_td}Vah&np0)u6an;Fd4
zUAfCxv_X>9*^oEkjmV+@?QyKBon~B`SDL@=&AcKj!?f^EfZhs~`od+WlubIkbZ5N_
z@Ne)HeE6l=%=!JgD)}?P7OOWrn{6hm)h4gSR+F}3o4;7pf|MOT`<+kEaF?@^y7(x&
z<=PV6{>|JvNj(1pOm4_Fm#(WQleDm37B9i;Kbv{4r=O$VVtY~N>-*JgUObNuz1$`k
za%}s;R11+0j&JXM5pL*eG6~pUaMMsAR&Vd*Nw*Ggn-)I1nY{S>$9rcsgt1gP7q~~U
z2dp%_n!HDHX^6yvzj`W`W(Vg;miQhi`k*nl&*|~SDZ3Y@Sv1%6q;!4S>02OgJ6&O+
zx|XD$Y5KNBmFZE(L<1i67#=&f!TWYh&63id+1j0R6*PEM<qmJS<fF*9SgW|U`|B#1
zt^C<a8Sg79i`A?*+$u`VXAjU;o9t;l>*-FVwc5)XY6R98ybiPa*1L=C^`=EBk{8Z;
zUR{tY;A*|`a_C?7g$Dz^Z!}*fdZ2Iq3AL_AVaH|Id>6(adfM5*QPt3}g?p*9i0<i4
zl_91dd=d<_CHxmnW4v`M)UCl$dc&uEQ}g=X6v%kkO{|zVk-y`sHtW>i{k5#3(tlZ2
z#2M@o6`FVYB(Kskt422?H)Fji-0cNT#%Irp8#1dpPrdA==hhLHCC<F6^;N6)RyNbc
za~Bk^UbIKR;9$wDJNH&e#CUF&^<w=S^vUtTMV~N^qVR=5A513+Wau|8I2kE*GV&qc
zghd-qscT5hQ4dh<F#lrv<4l;*g{|irN-mhDpH<5c>{?=UY3s|&8Nw?XH!-rrzY_Q-
zw%>f-k3tEFAGcJjnzwCSKP`A7LxO@-^Df>npD86=(-!5fNWD;bEOZht3!ieD)vf~>
zE0+J9?apDv!tuMpY(nzEW6~}cUj}^ozDAixmMJ<k!iFPc+l8wOZZny5IBIVe;gEB=
zqPo%Ca-BDu@D$%YM-&CU`LZurZc4bNd40#9J$zi(?*+-td8OBF){)2FDlGoX??=({
zM-Mi&TCI@Q^)Rq(IIQk<++YG9(<7C<4enN*(yUq&%&Lq(++yh0+`_gzQZ4A3>is9d
z0-=$fhgW`8)#6)GYW~-BL3V&_%&onXS848%FzKCCBeKf1bDm9AfQ&|n^G`v?#f7?)
zWHxTk5^4?L4-~2h+achYoODvLzf|t*#@Y`DRJ!g;ev#VMyubg`pPMcojkZg2dRL0w
zDEH9mxT~-^I7ayd;}egH;zXYgk$dq0;wzeuZGUp=%RIgAKYgD*?wjz$K1Xdu8(04;
zj%bF=-;;O)lu8ds?{b>6HjjsE!{r4gaUV-tdp;fecH+7~W=7K0L=joW&@0^v&5zge
zPASRYS<%C^|8De2u@4vAc4o?OA4!xr<)p$a{Yv@Rl1a-$`<i7Pms($8vC_FLQTQiP
z)aCqT%Q*8T&)EJpEDe#mb>@StpvB7Xip33WTN=MP+e;j5VzXD-^F`ynfTU@=(dLpJ
zyiXlkFHL;JVeaPGeA6RqW`_1sgK2Zht!Jsbgh?A*>FF2fx;WYFQip}#i4&oci)#XV
zCiWkirjQ@@Ldq)cRTR^vT%);l9e0yMYpcqSM;~0ZWu~%)!sDsQ59VAwp|EdeCBuD{
zD@`3+-Bz$x`MWGrX3{zC>$={D<LBbmj5jrs?^BXL3N>yhG2d`tk+W`q;$gO}43i~g
znpSqLaCPG9Y4WPAy4L2Wz~lEKyt6MQ`MTEj&6g9MZ%fVPTQfB{z(~(&4&$>`rLs}m
zg{Mn>VG7+`t|sd&xoX9}!%4Qhx2CMp3Hzn+#Y0Yu=?k|Ahwp)76XneB-G10Dt>rYu
z;qHc|tTJ1BzxU}cV7wrE??ng0hQMcTVmmHP(Y*9x=NlCc=QA&ETyt9;ZPRt-i_yo8
z;j$JQ@<;ADd~LYUt@1MPW$6`xMIzR(IIK+l#G1T34z~!YEt&gOV8P~d^L7}diL13F
z<qIACtf6*n&AAf^Z4*|iKX5Dk!=@)T>AgpP&~vAIn@=-*VQJWJ6gp{MyNhq5Tg2A2
z5^nxobqfL&vN}&Xi~P7+Ehc??#^m;x6_ZwKxh$K0X|mkY2d?M0Z8uW)T)T+H>Vh^$
z^u$w#Y_>1CKk=c_Bl(G~KeWEGPUtb~&!|{^*tsRW`xv7S?=4Xeabr_v10z27NnDfN
zf2D@-&$-dUa@gbi)|C&8D*FsOI5ltkP3aS|a6C~bUR7$dRnBSdRm&hH-y|i)k9B3v
z;pd&2nr>Jc|LHUnoW;t&>DBRxH`<h6EI6Svt<^LkcH)wdl{reUSIl9N6ip0hWm)5}
zd4ku8OKMJ$(^wigPNw9V9d2Mtj0tI4;&*MegnC(Wh`wS`%&+)@7-5}QwknI79g?$H
zJJpU91h_7DPu{b~RH~aPzQcFDuz!JyhuF$V&3b&BVpldzT6<{6pS-UBt6Y!l`mm=b
zQINsx-Ws(qwo^BjUT8L2R8rBF!oOyD(F2?B9ZZLI3*OCJ(4(>P&cbvDu7j7We!i&|
z*y*#we#56Dr<ndaeBwK)6J+RcJMB?)_)ERE6@k-cykTs+uzio}g2~QG>=$27NsTzY
zX|le`rnVJtx7<^|*;XZB@zf>VHDr~-$&|FZ-OWCHh2<T!3nmAw`grNY4YN~Sg;~@2
ztSt9(t4tAEu5xPwEBlSZv%B0<Jd|8r#I7xBIi1|9(W+wd_DtW=!xLv#Kj3_2E4(p8
zL(A8H#V73-EU#V|J6|*^<$7c7BhR?VZnLh;6~Ui&8utC=EB2_edK={j9cJ6eb)@I!
z!X-VaGq`0+1gBhZ6iN5XPto97v~>}SPFXXHep}01h3*RR$J;aBIWzJH^|<ibYdvXa
zDg4&U=&7S|tM%Kfz(<RNcR6ocyPUJsY?AbA_l2R6Q&uERJl5;MSMfX5D6#mPN3(m%
zM$Hq}0mjQLnJjk+WzDpjI^pBX%1uvB6?uL9pw+lbQ6?*GGJ|kj$M#9VG85~zoL;sv
zUM2T<cOR$3q^ZBHbaXnVtbG4v-qR!Pz9J2;^Rtev5qCT}^O)t|vyUE2@SIh+mvF5>
z{_s3r;q$XAEMCYSN;|QaO}l<`r^G2&kw{sq!#-TA*6h3bwPa1onJKfLTYLIDNk{XU
zSIPG>ch;o)R8N`T#LB4=z%Q|PLYs9$)WS#~b}65?0s#W2Tq0PWoZi%+sO0<5(f`D)
z>G9?c>pcYWCd`?V5n1#@mQ6%xmimXetP6cQcqE=o`}$3rv#=?@?;GQb=FTIQuJ5E*
z`KV3Xf1-KrlI^Shs%=wU_$!cex#InUdt#3r5sKql;_i6Wsb3@I;;{ou1NYdS{KfuN
zsc^Hqd4tMFmWw`}tIk<m)|vFY!cO>D2dCWkbPaJ&rI#6JT&1sG&^5T-R-kq1>_JH{
z6|+UGy!SE!o_k&6dvK*sGGxnyWhoXNfs7T#TPojVIM3lIH3$z$Ib<nqcky7){gQCa
zoZ0X8bA8wp7sV<j`B~RZ$!`{4Z&zuWMQD4e%#LHPT$eORrmkT(6BDs+u<ZDna9L-?
z^+jfh8IyD~T`CohwV6%{I4*QLKy`+@!Ksz?^Pd!)$U1I(T)9By(L6a9kqdjzaNKKK
zGTlYz{Y3U1j@*fNG_<TXUf|3*e&ZY;KWlSD+A;4%i+3wNRy9;pxgf-9y6*CY_U#Ae
zNM+n(&SHOaBEV5AboG%x(pT0^oFm!wFDCxo{m_fLE_W74^~zri^mM;2@#%el`vdF5
zz3e&B!J!iG`X|m(W$SBLuu`9Gl^8>M<e7O2>Ec=oRAPU+9XNcFVakz|JI@&V4m>hx
z{%~WS^u>c2!H4TEl!UAN(hxqvGbf{+$*ErDl}Dd=a2-o&bB^=#e=5_&M3=JXCSD3&
zoIZ((fko(&)Q7V=91FfmbjYwOmY6M4V)A|VV$G}<tlq|PT4$Gh{Hy;-J3O7&?_B!&
z6jry?PsJC6EsPjg_Oz$yUpU>=xBHLnCMNc;!D6BBt^Td~aY9dhV%vr@0zp-?Jek}7
zESt20^TAPnhWJ#MOWa?>L>rUdN9s=57I5Xr0$1TJ0fvwCLnLdAxx}6X<heI33^GV4
z(Rnmit-W7mgH3x_3770=pUVv2j3&6am``73o9ZDVm~m^#2c|EVMcf|m4{;Wb75@@>
zkip~B|1kYoA734GDsYfo((r)ml1ok3kpK>hGs|W~%<!1e>Cvbd_|i6cW=ora$s~=*
z$0b%ICy7T2xc}&q7BJSlBcJ+GC^tp2uII>ugyb(S37ONI-!aDB{n9z@k7lSsn7^FN
z>&!<YUf)yZNtf-BT(57mx`b1rk;{FN@sxd&^;W$Ux)*WrWOJvFWu$k~96l@a1~HQ=
zqa80FOj61Y&|^_-mT$=~|9j?#nSgb2Lv!;TMU`i*51y3=ObHKK(>%Mw&sHfy%39b#
z?Ucn*sYAW$y#Fd>juu`&-Fs<DQfb7UC%&ak>i1OlI7)ZCk1yIJpI=dRP)dH4*Q_}4
zO95Ln7Su}Y^~_-M3vp8}6)ljna^p}D;XTeKwbg3E#S?Q{D)v<T)|Q(su3G*5)THpq
zBGoEFm-|Jw_9t+uEaj;v<Lb@PdcI)6+?`VMD*7%SIq&K^lk<C#SKnd9gU_<goJ^1`
z&(*myRrBaWNx9SrKO5hlZ0{5UE^^5)>TX-MJu52spUf9A6_IIYIf@h>d9kuF-p;r^
z+4cBD?a3$YxROuyG<r#1y1BxIgOyuGWYc|*!zWB$iVN&<OxK!|P;#NksnI?CS<Z%7
zzI&Hvh^Kv8u$a&A@4g&s^+#=Dg85Trl9LaY*PYMteY*b2Pj}_JtT*NKb$p&xrmOkw
z{`j5eE9>(adYjFe*1b4V<^5f*-C!}V#FUdfYlT>~UdQOvx^yflU&DX0xbRwXrR(MR
z*Ze;?te-ram~`OGb&mHO4c^~7z9shfnt3(*s_o-t=ZtQ0I+?um6CZ!XvSjU#wX*jn
z+9-E3{B4=Cwd8=wLJp?g&6g|>&0J@%RwBI6x?%P&m;RpeYcFS|GAvS@$J6U{{9sjT
znJ1HzN?wt~ufunAu0%8OztJ(9v2ZCrFWat@3uJZ*XMYlCOJkjNy(4Dv(n6M`y(^7H
z-=)uFV{*&u^Lsus`GU~#HS;)98GBn43O8NmyA_mrZN`cN@+mdfKJGG_qgA>`PkY)T
zw>d3cj~F8b_fEa*yD&#2_fWyy5Lfd*KK|LlPuzRM{#<O@mTPh4*pvwN1wUuM`^hAz
zY5ar5Q`%z1VUt@Nc_AtimmQxJuP!N)`ox-2FFQ-1-;5)FT3geRnw0meyuucqTy_33
z^L;gIBg6aJ%8NElmpmsJU?`mwtZqDcvdaF@f1fQZIc%TLwA*)Q|L4-jHm<E%T-p;H
zl^06SRJA{@(k3f(Q~F58kzYzCS47mMkA=)B%Vhg?RO7G_hbWWhOI^hfHm1f9)n#!i
zO~QHthoUp3jb#owtd0@lyCC_I-{T;Ao3ZGXoi9!_Xz2(fzN|33#=Ge2`44aF|FIR{
ztp2ASm!x+p@ZI+r!q<4Fv93vPWfOS)@%o91fBc7@o#&k{E%0pbD&eMDx!r8%cG~^?
z_LBA5#=kS}6|a`Oq@E<Gu3o)x`}%!3@_pxJ?;c3<{_}3$f3F??c0N4M``^|4LjTp7
zomC=XKMe%3-)nVc?cO=<v9eIWm+$p8xq9qh>TBfQ-nZh9x;16*>O=f{cC8D%zC_#8
zYBI;i)>Dqtm#t=7BemgNiY;%%Vzw1h8;e}dEX%sE%5gQ@TB$9pws%ajK0khVKKsqC
zo!MNm*{^0fp5OKA?cP_qo1bj;iZx!>BBmY}dwoUrrBH_^F5#)0mS!H}mk2#x9pl?P
zi_P@<;bnokfA<`#XI}M3I&TG!<e%ToB2n?8E5jrX#z$;ja>1MVPspq(=NCQIPf-2s
z>)tlQ&1_cRnQI;A^xmA=+s{)c%Wu|q*~0I!LGNJ`zRMOqj}3fHC8dtPO#IbX@X$aj
z@1mv8bA#Gv$`bibrz%T+a=!>Ly!!Cb)>o&@XEGVZe+jUdA{b|S!RK8W?-MDHf4a-e
zb`>37%oqIS?SCnatOY9?r+7J5&(eM=9iNfq$*jg(@#W6722W-+z7xr7Uy1$CT=MsN
z-iwbDr<d8+o^7jL*CuwGf5YU*zf0;E3~#KPwtBkw242S7PJd_p-*xMBa)Zb{_tih!
zZ_amPUjF0bvM2WUzf4!~Q8`<mIHhyb{;H|mEdTGc{68oy6<g<far+l*EnluL?YTSm
z>CJUu`g!ur2|44J=gk`&<{k8~_HYcpe6_T9ms;%Rbzj&IS)9|)SnyxIg_UJe<LXe0
z=vj9EnackMOaFK+ZKLz+p}5qCc=Mb6<_-z7Ru{bEKD+<<RfUW4*Y=dfX|1xZx#jb1
z-@l_f-BcTLMOcz{Ia$n|7Ahm`^1?pw?XFz5i@kGaJv5#h&YI-9*7%oO`0Mmpsa>br
zpVz8?_0KNKx4-}8dzs)w3HdAg<7M2>&dZ(fe(?;RZ4(sBTu&_Z*!90@?w@RJ7fu0|
zhu^rTiPpydUwlgGRMMaGIqxppKfCy>UaWoI&gc6RP5)ouZPf|m)N9q*_Th)vrP)rW
z0#7Ebj>u8vUD=ZP)L`o25WZkb9}r`Y)bUK2<%Ki8omp95$hX-g|7DB`!;UXscZbxf
zyFaa2>>Jv(tobiH=f|LP*S=TU3FzB<+?p<x9<<=vo!eipUeaIkGBW!*`?~xI{{qh`
zN6*fluxXpzg%e9-<vcz&)>S$@(4GGN_OgQyy8lH!xAi{p_UEzxU6((szkgs_^4y9a
zy@en313oJG1=X#LVz~do;g@XXQOQ|*yVn0{_<PIj*~Ha(|K4vf`L*TQzw>+8L@&(Q
zyZK*q&z}G5q;>Qja+~%4=bgi6>Qna6Ja?Asuj2)->E7<X%Nn=2Sarrt+M}YqKjF31
zjg&)Lmjg{bQ@jLCIG?3m{%ycMAuREdf{&`lfvpMl7oUIN-Z_)a@TcNZ`9Ig+s7h7~
zIM1)KI<ZK9Q%Cltsd~|oUz&GhM9friWj@EHw#vY@&HiWg46(l_Z>UGlJ5yYA_oTGC
zJ9p3TqYS>vX1*IU4=iK6z2?C}#@nkl9Aw<(_GVVn^@dpHGTs}e3D+C;o?}e=*28t8
zb;0VFuXee`PF}~Pr2EfN?0@E4{!QQ5BY%fR@r3hcFe$D5+bAg<Xn1#Gs9wd@1~czF
znGWIt3zgkjkDrmM2>d0yPU?c`f~4jJZ#EUob?cdxbfuwdm3yLfjA_D^hK=VKO_HvN
zOlI?u`4GKloy~)(&!e;cExY?T&2sUR?n&xieHuR(K53gI{z_GU%A9~tjwhK_*{9U_
z7ulZt{AJQ}&p!2?bDxxE$t)|JaV$})z0wp?9?U&_LQ%DT%ALSZZhN2fPBQk|r%}1+
zNr#c;BA-5u$;w;@L;g6eTYHnu0mM7?q{YZ`;hAO0zHu6p#UZ>GDF&93K7MSI4ev>?
z95vfGYf;z#OE(jW%x28tRZDL^c{Ab6ZiAa{J{l)p|M_2Hld)h$<D)M=T87Q{*p{V!
zTE#e-%}2^0^MHx%WKU%^Utiw5%(;d)C-;;l9XBmLAgy=$lg43x<>i_G)t@RdS#qDa
z&+<&**M5^t|F_>s*gUT=MDOg!daXD5k8bTZoztKGfBGu>mk$4Gg7vo_jQjgJ?B21L
zG4?M{SGAR>cPBi3%{A}srukpjZjdo~KJks@$~yLo>(25#|N5<rUGn8?V;_&=|J^DF
zlato@dK`MBay@5LdcnV?;kR`5<o?Xpn{T7u{XFI4oZPq+qyMoc%{&JGZB9M${eSZ0
zKmI!oni~Snhrf2^`1CSoQPa-kFkJ}=3l^;uo{dtLslT*lq#1;&xOUIq#J6OT{wpal
zi!*cVr}c+UIWS>m02{x><(1A$cK<tj?{(o?u~RPl+HWSf@P1ged42bnm3tz;Jz_c{
zwZ%YmlBA)y{+><KIQ=?&doO6-=1FC{8y^y4o?RRGku~fV%gN8b{eD|2toyw{WPxxs
zOS+tq*S{0aLeEXSGZxHBozh<VhB4)7RaHk6+uoxqHfmO8&aMfvU%S71*=8xRxape9
z{#L1E9cE#>`^@O}l>)XZ(RnM=ZQil3lk8{oy0`M0yIP&o{fj9JxX+)J`~TACXR3A+
zr*@b7l@n$@OyaKXzgjsr`)OYNyJ_EHw}SQwMqFn)PBFxGHw8^wb}L-t$+Mq_V*Fk(
z-%g#fu=vd~;p>GJ{gZFK;n`GXso=$J_m(}S;me%;9Re#ImDrNq&SgiR&QJXvefpT>
zE9<<#Q_G!J_0H&(juLv9aJb#Nkw1vHX@zgHt)+xCLrh9^`P|E&g`Oy+cqWO5UYfY}
z>jK5ib7f1u+A#CTZ&LCTel09|ebtTJghzR2)V+>4A7VcDY18yId<_kyL3+o_SM2xb
z>v=t6OTq5kxkvq0T#DPYjA60MtZmI|>n=t{bpDk()azg;mmnpo_G5?WiOzN3tLtUn
z2=1|v=nd^J`(u0I-GaH&vsd|?mV25KxaBOr8wZ!+2lie6+aotE``;A)kdH}z-R`aP
z5>sR|D;jedpH$5G=Tdz_rMq$G^%y3`8xd|I)7Jgux+>RJ$&(T7`jR`-_c`N{6ZNin
zUkcctMzyIiNWA&tRJz+T(~B)<arr_;8P)EKdR1NQ?FZ)R2`qV8>?(UuqrBkJh7;3*
zIwSW-DW0m#)m_sdT(;rM#B1jsU!1n(z@+Bm+G5Y;zqTAVds6qnzPH=@ufC1U=JI>%
zKS;$yMbCSDa0<iGBbR^ds0=k#zw+kUao@IsMd}Nhzo{06UD#GM+oZQ5>r%_}@FG9u
z%9g*M8D_ADzRgk>X}tR7$kLyFBJZp$<`yX>I^NwsIpFy#Wy8+TOP)^X-lVg7=G3b1
z(`#G27N&6je)08#({(|{nHI9;Z<^<*Slv6sTcof+GV$;WpAS2f?O3-|+3CDtw}@t2
z_ISYoHFMSD=e~6vd8u*mamG5SzNB|Qtk*jJXt=>6dF$ta^2^7jl`3iSE!v@evuBsS
zqn5^#{>Z}ppKUjXYQMQOb1VCaVvi38Rv!_q+Z-#h-K={~?&_*v=2`4#i|@KBK9V#{
z?|o*G89hl(;L^*C`OgG3{5PyW&AgQ*_q5jf8r7y7T|0H-@|X^6)-X)j@#U;<hul7&
z)D|6iuYmL27gw&}X`OXQ(szf__2-Gcy7Rw^Xn6T&pMUuLZi`2GWbe_>Y2RwZi*8N1
z{`vM5#m~aqe@VAB`FVTWub!U$*r;4ODs~AQ^S#rrq$HG97hhjKeOsAb+hh3|UzApV
z35YduGWBd|+L@;+dX+8xmD#1Vx9g5Jz3sf~G54MKp`|%>*H(GQZ+~|`+<xEshd(A?
z`p>>wXp7ObjdfCj(tc_&=d)Ed`oGq#y?EC+FF<%zOIqyeCAU8}uD@DTb=jRoQGA-n
zfl?E_-FXa6%>}2p{#h7p+0-eg`|sV`r#EjE&Qq-T@VCt^La)!?!tBtBx!3P=7-<wU
z@1LCiD~tVJnb+M|yBR)})A(NBtkhPW>u;vXusGq6hx!|hIevR{SGiis8NB0O{!4a(
zcjgpNmipy&LL&a=-&ixh&OQ=&PjsSN-EGeGeg_V0nIOSqA3p!)#<b&!+Ar&FNJX-!
zFdw<%{W$k!t!cGG&CUYDlIt<Wb{E+<R@^>fI&)90;4;q%hgvFc?o+l2uoba!Sj${>
zao-wYk7l;hcM@`T-dFKhUtFl8HfQQN@x@}xHfApui&1>px8gX9cFjcQnq@C%zdp4x
z`K`FYr3oFgH?5nsyRb53{}aZufk~Y+K4j#4D>;&|ih<!x#PzwwB3FMEb$-Zuzxm{)
zTwR;fZOz4LJ%K-agK})=3+BtRI4<Adq2H|bZH~(PglaWm6K&T|@0!~y#BPYKxtOTQ
z^}S4~*Y%0V{69Y><Tj?N<_EbIbiUQPm%^RI6104)Vw<e?FV3Jf<tpMI9N&2I#JCnN
zoy7Uyt5#kz*IK^!CC|-k=XYg_{5d4FLgR7mp;;0Br*|g%ELLqVxUt4@a@1uT&B;$&
znUcORYy9l76T4lR+u9}MId9`1i`w17n<s0O*mxXy8S_^Dh4h4p55G*SdG>e7Z=)3r
z+m1IU`yEwlH7>Ve^X9&I`_K*JbK0DRwv+8-`mQW}!Qz|ovh?`b2F|pfIzkgJmx|e1
zN5s$boHE73;=mHehY`<}nF_AV)v{q&e1GOGon8K?^=DiO`hWdE`X4d3%&bb`WuIa>
zDmPp+@JUsFtvB!MraG~8D}-`?3w@fIS7fimnVodg?2YHel2c4x%Ux%jId=Y2=G?G{
z74fT<tELHCn>TLNHD~HlS;TyXO=vayv5DQmELyd)<<oz^X8HU{Vf&{v=GB&YYrI(u
zUJB{AeK56TQ+r|d(uO-LNG#*jJDsdbpS>rPR0QWM<|*l=1|>ea^8V^#E$xeHU!wOi
z+*MgHyQMs$KyOmxdv?Lthfb5<oz>Sc&08U|(`|Jy+rkguHbvz}oX*r0JfyN>|C-k4
zvsD)xyKMA2G?QbBtB#(NmBgRAHIdhSe;YaKYD&*#n*W6_$<=n^LesT#Idg<|ZGG;#
z-=}Ez9c3P_H=mxz)^PWx_jw99Kl<G48@Oq*<J$>Wwi|Z}we7Q3UlgO8E&4?(#((ig
zwOPBb`be%-JhxA~E`I4#?~e@0Lf_k3EaJB)C@=c+W>%K??&)9ocPKYXE4GD}DTJ(E
z@}uj~vdz!SW+@89?#YsR&p*d{cKu7)9R?pSowGCGf8B6V=IcX;x?kmn@xNv!wEKQu
z##&>QblCT|!nzODHxIlQC|=37ZzcDe`1zNrKUc^yTRFa77?*COxqjE_Et}I1l%IWI
zlJv<v{7O){X8jJ^$qZdim8-TJvuwJ&B2mcesYl4=CeN#KdnWvjeDowEWp1cIkwxB&
zwQqL5IT{uFi?`Oscah=QRW-Y>G#r-S6a8e#f011^wX>4B-zPt4ofX#T*1hJN%9QUf
z&6h^8-_>8qcU<m*;`sz0A&tXEy1Tx*2goZF-w$=rI>$3%x5j(-JHj<OvH!js{qAbM
z%=2;K$yHPNW{Tw8T9P2Zwcp01w3GK;wj$4ocTq2!|COwXiC#5zTP_RNGd<UyX@xJt
z{8!XU-7GJ>?^<paZ<fMfs`_xnw>Yl%NBH8OEWP8qr^jf<tw<HtWh)({>jHUeuAdQO
zsC)T&*V;g-8~TT4mTiyoGB-@VE3y1!l5x6-wb_NGEW0ioTbeod>HU=|rL!7NSbX2U
z#5(ummcUQyds)`jEjLy&EUBqtS(3G@B+61nEjDxe^ja4EUEK8|_GYfj4b0A5t}Jkp
zDmwFNkwbu=(Vq3y)>pfHe=lBi_0Zb{?yPdXm|1Ph9xuIM(QsSxcW}l`uQQ&P^5>oZ
zQ2w${|NQK&A05~J6^(kt=5cPn#+}~q`m>w1UH+S(*2(sv)>hMNqCJ1;PU%wJLeEEU
zA2V8ezW#ms@<&16CG%}2=4KY8{BqN^t>u?~!R%Seq<48$;oh*TOX}HQ@2_pXB|7Eo
z6!+OJXE@GR|H!(t`qv-h=C>mD7Z1Iald_E1mzr@k)$+vqfNz%{FZnb5_pitgnJ%qi
z=i6WZm|1=M{Aa7X*WOg#{Qt(Tgmrq`*KPae-m8nVJbNXl*hJS+FQRSBw5TV&kHf#L
zx%+pc{rN4$-PY>I=U5zNRn%N?k>z9Q+apc$=0B`G#N&BZh;vfUD^EwE$ihFdJ1c^C
z4&T!H)n|Vs@T+R!UtaUH7qbHI@;%hQ_Qht_Pu^BBdG1tY{fC>*zWwKQCR69myN;jh
z6;*rxWVZR=IdA;d$}_I<%ioT(d?}N!&8U|@e|USY$OW~^1ylcC{i72Q{af5m)AjW8
zl^3h*oL!dGUs$((<xex;_&rTkck*Z4`*Hp6W1e>|tLOUt{dOol=xd=t6`x#6x3s{Y
zJ{}Ri<>3lKM_(SE@vcRFg7C8q-JAloZhV*HqNjD=mU&rF5vMzs*K?-D?4^DG8rk(k
z<^81USJXeAayF(hdAr$XKbz)Hn^$YKo^S8bYrHD(a<=sQfGXLz#Y=*xvaI{kvoYwC
z{Q;}5pI>LTUBB~5mOmp>KInYfrH_&u+Y?skzR=Y*7Pz(P{kI+W{!7`<URJ$srOtcy
z$F5y>9?GBKh|E~@Gkbf_@;`A0wOzAMr@JKk6sYs>d+RQ?`y}_7El<zyf6H^xcFsi2
zC82D;dMl^Q6`NXdzrE;QZdHtZ%gv9B3w={X{&s(B*&TP`{I@1?Tg&r}2{IAO&+An+
zUFAPC{p!{TtMK=m6T*JUt=`%6?>hIE^^>+;yvO<R$ob@Xf`{HNlf9R}D?|87G!I{T
zddjDnzFv11eEuq#_FrlCdhw`qF3U=plS_PVs@>1OI7R+`7XPMy@1Nz(J`($a@vV`<
zc5AWqp<8_YKUZA8y>@Q@lC8TVzMYo5_S$3mWR=q73Bpy;f8P8uZuLD=zab)KtK8&6
zsh9lL<X%Y(=6Lt%;7whpe7C@Y<Hz#X-aq)$njvQYg4oi}AGh4ic<pmFDd|**^t--K
z(@(B$ZGUBHbu^`!WvSovf|}V~p7mm!^K6gTF6ERlkKBKn{o486tEJD&TwHv&@Aa(s
zukM!F7izbZB>k!nb@>18oM18E$G;NyovJ@KEp9hviMY+ZQRd6ue>;|5S-I+uu6)AH
z7h67V=-l^lPS3Fk99!0h&+tmVV=A{x?ah}uoyVoWJ~dz1$kIErw%}W}-1nc$0;2a%
zNZKrXcX5Zjokz#Q`~4c`<wrZKDjcT1{};=%^r0GmaC$ul$5PW!IlFL=dz1J5_z{&g
z>-OrylGpNH`o;I{Qs<ai`$2N2iP`j9Z!V?Zm2;U~yL;#V>7SNwdl}Klbfj(R=?BjY
z=XadH<rV$=?d(%og@)zRAFWu^y`;HF@J^p&htIp0lIQedFX+#@;o2C!YPIfk?a7Us
z@9;M(nOHIY)!6f44cE@71^FLkJ=+YwN}lyfk-aW4Z~0UHSGOdmBvu{!d*S^$o@rlO
zFRc79bA5^1%`2i4KX0;QUXgNk?iANkvh7tf1mYrYw|1|yO)F?W=c}y0@VLR{hzV0B
zOr9LWbmok$d$ZXuHygWPCARh{FAr9<&97fQ@95^IjfU%d*DZTk?OiTAbFo*caqF7h
zB767!31v=~$XuatcFW}KB9WbfEG94Ob1L*@ZRWX7pPv6~-tv|2)^n&XyZkD{OSI;u
zkMN{-;kO+3)C3F8eEvH?|Cma`jQP!dg^$~pyedlTZFe-;ygxdtD?NKrxcr7SbH0AB
zs=IkJWDiq>!ApUA%hwoXNeWjUHWU!q$e-+_e5z~licc~5-?#m~^77Ho2U~B&Rsa2N
zCE(n1b-Bz2cFt+vZaUQOIdX6L9M7Vt)N>IP_LefC+qay!ZF1jDpR05y`^Qy9&aayi
zdbhLMohv@3b9KiF!7Zm|8oAk*?*9BFfBLzOIqdPge78(zoTxc@QZ7u*X0pHC46|9a
zcMh>m|M%B+o95x%wY%nSTx_&<(nXi@PhqcR^J6<7`R6ZP_Ak?ZN9)_!CDBQTTQ8fx
zE4RJgYaz$!EvfY3)A<9x-`#&?r@G&MCdags3-8`bi#B_<D?c}Uckt%=!298U-rK*}
zxM=R1=ESmp7dySr{=3EaOMcJ7&ksr#Ieh1y_-YBq-<fmQ{Jzz7^<}igXVbqe=N9`v
zoy;-kTy))bky8ySTz}6PRaviJ`oG{$J+GCO%4VCs`t2`7x|PlexOhfAoZd2v|LOUZ
zhmYsPSW4QjDLB}dI8EHlS^bdDJ&o_ovcJw3FFs@{ZGEfqWy#ad_ctUj?7#SQ(U(r+
zM`El`BB!xj<m(i-X}>r1XV|)KxqUj(8=Rvoa-O{Pb6fH|sGhs%aH^>K%@ulkJ2$I2
z+jJUaEBZ-XykjdH^ZUN|g$rNJ>i_uMc-nQF;oa?d$DjJGPRugc+;eo={&cRltGn(7
z=Pu`L`(oIidUTccjz+d4H|}i~>wTTFSuH2=MSA#~#WMG6g+(poS-(0?^5@!;zFvQc
z%U?Ea+kM+>7Hu}&f3rGt@|sgzK?WBBJ5{qzAF^#un%lqg)s5{*26J>}TO;e{x#>tx
z3JUa!{^F{rIs4u13h{FiUo0jrGO%h)be&sWxYSZ-*5=4{S(X#FUvzR=KKDyo=CzO(
z#@P1NmD-<U4lVR`vRJt9<D!FWF22}b{eIJzGcz;xY4VkP&F%Kf=JknDnizd$X-?pa
z%8#pl_OJXNxs$2bbfvet-|@riij+!((~f$a<C$;sw<GdvUH6x#Yc6pY*D7%ayj?YC
zpRvV$z1`+}le6cVypHh=C~aleZP3a6>iYa~%eh6e@6x+>=hdZUZApo_eEfabveOB{
zVh{dJn7(1J+^5P^wyi&B9xdfk)r@j#Ni*l0yr?cZrDtJl``y5adrWo1QcroFR(h$L
zaoy-X)6wd-t4~^wtlcM{d)8dzYN|u}RlOS#=l(@|?H1PJ7iH{|KE14Zs)m42n#i5^
zotyT2SCWj_x%tIqx7uG1XRo}=YI#)7^rznT$C2+AUDrRlyJ{Qj+Pm>$Gp74QtT9$@
z^txpx{?EW5^OcyvrkzSYuQqv3T;^ZiHz#Dhd)3C=WBjY{Yh>+Nm}9<uzryo>OhNWp
z+od=DTz5gtxMtO!sgb)bZJV7d7<TEjUhwT^FK53v$&^WE)=KU|Jh>MIcV~RMzq-)#
z=8on1w<T}ZdcPEzzN?^r<wlS5d54lTSLsi7OA0dZIW_6)?u!SPJj|7woqm<G?Tn<E
z_o46pUu=GUWRtM17Y=<IeEQ}W4gTFn=AClf=W|N5cGiB4{TXlbC%1(7w~0>|J9f(6
z*yEMsk1yRtPZewHqvW4Io6YuG{Os|i6X#wGe3||&<w(J9?VA0!qWS;+oKdl3cDP;N
zli6&~W*_>hYY}60HGAreMH?KipZcQs^ys>OlU^NLuHR59=efPRw0B+C%qLf-PbjRi
zu+cwbZrhb*$h_BIRP2%0{#mLIkKdna()vs3H|LVYMK$*Rf4r>Aqg;YNeVLT%s&TP`
zG5e3xtXc0YeE)fz3vtaoymXf6JL6MZpQT^tbgsVjGtDpN_w+}<UoD#+|5oz+jKG+0
z(y9X9_rLgZE}r)-c;{)|<}YQFSI)T6^mNMQr>}3Va(?hn$S=P{Gh1(Nx8NLpqg?CX
zS0n!YIFhO||EOo*wA;s)p1-ql!TU<*`!kATye8*z&e1#i>i1zObIs<7hkW<#USo3G
zFZjxRrDGDIM=f70f8(Q`c`8hZ?QChN>V3`VdrP-}lsM^fTurulmbUVR;G5SsG<mJ!
za=Wrw>X5)5+e2IJZtszB_^fDsCiL7)RZgqqu<afWQ|~`DyEpfLky4#O>8>9U-_`B5
zUNZd?bn@%Zh7!xpkX+}d_A^)i@U1y*xWHS!bmi=lgEmjYJtvo>s^^>b`fE?%TbZ)5
z_fKo_1^$k^NA+2bHvOFOi_>V2&mJN3m6q;R@3>@}TIZc%XBV8(=(Huu-ucx28>ZUv
zq1X3l9bfbHN1B;Yx#X(2Qs1d<E=!;HdK`Nyykd^>X;Gh@`Q3KeiF;oJ<@4JY^M1L$
zS)b43zMsy$7~3n3>1Pif@jUZ$Cy)8j);lIUOxMkJ)R^m=U&g-tuJTR~4bfD`Q{`(z
zG>bzQD>s;CJ}Y{~`y;n;V{vJB{hM3G^NyN|y<c#6)9Pb4DsSu(5e{R!{b6nU@3+re
z_w3SraP{b&--nj&`|!fF@QXux!6xaFrk&Tf?z!g|oFTJjZeZz8{ZrABi<evadtY1^
z#4>4Gq=C=P+smXTmu!2Znx<WQ!)MmKX|=l#w!5Uh^W{=scY>qWcQRkg(eAH@mmf4a
z`7rRw2L+c&PcE*U+P*IRaM`y*E+L$M;_@$4$d)!bUG`q_^IK|!wc3rHj{3VKy6*6m
zOF6~2md)JPwaDY}>$Sgqa^!*?@BTGm{iUeZUny`{V}7V**!3-Kxvy@#s`8t#?0%EW
zn`HKe)SfjzkL-T8$bC`lzAu-r)jhttvPtFqTkn-;|MqCDQL$$(^f)=~sL{R6pI+$w
zT((S8{nvu(SugCDu0372e@)lFX}lN8RNqa0ny>O|cm4c_BBrY~vJ9%5i+&UrX!m|`
zEj(PkGV1;_arLO69wY19vQyosJQd&7aDBP8!;2`*()}}zMo3O$jk!AK@nMCj%{o_K
zKGe9Y;GBQZJn?Ud?1f3k7njsE9J|sj<2b+Jne|zbm2>1$RO(}RDxO#V%n1{p-~Ilk
z^h1;8O>4ADY^J<S40Z_@+O>Sw3P!)q`37@+C6=dIcF#>c6}jhCacpB~kILy&|EoAw
z-CNv#$!&kVOU$el>c6i0KGT?R#D9&DxoK*W*iAN-$rl%E{JuGLb0Kdbb6Bj+ij2Ka
zjq+zb@6P!le)Gu7>e{P6>_T~OKho*^(&(UfJ2Na)?3ls-2|16n=J>A94rGa*7+P^{
zz1_BNdS7HL0_83KeU)#NNL`?(^ZHY4*`><tZM(jOnMDQbUF-=f=S*|`5@0y<W%#C_
z61p*KOxoT?c=-4KJnEoT_9MvZ$o9)=4&S$&Df*yXRXlBq<r3qCOIco~E_Ha%_tsx~
zyY7S8D)IRfYpz{-8F%|hO{SBh*5yir$tu~C+-|-q_Byrno=WQKua%!`uLbM8lv7`>
zB(b99)GxR5ulV!cO;-DA9k{LTxc8E|cNfppFWH_GbmFI*q2S3i*Oq^qbiVzDx#1Q4
zOUpd1(mfAJYHD6R5W3f+EpGCoYtxp0(~-V$%e(*i3+>QFT1NN9W|o>uPt09%GA|_~
z%QBj=!8gK0@<+q_>eHPCY*v9cQ*P+4`YqFH%ia-VoYTaVz9v9;y2)SPoWq|i^qh~~
z<lY%}SFidh|BA`erG3TtC+il!|GO!7)7sSiY+G%tm>Qz&xbJU&dt|lw(Z2cS%)M{T
zMCNO3&Y1Um@{tMQbACvD75eYTvuifr8r{psB)>kY4Ak3rV#@L3t2}~dTU59O?x~;n
zaLb8%+XFc-%*!v*l|T6HEpOe#BR1EpXVl4bDxFDrD*eqXqiFuD*tlJ5mh*b@UG%Nd
z^ELHsKY!U+(%$#DYSzvruD=*lC!P(SF*Rs!itV$DYN=g3Z$h4Y7vP9^Qf<t+rLI3`
z*{ijec6@gG;xpms>Q!g1iMdXEf7^rkbnA@9-SghJh6LVB_~HG9O+G9`G?!^=$f@nm
z<heZ_$}By(xq6<*@`<xz-?m>oEBCxwL0}78NulA?U$Z#-v|}3IPhI(ap$ez)sq`0z
zU0q8>J?6w;F=kMd%{)`p7Vg_0Tz~afq31Tu6GDaaS)JJ!^38v#D(zS|H<Ly2@lz#d
z36(S9c~Os#L|VR7oVdC$_;+Oe-n(h9Vy<vJce`MsH(fd^(<kKCR?WBB#XBzWgt#Vd
zot_z)wXd@+@?*64`=CgpZ>;Z5E7khebw9tv7Q{RK&sBBvdrcSS<viazKkinB`x_Qs
z-sGvVJ3h{w@H+eRt_9N{G4`D+z3@P`Qex`T6AN6q4!!MJ(VM*QiO3Suh)<=-F_Gd@
z5jFGQig<cu?D+jW?}E+SPOkoGTO=Q?T9eEEJ?8Thqc3Yd7Ed$|eD(79-|#G_@3}8@
z{u<cKw>aizx%$VWXLVw=nT9nElAkWipW$7@^H(OrarqTz<pY;}8>JWhW8S@`>ioSt
zp6uV;2U#`eTyv<-2xce|?fhAqQxkdU<`4D#3~Q$wE9hlge)G9fTbvuXy<R{+bL}tg
zf6n14s;>DzmR|q3=;o7I&+2%q&Xix)&Mgn+Yd^d8xt8knL&_U#E}q!F_u}PT+Z_`N
zt@KLks~6-3GO2xfAbsm|dr7rz?$$*Up2j#e?VFW7^P9=V_1>`yd^erk^FCWzGG}Fz
zb(gHcRFh>3S8OtxlXO}A<8GOL)B2zJ24yFz?$2B_w{pjHVTsHKHa>gG7#A)R5>H&f
zTIg@rd;U-LpVzJ$Q>RUCH#_Ccd$1<F(Icd1#^;qkgqfe01;1{$P>=GGKNl3m*d_m>
zs5Q5pvH0npvU8s!!>eUJR%o>T_E;_*6}KsDRr;Ez)2Hw4%H6pA_1PT<-~Ik__RqH1
zWxXG=$`*uomTuhNd1z^9+4e*C9+aQ=Toh60$RRfOuiDSwg^D-#v46>&QWSUQv90s*
zzw2i4@jjfq^_Qi~)0*V^dn--9^Ll0oT;x~%!zZ+6#^)^^e<FA6>%EfJ@Xbl#)*bn6
zdvA8mi<&!eC-<i9U$=EW;xFl5c9OT2Z@Zyl>@-O)trI#S3G$5km05vBw{~hv-dHSS
z9Gx6<#YyVW^1wUx`OD8%-Hu-W$=5vo_Z@k@l~ZLu{C#|^lkwzYZPm{klGmr7Dq**o
zAbRGYAJ6e6yVn+|?nz4Kob8l8``-4WzhWwOim2w@Sop|({h3?rA-})8e=fb|F01xr
zGdJ<8>hCmD58g80G9_-I*6y;Y_Y<e5c*MV7Qnacex9etbnb1Oml_owbZ*ZN;X)WEv
z;oUp`X$+T!wZkvt*IV|L&CX4%WOBWI%GY4y%7vW$l8<ks?b*h0@U7y)x697!W|+;p
ztUFWGd&8HR3vQ&=#w15An%=%4QD^JAU9Xq|Jzj0rI<J3niMRGw_S}v`8R@NSmGk~C
zIiJ6<bKBIpi6*X-^**l&iMnxiP3_VhJEyquKjqoG(y%b^;S$q}wO3O4X0@KWb?b*I
z*Y_>{Uki8ozT_z?%ldO*rNxnBI+1g;^9$DZ*xB!1TJd?A%ubQi`C5^SU%dF{cxutQ
z$xmE&EV(i_D%zIASEk|Mq*<pH)#sgWb1uzUVe~NEFY&vrhR#aQj_K1oIhemb%ZXPG
zSj4nIPfT?C`Itv6JYjFv85!Cg%UtuM(M^BH@0oLN_&mtXTv@Sf+3S6p_ER#qual^J
zb@+OQck&07mmy3ScT{fm=YO+mRaNb5p<5C~m0xGA?6h#6nyz7~suOmya>a}TXRdY}
zuK96My?1T;acfz&&EAz$4?j4$;#NxNzKBZ)PN_E?ohj#jK67#JarqLdyZamNy?V>A
zE1u<iRiEe0>(x7i*50jMqIdQD(#p!5=Z&?(|7`v4c}@)2s}yTBwaOwy?#=Q%&#dO7
zs|<^0`vtXnsa|z0&XC)=HO0bx$NdM#xwxz6R7~BUd^_Rjla+=gQ=6Z3hQv&fyS-%I
zWv0-AKGqfd{ajBQq+eW)Y-rCZaM?YjELvgvqlKk^R0StJ{C_>9bzjf1zlx5cNgv-_
z+;dPv-|Jz}5ue^w(>*Jv`v@P4?2r<R|6FtY+wF6<f18i&dj3OwmC#bT`<I(CMDCVN
z){`+m?;~#X%}nRav-=XYiY=!D^e6N29-W%_#wX&U?9_&ejcNH)l%^|&FD%?-HD~qR
zl~v*;zwXDDzSJyvdnj>HN!4uW#aFIQNcz?lHP7<WT)s0lx6JInO#OVbc;m!rp-ys_
z{=43LdwQwszl9pF)vn(&kA1E6eMy(Yq~GTjcFZ~c|EZ<+qS<Blj2``($Fiqjw<hyE
z?~1K^%Ji>#AGqZ8E_0v4<x9#=TdL-<C@0HvS!&hyab8cI7G=C~;Y@{V=a-w`oO$nS
z!7P^SU!BGReJu51J{gnuT6?ukkY4)n#%dv!w5>M|7CpLh`SdNLM-$H1F)myF-9%XV
zcF?oL@4iQ;{yF*clf%8zE2S%D8VZMMd}V%gA=Siu!&>RzRwZ*8UVJzHzbN->;^a%y
zbVNK7#jmm)pKX82{BgSNGgsr~rxz|SE~^y&8!vtC%b$*@nu#|aS2A$&Cpsq!$_KA)
z7r*Q4`;YzLw)zvYrC;mP<o?dvzB{EN>aPB_tKIj1IQ03v%~9-pa#V8P<Q<HE=Y%cU
zsdI7VweOlcF0^b*`@ZtR%T3!?-{b4rS~}_F_gnrK+iKN%_%f>#K1)vQ_MYyxc*oRV
zHM95%E$7xPIXdM<-3sX}f8UooG>)!2wdkui^ZUoV>;A<bzpZV)^q95R+_bmhf99w!
z-eYg+l<IWho5$Qm=8tB(@qfwnnX=)wXJRdT$X?6SAKqNNRAOWHVzR4A75}7fVO$Iv
zAva#l{n6&Cv+tuwkwVJ{-**M2+g97mS$6M>!qen!(ciBbH$0NmojFBe{h22rM^<+~
z{I1t~r7rwoy-R0n%jOQ}smk?_OZqJ){%U$%sQ2qs-C}N+#aiDUJQn$Jz^YYc?ey!H
zBW)Iczn<||@#CDlPRkp1AM`3^WuED6+-+5|!{^O<NshYNWjfNQF1dWt686sB=y5-7
zq8gvBawErS8v|+9prh0BzfVv*yt7?8O-rW5%;ILureE>ret(`dv)#9by+B;)lG)u9
z(fP|tb46^~Cgsilsvn#;=gkeS8@6m(@3q}77yRQDGBcc7wX=U^*Zrw)rad^{cW}Pd
z?inU+H4kNjk`>zDFO$5t=kARoFP7XsJ<nm|-U%z_FN=M|%l16;d8UD2VDLNv;REyK
zCTP@`d|i>*{Nqc<n#l|H%<XjlS7~+a-I6<pjcx^n+z;G+w^izPOKO&Bz3T4vp!51R
zo6UNk3Ee*#xL)<Us(${{AHG%{z7Kcg|2dd0*632u|2?^C&w=gNUYeyo@r_M1+^pA=
z+10iq#q+Yq?I#r(Mu8q;ldmn2yXng}M_0hlbG6p$WZk%J?oCD2=h7?>PFX28-R|qY
zCBio~&M#jUXkKdl_{ZtPH$Pli_VQa_jnA@QZk*pYc$WJA&$+B^;ms+tt@BP^`Pp|#
z684w1=5MOK-+F2vUq^eD^u(9m)r);SXPok1{v`H9=fx)p)&XUKM{Qc%4`?j$uC6nD
zy<-dOeUYYIv-7hWz2{Ae+b3PT?dz|~D(Po6&OsH;xdG9JXGQLtEv>kpx}Dp^rnX}5
z5#4Z!g~zQg-^>$RJ>|p$Y3Acof}&YgzkN5iY-aD3`Eu80mTi?UGpy=AzvRDZAnT)$
znS7Hs-+7zZD0l19<zs=x-A0ej&Az<l&B^1F&naIouDKukrsMa<iT9Ka`Y(M@UM7(j
z^x^Re{)iaS%ReP+y`m<zoS79QKUsUS*T=rMbN=7usdnUu+@SaOM!&SJ-D-20*gWp(
z=_i-mo11s)d-9(*52vm<vuD=NUnR?p!a1I@vmSh7mpk?P^>cDgkBxs!U#$3;_s8Wv
zdtF<N=R4G8zRziEuGq0p|LIS&cbv;ZG+3to`g!gBzl(>Y``fv9K7HO=<W=%%?RUq7
zS6kkFn``%0bn?4UX^9(F*<m*N55J$5_%F3E{`UI=iKmoSr60&o`8{**ock}gPv6RV
z@%YZ|s>UXZ`l^n4-?Xs0c)Pd2VPDOu=HC|fT5rk=Hwr!|NUe{RPHgDf9UZ7y%$f1l
za^8mf;rDYFI`%K0U%UC~mZMKIByZb1>HU?se||%%dPrf9^{p@4_Re3uDe_%d>_X0`
zD>uY{x)a;^{_jtN3zI4heI}&Oo^fF7#;^<f+IlU1+_m&f&YOSi^4`g=*BH3{(zK-=
z-InXPxu36!?{GPM&%9N<VsggGBI8wZ$J!;nU9eBKeA_pFQFU*@0@2S|CcW9;Gd8_7
zax04ru@-f^Q4zk9D=O9eRq&3wPZ!iqKd*{hF7>(SYtg=S3p8FHv|BUjm}95iL<#5F
zCrY<g39rqGtvnQ5cH_}AiM@rQ`FaJV^=WC>>!<JJs;fJwt^fUEM)x22ml6)vFP`6;
z{H68$q!Xpa2VLea$XL$kwRd^og_iG!|77yNHhNrp;BvhFl}9(PFO5onv*xejvT1Wu
zBGNCsEC?@}u*mR_T;R9IJefWAIZ}1XhI8I7ViP#$FI%-;VvpiQ`ANEanvd`-oxkD9
znT0EA4)4z_xg!18xVX$TAe{R`<OORPL6!UOG_xnIl|RMv(C_(WZ|CIcbGMwf+PQO5
zu0AjG*{4dYls>JO+fx<&I7<8}*ThS_A4;!UM{Qg9GUi;Y*h~J?DvYy_U5gL7yX0)~
z{}{akC$i7~VP3V$!(+OE&Dw>rM?c00elGgv>z(<o{b&^T-Xr(acqg6jiN7f9CFALB
zwdCbfY2L=}sUo{ur#zoKS45oEjMZwb?Gn=u3!nXnC<=JmyCW>{*2!02H`$6VvB_8F
zJ)!(B=<PE#n~%vF=cPYvIe-7&gSzXN_40zNO4csBZDSt&>9bNqZDrZY`yrCMJM7FJ
z-?x7mzhmX|@4lDSPN%8QV7xYe%j}zy3-u1=Upkx<yeOsq{ssfRwfC2J2HM``ZMo`y
zx+v66W#_-$d)4gkIasGnPb^;KRvkb0v^f97Crf-gn>hAfoaGmE`Heu<?`Kwluf4+;
z-?480T-W5ZFyLs_3s14f`y)bTxy4VEFXc%)@$IO3==I%e*Cy979z1sCmd$hT=TlPl
z2TXC<(^dCZ`(0{Z%(-iKY7$qM+1=f;>;0R>w|Sek_{LA|e6d}>a<f2$)J)HXyKh!W
zJS$pOcDP`tlTvfjX1!fg+4rBZoG4yZVU`iGzu<nE+3KI`_`hzDz2f_^_@z_Nd@ZI^
zf8Ks-^1HrswW9V_yVir#bWbmnd-*(fb^7JhJ<E)G7QHOLe1koNkKJ&_&j<QuZ%tS~
zxK!3!nf6!DdD7}~-MM7Lw*`AEXS~zD{6O3MfVtuQ<xwRmTO%z#+|4y~xx~+=VSY+e
z&V0GXXVK@yljb>AZeFu1_S-){!Eduu-X^Q+9X)dO{1x-u(~t9Pe-*sBTW9cf!n61M
zCt9|y%$I-uGLf@DfA^6WCbJIO&fB@x`}5zjk5hx5r)q{rcTS1j>0>T_bncB^Pv=i}
zuDvZ9X(n|4^QXQa>|5U#y2vf?4eb>^eQWQ!vmYmHXKpz2f5#>3sR6eSS=+`g;Jkix
z-qpfDb+g-dH=lZNalP`rr3%7xOZE!{mR^iJf2(c%&okDx?@q|LCcNU5Jvz-SE`G(5
zihbD|ZQ7Tcy>mS7{#^gxp>xj3u3^hh?7ubnfrRSZh0lKO3Y={ha$ct_B<!=vUW4-*
z&ok#;&fTqW)_vmh(|uc0-(A|P&Gz)y`j0X?xyf35Gmq`JXl^{W^XGnpf+thYvh?*$
zdXz5uxzI$*S?RTZ?VDJs$<KGsDY$dv&0kMrot)WkGdEAF{H0KwGjGHGWS*7gLI$tG
z`mdiqul(^&y6`ex1N~dYJtn-)g?aa6UKJLF&R!wMdbKrYTK)vtn6mj|$8`SwShptf
zdb_oDHecwho@~o^3OBDwygoSnc+q6_zRd5RH%0B(Tkv*by}Vc5t=ZH18>J>LnEE^F
znf%)N8eP>jOTTZcxWjd?_H0U4`GmP`#+Dynw7AypEwm5m(yd!myZ>(g>KXs7mfv`5
zGWlWN4g2rEU91<bw+cC?bL#HW-_18Kl)U)ubS<vs^OqO?OW!zb*y0!*q`BbL^Na1q
z53O!5+%m7h#D??w?3(=l(+@ogt##aSzr0?;o`I`wL%a03D-w*?k9%^&e7&G~t-NNT
zw~*|aZxx3x+gHVK-ISTV_V!zuz<vAM&fc2!{)0|U_pSc*+vHcA-M;uov#tNOJzxIb
zT(I}`bje*ZhW33G--BWwPMeYved~8H%ax{do89b&D?aTzxocmhp?k-VxdB%)%lx*>
zF4<CicJ8+GQ(PbH4c?J&{;GP#QprOOHlbn{3YIZ!U3>Rp`HxJw?a3954wIHxNl$+C
zir-K|UfoON_L{u9<IFSX)@L?+DzksI@1vRj{rT7Svpsnj|GRLtoYiaI?yH^4;^n`5
z5hysoT2r@px&OEIH}cipAMNnIqj7qV!26J(ZAW&y^WV=`YuwL&eeL`Ywf|}(o$6#x
zT-85!Ijnd((~Gu@!iSytwU_3-RW|h8w9mux4EueNRVC9Wn5p~Uy|Cf5+WKILqZw9H
zeIz|!&CFfiYVzmpp_u<dXKGjP`M~p~<hcdM7mJFzijZkDd;b+2d0{c%Y~AVEk<q`K
zk7PWo?Ot6Qqp>?WuCM-=g26T83i*X*(cO2&D;-(y?_xEpm-&(UD#Q8iAI+Twjxm9o
zH+_4y^)tuu(w|~!1x2gugH5gOPmVfhyEl&Ml#b8*)9tlqSc)EQT=?<X?8o-OoaY<P
z&Ym&#PscwS0ULe&>)#)R?%QFLXFjF-PfyqH`vPxoduTlTH@W1KMb}hcy>A<S-YB`}
zz3*JEve46>>UG;*ZJYXZ-;Kr$!#P|kKdr9{e={t)_N+B8z~1@bvNq{74Tt>`lurK6
zne*}B#*;R)_$B|Uxr#4()Oaj9qvpO)hjf3#4uRBFV)y$$`igC@K7IG=7n`fwHeFj&
zw?Mq)UCh(6tj4f%*PC0WS$%)RESPEk;M-%H{4a0SCm+gOA}8K)+c+xvw}P@@`?mA7
z-Z#JAeEiALe)+W>=G{5xKiSPVV5+#~tb$#tO1{^=zi#`t$!*(vGgkQTOupSVPfvfg
zh;Ev1{GETnER)j7S$Tgy-&ET0$nxQhlo<<VKVlbtQRTt1Yn7F+J+mmE*Yxr)?{6G=
z<GAo&p~&<}xqlruE~r*K@Z61QGlOgW?dJQevC>jk_Df3_W<K)~ODb%h>nHlGzv@TB
zCAn*o878~Us^=Z6a;cbMcVttMp4x$UEfw2-jmb?u{P*8YxXu3Y)os~Vrz;D09XH?H
zblW`Q_3U%}{r~2j+@QIpl{GzKLkzS3<dd79RktVXZjqb1>068E@5}cK&ae9?cV#x`
z1Nlpg57+<kP0ajd&C_(?{INb?R`<^nY=55G?i;-2WV^4j(?Vy@>g{EUR%#C`CLVmM
z_35(p$Iu<Z&!ub3V}C#W81sPV>Wnm(E6scc|Griv8paw`*qy)lSvE3z*MXOglWp1Z
zy7}j{?%m<B;dhSC8jotVM;?5h{e2UU-~Dqv??cJACp*@#o6a|1^6AahqWinb?psZN
z=Wo7i<-X}oqLWSq#js}gpEqju))y(So>u<P^g>0AgXHv}vo_~EXGI(-S}DHeOy!(;
z*3ourPc42J%<5>qj{V{Jr8U2s!+b&n-+n7`vkGq9R%=`@5^3}F`Ro^;|2^o-e%<h^
z`O@$2j^_nAU#7k0DR_9^<?dzX|MfM>+}Hmn?5_Pa>CRtqo^NH%%hOLD`^?^XM?U1+
z?)_?aJU()BCD_&dtI`kIx7N&ip*4@xLK*F)^YUi?_p;ts7kbRbVfJZ-57XHt?%jF0
z=8TT!2ZLw1j=6&WPR$I{TR819!}hD;e!GgaN@r^?n;d@if7<bU-8t;3Gq<ko=$^7^
z->U{Qy_Tca`q5i=UVNFm{>$e~rL^f89;Ql}FZZZk-hU-?ec<7he*BK#ch8NtT>pkw
zJDOv$57*7QHD6Nx1^n8tD^YIEyg_hXZ-MF7E)5^kEbc@BiH%h&=6)^aPmfK=?eE%K
z&a|D|Y~7>sr*Cd2=)Cf&jb8TH&%$WkHIbj1_7A^E7FoyiZvMUP!PYjh<pPoC9~yaX
z%t%x$o<DI?iRYfG-(u&z?Vrb7*jjFB<#?KZo0^|~>NI!5{rbyTOCP?BUU_NWxxmK@
zYup%qefs|I{`1tBJ*ls+T!{Vo@mTu9fPLJZp_lky>Ree-W+$lsYl+d(%_UWg4M~w1
zJ45E3eVDpS^WE&)AJSF!(d92!-bzT6pXQN&C8t)Uca5Ia;hE`^_<n6EaXWO=$aVdN
zqdyNU5SzUG&BEiXd0k)sYG$;q@4p@AI&qx@{|ei-(;uGY7T(*opEc{g1lwMx`##Oe
z({i;ke_PkPE$m-h`^(*&J!(yv;0@1=?W{|U%@<ieGk+WP!e+;u>m~OWezx@Z?p()K
zcHVqr)+*VMiN~5Q&DFhJ`FW+7<G%2xDy;J+9<sF2dA4urqXkxOKW9$995-L`*imWM
zjh~|auBoeT%e;3%+rC36)%UY&@h_dNi>?S=mej6m$f>!%ZPikfMcw;?1C2S|OK*m1
z7cMk5nIUWKe<)~YP2B0qSfOKYltdzTcBxES=;IW!_}7(bVOLL`m&&gWSiA23j1T)&
zx7}5UIWOTOs<v`H>$4TX@$Y?D3KpMF+q~ERP>tc8-_ss<-k6?Qy}c|+G|b*WA)DVK
z`gvfw=EBbpgSo%zrXNsYyp=QKxW4(n#h0qq<;89DEL6+Poqh3d$?ml`9+b*$|0aA*
z&Ph~Ke`~Ju?t-Th=gM;CJzMKryMKe8XRiL_JIAtK86OGSx%a8!lMljLw)YbL7U`&@
zma1AC?tNm@;CW@U=9aHDmU6RMem&+{_kWGQ-iDLO+onn=%kL5JOIu`D-D5R1dh(v{
z{<FeW-c|mXc5!{wVxP_#q3>AjqBE=Y`O_~{R;b<FZ(`7vUV1?_pz1_Dvr3(yjBU(`
z#kc=1_@DJ@ALk_f?ddv6^3C}+vP)tD`wsXnDBgYH)V31meRi?pn+$HNdhcQh-@ff{
zSY6v<Z;M*9z@L>H?R%a)pBU4mw)x1i-Y?A&kF|K8?lb(p_KB9~&A?Uv-h@g0pCbE4
zNcrZ)$Je|c?F>9}*X9v_($QNB(zoAipZM<AZMKE2o0q8-tnvBp{NP@u$@G-F@73~@
z=2nIAi2J=LUs1EWZ<CQqhUkwcGTV!}-hJGwZTt77uH57IA66fq-#RIuePNr3{rxMK
zc`MSVEm`$6hjS}y+*EJXl&TBqZYRvmChdD-)GvSGZ1SFsL3R2;Gwr5mmuPBh{eAV(
z%lT2)%az_)NoJ?kER(&zyz7;m{cNRk$93*%>Y6Xm=VCjmZu?nqI@6auQ(rO%Di!%x
zf3%96x2~zkvU+vl<t4T+mCQc4U+zDD>DB404=?K9nq$8r;&n!9z-sw}m4DfF7oYhh
zEB+v6ie}I$b{5UA-##6mKCNh%KhGWcz3X@EFD#m`wm<aKvi^2EvlnLf4=`z+<=2gy
z|L<qox*Zi=M!qLcdG2gjbidY5{(fe3wZ*apXZLxWw*R^6&X#WFA1pI{F7ZqaJQ^zZ
zs#Ruv{_}vxVtt_@Ou}Bv-`u&~ccm;!Z)H_VjK`v@yA{|TOT4=y+%@T(o7eWFH|iIn
z|JW)g`}g=>&N?rBFJ^~4hkw}qyJx1IQO%YQIvQou`|9%UX-ghI-`8;^vr6&L0l7m-
z`*iR2pZPq?^82*&(pTQOZ(eI#9XrYP_UDrek503=cz#35l<M?nTC0psUx}HzN9J$!
z8Q}%|e=3%-c`xkAS+qBL`uX1<l;;1gx#zRB=;tffNxQ1#j;4wRoZdW7EI@MZp1AAp
zTf+Z#%+I{_*w0D1e#M!-zdr*sU+h>u|Ga|n`z<Q()Rjf-cZ5dzC)b7r&JgGQ>)Q5j
zQJaL&g5OIs9Xmd2+<kd}ySBZfV%(!`FY6V)*675@%So`ra=Q9HT*dPLrR#&L414d>
zKi|9(bMHUsc0p&puj-cT@m%bzP1m1Ehq8QNjX!^Q&-E7%tJ%t#Bx?m+*EK0QeSWy+
zf@fggb(?R>%N{?PWFdL`VCaqemHOH#q3i8l`;<&SQPC*SDfw1l+RsUM6Vhhg+4`-u
zHtm-Ak%cQgL+_;hS;*~c`$}qA%4b`zxYT#=zaO|PVwhKZI5|v|?b9XkxdP|)mt9^o
zXL{&-*`SiLn2j;E?Jt}(i;C8qIuuoPZ{L$MkwwS*G}qrZzt;D8%?bXZY1{5rq-kzq
z=)AN<)pADH?hD%$p9I`neCx7{rFGXG5!Y4o=N##+nUmM+r2kkhZlOfhoF6a6{j)u}
z9#;8={wq{IbtvW9svfP-xRB!iiaoow7f;fgD{ov?+w2fmzdvG9hFFNuj#IMcFQYU+
z|G6`{+RwwY-950XrQ*e=1BGIlvyV1sPUJYLkpJoRVb)^yIjh%Sf19!9`c1ZmKl5r?
z{9l}F>Zmt#IJ0EUm0i9Y10s_+j_PPCT?%LkK6=ih^V_<eFCX|6>dG@4<R(44tf%{}
zGVJX8=MU|^Zc04CzV+BEJOBG;zEZtc3#z7ie%#dkt-R>+ztpW0+&8awdwVr8(A+9`
zyX#i&iJN@eA1LN!tvz`EDbIA_<tMrWpG42pzwm2GOk?X}&Xa0SH{HIW@8q+4K8yMJ
z=V>;Vo*lladgT^xgMQnu{|hcCeky(7XZ^l6e`2#*<?X}Ev^EI7ZF$9V?x(`6Bfs{i
z@84ACrpz`oR{L7n*3T0fvTGMB?RlujxwYn$_0AjKhgk(K?K;F<T^-Z?b+=LRyiexe
znD{@|x%tdupBo>@H$_SQqFwm^%04;2mg<u+a__8~+Pl@>3i;nX^jN9B!u0uBrmy@f
zUJ6CW@)k=7Ub*@C!J6|&@9({*+Iz{sF!^Sw-raZBPiMQl+E?)WX_<ZUbM~zkule72
zB+dKHA9CaGz74bIf5@nKus>vXdezZF;nNA_uQpz0{>V5dC(KN^d%Ah!#ml$P?(>{f
z_Evm)n22=fY@gzDen$`49&UV^sJ_kX$QR$PCAXc*dYC7_EU&uwYU}kZuG`Ib{QI-7
zRh<rbUh?U&`Hp8F8CzA_HYwc_^w@vaGIouyrp^1s?Mt7Ysn2!QUi<q}ZI{M2E9srp
zySxkLncelaJ#%A$?&|HI1DRe+T1R|%->#KCPtWGS<z+2Pc{S2COIlg0=gq3MU~H~x
zWm&W9NpeSguvXB?8xFDOHqZYq^*vniBA5A#=SkY@w=FsQwJfSn!7%skB}JW<nYVWb
z_00I^HdCtT!d79&gim`;6$fVj^X*n$x^QFSN|o1@I#bwO?>9F+<=vmVOw2=fdXvPu
z#T~}~40t<gs~2bAa+jQF_;7BfWO{JyeAQlO`<bGO#$m1UnFe_tkA31p=Fi`_TJOdD
z1(!Ct#hUrtmJ*q?PW9;~bGO@O_6Hl;UUtU)-IjcO^`T0xGX`t6ScbDDT<83?a^bu)
zF=Z<+^#s1I4c_;=T{`NQ&FP|h1~uaH`Sxopj5WnLpT_jYdf8uo6(%{&R`IUo_q_S-
z^Z#2dJnY2g^l1Is7cH6jQZ@zIIo~X9+~T}^HacU@+`@oU`3rIb6rbNYxNlLg)xx8G
zC9l5CtY;|gPd6~ywexgmMWM^;xlb-l*xYlrdQ0G?OF6~gFZo&Cu3jJUb=_v6ON@QY
zUn3Y7pD@|6u&_eG>`dOftdz31%eUsa?EC%BYq4f=zD=Fsgt-g?ugp27g_-*Ih^;(x
z=|;uD8mTX<&ddsJ+mm0=U3QdTvvj%Tvp;s#RXfzTSe`qzSnTU9&o$?kELwl7VEMt*
zJHJkE^`0o&785@6<~G~UTmPG!Kij#Z?4GG$yzt9C{XQAzH*rVrnDzX0x84!%&0!CV
zqw9WDm5IzbsPjitWW#K(Zg=~BpTlue4fEAE-?}96K6vN9#b39anq$!@<U0G2<lTr{
zwYR^|%1iksY5LeId`e&GiM^*7Eb33EXI%)rqgcD3X3Gano2Rd%!fGRDU7i-pzNJfz
zvsL2!Z<Br<k0-tFp7JX!sa>tL=STcnlfV04v874*@0#_RZB4q+r$Q5{m1^xL(}mv`
z1`F;I$@Rb4$H-e7%65o1P}TXCt6qre#Pg;%*7&eR+MQP4QFtm_?cxO$wp~tBXDr`p
zo3J|I+v#mu*3-@{X|TG@^>^XLASKn+hyE$G98^)frFMR!Vb{s7u++lUbI#rBn0hdK
z_5`<eKNYJ^Mg6GZd7~sc^N3S+nO)2Sf#7XZo?Lz7mKfK{vQdBYuSz|oxVKK;Ywn(3
zwRK<O<QA3ea+Q^JkJe5ydjHqdCus6l_EU+rxw8fBKL6S!Q=_Jr^0(>K{D22L8vngn
zvf?rO_n>_7AK!C#@3q^_cT?`qnS1tJd~(;sZtHjm%D2euisqfDTWPi<DQ%&*+2WR`
zo1WWVNSPjU?8L{-fx)7FX^SpP3hLw>U75aAX8MY!4}GrM+~H)to^`$<B6vf(+P4nI
zBkPMZndJVpOWzgq`1R`k<FbpI>;K>RBs=AwwBz}=FLl-?6i6~HkA8Me!Y9%|kFmgW
zg8-vl^QWi(Y$ayKr|7uszYzH)_>t_^527XfHye}W%rC!x{rK}#-*3kZ-b(J$3%sP-
zUO7jm=a}WsgJD-cuJf~g8dvpp&soEJ0UsySz3!4J=WN^O*<<1#_v-GBH}++ko(F|Q
zr_b)}%{prSel>fzRbOSmG$Fec&xEttef<k|NIHnMeko`^Xsk0wx$4%FuTw25o(F6z
zD!wbnQ@1_VXLe_6JO}Hgl%umBsr@X;_EBJ%e^WgAsyOSlf0JfZ-})lzFFtY4-P8FS
zlm54wij}Q=U3WjSNBF$UWVJc#q<lPU`C?^Hd@@P&+w@2LwBp_P^&B?-_A3^)JSx{w
zuNAu!qx8?4!SapR`NHH)SMOgA{8+(gvp-gC$@%5q_WVqe*u2+1`H83A+ln10rmb2~
z*Ojq<%?Ad{SI)5?y|mbFe#)Hn@l3tKL%AmVt(&C!-WPaV)jnA}^<#3$##PO?oOf(9
zTrs`$Y5#&EAA|dDM^BW#bzIxOeMXSjyJbGEk+)uMd~30WZBb6s(gl-s&6m%eevjkm
zvvQ7-Nv&aW&v&rdtY4^Wv9)YoTKxKV=^f>ID_(B7-1hBNc988owMJemsVA&`ci$Vl
zf6BLJ-?KA2uD9>_w(jJL_Z1tSujQJvB&YQDBs2TuJEyAI61Uy%O5Aqoyhe53qMbK>
z&gKc0|7H9^H*)i{N6|ObHeB-y5`4z>JCo7mz}Zi-%R<d64H7QR659~To^?!ANXW!O
z({56mMQ_;oDGm82Cizy}iMWyeoqxr*rMfR0W9M5j=hOuUP1Dy>@(zBycGlZRh9*(A
z7sRCgrJYK*Pdl(DY?peRiPt9OZ`+@;*w`$(^5=PV^>6v>U*Ga>;WxW^cGjErYORk>
zaktpM7Z~Q>?tg8vG4Sz{HTxXZiV{s;zMblJHD%NHr}|6wuHVS?aqF%3ZyFXEJI{V-
zd?aJdZ|lk+y~s(6=d8Tx7Ipi0c(&}%+Rwb-ROel2_%U<o!<g$=gI*N9NZ%f3f9{pw
zm%Z8y+eH3J<w+XdS~)Xk`<-Q5Li}EzZNB(`?QKz+#Gg-Zmg-FGt$tcKP22kNgMG)v
zpR!g=GoAhIwCDcG4x1P1uHAe(XsgGRIZL`kmZ?80I<lgq{yD$(&8*pHcKA=ND5^Oh
zkh^Q@`<ycAow8dEbC2!iQJCrH7}iy}??n2RG&@hh)y&t@y{iOWCwP3>RFz)x!2cuj
z`h&b*f0iiRO02)+RWGqO*E*bK(<TSk<gCe0L+|=8KXv578J)#1&5O9!uH{eA;&=b|
z;#byrPw!aW)Lh{enbv<FtWsb8!}Y9QU&*)L`j+4Q<`K7Vyi;>D;(sQlvr?e7+l0gI
z>&nypS92HNa(nF4v~VkX%c{RUhmZ6BJR`jJ+V9@5_m>o;yUs*-duZK0cjdg%np2iK
zCtGHe`G1eyA$VTVHzsF~^2M+7+y69V{;jKe*)!)YtDftx1((k)nzk)Tcul=P)Sn>B
zQ=v`sSJzLNakD?S`=*iVipn2xNtSWG!Dr@Nul*nE>k+^1L({1PQumkSWY2baxUNf3
zdSyhyU9%1KoB7q&Hh+wmt)w0JzsR`I?WLRCvbSgKH@x`AGkgB|)$P5Gt=3udFRHHU
zc%?RX_kG2h59d}}-qBxl|McO%1v{S9e&hS$Dz`X)(;MzfPj2XHZ*OkOoErM$Q}l;`
z*!{~U9eK`>busrV$MiW~Z}aq9C!fFle74K`OO6&E(=P5=d4T2nzXOwJ<%=-Ap7qV?
zwvL3}$62p`F5JBIcI1P~J2z_=%S~SO%9%seC1*eXv$C=Y#w@w-Pwt+}|KCmE-;RZO
z{9YF26(=VKc&+PP=EFZH?Stw<zn!l`yszK3J8ztr%ziKP%-$DV1;yuF%H4U7UA_4!
zDW@<~`J3Z`ZxxA=dZi~G-#T;Yp8V|7f21~*y4p`&8hBs#_e<tVi~A+_!sfr?vR1W8
zKl1D9v=5KJ%*@(lDslF*#hnyGu`5eo?5Uaj<o4adlhvR1y)9=HzJFWI_@4WNX@@w4
z<sapLuQ%cN7qFu_;P#n3J<$aleA$+ls4MO%|CQNu^XL{6i{9<gV#&83#rf-h4~=6!
zwD!T=#Xj1VQH(EkmQ{y&=Q00wwX)7jnjR&dv+CddOsg598J9D^-Jf0WSz>qZ@*9cQ
ztZyGKDqsEP>z|lPZtk~oy$S36WsS4{tiAP8jANSgm*7aDFFel<*4|O|*m3_$>XBz_
zs!v^Nh!@EJ9sOk1&-<K9y@CSz7JpN|`u)u`25t!*{rxNNUa&Yh&CZ#z+ok!r{H2G{
zd-4xlTeMm*Z;SB#`&Vy$&$zggdu@HawD<WCx1M`eQ!WPw%}@Veaj(*JcH>52ORM8a
zR)^DeM!0{G*!1Lfwzc@#A0O6U^H$gR{kQYB%J0h8Qo1)=YkSO7760scR_A^-|Dn-Z
z;UBxtdQClO*Z0X<gZ=-%3-$KP9WMM7TYKV1Pq+KTWgpia=vy9fY|G1#+*Z+@R#F$2
z{X1-vnY#bZkr@a4Z<;>zJDz7=88vm;jBWqLjo15`Z^)aU)W`1e#*gnDHwSm{ublzM
z;!V6GW-NcPvt{eb$2)hLz4lv|dE&jX_<zZw674^C<rKg0a)_?sJUHX@!OCA&akZ<>
z6W1`^oGwu*fBSvPWEQntp^ufRX0vlDtKA&4w)r!B<^CS@ZAz=;1*c|q-|zb^Th6%8
zRc#CpyYWvfEMb=3t{odzyRnAGriorR+y6r1#il*wH&=Y0Eg!-3_IvEQc7Bty+e9R9
z6)bnSl_&Jqc%$djhgV-3S63cuC~?b@T9t3|_#F2N{=d_Lf`WshRbDQr+&k0kmAa<U
z_sey@)gk9zKQ%d5Z&4Mw!lkw*I^FWK%KcMUwc6|79j_J6eDkkh!AtH7hfDio&fYgx
z4}NhWiHYCmcl4AO-Kuqao*#F-c3Xdj!|EUU%aZwv)=p(_OZ^)8<+e<mXX5s+B@TWS
z`A_|-)*j0hc1g>so8eSxXTaq*Pv%EHSFX&;TCw`N?ZRK3GT%C`Gw-~3ZCmIy(SlpA
zJ-If1dw4|mK+bXFWBEDnFNOF#+T*D|>F-}h!$}o6vn~F5U!VIVZnBM^|EJO~@g8os
zJI-6=&Q4h^eLZr^KhE?C8?4-4+-BzH`s!T8_bxbViD%Qk#CGpVX1k~V$%<|iz5gU(
z^U?nrGu$L6`MY*aIb)_8(SN=uc7MdvL+!$6I3~_~RCiD9{t};KYwOn7J4jT`$h`Js
zM^y4OoiB66_U`GgI5z#}k+;p4p7RT71W($Wv(R1i?57`T{QPeYnj4;zZ#lZTS9xA)
z(0x^p@0q+`xg(a&x$U-d*IxP4{`2lV+os<ocy~n?L!YUk(-+mbcc0jQnDAlFl(#~x
z=2zXaXIt0WCHVwTRs9nYU9rC4W)Xj{edG1V>F-M289O_tY+l^6cdnyC^$|uUy(8Dp
zJ1y4;3APA|JDym5`j*CiuZ5}F*R2AR?$)^52+dD@*7QVpzUt!Lyd7sB#$K9~|LDu-
zgL^s-W=|7Rx3zWtz4?8ba7_HXbDzI`skyMEFYlU9=i`408|)-vekVnI6pvB~TAcj%
zpW5_@(;BZ2U#Ohtoj6_Q&&)pi=^K~H-G7)IS#JKP^3Kge2iY%7F0>6f{PWyv`!9Y|
z{+7ADyK(RMIsd~O>-@ua_PG6CP`5s&$yTIdb*12&x4SQ&nArAI@8$M&SCz89?-ppO
zV!QH@t?AIGYn5Atist;kIKQ-h*G<-KEez60b&rE{KiZtRb>Z05eNmNjo;}X6zjxrl
ze0iRyxs1ySAHMC8OD~&z=;X>k&c*+jxoYQBZ@*n7=>A)n$K&GjIV<G9|M+pId*O0n
zmDLR{_b)8}k+`DjYisF+AIi)@<~48cScFeFvwGg-C%>37^N-K}GC$^#-OQZz)%FMW
zKKpLD^Y6+{VSL)BlupJ7Zu@t!(fY`qj^GNm&tH6>p3IE2Jl$>d<L>sZ`}rxH*G|cQ
zUuXE7r}xRETeD;x7QOxTs^4n=N2h|Iiup|C;cqthZJ&FRmv?rx>Xj`{^%Dind-xJ3
zF}_QS{q&%Ft)2De5}CFCA58ys;pL1OA^x8jR%_Ya@z9Cgkk-m|Q-NQ@P;C<1j^h6-
zBEM~H`)<4YgxQsMiE~?0s}JnW<vnSz|Fccx=La`@*-DI-*PWZ$ke-$sa8A=-e&dFf
zms;<h*O87<-zm8Lz`0#YCnP<-`2L!6y!iXUeXRX%ZmT2r9?>?=O8-~8?8lk7J#A$r
z^IXD%vyR!VJ?Lkv6uJDdPT9B63%RphMS8BtuYWh!^oV`1z@J4G#Ujg&*K$g~of&7Y
zU+#3)f$d*ximT_#BBwq1%<(shg0=)dITQcI=<JzW)p;B9<205yHAL+^DYNSji_f#$
z-FjEj?VLB6eN4BMjXTr5``(^!(<aQxp3%KlNo%8Q#_pi-^IPx5i~Mp;nYXC@(k!Xm
zZuJX=j=8Opb1qceot}3(V#mAlscRCo@3FdZ{rVp0xA>sOPJhFEkGWf~z2VQ85~e9}
zZ_f7C>sOC6ng#Y4Sx=dE`Bmb{zkah4{yh?&vB_*rj9>o51J`zcdOO8rR#>X=w)nX7
z25KJ`pVZFNSh8zp*trS61v)ncvTXbQtU)^U-jW~Tn{(TmqBEyo{d(HYd(rk^XEx;*
z{N7*byw@aV<1d|vSzCk-zKiTiQN80CYIZc`b$;4zr%jqNFV6n{^Ne3fe(8qyKW2aG
zT2pIxX?E_McN%F^cU`%$pDSzfw9B5WbAr>}c=p)eyQjzUbkXAzzqIqVuT);P%1>?Y
zkz1_i{TI9`ci8i$Su61Lvt`y^-$QTqhOnH!^;^^CUP$w+r;`&|uX;?o_aL#W`O)G_
zVr%?kgUfu(PRzM{{pPeo0n>T!@n4MnWg^<Jw~&X~de^JU_f@_HqEQ{5lT}@`gnNBM
zt53FUKk~$-=xkXLyVv|;rkF*Ev6m;#OjXq1oVkDFyB#KMS5Ks7OyJU}s#t&E{o1!I
zRaz=DrzkV0O*^)2Rh^#5oAhO8Q+Dl5@{Ll=;AZ=N%;@%((4vGaPPP903Ws-p=X<Mv
zoNS?A<`sO;#7g+n8HJf2w`Nv*T$=XYsW#};9+uqgr71PSng1^@`T73t0S4zvuBM>H
zf0DW_zm*?4bKc~^QI0J>#a|g`mz=n$tvu7qE4TdcmE$jW-4{)mcD`nn{JVd9#Xfgm
zug%xrwS((i=5Fy0n{e?8nUk+(GTqH9dcVDL=XUFVXD`X_pRs2}w$11Ce)}2oYJxk>
zgm?0V-<fi4kLs!;XRExzZ=C7fr_ZDOd)nS-m)vabiK~m%u%$;I-1jwR#ixTJwa<$6
zzCKQ`DO~BK7s?*=C3#nK6;I`_*~dKmPum@P&A#>R<BSPc?3OJ1*(~<Ba@||$+fTRW
zf7z0K`nlZO39ow>?v4-dc%^m9`s%Zy+3#0hIc|ET<d>9i1H=D4F|(?KtnA)?ckoF~
z?Rec?&;Qi)^|n1nOy%$2Yg_f*RCn^vYY%#gbF+)4YTHB~>TUZhxxPZPy5sh>7^#QX
z<P>`6&tF)(@44{||JAuMyv=7*?!@))w7R&*D*Wc-_ilUty?$vt=cD!Yd*U58Ugh0V
z%sBierhIAA?LP`mmsghu)!SX2XydzJ`|YL-zuVq!K@G3Y3)C81|5tGN@sf;%w~uUF
zpyb|YeQ2GQBij@9fJ+iO3k{!KHgUO?+QpY`oH6-Nh?QoDZ=-g)%HBme>la^cv2j-w
z$v9>@ZO4p6@iIG^>-8pU->EK`yQ(3&O66hiMrrYgC(m;=Ouk(*vsSN;<6XP(TbTWm
zvwnNO&2K$Co9CFC^|Q=Nb&??=O%p$Car&or`rK#Z9T|7-=suhy>bXX%=Jy=gbie2a
z_f;4CUnVV4K3_uB>R@c>7Sms{Qd^G11=YD{CHCc<-RwSTefti^Y1_KCCsdo%hWjnD
zPiMTywEOJgr9Ta8w`;6ROiq{{b8=Twzr<(BB~9@*vHt&Wq-N~ybo8t~^UQW~p>coB
zGg)=%Q=B`Kgc&1@_H2n4ZShkIs=R8Lclxvi_qM%f&Q0C=^Ks?>1<Rf<*yc6y(Dv19
z4*Wb;{QHRb?n5tI{NodDe)+t_cl*;rduD`O4>=L!bIm_kcW+?WgA-1%6G~JMZ#v^J
z)hcz9?Csk%{OSRmVPRV}pSPa>ut<Yj?3azmTJ5i<Q}so3o{L$|Dg8BX&!!n~(hg+s
zJYD$TB$ruqZNlUU0`;O-o3p2%-}X%TsZ!w1sM_G$cj6v+2bP#VP_8(o_=m^i%hN|o
z@{AULSe5gfdF`n?+K%fu-t=CoP5fbBtksosZq{PYlBb(*|Jf!d7k2BLL(-{zxnJ$q
zt~(?4S+vNtC5!8r?4r}Jc(y5=z5Dc44BOkeAFrHd?96(ipt(~kaOJwWt}!>xl<`?j
zpS`C01X~#6wO6WgvsKEvH_r~Va1FgsYSMgK<wew9?m(7@+a{IipZVZXU02Gyv7-CT
zigt^?o3=@?i(YNtR-GH~R-K^mcG1opv5)!bC%A0)Yc7BDFs&$8d+~dHqqAlA7$-IA
z?p`WrpL^NxjpyzK=ARGlO?w_!-+wt<+lQ;~lYg7avCnoPhMV%IOuNr~M9+%H^XA$u
z?X@*m``0V~{Vnh#R<C=rZ~CTxz3cwJX05AcJ{-GR=i{-c{}b0#2Z*O@ntJmu^{wUB
zTj;QI&!l~VrLW8P{M;k%UUOKjAihNJZF`1i)Y9*_+F!`%P5%8nG1ttrRdeCx-oLNT
zi##~I;N{E-N4J#l+!rype`T-cDw)MbQIX1DQp2o!V?zBu{&L>kem&+6$L8A~_MSLW
zw)n}mg8N3(i@qmROgm{XhyU^E?H=sg&mVvOH0b{y$DL1Bce5ynep9t9pZQ1p<-9#I
zZ?t^^H_EuiU$I!}VPU&HCjPZx_i3Ki%C|y^VX^x2KdtWC%R0?}X~M-Pe>Xnui+M2p
zL3l{m(g`0n?eR*`IpuB@;lsanpUrtQx6H|Pp~mOU=Wah@zq0T3*Ps8Qx}4vdX!XdS
zHavVNsz`I*eT(u(4;$>)zn-4;p^#aZD>^pwf5H;$y=uoMzj?UL)jHr&=A@6y&g;%;
zR9NbnbSF|VWM@i$`lWwGE2r#xxm-@aX4kD-O#Q)gy0%YWGc)Iz@UI`UeJ)(tbM9@`
zA=y1wmZtxm@^;TouBa>gxyJLp9}ef<C0!R<=y|GL-pjddhl_dF+UNqG8%$2OEWH}k
zp2vHnl#1^Cx@B^<Vs_N=HrCK((|3K$*!?Bgzy4yC)68GNp(i(2|GeF3le^^B<?nC4
z#|taR?ApepFvaYm$gG0QWrctCO2%u})`!1ZZ(A|{^Ul9EqTP$v{WY<hv-fq{^GU|D
zZ%#4~@~ro`rZn|^iMGaG=39%BRR6|kPjI@O7xV6Wl1KE3>--G!tg<(`cRsMW>@t~Q
z2Uqu*oFn~{QYY(Juk@I8(8#7l;nTT0oStRAb)Pqwo>#8EYuqidI3Vmd&ve6PMSkhz
zm(M31TriQpQn}UD_4UlAdKd5W<${k{Io^nz6Vu<{B52EDwt9VxYKX?^xW0_#waahi
z&eIpT6VEl7J#CHFR8!gh%{%TNcHHRAzUFtuM~RlA%k67U%B#2CnB#9KfBg2ZHuJ7E
zr%X(8lF}+{GS26}O=n!(K6QR&m7(CaRYGsC2~SvQu>VHux1P-Y;Oy&*w?<rB{dNBR
z>n(TAFU;{;n&mP-{B%F_wpCm86rFbkS}cz#h}E5YKC-)B)-vBbVE!-Hsf&`0UVgiF
z?&;q@o9fuAo-_UWJ7KR!-=e6t4f#*ox1N&zD*kQX_VXg0RnH{l=Ki=K?is4NXJ*}}
z$4|~_FL6G%)OX$~`?_f?=QoGANz83Nx8U<~rpw2*eOU~ioDLDwiTm>Ttk1+>e-~9;
zem?orqUY}qiluW)mAKn2;G0{b_kO|Y#^*(+&UWd4+cU9$_P_rxdEdPIH2ct34vD#&
zHhi|YS{&&9FXcAhnTp8g^RAUVHkjME#3{N+;Z|%&2*1z`rPPwCemx}%*VdnBihIT_
z8q-&}t9F)5#<Bp@lN%;-r>y;>uG1^ODe>%i+n{`F$MD5s!aEJDjGmXj=7^J-U0&jL
z^;hub>pSLNG<kboUS-4Mg-5Pm)_?wf=4syxq5D4?RGpJMHTjbH^LDm9avJyj9}0&2
ze<q#rsHR}brP}YZRZCeD-CSe-p1;+(N0#r8z$N9xeP{c{R-6vh{-V3*#$M+;YWXXs
zykk5e&c1w`_VLFuvSOF#l(77Ieq-kIz};54DS}V$lurE+c1V)THT1LZ&63E{l&2{R
zV&BZt+E$vsZ0p;mzUbF3QdeT0Wb8V4W!0t=cJsQYMSuG?<@q5AgZw`sRwtI1Z_Zmf
z`SmH5#cw9vkM6CW>=eHBfy?U;%(sK%8+|_;uPoA^oRwbqcKZ1XIn!_2&sqBR`n1*O
zf1l@^TGLpuDtBUM*t~ug-!r-QZ0=6iyF6{#o|igmswYAZd;C<@EAGq-ddy>0ps&R>
zd;97)$F^;|R(h|4!?(8~^nLE<h;yA=noAtd=M_Kt+Ee>g#=k#Zwfc5gdx*^0{))-U
z+c(&9Y~j8UxY<s<LC*Zz+5c}NncHsnaLn9xu71DxlhV+qGwzzEY>0f*?$*%md0|CC
z>%ndD!KIvcq{QCMXEB=R-8)-Ebh2Es&=2$dcY-Y5f9tai{Jf}U!R#*&nc^F&1Dy1e
z@`KNu-1TsC!^cg&pVq(G;*oyR*|fz?z4>dbT%^`C&*>k0<9F^@9CL1(TC7WRj#bMS
zmg&haY_hGAw@rH59_w3s&{O5|HwIo|k01A=`6r#pH_v;;$W?GIZlBz;f_VjJY;#fz
z%~=ndmp**D>CNqH(WlC<9yt74O#W$E8q3ms>*r)guRh&xy#3+6qq}};oep_wk;;4N
zeYNY9=-C~5?Z!;WT_@h{-ypwn+4-A`pZ`1XVZ!?@$7^F-lD_+HDA_MFbxV=Cgv76f
z3<tA*SEe)6`#$;9o$<EtaOM6J*X8CrmdXVR{oQU}xP|@4-=!zlPvi@Jv$lMzx7hZz
z61%*`zWdIw=Z$&Q>$lnJDo^_>u3fn=_g2aIwQpCddc0}=;=_lpZg}&ODK?X9eO&*X
zbgiT{Dh@q6j%i+0|COZme$GMu!fSOGWo-T?spo8qT$Nt;MM3^%<eZ6CpFWisZq!hl
z<U7kJZc6+M^?9#8@2h;pK1pfDq3Q3q0=(KQ4!GXTGCRCswg1oVPj}K}ZQEsks`c9~
zUanhtK9uX3&D$FDo}X8OVs^1C37+b&^+R~kvv!SU|NsBW?>K1o9^s$#)iz?s278uc
zO*u9Dm$p5Az28jT<Ltz9Y)h9M*>8EJ-+Y11UMBzVX&b)=?OlBHx8{UL66^K8tl-wa
zQLMb_R#tey@w6N5JeJw9Tdux(`Z!8r*%OcNCmap8@i(k|I)k~U<@%mkr(%yDymIqp
z`J&&iv?aH%QuJ7|Ddc^1U)WmyiVHjD=H5Q+CSYo9d*RZ{#pli9^1g41%GKR)wO-It
zYWlTbwR7s?g1%pNzjN)nl9l*{_wLWG?)-JG^?Q-rHr<|=|HL=>{*TTtxElP5Mf3Ju
zru)B=Gg^N0U$pxopEKcoxxK9L=h-owfo=cUk0kiWY=3a=>hFKgO78pqnV{cl-aKQT
zm+;3WXN+1m)-91gzT(QR?ijXD$?p&4*vMZLw-t`=afqmJeSYMRwSJawfbHZr=RcU|
zF1RVV>`RDCrKVkv$N5uc{<{xfnet|R;OW0?+20R;|FZrg%SG8u`c}sKTBdHZ*O?Rc
z_jzf}?`x0Q%HBtWd08>bt!grS^`dwSpDf$`FPvua0u!Yje!ulRo_m<R-@f^LTGLj2
z_1m9&_X%{g{g%Bj_oTVld+p_0>-SqO%T<v*HRFBp)0thHo_H^)X?opn{pEfEvuXSC
ztibd?!HlcVb9B!BwfBeWUrF&@uFH>^UVo8u=ttqdlA1G<XPIreo$>1LRl^fp)qEnK
z*8UW(<?+g}l)o#wu|i3uQn%>tny_+{d-9_GnZ4(w(hob$dhy-y^wVPN^KJ2V!57ZA
zZ~YTn_~RqnFXnHjLissspH^L4a^%LcT@g1kT3Y^2_uOON`7P{y$VHX;(RcgoyG+U@
zx7Qzxk65=|A;HAXBGc>EmqRnx3#S_NZxZ>>U_SfMI+uva1$!d*Z$I1j@L5k%<Uh{E
zHSaHe{+;{#-SJE3rYdHn&idHGHS25pp5FNV7jJ#uR^xWdt$Xfj{aI@szjWY>N<8->
zebT$NQ-AN=xj%7HOtsqc>Ag2@^htGJU7HZF=<V{nr>vYOH>=+IB|A4YB8p*==DU@Z
zG5>uguGOD)INW`j+}3{&d%pRa7{6rQmMI|Tm-QjpLG|on`BNKzh_tKr%-+Jw&+o9{
z^Jn%cMV&MCHeS+KGGbXX|CZ1^_nz+^O7~M7w)dXf?czH>>62RQt|!5hKj!J?R3xTd
zSa!sJdcf;MrzdAP^vbU0AK~49qC!vel77)Q_PcG2N_ve`dv~8Q>GjwBr)M<Zt#!Z1
zGcTv*v2$k4*7y;bJDKO@B+<a@Kb!>~=Xy??clSr$^hJzIvgM~e`e!Ryu)l-N{@iOb
zr#liG?K@WNmwRbn%WQjav+J8<tuH^%`N?x=RmJkZZ+Lgt-aPT=!9S}x!P+&S^=Dnv
zW7_3^aPP`#9o~1oS8QLBB$=AMrKG1wP|tSidH0@YZ@Z6|i^pG|{?X{wvgebYo_AaS
zP(bNGqg;(>{Y1^<Qx~?IdtCb(%kizG@u^zbZ#9wAF@KqzoOgMfyshio7QgLoslRHd
zW=WD$PsnmE;rFfvciq?&;_4R13NT9Xtht>$HNV2wUT`0`{PSv_$j|4$+x%VKvu{D+
z6}vu1_e=3NA63QwKhAmecUyte+aGdGU$0Iqy}1ANEn~OC@ja(<doOZ4d)7aBiHCp2
z7Srw9zAem^_#kT$75Tt+-|PE(w*2jx<^O+It^NFcVz(Ywdhs~_7Pxm|`E+Bi$7h!t
z-@cr=>%U&xgI9hRc5?si$=drbFaKZp{R`)3ESeR_<zb&Ju=ax2{b@SBb&_`TEBK_&
zv9~OHs`B{5;m)OBO;3KBCGc=|y2Hg=Y7r&#XUTm$w$)^Ne*TlF$i(MY)aG$7n^-n`
z-_iXY>$m(~Ww35m{l=#(C(W3XMQSZ=KFmBeEtuir$6rOyPko=Qll_kC+@?k6Ti$ek
zd%#$~>WEMN?60fe?LMC|OVRE2s-A-%HtKB7pLEvrWQKL*?E9U1;#HR4wj8WGrhF@T
zev(veuBq>8&uupr+nt-$f7aumhDuL;<F5-_tPY6V8!ro*?`n7dSH^`uzor*xtT^IS
zqxNThpyFN0|MALVPh;Zdn_b=SC6yWI`^I%&#H!mm-zq&nF5DPeJ<U#_tN-_u7A=!y
zYB$|P=S@pzUGdguxhKmZ$>SNybI%66Z^_&J(av4UXEM(+{u#d?Kl}TN%_aYzNU+cD
zH|7%V&Urz?PtX2y;C>gsBLDaNJ&kS)&gU)M|4-^xW}QdYpRd^-Ggfy*&aG6{nrFAJ
zcYdbWs;<s8K^d>2rwUDp{1fzU@g(l4&(pmOjeYNm1m$);HH`dng<)x@hMDZ!WhXT@
zXqwG!QftXGYBe*Pl%%kd%VNf<cnS6endx))3fE}G-m+u&_tks1B>U>t*LIh^-BS(I
zuX+Ug39dWG*gdbfFW4gI`Ev2B+Mvk(E}x0lCr=f>;;-UlbA0K8Pf`!>2fnqxdErI&
zj#Eh;51!n+{(mCZML`bZIIda7Y6mx3wZD>CRqWO=wQtJuil<BBR%8gMCc9fLsnp#S
z7CU>L)P*KCgWimFtJzF=Go&t<EXZ`oE@tR`F>BH7s>>16%#N^#)ZO+GST7xO=vzIn
zkm>5fpH@U3U9;WYSNqywlg!O`DpcBM^BKjNCUi+DWFA>ks>VCRWI@%!%MTQ_{dl-?
z`HlD6YaE<;WFEt%ZCu4?`(y7V|LFd3g@xns+tvVWCSJAFl3O!AY4O~U4qPcAE`H<L
z|HbPhub-R!@ZuHjV**pf7iB(OJk>vJ-`Y>98A76=aaSv}dX6_f{d}x-d&#{^vH4GI
z=YB7l+p!>%&DS_3wtL;2nO03R4X(HBRCAg8Rn+&Q;<5`K$&X~&3hqA0oqLpdzmG-Q
zA>F8s%DKiJRRQzLA5Gz09^&e?l~L^u+p_ez@26Z>TQpJ6aG|GCsA*!)rz__^E6Zy(
zzn7X+)5br==tukm^W?iXM6a2I1TL<hVEb?9L;Lc>6XyEg=eaF(RdLsujDp(z`(8w|
zlrDN1I_>=W`}6ixJe~WEapviX9}*5Zb26v#RFu#DTkiNG`;z43bMMz%il6zpvf=dg
z+Ra577vkf;hu?|c_^oUHtNP=+{@wX-T6&+^#zbLBwu>7&bKmIREIoex&5=8WyFLB5
zz61#<^Sm(F@%KmPRo`CGIs5b~&K&*sQ@a1&b=F0#D*FFKuCh(*S4|Z<p}wfAv!X{m
zC+31`w|%Oen~&V|kI@hFS^EVudn2-4Zm;K-Q`Ye@RBUNulu`TG%3$bpghNQ(V~#@M
z&q4+vb&nX<_|Pj|3WZHargmy(u}r9#66yO<<rB|G!<lzL3S~f2oDdyNM?eg0A`@Mn
zvz*Mi?mNL_Rv(L}h$*XwlA{o((AA$^ES@SJQ|3r<PU4v05|}YDqe(?bWs{Q|h!Rxj
z1j&L_PV$)Gf+?g?q#>k&tjcAAN|TD93a5f6i>H!@kmDqdN!W!9)e6-bR0LHzQB`6J
zefSMEpi|)_i>I20k>jL~DPW3olFx*cCKX|onF=Q>YB7WuJjJ%Fd2EVMa-0NG`DEsu
zbMpK4t#IZFXe*KHd;T%_sDH-WIgy<<#y7Q_EmAx7%{*1}u4+a9e6A&3POIY{&gSX}
zZ+sLu<8e$6e{X-hy=7g{TchXB3+h!LiKnfojr|dD#GlhXut77&#M*A*xys<!*dK{M
z1CIH#+8Y|@N<<&iXK-Ou;9?A9>0ojYVOS&}(AuB@5<0b}(`*U9l9%G5n%K5Ejy5Y<
zyi^0U8a^#u*Ly{vluQ2H)sMmrL5w0#Z-sQ#lybDh@46cxlMtV|qbI&?O*kWmhCc7~
zzDZgi{yds^Cf=xC@=`3T!okZ@da|x7ALJiZp1MEe-zL_xEOpUs?As3Vb7x<_U}*iS
z;NVTGwihysKDRUK<i&8;q^t;(UEO*#?T=Gze#|j`8@u@F2j9<a$g{CKZ2h|6Qr*Ry
zZEq$fKGS@&$hfZahpJ-Y@{KOpv-uvF{FAXyx0&%X`<Z^cfwaBgWsU`I3|ko&u!Ywv
z*W9|#$@H`M@O<ficRu9ret06ubpEnek7p|@6<5~utq(i%e9zy~{7;+qZLGNS^<Gr-
zgY|YV)3e`8I)wHe-2UZpJ%2{q!_Ok$esEt7Tv^=f|G)C>{KJQRZR6PgJfHq`|FgD(
zZE^Rfzy5vR|I_>X3IEkh{?!TnjF)>9zx&`iww3JiPxiBm{5Q{&2>Il$WaJliFaL+g
zioaGGpQ>kcT;u;{(aF3t*Zl0iOO=0PpS<K}<hQ>!v3}R%+tb&4QfOb#yzN;?-+n<}
zlmFBHGha)$bnJT{$Z%%)eeU{erRJ9mzbsrI_-j|HRas%R{_T`oh89`uHJ|^BPyW`n
zzbeA;+tR+;*F|Yk-8TD{Ev$Vp=ZA99rOQej#uCie+BUyt4PSfk(BuPxlY95C{I%R}
z%cm2jlCt|(^nF`u@Nl)$`I(L@mwh_;+em%cJ<<5@Dm~3^zHO1qi{xi)*7+iz{ovsz
zSyyXiZJz8W3fFIMuw*-9aVl-=G8Xx{fnOhnWSnohJMnzsHDNZj<kYrKR^IaoIZys4
zPFN{_J>29s%Z|SG=d%0>^YeXD_tu;-OR#I4`Q`J6u)QtsWi4bmIMy6WG<;OIw<uCR
z!{_H|_Al%2J~-^uT%S6L<?{af{6=%k`jYMcT9*F%I7xKI1c&~xJ!UWeJSg?!?v!cI
zZ9A=gW?k^zzyBV;*1!Ar`9pjD|CK+^AFh}C_xZ#8_Wv{6`EM@lyvbi}wY0K1nX}%&
zi{*Rht!kyy@7?Mog<LqK_Z|K-FVH3OacY~Ng3+2!;ac8bLQU>YTVZS1)?4g(Wm?AJ
z>+|)!zRyXR=-ZLoXe#~dWMzIvNY0zNOWZd{&iL2z<dEgwT`F?TGY+q`u)VjVbcX<|
zQq`uy!}X`#{y`lwT|Q6wUyI0cnfkut9?@~EUILviv+EZ%aIWl8@+o+)AhD?Y;`8|$
zLe55)1&y1cWf_$(yBYe5aC|!uFSt?Gq&|hEm`CW(<N5y^XB+?Xf4cjw-Tt~=56?5y
zv;8nW@P9w^fwogGgrn<!{rW6={eI@(nb+s1?%ug>|LtF!^Yw4W{n+$&>bAX?x0OfT
zlD`p`eY-lhdi~bFjc>2++k5N#(RZuAf?3<mfA34WANTh5>e}k!JL0n2!;incT8PP9
z{`P8Sb@46nx!-K>#m(JjzP$EgW_9T;5c7HM#mef^dvSBOn`1C<#>H+kpMKdd-p}dg
zvY>4%r{t<BvuFI>;JY*@So5||T<L7DyUT*Mue`Fue#72fdF$8Ro}GPH{zhEv_VDd*
z&*oNtUY)x>@Am53-=1yL-<7*Q?>FcDdw0IMet&zlwz~BGy*uAgm~(f{IdR(W!l8X{
z9*W(pyXEQQ@w>gIDr31(qPp?IxANbf-u8R)#^s4$NS=DJ{%y{=b2;vIUV6KP&y{)G
z{4$3(SI+%m>6rWArr(F0<rUA=9p1Q{FI$vuebv~WF-g5<o3LHCdQm;o5x+k<>UG<K
z?Yq?--mqLKW6H2*-BQAI#E(UH?u>JG?I0#cu6oP1U?*>Nhc`zqlqF?Yn{KILI^rjg
ztKP9K*yS|H=!<2GGOR<l6fqs~lYpqUp6GNtbK%<~7s|3S)Qd8#ZMV!-xaBAK)<r0H
zdFM7^*VXEZ4_G?p>fQ2_$X(u3rqsLbvfJ&>#c!`%^m}r#>{o_$<DYm9{(q7Wz$qkO
zGC^R~F%9|n`ek3AuU@nN_OHv^*mJ8trQY7W%{Ke?;eED$KE2JnFZb<j;(fWgqi<jC
zu$9@~o_tgO#yr2<hcn|p?t8MucJ8+J&)*8}iO=1>97N0M{a$qXTggp1z1xRV<4f+&
zTX*|#Y4whG-Ro{&&aK{&%fG$$V($GHZy)Zn1;@<GeYSV}Gd8WC#q{ap<vz_s_o*A!
z-|<_q?eVO+1-b5Koom_a=1nM-PjugL`{o8&*4*Qc-OE$oMr2q^-Rf%>^FMvd@M2lW
z7FluKxzn~LFMj(X!`kjvU%&os!%JmTw#cflozqunIrnjGi>&_Ic?=tqm%Uw)X`L5k
z%b0ZB@Ak#ZWq-DmF&*h!xNY%@w-#A{Svq9B*Usm-<i}pF<GOg$+vRgL-e$ZiTa;-%
z>sQ~heT!GVkNEXT*7vWAn14y(@l(^^XS}qI`qej8&c9>d;-w&7Sd|rHlDqf)i<ipR
ze5qkN(l>42;?#=UAOHUPJoVcB)W0{c&EIDGW7FHf+j3jqM&6dI`}yw{D0IJVvfp4U
zyS@GKw}Ly+pq{g>{c`5f+m|z|Epqw6xyCA=|M}a8d-QkZ_)mZPa1(QNe92AZ@GXUf
zZ}qOX(t8o%yS@GQz6_U2y(KpVRo~1II`wj)?B%kM&BCkR?)bJzc-z|--}VQ{{V_f;
zpYb2hhx^6{{-2uA4J#}8zRYj`Z}SIUc>dqn{(sS#*mTjpz;tWl_nW^(3N7CL__SH@
zId{%?Kj+G-+BikrKi4I-;*#zDp4VaV_m@4swqxI!*Vne}(>=bnvNrPi+SJ<HebLt!
zMa6Gl7y7l}&aP=!uW!oHT`hll)7CZnrln>_{*9QnHudXPo#@lAj)P>@r{0VTvj@r4
z>Yl!~X@dCL>fhhqM$Kq{`#L!_@AmfW%y+l1AHQXjcYoc{%3J&F!k$<DF4H}K`rFry
zcI$6%&u-kOd!di@*EWY}?w;FBZzB@!h-G9q7Dg|S1@Uur4f43RB!l>CS&!v5mg#`_
zTMz8ez0eQh?-QG`jrG^Y16y=2TxYtx&A}ebk4m^D_To5*AJ6S`o9S-^h@ah<8J%DQ
zl0UZfz#d%#IqoyJnI1<a+!K@NXML92Sg31|&)svJ>GQS&JH%#eXFZnOSQ(uV15(xh
z>Tm{|&HlBYZ;Q`5u`=xv$K9E(-G6_SnXp_q>&q$f_%vs?quR4_c9nhKzGS2>lt2CF
z*uBlEz9)|_{-r0Uf8z7H8Xi+^wSFc3EEBVQuYG&-BbtxP?T^$K>rw0Ed^mkN-yXjm
zeT)CiHeP*b(<yD4)K>rfB1}gWWBsSkTrU`D73!L*t^YxD&yUHAU#`|tS^UOOu0XUx
z_FeBGF3}^6PFG&nRI|*njz3o=_*grnp!Y<^LH0ie6KbzKI{J9dJssuQ0SZE=HDd~|
zx4sUz7#H>;)iAEf@ZjrTX@;@a&Zf4;h#h{qwn2eC>A|(`rGhewrAm|LZVk1x4fL4s
z-=%84K>ml)lRwriuy}P{?#Ro_XB?)SIVGua@0C>4?Khvj96s%z6L#@62h+qD)?K~k
z4BHkLi_B`evc0OghihH)>E2CJb4z3QA87YZsJwoTyFUBRl@FTb@3j7UbB3m~R|d~F
zGgWrKe!J7@h{nRZU%U5L*|G(+-roMp@ZqHUub3B3`n&j!?ty=g41Z6Lw|Ws&wRyt2
z^Udiic3Q`5>kwjHEA#%(@o80iCvE6>B=PF!BWamc;+YY`Tw-w_WKV`EYHSd?q#U?o
z%0IrIzDTu%_c{?d2R<lfU2tFK%qJn(_(!&`&XeWc-}8wV{lhuCh4@wZ_q|!*?7|=$
z;5&1=<+=T4g`7S@6XJwARhT@VP3%!#=;qY3M9foD|D@_9XO#+%3lmZlJ)IUcyR=Sk
zVdi9h^lw&-nCl~hNB19HudiI{TGQ0Lw@X6EU3*T*(<NHZLoAn{vJ7opQ8edNp;r4!
znPo*YKzAo*G6f2s7pQy)yG&L1CI9~DOvULe#hM9vt#iI;JefVAa^eF8{}bCh3MQzl
zM5wr=^|EWtTUW8`(Zja(ex+@kKZQ6aMm}zATP{5>uVTl;hi&cFa_=g3Ox7&3x%;u?
z_+eT8a+^CR1C9npzTi-u{zCD1YmDiHwQQfRIvkTa5$SxnStfl`+HLK$Zp&`#ZrjQB
z`*VsiuHJZNAi{O#=p$8wLW!^@_w5IcY|x3Z=qZ+XZm{OK-3)fU;E)FvI`i#*_$1|s
z>B+PmJ=gw(>zKv^mA=Ebo;X{3^qfDW>Mz&jefQ^6d;f=Q-Tpn74z&hr&heV~N0~v`
zyEh}tC0Kc$l;G7>TC<C1XfX&!hrYklvCgf=RB4T1a<4|VFN4}Osf*V-wp~-%RwR+M
zPOx+>_kkmE%D<#8UhmlN#$uMVL9lf#_wSCQemZ4J8-+XPdc1K-QWvtG_~Yr3By}<C
zi8p+eG}{e356ZN8@H`ZhkK~yLQZapt%E@USZ(JTt-|dp;*Y5EqE7(B=YCuoi>mMur
zamuuT6oQPp(RoB)U2Jv2UL@mhOm6tRK1nyl<J43Um6?C}dZ+OQD2rTMVYFGld7jyq
z^Bkq>PPaP)qLfOeapX>Xy8AdYtG~ES*oTON$*ldNAjV6d?Em4{Lk!j4GzqLN<WM^2
za73`<#FLH#{|#Mlyyp34+3d)7$XvB>!M`8+a_?A}deYdcXB_;OYEa?0Yxx>}pMHlg
z^+~?}<@p-a+#g>F`rX*X94azPLx=J40X{~9h2J@@M0niI%y`bJ>(usO?uD<(D_&hF
zYwxm{C&^xX;uGgzv$hg;Q$ya9d|vEdC0>|hcH9=Yd3nM!p<jUu!XDKNIA!{n^`6}4
z_VDn+D^BlQ=QaNN&AC^#?ZJ$;8y0yo!E)1D7e+4H$!sE0lH*X!RwDmnSK$T?j=%{E
z6`U3dGd)r7@SiX*;D*p!t_w36VmU4pC^+Ud-T0EvBKRfqo6p1gg|+)_0>zKlFAP_&
z5&l?wZ0)6^h4v;Dr?1@kC@{6MM*d^%vALZ0dqN(s_ddI6mCKb2GXIaxki0KGi}lyV
zLrY)POHS5!>iPNCdHKWk%zvP5C1~x)&wt_Kr?t)kk_)*WJG&e25U4PWyP=p@A;u+p
zqgC$pp&g7hYxZnd9`nH9SmK#)ZpC6h7Zl1Iwvb)d$`|kC{&uF}<P$Sp&Z!uEW0Y6c
z|Kh?D>9=#0ey73WLn&=9t4n-VPyKuOL+J7P<Ml@Seh7seuP^L=_<JIg)w`1uGCzhM
zT`PH>Z>q4k-K0X}m?JZD8;n;U@AW?T;Qyy{yqB*@uy^hHQnUQGM#8j-3_peDxqtFM
zxkn^UOW%V>YUxvEm2W-sa%bATIumws$CCHUY&OMSO#NW}o^h^WK-A^;;y>?~JN)DM
zklq0G^8d-qJathAFC=ijWxQ+2p8V>?t{0+wnr?n|8|Po!Z&A{e%-ZWHVQn<WqIY4;
z;mV8LInQ}&93&p?U|Y<S*b$*wz`(os+XhvBW)qdJzP_HtYc-^1IVeb)mzc`r%Knik
z%t_|ES$I4n@3JI^Sqfu|^T+8IuU!%TD|fixWRAz(1i^PN?_Rv5ve}=@D_&iquwAgG
zVhWFtPkP7m=_~<)_dk6-Iz7aA-?<$jC6=;bzuUg8W%f91Y^Y}Ka7(cM@r%5lszs&S
zHoe_o&?}za5x7Fuq)vkQyxqlucIi2}ZG8(R=T+%h9A0W>6w^Mx-luxU>W7ljzL{(_
z$NDy$%wC+i{}OwvZ9nr3jlPybtH<6Jc8`|4n3(Y;wYOKM^K@rXU;nwUJXO;AE!OLu
z?6eDh^u08Xy2LJL`FC$$+?@3rmp#0__2I9QH!}_gHU)%DUvHr7#kc!}!->Mw!;{Z2
z80XAQe4>AoKl;pxB?><n<F)>TI9t{5^7FbIZ;>v2qo~0Dgtg|F)trM_91Kfde>u8y
z@#YsVtZW|@bEjDp<m`C1f1m7oZI*8iUkwZnN@iQ$voA~dvO)KQXV|wQ*Xk6%?3}+0
zE0q7dXI*iz(9u2i`oa@A4<ARE$sBUH>!bVNX#UNcKeM^!%C@z|C+#TG-n%U_K;GPy
zg-^O<A``!MpN^EF@s5-V<;LFRNH_KSGag%<d@QKIxX7U}kKx$4!xp=~g}mjfKAflb
zsmZwal2G#fefMr8Hp(YY^V>AJ!0R}R`xj&L^H*j{l``#fFk-NGI&tBgZBtjTpQFGA
z!~2f8Odmxej9Gs8mB=ZpZ;@j+Vrh}*alfG86nk;+)YJc?LX15g_3F-)y8W2Xri{g_
zpIxx}+=5gGhPyl1k0mV3$QQY}f^+%1*Jpc|aGI=r<kQd1;l}Lv=G=xLm3_@DQo4WR
zHCkT%T%=L5A^GIu@_^Rt|5vS)jXRR$Iu-^v3*Yv7Sgk1HAKL0J(p#atVVUB($Ju(1
z<6?C_3p+h$f0@C&E`pEaVZ?oghco6))Riz#ny}@BfZgekLc!H0|5jYdTD!@9%Gt*@
zeIFU3ggoEyXzML<m*CHRt!;kCWk$uSp5@#uc`@u#u9yCORMAX4vQhH0evh7n-Mzvx
z?Z=#ae-fu0y0Pe%XM~l8hx&A*fY=8;o2N^41g2<+$(QbVaCEcC0lrNI6*0_luPP1}
z*eidzbue{b$s;!LoyH+-cX9+S^m(W#m??Q#{d=7i$99HA`E8Eyzt#n=Iu;WaEof%`
zRa`1p^Vm^3K01}d(t3U9riCAEA}XKP9oyGe$LI9i!OzI0!7oIydd&mA$4qY|9xf5i
zO%`~m5MEJunfZW`#61C#gd-kzc)t8w_u&nny6MMd#nO&GitAM@mWZaZY_mJ%<*-C?
zuk<3GkBvv_EW~>s$euVoqcc%TGx6dbmiLAiLo~RiHTCp*-D`?n&ylcU#cbi2)5<A2
zn~HZ%)p%iAxSNrQQ={4M>bZT@QzY5urI)<k=Cg6Ra#o2<llTk=#sAXR)qb}1{dQb=
zWzX7)+-7Olz8svdY@ja8s&a&N+g;|x4i`Q~xFv4r-P~NUjD^9|uxSC?k(CesChS$}
z;P7;-|FC9pv?L2lo~1>7l1kn2NS`B%HO$-}UDy21z{s9|%&NuGVoP!t-_No;_9v$#
za$7A4Ojxv$*V0_hy)%)I*L|hosbq!hOUX00oH-IHd}@wGLCTx^8<yX6c({HqgQwe;
z-`wBJ69NP{d7gBg5K%JYwS3KO_#v0Ii@&Z<*?g(2bG}>f`l$!6Pd?#u^`MPVI>${#
zU+dfu<uIe59+?*wNf(8Kl?r_Jc1o~qnrjipJ#&eY8~byfv+rhJGcYL2S+r<X$BvoT
zH%_`Dxs|QZm)pZDH{plD)qcg@s&norYVbdC_|b8LTe2c?7iSC8$*u^4nU2%19s1m`
z$W*ery|{62x_Y(p3FF&G+mGyUPD$RYwZpQ!bb{Zvxd#g$i#b~y{9{z{=z7^cBj=AX
zcSSEM6i4{@UUBW^w?28qp}fuftI8qy2eE3==XV9&kl;G$u=?DglWW?|@~U{ljM}8!
zYu@y3TV`0yTp*RW^6^p5!oL=ale3tu+_)lVm@o3#WvKVMTGDKD;0oEm_y?IDiLZK+
zrJm?_>xwUvO}u}6&e56P;aXEVx0`(m+c5c{(mpZs7VE7tO9TFI4O@1#QT@gZrJPiT
z?aYD)A7^lPg?T0)W_;$;V=6GOvF3sVw?uw&obSXHwqemLI7{Oe9BrKL_GR*s*P7x?
z=bY1Z7|uMHe&?oB<ZM|zMa|IMdp@_4vd^lWZ8{~9YJR3W%3|u}2SIOta7~<gFnC{@
ziJ{yb!7bg2SqV?skD9BnC>3l9ZYrP08~#E$lieiW{L0f~@5JBLnZ_OQ;akbqz{sKf
zbG!OWug5BZN%<EJNO|~7wvaA#Ili3h=~LEYrxxs5e>KHJOO#znaMzTd?Yo?Q-BjGl
zR}dm6CG+Wm3ai>DpUH=;H0CAw7@Pfdohg6BUTHGR?)N`EJ{awiV>4wk5b97|8n?#u
z>CMMwTE)$CMT1tV6rMcs<iI8O$5-8)`j%ffz3Py~g9QJp8dVL4Tm#Pr-pQ75SKtYM
zB<>V^;(&^ld7QRtjQBAJiB4xrUZJA7NrBI&7@6`GNN>mtdeI@f`0D}%v7L*YB-6Mg
z?UeUPhVVXd@|@RP$rIqNc2JQ|bW&XYq)69$z1Isr9}wEHYuA%|Q5}NJ-E&^0C^!Fb
z`s20A+U)EL&6Pd3T&BH=NR$d_a|+QcPG2s${Pewr%@zx$AM9{cf4xw}L_xph_?glP
zhF<SV1AZO5mL-v?a7KW&ZC9jvq0zE?0pCQ7d0W^v>M%O;?GAQK5xpw*Y0){M>QbH<
zf#`pw#m<)w9GK+c`6+cvmub=RV{1#f3M{Oc8|JnY#Bd$q^9YGgV4OScmV^uQzGa~*
z+g458WFWby>9U=1LXhW*4U^-;73PN+FP|p$aDKB@Yw(1M4<E7`)F#~PcQJ2hv*ECw
zq`KDGRDXA)S};SoK$DuT?h4_#g?91L;U^BUyVWoX+4e8rA}cVn-XvUZ$@D$NIafLU
zothyx)4U_9)y?FQYb3A!iDjIE>dtO|qApJ`^j7_sG<(MC17>@jCrEh2ZIo}8d*~!R
z?O;!%QLEx*6Xh3t7JPB}ow`$vdt@$OczC7o<oCY<O$8N;Q;b)|CVX(dDrG&ZLSxxN
zvAJ{SYTaBQnapoBRcQH^@P;I{;KZk?61u{3HaR~DRL|(q<h*d?*uHhGT#0I~NylDt
zH?37TwK+y{k>4zjg<Ard<TgdTFyUa8Y|-co_qZ;2$HLiM^HoRe%41uVl;>6$ai%Qt
zF*z3>sBgIZ$m311VJkl#Vt;i1x_9CfJrg!(vwrThkn>ud>TC_13m$C<_t0m(_W9sV
zjY*};SbMzRndDu3^w&tTqw2<szf<<RMmcQU6_Wj^+W$$&gmuPw%>wSHf~Gb)8#GtD
z2W@dvwGo~?ds5L7=B7^To7*Q|NX%#JGYyKJ@qA^CHIEH%(WMQaE->=j?sE?5{PEAU
zLVrz+jm*TZpk=qNG$imjE^z$0ruB*T;}`sI*97mDKhiVX?;1CU)n5ko`aX`2tf42C
zJP<i(?3y3;MMT{_uBX82^RH7vTI_ZQ+T1$xRy;jy^=!%wqoZ3}y#1}_Om!$sih3fr
zcGIQsSK+>k4lbxyUea*u+Uh`_btxiqZSJJ1>1+``C}+Fw;PztMxE!7*`h_>wvcKcI
z;yKr?X`+;`bYqYAgFa5iHOU|6ruBQNS}?L2+)nJ366uR#U`RN2V{+$>WZhn-iq^OD
zZ-kbvE41(y`flo_up#G^)TBf<IhB<6a`zXWJ;=z^{XILSpycM$whd{qK}Nm;Pl`f0
zxtTqt9^WH6X|h}F9!1&J!9K5T9vK`ET>0U?4S)Jmc>($MY^G;Vxt4LA^8J`-aM{N=
zigjhPt3kv3;EN`YJ3024JoBCXqU%W%NA%%ePTabj>MPC*ILv8sdF;e>WMxal+=pkf
z8eQi~zt&hVnblOeqiU{H$l|#0m3KY`^l;48lP+HVMN|0N28V*)b&IrPx%N#yv2V^F
zO|Q2z(_-fsE^WW4$H#YXCsWe@ZUOEa+b7OxDK(vVD|5S+k(tShrH4y{CT`<zNfW%~
z(U!G&O?IwQ%-p+zvw4IMOFTF7D66=sbu#YFq8lDt78^Wiic>0goXWfSrRX*hxt}h_
zZly-7_UUO<Ro(Y;cjJzm2mEKvX(%c-(Q0)(w(4WvKAvUHKe*WXyC>cHoU-RkG1EH5
zv^4>b)+VuBkd@XvwL#MF@dE~b_QrO0Pp(wOr5ptxoVZ_3S()K5d6D6xBO19)E0}~O
zce9+(VwaIqm~%?yqpIR`<{z=KZ>nzHe(Gp^YscA4`<yS%4vOb{w{*<%_|$BfI_IyV
z&Ji7B3mFH$@}A4A-u*#Gzx}Y$;nd<?!Fj`Ahu0#-J*N+OIrn+~Y;h3qS+}qw!PaZ$
zmWz@%Pnm?ddj8n6t4L`s53^e2jWeMSIa1CY<zCU_;VTp*FI19zRb<z*malUcEIVm(
z;m%~&|C?1(ch1u4_70b7(^}x1*Js7YB5Lb#Ky#Agrb}L8RxcWOCLMLHwC0`gDP;Bz
zn-z-#qrM(HV7Bb=#IL3|n>@GQ{*oD5cl2%Ml)R;CJ8aIa6FI-J!|<brkPv_P)+4<S
z<K76XvaQtL^6bKdWT6yEIZpLUA950!I#vBM3Z&Edee5pz$*JUcly@4u;X59)^iY)y
z*USKA<3bntU2ix}KKWoImf5s_(lWd0xziY4U%DH6&o8jyoO1g=&kr7fv(70jmgGFv
zTR2OgVSkxo@xQ5g&&8*FX|ZB@qEpme#-Leh^|a8==x^IJm(vF<wEie`J`xMua>`_a
zilV~Pxm^)UE!*btdP#atsXw#&qLP1fpoXz_%)5+;h^PBkm{eW}uxMdi!JRtmg0!pM
z+60Y_U(^p)-n@TIE74$f>c>@jE7<2<bE<vA8n`yVOfstWiI9@)(=C1$r`F2<Q?JZ_
zz}&w6{z)0715<=#{~EpTd)<?D!ganU1AF+Z<{2gy&!oyZoWzzU{&MejTl89Nq5QOy
zZQW}GN+umxf7XL%y8Sgi{x=_1bq1*J^2*>}wo>|7apl$G38hR)nOv8yHV2<&c-td#
zeC5egi7&bMgSJjP<Kmg2V0xh|&TO-bolSC2f`G5dCiiQ4%G@ot<u|c<T?i1FCN<q{
zrx@Fm3wN0_w_Hd#<y(-#`Z&tv=A1K$T-KMZm3~D_b~8P4s(uu5-m2E)%FHL9)s{DY
zX86kcAZYSNUTw+MSqim+o1}R9G;SH1cr5<8Wx^cS6_Z4>s-05$gYU{(=xYbbAFJ7M
zbK~yyPP=b>@kpNe`2CB?yG!O>@ngD}7`;G7V(tvLa8c*|J^QrZ8mMwJesVQEnJT&H
zqwL0*Ed?o_#lEMeI9<ssXPRmLE2vj2Q23?Mvdc;7KkJOdh4VSp_`E-Jyfm4{UBsL0
zvPefQ>+z&H53Y87k>jhgP3log`z-bHzP?ec4{wZGlV1C_VA;j9RM~&oxOe(VRurbD
zohhm3TmM)fFmUgLdZSkz4?l3sF1lnY(WrUfp{9SqdC9mHiQ#g6e|-dlp6x%Tt5H&G
zJSWL&3GXJpge#MJ%vB<P%#jG5a>A1R5Z`k-Pc{C@7n+}&um}G2@@L3o+hU!6#=zpw
zhQM3iZR!l^TP9hg1-x3r!7ia2$K1{?>Eg2J@S10jzL^-6{NjtyV(zXo%y68?w7f+`
zLnV1zkY5aA;LMFa0a5JF!wWj@?3l6ncF$>b#n+1%8{3{}G#dpuxN@yBneQ_-g30=>
z!B5?rJ1-`=bpO_FlI45kcwDNl`P}L^8&h~noILoZxc)ZqE#;A2TITSmIW_p;#KmmB
zT_Mfft(k7WQfE~tWd{a*&@qwP&8IC}=I}GKq3B~pxX-!kTexGhos@5XsLwF$JQg)!
z)}k590-aW#))TfoXK~@EtCLg;YnE^lqio|eooy#$TW7X73Huc5eVi^9_bMk%$eHQw
z#tx&EFO=90x!;@Ua`VO0>Wgw_XI3tF%k^oIcEC03848Olrnl^r^iFYY(-d;Le`J56
z{<1Uf@?j<?=NY7#xSFWGnrReoB6WT-OL!>%=9hL|K@AI}e3%uNcDYTu+@Y{cBSv|t
z&ZN^)XPvibDz&X`h&5^ZrV!?MMAuVuS$uoZq{I83ge*A7R1)PP%gifO(s_YtR?M!q
z9a1yj1izns%6-O**-KbmcdBV@@m$;!q}`yj>Ewk)Y=`C<7P)Snzi~Zl%&yb!KB03q
z?)<a<r0(@VRgOnaS5vIM8DzUR%X-~c&<;GNc&qryl8Vqm;}gQhA`Ws^1zNYR#|p1k
zTG+a4!ky%~VFx};J9O|w=E~%aCkq#OWVw3u_D)*p8^-zYFi%0#wYXH56(&s$X6dV3
z^gSYlRzH1NwBxbi;Zv6z!Y_MXdBKvm#9)F|NY3<akqQL`QFnJ}^GMBDB)w8WNqEl1
z@&_+3EqmJA6~fT<fK^w++wz=>$lM}@E2q!RkjpCXZB*?|6kz5~G2))GBEfsr1<7vN
zIU6s(zar?Y_?fw7CyU9eFB>mJ$EI|z?#Vu3^tRx#pse_uM5b6qZVfgWR>7|AevQ>K
zS8pu|_T(}3STD3H_MlDTDf5t6tKKsXb!>U=!m8gTn;&wW>QGw1wMt^gVr9J}CwW*t
zs_k@5yY8SJ`|Jm&8}ovH*Y6k~)eOpK<=D%1c-<Bejh+TIRgN=k+B^=|GyhF0?`(AM
zoM_TnHk(WCZmd<|gI|er7CNpoY-ye*V&x{9Vir^0GoR0gfkUKYqxadATZMbIG(T)Q
z=)`Z_Ueh&+XKt9rLL2U>hFLo=-oAcvm6301@9tQaRaU)89rKr8PSvPg^2H=%%J&kf
zl8i}n+1k!Mn#`PXBWIFF_z$&|hX;z>dTv}cn3OPK3WE~U%k3T?k`IRek_)!#Irn7m
zr@hKIkGxI{>i4+%;)c%hBz{HruJA>{jGNBiIPJk<cBFlZo1ws$=?5=8ww)U~_W(P`
z8*ydp85b<x_@orgJh+zOV+iM0(<P0P%bm{Od%!PH`QaUxbcDB3uj@SKcO~Leik94U
zkZP5QUN<4DqueoFXlcNNFAllwJj#cr#yAGQU|Qh9!0$5G`%Twkjz^--bIPigT0LaD
zm8++@Tif)o=xKq>{^DyXsa)*J2^xGVk?Eb^It810`TE&p+HQp(owjL*jm#C+RVq(J
zCF`D^U(ViCvTE@xPv_n#et`@9SM%qdZhX4h;c(gwfnKcu{fxgXi}fZ)m+o55^!Ci%
z`^InNo|V^i|C?aUVS09k+vHzQJ92xv%+i_P1v+V5PM#(5NF`h~`IAIsV9FnVu6x}=
z&CXMgo~_6;N(`J{z-#Rtz{6wcwW(NXqjRT0=dBr)s!|7;V>6d?e7R^;n{H5YWP@XB
zdV_h&1;?g`Z2c@&iT4lrEKPjFUJ>}vZ--8)n{?L{r<T(fr=~7Bcj3;aEwa;P+V61w
z6iu3JynJH4!Y-fX_Q7wa+_`vviz9R3v6F8S4kq=r%R7r5S@hU$hEkCd%WN-Eru(U4
zFF9rS*R;<u{_rcQ#q)~Ksz1#xds;s9uyt1_1US4eI<S!?O;uO=%aN4h86lgWC5hT!
zvZzwLUuS4Duf2Xw3rlRDr>bY1Qs%8j$$-n5x1W@*3k($Gc{}fumxIGGlgaEI8#sNn
zr6mj7Q&)eUQqEO1cikzzqfP-bpSa#Qy)fmx*>cfx>O8rY`I(dFHY!_q8;f|vz4Q7q
z*U5b2f)g_94{Pt5wdu_nv!i}q$p@UR-nL~t5n8*}%f;)xXxJrzELU~z=9Ln=!(Pf6
zKfSH0@zGYhOK(qVM1|pkdCLR>SPLuj_)8YK{hm>zzUgEdQ@&5*4Q6S_$y?jQ?iTX2
zomrVs`M&4LqYGce&;Od$RhN3jn>jdN<HA)3lj8?O4;}4Z^kjxQlgk_3@+)h$CMbQs
z_Jqw$m4`*jDLAMm^wBTvUdB&HE~GRbSgLfz;of}1_D;Xj9c!%{U2-$$xb2wu@k!6o
z+}wZt>z-WUZ&%w})vRsaH`5^`UNd{XrjX``R<?sjSX-9_9A5U=oSCud&ynNe>rBiF
zK0AKwnI;r9?~U%#Re^eq;v!aIlZ;z_MPzY3XWG=N6MkMLJ4AB@Q;5~|o~rCA89Fn9
z1vZ+QX0(Jamz?h)z|hxNC}J>=PfLNxEFo;wqu2_aMLJenPxttpb1&=6s@NhV?>K4Y
zhY5Vnf*M|d9oCu-OB3d@hAuBso~^g4D(Gq!)0Hi+Pi_v>EH3P~vNbF?d1(LfovDX5
zXc=sKko)26!PP94(oq_(wjEy3^4TMGv+=qG!7hs$n~K&XE$3RuYj{PnMMXbi0gL^j
z4H`?QvrcEbpTD%R!O*`jH-GucUcbMIk*6%oYP(e(RQEb^p18yl=Q@l3K)J#<E6ysu
z_mXqY2s{#+u_64$6NU6e(rPCh<yN;8SD)V|V{qUEKYxDF;rqUmm`i<Tng^=8e9Z1$
z=U3%-uy?W2-LzR9!5hj>9rsdpy%<}i>#;*=CDRU$m8(vlnUTV}P?K5YLZJMHP2vYR
zgw}Xz^SMumNHoa0*!=C<N9K*Q<~?NQ-hIjK@uo1vAH6H@&F$#ez?qfXb942r`3qRs
zcP?+AIz5W{nba}MX^X{Q1uRW!n<4pZu}e_VTc(FoA2j9!8KvHTm?TxLDJ|@@VY1OJ
z)mPk8d+tq<IhQEJ^f%yHMe9wSUoSpgQHr+hJg)hQtG#Jo%k>`pGPhMb*Y9mzk$opx
zZ6$k0jqQcGA`kW~eYEC7-^?nb7ZF)OFEU=9ElTA#;`|@Jq)>WR?1#w4xP}#*_;3F1
z_#W8xVL{%63kTwl+;E)2e3Y$!lEaPNx0ZjgWj-$X_d&`M<2N(+9V&YxIywGOwcFIo
zEfb&KlF!!M;$q==+#_hlW~H@ro8y!}&FMO_VdnO8x(lp0x0yEeKls&?%C_s`lI97O
z&kh((jC`*4H1!wvmv8K^_#0odu3UP5;-c=rOEPv0QFkwutmx<xojCDahsxxo^EsOz
z&oSMZ>tQB&*EL2tW~Q;*x2IgiGLsA<7H*1`EOEa6{!7cnc`~BkXDo5qmU%predEeo
z(_KEEli+z^ykei(%|BT}uL2oZ@7wJ86gg*aQGHCEn1ue5{>2OYwhB$E3u<u)m*AN;
zJ9CXo0NbvO4R*%Nt(7OZx@4^KS)MBNFL8*O`E-2&&ygN?Bm1hl#Z!1g_?KQ3&e`sD
z^6JDBAsgIv{FbvXHeOkBW2NTPsRa?YBs*N^7oA%oWAf$fmNHgZo}%8xesz;~Cb4mz
zkm7xmQ6k;^oMo3Mi|eZMje=QQJ}JJOr#M63>X_t;#-3Ln=0yssweg=$TC!}8N@0md
z(u*jw(p{Q2%Wm1kcJB5P__Hf5Z?=Z`=iUi9r#wWiYB%N^8NQ$2mU`rY^qc3+cQ3R0
zr?31yQB5&d{M<6nYpFjByy8siHs|*k+nhYgH1`-M%fEnIt?JJ_g!qkr9nLW24X`&d
zjXaWidEOc2i2J7&y$t<!Goa+xlN?K@%1w*nG}p`AJl`B}F<ooxK8G9HSJ<C=_;2d9
zs9a;?(Q&g`vhC8=<C3c{cuZiP=`z*D<nrRr9P1=H4Q8g9EG}Ffv_NZ>#Q~YEPEILj
z=Jl2cE9$Us%f7WPVau6E&m7bG6hrp7DunQ{{*!(oKlik8kb<nrt2ZX6L}qSRR<@6i
zy`~#nz|EPwEPp|;ymFO64#TuIv*2#_T^u(i_H;cg-cTR*ldCG;=5)I5^doVm?FJJT
zXoRt9{Ytp(tg>*1-%9qC=~MMo1FD;3C13063&q_pntMSmcnQ0aZXjQiTX|jcPSHib
zxvWYn74{h{I5AaVWrSOnHfN3D>9kuDLT@~XbSPs}K9=}Nt^Z}rxuUOHt1KKEbo3T(
z>l8e7AWdXeZ0CA|&%G5}8)9e7+R>e1BwG1LElGKi^PS8aA6%GSryfu_Z}L)n*9Mo!
z6-S>WZcY}Re{7LhNu0?G_1=xO6VhkbIL}=byIgwp;;G_%y*=$*!mQrB!HXZ6nN@55
zo+WliEk8b7-ArvSJLi#vOjiE1d4g?BE8k6Y?~+ysjP2`p`s}q-Z{|}!exYp+Pgh*p
z)tz?n&b72f6%%F%yJfF#tJ9F3S1b9u`O&qx$FjmoCh*n#*rQQsZYA+DVOiiV!xu>*
z*QeX}oC!*m5OM2R)7<qjYF^5mg^M?(^ogGN72Z+3P%!SQ^09T{WjO~DleY?A+U8QQ
z$D>PHll>C2=6`2*y)7?#GE$Dr-10J_*-`1@dxq$OIg_H4R&ZOI{%H>G>hmcpb6Pdo
z+{LOUJ+}44gyK|=$!AS|?Vq7|=D_U}yd_p9%4Mv@3v_R!g|R%ee&M;?H=*sirO_lC
zLyc*zvr=>?W`5Q+UwX#3E6L>PlnpP!76$ny^PiCKtW%x2ta{FszRa_q{I)5x9b1-~
zbxk7mp;q!0w#~OglNlo%J0%{qaqc%XC|3=hUTf$h804nHmD=jv{dIM4w)5GwJxMF(
z?%^p~?Q+D+chRPvu*n}3cCIcAk9^+dv+80T8`H#>qNg4uHHuw`h*Vf{;+2vx-|KkQ
zK$cYzo45HSnFuDR8e4x1d)CVl%ezv0)mta2TO3MWNk!9m!<B!qJFlMPm2XfI=&He$
zoHzA+NQ(Kgl1E!4JWp@i7AP8deA%wEU-mzI?@@beqo(<(bMwuA%}J5<>x%RE)YF!E
zWI^{-yXjmXR!wl8EO1z24x=<5Z~nARW{&0>RVM3tD-Ar9U-4AEFS^1u^(9xpvlkQe
zg=##6XR)~4_^nis^FVyl?zEtd;if`s1J!3$S@4ylthc+c_xqU%3+rpxG%meJ@tNi}
zX^PWS$>$|1ru>Dv0l(`vdPJsv2^R1yiag2A@Q^LFLC8~`U7Ke^qQmPC-!5&xq&Y>@
zH|9mQ$iCjdb-c&cg$wOnmU=wZrS+(C-~_2RGm11bxn1jjSt&=xJ4CVtN1XcbQR`E0
zVvdeV$LdctE023PEVy%nUzBg==8)4Nerz2&mGiH5&&buCT(nHv@gV141I-J9N<|A8
zl#@ENLzINo73Ksrz5g1JcSUn!(5<HH6WP9bsJ11V>YdM-#w<2%f2KtGj-M7Du8TBl
z4$7W69J%h;l%C^3;h#=dPTHRGYo?F1&&q>SI#}%5MH<|ECY-Dg)qQC^bD9a~zw$>#
zMRP5^s(Yu2G^kwul<!lPzv%3#%_6yt(HmAiVsNpVmmMYbz}YcR>&MPwhlKdFh3g}C
zc7Hm1W2x9y=ap+Fr~In7e&wLG`e;s<)Rmb^!oQh)FR##JY-wH*K6BNK1vfaZYg^_O
zYMOldY<R8U*>j6`ej?X}cGa9|`J$+Bqer}>bgx*K%FeoLLQw+y%GB!k77Klz?$S8-
z3s;L~z(@70%RBtaShNhi(+)KH`90eb5jIc0Xm!i;P}a806Q4Mgel0L`Hw$GAR(A4l
zO`m0>^U9@j@|B}L_j}fIbj=s(ciWT0I!(i$H)%&o@uZ4pyo;DDE8{2V96o&dg~72K
z760?~w*9J4V0gxLf`d7=^P$KZomrD><VAZVJ=34ualL4uaN0do%ysXXBW5Rpj7}ey
z)a2UoAV1PeYQFxTwMGkOamgs%(|)?{R9n$hBRf5rww0grioy*(9uqBmeoA%m1OE9n
z0qh=PD_2jbYQMNNFgHZj<<>-H-ZMpNCwiU+=`LH_cK=jLdX3#AkxdTD*?t!JC3gP~
z&9c%>ke#$K>+^%)$Leu0)6cOBZkjwH@XVe^Ee6dTFMD3u$$P9xl{hG}f<=AV-iblG
zHB1i+r}a1`Jk|N)#jTNWsp;SGmXvFjnW@WuDSil&Jh$$WK<CeMDek_lff3?utrh|E
zJQEv@r<)oW_wJO~T*iBv_pz}48oQ58+-;4Uw%ofhd#Ab20mUOPe=HL-{`z&|-s&eG
zoQi_Fb|1W&rQDQc<6yc#?&}1H+S@)%OZ&HI2DPNQTLs>J_#pX%r}EL9FO8;Elcj7W
zH(Q9$k}Kd1sp^?5VZ$fh^}|PHv)2B7S_aqb?i%d)@OO@R(8=D=>fS)>W3mAzb;)N!
zl1&Ph++ATPxA3A;OZTjOLPlpcr)4r86W)+(BqAVo&C_68%Hmu>A&0k{4f+<ZW5^0H
z@?U*W;F89|LuCh*>KHaI^41hLmwF;~)GTI&^3tCQOBt$)(k|>-C3VTf_pj_#!HWFs
znk|I~<aQcxg)MW*U-q$p&BJAKsg3rsWfwVGz3qhNN?Kb^Tg${V(eS1~?8@bVL5ttA
z|37N3v+d|E-cGp{IaM_+3d=H2HupAJDop)m6QTZ4+9WA=mfLf+*Se>^uKZHASmIX3
zf;ktr&6uf|`fBnDr9%g^EV(2vH+HdWuKP75Aob#@uBc!$iF5kCt9x6G48!+|to;2a
ze^TL^ga0oCJXoi=ru2t`QGLS3j2r9H7dBYf=Kf$g^J-=t_f0pSZFy`mg$I;RxwN)D
zkxia*?Mz!|<}s}v?~jqs1m>?0F<i$zQ@t}tqb|ptcaKZd^ok?G!v9vNnaoo9sBoRd
z$NK<(h+tO9w#VZ4<X;Eost3-_I>X{rVU&A@cj6vi&pkbpvOZ`juyMp+lsT)_CAnpR
z`!&%=M<pLevmQUuKT}M!-*~FS)Q%7v7Jrq0)50$9@Ue1}+<H;8=|#5P#YjPi<}x>}
zo!8#&x~gu&<@_Z;exG4|-XV?6yTTs|o|zPVVrK?_x5o+X6;s$2zxlwQ-um5Wo7scY
zOZZE4Sndh-Z;@M)<oYi{f@AWJ&dR6*y-o=t^JY1%vygl9Ynp$F-Nm0S&sRBFOj;Zo
zv~okw+aDP@+s=Ns-u(B_#R_TH{MXVCx%68qC0GBuP<31KX~%!91v4M6+EFUfmDM4c
z_ag7am1AtrUtX{0p0~Pd#R(qg*Y2A4+dNhr<LGr`c1pI9*7*=(`t!$W3yF$o)y9=9
z&Z(0MpKjZH%qO{zNiIcUMJ8{lo}k_GHknm!YgaCmyu|VGy-3{W9x18me)4gLmmkXE
z2*1QSziH*{qU^=@cNuyu34Aw4H1b7Z_7*Xd)E}m8ht13E@<bw}Z%dk6IZ6K(FlW7M
zq9$oR^;^Eq@w2P$_ns5I+Lgz1D=<w`Zi(_#$B+)Sq>A~cjvUqcUGFUOwSe*BmT85>
zg<+qkvsO>LpK$Rbn_!jArCC}xOf?#Nza3e&BEaiJ@519=J7(`&#x{Mv&5EgwalId%
zJQ~v&M0AgzJD%v2vuSFuQ(lJ0#fc@hZ|-ktHQ$=SA@EZ||6V`Cmaaz=bOdL9bWTr*
z3Q*zSZxSYI6ecFcHpNd;bbo2u#!t0w?si#>hVvE&_swN}a^n_5XG71GSz9;q2DpE-
zi$3+(x{Yb`qs+$Wbq;}vC%eUs4`njGt(f(HcBvb4?&8m}JXY>4yjQbClNPjgnOnXM
z<5JOnYOqdfa;;VG>A91Vm#Y~sKO1rQoXPETO*?yvzb`J6(607jUwp_rlSQ68ZN*~)
zRWsF(3T*<e91k)&J6x6&8y!A!ewO`VwX1<U8zZ-VnPvE5mTKdK`~xPBFTB{CxNJg+
z`(x?3Wp?`xhNzo-G&f~%{LyVJf99p@f)^d#%g)@dvS?D6TYBnYNa~vSYp(OIRqdG}
zsWrpk^qQ4NM6xz2Zl3z}`iz1!FYCkKbvC>)+0a;Oy>HczWe*>ANrjr8Kj1bc#cb(o
zv$n?{g}E<FWk)M+R6Vrf^3xxat}`_rERoclv;L@Z)C@^^uO8=_w|8o+3wb)nr}e2x
z`nB&KvqXA+7liLi$#i)#<MDnS^Wzeyn?A9xSr96eEdDWcbx~PaM}w@YvQ_d9t0?($
zuPzP`<A29kc#pMS<TQ}g;JmXlxWPHV_rV({>!h3+xz}b&7%1~UU9!NOKRzomX8PtH
zW&@RjyW2W^?N(^HYkoNN)RNoJdLhGYuGpME!AD&rI~-ms?MvFH=j<41w9nwX-2Q7u
zpFbG?@s_ocXBGVPaNcrle@-WptE-Rn&(!%|Q<$RCUevkh)f^+0xQi<S`P2Tb;F+N_
zd&(-EZIb?m)rBcd7Ms^(E)Ti!QtF^<{JM&lhcaTen7w(^+4Mf`mHDi+0~<Q`|73Y>
z_~5trW#{DCvh}BCw|(T>9`(?nac=OM&$2%@CAoYHSS2xW;l)<L{>Jm-R?nG^^M~Hq
za@qTyWaE`cz3rE`FIw|^-JT!k0_1kywwl|sif!4eDHR*!&(4|PI_bwm@n?Bb+~&Kp
z7_+2eBUxG6g)_EZnrZPbIznn{Ln>>=E-7L4mvgRNoIdHK$d1VDMU!*>Y)?88{_*DO
zRZ715nPzpgzt8w|_hP(j=48>xV%fHzeRgbf{nD;lE#SDOS=~^q>}4Nv&q03f#ytfO
zT8o8tJ$^6tbIQk+XFRq2EkDI-?LL~c@rLrOy&n@U+14;#f9xo}?B1uR4T-9qE0Ye3
zv{=t=oSVAm!GqvGZU=1j<n^O#54K;5EX>YhH*H?$)iagxzSc66n0d-4n*)Vb2{5Hw
z9q#!U-@<g%V0X9Kln>GeB|7%|`u3gc>-aFSefrUS&${ExZ{}R^<@L3yZJ#UF;<vhR
zd340Ony}vw*K)mmZat^i_^Oznmu+Z_NpaYLZIX=3S#J9=e+q2T6u35J?`|)K)vDgh
zr%Fz*+hUM?NBhvS%YTn=N`1WZM4EfR{-4Fq&sR+1%=VF5TPiBlwlkvD!74e_daudc
zLq*GTcLk@t^D1~%;;}kwo0rEbht{Ms52Vv)#awCjDt|v`zV<~~{#~-Vj|#jW>Xxic
zkMrCwaPi=ziHu674xZv`FN?3=B&Z*pP!q6czh>2?Ek$vB;a2QvKi@BSu=k_uMgFXq
z1mpi_o|$GOg<fCB7<HAm_}Q$SM>a8`D<!A2Nz^Xcnra(Y$r7{Jh0pr%!ceYLmt{gc
zmUhk1Z#%Wgj$!Zi#)-}c@9`{}9ozI%^J2G8!tH3kr)T7i<-@O9vdr4N*Ck<N>&FF7
zpS51zd{d{oeQ(T_X-hZ?%FHfL_wD7LA$dwdLrwJb_qeasmmR#s-Cjtr&B%W-hj;N2
z-`U2@jXs5mTHi%N!dem!ZSyVUVN*_8qG@<uUh?_cERn9ub-vBXhnE@|t$iV!uiG>G
zeb<|3)7OihmM)E*cwdS6gL(0vtJ$kwe7xzLFwHMG*!=Y8Rqwyrwf)do$HH8)cmGRY
z<IYu@$1Xpcx?ec!+>_Add|tt;ADFD3e@-cLvtHk!$2%{bQ2yD~^SC4Av2kk05r;cY
z@e>Yx_Oi>9<X`nU=&8lr?;1_vZjI|4_E-K{yhzKJ^+Uyq<-cb$ru3A}<A3pe(fg}g
zooAW{&X6e8S$xQRG1n#6XZKq*F1R$Btm>0)c(_X|!|Cbyn1qLEXAd0H2<i7?;Wz%X
zMPN?n1&=k{>zBl&6|;Sd=JA|p>{;w>^U37Q`Cm?+58w9iCGqOir6s-A3GU92>Yety
zcjomKFKk_$M0Hg(x5zST?Q)de;B)jvn%?AT2a+6K`br-@U1}$>?17ZFQI+FUzsgy{
zOf?g4tu>$gSS|3*k{yX|H(A^Mu^qXXm}2)N>c94-YbUsLR>ih%WN~p5UJ<y^_?32%
zrhch~d8hQ5*#SNk3$E2q5l^--&54+E@^sCy2_MT%#Z6kLiSnr(p8m+(>V0Kt-$JR#
z9^P3OWg{8)MOIIlxxcxS>((;%TDH1#r>vzIw3cYjRORjzU48ob)|svGDvz{;XKvbN
z$KAepM*cO;Q(T{}R~H<8{L1CoW*zC|+cEsRzd5M0qy_|jpB&s_vg_xa^FIrvS6^6O
z@}(<rHhayHr7zrFw@C`xsN1em)UMq!*L=I)iq8vAownKi`QnnzMz^g*_)1wWPOTJO
zd1~3s{gDxiLeCbTE1zc@JM~U-mF>akkYk%Q;>684befv8+t!H-dCoP=nqsPU;YX>%
z-~Eo<8#uqNb(mu_f61SzlIq`+v{nhHP2G8Ss!#CIJ(=!&U)G(yWgS|&Y^B)QhTRHx
zQ?`k7a_!;^ZJ1LyA(U~^!mjYJ#05;#E==!Quk5f&MZh(u?dCVJ_!qvR%qzZ2Rc2ki
z8P5A;L6CdIE-r`3$r?hRG-9J%6SVr{(pFD&^9lKT>B7A%sij;J6BYN|sIl7K^k`bT
zLuj&(o^h|JkkxG2RePE=-=ChsTV1j-aOo@Y)f3EGt4uw4eT#B0bIsODFb}+<#u)i_
z;*3Mf8MChP&YD>hzwW4zknf@BV5?^l`pwzeckO3ROkedU>c{L)*FwdgFU>e)v$&`-
z@kMs{k9}ERgUeo+T;u<G^k=~A?V1T@AB}?*dl|Q^5MB9Ra@9wtk5d$$U0<qthRZHi
zeJgkDt__P`FDbkB;R*k`!mkPIqY^KCFFl*SP38bYA>RS{NfO=v=1cIJ?-p9HPr=tH
z`P-}eGU~=jd+LH%%y+SEDr;k9o2Rs;UDoi6vVP&dHf77l@uKeBS6uEcQdp3>?|S;Y
zxk_Ajo;_d3ZnAM*;}^4{y5z4c$Kqc{J!AHCY>-)@!83WAWLk!H@QJjuYEwH*%Dzf$
zJiTh&T#3G;tAaO+ca#Wnuc^wjI9RbF>ulGnlD}*A*{<nZFd^XDx(EMvzx=7u8@Amw
zbI0bxZkLliAFJuEeO>E#U%TVX7734C74;W;bM00zuAcTZYVS)f1<gQl$;Hd`C#A{E
z&U+wjZ7OZmA9En+@h9>9t|F_r?|ln98K7Sk^tf2zs#dyTSURVu`h}T-X?EOC%JxcC
z?&@MZ=Dqsf^e?}(-g;k@Hk-U(+g_Mu-LG$Yr_xsO*~Wc|i~JcP(EHRqSL;ZRm4T@H
z^Y5P&KfXJafBVnG`$GLwy;C<IWT|^#E!dP2=sjcZjU{0V%!)32-(&Rh*p$MIOV&QD
zZ+yFYU9WNkWOE1a;AQhD-dc5dMxv9!x}x8`mo^#hzb<jh#^_oom+ohG8P{#oWyJY{
zE3%(XH@+I%S9aN3;?nFAF`-B)yU%rbB|&D-C(KS$Z#wi+(8}|R^2!irp?B&Rr>i_T
zv&X^P%X-Gu*OR-_RZ>;d=9aNtW9i6!aYn45)F?dtf#kC}H~07JI`zEWr?WqO@_|G0
zqF2}1-O4$+cmnsy#-KDG_RjpTALfb9oq1WR^eRhJ-TcSr8chtmS0o+UrFf3@L$#CW
zp+$W!cDT*j8yS(WWAI^FoA=|hH*!tI!`xgI1NC&)S1h@zBXwc-KE3ygQ<p`3H!^P7
zzdx<nKd&q>d(xqh_Qkxe7hj&gb=t{Sz<9o;n&GQkUk=n5<S>hD>|S(M-=osdM8fFW
z$F=Wo{%qR6!djriqjq)H8SWD?k^FtK_tt(Y*--n*;Am*vkLafcC!)h*8IEbmoK^E(
z=C#${_EMONwLx6LH}|<`cRil-#AWWH^PkIk7yR7k6S_tr;n6DTE;fy-&2ttl^U0Ol
zb&A3B+J#H}U7MRTa)tK#<_fRPa=t3k70O=He13J_g`SwBoSuiZxQ?#gDlHdotRG$(
zIsa)|<@}SAd;h!DiT`}LR4QfS_7zWeYIUXkTXxESrhkat|5blB2CaJa=Zc8sfwLBu
zQZN7ZnC)74UEvA;$J#G)o0k<EX{JX1pRVWD=_p&ZbHh#t-W8!Yedf=3I%)C-Wp~~M
z@3@xDJ00-6@alKn*Q@SZ@BhA&<+@00(9+cc-K+QXtX{MryG`$e+9LJeeo`f>$@f40
zTjt5_zNwho>a@q#?WdCVU6DWk)9|vx<jU-)&2#;AxM#|}{btAIbjC?}$F%P{nq?x4
zGes|4J=C*u&Sw3uYZga*O}NbL`)twAMazCiU1#vk5AoESUfantztLNVMeLq?Jnv+w
zgjmT2H(idaCHLID^{zcA{pB&I_i(>Z{pN^P?>5)6+3PmzKFN8q<Xn-(g;nau8MW5N
z`>1nkIY>=u75A9*aW7kz(`~=0^TQ4nChP2)C3nRkl3DefhZvI{U*tt)S)Ip|G8P2y
z?LOrHq0_h_kni8kKNVtIw$1E1-E{j)uhIR^^@)Pw!G#>o3$<TJPhGrW`(~BG=?8kX
z@4wv~+WOJ<iL+kk?Wac$Ox4er<5jcygS7sW_xfKIyNh?tS#)agD(1Y0wpk4Qk=B<&
zCrF%3w6-tZtL;(apf|1M3A4;Pnb7I)nas18sy+wKa_OBFaWD4hF9V0G&A#SYOlI20
z%6L~4oxk>+VbA=djvra&zi`UGjC&xe7vH{lmi79kIjfm;Z(D0f-GBO}V9G=Xb;dX*
zJ=XuVD__oxa<t6v+{4OoswH9h;bgI1$)LT(A$8SP_C8YQ<8$cL-?@dW|8vf%p9;1|
zMOt&*4_9O!mTO%aXx+G>rgx5yeTKu?i7#scE7t{iO4u!X{<Hl&=a-+62{U{zY>Ajx
zYw04>$7gHt&O862xcHrj8x!j0uV4RD`F%~}EW>G?1<z-Pe7Uoz?&+RDQ=Pq6XEB{+
zoZBlp=jg1t^Mw3!^4F|8_HwP5A;%;CjJ+-SKMS}0Q>}h0!)A2-&AAC%!dK<H=*5^i
zzx>$QJd-W;<o_lA@;lu;c^1svDAQqMqi4$&&KqMoA?e9YQ5(-C;kygtDwVr4RgXT~
zw4ZtXAG^!fPJ7*ESoM0AnZoIiHy+o@B0kT}>R)_DdtKICmYT5D8KsPi*FBiY>-9PM
z?vnVa_k+q4|Li*TPeymfGJB6}a@#r%9-bQ>oMd>KTh^2(r|j13|L1qTQeCsSA*^QC
z#+L^z^Db}wRGl+{=V#44=3P>OGqW;2TuBX$of7o?(}o{9SIuozR<y2=pJbPP>}PGs
z<pt8oF^^`mYG&T`of53H+^kis&OLP(TkoS9gU@QAk_<;RY}j?gpXr!B4tbsCtgin`
z_?QhFo3hmD$I{8mW_s#KuxjReU8+6Xm!!b<c$MVtx3fb+-dEOym&RXY2#n8BE%Yg!
zo4#8$&h)j-`^&q0yOnuE3@x@6u64QTwaFp2XX?Aa1jd$znMH4GGqV`q_uL8JvuORf
zrp+dYb#^#O1U)b;{rtf6rPj>HFFs5%JrH=M@BPe=OI5xGor&p^=!$yPTKw+XGx1%4
zo`uVPUOuBNZPMlO!O<vIBSGe7sNuvDk*-IsemTE1^5>e%dgd2Tlt^nkU5r|xwr7iZ
z^zA*N&$-3xIv+|&D_e9|TZHPQpA}f1_4Lfc<ZaD{E%N+je9NtVm}wVB)t#TWb&prc
zr5GRG+41h3;yK~Luc|MHnOd<26uNy^y}P_hf6cOf`Aa%W&+JQ%VVXJ5M)8Nvj1s0-
zPlc^!z1&fI?eFyVV=qmMUb!sRPdxT?>W{boli#1crgFS+$;Ik=vHb!E{{D@B!*aGq
zN%N$#Qg&6^tND}T%cdW?tEs2ZA)D~fz3rh4i@0{d&UYp0dFn+^V=tT4<v;#5Z?jzM
z>)&hqzSc_2*FLUxVOGKJi;PxUW)C;7shr7d>Kjyit<)iC{;y!;C65%ocX{U?`aWyM
zsz;k2%rfV^9kh6lr}4z-qqAQWPEOGJ$~Bd9j&btd&uYFdb&}y5E3S$kIPpG^Z*Mi%
zjH~M!H0+q<BQGXaxa<>cj(gnSc|T3cPJ3a8&7#LWYwXW4WG*=_f4rx{rsMgi{?8{@
zy41=PZ(Mdgwz$Of!k(Nx3zFU^r2UQDx@FPVO<LT=Q9qt=tULO9McAsTOdS#pPu_`s
zC|E0#qWr>ZZ-a+p%>4zfpC<bR7ckb<`mJ;OQ}RIYrdcqHx6GPP+ryvr&vLpV@@2w>
zWl9yxq*W@8&-R@0E<R<YhLnJ6$I~STfB&kv7a{u9Zqv@p=_awij{f7@YhV|C)?GMb
zf#aIpxA)t#NINP%NVn&HF(GN|Mh(Z^8>Xr~exCBtwCn1IN%{ZlpE`wJ*ev`(!Mt_P
z))hKuj?J*iFXei3=Fo=87R^g5uea98T>b0rns|lhlJBa2*Y>Vj7J4C3x}rLFf|^9l
zoxqcy)(6Ry-WN^`WHI`hx2up@UH0!4RxiJ0;vzcjlN_bur<NtA@T@w}x2gG$)uV-3
zj}OT{XTJAW;l5YFqs5Ez4xbPR*}Fc^=dJI3gNB#$0`K-0Z;~lx*wmf$=vVC>--$nK
z=87&@ROj)^wukRaQjq)Nj=AZj3%+hB2@Q~3S-5(^9seIOTN4}~$Lut(RTOaPRnQK8
zeE1%Z!^Mow-o{)WF76Xg9d#o&-9Kl+r2NQ9=H<>Dj&=O+D+1nEom!JoHU0Za_qnhB
z>Dj+dd+YI&gZE0}=_)T4jXypwT;}~d6Qf^~UvuBRp|`7TQ_#|PRvcQF#6_Bq{gBV`
zJy!qod*lD2(-nUnMPK{f_J5y`eV_J?pKJkrtHKgYxBYvd-S~dLTcK&+^)Cs*pV;D_
zimd2m{h9r1b?7Hei@q&Jp&DtNOeI&Zbm;SHr23sTk)G73lCioZe8Q_%?ev)Y#e7@z
zmLHrqX`6fBI)%Hu{=QL@Rc8pl|ChM8<gDFGkB5#qK36I~UWrnE{@(1_>1SKo7(cEo
zV9m;CoHbJ^>euYf{1?~O9&umRXcv55(p2Ne?5u!?>ndBteJ6VF@;|ZaoXH)F5R*u$
z+bl(A=Po#Ba+_5%z)?zkpLpR(qmM@~I)*hb5_hv&kyPg68Fh#2=YtPtm@}B8YA&+Q
zy64k3d%>njj-tn!Y>FD?%wN>i$kdqFge3ZjpJA)(%ITDv+qm^I^WL0B(LKML=Qgnf
zy?$-{Fn|5AeqZMK7jH`a%Bb2TSs8lqqeQqMPj2alFA5PWQxsnd74KWq9&x0kizTsU
zp?H^+$W05kw})5Vch7ukW|hITmg~_Zf!BK5{$1(eD7vdF@!;wa-$myv)9iknv@GQO
zvv^a=5oU#TB}-UVr8as+C1m~%oZ^@0U32aGjx&s+cl_hZURb6Dex2fBs>gix@yx4@
zYR+EzwOS_(MW3vAHNRIjXwIrh{;PdHB~JM{eOW8ty%p=%u^z|~3lTFtvA^|uu7-0)
z{Ex6}r=@!KpIoZTt;6%HQ84%Kalc!?Qulqi<!W{+N7zzQXP@B?Pu7`C+R3bO6Q!kI
z`+kty>G1q|hufR{ZA;D{HeYeDef!<7yI+^{`e<&~Y%=huojmX4@;98FXF|inA11N7
zMSp#CY?owNNM!2lK*v7c=-Pho1V4F^S4#cX+e7p2mT(KD$|?F!acW+E>$sIw#~-DS
z?D@}ol>+P5{#z5ZMeX=%4gN^piw^7Ms}^o#j{BsySL0N#=+k3|a(!Q35TBE9BxG`v
z<wuP_3*Kk!uV1-p&#DCn8CTAlJ%iPL*RF|Y|7TQ%&%N>D^6a(?-*sPU9=N*Zou#Oh
z>MwDzug&wG+uRUbJ=M5!zVM|toP5eV`wI$OS;D{mE}kUyIr+k}ixuA|<xTx5D4doN
z@noy>MAm3!<q67nqyHW~WUjSaJ1463F#G=89O0jfFFl{YZYuNi@hh{)qd#TX9p5@#
zn0$B7GtWSFjfHPi;woNMuWj>C-^(nUv2B^@hb(^e^GDP7Uifiyt4v;Vw?yyiPKKoG
zla+TY<7;I9Q1STCC-ZziW1EdNR=a;KKj#0(Xanai{`{#s+QscQMQE%qJKmgYWS=M>
zX=!|k`C7bAiJhpQb2-DyZ+nj|>y0-nJIveC-xWK<ZF3;sj-KVq;(y*ezG_eA#&dif
zyS&)DCOI$uqtMj5=-{F+qCN3HY-^_-VEMKD*Yf85mit2{x3RyQ(eZJu_w7l$I_JbD
zE<VMlb$`tp_HdQ&)%Sj+J=;EY+Xt`1$&;VIybzqczvI#;Bh8j`DJwSy9tpW#H0zzT
zb$*L0d!sOGi<^7kub?Tjxx^V~U7Pncgx#z2N^PT+@RQdPv*#^3ku&3j(94Og%L8Wh
z-e=+#>YbvjsCue@$q5%PuAJqW-VYD3ewEs}qIr>ccU|~K+X}PCm9v{ogltwe?@l`~
zS>0DvP4HBqd&H!@JoYBK(`0w9^D2qGperqtsZ<>E>uVl^WoYlc4G%8<SeiHW5aa3h
zg*U%1G<Wo>y8p-WbNZU;PbH2&)m>baH^p3)Z#`%Jnn!<^{gE?k2rM(#n#1@0acH>t
zdY|7rs-~>>%i1}|tTA3~+mTF-FF~%NRtuw5Pp!60S{E9>x?bn;$!l8<xLf)b#4cHu
zDpl!Y{DF14<gDvoHB0B&OU!q_`uW25QoTkYr99<hk6mwfx21E>DZOWN!TzJ7GH+O1
z;EZQFJ}a_KOLNz+3tE%D&Z;_jTi{Z?^*^2)C-D3Gohb24ylfJ?KKolwko>=;8iF=n
z{nx%8JR)a1xsIL9yp_`>^3TT*=PQ0WN=JJa+E={#J?o0&<L*6|XInn$3HN_rSg$mH
zqQ9%>=d7t}n;k-0LZ<AIEiF}=XTz|e+HkkcMYDU^Zma)QK07|ioj1&8(hC*)MZ3-$
za-H6PELA<L<cG>C&rR8rT&AiN&O6|o*HWi3M}VR2{Nt5;ChBos8aIVyLwZ9jnXkVs
zSm5UADPHmEd(bkc84=0Tzo)F+sV4SOH<xv)So1OVddAD%Qv!sSo;xD3=E$5c^H|U5
zE8q98VcZ^~K4sORX8WYC$FjHd-af!}`{c2&YxJ+GGlm|%{Iw^FvAaud)(o*@H*_ag
z%}nW?F8!P*L2B97g$<rH4~y4CoIJbCGFoo_)eSF>uHx*=jwyZ2+TE@0dCGHL?~>>k
zVeeUAcOPXjzkWAqO?uw0rI`orC>E-wpW3>=bS;zE$rbwdzO%9CC_d}WI#N}ya&G3!
zmsM=GH=mw9*6`|>8@sD)$le>9=KYf8E<ehk+Bt{w=a0Q@9If|%Jqx{G<6^@Uy{ClT
z>p{O!AgkBeiN4DF&s<3{s(0S`C(d`y-;}b&zikYwuU+!pWzrUuQ6G3!n=|bo<96{^
z)n7S81vX#*QhVmctbGaWJkPJU&ezOXKUJZRb@kj5Mz;NrxXmU=Ydt*p|I;UCucw`f
zzM4i0|Gl2MvXYsJC&jC$$V)^d^7STruKiEWMyy#pL*nU_h;1&c%ICW6-ItlptD4!l
zCUC;U(mnR2j;2eNE?Ld-;mIF?jnmzt$|vo#IcZb3E=co^r(&|_vm%?xRva>&+b_8^
zhTN3Wh&*?G?`{{K#aDN?+Z^nkt}sC*twufS#P7uq8N&`f={+Ly@!z6#j@>!4Zm>;J
z6uiX~^oqasQU3Z<bz1hHCIntvF#p(>nZbr%eOi1zIeydMRCi&<601pmznS>2e}Cn{
znh|%WV@a>amuugazpidPCi?Ke-$0L`_Y1tWJ+IxI!t(A2hu4<p=h@}np4DdfSg~;R
zpJiK4=K1iO_5abAeo^H2<Xycj^Zu!iwLa}VbWtc?zb1vRnc@DCps-a|2Y1_<ELp~}
zT7I6CnOxQ1-c^%-=x@|*E)msfNPBi<N}tN@oL4jDY8Hh@%@yA4SfyNdDreGF>EDV|
z(_F<w&v|cgRKGb-Sb2ib<W-S^!udgWB#xz6>KLf=W>?L-xHoJ5VyhFr`Wc40uO&_L
zOJlw!$?o60cf+a+JuP3wY%PsW&P-~%v_mQQ=fxKag(}OPog4i!bSuRQ{g<U*+Whe6
ztU7@@$JcRF-HckoZuU*SHY3H^jXjcg!vU>0^~DK6p?Z>gC#@^-+T|#|{g%^(;K&sr
zM{lP}`*nTv+;zlUla1qh&`zrhXFQLt+Oe<ajOX@8OSQ6&U0PdH)O2_6eeW~-Dm8rr
z*x2mYZn<4BetY7t5Nq9`r!RIL3k+Q_Nsl{SaQl(p4FL{EzNXaoR2&u0aTna_`775d
z)Axe7rpF~Aq2{RpF5CCI^Y#>m>ltn9JGf-##2@Q-vp>#fuebi8E-S%R@cE9#qBq)y
z8AUu}GL?&NNFMr=^LTUIcC)zfX2s4vw#3)^8uCufGhfBdTYP-g_QyGP_RIcWFY4K7
z8J2td!s4i3Gd%CuFZ=s+oBo<>wRa7(Wo4okE`5Gm`AbgCmcNhp^~C7wFJJrB?tHE9
z!gsqxzeUR4KE9*Y>ezy;*SFW3N%3##xILw|{^qyx%XV+<R!V#?f4uZ7-;IcG{a-&7
z-LCWd-E`S*eaXzyTk6xwek{>`HlN|K_|cYKEde*9m_^_I(a5XbH~ZAL;;nBNhu*h3
zTr6vqf3~xmJ8#~NX=**w4(?{yTAu&=xzRQS{f`H;*_XasU@2HuWB2WRoSw_U^~+D{
z$+nz+Uh($9rGzU!7WVO0TWVe&&<nP^m3nGA-=&S^3Ay$P%t}^_mF*d)e{-gvx#}W!
zY{t6z<xi5H?bnt4vCsARKE=G>=f(eQ+!c6tk8+*4O3sg`x%TFz`SbN(OxeD2d)3UJ
ziMJo5+8<2O`21X|?8Meh-{)<=mUq1Rx%%;Q{%^iH-HKoF@zc(CvOieMuD!XGxUXb_
zzVV90`2T*3Px&3(^!D_6>+mdV<xcC_n(F#tuepz}zI}Yd<%fBmx6R)%t!m8p+`23G
z?2fm;-zdbXFz&DMvp%2muqfqmtE%e#Tj{qG5|_q4o3F=vX<zW+bH)E^x|A|MO?w(5
zBd#^`QmUo6BUjlKR=c?E-`A`Ecw&}So3pmo{=f45?3|}{ak|@TUx`oiKA~f5eZD%X
ztje|bZ^!bv)8cOUu9<$Y+)?>GH}_McvJB~jP5I98FJdeT)0VE*mhXtp|LE1GUz4WZ
zq8wGA;Bv$6*yBy_WLFfrPY+&qQrP2k+595kc{{e(+?j6sbe4_!wm*C3&)!lld-<Oc
zYgv52Y5n<^pB5i^-g@A@lGdxJwIL_<>~E)@KcmmPPRng?&Az-}p_}aWl;wUid^Kv^
z_5aSdhc?%0|89KyTL1U!-;ch}jNy3FDtD`O!421X%TuX$kK0`?yV?GDht84joxxnQ
zwY$X)-d;W}{wzpu@AAh@YSqt8->nbc6L)ae;>4To{nKr`#Up;QzNvF;oS?s9cSc6)
zf(xsT>V;^$Nw<C9qV)b#&ApdyAOEKA&ycd7e{=V)J+m&~`|LOIlixRW=j+es8cg5&
z^<L;+0dspsJC!?geS%pdzAUVe{B87+aiP&c%kOqEs*Mx%?Nlaqb5vgR`=j)?EdS8D
zo3YR87H&JJ|6X1^Wd{e_Nxi$V_rK5m_MH2*{kQLjQ*ZxqI)A$F*TYY*SudXU^Eth)
z<7uq^S@EO#zrGdUDl~q(ZBE5QFL6;_9v+GP?=Ge6J8)*6#Lr8&&z*Q|_vPHjZ+ExL
z3cpqQ`ioD0zd^HOQu~6^ML&N$+H|VoVeH3C2Q~llq$VEDxh<ZQxctJ)L#GbktGw~E
zJ^qcC_~S@vmW@7MkJIBb-5-`s-Fmn+z5Vnq{pQX44tVwdJ@iKJcYyvE+n6(bFO_Z_
z_5R(xEal^&F5P!*Z5u9oX}NdHR9+N)<?7Wt{b}5d&|S04`}S7bmu>udC1%U{<&z|y
z$F2P~*E@gTd|9d3>yxWQzw28S?Uu^*&$HNfW9o}d+HvAr&OE=Sf9=9kk1cc6es)H!
ze)_EItM;3+<XP{3RqYd)73?X!VYk8LlBsdu4_=<?^EP{F)ONjRr)D3TYW|j`N<FM@
zYu=k}EZ@67JlK8kvdVIuw!N}tVg09{bHAN)ZKmPhxk|Ao9B!2^SgF`;5HIp;_1=W(
z>5p9*dVfb$+86y?H8*~0+ZK7Q{TdnKP19fP`%rLkgTD1deXHw7zs2cY+cR-{Ro%Uv
z9{XklzikS!p8oA(<V=qD+Ua}e_3!b$>$m&(kD5~p3K^g7`o_I;<J}7n!!Q1Jdigze
zAIGP<iNF6YTK%+E`0b+2S59Ae=(N1QTk_kEe|15-J|1m7FZ1$~WxQQ{Z`Sdn8D($W
zly*y)yl%ZRe;!k<&+p@Z_XJ+O=Tl{^dj2toZrQ`5yDGdmx}T<H%6{vw{Qu(j=}h&W
z!hG59op<)c{63-=a;%(Xo_cKU^4b;MH}#+KswhABRkJr%!eN((S?2Nh)8ae17*$U<
zl&1GB|Cnf4zux8Dqs-ki4*MO=I3X$i>CJAQcOQTKd=%Che(;H#>i5O<iT;)8`gISV
z>{i_y6IN!MRi<dXdtXBC^b3bv6M3y~e~)e5ymxc=<d2UwZ|nYlSnt?Q*=J$;Hd|vW
zvUXS8zZDwGBVKD1!%@aD{rvS<e(@%mb>~*a#%!+f+Wfft^h}GZM?U_1Bz*GiCC8l~
zCL~QiyE`Xs?+1zf7QHoh@=NOark{U#>!4lJ?W?(ur=J&}c`ety^tP+^?aFs0*OzuW
zcAGwJI(1y(b~o4lo5#wkIePW?CEhvo<lFS}%w6x6uX#5kz&|i*etCMx&5!%uoLTnd
z@6&Da+`E6UzAgFEdtN;GurJ50i5LDl{aUwZ#oI}n<Nq3q8!nGII;+Uh@V<TYn(6oT
zchzf7{~B+xJ3fVXztQv4Oy`BB&F0%HuvR!*EHdHd&$)+}bqAD*w*OtVVfpE-_47ZM
ze*0Y(yJAlKp7^(`Sl-2Kd=pW#%RtN}CQi>fyj#5e^|=q1T@5s?=e*K;(h_?@GqwKb
zq1)cK)gFnvJvy$s<cPkS)b0fr_>4cVHmKxf=@ifXvc=3(G~wYVSDAZh{<{x)E&KNI
zaODHl*>9x84+n*<T59!@IZV82qCTTX>G2Jl{PpH`HCzkYe(p4Di1^bFQq3XyMmJkC
z-feodWlH9YWeb09l=<y`+kT$kbff88|M+<Qf8WjjXD74j&Uy3iem{SzbgpbmRprB9
zJ9FpH<gVJ6_d8uB{#<|8X1n{nzfV`@$9}uE^{sqvz2(%?Clde9(cd4v`^=s17X{z!
zuiIAgb5Hq+1Mi-4-}`vWSHM$TXl?%Q)VCb<@%sWUPXC;I!_LUGzH)A}>$G#hPxsvX
zVf1<1-#?$;DjPdLf4<9~rIX?H?Pc7P%Xy;hj{hxJZ9nIG-(LK_U0==<*%byizQ*|J
z-rkVY_DcS9?9=3VH`04nS{N?NonL&@ZrbVNdkrTSKYKpwTmPBz*(Wk?KR3+3*Sv&(
zjqrj^8o#To6I1vy?$+s^N{<RG`g<p^>gS)NvLiNoc=pHJDLd!<jQRFWKdpR8Y8A(d
z$Ut@79g9NxUvCobcL|DWQmfwb_p|?!wf$m;J{i7|fBZ{uW#{elmH%Pwl7CIFzg^tc
z?{sm`I>|{_cg{~en)>IiZ^{<01xNT_Y1{pGe_!KUkbARg-IL?xHzq`S%4!D8RQ@)7
zid$=b@_)03_hgqD2sQ`(xB7nUOx2=>3BR|q_RcJEUAW@Q`of8v!OLIeitYb#zJ6_@
z`BbCdPwNvu?C1S&_UHD8{KJG8_p@(kZ#fWf%X`z5Ps{Q*|GmBJXZ@;&3+vZE{`+_T
ze%^oe7Zv*0JnK_zyXTdfWc)gOY<1f8{J78B$LqE9q+aj+^8beA4As9iUj(H02AzBV
zkGJ9H{)-3By)}NqGw<G7^VL5dUt0dO+J1k1<%QLD+s?B0Rovfv@5@ux`<GXD@3pqy
z^3>U48vouh#d!W^xz-Q=o;Lk6`%%B*;Lf${1%4VnytH%i^CnqavllO(Ec|!pa(#lH
zY_RsTOaEsZDn58|eS6^+tN$;iuKX%`-evz;t+t=fJx?qBJ(n+%dQJQ3ujAT}avtn<
z^tddc{dHm0wwu#srWi=cY~eI9ethrymG<rr3d;()ezW$dJm6d+yglgK0<o`MM`izh
zmpgT+ekXrx^7P7YVNJgU%YNQ*Z&>@^e*Pb&c&Y#9?;cvdcr9bPo<-yOM$!L`8>N0H
zJ*{uhdg#=V)MByf{jPoC7w_@Sj=NN#;4WFL`SaJ=?7FMdFWlFiVVB3DDY4hRQl35U
z<J0|KORn&4`n7KVt*Z3(R>#&RPqAWKSv#S{E%~sv;Kud~TmIjX`ztMFVld-C#XULU
z1Fx%%^i)D8Gp=eo(JmOpXu(&KrJc8R>dviGcRsWWe&p3(U>VkSYjShiX*t$yd&RfQ
zU;f5<Bv4%czjVW`!|7L+sH^nq`)?EoO8D2?mAHJy52MeYy$>wAZtuF~On!RnW4=Q_
zj806lea0_nvZPM*p|Qe`J>d)Xw?=Pg2yf0vvSYpOD9+Ck{P_Jf>**3lZcpDBno-uZ
zUZAx~hF@wv!zW>fhxG-U|IN=x5MVhj8>i14|LUdd4w?0b?KW2#Enyeg=9_;{@22ww
zhKv&eJef(7iJR1!C!5E5^CVR@OnkKM4D-n=$3)&YADnL6C9&pTchBd(8O)B;`np*T
zzWcjJdbPhJ>#T!cs{a{2<~q2i(A&_WuHyAF&PPn|T^MA;H-2+wyth}$vcld=es#bv
z2WjTt>72!4>+J+)&M0hlRa+2nZq}L<k>|GecFnx4!Sf)mZ@Qz@gRRPpOvf4nZp1Z6
z2RSbHo_RC$&B8BV8@`6WYfIHC@Y~fT`NhJ3+4@;&<Bsp##Tpu`c-wBNALHv6{F0Wi
zLh-``kyG|_qVumlHa+NHJ+1UvuC0LV?+yCQ;<<ZDmV99ObWYIV@`2Y25BOGpbK;kj
zasKQSu}W!=&DRH}5uMWy2MBHw+<AZF!?inV`rhlP*DCBd7m+=y;+l1}q=fDcfls^n
z+P+Fo+O6VdWwhHPB;f4KJz9^7SU#-MyZVyL!Q(84irV>Kuhu>|CZ3RR<_%x5hK{D+
zyw(i=CH}HU=0{a7XMVT#^~Prjk)6@6j~~*L<>B65ue3xWp<vm~hFC)-`8`?>rMT@a
z8W*c;D02J^co*U5qxw1W*BgGh<RztziVJ*-&BCtPPq1zB<~L-m?vL*4FMK?``OEh=
zKN_k-7~1qT!rJD2dbTevmEp_$>nn{K;wJq#{qe%rBXh2c|B-V3>c}ban1AJ(_uo$+
zWQ{R*u=?T9&QOy6;L!D7!eQK|GZ!xM3pyaOW7$kgPS)e^PjF_<n8@>?=#ta28y>x0
zCvH9N3jAg}anhHm`(;`tFf^_>o-JwPvCJh+g>&JR>XlY)ll)rhlDDO+FWhm=q-Sf@
zafu5Kax*-B3NdMMDyS5)Gn&LIek;<nwK&43<e}K59Ny;Y)$s1EaHakvhZTQoc<)Fg
zYrl+7+-LDk{{+__t`i#s4)v`6Iw3?k@|F0;Bt=`^b1osfXIwZLx!g4WPWZPZ^SzS4
zN|)q5I?uvmlYjSED}zbw*P~@mg&JSv-@AV9_L5)qn<fb~=-Jln<ZoM&IXRbS`^GPf
zbHl41e#?#1yZ%w4*($hegOSK8&I<DdK~Gm7f4ARr{jVRk2HS*H8qe?Cb1osV?=WA&
zo;b$k#?#F?ise^2EPg4LaxZRi^urqq`WI<GWD2|f(kuV&@n2O-?`zyWe#!Oe+>E`I
z%l*<n)EfVCW0?2XN}l=fE92%aUx~CBE+<`??`;#OySQXZ_oo~=R?4fHWFerSapRcG
zrS(g{t*BF&`>~7FgEyXca@(6Nnu${voY=FD<@to`a*JJ`v0hp8_MU5dORtV>r%=0F
zM(y*tvfr83^S&{7Yvg>)?ordRoR}#e&$X<V6p7>Hb6F6Td!~jh^TE<<JGh>%)$TE7
zb8#p;Rxv@#=ot5b=S44E<MbLopU}Qiuz<VmaBuwHrhOY03#<uoXOv=}^=P-;dHrWs
zKJ#{j$4Z(SF6UbEQhm*?T}M=JEvb!_HI<60vUB2pv5ev3^0}4yr=_<nf4!t7Cw+?O
zasE{zdjF~(vmI_*JYi<xwNk5O%YW<NOK&@0nCWx)-PdAah4(LSoja91V}iiH);WKZ
z7y{NY9X+QJJ@MOT_T#=Aj~`@xWjE8@`SrmS_v%)aupR1r-Tm8?yLMls`8fsFw-;K|
zHOdc|uoSpz1<B6H&kA}GGW|%SV0hQDm{nI)y>#4m9d=O?n5Tc_U}>i(%LWf+qk9#X
z_@_-2v(UFt;ADTxXqa{`P`z}fz?6u_)8;9!6rM6#F84~Y`Lp9jQ;5hR?X(FeOnMI;
z?0(nKm)@S2$>OZ-5`W5d&&<0w_>vnWQtz#0N@5Y*6vWf|g6YrkRr@cjnOt~*C+<bF
zbptoQ>Q<&Cu5ElJo!kFfi88BtUyyRwk^la|$i1t8W$AI%ZwmrzSz9}Egtl^+t8SD^
z%@ug6cJ_cq4BK6`<L5TNi#QyA>cvZqK;fy(k6eTfzhRI2t~4{R`^r=s9dU)b4Z-Jj
zJzbK_lXp<ytW$dL6-_U5h2sZV6B(F7q%W-HOVwoiCRb&c5pFN{I7;ZHM^f&Tb&mV?
zz2JQJkcIKE?VX8#8y(Ch<xh5FvR!eQr^xVx#QQVWE4C%PHrC&AXxgJ~`zF3G3NSS|
z)z!PwIr+Hb^(l?~iw-||;PkdZqAh#U#y<rCJH(H-w=U{SzVK1*730h5YrVZ^MC@m@
zpWVxL`nk%SEQyYL;r@~#{WZ(v887VdnaUQj-aEiD!1}^QmXnVsvK)2JWV-XV^J3we
zg_07N!U~VqK6E^^U7>21soeD$AFjx{Zr}U$>X$Xa0k&dpRwV*~sr(`5t7RE8Sba=a
zuL*MCoEUz#WMx8(`QayLrKZg0IO4r#7lV7qk}!u=3>POY;h$e}KWurd<InErFOP|K
zU69cFtkh^X)vED=g|Td|f$j1itY2rW_^?chX>P;y-@Vz-8^atWcg<bVd!=x~^MU|v
zhH&Rrb}sidxy`%|dcquu?oBlT$`AG}I^^)!-*%(Gln<|a&#h!$I#p&Rzr@n72Gi>V
zLINZzF9+MN3SnC_-}DR9B7d<bQ%)_I`9UW|YE4FYa#g6xIZ?0b1+}u>{I}A2;$&Aa
zPtb|TI#l5p`RuR#oilOu%v1Dvj`mG?pZ(>SOYx>=o~Z{)9ymx8GBj!}sNNH+<D`9o
zah;FK5|252Pj`J>q~ShKdTZqx?hQUan7u;X&95;x2HdiD(J|i}`-7>_{pE6xiOp;7
z@V1MptbJb@{<d+NZlKv}p{<_NiW?U_uvq*cC|{=b0~ZfVCTDWlMCMgZ$>z$Jeigo(
zaFJi=;hs}c<^A5l1x`|{4B3i86Azr>o0fTSgXHE4#aMoJhF?uQ7J?E7mFn0_*p~iJ
z&EReEJTLKfvAEFDhKsv-+1r_w62jdD(k^YByC#xz3mfktPaZyzZ%g*?c+Fo@oT4Fc
z;&u0hKaKrubDruo)Cq1_XTRa|Id2c%2P}tYu=hsXHDQ>l6x^xcR<cf*@thyW{%OMP
z2H#lMtSfjlCrv|0_73}zUXItk2}kF;1P7|Aef{rJ^=8hjkV9E(<rj%`^3KgKyUpP8
zVV%Q=d#BgyzU^F*Kk3;KLCrm-T|Z1+o(r7Vf8_2mjZ#LR^E-GhZakdA?YD6mw|78U
zV8-N*#|%PC&T%~Bnya7?xoNeaNm=Qsc^l8&h%5<TxJynjVp>BEXBUfZ%7pSiyuxxR
z^<2fQ%8w2`Uguq~EmoXSuQjf6$@S=lvRD(T*l3F-23MuqJT@rLy~Xk;QScUvpa1=9
zv$p64XdLrAzV)?BXI{f<hlMO3LS&?~>fT<c<X>ZPss4@heM`Uj|HR)!J^1)0WCdgM
zv8gq+5BeP3Tih5`-pyGgZJ8RM9iw)pG^Q<tGpO}c)Z;B-3o2wB=B)HCnR8;|td`Y3
zl2T9Z=nGr&XoGagvqBLjqgO%3TKN-2AMM+)LO`A~F-2&0z^Ro!FJeyEm2llu@elde
zv3J7J+yhdF7&JsuRIfdKxbz@<x1`10+ZQxr!kj0pSRcFcie^f}f{&}qLwc=R7Z*9G
zsZL(h6=Z0zTkM(%<Km@amyZ9t^;oZ_(V;fCZgJ4VU7~ZB?k=sDp31e{N$<A2`o*0B
zD@5KDtXR?Uc9P%2kYlOtOG=+A1S_PODX9ugtGxZ!QR&?3r@q<@XHJN}xX*C(Rf)?w
zwh1TIgax)3Oif>K=4MDOqt^cqrYAn<FN#Px&VDj;@p?wHS<SmxE=JGGPG^;lb>_O3
z?6c;*I@gOCJHB36<gjbb^tdoZ@$6d{^ZzD2tC;f6_14iv6E|Et@GCHKukRD?ES6{N
zm90-!7fxCbuXBE4=_#YF9DB}sCDqPnvgO{eVMW~y=`z6={C;P}k9haF&C<KKhk5dq
zwE=|>G|oxw3z@=wocoGo#=O=;4@4d*?6Y{R7%;oXxb4X8=*#K?p2>5TW(RCtZG7&O
zqIKGYwvP%<o&qZTrx@~ADXS!<ytW9NV&-sI!QE2Om1$)l$HPkBE$N9(7iugQ@;Oc1
z*eBi_t<B7pA!+3kB6uQbMcd9k-cv8i*4@2tdt#PAF+*VYL&+7%Zd$@CRUWWc@da2a
zoGx4t#{PO!?c1P5O$R2o&)&TA@UEV4iDo97MH(;eS!~sstHik=wz4$Hgl%2pyswj(
zS8%zpPSvvEUm(NgH(Bh}vwf1Ql^0Gr$28~P;T57M<}ydQUKYIT>bL8`lr0Rh8mvYQ
z%{>>ldTKp030iRG{_Kh6=|1{0oHt$`sXY)UBC#;WWKEE!#AU_5i~1R_a2r|GhJI;s
zRKD=@@lyU9JL|+;cV%cEt}+(j(PC85KQRALSbWqX*WMC8(|HM+S?r}uCw9y=YN+^X
zu3XBr{Mxm6FSVwd-$k+*@4f0~VtxIK>9F2`qdtKh7mn<<4BHnG!I99-_QOE_1+$P;
z{6||E{+G5fTXxuOZwR{Z?BCaiSO0Q<u=>_f%D7LNbN$<63!N)cUq*jCB*A*B?f}OZ
zah9mwg12v_w^ueZrS>vU>U7k0IkYb!qF^D%iZ=d|GAjnY3eDynmD{gO_AcEjSZ}^6
zFDm)|EQX0pr54R+IO8Pxo(TM7HoW({B39|x(`|be{mBkGXyUfuu><p>1kZqjdwA|M
ztnrBb&|Yur{NeaC1vC8<GsU(z?GTV!-61pKH}lKnCF>V|-tZ!ed%NQTO%^U$hG?tQ
zj);X#97$>I@hVE+6nBdrQT)g}Z^Ggo4g0h^?W~X7Nu0{K``aPc4-Y(RFPyPbWqBEX
zyp#FR@nc--AMQFl;E%|BF(>LUONt{)nAYMG>r2ykZaT;m%TGMkDATaINQz}gq|B$f
z1%Zo|GMKK|Fc&LKD$SfPvOT#Z@347TU*m4u7i;)Cvq~jD{d1VnB<RS<Z8oW_xyf+%
zw)?FA9JD7dz9(zAvaHkI;UA{}Z-m_1%@6dxD|c)vKIJuKDfhBp6{@8#<-454=kEH$
z|54&<FH_5hORgssenoCvJo8`UE%tZ!9d<1@-hF5;L&R~tAMBnIqKoPmocnH?qHOqL
zrN)89d7Cp@^w$c+ez>l;b<t%pCELC)tGBB?XxuBuP_#&1G;3psS;v1XL%*N)4m;0>
zFmPn;P%O+@5b*Ue*XELEx!-QgdJ)3F<hx;RxopdJZx#C+8y4Ge1Z!+@YBD&ee<Ws(
z!@Aq)T)(b--Nsbq#qKmkNkBombT<!2h~iqN#;zyMZbDagDzKOdtUNO-YC%x^4Mq2g
z2P`i+_}?pD+jXKX@0{Arz7RI1FRvVO6J(+)B`b7}SRRVr`Zr-q1HadEV@J~lGl2(l
z7oF?x6g_2PlvM7`r4ioA!jLgNlU-LtF*qTH!B5sHb(UK14>f^}ZWm)xdd{`l{`y<I
z=gQAZ+51ur=hVAiQoeSBc`37{OAZImuT@*!?&#>0x=rD{AXCxtGGE~9W7c;j7cO0x
z+G3ax)tDkuD&)u_d4xslhUg~e(s@$)9?x$yc12r#oT|F9`AFpk@p(SXl5e(MJSS>u
zplhM9Z<A4jiG^=>OmA>&&A|hl<~rT)1WmQ7+k@xYbub(iS#{{3;KrA!4?NxVCmr#6
z%H-1Ix_r$X9?@lwwjcTG^oT`K)@7q`42$L0g~7q=drMYdc;!;^c7tjpkF#LGys8Rg
z!wMl=UY#Fjvt~@NdoW8VGbEm&#!a$>xv)g!j#1!o8ymJc4h!#G?a1$aR&i`9YX;+T
z9mdPgqSx^UoYbp7JljjLEq%{K^#=`2ey8qUy`^x=rhZ1@lE;Z3>eMG5QT=*Fn<?cE
z*J8m)f#WJ$PZoGRalFrM!ky>UC@0{u{LciR<_d-x8iMkV%+5)=I<J^@MeBs;F0Yph
z`dp1PPb4oo#GuGLYihH}x0Q?juM$`CUfSff;*6Q9?_EBLj146kGXK(=TAGb6ZQ<S}
z@xp*XDrK!yiQvVD0lE$+D}qJ)rIMT-gdN%9`6gL-xi@J9Ef?@?;bl<S=(MKJGUL(p
zIOBgDM&}t9IVK6NjA3DWy`UytKr`i!;3BC9_I5iC)^71&ToAluy1V!k;Va$yejWI9
zEhb8QhsM=X$+i0%4(<+`xG=gUu>ZQ_U4bbwuCYbaGaYYfoHDwu`K_yH9-BVf1&x41
z_AOEMDQ^px)Spn6Z`WrFT;6ePp@ip|&Rabmx1HmKOOETk+^hUJ#_PXs6aSaCnTe$z
zd88|DIyTO9Gyf~Quuw@)EQ4LIcSb9Ruqls4ev%?f3Rk_!g2Z)yHNw8A6<w83om4o#
zE#~JAALUK+rx!aMTk<eE;;7Ojxd0KN%XtUZ*r_mWJH@FI^7FNUZs0}tBTvI$JXVY_
zOsEaqCXldI(ZZx`-4)v_jE~D+pJmKt$`X0SW2__EzL<e6Hj5|u3{SV&oZqas7S-(E
zGV$=LK7;KYyehGMjlZvnvm5U3cAS_z>A5Qpw~!cjc#2WP_k<~ZoFCSHnYq0A{;Az^
zn(oK_E-vP&zMv`5%sJEC^`t6?YuA+mp(InmRn6BqAI#%-aq+&Oxya?B&YQ-`vc+%T
z*dCTDyHGKWxo~5<<X1i^=hn#&`g5c;%;FnPx2|Zr5r07JqtZ^rwrP!WA$wvr_K3Vv
zS|t^1c2fD+th_zqOgToUOvDxhy#L(2MVx8-BIPwk2hD%IIw!9X^?p_VowrS$ePJ)`
zrzYynU9x`__nY{JRf&ImRVPg1RnqyD_x0qJGpx_n*%>_9f9?Cc{ttmp*722d-<LdR
z`j^pu!g0F!Y{4B<e`-$DxW{xp@2$@Ch4W%RJD4ofY~!4L^Fcv^K-Z+;N7Yjv=H2Ml
zi!0Hd(wOx%Kr8CAf!F)@4>I4WePq6UF?ORtsEox$9xtz1wf4D>*MGd5{dZGG%EOef
zk9=*Wt{+ydGc&Nbzp&|mFcYKb?68Hk1^y4_bllSVVIXw9VS|kDOBP3A(_|CAmy)_S
z-FCQtxOifXb+L51xSAxR+!cP2vy#{PzJ9*au<T|>3}fix)gmRmd?ptJIC~dl&%L9t
z@S(-ZC9JO33;!=pW0qI`S(o2$DxWmNlY!$+Ba?rFh_K{Azqd>I_H=#8bY-<ZI`OXU
z|Efrp<xQK|f{tqoN935F5?VUpw4Z0fAN7-mEmSJnW7HQ^2AHkz4p2@_WAltR&rxV*
zj9z$R$B`ZWY}}Lnq_U~+(V3Vn_U~GH<E1}3U!+BylU50H>+oMt-66~V<cP;t-3Q-S
zTnH6#YQ2}@)L7`tlgS&PdEz>=ruO*<ryV$7_9^ds{QZ+!b2IbXvpcp=Zxm3H^Z2Iz
zC2vV{N75#z4VUj^sI)dd3Rc`3K1I0gZJCF%YR;N1y-||CI6_(4?KMN4f3T;lXnuU4
znRStc(!`b~o)Z%bE_{^<ILcA3mnXB}H4mfsfm*e1;X?9ko0_i}_FZ;(`(W`CRkOky
zY-?xQUGTAI=@d_T^JLmaF?PGqHT^#8*w;0FdTMoVVnAWQ;f|6VspOd+da<9PmWu@_
z`LLWi8L#{}M`nwDK$^{t#~sV<FDYbfu;6xavv*AiIB6KLc6<IUVGaM<lp|Xdq<+?F
zxM*v|J$$NGbohnpuZ1!x>n^fRbm!MMpJ&wK@<?QoXVQT+5z=BlZrAgy-s&~>UizRP
zd83~<?f?4ai~&K_J3>5ctuOZZeO0-%;+Y8dYQ5779$b3yQ(=OYaDhO;j7Y94!be&S
z?zt|WTff()Wn;BmN!;(amJ*>v{v(N9vT|D{K44$6$fDNIapP-4)(b~Fof_7=%RT9_
z;1vu=TWz`Y=(`GyfN&+|GMRaTZ`T+*bXA=Fu)?GISYxG2tP_hX2a~1FBE3(Nu0hJm
z3|v#hv_4553`!ITYu9ISyr%I~kLw-J28SaC8V_e*?D`Nj)!}tjQjXK{&5u01-Wy5^
ze0KP_i-liZymsA!fP{0gGb$%fRmrlJjkQjn?!IS!%FiO%hszwVtUX{c^Tk5`6Z08l
zHqQ0jv--t)Ior?6*7Au<r>foi;u$8V_pdKHx+3DmT*rspa}wt$7R4S9;LtBpsj7MF
z>1wj?npe0$^L^RZ&eNhTOy(wBx8J3#a<Rg{+TOSD!cE2)E2k~|A6h1~UUHb+QLwOo
zc4q9G!&+LKeme=tMoeUWB=ywfP&31y4ykwRD-LZ*E|fPkj7wqo`scAIdx`SG(+|`W
z-49J@y<*<W;pFTS!4&(-n`6?uhkTwTllT4dWxu6!rGQ;HX9uggqw)KD6OMGpD&;wP
z+<D;h!|AE{IT^<D4oCUg&+wi-taV_A_YbEFth)J~=UU`suez_9c_=OEm3Ll?O^(%$
zql?vZrrO21P2*gY=W*4`fyK|9&pkK4yf(pP!oo>gxlg=IKXT~&mn~N|vYyasJXer$
zTV|EdB7->D*RCShSF<Up@-6UN_1I2b<1RbXVxiAQakieUV$*K;cf48qp+`kY+OVsv
zAzV4EagBlG18(8l7Pbq8KOB*mw{GGDGldn3y+5wG$-Mg*`uvfUc({cB;^vwgKd=9I
zx3%u&^Z3UCsk^F$?%rL$u1Dq6qM2S^mo9E>=IHgfySPY|kNHXC?F$(lX*=Q;*j{#6
zVHN#ej=k8C)iXvRbfTv4#0N`eSE@Yj3Mf@%GLIIBIrr!Dkq>RmRh+$x*-eCHOjV{x
zhI4#pbE^y1jyxUt=d|#y|INH-+DrZxtFsC9&E0iP@YnP;&;Bpxo#3}U#;3`-Fl;|V
zyzT4QhRyTlABcIbUcG!X_m9^uVonzSyPn5q227HF&TU!jF2(&^Me%xKAK$&Nn;fLt
z8dmH$D=??x$y%u!hWq^gpX;>@=T2-i4z;>m_190T<g{egp=F&@`L0!0C>T$vH){TW
zl8vpAZFkG;))QZ6>`t1JWy2x$-`KiilGEvl4R=~RymCF2CTE%D9gyT;|9G<Wvt-Ol
z8`+rCWz#+#2x<9s`=-IgrMtGZuuT7#!dfp~lacV##=G(Q(@i>!TxmQ~mu<~vwx8U>
z+5AM|v`(;G*e7?%=$osv|I7}nUbpK;SHMy8FM8@*e}?=1y3V`iNUd$Aj9Ly~x#<LU
z2j&CU%-Q(W+paie>E_-o?D(zza^>e1qq=li!KD8d`@*ijlt`)n-LLcIe+2Vt=d-f^
z*Z3tL5^z4XKY3HV^q<@h+Yjzv#^_vF5cy%n({H|xo87FNf)`p!PpjG2({p39#-0D4
zZrS7)|GYoj{qOua?0egmWwAR-K5D!<tA@>X@jTD-Ytk#{v!46GTJ5q;a^9;AD-JK>
z>$cK)BXGIpv247U?bXb{<)L%KN~3D`{VD7{YjSj%rit&uOfKK-EUCSjUzS~*-Ed?>
zgxaqUFRNhY&Agm)R3~{#Vf`#C)6%S0SFWrq`u1;Y%0v6>3p5@+lP{n2(l>0zUv1;V
z0ve~b9C_fA9`b72W$Slq_B7?#?1>Ore*Qc6ef^sKAO4=&8CDj2(|5b8^k%J|jV2Wv
zW}nK_+P(74vdojce@(tEz7^Gb*QD(76#dL~>e7mKE%7S0?uQqtw;Mg3WN_n9Vst3$
z`UUz|CazWvckjBwX#4*Bg1)Ae%?mVHOGT~(2DmF<HO_Ezlx_Pqk2Rf1d9RzF#{5?X
zs}7&w`yDrXRrcR}<+t@~9l6i%+4A!kTSVd-!((-+%Rl^n+`eOPgOkV!gVVpcYdRTw
z*Z!UNf05X+8JgctN1A-Ob$G4$hAw$&*Z(^&$~(@8_{FZ*+_-VS^eNV+|94vc7xQW@
zo~{+#9>=wi>96M!`_3=w6aQ^k5>YR}q+oxaf$8SNg{vO8rRp&3FR*Q%7kGY2`nAn3
zPF6Jk_j7igrVzCwUL<7C%3W{v7c<x2KhW>#@W+YyU+Lt(UzndPb&M9+eN*##p}O=6
z>y|HlJ0|;1ne}T!3Cs0=uVR$9<*6>xyTkcLmMb9O<P!H)vD3~5T==oz&e@iFUwf9d
z#y+Qx-&w<Na(YTghr*%ut$D^DyX8L0-*G)MrLMSO-`k@ml@X=uYdU$4TsUNR>hx02
zUH_Zr{<$93QI&E|s7G3S-KkBhtmpmlpD!Eo^4I&Dcb4l1N+%yY^`C>U_Pp(X>G03<
zGbfx(nmX6pS2^&q#}$)H8gqh9BymUPC~|5hZEXE3@-6s=Z=!kYT@g@xn;MpW3qIVN
z@F(kE$e-A(UHr@q-tph(S1(zbU}my1Xp78`oSdL_KDTbxINoyPyDqNs*1O+WGvImm
z)phzF`zI_*zqIig_jbW04NtPqM>UsheaiZE&U)MIRYA|EiL>0lqo=>*t?|u+<{#$S
zUVAV7Z*~9ae%=OOj|AI!KM&34XWip8eSe5<L%!FdzoL~#6`Nu{By8EdrCX)ruG{1K
z_N5mm|1UqsytH(~odV`(`=x~czvk7^d8C*l_g}xjDrDmw9ggla!T-TK7A;;n`P-z-
z1?Q9%FM3a!o|1ZF*M}aVMZWG=RVJ=H)aYqsF8z+ZRAK3%B@POg8lo1MbUDxOk3VtN
zq~m0!*53#BH%(7@b3EZiv%mbK)9p9jDwgT25>HunaB~me6yDi47q2`vYvn6d$$t;t
zH|O&2tJ_c|v0tOAFN@*ThE<1G@vXL6<0u>QHNx~Ln7vs_;&a*6MB~y6R~C0YxcI34
z1S;a`{;vo^`&a#ATY1M(N@rf=c_XWU(p$~8mu}Uc0HO6p`zlsB3kyt~Cl)ce+rKp9
z)rG4IS38Hd>nS+Ja-VT{7%aQXO6F^EtSnop1dGe`lh>!pcR1~?Sao;-U$Yg@Vukk%
zNg@3cZR@(Px2o^?P~&}mf5!Juu6v(!Ptx|9r}1;)leS6XUVa)s7sP#XJjtrcKc&XM
z$oAxCmE)8D`04+g`=o5un`N6|`9Ey$i8`s@6WzW3Y3yA5q<50B*S;0gYXXa0PYS0N
zxy+OaRCAwlQqVN(xU$Ba_!C=I{imGdOf7PnIVV6(eacDBuSS&%ed08%Bz^qYCQm+?
z(ra!h?Q@PT*~EPR1(8_UT~;NfFJ4`^I+6c;%7rWE{_~%2w&HrFu|j$2j7v8Ye7K*A
z&x#6Z50rH)mAIN1(jGX^Gi35{@u+N&m|9uI;nbOOsXtdfG5Pv`k&TOlIvZR4@(Dse
z{(p*i5X~$tuFdk4b^ibJRerng{=c%P;#>X7Uz7RnNYC$$+V}U~E%`Mo1sRs;Z<fC_
zSEZ?R;v|oje+McYjlL(y$%d}{C+IRe$6`tA=E-IAWnaE#f1`Tw?{|+TVXJk%9fuxi
zT+i8)e8K+1=9g>&MQ!z)YK-QpESvn~X554mr}p=!O<=TJe5dZqq0g3o!`D9Rm)_xJ
z=y0~ba>fcD(d=nY%ZuAz>6Nm$tTdgzUbU~w!_~kyVU_NxTQ3}|1A;yr6`JNHlNM@n
zZA<tRhv_SKon*97_7AMtpze65*Z4l`E|)5f&5r#a4{fM)Jezclo8|Dgq%=?VdtU`^
zJw0Ag<`B3~$b?CCxw`ZVs{rw`%USkEYRz~oJ3|xp^q1$Qv2VY%@tSL(Xl_bq)GibI
z<8ND46Qx(5QF|8@nG$06fZy?_khza&^S&ezhUpu&_}HzoIAwq2V&JjK2HsOH*4-5j
zO=Mzr`m^EYlw|fQ!AYfp@g1Btleb8UUOcbu**Pbm%tTV-kavCWQtub9dYHpR`d0M$
z`Y^G-{rl?F<;=cgy360@^t|67T>8!<C~;!**PZ@7-N_r4i9RV#E=V~1=uXQ+qZ6lY
zmS66CZL#z5j4*3!am^hv0$0s%`d>U4r0YLBw9@9_l)!>z*ETcv+5Ee<sjx##SX86=
zk&qu#@MJTiuB<6<bNoLQdFWfXWNC3fJalP-NV4eczRWv;-uE2iraVycoPK)2@uxcj
zE>G4w!1<l${9l1>jqRr=-Rjz*c=u$$lb2~Pf1ftl@UKc*)R@Oc=KlQsVy=(*Za?EU
zc9E%B)aAcnfp&hkmBHKi8>-IM;;)MiEnao{?}Go%-_`h>ml|dqeJFg@ugdag&`I@Y
z`xK^iy=_!0X=z;cKe201U<Q+R@%0Ag>#|miRR(>1F4l)9HH6IO(qR|3Q1*$mZ2O$g
zy=u$m`n$~^q*8qn8JT0xFVX2et~*`2@t@c!y`rygiYzP`*VHc-JNw;OO4Ooosl=Ps
zV71oRZyz~ke>pg>*v8W)Zd)st+&+Ki4YDG-XMJ0@zOPwnr$0e;qWj{C$b?x}I=3Be
zJ(MoT8l_u5yNcOvjji9q6zhhVFT4iP?XFihmaM$jFU{Tj#7oJ#wENLqg@m7i7cYwK
za^#uLx7>w!+4b$0^c-3(OCBGPIy=qymDW9m%P+LNGk49{wNF3wQsxO8PqoWOvYFS-
z_{P!iGf}kd{A<frA!R2+0}GuUY=RjRURJ5|DNONyQD^o+v2v=ciQ9!23P+1yCuT}3
zCF;L?v^ZcwpN3CCVZgQ2@T5B*jvsZ5O9?A?=oQL(bN5-;uN_wfd+b(M%zRYge^5Ou
zRxj*P<wBud&wO3}o9}kv%qrZIWx3xv{KksY1__P}EF6s&fBM{b;%Oz<`Zp~PL~b1|
zxnFqTpN;I#V`t^Vc{6qDE^`DP;NRe^yrJRXzM?BD!xZmWEahJE+D4b7?fuol2zwnH
zS^M`2Dx3e7YlYv>Nsv`4XF9Yv?#l9~c6`5bnGdzA{V4Ij{&a6mr}zbhr=d<!$$Rxn
zCZ2L*&u_k7b^CyL#QU0dy=Aehn}3PC;E0?xDXMGkGMm=MobrX->BgN4V@|CP-f{2x
zoZEY2|NDRZ{JUsQeNf&11kt9bqrZ*kRoTp5di%XA_w#=~mv$YTQstv45uduQU+%Q^
zd9BKCzjU)E92PpLB_6o*nbAV#>-Q|!ujXezcy~qEFzd(D7e6Hhze_fqzE{7(TBp#k
z^w3{{iYITcdrb&k$Js3x!hb9E$=WM_6~B8fD)5TmY?07oHM2R`O|8m=sc+J=fSqiY
z`~u%yRWsX~uCZw5$yo;#Z>fFF<-OgPr;}(jap9!7W=ve=smm9fzASOj?%Ldn|97iC
z6xCQHOyyyWvDXr|yRdy_q4?&ooq|nfg5R%xJN*37gEGNG3nr>eIk?%grH}E$o*MUr
z2W~3QQ|pRvev+Q!9=X7x>)uA;@>s14?oU#uI9?6A>7}s4W|EX(&&S!PPo~YW)c@O7
z+JE`jF8wYAo%)Q)$Gm54FU-BN;K?Q*$ENlhxxWII@jPo~zcSl;dT8dA)_ryhi`P9>
ziS#rRm(jJftzUd0YWHQQn?G1*%9Lb&IH#+?lN7at`}@4(Z>wL<_;ZNWY5xJqg7}a3
zI8Oz=Q)@Qn&{^)vTIBgqN9yOIFMCREd|g_2{dOmBuD!vUEgI=>J#SV1idFrwGKP20
z>8G{N=B>TVU%0$Fce;6meYs`X#eD~-Z20`kdRd56EC&ngzxJndRA-4*R{l=ieDU|q
z4RH)YWpaL>!$dkKKeqC^?XxQ1w@Ais(uXhpw+g2&$($Z|>D(jDuc!H#yyX1uSr}-`
zMay$D&S_XtF19T7mszw_<9t5XCvLxX9JBM2&D|Hn+Fj9Zw$Nsa<$N1H{T`oV8Z7<I
zA7rF$=518bwU{l}aPgf?PRtwU>0T2eZ>^KPU#W7$-^cQ3>XqF$w!Z&vxK{D;{BHZm
z*+L@H?*BEW6i7RCRyI{WU3hY*@RK7uXK?O$)2)3trF#B0?)BnHdpH)CGB0@=TC$~R
zk;F}tt!wX`JG!GL$a=!(xjf7wJK77o3+MaXTcTjFZr!Pb`or8e_p1KAd|*w^Rz=nY
z?GjIWIJ>NbRZlkGdB^c(b428pt=$Uw;ur6zlt)YUFp4jW+4A)370w-tKm6>T^x%n#
zKwDev51G<SJ7g`oOL8aeUMnXr<^3dWL&C%xivoo?51oFM+PH6itXR+0GM<CyUd_E1
zv*E$bHF@1%a!Uiv)=WDt=PR`1j)?xTXQr0d^veFNI;>i_cVf)R6$dw{ZLxm(?&*hx
z8~(gfztFn+wN}RK{ZgNAJae@!ne^8}b5@BV7mJs`gvQMmB<8rYX?4^lFX-f6_vXF3
z$hzHgPpS0Ud-w|ZORQuz_B+9qu;y^Unoj9ZaU-RR4T_fGy94$<-qF}{yxKR$x+d~m
z%ro9xyN8#Lv(@>$SsHuGB*iK(i>0vT{%^tg{Jq}CzK6am?+W_*sOfn=)5}0T<4CLY
zhyzw9*oB$q{<@!%rLph8TfVY=`k_LTj!D+s$e0jN^7zA}m!}qMe~pT1u)8|@z{!h3
zRyMZBZ4xYerVGC}D_?$9@%g16Cnu_&pWk=m=?;(1lU?gRB$}MfIbOhiw|HjLn@5iK
z4k$(HPUQIH<}xeea|yd(Qqg%<snrYKA3eM8-^_D6Lu8#EEO>Q#!<Oy`MoqmBer6p0
zc(yvmI#Z9!WrdOJ(clwDws}TN9CfQN4>)DAa;JWnr2nkvQ}pC(EOT%BO|ScGHC=X9
z%yvh%Hwu40wsZ4nUh8RA4ccrq@!*Tn*^<>;zh8-RSD(CS&!=~M2i||QFy&s9H*MvE
zgw)Hc*St@(572XbJKr+o?;^iiKBcIdoGmrS(-*I8Pb~7_KK4}7_tB*nF&U4=uX@G=
zEDBuA+dk(L(>CqUFFOL>`**SJwf>gZt*pg!x+_gY-L&GTYv6CwjO}duGIVAxjF9`@
zS9_7|g;`J5cKO^7e|D6t{qdse4xcgOM}>t?SFL*~bMon%Nrh&SSC*cP)6Ku)`m<28
zVV>CMgg@)2*f%-+acsIFezYMjs!QGAfQ?e`)zei;6O8J%h83%O-Lg9P;_1y(o40HJ
ze(>?y*=-Z7z8V;nFVoR+H0{rB>scQ&V>R!lmehsEeb4=QF8y(1<2&<H-Zp`LlXqVG
z#?gE#SL@GL#~0koHRAiNxh{Ae)Y6m{`T1_uynt`HY)igOFUu7#(z>|d-EGV2C;PTN
z{`@Vh`p;YYJ^a3gbB@cbJ!@%xpV9rWQNBye)g{S`GS$}Co7TBrzpn3fJ^sIGUjE}f
zac8W875EOl(|*hG*NttFgZKFj_cl09GQ3$oh0&%jpmmAI&t*C0FRvWGqZXfX<u*s|
z^mYGoX0Kp!)m-(M;a`&P`MopiUF5x*lR0h|PGI|adcut0_?@+Gr|R1#Y&q4x(q>NT
ztqdjSck4vH-iqGJdozRezp!Q7yjH`xj#3#?>y14;uPy#{-d|(s3I5>4H>(pTY?$}+
z2<xoR$M18q*uH1~cH31yFx-BfBjW^_ClPF0>?H*9m;BHOzO((%`;z)QE)8uakKHTf
z+vFdxKWV9a{YcmC#kGXykeTzIySpE3zk11AVCw5l5%)6g>_4A<#P!~DiBt3EZxZb7
zcMHGlTamW#yXg^Y=ct^|la~b8|Gx1-Z|j_Q^JcgdEIuIm{`oBZEBOhs-&rrV*oCYp
znJ<0z(XBbbJo1jk9R9Obu?vTaO0+)NZ?pX3t@$5~vo<!*@jrjGn^AVA*%bc33r`kI
z{Qf6*+qxO|jCZ-%*esPz;yIN2<e+&&$L%G%9Q5?nt@WAmZ?(=A-)z#&^Y*melUA{(
z1@|3J|IWMYYn6FHw&s%ZjD!g*`S}Ib@!ppE;-T+T_il=8cUH_-X6^GSM@}j6TvV?5
zUT*NQns@pgSJ$#7y;GO)cITwXeXjSsocKKT`NEJei{J^{T5Y{{CrtWj?(z5Wiuns`
z0+-D=_nY(fQtO*qm-PjE<=wZs9{9g_=WV5w!;^%jHpEWd_`$Pb)%hyP-+N@7SI>9d
z_&z;6&o<W2f3N(TmrAj|(u#MsN>5Hoz2vm!+}=%P^FL2vE&I;4S7_#>s2F4STj>II
zyFWxT-}1hmv-YUKYoFRhp-)2u-^t}h%LM88ZEQ0+v4dk(&BuHDe4;wGp1)leEx^p6
z&$;AMpdRmgnF~>Or7tOcR+N~s)Iv_i@9#eu(MyY;Hy=B?G(2l>&)j+rhlPrjhc5P4
z9CtVpeIfOi^y;anQ$PGX6ZR`U;%-qw$_-1_Wbv*=j9meXUM<ZQ^2s(Ysfete)}{7E
zOmOcqcZQ#f%qM%UKVZ1~qLsazLBs3J@5){BUAm3N=MP<a?<bUdaJtKpkbC`%v$jmq
zleG1(dZ>IO$1<&^xP9~Ozap7SADY?kb1*5$Jf=VW+^cU2b!#RnuHBn|u6ny`zg1uE
zME(BD3!a^txz9g6apS%fOU0H2JP6{udptycmu~NmOCOhn?LM9^vHOed_w<`jObt)0
z{KcQYe&U>+Q{3GwU4Ghap0a|ywB7E_nv3jrmd5SsKUx?lH}AaB+Y|R?4AY!UMQ2tD
z?UGWKwzc7y{5L+C$7kk6zGVuYm$oR_vESd)b6@dJ%BJFDA`#Oh^|C~p4O4<tekc63
z`_U`O`Q*~w8kN0Ew^ZNJx+`-gyE?+K*>ck!LHRH8|HJ-r9XC^9_FH|WanYXz%jaag
zOgoe3^Qrv8hHb6acPObn_L|phHQyvauIu^R@2;)MU0;@+Kid#CJuhAPuiGk_M+?~7
z^t9wBC(CQCnbE@W+4j-)|ItDd%{M%m`}g7MmSyXGHZ~tj4L$ObIqYeyyngYP+d8k;
zIRqredPD`<>D$Ysy9cbX`@$*eer)!()8?h;sy4dV8C#n~wblGePn;=V@yDzGGf(+_
zwH^POe{Bt`UobgNZTsWDPgGJ)ZqUiU*x<KJy~XNgY>C#b6Gn>RhmIf3{VZeXsyZiy
z{no=BJGNik8T0<nx7$mNvz{Hi-xOTR`(vZ2^Tf!<mkobyi1V3q;BQ!*#@pX!+g@z(
z$%)vbBV5rc*;V?gdiVS{^CDlJT2=Q=>-W4TTi&y8-%vMstyj*C!u3VFPfv>4ulwbO
z=-gR#Z1*1TyC)UlXTGZ_JLGk)>9!^2=gK~pTu(prX}70sZRT`^JO7W|vaL1QFt1mt
zaOJJ}vJ3tnW9wC4F33LXz00bJ3Kw2~<t<&4DYV(FxpTc3->(Kck$5F<k1v=1ZhF9X
zbkCk2f4(`nzdkPC_y4`4`|_ko*FVTlOHcZGKyYcf$0Dmb&lNv|e))*mojW;I;NxK~
z^*0jSl?yK(7diT*ba&X4Ec^4lQd56yZ<RHEF-h9qj`^2#tgc6D$M0q7uTGaV_L>x{
za);Qfa;RQcseC!<&Cb3BoeMc-zxp{Xxg6wZ{lv?A)uW9TdO416s~`RqiSXa;H|c1V
z$v0M=I?J{D4ot|~njW&qI+t~u{%_f3-fLto1-X4~Hw`@}(j6|6y<EtHndf7&(#am#
z@B$|9)z`kSxaqW&v(rUp!zvf=Ib9~N#n*~9_XJj$Oyr$9!8}P({H3|X%fyYIF*={F
z-rM{vDbTA=snTtY*M(`a{F*$UbKWRQ8tvLGQ|nnb%Zul=P*50O<c#0Gl_dvr+A89E
z6clwiCQ8O0s92!3{J!~?w$i+*zx=!TW%D|Js~InCVvgwlG-qy~zm1=-pipNC_k^8Z
zyZ=V+eV`=2eWk($=k1TO=M<;!66n13Gj^WXDslB9PF0&WtsN?z$IYgEtJwE;)jr4n
zaX~W5x`l6MXeTS)(Gj*1d2X;)G%$rhD`w*6vXibBk8>aQEit(o^CeL&-X^%fisyLW
za?YUdcZ<#)6W`!><9*;ei&frFE`2`H{I>haq<o7d0Z&eynwR<5So=bXm#@p>WqNu!
zvm7&fUww`Uh++zhnI)Tfam6mRUGrQHZ7wh@6nSwi{hjly6Vay^PpeQDOO-k#*&efG
zg|3mg>TSkp?IyoBC&uxWtLY@NUkPzL@;O(ma`|Vq)}QtoO1GT1%Gh&Xy~-_ns@&@L
zYDM2O&ytlRmqg4=<?y?-YUiup)+!6t-`kYw>1@5R#BlSqlDWaAn)|m&n7wg4cza^$
z#ocS)7`>>!%KmFH+wKd~->(WVytwSd!H(?X`Vk@OhN`_etLA@Fbm`gnVY-CjWU;fx
z`_(M|22E3r-OrYLHTK?@?0ZY5%sBdFeeE>4*w$S^F1}ptp7j%EDr+u%@#^KRwT$||
z9Zi+y{_el?=i~g$2hP=zUzF{|JZ;WZg#MgV7kTo+D|_2t8+{xbKJWC{_SJF0m+nnZ
zj@PQ!E#%kLS{ZL5xj!S~lVkT4dkNJ)tB(Gu+Wp5d@zS4n&nETT{#+Ee^X|s)H<?m@
zJH3#J{p`Z7{YzN1M?bzoaP^PyowMeJX&*h28^87UT^nzeh%bt(w(;zUoBJx!Q&L9s
z!teX8FCR}mzv#`~rDhji&-pUdEN|cA$9u0#^y(5@#K<)53+H{I_Z2?J)U&nE%fFp>
z{m~k?$g4H{Uz6vY&8xU{WNW$M9{J0%e-7<FFLzmv$IU%bs&A7D`^|=5clB!z^F$k&
z=J)2T-CXLkd)4X_=YkR)+a4s(pSML%{@3N8RSD+H<7yMc-*<SgEYgfBQR+7e{9PC&
zvfecQx%A{7Z_Z7sk(Zq7-j&{79=iFI$n2---dYcDJ_-N5+$<=#llzy5^yV|GHnp@k
zGPh3svW%r<>NhFN%1xU;`gO%U^_f{AC%#hSw#hBg&^V3V!LvRVWN*o=+!k$k;Hh)i
zwAFQnpMOg)uARQZ=IZRHX7}cPk=t(^zDpwf^Y59<J|;MJ$@6<mUhbZ%9rAyY+OhTx
z2c-Kvj)qL!pkjXN@WSbBFT7alV!M8vE3>=0#P_@1w*@i+ySww*xtKpId=-9D{O_iQ
zUyvl<8RuV4Z9aebUY%(UKK;RTL)OXwcFwx|Sc{;}&8dm=m(J-uzV)tNckoQ(RMXBC
zTXw8BE4}rV`}v6=hN8z>Ry}HqlKD#5GoMO5iB4atlCh;XLZ_bhS@?=QmmVFp?78}(
zYvM$m**m}9^*GjQ9I<5YX&&BLY*)FZCI#sgDs7j){GeRuRLkY9Z+<ttSL3;Tgmc=R
z-5Tq7?#ZyuTY2ZY+4cu_POmK7(06T4mi0U1i2tV1Wm|lv-K#lQ<UViPAq!dQr5P=`
zvn8&)va3r?znaf+bH|48+p@J0_wzO@xAb1zDtmm2dhV*d9{eYzUU=;{o;>-q#r(3_
zOeZ_cREjEh2W)e@D(_igx#wWYNuAY~>n#p(_G&EryrubsSMe0qKgSRFOp;l2gs*e+
zf^&*HE_p_7@D$&4{&VmXu@gn2HV>F&d7O+NFS)Dl8v0JZH(B#&@26`@{!v1rJEk9g
zz-Q9-C9+QK&(le*KIzP!*N*@Cc6nCBrQX1~XLlUjvg~=;?0eVp_~KdI=7lWf-_AX2
zgJ0Fe>xqtsTHD^Ft!Fo?Gkc`+KFmyb;qs8uo0m7O@8Fx1nzAc;%l*u*TRy749G=<v
z70wR3u%#*CDfiYzFE$);VojWIJa_qvR7nvvo|+|lQp!B9Nw~_|++|EuzHQIpfBHi7
zl)cg}vNN9rKH=OU&}#HFYnRwQ#pUWL8PER91lwy_?#<|0wq@5t?>+vlH-B3MdvD4)
z^Se7WYH7R|*Sr&z+dD5S%`L6Ge0a)B!xKgAnZ_r5Q{9C*AH3Y5BB`<V;1s*>i<p}a
ziOl~MllDY7d9RRR%2t!U`rNSc1#f>%_wfGw>ExbMN)a||cI)ZfR4KUlR(<)g!`^$!
zzdcb*y>)Z*wVC>#m-^bC{!k}qGl%bz(7o9e+1>|~1!Gy#CK`o4by>ODKBaueD%Z>A
zvqYn&rCyR<I{WxEJ)hTe4jyPNdj4j6uA)G*&gq%o7QDaIuq`+6ZdvZ>_PyH0)lY6l
zb^SbIDiyhGl}(xAdZi~@ntVi(npIgJ8u#av%)hC{KFQxpP$ncoX!F%h<|ha2R)wTW
zmdyV!WyaDMR+9DQL3!Oz{H~_8B*cm8bS_u3oNOLiXA!q;1(&r|uTit*J?87vRtbg4
zulUw7@7mrIi@G=ds(j(S_`FL+rphw~#w#j9(YO6A@|k4~_IUJPQ%jqZkvQpnY-)YV
zT+@|tru*{!yca!LB-Ucfa!NINSIhb7^438+be|m0Tavay%iHX))}NCu;fMXo{@!Tx
zy<!@fy!C)WisbE=Ld#+ot@4%&Inyq`JL+SC#g5%f@lS8sz6jH=ep+ti!vCY-n#ekb
zFL@I`P50ea)VMaI`FCvjkDmCaO80b19elp=-&ADZ<umKqKNVJul!WM*o!nje{N;P^
zzp{;dQM%~XuBek%%Z^w~-|)oQM%n+wCjPXmM^6}C)0I6MDgMy@>71P&;d1Y;JP<dE
zI=6hyqr!FfR5^a`Sa0h)b;^>rPu@Gr2+ManKmF+H^IMI3>RgVV!ULv0hqroLm2`&Z
zp33j(3ov7QH#z8#(EZt`AF3LNKL}_KE4*=W1M8cju)BPm@_*x(P5r)&Bg#$BMJHG@
zZh!NM0ENjljN*I3I3~_q5Tnt*^KSDViRex5_T4wOkXn0M?IhEyi|p!_OZz$2-I;jL
zolUVm=J2YQ2Q})V`$cZdv~g7}TU%i<*>zcAZW+_bcKOpR;?c)$7yQ`u=IpilvcNLW
z2lGQ}_xl82+;Mq|$@XPJZ9Ea&d-g<RxjtML8fpKk@Uqqg#_uAjr}uaMFul9`aKa}i
z;lPD&eV%>GyQLiRq$TOr(Xd%*XUd*MZ4kIMzsibJOf%)sVrj0UzxGwG-ud+U>+NEa
zt9j=x_Pn{ry(funZEkaz6r=P(r^jInCD(aKy>pl+eVt{egkONky5@s6Htn3JB7%-6
zu-@IJoVoUSTgi;ukLs4Kw!H8tT;$kR4V8&r8rlE;7WCfOroU{{oSByddox|dHXk$<
z&pI;oRqW&EI{XPU>bj5b4fk`H{e1S`oo}-Et<O8CiuS(fjlSbAwYH95KkZh1-M0_*
zT^<XyUq0NmYwg8_yp84sroXmSdzIWh{H#upRiK|YHS_WsBSW!?)9<q_eXjaq#sA5M
z^VLo~^$U1#-KOLB!stJNAvfe*yrSm)JMi*(SK*gH#k;SY3io|B-*s)>#rDiu8qZ8x
z5=zalZQAf&qyP7W<GH2rHo9rwzZ!HLTY7Z;vdg<%f@Ym)exKuOx<PNNb?!RT^Fch_
z%}=A2#2gPb-NNbc*jKsmhIYI6ig^XAbJA9Q_de{uJZsat$k|8UJ9kFJ9#Qv@@qE1W
z_+<<8qP5A-xO+LpTa*t<8}3?sk<Tg0#J)qe*Uf0st;Cre-(OBG39t4D@#)NyUnM-%
z^p%ZGu;Rb68i8j?`@YsKJe_>EoYOzAUw+c8yyDK|n{!IdR!m-W;aY{E@WpvM-ZgR*
z`+Pr^k#PRg)%KsddlchXOlxkw5KX%HBL7fgW~{93-SYP8nuQV~ntCS=d|ESq^Q!sc
zlip1_<Nj*S^QeC_54k086MOx&*4X%L+uO?N&-SP{UHD;mU~2z;$41q3xi-K199Oo_
zd(|;%-U;W!KMzm4UcF%ARO>3A`7a#aS{p9-Whrrf*7ZVxQ<a^ZT*CSlOHZ}er0kQQ
z{^5bnO{1&FMQ6TiN_h0B>c*!dT2kBWbCy=t+865NJ)RK#p8fKtKkr<f|FSPqv%R#$
zLn-j*7WebzMo+Cz^Dkj3-q3sS#-imjzeKq)_UenjJ$tH_IW#a~ErWuu`I(jw?YEMm
zYwH{~o|pdZ%)FYvX^Q9PMb$5bw@)cb_$vL&bG_fhwNW$KUcdTVvHsoeRIW*0uO_T+
zU$dmv)>LA?-Je-zuTC`O`R-Jbe-h=T!tD_CgW>nKH*0j&&wq|}+rF@mvt6<!B6Zm`
zq1yV)HE-+Q%ux7T>hOBr)Wh77_5wn?*YMsys+oGS#dgC~sj!)?r5Xagr_1ZTrDC;K
z*k{;@*vx+7#M-}J@l2V{#%8nHi#&BJf}-s-CY0$cyRz-cq@5-%_MiCnCf~eV^=6iI
ztfOz*{J;mB^1e+vDibGh<Dpi7zVNKeH6iH^|28)jp3zM&UCnCl6J>REnz!3D-D$eh
zgWfp*T_qxX*UlsKO7@~XET=c>>{j{niQh@+?!Lpt8$PIQPj|e%G<Qdd1lzr};YQjm
zE4&iq=PAq23co$og-KPT{pBOQsk1&vUw(h^fZ2Z?^~H%xf8WwSHK~{9W|enPj73uH
zCrc6ar0J8CnB*2qdCsm|@#ke-qQ)ti<O2_d6*o%?dGV&sI3~xWHj{7Fv7gPg4b%QB
zy?2@T^uKW6t{F!UtZ<)cCb@ovu66OLw$%?AXC7+$^EI!sY`@3krT6$QxbG-x(0C^E
z<xb+-f=5dou3YHN+)||JCdYQ|w(-|Zg&QwBhWF`LU3g?UE2L#xZ#JiWb)Tlabw;=4
z$)y@UT@8y|kM0bS?U$btu|Gpsjq}+)i>V38f1f^Nt2DY7BeHAfhNU~s-<e%@SwAyu
zuM10Sb;~D__wQcL706s|qjP$b?OEmb;%`qT-<5XuiOuJ+^gH(CW%I7u=&xV7Pt0EJ
zJ5`I}aNF^tp2|&W{C+L%PZbN6{cmpB6QXQX^T)!L<HD;`sr3sZw;Ke^(NxR|;@}Wl
z!5g&i+7+?b!z(}i>77?7|MYtF&67gsk2)Wfk1dM-bm+_FFS85z{_I|oySG^DvEKnN
zrcjQ&6R*WNxB9jVYwmBFu3i3pclR}}@<kI~O-x^<X}T==UwqwKoumwFKUOd1ZpCD!
z;?QLl(*rJVO5GUa)VfS`=ef0OKT2|E-BdgH<MQ%ft({s*!c#UcO7;Dt!fSW^+hVC)
z<*c_i)DKQ7`{X9r$5OwDeQ{TR>qJ3SclW23;Y(YVmpt00zuM23duQrWqo~psyY47G
zUi&+Val(-kJKw1nOqx^KHM2!1MEVQ!r3;IVZ6YquzIuqo@A0eXyJ3yyeJ3}6&wD(n
zzWB|Okg^jW7u%@s%a+^XxXUKjc7x8nw*g=L&T4M`GIPg|jqQdf`qhn(7#7}e&Gq>z
z@Pd0)5#OH&F-fa_PJ3qMoRz`7Kf*?T-W_4hNq3GhTdF+ZRp$C!AFB7h$p4<9t3uqz
zO)V+20<ONX6W!(X%4OSEjazqWs}^sYyg6Ym=hXg_OMLod&dzb^JT-GQ+rf5~n$^{L
za=9~IK2PNfns)Y7h_;c%-%D?P-DpU!T`3f?Ls9k4#f;lF`yZ(AALib+HGiGbf@YQu
zzoqT}8uQ=jE!dtYbGKSczKqp0!KC$P^ONJr+cPF8hs^q1SbJSjRMT(mTwe8W8+493
zU*VkkL5uxb)i$ZcPtWUKGnwA)y1njDtDo_a_<s`*PAoecz3hf{-}JKZ4fQUpC1TSz
z2To}gtdV|jaNd122mgwnB1$hNT;T6w`eDJkZ-dO?4+>cqa;=jdiJR1%xVmg|LdmwW
zJsqLVn}hRq21v!4JxtU|H9AmxEAe2X2eX^pjZ9(d%)Rn1$q&w}{<4_Ip1RQL;(dR<
ziMeW9p7q-9>DcYNP<TpyY5e1iIq%l++ive{G!YM%T={k0lG<DESWkRf<2rNFDIY2A
z#GP>xRWliKZ{AOc7iHh+@xVl*{_$yp&d>Vy(taQOpB}_~DYR=+5tCx|#u*zvMjc<=
zwf3Z_!q@pum1;WMGM|6g!CA~O%Q#Mr_287E!$)4Wi>s{Ke{QwrsY9DS6$G3yPDstu
zJ-wiQM&H$K>qT#r`$TJ{eq{aqsyQdI+r($;Q~yb+8n^x&a?Cp1r~7k`&KIFJy~-b3
zr_40hlr&p%-)!MyWxM(lH{|T*pSOIf$a;P1-V?VY*-dxMIB?;F=8jn=LQ;|{C7!B_
z)R$;=)$}?BFIxWUOvK%bXWCR^-hX7@E^5&Hs7m;NUe}ITKcA-$3#yOh&zj$+`n9iQ
zUq{jXi2D7>!MV%z-aSs(vL*Qed(HI9AD6mS%A>!qP1f2dvF*Lf+Sy0dTvXk(=Fk3A
z8ur31=*IUKd)_pB=&mZ@TlBiyQzlPNIx<7&Zd@JP`Xy@HYb5IgA7mV^xe@dId`gS{
zEv}7{PCkmcA)+^3UE{d+y?ycdnW5MU|G10Qa_@T5GIg$L{5(+Wu47d{h2O=!TjXQa
z%-%1v%jTrtEOlLXsv=y{)&9cu8V`rFhs!r|U$489`n=n?XB$KO5xM@uX6B1N=?QA3
z^8eUX`bNC*%wvYUJBH2Y7Sso1T`PIq&oB96W#Bx?)o1LR<e%tqSMPEDA^vashn|xx
z2bk~Qe)j%SZr{msVjti9+;Oj}nDd}dxa9q*@7voVJ}La1eK+vv<4t__tcTwFzv6z>
z($i4(FOz@5@dr<DsFj@CJU4!E<hlN{9C?Y3%O*@deSDeyJKYquxz~OAepD`;BUGDR
zY{TKwYj!h!MZ~mpPWD-vcK&~lzg+C7dd2&%tb|Xb<x<;i(|O;q_r?GHR>Zq~{kEOg
zHTgpp@x0#R*)HK0{_Ie!j`L0P$VcKy3+!*JXfwv`oH|wEfokvTvmf*xuD+KdAI0{5
zb-wV|+fBhQ@(fR1Ie9Ybj^R8>m8r|l{NA_4q&l~JOJ?!+*<q=Rr>xG?R-55&_j#WF
zQ!oDXh(D9H6U?>rYi_!H(0_D)VXbY>x!Zr<X)RF=yVu7n*S_J&j0Pjg1?Q*Ro-~qu
zaWb!X)|PX%!Lu8BU7W?#9|V5ueDn7G^0b93&u{6K$Yf#hofv*PJ!0C5-es;EPwLz%
zIb1%`yjWz?4MWCGyN+oA&zj%5EDW?Sj5Vx@P6~5stc_4zTl4r@^&*~Lnd?`xRA!x-
z^(jaGhHs4O@x^zm7S(?6_YF5WWIFTPwN>mt?@9A^{g)N}ka_w~gI9WU<9b!!jdQN;
zS4+M(CtC0O8<ty->t@*ZM*EuW<hS_sx;5%X^Zt<U-D21LUk2wI*@a(ybj+edyf?P~
z1V^T)yjj=Gz;6jn(y6=p@7CmHKG)W>7xC?RyEca}dX0d7l*IhyUZ)eKdF_o;vW=h5
zy>6;i_xN^gitoCgu5yAGcfWA-PL$rNHtS4xd76jo6Rkd_73=2oRDO$^zV)cl$!C`m
zH}cL~cEj=N;tV78`i9P{3%P!t75d!4vn5i`c8P3*$wuDXoP%d`Z>%Y>DP6{JW7(;K
zbq|VnK5g4-y!G;(Gp)DeDm{cNsxKcaTl?hMrW;3}y8KXSyRS3N_Nn9|U+LE?UOePp
zf42C)+sc<)FY9C-wwUDQwODyZ`Vo2Y&1u0;p6f1Q+`(CY<L8_+6XhHCul;cR?>|Y!
zCq-L}+JD`=aOyKtL{P<@{H=cfl<wW&Q2oCodST$~m|uV1Ensea<}!KX*V)YWtA0)Q
zeBf&}O_ys@>{Rtd&o;ccrs=l!WOHiaiGsgJiZ4B@uloORdHojWZGx+8@BFNO@VwS}
zY0Rxf8je2mR$OsOS~a&RJNl0-|8>K2&aZa2+XZK23H2{k_<M6#=7w1AMdwVm-2JBI
zxaSrx`<5@M&GJ`tS7k1D&Xt%~|B(OJ;=hwW|IJ~}EK&aWbyM8!!jCM6LLE8&)s@Zi
zoNAaE8Fu^cjEl2!xH#(<xV4Kg?|$J`_hx|#uUz<2yEogm);hmdJ$$AA=!=VDvv$8I
z+?BFbbAl$f!r_Rm{1;wanwGJPtxNCW$5b}{2wARXk^NQYn||&%yDdi6?9$T8#3h;y
zY9|!09bH&m@?>pt+pPX`@0ohNA1?B^;ORT};g@|Kul24zJM-IU(avr6mxgsni5>fP
zZS^0=sh;mWnDp9nUbM{izMuWGwpi5uZ*jaxdHsbcckE;`x6AvOs65H`H?q{tx_<oU
zPs3Kj=ts#fJSsW9?#Wo)&#kxn$%@pL(-S8(X?=fs^M?GC)a}0;_s+M>S+70CyKbA{
zh3EDwvZMQLG_q^vZ&-D1!;0S@;+8kwlV8KQEGW@iVQKu4)O)#GEt>u|TP?7<#{cPr
z>)OS=kJBs_WJ~r7c;0<yWWD?G605oTyGoodojIMeCobi*+~FznO#LtGK6DXZxk+#7
zsl)njj7;=YV#9y<uktPkF6*e=m3L{;l__&epY=?Bx7w@x_9BaO&y^Oa9{aWDm0WT2
z{a1Oek8@YbuS;81$7LcXe$Y;R(gK4;h4U_7v6oi8v_|P*^z}fY-;u6&sunvbOZ|Hw
zSfpwDN!CXG;;xK+*Y6jJoOwR;^Nt(*=T)~n=Xq;ynBZe(`cHJ)ar@h~vPI|8Zn6Db
z^;YTP&*t+13)i2OuC?`BmmwUQynlmb>GEvBZHw>xI+g2ZdOmY{_31;GtXpeV|2A;r
zo|N-%?tA@v36Zz1En9V5-Qc!T<NqD{rf2)K1Q$HJU2<++*6$zD|F6HAB9!&x#+GU4
zH`|+A_AS`W@^#Vv3w^zFulzE;lc#O5{M1X++rKgjj~^<#KTAzfMZ@sHq<=~L^Je50
zTu#gv_Po8I?DFn=(`q#fXS1xH`o8Q>`1T+1>9?kydwcZo{`TG5KIB|}xBD)ym^z<)
z+53LRx4FM-nzlqMG3-&+-5VlnpBc1m{S!R}7cc8I%l?<}JMVja;Vz@W^2GUtpFeN-
z#vJ-{oB02P>etuK=6`Qlw$grO$c0*o`)l*6`t{5d+k!-*%=UlGTAVv0Zud@)rFN5D
zYm1NGylgK~Tg7ms==nXFwfE9~zj`<Q{k5A@pUPkS`#r&Ry5F4;E8p@0hPi9+1#bK8
zH*eYXxkd|YLTcj^tnEeD|NX*zF-v#$>+fgx-)P;xU8J(s;_^NNz5e_1=k1&FzL);p
z;lAtQ+eeW*F7J-lc_+R&ckhNR-<t1Qbxbj83_O&+;km5-+kFgwJ}kKW-_qvw-Gu!&
z>OSSR#P_;n7WLle7d~!#uenlxxA0Mwa=t@Xp2w%^w{Kc-$H3*q4%Whn+cLU#o&2+v
zuk_q^jrXgc|33fVjdy2by2Ej6@0WdRCO@C@Snbm%+iE+%FA<leN?-5je)VwjkM!fe
z#n$|~VW4+;cD$W;{ZpoU&-vst*z0$23r!I$|9#+A9m^@3@)>GdVjG;62Tbr2ye%4O
z_)@;7*FI3cYOnIYdgco^B2r$gv~oHn|II+~(9Gkl*QV$9{r1>*dw+C>*o@Wv4kw<*
zJe{AitVF+N8~@z5mloTv|NC<O8KL{!q4Pf)=qD`WSoni0l2K>*={ssOCb;kY_hQ%l
z+j^->Z|=@wo)Ka(Iqev?{v^Fig+D5P{5B8RuyE0-ey6N`ZZTd3&s+KA_jLQXR;K+|
za}_muP`TPW=FyqkJ^vQ;r|<Eod>MN9x8Z^l*K*?h{)?OJJXfQyeZHhgZhDcs-h_Xy
zuFH1hC-gm9QeK+3=Jk0NzSZS+IrcYMuGQ=M+`MM@Vq=*73CAM~MU{TP+yBP#;IdcS
z@91oP_0(RctmU+3VR_)ix8_otwrVeyvE80tcuYLLR9SIKW0q9PdJV@i-O%H=HmBII
zCq%tdIvQZO(ETC%<EJ&933{t`UD6Y1U47x{;y>!wbgBdT>)#w*Y8+i1Ao)Thfsax1
zG>eFBqP&etoy~$NB{^~X7d`oZm38jSf|tkHUB0<qU2ify?6(d3J|&eNc_%kZap5cP
zTV5`7k7V!rW-MCpTfwJxw{5q`&T_Nq|39%mtv8GLAwK6-c2L7x>&M+ieccx`fBlG?
z9NXhPeZqVj=G>WA>V9$9EmyKXTzT>Nf1aB8{|?Umv3l2uFM%5#eLdgS&7ZaP_paEk
zuNC*!9aRYZJ|V)bEtNy?|D9{b%T2!DJM1HNuYcYBr`0A|4-dJ`VtErQ@##x*;Cb=2
zU;O^OzANO<!ZrEIuKv~8`jvlot$Jjik?`|I!$-&0q4)ng{7?5!obg+d>ENl5F1`+}
z(*>eaZEd_6_pPXqvs+>R-?X4&hjNFvby$~j?}{zl`gh_LL-k5$@_ktQsb@>4-s|@E
zqn@pGKbwoX*)}b&*t1~9)w^@w^LxGdyW`02mtIXr_3tGAb1eLu?vUQkUpMpJ#5xw&
zng`GQ|3~=#Wq<uDSwH2*{s%L+v;A98pM2?M#k?)c(=X;{U+I(XZB<kMzG$6U>$`0t
z3(l4JIc|M?eP-yjywfMX<}jJ+UYt8kAbtP0&br=@A1<5}imj?I`d#(Vgj+n|Q*OiB
z)rU02&n>=E+o|cZzi8{{?Nx8i-r{)|lvq^RoOZYSRO*H!Pg3WL<(uuj*Q#f(>UCT7
z!0A$5x8ChbO{1bq9t*CE>^S!1`s&Z^oF+X=Ar0KkHb%<46`t>e-<bWl%6UI6`m@)B
zeOifGyLKoTTQ8rs*)U1?kb%gRyH-z+Zw|;`W%DnKB}X_W=IWhK@2WODeRSq}(u>>{
z#m6DOjZ;3|%nZHLIsNs!r`vRP<Z!z!xf*q<Wscs7=;_D!8sD@3;)(hzxBIi)`TvV{
zzb#7OTl;s0v_pGhQbf^z<@bV%cJVAbyJCWv{>1CYKcBp4@@?tm&)*-;xX5eqe$wA%
z57Q(2`(~P~lV4Hvs8@NDJnOG+q0fBReyv?*G+|oH<np^uUH>OlUEW&p?#oN5jB3Tn
z;<lczAIMMhD8IO)(S23uP5oU9=I%K0)I@7y>^je7kDonKPYCEdIr;0J{GMwS3KupX
zj5*ysZ=y<aZP3gI{Nh(GCNDZ#8T(Icg{}Vmt?70f{Z-GqeQ|p6_Jhx~e=|3@y?2>Z
z@BCG_u;9_A1uuS!waeb!Z+J#Or=;#@WqXSFAD;@&H`DTxu3XSq@kK-8yOl|;-uzFW
zFYmpla`Ju039ETxpL^b3VG=HGo+q6#&-2DqwxeFfUl-@S{`bhqI5T*^$qDYgC%2jC
zd0jM+-SkHN#INtAoIK|e<5znsXgy4M+1hbo25(k(#S-%}rmV_!PiLj3id&xe<SL-a
ztleGPEp<QX@o}~@E%PN4zQ^iWp9*s9+|;c;bGz`|75<;Kci(=Oni}vZGt>R}z8MGm
zUv8{D_-S98%5;92<9dfArTY)tX20;c@mlJgD#Lx1*O9Y31H;6-a<mkl@A5nqR{2(G
z`rbE=eMPIcZ~j&k-M;%cn^8cf>&B|Y!w+v%2>ZVe`f}}7Yq+O}F=ytQcebnCOjY+k
z+nxIGe8JgUPc=MNWQkmwv8eFn1(h2&1uoul{X1dDPyUnpQp}@S4WhHMC#uJ+I^FW~
zcYECT<UjK_O!L_B`LXr3dCTS;&S|b>=k&knQ}Wr>Yf8ySp9S@%?)G*oYV<x{)MtAA
z-r<qB;YIyVOTCx;VPBFbD1BFN!HmOqR~zN`dTeXjQ`TL2WL?hUcT4&IDy`h55@^Z0
zM8?CXboJ?&&*uxibZb-@`rB>p<F|e5@v*3L>Y2?_$IkIP^{h6U^!w1M>6;d*q%6J`
zenYhXpFihTy@eC6d&dM%@3Wov<MFPH7eAMrypkS%y?x%inSBZ-h1W{{N_emO@AzX*
zJHLi!&*itAv3D$P$*SDfQ<~no+3tFmic8M<DBJW0yH^z%=GskO{(lSW*BjXe|0DI6
zG2PkIml3$@)6#AFd+#vY*C(=m{LqtR=e6)&y+CD;JlE%)Uj#$VzZNW%KB&G}FW4_>
z-*FATz3n`EY--%@=Q7E?HxhBQFFJLkz{c(~-~JUAztxWMw{%$8YDu5D{N$q&@AK|(
z$zHa-l>zhO^m30p-dyo&m+yt8>(AzP+`qu*@#nw79pP0jVYl9t-q(L6bn-EC)00ny
zA963u$$wih_tW86j*R&2-?p3Oo?7rXeqnJ>+nw_p8g6|$T)U|MaOIB0i)8FO)sj+c
z|1gR4^XBZHl2dh5lKaSeZgU^K+sEyseK{_0hUPu#TKIZ@73a6L$9J|kc{r_|^0(?{
z-==(i&B}x?Zc9w9FSFQbRkZhB3EmR9>c;utz4GeZ`=7_`xsZ5q^(xzU9#7paTFeZ6
zT$=c|cV*O9XSFci==F9ssexKGw?2#h<GGntGw0LY^Rp~o?vno(zIXm{tM6H!^JUcj
z`)k}TS#*ofnQgkc*!P!ZHm-BUm;QgH^&wyUZq2+ykNW3KX|N56f0$&l?|^+vZJ>F8
z?$U(ldCumshlH2CTR(f-tvf7jT>2`D@2=F%e$;l}c*k3nFYnp~{(hRkT-bc?&biC}
zm12sI>x>FMG?!0NRe#2>HN)w1?X;hEt4$6}m~#7QU_FcJw^}3nbv4b=H_jS<(MsWb
z$Nykq?f*lfFBDJg>RRJ*$bL%3an{~R>d$R8-z`3Jv5qNXUAE*!ISaPtTZ`6h(zwLU
zb=TbA^+|gE@!Eg$|6SZX!M!L@tg+HgyTrDqJLheM$@)w3g=gJ)zD}{KZ2d4PE7et6
z=<w6_oq{~Y5)bRVH>j;QmW<30H{Q6_-a$rcO30ZUk-W0WKa@gB9<Kl2FezN~_ift`
z6{keEdowX>Dn5;#oc-|dfiJFaB>rlhuT%SFur6Yqb@2<SlSOf7ly?2eOWg4#G$nND
z#^O_-mvVc5FAU9I;l0?}dSUmX$EkB}<g99RDsL5V&TL)xNVX*Yf$Q1pX$xd*FL7n>
zXMFXjqN`XkSLV(8hWYB>lwKIFU?`VMeIm~^>$Y6k_eILdJKCpDU7p<hvgdQJ(y^a<
z;+MRY3_Qzzz1izIDLg3R$Fn@AWw(<ytWo7laCd5pp6tn=e~S5mjZa_YU-g_VtPC%0
z^)}!BQMAPU(fJ=up4yt-tBr%TlO&Er{&Rk!z&(A_+Y=V6UcZVd*c@!N@E8l*%GHMV
zG_KgK{-tv}e%U;6o}UZ$M?^k<d7Assw1p?n#2r0mx1#cs?cas}jvL<$nfLWomj-uk
zq)Pqn6PJsMUL5x@JoV|xf=}7AMZ_mRS##o#VZ*iXL!a(CEsy%0dL{jHuUAFY4j(o7
z_qyv2znZIOeD7a-gBHu;vMm!C?9QJPdYe9JeXVEWv)tv)!PgmQ&Xu_B+VlUg@s%>e
z=50T`S7-?T?2?g+<m>yol>H?8)=oQ?o}U~0Upeo!lDl#9vVC^(MfpEc_a#{WaD1OD
zGm9-Y%5mOAo%83d!v9zH)va;)JtN}f@yUJ#sq2N@|0*%p?Vh~ZTrsHU{EC;8f+K~F
zirl!IvOaz*>+a^4g%#6(crqXH*ytH`_^{tjUzVcyBMhtW=dZqfV~Tp1%Eohl*}2~Q
zt4v(}{rwi1U;0;b+%(rqD@Hpe&RFw1{jv1>TT&lCCu(}e=*?X>X>sg(tLSx`7i&$H
zP~7mWRrY%ibC|pAq#5Tn=?5wQ72FigQ~cbqT}U@t`&LVG<wWUM;+&TbuiW?YN&ETe
zN0W5)pH&;au{!+BMrT&j@orNkJNaeP<%3sC{?FRUY0BlYFQ9jd_{j&3vp=}Hnch8m
zl>6AEbtW=VX*I|At{wTk&|>CsZvGFU89k@n@`PUSyqvnzQoFHS((X#IkFIx`l*{^!
zc~YDaepgcX_jNp-;B#vB)+Ik;XUlOn)Nsw}KI9oJxMb-=1-a67?nSziVyizZiNCFw
zzj6O`DPi`_S3{q?a@GB5ZqBBu*(08F%Fe8>RdZ7Px35hPmT{kX+;F$lY~AD^k9<FV
z&vbYaogckSc1ocb^S#bz@145eAC|o+Al9waamCPlanZCZQ|uGMPoCd<O?Fq=-Uxjm
zE&Y3?(=*-o-pRGOuxhUT-M00||3xkGo@n;eY4t{@*>A46eb>0f&RwCq-PN{l-(Ker
zcH353PvBVayKR24kr9jLl+Lj5rN#Oo?a#I@y??CI{P&HomNm*L5mpWzzqj)Ce#nvf
zP+*(+F6qdJj$H;^^WSbNK7L0iEJceaYHFXa!AB+2yM8nEig*9Zwq8*&-CzCb!+GX*
z>l}~plv;j1y^eFuN2}+SOv!BSTXvs2Cg^YBv+7vZ{4Kw9rTtGoeySmz$+!ODt)zo*
z5A9Q5el#azSxo-HIyLu4ch7g6Hq~d<zZkvXl9%(NRskvNpeF?rC%%)Ov*_Z7_Sa`F
zl-@F)A#vfUkw#y2?O};Jf#tP_^%zg5Kd<C>+?=eRcu3*pz4h-;FMR0V{M$&d|1qoY
zk>@T_w<kW-vCKa%Rq#yQ>P30Nexte0dwl2JpIQ<9VCRXA7vvr+Z@Tr+%l7z|!>K<E
zJbumnWphK){jp{Gqzgt~3*NXb7X6YsE${C=EvZj7c2k;4cf>w=EMfW1yV7)OP{;Wz
zePK4PGjg6ApWC~9<?0JN0#fRwAI=P%?7`&x>FHkn4~rRBpAh*rf6cZ(>+=tY+%L4v
zh-8?#Z@KLDdl{|2+Wucki%MA`9eDn+(#Zwy^>}WosN8b>uzAm!ubFvjjHf=CH*rXL
zDHwOwsTsb1y6>LY#yts7Q+mRtG0BR3|Lngs$L+nt#K|X4JX*P*)5V~(VH-#8o-c<}
z)U>&r^o*9yop{Q4?|reto;9DGABwFm4>Dg~Bv|kBY5vvUnQ?2PTTPw5pEK%AIQ@Nr
z^}dy&u`WSVckM9PXBD0<)V3y0!{dMWHz}qgleZK{lr3uA%HEZzt0!GpllCsP>a|bK
z@msNHess)~ahP}P+Xk6$HHlrh6LUPKy{~h)7hk(G>C@$HyII0Ex=r29S-!GuUGU@7
zn0-=eKHCl`%{-O5rrJSry27!9{QYNAv_9zDm$__6*1WeS$wcqsDb;>mp*zMKi#kK+
zuia-B__zGZw3#i%iZ(y_dGD%u-0C^@xm%=E{bAYhmDB&bZ7Qy|^3L4&P4~$W`4-RM
zWnH(67($D_Jmn9%Y?=O_vF8k9(y0$~npo_<J8lv;@Ju@$ysr5C_p?uaw~8PCn)k}M
zBzDtc?;oEGc3d~vyE89yLEbXy)S{h%*K23%2g#@gPU<u|yf@W(GIQyKcaP_pJ>B&7
zh1o*Ui|-C@bvf27XzHvwCE!K(jgtahg8M2x`dy~ZOAN5FlZu?s`Gb?qT4#>!uSK_P
z?`$^OIr;3H$nEUe_jr3B)$zn^T6Lz*mF;-uCn=3BqHnj~y>USM>SmtF`&FLFmFt#F
zFM4(6*{u6Zt2F|ioy~r`^Mvf$)NC8ysr_35KR<X{yKZ+9-_aF|l3slLr`PuP@uqJN
zMcto&WZyeE;7^;)kuv+8+CPqWS9YGwdZ>J9rk2^=U(F>y4`1Cl!AAa8#jY3m+a!K4
z`xiZXdnlwhRU=fx?&#+DYjF)SUl;x>Qxt7Z_-S%>!)N)9?amLDtA+JhdYg1^Ul%dO
z<aGTn)eTRpW$Sy=cuwnnee)=^W6M)Mcb;j}r#;=+Idgx$;Q!LH@V6gLul}8S-Skpg
z=r^gluQL`uHeML<_gb=Y=!(krtGUZhEj`S+JZih&ZKIOgiuZObSasR2Ir8(;zqOk;
zIxkjxdT?L#F8^(J1NTk6*>iC9_ukxD-2$`1&Mdb+Y3VKXT+Cw2(N2kJuV%7`zuU&5
z6n$@fsK$BG^wcBQrz(A`X=k*ynUr?)==J282l|Rl?uM!{=Gfo3qc3%170)8y@}h%Z
zOwBvXdNg)NOzrf1YMPU0VaqD+<*+z>epk+fvXu+whQ?i5_T|9pektYjy1=zlA9Kt<
z+qNy*<o3zrjFnXjbhmt2Ig#n8@AigIc|Fc<C%hhCFW3BEXW4bVUhcc>zvtHkZZWUU
zNv-|3Z@zt$*jBy!FQ=WE@c4M;tO~`4KlbVVmfm4>>*F=)+b`lC+WIRy^eb=P@m{q|
zwJ+Lf(J$}$if+Pp=ilq{v+&P0<8A9-veoXNLA$zI%#~M~^Dboln7Q;}%=Evf=5Cht
z?Ja705aYjdgDzLY&ns_F7Y6U%5v=UI&d;0W?Bv5m#pY6X)OuXj+V4Hk;`YGLRO$Pv
z!z<*sU4F2Qg@38*qJ+5{6SvFQ#GVjTFS?*)%~3OF$GMo2X|EnVT~ubDbY<86!m6`I
zn?jxI6CQhBUNCXnJgyt=B}?>QOi+uMx+3s}p-bNLI<w4!H4o0|NPaqe?}9;OD(9L_
zEOF<=D=xR+`qko8X(zd3MrJSX_B(mJy82sG`3q9x_g%RYWSx0xC6jyLgFgqpeGhz8
zoA~G9zT#y@m0s*SzF(W-xbeg#ExWI4<R*Fk@l;*=zlu4UB_mqJU2XH7Me?zMf9GE;
zW{-|E%b2L)duYKa_V}k;e}D64x~;j`*5jytwZQjEM)#?qJG7R45HWi9=YdS(M4#0B
zbzDt9KR7LDPi=SF>;IBX%l=(d!`Aj-@BYWmrfx4bEbduywt+cYzC~nzNb#qov-WOf
z7dvHrYT5?&<dj|V_8uDP?27)+(%<g;zRue`){wb1&`3XiN!DudpPS#7xNfOjD9-P7
z&_yd#f@jUHS>3O<8Wy@%I@R%5&#X^4{`N;l^nIq&{jCB&Kid2+Ue)OH?`gNnyMK=S
zo~L_DpBIQ5p6~y&<#*XBG5=+MclVyue}47T-+~A2HTqUR-%RqVOFzuB?q`AAVu5q2
z{XyN)tFOE<KeU6TTvp&>w@2*Nr9XcJu-435<79rVHTz78?)K1K5<EI*H8%8HJTeQs
z`PNu#&oMRIl`W}lj(&ZyeY4dQeM)ybw>GAne%sV7@IG<k?tQVx?T#2<taZHbYr{;R
z=hD;OeZJ&spSpT&#@j_b6+1rE-ZQi2+4i$(snSEezsvl>C&r72%5$}+y!b7+V+E&P
zue)nreIn<-&}O5@Gd{(p&QrIryWHZvc<!8CUj*-&eFzAZ?CoO@`*ZioOo?50AK0DQ
zAFB57p_f2!Z};6LcJ~Y3cW1m`X!}^z{{*}BE^qdR=}#SJ?3H={_@Yut;M;Frrh0<*
zpI3EGK4;@8V)w^jSrLPYs>RyP-sckM2CnX(-0)@Bom)4bPY{#5_qMwB@tX_!N;fQ-
zg$vm{xQY*5OP#*s?2l-fmB+7rySD7sPwOpB*&=JZyyrV__MN$RpTe}d9a|^ci_X5?
z?01RpqeX6)vQGOclX5ntH|FBPN$)oM&dbt05<X4SVa9t$*Tomoj`$pD^n7ajPWs2z
z$kj!4>woQ$fBvgr;Ya?HuT0+<C?zaAxlAZ+>z(f$_SfGBE-K`HWvn5-cQWgoTZP%j
z=iJu%{8L{)=w7(7Zd-51!4)BqpTdG&_VFJ4r2M<Qw>!<^dr;j{!D(;*9m}1Rdw-YL
z<&t~vyRNBDwO_Nd*6p(U4xY)(|H2XzrS{aHIP+ib)bTo*|7*mhuFQUw_`<rnK=9JF
zyCN$;CYZ9n<6FA&oP^?aT?dm%{|)?Soz?G^xcOaC{Mm}{$t8cE7w&vJ)$?op4E3$1
z32*l(ao0?~`geDCNPnkh?zCfnHV5uBb~*h`hR5N*qx8QfweBxpwLYwg+<tzByZu4?
zy(!Jd>J~PID&9CFy2Lkb_O)qQ0>`|1-pa*o;h6tW@P<~HzroKsque__PrpjCuH0wG
zaOHAetjQVkrNJgIzbIG#6K|hza`9rm_md89-FJ48hq6eHlK3R6ON!ioH|;anK6Bc?
zWe@I{as6G_`#z=n+Yhr-GWLr%>HgHwmr8Y)`?~kWb7h5$3Hx|tPoHP6xW8tHvZ?NQ
z^I3)N30e00mrS|Rb*%QXiPq%}HmV1Y3p)wF@@bM<_j}5;^8JcYTP|NT*+19+R))Av
zq}Q=aA@*D>PkyUiYWe%n;qs34JEyAhnEpL5=Zz5emrd-;{_1*dIVohn?dQhH#a2I?
zIJcDkPqO-^a5<_jA>x~($b9Fv6%w~L$xh%+{>=PxtI#xGftfow&K+t_uRIebdH0pQ
z$gK;P7dihqJXb^9E1TbFuV>G;S=)q-PgH2nj!XM`xN~+wZJ3YVvy_c)CqHd_;T$G^
zZmFlqHuk_@)%JHLRQhhR_<KlPVoSV->*kNm=MKo1t=Y2egKoC`2UWS{CzfkJYCBn-
z7x!~QOmo?(q%GZFYRpPJFQ$D@iYaB2`>}CFciQCdff~G<MAnBkxpCc^5NB(1s^sk?
zHLt@Tm^BXZC|B-$d;7`pL$4m4iQAn!-TO?j=dLsNCwV(Q+T6n`zvcN$-zO!tvpnwD
zY1I5vs5PpzIQhVGj?_=d%0Cqey!$!|^6XzcmMg2gBL4DXZG=s&{Jl`W({CdT&fn|5
zs<8d1-tmr$vzA3~Kd_F??a1Wre;KJ(2d9<inLO9BEC|g0@%6@wZl1$Of@bbMzg2em
zZ}zC~<=-b(&XE4j-Rfebl=^03YQ5;kODE5N-|6)vd1=DE8`Y~nKI^V|9sbhz{l<KK
ze(rZI>J#73aCFy-eDF|mul)DZs!El6vR_so{-JYj)rRcJv0tVi-gG=Q_hXX%XZD|E
z`*zIIQEmR+Jf%wdmMTY6aBKXEK9A&m5*6>cC+eJzc$s){PeRP}XHEZlp3hI8zcc6l
zQ=91PZ>NXb`mtR|o&R)s(Hq_MpWiGyqM-a%U}>qXjBne%z2~K@eMNO8_N|(4y=vj=
z_F37gxr>f@-_NyWkLs6py=NreedLCJp7<9Ri@Vp|0&Yw?70qF>Yw_dCr7x`}W{KYr
zV%p>RNjg81x7xheSvx#xx@zsKr;qLKN*T00XJ1`0S0ZHRr(1ayWqrp5Hg(=;HOR7G
zaN+n)^WD8y|D;Pyn6z4Oak9~;OY+ao{<tWtviX?v>;r1&=4Eo-{_*uj!PdgdF&p(#
zHM!g8t6SW*X)Ku#<8$XsZ}@F~1N~~RiNBp6t}Z$A!%QT_r=82DXV;J8Yd?#}F4`r#
zSlG9k?`Ui3mzPfz_LVoh_t<}9^QjiYn9E|vk0z=&WUSa{F}tinR@_Or|GQL~b$*d$
z$*Np+<1I;go+5=S7xvU_%S*J_k-MRp%}sn>!;;ynHm2$?KcF2oPoMRk+WGIFAM0@&
zKM!p9XmH|2g;R#w>Q_a}j><V-S9?~)d2pl2_x|H<H|%?P{>IP!ZoAO#f9X6Oe}l3~
z6K8DLHLq5>@mx(r7DtS8<TcTE6^E=BEI!mDe#xl0DV~S@*PQZ44~tgD-<Iv#R&q@B
z*AdklPT_m3$}Vt}*i^2wp1sg>o@~szm=ww0yUQ-U`+7BQ!xFYg;llw^ZVm5>_BBNm
zWIXasocqryz0LY;xzA+ZvTrj|l{yRU1@BFh{aWJ8e8cSc-wI{9<_$aFoOl#x6e{XC
z@l*rLo0kj;x9&y>FTAt-ocyy{ZU<N<-hb!8n*D(xz{cb`YewsR<2#WVR+D{l+r4-u
z&$Bu9|48f9BisDhpItp*{lvcI)QY>Y`>YqfJY9SD_le(uOBMc47F0gESMc<y-u+u|
zJURc{Al{4H;xgw(ah|y^5@Wv1+&=ZN&F|dvb$32^w#DX#?VEGs%+Kx1mh71LV*d{N
z%EFIKmo~8Pje8UE(8%GAd1<F=<(+_*_9cN#iHuj27KTUnO<<|q-SM_;;#J>6R_il=
zH|WYU{;pqsaFV>wkJu>_IM=s!c?nluxR&-}iLTZ0?471(#BvXxI&{hFu3>WDAIBw&
zmwe^b4}Qe&z2V=MqgOh8I_K+&e&u(}4N91mG%F`C<c*`t{KePjrbX`cwotp<#`ZBd
zDMZlfrOdYlB|e|7yldFtpXGFN`QzM8mz1IflCPICUcCD}ak1VM3DM>~TI&xlX)|}R
z`{<X+e63B+^RdeM*)k53{1SpVPMNIQ?NWUCYsZxL*H>Q_k-8n?RDCs)Z|SeL6(@z1
zG*;d<baX8~SsR|W?8=1XhqF~4yk46)YxUcB_ebkg_T7C|a+q&=)B>rkJ)bYyYSj6h
z+H>zygEr5iH%A&34T5=AZD&-i(K)@j`q<a)C-(b<A7yA`OR$Oh=a_T<&5DXb;p?i0
zo~OFU8rglec5}&ziQ{_pVv+yOmW=I{K6l<Pe-!V%e*VL>;Jmcl)a98!)pp)mll7?X
zB9n07t=fg}^a7_}DNORYd9_LY&iU?VC4FK2--8u+M7rLaOo`{2UO(;5xj36Qom;F;
z9?9pl#P<DIJMs5ilil}Ck6hPS_Obh9_pG+qQ=ba%e6N!>yR*}#EBwRR)~m}^r>b0e
z+b~y=Rc@Z;7JbcB{pr^pKHO)q@s8}AQn%+~^(}9b!tPyvZCJDa-^~E;X>Nyi$?*Ht
z+)ED25UYPQP5yfZpXuA!n!14VcHh`ezOhU^bGdN-wB;Lj<iDA4zwbri)sF8OnH-<o
zUhlZLD^JI$a@Db)X7^*xtuKpIzBk+K+m9VFi<dGl`h7|ENsw6f4~7--4wdJ2`Gx0u
zZr3}%&Z|;p-IZF#sgu_h7cI@zzqBfs!-h}!LBV>vj=D>4whQHb<;(mNW!L;6=QHn&
z?N`I}+Ak&<h3LO?j{S0J#|g2zo3Foim$|pB^?m&4HcQ~_>~zWN@2)>M#J%;#D|1ze
zwQFtu7fA29dV5KRdh}|upopxu%CF|$z4+&s^Wx?8Ci~WwPn2Kg`!Mgrr%BBdH~1$k
zDGfhaa9nnI?5987ZZ9h>pC140@>);dQnq!$i!HkztLPpwTjr#3!6aM9!tjw!ZuR$d
z>9_jSV#-q)4hsKVS3Nhz_rbn@|MV|6co)C6yjdeTVe-B^nrC*cW6qg4&qS)R_u{;0
z`F-+2QBH5yR(E8k{_)kDvpl`vV#Q9e<#v9#xgk3jUA&yK!Q|%ef0`fv$Y&i`@KdAs
z-HQ7wcC9@4{=$N-Uz>Vu88@t+xpH3MgWVlcI~bR7MVjbIzwSzj^4q7f;M<%{&f2ey
z_@_@h{^Zu02Q4=j#287P7P6SVVqwtTx)VQYojRv(F<g<+J7vT9pKWUzfAcm5EuMSj
z;=k-o8`gb2k|F#kG-6e!1w+{K?Gvu0Prg;@8_+X-^JSI^DsOs|x=a+-^WJ6)y|;at
zuwB;L4Pn1#w3*%g^xJT9S6b}Dm`jt2E^L1BqA%oPYuk>Xu$e#NQwq&)w)^V$K8o;C
zO6{GwN5AF>J8R_XfC~=iHR}3S)+|ieSM8m#^<UebO>16H*`7T`_U#SfbD7i4BsZvT
zWa;z1{nhF2DHF?xdz{l=AIiD;U>akMR;S$awizww!&U$0UQYKsn$mJn^yUfQX?c@d
zZ-*Eyst?q>`Ml6k>T*Nw#5mt1?fbplQF_-?BjyD$Z4<w)GmZ81sY%lwRp$mw-V%Lj
z3zx4(arRE5=})Utt)3QHG`;hlP!TgDG341YCC*=G-0e!##3jWqa;`YKdaAcp+DqmQ
zoOf%Ace+hnw{Ynqz4%?+0W9}pJZm+kfAA=;yV@YJXNSY-E%w)4GZ(b|TJTrjXvbCd
z#A~nW+Qbfj|9-6VK*^=}BXfSFTzed!J3qK;e{1{I=dupVcYVK;6?N^8!rr5mP2~|0
zRsSPyE<bmqu41pfWWPkm?}XAVzxHPQwN8lmt#dD-Let=XqNm#X?oDR?Z%*I({_4Q|
z{<6QvKU`U~>rCvS6an6u3s1j|JL;gvd2t_G`;knM&D$27Ue)q<4}1C?r^u=u$}6@y
z+CN%T6Fz@&wngYl=eje#%d$31c)EW7?MKOf3%AwXIa*L6mZn*3FDCS`yUILea<RHo
zrGDWn<E3e@cfOH~x!m-@(#B1XsrS|SrSew-if%PGJv*-VL-*06h10(|zsUa56n|yg
z+VsroJb4}FJr6f?h5L%!Nxdq+VCpHhO!b+0Yv0*E+swngx;gdaqUq76pO+lDt{{8m
zyAnfJJx^q4>NP$wy`>_7Pp?Wk@^y0_EiG15J*4Hibk(EIM_zW9>`sV>%6-ev(%2E=
z{+++--+`vw^>!;ei+_~JPup}}i2uFUrg?qhU0z<jG2gG%<{UQuUMXNxzw*<(V6#nE
zQcs*;bvj4nkXr4}pl#hud|q8wiyw;Yy!J@eSs~~7_s>(<PX4uyEYIwh3yTsgI}wl)
z{VL4io067ow39jSjEv8p79KvbYjqO;uDyR+l235fClyVI__2`dVXpn0)QtIWMOs(5
z6n+Q}Jf?hR7h7E?XR6A_9Xvm8{{H{=kJr3UJC0BF;|_kzbj~dO)1#}WBj>SBKK_vZ
zjLMV+_M*jiYLDG7ds3LR!~6E%{7t)eY{~LEtbB5z*1d#v?T0=D-Ff@xwq3HG^b%{e
z4Qp-|xqHlPKeN<$zen3OlU2{147lv_-xo4|%jG($-uLsubM1DM``KRtL!ygaqHE$k
z({`>FkzcU-vuwJ?r*se3SAP@K`)*yz{>Q?4O6%j+d4Kz}KX2exc<(3Q^JM3$Pw$Qg
zy=Sl~J<C$)Ci|pDOI~j>>tYG9X_vmua#rZ)kCdC)YIFMIkI%eW^X$0Alop%uTF5VI
z5{pcGwpX;N<<sk@1!lJjPT2;tmVDd(>G97?PYhMh$!74YGuZ2h``UgNKj){RU3((u
zPhrxA#yXF@8H=~FW{PM`Ez=fkF15e1ZAbsDvYm_GmbKj7`K)Bw&Mb|mj2GhHI_>S?
zxbL?%?)=r1(5-)M;zX1558B>7e`KlGpOr6n9*){_ZJT4q^s9P@avmSby0a|iu)V{%
z*Y;Tf`&T@!Fw6K@<`Z>m->)0~yK_<=bIrP6rJ~8TYw_X->$bet;P{)g|L*TF`!ct)
zpQ6;}aa>fh{c7QTbMwF7+b4gnE&8%^r^tz|Y{@_FrtK)&K1JYB$a%Yv%@1UJSw7tO
z-Zjr^?)T)I^1FW9>3j9gdudl?Tz$;{7qh#NSB0DGuZJ;w4$~)}n`&$rx2}G}%m4Pd
z2VRHN1!~GYoWES;jg9})y7fGBr#G+c{Sf|q-n8FyZO)cWw*4}#`O*~ayM1z(1JlzJ
zXX&l^`sc6TY=4tw`4uZ3y@=KP|J6cNp~^n?-UW3XUU{MLvxgTmSjG1guPbrqi(e`x
zdTDLC>AuUVQ#$Xj3+DYi;nKxtUPV9EPEI>k6>BQ_$M)0OkI^&N?R?s5J@xtbIlr?t
z_I+;u^H;>~@9+6J4y&I92X@MHaL&zT-dpo*?;PGqWlC=&nm=^C=lE=CFEvlkUH5YP
zgX2Pi-ujOm8qR+_@+t8Z!_oq|C6_&BxkTOW>HE2K^1_!{-71Ub%$g^3?~~s9$8`}`
z<buUwf<nW83TN=`POH)_SH65=oB4@g{wdw(UHjX%dw+LIyLIH{;gV%~b1JJ-%jX|n
z^D#J6{>h0F_OzMn`iwsBoU!`qdfoU}YrN-bM-|-RJ;Z%G*zmvWTjQ67YFp1W>0Lh-
z<91~0=}Yx*E`6I6S}zrrUoPrZy;=2LK-k81uY>HYsgv_kF1?)+XPj}$D_-Tz9MiTf
zJ)16+T=o_Haxw0cp|J1-_r?vG_1RYWM_N|BF3XL&=X<N?&Za}>R&<|OsvmLFGcqzL
z=DelHw{1n5o0Dv!t#Z5ySCluFEh^=k^>yCv%D#OmPm6*VOI)A)xk#?wWqHhvNvms)
z?Ks!;ehI6}@sAdqQ6hRV7saNYY<pF-;-6dW&-@$BOMPZ_dG>E~ePGV{ttjqY&6%hj
zHE|Kzigwvqmsme6Y-ouM5sCPDE&BFa+t2#@Yy?}bzKQuaakpOYr8%Lmmh>BMk2tl&
z`$wU9r-x3_v{?SoQynIKho|pX{VJc?e%#^YWuuhDpVc>ZN8jn-uhpIU^XKckk9%*J
zzt!xnYhaprct#D+O`AtkjHji}C=+KrEM9sr^@4eH*X7@<q<(zdXLUB_)gjBJG57qp
z2~WRv!0FZVj`PuPf^@|U7iy$xW?H{{boG&qRL35Xy6GKn{&D{nJpC@_(1JVfdH8R7
zCvLv}mb+-a$GdKkvztZJn|PY-?XU36_?}aDZBo6u+K%@HpA8N^eYi-r>W7a>y0Yh&
z`({b`<v;%3e3I_ZlCtkU!y+H{>tCl_DpcwI-E7d`SG;qgX4nKN(dk_wtEBFKo_6rz
z<1ojL4Kd5#1W%jsx_`IJ(@P>zmd4YUc5LKgFnnaR+9g~=xVSyy!Tt82>OB)O`uE@9
zlrEQ^nqIf%g!{L9$$X_QkDo4?c}T9(c+$znPuzcxJihx*Q9sCfqR2;KyVCH0?j89*
zn!368<9H4)ah>EI*6n|Ly~;=S$}diDm+yJ*eRG=S+mdt>J>8>Ay&Np23NRluHaPp(
zIpN^*yZ`GKFt(Zm26|lk9_sWZmdn1iy8P*my+3zbI_B_|n9UKF{l9r#T~5QLj~iEs
zJQPwX*W4GV{<HE|xY!pBVf_kbw*=EG%PRVgZs1oenzCla@44Fj*@-EWlxHi;axJc9
z@HzQh=7H><71vv5dpY->^V=<a*DN?YerZ>7K(yA|ZOWzZzULi!UUqF~`&5Bd+m!u!
zqvwmruF^}{#U^%!^_q3tKgpsHy|wqIf621uGv9BTd0I(I{K7loteBd|hvIeL-#vGj
z#rtsnWY7Owd1Fi`NI804cWzMqXsFZ_^uXNF?u)#R0C)M6gn6}7^%XTfK4&zPU^}k+
zBgFXOqmBxV#HkWn_jUYBs#>LWWU|pF*VB5_?fx#lG<T}4$gVeA)WtSWS1tOq;&<ob
za;J`xDNo<e4b?P#UR=1+`j4-5#&5d|fr1tCZ<fowjEH=mSw7>|tljxP&jr0W^`l_z
z`(Nunv0Rkhq-SNgsb%U0`!|j9=g+)39;hy*v+(JgOY;O*_}Ks59DS%ft2o=G`=3Ce
zxWr|9fhWQ<j6U=~*gVnxZLnrt-V5Qzj2-cecV{FdKipRo_51oUMLpY>0d2c8jw*|~
zt?Okum0$29-c#cI)g_rJ_ZX$i`<%;<Fo(zfH?3aK_VL{67ngS|6}G)U=SQK7mEAMh
zOV2AKGhfyI-Xb==Gxd){Z_Kp$5my~|%f(KdU!rLLa-WCfn$<num-l#O*96%M3v9Wd
z?>;Z=L+J$Bizz4m%xk*GpZ#z7qYsbep0Mg>+^Sz-@U!aLlA|{k?TU!YXlbdPT$1m+
zWjlN1J(c;<GZ!!JE0T`1*#AI(&6?SQ>c14?(qsQQ?k$iw*39#)G4-BYjQj72leT0F
zCFTnDt&N!?{@~?O_h{AVf{FPijrIH1?|j!PpOG51n7#Q;r2UfVH8K}peZO7ge#@&{
zmNz!LA=$mFaK)>>I<=4Yvi8lN{hQ;~oT`bb^Q5<@Dwn(rk9_st<yPIPI!@8voY$`H
zZ~M;uvGd7Up!rU5`qo<KiECp^W*kUb=#^6`Rb<%}#2s=wb4hSc)`w&h)whe!Uk&>s
zA#_XN*wWk|9}AY%-iqHhv0#&xz}xTL9XyP$muE9=mpj~ZWlqmQG1+T1Kdm!<D5~i{
zTIy|Q8u|X5W~sVQRFnLL=9VaXHkWz1vYS@diJ9NgpS8xj=L>()H`ncnEgjwoPme}L
zdR;uB{xRM3$s3h<g~}}cce7h}KfnE(KPw{Y$o>P1r_R59Z(0ifQ_Ev^d8LdV*X<lX
zu^T<86XdO%bA4^Q!N+@t^_Q%!x4Tkb%v{xUymqZ{<crTX`Awo>t=4b9yMA+f?DdTO
z`}LP|cC`!qzqF-H@Io!eyRBX>BC?#B_O{P|eVVtaao%60n5sR>-z-^YZP%B(FY#XD
ziD69D+z!V+`*$o;Yx0t;zdcy}!`L8lhG(Tj)TVX%P8+-p;$qmi&qlW-#j`x>KkU4#
zyQjAQ{&^v$2bY#jSd~&`sOYA}9JEG!W6A5e`vvs(U)1Yh6m7K=dlmSoddJkXZAU+D
zT)p}9q`JJ%Us6AQ-kUW4p_Bim`8kC@_xy`iUuv^Wx73fL>Fd$Q)fd0SEuEVy!MlI+
zBdgs9T9?PyhyS-Ysg-fXa=X>{h4&8r<(Ra~x$(%dTDdL%dS>+}oBpn|%hlWRc0!26
zt<B==e4{$2Xiig}pT6zr@uh#vjah`wb?umDS981Z&bu35)+Z<a|H`&DZ*@wf#?i&`
z1|4Y`7gxJZTUPi}x@56y<wyC;TVzsY>P|%2xkP7H_HMYa#Iz;mxbc)&vs>0Py^DWu
zQxrd~Wc7OP+xE1?-#?da{Ko%bYTEr330|=mj%q7;Y?G8&{7C*T^ZbmQn@k<&?tOZ`
z>vy;5{0WP#cb`;~Nxjx<#P04X82bH6p8~_~%VPK3MOjY;7f<rN!KrmCW^vCpCsF?2
z@8$=8sr{bW{8z2A_uGx-Gu+mnF1U4eLqc|h!4<b92?9cG_6zJx*1FA%vOg0#wK{L^
z&!VraXV=#^Ik$IdwX`q(X{&YQe9!;#y-)Uf+4Zif_mavC?A@bNvvN_&;%U!SkG5aC
zrYL9o{^Il6E~oDrCH+aA>oRzw;>4eHH*>r>5M=OW$C@wuJ}iA$+h;PvP0~l+;QR5j
zzeAZ_^8bkh`|N&WF5&K-cSZHM_uLgB_K$zn=RW#>mNz5mr=n~j`;!y3Q~#BASY4_K
zOr5yGD3V90%JbIJ<zZjFPX@o()mtTIE4?x_Xk}ZR(CNr`0<s^f`rKCjd_D8)yv_5^
z<Vo8vbWjbw5LdHp(;vAxm1~q2+}2}SKb2FbUMNCW^Ti|2EY25wsecYE*mL$jyV@2J
zyMjr-WNutty>b1qeNxL?1Rgk8Hu_%=Zd=xTImvf}gV_1>i<@7q<nxuCWn~i1zD!OR
z#sQsH&uSXiwrJw7U0amP<`}%YE?abQsZPR6eRf~%6_Xc~&SNt@tCbPvxS1=pWI-mY
zueRTD{`sKGAXe}t2eAj*x~&ZkF9`0RUZ^C~b;EAThB}ShaFaJJYd!}pEeu?_t&-FC
zJhQ*=1Ov14Em2}Q2AYeF56#ePh}!>`vE+5>(U=73|NK>IHqz1N*1zAsKU9_=vAq0X
zIQM-WnY9I54t>{VuMX`CySZ$W?^TJM%Qu#6^h}-{e!0`a<gD{$n@RH`&o4^9I{E$H
zj-6UQ5^kvu!b-E0{`LG!sd!Z`zT=#6<)fPnF;{~1S<WhCuqB>R@SDszf0n0?`W>r0
z?vKf{sySwr2?(!Z(Txr4jGL!7<&Vpvh1ser`F51`2stda<(7(E)TnZ@dh*jJ8eP-$
zjSh;(uDHE;)u~?#mhKBU>Uqg&(wupod!OXpT>nPS%YT`fSEP*Zq&+;T@nOB*<&m2l
zr|o|8`SaJ{=KJLZEV@3%mMrU%8NxT)voXwS$Yn9RI%$c<6qUVdn)_5d+dYI<uIBk0
zd93GGSde9Chst*$PeE_~%s)0gQ$oz+4C+sEg=MVqi1NLtre3-2*ebVOURTZ?UzI-F
zI_|z&@4S6iRUW5JJP_5!6E($l%9FjJalrvX0>5~x`p$dKnAVZWztu{?-lX-1sqWP+
z9=m!@N3#~Z@RJbG+jLt!Gt)IHN;0eY!Sc7e>es0s3Y*JORn`4%$qD9=FL{B#oC2Ag
zCW^#8anW>spea;wD*9`~si>}1`)}P#-#h!W&(cDr$$NZO>t8fk-)QXQx^qK_rGe+<
zHCDg5{lpf!iV7^alM*>+^STXxe=X)%?j33^J4?-B#~F61{(FlU3&p3z1Qa=LUbAD?
zxzGY@ZrQhg7U}$sdQx_=RPlv}d#Zro*>A4t1wy)BCd<TT8Jv#(x;Z4wE3_eG+0)XM
z>Yv>w6-$aN$q=8U@zx^!xJJ*h(tuEj=g!lunxFKQzTjtz61mv0s3^s?>!Qa@o&34(
zK9f!bRnGRgy)5M2#p$>2eYZG$YWgBxYt6c)9*ec}ML8|2{kQ20y4&!#uJ<U_QBwL@
z)2A!9^-;*VgDF|z2V1m?BSjfkD;@s2)<_}kaeCh3sg8?Jc`ZH_%{npUp2yNj8FmRZ
z%90-zUVOabx7&ex8X*OVFBSweR>xb`Tz&A^!MZV;_4KmZvVBLg*}5_u>#_r^&a&SU
zxhoyAq+hhK!-Q$FOH)I<Fu%cV#+1}K6W@Iaw%d^__2>8(nG&DlRugPyD1AC%VB~y!
zXKmC5j>Wx4c38YPeM-3dO<L-ahCb!FE;lDJP2JIA<ac`G8a@7C$7`7<@<rcmRDAy;
zB=EOR(HilO@h7ag1Ftb^d^#@vku@~gYUZgG*Q77~JlH=qYUy^5S6XJhle;|AeWO;N
z+}2PQVtMY?9UYNVhb$6iNzGH4eV9M&Y4fM|=N~=}(VVbouUzQIjLkFhZYesZv?lWI
zROxqEzr<nUp~*2%zMQt2$jsUm;UhRvui2dUS*Xe)k=89Dh5wb_#owFc6v7qweN(F7
zf1PPlw!DpuNe`H!ct?fvW~PXboJF*2?c^K5^X4D^x8%<>w<m5vG7(`K;nOTkRXVRK
z&r3b!yEvhJnsHZRf3)xK|3@#mMEFeJr1<rs$Lyo8H@%R~b{GEh>cA<Z*E$Q=l`C-N
zDjhk@X}9>OX+w)pH?!l>DSzg?(mVHmmhU7L&m)Sf!<cSsI;%-8X8+amQY}*Ea&*u~
z@2_&FCva@Z%59x`u+eqB|D$QgGcR#J-yAoua;9Gu|D9gunU79nUoYJ3sQ8*`?^63{
zOTG<(SJv`*o|-1ff5aod^G50g|H4e+vgbwbT;-f5o|u(WaL8KTSib&;f=K8C%VRr5
z?L74CjDNglh`n^7`OwDLi5gZ{j-R<6;&%DmlV=mp*o4S0ye=YhwfO2!siKtjNV{`a
zcYV0~_mfPDQLjY*=cfyo_stFMoIRQGwdIZ89qTy1tZ#Im!mQ$5v^mD0Nb~rk&5L~W
zCw@C4nJG8(_!=4RFQ21L0-KsdsxKZ2JY814;<mtzQ#%73A4!NfNL!a_@F=EobVaRb
zSioiG+8wg0RO8UyLXNJ!l#7D8M!&!DT?+W)Dxs>gE5$kVQRv4&(WxO@PfgbfU3Fop
z&M6~}{##vUM|LdqkYP4$|FFtdj;q;)(TmSHNdLXf<H-FV?fCW?alBL%%Dw#G{fXqI
zxoOKcTs^ioySab%htogy?wE99XGE7p&zA?wCQ8>&dAf8h`)-fc%MT|kx_6}JOU~nt
zS1KpC^mDUbOmaT{Wnu8P(B(RlBHBc(Rrn*7RxP<&)2DRK{DN}#eg0UZ{Y#y^BxTCF
zgD;5)U0ccNT{of6B{TB)CAMBGm)GrAy$b(5>C4e6WUlwBKKu4hmr2@-h0gxz`9@3b
zUO1MjHR<XC-D7>9rV0AU)y&n<+@^MU!pjq0M<kyKNY8lwh<A~SnpBdKlGzCtX*E8l
zg?xuE3*2IFoj*Z()y@EysuIh$g3azvQn@FaOj4PsyZ82ins))RFU&PoEz>WJ;7R=;
zU8{XHwm)1>z3j`v;KS2flCAz5m3RfzbZ$Ipr}=rw5k61x%BSa?J5DXGtDVF(ZHY;`
zWWy#Nhsds=y47B5RU#t;k42s`+RA6_J!zZGd>f@mD-Su=JDUv*x^Jfze%2|?EI6+t
zo6>SB-AdF$X{+iY#cRh7PMDB=_)^F-KCvB(R`7V<lMZ@1Vf&{CpPW+W&iVNJ|BBg0
zep5egoT%UaufNTRv$&4u_C{VIjn~b8ri<KK)pA2?;kGX4u)-5ot2Wuro>}x=Q2JG%
z<E6v{C*&PgSQ|gcPW0pwyYsVr=am0uKiYYJS>A~*ndL6oJZH-D2Y*{PF;20)Fwe@V
zqW^nv*EQ=r*-h^PTAs*$40<Db>G;HdPX&I>GG4lHeP^BTtUqrTp7{}Oohtg}*q_g<
zYaM?#-RJ(LY0v*_`@#Lo4_=oGd$Ta;o0i$eZ|`3==ij*9w*I#C&R<{7%Afl9*wV1%
zYV41{=J$47y;=DG<cG<pm#)8F@r5V-eA6S%A5jnd8SPy~_ex5(?z+vr|6|8j_y3(0
zf4;u37o4d3dh40{Ms>=Uq&8mb*cRXaPdG7AxqjjWgNDN*lN}l)6q6V_g;n$>?6{%r
z@Q9`JwUANM5k?_(j~Io*K;?!b%__c{Uy^S8=u{|dJi;l&?h!LDwCTvw7Yr(TH$xQ)
zA!_Hq7%>n=Yo>+cBbH8P6|RY#Or8R!tR75`LL5SO-2@pXIZR+_R1r{V6p2!qrNGJJ
z$pDgL0SPyNgcT;f6K4?W{Okyq<ABL2pEeaxVFZaQfb=j-@|fV#s3N2ysL;vcsp6sJ
zD8z_UsLPm9Wg=9agAnH=jR`JIDohwcE%q)=DncM(22UlANe+`ZCbgIgJ5J)5<T62}
zK}Ar-Q^AwP^U#k@5T)cH1X7?fL8VD$Vsv+`u!^URAV@;U<C4aaQ}O%uX~c2`Se|#8
z?)m@ASMTj-WZt{nG&7u~*l4ACWV4Ffi-qR`jxN_XtA4QbOvLIl6OB5;-7EsrY|rIs
zz1c5OSGaFUwAmc{Muuk%(<A(JjY~JxpNcIz<}H%2>)eg{t{<G!dVTf>Y(MceUa0P5
zO{d$7JZ`Is{~|u6p%YK|gudF(ATYDx1dEHBf)T?+#txf2zp|Dv(Hj3i=8m8r(TV|F
zjX?}km==gIEtOckU({i#MV(Rrm+QVus)vMrSqYkS{$Tpi=RN%^7eix*ipAcK&fywd
zJX2bay;UkNe#sTWsB-Y~1BK)KvYuQ$WgItG|2-Hk=6$kpJ+D;dQ_;nI-11UN(i=>^
zT>RnvSYI~c&JM4q%k8Daemnj0|KBPY^Y>!?!;dR1?4}-Hz9}lVblIw~33v5H_Zx1v
zH-5X~zt9KnO`%%)f~yN%bdUTiKaeZHo2L24C;k2P#s8UpgzQY#Vhpiv&|(PTT=2_Z
zNbJAakJ}IOoByw5OJ8;5bj^(9W8vRlF|Pl4^~`>o+4DX9*C(!<IkV<2x9z0l-DYR+
ztw>$4bnac50ILtr7s%_^>!&vS*8l%kVA8J5H6LfcnX<0L;!V=jrYT|nUhZH2)w#UW
zX77dCJ0D-R-v9IcJ$HTlkKc#xi~rmA;d|?T{Xex2i4p3Kf6?jwf9j{`|J$WXJYpWV
z|1X{V=Q;12xVG2zi9+WOKKWlfdBgdGud*KgcV6%#KBwWq_y1=-OWL!4tYr5oIy!y%
zT4(;lD{uS1k9vJvPQ6gB<+9_S->o;6d%fL%V9)z**6(8XW&8S@b0q%8b9@r>(E2<x
z(&XfMChx!bB3EVVSKkZz|9j~o@8{olc-(fMtNSN1E&EQ^0{+}@m(nH&dhnf%j#CRh
zf6nstiPb-*{oh~l-1of1`^KrSg?@Ycx?b+t*|%Qs&;7#>KDa+fai3tMRIJRMDeHK(
zh1YSpU}Db2U-x_5{v@s9EcC0@J;qer5oD{iKWSR%dxOxU&KqMN9qBQBx$l<CmuBth
zyqo)e-ZM-;*>dY*p?~J<#+i&CQx~WIS+T2_JxxZ$-FkUTQbIg$+~VhlZ!VehzVI3|
z+cJh%Ei2BixF!7fKZlU6{PpRkzd3e%o9^DtzM)?*d0zAPn{rbhs-1WyD)(I|y}j#q
z?OwUbY4%l*7fGKE+_+`_!kc#v*Owirzk8tiNyd4lSBldffBAU-z4gnJDtujrFHX(8
z)Z$?){qoA0|F7$p*G#K-4c*zMvh3KEt*t^|GTwdGxBp+&`uTjtKT~h!=QlUZslWDl
zz1i=1hxg>{yi?wnQmoSL{dg(gN13zFGQy@GD|VRA{A<&O)Kok6FP00=f2@g|d!DhD
z??v&R4LSyA7!S)BocVZYlf(Oq$22$2QB*pAKCJl61|6d_j*n$Zb{bmhb@!b9ch>Q7
zOiGNH<aF0}n}7X#T*~{uSARLHKqixtzrz#Gjy()edch~A4jaaa&l@WH!l&KUD?VVv
zu2Z~c)0?uH%+o9rW<H*p+L+20@w}n5FMK-dHOqv#%-1aAV#Ok+U$$ajQ@m%J&W5v$
zw`De*{djBB0W<cT;sa*KvrjYK_EHdYD$x;n6miX>L#n>x^nLEB7N33{a?<GA^=EI-
zp3BQpUre=H`y_L*zjkAk<`u6+W>Q-PG^1Bc`ROHg-BrKkl$cj4_sUIE!)`jQ$ypdC
z#jV=;L%fSq;@_K<LO+@-``O>HK3;$Q$K?P6^$+Wno75wgM|S=#RXbu|eYT@#e^%?7
zdy|C@NQrcuw-3B0UoWaYd4t~lnai%%{PF!bA^2>HSnayPKbDgYSZwxMdima&w@$YY
zI`;fLaLm(Go&Dw9y}iMOX~i2g^IGOla4cCBa4S$H-ow_uWXrty6SDT#Y<fF+{d>jq
z`k#h*^DlkVe6K(4?eSf;x6+c2e_ND$+3>zy?&gg<_V4gL{^?uh?B1(?kLFtKV?4h7
z<_|uZ`^z3n7u3w|eShF1RIo+XB5}6gjRxUuH!+xZ8icpsJR)mRINJ}MxkT0?bGF~D
zhRNG*qA}<41*<v6Yh0D=HvFNGb7IPy`gG5eZHHIe2R?Rl@jltFApY^KQrN-c^H+RU
zKXvs}wZ`6QnqL%mvlY!-Y{dG+#(gF4lt2f!i{|xnd1a5@Rbbe)iSzgS$NK3PmmYZ7
z9Q=fvzrImG{_*>L=7kr^qBSH=WzCr%r=e$2f5j}&`1jAxul?hH+&*}p?_b%6|32(<
zFQuy4-}-a;`TPwcFWy`H_xXH&kBQ#r_g4RNPAi;l*doy*Kg}>#^75l!hm6+fKE7qN
z?&{ap)QtGHZ>1YwXVpsHn!d1B^5%8X-$q;Xoqrn@MTpBzTX@*0B*KmF^&$}6#&`Ne
zVr)r7Ti@vmg_4%*y8B+s={p}!eX>K(O3(Rt>WdvcbFR5APkph4XYRC#mv@2bg_n(r
zqQvJ;Tlm?iB+l*JZNGRwr?_R4rmuXGs<wWQkE(v~$<ra9j3$RiE(@Bra!RU}>Kc6=
z=jo|0Huc1u`gO>tB(iPY=@X@rKSQT4d~H+`>$dLoA`tEW!>A}${O+`cw~b1o-SS>9
z0?}=Gr%&X@mPD^S?G_i-RJP{yvZD|GWU3mv$Jj4<yL9R)8SR;tuYM)Q?py0}&P-{0
z;LfRM_U5RTY!lqodui>Gcd6PNH&4BB)9dXSbB8r7=fj%PwT~L#XV@rY8{Ii?ZOHi=
z<|AG|a#U-?%!>9iCaKnK6S9w9Ilq-xK$~&PREAq#jJkOZ5uFQ;_P$(G+##(H=9HnW
zxMiwP)E$P6Le9~h3)fz`aEYyBs=_TVf!w7X)65*!xLgcdl%XBEWe;PLYR@(yr)bTE
zxl1mDMP+CkZkZ}|%Z)F0Y5O)ImuSsJxhfaKW@TsxZrQ}iD6J5-D?>YR%T%LVZc@2R
zd$$R>N3UF*`{ZKSuMF+RKlK4}f1t@C|K$Q<t<ai}Khx*b@B8#;_c{CA%8S#?C3hIr
zMcmu^G&Mf1?&qg%`ue{=ZPM4@f9O+LL>%Ag(~9f$b@GoteY#^^MQl;by}r|@Gb=6E
zcKe;aompwAtN*<8Vr8YZzW(#4Pxt8U(#fCx^y#KOG1pwDKYhDvPt5e`uXnxJmG|XS
zQFI(AT0WIU$K^@STC;jtL(RV}NvAe0I<;}#9kY<=!n0;E(Gz`>UU$xy-n7nk4M(}P
z^tMF{)>>pFMqle{-L`1q+6pkUbK9atYj<Q6Mqlgd-L`1)+8-I(cKdRUdPE0a3Y)WK
zkL=?;b02SQ7u8>z(-2{JIqb?7QF9P;<K?g~TSV>G<~2kZUI`P~D(b#Aw;^KVm0NBT
z!mM^Ay*wRm@>SHg)-`y0@3OTSue4|V>RPrhNmJ+j=jFXRc_oEMPfcH+@lreLSJzZI
zGne>_m%_Jv5!JTMYlwg`b0JKddu4U@|7(u^ET30@?dNh`{e3x=SEq&NR$iYrJ$@hG
z?mcf#pPqMJeA@QIpNhcYyMDtS*=f@se=52Ywr7v*bPx-aMYdo5R1_8G_j=LiPi1j&
z=d?FmcYXf!?WR33)25%^^<vYW*lE*We=3R<kDb2#^{$JjPlHnWy}Z+>b1QeQ?RImy
z?=nj?uun9&Z_nN>Nw>PPwk^EXm92k!L$2%g|Lji>?w9@Z_CY@T|9O1><{q3<AOG+7
zr~A|Ezka&U{P+9Q{rdlFf4)Br7X5$hbiGFL^ABmaT)s}so4R=4tLqzk-%r}t{qWDK
zzO;Gg-_P3jdR=a9_%Yc4+w!6sb2s1g@~1Le(|cd-F23^l><i`A)yE~;i-Ue`Iv;WW
z=*<s4TCc-PRwP>QKb`I@op@45ZLQ1XdBUsoUthX%KL2)()UoL=BW4}@96j6j^3SI+
zljbY_JmGQWy7611sY_q}{Aan^pxyrYmstwWc%64dyNGi!3NPBgvVV^WvnsR3miYg^
zd!#fziY?Ged1=2_WbpyL*!<@U)>XdsyR_7KkEX`K7vCTG=iQyu_Re_y(L>zpxEpr;
zH8hT$@nW`Q%6*F}hIwXu=bqgCC%W#(OMT0Afdzja@SZ$cGxxIebspC{3QbQ_*6nzC
zu=GWcdfdY!r)P1v-MM9y7yW3j*B+<N>8cww6;AX?M1D-_TBZEVl~b&@{Cz={*Jg%Q
zu1eu8&L3V+{1AKWspa1Te9N7kdTNuTijTKVU!9%1xnG^}<o}|m$ispxryd8Wc*!$v
zcv~L)s;b(iU~B)^$1T4tE8d?M|M9`1;;;YX=KY~{tA0$8Dxb>sphNhsty6`HUBQx4
z{}8DYT3dG4eO!KiGjnWX%m2E+^8H7SA7rmnRut;|rOz)V_3H@3;}_-Ez5Hhx&V6qu
z+4ECA`qfv@O*&n<JKl5Ne-O~g9<ngu8T0*#Z_=BXv=vuYPIN!=!#|<qoKah$>DB8u
z-%AGzib`Z0xu(e8)8XKu^D}b-!=2yn4=<WO@6`XGN7Ie=#g$!BRBGT`vFyz87ti;b
z6{aR@^1fdBM_V;c^Rd^;Ft3e4c~d8?n&SI4NNVb*<x?z{_GqShX}Wst4D{-%(v<ef
z3^-MP>B%Gwq1}O9lZ$5mt5_OY(=_+1Pijp{{>;f*b3z`@;d&lov0O<&^jq)^|3q`>
zjl!wsp$e-WNh;~X?~qmcd;ERatdpt-V~l2atEt=SS9Njp_I*fjZE<$5*r24eK-o$C
z7dLCZ_@3AgMNezn9#2}_(eSL}LgzoHqqjx#!}r8|EIN9cSKoZkof9I`_wp)sKYZFY
zT{<o=WyPZv>^EOHE~^S~u{;sVzv*go%~yqF+Y^!ftHf`fPdK|RW@TE^xku-c&ObT7
z$E9VRj?E3;R;J>O6`VeA+RRz^^;~C?%4aaS_w9OWmdy>`_pgobolQ^d;Cj=3{k7!!
zOUCzRrSDNZUm-c~<0I>CCO-Ww#`j{=1C`i!Km1yJZ1yvqX;b6>J=cs$|2Q#Gq@qA;
zX7I^L^(+ZVD+PUfGqR2ZD=(5VyxOrUj3psy^}EByi&hK9_i|((iBLWjCEC!r)oqLE
zqP3x&Yj>@fbXTBsE$4wFamv4>E?)20@5W-5v_Y_SE$8o9?=~*c@l+{O+9=#P*W-;#
zlDg3TIT3G;B&iGS_EbqVS2LV(SD;hT=8>{Kw~CPU#2c!*o||N+rwUoD^EN2Gzx3U<
zE2h(SZJLD8>i9i-lQgPvYq)vAX7}%!H2JPTr!AMo|Ijt9Jf1rzHK{b6VqSET*{O@^
z&~av^t}4#OdWRGiX*k^Fyimq8Wg5d=RRyj&*LQyvH@ep*m3v{1q6gzo=9cFSXZRid
ze3%~*>^PsbR&1Kq5i6~<ORA<VQS8f3e!Dy&<h%jHZ|2r~uSX7l|HbdQ!_m^VT{<qi
zV(r7NZP%r`ROB>jFO)w1@A&iO{{Z$Kth?t&W!VQ5Ea-9)cCBo$ODIgu75~40agSoD
z`yHKb?UceuoxIkgf7g5D{+smrm-XL-N3+829PgSPUU*bXFZk%%l^2p1Ty~iekYL={
z%i<;&yW;D9wuLv;Z})1)Cne9%H4iut(RABGap{A1fp?6!icf|gwtJcv*}%GFHs^wc
z58V7??seYQ+%Huo#;V9=B;ulRXA0*}&PT#a)I%f>mrmQUOK5w@Bezo=J2{gs|5xZ(
z;Q!e#_MiBz@28bZ3;&2$l^!u?v{yHos@M1J!UNsF;^W>S&py^3_|B=Tyzk}ChWP$T
zDWRMi8Q1=I+X?SGZ^C_nPr#7#pMIi{rF~}H|KG<y<g@=5mNmJ<d$(aC@5fhjJ5#Q`
zI&HEsO)}J2H&!ypUaqtMjQhjp>HhM)`cl6Qqt`8Y7i2H@=GX<^-eV@hrwx39H{|T!
z^zGCh?dEklJ2u^%xB9mdFXz)Lp&W(y|GKT;bso=d`~U6#myVz7kBc9D_1{(cFhApV
zllr+k{BD2n?W|$cFTcsmzi(2-^5Q3R&*j|zSzMp;<*)n%y*b^FWS=~q_BGGk!oYjh
zjNMBnXFNTA^|IyX<cG|PbGAgiuK#n;GThx?UwMw*r21R4PE8igjJlAqF}psc|F^i@
z1ZbsKZ~f!;!TMzOxpT!n7-t-b@5)~$$iVt#*V1hVj5yyHY<CQ)Fx}P2d$sDN^P6Kf
zE_<vmua9mMY?U*(();>+vuP9WywhhUWlq$)lGpJ<$aTGT&`X`<nGYLg{Bzm<m~nGf
z`tcRBu08*tdg419-}Hlx_PY!kyb}LtnntMa{?-3zTfA{5+i8dH_t&0xG^Nz0tx%R&
zc(y#-!(8{<e6QvUmwoM7>&~s$-?wB|lljL#ld5(dwSL6)%jNUe=?AW=-*&w9p#ATx
zveO@~Ez&aXHa6#(6Op|m$nW)I$=Ms^<{f<T>(w%cDXY1Ax!<~W&aKqnd#yNmuSJAM
z_0ku7^XEMhQf^u9CGEe^P-wOs!y6~zT=T*Of24Q0uVI{27tpp*+Vr>EA~ge6S$EIZ
z`@YsX%qZo~Hb3;C|5nCSwW)pXeuw{G4?FNEpC!n=(c0`xqQUD++Cj4tO#D{dZiuzv
z;6D-6u63%gdLPT3dmpF231{Yh!5Gfspy1PZP-yDACaahEtWyenS|_M~VzR4MEU>j)
zbFI;=#LxCn;YEeg?=B1V{+7wSwwtwn{qKsI2j1<tr>*gA%Hny7LOU7bSKM#!zR~>7
zhw*~)^Q+(YIbVt7T+iIgrX!Uz`&`Sayzf#ICMvrw$Xg_JKwpsgV)})MZ@X_V-{tu~
zG-IDXQ+58=t<`%)w=zx<d!xQBU0Kvha(C#i<j35iw|lSMvpZfl<+t^rrc=qx8fNp3
z_T5T-!`CCS?T42Uf9X*ctJcg7N51G8_ov&xkliVLy4b3mKlXb?FSEkS+Ajf2+!bY7
z*Rm%|tK^*7K6!R?2iF!(i+<T3dmq1BsqNCnoTrf@QKod|<S9q{TKnGk)vH#x$3?2k
za<GN_-G8yq_^)fo*OUc34ik32YmRz6*Y8_;!q<{HyVg(C_7t9U@R!P$7shW^#>{-`
zCj8t<C}Ycyn#6#~yTwIX=ZWe3<W_dbPMV+ka0-`y`O<gSG=ow$mj6*`d9+a|nPWqg
zc*vWdSyOtK^q=3?_U{D8^jBLJ%WLbfmuy(S-g(1$zN4(AHqsfhSdSG&Eb6*F`)m5b
zu<TaX+rG<xwkF?MDCQ~b`8jZf;F5La*As)ezTbA4!Js>B_WOs{j_3Q_5(}2Ptl_mV
ztNk}KZdu;z_vzxt)edd)__J{Ls{Fw3Rc!bC0)Aa%mQ2ilJhAXz4nva2i^<Pd_X$=^
z;VWoXk$HEXkE2cgE2C9aS@p!`(>0n$r@j-};dAh;6=UuPkLHUmJH({_z1|hnFr)Ip
z#1oS@M|^N>I+V3z#jN`kpL*HX*7Hu2?~M8T{#@&?-w9879b8l|eSY&jlyNTqg_);x
zU(ay*7I8r4SM!aFkGt4zwK;ET&S|W!W6+s<zx0}FwA+2|A6Fc9R=3L@YS#$A@7@qP
z`Fvu<?1R^;|HboPeDSxW>3XW<Bj%O0Eb~k@ZE(9ExxJ@k&TA!ap&h?!_~w}{(5UF*
zt3K|VQ`M+^y6IM(?n}`lGq^0v_?qVJh-+_nGTCpr*JD))5y@TqcU}I_H>)$r-}#5Y
zpG}P?du0oMZT8x;ka_v^+j8MrA)5}Kk~q<)XX2%r)R3~Hxjb>t0S$rLYu6{fEG_Ne
z?i6BRIm*yfr4ZLxuHNvyOF_wu_r(N`c~0pDJ5tmC{Ecxrk+t=*#-l)i*~(wDXSp}E
z9A^nu<m1cTzhUv@|4y8x6EqLYHOyuZ-ao6cM3Px_&0_^E<<L1xy~^W{UpsMKXxk#;
z>25ztS3Nl}G5r4iynxMphvKUlC-o@!Y{@^cz**!2$CASZDFSnM&H36Y`C}KuasH!;
zyqU)f0=@os2+vG>&6fS@!?BY!{Xr)ia>SR-HCrWTEUe*LR#GByihq&Eee3l~ZR|Jw
z6pk_G^UpKpU=sMF67|h@O7+~1*+1TBckc3D(w!Z}P-diX-%7^9_HMR-1Dlqzn03(w
zOOAI=|L*qB4&3qT4c{RpF`q*hR&0|v@4PfCpj2nW*NHi&IX<s1EbLS5lasx|dS}k=
zTbj!QtqPXROzu~ln^4E;^D*0A*x4v1eBOg?v+@i~YyN&Jl=W1edBIg_qUV|3k^{{*
zsxD35bZoV@itKtR?>ifdSfjpHs&!4N&EH}gyRD;LBDUZE6|*}_#iO2YJ4-7jI5_n!
z;|jE>INGXs;{sEQe}j`w$fgrEDU1D*PT!dBvA<}3iD3FUkv)n_zwcf3WbFgBxy~XP
zI*<R!UkiPo{+^d#cSh4!-BmW9rn<DOUp%EyeO*V0qNx9o2<E3RPh9WMV7vb|;bmgw
zq%8+Sj7=9FW8Y$7aQQ-NMvvc4fr58d$!g2W+fJ@MJ7sI-d!ENEJ1Rp$nOf^sJ8xcK
z=Jt?Rc(EAY(=P>?WipN_OE*u^GZlWSn8<QT@0gau2^p6C^=C`nPCZ;N_as5<HqSY?
zRTrPX70LenF8y7dq1^%RfXwJ8LJf>O+&{Lvzw)y1_!F_P@#{=BIX(R;QY-h0#Knj#
z@M!9+3ZEpcbZXZD8OD2F3-`a)UNS$o<3?*bTl$}b51avfHkX%dEbCgHu=q$Udn4mp
z{tk^}@2q=y^`-0|RK0k@aiOE*xCRgZOQ$+1aX;QzJ{$ANUaqISj~sBBmuKN@@m_ED
z;p>9$w=uF;zcQ~2`SqvDp>ai;{^5cH&oBCmT;SI8QkY<#`G0D?NY@U%DUBvfSJk|V
zmqc8#SnT5an6-@A%$G}cL+Tc}Cw9}Df`n#hy8LLdFmJiSAiB_4y{Mn>)j7pq|GRng
ze4VZ-e#)z95(%Ai<Q%{1cQK*+eXf4Fn<hEdDAp-nb#<9>F*HbGU-NA4UqZ?}JiKhP
z-Bh=|ZeRIwh3niSeEV43c$^~JCIvL^5s;fajq#X=-x(8u1k?P+M|uZ$H6MDn$oH5)
zPRVq3V_OcsMUwpAGAo)=TcWrt1XU6@tm%`KbZm?7)n4RoJ&nb4l1W!+(ehJkOIOWm
zTYOpZh6l%Q<={6OlkGioZCsdk&6uyc?NyU;qEQO#^s}7nY84_Eong4s9K1>3Vn=Z+
zzs^4G9}S+u8GM(1&--ve@v~KBz=W5oT=k!m7I;qCFI(bJbV6~limIZLyBasU+lQ{r
zin;O~7i{uvJ{P`?ak@BJj&*{8ec}5Vs{6j>MQnT0;47@Rvun1q)`d0PGiPoxD(f^{
zJ$JMG1Z4#lvA9oyUl;J6PB_-kF*ReBV2QPa_3HN~x|eis&pcsJVo@4n@->*_&`sV+
zXD9hpvgd_w{nh6-uPbq7uQRWuMfvhwk~iyDad5n|&<{?0xHwu@Q020!T<rNaJyph8
zJ!k&*`}7)2RFd7ItZJv-Th@A~Z2@cGl8~+$512RIP~4^8EO6vw-94#8C05teHPj<F
zr7o0XfBcg1u9ywu7sDX6SI7Bh#AY@+NX_V(?wYmd1LKeHrW!q$W^Q|M#AF%Y=INam
zT=Z1Db~lJhr0;d8I61-5I@vnn<!VE_E%rf+OILH0XUb<CcY5FcW35$;N@VYE+0OU8
zMLo{D{1!L}Jo1R-y79eRY3}1$okAWLU+p}}_l$9_yOzSt9g}ONHD$joQ;uZ4+FCmA
z)b_&%f)&-jZg}R#Uvq!!de@Rb)r8Mh?^~z7R$x95_U&Lnh?c+^Ma~e7`K)`Grp!J#
z=kJG<jNs1de@d@eS?2aNOm>`qk@Lzd<+rTvi=Ny)t?L`s$B?O5`bfBDoq%kD(_)wF
zTZ@jTPMLB;=;_uD_s7g(%m3*b>VD`DTXeg7|0FF>Id>P6t!upucTDg%`BWhOd6|}K
zbxFP4zB#X5^1iipTKa4*J$|>(GX9W4FU#@h<#s1`vApw84qY_w?UbFe*4GRdUiu~I
z@Mr}q1B2Ql)jjXLS5Dc_o%`Qq{i5RD!!w`u8gMBX<eZY4l)%cS8Tq~>=fblGjEyIE
zyMHpf`E=KsG~Xa2T>+gRB_git$`nN9K6~{p4H8|?aXMe*a)QR>S$@(H$Lc5Zr9TvY
zl)U}v;XN@2LLL?yo#LN6Q|;D8t_;~FX&Z{Wtd32-aQ4mLmex}&R>FxB6Sxz(^<<W3
z2&AyQ`qUoW_acX<UEooO%NoD$X^xW)c$!Py;?s1S@TF?)u~Q!TGw!aq`zb_ZRk@^6
z`Lrv^B5v^|wG(`I#v~V6muh)`6It?&XSH7648>=>?1!GJXT>yakY}*oVfsP!s?#y)
z2oYH!w`p4!I_baNH~G`FM8l0y5em**H%o=DREl9R%nZ%3lCpWp8@J7Aleoi?xD9(`
zHu%ewXge4vlryeh{8V(C$h;pO`?_wW91vT_HEoKWS*m@&9T%_H4@-(eG+W!(gt8YE
zX0lHIxqI^tg@v}f$4*y#xAR-S>)Q67GkZIP9V(X?uDQ+Ap1YxW`NRW<TUA<(CWpuz
z-|>BG;F3j*zEy7h!i&rN<23^&9`tTlGl^00x#Zgvvo{qpU&OWu%~+P=G{a6OVBhgf
z->G>VFCMR)B<3mHS>kG85h%Zmo7?e`xQv5edCz5L@BSe5y4tmmH@HM(NySvmy{>S*
zt?h%GI4djH;;sK4l=_(@c^;bf@NniL<4vNHs|%Rdb6<_!Wo~q}!(hr{#|yi8n6#cJ
zvFc|j2Tg9dx|}06wKSQJ(ayYSmRfL(tK3t`r9m!MzG^`Uj@nzeMIGAj$dr9&i4d)t
z&=9QB+?dV6{Witn+X<aSQH575-s`f2XR}u>5x*?C;JHt?!t=#T#Y>-9E3P=aGF0Ph
zLbcY?=uA$%_RZQ?*)#3`DzIJ>coe{LkoB|HT`r|5FP7K^dl^O*`ApI|to+c$=w-Bl
zKVNmUef5Kn7A7J6$s%W$E!;VAhKq^a`4*iO0z2aj#2ap&m>}TGr*C!d!TKdj&!^6w
zy27io|INMSZ|;^n;j%aW!Su*jFZ><b=~E8A9tYxYU+`ZLIQR9;PWJO^GM88{Nj<yG
zvE;^!smB>TOJt&Cs&W<|7Zl$(!RV9Co(!puNdf|EJbc#_+<tgt!4yH)pTD;_$R?(*
z_!6#RTd-rXbY-C$=fk7fwl0?@lzx9zwK(U+RHOLjbOu3jlNqc+qH()9R^Qa_j5KZh
z`ylgd$O(m&si)NLwn*-1a<I32SNet{rtc`D>U-Y@jSf~)XZa>7S-(;7WL5k9%RR*T
z)Gr>U8#{wuA6at9Fnp(Tz}L+8##SDib&eic;P!~wY|hGd_ozEoh3<y8yy{%ce|<P0
z_g2@c_3R{p$6HDcv(I}H=HI?Xc!Kw1#pFrG4~4fF2S_bfyS>8g!I@Lv=cTx3TyZ$%
zsQk-<-E3{f0kw+tN4}S|TCSfX!f&GzwLpYXH8HbL_Kb<qf&AtZclFntW>8?b{ByTw
z<%vZ(4+Qk>TV<n4911q9GI`t4xc%<MPnTafYdu=iVra>`hux#Z)gi<-Qdcb0@n@6Z
ze_3_qfL0Y*?vh2UD<-jKRlB70J1=k(?A0<3dVNK<uIx?Tve=x8B^!Ny7&h_0yHpV*
z&FQk(eCb3JbE!L0m3&T1UhKIf7Q973Wu@B1G`6M*o^P3^wuDcT-sCkqsK@H9-_%d3
zYN9gJb~Zc8OgOV~>j4cz@t@*+!p%!dTS}x(tL|(wR0<Lk-dWh`mwsYW`C5ra1?G*a
z$?U(rZ_W{SW>&Y)`;@+W&v(mt5v`Bi#2nW1?Afu&SZ%r=m+Zp-?cZKTr4%o(zmt(}
zqwwld=hKn~mJ83Pw7*`Vt#849-0)2O)anUOf2~Wo5~8E<n!(Mqi+Q$3*i)f|-wLnK
zJSf=|`dNq5w!lwwwTx1qTRZQ$AnU2km-(fa>{YR3nzFIIpf=#u5)O7NSuGZkj!Ic^
z{uz%Im$67mN?jFu=2INnnzH0YLA#c4!<{G2)`hMvs%*xN*>4|yaoV7v<})QIW*<kS
zFt5dl@@Ae>ji(;Gl$iQ;lYZ}=$h;X}mU%8d{jI>pF@-aseQw2tjctz|{-?=(i!frV
zQdIdaShhTK)xzfrXWU;%>(1`#d!u9DCU}`?R?|w=s0t>N_0w6-`>fvhdc&pUb`7PQ
z^J8@Gb?+&hdF+V?pU%ws2m5p0oAfl9n%$D+y8EX6#rBjpZx$&|U8g-;W`W|lN#e5E
z;fd-Cudq}v^ZwJjX@@sUSytIO2CL=W+Fc(eR=NakW7}>2Hu3H(u3PIA^rhsR*OVJL
zvKg*F`doYaSwkBa!PH~BlNSa)_7wW{;t1Cf7b)XaOtX}}`b6qa_^Pem)YRZJh4Vuw
ztCUT9?F1)R)uWZ6Ngcf!F_XL`rM1^On`kPv={Cn6_@=RZn+(gOT1U0Dfqq4H5<3~h
z6jGOU*vM`1xS;CvcN4Q(u(E!0?I9M%e6?8N;|}*VbTu;66*qe><_XenP&%yU+%2V1
z?R1?rReqNXtKQuwO3Sp)U4AAa;hVR!tw^xe%e6T3#^EKhwi%wasa%0_DYA=f-Tk_h
z@}7Cl+Oom5XyT(vW|zp0xk?Ln&dXgO7j$vLzJn)T8u<8!_6fHJX}ff_F+FtRRJZH-
zp%%fKSmQCFERmJ5&@ZG@A;ojmwB@IJ?RKAgz}3&(wru)}OWZM@_H!P^J`OzM`;_s#
z_?f&rqH`K7Qp`0cEIDA>z1(}|G;=P)^?j368^pIPa53>v5$P>bnB&oFXxk;Uf=6d1
zPjkWqrOe7y9rxBRv6)QGWzrH{mv(9=X{6L;h=@qJXuU`ZnaQlWG)m&egp^wk*D2eq
zb3Www<43VjLWZ$f@QMY7s}_gsj8<)nvfXRZ;*&DN_=t3t-d+>IN0x`8|CQ}Jr*X}Y
z&H0{fg?_h*gw|61%Yx$1RM?tI%J|N?Zhq~k9h>xB@XSNjEB|LzFS2ocwU%Syx%J#%
zm0SfcEOBYPnb5}gM(S+*;k_!!t`FACX<XLJXnCcRpNZj^WYiO_ziTsBq?I0-QF=ah
zDenUfu79s8q&9e$%Ej=w-seiHQDzc-DLU=5$^j-OZIQrvg8zG>)@>0m-K6EQzfo^{
z^FDW$s!Lj8{SQlCddVwWrmj%<YaX?JQI3;A^X07Y32qKtFS+ig=v+UzORS`ArL?Dk
z-Yt)p=HBm`SL>bnF7Y_CRPFQB_p+R#Pq(aSf7ku`g5<S{8j9{+;Y-39H~IU!<mNAB
z4&hin^;Sr{X7keD7QIhx4{&hgu$39i-#B~bi6;vT!>^@SggVvoXF3#WmAona$1ssS
zS^UO<g5NGF3Qd|moAljMbXjllE(tub{%O~WI>&w1C2A9e``57lba`;7QF~{@vJ0#W
zeByQ}oQg;amO1JecX4Uv^59h`jY2e}G%qdHG(RkQTHwSUYu6>6XGNB|7Al;wh?d+W
zDXAFo^2(Y659=Ov-3Stz+}>>E%+szmZ&FV68|^28s*83xi0=H<JIP7fP2SzmOHKJg
z@KfbHkFco{sef7KpKFdbJ=WWBbXSGu;dhDk)4%TY+!O3GU1G|~XJ5R!JPNK%IQ69M
z%#Ig9k%=3;7T>xd;=jB*h;ONj440e4$LEeVrUx|A9QhJXUE8u*hO0@@&9kg;Ntxbf
z!9&wh0}f0zIKi}V?xO|$(|%gdT^Jmb>%o=RBF-f6l8wK`sxiKQqIdHReuHIa<`zW0
z>QS3AeT!4e>A+gUnQP}L?m5hE`-!9J4p)!X<QI||=O4dvZCq?}Ans95%YyQY{AtNk
zx@SCod13-*;EYQ%m5P*DW-IX?T5({%)5LAgzM7Lt<q9_iy^yNuRx<s#F4%n6zb7kK
z3i)T0p88<!Qf2l~!nS1FuJ}Tou#d`9@1`5&xgG!f=KExUg{NI3gPy8Q600*{_VT-}
zwY-?Kt1BVnyXJZsf$+IzALXXASx=bavWat%bGzAMzb9Vo+c|``N`{6zh6gXrIP_wU
zlmB_{hv~Oc(;sl~O?GO0@<h+&zG8_mXF;IAq5TzAuHnjS3mR1ZEi+-WS#tSZKx+7`
zuU#%GhW{KEPvTeSPF^XoD{P8d`3Yr_g-5n8(b)A^_b}7rmzkW5p#qbSfBQ0JdYALH
z6^aWpR(XEn+_ETaCdaBZwp(896Jf~Xn)dxzfL>+PuNRBY&1#C5{2Fxfini9)kbtxI
ziW^HD1BE75c$z6a`z_P6Qspd9OIK0-jsuT%3+LaE-}%QW*@1ukwDfHTM-<W*x*g9=
z<`bU#>O<7_=FdyEUFlc;bD|=6<BqL;|1VCKnYvSOXN!YzZ0hojysYmm{7&ENs9u@0
zeBA--HmR_=`vo})PIMSEGso11K0CN@=SjPZkCj)3gmS4}U7F<j(`iM&t6Q>}6k}!u
zpUK6hHH<qmrL{}eNnQ<O`kCr<(uI5e)rC&duRj~vH3~3D@V5mTOxY96D7b-dbBsjY
zfn>pyi<&7L&d3&}9aPkuv?^-C3b$*_X3q=^8;u;<%zp85vA$MWVYPMkv3HF-0=BMV
zFg;Th=@qc>!cp6-lA{OIvmXijsMtuHeci+V?x*U9vj^9*R7z)QaQ!ga)to4~rA(2j
z*-hB0Q}wvnAv^V#f+mVaC8ZXZIU1BSZT)wIT#?h<A`o(vr|)jA$<~W2?_V|BknEMF
zJcB`Xs^8*e2UmN1X1u}MlcD!H!Rnr})l9`BGhewbo7_C7Gj2w6T=Wkm%{kJnS5>%u
zCI@N;YAp$!A*o@h@v~7Ya=HGg$ki@e)RY|(mVBM5_GrrLyAhlHDwnL-CvhU4`|Xao
zS`D2W^rz39xaBMBxg4YImRzo)VoUQ13Pr<iNINFY+_<80Pe%S+`+(ICrQ{-ROfi#M
zwvlUNexT+87Z-`t6<ZC{{N4Wt)HUC^eD{R+TE^>7ExXp8zQ|p=f<>};u2D3P%9C3x
ziy3teF-I;<IkoTb(U7!d2J$S@nu}wX>}%MdEwTQEd7AmeiFL|uXIF^x{d!S!MM<a6
zi%ph6@A{XCsgG1<Su0<S$#`OvqP*1DAz$anGA`$;r4Jc38@`A;^vkVO51VPq5hk4Z
z%J)-hxsjyA<?|szx2#f=L^l>#`AN#IRp404;XHAP^PGtVF&Z<1WxY}sJl@@r(QGom
z@$p2tIbO|~!dLADw*1OU`W$%Asdeh*mWe4UZ?5G=Da~1stniZi*_s0`oBTEiR{Hra
zYI47(bx&d2)PRNiv?nR{JD4|ZvtJYVGJAzwdeOG|%l4nUy!5`<t|KD9xf^+2@`zMw
ze_iIYI{Jy*g|mHC&RknEBegCT<h|+$+nMWODtY&-(k8)mXC#$>e-eDI@qO8(Xvt#d
z^Y61O4rTE1cU3R>7xK#B@s-IprWCd*9kot2Xy{+?ONn!}%gt>X^-cNn{C+OUQ9IJ4
z`C-kBHR79it}DG+@NvDXD@Vaoeb48e$Glup7wFiZnKGwt!PCw~?uk2G(oB*jJH%up
zg;*SO)IBe#$+3OM6yA`D-B-RHEK9BIv=Zu+DocBk(){ik2gl1V0Z#5yP99uy&x@N!
zDY7DL>eY2Kn*G*IyjwQmp~|`$j!ZU-6#63MbJzH=rj{r@>RD(n&l9+KCco$R*4IWm
zBYS;fZ_X?zJn($xiV4Yo8>JYo*m)(n%bq%+dDHBcO|8m1EoTF{md*U9SSy)!F5__t
z+$;1$t@`3yFUtjntjmmr(t<89`=_t`JySKX)xW@ucaz@fcG=1KAO10<mrUwrHS*~0
zalWKyW0*ee$HA3lTW6#weZKgS=XY?1|1&|I{M3skH(C1DTvk7<`Q_llg0rDFl#^}!
zS6zDF8+Mf^Qa@OC3hN}U-aXIw<oB6%u56yN&bB9@)vKvyg;4%P7n4higZGOpWS255
zz8!IPQNV&z0)h``XH4BFdL_=m@P&F);U(p(yIt7pgpA)GidQx-?Xved+x#W<?u6il
zyI1izGzal&ZV${{9;$YFAFpWVHIt|MjZ6~^)PAx_eKmZ<ewZ!bu;2%uB@9ia3X79|
z+~H=s;8ZB}=*5FbkJG1&q^#z-e$?zeDm+KW(#Ybi$>GvfPmdKbUD?LGu&?K9T((S*
zey?58+zWD(SF#&v2l6#}<=3T7E$Vo;<n*VLE9UEjJ1vNlT(Q>WhH{*z+vTn;hq%r=
zEzR9-_+!z;<0k9W42%}!_Z;ft_m~j*@2-}c#_w5yX=SSW_9Z-f{V|KZEOEsnz0E2?
z6My?JGF@bu-zE~0Q?aCVkw?;m^jVUIGnvAqLiIHCz8x)*WSCm05#av7&30zBVrXOj
ziomUl-*Nt$d-B_=o_6O=tD+t|MxAy!X)q&dp@nmPeD!G!c~{q2ol8^q%&2ga;QA8u
zP?=xE++necrqNmL)?F!+%>PORsjiSIjViyWbL)1)+ryzEjS?HCiwa!GGnl^hf54qx
zm!o*UcZEE-{8aUC-@l{cwr9F7rCxAducZ1{cjb9y?~gn_RvIeRro2|?ZUoLW;(p7t
zZo78w2Q9ga=CZBEF_n9D%AFpGvGy-IYPOSs>uLRp69ExCli3(^cz!jmaarSXL!*;P
zb*o0U@3i$QMk_SDwuCH8N@ii|&24?AC$_ZF|Fx}AOD4<ul7_}K(KBWR|L6QW)8M<X
zv140;bf#d-`bqK=ZykDDn4(gd5+|f>+WSIwrdj8a!?#M<3MO3K{6cI?HP1bjW6nMe
zA9>r9JH>kiO)OSu&1=+}KJ^#F;-Zk`DN|Rk*lw^p)>N!PbV^6!C+1a}iFa4JO;T^p
z2?>+12zjrp8MAA(*&?@d0X1&QY%|shCEtw^N?NECn4p-FIM=6m1<Qqft2fzy?F^Z9
z+Cf2RuG0sHA0N&cH`{o!WiMxK|GJ)|`j|1x9rmJWfl020A6kX;Q}1#Z*eVt;o%AwH
z=%Uu^OpVLYV!O0Xu&i^MzU(6RlQjVWEB|Kf7hrSQT^R6^OQL5<YV_0u!y2x_iL+<?
zOXmokB00Ng@tHqG*Y|hX6a-G&Vv%DLQjmFMLHks@=}aG-I+9k+zBWDeYxsRvq1jg$
zyyQ$Tu8dM%X@2%WXXEDHV=UnlyeFC;-1J0l&D)~^yPUt-?aEuYXUX@OA#a}gE{f~u
zvtaMoT-;^v*wxhYrv1-bWxI)H>dEU&CM|JWVf1*;xh2o^*2{+ck6gUQK<Al&XHn!y
z=02aP&929~51sp@=;rZy>gBDMme~smi~o{Znx8x)yLF>>SKQ?dk2ftm^gvMc3j3Y2
zY>#|QAM~V28^*DpGMaHHDBz6C#C)FQWJSkQ=10WDHZG`f*tv18mW;;{$GtvZQyNYj
zUUW?JYed4#DZ;UwU+NTnyv$gwoLu-t5}znt_)=y%r~1S)_Ve-D%gYvPZ{}LM_E5Bj
zmPTx##qx~}9iEFVQW&juANQD>-?{EQN!;W2>Wb}2Ng1KR=kzxioQhTYJUO#j(`1Iw
z!~@SOasyS>PNeyy$+V=nd^@sNF**I5lF+*l?M;`aFwIbj`q!Jv;5ut+n&zcRb5`!x
zv1FLp=QpKlR{Mn6Ty^i>@Sm8zLi+cLJFbk&-%m=i%c#EQY4mr0pV5RBCvBL&1eCER
z`75}j%*+e?k!F;&X@;1Iv|_d5@v}iG$0W_F9%kAUn)C@>S30$~zvqjdLXV7>M74>|
z7Z0PqB|LGJ9EsAx_Pawn-JX7*`LOT=e__CT^@ERErd0458RmHZ4K*@aest4^KNBJy
z{c;v_@qC`}RM6#%V{L!Wk%#Ju7rQw+-@ESKl*D1hXI2<|%aO0uzsp@YZX1KQ1oPSE
z*{X9UA2H4k*d&|Ruvq8t;nOcRFfUR%Z2oWU(XjvgB@03rr4kotyS855Ryc2F2HUZi
z0?qKova=Zy0jXhPrb5ow9F^ZT8K_o&<`8jR!&aZ9C*;M-==mc#(P68|j#F#yHyhh%
zEHHh=9Q)^(`^1!=+CSQu-o>m;netmPe~WW?<3_L8#w*bY=kyI2MA|$>IJJ7srW>7&
z(7UU|``Xy)w%=*qxG9YtqEeB?hi+c_X?5*N24nKPrCy&8EI9Dd`rt9?stvqKWlBa}
zGs9&h(*zXe1>S$oEm+I`f#ubKiIMp#S9WaU_{y5#Bx}gU^Xl+iOZHdOuQ>m&HdLP3
z>X#+AvTo{|Xvy87fdwVq2h@T?C0kn)BRZL%&g7dli|^ScIUk>7JvMEx2k!h@yAHFT
z_CLUU*y5(0s({6_dkok9=sO?ho4RJ<zI)7v&dbCx^S_dxu~@V%maE9mm23Yi#l{~=
z4IcAyOI9d{w4@!+TJG<>(reRl*UCrC*+nizZesfy1CK9g4T;+JtGW1vPTxYdDIaHg
zruFC>g%vQd98K?aST<ciKWy8|oXdW#>w=_L_sm??!^XEoI&tEhRmIQbN`KGSTiW==
z$IN@TNJu{eZ%b>b(%P<!DGH1t%W@qTYj!@J9MQ^}AO5W>rR?1CiRWGfojO{w^Yh~q
zc1!w1zACjcSQWWl*wZC-%Eb5YUa#E+92pfG_N)DDY!Yp+V^o%W7d^E@?R7<xkMB}J
zuKp4=E9=~Z+Pf3Bgih#Lq<z(dE6e1sfw5Hl#v5YX<}T9;HU*y3VQ|hovADO%Q=#>n
z%?x$z#VT4$<Rs^=?%D3KUvBGm?WY&KxR<KvpYiFpx#G$4<B~bs7p{=0FCEsF1n$hS
z?VQ(sYD%r!W-VEs=i!|NJa1N>@_19^b2QgYbh^Yzi?E0FD^?VQ7ZwJ$DsO)D?{!{m
z)7x34Ob)A`W;Oq1o^ka_-DUqxrN;L<{_Z`eAsi|-soW<w#7wm>qV-=wpu~!WKTa4*
zTyH%y*&{l1iLCyo18$eBUWB~r%UyM<_=D>@&JM9gc@3oy;mPhFgm1SOh28G#X-!uX
zSTs2^Y26y#fQd&=Ypmqj?&KPHrR$LU8L<^Rb}ZR{fr*>@<3pZLx|u2GADWA)eK!c=
zOo$JRint^_(f-)xj2I=!#XVjAQ(qhHyBHzpuy@-T|DM;+O(rmSS1gEG7MJkSUF^%V
zEt2xDeKs$-6ZCab@H784ANzP@#dN%u+;F#i-IF47_)JQC%}KR~XI`)55n4B;#qq;W
zO|7lQ>bpXJxGk0J2xc@($=h#}GgF-_&3oOe0NtfE(?U-9q?}q3F!RNNt!YzgukfzX
zsEWR~#`2hhq19}Oy*&Obck|aZq_wSHlzX6FSUWj&sae>)>0iH|`1n8h{n<?_#|v{W
z)=PKSawORQ@7$~Kjm`Sx9b2Q#t3J&W*IpHrCsf^}P&FZImB9Nj4n2ue+h-|;v^Kh}
zPq{A7bIXT?KWy^Vi_>CrS$`}|kepGnn?FE=|4g57qwI<TuA|E%1ocnOS)sLKS=Px3
z2@B%{@34hA|Gl{2(Cceku7>!Du6oa<Y}CEYNlb!QtY&q=+HKrSr(SbSjc(m?;8vTv
zLR;!>_NuE|mtq4R7%VXH-*Lxlq0uJ$Gtaf13PQEQC3JQj%2<9$ZSm_4-~O%-U(<X-
zG6Gn83pJ+v58u3a-Lw-c{$J-xcQvU$;U0JOalXQ<is{edOcpoHHT99xe$si3H`qER
zp!D@qx$-Nfp}{{TpPsbXx2F9WYp#~d@l&rRTC#D;g{degg*QCnJ@ny}4Da-vduQ&f
zkp7l_G|73NfRfS8>1>Bhj=ec#?HHoyn6gmM-jK2SvHKRkhr1i5^J_>k#hqg)n%n0q
zp^#+PeY<gXM8iyjSrb2=t=YOugj3Le^UJ5347EBc8QTjy9A;lV_CHQ8<4e<>>%SlA
z|2Xl&gPVa-RC$Spk>G_1I?6KwdJkKQA9~7mXk+6<L+y**ax*FvUOMb=db3cuM1rY$
zN7Kc#-ij>mjoa2HyUToeXn41aRk6W7-XPc|LtWVAPng}}SFVOCQ_^@hG71W>XjDmA
z*T}Xo-`IRMlPy!S+YD>lDgidG-X9$wjMM%tD%<y#ku_i~&$aW0kCS(|iK@(16m4KU
zZ&@9)<&Sf3#m??7Uctu1kE{M~pP8(EX#L&?g=$}{H}iM&vM@$F^_V4Rz4bkwQQ-gX
zzi-}+8w*}%1@O%DVB0S+{hKOBUTU8%qg0aCz6Dzib|+-?WSk9W)(Ueuyz7YV;~X=i
zMR%Kwat+pI88~J0a9$Bue9|rTP_0Cb!=;S2<gEXxJ1lQRq;c0ba70-eO^G|aHC-jl
zm}^6L2FI0}nSRoN9IS6|X}QR7ZRc(lVkkVYkTrjUbLIt`1(McmCL0^KM>fYhEZ)6C
zTXVtLY=z4<Otmf#)C%IhHFRBbQgxd%v12mx%_X)Mq{0rSXn#n{`ym}9E2ZQRoH|SM
z#qu*ECXP?%KCI54?_FkJ!`*sSVA@p=jys3<GL=?-5olT?r&#BrvD1D3L#aRwoo)u^
zxrfx|JzL>XkYE%$VHU62b+yThR6W+^`6uQXcEo5-n0ZJoYUX<7PmIo)q37=Fsk^o$
zsc;_rZ?f}w`L?Ns8#nxQ;H_x8#lEAR?ZmpZQcoVmPP^h`a?e4`OoL_i*-epA?f>Gn
zLj@i(Zo4R{)-V2Q@uSvn>kr-D;^^7e$oc5UVb$(_k^3ccvNk+fxNWWD@s6&Wi-V&s
znoY7yT~Hx4yLnE|0xw+w)@H^cnYDcTC-YQu`mDIYHMP9$wnkO3OGDc_!$qY$r#}ao
zSp+XoKF1lg>EHb(wX4CQqTM1<Ii~erC#}uc#y{b)y7sKVvu%?WK5{QpYq2&oTHW|#
z!|d4?Z%NmvOU2m6ybf0Qwaj(C;(oU3!iWoov3$X`CKB(WbldJaZHTz7+_5*7GooPn
zfuKzX8#WmV_y2XtSln?nRY!owDPw1V@jlisE4JL<v1UTh$?lN&ZK@qRp6+Hmc3_v!
z9WL36Q+A!(JUQM?%+RSr^RjRCN&^<--~|cw$_s;AviIM-@0OyxbHO{4m63jX3L5L8
zQ=he{ok_p6z}lqP&c#XGM4u<PiGxk(r^Y<jF2gBnl$yEScg?$GAT@c#(X&flCS?2V
zHBe|#ecig1?d@F&W(mzmsg^U}GL)MmTkndCFL(Ukz9IFj(yj01$Lbzy#|Vj5C0MLE
zn7L(&pvR3RM|oH-C9KLy+vg>&-?*5)?x6Es=k}IIftw{Ag}xm;ntR5_oX^}crA2ht
z?qqYG7>R_niUmT!M<QqT7+vN$xoB;r3TsQx8#&f5Itwm0JBhzfRL+p7zj!A;!$xzB
zxAyXdUhbzk{?{I?I6trE{1H~8Kj&}sRaLA!*3~PT^{`yt^0-thuYr!~noDQg?(=Zo
z$Zss(wrOfZSSD}o3jVc~t68+#j<l@QdBp5cU45`<MR=R&+~u6d)^q3<9m)z=KIen(
z#$dO(+5z0gYRN~vXFU^rVC-Law5wMutMdtqfeq(B5j*WIEpNhY67<ikWtnx0JBMlB
zRR8+^g9kifc9@GP^a(sV*R)Tmt8Ig(-JyFkI0czqJ5TOEbYb>G4T-r;Ozvv-ng%|n
zf21<X>PPW3T-rTjsmJ}xQ?A#4Hs(6~$YbIC)J5DjE8D+`Fv(f<THZaPx`4xSWsL07
zpDMiGdIo$I=lkw-u5vqf`|M7M#ojY>los1o%-3!X(a0&(UmCc<DWpws@56-T{sv>o
zgyVt-j;UV|5G>oVo+nk{?dKw?)L+c|xn^Z*I5FQ7U&_?mR#*3rd5u=<$M*#*V*m2~
zHjJ{aW)(Xz(|^`}gVwDEPK&B6`)yXA*`_=1TVcq8g^Mor^mH#QNeXMTZ@%%1>yFO$
z93GZZm!obho)a1vzb$0tFTT68M|b1)qjElcAqB4!GX&M;ihc^lb6m<=SSrwz@X@^c
zaJAN@Bn7Acd7b<@@k$3xE(KT=D}A)x!){~DWSpQFqq|q>tVrYisZZ}0Tt4=*VP?~&
zwG6LY#ShPYWa7U0!&a*wo!sU{FO}Z5J)Bc=@ymB3XRGX4Q!V~81S&IaFesb4^=N9h
zfqKF+r`DU{2SR2%jPb}jkl$qXaPQ+bFNb!6<z|v%xkguVPkfkplC^bN>iV8*8b1~>
zx;rmyQ<Gv}koW!Usz+%*^VXGpG*`7c)b*-uzFfs|X#=^S9on4%b1I&zK6>sV&&l$+
zB-)@tx!_o5aF0wvs(nbg*4^xdE0WcWRN^<jyfH~rd-Z$A0|^%Qn-p8Rr`9@uj6QrR
z%xvzZ6Qa#J{NIF2jrS*6?9etkpiwNodFGdR6+_LXDL?prceTsMh4ubh66*gf<HglR
zfp^~;cW80C^CxeNvhb5%FTeN1ucdq41+H|feGE^NIL}iZ`}<tzLn{sT6Ve9z8JBQL
zn_iHLG1;?M=TE#l>o(3-6KBRt%XT*!R3<d84c=bl@UUllTOyZHUd#vK{R?70->^<?
zSr_r<?VFQvje9dnYIoN6+;$XOSMo46lA~cu_$D@^U5C3wq-riFU7nY-W5eP@Y%{8Q
zf7)~N+czu3x4L&J^4<--rK`B!$@?x-GN(y_<L<9*2Y=q)JR`27u;8jWa{^1mhgsZb
zSf7donJRH`ay33%)VN`%{(>zAdze02tlh1bq{pvc->7+L@_Y9MVkR94et(^xM(j;G
zoYAPGHPQd6P?*w8FXs2BH-9te>;Id^%qYs-lg)j_-mE`Ot@H29j}z-&^Yl1pJM~{>
zaQ19`sjYXwjjh4t&sBvr`7F9>;+ZP@?kqj9nQwNiVz2zqM#UbTic?2Vub=4ofx(V9
zDz;|Dp978BzZhL_cL=W3-T0~Xukkcxv6T}qB%EVh{et&F?USR2{wo*r#5B9tICi8w
zsWPrD{&1S(*)PFgf5kVkDV4Rw7}&JxuTYp_o;2;2`FmE~$eyfct!a8JEP9WO7si*G
zTStj#BwHIa{cyd1oqL0ZIIo@loj*-4I1TncGu~XU!o1tzDMu3PjLX(Kt2YJqyjt9s
zW4drs;L5udGcRscezQ@atZS#kiS8d#U9t(>SD(#vwy|FP>Gz4-m)krwlENSS_uW&c
z;C;I2;Om8ZPi0L!>vnF<L*u!XEfbYk!n<v6hCkb}Dm<n6=_7_Io5k&Nr@mP6y0M^T
z%~TFq%@<QHN;l}vZh!h9tMXaVAM=mWZHFc-<jC`~j@f8n7RIF}#u%z>vR3kq@aOvd
z5(?WlzIN$T*mUsFKc(IJDf=?^HfCS3d%!x)LGnSI<oceA3#;wlznpsixNq~6FK!0H
z8`GcqN81@?i8OHrNq&zwc6I+FQ-+!QLU@ige1CD-XM#!rm$I<*&8~l264<`zPd=dR
z*6*x%TahP0|J)6ZpZtkg+za>i{{43ISIxaP7YFs3`TJdd*dEv){^0&S`GD!_HJc}Q
ze3`U2$<12QmHXbMt8=T$64eb}_S$ew>=XXR^mOCJiSOnZtUM^Z`0yi_Mg1=d)s-6U
zh4#Cw<NM2`ddax`PNNIk`HpuJJ63qTkW_2Uzq9OHz@4>RhmEC+oNjPB_6a!d)A!8Z
z^)0<-aj5m>3}t2itqPGDwx1<Gid<o8YUE(wZz;w1fHnA=%Nd77@g8@gYjpdjIA47r
zoWL9rrYu(J^4r+q9k0r)4Yz;3^)wF@+s!+d_c8ZD(F1XxPFVB1h+GKZIHxwx`0}4S
zKW{g`+N_;Z#`ZqR|4&=YH8+hp{0`img)?m~%xgS<>eDH<7YBOTYF5pheV;d3Cu+;z
z>95!4FL6DnzKlI6<AS8wv4))rGBK6LQgO$;AA9W5;PYx~>SsM0AQ&HTrFnWxXKdev
z$cMIf!aS@KGXA_fxvRPVqV$Ioo(H)%F!Du696iP1Q~jy>8=Fz}vY?O`D|?;89kg#w
zexI3^K4q)vflTe^OU^iT1V?tX*={aWY%o&WSs~`XP&`EcPupds)y6ZY8|_+CwIx!d
znltp~xBCs#8~2OwPf0jc>?U&b<hp#fI<FOh0s9|BGfuzqVfCUVuIa7<Oo;|3ADX|e
z(VqL-?VaID8+EZA4WfVkKRN1P7!o8kbzjKWw~7I+ldC#+`&aFG;_<=b&BT+UEtb=J
zXKznWVV>$-T3-3;;MCxNJ^|?$jUKjQfnR@Yo9aGsjzj!ur#6xGzilRz@7b$((P(Ex
zd3NieD-K)sO8#ko`B7xQ5Q_<?YCNl`be0dNB$s(*e}#Zt`jL5SdMhGR#BErtif1T_
zD&PKU(8VhHQ2$2FNhuZMZ;h<l(yQJ~6x=!Glj@!CCU4zUHgx-k1k5jebV7uIafgAw
zLE?#&7ak%r54(5RKd`EfN@(V8NU1BW*ypCEzHHiN;j<M*6_2c%tZyD=SR$UhJ;vqa
zw?<K|gWE*Kj+F1<h%d?Bt+0Osn^08KWYIevER$K*-rRd|lC2U;2~T-qBJWf|&%__@
ztoD=Ts~xfn!(&+Ph@Fy~a*L<i{_K6p^L8)mUY$M~7+@43^?GycM~iFoRy$n~RP$_#
zUU_(f`mOh#*L|EmyPZ?EDL81%Ka<@#{eilhd#xdBo|~dt!}dVqRAr+$4U=<RzDtiP
z?f0;sVJy~}vW|V;pFHO<t5{9hJECEG=EaLdKL2<yL$EvIbl23U77U3;A}sD`PgR-s
z)MKi^p#srwF0NmnD!xuFn^@6ksoB0xW|t9z?j`FB67eg)Y+!TnoVZ}un~87tpQ$_+
z63`{YsH$^l^C88Eg&#hwFl_X0w<zLZO4=f&C1#SxeYD&2WN5~Nn@ip`=5$@)ki59b
z=ZJIJ4YqS^ll)tF-OXm)P-y?rWF5a?zILS0jsAr)^IkD${G0vt>gEMMQ>58iT9-dN
z(j>n}aJ3A7yt#{6%cJMlt7}{X52qQV9@-^t_M`1|_Jj1E+l$(cnimI4WI4yR%I)L1
zuaup(NB6}f&7?CO^4FGf*|{#Zo)~^--TJn&o@bxF@Jy_J`p8_!+>4j%Z-HUbwyp{P
zpG3bg=T58Nq{!zs<MPy-&MPMe8d-fyIe3h1_Iia=iZiZx<cPdH;?w3n^PjQdh7y*Q
zCN)vDy~<02v+mu>dQ`)D|M5@1gFKo{mes3Xu&kWY6nUW~!uaLemrFhh?cwIRC;62<
zalg{-r;}wxN-lg5uo6?=b%7)CzzX-wfA$|w#%-9;X&(LP?6i(DA=Th80r{MEqjJ5E
z&yw@JSD*hj>3aIRoYW(FPmNZtm)f|>=qyjzJgp7)9IE<Dq%I_XaX;YAQLg-AO2fp5
z?XK3r8fqrVou6-iHdYl$n!@NTA$eRu`NDRkYduZtKlk`;5HW4%-H^(=HvHSo8{I{{
zdPe!{n3LTM6ONpDJhAhTu*?<#3j-l;QLz-~#{JVLSRC4MZbr!$CYjPtO~2+<o;4~|
zNIq!To9%dEnw&OU{@WFbAEp{Lu-ZJGP!%+TX<zsnjmKr(E9|GPfB9DMzk#wztAN|H
zdk2~vf21%sin<3_mpuNq^uU*CJ(GD;z6C@~OrHK~e`KFu;iJdxbK0FUx|JHcc9d|J
zbM%SGePq}CXu+DggR}Qvjt<Ae{P`1(t}(sF!f&9Ze>D17NZ_8W&tA>mC}&(Jc|~c-
zWTl8*KQA==S-IlIhfNDjj~x{6KciXUJI86-shz6*2TwIy3af=w?F$z=Q4-~LL{h9r
z=R}3R!v|5m1g4jBg9I0y`>$cD=oQ-K`tE#9{C#ErnNs<y<i71c^zEXZc(uG@s#!r|
zM=RSIZ#UK00}Q;1>QgQ6&USx)^~vjo>xbUh8q2U|?RCrgbb$Y0eYnlk^9R_B=AT)4
zKzf5<NaByxSFYa3p2B=B$K`l$P4nvTDLIpC1ML3VYH7KzyE3cUIMYwg?9pSlLt6@u
zK1}URe=_U0P$A2#KdjX?)ix#GYh`=PIRiuvHp|{rGG2T5jbNbmqvS)|Tc%4ko$m@Y
zx#KjsYDd?OR+GuB{LRO?3;h0CSnV~KHbe6)XU#9Z!k$y_Wv02TKcDfwZ1&gmr+2!F
z%sS5>Qnw25le@U;H}|<KVHc|yc5#1;e|m;Dn`POR*H;VOHQG5h1a~bHdZm^s`DfRg
zk6jwG9h+54!af>4mUb7gWU-0!+iJ))&!Cy5Xb-2jQ^DWc`Fpl=9C13?b=AM-nkVbG
z<vS~!bu`cXZ2e|Y&|)kU(9n4BgKpb9>5cCK&b#bvENhH$PZcQ17SmW=STrZwQCg<E
zZ{wm(tg9K*Gd-IOg4w>D*`{(sqcO)&VY%!2ti{&_FW#8_U_tx6eLLzNCrR2IEUi)t
ztu<c1>W7YqKil`9s1sXHoiZ)lq%&_v$f1{;3KNg<TEF;qPkO=Ut_`k7i~|K8GoI{c
z{;K%bxP5sAZ-?WyMFRI)WS>4x3gxj}+!J}1ZQh#?X$RKp`MyoPFUHJdKSSQ#M!{CP
zLUC<u!<UoCp0ob=!VxfWzpym_V#Y9^oD++0tTdOgWSrOP#js&5&%ReY`4SR`3@5zY
zoc?&Sx4=!tB|iGE^_c=5PH1X%dYO|iY3up1yRyb-vp|BN_@dv>7cEIV%Hecc>iOg#
zSB_P`yU*O?PK|5auUsGgwA}2*rv6zb4hOk){EvJ*-{aw9z4d^=@`K4)Zm;gWaZWgR
zj7i2J-|ps@%hL7n-|mNRU_TdM6I^ru-VOJf&pdBdBq~hb5z}$(n*0i(fcwp>7w$jC
z?xhr-=AgBlp-d+rQTV||=@V0Snldtrd_N_+*R1b@lj^#lV;59@{s;`0sN-0;`xp1N
zwDOt-h0zV-{EJ=+T`YRi!JQSDuvdxw?8T&+CzDyV+HzDRA~vuWt&G}uMqOR{?e4CA
zar>$HftCvb9HIqPgp32#t|VMr+v48VaBq{W(Jz-1a&P+_S4g$~m~PDOSGkZcXa#%t
z;|ZI$UbyjT-p-YkC!>GZ?(4tjkjUt7FjMn!x6X{8dnZU2mj~Q><8Uxb-e~&a=|}5-
zXDwV*lv<@3{9r@H6Djupz3Da0-w!`Nw0NPO@)^U!IS1BvEGpZrTvnwKk;U@iov3h>
zZ-NfD9M8q;A69RCoH)1FFV11ZQ(Nb9&iVgl-=CY!`$x`5nnQxSAh>bMxggmM32DqT
zc}42<J9#VK96rS)7qxR^V;k40iX9F|e2%7hpHoacIWyW|M%3j^ZhIzPm*W3pq;+2S
z()I1`(|W#bFkR+suwxU$-|z-AoeS>n8QWXZYpade_a9=)jCAuqtfBQfeNxCg9+Tgb
zA3up`cyZE?_0`ffjA?t1tqQpmJoSs_*T92^|6CT+JK&=~qfp|tw)D3nt7G&PO7tX>
za?Bruub%Vl);9CdTU^(yA1swU!*=D`<eyI2>DvkzA5N~fRdP$7oX>E_jOoQfH%C@u
z^TdCr7@kg#R<7D}<XCJ5%cE~)sn$;npIEv_Bvlu&8Gi2kX2Ykv!DDfPl7iGK^@5!O
zt1b&g-emqXr%9<tA>HxLqo*s6+b-mq5_Uz!C0(_HPidvUN@yZ$_q?jaBj3KXRoqtJ
zbKOj+E28xLw_wZ6KE9nA`dlmY1h@p{5<0G*VVm%DZcpG6xz+Z;4?aH<e%x(#>5JPh
z-=5tGUphS`_!p$OIU4^}PR@&3zv9J4P5o;-1AO-I<_G<>WBqp9fG2sI^4G9yc5O|b
z9PjRXG+UfMC(|lkUN87VQH8JL{ZwAno(;G5G0a<VxPJrFYsqOlG6n9<H1g18OJh+N
zNR+yk9hKd2Ln^UJX5sx;{8v_FtX+_jI=w)+Sv-fuocXw=_LO_h{_keq)(~}1{n~t>
zN%6zV(~OEto0oCV@8m7`bLb_*ixcHH^39G|Dr|0lbZ#q~l(xy906+HI3+{+N-K0Hz
zZe@J#Ny`%DiZguQawSjS=ZMVbpKUiK_HX#+_Q(US1=fF)54s=quJQgcEkJo;M`wiO
zJUgYLw3ghMdWY{J23*e=KE0{tb(Ow%sa{ap!%QvcdTL<9%412B^_N8Q7}$JOnKv=V
zYsqy#m1dEbd?KE0ZfSC#wstOvQ~EiT<LWfm>D!M#m>+Oe!SYc^)8ozyL9P|+j{Hcw
zwCzZm5?eZFra;B{ryYU^&*d0=(^xU{<0dDuyg91I3uY9W&wnGW;Hx`1`a)np^N#t-
z>jNH77HzWUJ}!{vJn>IZiRn~TCXsWtpF5jMI#YHz%;SuCt^Znj)*_Dl866TY1y?QX
zNU##T)0%$erRl>4Cs!d|VMomk3wUSqv_D;*e*UPm#+rzThJ)XHHcPHF=}Pf<!zmV1
z%VJUaBt*$_>I=_9E3Rz(E+Ocbl62#>OMbvI_wEk6hHr0{>sm(i#;v=t_~Fy;$fZW>
zQqO;@UAS!Fq<bd&(rUK8O#6{;5Iei#MT~ary?L(>3x~Phx*548HthJqMYeONZeMjO
z&FIMEoC&+EHa$&ak`W9rcM=qySgzjW*WXjj)MXR2T5@KLi$KEZd*by%SEn7|46ur~
z)cQEDb_S#9l%$L&ueCSCb*s$y{drYQTuwt#{j1qiuDEVyPtkuP$NFYX_=Tds>T**q
z-8kduQXO-sYbvwnU2U^1kN3UG;CttG#%$Wciv`WL)k!usm(MH>-`Vrkc5CWe+b`Y|
zRVq}Zb2A^keX}`0Wo^3uQ{x9Sf7)%H)9SZH!dEBY=Rz^Ni_Ce(&qBL;IqO?~C!7ot
z;&!pgRO`K?)obm|tXaLu<|{vkrkw7&l}j`Cd|)_W{IBxZ`a|r`G9wv<pFZn2@c5?h
zj1T&bD(`nrmR-55`<tZPDZ7bJWV*{6#mzI>jwN%a?dO)($Z2SH3UU&f|80rIk|dT_
zCm-tZ@K{cH%gQyy$kJ)?jr%7iw>~;m=vw`|X~Tk~8NB>8M!^9BD>z-!n|Q<4yD2&!
zx!v)L)6}S9c2B<KeXg)ap+CgF&g4CQd2>)vVc}K1r<0SnGUQJ&-)*}$GrL8I|NmLe
zjM;2HVky=ioIiAKa(Ys2WzKzE=Vm>-SwO_bqLz8br)ug?Q9gbxVDrQoN3N#b?`aEM
zH~Uf<2jf{mztS~3r4CLN(Yt=iw0)1&!B^az_s<yeH#|Ss5jnrFm_siATl1;(2gaXI
za^<+~X?(;R5|*>(K!eQ;3)|DeU&T-U`B=>%`1^hLwB_5xHmrLkX&KO>8NjqJ&`H$6
zM48d=04LuJ&5TnlT*i&dd<@ok@f6Be%xg|(Ub`YNp!uNo#xpT&-f=T5qRcXF66%e1
z?Y$Jn_4Anqr@xQd^cf#62hF;jX1sHr?+n(H+!>$l|M1fd*>s$nKgFIQa$dsqDG5)%
zyiT0l8^I`H^tG!k%t2G>T57$>o}3qpKZ)-%JrNwo@j#&RjnW&*H<3SF+#@8oj{n|V
z#mrl)rJbOv<nvAF=+n+epIl!4n6c#Op`@b)A5XF|g*PvfPg&X-6KItk!OW@ea9X0p
z@s02QYyS>-XA3fYl4GB1sPkr3!ycQ1t~Lk7tqyKfZj5_tc~MI%ZIAAX_#fQz+Ot;v
zXf<YwVy>KIp|sdQIDhIB_W4Dh|DCBhZ`W8dX}x*u+qcZpNi0%toFCb#aBTS<Y{~TW
zl7_C#21k|n-!Z?lxzd9l6{#;P*xa$^N8QcEdnYmp@NZ@B@HJ7cm~lgS=8EnA#Z_m_
zHU7{z{Ytd!HSHI=3#$$)FP?u)al($T1>B3h`7R!*`<Al3kZ<n|gO}HDS8tCm(|0;E
z=fUNP8gqOXr2Te&Gci(rf5TgaNp6$Usu##g+c0f1$ct;@vD&%Vy@@lZ$x+BD>5wAx
zQU&j2`wq1|xy$u$#<jg`Cj33w#`XM?(fotQ&rZ|U4$(jGGxpE9k6N*=Q)HF@R4%$0
z@JVK7>++orX44$})j4JdvpzGQEc@qJYTUv{9x5@he<x}zU$RH={28`I^$AChuo>5<
z*KPk9daqQ3ORXwb#cQunu$Ahv3%)*cXKC?!KfUXBv0xR4PP=$cgR(iB%0=s4_R9N<
zpPc$4{lU?DsyK_&_A0yBCmNdtHs`A><@|DB!~AJ7I*P^7O)@*v0}rh4iWl8)BOxNt
zYbeXf*{IXd@yRps^wD>EBFwFKIQn|^j;@?|OO?+fMR<~=cI1NF-@XY*%+nV4Y*;sO
z)9o0c|GAY0|4cqCZ`^;y;rKkcHHYVkmhkI#N}h;Oe&oI97<b|U&h>g*B0j(O>-be3
zk<jz2@t?pO->6%4$9M6nt@wWK;M@79w|))1{m-~Ml~47ZviI%t+s=0%T>X5{?ut&!
zLvPJ*hKuq4_Go$&TPzo6d42CEyGc8)-&GgwW4*;*`ZMYH#Y=yx-lmrR*1WjdD&ol6
z#f!iE)0VyaZgJOdgQQ#a&lVTmw7>p0{dV25<U^VA5x3muek)vB5n|`~|5ly2;*zAs
zQ+(Iix{SD%@A|KE`+V0jx73xpN>xkcz3yc(i-g}$+3_<!dP(Fy;ehJboBH>D_SUxH
ztA6ltZyv7(W0~ZEt-KCfW%j@RXMO6X!wv2ootH8m{Lrg1Q795Qas2(3|NCxTY>?w!
zQtNd|rmIuF?1o^xqkiq06=&V2-JkB<u>ZzF?sfl9=ij|8U9i8e?&rS4{Xgo2{>n7m
zXXTV`cv*MqRb9j9#!IUC&bg)XKUJfjFMT`rvD3`^**DTIow`@?t@QJ)>(%@H&)u*5
z&1(@Mb)a}<!0W(;a=fQ3n`%r7{_*|(lRCft=Eb6W=9}Ja53+ve9G+|Kck6vuSxm)a
zujqAiS-Yp#u5|sjI&!J3cX;32u)Dvy*;Nl1+a%h(ot|cW?#B-EJv-ig-Rys_{qnbm
zOCDESC8!^Ly!-rgi|XSqs(QP%^q+=TDapTm`}pFn>-Y1;uV&AF`!o3asdpcqM(=$8
zq>wSgMla+1d%LLY-w`SH|98lX<xDM0+4WuQ^~Ig<=02}YTgNUjqh8+o+TPd9=UbVN
zo$Jn+WGZu~v;BK@+VSQ@pV<O}-#Xth2z@v0W#4gRZbFYWr^Sm|Hi`Kw=X}dt{@2XI
z`qZ2f-um~s$7M?#HoX26{q{Rk)=Le6?e~k`MjU?>wf6az6CeNnS$q8brt9J>-5ZlH
z`ycvewc`Cm_d63`*){B{&t;!=CF1+f<J<n~Rew$XR2^P@u~_QmoN~Kor5oa|@6%g5
z_wv1u+J29$1aH<IXWoCUwx;s_v*zzid$%;6y;s$HJYik3LE`%OZ}<9V>&fj1Kc9M?
z`NW}((;n`yWWT(1(?)KoTm5zS1YPZ3{M;k_#&!GrDg&wOagpB+RlENd4Lx-6eP8G4
zbEQ*c4X1Alzn31-e>-1ihwzT(p6JMi*63T|D`c4EXKX$5b<Tl;BIb8`@1~sabdr11
zx$N<Gzd6}9$LIBFXFfmn#VX<94FlO5H|yoU@<%$_b$-uYFaIs~@|MevPaT9G{@BA@
zb+gdT@`zOBd+qlMLgg>K*=H<|ohHmLcXXm`x+zEg|2x}m9hW<K*>xY!pM(|POmFD#
zDHIegDG!)3_tn+fy<hh5$Z%RMEiK?T%w6kg^Qze-y7L<Erqt<WCm2GGNBw3s65v!!
z%%821zw^YlQ*RgSlU4lW^61k1BcJ#Vzg@8PaG_~y@jdtF90nWy*?rh7=i;!pkFRWF
zvek=S75CVW|5f~`VD+f4$my=5?3s%#=XWp%US0gVqHMpv`P-D`aqNOMvMJf8qJO`e
z^veDDlW+Vtr@Xo++p_Mor|rqVlil-{Uoex||2;T>?XT$k2S&^PKG(Y_QuOEa*&>d~
z>;6{A^EAgb-!Hhbkyr11huYilWml@M_^EH*RXWel<m$JT&1<74RNnaRb-2=8mZPfO
z*mjAuwaf}Rj$7^qOGB&+6;tD_S02CGlDUrckl~-^MYevqpI)6l-@53I_VZx71=dF;
z`{!GqI(dBJ>EE4`gY^sL4qEW>9^a+BXUk=YhYvP9o$%^O^YdG+HcxkG9$xVH&(XTx
zCx2N^Fa|ffe)E0!+=wx5`a-!Yolk0COyX0pURP;(BVnrW-4o|-&s*{LM`eL-)dx{_
z=F$&YkGnpv<Dd8Rfz08?<?Dlas^_vE`uKZC`nt)hc76(cAh0TSdC3-8UKysdyW~GE
z)Oc`y%lo?)>+apq486DAsQ>XUjy83zzU$wN<?Rm#TE3rOJ~jN-`-YW=I({cU>n+UK
z`K@qnMB0V#Z?xXTzP)~S&#bvApJhMa-)*x?_J{r4DZW-m<bN&*xn#GsDyFYAetGlD
zzV>BrMcR4Kop{;vH|Moq^Xrt4e~(T+Q~I;^@TB*$S7okWKKK3A!4o{Qx*|TzxK()N
ziG%U-FFW7v@wzcF=|+An@B6<=w`cY@zqzMxxw2X6@%Op0HY?XZ_RD=f_tq1Q0);09
z8zw(g$#AmZlG$A+yz<_ZQ1<IaN4Wm|wh*eEkn-<QK-ZI6%cU~m9+p$oF7<Jhy8BdV
z^nT&0X0AHA*ukpqz3hrRSIpe|neY7hu6JNU^Oi|-P3^qp4^46QYrd$b@<vno_EANF
zeL9;q&fRkJ{bku=zp#e=<sR#;pT_eWJ#F|s>DzhhTe-hi3cNZxZT8zM`m*dlEdB@o
zdUxe*o%!oRuC^&R0xv77PpsV$#J6dg48sEU6P|0_8=GI><aC?(IZgh^?*Ddr&y(wn
z*f)H-z<udf{!!lOwb%6bi1IIbqtO0!!IUo#7(@G+%?kdWIv{`j*|Fw#f44Vt-YG2&
z_SqW1&;8`>)YEmRk0w7pwa+hee?j}697`tc_q&9JQ}>_g-@SK^8|!i3Mb{U8v0i$K
zoxe!n(xT&>U6r$U_^oW;UvR5s4TJLZe&ubwa!bujN^fYsz4CN{sEl{x#fS{y_kG=4
z<qy=%5V^|y=#<^Ak`v2cT`RMkvbw0Cy`4A6He(a>omYNq_&g<dm;L3ho3`-BgZ!KJ
zum2gpv2T3&zx7*Vz0{A(2j;VZt9X0fHTp)M0)lT%o;kO>##P*)B(FGZ-?jT|q}S@y
zI|yh$UOqd1L+Yn(_jxZLcjP*r-MszUMGH=q`G>7oxt_WEc3u0UURigog!%f%$u0B$
zFKc}BW999nRK70RV78O<^D^w(=3Ia3nL3wWx76+8E>Yg>z^LROd{fq{&26z@ic<Nd
zA6Z{$QoJGY?<=+Hf5F>#>A6K(&+Qg6U$KDaeEPFx-T!XyoSwoM?ys<5&tdM}awUJt
z9<86TukBNz{E~Ifd8O=K8+^kL@hLCq-@A10y3Mzy=cqS@Uf%Z1>GkA``CY%C?~NC_
zw!gu8OXas?jZ6>!aL-G&@LzO0>Z3ZZn|tUj-=EicSKJQU{7(AHw&H8s&OWL5b3Jh5
zTKSV~zt-=Nz4!OT)V1#Bg3|Vn*|Ia8g{pFo7c#d`xE8-E`sj|i$v5kcuD!VB`-}FA
zfoH=?R(ssuyivK--XuEmu3O%rrkkmH+9xhB9__EV-B*7*VSSE6?6)pynMwLmqT3Hn
ztv&Vez`9Kf+pY#J`>$iYZEso0tD5K6zFoT7W~7>L@V1OmxLt1k>3_xY;!B@1zs*0q
zXkPA|rLszQTa)hYW_@{l?}j&bm(`w0%&z*qQC8vmCJBj(Y}4&~bMDp4ncu0KysPh@
zy?<ZN)w_lp-*5!+eK=)pHFZ<{y5Hxw{k@XCdQW_zoa~*8N6V@&ynDTM-id5yf!}*~
zZP^umr|Xqj@%{$ci-GoA9v<|JdFkfE@pkv0O}F{$ZeO3U+<N89B|Bs_*X)mD-B$d$
zFS4#w`P0*?qAhE!#qRe--#oA{hhgUT;@Rd7{kPZeinp*f4cP7+@cpseHsyzAOPRa#
zRTjR{^}HSEdar+8K!`;{4zuz_-o8(Kjg`&*Ry{wg-dulB8u#M2#QNHv>vq3iE9_dA
zdm?rBJB{O?;uH68P1#?&{jzx3gp$XF?B8<NCB_%te7HE*G&cLWeYurcOX;4U3QNpg
z=53#ytv)qY)ZAfFPW<+++p4p_-?q%FeUe*{7=5cISM>F^X>FR%8`+jxueY0?+h4uq
zjro<We$nM;*m&<2o#tcIS@|OOnb3FJdxD8~C(e~Vo)Mv$+@>bIqqT6eA5&TL_X?R+
zwfSbBJicALUb?U?^{`t&eS+m_x7;an#WZiPnv#6-*0<ux%HQw*76`Z8w8ghi=WV~t
zPk&}acKh$ad~R*tpKorB9lsTLCRy!WnXuPbXi~Y{&N)u|eG4X)B^4-d=acl0vOl(V
z#f6D=f2Q<4n6TsX)Dr7g2id1TtiC&U!V>;~>Fe7{wD(-T_vlI4cK#iw#qQ>)<}dmY
zUvgMWXHBAVX^QaU^0;SuN0+m=FLjC7XUF*`HY)DB-TF|amC>#@f8T$0Rk=jZ{<NFf
zKc$vy@tNy)oL!u=_Mg(ZYw@|OBff9F?H0cM;*|~B4(tAGP|Xz=mYJkfK3`(Kxkj;l
zi6?);6EW#v!Hsjd_*WZd{F-aOpz>tPecoH`(Xl%I(K39`n(Wk@!*3t;+_k#DtJ^Ys
zS_0FLb8Am3+P!6dIo15dE3;z$qHhmezV6%4eE7|q;@i(3Fmm1c8WMT;?zgV~*PBnO
zJQ1kL>FK`osBX<&#kL#aw+`#=;CL)BciG?cJ2O+Jzl%F<r}L?Q?b9jWcZs;vI=o4p
z?0iW&xvz5mq&Gp~H`!uKUz_I$t>4vcy*=%<-A!59<!^bG`L38&R+)IGRp7S%U1{Iy
zqkm`Jym3$Ddgc;YnM-E<Q@&*vO7DE)d}_zWP4D&?eybK%^|}?A!rA_!xV<lX^5Z9q
z*S&KJmFh1Gz1G~d?hSWnUw6rV3(XIgjvGgphirWBe)G7?=G)<`{C7OO#U8jVb=~j3
zd+K#|8~nOuc581p`<v&yoZEft)JyF-nOpvT*ITrh`S^`}xgmM%miyj){@iETBWqyt
zHhJ0k^J)=MYJ2Lgyx!Zj;QP%}bt-pWd&!$luWbKa%aQu`>Nn=q$3tG5?Tg*FewF*(
z*js)pYkzx*e9>1}FMIq%bwtO%S<8?A`r%seMYp(WRi1r(7faLPvl|5G&XtSQ{_mui
z{ZuZkEc8#y*X(*d`|vMTjFS&u+_n5^SZMxTz2*1awt8`I=Y93-%hU8pvirhzZO+Lm
zWM9{N=)|U8bLIoP+j63}esNeOJuQCOtbna~Q{G<|nDkCGbI!fzqSG>8{W3n{o3!fo
z%3D|UQ<bOnw8V>L7>iD*{rcwlFVi=3<Ejt8wVhbMUpOzEQTb_a=cW9`Z-mdz{$T$0
z_Dl6r`yDRdnD5Hkm(S(e(>MLp?%31o{J-2&Yuy)aS28(Qyj<Q)^_`WK-TNDv`=^Ag
zTU>P4TX^oW4+RHTGC%H3`h3eex=8kQXCdEDlTeA-%W53Yn{eDdcf$L$vi0s@+0ysd
z7nlB6yG_t!*W|*UgC<e8R=2&Co3L1C+4C8nFYLT{zq}{&{^nYb-&<cNs+eD0vCjU|
zuVkhtj4swPGF@r530+y6?|tt&x&Cj#vALmB*|)KZ%s=+w%i6asJ72lY^tQ8@cb8$&
zoR~8Qs~&vTt9-%c`FcWhHN)pvjmPi)S+^?oD=j&EDn<Hy-oFQhc6J}^<l=K(?{SyU
z=KEB3+(+4gvHzwb%k7B8+jck`KYV^nGiQQ?nE|h~<<SDIf7-b!F-LPXr0?s0_58OX
zzI@YGFaAZ0y1zS%w9c@yNWWdad+FTv1_|z6=RfbgX_2E|zWvpcg|oCCJI~%V@!6NJ
zhDj^$oqwIE`ApzJ$?hEK#ns1mZ!^iAd|P*EnXvZFCrX8v4ZLq3-+cbcPM4X|dpNYp
zxhKz^C$IRKr8ACYW)Mr{^Iaia`|26x>Zh%Wo{;J_;irebO^UkA|IAZ6{4ciD9ob>O
zvgORmV=mEMQT|Vl`W!9O(=FZ|k<>eH!h`EZE-g#9-?93;FRjm5<?oNLiqQtuOC;Vl
z+?BUw_p6O=QvWRd+EaXj?pA|(sZTq^RHQUdRA0?Mb$ZF<o0~qd{(gU2b=tJOTNK(?
zMw~l2XX2jqQuBL4>b5<$T`pquEB3m_3k5gv&<orCDsr6dUvlB5+_XE3i&rKU{$77h
zQe3!NYVsY%g@^vHvaibe(r|`ZG;Y(GNr4m3tXC4RI{GO*C2+l1`~Ik7oEzsD?EkiK
z{#m;#Nh*_;HaR5E%nj>)9RKE`U`_s^n4Po!-u+O%S>S)|*I5hC#+{l`F#G!Y^!xAX
zggGKUC-gT=iN0x&xBi-7zCXi?X$zOm?9OjymM^OKDnGsUrA<w9eRQ+`-SzJN?H6?1
ztNe=#ZuZN?EM@FA(Y~$lD~0P}+Di_d@QmdKvlYMiZ8NIRcxhj!`)K|DZ#vm0y083F
zQreo+wo&(z*f*{-&l1bz#p}Lbk={S;kI{y$I$HbZo!Qm!Kg!Eq#r-Y+pG_>b)89@E
zU32Dgz(&(aGqx!{m>GCesP;>`b8hOVP2v7`3l47TaX-DuHsZC%v@6C5Ci}L<+a5i6
z_4AuAF%sXVo>h#VaPi1>Y0c%ogq$Uw+<dL~<YL{|7U}S34T}Gp435v&$gjMox5V$|
z?UmmgKCPSAx9GX_%cdXmDj!`pHo9GM=<+_J&r$VdGR3CyRnFH`s~<J1-rT|CT=IIw
zJ!zicUyJpZmMq${e&JHp^Dh?8f9m%zO3cgq&2Mf+fmlZC2qUAw4VO;rPQ77st6sXO
z;H#)$@Xnc<3lk)zKiGY`^4%x4^=Ew5w%K*UJ9wu5zQ19KsQ+m;b+<ydY1$QB4NDzj
z54g;dRc(GJ9`Iw@3}vU~7hYAT*_=P~=i<q*)7O-0$}&=q&Hd-Ht?J_&cMIt>R;io=
zt;u=;?;f0bbs+1FzrC4T;3`{XqeB`2vzatCCr+woiK>XK|66|l`#vjMNB{aq3nH52
zL(`^mru~^VVOf;0&;_jvEw;b!egF4MYZdQS=A8RlsZ!-zmwb6~t3-a<vpKB4HmBRZ
zR9eh&wIkIt{PBTbo6E1f&-Z>KSK!0slwDzVz+qe4onK#%rDj%(>liP8xUO(3*E~tj
z)4Xvu*Sya^Kj+$Wph92T;c<cVBqi57$sTpRbJ}Xy*8M&D<_P0#0j@yD1+k_^SJ~F>
zlrTKcvev5TfKjuym--T}P3oUlt+Edk{L=WK@z2elXP?+h{tPsZ6Wg<9$5Y+?*>3k@
zS4-RwFrLHJrTEt3OQpwwSwCDXm$02JlS~YIC;N^wI`Dj2;Dk?`_e^Y)6p3q}Sn$zQ
zI^8O0f<sffEBCZ5nnvG+iY6L$YHpru6qv4d_t2aL7dbzj4yYEf7oS%sw8+|9U#0Bh
z7oQ%F>X07?PIoCyTV~tn;jyP<lIo>*yvB}3!iU_K+fBIoE_B9b-%|w}HqHqN>X4W)
zf!9((IG9)Mj`Q3NdpZJUoa?%|JgLZ~VUEu{!<X^uhX3IGlK=AN7a}!Ta(<Wo)Lgo$
z|G%MW!hd54S+-Trdvki^wf9DB<*wg-YPs-LAHB@{<oax%6midGXFQTt?l%)N-am2G
zr6q=c*FII!*Wj5M_;P(OJ7X>P#iYHxkJ$hJtY)p(4R~!{_WOPL-}(g!>faRuvi?5`
z{HOV2`GfgQ^<4NFKjyDX&)y*PYMSOW2ZpTw^|w9#`-|@R@xFBL{(s+>ANa4&wdNfA
z`V{*~fA9Y}xAKQx*p}OOt}H#WTG#$bRQ-uxi<x8VKTbN}xAgn1v%0?Bf6a<5jiVkX
zEd8*1rtt<p>G%J5Uw^A#+|Vb~rMp<m<8AvMUDZRM?E1dz``&BIpDp(P@PZ>zv5qP+
zulq$^n)K{<i7KpWKf&f)^}oQWp6ic&h(gJU&f3ZG3lIBkXklCE?Ekt!_|1uO|L(lr
z%m4rGy0>UU*_YEgZS%{1<T6V8y#Ft1ed6gq<8OgwvPmjZZ?5I$sGI$p`o^36sa@;G
z<@_I~9;)G7VYc5q%y03G#MtWR?V6S8C&QG>R;`>p_j_YW^Xi;wEBF7q9rWGK?Dp>X
z=bFD4?J?-Qm;8R&?wY@cPTb_r)w{Mh*Xi53GmdBZQV({EzORef^T_(L?vd*0cXwQB
z4}28(YkmHujL)W0x)BHCC6;~Z+$wJx>}G7YbJL>LS;GGInXhg$g<c5TzOi~{<BpvT
z;xRR8FH%=#f4Ent5`8G^+WVX%>@lx@J~NHKdF|`cd^6?GAM&r&I<K3$Zk7AZw$?YZ
zZ!7)ZV<mI*=WkZ?_1Av;v2maMo%KTNQIAl1Idj0eX}7)9iyysgJN514nnlOYoqVdk
zR{TlFZpn>b8D1wdKeg-1)A#wb;P#(I<%V6#m)317mrQu|OK7phhFL0>M|iI5#UwN(
z9(wm*HRF20#D)(C<~M(ksNZ{}L%>zL%qLg-?Yoi(|En85edq4rTcT{mq1XF(eM{@#
z@}6(|Hz{x|6xpca@~H4&=AyUXu6y^y7)H4?<*#F@{lDS%`{S%u$9sZ>d7kQTp7-8t
zWAh`%gh%?$SN&FdM`RXqN*m@~fBI+Zr6~Q*1NK|y$G)`gEL=VPTUF)nh;8v@PP;uf
z{Y!ZDzc;qOSp9Ov%wul1uFk1HB;Y+OLfi83qO`YM=|P6li|6;9)+^h+=<K6=7kVWg
z9kPw#W$5{~qHO(l`Md9;-;}XOoZ4|u)&GpHWWeRU4snOH@A(uKo{s*=_@(URVdhV&
zsXh639tHigjQGDl(1B|M<DbyDFL(N<Ir;RkoZO&huC&D3M<v4Jw(Zi04C}4WUPYWa
zDZOQ8!>tXu;sM;#Czj|N7A=&&koS$xZRh`^TZ-KHT#J|NdN7HfpLxT3@m+>iiZi!Q
zkyx5A@9)Jq{(7H(@;%eLd^{m!wTXixd#Pd7!|OACrSzmU2-obN(VNrI&bEj%Vriby
z{eoy^nP!QP7h0>AR@+Ij2A=H>c6f2P{B}l%o4e0qr7f3Uo?5}NdF^|-%Wm!}JiJ@9
z@67J9WLnf)q<CS**OZT}dLE(zr>=Q)y%Wv7_F{VIn~Hx+-NGY+pD89B=~qy=`SjlG
ztP<8gHfNHLvxpuwTM+&}aH@k_h;rf;j&26`XVaxzFP;%}P`rNP>fx;iD%svGoo;#m
zK>ve9{`@k>dU*@IR5UKfGQ=OA`gG}z?<{_|*%(hXxHLFgm76m~vu~N+<oTcV(cQO4
z-r3IHbb~MUu!G3=N9$F8Zsiu!Ix1#-P*XC-jXiSro?Z67{IZR|Ux#Ne?LW6}eVxp&
zbIq5vPxLpOK0QZFX9dHC()g3pZ`}>$opxp2I+y3ScifqmB|cq2<fM0Dz9y6P&8?v`
z8n5N8jo<G4{Z5ZW)0fWZF1ZI`#hci?-32=zSw#DAZ799_U&XlPNa}5Y2Wi<xf){v_
zKE2IYSIN<RY42^G&)5CleCMAV^RIMKQvL1N>vg&j{J+v2KEB%S(%5`!y_t1G+4i&B
zcQ#zfQ_B?DB4%;xz|W&K+o}$~nq;vt`GVu#|1<xly<f=6%V8av@HpV;(JLP(&f;HQ
zQlQK9eQ6z|RGz^u^RI?UQL-#m)4$An_c83u#dA&$9gPYso)Y=b-(8l@(~ey+<^F=D
zzjqxxxJgpzMfauh#Xqmq?KQbBvmvaO{oalrO-XBwZK@u0UB9{RT}Ot+GUF3_&!1BM
zY1X&uaqcd@zgp36OWUV4EM5C0Ol-r&yNA0f{)w(Vp5rnhcf;IM5_{uJUD#8S^{PIU
zn=X%9bh_>dum4P^C%orWB(MCDN)syiko~Q*HgXzc<K@KTj+&45atiCD2=pzUvD~<3
z<*#SYQX2O?DE!7<w9EYB?TWoQdlk<Ynkl$WSseZ@R;;D`#EX#Mm(qMTNSmkMKD6)7
zUHR?5dCR8cU0-AvH?7N|Vt&e*4fhv3ePkx`dc*HR=^kn3_#DmM!IcJn@n`;iy1uEb
zUSo>EsgrlEpAqEB@OgOT3Zvoy_d7L@ns(N8d(L=}#I|hn=Eoh1ArJa;*R6b3ntM9q
zp&$#>is`u%e}4LV>tNSk?P*$<t@H|(xS8xamw9`ZSd7YFm)}P(tT~?`H2av{?((>9
zi(3cZ)it`$jMC6Iw3&W(s^sQ(9^bC6d7Y&fcJ`MZquz0Gg^M*Jx!c&@>VKAbw_4_I
zsKrg!1oIc`3upW{e0VjaIQ&@U8}*+?SCS3I&aAx7C7FJ|;FVP;^Pgmq9hS3i2{35}
zuAW<{rE6EWULaTT;dd>u@_F0mPF4sy5g&bnr7_2|qt@fkCnuvrD}uk(u4}7x*dige
zc}CSS-o=?ediLp0Q#+)repx@_+x^CC+YTj6KgFX{(7?SuQ9+tr|4AR05MS%XTS@DF
zGd};K)U@69pQ%*5n^iN@_wKv;&t_%+STVof;kkb?`&-|J|2qWuugTa?)L9#^pEP6T
z0YRA=oU;>mc(PAcZ%eJtVE^p;M!DSIjhic{Zpt>Uo8PbIxo-$_y4sVtGN&;2?Y?an
zJb5qaC<Z6XOj&E5k-WQ<D~9c?yZxcu3t@XFxC&L&&OW}dai*qGuHeSsZR?brWKv2^
z(tIXOII*Gmi(W!w&eh+ZJqeRpZX9xBx|1EgkXK3Z+_Z)O12@jN>3?@H<d*6*U%AO4
z+UU<1aru|Qo2j}Bdn=0<ZRg5QJY3lL?~&Vy*%7A-W{AFE$jIPXGe6Ogd)lSrX&q<6
zuHN8wPCFJC^P$u1h57aeC-h$|Uf;mD;3Jn&T0mu>#OB0o{dp$$4)6sY>IoOPFiSRw
zce|wP&Fk_v*0Rri!28wh57%RH260Z4MG~obr40{X+~k(I&hxA{Uf}ls9t&|cGw~%e
z3vzu`B!2rOv+i%`=Gymlo4{PZpu<Isv$IU>PqIB0EofHxz{PY=vnf(Q;7;4~y0dG9
zMV`&fFL7RGyfJo8?UknNTMBm#VmCz|+9^L@`G#pueOk=QeQWQEMsS(GX4<asY)|$<
zwpr_iYl<JL$!&B!vHM})@!G4lJC!$j6>_}b+bOtZ)dgky#?%d_E<C=sTl_Mkr))mO
zyFDPNj<1;Yyh2CCB8TSvVaq39Z)2;U`hvsuftk6~T?w{;oMYCHVpb_SbpL*{@9E$9
zCI;MP%27;uvuo0LEDzV8((qq!>X*birnG};!2)Z%9ahd*yhl-sadpC1V+mH5^{*5!
z@%;X}(``ro_p5Iv2yXmxwQss1S5n1Fb`7ruS~;nWOw36|8U+jfCbe|*9b%HuN{~AB
zy{DOB4`b=xhPOR?xR>m%f4ez0<(N+Sr`^XLUWtUdUy#h5Il0QvH+y>c*Be%0XFNn6
zf3MD$-M+Gup(n%bj+x65gR8UevivksW7M%q_WsIWt6SA?yUfq@MU}h9o7b~_{5K0O
zcJI!-^)l$OS8wIZKaEni_MdoT^rvH|nA!%7e7Bh=k10rGPVG@jj9@O73|TzCT6?ee
zV}I-Bgga5jY}a=5P06gg^8JO=47cjb5AAQrT-{Vxy6?P(&k+r^WxNG{&AVIAscIZ}
zd%cZqcINS0SFcQ+QDzk8D0$GSH!{J7v*OY1i9cD*FYbD``hKH)qm;G!rT0b2tMq~$
zv_AifoFTT<dLL_a5V!qh*4CFv;<aj=!t7gbNIuy2_u``UH#gMFemQ5eqHx~adT)lV
zrnp$HNe1u#E&I%W<5-46e)_4TD#dLHP7gNoW&N!>nY?x3)F*#GWV21!$B~y}(lnz^
zf3dNZ9<%&%o_mQcJPo__-SbqKMFO)5ysYMYfA{;YjDp686K%5m5}o%IPih>z_l9)~
zXO~0I>VP|p%alW6cF$eT@FzW4>kPl>#Kk;v2R}yjEK0Izl?v_(@b@yTJ1W`qV3m8U
zv64;mMuvuxO$%;Kk$yBaWcH?`8%*}3Jn{5A7E=*2DIsuXf$o<E->&A2g;RN$>JGX;
z*Zp-nLCWsP>4lr~S&v^zjdN=&Uv_BQe&O45O-#<5Ti~%kMMq-74XrQQ2b4tjN<MtZ
z#ZvcJp<syw`x&ckGt>p=xUvgR<`+Gb<NnarC&1latn!4y?LRfWn-{EDegC~f)Po3~
zY}10*ZQBkcN#`?e+}R+q!6SiVX>!MUMWxwMD^~FQYm(re==N(vkF?2~)JbBg$Hgsp
ztWUc|WE8wf`EoPGkY}yy1__4?l0I!g#m;k`6#SR<8YFK#%G0a4_^5<hb_b`LOX`hE
z=hVL?=>}3_jH`urGfIe@VVV%BeqYFm<>s!)IU3zo$$K&rawMnkbv^UfX+@T_{!i`%
z!4KkIe^)&;Dp<j|<k;>9Q5`dvi(Jg=dRXyzV#7J6A7Revb5CU@7#_Ii-ZG&kdO@M^
zvzv$1kKAVcp&7w5XK~$pX}fNLIZ{s!Sok(v@R+Y2U(>AZoOQ5FBJaboS8*L2YYe^`
zxH)Tvo2<EZer14HYlIa?Qf6)l<MM?T0v)M~W3;<DvnH)z+9Z+^xhT$Ux5-5lp0doQ
z<wjes{p@1soBt<(X>(&i*xc6!XVMDACFh)YvYd6h$Ng>Hd)V}T`7INPx#`d#R(_3P
z+e5A`yxoFp+8)+Dj^;V}Rg*=5<C2=>EY8JtQ>r*+XLY1<Haz`x_x^!^+m52)-Wq2T
zS~AW#+}jZ2#_C+csJ_Y7sd|3T!d+aYtqPkQ4H=gz?v7VJ!O7FMEA8xuxltxe&gE>6
z-53ws9I4=*p!xR$!_BSVJG^tQbaxtT_~}q_EKZ~46{}+B#wDyTJ@4({)1SAK=iMQ*
zoA2UpuV&C~3Eay2#-Tm<atX(V-A?RZ+6tl_=SuEXi+OwQ+3L&bvK_3oaU2du9GQ-$
zzG+}OX=t-juytLKdTqyrCHZcSCU#68zpvUTxWGY+XPStXC~M#0lO}Ra-CT`UE8<_Y
zURbzv>XMY}795vS_Ie&JD9ifwOQP$wsBTr{K?UDn=B~Au4<E_vykeDS94v6-%<Pj(
zds`2LvIO=!-E5fiPWG~K#e$&Ze4CcJ;vD@`Pw?8b<c6kwN#jm+3}x<ith_uim5I5r
zc!mPUqq}RCT&p-Iq%Xc=*P#V^R`NpHk<pV&X6^WWeZO4n;f<3^Bt==BmL+pcsQ<gm
z<G#aBHRWEhSO0r=-!*cOicWXB)1ZBO-L4O99h=(ipI$57*Jx&O?K11pjBRa&&82@Q
zon5z?w?k%8*)k>xYvHsOzr8cn&Ms33&}AulZ|kils+(23IZSBZ?Q_xX9sjcL{<WHW
zb48cK#HL$3j$S3l6fI86D&#Sg@obxR(M>GJRG-78sWdyLF8p1?MxC^|A_vZYmwL+k
zYV-TBbon)28$52QEnPWn{k?aq<4=pOeV){@)tGPDEM2j7y9WQw*~{crKJYUBSY^!H
z=;FFX&$m%@?S4Jhmpp<3C5B~+6KaiBdvte{iXM=iwT|&)^KOlOpRNA($5&i?BU!v;
z9b<W|J%{bz#-rEtJ+zxFmAmpc1T2)TVOY0q{k8lpdtdT2YtP*5d*x;T!^T-`Y5}Wm
zN3!}{6|rlp(Jj_gQ2r+Q#eTQW#+dwVSGLyF{uXFCn|7H!l<D^ifnc}xV|ky?g`Q5!
z63FNb@%cWl*LuOu<ww`G?%P__wyav9p{$KDfk9|ZW5P|}*UDbsJxpgLuo#3z-%5Jc
zV0N5gZu-;Am1-L}#Eu8_>c5z}yK!CtSJR3$jnCwGmp^UljDLUaf~k4%+p>4*D~@Nl
zwlABLebukjK<A!iN^|<0%x}UC9?V52CmL?Zvvw?s-0y2^kZB+%SkqX$XxG6<tE4&W
zZbrUkGH~JD8Ymf@@M(&6qoxAe>qp{;JuY}lI5idRP*O1tx}UOvweeaU(~%U33rQx;
z@}~l7q&FVRI4cnNwa`<!LGfQI!=>P4UM35b%_RmRSN%61UC0uaQWAVPc*(h=0io+f
zS$&+?ujpx-rCfdandyLvbLxpBof^pkJ}-r*RHvG#Go1gXbc?n5==3ui)-Ya~+gil-
z&fvhQ16~u}uyL3P?__RTE3t2rYHNq#g$n0}O<NAGI#D3TRNmyw@#Nnehp;1c+Y%<v
z-K-_)!}P6TqZsp3pJ{(~Fi+A{mon4UieTA)o-6d^wnrxR8=H6;`FJyOS$X<X``t1O
z1UQbhovF)pcl_eCp=074CY7&@LH5E`xArDgSU2i2+XdgKl~|n+efR0hwo7%1-F6WV
zObp%GPu4}pCW!JJS7~A8T~N9DZ7jo9*#kO17?<ZM&A4>uOKRJbB^R^Y8e>*5E=!#+
zl%>Yx89Q0@+rpW1`QDxG`LVO@MV;tYRfCZ9TbA76cR0do<9Q!;J`7rUW&NrP9Ve3<
zXI~7gn5?wIPr4?*%2aD_n(oTXKdzTpCu$Tj2pqq@Aw|jLs@sP*zotA35boJ{;A7jY
zCXdr6qz+Fr$)Du5;AQ(Yfn}0nU$t)PGkBlL;E@(+N!*ze;?~e{z{!YzLT&&<;}!Fg
z$N~>vo6MwZar0z%PYgbOQ%!A$&9?o5cO7O38X21QL@ETC%*mY1vM_r6mGXN6X=^!V
z=4luo-f=5Ko>_!7J3Hf`1pi?%vC6ltcbLS>b}?jGbLu;UP1tO{C922NKzQ@9&H4_>
zXYO)VY+b^5Ks1GqC;S-q&-yjRJqyZnBi(1+-Ql)=#!3D_)!Um|&%4#FT<EoRMX^`N
z%-E9+j3OVr7#Xhb^gKNEN6s|sJr;4(-n{V5?NmJNH?e#l|B~3v$$7^@=KeAIcHjbw
z#Cnas!;3R5noqaz2C`WMF#4;poC~{9@%Np%uKoj~Lw>Bwa+e)n(f6wRqtp!%mp{Du
zajBgq6As)vsn=z0(QB}WiMh9F%0BJC5u(=1w#~k9%k_rv%mTwEr(-<X)8jX%_#7)c
z@ZrdgY{xJE%~fuQx$T;D+&SIM$CE*^GXIJX=Yr@LcjUJSi#<M{R4jfhr2owg6A{55
z@2+<Cn>#a=oA^ESTUvF0jnw^PbEB%7?k&#TZ?#m?@#sy@jnXGP9&<=9m>yL>HLP;)
z@t&5hy~i2Ewj8*!bh+V%jP_bvZ}pqXHeH8~u1uNy)MZ`B_eV=x&TMjXxU{D1;jCY)
z4p%N|Sjb-5w<O*9{f6JZUpDtmn|MuY*R~g;`bmFo@bh0gb7$q9BuyrUT^1(a&KDX;
z?0aRn`15N2gAGrwJI(I6{3fMJB-#1v#UrUP>sh+FUX;gmW@p_>m~`>4$dU|Rv0&F5
zqIY~Hb!?W*oc1;2P{2c#*$FK#U9u!Tr=AOpNLMaAyX>febC}lm-5FLlj#y8fbaCNZ
z$3sOd)*joO*?1<L3HdzX)r3sBpnXwpGrn)tJr)s=absf5@jV688++GD>VM0=A|flk
zrQ&Lcr7gF<$I5*cM;9zR-lx+rGlh3rTK9#_Bd_MBZBFglGG~=Q7{l(c=8zKtai&aq
zzm9udFqDpQIlcYPalxdB9l|qzofk?o(GiL1dcza-i(_)~w}s|ee|^3tEZ{n<I>RW4
z<6?n=s7hY(PbnkghU=bcevO&_X5W==bj>IZ%<tZ~X0h<>>GIymx;Y2RYU4lh+T=`=
z4rhv(HB-pIzcsP&q2GbSCvHSOlVy0h{K6!c&Fw3WMY|{$oe?@RA)tlr>V^RKm2E4O
zE{HRKFXpu}+7j_MU+t5Kg7W1hE{9J=Ea3Ua^rvTLlY-W()Je65Ypw{2^q5#rn04)%
zTf*FZXSx`iJ|*p8-FAFKPTJvh4KHSAIvc<Fl6_4<`$dSry6^9{e{Y@M8(wubN+{*e
zQ;pj<R1Rm%yfSU!mIhuQi)R95hj!dqr?;!}qS&djy^ijUQSZJwId^capU%;}(EC8Q
zM0dJd%ihgtHzSW2+*!YNgU6R`(XlVH@61lLl;{^c8*A|CNqp*=iprZEPksfO-C2Iv
zhATYsxsph+GfP=;OZP+D%~L11n50NCB<$l@c5%Y|hVzqLYPj!-pXTqDOK{-WYrKs?
zeDT9ir*{a*m7NYf$7?Xz>~Le6r`~}^i=;Nz)@l07Vhr!@ku`Ky(+qZFWV&RqP|!TK
PJ$Z&eM`#1LuTCrgk|N3@

literal 0
HcmV?d00001

diff --git a/third_party/icu/data/icu_conversion_data.c.gz.aj b/third_party/icu/data/icu_conversion_data.c.gz.aj
new file mode 100644
index 0000000000000000000000000000000000000000..1ae6bce382a05570b46217e1a031414515439a42
GIT binary patch
literal 177689
zcmb37+EpljVfO4>hXsT8$M8z0W$oe0U@?AaSd%fQ<D=2l^+6dTIeQPvtayL!$7|K8
zA$<)DhadDvntsX1shmIg;fW<C*&9L)7+AXYtVlOhd1)!k<2%di@{WWDr&#p49d{(p
zx*@r1i;Cev-~GOq?mpb}aF_17jmt!KtzN^Meu#}#EQIAtf}P}z3Ekl~xyQaL{b@Ye
zk$>x5Ohg-#JD2fNUV;8cO2$hg>XH}h$a=Tfw#50xv-}8nkQB1ex!X{o%`mZ&twho>
z;^r3aDbf+u9D1TZj3mCNR&MKY>;53Ufm=uNUthpmmEs#2kJCR0Tzc7L-|DPb9K+D3
zCoK2faEGPvLxD>iLZX~@LS-pkhN9bkI!@Sl%CEtJl|!eb@s@h*Rd=?gcc%>QUP;~C
z@XkkiQKFE7KCgr02A-3AAs3P#WW9OidAM<Bm8_7mc*K5=t2+)TaLowr-L`4*o%iv*
zjTv4KtuMX~pW!U2wPAl_qpzfB_hQq@&IO$;b;8^BZ+pgIuJZHHyppn07h-pvh>e(i
zEY@X~Ys4x&XRBvRnWl!S^X6>dy|#M&M<X`XS>B(e@3>S2Yf7J;FXz?rsg2+7>E^xP
zUkKJ8{y0Z>@402HTJ^z!v$i|mub=Gm_2TR^2TXL;>J2hO<oCXOaQ_5n@(Md+<%yp8
z_c$jC9I=+J|8ghv!1*mLN|(9VPH&L#S<~|@;8W>_@};Rwl352%=t*=tuAi55(_`}8
z#+8af);)|qXD2L(HCd;6aM`-WuMeCJnyh#D{E-UpDAm~w+)5pFVR5m}7hXQd5;}gw
z+G0oisZ>o-;Yr=4#;d=z$yb{Cig<)3{}JxXahLV+y;La0v;MxoP1VJFpXSC)HJaz#
zq<KgCo1X6!`Hefy3bCnW{P}$&Cz*B0LG8r+>d&4RE7yx#w8l>_Ht%e@n6Uk@SIo(H
z<qa0!KD;qa?`g@<%zTqs6md%2%zQ?aisRK;)#cB2em}){U)cAh!u%7Reij{XC;U-6
zdoA?!af91wZl^P*&-UN`<sH+Rr`$HBFIKldoKo=bqNy@x(x0T9S%&OMT=Es)B$yBL
z8eZjCcQhwyJ4?@<l`qVB0(NCToZU7l)_l@}+pAt&IC1&OH<zn?`y8{ccNh3BzG-q}
zUB^q;4#s1z!s73?U2Q!ovhYX{Z@__nS&d=scaskvP*=V`H#g1S@5*n#fXVx!Bxk3d
zIQ4EznwoEIS=Z@fcWfRQR0?J=Ug*r&pZw0{<wH@%$<Gc~Cq`$o)TQ$LY)WJCIhlH3
z)&j2$dR6yrJLP@%oUsyrc`hL<#e?(T9I=YCZ@#X|4Q~?kH92<FPcvt!Q0Xas!8t*@
zy$$%a0;R-K+Z=y1h3;TeGhWj={nXu%0%OjFIxdY?dR%ov2d-*(KJ^SsIqbEh(kDG{
z=KiWyyLAi^CTp&$_Hb4SbuH=Gtd;tnv4FkG_@L85r?s4^a_2V8oIR^co1y!_1?8Ow
z*RUMeHFL{(PIsRhLUBJOnY|9iTX~<GUijzfab4$luGjP8BqC<VN}D-POXy^%HD+SB
zYQFmA^dwKo`qcSV#~By-cKar6-<#Un6(eUn)o_W}!t>`NZspW?GF=l`WMbNEbU5ps
z%jAa|8S}QLt$Z1GNOH1S(#;Fe4VoNJe&!pf{=cI7?>cvf;<e`;29r4J|93We9Gi7Y
zGWCJc60Y{>sP3hblMi0lIrH&6#!3I4)_wbboVz1-o$|A`*fqwd+U~9^=D+++JAePF
z+sbc5Up&1Qb~|cIUPEB&+mE+DcHccIEzlWzPTJ<5y4b2?g|7eDcL^%1t<W?~YI-Qn
zGku%i`~Oo75AVy0-fxy)rR>lXD{0yz#QA@rDodgAg|O~mb0=rd@AXVeZ#_71)#144
z47M$s8m7+l(wlHa=9Eg{oY(V3rtp1HaeFmGHoeJQEa6%XXYajC?Yer+6LN0H0w>vu
z$6gnVeaK|W&3V$h_l4o-3wr5-@|`oYxDVJaFqLdye|n!tjbzV(kZYTlGfi7~ap`Hd
z#Oq5}+Md4DBc(cJ@AqwMB#Og-CI%~4^b~ukO<b@}SbWPwLC!5oYzcPzR`s#&%3YUK
zonkw2RkQiu=LsHD!*|{+zs2OfqQ7@S)-pw7#db?c*Au%Nw&Wag^PHKQ;Whtv_@u~!
zzndkP@;$PB4{}*|vanyhvU5h(r~WMyX8jLxS%2^*-&~OeZj$<?e?*VPf1H2+ME$;o
zV<yihAHQ+>$LhZQw=`zd@c!#HmSQ#O{p*=rf9b~jFY!0zeKX4rY53J>&2Ky({zSg>
z{F(HROjGN(e4ktO=eo|X|G}&&IbH9&er)?9aJ=TJwDJu}7D1-npWn|qH)C3J&lSVj
z{tln_%RKMrUhzJdKkv-r$I?sp`AfMg+B>~U_o#I(S<{i^dPZ}CdSIXFukiIJ>o+@G
zNKS3iuQE_w&nosmyyuI5<9|2C%})*9&o?h=IaaWDy}6Q-QqKF+ht_i*-aKQ++ottP
zc?3IHgR41h4&5ueU_JHMK8YQ7?yx5uN{hYKY5KhC^XZKp#e2OHO17WAd(?eT;o%!%
zo6hb@)A5S`-*7=c?RfIV<2tWqxJqwUNWPf-<X_IlGt191*(*It&cD-SBb7Se{me7*
z5{A8npLI5!=oQ;|vB@;*3k$bzN5cuRP60&@-zoWQ9(%ivN^Gf!>~PFE$fDb}z?FZ?
zj7Bl73nsj3b+wT`4&7`Si44-YO13^5j+SqJo1vUzp3G>>W6*MITX)Zu=^M^+2xJ)k
z6uAGrNkQ`W`-e4p$3E05Hs~jRKP9wQyx_nWhOo#))0PF5T(3hiA`Z4ir_Qxx`^Ve<
zIGJ(A+Zn17G5nhZZ+(9q&9(Tp<F}kv(N8}EcRbnNzQgocUCZvj^5+9Tn@#(F*Wy)u
zvCM{>0upaH7IQpWS;J<hJRcmLY+CUKBB6=RN169;ICzMSTUsaMLR)5=|F=yaPP%bR
zug%%A>Ef!*pN?CaJrs~`zhKdFOwr`r%8lQT-IK1)k5LJJ%W~xO!Qh`fHEwpo^Jktn
zYR`PO{F~6)Srg+LCUE$hY2}1G`d<7;zu{E<!iL74f4y!-C7C6yn+zHl9T}LLFCExn
zXkgKP$n4}KmK)4p<5_t{uIR6b57@uJu6DN8td$whLNEHcT8o|yF$uotld9ria@pc(
zTbb0YM4zK=X;QN?zvPNP6>b&Sb1v=zv(KJ{hr+R(e;EWH_$J0=@n$^WHe2$}>sZ^J
zw4j>-3C@l;93D(yl1@tC*qpFgU7AZKut53DF*c>>9+{|==83#+W)c?@?q?@@9Q|*4
z;h!_B&N-))AFTUb&OYD$XU|r#V^23u&D<5M#SwJ#f!)@)b8<fqpJ|-V^~~#z?un#(
z+6ykF>PP?nnVVO*sb<Nx-_J7Dw~OViy?Hd*S@78Xzh|$SU0)I}`d@VSzj&>L=P!3p
z{kt}=fw|6E`P~l&!*aQXU)LrH>HNF2z`J@`;QxPT{_kWGHQOgDw?pivsY;CShbugs
z*If>oPQ2H1<K~7bW!^Ol?app+daJgOQ)kZK7V)sQ!$HT*PUgkA{)wzUFlE#1hGqX>
za7k>u;a_(!thsB8_>q9Sd|H?Mt~$9KxO&Fs)9=q-e`NnR{xyq@)%j5VL$K%HUZyJF
znzihw{<iP_bzamU>3ZPKopu-0zs%`vy<#AKW5@lVXWAbVUp?FLNp!;bqgUq6-oi1l
z>oN0Wr8!ccpVseDIlW0ze7U1Rj?<5vKLQr#4Fp$T=>1!GwWaQNTEl;X>$xgb295t8
z=*>#KEAH|2bXZHXx%axsJPwSP_!HkB{c?OoWqWshMVpxMw0YO0F9u6mE__$;>Asz{
znEhA#%{%?|b=g9HIQ{(l`|ga_H_8oC8ZRY92Fd-Iy!n^Dn8uVc(K{x*M(fJdlXMnH
z-jD2l7V<Z;`s@CAH>R^1WeB^mGpf%$;Hh44iE~Z~)Ae}=0uI+2su*114E+4te@pNM
zA4g->vm&7MB^u<MvAj*%@9)JQiZ_C~7G7bE@v6LX*W=SQC6&a@3j#tIPR=-Y$mwBY
zz%K*s3BS)iT;q^;&3RjZgMn6<rS^pSWh%}Z&ND>wY$qH%mwtMVvu--;wmIjgS%!3J
zPveTYf2WTt^LMZG=kBlbbg!-F{<pgS^mbOBSvpIef2}FK7xhXn?xNAvnoc|3U27iB
zJ*xdNET1uVl1t_OW!^UgZ<sZ@6lvLVtk7*aSap5bp+o=qR_;>!%l&vo#38OD=N(*2
zyHeipo#?nN&Oe!}yWQXUTmDj|pj+D_y<ATzO>Z=w<L>>d=u+DKe+BRVn2X78ygdKC
z4JY@*+(ifcP4-s`rWsXB`e{@=)Q<bHrY6WlHAYR)ZYnQF^pyP-s><P)i%fK$%<q0F
zBBW}3=}Y;>-g$8k62c2_9r}KBKhrF+i{|h8K9=79R_|3JvTpyj6jAOAm;M&mYzuY_
zyqf>vZvNR*hg81R_J3D4<T7jDmVROH{4mdWlW&s${&zkWz4X0^lly(=hh$-?r<HlJ
z5?eR|b?1B%t~2z0v6PY9YOjdN{}&vL!hSIe9<;S=vUFcBS|TsVbm&(>s@So}{{<8O
ze)DpgCsMa~>b3=c_wQc6Hm~=|-{(zlr!U=c>+_e}wGMBq^4{Mr^js-_@YVj>)Nr1o
zpW{#PrN?)_fAVX#lLPm*-_Z{~{$I=_w#!1P?#bk5Pt5~2KiKqS?#;U7`So!OOC&F|
z{qbg~`FmYVDKT2+ullqf`(ql;uvB)7e=R93Vh}kKsORpvYUM^z!)}G7sU~N7KB(v`
z=G8r!=9=RW8@gbfb3&_A<>#aKuk$Zib)j^(bHVTBuIcrw8B(OCpEindDGXF847^kj
zDCB6TACR7P{t<7;K8Nl@9^Vg5QJqk_Bjd2DnBJuw0&?v7T=R1_Y&dv|O<Q_R&IUHK
zfSXBA1;Ff0j?1{3Bpxm@Jpc5Vjd*XRp48_UlV>|K&J<sq@qCftdDrB5r;$XRTYVPP
z9iBJy{3Bk23r~*TaXQ4-%F87!V&=hKP`qjro5bCFNyWxVs`GLJZZ6p5nA&{G)g*D>
z-Uk9xd9|dMxbIx)q_fIankU9(scE~>mr1JKo7AOwatt;ceDLsNt>EHfj*O4qJM=6b
zvOb?3wZQqiv+-u}vu83=-IR@I&A51|hb?W3{^H7;-Op0D=q)akJbNl5)NQfRttl6m
zym6gu6g4Fjln-5v!3Dmy_M*9MWf?hc!NyUuE?(#<OEWR9zIdi5EX~A7_|}Y*Pb-gP
zWH?J3FYKLZc(lTCwW-uh&bOyqzNI?-Hd<Ia^TNJ2swM`)QZo+Ld1TnDY>^j|y2bIf
zqovHH=>liPK}9X;5VI9vUn-U^K73$d@BiI_g3??$8XFcKXls&|HIn7!>1=tK+EV7?
zWUaJNcH)ITj|_Jejty<%bFxA8g_=o4bE?+5mEvo5WWM50vHkzl`my|uQi-ZZt*-xc
zx{p8l?-KfZz9z#qPM67(X8z}u(_lFysl2}Tf4bfBvhVv%ogXyFXW2LKziU<hJ~RJ*
z^zU9vQyqp6XL8tos0$X#UJ;sbG5<w=!#lYtFO;so-@4`O<Sd;BM{auT?&quc#w^18
z^Ix^bp`BKzd^=hmdAvSd(Z1mR`=A*sM9q)H&$QE<f8@0K#fnl!CC&fAcRnUw_%8hU
zM%Vqo|ECZB`^@UVleWT0X1bK4e7Wx2%u1n?XX6=eaB6ceJ@I7gg15^v*2k|C%e_8X
zJw8n}KtQQRuYSqWg6Bm-qU!}6cloSU;@F?H#-%1;rR2@sUgp+Er^)HOTzk|*dmeXK
z@ivodsZm>IGkR|fJ=YPN;&^_dSN>lKcV>mx+SA%kcPa*n?kSt{`J1$lZqSwzDOT^w
z_xU*)XBW#{iWXURXm?M9r~J43Y5B7pkHzo{ERtS)<NJY*XU<31%9srmeFc0DWlGkV
zh(3&O3qHTL$<%wn^}0JkH<EPP1i!3n*1Ep$MQg*gleJ9C4@zmC&p24P;kKKETg!#-
z2PJ~{Y`HFAA>med@p{*xA~uP6-2J=e8L+gkHCVXI@Lx%<o=;xft2+~0KCBfJ*>%oS
zvRrKL|I;1z=X)LB9lgq>|ItFVVXEPD!H4O`1J@c!sasj_dGOxOU-P-OVTb?h&_b8D
zNp61ot)v{z|1)2n+ny0PU9#-OtKI<hNh>?0B|9xoS=^Uyk!ZZyf8Vtvc;d<vOZYn1
ziER8h<Ct@h@5Z3RP0YH5Tf%28+w9QX`SJXOrmDNYo;Tl+5u2X5RkXr+*WyJT!g=bo
zN9F$7Z)4+lxIFXy%@UoHM;S^d<)t6A+~qN+{G^$<aG=Y6YpwTvZHoo9E052gVefDx
z;D*+=#ZQkn{};c>YAqw$v`BOr`wQMbmhV?JIqX|Db>ro-mpk}){Yw;#JP))dD#d<T
z#C%9k%4d$Xnb_ARZeD>(Gf(IHncWZAAD%eSD}Lx}5?_R4aJP!rz8mh$8c`kT4EnlX
zdAu)t)je3M^_T7E3YX;m)(Ou#G&h^;9BNrFVambMyheEPf~hWx4|RsFO;W$}cw@_i
zmUA~^)fN}OsW{7=z46Oiju%J%p0Ix1dN^pKk<qRb7D{PR6ZVKlavk3)KI01W!tFOd
zRS6puDXYx2znW&4VcjNs>??~g%U!Xo><wokp4z?DH^0>~LtE=@gu$miQ<drRjr&FA
zBv$Xgkiz7X=@b}mboAD2n-W&F$xHIY&8(+B<<LHyJu9SCb+1L%)SZe;eYWJO1TETP
zYRLSAv7!I;B*&+-&U~>sbnLr8zePejD~qI@@PWYh6^B2ph;^|xvU{Pge(lsgo2I7^
zCA*oqd;*?_q}-S6ebc-Ax=O{xiSIWV8|E!ovZo+!)%WWALE*QrJ*(_#`f>52HhZ{n
zMoZ6Wql=gGFZfLS9DL7embi=3j_nope^QP+)aX9Tnse>=;xxVY&6)v=x&ppGz1IDn
zv%#=n%7evr502RIzpUg=?Bwh-ysO53cu`^2r(T7M7)Iq+kHlNgNIkpbGc}bpYS#9+
zKARsqZsd6M-AbyQH|6Bz@ah%65<k8CRp*|4X9urzh=tjeN%Le|R|S7K$#?klqc^rC
z-!?wh=T7KVxN9$<y-hFiaPLKDBb}Hd7gt3Jugux;>00I5?Y%WOd-t|H-TiIpyZ<xN
zZkOnI2POpHJEZAvrF433+UC!*D&89~&s)aolC|@!_4c*<4VNx@T)Q)^_?S?QfSKg-
zwlypLb(1W1Oz_?{RWpX+&1IK^w{JDsYDJt2k9-z+=&@+{r<||e0tZ`p99GZpIJ)O_
zjN)avUo%%!&1NoJ$;YtrVf)_a=?!<>Gq!*H>wNe6r_hZPM7u*if7_aLQtMgD<T9NM
zpX@yjCuiwS7Tm)5Mfb|hJ({|Ortg@%y;n7FYFwwwDzy6CSxW~0tPfY!Lbg7ix+TfU
zvGZzwuCmPbrrf5vmTE@f-%iaAiP_7`{Vgj(YO;lk)Y_tdfA8HouhVhu^MTM95jWK%
zE}}YDRxCfeZK4r-QigCte6D@RsoQaUx6aO(ZCB+H(=kOa{>z>{hnpg9eXip-2sK?+
z8@S@(@mb!5qSq!e9`?$6mB+pLfKrc?)^?w;mPSpcuL6xdxr_Y#4XPG@S9CdYA@|*K
z=YElCo^hSY;%E4IBu$rSPH=b_I>+wFLUT^HMf?6*Bp0(AOZuv6ypCCFIKT0bkgIjs
zl@;lKIp^!V?UP>V_uy5^ld6`$qfQ1Z*GqjkvN!dy^2s+pY!}72#a_FmG$*3%ylkZc
zXI{gpefJcZ7JGAxFJ1CLd52oSJ@4kZPrJ4-N(Nl~cDUI_`idZb34hLlGy6FHoq4D~
z{d-5+Mn3x^vf|sC<v3#wESQ|#^NICJ%UrP?&OWU*?A(r#WfA9Mn^ZotPW*RIXp&g<
z!73fE6~FHo?ND1_^XiQ9%rI*;wU&f0eLsG#T3E7XlG4=0)3?O`e;}ffWcWtz*xZ)_
zV(ZuTvMQcyUGaT&@8v7GTP9fA%l37gZP5rnCY8f2rF~uLUbyt?H+L4C+kIZEqGpPR
zLVsV&x?J0N3srOrX3L2!zMF6Ae~(AkZ-Ge6xwiM;mrY!5_C#cQ+<TRq`|gXg=y*g}
z*=xPm$(+2)sddHejT=f{y{;8dbgJu2pT);u{bDN9S9h6tAD3kCta)a1@zJG6zeT1P
z>{EErnz1%Ha3aTq@E-@x2eF-aD3!k8j+xrBknXqL^WMLZbYFOSu9^Q~PZkbIt{Fc<
zUvabb_}htb7(TGLoysQJIb~;zfr!bH<I?YDMKPc3Yb#&yYaWa4omZ_NrZ-O!zw{#L
z&!r82c3PJ&d+q7tko@4l$KR}{3Jce8*}3ZlCEU7a^zr^aR&H0lC0Ccm$#YHiY3EH=
z>^wYAc*0TdSB8Oi&zo*qUJyNpLqzD7n`r%nlf|v?R95SR?7q_KIWbD^p|{7*;OtLL
zHxm}mdmmsKB4;(}=b7Bj$===C+U6Nwp9YjMIfhSlUBHwvlQo3*c&5mdN=KG?8r%F#
zN@7{K8K<;eJR=u)BTr2(Tv>4M;?9**4VTLu-1*MPK=JE8huDcNjUOj;HajiLO?ADY
z{D6xu?!dPPo0Q@TvUZ<p6kY#%n~ue&>?^I_(>2eoyyx)z=hwgkTXM9I?Nd{6R?&N7
z7Hj^Ec|+6ps-w@DmptJuc`7cF5^lSO`9rUF_0(jQzI{q3v?Fxa$kqzD9(eYk`-g~0
zeZWqA`&W}iO;@Gd@G#>r%V>GYb<A(=(%qIv7tOtLQi_#_PbfUxBKGr!n+J<F^nK0?
zow@tUs>{a&HY_Ut#OUn5M@HVrVJ4fxtEQVhMH^0Vh+JVm^^jR~YU`F*rlc(?E#fW*
zD_?yxh|A(;Zizppf6RB@f1kh`3MZ8UGdZtGPAg_#`1o6ZxPa!@Pm*azS;Ko58g*@Z
zq~MkN`b1)aU|m(lDUpjS^)AW!$~<4<_C6^qQ(7zJlgoO;w{^7>MVlwsO!)psL3p;V
z?w5T+Z`WAmDKq(eoO@%{9`)m|dwt$~I<@=N!F|WG3zQh{`vf-6J)`+tM)XkKiYY92
z<1fE|b+D;+TXEEtnh7h8rz>t?6OUZN`skBMSzuA4*M^JDS0AnQ*tF7-t!4J-h;72w
zSuqAbg_D)u)krQ7$?N{4y`!XUahI_2o-YaS_Qk|K=;3cId01k!=9tO5-t3}lYkfX*
zES{pAv2*E~i_C?Y^V05q-6fl&bBdXDy7ufgCbj7oReYL^41E{t%!%62To`23KTE7I
z`HwYYWN*PghuAlhoBkRuIbUq}#;8-EN&j<aW6vK=o#S~^wwNaEou2M_F>cwLxyNs2
z|5-3~&E)pOOZoG=kFvceymyTMLhth_nbS7-eA{(#VSTN!o!E*OhWgL^Ezfdq-*&s`
zR(4M4!Rzdc40ww6$CVp2Oks9p{2lZpHg@0oO?k=;&+Jcao%>-WL+h^mmpOvfx7QtC
znRx5dkGJyo+TAu^&i;1o*^JrmxY->`Yp$R4y7_dfhrouMV}Ji=Rj_`&72lC_VZ%}0
z<m!W&ecPO#=>{k^|M;52&^&G4_3CDxptA~`O0BOh`l<=>{*j*h^-ff2-c-FUuh+J3
z3y#^pJv3pml4O*9i~Wqt#o3GG9XUi6l`C4yI31`j$xMD_JpFvb&+83BL3Q${4p+9t
zs2}qBaAZw-cBolGv&-M|4CxH58jC(Tg|zjT>$-%N|GMt4sd1X$x%o%UP7U8r%Y?-i
zNPXBpUG&Nx`CIzxdnH<Gt=VtoESjACGN5j;^XiF@lyeL3bM1Xxx5Hysl)?wK;8_lB
z@(jy65A*qMiJbH^l*8;*WlexZ%?#Zwyc6}Tx|i^AHT~Xkr{uHUv4Fb&Q+5hBidr4D
ze)v;r?|H?0`b%34=4!Co+&<p)@mt}F-Xx_%y)o-5?60o~*dOtFhd)PBMf&PXwK)Q7
zi;LMJ-$msach7pm@h2y9M(Vz9?=KD4>fYNir@wwtlVu-gAa!!W`Wk~f$9QjruYP+f
zEh29xEBE`4Q@)8d&n-ACTN5pC&&=nqk6E_m%}T|Z%YF~`JV=SsSP=hBQGbU<`)@X{
zd*#wv*Y%x(IDT^ea7p=aVqZ)4!ELwWohE-4_?l?uv!?J9XVAQh9aCBrn##VeD{a32
zD{JS@$=w+p(%xCOmuwTvyK%qJX4#K_dNErj%!~+|dukTT!$k{sJe{SooiYBWe#7s_
zNp;Z<e9ySzPR?I2Z_ku9U;ZXV{@fTI+41+j)NUP??VgcJe@|%7)wL50DsB9}#C%`u
zt3s8fb!w;g$tCXGS~5pvlk~Cf$vdxwzEDjp_cZD}dh#oWq@H5AQt7+dTYleAuidcP
zb-~#a3+_F7#c<h9|M_(80&Rw!uU=Jb<H!}2i<z*8XWPSdn&Mhp4SsH#y_}<ULfcCI
zwI94s%=FkDUnZ~4a`^CJ@o6=u13%Phd#$c^^vJf8Sgf?`MDwCkL2ulSv1@Gkke|K!
zL0jUh|02tLUT^EGI%=&lOP4)l{r!B+8LOOXPmA06bUwTzt8}=o`heP^3#S$csAy@=
zt+QEs_rOEdw=;ugt=>N2t*Ncv1ie2ddhbdLUGggY*RyjRsJZpG<5s&;ci=f=#;=<v
zv&{VX(0}Hs-xoDRE?u$VnyK@pGrlIqDzZ<CZQuH>r~az!wW?)$TDYTEcq#***Seoe
zpQMM%c}%|gY01{_XC)j`A8uTi-8^&pG0lSDU5)qKH^=PVwPB5r+Q9`WGNH9A*za>M
zwEt<p>CUQ;m2;==S@6pE?x$H3mI!d~*cQ=nZAy>o8a1WT<uiG%F>ABP?nsaNTB`nf
z<*Uhumzdbh&pa_}zeQHTqpY6!$>z6X_4^rIS|5F9&s*5|O7n7+Q^pL=KVlWnXV%T$
zp3Y%DEB8?H%Hz6L-zAR8ida9=a!g6PY<>8a$I06(f4^P2CHtjTh;qT%>IgZB_LM9G
zyASTaRIA0-t@W1J8McYLb)Ap@x_eLVKG-mGg>vJAv!$hTTN_p?t~GhwwQl9q4q^46
z#e%)}CBlp=)t8neet*E2C#u%SY#%M>`>E-Xo^#6IGitXNmL&>4{Qp}Zz|A9}<6YSC
zzxN+y@6`>yx8ALK>Y5kY%_@zK(W-}jF{D;LzkmA8TeDNUjAarE#7)8(&g?N;cX5JY
zlKd@`X&REJTKpgT{l8-Q$+hZdjbzlqrMH*YA7cJ4ly=e3D&gqW%2^NYAE?xslr#6i
zqEh+%o%j5?++Dm*UoqnqXcJ@Ib7$@ztveTt!Z)<^3*R~R(#%_xrz9mtRgrUv!rZhv
z<$89VBDoUNvP*u+N|Ca94;AdEuHpZ@wK2JH{}N5F-@AHr_883372`b{vxiCjwAIhU
zvFFRmr-fQgu+LsmT7Kl#cjoAb{RIbDP2Ow>-*`8*Q)z?mms>Lz&e|8h=XmX2(Ft?%
zemo6ZaqV_aO5xc#PjB66N@w}J+0l0Il*0@^=Fix@tiS1I;0nfRtNz?${KYSpVPVN@
zaG}IKh?j%$)+T*}6kp+)TQg78c)2XF{w0vj74F>B_rCpOs{_+B*Z3XBOQY)RivQH_
zD+%yW%aZw@WTM65E0ZYZkr`|6a%II7yTz^j$w5}BHO7Z`mTlN<Q}D&q(<JNlIfkgW
zzc)5MUUbRr5T9hg)1SwmxBhFI&YhzEXx~do8`F?<15w#8Gg|*$aP+)zQ9bbRbPet>
z<2?m_FISbDINcNHx#MaYlXPK{>q|v$2QS|D>WAx2)C(=W8y7Ks>BYxKUTx|NIBDA~
z98&nY>Gh`PiK{N3a^*Y59KA-b#_9Gxr`xt}wVzTu<Ygn5t+ncv76{$8&(?+ahE{Qg
zQ~OlG%tt9(1UF^!PEaxFH}kp|>GtHO>HHP*9hSDamKAz@UcD!M$4RH{3Q^2E*Lpvg
z7R#?E^0}tqn$M98*#?mSEiU1&HlhJ(k%=!ai#?myY!<5^v+POx0hJ}J8b)h`6T+{o
zFtO>&&ucH(_JnbdsF0GzTcL0LNxo7N3M$Vf6#E1Bu$gCG;rz+#Tq$bwU$C{2W1G)=
z3y+$Wfn8Mx^5zsbycU{pr(*i@kk_pTWzxE)Z|><1uFg64;CNuPWCBaoAs1;+kM|c0
z(%LHy%v7pK-f(zIj>{sJ<hNW?_jU6h+3>vA`he4#;;sd!?%H^qI<$6K_RYU0h3&8O
z3f&(jXC80g7Jc>h#aNw%atm!NH62WPcDwW%na+<p<#@DS`GFssDr<q#`trED1|``E
zYipe{4eJ7nY7Y3yRrFQZ2`)`Pl;Wg+ten5T+oW;FH1~HO`)p2$EI52_=Y-1W7X<;k
z_v!E_GCnhm;j20pHS5j<=fdt+w>3WXeQ!PU#ooj0lyiWB?8aZO8Yh2}egAKT_rs^3
zH=GM*a^@6kI%nbNnkQ|%-<)TP)w|2t`)8ii<F?FfxmUKDPwj}P(XmY&_D>pr-fk>=
zcyj5RFE8)kmiakvcWlAg@PpeQJP2N~=FP=-uNZwxmBJpJJ~qREKP|nsGH2(Z*ndpX
zV#>YCwnVM6Idtl?snqA+X#rZL%iX_zd8sma^9tS__n&OmJ~*Ri)dWSsPvRoYD!Yv5
z-r5u!^<fhKnqc;IFTeb5ZFq28;lpO8Y0sWE$K+?b&OIIZLu0R4m#yFIh*wJ|G5!c&
zd){<?y{PIDBTm+yll+gCL_T;`EHJ;yAiU~s`7`!Emk%9!b@1nlgA)VSpUr;Y*s)Ok
z`scUn&a9hvou89)UxL^6X(bO2guJ`(M`xo_aBAg?7vZU9@&Rjg_N}Rl{mb{d?bPww
z)+OvyU#%7ued%O3d127aaEGr|^JeJPh}=k$s9Nv!d&3T+stZ5<l`p(qeRT7MhSbjL
zFH+3*h0mAG-gnRY%^DYORStb0Df`A{N34AwH!WErmM&y;*?Afp&+03pI*)#ol}A56
zu(RH9k9@4|mmQDYyEDxlnwj)=GU)|=e73sodiDJU&SLzRq7@?_cg0?JTHbstYLPR~
z-}7cRMcJ=xt_vk4iXWX<wbl9XqLd?3&PYzVBFB_%AH}_S+b`Z)#dCp1zMCuxvQC^|
zoA-N5QIg~n)6(+JiGI?hrS+lvw0O=l+NL&dk6yZ6Nb!W@jC18B)6!oUedk@$!<T8J
zQ(U65d{=;P<gx?n4U$}LPQAO~V6<+`>CI=j*-Yg39G;RmPjK>rgNG_MvZO@B<vfo4
zx&EVe;POvO*;V$J1*~)&KE7qlydq?Hht0Xk$>wK5S4Fenro}vU(^fQ9T6}q;GWFca
z^~%>i>MWl;_tuXu+2+3tQ_5eo_H1mhSF@Mo?YEac;cIY?(Jo+rgJLtk$+qW?ifZ3(
zEy(ITkRI|cZr6h4Lg^=)%)BLA1?MdOaOy+=$Ikf5CP@W{?FU%eUq~jq>|k2elv&Gm
z@sg?aF8$1-GAq`*A5ni?BfVh8E;;FV#>w2ClE)rwb_^3MzjgM;FTI$<M-p2u-AkP%
z=bQM3vF*T=m`PIa(&l@ao7|~Xxx#UM>znGt_s^`haf)+j?kGAxRmOE*IlIWT6I#>w
z7T+?m6h6W>`*OzhT9IkxWsYBbr`^^4T$OL-zMDBH`nq+j^tFtL=Vl96UajR3?G4?_
zn11+4TyD6;Eg4O%RowoCj9d5Eq)7bT-nvn3VQyq^&cefOmrFi=*4m!Pu(qIBY1t0p
ztktRir*bS`7kNyy_k+XV!=lH!T!fZ5*EzjB7dlOMhfGjo6NB9Eteyi7;@8Ev&L>Hi
za-EVg+?Ws+U~O~cWy_|f2Wk>qUI_|3i#yq`DQR;dzWPaYLTmL6zBdldoC}})ReIsG
zF2B^$X34@G3{t;mNF{hFd+&;kyUp0NV24wA(S8xRuvG<`g+(uK-Qf#x?D*-#>t^<@
z`a?c<-p!+F{4Z)n4mrFJvU<qMkoEe;?M>&oxTY^nUy*-jS;3?S1=+u>XHHXmc3!08
z`9$rzT+K@&-mGI%^nAQ6`ob-XZJd157c6$+J1)p?v+(6UfkziPm>zDmuy^i%<dDNT
zeX`TqLya=0mTX_R?1$$s)fL=!x4%}WtP*#*xgg2>iRgQ0$CAb)DG!~CCw%Wp_1YR=
z5;TwDmqNIIqcW$pL0M3hLq5-))i))Dn*-R~OB#ZUEUYh_b$W5`dy^4Mi`cwBvCC3L
zb505^IHEfB^^V&;4vas)^D~Moy(*kJwe`gbw-5Jixhi6#H<j#O*7=a>s?p1t9@UNq
z-hTI*Y4rQd&u7VZCY(>_`pEytWnqo(!@Uy^Y?6p#deNC9<~1cS^2Md!z1usDy3D?;
zo2iyFGbxcj^=<U-n3QA|KH;ZzkMD0|@0QS5cy5CHUkhU+=ev{n=Y7b|d~-b^D>YAW
zYpS=xBGqZaOx3|hrq-H&^2+n;JjtRd5^+b-?*4q<EX%^Gjo!iSKbNsj{g^4jewvei
z$HJ@1d8u6O?@#NfY{)$6?)a(iQR>1k2f22>e13Am#3`be7JIP0>Yn><$tImQNBQF9
zf-WvPI4^L!!ta-F<ewDqN7SxUdA|R`+HQtF_x5gaHMWhHtJwbb(Y@=x81EMPd$@Vu
z_O}<>@N1Ubm1SEvb(uq2_B0yR2&(RNy0+FY^sSl3)YhdOyMIodBo->PT7PeuLSSUT
z<U1E8Y_{}fX_y+Zq+@-Xtm*5m`c@8+8aJP^FgG6G9_m@6?HR55*CSeZZ&}zA#}5VJ
zmita^mI_)bSG}}cgf(}!b~9I+L(Gl>*3)XI4|#64n#_FT(CfMPcg*d)SZe99P_?f!
zpv@_AS@W4Am8<+kn9quA<D6z4`14Z1bXkwR)+T$|cC5YNvdiPeiRCJ5u6SDBk(_CC
z?92?gnN0haF0^=T`Nw&cvaN<8Z~2)I%vE~N9WQ6h*&Ht9yK3F``D=6n?(X<Mx8$*{
z_*~yhtxJ9%sut@lkX*bzd&Z%BRn?U$hi@(Un$YX$FX(;Ob@{5kGXl@5{WyP?_OUX(
zKl@+@^R?!;TLh(A8@C16IY-sZ=1|dLH#(iL?0UIdn8@PjI^7L-53_XfJb%S>%lBzs
z3S0FwQKm2NVlOEy6$_F-=2y$X%gCkLH2Zbp+`E?x9sbSoRu8)NTVOJa+*7?TcOyf1
zFHcP6dYPu|Ci^gb=IPc+AFs{bt5?18>wFm=BZ0q@I~@8GFDX@YEqZn>WQ`)vGp0{&
zlX*37Z1Gy@xi>GS!?eM6tHHbY2wCo6$%6~_I_wEis9$_zUpL1_t&p^XtSQ`UA1|I;
zC(rTbTkoz7_uutDJe&9`?Bp!_9~#Gg?aMr}@ooCH&8`2k&Mh`ET2X#FRN65+dbw!R
zk+!9c32PjB!aYpCx6Ye=eMgmH&(n+<>^vO>Vv`N?JHxsaT3++5*xKj&^0i)fmbBIz
zhR(!=KW`k`;bO3^BRQg;`9HUFe_DW-z!6ce+kd{r7zm}E+SPP_=B*<Jr3(GO{qMJ!
zowtYzKJ`SdOzv19Pt`^4?K|e1dGN?<rJd;6w_WZ>%3r3fSDf1pti5oQ*YNBkEsd=5
z?{9a_e*O2rE`>Ph6Hanbn)7sA4DC((&Ua4?{_lR2;mCAvt+3|xl>*<Lx4+nS;Ew%-
z=@Y{KwlM`}Mz54pjeE^hxa)plz-3#Dd-pO-RU59xUg&;sRF3J#!RIlr>?%EOzWdy`
zuy4}sc-M8M8y|=)V!ylI?AQmZM-xxY$)2aEB9?kqa>d-^QwtAoY-rNaxtKa<UEHeV
zyINhG$x_Y%S2jwiHeFKt^jqN9v`W66Lg7jEFF%@`>y0@a^|~qf5MzNuMV|bd)^DmU
zB43zaUQ`HSURC<ibqD|YCDz<5OO#4>{1%8^xn|n?hmzb6Z+~rE(Ean;ug2-!g$EZ{
zyq_X>Rqeh-#NA`U>(iyOMOFx}cCf4Ln4-`e^S$we<+AETZH4nC*V;eoes{LvFsr_~
zAk-=0jsBSe#i?JXy|ZqqwcEgUqKjEUpzPb`@83VWT$(%UXUD?Nb55s=J-f79AT54Z
z=gtOad$FAJ)~3hp8d;bG<2Tj(%~2IFdcN@W7uLes?3KG(zp*HA+?C(9_g$G>)y~X!
zvOde|t~&ndlQ=Nxm+F>ZGh-rN?zth=$|>(*{JHh2GE>m*%p$e9GhaPk)3Vd>fzM94
zT{a834}IXjt=Y3<ZTOzx`xB;3?%OLLbn^3E?xuhI9({Y3PS|Op{z~!qyi%hVwkP>B
zjviCE-Kxy}UhPR~V&W5h@jky(wZf8&iM<gH@63JNBQ@Vjsw}Nz*m{2UZ(qih{4G-_
zNH=%Ay8PBN&GD7^4w0yZPCL!^%3izq`fFTh!)XDJr6J2V&n<hjkFU>dxvhh*^fH%*
zU2gNI@|WmoOi6W&`my0#O6io!%?CeyV9DBd`RyYesfQA7eN!Lh7+wGFWw-zL$L7uF
z);C9$^D92t{kI|D$&C4XyC;8I)Nxx;EGB$rW21vg<+{i77kR2iuj0QW&~ec%YGT{#
z)hD;abmuPenQPXgsTDbs-)Ct|_l&K&c6naw0@v?Ze{buY8&z-b$Xs#neWbvDE7IKZ
zl#-slfk+_RqvaXCiealROFo$Sa>ui2%2wBzq@B*mt~v8iR;~Mp_eqV_n^Q!WEI!4q
z#l6RGiCCiLtkmUw@xK}~nqJ*7<vuOD^pj7Gwxxh-JoDjwrp#t%Cuwz7TsK--oSd63
zvr(gQUr76^scR1VA2}qh5mW27X;;e9_M4L~G*s<nd3i8;`GGK7!QFai_%oswRo|L-
zLhxTpH^*zw2bwKcD~b{$wn%bI82;RTVKFaT-*LsTWuonCjoL*Vk{Tb@8(eC6on^2=
zs;b1v{ON_$9Xsx)91D?;V7F&{pj+lS>+Vyz*1k`A1~<5_n_myI$%%?M=eC!7O(_4N
z%~x*!JfyUC<z@MjC0_UZPgZ8$5VTl3Yx<9HPNnzvgf6>kRci6pIXscP*!le6t_6aZ
zT3f4r3LZY@zl8bY3CWEC^L+2P`{z#d(EO=m`$KDK;Jit}mwq(zUu%9TC}jJc)98_m
zQ^0OFR*lE|zOehR*tCQFw9xj9ccHsCuAFc|j?MF7p2gpwxlWJEz6Y4D+7VQ^S*LMX
zr{<KN8G4+nO|R{33fz^QG&TE9_AI-C56}1SK4P5TE0y;4_59Nn{G3Vh5Aw7#yJoZB
zp8M~7#~alLlWv^vKWr>D$5+$UYJ!OPjg5zsTV1)tm<+3#f9_zZ)j#yy_`0R>JfmvI
zW~Mk>qgriiwZk(jIkH3)-Lxtf%3caNo2YYkUFXEzKNUC6sk(KPD<y2h7q@@%Ecvso
zbYm2@Y%a_B;d*aZQ>^BJ2|Mc8pD<e;|G(1ew9A}Lhr<~XH=giND+sqbn9OCWdt3Z;
z;zpC!E2VkGz2^l2Pn&VfIA+&c<nzGlVe#C|6WV*)CMoaTCVyW;`cU?DI}Vqxiw|+X
z?s)1SFd-za{@~p0E8JIFeCJ_H6l489$9VlV|FwRae0sOI{%!jtAeJDupw)GU1Cz+(
zyZ%p7f}XEzJMWXQ;tGEQ^U{ghr<Pq>)4J1vSNp;%Gw#)eXIIzdo9VrCy55(h`RYr5
z$hjBiPB@%d8S*0X*UsHDxVbi1E}3ShEWNj^n{ySn=G`4?GTIya4el^s@tyjM{W|~a
z&EHoT?KiV|X@350=1Jy_zZGL_MUFFD2yMN>r?Nym){S|4nj4?ZCBC@HB~D>;jwXe0
zY*YHa(BXBE`h~T}gVaPEK5m`6i1k;CLhaep-YbkBooHNe;IpQ)PECiQLTR~X%L$PO
z@BXw|<<?%5eYI%L^I3_aa?<}_-QOcGcB8q>^3e28-NzO@jeH=?{y6o~Zha$j2l-?N
z{yMH#9r{7G4m%$n|M9Lh!#X)J-=gJb<CkY!xqS+pLo)bvm~Xkh({@r=;B8m1qR2)g
z<KEWZ&8|!>&mONamG~CFLA|L_p?&w0{JXcf*#EVKGhH>R3x0dXR&{@Qh5p-fhwHh8
zu0_P0^IUk_<HFizc2TF<+}788)EgOPIqWa|c<@*tAklKt!$y|IZ8y$-7cZzOux*b!
zBpods=b^v<qvLs*h;~aWlOG|=e1F@$*>J$IrANhj%Bhwh4Q`o63m(~H7Yk)(ureQ&
ztbg*D$tmMS^^TBbwj3=5)7uv(a`~*E@a%%go0o3!xlT$;E>A5gyrx>tFqKh!i%oHY
z(|-%a@7<=dLBHlsIKKNeAKUBwr=p)OxTd)OpHr)n%HMbosXvZ?HBuxmd8SUBXy{ZU
zcy7UmLjevGc!XBeFK}UOsXi8!bNJlDZEUwYgjCe@6^#|Yh&1v4aAuwKY5l~Ti*>F?
zl+8V1!yWYOno`oA_J<z1zhihS)@g2IUAyl<S?z+Xl!em{Ui;Hm&wpZ`{EM`sN1rXZ
zU$|5*?9-Cz@!l?Vio!~MvfFoBy~<<pYtH^r<5%jr+pqbRmuu@{KdtE1$!&K<(-uwW
zQ$4xW!NW1BCr5aS{#3Sit2Jf*hDk=bzv9b!G^J?oY==A7M9#^jKWdq46KLt@dz3No
zru8)gOWuwxF%{M9cCjb+=(C)QkB!bfZnUb-A?lXmN9H@htID@Hy^0gN=X4@ZXVVw!
zqSs2>cCJz}e7T74mi6T(m5V}mrS*S1y*TL9<2OT5%Hz>8jYTQ$m)34-U0ajJ%cT7@
znS;BK{pX9rt?PeDE?ux_&1$o?vm6gpKA1AM$A#}(Zuoq2fi%zB4<;&C3ig*wJG4j3
z(|E<Z1FUm}V`nV5uXs7G|Kf!Inb)hmE)=dcx>dP!!dK>XXT>|OztNgJ_rtYpQFZ$r
z6B*JJtS_|6Tzav|)?CPjC+$>T{kFMGU$)82cF2DFqFbh%qfzgbzRN>VwZNd|{qwk<
zzO3Qit{Hsz&dZeE?^BQMYhBN5x_3&;@?)=mX0)_h7nXS+ZJi{(>sjgc35Dm6|G2hj
zQbb#N?|$w1yT6Ni@yj%wn5%nv#~D}li$A`;td-o^DAzOL!QC$#xUW_2+{V~jxb5Su
zuBU$@c)a~OPAD`L{fzkXc+(I2#Vae?jPp*P7d|>+!L!+aB@bRvxF0LF>+sttUG35z
zJmv*zD?IvP)0)3LkyR^f|CRZ=?H4Al*mC^Sl*hHsq9J=j*FF4}C0#w&#F_PlLXK=D
zulT(~bKg(UO1=A{SAIgbRpgFbfrt-Xx@#kto*asa;db#{a8<gN<>jN=Z@F@f+YU@K
z-Of?`YR}WCJBsq>^h6(B_`Pq7Npx=chRovc(&?&lAE$LL+<4~Qs=d3*HEr)F?)|?c
z>_+S5({pwAu<!|ee6%ml=(uiGxWCgZZ{Abe<ir;~nZ>OtlG1-_mc~QAfOXp~eLTMH
z<JjBabVRj#(MPS{4&Od~58nLXTDlB-hlmc#kr~UcnCJvQadVcssS%Xxvm$PV>DLHJ
zYn2)PlOzi+I69Xtxp3gP#%9a=%Vxa%t{|Vr`BFju9>avjfG2ZwtD9!5y%>6fbE2!H
z?DF=rNeo3gyWV+!GcFWwDep}>Xu86A-*&ddC-q913{MuwAIKCbn4z0F>D>egGcJbb
zH;Q&z2RXETKl?*bAbZN=+J?*W{vn6*rXMRvpKz}3)3>Z0f2?%5K25ymXmGs3(c{}}
zait|6^)AowE8t&b-JetUHp|oVbKOgqUkh@4Z8=UXVD8eH)GfYb^7+H9S2k82e13Z8
z<T8PqzTtH#$CvP)oL=3vA?34cnew~2Dve7vRz~!?eSDIXDKY(l!Lc24maMw@Z)#!W
z`Dw3>jz_3z#a>%@qO6?ztc+yU;y{(ZiGoY5UN14%Jg1!C@NDOn3Ex~>D$ecP&KkBo
z&v-4{O^Mu6Y33)IxzAR<EB4IHa!%>(Je8-wX;@WVZM9ieV9KR^7AKuM;`TidFo@hT
zWqGLOlg)B_T4t}`c=1Z+yM^x0<D}0WaoCW!@_?Pqq)I9EB&XHJ9(DKJI3`FZa`gH)
zJvh}bz2WSq+qT=cdY!Mi6{EeXWa*vodXXugY&cyKzD-Gd&BAxd{SMPV;s2?vxAq)7
zzMK1$M2AW%(`|je{p#)-!XLSR8br>}u$0tkSmE9JT4iI{|Kpo9$_?rdSbz8$?`!_^
zrTmi_XV;#-8<Vg-cyGRjWI*H$4dsQ?8wy-ZL)ijvi{0hi9RF(K?sB<^yK5PYI##)T
zn#ptJ{aIP3`0~liXC!dW-FCXbb4~cpgG|#CZf0ysnYJzW;M=Ob;<j^7Z>(RC_N&aw
zGS@cnsC}2@-Cc_R{_;&}O^wRoJM{aQ7fY@E>c=O(aQrk}`L^zb!fdfyoApCN<QB8L
z%G$)gmfD@Nd{x67+t9a%uX*3@ueIHi_BBdHHP`V7_vPa%+jD=d+}b(i%Ps4S3$x27
zcEpD5k~sM6=%M+7Q;p}YSp8YziKw<uXVLAlFWd_I)U1~1TNssb`Ia+X&7E7Y>vrxv
z#-CAl7QCMOCU8@xf|Qles>}Kg;!N=;1$f)d7_SJ6OgY2xF{QDvh_y8-lK;<<#_LI@
zlg>U}R3Nu{d5TDM-TjhF{{N@&cpf~tJXB0IBk{}HPY<`Ou2Fu-xGvsG<@pNk>)#5e
zRQc}m6}v3+B2(e-wms_Hd6qx!9-ldV?zJaAhYHV~coY<9#r2wb_u3-$sCVoaw?$<J
zxb@ESKJ`^PEk9e(J^gB;;GtJ0=c+a=S{eLjJ^v?`#5Y@xt_kLUBz5wL*woE+2}%-w
zem7=W&T#VZ%-h8N!2K$7rBUs<oSdfAo9`v0+IL4N2xw0Ke)5F#>PJh0*|HbCoLMTG
zmcM+DT=(K9$4ve>>|o|}P3k&v|E#M)l;sUg-{*F^F#$LAuhpH*Sn}#=GmpdK`IlNJ
zc^ep2UA=qZ_`a*LBJDm~<%1GKWvBF8NoO#7JGdY8ZI~DP$==)aQqPQ!n`Yl}>U^L6
zccW>6gUd_)S8A%$U0jcR)?CZIyQc2+#B96%xyM{NESFuFxA$VbJCjkq?*H%!VF^=S
zcf{KMyp+@@p0a$}G4{FNJZ=9s)tz{plUpo2bM}G-77}ZXIqrt$<$k?=uYK0@4X5{}
zIIdXOWW7e)We>{+vw!C{C(T^)FxS%i_O10fUow}bF1)t??yOm9&Y}hZpL4Yque@bf
z_xgX-+!ANr;-sH$idR2XG9|BGeEYZRW4Ef-vfS@pYvc33JytoEVR%R_T3Ywrh8yp!
zZ+lE*`K2o(XZ>lxm77-LazD)+w_2W??e%9{j&G;*l8c5L;zOe!r=3~0+%@CQy_v6%
z?EJE{PgZO4oZQlBIp61Q-|>X+()2sgts84^*Y$6G8^jZRR9km#$UL`Av7636_cL<2
zVDI6TpgAGmokQiXVadc)6`_4Cub$6)DsA&QRr{A?&2`qL@$<Mnw%4g2hzswnG2Oj#
zij|AkhI<+U6Y3V4Y~Pfq#r$05<%?eL*WF8s{>D$0?lYJeXudOIlI%IY*SQDQue<Rx
zYt`JH|B6cG-M?*g2;J1r5cKO$$=S|>Td!EZDO-BxGRN*?#|s1_o=%T--@jt(x88$C
zm-9EQH$AoC-oghH4C{4sFa0XmSGx1ENnLG7eRl6|y|=gLCMoT8PA)v&5S?#oSM+*r
z(~ld8yw*?cth3D(j#Mv-*pW2Jiuco=uS@SPPn4HSVc5Ta)z@`Q5l`HHUOW7}t)TO|
zQRTZEWsS=xZl1H~*zU%U8-piqe^Iyjz~}11k~J?k*Rbxp&DW@R@Bt&wo`hS6!i>)U
z$$p#5^Hb}6@Wp@C-0Qq2v_2Q@|Gs+J>r=;KL)J+b_EdeU*|k8S-RAYR0`*si)wKfG
zeLq|GZDT@An|ZvQaOcyAd(YYA132wuT=X;qk8kZyyfkAM+nyhbg&)QgJdL{P(=%J@
z%Z^Lal9OB}N7S{SUy$7Y?f-45pZBC<4+#ir*55eR;(E@)WWu$Y!Y_AI3+lHoFxXvw
z^{kO+a+8J8=3Dxmt4jE}wtt^jp7G|@rsFT||1OqzoIUBavB|U4rH%;~Rx}4OpL{8C
zJ4UmZC*EG`^YYVzb<%5%<(eiu6)r0BP?|JjgU7@hH}=(U<a(r|D{A8KZuLub&l;5z
zKNjq3*?(s9^L-OGnsODUobK#qteLH*9w1lgIw5Gi)V=;NpG~{8l7HNPzv;@$IciTO
zqa9j<-yKLTKX53hSzbWp%8{d0>3?eC_Pe)w?2SMDX)aIV-|L5#)GW1Y-g<@qh+slS
z?8KwS;vyft<s^zebb4Q!H?{x1+w$$Fg)Q^W2KCn+=MLAmf4E}l-091<-<g*w6MN+$
zuf)WpwVEew8JWY)Zz^V<FZuk2A?_%5Ci62^tIm9vd7Fj8b{VkwT8mxXGQmsqMf}9Z
zlg#3ObG5y$Mcra{X3S-}Bi#`H%*k>6H;3(J7dBVfGKlRge6lr>qdD~C+=ptbuCsSs
zbJYk?OkexSrYEwsK*C;PO_I)`*8RyV+Ql^8cij5^!r>`%?$@V$x-L)e-I(*wTRw$l
z^Mj&^GjG+EhELdO%==`$Ud|rXwe54BWVe+Iddhxtt&IA0$hiK6xZUpnd;PD6e*VA5
zRrf1>)=BxJ{cUy37Z2_iFMOewd;7}Y*UK(0ds2Cs!JpUX!s>U^b{cq`b_=^RK|{v3
zJm&f<S?l}LlMgkuy?kY}q-bAn)pLe5G2F*RFMBk-Z<(}=eW${sH9Fc`4K{wyUU*OS
zH^VA}i5pfe7ht=Z*W?_oVEp#Dy~_Nbn;3c~Uz_q~$-8xPZ<>Ex5_|Z1Y*B*InPO(s
z9bSxL5*^Q6Pq&<q5K*sO`*3E_g;vWe9~ujPRi9q`kk7(v-z>Y?6V5n2*>izw%ljas
zTLD|G>q{T@tT0xZ>(0S1tit^768p6qTmeRlKQh>^+|Ai&x8cC|%OdZK3??bfzgnO+
zO|G!fZo|I%+>aNP-tW)euHrbQWw~l6|C9rd58VC|`gd>P!+HM?hE{7Y_&D>}#S&Zo
zsLRF|cZg<P=hc)}_b62hz998}JInWT7g{8<JFZJAzRbIPqEy7tOy99~`WefF+{?E_
zB|b{~Pha5lXF>Sg)v>}oTy0SXos;tC=A0Gf*`1)X^xb8j#8;bE_vC-Q)x9+{``Cf^
zdwA2IobOtx5c7OvcsHButZz)R^0o(`ZZHpCCb#nA4dIS0(k$j@3{qF>EIP3H`xT>u
zzuD9JOx8EndbYJPPm;1<dw*M7>AJ$nYb%a!+NX6bbcT%5;jJsC1>VR}vdNBGw|kBE
z`t)rM5_kVz-LRuKdofS>^zyfDI-b7q-G6o@$|ibfZ~3xh+Jw!A^7c&Gx#_mi!%aC;
zCM?i-uUp5Rcf^5J@pWSAfjChOmi-N1_E;<JS#aB$@BGo9@x{Um7W-_N|E~VY-JYqP
z<+rwYWPUUd-apS@pgpimV$ZAOhu1BZCdYg~ykF#xkA~CF$u?)5m+rby{=h=0Y2$Wf
z0jVpIol<*T^qYg9Y9De+y!&=xX-4&vY2sT#v!BZ^<Cr=n-CC(U>|2Y{1RL>HTWwTs
zPrM{GYoRkMW3$-8ZEq^?Cun|ZSd-ppIbD2;Ugx9#M=qR}FZ%2xdH7^z?7wG0*>U%_
zKDSPm_nOzb;8w!h4?VN~NeQyox-Yua(&+H^!I4y_!0)<z);sDIU&XJx^7GSYzQAKY
zB!4DtjNUad>|A3*?4rgGH%+%#%x~gfQmx>(AYnV(?s;mTdEUxhe9szsAegbwSa8>*
z<TWPg?-}#IcX6L<>G-EHLA+*is%Fd!ZzY?Q$m5f~elOh^axQVD`W5fV4qAyZ?^zFA
zcyLLJd#B5@diO<>p4a$^8(O|lI>{awBxBUAtYHw;wqnl1$=3d*>Do^lPg-#wJX^nO
z^WE}&ZxVRY)Pmm4)3EcuaMRAw;pWt4ea*hP%T?}6U2{EC=ga!>w94<*ne{&lxKBFn
zShzv{honQsJJ#Uk`umuitO9O*c{6EOLS~|iZK{}lWaiH1wCa=0wM&EYpTA$asjPf^
zXv@hfTt#1Z%sRaFZn(uo?=1<L$N#!M^7YuG*tM(uOxBkdXUk6>aSy1{QFv=1rLs+`
z!~Dk-8D1_<yTVIrW#4d~ys*DQbi%U9Q#B^5ZjgA_sl3E*#(jA{slST1>bCsn%;fI0
z+q-YqyUR=J9|#y0?|-nwH{UjPg?o_f`HtW2$yZ)8N3T%P{;H5tzfGlTLVwkTxn4n^
z^*eaPih`a-bltpbJLPK_#|rHL;~l#+TK2kXe7<%e^y3VVs2BD@Q@lQfF4apudw>06
zIh9v-np2P0{=0CAPefmJC+Bkm@mX~Wc~cY@hD_J4>)gtf@@!%4p>q?y)PLa$6PoPt
zw6H2Fc6aaR<CTsQUpf!3Jm0m2U)MI?CT1b$=0jidJ3RmQGTz_zPGx=XO$*jt4OVHZ
zU!<J4`tJ0nr!Ga2c{iRPm>D5@GdBM?M@f1zU*QYmu3d}0_s-V&x9rK+Tg{>Oxg_Ji
zPR*ZbkYstuQMV`3u1Udp^}7wfTDKgLFbdafNuLnhFIxTIp+wT&MJwc!F3;~(4-Y6i
zK7Sgk_LgP7$a2YY<JReI=O66hb$C2?-663Z(lYuAz30n2<<?x-H}fNN5>rjJNPp!n
z6FHG$yOJAWCqABMp8Pea-&t<HXIN^PyK21ocE!3z?;h<RiHAOaoxf3f=D+>V_sQpo
zUU<sf@Z{6R&ucHw$$vWG(J#e~EmPM=|BRk#_VUud^$zdZ9=?CR)!^E**?-pc-_6|7
ztR(8rnUq?!Oy<dEzq*&~*Vv+qcC2h^=M}mqo+EYjaDT2@hjdSQO3YMYuj}<+H`uJZ
zKEv%`meg<0ncp|5rruXd_{LB@k%P0M?ZzGFx$V4Po9{$A{Xai>ue>Iw{r<BP1d|_b
zUuEOuax~3x!c55@Ws2-lF{fU!_ivQi7It-O?!-5Bw`4V^aX*rNo-ACl(V@5Bd4GHk
zXY+b%rHZS+Z>}jmFV^oLE?#40)w_K4+icFo-`g_WrrnV%TIlpXby?Pbp%1gq2}geU
zSay^to=2vTV^)7HN8<b|xhtnx?9@HKPTE+mRZ}|Q*k&_xMz+)3zcR|tbT=G)_?aWm
zI<uy7hki}vhYLH_-n(;d;cJ<R*Eh^Ji21W1)m5kWXQN=E!l&~hKhxbT3$%90N^-9|
zaA+6*v)eQCUyGF$@|p<FboqPulY@G}9f1p4Da%^4m;BSWVoP$8JD;jDEAZ<Kzu%k=
zJ)4%gSMIP)@DrcOyktVUM!;_y*W2^g#(w`;|BF4>rR@50AD7QHVGGuMP&r&=+m*8L
zu*F|>XW0;qOV;}P5`uOp8p_B&b32f7h?iG9Iq*qSVf{|$Ejn+_8Q7WB-3mUhxTd9&
z%F*TZ<7&RcrzN}g-(l`;KDI+Oo995wg3{Jk9sWtuYt6TO)H=UQ^M$RSM*1Pe6~RWL
za>lC+?&*r_f03T~uq=hAGVIjp`=<`@&T}%q^7Qdvug0v!htuW==<>W_GG%XUmYZ<O
zK_#F%!nH4QS+AVQtu5=!&(!?*a$(c0We<b9ioMSA{|uL&sID^mWop8dPj5bclkZu5
zJMZWBCZ!{xXHIi1$&S9CSuE2z<1_PKiRqhNs$OmWrarOFGjXSLd7<$Yxw=~2Z!Fdh
zhrURQZ_7@W6Z1Q~$M$@Ei%+DO!d~0%2W9T=8OC+*`vi)#Qv;`_6?hvgmq_&KkTOct
zI?XplE^S6S%j~(AB1|oUE2iwJI2`tS#j9F=`L=y?*7?`;PAQ$UD@0+7MZ0nI)2ZJY
z|1N&_%KWF(oU_M%Y*~J5kxoV0L$)UiHm($Sc${OB*QY(cE5*f)jGeM(#WCGueD414
zRKXNiv*-_D+4fs(a(A6HH}Oqx-?`v=^=AHwRmU%H`2YWy!)cx*>lYIg=08v0s3LOd
z_O2f>%G_)3-?<u|{(Scnh1d0;Wwy?J({Se>hbpK2VJB9PWlkw;zc_G4ZduCDmQnR~
z@2btQ*AIXF_Qn6!oG<KePl+=z?>$tjKKFA2mrg+Uq?MXc$5-9&4_q%=7%!~#wd#=|
zpI?;cyjPCGbvq|-Hf0LxIlo}#l4!w8t{E~{G;YW5WZ$WOIr+O+kw_C~`A^R|hY$O$
z^kE2!^<Y{VpTD{*|J3DD7bE|#(mHE?RVD@Rd==#SSMfIQ>KCQT8qOL92Sfkr%y3?0
zwEZE!cg(XQlU+fRiqyKJKQT3J_1s_-@%PHQ-H(}_4qJsD+q{Nbl=Cj@Qwcq;y$foZ
zu4J>$F3=U-xKOH0;JU#L%Xxo3RC-Sdd8YX6`O?_sKj*f!7GJ3FP=E9}vFfGiPMQDV
zr+YUnZ2a748Wgvpnf;y9W2XYcM#)W^RiCw$tTVQAW<T_|Ryf1=aUuH)FN<TT%H~=n
zvp9l9=DrNuQzHLLA@uYk&GaYJ&r3ToAMq1C)x}?TY{leHZaO-9vV#{VdfRoLZDnkI
zaY92aT&;RT{`c=kTU$3Ac_v!Ew_W$6py1;sW83DEy$4MHtLzoJdwIgzDW8n&#hmmT
zZ(cjyVXii<e_z@2j4A3LQ`7$>O1OPno4HO=zBRUOeR)&H9tWYAW6N@r(v+pRbtm4i
zFk6ze<iR|9hlrEgH#$4baJy@};z`%U4OaRpTxTY1mOHsbcF(OJCDKlNzh!Z1d7PNT
zz2ub1#7PGiz6;yQFZ3xYPq_L`?ds;#I-TI}t%sEEn%DJs78p+0=`-QjrOP!3?F4p<
z%IW20AHFfWCZ~>PVPdJkod$<H$&(X&rLFf)@$Wd4&}y@3d9y%n@%J}wi=P$cL?0E~
zmbv88hL{sh$3y1(2c}=I&y6=yv47<t_O9;jJnu_xCv|S<Tkg$X+}ApHf>m{+?xN_+
zpV!0`t~q|Bq$7vx<O3#_34U_NqZ`^M9M0KnJpHlP{O#*|*%qBKefuMvtLe35hv9qG
z1<SXJ9XR83qe8*m(Rqu9p2(A0#%KYZIz5BqGs^lkC0aFG`{tQgR($VUQx;Mv;naWZ
zbKWtv>vE;6$FA><y}O+Gp@Q}GR0XNMOzew_6?mT(aO&<Wzn-;a*~=F%_T5;|<2s3x
zLst3tZI+E^3cGe0alART;?ct+%j~`s@F-f$+Mm2QeK&`;&2io-Q8o#u^#Y=%??1z_
zHQw9bd7eAR6+`}=lh(*oq|HCI+VF{U&9RyZA1*pAEx(X9C%fbFvY6uqqOL-jj>!Ti
zD?=tZot$<3_lA}U`J!H@{^iYh`F}ar0p^>*Yp10uUcRpJ<eF4V$iC0+2d)Nc2*((0
zSJ%Hinfbo#_bn6JrZnqK+0*kqv&QL{*C+8WouVpx)t2zIhCJ{tJ-cXQ$ONNzo|@h^
zf}Pe{4Cl>y>wC^CyLR5#V81a@AgE*0g<_AcM?qVhW<PVvylqf3uaF~U;iZD?x(;LI
zuYF&i1@Nu%@DTI5%fD6jpWyMiOTwpqKe;aT!QpA|&1zPhjJ(P`-DB_k(3RXAliY-Y
zx}r~~_Lnh4c>ZqfcC*;|t-*51+LYg4f3DLHKB&H@b;;e8ef<|Y`Bu!y;lJ#7`nc>(
zF(-D3U+b(~@;)esK3~2z{L-d9nvx6E?{vy!M6-IHE}0t3akRBnWm?d@?+ufumPo!i
z{#D}SlaKLLZyOY@3oV-Uv3SaqX_IWCId*-sh*zEZHPS#w_)dLW>BB7+>P?sb2Lz<*
zdCopoQzZ~u9->!1qw1wUhwaqmo2Q61K4=R1(zs2;>d1;U<qQ>frrQYwCZG9r*67ex
zj<{3B983O($mZ&wb`jF^H}W^VZ!Y|mTj<T#v{#>BX?c38$}T&Yzd>_u@UoQ*r;q7a
z@4RF1X5U$MNA^P^8H;>g7k1rTkze|y_U_)zQy$5uMMg2c&0Ef$uqUDA@f@X>&aoN+
z5wSbIK6m|g$%$*{nUB9T+6~)}`)6-h-@3@PmwRvBcf~p8Q+(WGr?NNR=k0k^$1^2G
zTeZ2>=%l7-vB0fO!YB0F9dsV}+<eMEap$qQns+tlTFD-Jw!37z+^G`**XKEZc3iM+
z)`O5$LDDk>R4hZDo!y!7DC?mA`tRXtALW;CE%z6;>Ak;UVZ>4W3(h&;C-2tVAAY9W
zGMrn!(N7}o)wOwnGvZtBZZ>+kvu9cJ)XA>w0tJ(djZB(fFV&ek?XSYcI*E4&U##H&
zc$9m?48iI0MjY!65>?g-|4TS^%G5p6_0d|*me8{y8~-WY_+%T}_*HGgE8S9)k6H}3
zisId0>*%YVP8Jlby+4QjRnEFo$);|G%aY8TdP2W#X#P5D@9q3GNmrx0c_S7+t6iIC
z9oonC-0JEs(_7Ql-QM?zS9q1p+X)LzV^7^Tzg>1hIWsO}_tM`9w`HG1&pev&z3Tpr
zV;cn)C_bn;^|mk2DQx>1=jbmziJd<y6Q5~y^KO1(TkKNVH`CkFUU>P5;#q7a6{>w)
zlh<E2YLI=VaB}qr#T#}f?&zPmt(Cf@_vcA2*~Ze7=J{o<5nLW~B+fcNYjd7ikic}}
zz^?#}NmG<nD$-nhW?o_5WM?XI<B!4_onMMJ-4@5AEpAmFcgXnK5UubdS@G5kZsVm&
ze~-J)Olg&Qv-$*6mCVu4^N-cG|KAnD^5~$~#>n}5-aY4Uay#c;zQ=XtlKMHzIX$KR
z)<%CbPe>DZx%TGk1(l9P+S^|`pYQOhnX%o)zeVU~+^;=J6Ry=3Z?#=F<LUR09oi2i
zmG>*e{_%OlsVl;_t!k3GfI{8AOK)PP+itv|!M;+tRJm-Odh87`Hiwc=-=6N;7<)sr
ze8=iwAx6)o-udsGnr}Cpz5cqqc#DJFosDis;xa#cNVVq6>#hxq(%@>bEdIIl{!5lV
zi$>-{k?tP~4_V#0EvU-l8UD2H$br)Oh5WN8=4<c%SJ11@$9(9PmBfbUM-va&ujtS#
zYMxpr$I9heEPnUA`jVOa<|d0K*KD>I2nh3H(N=9bF1h*l%YzGQ52aVy$xf@!GS!@Z
zCy!NC|A*@DNvW}MML8>Z_ZZ!H;Gr`|e%-&l(<|mn*xf1jm)O}K$MXG}l&rp3)vS-}
z>{`mZGrfxJcnft_GF}t^Ijink*Vo+wmo^1V_|Uvl(pyWGsm$PO&+h(M??|((g?g<m
zii_pvZ@DeY<I(#zB1lf-$#k0u|4!%yX{xR>ag<>>{x?s;RwqR3>G`Q<#`0P$8L{6&
zSA18zJI!mowY$RGhhLl|Up78oU;D9l`B#tWM!CXEY89fsa40SHj;PDz57)Z1NmopW
z>2TTJeG4Ppm=({T4!BkOf2yX()Mu=$ZKnFMD{@wHubtYGs9a{_T(kE<Kf7BZllV@)
z8_9E(o>Ymkow;z=Lx1|*m+jiSV&xOxoMye8?dVfw+h5OP`ruLhOQ*T3>mK-CnzSOn
zq<lhM)l+qj36A+Toxk4~EpMH%HTvA54^MtOZe;o?$No7ocIuZp+rx6tFROYT2)?xP
z!Mke>7MZv0R2|PZJKYgHcv#C)sZ`AIIj43&p{)7IFTaB#SMu$@a>+#DH;1~G@SIS)
z=EJAWUv-A+Ty845_sA{jKC8j08S0xKY~>FYe-wAvslV2;e{Xj>%O4}AoE52u-(G7j
zU*LVv{hX(4$#t>#^287mnJ)}RA9fj(%lBnk_I^`(AG}=3lV8nkuB2jZtvwez_rfP_
z67|zvs{dJO{^}3g=JR@a&GQw{6Z_7;c&M=@IZSnT(;v?Ejej*tZ10?l=apTzEPLL?
zCzl?2b#Ba->d1Sn6t?o^M8k`%pRQk+cTkx9Y4eQOlS(s$o;9Ac_51l%!@$a7zpY;1
zB<2$y^9(PX6`r_o-7-No>4qhaihE-BNW9)seU~j!XJYi@DS_;M?!BK<jD$RG+Wz=o
z5oa$ob5lBURkG#`xBBMBz{eh5cijG~?zxh!U9BCQT4Vm~;g*xB5;^~41D2f$i+9^v
zcOtRN-}PBoq;h8bUyGytd1sjSN$*Y(c_+>(mHTG1&peq8f!jIS65I}p|Cq@1QK&{n
zcGaodlQm{Y8OgeBo&PVi=VR#n?<r?4ZnCL=%ol%npI^ir&4()^wnx4DoihLR^@NKX
zc)wO!ii?RpTzsokn|=K~ugu^4o4wy}p2>8K?LkAXHf#OrCrbBsIc%8xH}>$7M1_4U
z)tM2e%hnf4r`+DY&S}!z?<}{@T>E);&h7Mo+8FWaemCy(RPy}%VZQHw{Q6H@<xk$v
zX8rX2*a_(xb$%(=j$_ii)^fR$;`3JdZB|O1r+nhdeo6HuZ`>7cu+)86xAWCsiMjtx
z^iJkK-1dL=yzpp7m-pHtH_W+S?)TEF_Vvg#j(Gn5Yu8-Hh;8Y&8COWpS}iWcdLzB=
z%bCFK=O?glJrwUN%XF8)(NT*rr%K+b<-BjU_a071ofBo--J)Jx{IKH&&+`7%OMg?k
zp5MFli>K*9j=WgNWxm)6kIk(zOg8;_H~)XLGe>zyKi{5;*0oy&n@;K;S1Xt~+hdo;
zh4^RU&&-?+FJxT1ZS&_t)Q`Sx3j6ZqYJN&iN&PsXamCr)HgC4rGx0vT?tXn^O?|`n
zvt8B!&H=TDCCeGuoWD5xaJrqmobSxWYx!jfm%Km6lC_uDW}R4kDd;=*64pRDm7lkN
zwpf%mPEyU#RrxJfrf}a}@X|YecBx$qeQR~5t$sQCv?t%?h0|?fZpAmb%npBa+~~fU
zQQKsGkA{dp?1v|^?+W{LxK&mD#K)Vd9VWYVZ!_LhFtc9}r<r(@Wzo#$3N;y<%gLKA
zy$t%VbTLg=x8_Lvoz9|!sLBPeTT`}N*)mtc{qfqInps!))${aT7ujsCQLI*9t+eEM
z>cj3Oc5{3cD|Psve7U~LdU<v07mJXS$E~)=2e@tiz<j@f-#W!B`kh{;e1ckB^P6NY
zo9SKWd3EIP6t9t~Reri)fxDxss%{bg&(P&-o8q`{YZ$$^7g*NV^u$zbqV;A=*3}Ou
zuFG!!toh^^zqjDBrzgEr@2l;d_FB(Re#h-HO;20Zt=#+uS+X8Z!37^0yXT#9=Luha
z!{sUS<>mY$8ZncEQx~rLe&YOr#n%siUBE55{1b2dy2Rz4r3)28V<lEqe?PS#R_i)b
z<)^wH`<=bzmTm?gM4s_();_JM)HnM}^vTN++xFb|C|<LKRs0&;`r5_(R&05zrnL0$
zjCFQSne#nd@~X+1-rtPDPF8{?+kU?@S2$`q!NKEk-2LO;m-s(+O`P{b%-2ucD%@Yo
z<h@$4J}>v57WGT-XKnN1X<P76@<{&e#7?8wE!Q;4|1ue8*DTAC4BQr&@~e4XdiUus
zj}-kbvj>%j#Vnnva{Oy!!rs!gOE}*i;<A6Bbk_XWgunOHoi>SuUDJDE!?0estg-&s
z$LATw)8EFP+gx&Y-=0^d?;8EsbmyPiI^E{@P1WwkPtnU)&AiSh<Z}0wa)W>J4jzep
zpNiYfvR^44i1w7e`ti~3wNDui7BPQ~ewK3f@VCj+&PE-wy8FCs>4c-|I}#o)*wLq)
z``T)P*6~|OtP|8fbI;A<tu`)p)=rC>u34M)^s(Jtt^>cGOFLH16$ml>d@HZO>{s*0
zn=EgdOtbDcTsXebeCOAzUw)f1IHfDzxRT=R%m29T>C1>ok`9Z`9%Vbj$KK8vCTX>=
zD)+;!4TWCXGdCTXxA}ar=WUzT3-e{l<`-;BzsdhPuB=_<YWs<(m$R-FiZL3+_Q*VR
z{G~r{PsObC1OD+6^JOMY-96_b%QAoU6FM!mlIy=T?_FJBxp_{}MdLZWu1#-#U2uP!
zT_($+9{2ugLHV-jJ1cgjx|<8+=;cWiaz4nBD~x?nkoeliL@Hsvg_!WHHEok@-IzZe
zvr%_oYtYS`o}q02&TL_c9QR~Rwo?knwS2dSr%j4FVsfG3Rt5iKS<hnr!j=l5Wp5V#
zUu|qTRh7r$c;^iEeX}<_H%WYx5xwEVhc&F18!Sbf66baH`&BsJd+pKul=Z65=SSi^
zop#k5GAxV)x2>!;vAd_j+P-&_XKwA)(&HH);%t1arTi8dg>N;Sxwug|Ugc4V%*np@
zjCUvGDg@k+m@~hiIDWtV%xP*LviN$-!y=Y%cp!Iu*Gs-qk=y5l=V-ci7TPP`ns)c<
z<0!o;>)HBj^sdQ7EKV*L)16{a)9HGG=h-dWuPg=6qMP-m{QmGU@!V-&-2#axd<kkM
z-(0(nbQebMX!>rjNbQE-_0XH)YHgj#a%<Zk6`eh@jsN(P4G*0JzN|Nq{OI?)_v#hZ
z$e;H<*OfRvuUCKWQK;WiQTESVefAf|o>O|i3{Sp}xMULcHq|IF`{?GW&vL)1o&4K1
ztKt8n%SNhs5q5t0$7G@w@1EN(7-6#X#IZd(**APM`6q@)Jlm!+J=(n7(6#bT#AZED
zxs=4nHLYFiH_0r0bK~=k>&^4Tg}$6H{?3?Xr}ysvT&C{thS%OFggmR#72(-AmAx-d
zD)H2z>3gP`q;D!v2|D$u{H|j1w?B<fm_@%G{p+?}=}6<>yzYbMoOivZ=Df+*+dAQ7
zrb*|d)|nI8jq1L|mETeQ>i3i5r~HZY?4~C*DxP>hX*8UfvFfh4lifxkResyMd!o3H
zx^3u6U$$g-mx~RDsJ_mPj?gDxSj1mEQFtF%#C>Vb^%fanOXJ#!CyY5H+l*2_yjo)Y
z*|O<h{O3IvPi!-s_x;rMm=YDfiOom5B68mIarWHmj;`2y^+iRI?sxSGKW07ZNuK(*
zl;zDNp*yX?<}U&^@$gLIzO0~@aWOmTx#`=Mry(68n(Ps0dL-McUQcj2VPtFhqD$yq
zn@5|N-BG^4BGy+iO%FO=vTv#r-WzqacaD1J{ONN;KQVkzDExhEM+jGr#@py8GuM19
zl#{&roAc-ONAI3*o$MK`d||VbQmwLU$}#^y+Z{()K7_>@?488Iwk&2!<Nj-#-lTQk
z&sZ<cm;EJT{rAbTy@oRJg@*Po^y>XD_J*0f$=m(A>&X2ZJeGZb)=qdO8@l?Q<;m&o
z?|vM7&>4GJ@9D*YyWi_yo88H;<(}F&>-5$6V(wD|wl|yU2$|U2zp=|={oW~eI@*3e
zsbGD7I&%5O{ENq@#=2bmU1Qp|{<%GO(WxgT0<2FM_ba}PYSj4q^4p&s?weO@${+sB
zeEZu*)5TZj1+HKA+D^yUZl`|A^!RTp3ZG6Zf1IuShE=Gv(IrmcVbO-OKh5sN99up2
zTgUfByt;ON;R?KlL3NquUWBx?{n!v-AMw=kmDScGs&{9}_)h(FBX;F)7A>~u=MTJY
zYdHz&rR|ov6IaMzo$+?b&Ag+#uKxDAef?7<r~6F(X_d2ckCr6lxIO%~s4_)uR-5Ox
z4~N6fi@)XLh@RPYa{Eq~+;g%UQ+DgI2<tz{S?zZ)W@Y=5?PvPq`8Hj>eL%rK+S#qC
zA?vO3kKShr>;5jgaM_;Y#Wj}K(+iR}D4$qWba2MEOie}JWxLOnwr8vM{%;bfPP_c4
zVwnQ3TU!{vEo<s^Sv^I^ZvoX!p$!t^Qmo=0T{)%i_bh+2ps6zahP=(f54+f3KV^9L
z`Tc*Rmc0UJ`wl)^?9B7aqpRn@rhPerQWhLNtZa*;rFY4Tcz8cO>vC#>)4wC9{Vsnl
za@?`g?bJRt_UDCL6+}e2Oj<V`*x%*w-<vg|Twe3t*A)|aZb^OFCpF=S+=Uyl+`@D8
z?o6mr4`w!VkeJNPl;Nl{nJ4p<zSaucYDPz<-964He;%wgzq~F{Orw2$N@PyH@Q$zz
z55BJm=Vv--Tf&etDNLFB=bMuq413}gG{kNN%BCE4dih%FAWxCflUGS1Zt)wew)RZ$
z;f}m``KzH^`tm|Yp@rt3l3EHXb#H77bq%tZSJ3~i>A+iWmm3bxuctp>V=3Yx)N;u`
zQD<+`EH=gK_I*1pNs1-1aV@>*baUf|N$rmpRckG}^`!PuUYv-=!A&1HYuxU=G@BBy
zFyp<HsK=K54(mD8tTW$9X-4n9#8x*UoSXHwvHX^k>n}StOwnwcut!-ZbzA<ar_=aK
zjJl+rw{dV@2xtA58+@Ge*^#v$xKf`xaqbBCq{|a<F+L>V`%UBR2M<PwafNy8T9g-a
zpy=d7#mXSgZF{G2Z_)_}6FV<9_nD|#jY*-UThyf~VrQow`<7JZ)%nfsv<$n7!9?bh
zPng`78kv84D!DG_s@}ZQt=CLXvvo6_xjEU&Z363Vtr?S^pZLyk^`g*e@u}}zMAGcj
zgXEcvZ63{&7iAF9e|5>|>7~*{xdSWhJQ&5IVqP%^YM0u(cRT#aR&;Hc{)+!iXi40S
zf9toMpLZyJUHg?<_MAHIk3lmx#AZc3`|EJ@Xyv5uUnhL|XSOV0E%$@%-`yRB-c-sj
zlJNXH>&U;GO(zZ>z7aR?VdKdQe@rX3r2a|yG57mP&)LhrtG><t_8?vO&nx9MvE0he
z<z2#U7CU67uKIO~`{{?B*RJkR@pGKHg*7Ma*qh&bCBE^dW<KoGTVko#Y5Xm&cH6mw
z0Vm~Nnwm?mIn=bQ{_kC}*f-l3Uw!R2Ejl@oE#!K(qxh~3)qbmc=5jpUJ7cfE{@k@s
zH(9s(t~`|fS!$Kwjn#JSGZP#y<?{2)j<Z(WF`-{v`)zw*xHr4)jU8)Ki|V!Hm#})c
zMscoo%2_w%$L~v%T8&?|cP2S#Jrn%G`thVu=4IulzhwB{@8Ala(W>CWP`=rv^w-pb
z_w-u)P4{HY^_dVF>7so~q`S>u=*^zA8qr&eYK}B--FT_gIbe6s#5YS`vERF|n0R}=
z|C+}y_T5dca@e?i=Df@0)mmvhn|G9c-?e|$KGE}E4lo=3ayF~ri7Ys(7h9K>p0?1S
zkgKd9%K8T9wk7ttZO#TfTf-DCr$oF7nQIucNp8=|NsF&+c+_2R=W?#w`yiv(wwrbn
zy>=BGZTlruE7VZ?>fAir55KRdSGn}<{MYb=;m^0&Lpu!XTn+9#-xzo0eC*GCS&Lh>
z&rIN~x-spL)=guJ_hC$I%k3qSPEPi@zGG1~Q+CV~o?O0Xty3<}^kQFV^vHaPdfDun
zsFTin|CBlxGVPqi`H#OddCHk>u}u5YS=ViEn<pEQDm0_)bkF|V=?`3ws=qD1wYK$b
zp47U(B05aMYtKcVS4_`1eed494Zn35RQPAtZnddP`TBO*iysl+6B^5`*GOmn_;Fy)
z({q_M5s5}?3!}{Hr*XM=u=J>hahL4fl_GH}z31G{xDC@bR&RXYV)%qhlsA|62>&x7
z--GL}U)|}FxMafT2cA<d@krR}yuSF2!~NJBJBz*)7g62okDuROmAUJ(MA=uJryC~C
zV|sJ5FJocOvf$W*t8Va?%)7|O>A&;${u!&@Ex+<)g-QNwq33EZ4m)nh)Un(!?>2X|
zvix1O{jxJWJ?wY*oVoWb?a-gdj4PX4+#H>@$v<N=-<1;iX`8o1+1J@lcbD<pno{dn
zzlHm&g4jNpDBJ#Pd$wf#-nZZDM%<akZRZ=7a{XBOa_2!S<D8shOW)q|yxrD$bqDXx
zw*3rqp4+$F`lVT&-rTu8`O>9tuad3b-Z;@FX|i`KM~=u0<0TPN+t*Lv*pg;{_jj1R
zS(?kc%7vy3F3|y5a~6IT?EiQA;7dD;j3+O`w_cq4Y4h2Ds+Nm;T@J3P<(S@R$n}k5
z)}f<OZ)RmP+Rm3Yihp|AbK?c$SKF_-%zG$&%c(q*^ZWv4-MK-wD?%P>?$zH@^tadd
z|Nl(JCFMUIpHvx3$R647$YyHwJn0o?JDCqBgl$~@be{5xlX@5U#p(h*PfYlKjg#%d
z)L-{oqRz&@-|~VbIit@v`n1K7Qx!~KrPc_PEcS|hDLkc(|J{Z~4=3`?6@4tDJ!R^;
zlTVg7?b>?gQ2Cye#Mj$%)+fpZt#~*^&nSM!JExxWIx`x3=9>pCdz81b;P|JC{*C+R
zu+EeJblv&Km-qW48H~;??-0GOAhhC<%$|~}Gd)b5))UKC7(VV^FYsAH-X-98;OPr`
z3f1lR%`zF6{<z6BQ_4?nfygmODW%IUS8gqPdxrlifA2~&p@~IVHIph%Tc6B-nXS7+
z^6yI_<&#0D8B*`O)ZVrvzb&Xte}cNT2kQbM`JHk-JGG2eo?Ix<nj_oBo^W&OYqbYI
z^vWhwUMp$a<iqMQb6&!SPp@WPn_lt$$qO!ZnevYxiryM-JaFhIXQZWKx_V>ffn1%u
z*=wi#Zl8NAuI6j|jIYzIC-ThEnbxU)^1REWlYx^qY*?*-RklcG`I;X`J7xKkj$}?@
zUCwQs?$BIaQKq)Tw3=m;W@oldenZPCp94luKRD&Jc||*=r<^PK#2r)?5+>Gu(~#}b
zy`Vk`y{64a54$bzs8Y{Rk9d9{@V(rE#ovz}7oROvS!lb`^!Z2g+fyG$1s!|(;fvIY
z$<tJKt~>7Nu%h5j*QO0VPS;qY3c1d)H}UM&oH`*`@RQ*tfj>@%=4GAzD>qrJ@BZnl
zU83_HtrrUX5SZq~Cw8=@Han>@^Za*lEq?~b9eeEl6$x`n`otNqWu5L#3VNWJbxh>z
zE^pTr{XxyV968S>Uh<y4-}7sHW<39!yIVQhQ+`%^?0ei+Gllc;*~j(1z4sVuCr^(5
z!Q6e|Wzy5rnV&P3%S_YLdlD<b6MKVoX-0esXU}@=>;>0<hb+{**=o2X=AQp1Vb$vm
z4DS{nf36$0NsN1Dz>6m?3n$(=seS6qGqtE$&gKUdI@kPS{wbLBOU5l|)pzdmf94)i
z+hm<8A$+2A;iAHAk4ly{X1-|WU@?CGJYDZ+dP>u)eR)#PiZU&BIP=b*_)N#8>)&ao
zjJNyh_M88>DI{iefAfS3;!ZR5Cfo337S<{}Tl+93FN5phN}D;sir%4)rxhk_&)y>$
zr=;_8Zs{eV;(vw9S4^4Xq<#3gWRFB}E3;BxpsI%J!&9$Q8tZa1)_!!jC{@2f_)+Dw
zQ=e<MTyXz%PgkhY<#FMq7PpBfE~#8u{HMYGV#W08fSFg=Cd@Kz^}Ba<gX<mhnnqVC
ze*LDz)J;_vioK8ZZ(Htsk-h56fp3%VT~?JiBU833-sIlqNmC6TO>yZFoX6OwAb*C(
z?)|_2%&ZPe1iPhDYv(VWxX;^SWkLSBnC-Ldr6*53%Xq47V#Uw(reAXzyZz<0AIs=H
zD|Y|UXLa%X<^7u)+-DXvu90SZ7$&{eTI}FH!HIHj7QNdUC-<DmTGh0^zvSYHZ;6Rs
za>)j&4-dwP=V-B*+DN_aE4f{Ataj=tY34gYncHPo7Vj??wA;4%ob8&3g<94RuI$*l
z@}jQqsXm4^Z{{TIb$;=l(f#O^SG8X2w;e9H96fQH&!R-8w{{EKPMZ7tx%%ne8yg?B
z%pWUvzudpA!Dvy?;Ym(k1$-v`$dF#>er-ea;tTRR+G6h%7|;Lligj8LeStaQfCgXm
z2hCH57Isu{bWBaSyl=w4JzpedbEqHv>v`I5y4_#q%X42wi&(wgvRiKP)T4c$R{id5
zE^BHinfmno-&KnqpZk3G>z;pyWT({1FId5G!vFV5SB<c1g{NwLQ@?(jUb8=F!Koh;
z(klLjXP#DgI-{hik<%%+{s!~jdHS34tLG+ckeIvJvyyFr(bq5MvpDZ=JiAsv<(J<h
zy=<Pp4h#GeE*-e9H-ATa>7@K~i~Sgy9@NFYVEZo3`F*DIt^Hzrg_)i0Mt3~5WmZ0z
zXQ868zu{rMYLd)m&)^e#^mFdy%dAkA%e(qDelmNG#QSp5CG*YG7vDYm^!7*RkA}rj
z{hv2Iy7K#5q28Ho=0}Au-dk9g`nPOeOw#<4g!v`&m7iXny6Ly$A}5LYH_pYdbG(=?
z%&++4)=9qfDam!c4)4=V|A%J%DCGXcBs$@Zy~2zdIa4FUO$B<B#6=vV;yvzk2Y>tI
zaEkk6vF7IT?u&Qs*gj{!&VA@>nz2Off_27|yBl;%c2+FdW20bqr}<fieV|HNtH>Ii
zEgPpamm3<`MopQzVXOBOZnp>P-)FC@PT}aD_+n9tq)hrR#^!$!4zI4yE@HpQRXF#k
z_m{>A=LPKpOWSP~GxbgH^Pk<z{LaVr#f;~Na&NMxiE4XqS-`Q=|E6(F>XMVuOYW2h
z9Q$pcxbX1b!&ctEnZ&)VcT8_su(+h4@>QW_YYK-1`>DHHW^6w$P7#`P@wtaim1XX;
z6pNocd*%8z`<*;3zQs3<`N*#-f#YsbwbLgS{0(2X@zUf8(^@M<WB>8Jx%={Q>$(u3
zWlkIKN(5ftSHbVKE7<ewtEb$bbe=45_-Z_NVbz8WUcUnE1#YkhGhY@^UD`DzcBWBb
zx%S2Mf5|37cN1FP_}z8xU;Qllul0Y|SAm`?(=yfm*cMG}KOwuXKY9m4$65QePs}YG
zzAA9-lfAyS)!<;<Vg3`V_TRf&zk?y<z&x2UChjS7mEY<t)VlL=pPtKh(RDk%`MfvX
z5>$Tsq5P>Ue5YdOCas^kMe(AC$M2VtpHJ_;-nMlOd+5rKd^eIgCS=R6x+n2oV2Pp3
zTE7DaV&AV^=Q)G>y-a05(4Tuoo-_7UUNc=86MG_F)pO4aF0nb=Cmh~2;gI}p^Idv}
z-r3gKdMW>%X2cqy>Rg-2q9UtU`F@t*-B}4YI8N)|=wf<Ub$97xR#}mm-<@{anVCrG
zy!>tCuixEQ<!ZN{OT=UU=7Uc^zsc{N@+o556-x!>75uL64?fuIC7qhUvj6_!QnSR)
zkov!|Umq)Ihc7<2@#;?gulw%wdk1|FWmuIt-+JwR=4Z+AzWe0-XA9^4_PX@zP2|sO
zYRgR8HIEp_zxnd`amc^J*Bu#;|H-=1vTc3gZsyDT-~Idbn>FfgT4zAu7AJcKjzj56
zt6TfpJbrR#G|N<8lyY7mqbXGPA!c7kgkpZL+VsM6FK69ODZ1kyS0M0e(}J2vfyu53
zb4{)P91T2Cw?}c&U*3Y&4d$0xCRy&Ekj3O7{mg)eBS~|YV)D&1ZEo2=i_iRXZR?-f
zEVMgNNJjHauLzU-<&CPT-=eSoIW4<c;c(~^FTQ1+O$kT694|;|u2fx;al^d6C*b?p
z_6J|Nmz#1V&gYTbs&}{0_%PF&M8<%^&j%bNc=k6|EBN<LSzjli-Td>8)MeFbwv=`I
znVrNRYqhjb{%NXm<oxx2=c7(m2h@55)r%WD^+-)@s^MItBKPR?hu4iAS3MU$xBa!p
zf8qr>&bJfA7hQ5;esqoX$D+p#wHq@S-)aWD7mnw;cbvm8bIFV^j79tJ?vWQYn0)p<
zPp4w<U#3S2`+I%n?rG`s`f9x9|K7&`VU2n}-ae^RzY;fV(cd33R^Q+EvEG~g|3y!X
z;+OM0*Z5qOCaC5WEShmGreBr6TShhJyZU24gG)_3$DbTgudQ~qH9vJ!UHgL6)pHT`
zYXAItEGFukP58N}VxGjmbvYY??A6|7`WUm8eF^%Oz_Z%+VgswB{haiL!sc4m6Q>8w
zs?^T?WIH+OeB+6wuNqH^II6F;5?W!hL397SnO8;5rfj+O;j~k0)8qGv4cgOAO<C4)
z=Rl1LV`l?X(>azStGUj|acw&@%h|wHnRCn8|LhigqBcC0Ukw+m40n#^%{ApY>9dwu
z!`No3|EaHfJlqp+GIeO#aL=FWZ>EtGaASdnw7@oviBQf4$HeASq0YOczFm2KRCZ$a
zn!@|~KD(x*MpivNkjQE#WN}_`|C1F=?xGf}0`@TSX%|F%l;Se8xtRdE1>x8c&<#3s
z)QeX-=Qu?M?8uO;H?%w8`aL5~IyqHz<p!JEI@^DG1bVt&P4$|6dXKH%WtZ4x&AWUS
zFkR%&ILckM{IrSh^p`Uo4#Y3}ztcy$WTp+<jmJ(mwx0IWSts6cL@zivrmRT+a$<~{
z({6#c&x~A;q&`}C`U}tWm7IFQKjPCS70+yX(3JWkb<^+DOTJByTpXtsz5h(fU!$K^
zN4|^m>`fFcm^t%B>ZRh2>w1%I;?>&s`K-(RaXzbikK(_!2fvM)-=5}K=UCTZyCdNM
zuMzL3^p2&85}bhy<?~+Z@ube!>ct+&kfUnUmw0=}efQ6?N4{u&yJTOknk(+L|A*we
zqq>>DKd<NLV4n2+>+UD&{}#k=5!s$<(ChvE;<^aA#kZ?C6eWN9Pu;RTGgqhAT|#W<
z-=C-6K0eh~xaIY4qyIBwGk&+M_2GUTd-?VOt!K$eJ^aaLe|aVgy$a@Ol<(imtz-Qn
zJm$3i$KDr@CV5P~@kuAOcE%%}ORJZ@Z{3)^KVjYd+qO;?b#D)+vGBb<uzBH@V|(Ya
z=5?R&=v<p)kg<N&-cyA&KVNevxp1y$?{%NxAH6F3UH8klv*+$58Wl<E&zsTJs};e&
zxW?_7nEkw&xgv**725cPOWdEyEPpM3Pu=dPU3#A59EGKFlRKvNTzws>^7w}MVinz0
z*@;`G3H@Nnn89%O+m17#eKK~wd~5Dc*jX@D;IJ)ki|8UnMI%O535RyK?eSd`)NfyI
zuoluVvZ!UYYD(IAp4H&m=O0o}IOV$7pWXYg_ThBFO+1Gh=I!)JJC|r%$jRhh^s)L-
zOs~s(??%RbJt5{7X58NWk6A?IM*oed(Dw3o-p76#mX`Cxd}cbjOSe^@vu^*6a7l#_
zi#@L#j)$$DAG9yu?*1!LyTvo|{RPD@CdJ<@3z)csuk3iR>}3_1X73sU2Nt!1U6mCv
z2@`tmoVK`GHf4vqi=)2r&qNNn^V>6zb7kjQC5LlreKQSzCoO)i!g<%}p2w0u{a$a{
zVc_!AQd!_bgilELwXjz&WH;v9KXrVX+qO_-*8jg%fvHy>I(4^)KFX=tF8!YOVGalH
z4)e^}0?bb*OZGK<tGgHQ-?x%sS>uFP7B$vM>|AH}<?L(zEp*Q1WuIu}Y7YIdoXQ4U
zzw@m<ezWxsI2~^Cn7#I1`ozwpKXXz}IKRxa$@gu~wXe82b?+X_spnU)cpN{eP|MIF
znX~0#{T)%6F8%KHOnt6uHfaS9nGgNAIA!bpTY80$+C4ZH2D=_kS{T)R#J{i4U_R3q
zrzKBB8aMDitYu|5b!6U_N!=!D8-rc<%&T2o>nQczi=*>UYH3GmmkdLd%D2U9Zf{Uj
zlw&ns<EOLb*OFkzzd_sW-kr9Yr+<K9>O1WWPHryqpFep2UkILlLREuLfW=?&<=W&`
zg)=wlXFXWNcEV5TZ0avRh7WN^^9>H=X5{z(X#d^(vQy9gF?Yd(Jr;?+8lV1c)zcPN
zaP8VMaYN{?9i6?RS1+zncX?!7c)lalL5^hw$HtjAT$*$n<1F<4cdb6<FC$wN(D3%t
ziO>Z_2NE7?GDe8+&yoJ6z^1X!rH=pOtjtY&&ezQUTaYrR>BQXG4=<@}ef@p-{G#<P
zkB(0KGvTE~(*Nt1V)`GfV0PV<oD;;|wUFuM(~m6Iv`qB3HpOI5TD;@%jGajnQtGBV
z?rmgSlt1Zm-kKHn1h#koaJ`bR`$<t~k5aVSBnH>eij=vVwDz!Yt+zkV_t10OG1c$|
zQ)D>ogs*<ve^cT5qooJ?+mcoNRu+6*<ZN|8hhdArOBXi}S&v%(1j7yKWlemkai<bL
z@lDH~#BkOB(h;!*^4yn7lQ-}wr@sDNR=CQ}*kkd98D-B7rOn?K9~u+y>VF{ZZ~na+
zuT5z-&a*zOQ}Ny$uJ$XisQ-C^N4nTI#XXFwKfdhjZ2Gz)Meh3LC)<N07bOcc9X**h
z|Fp}I6tABO>}L71Pb;39zw56Ww^!emN6MY{DYfziF$EQ;XMcaL5X*Hw{9|$Ho*#Fg
zuWj7#Ubk}oe-SkwH~WPa><SkP?kezxxCd#(Ps_?YWmMo*v((6Nm;L1XdyChmB(`@N
zS4OZU&3ZC#i6>|C{kXG#{yG=Cb)8?zXtZkWk2wsvPHYh;epKI2HWtpGb6R_nqTb{a
z8lQU>beT_;35xZbbiClpED;F-Ck3gVmZK*=G<;*)&LVNpgGHEejo>!HZ)cJ}m*pyQ
zM#YFW`qVMJXHZ@D@xmL`4Lc90cD*qzGP(J3d1r4%?1@zV)M};|&vrFEif+|ToUYTQ
zcViPDC+o)7hlEddg~|VW*89bG&&5`LF{M*`-smwpY>@J5JSOsVs<~v-EcxG6o*vuz
z9aQ<V$`zNt*|OmOx91Ey_MhHtsp+ouOm&HrW`ZhbnFohGCr5m|H%Dw+v7p1G`Iir_
z7u+cHep}_uNz0}>%W=)#xx?i650QG-`Tt5=-Yl7K>^S|i!D<hU&n=hQMRV9aH4Epp
zYCD-7jtnkb)-fq``oeu>g0CkiU7LEK%XwAB<bZcU$6hVIeRP_?k^c?p{>$%cVhSXe
zpEix1_u|a7({k+%MI2I<SKbMo`+97K(D~`zI^vqo$~IJ}&0Sqvn~+_9iuvM==h=A$
zsch=|s-7{`pJLf(SI}sm!W8cxv*7H}nQIlj3O1@=`2AY%eSt*jtXrK0`}O8#sLQ_C
z)Tem3K&q!bjU&C{>n3~g!xOGN)!LbHyj3T)^{S>o;m%X1S+(>cie_G%Ai6VTy3Pvk
z+_n$LQ(g7+Z?-z>&AaKMEuy_QW?hMGJj?w>PaZ|2T#(ZHTfT6YG1ne0_Jb~Z$F_=}
zeg3HBg2M5N6Pvuh&ONTO(#~`D^vOAXY*wN#a#y~|{avY+_*E&?ghRuA+Y#Y=yIwQ0
zSxLGGw_AMZbrzVH`e@oo#bb`nd*^f%MP*(HP?<KTOiD#`Y0A-$SJ<OC*G@?J<)XFM
zFY0C6(tUzI{`uT^z3JX`?&^~dtsBi!_W$u&d1eA@aKSezr-$t47(y<dleO^wW;Juu
z>>a*ydheuGHp_mtk9qO@kc0TQvvcF$A2n!c(4Wq9uce{e;27s|XO+)ZsYX+dc3ieu
zw`q5c<=kh9_p5X}X0j^;e2}a#_qkRt>b=iw->)dcurjua=gI9C4&1zVQhv8VTeZuy
z|C@V`F6C`sb9DXc=6f<<UTAXL@N&oOv_Hejw{?2<-niwvPG2?fJ1To?UGVy^MSHJa
zNw(a0lFRvFE7$IAZ=P;aj$ue$)979NMXh+z6t$n3mMLz_Ph|YDOAU1PFx9)1{&32v
zlr=FD8n05US_|ecX+Ef$q__EYYvX!F-X%8*Jb2sB^zrU^lgGg<%Mj}&QU39vfcCB{
z7g`(Jw|rQdyrSOuzs4QYxa+A|Z)AdNF5Hpr+AVHczHV>9tz%DH)-fw(zs^}>AFsat
z^_=H5&r&UZzTF@#JL5!wRsV}U8;!4j-km)0>#>vC(QmFy(NQitAR+Vi;MsjG7v?V4
zIrjB&e0Ev<{gq|=FTdFrvvs!YVcV<<dyR_t%llqN`@UcKqIT`9dx|Z#M_w%txaR*c
z+oS&fIz5&WhL@hF+-&bDNT%G|te38qwfxW2$0sb(o<@c3-*Klhqjj!r`OXhh?0SWt
zvfMU!si5E!%r{-h^6&$fF5#{@83oSGiNc50F3CM{_KNbag$X~O>4X$Lo8K|<$Rg>b
z7bH^6Sr@9Fc&)xnAusunnEcnG3nh=HR{uFE(xT${Pod)WweUZ2g)Xn6w5K=ke%Yhn
zUsU4NC0c#`kDU0S={mRWeNwY}J~haU)AZ?8vGle7S!A|;P56E*c;B*=Yb#}|Ki>S)
zv8(Qdy4T+8E3f|cT`HGv$IgHKrlozQ*Ta}HLk(Z1#JP72Ub27k$V=?lTO{N9cfqbF
zUa>;fsvpa%_g&T~d{KI}vUE$>@`@^EY4Z<@pDfU2;%7PCEf>|XWW|c?c{`MCHmMZG
zs3`GOwyU{!I4{`|*tC94x8(+|tF8WiTddpWYd?SUbMdvWU+3QXdSM&a%wMk_JeMfF
ze)EIV$KS_HEpKGl#ohE@v;AOjvsa7Zym~HcPmO&Q8*7(JoqGM&M$~0}uzQWxLAw_R
zJWlS5(N_=Ixbxtf3(sX=Srz`3D?FU7@q=T1aQgEuhq`cc|3u5L&d<9p9ne+eoA`{q
z*=601Ei2E+@;tl9kk(<-7d-WF)RFv3rcKe^AAZ+gRyoNztG~6x^2`3KQzy*0eER9}
zM;a3@wL9GOQfZd0babuv-y*zn730s}Z}zZ>GkME@Y?}CQ{d{}*lJER2x3w1Kee3-H
z+h*>6kssWwR(vd$8(-Y}HTUc5%qt-VF>*agd}RuCvIpZ2c(;C7pTBx?>XY?#v1@Pb
z`Qs-$S#9-%H5*fUmwsQr&FFyr?dl1#JNN6$rq_5K`p}*5|CsA8;qQOds_Q3L{o5|~
z=DN-*-zkjetE;Adzsvt5+>p)iZR+iaDc8EW=bU<x_a*-3b9>f*E1ATe-gEeM-St-Q
znbyuImib5QH2<`|SkR%L70;x_`fJw1;0^xAR&o^VaoBvlzH}M$)ENg=9|mvocZ=mt
znf%vluU5Rbqbi5fFW0N~HyeMrPX40L@WgMy7yT9213K4D<()4TWtP-2?ShKu+U%oF
z8e&cx1+usoZWHKg4bW-1<)k6j#K}IzDI}Uh_4KjReMi>U9&|F8+N<`Af17`TvBr|4
zPAha<k`$*1DS}jk*&-0_3;I_kDP9p{vRn8?j3=Tafm?8c03(Q)*8Wd6N0CWPDM3+(
z|FU(0B9mCt!D~8ckF9SwvFNxQaAJw>XlUh#01?{+4uaG%i7CP4bX+#Ha%_W|Wc5lG
zYUn{RrCUxeN_9!K(H$MF39}w`w03M0xG2`Nvr}2@&?1+;%3?}MiW*_&TpxJV<~{L0
z;>2)IS*&Scq%ufj!^fV8j*eD?mG7*R6g8B4A~+sb{n~nYe}bYCNZLim<%pAuj?0!-
zk8J{++!Hqla9)=aYdSecNbJy@757hyDJ3cji7DN5Qqgfa1R^%U6~g2a6({MsY-&wl
z>uZ6?9b6K@$vrU#Bn;CM(Q!mZ`LA34j!V+-zNAgJvP-p?_l*0|!JO>btoK>&vxM)v
zp?*~T$(K{7va;?w^rQqROFlWV-u=JGN6tliWg^dRGiZKuVu7_0M?cRg<}$IFOW0Fq
z+&rMr+<SJyG3oH6jrPBmcln<<JW+eK<dPJN?`b-o$0nsjPij)tjF@ytRdbsPt9Q@_
z5FxQ%wLCLnt3r&MzlejkVTR^52G;E04Q8y_hy8g9gEv^Hi5i_P`aiAbnAd{kLK(}h
z7zUqI6V(csBdVd+t+<Kl5RcUIb9&$PkM0WzS@k32<4J>9!}RCx-p^fFA1utzsQ=BY
z^xnopzc&Yzln9ppdvs{Fri*Xf?aCrk_Vo!jC(h;+f7AV_`_}Y?cqVPf7r$Q%>Tnhr
zi>?wZvGBd>7i?Pf^?JRp^`*PNXD^L26AU!VU0-|R$A!ovS&sKtRU3R;Kfz_@`4Ur!
zw~CCv+PnB8=EwJk%lGI89AygC3E0Y-^<VDQbN@zr)_)>9{!35njazgs?^Q?L-F(~M
zi+8a6xcYSN<nV7N65<wevu3(kM+K)>?EZB#H+X7jyx3LA)WzK8zf2zr?*H`Shivce
z_x{4%f1it=WNKWy-&$Bi$KZ&5lww8Cd_lg-BX+Ynt&Ruv-I%1j>w(XD%O|g$dCZvX
zj@xkE<n3FxSg_#F?!CGa(f@b5U*Bk9_s#j9SkJM4*R~&3-?abm?Iw|%_xG>6@kIF7
zSNBu@wqK0+b9?#JV_PmUDgHUUeCqKnoBXwZgNXjj)O;K311b5tdvrDLRTLJ;GDNoR
zSXS})==a4Hw^MocR6KrsZQZ>xS?2ZkD!w*`=kM-o4A0;5>%i*mcaI&IZGNxj!P(n+
z$q&v-=i7aFmb=~X!?WJgrM>%8{+J%q3zyJ;{6o8U|0kV@W2deDN$ou`qxr-Ow~{}m
zRTC|^`|^%$6qVfTF_+cd+)8cnV)40?C6BMqzV>JR#qM(_dyd2j_xi3$Va)w={^D7O
zTBD>jr+5<7A6J|&cy7Go;iRVWXIwKrCZ}^;RPB7{^VoE9;h7WfjQoG`Tryv5cg~{R
zo8z8jbnY?psU}sP#YW0<GnW|7EL@X1FI$p5`q;^7g<DeR<w~|kAA32ia8K&Id`b40
zW1Q0qkEG5klx&YVra8UvOzOO1N%q)dp3@7jq|PgqY>z#bIlb^s>b!DE_PAr6(+i)Z
z&a0Gck2|(<df}VYdDW8a@yAY1FZ_}^uU4`>{@Baug@01#)k|_Xurpm~WGGQ$kVR)^
z-+Pj5WF^qR$FhK#@r4TmL8kHM`F>VulT?KnUpO!ruX@b0X^+c>D~${#3LwMPnPxnh
z1U9Q=?{n^nb2Fwa$eUoGc4evk9Feo9G%Dmf8aS`i9}H@Ibm$+?f=xc}6DK}8#y`2`
z2fuLnuYNhnQ_=>$&odGiRR*-4;4_+|$jl)ZyyUOdi-j9Kwx@Jj)%C2+R!CyuQSq3b
z<-{AHadnA={sg6~N5oDxmptV<bm(86g37P^ve7B?XB@cJEBoJy>u1uHa~TJo*zY!2
z<6U+9{ia&K>)$qat6V>ox;{r!#I8*ARoqV5sh{JbzRlki&Ubvn`sIxirE0(LnEaQA
z<*Sgs@xI9HgftbI>rzYP-(Ed^|K#2o_u>^(ZvE4{*L~Ohr_#!uqT2J-9`F8QWj0mh
zV-)j-s@}R?PD-x&DxXh(dOrWt|Lv#0+y9L?ap3T#ed0;~ADebxKmN$3dHW-q8`~ef
zb$<PFd(`v59ir<`i&jna-zzcslCMrt9lzd5?yBCkSN<*)ytu^utVeg6dsxjT=9f8T
z+U50~X}5Exl`ZzrH+=spdUN*9-`?5N98X<bqj&aKt8GT((#YDZu<dE7Cf=*Vv~73(
zFWa_DZ1-O0*;coGHh-AHd}mv>ZdrA^+2I`H^M{%pJ+|+S>b`q#;aSPEHq#?3ivDl!
zh4497?$pcWvpkSH{aVuY@(kniX18^0RGGfpX7T54TYJ{%md<q@n^flQwjkxYF)x)g
zBn;f|PuzS*G9t=-#kRe7GH(TbS$SiM*qLviiVt4SI2zk_uBmAD$<X=S^MVWWmqs5f
z)_d1qB6g+NNY!S#_47yn0(COV=55JZ;(qzlXK@LaQ%2YS?|ZlY!+pKS9mlR5TYN}g
z<nNVB%m3HCdFbD2FZ%DXXkF~bmG|9W-ZxrcFZwU!M=(U3AfvvxHR%3su_y7lI{)PL
z40>|!r}h1FEj7-oQWT!|^vvtfGvAb-eCMzFRlmsL&u`y9%*`G3yF2%XE6nrz@c-7|
zTS71D?z>n`o%&s9{^R05r>8$U&U)s~kE#1(yZ&D?_}ppxPrUiZbM7hsuPWF*{ILI;
z>kn_;5|(T4Z|vLp?7|PXj_&EK8w|2pCC=F~KD^d@VZ+4l{d=8)Zi%`pdaph!Czg>v
zOaI`zu<O6?ZJOzS`spj%S&cy(EWeu-Y_|2g|7rg0mDBImq&17YG23E!OEziSiL^!8
z+XUJ!o!xLENB7Jd#<tjv*9{kI9!^c0oRV>BP2Cxv=ca$yd90Vlo#oqj^Ytvr2~Qc0
z%{05t@R@7F<kWQx&sY<-*|JU3-~7(7yWaTQ_9Oe#9_AnWo3<g@vOYd-Q*zs%w8QzF
z^=Aw2=X|?+{OhKMPZEo6zL;IPuX53>-rkvyl`pSX{k2!}O2o;BrPu1D*|+bnPrTbx
zUU!dWPxG?$3-P&6Z%?k?F7s=fw1vt+$&-gJ?u@+nvO&dWmig}WYq$0~)tr3%b8hS7
zd|ifDHjT&Lzh4&3y{^u4{X_9*`s_jyZk3M@`1d>N+iEmlFW4zL&*<>M)~~FEc@KG)
zD%+R*uXedo^nO)*Drfa0yNslDoN<RIdwt&@Z{jB+<NEutRQ%m<MYFsY6*-8kyW{5h
zY?J%sd%N@Z&6s>#<CX2cWAX8SK8Bo7pL$#4%DM2z{YGI0e@g#1$NBD^@IUm{ar?OV
zn$^h<9<Oh=ck{mb{S0I2&CMT=zy9~?v{Uk5{hfcmR#m;+*ZMks{qpJWwrkkj)3Dg~
z^Ywl^5uNS(<NWpSZVzZnPSUrC+s1i0_fV3(Ftc9zIr*;~i@rTK{XZw**Q>m0z89A@
zODyX84)3-q@Z&%GvdJ<hQ(XVlQQsnW|Bu&SK9o<IXjS*`zqpLh?&fJnPew*<y}jw?
zv{LqS`|8&F?9$UNj6R?!*247C`OS}Q=U3hr5|a%R@amZ8(GsWrA=ytXVufN{6}zzU
z>-UbwyRN=IeMQ<wtbg(GEdpz1@bXN!dRoEZ;%TL>8!M9cnfC;y?e5w0u&!a(ebGhj
z6{n{c+&cO1T^>6}?G~Pu@5A&ze43{3I??D=;+1`DKlXpyes|$+g*W^E&p&v|{;YoS
zmlLZ~ALidap88!bdEVUb7nSNRw>q2>eikKpxAW%Dzqe+`ZwWLCGMsh#r*@kA>-qow
zPTu3p&w8c*%@$4ZfB(xn{&NWZ64O<vU^wv6z1;QAPy5qzHVQR#sJ^U!`Qx?dvH696
zrXGv06w`Tp-s)f0=C?P`dZ)P?RZqG(`N+}_Z+z17jjAW_5}7Ca>P^7(PF>^6Hy57c
zEB&_o^qZ2h6`P&s{%uWdN-UE|Yft<pK8>^OX5tLlwwv)0Y7%Mfg=G@y^@TGNV!q`;
zS#xBM-AtVGcWG)<X4#Ci_RMe7r*SU3nK(;!+0FPUwHaycm1Q&1>nmp_%#(d~GcoQ!
zF_bm$@6^<$)Ut@Q_SA3T(>SNyOq?k@?Ph$WT0~lVX<0;ied)}Exw6-8CeHnPHMObK
zvT^fatKZqDS#H1CaF*}(oAtWFIhzlgS?6rtzs+aE+V(m}z3WA<)~EKJ%X{b?^DoZt
z-_+||ryuTr_38Vi&wcsR)`j<MUNndI`5KS(qgL9tZhkn|RFQJfW|8))X!X4c@{t?*
zbzZoK{ZZy-nv^QMWbOOs8gaYUY}eCXeA=Z{@|Ab^ne9qPQYGFsp5x;7i!oTJ_rhO%
z$@fdFlCPSWN*x<_DeRlB#}O!QvVvRc3hU3kChII}U*;aH-<4z=yJ4Z8grE44<(Y32
z50*Q=^jlZTlCxodxQr_Iq)Dvotvf>E&hJ^bVB-SG$bis}9ljS1H@T-c=ITD4$C$B3
zdV533j%iM*wh^b<m7k|?WIgB=+u63%De~0vQkjCnxAp&Tt=@XbsB_l#$ZFQu_Ytvq
znvol3>Xf%mDUH4Uw4*>*wNC#1XR}317w(EZUi^5PbynNQZGyT5TNlh+wfKJDom7$e
zxv#eV+56?|rL7m3Y-e5j{`%ufm1@bQVP}eZ*L}X_8o%MA_SC2EOE1|7zDWKlc!=xL
zY0op>;)yfkVxnX9WbVFee;RZt`){@7y3?_Vg?pa0v%2d}?><$vY`XQqjN%{nqBkz(
zzu9-8D^UE>O5JKMt6A;8s{BJX9&(d=uUqu@O~SQlyONS4wYEeaymYqruc>FrpAYk|
z{}DFeUh>=b;`;xe>sKfA%i1JY+y4E1J?qFnXVx6UH;W7(F{g+;;8B_;UZD8Sv1zJc
z`T|v(2A-rs=T!-TH4{%Q=nnV2z|toFX5Y>=vnE`(n-Q=qIn}U6r;YcBU*jG2KW}@g
ze3TtaB)d;5yCpH~>^#b6zRq9w%|_NsTQk<EsW^vCZ8v*s5q(be$z%sHw<kJIR(uPj
zT|e<D`&dnv<9S=ocHz<CIuGf;-3PhOPnNtLvQPVCp>5TLYkrsezc6L3WNX@<vD0n7
zsbEqh=fA{`-+_&OW_ON%*>7i=l>KwRT&w7WGN&W|lSKZ9bCn!iel*!|(YF7`4{cuZ
zPyRG#zyyf}|F=hGUfJlCv{gkRv*GXcf8U*~7&q=}(qZD&WPP*$#YL^vq3>RmadRdA
z-+C%*s$<UTP`)tBh}6T|PAc7y*1h*XdulUCq#<bgso353)YG>v4(*$<F0?P~X3+Y`
zo^L5pWrmrnW~7Abh=u)MAL<t-xiLpbZSI<~O}9>Jsjr^1wrJa_n_B5UR?%;Jwt|@;
zv5lu#C+mgnj(D3iwL4VrhUV&+HAM%POgz6Z>TTB4)uHRcc1AtTnyUT%W>)y>yK9Oz
zpE|2$zWUCZqQgsOU(E~K&FICU)-aj7p@=a^&*x6c9nMSdQx*wSGsMg&edJ`!mlwgH
zctr3CM{$calkCF21oj=9Rr1(2v_zXQKM-b^&+v=ofqKLK`lFhm>*S`zGdMFTykc$1
zw64<0-@P;ESnJUZeK+DCJ~Lsj`f#0zk5OG*n<e3uCA-y)@Bqz|?;8L2@74G2k1?NB
zyMF77#+fs|@878H!otyz;`!>^#=HCK>vQE-t#at;oWuU*l=UgyX0FEarAETBO*+ZD
zFP`Rb(N`C#5Rei`yP){t#*d|fCwk5QKi%_gnfTAO><<MWaX(^GH!4VCezRd^?3L4V
zQZGEPsrq)m+x^SmWbsw4LSMI~30yxaxn7sc|55nYW4g};jxqLToT%8Y!?otBE_>il
zQDr})Mas%~n^xrAijkN&`6Ew{$El9_I-IJ-f@&8svJTJd+}@*cx?}UPOLv~}{oB6&
zp#9|oJJ<fdqv7}EsY`tQohn0D?f2LIR4<me_)mW4jr!tdz1IQEW$S;KN`Kr|bt_@*
z;{XlA*ZDptDxc2Xez)?hyOHfioq31CH~hVS!+p&(ZudmLGh3VPT-g{tH^;)mS3T;E
z4ExoW+>6*&|4Y~RIi&dA$>RSbgNV2LEBpWNWDC2!|7TqE-i7z=%gq0s<=s_(=Bxes
z<@(R-?|owp<NKeu*Wqej^-o2%<+}OSJ2TY9-_||)G<`OU$;LGGn+9BldSRCH+b!1f
zSk{`nn_lKTp?s02-TT|TC7wNb+Zmr{)m^)~wlRA5zKo|k4%Y7%KEL9h*_6OzxArXx
z_`<x^&9>5O@r1U=HvAP2CWgl_9=!F|D)J}qw6j@ti)7MghunX2aP?{L_uJFoB~_np
zI%D?KZC}r$HB-NR`y;>m)ufaB{Jdx1${+BO^M6pNX};v{kEiRNed^zTuD*Wm|7D-&
zFD?GIH+FULWv(d^?AuwrU5!pJdF);JPxkd0_20WM{M*J@%lLc$QSk}?>&}W#5ant+
zxvIy01<RwS>y|hBw+5U$$hG>}MxRRtx0PO3czyZ1_I^#x-=Np~_5auYe1Ez={@?FU
z_ox5g_viec8Y8~LAH&UL{@e`y^O#>GDsi`y`5uMo`gN5(pPC!DibR|-UAf<X8pk@>
zxxE^HrYas+(~R0D)$@lr{$Rp!{*a{O9Cw!`Cf8{k?&D0e`Y-*KeU@)}<l<X$LQ%d)
z*H~`1`q#4OtmWQWH|E}xRG!wWWb;2{X^Dxh*5QQubB_O@j+q|6j^~Tvu9%nS|49{c
zXX&~sc3S;wnJl)^M|{?t9Z#<cwta~G{#xshMc~(?zb5$I`Bc=(c=4xq;-tGXwLYv@
z2v1%vD3SMG#CPW12WRK6&yW6dxc9f;xBtQa)@?t({=L+nw-55!>#cw6zs-1Mg<$=y
zKbN1+UvKcg@AG++|7W}(8~tDR`TRDXO*|()T>5y}Xqj$uR7`wC+t$>W=wn-V?c2cf
z_Vk5)dS$U|^__nkZP8c$y(=lc?OSR_yc^%^MTd<_BHH*)pGb@a(-#sYE!TAOoxV^g
zX{D!py!2wB?c>xJJ9=cUyB<${v4cl;`ozn-UToo+J8j`*qoOErFkKSm=J$HhWuua~
zwsWUXR9e40?R(;M$S0$zd(O19+8^hKvY&)bTRA0FD|Mz9=WFiM7g8m6=_pU%v~L4X
z?6ifijf!I3>aM1~*u@h&ed6m~Ao{xWzMh!tuG>>zY~#5*ZQ|`+V0z(gqoQc>yJ2rO
zEu1d8Q1<mE$x~JTwk(>VyiWeow@Y2AeWGV&O8;)qyC0^KE<Ne?hC5x);<hPS-4?jp
z_A)Hw`zFzwH@n{C94&ix+ieZU`LLFB?PJFG&5~TJwkg?0Z=CmYo1pfi&7vP8Z(R7h
zO;G#OX3?KFyZ-9rNN-efh!$M1R^q~=ZGzfDTXqP}E#)h`-61M?tBWJ|Xp7kP-iV0{
z*Pgf#mXncryF*m*R+m8T(T-``dLt$-TI+E!Y)wYx?G8~xi0W+%ouUh&s->;Ah{{7$
zZ(HaRT?kTr>KIFo^u|Z7(SnQD&bVlHB||%Ji>T$TF4<h?{%w!kqXnP*SFr!Fy?MXv
zpSKVEUoyxsa4F8KKlbyv_WAv<|6HCXo?H1j_4Mv-d$LcT-nZw^r>D8`ao?V%#>dsQ
zK7G4kkIb~`&T;V(_xetsE{y%S>&cEiveTzu{sfNC?I1et-0MZ3Kb6JBojZNHGPWf0
zp5E!xsg*m|bnBhIU0S(QPyc%9#ZucBPoHkvbLX1t_NQ;R?YU!S5E<^v_VZ6>>y%{H
zsmZZBa~EtoG%I(*Hm2D(t_j<jC&X4qFup4@&tY|pPP}kv+Zs`>Tvq4khhU~qE~{&F
z;>AbX)`&{wvbsk<yckwdbCYZ0wx%U(4KgctAFJGbJeyxTChDeHQmaq2<K?g$TQ=n$
z;hMXxY5Cd@nWeWoM4fV31EUkK+}b9nJtgY4z@;wsaE;oZT+8+~tz4h*YTLstqCT~(
z!SM&LK!q0`<<(yo@bi<XPA#i<{K89z*<NTz{NkFn&uQuLT;;G0Uqp3kTfO{myRAuh
zsU0EL_2vEj^Y-zD|K>iQU-~cebobtxLw^!0uTCq^t^E1)^#)Ktzp1QEthCnAZ-4r9
z!=67}r*9`#TI%T^&pmqjb|Hwhy!2vbrM0g9@~2O?=vnFKKY#jkPu{m(FZS$-oi_dS
zr=m#l*lF8Oe=3TNTlad=>rZ8|{8g1Z*LLTfzMWgSQ&)d`>BSEka~(D_o!aPhYU1|k
zVII+rXUzhl6M4%vKV81G{l8<~2XH}g`{4Y`2mdcQeH2_qet)`O{1<Ye`Tgntd#CrG
zD4ssCXrpLNzrZQ8BB^avb&>A74$jLhtw~RBJMwtO{ayRs{!@BBJ<;sz&s>|?Z9h|I
z`9A(*HsAO1pQ2?MHlO}%o9Fv{QPynV=c{fNn$10(RUBzH{pz{Yn=xUtPjA|}rcCej
zvrV^RrkzdQ3YK}cDMxShT#$^}^wi==j^41_+iI-EpB?}7b*Itxr?+>Se1H17I9G03
z{_{sRdhzpDf8J7KJ^gv*&tEa~r$4>D(;;3g<2h4p<bt!TZK;iAdImb&TXr7UpmX6I
zh;Jq?F^%;WnE#CF@FoXqFh3?CLhQwJ5Pv>*&uON=F$pnZFNzxrBNu@AwR#46++V=_
z^Q^~G8|%RQod<U47|a2wKOB<~B_?r>by;d-qMm^+H_vmX%bN}uh|ieD+E(0{8Tnw=
z@6yNw3ow7@fh{@)^SFIZGo9bEcKy7hcW0i)9ow~AGO6bFWa*WKUjwaJKMKn;`}Vv)
zH&<fk)8F=gXPjT9{p;CP+x}y#H>dRN^IRNiwfodBty@2q^|)P}{&AO?#L0T)H%}By
z0$n)v?<sj$dbm{f>-`oc*V@C6ABA42uR6)v6}#)+IpsL!na$R!Jtg<Mw*2|}@yF%o
zTOC8+-FU0B(f#8?)-#tUayH%6iE}zMbHR0sssncBeCMVJGD<$0+5PXR$(KI)8LQ?6
z90>mIy!cBF*PmBYcRpIie!Y=%vBu;=ZQt7xiKR!`bp`)FH4`;G%C||`DsS%$aYxZ}
zw;l*teS7q!MX~Xp^2gegX}@l~{X75v*`4~|Kl2>gkg3MtmGr6SUDDs8(zVIoK0a~#
zw0}<6wSqfL3u7#IeHGlW-S%bj@xHlkA`uGJRY&}uR;Bf_>A!uK)6Sl?xGdhp=126?
z<z4<OKZaz+zq;wn#B;8~K=;k_Z(rP{6<j)mZp7~YarOC4#<OWVY#sb3{?HKOb$hh;
z$HCKI9-020&i=MSvn)8~eNK#xv*5|RO|8>43R2}i%SXTZ>Ul(|=dx!)(q5jJRpNya
zq8w><A7n3vJqt@vJ>q_-!0_KWxw)Hr8tz57v(4mtIQQy;UY#Bxb)7c~_Q&$sIUDxY
z%Zt>07ul@wE#S|C(|$HC3>R3hPWdVE_fNL4_+b^cm#Wh|bS6~DeY8l*I&!2VTX6a(
zp?D=bm!5VB-a@fQDo3n3LMu8g1&kG5tv{)ns8Xi<OSkIpY3;L%gx{<Qzj4U@+ce9S
zhdf2ktt{|sE!MiV=ujQkZ_PK$Hq1Vhr?q?Kn`K$ut1{XyaD$keW}nJi74ZM#=Ba5x
zo&wW(9!>J}G!95%%+ylwWnHrJfs|;#vZm?Wado15Vn37}JT0mpz9;5m(b3bQ`ro-$
z3bK9b*wMkkT<N0Y($Z+7^hfcK@#_5-K9qH@%9cn}=;z4oFqq)(V(e0t*4QJbvWh)&
zN^{6pg-BbEaDJg{%~QUvsG9hf=hB%xk2$_aBCkX~nfXOB)lOOWX~hO<Bi3h1ACqR>
z=q)z4a+iBKt(ARuVcFw1Y8z%B$UBrEdc*%g3Da&)2%l+p=dC#>>XeoXA1&G-9oGNG
zM5KSyrPK9thf9UzUoLHBUp~ols?I-mX&(*Krj>#|y#-k;A<9dn9Itkn&A5=|+9R@C
zagP+^HI9vL6Sl76Rl6c}@EXUqu%p{HNjt3*ELoe`%(>U?gK5)x!Ma|CoR$P-j%eX8
zf?ab1-moO83s^Va;5_1|P{#D}H0M!2jWVW<vm=7al9QTm@-|AGQMOC7yTRLd<EU=u
zW>o>}MhVwFA0}Q_MCjTm+!61mQRcK!xMQvZ!a#j>5$lCegCSCZvp=tQ(yj3LG&My<
z@-JVf*@0w3hXme&2%g0gJf_<-ADPRLsqT=$IU$m1$x(-A(=x9yKd@hMjYC3{p`PKh
z48u>yr-%MEnu>nm+^Z0g9n-YnlpNdBd5*KwFWlS9^UJEa?%aY2%%6SzWERK=EYn%R
zB6TW?^*}Ylj&}ZOuNcbx<K!pEF}(PHWY$0avJ3BZTb#Y@1qu|poP=7{7R28;$8hZ2
zKS!n?4A*6Mm&om&!g0*H%u!IZT>15@Z%W@1_9Z`mn*98I((`u;YRAsqbbPTuhIyu1
zS(&s>wae3o=T5sW@&2R#F5u1Nx2_ZHZyvL|y>7vaIm%CZIP{io?%QbGCzvT{Fa1rk
z{H)V-o#b#~2dB<hW5L~aN@bari@a=)Jm-+((g{iDR{o*B!T-U$3Ac@!Dw*E1TwA%h
zQLsk%#<Kqp#2n|f6&?N&?{R<c3$B#HKjKBDN6Z=R)lH`A^nFXb5~`{3KHSRY>;Ff;
z8{P{?xz${D?zNX!^;|79!EN(@@mjT-`76a1$SEWV{fR%MQdBo-&%fW-VU^^4zJFyO
zz>TQGvL!ba4|7d>xaxwvNl4l2OU}#oT3<deG4sI*lZ5oBHO7D0K5YCD_%p%$SYFRv
ziQO9yi1+@L_?2m(YjQ(Ks_6WenJnj23e+0+@&Azcu|W0FglAJ$|9Zk9Y~-W1V&7!P
z$<roW-B^G1!~K8rTlDQDul|cSF|4`o>`*NovZ(vVss))@Yka5we4x$Qo2c|o{^q{Z
zsedP4{JFoTD0b`B-;E3OEbnL9Ss9!@!22_0UiAn6G}XQS%lP^au`@MSmHlLvss1lt
zUwH0nWM%*DFJ-GHnzenJ`nBPh1jpKs&%6J8);odl=i3MOZ#pht&dS0&>q6Ovyw{Ai
z>E;(iC6>IfyqP_Ft3Q`)<i3s9*%rR{Fn<)XtElv`>*)(~FMnjr>-bwKQOO~)@{Qw~
zL=i0?9fnGW|6wnc56)6+{2KN!_mk7US8P>0_GVKilx%rnY9%duFe1U|tLnqMY^%6-
zvuB0Z`7NE^_UDOB)SrwsPfjH%d(_HC+HVnG<oxi|xq~LVD(Z?DSpVtHkFO~*)wte#
zT>X0R%ReE-;Xl9lI&toed$i~4nOk@NKP$d<_oC^>S)AGjO?c)Ua8oP~_`k3`C&xM8
zvw0)$)aG4NcZ=UX(j@GYWqWAr?VrE5Fmul|zq0h@@5}xwPLIP^-qPRV`0$ML{Tvl{
zC;Q%is&C}$g0^^s+}iZZ@!Nr?9(K<+r(cnN*Xs3K`e8!g!#is?Og&klZTeESZ_c-S
zar;cmcbMKNTRYn>viwTqg)O@`?aFE8j4QHmwQ~CQygPN*i-fkt#$W!N{Qa=w<;;?d
z{|x%AM-!6MZ#Jzqe|<J^?bZ!OA`RldFJFIH64uPD;JaM^ob926yD#R?HgujMe3oH?
zUEH&@@&J1)6^mNkK!&fMS?nD5^NSa9v_3zg%aD@jEc>or|BV!{xXBgM*sdd5cIT4T
z-krvF!FLK%+#fMdx!)pE-8>6p)*8O~UA)nA_dRJ=3qI+Ti9Foe>vH&u^h@*{bMNr=
zt~eDO`nv1boPS&m3~>u~nBO>d?DP%quOf5#N-l4kal-hWNs#xK;@>hG9hvX{KGU0_
z{%G1a&a(IEa&lGM-W_nuzp?M*-PSeNtAGEgKc2L=Pe3B)%=YBji&vgo_wde!Yxj01
zu3W(qZgAgAJHy?{_uk&t)9+hlc;{54hN?M8`26_0;P>1$2KN_tq?(sm?JeRkjL=x+
zz<w?3(AwWJJB3Ucmy3!?>1#PgH44nWyjG;~y+fm$?|*d;#p-<qT)b~0KB>DfT(4QQ
z|JI9{jsl;X1-J}NHf<8wC+_euOf<r!`gGn}9S@eDH(wR&?OiK(%<7Khy3ewuSN5%6
z{GjacuU}%{UP_*9`lw*##a;5yxiU`V`Ky)ecK<b|WCaSnU0~0=u6X~k#mfrn_g`FV
z!7aaV|334*n~DRzT1PY#tnS&)&9TnjT7p${7MJG?g9o?1{7kPBU%>YH`8ttb!gD@2
z>$ldqEEU{nk}$zdEA_yxhM9G8fB#S2{PM{3^;T>ZAN6F{$m>hIJI~x~#pW>gP@Y;r
zz)b&{I}W+LE1Ak@`kG_QL1E4Y-u_8#|98Eb&S>Q5A$@6i8N*qngY1S}S{4gj!h;<R
z<F@?UD!OaJ<{wdtlCR#gU(I{_TmIIbUoQnL<d>+-WzAp_ophY()dl$mzL0qF8*R=j
znnNBQb++H2JNH4U8>3gxiNh~k?^``{>piHkBXiZ?yAw5kOyy|avZu%4T6GlX29qP@
zbEhiW2;3}xyr{6+`opF4MXTLD85ZWHeEitFomYWtQd38-)2*gp{elFCn{WQrwq)+y
z-NSgdsOZjKt*rJ<cUdm=E$5oSne&Hn`RcyIfesUL{F|?5>}SyzX!QAYeSYX4Uf%Dr
z!7J;cRYlkItjU~f$L`>##L>}d9Z|{O!sK_nR&wS!GrqisQ%^891oJqoVE=Q)z|Ld6
zfKtpPYfT2>$B*Axb|;pdl7HEGzd&61p+?6U!Jnp|Y#AJ;w7!2W@Z`azZ<<y=Zr`h0
zpm|u(q+oHwk@IN>V%^wHnFZ7?{5<hmRG_I@Nz<+VFW*|VQ&JP39br`d`)8NkSu2kE
zd8`3Od+x?(+-GoX4|w3Xs>4IkDfszjJ4WXFVht6`pYJGYzP&N8=cs$k$(|ky2V+<9
z&pdM;o-50{wQ}*ReUln<QrM5b>z0!gYRrBWdg6n^P0re_4?MZLue0w`eOSO0dAhFS
z(<-G0YIeW9k``Z=x?!rTptYs(-C+@XtM_7s#ny}OG%~+aR?=5_F!P~W0@o4kq=PDK
zzCqmYnI~OSnIrp-^_<<LxWd!+$B&x!eX({ixwg7t*5?rGS1ETTW6b3YX9%CsZ&a(3
zd#-&^^n7T7;MBC;$F8)*hWy?j=vz{-ygIns=@0vw(<-k${f(q!6<#J>onbNa=Y^`p
z;p(b;SJxf=ee*?zma_Q<|GT<d`DGqn@3-z!+R=BUUUr}HV>c1c>%UX$^baS78Ew7$
zn%jg|(Wkf0NA|Dk#HVLAD9@>$zgeR=kE_+_;x(^ZTt$~Wk34^5Rm@%xth=D@t3(+4
ztc1TE1yk5p$WGvhb59MJ;`NHXFW0Xng89+Q1MHoK%^$*!T}|0|t9I+mNdb?UgJd#Z
zs^2NS&{433S*GEp-rt~bucb59XP?a8d_vV?>7v8Zj?rt@MXE*yNGQK=s|@5?UQo5e
z#hOoX%caay&wJR8G+Qy1Uv%YN^5@8pwgv|aKeM6_=XO6Vs#ST{@Z@dVo;wzQ&s^3&
z*4}aQ<kiIF4Xv$Tq*L$z?)v4cbUaS6F?7KUtunI*UElm!51r<2nd(<Mf3@d>5H3BJ
zC+{XbfB%T<V><KoTiX|Zxb$F>d}vI9%2^Z6HL{OS9yfV-A^yqF#y`xfkGo&`Aow8F
z#I_`-<!pyT+mot|*F4VMx~+Mpv}juA(nU8^l_xTLzGwB+7PP$Qq`R8wX+yJ(sPbz@
z9`P2_>Z0}QWEhQOIN}R@6&ZzB$NbcJQV=V^oTgIp@Qc9Y*D2jn%cp2$Hr#6P5q4&u
zBe9iTa^DG7$x{ZwA3kvYJ9{9@L7eN;n<I<!OA`cS{sbI+xm9ak=!?5Q-t<~eIdJ9b
zk?+%dI$i`k5mDFXJfi+UJn3#-hMTwN!s}{_QhswK-%sUmX|*XYT2(mfi|nqB+V4Uf
zh96QaTU)puuzsGggHOoq<6b_W_{b<;+tP-YYaGPQPdoS>yCD5(p3*Y*^5y_BMup|~
zG!Jj?O5)B8Ozzry$@<&}-Tyf;ks`Yf)J{>&TPeDFiQiQB*C`JQN`JALDINNEBEkAV
zU~L}vhb>2rIa@p^j*84#y-JgN(m^eYug10hT!Ny0pFVRwn4Ix!N)Yp#Vu9ra+)Da#
zZO5+39Qkg0fj8P&szuoI(KW?c&qEz2pJ=caaGW}&Y=zL=Lc96y(>)JayVT^VoRZ_u
z``wWKFEW1T;<!ua>O$`R)K08Yx#y6@Diru#Jk@NEvz6}#3r?Fy&u3Zn&*Xk6e*8#8
ztKTht10mTH+~*cQR<>Dj*l2~}gk>K*Le6HC|F$cO%AXiIRd|Yw^Tmf#3O&F7O=&8q
zY_2fwidFa!d*zk7RHbIyBC(@$=PE@Wm;B2k)heXEeL7>3T4>_K#oD^cJ)7n}4U|m#
zqRDr`<(S?4nM{dFZb?U8a`&t)n^L~B!By5`qEDtO`<=>-ivtx)JQQ8Z_B8C~TX2kX
zlV@1q8z;kcX__xxCU+_*S1zjizbUKJ(o$~vyG<TF;x+>R?ATj_tb!$Z?%ABNU0e~?
zVRMl&gn@PX-iGRv7v`0pa+|isP={ga8|kUhS1R`Va4jm+`BC@bzLV~O2;W7f5BItT
zX%$@OJbU1X;ARom2L-bo_c9wsEftu@aWd4=auTECk-1N9cf2StlRc)jQSZuUk$tjU
z^SJC*W>;o3w9DK-u!^PT|En$HQF?KG4vw0EIja~9*p&kmYo_^pnOg95uIPc^Wu+;{
zoYh|*<+FOrpkB*Y@R2pl<H&=QgTj&dX<tNC-D7$ZtUmtqO472nacFnzTi5+G*816#
z8$vg?cDQ%%n$tS_QOlMC+EGiE>Q7m}Dx$gX|1NLFrE9}grDHraq%wDS`_5RY-ekAS
zo<IIwsr-!>C*~i%5?y}d?E+!xsVuf8v#gjjr#r|h%S8NsBz;w`)v17mBVk*t^i@Nd
zr3^U@lC}P93E#Jd7(HO!RqqsjaqqFGX9BNWKhfy0Su@w8NovR930oU#noZc5IZW=Y
zmpgcI$@JG}W|<bA3~`)rN|aUSi?7z6XQsk)Lry&seHhW36ZKfMovF{E_5ZR$OYZ*+
zzi;>o2k46ngeK%|;-5Q1ZPi7t09lhJ_7|3{zQ*@Dn)^@f$P|tfjOR{UogKeoq3R=+
zZ!dW|CQn=u<H2Sv`f&B--pH>BLC;UwaIHvjJjt;pZZS)!=5<l7?R!M-IxqJ;e&<B3
zSL;<%ftbbGFTEx=ym#58uU^X~RNWROUheZS@k)1dlli)a3;!SW$y#ieo6}Mny71P*
z?WcmwLQX8bTM{I&jlUsH=$1oMmh#%{l}<4?k`}G<SmKp+Wua`?Hm5D(vkc6b-k#vu
zS*AZ}m%{0`z^`1fT<dF85_5yrh_W4Xm@H(Mv72#sq@(|;o(7}h5U$qtDWSDFd~8=0
zb6DpdV=a7}DG+zelJVLD<7p>9iU~5E=$k6B^fga=(T03~=B9R5Po`A0Wfln!T176M
z2o7vib6otpbK-55AXb%2<($^5B?=-M3bYsBnBEb=cc8p*;_HR^(^=21cx<-(--b(k
zjEy#-kCeQWF8V*3Qt;h+v%sV)4-6PA&l>s(>*#v19)J9JlS_xm0X2og3$!M7eNf}o
z7PVA+D9m#((TJDT;`PMSOZv`vgtcwb>Nx!TvZ(M&6^Ve8PU-7-+#QPbq8t)kk|s6$
zda^)!^Rf#j>MQc^WUcbyUsXNnL%q;)o#5caqL)rb0}j5MFiW|gd3%?b$^ns!3Z9vY
zoNp@}xK6$~w5ONLBXX5*h}?~fC)QRiO_o0V(Bs#tl7mNXm)_XIwg1t_kfQBE!ZBj$
z;T`GmF7qoCxHy-ujC$i{uD^{@NcL)aR`H4xhK8F;`db87Mw=NNJTXN*WXIRt>`L-0
z)a{Stc$9GlyeT^zv+zJw9M{SKX63>b`B`5$E~b1iQp;@GJ86~O^t@@(*Hm`zUNiR)
zLvJ|W*U1SJmCWZhxCt}$X#X`i$WTAe>CqqVv*+|VUJ6#Sctoa%<=$9wBV)?(zaCqp
zqeS$WA8Kr9cG(ooa#883gGhVBgmSK=)>jfyW+zfz693t)_v*DNb#<2ud}!qEd|2zZ
z>%vpbU5C!`IK-TAZjC$nDU&s}W2r6YvD4oY&j_(?jTBm&b?a;McGF<F>c-I1PkeWU
z$nR)zaJJkk+*k5?`Crd#(l-tyzAhJPZ^-SM*jBf){C9DM>5?_)T^i<UZSC{X;whf0
z&Z5?R$KdVaNZX4g+}HWNRSri+aY%V4TwkW%c3LibIqSCQt1T;(d{wWo2VdoVx^vB|
zmlMjEax!gHR?GVtGVd^+AiG>eyLstpTaTMjLd>d)jZ>PVzU*0{dMQ(KF?(~tqKP&}
z<&vHXXZ}6VozT>_X2X#kH!g3zu(0KsT*2a*T)}cp%32cVb<R9f;cLDrc}?))eW!R4
z1@TwH?G@KDoEM!@nQgAg+rxe$pJDQ>DY+4z@u94*xF1|yP~dW@PtQ^6>hhT{bR4GW
z2;5e-sh;yFue~yJVpRCE!)N*~*QUNY+U6>C`b|B%{FS92mzgPbu-#?)dEh~d$*ff`
zex+`Sec5s{GDA6DWOB?bMXmb}PrLHE3z>(j`kZu~Tcy6Ba<T5!6@sP<^^z@g{iZ#h
zykkbj>BL1FLiz>#lXp5?4{&q&lyt~U?;K~?T8Gz(%dg)(p5b6Nzwz5&ovTMD99XK9
zVU%Hg)UYekXhDCqNamDA(eSKeE}zr4Ta}54-sJwNzPIlMAOChu^|)rC8Gm?hd}sJG
zC!zIK)ziEi{waz}|E;!Z^0{n&SF_FW)`8av{aI5!wP|qASFY_+(oAQ+k#|S=)RR=9
ziVyRaIY;cjAmlKI@#Q&fWgb84oC#khC2$^))phCPoMCnHFr!y@c>BXXwucj61gze<
z!geZ8?>6RTrw%NND~X!0^uS3)*_26!(VXgf?JHd3)i~D9{dC&WB`xO6&D<YPg(sRm
zY-sL!qR?vO<j~61r83`TVg!%%S%a6(dy1DH>yY{$&S3fGM01;I4Ex>im&u+I87%_x
z8ol3E&dPdW<?)8$IQ!XUbCj$3W=UwUr5j!T^2IykBg?ZzDvxKbG%1S>lX}DO$(ix#
zqK|s>=bX<>pLLDJZQFtR=}Agcw>p?c+WFeH22GAr+c3Xw;b!jEkPlo_)LfXPWOQfd
zOkU1q$~jYQ#j}_<`lfrP+&punP3Fq+L#LN4=#mhS&vX|_TRio>Yi&gMa;EL4Pj)YJ
z$b7$`iDyx4(z7=GiK&{)Iyh<*|L@v2!$v+g;^2{Lr<s9IFN$cUd({T6wBb*RnQD8c
z?WcwZLsN!<NXru~tw2B3h8cSbdS@7Q>#q*s4_wY6zm2CT@Z~M$cvi*j8VhHX$$s+W
z`#UMPfsOM*u3(wXiU|umTcj?$n3TKw`U!>kQu!S6#~JQxXlk4=aa`srlsX}TQDCB9
zo0O@-x0ZR_oBn<&KD*$OuUS&pqc3Nl*88UIY}0W}5bsjET+rsJUUftBOX(J&B{GZb
z+}*mu1x=al4PzKn7BVkgRCm|(qmsxhP41I6*H*CF*r^m;D2{p*vBGH1iOErNL4Ox4
z$(S7G^ua)8j>1-PO+l`y3J1=1DqZA`nRX*9a+l1riZdHNm?dOLb!zn<m@2^*(?02K
zkU2Ml-JB@%d0cbZ*j~HoaGYmZ{#`a>*OJW0#Z0UXYyLKgOi__hSL$-J6L^%m*q?vt
zZm;8$yv#fJSc7KDK3TEATWf;k=DAZgUVL}ufwSUQ<{3LV46?Rtybv{aX}9*5Y>%L~
zA1*wUEw{MKGD}8m0*^(8`-yv2lJDw5y{G!9nq4`tw`*mY<!71Z{R>}QT9Lx`;c}t%
z41=Bg@;uXn-2|AX2D`S)#uhUAEmH0X|EL?ZTkgsw)o=0(O&DI(pR=7fSLxfd<_oKZ
z-#Ga;-r-mg#B@04!l8uMQdi?Q>0SJ&acQzzP=2t(;@RG6c01}c(pOAaUGi{&T5DUN
z<;>E{vVWKJvoXk|bQF4BU2?0CSM+H`<idsRvz*Ob3)}p3r#Q^(JoRF0Oy=8cmC(*r
zn@(Lm=M>uOW%%R#<rkM)?UJgrv_2g-kTUU@)XUX&?8)TX(*-wNChC3n{cwo=vgE~N
z|GOQ{jsZ*)ICk9EIo8h^pIP1WF~n-}w^QHx5`^`ad)X?4EPl9QPLu5n8P2$gZmb{d
z{^~H7Z#ZPH627VN!2XhH?;e|^l`_dbNZbG7=D~(z>(Z4bdY9xK72}^65Z^2<QKz-#
zP3bR&Y3#}3FAijhWS!Y0^I^&CGELT3yefewHasm_$yQzd!cR$*t4U7d?SurD8I!mR
zxO^F&a3nNpBzOAOiL__9rZ1lxw(?jl^E)%~>32nDJ^Yl)apkyW@g}2|W;uft_C;GZ
zv;6kpKj_u$&dTHbMqA0-GenN<6;GhhpDvO8o1R~0a<sWR;p`G6cQ57D7uv41?|Rz&
za&^Puw%Y<zwFLB6{AF2qt~ttd)^fgAGiTTHY_DE%|EKONXFl~S7By17kES2qZg}GM
z9M&DxTnk;gGpBS<_L=(S<l=PKX`k#*RqA)}+I5-a9jl(aLp;^!`i0FhpBfciI?MZ9
z$~*+s!laG$S(GCdaDNqH&)vdv|0Rp8U5>!bmk)U5wk%GVagf=3ZgqUWpqKM~{tJuF
zOfJZL)ng^X*;+7tvF7QHxr#egJ+1ZZ_%;e}(r(d8_EY`aP(CxSUg@^xzQwfx?7R*y
z%{Mo)bX&|T<H;1*yslcktE2P6J&`VfA9{wK=MVmKsPeGnw~PN`z&XovV(o+m@rUyQ
z7Z%#T5V<gmr@*H%vg5Rso#9JEt)pu;M*Qp#`8LmUdwfa%^QZ646`iIf2Y61I>M`lt
zL5>xPTh?x}ny#>(V^<k}-O~o=V*!)p6B3oX!oq|ev7HTjtec<mMap+7Z&Pd8#?RUp
zxEEeisSvJuE)rGspl+FSnN{BdJ>GDI$er8=%MI?fH0hM?(Odr^%jc5u?F8$Sy^MUB
zTVAi3*u8a|l9*<l&-O`N;o4G>FMM3@Z*}@SL-U@VQ%AmMT*>iv5pI2}E5V{nYZSE3
zKW_91St%d<#>MERPeSj>gz5##D=vKtSbclF<8#ZWI`4bu#k`sI?a{>e;Ct)8RIOBa
zCCa%=YeL%l9Su7xCQg0xxuv<I_JoXSk?xEOdqZb7`Z6-Ji!L-gwO^~~Z|f<B$fOmU
z8XgCUWhlHmFLBsqvDKMty|Z01SKfD<G4bQG4@cLQ{^?)#<c43f+S;nQTH)U$8$;p^
zU(Puxbn;^>*TEC4vVrT`La*mrFv)BZo*%#C&<%&P*2j#qyjE0h;^z#VTEn?!!Gw*A
zdK9M|Pnv2i-tgAL=qt|_w~0+k4hsv<YHVlQc<ffOky2PY3-9+=7Jr0yu--V{TqhDR
zuS`dQ<=S^0@t?atM6qO+cukg?JEi}L(9I7kRPE-fBtKJq+ohB+MMu0msln-(Rc83y
zL%k)tPQ3CAT%EPJEzW;huhYEl^eY*ijh<rPY|rTy@^{N58@n~<@`b5>xIMvX;&xds
z)jRH7vwQ1Udv%0FMV=%-wVt~3c7pR<KaWjKvQHlD78l$3#pe6cL*JrTC@eT1JmdK7
z6<Lb|_h(NNTIU;G!#TloeKOM|ljhKgub3~$ZrJPBGOw@y;T#|FM>Pfs`j<~Qo^mlh
zqsUyTk#z2D@|33yOm1Zr5tFt5t$QS+k?gg%(?;=A{A!cZOGR8+GEZKb9den|vvsn4
zn1tq}cbBppkBD}NI=FU(#%3MU(B%k~JkZehr=fK#$F9i<fe{}ItvXY!7N_makl$73
z5O&z8#wjuQM#hX=S{C~auk02ScxdLhve&NK^?9+z0mo-A*F9Z&rD3PsU%As4xC>VV
z7&M>fc^t@OxYMukw`gMH8X=>l_a7!{Wot?^I~7a{y5)L`yS3$>h|IY>C7Hhyo>lC+
z>GS8srYlNOwq3_gzUG?Vw5{d(2mP|DRXfjbW?h|a6V0|#KH=vr$Kp$jk9Aqocb4~D
znA~?fX{v*}Z~i3B-HSbM>}Q&Kd%_z&dC3=B1zmqB{<ipAE24B~qqK#|1#!VLg{RFq
z&FqB=YhvFv|CZ}}{H3ly=+eV)KGBJ`=erK-A3y#<YdM$l$u0M<Pg>bokSA;Fx$qp<
zm9nxL-t)y=Lb2&_()|sOR8EAI+cmv3e)8~z+f=@oXOhgP9n$X3p0VuJ`vqJ7FZfeX
zu-9Sh7VAl+hDEdPvN^1cmSGc3+^A!6>BYp9OJ9r=Hr(H=wVC&#+TPt9*>}Q^^-QeU
zc}K=tohN9;i!iIZ7rt2QBwkTx|7q*SIkEIb;j7M^u5+rB_8MC;|0r0cA8NH<h_x!k
z!O`4*b&XJ}zK#6!xvZzSZ$3V;B6tnUa`~r<7oM;+KULGxemJEuOXtA(9*0DkO>>lX
zpOJ2`^iV!2Anx|_v@EmT<HjTX)9x!IGT(Gw`Gw`Cu24~_su1g&ciopW4rcIZy{*bB
zs{CnT`nKfAB410@BtETQZV|iZ@LBCT7OKA2GsbW(%aUHUB9mFYj>UYlJhya<%;&FK
z5qP?NLG`YLeHjwV7qcm?>hBj6O;q}EL5I!jRG^GIvwGXAROe;Vb8;7-yDYk6HN(O4
z>5n#Y78~B<IJoSKOW;1C8*0jr|E-K@<KO34fA6HN?+24<yXGefO)5*wk;;he;#Yp|
z`qcdOF^jvCV^l7zoYU}_e`jn&(m92X$rbeyTbHpP>38uItGjtc{WI4ze%Fgzl>WY%
zSo%99$1=8Z(w?}J>*riNFB^C<O>3iF^Uc#!*i$|FFZEhvuF>)MaI;ye?atQrhpPiR
zC-4h(D)nWwxG#SfbCEew=xT=B(=bKnsZ$FZ46|l-oRF}U&QNQL<}0~=CDt)>rpsCW
zm6le)9~D^^mS}9a@9-}pW}YDPOQpcH<w0uRdu3nz{F!wrZ|0on4hi90=H5>d2b=Sl
zFPK~rjJk8sb+)X>iWhe5f72(fU9i=2bIp^Y&NZRMM$CduNfQ?@DmS_tDYWV3lPZC#
zu#6z4ZO^wUC+%I<dFXVzkNTD4DJL6wyr(oSxHKnzVIJ#iySaX`Vjcb*CyZ2CxTZB;
z@?wegximA(MKmlzOkl2Kf8mNfp2<?vjeYli(y}|?kR~1+?aHw<!Pw~Rx&z@We(u_T
zt1(z&(hlxSBaxl|)l!s~INw-l@xg=DRq(LNX_JrQt2VG?u5f*_aC5TQ{8LNBOyWTM
zZ}!zbNMD`f+`A}tvE=H7Q^fUpr?eLdvwGULxg1Si^KIda?5u*-YifO`-jMiG*P+Pf
zdAh>zW8Vc)kIH_V33nN~vW#j{R)uq=o>VFRrI^MY(WkU_cY)Z|+)|b<`K02$xLd0h
z$$2%W#-IA<T=&{9)zsL^gUv2C?#T{q-X0T!V9nT<B^NipJ}u{#w9<s9NZI9zqhq}(
zuZiFFz)YjxTyFogf9wt5T>nP!?-g!svj>|DZ*vw#KPlVO(WPz3eu(+xe`mLIS6-Cl
zr~H_CB{8DeN$Ju%hS&m$IZ;o-(u1;Yu(ONHO1yb<XQkUMfs7Bbm7+|s=R_5KjDj=&
zC$$*J<}>qO$h^Q~A$q29SAuk);GO$@{!*n(hxeWnocLs--$tJ+D!H8@v*nF~z3!^$
zy9N7jPR>i^x*GD0;jiFJ@kN=lj(5J5xTf<u#kMvfCu5mgU!dALqpDnkU9py845Cju
z4(82xAT9ejL@4#9_+62MsVq$alXh?M*srxU?8zN3kG5MLrw?rJa@Col!gV<#)R6zt
zZY}?*m3baZw7#Y~2y})mRM9F6V3y(%W=YMAU2-V#-)fB>2f^$o(>lMX9g6cj>$hWS
zm9j&a^NX&c-2&}ld4iUPlBajA6#UY>E;MkZ`HNjER77UlZSP7ybxC@%bzxSIN@{fM
zhORBi&Uey&#BVP@V)o_66X~VT{kiM<F7dfn>bpd`^)EcKpmVC(^rp9=O6^)|ZFUy5
z`8Q10M0VX`xu@_taHrX^g9<$>F23A*GN4++_c~)@AKOozW9=3TyBu5g=`$R&s!`mP
z%_3SIsCZGRWU*%ZV<+}&ubA)smJl@B($CNCsx`Og;3bYplU5u3opWx%vvccYr~Hfe
z2?~4hw}eA==awh6%!zDEn}q_^txuP1Sm>O+{oAE&mrhJc^^JLuowBbtZk_0<b>SlW
zmZcqE>e6~ldD6sJPiE9;7IH_{-?C!%yx+KmM{WC*yvI|YOW(aQ!$ZLT=+B@&!Dfej
z8|Ny?D5Yyo)11r05c$V$v$)}{D6OY1!A*07Z><bkpy=|#fzPc)VESqoC7%XMRrdR*
zH1DhkN>bd;x>{NEO`{Xf-K>~tH&vK*OzNEt{mMVrP23a_e4|Oaqun^RS5u;Y<+_yV
zpEa^iZZMw7HzTH5Q-HICPmy8j1cj*|eWEV88R!-Yy?NjA^u?~?C*QP8HE&FEIlJHR
z#$GOS?Q=mpSkn%!>S0cJS+Ra)W&<zl#+3`6pW|b!?d;s=_Dt`S`QIs77dVgF<WG9}
z|LYm%l`F(HAIu6!<KlT^zhahK87G5G#NI;zM>!6<ZHV(I-0Zzz;@tjyZOV4XFIr8C
z?OyT8km-_jbHd?xg(;RgES^Dsm#AKG{%}j;$IYo;J@G<pWskHKmNhroJI+2;I!QU9
zbD|M5TS<`7@?)Dm{F@Nvx@WzP0gv-ESAl!%Qw=w5JTX&BuFA_MI5ja))$7g@wWME;
z_q&|`O<r{I!*tySp`$AsdByYnM9%MUOi@sOYklPOhfkNRd+f~a{d<zK`oDb1fe1#;
zL<OCTbDL(m22XjrxWPdAMb8xL)6#4iKhK;}T0dRz23wq0)=vW)S4Ocvx}F}7f6v@6
zY84XP(0sgF@qTDTz`6;?k2l{nD7okRjEn8w4x4u|8&gvLBu=&uVVvaoifhNcbtNlB
zq^Gh^ta{?Wy19olD0!z#_rlc=w+G#nywC5_>T}`59N}|q=dQ;`1TSV=(`X!)Cs(uT
z&yn`m7k?<llqo57NruZjOcOXTH}KAL?uk2jm^+wSADq}=o|GlNY+3c84}lglEV;K{
z;!`nL+5Yc%L&`JDm6a{O4t@+0Jh$$Oz{HQ|RNO;5180c0by`fA=b6{2Jnd^jid3?p
z+dHXc(#O>8X4F1n73PsmUAcE7|FL@p4DHr_hs~GHP+b50#a-hz@i~*7?(i-Pdo3sO
zm$i%W>6vzxH_`3_8o5hC=P89)U+ReRXO^GquVt9Fly}XgSs|GxB~I=P_|lNQ;6m~o
zm*s^O6ON`U%oM%(b!y|rEs1Rhn;#e7<4`qEduv+tR>I%m-jAtv!Bg)YXslZ7Ro~*(
zD<G%!>(t!DIi|tf7v?vlPF8F%*yz`?Nma76`vHTObZ4PVCF_LdNrh3OT@$?cM0atk
zeLO67diRu?!lh2@CgmP(HVTr_pTxaDHQnN@_G-n@%Tw-8lx@t-e!6<9?6w1~6E_;M
zRvGuS8#^e>+Va@b&UC7mhRr3(7e4n-?#oe4;P`Tq?X{O>?;NR8`ycxI5;y5S%k9|W
zpnCcBT~<RrU%3*I9!AY8&mFBfZ!ZwcdK_RCo|!*!o&U=1+BYwFu|EB<?dX{=C!FWG
zhzXX3eVTb<23yD;kFu-m?OMHIldjKNbg-a)Wz<@qDV;mB0$SdE-XC~sBiC<d*QVWE
z+ppy{d;DoUxoF{TxlX1-8@4<YR#>fiOLvL1;mchw3y$#l>kF$&O)Bk)i&B<SJMwV#
z1^Juqvz%)d1-R_jbD8H-wD8Bqd6plP3S*y$_VfH$JR@Lb*OBfRVFg_#y9ur<F5T#h
zzhO7U__p5?lTb-cmhUOAW@y_~YByE9R1HlGZQvK!x5#Q%q=;~VBlj8YBkh-uZ_kit
zwpYsvvFQn5N>vK_B(O*DXY?G;$3h<^DzA72bL?6F(=FMJq4d^^l~1B}uU^^z_e9A>
z=ei%Ke(c=n-g$Gc(fS0T`%b4^Ya<*S;@p;*Xr%70wK%zV`o68~`+9yk96oeV>><x<
zM~$!t{WJRvCeD{w&U)j-ot`t%lRZU$cl+M|)O*Za>V8Jd)xtRkFM6$<GV|EBdK0nR
z&wiZC*#B5(Pw$fLPnq9w>F=nN-25+~`u2;b9sf@)ocV55#w}sj&>uYEA?7I=$*mv1
zeBLklUbHeqW!J*3##8fK)kBuHyDeo}cxgVbR7KaUn!l-Uc($w+I4F|moH(KI;YQ=*
zKB+xSaZ8j}Wb&r!2%0VLl33-na^)h4pz@FTDRGxSNWJQJla4#O{7eQ%_yyMcO{-?t
zWG}tD>yr19N$+JOBQpwDUl9v&{SmtBaCn&Ax|9g*(~{v*PTH>p!Y%KHut|n5{kGoc
z_}Ml0rydi1+O@9aR$z)`+!Dp9ejy*!7FEx`bfi`3Z+)wLsX1fd)@c_ktT+GkZnnLY
zZ}75$mHTe|%23HSS}g~>YLljhYgj(<x}MB>hc`O7^`yVp6({%gUiFGC4wo6bW+gv=
z9iSNY^3+MiaFHb$9J}8A%71lewU>!Z!p|4__x#wmG(DN%BRKPe^Wx-NivDtSnP;?8
z&gfhb)#$a+s>|+4dRpP;Y*)m1asJ|o{j!`-B5pA>Hhj4vwR0oy?!Fhb)219RFJVnP
zc9~~$OxuD(Q^fV%T7sExeK7lP{;5aqN^jrB7bSfB(p^`*jyUj&N?U#nD^fZAGi04q
zbFI|W)4g+&7poXA^Nl}z%;5I1mYrXUzZ;iJ7+3Q!FTUfwlI1=3ven7qs$r}j53~z-
zawKGMesEb*XykbG{2cpO^{W$hHU@6pGV9{YS(=Rk`3DRhFTRv^*hBGT-!Z|tRd#&|
zr-Y?`&gN=3@b{=B|FbPh6H1Rb2S3YyYcW@0Zsn=OkfrP6FF8-UR>XIPL)2$R=CvtF
zTsv<F+&uLso5#Yqb?(DzwTLpU8xFf<<3($NKNdOmt(w%9tQ=%C`%tO&-H$&yJ2QKC
zt`)c;wBSmnQO(IL#^VXIdY+sMKm1bnOb0*fF~zfQBSgYimG~_?QZ&o>bG3^v*QweW
z+VPi64i%sI7{6@x;{H^(U(9J2PEFj`{bQAwq_uR!fwn1vy>>CYOZ%lQT`iVK)+aZx
z{XLYyBEdF=<JFxL2M(@$_TWul)vp@{x7HXMBvi<q@^H+S+jnKt4!v|Kb_b7^QeL^)
zHGz?Rj}Ed=f9_-P_73Zdu#BX~^S#u(+*z{3KZ-sseaUrd%Ey@-zyH|!M6i8+LVVVQ
zpC>vd@NGBospJw^v?@-s?#z;%&kr=tlj&0BI#uj7y`pk4+uj*J7Aqx8DhWEZDWcu}
z=!&BgcO31^^p)Ou$n7$R+^?cv-sx437i~#Q_7$lyciJCl!rbNi=dAgRe#ZOzCFG=D
zt@tt7euYH7!Rpkqk_|;&rhSe2+*K*OOPl0gO3i)wvEsvDpT~C&{_9uDx|N}t|HUE8
zXZr1o=)fzr!7(+9wfoQH_PV*HG6$D=?YVI-$*)7H^Un$PWb@w6*|9qrOnaBDV9IMR
zzU;NoaN~b-V^b~jX_5;vvwG$%DUM2y^<oQNu<@FUa^(Mf$Nu$i7Kep8&Zs+>cSwHU
z!V~4rwW6)QVlHP*HJ(Yok}#2)IrVjePK5G%##6lde;03H%q=<g<q*64oXH1d>#rZ!
z-jlED7gczwF<JAi*yN*+dRBZp)*YB@!EpU?yByEnrw?u%b`lA8X*<bbF2yW;x#Hjf
z_5Z!hxiMyO(|$GaE;W37&4SO1&3C2*x9q;DJ}YP2R8L^vt1?BA<ygrs$;bN(Sh<d`
z(2JUQXuWmvf&XS<l2+0-hpJ`bg!ZcbdaJ*o$jMmLBy)?bS(bp*)?>YCi8DXVeB18(
z^wxR9ImO0L#rn8xr^cHkhdFF}$+(>1wj1lGnJq6L#7+L5=f${Yl1^__n$4}qLu)>*
z*<s%M%zl~f>-Vf;)=fXo&X=w~Hk&6R`9g-K@kE0wN4;1sy;yxKV`*l}H0jdIy*9<c
z4Z({Yqars?bqeCtdXmg`+v4z(rnrzD)sO$gG`_32_*O@-U2UdsMwEYzQ2oKh&6>*m
z8?M~&nw2qkTgB4eikUKx`G5YNy+_?w`}t#&rJIwN{kMCv=lcE=zZ_e+bB@fvSs^zo
z;^O9&%~4kp&csaaozt$ZZ94bTgT?(Dwk^5Ny}n`f^aFYA)?0!cD`n<#t(YSC*)=@!
z8T*2--xfT-kSJfWNIJCXr{<-uGiSH)OHLK8_o<1EeEuNg+$SOS$+};}pUw*JIPdmV
zd*<8JMcXtS8#Zqi+?S&jbKpe4F9*XF5!=2my1&{gXoo?}@e2nf78aU)5i*xOSHNQQ
zvsLr%<%L_6N`$wicI`FR5M&PR%>3hFpP#yT!u)U98Zz2)t@AP?=ie~7oWJkn=GpQ4
zmg(nTD6XpG<~?xl%)j)tE7x3i=WU#5<?DJ^r+4|j>lGy*C&qJHWk~JMS|aEY`t)(;
z<CFJyW}kZyx}3)=boIB)6nk0EOKx#8JC7GHZ4Cb;`r^2t#xeCz0u%do%-p81^ZC;f
zvzPj-J_RM(_vSu1sJ+x-^5XkD{yD}?UB*(e=gQ$fzU)R-*8SpNs;_-tb!*{?*$N&!
zyQVq5o9)Q?lIijLz6EPJZ(UkolXrvXda$3cSN*eFVsh6Ca(I}&E^zpeykmY^Uiy<f
zmuIcv0eXuq`L9LWEV4;o^h~Vs>B<@MKU){wx&2c1h)l$vGY76sTqbtU<dtr3TjzO&
z_qG*Isk$nsw#xEpWi8-}Se8<vJk8TP!Dzt`N7=)tOYL7Qd+<v8Lbc;dzslLlGBtv?
z)=u;+Tx^gx<-Wn*H+;MP$$ePte7@%7)_>s()=iX*n6jIzHdFYe?*gY*zxC@6EZr0G
z@XnDugFGj@hwMxL8Sj<2yK>>-M?P`S4!8I0U%g|&$*nq@jWzc@*?;lIv#U4Si&8F{
zXH=A{SpHC(8<_NmPi@s*v%N14e=JV97sxK4=zCDd(?KUbcHjELXZ|P&g-M>6Qu4*q
zV2(@8<h;O1llLDxD7=2P(9HfVIbU*%_r9@WzALcA(e<9cpJKqvXOC@;A78rOXZEE?
z?Y3084}$$`<e%P5WBDNRb=5ANEf>wq*7@!}H=RrV@w>k8#qUmOyq<9L>PwgD$AedB
z%}o1$@QBdXjCslX9>06J@`2pt4|ndZF_{<nZ&${G+>3TQ(q2eSaCr44P<g$fS#P}j
ziT~FN8Tl60oDfv+H9uLe`s?IgiLO_7HXpk2&S}xglzUf{<+rTsy<_cKxlB`xw|%$s
zrjr`_MJHbV(ztcvA$#N#g({JYr<6FR92R;NRA`asA;_-Oe>Qo_-lO7DD_1aIjZ^m3
z-SzuaV~xgD!D;JQE-aDw)M67nT}*Mtq1DqPvX%%5P2K-xg}j#3QXYv$y*(B+Qrr1{
z>hia*@|qvxCUsi1M8rCziZ%GobWN#mYmyfGT-8rgTwC;Y)g!4@KW?oO4NE<+?$C-*
zo=vBe4O{$Wrz~QdH8UrE-O(cTRWr7)Dmb$*&g#0@ta>5a^HYABel|b4R$Kp^_q~?P
zil?6(uU}XDUi<Ww`uhtj*1Uhz{duX?tdogmpNyRqd-=AU5MBBH<*JWPpQbn;S?}p3
zAz3pkFH3lbiKFXf-`q0~Ps+#qeRVW!%OU6c(=F!*7BR@STx6{8P}2ETzr_5`3a^Gg
zO(7@0++1B(EODgc->*QqJ1eA4+>tE1@KI!)?V65-@;^C}CB8lRQa@E**Fm_ln=d!&
z(GI&uZ=M7`Ftan?yP(swbjd}JpS?GQU(~Nnm(zE#WZbk^$K2;tn#97j9*WYIv!^Is
zinzM?x1RRyvL)7HoBevD6ehX8Nw`*gnD>~sYxcD8<*(DPzD+D++1X-L&ienp&wF;y
zEm>kJ5Bs>JL)W{0mEO8#*+;%weRZSDNe5mg{aMpix@QGn_q3-`dtY|?1})H2a`TOM
zIcpJO`k<qC5^t~jy0bwKKZ);e6<?LT|E;s8v+Zl&ujl)wte$f+Vy<S$oZ06(3_ed(
z*q)hj<mD-LiDt20anqWKUsT`oN*}!6I)5kYx<B9a&ZM2<yUYG-M&M5of!>$yxg1Bn
zSQ)6gKmWc-abxMj?WKQC<Z~YHTDmDPfyutWXNN-M1tB+8vrf?yy)`eY?IcT*yDUsj
z$}VI1rgrdHQxVIGEv+Z-vdJFV7}a|rUE*QS>Kk>++jDx}o9;6E-ch>L$@`CB_{68G
zznkx-nCbp0YT0%ALY`EM-rm4+#YsyZ8sGlBVDTbL>-#2Jf+>FeyM<fr7X_KR9IO+m
z>%28bP&^|rkzKjEHsn*!ijE^IZu=dd#hfFpwOO{ps&n<_k37YGYwjNxQan}^Kh4f<
z;(;^rPp_`Cx}|ecc{=yUMxitx@y^A+J{lTX880(^v655fSM3*TtDxgP3%fX1J4$dr
z_<kqUpnKP%zmhAyr6uj%cc8gfOKxg!;Vmt_H62|(OXkds`^d9;ro{r&+PU`UCe75k
zd-{aHzkgD!c9l0GmwFmG%bqh5iQI1gQpZ@;@l4H;87&LauW~;=n*36sJ0?Z8&h13^
z!UGcw`}2QFOx6F<y^5K!a|(ZIZzbal?#~A|9@;H>k8hg)yHz_5vDa(AVJkd+irXQ+
zb@PoSo}nv0El<DERl)v?yYJVE#S!HalVui6{#yBG9iPel>6+c55<WIp6;?N|nV2eK
z>YDZWlBku3QTw#I3G-zmvV~vgYR%4YWnZ-@;EdG*HoJLQ3nX_=5*BI~Rp|=PG~cr>
zDQ?}DO?H>gZkl^?QtyAZI`JPbm&&9x`i6Xpk#rTV4}P}%h&k8&deMIoCqhl@R(+Cr
zkT~bzrjL(#X1f+$cRo<x(!crJjL^H28#Q15*`H!q#Ix$c>DFG>RnBWqx&3(JF<pjL
zR&xb&Xt=@Em^O?4<@?rq`v1N6V{gCej%6<%X~lL%#Jo(6kz|cEE_s`yQnBgX62~y3
zYqI(BA(;=o%;X%CrB;4hxMha^)#Jr=J|P!W%yerue(YVTczN|%yK^l9$`?82{Y+jW
zVm-N`hv$V>;<12pfzzv=1nN|6eATK|dQoD<s@U9D3$k36J}G)R-Q&Z{9Y<UejvtQ<
zKdizQ5zMk6kju$zZvSQ#@!}6wxhm%lJxuS~KXJ{WqlcAt`JN9HD-tORO0|s1cse<m
zz4exU+hkcUhbj%O_X=LezQ0<*l_Tw1m&ICrc*0kcy({`AS#`~sXu=XBvuIJZMZ_WH
zAjjo3;xp$b2rp+|%=2gQkB=T1xymBbIk#s@r|uDMPZSalF5+-rq<uoVb?=64n=J~(
z5BI7c{}#CFkOkk%3Ehv&Bn>}2VVD}sDev>=82iiL-lZoF&CpxE;>(Iv4YO~$u|6=&
zcHhCpq<k~?@$zLG`8M)*92J^elh>~8b~SakgR`%L^8A&aAuBpx-krLJUshyq#MJ=d
z4HFHwt#n>mz0J%jp<0dosCdoA<NrLqF|94QX*K8Oou60E?KHZuX;1T>kFjq2?oyl~
zLJuAvP=35$U(_~OQzL!y0X2pIHWq7MIps{B=db4O{dC*-y{P@h;+!cjCV8*=VDN5P
z5C5cKpSTlcVm;GM-Z%zGl?WA_4nKeKOS9D#8Od|vBG0uZw!BNOnqU88hJ*C{t-Oml
zW{I8Nt>4^n_`*W*>fJvS|2(>7u4cC6?(@&t|4vNPPcZ&vRCwQV(xiFO7uL@)-_&-L
zf9f)aLWlH3*7Qwj%a<NeJ}td^RpFn2P6>;T?o;9pZU1B8^?%YHvohv+*&*p2R@zU^
zl|=PsEqwCh#5te2Mo;TQ{%-eJs$zBF%nUvkF}dk7&C9LyUU}GkTEiuG^wR1(7V~pX
zq(lYo+qvfb0rma<SH2ouHZ!~Wwdb^&cjz0AYh@Xq=O*<lpVgkHHJ7C#Y-L6$<Ki_B
zI$6Cgx3BWt&$WN)8>j!JOaEGW8}L?7*kY?A*gRc&{;~r}%fzjUC2p+PV*YRbmn#Zq
z7B>XdWJP>^^kVy~$RD<R9$hv2&hy2#uJExmsdznUlb+X#;>d`aC9kB#gq}EE;SZc|
z`uIy>*yRP1i|r4w3kL=5mOi<h#V^-D_fOxU63Z*ypN}4KJ7vVsGqI9?hTfT(S;sW5
z^)~s<3svta<SB78S$5p~myhwJi0TrLir@+FvO`3ejb8Yly^(9XWKGTazq=QeMar_&
zZd`K2@M)>s(`i4htht=AymYphyUdD|0<FhU9GhA*8aGK8-&H=yB*3WrG@|-a2-Cfn
zJN4^b=TBvI&uW`k&|2-$c0A;4^Z6CaEBZDc@(yB853=qv{^Q;GN_ECg7DJV-R~nMv
zg&h~qTCu?5@}HH@Ch|#ns{Byu)MGLD_U6<Kj|VzVkF<V0pR}gtN#^vKE8KT|6;;gC
zbrH6E#qNE(M&rD=+#jL4p<-?YYTpaGBF@cG^t*UU^6>Et_Ql-xaXD}NDnI0=Jlpcc
zFzebU&1;^;hEsFu=4!?j#Dp$<AL<^ODdTWVa%*?i@*4d!%lhRXc`QA%D>;U#bDoak
zk2SWLcUDaAzkAWA`t!u^@nTZTub$8p{<^D$vp@XF_E!0qZN&w<XEdG;y*zj0+>WW`
zzPEqu?R{mNZpU@~d*s|#zYXld`_B4iH@$UZesSVP%lS*sQu(f!c%F@X+Mr_hb$Ob{
zLC0sJ({HrAH#JN>ntsSMKWUq)Tjk^wrOjRD4<D-?3wb1JBy5p>tme7zq>eX6>ry_R
z(r=h(?`HKSPctE8x6Gm+95pGG?GFWihVfMO+N;&gUio9q9oHj*vdh!|7Fu~I+x)Xw
zcKorU#q`=a{(h>j9?$4D-TQ9V;?<4MQXjLbR<j$vKEG<I=-TO#krI~=sk;~G-gDm;
zv{qzx2V=*(I7a^Kry9+gyk4<Za4o877CY%5<a?0i$4|?hQXeljx~JUs5z{H$aO!^C
zO#3XZz~BqcKC>r1n5pHl;n?iXneXCLIyIyOG&@q49Q^&OCTvYtmHfp!SG;F+)otAG
ztlx7!^z%g)sRDt|@4mg~f7@H8_#oY0_=Q07){Po(%M!JHkDWjHC|hN9qT2m`|4+4Q
zElhWR;F!i0nLRObMz3LISSi<=GlworwwSv#^0kwF-<yBRiw%m{R!&+~zhd30Oi_~?
zY+>tpC$Tj5OHK9puXQl+>%1M}uF=cx`HGqsT)yZZTXxd(OVgaE6O23_YhTuz(&H$3
zV`ABacz2sAxiK3a*Bsa-pAyaKS25*!u7qIIsqa^(uH5^QT_j<Z@v2`I>L&x)9d>)D
zt-J4@+jZvo>oW|ZU!8;ACzdP_pKK-N`Nn27^V;^6tD7!)^sZ_$vVT~7wP~yL^HY0Y
za_2f;;kaaEYWJp@PeV#~qv3*HzeL83HYZfQKi%eZU*I0op=>jq*YRtjEdQ%m|COS?
z?>Eh>y0Z4&`%jy-vfV$qXDr~0z80+G@G$+I-;=lJd%hm}e(Za##SM*24dc*N*Bk>T
z?Ng99`St%nhS9&%`}zM&cR%)8+<wX4{2%X6e|fBN@SHhAk;qo2%^Sb6$8*;FH=VS)
zaL+RK*^|sRObK3KB9^lL<@&IX%NJNoIyGhC3{8#;p~1@gOv*HivzFZy;8l6MEvTHe
zGfu<u@vnKs3(72cKTRpIF)Cp_nOBi@X_n4*%Z=aoKQGPx<{8e#@07ggO!x9@QkM1G
z4dcz0oAEUGH);nsaZfq(<l?3KJ~hsv+lAz3a{cnLOn<WALH;Sn)~L@#`m=PVr_7%)
zb#CU2LX8X~(`}ijP0RC2m)+9yXyBT(<HOzw-qX7D`}j6ViR?8v<oI-RGS{U?fhXA7
z)8%HHm38ay+P2Jm@<TO^wJ8B@8x^`^798c>Brne{&&!`WBe?PA0so1*6BipBlvgd7
zs%y|(UcVuDfrD3Vt=gZrSAT5Gd~n%1+F9P$-%NR})-JilA_pggvHSAh(q5{Q5|?yt
z`cHxG4nINV9{n#)fz3*?$4}HoeEsdQ_Kq97Z}jSx9WF;!9lbVxmf-f7t*p#zX01?p
zH7EI5^ZYr&;)m^xyxnXz9G%)W#bKjPfCslvuKlD-Cv!W`|1#{dRA~O~d797mxX!A1
zNh{q9FWAhBl+7?Y@@3NUCZ1r&wpIUb-cY=`QtRIqFI9QBS@oV#FCH-eyt<WP#o4Ay
zi`Z^%|FDlYk>f4*{Zmzu%^%HYZduyc>STXm@!J140;Bbxd9RL5S$xK9;^c#mr033_
zGC|Y8I4q&_&V?EIO{GtmdgBjEZ`^-#k{$o;1#I%y%C5g(ci+@t=N&cvLkb^#i`3>{
zQdbcQpSsQ|k+mxN>yu-vc&s<A*>v`V!trHm>w1(7mh*B|PINz;yENK7)KTfY@`FyH
zj1M8dBiz|1#;g2$@OF)cQ0nzp=U2IA7wNcGdSrL7RsFrd-7CA{#O^1HG>pThNOE7D
zb!nl##leo1YL>;10)IH(53K(m6jqt)*vuGYX>Q0}S5m4x_y5H&>#R0jTyD<0@O|`^
zVCnG9yPk)vnDQb%;tGG&yw3^lXTr`ssP*&M{Jc!!Vr@&K_Mx4tuAleJI@9mi?)LPq
z_omQi?mjctB~DCjR}$XlR;A!RYx}S6*8GsN@EcpW+W6~k-%$JH-g;h<FH7Q-`%AOT
zqd#TD9p5@#n0$B7Gv7dVkA-hs;woNMuWfTs-^nbSv2B^@hAr)rpFP?fm+|LK6kq?r
z)*iP^rJRkgPYT8avmb8$`@`YjC;l)$W1EdNQoDaGKj#0(XanaietEA#e(jo+O##!&
zjyLBT**}z-X=!|k=~}#wiJhpQvuQ&}?fQ+GkEi$E{lJj8I7xij#hfL~dXHDW46pmP
zQ7b=c&1UANb%z=k{kWiE-SN%sUP9m}u4DRt=9zgrFue`_d->q~Iq|DJ4>OypCH)Dw
zxNS+EQhNA}6`Pt*nQQ-c?0gdUyf;6sE_x;R`Vbql$a#FWrr)|1bhRhPyzrSNCGF`G
zI_+{u{+l%%-F!^yU9wWLt|z?%)3ijGOxD`IUdbmSw6dykm++F;60@f*I*~EsgviT@
zp35i9>b=jzB{X%4vV!WVK9%z>MO-<{Grb=;G*<P-TyebE?Ydw4rku{~k87SObh7xL
zdazFN!!ysIPS1{_D;(aJrax?7c65_+_)3@X#=MsgI)o)3Upl=z@w8^vmp0b^WBQ??
zT;e+QJxTZ5@3CmTto{7_%>0dU%aUb|M!P+`qmk_;6DAzCCaE^Kj$ihG$8PDTe(m2s
zu39}id|6$L?Wr*JjWKSs59kX=KQa@ETDeGQ?u9i%saIv}uAf?e)qdK?l<ceri+j{G
z#8+m{bd$TxQ*&@>TiQ;$)^nfPAAep^CHHIJIn@cyE~ZH~aaZ<cRBSsCoBp)%Unjq2
z=7vvQM@-U_oTgq4xfwS#XifSWtLo$}flGDt|3AN+$nWoWB4o*i%vtNMn^sq?<o_Kc
z!Y!xjzPiR>q5JtI_ZbiR<}n3I)!SSZ*eN_apm6bP{&~Olsg+*%p_E-^+&1UmtCz`F
z{GS+q($vqK7M2;r?NaIC`tf$)QsIwJSR3wjUwqoR>*GA*sGsMJ?WfrkZFn@XMg6DX
z&Dy?+5&ulP=B~K-P<W-<$90}Ep+d(i^W>70zc@c^*^v0dX5K4@;-o;QkMmwE$Xaqr
za>uH}jMA#Cu@6q(_nyg=nA8`0?PO4~ug;_BS6o`AY`y&dm=;Yxu~4mOZl_&Dr{POm
zt{L%e`{w^(S{0(cWYwW&`=qbOjBXuEPH20Zve@Re_fOA;s~<{i7l$2KvZ!5cS?k5}
z(<*UlnU7PCS<236%gW_QNQ(a<qa9YTIrH4K<LRb4RU5OK7Wb^yiZj?*A+&r>$}Nvo
zk7mudwC1|l<$XJ|x((;=T<LZBMlt*GsdE&s|690;D?(-OzKHj{Y&V=vNZ;)E`j2Cd
z@#V{3d8#&@p5DuNwYQhA%W6f{##Gy{)?)8IF?uCg3P1W-%_nHI|7-KoeLp)YShiKH
zWb=B^9~8)1batYz^8PbZQjF@Icm0{?JLhjo+2X2sGi-}rs)_b0ir<P4`R0|k`5WU^
z?yq~RS~xmxX8*EF%khohz`X18>IdmuR@tl_$p^F4BOA`G|Ix&o(&kn1VZTnD!#bOb
zYc*ZjuGHHZtLDF9kew;CcaD&x>$6>N`VZ+xN_JO@2=AREba|Ra;*rQ7#V%2gS5B8a
zA+^=`Qs4LGTp`WTURtgV>d)Ex_nI8qHTlX@&sQIdr*b{?E0XjMn*Mm^6~_l2YrKTF
zF73MJaH^>8-Cb2DiLLkUKIIJC!_uZ=_{ciY`JOy4{{$|!e3iD2@8Y3sx|=eSq!ZeX
z++8*~sQ!g;eRb%!;GYwx_v~SNYqIS8oUWDWOevFiH`P!4+I&pqnqc`Y@roV$!ri?b
zK0k6)k#$+Lb(?+c{p{=*IX)Ztoho1XthP^DGS}Dh_S_eao!|caJg~TS1B-&3e#un(
z6S<qDFE^i6{(qWH>s;rJzvt5*Y&VK=-4|{V_2_kL|Cz)G4`$Ce<rS$O@!HurtLwtA
zhbt2wFT8F)p;YL<>Xy_6>zY^t&d!mzG;zsnU)xK~{$IS-7AKi_<R9^$>AR^a+5X6-
zGba`usqD0Rp>nr)5qnTlfT?lPBi~KZ&NFqIS)VOQ^gVxhclg`l5cZufS#_7C+BQF1
z+qXJy&ExWI;vu4|CT_U5NZnn{YV!F2DODp;`CdLD_aZ5iCofvAJxYK0J*7&`R(jt0
zO!kN3SKftssx)U#eP@x`>uHrzGT|x%_nJkeL0oF7PbXc{Ub!SVh&483E9>IrTLM;g
zZSyvt^W^d5kR#?N**QvA#>i$oJCYO{67ThF$*PK#T(b>Jwdc=CFn|ADWV5`Em#|1<
zV}Ii=!7o0xMf!^l_-pj7GwM6VG&MwLj^`br*Nlpd9j{ORd-1VL|3;rqk?{B1C6{NF
z#jjBC^ztYPa}v7y&9Eq;Cu(QM4J)>ZX)cfC!`UB$PZD#t+Tc~rP$?Atszau#&A?vT
z-hR(Iwq{}1ruTZ=7VppgZ8%Hw;Zw_+HC1!voAc#^Yt60PSLaICJn@z<?u_4m_td|#
zyVK=9KMp!~z5Vyi9~r-o*Q(C^QMqr<wrhpIuiQHuJ$s4r<K=copC;e2`?x)S;nO=C
zmGd@#Gt8WOZ+g1)_sXL?_RC!USA5TY+TY~8d+uzTe|g_C#T$M7(=(UXCS~4VRI-;b
z|69(Rz?-wCGk^GV`g`%7{HxpleR%TZXRD2b*@BpcFD<)FzMq_@X1{BD?GB?pMg4t;
z%-XjHeSd4oSYBiI?ff}TKbD8rvs0(@9eP^4r>r2;;FJ2C`ER8~Qm1RmzcqS#x+g^F
zi7v|(^Xm@Q*FIfmaA^;Yw!3Z2bJqE1k;A$Ff`Prx*DIV{t^btoJ;auD@7wF(@82wQ
z7!oU^eXJj*Cro*Mbg#3L>6?m8Q}_Pfqq~%Q((Cl><9=~(zB%2BU-9wN&V9Qb79W28
z^$g#qR5sHb<teh^Q)MTo-ku&m|K$5lvHMN?wq-BZ4%^<Z7|XEGX8qr{QQm*03X&^~
zt9{kqifm~;eYT(ZlUw!uTh9ygkC)tH{dQ@oW7WL(fBqhqHekE4(!%BEdV}`sxsJ!L
z^BnWM`Sqi$)CulzgNJ@g67rrs-7-1XJp0bC`Ro41R9Qt%Yt1=QC3$E2!O#1QB-x)G
zY<GO;%3SL{!#bozI4A7SE}@PcU&YFMO#dX;&s=Kv+xp(Zo;}4@mCs8)wO&4Hy?k=H
zpZfbbe)|f*LZ@E-aS#*WQCTW(YybOc@vfS_AAi_8jQ>}#OFPaEzmt4-yTUg6Qtor2
zZ=8gRUDs<rQuNng2%YKO^G{Xm{9EZc)BCtDZ20{>Ktf;t$cG<?EbVW<-~UJc-*5I0
z^&6Y#-m!5!>+mtZ<9>ZT=l)-Z)IQpO`kN`V_r2xA`^%gkw%<8De|vL&;6c|b%!hXc
z<mJEraehk&d$`{NTdi*U)W%5>D|%0E+_ZWAzSYzJIC-#z^(9?@<eJwp{phj3wV%rW
zH}q8{zX;>KTKwPMZud8x*?Xk-?Y$WDhu`V_ZSG$>b$^Q0CG<aC-uQaepN`wlw-_IK
zIK`H;(eiCm-QF+H3U|kZywzh!*?n%Ew03#qUH$ooL(BThPgVwsz8BY6zOkbAp6KZ^
zEg`nQJ2iIoe`EOf`rg|rtL?dYmhXjpZnf&K7T3L{IWuduUQ+$8M{A$&-l6@}j8W_T
z<D;>h+m-e9Yx&-LS$TQR$*K)E%O3VO{ot}Z{<L^s4d3;b@0;pw7-rsLU8PtupXa5k
zcY$HRoR0E|+r!=!%4;A0n|#aKJN%~c+v$fM>wR1v`@_35zGBCTH`}zkKj}&H`F^c*
ztoi+ur{C88y@={`+1NR|wiKS3&iz?-?cTTvu?PKA&llaD9KRrMhe7VL<y#NeYX2>I
zQ)c=6?~K1nUt&_&?k6sb-STGozSihz3-!6W9xr-myKevd6?rv+`n-Ba{_l99(^vF%
zL7dc<zn@(Lqx5&|d-rSRtZ)6lTa$&JmzLZPtduL)cvr6Bw;|T>>&Cw|QW9ckcGZ0O
z`*cfi<Bsipzg?W}#`B$j^Fa1v=9&%r<-X56Ry(uo)y!khKcD&Gy8A*yN#ch~d<UnU
zua)NgcJ=C9yK8qDBK2ozy<2fq&uZJmO|PGr=}X+wcbif1x5n_+5$*gU=PAFR?hjb8
zCs<;?z~{RM0_9F#XId-na`{_v*6w#kGQV#mF8`x$C=+v`?)<mxwCTshFL%XGsr@}K
zdhdj*Ws0lcCf&}iv)r=5{6m4`O~n|ycz@fNfT9(@|9n2TG15<5X1;#8()82MtA5_m
zlbymXs##{a=XByV?x`Oi7u77Dbm^yMoZa$l_TL<N)A{swTzGrvS`bHUh4XFucPZh$
z>H=>zM9*LO>3>7O7yBCr&vpK}YVfZsM@#vi{D!8pSAOx!Jj!IN+SfHZ_>}!d#`O00
zK8G_8TE|zg2DtrUH$QOn#@_^1r6cFHf0!RwQ0C+GYria;`t>c`lMk!Dy36wV{D)KW
z1)cv6@QJik*OlG3*A^=;6L|Hs_4V^@wP|Z!u{`=~+~oY+{b%F+_s9M|-LqY<`t_}E
z{oD4>IhFckL;bnw_t(azz5D%w!!5!-dRNUo>m`qWeqznOz&7(&&V0T-=eKXqmG|?^
zJlCK2`@|K-)30T8jJD1<`hE7;^qJe=KA)a1FS*mo{(7wvM@YndGtrWJia9@JzTX$!
zW5r+lKc?ol#iFI(eOcMwZ(J6&E%7Fg>Wm$CMf9tj<R9~^AKCJ@Snu}nqMtj=(>`4I
z6Lb0B#E%ui61l%^o}IPw{dK^G&Fl82_1lv@j;*-%_WGiV)uJlb&)4P5kMFq`VX)@t
zUt^^k)6S>npMS-1TD<Faq@%j-hD8BWUT+d!?-F#QMXh?v-_PYM*7l1X`egV<{_!ut
zg_CceulyhJt^ez%qT6-<?p%MnI9Hx^QDv<0AKUYvb+mOpFAaUVH_KS>-j&@NR)-#$
zv))>D^5)$aht}QU*j=U_6SGw1SsvGlr;pbipR>=a@YR%Q^UD{z@3BaazmivGc+lbP
z#$8!&<{x&Q`{1~%)UnWrUALE(dWRR*Wz6_;^ZR)t&vO^o?VD>J|KRSkN5@@vtzT|3
z-_P~zs)BtpQvU2HDbH$r^yInN-`ndxEXw|zbkAz*(@L}bFRq18eSg&ObJ3f!O6#2D
z_E_WTmsqdq&+FbD^<OCP;kNm=zV_&gwZ^WT;aB?d#^t+i0-vt^DR~&bM)dR^eW_F3
zO-Ap{E$09FS@>&r?ZZ_sZ#wRb@p0O6f9<KY-LF>0eYFbu^QUr-7SGB94HIpS{Ira<
zd(Ll})~kPNDP!%$f8sm8g3qD_HyUmq{2#!6SI5t*_shZ)cmMs7|M+|Up`*7peYMfa
zzn|!wX7gf+Y~=r>dQVNM54msdt6lk$Qq|J3KKQ@Y_hV;PI4raMp3g01ypVZj$EE#l
z6J3_2uGk*==YjqIsKf8%4xae+|ImYa>3_LDwm-bzMu>4g`^IMbLnjw5uhx0mysqo5
z|LkY~!+H7tR?EuA|Nnja!GD)#3n`Vq6Tj+IUAws8CBO9gS+V;o&#ssGzfvRVPW8+G
z8;cFRfBk%sdGw2y<^JEoX3zfjvz5usSLD9B;j@hV8iBT^O{c%t)&2jpAiP#)HowHj
zy0pD7Pjm0{|5G1pzvZd3#Wa3feXDqWX8G0+9}lzs&HZ41<$=w$@Qyz-0xtY|;i=H~
zZgNS<mw^56U&_z$ZoksC^UMD<9}a_GKXYZ)p0lsIDqVZj`l9yJV+Q|i3L}@Cf5FbR
z@5R)jzmG)=%|G0ASd!T@^;f{&uG8xBQw*f$XbTslZJTFc{q>=ePDkdQLveTJa@a7I
zE_zg3em7`+#GQ-v@AF>#nf^OqN7>BseAgZM-}UA{@5p`q|GfI%6Vs3VKW1#hUORX3
zGf~E;R(+4;C01|L4--8crYbU9`<SeID*yHGS{#?YFI(;{e<1K!jMu)p&$Hscs(vZw
zUe^DrnRW5^9p@fz&Xk?Jise*YM(G2;3D>S!S6?;ElU;np(Bo~hiis3^9B0SfzkB}{
zr+-m5$xtxp2wuR<=-pg*ugOedXYbmq4^eV`LN|^x`sKE+-6phso6z>~^GpsAFYCIF
zdj(WGHQd}(ZJhu9=-cC4YMF}OEam!h{>GJezt*YQavuEgDTVtW>$CYC5@nk@^^NWK
zAG!DC7FWTHKQGr5+-;PIXN%i({Plt6NjKIXsutpyZ+iMy^okeVwWb@?o=q;4t=RZ~
z{)J2L6<_Tta&YhDC^C4a{+lao#qS4;Kkjs_%M19yFL&*~3p2B_#Vh9hJS8jDMI0W!
z-S>Uw)W`$23o_h(B|QE0Zew3B8~?!qmT4M_l|}-Uhc`-}sM$Y#ZM@ZlA9c2ZdaaTN
z&hBoP|H@gjsPNi+n-?DaN>3HO*Im2&HaOs;xWM0cTh7c-5O{w<QCh^lW<{55V&m6q
z4aY+{UQNiaUth*@tACT0)*+Jv#~l7e%5}CD^0|nnh8#bAXT^ymje_M|Ywk1dIuO{n
zA>jM_)CX)4mkb&XS_mAw*8RId`$fgd;|8gL)8xAQt=_V)wsSx7`|~=6^+{}Z`Oi4?
zMRv)Uwi>oKHQ6exXlyO_^q1U_q_9Hq!y}PT_H&~1uO_c`@UPBVdd6!(g6!`N`plxa
zIvEia%#r5>j+kutqO&5ifBPG~Ysnc`a#oADDt%&?(b+gVqGD3h1m5%CnE(C!;{E)`
zxgb+^rsST|9FFw8o2NHkXyQ^-c$DDLr#QFyhKIWzyOL-~m*UzkjXi8pS9e-X6=%QR
zqu_99&G~y=t2~*QlGDm(I4AB@x}@WhrI>ig@yE)kUmpbs?|v@CbkfTv%3iGKr9#AC
z8|Fid8+rTmWuA0ZMmek&@_O>~Ag7G&4K~J>_D!xLN1QMI{O@psb)BJ*WrM;379X>)
zx3AuAC}NLw_*>xa)tVe4T(;ofJR$7`Zpzyp7K;m7Tu8t8`u&$HNeBLSeCGPHTwMKM
zkxheA)3dKT>w7N>A7JTeW(dD_{`b=bOzU2K;<LI?&alZi;bHM#&eg3meBOJjYb9Lj
zUN+NGkoEZc1DrWCCjNW#^hm4EWqm^}MSu42o=HKSGnTCX^VFf`j=+X0ouAJfZB&aE
z3J7}bnZ6>~gZcE`{4?(_1!mYMK02^3?7aZ9%n5GclkrWCs!S~E?;IMIeLA8){X(<%
z8;7QpMHWR%qnAj$xMw%<+afd0rSs1>O5Y7S`qz4b{QAG#dzeo5NQoV?eVlz)^=iko
zxQ+jR?VK<#>u5`e_3R55BafTPuW0>~WWHDOS1HTthv!*1Qch|KRn+nSyt=EPQ{mD-
zGs_7rkE&|V=^LrcXTRvo|2x5BO3J0)g_o3rV$~NM3O{bB6La^WM6;Fh%?(B@f~<R_
zCw%>P^>Lnk$NFDC?iy}WR>_@TTsilElWe>2feKsRG<7|BhcmTHm^znbPl%KZ7F~XG
zLI0wn%Z#gES1jHBF1dE!r<)7he<UUTO_uNZYGvK{zWpLU)9;(!tx_Ce^Cew!-X*#n
zbo6?-!qX>8NH8&T&m=+i>#9pS9yT;6BuH;QR-?Do{Rz{Ho~O4DnAVv2NNsw&tR<95
zd9#SpyF@p`*eyS7c2C*zqOWN8o?Hbr&QJ0y5>j&?@>Msgp8P$pY@Xs{nZQ4`(&DLW
zj(@Xk_V1d;I?rw6Er|!LR~VR%MLj#>C#fUoq#pV8M{|tpu?ofyF?@SZ94g(>uXDbK
zJJvvE73cR?F-?7ka+g|ktu1WO-1mCp4kbU^uo(qcTLXm)S24`0JG^T3X}vzF!gz7_
z-mYnK-gjhoopkt;nSQP155v2cMw;c@gim$7W)I|YZM6@O-TP20W!dsKMsqh_x&Hh+
z@A2mv9ybfB{@!ExQ2jTqbkcF5^&X60Ip^$sAz={Sn2`QtZO8AQ&c~+;yxHJzb=9AQ
z2px{A?emr9G8VQub@F{ayDF{IV6XGhDT12E1(E_>xq@nq?KM0X28%9WQ(TwQyECNI
z`$<GkMY|_MLxmk9`_dbe1rjE@q^wzXLw}l}xJCDJ!B*z)43ft24(_Ehtu?kWwf<f5
zO5qu!)pC!NJ%2Q2x$AH}6E*fwndSEEforit`(|g+2aXrG4o<HTtW(lECBu9|V{@MJ
z1i^?CUS=X|xF5)`{>QSp?^vU8gw7Uz&NVKE{2nX)rQO;0v=p^EygcA5pt_=0dCK?v
z#`$wr_ugUT;oRA<wS{|fK*!Zb`!oeEpU7s`H1}k4duFsE$kh7$s{1|rMSAZ$E>h!S
zZ|PKNx~ab>YuzlZi>vRb3%7WhJ(#jtOQWpAvDNmWPRX^7xr<VmgJeYHkEuCqVZ8Th
zx6Z)>c5N{{cdo8Ku2HEW>A9r$j_HN#>UJ-482$LSFim1H;C|k7sAH0c0{d56=6D{H
zV^w$9kBZ6ErfrOo(bQ_}kNDcWsBvO=4$p6Sb-q2z=0rPK$+zlAFJX3ykY{_ryCL$x
zrOBEH^dC!=>`0%P!{8NWT5*6`BLBhex@!tQ=d=X#FkkxpMw9vV#B*P?8Rksayu|Rj
zluwJ@i+$HLhsr&Z4pex!U3<u{alT-!qNIe@=`Y-jbJ)|AH*Zn9(NM?ZP_?7fePK$&
z)h{Pv5^U3)ZA%0N4S3f)-<u)2glSpliZhGc_$H?6Sce;UMLMm#yS=Z{m7$`u@>)Vi
z*A%V}(F0LJUmprDWxB9;!5!u2Z`~@(Jb0#B`ZCx}HEKLyVLbJSLY3b)?nj1>51W%X
zQkbUkh0iKwj$obY^}>VqW%zHFUmIJSgBVvxH61-EW2hs=-VmXtaEOoP=K_}l#jY>%
zAN$*Cau_^(E<HDhy-3?KNY0?>b)w$i2Vss0DWW&)LN$3J_GiCf^_Z`6;^b6Eqld94
zLLvg+AN#P%V}|yVZw<e#MC7lWl~`*fz<waogh{SjKK+{5)5LFgcn{Pha+DpZnZMZi
zQic+{q*CL{LyU%xn3zHuzDMk|aMHTK`tIR<m)dhCr;gN|D|)DZ!K3qkz{v?BkK@m8
zy7>QvB*Vce-}nVj`^xK8FzyC5Q}P+(c53r!2SnKytHc?Is<x|ot}(k<q$eOEQ!Wzn
ziaA_<3lno=^r0DZpEj&`U~#r%PUMdAxs?mq7;oP{p0j+(gFBfW$0RzI+}L<w-?w|r
zZD(g3<$hSTgJFXBk^`L$9BJ?D6pS_g=sRaUnD<45x%P|G5($H<>kl6kHL$*U#Bs;R
zYRhC3?Vr7lO$AEL%qz_^^xu7AJ}V>1#PaBul1lx7#fq2R(jDvtBc{vWs9dgkgw3ES
z@l3PVnsrhQ9uqZP1eC0<cHTJG$6-HRnBDl>&dAsYM{LgusK{9ISnU>ExR>+H#+fU$
zPPHugH{YXsZtx_ph_zq6rzkC4c=_x)X$J0ltQ`LXroNb0^ssq_Wuc2vzN~T`-$hN9
zNBY7OGLJ>c))XI0s-3{99({SHD#z{<8~hIx<_PUL$MJ}3rh-Dmrf$KIveGm2HlDlT
zSt7o0myF<yj=3qEO)RD<56b^AYuSAGm-39uP3P}fJ@0~Tv0{umt#L8y!Uda4Vojuc
zqjxL`xGLS|aY1?RC6+gdf|pqG{OzyJD$x<pIOfa0pp>C=UBhaIg)MJa?l7D3=hnjC
zax-Ra|9^vjU+^5ezvuU6H(uYIypWAU>gm4U2V@)iI69de?r6G(7kTZ!w!>$|>m59u
zLPxl!F8#D(t;3;=oWf-~7ZVpL*Qj)b{hM&=kAdu(j-w0BUmSU?ap%N}rjsJ~lr}rn
zI|VA*6t!D7vE(^5<#7q`62HVh*T?LlaI3wOenp{lHE$Y&1lJxR;SdFL(<O2WD_uH|
zW@@~wkkokfIYcX!OH*RSpI5S>i+vuRST?!yq{<2@fsh4@M1QOlj`C8heRJOU-5GI#
zE6*RAbBX#*n`G3w+RUVtYq^8oU3vA3D+M-)d?`3l)%<vp&qI-8OWl{0Jyi%)NHJql
zWtw*IB~xr_>RhIMO<8vog9PH8_FqiD@Ug>(*I}}+mV!s3*IKuEn>CB2T>d3yPk4I2
zWur$Q|Dwxo(aa?#?5Uhh+jXuVs9hr`SJ`cSpLtIxYhR~3zb4m;)1TYBIwyZo>^6OS
zP2a80{bbycSRWDoT;476qDd{>wT?Xp{v22@P;2TZ^rL&_*G+km%S0Nt&$Qk#KX^eZ
zqm)=G&%cH^mQT&WpIARlS~zhPw}kYTivo|luW6hM>J*v6eVluW<cfK%2Ogw6nlZEO
zpTG&XGm_1Z->%IR?l@w2?vI(r^t+ttPbbJ4bsVk`R8+}O=l{eYFXG;D#N%4w+7r19
z?T!6KOkEN|%EE_!&suT*5R2l^BBxsJ)?-#NCUR><G!CS`FbH%}@LVYIY?s-IqnB5f
z)qhvWY&y>vp}Q+x;k?mCpCuC-<TETCo;BzkZj6#WbL3B@S0@uweq4L{V@`K2_9lrZ
ztU_*%-;b>fJt^WTeXBKG-EHw>sSlC<mJM6>iv&!K?Oys=Q^q1R%7gK@W4x^4RIgV*
z55B3;U*@(Tm1l+yizv?t8*8q)=N2)Da4lhAms*@TIpWO9B@RjN`K#{DwcBf;@?eSZ
zdB*g>1uaDnH+U^f@M-_iZN`?M|KY`!(2HDboetmoTOSn^{ng_t4GeDkn#LgE!c-7<
zquwaGHnUU8^wCbYuXZM`Zx{G5$Zbz%Q&|5mZHdo=DYt%kpKLMsJ=00&Ky+>3frQw&
z1-DsO%s9y{IH@B1_$e`NUdED(H|)-^e`=U<Ec}l|Tl>3buEM#`3m39!%3s-8UsqB3
zn`uqo-Q-y5`;96A0xd@sE_jJ6upe;d=(>9#*&txQ$LuoyXV;f1CM;M}*kELs!RmcO
zoQGGBh4JbAmNS0L2N<<e1x(-1stQ`EXZGm5+eP2(H}1j*f)qa2O|Ux1eRrbFnR<mI
z)$;3$kBD;R`8+HRI%wjy;GqMv(oKb{NwRl#=Q2bsIrro6zIg|KJob1nTRg>wJ4(@q
zqc_rp@5A55ABLB%UCfLq+1h$jAwZO=kxx3>Dy<`8;T(>nwD$QLO5YTBiyb-mG1>m0
zdm-cB5TV?%4?7QQG5f7G{=;}E@l|u$GtGz7>dNL=DCFdvDPX$IEAg*kYMb9mQy&Kr
zmjpBOO@E@&k4@k`VfbBerOXM&4O)vG8)hCo_?}(MRBMh}!&zemcAd!Q-#QDeU7nle
zBtMndaC`2#UlUgF&N=v=zd=(%P-4Rag_Y(S5}Un$&wa>$#i-PC0mr1Q=RcT#%xAdP
z(D}iN^IeVo91ZKGYgi&@C9LWX4|@Jj(W2zq%Fq83`X|mbY2ehKJC*0qd{OmTner=Y
zAN=FpQ7yGu`wZue*Eb~pEATCD`pQ4$r(JuF3!AM7Yr&OLvEJX_x-RK|O1;*;*jBo;
zX+iab&(EJV?vcCk#Kl_cNK$7`!vE6bIe+SZBse*9Fx<&Xuk+woEGsWl`^AH&NJP%J
zgo~k3>xSQMzIWTZRqbzVXtoguHs2n~#n@P;c<d2N&|SOW2g~)gi#`aJ5?I92$Z&7P
z>(qt?T!o?nAu6&)i&prt9^Am_nfmN7>$G1je3FNl)jgamen{l3<<3>nit5~^+}g;n
z(<(qpVtQB@TO4;_--m@PcLSmhur2*PQ$fl>ierOc;IrdLxGJSIH||zrow3>_kD*}t
zOBP)b#^43<40*CKsk790|Eno%bh{Rl(Q~Zz?(e^Q_h0#WDSKb4;hcKcN6OERvn^w`
zYRTsi`E_ct&kY@)V4n$`6J#nnGxG($HP1Qe!MLe+8jGPqRAYvSq)a2r%Ofm0H&{10
zSI%=t&?$W5a8P&dhf_)xj&E#EaOW>$<So0KvAj!ahLns#e5PcB)SOkW>yBwU?OX7m
zMRFSFW=^T8dk?ShlXhuH;0k>Bpu^&e(Ss$;;(<v^3m8=nE(s3vYvKz1*nZ=y(-Rgo
zS&xm%F&36z7w*2vy4UQ{TBo(b2e-c0y_h7?)bRP*;aQyrCx6IH*mjd`BGVoBfOoY$
z=k{&*D9P!~d*b56g3~RspEpPpFtyf%EAKH~_4sADm;&pZJuIN}7it;2%l=7U3o$a>
z^SIZ9QCy@V=-Ik!Y?q$>J=~+r%isLXdSZ&WX>XA$Lx1!_6D7_^Gme-^x^>!osBTO-
z=&U(`x#`0c`A3-#_!&f1|2W?{W~-dd5jZ)BRnt>kQ})Kj4t3Tl9}$TIDGr;8796t+
z{SyC`Nygfv$#-Vm$;?#;Di$0PVr6NNQJlwM9JG~tk3@z6gG9<&sS3e+4+V4^PDV_d
z8P2;;@j$0Tvp&0*Oy>PY5zSx@RVQ|aDG>+5_RCm&{JiF6J<Gi343`u(b%^LO%`Kf!
zWzQjc=|4vt?*@5!n+JALD$ExQFP&bTJze-n_r6~Tb_K;mrSCX#wN!5H{)U6Q112g)
zvt$`Fd9X(0a)^bzES)E|qIZo>^4z5J8qYH?B;`m+dfjNBVAsFIw(kRf<der6jvja|
za?c_~nbR_`U2tXfi|nR1H!Sz|d)QCERNuXTd2iyf=vbS(Y;k215|>@_t#|jB^P!vd
zMx)o^bVVmuUe<H$3k7(LX4-3AFt}bPs$C_#E9;BU$rTILgY7m6xqtgy#OE>Jq=tKK
zgv$kHp<T(hcqd$KZ#f)hY0eq>@7KI(8t;`4ebWB&@xdC24f`}=ITqY{AR}cR`bz9o
zL*edk&l*e^4Y_{yzMR%|I&cH~+-#oYGd#U!J-=BmEy~${Wuo(HKJ)Dz%qnqxjnA)%
zvzzVT>^L!b(lb{cZV@r=@RXp4^7JWuoEz4DnYq0A{;6GZC*6<tU0f_uJwfwDGv_>W
z*N>_Nu3b|KglbG5u4=x%^PsI<N5eFQV3&=HA}=s2TR*#XvbxRe&4N!l?1z%djbF){
z?BY^?YriO0;YLkfj2?^WkAF$|5A`QW*JQkW;dt=$)6*Wy^cH2QR!+95S(ACJ`rtt|
zW#42*j<0t1fz=P6g_X2~s@-slzx#vP;dSiSiM!(rE?(gJx%|@ErHj9Wm*)O{?snDS
z-!cJ@b8K>Af4BTf$$G|AEH3}zQ~c}Wa{V8sIa|k9&V66<nCV|e`vu46=CccnwI2s7
zIqqS-clVa3UaalzCye<%llg`DHXnT8pf19?{D|@Mvcr)j-xg&r<cV0lJ1B5=dtg;P
zdwy|vr@Td9URUd?M+bVB1qWSv^E+3%zW06H>*=l<+~%9R$}X(cYziyg*m&+IpCHRL
z0fQA5Tew~}r!zfvxe|D=ZQ>5L9ZzQjs0+;6Ai1}&<NS(~X78$5WP5vd8E-EvUpVRF
zG3MWli!JZo+E{l_>Vb*$JnaKnCR3T#Uu<BLQCcQgv39XEi;0Z-v@Htnt>)EV$w^37
zx%>X@#%Inf#+l4a)e47&4;*!=+7Mi|l6jp{ZdBkoze1Pt^ZVn(JXb2LVZ6%NYR#SM
zlDX(f(M?0nDgRw6CE7XOPk6~<!GDM?mD6iU&l!bRue*3%3?67ak(~bGyJx!NqWwP4
zTc)c9ojr8_m(GOM^IWYK)p+<+CNVx`+{DuVm|?zz=04Uxa!+@0xoBLUeI|B6yl`w3
zvkTXyuMTHUS^qFfaGLd)<Nc4lH4_dTZ20E5fBSSs!9(&Mztq3v9cliMwCPap@?94@
zxR`G)cdF6XSLeC?b%L7@hyM${aF@2f$}WO2|CR*s9+Yob$k)ZpEjHoN2NxlZON#Di
z9H$pIh$^4i>2cEkvZRb2!*8B9>s{<hQrK55mS4_z`=Hy&O;ZbBu&teG7vO6z(J5Z@
zCS}@2F?O5KHT^E@tk*Gqerk13Ft{+_P=`s5RPD?Tz1Tld$HfGcd^k>d#w-8Lk=da?
zA>F3p(TC;smmD%KSY&tily!M5pOoAXm4ClPO<>-y6CW}g%|9Ji6kn@R-#UHv1>U{W
zGun$5>~s@$k+a=%r{eSjq0VRzrB24pHMuhkBx85K+_qc7tmv@4O5%O98GrZBV{@>a
zlNhA<?Tv@E)zQfwOJ_~iU9;+<)3K!&KOOXa<#M3WVPTr4fd2z=r+V?;vVUJc2^@P_
zd7<`gy5ObG7F&U~lVueb9Gm0>x*vVBlsu|;T;z>jo`_71?I)%5ADJADMp0)^>XsdL
zZrmaCa>L^dJugz`@&&bI?{as2Wz2U*dDR;Y7Da<67dm!bNEh+y{lcIj5T$V<-NL)I
zF+z5mfJNkDp2-F~E+1r)JA9x;{!3WP)m1E6*M)0^t>%cj2ULAO$uOI}=elKu#LkRj
z);6ZXcW2J9dwMR~a{1ua;IhagzkR3aUVoS;)pD0>%H&S5UkyJv-zfJPcb19%y}CEu
zZpG@?M?Eu&THi}Kt+UhTsdx7ki4%)TpYVgJT%x=q<hr5Aoo<;%x%ufq4|~^dy(HrD
zy!`ktfu~}8zUnu=_UCbUhTIFj?;k3YQkJ8~t7yd@;pE}8a)Bqyjtj@nUeWuO_*C>}
zu3{(KniGtHkJNa-E9BI<@O=-r`M$-uP~Pxjd`ix%pU1WP?)Kh^wX)~55^B>qvM;kj
zK&s}D(1)OSCxLSQVvj{Lzf8SU8FAtvpVEzhA~i>&_tz#IDY`ExdPa!fqPSl1lXN;?
zdwN6C{q{4=XPvbj_IuxOy1;6h-+8R%o$OWjH8anoCB5*@yHok1B%pCUd(Kq5IJYUB
zd-6K2dNr{7h4Z<~=9||hm@HU0sg$eZ<?`c)zW>;ArI4jVr}0d|iyJbFe3XpiWM8+6
zUti6}q^h?dZ`I;?)-CUt`vN-)CD+R>VdC|EWA5@REW%Bwu~)#=_P}bvRSjVhJRdka
zbMMT%AXu@8-+#JChxCIh6WZ#tm3hkluPQF=?biOX{GzMfn?Kn#-=p?_`Mkcc)99>i
z=d*X=;%-8f7kv~}U%tBG*wD88UEofkcBW4bM{~6ny<8{0;=GB%D!*&Hj~<NK-K3zm
zVAY8UohRNdnOVv5m@BYUjVbiyj@WZQKD&HxV=m&Hx|m%>IL4G^iexy)XEwKg3qm$6
z+x+7&^RE8~*`77;{<lw<nd`V`>}QU@imQLx-(q-JVl2+)@#;(+<F2Y_yBSh#?HhNT
z^Zz#SoL<6u;V6!Wf3)Y-FKzVPH&_418QBb;FWbGfXNKKc^27eb^WwfQ0oT_&xw||j
zJfZ9|d$9ALiO-Al#P%>LF)k2T$XuiI>7Re{)`=yxmJ==dg6_4y{I2$4eon%~jQNSD
z)TXtSYV+FkCVdqCYi@sQf9dOkx(7=+r!&e3PYwuKVO5gfG<R!$N%sK@!zuh~SH3QP
z%XhIw$yh|@N8#kG#hRf!vl@F>?Kk?qpZA`L0ox;$)@4t&$ZULC*l6|t<0hLAJbbb;
zD_62@NOj6m`W(bIPqY47*7;EJ=!1MZ#!f9fGRFB==bJo~H#ac))toT#>iGk20vOLM
zX4~I?VZT%9N0n-Qx1Q!x5|-5~9&vw};W>{#j6JYAQZ?)tk5-_O_gkqOzduY~`9D58
zq10oO{ky{oV!eC(>Mun7Gy8G-L4Nc95awUEOfuA_ta;nrliJ(GwEResna<A|$vaPH
zeto-tkJ<U#G5@MRulRqm@`6r=SGs@*lbl4~$@B}xuV#KZQ#GyMxLI<4LO+N4;b2*@
z(ugaGf$gobNwYSrd3<`&ax2|eTUK3O<+u9$^!w{J&RllGq;;95iSNQpU*E;%z4uJ2
zE~{O9KXsx_&8~Tl=U1d(GJYNT%w(p}=@)Ky<}dT}ojc3Qv~<;iJ@#U{e~Nj#l(On}
zPxdL^_~%9ZOvM(4z+<v(>3bF>ZJzrqKRrvd<@Up)O71oL;(ydV{N4IK{M^<}vrpyi
z+H_OGEU{$T!REUmWx+RnQ_X$7^WH4qD6e`x>&>!_vzIk^6x-h9;8Xac^wLsF%+_*>
zW<<+95x%D@gGyLy7xu--?zEbu&G(_Qo~=~GN0!A`bH(BXeN9$U8|OG0bG<Uy(myj$
z?ZVV-mfBA%`4(ICyxA~+t7h%Bdg+Y+TiJANE?;^kEt$Zz(M|o=)H#PX^vgf`#$*!6
zp?LZ?cTJ~U_u9X6{wdu&Iz#i^>6xNmZXI50KA|gL+V%gwi~4U37Jd=WVPa0IH=B~p
z{ck7d|6*RPyOXt&o8vxBWcufM#J=-~y5N6<^oaTb_J+EBH#l1C+g9~RO+8ju_ppl1
zcJVyV^J~&CPUd6%?|0TkOgX9|E+u4-X6>8#l@0d!501Mi)E#83zmszRFGF4C12K#A
zvNNT3F0u)6E7mE$%Mj*TYVO@W?}z($sdL5JLZt`&65b0mG!-e?CcWJx)3)aj`?EZ?
zAKAYYGv+2H*|Ej%Js7!cg<}P$-I~(bo$*gP>OY7)H2HP(=+8~!m!B+LzxR@=nSdkr
zZw>wFDlh*r7kyg4R)F*P97{2Ctr$_a&~l5P{`X~<JpA=t=H2D#9(5ZXp4u}x#|NLU
zf1LGiCfiJvu!q~Hm<Xx%x_B!0p3!Jk**4*b*rEwaLeV$;6YX2?ij)Q4@J%$IdN)Kk
zFx}qtj=-`0XZ{E8t1kN;z_8;<>HmH1SzKw(lQmq=Hl9DG_VAR-+_T~;vQG+X9q;UW
zVlkPabYH~VUrkTAJ*(6tv-cH9J2H9f`?ki^BP=>D;QY^9o0n-7=2bfWo_+gi>y}zI
z%O6Sm)BC@^Zma)N`g0rO0#3nI$Bkoud|k*8{^C>nBGw1Kft&1|*6(n6z%4hWY+Bhg
zmUA~V{-kF}+5X&LEO%mV^V7E5Y5y)x__5x6!-fs*7b^eg2UsmxdB=pKJ4NunX2iv-
zmwc-|O#_}!6?m<B&U=qhNo<9i;l*XEvmTw8a_GR3nX`F6H(N~zNEA|d5bB^AAar!$
z`Q!XQ(zK2|aaH~Kz~9na=v%VD7suuN7oR#m`6gf~c8xpe@`Fe>cCPZXZ!S)8IXmUr
zB$oOM-yOHH$J)=3>xmb!b>FeyYR#(+s}8T?%MM@<wGH##2x7nX`F`|3Oz90DS(C3T
zbjq*%kiUeAoFCVFfYACa`y7?ZC$My<U)$WnH>K`dLjRR-@|QqJ+h2d)s|#B?JM6iA
zb{$=AWm5X$)rG4IS36%<bBf{SF?<*-yUR+3vnW=UtyE%5r{2`{+6)3SOFo9QJAy>q
zj@{kZ8EEVF?T2cN_`MH@e(PHPSGs#z?CT_5uVosMi&EMrNqU88NG?oit~$x4Dm+Ce
zV3Y01FDj=e^Z0MF-kVZB>E=pML9lAiu^Yar=38I6e!G05Ungjj`$-wq>?u8gn_Ty%
zbWhUsnx-*X@l;BOk>nzuwiOd)<l^rCaayJuxyZ*&W3u3>lolh&g+6X8rk~_W-Q;xf
zM1qmgj2<4f+&w36Bp8`V+}--Kbs5xO;qB}CY9iE{mzDjikC3|(7~t+GtJyZ?7+dlk
z_5ByFEVAl(Ww7e-3Q%dW$ww*HX5WRV#UQbbhoyDAziOQIS6#aDzx^lYJC9w?-(SxX
zl=shn>YI8mX6dQIO-%LY|EJq7zxU4GRR7*z`L2IQn7<!OUpy^;|MqX(*;#iO9!<Z=
z|IttA(5?bc7pMIT_9)D%4(RXO61;!wVqYtnM~7~D?mpl5<16nMp_l(G=RCM0dtK4R
zv2aSJ**(P<=5KPeK02QIV0~KsqNkFT=d`!19c%tnS~RnKkbLrOwfMaEFXgw+iqCs+
zYAb`G{j-IOjFv8moOb`Jtk;?=i~>8a#B7^p5hN%2Hj^nZH}Itk|EuOnbM(cw=N&S*
zJkKz5@BD}A0hSDDDVkmUzf?BtF0A5kp60=yXZFG{j>Am*f=<@n1GzWi&YZb<Ac=2b
zY+X+}d&*Po?QYR0UwX3d;=3~Q?v_{lfg9TN)Hh~HzkfWp{pR-Qw=>Qx((pAqq^27?
zYyRVJXC_|g4Q=BtU#DXfEcH#k=hM;m^OV?fkDmCTo0d7VCgj`X`z;qIE}vXI?Zo20
zJJf@p2z^tXSKW4|?DjHo#z<fF1g3LGa*w%)*IkoQEpWPVF-3tx|F7vYwS_B<`}Qgo
zyA<X)329`!4>$E)ZY7oKbNi;m{f8>AY$h%_qsT5?Fke9Yc!IC?iRT9&9BMndgY#(W
zglU`SFB862SbW@cZCROKNWo48uY^ta7d7vUn8&yJQ&p3O@?xL0>FhF<f7Yfx7Sh$y
zN?`k`I)`JKnrw>5HI3VM=e>A3A*O)iYDn6y)@6z(k87F5Uf$6>ZB6r<mIqFr)3eSm
zw$eNDWeJ!0L%Bmc;}4lVD2NP-W;9=7E!DN>?wQ>BnQYhoi(E1}@;AT6ct+aknWiao
z?XL(FUg*@aI@q&ThrKl6mi-0qwlckIsV(leru}yP@B3Lz&v~g)M(ac2D}FVWFN031
zKiPL+TGty!^^%6hCI8d9c%<gco?4mJ(D<5ju5{If{yq<D=gAlRx848d@$`5nJO8q_
zhktG}X}x_D-^v%K?5}B(aNzaK8B0IzdRpXsA^zPa?pWK|GJJjvukUYg{rtDj&C)N%
zkgfXRl+MKI#WshI)oyq$<1f-bJ@;T!`}@lW?(y|Zd$!Q&cKN=L`Qj($9A6x`N5{bT
zjmzDKO%ILNGi{w_&l`LCtBr)%a)Cu%zM(w#Wg-H9X@{Qu{ViePgqXmT<9UHwjMY{O
z&+pWW4Nl%sVqqyIu(xhX+0Bp@&gNWeF6Qi9yzUK8K<~OeQ<inFeEmB}ZJwFy)}}vd
z*I0Kr-FjS?vT02~P279U774{J#)FBCrW+iXeXm;=F)XTK`o6i5|J>>~(aCqDVwY6N
z*_z}wtG=;}u1R6KB-+ZQ$9HDe9IqbfKRYx|bRT)OFHqTCs(O3n6uVPqO&8Kr)hElp
zKl#ID?P*sp8~q5U(mO3*>WjRdED$@qdfMmB)ve9Dc$kG)dW21kx$N^N#LV0N!OL&Q
zT&CSK-l_NPwa<6o7x?*jXtQ*8yh)Szhr{<43VIw+h_`$drhPeg&K=g~gYsf5N55xT
ztjQOf*Z1CEV9w2d)~&1c`8M!Pvu1oSQQ+0xB6;?|n;5^hYyBwczmWQH?-4FzjzqD^
z*JfPcj$N|p;lbMvw%+6Wz<SC0{=&}8>8}**Sf!k%EjuzzX}P9<n!@e5FAv@IQPP<H
z=T*wO@21bo@~{67{_|&_%=7tA?%&U7RS4VYa4fh#eRIk0y%`T}=66PIw_Pd|yrXx$
z(RKIkr*qS$+Eo7)GVw^{OqkleBBpreZsyDPcCcT~&wlXoim*Y}kB2XQN(y|JY&vzX
ze#P&&MW0HX7&&Y{ean72Ep#1cw_F7OrPP$0FaJ&WsWQ=W>DovU1Gl?p95t1hZKcfn
z<~$SF$#%gn?A=v0ldaEVU5qE2H9K$VeSKSMo9vx}TJgk1ljfM^iF73{U+^m9OM?7s
zKb?Z@aeHLs^&(Uq8Lsx9QWDCzePu^?)ap9|j<XwnXZ?O!99Xd1AaR0+(5E{$U7U{J
z_^_wu_*sXO%IRD8?aaB!dv5ibwH96XHVT);DqV<vuynfP)xeuw3Oj5TNh$VxoPGL4
z+8oQ(RnEJQXC~hjcNCaqzs2*h>e<^n%)A1OB9|!~bADpB_oOr1*F(%N&Q3n<rTU5~
ze*W&_bx&O)J*~uJbR});l`cf>e(dz}$5vy@6<QzsbPw<(MIGV(KF_^!Z;4OU!&^@K
z-xz<ZZLNMDcBrf_P;!o6f~Bs~gl_KiuoL1tpD#VEr{9~Dr*5+4?;(S!i+Z-IXFY#g
z{`AK30@DY2)BBCT<;<<?u)ddVz5gJ~&eL1Fzp*Z9nS1TN;?kg(hkPeIu8(*ZxhQMm
z)tmb5z4y+yechy(xcp&AzUx8@$#sjDoL$0l%`EHQ@$V|GmkLFfXHE~i<oD?0S8skM
zA348U77E&O(em6(bEE^`tN5J$lDjR0$v(F0MDv%x<#v9Ox%*<aihta9`_5Cvvp<&<
zRwTL@xiMDAbUeTD?XjfSC8o_U6OC`pZR$N`YqFAM<<hseKF)P(`g>$baE#mD&9T2L
z<%6Y-{_T7*dFx5XjfG#f`*8L%&X{37!<5(e*jxqWbFRJ*QsZMJwQm335?ioW;c=_v
z+S_hXagpxY>Kzk(f^VIA8`E=IZEJqfzQu_-tX#TzcJC&in9aiS>uRXxoBRUBz1w%x
zYt0Z{Yqo>oi@@R@L&v!ELn@UEZp$|8OV{r1D@%Ox_Ry@^Kk~Gj4HhV!a*c}1pKDg0
zVpcy<c+M1-GZ!v!&tL2(vpxUZq1f3!@>U<8eJp9`OPz=#+cK`Ww0zi96}jMj`E<#|
zQkmwBo2!nSi)&Pbt-gJzHafP2clFba?!xSP#z(vNRr<DV?Vh*)RZ(YbzDsye)`hj4
zZ~gXF@2oi?v;P)nNMLC!Yw6$cRJq%GR<!$Fl2@;stmoC}z{g}$ASTiL$-|L#TRx9T
z;z7~Be>o17UJstR<0Jbz&o`X5r3;R6M)LKPYIK<vE`H(Uq#|Qr@jOd>O}Wf=-G^V}
zn6LB2Pcxog$vnIK!<TK1^1@Xsr<?K4?3*p&DDm)n-I3?sj~{)kefoRunvlIS9{ilW
z?Tg0rnL2YX>n6xP>E&v4`>U^H`Xv5A+5NJ8`hh}oj=ik8v0}o6^2Zw<tz33dv|3lq
zVg4iE2TxXT%1)b?9A^-->~wGW?B$nVJt$uJM<&y7Wwcb=ji<XjI!=CC_hF&Q*&O$B
zcF|2fj#VEOvJ(W?PwQZb{4_!9md#nmJBBl#GffSh@cq%U`1=0mF^l-hA1rv`y?qP!
zW1~5}2|qIqe>m&3R@PM9QYmERqDLoF9^FvUEfB7&3Op6Ea;JWjr2nkvPxRz-EOV{>
zrq_J7nl8J_XS*ZU8-<@AXLGftWFI>!v?{Xqgu)lC@MCdrlfBk1p4=H2S82}v;CscK
zNvw|M-k}8s<u9jR|Gpt!LwxbKYPQh*9l`JCc8Ez!o8JEznfW?#h0G<U!pf@2J5;R1
zZ~SmA5fT;2zOv%5;?g}iw?(z9Vm5rYc5J>T`#YmsQH$qnSDJ{rS;bG+fZrdl<?+@`
zj4+DbX!B0?w-WD#9Eq!W`)@7yS@<&Q-^H&xV$<@DFt<-zy=z&a%Jj|Nziy@^PFJhl
zx+YNk;c-=ls;Dc?ALBjiS=xT^vTxMy;@f*g<osENj~yXl`mZ}35B&&@K08?>^EJ!j
z>FK)X?=5@Nd_3Cxw$rOAhdu5{ZrWgR)n=Wf&Az0>byg>YrnSnM+kO7d|0tJP)ZkRN
zeW2gu9oN3?V4r$C<i~5~%i?}v`{c?67EEi344TmVEpOGl-EVI5biCAmd+XejkS52w
z{l(u-6z3lQ{dn!SAGhsm_<Zf=9G6*p*3$gGd0$uRG@F>COOl&1)z;RV*12B3&R=vn
z{=cbQ{^LDyJyw&I`R=^a{>riU36tZ3rO)@?Ti`Uu@MQfICL6x+Ru%uB%QDPQUO8@~
z7GH7YHb?IBb#-rNXfU~Qu6oS$FTwZx-kJMZ<%^n=Ic_IT;QD!b!i?bfoqO3&-Dmd5
zdU`xW?wwJo$wa;K@Sdu-Yj3gTTxt5xCGULt(2R2qydk~UUoKI38~FFHw3ptevsYGp
zi{nt)l5UsS`0CHc@0*(EedqrDbdmV_)%n*I*gg1)beM1H^K_U8)jZLB=l-yK&v}W%
z3(xT{O49p(=mYbEma5lDx^1toDKv}BocG+_`QYrUm%Ic-UvJ8|x8lzJ^VvUK?>(0|
zwf_F5hrRu7;a7bt)E0g>J!0(`Rq|PQX@Fhz?GNX+&UrR(MoYosZQA$Vo5rt*e^Yu#
z+(q=4z`{$lc_H1e3;fJ#EY2RZ&Fqrn37MKK()9n+9JlQKZs#?Q@|D=>itAgIq`V5a
zSK&Ch$xZCj?UGo-TA%kq7e7q-Cc9WTPnmaLZpRAWRj1EAeVX>X^`WnI@#mx|X%iRK
zTDI?but??|dq$;w?krDrzFqnApK!2Aq?|gO%zeErx4deJsG5EGslHazb)Ooirkf<C
zCbne=?!9d-@ni2U?{{mItS=qxy4>bwX4LmHep1Pe%A1)Qt)k~dR^%%7Ptub}`NtYi
zZ~f|b>^_cQp69jfZzs*odHPbk|Je4$w+=n{f9Xzc;--b3om>ajbwyMhcet9qv!^z0
z-NLKi*W9?iS$q3Dz3t2I>3=hsxNcRiz`IMmo*Rrj6~mrKM(#de`H5+FWph}kkDu<k
ziOO&Piq-A>5Y7C``+CM&SHagjwTr@@iU__F%Z=s<^6}fa%fw?R$C{dr_xAY&b!`29
zye?WmmO-C!$)rF%-uEIGqV7ruDStjDIc24V9FO1LZ_9*UUi{?v^wA~lrE$N`*;_0K
zm}v9;Vt+-u!;k0-DZhl*PCcFS;orHiTk$jQ7CEHc&}6L@@7lxI<*?}0(rf{rZ1aMO
z$m(ZZ&)(_kRQdKxKk2^XqqH}#?<Vi{pC1_*WBhl=2mT4jDV$T8Hm$B~@(S*KEK18h
zeSERNQ}?CN#m_3@c6qaPulMA?`~9YDziU+Hj<fyx0z5I3KAu+Hyvufj{OWX%Y<cs?
zdv7bb%O2l$V*2rw0mVms?l0dwAtL@r5cg!w0?zh#$*aU~PV=fcSs}Fg?&HsLyPw#8
zPrvuXRPlu7FaGTHf^&DaxK~-a+_c-=vVy&^-R#Ypi^unQ?<tY*e!R%W=6>p{iTigX
z^|WMp8GSxdVp?TZRgs|fd%v6f45PiWK7NyqWIC4cZOhc$@4Vw<%JZI+8+43gj%ZmW
zdo1<%bns8@2I<SfCscR;<f!t_+`c1pmZik?ZyS=Vic=z$@13y!xAcc-U#^GMoUl%2
z*N=|=h8HKDmAE@2<^6)B+~W0xl|IKrZP`oh3K!^zoIn1)o8!2^OW%33`PS;)Jm>bK
z*TwROBR@y1iyhl>yOan+L&0aekLUm2&Z3lmds6wQ_Vq%u_8Hyevzi{NxZFHoYI)_J
zGZ*tVui4Gu*jVbA>H2ZcpMp8EjvJ~jtGmc(=HEJZ|BB^nSK*&NZ!Tm=etJ2-HMMTR
zM{T=vm+#ije(;O`(e+t>8{Mi_-s$UCS5^#eOO5P4_AVvLNn`Te$mNTgb&8*RRqWo8
zJMZxF6)nPV9js(pPfGQk{_2;%KgMcrn%o=R9iidv^3lp7lBRaNi}_z4J+gCW{3q_E
zwRew+&79WA+dVZkFD|hqY3`?cdFs2}r|nv_YW~_&``9htK0m0uCf;SWkW|@?)i&=o
zwOq^Rwk>m9zU;mE`;RgCygJLJ<93>^`npXj`jB+`?#kV-jgx-fTrzLpmTuv9_8ZIm
z^F`OBFLsMq^(|Zc#r+!#K0Y%!<nUS6(^QEiYVW=SeWt!C*OnQ73_a{9ePQ+!ZxznY
zvR(3OcNAw&7oUB9-iwUCKhA#r|MS|#6*E#y|D65eb4Jc$&lkSJkmLM<rtx0yFCMyW
zxsyq0U*y9(*1w$M3ceUd8b;n-C%N?2_u|Das=sm*+j(X^Io5w(`fuwRF%_dDe=it6
zO`UbXZN|<f)>ZRmTAH#atts*R71MtC$b^=@DtE<6nVJh^jh3pWevIG|-^Q@`>W7a{
zBK-IIO*&R(@+~#`Pf>L3P3Kux=Q(yh+fur@*7Ci9Z_?8VQ;as*uUS~67QaU+-g>En
z)NO(LY?FNM7h0+tY|Z*BVHvGqrZ}O4Ic#Z0XYhoye~V0$CtT$2*V?2gv`I-UX<{AI
zL^-j!M_iTT-zGnc(cwBdQT_afP)@5&j}==I@AckQb@nQszEJ<9UodClT<23=irk;~
zob1<|;4A*VTS#RItAf#9?ic)5Qoj6Vx^8fOt<(RN9~U0f{di}|vMI$3p9AM@zWlZG
zxTa>4^28iQlkk<>?Kg|Z{CI4}$`bV=tfII09gnzDkoEr4!OmYFx#+EHIWE9DUE$-6
zwhMCSx5wTz&$#dXDS3(7xeyN7^fD7oxm7)lV#~HUsqWQ0Br2IbJ)-6EeY1BCUZH1Q
z<<dSJHD+6WuwYKbtms$!vd`W8v1x7b`*QvAn5##NUsgVG{MK6JX)Y6^k(9dh{FaY1
zB{Ph6sx<`$i;EliF5lwzu~J7vmrq;oWZ#aAlvv?7f2W6$8)jH=rMx!&rs(@%?bAT7
zJ<8oiq7T0~>j_;En>TyT+Z)r`4SsJ<jMFQB7U#sjVkJ-i^V>R~`kwnTeYQWLbjx|0
zjJ@#HtK4#@%58oxR`fmcELS;m$&8t)27Z%P?R@py+GU~oTbnX1osBnC42`ds%ndf>
z+`mo3?2Y4r+l{3Ycdvb8^J0G%^WU(h*bk?}BUead1g9jt$-MW9wd<A3vLh3He+J*%
z(Q#4y-y`L;NsB6<UuRe@|EcK6>+t1kcYU5+*E@TPz~&WyO3zJNyxJkWX@%zlgOmJ0
z#VV~LvAVIl%^sxQ?{GU({NLkgfBmvX{+Fo}`~O5(e6;u*n(F;4QO$Yv{_3yEGkX}G
z7eCB-&FuVAJn>}zZ_hsta#1cp`#p^6CT>V+7F$&x;PoT4>*vS9pUnrB{JeWrHNX0k
z>!PCF&hIy~o%+^#!D833PQIWo+Da1f`z|P@eOzy3YP&Y9Y2xjDS>Jb8O>@|A!dWEu
zWx-yltA~Y*EQ}U>-`{=cxb}S4i@Uw^7F@S{shxKx=lJpCliamVMt5*=WL$dwZ(2or
zvXEV7bmjjQ-}k7cn|@mLZ1-yUzd2<`C#$X7-2L#e;hlr}TYkSWe0wgzWXjS+4R50f
zy|MdmFU_Bj=)L6G+KFo`xAm<IFIBx6X{6&JSo7xc<SS2|>!N~0-@LlAn)gkTY$0c;
zo@2M$lZk3s`~A8fY(4h6X5pklPRDI5-c={g@6CI@#Vc%P&gM5co|-nZ-(>Ayt!%n<
z#{%DthNsqOd3gw!I7B$+b2)@(trwiEZY!M~^hGDV>&WLF1+F)jU0SE{+39BQr7!K#
zdwoxDzSqU4m%nIJ%+GGg`}beG-Cgli^S)01$Gj!qe@xwV$#uu9`|nMrvU3O2R6EYn
zvGs~x_-A^Eq1_JFdkHQ|%O<s`&J2{(+pFRduJGaYq$B0qK5MGwZu-8_`YFTB*c@vC
z2?_R<^AF7beBP~W(xHL`nM>*hDW}Q>(&c)h+SW6Mdbe2>e5ihXXrk8~UGCpHPjhTe
zmF+h1OFcK`rpls2kL+$=S$SQ5j&d;Li5~6u2CF*yEHBi0YhPEmZDp+LxG1+J>1MXU
zey4|1rS#7wu4z+sbK8*f>|(j9q_|)D#BYJivM+?Jh&wQ0M$t#%oquPr?G+AIo3Sp{
zp1sEW`v%_`38(gUKk#~fcH)DXhiY>_bDY~eM~MA%#@UtI9@{m4jo<v#<+RGnv(I(P
zA7s9kTb_Jaq__IE8q3yQ{qZ*=ze+op>AksH<}810`|CG54!ga-#h2VQ+3acDlGc>c
zlBIWN9z2<P$A7nPV}=X6(9S(`HKX^w>Q~tzb1h*~%Cf63?PVU`>D5^F`AV~gckz_e
z-~Eg;T`XBTYt_%SShatf>UOHhdDnv4d-IhxJ4{>ofKe<XL12cY`rA6sRYmogG2Wbc
zPqvHIU7EDvfUPW>|ALE4&wnz0q(5CFMM}D3)4gZg=ifRqDX-;K&N1d?X13dJf7w=6
z{9{q>6W`Du1-lzKKQ9rFloeBmDLAq2Tb0A(bY;I!>r#ystBOLW8|(h=%20@OTej2n
z`CPTtX9E_QXS{nWvOSa4S36^U&B3q~S+V#P3LKwy%oSOy!+f}fvH!{RGkOQ5lvg${
z|Lu08;~Rg|WuqrzuJ73r-P<ZfBAN6&7W(X)wX9YD!poh;A)ohuUnxIxPF#rV<mkH(
zRPQZ6^rH68O4SInXMc|xZM~$=%Ig1M&+8*AC!W)~_r)MJif7ZAL@S@5;FlL$m@2IG
z9ND<C4O-9d%{ZW7d~~_p^p`(f;@7)yM4I+Kj-P!~cE#?0iWh?R)(EZNWFW@>dL6f?
znPH5{?~^M&eh_+nZl1{!AKU6{R-aS%U0Io~c~8Hm{jx*Uq2tHy-Fo!F!$G_$a)(c<
zW#Ef9<`-qx<y_p-o3?XW>dF%LSI-KUa9^}aH%M4`NAvgFh#j4dv!43YUI<TgxV=#$
ze|O*0vp1(!+HA_vJz4YURj*EHsNC`i;bKKuj%pkm4+=GXnCE6T>wL~rW>0lh4n7^7
zzMHRF7=sezBUfz_+2vp0bz)Na+$sC5wYIkwFV8TJF<8$u?MATh{F7c&^Vdz+U37@g
zPx)*@U-|K^s;gVBvahi%_kSH%anUvMug#aG7pqnFm<SaLFlQxou6=7h=c>bTxwswf
z*}_JC8w@^vU$@2n(z#hK>!s}T+`OBfD2cV)Wj%E?de@Be)8(xLcIdu1p0^^cL(ALb
z@2NkYuHlFK%KqLkpS>Xa(y^>Y#{))vmyi1FjtCXE*O<?5U$*g)V_|_g>%LQ)D=(~#
zJ9p}RMu*sk#5I~RjUVnFdaB=^^K{epL%i?GZ$7lGnaW+g`3a-pOZz45G7pRjXZ;oy
zSD4<ITln;x#y*=nPwQ5HPgr=JGwXHM(bqGW9><)S#Q&|wMA6%R##C`-@5t>XBI$80
z^`|U~AFi{xv+*E*#g;k#aYrA=?Clo(6u92jcjA;4Z<oApmKB!mc7F1))%Uj&*VMTg
z(vNRkHxRw9_wK^cxUDDmU$l0*D6v9)X@lxM^XZ3`45pu5$g}pzhQ*287EjmgkxjDu
zwckhgeXih@o|p*{%L4Y)u}@prp#Jkt%!MdMm$Zf=Mf+!E<)0SipRD>-*Z&~nir#Ds
z?hg0-o+nT2lvWh6SIG;u{3$G5y^MA7k6c@!g!B(0tv5^`Kjaacd1~uTP7C=uo#Nd$
zjPA7^zMh)D?ce6^4c;vORbKs0o%QM1+*L~|O{P|Rv~NA|=%|o37ynwX-+}$JLmK%i
zmU%?~YdyYx<vYH)C%HpiT6Y<*yuCZCf47pd$YH&0S4@((8DDGayij-b(#hP34r&&=
z8UxnIn`fK$|B0&0ez@`01j|`1XLldhNV{;$EMuwAhQjC-ceFwdixwWPo&5Zu>Dw5d
z=V2!6n$OwnY3Do?G3mI1<lSA&nQNanezDzt^p8()@q%OPPxNLBdMLF9Jp1!|S<j6!
zy=7bG%)BEwHPcmUv!kha&XK9FVjn-(;ZB%Q)_r_)xR1l^@3Z&r{F245?(eRu+WVq6
z`cA&o);fOuv{&`}wtv|F*<+#R%Y`v_uf4d?ctC20R_!fYm0j=hitRrzbsT5gbR{!v
zr9}6I)BbZWeSTDO<-g}lf6*sF!Xh6`^&i!}6ss0t{l-3R(yZn22ETUJ$ke8IeBb3T
z=lz%7Wm{K&Pn@OIWy#AJad+xgpEuT*9@`x$jJ_-G-+Qyxwq#ME=f~Brw&=~9qW0y1
zb!0H#8t-?0v$M|{iylr&*r}#@y)=|BH*dklOCB=6P9+MhdcH5z%=qc=rH_^;n`VAr
z<NM^hn(G>_kD2OyOFmwDoI8ho=QG34tjAlp4+uWwmA`v2<Gxdt$^8woy<SF(ZY`Y2
z@%`o067lMe5TC}p{7u4BufDdiNml$<wkPpf(!Q^C2Tv#8E$8!3>(6hRl~>$(+&I6~
zY=!XRfa?{3!h!R5%xe@V_W6D+BjNm?tFwRVZc&VrFs-?nA)0jQh5ntym9et6cm3Pb
zYYs}JXzCp}@MX>W#jECvPkJ}!O!S*M&!g)66BZjqcYpm=7;JpD?d{Jg&-SP{1pGAI
zrY&!On|0ZoN(rmlg8}!eRvq@PV&WHl&hH-gzRxY}-2=niMa)~?pJ;gWT)`$g=J>(b
zbKU*{GYSq)(#n4{@n>Dko#xFIr$gk!Q&zDaJaF{url*{tAvyaky*~f?=(4e*r#XLb
zywB;+yTwJn=1-XQ#AC8Tr^C}=dE5IZpS;(xUm$empczwg_nfpz*)Lg5_QfnQ*Z$5N
z;?`)&+@NyTfH`RGmh^xvzu1o4lYe8*yT;BSNbO|zv!&%%f(|ua)&HQr*2;BPmaEj7
z)!!c{o8H$A76=bwkGHK*o%>qz;Eyd2uWw0oS1ZZx60JOZeUfsd*^*ZC>ig3sZ+-f#
za^L3{W=9MU8q5_~edUVKqWAi}Tko!yWlpiv&$zp|C}P4_UZstvZ+(cJ6EiL326L2V
z;?$WD?K~GX_WcgN@<Pb*uj41h$2X6NpQ@}WP286JQFcpa_p;B<Q*V57YuJ(;xOAK9
zv@@!Tzb1aaYdy_)?Qy-G7e%ft`6kJ!_j;Lr<dui5eX{FVz6LF{yy{+R^~9cA;<!yw
z+V-e~!zxRIb9TC<++1|Ci1Vo3k@a3?7vJ)RyY5sw<+UN^^rqrN`t^rZF21qWVdu^*
zs!rPyd#TO4ZQ<j)QIjI|Ttaz%Z;<&pVb3i+m7?ILq#1h8f;u_v1OLjE<b3{KV<Z(e
zYyDp(otqsK)^1lg+NyL-b)IvbQkkdbqO*&`l?yMM&$Pd*&*C~Uxx}KzDa9<-Npm63
z)f>%+RSr8|wb*}=Kj3qG%3O_0fA%|jiEowQ`08<4?WyR})O4Sl2BtO)!4VDn@0p%|
zC!V+I%5B!F-Ror(x-8gjzwe!9{c(~))`?>wQ9D_b`I%q6m8`z0@uuX$l*ipNTO#_@
z#1`f~o^|N_yxLRe`ED)pE8_I36YPnJE7S`-{#aV^nto7sXY)>dKg}C6>MLhB&YNN`
z-k7JC<N40`{Ia`GyjO+l%Q-%Lx6tP1?&=Elh+A6Rr#I#Je?IlwwdzyE_hSjc*Si#t
zd(N=gE4V#>^}agBy5g%VgIE^a{-80(#3rNcxl2TZoJ{&J`+^hRW=e;TZ@<jUymnp8
zFY#5S+_Fa{g%+u`N*`p|>Uik7)U8EEbG**q&yf5+v2O2FS>2CuHYtAsC;px!zG(ik
zyFJAZ?yKg$o*OJ_%BU?E^02Hqet*G=T&X!7KcDW2soWJW8l`R1>9odu*2-m5X7v56
zzqxhOgoSSnL{ISQboFqa37z;T#>w~e#G^%pqO+o&SoLo^maefPbvDai|M{Q9)w()(
z0?vJ!W_oe5$%~lV>$A4-uH2eDpC$O#NvWf?Prj)~8LWB{At4xOR1$KpIMu6yOFi%R
zrdJ`GCq#vDEe@T&?z3*ty<PfJ423;&Ue__JpPIRShNFh#PRIKUyt-D0n@rb6>Yh29
z8B@FZw$_JZhMC{)<;Fb9?~|SC#cx?R<>`^~lg(Bh)0eKbo_2J%T-UXgmP@DIQx>-`
zGiST`aY>tn8qe+xy_)h&xsIX78uNR+c}(vgidpW>Tl!GE_xaID)l9CFZdNd~OYB+b
zlK5x+rKhp4p2$vSP3oL}RP{2;s<iJbFNg%GZ&(|C@!8AEQd{rNVSVMZpvFfj)#ypa
zGZ!zBb1{}2_gEfA-%b9!;<U<)^%WEL<b*AWnt14Cd)D>iY<pgNPdjjkS@R*E+wF%x
zS=A~f<1VbN4eV-Q6JoGxmAl2hZ$+#__Te47z6aRN;VL?i!S$K_WB>7W7ZqGrggjqd
zb6rtT({JTmUiB{qx<{L@bGClaV!l?jU25;s^UBvurXN}qZU3-o`O8Q8zfUai*!@H|
z^o{H>@7>%t?l&=6b)UYg(RGkxAMb|+=ktr5&;R{&(&d7}Lis1GAIi9D5_h&eY`nVQ
zX4#J;`UyV-R{JU+e3A3EB5^fas`~w+gC<&L4-<7#g$~r-N^ETOV0DwbkST1na<9Bc
z@`3Y5e_PBGPd#XL(S9C}(rup=XQk6C63b`DsGqpMD(-Q{oOet3ZMSnahKPr2uKYT0
zNA0V3TPHq^>@s4VI#VU=-P0(8FGh^7HtlcNr^QwD@W9G|e}|?eCOwZ|llJ@AzjPty
zOH;ci6)>q)U!1Yw<E-P0yVjf(QTRCDr}9}`&gF9tKMFi!2ua`L!_=htq^<EXzkWw(
z{oJLRr<^u@G6*<hoscT0dwIeBnSEEctrxve?h~t3`H}V4E9U$}ZgZcBPs5v1Pu%)*
zz#+@IU-#b}l`oIl^y+?WT{6?0Q_^h5eY5i8?e+g2Y_#38e}3_)X6|*`l@qsa=F2Gv
zY+N{Ta-pfGN{CU%%Sj$C-X2rMf4*v7=HUNy=EhykGkN`X-22O)r=7rZbgR&Tb6q=P
z^L(B@EvRnGUp2o??JHl&z70k38TI>>f^(1Sy?dOrWsCCz>5A!9KQ47!lt({do2;=>
zW7~U=wX2Ul>+9+bu`&PoTIYJN%J%mcD{q<~(tmW&vg4Gvu;m>cvr88ucJ2MMXYZp~
zcRr>5Xl-1~_xWML-TR9L_hr29^A%4r%2Zx`;+)ndjUQ!;&bPbAFR-hud|$akc-GR5
z9*ci4f0x~m@h8wmfLB-O$m>+IN%^-5@1=%!#w`7~E~snY!Sz1{nrF4YO-f()cgN>*
z;>i-FjQcj&%C+a^RXmNg2|4xdL+R@q`aCm_Z`@hf&Lh=ar+jtI%18bDaxYc}&J$eS
zW8WnE<Q#YPp7$U4-{e1(oV1gXza)S5e(!CTYL?MAHa~w|_4@2l7UP)In(%#cQaLA?
zpXZmci}s(k`jf)3f8R>|4Z=Z8w|*|SePhF>f81<g@w~Txq|O%A<s5q}CX!-gxpPnM
zueUdw(l6DXUDEMh{FLU!ZDz-rov*Bz{I{_qwQ^a(f?1FLUfDm_PAcSG)k^!@Y{4OZ
znf-5<HvevW6uy6N%;C4IbKjMIVxFpS*d}+9V%w#u8w<qG27K$C{)18a#d)ifr+C-f
zoI0hOF!9({qdna}SDD{ypLOu}tJ^Gpw;d4K6+OkMG-#7pc}F^n=TfcDb$YsI<8I65
z+_L$*EcB+w(^XkV?>rMczvsC3&O?V^iv3UFx-s<>cdVJ>AMW4fFJky-e>U4cn`>p~
z)UC<4`<(-d&h#2-E;v8k*3(!v<78g(tSjegC(dr@ZE+S;eh~Pr^Ud4$+tUuNJint?
zAoC8#Dy8+a&Tr5OWcTe&njE!d<)QM4ro}3gZYVM~+I^T7@TB>z$HG9n!l@GbbPcr^
z9Jse;Qq;b{uk#d~Rgaf$mFm>myv*V@dy#PXxsMs!=dF<c6M9v2YJu;ny{4}n_kDM3
zeY9Vh*{*bEzQfBLg9lreoSYW?`}&eIvx`qZm7D8Od&m6q@}kwA>fhPi>h}b2@BTdf
zQ(fTUUwSpGd=EW86?NwY`^QC_UW;=on!a>)H&i|)7gC_?`+ei;^SW<Kj;6LR>e^VV
zG3&6H?vZY(W6xIx72j|>{GH|It)9yCoo+w#YbyKBoHX5k;;?d6y6p<h9OdgSldC@Z
z@o09$+zhn1I(4$a`MB4cwneDjim}q!_ICNn@_?Tjv#y-X|90e+K<l4nos}$XuXLu%
zUE0?evXM74r}1p=4V`kEQZ@D)%gz+&Juja5v~8R5*5JEmT5r9p^boE1y`b;ytdnP#
zZtOnY`=OI(e}qo;smrdjt*%YHcvw7Y_Vb^;A(yk4N8V_wQ<*x$E!@Dr-(D}RciD;a
z?jEcK!hbe=ww$SKpZ9Ouq5Si|s~aYqyn3qk`uVo7dEy7A99q2Zs?|^K>g2;y|IE#4
zb;}KU`FVFUo5(6*-&5E0_5PfA6|dH0ellVkhew&{JdRn;i=)-0x3ck1JF0N_tMIwb
z+4bB1J=|S?#d+JqRkmk-?tc9IuY+pLEhUXc-+2qJv?Q&X+mjvrMwb7n#B<fJcb(>+
zG%?+HTxddF?%j|Z>o^0`C2p;&7FCXW%f=k_tLOm#BeA1fE-x?>Id8v#{qLopCx6s!
zV>I8j@JChT`nMY@@(zVMaQyq2V=`GIdD5me+kTraHoN&m=wCxGA17Pc#HoL7Y@cLN
zF~zIaA~*YY<2CP=mGa$}7VBD+PkbDbvi0Od&1~nkO<D2_E-ul%R#GYwd*|`#61fdl
zENq;0U*%Xof18o3DQk9UX{5oWCl10X0<Yg(xNH6Cnqll&_vhtp$5aa@s=ipU%&)vE
z{zmEajN)Upk_TgszZcSO;pMKaem(X7f)h*1k1$Smwkmly+xuSj&&O$6b-$kP<9PQ^
zL38KcotN_NugK^)aedyBqNpS5wtxPdY?Qq1$MFjjK0SC<IWbILJf`ei;AxKO+NzA9
z&rWaNXwPvv?>l$({$h)`>DtrZ<Q!gje!t-LZE}@?VgKzML+9oNet)>vk9Dto2D8t~
zLt^e;`!}3kle<-->2I&a0-J06e@?irS={${nWchU$$kaT(DIpc?<NM#JtrQ!NwM%*
zs#%==p49J&Ugsw*U+HCW&~?!o?-iduFz@A=%I*1-t6qIK3xDc;L;vf}Q7yAJO+GiL
z@N%u_%RG0H<9}l$wz$O455HTR_aXZ3){HgNel0Tz{yKT##~lXWECpFxANGEhS@thZ
zb7GnpXIQMe*PC>~!q>e_y#+s-T~DZfIZ^hZ#`U$*&$#;2fz@{J=Nxyfuh3g)>z?ty
ziz#{1%8%;}Z2#SS_jyvWS+Mzu@Z9vybMiI2xN7C@y?9+%<l4SM=6CB^P3zdn8)Tne
z3(qTFRq40;PPEvvdzwq4@3t9638a30UsL~a+w`TmxvQ#bZly(Ae8@k)blJ^7Rfa1y
zv(McO&CRcW{Xc4Mj1zDE>}#LuzCBkw{;*`aLfjYeTN@`%j@o~J`)#gsD~qbmZnKNJ
zvv9-R{$-t<oLoFRTK3;)R!x(>Q*!e*%O$f7cekwD{xrtxj@rSTr@Q6;Up4)AeP-Fh
z;_8nd)_;$6uh^FP{qj4mZs9im-QV5Yziq3v+rA}RX+x!ZRMpB|b(5Cl#Gjn&*zu%n
zhVTECe9rq$U%1O)usrd7;^)5`zVU|r+$3JVxcc?2v-;;-R;`Rb8FC?4B7SXNRKK2;
z;;tZ(EVKO^vKHsg2;05Wqt$M*OKsqzH!tO<*xhAJTDjBQef9R6`*!_&y1KM1@n>|w
z|DW3yd8_AW%~>XG!Ei3@y~6EWb^pt+-6Rv_y7uW$nEQUx^?kn^6Sgip`}Oz3_%Daz
zv%B`#-N}sSn0|cy`sey@^R}1PR=cfwQMOU%&d0m_V&9uDZi|}{_1iIDw#7^GfW|Z9
z8=pI;m&G&GRZMtUUu*w5HgW%px=*<+^1Uv3MPKjp3m>z+H@8xMxv;BBIoF{J&*fe9
z+c!1b32=F_i#2fKHjl1VDgUC{t)5p;>0kZy_w{c#r;9M3mzI66vQ&DL->l$1>C>mb
zzxio(G08Xg%G$^0R<)PEnJ+7Eo__UsTj9)me?HCptv&s1?Yj?*<saIul^x{@uk){Y
z+rIcraeL;wZP|-2XtFGse6*}G%;yh#>f`bo-C^%L{@*vtkkL{4GDTMLsd@U09fxOL
zZ@o2Lf8B46eYf{V7l?JN=665wH16s9ie)AKHQV^-zP+^Ae*NE<^REcS8;8DsXrO=F
zN6hJ;=q08Izv(;A<~qrI{k3?t-S*f^UYpCGvKe$1sGaO#)t|)m(&CNHpTFI)8x$9v
z>T}B3R~6G$nC{qae$BOQk<H}#$qTsTKCF2qByRuNY_EOBV@dr<@l{tpe0&j*lD$QL
z`G4j)G3k4zPffRR?DpQtEFSP*Non$3a{>3jOS>)1uYJwtX}@YMyX}1m)9d|WYWdmu
zPa;<5|4=w|VVCmnm;2v19$5Bbd&soNr=Rl0tnWM(*|EFh#W!Kjo42M0^v%0%Z2h=<
z?Jhw<FGo|+P2m$1c1~N7{C1}CJZ1x(@`p(pGcGLp(EF)q-;rFsQ@bwdS#t$1Jni<=
z`j$?$K>z-?M^%lZs|6%8L=t!y4Nu?Eh&psH!sAZ`gXYN_I{Ozr_<xmUu4KW><I)b_
zT(7JTnI2Yc&%RGdp(j7a%~JgGy8D8co%=8G$-J%BdGM`m=C4l`VtPgIbM*c{<v;X4
zXU7G7iL2KaFx>V(CjL;S_2SB3KcXhZPVp9>KHo+*SMo~TEslHbO7`zRE1my;>)YPn
z?8T4zKPg>yJJ@smzP8@Ij;rrp7oNQS_~#C>){wigNiQX*9d!A#Z(X`y=DfY_k~(|h
zWA}glo^j<+YmbTH4Qt~kFQu2x(+_$n`}6v4wRsF8YO6}+muBl1{?!Wo$gdS~&ul^c
zg{W7*&7ahN6KUHuUr*qSkLvq|6<c`1!)_in7Jty)@%&-$kI(CK+uN-hb}UJ5wFv6;
z+WW!j@6({CLen!D?i-!=nDt^$T}8wkmlxCRIksmg)OH=`?ufj5_UAu^Blp&O?^(U>
zNW`B5HqY`|uHV01^6f{$^p|2UkFV|E|MKUh{N|PQi(_lww6gqWH~D<+fc%quA1m4O
zlfQjA^DDRaR^j1|6P~2Rg=7Yw$n$1;<?~p?#ky|p%+$BrdScG230xLBo5@-CEqnfp
z>o55EXX;#B@?HDtb?)VnI~peCGv~yL`t2-`Tl&u1C-K(Q)pPe>P0qizta!T6<kM0n
zpW{OGnl;t*pKpoSocUGHWT$4pEj_-tb(<ti?@hb5A~*1;@|4JqV-K#c{@BiF@<nM%
zGxuB@BU|1GmG7DFxBXpRTt7SatoHUVp*xo^JH$HkqUF{(y$@##v@Kfj?v|$gyqQ(O
z&pw7fK4$i|TYGl&JzxH9BIjy!yY3ycT{3^w$%#Q`)@!@eKNek$%RAk4dRsz8)Y_Fk
z9xt2LlxEHmcu}|D?1HQ77h1pn_eX!$Z9Va47jE@eGMGrH?Na@2|LL9Lsw9Kls@5$P
zANSbJ%kNvZQGf2V{pQzu(hfv=f9r00Uui3qc1goNAoYlJc*=gRE#j)jWY_$R_RUn#
z<y3pW>2&wEqhA~^KdQX6JafU@PM^3fDr=kfM=G6lUCh+)vU-#Ft_AaUoOohlG%@y+
z%H@LMjlvgJ9!WV_WoKTMZKLqwX2QCsuBSaFZL-t!{V<!Ubjys4!no=Cw_cq${rxTD
zxi^+iN>`3j{8Coo=Dq*;nu}HwQ|<@UdB@Dzp`-Du?nom0cYTr1+>&$S_t`7HWWBE(
zmsjR(ZkFXCB2x9Fr&@NF+4S;{KVQzvpY)=<B1P7}yRxm!tFd#V<9S|_^C~aAn3I?8
z{55fVX?@|rnO9cEOFdz|pAs!4-kLE(FY;UQ>0jSV3wh2Z#;@*F)Ox7$vbEuY1aB61
z^%C<kmaNKkPo+{*#qCafY8BLE*6Ob1l!|wHe4DMOWxim-_h3Ei6G0B0m%7zgZWr!d
z;s05C_sw^ysR@r(X1X8Wx8h)b=Ehpb&--?%Og9%fu6IUKy8pCo@ryHi!zJGxWxRjn
zb>!^Ez%cQi9503Et5j08_m@399rx{k`_8MkW2<+rJ$$#8`Q?f&O%Z!1JS@oB(z(6d
zvnu<o)9NEjX3j84`z{x`cve&V=h)33vNxo?EfPHvVrrDhb8$yWM$(&{4H<8h>N@WJ
z@!g_-?`#;`nQ3>TmT*pgW0d&s-?Hm}rTz=w5;82Wt8u+{`AM-vu0i&M57W*%n8)vM
zDHK1?@yosV%j4b?&y(c;9@_9P$NSGAwO_6KOEyl~-}p*w+r_t=M2mM+hdo;>crk3k
z)x3*+H>T!XvHiL*-Xc_7U2Ab;3-iX6d8IS6|JBL=J=7I<W%BtYkA3~E1NOw^s6JlP
zw&^pk@Z(cGPxejtq`Fo^@aBoHqGbmc{||Op+nt$G*_r*~(#GGy`(wpLZQrZR{FU>n
z>hfjdix>HPXP8cvPxks6Uoijk2R0SX!d3Gw-0WR^cl(pmu1_`vTzeif@s8l_#bM_q
zf9S1>37LKV#mo8E7RXieJ&S*RbV|eQryXJowd0F^Upi&`;4^!a0RMS~XUd|l{<1GT
z{zJ`hzPDUpSunr&9>E%+S4KZoj#c+Oxv=~6fzyYpKipM&@bR2;;cIuFnH+t`({DW#
z7fiQ5@`FuKaG85rbLDKCxE&8`Hfk-tc)*<h(B;#ry%jZS{;^kCVn5xoY5T$Yj^X%!
z>38!4R5u+;TmQUHaC(QFJcG*g?alek&uX%d7oFnYDsW+6-rM}#TTdK+?{|7EA-ePZ
zcc!G%=3iap+dmhuyGqsxg&yhp_2XcejoF5^8aKCeT^8@SFD^I3XIp=**(w1AVXeC-
zM4hhJe|_{Oy1$sS)tYl-(684|El$<h9C^}sPkQ35H$I|2PCSq`3-P~}b~4F6|7+b`
zoj-GmHZC}<6dqb_Gx1dOq9P;h<FB25Njq<SHQQs2)V8?Vh*OJPer!3e{YPrU)gKm5
zch5B~yi~gX&-$wU{iXA+3fWnP{Cn@P?WNn+TK<<Y_h#H#{`%vS(m0WStAifI>+Slv
zr}e13rl$F~fV$T1OFuUKSA5gGEBERou5C{(OM(UG8Kl;S-pcwc@Iq_Pf}XPD+s<@K
z-#h=HY|@0?@-bgdJM$dL{r;?QUfC&QPRa9=`<nCDEcUT9tPeOOvg-ZjE8inmGO{hd
zqu2IJdBw)}lYf>yy}hqBR@Qm7li1^b8*Hz<k6*#0%M!WtM3==M=f<*_3m%n4*E|d5
zHT%9vA1H}E-SN%A@WmCe&{K;%^fXNF+o(>QzgPDA&;7sMPdmvSbdBVA_H*5ZF9uvU
zZ+%=5x77BLRKLWllck$lJ_u!{y0Qx$e%f9sP*N=MuC9Bt+Uc1+8e6(&-MFRCz$fbE
zm1ZVnzPqzlP-EAF>-!l`YES?BRI<Y6XIJ#Z0}ehO|E{G(RTvxO3R=7DKec&or&*8o
zl+|f`zRHoux@Wpfzpu+OeOuR>H6~?#mHSE*U)p(XZQEtCYqIbx4Xc=AoM)HGXqbd^
z7>I<u@Ve*tnKkZ8O|I3ci%PoNzuk6=<?%1nTio0I_v*9foO=tZc)SkGb(i|d-u`Uc
z?z?+4PDng^nELePjc+Bte!3|>u6fR#sd~|@v%D&QujixipokaG);TS^nKWU|QT_yW
zr(MyLJ@~Uv9X?h$L*(->{|gzX*_QMNp2=H$dZJio{9<mkl}mJ%o%31QktdM;i+^FW
zPQ<Bg%#Tm3TU~wdu;0r&rfdctF})w#7JiGonttz}(bqi=C!~KKJ!Rz|A1APd$20Al
zmeDu2Gn3!G6n|=e@8q&CYr=zFA`4C*`gyx`{<MkC_HLU3r>V0}jlHe1V}?a~<+Faq
zE&GI~y%)2&dW(Or-R%5`LzkTmXV%o@)X9duubBDo+kK0WgOXdDdnG<pmW$=v3)jE(
z<}hA8m(MHq__;SHZs_Iw(Z9EF+h)ltN8@8$B~OH>TsUD=utwju!0v?amt%=%`fEev
zA6@<EaBkke$l2X>$=824X+Ch@G5^T~lZ;DTn|e|r>#ptm^}YB@vg@tmEnfEXQkt|Y
zMXg_n%Y8UKdCp$W2}w0<zT)#wP0>(0K2NRoua@qO+SzjL<s0XRIP?h2USeu9XUc6B
zh3~Dr8-9J=xU1K>Qpq*x^_~3(54?LXUKaJ=@9e_=0cVy?|5PP#NmOLl$zA)y_iem(
z?ZbS2Z?8v1FH4@vyp4LDqnYj=sCLmgp_kkGogn*}J}V2uyQ%Rz+-@l^SugSGTr-<$
z*0!(>oX0;Y@n6xGS<)U{bNMX)`~D+GBjc`oSGrNU>uhC&3TwZ3u1n26AAS2}A;$l%
z7A0pr>8Lp<rKvyXV6(|X?*6P*M~^Zkd#dubPm!Fzu-WqCK2N`AADx(MQl%ajO^{}{
zimh6jcaG~pY#aaAl;G1sGuf_XubJ(|v`pCaWwU;crN!h=2V;W%tn=+=-Bx2XtNVaw
zsNjyJi3)O+>)Z=;ImI@ARuX&r!=koc-%^dwZMF8vtKC7L{I%FLHF{XhmdZ<CKlJ3J
z{qL&A!eGYF7T?}kNniJ^*%<VvRx07=wXD-unlGJCdQ-mW*Y1GFzXOz|&LmA_pJ??g
zG34B&NuQ)&X#L!Ieb@VSvD~j+-<|5-`p)R<is`qGF|WDx^K$NwsyFLZXNjx~*1IBg
zdh1EQ%#!bmYA-ZgFN(hM_{DAOJNJ&8mEKp$yYTnGc}q#oCYDn!YgZrIDZa+}>#a-q
zkN3>p`}UX2zQXhwRt^onD|vc<-!ysfuqx=TYyZQCrHLtax1T=gt5i92BILyuEy>xp
zk2qzm@|&qwy!&7F@34*2{N0~Elry(GrI6I7b?(omY=-PT=PLUg<{y0ccHQp6BgOJ+
zVaeP4v;IzNUjFQ3(bQgx_S79_@ddjly!XsBlwPHIUF*jA7am)FOBU>WrutB|id!UV
zrN<rxr)2j}GI~LmY8it=r0Rmt@x(5-ZR+HU)oXpU;<fa#56qTt*~<ze?5xl4=AI>6
z%_VH*`Ms_-o=>!1>dy4e3K{(jB|G&?H~Lu8J1ecj+Ku-=aNM8&P5ZLNk*rUD)*ZRc
zbl$bEwT#_f^$K738_Q*~dWYK^FTKC<-0{4H)Okzc)@c)&Qp2UzM#rbUy0(7nBE|Es
zo%JH7-#%*BH}UblDdE9dtv`C+t~ffm{bZT_W1H$#ySS!uyZ+k#aGs=T=aO8HeS5yA
z{a<lFE2#DF<eSs}|6t?!DErj$^rnd~v+oqX?u}ZZ|3bfXnQPashhJ^DJlSf?6;G(C
zUDWHD|1@=-f3dM)z_k4dE*AoLW_Erx-dL&seQVT_Pfb%N8$?A4l|<ZmZ{ND<<z8k-
zKON=C%l{n_Ja~lZ)}bvMU$lGrhCOMCS>k1FJ2kyxzwR3e&!@AG=!U&tk>~gH(7zc!
z>{oxky(eQkS8>a`xv5GAr@i+puL;)PwPA@@X`y0G>AG`Acp~>EO!&9{jU@|*dgk&C
zZ(O*u`6UiV#qd3<@V|3<%e9#&`nFon{wQW-*`U+=Hqr9UPbZPvhi(e!-hbP$CcakF
z@aMZ5u{^6i7GJvCVjb>$dF4il>Fc{VC+ltKSYD+0TF&9v(T<BZZ|fV+WcqWuUsm8v
z#3XZV$=>d*MIV#7*}wO!iOIQIy>`8Cz<<eYsVWa+IP(8FJ8$2qnp^ewXKVMa#Ra<$
zUpZZW=;lt_yGyrZ{1z*D#D8XqX3){xoef@=zdo_AdpT!w{pQ{t#-tM;<}^vzeQ(?}
zJy+=bH1+uB^WL|g{LamN{A=E;&+m3sxQl;&ni#NtWmR!#=7OAMT}C_aY<OLFMqICN
zj>bioneXmhT6nU-D&Tt||LmfdWg)&9tXp;$>o$H|;C@zs^Qg$LLuHp*lXm3GO;Vn?
zG+jfaf4i5Mi_*T^&9(Y-Y=0@;vb_>H^UBGzFKcc$o4sdqE8Nef7Z;kfU#azRNF}ep
ztx4Z*uggjB{Hk;G3ID{M$78wf?wn(_c;~b7m+!cyL{_e~Ro}UITh7<zD>qFv^ODoK
z_k5LT@d<ZPN$39l^))x{>t)Nw)s@KYpJ=_aJAOg?jd$kdcmI7%imTZiRl!}dtjBlz
zzXP##!B%2(o;yqXi~l?;##;Zt_uLviAH(pjpq-B<+x|{`$NC{K|5LtrlA*yg|I0kq
ze;KQ-CF;DAp3d0pGP&Y)m#63EH_PYqMDKh4@py*Y_B%o6_2OC`qTl`QIQZ$(CL1%u
z&;7HH*U!BhT3zE?S)W?kdnIzJZF{`!vll<k#IC7(y-`s6%AWUE-!3_|^zcsqt$A~|
zrM}qqzT$Dysyu7aw6oK{eLsARvv<xE_TPCg?QWK_{|rkrwpxFWe_N&=Q%08I{H#gE
z(?ZVa{=OiloD{JsT`p$l?MEFsmHMj#<{6*!>Rdlb>08X<23a}J&5s_v-sqFiZn^56
zRx`sk`J8#;qB)^l7njY738+2myhu*jOFwLBQcxA|Y<55IgRH^<FRw0Fl3fyOl##x4
z`j(Tn8@>cCetw33)7B!HgPU(oDC<tM&g3x-<=@dd%g%Jt;vFlwG_sRrf}Fy%_V45R
z`Tcb0o`0Wr%>VE{+j#~5noCo^9{pYSe?{b#T{Xv}gPi;1pXVky9{N}tea^ff^~&S5
zqPH{R66Y>|xZwE2nRmWF+AZWhb;62&mwwAG?s^x#dQtKG<z}<F4!d88lCPg}SXf9e
z>1&Gr3ey_@O9kss#}z%#aSvVGqp;)pWc{3Oh6KCR-$i$h=*_#*F=2JEu;bS!7w+7d
z%C?<TIpB3T_ly9}I~(1eL{>`tJX|JoN9_5+Ed?THjLp7GYhK1}@rZ>lD}}wOU-+72
z_qmW*9lfmY+HNa<b1%N`qa1DeMcQbd39nY>Yt4qetnRm(_&l3J8>L#soR5^fE1xWN
zhW*u<DT?>!@QO|jopnIV_rcS;Mzg&q*Pf5nIB(9}Iy3Bk!RfcxE_`Z={&ZvAo72;0
zMQ5q|S-L*Z;F%+T;-8(@9{nEsJO1aUsQEu+_<nDel7?7Z)XC-Zjyel<#-CjBb-nq9
z*2ZnE#VH1-&;Gvn^y+^(oA0^Wrx|r+mUc>p{Cc3hzs$a_Fu^))3;Vhssr+y3xC=ru
zTUXa``Kvwl<3A@Tb}r=CGcENQ=K3sA5#M50eAm3Y?NtA6k%YGouU&fdV?wW>(Hjrt
z%*ziC+>%y1s(*7%&B?U<w~Z~E+-^Slmhi>r`(tUor*9OVc>Y;)_x!!klFLtx174_p
z{?xj9Th&AR+kN3Jv&CyZDv0<jnAGBUeAVh*JL05;!$j1d9KSI6r*zSszU>>nK27_%
zQ9;Rm-t&4t(+MZ`*E~FO_x=gR%#{)4eC3Is|D5n&p0A$&amlUyuV?%zir2lb&t7-p
z`;y7$)mA<!H}ZKt%lx|`*Mpcp8cTPWdY_FGJa%-)>n4tPms6|sP9AU9c<?z*bk;__
zkYu*(oonwLRY=IJYTVl@xSjJ<-h0P~rhG3wRdg=%76rX2x}(FHyegVk)M}FMRaSAe
zdxsB|*Y51A?MPqpn`!aaX9kkz%uVj5Uv#yvT&<h=R;j0A$A#Lk*|J=>YZ`+V7EG@R
zUcT~#ep45JqqEJIzg#g#Sf(FYtg_wy0?Ypujx+0ge(D*WmzJ4dcxdUxKEK#c9IJV4
zQbOGpKW-GMH!Wpr%M-oR{%gCJs*Mllp^b$XXJ2{F^?l!lZ*sScV?MR*xwv?D3PVJs
z+>D#2zuL)}I*DJ*J0J2m>7%i?@0^Nfor^ytwi_;GUZ8bw<=ogLzSm!(YUi-*xV|(i
zeZF&4MaA~@PkT2z@86K{oX6*cWOR^(@HV~OM{<wFAM&u>wr#uF)pPe1MPv46OpW;^
zGb{Jho3E_fzW84C`xlZU$8Y8NyYKkQV3|Bik!6)e35okPd^&cXH>>R0nh-y88AIMa
zcGg~}xq`}q>?%|H1M?po&suxx=iXPJE9bm+c6^-QbT#|N_YP^}P_wDkVW<E0d?@}a
zr6Se8z%Ku(Qn^uMuyx*BiQ?#`cK;I<pVpcNC*R<Z_~JEHEkD%t$!!52`%U%NcVz4;
z)_)l;^?lRb`#XF$y>$;?IqO{Y+6>W^f6~srwVh%2?sS0VgM3x7Wru#xKJjOL`Oh2c
zAFu!F)LIp6$5Wfmujce>*K8Y8xh>Bc?kbvGeX0Cp6_bnVPxH@9ZvDB`F|Rg1DY36=
zzuWbCx#?N!c~<>DYO!)HWASs*%m=f#yo(nNvQ=BMWv0~U>uyg^3h3P4Y|QYMJ@*Fp
zER##ugC0a~&U>%eU)FSAcavl4eusmr7;=tr1*xxh&h|FrNLD*mR<0j4<NSvmH@(8n
z=YIN|c58*t)31^&n)~c>RxFp@wPHp-*Yb=>FWtZV;n!1ac6XP%r`ncXJKJ+&oAwPB
zJ(kiX&QiZpYZK*+P5=6y+nKBKD^~jcMX|3RbC+1wv86^oii`_6)o1aldeiyiy$hUv
z8I|bVkALuY!y~>cS8VPV9FuL({QE~BI7mzC>#~(0%aSUKn)=OKw5~|9nEHIz)O}yq
zxMjui#aVT`=U=&~7qDqb&(c+ODjX-iJ1^n-eYSO3QGBs>w{+I8HjA4_#b2cIyZrW^
zkTF?hU+(8z_2(s@ScO)+{&%qCP2=*de;hWv;nlWl=LxjilDb<#`p7f(E7__#?FuHv
ziE~)l&;OaR)@Ik$`zg0BTvl@a^UQM!_p+_*^X?sSi}t<UE19ups;}PWU(b%6&8-#l
z(R-CL(e2}>Z6}<=<limrG~LD?_AA@^&Vx$dJr-{dO}EXkf6$fon0;oW{hNTy+=bEC
z?j7*9@tfwa`l#(xwOm}Ka7=SqSJD>l?=@y6g%{Jlr^J+6$^F>a(VaH=+eA&?OCswd
zo7^~VJ&3cl`Bd^&NX_f$hr?klFWo*B-R?ix-+JY!<lZuCJ@MJkAC*Y%SDoB+BVCjG
zp2zvivrfGH6*6(hUcn!K8h@vJD42ApSi<y^@rNHD9i(a!AKuwF@wn}qKP&X_EdG6=
z;+NeX?YYx#Z*-VfYm?S|=h^NU<*MvitMVFScsn}OMgNAKF3F3&xANkg)jtlnu6cO<
z@dAAt9%k>Q@9eIZ&3Pv8aX0>|@@E77ckE1^6)vA{D4+Uw>e154^WGPWo<8n#aLtYC
z(;uI4*S`*bXZ(I&zCJ(qn-=wH?`QbAdqqBYD7i=e{OO}gm3zEjRv-Rx=A72X?8$Lo
z+7E9!o|^k{k^N`(e`Wi2tkOB!{F`}tmGrHn98E#3@*RB~$@?@a-g67;d&j?AcyW(I
z%=2eW|9YO!PoKXt=l)Zh=<9E%huiuwU8tP@ba~Mm-SwZ}tU98g`Zi%{s9c|x^Zk42
zeRG%fh?T@&JuiFZ!d2&UTPK-?Jzn~L+dSs2$9r}1XLh?Lys_TQUG>QB?)779H>gh8
zUhp`huP@#6a;ef)z3S7l392Xc*DN)CbN3AM+L+APX<yb&-~PNL+d+O_{+36jP61C(
zW$!+C(^4+>gj$05yTIRUE%(mty_vh_+5Pv7hC8J$TB|AR{Y||0Mc#IM%Cm}$;?10!
z(`OtqtFtwW$h9$<zTP|X(xHdXch1ed=OE>wr<Hf?*y^{lZS|`?CjNGQw7SISznMyk
zPrHy!&z>L0*M1iFU9_t2;>2aPyB|4O|0*edaNl~{dk=ey&8Hd+r(||7F1WGNF(f3u
z$M<)gthkGCfB)6DWno2@C98AYjjtr>If@jXT-Z~wZC#>8Mb3q0wkq*?(w@el$*1S}
zHL6C<>t@QI{QURNk5gD@e%5lRlvt8uQ=Y-LdX>fHMg0pig^Ty@cn~q``SEP!7xTT^
z{;fY(<$vOQ{ap3S!ab9hsH<&Rmwx}4Z@P_+D$BZsI<L9PEfRTOXqZ1{-RiUaf%x6V
zd&lQi>=5~yz1#8Pv~xRL=5Jt$eZduee4ptpt9v%#XVW|;rL#|;)a|o)<Li)Hw)<~!
z&$^<Jn<U|MxZpzUyKn(cKHn{xYc9usIAgH<Ctv2Jljr0%ho~67Ge2~@Xwkg91ctlM
zR`i?yG2CH3T{!ZL_p&ug9zl`<4Ys@tJiDWdm8Sf@^7C-a<$@UsA8yO8SimbMEy91c
z^5&L9?_a*tF_`PPY}<P;9^v^mzy2R-opNNGKl`hz2c$mb$E03)ckce&-7mfOzN>xm
zch#hc|NS~AKDyVD`SjTRsGLvFi)ZLBO|`zfGg)85>f&LIFEh7Kb+-MTdcN+?1JABl
z*|2?gZk+kKec6H?f-m;Zu#enPar4p!>%DPbA|4t!+~F_%s9I?gu%ms+M5e@>D_Wh`
zZ;Meh{8W~B`|Y9CvuBh}%dBSB%{Tw{e?FUcjluW#N-bypMr~N3GLbzqcbDHSrQ$u)
zRwZupkPZ`@6!wZwviOO)j;!Tc{fwDR^)s5j?p9vrlfw0<q;tjo9aq&Dmz`XGQq1wF
zT*Md4s=I~Lu3EPzPs&{(U~lQBx^VKDM1H5^F8*)#34ZhQ(fDby$2TZy3HPG+ySN1I
z=E|9Ua_S7Ycyje788y{C2S0U|c}Z<Bd*Y#`9+_8^Gbx2>vO~bKpu0zB%|4|(X@9);
z@{^|9G+V!EZI<Es${RTOX`jN<vUV2Lv#MWXSI%^D?rY5xYFr<B*d+Y+{=Syj!kXRJ
zUb4yRZFI2Adii{@l}4S<nLS~Z4xG&zwuKA0&!jZJ%5^Z5KmF;AT;X5U5BirwjT#gg
zCDu>dpLpBcRx9q#jna)DDlG%W`_KRAdnhP9eR`ACt`os}YPWLZgunk@`y=_|*U$TI
zKAAl;`sT{0eTD1HHqY7-zvO_+gtGVzySuNdmdz0eF5CD(`txTc%ejlMK7M>skd5hR
zxX>y7w$t~E^V0R@zB$~gHGiZp(-OPx$J&X%WleYAS3PpwW7)^%liai0=AQmkaOZrT
zwAr1NHj~0XoNc|bTy=`dmA6fE^;qTRX>QS1Ox2%$<>AA978CE(TEG5sPWRuA&D~Lz
z@#~L&`1&i|SvOLM_j8er#K*|vYXWrtx#-@XcTpzm)$SiP%JcTV<(+f0_`r<ikL`5*
zlMCZtDDIcJ_*hHu{k=;GPkL_^E-t$}BjuAw@8{foJLkl|-Q%{$JonAV!kuou>=ECW
zT7OvTbL~Ug;<^o|i?5Z=s`B1dT3Hr)>T$u+cib!H<jkF{om)LID!2J@p#)!h^*6^a
zlhgK3{T=q+x;OsxZ(W<)H5<>~&eERloNuVPO8oOeuDvFDn_A_+RsCIbmoYK>@{S*7
z+gF_3YTTQ>J-gsxYf#SD*#bS+!sgZQ;Jx=MH|WabwUN>%b+&#J`*QBxin_ncE?&0R
zx_>R|N&Cxy3Mrkqq6@;`GNo7E4K=t^*q$kF|9??XmDuu{>iH|~rcS&(k)z9PwY5)>
za!#_=9oAK8y`1W8^OxqnzxUMV_Q96d>q-<VobH#NpS+s$&h`8MQ?&|K=Gm&x)@Lrd
zG=0{gGq0_->0X-Trn=$c?c&#mzk56RuBctXXSlV;zO(i7mp!o|aqCX5><^BTj$F6m
zi;1R=@2`FLUFZE@$W}6|Kj?YipWaimrT*-bzOpTD!A5q5GDG95m$_@w1DW{`Oi}DU
zm6{wk(dX<*b@nUsoXb|E#y)#E>61jv?5A@Kq^EehBuC1m3#*27-R2kE@2{|Nk(rg~
z+Id3X?CwplEZFZ5khNv8)vJ16scWY5B}$!iT9?l0NIv{(i&)U_93kuVj9nj#%4eCW
za8@nuP!c_neR$hjt=DT`w)MwopP6d6Y@ze^KOZGLUAL}F(986+d=Xhv>b~MdlXHyX
zYMwv(mn>#~IlOGTTagayM5E(A_r&e~G&gMt^~hL|E@FE;#Lm&+{#w;5QT2^+Gq06?
zx^3#!U-pLa`4#WkA~zmIFdbj|uIgZ3>M9wX_Z+7~4Wg}fq%zERT6DZpLG9V+(9ZhX
zTOxxDB=bsKt%@cuyS?P0+0vdD@gYUuDn&e!B{t~ZS>2iYbax@+>7_AkPfw~os0d8k
zs1p!2ZG*`D;^tX9?oL`3mhsN0f0}l2=DP4j!Ftn`SHv~lczlRsu@S$h<ArpaPYbPg
ze%SHB&p|J4ZA^?|uzZ8hice>k_9*!}d%MN%D@k9lqqbP}t77aS<$FH^nVmi)G3f^X
zj}c$kDD<lN%l;1!!sKT}uKq0>C2~JrTb=2!XZ?fXN0TG^>$mRZfBCn_evRFH2CLA!
z#j7(TKQn(9d(M0B`=Lw6x6iD7#q*~9>hHOqI{J3-N4c+#-+ayfHc$5Zo6<b`jL+ns
zrF`0_k^TADzBRSFe1CprT(2+j*FAOmj-`*)#R(F-&b$j{c_Jk6c6G7G952Ic?j38K
zZhTKK_@y~zX}fhpl=Khfb7#uG-rU_5Jk{dKahsGbHtwi@e|GDdUCU2@eS9vjdwSwB
zqwmR%g8Mqko6Rl#m8SF`^{;Kal4CpFwkbPH;NI@zCtMZ;{<ZwtEY;;-EpXw_4axnj
zu{K{W)p6E-jr-7jW7}7q%=2%1(;7tWRwrC-IT#!DcjH!}n+M*i1Ydo1FWK($1_SE|
zG5xBQQFhCfcd@-&P^ZJQ;KlI=p;Ob|COir%5)2KS`|!zz6#+3*xlam)6>5b_`acQ(
zB=)^=|B6i;>L)rMRJGc*f8u9$k(~6eem-*!FaN#aNv&9YMfilTCVM7?24y^{+V<|T
z#JrW~nIHdjp7?#vrin}RocD>xrY#mQdv|*Jme`bn2|;V-ajT!+s`g%jC((NU{Ou+k
z@9$sS<z4k<!=tVZ?|CA(sphj*=r>*3tX49)fzwQ1-pFUoZjEmWw@>?7w9WbL@4=VE
zeLqEU&)jxn-nX0QEfSb=f^VPqsfw1F*AM91E&QS)cXY!1wg2PKt$%W|zdGjQ76nti
z+c%&2#OdsvlNxCdDO2a@>%{jf*68@%v_Glta_TL+m)xzNeDA1g=wxX*N50U1Y{j-I
zZPTCRJ-`1=uIQ%1UB$-Cv(uy$Q{#-LpZTLC6?J&UJmG^Ue(tI5VY;zhQ)RxvGv|5h
zWiHj6zvv#8+bfv+@o|^Y>DVO|n|JTJU${$Y|NWI7JLR7onHXKwJ72*$^60*6oBs!z
z@h@li@@~t)j`^#--q-2;<Z}qV?jf-Jy5qU!A07rpUg%C#)XY6^$@1lgz|)B{Pi&Uy
zx4&O@_Hy+xr;IDhn4b5BUpRF7%#Q7cbQS+s?VQtVHK(XQgmK<q&HokqOfr2sKeOCA
z%zJ>}d*RCG{|?>ctXyh7vG=|okJSb9gS^FCwlVa|EWNZ&b<yT`v+sy~54?RYd`oQq
z2I=c_j+un>EwwTG;=VjUU+2*D;MLcwO+7<%@ArqZ-smnq|IIjYrBGc~)w>O1Yq#cF
zA4%HEZG3yj2C40u*FQ)fsQl~B`HH<#<~*D3y>*`6x5M*)XHIX<iE;cK-S6V+Xllik
zwY0X?F+=?K%bVMOZ8P?ocHQJ7ONZIVmB(kEn)mbm{yB5rPhNcXtg6F0ONqnh`3{Tj
zUfLkAyyBbMY}V%)P7TNF!nRy_xljJ|&kJ|H*9B&M>G^(f_D$oOOL7)X0f&rVT;O|d
z#IUQUWZjI;k41l5o&Ve4YIqt_H$hYG-F$WNH+%e_#)Y>XJMDO-?f2@>KBxDp`EQO*
z<=^>K!Tc%L_Qgj_p4^P=SjL@YTVH$WnQ%|0wA><#s_+x_wtkEq*ZilOe&O^wJo`ZC
z=MN_u&WS&sqkVSb;jopittGD`d*4@ZW<J_pv??(!Na|LMqh37A&7jEjL6Z;KyX`Tp
z)4pu7eNVyhPk-hW*Yh4M|2JX(yhFR||DR;Ll4Ykl^#|L;3pdOToKC-U`liB5l}mi4
z1?zus8P5N__|eOvt46sTbvkkzw<z&3RJ`}skzS@3!1dhKYU&ct6<dAZ&ybgroaGg*
zqWJX67m?4?%IoC6CQhvO2`uo8j7{}lVD(1o<<={1%O>UKPgpLaAv~X3o;Q2?E2p$q
zM=}qWwCT+$tWGtb@4WV7Y?l0y6D7=PGne%lJ>Ds?`s(Rv`jyvI&xz_<yyJS<`bKl+
zf2D6TpIA(eNOwHC)o}X5!nc>c$XjpU>vDB|*VX8?PM-POe#m-hf19BBLt$~wq+Xv*
za}Q;+`9*&0o-x<>_=IVJVsr9Vg*ZyTRh2U~WT;f(nEh+}vfpkVE3~#}2Y=3+l$J70
zw8TwRY5JaoRQJ?W&O+Pg3iocGQa#)CG4JIj%_9r%N#5eTW|?uV`pq-TUqVw)JNGKZ
z_#Zo6`S;=6B1P|YUnCdLVZHBpdYa7f$0{oVcLcabyQoPoI=%3Zbm`LliRIIbJhjv;
zkBPP2KXT)A#m>uy*$+QeDz3frac1yjwFYxel{ZR@98PZ!%iaFXXaA2+ECOkpi*LHW
z+hh_{9M(D6?%kb5nv-=N9*<R3+IC`QnQaKC@+1qs_<v&8>zCBpvw6&$rO18yzVrRN
zkJ@T7vRC$;kI(Kmce;ONxy~m>p%lK<PlgqrJEHyKr!Bk{%fb_Xgmt>p{Txrf_p37=
z96x?F<C&27InAe^J}$8?^yKwfQrGvR=7`?Kq(tE^r)iV=9-oWTJ+_$fg!iNJjVIpO
zUywI^d+sRjslR?71on!U)!yBw@SXF?zJiN;q6^thT=@Cf_gq6{`sck<{urAbti5l}
zU%cBUW{ETZ`-#tTRQmp%c6;-7U;TcwpErfgjqYnVXk0oNIxonmucEL%;n~GPJ#&+z
zJVDBjgC=@}K0dz2$Rg*C7l+`qqEmL2ojd>SC=Su{TsYOZD6&anGlPJV-exb8RwIM*
z4XppRYHvNx-SzE9^W+VIw^!I)UfgQGssHZtfN+zzyjdsTPMkW?hi!`d33a*mRs4|?
zZ*ea9F#m(*Zl>rXe-DU9OsW2`NJ33d$bM5+&8@geN92E8<lZ*_&piFi=x19l&%Ll?
ziO6&f#%CcuJQC**81$R>vA*~JU%!BHrb%L8!L{$9O<!Y$>|3kLpYGWEbF*b*E}x0n
zoYubj^{exx6-sN?c)9MdI2k)B|4L`wynj<!ZBDgthaKQJ({}60`HdRi4!ZPB5iMQ6
z^V1HoDmBYRmX{RGZ?*So{+z?Srdj;st`7k#S6sZCS-#|UTE^SeA%Q$z-b?TLEjjo3
zUg!^-b5`d!6t%B<+YtQtntEjWswSWC<4rpcRJC7-XYM<-^y=?Pak=depKVvmjdXd;
zbgR~>w>v-P$LgnhZ&%J-kohCi;HCWS15=q-7J6q^aa>Tbt8$sZ{pZlh_HWIp#}CP>
zNTkQ7dUJ67sbt{Ub?_s{y{A4qB$VWWB2=r^>aW?~F>e#s$7MYk7dL%g>i=J(B;A$2
z^_=zF#-ndiKhCjvRrfe!u7HAU>d$KRP_DC;^J8A^`@c+h%U_v{6&!2Wf3I_w3cFT$
ztJXia_wM!@bxo7hJ({m~|69Gs(WHG%x8E0?M71~O=?{(@8~!f4!pYX1ZMl0(I)m0_
zzyCSYKFHp_BNe#VKH^Slf2KUgCQcu@AGhCb7PNmGtWlTuLbzeYj(GE3h6j!vs(pIp
z+xlMTn5s)ldCISKg=_UhNE=RxKkz}mQ{w#9B`Z_znVY>6X@AzizG~OM>~9UckLQM6
zTvo`fUbSw{k1`i2yENXHpZBe?_*z#J)$M)MsBVFa-dX=OuNK6vUw6TO*Mj*a@k>Om
zg`WC7*?p;*-OBkp1+QGt51$wHp>%@mg(WBcNHyQn&;A!%_TwYZCnm9wxAqA$YW8MN
zO3D$8)zP<bI<v2H_x6LgZZqrWKRUnm%*BcBJ9{<e$X^goyLR@2aBaf+&0X~YdNN%f
z6<RA3W^HG`e6eonk+-uVBBEy`Yl}Z*E%^2SM)hVB_oA50FR|GdnHkUhzJG&vmgL+e
zPr7>6tb4WY^Zh@otZkp1o2k$tsjqv>@8%=1pKZcFUjL4|CHbQ<XxZH=1Fv~sxxY{9
zw~hK;7k#gBTVRsA<Mq4;77a;qtIC?@ckbC9cJKVzYW`buzB!$~BmVBTo5suVyN~{>
zoT_^@v!%;vo9)HKdiL~v{oGO-UAsS~zP+E2@><`<CuKu`<+eMeJ7rxrvUa^SxvR-%
z_T$>DNuMtM&d`=qYc=aEER>eZJGVvu=V@~hzd+_gfB!itH6QpDI_tpVqdRm=uOCg2
z=Fi^uSN7^(mC5FxF1gO1wWR#|gf=0y{A2tXjxKB4cPf3~#CI~(ezNpC{W)v8d%p0O
zeB;h{<`xt`c<M-$tIx$R>L07ko@A-SmMO=~D?7$juAl!k|Hg(r6YD3suf8+AcJ`#Z
zEzf8Ct~kP~5chk_r2P)e@AQ`()Ia|9no~@{`;Y8buExKAC9mh;`{B&>T@9^P=e32`
zEYRBbxbA*NzDVAlKMVftniBk8?TEd|+PwlV>^Q$)b5Y{zKT<PqUUuEnlA8{v|2f4}
zRk6Rx<`Bu>pHUNB)vn<Gap_f7i9H|F{Q@pNUzUA5uS9hJb|10n!E(FA*1nb&e52H3
zKJ{SRXR}2&%$t5Be_!xUDEI#5?~7R&cT8P*M5{H*XTro!4OgymNAI$Iu7Bcm{tj*@
z2j>O-ywz8}%*k5%G*{5htkg8}^ZL_w_O7YBy82sX><Z6cb9L9vfByfP$J*_w+0o2Q
ziGLS;tbO=JZfksM25)`b(OlEUmgVvF=Kn2DEW4VKos;|4xjO#cZPm=@S^_t}oS(Zb
zzWG^h{M~)>{_(<ZpL?C+c^k)lU3^{3X_3Pd*Bjry{5Is@L+-|o%?H<|mfSz@`tJRO
zFY9+7`uCY>uk7ZO2#uSI_d5t2y>KX8TG#jCC-Vnxou8K3FUzp>w5WN!Ri-6cv(k6M
zg(r7gVvZYMn`~w!9T{Bwdz*s#X$7n5bKe#O-l*KCx#ll##nKz*Cuan4?wX<0lrg)_
zkn5kJ$%mPTrX6m0k+rs_Z})!B&pV4k(#ub_Og^&9^U(tptwoytZ%vpsl;1g26XNMz
zS?D*3(|UTB(RG!DYaNb0-ue5wz~4KyrxNxT#x1t_c5d0l+t=on8RuP)GUwQusVTwQ
z^3i+)zw2txMc4R~L!Zpseyqm3*74)(_Xh$30$MAIP4=_5e);+6f28))o=N@-U%g-C
z<`p75xivp<O3txQHH9A+ZYiDQ{=NE3<@`wpY&q{tUgt7>E1T+i=?_agS<W4uWcbA=
z?3dlgwGVzvXd3tm&A8|APT%~!J8Qz;pW3sNPNnW|wD2mLJV_?Ma8c23p4b0x9{B%|
zNo3=nL$V)ZPQ3iB^-r6_I;iHN%ESzzNFR}3kG6RGu6;E9li$KuW-lYZzBv*Wxlqc@
zLnnQQv&n%kGIK*dU*Ei9-sZW7(yZ$h9aJMP2-a*f{3AD|vPXGAj`+mPRHei7UEI98
zylg!67|$B^$m?mmfBk<VpYBxtIZF)L%)Y+JS-pAfImLr47L)uQT>d&`fo4HO52sk=
z(a(`zbE;N31~)HbJA1WZrK>lT<I6iss${Ly<b^Ai8|uI9XwB|*-{tJL>soH4+tU?^
zuBJk!=e-v!3v}(($_m+dj?HAvio~T$e>@E4RWqG&CE-+-!pfsdDz#Y6e^zj+Crvn7
zHsNs9gxl*1ZfxCjZi$z1;L2^4#=iQ(e!kj?I&nL)T(g@UC8r9yl`|gsCClG%yF>d~
z)V81hSEL`?yt!oi>c4^Fo0y&L7juTKT@!gTIyHalS83~NlW*4|mTmM2me;u)xn#3P
z`b78Zog7kW7Yv-QZGHjr)~0*S6ILH+DRg5qVO(_b(fJqtZOixVp7z>mviO{{t=?6u
zoUU*zauayYaY<<Ahm{kTRIR<;kjq$iDx1IYGLz!M&I>_ZUR#Q$2YeFlVp~7;sFlK7
z3C#w-N->K~K8&3!l0Iw+o%SizOd$4TNxSRRd(AW63tfJnxarl`-;8y}_UGB#j@qvb
zT`_-o;0@2EA?q$(XEK_=_q+Db69&b=ms>Pm%cL&%Sag|dO|Zg+ROM?&Wm5~?BcG{9
z&hZy3pK|m;im#>ct`CQvY+~A{+8uu~YO~Hgh3=4hU)YK#o)AgV?D_O7_Db>0BzGeL
zevU1x+^%TWpP1!w^<Jvx>SrylCX4*Yj>{CCR~z@<vE-VkdsrnyYG%UI*&<I*6hB{T
z7+taZ%kDRycBLg3Te&{wxf7|y`0Q7x=k3iEtF`WV?fca$eUC$Woo+!c@2}o{+x7>2
zv^(zJ>l?aPtGrpOy-{n`POq1N>Wx9J3vvqLCr#U2JW26YO@V$%-KH;&dprUZ#C&gU
z=@eEwWXgQ>=jrobpM0@;9aK8UH7xaJhj?du_ouUUe*;-pgjVZ+G}~Gi`fGt#;mjhn
z<fBPfxz|q0o?4S9^ZZie`uDdZR*I!xySM9Y`qW7-Cv6!rqoi~j)6{>yzO`j1m%8cB
zn?`>-GHU;>e|2HmvcUAjLu-0ZN-aJ4R6E<O^MTK~Wm$%e**p16npRky4PAI>)r@H`
z&qd5Hoy{sabC=ZA*<Z7~{e$Hln<p6yv!01Orkees>p`Jue(?9zvYwId#@RiYew#Sf
zuoh;W7d@H#h=ce0#$(dwT`&E3X7TLH75iS(S3a4h6VI>iZ#^a?`Z4|8jhro0wp(%Y
zKY6V=d!5gnK+V`^Z++IBwKQFua6Mq|REeMa?bP`~mtRQ^w=4@?`e|j-#F8x=9JK6x
z4K{}vs4Zljx0Jc$<Ut=_ssHo*_i?{|e`V{`g<K{k_Oq+?uW4n@+a=n{c=nm)hQCwJ
z?P=Y`7#qIn+OiyrZ2jZ=Stl{E&es;$!sRF{<a}+p!veoEIvvKB3%+?L9y`&?ZCr6<
zqhB4DXGh#i;Rluns`ak6^vBKXzb@_|Z*}IJV2N(kmAxvgiyq~K<lg47bZ<Oxsqx(I
zJtvN3PU1+73QV2b<khs?YjIe~qUZ0z9v;&wo?*3qQswy&LGI9RN}=0JuIm1Enl0!&
z{i}|WLQHt|g*B7JJQ4%tKKK886C-BZA!`&`8=GzTa@q8=Gr14(&Jz9eV&0)Gow24N
zwvlzuo?V&qy=wohzmXE%N(a_=OWa6I`ae6PcHQ~1*?+I8wuVji;EFOix%Ks;Gn<ZV
zDSj`i9DP_I^=E*{kyY7_M{0yNgqlwHvoqt!*Lpz~E1y$b=5_B>&dRvYvpD}OQn_(P
z>tnUMHh<S~?QHft8ZEAr<@9~=uOntkFMNL7c6l=6g!{$Srm{=r)Nbf>2Wn1!_aO4k
zWF_I7hR5%!iEa^2c3tLQ{C#WFr?l_PO6raWjY2K9tZS6{?ayP*o1?aDcE}8cHS2c2
z+sIWE9rNJ1ouav|xY4YS_ex8@o}8r<F>}JL5MB1UMr$+XYN&o#_o&zMF4vQjX5vc4
zIjtw+E1C^Ia(!P^(8#rYaY5~~Pt%rv2-&aE^Wh`M%_Vg%mu-El&#Yd`p&Q<6GehOz
zlPRn69?rB|bS^?SG<b4w;Ton>oVP<Cy<4RmuKHjPr}nqU!uh)_Io`HTo6BN2cax{>
zYrDPsUdPSdzyJMFHKzG22?7VxK39s(5Vba6zI)Zogd;!pZ7LI;dgAW7bm8j@ekqB4
z`@btiyfq-qMIkhH!lT)0?+tFQcwCvf@l4R%P{!9wo<BPpJHKm>#cOj;X`eNM=38Fu
znzJUjvP)I)xf0u-S+TF9@Bdx7db{7H3+ul=kkp@Sr}Jw^e7VjI@mQT_y&8L8U3z&*
zw5;%AY(&Gp&Ek8_pZ80i3)T_3-|DLJFC!<8LHE^y^PeBSGC32bet%KyrO(Hw-wA!N
zby-BX@CFk<Hy(qJ&$gC)UO9Ju(DIXCI_yRD|IV<MRx=cgulwy3G`GSdB=!D>>it^(
zUOa#N^!wZ7zAx_D>Q;U>|L5%snPJiC^z_7fv**9BfBLNP`EHxH-YmaWx$`y@eN5Ne
zIqUY6nvRSK5eZxeT@C~vnbbOYbxT4hOZ?QS|DVL?F}3`cSo*K7(UVC=qxe<Ymk@=u
zp01^a+b4(56U_S+ee~*qcK*+n)q2kE86EClBxJ;R+3J$#InUsYw00IUcyMaJ_tGEB
z55(&j_+5@n65u=XY|mPbv*-0SdMC)cdc9c2nO<>k=L?(57Zo0#Q@JpobLypx37bCf
zzxlF1t##iqpH=^|7sNN~Nqq60@p669CN25apU=Z9z3yu53vT-0eCE&c1N#>rxGrb*
zW?{@WDYK2e->)1jdy{+o^}m+&W~G}Q?c!|I+m^L)mDeO6Q%#=pb3OYS!6lbMP33G4
z-`uQGYqi?7uItAXkuND#Q#7{=@B7%%d;Eus#lNZ^f0h*ORbfBYK9Fv_#<A6H!{4eN
z{}}Gs|FUCv4;RfSxfCPtxJzQ%k)0~*R10G#MF>5gv}00$<`+AtP@=L<r~ApxN3XrQ
zbyQuiGuv~H_oE1*>1=)8kCJX|R5_<wsM}fWDdYWU(!65Nm`x5koyDFp-j8C0j!*LN
zeIFzAe3Hd97d4?kZ=qJ-jVcqpg?6f(auWcvwt~2w3tJcM1ThdoVAf8RNr^5xo=+yZ
z=z<6^ORw|DVnL7zmrolOc`8j)De_dB4kD(h6nahq5kg>Mx=NuZn1w^^W!j-hDtexe
zKm?d|3C=<l<FrR8OVRVZHYr8V)9vgT)sr!k&ZwS@01;6j;)Q*&r_pqkVo#%KD#f0g
zK!j0#cI@?-No%YHy(jHdiS(Y7uHaYqvnr5XbU{Ji%tOCFzJ7iBdQp$RPmhI5>|@~#
z!EJnImz}gHZ<g7=Z{5uXuhwm5t+t{H>zRrt9jfSYer>u>VxNV!=IN%-!U74MD~=RI
zwXQ3E94@)VSZiW~<ffL7nh%5BRI@bNHYC=Z{1Ft<x#*hWX09{#EGI)Jt7^rD9uwDb
zdFUjf2O@M_4mwSlX?Z;{D){5PC2R~!KSe)X6r$&KbWw<o*V9EI$zRrYh5mdbdP`J%
z#iS>zo0ZlSo)LB{&b4{)I+A^D_(p*Z9+?#-E%j}F$u$d9wx@+k<^&w|dgPZ~QQ~r;
zJ^#dyfS(IjoqF~_{6iJbmc#|Us?xI?%p+PSEDvHal#`v<;!sgnn~_p?Y;p1?{vGu_
z`dc1I_J3?WzWm`26S?`vGM1iwx#~tk+5U<je)r!wZJ+#6^%3_TQK`uuTTOa5UjAXN
zB<;}tf_=X6{*;=7zqS<C9BbI&kgF4Ln8{Qx;I}qkc0Jc0%@4~D)UQ5}s`cMs{<q_w
zukQZ-BYAJszw+>pr(Zr@z4qt2n5FyP{gh;1X}`RO_sb@$*)^8E%|D+{;j-Pw(fIt{
z!_`xN*>7;+u-sX_=&wtgNz0ALVSlR*uvmR|?9S>p>2TB7<*1h#DpynUU!P;$ZZ2op
zDRqHA-EIG@{ILAce6D&e5*YtCaTNxuFZr{F-}w)JmiND{tZnyR1S@T;KdbP+h%w{*
zzvs;-{y%n%_!-LiUtjm=|4j|Q>+c-dzpml_@<+d}dVPtX`s2Fqm-|r(2SWaYax)*7
zvv_cMo$WH$3kOd8R=fFDA!pUkx8EIa@AIztXlv!YU+1H_@*eK`o2!pMe_xtZE`B$k
zr}=-?ZY8sW+m)ui&QGhl)-QQ9^Uqojzw6m4&&w*_p8eis-LAB6)|=IWEOoQ@YT3wk
zP02OLH9Z%oKC?|F_H!ra|84*6XGEs6Y;Qb%-|RbMc<jQP30ii0%2F>G_2_<dfAV0t
zrp=rE!iP&+SIjudcKI1g^2wCP`M35gvpW+w^<v2vA*S4g&X2W!eeGu}o_tmIbSmE&
ziJ$LQ`#4#9yvdO$)q2lcJf$Rg9xuQA$;5xsiAOD>j~#F5fBc@~bNZGIZ7ma$<5}f4
zKYgGRw7yE_YYg+`nRRWsg+UFyKkH{GUGM*_EmeDHSKq14N6v0KFBcp+mwi{pp%t8Z
zSDLa<Uw`4g^WL>Zp|kwbb!RZ$ULca3x^Vx-hRWL)_t&}X*L`r<tFmGZZ}GiKwV(IT
zOG`dK*tl@nNfD)g8wHv)eRK@dy3%83dMBjbzNozC`LB0X>{kl@KD{2_U-5V5y8U4?
zk2Y~#dwss_@73_zt7o_8-pKfxvRd6c`l0TNkFU;#<zL%&Xti^^>@Ty8xoP{XzFb|n
z{$I`Px$F6AOJA(szr)vH9pB?u2J8MkGINw)_SRBZ^w8OdT3dhh+*Wz~%4DtI_Ah5L
zXQ$lymR~ySn#yIflUsa0$z1(zAOHLVf7rR(PEWQ9r0TjvatCf})yP&n71gn+@zXZ2
zknFUp@psF1uj70DYQwsJueRMTEsY3oFZ~+6o%7n&#Aw-VSL1K1-3V*XExi#|pF2C@
zw%_Yl8`l2&Wp;3%)tl9a_x*Z(o8|Yb4eR-SzgnL!{KmTJhCtvA1D6}mEb`jFD(2Sb
zM<`9%d#Fvr>Gr>SRd$7!uYK*CTlS^o!E#y0w-Q<If$V*^CQQkVI921`{d<YIORDs;
zO}wEu`&Q>ThiyyL{@N<w`-l6YB2PWvewjax>bDO{t=TX4*}vJp{7HP{mzlhCrrx|)
zpz>h8@+tR&@oNvBFR3{2AZx;P_Wv$=jDJGsPH*8gI&ZS)-RJzEUYV?isfiu?m)I{q
zuIPTv<HYUBeJvW+`4cMt7Y5v|Y?g@mY9X@xwL-Vj-#<~3Gn}}W$O`^sz4H1v!|Siy
zALF*gm;L#)SI$W*NB;6hZTbDDA~(v(-~RYoqPKPNBH6=+_I0w0H<pCiv?V|NQQC9t
zR^&&CKKbs)$20!K$k?|%KK|&3p_IM&@&1QDHd@s6sLksZ?tgeBx$u$UtO`9pn>OM8
zqn*by9__UFDB))%rY^hK^ZBDC$rdjo=9KiP&F!At_i#ya;U&XaHF|z_ZIk<to;;rM
zX{W_c2|qhAb@|07pFet1Yw_hgYn@_fde8lb!W$3jmj8)mdTE_JM{i+=n8z!{7n>$u
zV^)6V6n63Xe2$4)XRdO0+Ev8u{gb*lL?li2qT`41ntz|p?~jn!Q>oH6uio+#-?2{N
z#gnhe>}K8`rPrKW!SgU|T|B=_XpH>U$dt=<e}8}6E&9*o!}3OZ-aqp*6Bf*Ms*jGY
z|Mg4w6Vu<h*XCFLjofyq^zYqk^J`zHyq-|h{4??O<!yGlZ$-D|<-WeYt#0jG(QWH$
zU*GzE^4;dIYqRg~`nERxe)^l}>-$RIMxQU7Uvg(xU-b3D*Ou=lgK4Wg+vByD3SV30
z#x7s`e2c%;+v+`~bGOaAZ1-|Y>D+DWE?+CYr#^T4y3f~2?^T`qy#z$>Iv0Jua(>CZ
zU3$^iQ(y0tvHn{mbjobXCf=nv%9T@cHBzO$Zngz&n`raOY|1X)rFqUP3)gQbjorTP
z^|j)))vaq^?<&2!ZQku`pBLxGZeRO)+t)kaJVEsQ4{M)q^WT*p`+e=}eP8c<_XN?U
zcel^`ZTE6t@^t3CZ#Z<{TwZqP;jb-=XH57e|8vEb<e3{5&%E$W{`Z$$_o6a|oy@1-
zX1q(ZUbLC{qU?-3_fr1b6X(7?ajq;Uy-w#y--jIcifzd@pOY2KmZV!>+ARBerabpX
zWxLyg^N%v$1u^Y!Pn`eu#`#AZ1+1C2$THsQW7L)9-l**8eSGQWenk+|>9*j)w=*tW
z+9+Ucv_)3(Rv)h}NTbVb!9{N+E`Hi5U>&qYR`XV$=vo=>jmmBi)qEit)_M@tZynmV
zJ$8nu=9`jX?YBi%Gs?E3Az86(Plk2U7Fpw4jZ(SCd$%dO-`==b_Ql1rKN;3df9yT@
z|7AY#XRep}vE9T#U}_W3&tJxC?d4NJX<=LFmbK3gZ`-x)>ejWJ@~6u0yZQ}8*RKB4
z5&d>YXkYZxh;LlcZ?}Z{{qFg$xqNNeEv@Hki$L*ut8(?6?Wa=bm)y*nzP9M5*7Vv-
zrK@ANpDJA)^WF3M+OpeP+t(J|R=@kL_ny}GwMF;U?{2?UyZR2l!Oe7Erk}sIID6f2
zRDEN-o%u>x#AfDeWeLJJs}4SATvN?$ePHig-rJ5B$|SZ__++;`-9C7s%mU1Gx$Std
zEMiNEPj<W8?SmJ~Dz?bV$!~LB@>U_!+9%4guh8<`$9sqPrib;n-cY<;c43RG*;+a7
zjn2#8KFGALiLw?;Y7e;Wc%@8YYn{#!zNOdrTedttDC;G^uX*MBgjd!xcF9`gvIpNk
zc%|InD@0g6<KuyD>kzvdcJKQKFO{$OQo-<xZ`!`*rSAh?zKrOQ)v0CozQ6F&=Zyl^
z5x@ARJ<PoR`|ES%wfj&1+Pv02ceRC+_genw(A%$Wh5dY3y*)bhzE<^3?VPQT!z}Zr
zJ`VdSxcye)>N(p^y`1qe%sO}K<uJ?llg~xJ-4l8)`sp73T{&A%hwXgRy%)rK9kw%X
z>TA1~yRNQ_e!49*@As11VY}Z-?+tzTd&%#x-T700hwXenxlQ&V(=6G5LwuSGWu}&S
z+;%+67jQfAZ6f=Yw-3J6Pmud_`M`X(e<mN^XFj-}8T51cr}?Mqwf?X8G@tn|l)dr~
zi1EMs)PB$6=^EnCpIv^rN@cH<-OA6g*LI7n56>0*T(tLfm3heRs_^B)UzOuxuiVVu
zH*0nH`opau@7KkxpS9O#zR?X0pY44QvkzRi^Oa<(;R_LqH2$(JJ}&drf}Z_{pIY%>
zT;jWb%fr3})As+H!{>hRx<S;`1$G9kD)*P~vng(ye<;Vg<>?G#8?h_jH0pR$&lb9v
zMxKqC{3q)9OE-~gD#1JEEIwp<^5IsS`?t)Fzj?iN?Fz=?rfq5x*H34c?OZoKyJ+p2
z?bojy+p=!^^LMf9*LU(p{`*>XHYbB=-SzCO_}Ar?4Qp45st7&2ot;)#{qnX}j&984
z?!>9FlULuqZEE@XYnku5DPPOZ>dMAtU6;6Iw#C#u+V_a*_E~!Gw^f|YJ@D>p8S|N#
z>#wtmoNEjA-hEJ%vTN>o@rYYb9mTh=J)OEY?y~mn+qtiIt-pTz%df9yYhSiLFS+D>
z{mrYY?FUmL^RHjgS-U-xdEVO2t;Ji{Zol0q9XxOS_O;c!WajH`Emka<cDXHVT61Dt
zv2>gD_1CQ-MREDt_jbSh<a6daOZ5E8)CWdJr|TXref`zUK0bf@Pu_@`>#zU$b?uYg
ze$%&mo`qk#z3HsJe)!3x<3{JV-~RLF(^8%6s=~c7mtRL;%r2|d-lcZ^ch&BLsZsgc
zch)Mde*M*Sd)U0oGdH&}x2?Z^{mKM~HF4dwYd7TP#uPhiz5Q9{tJiiqdCR)B+NpbW
zk9VxU{(9eUrkAhZ&boIZS;t^qZS;emn{qd=&bGg8y4$cVvh4Qu4YzV%Kj_Wz`uMAu
z*JAy3?&U2{rEXt)v(Mnlo=dk(cdi#M+<Rx6UQVib#I~>7-cSFTwfHuheeCw!`~G*o
zRh`w-^S?XqT-)>4yvcdKIh#M9K3uLGdg+?e?eCAZs?X+4s0e+%U2XcUs@;7t-@ca3
zj+sC2=)M&q4Yz7`pUq{sx+~@RqN%4jZyv6bz4G?&tk@gf`AZ^p-<e)JQS+fvan16e
zoa$O%gYR$O&U)~#Y+_<+b@iOxUw_R?xWk<<%=6n|9Vg$e_f@-}UH?{?bD>h!=zfN%
z+x>vd``Zli)^E4fxxGXDVA9sy_1l?mY;V+Gl**H`_ujLa^Dful5x>b4ZxQ<CufVUc
zs@dtW>&*|PzHoN^_RDSOm$eh;mfYlu->j2cy*v0yY4!dI8S&ivF6(iX9{Mi5A#dq-
z?hD&5N&CiLd)-^(d3lTah1P;e>7~_s)gljnt5Qz-^sVZ)_m`cj+4tLx;@_8?v72V<
z|7PQa!)_;gM7lox`|f%&q5KxBv3235d+RG_<*xtEy)pOw^^<#&cz*bu={d9P`tO=;
zwt=lqod<aSaD^1GN;J;j6SbpD$gAYc^XTJkXCKD6Ud~d8cl}%yxo*+Sg(b)8OZT<7
z+x_3uG|wh!f84j9MRT4?MkdY<-u|`9m+NF~`k&^|3Kf?<n>yBJKK)vDR$Jff>#S?X
zcCCxl+^Q{J+^eu|dv;Q@+^)FzVCM^pvae0IZi&0KF-o{QMs=b>@uJr3HQVCycBj66
z5_ef@`}fc?-yDWpU#rgMq^u1&!gW9G+Tp6&vpl~isEfq${x<E{@$Jknt=;c$eye&b
zxybM3Tf5y`;`9@~?7EkCIoLyc&${1Y!u&@}EyJIv?}?k79e#65?(Fs3uRoczc6xRm
zNAl9`*RTA#_G;G5qoy0SnC0Dh`}ysxnXyyKqHd?%zMXq}!s^I}7t9)TuHW9HJYieh
z`rhUmpZx8;+R@V4j~{LQT6uO|(eICCzVEt^h+m$4{m!wSK~pC0F<ZNK`t3VyA2RmE
z6-zIRt%}~3JG;&+@yDCk6%VCyZ%%mp=<}&rH<CV|+Ll|rnOi}eJ8)a9^lOXLalz6q
z_j&FInRUo;Zh8JXJ8$2KWxHgz7kaM9xw~zxZMw#cUA}pOOr;Nn>d(C}DD_>JpMQN_
z@0X3A-<s`g-1q#o>CVEq=eJFFI{x(tid?@_QC{ic_FRR?ZP%~gd9=v?7Td%kCg$|*
zbGQ90y1lK@@Y!o#-)~>b4l6&FIDfq~-iXunHru4bKeG4jTV^)R=i;7Uvu-)me13at
zYoX!!?b%s{3%}N#<rU0HyJ@;}$A<j<&&00pe3PB^_0*f2*XQxZSloYSJek+n<ye)q
zLRFg-_w8l8ZcnTBKQnVuk6XXJ_H|<Qe!n@#-+ndQ@B2&S+RVdxXD?-UMf`THjPqEO
zxBh##Ld;Je_S$08dDmXgwVG*He`LS7*1h=r**i}5n|(@o^MmK$wu|b8leKlZRP!!%
z_PRMr9Fg3R8>rbF8MWU0*XrE0Ya(xT{oQ_1vpHgSoc@)}-0Hm#r#rrhdH>jQ($A{h
z7mx2P-y6SM^82;zn_v9?ojdQh#C03<)?<qzbE<a>>NZF1h>O2`CbxQbn&q1G_1CYf
zoW6IY?DyN;nBbh*UCE21*f^{A?o;bv`SLlp`sTNvb+aFa)hNe0G{wFb-}B4tU2n}n
z*^<5S(VepM)|WrZd0(2FdSlssLBqmWmucIilV@LaJI-TQxA(q7ne-#KlMJhtv6RP3
z7gq1S|GirIy3OTXDLIQ%lWs`elvtD-Xja75u#@G?<}crhZu{Q8zy5aeZ10=-$F}T^
zpMCT8x2oAo@_VOGe!e?yeWVkkIqTKyI;a1Y?RKlF-dphIz=_PxS=T)dsXA?Ry}hLL
zXyo0@OIqpE88Sb+*{r*sJu6xJvyl1u$CtNn&z|&nM$CF~;T`+$s_oof=)39d-&_&D
zdvWsCGu6-UNS3>OPk&z0-N)MkEj9Djub1RgnLGW=Ppv(7r%ErJd^@aa_M<OyTmmt%
z>1*=VpRbL1FaF8#qiSW`g}K|lXDjc2wmB}}#qsX!NE83t@#{O;D%Y}m%wNB7$DixW
zN6u#-eEsd;tc+Xlp7(x|EL@i^anbcxL*Dt|usyxiF+V0>k3TH;{cp|eKGU6X`G?QF
zerxus@IEMgzI*+)&-+%@-nYK{@4ZXuY5!WaTP$bqUBQz*2i$UMV-Mz5?~SdyeY;h6
z`}FYZ%3-(cxCFO-l{liaD{Xt&w*1MvuZhMy`uZy>V#3nTxi`7i-;L(xUR1I#H~Qh$
z>g+_v$$o<8X5HDCQx_VexTM{sN;J+lrnx&?ppN(LMjtn(y=`q0!u31#R3rC3IPJVR
zpu?F}=gR*0|NcGyCTufcdwkj1Q2fS{)n%PoR*Np{hfI*rSGst$;LCCU_z5BI0n1Hf
zcDH?G(oUGu*YKEU*1v!$Ck}CM-Ta+n_v~}$Bxly_DLS!NhP~v^fp*7dKPN9Z@u6Ps
z%IWL-yEuMm%qhIy{P{qGHQV{6Cc4Q@v(@V|x_(SQ9TF;{eWFc5^U=W*xBE_C{cYkc
zb%(oM+|`sN;xofBp%dNf-H+`qJbEZt+fV%z&&|lRo^OoI{Na~xrOo%(Q~CdNL)Wgy
zLK;>}m9l%}Hf%c(IhA9X(BF_ndj${dRn`$%(d(78<Wt@1J%6?In@&|Pa?*O7Fj=Vc
zfQK1tzs<{yA72Q!a0D{#zAnE1_a3PQ^W}6_I?PlLdekZ7pmcP8N`&g<)&F-~cATKz
z!FRb>uKwX}?&-Y;7o6XpyK24vteGd7aue47IU+5ib#sepW5Pq`_Zi2+c8JX23_ROx
zmaqADs_8DbnGAbZo^@fJxT@X5w3pBFS_eZ*#MJ$-RLVXb|1To)_tnZ3+qlwKTWsUs
z{Xu}yfjz|eS@O2e^Lb+o(?nywEnKmQOMSJ)CN7252R3#|tPa@NCHOm_Ai}jZ)FR?&
zW9Xl7*1FIm;jLA*8pl>Q?bcegnr(XP#2Y<K+qIHj=bzv5VqQXW^MvF6+rlhXoN^59
zTUq2YHF;IkqElsBw^!yY+cfi3n$+yfoMoAvv)=J66gRtk!zbCe_pHgne;+o_6uUQR
zM+}$V^h2|^Ft~=r_=<)HAH5n<*rS!Vtm~BMRC&>@+F9Wtc`NP&>|9vn{Iu;<bEfBq
z6TTcrrX{SJB2c4yP}yAZUX+tYN8rN6W$thLPRUNS)+&!%u`7^y{lx>`rW0H^_k=R9
zzj~nB^g|Nwo=E3l*KNk@)65Q^zFA`Vy6oE5GrxZ(er|gvd!+lAhsccAUE&9iq|I4v
zxFoPsZEM8_$#mm$X-cy@bJYBT8sEC`sc)Kj0>a-kbIDQ99Q8sAPZjpro|X3&KDx2}
z_ow&A3m$j*^Kdt^w+C5FQTpZ1?W1AZv{KNgw;+opM0tsn<JDDSGcII>wusDD+#|(!
zjbo$Rgssb1)vib#yvDKZn#;CLl1}RcOV(yLa_)8eVA`}^u&$ROrzJs|BU=26VAtG$
zH!Ml&0@jT;IFI-#lrb58dGq7uj|5}2NmI_wIFVrHtiblMV}2{YnX^J!p!cF2kgB>J
z-A|qsaDdfs6z-Vo@P;MH<M)TjhZPa#Y!vR0_tPkIg2{uem@M|(zBTH?geBUZ9ziJ{
zjEYYiMg9lgF~8B>A^*~Iq4Sb8eTytlJ<_ODF7*Byykt+$BSWE}DRqH2TyKi0M%Evn
zqjhUd#^xjq-p?5lCl_4N_`YyXds^$nBpI(<kDAFs%URk!iFJju87wYy-`bZYd(~<c
zZ`6FpgN!fgkI(s6f8&6~_5{9@e;W=mYIFICcxoSOlx0bfdLnPppuXar$&Sw^J6r_r
z2`pW5a8>hd-++nDKm3}f`8V(LZr;T_d&WGE|5oReen@;U$((Ro;pXK9&n_)^DzvHS
zFMqg3@4;n;H;$iJW;!G4!`=tR3D@Vdy_~`Ja0c7W83_q3PckwKre$_;9$2_Uai)i}
z+(fqRe?kjl0u>fHI5)OPr6e-_W%<N;ML1}Ms(}B3xeivmMh1;}OgF#8$0)7(e!%{7
ze8GKw|0hR3>i>AvVlFt}xT{Oty|UoLw39m*3(c(D(_b+^(CFdWj@&<AS40L)?a0Y}
z9KU7e$9(}h4wepw6n^aQahy_gyz=k+{)h9~{z>!pcrS4X;{N>dWB&558+tk>eri8-
z)yXq_q1XJj#S`6TSA^I}$<Gz_otActeYT-la%R-R-a5skMQ0+MlDt|~S*P6yPl`KP
zs(O6tgKx9$if5hhaGIjX{PWlLM-MFi{d3y9Kxf}SNkNAFa`6({hrg!0Snwf{<7&>F
z9gVj?EK>_}c<NaF;GmY|QueR??5FyFSK4sh`X{}D`}q=wjhzn;2l4DOot<)~`O?Xd
zKdF&bsjI#jg;rK_t@~<USI~F$;-$ko{Nyj%r}}>N>)N7{G3CbB`;(6U<>o&D?l6Og
z)iNK*=On}|7yT}2@rdtr>|s{3IeC{Anypvddgwmw!F!9m>4~?^bHr9S%N*L{HrcFY
zvV2@N-}m(%@7W^OoMb(<V5iO(Prs0be2zyJXtXs3FW%H+IYqfA(D-T}V^U~yXY!)j
zhaZBoci3eLTE5|H+xT;al0C2Sv_y8+*~gd?9~JcfP_n72`ex|1BvM*=b%{Zfr*=+d
z#&V;ch_>Cg#TgmYSAJY{s8Y7g?6G)JGw(-V?dEUtk6u5#de{Bk;fL!Czii-Jzi;8%
zw<R-QB$&D#mn^v=;NJ6nv3;^&+D~JSSb1j=j{P4Kw;QQh1?e2##-GjQd}m@!wTDN$
z?%4?Wn?`((Rt6pm*6ca5%w4B!X@?jezjJNlvv#H3N*$V$y!jSByKy3sdA`RBi%+Hf
zYvpB~)<n;oDAZIhQEJi^!X{(M`aeFJv2%an#5N9@b4Jd875B?Z=~xxrJur6@qf+J!
zr)s@tCm(;~v46m_FtWSkALE(~Mn(fsChr}U?xsl}F237RdhnER#YVeiCB1gp<xOW3
ze%L)(>i=t@(iEoLPI-f=S`8B}#y?DA-}U>z6ZUVD?=(Do9H%f_?sfFez!>HRv9K#I
zH`|K!GU;&T>^|2L)Wv`O%#nFQ4(5}2A9$yTH*MFI)fF&5B;`A8U)t=gZx)BN*lwLD
z-SED4NuttUrr(qOteQ8v)#ZCtKTM8@a1U_Mj=OS5aM9=646*O{T12+J^y1<#J<Pz?
z_Hx3J9yRX%=N2!RcUadRp0m4sS^Hi#-qr_#!kJt@zy5DL>h`FVxwZGB+f8}bS>~q#
z%XR9OYc9TgAhTz?X3{4IC+4**uU;O{{4*=mdDGWpDjygORumm`>z^wrYTetqa95#G
z7K^0BZg#1l!~XemcCDW$9Vpy%`%uBue#1kF6=5@6OpA3~8r?emTR)p!*?U0a%9eyv
z@d6iL&bVS_TGC>$yt0--{FlcUvDIHzGQ9Z^@iM7nLR5H)!Y{3mo)!J)4ck9ze9)5E
zku0O_!(LHvX8mP}^~&XI`_k(#J(#;q=EGXfmM<UIJjq&Xq`dc~iG4v&qL=p^k2(Ec
zCOecCyy_@9-nGSmDd|ww#VO@8-n-cL-s9gh)o7vFqZ>UR#j|GGTzb%_c5;G_P~D1k
zQqd+`<}ow>esI=`@s>jV<B5{L4gcNWb!om@Xz7;&i$w~gQx1Np=xbf@^WqDJYu`n!
zx&>8K9m9W_7On6poXyK}YvIIWu9Fu;z4;+7_nG+@!%W8hGuAs_aPhflocU<I&N0R>
z_WOin|AMFLOxJ%@-*H-8x1{t{gv*ogz31xqn9_G`Hp{!&AQIOvI{k2kfK^-UMV8wV
zE!Gvf1?MvrE#B=8dwOZc2629=g7A(X?c16fF7ND*2;<#%WM%HLt#&nM+B@4WMo4Yp
zdDwWvj?vF*{RY3gHz)KQ=&y=d8>LsF`*{naOX2bt8~AS=Sr)Fb>A?;*aW?5m?r}a-
z&C5EYZ0}YzhHXsFW}CN4L2bhaMd$cjp2N)ck9`)MUOBzj!Dh$5W5NF}yRn}0Q1bh4
zK4X)@CGNY{>^lN9rgAbaddMUjAiqN+q1NnK;EK}H3EVS97&wk{v`8haH~8g#<-g(7
zBZWM_F1Z}pac<t{Z8GBZi%wP?k8H8eT(@P5<jwVse|H_5==jhuVH*Q``!%lGdD@y+
zI+yWOH{52KkkPv0bjFGVX10khS3cA+UG@7x)qyO@Xu}EK&wuf*dD1ZP_>KLF0h<{Q
z#aA;<IrAW>?e&_^VF$C?JZyMe*$!r2h+O!oG_{7|W}<J_@q|FTy0Bede39ns)OM5{
z+K{K#aO~6}=^mq1iN?a7u6zYwwBFCH{8}k&G-YqG$`Qsq=JS#qi~@B^x({m<7ipH}
z#;G5jQQ_VcbD2fxFYBDN6#h^0%@y7~jE`Qd47TjJ-TJ+Jp{ia7)BJzcqLXDi7EaDJ
zoY(GlZNbZkgxA9Bbsx{_-(W4NeeZ?wm#tSsw_BEP6`#b~Iem3;zvJA58p-L8uGgi_
zN;z;a`ORwYvh0e#4BNQ&sP%}Zt}&E)bA(y0bZxq=XZ8X2gae`Jcf@~bFA8z~qBmhm
za{d<6-gVlaA`Rb39%g;e)3>qrt3s1b$uWu9aW^ZQ11-$9Fg-91Qw%kV++(A%_}s2(
z8}&P@o*KSbHGAiT4{THIzODM0cQn^1uY;?P^-9W|n~#p)Y(A~1&ng{#rQ(V7M2^+l
zmsf~HtP<$H{Get|`^LSk?pq$*YpqYL5(~VwBZS*>;W_pl7BR~{o^0?r{D7%|q5eRZ
z^wx>mx_53lMM?j#yWrO_+dWKm(u?h8mfS_jD)$TwiXJBTHFs}L=6BM5>?Iju>?kMV
zrt)slTBeROogaAXCK)VHl<l(!S!Qx&$ITAYV>{-iyuI=F(B1XN*xw25xXTrrl(?Zy
z^t*Wt+g!cxziu9A?49T)r?-BI)XBXe1^PR}Gh;rQ&gAi0`pBl|%2L^dr>w4uADUk~
z>BiJ_wse~dKUfsntuG(h$k(Huy!cEkdlTck{tX((-dVTu@^jhuyi-*PZ1-sS_+H@P
z>3KZeET??;$h^Dbq%+k+Uy#Z8eaT_TLzU$xW%u<}EMEHm`rA({+-@;T=y7H9EnmVg
z_qatMgZ>H)Mu&Sx|C;{`^nA3-f$fS1kJEWI^|MTG3`}~Tn8@wOEASO|cF&Pq%HDA)
zWTMpKi@F7Km~QO6Xe{`nNl$Et?XjSKX{A$3GuF6>PZVvr*XzGvQQKsTcg`30PCX&+
z_xy>Pw0?*Ff%QjrZQw9o<)wADT;PhqrTd#2<yS4)Y1}Hm^YseVxkonENQxT@q|Wtm
zV*b!nab^~qp4+`6JlUKp(w2z76Ix>J5w3qH#pIw{acbPM#?y`lj}Pu+@t@E$A^Tt~
zi<d<BwT2!h<->yed9E#H_FAOj$#qmJfBCVsrJH88EjA9@AS~GDu6Q%>`d^i8h8(Os
zM*BN*R0GWwcV1|oXQaO38)M2$6Xt~VC&W*jT<WIr;(*^nktOPawbw7PJo^3cgF)D{
z!n=-vi>9mm(4F0&Hu+!K1ICMrj!vFIPAq+gv}FVswf}eD;@faRrBVB#-{ZTM-?AbY
zFYrchFV|9vmv)LU+4N2~K-GN96a^t`Cq6zu=5tvAF_ZYcO~QCpn6$qoKTx*OIBe8m
z=#Z4!m9}M9vgGkl`+}%NQDreo!51DrDGa=#uf&mV-eq>$>S=PJux!h*B<Bg5p1HGf
zJRaPb`z9^v_s)X~67wpj#D&_mT@i9KYJQ}sd_c{WF>0P+;3orV##GN95e@fKHD+9I
zweqF)bSEgS47$Oh+{B<ayYFqA1ILpe7WuMiR?M%AXG~VQwP{j+^M{amn;-iz)cGvw
zJob?}#oS9lz&iNQX)Uw34-7wkzcRUOD!S=`PyVuvY5c;AJm+`_l{0D@oU3B}80660
z*UihQuOQJRpuX5@Np$E2@1Ftu();E=+5V|ZFtzkuxze8U3x*v(o3d>tb||N<+_6ty
z<)zF=*K8HH?CWaQdi)=3k{ByHt=H!(FFSmXM<M&bt_kNR|19IEkb1Es@!Fl0jq<yn
zaeOqNQ2PAtZOMg8go8_j7FcNUGCW(rA~oZD*KVd6vyaVr{6VB3cxLs#2UojAOSG8Q
zasS%nr*M6ahCav4DdovJQn61BCWr-S^94^cPCFyCaN;8An4bp}3s#mDENG5PUeb4t
zeTm;=T}9nb9brw<wZ?NJJil1zEIOHMI_col?=H_}*Hv)yZwi{^uD;7}UGZ59Pi7(S
zn55T?y9>(R2?_N$giI0i*wu1-qSGoLyW5_{)@3;kPK&=NGaa4CeSm4kgy|2ee3nl9
zw{FT`;XSL)newGiG&|(c;IKh6)1ys9Vuj0`wViJ!oiTNOVyjcR`$<uA*k(PgnL-?G
z+c-m~txle()nBQ6{L+ao^BGUeCuIsu>GWOB)ARB7CcgBi!cQJgc<Q_-rf{Xx^OPm>
z$5Ll$y01P^x?on}+{srA=e2YMsPJ#-o47saLD__alOk7woxG25zA=8muP$r2&QQi_
zLNMnxE9cp)UQFtDH8Qx9dK_N%-kIDx?ViByiMyY!T#@pg%j2EWs^cC#b}N28Fnt=-
z>-pAXgU<Tp8r#ynN(+zpd~DW`I4*k6lGU;PoYPu{|1FuOeZLAh*L5x7<*j*nVeXaX
zEZsa}-b%{C7e6mBRn*flEDU{RC1vwcHfo#K7V*X-aufC_G%-%-nx_A;>|m_>N$nI(
zn~$B}wVM7hL|JHi1i9T`^0%ST*l_J}@3R+=h_HFCW}357lXd!!>X^bnm+E=q`fu;L
z+beh9xnUIdF4UZHhCA2cwlf8%*raRsFKpddVK}Adb>r=6JGM?<vWUUA=GX>d<#PX6
zO^1ocydxqlnVZfTpE{Xy>Lb^|-CQaLJ{MaAY9mx?`!COuzAEtGc(Jmckou$*T?Kbs
zIR3?)NV+t4!OJ5j898NM@0g)gcKdmv=Y%T;H(u??evs|#XrcVY&Lco#=~w4P!m&;|
zfj5L|EL=;DWxF0)#a^<7sZ{(uufHi1&&v#jFOJO&r+2PnI;PZ<G@0e9IOkoJ(#8CY
zb>(`duFE(AW-O6Z4R*0P>%GLODNIv*RfO!lXA^d(uL)hs#%bVcB4OzsvyhqBLoU)~
zjS*k$!FLahE}8Ow@z7m%+2Q<4v4gYwC&jt>c~4A_3({waWYBbY6&BOv{qRlE*T&oD
zEj%V#v=~fjC@8GFs>mwn!4neiu~b6UQq61Hvxz;I)a$q(oz%>iPUkPLNx9LuY=xWS
z9`BZf?benGJHCGQ@o3_j^m^jFL;SH#Eh#}2fgjmwRaI;CZm#hN4XWAmuJ+=+Sod|G
z%zO16?0!f5QrwVZa)pKU&sN^YEK)ajKAb60VYYC>Ivy60PPt=BOa0c`@OdnIG$HX~
zuiy&pFA6KUjX&AgXh?Z15=dC%;kTyf_QD$qQzy3kJiehpRxv%}O85!ef>n#H-#>B^
zY1)y|QJeQ+S2fSRK%*~iv(`HsGjwxvotdG^P{hCyC)M-3FtSv)`az~{$_a;+qL<X1
zTO=);J#2q6SASqVV4B0Fk(sZ=&vNIYMwy4oO+|G{5#}SDkxY{v91i3w94OTO+UO%C
zQrv6J5;p&n-QuP#9zi}FuUzcB(i|08RP&g3D(}pDT<c=`>&gKmA#<BwT#Dww)0uv<
zKWPYB{aoQC%Zp1QDn3P4HB9AhL64uLSl{HB6Z!Jq<|iC;WjuIA7M(9n@a|&EX!_>$
zecs0^mua1AtPMK0IA|~h9eVWWm{ITvhP}Kx<@*<`OJLk+@VL(0kl%F%uQ|)(c)xjf
zRv+4Jb@GR%`;lOa9VM?OtjY^pXP7UQ!T;a|SM!Rp*sU`{n4fjK{YanH;vqJvqKA{~
z*w@PsnlC=EveB~_xWcjYqKexq=H1Tc`|j#a>Gth4S;lGW_N8)n$-FCmEEiY$zH~V|
zmudSsYj(Aukhsil&!`O5lWKu!Y%KvDZ<S`YxKESX)H!=nuhmn(-Y=<&Vsj?#bWr9C
zcy=SoVTzvkOYx?cO-oB!Dx@x}?rJkonkXi`t8k*<^plhL*GaS~2ybvrc7OG~J4Jjl
z^Yhv}OV5{OzW3g>fa`c~IKx(>3y-|c&4{(ptYA4>w#J}!;hDL=3K!2YY`C&i==4g4
z&I9Ln`dq!35LY1QG~?0!W40PWwdQeGQgj?%C-g>L;>~f2><w;s(;S|D&{sXu^!%g)
z%+Es#C%4M<a!JowT4%^UUCykl`eYExiDdp~zYbi96ypoAG7->F_{^rm*74YJnG2Vs
z)lIQqKE+{EGnTw6IIbqxaOKHl>%vHvR4L=eY}sdDm^Li&o}r++yGC)7o{T`ndpFK0
z$}<mo8*06LIzuWyeP!aO$x6MBx1B$-Oj3PhC3B*=%T$K_mHFu@$3G=45OXS#zy40h
zw{(jP>raE)#VNtpCO$spUwCT^!}5fTnL_h8&b?MVv^gN_jn$efHx71rd{VzI`abm5
z4yVMTN4z?->JRNtc`wrQDC^A>>yTA9n?G#Nc=PCxV(%*LXmf}5xvDcZ?OZ)!@`S7=
zo6D~M)I9IFGFe+%FK3(^d~|BdkBUl_#BF>#?Vlz(&*HkHr>xH<zj#TxfTLS%-0XAV
zS+fmHT{tr@w<j%}{P?8OuN6nST3on{XYtGu`sw4T|KO{(YEw^x&kWAjt2s<g&2avy
z(AnkI)4XEkg~O9B8ZPqUl$KuVETVa+ZS8|y4Q~Ryay2<zelz(-sn2~<bD@YaykW*n
ztuGQ96Bl^3{COHN!>Mo2x!1x<92v$dXK@JEOx~fu`d#&`TAxwZT80jvnLMgl2i{2f
zhPTv8h=}IB)Dty~ESzVielR2GbZJYsg!3vX@8fc5Z!;FCd=2z)shm>M@kvHgb5)-4
z`A17wbr*G~oR^jqy|R$^DATmEsf^VgZq9$$beBICu{G87OYzbb*5u-1;Wmx?VDNUf
zf@sSJ#pYYBA}l8?C#y0|R9zJ{OV9k(n`5lul`<Dhx4QI3J#hQ;Q02W<|04@7FSm~k
zH+pwB2~SXatHw4-Vu$G6mr2W(_BNeet(?MnjXNelO(I39%g;){Yl+`%xkyeEsc9y&
z94~lG+_C54w8f3T@<bXNclVmGa&~C>Y<;nV+lsT5Z<5!;e)iKco|;>|4!7xC<SV~B
z^$aiXiMT`^m%b>0HP*hYy<xIjJ@0B6UAeXN=)+Sgvn-qUC_XoG5<WOj*=}k6@>e}6
zvbhFJ?lynh)YYsQ{L4VL(9v(1;)xyS3X22ZNNFrRon!Z~sDk0k{$ukz{ROK}J7wIt
zKDB1z8V-#hrGq&FhZEjf9gPp(tCHyYWW}7eWpmjqpLB|HNF0*%dLs38ZRMu4+*>nv
z&&Dq1Q3#Q&=e6kdUurj-T|72$JCpQ%7H8M2)}=O`6B-=2Tqi6)YQJ`xw5rS8aM2a_
zJ*Gx4cqiC4Z%WrvgQ<GWFBhvWXAwQ}-|c1SBp;5X0<X7PcQ`r}W1r35@|3G5+c9;<
zEJ4=ChkBRPb-%E4n=<8{;ZpBcXHJLjDz^x^;=M?&T`zCVxhZ`rf|f~pgtl@msO_0(
zcCy7xV0rJID>F2Em%7`}eP~-CSg<kkhGYFP#k5DN?0q{8)eavOdS$zU@#e~ajppyz
zC)m8$?Zoyi!*j<(uk;RCtBBI&tE7sU3q>wv@6=f(zu>(H>$L~M{Oj3Yx-2-<s5LY5
z^a`d4>iQu9rG6Vv_bGi?Cvjy<u;x-0xdV5fUB0yRwE5xG(*l*1{5>9mvrqbTKXI5;
zu<c@s@kOT%fh%t^9DaNxG}imni{w+kn)WC(3!jhNWV=oD^cJDOm<4R{KmEKG`Y-HP
zHc_1`xMJm}Ommf$US5;x9?jGHyzSK%>4T!LcX?-)@&DYt<b3RtxjDN^9A~EZ_3L!A
zxbv($bbErFc(c}OldzX5(!1ZhJXWjsuwKb-!He)u_f}Y&I%%W@awS~4Hf1x5<UxT&
zF4pQv*3*A*q@5Ph=y~j=FK|kesdUSi`;JVy6IRRm9i01`mDy$0LFMqd-S-cDS=#uP
zU2oZ;xdooDdX%QL?{J!NdSb0%=h``HdxF_*|8gw4!nH+9_@$)g`NwZs8z);F5Pa0r
zkziuTm$rOrx88B@iHd@Y6jEclK6GBV6;deR#wI7Yidp)|-NUPcU(U!#6f&LQ86^CU
zt!(it|5>xtKD7ibXR38xlqTGDalvNOFI~l(Hpyq^?ECO&S*gwWe-@madtxT3gk~uf
zK5Uc=2)bpaIX8PsD2wlI{VG;YeTRd=Ji&_WLX%HqH$5;2n;oVewDj?|4yCP`q1PM2
zi<L8-UeEFJ<$tt^{q3g91uX5J%N&A=!kgZ!?BZnUcM?jtH0#jn&LE35D*x0>*({b^
z($m<beY7f6Ws=@MkA;)`Rk?FlX6y=kswTcf&@16l^)yjg<-quFU#7G$Xup_r{2R~o
z>21!}PAV#9tm^#KIc1SqC&#QMwo_i&r7+|fP5bT_td|)zf!(9<TFM`<)zgo8?^O))
zX1*q)8y+C8qMFz_T~&F}e*Xo&s)cD!R-O3XF3jUU@psF=BQL|bnA<A1nrxn`#<rBh
zWJ19VJD-qM&e>=8tV_5n^>*h1cQ@agkG7fpclRxt^xW-{$P8m4xqvIYU5(riC%y6A
zJ$>2hE2$Ok!H1W9Hs@t*YOy$NaNw#~v8htH+FjL^HDbY6J!Y7QcTZiyt!O_jqxYsJ
z%aR9{K`z`|*d8p)SrfD~EhT#A34URZG_N0Pe8p}_%%A>^Glo$|Ia$wq)+Bx%1s1bw
zYeHtpe~57LEYZ@B_{C?r_rU`Z&oJGtkjGb>v(G3dF{Lr_=3J2u5xN$+`qh&>tJ&-i
znyzL=t<b!toz$e&srN2$xfYvM<Yik&_d^cZam#Bze9&*q<I7ULa67?S@WAwIvL>e&
zY~dD@5SbO=sbjUR_^r2?yrZC!Z}@VfUhV+brPH_S@ose2Qf2WqYc4LgcFhfW#lMaB
zx}oOgOrM4cUeg0EF9==ogW)=FONQRZ1itf`bA5y#Iel9caPr{n&Zrp<Yjrn=?=)G;
zWzWUwGd)1le(Lj0oka@$FSp1y3hA7#TN`-Y(`80QaPzb|c}jk}ZuJ_ze&XKsqI=@^
z?4>g039K<nIj2?ARPt&cq?vWzmT6q1)f`m5VuzOU8(-y(PSID4v~LAJ=f9D)s!d&_
zY-jH=)iq6TqbG4|FbX<J7UgcadHEv$r~C<3RUDNkO>K-0t_^ZzVr5((%T+wrB)ZGx
z$t{+}3_6a?H;<%v?K^zZq-xm(c@}BSg|SNe88&NktY2ZCX52Zg?y}q26XJe<UIbk^
zq|@iaCd#09wM0XxQ0UCuX<2#(MVy-+UXna$9_AF>tgtum1B1B3EA9jC{?~=G{Z*fL
zrO3)$JaWR&Dllb9!R@JGp-N$j+{+(LxpvX0X+pPwgs0Y#OwQw-kBYJ;clLzTge&GQ
zcy_wLCb0RkhQZrIzZl(C_pi&g^NiNmT-0T-sinnvm#=3_@yiU4E*;Tin*+vkkDE@O
zoY40CT8a6KTc<o;nMs_CShi4ldYR3d$uD1b$fXBui(j<AaC!IswIQ7+zlpO*-jVe9
zWcJCoHE8>h-UYK{KenZ0T<(rowDQgs!Q7zR0@;^#y>?7^yz5eW&-Lm4*6u6KpWgAB
zEuB>-A1v%vSTW_U^3UbBmPss+J+bVZ@f44HlD&*IQdh$RBTPNFx&9LV{%q-#^Pke1
zbZ79KU*%b)c(P!16Cc<As!6R3>Hh>u=V`8K31vI<$=T=RLwB!v9#tW;yhWmv%LI>1
zRdSn|%u-az=k2)k^deqQ)&-5Xk8jmJ>3wP9i6<Mlb=9pawx4Si@aI=6iJHk$_0`NP
zA>rhq&06crZp!eK@V?)6fLp!zn2FE>LDr%pe>9SpYHO?&PE6<dH&gP6)zO-QUF+ve
z_dJ`m=gQN$oTBU{^Jel`K9fJm*zi_aRqR%Zo8GBy!Ovb^E-Zhh8uUnV{*2R81&VHU
zYQ}MvOjdretw^<e*MXkymU$vq4g^P@GP&*cKK+pJ6kp-B-k%Z^?@FJMf3U@`D7Mk$
z(xan_S>m&1oc6XUSkbJjHcO+1$Kvq6kW-T@Stj<oxVfmM9KWo7Q1i#Zl?A%K7nBoi
z{a4+2-y3w5DQf)$-6<@SczU-y<CEQI*14{E$vRuV;8wR58IjKIT}rdwU941bm1uMO
za>vYV^9q#}Tplhj)T$QevhEa5;Ca=lFz1WQH@)D-`K2@YAFcmvRyxVP<Sg^oRObno
z5A4?BabOST)!IHeb9u-zkA1viotI30>N7G67^b~sv-)Xxi2W>^z+u7HGdmf#T$?B^
zdiJ@TKttH1%L?w#=dx~JqqK10<H*$cmn3ZtR;sE?Esssw(YW=FgkQs*vL~w!C)Xa4
zV!!IWus=k7%5jlTjbFT`G%X06eP3d!%o6L5n@WtbmU~PUI2rC%q|N!M!+qL?EeF?b
zaNK-B<MvyVeJ}PTTcrzUN*b8Cr5$AyNtsx;{iV{BnzJ6pyJh0#1wQjuZ*7R3v1W(2
z4rln2kDfc+ES`H#ZffK8pArxf$aBeOx6#$c1E-8nv7S6?IMb0?!HAFZ70Zo`>rdl4
zLl;QDN_l&tnyFqquG;s|Vb4g@vxWlKF9jN%^Ikh?&cbc$?Nz6QYq_49!+fzONyTxQ
zq2#nvTs2Q7K4B}EV&J;4V(FymXD%&k_Y7q^J#l%d__;-UcrVz6y1qQga-Y}a-2~mO
zCQJW1mc91NG`*so)Tlpo`VZ?@;__DvPJ2yv6Ibq<@?-lNGiSlak}pdFI=@BDtC+KG
zu~2H*Db|>+(y1S~<P6PZ8;xV?_VScFJ(;#eE_CwSAjXhO{}U!2+#tcm%W@<03v*Ow
z0O#gFCDx!U&r7qEVm%iH1`1_r`XBY<X5qT6bu^a8i+S#~;wp|{j(aN@n9r?}$bIt9
z^6N~&*-ngtsRzyOD(%=;RM(Qtz5V#ZiBCG~JbSK~tSwi$r6S;;c{%mK!R~pE+Yj7b
z{8`UPE`_NjUqGgF&x<V!4myT@;BncR^MpNP>SBq-Ju$BPTJOE>+Q_g%!S&3ZS85FI
zr|(SliJKPIwZKT^z4Gapstsm`-28%TqLke_)*9vBjS)#wRSr~COi7sQRJelU!p7B;
z?0Xwcx=uGJ7)^D$;PB(hxy#Kq-E6O?v$TI)zvJ7M^o}#f%nxfAE|PfH%AKcrw?n}8
zVDZwAmvUVe`ChslF@2j3U)Th`n5kO6i^WbzE?hA2_r<z{JRL=kS6mPkkmx+UP3=(9
zE0IfX<`@5-Pgtd4X!g>r^XJla^&y!LCaGj*o~k(UAgIHUSG!bC^fb3p*Sfq}@e{Aa
z=Y|L7Br$tbZt+=~<$dIyNOQW>xiT){7&pDpTP)g(stvZY`@U>?QX6u&sq*Cc(<>!3
zr|W;+{aNO~xpSWXTnxE{HnG3Fy}VW_)z2+H)56m?IQ5ogtl!z@IPR4Hl8e_|(oy43
z-MQq6p!+h<gGvuwQ<GI+C@uMNYGu?*@A(~#+%>&dwqHE6)$zvEmi3x93Qq<oe(+Fw
zb-dK)P)MPSUiP-k$D#t$4yH_NN>JNWvshA^nRC)U#VrOMtS^}__5MwI*r3Gp)9C1A
zMg{*)sdkf`M%AEmA*untS&cPMilkMAKVXiO*ePJLHCFby+?LD-KVN@+lsCm_^`u*?
z&Ny9cc)99~WiO+PkBsXCmN%1nF5kN|VST%XzR-8`yLUHrE)4Ot+uv|VrL^;i|1w^+
ziwQ+8his2;aqXR;Fw;=~v4(>3hV*y5`ena#4n6eSs3h&G88O-C<bFe*1p%Ay=(I*{
zy8LyzBD=%oz|Kp09|dZH3+G$kX8w@+>6o2z`NRW~yH#fMpPE-X$>)E*F^@`;kv_9+
zODy9};hjp8e7>tx7;~CR`f$(bZP@$im~K+aGfA_ohnhBpB7REOg%0iY@BiYc;3MzF
zQGG@ChezBm%QL%;I1;R7?b|~-ot{2_`cQC5JI9mXof|9^Eax%%`rN#<;#Bt;&5dhp
z?Dwp3Q2!>$+E#bs633xm3v8FWKe{s6;DsyG(eDdlBF$Ks*>>jzpK@$Eqh9KMIBpx0
zR|fOh=4h8W!bdOX2W*iwIxunC!-q>tXM8$d^<SJl`fvRL#+_UN9L#Z=C+BQC|Jldg
zIOdF~=T5C1*-?iXw2e2d5R^3HlPVPN&^$SN`9h5x(;uDtJUB#I6doS;YR)n&=#AUo
zsLZ2s!RlwDW8LHB0h_*vu2F2Zo~|wVvhISkt%j(?;wzy6dqp4p;bL@HI@3jT;U&%D
zmEJv1AL)LYVt07U+evp8EMjr#xo~2J@VQCnUd;$z&eBsijq5CPBhznr4r$Xb36c?S
zR4PO+ueZ2(Mxn_@dH;DX&Rq99oKG5s_ulJRF>xEmM^*)=xn^7@ubj`#Y51Y4`=Xw|
zO2znsr}pKm>p%XQc06ygsLWNR2`5cl-4YWd+#WepEOQ79UZAf%`I_0UB!fjPc2yx4
zWbGvCn4~0bo$OO&Jb3XtTlU%dOe-;wO>KXE8F1{cso*WY@WWAlt;N%z1;tV^KScRC
zPl(H^+`H-4YUSgqQI=Eo(&Wi1mCC3|YIa6y?vKu%IAAXDN@B+Zx0BZ@3V0rOEndl(
zSyMGh@@;pI;G7LjyV85ReRLJI%%)}PT-I+}H&Hr!%gn4TY<*Lt6DP^6Dt<C=%3pua
zOb6S~vzEqit#M~zKXkyVGssos)B*;ryW0+2ymID)XV0OgTfJ-B4juc+@pF@&YOK#x
z>w3TU&YwIyMLk&pCK~rX6TK3!DD%^M-#e|%4u_lHmR{snS#!fQ=(zc;-)5GJJgw$s
z24B`_Iiwulcg|1p#td_fTviq3mtJPQEmE%bUwV#dv)xX5&~`+?&+PC_R|gH&$|->t
zIF|&?WB-`4c&ZEYRpo`nlRt>PyKelgOj-7p#zK*&trK4cs#$X~$G<wp5I1p+k?4wE
zEv@Nm)Pp}?c(KVpJ8Wn3!#}44``T=M<vN@DGEbyknQ+A8&_=eu&dx2n+m871cAfcX
z-t6jJcGiONz}29wj{g`>yb9X?a&pSuE#^tp{ijd4Ol(^s8!XxC>v=gW(MCdqEk~nn
z6aTH!gP&Cb!=`E+_qN&4xr5JU>a66M%Rb1gIbq3oLFfUqV@uSs&m49eYu1Esnd=f=
zQN?u7YNpb&V>g(1{q}5`log<D!s43CQR9-|*mgv9u~R>f&XM-ZC$>L(Y1^L9<MXa?
zmyp9grR^KmNOj6z7B$fm?w#m%bp5eP$?Ot0mVniJKZw8mWyxc4pvYE0FIZIK*F?s>
zmN~bMTiw<1+-?$Dbm`Ab!+Q@G9&T`*DPes@`<%x?L!FXuN4>>5r^ov*o>-u)%F}W_
z-gDht)!?;<q<WJS_*gFl?D%_WgXlbs4qeOD3nILpa;*$`aU!H+A=g32VqK2kD|ps$
zR7KD4v2=4b)S8{Km&cdoYQA1`+N{-!a*phuIdNlGjI{LU)LL6Fp{rLk?&XTaXiUj(
zRt#C(cx*+>>wdOefrFE;hLmn=UA<Ole~4c5q`ZE{S)UKy%v9RKZ0fu4oTk|^?<V0;
z<?B}_@N|m3>OGj7VQL~bS8GqA@9hOeSs}INQWF(#bwwnYMf_ZQajTR#%hc<=THD<+
z8E-9fS7>j2tGu_Xt4Q3$if6|e;X_-3GP>UQfBtz)AjbEUC~J3~$*s&yCnc<sUIa?+
zlRM@T$`*P>;<(n&`exZuXQjm-^|{2;P3lj%%jF)spK#@);#pp!#S7+|`pQW^?R??3
zQkE(7uH3TYb4z@Awd@1tbv<5Pm2>aLvfz?8tL9BfUht_=X`<jxCDDA5HuI+$90k?Q
zJ1h3TIj@!EY$u^)^w@iO;;f{yk8=%IO)%VaVZwZe1NXj8T<iTK@5xi;?qesWYd73k
zkZjC;V#ay1w+9r(0@Ql=jP@N4&-R!o@o|fTe)xeE2G3Y!4{0*KU2*omZQ6p14m;n!
z@!tPJ`Ehs@1J?=>#iKJj9okY_54J?@np3P9^G-#^T2eBildUr0)4?UokGT#VUVkBx
z=Z&Fg&*eQ+oIV&P-?HL6q9`x-B21vA;a`p5vW_c$X<a`8_if0UATh~n^SK)hJ)BoO
zg|x0aEWfb*@Y!WS{R?IY7A#NaU*@o|Q(m%;?U{bx=U#SEhqam!`;VRJw~Yy#P}Ek!
z#Af^a1IMO6$-N@Wy1Cj;9N17}`tNN)PU?@X?6&88>e7E4xZxqPA@x`Wb8xpg^WuxP
zm49Y0PfT9EZgsFB%S4`fL%p|$CGK$b=}weN(%iRVYeTGosoSP!*%L*j7bmjLo>w@d
zH}+zl=ghKMVWm<FOIul9^;~QTbll<J%<m<YXz}bv_`_!z$&mv88U(j^1!?R(m(8yd
z#?ES?Rl=6VcOlc5YvIP&HBOHf&L~R^VKU&e*!o~@na12%>}Ji$3c}J=vrB)mmh9~I
zS7qH=IXC52kHdV49i4N;<w7nhg(N+a^t7>@;``{}wH;g&)?S#s^K-seX-bEKuc%4z
z75{@GCc#f*UsU&>-|Ayu({kt)&uK29rtcqCD8$aQY0+4n-XU-7`R-;uPq+wIieSKk
z;9Eyd%S`3uQ@E_U@@8fc@60JWj$X6Bi4=4vKJi+7W{TOxbv3e{{8C=Kv;XawGe@v%
zrb^5I&o;LExx?6!68|LdR<%81zU!=<dVPi4&L0z0OFNEPpFHH(ns8jl`jS;oec&pi
zL(I90xqaoBUvExI_`UFz+1rpw#R<nO{*|1(q`Z8&%yKK;$|bq4uNZDhnt4$(a7+Bj
zIb4Bzdd_<Kl?5&3a9NnpAmyGv{l4eIy)5b>8=byJzA!s=Z;s%CmDxTya}Up|)U=+f
zd1K-;7u}mbwr}#BzjBq9Qft_nXW{y+uH1G3ee(miY>;XTTJb2oOs&nDapqOeng`Fq
zwtVxu-{}|Lzw+x!k=oOX_)o0A95+X-#B2KE5c#=nyH_mT5X~djZM*13wHBlHd!Z<=
zGO2*F)JF&X+swRTVvjOCRPdR`z4k+NO26Lk-UcT<RsG=FL~n`0?00OE%q26oiq^RX
zOIe+FtZUQ>Y7x-5yl`)cRMSBbjYs)vC0-?O+yCY!nN4yiRbP^(`udRSueZK4g?DMr
z_gwx?;8{&iOJ2@Bk!3A{JUTA})5SE81~|F0iu)DWPE9mXzxc{z^0a2%IaQes9^Tgu
zY*l-ESCn~%W~5c?nTjvk&5?`mP8VAq{QvldR9V5d$E`Qq|Kl~?Wx3o8zU&gIwO_eZ
zcrrIKHI~d;b?D6Xr;k!gzC6&cT5<k@;>L3;Zgwen{(fMUduD<3^4T)Rhi2WmyHlD$
zoHgP#&zvP9g&M&TX)_fSZ{6xya$v&5!;2l}u@+Soyn2*=gG2Hd`^?|7d(?ePQ=gWs
zymZs(a6P-!`-dyPd&MM7|3CBAN6ENVtCPZ8!@kz`#U%SKV-uM+D=c%v#dgKUUu#Rw
zm2DNexoXYA#$PvX<%BvcoiM{>YHAHbRz9<CyIX41#j_>{|4d!sdM<|d<rL%nsimt{
z6i>}cJerbIqj`CcSB=@^_?1c1yKYUgVB|ahsAkr@-YBQ9*~SU_XV>=3y5(Tjcz)UQ
zn)3%Qc-+}y&LN=AG4otgvFjY(!r*5d?#Tj5Zk<A(|9Cm)w=NDS;o_8;`e(@{gS8*T
znXK3%c_v)qJ{9!jeCjFd`q#$Xmmi&2c|UX!x82hAd73O~FHD}6Y}RB<J>hx%)s8dj
zsS!daEI#bDn&lUwn|bf@9B;#2F(=iyZZH16Q`@8UvY7dfo~F5rjQU=9vP;`2oljXJ
z>#ia_-_u#;)<Zk4sg9e@YyZ?TmirTW(<fa}@`Ig<=nJDSv!B~vT6Ia}e_wmms{NVv
zYT3^t9a+6D20zpHO3dw=@M80w9-G&vw&fbXeH`l8)}?1`q}z77^QP#(o}||TTO%)T
zGUU1<DLPwF#gobHR(qJ;gI&T0raj0#>U^?ME8mt+_v4XZz51i#i?-ZKkv*ai@oVj)
zFY~-MNqAh@ul$kmn<}TzWS-V;wMp})e?M@hX8}(HPq_Aaj~PlYcCY?3Iq#N@yh(7u
zR?*D89|Cp+&wIwg9e?Gaf7}jF@98P#KF^mZT)xlXY-O_6YgYY|6%!dFM0RuKX5JJ!
z%Q<D`h0JfFmZGT^<`bpA%%0%;<He56LKhX!u2eic$+yj4+Ndt9$Z>M!&BYr`IrpU;
zxEQ%&bLTq7A6tHhh0ad<c{xV+#dZ~|WnFf>_URAi-ejooxV$dKvGCg5>8<lO*QiLG
zDZbToQQFb?+^nQa46}bt+;#bBoVV*-qltz;xt2An7KeV@$v(09VvTmYK(p3)-lzHf
z(b23{OSQv!HrqT<e=Sz~w4fqfq$%*Cp4f~p>=S293eu`!-}lLofBmXuKc|HHU(9%N
z^`OAJ9||#3eHSm^7_nOFxy*Z;D$lRk^>K~?%|4Hu10~KoRmav`UzH&n!+eD|VjfG-
z@?NP-UR|mCY%Tw#S1`;Da5WB5%H1hFV-kDVEh}Dab}_vgZl2(og~}KFZ31?iCBMI-
zU2-%nzkPX?>bFT3OK1Euyv(;LYw7BDr?eQR>=lyIc+;mHl=?xp+t=o5L8yBuug2F|
zpX$S8>sTH6LfyR-d6%#LvZaM5blR?|$Aoee0(O60a`0zynjvpbp}?(RMgt|?iib?X
zEAAw%a${3WRJ;^pB+&Mo$$R!8ZI1rayR<oq&wlvS=p#74s-86~v6<(YiJeJrY1kVX
z-HEQkpE9Eksi|B}c_5p1P44K&gZZWck%pIcYzz9^Tv;+hr7!)V`_Hx727PNd)7P*`
zc}e-kS2C6vGAw%NynMqx?k!VdmURD^rN?Ug+TaaW>Yo+-T!}3YG##hMi%&LS{!x-u
z`g7sSX0~^4+C_53-KK0~dAj;nxQ@GCkodxdbC_3HNI&?!q}%m>`ZI}LZ1EpiH79NS
z5c2Dpg}(Bv*Xpmn%b$@taYIrpqrxjbAlV@Qn9kAdH@Tyoda|CjhUu}e>0L7396vv=
zY?DU8(Xs%Rj|%s%J49?zXg+^n_I(N4#%I^7#J-+SNoWk|GYn&xb1yS8)WPtVOli>S
znNuB>@;y#<z3#W=c+w5cr%ZzSM%OPCGzhNBP2-z+Q!@7c`aQOPjW|=^aQ)kVdyV|j
zORHk$e7zADymd+1<g{x)BhT3^^AJ3?n$`bnsL47r?NtY*Oc<`e@%3LbNv1SiK}I+`
zRn48tGPQ>-<Fwh~qKvI+&x-!d{@Uvtcp`w!e5&j^#U5U-i$|0;Wi48^bwX{>zxVA-
zPs&Q?9u-tEjkp>g^gZ;<^fl%cy^omV3pisK-^G4BB_dn(<#SBj&*VEs9w+_Tj16sO
z*CqWtvci)i%_r3+Npkzosp~mUd|Dc$!L~1bzJcSW6`C<KZyX8hQE!qwvH#4Xz8N-r
zzB%oh4SRi$9XnD#BPe>to7@l8=gQ7K=TvdB;@tOl+KcZF|Kk?d|E>s#o%fShed0@1
z_oIi(xVppkO<leF%bR6>8JA|4h)j}E|IqYw=S9zVb4*q~oV@t)Bb9~yCXfA880&@V
zJfmcPvPMlx@7uxb%{6c0GIyoLS{u`6iP%-ne)Mop^xU>|iK4U{oXLFx$@}zA`|tcF
zUN<>(_oWPDWq;=hI@{vD_|!;c3myz;IecHLm+iv>&1$hUfs6S{?*!w$m9GZap5t7>
zsHffGDtqx`&y;H6&a`)BmFJes);WE5**Uk5tuMG5rdxfQ%r3<BLdPY2@=Rmlx_7^9
zn_sQg&MQ-V-=tn2tk*3#>zVU~q@);C{v*b(Zid{u+90#z=7Bj`7oYJTmT>7lwm+)A
z-bk0rUek6$aPztih78Y66%>43nNiWIe{dpCpv+X3sq)&hCo1jR^nxc&Xw$AePU%OV
zKi#HutI_%L_nB*~&l#R$OM9}!f_nuc?+uZoTr6tye#NCRyV`ndW_`NyY~ku9Q$Jj(
zo_VuzlN<A_ZP|Y7bXshthGtZxnXO|AII6OJ=F!VfSWUU?Uv63P%In$FF1^>rw{*H~
zeOBfCk^X7)YWnAcLSB{YIu_a7e7o|5|4dV@*Wq=mpG>l<H=X8E62_)zz&d%)pEULR
zTF-ZBf1hDGSC~aFp!H|{$D}0^vow0u;#Yl}qZr^jx#~vA{5c;dO+KK!+Fdo=;Asq-
zd4GBkv#;~qcb_gb?_TymO3}Q4O=<4Nj$1wXw@XBha`tRpljQNU>P&Ox@!yk$bWa>@
zi`Euh#&Gdp$OrkQ$BX}+61k8vttKSw&6<rWDLQxFRvvAhQRwy|B=^{9q1^_~3$)uM
zgAG^Rm2}&<N$D@^@0d+~Cwc5vBm_JA+G;6H_uhBpcighSJuG5hQdsA2J|{I(Sb;<Q
z1B-F1&$7NFGcEsceRPj$S7(xDVG{cD>PJn?%sEq{_|&65KKb#a)TMmW0){1y8>5>7
zjO<o83E%j(s_BoP_(O&5+Z=wfN}b5*^b0FdHu9Ua<=F?e(-kUQFGSukCGy4!dS?Dm
zXWc(Z{(QqVv)Dq(g4-(91)0VI^-=q|=O<r!eC7IrKo6sc-LF>1&MsV&zK$!|ea;e&
zyy<+*{@d&)t#fqxoN`XtuHv9J|4p8@bqD;r`hEp)-8tms#n2z9ovLgcXJLBI&sQ~R
zBKt-Df@4Ax*L-~teBUr-`sJyo58ll@YW=y}=(UY~gP7xysL&j(<%~0wj$L|qZ>h@6
z$i6$Qid^B1>e|0z<Tl1_=zhX|?xbnS^$W3#88h!W9{ZuUlwC?eRkca$LEwk=Me{U8
zr)+jf;dH4evtqFpl{3!rxpK*{Po}w{AuRdmsbd=hcFl;1X?36L`%>=6vKc)SBz3u!
z<Tf6b*dcYISSib5<-s*<rbpQC{#^7Y=IEpgbv;jie6eh>t6#nEyiP{(rn$_I4tW<h
zJz)1gru95|owTxa(8teT=fy{g+~`^66Z3r)+x(Ks-d|@5m9rwMw9Hupq(5u*d`SH>
zEoAeD+|=n_g3QWwQPZt{iuhmd-1D)lwqzUgtn6j!9=BF&)(ICrzO>*3|8(WB)jU7-
z_p$B|-Ed~P)TIO;%h;*5Ay$&&i<_5om6_gn&b3LVIWxAicha00M?dViRG-E=%}-^j
z-u!Djk9l6wY~Q}kdCvPM)oJ^JjhxRpoHGtBSr&5Yq)tWJnwfv*yj=2g(jIQ!d!k?4
z6ZcQLbv0Nvr0l{^14}XET^B?TIRy59`c;1|cyEE?=DcldX6tglP4Qe7FhPEUv#a&=
z($AXPRbQ>%Yx7#We4BRC^z4~6>t=0OrG1tsJWgxFMTfe$mnJKYOZ1<ZF7i(8lIBF^
z$NVnV#ae263OY|0KTAKQvqXd0L8A5e0)q?Njj#1gUO!uMZiXgTJMX5{xoge8-n`nK
z-P<%{`%30uH^ZbOiyjBI95R;OB4J^`$sH;h;vBet`vi++JI>81nZ;`I%7*(_@l#Xn
zLycXm%T3lzb&PgfBeCyjVDba)a0c$GQ-)ucXtMrUaU&w_mhPs1LHnLP@A+|Da-(LO
zK~r@zm)qrFUYF24O}94A&W!i+=}(!HwO~d2sxGPBTYpABeIk-x{AbE%4~||9p*1JG
zw`aA#@Gw;VXP`Q7*#j=~hnaS}xq^OJtF_G7n&A6GVezMeeW5<0&!bJQ&+Gac@q;_y
z^aQ_i2`@h^RQ!24V&mu3^z5$Yd}E{KAGa2CMXWlhdsbqN)eP-Kr^(N&g&TF2r@GAv
zJh-H(>$?-n{TB>rQ||2)QMCIhKV9v}QZL6R{}z3JTlrM8_`@>0!~e{0**z*g`rAd*
z_<*aTn3dsriK#Ljj8@KmyNY&wm)v{(a}4`Fv()#I4>clQ7cN!n|0VGJ{jue*C9gVM
z47L<4VJ>Ub;;oO}CABBbGU=>d$(^I-K3}`Ka}Tj!FW#5#di_LfRB@47?WFeDA3M4Y
z-bNKxMSh;S<ymp33Dfz{tyMMWDqeW3l|4AyMXT$@!E-qiU9Tlps9f{<GvlZ3=hTG{
zHW!LEMyCY%hfmB@T$^%m`oUy{Io|f?{FskV>RMtMcz&m|)Z*8>+fp;Hg>NapllJ(g
z$@W*Xe6MYOb&@Y;a{886ADxVkYAs@4w7a7|)|mZThu2E;RiFC`cqb=37xGQLc6Oc7
zAJGpLD${hA95k}d{uA-TSJ^{~Nxt86t0vbx&1RP3Ju}4)Rs4NyeqWhsqvxikUy<jx
ztvK+prQ7`Iq-AsBbNrdpEtHj6B}94T4NK<tlpe1Vc<OeOHA^<u@xn!&z_9C2Os=0Y
zGZEW=?9?faFz0hZo=q0PtY35@gz}~YZj+dKIVt4mi@#kNHnT4*(BE4-_gC)%qnb&t
z=gbWGyK>Jev-F2GT05qnc3Nj1nin}a^y8YPM!w;_vyJi|EO`7k$KjmzGBL^XM?8Dn
z)$IE&wfCHtn)lgMvF&D8QVmyGO#gyaGDV4!5qI6@zWEVyV7;QRZ0h_D#)I;w`jd?n
z_DWS4tc_jrh4b-j)-Q7e7fh;4H<eq@7Gk)1q2$qX>4!aqFGOZAG32DqS-tK<TcD!M
z;u&@}yXQqUE#cEhty-|(#bK#4mw4Aw?t5BQlRi)X^uuAZK&GJh<uB(`Jrj<)IC1yB
z)?BH?diAku;QLUc^(*ds?w|d7x73@|z9<uir`&q}k3Q;mdFWVgT_Lc%@YoffOM4$W
zha6nTBy}LeF8SBbllz(f)w<YJ{JRwV==Z-TcAsvaIdD#jVdt8sNhaGqCQLo?%Xo*>
z@7GluREksGmKAlS9A6}2Hc@VOpwG=MY5@(80(Wk_zD|%W?2?u!-`P(x_OE9h;x{b0
zUa|06!EtxV?XFcH4Q8zqSnGCf$AS(fYwfHHQo7S4ZNjD`aVS4Lpd@!@X;+e~zpvTD
z-M3}*>v!GTRFtyt#5N&MmDELN)+}5TJ+t45(RP!p)-M+w<6Co^0!leQ9!}v?dwSH$
zXQAA3Dfc<LPRX~6pDw$m+W(=tYJM7XC~I2NNhQhnpo5Fvhu*nS!ansZpXiz&0b68>
zBcHwB8MWp`)XYFGE*Vk(sK*xdpJ&c{^7jWvP06*;7R%Hby?>@gh-|k`UVi-4(lmzm
zfrgu}`W)NzX`w*v-2J<(_O01`^JFu}HoNc_dY6m;UHg7=t6OcqE3Zpm>yDK{QSBk@
zS`k+q0+~baXNNHxe@hH*l#JTBy73&>x{4hxN4$=<sa~J3<H;#qiG|l(u63w?j{1Bs
ze!-_Sr(fUZOjH$?i%7O)nXKJ?rQS8`@S>U$pEB>7ZO?t48LZ!+5fygvvPIXjU#==w
zeb}O&1wXzL(edJ_pX<w|7aBIj6^G7xu~O|<$gdR^hyR=w)4SxOf5^h;Yb@{iBdeqI
zl}dCal5@%*g@>HW&ix&%m8<?*_NNg0X5p>9Df_P6Iw+=RRP*V&a-qqL8ubU?v>W8I
zL&6$%c76(X)7rnx^ij|B8(X<!9q!BzzUf!lQ*lmP$8g>(N3E(&dnMZ^MlA6*5R~LC
zRV(1+x!UOxRuM2SIMPM%iNF!v)|pi`Y9R|RdCh7uiR4_^tnzBIXJ`oH#b9Rlp659p
z+iRayzYlj344HqkZkF2Gi-zK<6B@K81bBR5NZylXdO+uWQqQC>5szmsp6BS_r=Pgw
z=gBX#Wy+tv*gU~N-sNOh6Wia#NqLLH!{%(%Y<|Tew)y6<oJ(Kr+`rzo<T1GIS*5+@
z{K5jEW53N$D&(c5`z`CP-qZ0*U^Cm2@9Wx(+;42qZ&82sL3r%}w>H&Wsn+rUHp`?0
zwIZg@Wr>T`qLPnXa}j>x^7mTB2Djwwsj9PgEtxH`H$ix(q}92F0d;XzpJwj#4Jr)#
zRw)o-;AUHEt>Bc9vFp{$s}0BUXYn=q)v(vCRy^A{({@g|p45v~7k(ZxmCb9;j*mJ0
zE~@Nl?a@iuFBB>g*nhhm`=suC?Q!GF;G3p(r8WtS(@(tZ>7Ob;SO2H(=jgQFGfDy*
zwQOpWo}6*Hz2oD8cPooTPBEHneqSo~wnqEEYu<tzGgM;57cqGEE>)BF*({M+VKuoz
zZKI}VoV6!U&?M^sA<0=&Dl^vkDt(=FWWCC&Z33}!{oVCRp$<ilR;nDIyke=>ml)5N
zE0^R(o>k%F<II$($Uh}0+&nkYpd#d&PR&gXZuaz*JQ@b?t_9Z$FIk#8ORPlbiov_*
zQ^QxhUddHxxB8esxU=V<pjla3DoiG4_r5sURMZ)=%V92O+-v?XzP^Er%zcu2zPMZs
zNHUn~@-8Xuilphw1|`=?I>MfsD-_yv+n9g8yjnb$Hzah8)`EoU3pr|<y^|z_Y)&@)
zmSOatXE9a9PxY)yV$iDxw(duSXNrEaIxQXY@}kgU{tdQ&j!c?-RpP0Z++2P6Q(B=F
zy61E5H<~tjxn2Gf^+WWs?w9icyUb=UEDqddm%grl<_4{k=lCv`h0bmF`c|URziM3?
z*U`sW0lTa=J~d$U<5@9#MTh5^-IG0*s~@#gbe*^4s*jgm63>R#@1=IFS=m=uROaSO
zO+7pBxq-6wjw2V&To2FOJI%$QKRu(0%~WCM{x80IuNFOayxF}gopH`rx2U{7Pn?6K
z&dhQ7(&K$&N0G`(_oL^^maVs%y?FP;O}(O7Cd&@W&6DXrx25Q%scv$$UT$Q4+^?l7
zLN=YfZnG-p)QWm7`ud^7R(Zbb`|eYUB}z#qD;+MkhkZ@q-*a|KxMr#9AH%oIla{84
zdeofqnYA-?S6LWaP{pb}U-)@6=LuXF$&_N2XQ?=9um30X>jRsKq6Xf1=QwPPcXtZ?
zOYM4+{_aU<Nb!W)V?3$zU5nzn#ZPoQ&pP-q;oyhes~T6anP{k~ihS%`enhcX`NK>Z
zy@G@@3wy+M0$xAjaNKA=Q@#6W>|?Kr-wQS@D4a2uzphDBgY$~dBIC<!*&&lBD1OXM
z`g=yIb&v1E?T7NGYJXf&b^6z-Hsi~iy@~|4WF7o^B|`9l@26*({`|SUo{tRcvzhKq
zXB2MAaj)$tUbJe$m$-9hml;OS+`m|A#i|=7hh`oR)zptsetykk^Teq~L^r=*kQnp&
z*(TW}hvK&2z1QwI7Njmey*2x+=K6D6ekC0;w|FM_A<ZD^>h#Al&BqVdUHIBlC1xAy
zc=krlf^%%Mw8}yQ9OV3B<~@!4)y-4?N6?Aozg<dY*VYB4S5KTcsN~fs^wU>pmB*r)
zoCQk$1y6iDgVTa$%rr?|Ru~%A@q5K*nRK%Y$E2p3itL$mdQGasr$cKluJZiQ@Ob5m
zZ+g+1&t^|nvC*Cx6L!ddiOKOa?VWOJYnmRh+MK`tM^888*yDrjo8>3yoZpgtYD>|t
zs)S9Q+78ak0tGW#rzj=I_PIYjea>!H-N)$VHct;S@Od8ByEu>W{3><pM~6Ht?B?Z~
zUAVb?HB-3K#T0$-n0cyt=O^{WPj=DK3GH$8Hx<|_CG^)o+xJq}sYqttlzj@i%p8kv
zME_g+_~7)6iA<ja*<%fL3PJ<o=Dk#!_mX?=%Nr9t*Z0lI=oRI@A9ij1ucQ6FXHEV+
zl2l%M(8g0rP~*&+Xw~!0`a6GCi0uuZ?_yQCZhOqzkB+>WOuXL}v*nFgZ#{k~)tH?b
zBgS_}VdDDab&t2UY_`naxjATuqLTK0cAH<5H5q;u*m&e@l6=gZ)Xx+6{>OCHDRYfq
z9h`nCTJ`GcH@cgvo-W=nf1P5-j=2xT*6EqLbo~7gvOSY`Z_<oUuVwc>7Juj3HPg7(
zYE#SRnOELaNXWT<{n>q^PbpI^^H}rSgbzF?7VW5<Y4pNI*4D^(f)Vc|38Be+k~*%D
zo4$xjTzYfu$MH??LR*dIXRXN6+xtTD`Hrha!L4T88Fh~LP2;Ao?pU4NW`EvD>Xfm2
z+SSZ;Q+iV`U7mU9*`1D`iJK1J-*ZMfR3@_Jbx3{6Nln%KC!1F^n%J*M3Tt-V-@Pxn
z&bNGSOH*W=bY#%_HB<Z|HA^-JEq}J`@THRN-X?RTnNNRgmAY`^U_(l!dw=?+^4>dB
z>zeH{cD`&%cyVo}|8&b!0!iOaE-?v|llam3Bw^;v+eS}MD0ef}oBqrCW7_N}c(LK-
z2_KFY2Bme&TXY@ovWh9U-eK*F)jPVg`<5u*i6mj8o~Z&aw%Mn1^vu+r?%8l?>LzKu
z&i~tNB<iJpK+n8{9^nB#2^C4GCVYXxnn{N_OxMg8k+xZP|Kksy=)SL$H?jUTYEzmU
z8f<STxBt`4EW2N4j(04}e*E?Gi_JF!OC{DVFi2LN&f@v*&x}X^3U}R#dA#&M&HSF&
zRKeB%vpCa2&+I?G>6QE5XOF`zk16zT$vz-=KfV0#l#NgR+&H&a^3nQSyWVDrXcM!K
zx87~|U!%bH@|SITK#u%pftzoe-|4Tu<F7G6>gBJWpG^0h*>UFl%Uqe4Hx_*4J-ueu
z*BS2xO5Y{Fd;3Rn>+ZzWdk!Aj_=aH$>uRrKI~S~yn?G?=_V37Z^Yy1LeQvXhE8=6V
z{Ew~EXSd#)W!E2aW8vZ2Ygb!>E!6Jt1-!B@nO?l~O+#sY<f<ya<#W@o$NOdT{+?E8
zb3A0zw5)k2gzp90q}RV|YE9nvuJnF=KfB9rzTaO>WsB9s?%4h2)Oq+Ka>m>D$1h&@
zIiIWlYh|d_LG8<T5;9D}k3CG>emj14uW<I+y_VC=eZ?<SF;90E=sD^(<@;C3#<SP`
z)1-FReP#Z<$FD45YVCx?=sT?yOOM3_#C|((`BwXJ>i?<!4^AD|JiE_1%Fa*d!py((
zcN}j0dD!rM)tTQdvTr}>NcDG>h4_5^Q4zEIbHwZGm9|%y|A)sG?1(pgx9fj{=U=|v
zzuhAiythC6@3UfZ^H)nb`~Bwv*GM0bwSQ<Rx7~HA_*L#Fr~j;yE3k9=m~a2OaA{qv
z_wV&ZVK#o(uiI|^Z*|Vw`u*FoIlIevmdh@dj9R$&nav^n=UonVA%2N#1AK3<U!7L9
zknuUMSe^mv%2f`jvX6_kKE3(#_Tkg*d;h2%S;uCxPsBrif8~Y$TC8VsSYDsv`nab#
z+1`Hs^+bzzyQkmYFn>$wq@%&Rj&69aSY-b@^Z3^2{{CH)+JnzD&i)>pdi-y~@y^{7
zvdt$>_}J(6=F_@&>M3#aMSt)9I{W3(W*3px9q$a94s6i9b@10SzQv2b`o#)bMBU-u
zUgcxfQoQZ6lzqnAvOA1t`{L{F%z67jB+9Dp>Em!qZMiSL)*hR#eUJaUXZ3E`jNb~W
z=edoqPo7b|G^MY0!^v~4yw4KkOOC5dI24n6bXBo<{zqTCV>y$>XWRQf+IzG3`O!6U
zrO`Ir^L@95|Jo`ue{RC-GWN?|dv<FCxIVJgoVoZF=U=&ZVXtP~Ioa|mMn0y<B5!u7
zPhbAerH>k;{lwmgrpmfMTkz|V<IQ8@yMkBznZKy`ZhzqR=HI8^-k4dVlle|PWWMQn
z+o_3nj{cN6A{4osRp(mVgT$z-iW?4V8?Aa?6}xG2<+pR6gZw@^r<||r?%Zp;-1NTh
z{<k;c>nCb%d2DB`-&gmqN>Jm-XEmEt`zb{^_Mg3<7L`;hT~9u6_k8Ip_P$RA^Ztsn
zd{eTwWW4Wpvw3cjTD7IqH>J!|a~!SKb&K&OeZ6%ixh^bXtL%|%-R<A~EGMk{ab;Sc
z-)G&B8;ooIx-=i`@Bhj0r0z;i`wExuzy6e-^8I!5)xnVM&CB0CzxeX_@v1Ko{S&uu
zK3=8pyU47Vr}S8L<m=*l!pHLdeXqGyzn`&X{qmDSawVL`&jgL645U6q+O2zWsKooj
zf#V0u|Ndnze`Zu;uz2zR6+(AsDVDYU|M6vWXRw|?#~~);V&yq&l^L8SWIs~4_q1V&
zjY*%#)eY;{_VC4IUi<d7;k&EM|CA+N^6Soop7Sl)v~CO2x)QIIJm0>2)hL+R{o}h~
zrS%T6a4QLE87J?B>9Uiwtp2`n>9*6l`_^g48r_#-)*4Gan674+AMfC0<&U^$W))Xw
zd&nZ@#9zt6a{@2t#d$t9d^F8v7VD$8ms1bjdXafuV#N|pO~WIdMXSQOnx+2EmD)RL
z!jFBIyJq)_Zaw)SL$qH|a)Tm6M031Fv7x!dKgGA!#+{}KGs0%P+^y^_a>(U6x45f~
z(%(d`$Ggmxbkh#WJ-fX4?TRCp{n(|Sq$IJuSjKbV%-1t-E*%crsNBF=#H?ALuM%|h
z^tp_Mf&9y^95rVXextC=O1Z9FV2kX`ZnsC_OT$Z-IV`<juE@01f8nRsM)FYst8}bZ
z)azQ>)+9W#31So4^E3aE?Shml<DK)1zL>gAOo+2?caey5iZq)#@w^hh(wAqa9xq)Z
z<-vO5gIV+B3m236c6O=Hi5L1S%xuH%l*D#8k)?l43g@GA)k(qsbmr%IK3IM--2eKf
z0}A5&s#A6e32E$G9Pd9ZeVOAeQ|&@4BSy{3{hCWBXs@db(&+fJ=hFO^e{EHIkA6;Z
zl<#7&UwVZ7VlBtJ$B}OplqSo6{WImqeAoVeMRr<UOYbfHzkEe`sLjph625m5=B)ZV
z?|!U)sZnVl8LVfmD(hbVbb;1s$NAHmm*}W(jp|>zl3g>LziRWmsHwlseP3^P&h7lt
zjgz0RIxciQm|G<7MabsnAND7;_|;>B{nhRr_GEsZT)O=JDbo{0#xLG}R=lQaxa;E=
zb&;D-IF|poajx-7a?s1xnSSc|Q~mbY$R4@1W|x1`)*3y3i<5h1^zW(*^JQOKX%Z55
zta???vg==TYW&pZxx5biqEXoss`6=nf83w%<^?OC#eLcJH01ZZKTXGr{=IQe^;b`r
zI{nhu2i{AKogP?n%scAyUp``YXG@&RZT(fT);ANipPvu!DJwcqc52Qaht+i}KdQx9
z+TGTa^E<vFPAAAu@~CFXiQ{iSDv4RLzIrKhz;gQLBs;rhi;oAl{n@lnKfB<BlHXjf
zx{G(U8E+iCeycP6y1#wvnYq$}msfJlz9pgb<!iK?^=sEE%?a^!)2HA6x|K0V{Qtk;
zkINhPFJ_$Xz0UW3;9_ZQ+v}TUws1B+_GFu!cP~_Kk7$TG6KhK4CSUu_^?(2Thn}vV
z`QX1hZwZU9>=X7C8?V>Pz4p(Ta(sT_iA1INORaymzBp~KkT~zejSr#4Ed?ho+nDrT
zpYZOYtli^WxqQW6+B!N`xqIAL)U7^D-m?0!Zi;hntLL$;w-??y_vXFUkF5^1CqDeD
z-~ammZ`O%hIH$P&7w`Y%f6$)wzsR594|o~Bn+`i{zFaf;VFR=C>h}7(9{=ZSN!b1Q
z{^QF-|Hl8zx!>q5l&nwxBY!NU^V{wIu0@AkmgTzs`nJ9G?ePr^_w0BVG{<I#%r~=g
zy_foATHT>O$!kY`#eJ<kGG{*Xwz|E4%-7uhFU|VrSF^R6?}W$4PyG7zVMo2kb@ua{
zpWWHoe*Gs`fYHDG%^p0r``zz2s{Hz=ysFG+^HU4!-{xy>&S$C*Z_JK<+_11?ieDi^
zpnSpS4hM#hKYyP1KQ~!2wr}30AE9h7mNG7KuKeNg`Cd%DPt3Fml}{Ghn(_6^E441o
zIKQE%|L`oIxy$VKpR+Hjcx93{d+Db)I?UF~9Q9L6zRO(uAbHK~XxrgK%KiU@Oy*R|
z9qxJAW8YvkkyF+!<InmBENy`c&dfXYf^+%Dz=~h{4)3`BXzI?1OIMsWV*jXdKk?1|
z+lyCUTz&S?*TugN@JH|Ty0-IDsk^=MufrRXenq?WbZ{SA5<2x-phsxfajVp2nG%7@
zKNvlgc4QWqCrQ7}+IsW#>aT1Z!mY>li*D=Y-Fx^W>$z?%uM0BL)<sq?pT5|3d242g
zPr$42>vLrtWBGGy+NSr2ty;Wz;<ks}LUOk8vR^8Be!W?--SWaY&QsEC*Z*^f%0HXZ
z_MY>14bSWX8Pgwo9^@z$x}OM-^zG7o$(+CHA;Y3+?bq3jZ?YT_6WZFi;%ipY@6+q|
zKZ|8Kyxns3e;K`hB0qL>Niba4dB3}93-|vUu1@djc{3OmCW>=@G1z{4-@i*6Je&`n
zH9tJ5G1IgC_tf~3-p;NrKI{9g8Il<*T^--2m<u`Q$@A|z<q*!>qg;OS>a)Whk+G_f
zt2QnssyN1Nzxqr5ebt_AMt(M#nQxb@dndnPcUkPqX&n4-PsX%2Mq0UL+pF30ezz-F
z{_t7*>KPB8$%n7K<kD~;;gW8S{bY02w@YKi{~tDb$un72?_bzA(LZZT)P+4`%lAM0
zwA^v;)9lW5r{xo8{>#4owSG}UBiF7^BF!RF8;iJ#XEf|^Ov(wcE`4^4@#y3kDa<Tf
zkJp6!ST{kU%7AZj;%(k9$9FulGInM=@=s#h*9Fc#fs-e&Yw^_oG*LF^O|E_7ay)YG
z&3aGo3+`7vybl<Q+Ra;RtLDOJE7*R0b!QC&Yl4E}zJDisR){`XH|>Ft^o*&;y4U$N
zY1E{YJWT)f#OQX>(&ZW%JWDn|NwvTBEOCLh)#7*w4eofm6%Q+)hN!e&{BZoRN6LZK
zQ&z^`a8q^co;N{(ZOSo;3`xKD3g&I^746ubJ~!K;-z-shZt{w9i|>~{PxJfCcS`wE
zm*A3iUTx<~rxx^1&^AbDS}YzQAE@WHz<ytq>^`r?<pF$09H(3@s91a;g@vOq`FfEx
z^GC+n9_(KmOZLy~Tc5H|&|~8Dd5KCpzD-!io2ky(Qc`koD!X{%&fd>KK^yqb-gu!R
zr}x=kUEvqc<)*D?QsXusu9S-0?C`sE=Nm@uT2uD#ZVib`kE*g-it<h8fBv*}N<4#4
zC7by(x8>q{nNygA&L2Gzk$3&?%N;w8iJHbRSLPd9&Ut)mmB+<4os0*oj~zMbRVTa6
zu<(vVM}&#gfwc!;+RJ=D<{@x-rn0_4@#PPD556^6CNtAf-5~n!3(b!v7hVdQAIPY@
z^LGz(TXl`wF4JWi?@xU9UUo)8EA3-_-)siXx5_Sg4cF~1c|S5*p&UOYz3T!8_YR&3
zaveKZ*Y)q%(LVo^xXrUdfd^Kc&q_|@vjr>*G-(c#%)H+)x6S>|kInqwR`f+#Y~U5z
z<9Me(JbttDw$<zRi!LfpIFq=6TR)B2U{aC9_4%LvE?r)9uG#LOL!~X-lOuIJ@{<jh
z8TPbIG*{&Q*39a(w^6dK*(M<C=(p9~EhXPAZXd49F1b1HlTBsQ*+Pqa<!a5c51N+g
zhdrcsygV2^VP&s`YCw(&?~FYYCmwfrq#TxW;-lLZ;|*$$c^UM5j#wH6F{VB|S8vpJ
z<V6mTai;6Os(O{uV<`nblfGWf?5goqkiC8}Jm2|U0*C3rEosS>+@Fsb_OLby@w{Z@
zTD+Sh#`Mmoc9r}&J&h4FG^7^__ArX>pP*^X(#P`cgu!W@>9Tq+1z4Cw9z=d?eQXq7
zvc~-x&wd5v@?2-fdS?AY^Uo?c=|!Awh?(=a!&qMbGuNU6$p==%)J`m!z;U+FxPNU&
zk0ZkhN%kU@TTMEhH*d{5ArrT8kz~$)i&zd`hP?T&J9e15ee`#2Km7YcHoNxs^^=0#
zkH?p1y0$-FVA!qQ{7~vU;|B9}Q``LZH!M`hW4H3{NKbzpQr{SIc+#UMy`SD#^#09u
z-7NQG+rsB;;j7x#X$MVLzH_+Yh~R3b;-jutHqJa@alcTmP}+t~q?UJ~(N6uFxf2w>
zEIV@c)CBP}Zf!z>8?_6r&Di~KWy>DMzpfKm4u*s$Y8DuXsxqC7W8UQ=mz;J<Y>vV$
z=FDydo&yW`YJNIAV7u#n;^dzeaf^;E%U>`n%oQxp_)s0_YwUT$c*7QfIeE7f9XPyx
znN5fdyjH~Z*P)s5DgP?gBj*1_r)T)5WvM^X>~ec-Gw0B&1GbOv8wt-d-g?JD`H16*
zx1QU$+L;<F8-7e$w1|N}aqY1@264r(_YRT{?o%iDm!$CgU~YLL-k)9|*?a1SIomql
zqk@HO=MQKdc&}V^*Rgj&c}%c_FRS17gmpY7LYpS<Ry+He|FrLlqpHCt?HF%Ojh%C&
z$nAULB=@Cj%-U1}^lT5Y6oe!hdCy+aZPJ|4!#;V&q$;&8p?B<Eo0e;6Jd7^zlyM15
zvlHU-kZuo|#T?SPdY@Q<OpP{+*~CQ-k)nRbJ<eKBOFP4(!7Y-iJm2AnSZ6M8VMfDp
z!v?J=i5=QP*BW;TxEA#A-rx?F>Sbzf+|3a(mHAh2z^MqKw?5L{r_0`$8{GWC^(kh0
zb7>J5TbDp%QsX(ch_X#wZeGO<Yywg*PCj}f9<guY3Krk15{~T&hQV&l-_m3knX-i&
zDQ)9hV6J4gG4eo;pa=7xro{Q~)dzAruM|Iz)5tdXFgtX^wFCLdw~{5T`R%zFj`NC4
zJ*g!Txm+dgphH1(zr=|^7KL}LlfE$2^Ub=sZob+_%MIGgI_I4BSL0USe*Uih6R{m@
zJ@n(YM27Gvbbej%)$#p{8;y|(2f7@WGrU#g==v0Ew7)i?+=XqbOtaw@)_rF;Hz@2n
z(9n6|xPTeQS@9kl=JFZ=eKY<?4%bxnEwWi%q4?FIjq$}E4QAD_l^P3_J~BKIJTA0{
zN&L8A1@m3)H3weoPIoQNu<<?C;XQYSpo)>_M#Ia3wX84nA4ajgwp}6ZTC29qsEBLD
zd4UgXn<`tjGJlf}_I)C}Va|yOoyi7lhKH^RGNx4&1d54#WSQ^qq#;26NaMtb{X!S@
zlN6SyxHLDUd93R#4SeVOP+xs%n8%-`o*mbG^bT@t<WoyLQ_R5Nr(RgkIz7SSb;8pI
zUiTj3<1K=X+=m&hWZau|mN_o@cp$@FWou#PgCdQDTb$ntPD|&zH~l{7Zn7et%kRwf
z*T0rj$p;yyyT|)5txCDXGee+nhEaZBoyfn;0*=F%^BGufd|i~X>&1MT*3UDRu!tRW
zlwkevL{BpBxI)1;x6HI3vNBDd6&mcPvYxTvnlaaS#-SHm#kd#`s52h4)H}>_nuT3a
zxU*AJZjH}lD+yLV>*mmc)cvJLYE`aHv%Sm^@no(0eU2MFtnz{zdlsHNm{|X?p<~}c
zxAYA>i&ES+&oF*Ef8*+I!^uZFCr8NKY3`5pchHq6PYfwraCH4Sp|Y*~5|VG4gZmOX
z>JR>~=?LqN^-)Nl_=Dw5v$0F2o8K$J**t$9Y`x*k+8M|GNY?nolp>alb|Wi2iQ^OW
znO93j)D~@iEF_!LV$ABnpsd&)($;n~@FY((%O*Q5jrK#C43f>q-U+Z<{@9~%j%Q2b
z#MKKLlvys>?Pzk``bgAc^~3xuP90XpO9EzGj*qmGTpIP)C}hMmeLfgsaq;H|AKATX
z=N2#>GSY16((HfzJo`iQ_5#*vPCVKIT%sFUH8)AO#4`8r?lFmYaICu9FmWR@XW{o_
zrv>{3E}L$6%j-T}#(Qys%*z`m_ewH-=T*Pe$jk2kaL=Lr=N@**|8l8VSD0)Zr|a~A
z@!j#XQaSZCQT~%VN}g28KR7s{yydF?37I*p>uz%y&SbtPeNKrrw!_lh=o9BBy`Dbn
zMRm$yb0xAn<{pTYPPSM2qcoFyj==@y#^s$=$E-GnBu=rIA+GXCf;q5MO>SbV(1d9(
z^~~pMaO$+_i!q<*>29?aUb0y9?UWrooxD>_B_|lIP=BB||Drx)6weV3x8@T%T6uPT
zC5bm48Ooo?>o9FxeIU)EbLpB3h8%7d$M|*3`kw4~-*8l;l6ku0VZ9ik8R9>=lls&a
z&3VM<vGm9^H#gq8uuUS7?!p-y9y-h!2jde~Hg0P?%r^C^WUai&_7yYLKY2GCeQ>H@
zo7+Ms_>{YhSxPw9zgGJToEIcgR~|`Pa{h8m^#M@>$t}hyMn^K))aDo5N>q6BK`rda
zLx)})b@@ukg!O_4i%aI6$a=OYH0Oya!^2a0>$p2ku5LWPFz$dUXB*d=w#i@Amp?p}
zdRE}_f(KelQ~Ux>G@d#3qU6pyhqT`A*p?$3U$;&-36!hgJ(QhZe`Ec|a))f!gRu?Q
zm70$>UMx7~uP?z?Xj>|zr^_<QFi~3QC(k<T2dp#J3SHq@(c7h^XcmydA#VRcPpN!S
zwajrb)>ktdZRcM6J1MCw@ZQ;mkIY*gdrq9?pL8{OMdu#YhuwmQGjkZHv9z3w&{?#@
z&yuf4Lv8zAPtAu%n2tKGIMefO*Eg=8T^kp*_Oy1dWSceniP1JqufGOyO)U#&=84{Q
z+Qsuix8oN}UADrcrfF4kHYh}`5pWFZ<ZyG!=9ne1WWMlLLmO2C=`Rrp3NFk{j|7vi
zt*lfM3OCs=c=p;!3y1JW3DP@kFC=~0Ii)f1io@$kxdG*VQ_P&-cYbop__CzfXo6k@
z`@EZlj~XuTW8>JxzPtMc-<-U@6e*eOwobbyYP~jyaJjZ(IfG3RzoKzbQ0kt3rS%(h
z4EJ5Wb;5J1E7#0}rV|{Kf^1!uEy#0Rx@6&Rr@lxT=JRpOPBfcu=vI4N+I>em#7cA%
zcXwC8rIgR^q1lW)ha=nyx%rk&jc}}(WyQMu3wJk@1jFKn>Y#};9M(%;{pWx4fZ+j?
z3#=7P3udl(ZDzM}uH1_WE&N-!{VG|Tl=&ROEGA4m-rTl$$31~+OD{Ido<CBbrfo=e
zRyT2xc3h_#=6QapV092*%YWAk7ZcLl@>A})KY0C|$IY~A_l+E1hGXG;jfZcv9qh7m
zOkz}D5N`C1vnIaIXx*0|h8upa2$;9TsE|)Pd%~Q?gk^@=-0w7+S#?wm1sfkO>{+{^
zt~aHNt^B~m3HAp*wi_kRQgivjA$suvkCLi<v$Vo=4kb+{-_Vv_>K*PMSSFs4FX5VV
z<bs}iF3X$MJN`7Qc1%$2DW2dqU-Z=bkmoIuvieMS`7*;_-DtfS<hGqtd(zIO%r{eg
zggMeSu!xk^>DC|1<=UDsPq<d8<IIA)Rx=ojDj!@_-kQAAzI9dCf|3=f-GxRHQ&;yh
zc8Avo+;B@()p)?f_CfN{MW^e12@7sW7wr1zY5P2Z(cmy+ovFZzg}xKcKDlw-Gx+L?
zO&7FWxEA;9?%i#c=&0E)yL{oRDJB#42nv;_Hx#|TBpf(t!W2EPW3s*KtzxQ+!|&8B
zKF^d^8t^YUB7VJ8wM?lymn`dTmY4F!pYds}`g)W9?0+R^-sT_IZ3JJXCoJ6fwzlME
zW197a1`{q1Ddweue$NFoO#}W3a_nX<aVRhSV(~#lX<PEVDKpt;KbW1NA+Rz!t>*@(
zu}39mlxJ0|huw098l|tbVipgbZkd@lhA6+3cYNzo<gKEo5X9;v<(w@t(Pd_v$|r_Z
z(<B?^SItqBV9Glbrz~c#@xe<kA<56>Df`+MgKx3Xk9aQluL|15_QrvUS->^M<U;Ml
zm_+4cy-O|~U<z2z@XJ0ZX2wZ|4E92e8MX=&|MhQbGgzeIzDKEvi{%MZhlW{ePJ`bR
z0js31A#87+iZyf}Soms5N5cjoL7PA^77?~7tc5>!?C#hqkZChn)Zt*EQp5JPP3ss_
z-B(TiwdBN72S)9Nw+!iwE6i8QcNi_KY?k{T>H2`NUUj8U$3ZqprlQL`E{86fTj=Di
zW-h>K#OW=wwP{*w$gxX;lPw!0-Z-?}QJy3}qkYyb_IXmPg%0xG%wFZS@Vg6(HgnkN
zBt9hrv8N@S%*?+-Dy+p`oo=|X@4|%$h6QGKIvOXq%;@U<!@!rixR%#F_X?l#A-kNe
zj;BjXW`up%Z?tw-)>^Ny%UPPj@u?010*c2t*34k_+RAC7eMMY2t%9dw$HF#&yd@n1
z;&YcIU1KqNe1yrhEJK-BL+Ho724!|{F^SJ2C)PGpc0Ac8RJ=ei&ieDxMxI)Qr>YBF
zd_<qQxXldT<TE27$#BabHlLF&o>MA=5A?`S?d6@pvP)*cK^Xyy_3P&LGuNBXx-wyo
z(JNIBHNIq3LE{Ad2JJa)zSnEJw>0@*dd<H}JHUpgQH5o1-(s<aXUk7rXgH+xgF$!?
zUsvRn*%pl_ob6xuE$~!W(a!9x^r#|S<t<~qWD2KMdT_+|kF2kH&%F)lTFCkMN}}tn
zy_!jHXNbSaY0#8Cak_S*)P^UOUCwNaWLSP2u<&zQXTGfJStw(&^#YDh;f6kcc+(o9
zv^^p&3iIZ7DjRPWw0OCbLpk6EyYE8t%v}pL(?YTq{^UB>G}}ncU#FAxqlsqmoxU9^
z^PXszESd16U&j6?SDaS!N|vHEPaetMFm!(S-l1>e8qtEs64^}w6GcUiiFza@@Ju)4
z7cunvrT3Z7`Ibc`w_3@;DZ6H#J@kX~SF_Vzo!p)|$2ccsoLMTzQ?2yr^n<$%msU=b
zQ0nY@&-|26F{areBfw`9_aas`y<jQviPswLh@?foST4G;%)h5&#jnn#3mO&PM6|_i
zv0JHd(!&2t(hr8tG`(a8iAtV)ujPB)Q!i8n6`nfau0C_4<rF1HwizzRoIEB<nY4%M
zUO!gQ87zLK^pE08U#Tk}R?XI0u9#DMD5A)uYq9^bB)xARR(XpwA91q3z$P`{%wS`q
z{n4e?pVnmT6}oZF;jQnbNqi-fW9s`qbqcQuFnJ&+r<=mDS~jXp<@}QtPOl^m%PfhK
z^g~@@N=taEBsm^UTza(q(ow_D3kwqJAFLOio-E*OncOYZBQE?QTsqI3bKSa~vE5%!
z#AZhy4Di@Dq3x3S#q^~+ddk_f1vu5WylDMp%@`ZdV>xXZLz>%<<L-)QQp|2_;Na}C
zD2q&3#&SY($CjkP?D=UzoMJ(b-+OG3xW9_w`Ra<a6K5RD!pgd$*DeZ=`{sPh>(Kkv
zHO~*eK2ggS(X2P0E&PoBktZ(AXL18|A94FHYCe-2DZt^q)I}^^=1y4Zq2PqAj=DBe
z15fM}4(X3`wVBkzWYzMqK-<eiKz8SBp@)TUo}GBiQp&QV$4ORpH|L`@=1o#>B(1~D
z{k;?#y&S@%7DuhoSk9OCb>SVCThbj4FH@FO@~^UED4LqfHlanwwDX?Zk~Hu6oasir
z2b{!Ds9s8R@tkSr&aC4dSryLe_&|;4x5%PJDuT8X1UUlPgB#y^t7IRavhuK}((;DB
zI=30ITf<a87i@hZY~F0i#HX#Yx#3VlWWu!r9%~-hHFzrR;J3;-d4kjW?#h}8yArB4
zt4J+0soET1!PmS=!+5XW+lvf*1(tylCJ(rLp4hORW?RA*Tr3>M{lsS3&#ImOqTbIc
zkSfubIHCD8<BdL9hU<aL+L)QT_$$^;loz~ra*xK&${nWM&kdp~B7~Y)PYLg1=JxDy
zU)*^{w&57#gIlT*7hd*Fh+UdllYXFbN{~dMT)*gsc_$aKR`gfpCfH@&mSejjzWkBB
z<3^X_mQMHH9sLhWPP=S9BG9=+@Pl-K`n#Zn{YIjZ4h_9*Mkdx_Rb1MiA1%4(cwI7r
zY4+~Te~)VSdde(fV^`gIRb@ugBW?@B;0eBSI^sHom@E6)ud!B}e#v&|-lDT`muX;@
z)wJy&<@!au8SIrFhp~L0yP~l3h-k`|kC{A7oeFH+*Cp=n2t4I0W4WMaLtpQ0!2^*?
zGJpTwp!Z_&!@#IpQWH$J{A;=()T%CMaEw31GrIPoze1nz&;RX=p-hLWua}8U6y9*B
zDCwljs@{xELD>gOkJr9GGRYvH`K`}olUEuphf`jto@m_3!zbLk*J$J7JGydtO?EH7
z_Uz^heEh=h^8CXrwavdw=ABk4Q`zhs%*MB=i<QT5k@lOzz1p!a3MS9n@>ubt*r_v%
z?53Z*%Q}y5SMDT<cb=zTA1L=p{?s7L)?u7y<=+r&YZ$h*sh#0anTB)28zs(ND|Q<M
znmZ;LHg-ysPRugsx|12=n_%P9to(=dpQAe0uE{6n$<E?T(q#`~el2;>M&(wcz0FR)
zS<5STWgKC75~Ru8zUkq^16i^UcX}T3jb&LOcf+mhz>Np}4d+)#Ei-buwWg+2gGtXo
z<|lvt`gpM&uM@WNm#Nq;ch6t;h3{C*Pt9$1lW)s=GS64CcTjdwQId19V--EWQRstF
z<rVeX&(Ai0T)?|d<+Y4m!W6|PSx-b9GIspfQTWhq>D_kw$CH(mJ$mPrTxxU+R}%iw
zXshYayyUTG^Zt)=cllPEO#kb|-v02z^@2@xFT|%YF;`R?9GubLr+q`~d=P&>ukMUr
z7gGEFDR?_x`KJHp%7Hltl4@s!yKVI1bDpWQ<-7x{$sL(Od57D4uQhx`p68bsbnGg$
zGgAFB(c8+0Ik|aT*Q?7nE*3mEdV21WvY9tb+6o?gC|>%pY+;Ngqnl?;a+}`N9Ht!p
z|8q^sjkf-De?IS!ZPFL^w=Yj9`MOs%u;st;*&*rveWKxRhx4m0O0M@~+xgx@vD5cW
z^PjpWX6e>C0t>^m{EOy&FqUx(XnFkTSN31V7c-Ss_-rzCy6yOyxj^<-Z)0yIC!cf0
zrklyVr(bMiSoZ2g*t5RS2IgLNH;!8OdEXpFwY%)J!?t|zTfrbTEAGnc2{H9&wyuqd
z)=)mwGO^9h)7tDNchhX<Hv+77x0c*{m^XFhft;teR<#{%eUh9~&AO-WXvPfnpL_KR
z&N@Gw*!o`H!_6UljYq<p4VM=(#l<<B82-zWj1c^(zHw=YP(wG%YL?*TKIa#;H#RPE
zk7D?9LD={}$$^AphP%#OzNDxkv)60!)YJbg7o2~gl(cC{On;$EM)Qa497{I)yIqIB
zPS50S-^A0P{pwbSdV*oPL%@a9z{{PBxjB~?{#9Je*41=!M`EY)Lw<>@6MUJyA_fOp
zxQgc;XL!T<lR1L_Q|W}L-!>tFt9*Z*?eX&4^HAZ$rzQFF69jE0_DdVJJik(%;<WJS
z>zK$duXo4RuPrYWozuQ%0&C;5xa%!vN}DDWt8Yx0$iQEmsbrA0##q5%`Iegvhmxl3
zknr$Po2Vu(azeHzgXb`lRQHt~D(m*m(cK~VL~%#jf<1+fEvII)z3{oWW#;6*E7b{2
zkJC;~F<VqpRrg-{|NVjoW)Cji?$Gs7V9IX)f5;(heN1-ME9Og*Qx~59`X}6S-~a!6
z8IJA@f5Cfab+`&o%*xl{cj|0;%3uGQG2b|j{jb|Mx7U#yWkr8~dp-TJdCW0Yx!@I_
z_Whl%|8+S(TToLMzlPXxtA^_1bM0#Q{+&A3^p8Pnb?QC2MHUUe97=fdR=?YIO7F`z
z{xkAU9tO%+LMEBqyvBHhx8|v}=rqaj4;yCWEb3)6QYerpKj83);hECZX-sN6^tW|N
zt(_>SQPR!4koms!GR^I-pE5KQsxq#7_;2p~Vm<NSgDalj9AtfWD{SQJ{26@mQ}nVU
ziousV?`=GGD%4-mD1QOR#b&;96HXmk*z;L(Td6{uf`#O<Sqg1cle8u?s61?r$!%Kq
z&%@hljqB0~v&&2y13xhwiBTwL$W=SYoc}fP4SN-PMa{<PX6OH(efs(O!lS(lEnXfD
za?w7-Bh1obtgG&^f@R(XPbWo}gU&BE&Dk`uf3Nk))ozn+o@5H+DDq4&-F2aPpVpG)
z3Eh?T7XOYpf0{2|px>18z3T7Pb^FzBvUB#t9h3d>ztnhE{SVi=U#@jkKG}9Zt15LJ
z*XSGn4nLc5_RX#NJHDyM{+|4g`}gCQQLzEnLmn^Ry60uoy%OE`s&XH^?HIgc_MW-U
zvnjUzY~H`A)*tIvF}x2F`PeNz^I*^MdSioxK*hg;`5ImSzs9U%@@zg`HEsIAf7)z6
zA2Wq6&ae8mQ@rf$xjS;RZzq4$G7&%gEbB%L-|Fy%-x96=Pv!m;uaoe1yJLH%>1Xcl
zkNfAJn$J}4&dBy6fg|h)n;ffv+3S-(j(3HKKk;&Zzet@wNkH3i+AGNwUlT(%2D%1r
z$@ue)$2jhXv4NLGbCgry5r*@N`8-~&XvmJfkbkUYokqJsk%6Riyu=c2J`*l(>3JLn
zzdF2GYm%(+FJt3_x;BQ+j>M)FeEcp<Iz?>(70DexG%MOI1=1QSqC**XKlrm?#(s{A
z?Hd9+PJDOJu&bD<_(QJw-ND^n$9{5M`0b!$d+dUAsDw#=h~tDkdXitl9or5GD=%Z%
z9UEA4m}jN*iH(j2Pbuu;nSc7k&YoGP8+P_gJ#Da)-}S^+$ue~fp>-$q#inwyGBC47
znjU@Oo~r)ig7BoUWmo23@;NBEKHv4%e{-3>e%6d@Csz1#T)ixE`|Z*-@s9VK?v%YO
zaL%;(zxwGf&e^B`-u_?G7q?ILbbY?e+x?b&QHdG_lbim2d|7oVlkJk-z7thqwyWp)
z`kO{vO)xbLxSG^kwd&w0Hp5w~+*V6Xbz@?CUvcqD)@6g)@l);3Z(Ln)onx2V1=FT^
zSG~i($KEYJyYjTM4U4U>|MlQ*maj~4DYo77zDlXBasTMo8x@cmyn0!f?yH$~w#Wag
zp7>YGs%7d^zjvb3Ynh+MqQ7#*-ByLHFgT#j>)TtGUEyLEv42xt>fckpLO*`hlFpFH
z2H!<!2ELTg_ub>(w<g;z-|$H{o_qGhCI63n*0U>1wq3r-=cBje(lg_imV0-kS-QS(
zu`#a<|Co6`^Wur+Qj=b51(?md{Qj>|V~BGgud8Xv-*{eEQ<2S56WDES<}8|XT)B4s
z2cIm1l?Ruw1&1(u>MpBs?9hDh@V`{X|IJLc_ZIy=clF+u%b%FJ>ZZ7R@b2+XXKjgY
z;ywL)?=b@x*|bCU`|1V1dbVziUwr-T-Q$^$m;Fdx_+-D{muhVvjlaCCZ&@N|@TLBa
zDbV%&v77zxchQV39JyaE=9Kv6@U(D<OzCWL3yIbfDG*S7V<^mCt9(OV%ti7Jqg*RL
z+h^S`Ns@|YMn757Uj*7noL=pA^zYx}VGK(+YM!&KUHbAn_m!l>?OJI+lvRJZe2RR}
zdAI(u-TD7EIzEpMvHibwuug~JROY!uJN>s9RavHW>}nV2dg;=kTW>eHAoB;`iuH*<
zPR>*ilkW*on$7O5BEC2JPpr6A*}~}g7JJSwzvq6|SWku{_^1Dk(@#$FetskS^8VlF
zNgDq+Cw%^5_Vd2QJSp3phP-JLHpaw;JvyHDBI~n!jL!d?Jc5&j1Iq4A*^sea^5EzC
z<7P?@PwhAF+|DXc^J2F3*YyVie()!}`0vB$!YCDfY6D}%&Y*vx&7nHRtz{Z2(p-gs
zir;jUq_s}Ad2L!Kq<D&ZCFq7q5s8Iip()l=-<m*@!rPY3_YY`A>a<pPa+$>xar`gs
zp3?9`D<w%#Qz%k*MunU3<(;7d|3e;mvl<#2-#M1Tv!3Z}oK{D!5PNo#>~^m8wjJw*
zp6_l``FG_=^E$4qom;MRNv*zdCDieB^snsJ8nHL;_x!PadjHUQvuFR)7#oB;&X?PK
zzrKqpV&PHoX`FT6<e&U+unehsD!jIS4a<9rAnW%bd~a6tX&Z)%{wwzgvi*JEgpVc5
z<-oCTU%u?(i3*76-Ji6K{qp2@rxln|l(sV8UjJTd)!*x>|Bfxa^FRBgadA%JK~Ck|
zx*azJKk;2(XrU+0{_N47J+C|`q{Zw$VsWEp*1Rru@v8|w>=yrzsa@Z!mSp9Z>XxeC
z!?1FCt*p$EKX){*h?sw}|0`S2Au#p6#`k}#eWD%}&3Jy>@bBT|Oq*8c$R8gKcsC#T
z63jMZ=7fF0bN(5hY5lsLbC2jN8;&lQCiA-bZ@)YLKKxJ?e(sg|-EXZ6q&@~F8=tL9
zcKpAOi9`2C;$(X-X~x;9Z(rJ5B(w<q&5N}<?td|8c~0Rwul4fJzvV99``7>5qxYHz
zbU6N7FO+inRA=#qV?yWaV<l&ASnvL}KW0z;4u&-s@)ahX{<wdk`M)a0iMtlnEN#u`
zwwRjpT=8b#lEQ%14`+T6d73lnhsLj!o}b*m&e9H?WL&E{e^P*0=!##mU#$N)7`R`J
z7JE_kfA&)S{h<tokArQwC*J3Kv|0VP%n{x0XvZ#{4JRMDEV|-Up|x}JA=!zWd6FJ9
zOMExr<xsnCd-LVVJpa7PPg^ES&eO?W@VoQq$u$3E6-5^(KcBlMN1$Hrh{<Bctt+M|
zJt<t?=r{oh&0u4DSgE?@Rn`V=cTbM0S@YH{7n{h&CN%lD0&}whb8}{lKne&Q$nN5u
z5w&H-aw!wjjI0Y+7OZr<tmG6Yt~#fG(&6JTPioFAm@+}k-}FVPnZe_v?2bTQH&YGP
z&DwjoKb>)Dt4UxE)OF)ETy<~-+hi%9oDb1A)^$xeXsj1|=&@_N!e#NueJ4J+mgyT+
zSDtv|8m4blHSuJ(PF%_drE2k+yKGWkDS3<E+<D@RYnYx<QD)AvjgmY@M)Mz4Y?yg0
zO=|Yb9Jg=!M)j2^{<vP#H>#WXvRfxU<%iO4@yT~jC%dT{N$#7$6Yo=|Zzg>9)Vrf8
z-;)w=eLV5vXv&wQO_hc>_sqzN^D)!kl4)?$MB?y?Ppv-1KD@i6_Dnjn(lNYwUFhP%
z=94=Pwj`wQ37h*k<M#jMa}S%ebV)^lOWj_bRR?D-(_4Aagm>YjC?Bb)pk_~AHPaba
z5`vmnu5LUd7M~dT$}e;JEFF$p{6~cTzuEhdU*v&Oy6z|Qf3`7y_g}Vu`<?5C+Q*H(
z?Y{!OTP%<HGYB4#k2ILhTKqrsxbC6`d9P=k{GZI{`0SL5)_IYx^t{jdm{m)$telHO
z<c!xjJttn2mvFn3zjrp`-hcCwVcFYMi|4m*|DGgxzOErj>HdGo&>tHa40FWhoi=(E
z6!7}K(V3mP3C1TEKizLud(SqqykAKAt?tKlKkq;6`|Wst&WzhlyB7T`UGdvq+aYU1
zz?*qbIIj9E^xS^vf>X*&`x?gw3Tr0Ccj|lI_TFW!kWsL`<DsT^ii+%{i_g_Uyp7^y
zV)&T0@tCQrvhMC^lAO8p&cmt+dozEE%?#STPt>LR^D~Z@M$?@7D}%SVt~=6cz||~h
zJndTdn)4zN!ml$!vJ)zI%#m4mWlClEN8VHA?<*%9n0am8%1q6h_usblsfJq?unVkE
z?$+Dm_~w|@n^`szZymF>SMz?`aDtEB<+J7z#oZSrnfX>tEtK0Pw}az<XZnXTw;J9V
zz1%u~yUV^As~NlEtYZ)FJXXqdOX2%f=>*>?&lRLrEBU80X-Lf1{KXQ!$CTsI{4$Pd
z&!Rkkn>5$2othr2#xeP-?k27oKmNUJh*({=Xszr?llF5#AN?A<HydAbpE7T!p~V*I
z9ZpLnme$L7S%|NB{Nvo>TW?}EmB|UPOu7+Yko8vA;ehG;W6D{l)*et^^>X{xIUQz=
zFQe?+V?EwYHWr<fS(&HeFYkArw~e9Ev@iZaUeGIpZ96&d2p--2dd|Gy38t5tJwL=`
zcsld{bW(C@o*7d;ZyxVc@o=yG35B!6?>Cq~<L+K_)!*i!#Y_F&4XOzSZ!I_R{>bq5
zvUJ#8mTN9$dp`O3p`eOQZj79)i;5n-%h+W-Wj*uH#>3K+YO-Z@kG5vSxHWX#i2T1!
zn~%G-;__436!Skn6BfkCHLP0kuJPHn1!1MKOujq`O)3u#8_bL^)4j1+L(7dLzd53G
z4ui8#oI;j_wXK1}qLP$H#|~I*p2Hk2U{@xQ^FCaD?goxb$%O4(cg2(DoJ;=UaC^U_
zT~$T=nbiU+sga&#<vNQ}CdzE(U}?DK>#fJ)d1m+f=L&Z&pSIItJY&kk@lvgd=XPW0
zwPc--E@d-kuikik?V^D7H<d(lCi0f(@8Vj%R$MAcc%lFAhd+vhMdSp%tm;?nIas>3
z_WT}BjXx46w0~|5TK_+EQJPxxe#PE<-!J$jtv2lETX$n-zV*EoOxA`x7M*idjFr4@
zw8ncsi>cvoF&Ai1__|=ys;aH5rR#pwO<48r{W8C!xw#%|#iQCeG`w%jn(4~t{$RK2
zK@YBzF&w_}A3JWgy6YY2ljb<KjpMx7y>GKCL>64j^Zl_i#%hhv*UfXaySb%#dUO>M
z_UvC~`o@I!VS!7F#$3(l`yRm(@i_~g>?)|(zBgil<MVl{&;5!IE&F*!y4~D3v*plf
zq3`#0wCXSN+}`_$ndyX{P12riCH8+Zj(gPRx*R#Zg|Fke*ScuoQ~%cNwa$>||E0j;
zlf>g>rFZhPP3tDUm$JLOl~^96d<kN>T_`)N=&sKF$zmQEzvs@go%(r3;Wg=XuPg6X
zslI&hIZ3(0=gzcOZ<j>bS&DD)E1!J+;dN^+srSBVm3c4D#8vyO^cAvuY;r*1jcD8c
z*FSDE#VC7ycQ^a?eP!YPgU?-E^^0YLXV;Y4PIwyqeC^&wUA;G<t*2Its|mi52tJkT
z+UfK9*d_lffp2=#4p+*qj=Oc0!)J<G#}?NqF^5_Cf7;D`b^Z0J*WTYN_qTlrI~uX>
z*ljUOskjrS>($o3%H-Xp&r{>Xesl4uuWN3bWbfJkC~K$encvrzyPBB!Ey=uBx>jII
zJJa!J`MTNDZ?P)8t<$_$rT^Y&!gs-_x_w<70W0PzT;9>wn|<ZlpU4^6*DTb}Se+}d
zl5M!{*;=lZArL2eW95a|sfXH^2zzNR@BVn~=A%awBY1;m`WheFv*MuKy2eub;}uRm
zaaU8Ds-ia>=Y3YG|JdUF9?!>DZ|_W)>wP%pcCSR(XZaUv=eJJ_d40Mudyo2xNdHJ4
zp|30S!;BSD%PvVY#|hjyU;JX%TlsB*`5Edzocugp3TJf3nYoDWQ1Xa<^Yv8aZOyy|
zKF-`?J48;TAGRtxxFP?|m0O~fX6vq=?DpKcVe1)5p|32O4(n?t1)VpknQi3KClg$6
zn%LYR*t%)nEw{&CPV{>^c8To2=<2LD@uc*W$m>i;K1^KF{DZ%3ofD7De>SDO?7W%g
z_N`(;oBwtAZ>yX4{OYc-<p%vSy;^1Bne$J7>(jVlB;i?T{aNngn$+m*-UZw<-G4h=
zp5fB>B{@@n@oepgPsiMLdi-z+ox4}(M#9bsxw?4@%p1?KeNc-sSa-Gd!-cYm8;uWa
zd$9h>^QQ^>byZBSTQ7T~ZY#94?xJ|X<SV}!OeZcmkY2SoaNpt53t@)7XIm!uPWo9j
zbHUHm3D)AO+otanE%H5588l1XaBXssfRn}|hp5jx#XBm5U3ckPPB)XkZg0uj8PRZO
zXUo)(Lqad_94wA{%@ExuJz=h);5C!rUl$_8&kLI<MBllhrIIowFKqT|pRx<-FIwm5
zcNi45<|I5?u;JNHL5@wiUxMr6d+zmLR`E8^jh{FDinWALMce`YCz>t1(w0*n@9fS0
zExYN!hgPNX*r;od3ppnm-ukY(j5)u^UP(~<jRi+ph_>s?{<vd)cEwwxmM^flH^o<d
zrtjJb&p4J&o#3eQH#;?&f3^6Qk6hE2EdR2)e!AbAh}%WZ+cp_&{uaZUeQQNoq41XX
z*&zyN?ma$s=$T%Z_Zz0Cw~MaNcMe}?DBS7)d;3cx-KcqDE%O&;-96xNRWILV|JCWP
z<`<u<-I852M|=}QqyPHk8J&tJYd@QJ9^>s>l;WUfB`fE)ILUV9deg3bvTA~_uGu9>
zOnh-Am?uQ^+NIAzCQ|nAO6KjKlc{FVxhrg0gh#jFZ)@(|3z}~H&wZJ@_(rbsg|%}h
z$2kjJ)$8TdW!qg+^K@(BT#Jc6?ux6bCu@rc&(-to^i%mX_3f*!6Z^JD2Auv-CDq%i
z@%NliWctk=dDDLV2{`$z`~88nCSRrh-cpe2_B*Ei>vOQpYxnAj+Fz<~RL(f@*6Xlh
zN7FIJp2!=q0f8nJ%Eu3PpIVlYap;aF$Fg+~Qr}lRG2g!8!R?-HEARQf=FPcx!1`um
zvAmp^MzD<C)XfilRTLv0vh#o6zRG{!f%3G2o<+G<8?s%_O|7k1p|9L|JucbgY@eX5
z&6nTGYgj(Yi#sjj&oe)Lp6x{Yq}_ABO`Pk*do$keZc<Rt<M1tITSc!$E;?f2xu7C*
z_MuRQ=UGi#JMubrUDx}nw(CY^&ZaY})pdHZA0MtcRjM1pynB<c(8JTpWnW4^+D*26
z<r(#UrI)Hd`!WBU0o(V6>{zKbaqgR_%X+8xFPyi|@z=Tk6SJ3Hy~F+5uk_`XMQ<;i
z6rUv$e$V7UO7NPgdR-#b>hGFXe>m(MEPnIBzPqX>*WV>|zt3VzQ@CHa@HbQLtyXi-
zyNjPZo_O)adxzgSH_fB&x0ajzHOpXlu}6J(+UBpZD*G>ZSBihK%cxSgrX$20d`^K+
zdvC`An|aGmy-hj0FZShjCb{<;&qQm_n|`G6R4+eMLVs|~#)Zo_y@`pDd2pE3tnP67
z&nFYCp4#uR_#T%hA>l8*^QE~DXaDmMwTtWi1l>5IIrHL*OKscZC3fdeFJG3jU-yah
z^xzDGXGsm`riwWmOEWE;qf&W)Mw{QVBwgv9Yqm+eFV4U8TQ_FbtYz<7yK)#S9;W*E
zI4@sn6KvL2tfF`B;%BvNpZ}f>9s=j&S9BNqW>&}C4wRejndEV_)~++=vbNny*Q1}K
z*Kge`yCFKO_WhjSs&T*Lly927w9v1uimI8e$ddbe@}}7z%+~BW@X+z)X?wY*y&F$&
znrSz+CU(c$JBL-jJdL~hTX!1cyrR;!?M71y40>YxHp%{3@Go=UG?vo6d*0U8E7nLQ
zF1&jCqUpO_{_O7iT{r%2`G2qW#U=~h)#h9$ewKev7MK#aeEE_`G0#ob7GDb8<{^9B
z{K}fC@7as2FP+nUCnF>KEG3=Oz0vLGVUIo6IobaHeX)0Ij?!#TAr;Omlh3Gb>iNA}
z=BpaNoEXnf?vGVpF5J&GtKSovGI`REo0Z>A^FJ%OJ!|iQw>Lv0EzTZ_oN(#ym!QTX
zMy{{+TYf#>I9EC;Q~LDp%4IR?hHsygdpC;Bmc8Is_$&W{*_9w4o{Xl6PJ-2+7aM9R
z|9hpWn)x#^!<gBI>s(08K8MDrMoXXjVHfQ6FYPWnd*AItA)nFpkoLsa7wYaQYCS3O
zkKQBt<sV;mN&lQ6hu9Ro*T;V_KWMRdV|s>R&5336mgt^5dRt;;&JTm=RZ^4g6q`Nt
zs<A(O>*@E&$G(5npR?(uL&4i?c1{0&uCshT;a>jf)l;`D6#0}hpKICokMUZImpD(|
z61DQqd?llw^RHRWozbLJ(U`Jtr>$oA^=CetYO;1D3T?e9e(_|P+d)ls$t%C}@1+M9
z^aoB^`fI^KkBLTqn57<y&eHgI>u6%<_BTd~C%62uUs0*PCb;&)F8RpozInGlYzx)d
z7VG?TNx_TwDAfh`kL9Hvc2(V%yI^V&U+b%D`ALB@esKPB7C9+=?`M-<^^4W&+~>nL
zzZF(Gvf~E#{DiN}A5%+xr_H)5{h069n*6ZY@2@C*oik%npTqMhr>bY0GZd~rZO6U0
zK=?`YikzksTo!q|)sE%`eEik_Wa%dHcc)Sci*8TqoVlrY-~BaO_TRGfxBSajIN$q7
zFQmSVb3$~Dz0K-dZ0~O7zxjK+Qf2MWc-h3-+?TBRvu^O_EZOnexmw&mU&Tu&Zik%k
z;nYWdYu4s2{(OzS=gzb!mm<Zx9%WW7H@c5JT;DAIbk%OT-MLQD%YTc_4OP3nVat=*
zU!8Y^wXPGEZk)cn-_^qO$(#;}$@A*$6s2VM{W^H>gRa`WH%lK>w%O=qeU~*^z5gY5
z>uKwyW%rgEZcJ|fz}x3Ed+MW&AL_i)Y?c(S*)rjUkk)^P7Uy}ZyzYNW)RmrUCb#Xi
zNuFI|h2V<COFJD!Jxs!?K0kc<m-l{D{d0$VAKt6*-dg!R=E;*|0nzo2N8L*Ap5$(+
z*6VKzHE-f9I(%8~Ia|oPf4jE}G04?jJUStBe<APt*-t-fuq3|V-CuEA*Ya%bHT^@f
z+sj{7O;Eogd;ELhBGJMey=^y-p5JwMqT21U;@|#m_l|z#oZ-iH=(5m`$}{SpB6ZjN
z{nnejbw}jKofhujEw2@;YJB!D4)tk$U&$}7{~~Yx!-j?@L0)hF=<1wuon)@X_?PS0
zhepvwLN!*`vhE(O+;x6up`TvZx4##Tm-9V(8sc+yhsXCj{#ReHPTF|N_+f3S)k?W%
z+cUEte|9Md%sQkK>J}7i&g}H`mY+i3?U#!S+mH0^@m^tgMno*ZC`#{RoypO=OSyV?
z3HHky2(ESRGLew~q5Qwkwwt^4<+^v5iafth`|~zsMJQue;LLYPUw#FJ^NKa4wHaLy
znXTRy9@V);R_J4W^Zgrp5B|M!?aSIjOKywTI*Ui&4b_Xgux$5)i2awuF68Mb{jGUl
z6Xu#QW7^Y(c4_zQJ$bLE#J@M{5(`w3D^b0AJ|N8R-`-7|s{=ot%=ZY}xp3#M1v7it
z-Fe;KIM062VRUBoi*qylS1sMU+J42Op8_+BUVmeF$~sYIvF3ZO7Y&D<Ht2l*?f*Y8
za5MX^2(NmhX7h(TBvsdX+%{R*?<KOS$oxs@)3$3xU&9}NeZKwr2A^qiUveI>HLsqt
zF8NCE_Sx1yZ?C+p@_g>iqmI?aYMXq8rEi5c+*6l#kv^OE@NdlYf3=dgXFS}NcK6WE
zim7tb-p+MAa{U+IcD?5xZ_hiqs{RJQY}2=!Kc`*zP;}Y+U3qP}$m6n>%N}c<#_QjE
zyKf)A+Wl_XKF_V+FQ<L;d$VP0tnOc)y85^Sb`t;pu@`^;6KP=m|8>jKwEdM0|8_FX
zWnV7LB6rtc<!YzkuivFbNhh}Dl>SOP{O47*)6<LX7QuU@jLwSHNkn&;ESOX?eeu1;
z^Bb<!7yhZ=Qxf8lmMQZ;OGV4gXO3FOq`o$JrL48r|L*)-_Cc?{C;m)<uIyQz_;r%6
z1eQ$Xc>2tt{r6Qn?wd|)ujK5F>A!CN--qXpUkl%Py|;I>ww_Qp&+=}WNY<1uySHd>
z4nNl|wNEE-Lvz#)i=`LtUFfKY|CH~TwZq2PDKCmY+B+pzl4XwYM}?^LXukXZ>msFA
z-7B-7&3}B_svWrpYj1zPA?6y|Y%xRfYEtoi<C)(=XTH;Xa5;J1;ocn*UmvBd)_ct6
zHYN9wOZS@mZl+^mv+Bxb+WBptR=2Cua7M&*johN1e^ZnXH(TwFcU*ehx1!K*(aqx4
zS!;?OcW?SM_qk%$=|g+P&#Q4RYW&{CfB0fh;0#gkt%4^WObhq85frk@wom{0G4aA}
zFYk#dOc2X9(tdSGa^C|PZFT+A2K(PeF}rki)Ee?FN={91IQvj$L0=%375l{JE0?MM
zKE*3?U9{1WW1C)TQ_iu}rpa;ZR&IIbuzu+x35)AX%(cJFnV)~H_RGHWMPe6ng2esj
ztUI;thEj-d-_4N1#7w`*o0AP5DAieVE1&)8JB8ueI<MEeEgjG7zs$Lw|5Rz5OTgE=
zRvd?;?mTZx`+P|Do%f@CFYlgPy7%7qqFJnuUp(7BFJ5Z)7KZQb<($EP56ulv3V8gU
z{cfW4?OY~x!)pEHMRTQt4(Qmhh5fSHb*pFAwTmW)v@SnmSj#16RL%I<_OI?Irbl7(
zw{Ln@{v`17E{8)?HqQAKEwsBhG|!1?P4JFyO&d>dXzRY4|2}S$R<+)Yso%AK-B{88
z`KQem*Cn}00z21C{dTFgUjNTxy|sE)k5lJPH}h7~ahoIPsG2rgcz@;$i_*Bs-9o{S
znW76X@$9yZo~FP0N78EPUt50JzRW6}oE1@<-tzC)vm5Jo-sZhtBVN%hXBT?wlulUY
zZQsJ_Dh-qNh$kA=Dlb_z<)T5?yVcdZcI3?a|FS=2zvJtRiC<Xdw~5;wh?lW?+k5h2
zS@+qtUy`dr^ComE8X8KdrJekxSt8&m6esonfR*Cai+g@vHfL&*Z$8k^#j~P1^uqn@
zIJN1UHwN<a-wvH*Hao&FocVzLu`X-#m->raLi%;T8}L68lWW*#r~dY-;=Ti}JH#i|
z?Kat0xM5$RT@rhqo>t}jn{x#Y$M2XI|L&BO+kSgBxx6FV^3gXX3^a@CHpu8L%Tnlg
z8F6$%=Ir0oEp-0pUhCa@UinN7yYH>lb8lz9mMZudocA_$zU>{qGv;<(IUmJ2gZ#eT
zc;q%yGw#oi-9k%kP8b;7c2MfAw7CDXyQ87!-nuz_=S9u6Pp8hfc67qA{6$}Xz0*DX
zJFqEA?j(EaGX1zCZkw+C37Kn?ar^VTZLgDLSBC82tbbHjUu{_XaNV>L!6T2%Yabsx
zvHY(VQ`(+D&-I;CeIsPT9K$ax&^2N?HNBw6@10xZt9?4Rf0xfH`_&wKxl8?1!N)|s
zdp%aKw59L<y{M#K-}_{<#qp!Yk$3*I*xU;)y?!SmBl#%T$?W+irIRn|=Vm<Ey>^Fg
zLf08b+tlUOo|Cr+$+`SJR2X`$ylC2Y6GKkjNGo0=E!n@Kjy{1!e13h>5u&<__G#?S
zY~Jn}^@@F_xNgR~uV!}6HpX&}%P+oCN)VYe{q~M2|D_H8%w2K0bYf|}_2ns+XB?l%
z%S`Z7y_kILR)9soDfRcN2KtLmEIX{XV?*%9ea}?<E<K(5N9b8aZo>?Z^Xhkum;JwH
zu;K1Xai(O)dY@WG-u0$0RLoCWxM#iN5^ZXobw-+1a7LrkhN$_DPxhZM)!w}-J7Q|?
zwOw4sM($VVDjwJ2JYsZ~fph0nHH$;@rZ%VSV49hkb1hwWO<$bTyNAZl9cpcxZ5uzn
zWl36D<r3-TI)BBvyzIys6H~3;inVI#7jbHFJ-R;stBLKRb)QmGJ2lfye*AC?-L>aJ
z$@9)2;e_qKwtZZidwY*r!$+3y9+xlun)deHf<v2v(@V^v#giB9NSM8`q(a-y<wJYk
z{@YP|cjbNG*TA{6c3ax7ZTBo?JRV=%d9K!X|Mta2$<b%8C&j!iD*786|05t}(ZN-*
zPXZLO0(Tfp{=H4}jLKBq>Mb|DWX<+2Tb^-$UcP!)hU@Ifhdvvae{8i3vEUAPt!C(Z
zsX`>vW`}|&%S+KC>LpVnSG%S<PkcO)(<<)VZ&6W=Emsd**ZXpLLCME#&vzdduemAh
zBT$yZ$URlmby?ebSAN?gHp`<*?e%4}l-7Aao6DzM8@4p^?uM%mik^pEzwA7#i#us!
ze<tHJlTFWDSKRsQ?pV0^-IBiz4hs&Y+`9HFc;DH-ETV1G*I!Byk>ZF~I?cO%P3GFe
zwdXf3yBw0qoU7<xfB*I`BW<l1v)jehN8epueenMKb<q+(_V@)J?uycn7gq7yc>JDM
zl&U?eZpiHwL30v3txl=r-M!)KI`6|Dv$9|VCbt~v!z(AxpXgr5v+ID#lZxWILEXm6
zmw*2atg3Y^J!oF`S6gf9yq9YZC>&fb?3LO3JxC^J3(N87bstZ;S2Aimef8X<sODS?
zC;N;o+dSnbaaG<lSbD(rYAbJn$Dbu1eDcK9%I&nAJu*J7yuWqbW8dgaJ_b8G52<`s
z?0r`k{Vv>2Ut{@&8X;L$mj8vJMT$?lZo2AqX)N{%WZH6lc5iUU?WC)zOdeO$Cz{km
zf6{pqcTQMatl`#msrk#d-dx$Ns>tWR(A2**b)n#A{XM=hf?HOW=^7<!3vo}EOiA|p
zIQP}gghlu6{K((ydhg<-0P$1(b+fdte>3==df4>u(K#1i`5n}mELWs9eS7N?J2lDh
znYTAK%+vZD&bjj4;j^-j9&gCJXzRL}_0pBoHO0rK?0oUb>R}NhOMb@m3%filY7Q&U
z;COR(()nAR59PN7{z_?G_VPKmhJVmCmwqGR7cp=DxM|J4(Ov#w)<qFPj&HiJrUj%=
z_gb`9`Cw>-%c`&6FXu~J{<;yrueVq^Ouo4O-JQKYCF?rVmd@eSKWA8V<6G8ZGxNw_
zOXe->ij0VT*sb!yv@oydbZK&Hp0xYIJ$<jEL=0akh1t$=c%<}Q<!_CvbI_^nx=anZ
zN49z%x&ElJ%I1pri>#FEwqHySYdqLF?S$!}Wk!dV8JT?)$ahRA-L%wT=KrhT!)Lv$
zJiTjw^SS0%Wu2c3w$E&tS$%ZtN&fA<v)<>bvYjZJ_oq+5A|dDbn;%J@89CyMez45!
zZTMm>q@nNp(ez+d(A1U%-&3FcyL)G^T>K0T@z|74Pd<A_bWV9ZHBco<XVM(8lfTV$
z81H;moOd`rSoSBk>M|dp{gSDzDrfeb+V1>z(X5y$di>w!oN43KyX4D$-u3>@3~~Rb
zk2l1r6bKbP$qZC^-C)xhskmZak48qXVz?md<Rhkcryf(v4^AqYTdb0@S5*8v)7OsP
zWhYi^T@X7h-qn3y&584L=NYd`?b8$7rv7ZVR!!Fld=V4dxwPgq)24^}Dh}A3e|I{o
ztZSw36ZRV``4=jBEK<31gZWeJyuLr%qRWB>KQY|>d{4sm)bcamTV22Ikcm82|24)T
zjrqa*s&(PM=_g;;?v-WT9wGC|bjvqCiA@XUv;IA*vSZy`N0kYTY*U}zwvEa0c)HA#
z(>JPO-_7#=|5D~#znkwo==nvnKk%`U`8uD+B|8^R-Tth?ccNge!?z1o#T!n)cW%5}
zw@^>!`fu4<hhX!IH+~+|ZC+Rwzvq5Z{XWjaxr_Jz+S6;Uy~X-r<@@7l=A1&-*V`s-
z^fHVSy&9vrJbCH5tKNkt`K;8Pn3d)owW-{sryk~7a%<V`TU)i0@8-@e&)Ohrx&Ks6
z&h%gv!-b0X?pe)W@9Qe3t1?Ay%8^jd+t1$aJa#g3#k!B?lc(zb`||kTY0HJby^Hl0
z-t#RHk8;<1byVYy|E1|dhovVxEd9k@$h<e-J|ZeW_p4>@_7j$GIZV2%b>;_eKK-}W
z{QB>*$rUTgP8k>6)||KIlhKcrg*BP(i7#0-_Rg8O$TuPHyT{$y?b~y|d;4TweXk)<
zdn@dmz{h=k;z!l0Q!c#8Kkc#W^U_qV_p8pV+%7SxOZEDp&tdCb74NI%e*Ut@^!T!c
zecgBO?^T)}6aMME=FK}#H@*DXwpw$|)zaEy6BcZGrt7i0Z}p?LV#eN^_RBhov!+*w
z3JXP^5xG30Y0;?*1(BOVAMX3H=;WSbn_uQ<#736vt=Z6%a^>FLUkA-)gx{uenoijv
zz#N)ey>Ld}!YLXvf=nOX(iE&!Un{}%@o1k?fM1Hf&#lTiuhM?X`7>B>AN$a_!279K
zWW2NM1jgB7#qXEMP5i1av*zBer9ZxGp3-s8I{Z~j`4Jz<Cv{56(cf>&eof1sUuU%M
zZJmet=66@L3=O`Sr|p<>%I}8f$F#qXtgk7=eq7YD{a;&Aox#MKewX}c&E?mgZ9DTo
z#cpZa#k{u_pYKYhYTwu|d#ia<X=2}mdEOb4b=!n^W)=r`nB3&Q8M1SF@Vh;!1-lL#
zzBT#$Vdbu^zbZ3tw|tP^<2tS6(8EnH7qBcpX_50xKZfDZ?$5J34VAv05m7cW-u_!T
z$=L81ceID)kEstH&dc5PAVDUb|7C1T_%*Zdvz@b2<A0RTxXJbElt6jp)(!f9WF{x=
zO1b}@`CVC@#kP-mYH8Qa-kmXP>zi})wcvXF@|_Z@vY$^rx<8*STi)UA*ZuRwH+<_8
z_PHYw8Ztl6-;n#-zJp7zaD|@S`t<Dv@nuUd?w)a1jax&^<)3Njw4I^X&UTgTbDwh4
zBgSpT>rat8mlu03kZF2azURz$SLvm5Za%3jVcEN9;xjeAw~0(CQI=*WHy@4JANp!@
z(C(Y!^DdvAJ@xaJFM<mrB0_E-o4YmJ(WZsB>SSqXw7BHTC3+vf^Iu%jt^KvYxqX7v
z+eM<OJL*1sKK;g{cyiP6)nW$w0#~j6x}h`j_ou>Q!(KVDX)0^w&9-pOJ<8o|Eb&A@
ze61JDw?lur<KIlKy>n&R6}F$7_gXW{D1YCuZO+4;-`0FDdRe?`sbA)iV=_B`q|BFz
zRDQXl=aRWhNYEa(>cp7LX**_4xMf|aowhP}RqNInc|U7E|6G{WGiO7-Cy(V%g`NE?
zeymLUbh3BuCsRGIDU2bNv2&_ps+7g}mK6MDe)}fc>zAAEj{P43&X}`&{ULN?*Q5=<
zkNugjY_81A9DkpW{^uTEu9{TQm1ufz%PsfT-q6;IHg_9Oyu5tqqighnDUY^<&Z}%u
z^3*B%^Il10*E+YRt4lq^OTJ&Wp3K#(wfL>oW&1k~Nda3{?a0|;D0S!Q><ML;)E@`D
z_3E<rynF2U*5J6kCLeD{yn4yA{Zww{<-7Mvm*`#dpUJbTM6^_UgH_ST+I2Rs`ImN;
zd<>C$&bZUOUVS;2vE{0kKdr4rOV>V=oTv3MZ2tOZNA?`qw>kHvx6;!`n`@F+x9seg
z8#8Cpa{ihP`weWj3EDnq5iROdT(R7(RpW8)%cDYhKT;VN<Q8^cGf1j`9L%`m`7`Zf
zkGYq9S2ccTF1JddJAK--T=z}+$Afxi<eyw+JNE;VQ#n7U=Ik7^Z;j3MwI9P6R9>f5
z%{$>WrE2dohKnA%-YUgv<h^TUHu!L7pJUFy-V5R`JI-x(u@VsppSAIFqU!3;v*rc&
zdT+n=VTtq757TA@gl`L0)puAkAuy+H+56=NGu)r)tq5Esvr=!jPFnoUKY8s(E>2y&
z``PW2zkGJySR%F}rs~3f^`@86Ju8j8i``?Y*7i$B?bMzsG3omIoURL%f8$qeR$9Hg
zEKex0ww(E1!|s!u^MW(7kD0XIV(^lDt$8K-_MJ<%E?cVhEYExst8rXs-R9u=p5Aen
zZB3jTzf6BT`^^Epzb!#-UuSdauT#EmAMALk<gcrsV9;WddDTao7;>M!7k9ExzvdzC
zvBLLj(ENt8liv#Q@6NYb#}P1b+w#YUuU@sf!f$of<*3(Rh11;o{KP{ea~T)?d&2ak
z^o6<Ai@00|!#ky_b2O$cI(M_7#CV>6(zGJK>2Cec-}vruX1y^ha=(3jzv-3hH@mAi
zRx5ozq9~hm#r@p;U+f2W)qjvHRlUAU{qMV&-5nL3ch4723sziR<XY^{?crkJr(|8o
z{Hr_p;?{SU8CTAWPPkRZ)e<*J^!BalR~O5SE29$}j$W_pDtlqvSQ7F4uGvz-^@WC)
zKQMkQ)w%az<y^T|p_xMWo(E~XHaq!PNqx80#B2Vld&>1=B@cwXpE^6}e(ILzY`?f`
z&Q1QP7gHbD%DuW{?jo~CvqShp@_Z)T$dT_~>v(7O<c{=d)0SP{xYllchuc@jAm&F5
z4GXu0Jn;Ey`&}ZsXnCbKlVyCsqf@!zuDf?-TwQkY-OuR$V~=)jDDj#QmG)@vwjIAi
z&)*Z@^lkg%@7@y}k~Q-l#62|=`uo;luEE4#Q8%*BPWvw%s!*l>xH<pew5G3SGq;wD
zc%**Za_|1RqWW^%<#9>NSS_XQd906Iuy^vCBEHZaw@yp_Ys>9P-CwDpqABck{lp6G
z89yF3Ee!T?aQw67cHZl|%N=HkKX{VLwcKR4@b=Da?_yIXcXHiU%6XF8XfyAp+wWaZ
zH0I__>ORjZTJA8nQD(7yL4w+U3*qmlP3H#vn%o!u_#rP(U)b!tsp464?{(dGeJl3z
z{G5aHC+(h5(e~%kE(ysC4|f-|wQg_nHJ`hqno~qh<qr4tM;R9Te80QKPcpf`V}Znj
zy*anzOjZ<^vwSPvee{S@r_kNUJI?ICUwA+*^?km)t<K7~%}0LilQ4IWI`T=i|HqUC
zk8_h+-cFX<p?f5iTV;>SzRzB@y}R_QcItbx2=gb^PLix$+vji3bn$`Qb<0}m-h9jJ
zxk4v%_vao-nQ?=Qi!EM7dfn;}AMR9<$&=OzFDhN|nln1IZgtlhXZee@)h?Rb%j>6=
zNAB4FIzRKpp66cMK3_4J$G^Ro|JkKEbKGVhy3J}gTjI$)-u+xZxi!Cso%q$679-N0
zYqCAM>iw@7^%Vz0?N<Ko-ML|3=Z`}>CB=)RjkaCY+C8=1%JaPvvyg^(ZT6p>3-J~m
zy$M-IetwJ%eVZgMm-pWN^5fl$8<&^`-gsZCy8UOBt$C&BFSex{4wM}_v?)vNuUWFf
z&CcpSt%4st7w;|AuR5LT@nU1my-e|mPn9HVb|0Ls*>LRow5H>0_d2v|xK{1+<FtAo
zdfoTE;l2C^Eg}~bQdjx-9g6PUDjv7%yTHO14;~iwp1(Zht>@{_N&3N+fwjx5ADt_7
zE7Z$e_eoP}q1LPQE)Pvr13#7cPYqhKujc4W^Q9qb>mNLO`FGmWm|M2Sd$>HBrXFHn
zXSwfwXY}@<m>n|~eyF?q(=D@Rx0PLI>aJ^NroaF5<2KjR*%K0P7e*C(N_)h*EUss}
z^ih5VuVm=$ugi-L@8w_9mTc3$@Z7h3Q+_AK6!EQ`z@hcE_+tO*B3|{lsoeRe&kH}D
zu;SV6U(83ZXx_hj=F(*`6HQ&O1mm|6Wv{pH%f0k|^Y7l(@wd-UJN(2wWakI-jNfzl
z>fc|!&wlTb?vdC~52FcZRo@uo%$?DzExC8EGT(*i_n(CA71Y~PT<~V5_Prg;d$SuB
z?7l5F%b#iOUO|t?``^9yzI$Er#dD`0dS^0U&w8!?;KKE#fihLf_Ag%V`6_+lefDqb
z^i<<VYd<*^Po2tiI?jn}m(Bg(l@izZmIPa!?ey%Ob7<=gX{qb`+O`@z)0ff`D-pGs
z-ph1OFY%6Qn~kiKW^-?s`je6y6ZDfczBAMv@O-f8;twCT^oM^#;*^)AeEp})+CTqI
zb?<8j!)Mwxw`W<WZi@|@7t5%=>(k0`P2c5(ZF7@#FYk%xle%_Ju=JDRx{`D2n{`|p
z^1p9hQV{SxPvyvoDc_bZuv#=_?djl(OMIEZZp$wpZM$+-`@>J`U6sujzHiL>Ycns~
z;m(Jo9Z!DNhVT3R-u#8+e;&jB)dyH^?dnWe<QK4|nCa==(@jcy;(O+Q+Ms86kKN+$
zM;BeaRPn&oCUOt&F8!R<b2CLke)bdPqwaIRNlcme_g$+;;UBf*J2s!_mNsK3+}Hic
zOnZ;xlK2+gM@{ZcZ8iZplVbv`ekSgiVs$P-_U*n03*KsMir)9f=i(Jri9c!nl7EW+
z>IJWx5SKSig-81N(F=X21wR%}JEF8_@+yPBtFOoIJzIFTyTmnJ>2huPD%lSSn~GHy
z-cX(N&%jq{gJ#ZO#|OTrY`(wyyt8Y{`P{%Y`N_9LB6(MtdO7cY^mu2^v!!oO$M)UY
z8E=&B_hm_nhU@$DGta$Qk~=xZ{n8>n&8nYm&JR*tj|Tnm_;K!bY0w{~_{^7E0)FtE
zt2K%1Xv{ctsiW4U+v)a^+~4Nu6F&L<>nyu0x}c@F*!H9Dl;6BRzT7u=`uM>wOFmn*
zcU|t6U%xw7$*UdYIJ1b~EbN%srS=DAz7Om_#ILxrUAm^X|I`e<@I~>3=3adXrHXaJ
z^2d4tuK$~VonNuUtJU@I1pB+Guj+hGi~gSQ;@A3Rv;Rai6rIuA`HZDEby?N^g-H&%
z$&2I`EEKmXT{J6AJn(k;-Ofw#Rr|g(8^*gaynd7$(y8XzQ*XS^Bz2#0(aIpB+um<X
z-n_rI-E{Zcldra&PdXW=lEZD&d0=(H_4-B8b=n&=zh9g@RlMhZ+`8U>cJil|#Lw)g
zIP_4<hB182jIH1IE(o}#@^0_t6Kpe6_I?PKi<>sdqjK4by*CT1A3lw<v9x`|G-=<$
z{@$C#AMLly)QXyr=`MGU@7Ja2lh00m?t1gDr>NV^=mm$B?roTBXS1K-<_5mMC4bZE
z{3b8-n^LxX^1+V#=04m%Z~ow&{w#6TPM=<VeFkw${f7mijz8B&?s)bzX-kX_$KnXF
zs6N>XlHwoDE7zU!E&sTHU)=x3>Dfk4-kKCl$=?5`ZDx?ohw^};->nkuKWf%5U@h6t
zes1&a4J(Xq^As>_S-x_m*pJ_Cfh*=cQF*;rpj0ofW|h<Ru={#`|GtKt@YwxG$9uZn
zPOtj>2gb_5(_?2{dDOJ(TVCDPUEjB{{P}l|eMZyblX<}*M|RE%@O<-6YU5scwz3cB
zHf62&cj=M!&4mwUsjWTxYsDXn)$3|s*siP6yEoC|v+u&qckj3Mv_CgJB>Y?M=FOc3
zdU2;8^QV}LIPj#-oV!kL`|qzCr+#tbud2Jbr1$tr{qW}=1<yQZP370w*WLTsw}X)>
zR<QKG&Wl@`naOIm_-C%06+fkS`PF|OOSMFG|F56=E3dULbdl=oEA~N4Pqc3Me(3nR
zv_<O68g_Q7zk1YUvtRkZ>V(bx(pFA?3eK9_>D4Y!f8?*Us_xf;AJY1*T9?^k>=(0b
zw$$3N&i1{^@hM+VyeM1}{^9&bZ7uVUiKlD>BIndx$&PtiWx6#x?Ah*3Zn61ZVQxaJ
z_H57E=_Qc7x@6^jKJnBn`%_yAqFtRN_Qx)|U7PtcYenJz=*s!WLwD^jt?s%sy+5|w
z{o>}!zd7@-XnrjWS)mklPvidKgWkNmm&UKVy_|pU;nz7owXR*hn6m!;<QGa#i)@7a
z!ez4e3oy>&tByCYI+QJZBiv@vzHdwCC07}A{O;CGu3+adnE%_m!lh>WyIY4Mmwo?!
zQ~AOVmPZNsZ!cEVseg37uT`+9**@a^4MkZ&!Arm7wq4&Mv{E)ra)+mXVBC}C6`9#<
zoHak_O588czMnibKl<YrMxC}RpKsePT=S~^-_{(%db`T-)#aZQXC;MnS?|BtusFvi
zv|47B#^ek0<kxPu?~^|ye|_pNu9FIxaeL;ymF?c6ll#+j$N!4Ri;IL!mcKrCp@CuE
zTUlY=tEb+Y@toxi=#PkBtRDTp_3n}5jzU|^wDb1g{4XB(al5e`%a7kVr<|rZGk$r?
z`FAa=oYuzpyx^S|TaFeydMEGqj(zXkyE-SYbI&}>yG;LSh`WwXvDmu5xl9@<`Za&!
zqtAcSJF>s(op;yAYH!!3qVlVe1q)C7pP!f@lPnbZF{Z#O_y3XiR_i|J+PMlAZd)~b
zN&cBf-gOmLf@eOq{n`^8|F>dp>_$O5X0NXI)%GPZ*KOtQ&4^og_j|U;`?E(DFDZAB
zPx^58-N&4ZhqgPEp16Onuj-cr!?ohd{P5hg|5e&7g&)qgm+e}YtNeJY`?p`$KIiLh
zzuonmLGI<EO{FfAb%iB&UHQvY9cVc3!~ffR1>b(0QTX`t-`K-f5;>XjmUOi;Jo|jk
z$9{it;Ab}{)%`lg{jp7oZ?>ld|9kWEI9puC!-WriU3p~tG;rU-pZjx<o{CBf5)A&n
zG+A$7_390Z?L~*?Xg{_z4&CGWc#3)B3=84>V{7Ci#l9Q73lzEW>#Wv|l;EYoie~q(
z+P+j0u+x%x`QX#R>-UbVY2uml>--jzc99Z)iHwK4QeV7Sd|_pInftbC@$K7Ncgbhn
zKJxf0^SlL@p0-A4KY8&$YHQ!xXF<>7_9*vf$L)75h^Rg;)lu>{nQ7ngtw-K6?@oBj
zT%@7>Xo1D2vL@H+<6>J@+@1VpOZlBYiJMf8`sQrSjj_^sTW6X1I=i;^y5g7FsV)nB
z%kPSdAA1mB-lQYEJNME0wfpmrdR8AXo;J~Qlf&FY`cJ0c-c>wz^`oO-id`2PmdtAX
zqv3kt_NnjjC!F)53pbv<c7J<Y7UM1@fsH@h%D;a%=n6l&LGZ=vrAMYlHmxgaP@G(>
zE){f-u`~b7zr}$`x86LKFtK(_+_hcjX@t+}clJ^1mu=mluGlxVeD`wkeRt>ecG#TS
z@z&&+xx#B{asI!*<hGuOD?4V+cK4{|uDc$y-u+8XXkIJeE0m_Q`sftn@5kOpH2mQ8
z{=fL??+6C1qw`%Vm(6NY+*7Z+`^S;U-TAjwSNHSZ-m0*2tGLVI(?1^mvsL(8;9Hwm
zr50C{?szBv#qW>t_O_>d9WP6UC8<|@(tlDVXMf)HVf{UOgT`rJ7=$=&GM;QrSKH<8
zSs#1s)5eD7PlHc*{uTW<-*owb`ECL|OI4W}Th{XZ{OP4t{l|gz$FWD1e-8Zn<9_<#
zM>dC;H-@%q>bF*2&|G&r{#vMB*-Wnwm;ZEp*&2W6NjHzfhnaR~<35ShCnl|(%T;?z
z?f)yApS<>6*FN!^q|7_De&X@>Z@cb)=<4t;*exPoaew<e3F&|0OxJe*uZaKu<Xlbs
zZ2l=-&kL+G&daMr{;s#((|zo~$)~SBhwp!I_glfO)?b<*%4*_Y1wPT@I_GCOBeRBK
zolY?8_oS^}%#Zx8CN2wmZ(ejIoNsf{pQ+8iExw)-4DWj!c>AK{);%rP)2=TUJLe{O
zG>0ubvM4|G{E-0Z<p&?kcy5>bE9cc-?V8rK{51V-ZKpn8yvEnrwBW_g;75yU{N@_{
zsVmc#6{yY42;GrBUvtmNnNB}mJbAZ&8^eQ(ZzB#IE6t4w3s!4$a-SD=-RO`@qMNzC
z`s}A%4S&oyf_1*0+xj+2Iy`6R%;ZN}anp8wP<!(>#yx87tsSNkn^&%1S^lm~+qg7d
z_}y0)R*9ImV%nb1;ty_jeY9$V<*JaYwagEnZuz<V+S{)eSbXi)@GiJ#;C+4ieEmy}
z?L~LKq{>{n?Ct;b^Sj`s`_ixH&F`0*+E^U%cX!iQ<DK4reZnLreOscHKX1~PB8}eW
z_vt4LSIypZB|Gw2+j_;+Ga(x9toQ1$|CUr~<yxAn`{uC7N|RZYtw&<7R;;?Ip|dW?
zR`bVA?{_cQr<$vaT-bJG-j(#bDia(QM&wWBnZ2g^_G*nSqK~&`Z4cAd+2eaoJiva%
zsZ*Ei&N97U@jq=%bnA?5`#*or_`7XuKkJG`CsgEW+jLJ&da?TS7TJWbUo|=BzxtS4
zPknUr*6pZee=4~H7d$ub`s=u{cG{cxu9MT(IjSGyH}W~Jt@Yrl*2niIXSmEKzx+N~
zQG15!{{4l13wO+2RASe<@zbooJi68EGV`Jvt(6w#IK*trUGwRd!Q73Hu6H%2XGC9l
zUnHk2>>DyYMr5L4`_bJwh4C3Gw=<M^vwfc(6+9i6v9Nurc#_MK$)UUB&%XcnyYogu
z<bfN!7gN8orkq@=!g+kN%biH3r?Wp^4K7VDO?ni3V_ia$mWWy9&bil^!|d-^T)Wr!
zC9zcQo#^!R&<Cm)&GG|;52n{z`b6H=lWAR9{PdX7#GKh4mA7}76{qjM+oHH}?{;Zn
z$6MNOb+3zW=zO?ic16|p-K*NC*NRL`TXaccb`AGhqbeQqWp(>mv#-VXH+8JCTWoRH
z^5m<6E|=<vddv8~e>MKpbf`T#YsPeDTS4p|p*RE8y;l2A{>}LNFX{K*jc@<&%&_La
zANN{2OTCsa%t&<I+@-1iWY3j;elS1!-%|<OnuOmmUl!jFShGJyh+qH5g~OfvcklWC
zJkR!C`tFM|ekUK!54~+o8QNaO+b8Dd>0fu-%>2;i&ZW~XpUh-FvMB}MHIsS%tuOL^
z+Pk{_I)BpkJDTsDAMj8~`(>^v>pb}miSQoP?{BrjKV0gri&|;<_;YBAu6cB4+~!Tm
zdomgi&n$lUc#p=LpGQt^3BLQad)h7Y$@e;<g6&M%D*wxBTlIfe@a$OK5<fxu;a%xJ
zw>_uqie2N~`DLHdwR`V8@@5_1zBKA^-+soJ`kf4x3x9;KS8V;87PR#2?7qPCZB0kb
zKKyAEIr5lY;B$q1;J1g5@-&ny_6Nnh>0nxJXS44??~b0lS4xv#d@DS<$nGuMpSJ9u
z3az#mPb9DX`{mZ7*2gcI3*T$jxw*V#R#NgfUlAi6|L|q2l-}|*Nzs;kmHN7e)kk_N
zwB8FpZpnyvtp8`$^85mJj(_sYc<i|vrfU=}tc{%W=TMfA?LKCX8J+xRegyVK?upmS
zt-p73T4BN4+%Wf<(;aRk_;9uEyuXI^$F%qqkG*ftC0E?{^He)9^~Cqb?B0dZ2l~!r
zc^m#)U@KrBl6Pa-_S)>t!7U8)0unv*zO;mYe{1q1_u(w3r(Sjk!n`{6imjWs+u7$|
ztAI6sifYIe)|p?=EO)t-c6h<xyZgAF>ij!)_GpXH>vI|JclQ?cTDv|>3A}UTY~ggi
z=(o+zVcqVl^>QVHw=93@^mF#af_29(|F+zj%VJh>*!Hc>-?jVvcdk9VYI$#pb^qy$
z+3Gr9ig`1W?}<JVuzaZbcx4fD-#^<UnT`F7wI5v59`UKoyLWB#@#4;kowbXZja#R?
zK4m>V*V{F5$qIp|oUdPMxmG{i%3!;He$la;d(DsT*?)75?2Rv4A@;X@*DNlYXZvew
zrkB+g6S+Ot<&Bt*UR(BPML<ow%07KB#f7dfyu;V4_G-P!|M2bYnSLgoo4Y%cnZ(6g
zs`ps#5c#nG*HzKENu1M|llARa$UbOE6PYsgc`ln%l_g`(o2^^(-dnVk<$C(<UhIAJ
zZ1cZo&nCq@-u|I*=e=n?t6G(K9RmH@_|sc=vA><x>a6)^QGZ3`z8ljbT0JM4zwHh!
zm5sl*y_7BUwLxy`w1i2VA<Oh{WnbZ_-oMhGiSMarPj#6j$1cumu?t=_#s6pZ*cWi~
z{`<z|p5c3*-|CBG<XsdZ_kQ(muHV*AuJM<-h9vFDnY`&sfN{{0-8+8V*vZ0e85kZY
zvc!$;t>OEr=N|s@v68-h&7*dI--<;VL4hk4I4ul_HDq+Y*J_tsp~6^lsUTPKbf9Kz
zX8NaX0m~nCc5#+odVcU<-pA8S@e+1!M?dRz`P8Rhx#NG^`jz3bvWl~R961lite+au
zq}8eP@gMiP<<ne@_65w7Kbi1u?=@ZhQ;X*bJz>8p_W6`m@?M<;vxjSUsL!s7x9ER;
zXis|L=cWKHFD~9)ehpRqUjH^8U7c#Edctq2+3&+krm%%S5m?x)6rm;kz;IUD<gR5J
zOsNM0)*qQ$+1=~^?fYiURh>zHj$SaAJ8t(ue_Npf|Lq7i;nzuD_bB<uOq*I`&+=+s
z-Sq#DXTITj`fHX$#nz0{`B%6dbEWqmxZvvhQ(etmXkLguw|HmCw%%HUk6dAo_ccl@
zz5XjU>;AJFe-8-h)h`z=pZdY(?DOj$*DAHYdv54+GVqX1`0Uj1JMF`F1Hl_|V%`U0
z*3OEa%^iF$@dmH#|Hr&$6W?V!+3r=ewE7$xxk>z(yOr7Y{r8=EyRPsb`XpR1vqpOI
z{`6h<pC$eMyhMM_pOp>mKkDp!=CFlE1$IqTQg6!L`&ZhkOnZy+Ti4`D3rC06_;uld
zi~l(>{R`mJv$}eD@AIkMKB-Qx8s|Q?xvzhC!;j7HVy~z_<<Io`@4DdjZ@-^zji;VO
zdZ*4W_g#A~cIVD6m#aJfGcoVppY!r{J<E^GmOsH})xT`k<f^pB{I&EovC~t(T<KeW
zPq(7*?jwgHi|j|=W(E14{CwoU!3!^I%eYzHdkffj7Oc`0aDBPY!1}A8Q1}@Ios`@~
zF45Ij7H<d@`S^alNXncyv-#@&uwApc*D%4uKP`5qj_aLcbK;f=YTkLjxc%tWn7iA{
zS1bE}%oX8rw|d$0rB9+pIZ%F?NxV<ew5g7PF=@XPW~Juri9e$8`HPo-?hEy`u9ZI=
z8`sIqo~N!Z-*+^Cho#j!YKfu8ddpnaP~J)VE9>If1Ju5r><Fs4n#=h5$9i3-&rH`m
zJOx)Q-E~1aImw^@_S3J<^(L1tFKCQg)3Woy&y@+`PbSq*YVoK`Jo9qp(lwuBW}NvM
zo~&?ZN}cW5rMuG#nBqd_{hpoJ@Z|O+$6!7$>+YTRW?p)~ep$AaSG@N9kF&&Fvi202
zOSOB3{bVk=T68<>^zrPUdT&;E#69V4+jKVm)bXX<YyNKIo|66Whg<l5ubD+lFYGv{
zy8PNnf7#W0kLVwJ@cMV!oo{s^yW-=xI`50GJX(B+-D2h(^;Jx~g=X7WUH1k2xaaYF
z$L`v^Us|W8CjMynzkYA|%dIKbFCKaJQTyEmGqD~8)mD`+Csw?3WNy1_aHk<%{Awuc
zV)fA8sx65UfA8--+Ap%QBO-hK+?mhk?1}BH^;;LvGEc-cP(;3K$Kl8%cAMSXc5ME)
z>iK_xRPpWG*jYAQ^!)HH_>w{K-BKT`1uUQEEfF-xw+Yah`(As^f+=^trhSm9)ZDFa
zxb4A(bm?m~FE?**cyaRJ)qv24|6Y83C(Icx+}`z<iR+zt3#-}D*uzyvcXv-_2t9MA
z`38TG*Xzg~r*|@oEtztDg4Xh<C2TKdCNHi!cED;SQ{C<CIZbCibSB=Ov~i7_&#JzY
zS^*|kcPu$8RcJM_ee$7Ae`Qy*-E#Z=>KIGZrO>ODhvyV3?4PhV)7>JQ@hwNS`>s5z
z8wE{wZ_A0C`{Bc0x&Oe+0P_b=#9xa}XYkj&!05JX<!<&DrpFz=-g+D2_uf;`a;dlZ
zxq9g%i$b5aa7|b;MRDy4w)G+(zbVF_cx4;4j3KvoJ=e+wMg7H5pRA^>TX*xba2C7f
zykq+%Tu#ijb52Uz$itm*HS}G;p@hxf7HcSQ9WVN>(89BNmBE|#U$OsQ8Xve69=PA4
z&7tBOTWNv9x9iacN0x=AmWn=$-n`TK<nfI{szG)Wr+nuQn)R#1l0`5k^yID8J*Os5
zEUI6!My{AgU69L>TWikso}wpT<piE@k~|^mGG$6`i~Lpd5<!od4qQ3Xhnji36;r2s
z>{MTs$aYdUOVc1ldfVx0Mse16Z*EN$W_LNov})O>vxkECR<pWY-6yNia@X}4x5NpH
z6~fV8<~=jHZk60P#UHp^$>ZqEM=HX5*qjv~PgoJJ$@W!7EvbIr{_y=9l<!uqn1A8N
z&H0Zj4|O*GvfY<|#a}wPJfgwqehjOU)3>Y#-=;)9IIww=Wuu3A4{x47=fd4OM}*h>
zkN9a=5Nh+{oULx!)1wRCpM9S9TXNoaQNfJ<yg66Zd9|yL{^nBQlX+kjwkz*jw)hLV
ztNDfuKT5k+v`(!*d*X?}%N_GBZTxUqEKqT(H(Oo%&g5OO-s_H?ed2MvWV7M~snrIp
ztSipB|7^?WGFtC{fWum02A{Q+_S=-jH`Z-iv5zO=R_5L3uYUyWbCOXxbFehx#Pz9u
zuk((c+@?Q$Ut!t*JF&@UWgO=}$t$UO^mvLS>)aK2zYm`hC`u4ftjh_X(yG7bsy53W
zmo3IygD+ot&>8#F$*<AOvDe9*|LZ!Icw4ogJMz<`|K8+yKb`lPoLG;PCA;3*__^xm
zyEI#FC+;#=j)`jBpr=#wG~DvGnG9d{sp)y*|Nri*Wi;J-{iSeUy6cNQaxT|>btOuE
zo-Vp6#eXF8b<NyJ&7x~7GR1V)J^s=dP`~D4vX4t<%FosR+SgrXerfUCzi(c{qt&|V
z_l<LBAG~iTr?>a@>X1Jw-TU2F`QAAFeBGM~%kMmUSR~dEDDX*r&6CAvH?^Pg3YC_B
zb91+zM9aio@6%ZYZ?|mR)4zE48~a1|qz?CAj<`F0(N*c<oHbdi>R0G|dvzhcTsz}g
zn0y86rE8)+b#K0CuJ~%i5};hPa@Swe_3IK{%ugRw&QRi+qBWUi@vLopdl@@=*9*K@
z!1Y!xdC#Fzp0f|H+`4u4o$vS3s@;4mUKX`qSW~_HSLr(QnTp0CcLK}zFUqrDci@%j
z0+)C-Grst&D<^JS+s<4y*Yxl$S5=1tCtkP|AHOg&Mto+U={_ghdEH?i7o~5x`0Ft!
zKFD6Bb3-n7{)8aggI~>fD}Vo*SH>>WlEd3(y{P??!U?tc${t3WypGLkTWW4D{lvJB
z``~0Dp(k-~8P%u7mb&e=b`RwY=?H63@Q_Vgmv52l>dvR?U3QDrV!>Sd!v!Bxw*}rv
zbXldfF7B!{lU;A9JHv~MdLqd$d7tFyf6=pGsrX+Kqi^h>b|a*F!^$%TXH1tHT#xaS
zpS#8JLk7=Dxj7pO8hj2NP&n0{b}aO<PTBW<9>e(N{`;HnpUh!&-V|1`-}vR6y&JdQ
z-TUOtr<@Z!`JUOkuha<%O*{Vd-TG$Rz=sb$zg#!LQ`2*I;9@Va_18Ll%iC78Gc)Xb
zePGx6rnLEfn<ssa4)oAC`{k8eM#0Mqj^W4t1b3XS@(q1D&)1oC;yur{=*4nh+Pj$L
z-W~JF%go-YQIz7&=aCxg^;Bimi_93)`ogb8%isTbbuYVWa_Fg<a~G|f>0LGBmFc?P
z@J&U<{rr=2WLRd;$_kkf<v5`zYWCApjtp8~qTY(n<Wpro#UHpguV|C`((o>|t#|po
zl1z?PFIpUL9F!oTD4e^>DeL`~Y}VGRrW;qYa`@RkjA;JwW!~H+kJEOZpRv4TZy9UZ
zDpA)5OBIc4i+KBUUvcwo-Dl6t`fuix;*hehZoif%IqiDP^nUrA!)1aNUO{OSZt-q+
z*?QmKDUx$>m6hGXhrHYSIr5t2gTiJrt@rx6OSWZk+5xMl{}!@+eq%E=%v3dG(z+AH
z{~qtXlMu0AGNDo8RonF9{)0Q1=Lbr^pDAKzUGnI`m3es<DJ<IE&-7T{mfW*{7?8%R
z=~DEfB=*^wkLHnA*|bDbEeft)`PEf>Wqq{X`@~1TxPMHZXRshDG<j{fj*6}Fij@M|
z-upW}kKZl3^Zm6z{9o3_yc<2=q^HEq^1ShM<{rzmsD)2{6!+R5KJ~2i+4}CL-mT~N
zRE1VpN!S^#bJBPpGxO1u4>9jgt1tbKD&1CfcJtR6rc&EJEw(u7decSs^p(N~cOv_x
zRxDXju*dOGn{mhbHC3e<T8F)YqhxP(n|%N2EwK0Vftg{)o~b@<^>}woV%54;6E5xD
z6`wD(W_9<*>+kL!iC_{wZ_e}n^1DiI^Y&by52ionB&V7w&y<?={A9H58pZa@9Lqm@
z9?ks`5^J2a^6sR?0iRbZ`KdQ@-EMq*#O3_D;wVjv*Av?(`#(|8T7Rg1KkL<3w|s6F
z+L~E&%%9L|8XRrBQO;AK{K3AoQtyAQ#iD1Noll%iVtTO5De<xE7n8${(fQ%uTs}`T
zc>BGFMd^l}rNL%7<|p%1YqM@Mt$SO#tnO3z;Zx7#Ijj`duef6@niFow(w*QTERnV7
zVDgd-uYCm;N<3?RdgxSHh91i7Tjz6#`{Ng1f5BB#Mb$$8^D9Pg{{A3XCiAz^9`AW>
zpLfgf_&tC5=(|o(tn0j2Te;GI>9nZt3ys>u@m``sK}Y{^?WR0!QDbGFH{BYom%dJT
z{$cUj4I2u=6i)58$O`f3P3T`?n|Afn!kT>Ta_h;wvN=C3SyF0*?)?y+{B_#81J@UX
z&2VQ}d-e9qoVTW4C1)I7uit!7(Wmv!521eP?ebr5HGek{%;33Drr_otYv9Ns^sp}G
zmc^;9H~RveSKjU}Z`ib6HP8Cy^_9*KI0Bjb7p5q_{+DuQZ#}P`TImJlxmFFgUY6DE
zXL$N|m)!mLm-#N_%3R^=T(`YNpu9Zv+77`v-0#jUFj=RzWg{C)(}F*f>zs<};zYHd
zS4CxoOO>5Ga?{rS;HH18uLXJMu2C{$dZSQYFB)1}?(_1^qf`1@yE<>?ir(Gbr^fr;
zcg^IZ*F_m$)`UC$7Cv)0^q_g$?S?(~`!2L_HYqFZU6~LcXZxc?;YaeFs=Wo9Hs|?$
zKfsswT2Z6lde6}nkE&A+a(?4mu&1f<%A?LR{c`8Nrn&q-rqsf_vhs1M_qwI~3wl#t
zCG^Fbu3U7b^TI>Dqgughe7v+jdnG75x0_gX;>qqfb(@8mB4?r$rkk=|5w(vKo?Uat
zWbS+O9nWt}tvt%$G5@^D<23yPj4yn>IwwzTPt;tqDfRH{qIk(AE9~Oc@|z+ox$eyG
zk=(RvUHW^2!sB&^COxnedsH>Mf3|0f$nUy00`G6{$hzR7J3q&7N?4=Z-P~L20~WRG
ze!O~rcJ#xE{tCO#T@+^gExXWWowV51nzK7nJZ}8bDDW+rGsDzs-ID29U6U7-EQ@>S
z>Av?p6RX_Zyc@^9H~Q$G6Uaa35^Ff;74K%3Q|`gqrzZs(-FtngHK*jT;rfP$=leaE
zs88vgGJSc0+@i8#1*2;-ZhijyNz7u2jlQOS-|UCRlkOJm)6d$|-Vk+*y?{Mz#xLWV
zle;(9G|pW5F5t$iy`8;Jf`i?Hp8b5bDtG>y6Rw-C#Y{WZ$gcKjEx%1>ljA`no2kn-
z7p>=bT6*~M*H90Riyve)f&x?*md>i+UoGCX;`|@h7w1mxzUlhDZta3U#+O%L{#><?
z!<M&r;qNC0j+Ycmv~m^(E?j=n^PQpZQvbgp*}Sawmi@&a%G;!vzn{o)^0#XY<$o&D
zH}BE3Kb^Z*-?V8jWoh+o5@)`-mPh&19T)qlMrUU!dhO=A9C_2h_L_yUIq!pTjzhad
zCap5qRvX87a^vsN6eGX$nRi85r?|3o=PvuUoKyHI=iT$u_i3GGuRkc{diHu#`X}DT
z%aOB+*US^MWWB?==B-yn$=lVBnf$b$*6LmN5Lm1^c}wAg#rcK7H}7j-n9}=1IQYl0
zyFL#hEZ6^iR%5fimwEB2Q*Fh1;S90AZ`H2%dOt_*=)9k52cB99JWXF{l<RbMwTN0~
zUFWO4*9<Eret6~3qx$lGLe-(=+s@l26xfG7|FQht^s4D~e3?nNt}co#O>|zf<KXVr
zM^Y*~S7&#t89DxqGi0)|jCa;w=hPqNXR7Wo!FtENsx#*oHU56{$#vDL?a4B~S09}|
z=P~1h3x9JCL@Qj&Xw_9{*f2@|olkUBck2ALZ40+Oei>3T*S%foyn55eXAPMW6|e8D
z{VXqX@{no@>x5tG8eiTxFBjeLsn5Qhv18{~&gYNzIhoIyeYcpyMJ6lk-G;C^p{x6g
zVl5Wl>WfUiTCsBV#^~~bZ_f&jsGfW&WW1Z#sh;Wcu3MrTe^y7o(ri3?QftwTSb6ru
zJ$?d5kFIb#Dd93XhNoWW!^OX+1a;%1>#8f0&q=*GrjihSjUy?l!}Q(3pZi|kc{)|*
z$zQ>#`LV%K;-+&?@Tkl@I8BND?cLZ`p540jXO^%$Hsqgbowv$rliAcy7pffp#uROm
z56_t|fAXQur^8D=xw$KDkF;&gGz#w#ZCW_l=hAYv)ovT*liFIpo7RSAO~2V<wQ%hs
zwpyQwd(PQ0zcA<uHB7&i81;JQrKM>{n$K0)N=z|Yy_(Hg^F`}R>GmD`MppZS<~)CR
z^r4EiUZ;tPANLtP|5Z}5+Wn>tn`P8;vpTLjbQCNv44V4-X_30>p({(6j&4~!-MQF#
z3RA)4yn~*G)<@s{&N&!)V@KfQ*KSHd5w+zS>b+0p!&trte9Zjq>c8c*WyP;Is|`h~
zgx1aQxZB;a+l$xtb)lqxSx^T<X`I~dS?Q5)W%5KmO?B$DsSx_UCwo5gl~tb)Mr6sY
zV=jNwc$PVMJ<HQDZVmP@mG4Wfl@fe8H0H+}%;Grun`i&ed%Y%2n~z>Q_1CZFmf@$B
z0XN!QH)!cx{9GGh%yufswxFrQYAI9qkIFU5;(~8a&#K(7Aj0)3MfSO9bpnHIdBT!?
z<rXmwN?nB-%&q%=%XK_3dUw3!(O&;sD&~u(>l8TLJaG8(!x@(}zW61UJ-TIX<#%Jr
z8LO+>m;NhgzKeHhOHkw45Hm59vwp#%ue*MGxb0Yc^(I$)pMY&o))TSAnN6=sHy`<}
zt|M!)HoSV~t^WbL^`1&sZfE0MvqOGi+2O!?uB=PKKJ2}V3IY~di`_jvm&4O~U9+0)
z?Es6c2~{ylw#5#+ez&Lv++QENK>c`2R$;EiooP!~eNrfObAMa7`jp=@g;Qp$V*iS~
zp4m{I{L`aS%=S>=_F|PMlRL~aZLe-!%ii>=sI+i5i&W=kBZ)r}z3iu?WRDBKV%&G=
zka)-Z3y}^>R@rW-I)C(;bct%=-;)Br^}blU@n5c7x%^*>dtAiceG~3%n8$S`T-T&_
zi}HlES|7O|%Bf6q3=1r566#sMd&{w2gFi7A5?<QUKQ%v9m#;R<)M>nT@142FUa^pC
z<*C=Fw%a$Un>_uP_#*XIwxyU?)a6rE8$Vs}aV$Apv;XRTnJ4F`Fm^Rcxyi*%lf3lQ
z*xbEN`{=2i)`}Vqna>L5goKG-G29^cb?d5!ozHj4Wj);X>0G+`=hwogq|e?xT{o}e
zPk8YyVUu@{TWoe##fJYVjr=gxDF4j$SpI$D_g?J@x-4?d%&tbs?NN#9uQln)eb1LN
zb*{Shs`lvAumb1m;|0YAI?1-BA6|wq<ve;laox_kyw*97TO)pMSzov>Jz$;f6=nX{
zoo*7(J~syMD$dyR>`!l#>$PCcMe{u;e?Pu?`Bu*KZ%1-^^8K`9a&q*aOwf;hQlR!I
z;V;L=ifQjS=KBh)7Y>>FHlyauzjL$tt0bFwcuRF<1?v8uJS*g45qYK9SuWB0;l_t|
zKKu^&7H7Wy?cJ=MZ+oY#_<r{4Hj&I1*|%#~l*%SKf1iEcIFH%?tJ!-m<~z4|mpuJ!
zyXCr#{((JD4reSdSXT7vf=c$0>3xCUzQnDa&SPue`uWFeiQm^`-R`pY*G^EhR&$Ac
zGvk=`ylB~}^|uc0^Lw(S_2Y+&2X(&Iovq3&e}3NhZW-61?Ghqhvixsn@VXcZPW)v1
zNAXT5TcQ4S*W;%yIIHCbohj*$Td_1t+@L7q;Oa?lc9wOttPOKuTol=Kn{#)fT2&{j
z^pmBId8bc(W#3-M{pHpxL-F1zzcVcq4z%vLYE`|aQt{Oi_xQCp6?R`Wk~Gg0V4Ccf
z9A$NOx+90mf|Goy=fwkN%x$arn6~Ywi@3t`<J@0MSr@9W+%qZU-uJafK9`?j*!pwL
zrI@!mKlH;UzB~EHZ0YNmZrKMGX<|pt@7yw}Bz%#~o%4Nr`GVed`F&Y^aeh~P_?$%B
zrw0#xbJ%=w(zbQC9-6K=Cngh6cxT_n-+LW+6KnsligX|PYB%@H&ufkG-+QAgPd4rK
zyg0|7$L3LygW|12<x&T9Hcbslh`aJ4ZS6B*&Pm3VhK_0y`C-}rmxkLO*}|sq<=h&J
z?R@iJsxfc(TweWr_Jmijg!FgKbUx$x>H#0$Oi%wG#l=CpLudSVzR1zq9du~H3h!5f
zqVv*{o=DF5V&*X`G)6U#_ltS?`;+W!a<5Z6mHuTP+H`RDYWY;DD}@E(QZ;9%A9j1d
z;!(1q&1J_L4-uxioRTS*F9mE^diATxk?UzTvz-|>CL7r?3*23=Q>gdSif!W&^9GGy
zvRpfy7v4K?LtadSciRE2cr)GQQm5MfwQi7<mQ5@^ZLoUzgUOS#416vsa5?Jq#f9}B
zdKz%;GJo4Kt(|-pasI!(f^waMzU?@4V#Ti|U2)SZweOVrUOT|wQPouO<myE2XrEIn
zgEzf!{GFmvB5M6r^=bagBb?RGo+)RBFX!A|@_a3i!-EyEfw5gTYAzpO{Hk{(HsR39
z*qNGH%Z0rKYWBZTw0XAY@vRxP>*vk9wb%0A@(&Bj4j;M|&m0$I5q{xrCzJV9bCHP;
zZoPaOvQ^CY*e~78xl`|Y%D#NI;8W?71^f1|YyD;W`e*H&Glfzjt)B(Iia-CApY?VU
zqt>d>rEgfi-dR$u_<C)%v3%z94!hqmN)O~i&w1EB@V3ueSZ2>2?5}8Wqw1K|+15+_
zZ{EGXCHz*3N4WU-Zhs>`wzdk%^2>jBy_)}R+V|IKEC0-`HhErf?@_@X<7-E5_J`C8
zztD4v&R+L@;+d#w({CGCjrX3qA>wg-s>+8teZvIXB^&2`YWj2L_Yw2u+Z4E1LT1eK
zU=w(?S##Z_Nd;jW_f5{ey*u^vt%Fis|JgntV^U9SpXc^jcWL8yzO`Q?)(g(;^>4i-
z-*ohhM!xQsW$Y8?`&d7G&eF7VJL^;N4X-&1llQgHI{VFO_0N-qAI_?M{ZiTW@zkD4
z@oCK;&K}yEJN3Tpxm)ro%m>qr?FF~!{`L3ME;%+YTfJi0Y~k}ig0CN5<9=#GPv{-V
zb5rNcd$V!QJJ+0rUrQfunCF<#FFDt^_3+P@ps$Zb)Y%QsCF=iSjCpuA+<lXNqOEa9
zzy+JM&sBL%-`2Yt*<GtG$WioPq`G!dNV&*q%Sqw34FzQ!T#<A4$W2py{d3~y{z-W+
zOy5^eEwEvG_IvFo_S~Oyt{;uRF5q<ky3n7#yE!^*(jx=<Gw!o+f0o{;XcSfR%KP$y
zeHUuymkAa7m$O&R=U--b|Be3y<Gg0Q+n=^pOFl2KY(FIIz01-qV(;aI1q#)%n+~7e
zE1R3@->qh`<BPpb9?Kh9mbv|@N55Tt**`}!zek6!;_@2p%38l5?pX1&4^$T0l`4rZ
zypTCHUQ1=W?ct-RN~^3`G^Reg9y#|;^WKwt&l>IH^muSU{pr%*Qm<ZC#-9AmqW6^B
zr7UyhlCK|XZbyB;wm{Ekm5lHd&u@PZZQsBB)bD>giUR@~7l?Lj3hbx|U%#U6cFOJi
z&v(mK%YL76<Ja?CQ`cv!^=705J1=INlllF`wO6GJ72I+*xF4v9irc*HOKHlVeLS}D
zjaT0vFI~BOeZ#vKHP2rwG2ir*5@vRkm>+1tc>Q1N&v$QaPbpVU?I>M7bIx4Z1EM!h
z_pN=xWFBoU_ug(UL)c}Fn01Ed?8@UNrw6xW9`uct(0_OEOw1#W^IyLvOnA9Vt0sPt
zw#A2eviXGz%cpb39(*e``zYh<t<!F{Z~3nND?Xd;h)(5U>D?OPQkzdjIF<6=ZQ{E5
ze5%jRDTd2U?e&fo%?<0UDB{mI*u_yT@lNvW!rnzX_a|4no0OR(><v9O)wBA+r_PY$
zr(^!BiPn9x4Llxuk9WJ%9=2b5&M>`ux#d46^I_EmyasnCyT?S%)pxIoW!bXhSn*Hs
ztolQdXPqR^KK9FAv*ZoK@u_D$Pt2bmpc<3&(WdR!@5iO$d5&ML(;N1Dcs5P2JbwKJ
zZR=?cOCP9&AG$i*tJ?3$-?hyAN35QwmN{6Ruihc6BO}o8j@=|%*D6-^Z0VCn$D2Hl
zsooP$@RSv=-VvO<_j1yV9jCjK@7D5{vifJmg<Z*9<@u^`VrkG0?uYG#^DCW>3vdfe
zPhYd-#Jh<)?;b6@^lZ`XlY+^*>%L!7D*kaMpYQe62Wr3c(_-E%3wjWGO4%Y$-FC*w
zm-?A^KF)TY8=>9%-aW!^$#dgNkEK-S>zl=<?X%>XpL2C)*lVY?GR5wWbNl;V7kyZ|
z;JNIj&r+?%f=4%Vygb;p{JGrQx`dma%nlE(eJ=Upu%+;;xnS9pYo#+^Chp7Bi00b#
zroczh*39A7xvJC0)7}a?U0=iXJm}hwY^%z9E-Ugxl|@aZGR%8kuX*Wl>-FUA&u>=k
zFnP8}^j&{->Vs#m_ua3UoBGu=F;OYtSh=jst9&cQhX*9t+-HQ?&VHa~`Pz4$(6N$B
zc9#yHaI1<~c;II6g9k55HaKaV_i*)Z=izM65x*<2>i+%GAd5~7^&{N2%l<iqI5nPJ
z|FY=n%l37$w~YVp$~ke4yQWy-;ep@978VEe&;MEQ=5O73(-_0G&tKd2MQqi799ODt
zvElCrolK)UcPrlMXwNKXS7<LjeC(mj0WGcN0vw&1K6dZ>yB`V6?_+G&e;#bNR9hr+
z-{ZpFUuK-E@d|aGzv#2m_doKV<qAx#e8aCz|H-?PU*_79;IrnlXWsbZ(z-(UaQB3n
zlbJ1b+uyM9UYJpnl=<FJ>$dHVIKH&CFSn#vTJEt}d)sokXvlX{t?mDegtK-x-k)yu
z-c+;HJ}kfb$Fe(4A6}h3mR`K;xp;r@7p|Fe-WeJQgsfa0@56l|Jt*bH{oD&%FRg7|
zU&Hi#vcROT5*J>Z{16qnKH=Jdj^g!Pd4h@OvuuC3FwXqkRX*qb49-nE97^kT+gevo
zPcS%Fb?rfo(99K^78rQ7e&KpNcQgC*)eg+MUptp9Dmfr_rK>RFsX9B$3pvx?9TMVo
zQL$UTgvzsTPki4p{lSALDPeJWE>G8}^I!YiSTINHnf;T%8^_hJf7qV;|7Z#Wdz-1+
znWA0KtY;lw9KXq9+phcj{%`R=B&Z(4H>1|=*S1x5XBGBc3l%Qz?1~K!7wwgMUj1+B
zzE>{-W%UJ2lx)pD%HIn8tr)XAy}z6F)DNXy(^5_u2(Fl*ooVx1JK>e#Z-wb)>B^J4
zBBy>`aeCSHsWB(4x4W#Bx~TL}Amq2`&)a`%%zd6;tKpM&xRRlH;nP;XWvlz2K5M>x
zOVoCMi^ST@=j$x)Tz^<1du-BC_6rw(#;s3&AyILB!S9V>xA#O%l&M*5cG7=x<JIT@
zc8|i+2`)xIQvWia_<xz{w6WCziM=mF9gIHDknXKF<NY*y!sYAJ@;3z9O}W~wQnJHT
zCUe!QwJCMWf@-Z6J4&&2Gk5%RzaRcOIz;Fz_xEi)0@HcCioQt+Wa($S?OOCR`^V*V
z2C7z9SrQ&^dg$4^L$R~<)1<JQ-{1X_l%0O7K>4Ez%ezZC!E>3SGMTPOonn?-JfBDA
z($CqxZ!U+)@?VRNKJg|0PORh7=dDJDD*yGJY6X7(abA*tK5}~00d~Ln&HYzS{U}v?
z$GG{@E%%+yey?W!H9DH<D%A1i?8-*H6Cvwj8&-M+b?*7pS8njPq^z!u$3o?fMA1{>
zFNNz1`WgH<*UytF)Gu5Vxns74^qn0$9-Y-*FSAUCeG2!NFSkl==`+jKY|-5HV4j1r
zsIK-k)4$VRwr%()^O$FoU&noQcdhoa6R#(qdhNTXLXH3UtzwV6{=HNFxgKdg^Ur<b
znh7u5`F+h*#FPFS7};(Puvj@kF0M(%W@Urb$`4j6b$(r8(zN%xI$PZRaO~47`8nDJ
z*3})4Qg`i4w=A;wH?zZfWu#Zo!uRRX(>`sJ);#UiD7mca(3UO!AEO1>-!n$9dlyzO
z;AwGd)w$9$t&XQ8`YW~X%d{+Ax_(hkb4*^s(Knv+W-xl5?y8p7)b@Y7t2lm_$Emf!
zyVq^-HrL$nWQpL$RrB2!%<_|WKWyr9`C{7hr6~^{U0=J@x`OT2{IxG8JZF_%w$I3T
zN9T=Dt@`aY)e&t~Tnh@~9@N@?xVO^OiQ(4~Zcc4xpJSD;Z#I3GmU!^$lW<K^s!rrj
zk9*e*xxK$evg}=Y%rVlhsGiT~{;?+)lAr3{{PF03-=<HCmt5L;K>GU5H4#63!}O+=
zGkZLksIRb0+=gxb46Cqh6Ia@=zqp(2ET5^r2Y<^PtB`NG$G_I}+ijk<re?ytqFYtk
zI}P{8b<Xls57~Km*Q=LyxiYd9MeOI7KUgo&Z!gL9S>2@JXWW`ETjH-fsvWQP%ij=k
z|K64S*^JkBmL&>V<j2I!Ds4~-P1x4_r{w6Wwp{;r+tQXC{2O=j)8)KRGqux`uSW+T
z3pnYawS98Kiv!E8-Op)%E3liVZ+K$8zf<M8>r0QQ=SAdlKT2EwAY|E`lJg56yV-qH
zHJF$+zxwj2XGID1mfJ6{Xr3~Y+5U|Anh#4WAHFt9*%AJ>W9rve-t)Lt6+M_%B+M6-
zd%g6rsOk;}ww8@Gg-cdUSZW?6xvoo0>ul}9chm31X{~ws*=GCUwKm*gReYi`BDIN4
zuRqqcXYFkd$-TFIOUUvUd(KsSm295(%d_mxo<24O^L^K>ikYR({=QZtx2>%8yROGA
zQJ%H?Gq%+B&Dfs%_4m%r`}-QbY-?FnO-{_qkSkd}#p21d<0l!C!+vXO)Wx)QDHk7?
z`67Dc!koI~tbdD)zie}oTQ@1Bwno<G$yu?pdLJ%MP~CsPjAMfGDuz-8u6>akCTvX;
z-JB+P^2WhT`#s`H2WpF~IV0a~v9Fyfn0)nsx#28JHlB$&FYK1;FVb$=^Xhl#<1oc6
zrVg_$ac30UT(-ZzVUoY*&yy2BZnkl97G693itljLo`4HN2ZFn+B0hSvaBTL!++^Sz
zskrie1nYq{4N0o}0vBIxJN8n{S|PZ1#qsl*3*X6xD|LMLduzrKyxA-6thUdKzRX{$
zM-P4xy~UiE754De<b85qr`>%lz{~XMn*PEIeUB|vw#6&F6=<q+RjK@EeW%dIVNQI$
z&~{PBmRW+1))IO~GwV->=JH$$ecUteLg(|(Q$%lXPO8q^#klq5p$x@u6Q)Gzta`Ef
z_;tG_%lvmSYQF3YY@A`Tdd3&aBm2IKY_4C&!;|@jyQlewS#wc=fUQH$Z`S^G`vcn)
zf5^V6oK(O0PxQW(%eEVFY}~E5-+qDQ!RJlMf_40V_}I3sc;Hr=9w9jC{GluN*&a39
z+j+ZbzndX#rG8u_tTZERYH4S`<~GyVupO=pNfjn4rq?INyOp|MRateDFXZ+_5&0<=
ze(Iq@jkh<myl0X+uhiH&?MdVOF8P*}bs3uv1<udee&_jrA@j(6o67|6Td#VxV%6eR
zeR7>%n<`@Nd&Y10x9UR9`CrjrzF)Y%YX4cS3sRm(pDq!d6y_17nAy5S^wL7n3D-P)
z>$f^x<GNYZ@vv;J08@a<mgh6{d3J@h``$YIKg#n`+{&DQLx&%<mTAAV2%dY$*+~2b
z=kE!3qZc-<F-?<Azs9vWY{u3@YechV<aVf(-PrVCp5yL_Er%9~iZRdeTdky*xNSS%
z5fc;naL3&-OmB-6H=VXUueaUCT4wfJrL{Mn#<BO@&?~pveaH5FQ7iMkiuTxpdf~de
zIjlGAJDBvoD3R$BZ|3BVKTAs{FPklr7qr9kp<3%~k-S*063x3yU+T5)7I_!6!~3CL
zYqcw*=3V!fihJ5}WSH_|w!ByBy60-JcG2qCBfEBURF+SaY?f|WQz71(d16Yc_4G?2
zatg@_dlmn%a<kmjQ`u)N!c%ze+RW=4I1e3vJ7XnVORkr$F~^DR!d4G|*k2Cbc|}QP
zgT+b3pMR|$itgMkmH2_X@qxAJ$EEy5yO(V}HT8Iex$~FAQ)`q?%n5X3V6uF?hl@?A
ze13EHE&G!{a+!DfoE2F5$L#D0+0!etCVuswvGApD9w&QF)TMZlDX--jT`TXo@06}O
zzWDBGo+7WAug*noOuHT*_V<`t=8eo%YsHN&L?z6by~e2a`xa}CWw)QdS55vSFkklE
zlxL^6DfQmz?myN3=gO4-Pt9zb^&W>`cE5Cg(;qMAq8jZK-W&W0_eHi(?!SHHppcth
znPk^(Df`DymgYM*gmGV~>Y6ae;G3eE)n%cbXD1c4|I1R2`C|Th{nsuAiJk9GDHar6
zdvHk1$AT$Z?@G(dtnDl>*;a4)x#X@_Y1S#Ggj>Fc%2%j)w0(IZF6mnOP)~?EW6}?y
z<6E2kQiG<bd5143x;!It&5W3}|4LuH$Y>RCp2i@(O1sr}7mHi-r+Ht?m)?z6)?H!}
zu*-JBdhI`=iKS)cj>Rwkt(&oWt^7&xMS(BsEVlXk_aC`;;m6(e#y&ZdukiA2v1XjC
z*?RPrY==_0+~-A~cgS1G`zcA!$}`omD$3GS+GRPd?q%GB+RX22+q42U345J5pjF|-
z;(J*&W(teE)qK~5VJ7z%c9cjcNgR{2`gr_fN$#Ix?wU8%x^}trg#?&AU-VC`*Y53c
zuj!EwwKs<yk<`^z?z#T8>qJ|>t^1cFceE}1_HRhgQEPp3<1yD44Ryt!H^QtuU!8bY
zZI!lPT#~wd?x{?jv`wdZ4;=URHM_WCzu4WNtvBm5GS;k2zP2Q?_2`y`=I(p=7L*-u
z-yKyN$XT&;?yCJ~Z~y(V_C+s~pY=kQ^xy#P6FGKK4|vb)Hd?V@UDcYD*bJuh+0y5Y
zpC6kxW6}CVL+!*oFO^xj!j*Gt?@IBfT)z92<MaOcTnzt2g}!B0uI^r=`C+zS{^iWv
zHWS~IuE8?7+Lf<%c^Dsy;h$T~T6a9@{Vw&C*hhB_X734-EH<}F*I0Gn<thCQVF98X
zeLGFejqb+xZVq;sxXxwaolD!4R_iXgx#bh%?!1$`-@V_RH>3I!tHOS(*H?09X7;uV
zM?5`n^M>3}$2V%H0vPOnAFdbGaFsqG5m38n?a8CfGbf#DbK=(A^kvss)h}z355APU
zGnK(KY{m;_&D%bWQ;%gCc-17Sc=jt^^T{zv+b3`%^6{Dn|H^;sFt;`btch5&P-$!9
z0Xf%4GZjh#EN3si_jk&V$K9oO`Le#bTAy0k=JsvorP#H0OtVwZuxX|$+ujYZm{W83
zqxM>!-ScY9tQnVX`KRUm>iyd#B0Jhr{g;S(K8ZE@RdD6>@gLVOcdgECO^#lD??B1*
z_P<?1>((%;9zEN!I;pkD=3Cuo{_p$!7kYZBd7NZ<yDg;WZctmHd0U}*)e@5f({}1r
z&$+Z)@4C1kzx~{V1%0w>N-kfGEeVZg^xVF7K?2L!a_*1p5*dA=VI^mNt={?SU2%ER
z$yIqP_Dx9R_PctB@3Kw%xfO3eJrFCc6*&;GUF+^Ct#1;tb3Szx-x1Ax>-03jC2Co4
z9Z#fqy7RGDcP}rx&~KS^T7837$(zNWHf3LDPMGGJXx(Epc}u*iv+jcV*HgY~vqyw4
zc~_=U{w)2IiO3VqJxMAD<Q8Y9FLzJ*q(3z-Fd=!vV$HTM1zCYBrSE#&UH9bmg9|g(
zU6VP{mUF_1`O{LfRchrNMw?ZcV_mHaI7Pk-UE^);<qXnF4(Zq(xck7okNtvDHyJmy
z?mhj@+j{;r&6rExU$0+Untn(%-_&EuxkHOxJ51u26?LEFyQDwqyz$pNcFOa=&6`@B
zlDc!o+m3}b+mC)on=60q<Y$YPCHvLx2gD@b<+!=vxb|7&1uO5Ia!I(Yb==DPh+om#
zoeNhk$SL-`sg-7&cym|CsY^@t&R1G5$ru=US5v)O$-_Flz|sG1YTVq*J<soF$fdro
z{1<rt?acJumS6T1U%NR^w#?4@b<X$px!=!O-L`p?GIwJ|k=^{8Upw-iD_RF?x1}aW
zlp3ZeGo4t+o-eALqvswFlvBiD%XHiE&J!NBHiMrB(hhNpr)~JB_V?_s=r7m(zFpPI
z_?0x(f@P(v%`?BwR>pmCtsm|?7)|xQpc}Zo;IVSa|7*E*#%`%M*My55*&*jDbabY#
zd3`Ex&h_bmPolRuoL6q^4$JQ3X-wK+q<bs;Xjt;ODa}TG$NU8!Ki&8tmMiY@gkP7g
zZmrK{Y$-c_gR{B&6K~Xc;dd-atE0cAJoQ^={Jk^cqDz-r;GKJ7nyWbWg{z<46yh)B
zDr_lr-cYqa{oSn{4qS&;c?$pK*z8qzz<Ae$6Gwc@$^ru?ch2`Yv&D9@#NJ8vGkFrY
zINuojbBRzm-xVpXX*O{!!%>@{Jt{|Qjh<W&b=kJ+#E&%DzzJ<flb3kfe3`Pn^!6_Q
zcAIaHri=a;-Py1)Wwvp_w%fC=)QeirUElt;>so&8{5!34-Z{L<TRhX?aS3};xm?}T
zL?fNuk1jmB!C>>=X`Rh_E}J**I|Wi3e07ALX03OXQM+Z~?d{XGX~DjAr$6rTnzcya
zg-U7B%V6H$e|nc>K6(>8GvI`cVBJKI(D(O^8!x)@{r+~$SiHXeh0r$Xna?uvCN+L~
zm3B-o_06<{dTD06r_NNjTz#@``sBHW5h+)^_>0QwjE$C;97$PUrLrkG!8GIc!_`w5
z-dgz=)-n0m**GUjmT!BOTk`s#@A^lFQ+LOn%bkDcTy2?UJ7;~7b$kA`sb44M?cTkV
zS@WWopLpr*bo0_B+eP#qZ{F*Zx;A}byvY69eu<dn6SH>QR<oXRL(?TP+K4TpGUnB>
z$Fp=U99V4rG{t)LCgpOW^QWs`mML78yY0IzcFE<c$+qv0y>+^)fB9|Ny}T!(MfUm6
zzVc*le>;8mC-v!X<FC9uQN=lL{q&b*zw>yGni<~^n;9ngC0Bb5OY>Bn4N01>PubMR
zU!C)$*{^@m&d$8Z={B#s?nJ1{ow@#V^2%x@?ssM}vz=63x_*~^FWS}p^Hu1XvVGl_
z@kawp|DC;57{C6Nyv6RAj#n>_<dk1)H=H(^W7d`f<@|nZw$^dKHl^RK`||Jms+lRL
zqQoVIrDF}uE^gWFv(LbE)|N%xX_F1DgO7SX58>=tX)|-pyeY{EQ`c<p3Nu^wTJF!a
zAWQK(_jbrtmsxL{yYA=>7MJ$7ZpT#C3)Oq)|8o@Y_L{aRYw~@zHP<<Ae~o*%xx_X+
zVXd5g;?8?(ZuFI2651p^=d);U@Zztxj(j>kd)5s1iMJ~vZD+h%x3+l6oUE;pm9^j3
z3ALYncm3;~<#$eg54x?N-l5*%AQD=&&_`HX_?%Zr#LU95G@0rCMVqd$F3Ayjoq6d0
z_H29C^2;q7r!8zcbK!QV@DBZ+q8slHq-xK3(7W-^ybFJ(J~TgVV37V(iAOU@D&xk9
z{v*o0Te;rvDY%mOuS@RWq?e2Jmh9T=Hm_g*P;%$p2`*e3O}?%hGBqmYA3s&p=dY~f
zx%~g*zu9g-FHW?w`grL7yX%Uk#~3y1YikVRb4>~!AE-<TmS>fZ{AhAVwEvB4-|_>6
z|CU-$`V&6Q@eo5tApf%n-AK;(l;etr{_v@#)F}Nt@`=+sYmKg9=p=nkGp8tx%mXn$
zr#!q9TAP+Vf%j+76Yq;otn<5Ste#A^x@Gs_=>+{vQY*3)6AU?}CO`PN;jd)8)IO7+
zOu}kMBbhGM{pl#LyP3eq!KM(waPUTzsDMw_TkZKQ0&BLgHY6~ztiByue0o`2*%M)%
zSBxCzszN5~v(DZlrj?bYKA%N^$H9Q%AQOj}!v?0Nub2ElY&Hc6kiMLq5sU|`;zS%&
z7+shYgcv5W2zW9~6cLc(wd7Qo#Gu3|z}ax%wMv5rR|kuO3L{Glw+dMBDT{z7<3xrI
zl?D$ckPL&8=Vh=)PLO~~g9n2P6G%U&!XzdoMv$3~U#8w-7Kn74&MFYeIFYeKr{M%s
z2aiJvgA%vGCI%&K1tW%utOAh?6B#?!NOyxpnUt780;~eBn(d%0>{>&bCa?;)G6XVo
zFgb`Yvh29Ds6m6lMbJTn(S=cgiy@E&Bq<`m+MvPIp#+j*ROn(@#G%l|w1@#D#R8Io
zY6x@?0ZTD}q>}%X97}$ANM4M?lq)a%^o=`w53TtP=COBme0}Q}^n}5`vSFpj?w!sy
zJNqpH9y2Gc=+L}-d}TzJN{#0aqZt#FU)O(9e0=zUon2pM5X+upjA_nmKJaajK9#R=
z*f64Z@gfdB2lgX@Nvq2}?k2_FdXiovaoFDA5A$JrgBrt^t1oTudKk_VzvhotTYTu_
zh(~L<iW#S@Xc7tySi~e0dg#;%?}^*`FEzNZsvHU1z$Kxj5NO=;PpND|ezT_5p?kga
zKdyWbJkg)sPB!Yz0rP|ZxeQZVb=EuG`1aA*&a@^gYR$vw$@#MPt-4sh&eO{`-}vU|
ziz4RhmUVVUrxpg-)n2)%@HBS^Qx?m%50A@hl53nNs9M^dYCZ5{$_GxqMNv%Zw^Qz#
zM4I!yPLm0--x2A_JF!Ce;E#l2`?@D(l`8#;sreiD&ONG4;Bnq}s_pg@joVr(=XjT2
zPIRu}@%;0G|G<?8Hk%o}etD{_`jc^A;oXm^55wEyCF{2HT;9mBQT9*8ulZ8n=U%p+
zQMY8x`nD(OR?jxJ+8wdgYC5xd*B^tWe-BD6u(M|G4#}G@x~gvViMkK9R&(UucDa6G
zw!IN|JnpArfK-U?yzA{#v^zikU^ky|=6Ks~<+=YD($_Cnj_=nsK4`sn?YROexxnnI
zn7MXFn{)g!s*X5_dd^Y#<s6gH^IR!rhTUc780V9pkLPLsb5xlq_ej-cUTgQI=SDx&
zYX3<Dubj3aQB*o?W1?(vL_6>64T-X5Ym)gO^x;=_%e=21)>_Z@J*)Kk%f{rS6jPVg
zVdZl_2d@2;B|AU5Y_@EESmowE^VPf0_L;BV6M200)w^lOpVwVq`}ppwcbgyI)p~#J
zV_xa^n;-M0mcOppzUz4DyR?3j_;aOumj9e$n><s$|7Xba;LkmqC2u{yoYVSDqa)3D
z3W&vb-eUGbv-xS-Z<-(8<gpH%wRQu4Te@*^q0srT8|=m>g_c^*Fp2)y5NuK_bE)LH
zb8#hm`QdjTF29`g`xW2E#dq50{|@=K!cwtl-tMrn{2lL<v)QcgH80jIuYJ~c`$X*J
zhyd>wA8uaGS^6X4*`{SX*JbMEe46T8oOwCswu!j;&sWQK?#W!2KZ|=#P3z^DM<&-T
z&bs>TJC&(dJnQwGXHM$zFHNRfo!#nJym9jSvRS9+KC4>3^TEk*o3pC^#b+<ae0j2Z
z-m^{1cfQNitNAq5zxd_lnBOMi_CH@O-}xtVUHvSUhPezMR2XbH8T^?J_%T!nG0bCW
zP-jd~hYC$**dWZX?1a#b%Q;8EY9=!nxV9ZM@?$zM8Km(INOeJbrVS^9HRFeh^<aYp
zl+Nv(9Q^(HQLt{Xfe$<xZg4gj2ZH4x7VTtI{>*A~yHM!C_URvLQ_uU&n>hK4&$Kr?
zw;nf~|9kQ6V)1*q%i2yo_vw(hD)dsZxY<RH{n|_|{&mKRMb6$ozJ0nOwCBCTjSU9`
zIN8iwT$rvmyO{C02R#kk)5grhDJ0s%F67Hor+f9FXM)ad!#sYmgF&GSuQMF{<X5@!
zsc39m?x`r<o0q<ZehZ(yclS5vo9mv2?k)5Ex&Ozj3)(kWc;$S(w%nK?R>SEz@lU1E
z)ZmA^>in{e)}JiXx^w;f@mWj6OLL$1&9y3$YCSPC`Qg_2mHSs`pUDeqGJF<z<=H03
zseimH{!IP9|NS4OWuZDss#k=LT)02^9q;n%v-aLi3-8&Ta^1Ax<h}V<;`W8?)|=%0
zP0Z|u<HU_`JHj`vJX>*N;yu5QAB8NQ?b;`Dy!e%Jgx1@un|~)Rj9JzDbDe%*?)R+d
z&u`8dEb-oIc>n9tkkvINtKKe*x}B!JT42@N@~hvLMt#cq{qAP0#RdJV$7-%mUop*j
z>yw+YK?QR+&3OMy=<2cnU3MFT3Dah4KHi;GxzemXYjS$I=E2faN&8-Vhe&VO_B`ON
zh-A!iooBoExomY`v8|$Py8IoV$epXV>0G@QkQ-sE=d)wxrrn{}riZ4j->sh>-J9mI
z`I+|gnxAI(BV+t8r&OPOuEchJ#yjJAQ{&FQcsgg|I%m!AlIxQf{_dH6aLZpa<#_hX
z?~LQv4;w0*Ojx|M&@{GqGiS~-z7s{2@7{i@)wF$5!+*m6@&4OiMgPefP0>-mu5)Ig
z=)F%0AJ2Q$E^LfvtrPummV3*>X!9re?-#x~Z_xr43Vjd`kt4vkzxlup{|nC_7j`}T
z^X7fLV_S}-%h_kIcojk#8R8y$hAea`j|+Ku`D4Z6rTaU0elCCE^6#D2K3ivw%0E-J
zPI+IcjC4I|p6^||oo{FC{h0T0JPeOrSH$#p|9i7$`8M^bwblX4litc37QW><v_ACW
zy8|1ayePl?+kO3H`8keXCjVZ(;K$ruFU$D$XDX*(ZL&{Wk*?c3pG$wACi52SeT-sX
zYQ2AkxBWe~{Ni_$iThTz8Fqi@VBUJjF>FDF?LMv|2kyUXI*T*gLgq$E8dS2{zVuin
z;2rA5uX?^ci^C~@?qe%)W~S}38P^uQVCXX1t-;u|eYfmB37!tKr61}RJyqYvxaiq-
zPH|PQcS-CDY7B?2bljVF<4;GLc!|2}<-jjOr!|!3v}8QuOjfv+ziiF2nnp{V8_VUH
zwjQ{BX6gYuTb4JMKh)3v_Gh<?@Xh72`=qXL_A6|xbY=IO(c-)1fr`O;iHj%9^p_cL
zdfpW_d&a4pi)MajA007qGr5-XUnF=2@9i@)-+7%o>Hgi=@sCKb{_ktz-}a`9y}s{s
zvff2@lSSFTBHarYl{q7O=APC4?<X05@cZXu9!>JQ?><Q7__B3jd?@4nSKOSeJI^|O
znUqueTCX_NHRhpHdO>ybqLZGJukB%-zLjlb_QUn1KenmudvQ8$bIsh@?Ln#seLP<1
z`k2PH7X_cMbxOZ$^I7ha<|3A*pXYsRty{WV_gPSTbYoq#adzy*zAw9<X~k}7y}#~o
z$ff0`3wP9AYPp^C>hK-K?VS4NQQPl)4%y|q`h4aZ%UqqUe+mSaD4t>WPD(s;{By}&
zkF<~a^Kw5f*)spglk;8?A3lHFv+|qu&6$&R)PDWQ;`+C4a{nRDdvg7D#d4-0nQNsr
zl{OmBiClX$*KTjcj?*d4R=>_%Ilumy^IU;B{@2?VpY{0`^80f8+oRjftnKFK$))+8
zeRgI=;*8BdH^j#8J!A54|K+EdyQ?$ZxBu93ckdS4nae77yDXghbx%#zq>XX&w(Xwo
zmiaLKp3%AKcMEq}>rJw_7QAQ61D{WcZc>k5nu(t;waYjBs1sbXzjBJfJiYq!=e=0(
zbZ*w0KeN1z|G|S@6+iCI@vFYhZzKETv&ogq@9vlHly97{W#=sO@B0jFb#`#Ry7O$2
zHp}<Y2j_SWu|56Av8=QA-)%L4i1{Ucn=Oq`Cf$_#FK+L(JK_i5*8hJW)*lj3K5`&|
z`$+Yng&XW5I?I`+DpmAL?UkN$TH@S;uiFm35{yw!d-(Zb|GERMnc?PHvimk2w-R`~
zga6rDWv4?77gucPFyHY|vFm_kSqr<gtDD@9rP4<-Qhxm4m|vWI^u<%fKg&aHG>*96
z)9rNETV1IxTQ)KM$1CqWC7C`~PHc!gYhB|TH23L+>}JM);^}#Zv%9Kfw*No=%=zof
zAAE0X#dh=^n)aCO&9x-EJs)L1a<=&1{3D`p_I+v-XJ}GNq|JJs|Jsi}<zJ4>Ud>YV
z_xD7nricFd0czV%DgFJgz5Gwl=1;C+<^h)iBJOg2+@inb>b1LH1hUt$E^B>$dD^q&
zdH$((JA)UlJ$Kw}zRe1SN@M9t=O?-ox4T%Td^>g`S~<Tr>V(W9iK=E9HZiwj_Y|s|
z-@Gl9kgaZh!@Ftw^g|4j&1dX6_~h+Ug9A@^)$$E0t2KEHDw}UgpSc&u$9yL7-JXMQ
z-aa)r@P;=n-=Ml$m1jdm^HFJ?dvTo1I`<MjvUP6{Kg7^&UQyB9zP-GKQSRP@hrDu(
zwuc$`^FMrG<Ii8;#u%2rp{n`n?bV45SEbk7OZfUNOCsSb+ide2d-jU4-Pm*R8LwIX
z`c}qm8<_SUeD?OM!GU+YW%(Pbt8IBUR5#z1eseF5pZQJUMfOLV+C%rZ8_p_KwtMnH
zChT9`5vIhC?{#h3r-fGu&*~FVpERfSnad*Km3}JL+;=%!+L@jz_ASxlvl5W!y3)?{
zbD>AQhQNYEcFhvjolci0as)1L5;+;b-Kwm($e~kXf)dx34kg1y4!s%^l)1iiC^;^Y
z$*fth^X6`r@2A~Lo#P+sR$lU#ckXBX_Caxi8rPQ2Q=9g-II@3oX|xMe2@uf`VHI-K
z(vi3QdR+Hs@welT`rS{&{N4F8`TOyMr_S6FnYHACTC17nai)clTyKLirbp=r%zAdg
z<oLm37mG4xIbW8EdS;Qt=UbdH%lWh>pH=I!;0&wQX~8ljT)xXMlyK=Tw|F5k>)gVZ
zGEwIWFIb){%&=;oRxDG(=llFZ37_tCix)z(^cKDpiqb2*plTB9bXj#v%(a<*k2W2e
z77`bf>M5&ex-G%|>4jG1MHMoC1a}p*=*c90Fy-6VcJR8v52>hig&#~!Vx2FWZi#vP
zAUk7M>$2#Koku0}7rqpmmACMz82jBvr}%ugUnuHtepi_Leb<5gx+~?<nJ-Ry=02@^
zdCvcan98cp?mzwt-B<nh$?W;-`hV7^ElyOOo-RD?|KE`E^ZT9I8oNKNJ;c#8n`vI6
z^ZkYtl^%mfEE_*$^z44{=Q7{h-^Lp%r4|<&ELS`|L3m$BwO}~cqBHkd=etbP<dN>Y
z?p+mkKA~V;?JuX`&{<i3R*8Kq|1O&>^-j2+<pJAD<&PIPUNcM6clj!}G30}R#IA?G
zd=5E!gf#!Y))o+UXz8N8jx+DSzW6<TLc)^$rONdVY@7jIC)PJ_y1*>QA(_|pVEfbM
zuRg?kuq_O9KBRrH;nENHm~B%(_P4M}2^Jq$T+;K{UGVC{Ko75pQ<OB_g8cqZo)i-3
zz%_YFn&z372<K1hMbf?sx3BZaoOY?`#Uy73!EgIH9CfAkXm~!<Rm_+;#V^Rh^Px)X
zWRY`07RehTz0!HYxzaV~EG?LJD3U9kXP3uIrKQ~>dO;HZ-gb%T1?@<<;OZr=*?YmO
zUBqt52gO!?&6<P@?NjWQd;zgzmKIDs6v`FOv&-wH)>7>>FRpOSn56}i5AinV1?}>D
z37(w`+L1VKX~E$IyQj1jU+PN;gv`(h+O0~yAe*7I)b&E2Hq%eWWp5m&%I?@CepF+@
z>9~)~H8%r$7fP=RY+*i@yToF-$MOjFEy}Le4`dlXa?IE+SkT{4!}3OXfjr}1jvwmT
zEux<r8(4T$7A#0%U=lKz(9kqR<Zojui;2nuMMgo+4v)z2fNfIqFFPj4i?%vk3|+3x
zuglENY2o4E#QWjvmQVaA?i`t4J$uoIe;MnG&j=<c3WObUi{NfK>d=0PpNHji;|4hn
z?xqb&W`|}e=x`lcrLczU(53~eN;s}2q<YNxHu1<Kg*AMKUMcACANr-BBakJf%$wb^
z!GmLU%LcD2?fQvx1;n@#m*%u=b&Akwx#|@Aa*>c*fSZo^v7(z2uUj|9fruE1)2$mJ
zgvztEDvrq;;}9Yer=MD__nJS|A-U6FTE~v7Q&n~~q#lY;?Fl{Pxzc9UL8qG%+N~Rd
zKtzB9ck4zFv6)Hbby?%PhAiU~X3crV9OlhM#eoeXMqmCrf4eCz$q}paeqnL3ghFD&
zKQWFe*EN-UDppD*{I9xix8m8?f1i1$x+FN&d~Z}zViYNyf8u{zNm7Xa+hx7Wb88<w
zoXzsAQgox^q`un6J!cc<Hc#6V%^lBX({Zfx1s~I<G%4;c!l!IFoP~|9Iu%#1skyLD
zjO)$D+s%t&mp4`ZSL66t$r5dq&o|xb#IAWS)U@}`StaNG_f0-e!G?A7za(dtF3`5P
z|5@cG$IErw-#hBGRc~cu>w5k<K=UEj1Ge9hGqawq>RrorRQDHiQ{<EnO@S3RcbiE0
zEIw#rHO<Zc@{C54h8b)-HYd2rEk4gI7FlK2^(0WGUrAYMdy9(jUb~6O+YdT+KZsJf
zrnD-rxmxUIiSYKdN`2EV)EB+iHs4otz5Am))BI!+$#*gAhjlI`3rctI*}<C@$#TSZ
zN~(*R&c#<Q96Fcc1%G?_SWf)RxATq0Ww)7TU0VWETnZ*f<O)rCtJt0L!m6`nn{CR+
zQ$=qVd&s+1Z4B7wyxMP_^X-I=LgudB{&%X(659$d-drHldi$(ekAa>^;nYs=Gip8t
z?{uG@bNAv-ny^6Qq>@skj>?v{mLB12hMev0o2)!nXmUoXDV{vy{pVr4RPcV2WpNok
zp`3+N|9kXm{<G^{sekH(#_X!@{8`C%8yq&D;AHbqX~~-2Tz)Mf(Med=<E~i8wL@&C
zmKPr1y(p({m$&ccvC8FVrfSIFt~<5wr#5@g_M?`M4riHdyYN~_S8S^vqv7fMC%#pv
zuK2fbv$g1kl1-Cub@n>zRO`&(x~g4i-F(pJ{>RPxGWQxBwB7pZG7qzG^U0ToOT_KB
zKa%kAlV29~PlV}sLeVdQq?3_qt84VcufLuzs~`5$B%^%!hs}>K+y2R#rpCYdv0=;{
zJ0JTG5JpQz;k1)l_G=y+M$Jj<didyzqD%K?A4QOo{Pz$g5qERTY}&dnw+rZ=GtyNn
z{3NjGXy!5rJwJJ;Ef%Lm4jp~jeD!Yn@#zJH-%LbjmG^sYZRy^;TtYW5tt9nTrt2n`
z?#(^kJ1nZZE<Q>-CZTt3UYhn3#%<FI3co$Mb<=K!YRnwlqZx&(wyRuPp=j3dOek|n
z`f2gQ$0DzK-?6CfN_?Dl?CL#s|KgB+Gp_F_{ALpUk#+B}-p$J;%<|IaoIR7ex2<<`
zPjTwym|NL5cFZ&cN!{$y|6rOo$M#~+HrIV;uAeCUCc=FDY)>iMGpXo))+bTT$0MJq
zR?V?J`k>G_xkoPDzu05nm+LRUhCki!*1vhVgbm1WAEga!{hNEdzgSdv9ekX2Ou`oA
zct5^tAf<<wTi8Y{K7KQpN6y^XFMFdW|Fs8&(+WG~()|r9?wpZWeEjC+oTD!fN<lR7
zgEWaWA3xi(S@QnL=~$GWJOEY7hEO`wlkM7t!fAz*80V!V%(<d#+%zp!IIS#HL!f3-
zEW@R%atGX3eO@fjq@;etb`$?frMl}r-P_{0^sOH0Mp`j1o2Pa}Z1&5<B$chVx?66f
z8236z%xf#Tep$##r=*Z2;?%-!p(z$$4A}1_BnK%L?@t%~;<%@%@|?|$lQSpXJ~q>e
zE!Z+R^vG0W$%>9Ay5$cI)4N={j3)mPsL0Z?omBp$d6CL{4|#U?_ns-nlfEzPW=LDM
z%Ogq0a!218MafA1YOkbyFDpMK1lVeZFPLY;w*J-0S3RC7?p`mLIiJ7soRr}{Z`y~N
zc|Oh4zoy8^tX1zm(P~r6^X=N?iMf{FK0IXBbAKVsef+g&;E9ha{LijW_H6N)cTQ+t
zmfHKr5Axh!NSH|4-ejL`Gug8u)t~S7g$uhCD>(N4c337=^}W&8v_!A>_Ja$%RUS^Z
ziDZv{>p8Kya+CLl{+Z`CRoQ5M+dHo%?zE5E^9H5nD^B@s1_oc!I5eYP_=(+h_dgXi
z>!i}0o$BrHLSCG+IMw#9=0OC1!lA%BlOMNPmAqj-S@q%ud*9a!>y?VE{lA6p(tLE=
z`A^n?ug5pnAE=#EylMLW-_Ivo{ZBFq3gph%Rr7H3wVVFyOh3;~6IgpWT{6Sd;k=O7
zRDRK^2TITVP!f~+P;316obSFT*Fxu|O#FPT@b8b~PO0-^^NM{Q9?8GLJw5wL{<_D%
zjrKg(5$J0?#`4Wu?z;|q*n`gKS@$z-_6N6Bt6T3l%Kb<vuuFZq^8w>&Yov;&D7A1g
zPWe37<jZpDU;U>W&dxRAFiJVZF^5Mqf2-z%^GeJCOY<KdFEqKo;p@YYUS;ERhb`W$
zO4}h5T*!8J#{QM-?Y-ONUvw0AO|8#Re?0YI8sDR1^DR50vm(CBPG``oOzGs%)9HJZ
zwsH009Zx@($bFYtae4KfXH0Ad=Xa-Du5_2suDiBC=S^;vL)D`^zTgi$-{zk?IK{0d
zciNrBbC=vQ4=tLlIQRLtNy_z6$A2DeRj7NY_3rjxm1mz#bYoA&ziJcV+-~|SK==IH
z++MpgyN@q_*M48xztVKx^iZGYN8j`Bm~y_TChn-QefI6_{~hn-3aj7BfBdVr_o0X2
zPdDa#i{Lvg?LsR*np+3`+46l8ch{K;;cY%DM|YTZy<=a`$7;SI??LV&X@{Ww^MWMA
zBAup~KQ3s|6Xw}=M?CIJ7W=*Q=H6ddXUlBb;cgjoFQ=}!_&a-c$yVp%x_uvmgEn9A
zh`CkMzuGka-Rmcl=1n@PE+hDFDr0Jai{x$j2vy067X<I{YJOI^(#&;0v88$6+>8g0
zGE&RAx5dwx|L~d2^}Nf^74>?0iYM!xSasy>gpWm^4|uRk*sf&Nx%cS$oChaGwmTgA
zyHvILipFZenul%C@~aXgmRg_EI_5nu_I<fo!Gs+*7P6fS(z3YwG{8MPrBiCTX7eQ1
zE^X-#^LDIS$oJ&!1DkK{v1dQ<6}wJ6_vZ9O{&1r`&CR0!nD(sJahS&1(JfxESvoIf
z#)1IVR>8v#VhZ|iE?h5|r$2F9{58flsq3cF&It!5*dP25{)=t<o`g9PlUNfcM_e<=
zODte+zAZjYwZ~gLX2*j~smYP8J5mm^a_fahOyhnRQZT=1_mtk$oh#NGM(Vv<;WMrK
zSIp$Aha77368<jhw+!T#+Gv^~VQA}e&xB{@*_Z;4nY->R==izxN1JeZ0jHnzf<yd|
zpD!#`wG%i}<jFJr$-2fPc`q3+9}G-caim+Dd1Hi7v+7Pmg^l4&f72)B^(r|t#_3Ew
z65zSyuS%I$(lV7WucRd^eXK{EJb9+L#PPT4OHA&oRy6bz`p=LwQ^m{XkZVVtSZCy<
z2-VKONi%+SyH1i&=`5JDV3Yc^m4_LednO+7ivRNch=6B}%A^~LosyFx6gvecN!;#c
zos@C1_RYG*XEtnVnEBdZgTQ0ujWfkhD<`aw2;$#ryU|wYvhqe7p~K3C_2R3QCw`qI
zA=`O&;tZM2qZ3zr==?Ty#SzC+-a5{OMeY|Wniolj+;q9;xWTB&OwsW_Z&cm^MTOP`
zRz__W-6Q-xt2mt$`A_u+bTyX>etG*ut?|LN)-#{N0$kp7E%?2F$$z8vq_Yz@SPJ!Y
zE!f7tT1i6m;4%(1*Q^uTcHtb~TnyBkOa*&76TFvP+m~c#V5XrtH<3-Jm9_lH2ZwKp
zlXP2C#1$)C6mPnDY!j?HU=zXe$UvjXl)tC)$~5+6HXEKaD)PUnNI1r<Ehq7D1uwr(
zMZ&>kwj)g;N&A>z%Syao@s`i0JYnbJZO4+fGjEoSczYnjJ+~}j6LYt$#2Xgr6pdq@
zN6+2hTEccCoqJj5R=%Fngw@Q!vbT9!6gUobuI2M7O<2k7EW0&K%y@2~LzX~`u(a%q
z7YCf!*{n94Z>&7Gz#=GWLa$>d%gqSSi6U%<LNT``zIaS{yNmhC&Cjk+^p;s~xN}ff
z`yQ9ksiy`9OWa@BtgGitvwnR|S<NPHb;1gXPjfbL?&|XG&tYL{p7%Pe?72<(FQ-@U
z>esq&_mI8j#y%(5Yya0s1L1JJ#SYhMlYiOI)yrkA+<L*R_U10_=)&2HzxH#A%e}nU
zsCz5tmfiX1zS9o>Dfm&$ecn`T#cv;<?VC5brM;6W+HPCJ;k(sne*T`5ei1D4>o14C
zdd3%$diVUj=^tVq3sk>w<~l#mQ>?QwisQr?k*F05TW>wqa9(!op=HPwridc~PJs<f
z|4v*oFj*<G%S2EA;MId&P8Pv!yVxS!Dh}NgU4N`9TrVhWZ&6*r(#^BCUtM(PwT44=
zMq`M27h_U(t_G(=kQJBADXq;WukW5vb2_>z``P)@7`g2KEq@&Ut(tjd;p-h9tohvY
zu4WyT$T6JQzjnf=E4Nf%&%bag>&1kgm#l#;@)HdCk9{!~tG8DBr=iW;Wwy}dg^T%|
zhmo%jXg@vRcq*N7+0R6WR51zWt(KgVckeac-5j9x-hS7WuMGzm9Nb*(>2KBZcJcc{
zxgSqcj_$m5Z{`|{@Yx#gUOcE<+$nwH`stj5yVQ>?51%hD`&`Yh`vk{^^UW(bZdP#=
zt4_&2_u>NYhOLHw@^7fHncP&Gx2IWcc17s6sMLqfDjS=14``Y@N;F<x_hK>EW7Elp
z>MOP0dv{H1N?y(_e|gd=?^SD9`=2Jq26^Wfz3I&;aS)l{vPM)$RjqG=#J-h>mWv75
zhM9{P?Fo4PL-VLh&bH}G19G<O>wcXe5mT#Sx^2l1L;aPTmh^2+khz)gJvnRTlnPCc
z%%3~9czAbj+2)bmeR@Sj=6vyjzQ@jgc5FHC{X6DbQ|`|lTV_lXyL>AtD$2*ZJ2TU8
zt?shqpF4UiboYAJYR;Jc?nx@sr%SWvTx;^~uFN#l*L}A2=%y_Z)5K0+N{Wou;bu5x
z^fdLQ@5=0hORZJQ(!F%r{31*XFWhK|$oH1q(00HrO#bG(1FyH)eGe#PTrNGWK`C~g
z>*5lj<2&YZyuG$J<3sY@Die+xHrBWMR(>`5ZhP?1$r(M)p6C6-ck?crwZhX@=SG^*
zIoYrTnf5iptKPm;p1tP0o$<*dsdpo0$jL2Tbe!qgC*ePGPv+^U*T3E_u;t`CQPI~*
zUal8wGjv=YFJODPaOcwHHy18GD;u!vzS;I#{@c&bF<yxa>znq?u62fl#J)O-X3uFW
zS7*$;YTWjbC%O4rpxL}TOJtn4$|V|#I5a67W9Zqr(K7qZ`9%%ctSNgnM5J4NN{g0G
zWx70hmexUIF@g4G4~^ADK?XcK*b+`~iY&VuSU+c_8oM%wG{chf$=4Sj$oqb>|IOwq
z$K`u|^JpY>84JcTGu`Q2Z0D_X`=!V(%fNHXMROJ|a_60VyqL9fzRhE6R_+{E`E#*-
z&Tr1xf8$TiZn-8qeX6R;VYM6wZ*?~dLFo+())nUzGaQ-}GIwTS$Yy3wy~cY#swZ|n
z{>2=e;-j)`vBjz<C(ne(`#lbrQ!=M@&RaVTgNe@3x9#58Ok|Bq;Yr?oPtAkdM)IEH
zwl&Kdk9|&Y7D?BA@Owci+b_jOfk7(n#k1QQ-OdK=v~-kNa=PmJ$<N(6Dy&MYllyu;
zXPBQ)eel{@=7P!Wo0c<^^)E)XJl?D+k*ss<hxJ}T&BJfb<#l~ovv1O-QpfjfBCm5K
z4bHE1XbJi}a}M8I*Y~p{Dz9I9UbX7h&cai(q(gZl%-s$@akz3*y6XBHmO1K)cVu78
z@SJV%O?XL;c;OK<Z^8B~KJK&rCP`Ne`pq9XI9^OXVgGfHm00OggTrBgX}iNC#peE~
zF<r85Uw_w)yhQ<5XPazooS`i7$DdE((P8n$X+pf|vJS0^Y+=)c7j5*La*t=$L959D
zecP^9y$D^F8{BsB-lP>JH%mGU&ik#}ogTHVI;ZGlW5ipN>FSp_C!fC7v|&cr0^gRS
zH!r?mIbX`ES!KA<X-=QivmNde-YEvhJv(1H=kQUk4QYmQT0P5uiz$fdX$D>1a-RQK
ztjYc4ma}Px7ugyFw$@5*%1_eXX_wS^!o@ONeN&El>BAqhnW`tw37lyBb(cU-#|5Lb
z=2n?=&pK_6))vjrIQr%^_l#WKf(1KT)gES~3iT-)%xX^Fe8BajvP$Ql8od=8zDQfA
zm;E@>_2g{Fvx#!rYwdjgJ^i#ujFs!`cJ+muw!M7YsC|F#cR{lateRb%1xJcE7bciq
zSSrTFA)fTRU}2}8(vz8ogl6gN-4JVbGQq#(qx`wQDzo+q^B>>U^+d<l%zO7^uQEj;
zpJU%%ZP?uG&HcspQ)AMnr9}(GmW24+JMSp_yyK|esYSkDf9lAcoI2yu6PNl*ou5xP
zJ`wI$eYElW^N$gqwtCeW_p4U^c`fI?t3v7O`OtsXIgJ^*?=~LM&FJ4;_Nm6OX3g`R
ze8=OT2H(rwH#6<!j!RR*pWW^DteqOMpd~hcb)dO@U4Fc(g=M?`{ykIwd<y%tyHdNc
zN_tKA*QKHVPbZ#C{{NBB`<c|gr|VO;eY*S4>y9_4?U~ra+pBaoF1K6|f6vS2v3|bl
zw=buoPoDWP_ny}q&D9RyAFWo=KF57kCg{bhm3)2YSR6M!?Jj>IcTL%^+I8-X{-<^w
z{i(BO)cw7EPOyK$H9hU0GPl_$_+}qb4s`f4lS$z3ImyuCP!rqwEbG(vjhh{we<_l3
zV@v+?@%t?Wd9&2>Z<g21VyvI#xI|w`Z2D`?8ReZPul`^#*4enRVd+%`9)qP)-r{Bw
zSua(#Np8~rBf_k_%RYbps|^VY_gA{ly}h?=YP@X4{{8X^6U?i!+<!~+d@n!qAm-O;
zt1C9XZ})w&j5g`(-}Z68<MNj~ZhzmG_<()Q7T3(2yq)$>W&SoCy!QFm!C#`gQkJC}
zDD9T0uMf}vu{(Q)WZX~2A5p^P&sN>P=UFWGp}9@+FaHYL>@$xhHC?@X;B;F|R)L}Z
z-mmTs8!Dyy-xV`#pEP;-wA<1<R?Wy`d!p08`~Q_MLmW%X!}8B{4>`pi2j95Qoz4H;
zc5zMLWWH+4MERC>@mtDserHYL3zt2#GPz&wX^X{No-IpnE)Toq&hq}-rKsC)H|*z^
zY>~+=PrFktcIWn7&hK+szduvE^U2`s4t?3zZ26X!yO(a;Bh>dSY3<$z=S#M2pL;az
z*sD3ZJJK)fY0chxacTX(mpQLxefO?^{UGUgZSZ3)C*yPrf%U~L*R&6R5dYKEeby|d
zXMeuLzJuGOg*L7h{C1VCx;nSzubGpRz_zv5Zk4>=_>k*M!j#@GA0vzVc$Vs!&Q09q
zch~#!JKbAl_QtQ<EX|jsM4LR{H>)sy?T+mk$7FqXEN1;~wI{mxo#i~U^UPPiAOCuu
zHNo#(zDlA;WB1;PEq9siB0A<TP0|y*Jz-AJ#00~AUWfCZ&DLMHGH=tKIqz?u``lV{
zr|<nu`FD@@`|X{1&g8>3zT%Hh%s$>WUR=@fe$iE{)vxDoUwbWa%f7w&Ti+G`+V=I=
zvR}dZ(d+Y9UHsDd^_QB^3idVE{~MhCs{fcj$L)R2-S<9sw_8Vlw+{aP_T06<=dRT5
z-dwuhxb*%zWGVTIoXq<y)rPHcMLx5CtCsz|rgO~Q*!ZEfbWNx2{~3kOZ`Lo5m@55D
zzcNg@Ht^FA#sv#AtNz})Qs2GpUwH8U)idp#I_>V6e{Q#X^SIWy-i5#RLTYHN@4xM<
z&+bp3Ce{4%f7#Z*^Jm`w$Y@|3U&ClKS7`gGdDB;(KeMUScwa<@{<7`W8?N2E8zr6f
zG;Ol=+O4j;uk^GB8_2a~u9uW5d)#no-OZTmSN1z>4dj+}>srr$yF1@~?#wguejPo$
zY?4^jo{Af*ZkAvF_a$-nzO1KzHpIsJu702Sdhglg;{1x2w-l92fBd=S?p_nqM(`}|
zWWOy7?l&3!J-z&F%aSh@f4A)Ptgf(%s7wegvRziEd!A*(@p;)5c^e(R^P8ru(zW>>
zu%NPdL(=x=BJ+xM0uHq++q7ScdK1JQD|IAx{?V{hyLPc7Q%>6%v_BV`*B_x^CL8rY
zY~JAouD5U4Ex8cIwRWip_mdBbCCai7cOGM6yyx#IlfLWdvc2mU@z~8zp8iQLYuEfW
z7U8!y^S@Sc+wjcbg$&!=!wWi{xovs2v^Rdcuve~dWqaP23%zn`Tijz_F&b6RIAZJb
zNNyJkQ@;G6if;z<Z279zKXg7Y-*$^nwT8pOmoi+>11#L1XfM34Dx#OT%zMsDA@Mqf
zk4`d9jZ=eI1mqcyG&apy;dWrPRMhIiW!W<>N=2<+xV7pClYd5Fcf_9*TO0THHx3E^
z`rPv;XX3MtsrPF7UdUcIX_Z{sz$fGBCvn(o50g^S5;ehSaie1YD5b&*FP6A~-W|)B
ztxl|w&Q4l0*>}s{<4$JntP#!^r*_S};cAwd|M2GB<q5pL<|-m~1r>e|Eq4ViykH{x
z&~l59t{8{!XSsRKr_U^8XkTX8{``W`amV6}S%()`Zkh9FfvQjUPC3r=hj%rfU2taY
z<ReujbrP*-WwI^&4T^Ui?rlA4d34JvskV-+E4pc_Y4;4?C@!mgdr&QP!pTi~wy$4(
zTeDv53VV0ovZq`!=Vn}daZqiYKwzt-`NR$JIcEMs>W5b@k`(8gWSV=SY~HtRO3ky(
zDh~fVWO8k#9b4m!kMEK`yp=h;g-zI8<(bNo^Ib>E@@6IWmL@#ws5a1*%oER$Dl<3l
zx>M;VmcIS5;Cs!>o*wC<T!x24XPzxCWfsZIm5TG8F<C0j<lN!)oxcwq>HX+az2i|^
zYIAnk>8%MJ>WUk4jtaK#GmBre*!yYPqa|Ao#x=RjoIdyW)_rTc?=Cc)aCVpC=VW8u
zfLRuwZBLZ5Og{a<jmfY0Ox^0u&88Psbd$86H=k?Vk|I3sQ0OGyS##nz*;6JsZqfUE
zspQ9k<9|}LCsfU7x2>>#rS_1OK`l4;T}<YiY`#az?<%;r|7Oz`v)uSrHtfyO#CPSX
zrw<7lZpx^CQ26oY!=&o6T@__JZp@pjH&0e?Tl(_u^!DzzH(x%2NOiPxA8h#WUw`xA
z__X%<Gh~;sJ2m_lNo)Ud)T_>T^WjaiWaF5ZzW(}W?#+dgOkGcE-o{s`vK{$lqb;ev
ze(Mpjdnc9T>*cjseS4~xtowN^Wzn>aN?Q-{GtJ+^S@-9jSckmr`Hx<Ir!zj%XFd}D
z@KSxG{Hx3Qubr!_PX7wX-^w@jg5t;1|1Qkimr#Eq>*l96)<rYtb@C=14&83y;<;(|
zzfDemuBWzYtm}O8Gy1K-?OxfW+LC!`Esx4B@bI%=TD>}e_uJXuf3Er5-FSdGVS}qo
z@P=9jUtht!;*~iE<R@I_J}Y=HT9`FTbJ~;#ho{YW?Yp9BoBgwUVdrne{eEK~*O&aQ
zKz>!-H8#O!kHUQhjXg_+pBNbP?r)f<x8u`C@pdbY%_X4+mo1$wAmHDv@altjX?W(s
zrJkxq(~e!8uza%Djpy7yBEFOazMQtS-?LzX-0?$lyCNPoUbq~~alyx2*?HPD1%ZX#
zyhfb<hi`9NzIlS0zqrNEFXA8M-o4~{`Pj!?v@CMh;pI<jE|!ZPjgs28$#(5`7UQ`w
zK{+e5C!L(BH{sLSFO2GsjAIK8pX-+-o-1k32)N9|ze>?}$BNJ5T0Qe$svNm+`9uA)
zmrdQ)^^EEb^VT`3*xpsxIrk-_3tz0t1YdvUuIA;wKDjShcv<TwhEB`MbP@ahWuZ}$
zclg(1cJU@(L(?)n+|y?{7TNk<NcbSWqx_3W%BgwhCY(N35@<f-^U;qD3b!tWc5Gfg
zdqRePd9mdReW4Rumvg#QFALpaeVNnz?LVeIP21R(Db-T%{@KKazO#upIboW%>$RAY
zW?;wXD+~!9bpqEWcsXw~4E`DDw5nrCo?5><@5)~pidW@qm$3ci{wwLlw$RyiOPTFS
zelA<B=5MWbXJ_;&m3)2IX>vQbIj#D8OHy^#j<=tu?9iU2De)kBQ{UHznt{UeR(?Df
zY_xLT8UAnYdQQu}KfB}ImwP<7&fJ^Je`?MiYxxq(%KTY5hm2WUAI*H?{^?!Kg~w0!
z{#<bEWVLZd;iq>s0gs;C{kh=q$?DIJt((86U3l>3?n|X@Gc+%T&eN<*d4G1n-kZBW
zJ63Kkf8P8q?fqHyt7q@cb}ZgpzPy_uXPHY{+RDNJJ()=yzn*5+O_7=OGSJK8nIgl-
z6v5t1e|cxAM<;)2*s9zX=&9J`EwXW@+Qdl}L08t=YSzuRo$0dr(&fmQz2{OYWfsQE
zYt*;Y@xH2R>8n+!<&{-lutBwwoiAD_YR0@a_ct?rnvNOIWbIGi<T1zC|NP}MPflG|
z{$=APz5maf4m;y>t^c2ZsI+-97oS(&emeiV`<I!8W}lyQ*!5rBJX0OcGd`E*WwYqI
z!Atg%INP)RVrqYxuSd=J!XJ~Jc!_;sPU0o|g0-!l{|{`1$vEmn&-l{yL&8|oYwNt0
z|8HmdTu#)y6xpY29C_sVCm*>F1(_!cLj|qAG4{(@g|!x)NpUvmo!Rni@AAlw_T|EN
zww)^xd-d=o(`VQEMrlF+@P!KV@|+xPbfuGCJlNrV$C=++<g@KrCs&)f3;w;<-y|U9
zFTPM^o?XkNil}#;PU6<<9@+3_8EwBSD`e|0tT*cq>*1wuebkS9nDJ6a<mhGAw8{S-
zHB5dyb6v+GgUh}vx|WBQ*X>i7AmM-b*n!JE-JfRq74*E(-!wtaU;N15884ajI@5O@
zQac!LqUbsAUB}0SOt#WbTaIae5!YFA>N01cLz<Pc&<P)Yg|^u6E&C4b)9#8-d@TBO
z{f4hx`re7(Q@JILE*o@wFyWs1`3pnRjaFA*gPg;QCdzC}43y>hz2U<qk+P(Vetgjz
zUx-XQk=dkG73E~)RyJAj=$y<IoK;q<r%hPOF3^91<#E`Z%b_opET4VB(;sA+WYm_0
z-HSY>t0D@L-@Rl!A{F_l^(61?FO2zt#+o_n%1b}&Oq?vH|E5thPfPM*iupl~OQqpE
zJJdXnD$Pon+Ld86Gr;9?WUA6E9W9}YPZkOZ`i2R2Wb9gL<1)GG&-Iksp;C)HjS@SS
zg-V~9<msw7>x>rX#S}xwj%EB)n=>}8l5m;qyHsJ;8#NZkU8{YrO<JNP;JeAo!DOZn
z!=jgKKJojCiPmR)3V7MBwwlx4<nA)5St?&w<R7WNANnOR)97OR-hicRX8LLU+E5a9
zv3+TGr`?;Q3m5Twhl|!m>{|Ka@TI#h^1qtOL~A2vg?vfO1gV+aY4^tU+TwjnyXC&>
z)~<MQ_|jDu`EQF}2Hw9^cICRs`(S5&u)zkVM&9bSucED1zc_s9sEho!N8<f|qCcIO
zqx4*Bvv_6r)q8U?Sz14ogl(L(wEI`qe0_nELzllEv@E*z7DUL3XMA>TeR2Qt1cpkf
zsg+hU74FA9VQ=3XsZrl4^KD_0s&_cc<xk?iPrarcyJmdZ_iAcd)Q3ZQLIO!Qf;~mW
zC!SIY+Y}-h_sFKQBj#k#*TR;N>MIkZ)Oq~B{k`a%Rd7=Aq=0s$QD?A>fBK23I+jj;
zEl(1>or>NBEDRT7GE%XA;#sv=Y2q5EASJ2xFAKIQ&Qu8Al+qS(a#B;eqOF>tiuMz)
zX*mKqkwp(8UPyF0b*po3QuY3%+*-#gG~4O8-GRwY$5%M)OF7BRq~_h%J^#YSg+Ilu
z%+^hF+F#K=;p8Nhd$l%a`(>=YG0v0u^_6k1%rDauS$wj|A1}W<$rU+SRl3D@+M7j*
zrk6cMGHpC3m`6@hEZFpf{eaht8|)QO;yQbY*Nf{4$5cz+Fxx<gFWT<~EFvcz(Vw{d
zXJJcUZNqbpzrPyJ3I0+ox^rP8{|A|c$?^{@o%Y++^<U8c*dnhl?AmyIGE3uRRY@jw
zZ?(d#_b*pIT=ML}9{(%OM=bY;HazwFl-=rRJ}WtXvV?tGb&>k<N~QTHZvVF1xc(&T
z62<u%k-r4jg!-#?{x_a1`Kqt(&#wh6#-%-CX{!!RDPMLwfia!y)2)IRxAtZ<)qe8X
zbZ@%UrI)oUAI5xtw@mTu$A7#TZ|}W0p0Dn2Dk$_@)7xi??p)XJnaf->3ztPm?3*cL
z@X5`7!&=LLw%lj$_Wqx7;oakZ4qG2G7OTy^CnUX$cbj1O!5zjo(z_mXU7B=Z>edh;
zxg)P;dXznHdT~zn-CFJl-b{6ypc7VQ$MkniZQacFLrP#y-Nfy78#W}nA5&W<=_>Wn
z{ntrRKBk=W$!v={>g@wof7aJ(KJYS<t>S%g?Be790{HFB>k8|;F7A2C^`%twQCz;w
z*{>DRCm)@BZ7$KZ`%&zsE8DeKhO7KflohqzCbo6S`gI2%-4F4YZTU-E^tJqr%Rip)
zT6f(1`o~PR@aEdc_Mbu`TLX4k#Dv(_zdGQ*Y1h9${cpF(-`b!5z}}3xB<}x<xIb-@
zF?T-3$?ARmantRSblQTWxv%_hpOrVfVEz4peXRL@+veBp|F7KrQ`B~~^@acKv)*hO
z*Jn?@xLLWdl$Y!Ip78g#q`HL{stB?Qv)x;iv3KR&9lL5;>fI}fTVA{Eh&W>CTC-=?
za@Uo=J3n51yP>V#WJiXuz4USCyQ+t57wyoJi9aGZTVr~s`&QpRlMfCS`wkp^@I%Rt
z<!I&t3(h-7bRV2Ii2ayf$oivB^pVVkdmZ%-<$u-R{jFQSaSzv-4_npV$*lU{;Kw)T
zkKsbw7T>x%JDra!wfO!~mf7^gblT$+TU~k;er%Fe5V2V2m$P2(-1-Ug_0G?)vblff
z>Cf!^=jZp%IX$cF`Lj!(P9EM_ZC?KT+3n!>&(x1UyM1!=&*1iFv#X88SADx8YkAIO
zwf`sU=})4o+r&OMeC9n}Sp`xZzxrv`;qSS&FVl}yM!N1V4fvO-9P}mk+ID}QSL+u|
zk7+M@W>&Yk`Hp(&o~I#__nsvxb~Zo!cILE=j-n@L=kLDz>&`u1zwc4v{f^+LVt)@E
zo@cnLxqsb*!Y_9c)>qWW%sKqI`~LPulZ6KPUGXVD_Zg_yy|>x?gZ0aFtLoW`RlD|E
z?YbXV4W<??ua0qwZjd^XIQbB(n%@n*16M_j&5r)v^ttk#&%6+W-`k8UXISUGx1QW?
z`uN*C<DcIXYrk6L&skaX*1*9ferjZEM9eR(DbHU?$)$D8F5I#q#%6a@_;s_6nR?xY
z#@ouL*Dw8W)V`{uf1mB=emzkZ`Ij41@7()7kuUh5?k|u3h8Df2_HQvhAocOKq~Z3&
zJJ+t?FL!$uZTNM&)05xk68k(#<~gfeeja&e<KA2L``_QFYj%A0yzi}f;kADiPjA#U
zN54BSdpqA^mFL$*3qGFN_*YGR=j7ySnKxnPUvCH7e^n<;J&K7di8^BYM~&U7kW>HN
zgZD89yVZaA#vJVa{k`CewCkPA)_zS@U%y{+uP(W`Z|~#9w!1R!-}`t`E_O@)`yUzo
zdbhT(j{mkg`rC`X+GhK2tF6DSmM*JEyuV2L+v?w?EZdhinKn3bKbx?CkMmu}q`sU>
zDzi-vz7pTI?^R3K6JrC*wzY-_i^9z^n<nb{6w3>JY<1~yHrUsCQkN(9;Tb8jts9oK
z-nVSJ9&vTD&}ya|LJr#*)0hp!8sy!YoFis1JM86*@-4{Y5>?C4>y3@@;_m9K6529p
zK@ZFF!vUw-wd`ze3(72Zx*3;0-*uzhKmUc!{AVjIcvOB^wz{#IJX%=8SibDS)`msD
zZ-gz~GD%L8=MnQO=1<JO7PD}Ocueq_qOc&JPq%o5mBT`rtS1*9DBs-3(e)^RzkWvN
zVv%zJ-iH^ko^;`M)Yz7F*pZdhitU1K!z43}ZLb6l{E^-Ewz;T3{in3}2j{uD8XI}9
zn(kaCFaEGLWzrm#b9da*cCM43m9{8e&8%yYml{{fA~UvB!$3jdqXrh05>^R|eyaY8
zTlAFmuHMBLLbiGr%LKX|0<Q>t)d@V+q%}<=q4U;Mjp$Cr`7c5{&8EHxa4h!vved=f
zt7P$!e|}dMxcu}CnYZAD$}8VRS5#K{BpIqlc`sV6BIR|#J7_6u+jL!t9i5XVt`O^V
z61>vkQt1)W<`U@f<|5nCwwE&{=5QW;=D0EP%sjTEZLO1Un0A_8cAmn|epIYHPbbXl
z(KZL3Nk@1*d-`2GqczOBD<`f<?EbWMMNIeA2^rztex@2`hov~8E?=11)@IUEvw(+J
zG$SCV?IYimISX&`w5EmLQ&`aapgADtM|tuK6=z||U&jp=Hm))h>2cgBxy;a@L$K@Y
zhT6uX+;3u9l8iOX4o_nJrtP#*a+#Jvz)z*!s%|>-QzzDOoa^M|<_T~4W}Fj}F!R%d
zfE1l0^AoF?x`h}1)!8z?_1oqP{{j~zSe{g1VL$5NlCl3+M+1|=8@nsz1|c;IZ*W)h
zz4+i%EdFbk!DWdrabDbBU2=MB^cECxyGF0LerSQyTa5({+*>0hygA;6w_j~qlqAA<
z)BZ3!la74C2L@~Q9cB%I%sG+;&ysfNG#q4lBa$$W!JlP=YXc|KxtLcWqEgEPOj<)3
zw|Fh|;gSt1E#&SEyuo%riJ^>P!{=~wl|?W5jl~bw$*=ko@Sy#c-I-4d7xM4@<M?)|
z$%&Z@beiXJp6bf76?wJLZGGH|L|OjSyof{2`O>#;UAQRrOaEWz5|b+vjaIA>5pxlK
znzitMfMd#?2?kS^E!5)OU7g|TEFLAx#<y}u#f@aq{^iFFruz3k6ng*T$NoN^w~y!B
zS^WAqw`IHh&&(9LS6h<#J&G1jI;LhI)8A9?HgTa4x2)DQ*)$Q!t^<cH-~UJ#c^KH@
zF8J;m@5>0Or)~BpHwt>FR2csEb3USDRp%MwTbCm9^pl2hOq$)1ANHpM+$P#|@yJhK
zbVE%}>hcYaOAsWWzkDOV%EwKbkDi*CHJO+-shrb1?lwWC;O`A~7je_h3d`PJ1vujU
zSS0n^LF%`I(8nSvX-6q($EptPgH~N3jh{7wQVkAU#F?-icbnj{FepUIb+_<2uGu_k
z8YO+b3Y}?-GBUf=mb_5vidvQt(dC&bCG9A*^wdT{m5U!*s?_$BIYo<zEhy-knW7x9
z*Jp*BldlL@Pe4q_CXOWxVGOzr;!P4PM+DB?;Mn14c3DNjiD%6uE*4LZ2`S2s!a_ER
zjv_)nijJZ}Hi`#@y<Y5Jz{R<4f&w?^wh0TkIrmL?z@3_~e*q8Yxd1jV?}s^giq~Zw
z1UUUoQxpzr32nKx;zp@3XW7IB;+%DX2fHS&<7`_Sba2xZtq%95%51ZP&O$CK2VI1G
zV#=(oK0S`x_Sx0*X(3x{%EHGLL4Pl&z1VPZ;u?OdXD3#?)6c3dW?z3fI^*eXy~n}7
z*0tFvOzV4@X|raz<AzH;jHyLoVoZtWA5Dt7TYWwt;*o1q?WFSye2jg9=P}N=ndH)c
zaQzjB&NKDP<_WHKv)m+K@J3f})knMYFZ|^yUWS&evDa$3X6T$@+rP9_*y**}WnYf9
zvn`^H*`}ZE+u8f4&mmf0DbKcZb7}96Qu%ako7NAJZ36R;-K>(GaQbo1UcM7Y7an{3
z>C(i|K1;=n**mo6e+*c0ere+eTdA{a*+azKpFdl??<{|T|GIO(RabayGFx`O(lWA=
z*-(U=Y5EPOFX<AiZ{~{o+)UGYtH`SU^WVhGlc&5*w}^;DYn`vsi98*Y85<q5Yx3(!
zr7wNGybtavY5zQ_G}G|;*45t6FBu)%qd)c2t*c9mD_6FKuUM!1jMZXK$vPd`^Kvx>
z&8~YFHcz(sF*|$<m)<jHjs^)qPo=90f1C<7?00G5;`{k9OYWv~L6hpE5RMRmy;r*V
zYk3U?odjCS-c|27s;Bpn%c(v&XO+M%g%)RyM2;kzZMnU{-&AH@n&5Zx>w`O)I>9w*
zd$m$}+Y%-Qw`~aUHfKLH!6vLH>xAIzp3R3o?{(O%w5~;fY0tWsi5ve5++XwF?z69R
zP|A7zL%Yn5zfSCO*$}C_qxHN_l6<6v0B2pvpIK}4eWrEH_-fmFBTTxrf-m{+i!&#$
zI<Q|Av-5l{e<|f{PsmLjjn4B^dp2Z#HFP=lm&q+;f>8D1yU*8cKR%)2?2Ac^2e*|Q
z=qTN|CL8*&^1#U$m8FlKnfNz(aBp4G%5ur+%gYE2P1jqR;XFIO`#lIw)@l`55&I)z
z+Let<UPW!tm9O^U4QLn9Wqo~V?yPB7F3pa**5qBD1zIza70<P8sdX-+Uv$;wum@Yv
znekPZ!ykNEYF*3dhsqRxa3JE&q#}Repgl`2MH%SJpPpnjEhLtE+tcJXeG?{4h-y$4
zcX*f_737`2Vh!u`cL5;FGFbWa?>=O;*z=_={K1DyvT+TEz4I%ef|)OUqZ%hqd$)wu
z0)sjI-4j-e%9p-zjg!%tsV{va8-u64o5E_L`pxacLzUG|e7dSv3e>WEwk!?cU%Bfz
z%Tbo&PEKskm*41=`W?`}G~kU(@7xs|`E|b~W^WYMEn9TdZVRgbi1yoZqkS90qN9Fa
zZgk51z98-ZqOC6`%l~5Pkh8v+tX}n|`Gs|bW?&76e5L17)v4E;Qh!$HtlYKaR#d@D
zzPH|4FL`wX1g~~|Fw|eU)Vu`j@#r9g9}yle)|o2zTB3jHwN%whn}l9Q9N(ei78Rzf
zyL#d)t7EB-mo_P7MotX5Zm8EiX<De~=~YX1DIM5kkQq61%Nn07?Sg4XJx{+{l3E%R
z-MCpNASz5lS9{7d)h>>!4<9qlNu3%L-?&*PBq~fxS9|I&t~se2KW&Prj9k5EO{mK{
zKKaj+w7g$Ma9?|x407AFtz8;>9x8(zd^*b2<lcvm5kcPh)3$c4*z>P2a?PGKq0dAu
zPA5GD84@-V5pS2c=4>fZ*FE6rosYrPKXB4J{}b1oJtgYs%u`%*HkB;bIiTsCf9h0s
z%GG$el?(MAbSkT5d35-7*S}H>{61qr#EPOnQ<9XvEu03AEz@s_689=s9R<bFv}sOg
z5%=Xr`#y$6$FW2l#}1DzOJ00$5z$|{MeNM!<C~WBsfJ!}%Ka&^Te-_gaVvwXN&BJW
zTb4|_7ZAy<cX#2gCDZO*;F5{kb(p)L;wA6;0}q#4?_p@4eoqD*UCEM<HwnCqI9{=4
zt6p-9n6m6^iRoWGPB$${bqtC<zGIDNl$fe+_oP={45xQ3Nlm=8$zbK$1CiI%bh{@{
zJL;LsdTx_lX5@k`Isu?aH1#^Iv@~@hD7MxFM~P|aa)V=Q?Il=j?FivTS>u%Jse9ld
zC?cS-X%WCHe|l0>)#f#!9-4=bvs#>vN{wsWtP>U|rmdenwUYJRriiD@be2rJ_W@6A
z>Ee#9Pe`!^Vyd%Py#KvNsllYCZSwhG6TO&uU1y%MT+k`2=G!jlqxV8wZ|3^*x#6o#
z!`giEd0Hl3<>u^6>vfm8{gsVDW=X5<cm9Ok7jx%aGZES?vun0neEPhZ4`c$?Pd#S1
zbyk$e_t}#lO;-}?J-xpF)}DzsY7><A->dji>ojp{R_6cDJ`KTFH~K^>9}fPy?&nOW
zzh4dWZJvBs^;O{b@2z(mgZ4bVmcDe-{&Ui+lGTnKc=lA_?9BCN4VB`g9_DO*&GP)F
z&Jy)9WfuO`lj^@$Ys<XMm7D(B&i};2$mbz^4jI{T!VW8~pUn%mGFe~SqOI7<7f~AQ
z|G;3%qG??`j#C!iez%&jh`;*W=bh#I?Xu!}W%6?N3iwZ%=y>+vyU%+aVgxsR-t%lq
zqxs=^j89)x?$V4+Ddk;sJ-+nW_uZD3)6cXPoMCdlaADd%iLlVGhDp3%42p$i7qHgE
z1zPxIKP=ea=e(@9Vou0ZfsT&TZ906<-#V?caFm=Q@n>>JMQHb_RhOSC9n#pn|9zdf
z<`=K*_jRUSE55k>oTMKwxhIH4P*bwSMI<diCsH@N)b>_YqB}?7((}cvw^VxXm@b&q
zyX}(9v@;i%>=lXnRPFEaR_C(70*z0d4>rzpn>~4g?~$c#7arz`Cxy(N?s&P!&S76>
z#+xInC$Iloxzs3(;rA}PYo^)ZH<czC{8Cf#dL3y0eZ^%%@rgpS&qhSF-kr{KNL@<p
zU2(OOPg{!a7Y3zxhoW8iZjPpTN=Bl~1>f90Rd|ipP;1`vKU?p1zEt>fcFR&Gw$NFB
zMY2-N&+qo%9yec7i%sjao#EEsvX(!DZ_lpX>S&bpRfgYmUf9;7QXzYCc=;2jYDRM$
z+@Kn{H)!GPRZlni>r8l~n<wc$L-nwl<H8-HuX;-FuX^(7e%YQ2Z}wTcdDymHdX#uO
zq<)TEW(hlMK(RxQv!l^oqoP@^s%s=)CC;je=gHtacAH_BwL|JOo*lavr<T`pe-Jp9
zZnB(B<OjdU)HMzvvsX0suRP~)@>vY;qCWqibneNyC+5!4%x>I0`Mcxkq)RXCID78+
zzGPv!D)R6)kKN6mmZ4pNQnPjkn>9Kw&}=IRkW|dCFcLYZ-4SqSh2D{xWtU2Qg!F^n
z8ZaNb+VLj7e6np$%w55dC9~Qtt~~Ej+B^N#DbF2QfiJjua``9UJ<efP*?7v4)8j{A
z>a}n0y!W}SReT<-*l2s`pV`*?HM3i+mR__-G(UW;n#HVSkv8k5*)@e6&elIIo~vy5
zc}_E9-$iwIF@x&gGR?nS4rmtN+?V5eWx=c~Oz90@bA9d!MrrR8eq(2r^X*#G0>0|~
z%ayJ?tkAuXvCefV%fjD*i)TtL?wZ*2DLdfXH0jen&+uQLvs8%dN3nMQ3ijP5LZUgZ
z|4seB{r|SV+v_*~J#PR1>i)mw^<Ve@`+5Iw{{J$Y|N8$Q-v9qH|KG#+|6k7k|0Vwa
zL;L@4=l@%74^q7Q|7-g{@BbhFf3p79^#7mw|9?II|NH&FpZEV?{r^Y)|6lX}|2i*!
z^WW?He;?cbeO~|L_y33We=h(3zP$d=?)rbz>;L?=|NDIZ&*}F6FWdh)ZU67={l7o+
xugBE?TmJv~`v0fvf9~Ed`~Ls)|6l(9`+WcJ<^4Y|*WdnEUvfx=Nqc@i0{~{Q7hV7W

literal 0
HcmV?d00001

diff --git a/third_party/icu/udata.patch b/third_party/icu/udata.patch
new file mode 100644
index 0000000000..d6d59100e4
--- /dev/null
+++ b/third_party/icu/udata.patch
@@ -0,0 +1,53 @@
+--- /icu4c/source/common/udata.cpp.old	2018-06-19 22:34:56.000000000 -0700
++++ /icu4c/source/common/udata.cpp	2018-10-19 14:26:09.778950855 -0700
+@@ -18,15 +18,15 @@
+ 
+ #include "unicode/utypes.h"  /* U_PLATFORM etc. */
+ 
+-#ifdef __GNUC__
+-/* if gcc
+-#define ATTRIBUTE_WEAK __attribute__ ((weak))
+-might have to #include some other header
+-*/
++#if defined(__GNUC__) || defined(__SUNPRO_CC)
++#  define ATTRIBUTE_WEAK __attribute__ ((weak))
++#else
++#  define ATTRIBUTE_WEAK
+ #endif
+ 
+ #include "unicode/putil.h"
+ #include "unicode/udata.h"
++#include "unicode/umachine.h"
+ #include "unicode/uversion.h"
+ #include "charstr.h"
+ #include "cmemory.h"
+@@ -641,10 +641,11 @@
+  * partial-data-library access functions where each returns a pointer
+  * to its data package, if it is linked in.
+  */
+-/*
+-extern const void *uprv_getICUData_collation(void) ATTRIBUTE_WEAK;
+-extern const void *uprv_getICUData_conversion(void) ATTRIBUTE_WEAK;
+-*/
++
++//extern "C" const void *uprv_getICUData_collation(void);
++U_CDECL_BEGIN
++const void *uprv_getICUData_conversion(void) ATTRIBUTE_WEAK;
++U_CDECL_END
+ 
+ /*----------------------------------------------------------------------*
+  *                                                                      *
+@@ -702,10 +703,11 @@
+         if (uprv_getICUData_collation) {
+             setCommonICUDataPointer(uprv_getICUData_collation(), FALSE, pErrorCode);
+         }
++        */
+         if (uprv_getICUData_conversion) {
+-            setCommonICUDataPointer(uprv_getICUData_conversion(), FALSE, pErrorCode);
++          setCommonICUDataPointer(uprv_getICUData_conversion(), FALSE, pErrorCode);
+         }
+-        */
++
+ #if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP Platform does not support dll icu data at this time
+         setCommonICUDataPointer(&U_ICUDATA_ENTRY_POINT, FALSE, pErrorCode);
+         {
diff --git a/third_party/icu/workspace.bzl b/third_party/icu/workspace.bzl
index a4f653e026..f100836b41 100644
--- a/third_party/icu/workspace.bzl
+++ b/third_party/icu/workspace.bzl
@@ -2,6 +2,11 @@
 
 load("//third_party:repo.bzl", "third_party_http_archive")
 
+# Sanitize a dependency so that it works correctly from code that includes
+# TensorFlow as a submodule.
+def clean_dep(dep):
+    return str(Label(dep))
+
 def repo():
     third_party_http_archive(
         name = "icu",
@@ -13,4 +18,5 @@ def repo():
         ],
         build_file = "//third_party/icu:BUILD.bazel",
         system_build_file = "//third_party/icu:BUILD.system",
+        patch_file = clean_dep("//third_party/icu:udata.patch"),
     )
-- 
GitLab


From c10a3681aae5468d66888c6699f100cfef01bcf5 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 14 Nov 2018 11:38:32 -0800
Subject: [PATCH 0228/1554] [BufferAssignment] Hoist the calculation for
 eligibility of buffer set merging.

This avoids calling corresponding logic O(n^2) times, where n is number of colocated buffer sets, which can be expensive when there are many of colocated buffer sets.

PiperOrigin-RevId: 221481560
---
 .../compiler/xla/service/buffer_assignment.cc | 53 +++++++++++--------
 1 file changed, 30 insertions(+), 23 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index 40c012a5e4..4b3ab56dea 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -1434,33 +1434,40 @@ BufferAssigner::MergeColocatedBufferSets(
            computation == module->entry_computation();
   };
 
+  std::vector<bool> set_can_be_merged(colocated_buffer_sets.size(), true);
+
+  // Do not merge if one of the sets includes live outs, entry parameters or
+  // constants.
+  //
+  // Buffer liveness does not report the correct live range for entry
+  // parameter and live out buffers so we have to special case them here.  On
+  // backends that support constant buffer allocations, constant buffers are
+  // assigned globals in readonly storage so we can't merge colocated buffer
+  // sets containing constants with colocated buffer sets containing writing
+  // instructions or other constants.
+  //
+  // Moreover (on the CPU/GPU backends) the entry parameter buffers belong to
+  // the caller of the executable so we can't write to entry parameters
+  // either, and the argument for not merging constants also applies to entry
+  // parameters.
+  for (int64 i = 0; i < colocated_buffer_sets.size(); ++i) {
+    for (auto& buffer : colocated_buffer_sets[i]) {
+      if (buffer_liveness.MaybeLiveOut(*buffer) ||
+          is_entry_parameter(*buffer) ||
+          buffer->instruction()->opcode() == HloOpcode::kConstant) {
+        set_can_be_merged[i] = false;
+        break;
+      }
+    }
+  }
+
   // Returns true if the two colocated buffer sets (specified by their indices
   // into the colocated_buffer_sets) can be merged into a single set.
   auto cannot_merge_buffer_sets = [&colocated_buffer_sets, &buffer_liveness,
                                    &buffer_size,
-                                   &is_entry_parameter](int64 i, int64 j) {
-    // Do not merge if one of the sets includes live outs, entry parameters or
-    // constants.
-    //
-    // Buffer liveness does not report the correct live range for entry
-    // parameter and live out buffers so we have to special case them here.  On
-    // backends that support constant buffer allocations, constant buffers are
-    // assigned globals in readonly storage so we can't merge colocated buffer
-    // sets containing constants with colocated buffer sets containing writing
-    // instructions or other constants.
-    //
-    // Moreover (on the CPU/GPU backends) the entry parameter buffers belong to
-    // the caller of the executable so we can't write to entry parameters
-    // either, and the argument for not merging constants also applies to entry
-    // parameters.
-    for (int64 key : {i, j}) {
-      for (auto& buffer : colocated_buffer_sets[key]) {
-        if (buffer_liveness.MaybeLiveOut(*buffer) ||
-            is_entry_parameter(*buffer) ||
-            buffer->instruction()->opcode() == HloOpcode::kConstant) {
-          return true;
-        }
-      }
+                                   &set_can_be_merged](int64 i, int64 j) {
+    if (!set_can_be_merged[i] || !set_can_be_merged[j]) {
+      return true;
     }
 
     // Colocated sets satisfy the invariant that all buffers within a set have
-- 
GitLab


From 461ea88d9a6f2bb848da8cf9064fa939b89ff32b Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Wed, 14 Nov 2018 11:52:28 -0800
Subject: [PATCH 0229/1554] [XLA:GPU] Fix -Wreorder warning

tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h:120:11: error: field 'block_epilogue_generator_' will be initialized after field 'block_prologue_generator_' [-Werror,-Wreorder]
          block_epilogue_generator_(std::move(block_epilogue_generator)),

PiperOrigin-RevId: 221483918
---
 tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index aaad385d1d..97a1e10455 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -117,8 +117,8 @@ class IrEmitterUnnested : public IrEmitter {
         BlockPrologueGenerator block_prologue_generator = {},
         BlockEpilogueGenerator block_epilogue_generator = {})
         : tile_element_generator_(std::move(tile_element_generator)),
-          block_epilogue_generator_(std::move(block_epilogue_generator)),
-          block_prologue_generator_(std::move(block_prologue_generator)) {}
+          block_prologue_generator_(std::move(block_prologue_generator)),
+          block_epilogue_generator_(std::move(block_epilogue_generator)) {}
 
     const TileElementGenerator& GetTileElementGenerator() const {
       return tile_element_generator_;
-- 
GitLab


From c1399d6459f989740dcbce96582fb380716c0f1f Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Nov 2018 11:56:12 -0800
Subject: [PATCH 0230/1554] [tf.data] Ensure that all needed functions are
 available when initializing an iterator.

Previously, the function library would be captured when an `IteratorResource` is first created, which would mean that it is unable to use dataset containing functions defined subsequently, and (e.g.) initialized via a reinitializable iterator. This change moves function library capturing to the `MakeIteratorOp`, which runs in the same graph as the `Dataset` kernels that define the dataset.

This CL also cleans up the locking in `IteratorResource`, and fixes a potential data race when an iterator is being simultaneously used and restored.

Fixes #23334.

PiperOrigin-RevId: 221484449
---
 tensorflow/core/kernels/data/iterator_ops.cc  | 156 ++++++++++--------
 .../data/kernel_tests/iterator_ops_test.py    |  26 +++
 2 files changed, 114 insertions(+), 68 deletions(-)

diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 445718ba1e..93999dc095 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/common_runtime/renamed_device.h"
 #include "tensorflow/core/common_runtime/threadpool_device.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
@@ -33,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/public/session_options.h"
 
 namespace tensorflow {
@@ -65,11 +67,14 @@ class IteratorResource : public ResourceBase {
 
   Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
                  bool* end_of_sequence) {
-    std::shared_ptr<IteratorBase> captured_iterator(iterator_);
-    if (captured_iterator) {
-      CHECK_NOTNULL(lib_);
-      IteratorContext::Params params(ctx);
+    IteratorContext::Params params(ctx);
+    std::shared_ptr<IteratorBase> captured_iterator;
+    {
+      tf_shared_lock l(mu_);
+      captured_iterator = iterator_;
       params.lib = lib_;
+    }
+    if (captured_iterator) {
       return captured_iterator->GetNext(IteratorContext(std::move(params)),
                                         out_tensors, end_of_sequence);
     } else {
@@ -86,7 +91,11 @@ class IteratorResource : public ResourceBase {
   }
 
   Status Save(SerializationContext* ctx, IteratorStateWriter* writer) {
-    std::shared_ptr<IteratorBase> captured_iterator(iterator_);
+    std::shared_ptr<IteratorBase> captured_iterator;
+    {
+      tf_shared_lock l(mu_);
+      captured_iterator = iterator_;
+    }
     if (captured_iterator) {
       return captured_iterator->Save(ctx, writer);
     } else {
@@ -131,14 +140,18 @@ class IteratorResource : public ResourceBase {
     TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset));
 
     std::unique_ptr<IteratorBase> iterator;
-    IteratorContext::Params params(ctx);
-    params.lib = lib;
-    TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
-                                             "Iterator", &iterator));
-    TF_RETURN_IF_ERROR(set_iterator(std::move(iterator)));
-    std::shared_ptr<IteratorBase> captured_iterator(iterator_);
+    {
+      IteratorContext::Params params(ctx);
+      params.lib = lib;
+      TF_RETURN_IF_ERROR(dataset->MakeIterator(
+          IteratorContext(std::move(params)), "Iterator", &iterator));
+    }
+    TF_RETURN_IF_ERROR(
+        VerifyTypesMatch(output_dtypes_, iterator->output_dtypes()));
+    TF_RETURN_IF_ERROR(
+        VerifyShapesCompatible(output_shapes_, iterator->output_shapes()));
 
-    if (captured_iterator) {
+    {
       IteratorContext::Params params(ctx);
       params.lib = lib;
       DeviceBase* device = lib->device();
@@ -146,36 +159,64 @@ class IteratorResource : public ResourceBase {
         return device->GetAllocator(attrs);
       };
       IteratorContext iter_ctx(std::move(params));
-      TF_RETURN_IF_ERROR(captured_iterator->Restore(&iter_ctx, reader));
+      TF_RETURN_IF_ERROR(iterator->Restore(&iter_ctx, reader));
+    }
+
+    std::shared_ptr<IteratorBase> old_iterator;
+    {
       mutex_lock l(mu_);
-      device_mgr_ = std::move(device_mgr);
-      lib_def_ = std::move(flib_def);
-      pflr_ = std::move(pflr);
+      std::swap(device_mgr_, device_mgr);
+      std::swap(flib_def_, flib_def);
+      std::swap(pflr_, pflr);
       lib_ = lib;
-      return Status::OK();
-    } else {
-      return errors::FailedPrecondition(
-          "Failed to restore iterator. Make sure the checkpoint ",
-          "is not corrupt. If the checkpoint does not contain the GraphDef, ",
-          "you will need to initialize your iterator before restoring.");
+      old_iterator = iterator_;
+      iterator_ = std::move(iterator);
     }
-  }
 
-  std::shared_ptr<const FunctionLibraryDefinition> function_library() {
-    tf_shared_lock l(mu_);
-    return lib_def_;
+    return Status::OK();
   }
 
-  FunctionLibraryRuntime* function_library_runtime() { return lib_; }
+  Status AddLibrary(const FunctionLibraryDefinition& flib_def) {
+    mutex_lock l(mu_);
+    return flib_def_->AddLibrary(flib_def);
+  }
 
-  // Transfers ownership of iterator to this. This method is thread-safe.
-  Status set_iterator(std::unique_ptr<IteratorBase> iterator) {
-    if (iterator) {
-      TF_RETURN_IF_ERROR(
-          VerifyTypesMatch(output_dtypes_, iterator->output_dtypes()));
+  Status SetIteratorFromDataset(OpKernelContext* ctx, DatasetBase* dataset) {
+    mutex_lock l(mu_);
+    // Ensure that the iterator has access to all functions in the current
+    // subgraph, because some functions may have been defined after the resource
+    // was initially created.
+    Status s = flib_def_->AddLibrary(
+        *ctx->function_library()->GetFunctionLibraryDefinition());
+
+    if (!s.ok()) {
+      // Adding functions to `flib_def_` may fail, if there are clashes between
+      // the function names in (e.g.) a restored graph and the currently
+      // executing graph. In that case, we create a new function runtime for
+      // this iterator, based on the current `OpKernelContext`, which will have
+      // the functions we need.
+      iterator_.reset();
+      FunctionLibraryRuntime* lib;
+      std::unique_ptr<DeviceMgr> device_mgr(nullptr);
+      std::unique_ptr<FunctionLibraryDefinition> flib_def(nullptr);
+      std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
       TF_RETURN_IF_ERROR(
-          VerifyShapesCompatible(output_shapes_, iterator->output_shapes()));
+          ctx->function_library()->Clone(&flib_def, &pflr, &lib));
+      std::swap(device_mgr_, device_mgr);
+      std::swap(flib_def_, flib_def);
+      std::swap(pflr_, pflr);
+      lib_ = lib;
     }
+
+    std::unique_ptr<IteratorBase> iterator;
+    IteratorContext::Params params(ctx);
+    params.lib = lib_;
+    TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
+                                             "Iterator", &iterator));
+    TF_RETURN_IF_ERROR(
+        VerifyTypesMatch(output_dtypes_, iterator->output_dtypes()));
+    TF_RETURN_IF_ERROR(
+        VerifyShapesCompatible(output_shapes_, iterator->output_shapes()));
     iterator_.reset(iterator.release());
     return Status::OK();
   }
@@ -189,16 +230,12 @@ class IteratorResource : public ResourceBase {
   }
 
  private:
-  // The following (device_mgr_, flib_def_, pflr_) are only used when the
-  // IteratorResource is shared between sessions and in that case we create
-  // a new FLR. Otherwise these are set to null.
-  std::unique_ptr<DeviceMgr> device_mgr_;
-  std::unique_ptr<FunctionLibraryDefinition> flib_def_;
-  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
-  FunctionLibraryRuntime* lib_ = nullptr;  // not owned.
-  std::shared_ptr<IteratorBase> iterator_;
   mutex mu_;
-  std::shared_ptr<const FunctionLibraryDefinition> lib_def_ GUARDED_BY(mu_);
+  std::unique_ptr<DeviceMgr> device_mgr_ GUARDED_BY(mu_);
+  std::unique_ptr<FunctionLibraryDefinition> flib_def_ GUARDED_BY(mu_);
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_ GUARDED_BY(mu_);
+  FunctionLibraryRuntime* lib_ GUARDED_BY(mu_) = nullptr;  // not owned.
+  std::shared_ptr<IteratorBase> iterator_ GUARDED_BY(mu_);
   const DataTypeVector output_dtypes_;
   const std::vector<PartialTensorShape> output_shapes_;
 };
@@ -584,13 +621,7 @@ void MakeIteratorOp::Compute(OpKernelContext* ctx) {
   OP_REQUIRES_OK(
       ctx, LookupResource(ctx, HandleFromInput(ctx, 1), &iterator_resource));
   core::ScopedUnref unref(iterator_resource);
-
-  std::unique_ptr<IteratorBase> iterator;
-  IteratorContext::Params params(ctx);
-  params.lib = iterator_resource->function_library_runtime();
-  OP_REQUIRES_OK(ctx, dataset->MakeIterator(IteratorContext(std::move(params)),
-                                            "Iterator", &iterator));
-  OP_REQUIRES_OK(ctx, iterator_resource->set_iterator(std::move(iterator)));
+  OP_REQUIRES_OK(ctx, iterator_resource->SetIteratorFromDataset(ctx, dataset));
 }
 
 namespace {
@@ -916,13 +947,7 @@ class OneShotIteratorOp : public AsyncOpKernel {
     // factory function.
     DatasetBase* dataset;
     TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(return_values[0], &dataset));
-    std::unique_ptr<IteratorBase> iter;
-    IteratorContext::Params params(ctx);
-    params.lib = lib;
-    TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
-                                             "Iterator", &iter));
-    TF_RETURN_IF_ERROR((*iterator)->set_iterator(std::move(iter)));
-
+    TF_RETURN_IF_ERROR((*iterator)->SetIteratorFromDataset(ctx, dataset));
     (*iterator)->Ref();
     return Status::OK();
   }
@@ -976,10 +1001,8 @@ void IteratorGetNextOp::ComputeAsync(OpKernelContext* ctx, DoneCallback done) {
         std::vector<Tensor> components;
         bool end_of_sequence = false;
 
-        IteratorContext::Params params(ctx);
-        params.function_library = iterator->function_library();
-        Status s = iterator->GetNext(IteratorContext(std::move(params)),
-                                     &components, &end_of_sequence);
+        Status s = iterator->GetNext(IteratorContext(ctx), &components,
+                                     &end_of_sequence);
         // NOTE(mrry): We must unref the iterator before calling `done()`, to
         // avoid destruction races.
         iterator->Unref();
@@ -1005,10 +1028,9 @@ void IteratorGetNextSyncOp::Compute(OpKernelContext* ctx) {
   core::ScopedUnref unref_iterator(iterator);
   std::vector<Tensor> components;
   bool end_of_sequence = false;
-  IteratorContext::Params params(ctx);
-  params.function_library = iterator->function_library();
-  OP_REQUIRES_OK(ctx, iterator->GetNext(IteratorContext(std::move(params)),
-                                        &components, &end_of_sequence));
+
+  OP_REQUIRES_OK(ctx, iterator->GetNext(IteratorContext(ctx), &components,
+                                        &end_of_sequence));
   OP_REQUIRES(ctx, !end_of_sequence, errors::OutOfRange("End of sequence"));
 
   for (int i = 0; i < components.size(); ++i) {
@@ -1041,10 +1063,8 @@ class IteratorGetNextAsOptionalOp : public AsyncOpKernel {
           std::vector<Tensor> components;
           bool end_of_sequence = false;
 
-          IteratorContext::Params params(ctx);
-          params.function_library = iterator->function_library();
-          Status s = iterator->GetNext(IteratorContext(std::move(params)),
-                                       &components, &end_of_sequence);
+          Status s = iterator->GetNext(IteratorContext(ctx), &components,
+                                       &end_of_sequence);
           // NOTE(mrry): We must unref the iterator before calling `done()`, to
           // avoid destruction races.
           iterator->Unref();
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
index a2a3528cc6..490ca813dc 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
@@ -334,6 +334,32 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  def testReinitializableIteratorWithFunctions(self):
+
+    def g():
+      for i in range(10):
+        yield i
+
+    iterator = iterator_ops.Iterator.from_structure(dtypes.int64, [])
+    next_element = iterator.get_next()
+
+    with self.cached_session() as sess:
+      dataset_1 = dataset_ops.Dataset.from_generator(
+          g, output_types=dtypes.int64)
+      sess.run(iterator.make_initializer(dataset_1))
+      for expected in range(10):
+        self.assertEqual(expected, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
+      dataset_2 = dataset_ops.Dataset.from_generator(
+          g, output_types=dtypes.int64)
+      sess.run(iterator.make_initializer(dataset_2))
+      for expected in range(10):
+        self.assertEqual(expected, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testReinitializableIteratorStaticErrors(self):
     # Non-matching structure for types and shapes.
     with self.assertRaises(TypeError):
-- 
GitLab


From e767ba2c0d9a6b8a129f172fcae93828f8b46b70 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 14 Nov 2018 12:34:04 -0800
Subject: [PATCH 0231/1554] Fix cudnn_rnn_test failure due to cudnn rnn savable
 refactorization

PiperOrigin-RevId: 221490781
---
 tensorflow/contrib/cudnn_rnn/BUILD                   |  1 -
 .../cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py  | 12 +++++++++---
 .../contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py    |  2 +-
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD
index a4c1e968b9..cec8dc2bed 100644
--- a/tensorflow/contrib/cudnn_rnn/BUILD
+++ b/tensorflow/contrib/cudnn_rnn/BUILD
@@ -92,7 +92,6 @@ cuda_py_test(
     ],
     shard_count = 6,
     tags = [
-        "no_oss",  # http://b/119506830
         "noasan",  # http://b/62067814
     ],
 )
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 1954f6717b..6cc93dccb0 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -536,7 +536,9 @@ class CudnnRNNTestSaveRestore(test_util.TensorFlowTestCase):
       save_path = os.path.join(self.get_temp_dir(),
                                "save-restore-variable-test")
       saver = saver_lib.Saver()
-      weights, biases = model.rnn.saveable._OpaqueParamsToCanonical()
+      weights, biases = (
+          model.rnn.saveable.format_converter._opaque_to_cu_canonical(
+              model.rnn.saveable._variables))
       opaque_params = rnn.trainable_variables[0]
       # CudnnTestModel() creates CudnnOpaqueParamsSaveable that helps saver save
       # Cudnn vars in canonical format.
@@ -583,8 +585,12 @@ class CudnnRNNTestSaveRestore(test_util.TensorFlowTestCase):
             dtype=dtype)
       opaque_params = (model1.rnn.trainable_variables[0],
                        model2.rnn.trainable_variables[0])
-      weights1, biases1 = model1.rnn.saveable._OpaqueParamsToCanonical()
-      weights2, biases2 = model2.rnn.saveable._OpaqueParamsToCanonical()
+      saveable1 = model1.rnn.saveable
+      weights1, biases1 = saveable1.format_converter._opaque_to_cu_canonical(
+          saveable1._variables)
+      saveable2 = model1.rnn.saveable
+      weights2, biases2 = saveable2.format_converter._opaque_to_cu_canonical(
+          saveable2._variables)
       reset_params = [
           state_ops.assign(params,
                            array_ops.zeros_like(params, dtype=dtype))
diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
index d06d0c6bda..1ce29b42d5 100644
--- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
+++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
@@ -738,7 +738,7 @@ class CudnnOpaqueParamsSaveable(saver.BaseSaverBuilder.SaveableObject):
         self._variables, opaque_params, validate_shape=False)
 
   def _checkpointable_save(self, save_buffer):
-    weights, biases = self.format_converter.opaque_params_to_tf_canonical(
+    weights, biases = self.format_converter.opaque_to_tf_canonical(
         self._variables)
     for name, tensor in zip(self._param_names, weights + biases):
       save_buffer[name] = array_ops.identity(tensor)
-- 
GitLab


From f1f54862ae4d2d63e9702dee66aaa08efd24d41f Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 14 Nov 2018 12:59:23 -0800
Subject: [PATCH 0232/1554] Fix QuantizationMissingAllRangesTest: the network
 was too simple and TRT was chosing not to quantize it.

---
 .../contrib/tensorrt/test/quantization_test.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
index f32b15b038..83295ce2bd 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -43,6 +43,7 @@ def build_graph(input_name, input_dims, output_name,
   with g.as_default():
     x = array_ops.placeholder(
         dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+
     x = quantize(x, 10.0)
     x = x + 5
     x = quantize(x, 15.0)
@@ -50,6 +51,9 @@ def build_graph(input_name, input_dims, output_name,
     x = quantize(x, 10.0)
     x = x * 0.1
     x = quantize(x, 1.0)
+    w = constant_op.constant(np.ones((10, 1)), dtype=dtypes.float32)
+    x = math_ops.matmul(x, w)
+    x = quantize(x, 10.0)
     x = array_ops.identity(x, name=output_name)
   return g
 
@@ -58,7 +62,7 @@ class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
   def GetParams(self):
     """Create a graph containing single segment with no quantization ranges."""
     input_name = "input"
-    input_dims = [100, 100]
+    input_dims = [128, 10]
     output_name = "output"
     g = build_graph(input_name, input_dims, output_name,
                     add_quantization_nodes=False)
@@ -67,7 +71,7 @@ class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[(100, 100)])
+        expected_output_dims=[(128, 1)])
 
   def ShouldRunTest(self, run_params):
     return (run_params.precision_mode == "INT8" and
@@ -85,7 +89,7 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
   def GetParams(self):
     """Create a graph containing single segment with no quantization ranges."""
     input_name = "input"
-    input_dims = [100, 100]
+    input_dims = [128, 10]
     output_name = "output"
     g = build_graph(input_name, input_dims, output_name,
                     add_quantization_nodes=True)
@@ -94,7 +98,7 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[(100, 100)])
+        expected_output_dims=[(128, 1)])
 
   def ShouldRunTest(self, run_params):
     return (run_params.precision_mode == "INT8" and
@@ -117,7 +121,7 @@ class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
   def GetParams(self):
     """Create a graph containing single segment with no quantization ranges."""
     input_name = "input"
-    input_dims = [100, 100]
+    input_dims = [128, 10]
     output_name = "output"
     g = build_graph(input_name, input_dims, output_name,
                     add_quantization_nodes=True)
@@ -126,7 +130,7 @@ class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[(100, 100)])
+        expected_output_dims=[(128, 1)])
 
   def ShouldRunTest(self, run_params):
     return (run_params.precision_mode == "FP32" or
@@ -134,7 +138,7 @@ class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2"]
+    return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3"]
   
   def ExpectedAbsoluteTolerance(self, run_params):
     """The absolute tolerance to compare floating point results."""
-- 
GitLab


From 1ecd0a14f57ef2aa736839413741fec367981c2a Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 14 Nov 2018 13:02:34 -0800
Subject: [PATCH 0233/1554] Fusing BatchNorm into the Conv2D op.

PiperOrigin-RevId: 221495360
---
 .../core/grappler/optimizers/remapper.cc      |   4 +-
 tensorflow/core/kernels/conv_ops_fused.cc     | 296 ++++++--
 tensorflow/core/kernels/conv_ops_test.cc      | 697 ++++++++++++++----
 tensorflow/core/ops/nn_ops.cc                 |   3 +
 4 files changed, 818 insertions(+), 182 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index 32f603a949..b97a95f302 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -168,7 +168,7 @@ bool FindFusedBatchNorm(const GraphView& graph_view,
 
 void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
                           const std::vector<string>& fused_ops = {},
-                          int num_args = 1) {
+                          int num_args = 1, float epsilon = 0.0) {
   auto* attr = fused_conv2d->mutable_attr();
   auto src_attr = conv2d->attr();
 
@@ -184,6 +184,8 @@ void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
   }
 
   SetAttrValue(num_args, &(*attr)["num_args"]);
+  // Required only for FusedBatchNorm.
+  SetAttrValue(epsilon, &(*attr)["epsilon"]);
 }
 
 void AddFusedConv2DNode(const Conv2DWithBiasAdd& matched,
diff --git a/tensorflow/core/kernels/conv_ops_fused.cc b/tensorflow/core/kernels/conv_ops_fused.cc
index c75bb67932..4f30d625e0 100644
--- a/tensorflow/core/kernels/conv_ops_fused.cc
+++ b/tensorflow/core/kernels/conv_ops_fused.cc
@@ -929,9 +929,45 @@ template <typename Scalar, typename Index>
 using ContractionOutputMapper =
     Eigen::internal::blas_data_mapper<Scalar, Index, Eigen::ColMajor>;
 
-// Output kernel that fused BiasAdd operation into the output of tensor
-// contraction.
-template <typename T>
+// Returns input expression without any transformations.
+struct Identity {
+  template <typename XprType>
+  static auto apply(XprType expr) -> XprType {
+    return expr;
+  };
+};
+
+// Applies `Relu` to the passed input expression.
+struct Relu {
+  template <typename XprType>
+  static auto apply(XprType expr)
+      -> decltype(expr.cwiseMax(std::declval<typename XprType::Scalar>())) {
+    return expr.cwiseMax(static_cast<typename XprType::Scalar>(0));
+  };
+};
+
+// TensorContraction swaps lhs with rhs, and changes layout from RowMajor
+// (default in Tensorflow) to ColMajor (preferred in Eigen), and computes matmul
+// using these tensors.
+//
+// TensorContraction output matrix (before reshape) has a ColMajor layout, and
+// has dimensions:
+//  - rows: output_channels
+//  - cols: all other dimensions
+//
+// First element in every column is:
+//   [batch ??, height ??, width ??, out_channel = i]
+//
+// We do not know what are the values of the 'batch', 'height', and 'width' here
+// (if we know original dimensions, they can be computed from 'j').
+//
+// Each column of an output block is a continuous slice along the output channel
+// dimension, so we can use it to efficiently compute any transformation that
+// depends only on a channel value (e.g. add channel bias).
+
+// Output kernel that fuses BiasAdd operation into the output of tensor
+// contraction + any other transformation defined by Transform.
+template <typename T, typename Transform = Identity>
 struct BiasAddOutputKernel {
   explicit BiasAddOutputKernel(const T* bias_data) : bias_data(bias_data) {}
 
@@ -943,13 +979,13 @@ struct BiasAddOutputKernel {
     DCHECK(params.swapped_arguments);
 
     const T* bias_base = bias_data + i;
+    typename OutputTypes<T>::ConstTensor bias(bias_base, num_rows);
 
-    // TODO(ezhulenev): Use Eigen::Array with strides after upgrading Eigen.
     for (int col = 0; col < num_cols; ++col) {
       T* output_base = &output_mapper(0, col);
       typename OutputTypes<T>::Tensor output(output_base, num_rows);
-      typename OutputTypes<T>::ConstTensor bias(bias_base, num_rows);
-      output = output + bias;
+      const auto expr = output + bias;
+      output = Transform::template apply<decltype(expr)>(expr);
     }
   }
 
@@ -957,12 +993,16 @@ struct BiasAddOutputKernel {
   const T* bias_data;
 };
 
-// Output kernel that fused BiasAdd and Relu operations into the output of
-// tensor contraction.
-template <typename T>
-struct BiasAddWithReluOutputKernel {
-  explicit BiasAddWithReluOutputKernel(const T* bias_data)
-      : bias_data(bias_data) {}
+// Output kernel that fuses FusedBatchNorm operation into the output of tensor
+// contraction + any other transformation defined by Transform.
+template <typename T, typename Transform = Identity>
+struct FusedBatchNormOutputKernel {
+  FusedBatchNormOutputKernel(T epsilon, const T* scaling_factor_data,
+                             const T* offset_data, const T* estimated_mean_data)
+      : epsilon(epsilon),
+        scaling_factor_data(scaling_factor_data),
+        offset_data(offset_data),
+        estimated_mean_data(estimated_mean_data) {}
 
   template <typename Index, typename Scalar>
   EIGEN_ALWAYS_INLINE void operator()(
@@ -971,19 +1011,31 @@ struct BiasAddWithReluOutputKernel {
       Index num_rows, Index num_cols) const {
     DCHECK(params.swapped_arguments);
 
-    const T* bias_base = bias_data + i;
+    const T* scaling_factor_base = scaling_factor_data + i;
+    const T* offset_base = offset_data + i;
+    const T* mean_base = estimated_mean_data + i;
+
+    typename OutputTypes<T>::ConstTensor scaling_factor(scaling_factor_base,
+                                                        num_rows);
+    typename OutputTypes<T>::ConstTensor offset(offset_base, num_rows);
+    typename OutputTypes<T>::ConstTensor mean(mean_base, num_rows);
 
-    // TODO(ezhulenev): Use Eigen::Array with strides after upgrading Eigen.
     for (int col = 0; col < num_cols; ++col) {
       T* output_base = &output_mapper(0, col);
       typename OutputTypes<T>::Tensor output(output_base, num_rows);
-      typename OutputTypes<T>::ConstTensor bias(bias_base, num_rows);
-      output = (output + bias).cwiseMax(static_cast<T>(0));
+
+      auto scaled = (output - mean) * scaling_factor;
+      auto shifted = scaled + offset;
+
+      output = Transform::template apply<decltype(shifted)>(shifted);
     }
   }
 
  private:
-  const T* bias_data;
+  T epsilon;
+  const T* scaling_factor_data;
+  const T* offset_data;
+  const T* estimated_mean_data;
 };
 
 // Type aliases for the output kernels, purely for the sake of better launch
@@ -991,21 +1043,33 @@ struct BiasAddWithReluOutputKernel {
 template <typename T>
 using WithBiasAdd = BiasAddOutputKernel<T>;
 template <typename T>
-using WithBiasAddAndRelu = BiasAddWithReluOutputKernel<T>;
+using WithBiasAddAndRelu = BiasAddOutputKernel<T, Relu>;
+template <typename T>
+using WithFusedBatchNorm = FusedBatchNormOutputKernel<T>;
+template <typename T>
+using WithFusedBatchNormAndRelu = FusedBatchNormOutputKernel<T, Relu>;
 
 // Dispatch 2D convolution to the appropriate primitive operation:
 //   (1) MatMul for the case of 1x1 convolution.
 //   (2) MatMul for the case when filter size equals to the input size.
 //   (3) General spatial 2D convolution for all other cases.
-template <typename T, typename OutputKernel>
-struct LaunchConv2DWithOutputKernel {
-  void operator()(OpKernelContext* ctx, const Tensor& input,
-                  const Tensor& filter, int row_stride, int col_stride,
-                  int row_dilation, int col_dilation, const Padding& padding,
-                  const OutputKernel& output_kernel, Tensor* output,
-                  TensorFormat data_format) {
-    if (filter.dim_size(0) == 1 && filter.dim_size(1) == 1 && row_stride == 1 &&
-        col_stride == 1) {
+template <typename T>
+class LaunchConv2DWithOutputKernel {
+ public:
+  LaunchConv2DWithOutputKernel(int row_stride, int col_stride,      //
+                               int row_dilation, int col_dilation,  //
+                               Padding padding)
+      : row_stride_(row_stride),
+        col_stride_(col_stride),
+        row_dilation_(row_dilation),
+        col_dilation_(col_dilation),
+        padding_(padding) {}
+
+  template <typename OutputKernel>
+  void operator()(const OutputKernel& output_kernel, OpKernelContext* ctx,
+                  const Tensor& input, const Tensor& filter, Tensor* output) {
+    if (filter.dim_size(0) == 1 && filter.dim_size(1) == 1 &&
+        row_stride_ == 1 && col_stride_ == 1) {
       int conv_width = 1;  // Width for the convolution step.
       for (int i = 0; i < 3; ++i) {
         conv_width *= output->dim_size(i);
@@ -1021,8 +1085,8 @@ struct LaunchConv2DWithOutputKernel {
           dim_pair, output_kernel);
 
     } else if (filter.dim_size(0) == input.dim_size(1) &&
-               filter.dim_size(1) == input.dim_size(2) && row_dilation == 1 &&
-               col_dilation == 1 && padding == VALID) {
+               filter.dim_size(1) == input.dim_size(2) && row_dilation_ == 1 &&
+               col_dilation_ == 1 && padding_ == VALID) {
       // If the input data and filter have the same height/width,
       // reduce the 2D convolution to matrix multiplication.
       const auto k =  // Length of reduction dimension.
@@ -1040,11 +1104,18 @@ struct LaunchConv2DWithOutputKernel {
     } else {
       functor::SpatialConvolution<CPUDevice, T, OutputKernel>()(
           ctx->eigen_device<CPUDevice>(), output->tensor<T, 4>(),
-          input.tensor<T, 4>(), filter.tensor<T, 4>(), row_stride, col_stride,
-          row_dilation, col_dilation, BrainPadding2EigenPadding(padding),
+          input.tensor<T, 4>(), filter.tensor<T, 4>(), row_stride_, col_stride_,
+          row_dilation_, col_dilation_, BrainPadding2EigenPadding(padding_),
           output_kernel);
     }
   }
+
+ private:
+  int row_stride_;
+  int col_stride_;
+  int row_dilation_;
+  int col_dilation_;
+  const Padding padding_;
 };
 
 }  // namespace
@@ -1065,27 +1136,43 @@ class FusedConv2DOp : public OpKernel {
                 errors::InvalidArgument(
                     "Fused Conv2D must have at least one fused op."));
 
-    // Right now we always expect to have just one extra argument that is an
-    // input to the BiasAdd. In future we might fuse other types of computations
-    // taking additional arguments.
-
     int num_args;
     OP_REQUIRES_OK(context, context->GetAttr("num_args", &num_args));
-    OP_REQUIRES(context, num_args == 1,
-                errors::InvalidArgument(
-                    "Fused Conv2D must have one extra argument with a bias."));
 
     // TODO(ezhulenev): Add support for fusion element-wise op chains defined
     // at runtime, e.g. Relu+Sqrt+Tanh+etc...
 
+    // Match combination of fused ops to one of the supported fusions.
     if (FusedOpsMatches(fused_ops, {"BiasAdd"})) {
       fused_computation_ = FusedComputationType::kBiasAdd;
     } else if (FusedOpsMatches(fused_ops, {"BiasAdd", "Relu"})) {
       fused_computation_ = FusedComputationType::kBiasAddWithRelu;
+    } else if (FusedOpsMatches(fused_ops, {"FusedBatchNorm"})) {
+      fused_computation_ = FusedComputationType::kFusedBatchNorm;
+    } else if (FusedOpsMatches(fused_ops, {"FusedBatchNorm", "Relu"})) {
+      fused_computation_ = FusedComputationType::kFusedBatchNormWithRelu;
     } else {
       OP_REQUIRES(context, false,
-                  errors::Unimplemented("Fusion is not implemented: ",
-                                        str_util::Join(fused_ops, ",")));
+                  errors::Unimplemented("Fusion is not implemented: [",
+                                        str_util::Join(fused_ops, ","), "]"));
+    }
+
+    // Depending on a picked fusion type validate fusion-specific arguments.
+
+    if (fused_computation_ == FusedComputationType::kBiasAdd ||
+        fused_computation_ == FusedComputationType::kBiasAddWithRelu) {
+      OP_REQUIRES(context, num_args == 1,
+                  errors::InvalidArgument(
+                      "Fused Conv2D must have one extra argument: bias."));
+    }
+
+    if (fused_computation_ == FusedComputationType::kFusedBatchNorm ||
+        fused_computation_ == FusedComputationType::kFusedBatchNormWithRelu) {
+      OP_REQUIRES(
+          context, num_args == 4,
+          errors::InvalidArgument("Fused FusedBatchNorm must have four extra "
+                                  "arguments: scale, offset, mean, variance."));
+      OP_REQUIRES_OK(context, context->GetAttr("epsilon", &epsilon_));
     }
   }
 
@@ -1098,10 +1185,6 @@ class FusedConv2DOp : public OpKernel {
     // [ filter_rows, filter_cols, in_depth, out_depth]
     const Tensor& filter = context->input(1);
 
-    // Bias of the following dimensions:
-    // [ output_depth ]
-    const Tensor& bias = context->input(2);
-
     Conv2DDimensions dimensions;
     OP_REQUIRES_OK(context,
                    ComputeConv2DDimension(params_, input, filter, &dimensions));
@@ -1139,23 +1222,47 @@ class FusedConv2DOp : public OpKernel {
                 errors::Unimplemented("Fused conv implementation does not "
                                       "support grouped convolutions for now."));
 
-    auto bias_data = reinterpret_cast<const T*>(bias.tensor_data().data());
+    BiasAddArgs bias_add;
+    FusedBatchNormArgs fused_batch_norm;
 
-#define LAUNCH_CONV2D(KERNEL)                                                 \
-  LaunchConv2DWithOutputKernel<T, KERNEL>()(                                  \
-      context, input, filter, dimensions.stride_rows, dimensions.stride_cols, \
-      dimensions.dilation_rows, dimensions.dilation_cols, params_.padding,    \
-      KERNEL(bias_data), output, params_.data_format);                        \
-  break
+    LaunchConv2DWithOutputKernel<T> conv2d(
+        dimensions.stride_rows, dimensions.stride_cols,
+        dimensions.dilation_rows, dimensions.dilation_cols, params_.padding);
 
     switch (fused_computation_) {
       case FusedComputationType::kBiasAdd:
-        LAUNCH_CONV2D(WithBiasAdd<T>);
+        OP_REQUIRES_OK(context, InitBiasAddArgs(context, &bias_add));
+        conv2d(WithBiasAdd<T>(bias_add.bias_add_data), context, input, filter,
+               output);
+        break;
+
       case FusedComputationType::kBiasAddWithRelu:
-        LAUNCH_CONV2D(WithBiasAddAndRelu<T>);
+        OP_REQUIRES_OK(context, InitBiasAddArgs(context, &bias_add));
+        conv2d(WithBiasAddAndRelu<T>(bias_add.bias_add_data), context, input,
+               filter, output);
+        break;
+
+      case FusedComputationType::kFusedBatchNorm:
+        OP_REQUIRES_OK(context,
+                       InitFusedBatchNormArgs(context, &fused_batch_norm));
+        conv2d(WithFusedBatchNorm<T>(epsilon_,
+                                     fused_batch_norm.scaling_factor.data(),
+                                     fused_batch_norm.offset_data,
+                                     fused_batch_norm.estimated_mean_data),
+               context, input, filter, output);
+        break;
+
+      case FusedComputationType::kFusedBatchNormWithRelu:
+        OP_REQUIRES_OK(context,
+                       InitFusedBatchNormArgs(context, &fused_batch_norm));
+        conv2d(WithFusedBatchNormAndRelu<T>(
+                   epsilon_, fused_batch_norm.scaling_factor.data(),
+                   fused_batch_norm.offset_data,
+                   fused_batch_norm.estimated_mean_data),
+               context, input, filter, output);
+        break;
     }
   }
-#undef LAUNCH_CONV2D
 
  private:
   bool FusedOpsMatches(const std::vector<string>& fused_ops,
@@ -1163,13 +1270,92 @@ class FusedConv2DOp : public OpKernel {
     return fused_ops == expected;
   }
 
+  struct BiasAddArgs {
+    const T* bias_add_data = nullptr;
+  };
+
+  struct FusedBatchNormArgs {
+    const T* scale_data = nullptr;
+    const T* offset_data = nullptr;
+    const T* estimated_mean_data = nullptr;
+    const T* estimated_variance_data = nullptr;
+
+    // Precomputed expression:
+    //   scaling_factor = (estimated_variance + epsilon).rsqrt() * scale
+    Eigen::Tensor<T, 1, Eigen::RowMajor> scaling_factor;
+  };
+
+#define TF_REQUIRES(EXP, STATUS) \
+  if (!TF_PREDICT_TRUE(EXP)) return (STATUS)
+
+  void InitDataPtr(const Tensor& tensor, const T** ptr) const {
+    *ptr = reinterpret_cast<const T*>(tensor.tensor_data().data());
+  }
+
+  Status InitBiasAddArgs(OpKernelContext* context, BiasAddArgs* args) const {
+    // Bias of the following dimensions: [ output_depth ]
+    const Tensor& bias = context->input(2);
+
+    TF_REQUIRES(bias.dims() == 1,
+                errors::InvalidArgument("bias must be 1-dimensional",
+                                        bias.shape().DebugString()));
+
+    InitDataPtr(bias, &args->bias_add_data);
+
+    return Status::OK();
+  }
+
+  Status InitFusedBatchNormArgs(OpKernelContext* context,
+                                FusedBatchNormArgs* args) const {
+    const Tensor& scale = context->input(2);
+    const Tensor& offset = context->input(3);
+    const Tensor& estimated_mean = context->input(4);
+    const Tensor& estimated_variance = context->input(5);
+
+    TF_REQUIRES(scale.dims() == 1,
+                errors::InvalidArgument("scale must be 1-dimensional",
+                                        scale.shape().DebugString()));
+    TF_REQUIRES(offset.dims() == 1,
+                errors::InvalidArgument("offset must be 1-dimensional",
+                                        offset.shape().DebugString()));
+    TF_REQUIRES(estimated_mean.dims() == 1,
+                errors::InvalidArgument("estimated_mean must be 1-dimensional",
+                                        estimated_mean.shape().DebugString()));
+    TF_REQUIRES(
+        estimated_variance.dims() == 1,
+        errors::InvalidArgument("estimated_variance must be 1-dimensional",
+                                estimated_variance.shape().DebugString()));
+
+    InitDataPtr(scale, &args->scale_data);
+    InitDataPtr(offset, &args->offset_data);
+    InitDataPtr(estimated_mean, &args->estimated_mean_data);
+    InitDataPtr(estimated_variance, &args->estimated_variance_data);
+
+    // Precompute scaling factor once for all output blocks (kernels).
+    args->scaling_factor =
+        (estimated_variance.flat<T>() + static_cast<T>(epsilon_)).rsqrt() *
+        scale.flat<T>();
+
+    return Status::OK();
+  }
+
+#undef TF_REQUIRES
+
   // Element-wise ops applied to the result of Conv2D.
   // TODO(ezhulenev): Add support for runtime-defined op chains.
-  enum class FusedComputationType { kBiasAdd, kBiasAddWithRelu };
+  enum class FusedComputationType {
+    kBiasAdd,
+    kBiasAddWithRelu,
+    kFusedBatchNorm,
+    kFusedBatchNormWithRelu
+  };
 
   Conv2DParameters params_;
   FusedComputationType fused_computation_;
 
+  // FusedBatchNorm attributes.
+  float epsilon_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(FusedConv2DOp);
 };
 
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 6421cad367..87bbc30573 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <string>
+#include <vector>
+
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/image_ops.h"
 #include "tensorflow/cc/ops/nn_ops.h"
@@ -522,6 +525,7 @@ TEST_F(ConvOpTest, HandwrittenConv) { HandwrittenConv(); }
 
 TEST_F(ConvOpTest, AnisotropicStride) { AnisotropicStrides(); }
 
+template <typename T>
 class FusedConv2DOpTest : public OpsTestBase {
  protected:
   static constexpr int kDepth = 3;
@@ -529,10 +533,15 @@ class FusedConv2DOpTest : public OpsTestBase {
   static constexpr int kImageHeight = 32;
   static constexpr int kImageBatchCount = 8;
 
-  using GraphRunner =
+  using BiasAddGraphRunner =
       std::function<void(const Tensor& input_data, const Tensor& filter_data,
                          const Tensor& bias_data, Tensor* out)>;
 
+  using BatchNormGraphRunner = std::function<void(
+      const Tensor& input_data, const Tensor& filter_data,
+      const Tensor& scale_data, const Tensor& offset_data,
+      const Tensor& mean_data, const Tensor& variance_data, Tensor* out)>;
+
   // Runs a Tensorflow graph defined by the root scope, and fetches the result
   // of 'fetch' node into the output Tensor.
   void RunAndFetch(const tensorflow::Scope& root, const string& fetch,
@@ -550,8 +559,9 @@ class FusedConv2DOpTest : public OpsTestBase {
     *output = unfused_tensors[0];
   }
 
-  void RunConv2DOp(const Tensor& input_data, const Tensor& filter_data,
-                   const Tensor& bias_data, Tensor* output, int stride = 1) {
+  void RunConv2DWithBias(const Tensor& input_data, const Tensor& filter_data,
+                         const Tensor& bias_data, Tensor* output,
+                         int stride = 1) {
     auto root = tensorflow::Scope::NewRootScope();
 
     auto conv = ops::Conv2D(
@@ -567,9 +577,10 @@ class FusedConv2DOpTest : public OpsTestBase {
     RunAndFetch(root, "with_bias", output);
   }
 
-  void RunConv2DWithReluOp(const Tensor& input_data, const Tensor& filter_data,
-                           const Tensor& bias_data, Tensor* output,
-                           int stride = 1) {
+  void RunConv2DWithBiasAndRelu(const Tensor& input_data,
+                                const Tensor& filter_data,
+                                const Tensor& bias_data, Tensor* output,
+                                int stride = 1) {
     auto root = tensorflow::Scope::NewRootScope();
 
     auto conv = ops::Conv2D(
@@ -587,18 +598,79 @@ class FusedConv2DOpTest : public OpsTestBase {
     RunAndFetch(root, "with_relu", output);
   }
 
-  template <typename T>
+  void RunConv2DWithBatchNorm(const Tensor& input_data,
+                              const Tensor& filter_data,
+                              const Tensor& scale_data,
+                              const Tensor& offset_data,
+                              const Tensor& mean_data,
+                              const Tensor& variance_data, Tensor* output,
+                              int stride = 1) {
+    auto root = tensorflow::Scope::NewRootScope();
+
+    auto conv = ops::Conv2D(
+        root.WithOpName("conv"),
+        ops::Const(root.WithOpName("input"), Input::Initializer(input_data)),
+        ops::Const(root.WithOpName("filter"), Input::Initializer(filter_data)),
+        {1, stride, stride, 1}, "SAME");
+
+    ops::FusedBatchNorm::Attrs attr;
+    attr = attr.IsTraining(false);
+
+    auto with_fused_batch_norm = ops::FusedBatchNorm(
+        root.WithOpName("with_fused_batch_norm"), conv,
+        ops::Const(root.WithOpName("scale"), Input::Initializer(scale_data)),
+        ops::Const(root.WithOpName("offset"), Input::Initializer(offset_data)),
+        ops::Const(root.WithOpName("mean"), Input::Initializer(mean_data)),
+        ops::Const(root.WithOpName("var"), Input::Initializer(variance_data)),
+        attr);
+
+    RunAndFetch(root, "with_fused_batch_norm", output);
+  }
+
+  void RunConv2DWithBatchNormAndRelu(const Tensor& input_data,
+                                     const Tensor& filter_data,
+                                     const Tensor& scale_data,
+                                     const Tensor& offset_data,
+                                     const Tensor& mean_data,
+                                     const Tensor& variance_data,
+                                     Tensor* output, int stride = 1) {
+    auto root = tensorflow::Scope::NewRootScope();
+
+    auto conv = ops::Conv2D(
+        root.WithOpName("conv"),
+        ops::Const(root.WithOpName("input"), Input::Initializer(input_data)),
+        ops::Const(root.WithOpName("filter"), Input::Initializer(filter_data)),
+        {1, stride, stride, 1}, "SAME");
+
+    ops::FusedBatchNorm::Attrs attr;
+    attr = attr.IsTraining(false);
+
+    auto with_fused_batch_norm = ops::FusedBatchNorm(
+        root.WithOpName("with_fused_batch_norm"), conv,
+        ops::Const(root.WithOpName("scale"), Input::Initializer(scale_data)),
+        ops::Const(root.WithOpName("offset"), Input::Initializer(offset_data)),
+        ops::Const(root.WithOpName("mean"), Input::Initializer(mean_data)),
+        ops::Const(root.WithOpName("var"), Input::Initializer(variance_data)),
+        attr);
+
+    auto with_relu =
+        ops::Relu(root.WithOpName("with_relu"), with_fused_batch_norm.y);
+
+    RunAndFetch(root, "with_relu", output);
+  }
+
   void RunFusedConv2DOp(const Tensor& image, const Tensor& filter,
-                        const Tensor& bias,
+                        const std::vector<Tensor>& args,
                         const std::vector<string>& fused_ops, Tensor* output,
                         int stride = 1) {
     DataType dtype = DataTypeToEnum<T>::v();
+    int num_args = static_cast<int>(args.size());
 
     TF_EXPECT_OK(NodeDefBuilder("fused_conv_op", "_FusedConv2D")
                      .Input(FakeInput(dtype))
                      .Input(FakeInput(dtype))
-                     .Attr("num_args", 1)
-                     .Input(FakeInput(dtype))
+                     .Attr("num_args", num_args)
+                     .Input(FakeInput(num_args, dtype))
                      .Attr("T", dtype)
                      .Attr("strides", {1, stride, stride, 1})
                      .Attr("padding", "SAME")
@@ -609,18 +681,20 @@ class FusedConv2DOpTest : public OpsTestBase {
 
     AddInputFromArray<T>(image.shape(), image.flat<T>());
     AddInputFromArray<T>(filter.shape(), filter.flat<T>());
-    AddInputFromArray<T>(bias.shape(), bias.flat<T>());
+    for (const Tensor& arg : args)
+      AddInputFromArray<T>(arg.shape(), arg.flat<T>());
     TF_ASSERT_OK(RunOpKernel());
 
     *output = *GetOutput(0);
   }
 
-  template <typename T>
-  void VerifyTensorsNear(int depth, int image_width, int image_height,
-                         int image_batch_count, int filter_size,
-                         int filter_count, const GraphRunner& run_default,
-                         const GraphRunner& run_fused) {
+  void VerifyBiasAddTensorsNear(int depth, int image_width, int image_height,
+                                int image_batch_count, int filter_size,
+                                int filter_count,
+                                const BiasAddGraphRunner& run_default,
+                                const BiasAddGraphRunner& run_fused) {
     DataType dtype = DataTypeToEnum<T>::v();
+
     Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
     image.flat<T>() = image.flat<T>().setRandom();
 
@@ -643,111 +717,272 @@ class FusedConv2DOpTest : public OpsTestBase {
     test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-5);
   }
 
+  void VerifyFusedBatchNormTensorsNear(int depth, int image_width,
+                                       int image_height, int image_batch_count,
+                                       int filter_size, int filter_count,
+                                       const BatchNormGraphRunner& run_default,
+                                       const BatchNormGraphRunner& run_fused) {
+    DataType dtype = DataTypeToEnum<T>::v();
+
+    Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
+    image.flat<T>() = image.flat<T>().setRandom();
+
+    Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
+    filter.flat<T>() = filter.flat<T>().setRandom();
+
+    const int scale_size = filter_count;
+
+    Tensor scale(dtype, {scale_size});
+    scale.flat<T>() = scale.flat<T>().setRandom();
+
+    Tensor offset(dtype, {scale_size});
+    offset.flat<T>() = offset.flat<T>().setRandom();
+
+    Tensor mean(dtype, {scale_size});
+    mean.flat<T>() = mean.flat<T>().setRandom();
+
+    Tensor variance(dtype, {scale_size});
+    variance.flat<T>() = variance.flat<T>().setRandom();
+    variance.flat<T>() += variance.flat<T>().constant(static_cast<T>(0.5f));
+
+    Tensor conv_2d;
+    Tensor fused_conv_2d;
+
+    run_default(image, filter, scale, offset, mean, variance, &conv_2d);
+    run_fused(image, filter, scale, offset, mean, variance, &fused_conv_2d);
+
+    ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
+    ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
+
+    test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-3);
+  }
+
   // Verifies that computing Conv2D+BiasAdd in a graph is identical to
   // FusedConv2D.
-  template <typename T>
-  void VerifyConv2DWithBias(int depth, int image_width, int image_height,
-                            int image_batch_count, int filter_size,
-                            int filter_count) {
-    const GraphRunner run_default =
+  void VerifyConv2DWithBias(int filter_size, int filter_count,
+                            int depth = kDepth, int image_width = kImageWidth,
+                            int image_height = kImageHeight,
+                            int image_batch_count = kImageBatchCount) {
+    const BiasAddGraphRunner run_default =
         [this](const Tensor& input_data, const Tensor& filter_data,
                const Tensor& bias_data, Tensor* out) {
-          RunConv2DOp(input_data, filter_data, bias_data, out);
+          RunConv2DWithBias(input_data, filter_data, bias_data, out);
         };
 
-    const GraphRunner run_fused = [this](const Tensor& input_data,
-                                         const Tensor& filter_data,
-                                         const Tensor& bias_data, Tensor* out) {
-      RunFusedConv2DOp<T>(input_data, filter_data, bias_data, {"BiasAdd"}, out);
+    const BiasAddGraphRunner run_fused = [this](const Tensor& input_data,
+                                                const Tensor& filter_data,
+                                                const Tensor& bias_data,
+                                                Tensor* out) {
+      RunFusedConv2DOp(input_data, filter_data, {bias_data}, {"BiasAdd"}, out);
     };
 
-    VerifyTensorsNear<T>(depth, image_width, image_height, image_batch_count,
-                         filter_size, filter_count, run_default, run_fused);
+    VerifyBiasAddTensorsNear(depth, image_width, image_height,
+                             image_batch_count, filter_size, filter_count,
+                             run_default, run_fused);
   }
 
   // Verifies that computing Conv2D+BiasAdd+Relu in a graph is identical to
   // FusedConv2D.
-  template <typename T>
-  void VerifyConv2DWithBiasAndRelu(int depth, int image_width, int image_height,
-                                   int image_batch_count, int filter_size,
-                                   int filter_count) {
-    const GraphRunner run_default =
+  void VerifyConv2DWithBiasAndRelu(int filter_size, int filter_count,
+                                   int depth = kDepth,
+                                   int image_width = kImageWidth,
+                                   int image_height = kImageHeight,
+                                   int image_batch_count = kImageBatchCount) {
+    const BiasAddGraphRunner run_default =
         [this](const Tensor& input_data, const Tensor& filter_data,
                const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithReluOp(input_data, filter_data, bias_data, out);
+          RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out);
         };
 
-    const GraphRunner run_fused = [this](const Tensor& input_data,
-                                         const Tensor& filter_data,
-                                         const Tensor& bias_data, Tensor* out) {
-      RunFusedConv2DOp<T>(input_data, filter_data, bias_data,
-                          {"BiasAdd", "Relu"}, out);
-    };
+    const BiasAddGraphRunner run_fused =
+        [this](const Tensor& input_data, const Tensor& filter_data,
+               const Tensor& bias_data, Tensor* out) {
+          RunFusedConv2DOp(input_data, filter_data, {bias_data},
+                           {"BiasAdd", "Relu"}, out);
+        };
 
-    VerifyTensorsNear<T>(depth, image_width, image_height, image_batch_count,
-                         filter_size, filter_count, run_default, run_fused);
+    VerifyBiasAddTensorsNear(depth, image_width, image_height,
+                             image_batch_count, filter_size, filter_count,
+                             run_default, run_fused);
   }
-};
 
-#define FUSED_CONV2D_TESTS(dtype, name)                                       \
-  TEST_F(FusedConv2DOpTest, Conv2DWithBiasAddOneByOneConvolution##name) {     \
-    const int filter_size = 1;                                                \
-    const int filter_count = 12;                                              \
-                                                                              \
-    VerifyConv2DWithBias<dtype>(kDepth, kImageWidth, kImageHeight,            \
-                                kImageBatchCount, filter_size, filter_count); \
-  }                                                                           \
-                                                                              \
-  TEST_F(FusedConv2DOpTest, Conv2DWithBiasAddImageSizeConvolution##name) {    \
-    const int filter_size = 32;                                               \
-    const int filter_count = 12;                                              \
-                                                                              \
-    VerifyConv2DWithBias<dtype>(kDepth, kImageWidth, kImageHeight,            \
-                                kImageBatchCount, filter_size, filter_count); \
-  }                                                                           \
-                                                                              \
-  TEST_F(FusedConv2DOpTest, Conv2DWithBiasAddSpatialConvolution##name) {      \
-    const int filter_size = 3;                                                \
-    const int filter_count = 12;                                              \
-                                                                              \
-    VerifyConv2DWithBias<dtype>(kDepth, kImageWidth, kImageHeight,            \
-                                kImageBatchCount, filter_size, filter_count); \
-  }                                                                           \
-                                                                              \
-  TEST_F(FusedConv2DOpTest,                                                   \
-         Conv2DWithBiasAddAndReluOneByOneConvolution##name) {                 \
-    const int filter_size = 1;                                                \
-    const int filter_count = 12;                                              \
-                                                                              \
-    VerifyConv2DWithBiasAndRelu<dtype>(kDepth, kImageWidth, kImageHeight,     \
-                                       kImageBatchCount, filter_size,         \
-                                       filter_count);                         \
-  }                                                                           \
-                                                                              \
-  TEST_F(FusedConv2DOpTest,                                                   \
-         Conv2DWithBiasAddAndReluImageSizeConvolution##name) {                \
-    const int filter_size = 32;                                               \
-    const int filter_count = 12;                                              \
-                                                                              \
-    VerifyConv2DWithBiasAndRelu<dtype>(kDepth, kImageWidth, kImageHeight,     \
-                                       kImageBatchCount, filter_size,         \
-                                       filter_count);                         \
-  }                                                                           \
-                                                                              \
-  TEST_F(FusedConv2DOpTest,                                                   \
-         Conv2DWithBiasAddAndReluSpatialConvolution##name) {                  \
-    const int filter_size = 3;                                                \
-    const int filter_count = 12;                                              \
-                                                                              \
-    VerifyConv2DWithBiasAndRelu<dtype>(kDepth, kImageWidth, kImageHeight,     \
-                                       kImageBatchCount, filter_size,         \
-                                       filter_count);                         \
+  // Verifies that computing Conv2D+FusedBatchNorm in a graph is identical to
+  // FusedConv2D.
+  void VerifyConv2DWithBatchNorm(int filter_size, int filter_count,
+                                 int depth = kDepth,
+                                 int image_width = kImageWidth,
+                                 int image_height = kImageHeight,
+                                 int image_batch_count = kImageBatchCount) {
+    const BatchNormGraphRunner run_default =
+        [this](const Tensor& input_data, const Tensor& filter_data,
+               const Tensor& scale_data, const Tensor& offset_data,
+               const Tensor& mean_data, const Tensor& variance_data,
+               Tensor* out) {
+          RunConv2DWithBatchNorm(input_data, filter_data, scale_data,
+                                 offset_data, mean_data, variance_data, out);
+        };
+
+    const BatchNormGraphRunner run_fused =
+        [this](const Tensor& input_data, const Tensor& filter_data,
+               const Tensor& scale_data, const Tensor& offset_data,
+               const Tensor& mean_data, const Tensor& variance_data,
+               Tensor* out) {
+          RunFusedConv2DOp(input_data, filter_data,
+                           {scale_data, offset_data, mean_data, variance_data},
+                           {"FusedBatchNorm"}, out);
+        };
+
+    VerifyFusedBatchNormTensorsNear(depth, image_width, image_height,
+                                    image_batch_count, filter_size,
+                                    filter_count, run_default, run_fused);
   }
 
-FUSED_CONV2D_TESTS(float, F);
-FUSED_CONV2D_TESTS(double, D);
+  // Verifies that computing Conv2D+FusedBatchNorm+Relu in a graph is identical
+  // to FusedConv2D.
+  void VerifyConv2DWithBatchNormAndRelu(
+      int filter_size, int filter_count, int depth = kDepth,
+      int image_width = kImageWidth, int image_height = kImageHeight,
+      int image_batch_count = kImageBatchCount) {
+    const BatchNormGraphRunner run_default =
+        [this](const Tensor& input_data, const Tensor& filter_data,
+               const Tensor& scale_data, const Tensor& offset_data,
+               const Tensor& mean_data, const Tensor& variance_data,
+               Tensor* out) {
+          RunConv2DWithBatchNormAndRelu(input_data, filter_data, scale_data,
+                                        offset_data, mean_data, variance_data,
+                                        out);
+        };
+
+    const BatchNormGraphRunner run_fused =
+        [this](const Tensor& input_data, const Tensor& filter_data,
+               const Tensor& scale_data, const Tensor& offset_data,
+               const Tensor& mean_data, const Tensor& variance_data,
+               Tensor* out) {
+          RunFusedConv2DOp(input_data, filter_data,
+                           {scale_data, offset_data, mean_data, variance_data},
+                           {"FusedBatchNorm", "Relu"}, out);
+        };
+
+    VerifyFusedBatchNormTensorsNear(depth, image_width, image_height,
+                                    image_batch_count, filter_size,
+                                    filter_count, run_default, run_fused);
+  }
+};
+
+// Conv2D with BatchNorm can be tested only with `T=float`, because default
+// `FusedBatchNorm` kernel supports only floats for scale, mean and variance.
+
+template <typename T>
+class FusedConv2DWithBiasOpTest : public FusedConv2DOpTest<T> {};
+template <typename T>
+class FusedConv2DWithBatchNormOpTest : public FusedConv2DOpTest<T> {};
+
+TYPED_TEST_CASE_P(FusedConv2DWithBiasOpTest);
+TYPED_TEST_CASE_P(FusedConv2DWithBatchNormOpTest);
+
+// -------------------------------------------------------------------------- //
+// Conv2D + BiasAdd + {Relu}                                                  //
+// -------------------------------------------------------------------------- //
+
+TYPED_TEST_P(FusedConv2DWithBiasOpTest, OneByOneConvolution) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBias(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBiasOpTest, ImageSizeConvolution) {
+  const int filter_size = TestFixture::kImageWidth;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBias(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBiasOpTest, SpatialConvolution) {
+  const int filter_size = 3;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBias(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBiasOpTest, OneByOneConvolutionAndRelu) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBiasAndRelu(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBiasOpTest, ImageSizeConvolutionAndRelu) {
+  const int filter_size = TestFixture::kImageWidth;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBiasAndRelu(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBiasOpTest, SpatialConvolutionAndRelu) {
+  const int filter_size = 3;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBiasAndRelu(filter_size, filter_count);
+}
 
-#undef FUSED_CONV2D_TESTS
+// -------------------------------------------------------------------------- //
+// Conv2D + FusedBatchNorm + {Relu}                                           //
+// -------------------------------------------------------------------------- //
+
+TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, OneByOneConvolution) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBatchNorm(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, ImageSizeConvolution) {
+  const int filter_size = TestFixture::kImageWidth;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBatchNorm(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, SpatialConvolution) {
+  const int filter_size = 3;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBatchNorm(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, OneByOneConvolutionAndRelu) {
+  const int filter_size = 1;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBatchNormAndRelu(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, ImageSizeConvolutionAndRelu) {
+  const int filter_size = TestFixture::kImageWidth;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBatchNormAndRelu(filter_size, filter_count);
+}
+
+TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, SpatialConvolutionAndRelu) {
+  const int filter_size = 3;
+  const int filter_count = 12;
+  this->VerifyConv2DWithBatchNormAndRelu(filter_size, filter_count);
+}
+
+REGISTER_TYPED_TEST_CASE_P(FusedConv2DWithBiasOpTest,    //
+                           OneByOneConvolution,          //
+                           ImageSizeConvolution,         //
+                           SpatialConvolution,           //
+                           OneByOneConvolutionAndRelu,   //
+                           ImageSizeConvolutionAndRelu,  //
+                           SpatialConvolutionAndRelu);
+
+REGISTER_TYPED_TEST_CASE_P(FusedConv2DWithBatchNormOpTest,  //
+                           OneByOneConvolution,             //
+                           ImageSizeConvolution,            //
+                           SpatialConvolution,              //
+                           OneByOneConvolutionAndRelu,      //
+                           ImageSizeConvolutionAndRelu,     //
+                           SpatialConvolutionAndRelu);
+
+using FusedBiasAddDataTypes = ::testing::Types<float, double>;
+INSTANTIATE_TYPED_TEST_CASE_P(Test, FusedConv2DWithBiasOpTest,
+                              FusedBiasAddDataTypes);
+
+using FusedBatchNormDataTypes = ::testing::Types<float>;
+INSTANTIATE_TYPED_TEST_CASE_P(Test, FusedConv2DWithBatchNormOpTest,
+                              FusedBatchNormDataTypes);
 
 ////////////////////////////////////////////////////////////////////////////////
 // Performance benchmarks for the FusedConv2DWithBiasOp.                      //
@@ -771,6 +1006,19 @@ struct Conv2DWithBiasAndReluGraph {
   Node* relu;
 };
 
+struct Conv2DWithBatchNormGraph {
+  Graph* graph;
+  Node* conv2d;
+  Node* batch_norm;
+};
+
+struct Conv2DWithBatchNormAndReluGraph {
+  Graph* graph;
+  Node* conv2d;
+  Node* batch_norm;
+  Node* relu;
+};
+
 static Tensor MakeRandomTensor(const TensorShape& shape) {
   Tensor tensor(DT_FLOAT, TensorShape(shape));
   tensor.flat<float>() = tensor.flat<float>().setRandom();
@@ -800,7 +1048,7 @@ static Conv2DGraph Conv2D(int batch, int height, int width, int in_depth,
   return {graph, conv2d};
 }
 
-// Creates a Tensorflow graph with a Conv2D node followed by Relu.
+// Creates a Tensorflow graph with a Conv2D node followed by BiasAdd.
 static Conv2DWithBiasGraph Conv2DWithBias(int batch, int height, int width,
                                           int in_depth, int filter_w,
                                           int filter_h, int out_depth) {
@@ -846,11 +1094,68 @@ static Conv2DWithBiasAndReluGraph Conv2DWithBiasAndRelu(int batch, int height,
   return {graph, conv2d, bias, relu};
 }
 
-// Creates a tensorflow graph with a single FusedConv2D node and fuses into it
-// additional computations (e.g. BiasAdd or Relu).
-static Graph* FusedConv2D(int batch, int height, int width, int in_depth,
-                          int filter_w, int filter_h, int out_depth,
-                          const std::vector<string>& fused_ops = {}) {
+// Creates a Tensorflow graph with a Conv2D node followed by FusedBatchNorm.
+static Conv2DWithBatchNormGraph Conv2DWithBatchNorm(int batch, int height,
+                                                    int width, int in_depth,
+                                                    int filter_w, int filter_h,
+                                                    int out_depth) {
+  Conv2DGraph conv_graph =
+      Conv2D(batch, height, width, in_depth, filter_w, filter_h, out_depth);
+
+  Graph* graph = conv_graph.graph;
+  Node* conv2d = conv_graph.conv2d;
+
+  Tensor scale_t = MakeRandomTensor({out_depth});
+  Tensor offset_t = MakeRandomTensor({out_depth});
+  Tensor mean_t = MakeRandomTensor({out_depth});
+  Tensor variance_t = MakeRandomTensor({out_depth});
+
+  Node* scale = test::graph::Constant(graph, scale_t, "scale");
+  Node* offset = test::graph::Constant(graph, offset_t, "offset");
+  Node* mean = test::graph::Constant(graph, mean_t, "mean");
+  Node* variance = test::graph::Constant(graph, variance_t, "variance");
+
+  Node* out;
+  TF_CHECK_OK(NodeBuilder(graph->NewName("batch_norm"), "FusedBatchNorm")
+                  .Input(conv2d)
+                  .Input(scale)
+                  .Input(offset)
+                  .Input(mean)
+                  .Input(variance)
+                  .Attr("T", DT_FLOAT)
+                  .Attr("is_training", false)
+                  .Finalize(graph, &out));
+
+  return {graph, conv2d, out};
+}
+
+// Creates a Tensorflow graph with a Conv2D node followed by FusedBatchNorm and
+// Relu.
+static Conv2DWithBatchNormAndReluGraph Conv2DWithBatchNormAndRelu(
+    int batch, int height, int width, int in_depth, int filter_w, int filter_h,
+    int out_depth) {
+  Conv2DWithBatchNormGraph conv_graph = Conv2DWithBatchNorm(
+      batch, height, width, in_depth, filter_w, filter_h, out_depth);
+
+  Graph* graph = conv_graph.graph;
+  Node* conv2d = conv_graph.conv2d;
+  Node* batch_norm = conv_graph.batch_norm;
+
+  Node* relu;
+  TF_CHECK_OK(NodeBuilder(graph->NewName("relu"), "Relu")
+                  .Input(batch_norm)
+                  .Attr("T", DT_FLOAT)
+                  .Finalize(graph, &relu));
+
+  return {graph, conv2d, batch_norm, relu};
+}
+
+// Creates a tensorflow graph with a single FusedConv2D (with BiasAdd) node and
+// fuses into it additional computations (e.g. Relu).
+static Graph* FusedConv2DWithBias(int batch, int height, int width,
+                                  int in_depth, int filter_w, int filter_h,
+                                  int out_depth,
+                                  const std::vector<string>& fused_ops = {}) {
   Graph* graph = new Graph(OpRegistry::Global());
 
   Tensor images_t = MakeRandomTensor({batch, height, width, in_depth});
@@ -878,6 +1183,53 @@ static Graph* FusedConv2D(int batch, int height, int width, int in_depth,
   return graph;
 }
 
+// Creates a tensorflow graph with a single FusedConv2D (with FusedBatchNorm)
+// node and fuses into it additional computations (e.g. Relu).
+static Graph* FusedConv2DWithBatchNorm(
+    int batch, int height, int width, int in_depth, int filter_w, int filter_h,
+    int out_depth, const std::vector<string>& fused_ops = {}) {
+  Graph* graph = new Graph(OpRegistry::Global());
+
+  Tensor images_t = MakeRandomTensor({batch, height, width, in_depth});
+  Tensor filter_t = MakeRandomTensor({filter_w, filter_h, in_depth, out_depth});
+  Tensor scale_t = MakeRandomTensor({out_depth});
+  Tensor offset_t = MakeRandomTensor({out_depth});
+  Tensor mean_t = MakeRandomTensor({out_depth});
+  Tensor variance_t = MakeRandomTensor({out_depth});
+
+  Node* images = test::graph::Constant(graph, images_t, "images");
+  Node* filter = test::graph::Constant(graph, filter_t, "filter");
+  Node* scale = test::graph::Constant(graph, scale_t, "scale");
+  Node* offset = test::graph::Constant(graph, offset_t, "offset");
+  Node* mean = test::graph::Constant(graph, mean_t, "mean");
+  Node* variance = test::graph::Constant(graph, variance_t, "variance");
+
+  std::vector<NodeBuilder::NodeOut> args = {scale, offset, mean, variance};
+
+  Node* conv;
+  TF_CHECK_OK(NodeBuilder(graph->NewName("conv"), "_FusedConv2D")
+                  .Input(images)
+                  .Input(filter)
+                  .Attr("num_args", 4)
+                  .Input(args)
+                  .Attr("T", DT_FLOAT)
+                  .Attr("strides", {1, 1, 1, 1})
+                  .Attr("padding", "SAME")
+                  .Attr("fused_ops", fused_ops)
+                  .Finalize(graph, &conv));
+
+  return graph;
+}
+
+// Macro arguments names: --------------------------------------------------- //
+//    N: batch size
+//    H: height
+//    W: width
+//    C: channels
+//   FC: filter count
+//   FH: filter height
+//   FW: filter width
+
 #define BM_SETUP(N, H, W, C, type, LABEL, NAME)                               \
   testing::ItemsProcessed(static_cast<int64>(iters) * (N) * (H) * (W) * (C)); \
   testing::SetLabel(LABEL);
@@ -911,26 +1263,73 @@ static Graph* FusedConv2D(int batch, int height, int width, int in_depth,
   }                                                                       \
   BENCHMARK(BM_NAME(BM_Conv2DWithBiasAndRelu, type, N, H, W, C, FW, FH, FC));
 
-#define BM_FusedConv2D(N, H, W, C, FW, FH, FC, type, LABEL)                  \
-  static void BM_NAME(BM_FusedConv2D, type, N, H, W, C, FW, FH,              \
+#define BM_FusedConv2DWithBias(N, H, W, C, FW, FH, FC, type, LABEL)           \
+  static void BM_NAME(BM_FusedConv2DWithBias, type, N, H, W, C, FW, FH,       \
+                      FC)(int iters) {                                        \
+    BM_SETUP(N, H, W, C, type, LABEL, Conv2D);                                \
+    test::Benchmark(#type,                                                    \
+                    FusedConv2DWithBias(N, H, W, C, FW, FH, FC, {"BiasAdd"})) \
+        .Run(iters);                                                          \
+  }                                                                           \
+  BENCHMARK(BM_NAME(BM_FusedConv2DWithBias, type, N, H, W, C, FW, FH, FC));
+
+#define BM_FusedConv2DWithBiasAndRelu(N, H, W, C, FW, FH, FC, type, LABEL)     \
+  static void BM_NAME(BM_FusedConv2DWithBiasAndRelu, type, N, H, W, C, FW, FH, \
+                      FC)(int iters) {                                         \
+    BM_SETUP(N, H, W, C, type, LABEL, Conv2D);                                 \
+    test::Benchmark(#type, FusedConv2DWithBias(N, H, W, C, FW, FH, FC,         \
+                                               {"BiasAdd", "Relu"}))           \
+        .Run(iters);                                                           \
+  }                                                                            \
+  BENCHMARK(                                                                   \
+      BM_NAME(BM_FusedConv2DWithBiasAndRelu, type, N, H, W, C, FW, FH, FC));
+
+#define BM_Conv2DWithBatchNorm(N, H, W, C, FW, FH, FC, type, LABEL)           \
+  static void BM_NAME(BM_Conv2DWithBatchNorm, type, N, H, W, C, FW, FH,       \
+                      FC)(int iters) {                                        \
+    BM_SETUP(N, H, W, C, type, LABEL, Conv2D);                                \
+    test::Benchmark(#type, Conv2DWithBatchNorm(N, H, W, C, FW, FH, FC).graph) \
+        .Run(iters);                                                          \
+  }                                                                           \
+  BENCHMARK(BM_NAME(BM_Conv2DWithBatchNorm, type, N, H, W, C, FW, FH, FC));
+
+#define BM_Conv2DWithBatchNormAndRelu(N, H, W, C, FW, FH, FC, type, LABEL)     \
+  static void BM_NAME(BM_Conv2DWithBatchNormAndRelu, type, N, H, W, C, FW, FH, \
+                      FC)(int iters) {                                         \
+    BM_SETUP(N, H, W, C, type, LABEL, Conv2D);                                 \
+    test::Benchmark(#type,                                                     \
+                    Conv2DWithBatchNormAndRelu(N, H, W, C, FW, FH, FC).graph)  \
+        .Run(iters);                                                           \
+  }                                                                            \
+  BENCHMARK(                                                                   \
+      BM_NAME(BM_Conv2DWithBatchNormAndRelu, type, N, H, W, C, FW, FH, FC));
+
+#define BM_FusedConv2DWithBatchNorm(N, H, W, C, FW, FH, FC, type, LABEL)     \
+  static void BM_NAME(BM_FusedConv2DWithBatchNorm, type, N, H, W, C, FW, FH, \
                       FC)(int iters) {                                       \
     BM_SETUP(N, H, W, C, type, LABEL, Conv2D);                               \
-    test::Benchmark(#type, FusedConv2D(N, H, W, C, FW, FH, FC, {"BiasAdd"})) \
+    test::Benchmark(#type, FusedConv2DWithBatchNorm(N, H, W, C, FW, FH, FC,  \
+                                                    {"FusedBatchNorm"}))     \
         .Run(iters);                                                         \
   }                                                                          \
-  BENCHMARK(BM_NAME(BM_FusedConv2D, type, N, H, W, C, FW, FH, FC));
+  BENCHMARK(BM_NAME(BM_FusedConv2DWithBatchNorm, type, N, H, W, C, FW, FH, FC));
 
-#define BM_FusedConv2DAndRelu(N, H, W, C, FW, FH, FC, type, LABEL)            \
-  static void BM_NAME(BM_FusedConv2DAndRelu, type, N, H, W, C, FW, FH,        \
-                      FC)(int iters) {                                        \
+#define BM_FusedConv2DWithBatchNormAndRelu(N, H, W, C, FW, FH, FC, type,      \
+                                           LABEL)                             \
+  static void BM_NAME(BM_FusedConv2DWithBatchNormAndRelu, type, N, H, W, C,   \
+                      FW, FH, FC)(int iters) {                                \
     BM_SETUP(N, H, W, C, type, LABEL, Conv2D);                                \
     test::Benchmark(#type,                                                    \
-                    FusedConv2D(N, H, W, C, FW, FH, FC, {"BiasAdd", "Relu"})) \
+                    FusedConv2DWithBatchNorm(N, H, W, C, FW, FH, FC,          \
+                                             {"FusedBatchNorm", "Relu"}))     \
         .Run(iters);                                                          \
   }                                                                           \
-  BENCHMARK(BM_NAME(BM_FusedConv2DAndRelu, type, N, H, W, C, FW, FH, FC));
+  BENCHMARK(BM_NAME(BM_FusedConv2DWithBatchNormAndRelu, type, N, H, W, C, FW, \
+                    FH, FC));
 
+// -------------------------------------------------------------------------- //
 // Pixel CNN convolutions.
+// -------------------------------------------------------------------------- //
 
 // 1x1 Convolution: MatMulFunctor
 
@@ -938,6 +1337,8 @@ BM_Conv2D(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
 BM_Conv2D(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
 BM_Conv2D(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
 
+// 1) BiasAdd {+ Relu}
+
 BM_Conv2DWithBias(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
 BM_Conv2DWithBias(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
 BM_Conv2DWithBias(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
@@ -946,20 +1347,44 @@ BM_Conv2DWithBiasAndRelu(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
 BM_Conv2DWithBiasAndRelu(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
 BM_Conv2DWithBiasAndRelu(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
 
-BM_FusedConv2D(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
-BM_FusedConv2D(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
-BM_FusedConv2D(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
+BM_FusedConv2DWithBias(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
+BM_FusedConv2DWithBias(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
+BM_FusedConv2DWithBias(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
+
+BM_FusedConv2DWithBiasAndRelu(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
+BM_FusedConv2DWithBiasAndRelu(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
+BM_FusedConv2DWithBiasAndRelu(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
+
+// 2) FusedBatchNorm {+ Relu}
+
+BM_Conv2DWithBatchNorm(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
+BM_Conv2DWithBatchNorm(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
+BM_Conv2DWithBatchNorm(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
+
+BM_Conv2DWithBatchNormAndRelu(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
+BM_Conv2DWithBatchNormAndRelu(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
+BM_Conv2DWithBatchNormAndRelu(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
+
+BM_FusedConv2DWithBatchNorm(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
+BM_FusedConv2DWithBatchNorm(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
+BM_FusedConv2DWithBatchNorm(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
 
-BM_FusedConv2DAndRelu(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
-BM_FusedConv2DAndRelu(16, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 16");
-BM_FusedConv2DAndRelu(32, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 32");
+BM_FusedConv2DWithBatchNormAndRelu(8, 32, 32, 128, 1, 1, 1024, cpu, "1x1 /b 8");
+BM_FusedConv2DWithBatchNormAndRelu(16, 32, 32, 128, 1, 1, 1024, cpu,
+                                   "1x1 /b 16");
+BM_FusedConv2DWithBatchNormAndRelu(32, 32, 32, 128, 1, 1, 1024, cpu,
+                                   "1x1 /b 32");
 
+// -------------------------------------------------------------------------- //
 // 3x3 Convolution: SpatialConvolution
+// -------------------------------------------------------------------------- //
 
 BM_Conv2D(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
 BM_Conv2D(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
 BM_Conv2D(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
 
+// 1) BiasAdd {+ Relu}
+
 BM_Conv2DWithBias(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
 BM_Conv2DWithBias(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
 BM_Conv2DWithBias(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
@@ -968,12 +1393,32 @@ BM_Conv2DWithBiasAndRelu(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
 BM_Conv2DWithBiasAndRelu(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
 BM_Conv2DWithBiasAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
 
-BM_FusedConv2D(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
-BM_FusedConv2D(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
-BM_FusedConv2D(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
+BM_FusedConv2DWithBias(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
+BM_FusedConv2DWithBias(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
+BM_FusedConv2DWithBias(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
+
+BM_FusedConv2DWithBiasAndRelu(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
+BM_FusedConv2DWithBiasAndRelu(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
+BM_FusedConv2DWithBiasAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
+
+// 2) FusedBatchNorm {+ Relu}
+
+BM_Conv2DWithBatchNorm(8, 32, 32, 128, 3, 3, 1024, cpu, "1x1 /b 8");
+BM_Conv2DWithBatchNorm(16, 32, 32, 128, 3, 3, 1024, cpu, "1x1 /b 16");
+BM_Conv2DWithBatchNorm(32, 32, 32, 128, 3, 3, 1024, cpu, "1x1 /b 32");
+
+BM_Conv2DWithBatchNormAndRelu(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
+BM_Conv2DWithBatchNormAndRelu(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
+BM_Conv2DWithBatchNormAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
+
+BM_FusedConv2DWithBatchNorm(8, 32, 32, 128, 3, 3, 1024, cpu, "1x1 /b 8");
+BM_FusedConv2DWithBatchNorm(16, 32, 32, 128, 3, 3, 1024, cpu, "1x1 /b 16");
+BM_FusedConv2DWithBatchNorm(32, 32, 32, 128, 3, 3, 1024, cpu, "1x1 /b 32");
 
-BM_FusedConv2DAndRelu(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
-BM_FusedConv2DAndRelu(16, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 16");
-BM_FusedConv2DAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 32");
+BM_FusedConv2DWithBatchNormAndRelu(8, 32, 32, 128, 3, 3, 1024, cpu, "3x3 /b 8");
+BM_FusedConv2DWithBatchNormAndRelu(16, 32, 32, 128, 3, 3, 1024, cpu,
+                                   "3x3 /b 16");
+BM_FusedConv2DWithBatchNormAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu,
+                                   "3x3 /b 32");
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 4dfd95b019..efa84d6c22 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -327,6 +327,9 @@ REGISTER_OP("_FusedConv2D")
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Attr("fused_ops: list(string) = []")
+    // Attributes for the FusedBatchNorm ------------------------------------ //
+    .Attr("epsilon: float = 0.0001")
+    // ---------------------------------------------------------------------- //
     .SetShapeFn(shape_inference::Conv2DShape)
     .Doc(R"doc(
 *NOTE*: Do not invoke this operator directly in Python. Grappler is
-- 
GitLab


From daa219172809b56c590c8b929a636e92d4a19ad1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 13:06:27 -0800
Subject: [PATCH 0234/1554] Put TPUEstimatorSpecs through some of the same
 validation checks as EstimatorSpecs.

PiperOrigin-RevId: 221496146
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 7cb8c4aa7f..932367f4dd 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -298,9 +298,9 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec):  # pylint: disable=prote
       host_calls['host_call'] = host_call
     _OutfeedHostCall.validate(host_calls)
 
-    training_hooks = list(training_hooks or [])
-    evaluation_hooks = list(evaluation_hooks or [])
-    prediction_hooks = list(prediction_hooks or [])
+    training_hooks = tuple(training_hooks or [])
+    evaluation_hooks = tuple(evaluation_hooks or [])
+    prediction_hooks = tuple(prediction_hooks or [])
 
     for hook in training_hooks + evaluation_hooks + prediction_hooks:
       if not isinstance(hook, session_run_hook.SessionRunHook):
@@ -335,7 +335,7 @@ class TPUEstimatorSpec(model_fn_lib._TPUEstimatorSpec):  # pylint: disable=prote
     hooks = None
     if self.host_call is not None:
       hooks = [_OutfeedHostCallHook(host_call_ret['host_call'])]
-    hooks = list(hooks or [])
+    hooks = tuple(hooks or [])
     scaffold = self.scaffold_fn() if self.scaffold_fn else None
     return model_fn_lib.EstimatorSpec(
         mode=self.mode,
-- 
GitLab


From 7edeed733eda7eb3e591afe95562097e72d3b173 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 13:11:08 -0800
Subject: [PATCH 0235/1554] This CL fixes a crash that happens when fitting in
 Keras without scipy sparse libraries being installed.

PiperOrigin-RevId: 221496911
---
 tensorflow/python/keras/engine/training_arrays.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 9f3f3c7c24..7c603f4e01 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -365,8 +365,9 @@ def model_iteration(model,
                           'pass shuffle="batch".')
 
         # Sparse to dense conversion.
-        for i in indices_for_conversion_to_dense:
-          ins_batch[i] = ins_batch[i].toarray()
+        if issparse is not None:
+          for i in indices_for_conversion_to_dense:
+            ins_batch[i] = ins_batch[i].toarray()
 
         # Callbacks batch_begin.
         batch_logs = {'batch': batch_index, 'size': len(batch_ids)}
-- 
GitLab


From 89fa3c5ed66a4e543ba01c31febf3edb417808d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 13:21:31 -0800
Subject: [PATCH 0236/1554] Updating Eigen to commit
 https://bitbucket.org/eigen/eigen/commits/af207140728002141ce3ff17f6f14cc159965f93.

PiperOrigin-RevId: 221498624
---
 tensorflow/workspace.bzl         |  9 +++---
 third_party/eigen_reshaped.patch | 48 --------------------------------
 2 files changed, 4 insertions(+), 53 deletions(-)
 delete mode 100644 third_party/eigen_reshaped.patch

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 29559927bb..0c894c9634 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,12 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        patch_file = clean_dep("//third_party:eigen_reshaped.patch"),
-        sha256 = "d66cec3b54b3dfaa4666c1d49481a7197f93fc078cd53c54e2b4a8893a529c9f",
-        strip_prefix = "eigen-eigen-b4890dc6bc34",
+        sha256 = "1e045bef75e9b17d459b60cc30b34408f3fdab300c5053d3919d1a5921f3c86a",
+        strip_prefix = "eigen-eigen-af2071407280",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
         ],
     )
 
diff --git a/third_party/eigen_reshaped.patch b/third_party/eigen_reshaped.patch
deleted file mode 100644
index 7acfdcf9fe..0000000000
--- a/third_party/eigen_reshaped.patch
+++ /dev/null
@@ -1,48 +0,0 @@
---- a/Eigen/src/Core/util/ReshapedHelper.h	(date 1541195478000)
-+++ b/Eigen/src/Core/util/ReshapedHelper.h	(date 1541195478000)
-@@ -39,6 +39,11 @@
-   return total/other;
- }
-
-+template<int Flags, int Order>
-+struct get_compiletime_reshape_order {
-+  enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order };
-+};
-+
- }
-
- } // end namespace Eigen
---- a/Eigen/src/plugins/ReshapedMethods.h	(date 1541195254000)
-+++ b/Eigen/src/plugins/ReshapedMethods.h	(date 1541195254000)
-@@ -105,13 +105,13 @@
- inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,
-                 internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value,
-                 internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value,
--                (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+                internal::get_compiletime_reshape_order<Flags,Order>::value>
- reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST
- {
-   return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,
-                   internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value,
-                   internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value,
--                  (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+                  internal::get_compiletime_reshape_order<Flags,Order>::value>
-                 (derived(),
-                  internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()),
-                  internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size()));
-@@ -128,11 +128,13 @@
-
- template<int Order>
- EIGEN_DEVICE_FUNC
--inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1,
-+                internal::get_compiletime_reshape_order<Flags,Order>::value>
- reshaped() EIGEN_RESHAPED_METHOD_CONST
- {
-   EIGEN_STATIC_ASSERT(Order==RowMajor || Order==ColMajor || Order==AutoOrder, INVALID_TEMPLATE_PARAMETER);
--  return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+  return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1,
-+                  internal::get_compiletime_reshape_order<Flags,Order>::value>
-                 (derived(), size(), 1);
- }
- 
\ No newline at end of file
-- 
GitLab


From dfd22d1cea128e48ebee9de5301ad50def739d25 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Wed, 14 Nov 2018 13:22:15 -0800
Subject: [PATCH 0237/1554] Remove unnecessary 'typedef' keywords.

These generate a compiler warning:
tensorflow/c/kernels.h:38:1: warning: typedef requires a name [-Wmissing-declarations]

Also change the dependency from :c_api_internal to :c_api, only the latter exports c_api.h

PiperOrigin-RevId: 221498747
---
 tensorflow/c/BUILD     | 4 ++--
 tensorflow/c/kernels.h | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 183faaefa9..f9fc718f68 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -186,11 +186,11 @@ tf_cuda_library(
     visibility = ["//visibility:public"],
     deps = select({
         "//tensorflow:android": [
-            ":c_api_internal",
+            ":c_api",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
-            ":c_api_internal",
+            ":c_api",
             "//tensorflow/core:framework",
         ],
     }),
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index db51b2d535..2518789a3c 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -35,9 +35,9 @@ extern "C" {
 // `TF_RegisterKernelBuilder`, which will allow TF to construct user-provided
 // kernels when necessary.
 
-typedef struct TF_KernelBuilder;
-typedef struct TF_OpKernelConstruction;
-typedef struct TF_OpKernelContext;
+struct TF_KernelBuilder;
+struct TF_OpKernelConstruction;
+struct TF_OpKernelContext;
 
 // Allocates a new kernel builder and returns a pointer to it.
 //
-- 
GitLab


From e03aaa7e07353b5c165995c705ff922a5c2000eb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 13:24:46 -0800
Subject: [PATCH 0238/1554] Add tile and gather tests

PiperOrigin-RevId: 221499197
---
 tensorflow/lite/testing/generate_examples.py | 30 +++++++++++++++-----
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 2b129df766..1d52fda4af 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -370,7 +370,8 @@ def make_zip_of_tests(zip_path,
                       make_graph,
                       make_test_inputs,
                       extra_toco_options=ExtraTocoOptions(),
-                      use_frozen_graph=False):
+                      use_frozen_graph=False,
+                      expected_tf_success=None):
   """Helper to make a zip file of a bunch of TensorFlow models.
 
   This does a cartestian product of the dictionary of test_parameters and
@@ -390,6 +391,8 @@ def make_zip_of_tests(zip_path,
       `output_tensors` and returns tuple `(input_values, output_values)`.
     extra_toco_options: Additional toco options.
     use_frozen_graph: Whether or not freeze graph before toco converter.
+    expected_tf_success: Number of times tensorflow is supposed to succeed in
+      executing the input graphs. `None` means "unknown".
 
   Raises:
     RuntimeError: if there are toco errors that can't be ignored.
@@ -550,6 +553,11 @@ def make_zip_of_tests(zip_path,
                    " and %d TOCO converted graphs (%.1f%%"), zip_path,
                   total_conversions, tf_success, toco_success, percent)
 
+  if expected_tf_success is not None and tf_success != expected_tf_success:
+    raise RuntimeError(
+        "Expected TF to succeed %d times, but that happened %d times" %
+        (expected_tf_success, tf_success))
+
   if not FLAGS.ignore_toco_errors and toco_errors > 0:
     raise RuntimeError(
         "Found %d errors while generating toco models" % toco_errors)
@@ -1142,9 +1150,9 @@ def make_gather_tests(zip_path):
       # TODO(mgubin): add string tests when they are supported by Toco.
       # TODO(mgubin): add tests for Nd indices when they are supported by
       # TfLite.
-      "params_dtype": [tf.float32, tf.int32],
+      "params_dtype": [tf.float32, tf.int32, tf.int64],
       "params_shape": [[10], [1, 2, 20]],
-      "indices_dtype": [tf.int32],
+      "indices_dtype": [tf.int32, tf.int64],
       "indices_shape": [[3], [5]],
       "axis": [-1, 0, 1],
   }]
@@ -1172,7 +1180,13 @@ def make_gather_tests(zip_path):
     return [params, indices], sess.run(
         outputs, feed_dict=dict(zip(inputs, [params, indices])))
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  # Note that TF can't execute with index=1 and params_shape=[10].
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_success=60)
 
 
 def make_global_batch_norm_tests(zip_path):
@@ -3121,7 +3135,7 @@ def make_transpose_conv_tests(zip_path):
 def make_tile_tests(zip_path):
   """Make a set of tests to do tile."""
   test_parameters = [{
-      "input_dtype": [tf.float32, tf.int32],
+      "input_dtype": [tf.float32, tf.int32, tf.bool],
       "input_shape": [[3, 2, 1], [2, 2, 2]],
       "multiplier_dtype": [tf.int32, tf.int64],
       "multiplier_shape": [[3]]
@@ -3143,8 +3157,10 @@ def make_tile_tests(zip_path):
   def build_inputs(parameters, sess, inputs, outputs):
     input_value = create_tensor_data(parameters["input_dtype"],
                                      parameters["input_shape"])
-    multipliers_value = create_tensor_data(parameters["multiplier_dtype"],
-                                           parameters["multiplier_shape"])
+    multipliers_value = create_tensor_data(
+        parameters["multiplier_dtype"],
+        parameters["multiplier_shape"],
+        min_value=0)
     return [input_value, multipliers_value], sess.run(
         outputs,
         feed_dict={
-- 
GitLab


From 1a4e13725f04d8b27ed8f6c5feeed5ea7b2e54e9 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 14 Nov 2018 13:37:09 -0800
Subject: [PATCH 0239/1554] Enable V2 Optimizer With Keras model fit.

PiperOrigin-RevId: 221501649
---
 tensorflow/contrib/distribute/python/BUILD    | 12 +--
 .../python/keras_optimizer_v2_test.py         | 37 +++++++-
 tensorflow/python/keras/BUILD                 |  1 +
 tensorflow/python/keras/optimizer_v2/BUILD    |  1 -
 .../python/keras/optimizer_v2/optimizer_v2.py | 28 ++++--
 .../keras/optimizer_v2/optimizer_v2_test.py   | 89 ++++++++++++++++++-
 tensorflow/python/keras/optimizers.py         |  9 +-
 7 files changed, 147 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 24bcb98e09..119ed45625 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -480,17 +480,7 @@ cuda_py_test(
     name = "keras_optimizer_v2_test",
     srcs = ["keras_optimizer_v2_test.py"],
     additional_deps = [
-        ":combinations",
-        "@absl_py//absl/testing:parameterized",
-        "//third_party/py/numpy",
-        "//tensorflow/contrib/optimizer_v2:training",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/eager:test",
-        "//tensorflow/python/estimator:estimator_py",
-        "//tensorflow/python/feature_column",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:summary",
+        ":keras_test_lib",
     ],
     tags = [
         "multi_and_single_gpu",
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index e4e7717f8d..28f76ffadf 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -27,6 +27,7 @@ import six
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
@@ -39,6 +40,7 @@ from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.keras.optimizer_v2 import gradient_descent
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
@@ -140,6 +142,13 @@ class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase):
       shutil.rmtree(self._model_dir)
 
 
+def get_model():
+  x = keras.layers.Input(shape=(3,), name='input')
+  y = keras.layers.Dense(4, name='dense')(x)
+  model = keras.Model(x, y)
+  return model
+
+
 class MirroredStrategyOptimizerV2Test(test.TestCase):
 
   def testKerasOptimizerWithUnequalInput(self):
@@ -155,7 +164,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       train_op = optimizer.minimize(loss, var_list=[var])
       m = optimizer.get_slot(var, 'm')
       v = optimizer.get_slot(var, 'v')
-      return (var, m, v, train_op, optimizer.iteration)
+      return (var, m, v, train_op, optimizer.iterations)
 
     devices = ['/device:GPU:0', '/device:CPU:0']
     dist = mirrored_strategy.MirroredStrategy(devices)
@@ -235,6 +244,32 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
                               counter.get(devices[1])
                           ]))
 
+  def testOptimizerWithKerasModelAndNumpyArrays(self):
+    if context.num_gpus() < 1:
+      self.skipTest('Not enough GPUs.')
+
+    with self.cached_session():
+      model = get_model()
+      optimizer = gradient_descent.SGD(0.001)
+      loss = 'mse'
+      metrics = ['mae']
+      devices = ['/device:GPU:0', '/device:CPU:0']
+      dist = mirrored_strategy.MirroredStrategy(devices)
+      model.compile(optimizer, loss, metrics=metrics, distribute=dist)
+
+      inputs = np.zeros((64, 3), dtype=np.float32)
+      targets = np.zeros((64, 4), dtype=np.float32)
+
+      model.fit(
+          inputs,
+          targets,
+          epochs=1,
+          batch_size=2,
+          verbose=0,
+          validation_data=(inputs, targets))
+      model.evaluate(inputs, targets)
+      model.predict(inputs)
+
 
 def _replica_id():
   # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 9f85fe5dd7..db78eff86e 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -146,6 +146,7 @@ py_library(
     deps = [
         ":backend",
         "//tensorflow/python/data",
+        "//tensorflow/python/keras/optimizer_v2",
         "//tensorflow/python/training/checkpointable:data_structures",
         "//tensorflow/tools/docs:doc_controls",
         "@six_archive//:six",
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index fb43775fdc..6a973058f7 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -24,7 +24,6 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:state_ops",
-        "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
     ],
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index c6e1d57c5e..d15665dac8 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -33,7 +33,7 @@ from tensorflow.python.keras import initializers
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.ops import gradients
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops import variables
+from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer as optimizer_v1
@@ -327,9 +327,12 @@ class OptimizerV2(optimizer_v1.Optimizer):
       # this once b/118841692 is fixed.
       # with ops.control_dependencies(update_ops):
       #   apply_updates = self._iterations.assign_add(1).op
-      apply_updates = merge_update_step(update_ops, self.iteration)
+      apply_updates = merge_update_step(update_ops, self.iterations)
       return apply_updates
 
+  def get_updates(self, loss, params):
+    return [self.minimize(loss, params)]
+
   def _set_hyper(self, name, value):
     """set hyper `name` to value. value can be callable, tensor, numeric."""
     if name not in self._hyper:
@@ -393,7 +396,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
           "iter",
           shape=[],
           trainable=False,
-          aggregation=variables.VariableAggregation.ONLY_FIRST_REPLICA)
+          aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA)
+      self._weights.append(self._iterations)
     for name, value in self._hyper.items():
       if isinstance(value, ops.Tensor) or callable(value):
         pass
@@ -403,11 +407,12 @@ class OptimizerV2(optimizer_v1.Optimizer):
             shape=[],
             trainable=False,
             initializer=value,
-            aggregation=variables.VariableAggregation.ONLY_FIRST_REPLICA)
+            aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA)
+        self._weights.append(self._hyper[name])
     self._prepared = True
 
   @property
-  def iteration(self):
+  def iterations(self):
     if not self._prepared:
       self._prepare()
     return self._iterations
@@ -450,10 +455,14 @@ class OptimizerV2(optimizer_v1.Optimizer):
     value = self._get_hyper(hyperparameter_name)
     if callable(value):
       return value()
-    if isinstance(value, (ops.Tensor, variables.Variable)):
+    if isinstance(value, (ops.Tensor, tf_variables.Variable)):
       return backend.get_value(value)
     return value
 
+  def variables(self):
+    """Returns variables of this Optimizer based on the order created."""
+    return self._weights
+
   @property
   def weights(self):
     """Returns variables of this Optimizer based on the order created."""
@@ -490,15 +499,15 @@ class OptimizerV2(optimizer_v1.Optimizer):
                  dtype=None,
                  initializer="zeros",
                  trainable=None,
-                 synchronization=variables.VariableSynchronization.AUTO,
-                 aggregation=variables.VariableAggregation.NONE):
+                 synchronization=tf_variables.VariableSynchronization.AUTO,
+                 aggregation=tf_variables.VariableAggregation.NONE):
 
     if dtype is None:
       dtype = dtypes.float32
     if isinstance(initializer, six.string_types) or callable(initializer):
       initializer = initializers.get(initializer)
 
-    if synchronization == variables.VariableSynchronization.ON_READ:
+    if synchronization == tf_variables.VariableSynchronization.ON_READ:
       if trainable:
         raise ValueError(
             "Synchronization value can be set to "
@@ -522,6 +531,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
         use_resource=True,
         synchronization=synchronization,
         aggregation=aggregation)
+    backend.track_variable(variable)
 
     return variable
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 682deda23f..e43327ff12 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
@@ -25,6 +27,12 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import backend
+from tensorflow.python.keras import callbacks
+from tensorflow.python.keras.engine import input_layer
+from tensorflow.python.keras.engine import sequential
+from tensorflow.python.keras.engine import training
+from tensorflow.python.keras.layers import core
 from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
 from tensorflow.python.ops import array_ops
@@ -279,8 +287,8 @@ class OptimizerTest(test.TestCase):
   def testIterationWithoutMinimize(self):
     with self.cached_session():
       sgd = gradient_descent.SGD(3.0)
-      self.evaluate(sgd.iteration.initializer)
-      self.assertEqual(0, self.evaluate(sgd.iteration))
+      self.evaluate(sgd.iterations.initializer)
+      self.assertEqual(0, self.evaluate(sgd.iterations))
 
   @test_util.run_in_graph_and_eager_modes
   def testSerializationWithinDefun(self):
@@ -341,8 +349,8 @@ class OptimizerTest(test.TestCase):
       opt2.set_weights(weights)
       self.evaluate([opt_op_1, opt_op_2])
       self.assertAllClose(self.evaluate(var1), self.evaluate(var2))
-      self.assertEqual(1, self.evaluate(opt1.iteration))
-      self.assertEqual(1, self.evaluate(opt2.iteration))
+      self.assertEqual(1, self.evaluate(opt1.iterations))
+      self.assertEqual(1, self.evaluate(opt2.iterations))
 
       var3 = resource_variable_ops.ResourceVariable([1.0, 2.0, 3.0],
                                                     dtype=dtypes.float32)
@@ -411,6 +419,79 @@ class OptimizerTest(test.TestCase):
       # when adam updates beta_1_power.
       self.assertAllClose([-1.343838, -0.343838], fn())
 
+  @test_util.run_in_graph_and_eager_modes
+  def testOptimizerWithKerasModel(self):
+    a = input_layer.Input(shape=(3,), name='input_a')
+    b = input_layer.Input(shape=(3,), name='input_b')
+
+    dense = core.Dense(4, name='dense')
+    c = dense(a)
+    d = dense(b)
+    e = core.Dropout(0.5, name='dropout')(c)
+
+    model = training.Model([a, b], [d, e])
+
+    optimizer = gradient_descent.SGD(learning_rate=0.001)
+    loss = 'mse'
+    model.compile(optimizer, loss, metrics=['mae'])
+
+    input_a_np = np.random.random((10, 3))
+    input_b_np = np.random.random((10, 3))
+
+    output_d_np = np.random.random((10, 4))
+    output_e_np = np.random.random((10, 4))
+
+    model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
+              epochs=1,
+              batch_size=5)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testOptimizerWithCallbacks(self):
+    input_np = np.random.random((10, 3))
+    output_np = np.random.random((10, 4))
+    a = input_layer.Input(shape=(3,), name='input_a')
+    model = sequential.Sequential()
+    model.add(core.Dense(4, name='dense'))
+    model.add(core.Dropout(0.5, name='dropout'))
+    model(a)
+    optimizer = gradient_descent.SGD(learning_rate=0.1)
+    model.compile(optimizer, loss='mse', metrics=['mae'])
+    # This does not reduce the LR after the first epoch (due to low delta).
+    cbks = [
+        callbacks.ReduceLROnPlateau(
+            monitor='val_loss', factor=0.1, min_delta=0, patience=1, cooldown=5)
+    ]
+    model.fit(
+        input_np,
+        output_np,
+        batch_size=10,
+        validation_data=(input_np, output_np),
+        callbacks=cbks,
+        epochs=5,
+        verbose=0)
+    self.assertAllClose(
+        float(backend.get_value(model.optimizer.lr)), 0.1, atol=1e-4)
+
+    # This should reduce the LR after the first epoch (due to high delta).
+    cbks = [
+        callbacks.ReduceLROnPlateau(
+            monitor='val_loss',
+            factor=0.1,
+            min_delta=10,
+            patience=1,
+            cooldown=5)
+    ]
+    model.fit(
+        input_np,
+        output_np,
+        batch_size=10,
+        validation_data=(input_np, output_np),
+        callbacks=cbks,
+        epochs=5,
+        verbose=2)
+    self.assertAllClose(
+        float(backend.get_value(model.optimizer.lr)), 0.01, atol=1e-4)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 715d80a116..09dd708b93 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -23,6 +23,7 @@ import six
 from six.moves import zip  # pylint: disable=redefined-builtin
 
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import clip_ops
@@ -833,17 +834,17 @@ def get(identifier):
   Raises:
       ValueError: If `identifier` cannot be interpreted.
   """
+  if isinstance(identifier, (Optimizer, optimizer_v2.OptimizerV2)):
+    return identifier
   # Wrap TF optimizer instances
-  if isinstance(identifier, tf_optimizer_module.Optimizer):
+  elif isinstance(identifier, tf_optimizer_module.Optimizer):
     opt = TFOptimizer(identifier)
     K.track_tf_optimizer(opt)
     return opt
-  if isinstance(identifier, dict):
+  elif isinstance(identifier, dict):
     return deserialize(identifier)
   elif isinstance(identifier, six.string_types):
     config = {'class_name': str(identifier), 'config': {}}
     return deserialize(config)
-  if isinstance(identifier, Optimizer):
-    return identifier
   else:
     raise ValueError('Could not interpret optimizer identifier:', identifier)
-- 
GitLab


From 5c8a7cb381c023b5c4225c3475bd656f1b415979 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 14 Nov 2018 13:39:21 -0800
Subject: [PATCH 0240/1554] Revamp cudnn_rnn ops level unittests

* Use absl parameterized.named_tuple decorator
* Test training by comparing outputs and grads of cudnn and tf rnn, instead of comparing cudnn's symbolic and numerical gradients. Much more comfortable tolerance levels.
* Split different rnn type's tests, much easier to read. (though paying a little more cost in coding.)

PiperOrigin-RevId: 221502090
---
 tensorflow/contrib/cudnn_rnn/BUILD            |    6 +-
 .../python/kernel_tests/cudnn_rnn_ops_test.py | 1627 ++++++++++-------
 2 files changed, 958 insertions(+), 675 deletions(-)

diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD
index cec8dc2bed..8d35622e39 100644
--- a/tensorflow/contrib/cudnn_rnn/BUILD
+++ b/tensorflow/contrib/cudnn_rnn/BUILD
@@ -42,10 +42,11 @@ tf_custom_op_py_library(
 
 cuda_py_test(
     name = "cudnn_rnn_ops_test",
-    size = "large",
+    size = "medium",
     srcs = ["python/kernel_tests/cudnn_rnn_ops_test.py"],
     additional_deps = [
         ":cudnn_rnn_py",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/contrib/rnn:rnn_py",
         "//tensorflow/python/ops/losses:losses",
@@ -61,9 +62,8 @@ cuda_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
-    shard_count = 6,
+    shard_count = 2,
     tags = [
-        "no_oss",  # http://b/119506830
         "noasan",  # http://b/62067814
         "requires-gpu-sm35",
     ],
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py
index ae839108eb..1e2c9121d6 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py
@@ -18,24 +18,30 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import itertools
 import os
 import unittest
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops
 from tensorflow.core.protobuf import saver_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework.test_util import TensorFlowTestCase
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
-from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import rnn
+from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
 from tensorflow.python.platform import test
@@ -56,714 +62,991 @@ CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER
 CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER
 
 
-def _CreateModel(rnn_mode,
-                 num_layers,
-                 num_units,
-                 input_size,
-                 input_mode="linear_input",
-                 direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION,
-                 dtype=dtypes.float32,
-                 dropout=0.):
-  del input_mode
-  if rnn_mode == cudnn_rnn_ops.CUDNN_LSTM:
-    model_fn = cudnn_rnn_ops.CudnnLSTM
-  elif rnn_mode == cudnn_rnn_ops.CUDNN_GRU:
-    model_fn = cudnn_rnn_ops.CudnnGRU
-  elif rnn_mode == cudnn_rnn_ops.CUDNN_RNN_TANH:
-    model_fn = cudnn_rnn_ops.CudnnRNNTanh
-  elif rnn_mode == cudnn_rnn_ops.CUDNN_RNN_RELU:
-    model_fn = cudnn_rnn_ops.CudnnRNNRelu
+def RunLSTM(sess,
+            num_units,
+            input_size,
+            batch_size,
+            time,
+            num_layers=1,
+            is_training=True,
+            dropout=0.,
+            num_dirs=True,
+            dtype=dtypes.float32):
+  # TODO(jamesqin): add multi-layer tests.
+  # TODO(jamesqin): add multi-dir tests
+  assert num_layers == 1
+  assert num_dirs == 1
+  if is_training and not np.isclose(dropout, 0):
+    raise ValueError("dropout can not be 0. when test training.")
+
+  # set graph level random seed and numpy random seed.
+  random_seed.set_random_seed(0)
+  np.random.seed(0)
+
+  inputs = variable_scope.get_variable(
+      "inputs",
+      initializer=np.random.rand(time, batch_size,
+                                 input_size).astype(dtype.as_numpy_dtype),
+      dtype=dtype)
+  initial_h_op = variable_scope.get_variable(
+      "initial_h_op",
+      initializer=np.random.rand(batch_size,
+                                 num_units).astype(dtype.as_numpy_dtype),
+      dtype=dtype)
+  initial_c_op = variable_scope.get_variable(
+      "initial_c_op",
+      initializer=np.random.rand(batch_size,
+                                 num_units).astype(dtype.as_numpy_dtype),
+      dtype=dtype)
+
+  initializer = init_ops.random_uniform_initializer(
+      -0.01, 0.01, dtype=dtype, seed=19980904)
+
+  with variable_scope.variable_scope("test", initializer=initializer):
+    w = variable_scope.get_variable(
+        "rnn/lstm_cell/kernel",
+        shape=[input_size + num_units, num_units * 4],
+        dtype=dtype)
+    b = variable_scope.get_variable(
+        "rnn/lstm_cell/bias", shape=[num_units * 4], dtype=dtype)
+
+    # canonical lstm. must set forget_bias to 0. to align with cudnn lstm.
+    cell = rnn_cell_impl.LSTMCell(num_units, forget_bias=0., reuse=True)
+    outputs_op, state_tuple_op = rnn.dynamic_rnn(
+        cell,
+        inputs,
+        initial_state=rnn_cell_impl.LSTMStateTuple(
+            h=initial_h_op, c=initial_c_op),
+        dtype=dtype,
+        time_major=True,
+        scope=None)
+
+  # Convert to cudnn opaque param.
+  format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterLSTM(
+      num_layers, num_units, input_size)
+  opaque_params = format_converter.tf_canonical_to_opaque([w, b])
+
+  cu_initial_h_op = array_ops.expand_dims(initial_h_op, axis=0)
+  cu_initial_c_op = array_ops.expand_dims(initial_c_op, axis=0)
+  cu_outputs_op, cu_h_op, cu_c_op = cudnn_rnn_ops._cudnn_rnn(
+      inputs,
+      cu_initial_h_op,
+      cu_initial_c_op,
+      opaque_params,
+      dropout=dropout,
+      is_training=is_training,
+      rnn_mode=cudnn_rnn_ops.CUDNN_LSTM)
+  # Remove the trivial 1st dimension.
+  cu_state_tuple_op = rnn_cell_impl.LSTMStateTuple(
+      c=array_ops.squeeze(cu_c_op, axis=0),
+      h=array_ops.squeeze(cu_h_op, axis=0))
+
+  if is_training:
+    (inp_grad_op, hgrad_op,
+     cgrad_op, wgrad_op, bgrad_op) = gradients_impl.gradients(
+         outputs_op, [inputs, initial_h_op, initial_c_op, w, b])
+
+    (cu_inp_grad_op, cu_hgrad_op,
+     cu_cgrad_op, opaque_grad_op) = gradients_impl.gradients(
+         cu_outputs_op,
+         [inputs, cu_initial_h_op, cu_initial_c_op, opaque_params])
+    # Remove the trivial 1st dimension
+    cu_hgrad_op = array_ops.squeeze(cu_hgrad_op, axis=0)
+    # Remove the trivial 1st dimension
+    cu_cgrad_op = array_ops.squeeze(cu_cgrad_op, axis=0)
+
+    cu_wgrad_op, cu_bgrad_op = format_converter.opaque_to_tf_canonical(
+        opaque_grad_op)
+    cu_wgrad_op = cu_wgrad_op[0]
+    cu_bgrad_op = cu_bgrad_op[0]
+    # cudnn lstm has 2 biases each gate. When converting to tf canonical format,
+    # the two biases are summed into one. Thus here bias gradient should be
+    # halved when comparing with tf lstm.
+    cu_bgrad_op *= 0.5
+
+  init_op = variables.global_variables_initializer()
+  sess.run(init_op)
+
+  if is_training:
+    outputs, state_tuple, inp_grad, state_grad, wgrad, bgrad = sess.run([
+        outputs_op, state_tuple_op, inp_grad_op,
+        (hgrad_op, cgrad_op), wgrad_op, bgrad_op
+    ])
+    (cu_outputs, cu_state_tuple, cu_inp_grad, cu_state_grad, cu_wgrad,
+     cu_bgrad) = sess.run([
+         cu_outputs_op, cu_state_tuple_op, cu_inp_grad_op,
+         (cu_hgrad_op, cu_cgrad_op), cu_wgrad_op, cu_bgrad_op
+     ])
+
+    logging.vlog(1, "outputs: %s" % outputs)
+    logging.vlog(1, "cu_outputs: %s" % cu_outputs)
+    logging.vlog(1, "state_tuple: %s" % str(state_tuple))
+    logging.vlog(1, "cu_state_tuple: %s" % str(cu_state_tuple))
+    logging.vlog(1, "inp_grad: %s" % inp_grad)
+    logging.vlog(1, "cu_inp_grad: %s" % cu_inp_grad)
+    logging.vlog(1, "state_grad: %s" % str(state_grad))
+    logging.vlog(1, "cu_state_grad: %s" % str(cu_state_grad))
+    logging.vlog(1, "wgrad: %s" % str(wgrad))
+    logging.vlog(1, "bgrad: %s" % str(bgrad))
+    logging.vlog(1, "cu_wgrad: %s" % str(cu_wgrad))
+    logging.vlog(1, "cu_bgrad: %s" % str(cu_bgrad))
+    return (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad,
+            cu_inp_grad, state_grad, cu_state_grad, wgrad, bgrad, cu_wgrad,
+            cu_bgrad)
   else:
-    raise ValueError("Invalid rnn_mode: %s" % rnn_mode)
-  return model_fn(
-      num_layers,
-      num_units,
-      input_size,
-      direction=direction,
-      dtype=dtype,
-      dropout=dropout)
-
-
-def _CreateParamsSavable(params,
-                         model,
-                         base_variable_scope=None,
-                         name="params_canonical"):
-  """Create a RNNParamsSaveable for the weight and bias parameters.
+    outputs, state_tuple = sess.run([outputs_op, state_tuple_op])
+    cu_outputs, cu_state_tuple = sess.run([cu_outputs_op, cu_state_tuple_op])
+
+    logging.vlog(1, "outputs: %s" % outputs)
+    logging.vlog(1, "cu_outputs: %s" % cu_outputs)
+    logging.vlog(1, "state_tuple: %s" % str(state_tuple))
+    logging.vlog(1, "cu_state_tuple: %s" % str(cu_state_tuple))
+  return outputs, cu_outputs, state_tuple, cu_state_tuple
+
+
+# Basic set of RNN configs to test. They can be further extended in relevant
+# test (e.g. adding num_dirs).
+NAMED_RNN_TESTCASES = ({
+    "testcase_name": "xsmall",
+    "num_units": 1,
+    "input_size": 1,
+    "batch_size": 1,
+    "time": 1,
+    "num_layers": 1,
+}, {
+    "testcase_name": "small",
+    "num_units": 4,
+    "input_size": 4,
+    "batch_size": 4,
+    "time": 4,
+    "num_layers": 1,
+}, {
+    "testcase_name": "medium",
+    "num_units": 128,
+    "input_size": 64,
+    "batch_size": 8,
+    "time": 16,
+    "num_layers": 1,
+}, {
+    "testcase_name": "large",
+    "num_units": 128,
+    "input_size": 128,
+    "batch_size": 16,
+    "time": 32,
+    "num_layers": 1,
+})
+
+
+def ExpandNamedTestCases(inputs, *remove_keys, **extra_configs):
+  """Expands testcase with new config dimensions.
+
+  Example:
+    inputs = (
+      {'testcase_name': 'test1', 'gender': 'male'}
+      {'testcase_name': 'test2', 'gender': 'female'}
+    )
+    remove_keys:  empty
+    extra_configs = {
+      'age': [40, 80]
+      'height': [5, 6]
+    }
+
+    Returns:
+      (
+        {'testcase_name': 'test1_age_40_height_5','gender': 'male', 'age':
+        40,'height': 5}
+        {'testcase_name': 'test1_age_40_height_6', 'gender': 'male', 'age': 40,
+        'height': 6}
+        {'testcase_name': 'test1_age_80_height_5', 'gender': 'male', 'age': 80,
+        'height': 5}
+        {'testcase_name': 'test1_age_80_height_6', 'gender': 'male', 'age': 80,
+        'height': 6}
+
+        {'testcase_name': 'test2_age_40_height_5', 'gender': 'female', 'age':
+        40,
+        'height': 5}
+        {'testcase_name': 'test2_age_40_height_6', 'gender': 'female', 'age':
+        40,
+        'height': 6}
+        {'testcase_name': 'test2_age_80_height_5', 'gender': 'female', 'age':
+        80,
+        'height': 5}
+        {'testcase_name': 'test2_age_80_height_6', 'gender': 'female', 'age':
+        80,
+        'height': 6}
+      )
 
   Args:
-    params: a Variable for weight and bias parameters.
-    model: a CudnnRNN model.
-    base_variable_scope: a string, prefix of names of saved variables.
-    name: a string, name of the RNNParamsSaveable object.
+    inputs: A list of dictionary, each being a testcase.
+    *remove_keys: A list of keys into testcase which are not needed in new
+      testcases.
+    **extra_configs: A dict of new test dimension and applicable values in that
+      dimension.
+
   Returns:
-    a RNNParamsSaveable object.
+    A list of dictionary with expanded test cases.
   """
-  if model._rnn_mode == CUDNN_LSTM:
-    fn = cudnn_rnn_ops.CudnnLSTMSaveable
-  elif model._rnn_mode == CUDNN_GRU:
-    fn = cudnn_rnn_ops.CudnnGRUSaveable
-  elif model._rnn_mode == CUDNN_RNN_TANH:
-    fn = cudnn_rnn_ops.CudnnRNNTanhSaveable
-  elif model._rnn_mode == CUDNN_RNN_RELU:
-    fn = cudnn_rnn_ops.CudnnRNNReluSaveable
-  params_saveable = fn(
-      params,
-      model.num_layers,
-      model.num_units,
-      model.input_size,
-      model.input_mode,
-      model.direction,
-      scope=base_variable_scope,
-      name=name)
-  ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, params_saveable)
-  return params_saveable
-
-
-def _MinLSTMParamSize(num_layers,
-                      num_units,
-                      input_size,
-                      direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION):
-  if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION:
-    first_layer_weights = 4 * num_units * (num_units + input_size)
-    higher_layer_weights = 8 * (num_layers - 1) * num_units * num_units
-    all_biases = 8 * num_layers * num_units
-    return first_layer_weights + higher_layer_weights + all_biases
-  elif direction == cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION:
-    first_layer_weights = 4 * num_units * (num_units + input_size)
-    higher_layer_weights = (num_layers - 1) * (
-        4 * 2 * num_units * num_units + 4 * num_units**2)
-    all_biases = 8 * num_layers * num_units
-    return 2 * (first_layer_weights + higher_layer_weights + all_biases)
-  else:
-    raise ValueError("%s direction is not supported.")
+  res = []
+  ordered_extra_configs = collections.OrderedDict(extra_configs)
+  keys = ordered_extra_configs.keys()
+  # A list of list of configs.
+  # The outer loop is iterating keys, the innner is values of one key.
+  combined_kv = [[(k, v) for v in ordered_extra_configs[k]] for k in keys]
+  logging.info("combined_kv: %s", combined_kv)
 
+  for inp in inputs:
+    # Each inp is a dict
+    for config in itertools.product(*combined_kv):
+      new_inp = dict(inp)
+      # config is a list in the form of [(k_i, v_j), (k_p, v_q), ...]
+      suffix = ["%s_%s" % (p[0], str(p[1])) for p in config]
+      suffix = "_".join(suffix)
+      new_inp["testcase_name"] += "_" + suffix
+      for k, v in config:
+        new_inp[k] = v
+      # Remove not used keys from the new test case.
+      if remove_keys:
+        if not isinstance(remove_keys, (list, tuple)):
+          remove_keys = [remove_keys]
+        for k in remove_keys:
+          new_inp.pop(k, None)
+      logging.info("new_inp: %s", new_inp)
+      res.append(new_inp)
+  # Dedup, necessary if `remove_keys` is set.
+  return [dict(t) for t in {tuple(d.items()) for d in res}]
 
-class CudnnRNNTestSaveRestore(TensorFlowTestCase):
 
-  def _CompareWeights(self, lhs, rhs):
-    self.assertEqual(len(lhs), len(rhs))
-    for lw, rw in zip(lhs, rhs):
-      self.assertAllEqual(lw, rw)
+class CudnnLSTMTest(TensorFlowTestCase, parameterized.TestCase):
 
-  def _CompareBiases(self, lhs, rhs, rnn_mode, num_layers, direction):
-    self.assertEqual(len(lhs), len(rhs))
-    if rnn_mode == CUDNN_LSTM:
-      num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER
-    elif rnn_mode == CUDNN_GRU:
-      num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER
-    elif rnn_mode == CUDNN_RNN_TANH:
-      num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER
-    else:
-      num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER
-    num_dirs = 1 if direction == CUDNN_RNN_UNIDIRECTION else 2
-    num_params_per_layer *= num_dirs
-    self.assertEqual(num_params_per_layer * num_layers, len(lhs))
-
-    for i in range(num_layers):
-      layer_lhs = lhs[i * num_params_per_layer: (i+1) * num_params_per_layer]
-      layer_rhs = rhs[i * num_params_per_layer: (i+1) * num_params_per_layer]
-      if direction == CUDNN_RNN_UNIDIRECTION:
-        self._CompareSingleLayerBiases(layer_lhs, layer_rhs)
-      else:
-        size = len(layer_lhs)
-        fw_lhs, bw_lhs = layer_lhs[:size//2], layer_lhs[size//2:]
-        fw_rhs, bw_rhs = layer_rhs[:size//2], layer_rhs[size//2:]
-        self._CompareSingleLayerBiases(fw_lhs, fw_rhs)
-        self._CompareSingleLayerBiases(bw_lhs, bw_rhs)
-
-  def _CompareSingleLayerBiases(self, lhs, rhs):
-    self.assertEqual(len(lhs), len(rhs))
-
-    lf_lhs, rt_lhs = lhs[:len(lhs)//2], lhs[len(lhs)//2:]
-    lf_rhs, rt_rhs = rhs[:len(rhs)//2], rhs[len(rhs)//2:]
-    self.assertEqual(len(lf_lhs), len(rt_lhs))
-    self.assertEqual(len(lf_rhs), len(rt_rhs))
-
-    sum_lhs, sum_rhs = [], []
-    for lf, rt in zip(lf_lhs, rt_lhs):
-      sum_lhs.append(lf + rt)
-    for lf, rt in zip(lf_rhs, rt_rhs):
-      sum_rhs.append(lf + rt)
-    self.assertEqual(len(sum_lhs), len(sum_rhs))
-    for lf, rt in zip(sum_lhs, sum_rhs):
-      self.assertAllEqual(lf, rt)
+  def _test_training_helper(self,
+                            num_units,
+                            input_size,
+                            batch_size,
+                            time,
+                            num_layers,
+                            dtype,
+                            rtol=2e-6,
+                            atol=2e-6):
+    with self.session(use_gpu=True) as sess:
+      (outputs, cu_outputs, state_tuple, cu_state_tuple, inp_grad, cu_inp_grad,
+       state_grad, cu_state_grad, wgrad, bgrad, cu_wgrad, cu_bgrad) = RunLSTM(
+           sess, num_units, input_size, batch_size, time, num_layers)
 
-  def _testSaveRestoreVariable(self, rnn_mode, direction, dtype):
-    num_layers = 2
-    num_units = 7
-    input_size = 3
-    with ops.Graph().as_default():
-      model = _CreateModel(
-          rnn_mode,
-          num_layers=num_layers,
-          num_units=num_units,
-          input_size=input_size,
-          direction=direction,
-          dtype=dtype)
-      random_seed.set_random_seed(1234)
-      params_size_t = model.params_size()
-      params = variables.VariableV1(
-          random_ops.random_uniform([params_size_t], dtype=dtype),
-          dtype=dtype,
-          validate_shape=False)
-      saveable = _CreateParamsSavable(params, model)
-      weights, biases = saveable.format_converter._opaque_to_cu_canonical(
-          saveable._variables)
-      reset_params = state_ops.assign(
-          params,
-          array_ops.zeros([params_size_t], dtype=dtype),
-          validate_shape=False)
-      save_path = os.path.join(self.get_temp_dir(),
-                               "save-restore-variable-test")
-      saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
-      # Passing graph explicitly, otherwise an old sess would be reused.
-      with self.test_session(
-          use_gpu=True, graph=ops.get_default_graph()) as sess:
-        sess.run(variables.global_variables_initializer())
-        val = saver.save(sess, save_path)
-        self.assertEqual(save_path, val)
+      self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol)
+      for s, cu_s in zip(state_tuple, cu_state_tuple):
+        self.assertAllClose(s, cu_s, rtol=rtol, atol=atol)
+      for sg, cu_sg in zip(state_grad, cu_state_grad):
+        self.assertAllClose(sg, cu_sg, rtol=rtol, atol=atol)
+      self.assertAllClose(inp_grad, cu_inp_grad, rtol=rtol, atol=atol)
+      self.assertAllClose(bgrad, cu_bgrad, rtol=rtol, atol=atol)
+      self.assertAllClose(wgrad, cu_wgrad, rtol=rtol, atol=atol)
 
-        weights_v, biases_v = sess.run([weights, biases])
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_training(self, num_units, input_size, batch_size, time, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_training_helper(num_units, input_size, batch_size, time,
+                               num_layers, dtypes.float32)
 
-        sess.run(reset_params)
-        saver.restore(sess, save_path)
-        weights_v_restored, biases_v_restored = sess.run([weights, biases])
-
-        self._CompareWeights(weights_v, weights_v_restored)
-        self._CompareBiases(biases_v, biases_v_restored, rnn_mode, num_layers,
-                            direction)
-
-  def _testSaveRestoreTwoVariables(self, rnn_mode, direction, dtype):
-    num_layers = 2
-    num_units = 7
-    input_size = 3
-    with ops.Graph().as_default():
-      model = _CreateModel(
-          rnn_mode,
-          num_layers=num_layers,
-          num_units=num_units,
-          input_size=input_size,
-          direction=direction,
-          dtype=dtype)
-      random_seed.set_random_seed(1234)
-      params_size_t = model.params_size()
-      names = ["rnn_1", "rnn_2"]
-      param_vars = [
-          variables.VariableV1(
-              random_ops.random_uniform([params_size_t], dtype=dtype),
-              dtype=dtype,
-              validate_shape=False) for name in names
-      ]
-      saveables = []
-      for name, params in zip(names, param_vars):
-        saveables.append(_CreateParamsSavable(params, model, name, name))
-      weights1, biases1 = saveables[0].format_converter._opaque_to_cu_canonical(
-          saveables[0]._variables)
-      weights2, biases2 = saveables[1].format_converter._opaque_to_cu_canonical(
-          saveables[1]._variables)
-      reset_params = [
-          state_ops.assign(
-              params,
-              array_ops.zeros([params_size_t], dtype=dtype),
-              validate_shape=False) for params in param_vars
-      ]
-      save_path = os.path.join(self.get_temp_dir(),
-                               "save-restore-variable-test")
-      saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
-      # Passing graph explicitly, otherwise an old sess would be reused.
-      with self.test_session(use_gpu=True,
-                             graph=ops.get_default_graph()) as sess:
-        sess.run(variables.global_variables_initializer())
-        val = saver.save(sess, save_path)
-        self.assertEqual(save_path, val)
-        weights1_v, biases1_v = sess.run([weights1, biases1])
-        weights2_v, biases2_v = sess.run([weights2, biases2])
-
-        sess.run(reset_params)
-        saver.restore(sess, save_path)
-        weights1_v_restored, biases1_v_restored = sess.run([weights1, biases1])
-        weights2_v_restored, biases2_v_restored = sess.run([weights2, biases2])
-
-        self._CompareWeights(weights1_v, weights1_v_restored)
-        self._CompareWeights(weights2_v, weights2_v_restored)
-        self._CompareBiases(biases1_v, biases1_v_restored, rnn_mode, num_layers,
-                            direction)
-        self._CompareBiases(biases2_v, biases2_v_restored, rnn_mode, num_layers,
-                            direction)
-
-  def _testSaveRestoreOutput(self, rnn_mode, direction, dtype):
-    with ops.Graph().as_default():
-      num_layers = 2
-      num_units = 7
-      input_size = 7
-      seq_length = 10
-      batch_size = 5
-      dir_count = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2
-      model = _CreateModel(
-          rnn_mode,
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_training_fp16(self, num_units, input_size, batch_size, time,
+                         num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_training_helper(
+        num_units,
+        input_size,
+        batch_size,
+        time,
+        num_layers,
+        dtypes.float16,
+        rtol=5e-3,
+        atol=5e-4)
+
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_inference(self, num_units, input_size, batch_size, time, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with self.session(use_gpu=True) as sess:
+      (outputs, cu_outputs, state_tuple, cu_state_tuple) = RunLSTM(
+          sess,
+          num_units,
+          input_size,
+          batch_size,
+          time,
           num_layers,
+          is_training=False)
+
+      self.assertAllClose(outputs, cu_outputs)
+      # h
+      self.assertAllClose(state_tuple.h, cu_state_tuple.h)
+      # c
+      self.assertAllClose(state_tuple.c, cu_state_tuple.c)
+
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_inference_fp16(self, num_units, input_size, batch_size, time,
+                          num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with self.session(use_gpu=True) as sess:
+      (outputs, cu_outputs, state_tuple, cu_state_tuple) = RunLSTM(
+          sess,
           num_units,
           input_size,
-          direction=direction,
-          dtype=dtype)
-      params_size_t = model.params_size()
-      params = variables.VariableV1(
-          array_ops.ones([params_size_t], dtype=dtype),
-          validate_shape=False,
-          dtype=dtype)
-      _CreateParamsSavable(params, model)
-      save_path = os.path.join(self.get_temp_dir(), "save-restore-output-test")
-      saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
+          batch_size,
+          time,
+          num_layers,
+          is_training=False,
+          dtype=dtypes.float16)
 
-      np.random.seed(1234)
-      has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM)
-      input_data = constant_op.constant(
-          np.random.randn(seq_length, batch_size, input_size), dtype=dtype)
-      input_h = constant_op.constant(
-          np.random.randn(num_layers * dir_count, batch_size, num_units),
-          dtype=dtype)
-      if has_input_c:
-        input_c = constant_op.constant(
-            np.random.randn(num_layers * dir_count, batch_size, num_units),
-            dtype=dtype)
-        outputs = model(
-            input_data=input_data,
-            input_h=input_h,
-            input_c=input_c,
-            params=params,
-            is_training=False)
-      else:
-        outputs = model(
-            input_data=input_data,
-            input_h=input_h,
-            params=params,
-            is_training=False)
-      total_sum = sum(map(math_ops.reduce_sum, outputs))
-      # Passing graph explicitly, otherwise an old sess would be reused.
-      with self.test_session(
-          use_gpu=True, graph=ops.get_default_graph()) as sess:
-        sess.run(variables.global_variables_initializer())
-        total_sum_v = sess.run(total_sum)
-        val = saver.save(sess, save_path)
-        self.assertEqual(save_path, val)
-      # Passing graph explicitly, otherwise an old sess would be reused.
-      with self.test_session(
-          use_gpu=True, graph=ops.get_default_graph()) as sess:
-        reset_params = state_ops.assign(
-            params,
-            array_ops.zeros([params_size_t], dtype=dtype),
-            validate_shape=False)
-        sess.run(reset_params)
-        saver.restore(sess, save_path)
-        total_sum_v_restored = sess.run(total_sum)
-        self.assertAllClose(total_sum_v, total_sum_v_restored, atol=1e-5)
+      rtol, atol = 5e-3, 5e-4
+      self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol)
+      # h
+      self.assertAllClose(
+          state_tuple.h, cu_state_tuple.h, rtol=rtol, atol=atol)
+      # c
+      self.assertAllClose(
+          state_tuple.c, cu_state_tuple.c, rtol=rtol, atol=atol)
 
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
   @unittest.skipUnless(test.is_built_with_cuda(),
                        "Test only applicable when running on GPUs")
-  def testSaveRestore(self):
-    rnn_modes = [
-        cudnn_rnn_ops.CUDNN_LSTM, cudnn_rnn_ops.CUDNN_GRU,
-        cudnn_rnn_ops.CUDNN_RNN_TANH, cudnn_rnn_ops.CUDNN_RNN_RELU
-    ]
-    directions = [
-        cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION,
-        cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION
-    ]
-    dtype_list = [dtypes.float32, dtypes.float64]
-    for rnn_mode, direction, dtype in itertools.product(rnn_modes, directions,
-                                                        dtype_list):
-      self._testSaveRestoreVariable(rnn_mode, direction, dtype)
-      self._testSaveRestoreTwoVariables(rnn_mode, direction, dtype)
-      self._testSaveRestoreOutput(rnn_mode, direction, dtype)
-
-
-class CudnnRNNTestParamsSize(TensorFlowTestCase):
-
-  def _testOneLSTMParamsSize(self, num_layers, num_units, input_size,
-                             direction):
-    logging.info("Testing one lstm param size with config: %s", locals())
-    min_params_size = _MinLSTMParamSize(num_layers, num_units, input_size,
-                                        direction)
-    model = _CreateModel(
-        cudnn_rnn_ops.CUDNN_LSTM,
-        num_layers,
+  def test_inference_with_dropout(self, num_units, input_size, batch_size, time,
+                                  num_layers):
+    """Validates that dropout does not affect Cudnn Rnn inference."""
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    # Hand-picked dropouts are used below (0. and 1.)
+    with ops.Graph().as_default() as g:
+      with self.session(use_gpu=True, graph=g) as sess:
+        # 1st time w/o dropout.
+        (_, cu_outputs, _, cu_state_tuple) = RunLSTM(
+            sess,
+            num_units,
+            input_size,
+            batch_size,
+            time,
+            num_layers,
+            is_training=False,
+            dropout=0.)
+
+    with ops.Graph().as_default() as g:
+      with self.session(use_gpu=True, graph=g) as sess:
+        (_, cu_outputs2, _, cu_state_tuple2) = RunLSTM(
+            sess,
+            num_units,
+            input_size,
+            batch_size,
+            time,
+            num_layers,
+            is_training=False,
+            dropout=1.)
+
+    self.assertAllClose(cu_outputs, cu_outputs2)
+    # h
+    self.assertAllClose(cu_state_tuple.h, cu_state_tuple2.h)
+    # c
+    self.assertAllClose(cu_state_tuple.c, cu_state_tuple2.c)
+
+
+def RunGRU(sess,
+           num_units,
+           input_size,
+           batch_size,
+           time,
+           num_layers=1,
+           is_training=True,
+           dropout=0.,
+           num_dirs=True,
+           dtype=dtypes.float32):
+  # TODO(jamesqin): add multi-layer tests.
+  # TODO(jamesqin): add multi-dir tests
+  assert num_layers == 1
+  assert num_dirs == 1
+  if is_training and not np.isclose(dropout, 0):
+    raise ValueError("dropout can not be 0. when test training.")
+
+  # set graph level random seed and numpy random seed.
+  random_seed.set_random_seed(0)
+  np.random.seed(0)
+
+  inputs = variable_scope.get_variable(
+      "inputs",
+      initializer=np.random.rand(time, batch_size,
+                                 input_size).astype(dtype.as_numpy_dtype),
+      dtype=dtype)
+  initial_h_op = variable_scope.get_variable(
+      "initial_h_op",
+      initializer=np.random.rand(batch_size,
+                                 num_units).astype(dtype.as_numpy_dtype),
+      dtype=dtype)
+
+  initializer = init_ops.random_uniform_initializer(
+      -0.01, 0.01, dtype=dtype, seed=19980904)
+  with variable_scope.variable_scope("test", initializer=initializer):
+    gate_kernel = variable_scope.get_variable(
+        "rnn/cudnn_compatible_gru_cell/gates/kernel",
+        shape=[input_size + num_units, num_units * 2],
+        dtype=dtype)
+    gate_bias = variable_scope.get_variable(
+        "rnn/cudnn_compatible_gru_cell/gates/bias",
+        shape=[num_units * 2],
+        dtype=dtype)
+    candidate_inp_kernel = variable_scope.get_variable(
+        "rnn/cudnn_compatible_gru_cell/candidate/input_projection/kernel",
+        shape=[input_size, num_units],
+        dtype=dtype)
+    candidate_inp_bias = variable_scope.get_variable(
+        "rnn/cudnn_compatible_gru_cell/candidate/input_projection/bias",
+        shape=[num_units],
+        dtype=dtype)
+    candidate_hid_kernel = variable_scope.get_variable(
+        "rnn/cudnn_compatible_gru_cell/candidate/hidden_projection/kernel",
+        shape=[num_units, num_units],
+        dtype=dtype)
+    candidate_hid_bias = variable_scope.get_variable(
+        "rnn/cudnn_compatible_gru_cell/candidate/hidden_projection/bias",
+        shape=[num_units],
+        dtype=dtype)
+
+    cell = cudnn_rnn_ops.CudnnCompatibleGRUCell(num_units, reuse=True)
+    outputs_op, h_op = rnn.dynamic_rnn(
+        cell,
+        inputs,
+        initial_state=initial_h_op,
+        dtype=dtype,
+        time_major=True,
+        scope=None)
+
+  ws = [gate_kernel, candidate_inp_kernel, candidate_hid_kernel]
+  bs = [gate_bias, candidate_inp_bias, candidate_hid_bias]
+  # Convert to cudnn opaque param.
+  format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterGRU(
+      num_layers, num_units, input_size)
+  opaque_params = format_converter.tf_canonical_to_opaque(ws + bs)
+
+  cu_initial_h_op = array_ops.expand_dims(initial_h_op, axis=0)
+  cu_outputs_op, cu_h_op, _ = cudnn_rnn_ops._cudnn_rnn(
+      inputs,
+      cu_initial_h_op,
+      array_ops.zeros_like(cu_initial_h_op),  # not used
+      opaque_params,
+      dropout=dropout,
+      is_training=is_training,
+      rnn_mode=cudnn_rnn_ops.CUDNN_GRU)
+
+  if is_training:
+    (inp_grad_op, hgrad_op, gk_grad_op, cik_grad_op, chk_grad_op, gb_grad_op,
+     cib_grad_op, chb_grad_op) = gradients_impl.gradients(
+         outputs_op, [inputs, initial_h_op] + ws + bs)
+
+    (cu_inp_grad_op, cu_hgrad_op, opaque_grad_op) = gradients_impl.gradients(
+        cu_outputs_op, [inputs, cu_initial_h_op, opaque_params])
+    # Remove the trivial 1st dimension
+    cu_hgrad_op = array_ops.squeeze(cu_hgrad_op, axis=0)
+
+    cu_wgrad_op, cu_bgrad_op = format_converter.opaque_to_tf_canonical(
+        opaque_grad_op)
+    (cu_gk_grad_op, cu_cik_grad_op, cu_chk_grad_op) = cu_wgrad_op
+    (cu_gb_grad_op, cu_cib_grad_op, cu_chb_grad_op) = cu_bgrad_op
+    # cudnn gru has 2 biases for reset and update gates. When converting to tf
+    # canonical format, the two biases are summed into one.  Thus here relevant
+    # bias gradient should be halved before comparing with tf gru.
+    cu_gb_grad_op *= 0.5
+
+  init_op = variables.global_variables_initializer()
+  sess.run(init_op)
+
+  if is_training:
+    outputs, h, inp_grad, hgrad, wgrad, bgrad = sess.run([
+        outputs_op, h_op, inp_grad_op, hgrad_op,
+        (gk_grad_op, cik_grad_op, chk_grad_op),
+        (gb_grad_op, cib_grad_op, chb_grad_op)
+    ])
+    (cu_outputs, cu_h, cu_inp_grad, cu_hgrad, cu_wgrad, cu_bgrad) = sess.run([
+        cu_outputs_op, cu_h_op, cu_inp_grad_op, cu_hgrad_op,
+        (cu_gk_grad_op, cu_cik_grad_op, cu_chk_grad_op),
+        (cu_gb_grad_op, cu_cib_grad_op, cu_chb_grad_op)
+    ])
+    # Remove the trivial 1st dimension
+    cu_h = np.squeeze(cu_h, axis=0)
+
+    logging.vlog(1, "outputs: %s" % outputs)
+    logging.vlog(1, "cu_outputs: %s" % cu_outputs)
+    logging.vlog(1, "h: %s" % h)
+    logging.vlog(1, "cu_h: %s" % h)
+    logging.vlog(1, "inp_grad: %s" % inp_grad)
+    logging.vlog(1, "cu_inp_grad: %s" % cu_inp_grad)
+    logging.vlog(1, "hgrad: %s" % hgrad)
+    logging.vlog(1, "cu_hgrad: %s" % cu_hgrad)
+    logging.vlog(1, "wgrad: %s" % str(wgrad))
+    logging.vlog(1, "bgrad: %s" % str(bgrad))
+    logging.vlog(1, "cu_wgrad: %s" % str(cu_wgrad))
+    logging.vlog(1, "cu_bgrad: %s" % str(cu_bgrad))
+    return (outputs, cu_outputs, h, cu_h, inp_grad, cu_inp_grad, hgrad,
+            cu_hgrad, wgrad, bgrad, cu_wgrad, cu_bgrad)
+  else:
+    outputs, h = sess.run([outputs_op, h_op])
+    cu_outputs, cu_h = sess.run([cu_outputs_op, cu_h_op])
+    # Remove the trivial 1st dimension.
+    cu_h = np.squeeze(cu_h, axis=0)
+
+    logging.vlog(1, "outputs: %s" % outputs)
+    logging.vlog(1, "cu_outputs: %s" % cu_outputs)
+    logging.vlog(1, "h: %s" % h)
+    logging.vlog(1, "cu_h: %s" % h)
+  return outputs, cu_outputs, h, cu_h
+
+
+class CudnnGRUTest(TensorFlowTestCase, parameterized.TestCase):
+
+  def _test_training_helper(self,
+                            num_units,
+                            input_size,
+                            batch_size,
+                            time,
+                            num_layers,
+                            dtype,
+                            rtol=2e-6,
+                            atol=2e-6):
+    with self.session(use_gpu=True) as sess:
+      (outputs, cu_outputs, h, cu_h, inp_grad, cu_inp_grad, hgrad,
+       cu_hgrad, wgrad, bgrad, cu_wgrad, cu_bgrad) = RunGRU(
+           sess, num_units, input_size, batch_size, time, num_layers)
+
+      self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol)
+      self.assertAllClose(h, cu_h, rtol=rtol, atol=atol)
+      self.assertAllClose(hgrad, cu_hgrad, rtol=rtol, atol=atol)
+      self.assertAllClose(inp_grad, cu_inp_grad, rtol=rtol, atol=atol)
+      for bg, cu_bg in zip(bgrad, cu_bgrad):
+        self.assertAllClose(bg, cu_bg, rtol=rtol, atol=atol)
+      for wg, cu_wg in zip(wgrad, cu_wgrad):
+        self.assertAllClose(wg, cu_wg, rtol=rtol, atol=atol)
+
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_training(self, num_units, input_size, batch_size, time, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_training_helper(num_units, input_size, batch_size, time,
+                               num_layers, dtypes.float32)
+
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_training_fp16(self, num_units, input_size, batch_size, time,
+                         num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_training_helper(
         num_units,
         input_size,
-        direction=direction)
-    params_size = model.params_size()
-    with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess:
-      params_size_v = sess.run(params_size)
-      self.assertLessEqual(min_params_size, params_size_v)
+        batch_size,
+        time,
+        num_layers,
+        dtypes.float16,
+        rtol=5e-3,
+        atol=5e-4)
 
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
   @unittest.skipUnless(test.is_built_with_cuda(),
                        "Test only applicable when running on GPUs")
-  def testLSTMParamsSize(self):
-    test_configs = [
-        [4, 200, 200],
-        [4, 200, 300],
-        [4, 200, 100],
-        [1, 100, 200],
-        [2, 200, 100],
-        [3, 200, 400],
-    ]
-    directions = [
-        cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION,
-        cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION
-    ]
-    for (config, direction) in itertools.product(test_configs, directions):
-      num_layers, num_units, input_size = config
-      with ops.Graph().as_default():
-        self._testOneLSTMParamsSize(num_layers, num_units, input_size,
-                                    direction)
+  def test_inference(self, num_units, input_size, batch_size, time, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with self.session(use_gpu=True) as sess:
+      (outputs, cu_outputs, h, cu_h) = RunGRU(
+          sess,
+          num_units,
+          input_size,
+          batch_size,
+          time,
+          num_layers,
+          is_training=False)
+      self.assertAllClose(outputs, cu_outputs)
+      self.assertAllClose(h, cu_h)
 
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
   @unittest.skipUnless(test.is_built_with_cuda(),
                        "Test only applicable when running on GPUs")
-  def testLSTMParamsSizeShape(self):
-    with self.assertRaisesRegexp(
-        ValueError, "Shape must be rank 0 but is rank 1"):
-      model = _CreateModel(
-          cudnn_rnn_ops.CUDNN_LSTM,
-          constant_op.constant([4]), 200, 200,
-          direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION)
-      _ = model.params_size()
-    with self.assertRaisesRegexp(
-        ValueError, "Shape must be rank 0 but is rank 1"):
-      model = _CreateModel(
-          cudnn_rnn_ops.CUDNN_LSTM,
-          4, constant_op.constant([200]), 200,
-          direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION)
-      _ = model.params_size()
-    with self.assertRaisesRegexp(
-        ValueError, "Shape must be rank 0 but is rank 1"):
-      model = _CreateModel(
+  def test_inference_fp16(self, num_units, input_size, batch_size, time,
+                          num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with self.session(use_gpu=True) as sess:
+      (outputs, cu_outputs, h, cu_h) = RunGRU(
+          sess,
+          num_units,
+          input_size,
+          batch_size,
+          time,
+          num_layers,
+          is_training=False,
+          dtype=dtypes.float16)
+
+      rtol, atol = 5e-3, 5e-4
+      self.assertAllClose(outputs, cu_outputs, rtol=rtol, atol=atol)
+      self.assertAllClose(h, cu_h, rtol=rtol, atol=atol)
+
+  @parameterized.named_parameters(*NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_inference_with_dropout(self, num_units, input_size, batch_size, time,
+                                  num_layers):
+    """Validates that dropout does not affect Cudnn Rnn inference."""
+    # Hand-picked dropouts are used below (0. and 1.)
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with ops.Graph().as_default() as g:
+      with self.session(use_gpu=True, graph=g) as sess:
+        # 1st time w/o dropout.
+        (_, cu_outputs, _, cu_h) = RunGRU(
+            sess,
+            num_units,
+            input_size,
+            batch_size,
+            time,
+            num_layers,
+            is_training=False,
+            dropout=0.)
+
+    with ops.Graph().as_default() as g:
+      with self.session(use_gpu=True, graph=g) as sess:
+        (_, cu_outputs2, _, cu_h2) = RunGRU(
+            sess,
+            num_units,
+            input_size,
+            batch_size,
+            time,
+            num_layers,
+            is_training=False,
+            dropout=1.)
+
+    self.assertAllClose(cu_outputs, cu_outputs2)
+    self.assertAllClose(cu_h[0], cu_h2[0])
+
+
+class CudnnParamsFormatConverterTest(TensorFlowTestCase,
+                                     parameterized.TestCase):
+  """Class for testing various format converters."""
+
+  def _test_lstm_helper(self, num_units, input_size, num_layers, direction):
+    with self.session(use_gpu=True) as sess:
+      random_seed.set_random_seed(0)
+      np.random.seed(0)
+
+      num_dirs = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2
+      format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterLSTM(
+          num_layers, num_units, input_size, direction=direction)
+
+      ws, bs = [], []
+      for _ in range(num_layers * num_dirs):
+        w = constant_op.constant(
+            np.random.rand(input_size + num_units, 4 * num_units),
+            dtype=dtypes.float32)
+        b = constant_op.constant(
+            np.random.rand(4 * num_units), dtype=dtypes.float32)
+        ws.append(w)
+        bs.append(b)
+
+      opaque_params = format_converter.tf_canonical_to_opaque(ws + bs)
+      opaque_params_size = cudnn_rnn_ops.cudnn_rnn_opaque_params_size(
           cudnn_rnn_ops.CUDNN_LSTM,
-          4, 200, constant_op.constant([200]),
-          direction=cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION)
-      _ = model.params_size()
+          num_layers,
+          num_units,
+          input_size,
+          direction=direction)
 
+      ws_r, bs_r = format_converter.opaque_to_tf_canonical(opaque_params)
 
-class CudnnRNNTestInference(TensorFlowTestCase):
+      # Test tf_canonical_to_opaque() followed by opaque_to_tf_canonical()
+      # returns the original input.
+      ws, ws_r, bs, bs_r = sess.run([ws, ws_r, bs, bs_r])
+      for w, w_r in zip(ws, ws_r):
+        self.assertAllClose(w, w_r)
+      for b, b_r in zip(bs, bs_r):
+        self.assertAllClose(b, b_r)
 
-  def _testOneSimpleInference(self, rnn_mode, num_layers, num_units, input_size,
-                              batch_size, seq_length, dir_count, dropout,
-                              expected, tolerance):
-    random_seed.set_random_seed(5678)
-    model = _CreateModel(
-        rnn_mode,
-        num_layers,
-        num_units,
-        input_size,
-        input_mode="auto_select",
-        direction=(cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION if dir_count == 1
-                   else cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION),
-        dropout=dropout)
-    has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM)
-    params_size_t = model.params_size()
-    input_data = array_ops.ones([seq_length, batch_size, input_size])
-    input_h = array_ops.ones([num_layers * dir_count, batch_size, num_units])
-    params = variables.VariableV1(
-        array_ops.ones([params_size_t]), validate_shape=False)
-    if has_input_c:
-      input_c = array_ops.ones([num_layers * dir_count, batch_size, num_units])
-      output, output_h, output_c = model(
-          input_data=input_data,
-          input_h=input_h,
-          input_c=input_c,
-          params=params,
-          is_training=False)
-    else:
-      output, output_h = model(
-          input_data=input_data,
-          input_h=input_h,
-          params=params,
-          is_training=False)
-    output_sum = math_ops.reduce_sum(output)
-    output_h_sum = math_ops.reduce_sum(output_h)
-    total_sum = output_sum + output_h_sum
-    if has_input_c:
-      output_c_sum = math_ops.reduce_sum(output_c)
-      total_sum += output_c_sum
-    with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess:
-      sess.run(variables.global_variables_initializer())
-      total_sum_v = sess.run([total_sum])
+      # Test opaque_params size lower bound
+      opaque_params_size_v = sess.run(opaque_params_size)
+      min_params_size = (
+          np.sum([x.size for x in ws]) + np.sum([x.size for x in bs]))
+      logging.info("min_parm_size: %d vs actual_opaque_param_size: %d",
+                   min_params_size, opaque_params_size_v)
+      self.assertLessEqual(min_params_size, opaque_params_size_v)
 
-      self.assertAllClose(
-          total_sum_v[0], expected, atol=tolerance, rtol=tolerance)
+  @parameterized.named_parameters((c["testcase_name"], c["num_units"],
+                                   c["input_size"], c["num_layers"])
+                                  for c in NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_lstm(self, num_units, input_size, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_lstm_helper(num_units, input_size, num_layers,
+                           cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION)
 
+  @parameterized.named_parameters((c["testcase_name"], c["num_units"],
+                                   c["input_size"], c["num_layers"])
+                                  for c in NAMED_RNN_TESTCASES)
   @unittest.skipUnless(test.is_built_with_cuda(),
                        "Test only applicable when running on GPUs")
-  def testSimpleInference(self):
-    test_configs = [
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_LSTM,
-            "expected": 231833.22,
-            "tolerance": 1e-2,
-            "shape": {
-                "num_layers": 4,
-                "num_units": 200,
-                "input_size": 200,
-                "batch_size": 20,
-                "seq_length": 10,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_GRU,
-            "expected": 56000,
-            "tolerance": 1e-2,
-            "shape": {
-                "num_layers": 4,
-                "num_units": 200,
-                "input_size": 200,
-                "batch_size": 20,
-                "seq_length": 10,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_TANH,
-            "expected": 56000,
-            "tolerance": 1e-2,
-            "shape": {
-                "num_layers": 4,
-                "num_units": 200,
-                "input_size": 200,
-                "batch_size": 20,
-                "seq_length": 10,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_RELU,
-            "expected": 130688,
-            "tolerance": 1e-2,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 8,
-                "input_size": 4,
-                "batch_size": 4,
-                "seq_length": 2,
-                "dir_count": 1,
-            },
-        },
-    ]
-    # Cudnn scales result for dropout during training, therefore dropout has no
-    # impact for inference results.
-    # (lstm, gru, rnn_tanh are saturated in the test. rnn_relu case is most
-    # demonstrative of the dropout-invariant nature of CudnnRnn.)
-    dropouts = [0., 0.5, 1.]
-    for (config, dropout) in itertools.product(test_configs, dropouts):
-      rnn_mode = config["rnn_mode"]
-      expected = config["expected"]
-      tolerance = config["tolerance"]
-      shape = config["shape"]
-      with ops.Graph().as_default():
-        self._testOneSimpleInference(
-            rnn_mode, shape["num_layers"], shape["num_units"],
-            shape["input_size"], shape["batch_size"], shape["seq_length"],
-            shape["dir_count"], dropout, expected, tolerance)
-
-
-class CudnnRNNTestTraining(TensorFlowTestCase):
-
-  def _testOneSimpleTraining(self, rnn_mode, num_layers, num_units, input_size,
-                             batch_size, seq_length, dir_count, dropout, dtype,
-                             delta, tolerance):
-    # Gradient checking runs two forward ops with almost the same input. Need to
-    # make sure the drop patterns across the two runs are the same.
-    logging.info("Training test with config: %s", locals())
-    old_env_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE", str(False))
-    os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True)
-    has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM)
-    random_seed.set_random_seed(5678)
-    direction = (cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION if dir_count == 1
-                 else cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION)
-    model = _CreateModel(
-        rnn_mode,
-        num_layers,
-        num_units,
-        input_size,
-        direction=direction,
-        dtype=dtype,
-        dropout=dropout)
-    params_size_t = model.params_size()
-    input_data = variables.VariableV1(
-        random_ops.random_uniform(
-            [seq_length, batch_size, input_size], dtype=dtype),
-        dtype=dtype)
-    input_h = variables.VariableV1(
-        random_ops.random_uniform(
-            [num_layers * dir_count, batch_size, num_units], dtype=dtype),
-        dtype=dtype)
-    params = variables.VariableV1(
-        random_ops.random_uniform([params_size_t], dtype=dtype),
-        validate_shape=False,
-        dtype=dtype)
-    if has_input_c:
-      input_c = variables.VariableV1(
-          random_ops.random_uniform(
-              [num_layers * dir_count, batch_size, num_units], dtype=dtype),
-          dtype=dtype)
-
-      output, output_h, output_c = model(
-          input_data=input_data,
-          input_h=input_h,
-          input_c=input_c,
-          params=params)
-    else:
-      output, output_h = model(
-          input_data=input_data, input_h=input_h, params=params)
-    output_sum = math_ops.reduce_sum(output)
-    output_h_sum = math_ops.reduce_sum(output_h)
-    total_sum = output_sum + output_h_sum
-    if has_input_c:
-      output_c_sum = math_ops.reduce_sum(output_c)
-      total_sum += output_c_sum
-
-    with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess:
-      params_size_v = sess.run(params_size_t)
-      inputs_and_shapes = [
-          (input_data, [seq_length, batch_size, input_size]),
-          (input_h, [num_layers * dir_count, batch_size, num_units]),
-          (params, [params_size_v]),
-      ]
-      if has_input_c:
-        inputs_and_shapes.append(
-            (input_c, [num_layers * dir_count, batch_size, num_units]),)
-      sess.run(variables.global_variables_initializer())
-      all_inputs = [entry[0] for entry in inputs_and_shapes]
-      all_shapes = [entry[1] for entry in inputs_and_shapes]
-
-      err = gradient_checker.compute_gradient_error(
-          all_inputs, all_shapes, total_sum, [1], delta=delta)
-
-      self.assertLess(err, tolerance)
-      os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = old_env_state
+  def test_lstm_bidi(self, num_units, input_size, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_lstm_helper(num_units, input_size, num_layers,
+                           cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION)
+
+  def _test_gru_helper(self, num_units, input_size, num_layers, direction):
+    with self.session(use_gpu=True) as sess:
+      random_seed.set_random_seed(0)
+      np.random.seed(0)
+
+      num_dirs = 1 if direction == cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION else 2
+      format_converter = cudnn_rnn_ops.CudnnParamsFormatConverterGRU(
+          num_layers, num_units, input_size, direction=direction)
+
+      ws, bs = [], []
+      for _ in range(num_layers * num_dirs):
+        gate_kernel = constant_op.constant(
+            np.random.rand(input_size + num_units, num_units * 2),
+            dtype=dtypes.float32)
+        gate_bias = constant_op.constant(
+            np.random.rand(num_units * 2), dtype=dtypes.float32)
+        candidate_inp_kernel = constant_op.constant(
+            np.random.rand(input_size, num_units), dtype=dtypes.float32)
+        candidate_inp_bias = constant_op.constant(
+            np.random.rand(num_units), dtype=dtypes.float32)
+        candidate_hid_kernel = constant_op.constant(
+            np.random.rand(num_units, num_units), dtype=dtypes.float32)
+        candidate_hid_bias = constant_op.constant(
+            np.random.rand(num_units), dtype=dtypes.float32)
+        ws.extend([gate_kernel, candidate_inp_kernel, candidate_hid_kernel])
+        bs.extend([gate_bias, candidate_inp_bias, candidate_hid_bias])
 
+      opaque_params = format_converter.tf_canonical_to_opaque(ws + bs)
+      opaque_params_size = cudnn_rnn_ops.cudnn_rnn_opaque_params_size(
+          cudnn_rnn_ops.CUDNN_GRU,
+          num_layers,
+          num_units,
+          input_size,
+          direction=direction)
+
+      ws_r, bs_r = format_converter.opaque_to_tf_canonical(opaque_params)
+
+      # Test tf_canonical_to_opaque() followed by opaque_to_tf_canonical()
+      # returns the original input.
+      ws, ws_r, bs, bs_r = sess.run([ws, ws_r, bs, bs_r])
+      for w, w_r in zip(ws, ws_r):
+        self.assertAllClose(w, w_r)
+      for b, b_r in zip(bs, bs_r):
+        self.assertAllClose(b, b_r)
+
+      # Test opaque_params size lower bound
+      opaque_params_size_v = sess.run(opaque_params_size)
+      min_params_size = (
+          np.sum([x.size for x in ws]) + np.sum([x.size for x in bs]))
+      logging.info("min_parm_size: %d vs actual_opaque_param_size: %d",
+                   min_params_size, opaque_params_size_v)
+      self.assertLessEqual(min_params_size, opaque_params_size_v)
+
+  @parameterized.named_parameters((c["testcase_name"], c["num_units"],
+                                   c["input_size"], c["num_layers"])
+                                  for c in NAMED_RNN_TESTCASES)
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_gru(self, num_units, input_size, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_gru_helper(num_units, input_size, num_layers,
+                          cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION)
+
+  @parameterized.named_parameters((c["testcase_name"], c["num_units"],
+                                   c["input_size"], c["num_layers"])
+                                  for c in NAMED_RNN_TESTCASES)
   @unittest.skipUnless(test.is_built_with_cuda(),
                        "Test only applicable when running on GPUs")
-  def DISABLED_testSimpleTraining(self):
-    # TODO(jamesqin): fix b/117989214
-    test_configs = [
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_LSTM,
-            "dtype": dtypes.float64,
-            "delta": 1e-4,
-            "tolerance": 5e-6,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_GRU,
-            "dtype": dtypes.float64,
-            "delta": 1e-4,
-            "tolerance": 5e-6,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_TANH,
-            "dtype": dtypes.float64,
-            "delta": 1e-4,
-            "tolerance": 5e-6,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_RELU,
-            "dtype": dtypes.float64,
-            "delta": 1e-4,
-            "tolerance": 5e-6,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-                "dir_count": 1,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_LSTM,
-            "dtype": dtypes.float32,
-            "tolerance": 1.5e-2,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_GRU,
-            "dtype": dtypes.float32,
-            "tolerance": 4e-3,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_TANH,
-            "dtype": dtypes.float32,
-            "tolerance": 5e-3,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-            },
-        },
-        {
-            "rnn_mode": cudnn_rnn_ops.CUDNN_RNN_RELU,
-            "dtype": dtypes.float32,
-            "tolerance": 5e-1,
-            "shape": {
-                "num_layers": 2,
-                "num_units": 3,
-                "input_size": 4,
-                "batch_size": 3,
-                "seq_length": 4,
-            },
-        },
-    ]
-    dropouts = [0., 0.5, 1.]
-    dir_counts = [1]
-    for config, dropout, dir_count in itertools.product(test_configs, dropouts,
-                                                        dir_counts):
-      rnn_mode = config["rnn_mode"]
-      dtype = config.get("dtype", dtypes.float32)
-      delta = config.get("delta", 1e-3)
-      tolerance = config["tolerance"]
-      shape = config["shape"]
-      with ops.Graph().as_default():
-        self._testOneSimpleTraining(rnn_mode, shape["num_layers"],
-                                    shape["num_units"], shape["input_size"],
-                                    shape["batch_size"], shape["seq_length"],
-                                    dir_count, dropout, dtype, delta, tolerance)
+  def test_gru_bidi(self, num_units, input_size, num_layers):
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    self._test_gru_helper(num_units, input_size, num_layers,
+                          cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION)
+
+
+class CudnnRnnSaveRestoreTest(TensorFlowTestCase, parameterized.TestCase):
+  """Class for testing various Cudnn Rnn SaveableObjects."""
+
+  def _create_opaque_param(self,
+                           rnn_mode,
+                           num_units,
+                           input_size,
+                           num_layers,
+                           direction,
+                           name=None):
+    param_size_t = cudnn_rnn_ops.cudnn_rnn_opaque_params_size(
+        rnn_mode, num_layers, num_units, input_size, direction=direction)
+    init_val = random_ops.random_uniform([param_size_t])
+    return variable_scope.get_variable(
+        name or "opaque_param", initializer=init_val, validate_shape=False)
+
+  def _create_saveable(self, opaque_param, rnn_mode, num_units, input_size,
+                       num_layers, direction):
+    if rnn_mode == CUDNN_LSTM:
+      fn = cudnn_rnn_ops.CudnnLSTMSaveable
+    elif rnn_mode == CUDNN_GRU:
+      fn = cudnn_rnn_ops.CudnnGRUSaveable
+    elif rnn_mode == CUDNN_RNN_TANH:
+      fn = cudnn_rnn_ops.CudnnRNNTanhSaveable
+    elif rnn_mode == CUDNN_RNN_RELU:
+      fn = cudnn_rnn_ops.CudnnRNNReluSaveable
+    saveable = fn(
+        opaque_param, num_layers, num_units, input_size, direction=direction)
+    return saveable
+
+  def _compare_weights(self, lhs, rhs):
+    self.assertLen(rhs, len(lhs))
+    for lw, rw in zip(lhs, rhs):
+      self.assertAllEqual(lw, rw)
+
+  def _compare_biases(self, lhs, rhs):
+    self.assertLen(rhs, len(lhs))
+    for lf, rt in zip(lhs, rhs):
+      self.assertAllEqual(lf, rt)
+
+  @parameterized.named_parameters(
+      ExpandNamedTestCases(
+          NAMED_RNN_TESTCASES, "time", "batch_size", **{
+              "rnn_mode": [
+                  CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH
+              ],
+              "direction": [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION]
+          }))
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_save_restore_variable(self, rnn_mode, num_units, input_size,
+                                 num_layers, direction):
+    # Verify the restored opaque param, once converted to tf_canonical format,
+    # is the same as the tf canonicals of the pre-restored param.
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with self.session(use_gpu=True) as sess:
+      opaque_param = self._create_opaque_param(rnn_mode, num_units, input_size,
+                                               num_layers, direction)
+      saveable = self._create_saveable(opaque_param, rnn_mode, num_units,
+                                       input_size, num_layers, direction)
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
+      weights_op, biases_op = saveable.format_converter.opaque_to_tf_canonical(
+          saveable._variables)
+
+      save_path = os.path.join(self.get_temp_dir(), "save_restore_var_test")
+      saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
+
+      init_op = variables.global_variables_initializer()
+      reset_op = state_ops.assign(opaque_param,
+                                  array_ops.zeros_like(opaque_param))
+      sess.run(init_op)
+      self.assertEqual(save_path, saver.save(sess, save_path))
+
+      # Get the tf canonical vals before reset-restore
+      weights, biases = sess.run([weights_op, biases_op])
+
+      # Reset the opaque param value
+      sess.run(reset_op)
+      # Assert reset happened.
+      weights_z, biases_z = sess.run([weights_op, biases_op])
+      for w in weights_z:
+        self.assertAllClose(w, np.zeros_like(w))
+      for b in biases_z:
+        self.assertAllClose(b, np.zeros_like(b))
+
+      # Restore opaque param value from checkpoint.
+      saver.restore(sess, save_path)
+      weights_r, biases_r = sess.run([weights_op, biases_op])
+      self._compare_weights(weights, weights_r)
+      self._compare_biases(biases, biases_r)
+
+  @parameterized.named_parameters(
+      ExpandNamedTestCases(
+          NAMED_RNN_TESTCASES, "time", "batch_size", **{
+              "rnn_mode": [
+                  CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH
+              ],
+              "direction": [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION]
+          }))
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def test_save_restore_multi_variables(self, rnn_mode, num_units, input_size,
+                                        num_layers, direction):
+    # Verify the restored opaque param, once converted to tf_canonical format,
+    # is the same as the tf canonicals of the pre-restored param.
+    if not context.context().num_gpus():
+      self.skipTest("No GPUs found")
+    with self.session(use_gpu=True) as sess:
+      opaque_params = []
+      saveables = []
+      num_opaque_params = 2
+      for i in range(num_opaque_params):
+        opaque_params.append(
+            self._create_opaque_param(
+                rnn_mode,
+                num_units,
+                input_size,
+                num_layers,
+                direction,
+                name="opaque_param_%d" % i))
+        saveable = self._create_saveable(opaque_params[i], rnn_mode, num_units,
+                                         input_size, num_layers, direction)
+        ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
+        saveables.append(saveable)
+
+      weights_ops, biases_ops = [], []
+      for i in range(num_opaque_params):
+        weights_op, biases_op = (
+            saveables[i].format_converter.opaque_to_tf_canonical(
+                saveables[i]._variables))
+        weights_ops.append(weights_op)
+        biases_ops.append(biases_op)
+
+      save_path = os.path.join(self.get_temp_dir(), "save_restore_var_test")
+      saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
+
+      init_op = variables.global_variables_initializer()
+      reset_ops = []
+      for i in range(num_opaque_params):
+        reset_ops.append(
+            state_ops.assign(opaque_params[i],
+                             array_ops.zeros_like(opaque_params[i])))
+      sess.run(init_op)
+      self.assertEqual(save_path, saver.save(sess, save_path))
+
+      # Get the tf canonical vals before reset-restore
+      for i in range(num_opaque_params):
+        weights, biases = sess.run([weights_ops[i], biases_ops[i]])
+
+        # Reset the opaque param value
+        sess.run(reset_ops[i])
+
+        # Assert reset happened.
+        weights_z, biases_z = sess.run([weights_ops[i], biases_ops[i]])
+        for w in weights_z:
+          self.assertAllClose(w, np.zeros_like(w))
+        for b in biases_z:
+          self.assertAllClose(b, np.zeros_like(b))
+
+        # Restore opaque param value from checkpoint.
+        saver.restore(sess, save_path)
+        weights_r, biases_r = sess.run([weights_ops[i], biases_ops[i]])
+        self._compare_weights(weights, weights_r)
+        self._compare_biases(biases, biases_r)
 
 
 if __name__ == "__main__":
-- 
GitLab


From ebdc11520bd79a58b74c0d04a258917f887a24a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 13:45:52 -0800
Subject: [PATCH 0241/1554] Bug fix: platform_strings should include powerpc
 options on 32-bit powerpc machines

PiperOrigin-RevId: 221503303
---
 tensorflow/core/platform/platform_strings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/platform_strings.h b/tensorflow/core/platform/platform_strings.h
index c9d29e6362..24c2929e41 100644
--- a/tensorflow/core/platform/platform_strings.h
+++ b/tensorflow/core/platform/platform_strings.h
@@ -297,7 +297,7 @@ limitations under the License.
 #undef TF_PLAT_STR_LIST___x86_64__
 #define TF_PLAT_STR_LIST___x86_64__()
 #endif
-#if !defined(__powerpc64__)
+#if !defined(__powerpc64__) && !defined(__powerpc__)
 #undef TF_PLAT_STR_LIST___powerpc64__
 #define TF_PLAT_STR_LIST___powerpc64__()
 #endif
-- 
GitLab


From 09efdc5aa73127a1aca84b32f29c5a20f1bd097a Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 14 Nov 2018 14:00:24 -0800
Subject: [PATCH 0242/1554] Identified a few more situations where quantization
 range needs to be inferred. In some cases, it is impossible to infer these
 ranges.

---
 .../contrib/tensorrt/convert/convert_nodes.cc | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index ca06152fac..bc29bf8ffa 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1467,6 +1467,10 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
           *const_cast<nvinfer1::ITensor*>(tensor),
           nvinfer1::UnaryOperation::kNEG);
       TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+      // Since quantization ranges are symmetric, the same range as the input
+      // will work for the negation of the input.
+      params->converter->MarkQuantizationRangesAsInferrable(
+          const_cast<nvinfer1::ITensor*>(tensor), layer->getOutput(0));
       tensor = layer->getOutput(0);
     } else {
       TRT_ShapedWeights neg_weights =
@@ -1478,6 +1482,23 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
     }
   } else if (node_def.op() == "Div" || node_def.op() == "RealDiv") {
     if (swapped_inputs) {
+      // We need to infer the quantization range for this intermediate
+      // tensor.
+      // x -> [Recip] -> 1/x -> [Scale] -> s/x
+      //                  ^
+      //          need range for this
+      // We have the quantization scales for x and s/x - can we divide the scale
+      // for s/x by s? Only if it was a scalar...
+      // Because of this issue, fall back to BinaryTensorOpTensor if we are
+      // doing INT8 with no calibration. There is most likely no performance
+      // penalty by falling back here.
+      if (params->converter->precision_mode() == INT8MODE &&
+          !params->converter->use_calibration()) {
+        return tensorflow::errors::Unimplemented(
+            "Intermediate quantization range cannot be determined without"
+            " calibration. Falling back to BinaryTensorOpTensor for ",
+            node_def.op(), ", at ", node_def.name());
+      }
       scale_weights = weights;
       nvinfer1::IUnaryLayer* layer = params->converter->network()->addUnary(
           *const_cast<nvinfer1::ITensor*>(tensor),
@@ -2409,6 +2430,12 @@ tensorflow::Status ConvertBinary(OpConverterParams* params) {
         node_def.name());
   }
 
+  // TODO(tmorris): TRT plans to deprecate IScaleLayer and will replace it with
+  // IElementwiseLayer. At that point, we can remove BinaryTensorOpWeight. For
+  // now, the performance will be slightly better with IScaleLayer because it
+  // can be fused in more situations. However, most of the benefits of
+  // IScaleLayer are when the layer performs both a shift and a scale, which we
+  // don't do except for convolutions.
   // Try to convert into Scale layer first (for better performance)
   // Since scale layer supports restricted broadcast policy and op types, we
   // allow failure and try to handle it through Elementwise op
@@ -2451,6 +2478,18 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
 
   nvinfer1::IUnaryLayer* layer;
   if (node_def.op() == "Rsqrt") {
+    // We will need a quantization range for intermediate tensor
+    // if not using calibration.
+    // x -> [Sqrt] -> sqrt(x) -> [Recip] -> 1/sqrt(x)
+    //                   ^
+    //             need range here
+    if (params->converter->precision_mode() == INT8MODE &&
+        !params->converter->use_calibration()) {
+        return tensorflow::errors::Unimplemented(
+            "Intermediate quantization range cannot be determined without"
+            " calibration for Rsqrt, consider replacing with "
+            "Sqrt -> FakeQuant -> Reciprocal ops, at ", node_def.name());
+    }
     layer = params->converter->network()->addUnary(
         *const_cast<nvinfer1::ITensor*>(tensor),
         nvinfer1::UnaryOperation::kSQRT);
-- 
GitLab


From 087144dd8e85242dcb2129bda5314651399f995f Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Wed, 14 Nov 2018 13:57:50 -0800
Subject: [PATCH 0243/1554] Delete remaining instances of `num_replicas` usage.

PiperOrigin-RevId: 221505311
---
 tensorflow/python/training/distribute.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 4ed2fb1925..88ee5376fb 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1092,11 +1092,6 @@ class ReplicaContext(object):
     finally:
       _pop_per_thread_mode()
 
-  @property
-  def num_replicas(self):
-    """Returns number of replicas, for purposes of averaging across replicas."""
-    return self._distribution_strategy.num_replicas
-
   @property
   def num_replicas_in_sync(self):
     """Returns number of replicas over which gradients are aggregated."""
@@ -1204,10 +1199,6 @@ class _DefaultDistributionStrategy(DistributionStrategy):
   def value_container(self, value):
     return value
 
-  @property
-  def num_replicas(self):
-    return 1
-
   @property
   def num_replicas_in_sync(self):
     return 1
-- 
GitLab


From b7a2128b28a2755dba2a51d065c429c0cfb0a0f5 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Wed, 14 Nov 2018 14:21:52 -0800
Subject: [PATCH 0244/1554] Add missing build dependencies

PiperOrigin-RevId: 221510037
---
 tensorflow/compiler/jit/BUILD         | 1 +
 tensorflow/compiler/tf2xla/BUILD      | 2 ++
 tensorflow/compiler/xla/service/BUILD | 3 +++
 3 files changed, 6 insertions(+)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 5f25e4626a..f5ffe6ff6a 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -268,6 +268,7 @@ cc_library(
         "//tensorflow/core/kernels:variable_ops",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index e017141549..ab018d8507 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -204,6 +204,7 @@ cc_library(
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -221,6 +222,7 @@ cc_library(
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
     ],
     alwayslink = 1,
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 3fc804b9f6..779a700099 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -403,6 +403,7 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "@com_google_absl//absl/base",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/types:span",
     ],
@@ -1336,6 +1337,7 @@ cc_library(
         ":hlo",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -2347,6 +2349,7 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
-- 
GitLab


From 70148b32e354fd3c578d54a0d4ac7bcc7a7deea5 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Wed, 14 Nov 2018 14:41:34 -0800
Subject: [PATCH 0245/1554] Add methods to TPUClusterResolvers in preparation
 for Distribute Coordinators

PiperOrigin-RevId: 221513705
---
 .../python/training/tpu_cluster_resolver.py   | 116 ++++++++++++++----
 .../training/tpu_cluster_resolver_test.py     |  62 ++++++++++
 2 files changed, 153 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
index 1f6803a9ff..1a7a31c15c 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
@@ -24,6 +24,7 @@ from six.moves.urllib.request import Request
 from six.moves.urllib.request import urlopen
 
 from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
+from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import format_master_url
 from tensorflow.python.training import server_lib
 from tensorflow.python.util import compat
 
@@ -85,6 +86,8 @@ class TPUClusterResolver(ClusterResolver):
     return compat.as_bytes(resp.read())
 
   def _shouldResolve(self):
+    if isinstance(self._should_resolve_override, bool):
+      return self._should_resolve_override
     if (self._tpu == compat.as_bytes('') or
         self._tpu == compat.as_bytes('local') or
         self._tpu.startswith(compat.as_bytes('/bns')) or
@@ -181,7 +184,39 @@ class TPUClusterResolver(ClusterResolver):
       raise ValueError('Please provide a TPU Name to connect to.')
 
     self._tpu = compat.as_bytes(tpu)  # self._tpu is always bytes
-    self._job_name = job_name
+
+    # By default the task_type is 'worker` and the task_index is 0 (which is the
+    # first worker in the task).
+    self.task_type = job_name
+    self.task_index = 0
+
+    if tpu.startswith('grpc://'):
+      # Cloud environment, where we are using GRPC to communicate to TPUs.
+      self.environment = ''
+    elif tpu == 'local' or not tpu:
+      # Google environment, where the TPU is attached to the host.
+      self.environment = 'google'
+    elif tpu.startswith('/bns'):
+      # Google environment, where we reach the TPU through BNS.
+      self.environment = 'google'
+
+    # If TPU is in the Google environment or exists locally, we don't use any
+    # RPC layer.
+    if tpu.startswith('/bns') or tpu == 'local' or not tpu:
+      self.rpc_layer = None
+    else:
+      self.rpc_layer = 'grpc'
+
+    # Setting this overrides the return value of self._shouldResolve()
+    self._should_resolve_override = None
+
+    # We strip out the protocol if it is included, and override the
+    # shouldResolve function to never resolve. We are adding the protocol back
+    # in later in self.master().
+    if self.rpc_layer is not None and tpu.startswith(self.rpc_layer + '://'):
+      tpu = tpu[len(self.rpc_layer + '://'):]
+      self._tpu = tpu
+      self._should_resolve_override = False
 
     # Whether we should actually attempt to contact Cloud APIs
     should_resolve = self._shouldResolve()
@@ -222,7 +257,7 @@ class TPUClusterResolver(ClusterResolver):
     else:
       self._coordinator_address = coordinator_address
 
-  def master(self, task_type=None, task_index=None):
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
     """Get the Master string to be used for the session.
 
     In the normal case, this returns the grpc path (grpc://1.2.3.4:8470) of
@@ -234,8 +269,12 @@ class TPUClusterResolver(ClusterResolver):
     'grpc://10.240.1.2:8470' will be returned).
 
     Args:
-      task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
+      task_type: (Optional, string) The type of the TensorFlow task of the
+        master.
+      task_index: (Optional, integer) The index of the TensorFlow task of the
+        master.
+      rpc_layer: (Optional, string) The RPC protocol TensorFlow should use to
+        communicate with TPUs.
 
     Returns:
       string, the connection string to use when creating a session.
@@ -243,25 +282,34 @@ class TPUClusterResolver(ClusterResolver):
     Raises:
       ValueError: If none of the TPUs specified exists.
     """
-    if not self._shouldResolve():
-      return self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0]
-
-    cluster_spec = self.cluster_spec()
-    if task_type and task_index:
-      return cluster_spec.task_address(task_type, task_index)
-
-    job_tasks = cluster_spec.job_tasks(self._job_name)
-    if not job_tasks:
-      raise ValueError('No TPUs exists with the specified names exist.')
-
-    return 'grpc://' + job_tasks[0]
+    if self._shouldResolve():
+      # We are going to communicate with the Cloud TPU APIs to get a Cluster.
+      cluster_spec = self.cluster_spec()
+      if task_type is not None and task_index is not None:
+        # task_type and task_index is from the function parameter
+        master = cluster_spec.task_address(task_type, task_index)
+      elif self.task_type is not None and self.task_index is not None:
+        # task_type and task_index is from the object
+        master = cluster_spec.task_address(self.task_type, self.task_index)
+      else:
+        # by default we take the first item in the cluster with the right name
+        job_tasks = cluster_spec.job_tasks(self.task_type)
+        if not job_tasks:
+          raise ValueError('No TPUs with the specified names exist.')
+        master = job_tasks[0]
+    else:
+      if isinstance(self._tpu, (bytes, bytearray)):
+        master = self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0]
+      else:
+        master = self._tpu.split(_ENDPOINTS_SEPARATOR)[0]
+    return format_master_url(master, rpc_layer or self.rpc_layer)
 
   def get_master(self):
     return self.master()
 
   def get_job_name(self):
     if self._shouldResolve():
-      return self._job_name
+      return self.task_type
 
   def cluster_spec(self):
     """Returns a ClusterSpec object based on the latest TPU information.
@@ -313,18 +361,23 @@ class TPUClusterResolver(ClusterResolver):
         instance_url = '%s:%s' % (response['ipAddress'], response['port'])
         worker_list = [instance_url]
 
-      cluster_spec = {self._job_name: worker_list}
+      cluster_spec = {self.task_type: worker_list}
     else:
-      if not self._tpu.startswith(compat.as_bytes('grpc://')):
+      if self.rpc_layer is None:
         # Case 3.
         return None
       # Case 2.
-      cluster_spec = {
-          self._job_name: [
-              x[len(compat.as_bytes('grpc://')):]
-              for x in self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))
-          ]
-      }
+      tpus = []
+      for tpu in self._tpu.split(_ENDPOINTS_SEPARATOR):
+        # We are working around the fact that GKE environment variable that is
+        # supplied to us has the protocol string embedded in it, but we want
+        # to strip it out for the ClusterSpec.
+        if (self.rpc_layer is not None and
+            tpu.startswith(self.rpc_layer + '://')):
+          tpus.append(tpu[len(self.rpc_layer + '://'):])
+        else:
+          tpus.append(tpu)
+      cluster_spec = {self.task_type: tpus}
 
     if self._coordinator_address:
       # {1, 2}.a
@@ -332,6 +385,19 @@ class TPUClusterResolver(ClusterResolver):
 
     return server_lib.ClusterSpec(cluster_spec)
 
+  def num_accelerators_per_worker(self, session_config=None):
+    """Returns the number of TPU cores per worker.
+
+    This defaults to 8 for all current TPU configurations, and we do not need
+    to query any remote systems for this.
+
+    Args:
+      session_config: Unused. Not currently necessary to query anything as this
+        number is 8 for all TPU configurations.
+    """
+    del session_config  # Unused. Not necessary to query anything.
+    return 8
+
   def _start_local_server(self):
     address = self._requestComputeMetadata('instance/network-interfaces/0/ip')
     self._server = server_lib.Server(
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
index 478c82967b..365bd52ee2 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
@@ -132,6 +132,7 @@ class TPUClusterResolverTest(test.TestCase):
     }
     """ % tpu_cluster_resolver._coordinator_port
     self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto))
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
 
   @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
                      mock_request_compute_metadata)
@@ -157,6 +158,7 @@ class TPUClusterResolverTest(test.TestCase):
     job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
 
   @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
                      mock_request_compute_metadata)
@@ -226,6 +228,7 @@ class TPUClusterResolverTest(test.TestCase):
     job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
 
   def testNewNetworkEndpointFormat(self):
     tpu_map = {
@@ -304,6 +307,7 @@ class TPUClusterResolverTest(test.TestCase):
     }
     """ % tpu_cluster_resolver._coordinator_port
     self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto))
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470')
 
   def testPodResolutionNoCoordinator(self):
     tpu_map = {
@@ -350,6 +354,7 @@ class TPUClusterResolverTest(test.TestCase):
     }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470')
 
   def testGetMasterNoEntries(self):
     tpu_map = {}
@@ -464,5 +469,62 @@ class TPUClusterResolverTest(test.TestCase):
     self.assertEqual('https://{api}.internal/{apiVersion}',
                      TPUClusterResolver._environmentDiscoveryUrl())
 
+  def testEnvironmentAndRpcDetectionForGoogle(self):
+    tpu_cluster_resolver = TPUClusterResolver(tpu='/bns/ab/cd/ef')
+    self.assertEqual(tpu_cluster_resolver.environment, 'google')
+    self.assertEqual(tpu_cluster_resolver.rpc_layer, None)
+
+  def testEnvironmentAndRpcDetectionForGrpcString(self):
+    tpu_cluster_resolver = TPUClusterResolver(tpu='grpc://10.1.2.3:8470')
+    self.assertEqual(tpu_cluster_resolver.environment, '')
+    self.assertEqual(tpu_cluster_resolver.rpc_layer, 'grpc')
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
+
+  def testOverrideTaskTypeAndIndexAndGetMaster(self):
+    tpu_map = {
+        'projects/test-project/locations/us-central1-c/nodes/test-tpu-1': {
+            'health':
+                'HEALTHY',
+            'networkEndpoints': [
+                {
+                    'ipAddress': '10.2.3.4',
+                    'port': 8470,
+                },
+                {
+                    'ipAddress': '10.2.3.5',
+                    'port': 8470,
+                },
+                {
+                    'ipAddress': '10.2.3.6',
+                    'port': 8470,
+                },
+                {
+                    'ipAddress': '10.2.3.7',
+                    'port': 8470,
+                },
+            ]
+        }
+    }
+
+    tpu_cluster_resolver = TPUClusterResolver(
+        project='test-project',
+        zone='us-central1-c',
+        tpu='test-tpu-1',
+        coordinator_name=None,
+        credentials=None,
+        service=self.mock_service_client(tpu_map=tpu_map))
+
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470')
+
+    tpu_cluster_resolver.task_type = 'worker'
+    tpu_cluster_resolver.task_index = 3
+    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.7:8470')
+
+    self.assertEqual(
+        tpu_cluster_resolver.master(
+            task_type='worker', task_index=2, rpc_layer='test'),
+        'test://10.2.3.6:8470')
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 368861aedb8105847f05284ea9fac235f1e9fe7a Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 14 Nov 2018 14:46:59 -0800
Subject: [PATCH 0246/1554] Deprecate `SUM_BY_NONZERO_WEIGHTS`,
 `SUM_OVER_NONZERO_WEIGHTS`, `MEAN` loss reductions in V1 and remove them in
 V2.

PiperOrigin-RevId: 221514670
---
 tensorflow/python/ops/losses/losses_impl.py   | 37 ++++++++++++++-----
 .../v1/tensorflow.losses.-reduction.pbtxt     |  1 +
 .../v2/tensorflow.losses.-reduction.pbtxt     | 14 +------
 3 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 53c09ee8dd..ffcd12b38e 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -34,28 +34,45 @@ from tensorflow.python.util.deprecation import deprecated_argument_lookup
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("losses.Reduction")
-class Reduction(object):
+@tf_export("losses.Reduction", v1=[])
+class ReductionV2(object):
   """Types of loss reduction.
 
   Contains the following values:
   `NONE`: Un-reduced weighted losses with the same shape as input.
   `SUM`: Scalar sum of weighted losses.
-  `MEAN`: Scalar `SUM` divided by sum of weights.
   `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
-  `SUM_OVER_NONZERO_WEIGHTS`: Scalar `SUM` divided by number of non-zero
-     weights.
-  `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`.
   """
 
   NONE = "none"
-
   SUM = "weighted_sum"
+  SUM_OVER_BATCH_SIZE = "weighted_sum_over_batch_size"
 
-  MEAN = "weighted_mean"
+  @classmethod
+  def all(cls):
+    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
+
+  @classmethod
+  def validate(cls, key):
+    if key not in cls.all():
+      raise ValueError("Invalid Reduction Key %s." % key)
 
-  SUM_OVER_BATCH_SIZE = "weighted_sum_over_batch_size"
 
+@tf_export(v1=["losses.Reduction"])
+class Reduction(ReductionV2):
+  """Types of loss reduction.
+
+  Contains the following values:
+  `NONE`: Un-reduced weighted losses with the same shape as input.
+  `SUM`: Scalar sum of weighted losses.
+  `MEAN`: Scalar `SUM` divided by sum of weights. DEPRECATED.
+  `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+  `SUM_OVER_NONZERO_WEIGHTS`: Scalar `SUM` divided by number of non-zero
+     weights. DEPRECATED.
+  `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`.
+  """
+
+  MEAN = "weighted_mean"
   SUM_BY_NONZERO_WEIGHTS = "weighted_sum_by_nonzero_weights"
   SUM_OVER_NONZERO_WEIGHTS = SUM_BY_NONZERO_WEIGHTS
 
@@ -72,7 +89,7 @@ class Reduction(object):
   @classmethod
   def validate(cls, key):
     if key not in cls.all():
-      raise ValueError("Invalid ReductionKey %s." % key)
+      raise ValueError("Invalid Reduction Key %s." % key)
 
 
 def _safe_div(numerator, denominator, name="value"):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt
index 258ad5047e..b2adb52660 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.losses.Reduction"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.Reduction\'>"
+  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "MEAN"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
index 258ad5047e..6a44e4ce66 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
@@ -1,11 +1,7 @@
 path: "tensorflow.losses.Reduction"
 tf_class {
-  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.Reduction\'>"
+  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "MEAN"
-    mtype: "<type \'str\'>"
-  }
   member {
     name: "NONE"
     mtype: "<type \'str\'>"
@@ -14,18 +10,10 @@ tf_class {
     name: "SUM"
     mtype: "<type \'str\'>"
   }
-  member {
-    name: "SUM_BY_NONZERO_WEIGHTS"
-    mtype: "<type \'str\'>"
-  }
   member {
     name: "SUM_OVER_BATCH_SIZE"
     mtype: "<type \'str\'>"
   }
-  member {
-    name: "SUM_OVER_NONZERO_WEIGHTS"
-    mtype: "<type \'str\'>"
-  }
   member_method {
     name: "__init__"
   }
-- 
GitLab


From 6e5c26a2ab63d51ec16d88a6bdbb27c3e5f6f378 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Wed, 14 Nov 2018 14:47:32 -0800
Subject: [PATCH 0247/1554] Add new flag zero_output_for_mask to RNN.

Current RNN layer will use previous timestep output as the output if the step is masked. It is useful for certain case, eg propagate the output to final timestep when data is right padded, and user only want the output for final timestep. However, if user cares about output for all timestep, then use zero filed output is more reasonable when input is masked.

Since this is change of behavior, added a new flag in RNN as well as backend.rnn().

PiperOrigin-RevId: 221514766
---
 tensorflow/python/keras/backend.py            | 33 ++++++++++++++-----
 tensorflow/python/keras/layers/recurrent.py   |  8 ++++-
 .../python/keras/layers/recurrent_test.py     | 26 +++++++++++++++
 tensorflow/python/keras/layers/wrappers.py    |  4 +++
 .../python/keras/layers/wrappers_test.py      | 28 ++++++++++++++++
 .../golden/v1/tensorflow.keras.backend.pbtxt  |  2 +-
 .../golden/v2/tensorflow.keras.backend.pbtxt  |  2 +-
 7 files changed, 91 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index aeb9bd4cb0..6174478e59 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3237,7 +3237,8 @@ def rnn(step_function,
         constants=None,
         unroll=False,
         input_length=None,
-        time_major=False):
+        time_major=False,
+        zero_output_for_mask=False):
   """Iterates over the time dimension of a tensor.
 
   Arguments:
@@ -3275,7 +3276,9 @@ def rnn(step_function,
           RNN calculation. However, most TensorFlow data is batch-major, so by
           default this function accepts input and emits output in batch-major
           form.
-
+      zero_output_for_mask: Boolean. If True, the output for masked timestep
+          will be zeros, whereas in the False case, output from previous
+          timestep is returned.
   Returns:
       A tuple, `(last_output, outputs, new_states)`.
           last_output: the latest output of the rnn, of shape `(samples, ...)`
@@ -3327,13 +3330,13 @@ def rnn(step_function,
   # So we need to broadcast the mask to match the shape of inputs.
   # That's what the tile call does, it just repeats the mask along its
   # second dimension n times.
-  def _expand_mask(mask_t, input_t):
+  def _expand_mask(mask_t, input_t, fixed_dim=1):
     assert not nest.is_sequence(mask_t)
     assert not nest.is_sequence(input_t)
     rank_diff = len(input_t.shape) - len(mask_t.shape)
     for _ in range(rank_diff):
       mask_t = array_ops.expand_dims(mask_t)
-    expand_dims = [1] + input_t.shape.as_list()[1:]
+    expand_dims = [1] * fixed_dim + input_t.shape.as_list()[fixed_dim:]
     return array_ops.tile(mask_t, expand_dims)
 
   if unroll:
@@ -3392,6 +3395,17 @@ def rnn(step_function,
       last_output = successive_outputs[-1]
       new_states = successive_states[-1]
       outputs = array_ops.stack(successive_outputs)
+
+      if zero_output_for_mask:
+        last_output = array_ops.where(
+            _expand_mask(mask_list[-1], last_output),
+            last_output,
+            zeros_like(last_output))
+        outputs = array_ops.where(
+            _expand_mask(mask, outputs, fixed_dim=2),
+            outputs,
+            zeros_like(outputs))
+
     else:
       for i in range(time_steps):
         inp = _get_input_tensor(i)
@@ -3485,11 +3499,12 @@ def rnn(step_function,
                                            tuple(states) + tuple(constants))
         # mask output
         flat_output = nest.flatten(output)
-        flat_previous_output = nest.flatten(prev_output)
+        flat_mask_output = (flat_zero_output if zero_output_for_mask
+                            else nest.flatten(prev_output))
         tiled_mask_t = tuple(_expand_mask(mask_t, o) for o in flat_output)
         flat_new_output = tuple(
-            array_ops.where(m, o, po) for m, o, po in zip(
-                tiled_mask_t, flat_output, flat_previous_output))
+            array_ops.where(m, o, zo) for m, o, zo in zip(
+                tiled_mask_t, flat_output, flat_mask_output))
 
         # mask states
         flat_state = nest.flatten(states)
@@ -3498,8 +3513,8 @@ def rnn(step_function,
           new_state.set_shape(state.shape)
         tiled_mask_t = tuple(_expand_mask(mask_t, s) for s in flat_state)
         flat_final_state = tuple(
-            array_ops.where(m, o, po)
-            for m, o, po in zip(tiled_mask_t, flat_new_state, flat_state))
+            array_ops.where(m, s, ps)
+            for m, s, ps in zip(tiled_mask_t, flat_new_state, flat_state))
         new_states = nest.pack_sequence_as(new_states, flat_final_state)
 
         output_ta_t = tuple(
diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 979187c719..d04533815e 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -446,6 +446,9 @@ class RNN(Layer):
                        'an attribute `state_size` '
                        '(tuple of integers, '
                        'one integer per RNN state).')
+    # If True, the output for masked timestep will be zeros, whereas in the
+    # False case, output from previous timestep is returned for masked timestep.
+    self.zero_output_for_mask = kwargs.pop('zero_output_for_mask', False)
     super(RNN, self).__init__(**kwargs)
     self.cell = cell
     if isinstance(cell, checkpointable.CheckpointableBase):
@@ -829,7 +832,8 @@ class RNN(Layer):
         mask=mask,
         unroll=self.unroll,
         input_length=timesteps,
-        time_major=self.time_major)
+        time_major=self.time_major,
+        zero_output_for_mask=self.zero_output_for_mask)
     if self.stateful:
       updates = []
       for i in range(len(states)):
@@ -923,6 +927,8 @@ class RNN(Layer):
     }
     if self._num_constants is not None:
       config['num_constants'] = self._num_constants
+    if self.zero_output_for_mask:
+      config['zero_output_for_mask'] = self.zero_output_for_mask
 
     cell_config = self.cell.get_config()
     config['cell'] = {
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index bb14a7a505..7172571f7c 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -1079,6 +1079,32 @@ class RNNTest(test.TestCase):
     # Expect last output to be the same as last output before masking
     self.assertAllClose(y_np, x_np[:, 1, :])
 
+  def test_zero_output_for_masking(self):
+
+    for unroll in [True, False]:
+      cell = keras.layers.SimpleRNNCell(5)
+      x = keras.Input((5, 5))
+      mask = keras.layers.Masking()
+      layer = keras.layers.RNN(
+          cell, return_sequences=True, zero_output_for_mask=True, unroll=unroll)
+      masked_input = mask(x)
+      y = layer(masked_input)
+      model = keras.models.Model(x, y)
+      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
+                    loss='mse')
+
+      np_x = np.ones((6, 5, 5))
+      result_1 = model.predict(np_x)
+
+      # set the time 4 and 5 for last record to be zero (masked).
+      np_x[5, 3:] = 0
+      result_2 = model.predict(np_x)
+
+      # expect the result_2 has same output, except the time 4,5 for last
+      # record.
+      result_1[5, 3:] = 0
+      self.assertAllClose(result_1, result_2)
+
 
 class Minimal2DRNNCell(keras.layers.Layer):
   """The minimal 2D RNN cell is a simple combination of 2 1-D RNN cell.
diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index d40f7a2e80..27419a1ffc 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -389,6 +389,10 @@ class Bidirectional(Wrapper):
       raise ValueError('Invalid merge mode. '
                        'Merge mode should be one of '
                        '{"sum", "mul", "ave", "concat", None}')
+    if getattr(layer, 'zero_output_for_mask', None) is not None:
+      # Force the zero_output_for_mask to be True if it presents.
+      layer.zero_output_for_mask = True
+
     self.forward_layer = copy.copy(layer)
     config = layer.get_config()
     config['go_backwards'] = not config['go_backwards']
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 9584b0186c..bbafa96aab 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -635,6 +635,34 @@ class BidirectionalTest(test.TestCase):
       y_np_3 = model.predict([x_np, s_fw_np, s_bk_np, c_np])
       self.assertAllClose(y_np, y_np_3, atol=1e-4)
 
+  def test_Bidirectional_with_masking(self):
+    rnn = keras.layers.LSTM
+    samples = 2
+    dim = 5
+    timesteps = 3
+    units = 3
+    merge_mode = 'concat'
+    x = np.random.rand(samples, timesteps, dim)
+    # clear the first record's timestep 2, and expect the output of timestep 2
+    # is also 0s.
+    x[0, 2] = 0
+
+    with self.cached_session():
+      inputs = keras.Input((timesteps, dim))
+      masked_inputs = keras.layers.Masking()(inputs)
+      wrapped = keras.layers.Bidirectional(
+          rnn(units, return_sequences=True),
+          merge_mode=merge_mode)
+      outputs = _to_list(wrapped(masked_inputs, training=True))
+      self.assertEqual(len(outputs), 1)
+      self.assertEqual(outputs[0].get_shape().as_list(),
+                       [None, timesteps, units * 2])
+
+      model = keras.Model(inputs, outputs)
+      y = _to_list(model.predict(x))
+      self.assertEqual(len(y), 1)
+      self.assertAllClose(y[0][0, 2], np.zeros(units * 2))
+
 
 def _to_list(ls):
   if isinstance(ls, list):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
index b0e5d2bde7..85e2a71ff3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
@@ -398,7 +398,7 @@ tf_module {
   }
   member_method {
     name: "rnn"
-    argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\'], "
+    argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\', \'zero_output_for_mask\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\', \'False\'], "
   }
   member_method {
     name: "round"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
index b30778b2a0..38105c540c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
@@ -394,7 +394,7 @@ tf_module {
   }
   member_method {
     name: "rnn"
-    argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\'], "
+    argspec: "args=[\'step_function\', \'inputs\', \'initial_states\', \'go_backwards\', \'mask\', \'constants\', \'unroll\', \'input_length\', \'time_major\', \'zero_output_for_mask\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'False\', \'None\', \'False\', \'False\'], "
   }
   member_method {
     name: "round"
-- 
GitLab


From 958a14b27f45390f5e3ab3a234dde3c18fd60241 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Wed, 14 Nov 2018 15:00:18 -0800
Subject: [PATCH 0248/1554] Update tf.pad for v2 API.

PiperOrigin-RevId: 221517062
---
 tensorflow/python/ops/array_ops.py            | 60 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  5 +-
 3 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 8925af6664..d0efbdbaf8 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2005,7 +2005,65 @@ def sparse_placeholder(dtype, shape=None, name=None):
 # pylint: enable=redefined-outer-name
 
 
-@tf_export("pad")
+@tf_export("pad", v1=[])
+def pad_v2(tensor, paddings, mode="CONSTANT", constant_values=0, name=None):
+  """Pads a tensor.
+
+  This operation pads a `tensor` according to the `paddings` you specify.
+  `paddings` is an integer tensor with shape `[n, 2]`, where n is the rank of
+  `tensor`. For each dimension D of `input`, `paddings[D, 0]` indicates how
+  many values to add before the contents of `tensor` in that dimension, and
+  `paddings[D, 1]` indicates how many values to add after the contents of
+  `tensor` in that dimension. If `mode` is "REFLECT" then both `paddings[D, 0]`
+  and `paddings[D, 1]` must be no greater than `tensor.dim_size(D) - 1`. If
+  `mode` is "SYMMETRIC" then both `paddings[D, 0]` and `paddings[D, 1]` must be
+  no greater than `tensor.dim_size(D)`.
+
+  The padded size of each dimension D of the output is:
+
+  `paddings[D, 0] + tensor.dim_size(D) + paddings[D, 1]`
+
+  For example:
+
+  ```python
+  t = tf.constant([[1, 2, 3], [4, 5, 6]])
+  paddings = tf.constant([[1, 1,], [2, 2]])
+  # 'constant_values' is 0.
+  # rank of 't' is 2.
+  tf.pad(t, paddings, "CONSTANT")  # [[0, 0, 0, 0, 0, 0, 0],
+                                   #  [0, 0, 1, 2, 3, 0, 0],
+                                   #  [0, 0, 4, 5, 6, 0, 0],
+                                   #  [0, 0, 0, 0, 0, 0, 0]]
+
+  tf.pad(t, paddings, "REFLECT")  # [[6, 5, 4, 5, 6, 5, 4],
+                                  #  [3, 2, 1, 2, 3, 2, 1],
+                                  #  [6, 5, 4, 5, 6, 5, 4],
+                                  #  [3, 2, 1, 2, 3, 2, 1]]
+
+  tf.pad(t, paddings, "SYMMETRIC")  # [[2, 1, 1, 2, 3, 3, 2],
+                                    #  [2, 1, 1, 2, 3, 3, 2],
+                                    #  [5, 4, 4, 5, 6, 6, 5],
+                                    #  [5, 4, 4, 5, 6, 6, 5]]
+  ```
+
+  Args:
+    tensor: A `Tensor`.
+    paddings: A `Tensor` of type `int32`.
+    mode: One of "CONSTANT", "REFLECT", or "SYMMETRIC" (case-insensitive)
+    constant_values: In "CONSTANT" mode, the scalar pad value to use. Must be
+      same type as `tensor`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `tensor`.
+
+  Raises:
+    ValueError: When mode is not one of "CONSTANT", "REFLECT", or "SYMMETRIC".
+  """
+  return pad(tensor, paddings, mode, name, constant_values)
+
+
+@tf_export(v1=["pad"])
 def pad(tensor, paddings, mode="CONSTANT", name=None, constant_values=0):  # pylint: disable=invalid-name
   """Pads a tensor.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 665b3f5d81..e8435a401a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -930,7 +930,7 @@ tf_module {
   }
   member_method {
     name: "pad"
-    argspec: "args=[\'tensor\', \'paddings\', \'mode\', \'name\', \'constant_values\'], varargs=None, keywords=None, defaults=[\'CONSTANT\', \'None\', \'0\'], "
+    argspec: "args=[\'tensor\', \'paddings\', \'mode\', \'constant_values\', \'name\'], varargs=None, keywords=None, defaults=[\'CONSTANT\', \'0\', \'None\'], "
   }
   member_method {
     name: "parallel_stack"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 232ad48ac4..fbf4320e94 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -59,10 +59,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # for safety, if positional arguments are used. If you have reversed the
     # positional arguments yourself, this could do the wrong thing.
     self.function_reorders = {
-        "tf.convert_to_tensor": ["value", "dtype", "preferred_dtype", "name"],
-        "tf.argmin": ["input", "axis", "output_type", "name"],
         "tf.argmax": ["input", "axis", "output_type", "name"],
+        "tf.argmin": ["input", "axis", "output_type", "name"],
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
+        "tf.convert_to_tensor": ["value", "dtype", "preferred_dtype", "name"],
+        "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
     }
 
     # Specially handled functions.
-- 
GitLab


From bd55d0dedb918a94e0ce54aaa53a24d334eefe06 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 14 Nov 2018 15:33:36 -0800
Subject: [PATCH 0249/1554] Export V2 Estimator without export_savedmodel
 function

PiperOrigin-RevId: 221523075
---
 .../v1/tensorflow.estimator.-baseline-classifier.pbtxt      | 1 +
 .../v1/tensorflow.estimator.-baseline-estimator.pbtxt       | 1 +
 .../v1/tensorflow.estimator.-baseline-regressor.pbtxt       | 1 +
 .../v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt | 1 +
 .../v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt  | 1 +
 .../golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt  | 1 +
 .../golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt   | 1 +
 ...orflow.estimator.-d-n-n-linear-combined-classifier.pbtxt | 1 +
 ...sorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt | 1 +
 ...sorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt | 1 +
 .../golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt   | 1 +
 .../api/golden/v1/tensorflow.estimator.-estimator.pbtxt     | 1 +
 .../golden/v1/tensorflow.estimator.-linear-classifier.pbtxt | 1 +
 .../golden/v1/tensorflow.estimator.-linear-estimator.pbtxt  | 1 +
 .../golden/v1/tensorflow.estimator.-linear-regressor.pbtxt  | 1 +
 .../v2/tensorflow.estimator.-baseline-classifier.pbtxt      | 1 +
 .../v2/tensorflow.estimator.-baseline-estimator.pbtxt       | 1 +
 .../v2/tensorflow.estimator.-baseline-regressor.pbtxt       | 1 +
 .../v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt | 1 +
 .../v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt  | 1 +
 .../golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt  | 1 +
 .../golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt   | 1 +
 ...orflow.estimator.-d-n-n-linear-combined-classifier.pbtxt | 1 +
 ...sorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt | 1 +
 ...sorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt | 1 +
 .../golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt   | 1 +
 .../api/golden/v2/tensorflow.estimator.-estimator.pbtxt     | 6 +-----
 .../golden/v2/tensorflow.estimator.-linear-classifier.pbtxt | 1 +
 .../golden/v2/tensorflow.estimator.-linear-estimator.pbtxt  | 1 +
 .../golden/v2/tensorflow.estimator.-linear-regressor.pbtxt  | 1 +
 30 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
index 32b84e90ce..c50527ed63 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt
index 94933e7ffd..d218773dfc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
index db7776b5bf..ee37a060ae 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt
index d530c71482..970abd8622 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees.BoostedTreesClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees._BoostedTreesBase\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt
index 4703c0f561..b5bbad965e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees.BoostedTreesRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees._BoostedTreesBase\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
index ce6040d0f2..22c9e68da1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt
index 4635a1544c..85ff5a4fb1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index e85007e16e..3f5f9ec724 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
index a23f5daeac..ac13dad2d4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index 8a55bb835f..6e4018122b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 2c4128ec48..93fa79403b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt
index 9d270a87ab..eee57462fb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.Estimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
index 4b5de2e245..1cadbbca57 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt
index 3d6b03098a..023edec819 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
index 0d1510e9ab..95226b3bcf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
index 32b84e90ce..c50527ed63 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
index 94933e7ffd..d218773dfc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
index db7776b5bf..ee37a060ae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.BaselineRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt
index d530c71482..970abd8622 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees.BoostedTreesClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees._BoostedTreesBase\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt
index 4703c0f561..b5bbad965e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt
@@ -3,6 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees.BoostedTreesRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.boosted_trees._BoostedTreesBase\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
index ce6040d0f2..22c9e68da1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
index 4635a1544c..85ff5a4fb1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index e85007e16e..3f5f9ec724 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifier\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
index a23f5daeac..ac13dad2d4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index 8a55bb835f..6e4018122b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNLinearCombinedRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 2c4128ec48..93fa79403b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.DNNRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt
index 9d270a87ab..71531fd217 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.Estimator"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
@@ -34,10 +34,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
index 4acbff2cff..72c226b25d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
index 3d6b03098a..023edec819 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearEstimator"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearEstimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
index 0d1510e9ab..95226b3bcf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.estimator.LinearRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressor\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "config"
-- 
GitLab


From 785b7beec1bb9d21483fb23aeda04960afdf18ec Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 14 Nov 2018 15:35:58 -0800
Subject: [PATCH 0250/1554] Fix to remove the need to pass the batch size when
 using distribution strategy when using predict on a numpy array if the number
 of samples is less than the default batch size.

PiperOrigin-RevId: 221523550
---
 .../contrib/distribute/python/keras_test.py   | 19 +++------------
 .../engine/distributed_training_utils.py      | 17 +++++++++++++-
 tensorflow/python/keras/engine/training.py    | 23 ++++++++++---------
 3 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 790b683e58..c16fc52c9a 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -220,7 +220,7 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
   # TODO(b/118776054): Use global batch size for Keras/DS support.
   use_per_core_batch_size = (
       with_distribution and
-      with_distribution.__class__.__name__ != 'TPUStrategy')
+      not isinstance(with_distribution, tpu_strategy.TPUStrategy))
   if use_per_core_batch_size:
     batch_size //= with_distribution.num_replicas_in_sync
 
@@ -237,19 +237,7 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
         'x': x_train,
         'y': y_train,
     }
-    # TODO(b/119318587): We should not require batch_size when distribution
-    # is enabled.
-    if with_distribution:
-      if use_per_core_batch_size:
-        predict_batch_size = (
-            len(x_predict) // with_distribution.num_replicas_in_sync)
-      else:
-        predict_batch_size = len(x_predict)
-    else:
-      predict_batch_size = None
-
     predict_inputs = {
-        'batch_size': predict_batch_size,
         'x': np.array(x_predict, dtype=np.float32),
     }
   else:
@@ -280,7 +268,6 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
     predict_dataset = batch_wrapper(predict_dataset,
                                     predict_batch_size, with_distribution)
     predict_inputs = {
-        'batch_size': None,
         'steps': 1,
         'x': predict_dataset,
     }
@@ -486,8 +473,8 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                                      '/device:CPU:0',
                                                      '/device:GPU:1'])
 
-      with self.assertRaisesRegexp(ValueError, 'Please specify a batch_size '
-                                               'that is smaller than'):
+      with self.assertRaisesRegexp(ValueError, 'The number of samples is not '
+                                   'divisible by batch size.'):
         # The batch size(128) is larger than the number of input
         # samples(64).
         distributed_training_utils.get_input_batch_params(inputs,
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 7e3147bb32..7d38df410f 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -364,9 +364,24 @@ def get_input_batch_params(first_x_value, batch_size, distribution_strategy):
     ValueError: If the number of batches or steps evaluates to 0.
 
   """
+  if batch_size is None:
+    # Default the global batch size to the minimum of 32 and the size of
+    # the numpy array. 32 is chosen to guarantee backward compatibility.
+    batch_size = min(first_x_value.shape[0], 32)
+    if distribution_strategy.__class__.__name__ != 'TPUStrategy':
+      if batch_size % distribution_strategy.num_replicas_in_sync:
+        raise ValueError(
+            'The batch size (%s) could not be sharded evenly across the sync '
+            'replicas (%s) in the distribution strategy.' % (
+                batch_size, distribution_strategy.num_replicas_in_sync))
+      batch_size = batch_size // distribution_strategy.num_replicas_in_sync
+
+  # Calculate number of global batches
+  if first_x_value.shape[0] % batch_size:
+    raise ValueError('The number of samples is not divisible by batch size.')
   num_batches = first_x_value.shape[0] // batch_size
   if not num_batches:
-    raise ValueError('Please specify a batch_size that is smaller than'
+    raise ValueError('Please specify a batch_size that is smaller than '
                      'the number of input samples %d.' % first_x_value.shape[0])
   # TODO(b/118776054): Use global batch size for Keras/DS support.
   # The Keras API supports using the global batch size which is currently only
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 4c56e1eea7..3562060707 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1567,9 +1567,6 @@ class Model(Network):
           shuffle=shuffle,
           initial_epoch=initial_epoch)
 
-    # Backwards compatibility
-    if batch_size is None and steps_per_epoch is None:
-      batch_size = 32
     # Legacy support
     if 'nb_epoch' in kwargs:
       logging.warning(
@@ -1591,6 +1588,10 @@ class Model(Network):
         steps_per_epoch = distributed_training_utils.get_input_batch_params(
             first_x_value, batch_size, self._distribution_strategy)
 
+    # Backwards compatibility
+    if batch_size is None and steps_per_epoch is None:
+      batch_size = 32
+
     x, y, sample_weights = self._standardize_user_data(
         x,
         y,
@@ -1798,10 +1799,6 @@ class Model(Network):
           workers=workers,
           use_multiprocessing=use_multiprocessing)
 
-    # Backwards compatibility.
-    if batch_size is None and steps is None:
-      batch_size = 32
-
     # Validate and standardize user data.
     if self._distribution_strategy:
       distributed_training_utils.validate_inputs(
@@ -1811,6 +1808,10 @@ class Model(Network):
         steps = distributed_training_utils.get_input_batch_params(
             first_x_value, batch_size, self._distribution_strategy)
 
+    # Backwards compatibility.
+    if batch_size is None and steps is None:
+      batch_size = 32
+
     x, y, sample_weights = self._standardize_user_data(
         x,
         y,
@@ -1909,10 +1910,6 @@ class Model(Network):
           workers=workers,
           use_multiprocessing=use_multiprocessing)
 
-    # Backwards compatibility.
-    if batch_size is None and steps is None:
-      batch_size = 32
-
     if self._distribution_strategy:
       distributed_training_utils.validate_inputs(
           x, None, self._distribution_strategy)
@@ -1920,6 +1917,10 @@ class Model(Network):
       if isinstance(first_x_value, np.ndarray) and not steps:
         steps = distributed_training_utils.get_input_batch_params(
             first_x_value, batch_size, self._distribution_strategy)
+    # Backwards compatibility.
+    if batch_size is None and steps is None:
+      batch_size = 32
+
     # Validate and standardize user data.
     # TODO(anjalisridhar): We don't pass batch_size here for some reason. This
     # means that we end up calculating it twice which we should avoid.
-- 
GitLab


From 4ed1ea78f4df736e2647892918de121edd0137b9 Mon Sep 17 00:00:00 2001
From: Nick Hamatake <hamatake@google.com>
Date: Wed, 14 Nov 2018 15:38:05 -0800
Subject: [PATCH 0251/1554] Internal change

PiperOrigin-RevId: 221523998
---
 third_party/icu/data/{BUILD => BUILD.bazel} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename third_party/icu/data/{BUILD => BUILD.bazel} (100%)

diff --git a/third_party/icu/data/BUILD b/third_party/icu/data/BUILD.bazel
similarity index 100%
rename from third_party/icu/data/BUILD
rename to third_party/icu/data/BUILD.bazel
-- 
GitLab


From 65e1e1ebbc86a3c4740168445dae4c2075ea2932 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 14 Nov 2018 15:40:21 -0800
Subject: [PATCH 0252/1554] Small changes to API generation to help creation of
 virtual pip.

- Underscore some imports like _print_function so we don't have to
  delete them. Deleting them doesn't work well since they get added
  to our __all__ list before being deleted which makes "import *" fail.
- Give a more unique name to some generated target in the api genrule.
  Doing this since I want multiple genrules that generate the same version
  of the API. (a compat_v1 and root_v1 for example).

PiperOrigin-RevId: 221524470
---
 tensorflow/python/tools/api/generator/api_gen.bzl          | 2 +-
 tensorflow/python/tools/api/generator/create_python_api.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/tools/api/generator/api_gen.bzl b/tensorflow/python/tools/api/generator/api_gen.bzl
index 2e5d875a58..8ee5e0ac94 100644
--- a/tensorflow/python/tools/api/generator/api_gen.bzl
+++ b/tensorflow/python/tools/api/generator/api_gen.bzl
@@ -60,7 +60,7 @@ def gen_api_init_files(
         root_init_template_flag = "--root_init_template=$(location " + root_init_template + ")"
 
     primary_package = packages[0]
-    api_gen_binary_target = ("create_" + primary_package + "_api_%d") % api_version
+    api_gen_binary_target = ("create_" + primary_package + "_api_%d_%s") % (api_version, name)
     native.py_binary(
         name = api_gen_binary_target,
         srcs = ["//tensorflow/python/tools/api/generator:create_python_api.py"],
diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py
index f625803421..be988c572c 100644
--- a/tensorflow/python/tools/api/generator/create_python_api.py
+++ b/tensorflow/python/tools/api/generator/create_python_api.py
@@ -45,10 +45,10 @@ _GENERATED_FILE_HEADER = """# This file is MACHINE GENERATED! Do not edit.
 \"\"\"%s
 \"\"\"
 
-from __future__ import print_function
+from __future__ import print_function as _print_function
 
 """
-_GENERATED_FILE_FOOTER = '\n\ndel print_function\n'
+_GENERATED_FILE_FOOTER = '\n\ndel _print_function\n'
 
 
 class SymbolExposedTwiceError(Exception):
-- 
GitLab


From bba6d55aa28b14bf59c09897290e6aaf9bf07a6d Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Wed, 14 Nov 2018 15:49:44 -0800
Subject: [PATCH 0253/1554] Update to new TF Lite path

PiperOrigin-RevId: 221526017
---
 tensorflow/lite/lib_package/create_ios_frameworks.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/lib_package/create_ios_frameworks.sh b/tensorflow/lite/lib_package/create_ios_frameworks.sh
index fa466ed5bc..7901655b7c 100755
--- a/tensorflow/lite/lib_package/create_ios_frameworks.sh
+++ b/tensorflow/lite/lib_package/create_ios_frameworks.sh
@@ -30,7 +30,7 @@ echo "Creating target Headers directories"
 mkdir -p $FW_DIR_TFLITE_HDRS
 
 echo "Headers, populating: TensorFlow Lite"
-cd $TFLITE_DIR/../../..
+cd $TFLITE_DIR/../..
 
 find tensorflow/lite -name '*.h' \
     -not -path 'tensorflow/lite/tools/*' \
@@ -51,10 +51,10 @@ cd $FW_DIR_TFLITE_HDRS
 tar xf tmp.tar
 rm -f tmp.tar
 
-cd $TFLITE_DIR/../../..
+cd $TFLITE_DIR/../..
 echo "Generate master LICENSE file and copy to target"
 bazel build //tensorflow/tools/lib_package:clicenses_generate
-cp $TFLITE_DIR/../../../bazel-genfiles/tensorflow/tools/lib_package/include/tensorflow/c/LICENSE \
+cp $TFLITE_DIR/../../bazel-genfiles/tensorflow/tools/lib_package/include/tensorflow/c/LICENSE \
    $FW_DIR_TFLITE
 
 echo "Copying static libraries"
-- 
GitLab


From b8fbb495b6dcb3e993f310b296e1c70a6a230a37 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 14 Nov 2018 16:06:02 -0800
Subject: [PATCH 0254/1554] [TF:XLA] Bump open source abseil revision to
 7b46e1d31a6b08b1c6da2a13e7b151a20446fa07

PiperOrigin-RevId: 221528980
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0c894c9634..4db44278f3 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "3cf6132129ba87f0781c383bfaf381b7174b5818e81fffcc5d04bb451154f0f2",
-        strip_prefix = "abseil-cpp-f95179062eb65ce40895cc76f1398cce25394369",
+        sha256 = "f8536445cd480be7ec89bac19deaf766a1038330470fb0b469a98bce09e5c5ce",
+        strip_prefix = "abseil-cpp-7b46e1d31a6b08b1c6da2a13e7b151a20446fa07",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/7b46e1d31a6b08b1c6da2a13e7b151a20446fa07.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/7b46e1d31a6b08b1c6da2a13e7b151a20446fa07.tar.gz",
         ],
     )
 
-- 
GitLab


From ba5cd08179fecabbec3cbb21fce8232e667df72a Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 14 Nov 2018 16:06:13 -0800
Subject: [PATCH 0255/1554] Remove random.get_seed from the API of TF 2.0.

PiperOrigin-RevId: 221529008
---
 tensorflow/python/framework/random_seed.py             | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt | 4 ----
 tensorflow/tools/compatibility/renames_v2.py           | 3 ++-
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py
index 777bb2fe8c..88c8b878dd 100644
--- a/tensorflow/python/framework/random_seed.py
+++ b/tensorflow/python/framework/random_seed.py
@@ -34,7 +34,7 @@ def _truncate_seed(seed):
   return seed % _MAXINT32  # Truncate to fit into 32-bit integer
 
 
-@tf_export('random.get_seed', v1=['random.get_seed', 'get_seed'])
+@tf_export(v1=['random.get_seed', 'get_seed'])
 @deprecation.deprecated_endpoints('get_seed')
 def get_seed(op_seed):
   """Returns the local seeds an operation should use given an op-specific seed.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index 160c09798d..bfc38fbf3c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -4,10 +4,6 @@ tf_module {
     name: "gamma"
     argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
   }
-  member_method {
-    name: "get_seed"
-    argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "log_uniform_candidate_sampler"
     argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5975eadead..06a33ef41e 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -158,7 +158,7 @@ renames = {
     'tf.floordiv': 'tf.math.floordiv',
     'tf.get_default_session': 'tf.compat.v1.get_default_session',
     'tf.get_local_variable': 'tf.compat.v1.get_local_variable',
-    'tf.get_seed': 'tf.random.get_seed',
+    'tf.get_seed': 'tf.compat.v1.get_seed',
     'tf.get_session_handle': 'tf.compat.v1.get_session_handle',
     'tf.get_session_tensor': 'tf.compat.v1.get_session_tensor',
     'tf.get_variable': 'tf.compat.v1.get_variable',
@@ -293,6 +293,7 @@ renames = {
     'tf.qr': 'tf.linalg.qr',
     'tf.quantize': 'tf.quantization.quantize',
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
+    'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
-- 
GitLab


From 65ebe9f5f714ab74c43e2b4479f8706b28386e25 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 14 Nov 2018 16:09:49 -0800
Subject: [PATCH 0256/1554] Update the arguments for `tf.nn.pool` and exporting
 in v2.

PiperOrigin-RevId: 221529633
---
 tensorflow/python/ops/nn_ops.py               | 101 +++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |   7 ++
 .../tools/compatibility/tf_upgrade_v2_test.py |  10 ++
 4 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index bc195993c2..4c69201b49 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -873,7 +873,7 @@ class Convolution(object):
     return self.conv_op(inp, filter)
 
 
-@tf_export("nn.pool")
+@tf_export(v1=["nn.pool"])
 def pool(
     input,  # pylint: disable=redefined-builtin
     window_shape,
@@ -1044,6 +1044,105 @@ def pool(
         filter_shape=window_shape)
 
 
+@tf_export("nn.pool", v1=[])
+def pool_v2(
+    input,  # pylint: disable=redefined-builtin
+    window_shape,
+    pooling_type,
+    strides=None,
+    padding="VALID",
+    data_format=None,
+    dilations=None,
+    name=None):
+  # pylint: disable=line-too-long
+  """Performs an N-D pooling operation.
+
+  In the case that `data_format` does not start with "NC", computes for
+      0 <= b < batch_size,
+      0 <= x[i] < output_spatial_shape[i],
+      0 <= c < num_channels:
+
+  ```
+    output[b, x[0], ..., x[N-1], c] =
+      REDUCE_{z[0], ..., z[N-1]}
+        input[b,
+              x[0] * strides[0] - pad_before[0] + dilation_rate[0]*z[0],
+              ...
+              x[N-1]*strides[N-1] - pad_before[N-1] + dilation_rate[N-1]*z[N-1],
+              c],
+  ```
+
+  where the reduction function REDUCE depends on the value of `pooling_type`,
+  and pad_before is defined based on the value of `padding` as described in
+  the "returns" section of `tf.nn.convolution` for details.
+  The reduction never includes out-of-bounds positions.
+
+  In the case that `data_format` starts with `"NC"`, the `input` and output are
+  simply transposed as follows:
+
+  ```
+    pool(input, data_format, **kwargs) =
+      tf.transpose(pool(tf.transpose(input, [0] + range(2,N+2) + [1]),
+                        **kwargs),
+                   [0, N+1] + range(1, N+1))
+  ```
+
+  Args:
+    input: Tensor of rank N+2, of shape `[batch_size] + input_spatial_shape +
+      [num_channels]` if data_format does not start with "NC" (default), or
+      `[batch_size, num_channels] + input_spatial_shape` if data_format starts
+      with "NC".  Pooling happens over the spatial dimensions only.
+    window_shape: Sequence of N ints >= 1.
+    pooling_type: Specifies pooling operation, must be "AVG" or "MAX".
+    strides: Optional. Sequence of N ints >= 1.  Defaults to [1]*N. If any value of
+      strides is > 1, then all values of dilation_rate must be 1.
+    padding: The padding algorithm, must be "SAME" or "VALID". Defaults to "SAME".
+      See the "returns" section of `tf.nn.convolution` for details.
+    data_format: A string or None.  Specifies whether the channel dimension of
+      the `input` and output is the last dimension (default, or if `data_format`
+      does not start with "NC"), or the second dimension (if `data_format`
+      starts with "NC").  For N=1, the valid values are "NWC" (default) and
+      "NCW".  For N=2, the valid values are "NHWC" (default) and "NCHW". For
+      N=3, the valid values are "NDHWC" (default) and "NCDHW".
+    dilations: Optional.  Dilation rate.  List of N ints >= 1. Defaults to
+      [1]*N.  If any value of dilation_rate is > 1, then all values of strides
+      must be 1.
+    name: Optional. Name of the op.
+
+  Returns:
+    Tensor of rank N+2, of shape
+      [batch_size] + output_spatial_shape + [num_channels]
+
+    if data_format is None or does not start with "NC", or
+
+      [batch_size, num_channels] + output_spatial_shape
+
+    if data_format starts with "NC",
+    where `output_spatial_shape` depends on the value of padding:
+
+    If padding = "SAME":
+      output_spatial_shape[i] = ceil(input_spatial_shape[i] / strides[i])
+
+    If padding = "VALID":
+      output_spatial_shape[i] =
+        ceil((input_spatial_shape[i] - (window_shape[i] - 1) * dilation_rate[i])
+             / strides[i]).
+
+  Raises:
+    ValueError: if arguments are invalid.
+
+  """
+  return pool(
+      input=input,
+      window_shape=window_shape,
+      pooling_type=pooling_type,
+      padding=padding,
+      dilation_rate=dilations,
+      strides=strides,
+      name=name,
+      data_format=data_format)
+
+
 @tf_export("nn.atrous_conv2d")
 def atrous_conv2d(value, filters, rate, padding, name=None):
   """Atrous convolution (a.k.a. convolution with holes or dilated convolution).
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 2dc5c48aa6..d1c04f5acf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -222,7 +222,7 @@ tf_module {
   }
   member_method {
     name: "pool"
-    argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'padding\', \'dilation_rate\', \'strides\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'VALID\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "quantized_avg_pool"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index fbf4320e94..939e4109a7 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -37,6 +37,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
+        "tf.nn.pool": {
+            "dilation_rate": "dilations"
+        }
     }
 
     # Mapping from function to the new name of the function
@@ -64,6 +67,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
         "tf.convert_to_tensor": ["value", "dtype", "preferred_dtype", "name"],
         "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
+        "tf.nn.pool": [
+            "input", "window_shape", "pooling_type", "padding", "dilation_rate",
+            "strides", "name", "data_format"
+        ]
     }
 
     # Specially handled functions.
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 9060b1c71f..66d18e146a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -72,6 +72,16 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, "some_call(tf.sysconfig.MONOLITHIC_BUILD)\n")
 
+  def testRenameArgs(self):
+    text = ("tf.nn.pool(input_a, window_shape_a, pooling_type_a, padding_a, "
+            "dilation_rate_a, strides_a, name_a, data_format_a)\n")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text,
+                     ("tf.nn.pool(input=input_a, window_shape=window_shape_a,"
+                      " pooling_type=pooling_type_a, padding=padding_a, "
+                      "dilations=dilation_rate_a, strides=strides_a, "
+                      "name=name_a, data_format=data_format_a)\n"))
+
   def testReorder(self):
     text = "tf.boolean_mask(a, b, c, d)\n"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
-- 
GitLab


From 0f59b6068d5c9cc180ea2b102a13529fc64ada8c Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Wed, 14 Nov 2018 16:13:30 -0800
Subject: [PATCH 0257/1554] Improve error messages when no server factory can
 be found

The new error appends the names of the available factories, e.g.:
"The available server factories are: [ GRPC_SERVER ]"

PiperOrigin-RevId: 221530310
---
 tensorflow/core/distributed_runtime/BUILD     | 11 ++++++
 .../core/distributed_runtime/server_lib.cc    | 10 ++++--
 .../distributed_runtime/server_lib_test.cc    | 35 +++++++++++++++++++
 3 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/core/distributed_runtime/server_lib_test.cc

diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 3a534dcfb7..818324746f 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -467,6 +467,17 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "server_lib_test",
+    srcs = ["server_lib_test.cc"],
+    deps = [
+        ":server_lib",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 cc_library(
     name = "rpc_collective_executor_mgr",
     srcs = ["rpc_collective_executor_mgr.cc"],
diff --git a/tensorflow/core/distributed_runtime/server_lib.cc b/tensorflow/core/distributed_runtime/server_lib.cc
index 7d308bb723..fe9369e884 100644
--- a/tensorflow/core/distributed_runtime/server_lib.cc
+++ b/tensorflow/core/distributed_runtime/server_lib.cc
@@ -49,16 +49,22 @@ void ServerFactory::Register(const string& server_type,
 Status ServerFactory::GetFactory(const ServerDef& server_def,
                                  ServerFactory** out_factory) {
   mutex_lock l(*get_server_factory_lock());
-  // TODO(mrry): Improve the error reporting here.
   for (const auto& server_factory : *server_factories()) {
     if (server_factory.second->AcceptsOptions(server_def)) {
       *out_factory = server_factory.second;
       return Status::OK();
     }
   }
+
+  std::vector<string> server_names;
+  for (const auto& server_factory : *server_factories()) {
+    server_names.push_back(server_factory.first);
+  }
+
   return errors::NotFound(
       "No server factory registered for the given ServerDef: ",
-      server_def.DebugString());
+      server_def.DebugString(), "\nThe available server factories are: [ ",
+      str_util::Join(server_names, ", "), " ]");
 }
 
 // Creates a server based on the given `server_def`, and stores it in
diff --git a/tensorflow/core/distributed_runtime/server_lib_test.cc b/tensorflow/core/distributed_runtime/server_lib_test.cc
new file mode 100644
index 0000000000..29aafc15d6
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/server_lib_test.cc
@@ -0,0 +1,35 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/distributed_runtime/server_lib.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+TEST(ServerLibTest, NewServerNoFactoriesAccept) {
+  ServerDef server_def;
+  server_def.set_protocol("fake_protocol");
+  std::unique_ptr<ServerInterface> server;
+  Status s = NewServer(server_def, &server);
+  ASSERT_NE(s, Status::OK());
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "No server factory registered for the given ServerDef"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(),
+                                    "The available server factories are: ["));
+}
+
+}  // namespace tensorflow
-- 
GitLab


From 3732d473b6a7186c08ae560478ee0ac939c0bea9 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 14 Nov 2018 16:21:11 -0800
Subject: [PATCH 0258/1554] Fix AttributeError caused by renames being a set,
 not a dictionary.

PiperOrigin-RevId: 221531604
---
 .../tools/compatibility/update/generate_v2_renames_map.py     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
index 43aa8e057e..9a7bae0da3 100644
--- a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
+++ b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
@@ -144,8 +144,8 @@ def collect_function_renames():
   # It is possible that a different function is exported with the
   # same name. For e.g. when creating a different function to
   # rename arguments. Exclude it from renames in this case.
-  renames = {name: new_name for name, new_name in renames.items()
-             if name not in v2_names}
+  renames = set((name, new_name) for name, new_name in renames
+                if name not in v2_names)
   return renames
 
 
-- 
GitLab


From 1c2d515477ada15b363d8c210d668dd20784d392 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 16:32:07 -0800
Subject: [PATCH 0259/1554] * Improve performance of cumsum and cumprod by up
 to 300x. * Add benchmarks.

The vast majority of cases are much faster, but there are a small number of cases
that do get slower.

PiperOrigin-RevId: 221533615
---
 tensorflow/core/graph/testlib.cc              |  11 +
 tensorflow/core/graph/testlib.h               |   4 +
 tensorflow/core/kernels/BUILD                 |  25 +-
 tensorflow/core/kernels/scan_ops_gpu.cu.cc    | 263 +++++++++++++++++-
 tensorflow/core/kernels/scan_ops_test.cc      | 146 ++++++++++
 .../core/util/permutation_input_iterator.h    |   6 +-
 .../core/util/permutation_output_iterator.h   | 129 +++++++++
 .../python/kernel_tests/conv_ops_test.py      |   2 +-
 .../python/kernel_tests/scan_ops_test.py      |   5 +
 9 files changed, 585 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/core/kernels/scan_ops_test.cc
 create mode 100644 tensorflow/core/util/permutation_output_iterator.h

diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc
index 0a38aa1c91..0e74a30c7a 100644
--- a/tensorflow/core/graph/testlib.cc
+++ b/tensorflow/core/graph/testlib.cc
@@ -123,6 +123,17 @@ Node* Assign(Graph* g, Node* var, Node* val) {
   return ret;
 }
 
+Node* Cumsum(Graph* g, Node* data, Node* axes, bool exclusive, bool reverse) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Cumsum")
+                  .Input(data)
+                  .Input(axes)
+                  .Attr("exclusive", exclusive)
+                  .Attr("reverse", reverse)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
 Node* Reduce(Graph* g, const string& reduce, Node* data, Node* axes,
              bool keep_dims) {
   Node* ret;
diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h
index b00196f587..0c7233161f 100644
--- a/tensorflow/core/graph/testlib.h
+++ b/tensorflow/core/graph/testlib.h
@@ -68,6 +68,10 @@ Node* Recv(Graph* g, const string& tensor, const string& type,
            const string& sender, const uint64 sender_incarnation,
            const string& receiver);
 
+// Adds a cumsum "node" in "g" doing cumsum(data, axes).
+Node* Cumsum(Graph* g, Node* data, Node* axes, bool exclusive = false,
+             bool reverse = false);
+
 // Adds a reduction "node" in "g" doing sum(data, axes).  "reduce" is
 // a reduction, e.g., Sum, Max, Min, Mean, etc.
 Node* Reduce(Graph* g, const string& reduce, Node* data, Node* axes,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 5fd0f5d5d1..9065989657 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3197,7 +3197,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "scan_ops",
     prefix = "scan_ops",
-    deps = MATH_DEPS,
+    deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
 tf_kernel_library(
@@ -3375,6 +3375,29 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "scan_ops_test",
+    size = "small",
+    srcs = ["scan_ops_test.cc"],
+    linkopts = select({
+        "//tensorflow:darwin": ["-headerpad_max_install_names"],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":host_constant_op",
+        ":ops_testutil",
+        ":ops_util",
+        ":scan_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "reduction_ops_test",
     size = "small",
diff --git a/tensorflow/core/kernels/scan_ops_gpu.cu.cc b/tensorflow/core/kernels/scan_ops_gpu.cu.cc
index ed6c6affce..83f6cfd8c4 100644
--- a/tensorflow/core/kernels/scan_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/scan_ops_gpu.cu.cc
@@ -1,4 +1,4 @@
-/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,9 +16,18 @@ limitations under the License.
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
+#define CUB_USE_COOPERATIVE_GROUPS
 
+#include "third_party/cub/block/block_load.cuh"
+#include "third_party/cub/block/block_scan.cuh"
+#include "third_party/cub/block/block_store.cuh"
+#include "third_party/cub/iterator/counting_input_iterator.cuh"
+#include "third_party/cub/iterator/transform_input_iterator.cuh"
+#include "cuda/include/cuComplex.h"
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/util/permutation_input_iterator.h"
+#include "tensorflow/core/util/permutation_output_iterator.h"
 
 #include "tensorflow/core/kernels/scan_ops.h"
 
@@ -27,6 +36,258 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 typedef Eigen::Index Index;
 
+namespace functor {
+
+// Map a contiguous range to the actual memory locations depending on which
+// axis the scan is taking place over and whether or not reversed.
+struct MapIndexToLocation {
+  __host__ __device__ MapIndexToLocation(int dimx, int dimy, int dimz,
+                                         bool reverse = false)
+      : dimx_(dimx), dimy_(dimy), dimz_(dimz), reverse_(reverse) {}
+
+  __host__ __device__ int operator()(int id) const {
+    if (dimx_ == 1) {
+      int row = id % dimy_;
+      int col = id / dimy_;
+
+      if (reverse_) return (dimy_ - row - 1) * dimz_ + col;
+
+      return row * dimz_ + col;
+    } else if (dimz_ == 1) {
+      if (reverse_) {
+        int row = id / dimy_;
+        int col = id % dimy_;
+        return row * dimy_ + (dimy_ - col - 1);
+      }
+      return id;
+    } else {
+      int col = id % dimy_;
+      int tmp = id / dimy_;
+
+      int row1 = id / (dimy_ * dimz_);
+      int col1 = tmp % dimz_;
+
+      if (reverse_)
+        return row1 * dimy_ * dimz_ + (dimy_ - col - 1) * dimz_ + col1;
+
+      return row1 * dimy_ * dimz_ + col * dimz_ + col1;
+    }
+  }
+
+  int dimx_;
+  int dimy_;
+  int dimz_;
+  bool reverse_;
+};
+
+template <typename T, typename Op>
+struct BlockPrefixCallbackOp {
+  // Running prefix
+  T running_total_;
+  Op op_;
+
+  __device__ BlockPrefixCallbackOp(T running_total, Op op)
+      : running_total_(running_total), op_(op) {}
+
+  // Callback operator to be entered by the first warp of threads in the block.
+  // tid 0 is responsible for returning a value for seeding the block-wide scan.
+  __device__ T operator()(T block_aggregate) {
+    T old_prefix = running_total_;
+    running_total_ = op_(old_prefix, block_aggregate);
+    return old_prefix;
+  }
+};
+
+template <typename T>
+struct Sum {
+  __host__ __device__ T operator()(const T& a, const T& b) const {
+    return a + b;
+  }
+};
+
+template <typename T>
+struct Prod {
+  __host__ __device__ T operator()(const T& a, const T& b) const {
+    return a * b;
+  }
+};
+
+template <typename T, typename Op>
+struct IsSum {
+  constexpr static bool value =
+      (std::is_same<Op, Sum<T>>::value ||
+       std::is_same<Op, Eigen::internal::SumReducer<T>>::value);
+};
+
+template <typename T, typename Op>
+struct IsProd {
+  constexpr static bool value =
+      (std::is_same<Op, Prod<T>>::value ||
+       std::is_same<Op, Eigen::internal::ProdReducer<T>>::value);
+};
+
+template <typename T, typename Op>
+struct IdentityValue {
+  static_assert(IsSum<T, Op>::value || IsProd<T, Op>::value,
+                "IdentityValue not yet defined for this type.");
+
+  template <typename U = T, typename OpCopy = Op>
+  __host__ __device__ U operator()(
+      typename std::enable_if<IsSum<U, OpCopy>::value, U>::type t = U(0)) {
+    return t;
+  }
+
+  template <typename U = T, typename OpCopy = Op>
+  __host__ __device__ U operator()(
+      typename std::enable_if<IsProd<U, OpCopy>::value, U>::type t = U(1)) {
+    return t;
+  }
+};
+
+// Each block is mapped to one sequence.  A contiguous range is mapped to the
+// appropriate locations in memory by the permutation iterators.  This is
+// ideal for 1-D and row based scans.  Column scans would be better if they
+// did a block load and then locally transposed.  CUB's device wide scan is not
+// used in the large 1D case, even though it would be more efficient, because
+// it is not deterministic.
+template <typename T, typename Op, int BlockDim = 128, int ItemsPerThread = 4>
+__global__ void scan_kernel(const T* in, T* out, int dimx, int dimy, int dimz,
+                            bool exclusive, bool reverse, Op op) {
+  typedef cub::BlockLoad<T, BlockDim, ItemsPerThread, cub::BLOCK_LOAD_TRANSPOSE>
+      BlockLoad;
+  typedef cub::BlockStore<T, BlockDim, ItemsPerThread,
+                          cub::BLOCK_STORE_TRANSPOSE>
+      BlockStore;
+  typedef cub::BlockScan<T, BlockDim> BlockScan;
+
+  // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
+  __shared__ union {
+    typename BlockLoad::TempStorage load;
+    typename BlockScan::TempStorage scan;
+    typename BlockStore::TempStorage store;
+  } temp_storage;
+
+  int problem_length = dimy;
+
+  // Initialize running total
+  BlockPrefixCallbackOp<T, Op> prefix_op(IdentityValue<T, Op>()(), op);
+
+  MapIndexToLocation map_op(dimx, dimy, dimz, reverse);
+  int block_start = problem_length * blockIdx.x;
+  // Have the block iterate over segments of items
+  for (int block_offset = block_start;
+       block_offset < block_start + problem_length;
+       block_offset += BlockDim * ItemsPerThread) {
+    int valid_items = min(BlockDim * ItemsPerThread,
+                          problem_length - (block_offset % problem_length));
+
+    // first construct a counting iterator that has the desired start point
+    typedef cub::TransformInputIterator<int, MapIndexToLocation,
+                                        cub::CountingInputIterator<int>>
+        MapIterType;
+
+    cub::CountingInputIterator<int> counting_iter(block_offset);
+
+    // Next map the iterator to the actual locations in memory
+    MapIterType map_iter(counting_iter, map_op);
+
+    PermutationInputIterator<T, const T*, MapIterType> permutein_iter(in,
+                                                                      map_iter);
+    PermutationOutputIterator<T, T*, MapIterType> permuteout_iter(out,
+                                                                  map_iter);
+
+    // Load a segment of consecutive items that are blocked across threads
+    T thread_data[ItemsPerThread];
+    BlockLoad(temp_storage.load).Load(permutein_iter, thread_data, valid_items);
+    __syncthreads();
+
+    // Collectively compute the block-wide scan
+    if (exclusive) {
+      BlockScan(temp_storage.scan)
+          .ExclusiveScan(thread_data, thread_data, op, prefix_op);
+    } else {
+      BlockScan(temp_storage.scan)
+          .InclusiveScan(thread_data, thread_data, op, prefix_op);
+    }
+    __syncthreads();
+
+    // Store scanned items to output segment
+    BlockStore(temp_storage.store)
+        .Store(permuteout_iter, thread_data, valid_items);
+    __syncthreads();
+  }
+}
+
+template <typename T, typename Op>
+void LaunchScan(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
+                typename TTypes<T, 3>::Tensor out, Op op, const bool reverse,
+                const bool exclusive) {
+  const int items_per_thread = 4;
+
+  int dimx = in.dimension(0);
+  int dimy = in.dimension(1);
+  int dimz = in.dimension(2);
+  int num_blocks = dimx * dimz;
+
+  int ideal_block_size = dimy / items_per_thread;
+
+  // There seems to be a bug when the type is not float and block_size 1024.
+  // Launch on the smallest power of 2 block size that we can.
+  if (ideal_block_size >= 1024 && std::is_same<T, float>::value) {
+    const int block_size = 1024;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 512) {
+    const int block_size = 512;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 256) {
+    const int block_size = 256;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 128) {
+    const int block_size = 128;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 64) {
+    const int block_size = 64;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else {
+    const int block_size = 32;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  }
+}
+
+template <typename T>
+struct Scan<GPUDevice, Eigen::internal::SumReducer<T>, T> {
+  void operator()(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
+                  typename TTypes<T, 3>::Tensor out,
+                  const Eigen::internal::SumReducer<T>& reducer,
+                  const bool reverse, const bool exclusive) {
+    LaunchScan<T, Sum<T>>(d, in, out, Sum<T>(), reverse, exclusive);
+  }
+};
+
+template <typename T>
+struct Scan<GPUDevice, Eigen::internal::ProdReducer<T>, T> {
+  void operator()(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
+                  typename TTypes<T, 3>::Tensor out,
+                  const Eigen::internal::ProdReducer<T>& reducer,
+                  const bool reverse, const bool exclusive) {
+    LaunchScan<T, Prod<T>>(d, in, out, Prod<T>(), reverse, exclusive);
+  }
+};
+
+}  // namespace functor
+
 #define DEFINE(REDUCER, T) template struct functor::Scan<GPUDevice, REDUCER, T>;
 
 #define DEFINE_FOR_ALL_REDUCERS(T)           \
diff --git a/tensorflow/core/kernels/scan_ops_test.cc b/tensorflow/core/kernels/scan_ops_test.cc
new file mode 100644
index 0000000000..588b606a99
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_test.cc
@@ -0,0 +1,146 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+template <typename T>
+static Graph* LargeOneDCumsum(int num_x, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DataTypeToEnum<T>::value, TensorShape({num_x}));
+  data.flat<T>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 0;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+static Graph* ColCumsum(int num_x, int num_y, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DT_FLOAT, TensorShape({num_x, num_y}));
+  data.flat<float>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 0;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+static Graph* RowCumsum(int num_x, int num_y, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DT_FLOAT, TensorShape({num_x, num_y}));
+  data.flat<float>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 1;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+static Graph* ThreeDYCumsum(int num_y, int num_z, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DT_FLOAT, TensorShape({32, num_y, num_z}));
+  data.flat<float>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 1;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+template <typename T>
+static void LargeOneDimensional(int iters, const string& device, int num_x,
+                                bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * sizeof(T));
+  test::Benchmark(device, LargeOneDCumsum<T>(num_x, reverse)).Run(iters);
+}
+
+static void DoRowCumsum(int iters, const string& device, int num_x, int num_y,
+                        bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
+                          sizeof(float));
+  test::Benchmark(device, RowCumsum(num_x, num_y, reverse)).Run(iters);
+}
+
+static void DoColCumsum(int iters, const string& device, int num_x, int num_y,
+                        bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
+                          sizeof(float));
+  test::Benchmark(device, ColCumsum(num_x, num_y, reverse)).Run(iters);
+}
+
+static void Do3DYCumsum(int iters, const string& device, int num_x, int num_y,
+                        bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
+                          sizeof(float));
+  test::Benchmark(device, ThreeDYCumsum(num_x, num_y, reverse)).Run(iters);
+}
+
+static void BM_OneDCumsumGPU(int iters, int num_x) {
+  LargeOneDimensional<float>(iters, "gpu", num_x);
+}
+BENCHMARK(BM_OneDCumsumGPU)->Range(1, 1 << 21);
+
+static void BM_OneDCumsumGPUHalf(int iters, int num_x) {
+  LargeOneDimensional<Eigen::half>(iters, "gpu", num_x);
+}
+BENCHMARK(BM_OneDCumsumGPUHalf)->Range(1, 1 << 21);
+
+static void BM_Sum2DRowCumsumGPU(int iters, int num_x, int num_y) {
+  DoRowCumsum(iters, "gpu", num_x, num_y);
+}
+BENCHMARK(BM_Sum2DRowCumsumGPU)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum2DColumnCumsumGPU(int iters, int num_x, int num_y) {
+  DoColCumsum(iters, "gpu", num_x, num_y);
+}
+BENCHMARK(BM_Sum2DColumnCumsumGPU)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum3DYCumsumGPU(int iters, int num_x, int num_y) {
+  Do3DYCumsum(iters, "gpu", num_x, num_y);
+}
+BENCHMARK(BM_Sum3DYCumsumGPU)->RangePair(64, 4096, 64, 4096);
+
+static void BM_OneDCumsumGPU_reverse(int iters, int num_x) {
+  LargeOneDimensional<float>(iters, "gpu", num_x, true);
+}
+BENCHMARK(BM_OneDCumsumGPU_reverse)->Range(1, 1 << 21);
+
+static void BM_Sum2DRowCumsumGPU_reverse(int iters, int num_x, int num_y) {
+  DoRowCumsum(iters, "gpu", num_x, num_y, true);
+}
+BENCHMARK(BM_Sum2DRowCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum2DColumnCumsumGPU_reverse(int iters, int num_x, int num_y) {
+  DoColCumsum(iters, "gpu", num_x, num_y, true);
+}
+BENCHMARK(BM_Sum2DColumnCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum3DYCumsumGPU_reverse(int iters, int num_x, int num_y) {
+  Do3DYCumsum(iters, "gpu", num_x, num_y, true);
+}
+BENCHMARK(BM_Sum3DYCumsumGPU_reverse)->RangePair(32, 2048, 32, 2048);
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/util/permutation_input_iterator.h b/tensorflow/core/util/permutation_input_iterator.h
index f6375b2515..649318ebf3 100644
--- a/tensorflow/core/util/permutation_input_iterator.h
+++ b/tensorflow/core/util/permutation_input_iterator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
-#define TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#define TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
 
 #include <iostream>
 #include <iterator>
@@ -131,4 +131,4 @@ class PermutationInputIterator {
 
 }  // end namespace tensorflow
 
-#endif  // TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#endif  // TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
diff --git a/tensorflow/core/util/permutation_output_iterator.h b/tensorflow/core/util/permutation_output_iterator.h
new file mode 100644
index 0000000000..638c0f4545
--- /dev/null
+++ b/tensorflow/core/util/permutation_output_iterator.h
@@ -0,0 +1,129 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
+#define TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
+
+#include <iostream>
+#include <iterator>
+
+namespace tensorflow {
+
+template <typename ValueType, typename OutputIteratorT, typename IndexIteratorT,
+          typename OffsetT = ptrdiff_t>
+class PermutationOutputIterator {
+ public:
+  // Required iterator traits
+  typedef PermutationOutputIterator self_type;  ///< My own type
+  typedef OffsetT difference_type;  ///< Type to express the result of
+                                    ///< subtracting one iterator from another
+  typedef ValueType
+      value_type;  ///< The type of the element the iterator can point to
+  typedef ValueType* pointer;    ///< The type of a pointer to an element the
+                                 ///< iterator can point to
+  typedef ValueType& reference;  ///< The type of a reference to an element the
+                                 ///< iterator can point to
+
+  typedef std::random_access_iterator_tag
+      iterator_category;  ///< The iterator category
+
+ private:
+  OutputIteratorT output_itr;
+  IndexIteratorT index_itr;
+
+ public:
+  /// Constructor
+  __host__ __device__ __forceinline__ PermutationOutputIterator(
+      OutputIteratorT output_itr,  ///< Input iterator to wrap
+      IndexIteratorT index_itr)    ///< Conversion functor to wrap
+      : output_itr(output_itr), index_itr(index_itr) {}
+
+  /// Postfix increment
+  __host__ __device__ __forceinline__ self_type operator++(int) {
+    self_type retval = *this;
+    index_itr++;
+    return retval;
+  }
+
+  /// Prefix increment
+  __host__ __device__ __forceinline__ self_type operator++() {
+    index_itr++;
+    return *this;
+  }
+
+  /// Indirection
+  __host__ __device__ __forceinline__ reference operator*() const {
+    return output_itr[*index_itr];
+  }
+
+  /// Addition
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type operator+(Distance n) const {
+    self_type retval(output_itr, index_itr + n);
+    return retval;
+  }
+
+  /// Addition assignment
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type& operator+=(Distance n) {
+    index_itr += n;
+    return *this;
+  }
+
+  /// Subtraction
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type operator-(Distance n) const {
+    self_type retval(output_itr, index_itr - n);
+    return retval;
+  }
+
+  /// Subtraction assignment
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type& operator-=(Distance n) {
+    index_itr -= n;
+    return *this;
+  }
+
+  /// Distance
+  __host__ __device__ __forceinline__ difference_type
+  operator-(self_type other) const {
+    return index_itr - other.index_itr;
+  }
+
+  /// Array subscript
+  template <typename Distance>
+  __host__ __device__ __forceinline__ reference operator[](Distance n) const {
+    return output_itr[index_itr[n]];
+  }
+
+  /// Equal to
+  __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) {
+    return (index_itr == rhs.index_itr && output_itr == rhs.output_itr);
+  }
+
+  /// Not equal to
+  __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) {
+    return !(*this == rhs);
+  }
+
+  /// ostream operator
+  friend std::ostream& operator<<(std::ostream& os, const self_type& itr) {
+    return os;
+  }
+};
+
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 0ccbbf155c..835cc1504d 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -1669,7 +1669,7 @@ class SeparableConv2DTest(test.TestCase):
 
       value = sess.run(conv)
     tf_logging.info("value = ", value)
-    self.assertArrayNear(expected, np.ravel(value), 1e-5)
+    self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
 
   def _testSeparableConv2D(self, data_format):
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index b369222565..c48e0e2e67 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -126,6 +126,11 @@ class CumsumTest(test.TestCase):
       for axis in range(-6, 6, 3):
         self._compareAll(x, axis)
 
+  def testLarge(self):
+    for dtype in self.valid_dtypes:
+      x = np.ones([1000000], dtype=dtype) / 1024
+      self._compareAll(x, 0)
+
   def testInvalidAxis(self):
     x = np.arange(0, 10).reshape([2, 5]).astype(np.float32)
     input_tensor = ops.convert_to_tensor(x)
-- 
GitLab


From 2ba2e6b25891e63c425c7f239ac75bcb7f8f9bda Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 14 Nov 2018 16:47:42 -0800
Subject: [PATCH 0260/1554] Change signature of tf.count_nonzero for TF 2.0.

PiperOrigin-RevId: 221536352
---
 tensorflow/python/ops/math_ops.py             | 73 +++++++++++++++++--
 .../tools/api/golden/v2/tensorflow.math.pbtxt |  2 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 -
 .../tools/compatibility/tf_upgrade_v2.py      |  7 +-
 .../tools/compatibility/tf_upgrade_v2_test.py | 12 +++
 5 files changed, 85 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 4ff7a8e3a6..79387bd99d 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1408,7 +1408,7 @@ def reduce_sum(input_tensor,
                                    name=name))
 
 
-@tf_export("math.count_nonzero", "count_nonzero")
+@tf_export(v1=["math.count_nonzero", "count_nonzero"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
 def count_nonzero(input_tensor,
@@ -1469,20 +1469,79 @@ def count_nonzero(input_tensor,
   """
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis,
+      "reduction_indices", reduction_indices
+      )
   if keepdims is None:
     keepdims = False
 
-  with ops.name_scope(name, "count_nonzero", [input_tensor]):
-    input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")
+  return count_nonzero_v2(input_tensor, axis, keepdims, dtype, name)
+
+
+@tf_export("math.count_nonzero", v1=[])
+def count_nonzero_v2(input,  # pylint: disable=redefined-builtin
+                     axis=None,
+                     keepdims=None,
+                     dtype=dtypes.int64,
+                     name=None):
+  """Computes number of nonzero elements across dimensions of a tensor.
+
+  Reduces `input` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
+
+  If `axis` has no entries, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  **NOTE** Floating point comparison to zero is done by exact floating point
+  equality check.  Small values are **not** rounded to zero for purposes of
+  the nonzero check.
+
+  For example:
+
+  ```python
+  x = tf.constant([[0, 1, 0], [1, 1, 0]])
+  tf.count_nonzero(x)  # 3
+  tf.count_nonzero(x, 0)  # [1, 2, 0]
+  tf.count_nonzero(x, 1)  # [1, 2]
+  tf.count_nonzero(x, 1, keepdims=True)  # [[1], [2]]
+  tf.count_nonzero(x, [0, 1])  # 3
+  ```
+
+  **NOTE** Strings are compared against zero-length empty string `""`. Any
+  string with a size greater than zero is already considered as nonzero.
+
+  For example:
+  ```python
+  x = tf.constant(["", "a", "  ", "b", ""])
+  tf.count_nonzero(x) # 3, with "a", "  ", and "b" as nonzero strings.
+  ```
+
+  Args:
+    input: The tensor to reduce. Should be of numeric type, `bool`,
+      or `string`.
+    axis: The dimensions to reduce. If `None` (the default),
+      reduces all dimensions. Must be in the range
+      `[-rank(input), rank(input))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    dtype: The output dtype; defaults to `tf.int64`.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor (number of nonzero values).
+  """
+  with ops.name_scope(name, "count_nonzero", [input]):
+    input = ops.convert_to_tensor(input, name="input")
     # A scalar of 'zero' is enough as `not_equal` will broadcast.
-    zero = array_ops.zeros([], dtype=input_tensor.dtype)
+    zero = array_ops.zeros([], dtype=input.dtype)
     return cast(
         reduce_sum(
             # int64 reduction happens on GPU
-            to_int64(gen_math_ops.not_equal(input_tensor, zero)),
+            to_int64(gen_math_ops.not_equal(input, zero)),
             axis=axis,
-            keepdims=keepdims,
-            reduction_indices=reduction_indices),
+            keepdims=keepdims),
         dtype=dtype)
 
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index a441e42b0a..a7638ffde6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -102,7 +102,7 @@ tf_module {
   }
   member_method {
     name: "count_nonzero"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"<dtype: \'int64\'>\", \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'axis\', \'keepdims\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"<dtype: \'int64\'>\", \'None\'], "
   }
   member_method {
     name: "cumprod"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index e8435a401a..c41e9dcd57 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -624,10 +624,6 @@ tf_module {
     name: "cosh"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "count_nonzero"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'dtype\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"<dtype: \'int64\'>\", \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "create_partitioned_variables"
     argspec: "args=[\'shape\', \'slicing\', \'initializer\', \'dtype\', \'trainable\', \'collections\', \'name\', \'reuse\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'True\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 939e4109a7..a6882e35d3 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -37,9 +37,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
+        "tf.math.count_nonzero": {
+            "input_tensor": "input",
+            "keep_dims": "keepdims",
+            "reduction_indices": "axis",
+        },
         "tf.nn.pool": {
             "dilation_rate": "dilations"
-        }
+        },
     }
 
     # Mapping from function to the new name of the function
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 66d18e146a..2dfeb64b32 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -110,6 +110,18 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                               % "tf.estimator.LinearClassifier"])
     self.assertIn("loss_reduction has been changed", report)
 
+  def testCountNonZeroChanges(self):
+    text = (
+        "tf.math.count_nonzero(input_tensor=input, dtype=dtype, name=name, "
+        "reduction_indices=axis, keep_dims=keepdims)\n"
+        )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    expected_text = (
+        "tf.math.count_nonzero(input=input, dtype=dtype, name=name, "
+        "axis=axis, keepdims=keepdims)\n"
+        )
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 0773b244a01e738459354b2697352feb04646c21 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 14 Nov 2018 16:47:46 -0800
Subject: [PATCH 0261/1554] Disable LOG and VLOG in fuzzing mode.

PiperOrigin-RevId: 221536365
---
 tensorflow/core/platform/default/logging.cc | 28 +++++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/platform/default/logging.cc b/tensorflow/core/platform/default/logging.cc
index ad3feda701..133ae45a55 100644
--- a/tensorflow/core/platform/default/logging.cc
+++ b/tensorflow/core/platform/default/logging.cc
@@ -117,13 +117,33 @@ int64 LogLevelStrToInt(const char* tf_env_var_val) {
 }  // namespace
 
 int64 MinLogLevelFromEnv() {
+  // We don't want to print logs during fuzzing as that would slow fuzzing down
+  // by almost 2x. So, if we are in fuzzing mode (not just running a test), we
+  // return a value so that nothing is actually printed. Since LOG uses >=
+  // (see ~LogMessage in this file) to see if log messages need to be printed,
+  // the value we're interested on to disable printing is the maximum severity.
+  // See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  return tensorflow::NUM_SEVERITIES;
+#else
   const char* tf_env_var_val = getenv("TF_CPP_MIN_LOG_LEVEL");
   return LogLevelStrToInt(tf_env_var_val);
+#endif
 }
 
 int64 MinVLogLevelFromEnv() {
+  // We don't want to print logs during fuzzing as that would slow fuzzing down
+  // by almost 2x. So, if we are in fuzzing mode (not just running a test), we
+  // return a value so that nothing is actually printed. Since VLOG uses <=
+  // (see VLOG_IS_ON in logging.h) to see if log messages need to be printed,
+  // the value we're interested on to disable printing is 0.
+  // See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  return 0;
+#else
   const char* tf_env_var_val = getenv("TF_CPP_MIN_VLOG_LEVEL");
   return LogLevelStrToInt(tf_env_var_val);
+#endif
 }
 
 LogMessage::~LogMessage() {
@@ -133,16 +153,8 @@ LogMessage::~LogMessage() {
 }
 
 int64 LogMessage::MinVLogLevel() {
-  // We don't want to print logs during fuzzing as that would slow fuzzing down
-  // by almost 2x. So, if we are in fuzzing mode (not just running a test), we
-  // return minimum value so that nothing is actually printed
-  // See also http://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
-  return 0;
-#else
   static int64 min_vlog_level = MinVLogLevelFromEnv();
   return min_vlog_level;
-#endif
 }
 
 LogMessageFatal::LogMessageFatal(const char* file, int line)
-- 
GitLab


From 6c6056eb4d36f2ba54456dc94f15b9f55d922e42 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 14 Nov 2018 16:57:41 -0800
Subject: [PATCH 0262/1554] [XLA] Make ParseFlagsFromEnv take an envvar
 parameter.

Previously we hardcoded TF_XLA_FLAGS.

PiperOrigin-RevId: 221538332
---
 .../legacy_flags/build_xla_ops_pass_flags.cc  |  2 +-
 .../mark_for_compilation_pass_flags.cc        |  2 +-
 .../jit/legacy_flags/xla_device_flags.cc      |  2 +-
 .../jit/legacy_flags/xla_ops_common_flags.cc  |  2 +-
 .../compiler/tf2xla/dump_graph_flags.cc       |  2 +-
 .../compiler/xla/debug_options_flags.cc       |  2 +-
 .../compiler/xla/parse_flags_from_env.cc      | 69 ++++++++++---------
 .../compiler/xla/parse_flags_from_env.h       | 53 +++++++-------
 .../compiler/xla/parse_flags_from_env_test.cc |  8 +--
 9 files changed, 75 insertions(+), 67 deletions(-)

diff --git a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc
index 961c17c17e..1f02cfaa95 100644
--- a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc
+++ b/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc
@@ -34,7 +34,7 @@ void AllocateAndParseFlags() {
       Flag("tf_xla_enable_lazy_compilation",
            &flags->tf_xla_enable_lazy_compilation, ""),
   });
-  xla::ParseFlagsFromEnv(*flag_list);
+  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
index bad306e0b0..043012bd0b 100644
--- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
+++ b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
@@ -65,7 +65,7 @@ static void AllocateFlags() {
        Flag("tf_xla_fusion_only", &flags->tf_xla_fusion_only,
             "enable fusion of element-wise operations only using XLA when "
             "global_jit_level is ON*.")});
-  xla::ParseFlagsFromEnv(*flag_list);
+  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
 
   if (VLOG_IS_ON(1)) {
     VLOG(1) << "Parsed MarkForCompilationPassFlags:";
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
index 76b80d3034..6389cf10d4 100644
--- a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
+++ b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
@@ -41,7 +41,7 @@ static void AllocateFlags() {
            "Switch a device into 'on-demand' mode, where instead of "
            "autoclustering ops are compiled one by one just-in-time."),
   });
-  xla::ParseFlagsFromEnv(*flag_list);
+  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
 }
 
 // Return a pointer to the XlaDeviceFlags struct;
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc
index 1443d48a73..bf8bd0d47f 100644
--- a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc
+++ b/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc
@@ -35,7 +35,7 @@ void AllocateAndParseFlags() {
       Flag("tf_xla_always_defer_compilation",
            &flags->tf_xla_always_defer_compilation, ""),
   });
-  xla::ParseFlagsFromEnv(*flag_list);
+  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
 
   if (VLOG_IS_ON(1)) {
     VLOG(1) << "Parsed XlaOpsCommonFlags:";
diff --git a/tensorflow/compiler/tf2xla/dump_graph_flags.cc b/tensorflow/compiler/tf2xla/dump_graph_flags.cc
index 2eb1f8cd84..32066bfd5b 100644
--- a/tensorflow/compiler/tf2xla/dump_graph_flags.cc
+++ b/tensorflow/compiler/tf2xla/dump_graph_flags.cc
@@ -41,7 +41,7 @@ static void AllocateFlags() {
            "Path prefix to which graphs dumped during debugging should be "
            "written."),
   });
-  xla::ParseFlagsFromEnv(*flag_list);
+  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
 }
 
 // Append to *append_to flag definitions associated with the XLA bridge's
diff --git a/tensorflow/compiler/xla/debug_options_flags.cc b/tensorflow/compiler/xla/debug_options_flags.cc
index 033887d7c1..a41bd7f9ae 100644
--- a/tensorflow/compiler/xla/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/debug_options_flags.cc
@@ -335,7 +335,7 @@ void AllocateFlags() {
           "behavior to help run tests on the host that run models in parallel "
           "across multiple devices."),
   });
-  ParseFlagsFromEnv(*flag_objects);
+  ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_objects);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/parse_flags_from_env.cc b/tensorflow/compiler/xla/parse_flags_from_env.cc
index 40481331b6..d575c5bc12 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <memory>
+#include <unordered_map>
 #include <vector>
 
 #include "tensorflow/compiler/xla/parse_flags_from_env.h"
@@ -32,7 +34,6 @@ limitations under the License.
 
 namespace xla {
 
-static const char kEnvVar[] = "TF_XLA_FLAGS";  // environment variable queried
 static const char kWS[] = " \t\r\n";           // whitespace
 
 // The following struct represents an argv[]-style array, parsed
@@ -42,12 +43,20 @@ static const char kWS[] = " \t\r\n";           // whitespace
 // constructor/destructor collisions with other "private" types
 // in the same named namespace.
 namespace {
+
+// Functor which deletes objects by calling `free`.  Necessary to free strdup'ed
+// strings created by AppendToEnvArgv.
+struct FreeDeleter {
+  void operator()(char* ptr) { free(ptr); }
+};
+
 struct EnvArgv {
   EnvArgv() : initialized(false), argc(0) {}
   bool initialized;         // whether the other fields have been set.
   int argc;                 // elements used in argv[]
   std::vector<char*> argv;  // flag arguments parsed from environment string.
-  std::vector<char*> argv_save;  // saved values from argv[] to avoid leaks
+  // saved values from argv[] to avoid leaks
+  std::vector<std::unique_ptr<char, FreeDeleter>> argv_save;
 };
 }  // anonymous namespace
 
@@ -63,7 +72,7 @@ static void AppendToEnvArgv(const char* s0, size_t s0len, const char* s1,
     string s = string(s0, s0len) + string(s1, s1len);
     char* str = strdup(s.c_str());
     a->argv.push_back(str);
-    a->argv_save.push_back(str);
+    a->argv_save.emplace_back(str);
     a->argc++;
   }
 }
@@ -127,14 +136,14 @@ static void ParseArgvFromString(const string& flag_str, EnvArgv* a) {
   }
 }
 
-// Call ParseArgvFromString(..., a) on a string derived from the setting of an
-// environment variable kEnvVar, or a file it points to.
-static void SetArgvFromEnv(EnvArgv* a) {
+// Call ParseArgvFromString(..., a) on a string derived from the setting of the
+// environment variable `envvar`, or a file it points to.
+static void SetArgvFromEnv(absl::string_view envvar, EnvArgv* a) {
   if (!a->initialized) {
     static const char kDummyArgv[] = "<argv[0]>";
     AppendToEnvArgv(kDummyArgv, strlen(kDummyArgv), nullptr, 0,
                     a);  // dummy argv[0]
-    const char* env = getenv(kEnvVar);
+    const char* env = getenv(string(envvar).c_str());
     if (env == nullptr || env[0] == '\0') {
       // nothing
     } else if (env[strspn(env, kWS)] == '-') {  // flags in env var value
@@ -157,23 +166,25 @@ static void SetArgvFromEnv(EnvArgv* a) {
   }
 }
 
-// The simulated argv[] parsed from the environment.
-static EnvArgv* env_argv;
+// The simulated argv[] parsed from the environment, one for each different
+// environment variable we've seen.
+static std::unordered_map<string, EnvArgv>& EnvArgvs() {
+  static auto* env_argvs = new std::unordered_map<string, EnvArgv>();
+  return *env_argvs;
+}
 
-// Used to protect accesses to env_argv.
+// Used to protect accesses to env_argvs.
 static tensorflow::mutex env_argv_mu(tensorflow::LINKER_INITIALIZED);
 
 // Call Flags::Parse(argc, argv, flag_list) against any as yet unrecognized
 // flags passed in from the environment.
-bool ParseFlagsFromEnv(const std::vector<tensorflow::Flag>& flag_list) {
-  env_argv_mu.lock();
-  if (env_argv == nullptr) {
-    env_argv = new EnvArgv;
-  }
-  SetArgvFromEnv(env_argv);  // a no-op if already initialized
+bool ParseFlagsFromEnv(absl::string_view envvar,
+                       const std::vector<tensorflow::Flag>& flag_list) {
+  tensorflow::mutex_lock lock(env_argv_mu);
+  auto* env_argv = &EnvArgvs()[string(envvar)];
+  SetArgvFromEnv(envvar, env_argv);  // a no-op if already initialized
   bool result =
       tensorflow::Flags::Parse(&env_argv->argc, &env_argv->argv[0], flag_list);
-  env_argv_mu.unlock();
   return result;
 }
 
@@ -182,23 +193,13 @@ bool ParseFlagsFromEnv(const std::vector<tensorflow::Flag>& flag_list) {
 // will parse the environment variable (or the file it points to) anew, and set
 // *pargc, and *pargv to point to the internal locations of the argc and argv
 // constructed from the environment.
-void ResetFlagsFromEnvForTesting(int** pargc, std::vector<char*>** pargv) {
-  env_argv_mu.lock();
-  if (env_argv == nullptr) {
-    env_argv = new EnvArgv;
-  }
-  if (!env_argv->argv_save.empty()) {
-    for (int i = 0; env_argv->argv_save[i] != nullptr; i++) {
-      free(env_argv->argv_save[i]);
-    }
-  }
-  env_argv->initialized = false;
-  env_argv->argc = 0;
-  env_argv->argv.clear();
-  env_argv->argv_save.clear();
-  env_argv_mu.unlock();
-  *pargc = &env_argv->argc;
-  *pargv = &env_argv->argv;
+void ResetFlagsFromEnvForTesting(absl::string_view envvar, int** pargc,
+                                 std::vector<char*>** pargv) {
+  tensorflow::mutex_lock lock(env_argv_mu);
+  EnvArgvs().erase(string(envvar));
+  auto& env_argv = EnvArgvs()[string(envvar)];
+  *pargc = &env_argv.argc;
+  *pargv = &env_argv.argv;
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/parse_flags_from_env.h b/tensorflow/compiler/xla/parse_flags_from_env.h
index fe86ee687f..c0742e8b4f 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.h
+++ b/tensorflow/compiler/xla/parse_flags_from_env.h
@@ -17,47 +17,54 @@ limitations under the License.
 #define TENSORFLOW_COMPILER_XLA_PARSE_FLAGS_FROM_ENV_H_
 
 // This module exports ParseFlagsFromEnv(), which allows other modules to parse
-// flags from the environtment variable TF_XLA_FLAGS, or (if the first
-// non-whitespace in the variable value is not '-'), a file named by that
-// environment variable.  The accepted syntax is that flags arguments are of
-// the form --flag=value or (for boolean flags) --flag, and are whitespace
-// separated.  The <value> may be one of:
-// - <non-whitespace, non-nul not starting with single-quote or double-quote>
-//   in which case the effective value is the string itself
-// - <single-quote><characters string not containing nul or
-//   single-quote><single_quote> in which case the effective value is the
-//   string with the single-quotes removed
-// - <double-quote><character string not containing nul or unesecaped
-//   double-quote><double_quote> in which case the effective value if the
-//   string with the double-quotes removed, and escaped sequences of
-//   <backslash><char> replaced by <char>.
+// flags from an environtment variable, or (if the first non-whitespace in the
+// variable value is not '-'), a file named by that environment variable.
+//
+// The accepted syntax is that flags arguments are of the form --flag=value or
+// (for boolean flags) --flag, and are whitespace separated.  The <value> may be
+// one of:
+//
+//  - <non-whitespace, non-nul not starting with single-quote or double-quote>
+//    in which case the effective value is the string itself
+//  - <single-quote><characters string not containing nul or
+//    single-quote><single_quote> in which case the effective value is the
+//    string with the single-quotes removed
+//  - <double-quote><character string not containing nul or unesecaped
+//    double-quote><double_quote> in which case the effective value if the
+//    string with the double-quotes removed, and escaped sequences of
+//    <backslash><char> replaced by <char>.
 //
 // Flags values inconsistent with the type of the flag will be rejected by the
 // flag parser.
 //
 // Examples:
-//    TF_XLA_FLAGS="--foo=bar  --wombat='value with a space'"
 //
-//    TF_XLA_FLAGS=/tmp/flagfile
+//  - TF_XLA_FLAGS="--foo=bar  --wombat='value with a space'"
+//  - TF_XLA_FLAGS=/tmp/flagfile
+//
 // where /tmp/flagfile might contain
-//    --some_flag="This is a string containing a \" and a '."
-//    --another_flag=wombats
+//
+//  --some_flag="This is a string containing a \" and a '."
+//  --another_flag=wombats
 
 #include <vector>
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/command_line_flags.h"
 
 namespace xla {
 
-// Call tensorflow::Flags::Parse(argc, argv, flag_list) against any as yet
-// unrecognized flags passed in from the environment, and return its
-// return value.
-bool ParseFlagsFromEnv(const std::vector<tensorflow::Flag>& flag_list);
+// Calls tensorflow::Flags::Parse(argc, argv, flag_list) against any as yet
+// unrecognized flags passed in the environment variable `envvar`, and returns
+// its return value.
+bool ParseFlagsFromEnv(absl::string_view envvar,
+                       const std::vector<tensorflow::Flag>& flag_list);
 
 // Used only for testing.  Not to be used by clients.
-void ResetFlagsFromEnvForTesting(int** pargc, std::vector<char*>** pargv);
+void ResetFlagsFromEnvForTesting(absl::string_view envvar, int** pargc,
+                                 std::vector<char*>** pargv);
 
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/parse_flags_from_env_test.cc b/tensorflow/compiler/xla/parse_flags_from_env_test.cc
index edd6538402..26c15c3e98 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env_test.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env_test.cc
@@ -37,12 +37,12 @@ static void TestParseFlagsFromEnv(const char* msg) {
   // Initialize module under test.
   int* pargc;
   std::vector<char*>* pargv;
-  ResetFlagsFromEnvForTesting(&pargc, &pargv);
+  ResetFlagsFromEnvForTesting("TF_XLA_FLAGS", &pargc, &pargv);
 
   // Ensure that environment variable can be parsed when
   // no flags are expected.
   std::vector<tensorflow::Flag> empty_flag_list;
-  bool parsed_ok = ParseFlagsFromEnv(empty_flag_list);
+  bool parsed_ok = ParseFlagsFromEnv("TF_XLA_FLAGS", empty_flag_list);
   CHECK(parsed_ok) << msg;
   const std::vector<char*>& argv_first = *pargv;
   CHECK_NE(argv_first[0], nullptr) << msg;
@@ -65,7 +65,7 @@ static void TestParseFlagsFromEnv(const char* msg) {
       tensorflow::Flag("single_quoted", &single_quoted, ""),
       tensorflow::Flag("double_quoted", &double_quoted, ""),
   };
-  parsed_ok = ParseFlagsFromEnv(flag_list);
+  parsed_ok = ParseFlagsFromEnv("TF_XLA_FLAGS", flag_list);
   CHECK_EQ(*pargc, 1) << msg;
   const std::vector<char*>& argv_second = *pargv;
   CHECK_NE(argv_second[0], nullptr) << msg;
@@ -171,7 +171,7 @@ int main(int argc, char* argv[]) {
       tensorflow::Flag("int_flag", &int_flag, "An integer flag to test with"),
   };
   xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
-  bool parse_ok = xla::ParseFlagsFromEnv(flag_list);
+  bool parse_ok = xla::ParseFlagsFromEnv("TF_XLA_FLAGS", flag_list);
   if (!parse_ok) {
     LOG(QFATAL) << "can't parse from environment\n" << usage;
   }
-- 
GitLab


From 627e22a24758b8568d9aaae3d002827b61bb6095 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 16:59:59 -0800
Subject: [PATCH 0263/1554] Do not lower cond_v2 or while_v2 to v1 control flow
 when the context requires the single threaded executor.

the SingleThreadExecutor does not support v1 control flow ops.

PiperOrigin-RevId: 221538796
---
 tensorflow/python/kernel_tests/cond_v2_test.py | 18 ++++++++++++++++++
 tensorflow/python/ops/control_flow_util_v2.py  | 14 ++++++++++----
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index b077b853ed..cf41134e7c 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -666,6 +667,23 @@ class CondV2Test(test.TestCase):
           if_found,
           "An `If` op was not found, but the graph should not be lowered.")
 
+  def testLoweringDisabledWithSingleThreadedExecutorContext(self):
+    with self.session(graph=ops.Graph()) as sess:
+      @function.defun
+      def _add_cond(x):
+        return cond_v2.cond_v2(
+            constant_op.constant(True, name="pred"),
+            lambda: x,
+            lambda: x + 1)
+
+      x = array_ops.placeholder(shape=None, dtype=dtypes.float32)
+      with context.function_executor_type("SINGLE_THREADED_EXECUTOR"):
+        out_cond = _add_cond(x)
+
+      # The fact that sess.run() succeeds means lowering is disabled, because
+      # the single threaded executor does not support cond v1 ops.
+      sess.run(out_cond, feed_dict={x: 1.0})
+
 
 class CondV2CollectionTest(test.TestCase):
 
diff --git a/tensorflow/python/ops/control_flow_util_v2.py b/tensorflow/python/ops/control_flow_util_v2.py
index 2eec423991..5f56850884 100644
--- a/tensorflow/python/ops/control_flow_util_v2.py
+++ b/tensorflow/python/ops/control_flow_util_v2.py
@@ -103,14 +103,20 @@ def maybe_set_lowering_attr(op):
   pruning. This brings v2 control flow closer to feature parity with v1 control
   flow.
 
-  However, we do not lower `If` and `While` ops in the XLA context because it is
-  easier for XLA to apply its own optimizations when dealing with un-lowered
-  control flow operators than with low-level control flow primitives.
+  However, we do not lower in the following cases:
+    - When the `If` or `While` ops are in the XLA context. Because it is easier
+      for XLA to apply its own optimizations when dealing with un-lowered
+      control flow operators than with low-level control flow primitives.
+    - When the eager execution context specifies the executor of functions to
+      be the single threaded executor (see context.function_executor_type()).
+      Because the single threaded executor does not support v1 control flow ops.
 
   Args:
     op: An `If` or `While` Operation.
   """
-  if not control_flow_util.IsInXLAContext(op):
+  if (not control_flow_util.IsInXLAContext(op) and
+      context.context().get_function_call_options().executor_type
+      != "SINGLE_THREADED_EXECUTOR"):
     # pylint: disable=protected-access
     op._set_attr("_lower_using_switch_merge", attr_value_pb2.AttrValue(b=True))
     # pylint: enable=protected-access
-- 
GitLab


From 3e5d507b1d1803c8b7b4419cab02abb3fc6a35ac Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 17:08:00 -0800
Subject: [PATCH 0264/1554] Step 1 of *args, **kwargs -> args, kwargs
 transition for DistributionStrategy.update() and .update_non_slot().

PiperOrigin-RevId: 221540518
---
 .../distribute/python/mirrored_strategy.py    | 12 +--
 .../distribute/python/one_device_strategy.py  | 10 +--
 .../python/parameter_server_strategy.py       | 12 +--
 .../contrib/distribute/python/tpu_strategy.py | 17 ++---
 .../contrib/distribute/python/values.py       |  4 +-
 tensorflow/python/training/distribute.py      | 75 +++++++++++++------
 6 files changed, 73 insertions(+), 57 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index bf065713a0..4bb2cb1990 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -612,11 +612,9 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     return self._get_cross_tower_ops().batch_reduce(reduce_op,
                                                     value_destination_pairs)
 
-  def _update(self, var, options, fn, *args, **kwargs):
+  def _update(self, var, fn, args, kwargs, group):
     # TODO(josh11b): In eager mode, use one thread per device.
     assert isinstance(var, values.DistributedVariable)
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
     updates = {}
     for d, v in var._index.items():  # pylint: disable=protected-access
       name = "update_%d" % self._device_index.get(d)
@@ -625,12 +623,10 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
         updates[d] = fn(v,
                         *values.select_device_mirrored(d, args),
                         **values.select_device_mirrored(d, kwargs))
-    return values.update_regroup(self, updates, should_group)
+    return values.update_regroup(self, updates, group)
 
-  def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs):
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
     assert isinstance(colocate_with, list)
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
     # TODO(josh11b): In eager mode, use one thread per device.
     updates = {}
     for d in colocate_with:
@@ -638,7 +634,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
       with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
         updates[d] = fn(*values.select_device_mirrored(d, args),
                         **values.select_device_mirrored(d, kwargs))
-    return values.update_regroup(self, updates, should_group)
+    return values.update_regroup(self, updates, group)
 
   def read_var(self, replica_local_var):
     """Read the aggregate value of a replica-local variable."""
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 8ce8e11589..96f5f739d5 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -126,18 +126,16 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     del reduce_op, destinations
     return value
 
-  def _update(self, var, options, fn, *args, **kwargs):
+  def _update(self, var, fn, args, kwargs, group):
     # The implementations of _update() and _update_non_slot() are identical
     # except _update() passes `var` as the first argument to `fn()`.
-    return self._update_non_slot(var, options, fn, var, *args, **kwargs)
+    return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
 
-  def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs):
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
     del colocate_with
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
     with ops.device(self._device), distribute_lib.UpdateContext(self._device):
       result = fn(*args, **kwargs)
-      if should_group:
+      if group:
         return result
       else:
         return nest.map_structure(self._unwrap, result)
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 49733dd7a6..9cd71293f8 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -350,30 +350,26 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
 
     return nest.map_structure(_select_fn, structured)
 
-  def _update(self, var, options, fn, *args, **kwargs):
+  def _update(self, var, fn, args, kwargs, group):
     if isinstance(var, values.AggregatingVariable):
       var = var.get()
     if not isinstance(var, resource_variable_ops.ResourceVariable):
       raise ValueError(
           "You can not update `var` %r. It must be a Variable." % var)
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
     with ops.colocate_with(var), distribute_lib.UpdateContext(var.device):
       result = fn(var, *self._select_single_value(args),
                   **self._select_single_value(kwargs))
-      if should_group:
+      if group:
         return result
       else:
         return nest.map_structure(self._unwrap, result)
 
   # TODO(yuefengz): does it need to call _select_single_value?
-  def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs):
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
     with ops.device(
         colocate_with.device), distribute_lib.UpdateContext(colocate_with):
       result = fn(*args, **kwargs)
-      if should_group:
+      if group:
         return result
       else:
         return nest.map_structure(self._unwrap, result)
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index ae5c55a346..adc075011d 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -467,13 +467,10 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
       return output * (1. / len(value))
     return output
 
-  def _update(self, var, options, fn, *args, **kwargs):
+  def _update(self, var, fn, args, kwargs, group):
     assert isinstance(var, values.TPUMirroredVariable)
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
-
     if values._enclosing_tpu_context() is not None:  # pylint: disable=protected-access
-      if should_group:
+      if group:
         return fn(var, *args, **kwargs)
       else:
         return [fn(var, *args, **kwargs)]
@@ -488,9 +485,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
         updates[d] = fn(v,
                         *values.select_device_mirrored(d, args),
                         **values.select_device_mirrored(d, kwargs))
-    return values.update_regroup(self, updates, should_group)
-
-  # TODO(josh11b): Need to implement _update_non_slot()!
+    return values.update_regroup(self, updates, group)
 
   def read_var(self, var):
     assert isinstance(var, values.TPUMirroredVariable)
@@ -553,14 +548,12 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def non_slot_devices(self, var_list):
     return self._host_device
 
-  def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs):
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
     del colocate_with
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
     with ops.device(self._host_device), distribute_lib.UpdateContext(
         self._host_device):
       result = fn(*args, **kwargs)
-      if should_group:
+      if group:
         return result
       else:
         return nest.map_structure(self._unwrap, result)
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index e2931fcacd..f565381b49 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -1063,10 +1063,10 @@ def select_device_mirrored(device, structured):
   return nest.map_structure(_get_mirrored, structured)
 
 
-def update_regroup(strategy, updates, should_group):
+def update_regroup(strategy, updates, group):
   """Regroup for an update, with dependencies to ensure all updates execute."""
   regrouped = regroup(updates, Mirrored)
-  if not should_group:
+  if not group:
     return nest.map_structure(strategy.unwrap, regrouped)
   grouped_flat = []
   for u in nest.flatten(regrouped):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 88ee5376fb..883a22389a 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -514,7 +514,7 @@ class DistributionStrategy(object):
         # operates on v1 from var1, v2 from var2, and v3 from var3
 
       # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too.
-      distribution_strategy.update(v1, fn, v2, v3)
+      distribution_strategy.update(v1, fn, args=(v2, v3))
     ```
 
     Args:
@@ -838,7 +838,7 @@ class DistributionStrategy(object):
     results = {}
     for device, v in var:
       with tf.device(device):
-        # *args and **kwargs will be unwrapped if they are mirrored.
+        # args and kwargs will be unwrapped if they are mirrored.
         results[device] = fn(v, *args, **kwargs)
     return merged(results)
     ```
@@ -852,23 +852,41 @@ class DistributionStrategy(object):
     Args:
       var: Variable, possibly mirrored to multiple devices, to operate on.
       fn: Function to call. Should take the variable as the first argument.
-      *args: Additional positional arguments to pass to `fn()`.
-      **kwargs: Keyword arguments to pass to `fn()`. If "grouped=False" is
-        specified, the return value will be unwrapped.
+      args: Tuple or list. Additional positional arguments to pass to `fn()`.
+      kwargs: Dict with keyword arguments to pass to `fn()`.
+      group: Boolean. Defaults to True. If False, the return value will be
+        unwrapped.
 
     Returns:
       By default, the merged return value of `fn` across all replicas.  The
       merged result has dependencies to make sure that if it is evaluated at
       all, the side effects (updates) will happen on every replica. If instead
-      "grouped=False" is specified, this function will return a nest of lists
+      "group=False" is specified, this function will return a nest of lists
       where each list has an element per replica, and the caller is responsible
       for ensuring all elements are executed.
     """
     _require_cross_replica_context(self)
-    options = {"grouped": kwargs.pop("grouped", True)}
-    return self._update(var, options, fn, *args, **kwargs)
+    group = kwargs.pop("group", True)
+    # We temporarily support "grouped" in addition to "group" for backward-
+    # compatibility.
+    group = kwargs.pop("grouped", True) and group
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to update")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to update")
+      kwargs = k
+    return self._update(var, fn, args, kwargs, group)
 
-  def _update(self, var, options, fn, *args, **kwargs):
+  def _update(self, var, fn, args, kwargs, group):
     raise NotImplementedError("must be implemented in descendants")
 
   def update_non_slot(self, colocate_with, fn, *args, **kwargs):
@@ -877,19 +895,36 @@ class DistributionStrategy(object):
     Args:
       colocate_with: The return value of `non_slot_devices()`.
       fn: Function to execute.
-      *args: Positional arguments to pass to `fn()`.
-      **kwargs: Keyword arguments to pass to `fn()`. If "grouped=False" is
-        specified, the return value will be unwrapped and the caller is
-        responsible for ensuring all elements are executed.
+      args: Tuple or list. Positional arguments to pass to `fn()`.
+      kwargs: Dict with keyword arguments to pass to `fn()`.
+      group: Boolean. Defaults to True. If False, the return value will be
+        unwrapped.
 
     Returns:
       Return value of `fn`, possibly merged across devices.
     """
     _require_cross_replica_context(self)
-    options = {"grouped": kwargs.pop("grouped", True)}
-    return self._update_non_slot(colocate_with, options, fn, *args, **kwargs)
+    group = kwargs.pop("group", True)
+    # We temporarily support "grouped" in addition to "group" for backward-
+    # compatibility.
+    group = kwargs.pop("grouped", True) and group
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to update_non_slot")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to update_non_slot")
+      kwargs = k
+    return self._update_non_slot(colocate_with, fn, args, kwargs, group)
 
-  def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs):
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
     raise NotImplementedError("must be implemented in descendants")
 
   def unwrap(self, value):
@@ -1173,14 +1208,12 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     del reduce_op, destinations
     return value
 
-  def _update(self, var, options, fn, *args, **kwargs):
+  def _update(self, var, fn, args, kwargs, group):
     # The implementations of _update() and _update_non_slot() are identical
     # except _update() passes `var` as the first argument to `fn()`.
-    return self._update_non_slot(var, options, fn, var, *args, **kwargs)
+    return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
 
-  def _update_non_slot(self, colocate_with, options, fn, *args, **kwargs):
-    should_group = options.pop("grouped")
-    assert not options  # Validate that we are processing all of the options.
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, should_group):
     # TODO(josh11b): Figure out what we should be passing to UpdateContext()
     # once that value is used for something.
     with ops.colocate_with(colocate_with), UpdateContext(colocate_with):
-- 
GitLab


From df74b804064bd16e1fe4aed2940c5f536c993dfc Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 14 Nov 2018 17:25:18 -0800
Subject: [PATCH 0265/1554] Disable flaky testRunMetadata

PiperOrigin-RevId: 221542805
---
 tensorflow/python/eager/function_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index ff1d947bf7..fad827953a 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -645,7 +645,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     # Ensure that v is watched again.
     self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
 
-  def testRunMetadata(self):
+  def disabled_testRunMetadata(self):
 
     @def_function.function
     def f(x):
-- 
GitLab


From 6f99a8181d8bf7d9e48673c65d26139432a0af67 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 14 Nov 2018 17:29:35 -0800
Subject: [PATCH 0266/1554] Implement Keras V2 Adam Optimizer.

PiperOrigin-RevId: 221543342
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  30 +-
 tensorflow/python/keras/optimizer_v2/adam.py  |  62 +++-
 .../python/keras/optimizer_v2/adam_test.py    | 300 ++++++++++++++++++
 .../python/keras/optimizer_v2/optimizer_v2.py |   1 +
 .../keras/optimizer_v2/optimizer_v2_test.py   |   6 +-
 5 files changed, 381 insertions(+), 18 deletions(-)
 create mode 100644 tensorflow/python/keras/optimizer_v2/adam_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 6a973058f7..eaa764992f 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -7,6 +7,7 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 py_library(
@@ -29,6 +30,25 @@ py_library(
     ],
 )
 
+cuda_py_test(
+    name = "adam_test",
+    size = "medium",
+    srcs = ["adam_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+    shard_count = 4,
+)
+
 cuda_py_test(
     name = "gradient_descent_test",
     size = "medium",
@@ -48,21 +68,21 @@ cuda_py_test(
     shard_count = 4,
 )
 
-cuda_py_test(
+py_test(
     name = "optimizer_v2_test",
     size = "medium",
     srcs = ["optimizer_v2_test.py"],
-    additional_deps = [
+    deps = [
         ":optimizer_v2",
-        "//tensorflow/python/eager:def_function",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:clip_ops",
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:clip_ops",
         "//tensorflow/python:gradients",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variables",
+        "//tensorflow/python/eager:def_function",
     ],
 )
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index b05811c419..4b3f8563ff 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -1,4 +1,4 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,8 +17,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.training import training_ops
 
 
@@ -106,23 +110,63 @@ class Adam(optimizer_v2.OptimizerV2):
       self.add_slot(var, 'v')
 
   def _resource_apply_dense(self, grad, var):
+    grad_dtype = grad.dtype.base_dtype
     m = self.get_slot(var, 'm')
     v = self.get_slot(var, 'v')
-    # TODO(tanzheny): let optimizer have its own step counter, and let
-    # beta1_power and beta2_power depend on it.
+    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
+    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
+    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_power = math_ops.pow(beta_1_t, local_step)
+    beta_2_power = math_ops.pow(beta_2_t, local_step)
     return training_ops.resource_apply_adam(
         var.handle,
         m.handle,
         v.handle,
-        math_ops.cast(self._get_hyper('beta_1'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('beta_2'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('learning_rate'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('beta_1'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('beta_2'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('epsilon'), grad.dtype.base_dtype),
+        beta_1_power,
+        beta_2_power,
+        math_ops.cast(self._get_hyper('learning_rate'), grad_dtype),
+        beta_1_t,
+        beta_2_t,
+        math_ops.cast(self._get_hyper('epsilon'), grad_dtype),
         grad,
         use_locking=self._use_locking)
 
+  def _resource_apply_sparse(self, grad, var, indices):
+
+    def _resource_scatter_add(x, i, v):
+      with ops.control_dependencies(
+          [resource_variable_ops.resource_scatter_add(x.handle, i, v)]):
+        return x.value()
+
+    var_dtype = var.dtype.base_dtype
+    local_step = math_ops.cast(self.iterations + 1, var_dtype)
+    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), var_dtype)
+    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), var_dtype)
+    beta_1_power = math_ops.pow(beta_1_t, local_step)
+    beta_2_power = math_ops.pow(beta_2_t, local_step)
+    lr_t = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
+    epsilon_t = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    lr = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power))
+
+    # m_t = beta1 * m + (1 - beta1) * g_t
+    m = self.get_slot(var, 'm')
+    m_scaled_g_values = grad * (1 - beta_1_t)
+    m_t = state_ops.assign(m, m * beta_1_t, use_locking=self._use_locking)
+    with ops.control_dependencies([m_t]):
+      m_t = _resource_scatter_add(m, indices, m_scaled_g_values)
+
+    # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
+    v = self.get_slot(var, 'v')
+    v_scaled_g_values = (grad * grad) * (1 - beta_2_t)
+    v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking)
+    with ops.control_dependencies([v_t]):
+      v_t = _resource_scatter_add(v, indices, v_scaled_g_values)
+
+    v_sqrt = math_ops.sqrt(v_t)
+    var_update = state_ops.assign_sub(
+        var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
+    return control_flow_ops.group(*[var_update, m_t, v_t])
+
   def get_config(self):
     config = super(Adam, self).get_config()
     config.update({
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
new file mode 100644
index 0000000000..69b601ffc7
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -0,0 +1,300 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Adam."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+def adam_update_numpy(param,
+                      g_t,
+                      t,
+                      m,
+                      v,
+                      alpha=0.001,
+                      beta1=0.9,
+                      beta2=0.999,
+                      epsilon=1e-8):
+  alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
+
+  m_t = beta1 * m + (1 - beta1) * g_t
+  v_t = beta2 * v + (1 - beta2) * g_t * g_t
+
+  param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon)
+  return param_t, m_t, v_t
+
+
+def get_beta_accumulators(opt, dtype):
+  local_step = math_ops.cast(opt.iterations + 1, dtype)
+  beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype)
+  beta_1_power = math_ops.pow(beta_1_t, local_step)
+  beta_2_t = math_ops.cast(opt._get_hyper("beta_2"), dtype)
+  beta_2_power = math_ops.pow(beta_2_t, local_step)
+  return (beta_1_power, beta_2_power)
+
+
+class AdamOptimizerTest(test.TestCase):
+
+  def testSparse(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0_np_indices = np.array([0, 1], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np),
+            constant_op.constant(grads0_np_indices), constant_op.constant([2]))
+        grads1_np_indices = np.array([0, 1], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np),
+            constant_op.constant(grads1_np_indices), constant_op.constant([2]))
+        opt = adam.Adam()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSparseDevicePlacement(self):
+    for index_dtype in [dtypes.int32, dtypes.int64]:
+      with self.cached_session(force_gpu=test.is_gpu_available()):
+        # If a GPU is available, tests that all optimizer ops can be placed on
+        # it (i.e. they have GPU kernels).
+        var = variables.Variable([[1.0], [2.0]])
+        indices = constant_op.constant([0, 1], dtype=index_dtype)
+        gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices))
+        optimizer = adam.Adam(3.0)
+        minimize_op = optimizer.minimize(gathered_sum, var_list=[var])
+        variables.global_variables_initializer().run()
+        minimize_op.run()
+
+  def testSparseRepeatedIndices(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        repeated_index_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        aggregated_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        grad_repeated_index = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1, 0.1], shape=[2, 1], dtype=dtype),
+            constant_op.constant([1, 1]),
+            constant_op.constant([2, 1]))
+        grad_aggregated = ops.IndexedSlices(
+            constant_op.constant(
+                [0.2], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        repeated_update = adam.Adam().apply_gradients(
+            [(grad_repeated_index, repeated_index_update_var)])
+        aggregated_update = adam.Adam().apply_gradients(
+            [(grad_aggregated, aggregated_update_var)])
+        variables.global_variables_initializer().run()
+        self.assertAllClose(aggregated_update_var.eval(),
+                            repeated_index_update_var.eval())
+        for _ in range(3):
+          repeated_update.run()
+          aggregated_update.run()
+          self.assertAllClose(aggregated_update_var.eval(),
+                              repeated_index_update_var.eval())
+
+  def doTestBasic(self, use_callable_params=False):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = lambda: 0.001
+        beta1 = lambda: 0.9
+        beta2 = lambda: 0.999
+        epsilon = lambda: 1e-8
+        if not use_callable_params:
+          learning_rate = learning_rate()
+          beta1 = beta1()
+          beta2 = beta2()
+          epsilon = epsilon()
+
+        opt = adam.Adam(learning_rate=learning_rate)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+        self.evaluate(variables.global_variables_initializer())
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          if not context.executing_eagerly():
+            self.evaluate(update)
+          elif t > 1:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta_2_power))
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testResourceBasic(self):
+    self.doTestBasic()
+
+  def testBasicCallableParams(self):
+    with context.eager_mode():
+      self.doTestBasic(use_callable_params=True)
+
+  def testTensorLearningRate(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = adam.Adam(constant_op.constant(0.001))
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSharing(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = adam.Adam()
+        update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 3 steps of intertwined Adam1 and Adam2.
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          if t % 2 == 0:
+            update1.run()
+          else:
+            update2.run()
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSlotsUniqueEager(self):
+    with context.eager_mode():
+      v1 = resource_variable_ops.ResourceVariable(1.)
+      v2 = resource_variable_ops.ResourceVariable(1.)
+      opt = adam.Adam(1.)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, hyper variables, and two unique slot
+      # variables for v1 and v2 respectively.
+      self.assertEqual(9, len(set(opt.variables())))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index d15665dac8..68b5368711 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -395,6 +395,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       self._iterations = self.add_weight(
           "iter",
           shape=[],
+          dtype=dtypes.int64,
           trainable=False,
           aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA)
       self._weights.append(self._iterations)
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index e43327ff12..cb0886ac9a 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -414,10 +414,8 @@ class OptimizerTest(test.TestCase):
         opt.minimize(loss, [var])
         return var
 
-      self.assertAllClose([0., 1.], fn())
-      # This is just to test tf.function. The values needs to be updated
-      # when adam updates beta_1_power.
-      self.assertAllClose([-1.343838, -0.343838], fn())
+      self.assertAllClose([0., 1.], fn(), atol=1e-4)
+      self.assertAllClose([-1, 0.], fn(), atol=1e-4)
 
   @test_util.run_in_graph_and_eager_modes
   def testOptimizerWithKerasModel(self):
-- 
GitLab


From 5847293aeb9ab45a02c4231c40569a15bd4541c6 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Wed, 14 Nov 2018 17:30:43 -0800
Subject: [PATCH 0267/1554] Create a simple PIP package generator for
 standalone TF Lite.

Based on original dkovalev@

PiperOrigin-RevId: 221543483
---
 tensorflow/lite/tools/make/Makefile           |   3 +
 tensorflow/lite/tools/pip_package/MANIFEST.in |   1 +
 tensorflow/lite/tools/pip_package/README.md   |  33 ++++
 .../tools/pip_package/build_pip_package.sh    |  54 +++++++
 tensorflow/lite/tools/pip_package/setup.py    | 150 ++++++++++++++++++
 5 files changed, 241 insertions(+)
 create mode 100644 tensorflow/lite/tools/pip_package/MANIFEST.in
 create mode 100644 tensorflow/lite/tools/pip_package/README.md
 create mode 100644 tensorflow/lite/tools/pip_package/build_pip_package.sh
 create mode 100644 tensorflow/lite/tools/pip_package/setup.py

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 8f12355854..28374f13b7 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -208,6 +208,9 @@ $(BENCHMARK_BINARY) : $(BENCHMARK_LIB)
 
 benchmark: $(BENCHMARK_BINARY)
 
+libdir:
+	@echo $(LIBDIR)
+
 # Gets rid of all generated files.
 clean:
 	rm -rf $(MAKEFILE_DIR)/gen
diff --git a/tensorflow/lite/tools/pip_package/MANIFEST.in b/tensorflow/lite/tools/pip_package/MANIFEST.in
new file mode 100644
index 0000000000..bb574e63a3
--- /dev/null
+++ b/tensorflow/lite/tools/pip_package/MANIFEST.in
@@ -0,0 +1 @@
+recursive-include * *.py
diff --git a/tensorflow/lite/tools/pip_package/README.md b/tensorflow/lite/tools/pip_package/README.md
new file mode 100644
index 0000000000..8190782c39
--- /dev/null
+++ b/tensorflow/lite/tools/pip_package/README.md
@@ -0,0 +1,33 @@
+# Building TensorFlow Lite Standalone Pip
+
+Many users would like to deploy TensorFlow lite interpreter and use it from
+Python without requiring the rest of TensorFlow.
+
+## Steps
+
+To build a binary wheel run this script:
+```
+sudo apt install swig libjpeg-dev zlib1g-dev python3-dev python3-numpy
+sh tensorflow/lite/tools/pip_package/build_pip_package.sh
+```
+That will print out some output and a .whl file. You can then install that
+```
+pip install --upgrade <wheel>
+```
+
+Note, unlike tensorflow this will be installed to a tflite_runtime namespace.
+You can then use the Tensorflow Lite interpreter as.
+```
+import tflite_runtime as tflr
+interpreter = tflr.lite.Interpreter(model_path="foo.tflite")
+```
+
+This currently works to build on Linux machines including Raspberry Pi. In
+the future, cross compilation to smaller SOCs like Raspberry Pi from
+bigger host will be supported.
+
+## Caveats
+
+* You cannot use TensorFlow Select ops, only TensorFlow Lite builtins.
+* Currently custom ops and delegates cannot be registered.
+
diff --git a/tensorflow/lite/tools/pip_package/build_pip_package.sh b/tensorflow/lite/tools/pip_package/build_pip_package.sh
new file mode 100644
index 0000000000..2887ce8471
--- /dev/null
+++ b/tensorflow/lite/tools/pip_package/build_pip_package.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+# Find where this script lives and then the Tensorflow root.
+MY_DIRECTORY=`dirname $0`
+export TENSORFLOW_SRC_ROOT=`realpath $MY_DIRECTORY/../../../..`
+
+export TENSORFLOW_VERSION=`grep "_VERSION = " $TENSORFLOW_SRC_ROOT/tensorflow/tools/pip_package/setup.py  | cut -d'=' -f 2 | sed "s/[ '-]//g"`;
+
+
+# Build a pip build tree.
+BUILD_ROOT=/tmp/tflite_pip
+rm -rf $BUILD_ROOT
+mkdir -p $BUILD_ROOT/tflite_runtime/lite
+mkdir -p $BUILD_ROOT/tflite_runtime/lite/python
+
+# Build an importable module tree
+cat > $BUILD_ROOT/tflite_runtime/__init__.py <<EOF;
+import tflite_runtime.lite.interpreter
+EOF
+
+cat > $BUILD_ROOT/tflite_runtime/lite/__init__.py <<EOF;
+from interpreter import Interpreter as Interpreter
+EOF
+
+cat > $BUILD_ROOT/tflite_runtime/lite/python/__init__.py <<EOF;
+# Python module for TensorFlow Lite
+EOF
+
+# Copy necessary source files
+TFLITE_ROOT=$TENSORFLOW_SRC_ROOT/tensorflow/lite
+cp -r  $TFLITE_ROOT/python/interpreter_wrapper $BUILD_ROOT
+cp $TFLITE_ROOT/python/interpreter.py $BUILD_ROOT/tflite_runtime/lite/
+cp $TFLITE_ROOT/tools/pip_package/setup.py $BUILD_ROOT
+cp $TFLITE_ROOT/tools/pip_package/MANIFEST.in $BUILD_ROOT
+
+# Build the Pip
+cd $BUILD_ROOT
+python setup.py bdist_wheel
diff --git a/tensorflow/lite/tools/pip_package/setup.py b/tensorflow/lite/tools/pip_package/setup.py
new file mode 100644
index 0000000000..64d62ee1f2
--- /dev/null
+++ b/tensorflow/lite/tools/pip_package/setup.py
@@ -0,0 +1,150 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TensorFlow Lite is for mobile and embedded devices.
+
+TensorFlow Lite is the official solution for running machine learning models on
+mobile and embedded devices. It enables on-device machine learning inference
+with low latency and a small binary size on Android, iOS, and other operating
+systems.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import multiprocessing
+import os
+import subprocess
+
+from distutils.command.build_ext import build_ext
+import numpy
+
+from setuptools import Extension
+from setuptools import find_packages
+from setuptools import setup
+from setuptools.command.build_py import build_py
+PACKAGE_NAME = 'tflite-runtime'
+PACKAGE_VERSION = os.environ['TENSORFLOW_VERSION']
+DOCLINES = __doc__.split('\n')
+PACKAGE = 'tflite_runtime.lite.python'
+TENSORFLOW_DIR = os.environ['TENSORFLOW_SRC_ROOT']
+
+# Setup cross compiling
+TARGET = (
+    os.environ['TENSORFLOW_TARGET'] if 'TENSORFLOW_TARGET' in os.environ
+    else None)
+if TARGET == 'rpi':
+  os.environ['CXX'] = 'arm-linux-gnueabihf-g++'
+  os.environ['CC'] = 'arm-linux-gnueabihf-g++'
+MAKE_CROSS_OPTIONS = ['TARGET=%s' % TARGET]  if TARGET else []
+
+RELATIVE_MAKE_DIR = os.path.join('tensorflow', 'lite', 'tools', 'make')
+MAKE_DIR = os.path.join(TENSORFLOW_DIR, RELATIVE_MAKE_DIR)
+DOWNLOADS_DIR = os.path.join(MAKE_DIR, 'downloads')
+RELATIVE_MAKEFILE_PATH = os.path.join(RELATIVE_MAKE_DIR, 'Makefile')
+DOWNLOAD_SCRIPT_PATH = os.path.join(MAKE_DIR, 'download_dependencies.sh')
+
+
+def make_args(target='', quiet=True):
+  """Construct make command line."""
+  args = (['make', 'SHELL=/bin/bash', '-C', TENSORFLOW_DIR]
+          + MAKE_CROSS_OPTIONS +
+          ['-f', RELATIVE_MAKEFILE_PATH, '-j',
+           str(multiprocessing.cpu_count())])
+  if quiet:
+    args.append('--quiet')
+  if target:
+    args.append(target)
+  return args
+
+
+def make_output(target):
+  """Invoke make on the target and return output."""
+  return subprocess.check_output(make_args(target)).decode('utf-8').strip()
+
+
+def make():
+  """Invoke make to build tflite C++ sources.
+
+  Build dependencies:
+     apt-get install swig libjpeg-dev zlib1g-dev python3-dev python3-nump
+  """
+  subprocess.check_call(make_args(quiet=False))
+
+
+def download_dependencies():
+  """Download build dependencies if haven't done yet."""
+  if not os.path.isdir(DOWNLOADS_DIR) or not os.listdir(DOWNLOADS_DIR):
+    subprocess.check_call(DOWNLOAD_SCRIPT_PATH)
+
+
+class CustomBuildExt(build_ext, object):
+
+  def run(self):
+    download_dependencies()
+    make()
+
+    return super(CustomBuildExt, self).run()
+
+
+class CustomBuildPy(build_py, object):
+
+  def run(self):
+    self.run_command('build_ext')
+    return super(CustomBuildPy, self).run()
+
+
+LIB_TFLITE = 'tensorflow-lite'
+LIB_TFLITE_DIR = make_output('libdir')
+
+ext = Extension(
+    name='%s._interpreter_wrapper' % PACKAGE,
+    language='c++',
+    sources=['interpreter_wrapper/interpreter_wrapper.i',
+             'interpreter_wrapper/interpreter_wrapper.cc'],
+    swig_opts=['-c++',
+               '-I%s' % TENSORFLOW_DIR,
+               '-module', 'interpreter_wrapper',
+               '-outdir', '.'],
+    extra_compile_args=['-std=c++11'],
+    include_dirs=[TENSORFLOW_DIR,
+                  os.path.join(TENSORFLOW_DIR, 'tensorflow', 'lite', 'tools',
+                               'pip_package'),
+                  numpy.get_include(),
+                  os.path.join(DOWNLOADS_DIR, 'flatbuffers', 'include'),
+                  os.path.join(DOWNLOADS_DIR, 'absl')],
+    libraries=[LIB_TFLITE],
+    library_dirs=[LIB_TFLITE_DIR])
+
+
+setup(
+    name=PACKAGE_NAME,
+    version=PACKAGE_VERSION,
+    description=DOCLINES[0],
+    long_description='\n'.join(DOCLINES[2:]),
+    url='https://www.tensorflow.org/lite/',
+    author='Google Inc.',
+    author_email='opensource@google.com',
+    license='Apache 2.0',
+    include_package_data=True,
+    keywords='tflite tensorflow tensor machine learning',
+    packages=find_packages(exclude=[]),
+    ext_modules=[ext],
+    package_dir={PACKAGE: '.'},
+    cmdclass={
+        'build_ext': CustomBuildExt,
+        'build_py': CustomBuildPy,
+    }
+)
-- 
GitLab


From 13440cc25460113c4828f4355bcc7544a613daa1 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 14 Nov 2018 17:41:08 -0800
Subject: [PATCH 0268/1554] Update MKL-DNN to incorporate sgemm thread safety
 bug fix. Commit with the fix:
 https://github.com/intel/mkl-dnn/commit/af280290dc063a8f45fc14cabd3871984fd6123b
 Latest commit we are updating to:
 https://github.com/intel/mkl-dnn/commit/733fc908874c71a5285043931a1cf80aa923165c

PiperOrigin-RevId: 221544975
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 4db44278f3..b4f7aba046 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -112,11 +112,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "mkl_dnn",
         build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"),
-        sha256 = "363cc9239eacf8e7917753c6d8c94f767e4cd049160d0654a61ef32d5e1b3049",
-        strip_prefix = "mkl-dnn-4e333787e0d66a1dca1218e99a891d493dbc8ef1",
+        sha256 = "b100f57af4a2b59a3a37a1ba38f77b644d2107d758a1a7f4e51310063cd21e73",
+        strip_prefix = "mkl-dnn-733fc908874c71a5285043931a1cf80aa923165c",
         urls = [
-            "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/4e333787e0d66a1dca1218e99a891d493dbc8ef1.tar.gz",
-            "https://github.com/intel/mkl-dnn/archive/4e333787e0d66a1dca1218e99a891d493dbc8ef1.tar.gz",
+            "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/733fc908874c71a5285043931a1cf80aa923165c.tar.gz",
+            "https://github.com/intel/mkl-dnn/archive/733fc908874c71a5285043931a1cf80aa923165c.tar.gz",
         ],
     )
 
-- 
GitLab


From 450a56fcafd47742f0ac3ba44c8249e3a1a26656 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 17:42:05 -0800
Subject: [PATCH 0269/1554] Adding 'run_in_graph_and_eager_modes' and
 'run_all_in_graph_and_eager_modes' to kernel_tests/array_ops_test.py

PiperOrigin-RevId: 221545108
---
 .../python/kernel_tests/array_ops_test.py     | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 5452ec4804..5caed54c4d 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -48,23 +48,24 @@ from tensorflow.python.platform import test as test_lib
 
 class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_in_graph_and_eager_modes
   def testNonBatchMatrix(self):
     matrix = [[1, 2, 3], [4, 5, 6]]  # Shape (2, 3)
     expected_transposed = [[1, 4], [2, 5], [3, 6]]  # Shape (3, 2)
-    with self.cached_session():
-      transposed = array_ops.matrix_transpose(matrix)
-      self.assertEqual((3, 2), transposed.get_shape())
-      self.assertAllEqual(expected_transposed, transposed.eval())
+    transposed = array_ops.matrix_transpose(matrix)
+    self.assertEqual((3, 2), transposed.get_shape())
+    self.assertAllEqual(expected_transposed, transposed)
 
+  @test_util.run_in_graph_and_eager_modes
   def testConjugate(self):
     m = [[1 + 1j, 2 + 2j, 3 + 3j], [4 + 4j, 5 + 5j, 6 + 6j]]
     expected_transposed = [[1 - 1j, 4 - 4j], [2 - 2j, 5 - 5j], [3 - 3j, 6 - 6j]]
-    with self.cached_session():
-      matrix = ops.convert_to_tensor(m)
-      transposed = array_ops.matrix_transpose(matrix, conjugate=True)
-      self.assertEqual((3, 2), transposed.get_shape())
-      self.assertAllEqual(expected_transposed, transposed.eval())
+    matrix = ops.convert_to_tensor(m)
+    transposed = array_ops.matrix_transpose(matrix, conjugate=True)
+    self.assertEqual((3, 2), transposed.get_shape())
+    self.assertAllEqual(expected_transposed, transposed)
 
+  @test_util.run_in_graph_and_eager_modes
   def testBatchMatrix(self):
     matrix_0 = [[1, 2, 3], [4, 5, 6]]
     matrix_0_t = [[1, 4], [2, 5], [3, 6]]
@@ -72,10 +73,9 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
     matrix_1_t = [[11, 44], [22, 55], [33, 66]]
     batch_matrix = [matrix_0, matrix_1]  # Shape (2, 2, 3)
     expected_transposed = [matrix_0_t, matrix_1_t]  # Shape (2, 3, 2)
-    with self.cached_session():
-      transposed = array_ops.matrix_transpose(batch_matrix)
-      self.assertEqual((2, 3, 2), transposed.get_shape())
-      self.assertAllEqual(expected_transposed, transposed.eval())
+    transposed = array_ops.matrix_transpose(batch_matrix)
+    self.assertEqual((2, 3, 2), transposed.get_shape())
+    self.assertAllEqual(expected_transposed, transposed)
 
   def testNonBatchMatrixDynamicallyDefined(self):
     matrix = [[1, 2, 3], [4, 5, 6]]  # Shape (2, 3)
@@ -104,11 +104,11 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
               batch_matrix_ph: batch_matrix
           }))
 
+  @test_util.run_in_graph_and_eager_modes
   def testTensorWithStaticRankLessThanTwoRaisesBecauseNotAMatrix(self):
     vector = [1, 2, 3]
-    with self.cached_session():
-      with self.assertRaisesRegexp(ValueError, "should be a "):
-        array_ops.matrix_transpose(vector)
+    with self.assertRaisesRegexp(ValueError, "should be a "):
+      array_ops.matrix_transpose(vector)
 
 
 class BooleanMaskTest(test_util.TensorFlowTestCase):
-- 
GitLab


From 4b7ed4e8780ad67fbe1a5cec43e9c6e20877ffc1 Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Wed, 14 Nov 2018 18:00:38 -0800
Subject: [PATCH 0270/1554] Remove tf.get_collection, tf.get_collection_ref &
 tf.get_default_graph from TF 2.0

PiperOrigin-RevId: 221547523
---
 tensorflow/python/framework/ops.py            |  6 +--
 .../tools/api/golden/v2/tensorflow.pbtxt      | 12 -----
 tensorflow/tools/compatibility/renames_v2.py  | 44 +++++++++++++++++--
 3 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 84f7dc5c7c..11a73eb11d 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5677,7 +5677,7 @@ def reset_default_graph():
   _default_graph_stack.reset()
 
 
-@tf_export("get_default_graph")
+@tf_export(v1=["get_default_graph"])
 def get_default_graph():
   """Returns the default graph for the current thread.
 
@@ -6010,7 +6010,7 @@ def add_to_collections(names, value):
   get_default_graph().add_to_collections(names, value)
 
 
-@tf_export("get_collection_ref")
+@tf_export(v1=["get_collection_ref"])
 def get_collection_ref(key):
   """Wrapper for `Graph.get_collection_ref()` using the default graph.
 
@@ -6034,7 +6034,7 @@ def get_collection_ref(key):
   return get_default_graph().get_collection_ref(key)
 
 
-@tf_export("get_collection")
+@tf_export(v1=["get_collection"])
 def get_collection(key, scope=None):
   """Wrapper for `Graph.get_collection()` using the default graph.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index c41e9dcd57..9ce326edd2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -736,18 +736,6 @@ tf_module {
     name: "gather_nd"
     argspec: "args=[\'params\', \'indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "get_collection"
-    argspec: "args=[\'key\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "get_collection_ref"
-    argspec: "args=[\'key\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_default_graph"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "gradients"
     argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 06a33ef41e..cf93bd2f53 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -139,11 +139,13 @@ renames = {
     'tf.distributions.StudentT': 'tf.compat.v1.distributions.StudentT',
     'tf.distributions.Uniform': 'tf.compat.v1.distributions.Uniform',
     'tf.distributions.kl_divergence': 'tf.compat.v1.distributions.kl_divergence',
+    'tf.div': 'tf.compat.v1.div',
     'tf.enable_resource_variables': 'tf.compat.v1.enable_resource_variables',
     'tf.enable_v2_tensorshape': 'tf.compat.v1.enable_v2_tensorshape',
     'tf.encode_base64': 'tf.io.encode_base64',
     'tf.erf': 'tf.math.erf',
     'tf.erfc': 'tf.math.erfc',
+    'tf.expand_dims': 'tf.compat.v1.expand_dims',
     'tf.expm1': 'tf.math.expm1',
     'tf.extract_image_patches': 'tf.image.extract_image_patches',
     'tf.fake_quant_with_min_max_args': 'tf.quantization.fake_quant_with_min_max_args',
@@ -152,10 +154,27 @@ renames = {
     'tf.fake_quant_with_min_max_vars_gradient': 'tf.quantization.fake_quant_with_min_max_vars_gradient',
     'tf.fake_quant_with_min_max_vars_per_channel': 'tf.quantization.fake_quant_with_min_max_vars_per_channel',
     'tf.fake_quant_with_min_max_vars_per_channel_gradient': 'tf.quantization.fake_quant_with_min_max_vars_per_channel_gradient',
+    'tf.feature_column.bucketized_column': 'tf.compat.v1.feature_column.bucketized_column',
+    'tf.feature_column.categorical_column_with_hash_bucket': 'tf.compat.v1.feature_column.categorical_column_with_hash_bucket',
+    'tf.feature_column.categorical_column_with_identity': 'tf.compat.v1.feature_column.categorical_column_with_identity',
+    'tf.feature_column.categorical_column_with_vocabulary_file': 'tf.compat.v1.feature_column.categorical_column_with_vocabulary_file',
+    'tf.feature_column.categorical_column_with_vocabulary_list': 'tf.compat.v1.feature_column.categorical_column_with_vocabulary_list',
+    'tf.feature_column.crossed_column': 'tf.compat.v1.feature_column.crossed_column',
+    'tf.feature_column.embedding_column': 'tf.compat.v1.feature_column.embedding_column',
+    'tf.feature_column.indicator_column': 'tf.compat.v1.feature_column.indicator_column',
+    'tf.feature_column.input_layer': 'tf.compat.v1.feature_column.input_layer',
+    'tf.feature_column.linear_model': 'tf.compat.v1.feature_column.linear_model',
+    'tf.feature_column.make_parse_example_spec': 'tf.compat.v1.feature_column.make_parse_example_spec',
+    'tf.feature_column.numeric_column': 'tf.compat.v1.feature_column.numeric_column',
+    'tf.feature_column.shared_embedding_columns': 'tf.compat.v1.feature_column.shared_embedding_columns',
+    'tf.feature_column.weighted_categorical_column': 'tf.compat.v1.feature_column.weighted_categorical_column',
     'tf.fft': 'tf.signal.fft',
     'tf.fft2d': 'tf.signal.fft2d',
     'tf.fft3d': 'tf.signal.fft3d',
     'tf.floordiv': 'tf.math.floordiv',
+    'tf.get_collection': 'tf.compat.v1.get_collection',
+    'tf.get_collection_ref': 'tf.compat.v1.get_collection_ref',
+    'tf.get_default_graph': 'tf.compat.v1.get_default_graph',
     'tf.get_default_session': 'tf.compat.v1.get_default_session',
     'tf.get_local_variable': 'tf.compat.v1.get_local_variable',
     'tf.get_seed': 'tf.compat.v1.get_seed',
@@ -239,6 +258,7 @@ renames = {
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log_sigmoid': 'tf.math.log_sigmoid',
     'tf.logical_xor': 'tf.math.logical_xor',
+    'tf.losses.Reduction': 'tf.compat.v1.losses.Reduction',
     'tf.make_template': 'tf.compat.v1.make_template',
     'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto',
     'tf.manip.batch_to_space_nd': 'tf.batch_to_space_nd',
@@ -264,7 +284,6 @@ renames = {
     'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
-    'tf.nn.ctc_beam_search_decoder': 'tf.compat.v1.nn.ctc_beam_search_decoder',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
@@ -273,12 +292,12 @@ renames = {
     'tf.nn.rnn_cell.BasicRNNCell': 'tf.compat.v1.nn.rnn_cell.BasicRNNCell',
     'tf.nn.rnn_cell.GRUCell': 'tf.compat.v1.nn.rnn_cell.GRUCell',
     'tf.nn.rnn_cell.LSTMCell': 'tf.compat.v1.nn.rnn_cell.LSTMCell',
-    'tf.nn.softmax_cross_entropy_with_logits': 'tf.compat.v1.nn.softmax_cross_entropy_with_logits',
     'tf.nn.softmax_cross_entropy_with_logits_v2': 'tf.nn.softmax_cross_entropy_with_logits',
     'tf.nn.static_rnn': 'tf.compat.v1.nn.static_rnn',
     'tf.nn.uniform_candidate_sampler': 'tf.random.uniform_candidate_sampler',
     'tf.op_scope': 'tf.compat.v1.op_scope',
     'tf.orthogonal_initializer': 'tf.keras.initializers.Orthogonal',
+    'tf.pad': 'tf.compat.v1.pad',
     'tf.parse_example': 'tf.io.parse_example',
     'tf.parse_single_example': 'tf.io.parse_single_example',
     'tf.parse_single_sequence_example': 'tf.io.parse_single_sequence_example',
@@ -286,6 +305,16 @@ renames = {
     'tf.placeholder': 'tf.compat.v1.placeholder',
     'tf.placeholder_with_default': 'tf.compat.v1.placeholder_with_default',
     'tf.polygamma': 'tf.math.polygamma',
+    'tf.profiler.AdviceProto': 'tf.compat.v1.profiler.AdviceProto',
+    'tf.profiler.GraphNodeProto': 'tf.compat.v1.profiler.GraphNodeProto',
+    'tf.profiler.MultiGraphNodeProto': 'tf.compat.v1.profiler.MultiGraphNodeProto',
+    'tf.profiler.OpLogProto': 'tf.compat.v1.profiler.OpLogProto',
+    'tf.profiler.ProfileOptionBuilder': 'tf.compat.v1.profiler.ProfileOptionBuilder',
+    'tf.profiler.Profiler': 'tf.compat.v1.profiler.Profiler',
+    'tf.profiler.advise': 'tf.compat.v1.profiler.advise',
+    'tf.profiler.profile': 'tf.compat.v1.profiler.profile',
+    'tf.profiler.write_op_log': 'tf.compat.v1.profiler.write_op_log',
+    'tf.py_func': 'tf.compat.v1.py_func',
     'tf.python_io.TFRecordCompressionType': 'tf.io.TFRecordCompressionType',
     'tf.python_io.TFRecordOptions': 'tf.io.TFRecordOptions',
     'tf.python_io.TFRecordWriter': 'tf.io.TFRecordWriter',
@@ -315,12 +344,13 @@ renames = {
     'tf.rint': 'tf.math.rint',
     'tf.rsqrt': 'tf.math.rsqrt',
     'tf.saved_model.Builder': 'tf.compat.v1.saved_model.Builder',
+    'tf.saved_model.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.LEGACY_INIT_OP_KEY',
     'tf.saved_model.TRAINING': 'tf.saved_model.TRANING',
     'tf.saved_model.build_tensor_info': 'tf.compat.v1.saved_model.build_tensor_info',
     'tf.saved_model.builder.SavedModelBuilder': 'tf.compat.v1.saved_model.builder.SavedModelBuilder',
     'tf.saved_model.constants.ASSETS_DIRECTORY': 'tf.saved_model.ASSETS_DIRECTORY',
     'tf.saved_model.constants.ASSETS_KEY': 'tf.saved_model.ASSETS_KEY',
-    'tf.saved_model.constants.LEGACY_INIT_OP_KEY': 'tf.saved_model.LEGACY_INIT_OP_KEY',
+    'tf.saved_model.constants.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.constants.LEGACY_INIT_OP_KEY',
     'tf.saved_model.constants.MAIN_OP_KEY': 'tf.saved_model.MAIN_OP_KEY',
     'tf.saved_model.constants.SAVED_MODEL_FILENAME_PB': 'tf.saved_model.SAVED_MODEL_FILENAME_PB',
     'tf.saved_model.constants.SAVED_MODEL_FILENAME_PBTXT': 'tf.saved_model.SAVED_MODEL_FILENAME_PBTXT',
@@ -419,8 +449,11 @@ renames = {
     'tf.string_to_number': 'tf.strings.to_number',
     'tf.svd': 'tf.linalg.svd',
     'tf.tables_initializer': 'tf.compat.v1.tables_initializer',
+    'tf.test.assert_equal_graph_def': 'tf.compat.v1.test.assert_equal_graph_def',
+    'tf.test.compute_gradient': 'tf.compat.v1.test.compute_gradient',
     'tf.test.compute_gradient_error': 'tf.compat.v1.test.compute_gradient_error',
     'tf.test.get_temp_dir': 'tf.compat.v1.test.get_temp_dir',
+    'tf.test.mock': 'tf.compat.v1.test.mock',
     'tf.test.test_src_dir_path': 'tf.compat.v1.test.test_src_dir_path',
     'tf.to_bfloat16': 'tf.compat.v1.to_bfloat16',
     'tf.to_complex128': 'tf.compat.v1.to_complex128',
@@ -431,6 +464,8 @@ renames = {
     'tf.to_int64': 'tf.compat.v1.to_int64',
     'tf.trace': 'tf.linalg.trace',
     'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession',
+    'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader',
+    'tf.train.ProfilerHook': 'tf.compat.v1.train.ProfilerHook',
     'tf.train.QueueRunner': 'tf.compat.v1.train.QueueRunner',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
     'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef',
@@ -453,6 +488,7 @@ renames = {
     'tf.train.get_or_create_global_step': 'tf.compat.v1.train.get_or_create_global_step',
     'tf.train.global_step': 'tf.compat.v1.train.global_step',
     'tf.train.import_meta_graph': 'tf.compat.v1.train.import_meta_graph',
+    'tf.train.init_from_checkpoint': 'tf.compat.v1.train.init_from_checkpoint',
     'tf.train.input_producer': 'tf.compat.v1.train.input_producer',
     'tf.train.inverse_time_decay': 'tf.compat.v1.train.inverse_time_decay',
     'tf.train.limit_epochs': 'tf.compat.v1.train.limit_epochs',
@@ -471,6 +507,7 @@ renames = {
     'tf.train.queue_runner.start_queue_runners': 'tf.compat.v1.train.queue_runner.start_queue_runners',
     'tf.train.range_input_producer': 'tf.compat.v1.train.range_input_producer',
     'tf.train.remove_checkpoint': 'tf.compat.v1.train.remove_checkpoint',
+    'tf.train.replica_device_setter': 'tf.compat.v1.train.replica_device_setter',
     'tf.train.shuffle_batch': 'tf.compat.v1.train.shuffle_batch',
     'tf.train.shuffle_batch_join': 'tf.compat.v1.train.shuffle_batch_join',
     'tf.train.slice_input_producer': 'tf.compat.v1.train.slice_input_producer',
@@ -492,6 +529,7 @@ renames = {
     'tf.variables_initializer': 'tf.compat.v1.variables_initializer',
     'tf.variance_scaling_initializer': 'tf.keras.initializers.VarianceScaling',
     'tf.verify_tensor_all_finite': 'tf.debugging.assert_all_finite',
+    'tf.wrap_function': 'tf.compat.v1.wrap_function',
     'tf.write_file': 'tf.io.write_file',
     'tf.zeta': 'tf.math.zeta'
 }
-- 
GitLab


From 98e0558f5fc2caad195fd9a9a24906dff1ac286b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 14 Nov 2018 18:02:57 -0800
Subject: [PATCH 0271/1554] Delete stale comment

PiperOrigin-RevId: 221547963
---
 tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index 425e769346..66206909a9 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -287,11 +287,6 @@ class XlaCompiledCpuFunction {
 
   // Argument i needs to be placed in buffer_table_[arg_index_to_temp_index_[i]]
   // for XLA generated code to be able to find it.
-  //
-  // For now we need to keep around the args_ array because there is code that
-  // depends on args() returning a void**.  However, in the future we may remove
-  // args_ in favor of using buffer_table_ as the sole storage for the
-  // arguments.
   const int32* const arg_index_table_;
 
   // The number of incoming arguments.
-- 
GitLab


From 375c51ed8ecb94941ba2bd9392e1f890c7a2d057 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Nov 2018 18:11:29 -0800
Subject: [PATCH 0272/1554] Prune unreachable functions in the `GraphDef`
 objects returned from partitioning.

We currently make a full copy of the `FunctionDefLibrary` for each partition of a partitioned graph, and transfer it to remote workers in the remote setting. This change should reduce the memory consumption of `FunctionLibraryDefinition` objects throughout the runtime, by eliminating functions that will never be executed.

This change moves the implementation (and corresponding tests) of the functions `tensorflow::grappler::ReachableFunctionDefinitions()` into methods `tensorflow::FunctionLibraryDefinition::ReachableDefinitions()`. It also removes the `tensorflow::grappler::ReachableFunctions()` functions, which were unused except in tests.

PiperOrigin-RevId: 221549148
---
 tensorflow/core/BUILD                         |   4 +
 tensorflow/core/framework/function.cc         | 109 ++++++++++++++++++
 tensorflow/core/framework/function.h          |   5 +
 tensorflow/core/framework/function_test.cc    |  54 +++++++++
 tensorflow/core/graph/graph_partition.cc      |   4 +-
 tensorflow/core/graph/graph_partition_test.cc |  10 +-
 tensorflow/core/grappler/utils/BUILD          |   1 -
 tensorflow/core/grappler/utils/functions.cc   | 108 +----------------
 tensorflow/core/grappler/utils/functions.h    |   8 --
 .../core/grappler/utils/functions_test.cc     |  59 ----------
 10 files changed, 185 insertions(+), 177 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 4f998cf9b3..c69f3afd8b 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1671,6 +1671,7 @@ cc_library(
 cc_library(
     name = "mobile_additional_lib_deps",
     deps = tf_additional_lib_deps() + [
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -1775,6 +1776,7 @@ cc_library(
     deps = [
         ":protos_all_cc_impl",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@double_conversion//:double-conversion",
         "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
@@ -1799,6 +1801,7 @@ cc_library(
     deps = [
         ":protos_all_cc_impl",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@double_conversion//:double-conversion",
         "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
@@ -2640,6 +2643,7 @@ tf_cuda_library(
         ":stats_calculator_portable",
         ":version_lib",
         "@com_google_absl//absl/base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/platform/default/build_config:platformlib",
         "//tensorflow/core/kernels:bounds_check",
         "//third_party/eigen3",
diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index 9d0933e680..cb894099b0 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/function.pb_text.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -1279,6 +1280,114 @@ GET_ATTR(string)
 GET_ATTR(bool)
 #undef GET_ATTR
 
+namespace {
+
+absl::flat_hash_set<string> ReachableFunctions(
+    const FunctionLibraryDefinition& flib,
+    const protobuf::RepeatedPtrField<NodeDef>& nodes) {
+  // Functions that are reachable from the graph.
+  absl::flat_hash_set<string> reachable_funcs;
+
+  // Functions might be reachable from the nested function calls, so we keep a
+  // queue of functions that we have to check.
+  gtl::InlinedVector<const FunctionDef*, 4> func_queue;
+
+  // Add reachable and not already processed functions to the functions queue.
+  const auto add_to_func_queue = [&](const string& func_name) {
+    const FunctionDef* func = flib.Find(func_name);
+    if (func && reachable_funcs.find(func_name) == reachable_funcs.end()) {
+      func_queue.push_back(func);
+    }
+  };
+
+  // Add all the functions that are reachable from the given node to the queue.
+  const auto process_node = [&](const NodeDef& node) {
+    // Node itself can be a call to the function.
+    add_to_func_queue(node.op());
+
+    // Or node can have an attribute referencing a function.
+    for (const auto& attr : node.attr()) {
+      const auto& attr_value = attr.second;
+
+      // 1. AttrValue.func
+      if (attr_value.has_func()) {
+        add_to_func_queue(attr_value.func().name());
+      }
+
+      // 2. AttrValue.ListValue.func
+      if (attr_value.has_list()) {
+        for (const auto& func : attr_value.list().func()) {
+          add_to_func_queue(func.name());
+        }
+      }
+    }
+  };
+
+  // Add all functions that are directly called from the optimized graph.
+  std::for_each(nodes.begin(), nodes.end(), process_node);
+
+  // Process all reachable functions.
+  while (!func_queue.empty()) {
+    const FunctionDef* func = func_queue.back();
+    func_queue.pop_back();
+
+    const string& func_name = func->signature().name();
+    reachable_funcs.insert(func_name);
+
+    // Find all the functions called from the function body.
+    const auto& func_body = func->node_def();
+    std::for_each(func_body.begin(), func_body.end(), process_node);
+
+    // Check if the function has a registered gradient.
+    const string grad_func_name = flib.FindGradient(func_name);
+    if (!grad_func_name.empty()) add_to_func_queue(grad_func_name);
+  }
+
+  return reachable_funcs;
+}
+
+FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
+    const FunctionLibraryDefinition& flib,
+    const protobuf::RepeatedPtrField<NodeDef>& nodes) {
+  absl::flat_hash_set<string> reachable_funcs = ReachableFunctions(flib, nodes);
+
+  FunctionLibraryDefinition reachable_flib(flib.default_registry(),
+                                           FunctionDefLibrary());
+
+  for (const string& func_name : reachable_funcs) {
+    const FunctionDef* func = flib.Find(func_name);
+    DCHECK_NE(func, nullptr);
+    // That should never fail, because we copy functions from valid flib and use
+    // the same default registry.
+    const Status added = reachable_flib.AddFunctionDef(*func);
+    DCHECK(added.ok());
+
+    const string grad_func_name = flib.FindGradient(func_name);
+    if (!grad_func_name.empty()) {
+      GradientDef grad;
+      grad.set_function_name(func_name);
+      grad.set_gradient_func(grad_func_name);
+      // It can only fail if function already has a gradient function.
+      const Status added_grad = reachable_flib.AddGradientDef(grad);
+      DCHECK(added_grad.ok());
+    }
+  }
+
+  return reachable_flib;
+}
+
+}  // namespace
+
+FunctionLibraryDefinition FunctionLibraryDefinition::ReachableDefinitions(
+    const GraphDef& graph) const {
+  return ReachableFunctionLibraryDefinition(*this, graph.node());
+}
+
+FunctionLibraryDefinition FunctionLibraryDefinition::ReachableDefinitions(
+    const FunctionDef& func) const {
+  return ReachableFunctionLibraryDefinition(*this, func.node_def());
+}
+
 void FunctionDefHelper::AttrValueWrapper::InitFromString(StringPiece val) {
   if (val.size() >= 2 && val[0] == '$') {
     proto.set_placeholder(val.data() + 1, val.size() - 1);
diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h
index 4cc1b858e3..6792cf1653 100644
--- a/tensorflow/core/framework/function.h
+++ b/tensorflow/core/framework/function.h
@@ -411,6 +411,11 @@ class FunctionLibraryDefinition : public OpRegistryInterface {
     return default_registry_;
   }
 
+  // Returns a copy of `*this` with only the subset of functions that are
+  // reachable from the nodes of `graph` or `func`.
+  FunctionLibraryDefinition ReachableDefinitions(const GraphDef& graph) const;
+  FunctionLibraryDefinition ReachableDefinitions(const FunctionDef& func) const;
+
  private:
   // Shape inference for functions is handled separately by ShapeRefiner.
 
diff --git a/tensorflow/core/framework/function_test.cc b/tensorflow/core/framework/function_test.cc
index 10392a9f32..28c2318c76 100644
--- a/tensorflow/core/framework/function_test.cc
+++ b/tensorflow/core/framework/function_test.cc
@@ -1293,6 +1293,60 @@ TEST(FunctionLibraryDefinitionTest, GetAttr_Gradient) {
   EXPECT_EQ(annotation, false);  // WXPlusB has no custom gradient.
 }
 
+TEST(FunctionLibraryDefinitionTest, ReachableDefinitions) {
+  using ::tensorflow::test::function::GDef;
+  using ::tensorflow::test::function::NDef;
+  using FDH = ::tensorflow::FunctionDefHelper;
+
+  const auto make_simple_fdef = [](const string& name) {
+    return FDH::Create(
+        name, {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+        {{{"output"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+        /* Mapping between function returns and function node outputs. */
+        {{"z", "output:z:0"}});
+  };
+
+  FunctionDef func_1 = make_simple_fdef("Func1");
+  FunctionDef func_2 = make_simple_fdef("Func2");
+  FunctionDef func_3 = make_simple_fdef("Func3");
+
+  FunctionDef func_2_grad = make_simple_fdef("Func2_grad");
+
+  constexpr char kDevice[] = "/device:CPU:0";
+
+  GraphDef graph = GDef(
+      {
+          NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+          NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+          NDef("x", "Func1", {"a", "b"}, {{"T", DT_FLOAT}}, kDevice),
+          NDef("y", "PartitionedCall", {"a", "b"},
+               {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+                {"Tout", DataTypeSlice{DT_FLOAT}},
+                {"f", FDH::FunctionRef("Func2", {{"T", DT_FLOAT}})}},
+               kDevice),
+      },
+      // FunctionLib
+      {func_1, func_2, func_3, func_2_grad});
+
+  // Register custom function gradient after the graph was constructed.
+  GradientDef* func3_grad_def = graph.mutable_library()->add_gradient();
+  func3_grad_def->set_function_name("Func2");
+  func3_grad_def->set_gradient_func("Func2_grad");
+
+  FunctionLibraryDefinition flib(OpRegistry::Global(), graph.library());
+
+  // - 'Func1' is called directly from the graph.
+  // - 'Func2' is called indirectly via a PartitionedCall attribute, and it also
+  //   has a custom gradient ('Func2_grad') that must remain in the library.
+  // - 'Func3' is unreachable and has to be removed from the library.
+  FunctionLibraryDefinition reachable_flib = flib.ReachableDefinitions(graph);
+  EXPECT_EQ(reachable_flib.num_functions(), 3);
+  EXPECT_TRUE(reachable_flib.Contains("Func1"));
+  EXPECT_TRUE(reachable_flib.Contains("Func2"));
+  EXPECT_TRUE(reachable_flib.Contains("Func2_grad"));
+  EXPECT_FALSE(reachable_flib.Contains("Func3"));
+}
+
 // TODO(skyewm): this could be more thorough
 TEST(FunctionDefsEqualTest, TestFunctionDefsEqual) {
   // Equal functions
diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index 1dbcebab59..9c640c42a5 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/memory_types.h"
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor.pb.h"
@@ -1186,7 +1187,8 @@ Status Partition(const PartitionOptions& opts, Graph* g,
   for (auto& it : *partitions) {
     GraphDef* gdef = &it.second;
     *gdef->mutable_versions() = g->versions();
-    *gdef->mutable_library() = flib_def->ToProto();
+    // Prune unreachable functions from `flib_def` before adding them to `gdef`.
+    *gdef->mutable_library() = flib_def->ReachableDefinitions(*gdef).ToProto();
 
     // Traverse the graph to fill every send/recv op's incarnation
     // information.
diff --git a/tensorflow/core/graph/graph_partition_test.cc b/tensorflow/core/graph/graph_partition_test.cc
index f44ed47a6e..29d8034d2a 100644
--- a/tensorflow/core/graph/graph_partition_test.cc
+++ b/tensorflow/core/graph/graph_partition_test.cc
@@ -470,13 +470,19 @@ TEST_F(GraphPartitionTest, Functions) {
   ConstructOp(in_.WithOpName("A2"), "XTimesTwo", {a1});
   ConstructOp(in_.WithOpName("B2"), "XTimesFour", {b1});
 
+  // The `Partition()` helper function uses the first letter of the op name ('A'
+  // or 'B') to choose a device for each node.
   Partition(ToGraphDef(), &partitions_);
   EXPECT_EQ(2, partitions_.size());
 
-  // Test that partition graphs inherit function library from original graph
+  // Test that partition graphs inherit function library from original graph.
   string a = "/job:a/replica:0/task:0/cpu:0";
   string b = "/job:a/replica:0/task:0/cpu:1";
-  ExpectFunctions(partitions_[a].library(), {"XTimesTwo", "XTimesFour"});
+
+  // Node "A2" is placed in part `a`, and uses only "XTimesTwo".
+  ExpectFunctions(partitions_[a].library(), {"XTimesTwo"});
+  // Node "B2" is placed in part `b`, and uses both "XTimesFour" directly,
+  // and "XTimesTwo" in the body of "XTimesFour".
   ExpectFunctions(partitions_[b].library(), {"XTimesTwo", "XTimesFour"});
 }
 
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index dbe425b75f..2b9448e403 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -172,7 +172,6 @@ cc_library(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
-        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index c806f3874d..c60a450573 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -74,120 +74,16 @@ Status ResolveFunctionBodyNodeAttrPlaceholders(
   return Status::OK();
 }
 
-absl::flat_hash_set<string> ReachableFunctions(
-    const FunctionLibraryDefinition& flib,
-    const protobuf::RepeatedPtrField<NodeDef>& nodes) {
-  // Functions that are reachable from the graph.
-  absl::flat_hash_set<string> reachable_funcs;
-
-  // Functions might be reachable from the nested function calls, so we keep a
-  // queue of functions that we have to check.
-  gtl::InlinedVector<const FunctionDef*, 4> func_queue;
-
-  // Add reachable and not already processed functions to the functions queue.
-  const auto add_to_func_queue = [&](const string& func_name) {
-    const FunctionDef* func = flib.Find(func_name);
-    if (func && reachable_funcs.find(func_name) == reachable_funcs.end()) {
-      func_queue.push_back(func);
-    }
-  };
-
-  // Add all the functions that are reachable from the given node to the queue.
-  const auto process_node = [&](const NodeDef& node) {
-    // Node itself can be a call to the function.
-    add_to_func_queue(node.op());
-
-    // Or node can have an attribute referencing a function.
-    for (const auto& attr : node.attr()) {
-      const auto& attr_value = attr.second;
-
-      // 1. AttrValue.func
-      if (attr_value.has_func()) {
-        add_to_func_queue(attr_value.func().name());
-      }
-
-      // 2. AttrValue.ListValue.func
-      if (attr_value.has_list()) {
-        for (const auto& func : attr_value.list().func()) {
-          add_to_func_queue(func.name());
-        }
-      }
-    }
-  };
-
-  // Add all functions that are directly called from the optimized graph.
-  std::for_each(nodes.begin(), nodes.end(), process_node);
-
-  // Process all reachable functions.
-  while (!func_queue.empty()) {
-    const FunctionDef* func = func_queue.back();
-    func_queue.pop_back();
-
-    const string& func_name = func->signature().name();
-    reachable_funcs.insert(func_name);
-
-    // Find all the functions called from the function body.
-    const auto& func_body = func->node_def();
-    std::for_each(func_body.begin(), func_body.end(), process_node);
-
-    // Check if the function has a registered gradient.
-    const string grad_func_name = flib.FindGradient(func_name);
-    if (!grad_func_name.empty()) add_to_func_queue(grad_func_name);
-  }
-
-  return reachable_funcs;
-}
-
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib,
-    const protobuf::RepeatedPtrField<NodeDef>& nodes) {
-  absl::flat_hash_set<string> reachable_funcs = ReachableFunctions(flib, nodes);
-
-  FunctionLibraryDefinition reachable_flib(flib.default_registry(),
-                                           FunctionDefLibrary());
-
-  for (const string& func_name : reachable_funcs) {
-    const FunctionDef* func = flib.Find(func_name);
-    DCHECK_NE(func, nullptr);
-    // That should never fail, because we copy functions from valid flib and use
-    // the same default registry.
-    const Status added = reachable_flib.AddFunctionDef(*func);
-    DCHECK(added.ok());
-
-    const string grad_func_name = flib.FindGradient(func_name);
-    if (!grad_func_name.empty()) {
-      GradientDef grad;
-      grad.set_function_name(func_name);
-      grad.set_gradient_func(grad_func_name);
-      // It can only fail if function already has a gradient function.
-      const Status added_grad = reachable_flib.AddGradientDef(grad);
-      DCHECK(added_grad.ok());
-    }
-  }
-
-  return reachable_flib;
-}
-
 }  // namespace
 
-absl::flat_hash_set<string> ReachableFunctions(
-    const FunctionLibraryDefinition& flib, const GraphDef& graph) {
-  return ReachableFunctions(flib, graph.node());
-}
-
-absl::flat_hash_set<string> ReachableFunctions(
-    const FunctionLibraryDefinition& flib, const FunctionDef& func) {
-  return ReachableFunctions(flib, func.node_def());
-}
-
 FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
     const FunctionLibraryDefinition& flib, const GraphDef& graph) {
-  return ReachableFunctionLibraryDefinition(flib, graph.node());
+  return flib.ReachableDefinitions(graph);
 }
 
 FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
     const FunctionLibraryDefinition& flib, const FunctionDef& func) {
-  return ReachableFunctionLibraryDefinition(flib, func.node_def());
+  return flib.ReachableDefinitions(func);
 }
 
 void GrapplerFunctionConnectivity::RegisterInputArgExpansion(
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index 72b3c0f31a..2eec83e998 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -19,7 +19,6 @@ limitations under the License.
 #include <memory>
 #include <string>
 #include <unordered_map>
-#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -31,13 +30,6 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-// Returns a set of functions from the function library, that are reachable from
-// the nodes of the graph.
-absl::flat_hash_set<string> ReachableFunctions(
-    const FunctionLibraryDefinition& flib, const GraphDef& graph);
-absl::flat_hash_set<string> ReachableFunctions(
-    const FunctionLibraryDefinition& flib, const FunctionDef& func);
-
 // Returns a copy of FunctionLibraryDefinition with subset of functions that are
 // reachable from the nodes of the graph.
 FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 16834acecf..31b9d7e49c 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -32,65 +32,6 @@ constexpr char kDevice[] = "/device:CPU:0";
 
 class FunctionsTest : public ::testing::Test {};
 
-TEST_F(FunctionsTest, ReachableFunctions) {
-  using ::tensorflow::test::function::GDef;
-  using ::tensorflow::test::function::NDef;
-  using FDH = ::tensorflow::FunctionDefHelper;
-
-  const auto make_simple_fdef = [](const string &name) {
-    return FDH::Create(
-        name, {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-        {{{"output"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-        /* Mapping between function returns and function node outputs. */
-        {{"z", "output:z:0"}});
-  };
-
-  FunctionDef func_1 = make_simple_fdef("Func1");
-  FunctionDef func_2 = make_simple_fdef("Func2");
-  FunctionDef func_3 = make_simple_fdef("Func3");
-
-  FunctionDef func_2_grad = make_simple_fdef("Func2_grad");
-
-  GraphDef graph = GDef(
-      {
-          NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-          NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-          NDef("x", "Func1", {"a", "b"}, {{"T", DT_FLOAT}}, kDevice),
-          NDef("y", "PartitionedCall", {"a", "b"},
-               {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
-                {"Tout", DataTypeSlice{DT_FLOAT}},
-                {"f", FDH::FunctionRef("Func2", {{"T", DT_FLOAT}})}},
-               kDevice),
-      },
-      // FunctionLib
-      {func_1, func_2, func_3, func_2_grad});
-
-  // Register custom function gradient after the graph was constructed.
-  GradientDef *func3_grad_def = graph.mutable_library()->add_gradient();
-  func3_grad_def->set_function_name("Func2");
-  func3_grad_def->set_gradient_func("Func2_grad");
-
-  FunctionLibraryDefinition flib(OpRegistry::Global(), graph.library());
-
-  // - 'Func1' called directly from the graph
-  // - 'Func2' called indirectly via PartitionedCall attribute, and it also
-  //   has a custom gradient ('Func2_grad') that must remain in the library
-  // - 'Func3' in unreachable and has to be removed from the library
-
-  absl::flat_hash_set<string> reachable_funcs = ReachableFunctions(flib, graph);
-  ASSERT_EQ(reachable_funcs.size(), 3);
-  EXPECT_NE(reachable_funcs.find("Func1"), reachable_funcs.end());
-  EXPECT_NE(reachable_funcs.find("Func2"), reachable_funcs.end());
-  EXPECT_NE(reachable_funcs.find("Func2_grad"), reachable_funcs.end());
-
-  FunctionLibraryDefinition reachable_flib =
-      ReachableFunctionLibraryDefinition(flib, graph);
-  ASSERT_EQ(reachable_flib.num_functions(), 3);
-  EXPECT_TRUE(reachable_flib.Contains("Func1"));
-  EXPECT_TRUE(reachable_flib.Contains("Func2"));
-  EXPECT_TRUE(reachable_flib.Contains("Func2_grad"));
-}
-
 TEST_F(FunctionsTest, IsParametrized) {
   // Function is defined for multiple input types.
   FunctionDef parametrized_func = FunctionDefHelper::Create(
-- 
GitLab


From f1e2b518e1dd14d981de6fba893cf6611028a909 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 18:16:35 -0800
Subject: [PATCH 0273/1554] Add cholesky factorization to LinearOperator.

PiperOrigin-RevId: 221549742
---
 tensorflow/python/kernel_tests/linalg/BUILD   |  16 +++
 .../linalg/linear_operator_algebra_test.py    |  82 ++++++++++++++
 .../linalg/linear_operator_block_diag_test.py |  45 +++++++-
 .../linalg/linear_operator_circulant_test.py  |  50 +++++++--
 .../linear_operator_composition_test.py       |   6 +-
 .../linalg/linear_operator_diag_test.py       |  24 +++-
 .../linear_operator_full_matrix_test.py       |  25 ++++-
 .../linalg/linear_operator_identity_test.py   |  43 ++++++-
 .../linalg/linear_operator_kronecker_test.py  |  48 +++++++-
 .../linear_operator_low_rank_update_test.py   |  13 ++-
 .../linear_operator_lower_triangular_test.py  |   5 +
 .../linalg/linear_operator_zeros_test.py      |   7 +-
 .../ops/linalg/cholesky_registrations.py      | 101 +++++++++++++++++
 tensorflow/python/ops/linalg/linalg.py        |   2 +
 .../python/ops/linalg/linear_operator.py      |  46 ++++++--
 .../ops/linalg/linear_operator_algebra.py     | 106 ++++++++++++++++++
 .../ops/linalg/linear_operator_block_diag.py  |   4 +-
 .../ops/linalg/linear_operator_kronecker.py   |   4 +-
 .../ops/linalg/linear_operator_test_util.py   |  26 ++++-
 ...w.linalg.-linear-operator-block-diag.pbtxt |   4 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   4 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   4 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   4 +
 ....linalg.-linear-operator-composition.pbtxt |   4 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   4 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   4 +
 ...low.linalg.-linear-operator-identity.pbtxt |   4 +
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   4 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   4 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   4 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   4 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   4 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   4 +
 ...w.linalg.-linear-operator-block-diag.pbtxt |   4 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   4 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   4 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   4 +
 ....linalg.-linear-operator-composition.pbtxt |   4 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   4 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   4 +
 ...low.linalg.-linear-operator-identity.pbtxt |   4 +
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   4 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   4 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   4 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   4 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   4 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   4 +
 47 files changed, 720 insertions(+), 45 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
 create mode 100644 tensorflow/python/ops/linalg/cholesky_registrations.py
 create mode 100644 tensorflow/python/ops/linalg/linear_operator_algebra.py

diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index be2e31cb5a..599e08ffa1 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -40,6 +40,22 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "linear_operator_algebra_test",
+    size = "small",
+    srcs = ["linear_operator_algebra_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 cuda_py_test(
     name = "linear_operator_block_diag_test",
     size = "medium",
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
new file mode 100644
index 0000000000..98091f9758
--- /dev/null
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
@@ -0,0 +1,82 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for registration mechanisms."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops.linalg import cholesky_registrations  # pylint: disable=unused-import
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.platform import test
+
+# pylint: disable=protected-access
+_CHOLESKY_DECOMPS = linear_operator_algebra._CHOLESKY_DECOMPS
+_registered_cholesky = linear_operator_algebra._registered_cholesky
+# pylint: enable=protected-access
+
+
+class CholeskyTest(test.TestCase):
+
+  def testRegistration(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+
+      def _matmul(self, a):
+        pass
+
+      def _shape(self):
+        pass
+
+      def _shape_tensor(self):
+        pass
+
+    # Register Cholesky to a lambda that spits out the name parameter
+    @linear_operator_algebra.RegisterCholesky(CustomLinOp)
+    def _cholesky(a):  # pylint: disable=unused-argument,unused-variable
+      return "OK"
+
+    with self.assertRaisesRegexp(ValueError, "positive definite"):
+      CustomLinOp(dtype=None, is_self_adjoint=True).cholesky()
+
+    with self.assertRaisesRegexp(ValueError, "self adjoint"):
+      CustomLinOp(dtype=None, is_positive_definite=True).cholesky()
+
+    self.assertEqual("OK", CustomLinOp(
+        dtype=None, is_self_adjoint=True, is_positive_definite=True).cholesky())
+
+  def testRegistrationFailures(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+      pass
+
+    with self.assertRaisesRegexp(TypeError, "must be callable"):
+      linear_operator_algebra.RegisterCholesky(CustomLinOp)("blah")
+
+    # First registration is OK
+    linear_operator_algebra.RegisterCholesky(CustomLinOp)(lambda a: None)
+
+    # Second registration fails
+    with self.assertRaisesRegexp(ValueError, "has already been registered"):
+      linear_operator_algebra.RegisterCholesky(CustomLinOp)(lambda a: None)
+
+  def testExactRegistrationsAllMatch(self):
+    for (k, v) in _CHOLESKY_DECOMPS.items():
+      self.assertEqual(v, _registered_cholesky(k))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
index 30951b1b0e..f0cc5d709f 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_block_diag as block_diag
+from tensorflow.python.ops.linalg import linear_operator_lower_triangular as lower_triangular
 from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import test
@@ -78,7 +79,9 @@ class SquareLinearOperatorBlockDiagTest(
         build_info((2, 1, 5, 5), blocks=[(2, 1, 2, 2), (1, 3, 3)]),
     ]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     shape = list(build_info.shape)
     expected_blocks = (
         build_info.__dict__["blocks"] if "blocks" in build_info.__dict__
@@ -98,7 +101,11 @@ class SquareLinearOperatorBlockDiagTest(
 
     operator = block_diag.LinearOperatorBlockDiag(
         [linalg.LinearOperatorFullMatrix(
-            l, is_square=True) for l in lin_op_matrices])
+            l,
+            is_square=True,
+            is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
+            is_positive_definite=True if ensure_self_adjoint_and_pd else None)
+         for l in lin_op_matrices])
 
     # Should be auto-set.
     self.assertTrue(operator.is_square)
@@ -129,6 +136,40 @@ class SquareLinearOperatorBlockDiagTest(
     self.assertTrue(operator.is_non_singular)
     self.assertFalse(operator.is_self_adjoint)
 
+  def test_block_diag_cholesky_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = block_diag.LinearOperatorBlockDiag(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_positive_definite=True,
+                is_self_adjoint=True,
+            ),
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_positive_definite=True,
+                is_self_adjoint=True,
+            ),
+        ],
+        is_positive_definite=True,
+        is_self_adjoint=True,
+    )
+    cholesky_factor = operator.cholesky()
+    self.assertTrue(isinstance(
+        cholesky_factor,
+        block_diag.LinearOperatorBlockDiag))
+    self.assertEqual(2, len(cholesky_factor.operators))
+    self.assertTrue(
+        isinstance(
+            cholesky_factor.operators[0],
+            lower_triangular.LinearOperatorLowerTriangular)
+    )
+    self.assertTrue(
+        isinstance(
+            cholesky_factor.operators[1],
+            lower_triangular.LinearOperatorLowerTriangular)
+    )
+
   def test_is_non_singular_auto_set(self):
     # Matrix with two positive eigenvalues, 11 and 8.
     # The matrix values do not effect auto-setting of the flags.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
index f1e151ebd8..dd342f1669 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
@@ -97,7 +97,9 @@ class LinearOperatorCirculantTestSelfAdjointOperator(
     # real, the matrix will not be real.
     return [dtypes.complex64]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     shape = build_info.shape
     # For this test class, we are creating real spectrums.
     # We also want the spectrum to have eigenvalues bounded away from zero.
@@ -105,6 +107,8 @@ class LinearOperatorCirculantTestSelfAdjointOperator(
     # spectrum is bounded away from zero.
     spectrum = linear_operator_test_util.random_sign_uniform(
         shape=self._shape_to_spectrum_shape(shape), minval=1., maxval=2.)
+    if ensure_self_adjoint_and_pd:
+      spectrum = math_ops.abs(spectrum)
     # If dtype is complex, cast spectrum to complex.  The imaginary part will be
     # zero, so the operator will still be self-adjoint.
     spectrum = math_ops.cast(spectrum, dtype)
@@ -115,7 +119,10 @@ class LinearOperatorCirculantTestSelfAdjointOperator(
       lin_op_spectrum = array_ops.placeholder_with_default(spectrum, shape=None)
 
     operator = linalg.LinearOperatorCirculant(
-        lin_op_spectrum, is_self_adjoint=True, input_output_dtype=dtype)
+        lin_op_spectrum,
+        is_self_adjoint=True,
+        is_positive_definite=True if ensure_self_adjoint_and_pd else None,
+        input_output_dtype=dtype)
 
     mat = self._spectrum_to_circulant_1d(spectrum, shape, dtype=dtype)
 
@@ -146,7 +153,9 @@ class LinearOperatorCirculantTestHermitianSpectrum(
   def _dtypes_to_test(self):
     return [dtypes.float32, dtypes.complex64]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     shape = build_info.shape
     # For this test class, we are creating Hermitian spectrums.
     # We also want the spectrum to have eigenvalues bounded away from zero.
@@ -175,7 +184,11 @@ class LinearOperatorCirculantTestHermitianSpectrum(
       lin_op_spectrum = array_ops.placeholder_with_default(spectrum, shape=None)
 
     operator = linalg.LinearOperatorCirculant(
-        lin_op_spectrum, input_output_dtype=dtype)
+        lin_op_spectrum,
+        input_output_dtype=dtype,
+        is_positive_definite=True if ensure_self_adjoint_and_pd else None,
+        is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
+    )
 
     mat = self._spectrum_to_circulant_1d(spectrum, shape, dtype=dtype)
 
@@ -205,7 +218,16 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
   def _dtypes_to_test(self):
     return [dtypes.complex64]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  # Skip Cholesky since we are explicitly testing non-hermitian
+  # spectra.
+  @property
+  def _tests_to_skip(self):
+    return ["cholesky"]
+
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+    del ensure_self_adjoint_and_pd
     shape = build_info.shape
     # Will be well conditioned enough to get accurate solves.
     spectrum = linear_operator_test_util.random_sign_uniform(
@@ -421,7 +443,9 @@ class LinearOperatorCirculant2DTestHermitianSpectrum(
   def _dtypes_to_test(self):
     return [dtypes.float32, dtypes.complex64]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     shape = build_info.shape
     # For this test class, we are creating Hermitian spectrums.
     # We also want the spectrum to have eigenvalues bounded away from zero.
@@ -450,7 +474,10 @@ class LinearOperatorCirculant2DTestHermitianSpectrum(
       lin_op_spectrum = array_ops.placeholder_with_default(spectrum, shape=None)
 
     operator = linalg.LinearOperatorCirculant2D(
-        lin_op_spectrum, input_output_dtype=dtype)
+        lin_op_spectrum,
+        is_positive_definite=True if ensure_self_adjoint_and_pd else None,
+        is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
+        input_output_dtype=dtype)
 
     mat = self._spectrum_to_circulant_2d(spectrum, shape, dtype=dtype)
 
@@ -470,7 +497,14 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
   def _dtypes_to_test(self):
     return [dtypes.complex64]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  @property
+  def _tests_to_skip(self):
+    return ["cholesky"]
+
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+    del ensure_self_adjoint_and_pd
     shape = build_info.shape
     # Will be well conditioned enough to get accurate solves.
     spectrum = linear_operator_test_util.random_sign_uniform(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
index 02f56db596..3f19dc4bff 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
@@ -42,8 +42,12 @@ class SquareLinearOperatorCompositionTest(
     self._rtol[dtypes.float32] = 1e-4
     self._rtol[dtypes.complex64] = 1e-4
 
+  @property
+  def _tests_to_skip(self):
+    # Cholesky not implemented.
+    return ["cholesky"]
+
   def _operator_and_matrix(self, build_info, dtype, use_placeholder):
-    sess = ops.get_default_session()
     shape = list(build_info.shape)
 
     # Either 1 or 2 matrices, depending.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index 0758349531..d8e53fdcf5 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -32,17 +32,26 @@ class LinearOperatorDiagTest(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """Most tests done in the base class LinearOperatorDerivedClassTest."""
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     shape = list(build_info.shape)
     diag = linear_operator_test_util.random_sign_uniform(
         shape[:-1], minval=1., maxval=2., dtype=dtype)
 
+    if ensure_self_adjoint_and_pd:
+      # Abs on complex64 will result in a float32, so we cast back up.
+      diag = math_ops.cast(math_ops.abs(diag), dtype=dtype)
+
     lin_op_diag = diag
 
     if use_placeholder:
       lin_op_diag = array_ops.placeholder_with_default(diag, shape=None)
 
-    operator = linalg.LinearOperatorDiag(lin_op_diag)
+    operator = linalg.LinearOperatorDiag(
+        lin_op_diag,
+        is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
+        is_positive_definite=True if ensure_self_adjoint_and_pd else None)
 
     matrix = array_ops.matrix_diag(diag)
 
@@ -145,6 +154,17 @@ class LinearOperatorDiagTest(
       self.assertAllEqual(operator_solve.get_shape(), mat_solve.get_shape())
       self.assertAllClose(*sess.run([operator_solve, mat_solve]))
 
+  def test_diag_cholesky_type(self):
+    diag = [1., 3., 5., 8.]
+    operator = linalg.LinearOperatorDiag(
+        diag,
+        is_positive_definite=True,
+        is_self_adjoint=True,
+    )
+    self.assertTrue(isinstance(
+        operator.cholesky(),
+        linalg.LinearOperatorDiag))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
index 8c2d2cf077..36575ceec3 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
@@ -33,7 +33,9 @@ class SquareLinearOperatorFullMatrixTest(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """Most tests done in the base class LinearOperatorDerivedClassTest."""
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     shape = list(build_info.shape)
 
     matrix = linear_operator_test_util.random_positive_definite_matrix(
@@ -44,7 +46,12 @@ class SquareLinearOperatorFullMatrixTest(
     if use_placeholder:
       lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None)
 
-    operator = linalg.LinearOperatorFullMatrix(lin_op_matrix, is_square=True)
+    # Set the hints to none to test non-symmetric PD code paths.
+    operator = linalg.LinearOperatorFullMatrix(
+        lin_op_matrix,
+        is_square=True,
+        is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
+        is_positive_definite=True if ensure_self_adjoint_and_pd else None)
 
     return operator, matrix
 
@@ -123,7 +130,13 @@ class SquareLinearOperatorFullMatrixSymmetricPositiveDefiniteTest(
   def _dtypes_to_test(self):
     return [dtypes.float32, dtypes.float64]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+
+    # Matrix is always symmetric and positive definite in this class.
+    del ensure_self_adjoint_and_pd
+
     shape = list(build_info.shape)
 
     matrix = linear_operator_test_util.random_positive_definite_matrix(
@@ -134,7 +147,11 @@ class SquareLinearOperatorFullMatrixSymmetricPositiveDefiniteTest(
     if use_placeholder:
       lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None)
 
-    operator = linalg.LinearOperatorFullMatrix(lin_op_matrix, is_square=True)
+    operator = linalg.LinearOperatorFullMatrix(
+        lin_op_matrix,
+        is_square=True,
+        is_self_adjoint=True,
+        is_positive_definite=True)
 
     return operator, matrix
 
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 465a8194dd..d47a21c1c3 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -22,6 +22,7 @@ import numpy as np
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_test_util
@@ -41,7 +42,12 @@ class LinearOperatorIdentityTest(
     # 16bit.
     return [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+    # Identity matrix is already Hermitian Positive Definite.
+    del ensure_self_adjoint_and_pd
+
     shape = list(build_info.shape)
     assert shape[-1] == shape[-2]
 
@@ -242,6 +248,16 @@ class LinearOperatorIdentityTest(
           is_non_singular=None,
       )
 
+  def test_identity_cholesky_type(self):
+    operator = linalg_lib.LinearOperatorIdentity(
+        num_rows=2,
+        is_positive_definite=True,
+        is_self_adjoint=True,
+    )
+    self.assertTrue(isinstance(
+        operator.cholesky(),
+        linalg_lib.LinearOperatorIdentity))
+
 
 class LinearOperatorScaledIdentityTest(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
@@ -253,7 +269,10 @@ class LinearOperatorScaledIdentityTest(
     # 16bit.
     return [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+
     shape = list(build_info.shape)
     assert shape[-1] == shape[-2]
 
@@ -266,6 +285,9 @@ class LinearOperatorScaledIdentityTest(
     multiplier = linear_operator_test_util.random_sign_uniform(
         shape=batch_shape, minval=1., maxval=2., dtype=dtype)
 
+    if ensure_self_adjoint_and_pd:
+      # Abs on complex64 will result in a float32, so we cast back up.
+      multiplier = math_ops.cast(math_ops.abs(multiplier), dtype=dtype)
 
     # Nothing to feed since LinearOperatorScaledIdentity takes no Tensor args.
     lin_op_multiplier = multiplier
@@ -275,7 +297,10 @@ class LinearOperatorScaledIdentityTest(
           multiplier, shape=None)
 
     operator = linalg_lib.LinearOperatorScaledIdentity(
-        num_rows, lin_op_multiplier)
+        num_rows,
+        lin_op_multiplier,
+        is_self_adjoint=True if ensure_self_adjoint_and_pd else None,
+        is_positive_definite=True if ensure_self_adjoint_and_pd else None)
 
     multiplier_matrix = array_ops.expand_dims(
         array_ops.expand_dims(multiplier, -1), -1)
@@ -420,6 +445,18 @@ class LinearOperatorScaledIdentityTest(
     self.assertTrue(operator.is_non_singular)
     self.assertTrue(operator.is_self_adjoint is None)
 
+  def test_scaled_identity_cholesky_type(self):
+    operator = linalg_lib.LinearOperatorScaledIdentity(
+        num_rows=2,
+        multiplier=3.,
+        is_positive_definite=True,
+        is_self_adjoint=True,
+    )
+    self.assertTrue(isinstance(
+        operator.cholesky(),
+        linalg_lib.LinearOperatorScaledIdentity))
+
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
index f039b60f64..df6954f520 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_kronecker as kronecker
+from tensorflow.python.ops.linalg import linear_operator_lower_triangular as lower_triangular
 from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import test
@@ -99,7 +100,12 @@ class SquareLinearOperatorKroneckerTest(
   def _tests_to_skip(self):
     return ["det", "solve", "solve_with_broadcast"]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+    # Kronecker products constructed below will be from symmetric
+    # positive-definite matrices.
+    del ensure_self_adjoint_and_pd
     shape = list(build_info.shape)
     expected_factors = build_info.__dict__["factors"]
     matrices = [
@@ -116,7 +122,11 @@ class SquareLinearOperatorKroneckerTest(
 
     operator = kronecker.LinearOperatorKronecker(
         [linalg.LinearOperatorFullMatrix(
-            l, is_square=True) for l in lin_op_matrices])
+            l,
+            is_square=True,
+            is_self_adjoint=True,
+            is_positive_definite=True)
+         for l in lin_op_matrices])
 
     matrices = linear_operator_util.broadcast_matrix_batch_dims(matrices)
 
@@ -180,6 +190,40 @@ class SquareLinearOperatorKroneckerTest(
     with self.assertRaisesRegexp(ValueError, ">=1 operators"):
       kronecker.LinearOperatorKronecker([])
 
+  def test_kronecker_cholesky_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = kronecker.LinearOperatorKronecker(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_positive_definite=True,
+                is_self_adjoint=True,
+            ),
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_positive_definite=True,
+                is_self_adjoint=True,
+            ),
+        ],
+        is_positive_definite=True,
+        is_self_adjoint=True,
+    )
+    cholesky_factor = operator.cholesky()
+    self.assertTrue(isinstance(
+        cholesky_factor,
+        kronecker.LinearOperatorKronecker))
+    self.assertEqual(2, len(cholesky_factor.operators))
+    self.assertTrue(
+        isinstance(
+            cholesky_factor.operators[0],
+            lower_triangular.LinearOperatorLowerTriangular)
+    )
+    self.assertTrue(
+        isinstance(
+            cholesky_factor.operators[1],
+            lower_triangular.LinearOperatorLowerTriangular)
+    )
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py
index 207e5edf81..2920f3ae7e 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py
@@ -69,7 +69,8 @@ class BaseLinearOperatorLowRankUpdatetest(object):
     return linear_operator_test_util.random_uniform(
         diag_shape, minval=1e-4, maxval=1., dtype=dtype)
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(self, build_info, dtype, use_placeholder,
+                           ensure_self_adjoint_and_pd=False):
     # Recall A = L + UDV^H
     shape = list(build_info.shape)
     diag_shape = shape[:-1]
@@ -93,7 +94,7 @@ class BaseLinearOperatorLowRankUpdatetest(object):
     lin_op_v = v
 
     # D
-    if self._is_diag_update_positive:
+    if self._is_diag_update_positive or ensure_self_adjoint_and_pd:
       diag_update = self._gen_positive_diag(dtype, diag_update_shape)
     else:
       diag_update = linear_operator_test_util.random_normal(
@@ -178,6 +179,10 @@ class LinearOperatorLowRankUpdatetestWithDiagCannotUseCholesky(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """A = L + UDU^H, D !> 0, L > 0 ==> A !> 0 and we cannot use a Cholesky."""
 
+  @property
+  def _tests_to_skip(self):
+    return ["cholesky"]
+
   _use_diag_update = True
   _is_diag_update_positive = False
   _use_v = False
@@ -217,6 +222,10 @@ class LinearOperatorLowRankUpdatetestNoDiagCannotUseCholesky(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """A = L + UV^H, L > 0 ==> A is not symmetric and we cannot use a Cholesky."""
 
+  @property
+  def _tests_to_skip(self):
+    return ["cholesky"]
+
   _use_diag_update = False
   _is_diag_update_positive = None
   _use_v = True
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
index e3c8f5cb68..70a3e36c41 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
@@ -29,6 +29,11 @@ class LinearOperatorLowerTriangularTest(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """Most tests done in the base class LinearOperatorDerivedClassTest."""
 
+  @property
+  def _tests_to_skip(self):
+    # Cholesky does not make sense for triangular matrices.
+    return ["cholesky"]
+
   def _operator_and_matrix(self, build_info, dtype, use_placeholder):
     shape = list(build_info.shape)
     # Upper triangle will be nonzero, but ignored.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
index ad97d1a93e..d1bcb89994 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
@@ -35,7 +35,7 @@ class LinearOperatorZerosTest(
 
   @property
   def _tests_to_skip(self):
-    return ["log_abs_det", "solve", "solve_with_broadcast"]
+    return ["cholesky", "log_abs_det", "solve", "solve_with_broadcast"]
 
   @property
   def _operator_build_infos(self):
@@ -46,7 +46,10 @@ class LinearOperatorZerosTest(
         build_info((3, 4, 4)),
         build_info((2, 1, 4, 4))]
 
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
+    del ensure_self_adjoint_and_pd
     del use_placeholder
     shape = list(build_info.shape)
     assert shape[-1] == shape[-2]
diff --git a/tensorflow/python/ops/linalg/cholesky_registrations.py b/tensorflow/python/ops/linalg/cholesky_registrations.py
new file mode 100644
index 0000000000..e5284cf22a
--- /dev/null
+++ b/tensorflow/python/ops/linalg/cholesky_registrations.py
@@ -0,0 +1,101 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Registrations for LinearOperator.cholesky."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.ops.linalg import linear_operator_block_diag
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_identity
+from tensorflow.python.ops.linalg import linear_operator_kronecker
+from tensorflow.python.ops.linalg import linear_operator_lower_triangular
+
+
+# By default, compute the Cholesky of the dense matrix, and return a
+# LowerTriangular operator. Methods below specialize this registration.
+@linear_operator_algebra.RegisterCholesky(linear_operator.LinearOperator)
+def _cholesky_linear_operator(linop):
+  return linear_operator_lower_triangular.LinearOperatorLowerTriangular(
+      linalg_ops.cholesky(linop.to_dense()),
+      is_non_singular=True,
+      is_self_adjoint=False,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterCholesky(
+    linear_operator_diag.LinearOperatorDiag)
+def _cholesky_diag(diag_operator):
+  return linear_operator_diag.LinearOperatorDiag(
+      math_ops.sqrt(diag_operator.diag),
+      is_non_singular=True,
+      is_self_adjoint=True,
+      is_positive_definite=True,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterCholesky(
+    linear_operator_identity.LinearOperatorIdentity)
+def _cholesky_identity(identity_operator):
+  return linear_operator_identity.LinearOperatorIdentity(
+      num_rows=identity_operator._num_rows,  # pylint: disable=protected-access
+      batch_shape=identity_operator.batch_shape,
+      dtype=identity_operator.dtype,
+      is_non_singular=True,
+      is_self_adjoint=True,
+      is_positive_definite=True,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterCholesky(
+    linear_operator_identity.LinearOperatorScaledIdentity)
+def _cholesky_scaled_identity(identity_operator):
+  return linear_operator_identity.LinearOperatorScaledIdentity(
+      num_rows=identity_operator._num_rows,  # pylint: disable=protected-access
+      multiplier=math_ops.sqrt(identity_operator.multiplier),
+      is_non_singular=True,
+      is_self_adjoint=True,
+      is_positive_definite=True,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterCholesky(
+    linear_operator_block_diag.LinearOperatorBlockDiag)
+def _cholesky_block_diag(block_diag_operator):
+    # We take the cholesky of each block on the diagonal.
+  return linear_operator_block_diag.LinearOperatorBlockDiag(
+      operators=[
+          operator.cholesky() for operator in block_diag_operator.operators],
+      is_non_singular=True,
+      is_self_adjoint=False,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterCholesky(
+    linear_operator_kronecker.LinearOperatorKronecker)
+def _cholesky_kronecker(kronecker_operator):
+    # Cholesky decomposition of a Kronecker product is the Kronecker product
+    # of cholesky decompositions.
+  return linear_operator_kronecker.LinearOperatorKronecker(
+      operators=[
+          operator.cholesky() for operator in kronecker_operator.operators],
+      is_non_singular=True,
+      is_self_adjoint=False,
+      is_square=True)
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index c29b5033bb..e29c45c5f5 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import,unused-import
+from tensorflow.python.ops.linalg import cholesky_registrations as _cholesky_registrations
+from tensorflow.python.ops.linalg import linear_operator_algebra as _linear_operator_algebra
 from tensorflow.python.ops.linalg.linalg_impl import *
 from tensorflow.python.ops.linalg.linear_operator import *
 from tensorflow.python.ops.linalg.linear_operator_block_diag import *
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 9ef6c42b04..b8683a24fc 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -32,6 +32,7 @@ from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.linalg import linalg_impl as linalg
+from tensorflow.python.ops.linalg import linear_operator_algebra
 from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
@@ -284,7 +285,7 @@ class LinearOperator(object):
     `[B1,...,Bb, M, N]`, equivalent to `tf.shape(A)`.
 
     Args:
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       `int32` `Tensor`
@@ -318,7 +319,7 @@ class LinearOperator(object):
     `[B1,...,Bb]`.
 
     Args:
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       `int32` `Tensor`
@@ -340,7 +341,7 @@ class LinearOperator(object):
     `A.shape = [B1,...,Bb, M, N]`, then this returns `b + 2`.
 
     Args:
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       Python integer, or None if the tensor rank is undefined.
@@ -356,7 +357,7 @@ class LinearOperator(object):
     `A.shape = [B1,...,Bb, M, N]`, then this returns `b + 2`.
 
     Args:
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       `int32` `Tensor`, determined at runtime.
@@ -586,7 +587,7 @@ class LinearOperator(object):
       adjoint: Python `bool`.  If `True`, left multiply by the adjoint: `A^H x`.
       adjoint_arg:  Python `bool`.  If `True`, compute `A x^H` where `x^H` is
         the hermitian transpose (transposition and complex conjugation).
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       A `Tensor` with shape `[..., M, R]` and same `dtype` as `self`.
@@ -630,7 +631,7 @@ class LinearOperator(object):
         dimensions, the last dimension defines a vector.
         See class docstring for definition of compatibility.
       adjoint: Python `bool`.  If `True`, left multiply by the adjoint: `A^H x`.
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       A `Tensor` with shape `[..., M]` and same `dtype` as `self`.
@@ -655,7 +656,7 @@ class LinearOperator(object):
     """Determinant for every batch member.
 
     Args:
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       `Tensor` with shape `self.batch_shape` and same `dtype` as `self`.
@@ -684,7 +685,7 @@ class LinearOperator(object):
     """Log absolute value of determinant for every batch member.
 
     Args:
-      name:  A name for this `Op.
+      name:  A name for this `Op`.
 
     Returns:
       `Tensor` with shape `self.batch_shape` and same `dtype` as `self`.
@@ -830,6 +831,31 @@ class LinearOperator(object):
 
       return self._solvevec(rhs, adjoint=adjoint)
 
+  def cholesky(self, name="cholesky"):
+    """Returns a Cholesky factor as a `LinearOperator`.
+
+    Given `A` representing this `LinearOperator`, if `A` is positive definite
+    self-adjoint, return `L`, where `A = L L^T`, i.e. the cholesky
+    decomposition.
+
+    Args:
+      name:  A name for this `Op`.
+
+    Returns:
+      `LinearOperator` which represents the lower triangular matrix
+      in the Cholesky decomposition.
+
+    Raises:
+      ValueError: When the `LinearOperator` is not hinted to be positive
+        definite and self adjoint.
+    """
+
+    if not self._can_use_cholesky():
+      raise ValueError("Cannot take the Cholesky decomposition: "
+                       "Not a positive definite self adjoint matrix.")
+    with self._name_scope(name):
+      return linear_operator_algebra.cholesky(self)
+
   def _to_dense(self):
     """Generic and often inefficient implementation.  Override often."""
     logging.warn("Using (possibly slow) default implementation of to_dense."
@@ -922,6 +948,4 @@ class LinearOperator(object):
       return self._add_to_tensor(x)
 
   def _can_use_cholesky(self):
-    # TODO(langmore) Add complex types when tf.cholesky can use them.
-    return (not self.dtype.is_complex and self.is_self_adjoint and
-            self.is_positive_definite)
+    return self.is_self_adjoint and self.is_positive_definite
diff --git a/tensorflow/python/ops/linalg/linear_operator_algebra.py b/tensorflow/python/ops/linalg/linear_operator_algebra.py
new file mode 100644
index 0000000000..ba9b732797
--- /dev/null
+++ b/tensorflow/python/ops/linalg/linear_operator_algebra.py
@@ -0,0 +1,106 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Registration mechanisms for various n-ary operations on LinearOperators."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.util import tf_inspect
+
+
+_CHOLESKY_DECOMPS = {}
+
+
+def _registered_cholesky(type_a):
+  """Get the Cholesky function registered for class a."""
+  hierarchy_a = tf_inspect.getmro(type_a)
+  distance_to_children = None
+  cholesky_fn = None
+  for mro_to_a, parent_a in enumerate(hierarchy_a):
+    candidate_dist = mro_to_a
+    candidate_cholesky_fn = _CHOLESKY_DECOMPS.get(parent_a, None)
+    if not cholesky_fn or (
+        candidate_cholesky_fn and candidate_dist < distance_to_children):
+      distance_to_children = candidate_dist
+      cholesky_fn = candidate_cholesky_fn
+  return cholesky_fn
+
+
+def cholesky(lin_op_a, name=None):
+  """Get the Cholesky factor associated to lin_op_a.
+
+  Args:
+    lin_op_a: The LinearOperator to decompose.
+    name: Name to use for this operation.
+
+  Returns:
+    A LinearOperator that represents the lower Cholesky factor of `lin_op_a`.
+
+  Raises:
+    NotImplementedError: If no Cholesky method is defined for the LinearOperator
+      type of `lin_op_a`.
+  """
+  cholesky_fn = _registered_cholesky(type(lin_op_a))
+  if cholesky_fn is None:
+    raise ValueError("No cholesky decomposition registered for {}".format(
+        type(lin_op_a)))
+
+  with ops.name_scope(name, "Cholesky"):
+    return cholesky_fn(lin_op_a)
+
+
+class RegisterCholesky(object):
+  """Decorator to register a Cholesky implementation function.
+
+  Usage:
+
+  @linear_operator_algebra.RegisterCholesky(lin_op.LinearOperatorIdentity)
+  def _cholesky_identity(lin_op_a):
+    # Return the identity matrix.
+  """
+
+  def __init__(self, lin_op_cls_a):
+    """Initialize the LinearOperator registrar.
+
+    Args:
+      lin_op_cls_a: the class of the LinearOperator to decompose.
+    """
+    self._key = lin_op_cls_a
+
+  def __call__(self, cholesky_fn):
+    """Perform the Cholesky registration.
+
+    Args:
+      cholesky_fn: The function to use for the Cholesky.
+
+    Returns:
+      cholesky_fn
+
+    Raises:
+      TypeError: if cholesky_fn is not a callable.
+      ValueError: if a Cholesky function has already been registered for
+        the given argument classes.
+    """
+    if not callable(cholesky_fn):
+      raise TypeError(
+          "cholesky_fn must be callable, received: {}".format(cholesky_fn))
+    if self._key in _CHOLESKY_DECOMPS:
+      raise ValueError("Cholesky({}) has already been registered to: {}".format(
+          self._key.__name__, _CHOLESKY_DECOMPS[self._key]))
+    _CHOLESKY_DECOMPS[self._key] = cholesky_fn
+    return cholesky_fn
diff --git a/tensorflow/python/ops/linalg/linear_operator_block_diag.py b/tensorflow/python/ops/linalg/linear_operator_block_diag.py
index 438c3496bd..b0b418c997 100644
--- a/tensorflow/python/ops/linalg/linear_operator_block_diag.py
+++ b/tensorflow/python/ops/linalg/linear_operator_block_diag.py
@@ -29,9 +29,7 @@ from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.util.tf_export import tf_export
 
-__all__ = [
-    "LinearOperatorBlockDiag",
-]
+__all__ = ["LinearOperatorBlockDiag"]
 
 
 @tf_export("linalg.LinearOperatorBlockDiag")
diff --git a/tensorflow/python/ops/linalg/linear_operator_kronecker.py b/tensorflow/python/ops/linalg/linear_operator_kronecker.py
index 1fd5073c17..f7e785caa5 100644
--- a/tensorflow/python/ops/linalg/linear_operator_kronecker.py
+++ b/tensorflow/python/ops/linalg/linear_operator_kronecker.py
@@ -30,9 +30,7 @@ from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.util.tf_export import tf_export
 
-__all__ = [
-    "LinearOperatorKronecker",
-]
+__all__ = ["LinearOperatorKronecker"]
 
 
 def _vec(x):
diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
index 76d659f109..e50f572b5f 100644
--- a/tensorflow/python/ops/linalg/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -102,7 +102,9 @@ class LinearOperatorDerivedClassTest(test.TestCase):
     raise NotImplementedError("operator_build_infos has not been implemented.")
 
   @abc.abstractmethod
-  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+  def _operator_and_matrix(
+      self, build_info, dtype, use_placeholder,
+      ensure_self_adjoint_and_pd=False):
     """Build a batch matrix and an Operator that should have similar behavior.
 
     Every operator acts like a (batch) matrix.  This method returns both
@@ -114,6 +116,11 @@ class LinearOperatorDerivedClassTest(test.TestCase):
       dtype:  Numpy dtype.  Data type of returned array/operator.
       use_placeholder:  Python bool.  If True, initialize the operator with a
         placeholder of undefined shape and correct dtype.
+      ensure_self_adjoint_and_pd: If `True`,
+        construct this operator to be Hermitian Positive Definite, as well
+        as ensuring the hints `is_positive_definite` and `is_self_adjoint`
+        are set.
+        This is useful for testing methods such as `cholesky`.
 
     Returns:
       operator:  `LinearOperator` subclass instance.
@@ -271,6 +278,21 @@ class LinearOperatorDerivedClassTest(test.TestCase):
     self._skip_if_tests_to_skip_contains("matmul_with_broadcast")
     self._test_matmul(with_batch=False)
 
+  def test_cholesky(self):
+    self._skip_if_tests_to_skip_contains("cholesky")
+    for use_placeholder in self._use_placeholder_options:
+      for build_info in self._operator_build_infos:
+        for dtype in self._dtypes_to_test:
+          with self.test_session(graph=ops.Graph()) as sess:
+            sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED
+            operator, mat = self._operator_and_matrix(
+                build_info, dtype, use_placeholder=use_placeholder,
+                ensure_self_adjoint_and_pd=True)
+            op_chol = operator.cholesky().to_dense()
+            mat_chol = linalg_ops.cholesky(mat)
+            op_chol_v, mat_chol_v = sess.run([op_chol, mat_chol])
+            self.assertAC(mat_chol_v, op_chol_v)
+
   def _test_solve(self, with_batch):
     for use_placeholder in self._use_placeholder_options:
       for build_info in self._operator_build_infos:
@@ -441,7 +463,7 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest):
   @property
   def _tests_to_skip(self):
     """List of test names to skip."""
-    return ["solve", "solve_with_broadcast", "det", "log_abs_det"]
+    return ["cholesky", "solve", "solve_with_broadcast", "det", "log_abs_det"]
 
   @property
   def _operator_build_infos(self):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index 973705dae2..773c74e64d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
index de917706d5..533544d21f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "block_shape_tensor"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "convolution_kernel"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index c4e6a21c3a..e3926eb6d4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "block_shape_tensor"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "convolution_kernel"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index 2e085a8e28..ba209df782 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "block_shape_tensor"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "convolution_kernel"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
index 42d22bce42..081fb0e08b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
index d6749fdcec..2014a04301 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index d9f363d133..9a87ae9687 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
index aac7ee31ed..33afb835ce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -76,6 +76,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index c11d390829..a9078c8ab5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 3ee800269e..4cfa3bb30d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index 63a1bc2321..a87649133f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index e2c5a505a7..3265646784 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -80,6 +80,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
index a1b0e06b47..49d8890c89 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
index 6d849dc040..c89dc067b3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index 973705dae2..773c74e64d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
index de917706d5..533544d21f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "block_shape_tensor"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "convolution_kernel"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index c4e6a21c3a..e3926eb6d4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "block_shape_tensor"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "convolution_kernel"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index 2e085a8e28..ba209df782 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "block_shape_tensor"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "convolution_kernel"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'convolution_kernel\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
index 42d22bce42..081fb0e08b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
index d6749fdcec..2014a04301 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index d9f363d133..9a87ae9687 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
index aac7ee31ed..33afb835ce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -76,6 +76,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index c11d390829..a9078c8ab5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -79,6 +79,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 3ee800269e..4cfa3bb30d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index 63a1bc2321..a87649133f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index e2c5a505a7..3265646784 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -80,6 +80,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
index a1b0e06b47..49d8890c89 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
index 6d849dc040..c89dc067b3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "batch_shape_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
   }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
   member_method {
     name: "determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
-- 
GitLab


From 2c3c2f47e276bb826c0708e592aeee4e3583cd07 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 18:18:09 -0800
Subject: [PATCH 0274/1554] Modify ShouldUseRunHandlerPool() to also be used
 when session_inter_op_thread_pool() option is enabled in SessionConfig. Use
 it while running session-run(s) on the default inter_op_thread_pool() = 0.

PiperOrigin-RevId: 221549960
---
 .../core/common_runtime/direct_session.cc     | 19 +++++++++++++------
 .../core/common_runtime/direct_session.h      |  6 ++++--
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 52c1cd2691..40b7071f40 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -253,11 +253,19 @@ static RunHandlerPool* GetOrCreateRunHandlerPool(
   return pool;
 }
 
-bool DirectSession::ShouldUseRunHandlerPool() const {
-  if (options_.config.session_inter_op_thread_pool_size() > 0 ||
-      options_.config.use_per_session_threads()) {
+bool DirectSession::ShouldUseRunHandlerPool(
+    const RunOptions& run_options) const {
+  if (options_.config.use_per_session_threads()) return false;
+  if (options_.config.session_inter_op_thread_pool_size() > 0 &&
+      run_options.inter_op_thread_pool() > 0)
     return false;
-  }
+  // Only use RunHandlerPool when:
+  // a. Single global thread pool is used for inter-op parallelism.
+  // b. When multiple inter_op_thread_pool(s) are created, use it only while
+  // running sessions on the default inter_op_thread_pool=0. Typically,
+  // servo-team uses inter_op_thread_pool > 0 for model loading.
+  // TODO(crk): Revisit whether we'd want to create one (static) RunHandlerPool
+  // per entry in session_inter_op_thread_pool() in the future.
   return true;
 }
 
@@ -603,9 +611,8 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
   }
 
   std::unique_ptr<RunHandler> handler;
-  if (ShouldUseRunHandlerPool() &&
+  if (ShouldUseRunHandlerPool(run_options) &&
       run_options.experimental().use_run_handler_pool()) {
-    // Non-null only when a global inter-op pool is used.
     VLOG(1) << "Using RunHandler to scheduler inter-op closures.";
     handler = GetOrCreateRunHandlerPool(options_)->Get();
   }
diff --git a/tensorflow/core/common_runtime/direct_session.h b/tensorflow/core/common_runtime/direct_session.h
index 3a168bbe3f..6754e9cfb7 100644
--- a/tensorflow/core/common_runtime/direct_session.h
+++ b/tensorflow/core/common_runtime/direct_session.h
@@ -247,8 +247,10 @@ class DirectSession : public Session {
                                    ExecutorsAndKeys* executors_and_keys,
                                    RunMetadata* run_metadata);
 
-  // Returns whether inter-op execution uses a global pool.
-  bool ShouldUseRunHandlerPool() const;
+  // Returns whether inter-op execution uses a global pool or the input
+  // `run_options` requests being run on inter_op_thread_pool = 0 in case
+  // multiple pools are configured.
+  bool ShouldUseRunHandlerPool(const RunOptions& run_options) const;
 
   ::tensorflow::Status ExtendLocked(const GraphDef& graph)
       EXCLUSIVE_LOCKS_REQUIRED(graph_state_lock_);
-- 
GitLab


From 010199ab1a8ae0412fa34644ea859ae8a0b9a2a1 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 14 Nov 2018 18:24:17 -0800
Subject: [PATCH 0275/1554] [TF:XLA] Consolidate TF:XLA flags into a single
 file.

Preparation for rejecting unknown TF_XLA_FLAGS.

PiperOrigin-RevId: 221550561
---
 tensorflow/c/BUILD                            |   2 +-
 tensorflow/c/c_api_experimental.cc            |   2 +-
 tensorflow/compiler/jit/BUILD                 |  17 +-
 tensorflow/compiler/jit/build_xla_ops_pass.cc |   2 +-
 tensorflow/compiler/jit/flags.cc              | 154 ++++++++++++++++++
 ...k_for_compilation_pass_flags.h => flags.h} |  63 +++++--
 .../increase_dynamism_for_auto_jit_pass.cc    |   2 +-
 tensorflow/compiler/jit/kernels/BUILD         |   2 +-
 tensorflow/compiler/jit/kernels/xla_ops.cc    |   2 +-
 tensorflow/compiler/jit/legacy_flags/BUILD    |  65 --------
 .../legacy_flags/build_xla_ops_pass_flags.cc  |  47 ------
 .../legacy_flags/build_xla_ops_pass_flags.h   |  37 -----
 .../mark_for_compilation_pass_flags.cc        |  98 -----------
 .../jit/legacy_flags/xla_device_flags.cc      |  56 -------
 .../jit/legacy_flags/xla_device_flags.h       |  47 ------
 .../jit/legacy_flags/xla_ops_common_flags.cc  |  52 ------
 .../jit/legacy_flags/xla_ops_common_flags.h   |  36 ----
 .../compiler/jit/mark_for_compilation_pass.cc |   2 +-
 tensorflow/compiler/jit/xla_cpu_device.cc     |   2 +-
 tensorflow/compiler/tf2xla/BUILD              |   5 +-
 tensorflow/compiler/tf2xla/dump_graph.cc      |   2 +-
 .../compiler/tf2xla/dump_graph_flags.cc       |  63 -------
 tensorflow/compiler/tf2xla/dump_graph_flags.h |  48 ------
 tensorflow/compiler/tf2xla/xla_op_registry.cc |   2 +-
 24 files changed, 230 insertions(+), 578 deletions(-)
 create mode 100644 tensorflow/compiler/jit/flags.cc
 rename tensorflow/compiler/jit/{legacy_flags/mark_for_compilation_pass_flags.h => flags.h} (57%)
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/BUILD
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_device_flags.h
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc
 delete mode 100644 tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h
 delete mode 100644 tensorflow/compiler/tf2xla/dump_graph_flags.cc
 delete mode 100644 tensorflow/compiler/tf2xla/dump_graph_flags.h

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index f9fc718f68..84238ffc1f 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -121,7 +121,7 @@ tf_cuda_library(
         ":c_api",
         ":c_api_internal",
         "//tensorflow/c/eager:c_api",
-        "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags",
+        "//tensorflow/compiler/jit:flags",
         "//tensorflow/contrib/tpu:all_ops",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index fabe2fa0f6..bc434e7593 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #include "tensorflow/c/c_api_experimental.h"
 
 #include "tensorflow/c/c_api_internal.h"
-#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/graph/graph.h"
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index f5ffe6ff6a..682c0f0cb0 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -76,10 +76,10 @@ cc_library(
     srcs = ["xla_cpu_device.cc"],
     visibility = [":friends"],
     deps = [
+        ":flags",
         ":jit_compilation_passes",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_ops",
-        "//tensorflow/compiler/jit/legacy_flags:xla_device_flags",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/xla/service:cpu_plugin",  # buildcleaner: keep
@@ -210,6 +210,18 @@ cc_library(
 
 # Internal targets below this point.
 
+cc_library(
+    name = "flags",
+    srcs = ["flags.cc"],
+    hdrs = ["flags.h"],
+    visibility = [":friends"],
+    deps = [
+        "//tensorflow/compiler/xla:parse_flags_from_env",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+    ],
+)
+
 cc_library(
     name = "common",
     srcs = [
@@ -488,6 +500,7 @@ cc_library(
     deps = [
         ":common",
         ":encapsulate_util",
+        ":flags",
         ":shape_inference_helpers",
         ":union_find",
         ":xla_cluster_util",
@@ -495,8 +508,6 @@ cc_library(
         "//tensorflow/cc:ops",
         "//tensorflow/cc:scope_internal",
         "//tensorflow/compiler/jit/graphcycles",
-        "//tensorflow/compiler/jit/legacy_flags:build_xla_ops_pass_flags",
-        "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags",
         "//tensorflow/compiler/jit/ops:xla_ops",
         "//tensorflow/compiler/tf2xla:dump_graph",
         "//tensorflow/compiler/tf2xla:resource_operation_table",
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass.cc b/tensorflow/compiler/jit/build_xla_ops_pass.cc
index 93637a69d5..01bea34af9 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass.cc
@@ -23,7 +23,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/control_flow_ops.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/encapsulate_subgraphs_pass.h"
-#include "tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/compiler/tf2xla/cc/ops/xla_jit_ops.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc
new file mode 100644
index 0000000000..7b055893cb
--- /dev/null
+++ b/tensorflow/compiler/jit/flags.cc
@@ -0,0 +1,154 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <mutex>  // NOLINT
+
+#include "tensorflow/compiler/jit/flags.h"
+#include "tensorflow/compiler/xla/parse_flags_from_env.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+namespace tensorflow {
+namespace legacy_flags {
+namespace {
+
+BuildXlaOpsPassFlags* build_ops_flags;
+DumpGraphFlags* dump_graph_flags;
+MarkForCompilationPassFlags* mark_for_compilation_flags;
+XlaDeviceFlags* device_flags;
+XlaOpsCommonFlags* ops_flags;
+
+std::vector<Flag>* flag_list;
+std::once_flag flags_init;
+
+void AppendDumpGraphFlagsInternal(std::vector<Flag>* flag_list) {
+  std::vector<Flag> new_flags = {
+      Flag("tf_dump_graph_prefix", &dump_graph_flags->tf_dump_graph_prefix,
+           "Path prefix to which graphs dumped during debugging should be "
+           "written."),
+  };
+  flag_list->insert(flag_list->end(), new_flags.begin(), new_flags.end());
+}
+
+void AppendMarkForCompilationPassFlagsInternal(std::vector<Flag>* flag_list) {
+  std::vector<Flag> new_flags = {
+      Flag("tf_xla_auto_jit", &mark_for_compilation_flags->tf_xla_auto_jit,
+           "Control compilation of operators into XLA computations on CPU and "
+           "GPU devices.  0 = use ConfigProto setting; -1 = off; 1 = on for "
+           "things very likely to be improved; 2 = on for everything.  "
+           "Experimental."),
+      Flag("tf_xla_min_cluster_size",
+           &mark_for_compilation_flags->tf_xla_min_cluster_size,
+           "Minimum number of operators in an XLA compilation. Ignored for "
+           "operators placed on an XLA device or operators explicitly marked "
+           "for compilation."),
+      Flag("tf_xla_max_cluster_size",
+           &mark_for_compilation_flags->tf_xla_max_cluster_size,
+           "Maximum number of operators in an XLA compilation."),
+      Flag("tf_xla_clustering_debug",
+           &mark_for_compilation_flags->tf_xla_clustering_debug,
+           "Dump graphs during XLA compilation."),
+      Flag("tf_xla_cpu_global_jit",
+           &mark_for_compilation_flags->tf_xla_cpu_global_jit,
+           "Enables global JIT compilation for CPU via SessionOptions."),
+      Flag("tf_xla_clustering_fuel",
+           &mark_for_compilation_flags->tf_xla_clustering_fuel,
+           "Places an artificial limit on the number of ops marked as "
+           "eligible for clustering."),
+      Flag("tf_xla_fusion_only",
+           &mark_for_compilation_flags->tf_xla_fusion_only,
+           "enable fusion of element-wise operations only using XLA when "
+           "global_jit_level is ON*.")};
+  flag_list->insert(flag_list->end(), new_flags.begin(), new_flags.end());
+}
+
+void AllocateAndParseFlags() {
+  build_ops_flags = new BuildXlaOpsPassFlags;
+  build_ops_flags->tf_xla_enable_lazy_compilation = true;
+
+  dump_graph_flags = new DumpGraphFlags;
+  dump_graph_flags->tf_dump_graph_prefix = "/tmp/";
+
+  mark_for_compilation_flags = new MarkForCompilationPassFlags;
+  mark_for_compilation_flags->tf_xla_auto_jit = 0;
+  mark_for_compilation_flags->tf_xla_min_cluster_size = 2;
+  mark_for_compilation_flags->tf_xla_max_cluster_size =
+      std::numeric_limits<int32>::max();
+  mark_for_compilation_flags->tf_xla_clustering_debug = false;
+  mark_for_compilation_flags->tf_xla_cpu_global_jit = false;
+  mark_for_compilation_flags->tf_xla_clustering_fuel =
+      std::numeric_limits<int64>::max();
+  mark_for_compilation_flags->tf_xla_fusion_only = false;
+
+  device_flags = new XlaDeviceFlags;
+  device_flags->tf_xla_compile_on_demand = false;
+
+  ops_flags = new XlaOpsCommonFlags;
+  ops_flags->tf_xla_always_defer_compilation = false;
+
+  flag_list = new std::vector<Flag>({
+      Flag("tf_xla_enable_lazy_compilation",
+           &build_ops_flags->tf_xla_enable_lazy_compilation, ""),
+
+      Flag("tf_xla_compile_on_demand", &device_flags->tf_xla_compile_on_demand,
+           "Switch a device into 'on-demand' mode, where instead of "
+           "autoclustering ops are compiled one by one just-in-time."),
+
+      Flag("tf_xla_always_defer_compilation",
+           &ops_flags->tf_xla_always_defer_compilation, ""),
+  });
+  AppendDumpGraphFlagsInternal(flag_list);
+  AppendMarkForCompilationPassFlagsInternal(flag_list);
+  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
+}
+
+}  // namespace
+
+const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags() {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  return *build_ops_flags;
+}
+
+DumpGraphFlags* GetDumpGraphFlags() {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  return dump_graph_flags;
+}
+
+MarkForCompilationPassFlags* GetMarkForCompilationPassFlags() {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  return mark_for_compilation_flags;
+}
+
+XlaDeviceFlags* GetXlaDeviceFlags() {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  return device_flags;
+}
+
+const XlaOpsCommonFlags& GetXlaOpsCommonFlags() {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  return *ops_flags;
+}
+
+void AppendMarkForCompilationPassFlags(std::vector<Flag>* flag_list) {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  AppendMarkForCompilationPassFlagsInternal(flag_list);
+}
+
+void AppendDumpGraphFlags(std::vector<Flag>* flag_list) {
+  std::call_once(flags_init, &AllocateAndParseFlags);
+  AppendDumpGraphFlagsInternal(flag_list);
+}
+
+}  // namespace legacy_flags
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h b/tensorflow/compiler/jit/flags.h
similarity index 57%
rename from tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h
rename to tensorflow/compiler/jit/flags.h
index 79b47357a1..50f4f35836 100644
--- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h
+++ b/tensorflow/compiler/jit/flags.h
@@ -13,10 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_MARK_FOR_COMPILATION_PASS_FLAGS_H_
-#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_MARK_FOR_COMPILATION_PASS_FLAGS_H_
-
-// Legacy flags for the XLA bridge's mark_for_compilation_pass module.
+#ifndef TENSORFLOW_COMPILER_JIT_FLAGS_H_
+#define TENSORFLOW_COMPILER_JIT_FLAGS_H_
 
 #include <vector>
 
@@ -26,13 +24,7 @@ limitations under the License.
 namespace tensorflow {
 namespace legacy_flags {
 
-// Append to *flag_list flag definitions associated with the XLA bridge's
-// mark_for_compilation_pass module.
-void AppendMarkForCompilationPassFlags(
-    std::vector<tensorflow::Flag>* flag_list);
-
-// The values of flags associated with the XLA bridge's
-// mark_for_compilation_pass module.
+// Flags associated with the XLA bridge's mark_for_compilation_pass module.
 struct MarkForCompilationPassFlags {
   int32 tf_xla_auto_jit;  // Control compilation of operators into XLA
                           // computations on CPU and GPU devices.  0 = use
@@ -57,12 +49,57 @@ struct MarkForCompilationPassFlags {
                             // only using XLA.
 };
 
-// Return a pointer to the MarkForCompilationPassFlags struct;
+// Flags associated with the XLA bridge's xla_device module.
+struct XlaDeviceFlags {
+  // Switch the CPU device into "on-demand" mode, where instead of
+  // autoclustering ops are compiled one by one just-in-time.
+  // Enabling this mode by a legacy flag is a temporary mechanism. When this
+  // feature is battle-tested, we will switch this to be a session option.
+  bool tf_xla_compile_on_demand;
+};
+
+// Flags common to the _Xla* ops and their kernels.
+struct XlaOpsCommonFlags {
+  // If true, _XlaCompile always refuses to compile the cluster, which means the
+  // XLA clusters always run in the TF executor.  Defaults to false.
+  bool tf_xla_always_defer_compilation;
+};
+
+// Flags for the build_xla_ops pass.
+struct BuildXlaOpsPassFlags {
+  // Enables lazy compilation for TF/XLA (only when auto-clustering) if true.
+  // Defaults to true.
+  bool tf_xla_enable_lazy_compilation;
+};
+
+// Flags for the XLA bridge's dump_graph module.
+struct DumpGraphFlags {
+  // Path prefix to which graphs dumped during debugging should be written.
+  string tf_dump_graph_prefix;
+};
+
+// Return a pointer to the DumpGraphFlags struct;
 // repeated calls return the same pointer.
 // This should be called only after Flags::Parse() has returned.
+
+// Getters for flags structs defined above.  The first call to any of these
+// parses TF_XLA_FLAGS for all of them.  Those functions which return a pointer
+// always return the same pointer.
 MarkForCompilationPassFlags* GetMarkForCompilationPassFlags();
+const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags();
+XlaDeviceFlags* GetXlaDeviceFlags();
+const XlaOpsCommonFlags& GetXlaOpsCommonFlags();
+DumpGraphFlags* GetDumpGraphFlags();
+
+// Appends the flag definitions associated with
+// MarkForCompilationPassFlags/DumpGraphFlags to `flag_list`.
+//
+// Has the side-effect of parsing TF_XLA_FLAGS if that hasn't happened yet.
+void AppendMarkForCompilationPassFlags(
+    std::vector<tensorflow::Flag>* flag_list);
+void AppendDumpGraphFlags(std::vector<tensorflow::Flag>* flag_list);
 
 }  // namespace legacy_flags
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_MARK_FOR_COMPILATION_PASS_FLAGS_H_
+#endif  // TENSORFLOW_COMPILER_JIT_FLAGS_H_
diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
index d984ca15cb..fb0b0a89d9 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
@@ -23,7 +23,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/math_ops.h"
-#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/compiler/tf2xla/cc/ops/xla_ops.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
diff --git a/tensorflow/compiler/jit/kernels/BUILD b/tensorflow/compiler/jit/kernels/BUILD
index 830db9ebdd..0583774714 100644
--- a/tensorflow/compiler/jit/kernels/BUILD
+++ b/tensorflow/compiler/jit/kernels/BUILD
@@ -12,10 +12,10 @@ cc_library(
     hdrs = ["xla_ops.h"],
     deps = [
         "//tensorflow/compiler/jit:common",
+        "//tensorflow/compiler/jit:flags",
         "//tensorflow/compiler/jit:xla_compilation_cache",
         "//tensorflow/compiler/jit:xla_device",
         "//tensorflow/compiler/jit:xla_launch_util",
-        "//tensorflow/compiler/jit/legacy_flags:xla_ops_common_flags",
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:tf2xla_util",
         "//tensorflow/compiler/tf2xla:xla_compiler",
diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc
index 055de7afcc..2dea0c5330 100644
--- a/tensorflow/compiler/jit/kernels/xla_ops.cc
+++ b/tensorflow/compiler/jit/kernels/xla_ops.cc
@@ -18,7 +18,7 @@ limitations under the License.
 #include "absl/container/flat_hash_map.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/jit/defs.h"
-#include "tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/tf2xla_util.h"
 #include "tensorflow/compiler/tf2xla/xla_compiler.h"
diff --git a/tensorflow/compiler/jit/legacy_flags/BUILD b/tensorflow/compiler/jit/legacy_flags/BUILD
deleted file mode 100644
index 5fa6c85f06..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/BUILD
+++ /dev/null
@@ -1,65 +0,0 @@
-# Legacy command line flags for the XLA bridge libraries.
-
-# Please do not add more flags to this package.
-
-# The XLA bridge libraries were written in an environment that allowed
-# command-line flags to be scattered freely throughout the libraries.  This
-# model, while initially convenient, leads to a proliferation in unused command
-# line flags in tests and binaries, and serious problems in servers, where one
-# might wish parameters to be different in independent RPC calls to the same
-# routine.
-#
-# Please don't add more flags.  If you're a library author, pass options and
-# parameters explicitly through the library's interface.
-
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//tensorflow:internal"])
-
-cc_library(
-    name = "mark_for_compilation_pass_flags",
-    srcs = ["mark_for_compilation_pass_flags.cc"],
-    hdrs = ["mark_for_compilation_pass_flags.h"],
-    deps =
-        [
-            "//tensorflow/compiler/xla:parse_flags_from_env",
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-        ],
-)
-
-cc_library(
-    name = "xla_device_flags",
-    srcs = ["xla_device_flags.cc"],
-    hdrs = ["xla_device_flags.h"],
-    deps =
-        [
-            "//tensorflow/compiler/xla:parse_flags_from_env",
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-        ],
-)
-
-cc_library(
-    name = "build_xla_ops_pass_flags",
-    srcs = ["build_xla_ops_pass_flags.cc"],
-    hdrs = ["build_xla_ops_pass_flags.h"],
-    deps =
-        [
-            "//tensorflow/compiler/xla:parse_flags_from_env",
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-        ],
-)
-
-cc_library(
-    name = "xla_ops_common_flags",
-    srcs = ["xla_ops_common_flags.cc"],
-    hdrs = ["xla_ops_common_flags.h"],
-    deps =
-        [
-            "//tensorflow/compiler/xla:parse_flags_from_env",
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-        ],
-)
diff --git a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc
deleted file mode 100644
index 1f02cfaa95..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <mutex>  // NOLINT
-
-#include "tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h"
-#include "tensorflow/compiler/xla/parse_flags_from_env.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-namespace {
-
-BuildXlaOpsPassFlags* flags;
-std::vector<Flag>* flag_list;
-std::once_flag flags_init;
-
-void AllocateAndParseFlags() {
-  flags = new BuildXlaOpsPassFlags;
-  flags->tf_xla_enable_lazy_compilation = true;
-  flag_list = new std::vector<Flag>({
-      Flag("tf_xla_enable_lazy_compilation",
-           &flags->tf_xla_enable_lazy_compilation, ""),
-  });
-  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
-}
-
-}  // namespace
-
-const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags() {
-  std::call_once(flags_init, &AllocateAndParseFlags);
-  return *flags;
-}
-}  // namespace legacy_flags
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h b/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h
deleted file mode 100644
index 9aa5cf64d6..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/build_xla_ops_pass_flags.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_BUILD_XLA_OPS_PASS_FLAGS_H_
-#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_BUILD_XLA_OPS_PASS_FLAGS_H_
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// Flags for the build_xla_ops pass.
-struct BuildXlaOpsPassFlags {
-  // Enables lazy compilation for TF/XLA (only when auto-clustering) if true.
-  // Defaults to true.
-  bool tf_xla_enable_lazy_compilation;
-};
-
-// Parses the flags in BuildXlaOpsPassFlags from the TF_XLA_FLAGS environment
-// variable and returns a reference to the parsed copy.  Parses TF_XLA_FLAGS
-// only the first time this routine is called.
-const BuildXlaOpsPassFlags& GetBuildXlaOpsPassFlags();
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_BUILD_XLA_OPS_PASS_FLAGS_H_
diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
deleted file mode 100644
index 043012bd0b..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
+++ /dev/null
@@ -1,98 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Legacy flags for the XLA bridge's mark_for_compilation_pass module.
-
-#include <mutex>
-#include <vector>
-
-#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
-#include "tensorflow/compiler/xla/parse_flags_from_env.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// Pointers to the parsed value of the flags and flag descriptors, initialized
-// via flags_init.
-static MarkForCompilationPassFlags* flags;
-static std::vector<Flag>* flag_list;
-static std::once_flag flags_init;
-
-// Allocate *flags.  Called via call_once(&flags_init,...).
-static void AllocateFlags() {
-  flags = new MarkForCompilationPassFlags;
-  flags->tf_xla_auto_jit = 0;
-  flags->tf_xla_min_cluster_size = 2;
-  flags->tf_xla_max_cluster_size = std::numeric_limits<int32>::max();
-  flags->tf_xla_clustering_debug = false;
-  flags->tf_xla_cpu_global_jit = false;
-  flags->tf_xla_clustering_fuel = std::numeric_limits<int64>::max();
-  flags->tf_xla_fusion_only = false;
-  flag_list = new std::vector<Flag>(
-      {Flag("tf_xla_auto_jit", &flags->tf_xla_auto_jit,
-            "Control compilation of operators into XLA computations on CPU and "
-            "GPU devices.  0 = use ConfigProto setting; -1 = off; 1 = on for "
-            "things very likely to be improved; 2 = on for everything.  "
-            "Experimental."),
-       Flag("tf_xla_min_cluster_size", &flags->tf_xla_min_cluster_size,
-            "Minimum number of operators in an XLA compilation. Ignored for "
-            "operators placed on an XLA device or operators explicitly marked "
-            "for compilation."),
-       Flag("tf_xla_max_cluster_size", &flags->tf_xla_max_cluster_size,
-            "Maximum number of operators in an XLA compilation."),
-       Flag("tf_xla_clustering_debug", &flags->tf_xla_clustering_debug,
-            "Dump graphs during XLA compilation."),
-       Flag("tf_xla_cpu_global_jit", &flags->tf_xla_cpu_global_jit,
-            "Enables global JIT compilation for CPU via SessionOptions."),
-       Flag("tf_xla_clustering_fuel", &flags->tf_xla_clustering_fuel,
-            "Places an artificial limit on the number of ops marked as "
-            "eligible for clustering."),
-       Flag("tf_xla_fusion_only", &flags->tf_xla_fusion_only,
-            "enable fusion of element-wise operations only using XLA when "
-            "global_jit_level is ON*.")});
-  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
-
-  if (VLOG_IS_ON(1)) {
-    VLOG(1) << "Parsed MarkForCompilationPassFlags:";
-    VLOG(1) << "  tf_xla_auto_jit = " << flags->tf_xla_auto_jit;
-    VLOG(1) << "  tf_xla_min_cluster_size = " << flags->tf_xla_min_cluster_size;
-    VLOG(1) << "  tf_xla_max_cluster_size = " << flags->tf_xla_max_cluster_size;
-    VLOG(1) << "  tf_xla_clustering_debug = " << flags->tf_xla_clustering_debug;
-    VLOG(1) << "  tf_xla_cpu_global_jit = " << flags->tf_xla_cpu_global_jit;
-    VLOG(1) << "  tf_xla_clustering_fuel = " << flags->tf_xla_clustering_fuel;
-    VLOG(1) << "  tf_xla_fusion_only = " << flags->tf_xla_fusion_only;
-  }
-}
-
-// Append to *append_to flag definitions associated with the XLA bridge's
-// mark_for_compilation_pass module.
-void AppendMarkForCompilationPassFlags(std::vector<Flag>* append_to) {
-  std::call_once(flags_init, &AllocateFlags);
-  append_to->insert(append_to->end(), flag_list->begin(), flag_list->end());
-}
-
-// Return a pointer to the MarkForCompilationPassFlags struct;
-// repeated calls return the same pointer.
-// This should be called only after Flags::Parse() has returned.
-MarkForCompilationPassFlags* GetMarkForCompilationPassFlags() {
-  std::call_once(flags_init, &AllocateFlags);
-  return flags;
-}
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
deleted file mode 100644
index 6389cf10d4..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Legacy flags for the XLA bridge's xla_device module.
-
-#include <mutex>
-#include <vector>
-
-#include "tensorflow/compiler/jit/legacy_flags/xla_device_flags.h"
-#include "tensorflow/compiler/xla/parse_flags_from_env.h"
-#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// Pointers to the parsed value of the flags and flag descriptors, initialized
-// via flags_init.
-static XlaDeviceFlags* flags;
-static std::vector<Flag>* flag_list;
-static std::once_flag flags_init;
-
-// Allocate *flags.  Called via call_once(&flags_init,...).
-static void AllocateFlags() {
-  flags = new XlaDeviceFlags;
-  flags->tf_xla_compile_on_demand = false;
-  flag_list = new std::vector<Flag>({
-      Flag("tf_xla_compile_on_demand", &flags->tf_xla_compile_on_demand,
-           "Switch a device into 'on-demand' mode, where instead of "
-           "autoclustering ops are compiled one by one just-in-time."),
-  });
-  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
-}
-
-// Return a pointer to the XlaDeviceFlags struct;
-// repeated calls return the same pointer.
-// This should be called only after Flags::Parse() has returned.
-XlaDeviceFlags* GetXlaDeviceFlags() {
-  std::call_once(flags_init, &AllocateFlags);
-  return flags;
-}
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h
deleted file mode 100644
index 27b22121ac..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_
-#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_
-
-// Legacy flags for the XLA bridge's xla_device module.
-
-#include <vector>
-
-#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// The values of flags associated with the XLA bridge's
-// xla_device module.
-typedef struct {
-  // Switch the CPU device into "on-demand" mode, where instead of
-  // autoclustering ops are compiled one by one just-in-time.
-  // Enabling this mode by a legacy flag is a temporary mechanism. When this
-  // feature is battle-tested, we will switch this to be a session option.
-  bool tf_xla_compile_on_demand;
-} XlaDeviceFlags;
-
-// Return a pointer to the XlaDeviceFlags struct;
-// repeated calls return the same pointer.
-// This should be called only after Flags::Parse() has returned.
-XlaDeviceFlags* GetXlaDeviceFlags();
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc
deleted file mode 100644
index bf8bd0d47f..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.cc
+++ /dev/null
@@ -1,52 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <mutex>  // NOLINT
-#include <vector>
-
-#include "tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h"
-#include "tensorflow/compiler/xla/parse_flags_from_env.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-
-XlaOpsCommonFlags* flags;
-std::vector<Flag>* flag_list;
-std::once_flag flags_init;
-
-void AllocateAndParseFlags() {
-  flags = new XlaOpsCommonFlags;
-  flags->tf_xla_always_defer_compilation = false;
-  flag_list = new std::vector<Flag>({
-      Flag("tf_xla_always_defer_compilation",
-           &flags->tf_xla_always_defer_compilation, ""),
-  });
-  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
-
-  if (VLOG_IS_ON(1)) {
-    VLOG(1) << "Parsed XlaOpsCommonFlags:";
-    VLOG(1) << "  tf_xla_always_defer_compilation = "
-            << flags->tf_xla_always_defer_compilation;
-  }
-}
-
-const XlaOpsCommonFlags& GetXlaOpsCommonFlags() {
-  std::call_once(flags_init, &AllocateAndParseFlags);
-  return *flags;
-}
-}  // namespace legacy_flags
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h b/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h
deleted file mode 100644
index 7c5c1818ef..0000000000
--- a/tensorflow/compiler/jit/legacy_flags/xla_ops_common_flags.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_OPS_COMMON_FLAGS_H_
-#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_OPS_COMMON_FLAGS_H_
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// Flags common to the _Xla* ops and their kernels.
-struct XlaOpsCommonFlags {
-  // If true, _XlaCompile always refuses to compile the cluster, which means the
-  // XLA clusters always run in the TF executor.  Defaults to false.
-  bool tf_xla_always_defer_compilation;
-};
-
-// Parses the flags in XlaOpsCommonFlags from the TF_XLA_FLAGS environment
-// variable and returns a reference to the parsed copy.  Parses TF_XLA_FLAGS
-// only the first time this routine is called.
-const XlaOpsCommonFlags& GetXlaOpsCommonFlags();
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_OPS_COMMON_FLAGS_H_
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 70033cae0a..9b02b0c3f9 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -24,8 +24,8 @@ limitations under the License.
 #include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
-#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
 #include "tensorflow/compiler/jit/union_find.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/compiler/tf2xla/const_analysis.h"
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 116e075603..13126e0878 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -17,8 +17,8 @@ limitations under the License.
 // operators using XLA via the XLA "Host" (CPU) backend.
 
 #include "absl/memory/memory.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/compiler/jit/kernels/xla_ops.h"
-#include "tensorflow/compiler/jit/legacy_flags/xla_device_flags.h"
 #include "tensorflow/compiler/jit/xla_compile_on_demand_op.h"
 #include "tensorflow/compiler/jit/xla_device.h"
 #include "tensorflow/compiler/jit/xla_device_ops.h"
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index ab018d8507..486b4d8a8c 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -195,8 +195,8 @@ cc_library(
         ":sharding_util",
         ":side_effect_util",
         ":tf2xla_util",
+        "//tensorflow/compiler/jit:flags",
         "//tensorflow/compiler/jit:xla_cluster_util",
-        "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags",
         "//tensorflow/compiler/tf2xla/lib:util",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -439,13 +439,12 @@ cc_library(
     name = "dump_graph",
     srcs = [
         "dump_graph.cc",
-        "dump_graph_flags.cc",
-        "dump_graph_flags.h",
     ],
     hdrs = [
         "dump_graph.h",
     ],
     deps = [
+        "//tensorflow/compiler/jit:flags",
         "//tensorflow/compiler/xla:parse_flags_from_env",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/tf2xla/dump_graph.cc b/tensorflow/compiler/tf2xla/dump_graph.cc
index 380c6a7e23..34f891890a 100644
--- a/tensorflow/compiler/tf2xla/dump_graph.cc
+++ b/tensorflow/compiler/tf2xla/dump_graph.cc
@@ -19,7 +19,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 
 #include "absl/strings/str_cat.h"
-#include "tensorflow/compiler/tf2xla/dump_graph_flags.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 
diff --git a/tensorflow/compiler/tf2xla/dump_graph_flags.cc b/tensorflow/compiler/tf2xla/dump_graph_flags.cc
deleted file mode 100644
index 32066bfd5b..0000000000
--- a/tensorflow/compiler/tf2xla/dump_graph_flags.cc
+++ /dev/null
@@ -1,63 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Legacy flags for the XLA bridge's dump_graph module.
-
-#include <mutex>
-#include <vector>
-
-#include "tensorflow/compiler/tf2xla/dump_graph_flags.h"
-#include "tensorflow/compiler/xla/parse_flags_from_env.h"
-#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// Pointers to the parsed value of the flags and flag descriptors, initialized
-// via flags_init.
-static DumpGraphFlags* flags;
-static std::vector<Flag>* flag_list;
-static std::once_flag flags_init;
-
-// Allocate *flags.  Called via call_once(&flags_init,...).
-static void AllocateFlags() {
-  flags = new DumpGraphFlags;
-  flags->tf_dump_graph_prefix = "/tmp/";
-  flag_list = new std::vector<Flag>({
-      Flag("tf_dump_graph_prefix", &flags->tf_dump_graph_prefix,
-           "Path prefix to which graphs dumped during debugging should be "
-           "written."),
-  });
-  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
-}
-
-// Append to *append_to flag definitions associated with the XLA bridge's
-// dump_graph module.
-void AppendDumpGraphFlags(std::vector<Flag>* append_to) {
-  std::call_once(flags_init, &AllocateFlags);
-  append_to->insert(append_to->end(), flag_list->begin(), flag_list->end());
-}
-
-// Return a pointer to the DumpGraphFlags struct;
-// repeated calls return the same pointer.
-// This should be called only after Flags::Parse() has returned.
-DumpGraphFlags* GetDumpGraphFlags() {
-  std::call_once(flags_init, &AllocateFlags);
-  return flags;
-}
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/dump_graph_flags.h b/tensorflow/compiler/tf2xla/dump_graph_flags.h
deleted file mode 100644
index 80a3307d92..0000000000
--- a/tensorflow/compiler/tf2xla/dump_graph_flags.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_TF2XLA_DUMP_GRAPH_FLAGS_H_
-#define TENSORFLOW_COMPILER_TF2XLA_DUMP_GRAPH_FLAGS_H_
-
-// Legacy flags for the XLA bridge's dump_graph module.
-
-#include <vector>
-
-#include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
-
-namespace tensorflow {
-namespace legacy_flags {
-
-// Append to *flag_list flag definitions associated with the XLA bridge's
-// dump_graph module.
-void AppendDumpGraphFlags(std::vector<tensorflow::Flag>* flag_list);
-
-// The values of flags associated with the XLA bridge's
-// dump_graph module.
-typedef struct {
-  string tf_dump_graph_prefix;  // Path prefix to which graphs dumped during
-                                // debugging should be written.
-} DumpGraphFlags;
-
-// Return a pointer to the DumpGraphFlags struct;
-// repeated calls return the same pointer.
-// This should be called only after Flags::Parse() has returned.
-DumpGraphFlags* GetDumpGraphFlags();
-
-}  // namespace legacy_flags
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_TF2XLA_DUMP_GRAPH_FLAGS_H_
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.cc b/tensorflow/compiler/tf2xla/xla_op_registry.cc
index dcd0e9c5c1..49a6ebbbde 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.cc
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.cc
@@ -18,7 +18,7 @@ limitations under the License.
 #include <functional>
 #include <memory>
 
-#include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
+#include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
-- 
GitLab


From 8c41a3aa72b18e2eca6f2f43bef85244ce45a156 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 14 Nov 2018 18:41:40 -0800
Subject: [PATCH 0276/1554] Change __init__.py file names under _api/v1 and
 _api/v2 to v1.py or v2.py. We copy this file over to tensorflow/ directory.
 Basically, we have tensorflow/__init__.py and tensorflow/compat/<v1 or
 v2>/__init__.py which is the same file. This is both confusing and causes
 issues.

PiperOrigin-RevId: 221552254
---
 tensorflow/BUILD                                     | 6 ++++--
 tensorflow/python/tools/api/generator/api_gen.bzl    | 9 ++++++++-
 tensorflow/tools/api/tests/api_compatibility_test.py | 2 +-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index a81f132fe2..7c87a616a6 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -554,8 +554,8 @@ genrule(
     }),
     outs = ["__init__.py"],
     cmd = select({
-        "api_version_2": "cp $(@D)/_api/v2/__init__.py $(OUTS)",
-        "//conditions:default": "cp $(@D)/_api/v1/__init__.py $(OUTS)",
+        "api_version_2": "cp $(@D)/_api/v2/v2.py $(OUTS)",
+        "//conditions:default": "cp $(@D)/_api/v1/v1.py $(OUTS)",
     }),
 )
 
@@ -566,6 +566,7 @@ gen_api_init_files(
     output_dir = "_api/v1/",
     output_files = TENSORFLOW_API_INIT_FILES_V1,
     output_package = "tensorflow._api.v1",
+    root_file_name = "v1.py",
     root_init_template = "api_template_v1.__init__.py",
 )
 
@@ -581,6 +582,7 @@ gen_api_init_files(
     output_dir = "_api/v2/",
     output_files = TENSORFLOW_API_INIT_FILES_V2,
     output_package = "tensorflow._api.v2",
+    root_file_name = "v2.py",
     root_init_template = "api_template.__init__.py",
 )
 
diff --git a/tensorflow/python/tools/api/generator/api_gen.bzl b/tensorflow/python/tools/api/generator/api_gen.bzl
index 8ee5e0ac94..5e64cc64d2 100644
--- a/tensorflow/python/tools/api/generator/api_gen.bzl
+++ b/tensorflow/python/tools/api/generator/api_gen.bzl
@@ -20,7 +20,8 @@ def gen_api_init_files(
         packages = ["tensorflow.python", "tensorflow.lite.python.lite"],
         package_deps = ["//tensorflow/python:no_contrib"],
         output_package = "tensorflow",
-        output_dir = ""):
+        output_dir = "",
+        root_file_name = "__init__.py"):
     """Creates API directory structure and __init__.py files.
 
     Creates a genrule that generates a directory structure with __init__.py
@@ -54,6 +55,7 @@ def gen_api_init_files(
       output_package: Package where generated API will be added to.
       output_dir: Subdirectory to output API to.
         If non-empty, must end with '/'.
+      root_file_name: Name of the root file with all the root imports.
     """
     root_init_template_flag = ""
     if root_init_template:
@@ -73,6 +75,11 @@ def gen_api_init_files(
         ],
     )
 
+    # Replace name of root file with root_file_name.
+    output_files = [
+        root_file_name if f == "__init__.py" else f
+        for f in output_files
+    ]
     all_output_files = ["%s%s" % (output_dir, f) for f in output_files]
     compat_api_version_flags = ""
     for compat_api_version in compat_api_versions:
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index fb489ea80f..b0f3742af1 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -33,7 +33,7 @@ import re
 import sys
 
 import tensorflow as tf
-from tensorflow._api import v2 as tf_v2
+from tensorflow._api.v2 import v2 as tf_v2
 
 from google.protobuf import message
 from google.protobuf import text_format
-- 
GitLab


From f18df6b5c362feead21edf094f654331dbab38fb Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Wed, 14 Nov 2018 18:50:39 -0800
Subject: [PATCH 0277/1554] Small updates to LinearOperatorComposition. 
 Noticed while writing LinearOperatorInversion.

PiperOrigin-RevId: 221553102
---
 tensorflow/python/ops/linalg/linear_operator_composition.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/python/ops/linalg/linear_operator_composition.py b/tensorflow/python/ops/linalg/linear_operator_composition.py
index 0292bc51dc..f499b30661 100644
--- a/tensorflow/python/ops/linalg/linear_operator_composition.py
+++ b/tensorflow/python/ops/linalg/linear_operator_composition.py
@@ -275,6 +275,3 @@ class LinearOperatorComposition(linear_operator.LinearOperator):
     for operator in solve_order_list[1:]:
       solution = operator.solve(solution, adjoint=adjoint)
     return solution
-
-  def _add_to_tensor(self, x):
-    return self.to_dense() + x
-- 
GitLab


From eedc97615c381ed621871ea62e4ad9ee0feafe05 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 14 Nov 2018 18:53:22 -0800
Subject: [PATCH 0278/1554] Split partially_decluster_pass.cc into two
 namespaces

This reflects the two reasons why we decluster nodes:

 - To reduce device to host copies
 - To reduce recompilations

I'm about to add a third so I thought this cleanup made sense.

PiperOrigin-RevId: 221553369
---
 .../compiler/jit/partially_decluster_pass.cc  | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/jit/partially_decluster_pass.cc b/tensorflow/compiler/jit/partially_decluster_pass.cc
index 36b345ecbf..42ea3926e1 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass.cc
@@ -26,6 +26,10 @@ limitations under the License.
 
 namespace tensorflow {
 namespace {
+
+bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); }
+
+namespace reduce_device_to_host_copies {
 Status FindNodesToDecluster(const Graph& graph,
                             absl::flat_hash_set<Node*>* result,
                             absl::Span<Node* const> post_order) {
@@ -140,8 +144,6 @@ Status PartiallyDeclusterNode(Graph* graph, Node* n) {
   return Status::OK();
 }
 
-bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); }
-
 // Clones nodes to outside their cluster to avoid device-to-host copies.  For
 // instance, converts this:
 //
@@ -168,7 +170,7 @@ bool NotBackedge(const Edge& edge) { return !edge.src()->IsNextIteration(); }
 // where the ===> arrow has a hostmem source and destination and would entail a
 // device to host copy if the source and destination were not in the same XLA
 // cluster.
-Status PartiallyDeclusterToRemoveDeviceToHostCopies(Graph* graph) {
+Status PartiallyDeclusterGraph(Graph* graph) {
   // When deciding whether to decluster a particular node, we base our decision
   // on if we've decided that some of its consumers have to be declustered too.
   // Iterating the graph in post-order guarantees that consumers have been
@@ -206,7 +208,9 @@ Status PartiallyDeclusterToRemoveDeviceToHostCopies(Graph* graph) {
 
   return Status::OK();
 }
+}  // namespace reduce_device_to_host_copies
 
+namespace reduce_recompilation {
 bool IsIntraClusterEdge(const Edge& edge) {
   absl::optional<absl::string_view> src_cluster_name =
       GetXlaClusterForNode(*edge.src());
@@ -269,7 +273,7 @@ Status MustCompileNode(const Node* n, bool* must_compile) {
 // regress performance in any significant manner.  We will have to revisit this
 // algorith with a more complex cost model if this assumption turns out to be
 // incorrect.
-Status DeclusterNodesToReduceRecompilations(Graph* graph) {
+Status PartiallyDeclusterGraph(Graph* graph) {
   std::vector<bool> compile_time_const_nodes(graph->num_node_ids());
   TF_RETURN_IF_ERROR(BackwardsConstAnalysis(
       *graph, nullptr, &compile_time_const_nodes, IsIntraClusterEdge));
@@ -322,7 +326,7 @@ Status DeclusterNodesToReduceRecompilations(Graph* graph) {
 
   return Status::OK();
 }
-
+}  // namespace reduce_recompilation
 }  // namespace
 
 Status PartiallyDeclusterPass::Run(
@@ -334,8 +338,9 @@ Status PartiallyDeclusterPass::Run(
 
   Graph* graph = options.graph->get();
 
-  TF_RETURN_IF_ERROR(PartiallyDeclusterToRemoveDeviceToHostCopies(graph));
-  TF_RETURN_IF_ERROR(DeclusterNodesToReduceRecompilations(graph));
+  TF_RETURN_IF_ERROR(
+      reduce_device_to_host_copies::PartiallyDeclusterGraph(graph));
+  TF_RETURN_IF_ERROR(reduce_recompilation::PartiallyDeclusterGraph(graph));
 
   return Status::OK();
 }
-- 
GitLab


From ddfa238d6e0f2a4b53ac64c41736905f8e4681f8 Mon Sep 17 00:00:00 2001
From: RJ Ryan <rjryan@google.com>
Date: Wed, 14 Nov 2018 19:35:38 -0800
Subject: [PATCH 0279/1554] Remove tf.signal's __init__.py.

An attempt to avoid conflicts with the Python standard library 'signal' module.

PiperOrigin-RevId: 221557063
---
 tensorflow/compiler/tests/fft_test.py              |  2 +-
 tensorflow/python/__init__.py                      |  2 +-
 .../python/ops/signal/{__init__.py => signal.py}   | 14 ++++++++++++++
 tensorflow/python/tools/api/generator/doc_srcs.py  |  2 +-
 4 files changed, 17 insertions(+), 3 deletions(-)
 rename tensorflow/python/ops/signal/{__init__.py => signal.py} (67%)

diff --git a/tensorflow/compiler/tests/fft_test.py b/tensorflow/compiler/tests/fft_test.py
index e92afd5d6f..fab356991a 100644
--- a/tensorflow/compiler/tests/fft_test.py
+++ b/tensorflow/compiler/tests/fft_test.py
@@ -27,8 +27,8 @@ from tensorflow.compiler.tests import xla_test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
-from tensorflow.python.ops import signal
 from tensorflow.python.ops import spectral_ops
+from tensorflow.python.ops.signal import signal
 from tensorflow.python.platform import googletest
 
 BATCH_DIMS = (3, 5)
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 5da304e38c..f0d8079ce7 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -91,7 +91,7 @@ from tensorflow.python.ops import spectral_ops as spectral
 from tensorflow.python.ops.distributions import distributions
 from tensorflow.python.ops.linalg import linalg
 from tensorflow.python.ops.losses import losses
-from tensorflow.python.ops import signal
+from tensorflow.python.ops.signal import signal
 from tensorflow.python.profiler import profiler
 from tensorflow.python.saved_model import saved_model
 from tensorflow.python.summary import summary
diff --git a/tensorflow/python/ops/signal/__init__.py b/tensorflow/python/ops/signal/signal.py
similarity index 67%
rename from tensorflow/python/ops/signal/__init__.py
rename to tensorflow/python/ops/signal/signal.py
index 3fa4e94e58..4ddd30533c 100644
--- a/tensorflow/python/ops/signal/__init__.py
+++ b/tensorflow/python/ops/signal/signal.py
@@ -1,3 +1,17 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
 """Signal processing operations.
 
 See the [tf.signal](https://tensorflow.org/api_guides/python/contrib.signal)
diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py
index 479d5006d1..7613104749 100644
--- a/tensorflow/python/tools/api/generator/doc_srcs.py
+++ b/tensorflow/python/tools/api/generator/doc_srcs.py
@@ -56,7 +56,7 @@ _TENSORFLOW_DOC_SOURCES = {
     'resource_loader': DocSource(
         docstring_module_name='platform.resource_loader'),
     'sets': DocSource(docstring_module_name='ops.sets'),
-    'signal': DocSource(docstring_module_name='ops.signal'),
+    'signal': DocSource(docstring_module_name='ops.signal.signal'),
     'sparse': DocSource(docstring_module_name='ops.sparse_ops'),
     'spectral': DocSource(docstring_module_name='ops.spectral_ops'),
     'strings': DocSource(docstring_module_name='ops.string_ops'),
-- 
GitLab


From 35228fbf0caf18a820a7306809f1fae5e111a64e Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 14 Nov 2018 19:53:57 -0800
Subject: [PATCH 0280/1554] Use new enum `ReduceOp` for all callers of
 strategy.reduce. Also use it for loss reduction as well as OutputContext
 which were previously using VariableAggregation.

PiperOrigin-RevId: 221558489
---
 .../distribute/python/minimize_loss_test.py   | 47 +++++++++----------
 .../distribute/python/mirrored_strategy.py    |  7 ++-
 .../contrib/distribute/python/tpu_strategy.py |  9 ++--
 .../contrib/distribute/python/values.py       | 36 +++++++-------
 tensorflow/contrib/optimizer_v2/BUILD         |  1 +
 .../contrib/optimizer_v2/optimizer_v2.py      |  6 +--
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/keras/BUILD                 |  1 +
 .../keras/engine/training_distributed.py      | 18 +++----
 tensorflow/python/keras/optimizer_v2/BUILD    |  1 +
 .../python/keras/optimizer_v2/optimizer_v2.py |  4 +-
 tensorflow/python/training/distribute.py      |  6 +--
 tensorflow/python/training/moving_averages.py |  4 +-
 tensorflow/python/training/optimizer.py       |  8 ++--
 14 files changed, 73 insertions(+), 76 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 5d3b5d8922..1f57dd1754 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -403,9 +403,9 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         train_op = optimizer.minimize(loss_fn)
         loss = loss_fn()
         output_context.set_last_step_output(
-            name="replica_loss_agg",
+            name="replica_loss_reduced",
             output=loss,
-            aggregation=variables_lib.VariableAggregation.MEAN)
+            reduce_op=reduce_util.ReduceOp.MEAN)
         output_context.set_non_tensor_output(key1, value1)
         return (train_op, loss)
 
@@ -413,11 +413,11 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         (train_op, loss) = distribution.call_for_each_replica(
             model_fn, args=(output_context,) + inputs)
         output_context.set_last_step_output(
-            name="cross_replica_loss_agg",
+            name="cross_replica_loss_reduced",
             output=loss,
-            aggregation=variables_lib.VariableAggregation.MEAN)
+            reduce_op=reduce_util.ReduceOp.MEAN)
         output_context.set_last_step_output(
-            name="cross_replica_loss_noagg",
+            name="cross_replica_loss_not_reduced",
             output=loss)
         return distribution.group(train_op)
 
@@ -425,16 +425,16 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
       def run_step():
         initial_loss = lambda: constant_op.constant(1e7)
-        # Initial values corresponding to aggregated losses are just single
-        # tensors. But for non aggregated losses, we need to have initial
+        # Initial values corresponding to reduced losses are just single
+        # tensors. But for non reduced losses, we need to have initial
         # values that are of the same structure as non reduced losses. In
         # MirroredStrategy, this will be a list of losses, in TPUStrategy
         # it will be single tensor. Using `broadcast` followed by `unwrap`
         # gives us the desired initial value structure.
         initial_loop_values = {
-            "replica_loss_agg": initial_loss(),
-            "cross_replica_loss_agg": initial_loss(),
-            "cross_replica_loss_noagg":
+            "replica_loss_reduced": initial_loss(),
+            "cross_replica_loss_reduced": initial_loss(),
+            "cross_replica_loss_not_reduced":
             distribution.unwrap(distribution.broadcast(initial_loss()))
         }
         ctx = distribution.run_steps_on_dataset(
@@ -444,17 +444,17 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         self.assertEqual({key1: [value1]}, ctx.non_tensor_outputs)
         self._verify_loss_output(
             initial_loss(),
-            loss_output=ctx.last_step_outputs["replica_loss_agg"],
-            aggregated=True, distribution=distribution)
+            loss_output=ctx.last_step_outputs["replica_loss_reduced"],
+            reduced=True, distribution=distribution)
         self._verify_loss_output(
             initial_loss(),
-            loss_output=ctx.last_step_outputs["cross_replica_loss_agg"],
-            aggregated=True, distribution=distribution)
+            loss_output=ctx.last_step_outputs["cross_replica_loss_reduced"],
+            reduced=True, distribution=distribution)
         self._verify_loss_output(
             initial_loss(),
-            loss_output=ctx.last_step_outputs["cross_replica_loss_noagg"],
-            aggregated=False, distribution=distribution)
-        return (ctx.run_op, ctx.last_step_outputs["replica_loss_agg"])
+            loss_output=ctx.last_step_outputs["cross_replica_loss_not_reduced"],
+            reduced=False, distribution=distribution)
+        return (ctx.run_op, ctx.last_step_outputs["replica_loss_reduced"])
 
       self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
@@ -479,17 +479,16 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       error_is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
       self.assertTrue(error_is_not_increasing)
 
-  def _verify_loss_output(self, initial_loss, loss_output, aggregated,
+  def _verify_loss_output(self, initial_loss, loss_output, reduced,
                           distribution):
-    if not aggregated:
-      self.assertEqual(distribution.num_replicas_in_sync,
-                       len(distribution.unwrap(loss_output)))
+    if not reduced:
+      self.assertLen(distribution.unwrap(loss_output),
+                     distribution.num_replicas_in_sync)
       loss_output = distribution.reduce(
-          aggregation=reduce_util.ReduceOp.MEAN,
-          value=loss_output, destinations="/device:CPU:0")
+          reduce_util.ReduceOp.MEAN, loss_output, destinations="/device:CPU:0")
 
     unwrapped_output = distribution.unwrap(loss_output)
-    self.assertEqual(1, len(unwrapped_output))
+    self.assertLen(unwrapped_output, 1)
     loss_tensor = unwrapped_output[0]
     self.assertEqual(initial_loss.dtype, loss_tensor.dtype)
     self.assertEqual(initial_loss.shape, loss_tensor.shape)
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 4bb2cb1990..9795df8d13 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -36,7 +36,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.training import coordinator
 from tensorflow.python.training import device_util
 from tensorflow.python.training import distribute as distribute_lib
@@ -532,11 +531,11 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     last_step_tensor_outputs_dict = nest.pack_sequence_as(
         ctx.last_step_outputs, last_step_tensor_outputs)
 
-    for (name, aggregation) in ctx._last_step_outputs_aggregations.items():  # pylint: disable=protected-access
+    for name, reduce_op in ctx._last_step_outputs_reduce_ops.items():  # pylint: disable=protected-access
       output = last_step_tensor_outputs_dict[name]
-      # For outputs that have already been aggregated, wrap them in a Mirrored
+      # For outputs that have already been reduced, wrap them in a Mirrored
       # container, else in a PerReplica container.
-      if aggregation is variables_lib.VariableAggregation.NONE:
+      if reduce_op is None:
         last_step_tensor_outputs_dict[name] = values.regroup(
             {d: t for d, t in zip(self._devices, output)}, values.PerReplica)
       else:
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index adc075011d..dcc2c64438 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -41,7 +41,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.training import device_util
 from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
@@ -360,14 +359,14 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     last_step_tensor_outputs_dict = nest.pack_sequence_as(
         ctx.last_step_outputs, last_step_tensor_outputs)
 
-    for (name, aggregation) in ctx._last_step_outputs_aggregations.items():  # pylint: disable=protected-access
+    for name, reduce_op in ctx._last_step_outputs_reduce_ops.items():  # pylint: disable=protected-access
       output = last_step_tensor_outputs_dict[name]
-      # For outputs that have already been aggregated, take the first value
+      # For outputs that have already been reduced, take the first value
       # from the list as each value should be the same. Else return the full
       # list of values.
-      # TODO(josh11b): If aggregation is NONE, we should return a PerReplica
+      # TODO(josh11b): If reduce_op is NONE, we should return a PerReplica
       # value.
-      if aggregation is not variables_lib.VariableAggregation.NONE:
+      if reduce_op is not None:
         # TODO(priyag): Should this return the element or a list with 1 element
         last_step_tensor_outputs_dict[name] = output[0]
     ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index f565381b49..bf8c958efa 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -41,7 +41,6 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.training import device_util
 from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import distribution_strategy_context
@@ -1508,7 +1507,7 @@ class MultiStepContext(object):
       A context object.
     """
     self._last_step_outputs = {}
-    self._last_step_outputs_aggregations = {}
+    self._last_step_outputs_reduce_ops = {}
     self._non_tensor_outputs = {}
 
   @property
@@ -1518,8 +1517,8 @@ class MultiStepContext(object):
     Keys in the dictionary are names of tensors to be captured, as specified
     when `set_last_step_output` is called.
     Values in the dictionary are the tensors themselves. If
-    `set_last_step_output` was called with an `aggregation` for this output,
-    then the value is the aggregated value.
+    `set_last_step_output` was called with a `reduce_op` for this output,
+    then the value is the reduced value.
 
     Returns:
       A dictionary with last step outputs.
@@ -1532,8 +1531,7 @@ class MultiStepContext(object):
       raise ValueError("Need a dictionary to set last_step_outputs.")
     self._last_step_outputs = outputs
 
-  def set_last_step_output(self, name, output,
-                           aggregation=variables_lib.VariableAggregation.NONE):
+  def set_last_step_output(self, name, output, reduce_op=None):
     """Set `output` with `name` to be outputted from the last step.
 
     Args:
@@ -1541,37 +1539,35 @@ class MultiStepContext(object):
         name.
       output: The tensors that should be outputted with `name`. See below for
         actual types supported.
-      aggregation: Aggregation method to use to aggregate outputs from multiple
+      reduce_op: Reduction method to use to reduce outputs from multiple
         replicas. Required if `set_last_step_output` is called in a replica
         context. Optional in cross_replica_context.
-        When present, the outputs from all the replicas are aggregated using the
+        When present, the outputs from all the replicas are reduced using the
         current distribution strategy's `reduce` method. Hence, the type of
         `output` must be what's supported by the corresponding `reduce` method.
-        For e.g. if using MirroredStrategy and aggregation is set, output
+        For e.g. if using MirroredStrategy and reduction is set, output
         must be a `PerReplica` value.
-        The aggregation method is also recorded in a dictionary
-        `_last_step_outputs_aggregations` for later interpreting of the
+        The reduce method is also recorded in a dictionary
+        `_last_step_outputs_reduce_ops` for later interpreting of the
         outputs as already reduced or not.
-        # TODO(priyag): Change aggregation type used here.
-
     """
     if distribution_strategy_context.get_cross_replica_context():
-      self._last_step_outputs_aggregations[name] = aggregation
-      if aggregation is variables_lib.VariableAggregation.NONE:
+      self._last_step_outputs_reduce_ops[name] = reduce_op
+      if reduce_op is None:
         self._last_step_outputs[name] = output
       else:
         distribution = distribution_strategy_context.get_distribution_strategy()
         self._last_step_outputs[name] = distribution.reduce(
-            aggregation, output, destinations="/device:CPU:0")
+            reduce_op, output, destinations="/device:CPU:0")
     else:
-      assert aggregation is not variables_lib.VariableAggregation.NONE
+      assert reduce_op is not None
       def merge_fn(distribution, value):
         self._last_step_outputs[name] = distribution.reduce(
-            aggregation, value, destinations="/device:CPU:0")
+            reduce_op, value, destinations="/device:CPU:0")
         # Setting this inside the `merge_fn` because all replicas share the same
         # context object, so it's more robust to set it only once (even if all
         # the replicas are trying to set the same value).
-        self._last_step_outputs_aggregations[name] = aggregation
+        self._last_step_outputs_reduce_ops[name] = reduce_op
 
       distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, output)
@@ -1588,7 +1584,7 @@ class MultiStepContext(object):
     else:
       def merge_fn(distribution, value):
         # NOTE(priyag): For non tensor outputs, we simply return all the values
-        # in a list as aggregation doesn't make sense on non tensors.
+        # in a list as reduction doesn't make sense on non tensors.
         self._non_tensor_outputs[name] = distribution.unwrap(value)
       distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, output)
diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD
index 3ba3ee29ec..835fb4aec4 100644
--- a/tensorflow/contrib/optimizer_v2/BUILD
+++ b/tensorflow/contrib/optimizer_v2/BUILD
@@ -56,6 +56,7 @@ py_library(
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:reduce_util",
     ],
 )
 
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 467dd86d8f..1b6e70d3a0 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -24,6 +24,7 @@ import abc
 
 import six
 
+from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -848,8 +849,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
     """Scale loss for the number of replicas."""
     if scale_loss_by_num_replicas is None:
       scale_loss_by_num_replicas = (
-          distribute_lib.get_loss_reduction() == variable_scope
-          .VariableAggregation.MEAN)
+          distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN)
     if scale_loss_by_num_replicas:
       num_replicas = \
         distribute_ctx.get_distribution_strategy().num_replicas_in_sync
@@ -928,7 +928,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
   def _distributed_apply(self, distribution, grads_and_vars, global_step, name):
     """`apply_gradients` for use with a `DistributionStrategy`."""
     reduced_grads = distribution.batch_reduce(
-        variable_scope.VariableAggregation.SUM, grads_and_vars)
+        ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     var_list = [v for _, v in grads_and_vars]
     grads_and_vars = zip(reduced_grads, var_list)
 
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a6e5c110b7..18f226beb5 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3476,6 +3476,7 @@ py_library(
         "@six_archive//:six",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:distribute_coordinator_context",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index db78eff86e..365ae4746d 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -146,6 +146,7 @@ py_library(
     deps = [
         ":backend",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/keras/optimizer_v2",
         "//tensorflow/python/training/checkpointable:data_structures",
         "//tensorflow/tools/docs:doc_controls",
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 808d7c9f33..3a2373b4cf 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 import enum
 import numpy as np
 
+from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -34,7 +35,6 @@ from tensorflow.python.keras.engine import distributed_training_utils
 from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
@@ -288,12 +288,12 @@ def _experimental_fit_loop(
 
     for label, output in zip(out_labels, combined_fn.outputs):
       if label == 'loss':
-        aggregation = distribute_lib.get_loss_reduction()
+        reduce_op = distribute_lib.get_loss_reduction()
       else:
-        # We aggregate all other metrics using mean for now. This is temporary
+        # We reduce all other metrics using mean for now. This is temporary
         # workaround until new metrics are in place.
-        aggregation = variable_scope.VariableAggregation.MEAN
-      ctx.set_last_step_output(label, output, aggregation)
+        reduce_op = ds_reduce_util.ReduceOp.MEAN
+      ctx.set_last_step_output(label, output, reduce_op)
 
     # TODO(priyag, sourabhbajaj): Ignoring these things from the combined_fn:
     # feed_dict, session kwargs, run options, run_metadata for now. These should
@@ -571,12 +571,12 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None,
 
     for label, output in zip(model.metrics_names, combined_fn.outputs):
       if label == 'loss':
-        aggregation = distribute_lib.get_loss_reduction()
+        reduce_op = distribute_lib.get_loss_reduction()
       else:
-        # We aggregate all other metrics using mean for now. This is temporary
+        # We reduce all other metrics using mean for now. This is temporary
         # workaround until new metrics are in place.
-        aggregation = variable_scope.VariableAggregation.MEAN
-      ctx.set_last_step_output(label, output, aggregation)
+        reduce_op = ds_reduce_util.ReduceOp.MEAN
+      ctx.set_last_step_output(label, output, reduce_op)
 
     return combined_fn.updates_op
 
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index eaa764992f..e0ff587549 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -27,6 +27,7 @@ py_library(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:reduce_util",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 68b5368711..8e3aea2846 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -24,6 +24,7 @@ import abc
 
 import six
 
+from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -32,7 +33,6 @@ from tensorflow.python.keras import backend
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.ops import gradients
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import distribution_strategy_context
@@ -580,7 +580,7 @@ def merge_grads(grads_and_vars):
 
   def merge_grad_fn(strategy, grads_and_vars):
     reduced_grads = strategy.batch_reduce(
-        variable_scope.VariableAggregation.MEAN, grads_and_vars)
+        ds_reduce_util.ReduceOp.MEAN, grads_and_vars)
     return reduced_grads
 
   return distribution_strategy_context.get_replica_context().merge_call(
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 883a22389a..aed2a413ae 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -71,11 +71,11 @@ class UpdateContext(object):
 
 
 def get_loss_reduction():
-  """Reduce `aggregation` corresponding to the last loss reduction."""
+  """Reduce op corresponding to the last loss reduction."""
   loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
   if loss_reduction == losses_impl.Reduction.SUM:
-    return variable_scope.VariableAggregation.SUM
-  return variable_scope.VariableAggregation.MEAN
+    return reduce_util.ReduceOp.SUM
+  return reduce_util.ReduceOp.MEAN
 
 
 # ------------------------------------------------------------------------------
diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index fc9eb479cc..957c8810ac 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
@@ -95,8 +96,7 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None):
       # In a replica context, we update variable using the mean of value across
       # replicas.
       def merge_fn(strategy, v, value):
-        value = strategy.reduce(
-            variable_scope.VariableAggregation.MEAN, value, v)
+        value = strategy.reduce(ds_reduce_util.ReduceOp.MEAN, value, v)
         return strategy.update(v, update_fn, value)
 
       return replica_context.merge_call(merge_fn, variable, value)
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 9dfa9d2afb..ada8a7d616 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -24,6 +24,7 @@ import abc
 
 import six
 
+from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -520,8 +521,7 @@ class Optimizer(
 
   @staticmethod
   def _scale_loss(loss_value):
-    if (distribute_lib.get_loss_reduction() ==
-        variable_scope.VariableAggregation.MEAN):
+    if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
       num_replicas = \
         distribute_ctx.get_distribution_strategy().num_replicas_in_sync
       if num_replicas > 1:
@@ -658,10 +658,10 @@ class Optimizer(
     Returns:
       An `Operation` that applies the specified gradients across all
       replicas. If `global_step` was not None, that operation also
-      increments `global_step`.
+      increments `global_step`
     """
     reduced_grads = distribution.batch_reduce(
-        variable_scope.VariableAggregation.SUM, grads_and_vars)
+        ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     var_list = [v for _, v in grads_and_vars]
     grads_and_vars = zip(reduced_grads, var_list)
     # Note that this is called in a cross-replica context.
-- 
GitLab


From 5b61eed60089e1b28093ecc180034bc7ca307e3a Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Nov 2018 20:23:35 -0800
Subject: [PATCH 0281/1554] [tf.data] Duplicate the internal `Dataset` class
 into `DatasetV1` and `DatasetV2`.

The implementation of `tf.data.Dataset` now depends on the version of
TensorFlow: in 1.x we export `DatasetV1` and in 2.x we export `DatasetV2`.
Currently, the internal `dataset_ops.Dataset` symbol maps to `DatasetV1`, but
this will change after all internal tests are updated to 2.x compatibility.

This change also removes the deprecated `Dataset.from_sparse_tensor_slices()`
method from `DatasetV2`, since its replacement has long been available in
`Dataset.from_tensor_slices()`.

PiperOrigin-RevId: 221560852
---
 .../contrib/distribute/python/input_ops.py    |   5 +
 .../contrib/distribute/python/tpu_strategy.py |   2 +
 tensorflow/contrib/eager/python/datasets.py   |   8 +-
 .../python/data/experimental/ops/counter.py   |  14 +-
 .../experimental/ops/indexed_dataset_ops.py   |   2 +
 .../data/experimental/ops/interleave_ops.py   |  28 ++-
 .../data/experimental/ops/random_ops.py       |  23 +-
 .../python/data/experimental/ops/readers.py   | 144 +++++++++--
 tensorflow/python/data/ops/dataset_ops.py     | 225 +++++++++++++++---
 tensorflow/python/data/ops/readers.py         | 112 +++++++--
 .../golden/v1/tensorflow.data.-dataset.pbtxt  |   3 +-
 ...ow.data.-fixed-length-record-dataset.pbtxt |   7 +-
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   6 +-
 .../tensorflow.data.-text-line-dataset.pbtxt  |   7 +-
 ...rflow.data.experimental.-csv-dataset.pbtxt |   7 +-
 ...ow.data.experimental.-random-dataset.pbtxt |   7 +-
 ...rflow.data.experimental.-sql-dataset.pbtxt |   7 +-
 .../v1/tensorflow.data.experimental.pbtxt     |   2 +-
 .../golden/v2/tensorflow.data.-dataset.pbtxt  |   7 +-
 ...ow.data.-fixed-length-record-dataset.pbtxt |   8 +-
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   8 +-
 .../tensorflow.data.-text-line-dataset.pbtxt  |   8 +-
 ...rflow.data.experimental.-csv-dataset.pbtxt |   8 +-
 ...ow.data.experimental.-random-dataset.pbtxt |   8 +-
 ...rflow.data.experimental.-sql-dataset.pbtxt |   8 +-
 .../v2/tensorflow.data.experimental.pbtxt     |   2 +-
 26 files changed, 526 insertions(+), 140 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/input_ops.py b/tensorflow/contrib/distribute/python/input_ops.py
index ac1ccd64b3..c40b2bf27a 100644
--- a/tensorflow/contrib/distribute/python/input_ops.py
+++ b/tensorflow/contrib/distribute/python/input_ops.py
@@ -102,6 +102,11 @@ def auto_shard_dataset(dataset, num_shards, index):
               dataset._input_dataset, found_reader_op)
           return dataset
 
+    if isinstance(dataset, dataset_ops.DatasetV1Adapter):
+      dataset._dataset = _auto_shard_impl(
+          dataset._dataset, found_reader_op)
+      return dataset
+
     # TODO(priyag): Make _input_dataset(s) a common property of all datasets to
     # make this check more robust.
     if hasattr(dataset, "_input_dataset"):
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index dcc2c64438..094a57c4e6 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -239,6 +239,8 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     def _get_dataset_batch_size(dataset):
       """Get the global batch size from the dataset object."""
       # pylint: disable=protected-access
+      if isinstance(dataset, dataset_ops.DatasetV1Adapter):
+        dataset = dataset._dataset
       if isinstance(dataset, dataset_ops.BatchDataset):
         return tensor_util.constant_value(dataset._batch_size)
       elif isinstance(dataset, batching._MapAndBatchDataset):
diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index 3aed121233..db77a39626 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import prefetching_ops
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -52,11 +53,16 @@ class Iterator(iterator_ops.EagerIterator):
       TypeError: If `dataset` is an unsupported type.
       RuntimeError: When invoked without eager execution enabled.
     """
-    if isinstance(dataset, prefetching_ops._PrefetchToDeviceDataset):  # pylint: disable=protected-access
+    # pylint: disable=protected-access
+    if (isinstance(dataset, prefetching_ops._PrefetchToDeviceDataset)
+        or (isinstance(dataset, dataset_ops.DatasetV1Adapter)
+            and isinstance(
+                dataset._dataset, prefetching_ops._PrefetchToDeviceDataset))):
       raise TypeError(
           "`tf.data.experimental.prefetch_to_device()` is not compatible with "
           "`tf.contrib.eager.Iterator`. Use `for ... in dataset:` to iterate "
           "over the dataset instead.")
+    # pylint: enable=protected-access
 
     if not context.context().device_spec.device_type:
       is_remote_device = False
diff --git a/tensorflow/python/data/experimental/ops/counter.py b/tensorflow/python/data/experimental/ops/counter.py
index 42200eaef9..652eb9d002 100644
--- a/tensorflow/python/data/experimental/ops/counter.py
+++ b/tensorflow/python/data/experimental/ops/counter.py
@@ -25,8 +25,8 @@ from tensorflow.python.framework import ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("data.experimental.Counter")
-def Counter(start=0, step=1, dtype=dtypes.int64):
+@tf_export("data.experimental.Counter", v1=[])
+def CounterV2(start=0, step=1, dtype=dtypes.int64):
   """Creates a `Dataset` that counts from `start` in steps of size `step`.
 
   For example:
@@ -53,3 +53,13 @@ def Counter(start=0, step=1, dtype=dtypes.int64):
     step = ops.convert_to_tensor(step, dtype=dtype, name="step")
     return dataset_ops.Dataset.from_tensors(0).repeat(None).apply(
         scan_ops.scan(start, lambda state, _: (state + step, state)))
+
+
+@tf_export(v1=["data.experimental.Counter"])
+def CounterV1(start=0, step=1, dtype=dtypes.int64):
+  return dataset_ops.DatasetV1Adapter(CounterV2(start, step, dtype))
+CounterV1.__doc__ = CounterV2.__doc__
+
+# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep
+# this alias in place.
+Counter = CounterV1  # pylint: disable=invalid-name
diff --git a/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py b/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py
index 9c06474a2f..570f0116f7 100644
--- a/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py
+++ b/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py
@@ -65,6 +65,7 @@ class MaterializedIndexedDataset(object):
             sparse.as_dense_types(self._output_shapes, self._output_classes)))
 
 
+# TODO(saeta): Add a `DatasetV1` wrapper if this is exposed via the public API.
 class IndexedDataset(dataset_ops.Dataset):
   """IndexedDataset is highly experimental!
   """
@@ -149,6 +150,7 @@ class IndexedDataset(dataset_ops.Dataset):
     raise NotImplementedError("IndexedDataset._as_variant_tensor")
 
 
+# TODO(saeta): Add a `DatasetV1` wrapper if this is exposed via the public API.
 class IdentityIndexedDataset(IndexedDataset):
   """IdentityIndexedDataset is a trivial indexed dataset used for testing.
   """
diff --git a/tensorflow/python/data/experimental/ops/interleave_ops.py b/tensorflow/python/data/experimental/ops/interleave_ops.py
index a3c094859e..5605e1d60e 100644
--- a/tensorflow/python/data/experimental/ops/interleave_ops.py
+++ b/tensorflow/python/data/experimental/ops/interleave_ops.py
@@ -133,8 +133,8 @@ class _DirectedInterleaveDataset(dataset_ops.Dataset):
     return self._data_inputs[0].output_types
 
 
-@tf_export("data.experimental.sample_from_datasets")
-def sample_from_datasets(datasets, weights=None, seed=None):
+@tf_export("data.experimental.sample_from_datasets", v1=[])
+def sample_from_datasets_v2(datasets, weights=None, seed=None):
   """Samples elements at random from the datasets in `datasets`.
 
   Args:
@@ -217,8 +217,15 @@ def sample_from_datasets(datasets, weights=None, seed=None):
   return _DirectedInterleaveDataset(selector_input, datasets)
 
 
-@tf_export("data.experimental.choose_from_datasets")
-def choose_from_datasets(datasets, choice_dataset):
+@tf_export(v1=["data.experimental.sample_from_datasets"])
+def sample_from_datasets_v1(datasets, weights=None, seed=None):
+  return dataset_ops.DatasetV1Adapter(
+      sample_from_datasets_v2(datasets, weights, seed))
+sample_from_datasets_v1.__doc__ = sample_from_datasets_v2.__doc__
+
+
+@tf_export("data.experimental.choose_from_datasets", v1=[])
+def choose_from_datasets_v2(datasets, choice_dataset):
   """Creates a dataset that deterministically chooses elements from `datasets`.
 
   For example, given the following datasets:
@@ -260,3 +267,16 @@ def choose_from_datasets(datasets, choice_dataset):
     raise TypeError("`choice_dataset` must be a dataset of scalar "
                     "`tf.int64` tensors.")
   return _DirectedInterleaveDataset(choice_dataset, datasets)
+
+
+@tf_export(v1=["data.experimental.choose_from_datasets"])
+def choose_from_datasets_v1(datasets, choice_dataset):
+  return dataset_ops.DatasetV1Adapter(
+      choose_from_datasets_v2(datasets, choice_dataset))
+choose_from_datasets_v1.__doc__ = choose_from_datasets_v2.__doc__
+
+
+# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep
+# these aliases in place.
+choose_from_datasets = choose_from_datasets_v1
+sample_from_datasets = sample_from_datasets_v1
diff --git a/tensorflow/python/data/experimental/ops/random_ops.py b/tensorflow/python/data/experimental/ops/random_ops.py
index e3a2aeab31..7bf703502b 100644
--- a/tensorflow/python/data/experimental/ops/random_ops.py
+++ b/tensorflow/python/data/experimental/ops/random_ops.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import random_seed
 from tensorflow.python.framework import dtypes
@@ -26,13 +28,13 @@ from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("data.experimental.RandomDataset")
-class RandomDataset(dataset_ops.DatasetSource):
+@tf_export("data.experimental.RandomDataset", v1=[])
+class RandomDatasetV2(dataset_ops.DatasetSource):
   """A `Dataset` of pseudorandom values."""
 
   def __init__(self, seed=None):
     """A `Dataset` of pseudorandom values."""
-    super(RandomDataset, self).__init__()
+    super(RandomDatasetV2, self).__init__()
     self._seed, self._seed2 = random_seed.get_seed(seed)
 
   def _as_variant_tensor(self):
@@ -52,3 +54,18 @@ class RandomDataset(dataset_ops.DatasetSource):
   @property
   def output_types(self):
     return dtypes.int64
+
+
+@tf_export(v1=["data.experimental.RandomDataset"])
+class RandomDatasetV1(dataset_ops.DatasetV1Adapter):
+  """A `Dataset` of pseudorandom values."""
+
+  @functools.wraps(RandomDatasetV2.__init__)
+  def __init__(self, seed=None):
+    wrapped = RandomDatasetV2(seed)
+    super(RandomDatasetV1, self).__init__(wrapped)
+
+
+# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep
+# this alias in place.
+RandomDataset = RandomDatasetV1
diff --git a/tensorflow/python/data/experimental/ops/readers.py b/tensorflow/python/data/experimental/ops/readers.py
index fe60192586..1ba26ed5b9 100644
--- a/tensorflow/python/data/experimental/ops/readers.py
+++ b/tensorflow/python/data/experimental/ops/readers.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import collections
 import csv
+import functools
 
 import numpy as np
 
@@ -307,8 +308,8 @@ def make_tf_record_dataset(file_pattern,
     return dataset.prefetch(buffer_size=prefetch_buffer_size)
 
 
-@tf_export("data.experimental.make_csv_dataset")
-def make_csv_dataset(
+@tf_export("data.experimental.make_csv_dataset", v1=[])
+def make_csv_dataset_v2(
     file_pattern,
     batch_size,
     column_names=None,
@@ -507,11 +508,42 @@ def make_csv_dataset(
   return dataset
 
 
+@tf_export(v1=["data.experimental.make_csv_dataset"])
+def make_csv_dataset_v1(
+    file_pattern,
+    batch_size,
+    column_names=None,
+    column_defaults=None,
+    label_name=None,
+    select_columns=None,
+    field_delim=",",
+    use_quote_delim=True,
+    na_value="",
+    header=True,
+    num_epochs=None,
+    shuffle=True,
+    shuffle_buffer_size=10000,
+    shuffle_seed=None,
+    prefetch_buffer_size=optimization.AUTOTUNE,
+    num_parallel_reads=1,
+    sloppy=False,
+    num_rows_for_inference=100,
+    compression_type=None,
+):  # pylint: disable=missing-docstring
+  return dataset_ops.DatasetV1Adapter(make_csv_dataset_v2(
+      file_pattern, batch_size, column_names, column_defaults, label_name,
+      select_columns, field_delim, use_quote_delim, na_value, header,
+      num_epochs, shuffle, shuffle_buffer_size, shuffle_seed,
+      prefetch_buffer_size, num_parallel_reads, sloppy, num_rows_for_inference,
+      compression_type))
+make_csv_dataset_v1.__doc__ = make_csv_dataset_v2.__doc__
+
+
 _DEFAULT_READER_BUFFER_SIZE_BYTES = 4 * 1024 * 1024  # 4 MB
 
 
-@tf_export("data.experimental.CsvDataset")
-class CsvDataset(dataset_ops.DatasetSource):
+@tf_export("data.experimental.CsvDataset", v1=[])
+class CsvDatasetV2(dataset_ops.DatasetSource):
   """A Dataset comprising lines from one or more CSV files."""
 
   def __init__(self,
@@ -594,7 +626,7 @@ class CsvDataset(dataset_ops.DatasetSource):
         the input data. If specified, only this subset of columns will be
         parsed. Defaults to parsing all columns.
     """
-    super(CsvDataset, self).__init__()
+    super(CsvDatasetV2, self).__init__()
     self._filenames = ops.convert_to_tensor(
         filenames, dtype=dtypes.string, name="filenames")
     self._compression_type = convert.optional_param_to_tensor(
@@ -658,22 +690,43 @@ class CsvDataset(dataset_ops.DatasetSource):
     return self._output_classes
 
 
-@tf_export("data.experimental.make_batched_features_dataset")
-def make_batched_features_dataset(file_pattern,
-                                  batch_size,
-                                  features,
-                                  reader=core_readers.TFRecordDataset,
-                                  label_key=None,
-                                  reader_args=None,
-                                  num_epochs=None,
-                                  shuffle=True,
-                                  shuffle_buffer_size=10000,
-                                  shuffle_seed=None,
-                                  prefetch_buffer_size=optimization.AUTOTUNE,
-                                  reader_num_threads=1,
-                                  parser_num_threads=2,
-                                  sloppy_ordering=False,
-                                  drop_final_batch=False):
+@tf_export(v1=["data.experimental.CsvDataset"])
+class CsvDatasetV1(dataset_ops.DatasetV1Adapter):
+  """A Dataset comprising lines from one or more CSV files."""
+
+  @functools.wraps(CsvDatasetV2.__init__)
+  def __init__(self,
+               filenames,
+               record_defaults,
+               compression_type=None,
+               buffer_size=None,
+               header=False,
+               field_delim=",",
+               use_quote_delim=True,
+               na_value="",
+               select_cols=None):
+    wrapped = CsvDatasetV2(filenames, record_defaults, compression_type,
+                           buffer_size, header, field_delim, use_quote_delim,
+                           na_value, select_cols)
+    super(CsvDatasetV1, self).__init__(wrapped)
+
+
+@tf_export("data.experimental.make_batched_features_dataset", v1=[])
+def make_batched_features_dataset_v2(file_pattern,
+                                     batch_size,
+                                     features,
+                                     reader=core_readers.TFRecordDataset,
+                                     label_key=None,
+                                     reader_args=None,
+                                     num_epochs=None,
+                                     shuffle=True,
+                                     shuffle_buffer_size=10000,
+                                     shuffle_seed=None,
+                                     prefetch_buffer_size=optimization.AUTOTUNE,
+                                     reader_num_threads=1,
+                                     parser_num_threads=2,
+                                     sloppy_ordering=False,
+                                     drop_final_batch=False):
   """Returns a `Dataset` of feature dictionaries from `Example` protos.
 
   If label_key argument is provided, returns a `Dataset` of tuple
@@ -819,6 +872,31 @@ def make_batched_features_dataset(file_pattern,
   return dataset
 
 
+@tf_export(v1=["data.experimental.make_batched_features_dataset"])
+def make_batched_features_dataset_v1(file_pattern,  # pylint: disable=missing-docstring
+                                     batch_size,
+                                     features,
+                                     reader=core_readers.TFRecordDataset,
+                                     label_key=None,
+                                     reader_args=None,
+                                     num_epochs=None,
+                                     shuffle=True,
+                                     shuffle_buffer_size=10000,
+                                     shuffle_seed=None,
+                                     prefetch_buffer_size=optimization.AUTOTUNE,
+                                     reader_num_threads=1,
+                                     parser_num_threads=2,
+                                     sloppy_ordering=False,
+                                     drop_final_batch=False):
+  return dataset_ops.DatasetV1Adapter(make_batched_features_dataset_v2(
+      file_pattern, batch_size, features, reader, label_key, reader_args,
+      num_epochs, shuffle, shuffle_buffer_size, shuffle_seed,
+      prefetch_buffer_size, reader_num_threads, parser_num_threads,
+      sloppy_ordering, drop_final_batch))
+make_batched_features_dataset_v2.__doc__ = (
+    make_batched_features_dataset_v1.__doc__)
+
+
 def _get_file_names(file_pattern, shuffle):
   """Parse list of file names from pattern, optionally shuffled.
 
@@ -850,8 +928,8 @@ def _get_file_names(file_pattern, shuffle):
   return file_names
 
 
-@tf_export("data.experimental.SqlDataset")
-class SqlDataset(dataset_ops.DatasetSource):
+@tf_export("data.experimental.SqlDataset", v1=[])
+class SqlDatasetV2(dataset_ops.DatasetSource):
   """A `Dataset` consisting of the results from a SQL query."""
 
   def __init__(self, driver_name, data_source_name, query, output_types):
@@ -883,7 +961,7 @@ class SqlDataset(dataset_ops.DatasetSource):
       output_types: A tuple of `tf.DType` objects representing the types of the
         columns returned by `query`.
     """
-    super(SqlDataset, self).__init__()
+    super(SqlDatasetV2, self).__init__()
     self._driver_name = ops.convert_to_tensor(
         driver_name, dtype=dtypes.string, name="driver_name")
     self._data_source_name = ops.convert_to_tensor(
@@ -910,3 +988,21 @@ class SqlDataset(dataset_ops.DatasetSource):
   @property
   def output_types(self):
     return self._output_types
+
+
+@tf_export(v1=["data.experimental.SqlDataset"])
+class SqlDatasetV1(dataset_ops.DatasetV1Adapter):
+  """A `Dataset` consisting of the results from a SQL query."""
+
+  @functools.wraps(SqlDatasetV2.__init__)
+  def __init__(self, driver_name, data_source_name, query, output_types):
+    wrapped = SqlDatasetV2(driver_name, data_source_name, query, output_types)
+    super(SqlDatasetV1, self).__init__(wrapped)
+
+
+# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep
+# these aliases in place.
+CsvDataset = CsvDatasetV1
+SqlDataset = SqlDatasetV1
+make_batched_features_dataset = make_batched_features_dataset_v1
+make_csv_dataset = make_csv_dataset_v1
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 3836a68e7d..e627cc3469 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import abc
+import functools
 import threading
 import warnings
 
@@ -52,19 +53,16 @@ from tensorflow.python.util import function_utils
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("data.Dataset")
-@six.add_metaclass(abc.ABCMeta)
-class Dataset(object):
+@tf_export("data.Dataset", v1=[])
+class DatasetV2(object):
   """Represents a potentially large set of elements.
 
   A `Dataset` can be used to represent an input pipeline as a
   collection of elements (nested structures of tensors) and a "logical
+
   plan" of transformations that act on those elements.
   """
 
-  def __init__(self):
-    pass
-
   def _as_serialized_graph(self):
     """Produces serialized graph representation of the dataset.
 
@@ -311,19 +309,6 @@ class Dataset(object):
     """
     return TensorSliceDataset(tensors)
 
-  @staticmethod
-  @deprecation.deprecated(None, "Use `tf.data.Dataset.from_tensor_slices()`.")
-  def from_sparse_tensor_slices(sparse_tensor):
-    """Splits each rank-N `tf.SparseTensor` in this dataset row-wise.
-
-    Args:
-      sparse_tensor: A `tf.SparseTensor`.
-
-    Returns:
-      Dataset: A `Dataset` of rank-(N-1) sparse tensors.
-    """
-    return SparseTensorSliceDataset(sparse_tensor)
-
   class _GeneratorState(object):
     """Stores outstanding iterators created from a Python generator.
 
@@ -435,7 +420,7 @@ class Dataset(object):
     flattened_types = [dtypes.as_dtype(dt) for dt in nest.flatten(output_types)]
     flattened_shapes = nest.flatten(output_shapes)
 
-    generator_state = Dataset._GeneratorState(generator)
+    generator_state = DatasetV2._GeneratorState(generator)
 
     def get_iterator_id_fn(unused_dummy):
       """Creates a unique `iterator_id` for each pass over the dataset.
@@ -1217,7 +1202,7 @@ class Dataset(object):
           dataset.
     """
     dataset = transformation_func(self)
-    if not isinstance(dataset, Dataset):
+    if not isinstance(dataset, DatasetV2):
       raise TypeError("`transformation_func` must return a Dataset.")
     dataset._input_datasets = [self]  # pylint: disable=protected-access
     return dataset
@@ -1401,6 +1386,188 @@ class Dataset(object):
     return _OptionsDataset(self, options)
 
 
+@tf_export(v1=["data.Dataset"])
+class DatasetV1(DatasetV2):
+  """Represents a potentially large set of elements.
+
+  A `Dataset` can be used to represent an input pipeline as a
+  collection of elements (nested structures of tensors) and a "logical
+  plan" of transformations that act on those elements.
+  """
+
+  def __init__(self):
+    pass
+
+  @staticmethod
+  @functools.wraps(DatasetV2.from_tensors)
+  def from_tensors(tensors):
+    return DatasetV1Adapter(DatasetV2.from_tensors(tensors))
+
+  @staticmethod
+  @functools.wraps(DatasetV2.from_tensor_slices)
+  def from_tensor_slices(tensors):
+    return DatasetV1Adapter(DatasetV2.from_tensor_slices(tensors))
+
+  @staticmethod
+  @deprecation.deprecated(None, "Use `tf.data.Dataset.from_tensor_slices()`.")
+  def from_sparse_tensor_slices(sparse_tensor):
+    """Splits each rank-N `tf.SparseTensor` in this dataset row-wise.
+
+    Args:
+      sparse_tensor: A `tf.SparseTensor`.
+
+    Returns:
+      Dataset: A `Dataset` of rank-(N-1) sparse tensors.
+    """
+    return DatasetV1Adapter(SparseTensorSliceDataset(sparse_tensor))
+
+  @staticmethod
+  @functools.wraps(DatasetV2.from_generator)
+  def from_generator(generator, output_types, output_shapes=None, args=None):
+    return DatasetV1Adapter(DatasetV2.from_generator(
+        generator, output_types, output_shapes, args))
+
+  @staticmethod
+  @functools.wraps(DatasetV2.range)
+  def range(*args):
+    return DatasetV1Adapter(DatasetV2.range(*args))
+
+  @staticmethod
+  @functools.wraps(DatasetV2.zip)
+  def zip(datasets):
+    return DatasetV1Adapter(DatasetV2.zip(datasets))
+
+  @functools.wraps(DatasetV2.concatenate)
+  def concatenate(self, dataset):
+    return DatasetV1Adapter(super(DatasetV1, self).concatenate(dataset))
+
+  @functools.wraps(DatasetV2.prefetch)
+  def prefetch(self, buffer_size):
+    return DatasetV1Adapter(super(DatasetV1, self).prefetch(buffer_size))
+
+  @staticmethod
+  @functools.wraps(DatasetV2.list_files)
+  def list_files(file_pattern, shuffle=None, seed=None):
+    return DatasetV1Adapter(DatasetV2.list_files(file_pattern, shuffle, seed))
+
+  @functools.wraps(DatasetV2.repeat)
+  def repeat(self, count=None):
+    return DatasetV1Adapter(super(DatasetV1, self).repeat(count))
+
+  @functools.wraps(DatasetV2.shuffle)
+  def shuffle(self, buffer_size, seed=None, reshuffle_each_iteration=None):
+    return DatasetV1Adapter(super(DatasetV1, self).shuffle(
+        buffer_size, seed, reshuffle_each_iteration))
+
+  @functools.wraps(DatasetV2.cache)
+  def cache(self, filename=""):
+    return DatasetV1Adapter(super(DatasetV1, self).cache(filename))
+
+  @functools.wraps(DatasetV2.take)
+  def take(self, count):
+    return DatasetV1Adapter(super(DatasetV1, self).take(count))
+
+  @functools.wraps(DatasetV2.skip)
+  def skip(self, count):
+    return DatasetV1Adapter(super(DatasetV1, self).skip(count))
+
+  @functools.wraps(DatasetV2.shard)
+  def shard(self, num_shards, index):
+    return DatasetV1Adapter(super(DatasetV1, self).shard(num_shards, index))
+
+  @functools.wraps(DatasetV2.batch)
+  def batch(self, batch_size, drop_remainder=False):
+    return DatasetV1Adapter(super(DatasetV1, self).batch(
+        batch_size, drop_remainder))
+
+  @functools.wraps(DatasetV2.padded_batch)
+  def padded_batch(self,
+                   batch_size,
+                   padded_shapes,
+                   padding_values=None,
+                   drop_remainder=False):
+    return DatasetV1Adapter(super(DatasetV1, self).padded_batch(
+        batch_size, padded_shapes, padding_values, drop_remainder))
+
+  @functools.wraps(DatasetV2.map)
+  def map(self, map_func, num_parallel_calls=None):
+    return DatasetV1Adapter(super(DatasetV1, self).map(
+        map_func, num_parallel_calls))
+
+  @functools.wraps(DatasetV2.flat_map)
+  def flat_map(self, map_func):
+    return DatasetV1Adapter(super(DatasetV1, self).flat_map(map_func))
+
+  @functools.wraps(DatasetV2.interleave)
+  def interleave(self,
+                 map_func,
+                 cycle_length,
+                 block_length=1,
+                 num_parallel_calls=None):
+    return DatasetV1Adapter(super(DatasetV1, self).interleave(
+        map_func, cycle_length, block_length, num_parallel_calls))
+
+  @functools.wraps(DatasetV2.filter)
+  def filter(self, predicate):
+    return DatasetV1Adapter(super(DatasetV1, self).filter(predicate))
+
+  @functools.wraps(DatasetV2.apply)
+  def apply(self, transformation_func):
+    return DatasetV1Adapter(super(DatasetV1, self).apply(transformation_func))
+
+  @functools.wraps(DatasetV2.window)
+  def window(self, size, shift=None, stride=1, drop_remainder=False):
+    return DatasetV1Adapter(super(DatasetV1, self).window(
+        size, shift, stride, drop_remainder))
+
+  @functools.wraps(DatasetV2.with_options)
+  def with_options(self, options):
+    return DatasetV1Adapter(super(DatasetV1, self).with_options(options))
+
+
+# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep
+# this alias in place.
+Dataset = DatasetV1
+
+
+class DatasetV1Adapter(DatasetV1):
+  """Wraps a V2 `Dataset` object in the `tf.compat.v1.data.Dataset` API."""
+
+  def __init__(self, dataset):
+    super(DatasetV1Adapter, self).__init__()
+    self._dataset = dataset
+
+  def _as_variant_tensor(self):
+    return self._dataset._as_variant_tensor()  # pylint: disable=protected-access
+
+  def _inputs(self):
+    return self._dataset._inputs()  # pylint: disable=protected-access
+
+  def options(self):
+    return self._dataset.options()
+
+  @property
+  def output_classes(self):
+    return self._dataset.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._dataset.output_shapes
+
+  @property
+  def output_types(self):
+    return self._dataset.output_types
+
+  def make_initializable_iterator(self, shared_name=None):
+    return self._dataset.make_initializable_iterator(shared_name)
+
+  def __iter__(self):
+    return iter(self._dataset)
+
+  def make_one_shot_iterator(self):
+    return self._dataset.make_one_shot_iterator()
+
+
 @tf_export("data.Options")
 class Options(object):
   """Represents options for tf.data.Dataset.
@@ -1546,14 +1713,14 @@ class Options(object):
     return result
 
 
-class DatasetSource(Dataset):
+class DatasetSource(DatasetV2):
   """Abstract class representing a dataset with no inputs."""
 
   def _inputs(self):
     return []
 
 
-class UnaryDataset(Dataset):
+class UnaryDataset(DatasetV2):
   """Abstract class representing a dataset with one input."""
 
   def __init__(self, input_dataset):
@@ -1740,7 +1907,7 @@ class _NestedDatasetComponent(object):
     return self._output_types
 
 
-class _VariantDataset(Dataset):
+class _VariantDataset(DatasetV2):
   """A Dataset wrapper around a `tf.variant`-typed function argument."""
 
   def __init__(self, dataset_variant, structure):
@@ -1897,7 +2064,7 @@ class StructuredFunctionWrapper(object):
           flat_classes.append(sparse_tensor_lib.SparseTensor)
           flat_shapes.append(t.get_shape())
           flat_types.append(t.dtype)
-        elif isinstance(t, Dataset):
+        elif isinstance(t, DatasetV2):
           if not self._nested_dataset_support:
             raise NotImplementedError(
                 "The %s transformation does not currently support nested "
@@ -2108,14 +2275,14 @@ class _GeneratorDataset(DatasetSource):
     return "Dataset.from_generator()"
 
 
-class ZipDataset(Dataset):
+class ZipDataset(DatasetV2):
   """A `Dataset` that zips its inputs together."""
 
   def __init__(self, datasets):
     """See `Dataset.zip()` for details."""
     super(ZipDataset, self).__init__()
     for ds in nest.flatten(datasets):
-      if not isinstance(ds, Dataset):
+      if not isinstance(ds, DatasetV2):
         if isinstance(ds, list):
           message = ("The argument to `Dataset.zip()` must be a nested "
                      "structure of `Dataset` objects. Nested structures do not "
@@ -2155,7 +2322,7 @@ class ZipDataset(Dataset):
         [ds.output_types for ds in nest.flatten(self._datasets)])
 
 
-class ConcatenateDataset(Dataset):
+class ConcatenateDataset(DatasetV2):
   """A `Dataset` that concatenates its input with given dataset."""
 
   def __init__(self, input_dataset, dataset_to_concatenate):
@@ -2737,7 +2904,7 @@ class MapDataset(UnaryDataset):
     return "Dataset.map()"
 
 
-class MatchingFilesDataset(Dataset):
+class MatchingFilesDataset(DatasetV2):
   """A `Dataset` that list the files according to the input patterns."""
 
   def __init__(self, patterns):
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index 7e165a052d..8a563122a0 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -32,8 +32,8 @@ from tensorflow.python.util.tf_export import tf_export
 _DEFAULT_READER_BUFFER_SIZE_BYTES = 256 * 1024  # 256 KB
 
 
-@tf_export("data.TextLineDataset")
-class TextLineDataset(dataset_ops.DatasetSource):
+@tf_export("data.TextLineDataset", v1=[])
+class TextLineDatasetV2(dataset_ops.DatasetSource):
   """A `Dataset` comprising lines from one or more text files."""
 
   def __init__(self, filenames, compression_type=None, buffer_size=None):
@@ -47,7 +47,7 @@ class TextLineDataset(dataset_ops.DatasetSource):
         to buffer. A value of 0 results in the default buffering values chosen
         based on the compression type.
     """
-    super(TextLineDataset, self).__init__()
+    super(TextLineDatasetV2, self).__init__()
     self._filenames = ops.convert_to_tensor(
         filenames, dtype=dtypes.string, name="filenames")
     self._compression_type = convert.optional_param_to_tensor(
@@ -75,6 +75,24 @@ class TextLineDataset(dataset_ops.DatasetSource):
     return dtypes.string
 
 
+@tf_export(v1=["data.TextLineDataset"])
+class TextLineDatasetV1(dataset_ops.DatasetV1Adapter):
+  """A `Dataset` comprising lines from one or more text files."""
+
+  def __init__(self, filenames, compression_type=None, buffer_size=None):
+    wrapped = TextLineDatasetV2(filenames, compression_type, buffer_size)
+    super(TextLineDatasetV1, self).__init__(wrapped)
+  __init__.__doc__ = TextLineDatasetV2.__init__.__doc__
+
+  @property
+  def _filenames(self):
+    return self._dataset._filenames  # pylint: disable=protected-access
+
+  @_filenames.setter
+  def _filenames(self, value):
+    self._dataset._filenames = value  # pylint: disable=protected-access
+
+
 class _TFRecordDataset(dataset_ops.DatasetSource):
   """A `Dataset` comprising records from one or more TFRecord files."""
 
@@ -156,8 +174,8 @@ class ParallelInterleaveDataset(dataset_ops.InterleaveDataset):
     return "tf.data.experimental.parallel_interleave()"
 
 
-@tf_export("data.TFRecordDataset")
-class TFRecordDataset(dataset_ops.Dataset):
+@tf_export("data.TFRecordDataset", v1=[])
+class TFRecordDatasetV2(dataset_ops.DatasetV2):
   """A `Dataset` comprising records from one or more TFRecord files."""
 
   def __init__(self, filenames, compression_type=None, buffer_size=None,
@@ -182,7 +200,7 @@ class TFRecordDataset(dataset_ops.Dataset):
       TypeError: If any argument does not have the expected type.
       ValueError: If any argument does not have the expected shape.
     """
-    super(TFRecordDataset, self).__init__()
+    super(TFRecordDatasetV2, self).__init__()
     if isinstance(filenames, dataset_ops.Dataset):
       if filenames.output_types != dtypes.string:
         raise TypeError(
@@ -194,7 +212,7 @@ class TFRecordDataset(dataset_ops.Dataset):
     else:
       filenames = ops.convert_to_tensor(filenames, dtype=dtypes.string)
       filenames = array_ops.reshape(filenames, [-1], name="flat_filenames")
-      filenames = dataset_ops.Dataset.from_tensor_slices(filenames)
+      filenames = dataset_ops.DatasetV2.from_tensor_slices(filenames)
 
     self._filenames = filenames
     self._compression_type = compression_type
@@ -217,10 +235,10 @@ class TFRecordDataset(dataset_ops.Dataset):
              compression_type=None,
              buffer_size=None,
              num_parallel_reads=None):
-    return TFRecordDataset(filenames or self._filenames,
-                           compression_type or self._compression_type,
-                           buffer_size or self._buffer_size,
-                           num_parallel_reads or self._num_parallel_reads)
+    return TFRecordDatasetV2(filenames or self._filenames,
+                             compression_type or self._compression_type,
+                             buffer_size or self._buffer_size,
+                             num_parallel_reads or self._num_parallel_reads)
 
   def _as_variant_tensor(self):
     return self._impl._as_variant_tensor()  # pylint: disable=protected-access
@@ -241,8 +259,40 @@ class TFRecordDataset(dataset_ops.Dataset):
     return self._impl.output_types
 
 
-@tf_export("data.FixedLengthRecordDataset")
-class FixedLengthRecordDataset(dataset_ops.DatasetSource):
+@tf_export(v1=["data.TFRecordDataset"])
+class TFRecordDatasetV1(dataset_ops.DatasetV1Adapter):
+  """A `Dataset` comprising records from one or more TFRecord files."""
+
+  def __init__(self, filenames, compression_type=None, buffer_size=None,
+               num_parallel_reads=None):
+    wrapped = TFRecordDatasetV2(
+        filenames, compression_type, buffer_size, num_parallel_reads)
+    super(TFRecordDatasetV1, self).__init__(wrapped)
+  __init__.__doc__ = TFRecordDatasetV2.__init__.__doc__
+
+  def _clone(self,
+             filenames=None,
+             compression_type=None,
+             buffer_size=None,
+             num_parallel_reads=None):
+    # pylint: disable=protected-access
+    return TFRecordDatasetV1(
+        filenames or self._dataset._filenames,
+        compression_type or self._dataset._compression_type,
+        buffer_size or self._dataset._buffer_size,
+        num_parallel_reads or self._dataset._num_parallel_reads)
+
+  @property
+  def _filenames(self):
+    return self._dataset._filenames  # pylint: disable=protected-access
+
+  @_filenames.setter
+  def _filenames(self, value):
+    self._dataset._filenames = value  # pylint: disable=protected-access
+
+
+@tf_export("data.FixedLengthRecordDataset", v1=[])
+class FixedLengthRecordDatasetV2(dataset_ops.DatasetSource):
   """A `Dataset` of fixed-length records from one or more binary files."""
 
   def __init__(self,
@@ -267,7 +317,7 @@ class FixedLengthRecordDataset(dataset_ops.DatasetSource):
       compression_type: (Optional.) A `tf.string` scalar evaluating to one of
         `""` (no compression), `"ZLIB"`, or `"GZIP"`.
     """
-    super(FixedLengthRecordDataset, self).__init__()
+    super(FixedLengthRecordDatasetV2, self).__init__()
     self._filenames = ops.convert_to_tensor(
         filenames, dtype=dtypes.string, name="filenames")
     self._record_bytes = ops.convert_to_tensor(
@@ -296,7 +346,6 @@ class FixedLengthRecordDataset(dataset_ops.DatasetSource):
           self._filenames, self._header_bytes, self._record_bytes,
           self._footer_bytes, self._buffer_size)
 
-
   @property
   def output_classes(self):
     return ops.Tensor
@@ -308,3 +357,36 @@ class FixedLengthRecordDataset(dataset_ops.DatasetSource):
   @property
   def output_types(self):
     return dtypes.string
+
+
+@tf_export(v1=["data.FixedLengthRecordDataset"])
+class FixedLengthRecordDatasetV1(dataset_ops.DatasetV1Adapter):
+  """A `Dataset` of fixed-length records from one or more binary files."""
+
+  def __init__(self,
+               filenames,
+               record_bytes,
+               header_bytes=None,
+               footer_bytes=None,
+               buffer_size=None,
+               compression_type=None):
+    wrapped = FixedLengthRecordDatasetV2(
+        filenames, record_bytes, header_bytes, footer_bytes, buffer_size,
+        compression_type)
+    super(FixedLengthRecordDatasetV1, self).__init__(wrapped)
+  __init__.__doc__ = FixedLengthRecordDatasetV2.__init__.__doc__
+
+  @property
+  def _filenames(self):
+    return self._dataset._filenames  # pylint: disable=protected-access
+
+  @_filenames.setter
+  def _filenames(self, value):
+    self._dataset._filenames = value  # pylint: disable=protected-access
+
+
+# TODO(b/119044825): Until all `tf.data` unit tests are converted to V2, keep
+# these aliases in place.
+FixedLengthRecordDataset = FixedLengthRecordDatasetV1
+TFRecordDataset = TFRecordDatasetV1
+TextLineDataset = TextLineDatasetV1
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
index 8b7f63e43e..f59082baeb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.data.Dataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
index 81358cecbc..d73168b070 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -1,8 +1,9 @@
 path: "tensorflow.data.FixedLengthRecordDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.readers.FixedLengthRecordDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.readers.FixedLengthRecordDatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
index 7b7a9ebaf0..51224cd6b4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -1,7 +1,9 @@
 path: "tensorflow.data.TFRecordDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.readers.TFRecordDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.readers.TFRecordDatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
index 1c305abf68..a10add1b7e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-text-line-dataset.pbtxt
@@ -1,8 +1,9 @@
 path: "tensorflow.data.TextLineDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.readers.TextLineDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.readers.TextLineDatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
index 2520e28a3c..71b597c19c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -1,8 +1,9 @@
 path: "tensorflow.data.experimental.CsvDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.CsvDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.CsvDatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
index 1dd53b1eab..20646e87b5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -1,8 +1,9 @@
 path: "tensorflow.data.experimental.RandomDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.experimental.ops.random_ops.RandomDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.random_ops.RandomDatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
index 8fdd9dc52e..86c5ff5b0b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -1,8 +1,9 @@
 path: "tensorflow.data.experimental.SqlDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.SqlDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.SqlDatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV1\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index 4c253bb8ad..244b24519c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -90,7 +90,7 @@ tf_module {
   }
   member_method {
     name: "make_batched_features_dataset"
-    argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"<class \'tensorflow.python.data.ops.readers.TFRecordDataset\'>\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], "
+    argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"<class \'tensorflow.python.data.ops.readers.TFRecordDatasetV1\'>\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], "
   }
   member_method {
     name: "make_csv_dataset"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
index 8b7f63e43e..9394f4b767 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.data.Dataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -16,7 +16,6 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "apply"
@@ -46,10 +45,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
index 81358cecbc..8c32c773b7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.data.FixedLengthRecordDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.readers.FixedLengthRecordDataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.readers.FixedLengthRecordDatasetV2\'>"
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -48,10 +48,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
index 7b7a9ebaf0..9f32bce109 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.data.TFRecordDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.readers.TFRecordDataset\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.readers.TFRecordDatasetV2\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -47,10 +47,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
index 1c305abf68..0eedfdbfe1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.data.TextLineDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.readers.TextLineDataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.readers.TextLineDatasetV2\'>"
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -48,10 +48,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
index 2520e28a3c..08214ec3cf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.data.experimental.CsvDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.CsvDataset\'>"
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.CsvDatasetV2\'>"
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -48,10 +48,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
index 1dd53b1eab..608253298e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.data.experimental.RandomDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.experimental.ops.random_ops.RandomDataset\'>"
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.random_ops.RandomDatasetV2\'>"
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -48,10 +48,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
index 8fdd9dc52e..3335eb1dc7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.data.experimental.SqlDataset"
 tf_class {
-  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.SqlDataset\'>"
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.readers.SqlDatasetV2\'>"
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetSource\'>"
-  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Dataset\'>"
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
@@ -48,10 +48,6 @@ tf_class {
     name: "from_generator"
     argspec: "args=[\'generator\', \'output_types\', \'output_shapes\', \'args\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "from_sparse_tensor_slices"
-    argspec: "args=[\'sparse_tensor\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "from_tensor_slices"
     argspec: "args=[\'tensors\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index 4c253bb8ad..244b24519c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -90,7 +90,7 @@ tf_module {
   }
   member_method {
     name: "make_batched_features_dataset"
-    argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"<class \'tensorflow.python.data.ops.readers.TFRecordDataset\'>\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], "
+    argspec: "args=[\'file_pattern\', \'batch_size\', \'features\', \'reader\', \'label_key\', \'reader_args\', \'num_epochs\', \'shuffle\', \'shuffle_buffer_size\', \'shuffle_seed\', \'prefetch_buffer_size\', \'reader_num_threads\', \'parser_num_threads\', \'sloppy_ordering\', \'drop_final_batch\'], varargs=None, keywords=None, defaults=[\"<class \'tensorflow.python.data.ops.readers.TFRecordDatasetV1\'>\", \'None\', \'None\', \'None\', \'True\', \'10000\', \'None\', \'-1\', \'1\', \'2\', \'False\', \'False\'], "
   }
   member_method {
     name: "make_csv_dataset"
-- 
GitLab


From bf705e98d1cec46211b13c190c6bdf4f9f221b88 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Nov 2018 20:47:37 -0800
Subject: [PATCH 0282/1554] Avoid using offsetof in `ValueAndTensorBuffer`
 because it raises a warning before C++17.

Also define a class-specific placement deallocation function for `ValueAndTensorBuffer`, because some compilers warn if it is not present.

PiperOrigin-RevId: 221562487
---
 tensorflow/core/framework/tensor.h | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index eff1dc3736..797ce5f0be 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_FRAMEWORK_TENSOR_H_
 
 #include <cstdint>
+#include <type_traits>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -893,8 +894,25 @@ Tensor::Tensor(T value, host_scalar_tag tag) {
       // `core::Refcounted::Unref()` for an object of this type will free
       // the enclosing `ValueAndTensorBuffer` for the tensor buffer.
       static void operator delete(void* ptr) {
-        port::AlignedFree(static_cast<char*>(ptr) -
-                          offsetof(ValueAndTensorBuffer, tensor_buffer));
+        // Use a dummy object to compute to offset of
+        // `ValueAndTensorBuffer::tensor_buffer`, because `offsetof()` is not
+        // necessarily defined on this non-POD type (until C++17).
+        typename std::aligned_storage<sizeof(ValueAndTensorBuffer),
+                                      alignof(ValueAndTensorBuffer)>::type
+            dummy_storage_;
+        ValueAndTensorBuffer* dummy_object =
+            reinterpret_cast<ValueAndTensorBuffer*>(&dummy_storage_);
+        intptr_t offset =
+            reinterpret_cast<intptr_t>(&dummy_object->tensor_buffer) -
+            reinterpret_cast<intptr_t>(dummy_object);
+
+        port::AlignedFree(static_cast<char*>(ptr) - offset);
+      }
+
+      static void operator delete(void*, void*) {
+        // Some compilers require an overridden class-specific deallocation
+        // function, which will be called if placement `new` throws an
+        // exception.
       }
 
      private:
-- 
GitLab


From 6f0f058bced3bd07e3df33ed66163715db28757d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Nov 2018 21:03:49 -0800
Subject: [PATCH 0283/1554] Automated rollback of commit
 1c2d515477ada15b363d8c210d668dd20784d392

PiperOrigin-RevId: 221563772
---
 tensorflow/core/graph/testlib.cc              |  11 -
 tensorflow/core/graph/testlib.h               |   4 -
 tensorflow/core/kernels/BUILD                 |  25 +-
 tensorflow/core/kernels/scan_ops_gpu.cu.cc    | 263 +-----------------
 tensorflow/core/kernels/scan_ops_test.cc      | 146 ----------
 .../core/util/permutation_input_iterator.h    |   6 +-
 .../core/util/permutation_output_iterator.h   | 129 ---------
 .../python/kernel_tests/conv_ops_test.py      |   2 +-
 .../python/kernel_tests/scan_ops_test.py      |   5 -
 9 files changed, 6 insertions(+), 585 deletions(-)
 delete mode 100644 tensorflow/core/kernels/scan_ops_test.cc
 delete mode 100644 tensorflow/core/util/permutation_output_iterator.h

diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc
index 0e74a30c7a..0a38aa1c91 100644
--- a/tensorflow/core/graph/testlib.cc
+++ b/tensorflow/core/graph/testlib.cc
@@ -123,17 +123,6 @@ Node* Assign(Graph* g, Node* var, Node* val) {
   return ret;
 }
 
-Node* Cumsum(Graph* g, Node* data, Node* axes, bool exclusive, bool reverse) {
-  Node* ret;
-  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Cumsum")
-                  .Input(data)
-                  .Input(axes)
-                  .Attr("exclusive", exclusive)
-                  .Attr("reverse", reverse)
-                  .Finalize(g, &ret));
-  return ret;
-}
-
 Node* Reduce(Graph* g, const string& reduce, Node* data, Node* axes,
              bool keep_dims) {
   Node* ret;
diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h
index 0c7233161f..b00196f587 100644
--- a/tensorflow/core/graph/testlib.h
+++ b/tensorflow/core/graph/testlib.h
@@ -68,10 +68,6 @@ Node* Recv(Graph* g, const string& tensor, const string& type,
            const string& sender, const uint64 sender_incarnation,
            const string& receiver);
 
-// Adds a cumsum "node" in "g" doing cumsum(data, axes).
-Node* Cumsum(Graph* g, Node* data, Node* axes, bool exclusive = false,
-             bool reverse = false);
-
 // Adds a reduction "node" in "g" doing sum(data, axes).  "reduce" is
 // a reduction, e.g., Sum, Max, Min, Mean, etc.
 Node* Reduce(Graph* g, const string& reduce, Node* data, Node* axes,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 9065989657..5fd0f5d5d1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3197,7 +3197,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "scan_ops",
     prefix = "scan_ops",
-    deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]),
+    deps = MATH_DEPS,
 )
 
 tf_kernel_library(
@@ -3375,29 +3375,6 @@ tf_cuda_cc_test(
     ],
 )
 
-tf_cuda_cc_test(
-    name = "scan_ops_test",
-    size = "small",
-    srcs = ["scan_ops_test.cc"],
-    linkopts = select({
-        "//tensorflow:darwin": ["-headerpad_max_install_names"],
-        "//conditions:default": [],
-    }),
-    deps = [
-        ":host_constant_op",
-        ":ops_testutil",
-        ":ops_util",
-        ":scan_ops",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
 tf_cuda_cc_test(
     name = "reduction_ops_test",
     size = "small",
diff --git a/tensorflow/core/kernels/scan_ops_gpu.cu.cc b/tensorflow/core/kernels/scan_ops_gpu.cu.cc
index 83f6cfd8c4..ed6c6affce 100644
--- a/tensorflow/core/kernels/scan_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/scan_ops_gpu.cu.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,18 +16,9 @@ limitations under the License.
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
-#define CUB_USE_COOPERATIVE_GROUPS
 
-#include "third_party/cub/block/block_load.cuh"
-#include "third_party/cub/block/block_scan.cuh"
-#include "third_party/cub/block/block_store.cuh"
-#include "third_party/cub/iterator/counting_input_iterator.cuh"
-#include "third_party/cub/iterator/transform_input_iterator.cuh"
-#include "cuda/include/cuComplex.h"
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/util/permutation_input_iterator.h"
-#include "tensorflow/core/util/permutation_output_iterator.h"
 
 #include "tensorflow/core/kernels/scan_ops.h"
 
@@ -36,258 +27,6 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 typedef Eigen::Index Index;
 
-namespace functor {
-
-// Map a contiguous range to the actual memory locations depending on which
-// axis the scan is taking place over and whether or not reversed.
-struct MapIndexToLocation {
-  __host__ __device__ MapIndexToLocation(int dimx, int dimy, int dimz,
-                                         bool reverse = false)
-      : dimx_(dimx), dimy_(dimy), dimz_(dimz), reverse_(reverse) {}
-
-  __host__ __device__ int operator()(int id) const {
-    if (dimx_ == 1) {
-      int row = id % dimy_;
-      int col = id / dimy_;
-
-      if (reverse_) return (dimy_ - row - 1) * dimz_ + col;
-
-      return row * dimz_ + col;
-    } else if (dimz_ == 1) {
-      if (reverse_) {
-        int row = id / dimy_;
-        int col = id % dimy_;
-        return row * dimy_ + (dimy_ - col - 1);
-      }
-      return id;
-    } else {
-      int col = id % dimy_;
-      int tmp = id / dimy_;
-
-      int row1 = id / (dimy_ * dimz_);
-      int col1 = tmp % dimz_;
-
-      if (reverse_)
-        return row1 * dimy_ * dimz_ + (dimy_ - col - 1) * dimz_ + col1;
-
-      return row1 * dimy_ * dimz_ + col * dimz_ + col1;
-    }
-  }
-
-  int dimx_;
-  int dimy_;
-  int dimz_;
-  bool reverse_;
-};
-
-template <typename T, typename Op>
-struct BlockPrefixCallbackOp {
-  // Running prefix
-  T running_total_;
-  Op op_;
-
-  __device__ BlockPrefixCallbackOp(T running_total, Op op)
-      : running_total_(running_total), op_(op) {}
-
-  // Callback operator to be entered by the first warp of threads in the block.
-  // tid 0 is responsible for returning a value for seeding the block-wide scan.
-  __device__ T operator()(T block_aggregate) {
-    T old_prefix = running_total_;
-    running_total_ = op_(old_prefix, block_aggregate);
-    return old_prefix;
-  }
-};
-
-template <typename T>
-struct Sum {
-  __host__ __device__ T operator()(const T& a, const T& b) const {
-    return a + b;
-  }
-};
-
-template <typename T>
-struct Prod {
-  __host__ __device__ T operator()(const T& a, const T& b) const {
-    return a * b;
-  }
-};
-
-template <typename T, typename Op>
-struct IsSum {
-  constexpr static bool value =
-      (std::is_same<Op, Sum<T>>::value ||
-       std::is_same<Op, Eigen::internal::SumReducer<T>>::value);
-};
-
-template <typename T, typename Op>
-struct IsProd {
-  constexpr static bool value =
-      (std::is_same<Op, Prod<T>>::value ||
-       std::is_same<Op, Eigen::internal::ProdReducer<T>>::value);
-};
-
-template <typename T, typename Op>
-struct IdentityValue {
-  static_assert(IsSum<T, Op>::value || IsProd<T, Op>::value,
-                "IdentityValue not yet defined for this type.");
-
-  template <typename U = T, typename OpCopy = Op>
-  __host__ __device__ U operator()(
-      typename std::enable_if<IsSum<U, OpCopy>::value, U>::type t = U(0)) {
-    return t;
-  }
-
-  template <typename U = T, typename OpCopy = Op>
-  __host__ __device__ U operator()(
-      typename std::enable_if<IsProd<U, OpCopy>::value, U>::type t = U(1)) {
-    return t;
-  }
-};
-
-// Each block is mapped to one sequence.  A contiguous range is mapped to the
-// appropriate locations in memory by the permutation iterators.  This is
-// ideal for 1-D and row based scans.  Column scans would be better if they
-// did a block load and then locally transposed.  CUB's device wide scan is not
-// used in the large 1D case, even though it would be more efficient, because
-// it is not deterministic.
-template <typename T, typename Op, int BlockDim = 128, int ItemsPerThread = 4>
-__global__ void scan_kernel(const T* in, T* out, int dimx, int dimy, int dimz,
-                            bool exclusive, bool reverse, Op op) {
-  typedef cub::BlockLoad<T, BlockDim, ItemsPerThread, cub::BLOCK_LOAD_TRANSPOSE>
-      BlockLoad;
-  typedef cub::BlockStore<T, BlockDim, ItemsPerThread,
-                          cub::BLOCK_STORE_TRANSPOSE>
-      BlockStore;
-  typedef cub::BlockScan<T, BlockDim> BlockScan;
-
-  // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
-  __shared__ union {
-    typename BlockLoad::TempStorage load;
-    typename BlockScan::TempStorage scan;
-    typename BlockStore::TempStorage store;
-  } temp_storage;
-
-  int problem_length = dimy;
-
-  // Initialize running total
-  BlockPrefixCallbackOp<T, Op> prefix_op(IdentityValue<T, Op>()(), op);
-
-  MapIndexToLocation map_op(dimx, dimy, dimz, reverse);
-  int block_start = problem_length * blockIdx.x;
-  // Have the block iterate over segments of items
-  for (int block_offset = block_start;
-       block_offset < block_start + problem_length;
-       block_offset += BlockDim * ItemsPerThread) {
-    int valid_items = min(BlockDim * ItemsPerThread,
-                          problem_length - (block_offset % problem_length));
-
-    // first construct a counting iterator that has the desired start point
-    typedef cub::TransformInputIterator<int, MapIndexToLocation,
-                                        cub::CountingInputIterator<int>>
-        MapIterType;
-
-    cub::CountingInputIterator<int> counting_iter(block_offset);
-
-    // Next map the iterator to the actual locations in memory
-    MapIterType map_iter(counting_iter, map_op);
-
-    PermutationInputIterator<T, const T*, MapIterType> permutein_iter(in,
-                                                                      map_iter);
-    PermutationOutputIterator<T, T*, MapIterType> permuteout_iter(out,
-                                                                  map_iter);
-
-    // Load a segment of consecutive items that are blocked across threads
-    T thread_data[ItemsPerThread];
-    BlockLoad(temp_storage.load).Load(permutein_iter, thread_data, valid_items);
-    __syncthreads();
-
-    // Collectively compute the block-wide scan
-    if (exclusive) {
-      BlockScan(temp_storage.scan)
-          .ExclusiveScan(thread_data, thread_data, op, prefix_op);
-    } else {
-      BlockScan(temp_storage.scan)
-          .InclusiveScan(thread_data, thread_data, op, prefix_op);
-    }
-    __syncthreads();
-
-    // Store scanned items to output segment
-    BlockStore(temp_storage.store)
-        .Store(permuteout_iter, thread_data, valid_items);
-    __syncthreads();
-  }
-}
-
-template <typename T, typename Op>
-void LaunchScan(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
-                typename TTypes<T, 3>::Tensor out, Op op, const bool reverse,
-                const bool exclusive) {
-  const int items_per_thread = 4;
-
-  int dimx = in.dimension(0);
-  int dimy = in.dimension(1);
-  int dimz = in.dimension(2);
-  int num_blocks = dimx * dimz;
-
-  int ideal_block_size = dimy / items_per_thread;
-
-  // There seems to be a bug when the type is not float and block_size 1024.
-  // Launch on the smallest power of 2 block size that we can.
-  if (ideal_block_size >= 1024 && std::is_same<T, float>::value) {
-    const int block_size = 1024;
-    scan_kernel<T, Op, block_size, items_per_thread>
-        <<<num_blocks, block_size, 0, d.stream()>>>(
-            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
-  } else if (ideal_block_size >= 512) {
-    const int block_size = 512;
-    scan_kernel<T, Op, block_size, items_per_thread>
-        <<<num_blocks, block_size, 0, d.stream()>>>(
-            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
-  } else if (ideal_block_size >= 256) {
-    const int block_size = 256;
-    scan_kernel<T, Op, block_size, items_per_thread>
-        <<<num_blocks, block_size, 0, d.stream()>>>(
-            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
-  } else if (ideal_block_size >= 128) {
-    const int block_size = 128;
-    scan_kernel<T, Op, block_size, items_per_thread>
-        <<<num_blocks, block_size, 0, d.stream()>>>(
-            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
-  } else if (ideal_block_size >= 64) {
-    const int block_size = 64;
-    scan_kernel<T, Op, block_size, items_per_thread>
-        <<<num_blocks, block_size, 0, d.stream()>>>(
-            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
-  } else {
-    const int block_size = 32;
-    scan_kernel<T, Op, block_size, items_per_thread>
-        <<<num_blocks, block_size, 0, d.stream()>>>(
-            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
-  }
-}
-
-template <typename T>
-struct Scan<GPUDevice, Eigen::internal::SumReducer<T>, T> {
-  void operator()(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
-                  typename TTypes<T, 3>::Tensor out,
-                  const Eigen::internal::SumReducer<T>& reducer,
-                  const bool reverse, const bool exclusive) {
-    LaunchScan<T, Sum<T>>(d, in, out, Sum<T>(), reverse, exclusive);
-  }
-};
-
-template <typename T>
-struct Scan<GPUDevice, Eigen::internal::ProdReducer<T>, T> {
-  void operator()(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
-                  typename TTypes<T, 3>::Tensor out,
-                  const Eigen::internal::ProdReducer<T>& reducer,
-                  const bool reverse, const bool exclusive) {
-    LaunchScan<T, Prod<T>>(d, in, out, Prod<T>(), reverse, exclusive);
-  }
-};
-
-}  // namespace functor
-
 #define DEFINE(REDUCER, T) template struct functor::Scan<GPUDevice, REDUCER, T>;
 
 #define DEFINE_FOR_ALL_REDUCERS(T)           \
diff --git a/tensorflow/core/kernels/scan_ops_test.cc b/tensorflow/core/kernels/scan_ops_test.cc
deleted file mode 100644
index 588b606a99..0000000000
--- a/tensorflow/core/kernels/scan_ops_test.cc
+++ /dev/null
@@ -1,146 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/platform/test.h"
-#include "tensorflow/core/platform/test_benchmark.h"
-
-namespace tensorflow {
-
-template <typename T>
-static Graph* LargeOneDCumsum(int num_x, bool reverse = false) {
-  auto* g = new Graph(OpRegistry::Global());
-  Tensor data(DataTypeToEnum<T>::value, TensorShape({num_x}));
-  data.flat<T>().setRandom();
-  Tensor axes(DT_INT32, TensorShape({}));
-  axes.flat<int32>()(0) = 0;
-  test::graph::Cumsum(g, test::graph::Constant(g, data),
-                      test::graph::Constant(g, axes));
-  return g;
-}
-
-static Graph* ColCumsum(int num_x, int num_y, bool reverse = false) {
-  auto* g = new Graph(OpRegistry::Global());
-  Tensor data(DT_FLOAT, TensorShape({num_x, num_y}));
-  data.flat<float>().setRandom();
-  Tensor axes(DT_INT32, TensorShape({}));
-  axes.flat<int32>()(0) = 0;
-  test::graph::Cumsum(g, test::graph::Constant(g, data),
-                      test::graph::Constant(g, axes));
-  return g;
-}
-
-static Graph* RowCumsum(int num_x, int num_y, bool reverse = false) {
-  auto* g = new Graph(OpRegistry::Global());
-  Tensor data(DT_FLOAT, TensorShape({num_x, num_y}));
-  data.flat<float>().setRandom();
-  Tensor axes(DT_INT32, TensorShape({}));
-  axes.flat<int32>()(0) = 1;
-  test::graph::Cumsum(g, test::graph::Constant(g, data),
-                      test::graph::Constant(g, axes));
-  return g;
-}
-
-static Graph* ThreeDYCumsum(int num_y, int num_z, bool reverse = false) {
-  auto* g = new Graph(OpRegistry::Global());
-  Tensor data(DT_FLOAT, TensorShape({32, num_y, num_z}));
-  data.flat<float>().setRandom();
-  Tensor axes(DT_INT32, TensorShape({}));
-  axes.flat<int32>()(0) = 1;
-  test::graph::Cumsum(g, test::graph::Constant(g, data),
-                      test::graph::Constant(g, axes));
-  return g;
-}
-
-template <typename T>
-static void LargeOneDimensional(int iters, const string& device, int num_x,
-                                bool reverse = false) {
-  testing::ItemsProcessed(static_cast<int64>(iters) * num_x);
-  testing::BytesProcessed(static_cast<int64>(iters) * num_x * sizeof(T));
-  test::Benchmark(device, LargeOneDCumsum<T>(num_x, reverse)).Run(iters);
-}
-
-static void DoRowCumsum(int iters, const string& device, int num_x, int num_y,
-                        bool reverse = false) {
-  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
-  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
-                          sizeof(float));
-  test::Benchmark(device, RowCumsum(num_x, num_y, reverse)).Run(iters);
-}
-
-static void DoColCumsum(int iters, const string& device, int num_x, int num_y,
-                        bool reverse = false) {
-  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
-  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
-                          sizeof(float));
-  test::Benchmark(device, ColCumsum(num_x, num_y, reverse)).Run(iters);
-}
-
-static void Do3DYCumsum(int iters, const string& device, int num_x, int num_y,
-                        bool reverse = false) {
-  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
-  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
-                          sizeof(float));
-  test::Benchmark(device, ThreeDYCumsum(num_x, num_y, reverse)).Run(iters);
-}
-
-static void BM_OneDCumsumGPU(int iters, int num_x) {
-  LargeOneDimensional<float>(iters, "gpu", num_x);
-}
-BENCHMARK(BM_OneDCumsumGPU)->Range(1, 1 << 21);
-
-static void BM_OneDCumsumGPUHalf(int iters, int num_x) {
-  LargeOneDimensional<Eigen::half>(iters, "gpu", num_x);
-}
-BENCHMARK(BM_OneDCumsumGPUHalf)->Range(1, 1 << 21);
-
-static void BM_Sum2DRowCumsumGPU(int iters, int num_x, int num_y) {
-  DoRowCumsum(iters, "gpu", num_x, num_y);
-}
-BENCHMARK(BM_Sum2DRowCumsumGPU)->RangePair(1, 8192, 1, 8192);
-
-static void BM_Sum2DColumnCumsumGPU(int iters, int num_x, int num_y) {
-  DoColCumsum(iters, "gpu", num_x, num_y);
-}
-BENCHMARK(BM_Sum2DColumnCumsumGPU)->RangePair(1, 8192, 1, 8192);
-
-static void BM_Sum3DYCumsumGPU(int iters, int num_x, int num_y) {
-  Do3DYCumsum(iters, "gpu", num_x, num_y);
-}
-BENCHMARK(BM_Sum3DYCumsumGPU)->RangePair(64, 4096, 64, 4096);
-
-static void BM_OneDCumsumGPU_reverse(int iters, int num_x) {
-  LargeOneDimensional<float>(iters, "gpu", num_x, true);
-}
-BENCHMARK(BM_OneDCumsumGPU_reverse)->Range(1, 1 << 21);
-
-static void BM_Sum2DRowCumsumGPU_reverse(int iters, int num_x, int num_y) {
-  DoRowCumsum(iters, "gpu", num_x, num_y, true);
-}
-BENCHMARK(BM_Sum2DRowCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192);
-
-static void BM_Sum2DColumnCumsumGPU_reverse(int iters, int num_x, int num_y) {
-  DoColCumsum(iters, "gpu", num_x, num_y, true);
-}
-BENCHMARK(BM_Sum2DColumnCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192);
-
-static void BM_Sum3DYCumsumGPU_reverse(int iters, int num_x, int num_y) {
-  Do3DYCumsum(iters, "gpu", num_x, num_y, true);
-}
-BENCHMARK(BM_Sum3DYCumsumGPU_reverse)->RangePair(32, 2048, 32, 2048);
-
-}  // end namespace tensorflow
diff --git a/tensorflow/core/util/permutation_input_iterator.h b/tensorflow/core/util/permutation_input_iterator.h
index 649318ebf3..f6375b2515 100644
--- a/tensorflow/core/util/permutation_input_iterator.h
+++ b/tensorflow/core/util/permutation_input_iterator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
-#define TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#ifndef TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#define TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
 
 #include <iostream>
 #include <iterator>
@@ -131,4 +131,4 @@ class PermutationInputIterator {
 
 }  // end namespace tensorflow
 
-#endif  // TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#endif  // TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
diff --git a/tensorflow/core/util/permutation_output_iterator.h b/tensorflow/core/util/permutation_output_iterator.h
deleted file mode 100644
index 638c0f4545..0000000000
--- a/tensorflow/core/util/permutation_output_iterator.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
-#define TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
-
-#include <iostream>
-#include <iterator>
-
-namespace tensorflow {
-
-template <typename ValueType, typename OutputIteratorT, typename IndexIteratorT,
-          typename OffsetT = ptrdiff_t>
-class PermutationOutputIterator {
- public:
-  // Required iterator traits
-  typedef PermutationOutputIterator self_type;  ///< My own type
-  typedef OffsetT difference_type;  ///< Type to express the result of
-                                    ///< subtracting one iterator from another
-  typedef ValueType
-      value_type;  ///< The type of the element the iterator can point to
-  typedef ValueType* pointer;    ///< The type of a pointer to an element the
-                                 ///< iterator can point to
-  typedef ValueType& reference;  ///< The type of a reference to an element the
-                                 ///< iterator can point to
-
-  typedef std::random_access_iterator_tag
-      iterator_category;  ///< The iterator category
-
- private:
-  OutputIteratorT output_itr;
-  IndexIteratorT index_itr;
-
- public:
-  /// Constructor
-  __host__ __device__ __forceinline__ PermutationOutputIterator(
-      OutputIteratorT output_itr,  ///< Input iterator to wrap
-      IndexIteratorT index_itr)    ///< Conversion functor to wrap
-      : output_itr(output_itr), index_itr(index_itr) {}
-
-  /// Postfix increment
-  __host__ __device__ __forceinline__ self_type operator++(int) {
-    self_type retval = *this;
-    index_itr++;
-    return retval;
-  }
-
-  /// Prefix increment
-  __host__ __device__ __forceinline__ self_type operator++() {
-    index_itr++;
-    return *this;
-  }
-
-  /// Indirection
-  __host__ __device__ __forceinline__ reference operator*() const {
-    return output_itr[*index_itr];
-  }
-
-  /// Addition
-  template <typename Distance>
-  __host__ __device__ __forceinline__ self_type operator+(Distance n) const {
-    self_type retval(output_itr, index_itr + n);
-    return retval;
-  }
-
-  /// Addition assignment
-  template <typename Distance>
-  __host__ __device__ __forceinline__ self_type& operator+=(Distance n) {
-    index_itr += n;
-    return *this;
-  }
-
-  /// Subtraction
-  template <typename Distance>
-  __host__ __device__ __forceinline__ self_type operator-(Distance n) const {
-    self_type retval(output_itr, index_itr - n);
-    return retval;
-  }
-
-  /// Subtraction assignment
-  template <typename Distance>
-  __host__ __device__ __forceinline__ self_type& operator-=(Distance n) {
-    index_itr -= n;
-    return *this;
-  }
-
-  /// Distance
-  __host__ __device__ __forceinline__ difference_type
-  operator-(self_type other) const {
-    return index_itr - other.index_itr;
-  }
-
-  /// Array subscript
-  template <typename Distance>
-  __host__ __device__ __forceinline__ reference operator[](Distance n) const {
-    return output_itr[index_itr[n]];
-  }
-
-  /// Equal to
-  __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) {
-    return (index_itr == rhs.index_itr && output_itr == rhs.output_itr);
-  }
-
-  /// Not equal to
-  __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) {
-    return !(*this == rhs);
-  }
-
-  /// ostream operator
-  friend std::ostream& operator<<(std::ostream& os, const self_type& itr) {
-    return os;
-  }
-};
-
-}  // end namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 835cc1504d..0ccbbf155c 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -1669,7 +1669,7 @@ class SeparableConv2DTest(test.TestCase):
 
       value = sess.run(conv)
     tf_logging.info("value = ", value)
-    self.assertArrayNear(expected, np.ravel(value), 1e-3)
+    self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
 
   def _testSeparableConv2D(self, data_format):
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index c48e0e2e67..b369222565 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -126,11 +126,6 @@ class CumsumTest(test.TestCase):
       for axis in range(-6, 6, 3):
         self._compareAll(x, axis)
 
-  def testLarge(self):
-    for dtype in self.valid_dtypes:
-      x = np.ones([1000000], dtype=dtype) / 1024
-      self._compareAll(x, 0)
-
   def testInvalidAxis(self):
     x = np.arange(0, 10).reshape([2, 5]).astype(np.float32)
     input_tensor = ops.convert_to_tensor(x)
-- 
GitLab


From 8dd83e34a00d051bb444b0474096d4f468a13663 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Nov 2018 22:22:21 -0800
Subject: [PATCH 0284/1554] [tf.data] Add `UnaryUnchangedStructureDataset`.

This reduces the structure-handling boilerplate in `Dataset` implementations that have matching input and output structure.

PiperOrigin-RevId: 221569355
---
 .../python/data/experimental/ops/error_ops.py |  14 +-
 .../data/experimental/ops/optimization.py     |  28 +--
 .../data/experimental/ops/prefetching_ops.py  |  28 +--
 .../data/experimental/ops/shuffle_ops.py      |  14 +-
 .../python/data/experimental/ops/sleep.py     |   2 +-
 .../python/data/experimental/ops/stats_ops.py |  14 +-
 .../data/experimental/ops/threadpool.py       |  14 +-
 .../python/data/experimental/ops/unique.py    |  14 +-
 tensorflow/python/data/ops/dataset_ops.py     | 175 +++---------------
 9 files changed, 38 insertions(+), 265 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/error_ops.py b/tensorflow/python/data/experimental/ops/error_ops.py
index 82e274b70c..879b13ce09 100644
--- a/tensorflow/python/data/experimental/ops/error_ops.py
+++ b/tensorflow/python/data/experimental/ops/error_ops.py
@@ -52,7 +52,7 @@ def ignore_errors():
   return _apply_fn
 
 
-class _IgnoreErrorsDataset(dataset_ops.UnaryDataset):
+class _IgnoreErrorsDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that silently ignores errors when computing its input."""
 
   def __init__(self, input_dataset):
@@ -64,15 +64,3 @@ class _IgnoreErrorsDataset(dataset_ops.UnaryDataset):
     return gen_experimental_dataset_ops.experimental_ignore_errors_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         **dataset_ops.flat_structure(self))
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
diff --git a/tensorflow/python/data/experimental/ops/optimization.py b/tensorflow/python/data/experimental/ops/optimization.py
index b744db7f1e..c6c7de9265 100644
--- a/tensorflow/python/data/experimental/ops/optimization.py
+++ b/tensorflow/python/data/experimental/ops/optimization.py
@@ -100,7 +100,7 @@ def optimize(optimizations=None):
   return _apply_fn
 
 
-class _AssertNextDataset(dataset_ops.UnaryDataset):
+class _AssertNextDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that asserts which transformations happen next."""
 
   def __init__(self, input_dataset, transformations):
@@ -118,20 +118,8 @@ class _AssertNextDataset(dataset_ops.UnaryDataset):
         self._transformations,
         **dataset_ops.flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
 
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-
-class _NonSerializableDataset(dataset_ops.UnaryDataset):
+class _NonSerializableDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that performs non-serializable identity transformation."""
 
   def __init__(self, input_dataset):
@@ -143,15 +131,3 @@ class _NonSerializableDataset(dataset_ops.UnaryDataset):
     return gen_experimental_dataset_ops.experimental_non_serializable_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         **dataset_ops.flat_structure(self))
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index d34f9f25bd..1c4e14ce32 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -265,7 +265,7 @@ class _PrefetchToDeviceEagerIterator(iterator_ops.EagerIterator):
 # pylint: enable=protected-access
 
 
-class _PrefetchToDeviceDataset(dataset_ops.UnaryDataset):
+class _PrefetchToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` whose iterator prefetches elements to another device."""
 
   def __init__(self, input_dataset, device, buffer_size):
@@ -322,18 +322,6 @@ class _PrefetchToDeviceDataset(dataset_ops.UnaryDataset):
     raise NotImplementedError("`prefetch_to_device()` must be the last "
                               "transformation in a dataset pipeline.")
 
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
 
 @tf_export("data.experimental.prefetch_to_device")
 def prefetch_to_device(device, buffer_size=None):
@@ -380,7 +368,7 @@ def copy_to_device(target_device, source_device="/cpu:0"):
 # TODO(rohanj): Use the _input_hostmem attr on the RemoteCall ops to indicate
 # all inputs to the Op are in host memory, thereby avoiding some unnecessary
 # Sends and Recvs.
-class _CopyToDeviceDataset(dataset_ops.UnaryDataset):
+class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that copies elements to another device."""
 
   def __init__(self, input_dataset, target_device, source_device="/cpu:0"):
@@ -529,18 +517,6 @@ class _CopyToDeviceDataset(dataset_ops.UnaryDataset):
           output_types=self._flat_output_types,
           output_shapes=self._flat_output_shapes)
 
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
 
 class _MapOnGpuDataset(dataset_ops.UnaryDataset):
   """A `Dataset` that maps a function over elements in its using a GPU."""
diff --git a/tensorflow/python/data/experimental/ops/shuffle_ops.py b/tensorflow/python/data/experimental/ops/shuffle_ops.py
index a4307212da..d12328a714 100644
--- a/tensorflow/python/data/experimental/ops/shuffle_ops.py
+++ b/tensorflow/python/data/experimental/ops/shuffle_ops.py
@@ -26,7 +26,7 @@ from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-class _ShuffleAndRepeatDataset(dataset_ops.UnaryDataset):
+class _ShuffleAndRepeatDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that fuses `shuffle` and `repeat`."""
 
   def __init__(self, input_dataset, buffer_size, count=None, seed=None):
@@ -53,18 +53,6 @@ class _ShuffleAndRepeatDataset(dataset_ops.UnaryDataset):
         **dataset_ops.flat_structure(self))
     # pylint: enable=protected-access
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
 
 @tf_export("data.experimental.shuffle_and_repeat")
 def shuffle_and_repeat(buffer_size, count=None, seed=None):
diff --git a/tensorflow/python/data/experimental/ops/sleep.py b/tensorflow/python/data/experimental/ops/sleep.py
index 7e7d370f70..5e9d021ada 100644
--- a/tensorflow/python/data/experimental/ops/sleep.py
+++ b/tensorflow/python/data/experimental/ops/sleep.py
@@ -21,7 +21,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.ops import gen_experimental_dataset_ops
 
 
-class _SleepDataset(dataset_ops.UnaryDataset):
+class _SleepDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that sleeps before producing each upstream element."""
 
   def __init__(self, input_dataset, sleep_microseconds):
diff --git a/tensorflow/python/data/experimental/ops/stats_ops.py b/tensorflow/python/data/experimental/ops/stats_ops.py
index ca2f5f2a88..95689433bd 100644
--- a/tensorflow/python/data/experimental/ops/stats_ops.py
+++ b/tensorflow/python/data/experimental/ops/stats_ops.py
@@ -94,7 +94,7 @@ def latency_stats(tag):
   return _apply_fn
 
 
-class _StatsDataset(dataset_ops.UnaryDataset):
+class _StatsDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that acts as an identity, and also records statistics."""
 
   def __init__(self, input_dataset, op_function, tag):
@@ -108,15 +108,3 @@ class _StatsDataset(dataset_ops.UnaryDataset):
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._tag,
         **dataset_ops.flat_structure(self))
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
diff --git a/tensorflow/python/data/experimental/ops/threadpool.py b/tensorflow/python/data/experimental/ops/threadpool.py
index 3ea017c6e8..69e8829d68 100644
--- a/tensorflow/python/data/experimental/ops/threadpool.py
+++ b/tensorflow/python/data/experimental/ops/threadpool.py
@@ -60,7 +60,7 @@ class PrivateThreadPool(object):
           display_name=display_name)
 
 
-class _ThreadPoolDataset(dataset_ops.UnaryDataset):
+class _ThreadPoolDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` that acts as an identity, and sets a custom threadpool."""
 
   def __init__(self, input_dataset, thread_pool):
@@ -74,18 +74,6 @@ class _ThreadPoolDataset(dataset_ops.UnaryDataset):
         self._thread_pool._resource,  # pylint: disable=protected-access
         **dataset_ops.flat_structure(self))
 
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
 
 # TODO(b/73383364): Properly export in the `tf.data.experimental` API when
 # stable or make private / remove.
diff --git a/tensorflow/python/data/experimental/ops/unique.py b/tensorflow/python/data/experimental/ops/unique.py
index 2a7775c456..55ed98d854 100644
--- a/tensorflow/python/data/experimental/ops/unique.py
+++ b/tensorflow/python/data/experimental/ops/unique.py
@@ -48,7 +48,7 @@ def unique():
   return _apply_fn
 
 
-class _UniqueDataset(dataset_ops.UnaryDataset):
+class _UniqueDataset(dataset_ops.UnaryUnchangedStructureDataset):
   """A `Dataset` contains the unique elements from its input."""
 
   def __init__(self, input_dataset):
@@ -65,15 +65,3 @@ class _UniqueDataset(dataset_ops.UnaryDataset):
     return gen_experimental_dataset_ops.experimental_unique_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         **dataset_ops.flat_structure(self))
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index e627cc3469..7f0903c330 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1731,6 +1731,22 @@ class UnaryDataset(DatasetV2):
     return [self._input_dataset]
 
 
+class UnaryUnchangedStructureDataset(UnaryDataset):
+  """Represents a unary dataset with the same input and output structure."""
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes  # pylint: disable=protected-access
+
+  @property
+  def output_shapes(self):
+    return self._input_dataset.output_shapes  # pylint: disable=protected-access
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types  # pylint: disable=protected-access
+
+
 class TensorDataset(DatasetSource):
   """A `Dataset` with a single element, viz. a nested structure of tensors."""
 
@@ -2377,7 +2393,7 @@ class ConcatenateDataset(DatasetV2):
     return self._output_types
 
 
-class RepeatDataset(UnaryDataset):
+class RepeatDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that repeats its input several times."""
 
   def __init__(self, input_dataset, count):
@@ -2396,18 +2412,6 @@ class RepeatDataset(UnaryDataset):
         count=self._count,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
 
 class RangeDataset(DatasetSource):
   """A `Dataset` of a step separated range of values."""
@@ -2457,7 +2461,7 @@ class RangeDataset(DatasetSource):
     return dtypes.int64
 
 
-class CacheDataset(UnaryDataset):
+class CacheDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that caches elements of its input."""
 
   def __init__(self, input_dataset, filename):
@@ -2473,20 +2477,8 @@ class CacheDataset(UnaryDataset):
         filename=self._filename,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
 
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-
-class ShuffleDataset(UnaryDataset):
+class ShuffleDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that randomly shuffles the elements of its input."""
 
   def __init__(self,
@@ -2534,20 +2526,8 @@ class ShuffleDataset(UnaryDataset):
         reshuffle_each_iteration=self._reshuffle_each_iteration,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
 
-
-class TakeDataset(UnaryDataset):
+class TakeDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` containing the first `count` elements from its input."""
 
   def __init__(self, input_dataset, count):
@@ -2562,20 +2542,8 @@ class TakeDataset(UnaryDataset):
         count=self._count,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
 
-
-class SkipDataset(UnaryDataset):
+class SkipDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` skipping the first `count` elements from its input."""
 
   def __init__(self, input_dataset, count):
@@ -2590,18 +2558,6 @@ class SkipDataset(UnaryDataset):
         count=self._count,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
 
 class BatchDataset(UnaryDataset):
   """A `Dataset` that batches contiguous elements from its input."""
@@ -2904,7 +2860,7 @@ class MapDataset(UnaryDataset):
     return "Dataset.map()"
 
 
-class MatchingFilesDataset(DatasetV2):
+class MatchingFilesDataset(DatasetSource):
   """A `Dataset` that list the files according to the input patterns."""
 
   def __init__(self, patterns):
@@ -2915,9 +2871,6 @@ class MatchingFilesDataset(DatasetV2):
   def _as_variant_tensor(self):
     return gen_dataset_ops.matching_files_dataset(self._patterns)
 
-  def _inputs(self):
-    return []
-
   @property
   def output_classes(self):
     return ops.Tensor
@@ -3057,7 +3010,7 @@ class ParallelInterleaveDataset(FlatMapDataset):
     return "Dataset.interleave()"
 
 
-class FilterDataset(UnaryDataset):
+class FilterDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that filters its input according to a predicate function."""
 
   def __init__(self, input_dataset, predicate):
@@ -3079,23 +3032,11 @@ class FilterDataset(UnaryDataset):
         predicate=self._predicate,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
   def _transformation_name(self):
     return "Dataset.filter()"
 
 
-class PrefetchDataset(UnaryDataset):
+class PrefetchDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that asynchronously prefetches its input."""
 
   def __init__(self, input_dataset, buffer_size):
@@ -3113,18 +3054,6 @@ class PrefetchDataset(UnaryDataset):
         buffer_size=self._buffer_size,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
 
 class WindowDataset(UnaryDataset):
   """A dataset that creates window datasets from the input elements."""
@@ -3176,7 +3105,7 @@ class WindowDataset(UnaryDataset):
     return self._output_types
 
 
-class _OptionsDataset(UnaryDataset):
+class _OptionsDataset(UnaryUnchangedStructureDataset):
   """An identity `Dataset` that stores options."""
 
   def __init__(self, input_dataset, options):
@@ -3194,20 +3123,8 @@ class _OptionsDataset(UnaryDataset):
   def options(self):
     return self._options
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
 
-class _ModelDataset(UnaryDataset):
+class _ModelDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that acts as an identity, and models performance."""
 
   def __init__(self, input_dataset):
@@ -3220,20 +3137,8 @@ class _ModelDataset(UnaryDataset):
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
 
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-
-class _OptimizeDataset(UnaryDataset):
+class _OptimizeDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that acts as an identity, and applies optimizations."""
 
   def __init__(self, input_dataset, optimizations):
@@ -3251,20 +3156,8 @@ class _OptimizeDataset(UnaryDataset):
         self._optimizations,
         **flat_structure(self))
 
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
 
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-
-class _SetStatsAggregatorDataset(UnaryDataset):
+class _SetStatsAggregatorDataset(UnaryUnchangedStructureDataset):
   """A `Dataset` that acts as an identity, and sets stats aggregator."""
 
   def __init__(self, input_dataset, aggregator, prefix, counter_prefix):
@@ -3281,15 +3174,3 @@ class _SetStatsAggregatorDataset(UnaryDataset):
         self._prefix,
         self._counter_prefix,
         **flat_structure(self))
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-- 
GitLab


From 387a062dd31406360733a5ccc72d783153b86096 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 14 Nov 2018 23:56:51 -0800
Subject: [PATCH 0285/1554] [XLA:GPU] Miscellaneous clean up for 0-2-1
 transpose related code.

We use the terms reduced shape and unreduced shape to refer to the logical
shapes and the original shapes of a 0-2-1 tranpose. Since reduced shape and
unreduced shape could also refer to the result shape and source shape of a
reduction operation, the purpose of this CL is mainly to change the 0-2-1
transpose related code outside reduction implementation to use the words
normalized/unnormalized instead of reduced/unreduced. The reduction
implementation will be fixed in another CL that migrates the implementation to
use the kernel mapping scheme.

PiperOrigin-RevId: 221575772
---
 .../xla/service/gpu/ir_emitter_unnested.cc    |  2 +-
 .../xla/service/llvm_ir/kernel_tiling.cc      | 95 ++++++++++---------
 .../xla/service/llvm_ir/kernel_tiling.h       | 24 ++---
 3 files changed, 58 insertions(+), 63 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 9976b521b3..5751cfd952 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -3375,7 +3375,7 @@ void IrEmitterUnnested::EmitTileElementForFusion(
   fused_emitter.SetTiledParameterInfo(tiled_param_info);
   TF_CHECK_OK(hlo->fused_expression_root()->Accept(&fused_emitter));
   IrArray::Index untiled_index =
-      kernel_info->GetKernelMappingScheme()->GetReshapedOutputIndex(
+      kernel_info->GetKernelMappingScheme()->GetUnnormalizedIndex(
           index, output_arrays[0].GetShape());
   const llvm_ir::ElementGenerator& output_generator =
       fused_emitter.GetRootGenerator();
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index 286b987d51..c26711e526 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -52,6 +52,29 @@ Shape MergeDimensions(absl::Span<const size_t> segs, const Shape& shape) {
   return ShapeUtil::MakeShapeWithDescendingLayout(shape.element_type(),
                                                   dimensions);
 }
+
+// Given an index for a shape, return the equivalent new index if the shape is
+// reshaped to another shape.
+IrArray::Index GetReshapedIndex(const IrArray::Index& index, const Shape& shape,
+                                const Shape& reshaped_shape,
+                                llvm::IRBuilder<>* b) {
+  auto bounds = shape.dimensions();
+  auto minor_to_major = shape.layout().minor_to_major();
+  llvm::Value* linear_index = index.GetConstantWithIndexType(0);
+  int64 multiplier = 1;
+  for (int i = 0; i < index.size(); ++i) {
+    int64 dim = minor_to_major[i];
+    llvm::Value* addend = b->CreateMul(
+        index[dim], index.GetConstantWithIndexType(multiplier), "linearizing",
+        /*HasNUW=*/true, /*HasNSW=*/true);
+    linear_index = b->CreateAdd(linear_index, addend, "",
+                                /*HasNUW=*/true, /*HasNSW=*/true);
+    multiplier *= bounds[dim];
+  }
+
+  return IrArray::Index(linear_index, reshaped_shape, b);
+}
+
 }  // namespace
 
 absl::optional<std::vector<int64> > FindTranspose021(const Shape& a,
@@ -60,28 +83,30 @@ absl::optional<std::vector<int64> > FindTranspose021(const Shape& a,
     return absl::nullopt;
   }
 
-  std::vector<int64> perm(a.dimensions().size());
-  {
-    auto layout_a_orig = LayoutUtil::MinorToMajor(a);
-    std::vector<int64> layout_a(layout_a_orig.rbegin(), layout_a_orig.rend());
-    auto layout_b_orig = LayoutUtil::MinorToMajor(b);
-    std::vector<int64> layout_b(layout_b_orig.rbegin(), layout_b_orig.rend());
-    for (size_t i = 0; i < perm.size(); ++i) {
-      perm[i] = PositionInContainer(layout_b, layout_a[i]);
-    }
+  std::vector<int64> permutation(a.dimensions().size());
+  absl::Span<const int64> minor_to_major_a = LayoutUtil::MinorToMajor(a);
+  std::vector<int64> major_to_minor_a(minor_to_major_a.rbegin(),
+                                      minor_to_major_a.rend());
+  absl::Span<const int64> minor_to_major_b = LayoutUtil::MinorToMajor(b);
+  std::vector<int64> major_to_minor_b(minor_to_major_b.rbegin(),
+                                      minor_to_major_b.rend());
+  for (size_t i = 0; i < permutation.size(); ++i) {
+    permutation[i] = PositionInContainer(major_to_minor_b, major_to_minor_a[i]);
   }
-  auto segs = ConsecutiveSegments(perm);
-  if ((3 == segs.size() && 0 == perm[0]) || 2 == segs.size()) {
-    Shape norm_a =
+
+  std::vector<size_t> segments = ConsecutiveSegments(permutation);
+  if ((3 == segments.size() && 0 == permutation[0]) || 2 == segments.size()) {
+    Shape descending_layout_shape =
         ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(a);
-    Shape reduced_a = MergeDimensions(segs, norm_a);
-    auto reduced_a_dims = reduced_a.dimensions();
+    Shape normalized_shape = MergeDimensions(segments, descending_layout_shape);
+    absl::Span<const int64> normalized_dims =
+        AsInt64Slice(normalized_shape.dimensions());
     std::vector<int64> dims_021;
-    if (2 == segs.size()) {
+    if (2 == segments.size()) {
       // The logical component-0 is of size one.
-      dims_021 = {1, reduced_a_dims[1], reduced_a_dims[0]};
+      dims_021 = {1, normalized_dims[1], normalized_dims[0]};
     } else {
-      dims_021 = {reduced_a_dims[0], reduced_a_dims[2], reduced_a_dims[1]};
+      dims_021 = {normalized_dims[0], normalized_dims[2], normalized_dims[1]};
     }
 
     return dims_021;
@@ -90,29 +115,6 @@ absl::optional<std::vector<int64> > FindTranspose021(const Shape& a,
   return absl::nullopt;
 }
 
-IrArray::Index GetUnreducedOutputIndex(
-    const IrArray::Index& reduced_output_index,
-    const Shape& reduced_output_shape, const Shape& unreduced_output_shape,
-    llvm::IRBuilder<>* b) {
-  auto bounds = reduced_output_shape.dimensions();
-  auto minor_to_major = reduced_output_shape.layout().minor_to_major();
-  llvm::Value* linear_index = reduced_output_index.GetConstantWithIndexType(0);
-  int64 multiplier = 1;
-  for (int i = 0; i < reduced_output_index.size(); ++i) {
-    int64 dim = minor_to_major[i];
-    llvm::Value* addend =
-        b->CreateMul(reduced_output_index[dim],
-                     reduced_output_index.GetConstantWithIndexType(multiplier),
-                     "linearizing",
-                     /*HasNUW=*/true, /*HasNSW=*/true);
-    linear_index = b->CreateAdd(linear_index, addend, "",
-                                /*HasNUW=*/true, /*HasNSW=*/true);
-    multiplier *= bounds[dim];
-  }
-
-  return IrArray::Index(linear_index, unreduced_output_shape, b);
-}
-
 KernelMappingScheme::KernelMappingScheme(
     absl::Span<const int64> dims_in_elems, int64 tile_size_y, int64 tile_size_x,
     absl::Span<const int64> req_block_sizes, int64 num_threads_y,
@@ -143,13 +145,14 @@ KernelMappingScheme::KernelMappingScheme(
            << "]";
 }
 
-IrArray::Index KernelMappingScheme::GetReshapedOutputIndex(
-    const IrArray::Index& output_index, const Shape& reshaped_output_shape) {
-  DCHECK_EQ(output_index.size(), dims_in_elems_.size());
+IrArray::Index KernelMappingScheme::GetUnnormalizedIndex(
+    const IrArray::Index& normalized_shape_index,
+    const Shape& unnormalized_shape) {
+  DCHECK_EQ(normalized_shape_index.size(), dims_in_elems_.size());
   Shape output_shape = ShapeUtil::MakeShapeWithDescendingLayout(
-      reshaped_output_shape.element_type(), GetDimensionsInElements());
-  return llvm_ir::GetUnreducedOutputIndex(output_index, output_shape,
-                                          reshaped_output_shape, b_);
+      unnormalized_shape.element_type(), GetDimensionsInElements());
+  return GetReshapedIndex(normalized_shape_index, output_shape,
+                          unnormalized_shape, b_);
 }
 
 IrArray::Index KernelMappingScheme::EmitBlockIndex(llvm::Type* index_ty) {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index fc1fa9c3d2..c44ffb9e07 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -28,24 +28,15 @@ namespace llvm_ir {
 // If a shape can be viewed as three logical components 0-1-2 in the order of
 // major to minor, a 0-2-1-transpose changes the order of such logical
 // components to 0-2-1. We call the shape being transposed the input shape and
-// the transposed shape the output shape. The logical view of the input and
-// output shapes for the transpose are called the 0-1-2 shape or reduced input
-// shape and the 0-2-1 shape or the reduced output shape respectively. The
-// original input and output shapes are called the unreduced input and output
-// shapes.
-
+// the transposed shape the output shape. The logical view of the input/output
+// shapes for the transpose are called the 0-1-2/0-2-1 shapes or the normalized
+// shapes. The original input/output shapes are called unnormalized shapes.
+//
 // If `b` is a 0-2-1 transpose of `a` in 0-1-2, return the dimensions for the
-// reduced shape of `b` or the 0-2-1 shape.
+// normalized shape of `b` or the 0-2-1 shape.
 absl::optional<std::vector<int64> > FindTranspose021(const Shape& a,
                                                      const Shape& b);
 
-// Return the unreduced output index corresponding to the given reduced output
-// index.
-IrArray::Index GetUnreducedOutputIndex(
-    const IrArray::Index& reduced_output_index,
-    const Shape& reduced_output_shape, const Shape& unreduced_output_shape,
-    llvm::IRBuilder<>* b);
-
 // A tile is a spatial subdivision of a tensor. We group tensor elements into
 // tiles so that we can launch kernels to process the tensor elements in blocks
 // of tiles.
@@ -158,8 +149,9 @@ class KernelMappingScheme {
   std::tuple<llvm::Value*, llvm::Value*> EmitThreadYXCoordinate(
       llvm::Type* index_ty);
 
-  IrArray::Index GetReshapedOutputIndex(const IrArray::Index& output_index,
-                                        const Shape& reshaped_output_shape);
+  IrArray::Index GetUnnormalizedIndex(
+      const IrArray::Index& normalized_shape_index,
+      const Shape& unnormalized_shape);
 
   llvm::GlobalVariable* GetSharedMemoryBufferForElementType(
       llvm::Type* elem_ty, absl::string_view buffer_name);
-- 
GitLab


From fa5e5ecd88673492a91117d5d96d7be5f68e3443 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 15 Nov 2018 00:58:02 -0800
Subject: [PATCH 0286/1554] Add make_input_fn_iterator method to distribution
 strategies which takes an input function and (will, in a following cl,)
 return an iterator object Add InputContext class which will be an argument
 for input_fn to the `make_input_fn_iterator` method.

PiperOrigin-RevId: 221581165
---
 tensorflow/contrib/distribute/python/BUILD    |   1 +
 .../python/collective_all_reduce_strategy.py  |  22 ++-
 .../collective_all_reduce_strategy_test.py    |  26 ++++
 .../distribute/python/mirrored_strategy.py    |  24 ++++
 .../python/mirrored_strategy_test.py          |  22 +++
 .../distribute/python/one_device_strategy.py  |   8 ++
 .../python/one_device_strategy_test.py        |   9 ++
 .../python/parameter_server_strategy.py       |  21 +++
 .../python/parameter_server_strategy_test.py  |  24 ++++
 .../distribute/python/strategy_test_lib.py    |  28 ++++
 .../contrib/distribute/python/values.py       |  24 +++-
 .../python/distribute/multi_worker_util.py    |  98 +++++++++++--
 .../distribute/multi_worker_util_test.py      |  91 +++++++++++-
 tensorflow/python/training/distribute.py      | 133 +++++++++++++++++-
 tensorflow/python/training/distribute_test.py |  23 ++-
 15 files changed, 529 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 119ed45625..e642c8ca14 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -118,6 +118,7 @@ cuda_py_test(
         ":combinations",
         ":multi_worker_test_base",
         ":parameter_server_strategy",
+        ":strategy_test_lib",
         ":values",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index efa99d1fc5..a2922b9881 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import collective_ops
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import distribute as distribute_lib
 
 
 # TODO(yuefengz): support in-graph replication.
@@ -94,8 +95,7 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
           "Unrecognized task_type: %r, valid task types are: \"chief\", "
           "\"worker\"." % task_type)
     cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-    self._num_workers = len(cluster_spec.as_dict().get("worker", [])) + len(
-        cluster_spec.as_dict().get("chief", []))
+    self._num_workers = multi_worker_util.worker_count(cluster_spec, task_type)
     if not self._num_workers:
       raise ValueError("No `worker` or `chief` tasks can be found in "
                        "`cluster_spec`.")
@@ -208,6 +208,23 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._devices, True)
 
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    """Distributes the dataset to each local GPU."""
+    if self._cluster_spec is None:
+      input_pipeline_id = 0
+    else:
+      input_pipeline_id = multi_worker_util.id_in_cluster(
+          self._cluster_spec, self._task_type, self._task_id)
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=self._num_workers,
+        input_pipeline_id=input_pipeline_id,
+        num_replicas_in_sync=self.num_replicas_in_sync)
+    return values.PerReplicaDataset(
+        self._call_dataset_fn(input_fn, input_context), self._devices, True)
+
   def configure(self,
                 session_config=None,
                 cluster_spec=None,
@@ -289,4 +306,3 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
   @property
   def num_replicas_in_sync(self):
     return len(self._devices) * self._num_workers
-
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 219c5f531c..28e69d2a5f 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -25,6 +25,7 @@ from tensorflow.contrib.distribute.python import collective_all_reduce_strategy
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import multi_worker_test_base
+from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import keras
 from tensorflow.python.distribute import reduce_util
@@ -339,5 +340,30 @@ class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase,
     self._test_complex_model(None, None, num_gpus)
 
 
+class InputContextTest(strategy_test_lib.DistributionTestBase):
+
+  def testInputContextPropertyLocal(self):
+    d = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
+        num_gpus_per_worker=2)
+    with context.graph_mode():
+      input_fn = self._input_fn_to_test_input_context(
+          expected_num_replicas_in_sync=2,
+          expected_num_input_pipelines=1,
+          expected_input_pipeline_id=0)
+      d.make_input_fn_iterator(input_fn)
+
+  def testInputContextPropertyMultiWorker(self):
+    d = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
+        num_gpus_per_worker=2)
+    cluster_spec = {'worker': ['worker1', 'worker2', 'worker3'], 'ps': ['ps1']}
+    d.configure(cluster_spec=cluster_spec, task_type='worker', task_id=1)
+    with context.graph_mode():
+      input_fn = self._input_fn_to_test_input_context(
+          expected_num_replicas_in_sync=6,
+          expected_num_input_pipelines=3,
+          expected_input_pipeline_id=1)  # because task_id = 1
+      d.make_input_fn_iterator(input_fn)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 9795df8d13..74d69abcfd 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -485,6 +485,30 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
       return values.PerReplicaDataset(
           self._call_dataset_fn(dataset_fn), self._devices)
 
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    if self._cluster_spec:
+      input_fns = []
+      for i in range(len(self._worker_devices)):
+        input_context = distribute_lib.InputContext(
+            num_input_pipelines=len(self._worker_devices),
+            input_pipeline_id=i,
+            num_replicas_in_sync=self.num_replicas_in_sync)
+        input_fns.append(
+            partial(self._call_dataset_fn, input_fn, input_context))
+
+      return values.MultiWorkerDataset(input_fns, self._worker_devices,
+                                       self._auto_shard_dataset)
+    else:
+      input_context = distribute_lib.InputContext(
+          num_input_pipelines=1,
+          input_pipeline_id=0,
+          num_replicas_in_sync=self.num_replicas_in_sync)
+      return values.PerReplicaDataset(
+          self._call_dataset_fn(input_fn, input_context), self._devices)
+
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   def _run_steps_on_dataset(self, fn, iterator, iterations,
                             initial_loop_values=None):
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
index 7c7026052d..001ee89c82 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -47,6 +47,28 @@ class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
   def testCallAndMergeExceptions(self):
     self._test_call_and_merge_exceptions(self._get_distribution_strategy())
 
+  @test_util.run_in_graph_and_eager_modes
+  def testInputContextPropertyLocal(self):
+    d = mirrored_strategy.MirroredStrategy(num_gpus_per_worker=2)
+    input_fn = self._input_fn_to_test_input_context(
+        expected_num_replicas_in_sync=2,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)
+    d.make_input_fn_iterator(input_fn)
+
+  def testInputContextPropertyMultiWorker(self):
+    d = mirrored_strategy.MirroredStrategy(num_gpus_per_worker=2)
+    cluster_spec = {"worker": ["worker1", "worker2", "worker3"]}
+    d.configure(cluster_spec=cluster_spec)
+    with context.graph_mode():
+      # `expected_input_pipeline_id` is None because the input_fn will be called
+      # multiple times, each with a different input_pipeline_id.
+      input_fn = self._input_fn_to_test_input_context(
+          expected_num_replicas_in_sync=6,
+          expected_num_input_pipelines=3,
+          expected_input_pipeline_id=None)
+      d.make_input_fn_iterator(input_fn)
+
 
 class VariableCreatorStackTest(test.TestCase):
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 96f5f739d5..cbd5d1ab40 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -67,6 +67,14 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), [self._device])
 
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    return values.PerReplicaDataset(
+        self._call_dataset_fn(input_fn, distribute_lib.InputContext()),
+        [self._device])
+
   def _broadcast(self, tensor, destinations):
     del destinations
     return tensor
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 79f9c39cc8..cf6d890390 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -42,6 +42,15 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
   def testCallAndMergeExceptions(self):
     self._test_call_and_merge_exceptions(self._get_distribution_strategy())
 
+  @test_util.run_in_graph_and_eager_modes
+  def testInputContextPropertyLocal(self):
+    d = one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+    input_fn = self._input_fn_to_test_input_context(
+        expected_num_replicas_in_sync=1,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)
+    d.make_input_fn_iterator(input_fn)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 9cd71293f8..11d6be9e86 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -227,6 +227,27 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._compute_devices, True)
 
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    """Distributes the dataset to each local GPU."""
+    if self._cluster_spec:
+      input_pipeline_id = multi_worker_util.id_in_cluster(
+          self._cluster_spec, self._task_type, self._task_id)
+      num_input_pipelines = multi_worker_util.worker_count(
+          self._cluster_spec, self._task_type)
+    else:
+      input_pipeline_id = 0
+      num_input_pipelines = 1
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=num_input_pipelines,
+        input_pipeline_id=input_pipeline_id,
+        num_replicas_in_sync=self.num_replicas_in_sync)
+    return values.PerReplicaDataset(
+        self._call_dataset_fn(input_fn, input_context), self._compute_devices,
+        True)
+
   def _broadcast(self, tensor, destinations):
     if not cross_tower_ops_lib.check_destinations(destinations):
       destinations = self._compute_devices
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index ec61f62dc7..d25b5836c5 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -25,6 +25,7 @@ from absl.testing import parameterized
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import parameter_server_strategy
+from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.distribute import multi_worker_util
@@ -622,5 +623,28 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
       distribution.call_for_each_replica(f)
 
 
+class InputContextTest(strategy_test_lib.DistributionTestBase):
+
+  def testInputContextPropertyLocal(self):
+    d = parameter_server_strategy.ParameterServerStrategy(num_gpus_per_worker=2)
+    with context.graph_mode():
+      input_fn = self._input_fn_to_test_input_context(
+          expected_num_replicas_in_sync=2,
+          expected_num_input_pipelines=1,
+          expected_input_pipeline_id=0)
+      d.make_input_fn_iterator(input_fn)
+
+  def testInputContextPropertyMultiWorker(self):
+    d = parameter_server_strategy.ParameterServerStrategy(num_gpus_per_worker=2)
+    cluster_spec = {'worker': ['worker1', 'worker2', 'worker3'], 'ps': ['ps1']}
+    d.configure(cluster_spec=cluster_spec, task_type='worker', task_id=1)
+    with context.graph_mode():
+      input_fn = self._input_fn_to_test_input_context(
+          expected_num_replicas_in_sync=2,
+          expected_num_input_pipelines=3,
+          expected_input_pipeline_id=1)  # because task_id =1
+      d.make_input_fn_iterator(input_fn)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 31d7490290..ceea898566 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -211,3 +212,30 @@ class DistributionTestBase(test.TestCase):
         dist.call_for_each_replica(_merge_call_raises_fn)
       with self.assertRaises(_TestException):
         dist.call_for_each_replica(_merge_call_merge_raises_fn)
+
+  def _input_fn_to_test_input_context(self, expected_num_replicas_in_sync,
+                                      expected_num_input_pipelines,
+                                      expected_input_pipeline_id):
+    # Use a list of one element as counter so that it can be captured by the
+    # `_input_fn`. This counter is incremented by 1 each time an input_fn is
+    # called. We use this counter to check whether the `input_pipeline_id`
+    # matches the counter in the in-graph replication.
+    worker_id_counter = [0]
+
+    def _input_fn(input_context):
+      """Input fn for testing."""
+      self.assertIsNotNone(input_context)
+      self.assertEqual(expected_num_replicas_in_sync,
+                       input_context.num_replicas_in_sync)
+      self.assertEqual(expected_num_input_pipelines,
+                       input_context.num_input_pipelines)
+      if expected_input_pipeline_id is not None:
+        self.assertEqual(expected_input_pipeline_id,
+                         input_context.input_pipeline_id)
+      else:
+        self.assertEqual(worker_id_counter[0], input_context.input_pipeline_id)
+        worker_id_counter[0] += 1
+
+      return dataset_ops.Dataset.from_tensors([[1.]]).repeat()
+
+    return _input_fn
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index bf8c958efa..2f4ef9de89 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -1256,22 +1256,34 @@ class MultiWorkerDataset(object):
     """Initialize the MultiWorkerDataset object.
 
     Args:
-      dataset_fn: a function that returns a `tf.data.Dataset`.
+      dataset_fn: a function or a list of functions that returns a
+        `tf.data.Dataset`.
       worker_device_pairs: a list of (worker, list of devices on that worker)
-        pairs.
+        pairs; it must have same length with `dataset_fn` if `dataset_fn` is a
+        list.
       prefetch_on_device: whether to prefetch to devices.
       auto_shard: whether to auto-shard the dataset.
     """
+    if isinstance(dataset_fn, list):
+      if len(dataset_fn) != len(worker_device_pairs):
+        raise ValueError("If `dataset_fn` is a list, it must have same length "
+                         "as `worker_device_pairs`")
+      if auto_shard:
+        raise ValueError(
+            "If `dataset_fn` is a list, `auto_shard` is not supported.")
     self._worker_device_pairs = worker_device_pairs
     self._datasets = []
     # TODO(yuefengz, priyag): support different set of jobs for input
     # processing.
     for i, (worker, worker_devices) in enumerate(worker_device_pairs):
       with ops.device(worker):
-        worker_input = dataset_fn()
-        if auto_shard:
-          worker_input = input_ops.auto_shard_dataset(
-              worker_input, len(worker_device_pairs), i)
+        if isinstance(dataset_fn, list):
+          worker_input = dataset_fn[i]()
+        else:
+          worker_input = dataset_fn()
+          if auto_shard:
+            worker_input = input_ops.auto_shard_dataset(
+                worker_input, len(worker_device_pairs), i)
         dataset = PerReplicaDataset(
             worker_input, worker_devices, prefetch_on_device=prefetch_on_device)
         self._datasets.append((worker, dataset))
diff --git a/tensorflow/python/distribute/multi_worker_util.py b/tensorflow/python/distribute/multi_worker_util.py
index 360733eff6..2986a6726a 100644
--- a/tensorflow/python/distribute/multi_worker_util.py
+++ b/tensorflow/python/distribute/multi_worker_util.py
@@ -45,6 +45,33 @@ def normalize_cluster_spec(cluster_spec):
   return cluster_spec
 
 
+# TODO(yuefengz): add more validations.
+def _validate_cluster_spec(cluster_spec, task_type, task_id):
+  """Validates `cluster_spec`.
+
+  It checks
+  1) whether there is such a task type as `task_type` in the
+  `cluster_spec`.
+  2) whether there is at most one "chief" job.
+  3) whether the `task_id` is smaller than the number of `task_type`.
+
+  Args:
+    cluster_spec: a dict, `ClusterDef` or `ClusterSpec` object to be validated.
+    task_type: string indicating the type of the task.
+    task_id: task_id: the id of the `task_type` in this cluster.
+  Throws:
+    ValueError: if `cluster_spec` fails any check.
+  """
+  cluster_spec = normalize_cluster_spec(cluster_spec).as_dict()
+  if task_type and task_type not in cluster_spec:
+    raise ValueError("`task_type` %r not found in cluster_spec." % task_type)
+  if len(cluster_spec.get("chief", [])) > 1:
+    raise ValueError("There must be at most one 'chief' job.")
+  if task_id >= len(cluster_spec[task_type]):
+    raise ValueError(
+        "The `task_id` %d exceeds the maximum id of %s." % (task_id, task_type))
+
+
 def is_chief(cluster_spec, task_type, task_id):
   """Returns whether the given task is chief in the cluster.
 
@@ -61,20 +88,73 @@ def is_chief(cluster_spec, task_type, task_id):
     ValueError: if `task_type` is not in the `cluster_spec` or `task_id` exceeds
       the maximum id of the `task_type`.
   """
-  cluster_spec = normalize_cluster_spec(cluster_spec)
-  if task_type not in cluster_spec.jobs:
-    raise ValueError(
-        "The task_type \"%s\" is not in the `cluster_spec`." % task_type)
-  if task_id >= cluster_spec.num_tasks(task_type):
-    raise ValueError("The `task_id` %d exceeds the maximum id of %s." % (
-        task_id, task_type))
+  _validate_cluster_spec(cluster_spec, task_type, task_id)
+  cluster_spec = normalize_cluster_spec(cluster_spec).as_dict()
 
   if task_type == "chief":
     return True
 
   # If chief not in the cluster_spec, use the first worker as chief. This is
   # common in CollectiveAllReduceStrategy.
-  if ("chief" not in cluster_spec.jobs and task_type == "worker" and
-      task_id == 0):
+  if ("chief" not in cluster_spec and task_type == "worker" and task_id == 0):
     return True
   return False
+
+
+def worker_count(cluster_spec, task_type):
+  """Returns the number of workers in the cluster."""
+  _validate_cluster_spec(cluster_spec, task_type, task_id=0)
+  cluster_spec = normalize_cluster_spec(cluster_spec).as_dict()
+
+  # Other jobs such as "ps" shouldn't call this function.
+  if task_type not in ["chief", "worker", "evaluator"]:
+    raise ValueError("Unexpected `task_type` %r" % task_type)
+
+  if task_type == "evaluator":
+    # The "evaluator" is in its own cluster or its own partition of a cluster.
+    # So we don't have to count "chief" or "worker" if the current task is an
+    # "evaluator".
+    return len(cluster_spec["evaluator"])
+  else:
+    # In the non-evaluator case, we return the total number of "chief" and
+    # "worker" tasks as the "chief" is also a worker.
+    return (len(cluster_spec.get("chief", [])) + len(
+        cluster_spec.get("worker", [])))
+
+
+def id_in_cluster(cluster_spec, task_type, task_id):
+  """Returns a unique id for the task in the `task_type`'s cluster.
+
+  It returns an id ranging from [0, `worker_count(task_type, task_id)`).
+
+  Note: this function assumes that "evaluate" job is in its own cluster or its
+  own partition of a cluster.
+
+  Args:
+    cluster_spec: a dict, `ClusterDef` or `ClusterSpec` object to be validated.
+    task_type: string indicating the type of the task.
+    task_id: the id of the `task_type` in this cluster.
+
+  Returns:
+    an int indicating the unique id.
+
+  Throws:
+    ValueError: if `task_type` is not "chief", "worker" or "evaluator".
+  """
+  _validate_cluster_spec(cluster_spec, task_type, task_id)
+  cluster_spec = normalize_cluster_spec(cluster_spec).as_dict()
+
+  # The "chief" job has always id 0 and there is at most one and "worker" jobs
+  # come after it.
+  if task_type == "chief":
+    return 0
+
+  if task_type == "worker":
+    return task_id + len(cluster_spec.get("chief", []))
+
+  # The "evaluator" is in its own cluster or its own partition of a cluster.
+  if task_type == "evaluator":
+    return task_id
+
+  # We currently don't assign ids to other tasks.
+  raise ValueError("There is no id for task_type %r" % task_type)
diff --git a/tensorflow/python/distribute/multi_worker_util_test.py b/tensorflow/python/distribute/multi_worker_util_test.py
index bdc49725c7..9e1596eefd 100644
--- a/tensorflow/python/distribute/multi_worker_util_test.py
+++ b/tensorflow/python/distribute/multi_worker_util_test.py
@@ -95,7 +95,7 @@ class IsChiefTest(test.TestCase):
     self.assertFalse(multi_worker_util.is_chief(cluster_spec, "worker", 1))
 
     with self.assertRaisesRegexp(
-        ValueError, "The task_type \"chief\" is not in the `cluster_spec`."):
+        ValueError, "`task_type` 'chief' not found in cluster_spec."):
       multi_worker_util.is_chief(cluster_spec, "chief", 0)
 
     with self.assertRaisesRegexp(
@@ -103,5 +103,94 @@ class IsChiefTest(test.TestCase):
       multi_worker_util.is_chief(cluster_spec, "worker", 2)
 
 
+class NumWorkersTest(test.TestCase):
+
+  def testCountWorker(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:1234"],
+        "worker": ["127.0.0.1:8964", "127.0.0.1:2333"],
+        "ps": ["127.0.0.1:1926", "127.0.0.1:3141"]
+    }
+    self.assertEqual(
+        multi_worker_util.worker_count(cluster_spec, task_type="chief"), 3)
+    self.assertEqual(
+        multi_worker_util.worker_count(cluster_spec, task_type="worker"), 3)
+
+  def testCountEvaluator(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:1234"],
+        "worker": ["127.0.0.1:8964", "127.0.0.1:2333"],
+        "evaluator": ["127.0.0.1:7566"]
+    }
+    self.assertEqual(
+        multi_worker_util.worker_count(cluster_spec, task_type="evaluator"), 1)
+
+  def testTaskTypeNotFound(self):
+    cluster_spec = {}
+    with self.assertRaisesRegexp(
+        ValueError, "`task_type` 'worker' not found in cluster_spec."):
+      multi_worker_util.worker_count(cluster_spec, task_type="worker")
+
+  def testCountPs(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:1234"],
+        "ps": ["127.0.0.1:1926", "127.0.0.1:3141"]
+    }
+    # A "ps" job shouldn't call this method.
+    with self.assertRaisesRegexp(ValueError, "Unexpected `task_type` 'ps'"):
+      multi_worker_util.worker_count(cluster_spec, task_type="ps")
+
+
+class IdInClusterTest(test.TestCase):
+
+  def testChiefId(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:1234"],
+        "worker": ["127.0.0.1:8964", "127.0.0.1:2333"],
+        "ps": ["127.0.0.1:1926", "127.0.0.1:3141"]
+    }
+    self.assertEqual(
+        multi_worker_util.id_in_cluster(cluster_spec, "chief", 0), 0)
+
+  def testWorkerId(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:1234"],
+        "worker": ["127.0.0.1:8964", "127.0.0.1:2333"],
+        "ps": ["127.0.0.1:1926", "127.0.0.1:3141"]
+    }
+    self.assertEqual(
+        multi_worker_util.id_in_cluster(cluster_spec, "worker", 1), 2)
+
+    cluster_spec = {
+        "worker": ["127.0.0.1:8964", "127.0.0.1:2333"],
+        "ps": ["127.0.0.1:1926", "127.0.0.1:3141"]
+    }
+    self.assertEqual(
+        multi_worker_util.id_in_cluster(cluster_spec, "worker", 1), 1)
+
+  def testEvaluatorId(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:1234"],
+        "worker": ["127.0.0.1:8964", "127.0.0.1:2333"],
+        "evaluator": ["127.0.0.1:7566"]
+    }
+    self.assertEqual(
+        multi_worker_util.id_in_cluster(cluster_spec, "evaluator", 0), 0)
+
+  def testPsId(self):
+    cluster_spec = {"chief": ["127.0.0.1:1234"], "ps": ["127.0.0.1:7566"]}
+    with self.assertRaisesRegexp(ValueError,
+                                 "There is no id for task_type 'ps'"):
+      multi_worker_util.id_in_cluster(cluster_spec, "ps", 0)
+
+  def testMultipleChiefs(self):
+    cluster_spec = {
+        "chief": ["127.0.0.1:8258", "127.0.0.1:7566"],
+    }
+    with self.assertRaisesRegexp(ValueError,
+                                 "There must be at most one 'chief' job."):
+      multi_worker_util.id_in_cluster(cluster_spec, "chief", 0)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index aed2a413ae..2ff6ff9bda 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import threading
+import enum
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
@@ -184,6 +185,80 @@ class _SameScopeAgainContext(object):
     del exception_type, exception_value, traceback
 
 
+# TODO(yuefengz): add more replication modes.
+class InputReplicationMode(enum.Enum):
+  """Replication mode for input function."""
+
+  # The input function will be called on each worker independently, creating as
+  # many input pipelines as number of workers. Replicas will dequeue from the
+  # local Dataset on their worker. Distribution Strategy doesn't manage any
+  # state sharing between such separate input pipelines.
+  PER_WORKER = 0
+
+
+class InputContext(object):
+  """A class wrapping information needed by an input function.
+
+  This is a context class that is passed to the user's input fn and contains
+  information about the compute replicas and input pipelines. The number of
+  compute replicas (in sync training) helps compute per input pipeline batch
+  size from the desired global batch size. Input pipeline information can be
+  used to return a different subset of the input in each input pipeline (for
+  e.g. shard the input pipeline, use a different input source etc).
+  """
+
+  def __init__(self,
+               num_input_pipelines=1,
+               input_pipeline_id=0,
+               num_replicas_in_sync=1):
+    """Initializes an InputContext object.
+
+    Args:
+      num_input_pipelines: the number of input pipelines in a cluster.
+      input_pipeline_id: the current input pipeline id, should be an int in
+        [0,`num_input_pipelines`).
+      num_replicas_in_sync: the number of replicas that are in sync.
+    """
+    self._num_input_pipelines = num_input_pipelines
+    self._input_pipeline_id = input_pipeline_id
+    self._num_replicas_in_sync = num_replicas_in_sync
+
+  @property
+  def num_replicas_in_sync(self):
+    """Returns the number of compute replicas in sync."""
+    return self._num_replicas_in_sync
+
+  @property
+  def input_pipeline_id(self):
+    """Returns the input pipeline ID."""
+    return self._input_pipeline_id
+
+  @property
+  def num_input_pipelines(self):
+    """Returns the number of input pipelines."""
+    return self._num_input_pipelines
+
+  def get_per_replica_batch_size(self, global_batch_size):
+    """Returns the per-replica batch size.
+
+    Args:
+      global_batch_size: the global batch size which should be divisible by
+        `num_replicas_in_sync`.
+
+    Returns:
+      the per-replica batch size.
+
+    Raises:
+      ValueError: if `global_batch_size` not divisible by
+        `num_replicas_in_sync`.
+    """
+    if global_batch_size % self._num_replicas_in_sync != 0:
+      raise ValueError("The `global_batch_size` %r is not divisible by "
+                       "`num_replicas_in_sync` %r " %
+                       (global_batch_size, self._num_replicas_in_sync))
+    return global_batch_size // self._num_replicas_in_sync
+
+
 # ------------------------------------------------------------------------------
 # Base classes for all distribution strategies.
 
@@ -534,8 +609,15 @@ class DistributionStrategy(object):
     _require_distribution_strategy_scope(self)
     return variable_scope.variable_creator_scope(create_colocated_variable)
 
-  def _call_dataset_fn(self, dataset_fn):
-    result = dataset_fn()
+  def _call_dataset_fn(self, dataset_fn, input_context=None):
+    """Call the `dataset_fn` with `input_context` as argument."""
+    # This method is invoked by both `make_input_fn_iterator` and
+    # `distribute_dataset`. The `dataset_fn` for the former one accepts an
+    # input_context while the latter one doesn't.
+    if input_context:
+      result = dataset_fn(input_context)
+    else:
+      result = dataset_fn()
     if not isinstance(result, dataset_ops.Dataset):
       raise ValueError(
           "dataset_fn() must return a tf.data.Dataset when using a "
@@ -580,7 +662,7 @@ class DistributionStrategy(object):
       return tf.data.Dataset.from_tensors([[1.]]).repeat()
     with distribution_strategy.scope():
       distributed_dataset = distribution_strategy.distribute_dataset(dataset_fn)
-      iterator = distributed_dataset.make_one_shot_iterator()
+      iterator = distributed_dataset.make_initializable_iterator()
       replica_results = distribution_strategy.call_for_each_replica(
           replica_fn, args=(iterator.get_next(),))
     ```
@@ -593,6 +675,46 @@ class DistributionStrategy(object):
     """
     raise NotImplementedError("must be implemented in descendants")
 
+  def make_input_fn_iterator(self,
+                             input_fn,
+                             replication_mode=InputReplicationMode.PER_WORKER):
+    """Returns an iterator split across replicas created from an input function.
+
+    The `input_fn` should take an `InputContext` object where information about
+    input sharding can be accessed:
+
+    ```
+    def input_fn(input_context):
+      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
+      return d.shard(input_context.num_input_pipelines,
+                     input_context.input_pipeline_id)
+    with distribution_strategy.scope():
+      iterator = distribution_strategy.make_input_fn_iterator(
+          input_fn)
+      replica_results = distribution_strategy.call_for_each_replica(
+          replica_fn, iterator.get_next())
+    ```
+
+    Args:
+      input_fn: A function that returns a `tf.data.Dataset`. This function is
+        expected to take an `InputContext` object.
+      replication_mode: an enum value of `InputReplicationMode`. Only
+        `PER_WORKER` is supported currently.
+
+    Returns:
+      An iterator object that can be initialized and fetched next element.
+    """
+    if replication_mode != InputReplicationMode.PER_WORKER:
+      raise ValueError(
+          "Input replication mode not supported: %r" % replication_mode)
+    return self._make_input_fn_iterator(
+        input_fn, replication_mode=replication_mode)
+
+  def _make_input_fn_iterator(self,
+                              input_fn,
+                              replication_mode=InputReplicationMode.PER_WORKER):
+    raise NotImplementedError("must be implemented in descendants")
+
   def broadcast(self, tensor, destinations=None):
     """Mirror a tensor on one device to all worker devices.
 
@@ -1193,6 +1315,11 @@ class _DefaultDistributionStrategy(DistributionStrategy):
   def distribute_dataset(self, dataset_fn):
     return self._call_dataset_fn(dataset_fn)
 
+  def _make_input_fn_iterator(self,
+                              input_fn,
+                              replication_mode=InputReplicationMode.PER_WORKER):
+    return self._call_dataset_fn(input_fn, InputContext())
+
   def _broadcast(self, tensor, destinations):
     if destinations is None:
       return tensor
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index 0a7bbd5687..20b6d4f977 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -20,11 +20,11 @@ from __future__ import print_function
 
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
-from tensorflow.python.training import distribute
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import distribution_strategy_context
 
 
-class _TestReplicaContext(distribute.ReplicaContext):
+class _TestReplicaContext(distribute_lib.ReplicaContext):
 
   def merge_call(self, fn, *args, **kwargs):
     return kwargs["test_arg"]
@@ -38,7 +38,7 @@ def _get_test_variable(name, synchronization, aggregation):
   }
 
 
-class _TestStrategy(distribute.DistributionStrategy):
+class _TestStrategy(distribute_lib.DistributionStrategy):
 
   def _call_for_each_replica(self, fn, args, kwargs):
     with _TestReplicaContext(self, replica_id=0):
@@ -144,5 +144,22 @@ class DefaultDistributionStrategyTest(test.TestCase):
     _assert_in_default_state(self)
 
 
+class InputContextTest(test.TestCase):
+
+  def testProperties(self):
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=2, input_pipeline_id=1, num_replicas_in_sync=6)
+    self.assertEqual(6, input_context.num_replicas_in_sync)
+    self.assertEqual(1, input_context.input_pipeline_id)
+    self.assertEqual(2, input_context.num_input_pipelines)
+
+  def testPerReplicaBatchSize(self):
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=2, input_pipeline_id=1, num_replicas_in_sync=6)
+    self.assertEqual(2, input_context.get_per_replica_batch_size(12))
+    with self.assertRaises(ValueError):
+      input_context.get_per_replica_batch_size(13)
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 9a608da83605d162be891e69db903f581ca9566b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 01:02:45 -0800
Subject: [PATCH 0287/1554] compat: Update forward compatibility horizon to
 2018-11-15

PiperOrigin-RevId: 221581897
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 6a20e63e89..eea6567774 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 14)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 15)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From cfdfcc311c10eacad1c9a6506c53987372b132f3 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Thu, 15 Nov 2018 04:27:12 -0800
Subject: [PATCH 0288/1554] Rename `ReplicaContext.replica_id` to
 `replica_id_in_sync_group`.

PiperOrigin-RevId: 221602399
---
 .../python/keras_optimizer_v2_test.py         |  6 +--
 .../distribute/python/mirrored_strategy.py    |  3 +-
 .../python/mirrored_strategy_multigpu_test.py | 54 +++++++------------
 .../python/mirrored_strategy_test.py          |  9 ++--
 .../distribute/python/one_device_strategy.py  |  2 +-
 .../python/parameter_server_strategy_test.py  |  8 +--
 .../distribute/python/strategy_test_lib.py    | 14 ++---
 .../contrib/distribute/python/tpu_strategy.py |  5 +-
 tensorflow/python/ops/summary_op_util.py      |  3 +-
 tensorflow/python/training/distribute.py      | 14 ++---
 tensorflow/python/training/distribute_test.py |  2 +-
 .../training/distribution_strategy_context.py |  2 +-
 12 files changed, 53 insertions(+), 69 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 28f76ffadf..402a2cfcdf 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -47,7 +47,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
-from tensorflow.python.training import distribution_strategy_context
+from tensorflow.python.training import distribution_strategy_context as ds_context
 
 
 class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase):
@@ -272,9 +272,9 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
 
 
 def _replica_id():
-  # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
+  # TODO(cjfj): Return `replica_id_...` directly, once it is a `Tensor`.
   return constant_op.constant(
-      distribution_strategy_context.get_replica_context().replica_id)
+      ds_context.get_replica_context().replica_id_in_sync_group)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 74d69abcfd..b8d4fea298 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -829,4 +829,5 @@ class MirroredReplicaContext(distribute_lib.ReplicaContext):
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
-    return [self._distribution_strategy.worker_devices[self._replica_id]]
+    ds = self._distribution_strategy
+    return [ds.worker_devices[self._replica_id_in_sync_group]]
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 23379a72d9..463459bb23 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -48,7 +48,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import device_util
-from tensorflow.python.training import distribution_strategy_context
+from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import optimizer as optimizer_lib
 from tensorflow.python.training import server_lib
@@ -183,8 +183,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       # This variable should be created only once across the threads because of
       # special variable_creator functions used by `dist.call_for_each_replica`.
       v = variable_scope.variable(1.0, name="foo")
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -201,8 +200,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     def model_fn():
       v = variable_scope.variable(1.0)
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -222,8 +220,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       vs = []
       for i in range(5):
         vs.append(variable_scope.variable(1.0, name="foo" + str(i)))
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return vs
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -245,8 +242,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       vs.append(variable_scope.variable(1.0, name="foo_1/bar"))
       vs.append(variable_scope.variable(1.0, name="foo_1/bar_1"))
       vs.append(variable_scope.variable(1.0, name="foo/bar_1"))
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return vs
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -269,8 +265,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     def model_fn():
       replica_id = self.evaluate(_replica_id())
       v = variable_scope.variable(1.0, name="foo_" + str(replica_id))
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -292,8 +287,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         layer2 = core.Dense(1)
         layer2(features)
         # This will pause the current thread, and execute the other thread.
-        distribution_strategy_context.get_replica_context().merge_call(
-            lambda _: _)
+        ds_context.get_replica_context().merge_call(lambda _: _)
         layer3 = core.Dense(1)
         layer3(features)
         return [(layer1.kernel, layer1.bias),
@@ -330,8 +324,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       with variable_scope.variable_scope("common"):
         v1 = variable_scope.variable(1.0, name="var1")
         # This will pause the current thread, and execute the other thread.
-        distribution_strategy_context.get_replica_context().merge_call(
-            lambda _: _)
+        ds_context.get_replica_context().merge_call(lambda _: _)
         v2 = variable_scope.variable(
             1.0,
             name="var2",
@@ -374,8 +367,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       with variable_scope.variable_scope("common"):
         v1 = variable_scope.get_variable("var1", [1])
         # This will pause the current thread, and execute the other thread.
-        distribution_strategy_context.get_replica_context().merge_call(
-            lambda _: _)
+        ds_context.get_replica_context().merge_call(lambda _: _)
         v2 = variable_scope.get_variable(
             "var2", [1],
             synchronization=variable_scope.VariableSynchronization.ON_READ,
@@ -564,8 +556,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     def model_fn():
       v = variable_scope.variable(1.0, name="foo")
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -582,8 +573,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     def model_fn(name):
       v = variable_scope.variable(1.0, name=name)
-      distribution_strategy_context.get_replica_context().merge_call(
-          lambda _: _)
+      ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -683,8 +673,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     def model_fn():
       with ops.name_scope("foo"):
         a = constant_op.constant(1.0, name="a")
-        distribution_strategy_context.get_replica_context().merge_call(
-            lambda _: _)
+        ds_context.get_replica_context().merge_call(lambda _: _)
         b = constant_op.constant(1.0, name="b")
       return a, b
 
@@ -705,8 +694,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     def model_fn():
       with ops.name_scope(None, "foo"):
         a = constant_op.constant(1.0, name="a")
-        distribution_strategy_context.get_replica_context().merge_call(
-            lambda _: _)
+        ds_context.get_replica_context().merge_call(lambda _: _)
         b = constant_op.constant(2.0, name="b")
       return a, b
 
@@ -734,8 +722,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     def model_fn():
       b = variable_scope.variable(1.0, name="b")
       with ops.name_scope("foo"):
-        c = distribution_strategy_context.get_replica_context().merge_call(
-            in_cross_replica)
+        c = ds_context.get_replica_context().merge_call(in_cross_replica)
       return b, c
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -767,8 +754,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     def model_fn():
       b = variable_scope.get_variable("b", [1])
       with ops.name_scope("foo"):
-        c = distribution_strategy_context.get_replica_context().merge_call(
-            in_cross_replica)
+        c = ds_context.get_replica_context().merge_call(in_cross_replica)
       return b, c
 
     dist = mirrored_strategy.MirroredStrategy(
@@ -951,7 +937,7 @@ class MirroredVariableUpdateTest(test.TestCase):
 
       def model_fn():
         value = math_ops.cast(
-            distribution_strategy_context.get_replica_context().replica_id,
+            ds_context.get_replica_context().replica_id_in_sync_group,
             mirrored_var.dtype)
         return mirrored_var.assign(value)
 
@@ -1025,7 +1011,7 @@ class MirroredVariableUpdateTest(test.TestCase):
 
       def model_fn():
         value = math_ops.cast(
-            distribution_strategy_context.get_replica_context().replica_id,
+            ds_context.get_replica_context().replica_id_in_sync_group,
             mirrored_var.dtype)
         return mirrored_var.assign_add(value)
 
@@ -1091,7 +1077,7 @@ class MirroredVariableUpdateTest(test.TestCase):
 
       def model_fn():
         value = math_ops.cast(
-            distribution_strategy_context.get_replica_context().replica_id,
+            ds_context.get_replica_context().replica_id_in_sync_group,
             mirrored_var.dtype)
         return mirrored_var.assign_sub(value)
 
@@ -1463,9 +1449,9 @@ class MultiWorkerMirroredStrategyTestWithChief(
 
 
 def _replica_id():
-  # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
+  # TODO(cjfj): Return `replica_id_...` directly, once it is a `Tensor`.
   return constant_op.constant(
-      distribution_strategy_context.get_replica_context().replica_id)
+      ds_context.get_replica_context().replica_id_in_sync_group)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
index 001ee89c82..79318c3e2d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -26,7 +26,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.training import distribution_strategy_context
+from tensorflow.python.training import distribution_strategy_context as ds_context
 
 
 class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
@@ -87,8 +87,7 @@ class VariableCreatorStackTest(test.TestCase):
         v = variable_scope.variable(1.0)
 
         # This will pause the current thread, and execute the other thread.
-        distribution_strategy_context.get_replica_context().merge_call(
-            lambda _: _)
+        ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     def main_thread_creator(next_creator, *args, **kwargs):
@@ -106,9 +105,9 @@ class VariableCreatorStackTest(test.TestCase):
 
 
 def _replica_id():
-  # TODO(cjfj): Return `replica_id` directly, once it is a `Tensor`.
+  # TODO(cjfj): Return `replica_id_...` directly, once it is a `Tensor`.
   return constant_op.constant(
-      distribution_strategy_context.get_replica_context().replica_id)
+      ds_context.get_replica_context().replica_id_in_sync_group)
 
 
 class MultiWorkerMirroredStrategyTest(test.TestCase):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index cbd5d1ab40..a4ce956a9f 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -180,7 +180,7 @@ class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
 
   def __init__(self, distribution_strategy):
     distribute_lib.ReplicaContext.__init__(
-        self, distribution_strategy, replica_id=0)
+        self, distribution_strategy, replica_id_in_sync_group=0)
 
   @property
   def device(self):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index d25b5836c5..be42686322 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -44,7 +44,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import device_util
-from tensorflow.python.training import distribution_strategy_context
+from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import training_util
 
 CHIEF = run_config.TaskType.CHIEF
@@ -98,7 +98,7 @@ class ParameterServerStrategyTestBase(
         else:
           last_part_device = (
               'device:GPU:%d' %
-              distribution_strategy_context.get_replica_context().replica_id)
+              ds_context.get_replica_context().replica_id_in_sync_group)
 
         a = constant_op.constant(1.0)
         b = constant_op.constant(2.0)
@@ -265,7 +265,7 @@ class ParameterServerStrategyTestBase(
         else:
           replica_compute_device = (
               '/device:GPU:%d' %
-              distribution_strategy_context.get_replica_context().replica_id)
+              ds_context.get_replica_context().replica_id_in_sync_group)
         replica_compute_device = device_util.canonicalize(
             replica_compute_device)
 
@@ -274,7 +274,7 @@ class ParameterServerStrategyTestBase(
         else:
           replica_variable_device = (
               '/device:GPU:%d' %
-              distribution_strategy_context.get_replica_context().replica_id)
+              ds_context.get_replica_context().replica_id_in_sync_group)
         replica_variable_device = device_util.canonicalize(
             replica_variable_device)
 
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index ceea898566..7c6f7123da 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -29,7 +29,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribution_strategy_context
+from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import optimizer
 
 
@@ -46,8 +46,7 @@ def _raise_exception_fn(_=None):
 # Must be the argument to a distribution.call_for_each_replica() call, calls a
 # get_replica_context().merge_call() that raises an exception.
 def _merge_raises_fn():
-  distribution_strategy_context.get_replica_context().merge_call(
-      _raise_exception_fn)
+  ds_context.get_replica_context().merge_call(_raise_exception_fn)
 
 
 # Must be the argument to a get_replica_context().merge_call() call, calls
@@ -60,8 +59,7 @@ def _call_raises_fn(dist):
 # calls a get_replica_context().merge_call() that calls a
 # call_for_each_replica() that raises an exception.
 def _merge_call_raises_fn():
-  distribution_strategy_context.get_replica_context().merge_call(
-      _call_raises_fn)
+  ds_context.get_replica_context().merge_call(_call_raises_fn)
 
 
 # Must be the argument to a get_replica_context().merge_call() call, calls
@@ -75,8 +73,7 @@ def _call_merge_raises_fn(dist):
 # get_replica_context().merge_call() that calls a call_for_each_replica() that
 # calls a get_replica_context().merge_call() that raises an exception.
 def _merge_call_merge_raises_fn():
-  distribution_strategy_context.get_replica_context().merge_call(
-      _call_merge_raises_fn)
+  ds_context.get_replica_context().merge_call(_call_merge_raises_fn)
 
 
 class DistributionTestBase(test.TestCase):
@@ -193,8 +190,7 @@ class DistributionTestBase(test.TestCase):
       expected_devices = [False] * len(d.worker_devices)
 
       def mark_devices_fn():
-        replica_id = (
-            distribution_strategy_context.get_replica_context().replica_id)
+        replica_id = ds_context.get_replica_context().replica_id_in_sync_group
         self.assertLess(replica_id, len(d.worker_devices))
         self.assertFalse(expected_devices[replica_id])
         expected_devices[replica_id] = True
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 094a57c4e6..53fe8ae244 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -587,7 +587,7 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
   # TODO(sourabhbajaj): Call for each tower should be updating this.
   def __init__(self, distribution_strategy):
     distribute_lib.ReplicaContext.__init__(
-        self, distribution_strategy, replica_id=0)
+        self, distribution_strategy, replica_id_in_sync_group=0)
 
   @property
   def device(self):
@@ -596,4 +596,5 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
-    return [self._distribution_strategy.worker_devices[self._replica_id]]
+    ds = self._distribution_strategy
+    return [ds.worker_devices[self._replica_id_in_sync_group]]
diff --git a/tensorflow/python/ops/summary_op_util.py b/tensorflow/python/ops/summary_op_util.py
index 14aa44a920..fe659532b0 100644
--- a/tensorflow/python/ops/summary_op_util.py
+++ b/tensorflow/python/ops/summary_op_util.py
@@ -50,7 +50,8 @@ def skip_summary():
   # alternatives to override default behavior. (e.g. run on last replica,
   # compute sum or mean across replicas).
   replica_context = distribution_strategy_context.get_replica_context()
-  return replica_context and replica_context.replica_id > 0
+  # TODO(cjfj): Also check is sync group ID > 0?
+  return replica_context and replica_context.replica_id_in_sync_group > 0
 
 
 def clean_tag(name):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 2ff6ff9bda..ceb73ebc4e 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -814,7 +814,7 @@ class DistributionStrategy(object):
     """Run `fn` once per replica.
 
     `fn` may call `tf.get_replica_context()` to access methods such as
-    `replica_id()` and `merge_call()`.
+    `replica_id_in_sync_group` and `merge_call()`.
 
     `merge_call()` is used to communicate between the replicas and
     re-enter the cross-replica context. All replicas pause their execution
@@ -832,7 +832,7 @@ class DistributionStrategy(object):
     # Called once per replica in `distribution`, in a "replica" context.
     def fn(three):
       replica_ctx = tf.get_replica_context()
-      v = three + replica_ctx.replica_id
+      v = three + replica_ctx.replica_id_in_sync_group
       # Computes the sum of the `v` values across all replicas.
       s = replica_ctx.merge_call(merge_fn, args=(v,))
       return s + v
@@ -1182,11 +1182,11 @@ class DistributionStrategy(object):
 class ReplicaContext(object):
   """DistributionStrategy API inside a `call_for_each_replica()` call."""
 
-  def __init__(self, distribution_strategy, replica_id):
+  def __init__(self, distribution_strategy, replica_id_in_sync_group):
     self._distribution_strategy = distribution_strategy
     self._thread_context = distribution_strategy_context._InReplicaThreadMode(  # pylint: disable=protected-access
         self)
-    self._replica_id = replica_id
+    self._replica_id_in_sync_group = replica_id_in_sync_group
 
   def __enter__(self):
     _push_per_thread_mode(self._thread_context)
@@ -1255,10 +1255,10 @@ class ReplicaContext(object):
     return self._distribution_strategy.num_replicas_in_sync
 
   @property
-  def replica_id(self):
+  def replica_id_in_sync_group(self):
     """Which replica is being defined, a number from 0 to `num_replicas - 1`."""
     require_replica_context(self)
-    return self._replica_id
+    return self._replica_id_in_sync_group
 
   @property
   def distribution_strategy(self):
@@ -1327,7 +1327,7 @@ class _DefaultDistributionStrategy(DistributionStrategy):
       raise NotImplementedError("TODO")
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    with ReplicaContext(self, replica_id=0):
+    with ReplicaContext(self, replica_id_in_sync_group=0):
       return fn(*args, **kwargs)
 
   def _reduce(self, reduce_op, value, destinations):
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index 20b6d4f977..39437ad9b5 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -41,7 +41,7 @@ def _get_test_variable(name, synchronization, aggregation):
 class _TestStrategy(distribute_lib.DistributionStrategy):
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    with _TestReplicaContext(self, replica_id=0):
+    with _TestReplicaContext(self, replica_id_in_sync_group=0):
       return fn(*args, **kwargs)
 
   def _create_variable(self, next_creator, *args, **kwargs):
diff --git a/tensorflow/python/training/distribution_strategy_context.py b/tensorflow/python/training/distribution_strategy_context.py
index 278f35b97e..9a33f5f416 100644
--- a/tensorflow/python/training/distribution_strategy_context.py
+++ b/tensorflow/python/training/distribution_strategy_context.py
@@ -196,7 +196,7 @@ def _get_default_distribution_strategy():
 def _get_default_replica_context():
   if _defaults["replica_context"] is None:
     _defaults["replica_context"] = distribute_lib.ReplicaContext(
-        _get_default_distribution_strategy(), replica_id=0)
+        _get_default_distribution_strategy(), replica_id_in_sync_group=0)
   return _defaults["replica_context"]
 
 
-- 
GitLab


From fa48fd8be8fb632343e7c15ca82c0193c265e07d Mon Sep 17 00:00:00 2001
From: Yash Gaurkar <yashgaurkar@gmail.com>
Date: Thu, 15 Nov 2018 19:47:03 +0530
Subject: [PATCH 0289/1554] Unicode error while testing

Error for inverted question mark : UnicodeEncodeError: 'ascii' codec can't encode character u'\xbf' in position 8: ordinal not in range(128)
Error for not adding u' ' : TypeError: normalize() argument 2 must be unicode, not str
---
 .../examples/nmt_with_attention/nmt_with_attention.ipynb  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
index 480777d948..66d52a7494 100644
--- a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
+++ b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
@@ -768,7 +768,7 @@
       },
       "outputs": [],
       "source": [
-        "translate('hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
+        "translate(u'hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
       ]
     },
     {
@@ -781,7 +781,7 @@
       },
       "outputs": [],
       "source": [
-        "translate('esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
+        "translate(u'esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
       ]
     },
     {
@@ -794,7 +794,7 @@
       },
       "outputs": [],
       "source": [
-        "translate('¿todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
+        "translate(u'todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
       ]
     },
     {
@@ -808,7 +808,7 @@
       "outputs": [],
       "source": [
         "# wrong translation\n",
-        "translate('trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
+        "translate(u'trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
       ]
     },
     {
-- 
GitLab


From 5704cba5777d254471dbcda619764713e7f79135 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 07:32:50 -0800
Subject: [PATCH 0290/1554] Make DatasetV2 an abc again.

PiperOrigin-RevId: 221620796
---
 tensorflow/python/data/ops/dataset_ops.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 7f0903c330..cf51fdffdd 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -54,6 +54,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 
 @tf_export("data.Dataset", v1=[])
+@six.add_metaclass(abc.ABCMeta)
 class DatasetV2(object):
   """Represents a potentially large set of elements.
 
-- 
GitLab


From 770561637857bf93947328d43497f0d4e37594c6 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Thu, 15 Nov 2018 07:44:18 -0800
Subject: [PATCH 0291/1554] Improve coverage of server_lib.cc testing

We add one more test method to cover the successful case. Testing now covers all lines except the LOG(ERROR) statement.

PiperOrigin-RevId: 221622246
---
 .../distributed_runtime/server_lib_test.cc    | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tensorflow/core/distributed_runtime/server_lib_test.cc b/tensorflow/core/distributed_runtime/server_lib_test.cc
index 29aafc15d6..460372523c 100644
--- a/tensorflow/core/distributed_runtime/server_lib_test.cc
+++ b/tensorflow/core/distributed_runtime/server_lib_test.cc
@@ -14,11 +14,32 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/distributed_runtime/server_lib.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 
+class TestServerFactory : public ServerFactory {
+ public:
+  bool AcceptsOptions(const ServerDef& server_def) override {
+    return server_def.protocol() == "test_protocol";
+  }
+
+  Status NewServer(const ServerDef& server_def,
+                   std::unique_ptr<ServerInterface>* out_server) override {
+    return Status::OK();
+  }
+};
+
+TEST(ServerLibTest, NewServerFactoryAccepts) {
+  ServerFactory::Register("TEST_SERVER", new TestServerFactory());
+  ServerDef server_def;
+  server_def.set_protocol("test_protocol");
+  std::unique_ptr<ServerInterface> server;
+  TF_EXPECT_OK(NewServer(server_def, &server));
+}
+
 TEST(ServerLibTest, NewServerNoFactoriesAccept) {
   ServerDef server_def;
   server_def.set_protocol("fake_protocol");
-- 
GitLab


From 1696d69b07f40a2a3d5c6b722a5990011af25842 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Nov 2018 08:18:59 -0800
Subject: [PATCH 0292/1554] Add a comment in tf_upgrade_v2.py to indicate order
 of arguments in self.function_reorders should match old order as opposed to
 new order.

PiperOrigin-RevId: 221626603
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index a6882e35d3..8de288d6ba 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -66,6 +66,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Functions that were reordered should be changed to the new keyword args
     # for safety, if positional arguments are used. If you have reversed the
     # positional arguments yourself, this could do the wrong thing.
+    # IMPORTANT: order here should correspond to OLD argument order.
+    # We just prepend "arg_name=" to all arguments in function calls.
     self.function_reorders = {
         "tf.argmax": ["input", "axis", "output_type", "name"],
         "tf.argmin": ["input", "axis", "output_type", "name"],
-- 
GitLab


From ec5a7049052b61d461c0a155a1cdb250ce0e8d9e Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 15 Nov 2018 08:31:44 -0800
Subject: [PATCH 0293/1554] Add tests for ast_edits.

PiperOrigin-RevId: 221628173
---
 tensorflow/tools/compatibility/BUILD          |  12 +
 .../tools/compatibility/ast_edits_test.py     | 396 ++++++++++++++++++
 2 files changed, 408 insertions(+)
 create mode 100644 tensorflow/tools/compatibility/ast_edits_test.py

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 5f619c4e62..f46e36bf32 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -14,6 +14,18 @@ py_library(
     srcs_version = "PY2AND3",
 )
 
+py_test(
+    name = "ast_edits_test",
+    srcs = ["ast_edits_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ast_edits",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "@six_archive//:six",
+    ],
+)
+
 py_binary(
     name = "tf_upgrade",
     srcs = ["tf_upgrade.py"],
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
new file mode 100644
index 0000000000..08f4ae3fcc
--- /dev/null
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -0,0 +1,396 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for ast_edits which is used in tf upgraders.
+
+All of the tests assume that we want to change from an API containing
+
+    def f(a, b, kw1, kw2): ...
+    def g(a, b, kw1, c, kw1_alias): ...
+    def g2(a, b, kw1, c, d, kw1_alias): ...
+    def h(a, kw1, kw2, kw1_alias, kw2_alias): ...
+
+and the changes to the API consist of renaming, reordering, and/or removing
+arguments. Thus, we want to be able to generate changes to produce each of the
+following new APIs:
+
+    def f(a, b, kw1, kw3): ...
+    def f(a, b, kw2, kw1): ...
+    def f(a, b, kw3, kw1): ...
+    def g(a, b, kw1, c): ...
+    def g(a, b, c, kw1): ...
+    def g2(a, b, kw1, c, d): ...
+    def g2(a, b, c, d, kw1): ...
+    def h(a, kw1, kw2): ...
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import six
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test as test_lib
+from tensorflow.tools.compatibility import ast_edits
+
+
+class NoUpdateSpec(ast_edits.APIChangeSpec):
+  """A specification of an API change which doesn't change anything."""
+
+  def __init__(self):
+    self.function_handle = {}
+    self.function_reorders = {}
+    self.function_keyword_renames = {}
+
+
+class RenameKeywordSpec(NoUpdateSpec):
+  """A specification where kw2 gets renamed to kw3.
+
+  The new API is
+
+    def f(a, b, kw1, kw3): ...
+
+  """
+
+  def __init__(self):
+    NoUpdateSpec.__init__(self)
+    self.update_renames()
+
+  def update_renames(self):
+    self.function_keyword_renames["f"] = {"kw2": "kw3"}
+
+
+class ReorderKeywordSpec(NoUpdateSpec):
+  """A specification where kw2 gets moved in front of kw1.
+
+  The new API is
+
+    def f(a, b, kw2, kw1): ...
+
+  """
+
+  def __init__(self):
+    NoUpdateSpec.__init__(self)
+    self.update_reorders()
+
+  def update_reorders(self):
+    # Note that these should be in the old order.
+    self.function_reorders["f"] = ["a", "b", "kw1", "kw2"]
+
+
+class ReorderAndRenameKeywordSpec(ReorderKeywordSpec, RenameKeywordSpec):
+  """A specification where kw2 gets moved in front of kw1 and is changed to kw3.
+
+  The new API is
+
+    def f(a, b, kw3, kw1): ...
+
+  """
+
+  def __init__(self):
+    ReorderKeywordSpec.__init__(self)
+    RenameKeywordSpec.__init__(self)
+    self.update_renames()
+    self.update_reorders()
+
+
+class RemoveDeprecatedAliasKeyword(NoUpdateSpec):
+  """A specification where kw1_alias is removed in g.
+
+  The new API is
+
+    def g(a, b, kw1, c): ...
+    def g2(a, b, kw1, c, d): ...
+
+  """
+
+  def __init__(self):
+    NoUpdateSpec.__init__(self)
+    self.function_keyword_renames["g"] = {"kw1_alias": "kw1"}
+    self.function_keyword_renames["g2"] = {"kw1_alias": "kw1"}
+
+
+class RemoveDeprecatedAliasAndReorderRest(RemoveDeprecatedAliasKeyword):
+  """A specification where kw1_alias is removed in g.
+
+  The new API is
+
+    def g(a, b, c, kw1): ...
+    def g2(a, b, c, d, kw1): ...
+
+  """
+
+  def __init__(self):
+    RemoveDeprecatedAliasKeyword.__init__(self)
+    # Note that these should be in the old order.
+    self.function_reorders["g"] = ["a", "b", "kw1", "c"]
+    self.function_reorders["g2"] = ["a", "b", "kw1", "c", "d"]
+
+
+class RemoveMultipleKeywordArguments(NoUpdateSpec):
+  """A specification where both keyword aliases are removed from h.
+
+  The new API is
+
+    def h(a, kw1, kw2): ...
+
+  """
+
+  def __init__(self):
+    NoUpdateSpec.__init__(self)
+    self.function_keyword_renames["h"] = {
+        "kw1_alias": "kw1",
+        "kw2_alias": "kw2",
+    }
+
+
+class TestAstEdits(test_util.TensorFlowTestCase):
+
+  def _upgrade(self, spec, old_file_text):
+    in_file = six.StringIO(old_file_text)
+    out_file = six.StringIO()
+    upgrader = ast_edits.ASTCodeUpgrader(spec)
+    count, report, errors = (
+        upgrader.process_opened_file("test.py", in_file,
+                                     "test_out.py", out_file))
+    return (count, report, errors), out_file.getvalue()
+
+  def testNoTransformIfNothingIsSupplied(self):
+    text = "f(a, b, kw1=c, kw2=d)\n"
+    _, new_text = self._upgrade(NoUpdateSpec(), text)
+    self.assertEqual(new_text, text)
+
+    text = "f(a, b, c, d)\n"
+    _, new_text = self._upgrade(NoUpdateSpec(), text)
+    self.assertEqual(new_text, text)
+
+  def testKeywordRename(self):
+    """Test that we get the expected result if renaming kw2 to kw3."""
+    text = "f(a, b, kw1=c, kw2=d)\n"
+    expected = "f(a, b, kw1=c, kw3=d)\n"
+    _, new_text = self._upgrade(RenameKeywordSpec(), text)
+    self.assertEqual(new_text, expected)
+
+    # No keywords specified, no reordering, so we should get input as output
+    text = "f(a, b, c, d)\n"
+    _, new_text = self._upgrade(RenameKeywordSpec(), text)
+    self.assertEqual(new_text, text)
+
+  def testKeywordReorder(self):
+    """Test that we get the expected result if kw2 is now before kw1."""
+    text = "f(a, b, kw1=c, kw2=d)\n"
+    acceptable_outputs = [
+        # No change is a valid output
+        text,
+        # Just reordering the kw.. args is also ok
+        "f(a, b, kw2=d, kw1=c)\n",
+        # Also cases where all arguments are fully specified are allowed
+        "f(a=a, b=b, kw1=c, kw2=d)\n",
+        "f(a=a, b=b, kw2=d, kw1=c)\n",
+    ]
+    _, new_text = self._upgrade(ReorderKeywordSpec(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # Keywords are reordered, so we should reorder arguments too
+    text = "f(a, b, c, d)\n"
+    acceptable_outputs = [
+        "f(a, b, d, c)\n",
+        "f(a=a, b=b, kw1=c, kw2=d)\n",
+        "f(a=a, b=b, kw2=d, kw1=c)\n",
+    ]
+    _, new_text = self._upgrade(ReorderKeywordSpec(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+  def testKeywordReorderAndRename(self):
+    """Test that we get the expected result if kw2 is renamed and moved."""
+    text = "f(a, b, kw1=c, kw2=d)\n"
+    acceptable_outputs = [
+        "f(a, b, kw3=d, kw1=c)\n",
+        "f(a=a, b=b, kw1=c, kw3=d)\n",
+        "f(a=a, b=b, kw3=d, kw1=c)\n",
+    ]
+    _, new_text = self._upgrade(ReorderAndRenameKeywordSpec(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # Keywords are reordered, so we should reorder arguments too
+    text = "f(a, b, c, d)\n"
+    acceptable_outputs = [
+        "f(a, b, d, c)\n",
+        "f(a=a, b=b, kw1=c, kw3=d)\n",
+        "f(a=a, b=b, kw3=d, kw1=c)\n",
+    ]
+    _, new_text = self._upgrade(ReorderAndRenameKeywordSpec(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+  def testRemoveDeprecatedKeywordAlias(self):
+    """Test that we get the expected result if a keyword alias is removed."""
+    text = "g(a, b, kw1=x, c=c)\n"
+    acceptable_outputs = [
+        # Not using deprecated alias, so original is ok
+        text,
+        "g(a=a, b=b, kw1=x, c=c)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # No keyword used, should be no change
+    text = "g(a, b, x, c)\n"
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertEqual(new_text, text)
+
+    # If we used the alias, it should get renamed
+    text = "g(a, b, kw1_alias=x, c=c)\n"
+    acceptable_outputs = [
+        "g(a, b, kw1=x, c=c)\n",
+        "g(a, b, c=c, kw1=x)\n",
+        "g(a=a, b=b, kw1=x, c=c)\n",
+        "g(a=a, b=b, c=c, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # It should get renamed even if it's last
+    text = "g(a, b, c=c, kw1_alias=x)\n"
+    acceptable_outputs = [
+        "g(a, b, kw1=x, c=c)\n",
+        "g(a, b, c=c, kw1=x)\n",
+        "g(a=a, b=b, kw1=x, c=c)\n",
+        "g(a=a, b=b, c=c, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+  def testRemoveDeprecatedKeywordAndReorder(self):
+    """Test for when a keyword alias is removed and args are reordered."""
+    text = "g(a, b, kw1=x, c=c)\n"
+    acceptable_outputs = [
+        "g(a, b, c=c, kw1=x)\n",
+        "g(a=a, b=b, kw1=x, c=c)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # Keywords are reordered, so we should reorder arguments too
+    text = "g(a, b, x, c)\n"
+    # Don't accept an output which doesn't reorder c and d
+    acceptable_outputs = [
+        "g(a, b, c, x)\n",
+        "g(a=a, b=b, kw1=x, c=c)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # If we used the alias, it should get renamed
+    text = "g(a, b, kw1_alias=x, c=c)\n"
+    acceptable_outputs = [
+        "g(a, b, kw1=x, c=c)\n",
+        "g(a, b, c=c, kw1=x)\n",
+        "g(a=a, b=b, kw1=x, c=c)\n",
+        "g(a=a, b=b, c=c, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # It should get renamed and reordered even if it's last
+    text = "g(a, b, c=c, kw1_alias=x)\n"
+    acceptable_outputs = [
+        "g(a, b, kw1=x, c=c)\n",
+        "g(a, b, c=c, kw1=x)\n",
+        "g(a=a, b=b, kw1=x, c=c)\n",
+        "g(a=a, b=b, c=c, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+  def testRemoveDeprecatedKeywordAndReorder2(self):
+    """Same as testRemoveDeprecatedKeywordAndReorder but on g2 (more args)."""
+    text = "g2(a, b, kw1=x, c=c, d=d)\n"
+    acceptable_outputs = [
+        "g2(a, b, c=c, d=d, kw1=x)\n",
+        "g2(a=a, b=b, kw1=x, c=c, d=d)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # Keywords are reordered, so we should reorder arguments too
+    text = "g2(a, b, x, c, d)\n"
+    # Don't accept an output which doesn't reorder c and d
+    acceptable_outputs = [
+        "g2(a, b, c, d, x)\n",
+        "g2(a=a, b=b, kw1=x, c=c, d=d)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasAndReorderRest(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # If we used the alias, it should get renamed
+    text = "g2(a, b, kw1_alias=x, c=c, d=d)\n"
+    acceptable_outputs = [
+        "g2(a, b, kw1=x, c=c, d=d)\n",
+        "g2(a, b, c=c, d=d, kw1=x)\n",
+        "g2(a=a, b=b, kw1=x, c=c, d=d)\n",
+        "g2(a=a, b=b, c=c, d=d, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # It should get renamed and reordered even if it's not in order
+    text = "g2(a, b, d=d, c=c, kw1_alias=x)\n"
+    acceptable_outputs = [
+        "g2(a, b, kw1=x, c=c, d=d)\n",
+        "g2(a, b, c=c, d=d, kw1=x)\n",
+        "g2(a, b, d=d, c=c, kw1=x)\n",
+        "g2(a=a, b=b, kw1=x, c=c, d=d)\n",
+        "g2(a=a, b=b, c=c, d=d, kw1=x)\n",
+        "g2(a=a, b=b, d=d, c=c, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveDeprecatedAliasKeyword(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+  def testRemoveMultipleKeywords(self):
+    """Remove multiple keywords at once."""
+    # Not using deprecated keywords -> no rename
+    text = "h(a, kw1=x, kw2=y)\n"
+    _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
+    self.assertEqual(new_text, text)
+
+    # Using positional arguments (in proper order) -> no change
+    text = "h(a, x, y)\n"
+    _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
+    self.assertEqual(new_text, text)
+
+    # Use only the old names, in order
+    text = "h(a, kw1_alias=x, kw2_alias=y)\n"
+    acceptable_outputs = [
+        "h(a, x, y)\n",
+        "h(a, kw1=x, kw2=y)\n",
+        "h(a=a, kw1=x, kw2=y)\n",
+        "h(a, kw2=y, kw1=x)\n",
+        "h(a=a, kw2=y, kw1=x)\n",
+    ]
+    _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # Use only the old names, in reverse order, should give one of same outputs
+    text = "h(a, kw2_alias=y, kw1_alias=x)\n"
+    _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+    # Mix old and new names
+    text = "h(a, kw1=x, kw2_alias=y)\n"
+    _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
+
+if __name__ == "__main__":
+  test_lib.main()
-- 
GitLab


From 82eb2d85c1f6ae56cc6c092bdc3f386278241e6e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 08:50:47 -0800
Subject: [PATCH 0294/1554] Automated rollback of commit
 89fa3c5ed66a4e543ba01c31febf3edb417808d0

PiperOrigin-RevId: 221630636
---
 tensorflow/workspace.bzl         |  9 +++---
 third_party/eigen_reshaped.patch | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 4 deletions(-)
 create mode 100644 third_party/eigen_reshaped.patch

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b4f7aba046..42d92f7b49 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "1e045bef75e9b17d459b60cc30b34408f3fdab300c5053d3919d1a5921f3c86a",
-        strip_prefix = "eigen-eigen-af2071407280",
+        patch_file = clean_dep("//third_party:eigen_reshaped.patch"),
+        sha256 = "d66cec3b54b3dfaa4666c1d49481a7197f93fc078cd53c54e2b4a8893a529c9f",
+        strip_prefix = "eigen-eigen-b4890dc6bc34",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz",
         ],
     )
 
diff --git a/third_party/eigen_reshaped.patch b/third_party/eigen_reshaped.patch
new file mode 100644
index 0000000000..7acfdcf9fe
--- /dev/null
+++ b/third_party/eigen_reshaped.patch
@@ -0,0 +1,48 @@
+--- a/Eigen/src/Core/util/ReshapedHelper.h	(date 1541195478000)
++++ b/Eigen/src/Core/util/ReshapedHelper.h	(date 1541195478000)
+@@ -39,6 +39,11 @@
+   return total/other;
+ }
+
++template<int Flags, int Order>
++struct get_compiletime_reshape_order {
++  enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order };
++};
++
+ }
+
+ } // end namespace Eigen
+--- a/Eigen/src/plugins/ReshapedMethods.h	(date 1541195254000)
++++ b/Eigen/src/plugins/ReshapedMethods.h	(date 1541195254000)
+@@ -105,13 +105,13 @@
+ inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,
+                 internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value,
+                 internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value,
+-                (Order==AutoOrder?Flags&RowMajorBit:Order)>
++                internal::get_compiletime_reshape_order<Flags,Order>::value>
+ reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST
+ {
+   return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,
+                   internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value,
+                   internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value,
+-                  (Order==AutoOrder?Flags&RowMajorBit:Order)>
++                  internal::get_compiletime_reshape_order<Flags,Order>::value>
+                 (derived(),
+                  internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()),
+                  internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size()));
+@@ -128,11 +128,13 @@
+
+ template<int Order>
+ EIGEN_DEVICE_FUNC
+-inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, (Order==AutoOrder?Flags&RowMajorBit:Order)>
++inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1,
++                internal::get_compiletime_reshape_order<Flags,Order>::value>
+ reshaped() EIGEN_RESHAPED_METHOD_CONST
+ {
+   EIGEN_STATIC_ASSERT(Order==RowMajor || Order==ColMajor || Order==AutoOrder, INVALID_TEMPLATE_PARAMETER);
+-  return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, (Order==AutoOrder?Flags&RowMajorBit:Order)>
++  return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1,
++                  internal::get_compiletime_reshape_order<Flags,Order>::value>
+                 (derived(), size(), 1);
+ }
+ 
\ No newline at end of file
-- 
GitLab


From a30288517169434adc83a5c989984b912850af6b Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 15 Nov 2018 09:25:11 -0800
Subject: [PATCH 0295/1554] Add the default worker name to `AbortedError`
 messages.

This should help track down which worker is "this worker" in "this worker has restarted" errors.

PiperOrigin-RevId: 221635865
---
 tensorflow/contrib/verbs/rdma.cc                   | 6 ++++--
 tensorflow/core/distributed_runtime/session_mgr.cc | 4 +++-
 tensorflow/core/distributed_runtime/worker.cc      | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/verbs/rdma.cc b/tensorflow/contrib/verbs/rdma.cc
index f7c979e863..9db80f6b57 100644
--- a/tensorflow/contrib/verbs/rdma.cc
+++ b/tensorflow/contrib/verbs/rdma.cc
@@ -30,7 +30,6 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/session_mgr.h"
-#include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/framework/rendezvous.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -1028,7 +1027,10 @@ Status RdmaTensorResponse::PrepareRecvTensor(
     return errors::Aborted(
         "RecvTensor expects a different device incarnation: ",
         parsed.src_incarnation, " vs. ", (*src_dev)->attributes().incarnation(),
-        ". Your worker job was probably restarted. Check your "
+        ". Your worker job (\"",
+        channel_->adapter_->worker_env_->session_mgr->LegacySession()
+            ->worker_name,
+        "\") was probably restarted. Check your "
         "worker job for the reason why it was restarted.");
   }
 
diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc
index 95b31c6991..38833bd202 100644
--- a/tensorflow/core/distributed_runtime/session_mgr.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr.cc
@@ -122,7 +122,9 @@ Status SessionMgr::WorkerSessionForSessionLocked(
     auto it = sessions_.find(session_handle);
     if (it == sessions_.end()) {
       return errors::Aborted("Session handle is not found: ", session_handle,
-                             ". Possibly this worker just restarted.");
+                             ". Possibly this worker (\"",
+                             legacy_session_->worker_name,
+                             "\") just restarted.");
     } else {
       *out_session = it->second;
     }
diff --git a/tensorflow/core/distributed_runtime/worker.cc b/tensorflow/core/distributed_runtime/worker.cc
index 079c09859f..f42143e582 100644
--- a/tensorflow/core/distributed_runtime/worker.cc
+++ b/tensorflow/core/distributed_runtime/worker.cc
@@ -438,7 +438,9 @@ Status Worker::PrepareRecvTensor(const Rendezvous::ParsedKey& parsed,
     return errors::Aborted(
         "RecvTensor expects a different device incarnation: ",
         parsed.src_incarnation, " vs. ", (*src_dev)->attributes().incarnation(),
-        ". Your worker job was probably restarted. Check your "
+        ". Your worker job (\"",
+        env_->session_mgr->LegacySession()->worker_name,
+        "\") was probably restarted. Check your "
         "worker job for the reason why it was restarted.");
   }
 
-- 
GitLab


From 299469c1ebc82d02dcca70be58f9bc21fdb74112 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 15 Nov 2018 09:34:56 -0800
Subject: [PATCH 0296/1554] cond_v2: make FakeParam output tensor with correct
 size and device.

Allocating tensors of the expected size is necessary for adding them
to TensorLists in the case of cond_v2 nested in while_v2.

PiperOrigin-RevId: 221637330
---
 tensorflow/core/kernels/functional_ops.cc     | 31 ++++++++--
 .../kernel_tests/control_flow_ops_py_test.py  | 57 +++++++++++++++++++
 2 files changed, 82 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/functional_ops.cc b/tensorflow/core/kernels/functional_ops.cc
index 1529d2e336..5ecb203cbc 100644
--- a/tensorflow/core/kernels/functional_ops.cc
+++ b/tensorflow/core/kernels/functional_ops.cc
@@ -526,21 +526,40 @@ REGISTER_KERNEL_BUILDER(Name("For")
                             .HostMemory("delta"),
                         ForOp);
 
+// FakeParamOp allocates a tensor with a shape conforming to the expected
+// output. This is necessary if the value will be stored in a while_loop's
+// TensorList. The output is otherwise not expected to be consumed by anything
+// else.
 class FakeParamOp : public OpKernel {
  public:
   explicit FakeParamOp(OpKernelConstruction* context) : OpKernel(context) {
-    OP_REQUIRES_OK(context, context->GetAttr("dtype", &dtype_));
+    DataType dtype;
+    OP_REQUIRES_OK(context, context->GetAttr("dtype", &dtype));
+
+    // Set shape to the specified shape, setting unknown dimensions to empty.
+    // If the specified shape is unknown, leave as an empty shape.
+    TensorShape shape;
+    PartialTensorShape partial_shape;
+    OP_REQUIRES_OK(context, context->GetAttr("shape", &partial_shape));
+    if (!partial_shape.unknown_rank()) {
+      for (int64 d : partial_shape.dim_sizes()) {
+        shape.AddDim(d == -1 ? 0 : d);
+      }
+    }
+
+    // Create a persistent tensor that we can repeatedly return to save memory.
+    // TODO(b/119612758): add optimization to prevent sending this across
+    // devices on each Compute() call.
+    OP_REQUIRES_OK(context, context->allocate_persistent(
+                                dtype, shape, &value_handle_, nullptr));
   }
 
   void Compute(OpKernelContext* context) override {
-    // We must produce something (only Switch and Recvs are allowed to output
-    // dead tensors). This output is not expected to be consumed by anything.
-    Tensor output_tensor(dtype_, TensorShape({}));
-    context->set_output(0, output_tensor);
+    context->set_output(0, *value_handle_.AccessTensor(context));
   }
 
  private:
-  DataType dtype_;
+  PersistentTensor value_handle_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("FakeParam").Device(DEVICE_CPU), FakeParamOp);
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 5b78113f4f..ec0492dd14 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -711,6 +711,34 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(980.0, r.eval(feed_dict={c: 1}))
       self.assertAllEqual(30.0, r.eval(feed_dict={c: 3}))
 
+  def testCondGradMultiDevice(self):
+    config = config_pb2.ConfigProto(device_count={"CPU": 2},
+                                    allow_soft_placement=True)
+    with self.cached_session(use_gpu=True, config=config) as sess:
+      pred = array_ops.placeholder(dtypes.bool, [])
+      x = array_ops.placeholder(dtypes.float32)
+      y = array_ops.placeholder(dtypes.float32)
+
+      with ops.device("/cpu:0"):
+        z = control_flow_ops.cond(pred, lambda: x * y * 2.0, lambda: 2.0)
+
+      with ops.device("/cpu:1"):
+        grad = gradients_impl.gradients(z, x)[0]
+
+      self.assertEqual(sess.run(grad, {pred: True, x: 1.0, y: 2.0}), 4.0)
+      self.assertEqual(sess.run(grad, {pred: False, x: 1.0, y: 2.0}), 0.0)
+
+      with ops.device("/cpu:0"):
+        grad_grad = gradients_impl.gradients(grad, x)[0]
+
+      # v1 control flow gets None second derivative for some reason.
+      if not control_flow_ops.ENABLE_COND_V2:
+        self.assertIsNone(grad_grad)
+        return
+
+      self.assertEqual(sess.run(grad_grad, {pred: True, x: 1.0, y: 2.0}), 0.0)
+      self.assertEqual(sess.run(grad_grad, {pred: False, x: 1.0, y: 2.0}), 0.0)
+
   def testNestedCond_Simple(self):
     with self.cached_session():
       x = constant_op.constant(0., name="X")
@@ -1657,6 +1685,35 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [n])
       self.assertAllEqual(10, r.eval())
 
+  def testWhileCondGradMultiDevice(self):
+    config = config_pb2.ConfigProto(device_count={"CPU": 2},
+                                    allow_soft_placement=True)
+    with self.cached_session(use_gpu=True, config=config) as sess:
+      pred = array_ops.placeholder(dtypes.bool, [])
+      x_init = constant_op.constant(1.0)
+
+      with ops.device("/cpu:0"):
+        z = control_flow_ops.while_loop(
+            lambda i, _: i < 3,
+            lambda i, x: (i + 1, control_flow_ops.cond(
+                pred, lambda: x * 2.0, lambda: 10.0)),
+            [0, x_init])
+
+      with ops.device("/cpu:1"):
+        grad = gradients_impl.gradients(z, x_init)[0]
+
+      self.assertEqual(sess.run(grad, {pred: True}), 8.0)
+      self.assertEqual(sess.run(grad, {pred: False}), 0.0)
+
+      if not control_flow_ops.ENABLE_WHILE_V2:
+        return
+
+      with ops.device("/cpu:0"):
+        grad_grad = gradients_impl.gradients(grad, x_init)[0]
+
+      self.assertEqual(sess.run(grad_grad, {pred: True}), 0.0)
+      self.assertEqual(sess.run(grad_grad, {pred: False}), 0.0)
+
   # NOTE: It is ok to have parallel_iterations > 1
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileUpdateVariable_1(self):
-- 
GitLab


From 9771acf677b61a734d2df6897a45262a2a5f77cd Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 15 Nov 2018 09:47:05 -0800
Subject: [PATCH 0297/1554] Break down function_test a bit more.

PiperOrigin-RevId: 221639302
---
 tensorflow/python/eager/BUILD                 |  19 +-
 .../python/eager/function_gradients_test.py   | 755 ++++++++++++++++++
 tensorflow/python/eager/function_test.py      | 708 ----------------
 3 files changed, 773 insertions(+), 709 deletions(-)
 create mode 100644 tensorflow/python/eager/function_gradients_test.py

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 362e8e3b83..db5c6b2eb7 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -174,6 +174,23 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "function_gradients_test",
+    size = "medium",
+    srcs = ["function_gradients_test.py"],
+    additional_deps = [
+        ":backprop",
+        ":context",
+        ":def_function",
+        ":function",
+        ":test",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+    ],
+    shard_count = 5,
+)
+
 cuda_py_test(
     name = "function_test",
     size = "medium",
@@ -193,7 +210,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:resource_variable_ops",
     ],
-    shard_count = 20,
+    shard_count = 15,
 )
 
 py_library(
diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py
new file mode 100644
index 0000000000..d4f8aaa7e3
--- /dev/null
+++ b/tensorflow/python/eager/function_gradients_test.py
@@ -0,0 +1,755 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
+from tensorflow.python.eager import function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+class FunctionGradientsTest(test.TestCase, parameterized.TestCase):
+
+  def testGraphModeWithGradients(self):
+    v = resource_variable_ops.ResourceVariable(1.0, name='v')
+
+    @def_function.function
+    def step():
+      def inner():
+        return v * v
+
+      return backprop.implicit_grad(inner)()[0][0]
+
+    self.assertAllEqual(step(), 2.0)
+
+  def testGraphGradientVariable(self):
+    with ops.Graph().as_default(), self.cached_session():
+      v = variables.Variable(1.0)
+
+      @def_function.function
+      def f():
+        return 2.0 * v
+
+      node = f()
+      grads, = gradients_impl.gradients(node, v)
+      v.initializer.run()
+      self.assertAllEqual(grads.eval(), 2.0)
+      self.assertEqual(grads.shape, v.shape)
+
+  def testSymGradGatherNd(self):
+    with ops.Graph().as_default(), self.cached_session() as sess:
+
+      @def_function.function
+      def f(x):
+        return array_ops.gather_nd(x, [[0]])
+
+      c = constant_op.constant([[2.]])
+      f_c = f(c)
+      g, = gradients_impl.gradients(f_c, c)
+      self.assertAllEqual(sess.run(g).values, [[1.0]])
+
+  def testNoSymGradNestedDefun(self):
+
+    @def_function.function
+    def outer():
+
+      @def_function.function
+      def f(x):
+        return array_ops.gather_nd(x, [[0]])
+
+      c = constant_op.constant([[2.]])
+      f_c = f(c)
+      g, = gradients_impl.gradients(f_c, c)
+      self.assertIsInstance(g, ops.IndexedSlices)
+
+    outer()
+
+  def testGraphFunctionWithGradients(self):
+    v = resource_variable_ops.ResourceVariable(1.0, name='v')
+
+    @def_function.function
+    def step():
+      def inner():
+        return v * v
+
+      return backprop.implicit_grad(inner)()[0][0]
+
+    step_op = step.get_concrete_function()
+    self.assertEqual(step_op.output_dtypes, dtypes.float32)
+    self.assertEqual(step_op.output_shapes, tensor_shape.TensorShape([]))
+    self.assertAllEqual(step_op(), 2.0)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testDefunCondGradient(self):
+
+    @def_function.function
+    def f(x):
+      return control_flow_ops.cond(x > 0.5, lambda: 2 * x, lambda: 3 * x)
+
+    with backprop.GradientTape() as t:
+      x = constant_op.constant(1.0)
+      t.watch(x)
+      y = f(x)
+    self.assertAllEqual(self.evaluate(t.gradient(y, x)), 2.0)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testGraphLoopGradient(self):
+
+    @def_function.function
+    def f(x):
+      return control_flow_ops.while_loop(lambda _, i: i < 2,
+                                         lambda x, i: (2*x, i + 1),
+                                         [x, 0])[0]
+
+    with backprop.GradientTape() as t:
+      x = constant_op.constant(1.0)
+      t.watch(x)
+      y = f(x)
+    self.assertAllEqual(self.evaluate(t.gradient(y, x)), 4.0)
+
+  def testDefunDifferentiable(self):
+    v = resource_variable_ops.ResourceVariable(1.0)
+
+    @def_function.function
+    def f():
+      return v * v
+
+    self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
+
+  def testDefunCanBeDifferentiatedTwice(self):
+    v = resource_variable_ops.ResourceVariable(1.0)
+
+    @def_function.function
+    def f():
+      return v * v
+
+    self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
+    # Ensure that v is watched again.
+    self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
+
+  def testSymbolicGradientVariableNoneNotZerosLike(self):
+    with ops.Graph().as_default():
+      v = variables.Variable(1.0)
+
+      @def_function.function
+      def f(x, v):
+        v.read_value()
+        return x * x
+
+      x = constant_op.constant(1.0)
+      l = f(x, v)
+      _, dv = gradients_impl.gradients(l, [x, v])
+      with self.cached_session():
+        v.initializer.run()
+        self.assertEqual(dv, None)
+
+  def testDefunCallBackprop(self):
+
+    @def_function.function
+    def f(x):
+      return math_ops.add(x, x)
+
+    @def_function.function
+    def g(x):
+      return backprop.gradients_function(f, [0])(x)[0]
+
+    self.assertAllEqual(2, g(constant_op.constant(2.)))
+
+  def testGraphModeEagerGradError(self):
+    with context.graph_mode():
+      def f():
+        x = variable_scope.get_variable(
+            'v', initializer=constant_op.constant(1.0))
+        return x * constant_op.constant(2.0)
+
+      with self.assertRaisesRegexp(ValueError,
+                                   'No trainable variables were accessed'):
+        backprop.implicit_val_and_grad(f)()
+
+  def testDefunCallBackpropUsingSameObjectForMultipleArguments(self):
+
+    @def_function.function
+    def g(x):
+      return backprop.gradients_function(math_ops.multiply, [0, 1])(x, x)
+
+    def np_g(x):
+      return [d.numpy() for d in g(x)]
+
+    x = constant_op.constant(1.)
+    self.assertAllEqual([1., 1.], np_g(x))
+    self.assertAllEqual([1., 1.], np_g(1.))
+
+  def testGradientTensorConversionWithDefun(self):
+    three = resource_variable_ops.ResourceVariable(3.0, name='v')
+
+    @def_function.function
+    def f(x):
+      return math_ops.add(x, three)
+
+    def g(x):
+      return f(x)
+
+    g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0]
+    self.assertAllEqual(g, 1.0)
+
+  def testGradient(self):
+    matmul = def_function.function(math_ops.matmul)
+
+    def sq(x):
+      return matmul(x, x, transpose_a=True)
+
+    t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
+    grad_t, = backprop.gradients_function(sq, [0])(t)
+    self.assertAllEqual(grad_t, [[6, 6], [14, 14]])
+
+  def testGradientInFunction(self):
+
+    @def_function.function
+    def f(x):
+      return backprop.gradients_function(lambda y: y * y, [0])(x)[0]
+
+    self.assertAllEqual(f(constant_op.constant(1.0)), 2.0)
+
+  def testGradientOfGatherWithDefun(self):
+    v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0])
+
+    def sum_gather():
+      return math_ops.reduce_sum(array_ops.gather(v, [1, 2]))
+
+    grad_fn = backprop.implicit_grad(sum_gather)
+    gradient = grad_fn()
+    defun_grad_fn = backprop.implicit_grad(def_function.function(sum_gather))
+    defun_gradient = defun_grad_fn()
+    self.assertEqual(len(gradient), len(defun_gradient))
+
+    gradient = gradient[0][0]
+    defun_gradient = defun_gradient[0][0]
+    self.assertAllEqual(gradient.values, defun_gradient.values)
+    self.assertAllEqual(gradient.indices, defun_gradient.indices)
+    self.assertAllEqual(gradient.dense_shape, defun_gradient.dense_shape)
+
+  def testDifferentiableFunctionNoneOutputs(self):
+
+    @def_function.function
+    def my_function(x):
+      return x, None
+
+    def wrapper(x):
+      return my_function(x)[0]
+
+    g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0))
+    self.assertAllEqual(g[0], 1.)
+
+    @def_function.function
+    def foo(a):
+      return None, a * a
+
+    x = constant_op.constant(5.0)
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      none, r = foo(x)
+    g = tp.gradient(r, x)
+
+    self.assertIs(none, None)
+    self.assertAllEqual(r, 25.0)
+    self.assertAllEqual(g, 2 * 5.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testNestedDifferentiableFunction(self):
+    @def_function.function
+    def inner_fn(a, b):
+      return a * math_ops.add(a, b)
+
+    @def_function.function
+    def outer_fn(x):
+      return inner_fn(x, 1.0)
+
+    x = constant_op.constant(5.0)
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunction(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def outer_fn(x):
+      return middle_fn(x, 1.0)
+
+    x = constant_op.constant(5.0)
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionWithMultipleGradCalls(self):
+    @def_function.function
+    def inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return math_ops.mul(a, inner_fn(a, b))
+
+    @def_function.function
+    def outer_fn(x):
+      return middle_fn(x, 3.0)
+
+    x = constant_op.constant(5.0)
+    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
+
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
+    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
+    self.assertAllEqual(middle_fn(3.0, x), 3.0 * (3.0 + 5.0))
+
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
+
+    y = constant_op.constant(4.0)
+    with backprop.GradientTape() as tp:
+      tp.watch(y)
+      result = outer_fn(y)
+    grad = tp.gradient(result, y)
+
+    self.assertAllEqual(grad, 2 * 4.0 + 3.0)
+
+    with backprop.GradientTape() as tp:
+      tp.watch(y)
+      result = inner_fn(y, y)
+    grad = tp.gradient(result, y)
+
+    self.assertAllEqual(grad, 2.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionGradientTapeInDefun(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def outer_fn(x):
+      with backprop.GradientTape() as tp:
+        tp.watch(x)
+        result = middle_fn(x, 1.0)
+      grad = tp.gradient(result, x)
+      return grad
+
+    x = constant_op.constant(5.0)
+    grad = outer_fn(x)
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionGradientTapeInNestedDefun(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def almost_outer_fn(x):
+      with backprop.GradientTape() as tp:
+        tp.watch(x)
+        result = middle_fn(x, 1.0)
+      grad = tp.gradient(result, x)
+      return grad
+
+    @def_function.function
+    def outer_fn(x):
+      return almost_outer_fn(x)
+
+    x = constant_op.constant(5.0)
+    grad = outer_fn(x)
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionGradientTapeInMultNestedDefun(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def almost_outer_fn(x):
+      with backprop.GradientTape() as tp:
+        tp.watch(x)
+        result = middle_fn(x, 1.0)
+      grad = tp.gradient(result, x)
+      return grad
+
+    @def_function.function
+    def outer_fn(x):
+      return almost_outer_fn(x)
+
+    @def_function.function
+    def outer_outer_fn(x):
+      return outer_fn(x)
+
+    x = constant_op.constant(5.0)
+    grad = outer_outer_fn(x)
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionTFGradientInDefun(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def outer_fn(x):
+      result = middle_fn(x, 1.0)
+      return gradients_impl.gradients(result, [x])[0]
+
+    x = constant_op.constant(5.0)
+    grad = outer_fn(x)
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionTFGradientInNestedDefun(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def almost_outer_fn(x):
+      result = middle_fn(x, 1.0)
+      return gradients_impl.gradients(result, [x])[0]
+
+    @def_function.function
+    def outer_fn(x):
+      return almost_outer_fn(x)
+
+    x = constant_op.constant(5.0)
+    grad = outer_fn(x)
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testDeeplyNestedDifferentiableFunctionTFGradientInMultNestedDefun(self):
+    @def_function.function
+    def inner_inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def inner_fn(a, b):
+      return inner_inner_fn(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def almost_outer_fn(x):
+      result = middle_fn(x, 1.0)
+      return gradients_impl.gradients(result, [x])[0]
+
+    @def_function.function
+    def outer_fn(x):
+      return almost_outer_fn(x)
+
+    @def_function.function
+    def outer_outer_fn(x):
+      return outer_fn(x)
+
+    x = constant_op.constant(5.0)
+    grad = outer_outer_fn(x)
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  def testDeeplyNestedDifferentiableFunctionWithVariable(self):
+    var = variables.Variable(constant_op.constant(1.0))
+
+    @def_function.function
+    def inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return a * inner_fn(a, b)
+
+    @def_function.function
+    def outer_fn(x):
+      return middle_fn(x, var)
+
+    x = constant_op.constant(5.0)
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
+
+  def testDeeplyNestedDifferentiableFunctionWithVariableMultipleGradCalls(self):
+    v = variables.Variable(constant_op.constant(3.0))
+
+    @def_function.function
+    def inner_fn(a, b):
+      return math_ops.add(a, b)
+
+    @def_function.function
+    def middle_fn(a, b):
+      return math_ops.mul(a, inner_fn(a, b))
+
+    @def_function.function
+    def outer_fn(x):
+      return middle_fn(x, v)
+
+    x = constant_op.constant(5.0)
+    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
+
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
+    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
+    self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0))
+
+    with backprop.GradientTape() as tp:
+      tp.watch(x)
+      result = outer_fn(x)
+    grad = tp.gradient(result, x)
+
+    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
+
+    y = constant_op.constant(4.0)
+    with backprop.GradientTape() as tp:
+      tp.watch(y)
+      result = outer_fn(y)
+    grad = tp.gradient(result, y)
+
+    self.assertAllEqual(grad, 2 * 4.0 + 3.0)
+
+    v.assign(constant_op.constant(1.5))
+    with backprop.GradientTape() as tp:
+      tp.watch(y)
+      result = outer_fn(y)
+    grad = tp.gradient(result, y)
+
+    self.assertAllEqual(grad, 2 * 4.0 + 1.5)
+
+    with backprop.GradientTape() as tp:
+      tp.watch(y)
+      result = inner_fn(y, v)
+    grad = tp.gradient(result, y)
+
+    self.assertAllEqual(grad, 1.0)
+
+  def testDeeplyNestedDifferentiableFunctionWithVariableMultipleTFGrads(self):
+    with context.graph_mode(), self.cached_session():
+      v = resource_variable_ops.ResourceVariable(3.0)
+      v.initializer.run()
+
+      @def_function.function
+      def inner_fn(a, b):
+        return math_ops.add(a, b)
+
+      @def_function.function
+      def middle_fn(a, b):
+        return math_ops.mul(a, inner_fn(a, b))
+
+      @def_function.function
+      def outer_fn(x):
+        return middle_fn(x, v)
+
+      x = constant_op.constant(5.0)
+      self.assertAllEqual(outer_fn(x).eval(), 5.0 * (5.0 + 3.0))
+
+      grad, = gradients_impl.gradients(outer_fn(x), x)
+
+      self.assertAllEqual(grad, 2 * 5.0 + 3.0)
+      self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
+      self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0))
+
+      grad, = gradients_impl.gradients(outer_fn(x), x)
+
+      self.assertAllEqual(grad, 2 * 5.0 + 3.0)
+
+      y = constant_op.constant(4.0)
+      grad, = gradients_impl.gradients(outer_fn(y), y)
+      self.assertAllEqual(grad, 2 * 4.0 + 3.0)
+
+      self.evaluate(v.assign(constant_op.constant(1.5)))
+      grad, = gradients_impl.gradients(outer_fn(y), y)
+
+      self.assertAllEqual(grad, 2 * 4.0 + 1.5)
+
+      grad, = gradients_impl.gradients(inner_fn(y, v), y)
+      self.assertAllEqual(grad, 1.0)
+
+  def testNestedDifferentiableFunctionNoneOutputs(self):
+    @def_function.function
+    def foo(a, b):
+      return None, a * math_ops.add(a, b), None, 2*a
+
+    @def_function.function
+    def bar(x):
+      return foo(x, 1.0)
+
+    x = constant_op.constant(5.0)
+    with backprop.GradientTape(persistent=True) as tp:
+      tp.watch(x)
+      none1, r1, none2, r2 = bar(x)
+    g1 = tp.gradient(r1, x)
+    g2 = tp.gradient(r2, x)
+
+    self.assertAllEqual(r1, 30.0)
+    self.assertAllEqual(r2, 10.0)
+    self.assertIs(none1, None)
+    self.assertIs(none2, None)
+    self.assertAllEqual(g1, 2 * 5.0 + 1.0)
+    self.assertAllEqual(g2, 2.0)
+
+  def testGradientWithKeywordArguments(self):
+    matmul = def_function.function(math_ops.matmul)
+
+    def sq(x):
+      return matmul(a=x, b=x, transpose_a=True)
+
+    t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
+    grad_t, = backprop.gradients_function(sq, [0])(t)
+    self.assertAllEqual(grad_t, [[6, 6], [14, 14]])
+
+    with backprop.GradientTape(persistent=True) as tape:
+      tape.watch(t)
+      one = matmul(t, b=t, transpose_a=True)
+      two = matmul(b=t, a=t, transpose_a=True)
+      three = matmul(a=t, b=t, transpose_a=True)
+
+    for output in [one, two, three]:
+      self.assertAllEqual(tape.gradient(output, t), [[6, 6], [14, 14]])
+
+  def testGradientInFunctionWithKeywordArguments(self):
+
+    @def_function.function
+    def f(x):
+      return backprop.gradients_function(lambda y: y * y, [0])(x)[0]
+
+    self.assertAllEqual(f(x=constant_op.constant(1.0)), 2.0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testBackwardNone(self):
+    model = variables.Variable(1.0, name='model')
+    count = variables.Variable(0)
+
+    @function.defun
+    def forward_pass(value):
+      count.assign_add(1)
+      residuals = value - model
+      loss = 0.5 * math_ops.reduce_mean(math_ops.pow(residuals, 2))
+      # Note: count is an integer, so its doutput will be None
+      return loss, count
+
+    def reduce_fn(x):
+      if context.executing_eagerly():
+        with backprop.GradientTape() as t:
+          loss, count = forward_pass(x)
+        return t.gradient(loss, model), count
+      loss, count = forward_pass(x)
+      grad_only = gradients_impl.gradients(loss, model)
+      return grad_only, count
+
+    g, _ = reduce_fn(constant_op.constant([7.0]))
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllEqual(nest.flatten(self.evaluate(g)), [-6.0])
+
+
+if __name__ == '__main__':
+  ops.enable_eager_execution(
+      config=config_pb2.ConfigProto(device_count={'CPU': 4}))
+  test.main()
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index fad827953a..98040dc68c 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -29,7 +29,6 @@ import numpy
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
-from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
@@ -48,7 +47,6 @@ from tensorflow.python.layers import convolutional
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -149,32 +147,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     out = a_times_b(pair({'a': t}, {'b': t}))
     self.assertAllEqual(out, math_ops.matmul(t, t).numpy())
 
-  def testGraphModeWithGradients(self):
-    v = resource_variable_ops.ResourceVariable(1.0, name='v')
-
-    @def_function.function
-    def step():
-      def inner():
-        return v * v
-
-      return backprop.implicit_grad(inner)()[0][0]
-
-    self.assertAllEqual(step(), 2.0)
-
-  def testGraphGradientVariable(self):
-    with ops.Graph().as_default(), self.cached_session():
-      v = variables.Variable(1.0)
-
-      @def_function.function
-      def f():
-        return 2.0 * v
-
-      node = f()
-      grads, = gradients_impl.gradients(node, v)
-      v.initializer.run()
-      self.assertAllEqual(grads.eval(), 2.0)
-      self.assertEqual(grads.shape, v.shape)
-
   def testGraphEagerIsolation(self):
 
     @function.defun
@@ -314,34 +286,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     random_seed.set_random_seed(1)
     self.assertAllEqual(f(), x)
 
-  def testSymGradGatherNd(self):
-    with ops.Graph().as_default(), self.cached_session() as sess:
-
-      @def_function.function
-      def f(x):
-        return array_ops.gather_nd(x, [[0]])
-
-      c = constant_op.constant([[2.]])
-      f_c = f(c)
-      g, = gradients_impl.gradients(f_c, c)
-      self.assertAllEqual(sess.run(g).values, [[1.0]])
-
-  def testNoSymGradNestedDefun(self):
-
-    @def_function.function
-    def outer():
-
-      @def_function.function
-      def f(x):
-        return array_ops.gather_nd(x, [[0]])
-
-      c = constant_op.constant([[2.]])
-      f_c = f(c)
-      g, = gradients_impl.gradients(f_c, c)
-      self.assertIsInstance(g, ops.IndexedSlices)
-
-    outer()
-
   def testNestedInputsGraphFunction(self):
     matmul = def_function.function(math_ops.matmul)
 
@@ -378,21 +322,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual(a, math_ops.matmul(t, t).numpy())
     self.assertAllEqual(b['b'].numpy(), 1.0)
 
-  def testGraphFunctionWithGradients(self):
-    v = resource_variable_ops.ResourceVariable(1.0, name='v')
-
-    @def_function.function
-    def step():
-      def inner():
-        return v * v
-
-      return backprop.implicit_grad(inner)()[0][0]
-
-    step_op = step.get_concrete_function()
-    self.assertEqual(step_op.output_dtypes, dtypes.float32)
-    self.assertEqual(step_op.output_shapes, tensor_shape.TensorShape([]))
-    self.assertAllEqual(step_op(), 2.0)
-
   def testGraphFunctionNoneOutput(self):
     @def_function.function
     def fn(unused_a, unused_b):
@@ -404,34 +333,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(fn_op.output_shapes, None)
     self.assertAllEqual(fn_op(x, x), None)
 
-  @test_util.run_in_graph_and_eager_modes()
-  def testDefunCondGradient(self):
-
-    @def_function.function
-    def f(x):
-      return control_flow_ops.cond(x > 0.5, lambda: 2 * x, lambda: 3 * x)
-
-    with backprop.GradientTape() as t:
-      x = constant_op.constant(1.0)
-      t.watch(x)
-      y = f(x)
-    self.assertAllEqual(self.evaluate(t.gradient(y, x)), 2.0)
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testGraphLoopGradient(self):
-
-    @def_function.function
-    def f(x):
-      return control_flow_ops.while_loop(lambda _, i: i < 2,
-                                         lambda x, i: (2*x, i + 1),
-                                         [x, 0])[0]
-
-    with backprop.GradientTape() as t:
-      x = constant_op.constant(1.0)
-      t.watch(x)
-      y = f(x)
-    self.assertAllEqual(self.evaluate(t.gradient(y, x)), 4.0)
-
   def testDefunNumpyArraysConvertedToTensors(self):
 
     def f(x):
@@ -625,26 +526,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertIsInstance(
         self.v, resource_variable_ops.ResourceVariable)
 
-  def testDefunDifferentiable(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
-
-    @def_function.function
-    def f():
-      return v * v
-
-    self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
-
-  def testDefunCanBeDifferentiatedTwice(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
-
-    @def_function.function
-    def f():
-      return v * v
-
-    self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
-    # Ensure that v is watched again.
-    self.assertAllEqual(backprop.implicit_grad(f)()[0][0], 2.0)
-
   def disabled_testRunMetadata(self):
 
     @def_function.function
@@ -685,22 +566,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       op = call()
       self.assertAllEqual(sess.run(op), 2.0)
 
-  def testSymbolicGradientVariableNoneNotZerosLike(self):
-    with ops.Graph().as_default():
-      v = variables.Variable(1.0)
-
-      @def_function.function
-      def f(x, v):
-        v.read_value()
-        return x * x
-
-      x = constant_op.constant(1.0)
-      l = f(x, v)
-      _, dv = gradients_impl.gradients(l, [x, v])
-      with self.cached_session():
-        v.initializer.run()
-        self.assertEqual(dv, None)
-
   def testGraphModeManyFunctions(self):
     with ops.Graph().as_default(), self.cached_session():
 
@@ -742,42 +607,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     self.assertAllEqual(8, g(constant_op.constant(2)))
 
-  def testDefunCallBackprop(self):
-
-    @def_function.function
-    def f(x):
-      return math_ops.add(x, x)
-
-    @def_function.function
-    def g(x):
-      return backprop.gradients_function(f, [0])(x)[0]
-
-    self.assertAllEqual(2, g(constant_op.constant(2.)))
-
-  def testGraphModeEagerGradError(self):
-    with context.graph_mode():
-      def f():
-        x = variable_scope.get_variable(
-            'v', initializer=constant_op.constant(1.0))
-        return x * constant_op.constant(2.0)
-
-      with self.assertRaisesRegexp(ValueError,
-                                   'No trainable variables were accessed'):
-        backprop.implicit_val_and_grad(f)()
-
-  def testDefunCallBackpropUsingSameObjectForMultipleArguments(self):
-
-    @def_function.function
-    def g(x):
-      return backprop.gradients_function(math_ops.multiply, [0, 1])(x, x)
-
-    def np_g(x):
-      return [d.numpy() for d in g(x)]
-
-    x = constant_op.constant(1.)
-    self.assertAllEqual([1., 1.], np_g(x))
-    self.assertAllEqual([1., 1.], np_g(1.))
-
   def testCallShape(self):
 
     @def_function.function
@@ -808,37 +637,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     g(three)
 
-  def testGradientTensorConversionWithDefun(self):
-    three = resource_variable_ops.ResourceVariable(3.0, name='v')
-
-    @def_function.function
-    def f(x):
-      return math_ops.add(x, three)
-
-    def g(x):
-      return f(x)
-
-    g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0]
-    self.assertAllEqual(g, 1.0)
-
-  def testGradient(self):
-    matmul = def_function.function(math_ops.matmul)
-
-    def sq(x):
-      return matmul(x, x, transpose_a=True)
-
-    t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
-    grad_t, = backprop.gradients_function(sq, [0])(t)
-    self.assertAllEqual(grad_t, [[6, 6], [14, 14]])
-
-  def testGradientInFunction(self):
-
-    @def_function.function
-    def f(x):
-      return backprop.gradients_function(lambda y: y * y, [0])(x)[0]
-
-    self.assertAllEqual(f(constant_op.constant(1.0)), 2.0)
-
   def testGatherResourceWithDefun(self):
     with ops.device('cpu:0'):
       v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0])
@@ -849,24 +647,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     defined = def_function.function(sum_gather)
     self.assertAllEqual(sum_gather(), defined())
 
-  def testGradientOfGatherWithDefun(self):
-    v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0])
-
-    def sum_gather():
-      return math_ops.reduce_sum(array_ops.gather(v, [1, 2]))
-
-    grad_fn = backprop.implicit_grad(sum_gather)
-    gradient = grad_fn()
-    defun_grad_fn = backprop.implicit_grad(def_function.function(sum_gather))
-    defun_gradient = defun_grad_fn()
-    self.assertEqual(len(gradient), len(defun_gradient))
-
-    gradient = gradient[0][0]
-    defun_gradient = defun_gradient[0][0]
-    self.assertAllEqual(gradient.values, defun_gradient.values)
-    self.assertAllEqual(gradient.indices, defun_gradient.indices)
-    self.assertAllEqual(gradient.dense_shape, defun_gradient.dense_shape)
-
   def testReturningIndexedSlicesWithDefun(self):
 
     def validate(indexed_slice):
@@ -1012,440 +792,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     shape = constant_op.constant([2, 1]).gpu()
     reshape(value, shape)  # No error is raised
 
-  def testDifferentiableFunctionNoneOutputs(self):
-
-    @def_function.function
-    def my_function(x):
-      return x, None
-
-    def wrapper(x):
-      return my_function(x)[0]
-
-    g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0))
-    self.assertAllEqual(g[0], 1.)
-
-    @def_function.function
-    def foo(a):
-      return None, a * a
-
-    x = constant_op.constant(5.0)
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      none, r = foo(x)
-    g = tp.gradient(r, x)
-
-    self.assertIs(none, None)
-    self.assertAllEqual(r, 25.0)
-    self.assertAllEqual(g, 2 * 5.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testNestedDifferentiableFunction(self):
-    @def_function.function
-    def inner_fn(a, b):
-      return a * math_ops.add(a, b)
-
-    @def_function.function
-    def outer_fn(x):
-      return inner_fn(x, 1.0)
-
-    x = constant_op.constant(5.0)
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunction(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def outer_fn(x):
-      return middle_fn(x, 1.0)
-
-    x = constant_op.constant(5.0)
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionWithMultipleGradCalls(self):
-    @def_function.function
-    def inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return math_ops.mul(a, inner_fn(a, b))
-
-    @def_function.function
-    def outer_fn(x):
-      return middle_fn(x, 3.0)
-
-    x = constant_op.constant(5.0)
-    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
-
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
-    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
-    self.assertAllEqual(middle_fn(3.0, x), 3.0 * (3.0 + 5.0))
-
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
-
-    y = constant_op.constant(4.0)
-    with backprop.GradientTape() as tp:
-      tp.watch(y)
-      result = outer_fn(y)
-    grad = tp.gradient(result, y)
-
-    self.assertAllEqual(grad, 2 * 4.0 + 3.0)
-
-    with backprop.GradientTape() as tp:
-      tp.watch(y)
-      result = inner_fn(y, y)
-    grad = tp.gradient(result, y)
-
-    self.assertAllEqual(grad, 2.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionGradientTapeInDefun(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def outer_fn(x):
-      with backprop.GradientTape() as tp:
-        tp.watch(x)
-        result = middle_fn(x, 1.0)
-      grad = tp.gradient(result, x)
-      return grad
-
-    x = constant_op.constant(5.0)
-    grad = outer_fn(x)
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionGradientTapeInNestedDefun(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def almost_outer_fn(x):
-      with backprop.GradientTape() as tp:
-        tp.watch(x)
-        result = middle_fn(x, 1.0)
-      grad = tp.gradient(result, x)
-      return grad
-
-    @def_function.function
-    def outer_fn(x):
-      return almost_outer_fn(x)
-
-    x = constant_op.constant(5.0)
-    grad = outer_fn(x)
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionGradientTapeInMultNestedDefun(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def almost_outer_fn(x):
-      with backprop.GradientTape() as tp:
-        tp.watch(x)
-        result = middle_fn(x, 1.0)
-      grad = tp.gradient(result, x)
-      return grad
-
-    @def_function.function
-    def outer_fn(x):
-      return almost_outer_fn(x)
-
-    @def_function.function
-    def outer_outer_fn(x):
-      return outer_fn(x)
-
-    x = constant_op.constant(5.0)
-    grad = outer_outer_fn(x)
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionTFGradientInDefun(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def outer_fn(x):
-      result = middle_fn(x, 1.0)
-      return gradients_impl.gradients(result, [x])[0]
-
-    x = constant_op.constant(5.0)
-    grad = outer_fn(x)
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionTFGradientInNestedDefun(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def almost_outer_fn(x):
-      result = middle_fn(x, 1.0)
-      return gradients_impl.gradients(result, [x])[0]
-
-    @def_function.function
-    def outer_fn(x):
-      return almost_outer_fn(x)
-
-    x = constant_op.constant(5.0)
-    grad = outer_fn(x)
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testDeeplyNestedDifferentiableFunctionTFGradientInMultNestedDefun(self):
-    @def_function.function
-    def inner_inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def inner_fn(a, b):
-      return inner_inner_fn(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def almost_outer_fn(x):
-      result = middle_fn(x, 1.0)
-      return gradients_impl.gradients(result, [x])[0]
-
-    @def_function.function
-    def outer_fn(x):
-      return almost_outer_fn(x)
-
-    @def_function.function
-    def outer_outer_fn(x):
-      return outer_fn(x)
-
-    x = constant_op.constant(5.0)
-    grad = outer_outer_fn(x)
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  def testDeeplyNestedDifferentiableFunctionWithVariable(self):
-    var = variables.Variable(constant_op.constant(1.0))
-
-    @def_function.function
-    def inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return a * inner_fn(a, b)
-
-    @def_function.function
-    def outer_fn(x):
-      return middle_fn(x, var)
-
-    x = constant_op.constant(5.0)
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 1.0)
-
-  def testDeeplyNestedDifferentiableFunctionWithVariableMultipleGradCalls(self):
-    v = variables.Variable(constant_op.constant(3.0))
-
-    @def_function.function
-    def inner_fn(a, b):
-      return math_ops.add(a, b)
-
-    @def_function.function
-    def middle_fn(a, b):
-      return math_ops.mul(a, inner_fn(a, b))
-
-    @def_function.function
-    def outer_fn(x):
-      return middle_fn(x, v)
-
-    x = constant_op.constant(5.0)
-    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
-
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
-    self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
-    self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0))
-
-    with backprop.GradientTape() as tp:
-      tp.watch(x)
-      result = outer_fn(x)
-    grad = tp.gradient(result, x)
-
-    self.assertAllEqual(grad, 2 * 5.0 + 3.0)
-
-    y = constant_op.constant(4.0)
-    with backprop.GradientTape() as tp:
-      tp.watch(y)
-      result = outer_fn(y)
-    grad = tp.gradient(result, y)
-
-    self.assertAllEqual(grad, 2 * 4.0 + 3.0)
-
-    v.assign(constant_op.constant(1.5))
-    with backprop.GradientTape() as tp:
-      tp.watch(y)
-      result = outer_fn(y)
-    grad = tp.gradient(result, y)
-
-    self.assertAllEqual(grad, 2 * 4.0 + 1.5)
-
-    with backprop.GradientTape() as tp:
-      tp.watch(y)
-      result = inner_fn(y, v)
-    grad = tp.gradient(result, y)
-
-    self.assertAllEqual(grad, 1.0)
-
-  def testDeeplyNestedDifferentiableFunctionWithVariableMultipleTFGrads(self):
-    with context.graph_mode(), self.cached_session():
-      v = resource_variable_ops.ResourceVariable(3.0)
-      v.initializer.run()
-
-      @def_function.function
-      def inner_fn(a, b):
-        return math_ops.add(a, b)
-
-      @def_function.function
-      def middle_fn(a, b):
-        return math_ops.mul(a, inner_fn(a, b))
-
-      @def_function.function
-      def outer_fn(x):
-        return middle_fn(x, v)
-
-      x = constant_op.constant(5.0)
-      self.assertAllEqual(outer_fn(x).eval(), 5.0 * (5.0 + 3.0))
-
-      grad, = gradients_impl.gradients(outer_fn(x), x)
-
-      self.assertAllEqual(grad, 2 * 5.0 + 3.0)
-      self.assertAllEqual(outer_fn(x), 5.0 * (5.0 + 3.0))
-      self.assertAllEqual(middle_fn(v, x), 3.0 * (3.0 + 5.0))
-
-      grad, = gradients_impl.gradients(outer_fn(x), x)
-
-      self.assertAllEqual(grad, 2 * 5.0 + 3.0)
-
-      y = constant_op.constant(4.0)
-      grad, = gradients_impl.gradients(outer_fn(y), y)
-      self.assertAllEqual(grad, 2 * 4.0 + 3.0)
-
-      self.evaluate(v.assign(constant_op.constant(1.5)))
-      grad, = gradients_impl.gradients(outer_fn(y), y)
-
-      self.assertAllEqual(grad, 2 * 4.0 + 1.5)
-
-      grad, = gradients_impl.gradients(inner_fn(y, v), y)
-      self.assertAllEqual(grad, 1.0)
-
-  def testNestedDifferentiableFunctionNoneOutputs(self):
-    @def_function.function
-    def foo(a, b):
-      return None, a * math_ops.add(a, b), None, 2*a
-
-    @def_function.function
-    def bar(x):
-      return foo(x, 1.0)
-
-    x = constant_op.constant(5.0)
-    with backprop.GradientTape(persistent=True) as tp:
-      tp.watch(x)
-      none1, r1, none2, r2 = bar(x)
-    g1 = tp.gradient(r1, x)
-    g2 = tp.gradient(r2, x)
-
-    self.assertAllEqual(r1, 30.0)
-    self.assertAllEqual(r2, 10.0)
-    self.assertIs(none1, None)
-    self.assertIs(none2, None)
-    self.assertAllEqual(g1, 2 * 5.0 + 1.0)
-    self.assertAllEqual(g2, 2.0)
-
   def testNoneOutput(self):
 
     @def_function.function
@@ -2039,33 +1385,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual(six, 2.0)
     self.assertAllEqual(seven, 2.0)
 
-  def testGradientWithKeywordArguments(self):
-    matmul = def_function.function(math_ops.matmul)
-
-    def sq(x):
-      return matmul(a=x, b=x, transpose_a=True)
-
-    t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
-    grad_t, = backprop.gradients_function(sq, [0])(t)
-    self.assertAllEqual(grad_t, [[6, 6], [14, 14]])
-
-    with backprop.GradientTape(persistent=True) as tape:
-      tape.watch(t)
-      one = matmul(t, b=t, transpose_a=True)
-      two = matmul(b=t, a=t, transpose_a=True)
-      three = matmul(a=t, b=t, transpose_a=True)
-
-    for output in [one, two, three]:
-      self.assertAllEqual(tape.gradient(output, t), [[6, 6], [14, 14]])
-
-  def testGradientInFunctionWithKeywordArguments(self):
-
-    @def_function.function
-    def f(x):
-      return backprop.gradients_function(lambda y: y * y, [0])(x)[0]
-
-    self.assertAllEqual(f(x=constant_op.constant(1.0)), 2.0)
-
   def testDefuningInstanceMethod(self):
 
     integer = constant_op.constant(2, dtypes.int64)
@@ -2339,33 +1658,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
         # pylint: disable=protected-access
         self.assertEqual(len(graph._functions), 3)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testBackwardNone(self):
-    model = variables.Variable(1.0, name='model')
-    count = variables.Variable(0)
-
-    @function.defun
-    def forward_pass(value):
-      count.assign_add(1)
-      residuals = value - model
-      loss = 0.5 * math_ops.reduce_mean(math_ops.pow(residuals, 2))
-      # Note: count is an integer, so its doutput will be None
-      return loss, count
-
-    def reduce_fn(x):
-      if context.executing_eagerly():
-        with backprop.GradientTape() as t:
-          loss, count = forward_pass(x)
-        return t.gradient(loss, model), count
-      loss, count = forward_pass(x)
-      grad_only = gradients_impl.gradients(loss, model)
-      return grad_only, count
-
-    g, _ = reduce_fn(constant_op.constant([7.0]))
-
-    self.evaluate(variables.global_variables_initializer())
-    self.assertAllEqual(nest.flatten(self.evaluate(g)), [-6.0])
-
   def testCallingFunctionWithDifferentVariables(self):
 
     @function.defun
-- 
GitLab


From e68d8bb2f59a9b99432eae99fd531ad1737f7576 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Thu, 15 Nov 2018 09:53:18 -0800
Subject: [PATCH 0298/1554] Update the arguments for `tf.nn.separable_conv2d`
 and exporting in v2. Renaming 'rate' argument to 'dilations'.

PiperOrigin-RevId: 221640289
---
 tensorflow/python/ops/nn_impl.py              | 68 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  9 ++-
 3 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index ef763a4b61..b46cc6488c 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -501,7 +501,7 @@ def depthwise_conv2d(input,
 
 
 # pylint: disable=redefined-builtin,line-too-long
-@tf_export("nn.separable_conv2d")
+@tf_export(v1=["nn.separable_conv2d"])
 def separable_conv2d(input,
                      depthwise_filter,
                      pointwise_filter,
@@ -599,6 +599,72 @@ def separable_conv2d(input,
         name=name)
 
 
+@tf_export("nn.separable_conv2d", v1=[])
+def separable_conv2d_v2(
+    input,
+    depthwise_filter,
+    pointwise_filter,
+    strides,
+    padding,
+    data_format=None,
+    dilations=None,
+    name=None,
+):
+  """2-D convolution with separable filters.
+
+  Performs a depthwise convolution that acts separately on channels followed by
+  a pointwise convolution that mixes channels.  Note that this is separability
+  between dimensions `[1, 2]` and `3`, not spatial separability between
+  dimensions `1` and `2`.
+
+  In detail,
+
+      output[b, i, j, k] = sum_{di, dj, q, r}
+          input[b, strides[1] * i + di, strides[2] * j + dj, q] *
+          depthwise_filter[di, dj, q, r] *
+          pointwise_filter[0, 0, q * channel_multiplier + r, k]
+
+  `strides` controls the strides for the depthwise convolution only, since
+  the pointwise convolution has implicit strides of `[1, 1, 1, 1]`.  Must have
+  `strides[0] = strides[3] = 1`.  For the most common case of the same
+  horizontal and vertical strides, `strides = [1, stride, stride, 1]`.
+  If any value in `rate` is greater than 1, we perform atrous depthwise
+  convolution, in which case all values in the `strides` tensor must be equal
+  to 1.
+
+  Args:
+    input: 4-D `Tensor` with shape according to `data_format`.
+    depthwise_filter: 4-D `Tensor` with shape `[filter_height, filter_width,
+      in_channels, channel_multiplier]`. Contains `in_channels` convolutional
+      filters of depth 1.
+    pointwise_filter: 4-D `Tensor` with shape `[1, 1, channel_multiplier *
+      in_channels, out_channels]`.  Pointwise filter to mix channels after
+      `depthwise_filter` has convolved spatially.
+    strides: 1-D of size 4.  The strides for the depthwise convolution for each
+      dimension of `input`.
+    padding: A string, either `'VALID'` or `'SAME'`.  The padding algorithm. See
+      the "returns" section of `tf.nn.convolution` for details.
+    data_format: The data format for input. Either "NHWC" (default) or "NCHW".
+    dilations: 1-D of size 2. The dilation rate in which we sample input values
+      across the `height` and `width` dimensions in atrous convolution. If it is
+      greater than 1, then all values of strides must be 1.
+    name: A name for this operation (optional).
+
+  Returns:
+    A 4-D `Tensor` with shape according to 'data_format'. For
+      example, with data_format="NHWC", shape is [batch, out_height,
+      out_width, out_channels].
+  """
+  return separable_conv2d(
+      input,
+      depthwise_filter,
+      pointwise_filter,
+      strides,
+      padding,
+      rate=dilations,
+      name=name,
+      data_format=data_format)
+
 # pylint: enable=redefined-builtin,line-too-long
 
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index d1c04f5acf..c4da3d670a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -266,7 +266,7 @@ tf_module {
   }
   member_method {
     name: "separable_conv2d"
-    argspec: "args=[\'input\', \'depthwise_filter\', \'pointwise_filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'depthwise_filter\', \'pointwise_filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "sigmoid"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 8de288d6ba..1e050cca9b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -45,6 +45,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.pool": {
             "dilation_rate": "dilations"
         },
+        "tf.nn.separable_conv2d": {
+            "rate": "dilations"
+        }
     }
 
     # Mapping from function to the new name of the function
@@ -77,7 +80,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
-        ]
+        ],
+        "tf.nn.separable_conv2d": [
+            "input", "depthwise_filter", "pointwise_filter", "strides",
+            "padding", "data_format", "dilations", "name"
+        ],
     }
 
     # Specially handled functions.
-- 
GitLab


From 829a53908b6005f7ef2c23e3e1321ca1979b2d81 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 09:54:23 -0800
Subject: [PATCH 0299/1554] [XLA] Split XLA_FLAGS out from TF_XLA_FLAGS, and
 die if an unknown flag is passed.

PiperOrigin-RevId: 221640411
---
 tensorflow/compiler/jit/flags.cc              |  2 +-
 tensorflow/compiler/xla/BUILD                 |  2 +
 .../compiler/xla/debug_options_flags.cc       |  2 +-
 .../compiler/xla/parse_flags_from_env.cc      | 51 +++++++++++++++----
 .../compiler/xla/parse_flags_from_env.h       | 13 +++--
 .../compiler/xla/parse_flags_from_env_test.cc | 18 ++-----
 6 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc
index 7b055893cb..702197e1b1 100644
--- a/tensorflow/compiler/jit/flags.cc
+++ b/tensorflow/compiler/jit/flags.cc
@@ -110,7 +110,7 @@ void AllocateAndParseFlags() {
   });
   AppendDumpGraphFlagsInternal(flag_list);
   AppendMarkForCompilationPassFlagsInternal(flag_list);
-  xla::ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_list);
+  xla::ParseFlagsFromEnvAndDieIfUnknown("TF_XLA_FLAGS", *flag_list);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 91096cf1d0..d914e97b6b 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -745,6 +745,8 @@ cc_library(
             "//tensorflow/core:framework_internal",
             "//tensorflow/core:lib",
             "@com_google_absl//absl/strings",
+            "@com_google_absl//absl/strings:str_format",
+            "@com_google_absl//absl/types:span",
         ],
 )
 
diff --git a/tensorflow/compiler/xla/debug_options_flags.cc b/tensorflow/compiler/xla/debug_options_flags.cc
index a41bd7f9ae..a40330a9b1 100644
--- a/tensorflow/compiler/xla/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/debug_options_flags.cc
@@ -335,7 +335,7 @@ void AllocateFlags() {
           "behavior to help run tests on the host that run models in parallel "
           "across multiple devices."),
   });
-  ParseFlagsFromEnv("TF_XLA_FLAGS", *flag_objects);
+  ParseFlagsFromEnvAndDieIfUnknown("XLA_FLAGS", *flag_objects);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/parse_flags_from_env.cc b/tensorflow/compiler/xla/parse_flags_from_env.cc
index d575c5bc12..5b568888d1 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env.cc
@@ -13,9 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// This module exports ParseFlagsFromEnv(), which allows other modules to parse
-// flags from an environtment variable, or a file named by the environment
-// variable.
+// This module exports ParseFlagsFromEnvAndDieIfUnknown(), which allows other
+// modules to parse flags from an environtment variable, or a file named by the
+// environment variable.
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -24,6 +24,9 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_join.h"
+#include "absl/types/span.h"
 #include "tensorflow/compiler/xla/parse_flags_from_env.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/platform/logging.h"
@@ -176,23 +179,49 @@ static std::unordered_map<string, EnvArgv>& EnvArgvs() {
 // Used to protect accesses to env_argvs.
 static tensorflow::mutex env_argv_mu(tensorflow::LINKER_INITIALIZED);
 
-// Call Flags::Parse(argc, argv, flag_list) against any as yet unrecognized
-// flags passed in from the environment.
-bool ParseFlagsFromEnv(absl::string_view envvar,
-                       const std::vector<tensorflow::Flag>& flag_list) {
+bool ParseFlagsFromEnvAndDieIfUnknown(
+    absl::string_view envvar, const std::vector<tensorflow::Flag>& flag_list) {
   tensorflow::mutex_lock lock(env_argv_mu);
   auto* env_argv = &EnvArgvs()[string(envvar)];
   SetArgvFromEnv(envvar, env_argv);  // a no-op if already initialized
   bool result =
       tensorflow::Flags::Parse(&env_argv->argc, &env_argv->argv[0], flag_list);
+
+  // There's always at least one unparsed argc, namely the fake argv[0].
+  if (result && env_argv->argc != 1) {
+    // Skip the first argv, which is the fake argv[0].
+    auto unknown_flags = absl::MakeSpan(env_argv->argv);
+    unknown_flags.remove_prefix(1);
+
+    // Some flags are set on XLA_FLAGS, others on TF_XLA_FLAGS.  If we find an
+    // unrecognized flag, suggest the alternative.
+    string alternate_envvar;
+    if (envvar == "TF_XLA_FLAGS") {
+      alternate_envvar = "XLA_FLAGS";
+    } else if (envvar == "XLA_FLAGS") {
+      alternate_envvar = "TF_XLA_FLAGS";
+    }
+    string did_you_mean;
+    if (!alternate_envvar.empty()) {
+      did_you_mean = absl::StrFormat(
+          "\nPerhaps you meant to specify these on the %s envvar?",
+          alternate_envvar);
+    }
+
+    LOG(FATAL) << "Unknown flag" << (unknown_flags.size() > 1 ? "s" : "")
+               << " in " << envvar << ": " << absl::StrJoin(unknown_flags, " ")
+               << did_you_mean;
+    return false;
+  }
   return result;
 }
 
 // Testing only.
-// Reset the env_argv struct so that subsequent calls to ParseFlagsFromEnv()
-// will parse the environment variable (or the file it points to) anew, and set
-// *pargc, and *pargv to point to the internal locations of the argc and argv
-// constructed from the environment.
+//
+// Resets the env_argv struct so that subsequent calls to
+// ParseFlagsFromEnvAndDieIfUnknown() will parse the environment variable (or
+// the file it points to) anew, and set *pargc, and *pargv to point to the
+// internal locations of the argc and argv constructed from the environment.
 void ResetFlagsFromEnvForTesting(absl::string_view envvar, int** pargc,
                                  std::vector<char*>** pargv) {
   tensorflow::mutex_lock lock(env_argv_mu);
diff --git a/tensorflow/compiler/xla/parse_flags_from_env.h b/tensorflow/compiler/xla/parse_flags_from_env.h
index c0742e8b4f..76940a4299 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.h
+++ b/tensorflow/compiler/xla/parse_flags_from_env.h
@@ -16,9 +16,10 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_PARSE_FLAGS_FROM_ENV_H_
 #define TENSORFLOW_COMPILER_XLA_PARSE_FLAGS_FROM_ENV_H_
 
-// This module exports ParseFlagsFromEnv(), which allows other modules to parse
-// flags from an environtment variable, or (if the first non-whitespace in the
-// variable value is not '-'), a file named by that environment variable.
+// This module exports ParseFlagsFromEnvAndDieIfUnknown(), which allows other
+// modules to parse flags from an environtment variable, or (if the first
+// non-whitespace in the variable value is not '-'), a file named by that
+// environment variable.
 //
 // The accepted syntax is that flags arguments are of the form --flag=value or
 // (for boolean flags) --flag, and are whitespace separated.  The <value> may be
@@ -59,8 +60,10 @@ namespace xla {
 // Calls tensorflow::Flags::Parse(argc, argv, flag_list) against any as yet
 // unrecognized flags passed in the environment variable `envvar`, and returns
 // its return value.
-bool ParseFlagsFromEnv(absl::string_view envvar,
-                       const std::vector<tensorflow::Flag>& flag_list);
+//
+// Raises a fatal error if any flags in `envvar` were not recognized.
+bool ParseFlagsFromEnvAndDieIfUnknown(
+    absl::string_view envvar, const std::vector<tensorflow::Flag>& flag_list);
 
 // Used only for testing.  Not to be used by clients.
 void ResetFlagsFromEnvForTesting(absl::string_view envvar, int** pargc,
diff --git a/tensorflow/compiler/xla/parse_flags_from_env_test.cc b/tensorflow/compiler/xla/parse_flags_from_env_test.cc
index 26c15c3e98..3465552ebb 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env_test.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env_test.cc
@@ -39,19 +39,6 @@ static void TestParseFlagsFromEnv(const char* msg) {
   std::vector<char*>* pargv;
   ResetFlagsFromEnvForTesting("TF_XLA_FLAGS", &pargc, &pargv);
 
-  // Ensure that environment variable can be parsed when
-  // no flags are expected.
-  std::vector<tensorflow::Flag> empty_flag_list;
-  bool parsed_ok = ParseFlagsFromEnv("TF_XLA_FLAGS", empty_flag_list);
-  CHECK(parsed_ok) << msg;
-  const std::vector<char*>& argv_first = *pargv;
-  CHECK_NE(argv_first[0], nullptr) << msg;
-  int i = 0;
-  while (argv_first[i] != nullptr) {
-    i++;
-  }
-  CHECK_EQ(i, *pargc) << msg;
-
   // Check that actual flags can be parsed.
   bool simple = false;
   string with_value;
@@ -65,7 +52,7 @@ static void TestParseFlagsFromEnv(const char* msg) {
       tensorflow::Flag("single_quoted", &single_quoted, ""),
       tensorflow::Flag("double_quoted", &double_quoted, ""),
   };
-  parsed_ok = ParseFlagsFromEnv("TF_XLA_FLAGS", flag_list);
+  bool parsed_ok = ParseFlagsFromEnvAndDieIfUnknown("TF_XLA_FLAGS", flag_list);
   CHECK_EQ(*pargc, 1) << msg;
   const std::vector<char*>& argv_second = *pargv;
   CHECK_NE(argv_second[0], nullptr) << msg;
@@ -171,7 +158,8 @@ int main(int argc, char* argv[]) {
       tensorflow::Flag("int_flag", &int_flag, "An integer flag to test with"),
   };
   xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
-  bool parse_ok = xla::ParseFlagsFromEnv("TF_XLA_FLAGS", flag_list);
+  bool parse_ok =
+      xla::ParseFlagsFromEnvAndDieIfUnknown("TF_XLA_FLAGS", flag_list);
   if (!parse_ok) {
     LOG(QFATAL) << "can't parse from environment\n" << usage;
   }
-- 
GitLab


From 0e5e47022f4c32b54a267b8f12dd5ceb2fc680f4 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 15 Nov 2018 10:13:19 -0800
Subject: [PATCH 0300/1554] Fix two bugs in IncreaseDynamismForAutoJitPass:

 - We'd miscompile rewriteable slices that have other rewritable slices as
   input.  This was because we were caching the slice inputs from the first time
   we looked at a rewriteable Slice when a rewrite could have changed one of the
   inputs to that Slice.  Fix this by not caching SliceInputs.

 - We'd sometimes try to create (trivial) ConcatV2 nodes with one input, which
   isn't legal.  Fix this by not creating these trivial ConcatV2 nodes.

PiperOrigin-RevId: 221644004
---
 .../increase_dynamism_for_auto_jit_pass.cc    |  51 +++----
 ...ncrease_dynamism_for_auto_jit_pass_test.cc | 130 +++++++++++++++++-
 2 files changed, 152 insertions(+), 29 deletions(-)

diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
index fb0b0a89d9..a8295bb77b 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
@@ -208,8 +208,12 @@ Status ComputeSliceSize(const Scope& host_scope,
     DCHECK_EQ(slice_size.back().type(), DT_INT64);
   }
 
-  *size = ops::Concat(host_scope.WithOpName("slice_size"), slice_size,
-                      ops::Const(host_scope.WithOpName("concat_axis"), 0));
+  // Trivial ConcatV2 nodes (with exactly one input) are disallowed.
+  *size =
+      slice_size.size() == 1
+          ? slice_size[0]
+          : ops::Concat(host_scope.WithOpName("slice_size"), slice_size,
+                        ops::Const(host_scope.WithOpName("concat_axis"), 0));
   return Status::OK();
 }
 
@@ -242,6 +246,9 @@ Status ConvertTensorFlowSliceToStaticShapedSlice(
                      .WithOpName("static_shaped_slice"),
                  slice_inputs_int64.input, slice_inputs_int64.begin, slice_size)
           .node();
+
+  TF_RETURN_IF_ERROR(main_scope.status());
+
   std::vector<string> compile_time_const_inputs;
   compile_time_const_inputs.push_back("size");
   (*result)->AddAttr(kXlaCompileTimeConstantInputsAttr,
@@ -284,49 +291,45 @@ Status RewriteSlice(Graph* g, Node* slice, const SliceInputs& slice_inputs,
   return Status::OK();
 }
 
-// If `n` is a slice we can rewrite to have a static shape (i.e. have the output
-// shape only depend on the "size" input) then returns the a SliceInputs
-// representing the inputs to `n`.  Otherwise returns nullopt.
-StatusOrOptional<SliceInputs> IsRewritableSlice(Node* n) {
+// Return true if `n` is a slice we can rewrite to have a static shape
+// (i.e. have the output shape only depend on the "size" input).
+xla::StatusOr<bool> IsRewritableSlice(Node* n) {
   if (n->type_string() != "Slice") {
-    return {absl::nullopt};
+    return false;
   }
 
   if (!GetXlaClusterForNode(*n).has_value()) {
     // There is no need to change slice ops outside XLA clusters.
-    return {absl::nullopt};
+    return false;
   }
 
   TF_ASSIGN_OR_RETURN(absl::optional<SliceInputs> slice_inputs,
                       GetSliceInputs(n));
   if (!slice_inputs.has_value()) {
-    return {absl::nullopt};
+    return false;
   }
 
   // If slice_size[i] < -1 for any i then executing the slice will throw an
   // error, and we don't do anything here.
-  bool slice_is_ok = absl::c_all_of(slice_inputs->size_as_vector,
-                                    [](int64 size_i) { return size_i >= -1; });
-  if (!slice_is_ok) {
-    return {absl::nullopt};
-  }
-
-  return slice_inputs;
+  return absl::c_all_of(slice_inputs->size_as_vector,
+                        [](int64 size_i) { return size_i >= -1; });
 }
 
 Status FindAndRewriteSlices(Graph* g, bool* changed) {
-  std::vector<std::pair<Node*, SliceInputs>> slices_to_rewrite;
+  std::vector<Node*> slices_to_rewrite;
   for (Node* n : g->nodes()) {
-    TF_ASSIGN_OR_RETURN(absl::optional<SliceInputs> slice_inputs,
-                        IsRewritableSlice(n));
-    if (slice_inputs.has_value()) {
-      slices_to_rewrite.push_back({n, std::move(*slice_inputs)});
+    TF_ASSIGN_OR_RETURN(bool is_rewritable, IsRewritableSlice(n));
+    if (is_rewritable) {
+      slices_to_rewrite.push_back(n);
     }
   }
 
-  for (const auto& pair : slices_to_rewrite) {
-    TF_RETURN_IF_ERROR(RewriteSlice(g, pair.first, pair.second,
-                                    *GetXlaClusterForNode(*pair.first)));
+  for (Node* n : slices_to_rewrite) {
+    TF_ASSIGN_OR_RETURN(absl::optional<SliceInputs> slice_inputs,
+                        GetSliceInputs(n));
+    TF_RET_CHECK(slice_inputs.has_value());
+    TF_RETURN_IF_ERROR(
+        RewriteSlice(g, n, *slice_inputs, *GetXlaClusterForNode(*n)));
   }
 
   if (!slices_to_rewrite.empty()) {
diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
index 0f6f612e96..a2f1b831ad 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass_test.cc
@@ -27,6 +27,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
+using ::testing::_;
 using testing::matchers::AssignedDevice;
 using testing::matchers::Attr;
 using testing::matchers::Const;
@@ -142,6 +143,26 @@ TEST(SliceToDynamicSliceRewriteTest, Basic) {
   EXPECT_THAT(static_shaped_slice, m_dynamic_slice);
 }
 
+TEST(SliceToDynamicSliceRewriteTest, SliceFromVector) {
+  Scope root = Scope::NewRootScope()
+                   .ExitOnError()
+                   .WithAssignedDevice(kDeviceName)
+                   .WithXlaCluster("cluster_0");
+
+  Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
+  Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT32);
+  Output size = ops::Const(root.WithOpName("size"), {-1});
+  Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size);
+
+  std::unique_ptr<Graph> result;
+  TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result));
+
+  Node* static_shaped_slice = testing::FindNodeByName(
+      result.get(), "slice/static_shaped_slice/static_shaped_slice");
+  EXPECT_NE(static_shaped_slice, nullptr);
+  EXPECT_THAT(result->nodes(), Not(Contains(NodeWith(Op("ConcatV2")))));
+}
+
 TEST(SliceToDynamicSliceRewriteTest, ControlDependencePreserved) {
   Scope root = Scope::NewRootScope()
                    .ExitOnError()
@@ -166,18 +187,18 @@ TEST(SliceToDynamicSliceRewriteTest, ControlDependencePreserved) {
                        CtrlDeps(NodeWith(Op("Placeholder"), Name("control")))));
 }
 
+int64 ToInt64(int v) { return static_cast<int64>(v); }
+
 TEST(SliceToDynamicSliceRewriteTest, Int64Indices) {
   Scope root = Scope::NewRootScope()
                    .ExitOnError()
                    .WithAssignedDevice(kDeviceName)
                    .WithXlaCluster("cluster_0");
 
-  auto to_int64 = [](int v) { return static_cast<int64>(v); };
-
   Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
   Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT64);
   Output size =
-      ops::Const(root.WithOpName("size"), {to_int64(-1), to_int64(500)});
+      ops::Const(root.WithOpName("size"), {ToInt64(-1), ToInt64(500)});
   Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size);
 
   std::unique_ptr<Graph> result;
@@ -252,13 +273,35 @@ TEST(SliceToDynamicSliceRewriteTest, DontRewriteSliceWithNonConstSize) {
                                     Attr(kXlaCompileTimeConstantInputsAttr)))));
 }
 
+TEST(SliceToDynamicSliceRewriteTest, ScalarSlice) {
+  Scope root = Scope::NewRootScope()
+                   .ExitOnError()
+                   .WithAssignedDevice(kDeviceName)
+                   .WithXlaCluster("cluster_0");
+
+  Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
+  Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT64);
+  Output size = ops::Const<int64>(root.WithOpName("size"), {});
+  Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size);
+
+  std::unique_ptr<Graph> result;
+  TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result));
+
+  Node* static_shaped_slice = testing::FindNodeByName(
+      result.get(), "slice/static_shaped_slice/static_shaped_slice");
+  ASSERT_NE(static_shaped_slice, nullptr);
+  EXPECT_THAT(static_shaped_slice,
+              NodeWith(Op("Slice"), Attr(kXlaCompileTimeConstantInputsAttr),
+                       Inputs(_, _, Out(NodeWith(Name(size.node()->name()))))));
+}
+
 TEST(SliceToDynamicSliceRewriteTest, IndicesNotVector) {
   Scope root = Scope::NewRootScope()
                    .ExitOnError()
                    .WithAssignedDevice(kDeviceName)
                    .WithXlaCluster("cluster_0");
 
-  auto to_int64 = [](int v) { return static_cast<int64>(v); };
+  auto ToInt64 = [](int v) { return static_cast<int64>(v); };
 
   Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
   Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT64);
@@ -271,7 +314,7 @@ TEST(SliceToDynamicSliceRewriteTest, IndicesNotVector) {
       ops::Slice(root.WithOpName("slice"), input, begin, size_placeholder);
 
   Output size =
-      ops::Const(root.WithOpName("size"), {{to_int64(-1)}, {to_int64(500)}});
+      ops::Const(root.WithOpName("size"), {{ToInt64(-1)}, {ToInt64(500)}});
   TF_ASSERT_OK(root.graph()->UpdateEdge(size.node(), 0, slice.node(), 2));
 
   std::unique_ptr<Graph> result;
@@ -281,5 +324,82 @@ TEST(SliceToDynamicSliceRewriteTest, IndicesNotVector) {
               Not(Contains(NodeWith(Op("Slice"),
                                     Attr(kXlaCompileTimeConstantInputsAttr)))));
 }
+
+TEST(SliceToDynamicSliceRewriteTest, SliceWithSliceInput) {
+  Scope root = Scope::NewRootScope()
+                   .ExitOnError()
+                   .WithAssignedDevice(kDeviceName)
+                   .WithXlaCluster("cluster_0");
+
+  Output input = ops::Placeholder(root.WithOpName("input"), DT_FLOAT);
+  Output begin = ops::Placeholder(root.WithOpName("begin"), DT_INT32);
+  Output size_a = ops::Const(root.WithOpName("size_a"), {-1, 500});
+  Output slice = ops::Slice(root.WithOpName("slice"), input, begin, size_a);
+
+  Output size_b = ops::Const(root.WithOpName("size_a"), {-1, 200});
+  Output slice_with_slice_input = ops::Slice(
+      root.WithOpName("slice_with_slice_input"), slice, begin, size_b);
+
+  std::unique_ptr<Graph> result;
+  TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result));
+
+  Node* static_shaped_slice = testing::FindNodeByName(
+      result.get(),
+      "slice_with_slice_input/static_shaped_slice/static_shaped_slice");
+  ASSERT_NE(static_shaped_slice, nullptr);
+  EXPECT_EQ(static_shaped_slice->output_type(0), DT_FLOAT)
+      << "Expected DT_FLOAT, was "
+      << DataType_Name(static_shaped_slice->output_type(0));
+  EXPECT_THAT(
+      static_shaped_slice,
+      NodeWith(
+          Op("Slice"),
+          Inputs(Out(NodeWith(
+                     Op("Slice"),
+                     Name("slice/static_shaped_slice/static_shaped_slice"))),
+                 _, _)));
+}
+
+TEST(SliceToDynamicSliceRewriteTest, SliceWithSliceBegin) {
+  Scope root = Scope::NewRootScope()
+                   .ExitOnError()
+                   .WithAssignedDevice(kDeviceName)
+                   .WithXlaCluster("cluster_0");
+
+  Output input_float =
+      ops::Placeholder(root.WithOpName("input_float"), DT_FLOAT);
+  Output input_i64 = ops::Placeholder(root.WithOpName("input_i64"), DT_INT64);
+
+  Output begin_begin =
+      ops::Placeholder(root.WithOpName("begin_begin"), DT_INT32);
+  Output begin_size = ops::Const(root.WithOpName("begin_size"), {-1});
+  Output begin =
+      ops::Slice(root.WithOpName("begin"), input_i64, begin_begin, begin_size);
+
+  Output size =
+      ops::Const(root.WithOpName("size"), {ToInt64(-1), ToInt64(200)});
+  Output slice_with_slice_begin = ops::Slice(
+      root.WithOpName("slice_with_slice_begin"), input_float, begin, size);
+
+  std::unique_ptr<Graph> result;
+  TF_ASSERT_OK(IncreaseDynamismForAutoJit(root, &result));
+
+  Node* static_shaped_slice = testing::FindNodeByName(
+      result.get(),
+      "slice_with_slice_begin/static_shaped_slice/static_shaped_slice");
+  ASSERT_NE(static_shaped_slice, nullptr);
+  EXPECT_EQ(static_shaped_slice->output_type(0), DT_FLOAT)
+      << "Expected DT_FLOAT, was "
+      << DataType_Name(static_shaped_slice->output_type(0));
+  EXPECT_THAT(
+      static_shaped_slice,
+      NodeWith(
+          Op("Slice"),
+          Inputs(_,
+                 Out(NodeWith(
+                     Op("Slice"),
+                     Name("begin/static_shaped_slice/static_shaped_slice"))),
+                 _)));
+}
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 73e3215c3a2edadbf9111cca44ab3d5ca146c327 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 10:17:39 -0800
Subject: [PATCH 0301/1554] Apparently some users still haven't migrated to
 cuda 9.  Fixes for them.

Automated rollback of commit 6f0f058bced3bd07e3df33ed66163715db28757d

PiperOrigin-RevId: 221644777
---
 tensorflow/core/graph/testlib.cc              |  11 +
 tensorflow/core/graph/testlib.h               |   4 +
 tensorflow/core/kernels/BUILD                 |  25 +-
 tensorflow/core/kernels/scan_ops_gpu.cu.cc    | 266 +++++++++++++++++-
 tensorflow/core/kernels/scan_ops_test.cc      | 146 ++++++++++
 .../core/util/permutation_input_iterator.h    |   6 +-
 .../core/util/permutation_output_iterator.h   | 129 +++++++++
 .../python/kernel_tests/conv_ops_test.py      |   2 +-
 .../python/kernel_tests/scan_ops_test.py      |   5 +
 9 files changed, 588 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/core/kernels/scan_ops_test.cc
 create mode 100644 tensorflow/core/util/permutation_output_iterator.h

diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc
index 0a38aa1c91..0e74a30c7a 100644
--- a/tensorflow/core/graph/testlib.cc
+++ b/tensorflow/core/graph/testlib.cc
@@ -123,6 +123,17 @@ Node* Assign(Graph* g, Node* var, Node* val) {
   return ret;
 }
 
+Node* Cumsum(Graph* g, Node* data, Node* axes, bool exclusive, bool reverse) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Cumsum")
+                  .Input(data)
+                  .Input(axes)
+                  .Attr("exclusive", exclusive)
+                  .Attr("reverse", reverse)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
 Node* Reduce(Graph* g, const string& reduce, Node* data, Node* axes,
              bool keep_dims) {
   Node* ret;
diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h
index b00196f587..0c7233161f 100644
--- a/tensorflow/core/graph/testlib.h
+++ b/tensorflow/core/graph/testlib.h
@@ -68,6 +68,10 @@ Node* Recv(Graph* g, const string& tensor, const string& type,
            const string& sender, const uint64 sender_incarnation,
            const string& receiver);
 
+// Adds a cumsum "node" in "g" doing cumsum(data, axes).
+Node* Cumsum(Graph* g, Node* data, Node* axes, bool exclusive = false,
+             bool reverse = false);
+
 // Adds a reduction "node" in "g" doing sum(data, axes).  "reduce" is
 // a reduction, e.g., Sum, Max, Min, Mean, etc.
 Node* Reduce(Graph* g, const string& reduce, Node* data, Node* axes,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 5fd0f5d5d1..9065989657 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3197,7 +3197,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "scan_ops",
     prefix = "scan_ops",
-    deps = MATH_DEPS,
+    deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
 tf_kernel_library(
@@ -3375,6 +3375,29 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "scan_ops_test",
+    size = "small",
+    srcs = ["scan_ops_test.cc"],
+    linkopts = select({
+        "//tensorflow:darwin": ["-headerpad_max_install_names"],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":host_constant_op",
+        ":ops_testutil",
+        ":ops_util",
+        ":scan_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "reduction_ops_test",
     size = "small",
diff --git a/tensorflow/core/kernels/scan_ops_gpu.cu.cc b/tensorflow/core/kernels/scan_ops_gpu.cu.cc
index ed6c6affce..ed66c02dc5 100644
--- a/tensorflow/core/kernels/scan_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/scan_ops_gpu.cu.cc
@@ -1,4 +1,4 @@
-/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,8 +17,20 @@ limitations under the License.
 
 #define EIGEN_USE_GPU
 
+#if CUDA_VERSION >= 9000
+#define CUB_USE_COOPERATIVE_GROUPS
+#endif  // CUDA_VERSION >= 9000
+
+#include "third_party/cub/block/block_load.cuh"
+#include "third_party/cub/block/block_scan.cuh"
+#include "third_party/cub/block/block_store.cuh"
+#include "third_party/cub/iterator/counting_input_iterator.cuh"
+#include "third_party/cub/iterator/transform_input_iterator.cuh"
+#include "cuda/include/cuComplex.h"
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/util/permutation_input_iterator.h"
+#include "tensorflow/core/util/permutation_output_iterator.h"
 
 #include "tensorflow/core/kernels/scan_ops.h"
 
@@ -27,6 +39,258 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 typedef Eigen::Index Index;
 
+namespace functor {
+
+// Map a contiguous range to the actual memory locations depending on which
+// axis the scan is taking place over and whether or not reversed.
+struct MapIndexToLocation {
+  __host__ __device__ MapIndexToLocation(int dimx, int dimy, int dimz,
+                                         bool reverse = false)
+      : dimx_(dimx), dimy_(dimy), dimz_(dimz), reverse_(reverse) {}
+
+  __host__ __device__ int operator()(int id) const {
+    if (dimx_ == 1) {
+      int row = id % dimy_;
+      int col = id / dimy_;
+
+      if (reverse_) return (dimy_ - row - 1) * dimz_ + col;
+
+      return row * dimz_ + col;
+    } else if (dimz_ == 1) {
+      if (reverse_) {
+        int row = id / dimy_;
+        int col = id % dimy_;
+        return row * dimy_ + (dimy_ - col - 1);
+      }
+      return id;
+    } else {
+      int col = id % dimy_;
+      int tmp = id / dimy_;
+
+      int row1 = id / (dimy_ * dimz_);
+      int col1 = tmp % dimz_;
+
+      if (reverse_)
+        return row1 * dimy_ * dimz_ + (dimy_ - col - 1) * dimz_ + col1;
+
+      return row1 * dimy_ * dimz_ + col * dimz_ + col1;
+    }
+  }
+
+  int dimx_;
+  int dimy_;
+  int dimz_;
+  bool reverse_;
+};
+
+template <typename T, typename Op>
+struct BlockPrefixCallbackOp {
+  // Running prefix
+  T running_total_;
+  Op op_;
+
+  __device__ BlockPrefixCallbackOp(T running_total, Op op)
+      : running_total_(running_total), op_(op) {}
+
+  // Callback operator to be entered by the first warp of threads in the block.
+  // tid 0 is responsible for returning a value for seeding the block-wide scan.
+  __device__ T operator()(T block_aggregate) {
+    T old_prefix = running_total_;
+    running_total_ = op_(old_prefix, block_aggregate);
+    return old_prefix;
+  }
+};
+
+template <typename T>
+struct Sum {
+  __host__ __device__ T operator()(const T& a, const T& b) const {
+    return a + b;
+  }
+};
+
+template <typename T>
+struct Prod {
+  __host__ __device__ T operator()(const T& a, const T& b) const {
+    return a * b;
+  }
+};
+
+template <typename T, typename Op>
+struct IsSum {
+  constexpr static bool value =
+      (std::is_same<Op, Sum<T>>::value ||
+       std::is_same<Op, Eigen::internal::SumReducer<T>>::value);
+};
+
+template <typename T, typename Op>
+struct IsProd {
+  constexpr static bool value =
+      (std::is_same<Op, Prod<T>>::value ||
+       std::is_same<Op, Eigen::internal::ProdReducer<T>>::value);
+};
+
+template <typename T, typename Op>
+struct IdentityValue {
+  static_assert(IsSum<T, Op>::value || IsProd<T, Op>::value,
+                "IdentityValue not yet defined for this type.");
+
+  template <typename U = T, typename OpCopy = Op>
+  __host__ __device__ U operator()(
+      typename std::enable_if<IsSum<U, OpCopy>::value, U>::type t = U(0)) {
+    return t;
+  }
+
+  template <typename U = T, typename OpCopy = Op>
+  __host__ __device__ U operator()(
+      typename std::enable_if<IsProd<U, OpCopy>::value, U>::type t = U(1)) {
+    return t;
+  }
+};
+
+// Each block is mapped to one sequence.  A contiguous range is mapped to the
+// appropriate locations in memory by the permutation iterators.  This is
+// ideal for 1-D and row based scans.  Column scans would be better if they
+// did a block load and then locally transposed.  CUB's device wide scan is not
+// used in the large 1D case, even though it would be more efficient, because
+// it is not deterministic.
+template <typename T, typename Op, int BlockDim = 128, int ItemsPerThread = 4>
+__global__ void scan_kernel(const T* in, T* out, int dimx, int dimy, int dimz,
+                            bool exclusive, bool reverse, Op op) {
+  typedef cub::BlockLoad<T, BlockDim, ItemsPerThread, cub::BLOCK_LOAD_TRANSPOSE>
+      BlockLoad;
+  typedef cub::BlockStore<T, BlockDim, ItemsPerThread,
+                          cub::BLOCK_STORE_TRANSPOSE>
+      BlockStore;
+  typedef cub::BlockScan<T, BlockDim> BlockScan;
+
+  // Allocate aliased shared memory for BlockLoad, BlockStore, and BlockScan
+  __shared__ union {
+    typename BlockLoad::TempStorage load;
+    typename BlockScan::TempStorage scan;
+    typename BlockStore::TempStorage store;
+  } temp_storage;
+
+  int problem_length = dimy;
+
+  // Initialize running total
+  BlockPrefixCallbackOp<T, Op> prefix_op(IdentityValue<T, Op>()(), op);
+
+  MapIndexToLocation map_op(dimx, dimy, dimz, reverse);
+  int block_start = problem_length * blockIdx.x;
+  // Have the block iterate over segments of items
+  for (int block_offset = block_start;
+       block_offset < block_start + problem_length;
+       block_offset += BlockDim * ItemsPerThread) {
+    int valid_items = min(BlockDim * ItemsPerThread,
+                          problem_length - (block_offset % problem_length));
+
+    // first construct a counting iterator that has the desired start point
+    typedef cub::TransformInputIterator<int, MapIndexToLocation,
+                                        cub::CountingInputIterator<int>>
+        MapIterType;
+
+    cub::CountingInputIterator<int> counting_iter(block_offset);
+
+    // Next map the iterator to the actual locations in memory
+    MapIterType map_iter(counting_iter, map_op);
+
+    PermutationInputIterator<T, const T*, MapIterType> permutein_iter(in,
+                                                                      map_iter);
+    PermutationOutputIterator<T, T*, MapIterType> permuteout_iter(out,
+                                                                  map_iter);
+
+    // Load a segment of consecutive items that are blocked across threads
+    T thread_data[ItemsPerThread];
+    BlockLoad(temp_storage.load).Load(permutein_iter, thread_data, valid_items);
+    __syncthreads();
+
+    // Collectively compute the block-wide scan
+    if (exclusive) {
+      BlockScan(temp_storage.scan)
+          .ExclusiveScan(thread_data, thread_data, op, prefix_op);
+    } else {
+      BlockScan(temp_storage.scan)
+          .InclusiveScan(thread_data, thread_data, op, prefix_op);
+    }
+    __syncthreads();
+
+    // Store scanned items to output segment
+    BlockStore(temp_storage.store)
+        .Store(permuteout_iter, thread_data, valid_items);
+    __syncthreads();
+  }
+}
+
+template <typename T, typename Op>
+void LaunchScan(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
+                typename TTypes<T, 3>::Tensor out, Op op, const bool reverse,
+                const bool exclusive) {
+  const int items_per_thread = 4;
+
+  int dimx = in.dimension(0);
+  int dimy = in.dimension(1);
+  int dimz = in.dimension(2);
+  int num_blocks = dimx * dimz;
+
+  int ideal_block_size = dimy / items_per_thread;
+
+  // There seems to be a bug when the type is not float and block_size 1024.
+  // Launch on the smallest power of 2 block size that we can.
+  if (ideal_block_size >= 1024 && std::is_same<T, float>::value) {
+    const int block_size = 1024;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 512) {
+    const int block_size = 512;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 256) {
+    const int block_size = 256;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 128) {
+    const int block_size = 128;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else if (ideal_block_size >= 64) {
+    const int block_size = 64;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  } else {
+    const int block_size = 32;
+    scan_kernel<T, Op, block_size, items_per_thread>
+        <<<num_blocks, block_size, 0, d.stream()>>>(
+            in.data(), out.data(), dimx, dimy, dimz, exclusive, reverse, op);
+  }
+}
+
+template <typename T>
+struct Scan<GPUDevice, Eigen::internal::SumReducer<T>, T> {
+  void operator()(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
+                  typename TTypes<T, 3>::Tensor out,
+                  const Eigen::internal::SumReducer<T>& reducer,
+                  const bool reverse, const bool exclusive) {
+    LaunchScan<T, Sum<T>>(d, in, out, Sum<T>(), reverse, exclusive);
+  }
+};
+
+template <typename T>
+struct Scan<GPUDevice, Eigen::internal::ProdReducer<T>, T> {
+  void operator()(const GPUDevice& d, typename TTypes<T, 3>::ConstTensor in,
+                  typename TTypes<T, 3>::Tensor out,
+                  const Eigen::internal::ProdReducer<T>& reducer,
+                  const bool reverse, const bool exclusive) {
+    LaunchScan<T, Prod<T>>(d, in, out, Prod<T>(), reverse, exclusive);
+  }
+};
+
+}  // namespace functor
+
 #define DEFINE(REDUCER, T) template struct functor::Scan<GPUDevice, REDUCER, T>;
 
 #define DEFINE_FOR_ALL_REDUCERS(T)           \
diff --git a/tensorflow/core/kernels/scan_ops_test.cc b/tensorflow/core/kernels/scan_ops_test.cc
new file mode 100644
index 0000000000..588b606a99
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_test.cc
@@ -0,0 +1,146 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+template <typename T>
+static Graph* LargeOneDCumsum(int num_x, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DataTypeToEnum<T>::value, TensorShape({num_x}));
+  data.flat<T>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 0;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+static Graph* ColCumsum(int num_x, int num_y, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DT_FLOAT, TensorShape({num_x, num_y}));
+  data.flat<float>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 0;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+static Graph* RowCumsum(int num_x, int num_y, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DT_FLOAT, TensorShape({num_x, num_y}));
+  data.flat<float>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 1;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+static Graph* ThreeDYCumsum(int num_y, int num_z, bool reverse = false) {
+  auto* g = new Graph(OpRegistry::Global());
+  Tensor data(DT_FLOAT, TensorShape({32, num_y, num_z}));
+  data.flat<float>().setRandom();
+  Tensor axes(DT_INT32, TensorShape({}));
+  axes.flat<int32>()(0) = 1;
+  test::graph::Cumsum(g, test::graph::Constant(g, data),
+                      test::graph::Constant(g, axes));
+  return g;
+}
+
+template <typename T>
+static void LargeOneDimensional(int iters, const string& device, int num_x,
+                                bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * sizeof(T));
+  test::Benchmark(device, LargeOneDCumsum<T>(num_x, reverse)).Run(iters);
+}
+
+static void DoRowCumsum(int iters, const string& device, int num_x, int num_y,
+                        bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
+                          sizeof(float));
+  test::Benchmark(device, RowCumsum(num_x, num_y, reverse)).Run(iters);
+}
+
+static void DoColCumsum(int iters, const string& device, int num_x, int num_y,
+                        bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
+                          sizeof(float));
+  test::Benchmark(device, ColCumsum(num_x, num_y, reverse)).Run(iters);
+}
+
+static void Do3DYCumsum(int iters, const string& device, int num_x, int num_y,
+                        bool reverse = false) {
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_x * num_y);
+  testing::BytesProcessed(static_cast<int64>(iters) * num_x * num_y *
+                          sizeof(float));
+  test::Benchmark(device, ThreeDYCumsum(num_x, num_y, reverse)).Run(iters);
+}
+
+static void BM_OneDCumsumGPU(int iters, int num_x) {
+  LargeOneDimensional<float>(iters, "gpu", num_x);
+}
+BENCHMARK(BM_OneDCumsumGPU)->Range(1, 1 << 21);
+
+static void BM_OneDCumsumGPUHalf(int iters, int num_x) {
+  LargeOneDimensional<Eigen::half>(iters, "gpu", num_x);
+}
+BENCHMARK(BM_OneDCumsumGPUHalf)->Range(1, 1 << 21);
+
+static void BM_Sum2DRowCumsumGPU(int iters, int num_x, int num_y) {
+  DoRowCumsum(iters, "gpu", num_x, num_y);
+}
+BENCHMARK(BM_Sum2DRowCumsumGPU)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum2DColumnCumsumGPU(int iters, int num_x, int num_y) {
+  DoColCumsum(iters, "gpu", num_x, num_y);
+}
+BENCHMARK(BM_Sum2DColumnCumsumGPU)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum3DYCumsumGPU(int iters, int num_x, int num_y) {
+  Do3DYCumsum(iters, "gpu", num_x, num_y);
+}
+BENCHMARK(BM_Sum3DYCumsumGPU)->RangePair(64, 4096, 64, 4096);
+
+static void BM_OneDCumsumGPU_reverse(int iters, int num_x) {
+  LargeOneDimensional<float>(iters, "gpu", num_x, true);
+}
+BENCHMARK(BM_OneDCumsumGPU_reverse)->Range(1, 1 << 21);
+
+static void BM_Sum2DRowCumsumGPU_reverse(int iters, int num_x, int num_y) {
+  DoRowCumsum(iters, "gpu", num_x, num_y, true);
+}
+BENCHMARK(BM_Sum2DRowCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum2DColumnCumsumGPU_reverse(int iters, int num_x, int num_y) {
+  DoColCumsum(iters, "gpu", num_x, num_y, true);
+}
+BENCHMARK(BM_Sum2DColumnCumsumGPU_reverse)->RangePair(1, 8192, 1, 8192);
+
+static void BM_Sum3DYCumsumGPU_reverse(int iters, int num_x, int num_y) {
+  Do3DYCumsum(iters, "gpu", num_x, num_y, true);
+}
+BENCHMARK(BM_Sum3DYCumsumGPU_reverse)->RangePair(32, 2048, 32, 2048);
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/util/permutation_input_iterator.h b/tensorflow/core/util/permutation_input_iterator.h
index f6375b2515..649318ebf3 100644
--- a/tensorflow/core/util/permutation_input_iterator.h
+++ b/tensorflow/core/util/permutation_input_iterator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
-#define TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#define TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
 
 #include <iostream>
 #include <iterator>
@@ -131,4 +131,4 @@ class PermutationInputIterator {
 
 }  // end namespace tensorflow
 
-#endif  // TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_
+#endif  // TENSORFLOW_CORE_UTIL_PERMUTATION_INPUT_ITERATOR_H_
diff --git a/tensorflow/core/util/permutation_output_iterator.h b/tensorflow/core/util/permutation_output_iterator.h
new file mode 100644
index 0000000000..638c0f4545
--- /dev/null
+++ b/tensorflow/core/util/permutation_output_iterator.h
@@ -0,0 +1,129 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
+#define TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
+
+#include <iostream>
+#include <iterator>
+
+namespace tensorflow {
+
+template <typename ValueType, typename OutputIteratorT, typename IndexIteratorT,
+          typename OffsetT = ptrdiff_t>
+class PermutationOutputIterator {
+ public:
+  // Required iterator traits
+  typedef PermutationOutputIterator self_type;  ///< My own type
+  typedef OffsetT difference_type;  ///< Type to express the result of
+                                    ///< subtracting one iterator from another
+  typedef ValueType
+      value_type;  ///< The type of the element the iterator can point to
+  typedef ValueType* pointer;    ///< The type of a pointer to an element the
+                                 ///< iterator can point to
+  typedef ValueType& reference;  ///< The type of a reference to an element the
+                                 ///< iterator can point to
+
+  typedef std::random_access_iterator_tag
+      iterator_category;  ///< The iterator category
+
+ private:
+  OutputIteratorT output_itr;
+  IndexIteratorT index_itr;
+
+ public:
+  /// Constructor
+  __host__ __device__ __forceinline__ PermutationOutputIterator(
+      OutputIteratorT output_itr,  ///< Input iterator to wrap
+      IndexIteratorT index_itr)    ///< Conversion functor to wrap
+      : output_itr(output_itr), index_itr(index_itr) {}
+
+  /// Postfix increment
+  __host__ __device__ __forceinline__ self_type operator++(int) {
+    self_type retval = *this;
+    index_itr++;
+    return retval;
+  }
+
+  /// Prefix increment
+  __host__ __device__ __forceinline__ self_type operator++() {
+    index_itr++;
+    return *this;
+  }
+
+  /// Indirection
+  __host__ __device__ __forceinline__ reference operator*() const {
+    return output_itr[*index_itr];
+  }
+
+  /// Addition
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type operator+(Distance n) const {
+    self_type retval(output_itr, index_itr + n);
+    return retval;
+  }
+
+  /// Addition assignment
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type& operator+=(Distance n) {
+    index_itr += n;
+    return *this;
+  }
+
+  /// Subtraction
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type operator-(Distance n) const {
+    self_type retval(output_itr, index_itr - n);
+    return retval;
+  }
+
+  /// Subtraction assignment
+  template <typename Distance>
+  __host__ __device__ __forceinline__ self_type& operator-=(Distance n) {
+    index_itr -= n;
+    return *this;
+  }
+
+  /// Distance
+  __host__ __device__ __forceinline__ difference_type
+  operator-(self_type other) const {
+    return index_itr - other.index_itr;
+  }
+
+  /// Array subscript
+  template <typename Distance>
+  __host__ __device__ __forceinline__ reference operator[](Distance n) const {
+    return output_itr[index_itr[n]];
+  }
+
+  /// Equal to
+  __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) {
+    return (index_itr == rhs.index_itr && output_itr == rhs.output_itr);
+  }
+
+  /// Not equal to
+  __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) {
+    return !(*this == rhs);
+  }
+
+  /// ostream operator
+  friend std::ostream& operator<<(std::ostream& os, const self_type& itr) {
+    return os;
+  }
+};
+
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_UTIL_PERMUTATION_OUTPUT_ITERATOR_H_
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 0ccbbf155c..835cc1504d 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -1669,7 +1669,7 @@ class SeparableConv2DTest(test.TestCase):
 
       value = sess.run(conv)
     tf_logging.info("value = ", value)
-    self.assertArrayNear(expected, np.ravel(value), 1e-5)
+    self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
 
   def _testSeparableConv2D(self, data_format):
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index b369222565..c48e0e2e67 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -126,6 +126,11 @@ class CumsumTest(test.TestCase):
       for axis in range(-6, 6, 3):
         self._compareAll(x, axis)
 
+  def testLarge(self):
+    for dtype in self.valid_dtypes:
+      x = np.ones([1000000], dtype=dtype) / 1024
+      self._compareAll(x, 0)
+
   def testInvalidAxis(self):
     x = np.arange(0, 10).reshape([2, 5]).astype(np.float32)
     input_tensor = ops.convert_to_tensor(x)
-- 
GitLab


From 848dd8dd287c93c3e109af68654283845a8900db Mon Sep 17 00:00:00 2001
From: sahilbadyal <sbadyals@gmail.com>
Date: Fri, 16 Nov 2018 00:12:48 +0530
Subject: [PATCH 0302/1554] changed the download version of protobuf to 3.6.0

---
 tensorflow/contrib/makefile/download_dependencies.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 0a07588f07..b396c52767 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -34,7 +34,7 @@ NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.
 # 1.10 branch does not work. `make distclean` fails and blocks the build
 # process. For now we're hardcoding to the version which is used by
 # TensorFlow 1.9.
-PROTOBUF_URL="https://mirror.bazel.build/github.com/google/protobuf/archive/396336eb961b75f03b25824fe86cf6490fb75e3a.tar.gz"
+PROTOBUF_URL="https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.0.tar.gz"
 # TODO (yongtang): Replace the following with 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' once
 # the archive has been propagated in mirror.bazel.build.
 RE2_URL="$(grep -o 'https://github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
-- 
GitLab


From 56e2c8224e99e2d97c63f8baf6bd1034ed61da36 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 10:21:11 -0800
Subject: [PATCH 0303/1554] Automated rollback of commit
 82eb2d85c1f6ae56cc6c092bdc3f386278241e6e

PiperOrigin-RevId: 221645406
---
 tensorflow/workspace.bzl         |  9 +++---
 third_party/eigen_reshaped.patch | 48 --------------------------------
 2 files changed, 4 insertions(+), 53 deletions(-)
 delete mode 100644 third_party/eigen_reshaped.patch

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 42d92f7b49..b4f7aba046 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,12 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        patch_file = clean_dep("//third_party:eigen_reshaped.patch"),
-        sha256 = "d66cec3b54b3dfaa4666c1d49481a7197f93fc078cd53c54e2b4a8893a529c9f",
-        strip_prefix = "eigen-eigen-b4890dc6bc34",
+        sha256 = "1e045bef75e9b17d459b60cc30b34408f3fdab300c5053d3919d1a5921f3c86a",
+        strip_prefix = "eigen-eigen-af2071407280",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/b4890dc6bc34.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
         ],
     )
 
diff --git a/third_party/eigen_reshaped.patch b/third_party/eigen_reshaped.patch
deleted file mode 100644
index 7acfdcf9fe..0000000000
--- a/third_party/eigen_reshaped.patch
+++ /dev/null
@@ -1,48 +0,0 @@
---- a/Eigen/src/Core/util/ReshapedHelper.h	(date 1541195478000)
-+++ b/Eigen/src/Core/util/ReshapedHelper.h	(date 1541195478000)
-@@ -39,6 +39,11 @@
-   return total/other;
- }
-
-+template<int Flags, int Order>
-+struct get_compiletime_reshape_order {
-+  enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order };
-+};
-+
- }
-
- } // end namespace Eigen
---- a/Eigen/src/plugins/ReshapedMethods.h	(date 1541195254000)
-+++ b/Eigen/src/plugins/ReshapedMethods.h	(date 1541195254000)
-@@ -105,13 +105,13 @@
- inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,
-                 internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value,
-                 internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value,
--                (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+                internal::get_compiletime_reshape_order<Flags,Order>::value>
- reshaped(NRowsType nRows, NColsType nCols) EIGEN_RESHAPED_METHOD_CONST
- {
-   return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived,
-                   internal::get_compiletime_reshape_size<NRowsType,NColsType,SizeAtCompileTime>::value,
-                   internal::get_compiletime_reshape_size<NColsType,NRowsType,SizeAtCompileTime>::value,
--                  (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+                  internal::get_compiletime_reshape_order<Flags,Order>::value>
-                 (derived(),
-                  internal::get_runtime_reshape_size(nRows,internal::get_runtime_value(nCols),size()),
-                  internal::get_runtime_reshape_size(nCols,internal::get_runtime_value(nRows),size()));
-@@ -128,11 +128,13 @@
-
- template<int Order>
- EIGEN_DEVICE_FUNC
--inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+inline Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1,
-+                internal::get_compiletime_reshape_order<Flags,Order>::value>
- reshaped() EIGEN_RESHAPED_METHOD_CONST
- {
-   EIGEN_STATIC_ASSERT(Order==RowMajor || Order==ColMajor || Order==AutoOrder, INVALID_TEMPLATE_PARAMETER);
--  return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1, (Order==AutoOrder?Flags&RowMajorBit:Order)>
-+  return Reshaped<EIGEN_RESHAPED_METHOD_CONST Derived, SizeAtCompileTime, 1,
-+                  internal::get_compiletime_reshape_order<Flags,Order>::value>
-                 (derived(), size(), 1);
- }
- 
\ No newline at end of file
-- 
GitLab


From ffde93148d82093c9cd8703e8d7d81bff67a1dc6 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 15 Nov 2018 10:25:28 -0800
Subject: [PATCH 0304/1554] [Grappler] Remapper optimizer for
 Conv2D+FusedBatchNorm:

 (1) Conv2D + FusedBatchNorm
 (2) Conv2D + FusedBatchNorm + Relu

PiperOrigin-RevId: 221646158
---
 tensorflow/core/grappler/graph_view.cc        |   6 +
 tensorflow/core/grappler/graph_view.h         |   5 +-
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../core/grappler/optimizers/remapper.cc      | 316 +++++++++++++++---
 .../core/grappler/optimizers/remapper_test.cc | 152 +++++++++
 5 files changed, 427 insertions(+), 54 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.cc b/tensorflow/core/grappler/graph_view.cc
index 35675fb1a2..ba9d2eb321 100644
--- a/tensorflow/core/grappler/graph_view.cc
+++ b/tensorflow/core/grappler/graph_view.cc
@@ -70,6 +70,12 @@ bool HasSingleFanoutNode(const GraphView& graph_view, const NodeDef* node,
   return fanout.size() <= 1;
 }
 
+bool HasFanouts(const GraphView& graph_view, const NodeDef* node, int port) {
+  const auto output = GraphView::OutputPort(node, port);
+  const auto fanout = graph_view.GetFanout(output);
+  return !fanout.empty();
+}
+
 bool NoControlFanin(const GraphView& graph_view, const NodeDef* node) {
   const auto control_port = GraphView::InputPort(node, -1);
   return graph_view.GetFanin(control_port).empty();
diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index 89cec2eb2e..0a47b22565 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -342,10 +342,13 @@ class GraphView
   }
 };
 
-// Returns true if node has one (or zero) fanout nodes at given port.
+// Returns true if node has one (or zero) fanout nodes at given output port.
 bool HasSingleFanoutNode(const GraphView& graph_view, const NodeDef* node,
                          int port = 0);
 
+// Returns true if node has at least one fanout node at given output port.
+bool HasFanouts(const GraphView& graph_view, const NodeDef* node, int port = 0);
+
 bool NoControlFanin(const GraphView& graph_view, const NodeDef* node);
 bool NoControlFanout(const GraphView& graph_view, const NodeDef* node);
 bool NoControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node);
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 3a5b1334d3..904fd42844 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -708,6 +708,8 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/utils:topological_sort",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index b97a95f302..8b93411358 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/remapper.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/graph_view.h"
@@ -22,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
@@ -29,22 +31,55 @@ namespace grappler {
 
 namespace {
 
+constexpr char kFusedConv2D[] = "_FusedConv2D";
+
+constexpr char kDataFormat[] = "data_format";
+constexpr char kIsTraining[] = "is_training";
+
+struct RemapperContext {
+  RemapperContext(const GrapplerItem& item)
+      : nodes_to_preserve(item.NodesToPreserve()),
+        graph_view(&item.graph),
+        graph_properties(item),
+        inferred_graph_properties(false) {}
+
+  std::unordered_set<string> nodes_to_preserve;
+  GraphView graph_view;
+  GraphProperties graph_properties;
+  bool inferred_graph_properties;
+};
+
 // FusedBatchNorm that can be replaced with a cheaper set of primitives.
 struct FusedBatchNorm {
-  const NodeDef* fused_batch_norm;
+  const NodeDef* fused_batch_norm = nullptr;
 };
 
 // Conv2D node followed by a BiasAdd.
 struct Conv2DWithBiasAdd {
-  const NodeDef* conv2d;
-  const NodeDef* bias_add;
+  const NodeDef* conv2d = nullptr;
+  const NodeDef* bias_add = nullptr;
 };
 
 // Conv2D node followed by a BiasAdd and Relu.
 struct Conv2DWithBiasAddAndRelu {
-  const NodeDef* conv2d;
-  const NodeDef* bias_add;
-  const NodeDef* relu;
+  const NodeDef* conv2d = nullptr;
+  const NodeDef* bias_add = nullptr;
+  const NodeDef* relu = nullptr;
+};
+
+// Conv2D node followed by a FusedBatchNorm.
+struct Conv2DWithBatchNorm {
+  const NodeDef* conv2d = nullptr;
+  const NodeDef* fused_batch_norm = nullptr;
+  float epsilon = 0.0;
+};
+
+// Conv2D node followed by a FusedBatchNorm and Relu.
+struct Conv2DWithBatchNormAndRelu {
+  const NodeDef* conv2d = nullptr;
+  const NodeDef* fused_batch_norm = nullptr;
+  const NodeDef* relu = nullptr;
+  float epsilon = 0.0;
 };
 
 bool IsFloatOrDoubleDataType(const NodeDef* node,
@@ -54,7 +89,7 @@ bool IsFloatOrDoubleDataType(const NodeDef* node,
 }
 
 bool HaveSameDataType(const NodeDef* lhs, const NodeDef* rhs,
-                      const string& type_attr) {
+                      const string& type_attr = "T") {
   DataType lhs_attr = GetDataTypeFromAttr(*lhs, type_attr);
   DataType rhs_attr = GetDataTypeFromAttr(*rhs, type_attr);
 
@@ -62,25 +97,36 @@ bool HaveSameDataType(const NodeDef* lhs, const NodeDef* rhs,
          lhs_attr == rhs_attr;
 }
 
-bool FindConv2DWithBias(const GraphView& graph_view, const NodeDef* node,
+bool HasDataType(const NodeDef* node, const DataType& expected,
+                 const string& type_attr = "T") {
+  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
+  return dtype == expected;
+}
+
+bool IsInPreserveSet(const RemapperContext& ctx, const NodeDef* node) {
+  return ctx.nodes_to_preserve.count(node->name()) > 0;
+}
+
+bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* node,
                         Conv2DWithBiasAdd* matched) {
   // Root of the pattern must be a BiasAdd.
   if (!node) return false;
   if (!IsBiasAdd(*node)) return false;
   if (!NodeIsOnCpu(node)) return false;
   if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(graph_view, node)) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
 
   // Input to the BiasAdd must be a Conv2D in NHWC format.
   const auto input_port = GraphView::InputPort(node, 0);
-  const auto conv2d = graph_view.GetRegularFanin(input_port);
+  const auto conv2d = ctx.graph_view.GetRegularFanin(input_port);
   if (!conv2d.node) return false;
   if (!IsConv2D(*conv2d.node)) return false;
-  if (conv2d.node->attr().at("data_format").s() != "NHWC") return false;
+  if (conv2d.node->attr().at(kDataFormat).s() != "NHWC") return false;
   if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node, "T")) return false;
-  if (!NoControlFaninOrFanout(graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(graph_view, conv2d.node)) return false;
+  if (!HaveSameDataType(node, conv2d.node)) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
+  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
+  if (IsInPreserveSet(ctx, conv2d.node)) return false;
 
   // We successfully found a Conv2D+BiasAdd pattern.
   matched->conv2d = conv2d.node;
@@ -89,23 +135,24 @@ bool FindConv2DWithBias(const GraphView& graph_view, const NodeDef* node,
   return true;
 }
 
-bool FindConv2DWithBiasAndRelu(const GraphView& graph_view, const NodeDef* node,
+bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* node,
                                Conv2DWithBiasAddAndRelu* matched) {
   // Root of the pattern must be a Relu.
   if (!node) return false;
   if (!IsRelu(*node)) return false;
   if (!NodeIsOnCpu(node)) return false;
   if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(graph_view, node)) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
 
   // And input to Relu must match Conv2DWithBiasAdd pattern.
   const auto input_port = GraphView::InputPort(node, 0);
-  const auto bias_add = graph_view.GetRegularFanin(input_port);
+  const auto bias_add = ctx.graph_view.GetRegularFanin(input_port);
 
   Conv2DWithBiasAdd base;
-  if (!FindConv2DWithBias(graph_view, bias_add.node, &base)) return false;
-  if (!HasSingleFanoutNode(graph_view, base.bias_add)) return false;
-  if (!HaveSameDataType(node, base.bias_add, "T")) return false;
+  if (!FindConv2DWithBias(ctx, bias_add.node, &base)) return false;
+  if (!HasSingleFanoutNode(ctx.graph_view, base.bias_add)) return false;
+  if (!HaveSameDataType(node, base.bias_add)) return false;
+  if (IsInPreserveSet(ctx, base.bias_add)) return false;
 
   // We successfully found a Conv2D+BiasAdd+Relu pattern.
   matched->conv2d = base.conv2d;
@@ -115,27 +162,97 @@ bool FindConv2DWithBiasAndRelu(const GraphView& graph_view, const NodeDef* node,
   return true;
 }
 
+bool FindConv2DWithBatchNorm(const RemapperContext& ctx, const NodeDef* node,
+                             Conv2DWithBatchNorm* matched) {
+  // Root of the pattern must be a FusedBatchNorm or a FusedBatchNormV2.
+  if (node == nullptr) return false;
+  if (!IsFusedBatchNorm(*node)) return false;
+  if (!NodeIsOnCpu(node)) return false;
+  if (!HasDataType(node, DT_FLOAT)) return false;
+
+  // V2 has a separate data type for the scale/offset/mean/variance inputs.
+  if (node->op() == "FusedBatchNormV2" && !HasDataType(node, DT_FLOAT, "U"))
+    return false;
+
+  // Check that batch normalization is in inference mode.
+  const auto& attr = node->attr();
+  if (attr.count(kIsTraining) > 0 && attr.at(kIsTraining).b()) return false;
+
+  // Check that only 0th output is consumed by other nodes.
+  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (HasFanouts(ctx.graph_view, node, 1)) return false;  // batch_mean
+  if (HasFanouts(ctx.graph_view, node, 2)) return false;  // batch_variance
+  if (HasFanouts(ctx.graph_view, node, 3)) return false;  // reserve_space_1
+  if (HasFanouts(ctx.graph_view, node, 4)) return false;  // reserve_space_2
+
+  // Input to the FusedBatchNorm must be a Conv2D in NHWC format.
+  const auto input_port = GraphView::InputPort(node, 0);
+  const auto conv2d = ctx.graph_view.GetRegularFanin(input_port);
+  if (conv2d.node == nullptr) return false;
+  if (!IsConv2D(*conv2d.node)) return false;
+  if (conv2d.node->attr().at(kDataFormat).s() != "NHWC") return false;
+  if (!NodeIsOnCpu(conv2d.node)) return false;
+  if (!HaveSameDataType(node, conv2d.node)) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
+  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
+  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  // We successfully found a Conv2D+FusedBatchNorm pattern.
+  matched->conv2d = conv2d.node;
+  matched->fused_batch_norm = node;
+  if (!GetNodeAttr(*node, "epsilon", &matched->epsilon).ok()) return false;
+
+  return true;
+}
+
+bool FindConv2DWithBatchNormAndRelu(const RemapperContext& ctx,
+                                    const NodeDef* node,
+                                    Conv2DWithBatchNormAndRelu* matched) {
+  // Root of the pattern must be a Relu.
+  if (node == nullptr) return false;
+  if (!IsRelu(*node)) return false;
+  if (!NodeIsOnCpu(node)) return false;
+  if (!IsFloatOrDoubleDataType(node)) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+
+  // And input to Relu must match Conv2DWithBatchNorm pattern.
+  const auto input_port = GraphView::InputPort(node, 0);
+  const auto batch_norm = ctx.graph_view.GetRegularFanin(input_port);
+
+  Conv2DWithBatchNorm base;
+  if (!FindConv2DWithBatchNorm(ctx, batch_norm.node, &base)) return false;
+  if (!HasSingleFanoutNode(ctx.graph_view, base.fused_batch_norm)) return false;
+  if (!HaveSameDataType(node, base.fused_batch_norm)) return false;
+  if (IsInPreserveSet(ctx, base.fused_batch_norm)) return false;
+
+  // We successfully found a Conv2D+FusedBatchNorm+Relu pattern.
+  matched->conv2d = base.conv2d;
+  matched->fused_batch_norm = base.fused_batch_norm;
+  matched->relu = node;
+  matched->epsilon = base.epsilon;
+
+  return true;
+}
+
 // Check that given node meets some basic FusedBatchNorm optimization
 // preconditions. We use this check to lazily infer graph properties which is
 // rather expensive.
-bool IsFusedBatchNormCandidate(const GraphView& graph_view,
-                               const NodeDef& node) {
+bool IsFusedBatchNormCandidate(const NodeDef& node) {
   if (!IsFusedBatchNorm(node)) return false;
   if (GetDataTypeFromAttr(node, "T") != DT_FLOAT) return false;
 
   // Check that the node is in inference mode.
   const auto& attr = node.attr();
-  if (attr.count("is_training") > 0 && attr.at("is_training").b()) return false;
+  if (attr.count(kIsTraining) > 0 && attr.at(kIsTraining).b()) return false;
 
   return true;
 }
 
-bool FindFusedBatchNorm(const GraphView& graph_view,
-                        const GraphProperties& graph_properties,
-                        const NodeDef* node, FusedBatchNorm* matched) {
-  if (!IsFusedBatchNormCandidate(graph_view, *node)) return false;
+bool FindFusedBatchNorm(const RemapperContext& ctx, const NodeDef* node,
+                        FusedBatchNorm* matched) {
+  if (!IsFusedBatchNormCandidate(*node)) return false;
 
-  const auto& props = graph_properties.GetInputProperties(node->name());
+  const auto& props = ctx.graph_properties.GetInputProperties(node->name());
 
   // a. Scaling factor can be const folded:
   //      scaling_factor = (variance + epsilon).rsqrt() * scale
@@ -155,7 +272,7 @@ bool FindFusedBatchNorm(const GraphView& graph_view,
   if (!can_remap) return false;
 
   // The optimized version only generates the first output.
-  for (GraphView::Edge edge : graph_view.GetFanoutEdges(*node, false)) {
+  for (GraphView::Edge edge : ctx.graph_view.GetFanoutEdges(*node, false)) {
     if (edge.src.port_id != 0) return false;
   }
 
@@ -188,37 +305,98 @@ void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
   SetAttrValue(epsilon, &(*attr)["epsilon"]);
 }
 
-void AddFusedConv2DNode(const Conv2DWithBiasAdd& matched,
-                        GraphDef* optimized_graph) {
+void AddFusedConv2DNode(
+    const Conv2DWithBiasAdd& matched, GraphDef* optimized_graph,
+    absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
   VLOG(2) << "Fuse Conv2D with BiasAdd: bias_add=" << matched.bias_add->name()
           << " conv2d=" << matched.conv2d->name();
 
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.bias_add->name());
-  fused_conv2d->set_op("_FusedConv2D");
+  fused_conv2d->set_op(kFusedConv2D);
   fused_conv2d->set_device(matched.bias_add->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
   CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd"});
+
+  invalidated_nodes->insert(matched.bias_add);
+  invalidated_nodes->insert(matched.conv2d);
 }
 
-void AddFusedConv2DNode(const Conv2DWithBiasAddAndRelu& matched,
-                        GraphDef* optimized_graph) {
+void AddFusedConv2DNode(
+    const Conv2DWithBiasAddAndRelu& matched, GraphDef* optimized_graph,
+    absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
   VLOG(2) << "Fuse Conv2D with BiasAdd and Relu: relu=" << matched.relu->name()
           << " bias_add=" << matched.bias_add->name()
           << " conv2d=" << matched.conv2d->name();
 
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.relu->name());
-  fused_conv2d->set_op("_FusedConv2D");
+  fused_conv2d->set_op(kFusedConv2D);
   fused_conv2d->set_device(matched.relu->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
   CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd", "Relu"});
+
+  invalidated_nodes->insert(matched.relu);
+  invalidated_nodes->insert(matched.bias_add);
+  invalidated_nodes->insert(matched.conv2d);
+}
+
+void AddFusedConv2DNode(
+    const Conv2DWithBatchNorm& matched, GraphDef* optimized_graph,
+    absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
+  VLOG(2) << "Fuse Conv2D with BatchNorm: batch_norm="
+          << matched.fused_batch_norm->name()
+          << " conv2d=" << matched.conv2d->name();
+
+  NodeDef* fused_conv2d = optimized_graph->add_node();
+  fused_conv2d->set_name(matched.fused_batch_norm->name());
+  fused_conv2d->set_op(kFusedConv2D);
+  fused_conv2d->set_device(matched.fused_batch_norm->device());
+  fused_conv2d->add_input(matched.conv2d->input(0));            // 0: input
+  fused_conv2d->add_input(matched.conv2d->input(1));            // 1: filter
+  fused_conv2d->add_input(matched.fused_batch_norm->input(1));  // 2: scale
+  fused_conv2d->add_input(matched.fused_batch_norm->input(2));  // 3: offset
+  fused_conv2d->add_input(matched.fused_batch_norm->input(3));  // 4: mean
+  fused_conv2d->add_input(matched.fused_batch_norm->input(4));  // 5: variance
+
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"FusedBatchNorm"},
+                       /*num_args*/ 4, /*epsilon*/ matched.epsilon);
+
+  invalidated_nodes->insert(matched.fused_batch_norm);
+  invalidated_nodes->insert(matched.conv2d);
+}
+
+void AddFusedConv2DNode(
+    const Conv2DWithBatchNormAndRelu& matched, GraphDef* optimized_graph,
+    absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
+  VLOG(2) << "Fuse Conv2D with BatchNorm and Relu: relu="
+          << matched.relu->name()
+          << " batch_norm=" << matched.fused_batch_norm->name()
+          << " conv2d=" << matched.conv2d->name();
+
+  NodeDef* fused_conv2d = optimized_graph->add_node();
+  fused_conv2d->set_name(matched.relu->name());
+  fused_conv2d->set_op(kFusedConv2D);
+  fused_conv2d->set_device(matched.fused_batch_norm->device());
+  fused_conv2d->add_input(matched.conv2d->input(0));            // 0: input
+  fused_conv2d->add_input(matched.conv2d->input(1));            // 1: filter
+  fused_conv2d->add_input(matched.fused_batch_norm->input(1));  // 2: scale
+  fused_conv2d->add_input(matched.fused_batch_norm->input(2));  // 3: offset
+  fused_conv2d->add_input(matched.fused_batch_norm->input(3));  // 4: mean
+  fused_conv2d->add_input(matched.fused_batch_norm->input(4));  // 5: variance
+
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"FusedBatchNorm", "Relu"},
+                       /*num_args*/ 4, /*epsilon*/ matched.epsilon);
+
+  invalidated_nodes->insert(matched.relu);
+  invalidated_nodes->insert(matched.fused_batch_norm);
+  invalidated_nodes->insert(matched.conv2d);
 }
 
 void AddBatchNormNodes(const FusedBatchNorm& matched,
@@ -233,7 +411,7 @@ void AddBatchNormNodes(const FusedBatchNorm& matched,
   string mean = fused_node.input(3);
   string variance = fused_node.input(4);
 
-  if (fused_node.attr().at("data_format").s() == "NCHW") {
+  if (fused_node.attr().at(kDataFormat).s() == "NCHW") {
     // Need to reshape the last 4 inputs
     NodeDef* new_shape = optimized_graph->add_node();
     new_shape->set_name(AddPrefixToNodeName("NCHWShape", fused_node.name()));
@@ -367,38 +545,70 @@ void AddBatchNormNodes(const FusedBatchNorm& matched,
 
 Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
                           GraphDef* optimized_graph) {
-  GraphProperties properties(item);
-  bool inferred_properties = false;
-  GraphView graph(const_cast<GraphDef*>(&item.graph));
-
   // Supported graph patterns.
-  FusedBatchNorm fused_batch_norm{};
-  Conv2DWithBiasAdd conv2d_with_bias{};
-  Conv2DWithBiasAddAndRelu conv2d_with_bias_and_relu{};
+  // clang-format off
+  FusedBatchNorm              fused_batch_norm;
+  Conv2DWithBiasAdd           conv2d_with_bias;
+  Conv2DWithBiasAddAndRelu    conv2d_with_bias_and_relu;
+  Conv2DWithBatchNorm         conv2d_with_batch_norm;
+  Conv2DWithBatchNormAndRelu  conv2d_with_batch_norm_and_relu;
+  // clang-format on
+
+  // Processing graph in reverse-topological sorted order allows to remap
+  // longer chains of dependent ops in one pass.
+  GraphDef topo_sorted_graph = item.graph;
+  TF_RETURN_IF_ERROR(TopologicalSort(&topo_sorted_graph));
+  std::reverse(topo_sorted_graph.mutable_node()->begin(),
+               topo_sorted_graph.mutable_node()->end());
+
+  RemapperContext ctx(item);
+
+  // Skip nodes that were invalidated by a remapper, e.g. do not process BiasAdd
+  // and Relu nodes that were fused into a Conv2D node.
+  absl::flat_hash_set<const NodeDef*> invalidated_nodes;
 
   optimized_graph->mutable_node()->Reserve(item.graph.node_size());
   for (const NodeDef& node : item.graph.node()) {
-    // Remap Conv2D+BiasAdd into the _FusedConv2DWithBias.
-    if (FindConv2DWithBias(graph, &node, &conv2d_with_bias)) {
-      AddFusedConv2DNode(conv2d_with_bias, optimized_graph);
+    // Check if node was invalidated by one of the previous remaps.
+    if (invalidated_nodes.count(&node) > 0) continue;
+
+    // Remap Conv2D+BiasAdd into the _FusedConv2D.
+    if (FindConv2DWithBias(ctx, &node, &conv2d_with_bias)) {
+      AddFusedConv2DNode(conv2d_with_bias, optimized_graph, &invalidated_nodes);
+      continue;
+    }
+
+    // Remap Conv2D+BiasAdd+Relu into the _FusedConv2D.
+    if (FindConv2DWithBiasAndRelu(ctx, &node, &conv2d_with_bias_and_relu)) {
+      AddFusedConv2DNode(conv2d_with_bias_and_relu, optimized_graph,
+                         &invalidated_nodes);
+      continue;
+    }
+
+    // Remap Conv2D+FusedBatchNorm into the _FusedConv2D;
+    if (FindConv2DWithBatchNorm(ctx, &node, &conv2d_with_batch_norm)) {
+      AddFusedConv2DNode(conv2d_with_batch_norm, optimized_graph,
+                         &invalidated_nodes);
       continue;
     }
 
-    // Remap Conv2D+BiasAdd+Relu into the _FusedConv2DWithBias(Relu).
-    if (FindConv2DWithBiasAndRelu(graph, &node, &conv2d_with_bias_and_relu)) {
-      AddFusedConv2DNode(conv2d_with_bias_and_relu, optimized_graph);
+    // Remap Conv2D+FusedBatchNorm+Relu into the _FusedConv2D;
+    if (FindConv2DWithBatchNormAndRelu(ctx, &node,
+                                       &conv2d_with_batch_norm_and_relu)) {
+      AddFusedConv2DNode(conv2d_with_batch_norm_and_relu, optimized_graph,
+                         &invalidated_nodes);
       continue;
     }
 
     // Infer properties lazily in case they are not needed.
-    if (!inferred_properties && IsFusedBatchNormCandidate(graph, node)) {
-      TF_RETURN_IF_ERROR(properties.InferStatically(false));
-      inferred_properties = true;
+    if (!ctx.inferred_graph_properties && IsFusedBatchNormCandidate(node)) {
+      TF_RETURN_IF_ERROR(ctx.graph_properties.InferStatically(false));
+      ctx.inferred_graph_properties = true;
     }
 
     // During inference, most of the inputs to FusedBatchNorm are constant, and
     // we can therefore replace the op with a much cheaper set of primitives.
-    if (FindFusedBatchNorm(graph, properties, &node, &fused_batch_norm)) {
+    if (FindFusedBatchNorm(ctx, &node, &fused_batch_norm)) {
       AddBatchNormNodes(fused_batch_norm, optimized_graph);
       continue;
     }
diff --git a/tensorflow/core/grappler/optimizers/remapper_test.cc b/tensorflow/core/grappler/optimizers/remapper_test.cc
index 249ca70673..1536c91ef7 100644
--- a/tensorflow/core/grappler/optimizers/remapper_test.cc
+++ b/tensorflow/core/grappler/optimizers/remapper_test.cc
@@ -215,5 +215,157 @@ TEST_F(RemapperTest, FuseConv2DWithBiasAndRelu) {
   test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
+TEST_F(RemapperTest, FuseConv2DWithBatchNorm) {
+  using ops::Placeholder;
+
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  auto input_shape = ops::Placeholder::Shape({8, 32, 32, 3});
+  auto filter_shape = ops::Placeholder::Shape({1, 1, 3, 128});
+  auto scale_shape = ops::Placeholder::Shape({128});
+
+  auto input = Placeholder(s.WithOpName("input"), DT_FLOAT, input_shape);
+  auto filter = Placeholder(s.WithOpName("filter"), DT_FLOAT, filter_shape);
+  auto scale = Placeholder(s.WithOpName("scale"), DT_FLOAT, scale_shape);
+  auto offset = Placeholder(s.WithOpName("offset"), DT_FLOAT, scale_shape);
+  auto mean = Placeholder(s.WithOpName("mean"), DT_FLOAT, scale_shape);
+  auto variance = Placeholder(s.WithOpName("variance"), DT_FLOAT, scale_shape);
+
+  std::vector<int> strides = {1, 1, 1, 1};
+  auto conv = ops::Conv2D(s.WithOpName("conv"), input, filter, strides, "SAME");
+  ops::FusedBatchNorm::Attrs attrs;
+  attrs = attrs.IsTraining(false);
+  auto batch_norm = ops::FusedBatchNorm(s.WithOpName("batch_norm"), conv, scale,
+                                        offset, mean, variance, attrs);
+  auto fetch = ops::Identity(s.WithOpName("fetch"), batch_norm.y);
+
+  auto input_t = GenerateRandomTensor<DT_FLOAT>({8, 32, 32, 3});
+  auto filter_t = GenerateRandomTensor<DT_FLOAT>({1, 1, 3, 128});
+  auto scale_t = GenerateRandomTensor<DT_FLOAT>({128});
+  auto offset_t = GenerateRandomTensor<DT_FLOAT>({128});
+  auto mean_t = GenerateRandomTensor<DT_FLOAT>({128});
+  auto variance_t = GenerateRandomTensor<DT_FLOAT>({128});
+
+  GrapplerItem item;
+  item.fetch = {"fetch"};
+  item.feed = {{"input", input_t}, {"filter", filter_t},
+               {"scale", scale_t}, {"offset", offset_t},
+               {"mean", mean_t},   {"variance", variance_t}};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  // Place all nodes on CPU.
+  for (int i = 0; i < item.graph.node_size(); ++i) {
+    item.graph.mutable_node(i)->set_device("/device:CPU:0");
+  }
+
+  Remapper optimizer(RewriterConfig::ON);
+  GraphDef output;
+  TF_CHECK_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int found = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "batch_norm") {
+      EXPECT_EQ("_FusedConv2D", node.op());
+      EXPECT_EQ("input", node.input(0));
+      EXPECT_EQ("filter", node.input(1));
+
+      EXPECT_EQ(4, node.attr().at("num_args").i());
+      EXPECT_EQ("scale", node.input(2));
+      EXPECT_EQ("offset", node.input(3));
+      EXPECT_EQ("mean", node.input(4));
+      EXPECT_EQ("variance", node.input(5));
+
+      const auto fused_ops = node.attr().at("fused_ops").list().s();
+      EXPECT_EQ(1, fused_ops.size());
+      EXPECT_EQ("FusedBatchNorm", fused_ops[0]);
+      found++;
+    }
+  }
+  EXPECT_EQ(1, found);
+
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed);
+  auto tensors = EvaluateNodes(output, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors_expected.size());
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
+}
+
+TEST_F(RemapperTest, FuseConv2DWithBatchNormAndRelu) {
+  using ops::Placeholder;
+
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  auto input_shape = ops::Placeholder::Shape({8, 32, 32, 3});
+  auto filter_shape = ops::Placeholder::Shape({1, 1, 3, 128});
+  auto scale_shape = ops::Placeholder::Shape({128});
+
+  auto input = Placeholder(s.WithOpName("input"), DT_FLOAT, input_shape);
+  auto filter = Placeholder(s.WithOpName("filter"), DT_FLOAT, filter_shape);
+  auto scale = Placeholder(s.WithOpName("scale"), DT_FLOAT, scale_shape);
+  auto offset = Placeholder(s.WithOpName("offset"), DT_FLOAT, scale_shape);
+  auto mean = Placeholder(s.WithOpName("mean"), DT_FLOAT, scale_shape);
+  auto variance = Placeholder(s.WithOpName("variance"), DT_FLOAT, scale_shape);
+
+  std::vector<int> strides = {1, 1, 1, 1};
+  auto conv = ops::Conv2D(s.WithOpName("conv"), input, filter, strides, "SAME");
+  ops::FusedBatchNorm::Attrs attrs;
+  attrs = attrs.IsTraining(false);
+  auto batch_norm = ops::FusedBatchNorm(s.WithOpName("batch_norm"), conv, scale,
+                                        offset, mean, variance, attrs);
+  auto relu = ops::Relu(s.WithOpName("relu"), batch_norm.y);
+  auto fetch = ops::Identity(s.WithOpName("fetch"), relu);
+
+  auto input_t = GenerateRandomTensor<DT_FLOAT>({8, 32, 32, 3});
+  auto filter_t = GenerateRandomTensor<DT_FLOAT>({1, 1, 3, 128});
+  auto scale_t = GenerateRandomTensor<DT_FLOAT>({128});
+  auto offset_t = GenerateRandomTensor<DT_FLOAT>({128});
+  auto mean_t = GenerateRandomTensor<DT_FLOAT>({128});
+  auto variance_t = GenerateRandomTensor<DT_FLOAT>({128});
+
+  GrapplerItem item;
+  item.fetch = {"fetch"};
+  item.feed = {{"input", input_t}, {"filter", filter_t},
+               {"scale", scale_t}, {"offset", offset_t},
+               {"mean", mean_t},   {"variance", variance_t}};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  // Place all nodes on CPU.
+  for (int i = 0; i < item.graph.node_size(); ++i) {
+    item.graph.mutable_node(i)->set_device("/device:CPU:0");
+  }
+
+  Remapper optimizer(RewriterConfig::ON);
+  GraphDef output;
+  TF_CHECK_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int found = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "relu") {
+      EXPECT_EQ("_FusedConv2D", node.op());
+      EXPECT_EQ("input", node.input(0));
+      EXPECT_EQ("filter", node.input(1));
+
+      EXPECT_EQ(4, node.attr().at("num_args").i());
+      EXPECT_EQ("scale", node.input(2));
+      EXPECT_EQ("offset", node.input(3));
+      EXPECT_EQ("mean", node.input(4));
+      EXPECT_EQ("variance", node.input(5));
+
+      const auto fused_ops = node.attr().at("fused_ops").list().s();
+      EXPECT_EQ(2, fused_ops.size());
+      EXPECT_EQ("FusedBatchNorm", fused_ops[0]);
+      EXPECT_EQ("Relu", fused_ops[1]);
+      found++;
+    }
+  }
+  EXPECT_EQ(1, found);
+
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed);
+  auto tensors = EvaluateNodes(output, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors_expected.size());
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
+}
+
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 406591972c09fb52ee13ce6d3bcc8dc6fd118d35 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Thu, 15 Nov 2018 10:37:02 -0800
Subject: [PATCH 0305/1554] Make the API of tf.nn.fractional_*_pool ops
 consistent with other random ops.

Before this change, fractional_avg_pool and fractional_max_pool would take 3
randomness-related args: `seed`, `seed2`, and `deterministic`.  The intended
behavior was to get a deterministic execution if `deterministic` was true using
`seed` and `seed2`.

After this change, these ops take a single `seed` arg.  If `seed` is zero, the
execution is random.  Otherwise, we use the graph-level random ops to generate
seed and seed2 from `seed`, and pass that to the kernel with deterministic set
to True.

PiperOrigin-RevId: 221648311
---
 .../api_def_FractionalAvgPool.pbtxt           |   4 +-
 .../api_def_FractionalMaxPool.pbtxt           |   4 +-
 .../fractional_avg_pool_op_test.py            |  70 +----
 .../fractional_max_pool_op_test.py            |  38 +--
 tensorflow/python/ops/nn_ops.py               | 287 ++++++++++++++++++
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   4 +-
 6 files changed, 318 insertions(+), 89 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_FractionalAvgPool.pbtxt b/tensorflow/core/api_def/python_api/api_def_FractionalAvgPool.pbtxt
index 16ed9b56f2..cbe87777a7 100644
--- a/tensorflow/core/api_def/python_api/api_def_FractionalAvgPool.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_FractionalAvgPool.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "FractionalAvgPool"
-  endpoint {
-    name: "nn.fractional_avg_pool"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_FractionalMaxPool.pbtxt b/tensorflow/core/api_def/python_api/api_def_FractionalMaxPool.pbtxt
index 6955595208..02470b4345 100644
--- a/tensorflow/core/api_def/python_api/api_def_FractionalMaxPool.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_FractionalMaxPool.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "FractionalMaxPool"
-  endpoint {
-    name: "nn.fractional_max_pool"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
index f89d2062f1..114240fd85 100644
--- a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
@@ -37,7 +37,6 @@ class FractionalAvgTest(test.TestCase):
   # Random number generate with seed.
   _PRNG = np.random.RandomState(341261000)
   _SEED = 341261001
-  _SEED2 = 341261002
 
   def _AvgPoolAlongRows(self, input_matrix, row_seq, overlapping):
     """Perform average pool along row of a 2-D matrix based on row_seq.
@@ -128,14 +127,12 @@ class FractionalAvgTest(test.TestCase):
       None
     """
     with self.cached_session() as sess:
-      p, r, c = nn_ops.fractional_avg_pool(
+      p, r, c = nn_ops.fractional_avg_pool_v2(
           input_tensor,
           pooling_ratio,
           pseudo_random,
           overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
+          seed=self._SEED)
       actual, row_seq, col_seq = sess.run([p, r, c])
       expected = self._GetExpectedFractionalAvgPoolResult(input_tensor, row_seq,
                                                           col_seq, overlapping)
@@ -161,14 +158,12 @@ class FractionalAvgTest(test.TestCase):
       rand_mat = self._PRNG.randint(10, size=tensor_shape)
       pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1]
       with self.cached_session() as sess:
-        p, r, c = nn_ops.fractional_avg_pool(
+        p, r, c = nn_ops.fractional_avg_pool_v2(
             rand_mat.astype(np.float32),
             pooling_ratio,
             pseudo_random,
             overlapping,
-            deterministic=True,
-            seed=self._SEED,
-            seed2=self._SEED2)
+            seed=self._SEED)
         tensor_output, row_seq, col_seq = sess.run([p, r, c])
         expected_result = self._GetExpectedFractionalAvgPoolResult(
             rand_mat.astype(np.float32), row_seq, col_seq, overlapping)
@@ -214,12 +209,6 @@ class FractionalAvgTest(test.TestCase):
 
   def testIntegerTensorInput(self):
     """Test FractionalAvgPool works fine when input tensor is integer type.
-
-    I would have used _ValidateFractionalAvgPoolResult function to automate this
-    process, however, there's rounding issue. It is caused by numpy.mean cast
-    integer input to numpy.float64 for intermediate use. While for
-    fractional_avg_pool, the mean operation is integer division (trucated).  So,
-    for this test case, I will hard code a simple matrix.
     """
     pseudo_random = True
     overlapping = True
@@ -234,29 +223,9 @@ class FractionalAvgTest(test.TestCase):
         [4, 4, 5, 9, 7, 2]
     ])
     # pyformat: enable
-    with self.cached_session() as sess:
-      # Since deterministic = True, seed and seed2 are fixed. Therefore r, and c
-      # are the same each time. We can have an expected result precomputed.
-      # r = [0, 2, 4, 6]
-      # c = [0, 1, 3, 4, 6]
-
-      # pyformat: disable
-      expected = np.array([
-          [6, 5, 3, 5],
-          [5, 5, 4, 5],
-          [5, 4, 7, 5]
-      ]).reshape((1, 3, 4, 1))
-      # pyformat: enable
-      p, unused_r, unused_c = nn_ops.fractional_avg_pool(
-          mat.reshape(tensor_shape), [1, math.sqrt(3), math.sqrt(2), 1],
-          pseudo_random,
-          overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
-      actual = sess.run(p)
-      self.assertShapeEqual(expected, p)
-      self.assertAllClose(expected, actual)
+    self._ValidateFractionalAvgPoolResult(mat.reshape(tensor_shape),
+                                          [1, math.sqrt(3), math.sqrt(2), 1],
+                                          pseudo_random, overlapping)
 
   def testDifferentTensorShapes(self):
     """Test different shapes of input tensor.
@@ -320,14 +289,12 @@ class FractionalAvgTest(test.TestCase):
       pooling_ratio = [1, 1.5, 1.5, 1]
       pseudo_random = False
       overlapping = False
-      p, r, c = nn_ops.fractional_avg_pool(
+      p, r, c = nn_ops.fractional_avg_pool_v2(
           input_holder,
           pooling_ratio,
           pseudo_random,
           overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
+          seed=self._SEED)
       # First run.
       input_a = np.zeros([3, 32, 32, 3])
       actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_a})
@@ -372,7 +339,6 @@ class FractionalAvgPoolGradTest(test.TestCase):
   """
   _PRNG = np.random.RandomState(341261004)
   _SEED = 341261005
-  _SEED2 = 341261006
 
   def _GenerateRandomInputTensor(self, shape):
     num_elements = 1
@@ -470,14 +436,12 @@ class FractionalAvgPoolGradTest(test.TestCase):
       for overlapping in True, False:
         with self.cached_session() as _:
           input_tensor = constant_op.constant(input_data, shape=input_shape)
-          output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool(
+          output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2(
               input_tensor,
               pooling_ratio,
               pseudo_random=pseudo_random,
               overlapping=overlapping,
-              deterministic=True,
-              seed=self._SEED,
-              seed2=self._SEED2)
+              seed=self._SEED)
           output_data = output_tensor.eval()
           output_shape = output_data.shape
           # error_margin and delta setting is similar to avg_pool_grad.
@@ -503,14 +467,12 @@ class FractionalAvgPoolGradTest(test.TestCase):
             input_data = self._GenerateRandomInputTensor(input_shape)
             with self.cached_session() as _:
               input_tensor = constant_op.constant(input_data, shape=input_shape)
-              output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool(
+              output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2(
                   input_tensor,
                   pooling_ratio,
                   pseudo_random=pseudo_random,
                   overlapping=overlapping,
-                  deterministic=True,
-                  seed=self._SEED,
-                  seed2=self._SEED2)
+                  seed=self._SEED)
               output_data = output_tensor.eval()
               output_shape = output_data.shape
               # error_margin and delta setting is similar to avg_pool_grad.
@@ -534,14 +496,12 @@ class FractionalAvgPoolGradTest(test.TestCase):
 
     with self.cached_session() as _:
       input_tensor = constant_op.constant(input_data, shape=input_shape)
-      output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool(
+      output_tensor, unused_a, unused_b = nn_ops.fractional_avg_pool_v2(
           input_tensor,
           pooling_ratio,
           pseudo_random=pseudo_random,
           overlapping=overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
+          seed=self._SEED)
       # error_margin and delta setting is similar to avg_pool_grad.
       error_margin = 1e-4
       gradient_error = gradient_checker.compute_gradient_error(
diff --git a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
index 9b94ca8554..1b536ba28a 100644
--- a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
@@ -37,7 +37,6 @@ class FractionalMaxPoolTest(test.TestCase):
   # Random number generate with seed.
   _PRNG = np.random.RandomState(341261)
   _SEED = 123456
-  _SEED2 = 654321
 
   def _MaxPoolAlongRows(self, input_matrix, row_seq, overlapping):
     """Perform max pool along row of a 2-D matrix based on row_seq.
@@ -128,14 +127,12 @@ class FractionalMaxPoolTest(test.TestCase):
       None
     """
     with self.cached_session() as sess:
-      p, r, c = nn_ops.fractional_max_pool(
+      p, r, c = nn_ops.fractional_max_pool_v2(
           input_tensor,
           pooling_ratio,
           pseudo_random,
           overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
+          seed=self._SEED)
       actual, row_seq, col_seq = sess.run([p, r, c])
       expected = self._GetExpectedFractionalMaxPoolResult(input_tensor, row_seq,
                                                           col_seq, overlapping)
@@ -161,14 +158,12 @@ class FractionalMaxPoolTest(test.TestCase):
       rand_mat = self._PRNG.randint(10, size=tensor_shape)
       pooling_ratio = [1, math.sqrt(2), math.sqrt(2), 1]
       with self.cached_session() as sess:
-        p, r, c = nn_ops.fractional_max_pool(
+        p, r, c = nn_ops.fractional_max_pool_v2(
             rand_mat,
             pooling_ratio,
             pseudo_random,
             overlapping,
-            deterministic=True,
-            seed=self._SEED,
-            seed2=self._SEED2)
+            seed=self._SEED)
         tensor_output, row_seq, col_seq = sess.run([p, r, c])
         expected_result = self._GetExpectedFractionalMaxPoolResult(rand_mat,
                                                                    row_seq,
@@ -291,14 +286,12 @@ class FractionalMaxPoolTest(test.TestCase):
       pooling_ratio = [1, 1.5, 1.5, 1]
       pseudo_random = False
       overlapping = False
-      p, r, c = nn_ops.fractional_max_pool(
+      p, r, c = nn_ops.fractional_max_pool_v2(
           input_holder,
           pooling_ratio,
           pseudo_random,
           overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
+          seed=self._SEED)
       # First run.
       input_a = np.zeros([3, 32, 32, 3])
       actual, row_seq, col_seq = sess.run([p, r, c], {input_holder: input_a})
@@ -344,7 +337,6 @@ class FractionalMaxPoolGradTest(test.TestCase):
 
   _PRNG = np.random.RandomState(341261)
   _SEED = 123456
-  _SEED2 = 654321
 
   def _GenerateUniqueRandomInputTensor(self, shape):
     """Generate 'unqiue' random input tensor.
@@ -449,14 +441,12 @@ class FractionalMaxPoolGradTest(test.TestCase):
       for overlapping in True, False:
         with self.cached_session() as _:
           input_tensor = constant_op.constant(input_data, shape=input_shape)
-          output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool(
+          output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2(
               input_tensor,
               pooling_ratio,
               pseudo_random=pseudo_random,
               overlapping=overlapping,
-              deterministic=True,
-              seed=self._SEED,
-              seed2=self._SEED2)
+              seed=self._SEED)
           output_data = output_tensor.eval()
           output_shape = output_data.shape
           # error_margin and delta setting is similar to max_pool_grad.
@@ -484,14 +474,12 @@ class FractionalMaxPoolGradTest(test.TestCase):
             input_data += self._PRNG.random_sample(input_shape)
             with self.cached_session() as _:
               input_tensor = constant_op.constant(input_data, shape=input_shape)
-              output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool(
+              output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2(
                   input_tensor,
                   pooling_ratio,
                   pseudo_random=pseudo_random,
                   overlapping=overlapping,
-                  deterministic=True,
-                  seed=self._SEED,
-                  seed2=self._SEED2)
+                  seed=self._SEED)
               output_data = output_tensor.eval()
               output_shape = output_data.shape
               # error_margin and delta setting is similar to max_pool_grad.
@@ -517,14 +505,12 @@ class FractionalMaxPoolGradTest(test.TestCase):
 
     with self.cached_session() as _:
       input_tensor = constant_op.constant(input_data, shape=input_shape)
-      output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool(
+      output_tensor, unused_a, unused_b = nn_ops.fractional_max_pool_v2(
           input_tensor,
           pooling_ratio,
           pseudo_random=pseudo_random,
           overlapping=overlapping,
-          deterministic=True,
-          seed=self._SEED,
-          seed2=self._SEED2)
+          seed=self._SEED)
       # error_margin and delta setting is similar to max_pool_grad.
       error_margin = 1e-3
       gradient_error = gradient_checker.compute_gradient_error(
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 4c69201b49..651bd32e5f 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import graph_util
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -2501,6 +2502,292 @@ def nth_element(input, n, reverse=False, name=None):  # pylint: disable=redefine
   return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name)
 
 
+@tf_export(v1=["nn.fractional_max_pool"])
+@deprecation.deprecated(date=None, instructions="`seed2` and `deterministic` "
+                        "args are deprecated.  Use fractional_max_pool_v2.")
+def fractional_max_pool(value,
+                        pooling_ratio,
+                        pseudo_random=False,
+                        overlapping=False,
+                        deterministic=False,
+                        seed=0,
+                        seed2=0,
+                        name=None):   # pylint: disable=redefined-builtin
+  r"""Performs fractional max pooling on the input.
+
+  This is a deprecated version of `fractional_max_pool`.
+
+  Fractional max pooling is slightly different than regular max pooling.  In
+  regular max pooling, you downsize an input set by taking the maximum value of
+  smaller N x N subsections of the set (often 2x2), and try to reduce the set by
+  a factor of N, where N is an integer.  Fractional max pooling, as you might
+  expect from the word "fractional", means that the overall reduction ratio N
+  does not have to be an integer.
+
+  The sizes of the pooling regions are generated randomly but are fairly
+  uniform.  For example, let's look at the height dimension, and the constraints
+  on the list of rows that will be pool boundaries.
+
+  First we define the following:
+
+  1.  input_row_length : the number of rows from the input set
+  2.  output_row_length : which will be smaller than the input
+  3.  alpha = input_row_length / output_row_length : our reduction ratio
+  4.  K = floor(alpha)
+  5.  row_pooling_sequence : this is the result list of pool boundary rows
+
+  Then, row_pooling_sequence should satisfy:
+
+  1.  a[0] = 0 : the first value of the sequence is 0
+  2.  a[end] = input_row_length : the last value of the sequence is the size
+  3.  K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size
+  4.  length(row_pooling_sequence) = output_row_length+1
+
+  For more details on fractional max pooling, see this paper: [Benjamin Graham,
+  Fractional Max-Pooling](http://arxiv.org/abs/1412.6071)
+
+  Args:
+    value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`.
+    pooling_ratio: A list of `floats` that has length >= 4.  Pooling ratio for
+      each dimension of `value`, currently only supports row and col dimension
+      and should be >= 1.0. For example, a valid pooling ratio looks like [1.0,
+      1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't
+      allow pooling on batch and channels dimensions.  1.44 and 1.73 are pooling
+      ratio on height and width dimensions respectively.
+    pseudo_random: An optional `bool`.  Defaults to `False`. When set to `True`,
+      generates the pooling sequence in a pseudorandom fashion, otherwise, in a
+      random fashion. Check paper [Benjamin Graham, Fractional
+      Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between
+      pseudorandom and random.
+    overlapping: An optional `bool`.  Defaults to `False`.  When set to `True`,
+      it means when pooling, the values at the boundary of adjacent pooling
+      cells are used by both cells. For example:
+      `index  0  1  2  3  4`
+      `value  20 5  16 3  7`
+      If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used
+      twice.  The result would be [20, 16] for fractional max pooling.
+    deterministic: An optional `bool`.  Deprecated; use `fractional_max_pool_v2`
+      instead.
+    seed: An optional `int`.  Defaults to `0`.  If set to be non-zero, the
+      random number generator is seeded by the given seed.  Otherwise it is
+      seeded by a random seed.
+    seed2: An optional `int`.  Deprecated; use `fractional_max_pool_v2` instead.
+    name: A name for the operation (optional).
+
+  Returns:
+  A tuple of `Tensor` objects (`output`, `row_pooling_sequence`,
+  `col_pooling_sequence`).
+    output: Output `Tensor` after fractional max pooling.  Has the same type as
+      `value`.
+    row_pooling_sequence: A `Tensor` of type `int64`.
+    col_pooling_sequence: A `Tensor` of type `int64`.
+  """
+  return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random,
+                                        overlapping, deterministic, seed, seed2,
+                                        name)
+
+
+@tf_export("nn.fractional_max_pool", v1=[])
+def fractional_max_pool_v2(value,
+                           pooling_ratio,
+                           pseudo_random=False,
+                           overlapping=False,
+                           seed=0,
+                           name=None):  # pylint: disable=redefined-builtin
+  r"""Performs fractional max pooling on the input.
+
+  Fractional max pooling is slightly different than regular max pooling.  In
+  regular max pooling, you downsize an input set by taking the maximum value of
+  smaller N x N subsections of the set (often 2x2), and try to reduce the set by
+  a factor of N, where N is an integer.  Fractional max pooling, as you might
+  expect from the word "fractional", means that the overall reduction ratio N
+  does not have to be an integer.
+
+  The sizes of the pooling regions are generated randomly but are fairly
+  uniform.  For example, let's look at the height dimension, and the constraints
+  on the list of rows that will be pool boundaries.
+
+  First we define the following:
+
+  1.  input_row_length : the number of rows from the input set
+  2.  output_row_length : which will be smaller than the input
+  3.  alpha = input_row_length / output_row_length : our reduction ratio
+  4.  K = floor(alpha)
+  5.  row_pooling_sequence : this is the result list of pool boundary rows
+
+  Then, row_pooling_sequence should satisfy:
+
+  1.  a[0] = 0 : the first value of the sequence is 0
+  2.  a[end] = input_row_length : the last value of the sequence is the size
+  3.  K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size
+  4.  length(row_pooling_sequence) = output_row_length+1
+
+  For more details on fractional max pooling, see this paper: [Benjamin Graham,
+  Fractional Max-Pooling](http://arxiv.org/abs/1412.6071)
+
+  Args:
+    value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`.
+    pooling_ratio: A list of `floats` that has length >= 4.  Pooling ratio for
+      each dimension of `value`, currently only supports row and col dimension
+      and should be >= 1.0. For example, a valid pooling ratio looks like [1.0,
+      1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't
+      allow pooling on batch and channels dimensions.  1.44 and 1.73 are pooling
+      ratio on height and width dimensions respectively.
+    pseudo_random: An optional `bool`.  Defaults to `False`. When set to `True`,
+      generates the pooling sequence in a pseudorandom fashion, otherwise, in a
+      random fashion. Check paper [Benjamin Graham, Fractional
+      Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between
+      pseudorandom and random.
+    overlapping: An optional `bool`.  Defaults to `False`.  When set to `True`,
+      it means when pooling, the values at the boundary of adjacent pooling
+      cells are used by both cells. For example:
+      `index  0  1  2  3  4`
+      `value  20 5  16 3  7`
+      If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used
+      twice.  The result would be [20, 16] for fractional max pooling.
+    seed: An optional `int`.  Defaults to `0`.  If set to be non-zero, the
+      random number generator is seeded by the given seed.  Otherwise it is
+      seeded by a random seed.
+    name: A name for the operation (optional).
+
+  Returns:
+  A tuple of `Tensor` objects (`output`, `row_pooling_sequence`,
+  `col_pooling_sequence`).
+    output: Output `Tensor` after fractional max pooling.  Has the same type as
+      `value`.
+    row_pooling_sequence: A `Tensor` of type `int64`.
+    col_pooling_sequence: A `Tensor` of type `int64`.
+  """
+  if seed == 0:
+    return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random,
+                                          overlapping, deterministic=False,
+                                          seed=0, seed2=0, name=name)
+  else:
+    seed1, seed2 = random_seed.get_seed(seed)
+    return gen_nn_ops.fractional_max_pool(value, pooling_ratio, pseudo_random,
+                                          overlapping, deterministic=True,
+                                          seed=seed1, seed2=seed2, name=name)
+
+
+@tf_export(v1=["nn.fractional_avg_pool"])
+@deprecation.deprecated(date=None, instructions="`seed2` and `deterministic` "
+                        "args are deprecated.  Use fractional_avg_pool_v2.")
+def fractional_avg_pool(value,
+                        pooling_ratio,
+                        pseudo_random=False,
+                        overlapping=False,
+                        deterministic=False,
+                        seed=0,
+                        seed2=0,
+                        name=None):  # pylint: disable=redefined-builtin
+  r"""Performs fractional average pooling on the input.
+
+  This is a deprecated version of `fractional_avg_pool`.
+
+  Fractional average pooling is similar to Fractional max pooling in the pooling
+  region generation step. The only difference is that after pooling regions are
+  generated, a mean operation is performed instead of a max operation in each
+  pooling region.
+
+  Args:
+    value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`.
+    pooling_ratio: A list of `floats` that has length >= 4.  Pooling ratio for
+      each dimension of `value`, currently only supports row and col dimension
+      and should be >= 1.0. For example, a valid pooling ratio looks like [1.0,
+      1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't
+      allow pooling on batch and channels dimensions.  1.44 and 1.73 are pooling
+      ratio on height and width dimensions respectively.
+    pseudo_random: An optional `bool`.  Defaults to `False`. When set to `True`,
+      generates the pooling sequence in a pseudorandom fashion, otherwise, in a
+      random fashion. Check paper [Benjamin Graham, Fractional
+      Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between
+      pseudorandom and random.
+    overlapping: An optional `bool`.  Defaults to `False`.  When set to `True`,
+      it means when pooling, the values at the boundary of adjacent pooling
+      cells are used by both cells. For example:
+      `index  0  1  2  3  4`
+      `value  20 5  16 3  7`
+      If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used
+      twice.  The result would be [20, 16] for fractional avg pooling.
+    deterministic: An optional `bool`.  Deprecated; use `fractional_avg_pool_v2`
+      instead.
+    seed: An optional `int`.  Defaults to `0`.  If set to be non-zero, the
+      random number generator is seeded by the given seed.  Otherwise it is
+      seeded by a random seed.
+    seed2: An optional `int`.  Deprecated; use `fractional_avg_pool_v2` instead.
+    name: A name for the operation (optional).
+
+  Returns:
+  A tuple of `Tensor` objects (`output`, `row_pooling_sequence`,
+  `col_pooling_sequence`).
+    output: Output `Tensor` after fractional avg pooling.  Has the same type as
+      `value`.
+    row_pooling_sequence: A `Tensor` of type `int64`.
+    col_pooling_sequence: A `Tensor` of type `int64`.
+  """
+  return gen_nn_ops.fractional_avg_pool(value, pooling_ratio, pseudo_random,
+                                        overlapping, deterministic, seed, seed2,
+                                        name=name)
+
+
+@tf_export("nn.fractional_avg_pool", v1=[])
+def fractional_avg_pool_v2(value,
+                           pooling_ratio,
+                           pseudo_random=False,
+                           overlapping=False,
+                           seed=0,
+                           name=None):  # pylint: disable=redefined-builtin
+  r"""Performs fractional average pooling on the input.
+
+  Fractional average pooling is similar to Fractional max pooling in the pooling
+  region generation step. The only difference is that after pooling regions are
+  generated, a mean operation is performed instead of a max operation in each
+  pooling region.
+
+  Args:
+    value: A `Tensor`. 4-D with shape `[batch, height, width, channels]`.
+    pooling_ratio: A list of `floats` that has length >= 4.  Pooling ratio for
+      each dimension of `value`, currently only supports row and col dimension
+      and should be >= 1.0. For example, a valid pooling ratio looks like [1.0,
+      1.44, 1.73, 1.0]. The first and last elements must be 1.0 because we don't
+      allow pooling on batch and channels dimensions.  1.44 and 1.73 are pooling
+      ratio on height and width dimensions respectively.
+    pseudo_random: An optional `bool`.  Defaults to `False`. When set to `True`,
+      generates the pooling sequence in a pseudorandom fashion, otherwise, in a
+      random fashion. Check paper [Benjamin Graham, Fractional
+      Max-Pooling](http://arxiv.org/abs/1412.6071) for difference between
+      pseudorandom and random.
+    overlapping: An optional `bool`.  Defaults to `False`.  When set to `True`,
+      it means when pooling, the values at the boundary of adjacent pooling
+      cells are used by both cells. For example:
+      `index  0  1  2  3  4`
+      `value  20 5  16 3  7`
+      If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used
+      twice.  The result would be [20, 16] for fractional avg pooling.
+    seed: An optional `int`.  Defaults to `0`.  If set to be non-zero, the
+      random number generator is seeded by the given seed.  Otherwise it is
+      seeded by a random seed.
+    name: A name for the operation (optional).
+
+  Returns:
+  A tuple of `Tensor` objects (`output`, `row_pooling_sequence`,
+  `col_pooling_sequence`).
+    output: Output `Tensor` after fractional avg pooling.  Has the same type as
+      `value`.
+    row_pooling_sequence: A `Tensor` of type `int64`.
+    col_pooling_sequence: A `Tensor` of type `int64`.
+  """
+  if seed == 0:
+    return gen_nn_ops.fractional_avg_pool(value, pooling_ratio, pseudo_random,
+                                          overlapping, deterministic=False,
+                                          seed=0, seed2=0, name=name)
+  else:
+    seed1, seed2 = random_seed.get_seed(seed)
+    return gen_nn_ops.fractional_avg_pool(value, pooling_ratio, pseudo_random,
+                                          overlapping, deterministic=True,
+                                          seed=seed1, seed2=seed2, name=name)
+
+
 @tf_export("nn.conv1d")
 @deprecation.deprecated_arg_values(
     None,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index c4da3d670a..2fce093e16 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -150,11 +150,11 @@ tf_module {
   }
   member_method {
     name: "fractional_avg_pool"
-    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'0\', \'0\', \'None\'], "
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'0\', \'None\'], "
   }
   member_method {
     name: "fractional_max_pool"
-    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'deterministic\', \'seed\', \'seed2\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'0\', \'0\', \'None\'], "
+    argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'0\', \'None\'], "
   }
   member_method {
     name: "fused_batch_norm"
-- 
GitLab


From 7ba1642f782a3db29c55987d1953198fa4302acb Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 15 Nov 2018 10:51:00 -0800
Subject: [PATCH 0306/1554] Changing public feature column method imports to
 come from feature_column_lib rather than the base feature_column file. This
 enables better file management of the feature column repository.

PiperOrigin-RevId: 221651041
---
 tensorflow/contrib/layers/BUILD                               | 4 ++--
 .../contrib/layers/python/layers/feature_column_ops_test.py   | 2 +-
 .../contrib/layers/python/layers/feature_column_test.py       | 2 +-
 tensorflow/contrib/learn/python/learn/estimators/dnn.py       | 4 ++--
 .../learn/python/learn/estimators/dnn_linear_combined.py      | 2 +-
 .../learn/python/learn/estimators/dnn_linear_combined_test.py | 2 +-
 tensorflow/contrib/learn/python/learn/estimators/dnn_test.py  | 2 +-
 tensorflow/contrib/learn/python/learn/estimators/linear.py    | 2 +-
 .../contrib/learn/python/learn/estimators/linear_test.py      | 2 +-
 9 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index e6596bfdfb..795591ea62 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -253,7 +253,7 @@ py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
-        "//tensorflow/python/feature_column",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
     ],
 )
@@ -277,7 +277,7 @@ py_test(
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
-        "//tensorflow/python/feature_column",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py
index 6fb4b9ff35..7e6eafaa0d 100644
--- a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py
+++ b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py
@@ -27,7 +27,7 @@ from tensorflow.contrib.layers.python.layers import feature_column
 from tensorflow.contrib.layers.python.layers import feature_column_ops
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/layers/python/layers/feature_column_test.py b/tensorflow/contrib/layers/python/layers/feature_column_test.py
index d90d6ecf7f..cab8da808b 100644
--- a/tensorflow/contrib/layers/python/layers/feature_column_test.py
+++ b/tensorflow/contrib/layers/python/layers/feature_column_test.py
@@ -27,7 +27,7 @@ import numpy as np
 
 from tensorflow.contrib.layers.python.layers import feature_column as fc
 from tensorflow.contrib.layers.python.layers import feature_column_ops
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn.py b/tensorflow/contrib/learn/python/learn/estimators/dnn.py
index eabebb7e88..18ca4214a1 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn.py
@@ -28,7 +28,6 @@ import six
 from tensorflow.contrib import layers
 from tensorflow.contrib.framework import deprecated
 from tensorflow.contrib.framework import deprecated_arg_values
-from tensorflow.python.training import training_util
 from tensorflow.contrib.layers.python.layers import feature_column
 from tensorflow.contrib.layers.python.layers import optimizers
 from tensorflow.contrib.learn.python.learn import metric_spec
@@ -38,11 +37,12 @@ from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
 from tensorflow.contrib.learn.python.learn.estimators import model_fn
 from tensorflow.contrib.learn.python.learn.estimators import prediction_key
 from tensorflow.contrib.learn.python.learn.utils import export
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.summary import summary
+from tensorflow.python.training import training_util
 
 # The default learning rate of 0.05 is a historical artifact of the initial
 # implementation, but seems a reasonable choice.
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py
index 3d85533d92..7a3cc8bd98 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py
@@ -38,7 +38,7 @@ from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
 from tensorflow.contrib.learn.python.learn.estimators import model_fn
 from tensorflow.contrib.learn.python.learn.estimators import prediction_key
 from tensorflow.contrib.learn.python.learn.utils import export
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import nn
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py
index 4e65c180d8..d46a873bfa 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined_test.py
@@ -36,7 +36,7 @@ from tensorflow.contrib.learn.python.learn.estimators import run_config
 from tensorflow.contrib.learn.python.learn.estimators import test_data
 from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec
 from tensorflow.contrib.metrics.python.ops import metric_ops
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py
index 2bd57597c2..ee25cebd48 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py
@@ -38,7 +38,7 @@ from tensorflow.contrib.learn.python.learn.estimators import run_config
 from tensorflow.contrib.learn.python.learn.estimators import test_data
 from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec
 from tensorflow.contrib.metrics.python.ops import metric_ops
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py
index e100bc7a1e..439b17e505 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py
@@ -37,7 +37,7 @@ from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
 from tensorflow.contrib.learn.python.learn.estimators import prediction_key
 from tensorflow.contrib.learn.python.learn.utils import export
 from tensorflow.contrib.linear_optimizer.python import sdca_optimizer
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py
index 597ca4e86d..64ee2eef0a 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py
@@ -37,7 +37,7 @@ from tensorflow.contrib.learn.python.learn.estimators import test_data
 from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec
 from tensorflow.contrib.linear_optimizer.python import sdca_optimizer as sdca_optimizer_lib
 from tensorflow.contrib.metrics.python.ops import metric_ops
-from tensorflow.python.feature_column import feature_column as fc_core
+from tensorflow.python.feature_column import feature_column_lib as fc_core
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
-- 
GitLab


From 181b59b22052ca1ad090a0bdbbf626726b7e489b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 10:56:29 -0800
Subject: [PATCH 0307/1554] Fix SIGFPE when concatenating two "empty" ragged
 tensors.

PiperOrigin-RevId: 221652186
---
 tensorflow/core/kernels/ragged_gather_op.cc           | 6 ++++--
 tensorflow/python/ops/ragged/ragged_concat_op_test.py | 5 +++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/ragged_gather_op.cc b/tensorflow/core/kernels/ragged_gather_op.cc
index b2a342f637..903a97a960 100644
--- a/tensorflow/core/kernels/ragged_gather_op.cc
+++ b/tensorflow/core/kernels/ragged_gather_op.cc
@@ -236,8 +236,10 @@ class RaggedGatherOpBase : public OpKernel {
     values_shape.set_dim(0, num_values);
     TF_RETURN_IF_ERROR(
         context->allocate_output(values_index, values_shape, &values_out));
-    int64 value_size = params_dense_values_in.NumElements() /
-                       params_dense_values_in.dim_size(0);
+    const int64 num_elements = params_dense_values_in.NumElements();
+    const int64 value_size =
+        num_elements == 0 ? 0
+                          : (num_elements / params_dense_values_in.dim_size(0));
     CallWriteValueSlices(params_dense_values_in, value_slices, value_size,
                          values_out);
     return ::tensorflow::Status::OK();
diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
index 6b1a602d04..bddc5d8580 100644
--- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
@@ -41,6 +41,11 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     ]
 
   @parameterized.parameters(
+      dict(
+          descr='Two rank-2 inputs with empty value axis=1',
+          rt_inputs=([[]], [[]]),
+          axis=1,
+          expected=[[]]),
       dict(
           descr='Two rank-2 inputs (ragged_rank=1), axis=0',
           rt_inputs=(
-- 
GitLab


From 1ffa8477eb7fd69cd1f513b330ebaabdf6433a76 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 15 Nov 2018 10:56:52 -0800
Subject: [PATCH 0308/1554] Changing public feature column method imports to
 come from feature_column_lib rather than the base feature_column file. This
 enables better file management of the feature column repository.

PiperOrigin-RevId: 221652243
---
 tensorflow/python/BUILD                                         | 2 +-
 .../python/keras/engine/feature_columns_integration_test.py     | 2 +-
 tensorflow/python/training/warm_starting_util_test.py           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 18f226beb5..7cc1237fe7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4904,7 +4904,7 @@ py_test(
         ":training",
         ":variable_scope",
         ":variables",
-        "//tensorflow/python/feature_column",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index 9ee6a0a27a..c187b1f8e5 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -22,7 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.feature_column import feature_column_v2 as fc
+from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.platform import test
diff --git a/tensorflow/python/training/warm_starting_util_test.py b/tensorflow/python/training/warm_starting_util_test.py
index 91a0b53b3a..b575b8d364 100644
--- a/tensorflow/python/training/warm_starting_util_test.py
+++ b/tensorflow/python/training/warm_starting_util_test.py
@@ -22,7 +22,7 @@ import os
 import numpy as np
 import six
 
-from tensorflow.python.feature_column import feature_column as fc
+from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-- 
GitLab


From 76d204f38757c2b4a3a82020b90b7e739b2c90b6 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 15 Nov 2018 11:01:24 -0800
Subject: [PATCH 0309/1554] Keep side effectful ops in grappler function items

PiperOrigin-RevId: 221653198
---
 tensorflow/core/grappler/op_types.cc          |  4 +++
 tensorflow/core/grappler/utils/functions.cc   | 10 ++-----
 tensorflow/core/grappler/utils/functions.h    | 16 ++++++-----
 .../core/grappler/utils/functions_test.cc     | 27 +++++++++++++++++++
 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index ebc4e9c466..08d4e39ec2 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -571,6 +571,10 @@ bool IsFreeOfSideEffect(const NodeDef& node) {
   if (node.op().find("Queue") != string::npos) {
     return false;
   }
+  // Sending a tensor via a network is a side effect.
+  if (IsSend(node)) {
+    return false;
+  }
   return !ModifiesInputsInPlace(node);
 }
 
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index c60a450573..7756c73967 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -347,12 +347,6 @@ GrapplerFunctionItem::GrapplerFunctionItem(
       fetch.push_back(output_tensor);
     }
   }
-  // Stateful and Send (it's not stateful) nodes must be preserved in the graph.
-  for (const NodeDef& node : graph.node()) {
-    if (IsSend(node)) {
-      keep_ops.push_back(node.name());
-    }
-  }
 }
 
 const string& GrapplerFunctionItem::description() const { return description_; }
@@ -584,8 +578,8 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
     TF_RETURN_IF_ERROR(RegisterFunctionBodyOutputs(*registration, func_def_node,
                                                    &connectivity));
 
-    // Stateful and Send nodes must be preserved in a function body
-    if (registration->op_def.is_stateful() || IsSend(func_def_node)) {
+    // Ops with side effects must be preserved in a function body.
+    if (!IsFreeOfSideEffect(func_def_node)) {
       keep_nodes.push_back(func_def_node.name());
     }
   }
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index 2eec83e998..ba9950e484 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -142,12 +142,6 @@ class GrapplerFunctionItemInstantiation {
 class GrapplerFunctionItem : public GrapplerItem {
  public:
   GrapplerFunctionItem() = default;
-  GrapplerFunctionItem(string func_name, string description,
-                       AttrSlice func_attr,
-                       std::vector<InputArgExpansion> input_arg_expansions,
-                       std::vector<OutputArgExpansion> output_arg_expansions,
-                       std::vector<string> keep_nodes, int graph_def_version,
-                       bool is_stateful, GraphDef&& function_body);
 
   const string& description() const;
 
@@ -170,12 +164,22 @@ class GrapplerFunctionItem : public GrapplerItem {
   GrapplerFunctionItem& SwapFunctionBody(GraphDef&& other);
 
  private:
+  friend Status MakeGrapplerFunctionItem(const FunctionDef&, const AttrSlice&,
+                                         const FunctionLibraryDefinition&, int,
+                                         GrapplerFunctionItem*);
   friend Status ReplaceInputWithConst(const NodeDef&, int,
                                       GrapplerFunctionItem*);
   friend Status RemoveUnusedOutputs(
       const gtl::FlatSet<int>& active_outputs, GrapplerFunctionItem* item,
       std::vector<std::pair<int, int>>* output_mapping);
 
+  GrapplerFunctionItem(string func_name, string description,
+                       AttrSlice func_attr,
+                       std::vector<InputArgExpansion> input_arg_expansions,
+                       std::vector<OutputArgExpansion> output_arg_expansions,
+                       std::vector<string> keep_nodes, int graph_def_version,
+                       bool is_stateful, GraphDef&& function_body);
+
   string description_;
   AttrSlice func_attr_;  // Attributes specific to function definition that
                          // produced this item (FuncDef.attr field).
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 31b9d7e49c..8639dec05a 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -576,6 +576,33 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) {
   EXPECT_EQ("two", cast.input(0));
 }
 
+TEST_F(FunctionsTest, FromFunctionDefWithSideEffectfulOps) {
+  const Tensor kOne = test::AsScalar<float>(1.0);
+  FunctionDef func = FunctionDefHelper::Define(
+      /* Name */ "SideEffects",
+      /* Args */ {"x: Ref(float)"},
+      /* Return values */ {},
+      /* Attr def */ {},
+      /* Nodes */
+      {{{"one"}, "Const", {}, {{"value", kOne}, {"dtype", DT_FLOAT}}},
+       {{"update"}, "AssignAdd", {"x", "one"}, {{"T", DT_FLOAT}}}});
+
+  protobuf::Map<string, AttrValue> func_instantiation_attr;
+  FunctionLibraryDefinition flib(OpRegistry::Global(), FunctionDefLibrary());
+
+  GrapplerFunctionItem item;
+  TF_EXPECT_OK(MakeGrapplerFunctionItem(func,
+                                        AttrSlice(&func_instantiation_attr),
+                                        flib, TF_GRAPH_DEF_VERSION, &item));
+
+  EXPECT_EQ("SideEffects", item.id);
+  EXPECT_EQ(3, item.function_body().node_size());
+  EXPECT_EQ(1, item.input_size());
+  EXPECT_EQ(0, item.output_size());
+  ASSERT_EQ(1, item.keep_ops.size());
+  EXPECT_EQ("update", item.keep_ops[0]);
+}
+
 TEST_F(FunctionsTest, MakeFunctionDef) {
   const Tensor kTwo = test::AsScalar<int64>(2);
   FunctionDef func = FunctionDefHelper::Define(
-- 
GitLab


From c7d57d435c2163bd314bf0d39a19d6ce9321196b Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Thu, 15 Nov 2018 11:08:45 -0800
Subject: [PATCH 0310/1554] Update the arguments for
 `tf.nn.sufficient_statistics` and exporting in v2. Renaming 'keep_dims'
 argument to 'keepdims'.

PiperOrigin-RevId: 221654843
---
 tensorflow/python/ops/nn_impl.py              | 31 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  3 ++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index b46cc6488c..245aa0e52f 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -668,7 +668,7 @@ def separable_conv2d_v2(
 # pylint: enable=redefined-builtin,line-too-long
 
 
-@tf_export("nn.sufficient_statistics")
+@tf_export(v1=["nn.sufficient_statistics"])
 def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None):
   """Calculate the sufficient statistics for the mean and variance of `x`.
 
@@ -718,6 +718,35 @@ def sufficient_statistics(x, axes, shift=None, keep_dims=False, name=None):
   return counts, m_ss, v_ss, shift
 
 
+@tf_export("nn.sufficient_statistics", v1=[])
+def sufficient_statistics_v2(x, axes, shift=None, keepdims=False, name=None):
+  """Calculate the sufficient statistics for the mean and variance of `x`.
+
+  These sufficient statistics are computed using the one pass algorithm on
+  an input that's optionally shifted. See:
+  https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Computing_shifted_data
+
+  Args:
+    x: A `Tensor`.
+    axes: Array of ints. Axes along which to compute mean and variance.
+    shift: A `Tensor` containing the value by which to shift the data for
+      numerical stability, or `None` if no shift is to be performed. A shift
+      close to the true mean provides the most numerically stable results.
+    keepdims: produce statistics with the same dimensionality as the input.
+    name: Name used to scope the operations that compute the sufficient stats.
+
+  Returns:
+    Four `Tensor` objects of the same type as `x`:
+
+    * the count (number of elements to average over).
+    * the (possibly shifted) sum of the elements in the array.
+    * the (possibly shifted) sum of squares of the elements in the array.
+    * the shift by which the mean must be corrected or None if `shift` is None.
+  """
+  return sufficient_statistics(
+      x=x, axes=axes, shift=shift, keep_dims=keepdims, name=name)
+
+
 @tf_export("nn.normalize_moments")
 def normalize_moments(counts, mean_ss, variance_ss, shift, name=None):
   """Calculate the mean and variance of based on the sufficient statistics.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 2fce093e16..881f566fe2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -314,7 +314,7 @@ tf_module {
   }
   member_method {
     name: "sufficient_statistics"
-    argspec: "args=[\'x\', \'axes\', \'shift\', \'keep_dims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'x\', \'axes\', \'shift\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "tanh"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 1e050cca9b..82044ff43f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -47,6 +47,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         },
         "tf.nn.separable_conv2d": {
             "rate": "dilations"
+        },
+        "tf.nn.sufficient_statistics": {
+            "keep_dims": "keepdims"
         }
     }
 
-- 
GitLab


From 2e21bcf118a549a65c9a6d036fa06eba0f2346e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 11:21:10 -0800
Subject: [PATCH 0311/1554] Avoid constant folding partial output of a node. It
 fixes issues #22383. Its root cause is that tf only fold values but not
 indices when running TopK on GPU with dtype float. TopK generates different
 results on GPU and CPU.

PiperOrigin-RevId: 221657401
---
 tensorflow/core/BUILD                         |  1 +
 .../core/common_runtime/constant_folding.cc   | 26 ++++++-----
 .../common_runtime/constant_folding_test.cc   | 46 +++++++++++++++++++
 3 files changed, 62 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c69f3afd8b..3ee31e0b29 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -4106,6 +4106,7 @@ tf_cc_test(
         "//tensorflow/core/kernels:identity_op",
         "//tensorflow/core/kernels:immutable_constant_op",
         "//tensorflow/core/kernels:matmul_op",
+        "//tensorflow/core/kernels:topk_op",
         "//third_party/eigen3",
     ],
 )
diff --git a/tensorflow/core/common_runtime/constant_folding.cc b/tensorflow/core/common_runtime/constant_folding.cc
index 6d5c7f951e..5c226ec56e 100644
--- a/tensorflow/core/common_runtime/constant_folding.cc
+++ b/tensorflow/core/common_runtime/constant_folding.cc
@@ -471,10 +471,10 @@ bool ReplaceTensorWithConstant(
   // Be conservative when replacing a tensor with a constant, when not
   // running on CPU.
   // 1) Do not replace another constant.
-  // 2) If the destination tensor is not an int32 tensor, and has HOST_MEMORY
-  // constraint, do not replace it.
-  // 3) If the destination tensor is an int32 tensor, and has DEVICE_MEMORY
-  // constraint, do not replace it.
+  // 2) If the destination tensor or any other tensor from the same node is not
+  // an int32 tensor, and has HOST_MEMORY constraint, do not replace it.
+  // 3) If the destination tensor or any other tensor from the same node is an
+  // int32 tensor, and has DEVICE_MEMORY constraint, do not replace it.
   // 4) If the size of the constant in bytes is too large (>
   // max_constant_in_bytes), do not replace it. This prevents the size of the
   // Graph from growing too large.
@@ -490,16 +490,20 @@ bool ReplaceTensorWithConstant(
                                ? DeviceType{partition_device->device_type()}
                                : DEVICE_CPU;
   if (partition_device && device_type != DEVICE_CPU) {
-    MemoryType memory_type;
-    if (!MemoryTypeForOutput(device_type, graph, tensor.first, tensor.second,
-                             &memory_type)
+    MemoryTypeVector input_mvec;
+    MemoryTypeVector output_mvec;
+    if (!MemoryTypesForNode(graph->op_registry(), device_type,
+                            tensor.first->def(), &input_mvec, &output_mvec)
              .ok()) {
       return false;
     }
-    bool is_int32 = tensor.first->output_type(tensor.second) == DT_INT32;
-    if ((memory_type == HOST_MEMORY && !is_int32) ||
-        (memory_type == DEVICE_MEMORY && is_int32)) {
-      return false;
+    for (int i = 0; i < output_mvec.size(); i++) {
+      MemoryType memory_type = output_mvec[i];
+      bool is_int32 = tensor.first->output_type(i) == DT_INT32;
+      if ((memory_type == HOST_MEMORY && !is_int32) ||
+          (memory_type == DEVICE_MEMORY && is_int32)) {
+        return false;
+      }
     }
   }
   if (constant.TotalBytes() > max_constant_size_in_bytes) {
diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc
index 98aefcde27..1d4586f3da 100644
--- a/tensorflow/core/common_runtime/constant_folding_test.cc
+++ b/tensorflow/core/common_runtime/constant_folding_test.cc
@@ -18,13 +18,16 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "tensorflow/cc/ops/nn_ops.h"
 #include "tensorflow/core/common_runtime/constant_folding.h"
 
 #include "tensorflow/cc/ops/array_ops_internal.h"
 #include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/framework/device_attributes.pb.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -90,6 +93,24 @@ class ConstantFoldingTest : public ::testing::Test {
   }
 };
 
+class FakeDevice : public Device {
+ private:
+  explicit FakeDevice(const DeviceAttributes& device_attributes)
+      : Device(nullptr, device_attributes) {}
+
+ public:
+  Status Sync() override { return errors::Unimplemented("FakeDevice::Sync()"); }
+
+  Allocator* GetAllocator(AllocatorAttributes attr) override { return nullptr; }
+
+  static std::unique_ptr<Device> Make(const string& name, const string& type) {
+    DeviceAttributes device_attributes;
+    device_attributes.set_name(name);
+    device_attributes.set_device_type(DeviceType(type).type());
+    return std::unique_ptr<Device>(new FakeDevice(device_attributes));
+  }
+};
+
 TEST_F(ConstantFoldingTest, Basic) {
   Scope s = Scope::NewRootScope();
   BuildSimpleGraph(&s);
@@ -610,6 +631,31 @@ TEST_F(ConstantFoldingTest, ConstShapeKnown) {
   }
 }
 
+TEST_F(ConstantFoldingTest, NoReplacePartialOutput) {
+  Graph g(OpRegistry::Global());
+  {
+    Scope s = Scope::NewRootScope().ExitOnError().WithAssignedDevice("/gpu:0");
+
+    auto c0 = ops::Const<float>(s.WithOpName("c0"), {5.0, 2.0, 8.0, 1.0}, {4});
+    auto k = ops::Const<int>(s.WithOpName("k"), 3);
+    auto topK =
+        ops::TopK(s.WithOpName("topK"), c0, k, ops::TopK::Sorted(false));
+    auto send_values = ops::_Send(s.WithOpName("send_values"), topK.values,
+                                  "send_values", "sender", 0, "receiver");
+    auto send_indices = ops::_Send(s.WithOpName("send_indices"), topK.indices,
+                                   "send_indices", "sender", 0, "receiver");
+    TF_ASSERT_OK(s.ToGraph(&g));
+  }
+  bool was_mutated;
+  TF_EXPECT_OK(ConstantFold(
+      ConstantFoldingOptions{}, nullptr, Env::Default(),
+      FakeDevice::Make("/job:tpu_worker/replica:0/task:0/device:GPU:0",
+                       DEVICE_GPU)
+          .get(),
+      &g, &was_mutated));
+  EXPECT_FALSE(was_mutated);
+}
+
 namespace {
 
 const char kTestMemRegionName[] = "test://test";
-- 
GitLab


From 71ba0ec86e7cde759006f17979f71e863602182c Mon Sep 17 00:00:00 2001
From: Ouwen Huang <ouwen.huang@duke.edu>
Date: Thu, 15 Nov 2018 14:28:48 -0500
Subject: [PATCH 0312/1554] Update weight_decay_optimizers.py

---
 .../opt/python/training/weight_decay_optimizers.py        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py b/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py
index 1e8351b70f..8b8065c678 100644
--- a/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py
+++ b/tensorflow/contrib/opt/python/training/weight_decay_optimizers.py
@@ -64,10 +64,10 @@ class DecoupledWeightDecayExtension(object):
   the decay to the `weight_decay` as well. For example:
 
   ```python
-    decay = tf.train.piecewise_constant(tf.train.get_global_step(), 
-                                        [10000, 15000], [1e-1, 1e-2, 1e-3])
-    lr = 1*decay
-    wd = 1e-4*decay
+    schedule = tf.train.piecewise_constant(tf.train.get_global_step(), 
+                                           [10000, 15000], [1e-0, 1e-1, 1e-2])
+    lr = 1e-1 * schedule()
+    wd = lambda: 1e-4 * schedule()
 
     # ...
 
-- 
GitLab


From 8aff5e540a4771a56926ac30be0ebb4cc512e606 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 11:21:55 -0800
Subject: [PATCH 0313/1554] Apply EIGEN_STRONG_INLINE to all
 tf_custom_op_library when --define=override_eigen_strong_inline=true

This avoids hitting #10521 for tensorflow/contrib/rnn/kernels/lstm_ops.cc

PiperOrigin-RevId: 221657546
---
 tensorflow/tensorflow.bzl | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 8e5ab94b53..f01dc64bcf 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1493,7 +1493,7 @@ check_deps = rule(
     },
 )
 
-def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [], **kwargs):
+def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [], copts = [], **kwargs):
     """Helper to build a dynamic library (.so) from the sources containing implementations of custom ops and kernels.
     """
     cuda_deps = [
@@ -1505,12 +1505,18 @@ def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [
         clean_dep("//tensorflow/core:stream_executor_headers_lib"),
     ]
     deps = deps + tf_custom_op_library_additional_deps()
+
+    # Override EIGEN_STRONG_INLINE to inline when
+    # --define=override_eigen_strong_inline=true to avoid long compiling time.
+    # See https://github.com/tensorflow/tensorflow/issues/10521
+    copts = copts + if_override_eigen_strong_inline(["/DEIGEN_STRONG_INLINE=inline"])
+
     if gpu_srcs:
         basename = name.split(".")[0]
         native.cc_library(
             name = basename + "_gpu",
             srcs = gpu_srcs,
-            copts = _cuda_copts() + if_tensorrt(["-DGOOGLE_TENSORRT=1"]),
+            copts = copts + _cuda_copts() + if_tensorrt(["-DGOOGLE_TENSORRT=1"]),
             features = if_cuda(["-use_header_modules"]),
             deps = deps + if_cuda_is_configured_compat(cuda_deps) + if_rocm_is_configured(rocm_deps),
             **kwargs
@@ -1531,7 +1537,7 @@ def tf_custom_op_library(name, srcs = [], gpu_srcs = [], deps = [], linkopts = [
         srcs = srcs,
         deps = deps + if_cuda_is_configured_compat(cuda_deps) + if_rocm_is_configured(rocm_deps),
         data = if_static([name + "_check_deps"]),
-        copts = tf_copts(is_external = True),
+        copts = copts + tf_copts(is_external = True),
         features = ["windows_export_all_symbols"],
         linkopts = linkopts + select({
             "//conditions:default": [
-- 
GitLab


From e0ef2053ac909caf6613527e37838bef346bedea Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 15 Nov 2018 11:34:39 -0800
Subject: [PATCH 0314/1554] Fix and test issue with incorrect target shape
 assumption check.

PiperOrigin-RevId: 221660032
---
 tensorflow/python/keras/engine/training.py    |  4 ++-
 .../python/keras/engine/training_test.py      | 30 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 3562060707..856f315a76 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1284,7 +1284,9 @@ class Model(Network):
       y = training_utils.standardize_input_data(
           y,
           feed_output_names,
-          feed_output_shapes,
+          # Don't enforce target shapes to match output shapes.
+          # Precise checks will be run in `check_loss_and_target_compatibility`.
+          shapes=None,
           check_batch_axis=False,  # Don't enforce the batch size.
           exception_prefix='target')
 
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 8ba19859d0..359b4da0db 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -544,6 +544,36 @@ class TrainingTest(test.TestCase):
               'val_loss', 'val_weighted_mean_absolute_error'
           ]))
 
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_mismatched_output_shape_and_target_shape(self):
+    model = keras.Sequential([
+        keras.layers.Dense(2, input_shape=(3, 4)),
+        keras.layers.Dense(5),
+    ])
+    model.compile(RMSPropOptimizer(learning_rate=0.001),
+                  loss='sparse_categorical_crossentropy')
+    # Test with Numpy data
+    x_train = np.random.random((10, 3, 4))
+    y_train = np.random.randint(0, 5, size=(10, 3))
+    model.fit(x_train, y_train, batch_size=5, epochs=1)
+
+    # Test with iterator
+    dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+    dataset = dataset.repeat(10)
+    dataset = dataset.batch(10)
+    iterator = dataset.make_one_shot_iterator()
+    model.fit(iterator, epochs=1, steps_per_epoch=2)
+
+    if context.executing_eagerly():
+      # Test with eager execution
+      model.compile(RMSPropOptimizer(learning_rate=0.001),
+                    loss='sparse_categorical_crossentropy',
+                    run_eagerly=True)
+      model.fit(x_train, y_train, batch_size=5, epochs=1)
+
+      # Test with eager execution and iterator
+      model.fit(iterator, epochs=1, steps_per_epoch=2)
+
 
 class TestExceptionsAndWarnings(test.TestCase):
 
-- 
GitLab


From dcf390ede49516cf52445520bb6a918e9bb1249f Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 15 Nov 2018 11:36:11 -0800
Subject: [PATCH 0315/1554] Add abstract methods/properties to the base Cluster
 Resolver class.

Also fixes a bug in KubernetesClusterResolver.master() where we were not getting the attribute correctly and added a test for it.

PiperOrigin-RevId: 221660325
---
 .../python/training/cluster_resolver.py       | 36 +++++++++++++++++--
 .../training/kubernetes_cluster_resolver.py   |  4 +--
 .../kubernetes_cluster_resolver_test.py       |  9 ++---
 .../python/training/tpu_cluster_resolver.py   | 11 ++++--
 4 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
index 3583501a45..7774ac0e12 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
@@ -44,6 +44,17 @@ class ClusterResolver(object):
   automatically discover and resolve IP addresses for various TensorFlow
   workers. This will eventually allow us to automatically recover from
   underlying machine failures and scale TensorFlow worker clusters up and down.
+
+  Note to Implementors: In addition to these abstract methods, you must also
+  implement the task_type, task_index, and rpc_layer attributes. You may choose
+  to implement them either as properties with getters or setters or directly
+  set the attributes.
+
+  - task_type is the name of the server's current named job (e.g. 'worker',
+     'ps' in a distributed parameterized training job).
+  - task_index is the ordinal index of the server within the task type.
+  - rpc_layer is the protocol used by TensorFlow to communicate with other
+      TensorFlow servers in a distributed environment.
   """
 
   @abc.abstractmethod
@@ -60,8 +71,7 @@ class ClusterResolver(object):
     management system every time this function is invoked and reconstructing
     a cluster_spec, rather than attempting to cache anything.
     """
-    raise NotImplementedError(
-        'cluster_spec is not implemented for {}.'.format(self))
+    raise NotImplementedError()
 
   @abc.abstractmethod
   def master(self, task_type=None, task_index=None, rpc_layer=None):
@@ -79,7 +89,27 @@ class ClusterResolver(object):
     returned is up-to-date at the time to calling this function. This usually
     means retrieving the master every time this function is invoked.
     """
-    raise NotImplementedError('master is not implemented for {}.'.format(self))
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def num_accelerators_per_worker(self, session_config=None):
+    """Returns the number of accelerator cores per worker.
+
+    This returns the number of accelerator cores (such as GPUs and TPUs)
+    available per worker. If workers only has CPU cores available, then this
+    should return 0. This method will query the master for this information
+    if it is not otherwise known.
+
+    Args:
+      session_config: (Optional) Configuration for starting a new session to
+        query how many accelerator cores it has.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractproperty
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in."""
+    raise NotImplementedError()
 
 
 class SimpleClusterResolver(ClusterResolver):
diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
index a705b99374..eab1359a5b 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
@@ -113,9 +113,9 @@ class KubernetesClusterResolver(ClusterResolver):
           self.cluster_spec().task_address(task_type, task_index),
           rpc_layer or self.rpc_layer)
 
-    if self._task_type is not None and self._task_index is not None:
+    if self.task_type is not None and self.task_index is not None:
       return format_master_url(
-          self.cluster_spec().task_address(self._task_type, self._task_index),
+          self.cluster_spec().task_address(self.task_type, self.task_index),
           rpc_layer or self.rpc_layer)
 
     return ''
diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
index 9ab1fcb309..c63a98af6c 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
@@ -118,10 +118,11 @@ class KubernetesClusterResolverTest(test.TestCase):
     cluster_resolver = KubernetesClusterResolver(
         override_client=_mock_kubernetes_client(
             {'job-name=tensorflow': ret}))
-    cluster_resolver.task_type = 'blah'
-    cluster_resolver.task_index = 1
-    self.assertEqual(cluster_resolver.task_type, 'blah')
-    self.assertEqual(cluster_resolver.task_index, 1)
+    cluster_resolver.task_type = 'worker'
+    cluster_resolver.task_index = 0
+    self.assertEqual(cluster_resolver.task_type, 'worker')
+    self.assertEqual(cluster_resolver.task_index, 0)
+    self.assertEqual(cluster_resolver.master(), 'grpc://10.1.2.3:8470')
     self.assertEqual(cluster_resolver.master('worker', 2),
                      'grpc://10.1.2.5:8470')
 
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
index 1a7a31c15c..d5537a4100 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
@@ -192,13 +192,13 @@ class TPUClusterResolver(ClusterResolver):
 
     if tpu.startswith('grpc://'):
       # Cloud environment, where we are using GRPC to communicate to TPUs.
-      self.environment = ''
+      self._environment = ''
     elif tpu == 'local' or not tpu:
       # Google environment, where the TPU is attached to the host.
-      self.environment = 'google'
+      self._environment = 'google'
     elif tpu.startswith('/bns'):
       # Google environment, where we reach the TPU through BNS.
-      self.environment = 'google'
+      self._environment = 'google'
 
     # If TPU is in the Google environment or exists locally, we don't use any
     # RPC layer.
@@ -398,6 +398,11 @@ class TPUClusterResolver(ClusterResolver):
     del session_config  # Unused. Not necessary to query anything.
     return 8
 
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in."""
+    return self._environment
+
   def _start_local_server(self):
     address = self._requestComputeMetadata('instance/network-interfaces/0/ip')
     self._server = server_lib.Server(
-- 
GitLab


From 4ea51ae99cdeeb7abb9bc8e5efff188c4729fca3 Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Thu, 15 Nov 2018 11:37:50 -0800
Subject: [PATCH 0316/1554] Change _Send/_Recv attrs in VirtualScheduler

Change src_device_ to send_device and dst_device_ to recv_device. This complies
with tensorflow naming, so that VirtualScheduler can handle graphs generated
on inspectz with _Send/_Recv nodes and AutoGrappler does not need to remove
them.

PiperOrigin-RevId: 221660625
---
 tensorflow/core/grappler/costs/virtual_scheduler.cc | 4 ++++
 tensorflow/core/grappler/costs/virtual_scheduler.h  | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 424994125f..b9b240e72c 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -639,6 +639,8 @@ std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
   send->set_device(ChannelDeviceName(from, to));
   auto& send_attr = *(send->mutable_attr());
   send_attr[kAttrInputSrc].set_s(input_name);
+  // Use input_name as tensor_name.
+  send_attr[kAttrTensorName].set_s(input_name);
   send_attr[kAttrSrcDevice].set_s(DeviceName(from));
   send_attr[kAttrDstDevice].set_s(DeviceName(to));
 
@@ -650,6 +652,8 @@ std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
   recv->set_device(DeviceName(to));
   auto& recv_attr = *(recv->mutable_attr());
   recv_attr[kAttrInputSrc].set_s(input_name);
+  // Use input_name as tensor_name.
+  recv_attr[kAttrTensorName].set_s(input_name);
 
   // NodeState for _Send op.
   auto& send_node_state = GetNodeStateOrCreateIt(send);
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index 89dff9686d..92e0a88782 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -308,8 +308,9 @@ class VirtualScheduler {
  private:
   // Constants.
   const string kAttrInputSrc = "input_source_";
-  const string kAttrSrcDevice = "src_device_";
-  const string kAttrDstDevice = "dst_device_";
+  const string kAttrSrcDevice = "send_device";
+  const string kAttrDstDevice = "recv_device";
+  const string kAttrTensorName = "tensor_name";
   const string kChannelDevice = "Channel";
 
   // Methods called from Init(). Fails if initialize_ is set.
-- 
GitLab


From 02d069baba34da897b0739ac7406688408537ab1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 11:38:36 -0800
Subject: [PATCH 0317/1554] Removes another potential noisy error message from
 gradient tape.

PiperOrigin-RevId: 221660789
---
 tensorflow/python/eager/backprop.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 844c9b52e7..5b6b42155f 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -776,6 +776,8 @@ class GradientTape(object):
         context.context().end_step()
       except AttributeError:
         pass
+      except TypeError:
+        pass
 
   def watch(self, tensor):
     """Ensures that `tensor` is being traced by this tape.
-- 
GitLab


From af832feff72da49f78d4d7b8fd266d0b8f783365 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 15 Nov 2018 11:41:21 -0800
Subject: [PATCH 0318/1554] Changing public feature column method imports to
 come from feature_column_lib rather than the base feature_column file. This
 enables better file management of the feature column repository.

PiperOrigin-RevId: 221661326
---
 .../estimator_batch/dnn_tree_combined_estimator.py              | 2 +-
 .../contrib/distribute/python/estimator_integration_test.py     | 2 +-
 tensorflow/contrib/distribute/python/estimator_training_test.py | 2 +-
 tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py | 2 +-
 tensorflow/contrib/factorization/python/ops/kmeans.py           | 2 +-
 tensorflow/contrib/factorization/python/ops/kmeans_test.py      | 2 +-
 tensorflow/contrib/timeseries/python/timeseries/estimators.py   | 2 +-
 .../contrib/timeseries/python/timeseries/estimators_test.py     | 2 +-
 tensorflow/contrib/timeseries/python/timeseries/head_test.py    | 2 +-
 tensorflow/contrib/timeseries/python/timeseries/model.py        | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py
index ca73e4af2f..358404cd94 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py
@@ -36,7 +36,7 @@ from tensorflow.contrib.learn.python.learn.estimators import estimator
 from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
 from tensorflow.python.estimator import estimator as core_estimator
 from tensorflow.contrib.learn.python.learn.estimators import model_fn
-from tensorflow.python.feature_column import feature_column as feature_column_lib
+from tensorflow.python.feature_column import feature_column_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
diff --git a/tensorflow/contrib/distribute/python/estimator_integration_test.py b/tensorflow/contrib/distribute/python/estimator_integration_test.py
index a1355c0b09..ecda5337bb 100644
--- a/tensorflow/contrib/distribute/python/estimator_integration_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_integration_test.py
@@ -34,7 +34,7 @@ from tensorflow.python.estimator.canned import dnn_linear_combined
 from tensorflow.python.estimator.canned import prediction_keys
 from tensorflow.python.estimator.export import export
 from tensorflow.python.estimator.inputs import numpy_io
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import gfile
 from tensorflow.python.summary.writer import writer_cache
diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 8f82b4c92a..cc15fc80a9 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -45,7 +45,7 @@ from tensorflow.python.estimator import training as estimator_training
 from tensorflow.python.estimator.canned import dnn_linear_combined
 from tensorflow.python.estimator.canned import prediction_keys
 from tensorflow.python.estimator.export import export as export_lib
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary import summary_iterator
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 402a2cfcdf..54b79e2f97 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -36,7 +36,7 @@ from tensorflow.python.estimator.canned import dnn_linear_combined
 from tensorflow.python.estimator.canned import prediction_keys
 from tensorflow.python.estimator.export import export
 from tensorflow.python.estimator.inputs import numpy_io
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.keras.optimizer_v2 import adam
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py
index f384d761a8..3eb396a29c 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans.py
@@ -26,7 +26,7 @@ from tensorflow.contrib.factorization.python.ops import clustering_ops
 from tensorflow.python.estimator import estimator
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator.export import export_output
-from tensorflow.python.feature_column import feature_column as fc
+from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index 1ab5418fe4..2f7cd131d3 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -27,7 +27,7 @@ from sklearn.cluster import KMeans as SklearnKMeans
 # pylint: disable=g-import-not-at-top
 from tensorflow.contrib.factorization.python.ops import kmeans as kmeans_lib
 from tensorflow.python.estimator import run_config
-from tensorflow.python.feature_column import feature_column as fc
+from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators.py b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
index af68aa03cf..146ed9f271 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
@@ -32,7 +32,7 @@ from tensorflow.contrib.timeseries.python.timeseries.state_space_models.filterin
 from tensorflow.python.estimator import estimator_lib
 from tensorflow.python.estimator.canned import optimizers
 from tensorflow.python.estimator.export import export_lib
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
index ffd838be40..7d780559f9 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
@@ -30,7 +30,7 @@ from tensorflow.contrib.timeseries.python.timeseries import saved_model_utils
 
 from tensorflow.python.client import session
 from tensorflow.python.estimator import estimator_lib
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head_test.py b/tensorflow/contrib/timeseries/python/timeseries/head_test.py
index 90c7d8ac1a..8f692d94da 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/head_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/head_test.py
@@ -38,7 +38,7 @@ from tensorflow.core.example import example_pb2
 
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.estimator import estimator_lib
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
diff --git a/tensorflow/contrib/timeseries/python/timeseries/model.py b/tensorflow/contrib/timeseries/python/timeseries/model.py
index edd97b2a4c..a8cd4287e0 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/model.py
@@ -27,7 +27,7 @@ from tensorflow.contrib.timeseries.python.timeseries import math_utils
 from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures
 from tensorflow.contrib.timeseries.python.timeseries.feature_keys import TrainEvalFeatures
 
-from tensorflow.python.feature_column import feature_column
+from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-- 
GitLab


From d9361134494d90cc945c5a77f878e106d663931d Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 11:41:37 -0800
Subject: [PATCH 0319/1554] Update documentation to reflect that XLA_FLAGS and
 TF_XLA_FLAGS are now split.

PiperOrigin-RevId: 221661379
---
 tensorflow/compiler/xla/g3doc/jit.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/g3doc/jit.md b/tensorflow/compiler/xla/g3doc/jit.md
index b61a0358bc..85fa16ccc7 100644
--- a/tensorflow/compiler/xla/g3doc/jit.md
+++ b/tensorflow/compiler/xla/g3doc/jit.md
@@ -144,7 +144,7 @@ Execute the python script to train the model with XLA and turn on a debugging
 feature of XLA via an environmental variable that outputs the XLA graph.
 
 ```shell
-TF_XLA_FLAGS="--xla_hlo_graph_path=/tmp --xla_generate_hlo_graph=.*" python mnist_softmax_xla.py
+XLA_FLAGS="--xla_hlo_graph_path=/tmp --xla_generate_hlo_graph=.*" python mnist_softmax_xla.py
 ```
 
 Open the timeline file created (`timeline.ctf.json`).  The rendered timeline
-- 
GitLab


From 24a06a12201819779b647819d726e486c2b88a68 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 15 Nov 2018 11:44:49 -0800
Subject: [PATCH 0320/1554] Add additional guidance for errors that can
 possibly be caused by buggy inspect.getsource.

PiperOrigin-RevId: 221661908
---
 tensorflow/python/autograph/impl/conversion.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 328a4b5fe4..48a9307cab 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -295,11 +295,17 @@ def function_to_graph(f,
           ' matching signature. To avoid ambiguity, define each lambda'
           ' in a separate expression.'.format(f, source))
     else:
+      # The inspect.getsource bug is currently known to occur in the Windows
+      # integration tests which run Python 3.6.
+      # TODO(mdan): Find out eaxctly which distribution of Python is that.
       raise ValueError(
           'Unable to identify source code of function {}. The source code'
           ' reported by Python did not include exactly one matching signature:'
           '\n{}\nTo avoid ambiguity, use a unique name for each'
-          ' function.'.format(f, source))
+          ' function.\nNote that some distributions of Python may report source'
+          ' code incorrectly. It may be possible to avoid that bug by'
+          ' organizing the code into smaller units (smaller files, functions or'
+          ' classes), or by turning AutoGraph off.'.format(f, source))
   node, = nodes
 
   # TODO(znado): Place inside standard_analysis.
-- 
GitLab


From 87ca7af6920a284447c396fc4a39aa2cb738f7a7 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 15 Nov 2018 11:47:48 -0800
Subject: [PATCH 0321/1554] Break down the tests in optimizer_v2_test to work
 around the bug in inspect.getsource that manifests in windows builds.

PiperOrigin-RevId: 221662383
---
 .../keras/optimizer_v2/optimizer_v2_test.py   | 35 +++++++++++--------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index cb0886ac9a..305267d73e 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -402,21 +402,6 @@ class OptimizerTest(test.TestCase):
     with self.assertRaises(AttributeError):
       opt.not_an_attr += 3
 
-  def testOptimizerWithFunction(self):
-    with context.eager_mode():
-      var = resource_variable_ops.ResourceVariable([1.0, 2.0],
-                                                   dtype=dtypes.float32)
-      loss = lambda: 3 * var
-      opt = adam.Adam(learning_rate=1.0)
-
-      @def_function.function
-      def fn():
-        opt.minimize(loss, [var])
-        return var
-
-      self.assertAllClose([0., 1.], fn(), atol=1e-4)
-      self.assertAllClose([-1, 0.], fn(), atol=1e-4)
-
   @test_util.run_in_graph_and_eager_modes
   def testOptimizerWithKerasModel(self):
     a = input_layer.Input(shape=(3,), name='input_a')
@@ -491,5 +476,25 @@ class OptimizerTest(test.TestCase):
         float(backend.get_value(model.optimizer.lr)), 0.01, atol=1e-4)
 
 
+# Note: These tests are kept in a separate class to avoid bugs in some
+# distributions of Python that break AutoGraph which is used by tf.function.
+class OptimizerWithFunctionTest(test.TestCase):
+
+  def testBasic(self):
+    with context.eager_mode():
+      var = resource_variable_ops.ResourceVariable([1.0, 2.0],
+                                                   dtype=dtypes.float32)
+      loss = lambda: 3 * var
+      opt = adam.Adam(learning_rate=1.0)
+
+      @def_function.function
+      def fn():
+        opt.minimize(loss, [var])
+        return var
+
+      self.assertAllClose([0., 1.], fn(), atol=1e-4)
+      self.assertAllClose([-1, 0.], fn(), atol=1e-4)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 79e73ee2294c06866e4b01a02eb3675eb08a5969 Mon Sep 17 00:00:00 2001
From: Taylor Robie <taylorrobie@google.com>
Date: Thu, 15 Nov 2018 11:49:38 -0800
Subject: [PATCH 0322/1554] Move `name` argument to the end of the `shape`,
 `size`, and `sparse.concat` functions.

Add a rewrite rule for `sparse_concat` to `sparse.concat`.

PiperOrigin-RevId: 221662719
---
 tensorflow/python/ops/array_ops.py               | 16 ++++++++++++++--
 tensorflow/python/ops/sparse_ops.py              |  8 +++++++-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt  |  6 +++---
 .../tools/api/golden/v2/tensorflow.sparse.pbtxt  |  2 +-
 tensorflow/tools/compatibility/tf_upgrade_v2.py  | 14 ++++++++++----
 5 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index d0efbdbaf8..5a81442f3a 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -269,7 +269,13 @@ def broadcast_static_shape(shape_x, shape_y):
   return common_shapes.broadcast_shape(shape_x, shape_y)
 
 
-@tf_export("shape")
+@tf_export("shape", v1=[])
+def shape_v2(input, out_type=dtypes.int32, name=None):
+  # pylint: disable=redefined-builtin
+  return shape(input, name, out_type)
+
+
+@tf_export(v1=["shape"])
 def shape(input, name=None, out_type=dtypes.int32):
   # pylint: disable=redefined-builtin
   """Returns the shape of a tensor.
@@ -342,7 +348,13 @@ def shape_n(input, out_type=dtypes.int32, name=None):
   return gen_array_ops.shape_n(input, out_type=out_type, name=name)
 
 
-@tf_export("size")
+@tf_export("size", v1=[])
+def size_v2(input, out_type=dtypes.int32, name=None):
+  # pylint: disable=redefined-builtin
+  return size(input, name, out_type)
+
+
+@tf_export(v1=["size"])
 def size(input, name=None, out_type=dtypes.int32):
   # pylint: disable=redefined-builtin
   """Returns the size of a tensor.
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 80d93f3de1..8a7cfd45b2 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -185,8 +185,14 @@ def sparse_eye(num_rows,
         dense_shape=[num_rows, num_columns])
 
 
+@tf_export("sparse.concat", "sparse_concat", v1=[])
+def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim=False,
+                     concat_dim=None, name=None):
+  return sparse_concat(axis, sp_inputs, name, expand_nonconcat_dim, concat_dim)
+
+
 # pylint: disable=protected-access
-@tf_export("sparse.concat", "sparse_concat")
+@tf_export(v1=["sparse.concat", "sparse_concat"])
 @deprecation.deprecated_endpoints("sparse_concat")
 @deprecation.deprecated_args(
     None, "concat_dim is deprecated, use axis instead", "concat_dim")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 9ce326edd2..87ce1b2328 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1058,7 +1058,7 @@ tf_module {
   }
   member_method {
     name: "shape"
-    argspec: "args=[\'input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'int32\'>\"], "
+    argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
   }
   member_method {
     name: "shape_n"
@@ -1082,7 +1082,7 @@ tf_module {
   }
   member_method {
     name: "size"
-    argspec: "args=[\'input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'int32\'>\"], "
+    argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
   }
   member_method {
     name: "slice"
@@ -1094,7 +1094,7 @@ tf_module {
   }
   member_method {
     name: "sparse_concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'concat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\'], "
   }
   member_method {
     name: "sparse_to_dense"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index 9c9c4d838e..cf63924bb1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -14,7 +14,7 @@ tf_module {
   }
   member_method {
     name: "concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'name\', \'expand_nonconcat_dim\', \'concat_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'concat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\'], "
   }
   member_method {
     name: "cross"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 82044ff43f..35592918ae 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -58,6 +58,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # pylint: disable=line-too-long
     # Add additional renames not in renames_v2.py here.
     self.symbol_renames.update({
+        "tf.sparse_concat": "tf.sparse.concat",
     })
     # pylint: enable=line-too-long
 
@@ -75,11 +76,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # IMPORTANT: order here should correspond to OLD argument order.
     # We just prepend "arg_name=" to all arguments in function calls.
     self.function_reorders = {
-        "tf.argmax": ["input", "axis", "output_type", "name"],
-        "tf.argmin": ["input", "axis", "output_type", "name"],
+        "tf.argmax": ["input", "axis", "name", "dimension", "output_type"],
+        "tf.argmin": ["input", "axis", "name", "dimension", "output_type"],
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
-        "tf.convert_to_tensor": ["value", "dtype", "preferred_dtype", "name"],
-        "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
+        "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
@@ -88,6 +88,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "depthwise_filter", "pointwise_filter", "strides",
             "padding", "data_format", "dilations", "name"
         ],
+        "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
+        "tf.shape": ["input", "name", "out_type"],
+        "tf.size": ["input", "name", "out_type"],
+        "tf.sparse.concat": [
+            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
+        ],
     }
 
     # Specially handled functions.
-- 
GitLab


From ed2c7cdf015a858bb0404826f9336dd224e28533 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Thu, 15 Nov 2018 11:50:48 -0800
Subject: [PATCH 0323/1554] Add tensorflow/python/platform/__init__.py for
 clearer error msg.

Without this init file, when trying to import tensorflow python code
from the source tree you get this error....

File "tensorflow/tensorflow/python/pywrap_tensorflow.py", line 25, in <module>
  from tensorflow.python.platform import self_check
ImportError: No module named platform

With this init file you get a much clearer error message telling you
not to do this...

File "tensorflow/tensorflow/python/pywrap_tensorflow.py", line 25, in <module>
  from tensorflow.python.platform import self_check
File "tensorflow/tensorflow/python/platform/self_check.py", line 27, in <module>
    raise ImportError("Could not import tensorflow. Do not import tensorflow "
ImportError: Could not import tensorflow. Do not import tensorflow from its source directory; change directory to outside the TensorFlow source tree, and relaunch your Python interpreter from there.
PiperOrigin-RevId: 221662931
---
 tensorflow/python/eager/test.py        | 2 +-
 tensorflow/python/platform/__init__.py | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 tensorflow/python/platform/__init__.py

diff --git a/tensorflow/python/eager/test.py b/tensorflow/python/eager/test.py
index 33ee797678..a45deac962 100644
--- a/tensorflow/python/eager/test.py
+++ b/tensorflow/python/eager/test.py
@@ -24,6 +24,6 @@ from tensorflow.python.platform.test import *  # pylint: disable=wildcard-import
 
 
 # TODO(akshayka): Do away with this file.
-def main(argv=None):
+def main(argv=None):  # pylint: disable=function-redefined
   _ops.enable_eager_execution()
   _test.main(argv)
diff --git a/tensorflow/python/platform/__init__.py b/tensorflow/python/platform/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
-- 
GitLab


From a2833694c1b2e8b26c2409dd7e40e65d4c4b8ea3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 11:54:36 -0800
Subject: [PATCH 0324/1554]   export keras.backend.tile

PiperOrigin-RevId: 221663541
---
 tensorflow/python/keras/backend.py                            | 2 +-
 tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 6174478e59..1d23e14d1d 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -2547,7 +2547,7 @@ def arange(start, stop=None, step=1, dtype='int32'):
     result = cast(result, dtype)
   return result
 
-
+@tf_export('keras.backend.tile')
 def tile(x, n):
   """Creates a tensor by tiling `x` by `n`.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
index 85e2a71ff3..8cd0c6ea5f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.backend.pbtxt
@@ -512,6 +512,10 @@ tf_module {
     name: "temporal_padding"
     argspec: "args=[\'x\', \'padding\'], varargs=None, keywords=None, defaults=[\'(1, 1)\'], "
   }
+  member_method {
+    name: "tile"
+    argspec: "args=[\'x\', \'n\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "to_dense"
     argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
index 38105c540c..d200d3d26d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.backend.pbtxt
@@ -508,6 +508,10 @@ tf_module {
     name: "temporal_padding"
     argspec: "args=[\'x\', \'padding\'], varargs=None, keywords=None, defaults=[\'(1, 1)\'], "
   }
+  member_method {
+    name: "tile"
+    argspec: "args=[\'x\', \'n\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "to_dense"
     argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From dd7dd8e16c0b564dce9893d65fb92e8b6eb79e93 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Thu, 15 Nov 2018 12:11:47 -0800
Subject: [PATCH 0325/1554] [TF:XLA][XlaCompiler]Support non-default layout in
 return type.

- Add a layout field in XlaContext RetVal object.

- Change retval_op to also return the layout of the result of shape_representation_fn.

- Resets the layout based on the layout field for return shapes in xla_compiler.

- Add tests.

PiperOrigin-RevId: 221666728
---
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 39 ++++++----
 .../compiler/tf2xla/xla_compiler_test.cc      | 77 +++++++++++++++++++
 2 files changed, 102 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index a08d030ce7..8036bc6844 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -158,7 +158,8 @@ Status BuildComputation(
     xla::XlaBuilder* builder, xla::XlaComputation* computation,
     int* num_computation_outputs, int* num_nonconst_outputs,
     std::vector<XlaCompiler::OutputDescription>* outputs,
-    std::vector<XlaCompiler::ResourceUpdate>* resource_updates) {
+    std::vector<XlaCompiler::ResourceUpdate>* resource_updates,
+    xla::Shape* output_shape) {
   // Attach a common operator name as metadata. This has no semantic effect — it
   // merely makes the HLO graph more readable when visualized via TensorBoard,
   // since TensorBoard forms groups out of operators with similar names.
@@ -176,6 +177,10 @@ Status BuildComputation(
 
   std::vector<xla::XlaOp> elems;
   elems.reserve(retvals.size());
+
+  // Keeps track of which retvals have layout to update. The first element is
+  // the output index, second element is the new layout.
+  std::vector<std::pair<int64, xla::Layout>> retval_to_update_layout;
   for (int i = 0; i < retvals.size(); ++i) {
     XlaCompiler::OutputDescription& output = (*outputs)[i];
     const XlaExpression& retval = retvals[i];
@@ -202,10 +207,12 @@ Status BuildComputation(
           TF_ASSIGN_OR_RETURN(xla::Shape shape, shape_representation_fn(
                                                     output.shape, output.type));
           value = xla::Reshape(value, xla::AsInt64Slice(shape.dimensions()));
+          retval_to_update_layout.emplace_back(elems.size(), shape.layout());
         } else if (it != retval_cores.end()) {
           // Apply the sharding to the output, if there is a core assignment.
           value = identity_op(value);
         }
+
         elems.push_back(value);
         break;
       }
@@ -297,6 +304,21 @@ Status BuildComputation(
     return computation_status.status();
   }
   *computation = computation_status.ConsumeValueOrDie();
+
+  TF_ASSIGN_OR_RETURN(const auto& program_shape,
+                      computation->GetProgramShape());
+  *output_shape = program_shape.result();
+  // Update the output layout to the layout of retval.
+  for (auto& update : retval_to_update_layout) {
+    if (!always_return_tuple && elems.size() == 1) {
+      *output_shape->mutable_layout() = update.second;
+      continue;
+    }
+
+    xla::Shape* output_sub_shape =
+        xla::ShapeUtil::GetMutableSubshape(output_shape, {update.first});
+    *output_sub_shape->mutable_layout() = update.second;
+  }
   return Status::OK();
 }
 
@@ -988,23 +1010,12 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
       options.return_updated_values_for_all_resources,
       options.always_return_tuple, &builder, result->computation.get(),
       &num_computation_outputs, &num_nonconst_outputs, &result->outputs,
-      &result->resource_updates));
+      &result->resource_updates, &result->xla_output_shape));
 
   VLOG(2) << "Outputs: total: " << context->retvals().size()
           << " nonconstant: " << num_nonconst_outputs;
-
-  // Compute the XLA output shape, if there is a computation with non-constant
-  // outputs.
-  TF_ASSIGN_OR_RETURN(std::unique_ptr<xla::ProgramShape> computation_shape,
-                      client()->GetComputationShape(*result->computation));
-
-  result->xla_output_shape.Swap(computation_shape->mutable_result());
   VLOG(2) << "XLA output shape: "
-          << xla::ShapeUtil::HumanString(result->xla_output_shape);
-
-  // Tensorflow expects a major-to-minor order of results.
-  xla::LayoutUtil::SetToDefaultLayout(&result->xla_output_shape);
-
+          << xla::ShapeUtil::HumanStringWithLayout(result->xla_output_shape);
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index aaee208f63..eba5d77efa 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/function_ops.h"
 #include "tensorflow/cc/ops/resource_variable_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/side_effect_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
@@ -910,6 +911,82 @@ TEST_F(XlaCompilerTest, Variables) {
   RunAndCheckVariablesComputation(client_, result);
 }
 
+TEST_F(XlaCompilerTest, ResultLayoutSingle) {
+  Scope scope = Scope::NewRootScope().ExitOnError();
+  auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
+  auto b = ops::_Retval(scope.WithOpName("RET"), a, 0);
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  TF_ASSERT_OK(scope.ToGraph(graph.get()));
+
+  // Builds a description of the arguments.
+  std::vector<XlaCompiler::Argument> args(1);
+  args[0].kind = XlaCompiler::Argument::kParameter;
+  args[0].type = DT_INT32;
+  args[0].shape = TensorShape({2, 3});
+
+  auto options = DefaultOptions();
+  // Sets the representation function to return a non-default layout.
+  options.shape_representation_fn =
+      [](const TensorShape& shape, DataType type) -> xla::StatusOr<xla::Shape> {
+    xla::Shape xla_shape;
+    TF_RETURN_IF_ERROR(TensorShapeToXLAShape(type, shape, &xla_shape));
+    *xla_shape.mutable_layout() = xla::LayoutUtil::MakeLayout({0, 1});
+    return xla_shape;
+  };
+
+  // Compiles the graph.
+  XlaCompiler compiler(options);
+
+  XlaCompiler::CompilationResult result;
+  auto compile_options = XlaCompiler::CompileOptions();
+  compile_options.always_return_tuple = false;
+  TF_ASSERT_OK(compiler.CompileGraph(compile_options, "id", std::move(graph),
+                                     args, &result));
+  EXPECT_TRUE(xla::ShapeUtil::Equal(
+      result.xla_output_shape,
+      xla::ShapeUtil::MakeShapeWithLayout(xla::S32, {2, 3}, {0, 1})));
+}
+
+TEST_F(XlaCompilerTest, ResultLayoutMultiple) {
+  Scope scope = Scope::NewRootScope().ExitOnError();
+  auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
+  auto b = ops::_Retval(scope.WithOpName("RET1"), a, 0);
+  auto c = ops::_Retval(scope.WithOpName("RET2"), a, 1);
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  TF_ASSERT_OK(scope.ToGraph(graph.get()));
+
+  // Builds a description of the arguments.
+  std::vector<XlaCompiler::Argument> args(1);
+  args[0].kind = XlaCompiler::Argument::kParameter;
+  args[0].type = DT_INT32;
+  args[0].shape = TensorShape({2, 3});
+
+  auto options = DefaultOptions();
+  // Sets the representation function to return a non-default layout.
+  options.shape_representation_fn =
+      [](const TensorShape& shape, DataType type) -> xla::StatusOr<xla::Shape> {
+    xla::Shape xla_shape;
+    TF_RETURN_IF_ERROR(TensorShapeToXLAShape(type, shape, &xla_shape));
+    *xla_shape.mutable_layout() = xla::LayoutUtil::MakeLayout({0, 1});
+    return xla_shape;
+  };
+
+  // Compiles the graph.
+  XlaCompiler compiler(options);
+
+  XlaCompiler::CompilationResult result;
+  TF_ASSERT_OK(compiler.CompileGraph(XlaCompiler::CompileOptions(), "id",
+                                     std::move(graph), args, &result));
+  xla::Shape result_shape =
+      xla::ShapeUtil::MakeShapeWithLayout(xla::S32, {2, 3}, {0, 1});
+
+  EXPECT_TRUE(xla::ShapeUtil::Equal(
+      result.xla_output_shape,
+      xla::ShapeUtil::MakeTupleShape({result_shape, result_shape})));
+}
+
 // Tests a simple graph that reads and writes a variable.
 TEST_F(XlaCompilerTest, ReturnResourceHandleOnly) {
   Scope scope = Scope::NewRootScope().ExitOnError();
-- 
GitLab


From 219bb5d8dc9d16fa2e9895d8d4c6adaf45ee3c34 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 15 Nov 2018 12:16:59 -0800
Subject: [PATCH 0326/1554] Use protos for dnn.h data structure
 implementations.

This is the first of a series patches that log StreamExecutor
convolution calls. This patch introduced structured (proto) logging,
suitable for serialize and potentially deserialize.

PiperOrigin-RevId: 221667516
---
 .../xla/service/gpu/cudnn_conv_runner.cc      |   8 +-
 tensorflow/stream_executor/BUILD              |  12 +
 tensorflow/stream_executor/cuda/cuda_dnn.cc   |   2 +
 tensorflow/stream_executor/dnn.cc             | 134 +++++----
 tensorflow/stream_executor/dnn.h              | 265 +++++++++---------
 tensorflow/stream_executor/dnn.proto          | 102 +++++++
 tensorflow/stream_executor/stream.cc          |   3 +-
 7 files changed, 309 insertions(+), 217 deletions(-)
 create mode 100644 tensorflow/stream_executor/dnn.proto

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
index 3df4ab96e1..3425e1b494 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
@@ -370,14 +370,12 @@ StatusOr<CudnnConvParams> GetCudnnConvParams(
       params.output_shape = &conv_result_shape;
       params.fusion.emplace();
       auto& fusion = *params.fusion;
-      if (backend_config.activation_mode() <
-          static_cast<int64>(se::dnn::ActivationMode::kNumActivationModes)) {
-        fusion.mode = static_cast<se::dnn::ActivationMode>(
-            backend_config.activation_mode());
-      } else {
+      if (!se::dnn::ActivationMode_IsValid(backend_config.activation_mode())) {
         return InternalError("Bad activation mode: %s",
                              backend_config.ShortDebugString());
       }
+      fusion.mode = static_cast<se::dnn::ActivationMode>(
+          backend_config.activation_mode());
       fusion.side_input_scale = backend_config.side_input_scale();
       params.input_buf = operand_buffers[0];
       params.filter_buf = operand_buffers[1];
diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 5c9d85acf4..2526e1adaa 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -1,6 +1,8 @@
 licenses(["restricted"])
 
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos")
 load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
 load("//tensorflow:tensorflow.bzl", "cc_header_only_library")
 
@@ -13,6 +15,14 @@ STREAM_EXECUTOR_HEADERS = glob([
     "platform/**/*.h",
 ])
 
+tf_proto_library(
+    name = "dnn_proto",
+    srcs = ["dnn.proto"],
+    cc_api_version = 2,
+    default_header = True,
+    protodeps = tf_additional_all_protos(),
+)
+
 cc_library(
     name = "stream_executor_impl",
     srcs = glob(
@@ -35,6 +45,7 @@ cc_library(
     }),
     visibility = ["//visibility:public"],
     deps = [
+        ":dnn_proto_cc_impl",
         "//tensorflow/core:lib",
         "//tensorflow/core:ptr_util",
         "@com_google_absl//absl/container:flat_hash_map",
@@ -51,6 +62,7 @@ cc_library(
     hdrs = STREAM_EXECUTOR_HEADERS,
     visibility = ["//visibility:public"],
     deps = [
+        ":dnn_proto_cc",
         "//tensorflow/core:lib",
         "//tensorflow/core:ptr_util",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index b5583cbb94..3dd5b77ddb 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -161,6 +161,8 @@ cudnnDataType_t GetCudnnDataType<int8>(dnn::DataLayout layout) {
       return CUDNN_DATA_INT8;
     case dnn::DataLayout::kBatchDepthYX4:
       return CUDNN_DATA_INT8x4;
+    default:
+      LOG(FATAL) << "Unsupported layout: " << static_cast<int32_t>(layout);
   }
 }
 
diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc
index 0448ffce41..faa662211e 100644
--- a/tensorflow/stream_executor/dnn.cc
+++ b/tensorflow/stream_executor/dnn.cc
@@ -23,7 +23,7 @@ namespace stream_executor {
 namespace dnn {
 
 uint64 AlgorithmDesc::hash() const {
-  return ::tensorflow::Hash64Combine(algo_, tensor_ops_enabled_);
+  return ::tensorflow::Hash64Combine(algo_id(), tensor_ops_enabled());
 }
 
 bool DnnSupport::GetConvolveAlgorithms(
@@ -187,6 +187,9 @@ std::tuple<int, int, int> GetDimIndices(const DataLayout& layout,
       batch_idx = 0;
       spatial_idx = 2;
       break;
+
+    default:
+      LOG(FATAL) << "Unknown layout " << layout;
   }
 
   return std::make_tuple(depth_idx, batch_idx, spatial_idx);
@@ -233,28 +236,27 @@ string AlgorithmConfig::ToString() const {
 // -- BatchDescriptor
 
 BatchDescriptor::BatchDescriptor(int ndims)
-    : count_(0),
-      feature_map_count_(0),
-      spatial_size_(ndims, 0),
-      value_max_(0.0),
+    : value_max_(0.0),
       value_min_(0.0),
-      layout_(DataLayout::kYXDepthBatch),
-      ndims_(ndims),
-      quantized_activation_mode_(QuantizedActivationMode::k8Bit) {}
+      quantized_activation_mode_(QuantizedActivationMode::k8Bit) {
+  tensor_.mutable_dimensions()->Resize(ndims + 2, 0);
+  set_layout(DataLayout::kYXDepthBatch);
+}
 
 BatchDescriptor::BatchDescriptor() : BatchDescriptor(/*ndims=*/2) {}
 
 std::vector<int64> BatchDescriptor::full_dims(const DataLayout& layout) const {
-  std::vector<int64> bdyx_dims(ndims_ + 2);
+  std::vector<int64> bdyx_dims(ndims() + 2);
   bdyx_dims[0] = count();
   bdyx_dims[1] = feature_map_count();
-  std::copy(spatial_size_.begin(), spatial_size_.end(), bdyx_dims.begin() + 2);
+  std::copy(spatial_size().begin(), spatial_size().end(),
+            bdyx_dims.begin() + 2);
   return ReorderDims(bdyx_dims, DataLayout::kBatchDepthYX, layout);
 }
 
 std::vector<int64> BatchDescriptor::full_strides(
     const DataLayout& layout) const {
-  if (layout_ == DataLayout::kBatchDepthYX4) {
+  if (this->layout() == DataLayout::kBatchDepthYX4) {
     LOG(FATAL)
         << "Cannot compute full strides for batch descriptor " << ToString()
         << ", because its layout is kBatchDepthYX4. In fact, "
@@ -262,36 +264,32 @@ std::vector<int64> BatchDescriptor::full_strides(
            "Use cudnnSetTensor4DDescriptor to set cudnnTensorDescriptor_t "
            "instead.";
   }
-  std::vector<int64> phys_dims = full_dims(layout_);
+  std::vector<int64> phys_dims = full_dims(this->layout());
   std::vector<int64> phys_strides(phys_dims.size());
-  phys_strides[ndims_ + 1] = 1;
-  for (int i = ndims_; i >= 0; i--) {
+  phys_strides[ndims() + 1] = 1;
+  for (int i = ndims(); i >= 0; i--) {
     phys_strides[i] = phys_strides[i + 1] * phys_dims[i + 1];
   }
-  return ReorderDims(phys_strides, layout_, layout);
+  return ReorderDims(phys_strides, this->layout(), layout);
 }
 
 void BatchDescriptor::CloneFrom(const BatchDescriptor& other) {
-  count_ = other.count_;
-  feature_map_count_ = other.feature_map_count_;
-  spatial_size_ = other.spatial_size_;
+  tensor_ = other.tensor_;
   value_max_ = other.value_max_;
   value_min_ = other.value_min_;
-  layout_ = other.layout_;
-  ndims_ = other.ndims_;
   quantized_activation_mode_ = other.quantized_activation_mode_;
 }
 
 string BatchDescriptor::ToString() const {
   string spatial;
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&spatial, "%lld ", spatial_size_[i]);
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&spatial, "%lld ", spatial_size()[i]);
   }
   return port::Printf(
       "{count: %lld feature_map_count: %lld spatial: %s "
       "value_min: %f value_max: %f layout: %s}",
-      count_, feature_map_count_, spatial.c_str(), value_min_, value_max_,
-      DataLayoutString(layout_).c_str());
+      count(), feature_map_count(), spatial.c_str(), value_min_, value_max_,
+      DataLayoutString(layout()).c_str());
 }
 
 string BatchDescriptor::ToShortString() const {
@@ -302,8 +300,8 @@ string BatchDescriptor::ToShortString() const {
   string batch = absl::StrCat("b", count());
 
   string spatial = "s";
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&spatial, "%lld ", spatial_size_[i]);
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&spatial, "%lld ", spatial_size()[i]);
   }
 
   string suffix;
@@ -333,18 +331,18 @@ string BatchDescriptor::ToShortString() const {
 
 int64 BatchDescriptor::NodesPerFeatureMap() const {
   int64 ret = 1;
-  for (int i = 0; i < ndims_; i++) {
-    ret *= spatial_size_[i];
+  for (int i = 0; i < ndims(); i++) {
+    ret *= spatial_size()[i];
   }
   return ret;
 }
 
 int64 BatchDescriptor::NodesAcrossFeatureMaps() const {
-  return NodesPerFeatureMap() * feature_map_count_;
+  return NodesPerFeatureMap() * feature_map_count();
 }
 
 int64 BatchDescriptor::ElementCount() const {
-  return count_ * feature_map_count_ * NodesPerFeatureMap();
+  return count() * feature_map_count() * NodesPerFeatureMap();
 }
 
 int64 BatchDescriptor::FullyConnectedWeightCount(
@@ -372,33 +370,27 @@ BatchDescriptor BatchDescriptor::DepthConcatenateOutputDescriptor(
 
 // -- FilterDescriptor
 
-FilterDescriptor::FilterDescriptor(int ndims)
-    : output_feature_map_count_(0),
-      input_feature_map_count_(0),
-      input_filter_dims_(ndims, 0),
-      ndims_(ndims),
-      layout_(FilterLayout::kOutputInputYX) {}
+FilterDescriptor::FilterDescriptor(int ndims) {
+  tensor_.mutable_dimensions()->Resize(ndims + 2, 0);
+  set_layout(FilterLayout::kOutputInputYX);
+}
 
 FilterDescriptor::FilterDescriptor() : FilterDescriptor(/*ndims=*/2) {}
 
 FilterDescriptor::~FilterDescriptor() {}
 
 void FilterDescriptor::CloneFrom(const FilterDescriptor& other) {
-  set_output_feature_map_count(other.output_feature_map_count())
-      .set_input_feature_map_count(other.input_feature_map_count())
-      .set_layout(other.layout());
-  input_filter_dims_ = other.input_filter_dims_;
-  ndims_ = other.ndims_;
+  tensor_ = other.tensor_;
 }
 
 string FilterDescriptor::ToString() const {
   string desc = port::Printf(
       "{output_feature_map_count: %lld input_feature_map_count: %lld "
       "layout: %s shape: ",
-      output_feature_map_count_, input_feature_map_count_,
-      FilterLayoutString(layout_).c_str());
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&desc, "%lld ", input_filter_dims_[i]);
+      output_feature_map_count(), input_feature_map_count(),
+      FilterLayoutString(layout()).c_str());
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&desc, "%lld ", input_filter_dims()[i]);
   }
   absl::StrAppend(&desc, "}");
 
@@ -409,15 +401,15 @@ string FilterDescriptor::ToShortString() const {
   // All the constituent strings are less than 15 characters, so the
   // small string optimization ensures that there will be at most one
   // heap memory allocation.
-  string od = absl::StrCat("od", output_feature_map_count_);
-  string id = absl::StrCat("id", input_feature_map_count_);
+  string od = absl::StrCat("od", output_feature_map_count());
+  string id = absl::StrCat("id", input_feature_map_count());
 
   string spatial = "s";
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&spatial, "%lld ", input_filter_dims_[i]);
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&spatial, "%lld ", input_filter_dims()[i]);
   }
 
-  switch (layout_) {
+  switch (layout()) {
     case FilterLayout::kOutputInputYX:
       return absl::StrCat(od, id, spatial);
     case FilterLayout::kOutputYXInput:
@@ -429,28 +421,28 @@ string FilterDescriptor::ToShortString() const {
     case FilterLayout::kYXInputOutput:
       return absl::StrCat(spatial, id, od);
     default:
-      LOG(FATAL) << "Unknown layout " << static_cast<int32>(layout_);
+      LOG(FATAL) << "Unknown layout " << static_cast<int32>(layout());
       return "";  // Avoid return warning (unreachable)
   }
 }
 
 int64 FilterDescriptor::ComputeWeightCount() const {
-  int64 ret = output_feature_map_count_ * input_feature_map_count_;
-  for (int i = 0; i < ndims_; i++) {
-    ret *= input_filter_dims_[i];
+  int64 ret = output_feature_map_count() * input_feature_map_count();
+  for (int i = 0; i < ndims(); i++) {
+    ret *= input_filter_dims()[i];
   }
   return ret;
 }
 
 // -- ConvolutionDescriptor
 
-ConvolutionDescriptor::ConvolutionDescriptor(int ndims)
-    : zero_padding_(ndims, 0),
-      filter_strides_(ndims, 1),
-      dilation_rates_(ndims, 1),
-      group_count_(1),
-      ndims_(ndims),
-      convolution_not_crosscorr_(false) {}
+ConvolutionDescriptor::ConvolutionDescriptor(int ndims) {
+  proto_.mutable_paddings()->Resize(ndims, 0);
+  proto_.mutable_strides()->Resize(ndims, 1);
+  proto_.mutable_dilations()->Resize(ndims, 1);
+  proto_.set_group_count(1);
+  proto_.set_convolution_mode(ConvolutionMode::CROSS_CORRELATION);
+}
 
 ConvolutionDescriptor::ConvolutionDescriptor()
     : ConvolutionDescriptor(/*ndims=*/2) {}
@@ -461,10 +453,10 @@ string ConvolutionDescriptor::ToString() const {
   string padding;
   string strides;
   string dilations;
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&padding, "%lld ", zero_padding_[i]);
-    port::Appendf(&strides, "%lld ", filter_strides_[i]);
-    port::Appendf(&dilations, "%lld ", dilation_rates_[i]);
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&padding, "%lld ", this->padding()[i]);
+    port::Appendf(&strides, "%lld ", this->strides()[i]);
+    port::Appendf(&dilations, "%lld ", this->dilations()[i]);
   }
 
   return port::Printf(
@@ -476,15 +468,15 @@ string ConvolutionDescriptor::ToString() const {
 
 string ConvolutionDescriptor::ToShortString() const {
   string desc;
-  for (int i = 0; i < ndims_; i++) {
+  for (int i = 0; i < ndims(); i++) {
     if (i > 0) port::Appendf(&desc, "_");
-    port::Appendf(&desc, "p%d:%lld", i, zero_padding_[i]);
+    port::Appendf(&desc, "p%d:%lld", i, padding()[i]);
   }
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&desc, "_s%d:%lld", i, filter_strides_[i]);
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&desc, "_s%d:%lld", i, strides()[i]);
   }
-  for (int i = 0; i < ndims_; i++) {
-    port::Appendf(&desc, "_d%d:%lld", i, dilation_rates_[i]);
+  for (int i = 0; i < ndims(); i++) {
+    port::Appendf(&desc, "_d%d:%lld", i, dilations()[i]);
   }
   return desc;
 }
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index f4270c8575..ab88345873 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -29,7 +29,9 @@ limitations under the License.
 
 #include "absl/types/optional.h"
 #include "absl/types/span.h"
+#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/stream_executor/device_memory.h"
+#include "tensorflow/stream_executor/dnn.pb.h"
 #include "tensorflow/stream_executor/lib/array_slice.h"
 #include "tensorflow/stream_executor/lib/status.h"
 #include "tensorflow/stream_executor/lib/statusor.h"
@@ -48,19 +50,6 @@ class ScratchAllocator;
 
 namespace dnn {
 
-// Describes how an input or output layer's data is formatted.
-// Specify int64 so there's no padding in BatchDescriptor.
-enum class DataLayout : int64 {
-  kYXDepthBatch = 0,  // Same as dist_belief::DF_DEPTH_MAJOR.
-  kYXBatchDepth,      // Same as dist_belief::DF_BATCH_MAJOR.
-  kBatchYXDepth,      // Same as run_brain output, and tensorflow's layout.
-  kBatchDepthYX,      // cuDNN's NCHW layout, data laid out as image, feature
-                      // maps, rows, columns.
-  kBatchDepthYX4,     // cuDNN's NCHW_VECT_C layout, data laid out the same as
-                      // kBatchDepthYX but each element is a vector of 4 feature
-                      // maps.
-};
-
 // Specifies an index to use when accessing specific spatial dimensions.
 enum class DimIndex : int {
   X = 0,
@@ -73,8 +62,27 @@ inline int64 GetDim(absl::Span<const int64> data, DimIndex dim) {
   return data.rbegin()[static_cast<int64>(dim)];
 }
 
+inline void SetDim(absl::Span<int64> data, DimIndex dim, int64 value) {
+  data.rbegin()[static_cast<int64>(dim)] = value;
+}
+
 inline void SetDim(std::vector<int64>* data, DimIndex dim, int64 value) {
-  data->rbegin()[static_cast<int64>(dim)] = value;
+  return SetDim(absl::MakeSpan(*data), dim, value);
+}
+
+// tensorflow::int64 is not the same type as tensorflow::protobuf_int64 in
+// open-source. Wrapper function that gives an int64 array slice view of a
+// repeated int64 protobuf field.
+inline absl::Span<const int64> AsInt64Slice(
+    const tensorflow::protobuf::RepeatedField<tensorflow::protobuf_int64>& v) {
+  return absl::Span<const int64>(reinterpret_cast<const int64*>(v.data()),
+                                 v.size());
+}
+
+inline absl::Span<int64> AsInt64Slice(
+    tensorflow::protobuf::RepeatedField<tensorflow::protobuf_int64>* v) {
+  return absl::Span<int64>(reinterpret_cast<int64*>(v->mutable_data()),
+                           v->size());
 }
 
 // Returns a string representation of the given data layout.
@@ -87,14 +95,6 @@ enum class QuantizedActivationMode {
   k32Bit = 4,
 };
 
-// Specifies the data type used by an operation.
-enum class DataType {
-  kFloat = 0,
-  kDouble = 1,
-  kHalf = 2,
-  kInt8 = 3,
-};
-
 // A helper class to convert C/C++ types to the proper enums.
 template <typename T>
 struct ToDataType;
@@ -245,15 +245,15 @@ class BatchDescriptor {
   string ToShortString() const;
 
   // Accessors.
-  int64 count() const { return count_; }
-  int64 feature_map_count() const { return feature_map_count_; }
-  int64 height() const { return GetDim(spatial_size_, DimIndex::Y); }
-  int64 width() const { return GetDim(spatial_size_, DimIndex::X); }
-  int64 spatial_dim(DimIndex dim) const { return GetDim(spatial_size_, dim); }
-  int ndims() const { return ndims_; }
+  int64 count() const { return tensor_.dimensions(0); }
+  int64 feature_map_count() const { return tensor_.dimensions(1); }
+  int64 height() const { return GetDim(spatial_size(), DimIndex::Y); }
+  int64 width() const { return GetDim(spatial_size(), DimIndex::X); }
+  int64 spatial_dim(DimIndex dim) const { return GetDim(spatial_size(), dim); }
+  int ndims() const { return spatial_size().size(); }
   float value_max() const { return value_max_; }
   float value_min() const { return value_min_; }
-  DataLayout layout() const { return layout_; }
+  DataLayout layout() const { return tensor_.data_layout(); }
   QuantizedActivationMode quantized_activation_mode() const {
     return quantized_activation_mode_;
   }
@@ -267,23 +267,23 @@ class BatchDescriptor {
 
   // Named-argument helpers for avoiding user error during construction.
   BatchDescriptor& set_count(int64 value) {
-    count_ = value;
+    tensor_.set_dimensions(0, value);
     return *this;
   }
   BatchDescriptor& set_feature_map_count(int64 value) {
-    feature_map_count_ = value;
+    tensor_.set_dimensions(1, value);
     return *this;
   }
   BatchDescriptor& set_height(int64 value) {
-    SetDim(&spatial_size_, DimIndex::Y, value);
+    SetDim(spatial_size(), DimIndex::Y, value);
     return *this;
   }
   BatchDescriptor& set_width(int64 value) {
-    SetDim(&spatial_size_, DimIndex::X, value);
+    SetDim(spatial_size(), DimIndex::X, value);
     return *this;
   }
   BatchDescriptor& set_spatial_dim(DimIndex dim, int64 value) {
-    SetDim(&spatial_size_, dim, value);
+    SetDim(spatial_size(), dim, value);
     return *this;
   }
   BatchDescriptor& set_value_max(float value) {
@@ -295,7 +295,7 @@ class BatchDescriptor {
     return *this;
   }
   BatchDescriptor& set_layout(DataLayout layout) {
-    layout_ = layout;
+    tensor_.set_data_layout(layout);
     return *this;
   }
   BatchDescriptor& set_quantized_activation_mode(
@@ -334,31 +334,20 @@ class BatchDescriptor {
       port::ArraySlice<dnn::BatchDescriptor> inputs);
 
  private:
-  int64 count_;
-  int64 feature_map_count_;
-  // Stored as: ..., y, x.
-  std::vector<int64> spatial_size_;
+  absl::Span<const int64> spatial_size() const {
+    return AsInt64Slice(tensor_.dimensions()).subspan(2);
+  }
+
+  absl::Span<int64> spatial_size() {
+    return AsInt64Slice(tensor_.mutable_dimensions()).subspan(2);
+  }
+
+  TensorDescriptorProto tensor_;
   float value_max_;
   float value_min_;
-  DataLayout layout_;
-  int ndims_;
   QuantizedActivationMode quantized_activation_mode_;
 };
 
-// Describes how a filter is laid out in the memory.
-// Specify int64 so there's no padding in FilterDescriptor.
-enum class FilterLayout : int64 {
-  kOutputInputYX = 0,  // cuDNN's default filter layout, laid out as:
-                       // (major) output feature maps >> input feature maps >>
-                       // rows >> columns (minor).
-  kOutputYXInput,      // major to minor:
-                       //   (output features, row, columns, input features)
-  kOutputInputYX4,  // laid out the same as kOutputInputYX but each element is a
-                    // vector of 4 feature maps.
-  kInputYXOutput,   // Same as dist_belief's default filter layout.
-  kYXInputOutput,   // Same as tensorflow's default filter layout.
-};
-
 // Returns a string representation of the given filter layout.
 string FilterLayoutString(FilterLayout layout);
 
@@ -398,30 +387,30 @@ class FilterDescriptor {
 
   // Named-argument helpers for avoiding user error during construction.
   FilterDescriptor& set_output_feature_map_count(int64 value) {
-    output_feature_map_count_ = value;
+    tensor_.set_dimensions(0, value);
     return *this;
   }
   FilterDescriptor& set_input_feature_map_count(int64 value) {
-    input_feature_map_count_ = value;
+    tensor_.set_dimensions(1, value);
     return *this;
   }
   FilterDescriptor& set_input_filter_height(int64 value) {
-    SetDim(&input_filter_dims_, DimIndex::Y, value);
+    SetDim(input_filter_dims(), DimIndex::Y, value);
     return *this;
   }
   FilterDescriptor& set_input_filter_width(int64 value) {
-    SetDim(&input_filter_dims_, DimIndex::X, value);
+    SetDim(input_filter_dims(), DimIndex::X, value);
     return *this;
   }
   FilterDescriptor& set_layout(FilterLayout layout) {
-    layout_ = layout;
+    tensor_.set_filter_layout(layout);
     return *this;
   }
   FilterDescriptor& set_spatial_dim(DimIndex dim, int64 value) {
-    SetDim(&input_filter_dims_, dim, value);
+    SetDim(input_filter_dims(), dim, value);
     return *this;
   }
-  int ndims() const { return ndims_; }
+  int ndims() const { return input_filter_dims().size(); }
 
   void CloneFrom(const FilterDescriptor& other);
 
@@ -434,32 +423,32 @@ class FilterDescriptor {
 
   // Returns the number of biases required as parameters for a convolution
   // using this filter descriptor.
-  int64 bias_count() const { return output_feature_map_count_; }
+  int64 bias_count() const { return output_feature_map_count(); }
 
-  int64 output_feature_map_count() const { return output_feature_map_count_; }
-  int64 input_feature_map_count() const { return input_feature_map_count_; }
+  int64 output_feature_map_count() const { return tensor_.dimensions(0); }
+  int64 input_feature_map_count() const { return tensor_.dimensions(1); }
   int64 input_filter_height() const {
-    return GetDim(input_filter_dims_, DimIndex::Y);
+    return GetDim(input_filter_dims(), DimIndex::Y);
   }
   int64 input_filter_width() const {
-    return GetDim(input_filter_dims_, DimIndex::X);
+    return GetDim(input_filter_dims(), DimIndex::X);
   }
   int64 input_filter_dim(DimIndex dim) const {
-    return GetDim(input_filter_dims_, dim);
+    return GetDim(input_filter_dims(), dim);
   }
 
-  FilterLayout layout() const { return layout_; }
+  FilterLayout layout() const { return tensor_.filter_layout(); }
+
   absl::Span<const int64> input_filter_dims() const {
-    return input_filter_dims_;
+    return AsInt64Slice(tensor_.dimensions()).subspan(2);
   }
 
  private:
-  int64 output_feature_map_count_;
-  int64 input_feature_map_count_;
-  // Stored as: ..., y, x.
-  std::vector<int64> input_filter_dims_;
-  int ndims_;
-  FilterLayout layout_;
+  absl::Span<int64> input_filter_dims() {
+    return AsInt64Slice(tensor_.mutable_dimensions()).subspan(2);
+  }
+
+  TensorDescriptorProto tensor_;
 };
 
 // Describes how padding should be aligned when the total number of pad
@@ -518,90 +507,102 @@ class ConvolutionDescriptor {
   string ToShortString() const;
 
   ConvolutionDescriptor& set_zero_padding_height(int64 value) {
-    SetDim(&zero_padding_, DimIndex::Y, value);
+    SetDim(padding(), DimIndex::Y, value);
     return *this;
   }
   ConvolutionDescriptor& set_zero_padding_width(int64 value) {
-    SetDim(&zero_padding_, DimIndex::X, value);
+    SetDim(padding(), DimIndex::X, value);
     return *this;
   }
   ConvolutionDescriptor& set_zero_padding(DimIndex dim, int64 value) {
-    SetDim(&zero_padding_, dim, value);
+    SetDim(padding(), dim, value);
     return *this;
   }
   ConvolutionDescriptor& set_vertical_filter_stride(int64 value) {
-    SetDim(&filter_strides_, DimIndex::Y, value);
+    SetDim(strides(), DimIndex::Y, value);
     return *this;
   }
   ConvolutionDescriptor& set_horizontal_filter_stride(int64 value) {
-    SetDim(&filter_strides_, DimIndex::X, value);
+    SetDim(strides(), DimIndex::X, value);
     return *this;
   }
   ConvolutionDescriptor& set_filter_stride(DimIndex dim, int64 value) {
-    SetDim(&filter_strides_, dim, value);
+    SetDim(strides(), dim, value);
     return *this;
   }
   ConvolutionDescriptor& set_vertical_dilation_rate(int64 value) {
-    SetDim(&dilation_rates_, DimIndex::Y, value);
+    SetDim(dilations(), DimIndex::Y, value);
     return *this;
   }
   ConvolutionDescriptor& set_horizontal_dilation_rate(int64 value) {
-    SetDim(&dilation_rates_, DimIndex::X, value);
+    SetDim(dilations(), DimIndex::X, value);
     return *this;
   }
   ConvolutionDescriptor& set_dilation_rate(DimIndex dim, int64 value) {
-    SetDim(&dilation_rates_, dim, value);
+    SetDim(dilations(), dim, value);
     return *this;
   }
   ConvolutionDescriptor& set_group_count(int group_count) {
-    group_count_ = group_count;
+    proto_.set_group_count(group_count);
     return *this;
   }
   ConvolutionDescriptor& set_convolution_not_crosscorr(bool conv) {
-    convolution_not_crosscorr_ = conv;
+    proto_.set_convolution_mode(conv ? ConvolutionMode::CONVOLUTION
+                                     : ConvolutionMode::CROSS_CORRELATION);
     return *this;
   }
-  int64 zero_padding_height() const {
-    return GetDim(zero_padding_, DimIndex::Y);
-  }
-  int64 zero_padding_width() const {
-    return GetDim(zero_padding_, DimIndex::X);
-  }
+  int64 zero_padding_height() const { return GetDim(padding(), DimIndex::Y); }
+  int64 zero_padding_width() const { return GetDim(padding(), DimIndex::X); }
   int64 vertical_filter_stride() const {
-    return GetDim(filter_strides_, DimIndex::Y);
+    return GetDim(strides(), DimIndex::Y);
   }
   int64 horizontal_filter_stride() const {
-    return GetDim(filter_strides_, DimIndex::X);
+    return GetDim(strides(), DimIndex::X);
   }
   int64 vertical_dilation_rate() const {
-    return GetDim(dilation_rates_, DimIndex::Y);
+    return GetDim(dilations(), DimIndex::Y);
   }
   int64 horizontal_dilation_rate() const {
-    return GetDim(dilation_rates_, DimIndex::X);
+    return GetDim(dilations(), DimIndex::X);
   }
 
-  int zero_padding(DimIndex dim) const { return GetDim(zero_padding_, dim); }
-  int filter_stride(DimIndex dim) const { return GetDim(filter_strides_, dim); }
-  int dilation_rate(DimIndex dim) const { return GetDim(dilation_rates_, dim); }
+  int zero_padding(DimIndex dim) const { return GetDim(padding(), dim); }
+  int filter_stride(DimIndex dim) const { return GetDim(strides(), dim); }
+  int dilation_rate(DimIndex dim) const { return GetDim(dilations(), dim); }
   // TODO(timshen): remove this function. No users of this class is setting a
   // non-default pad alignment.
   PadAlignment pad_alignment() const { return PadAlignment::kDefault; }
-  int group_count() const { return group_count_; }
-  int ndims() const { return ndims_; }
-  bool convolution_not_crosscorr() const { return convolution_not_crosscorr_; }
+  int group_count() const { return proto_.group_count(); }
+  int ndims() const { return padding().size(); }
+  bool convolution_not_crosscorr() const {
+    return proto_.convolution_mode() == ConvolutionMode::CONVOLUTION;
+  }
+
+  absl::Span<const int64> strides() const {
+    return AsInt64Slice(proto_.strides());
+  }
+
+  absl::Span<const int64> dilations() const {
+    return AsInt64Slice(proto_.dilations());
+  }
 
-  absl::Span<const int64> strides() const { return filter_strides_; }
-  absl::Span<const int64> dilations() const { return dilation_rates_; }
-  absl::Span<const int64> padding() const { return zero_padding_; }
+  absl::Span<const int64> padding() const {
+    return AsInt64Slice(proto_.paddings());
+  }
 
  private:
-  // Stored as: .. y, x.
-  std::vector<int64> zero_padding_;
-  std::vector<int64> filter_strides_;
-  std::vector<int64> dilation_rates_;
-  int group_count_;
-  int ndims_;
-  bool convolution_not_crosscorr_;
+  absl::Span<int64> strides() { return AsInt64Slice(proto_.mutable_strides()); }
+
+  absl::Span<int64> dilations() {
+    return AsInt64Slice(proto_.mutable_dilations());
+  }
+
+  absl::Span<int64> padding() {
+    return AsInt64Slice(proto_.mutable_paddings());
+  }
+
+  ConvolutionDescriptorProto proto_;
+
   // TODO(leary) cudnn provides these fields, but need to characterize what
   // their effect is -- they may be boolean rather than integral.
   // int64 upscale_input_x;
@@ -725,21 +726,23 @@ class PoolingDescriptor {
 class AlgorithmDesc {
  public:
   typedef int64 Index;
-  AlgorithmDesc(Index a, bool use_tensor_ops)
-      : algo_(a), tensor_ops_enabled_(use_tensor_ops) {
-    DCHECK_NE(a, -1);
+  AlgorithmDesc(Index a, bool use_tensor_ops) {
+    proto_.set_algo_id(a);
+    proto_.set_math_type(use_tensor_ops ? AlgorithmProto::TENSOR_OP_MATH
+                                        : AlgorithmProto::DEFAULT_MATH);
+  }
+  bool tensor_ops_enabled() const {
+    return proto_.math_type() == AlgorithmProto::TENSOR_OP_MATH;
   }
-  bool tensor_ops_enabled() const { return tensor_ops_enabled_; }
-  Index algo_id() const { return algo_; }
+  Index algo_id() const { return proto_.algo_id(); }
   bool operator==(const AlgorithmDesc& other) const {
-    return this->algo_ == other.algo_ &&
-           this->tensor_ops_enabled_ == other.tensor_ops_enabled_;
+    return algo_id() == other.algo_id() &&
+           tensor_ops_enabled() == other.tensor_ops_enabled();
   }
   uint64 hash() const;
 
  private:
-  Index algo_;
-  bool tensor_ops_enabled_;
+  AlgorithmProto proto_;
 };
 
 // Describes the result from a perf experiment.
@@ -883,24 +886,6 @@ class NormalizeDescriptor {
   int32 segment_size_;
 };
 
-// Describes a kind of non-linearity (threshold-like mathematical function).
-enum class ActivationMode {
-  kNone = 0,
-  kSigmoid,
-  // Rectified linear activation: f(x) = x < 0 ? 0 : x
-  kRelu,
-  // Rectified linear activation, where upper maximum is 6.0.
-  kRelu6,
-  // Rectified linear activation, where upper maximum specified by
-  // BatchDescriptor::value_max().
-  kReluX,
-  kTanh,
-  // Like ReluX, but passes all values in the range [-X,X].
-  kBandPass,
-
-  kNumActivationModes,  // Always in the end.
-};
-
 // Returns a string representation of the given activation mode.
 string ActivationModeString(ActivationMode mode);
 
diff --git a/tensorflow/stream_executor/dnn.proto b/tensorflow/stream_executor/dnn.proto
new file mode 100644
index 0000000000..91943cc8e0
--- /dev/null
+++ b/tensorflow/stream_executor/dnn.proto
@@ -0,0 +1,102 @@
+// LINT: LEGACY_NAMES
+syntax = "proto3";
+
+package stream_executor.dnn;
+
+// Specifies the data type used by an operation.
+enum DataType {
+  kFloat = 0;
+  kDouble = 1;
+  kHalf = 2;
+  kInt8 = 3;
+}
+
+// Describes how a convolution input or output layer's data is formatted.
+enum DataLayout {
+  // Naming convention:
+  // Y <-> row or height
+  // X <-> column or width
+  // Batch <-> batch, or N
+  // Depth <-> feature, or channel
+  // TODO(timshen): turn them into cuDNN names, e.g. kNCHW.
+  kYXDepthBatch = 0;
+  kYXBatchDepth = 1;
+  kBatchYXDepth = 2;   // cuDNN's NHWC layout
+  kBatchDepthYX = 3;   // cuDNN's NCHW layout
+  kBatchDepthYX4 = 4;  // cuDNN's NCHW_VECT_C layout
+}
+
+// Describes how a convolution filter is laid out in the memory.
+enum FilterLayout {
+  // Naming convention:
+  // Y <-> row or height
+  // X <-> column or width
+  // Output <-> output feature, or N
+  // Input <-> input feature, or N
+  // TODO(timshen): turn them into cuDNN names, e.g. kNCHW.
+  kOutputInputYX = 0;   // cuDNN's NCHW layout
+  kOutputYXInput = 1;   // cuDNN's NHWC layout
+  kOutputInputYX4 = 2;  // cuDNN's NCHW_VECT_C layout
+  kInputYXOutput = 3;
+  kYXInputOutput = 4;
+}
+
+// Describes a kind of non-linearity (threshold-like mathematical function).
+enum ActivationMode {
+  kNone = 0;
+  kSigmoid = 1;
+  // Rectified linear activation: f(x) = x < 0 ? 0 : x
+  kRelu = 2;
+  // Rectified linear activation; where upper maximum is 6.0.
+  kRelu6 = 3;
+  // Rectified linear activation; where upper maximum specified by
+  // BatchDescriptor::value_max().
+  kReluX = 4;
+  kTanh = 5;
+  // Like ReluX; but passes all values in the range [-X,X].
+  kBandPass = 6;
+}
+
+// Describe the math definition for the conv op. The popular behavior is
+// actually called cross-correlation in math, despite the operation is often
+// referred as convolution. See cuDNN cudnnConvolutionMode_t.
+enum ConvolutionMode {
+  CROSS_CORRELATION = 0;
+  CONVOLUTION = 1;
+}
+
+// Generic tensor representation.
+message TensorDescriptorProto {
+  repeated int64 dimensions = 1;
+  DataType data_type = 2;
+  oneof layout_oneof {
+    DataLayout data_layout = 3;
+    FilterLayout filter_layout = 4;
+  }
+}
+
+// Generic algorithm representation.
+message AlgorithmProto {
+  enum MathType {
+    DEFAULT_MATH = 0;
+    // The GPU may operate 4x4 matrix FMA.
+    // See cuDNN's documentation for CUDNN_TENSOR_OP_MATH.
+    TENSOR_OP_MATH = 1;
+  }
+  int64 algo_id = 1;
+  MathType math_type = 2;
+}
+
+// Convolution-specific parameters.
+message ConvolutionDescriptorProto {
+  repeated int64 paddings = 1;
+  repeated int64 strides = 2;
+  repeated int64 dilations = 3;
+  // The "accumulator" type. For example, use F32 as an accumulator for F16
+  // convolutions.
+  // See cuDNN's cudnnConvolutionMode_t.
+  DataType compute_mode = 4;
+  // See cuDNN's group count.
+  int32 group_count = 5;
+  ConvolutionMode convolution_mode = 6;
+}
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 5421e4f4a5..4dc7c59921 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -191,8 +191,9 @@ string ToVlogString(dnn::DataType data_type) {
       return "dnn::DataType::kHalf";
     case dnn::DataType::kInt8:
       return "dnn::DataType::kInt8";
+    default:
+      return "unknown DataType";
   }
-  return "unknown DataType";
 }
 
 // Used together with PARAM to VLOG calls made to the stream. Intended
-- 
GitLab


From c6f2390654dd3cbbdacb7c4e22d2ec716b63fcc2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 12:25:51 -0800
Subject: [PATCH 0327/1554] Allow real data to be a list or dict.

PiperOrigin-RevId: 221668879
---
 tensorflow/contrib/gan/python/train.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index 7ee39f304a..73185c79fc 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -114,7 +114,7 @@ def gan_model(
     discriminator_gen_outputs = discriminator_fn(generated_data,
                                                  generator_inputs)
   with variable_scope.variable_scope(dis_scope, reuse=True):
-    real_data = ops.convert_to_tensor(real_data)
+    real_data = _convert_tensor_or_l_or_d(real_data)
     discriminator_real_outputs = discriminator_fn(real_data, generator_inputs)
 
   if check_shapes:
-- 
GitLab


From 5c85435208629b50f7dc38e124611939befc9ed8 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 15 Nov 2018 12:26:15 -0800
Subject: [PATCH 0328/1554] Add LeakyRelu to the op schema

PiperOrigin-RevId: 221668946
---
 tensorflow/lite/builtin_ops.h                 |   1 +
 tensorflow/lite/c/builtin_op_data.h           |   4 +
 .../lite/core/api/flatbuffer_conversions.cc   |  10 ++
 .../writer/option_writer_generator.cc         |   1 +
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   6 +
 tensorflow/lite/schema/schema_generated.h     | 141 +++++++++++++++++-
 7 files changed, 158 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index b8c05f57bb..89f5aac922 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -123,6 +123,7 @@ typedef enum {
   kTfLiteBuiltinFloorMod = 95,
   kTfLiteBuiltinRange = 96,
   kTfLiteBuiltinResizeNearestNeighbor = 97,
+  kTfLiteBuiltinLeakyRelu = 98,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h
index 855983d60d..5a2f1fa4b1 100644
--- a/tensorflow/lite/c/builtin_op_data.h
+++ b/tensorflow/lite/c/builtin_op_data.h
@@ -328,6 +328,10 @@ typedef struct {
   int axis;
 } TfLiteUnpackParams;
 
+typedef struct {
+  float alpha;
+} TfLiteLeakyReluParams;
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif  // __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 8cd3faabb7..e63a3ec5d6 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -618,6 +618,16 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       break;
     }
 
+    case BuiltinOperator_LEAKY_RELU: {
+      TfLiteLeakyReluParams* params =
+          allocator->AllocatePOD<TfLiteLeakyReluParams>();
+      if (auto* leaky_relu_params = op->builtin_options_as_LeakyReluOptions()) {
+        params->alpha = leaky_relu_params->alpha();
+      }
+      *builtin_data = reinterpret_cast<void*>(params);
+      break;
+    }
+
     // Below are the ops with no builtin_data strcture.
     case BuiltinOperator_BATCH_TO_SPACE_ND:
     // TODO(aselle): Implement call in BuiltinOptions, but nullptrs are
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index 036809e94a..26d4a91c71 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -66,6 +66,7 @@ static const char* param_structs[] = {"TfLiteConvParams",
                                       "TfLiteFakeQuantParams",
                                       "TfLitePackParams",
                                       "TfLiteOneHotParams",
+                                      "TfLiteLeakyReluParams",
                                       nullptr};
 }  // namespace
 
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 950bdb3942..a1be2a5abc 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -682,6 +682,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_FILL:
       case tflite::BuiltinOperator_FLOOR_MOD:
       case tflite::BuiltinOperator_RANGE:
+      case tflite::BuiltinOperator_LEAKY_RELU:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 9b0eae74c3..3eeae708fd 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -200,6 +200,7 @@ enum BuiltinOperator : byte {
   FLOOR_MOD = 95,
   RANGE = 96,
   RESIZE_NEAREST_NEIGHBOR = 97,
+  LEAKY_RELU = 98,
 }
 
 // Options for the builtin operators.
@@ -278,6 +279,7 @@ union BuiltinOptions {
   FloorModOptions,
   RangeOptions,
   ResizeNearestNeighborOptions,
+  LeakyReluOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -658,6 +660,10 @@ table FloorModOptions {
 table RangeOptions {
 }
 
+table LeakyReluOptions {
+  alpha:float;
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index b7885cfcc5..b054f6bb0c 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -253,6 +253,9 @@ struct FloorModOptionsT;
 struct RangeOptions;
 struct RangeOptionsT;
 
+struct LeakyReluOptions;
+struct LeakyReluOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -500,11 +503,12 @@ enum BuiltinOperator {
   BuiltinOperator_FLOOR_MOD = 95,
   BuiltinOperator_RANGE = 96,
   BuiltinOperator_RESIZE_NEAREST_NEIGHBOR = 97,
+  BuiltinOperator_LEAKY_RELU = 98,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_RESIZE_NEAREST_NEIGHBOR
+  BuiltinOperator_MAX = BuiltinOperator_LEAKY_RELU
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[97] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[98] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -602,7 +606,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[97] {
     BuiltinOperator_FILL,
     BuiltinOperator_FLOOR_MOD,
     BuiltinOperator_RANGE,
-    BuiltinOperator_RESIZE_NEAREST_NEIGHBOR
+    BuiltinOperator_RESIZE_NEAREST_NEIGHBOR,
+    BuiltinOperator_LEAKY_RELU
   };
   return values;
 }
@@ -707,6 +712,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "FLOOR_MOD",
     "RANGE",
     "RESIZE_NEAREST_NEIGHBOR",
+    "LEAKY_RELU",
     nullptr
   };
   return names;
@@ -793,11 +799,12 @@ enum BuiltinOptions {
   BuiltinOptions_FloorModOptions = 72,
   BuiltinOptions_RangeOptions = 73,
   BuiltinOptions_ResizeNearestNeighborOptions = 74,
+  BuiltinOptions_LeakyReluOptions = 75,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_ResizeNearestNeighborOptions
+  BuiltinOptions_MAX = BuiltinOptions_LeakyReluOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[75] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[76] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -873,7 +880,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[75] {
     BuiltinOptions_UnidirectionalSequenceLSTMOptions,
     BuiltinOptions_FloorModOptions,
     BuiltinOptions_RangeOptions,
-    BuiltinOptions_ResizeNearestNeighborOptions
+    BuiltinOptions_ResizeNearestNeighborOptions,
+    BuiltinOptions_LeakyReluOptions
   };
   return values;
 }
@@ -955,6 +963,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "FloorModOptions",
     "RangeOptions",
     "ResizeNearestNeighborOptions",
+    "LeakyReluOptions",
     nullptr
   };
   return names;
@@ -1265,6 +1274,10 @@ template<> struct BuiltinOptionsTraits<ResizeNearestNeighborOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_ResizeNearestNeighborOptions;
 };
 
+template<> struct BuiltinOptionsTraits<LeakyReluOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_LeakyReluOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1888,6 +1901,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_ResizeNearestNeighborOptions ?
       reinterpret_cast<const ResizeNearestNeighborOptionsT *>(value) : nullptr;
   }
+  LeakyReluOptionsT *AsLeakyReluOptions() {
+    return type == BuiltinOptions_LeakyReluOptions ?
+      reinterpret_cast<LeakyReluOptionsT *>(value) : nullptr;
+  }
+  const LeakyReluOptionsT *AsLeakyReluOptions() const {
+    return type == BuiltinOptions_LeakyReluOptions ?
+      reinterpret_cast<const LeakyReluOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -6633,6 +6654,60 @@ inline flatbuffers::Offset<RangeOptions> CreateRangeOptions(
 
 flatbuffers::Offset<RangeOptions> CreateRangeOptions(flatbuffers::FlatBufferBuilder &_fbb, const RangeOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct LeakyReluOptionsT : public flatbuffers::NativeTable {
+  typedef LeakyReluOptions TableType;
+  float alpha;
+  LeakyReluOptionsT()
+      : alpha(0.0f) {
+  }
+};
+
+struct LeakyReluOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef LeakyReluOptionsT NativeTableType;
+  enum {
+    VT_ALPHA = 4
+  };
+  float alpha() const {
+    return GetField<float>(VT_ALPHA, 0.0f);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyField<float>(verifier, VT_ALPHA) &&
+           verifier.EndTable();
+  }
+  LeakyReluOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(LeakyReluOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<LeakyReluOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct LeakyReluOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_alpha(float alpha) {
+    fbb_.AddElement<float>(LeakyReluOptions::VT_ALPHA, alpha, 0.0f);
+  }
+  explicit LeakyReluOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  LeakyReluOptionsBuilder &operator=(const LeakyReluOptionsBuilder &);
+  flatbuffers::Offset<LeakyReluOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<LeakyReluOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<LeakyReluOptions> CreateLeakyReluOptions(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    float alpha = 0.0f) {
+  LeakyReluOptionsBuilder builder_(_fbb);
+  builder_.add_alpha(alpha);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<LeakyReluOptions> CreateLeakyReluOptions(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -6988,6 +7063,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const ResizeNearestNeighborOptions *builtin_options_as_ResizeNearestNeighborOptions() const {
     return builtin_options_type() == BuiltinOptions_ResizeNearestNeighborOptions ? static_cast<const ResizeNearestNeighborOptions *>(builtin_options()) : nullptr;
   }
+  const LeakyReluOptions *builtin_options_as_LeakyReluOptions() const {
+    return builtin_options_type() == BuiltinOptions_LeakyReluOptions ? static_cast<const LeakyReluOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7315,6 +7393,10 @@ template<> inline const ResizeNearestNeighborOptions *Operator::builtin_options_
   return builtin_options_as_ResizeNearestNeighborOptions();
 }
 
+template<> inline const LeakyReluOptions *Operator::builtin_options_as<LeakyReluOptions>() const {
+  return builtin_options_as_LeakyReluOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -9806,6 +9888,32 @@ inline flatbuffers::Offset<RangeOptions> CreateRangeOptions(flatbuffers::FlatBuf
       _fbb);
 }
 
+inline LeakyReluOptionsT *LeakyReluOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new LeakyReluOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void LeakyReluOptions::UnPackTo(LeakyReluOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+  { auto _e = alpha(); _o->alpha = _e; };
+}
+
+inline flatbuffers::Offset<LeakyReluOptions> LeakyReluOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateLeakyReluOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<LeakyReluOptions> CreateLeakyReluOptions(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const LeakyReluOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  auto _alpha = _o->alpha;
+  return tflite::CreateLeakyReluOptions(
+      _fbb,
+      _alpha);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -10360,6 +10468,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const ResizeNearestNeighborOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_LeakyReluOptions: {
+      auto ptr = reinterpret_cast<const LeakyReluOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -10674,6 +10786,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const ResizeNearestNeighborOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_LeakyReluOptions: {
+      auto ptr = reinterpret_cast<const LeakyReluOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -10976,6 +11092,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const ResizeNearestNeighborOptionsT *>(value);
       return CreateResizeNearestNeighborOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_LeakyReluOptions: {
+      auto ptr = reinterpret_cast<const LeakyReluOptionsT *>(value);
+      return CreateLeakyReluOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -11278,6 +11398,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new ResizeNearestNeighborOptionsT(*reinterpret_cast<ResizeNearestNeighborOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_LeakyReluOptions: {
+      value = new LeakyReluOptionsT(*reinterpret_cast<LeakyReluOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -11655,6 +11779,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_LeakyReluOptions: {
+      auto ptr = reinterpret_cast<LeakyReluOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
GitLab


From c9734ae1cd0705d9be39b58fa328f4f3d5b3547c Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 15 Nov 2018 12:27:55 -0800
Subject: [PATCH 0329/1554] [XLA] Change GetDimensionSize type to U32.

PiperOrigin-RevId: 221669196
---
 tensorflow/compiler/xla/service/shape_inference.cc | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 61a60ef9ef..2bfc1676bd 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2038,7 +2038,16 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
                            dimension);
   }
 
-  return ShapeUtil::MakeShape(S64, {});
+  // TODO(b/119580730): Remove this restriction when very large dimension size
+  // is needed.
+  if (shape.dimensions(dimension) > std::numeric_limits<uint32>::max()) {
+    return InvalidArgument(
+        "GetDimensionSize's input shape is %s, the %dth dimension exceeds the "
+        "UINT_MAX limit.",
+        ShapeUtil::HumanString(shape), dimension);
+  }
+
+  return ShapeUtil::MakeShape(U32, {});
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferSliceShape(
-- 
GitLab


From b32ddc60bb320a58e532c15c5029f2b105a0c13a Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 15 Nov 2018 12:50:39 -0800
Subject: [PATCH 0330/1554] Remove alias tf.set_random_seed in TF 2.0 API.

PiperOrigin-RevId: 221672774
---
 tensorflow/python/framework/random_seed.py      | 7 ++++---
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py
index 88c8b878dd..0d20693429 100644
--- a/tensorflow/python/framework/random_seed.py
+++ b/tensorflow/python/framework/random_seed.py
@@ -45,7 +45,7 @@ def get_seed(op_seed):
   graph, or for only specific operations.
 
   For details on how the graph-level seed interacts with op seeds, see
-  `tf.set_random_seed`.
+  `tf.random.set_random_seed`.
 
   Args:
     op_seed: integer.
@@ -82,7 +82,8 @@ def get_seed(op_seed):
   return seeds
 
 
-@tf_export('random.set_random_seed', 'set_random_seed')
+@tf_export('random.set_random_seed',
+           v1=['random.set_random_seed', 'set_random_seed'])
 def set_random_seed(seed):
   """Sets the graph-level random seed.
 
@@ -154,7 +155,7 @@ def set_random_seed(seed):
   sessions, set a graph-level seed:
 
   ```python
-  tf.set_random_seed(1234)
+  tf.random.set_random_seed(1234)
   a = tf.random_uniform([1])
   b = tf.random_normal([1])
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 87ce1b2328..e5331649d7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1052,10 +1052,6 @@ tf_module {
     name: "sequence_mask"
     argspec: "args=[\'lengths\', \'maxlen\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'bool\'>\", \'None\'], "
   }
-  member_method {
-    name: "set_random_seed"
-    argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shape"
     argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
-- 
GitLab


From f15948957d31f6632c343ec6b7261c1d01acc79e Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 15 Nov 2018 13:10:55 -0800
Subject: [PATCH 0331/1554] Start moving most of the implementation of
 interpreter into subgraph.

This prepares for adding control flow by allowing multiple subgraphs
to be created and call into each other.

PiperOrigin-RevId: 221676178
---
 tensorflow/lite/BUILD            |  2 +
 tensorflow/lite/core/subgraph.cc | 32 +++++++++++
 tensorflow/lite/core/subgraph.h  | 97 ++++++++++++++++++++++++++++++++
 tensorflow/lite/interpreter.cc   | 90 +++++++++++++++--------------
 tensorflow/lite/interpreter.h    | 62 ++++++++++----------
 5 files changed, 212 insertions(+), 71 deletions(-)
 create mode 100644 tensorflow/lite/core/subgraph.cc
 create mode 100644 tensorflow/lite/core/subgraph.h

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index be84fc5db1..bb2c53b8c9 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -131,6 +131,7 @@ cc_library(
     name = "framework",
     srcs = [
         "allocation.cc",
+        "core/subgraph.cc",
         "graph_info.cc",
         "interpreter.cc",
         "model.cc",
@@ -155,6 +156,7 @@ cc_library(
         "allocation.h",
         "context.h",
         "context_util.h",
+        "core/subgraph.h",
         "error_reporter.h",
         "graph_info.h",
         "interpreter.h",
diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
new file mode 100644
index 0000000000..1033fecbb5
--- /dev/null
+++ b/tensorflow/lite/core/subgraph.cc
@@ -0,0 +1,32 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/core/subgraph.h"
+
+namespace tflite {
+
+Subgraph::~Subgraph() {
+  for (auto& node_and_reg : nodes_and_registration_) {
+    TfLiteNode& node = node_and_reg.first;
+    TfLiteIntArrayFree(node.inputs);
+    TfLiteIntArrayFree(node.outputs);
+    TfLiteIntArrayFree(node.temporaries);
+    if (node.builtin_data) free(node.builtin_data);
+    OpFree(node_and_reg.second, node.user_data);
+    node.builtin_data = nullptr;
+  }
+}
+
+}  // namespace tflite
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
new file mode 100644
index 0000000000..e53f752631
--- /dev/null
+++ b/tensorflow/lite/core/subgraph.h
@@ -0,0 +1,97 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_CORE_SUBGRAPH_H_
+#define TENSORFLOW_LITE_CORE_SUBGRAPH_H_
+
+#include <cstdlib>
+#include <vector>
+
+#include "tensorflow/lite/allocation.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+
+namespace tflite {
+
+class Subgraph {
+ public:
+  Subgraph(TfLiteContext* context) : context_(context) {}
+
+  virtual ~Subgraph();
+
+  // Read only access to list of inputs.
+  const std::vector<int>& inputs() const { return inputs_; }
+
+  // Read only access to list of outputs.
+  const std::vector<int>& outputs() const { return outputs_; }
+
+  // Read only access to list of variable tensors.
+  const std::vector<int>& variables() const { return variables_; }
+
+  // Read only access to list of inputs.
+  std::vector<int>& inputs() { return inputs_; }
+
+  // Read only access to list of outputs.
+  std::vector<int>& outputs() { return outputs_; }
+
+  // Read only access to list of variable tensors.
+  std::vector<int>& variables() { return variables_; }
+
+  // Mutable form of tensors (TEMPORARY for refactor).
+  // TODO(b/119495520): remove when refactoring complete.
+  std::vector<TfLiteTensor>& tensors() { return tensors_; }
+  // Mutable form of tensors (TEMPORARY for refactor).
+  // TODO(b/119495520): remove when refactoring complete.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
+  nodes_and_registration() {
+    return nodes_and_registration_;
+  }
+
+  const std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
+  nodes_and_registration() const {
+    return nodes_and_registration_;
+  }
+
+ private:
+  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
+  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
+    if (op_reg.free == nullptr) return;
+    if (buffer) {
+      op_reg.free(context_, buffer);
+    }
+  }
+
+  // TODO(b/119495520): Make this be the authoritative copy.
+  TfLiteContext* context_;
+
+  std::vector<TfLiteTensor> tensors_;
+
+  // Array of indices representing the tensors that are inputs to the
+  // interpreter.
+  std::vector<int> inputs_;
+
+  // Array of indices representing the tensors that are outputs to the
+  // interpreter.
+  std::vector<int> outputs_;
+
+  // Array of indices representing the tensors that are variable tensors.
+  std::vector<int> variables_;
+
+  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
+  // function pointers to actual implementation.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
+      nodes_and_registration_;
+};
+
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_CORE_SUBGRAPH_H_
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index c90fc3be87..39b105042d 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -117,6 +117,7 @@ class InterpreterInfo : public GraphInfo {
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
+  subgraphs_.emplace_back(&context_);
   context_.impl_ = static_cast<void*>(this);
   context_.ResizeTensor = ResizeTensor;
   context_.ReportError = ReportError;
@@ -132,8 +133,9 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
   SwitchToKernelContext();
 
   // Reserve some space for the tensors to avoid excessive resizing.
-  tensors_.reserve(kTensorsReservedCapacity);
-  nodes_and_registration_.reserve(kTensorsReservedCapacity);
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  tensors.reserve(kTensorsReservedCapacity);
+  primary_subgraph().nodes_and_registration().reserve(kTensorsReservedCapacity);
   next_execution_plan_index_to_prepare_ = 0;
 
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
@@ -144,16 +146,6 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
 }
 
 Interpreter::~Interpreter() {
-  for (auto& nodeAndReg : nodes_and_registration_) {
-    TfLiteNode& node = nodeAndReg.first;
-    TfLiteIntArrayFree(node.inputs);
-    TfLiteIntArrayFree(node.outputs);
-    TfLiteIntArrayFree(node.temporaries);
-    if (node.builtin_data) free(node.builtin_data);
-    OpFree(nodeAndReg.second, node.user_data);
-    node.builtin_data = nullptr;
-  }
-
   for (size_t i = 0; i < context_.tensors_size; i++) {
     TfLiteTensor* tensor = &context_.tensors[i];
     if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
@@ -261,6 +253,8 @@ TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
                                            &node_subsets);
 
   execution_plan_.clear();
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+
   for (auto& node_subset : node_subsets) {
     // Subsets calimed by the delegate should have a "macro" op created, the
     // other node_subsets (kTfNonPartition) just have their nodes added back to
@@ -283,14 +277,15 @@ TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
 
         // Initialize the output tensors's delegate-related fields.
         for (int tensor_index : node_subset.output_tensors) {
-          TfLiteTensor* tensor = &tensors_[tensor_index];
+          TfLiteTensor* tensor = &tensors[tensor_index];
           TF_LITE_ENSURE(&context_, tensor->delegate == nullptr ||
                                         tensor->delegate == delegate);
           tensor->delegate = delegate;
         }
 
         // Associate the node with the delegate.
-        TfLiteNode* node = &nodes_and_registration_[node_index].first;
+        TfLiteNode* node =
+            &primary_subgraph().nodes_and_registration()[node_index].first;
         node->delegate = delegate;
       } break;
       case NodeSubset::kTfUnexplored:
@@ -353,21 +348,21 @@ TfLiteStatus Interpreter::GetExecutionPlan(struct TfLiteContext* context,
 TfLiteStatus Interpreter::SetInputs(std::vector<int> inputs) {
   TF_LITE_ENSURE_OK(&context_,
                     CheckTensorIndices("inputs", inputs.data(), inputs.size()));
-  inputs_ = std::move(inputs);
+  primary_subgraph().inputs() = std::move(inputs);
   return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetOutputs(std::vector<int> outputs) {
   TF_LITE_ENSURE_OK(
       &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
-  outputs_ = std::move(outputs);
+  primary_subgraph().outputs() = std::move(outputs);
   return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetVariables(std::vector<int> variables) {
   TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
                                                   variables.size()));
-  variables_ = std::move(variables);
+  primary_subgraph().variables() = std::move(variables);
   return kTfLiteOk;
 }
 
@@ -439,7 +434,8 @@ TfLiteStatus Interpreter::AllocateTensors() {
   // Explicit (re)allocation is necessary if nodes have been changed or tensors
   // have been resized. For inputs marked as dynamic, we can't short-circuit the
   // allocation as the client may have done the resize manually.
-  if (state_ != kStateUninvokable && !HasDynamicTensorImpl(context_, inputs_)) {
+  if (state_ != kStateUninvokable &&
+      !HasDynamicTensorImpl(context_, inputs())) {
     return kTfLiteOk;
   }
 
@@ -463,7 +459,8 @@ TfLiteStatus Interpreter::AllocateTensors() {
 
 // TODO(ycling): Support non-zero default values.
 TfLiteStatus Interpreter::ResetVariableTensors() {
-  for (auto& tensor : tensors_) {
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  for (auto& tensor : tensors) {
     if (!tensor.is_variable) {
       continue;
     }
@@ -480,7 +477,7 @@ TfLiteStatus Interpreter::ResetVariableTensors() {
 }
 
 void Interpreter::ReserveNodes(int count) {
-  nodes_and_registration_.reserve(count);
+  primary_subgraph().nodes_and_registration().reserve(count);
 }
 
 TfLiteStatus Interpreter::AddNodeWithParameters(
@@ -503,10 +500,11 @@ TfLiteStatus Interpreter::AddNodeWithParameters(
       &context_,
       CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
 
-  int new_node_index = nodes_and_registration_.size();
+  int new_node_index = primary_subgraph().nodes_and_registration().size();
   if (node_index) *node_index = new_node_index;
-  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
-  auto& node_and_reg = nodes_and_registration_.back();
+  primary_subgraph().nodes_and_registration().resize(
+      primary_subgraph().nodes_and_registration().size() + 1);
+  auto& node_and_reg = primary_subgraph().nodes_and_registration().back();
   TfLiteNode& node = node_and_reg.first;
   if (node.inputs) TfLiteIntArrayFree(node.inputs);
   if (node.outputs) TfLiteIntArrayFree(node.outputs);
@@ -577,12 +575,13 @@ bool HasDynamicTensor(const TfLiteContext& context,
 
 TfLiteStatus Interpreter::PrepareOpsStartingAt(
     int first_execution_plan_index, int* last_execution_plan_index_prepared) {
+  auto& subgraph = primary_subgraph();
   for (int execution_plan_index = first_execution_plan_index;
        execution_plan_index < execution_plan_.size(); execution_plan_index++) {
     int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    TfLiteNode& node = subgraph.nodes_and_registration()[node_index].first;
     const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
+        subgraph.nodes_and_registration()[node_index].second;
     EnsureTensorsVectorCapacity();
     if (OpPrepare(registration, &node) == kTfLiteError) {
       return ReportOpError(&context_, node, registration, node_index,
@@ -621,6 +620,8 @@ TfLiteStatus Interpreter::PrepareOpsAndTensors() {
 }
 
 TfLiteStatus Interpreter::Invoke() {
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+
   if (!consistent_) {
     ReportError(&context_, "Invoke called on model that is not consistent.");
     return kTfLiteError;
@@ -657,9 +658,10 @@ TfLiteStatus Interpreter::Invoke() {
                                     execution_plan_index);
     }
     int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    TfLiteNode& node =
+        primary_subgraph().nodes_and_registration()[node_index].first;
     const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
+        primary_subgraph().nodes_and_registration()[node_index].second;
     SCOPED_OPERATOR_PROFILE(profiler_, node_index);
 
     // TODO(ycling): This is an extra loop through inputs to check if the data
@@ -671,7 +673,7 @@ TfLiteStatus Interpreter::Invoke() {
       if (tensor_index == kOptionalTensor) {
         continue;
       }
-      TfLiteTensor* tensor = &tensors_[tensor_index];
+      TfLiteTensor* tensor = &tensors[tensor_index];
       if (tensor->delegate && tensor->delegate != node.delegate &&
           tensor->data_is_stale) {
         EnsureTensorDataIsReadable(tensor_index);
@@ -694,7 +696,7 @@ TfLiteStatus Interpreter::Invoke() {
   }
 
   if (!allow_buffer_handle_output_) {
-    for (int tensor_index : outputs_) {
+    for (int tensor_index : outputs()) {
       EnsureTensorDataIsReadable(tensor_index);
     }
   }
@@ -729,15 +731,17 @@ void Interpreter::ReportError(TfLiteContext* context, const char* format, ...) {
 
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
                                      int* first_new_tensor_index) {
-  const size_t base_index = tensors_.size();
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+
+  const size_t base_index = tensors.size();
   if (first_new_tensor_index) *first_new_tensor_index = base_index;
-  tensors_.resize(tensors_.size() + tensors_to_add);
-  for (size_t i = base_index; i < tensors_.size(); i++) {
-    memset(&tensors_[i], 0, sizeof(tensors_[i]));
-    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
+  tensors.resize(tensors.size() + tensors_to_add);
+  for (size_t i = base_index; i < tensors.size(); i++) {
+    memset(&tensors[i], 0, sizeof(tensors[i]));
+    tensors[i].buffer_handle = kTfLiteNullBufferHandle;
   }
-  context_.tensors = tensors_.data();
-  context_.tensors_size = tensors_.size();
+  context_.tensors = tensors.data();
+  context_.tensors_size = tensors.size();
   return kTfLiteOk;
 }
 
@@ -755,8 +759,9 @@ TfLiteStatus Interpreter::GetNodeAndRegistration(
   TF_LITE_ENSURE(&context_, node_index >= 0);
   TF_LITE_ENSURE(&context_, static_cast<size_t>(node_index) < nodes_size());
   TF_LITE_ENSURE(&context_, node != nullptr && registration != nullptr);
-  *node = &nodes_and_registration_[node_index].first;
-  *registration = &nodes_and_registration_[node_index].second;
+  auto& node_and_reg = primary_subgraph().nodes_and_registration()[node_index];
+  *node = &node_and_reg.first;
+  *registration = &node_and_reg.second;
   return kTfLiteOk;
 }
 
@@ -946,7 +951,8 @@ TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
       // If all the nodes can be prepared, check if the last node has dynamic
       // tensors.
       int node_index = execution_plan_[last_execution_plan_index_prepared];
-      TfLiteNode& node = nodes_and_registration_[node_index].first;
+      TfLiteNode& node =
+          primary_subgraph().nodes_and_registration()[node_index].first;
       if (!HasDynamicTensor(context_, node.outputs)) {
         has_dynamic_tensors = false;
       }
@@ -988,7 +994,8 @@ TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle buffer_handle,
                                           TfLiteDelegate* delegate) {
   TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  TfLiteTensor* tensor = &tensors_[tensor_index];
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  TfLiteTensor* tensor = &tensors[tensor_index];
 
   TF_LITE_ENSURE(&context_,
                  tensor->delegate == nullptr || tensor->delegate == delegate);
@@ -1007,7 +1014,8 @@ TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle* buffer_handle,
                                           TfLiteDelegate** delegate) {
   TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  TfLiteTensor* tensor = &tensors_[tensor_index];
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  TfLiteTensor* tensor = &tensors[tensor_index];
 
   *delegate = tensor->delegate;
   *buffer_handle = tensor->buffer_handle;
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index 415c5f0979..1cadb5eb5d 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/memory_planner.h"
 #include "tensorflow/lite/profiling/profiler.h"
 #include "tensorflow/lite/stderr_reporter.h"
@@ -197,31 +198,37 @@ class Interpreter {
   // Functions to access tensor data
 
   // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return inputs_; }
+  const std::vector<int>& inputs() const { return primary_subgraph().inputs(); }
 
   // Return the name of a given input. The given index must be between 0 and
   // inputs().size().
   const char* GetInputName(int index) const {
-    return context_.tensors[inputs_[index]].name;
+    return context_.tensors[inputs()[index]].name;
   }
 
   // Read only access to list of outputs.
-  const std::vector<int>& outputs() const { return outputs_; }
+  const std::vector<int>& outputs() const {
+    return primary_subgraph().outputs();
+  }
 
   // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const { return variables_; }
+  const std::vector<int>& variables() const {
+    return primary_subgraph().variables();
+  }
 
   // Return the name of a given output. The given index must be between 0 and
   // outputs().size().
   const char* GetOutputName(int index) const {
-    return context_.tensors[outputs_[index]].name;
+    return context_.tensors[outputs()[index]].name;
   }
 
   // Return the number of tensors in the model.
   size_t tensors_size() const { return context_.tensors_size; }
 
   // Return the number of ops in the model.
-  size_t nodes_size() const { return nodes_and_registration_.size(); }
+  size_t nodes_size() const {
+    return primary_subgraph().nodes_and_registration().size();
+  }
 
   // WARNING: Experimental interface, subject to change
   const std::vector<int>& execution_plan() const { return execution_plan_; }
@@ -251,10 +258,9 @@ class Interpreter {
   // Get a pointer to an operation and registration data structure if in bounds.
   const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
       int node_index) const {
-    if (node_index < 0 ||
-        static_cast<size_t>(node_index) >= nodes_and_registration_.size())
+    if (node_index < 0 || static_cast<size_t>(node_index) >= nodes_size())
       return nullptr;
-    return &nodes_and_registration_[node_index];
+    return &primary_subgraph().nodes_and_registration()[node_index];
   }
 
   // Perform a checked cast to the appropriate tensor type (mutable pointer
@@ -285,28 +291,28 @@ class Interpreter {
   // index must be between 0 and inputs().size().
   template <class T>
   T* typed_input_tensor(int index) {
-    return typed_tensor<T>(inputs_[index]);
+    return typed_tensor<T>(inputs()[index]);
   }
 
   // Return an immutable pointer into the data of a given input tensor. The
   // given index must be between 0 and inputs().size().
   template <class T>
   const T* typed_input_tensor(int index) const {
-    return typed_tensor<T>(inputs_[index]);
+    return typed_tensor<T>(inputs()[index]);
   }
 
   // Return a mutable pointer into the data of a given output tensor. The given
   // index must be between 0 and outputs().size().
   template <class T>
   T* typed_output_tensor(int index) {
-    return typed_tensor<T>(outputs_[index]);
+    return typed_tensor<T>(outputs()[index]);
   }
 
   // Return an immutable pointer into the data of a given output tensor. The
   // given index must be between 0 and outputs().size().
   template <class T>
   const T* typed_output_tensor(int index) const {
-    return typed_tensor<T>(outputs_[index]);
+    return typed_tensor<T>(outputs()[index]);
   }
 
   // Change the dimensionality of a given tensor. Note, this is only acceptable
@@ -446,6 +452,14 @@ class Interpreter {
   friend class InterpreterBuilder;
   friend class InterpreterTest;
 
+  Subgraph& primary_subgraph() {
+    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
+  }
+
+  const Subgraph& primary_subgraph() const {
+    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
+  }
+
   // Prevent 'context_' from accessing functions that are only available to
   // delegated kernels.
   void SwitchToKernelContext();
@@ -498,7 +512,7 @@ class Interpreter {
   // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
   // `context_` whenever this std::vector is reallocated. Currently this
   // only happens in `AddTensors()`.
-  std::vector<TfLiteTensor> tensors_;
+  // std::vector<TfLiteTensor> tensors_;
 
   // Check if an array of tensor indices are valid with respect to the Tensor
   // array.
@@ -592,6 +606,7 @@ class Interpreter {
   // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
   // more tensors won't invalidate the pointer to existing tensors.
   void EnsureTensorsVectorCapacity() {
+    std::vector<TfLiteTensor>& tensors_ = primary_subgraph().tensors();
     const size_t required_capacity = tensors_size() + kTensorsCapacityHeadroom;
     if (required_capacity > tensors_.capacity()) {
       tensors_.reserve(required_capacity);
@@ -618,27 +633,11 @@ class Interpreter {
   // structure to store tensors.
   TfLiteContext context_;
 
-  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
-  // function pointers to actual implementation.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
-      nodes_and_registration_;
-
   // Whether the model is consistent. That is to say if the inputs and outputs
   // of every node and the global inputs and outputs are valid indexes into
   // the tensor array.
   bool consistent_ = true;
 
-  // Array of indices representing the tensors that are inputs to the
-  // interpreter.
-  std::vector<int> inputs_;
-
-  // Array of indices representing the tensors that are outputs to the
-  // interpreter.
-  std::vector<int> outputs_;
-
-  // Array of indices representing the tensors that are variable tensors.
-  std::vector<int> variables_;
-
   // The error reporter delegate that tflite will forward queries errors to.
   ErrorReporter* error_reporter_;
 
@@ -686,6 +685,9 @@ class Interpreter {
 
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
+
+  // Subgraphs
+  std::vector<Subgraph> subgraphs_;
 };
 
 }  // namespace tflite
-- 
GitLab


From c418938e751699b7f3ec30d8c4a68c0b4ae1a271 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 13:11:05 -0800
Subject: [PATCH 0332/1554] Delete ReplicaContext.device. All users were moved
 to .devices in an earlier change.

PiperOrigin-RevId: 221676215
---
 tensorflow/contrib/distribute/python/mirrored_strategy.py   | 4 ----
 tensorflow/contrib/distribute/python/one_device_strategy.py | 4 ----
 tensorflow/contrib/distribute/python/tpu_strategy.py        | 4 ----
 tensorflow/python/training/distribute.py                    | 5 -----
 4 files changed, 17 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index b8d4fea298..6aec4d27a3 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -822,10 +822,6 @@ class MirroredReplicaContext(distribute_lib.ReplicaContext):
       raise _RequestedStop()
     return t.merge_result
 
-  @property
-  def device(self):
-    raise RuntimeError("Use .devices instead")
-
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index a4ce956a9f..3582d99405 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -182,10 +182,6 @@ class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
     distribute_lib.ReplicaContext.__init__(
         self, distribution_strategy, replica_id_in_sync_group=0)
 
-  @property
-  def device(self):
-    raise RuntimeError("Use .devices instead")
-
   @property
   def devices(self):
     return [self._distribution_strategy.worker_devices[0]]
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 53fe8ae244..5f7838c6ce 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -589,10 +589,6 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
     distribute_lib.ReplicaContext.__init__(
         self, distribution_strategy, replica_id_in_sync_group=0)
 
-  @property
-  def device(self):
-    raise RuntimeError("Use .devices instead")
-
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index ceb73ebc4e..dfa368026b 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1265,11 +1265,6 @@ class ReplicaContext(object):
     """The current `DistributionStrategy` object."""
     return self._distribution_strategy
 
-  @property
-  def device(self):
-    """BEING DELETED: use .devices instead."""
-    raise RuntimeError("Use .devices instead")
-
   @property
   def devices(self):
     """The devices this replica is to be executed on, as a list of strings."""
-- 
GitLab


From 31f48a5a60cbfec9163b981cb2bdb5486f0fd257 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Nov 2018 13:17:46 -0800
Subject: [PATCH 0333/1554] Add compat.v1 for v1 build as well.

PiperOrigin-RevId: 221677301
---
 tensorflow/BUILD | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 7c87a616a6..2dc70c359c 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -43,6 +43,11 @@ TENSORFLOW_API_INIT_FILES_V2 = (
     TENSORFLOW_API_INIT_FILES + get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1)
 )
 
+# @unused
+TENSORFLOW_API_INIT_FILES_V1_WITH_COMPAT = (
+    TENSORFLOW_API_INIT_FILES_V1 + get_compat_files(TENSORFLOW_API_INIT_FILES_V1, 1)
+)
+
 # Config setting used when building for products
 # which requires restricted licenses to be avoided.
 config_setting(
@@ -561,10 +566,15 @@ genrule(
 
 gen_api_init_files(
     name = "tf_python_api_gen_v1",
-    srcs = ["api_template_v1.__init__.py"],
+    srcs = [
+        "api_template_v1.__init__.py",
+        "compat_template_v1.__init__.py",
+    ],
     api_version = 1,
+    compat_api_versions = [1],
+    compat_init_templates = ["compat_template_v1.__init__.py"],
     output_dir = "_api/v1/",
-    output_files = TENSORFLOW_API_INIT_FILES_V1,
+    output_files = TENSORFLOW_API_INIT_FILES_V1_WITH_COMPAT,
     output_package = "tensorflow._api.v1",
     root_file_name = "v1.py",
     root_init_template = "api_template_v1.__init__.py",
-- 
GitLab


From 2e902b354bcea0d7eb1319de2beeaf603c7d5ee5 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Thu, 15 Nov 2018 13:25:33 -0800
Subject: [PATCH 0334/1554] Support attributes on tf.function calls

While FunctionDef has an OpDef that can contain attribute
specification for each function, we don't use per function
attribute specificaton at the moment. Instead we have just two
attributes "executor_type" and "config" shared by all functions.
These attributes are declared in PartitionedCall.

This CL makes regular functions calls (not going through
PartitionedCall) have the same two attributes. It is done
by hardcoding them in eager/attr_builder. This is a bit hacky
but quick and efficient. A cleaner alternative could be to have
TF_GraphToFunction always add these attributes to OpDef of every
FunctionDef it builds. Hardcoding these attributes in C API
instead of letting users specify them from Python should be fine
because they are consumed by FunctionLibraryRuntime.

PiperOrigin-RevId: 221678560
---
 tensorflow/c/eager/c_api.cc                   | 26 ++++++++--------
 tensorflow/c/eager/c_api_internal.h           |  7 ++---
 .../core/common_runtime/eager/attr_builder.cc | 30 +++++++++++++++++--
 .../core/common_runtime/eager/attr_builder.h  |  6 +++-
 .../common_runtime/eager/attr_builder_test.cc | 17 ++++++++---
 .../common_runtime/eager/eager_operation.h    | 14 +++++----
 .../core/common_runtime/eager/execute.cc      | 15 +++++++---
 .../eager/eager_service_impl.cc               | 26 ++++++++--------
 tensorflow/lite/delegates/flex/kernel.cc      | 24 ++++++++++-----
 9 files changed, 112 insertions(+), 53 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 408277468d..192044915f 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/c_api_internal.h"
+#include "tensorflow/core/platform/host_info.h"
 #ifdef TENSORFLOW_EAGER_USE_XLA
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #endif  // TENSORFLOW_EAGER_USE_XLA
@@ -458,13 +459,20 @@ TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name,
                   TF_Status* status) {
   const char* name = op_or_function_name;  // Shorthand
   const tensorflow::AttrTypeMap* types;
-  status->status = tensorflow::AttrTypeMapForOp(name, &types);
-  if (status->status.ok()) return new TFE_Op(ctx, name, types);
-  if (TF_GetCode(status) == TF_NOT_FOUND) {
-    if (ctx->context.FindFunctionByName(name)) {
-      status->status = tensorflow::Status::OK();
-      return new TFE_Op(ctx, name, nullptr);
+  bool is_function = false;
+  status->status = tensorflow::AttrTypeMapForOp(name, &types, &is_function);
+  if (status->status.ok()) {
+    if (is_function && !ctx->context.FindFunctionByName(name)) {
+      status->status = tensorflow::errors::NotFound(
+          "'", name,
+          "' is neither a type of a primitive operation nor a name "
+          "of a function registered in binary running on ",
+          tensorflow::port::Hostname(),
+          ". Make sure the operation or function is "
+          "registered in the binary running in this process.");
+      return nullptr;
     }
+    return new TFE_Op(ctx, name, is_function, types);
   }
   return nullptr;
 }
@@ -497,12 +505,6 @@ void TFE_OpAddInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) {
 TF_AttrType TFE_OpGetAttrType(TFE_Op* op, const char* attr_name,
                               unsigned char* is_list, TF_Status* status) {
   TF_AttrType ret;
-  if (op->operation.is_function()) {
-    status->status = tensorflow::errors::Unimplemented(
-        "TODO(apassos): Support for attributes for TensorFlow functions is not "
-        "ready yet.");
-    return TF_ATTR_INT;  // The compiler requires that we return something.
-  }
   status->status = tensorflow::AttrTypeByName(*op->operation.AttrTypes(),
                                               attr_name, &ret, is_list);
   return ret;
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index fa1b22e3af..67bc1bcd24 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -93,10 +93,9 @@ struct TFE_TensorDebugInfo {
 };
 
 struct TFE_Op {
-  // t is NULL iff the TFE_Op corresponds to a TensorFlow function instead of a
-  // primitive operation.
-  TFE_Op(TFE_Context* ctx, const char* op, const tensorflow::AttrTypeMap* t)
-      : operation(&ctx->context, op, t) {}
+  TFE_Op(TFE_Context* ctx, const char* op, bool is_function,
+         const tensorflow::AttrTypeMap* t)
+      : operation(&ctx->context, op, is_function, t) {}
 
   tensorflow::EagerOperation operation;
 };
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.cc b/tensorflow/core/common_runtime/eager/attr_builder.cc
index 201f06242f..aae3392d0e 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder.cc
@@ -39,6 +39,18 @@ std::unordered_map<string, const AttrTypeMap*>* OpNameToAttrTypeMap() {
 
 const uint32 kIsList = 1U << 31;
 
+AttrTypeMap* DefaultFunctionAttrTypeMap() {
+  AttrTypeMap* map = new AttrTypeMap();
+  (*map)["executor_type"] = TF_ATTR_STRING;
+  (*map)["config"] = TF_ATTR_STRING;
+  return map;
+}
+
+const AttrTypeMap* GetDefaultFunctionAttrTypeMap() {
+  static const AttrTypeMap* map = DefaultFunctionAttrTypeMap();
+  return map;
+}
+
 }  // namespace
 
 Status OpDefForOp(const char* op_name, const OpDef** op_def) {
@@ -50,13 +62,27 @@ Status OpDefForOp(const char* op_name, const OpDef** op_def) {
   return s;
 }
 
-Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out) {
+Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out,
+                        bool* is_function) {
   mutex_lock l(g_op_name_to_attr_type_map_lock);
+  *is_function = false;
   *out = gtl::FindPtrOrNull(*OpNameToAttrTypeMap(), op_name);
   if (*out != nullptr) return Status::OK();
   const OpDef* op_def = nullptr;
   Status s = OpDefForOp(op_name, &op_def);
-  if (!s.ok()) return s;
+  if (errors::IsNotFound(s)) {
+    // If we did not find the op def, we assume `op_name` is a function.
+    // If it is actually a misspelled op, user will get another error when
+    // trying to run it.
+    // TODO(iga): If we ever have a use case for different attribute specs
+    // in different functions, we will need to look at the OpDef in the
+    // function def to retrieve their types.
+    *out = GetDefaultFunctionAttrTypeMap();
+    *is_function = true;
+    return Status::OK();
+  } else if (!s.ok()) {
+    return s;
+  }
   std::unique_ptr<AttrTypeMap> m(new AttrTypeMap);
   // TODO(agarwal): Avoid having to create this "registry" at runtime,
   // perhaps can be done at op registration time?
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.h b/tensorflow/core/common_runtime/eager/attr_builder.h
index af5b7d80c3..41dd275a66 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.h
+++ b/tensorflow/core/common_runtime/eager/attr_builder.h
@@ -43,7 +43,11 @@ typedef std::unordered_map<string, uint32> AttrTypeMap;
 Status OpDefForOp(const char* op_name, const OpDef** op_def);
 
 // Returns the AttrTypeMap for the TensorFlow operation named op_name.
-Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out);
+// If op_name is not registered in global op registry, AttrTypeMapForOp assumes
+// the op to be a function and returns the default attributes for a function.
+// `is_function` is set to true in this case.
+Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out,
+                        bool* is_function);
 
 // Looks for 'attr_name' in 'm' and sets 'out' and 'is_list'.
 Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
diff --git a/tensorflow/core/common_runtime/eager/attr_builder_test.cc b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
index 79b094f2e0..220cc6f5ce 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder_test.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
@@ -35,9 +35,18 @@ namespace {
 
 TEST(AttrTypeMap, Lookup) {
   const AttrTypeMap* m = nullptr;
-  Status s = AttrTypeMapForOp("ThisOpCannotPossiblyExist", &m);
-  EXPECT_FALSE(s.ok());
-  s = AttrTypeMapForOp("MatMul", &m);
+  // Unknown ops are assumed to be functions.
+  // Their maps are filled with default attributes.
+  bool is_function = false;
+  Status s = AttrTypeMapForOp("SomeFunctionName", &m, &is_function);
+  EXPECT_TRUE(s.ok());
+  EXPECT_TRUE(is_function);
+  EXPECT_EQ(TF_ATTR_STRING, m->find("executor_type")->second);
+  EXPECT_EQ(TF_ATTR_STRING, m->find("config")->second);
+
+  is_function = true;
+  s = AttrTypeMapForOp("MatMul", &m, &is_function);
+  EXPECT_FALSE(is_function);
   ASSERT_TRUE(s.ok()) << s;
 
   TF_AttrType t;
@@ -50,7 +59,7 @@ TEST(AttrTypeMap, Lookup) {
   EXPECT_EQ(TF_ATTR_BOOL, t);
   EXPECT_EQ(is_list, 0);
 
-  s = AttrTypeMapForOp("Squeeze", &m);
+  s = AttrTypeMapForOp("Squeeze", &m, &is_function);
   ASSERT_TRUE(s.ok()) << s;
   s = AttrTypeByName(*m, "squeeze_dims", &t, &is_list);
   ASSERT_TRUE(s.ok()) << s;
diff --git a/tensorflow/core/common_runtime/eager/eager_operation.h b/tensorflow/core/common_runtime/eager/eager_operation.h
index fcf62c7715..935ca7f9aa 100644
--- a/tensorflow/core/common_runtime/eager/eager_operation.h
+++ b/tensorflow/core/common_runtime/eager/eager_operation.h
@@ -22,11 +22,14 @@ limitations under the License.
 namespace tensorflow {
 class EagerOperation {
  public:
-  // t is NULL iff the EagerOperation corresponds to a TensorFlow function
-  // instead of a primitive operation.
   EagerOperation(tensorflow::EagerContext* ctx, const char* op,
-                 const tensorflow::AttrTypeMap* t)
-      : ctx_(ctx), name_(op), attrs_(op), attr_types_(t), device_(nullptr) {}
+                 bool is_function, const tensorflow::AttrTypeMap* t)
+      : ctx_(ctx),
+        name_(op),
+        attrs_(op),
+        attr_types_(t),
+        device_(nullptr),
+        is_function_(is_function) {}
 
   ~EagerOperation() {
     for (tensorflow::TensorHandle* h : inputs_) {
@@ -34,7 +37,7 @@ class EagerOperation {
     }
   }
 
-  bool is_function() const { return attr_types_ == nullptr; }
+  bool is_function() const { return is_function_; }
 
   tensorflow::EagerContext* EagerContext() { return ctx_; }
 
@@ -68,6 +71,7 @@ class EagerOperation {
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 4> inputs_;
   tensorflow::Device* device_;
   bool use_xla_ = false;
+  const bool is_function_;
 };
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index a708033c65..5bf7888fad 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/execute_node.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/lib/core/errors.h"
 #ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
 #include "tensorflow/core/distributed_runtime/eager/remote_execute_node.h"
@@ -827,8 +828,11 @@ Status ExecuteSend(EagerContext* ctx, tensorflow::Device* device,
                    TensorHandle* h, StringPiece wire_id,
                    const string& recv_device) {
   const tensorflow::AttrTypeMap* types;
-  TF_RETURN_IF_ERROR(tensorflow::AttrTypeMapForOp("_Send", &types));
-  tensorflow::EagerOperation op(ctx, "_Send", types);
+  bool is_function = false;
+  TF_RETURN_IF_ERROR(
+      tensorflow::AttrTypeMapForOp("_Send", &types, &is_function));
+  DCHECK(!is_function);
+  tensorflow::EagerOperation op(ctx, "_Send", /*is_function=*/false, types);
 
   op.AddInput(h);
 
@@ -855,8 +859,11 @@ Status ExecuteRecv(EagerContext* ctx, tensorflow::Device* device,
                    const string& send_device, int64 send_device_incarnation,
                    TensorHandle** result) {
   const tensorflow::AttrTypeMap* types;
-  TF_RETURN_IF_ERROR(tensorflow::AttrTypeMapForOp("_Recv", &types));
-  tensorflow::EagerOperation op(ctx, "_Recv", types);
+  bool is_function = false;
+  TF_RETURN_IF_ERROR(
+      tensorflow::AttrTypeMapForOp("_Recv", &types, &is_function));
+  DCHECK(!is_function);
+  tensorflow::EagerOperation op(ctx, "_Recv", /*is_function=*/false, types);
 
   op.SetDevice(device);
 
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
index b8af63724a..5b0a420fad 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
@@ -36,6 +36,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/host_info.h"
 
 namespace tensorflow {
 namespace eager {
@@ -152,20 +153,19 @@ Status EagerServiceImpl::ExecuteOp(const Operation& operation,
   std::unique_ptr<tensorflow::EagerOperation> op;
   const char* name = operation.name().c_str();  // Shorthand
   const tensorflow::AttrTypeMap* types;
-  auto status = tensorflow::AttrTypeMapForOp(name, &types);
-  if (status.ok()) {
-    op.reset(
-        new tensorflow::EagerOperation(server_context->Context(), name, types));
-  } else if (errors::IsNotFound(status)) {
-    if (server_context->Context()->FindFunctionByName(name)) {
-      op.reset(new tensorflow::EagerOperation(server_context->Context(), name,
-                                              nullptr));
-    } else {
-      return status;
-    }
-  } else {
-    return status;
+  bool is_function = false;
+  TF_RETURN_IF_ERROR(tensorflow::AttrTypeMapForOp(name, &types, &is_function));
+  if (is_function && !server_context->Context()->FindFunctionByName(name)) {
+    return errors::NotFound(
+        "'", name,
+        "' is neither a type of a primitive operation nor a name "
+        "of a function registered in binary running on ",
+        port::Hostname(),
+        ". Make sure the operation or function is "
+        "registered in the binary running in this process.");
   }
+  op.reset(new tensorflow::EagerOperation(server_context->Context(), name,
+                                          is_function, types));
 
   TF_RETURN_IF_ERROR(op->SetDevice(operation.device().c_str()));
 
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index c4fe142dff..02da1d1a22 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -15,6 +15,12 @@ limitations under the License.
 #include "tensorflow/lite/delegates/flex/kernel.h"
 
 #include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/execute.h"
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/node_def_util.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/lite/builtin_ops.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/context_util.h"
@@ -22,11 +28,6 @@ limitations under the License.
 #include "tensorflow/lite/delegates/flex/util.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
 #include "tensorflow/lite/string.h"
-#include "tensorflow/core/common_runtime/eager/context.h"
-#include "tensorflow/core/common_runtime/eager/execute.h"
-#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
-#include "tensorflow/core/framework/node_def.pb.h"
-#include "tensorflow/core/framework/node_def_util.h"
 
 // Note: this is part of TF Lite's Flex delegation code which is to be
 // completed soon.
@@ -78,11 +79,18 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
                                  const std::vector<int>& inputs,
                                  const std::vector<int>& outputs) {
   const tensorflow::AttrTypeMap* attr_types;
+  bool is_function = false;
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      tensorflow::AttrTypeMapForOp(op_name.c_str(), &attr_types),
+      tensorflow::AttrTypeMapForOp(op_name.c_str(), &attr_types, &is_function),
       " (while processing attributes of '", op_name, "')");
-
-  tensorflow::EagerOperation op(eager_context, op_name.c_str(), attr_types);
+  if (is_function) {
+    return tensorflow::errors::NotFound(
+        "Operation '", op_name,
+        "' is not registered.  (while processing attributes of '", op_name,
+        "')");
+  }
+  tensorflow::EagerOperation op(eager_context, op_name.c_str(),
+                                /*is_function=*/false, attr_types);
   for (const auto& attr : nodedef.attr()) {
     op.MutableAttrs()->Set(attr.first, attr.second);
   }
-- 
GitLab


From a68fec0c77a4c99d4306cd3e987b4f6f548a8112 Mon Sep 17 00:00:00 2001
From: RJ Ryan <rjryan@google.com>
Date: Thu, 15 Nov 2018 13:30:07 -0800
Subject: [PATCH 0335/1554] Merge tf.spectral into tf.signal for TensorFlow
 2.0.

PiperOrigin-RevId: 221679418
---
 tensorflow/compiler/tests/BUILD               |   1 -
 tensorflow/compiler/tests/fft_test.py         |  25 +-
 tensorflow/contrib/distributions/BUILD        |   2 +-
 .../distributions/python/ops/sample_stats.py  |   6 +-
 tensorflow/python/BUILD                       |  32 +-
 tensorflow/python/__init__.py                 |   1 -
 tensorflow/python/kernel_tests/BUILD          |  29 --
 tensorflow/python/kernel_tests/linalg/BUILD   |   3 +-
 .../linalg/linear_operator_circulant_test.py  |  29 +-
 tensorflow/python/kernel_tests/signal/BUILD   |  31 +-
 .../kernel_tests/{ => signal}/dct_ops_test.py |  22 +-
 .../kernel_tests/{ => signal}/fft_ops_test.py |  26 +-
 tensorflow/python/ops/gradients_impl.py       |   1 -
 tensorflow/python/ops/linalg/BUILD            |   1 +
 .../ops/linalg/linear_operator_circulant.py   |   5 +-
 tensorflow/python/ops/math_ops.py             |  11 -
 tensorflow/python/ops/signal/BUILD            |  17 +-
 .../{spectral_ops.py => signal/dct_ops.py}    | 152 +-------
 tensorflow/python/ops/signal/fft_ops.py       | 330 ++++++++++++++++++
 tensorflow/python/ops/signal/mfcc_ops.py      |   4 +-
 tensorflow/python/ops/signal/shape_ops.py     |   2 +-
 tensorflow/python/ops/signal/signal.py        |  14 +
 tensorflow/python/ops/signal/spectral_ops.py  |   8 +-
 tensorflow/python/ops/spectral_grad.py        | 185 ----------
 tensorflow/python/ops/standard_ops.py         |   1 -
 .../tools/api/generator/api_init_files.bzl    |   1 -
 .../python/tools/api/generator/doc_srcs.py    |   1 -
 .../api/golden/v1/tensorflow.signal.pbtxt     |  32 ++
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 .../api/golden/v2/tensorflow.signal.pbtxt     |  32 ++
 .../api/golden/v2/tensorflow.spectral.pbtxt   |  35 --
 tensorflow/tools/compatibility/renames_v2.py  |   8 +
 32 files changed, 538 insertions(+), 513 deletions(-)
 rename tensorflow/python/kernel_tests/{ => signal}/dct_ops_test.py (91%)
 rename tensorflow/python/kernel_tests/{ => signal}/fft_ops_test.py (98%)
 rename tensorflow/python/ops/{spectral_ops.py => signal/dct_ops.py} (53%)
 create mode 100644 tensorflow/python/ops/signal/fft_ops.py
 delete mode 100644 tensorflow/python/ops/spectral_grad.py
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 6b8e6bba1e..2b88a64fed 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -474,7 +474,6 @@ tf_xla_py_test(
         "//tensorflow/python:extra_py_tests_deps",
         "//tensorflow/python:framework",
         "//tensorflow/python:platform_test",
-        "//tensorflow/python:spectral_ops",
         "//tensorflow/python/ops/signal",
     ],
 )
diff --git a/tensorflow/compiler/tests/fft_test.py b/tensorflow/compiler/tests/fft_test.py
index fab356991a..61abf9c9c0 100644
--- a/tensorflow/compiler/tests/fft_test.py
+++ b/tensorflow/compiler/tests/fft_test.py
@@ -27,7 +27,6 @@ from tensorflow.compiler.tests import xla_test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
-from tensorflow.python.ops import spectral_ops
 from tensorflow.python.ops.signal import signal
 from tensorflow.python.platform import googletest
 
@@ -107,39 +106,39 @@ class FFTTest(xla_test.XLATestCase):
 
   def testFFT(self):
     self._VerifyFftMethod(INNER_DIMS_1D, lambda x: x, np.fft.fft,
-                          spectral_ops.fft)
+                          signal.fft)
 
   def testFFT2D(self):
     self._VerifyFftMethod(INNER_DIMS_2D, lambda x: x, np.fft.fft2,
-                          spectral_ops.fft2d)
+                          signal.fft2d)
 
   def testFFT3D(self):
     self._VerifyFftMethod(INNER_DIMS_3D, lambda x: x,
                           lambda x: np.fft.fftn(x, axes=(-3, -2, -1)),
-                          spectral_ops.fft3d)
+                          signal.fft3d)
 
   def testIFFT(self):
     self._VerifyFftMethod(INNER_DIMS_1D, lambda x: x, np.fft.ifft,
-                          spectral_ops.ifft)
+                          signal.ifft)
 
   def testIFFT2D(self):
     self._VerifyFftMethod(INNER_DIMS_2D, lambda x: x, np.fft.ifft2,
-                          spectral_ops.ifft2d)
+                          signal.ifft2d)
 
   def testIFFT3D(self):
     self._VerifyFftMethod(INNER_DIMS_3D, lambda x: x,
                           lambda x: np.fft.ifftn(x, axes=(-3, -2, -1)),
-                          spectral_ops.ifft3d)
+                          signal.ifft3d)
 
   def testRFFT(self):
     self._VerifyFftMethod(
         INNER_DIMS_1D, np.real, lambda x: np.fft.rfft(x, n=x.shape[-1]),
-        lambda x: spectral_ops.rfft(x, fft_length=[x.shape[-1].value]))
+        lambda x: signal.rfft(x, fft_length=[x.shape[-1].value]))
 
   def testRFFT2D(self):
 
     def _tf_fn(x):
-      return spectral_ops.rfft2d(
+      return signal.rfft2d(
           x, fft_length=[x.shape[-2].value, x.shape[-1].value])
 
     self._VerifyFftMethod(
@@ -153,7 +152,7 @@ class FFTTest(xla_test.XLATestCase):
           x, axes=(-3, -2, -1), s=[x.shape[-3], x.shape[-2], x.shape[-1]])
 
     def _tf_fn(x):
-      return spectral_ops.rfft3d(
+      return signal.rfft3d(
           x,
           fft_length=[x.shape[-3].value, x.shape[-2].value, x.shape[-1].value])
 
@@ -162,7 +161,7 @@ class FFTTest(xla_test.XLATestCase):
   def testIRFFT(self):
 
     def _tf_fn(x):
-      return spectral_ops.irfft(x, fft_length=[2 * (x.shape[-1].value - 1)])
+      return signal.irfft(x, fft_length=[2 * (x.shape[-1].value - 1)])
 
     self._VerifyFftMethod(
         INNER_DIMS_1D, lambda x: np.fft.rfft(np.real(x), n=x.shape[-1]),
@@ -171,7 +170,7 @@ class FFTTest(xla_test.XLATestCase):
   def testIRFFT2D(self):
 
     def _tf_fn(x):
-      return spectral_ops.irfft2d(
+      return signal.irfft2d(
           x, fft_length=[x.shape[-2].value, 2 * (x.shape[-1].value - 1)])
 
     self._VerifyFftMethod(
@@ -195,7 +194,7 @@ class FFTTest(xla_test.XLATestCase):
           s=[x.shape[-3], x.shape[-2], 2 * (x.shape[-1] - 1)])
 
     def _tf_fn(x):
-      return spectral_ops.irfft3d(
+      return signal.irfft3d(
           x,
           fft_length=[
               x.shape[-3].value, x.shape[-2].value, 2 * (x.shape[-1].value - 1)
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 60f6b90edc..3079175015 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -72,7 +72,6 @@ py_library(
         "//tensorflow/python:nn",
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:random_ops",
-        "//tensorflow/python:spectral_ops",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
@@ -80,6 +79,7 @@ py_library(
         "//tensorflow/python:variables",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
+        "//tensorflow/python/ops/signal",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/distributions/python/ops/sample_stats.py b/tensorflow/contrib/distributions/python/ops/sample_stats.py
index aa680a92be..978e627d66 100644
--- a/tensorflow/contrib/distributions/python/ops/sample_stats.py
+++ b/tensorflow/contrib/distributions/python/ops/sample_stats.py
@@ -29,8 +29,8 @@ from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
-from tensorflow.python.ops import spectral_ops
 from tensorflow.python.ops.distributions import util
+from tensorflow.python.ops.signal import fft_ops
 
 __all__ = [
     "auto_correlation",
@@ -157,11 +157,11 @@ def auto_correlation(
                                        dtype.real_dtype.as_numpy_dtype(0.))
 
     # Autocorrelation is IFFT of power-spectral density (up to some scaling).
-    fft_x_rotated_pad = spectral_ops.fft(x_rotated_pad)
+    fft_x_rotated_pad = fft_ops.fft(x_rotated_pad)
     spectral_density = fft_x_rotated_pad * math_ops.conj(fft_x_rotated_pad)
     # shifted_product is R[m] from above detailed explanation.
     # It is the inner product sum_n X[n] * Conj(X[n - m]).
-    shifted_product = spectral_ops.ifft(spectral_density)
+    shifted_product = fft_ops.ifft(spectral_density)
 
     # Cast back to real-valued if x was real to begin with.
     shifted_product = math_ops.cast(shifted_product, dtype)
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 7cc1237fe7..d656d8ef16 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -124,7 +124,6 @@ py_library(
         ":session_ops",
         ":sets",
         ":sparse_ops",
-        ":spectral_ops",
         ":spectral_ops_test_util",
         ":standard_ops",
         ":state_ops",
@@ -1837,6 +1836,7 @@ tf_gen_op_wrapper_private_py(
 
 tf_gen_op_wrapper_private_py(
     name = "spectral_ops_gen",
+    visibility = ["//tensorflow/python/ops/signal:__pkg__"],
 )
 
 tf_gen_op_wrapper_private_py(
@@ -2266,7 +2266,6 @@ py_library(
         ":platform",
         ":random_grad",
         ":resource_variable_ops",
-        ":spectral_grad",
         ":tensor_array_ops",
         ":tensor_util",
         ":unconnected_gradients",
@@ -2508,7 +2507,6 @@ py_library(
         ":nn_ops_gen",
         ":sparse_ops_gen",
         ":sparse_tensor",
-        ":spectral_ops_gen",
         ":state_ops",
         ":state_ops_gen",
         ":tensor_shape",
@@ -2817,33 +2815,6 @@ py_test(
     ],
 )
 
-py_library(
-    name = "spectral_grad",
-    srcs = ["ops/spectral_grad.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":array_ops",
-        ":framework",
-        ":framework_for_generated_wrappers",
-        ":math_ops",
-        ":spectral_ops",
-        "//third_party/py/numpy",
-    ],
-)
-
-py_library(
-    name = "spectral_ops",
-    srcs = ["ops/spectral_ops.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":array_ops",
-        ":dtypes",
-        ":framework_ops",
-        ":math_ops",
-        ":spectral_ops_gen",
-    ],
-)
-
 py_library(
     name = "spectral_ops_test_util",
     srcs = ["ops/spectral_ops_test_util.py"],
@@ -2961,7 +2932,6 @@ py_library(
         ":sparse_grad",
         ":sparse_ops",
         ":special_math_ops",
-        ":spectral_grad",
         ":state_grad",
         ":state_ops",
         ":stateless_random_ops",
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index f0d8079ce7..c93ba4dcee 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -87,7 +87,6 @@ from tensorflow.python.ops import manip_ops as manip
 from tensorflow.python.ops import metrics
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import sets
-from tensorflow.python.ops import spectral_ops as spectral
 from tensorflow.python.ops.distributions import distributions
 from tensorflow.python.ops.linalg import linalg
 from tensorflow.python.ops.losses import losses
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 002d5fbca7..43353f7a51 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2626,35 +2626,6 @@ cuda_py_test(
     tags = ["manual"],
 )
 
-cuda_py_test(
-    name = "dct_ops_test",
-    srcs = ["dct_ops_test.py"],
-    additional_deps = [
-        "@absl_py//absl/testing:parameterized",
-        "//third_party/py/numpy",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:spectral_ops",
-        "//tensorflow/python:spectral_ops_test_util",
-    ],
-)
-
-cuda_py_test(
-    name = "fft_ops_test",
-    size = "medium",
-    srcs = ["fft_ops_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:spectral_ops",
-        "//tensorflow/python:spectral_ops_test_util",
-    ],
-    shard_count = 4,
-    tags = ["optonly"],
-)
-
 cuda_py_test(
     name = "pooling_ops_3d_test",
     size = "medium",
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index 599e08ffa1..c2f14a119a 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -105,7 +105,6 @@ cuda_py_test(
     size = "medium",
     srcs = ["linear_operator_circulant_test.py"],
     additional_deps = [
-        "//tensorflow/python/ops/linalg",
         "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:spectral_ops_test_util",
@@ -115,6 +114,8 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python/ops/signal",
     ],
     shard_count = 5,
     tags = [
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
index dd342f1669..bd666c0db3 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import spectral_ops_test_util
 from tensorflow.python.ops.linalg import linalg
 from tensorflow.python.ops.linalg import linear_operator_circulant
 from tensorflow.python.ops.linalg import linear_operator_test_util
+from tensorflow.python.ops.signal import fft_ops
 from tensorflow.python.platform import test
 
 rng = np.random.RandomState(0)
@@ -75,8 +76,8 @@ class LinearOperatorCirculantBaseTest(object):
       x = np.zeros([domain_dimension])
       # x is a basis vector.
       x[m] = 1.0
-      fft_x = math_ops.fft(x.astype(np.complex64))
-      h_convolve_x = math_ops.ifft(spectrum * fft_x)
+      fft_x = fft_ops.fft(x.astype(np.complex64))
+      h_convolve_x = fft_ops.ifft(spectrum * fft_x)
       matrix_rows.append(h_convolve_x)
     matrix = array_ops.stack(matrix_rows, axis=-1)
     return math_ops.cast(matrix, dtype)
@@ -169,14 +170,14 @@ class LinearOperatorCirculantTestHermitianSpectrum(
     #  = IFFT[EvenPartOf[pre_spectrum]]
     # is the IFFT of something that is also bounded away from zero.
     # Therefore, FFT[pre_h] would be a well-conditioned spectrum.
-    pre_h = math_ops.ifft(pre_spectrum_c)
+    pre_h = fft_ops.ifft(pre_spectrum_c)
 
     # A spectrum is Hermitian iff it is the DFT of a real convolution kernel.
     # So we will make spectrum = FFT[h], for real valued h.
     h = math_ops.real(pre_h)
     h_c = _to_complex(h)
 
-    spectrum = math_ops.fft(h_c)
+    spectrum = fft_ops.fft(h_c)
 
     lin_op_spectrum = spectrum
 
@@ -273,7 +274,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
   def test_defining_operator_using_real_convolution_kernel(self):
     with self.cached_session():
       convolution_kernel = [1., 2., 1.]
-      spectrum = math_ops.fft(
+      spectrum = fft_ops.fft(
           math_ops.cast(convolution_kernel, dtypes.complex64))
 
       # spectrum is shape [3] ==> operator is shape [3, 3]
@@ -291,7 +292,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       # Make spectrum the FFT of a real convolution kernel h.  This ensures that
       # spectrum is Hermitian.
       h = linear_operator_test_util.random_normal(shape=(3, 4))
-      spectrum = math_ops.fft(math_ops.cast(h, dtypes.complex64))
+      spectrum = fft_ops.fft(math_ops.cast(h, dtypes.complex64))
       operator = linalg.LinearOperatorCirculant(
           spectrum, input_output_dtype=dtypes.complex64)
       matrix = operator.to_dense()
@@ -419,8 +420,8 @@ class LinearOperatorCirculant2DBaseTest(object):
         x = np.zeros(block_shape)
         # x is a basis vector.
         x[n0, n1] = 1.0
-        fft_x = math_ops.fft2d(x.astype(np.complex64))
-        h_convolve_x = math_ops.ifft2d(spectrum * fft_x)
+        fft_x = fft_ops.fft2d(x.astype(np.complex64))
+        h_convolve_x = fft_ops.ifft2d(spectrum * fft_x)
         # We want the flat version of the action of the operator on a basis
         # vector, not the block version.
         h_convolve_x = array_ops.reshape(h_convolve_x, shape[:-1])
@@ -459,14 +460,14 @@ class LinearOperatorCirculant2DTestHermitianSpectrum(
     #  = IFFT[EvenPartOf[pre_spectrum]]
     # is the IFFT of something that is also bounded away from zero.
     # Therefore, FFT[pre_h] would be a well-conditioned spectrum.
-    pre_h = math_ops.ifft2d(pre_spectrum_c)
+    pre_h = fft_ops.ifft2d(pre_spectrum_c)
 
     # A spectrum is Hermitian iff it is the DFT of a real convolution kernel.
     # So we will make spectrum = FFT[h], for real valued h.
     h = math_ops.real(pre_h)
     h_c = _to_complex(h)
 
-    spectrum = math_ops.fft2d(h_c)
+    spectrum = fft_ops.fft2d(h_c)
 
     lin_op_spectrum = spectrum
 
@@ -636,7 +637,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       convolution_kernel = linear_operator_test_util.random_normal(
           shape=(2, 2, 3, 5), dtype=dtypes.float32)
       # Convolution kernel is real ==> spectrum is Hermitian.
-      spectrum = math_ops.fft3d(
+      spectrum = fft_ops.fft3d(
           math_ops.cast(convolution_kernel, dtypes.complex64))
 
       # spectrum is Hermitian ==> operator is real.
@@ -668,7 +669,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       #         =      H1  +      H2
       # where H1 is real since it is Hermitian,
       # and H2 is imaginary since it is anti-Hermitian.
-      ifft_s = math_ops.ifft3d(math_ops.cast(s, dtypes.complex64))
+      ifft_s = fft_ops.ifft3d(math_ops.cast(s, dtypes.complex64))
 
       # Throw away H2, keep H1.
       real_ifft_s = math_ops.real(ifft_s)
@@ -676,7 +677,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       # This is the perfect spectrum!
       # spectrum = DFT[H1]
       #          = S1,
-      fft_real_ifft_s = math_ops.fft3d(
+      fft_real_ifft_s = fft_ops.fft3d(
           math_ops.cast(real_ifft_s, dtypes.complex64))
 
       # S1 is Hermitian ==> operator is real.
@@ -699,7 +700,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       # S2 is anti-Hermitian ==> operator is imaginary.
       # S2 is real ==> operator is self-adjoint.
       imag_ifft_s = math_ops.imag(ifft_s)
-      fft_imag_ifft_s = math_ops.fft3d(
+      fft_imag_ifft_s = fft_ops.fft3d(
           1j * math_ops.cast(imag_ifft_s, dtypes.complex64))
       operator_imag = linalg.LinearOperatorCirculant3D(fft_imag_ifft_s)
 
diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD
index 56d4d46d46..8f4e31abe3 100644
--- a/tensorflow/python/kernel_tests/signal/BUILD
+++ b/tensorflow/python/kernel_tests/signal/BUILD
@@ -18,6 +18,35 @@ py_library(
     ],
 )
 
+cuda_py_tests(
+    name = "dct_ops_test",
+    srcs = ["dct_ops_test.py"],
+    additional_deps = [
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:spectral_ops_test_util",
+        "//tensorflow/python/ops/signal",
+    ],
+)
+
+cuda_py_tests(
+    name = "fft_ops_test",
+    size = "medium",
+    srcs = ["fft_ops_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:spectral_ops_test_util",
+        "//tensorflow/python/ops/signal",
+    ],
+    shard_count = 4,
+    tags = ["optonly"],
+)
+
 cuda_py_tests(
     name = "mel_ops_test",
     srcs = ["mel_ops_test.py"],
@@ -91,9 +120,9 @@ cuda_py_tests(
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:spectral_ops_test_util",
+        "//tensorflow/python/ops/signal",
     ],
     tags = ["nomac"],
 )
diff --git a/tensorflow/python/kernel_tests/dct_ops_test.py b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
similarity index 91%
rename from tensorflow/python/kernel_tests/dct_ops_test.py
rename to tensorflow/python/kernel_tests/signal/dct_ops_test.py
index 27fdf8775e..af4939332f 100644
--- a/tensorflow/python/kernel_tests/dct_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
@@ -23,8 +23,8 @@ import importlib
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.python.ops import spectral_ops
 from tensorflow.python.ops import spectral_ops_test_util
+from tensorflow.python.ops.signal import dct_ops
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 
@@ -105,10 +105,10 @@ class DCTOpsTest(parameterized.TestCase, test.TestCase):
   def _compare(self, signals, norm, dct_type, atol=5e-4, rtol=5e-4):
     """Compares (I)DCT to SciPy (if available) and a NumPy implementation."""
     np_dct = NP_DCT[dct_type](signals, norm)
-    tf_dct = spectral_ops.dct(signals, type=dct_type, norm=norm).eval()
+    tf_dct = dct_ops.dct(signals, type=dct_type, norm=norm).eval()
     self.assertAllClose(np_dct, tf_dct, atol=atol, rtol=rtol)
     np_idct = NP_IDCT[dct_type](signals, norm)
-    tf_idct = spectral_ops.idct(signals, type=dct_type, norm=norm).eval()
+    tf_idct = dct_ops.idct(signals, type=dct_type, norm=norm).eval()
     self.assertAllClose(np_idct, tf_idct, atol=atol, rtol=rtol)
     if fftpack:
       scipy_dct = fftpack.dct(signals, type=dct_type, norm=norm)
@@ -116,9 +116,9 @@ class DCTOpsTest(parameterized.TestCase, test.TestCase):
       scipy_idct = fftpack.idct(signals, type=dct_type, norm=norm)
       self.assertAllClose(scipy_idct, tf_idct, atol=atol, rtol=rtol)
     # Verify inverse(forward(s)) == s, up to a normalization factor.
-    tf_idct_dct = spectral_ops.idct(
+    tf_idct_dct = dct_ops.idct(
         tf_dct, type=dct_type, norm=norm).eval()
-    tf_dct_idct = spectral_ops.dct(
+    tf_dct_idct = dct_ops.dct(
         tf_idct, type=dct_type, norm=norm).eval()
     if norm is None:
       if dct_type == 1:
@@ -147,20 +147,20 @@ class DCTOpsTest(parameterized.TestCase, test.TestCase):
     signals = np.random.rand(10)
     # Unsupported type.
     with self.assertRaises(ValueError):
-      spectral_ops.dct(signals, type=5)
+      dct_ops.dct(signals, type=5)
     # DCT-I normalization not implemented.
     with self.assertRaises(ValueError):
-      spectral_ops.dct(signals, type=1, norm="ortho")
+      dct_ops.dct(signals, type=1, norm="ortho")
     # DCT-I requires at least two inputs.
     with self.assertRaises(ValueError):
-      spectral_ops.dct(np.random.rand(1), type=1)
+      dct_ops.dct(np.random.rand(1), type=1)
     # Unknown normalization.
     with self.assertRaises(ValueError):
-      spectral_ops.dct(signals, norm="bad")
+      dct_ops.dct(signals, norm="bad")
     with self.assertRaises(NotImplementedError):
-      spectral_ops.dct(signals, n=10)
+      dct_ops.dct(signals, n=10)
     with self.assertRaises(NotImplementedError):
-      spectral_ops.dct(signals, axis=0)
+      dct_ops.dct(signals, axis=0)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
similarity index 98%
rename from tensorflow/python/kernel_tests/fft_ops_test.py
rename to tensorflow/python/kernel_tests/signal/fft_ops_test.py
index 8592550f99..3eeecc12a8 100644
--- a/tensorflow/python/kernel_tests/fft_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
@@ -29,8 +29,8 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_spectral_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import spectral_ops
 from tensorflow.python.ops import spectral_ops_test_util
+from tensorflow.python.ops.signal import fft_ops
 from tensorflow.python.platform import test
 
 VALID_FFT_RANKS = (1, 2, 3)
@@ -139,21 +139,21 @@ class FFTOpsTest(BaseFFTOpsTest):
 
   def _tfFFTForRank(self, rank):
     if rank == 1:
-      return spectral_ops.fft
+      return fft_ops.fft
     elif rank == 2:
-      return spectral_ops.fft2d
+      return fft_ops.fft2d
     elif rank == 3:
-      return spectral_ops.fft3d
+      return fft_ops.fft3d
     else:
       raise ValueError("invalid rank")
 
   def _tfIFFTForRank(self, rank):
     if rank == 1:
-      return spectral_ops.ifft
+      return fft_ops.ifft
     elif rank == 2:
-      return spectral_ops.ifft2d
+      return fft_ops.ifft2d
     elif rank == 3:
-      return spectral_ops.ifft3d
+      return fft_ops.ifft3d
     else:
       raise ValueError("invalid rank")
 
@@ -312,21 +312,21 @@ class RFFTOpsTest(BaseFFTOpsTest):
 
   def _tfFFTForRank(self, rank):
     if rank == 1:
-      return spectral_ops.rfft
+      return fft_ops.rfft
     elif rank == 2:
-      return spectral_ops.rfft2d
+      return fft_ops.rfft2d
     elif rank == 3:
-      return spectral_ops.rfft3d
+      return fft_ops.rfft3d
     else:
       raise ValueError("invalid rank")
 
   def _tfIFFTForRank(self, rank):
     if rank == 1:
-      return spectral_ops.irfft
+      return fft_ops.irfft
     elif rank == 2:
-      return spectral_ops.irfft2d
+      return fft_ops.irfft2d
     elif rank == 3:
-      return spectral_ops.irfft3d
+      return fft_ops.irfft3d
     else:
       raise ValueError("invalid rank")
 
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index 4f0fb54dca..53c0709e32 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -51,7 +51,6 @@ from tensorflow.python.ops import math_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import spectral_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops.unconnected_gradients import UnconnectedGradients
 from tensorflow.python.platform import tf_logging as logging
diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD
index c7314d7774..5df2d6b838 100644
--- a/tensorflow/python/ops/linalg/BUILD
+++ b/tensorflow/python/ops/linalg/BUILD
@@ -18,6 +18,7 @@ py_library(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python/ops/signal",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/python/ops/linalg/linear_operator_circulant.py b/tensorflow/python/ops/linalg/linear_operator_circulant.py
index 021ef47383..09f0c518e7 100644
--- a/tensorflow/python/ops/linalg/linear_operator_circulant.py
+++ b/tensorflow/python/ops/linalg/linear_operator_circulant.py
@@ -30,6 +30,7 @@ from tensorflow.python.ops.distributions import util as distribution_util
 from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_util
+from tensorflow.python.ops.signal import fft_ops
 from tensorflow.python.util.tf_export import tf_export
 
 __all__ = [
@@ -39,8 +40,8 @@ __all__ = [
 ]
 
 # Different FFT Ops will be used for different block depths.
-_FFT_OP = {1: math_ops.fft, 2: math_ops.fft2d, 3: math_ops.fft3d}
-_IFFT_OP = {1: math_ops.ifft, 2: math_ops.ifft2d, 3: math_ops.ifft3d}
+_FFT_OP = {1: fft_ops.fft, 2: fft_ops.fft2d, 3: fft_ops.fft3d}
+_IFFT_OP = {1: fft_ops.ifft, 2: fft_ops.ifft2d, 3: fft_ops.ifft3d}
 
 # This is the only dtype allowed with fft ops.
 # TODO(langmore) Add other types once available.
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 79387bd99d..a769c0dff4 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -36,7 +36,6 @@ from tensorflow.python.ops import gen_data_flow_ops
 from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gen_sparse_ops
-from tensorflow.python.ops import gen_spectral_ops
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_math_ops import *
@@ -3417,13 +3416,3 @@ def bessel_i1e(x, name=None):
           indices=x.indices, values=x_i1e, dense_shape=x.dense_shape)
     else:
       return gen_math_ops.bessel_i1e(x, name=name)
-
-
-# FFT ops were moved to tf.spectral. tf.fft symbols were part of the TensorFlow
-# 1.0 API so we leave these here for backwards compatibility.
-fft = gen_spectral_ops.fft
-ifft = gen_spectral_ops.ifft
-fft2d = gen_spectral_ops.fft2d
-ifft2d = gen_spectral_ops.ifft2d
-fft3d = gen_spectral_ops.fft3d
-ifft3d = gen_spectral_ops.ifft3d
diff --git a/tensorflow/python/ops/signal/BUILD b/tensorflow/python/ops/signal/BUILD
index 0d04dc0c1b..da2bf9c1d2 100644
--- a/tensorflow/python/ops/signal/BUILD
+++ b/tensorflow/python/ops/signal/BUILD
@@ -6,16 +6,29 @@ exports_files(["LICENSE"])
 
 py_library(
     name = "signal",
-    srcs = glob(["*.py"]),
+    srcs = [
+        "dct_ops.py",
+        "fft_ops.py",
+        "mel_ops.py",
+        "mfcc_ops.py",
+        "reconstruction_ops.py",
+        "shape_ops.py",
+        "signal.py",
+        "spectral_ops.py",
+        "util_ops.py",
+        "window_ops.py",
+    ],
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
-        "//tensorflow/python:spectral_ops",
+        "//tensorflow/python:spectral_ops_gen",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/ops/spectral_ops.py b/tensorflow/python/ops/signal/dct_ops.py
similarity index 53%
rename from tensorflow/python/ops/spectral_ops.py
rename to tensorflow/python/ops/signal/dct_ops.py
index 9d914513f1..d042c95c04 100644
--- a/tensorflow/python/ops/spectral_ops.py
+++ b/tensorflow/python/ops/signal/dct_ops.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Spectral operators (e.g. DCT, FFT, RFFT)."""
+"""Discrete Cosine Transform ops."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -22,149 +22,12 @@ import math as _math
 from tensorflow.python.framework import dtypes as _dtypes
 from tensorflow.python.framework import ops as _ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util as _tensor_util
 from tensorflow.python.ops import array_ops as _array_ops
-from tensorflow.python.ops import gen_spectral_ops
 from tensorflow.python.ops import math_ops as _math_ops
+from tensorflow.python.ops.signal import fft_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _infer_fft_length_for_rfft(input_tensor, fft_rank):
-  """Infers the `fft_length` argument for a `rank` RFFT from `input_tensor`."""
-  # A TensorShape for the inner fft_rank dimensions.
-  fft_shape = input_tensor.get_shape()[-fft_rank:]
-
-  # If any dim is unknown, fall back to tensor-based math.
-  if not fft_shape.is_fully_defined():
-    return _array_ops.shape(input_tensor)[-fft_rank:]
-
-  # Otherwise, return a constant.
-  return _ops.convert_to_tensor(fft_shape.as_list(), _dtypes.int32)
-
-
-def _infer_fft_length_for_irfft(input_tensor, fft_rank):
-  """Infers the `fft_length` argument for a `rank` IRFFT from `input_tensor`."""
-  # A TensorShape for the inner fft_rank dimensions.
-  fft_shape = input_tensor.get_shape()[-fft_rank:]
-
-  # If any dim is unknown, fall back to tensor-based math.
-  if not fft_shape.is_fully_defined():
-    fft_length = _array_ops.unstack(_array_ops.shape(input_tensor)[-fft_rank:])
-    fft_length[-1] = _math_ops.maximum(0, 2 * (fft_length[-1] - 1))
-    return _array_ops.stack(fft_length)
-
-  # Otherwise, return a constant.
-  fft_length = fft_shape.as_list()
-  if fft_length:
-    fft_length[-1] = max(0, 2 * (fft_length[-1] - 1))
-  return _ops.convert_to_tensor(fft_length, _dtypes.int32)
-
-
-def _maybe_pad_for_rfft(input_tensor, fft_rank, fft_length, is_reverse=False):
-  """Pads `input_tensor` to `fft_length` on its inner-most `fft_rank` dims."""
-  fft_shape = _tensor_util.constant_value_as_shape(fft_length)
-
-  # Edge case: skip padding empty tensors.
-  if (input_tensor.shape.ndims is not None and
-      any(dim.value == 0 for dim in input_tensor.shape.dims)):
-    return input_tensor
-
-  # If we know the shapes ahead of time, we can either skip or pre-compute the
-  # appropriate paddings. Otherwise, fall back to computing paddings in
-  # TensorFlow.
-  if fft_shape.is_fully_defined() and input_tensor.shape.ndims is not None:
-    # Slice the last FFT-rank dimensions from input_tensor's shape.
-    input_fft_shape = input_tensor.shape[-fft_shape.ndims:]
-
-    if input_fft_shape.is_fully_defined():
-      # In reverse, we only pad the inner-most dimension to fft_length / 2 + 1.
-      if is_reverse:
-        fft_shape = fft_shape[:-1].concatenate(
-            fft_shape.dims[-1].value // 2 + 1)
-
-      paddings = [[0, max(fft_dim.value - input_dim.value, 0)]
-                  for fft_dim, input_dim in zip(
-                      fft_shape.dims, input_fft_shape.dims)]
-      if any(pad > 0 for _, pad in paddings):
-        outer_paddings = [[0, 0]] * max((input_tensor.shape.ndims -
-                                         fft_shape.ndims), 0)
-        return _array_ops.pad(input_tensor, outer_paddings + paddings)
-      return input_tensor
-
-  # If we can't determine the paddings ahead of time, then we have to pad. If
-  # the paddings end up as zero, tf.pad has a special-case that does no work.
-  input_rank = _array_ops.rank(input_tensor)
-  input_fft_shape = _array_ops.shape(input_tensor)[-fft_rank:]
-  outer_dims = _math_ops.maximum(0, input_rank - fft_rank)
-  outer_paddings = _array_ops.zeros([outer_dims], fft_length.dtype)
-  # In reverse, we only pad the inner-most dimension to fft_length / 2 + 1.
-  if is_reverse:
-    fft_length = _array_ops.concat([fft_length[:-1],
-                                    fft_length[-1:] // 2 + 1], 0)
-  fft_paddings = _math_ops.maximum(0, fft_length - input_fft_shape)
-  paddings = _array_ops.concat([outer_paddings, fft_paddings], 0)
-  paddings = _array_ops.stack([_array_ops.zeros_like(paddings), paddings],
-                              axis=1)
-  return _array_ops.pad(input_tensor, paddings)
-
-
-def _rfft_wrapper(fft_fn, fft_rank, default_name):
-  """Wrapper around gen_spectral_ops.rfft* that infers fft_length argument."""
-
-  def _rfft(input_tensor, fft_length=None, name=None):
-    with _ops.name_scope(name, default_name,
-                         [input_tensor, fft_length]) as name:
-      input_tensor = _ops.convert_to_tensor(input_tensor, _dtypes.float32)
-      input_tensor.shape.with_rank_at_least(fft_rank)
-      if fft_length is None:
-        fft_length = _infer_fft_length_for_rfft(input_tensor, fft_rank)
-      else:
-        fft_length = _ops.convert_to_tensor(fft_length, _dtypes.int32)
-      input_tensor = _maybe_pad_for_rfft(input_tensor, fft_rank, fft_length)
-      return fft_fn(input_tensor, fft_length, name)
-  _rfft.__doc__ = fft_fn.__doc__
-  return _rfft
-
-
-def _irfft_wrapper(ifft_fn, fft_rank, default_name):
-  """Wrapper around gen_spectral_ops.irfft* that infers fft_length argument."""
-
-  def _irfft(input_tensor, fft_length=None, name=None):
-    with _ops.name_scope(name, default_name,
-                         [input_tensor, fft_length]) as name:
-      input_tensor = _ops.convert_to_tensor(input_tensor, _dtypes.complex64)
-      input_tensor.shape.with_rank_at_least(fft_rank)
-      if fft_length is None:
-        fft_length = _infer_fft_length_for_irfft(input_tensor, fft_rank)
-      else:
-        fft_length = _ops.convert_to_tensor(fft_length, _dtypes.int32)
-      input_tensor = _maybe_pad_for_rfft(input_tensor, fft_rank, fft_length,
-                                         is_reverse=True)
-      return ifft_fn(input_tensor, fft_length, name)
-  _irfft.__doc__ = ifft_fn.__doc__
-  return _irfft
-
-
-fft = gen_spectral_ops.fft
-ifft = gen_spectral_ops.ifft
-fft2d = gen_spectral_ops.fft2d
-ifft2d = gen_spectral_ops.ifft2d
-fft3d = gen_spectral_ops.fft3d
-ifft3d = gen_spectral_ops.ifft3d
-rfft = _rfft_wrapper(gen_spectral_ops.rfft, 1, "rfft")
-tf_export("spectral.rfft")(rfft)
-irfft = _irfft_wrapper(gen_spectral_ops.irfft, 1, "irfft")
-tf_export("spectral.irfft")(irfft)
-rfft2d = _rfft_wrapper(gen_spectral_ops.rfft2d, 2, "rfft2d")
-tf_export("spectral.rfft2d")(rfft2d)
-irfft2d = _irfft_wrapper(gen_spectral_ops.irfft2d, 2, "irfft2d")
-tf_export("spectral.irfft2d")(irfft2d)
-rfft3d = _rfft_wrapper(gen_spectral_ops.rfft3d, 3, "rfft3d")
-tf_export("spectral.rfft3d")(rfft3d)
-irfft3d = _irfft_wrapper(gen_spectral_ops.irfft3d, 3, "irfft3d")
-tf_export("spectral.irfft3d")(irfft3d)
-
-
 def _validate_dct_arguments(input_tensor, dct_type, n, axis, norm):
   """Checks that DCT/IDCT arguments are compatible and well formed."""
   if n is not None:
@@ -186,7 +49,7 @@ def _validate_dct_arguments(input_tensor, dct_type, n, axis, norm):
 
 
 # TODO(rjryan): Implement `n` and `axis` parameters.
-@tf_export("spectral.dct")
+@tf_export("signal.dct", v1=["signal.dct", "spectral.dct"])
 def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disable=redefined-builtin
   """Computes the 1D [Discrete Cosine Transform (DCT)][dct] of `input`.
 
@@ -235,7 +98,7 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
 
     if type == 1:
       dct1_input = _array_ops.concat([input, input[..., -2:0:-1]], axis=-1)
-      dct1 = _math_ops.real(rfft(dct1_input))
+      dct1 = _math_ops.real(fft_ops.rfft(dct1_input))
       return dct1
 
     if type == 2:
@@ -247,7 +110,8 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
       # TODO(rjryan): Benchmark performance and memory usage of the various
       # approaches to computing a DCT via the RFFT.
       dct2 = _math_ops.real(
-          rfft(input, fft_length=[2 * axis_dim])[..., :axis_dim] * scale)
+          fft_ops.rfft(
+              input, fft_length=[2 * axis_dim])[..., :axis_dim] * scale)
 
       if norm == "ortho":
         n1 = 0.5 * _math_ops.rsqrt(axis_dim_float)
@@ -277,7 +141,7 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
               _math_ops.range(axis_dim_float) * _math.pi * 0.5 /
               axis_dim_float))
       dct3 = _math_ops.real(
-          irfft(
+          fft_ops.irfft(
               scale * _math_ops.complex(input, 0.0),
               fft_length=[2 * axis_dim]))[..., :axis_dim]
 
@@ -285,7 +149,7 @@ def dct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disabl
 
 
 # TODO(rjryan): Implement `n` and `axis` parameters.
-@tf_export("spectral.idct")
+@tf_export("signal.idct", v1=["signal.idct", "spectral.idct"])
 def idct(input, type=2, n=None, axis=-1, norm=None, name=None):  # pylint: disable=redefined-builtin
   """Computes the 1D [Inverse Discrete Cosine Transform (DCT)][idct] of `input`.
 
diff --git a/tensorflow/python/ops/signal/fft_ops.py b/tensorflow/python/ops/signal/fft_ops.py
new file mode 100644
index 0000000000..2d14b2bbd7
--- /dev/null
+++ b/tensorflow/python/ops/signal/fft_ops.py
@@ -0,0 +1,330 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Fast-Fourier Transform ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes as _dtypes
+from tensorflow.python.framework import ops as _ops
+from tensorflow.python.framework import tensor_util as _tensor_util
+from tensorflow.python.ops import array_ops as _array_ops
+from tensorflow.python.ops import gen_spectral_ops
+from tensorflow.python.ops import math_ops as _math_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+def _infer_fft_length_for_rfft(input_tensor, fft_rank):
+  """Infers the `fft_length` argument for a `rank` RFFT from `input_tensor`."""
+  # A TensorShape for the inner fft_rank dimensions.
+  fft_shape = input_tensor.get_shape()[-fft_rank:]
+
+  # If any dim is unknown, fall back to tensor-based math.
+  if not fft_shape.is_fully_defined():
+    return _array_ops.shape(input_tensor)[-fft_rank:]
+
+  # Otherwise, return a constant.
+  return _ops.convert_to_tensor(fft_shape.as_list(), _dtypes.int32)
+
+
+def _infer_fft_length_for_irfft(input_tensor, fft_rank):
+  """Infers the `fft_length` argument for a `rank` IRFFT from `input_tensor`."""
+  # A TensorShape for the inner fft_rank dimensions.
+  fft_shape = input_tensor.get_shape()[-fft_rank:]
+
+  # If any dim is unknown, fall back to tensor-based math.
+  if not fft_shape.is_fully_defined():
+    fft_length = _array_ops.unstack(_array_ops.shape(input_tensor)[-fft_rank:])
+    fft_length[-1] = _math_ops.maximum(0, 2 * (fft_length[-1] - 1))
+    return _array_ops.stack(fft_length)
+
+  # Otherwise, return a constant.
+  fft_length = fft_shape.as_list()
+  if fft_length:
+    fft_length[-1] = max(0, 2 * (fft_length[-1] - 1))
+  return _ops.convert_to_tensor(fft_length, _dtypes.int32)
+
+
+def _maybe_pad_for_rfft(input_tensor, fft_rank, fft_length, is_reverse=False):
+  """Pads `input_tensor` to `fft_length` on its inner-most `fft_rank` dims."""
+  fft_shape = _tensor_util.constant_value_as_shape(fft_length)
+
+  # Edge case: skip padding empty tensors.
+  if (input_tensor.shape.ndims is not None and
+      any(dim.value == 0 for dim in input_tensor.shape.dims)):
+    return input_tensor
+
+  # If we know the shapes ahead of time, we can either skip or pre-compute the
+  # appropriate paddings. Otherwise, fall back to computing paddings in
+  # TensorFlow.
+  if fft_shape.is_fully_defined() and input_tensor.shape.ndims is not None:
+    # Slice the last FFT-rank dimensions from input_tensor's shape.
+    input_fft_shape = input_tensor.shape[-fft_shape.ndims:]
+
+    if input_fft_shape.is_fully_defined():
+      # In reverse, we only pad the inner-most dimension to fft_length / 2 + 1.
+      if is_reverse:
+        fft_shape = fft_shape[:-1].concatenate(
+            fft_shape.dims[-1].value // 2 + 1)
+
+      paddings = [[0, max(fft_dim.value - input_dim.value, 0)]
+                  for fft_dim, input_dim in zip(
+                      fft_shape.dims, input_fft_shape.dims)]
+      if any(pad > 0 for _, pad in paddings):
+        outer_paddings = [[0, 0]] * max((input_tensor.shape.ndims -
+                                         fft_shape.ndims), 0)
+        return _array_ops.pad(input_tensor, outer_paddings + paddings)
+      return input_tensor
+
+  # If we can't determine the paddings ahead of time, then we have to pad. If
+  # the paddings end up as zero, tf.pad has a special-case that does no work.
+  input_rank = _array_ops.rank(input_tensor)
+  input_fft_shape = _array_ops.shape(input_tensor)[-fft_rank:]
+  outer_dims = _math_ops.maximum(0, input_rank - fft_rank)
+  outer_paddings = _array_ops.zeros([outer_dims], fft_length.dtype)
+  # In reverse, we only pad the inner-most dimension to fft_length / 2 + 1.
+  if is_reverse:
+    fft_length = _array_ops.concat([fft_length[:-1],
+                                    fft_length[-1:] // 2 + 1], 0)
+  fft_paddings = _math_ops.maximum(0, fft_length - input_fft_shape)
+  paddings = _array_ops.concat([outer_paddings, fft_paddings], 0)
+  paddings = _array_ops.stack([_array_ops.zeros_like(paddings), paddings],
+                              axis=1)
+  return _array_ops.pad(input_tensor, paddings)
+
+
+def _rfft_wrapper(fft_fn, fft_rank, default_name):
+  """Wrapper around gen_spectral_ops.rfft* that infers fft_length argument."""
+
+  def _rfft(input_tensor, fft_length=None, name=None):
+    """Wrapper around gen_spectral_ops.rfft* that infers fft_length argument."""
+    with _ops.name_scope(name, default_name,
+                         [input_tensor, fft_length]) as name:
+      input_tensor = _ops.convert_to_tensor(input_tensor, _dtypes.float32)
+      input_tensor.shape.with_rank_at_least(fft_rank)
+      if fft_length is None:
+        fft_length = _infer_fft_length_for_rfft(input_tensor, fft_rank)
+      else:
+        fft_length = _ops.convert_to_tensor(fft_length, _dtypes.int32)
+      input_tensor = _maybe_pad_for_rfft(input_tensor, fft_rank, fft_length)
+      return fft_fn(input_tensor, fft_length, name)
+  _rfft.__doc__ = fft_fn.__doc__
+  return _rfft
+
+
+def _irfft_wrapper(ifft_fn, fft_rank, default_name):
+  """Wrapper around gen_spectral_ops.irfft* that infers fft_length argument."""
+
+  def _irfft(input_tensor, fft_length=None, name=None):
+    """Wrapper irfft* that infers fft_length argument."""
+    with _ops.name_scope(name, default_name,
+                         [input_tensor, fft_length]) as name:
+      input_tensor = _ops.convert_to_tensor(input_tensor, _dtypes.complex64)
+      input_tensor.shape.with_rank_at_least(fft_rank)
+      if fft_length is None:
+        fft_length = _infer_fft_length_for_irfft(input_tensor, fft_rank)
+      else:
+        fft_length = _ops.convert_to_tensor(fft_length, _dtypes.int32)
+      input_tensor = _maybe_pad_for_rfft(input_tensor, fft_rank, fft_length,
+                                         is_reverse=True)
+      return ifft_fn(input_tensor, fft_length, name)
+  _irfft.__doc__ = ifft_fn.__doc__
+  return _irfft
+
+
+# FFT/IFFT 1/2/3D are exported via
+# third_party/tensorflow/core/api_def/python_api/
+fft = gen_spectral_ops.fft
+ifft = gen_spectral_ops.ifft
+fft2d = gen_spectral_ops.fft2d
+ifft2d = gen_spectral_ops.ifft2d
+fft3d = gen_spectral_ops.fft3d
+ifft3d = gen_spectral_ops.ifft3d
+rfft = _rfft_wrapper(gen_spectral_ops.rfft, 1, "rfft")
+tf_export("signal.rfft", v1=["signal.rfft", "spectral.rfft"])(rfft)
+irfft = _irfft_wrapper(gen_spectral_ops.irfft, 1, "irfft")
+tf_export("signal.irfft", v1=["signal.irfft", "spectral.irfft"])(irfft)
+rfft2d = _rfft_wrapper(gen_spectral_ops.rfft2d, 2, "rfft2d")
+tf_export("signal.rfft2d", v1=["signal.rfft2d", "spectral.rfft2d"])(rfft2d)
+irfft2d = _irfft_wrapper(gen_spectral_ops.irfft2d, 2, "irfft2d")
+tf_export("signal.irfft2d", v1=["signal.irfft2d", "spectral.irfft2d"])(irfft2d)
+rfft3d = _rfft_wrapper(gen_spectral_ops.rfft3d, 3, "rfft3d")
+tf_export("signal.rfft3d", v1=["signal.rfft3d", "spectral.rfft3d"])(rfft3d)
+irfft3d = _irfft_wrapper(gen_spectral_ops.irfft3d, 3, "irfft3d")
+tf_export("signal.irfft3d", v1=["signal.irfft3d", "spectral.irfft3d"])(irfft3d)
+
+
+def _fft_size_for_grad(grad, rank):
+  return _math_ops.reduce_prod(_array_ops.shape(grad)[-rank:])
+
+
+@_ops.RegisterGradient("FFT")
+def _fft_grad(_, grad):
+  size = _math_ops.cast(_fft_size_for_grad(grad, 1), grad.dtype)
+  return ifft(grad) * size
+
+
+@_ops.RegisterGradient("IFFT")
+def _ifft_grad(_, grad):
+  rsize = _math_ops.cast(
+      1. / _math_ops.cast(_fft_size_for_grad(grad, 1), grad.dtype.real_dtype),
+      grad.dtype)
+  return fft(grad) * rsize
+
+
+@_ops.RegisterGradient("FFT2D")
+def _fft2d_grad(_, grad):
+  size = _math_ops.cast(_fft_size_for_grad(grad, 2), grad.dtype)
+  return ifft2d(grad) * size
+
+
+@_ops.RegisterGradient("IFFT2D")
+def _ifft2d_grad(_, grad):
+  rsize = _math_ops.cast(
+      1. / _math_ops.cast(_fft_size_for_grad(grad, 2), grad.dtype.real_dtype),
+      grad.dtype)
+  return fft2d(grad) * rsize
+
+
+@_ops.RegisterGradient("FFT3D")
+def _fft3d_grad(_, grad):
+  size = _math_ops.cast(_fft_size_for_grad(grad, 3), grad.dtype)
+  return ifft3d(grad) * size
+
+
+@_ops.RegisterGradient("IFFT3D")
+def _ifft3d_grad(_, grad):
+  rsize = _math_ops.cast(
+      1. / _math_ops.cast(_fft_size_for_grad(grad, 3), grad.dtype.real_dtype),
+      grad.dtype)
+  return fft3d(grad) * rsize
+
+
+def _rfft_grad_helper(rank, irfft_fn):
+  """Returns a gradient function for an RFFT of the provided rank."""
+  # Can't happen because we don't register a gradient for RFFT3D.
+  assert rank in (1, 2), "Gradient for RFFT3D is not implemented."
+
+  def _grad(op, grad):
+    """A gradient function for RFFT with the provided `rank` and `irfft_fn`."""
+    fft_length = op.inputs[1]
+    input_shape = _array_ops.shape(op.inputs[0])
+    is_even = _math_ops.cast(1 - (fft_length[-1] % 2), _dtypes.complex64)
+
+    def _tile_for_broadcasting(matrix, t):
+      expanded = _array_ops.reshape(
+          matrix,
+          _array_ops.concat([
+              _array_ops.ones([_array_ops.rank(t) - 2], _dtypes.int32),
+              _array_ops.shape(matrix)
+          ], 0))
+      return _array_ops.tile(
+          expanded, _array_ops.concat([_array_ops.shape(t)[:-2], [1, 1]], 0))
+
+    def _mask_matrix(length):
+      """Computes t_n = exp(sqrt(-1) * pi * n^2 / line_len)."""
+      # TODO(rjryan): Speed up computation of twiddle factors using the
+      # following recurrence relation and cache them across invocations of RFFT.
+      #
+      # t_n = exp(sqrt(-1) * pi * n^2 / line_len)
+      # for n = 0, 1,..., line_len-1.
+      # For n > 2, use t_n = t_{n-1}^2 / t_{n-2} * t_1^2
+      a = _array_ops.tile(
+          _array_ops.expand_dims(_math_ops.range(length), 0), (length, 1))
+      b = _array_ops.transpose(a, [1, 0])
+      return _math_ops.exp(
+          -2j * np.pi * _math_ops.cast(a * b, _dtypes.complex64) /
+          _math_ops.cast(length, _dtypes.complex64))
+
+    def _ymask(length):
+      """A sequence of [1+0j, -1+0j, 1+0j, -1+0j, ...] with length `length`."""
+      return _math_ops.cast(1 - 2 * (_math_ops.range(length) % 2),
+                            _dtypes.complex64)
+
+    y0 = grad[..., 0:1]
+    if rank == 1:
+      ym = grad[..., -1:]
+      extra_terms = y0 + is_even * ym * _ymask(input_shape[-1])
+    elif rank == 2:
+      # Create a mask matrix for y0 and ym.
+      base_mask = _mask_matrix(input_shape[-2])
+
+      # Tile base_mask to match y0 in shape so that we can batch-matmul the
+      # inner 2 dimensions.
+      tiled_mask = _tile_for_broadcasting(base_mask, y0)
+
+      y0_term = _math_ops.matmul(tiled_mask, _math_ops.conj(y0))
+      extra_terms = y0_term
+
+      ym = grad[..., -1:]
+      ym_term = _math_ops.matmul(tiled_mask, _math_ops.conj(ym))
+
+      inner_dim = input_shape[-1]
+      ym_term = _array_ops.tile(
+          ym_term,
+          _array_ops.concat([
+              _array_ops.ones([_array_ops.rank(grad) - 1], _dtypes.int32),
+              [inner_dim]
+          ], 0)) * _ymask(inner_dim)
+
+      extra_terms += is_even * ym_term
+
+    # The gradient of RFFT is the IRFFT of the incoming gradient times a scaling
+    # factor, plus some additional terms to make up for the components dropped
+    # due to Hermitian symmetry.
+    input_size = _math_ops.to_float(_fft_size_for_grad(op.inputs[0], rank))
+    the_irfft = irfft_fn(grad, fft_length)
+    return 0.5 * (the_irfft * input_size + _math_ops.real(extra_terms)), None
+
+  return _grad
+
+
+def _irfft_grad_helper(rank, rfft_fn):
+  """Returns a gradient function for an IRFFT of the provided rank."""
+  # Can't happen because we don't register a gradient for IRFFT3D.
+  assert rank in (1, 2), "Gradient for IRFFT3D is not implemented."
+
+  def _grad(op, grad):
+    """A gradient function for IRFFT with the provided `rank` and `rfft_fn`."""
+    # Generate a simple mask like [1.0, 2.0, ..., 2.0, 1.0] for even-length FFTs
+    # and [1.0, 2.0, ..., 2.0] for odd-length FFTs. To reduce extra ops in the
+    # graph we special-case the situation where the FFT length and last
+    # dimension of the input are known at graph construction time.
+    fft_length = op.inputs[1]
+    is_odd = _math_ops.mod(fft_length[-1], 2)
+    input_last_dimension = _array_ops.shape(op.inputs[0])[-1]
+    mask = _array_ops.concat(
+        [[1.0], 2.0 * _array_ops.ones([input_last_dimension - 2 + is_odd]),
+         _array_ops.ones([1 - is_odd])], 0)
+
+    rsize = _math_ops.reciprocal(_math_ops.to_float(
+        _fft_size_for_grad(grad, rank)))
+
+    # The gradient of IRFFT is the RFFT of the incoming gradient times a scaling
+    # factor and a mask. The mask scales the gradient for the Hermitian
+    # symmetric components of the RFFT by a factor of two, since these
+    # components are de-duplicated in the RFFT.
+    the_rfft = rfft_fn(grad, fft_length)
+    return the_rfft * _math_ops.cast(rsize * mask, _dtypes.complex64), None
+
+  return _grad
+
+
+_ops.RegisterGradient("RFFT")(_rfft_grad_helper(1, irfft))
+_ops.RegisterGradient("IRFFT")(_irfft_grad_helper(1, rfft))
+_ops.RegisterGradient("RFFT2D")(_rfft_grad_helper(2, irfft2d))
+_ops.RegisterGradient("IRFFT2D")(_irfft_grad_helper(2, rfft2d))
diff --git a/tensorflow/python/ops/signal/mfcc_ops.py b/tensorflow/python/ops/signal/mfcc_ops.py
index 6ae3b222ba..601409dea9 100644
--- a/tensorflow/python/ops/signal/mfcc_ops.py
+++ b/tensorflow/python/ops/signal/mfcc_ops.py
@@ -22,7 +22,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import spectral_ops
+from tensorflow.python.ops.signal import dct_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -106,5 +106,5 @@ def mfccs_from_log_mel_spectrograms(log_mel_spectrograms, name=None):
     else:
       num_mel_bins = array_ops.shape(log_mel_spectrograms)[-1]
 
-    dct2 = spectral_ops.dct(log_mel_spectrograms)
+    dct2 = dct_ops.dct(log_mel_spectrograms, type=2)
     return dct2 * math_ops.rsqrt(math_ops.to_float(num_mel_bins) * 2.0)
diff --git a/tensorflow/python/ops/signal/shape_ops.py b/tensorflow/python/ops/signal/shape_ops.py
index 02dd7c97e8..ae9c2ef28e 100644
--- a/tensorflow/python/ops/signal/shape_ops.py
+++ b/tensorflow/python/ops/signal/shape_ops.py
@@ -71,7 +71,7 @@ def frame(signal, frame_length, frame_step, pad_end=False, pad_value=0, axis=-1,
   ```python
   pcm = tf.placeholder(tf.float32, [None, 9152])
   frames = tf.signal.frame(pcm, 512, 180)
-  magspec = tf.abs(tf.spectral.rfft(frames, [512]))
+  magspec = tf.abs(tf.signal.rfft(frames, [512]))
   image = tf.expand_dims(magspec, 3)
   ```
 
diff --git a/tensorflow/python/ops/signal/signal.py b/tensorflow/python/ops/signal/signal.py
index 4ddd30533c..cdc4d1c191 100644
--- a/tensorflow/python/ops/signal/signal.py
+++ b/tensorflow/python/ops/signal/signal.py
@@ -39,6 +39,20 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import
+from tensorflow.python.ops.signal.dct_ops import dct
+from tensorflow.python.ops.signal.dct_ops import idct
+from tensorflow.python.ops.signal.fft_ops import fft
+from tensorflow.python.ops.signal.fft_ops import fft2d
+from tensorflow.python.ops.signal.fft_ops import fft3d
+from tensorflow.python.ops.signal.fft_ops import ifft
+from tensorflow.python.ops.signal.fft_ops import ifft2d
+from tensorflow.python.ops.signal.fft_ops import ifft3d
+from tensorflow.python.ops.signal.fft_ops import irfft
+from tensorflow.python.ops.signal.fft_ops import irfft2d
+from tensorflow.python.ops.signal.fft_ops import irfft3d
+from tensorflow.python.ops.signal.fft_ops import rfft
+from tensorflow.python.ops.signal.fft_ops import rfft2d
+from tensorflow.python.ops.signal.fft_ops import rfft3d
 from tensorflow.python.ops.signal.mel_ops import linear_to_mel_weight_matrix
 from tensorflow.python.ops.signal.mfcc_ops import mfccs_from_log_mel_spectrograms
 from tensorflow.python.ops.signal.reconstruction_ops import overlap_and_add
diff --git a/tensorflow/python/ops/signal/spectral_ops.py b/tensorflow/python/ops/signal/spectral_ops.py
index b0b7d964b9..f029e0a8b5 100644
--- a/tensorflow/python/ops/signal/spectral_ops.py
+++ b/tensorflow/python/ops/signal/spectral_ops.py
@@ -25,7 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import spectral_ops
+from tensorflow.python.ops.signal import fft_ops
 from tensorflow.python.ops.signal import reconstruction_ops
 from tensorflow.python.ops.signal import shape_ops
 from tensorflow.python.ops.signal import window_ops
@@ -86,9 +86,9 @@ def stft(signals, frame_length, frame_step, fft_length=None,
       window = window_fn(frame_length, dtype=framed_signals.dtype)
       framed_signals *= window
 
-    # spectral_ops.rfft produces the (fft_length/2 + 1) unique components of the
+    # fft_ops.rfft produces the (fft_length/2 + 1) unique components of the
     # FFT of the real windowed signals in framed_signals.
-    return spectral_ops.rfft(framed_signals, [fft_length])
+    return fft_ops.rfft(framed_signals, [fft_length])
 
 
 @tf_export('signal.inverse_stft_window_fn')
@@ -232,7 +232,7 @@ def inverse_stft(stfts,
       fft_length = ops.convert_to_tensor(fft_length, name='fft_length')
       fft_length.shape.assert_has_rank(0)
 
-    real_frames = spectral_ops.irfft(stfts, [fft_length])
+    real_frames = fft_ops.irfft(stfts, [fft_length])
 
     # frame_length may be larger or smaller than fft_length, so we pad or
     # truncate real_frames to frame_length.
diff --git a/tensorflow/python/ops/spectral_grad.py b/tensorflow/python/ops/spectral_grad.py
deleted file mode 100644
index 0af24114ac..0000000000
--- a/tensorflow/python/ops/spectral_grad.py
+++ /dev/null
@@ -1,185 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Gradients for operators defined in spectral_ops.py."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import spectral_ops
-
-
-def _FFTSizeForGrad(grad, rank):
-  return math_ops.reduce_prod(array_ops.shape(grad)[-rank:])
-
-
-@ops.RegisterGradient("FFT")
-def _FFTGrad(_, grad):
-  size = math_ops.cast(_FFTSizeForGrad(grad, 1), grad.dtype)
-  return spectral_ops.ifft(grad) * size
-
-
-@ops.RegisterGradient("IFFT")
-def _IFFTGrad(_, grad):
-  rsize = math_ops.cast(
-      1. / math_ops.cast(_FFTSizeForGrad(grad, 1), grad.dtype.real_dtype),
-      grad.dtype)
-  return spectral_ops.fft(grad) * rsize
-
-
-@ops.RegisterGradient("FFT2D")
-def _FFT2DGrad(_, grad):
-  size = math_ops.cast(_FFTSizeForGrad(grad, 2), grad.dtype)
-  return spectral_ops.ifft2d(grad) * size
-
-
-@ops.RegisterGradient("IFFT2D")
-def _IFFT2DGrad(_, grad):
-  rsize = math_ops.cast(
-      1. / math_ops.cast(_FFTSizeForGrad(grad, 2), grad.dtype.real_dtype),
-      grad.dtype)
-  return spectral_ops.fft2d(grad) * rsize
-
-
-@ops.RegisterGradient("FFT3D")
-def _FFT3DGrad(_, grad):
-  size = math_ops.cast(_FFTSizeForGrad(grad, 3), grad.dtype)
-  return spectral_ops.ifft3d(grad) * size
-
-
-@ops.RegisterGradient("IFFT3D")
-def _IFFT3DGrad(_, grad):
-  rsize = math_ops.cast(
-      1. / math_ops.cast(_FFTSizeForGrad(grad, 3), grad.dtype.real_dtype),
-      grad.dtype)
-  return spectral_ops.fft3d(grad) * rsize
-
-
-def _RFFTGradHelper(rank, irfft_fn):
-  """Returns a gradient function for an RFFT of the provided rank."""
-  # Can't happen because we don't register a gradient for RFFT3D.
-  assert rank in (1, 2), "Gradient for RFFT3D is not implemented."
-
-  def _Grad(op, grad):
-    """A gradient function for RFFT with the provided `rank` and `irfft_fn`."""
-    fft_length = op.inputs[1]
-    input_shape = array_ops.shape(op.inputs[0])
-    is_even = math_ops.cast(1 - (fft_length[-1] % 2), dtypes.complex64)
-
-    def _TileForBroadcasting(matrix, t):
-      expanded = array_ops.reshape(
-          matrix,
-          array_ops.concat([
-              array_ops.ones([array_ops.rank(t) - 2], dtypes.int32),
-              array_ops.shape(matrix)
-          ], 0))
-      return array_ops.tile(
-          expanded, array_ops.concat([array_ops.shape(t)[:-2], [1, 1]], 0))
-
-    def _MaskMatrix(length):
-      # TODO(rjryan): Speed up computation of twiddle factors using the
-      # following recurrence relation and cache them across invocations of RFFT.
-      #
-      # t_n = exp(sqrt(-1) * pi * n^2 / line_len)
-      # for n = 0, 1,..., line_len-1.
-      # For n > 2, use t_n = t_{n-1}^2 / t_{n-2} * t_1^2
-      a = array_ops.tile(
-          array_ops.expand_dims(math_ops.range(length), 0), (length, 1))
-      b = array_ops.transpose(a, [1, 0])
-      return math_ops.exp(-2j * np.pi * math_ops.cast(a * b, dtypes.complex64) /
-                          math_ops.cast(length, dtypes.complex64))
-
-    def _YMMask(length):
-      """A sequence of [1+0j, -1+0j, 1+0j, -1+0j, ...] with length `length`."""
-      return math_ops.cast(1 - 2 * (math_ops.range(length) % 2),
-                           dtypes.complex64)
-
-    y0 = grad[..., 0:1]
-    if rank == 1:
-      ym = grad[..., -1:]
-      extra_terms = y0 + is_even * ym * _YMMask(input_shape[-1])
-    elif rank == 2:
-      # Create a mask matrix for y0 and ym.
-      base_mask = _MaskMatrix(input_shape[-2])
-
-      # Tile base_mask to match y0 in shape so that we can batch-matmul the
-      # inner 2 dimensions.
-      tiled_mask = _TileForBroadcasting(base_mask, y0)
-
-      y0_term = math_ops.matmul(tiled_mask, math_ops.conj(y0))
-      extra_terms = y0_term
-
-      ym = grad[..., -1:]
-      ym_term = math_ops.matmul(tiled_mask, math_ops.conj(ym))
-
-      inner_dim = input_shape[-1]
-      ym_term = array_ops.tile(
-          ym_term,
-          array_ops.concat([
-              array_ops.ones([array_ops.rank(grad) - 1], dtypes.int32),
-              [inner_dim]
-          ], 0)) * _YMMask(inner_dim)
-
-      extra_terms += is_even * ym_term
-
-    # The gradient of RFFT is the IRFFT of the incoming gradient times a scaling
-    # factor, plus some additional terms to make up for the components dropped
-    # due to Hermitian symmetry.
-    input_size = math_ops.to_float(_FFTSizeForGrad(op.inputs[0], rank))
-    irfft = irfft_fn(grad, fft_length)
-    return 0.5 * (irfft * input_size + math_ops.real(extra_terms)), None
-
-  return _Grad
-
-
-def _IRFFTGradHelper(rank, rfft_fn):
-  """Returns a gradient function for an IRFFT of the provided rank."""
-  # Can't happen because we don't register a gradient for IRFFT3D.
-  assert rank in (1, 2), "Gradient for IRFFT3D is not implemented."
-
-  def _Grad(op, grad):
-    """A gradient function for IRFFT with the provided `rank` and `rfft_fn`."""
-    # Generate a simple mask like [1.0, 2.0, ..., 2.0, 1.0] for even-length FFTs
-    # and [1.0, 2.0, ..., 2.0] for odd-length FFTs. To reduce extra ops in the
-    # graph we special-case the situation where the FFT length and last
-    # dimension of the input are known at graph construction time.
-    fft_length = op.inputs[1]
-    is_odd = math_ops.mod(fft_length[-1], 2)
-    input_last_dimension = array_ops.shape(op.inputs[0])[-1]
-    mask = array_ops.concat(
-        [[1.0], 2.0 * array_ops.ones([input_last_dimension - 2 + is_odd]),
-         array_ops.ones([1 - is_odd])], 0)
-
-    rsize = math_ops.reciprocal(math_ops.to_float(_FFTSizeForGrad(grad, rank)))
-
-    # The gradient of IRFFT is the RFFT of the incoming gradient times a scaling
-    # factor and a mask. The mask scales the gradient for the Hermitian
-    # symmetric components of the RFFT by a factor of two, since these
-    # components are de-duplicated in the RFFT.
-    rfft = rfft_fn(grad, fft_length)
-    return rfft * math_ops.cast(rsize * mask, dtypes.complex64), None
-
-  return _Grad
-
-
-ops.RegisterGradient("RFFT")(_RFFTGradHelper(1, spectral_ops.irfft))
-ops.RegisterGradient("IRFFT")(_IRFFTGradHelper(1, spectral_ops.rfft))
-ops.RegisterGradient("RFFT2D")(_RFFTGradHelper(2, spectral_ops.irfft2d))
-ops.RegisterGradient("IRFFT2D")(_IRFFTGradHelper(2, spectral_ops.rfft2d))
diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py
index cc779e6901..03e491a315 100644
--- a/tensorflow/python/ops/standard_ops.py
+++ b/tensorflow/python/ops/standard_ops.py
@@ -31,7 +31,6 @@ from tensorflow.python.ops import manip_grad
 from tensorflow.python.ops import math_grad
 from tensorflow.python.ops import random_grad
 from tensorflow.python.ops import sparse_grad
-from tensorflow.python.ops import spectral_grad
 from tensorflow.python.ops import state_grad
 from tensorflow.python.ops import tensor_array_grad
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 60a5540b83..a9e8ed4578 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -74,7 +74,6 @@ TENSORFLOW_API_INIT_FILES = [
     "sets/__init__.py",
     "signal/__init__.py",
     "sparse/__init__.py",
-    "spectral/__init__.py",
     "strings/__init__.py",
     "summary/__init__.py",
     "sysconfig/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py
index 7613104749..0524b5e35c 100644
--- a/tensorflow/python/tools/api/generator/doc_srcs.py
+++ b/tensorflow/python/tools/api/generator/doc_srcs.py
@@ -58,7 +58,6 @@ _TENSORFLOW_DOC_SOURCES = {
     'sets': DocSource(docstring_module_name='ops.sets'),
     'signal': DocSource(docstring_module_name='ops.signal.signal'),
     'sparse': DocSource(docstring_module_name='ops.sparse_ops'),
-    'spectral': DocSource(docstring_module_name='ops.spectral_ops'),
     'strings': DocSource(docstring_module_name='ops.string_ops'),
     'sysconfig': DocSource(docstring_module_name='platform.sysconfig'),
     'test': DocSource(docstring_module_name='platform.test'),
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt
index 2c50c41f18..ea717b4d71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.signal.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.signal"
 tf_module {
+  member_method {
+    name: "dct"
+    argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], "
+  }
   member_method {
     name: "fft"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -24,6 +28,10 @@ tf_module {
     name: "hann_window"
     argspec: "args=[\'window_length\', \'periodic\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \"<dtype: \'float32\'>\", \'None\'], "
   }
+  member_method {
+    name: "idct"
+    argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], "
+  }
   member_method {
     name: "ifft"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -44,6 +52,18 @@ tf_module {
     name: "inverse_stft_window_fn"
     argspec: "args=[\'frame_step\', \'forward_window_fn\', \'name\'], varargs=None, keywords=None, defaults=[\'<function hann_window instance>\', \'None\'], "
   }
+  member_method {
+    name: "irfft"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "irfft2d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "irfft3d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "linear_to_mel_weight_matrix"
     argspec: "args=[\'num_mel_bins\', \'num_spectrogram_bins\', \'sample_rate\', \'lower_edge_hertz\', \'upper_edge_hertz\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'20\', \'129\', \'8000\', \'125.0\', \'3800.0\', \"<dtype: \'float32\'>\", \'None\'], "
@@ -56,6 +76,18 @@ tf_module {
     name: "overlap_and_add"
     argspec: "args=[\'signal\', \'frame_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "rfft"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "rfft2d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "rfft3d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "stft"
     argspec: "args=[\'signals\', \'frame_length\', \'frame_step\', \'fft_length\', \'window_fn\', \'pad_end\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'<function hann_window instance>\', \'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index e5331649d7..a2ac908195 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -388,10 +388,6 @@ tf_module {
     name: "sparse"
     mtype: "<type \'module\'>"
   }
-  member {
-    name: "spectral"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "string"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt
index 2c50c41f18..ea717b4d71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.signal.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.signal"
 tf_module {
+  member_method {
+    name: "dct"
+    argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], "
+  }
   member_method {
     name: "fft"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -24,6 +28,10 @@ tf_module {
     name: "hann_window"
     argspec: "args=[\'window_length\', \'periodic\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \"<dtype: \'float32\'>\", \'None\'], "
   }
+  member_method {
+    name: "idct"
+    argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], "
+  }
   member_method {
     name: "ifft"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -44,6 +52,18 @@ tf_module {
     name: "inverse_stft_window_fn"
     argspec: "args=[\'frame_step\', \'forward_window_fn\', \'name\'], varargs=None, keywords=None, defaults=[\'<function hann_window instance>\', \'None\'], "
   }
+  member_method {
+    name: "irfft"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "irfft2d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "irfft3d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "linear_to_mel_weight_matrix"
     argspec: "args=[\'num_mel_bins\', \'num_spectrogram_bins\', \'sample_rate\', \'lower_edge_hertz\', \'upper_edge_hertz\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'20\', \'129\', \'8000\', \'125.0\', \'3800.0\', \"<dtype: \'float32\'>\", \'None\'], "
@@ -56,6 +76,18 @@ tf_module {
     name: "overlap_and_add"
     argspec: "args=[\'signal\', \'frame_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "rfft"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "rfft2d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "rfft3d"
+    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "stft"
     argspec: "args=[\'signals\', \'frame_length\', \'frame_step\', \'fft_length\', \'window_fn\', \'pad_end\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'<function hann_window instance>\', \'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt
deleted file mode 100644
index b0f0783e30..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.spectral.pbtxt
+++ /dev/null
@@ -1,35 +0,0 @@
-path: "tensorflow.spectral"
-tf_module {
-  member_method {
-    name: "dct"
-    argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "idct"
-    argspec: "args=[\'input\', \'type\', \'n\', \'axis\', \'norm\', \'name\'], varargs=None, keywords=None, defaults=[\'2\', \'None\', \'-1\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "irfft"
-    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "irfft2d"
-    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "irfft3d"
-    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "rfft"
-    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "rfft2d"
-    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "rfft3d"
-    argspec: "args=[\'input_tensor\', \'fft_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-}
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index cf93bd2f53..de87f405bc 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -434,12 +434,20 @@ renames = {
     'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense',
     'tf.sparse_to_indicator': 'tf.sparse.to_indicator',
     'tf.sparse_transpose': 'tf.sparse.transpose',
+    'tf.spectral.dct': 'tf.signal.dct',
     'tf.spectral.fft': 'tf.signal.fft',
     'tf.spectral.fft2d': 'tf.signal.fft2d',
     'tf.spectral.fft3d': 'tf.signal.fft3d',
+    'tf.spectral.idct': 'tf.signal.idct',
     'tf.spectral.ifft': 'tf.signal.ifft',
     'tf.spectral.ifft2d': 'tf.signal.ifft2d',
     'tf.spectral.ifft3d': 'tf.signal.ifft3d',
+    'tf.spectral.irfft': 'tf.signal.irfft',
+    'tf.spectral.irfft2d': 'tf.signal.irfft2d',
+    'tf.spectral.irfft3d': 'tf.signal.irfft3d',
+    'tf.spectral.rfft': 'tf.signal.rfft',
+    'tf.spectral.rfft2d': 'tf.signal.rfft2d',
+    'tf.spectral.rfft3d': 'tf.signal.rfft3d',
     'tf.squared_difference': 'tf.math.squared_difference',
     'tf.string_join': 'tf.strings.join',
     'tf.string_strip': 'tf.strings.strip',
-- 
GitLab


From 05a5a85c3381f9886e61028c44745fbef2a870af Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 13:37:13 -0800
Subject: [PATCH 0336/1554] Don't load libraries requiring missing CPU
 features.

This CL attempts to read the platform strings from a given plugin shared object
before that shared object is loaded. It also moves the GetStrings function in
platform_strings_test.cc into a new source file so that it may be used by the
plugin loader.

PiperOrigin-RevId: 221680960
---
 tensorflow/core/BUILD                         | 21 ++++--
 tensorflow/core/framework/op_kernel.cc        | 56 ++++++++++++++--
 tensorflow/core/platform/platform_strings.cc  | 64 +++++++++++++++++++
 tensorflow/core/platform/platform_strings.h   | 19 ++++++
 .../core/platform/platform_strings_test.cc    | 36 +----------
 5 files changed, 151 insertions(+), 45 deletions(-)
 create mode 100644 tensorflow/core/platform/platform_strings.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3ee31e0b29..73e8db58a8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -300,6 +300,7 @@ filegroup(
         "platform/env_time.h",
         "platform/logging.h",
         "platform/macros.h",
+        "platform/platform_strings.h",
         "platform/types.h",
     ],
     visibility = ["//visibility:private"],
@@ -519,6 +520,19 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "platform_strings",
+    srcs = tf_platform_srcs([
+        "platform/platform_strings.cc",
+        "platform/platform_strings_computed.h",
+    ]),
+    hdrs = [
+        "platform/platform_strings.h",
+    ],
+    visibility = ["//tensorflow/core:__subpackages__"],
+    deps = [":lib"],
+)
+
 filegroup(
     name = "platform_other_hdrs",
     srcs = [
@@ -3412,12 +3426,6 @@ tf_cc_test(
     ],
 )
 
-cc_library(
-    name = "platform_strings",
-    srcs = ["platform/platform_strings_computed.h"],
-    hdrs = ["platform/platform_strings.h"],
-)
-
 tf_cc_test(
     name = "platform_strings_test",
     size = "small",
@@ -4879,6 +4887,7 @@ transitive_hdrs(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:platform_strings",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:stream_executor",
     ],
diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index 9e2d98b70a..1213e25116 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -39,9 +39,11 @@ limitations under the License.
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/platform_strings.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/ptr_util.h"
 
@@ -945,6 +947,44 @@ static const char kKernelLibPattern[] = "libtfkernel*.dylib";
 static const char kKernelLibPattern[] = "libtfkernel*.so";
 #endif
 
+#define FEATURE(x) \
+  { x, #x }
+
+// Returns Status::OK if the dynamic library at the given path is safe to
+// load with some level of confidence.
+static Status IsProbablySafeToLoad(const string& path) {
+  // A map of platform string to required CPU feature.
+  using port::CPUFeature;
+  static const auto* feature_map =
+      new std::map<string, std::pair<CPUFeature, string>>{
+          {"__AVX512VL__=1", FEATURE(CPUFeature::AVX512VL)},
+      };
+
+  std::vector<std::string> platform_strings;
+  int result = GetPlatformStrings(path, &platform_strings);
+  if (result) {
+    return Status(error::Code::UNKNOWN, strerror(result));
+  }
+  if (platform_strings.empty()) {
+    return Status(error::Code::FAILED_PRECONDITION,
+                  "Didn't find any platform strings");
+  }
+  std::vector<std::string> missing_features;
+  for (const auto& platform_string : platform_strings) {
+    const auto& entry = feature_map->find(platform_string);
+    if (entry != feature_map->end() &&
+        !port::TestCPUFeature(entry->second.first)) {
+      missing_features.emplace_back(entry->second.second);
+    }
+  }
+  if (!missing_features.empty()) {
+    string errmsg = "Missing CPU features: ";
+    errmsg.append(str_util::Join(missing_features, ", "));
+    return Status(errors::Code::FAILED_PRECONDITION, errmsg);
+  }
+  return Status::OK();
+}
+
 void LoadDynamicKernelsInternal() {
   Env* env = Env::Default();
   string bazel_kernel_dir = io::JoinPath(env->GetRunfilesDir(),
@@ -955,12 +995,18 @@ void LoadDynamicKernelsInternal() {
   Status s_kernel_dir = env->GetChildren(bazel_kernel_dir, &files);
   if (s_kernel_dir.ok()) {
     string dll_spec = io::JoinPath(bazel_kernel_dir, kKernelLibPattern);
-    for (const auto&  file : files) {
-      string fullpath =  io::JoinPath(bazel_kernel_dir, file);
+    for (const auto& file : files) {
+      string fullpath = io::JoinPath(bazel_kernel_dir, file);
       if (env->MatchPath(fullpath, dll_spec)) {
-        // TODO(gunan): Store the handles to the opened files.
-        void* unused_filehandle;
-        TF_CHECK_OK(env->LoadLibrary(fullpath.c_str(), &unused_filehandle));
+        Status s = IsProbablySafeToLoad(fullpath);
+        if (s.ok()) {
+          // TODO(gunan): Store the handles to the opened files.
+          void* unused_filehandle;
+          TF_CHECK_OK(env->LoadLibrary(fullpath.c_str(), &unused_filehandle));
+        } else {
+          LOG(WARNING) << "Not loading plugin library " << fullpath << ": "
+                       << s.error_message();
+        }
       }
     }
   }
diff --git a/tensorflow/core/platform/platform_strings.cc b/tensorflow/core/platform/platform_strings.cc
new file mode 100644
index 0000000000..c1852633d5
--- /dev/null
+++ b/tensorflow/core/platform/platform_strings.cc
@@ -0,0 +1,64 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/platform_strings.h"
+
+#include <cstdio>
+#include <cstring>
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+int GetPlatformStrings(const std::string& path,
+                       std::vector<std::string>* found) {
+  int result;
+  FILE* ifp = fopen(path.c_str(), "rb");
+  if (ifp != nullptr) {
+    static const char prefix[] = TF_PLAT_STR_MAGIC_PREFIX_;
+    int first_char = prefix[1];
+    int last_char = -1;
+    int c;
+    while ((c = getc(ifp)) != EOF) {
+      if (c == first_char && last_char == 0) {
+        int i = 2;
+        while (prefix[i] != 0 && (c = getc(ifp)) == prefix[i]) {
+          i++;
+        }
+        if (prefix[i] == 0) {
+          std::string str;
+          while ((c = getc(ifp)) != EOF && c != 0) {
+            str.push_back(c);
+          }
+          if (!str.empty()) {
+            found->push_back(str);
+          }
+        }
+      }
+      last_char = c;
+    }
+
+    result = (ferror(ifp) == 0) ? 0 : errno;
+    fclose(ifp);
+  } else {
+    result = errno;
+  }
+  return result;
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/platform_strings.h b/tensorflow/core/platform/platform_strings.h
index 24c2929e41..5b1dbd130e 100644
--- a/tensorflow/core/platform/platform_strings.h
+++ b/tensorflow/core/platform/platform_strings.h
@@ -24,6 +24,9 @@ limitations under the License.
 // built with the same options (or at least, the strings should be embedded in
 // the compilation unit built with the most restrictive options).
 
+// The platform strings embedded into a binary may be retrieved with the
+// GetPlatformStrings function.
+
 // Rationale:
 // We wish to load only those libraries that this CPU can execute.  For
 // example, we should not load a library compiled with avx256 instructions on a
@@ -58,6 +61,9 @@ limitations under the License.
 
 #include <stdio.h>
 
+#include <string>
+#include <vector>
+
 // Aside from the header guard, the internal macros defined here have the form:
 //   TF_PLAT_STR_*
 
@@ -342,4 +348,17 @@ limitations under the License.
     }  /* anonymous namespace */
 // clang-format on
 
+namespace tensorflow {
+
+class Status;
+
+// Retrieves the platform strings from the file at the given path and appends
+// them to the given vector. If the returned int is non-zero, an error occurred
+// reading the file and vector may or may not be modified. The returned error
+// code is suitable for use with strerror().
+int GetPlatformStrings(const std::string& path,
+                       std::vector<std::string>* found);
+
+}  // namespace tensorflow
+
 #endif  // TENSORFLOW_CORE_PLATFORM_PLATFORM_STRINGS_H_
diff --git a/tensorflow/core/platform/platform_strings_test.cc b/tensorflow/core/platform/platform_strings_test.cc
index 9584333bb8..5251f10d41 100644
--- a/tensorflow/core/platform/platform_strings_test.cc
+++ b/tensorflow/core/platform/platform_strings_test.cc
@@ -38,45 +38,13 @@ typedef std::vector<std::string> string_vec;
 
 // Append to *found the strings within the named file with the platform_strings
 // magic prefix, and return true; or return false on error.
-static bool GetStrings(const std::string &file_name, string_vec *found) {
-  bool result = false;
-  FILE *ifp = fopen(file_name.c_str(), "rb");
-  if (ifp != nullptr) {
-    static const char prefix[] = TF_PLAT_STR_MAGIC_PREFIX_;
-    int first_char = prefix[1];
-    int last_char = -1;
-    int c;
-    while ((c = getc(ifp)) != EOF) {
-      if (c == first_char && last_char == 0) {
-        int i = 2;
-        while (prefix[i] != 0 && (c = getc(ifp)) == prefix[i]) {
-          i++;
-        }
-        if (prefix[i] == 0) {
-          std::string str;
-          while ((c = getc(ifp)) != EOF && c != 0) {
-            str.push_back(c);
-          }
-          if (!str.empty()) {
-            found->push_back(str);
-          }
-        }
-      }
-      last_char = c;
-    }
-
-    result = (ferror(ifp) == 0);
-    fclose(ifp);
-  }
-  return result;
-}
 
 // Print the platform strings embedded in the binary file_name and return 0,
 // on on error return 2.
 static int PrintStrings(const std::string file_name) {
   int rc = 0;
   string_vec str;
-  if (GetStrings(file_name, &str)) {
+  if (!tensorflow::GetPlatformStrings(file_name, &str)) {
     for (int i = 0; i != str.size(); i++) {
       printf("%s\n", str[i].c_str());
     }
@@ -145,7 +113,7 @@ static int RunTest(const std::string &binary_name) {
   int rc = 0;
   string_vec str;
 
-  if (GetStrings(binary_name, &str)) {
+  if (!tensorflow::GetPlatformStrings(binary_name, &str)) {
     CheckStr(str, "__linux__", AS_STR(__linux__));
     CheckStr(str, "_WIN32", AS_STR(_WIN32));
     CheckStr(str, "__APPLE__", AS_STR(__APPLE__));
-- 
GitLab


From a7b6dd64b4411d81ceedb5c8f9aae44fc5be1b00 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 15 Nov 2018 13:42:33 -0800
Subject: [PATCH 0337/1554] Update default value of loss reduction in  
 tf.estimator.LinearClassifier,   tf.estimator.LinearRegressor,  
 tf.estimator.DNNLinearCombinedClassifier,  
 tf.estimator.DNNLinearCombinedRegressor,   tf.estimator.DNNRegressor,  
 tf.estimator.DNNClassifier to SUM_OVER_BATCH_SIZE for TF V2. It's SUM in V1.

PiperOrigin-RevId: 221682023
---
 ...nsorflow.estimator.-d-n-n-classifier.pbtxt |  1 +
 ...or.-d-n-n-linear-combined-classifier.pbtxt |  1 +
 ...tor.-d-n-n-linear-combined-regressor.pbtxt |  1 +
 ...ensorflow.estimator.-d-n-n-regressor.pbtxt |  1 +
 ...sorflow.estimator.-linear-classifier.pbtxt |  1 +
 ...nsorflow.estimator.-linear-regressor.pbtxt |  1 +
 ...nsorflow.estimator.-d-n-n-classifier.pbtxt |  4 +-
 ...or.-d-n-n-linear-combined-classifier.pbtxt |  4 +-
 ...tor.-d-n-n-linear-combined-regressor.pbtxt |  4 +-
 ...ensorflow.estimator.-d-n-n-regressor.pbtxt |  4 +-
 ...nsorflow.estimator.-linear-regressor.pbtxt |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 40 ++++++++++++++-----
 .../tools/compatibility/tf_upgrade_v2_test.py | 19 +++++----
 13 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
index 22c9e68da1..77e60d426e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.DNNClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index 3f5f9ec724..07aefed63d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.DNNLinearCombinedClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index 6e4018122b..852e8d2f54 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.DNNLinearCombinedRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 93fa79403b..2779cbe90e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.DNNRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
index 1cadbbca57..6569e92c6a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.LinearClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
index 95226b3bcf..d74bf4f197 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.LinearRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
index 22c9e68da1..c542edf64d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.DNNClassifier"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Adagrad\', \'<function relu instance>\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\'], "
+    argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Adagrad\', \'<function relu instance>\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index 3f5f9ec724..623cbc3648 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedClassifier"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'<function relu instance>\', \'None\', \'2\', \'None\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\', \'sum\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'<function relu instance>\', \'None\', \'2\', \'None\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\', \'sum\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index 6e4018122b..f45e76537a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedRegressor"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'label_dimension\', \'weight_column\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'<function relu instance>\', \'None\', \'1\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\', \'sum\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'linear_feature_columns\', \'linear_optimizer\', \'dnn_feature_columns\', \'dnn_optimizer\', \'dnn_hidden_units\', \'dnn_activation_fn\', \'dnn_dropout\', \'label_dimension\', \'weight_column\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\', \'linear_sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Ftrl\', \'None\', \'Adagrad\', \'None\', \'<function relu instance>\', \'None\', \'1\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\', \'sum\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 93fa79403b..8db2196512 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.DNNRegressor"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Adagrad\', \'<function relu instance>\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum\', \'False\'], "
+    argspec: "args=[\'self\', \'hidden_units\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'activation_fn\', \'dropout\', \'input_layer_partitioner\', \'config\', \'warm_start_from\', \'loss_reduction\', \'batch_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Adagrad\', \'<function relu instance>\', \'None\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'False\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
index 95226b3bcf..c4bb19612a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.LinearRegressor"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'partitioner\', \'warm_start_from\', \'loss_reduction\', \'sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'None\', \'None\', \'weighted_sum\', \'sum\'], "
+    argspec: "args=[\'self\', \'feature_columns\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'partitioner\', \'warm_start_from\', \'loss_reduction\', \'sparse_combiner\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'None\', \'None\', \'weighted_sum_over_batch_size\', \'sum\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 35592918ae..94eff59182 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -116,16 +116,36 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
     self.function_warnings = {
-        "tf.train.exponential_decay": decay_function_comment,
-        "tf.train.piecewise_constant": decay_function_comment,
-        "tf.train.polynomial_decay": decay_function_comment,
-        "tf.train.natural_exp_decay": decay_function_comment,
-        "tf.train.inverse_time_decay": decay_function_comment,
-        "tf.train.cosine_decay": decay_function_comment,
-        "tf.train.cosine_decay_restarts": decay_function_comment,
-        "tf.train.linear_cosine_decay": decay_function_comment,
-        "tf.train.noisy_linear_cosine_decay": decay_function_comment,
-        "tf.estimator.LinearClassifier": default_loss_reduction_changed,
+        "tf.train.exponential_decay":
+            decay_function_comment,
+        "tf.train.piecewise_constant":
+            decay_function_comment,
+        "tf.train.polynomial_decay":
+            decay_function_comment,
+        "tf.train.natural_exp_decay":
+            decay_function_comment,
+        "tf.train.inverse_time_decay":
+            decay_function_comment,
+        "tf.train.cosine_decay":
+            decay_function_comment,
+        "tf.train.cosine_decay_restarts":
+            decay_function_comment,
+        "tf.train.linear_cosine_decay":
+            decay_function_comment,
+        "tf.train.noisy_linear_cosine_decay":
+            decay_function_comment,
+        "tf.estimator.LinearClassifier":
+            default_loss_reduction_changed,
+        "tf.estimator.LinearRegressor":
+            default_loss_reduction_changed,
+        "tf.estimator.DNNLinearCombinedClassifier":
+            default_loss_reduction_changed,
+        "tf.estimator.DNNLinearCombinedRegressor":
+            default_loss_reduction_changed,
+        "tf.estimator.DNNRegressor":
+            default_loss_reduction_changed,
+        "tf.estimator.DNNClassifier":
+            default_loss_reduction_changed,
     }
     # Right now we can't have both a rename and a warning.
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 2dfeb64b32..fa6fc5a362 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -102,13 +102,18 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       self.assertEqual(errors, ["test.py:1: %s requires manual check." % decay])
       self.assertIn("%s has been changed" % decay, report)
 
-  def testEstimatorLossReductionChangege(self):
-    text = "tf.estimator.LinearClassifier(a, b)\n"
-    _, report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors, ["test.py:1: %s requires manual check."
-                              % "tf.estimator.LinearClassifier"])
-    self.assertIn("loss_reduction has been changed", report)
+  def testEstimatorLossReductionChange(self):
+    classes = [
+        "LinearClassifier", "LinearRegressor", "DNNLinearCombinedClassifier",
+        "DNNLinearCombinedRegressor", "DNNRegressor", "DNNClassifier"
+    ]
+    for c in classes:
+      ns = "tf.estimator." + c
+      text = ns + "(a, b)"
+      _, report, errors, new_text = self._upgrade(text)
+      self.assertEqual(text, new_text)
+      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("loss_reduction has been changed", report)
 
   def testCountNonZeroChanges(self):
     text = (
-- 
GitLab


From 24dd4cd186397b97f60519625430f1eac7338a3d Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 15 Nov 2018 13:55:55 -0800
Subject: [PATCH 0338/1554] [TF:XLA] Bump open source llvm revision to r346880

PiperOrigin-RevId: 221684552
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b4f7aba046..435ea2d733 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "016e265dc5978f424f23a7893044bb41c68cd58a1c993b1fac99187d92176ae4",
-        strip_prefix = "llvm-5e1b99cf3b50e2d013904a4ecc096cee77d91b86",
+        sha256 = "9d4ceba32fc6cedea2e299c32005f36e273f6174d4f2a7ad808231bd4ab43396",
+        strip_prefix = "llvm-45c215a28e72a6a6b301d0ee3466d999bd1e39d3",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/5e1b99cf3b50e2d013904a4ecc096cee77d91b86.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/5e1b99cf3b50e2d013904a4ecc096cee77d91b86.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/45c215a28e72a6a6b301d0ee3466d999bd1e39d3.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/45c215a28e72a6a6b301d0ee3466d999bd1e39d3.tar.gz",
         ],
     )
 
-- 
GitLab


From ccd7ddc3e9cebabb322f8210430a85fc7d636e8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 13:58:48 -0800
Subject: [PATCH 0339/1554] Avoid a potential loop while copying the shape of
 inputs to the 'axis' parameters of reduce ops.

PiperOrigin-RevId: 221685018
---
 tensorflow/lite/testing/generate_examples.py              | 8 +++++++-
 .../graph_transformations/resolve_reduce_attributes.cc    | 5 +++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 1d52fda4af..71db1bcdb4 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -901,7 +901,13 @@ def make_reduce_tests(reduce_op,
     }, {
         "input_dtype": [tf.float32],
         "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
-        "axis": [None],
+        "axis": [[]],   # shape is: [0]
+        "const_axis": [False],
+        "keepdims": [True, False],
+    }, {
+        "input_dtype": [tf.float32],
+        "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
+        "axis": [None],  # shape is: []
         "const_axis": [True],
         "keepdims": [True, False],
     }]
diff --git a/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc b/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc
index ea5d33009b..9ceba45e93 100644
--- a/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc
+++ b/tensorflow/lite/toco/graph_transformations/resolve_reduce_attributes.cc
@@ -35,6 +35,11 @@ bool ResolveAttributes(Model* model, T* op) {
 
   const Array& indices_array = model->GetArray(op->inputs[1]);
   if (!indices_array.has_shape()) return false;
+
+  // It is ok for indices_array to have a shape for an empty tensor. In that
+  // case, we don't bother setting 'axis'.
+  if (indices_array.buffer->Length() == 0) return false;
+
   op->axis = indices_array.GetBuffer<ArrayDataType::kInt32>().data;
   return true;
 }
-- 
GitLab


From f3c0d1a9d7d711d78e235e4da3038f5eb4f5a7f4 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 15 Nov 2018 14:10:38 -0800
Subject: [PATCH 0340/1554] Disable broken ragged_tensor tests on windows

PiperOrigin-RevId: 221687416
---
 tensorflow/python/ops/ragged/BUILD | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 152c6dc841..9ff5f26804 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -256,6 +256,9 @@ py_test(
     size = "medium",
     srcs = ["ragged_tensor_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "no_windows",
+    ],
     deps = [
         ":ragged",
         "//tensorflow/python:array_ops",
@@ -407,6 +410,9 @@ py_test(
     name = "ragged_to_sparse_op_test",
     srcs = ["ragged_to_sparse_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "no_windows",
+    ],
     deps = [
         ":ragged",
         "//tensorflow/python:array_ops",
@@ -513,6 +519,9 @@ py_test(
     name = "ragged_constant_value_op_test",
     srcs = ["ragged_constant_value_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "no_windows",
+    ],
     deps = [
         ":ragged",
         "//tensorflow/python:framework_test_lib",
-- 
GitLab


From c90fa478b97d8a876d67b27ba017dbed768d9f1f Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 15 Nov 2018 14:12:07 -0800
Subject: [PATCH 0341/1554] Update default value of loss reduction in  
 tf.estimator.BaselineClassifier,   tf.estimator.BaselineRegressor, to
 SUM_OVER_BATCH_SIZE for TF V2. It's SUM in V1.

PiperOrigin-RevId: 221687673
---
 .../golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt | 1 +
 .../golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt  | 1 +
 .../golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt | 4 ++--
 .../golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt  | 4 ++--
 tensorflow/tools/compatibility/tf_upgrade_v2.py               | 4 ++++
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py          | 3 ++-
 6 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
index c50527ed63..af1659528b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.BaselineClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
index ee37a060ae..e5794252e4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.estimator.BaselineRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
index c50527ed63..07483df83e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.BaselineClassifier"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifier\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Ftrl\', \'None\', \'weighted_sum\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'2\', \'None\', \'None\', \'Ftrl\', \'None\', \'weighted_sum_over_batch_size\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
index ee37a060ae..292b5f32d8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.BaselineRegressor"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressor\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
@@ -22,7 +22,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'weighted_sum\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'label_dimension\', \'weight_column\', \'optimizer\', \'config\', \'loss_reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'Ftrl\', \'None\', \'weighted_sum_over_batch_size\'], "
   }
   member_method {
     name: "eval_dir"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 94eff59182..7e741fa1d9 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -146,6 +146,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             default_loss_reduction_changed,
         "tf.estimator.DNNClassifier":
             default_loss_reduction_changed,
+        "tf.estimator.BaselineClassifier":
+            default_loss_reduction_changed,
+        "tf.estimator.BaselineRegressor":
+            default_loss_reduction_changed,
     }
     # Right now we can't have both a rename and a warning.
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index fa6fc5a362..4b83d50036 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -105,7 +105,8 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   def testEstimatorLossReductionChange(self):
     classes = [
         "LinearClassifier", "LinearRegressor", "DNNLinearCombinedClassifier",
-        "DNNLinearCombinedRegressor", "DNNRegressor", "DNNClassifier"
+        "DNNLinearCombinedRegressor", "DNNRegressor", "DNNClassifier",
+        "BaselineClassifier", "BaselineRegressor"
     ]
     for c in classes:
       ns = "tf.estimator." + c
-- 
GitLab


From 60c32eaddfc3bcd9637c2c33b649b6c5b4a6c839 Mon Sep 17 00:00:00 2001
From: Shining Sun <shiningsun@google.com>
Date: Thu, 15 Nov 2018 14:17:05 -0800
Subject: [PATCH 0342/1554] Change replica_id from int to tensor

PiperOrigin-RevId: 221688437
---
 tensorflow/contrib/distribute/python/BUILD    |  6 +++++
 .../python/keras_optimizer_v2_test.py         |  8 ++++---
 .../distribute/python/mirrored_strategy.py    |  9 +++++---
 .../python/mirrored_strategy_multigpu_test.py |  7 +++---
 .../python/mirrored_strategy_test.py          |  7 +++---
 .../distribute/python/one_device_strategy.py  |  9 +++++++-
 .../python/parameter_server_strategy_test.py  | 23 +++++++++++--------
 .../contrib/distribute/python/tpu_strategy.py | 15 +++++++++---
 tensorflow/python/BUILD                       |  4 ++++
 tensorflow/python/ops/summary_op_util.py      | 22 ++++++++++++++----
 tensorflow/python/training/distribute.py      |  6 ++++-
 tensorflow/python/training/distribute_test.py |  5 +++-
 12 files changed, 90 insertions(+), 31 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index e642c8ca14..4b96179077 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -78,8 +78,10 @@ py_library(
         "//tensorflow/python:device",
         "//tensorflow/python:device_util",
         "//tensorflow/python:distribute",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:pywrap_tensorflow",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
@@ -130,6 +132,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:layers",
         "//tensorflow/python:session",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
@@ -152,6 +155,7 @@ py_library(
         "//tensorflow/contrib/eager/python:datasets",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:distribute",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/distribute:reduce_util",
@@ -345,7 +349,9 @@ py_library(
         "//tensorflow/contrib/tpu:tpu_lib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:reduce_util",
     ],
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 54b79e2f97..6a4cbb113a 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
 from tensorflow.python.ops import math_ops
@@ -272,9 +273,10 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
 
 
 def _replica_id():
-  # TODO(cjfj): Return `replica_id_...` directly, once it is a `Tensor`.
-  return constant_op.constant(
-      ds_context.get_replica_context().replica_id_in_sync_group)
+  replica_id = ds_context.get_replica_context().replica_id_in_sync_group
+  if not isinstance(replica_id, ops.Tensor):
+    replica_id = constant_op.constant(replica_id)
+  return replica_id
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 6aec4d27a3..990f36ce2e 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -32,7 +32,9 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import variable_scope
@@ -783,7 +785,8 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
             context.context()._mode(self.context_mode), \
             context.context().device_policy(self.context_device_policy), \
             _enter_graph(self.graph), \
-            MirroredReplicaContext(self.distribution, self.replica_id), \
+            MirroredReplicaContext(self.distribution, constant_op.constant(
+                self.replica_id, dtypes.int32)), \
             ops.device(self.device), \
             ops.name_scope(self._name_scope), \
             variable_scope.variable_scope(
@@ -825,5 +828,5 @@ class MirroredReplicaContext(distribute_lib.ReplicaContext):
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
-    ds = self._distribution_strategy
-    return [ds.worker_devices[self._replica_id_in_sync_group]]
+    replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
+    return [self._distribution_strategy.worker_devices[replica_id]]
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 463459bb23..9c46ce5665 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -1449,9 +1449,10 @@ class MultiWorkerMirroredStrategyTestWithChief(
 
 
 def _replica_id():
-  # TODO(cjfj): Return `replica_id_...` directly, once it is a `Tensor`.
-  return constant_op.constant(
-      ds_context.get_replica_context().replica_id_in_sync_group)
+  replica_id = ds_context.get_replica_context().replica_id_in_sync_group
+  if not isinstance(replica_id, ops.Tensor):
+    replica_id = constant_op.constant(replica_id)
+  return replica_id
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
index 79318c3e2d..0886d0ef34 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -105,9 +105,10 @@ class VariableCreatorStackTest(test.TestCase):
 
 
 def _replica_id():
-  # TODO(cjfj): Return `replica_id_...` directly, once it is a `Tensor`.
-  return constant_op.constant(
-      ds_context.get_replica_context().replica_id_in_sync_group)
+  replica_id = ds_context.get_replica_context().replica_id_in_sync_group
+  if not isinstance(replica_id, ops.Tensor):
+    replica_id = constant_op.constant(replica_id)
+  return replica_id
 
 
 class MultiWorkerMirroredStrategyTest(test.TestCase):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 3582d99405..681aa31432 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -22,6 +22,7 @@ import six
 
 from tensorflow.contrib.distribute.python import values
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -180,7 +181,13 @@ class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
 
   def __init__(self, distribution_strategy):
     distribute_lib.ReplicaContext.__init__(
-        self, distribution_strategy, replica_id_in_sync_group=0)
+        self,
+        distribution_strategy,
+        replica_id_in_sync_group=constant_op.constant(0, dtypes.int32))
+
+  @property
+  def device(self):
+    raise RuntimeError("Use .devices instead")
 
   @property
   def devices(self):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index be42686322..7561d7b3d5 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -52,6 +53,13 @@ WORKER = run_config.TaskType.WORKER
 PS = run_config.TaskType.PS
 
 
+def _get_replica_id_integer():
+  replica_id = ds_context.get_replica_context().replica_id_in_sync_group
+  if isinstance(replica_id, ops.Tensor):
+    replica_id = tensor_util.constant_value(replica_id)
+  return replica_id
+
+
 class ParameterServerStrategyTestBase(
     multi_worker_test_base.MultiWorkerTestBase):
 
@@ -96,9 +104,8 @@ class ParameterServerStrategyTestBase(
         if num_gpus == 0:
           last_part_device = 'device:CPU:0'
         else:
-          last_part_device = (
-              'device:GPU:%d' %
-              ds_context.get_replica_context().replica_id_in_sync_group)
+          replica_id = _get_replica_id_integer()
+          last_part_device = ('device:GPU:%d' % replica_id)
 
         a = constant_op.constant(1.0)
         b = constant_op.constant(2.0)
@@ -263,18 +270,16 @@ class ParameterServerStrategyTestBase(
         if 'CPU' in compute_device:
           replica_compute_device = '/device:CPU:0'
         else:
-          replica_compute_device = (
-              '/device:GPU:%d' %
-              ds_context.get_replica_context().replica_id_in_sync_group)
+          replica_id = _get_replica_id_integer()
+          replica_compute_device = ('/device:GPU:%d' % replica_id)
         replica_compute_device = device_util.canonicalize(
             replica_compute_device)
 
         if 'CPU' in variable_device:
           replica_variable_device = '/device:CPU:0'
         else:
-          replica_variable_device = (
-              '/device:GPU:%d' %
-              ds_context.get_replica_context().replica_id_in_sync_group)
+          replica_id = _get_replica_id_integer()
+          replica_variable_device = ('/device:GPU:%d' % replica_id)
         replica_variable_device = device_util.canonicalize(
             replica_variable_device)
 
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 5f7838c6ce..b917334790 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -35,6 +35,7 @@ from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -404,7 +405,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
       return [tpu.shutdown_system()]
 
   def _get_devices_from(self, colocate_with=None):
-     # TODO(jhseu): Change this when we support model parallelism.
+    # TODO(jhseu): Change this when we support model parallelism.
     return self._tpu_devices
 
   def _create_variable(self, next_creator, *args, **kwargs):
@@ -587,10 +588,18 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
   # TODO(sourabhbajaj): Call for each tower should be updating this.
   def __init__(self, distribution_strategy):
     distribute_lib.ReplicaContext.__init__(
-        self, distribution_strategy, replica_id_in_sync_group=0)
+        self,
+        distribution_strategy,
+        # TODO(b/118385803): properly initialize replica_id, instead of always 0
+        replica_id_in_sync_group=constant_op.constant(0, dtypes.int32))
+
+  @property
+  def device(self):
+    raise RuntimeError("Use .devices instead")
 
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
     ds = self._distribution_strategy
-    return [ds.worker_devices[self._replica_id_in_sync_group]]
+    replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
+    return [ds.worker_devices[replica_id]]
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d656d8ef16..3499bbca2f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3552,8 +3552,10 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":array_ops",
+        ":constant_op",
         ":control_flow_ops",
         ":device_util",
+        ":dtypes",
         ":framework_ops",
         ":platform",
         ":resource_variable_ops",
@@ -3573,7 +3575,9 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":client_testlib",
+        ":constant_op",
         ":distribute",
+        ":dtypes",
         ":variable_scope",
     ],
 )
diff --git a/tensorflow/python/ops/summary_op_util.py b/tensorflow/python/ops/summary_op_util.py
index fe659532b0..c72a9aefc3 100644
--- a/tensorflow/python/ops/summary_op_util.py
+++ b/tensorflow/python/ops/summary_op_util.py
@@ -22,6 +22,7 @@ import contextlib
 import re
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.training import distribution_strategy_context
 
@@ -44,14 +45,27 @@ _INVALID_TAG_CHARACTERS = re.compile(r'[^-/\w\.]')
 
 
 def skip_summary():
-  # If using multiple replicas in distributed strategy, skip summaries on all
-  # replicas except the first one (replica_id=0).
+  """Determines if summary should be skipped.
+
+  If using multiple replicas in distributed strategy, skip summaries on all
+  replicas except the first one (replica_id=0).
+
+  Returns:
+    True if the summary is skipped; False otherwise.
+  """
+
   # TODO(priyag): Add a new optional argument that will provide multiple
   # alternatives to override default behavior. (e.g. run on last replica,
   # compute sum or mean across replicas).
   replica_context = distribution_strategy_context.get_replica_context()
-  # TODO(cjfj): Also check is sync group ID > 0?
-  return replica_context and replica_context.replica_id_in_sync_group > 0
+  if not replica_context:
+    return False
+  # TODO(b/118385803): when replica_id of _TPUReplicaContext is properly
+  # initialized, remember to change here as well.
+  replica_id = replica_context.replica_id_in_sync_group
+  if isinstance(replica_id, ops.Tensor):
+    replica_id = tensor_util.constant_value(replica_id)
+  return replica_id and replica_id > 0
 
 
 def clean_tag(name):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index dfa368026b..1cd3bc2e04 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -23,6 +23,8 @@ import enum
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -1322,7 +1324,9 @@ class _DefaultDistributionStrategy(DistributionStrategy):
       raise NotImplementedError("TODO")
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    with ReplicaContext(self, replica_id_in_sync_group=0):
+    with ReplicaContext(self, \
+                        replica_id_in_sync_group= \
+                        constant_op.constant(0, dtypes.int32)):
       return fn(*args, **kwargs)
 
   def _reduce(self, reduce_op, value, destinations):
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index 39437ad9b5..d8c7d64088 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 from tensorflow.python.training import distribute as distribute_lib
@@ -41,7 +43,8 @@ def _get_test_variable(name, synchronization, aggregation):
 class _TestStrategy(distribute_lib.DistributionStrategy):
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    with _TestReplicaContext(self, replica_id_in_sync_group=0):
+    with _TestReplicaContext(
+        self, replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
       return fn(*args, **kwargs)
 
   def _create_variable(self, next_creator, *args, **kwargs):
-- 
GitLab


From d350de2d22627742e3d58d10159288bc3178132e Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 15 Nov 2018 14:20:59 -0800
Subject: [PATCH 0343/1554] Implement RMSProp Optimizer.

PiperOrigin-RevId: 221689113
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  20 ++
 .../python/keras/optimizer_v2/rmsprop.py      | 193 ++++++++++
 .../python/keras/optimizer_v2/rmsprop_test.py | 335 ++++++++++++++++++
 3 files changed, 548 insertions(+)
 create mode 100644 tensorflow/python/keras/optimizer_v2/rmsprop.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/rmsprop_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index e0ff587549..2421decc32 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -16,6 +16,7 @@ py_library(
         "adam.py",
         "gradient_descent.py",
         "optimizer_v2.py",
+        "rmsprop.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
@@ -87,3 +88,22 @@ py_test(
         "//tensorflow/python/eager:def_function",
     ],
 )
+
+cuda_py_test(
+    name = "rmsprop_test",
+    size = "medium",
+    srcs = ["rmsprop_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python/eager:def_function",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:variables",
+    ],
+)
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
new file mode 100644
index 0000000000..4b036c005f
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -0,0 +1,193 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""RMSProp for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import math_ops
+from tensorflow.python.training import training_ops
+
+
+class RMSProp(optimizer_v2.OptimizerV2):
+  r"""Optimizer that implements the RMSProp algorithm.
+
+  A detailed description of rmsprop.
+
+    - maintain a moving (discounted) average of the square of gradients
+    - divide gradient by the root of this average
+
+  $$mean_square_t = rho * mean_square{t-1} + (1-rho) * gradient ** 2$$
+  $$mom_t = momentum * mom_{t-1} + learning_rate * gradient / \sqrt{ /
+      mean_square_t + \epsilon}$$
+  $$variable_t := variable_{t-1} - mom_t
+
+  This implementation of RMSProp uses plain momentum, not Nesterov momentum.
+
+  The centered version additionally maintains a moving average of the
+  gradients, and uses that average to estimate the variance:
+
+  $$mean_grad_t = rho * mean_grad_{t-1} + (1-rho) * gradient$$
+  $$mean_square_t = rho * mean_square_{t-1} + (1-rho) * gradient ** 2$$
+  $$mom_t = momentum * mom_{t-1} + learning_rate * gradient /
+      sqrt(mean_square_t - mean_grad_t**2 + epsilon)$$
+  $$variable_t := variable_{t-1} - mom_t
+
+  References
+    See ([pdf]
+      http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf).
+  """
+
+  def __init__(self,
+               learning_rate=0.001,
+               rho=0.9,
+               momentum=0.0,
+               epsilon=1e-10,
+               centered=False,
+               name="RMSProp"):
+    """Construct a new RMSProp optimizer.
+
+    Note that in the dense implementation of this algorithm, variables and their
+    corresponding accumulators (momentum, gradient moving average, square
+    gradient moving average) will be updated even if the gradient is zero
+    (i.e. accumulators will decay, momentum will be applied). The sparse
+    implementation (used when the gradient is an `IndexedSlices` object,
+    typically because of `tf.gather` or an embedding lookup in the forward pass)
+    will not update variable slices or their accumulators unless those slices
+    were used in the forward pass (nor is there an "eventual" correction to
+    account for these omitted updates). This leads to more efficient updates for
+    large embedding lookup tables (where most of the slices are not accessed in
+    a particular graph execution), but differs from the published algorithm.
+
+    Args:
+      learning_rate: A Tensor or a floating point value.  The learning rate.
+      rho: Discounting factor for the history/coming gradient
+      momentum: A scalar tensor.
+      epsilon: Small value to avoid zero denominator.
+      centered: If True, gradients are normalized by the estimated variance of
+        the gradient; if False, by the uncentered second moment. Setting this to
+        True may help with training, but is slightly more expensive in terms of
+        computation and memory. Defaults to False.
+      name: Optional name prefix for the operations created when applying
+        gradients. Defaults to "RMSProp".
+
+    @compatibility(eager)
+    When eager execution is enabled, `learning_rate`, `decay`, `momentum`, and
+    `epsilon` can each be a callable that takes no arguments and returns the
+    actual value to use. This can be useful for changing these values across
+    different invocations of optimizer functions.
+    @end_compatibility
+    """
+    super(RMSProp, self).__init__(name)
+    self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("rho", rho)
+
+    self._momentum = False
+    if isinstance(momentum, ops.Tensor) or callable(momentum) or momentum > 0:
+      self._momentum = True
+    if isinstance(momentum, (int, float)) and (momentum < 0 or momentum > 1):
+      raise ValueError("`momentum` must be between [0, 1].")
+    self._set_hyper("momentum", momentum)
+
+    self._set_hyper("epsilon", epsilon)
+    self._centered = centered
+
+  def _create_slots(self, var_list):
+    for var in var_list:
+      self.add_slot(var, "rms")
+      self.add_slot(var, "momentum")
+      if self._centered:
+        self.add_slot(var, "mg")
+
+  def _resource_apply_dense(self, grad, var):
+    rms = self.get_slot(var, "rms")
+    mom = self.get_slot(var, "momentum")
+    learning_rate = math_ops.cast(
+        self._get_hyper("learning_rate"), grad.dtype.base_dtype)
+    rho = math_ops.cast(self._get_hyper("rho"), grad.dtype.base_dtype)
+    momentum = math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype)
+    epsilon = math_ops.cast(self._get_hyper("epsilon"), grad.dtype.base_dtype)
+    if self._centered:
+      mg = self.get_slot(var, "mg")
+      return training_ops.resource_apply_centered_rms_prop(
+          var.handle,
+          mg.handle,
+          rms.handle,
+          mom.handle,
+          learning_rate,
+          rho,
+          momentum,
+          epsilon,
+          grad,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.resource_apply_rms_prop(
+          var.handle,
+          rms.handle,
+          mom.handle,
+          learning_rate,
+          rho,
+          momentum,
+          epsilon,
+          grad,
+          use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices):
+    rms = self.get_slot(var, "rms")
+    mom = self.get_slot(var, "momentum")
+    learning_rate = math_ops.cast(
+        self._get_hyper("learning_rate"), grad.dtype.base_dtype)
+    rho = math_ops.cast(self._get_hyper("rho"), grad.dtype.base_dtype)
+    momentum = math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype)
+    epsilon = math_ops.cast(self._get_hyper("epsilon"), grad.dtype.base_dtype)
+    if self._centered:
+      mg = self.get_slot(var, "mg")
+      return training_ops.resource_sparse_apply_centered_rms_prop(
+          var.handle,
+          mg.handle,
+          rms.handle,
+          mom.handle,
+          learning_rate,
+          rho,
+          momentum,
+          epsilon,
+          grad,
+          indices,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.resource_sparse_apply_rms_prop(
+          var.handle,
+          rms.handle,
+          mom.handle,
+          learning_rate,
+          rho,
+          momentum,
+          epsilon,
+          grad,
+          indices,
+          use_locking=self._use_locking)
+
+  def get_config(self):
+    config = super(RMSProp, self).get_config()
+    config.update({
+        "learning_rate": self._serialize_hyperparameter("learning_rate"),
+        "rho": self._serialize_hyperparameter("rho"),
+        "momentum": self._serialize_hyperparameter("momentum"),
+        "epsilon": self._serialize_hyperparameter("epsilon"),
+        "centered": self._centered,
+    })
+    return config
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
new file mode 100644
index 0000000000..57a2c7789f
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -0,0 +1,335 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for rmsprop."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+import itertools
+import math
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import rmsprop
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+_DATA_TYPES = [dtypes.half, dtypes.float32]
+
+_TEST_PARAM_VALUES = [
+    # learning_rate, rho, momentum, epsilon, centered
+    [0.05, 0.9, 0.0, 1e-3, True],
+    [0.05, 0.9, 0.0, 1e-3, False],
+    [0.1, 0.9, 0.0, 1e-3, True],
+    [0.01, 0.9, 0.0, 1e-5, True],
+    [0.01, 0.9, 0.9, 1e-5, True],
+]
+
+_TESTPARAMS = [
+    [data_type] + values
+    for data_type, values in itertools.product(_DATA_TYPES, _TEST_PARAM_VALUES)
+]
+
+
+class RMSPropOptimizerTest(test.TestCase):
+
+  def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, rho, momentum,
+                            epsilon, centered):
+    rms_t = rms * rho + (1 - rho) * g * g
+    denom_t = rms_t + epsilon
+    if centered:
+      mg_t = mg * rho + (1 - rho) * g
+      denom_t -= mg_t * mg_t
+    else:
+      mg_t = mg
+    mom_t = momentum * mom + lr * g / np.sqrt(denom_t, dtype=denom_t.dtype)
+    var_t = var - mom_t
+    return var_t, mg_t, rms_t, mom_t
+
+  def _sparse_rmsprop_update_numpy(self, var, gindexs, gvalues, mg, rms, mom,
+                                   lr, rho, momentum, epsilon, centered):
+    mg_t = copy.deepcopy(mg)
+    rms_t = copy.deepcopy(rms)
+    mom_t = copy.deepcopy(mom)
+    var_t = copy.deepcopy(var)
+    for i in range(len(gindexs)):
+      gindex = gindexs[i]
+      gvalue = gvalues[i]
+      rms_t[gindex] = rms[gindex] * rho + (1 - rho) * gvalue * gvalue
+      denom_t = rms_t[gindex] + epsilon
+      if centered:
+        mg_t[gindex] = mg_t[gindex] * rho + (1 - rho) * gvalue
+        denom_t -= mg_t[gindex] * mg_t[gindex]
+      mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t)
+      var_t[gindex] = var[gindex] - mom_t[gindex]
+    return var_t, mg_t, rms_t, mom_t
+
+  def testDense(self):
+    for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
+      with self.cached_session(use_gpu=True):
+        # Initialize variables for numpy implementation.
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(var0_np, dtype=dtype)
+        var1 = resource_variable_ops.ResourceVariable(var1_np, dtype=dtype)
+        grads0 = constant_op.constant(grads0_np, dtype=dtype)
+        grads1 = constant_op.constant(grads1_np, dtype=dtype)
+        opt = rmsprop.RMSProp(
+            learning_rate=learning_rate,
+            rho=rho,
+            momentum=momentum,
+            epsilon=epsilon,
+            centered=centered)
+
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        if centered:
+          mg0 = opt.get_slot(var0, "mg")
+          mg1 = opt.get_slot(var1, "mg")
+        else:
+          mg0 = None
+          mg1 = None
+
+        rms0 = opt.get_slot(var0, "rms")
+        self.assertTrue(rms0 is not None)
+        rms1 = opt.get_slot(var1, "rms")
+        self.assertTrue(rms1 is not None)
+        mom0 = opt.get_slot(var0, "momentum")
+        self.assertTrue(mom0 is not None)
+        mom1 = opt.get_slot(var1, "momentum")
+        self.assertTrue(mom1 is not None)
+
+        mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        rms0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        rms1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 4 steps of RMSProp
+        for _ in range(1, 5):
+          update.run()
+
+          var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
+              var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate, rho,
+              momentum, epsilon, centered)
+          var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
+              var1_np, grads1_np, mg1_np, rms1_np, mom1_np, learning_rate, rho,
+              momentum, epsilon, centered)
+
+          # Validate updated params
+          if centered:
+            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
+            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
+          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
+          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
+          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
+          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = rmsprop.RMSProp(
+            learning_rate=1.0,
+            rho=0.0,
+            momentum=0.0,
+            epsilon=0.0,
+            centered=False).minimize(
+                loss, var_list=[var0])
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[0., 1.]], var0.eval(), atol=0.01)
+
+  def testMinimizeSparseResourceVariableCentered(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = rmsprop.RMSProp(
+            learning_rate=1.0,
+            rho=0.0,
+            momentum=0.0,
+            epsilon=1.0,
+            centered=True).minimize(
+                loss, var_list=[var0])
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[-111, -138]], var0.eval(), atol=0.01)
+
+  def testSparse(self):
+    for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
+      with self.cached_session(use_gpu=True):
+        # Initialize variables for numpy implementation.
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0_np_indices = np.array([0], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np),
+            constant_op.constant(grads0_np_indices), constant_op.constant([1]))
+        grads1_np_indices = np.array([1], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np),
+            constant_op.constant(grads1_np_indices), constant_op.constant([1]))
+        opt = rmsprop.RMSProp(
+            learning_rate=learning_rate,
+            rho=rho,
+            momentum=momentum,
+            epsilon=epsilon,
+            centered=centered)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        if centered:
+          mg0 = opt.get_slot(var0, "mg")
+          self.assertEqual(mg0 is not None, centered)
+          mg1 = opt.get_slot(var1, "mg")
+          self.assertEqual(mg1 is not None, centered)
+        else:
+          mg0 = None
+          mg1 = None
+        rms0 = opt.get_slot(var0, "rms")
+        self.assertTrue(rms0 is not None)
+        rms1 = opt.get_slot(var1, "rms")
+        self.assertTrue(rms1 is not None)
+        mom0 = opt.get_slot(var0, "momentum")
+        self.assertTrue(mom0 is not None)
+        mom1 = opt.get_slot(var1, "momentum")
+        self.assertTrue(mom1 is not None)
+
+        mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        rms0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        rms1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 4 steps of RMSProp
+        for _ in range(1, 5):
+          update.run()
+
+          var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
+              var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np,
+              learning_rate, rho, momentum, epsilon, centered)
+          var1_np, mg1_np, rms1_np, mom1_np = self._sparse_rmsprop_update_numpy(
+              var1_np, grads1_np_indices, grads1_np, mg1_np, rms1_np, mom1_np,
+              learning_rate, rho, momentum, epsilon, centered)
+
+          # Validate updated params
+          if centered:
+            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
+            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
+          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
+          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
+          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
+          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testCallableParams(self):
+    with context.eager_mode():
+      for dtype in [dtypes.half, dtypes.float32]:
+        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
+        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+
+        learning_rate = lambda: 2.0
+        rho = lambda: 0.9
+        momentum = lambda: 0.0
+        epsilon = lambda: 1.0
+        opt = rmsprop.RMSProp(learning_rate, rho, momentum, epsilon)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+        # Step 1: the rms accumulators where 1. So we should see a normal
+        # update: v -= grad * learning_rate
+        opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        # Check the parameters.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)),
+                2.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0))
+            ]), self.evaluate(var0))
+        self.assertAllCloseAccordingToType(
+            np.array([
+                3.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)),
+                4.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0))
+            ]), self.evaluate(var1))
+        # Step 2: the root mean square accumulators contain the previous update.
+        opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        # Check the parameters.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) -
+                (0.1 * 2.0 / math.sqrt(0.001 * 0.9 + 0.001 + 1.0)),
+                2.0 - (0.1 * 2.0 / math.sqrt(0.001 + 1.0)) -
+                (0.1 * 2.0 / math.sqrt(0.001 * 0.9 + 0.001 + 1.0))
+            ]), self.evaluate(var0))
+        self.assertAllCloseAccordingToType(
+            np.array([
+                3.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) -
+                (0.01 * 2.0 / math.sqrt(0.00001 * 0.9 + 1e-5 + 1.0)),
+                4.0 - (0.01 * 2.0 / math.sqrt(0.00001 + 1.0)) -
+                (0.01 * 2.0 / math.sqrt(0.00001 * 0.9 + 1e-5 + 1.0))
+            ]), self.evaluate(var1))
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 45372d7cbaed40eae35536bf6466e5029767550e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 14:31:19 -0800
Subject: [PATCH 0344/1554] Write string tensor in a way compatible with
 tensor_util.MakeNdarray

PiperOrigin-RevId: 221691033
---
 .../tensorboard/db/summary_file_writer.cc      | 11 ++++++++++-
 .../tensorboard/db/summary_file_writer_test.cc | 18 ++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc b/tensorflow/contrib/tensorboard/db/summary_file_writer.cc
index 3f24f58f03..22b6f09d0c 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc
+++ b/tensorflow/contrib/tensorboard/db/summary_file_writer.cc
@@ -73,7 +73,16 @@ class SummaryFileWriter : public SummaryWriterInterface {
     e->set_step(global_step);
     e->set_wall_time(GetWallTime());
     Summary::Value* v = e->mutable_summary()->add_value();
-    t.AsProtoTensorContent(v->mutable_tensor());
+
+    if (t.dtype() == DT_STRING) {
+      // Treat DT_STRING specially, so that tensor_util.MakeNdarray in Python
+      // can convert the TensorProto to string-type numpy array. MakeNdarray
+      // does not work with strings encoded by AsProtoTensorContent() in
+      // tensor_content.
+      t.AsProtoField(v->mutable_tensor());
+    } else {
+      t.AsProtoTensorContent(v->mutable_tensor());
+    }
     v->set_tag(tag);
     if (!serialized_metadata.empty()) {
       v->mutable_metadata()->ParseFromString(serialized_metadata);
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc b/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
index cd3f712256..ffbfb9533e 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
+++ b/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
 
 #include "tensorflow/core/framework/summary.pb.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/io/path.h"
@@ -104,6 +105,23 @@ TEST_F(SummaryFileWriterTest, WriteTensor) {
                                   CHECK_EQ(e.summary().value_size(), 1);
                                   EXPECT_EQ(e.summary().value(0).tag(), "name");
                                 }));
+  TF_CHECK_OK(SummaryTestHelper(
+      "string_tensor_test",
+      [](SummaryWriterInterface* writer) {
+        Tensor hello(DT_STRING, TensorShape({}));
+        hello.scalar<string>()() = "hello";
+        TF_RETURN_IF_ERROR(writer->WriteTensor(
+            2, hello, "name", SummaryMetadata().SerializeAsString()));
+        TF_RETURN_IF_ERROR(writer->Flush());
+        return Status::OK();
+      },
+      [](const Event& e) {
+        EXPECT_EQ(e.step(), 2);
+        CHECK_EQ(e.summary().value_size(), 1);
+        EXPECT_EQ(e.summary().value(0).tag(), "name");
+        EXPECT_EQ(e.summary().value(0).tensor().dtype(), DT_STRING);
+        EXPECT_EQ(e.summary().value(0).tensor().string_val()[0], "hello");
+      }));
 }
 
 TEST_F(SummaryFileWriterTest, WriteScalar) {
-- 
GitLab


From a81e966cf236e07668009e9751c150e5cec13a0a Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 15 Nov 2018 14:41:35 -0800
Subject: [PATCH 0345/1554] Prototype for Keras multi worker: make session a
 per-thread object; configure session if running under distribute coordinator;
 initialize variables or wait for variables to be initialized. The changes to
 Keras should be noop.

PiperOrigin-RevId: 221692888
---
 .../distribute/distribute_coordinator.py      |  9 +++
 tensorflow/python/keras/backend.py            | 34 +++++++-----
 .../engine/distributed_training_utils.py      | 55 +++++++++++++++++--
 tensorflow/python/keras/engine/training.py    | 10 ++--
 .../keras/engine/training_distributed.py      |  7 +++
 5 files changed, 92 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 520413102b..07d291e037 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -261,6 +261,10 @@ class _WorkerContext(object):
           config=session_config,
           max_wait_secs=max_wait_secs)
 
+  @property
+  def session_config(self):
+    return copy.deepcopy(self._session_config)
+
   @property
   def has_barrier(self):
     """Whether the barrier is set or not."""
@@ -301,6 +305,11 @@ class _WorkerContext(object):
     """Returns number of workers in the cluster, including chief."""
     return self._num_workers
 
+  @property
+  def should_init(self):
+    """Whether to run init ops."""
+    return self._strategy.should_init
+
   @property
   def should_checkpoint(self):
     """Whether to save checkpoint."""
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 1d23e14d1d..c67b7a5b0b 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -25,6 +25,7 @@ import collections
 import itertools
 import json
 import os
+import threading
 import weakref
 
 import numpy as np
@@ -73,9 +74,9 @@ py_sum = sum
 # while executing eagerly (such as the functional API for model-building).
 _GRAPH = None
 
-# This is the default internal TF session used by Keras.
-# It can be set manually via `set_session(sess)`.
-_SESSION = None
+# This is a thread local object that will hold the default internal TF session
+# used by Keras. It can be set manually via `set_session(sess)`.
+_SESSION = threading.local()
 
 # This dictionary holds a mapping {graph: learning_phase}.
 # A learning phase is a bool tensor used to run Keras models in
@@ -337,7 +338,7 @@ def clear_session():
   global _GRAPH_TF_OPTIMIZERS  # pylint: disable=global-variable-not-assigned
   ops.reset_default_graph()
   reset_uids()
-  _SESSION = None
+  _SESSION.session = None
   graph = get_graph()
   with graph.as_default():
     phase = array_ops.placeholder_with_default(
@@ -444,6 +445,20 @@ def learning_phase_scope(value):
         _GRAPH_LEARNING_PHASES[get_graph()] = previous_value
 
 
+def _get_session():
+  """Returns the session object for the current thread."""
+  global _SESSION
+  default_session = ops.get_default_session()
+  if default_session is not None:
+    session = default_session
+  else:
+    if getattr(_SESSION, 'session', None) is None:
+      _SESSION.session = session_module.Session(
+          config=get_default_session_config())
+    session = _SESSION.session
+  return session
+
+
 @tf_export(v1=['keras.backend.get_session'])
 def get_session():
   """Returns the TF session to be used by the backend.
@@ -461,14 +476,7 @@ def get_session():
   Returns:
       A TensorFlow session.
   """
-  global _SESSION
-  default_session = ops.get_default_session()
-  if default_session is not None:
-    session = default_session
-  else:
-    if _SESSION is None:
-      _SESSION = session_module.Session(config=get_default_session_config())
-    session = _SESSION
+  session = _get_session()
   if not _MANUAL_VAR_INIT:
     with session.graph.as_default():
       _initialize_variables(session)
@@ -493,7 +501,7 @@ def set_session(session):
       session: A TF Session.
   """
   global _SESSION
-  _SESSION = session
+  _SESSION.session = session
 
 
 def get_default_session_config():
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 7d38df410f..8dd9ef470b 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -22,6 +22,7 @@ import numpy as np
 from tensorflow.python.client import session as session_module
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.distribute import distribute_coordinator_context as dc_context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -293,19 +294,63 @@ def validate_all_tensor_shapes(x, x_values):
                        ' inputs {}'.format(x))
 
 
+def _wait_for_variable_initialization(session):
+  """Utility to wait for variables to be initialized."""
+  all_variables = K._get_variables(K.get_graph())  # pylint: disable=protected-access
+  candidate_vars = []
+  for v in all_variables:
+    if not getattr(v, '_keras_initialized', False):
+      candidate_vars.append(v)
+
+  if not candidate_vars:
+    return
+
+  while True:
+    is_initialized = session.run(
+        [variables.is_variable_initialized(v) for v in candidate_vars])
+    uninitialized_vars = []
+    for flag, v in zip(is_initialized, candidate_vars):
+      if not flag:
+        uninitialized_vars.append(v)
+      v._keras_initialized = True  # pylint: disable=protected-access
+    if not uninitialized_vars:
+      break
+
+
+def init_restore_or_wait_for_variables():
+  """Initialize or restore variables or wait for variables to be initialized."""
+  session = K._get_session()  # pylint: disable=protected-access
+  worker_context = dc_context.get_current_worker_context()
+  if not worker_context or worker_context.should_init:
+    # TODO(yuefengz): if checkpoints exit, restore from checkpoint.
+    K._initialize_variables(session)  # pylint: disable=protected-access
+  else:
+    _wait_for_variable_initialization(session)
+
+
 def configure_and_create_session(distribution_strategy):
   """Configure session config and create a session with it."""
   # TODO(priyag): Throw error if a session already exists.
   session_config = K.get_default_session_config()
-  distribution_strategy.configure(session_config)
 
-  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
-    # TODO(priyag): Remove this workaround when Distributed Coordinator is
-    # integrated with keras and we can create a session from there.
+  if type(distribution_strategy).__name__ == 'TPUStrategy':
+    # TODO(priyag, yuefengz): Remove this workaround when Distribute
+    # Coordinator is integrated with keras and we can create a session from
+    # there.
+    distribution_strategy.configure(session_config)
     master = distribution_strategy._tpu_cluster_resolver.master()  # pylint: disable=protected-access
     session = session_module.Session(config=session_config, target=master)
   else:
-    session = session_module.Session(config=session_config)
+    worker_context = dc_context.get_current_worker_context()
+    if worker_context:
+      dc_session_config = worker_context.session_config
+      # Merge the default session config to the one from distribute coordinator,
+      # which is fine for now since they don't have conflicting configurations.
+      dc_session_config.MergeFrom(session_config)
+      session = session_module.Session(
+          config=dc_session_config, target=worker_context.master_target)
+    else:
+      session = session_module.Session(config=session_config)
 
   K.set_session(session)
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 856f315a76..9fd12c4b56 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -971,13 +971,13 @@ class Model(Network):
         x = x.batch(batch_size, drop_remainder=drop_remainder)
 
     assert isinstance(x, dataset_ops.Dataset)
-    if self._distribution_strategy.__class__.__name__ == 'TPUStrategy':
-      iterator = self._distribution_strategy.make_dataset_iterator(x)
-    else:
-      dataset = self._distribution_strategy.distribute_dataset(lambda: x)
-      iterator = dataset.make_initializable_iterator()
 
     with self._distribution_strategy.scope():
+      if type(self._distribution_strategy).__name__ == 'TPUStrategy':
+        iterator = self._distribution_strategy.make_dataset_iterator(x)
+      else:
+        dataset = self._distribution_strategy.distribute_dataset(lambda: x)
+        iterator = dataset.make_initializable_iterator()
       K.get_session().run(iterator.initializer)
 
     training_utils.validate_iterator_input(x, y, sample_weight,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 3a2373b4cf..3c64233215 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -105,6 +105,13 @@ def fit_loop(
     (grouped_inputs, grouped_outputs, grouped_updates,
      grouped_session_args) = current_strategy.call_for_each_replica(
          _per_device_fit_function, args=(model._grouped_model,))
+
+    # Initialize the variables in the replicated model. This is necessary for
+    # multi-worker training because on some workers, initialization is not
+    # needed. This method does initialization or waiting for initialization
+    # according to the context object of distribute coordinator.
+    distributed_training_utils.init_restore_or_wait_for_variables()
+
     # Unwrap all the per device values returned from `call_for_each_replica`.
     # Unwrapping per device values gives you a list of values that can be
     # used to construct a new train function that is composed of update ops on
-- 
GitLab


From e5a31083cd54a6a7939b0c55b3d03b18e0d08785 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Thu, 15 Nov 2018 14:50:11 -0800
Subject: [PATCH 0346/1554] Use NumPy types for object/bool in _NP_TO_TF

np.object and np.bool are just references to the object/bool builtins
and not NumPy types.

PiperOrigin-RevId: 221694469
---
 tensorflow/python/framework/dtypes.py      | 10 ++++++----
 tensorflow/python/framework/dtypes_test.py |  4 ++--
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 48e9f0524e..f7a12d27df 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
+from six.moves import builtins
 
 from tensorflow.core.framework import types_pb2
 from tensorflow.python import pywrap_tensorflow
@@ -548,8 +549,8 @@ _NP_TO_TF = frozenset([
     (np.int8, int8),
     (np.complex64, complex64),
     (np.complex128, complex128),
-    (np.object, string),
-    (np.bool, bool),
+    (np.object_, string),
+    (np.bool_, bool),
     (_np_qint8, qint8),
     (_np_quint8, quint8),
     (_np_qint16, qint16),
@@ -658,8 +659,9 @@ tf_export(
         __name__, "QUANTIZED_DTYPES")
 
 _PYTHON_TO_TF = {
-    float: float32,
-    bool: bool,
+    builtins.float: float32,
+    builtins.bool: bool,
+    builtins.object: string
 }
 
 
diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py
index a873670e04..719fdc0953 100644
--- a/tensorflow/python/framework/dtypes_test.py
+++ b/tensorflow/python/framework/dtypes_test.py
@@ -81,10 +81,10 @@ class TypesTest(test_util.TensorFlowTestCase):
     self.assertIs(dtypes.int8, dtypes.as_dtype(np.int8))
     self.assertIs(dtypes.complex64, dtypes.as_dtype(np.complex64))
     self.assertIs(dtypes.complex128, dtypes.as_dtype(np.complex128))
-    self.assertIs(dtypes.string, dtypes.as_dtype(np.object))
+    self.assertIs(dtypes.string, dtypes.as_dtype(np.object_))
     self.assertIs(dtypes.string,
                   dtypes.as_dtype(np.array(["foo", "bar"]).dtype))
-    self.assertIs(dtypes.bool, dtypes.as_dtype(np.bool))
+    self.assertIs(dtypes.bool, dtypes.as_dtype(np.bool_))
     with self.assertRaises(TypeError):
       dtypes.as_dtype(np.dtype([("f1", np.uint), ("f2", np.int32)]))
 
-- 
GitLab


From cc6113fd3b64b282fa9a77c1112743be9f224099 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 15 Nov 2018 15:03:54 -0800
Subject: [PATCH 0347/1554] Disable c_api_test in OSS

PiperOrigin-RevId: 221697081
---
 tensorflow/c/eager/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index ba3d8533db..c25b78434f 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -133,6 +133,7 @@ tf_cuda_cc_test(
     tags = [
         "guitar",
         "multi_gpu",
+        "no_oss",
     ],
     deps = [
         ":c_api",
-- 
GitLab


From 67c74b1ab0e05531fc0cb1e864ac8a4c5e9b84c1 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 15 Nov 2018 15:07:08 -0800
Subject: [PATCH 0348/1554] [XLA] Add the HloInstruction::dimension() method to
 delegate the HloGetDimensionSizeInstruction::dimension.

PiperOrigin-RevId: 221697647
---
 tensorflow/compiler/xla/service/hlo_instruction.cc | 4 ++++
 tensorflow/compiler/xla/service/hlo_instruction.h  | 3 +++
 tensorflow/compiler/xla/service/hlo_verifier.cc    | 6 +++---
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index b8f7726c39..ce255292d2 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -3069,6 +3069,10 @@ int64 HloInstruction::concatenate_dimension() const {
   return Cast<HloConcatenateInstruction>(this)->concatenate_dimension();
 }
 
+int64 HloInstruction::dimension() const {
+  return Cast<HloGetDimensionSizeInstruction>(this)->dimension();
+}
+
 bool HloInstruction::IsRank2Transpose() const {
   auto transpose = DynCast<HloTransposeInstruction>(this);
   return transpose != nullptr && transpose->IsRank2Transpose();
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index b6bbc560d9..95ad29235a 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -1323,6 +1323,9 @@ class HloInstruction {
   // Delegates to HloConcatenateInstruction::concatenate_dimension.
   int64 concatenate_dimension() const;
 
+  // Delegates to HloGetDimensionSizeInstruction::dimension.
+  int64 dimension() const;
+
   // Returns whether this instruction does a rank-2 transposition.
   bool IsRank2Transpose() const;
 
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 27fd685a69..c5688a445a 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -757,9 +757,9 @@ Status ShapeVerifier::HandleAfterAll(HloInstruction* token) {
 }
 
 Status ShapeVerifier::HandleGetDimensionSize(HloInstruction* get_size) {
-  return CheckShape(
-      get_size, ShapeInference::InferGetDimensionSizeShape(
-                    get_size->operand(0)->shape(), get_size->dimensions(0)));
+  return CheckShape(get_size,
+                    ShapeInference::InferGetDimensionSizeShape(
+                        get_size->operand(0)->shape(), get_size->dimension()));
 }
 
 Status ShapeVerifier::CheckShape(const HloInstruction* instruction,
-- 
GitLab


From 8ba1b3462a19529597e4a57556cbda95d25c492e Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Thu, 15 Nov 2018 15:10:39 -0800
Subject: [PATCH 0349/1554] [TF:XLA] Delete unused method
 ParseAndReturnUnverifiedModule and deprecate CreateNewUnverifiedModule to
 avoid creating unverified modules in unit tests. The only legitimate use case
 for creating an unverified module is in hlo_verifier_test, and I added a
 helper method there for that.

PiperOrigin-RevId: 221698267
---
 .../compiler/xla/service/hlo_verifier_test.cc    | 16 ++++++++++------
 tensorflow/compiler/xla/tests/hlo_test_base.cc   |  8 --------
 tensorflow/compiler/xla/tests/hlo_test_base.h    |  8 ++------
 3 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 5ddfe0a944..4bc557e4e6 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -35,6 +35,10 @@ namespace {
 
 using ::testing::HasSubstr;
 
+std::unique_ptr<HloModule> CreateUnverifiedModule() {
+  return absl::make_unique<HloModule>("module", HloModuleConfig());
+}
+
 // This class cannot be converted to use HloTestBase. It explicitly
 // uses HloTestBase to create and test malformed HLOs.
 class HloVerifierTest : public HloTestBase {
@@ -66,7 +70,7 @@ TEST_F(HloVerifierTest, NullInstructionParent) {
       HloInstruction::CreateParameter(0, scalar_shape, "param"));
   HloInstruction* negate = builder.AddInstruction(
       HloInstruction::CreateUnary(scalar_shape, HloOpcode::kNegate, param));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateUnverifiedModule();
   module->AddEntryComputation(builder.Build());
 
   TF_ASSERT_OK(verifier().Run(module.get()).status());
@@ -85,7 +89,7 @@ TEST_F(HloVerifierTest, NullComputationParent) {
       HloInstruction::CreateParameter(0, scalar_shape, "param"));
   builder.AddInstruction(
       HloInstruction::CreateUnary(scalar_shape, HloOpcode::kNegate, param));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateUnverifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
   TF_ASSERT_OK(verifier().Run(module.get()).status());
@@ -104,7 +108,7 @@ TEST_F(HloVerifierTest, DifferentOperandParents) {
       HloInstruction::CreateParameter(0, scalar_shape, "param"));
   HloInstruction* negate = builder.AddInstruction(
       HloInstruction::CreateUnary(scalar_shape, HloOpcode::kNegate, param));
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateUnverifiedModule();
   module->AddEntryComputation(builder.Build());
 
   HloComputation::Builder emb_builder(TestName());
@@ -138,7 +142,7 @@ TEST_F(HloVerifierTest, ResetsShapeVerifierState) {
   builder.AddInstruction(
       HloInstruction::CreateBinary(s2, HloOpcode::kMultiply, add, add));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateUnverifiedModule();
   module->AddEntryComputation(builder.Build());
 
   // Run the verifier twice.  It should fail both times, because it shouldn't
@@ -303,7 +307,7 @@ TEST_F(HloVerifierTest, NegativeInteriorPaddingNotAllowed) {
           HloInstruction::CreateConstant(LiteralUtil::Zero(F32))),
       padding_config));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateUnverifiedModule();
   module->AddEntryComputation(builder.Build());
 
   auto status = verifier().Run(module.get()).status();
@@ -327,7 +331,7 @@ TEST_F(HloVerifierTest, PadNegativeInteriorDilationNotAllowed) {
           HloInstruction::CreateConstant(LiteralUtil::Zero(F32).Clone())),
       padding_config));
 
-  auto module = CreateNewUnverifiedModule();
+  auto module = CreateUnverifiedModule();
   module->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(verifier().Run(module.get()).status().error_message(),
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index d8fa00272f..2f467a26b0 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -140,14 +140,6 @@ std::unique_ptr<VerifiedHloModule> HloTestBase::CreateNewVerifiedModule(
       allow_mixed_precision_in_hlo_verifier_);
 }
 
-StatusOr<std::unique_ptr<HloModule>>
-HloTestBase::ParseAndReturnUnverifiedModule(absl::string_view hlo_text,
-                                            const HloModuleConfig& config) {
-  auto module = absl::make_unique<HloModule>(TestName(), config);
-  TF_RETURN_IF_ERROR(ParseHloString(hlo_text, module.get()));
-  return std::move(module);
-}
-
 StatusOr<std::unique_ptr<VerifiedHloModule>>
 HloTestBase::ParseAndReturnVerifiedModule(absl::string_view hlo_text,
                                           const HloModuleConfig& config) {
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h
index 366726d90b..1d1e7f4372 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/base/macros.h"
 #include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/backend.h"
@@ -100,6 +101,7 @@ class HloTestBase : public ::testing::Test {
   //
   // This returns a vanilla HloModule that doesn't run the HLO verifier on
   // destruction.
+  ABSL_DEPRECATED("Use CreateNewVerifiedModule instead.")
   std::unique_ptr<HloModule> CreateNewUnverifiedModule(
       const string& name = TestName());
 
@@ -108,12 +110,6 @@ class HloTestBase : public ::testing::Test {
   std::unique_ptr<VerifiedHloModule> CreateNewVerifiedModule(
       const string& name = TestName());
 
-  // Parses the given string and returns module as a vanilla, unverified
-  // HloModule.
-  StatusOr<std::unique_ptr<HloModule>> ParseAndReturnUnverifiedModule(
-      absl::string_view hlo_text,
-      const HloModuleConfig& config = HloModuleConfig());
-
   // Parses the given string and returns module as a VerifiedHloModule.
   StatusOr<std::unique_ptr<VerifiedHloModule>> ParseAndReturnVerifiedModule(
       absl::string_view hlo_text,
-- 
GitLab


From a85df480c4b7c06c0f8adaf0003f038e2c440f50 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Thu, 15 Nov 2018 15:10:48 -0800
Subject: [PATCH 0350/1554] [TF:XLA] Handle literals of arbitrary rank when
 converting a literal to a string.

Removed duplication and special-casing of ranks 1 to 5. See DenseArrayToStringHelper.
The rest of the diff is because of splitting out the other cases into helper methods.

PiperOrigin-RevId: 221698285
---
 tensorflow/compiler/xla/literal.cc            | 275 ++++++++----------
 tensorflow/compiler/xla/literal_test.cc       | 142 ++++++---
 .../compiler/xla/service/hlo_parser_test.cc   |   4 +-
 .../service/indexed_array_analysis_test.cc    |   8 +-
 .../xla/tests/array_elementwise_ops_test.cc   |  12 +-
 5 files changed, 237 insertions(+), 204 deletions(-)

diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index cb00a0ab16..fcc59f6d21 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
+#include "absl/types/span.h"
 #include "tensorflow/compiler/xla/index_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
@@ -1012,166 +1013,144 @@ void LiteralBase::Piece::SortSparseElementsInternal() {
 
 namespace {
 
-void ToStringHelper(const LiteralBase& literal, const ShapeIndex& shape_index,
-                    bool print_layout, std::vector<string>* pieces) {
-  const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index);
-  CHECK(LayoutUtil::HasLayout(literal.shape()));
-  CHECK(LayoutUtil::HasLayout(subshape));
+string ShapeToString(bool print_layout, const Shape& shape) {
+  return print_layout ? ShapeUtil::HumanStringWithLayout(shape)
+                      : ShapeUtil::HumanString(shape);
+}
 
-  auto shape_to_string = [print_layout](const Shape& shape) {
-    if (print_layout) {
-      return ShapeUtil::HumanStringWithLayout(shape);
-    } else {
-      return ShapeUtil::HumanString(shape);
-    }
-  };
+void ToStringHelper(const LiteralBase& literal, const ShapeIndex& shape_index,
+                    bool print_layout, std::vector<string>* pieces);
 
-  // TODO(b/32894291): refactor this code to reduce code duplication.
-  if (ShapeUtil::IsTuple(subshape)) {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back(" (\n");
-    std::vector<string> tuple_pieces;
-    for (int i = 0; i < ShapeUtil::TupleElementCount(subshape); ++i) {
-      ShapeIndex element_index = shape_index;
-      element_index.push_back(i);
-      std::vector<string> element_pieces;
-      ToStringHelper(literal, element_index, print_layout, &element_pieces);
-      tuple_pieces.push_back(absl::StrJoin(element_pieces, ""));
+void TupleToStringHelper(const LiteralBase& literal,
+                         const ShapeIndex& shape_index, bool print_layout,
+                         std::vector<string>* pieces) {
+  const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index);
+  pieces->push_back(ShapeToString(print_layout, subshape));
+  pieces->push_back(" (\n");
+  std::vector<string> tuple_pieces;
+  for (int i = 0; i < ShapeUtil::TupleElementCount(subshape); ++i) {
+    ShapeIndex element_index = shape_index;
+    element_index.push_back(i);
+    std::vector<string> element_pieces;
+    ToStringHelper(literal, element_index, print_layout, &element_pieces);
+    tuple_pieces.push_back(absl::StrJoin(element_pieces, ""));
+  }
+  pieces->push_back(absl::StrJoin(tuple_pieces, ",\n"));
+  pieces->push_back("\n)");
+}
+
+void SparseArrayToStringHelper(const LiteralBase& literal,
+                               const Shape& subshape, bool print_layout,
+                               std::vector<string>* pieces) {
+  pieces->push_back(ShapeToString(print_layout, subshape));
+  pieces->push_back("{");
+  int64 rank = ShapeUtil::Rank(subshape);
+  int64 num_elements = literal.sparse_element_count();
+  for (int64 i = 0; i < num_elements; ++i) {
+    if (i > 0) {
+      pieces->push_back(", ");
     }
-    pieces->push_back(absl::StrJoin(tuple_pieces, ",\n"));
-    pieces->push_back("\n)");
-    return;
-  }
-
-  if (ShapeUtil::IsToken(subshape)) {
-    pieces->push_back("token");
-    return;
-  }
-
-  if (LayoutUtil::IsSparseArray(subshape)) {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back("{");
-    int64 rank = ShapeUtil::Rank(subshape);
-    int64 num_elements = literal.sparse_element_count();
-    for (int64 i = 0; i < num_elements; ++i) {
-      if (i > 0) {
-        pieces->push_back(", ");
-      }
-      if (rank == 1) {
-        pieces->push_back(StrCat(literal.GetSparseIndex(i)[0]));
-        pieces->push_back(": ");
-      } else {
-        pieces->push_back("[");
-        pieces->push_back(absl::StrJoin(literal.GetSparseIndex(i), ", "));
-        pieces->push_back("]: ");
-      }
-      pieces->push_back(literal.GetSparseElementAsString(i));
+    if (rank == 1) {
+      pieces->push_back(StrCat(literal.GetSparseIndex(i)[0]));
+      pieces->push_back(": ");
+    } else {
+      pieces->push_back("[");
+      pieces->push_back(absl::StrJoin(literal.GetSparseIndex(i), ", "));
+      pieces->push_back("]: ");
     }
-    pieces->push_back("}");
-    return;
+    pieces->push_back(literal.GetSparseElementAsString(i));
   }
+  pieces->push_back("}");
+}
 
-  CHECK(LayoutUtil::IsDenseArray(subshape));
-
-  auto element_to_string = [&](absl::Span<const int64> indices) -> string {
-    PrimitiveType element_type = subshape.element_type();
-    // We display predicates as 0s and 1s so that the string is more dense.
-    string elem = element_type == PRED
-                      ? literal.Get<bool>(indices, shape_index) ? "1" : "0"
-                      : literal.GetAsString(indices, shape_index);
-    return ((!indices.empty() && indices.back() > 0) ? ", " : "") + elem;
-  };
+void DenseArrayToStringHelper(const LiteralBase& literal,
+                              const ShapeIndex& shape_index, bool print_layout,
+                              std::vector<string>* pieces) {
+  const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index);
+  int64 rank = ShapeUtil::Rank(subshape);
+
+  std::function<void(absl::Span<const int64> dimensions, std::vector<int64>*)>
+      to_string_recursive = [&](absl::Span<const int64> dimensions,
+                                std::vector<int64>* accum_indices) {
+        // dimensions.size() decreases by 1 at each recursive call,
+        // and accum_indices->size() increases by 1.
+        // Their sum is equal to the rank of the tensor.
+        CHECK_EQ(rank, dimensions.size() + accum_indices->size());
+
+        auto brace_to_string = [&](string brace) -> string {
+          // Handle 1D tensor
+          if (rank == 1) {
+            return brace;
+          }
+          // Handle the innermost tensor of a 2D+ tensor.
+          if (dimensions.size() == 1 && brace == "{") {
+            return StrCat("  ", brace, dimensions[0] <= 1 ? "" : " ");
+          }
+          if (dimensions.size() == 1 && brace == "}") {
+            return StrCat(dimensions[0] <= 1 ? "" : " ", brace);
+          }
+          // Handle the non-innermost tensors of a 2D+ tensor.
+          if (brace == "{") {
+            if (rank > 3 && !accum_indices->empty() &&
+                accum_indices->size() < rank) {
+              int index = accum_indices->size() - 1;
+              int value = accum_indices->back();
+              return StrCat(brace, " /*i", index, "=", value, "*/\n");
+            }
+            return StrCat(brace, "\n");
+          }
+          return StrCat("\n", brace);
+        };
 
-  if (ShapeUtil::Rank(subshape) == 0) {
-    pieces->push_back(literal.GetAsString({}, shape_index));
-  } else if (ShapeUtil::Rank(subshape) == 1) {
-    pieces->push_back("{");
-    for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) {
-      pieces->push_back(element_to_string({i0}));
-    }
-    pieces->push_back("}");
-  } else if (ShapeUtil::Rank(subshape) == 2) {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back(" {\n");
-    for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) {
-      pieces->push_back("  { ");
-      for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) {
-        pieces->push_back(element_to_string({i0, i1}));
-      }
-      pieces->push_back(" ");
-      pieces->push_back(i0 == subshape.dimensions(0) - 1 ? "}\n" : "},\n");
-    }
-    pieces->push_back("}");
-  } else if (ShapeUtil::Rank(subshape) == 3) {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back(" {\n");
-    for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) {
-      pieces->push_back(i0 > 0 ? ",\n{" : "{");
-      for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) {
-        pieces->push_back(i1 > 0 ? ",\n  { " : " { ");
-        for (int64 i2 = 0; i2 < subshape.dimensions(2); ++i2) {
-          pieces->push_back(element_to_string({i0, i1, i2}));
-        }
-        pieces->push_back(" }");
-      }
-      pieces->push_back(" }");
-    }
-    pieces->push_back("\n}");
-  } else if (ShapeUtil::Rank(subshape) == 4) {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back(" {\n");
-    for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) {
-      pieces->push_back(StrFormat("  {  /*i0=%d*/\n", i0));
-      for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) {
-        pieces->push_back(StrFormat("    {  /*i1=%d*/\n", i1));
-        for (int64 i2 = 0; i2 < subshape.dimensions(2); ++i2) {
-          pieces->push_back("      {");
-          for (int64 i3 = 0; i3 < subshape.dimensions(3); ++i3) {
-            pieces->push_back(element_to_string({i0, i1, i2, i3}));
+        if (dimensions.empty()) {
+          // Display predicates as 0s and 1s so that the string is more dense.
+          string elem;
+          if (subshape.element_type() == PRED && rank > 0) {
+            elem = literal.Get<bool>(*accum_indices, shape_index) ? "1" : "0";
+          } else {
+            elem = literal.GetAsString(*accum_indices, shape_index);
           }
-          pieces->push_back(i2 == subshape.dimensions(2) - 1 ? "}\n" : "},\n");
-        }
-        pieces->push_back(i1 == subshape.dimensions(1) - 1 ? "    }\n"
-                                                           : "    },\n");
-      }
-      pieces->push_back(i0 == subshape.dimensions(0) - 1 ? "  }\n" : "  },\n");
-    }
-    pieces->push_back("}");
-  } else if (ShapeUtil::Rank(subshape) == 5) {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back(" {\n");
-    for (int64 i0 = 0; i0 < subshape.dimensions(0); ++i0) {
-      pieces->push_back(StrFormat("  {  /*i0=%d*/\n", i0));
-      for (int64 i1 = 0; i1 < subshape.dimensions(1); ++i1) {
-        pieces->push_back(StrFormat("    {  /*i1=%d*/\n", i1));
-        for (int64 i2 = 0; i2 < subshape.dimensions(2); ++i2) {
-          pieces->push_back(StrFormat("      {  /*i2=%d*/\n", i2));
-          for (int64 i3 = 0; i3 < subshape.dimensions(3); ++i3) {
-            pieces->push_back("        {");
-            for (int64 i4 = 0; i4 < subshape.dimensions(4); ++i4) {
-              pieces->push_back(element_to_string({i0, i1, i2, i3, i4}));
+          pieces->push_back(elem);
+        } else {
+          pieces->push_back(brace_to_string("{"));
+          for (int i = 0; i < dimensions[0]; ++i) {
+            std::vector<int64> cloned_indices(*accum_indices);
+            cloned_indices.push_back(i);
+            to_string_recursive(dimensions.subspan(1), &cloned_indices);
+            if (i < dimensions[0] - 1) {
+              pieces->push_back(",");
+              pieces->push_back(dimensions.size() > 1 ? "\n" : " ");
             }
-            pieces->push_back(i3 == subshape.dimensions(3) - 1 ? "}\n"
-                                                               : "},\n");
           }
-          pieces->push_back(i2 == subshape.dimensions(2) - 1 ? "      }\n"
-                                                             : "      },\n");
+          pieces->push_back(brace_to_string("}"));
+          return;
         }
-        pieces->push_back(i1 == subshape.dimensions(1) - 1 ? "    }\n"
-                                                           : "    },\n");
-      }
-      pieces->push_back(i0 == subshape.dimensions(0) - 1 ? "  }\n" : "  },\n");
-    }
-    pieces->push_back("}");
+      };
+
+  if (rank > 1) {
+    pieces->push_back(ShapeToString(print_layout, subshape));
+    pieces->push_back(" ");
+  }
+  std::vector<int64> indices = {};
+  std::vector<int64> dimensions(subshape.dimensions().begin(),
+                                subshape.dimensions().end());
+  to_string_recursive(dimensions, &indices);
+}
+
+void ToStringHelper(const LiteralBase& literal, const ShapeIndex& shape_index,
+                    bool print_layout, std::vector<string>* pieces) {
+  const Shape& subshape = ShapeUtil::GetSubshape(literal.shape(), shape_index);
+  CHECK(LayoutUtil::HasLayout(literal.shape()));
+  CHECK(LayoutUtil::HasLayout(subshape));
+  if (ShapeUtil::IsTuple(subshape)) {
+    TupleToStringHelper(literal, shape_index, print_layout, pieces);
+  } else if (ShapeUtil::IsToken(subshape)) {
+    pieces->push_back("token");
+  } else if (LayoutUtil::IsSparseArray(subshape)) {
+    SparseArrayToStringHelper(literal, subshape, print_layout, pieces);
   } else {
-    pieces->push_back(shape_to_string(subshape));
-    pieces->push_back(" {");
-    literal.EachCellAsString(
-        [&](absl::Span<const int64> indices, const string& value) {
-          pieces->push_back(" ");
-          pieces->push_back(value);
-        });
-    pieces->push_back("}");
+    CHECK(LayoutUtil::IsDenseArray(subshape));
+    DenseArrayToStringHelper(literal, shape_index, print_layout, pieces);
   }
 }
 
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index 8cec37897a..dac6067861 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -150,12 +150,58 @@ TEST_F(LiteralUtilTest, R3ToString) {
   const auto literal =
       LiteralUtil::CreateR3({{{1}, {2}}, {{3}, {4}}, {{5}, {6}}});
   const string expected = R"(s32[3,2,1] {
-{ { 1 },
-  { 2 } },
-{ { 3 },
-  { 4 } },
-{ { 5 },
-  { 6 } }
+{
+  {1},
+  {2}
+},
+{
+  {3},
+  {4}
+},
+{
+  {5},
+  {6}
+}
+})";
+  EXPECT_EQ(expected, literal.ToString());
+}
+
+TEST_F(LiteralUtilTest, R6ToString) {
+  const auto literal =
+      LiteralUtil::CreateFromDimensions(S32, {2, 2, 1, 1, 1, 2});
+  const string expected = R"(s32[2,2,1,1,1,2] {
+{ /*i0=0*/
+{ /*i1=0*/
+{ /*i2=0*/
+{ /*i3=0*/
+  { 0, 0 }
+}
+}
+},
+{ /*i1=1*/
+{ /*i2=0*/
+{ /*i3=0*/
+  { 0, 0 }
+}
+}
+}
+},
+{ /*i0=1*/
+{ /*i1=0*/
+{ /*i2=0*/
+{ /*i3=0*/
+  { 0, 0 }
+}
+}
+},
+{ /*i1=1*/
+{ /*i2=0*/
+{ /*i3=0*/
+  { 0, 0 }
+}
+}
+}
+}
 })";
   EXPECT_EQ(expected, literal.ToString());
 }
@@ -190,12 +236,16 @@ TEST_F(LiteralUtilTest, CreateR3FromArray3d) {
   EXPECT_THAT(literal.shape().dimensions(), ElementsAre(2, 3, 2));
   string result = literal.ToString();
   const string expected = R"(f32[2,3,2] {
-{ { 1, 2 },
+{
+  { 1, 2 },
   { 3, 4 },
-  { 5, 6 } },
-{ { 7, 8 },
+  { 5, 6 }
+},
+{
+  { 7, 8 },
   { 9, 10 },
-  { 11, 12 } }
+  { 11, 12 }
+}
 })";
   EXPECT_EQ(expected, result);
 }
@@ -247,18 +297,18 @@ TEST_F(LiteralUtilTest, LiteralR4F32ProjectedStringifies) {
   EXPECT_THAT(literal.shape().dimensions(), ElementsAre(1, 2, 3, 2));
   string result = literal.ToString();
   const string expected = R"(f32[1,2,3,2] {
-  {  /*i0=0*/
-    {  /*i1=0*/
-      {1, 2},
-      {1001, 1002},
-      {2001, 2002}
-    },
-    {  /*i1=1*/
-      {1, 2},
-      {1001, 1002},
-      {2001, 2002}
-    }
-  }
+{ /*i0=0*/
+{ /*i1=0*/
+  { 1, 2 },
+  { 1001, 1002 },
+  { 2001, 2002 }
+},
+{ /*i1=1*/
+  { 1, 2 },
+  { 1001, 1002 },
+  { 2001, 2002 }
+}
+}
 })";
   EXPECT_EQ(expected, result);
 }
@@ -268,30 +318,30 @@ TEST_F(LiteralUtilTest, LiteralR4F32Stringifies) {
               ElementsAre(2, 2, 3, 3));
   string result = literal_r4_2x2x3x3_dim0major_.ToString();
   const string expected = R"(f32[2,2,3,3] {
-  {  /*i0=0*/
-    {  /*i1=0*/
-      {1, 2, 3},
-      {4, 5, 6},
-      {7, 8, 9}
-    },
-    {  /*i1=1*/
-      {11, 12, 13},
-      {14, 15, 16},
-      {17, 18, 19}
-    }
-  },
-  {  /*i0=1*/
-    {  /*i1=0*/
-      {101, 102, 103},
-      {104, 105, 106},
-      {107, 108, 109}
-    },
-    {  /*i1=1*/
-      {201, 202, 203},
-      {204, 205, 206},
-      {207, 208, 209}
-    }
-  }
+{ /*i0=0*/
+{ /*i1=0*/
+  { 1, 2, 3 },
+  { 4, 5, 6 },
+  { 7, 8, 9 }
+},
+{ /*i1=1*/
+  { 11, 12, 13 },
+  { 14, 15, 16 },
+  { 17, 18, 19 }
+}
+},
+{ /*i0=1*/
+{ /*i1=0*/
+  { 101, 102, 103 },
+  { 104, 105, 106 },
+  { 107, 108, 109 }
+},
+{ /*i1=1*/
+  { 201, 202, 203 },
+  { 204, 205, 206 },
+  { 207, 208, 209 }
+}
+}
 })";
   EXPECT_EQ(expected, result);
 }
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index c59bdc0a0b..88682e55fb 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -195,7 +195,7 @@ ENTRY %add_constants () -> f32[] {
 R"(HloModule TupleConstant_module
 
 ENTRY %TupleConstant.v1 () -> (f32[2,1], f32[2]) {
-  ROOT %constant = (f32[2,1]{1,0}, f32[2]{0}) constant((f32[2,1], f32[2]) ( f32[2,1] { { 1 }, { 2 } }, {2, 42} ))
+  ROOT %constant = (f32[2,1]{1,0}, f32[2]{0}) constant((f32[2,1], f32[2]) ( f32[2,1] { {1}, {2} }, {2, 42} ))
 }
 
 )"
@@ -587,7 +587,7 @@ ENTRY %DynamicUpdateSlice.v4 (input: s32[1,1,25,1], update: s32[1,1,2,1], start_
 R"(HloModule BasicTraining_module
 
 ENTRY %BasicTraining.v4 () -> (f32[2,2,1,2], f32[2], f32[2]) {
-  %constant = f32[2,2,1,2]{3,2,1,0} constant(f32[2,2,1,2] { { /*i0=0*/ { /*i1=0*/ {1, 2} }, { /*i1=1*/ {3, 4} } }, { /*i0=1*/ { /*i1=0*/ {5, 6} }, { /*i1=1*/ {7, 8} } } })
+  %constant = f32[2,2,1,2]{3,2,1,0} constant(f32[2,2,1,2] { { /*i0=0*/ { /*i1=0*/ { 1, 2 } }, { /*i1=1*/ { 3, 4 } } }, { /*i0=1*/ { /*i1=0*/ { 5, 6 } }, { /*i1=1*/ { 7, 8 } } } })
   %constant.1 = f32[2]{0} constant({2, 3})
   %constant.2 = f32[2]{0} constant({1, 2})
   ROOT %batch-norm-training = (f32[2,2,1,2]{3,2,1,0}, f32[2]{0}, f32[2]{0}) batch-norm-training(f32[2,2,1,2]{3,2,1,0} %constant, f32[2]{0} %constant.1, f32[2]{0} %constant.2), epsilon=0.001, feature_index=3
diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
index 20cc18f981..98246d5403 100644
--- a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
@@ -481,8 +481,8 @@ ENTRY main {
   const char* expected_root_expression = R"(
 (scalar-indexed-const
   (constant s32[2,1,1,1,6] s32[2,1,1,1,6] {
-    { /*i0=0*/ { /*i1=0*/ { /*i2=0*/ {1, 2, 3, 4, 5, 6} } } },
-    { /*i0=1*/ { /*i1=0*/ { /*i2=0*/ {1, 2, 3, 4, 5, 6} } } } })
+    { /*i0=0*/ { /*i1=0*/ { /*i2=0*/ { 1, 2, 3, 4, 5, 6 } } } },
+    { /*i0=1*/ { /*i1=0*/ { /*i2=0*/ { 1, 2, 3, 4, 5, 6 } } } } })
   (reshape %indices to s32[])
   0->[])
 )";
@@ -512,8 +512,8 @@ ENTRY main {
   const char* expected_root_expression = R"(
 (scalar-indexed-const
   (constant s32[2,1,1,6] s32[2,1,1,6] {
-    { /*i0=0*/ { /*i1=0*/ {1, 2, 3, 4, 5, 6} } },
-    { /*i0=1*/ { /*i1=0*/ {1, 2, 3, 4, 5, 6} } } })
+    { /*i0=0*/ { /*i1=0*/ { 1, 2, 3, 4, 5, 6 } } },
+    { /*i0=1*/ { /*i1=0*/ { 1, 2, 3, 4, 5, 6 } } } })
   (reshape %indices to s32[5])
   0->[2])
 )";
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 2180b22cb3..0615f9425c 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -2744,12 +2744,16 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGtR3F32sWithDegenerateDim2) {
   Array3D<int> expected_3d(
       {{{0, 1}, {0, 0}, {0, 0}}, {{0, 1}, {1, 0}, {0, 1}}});
   const string expected = R"(pred[2,3,2] {
-{ { 0, 1 },
+{
+  { 0, 1 },
   { 0, 0 },
-  { 0, 0 } },
-{ { 0, 1 },
+  { 0, 0 }
+},
+{
+  { 0, 1 },
   { 1, 0 },
-  { 0, 1 } }
+  { 0, 1 }
+}
 })";
   EXPECT_EQ(expected, ExecuteToString(&builder, {}));
 }
-- 
GitLab


From 837072ef149aaf7c2563271a5f52de98d2af72e1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 15:20:59 -0800
Subject: [PATCH 0351/1554] Another try at handling ops that are known to TOCO
 but not to TFLite. If they aren't builtins, and we are in flex mode, and we
 have a NodeDef, we must export them as flex ops.

PiperOrigin-RevId: 221700244
---
 tensorflow/lite/toco/tflite/export.cc      | 30 ++++++++++++----
 tensorflow/lite/toco/tflite/export_test.cc | 40 +++++++++++++++-------
 2 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/tensorflow/lite/toco/tflite/export.cc b/tensorflow/lite/toco/tflite/export.cc
index 489c21295e..f17ce900eb 100644
--- a/tensorflow/lite/toco/tflite/export.cc
+++ b/tensorflow/lite/toco/tflite/export.cc
@@ -332,6 +332,11 @@ Offset<Vector<Offset<Operator>>> ExportOperators(
     std::set<int32_t>* variable_tensor_indices, const ExportParams& params) {
   variable_tensor_indices->clear();
 
+  auto is_tflite_builtin = [](const BaseOperator* op) {
+    const auto& tflite_builtins = GetBuiltinOpsMap();
+    return (op && tflite_builtins.find(op->name()) != tflite_builtins.end());
+  };
+
   // The operators are in execution order, so we just follow tf.mini order.
   std::vector<Offset<Operator>> op_vector;
   for (const auto& op : model.operators) {
@@ -360,7 +365,19 @@ Offset<Vector<Offset<Operator>>> ExportOperators(
     auto options = Options::Custom(0);
 
     std::vector<bool> mutating_input_variables;
-    if (tflite_op) {
+
+    // It is conceivable that an op is exportable via Serialize() but does not
+    // have a corresponding TFLITE builtin. In that case, when flex mode is
+    // enabled we should export it as a flex op, not as a native.
+    bool export_as_flex_op = !is_tflite_builtin(tflite_op) &&
+                             key.is_flex_op() &&
+                             !op->tensorflow_node_def.empty();
+    if (export_as_flex_op) {
+      auto fbb = WriteFlexOpOptions(op->tensorflow_node_def);
+      if (fbb) {
+        options = Options::Custom(builder->CreateVector(fbb->GetBuffer()));
+      }
+    } else if (tflite_op) {
       options = tflite_op->Serialize(*op, builder);
       mutating_input_variables = tflite_op->GetMutatingInputVariables(*op);
 
@@ -373,12 +390,13 @@ Offset<Vector<Offset<Operator>>> ExportOperators(
           variable_tensor_indices->insert(variable_tensor_index);
         }
       }
-    } else if (key.is_flex_op() && !op->tensorflow_node_def.empty()) {
-      auto fbb = WriteFlexOpOptions(op->tensorflow_node_def);
-      if (fbb) {
-        options = Options::Custom(builder->CreateVector(fbb->GetBuffer()));
-      }
+    } else {
+      // We don't know much about this op. It doesn't have a serializer and
+      // it is not supposed to be exported as a flex op. We will treat it as
+      // a regular custom op: we will still create an operator for it, but it
+      // will not have any 'options'.
     }
+
     // The only supported CustomOptionFormat is FLEXBUFFERS now.
     op_vector.push_back(CreateOperator(
         *builder, op_index, builder->CreateVector(inputs),
diff --git a/tensorflow/lite/toco/tflite/export_test.cc b/tensorflow/lite/toco/tflite/export_test.cc
index b6c67772ac..b371296784 100644
--- a/tensorflow/lite/toco/tflite/export_test.cc
+++ b/tensorflow/lite/toco/tflite/export_test.cc
@@ -46,6 +46,18 @@ class ExportTest : public ::testing::Test {
         input_model_.operators.emplace_back(new AddOperator);
       } else if (name == "Sub") {
         input_model_.operators.emplace_back(new SubOperator);
+      } else if (name == "Assert") {
+        auto* op = new TensorFlowAssertOperator;
+
+        // Even though assert is known to TOCO, it doesn't have a tflite
+        // serializer, so it has to be exported as a custom op. If we attach a
+        // NodeDef to it, however, it will be exported as a flex op instead.
+        ::tensorflow::NodeDef node_def;
+        node_def.set_name("Assert");
+        node_def.set_op("Assert");
+        node_def.SerializeToString(&op->tensorflow_node_def);
+
+        input_model_.operators.emplace_back(op);
       } else {
         auto* op = new TensorFlowUnsupportedOperator;
         op->tensorflow_op = name;
@@ -232,37 +244,38 @@ class OpSetsTest : public ExportTest {
 TEST_F(OpSetsTest, BuiltinsOnly) {
   // --target_op_set=TFLITE_BUILTINS
   SetAllowedOpSets({kTfLiteBuiltins});
-  EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold"}),
+  EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold", "Assert"}),
               ElementsAre());
   EXPECT_THAT(ImportExport({"Add"}), ElementsAre("builtin:ADD"));
 
   // --target_op_set=TFLITE_BUILTINS --allow_custom_ops
   SetAllowedOpSets({kTfLiteBuiltins, kCustomOps});
-  EXPECT_THAT(
-      ImportExport({"Add", "AdjustHue", "UnrollAndFold"}),
-      ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:UnrollAndFold"));
+  EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold", "Assert"}),
+              ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:Assert",
+                          "custom:UnrollAndFold"));
 }
 
 TEST_F(OpSetsTest, TfSelectOnly) {
   // --target_op_set=SELECT_TF_OPS
   SetAllowedOpSets({kSelectTfOps});
-  EXPECT_THAT(
-      ImportExport({"Add", "AdjustHue", "RandomUniform", "UnrollAndFold"}),
-      ElementsAre());
+  EXPECT_THAT(ImportExport({"Add", "AdjustHue", "RandomUniform",
+                            "UnrollAndFold", "Assert"}),
+              ElementsAre());
   EXPECT_THAT(ImportExport({"Add"}), ElementsAre("custom:FlexAdd"));
 
   // --target_op_set=SELECT_TF_OPS --allow_custom_ops
   SetAllowedOpSets({kSelectTfOps, kCustomOps});
   EXPECT_THAT(
-      ImportExport({"Add", "AdjustHue", "RandomUniform", "UnrollAndFold"}),
-      ElementsAre("custom:AdjustHue", "custom:FlexAdd",
+      ImportExport(
+          {"Add", "AdjustHue", "RandomUniform", "UnrollAndFold", "Assert"}),
+      ElementsAre("custom:AdjustHue", "custom:FlexAdd", "custom:FlexAssert",
                   "custom:FlexRandomUniform", "custom:UnrollAndFold"));
 }
 
 TEST_F(OpSetsTest, BuiltinsAndTfSelect) {
   // --target_op_set=TFLITE_BUILTINS,SELECT_TF_OPS
   SetAllowedOpSets({kTfLiteBuiltins, kSelectTfOps});
-  EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold"}),
+  EXPECT_THAT(ImportExport({"Add", "AdjustHue", "UnrollAndFold", "Assert"}),
               ElementsAre());
   EXPECT_THAT(ImportExport({"Add", "RandomUniform"}),
               ElementsAre("builtin:ADD", "custom:FlexRandomUniform"));
@@ -270,9 +283,10 @@ TEST_F(OpSetsTest, BuiltinsAndTfSelect) {
   // --target_op_set=TFLITE_BUILTINS,SELECT_TF_OPS --allow_custom_ops
   SetAllowedOpSets({kTfLiteBuiltins, kSelectTfOps, kCustomOps});
   EXPECT_THAT(
-      ImportExport({"Add", "AdjustHue", "RandomUniform", "UnrollAndFold"}),
-      ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:FlexRandomUniform",
-                  "custom:UnrollAndFold"));
+      ImportExport(
+          {"Add", "AdjustHue", "RandomUniform", "UnrollAndFold", "Assert"}),
+      ElementsAre("builtin:ADD", "custom:AdjustHue", "custom:FlexAssert",
+                  "custom:FlexRandomUniform", "custom:UnrollAndFold"));
 }
 
 // This test is based on a hypothetical scenario that dilation is supported
-- 
GitLab


From 0edab17ef54ed8211818c3365499928b4850f621 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 15:22:57 -0800
Subject: [PATCH 0352/1554] Step 1 of DistributionStrategyExtended transition.

PiperOrigin-RevId: 221700595
---
 tensorflow/contrib/distribute/__init__.py     |   1 +
 .../python/collective_all_reduce_strategy.py  |  55 +-
 .../collective_all_reduce_strategy_test.py    |   9 +-
 .../contrib/distribute/python/keras_test.py   |   2 +-
 .../distribute/python/metrics_v1_test.py      |   5 +-
 .../distribute/python/mirrored_strategy.py    |  73 +-
 .../python/mirrored_strategy_multigpu_test.py |   8 +-
 .../distribute/python/one_device_strategy.py  |  38 +-
 .../python/parameter_server_strategy.py       |  47 +-
 .../python/parameter_server_strategy_test.py  |  19 +-
 .../contrib/distribute/python/tpu_strategy.py |  60 +-
 .../contrib/distribute/python/values.py       |   8 +-
 tensorflow/python/BUILD                       |   1 +
 .../engine/distributed_training_utils.py      |   6 +-
 .../keras/engine/training_distributed.py      |  11 +-
 tensorflow/python/ops/metrics_impl.py         |   8 +-
 tensorflow/python/training/distribute.py      | 886 +++++++++++-------
 tensorflow/python/training/distribute_test.py |   9 +-
 18 files changed, 770 insertions(+), 476 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index ab2f221dc6..69784c737a 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -46,6 +46,7 @@ _allowed_symbols = [
     'CrossDeviceOps',
     'DistributeConfig',
     'DistributionStrategy',
+    'DistributionStrategyExtended',
     'MirroredStrategy',
     'Monitor',
     'MultiWorkerAllReduce',
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index a2922b9881..dd7d29ac2b 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -33,7 +33,7 @@ from tensorflow.python.training import distribute as distribute_lib
 
 
 # TODO(yuefengz): support in-graph replication.
-class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
+class CollectiveAllReduceStrategy(distribute_lib.DistributionStrategy):
   """Distribution strategy that uses collective ops for all-reduce.
 
   It is similar to the MirroredStrategy but it uses collective ops for
@@ -54,10 +54,20 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
       num_gpus_per_worker: number of local GPUs or GPUs per worker, the default
         is 0 meaning CPU only.
     """
+    super(CollectiveAllReduceStrategy, self).__init__(
+        CollectiveAllReduceExtended(self, num_gpus_per_worker))
+
+
+class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
+  """Implementation of CollectiveAllReduceStrategy."""
+
+  def __init__(self, container_strategy, num_gpus_per_worker):
+    distribute_lib.DistributionStrategyExtended.__init__(
+        self, container_strategy)
     self._num_gpus_per_worker = num_gpus_per_worker
-    self._initialize_local_worker(num_gpus_per_worker)
+    self._initialize_local_worker(container_strategy, num_gpus_per_worker)
 
-  def _initialize_local_worker(self, num_gpus_per_worker):
+  def _initialize_local_worker(self, container_strategy, num_gpus_per_worker):
     """Initializes the object for local training."""
     self._is_chief = True
     self._num_workers = 1
@@ -70,9 +80,10 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
       local_devices = ["/device:CPU:0"]
 
     self._collective_keys = cross_tower_utils.CollectiveKeys()
-    super(CollectiveAllReduceStrategy, self).__init__(
+    super(CollectiveAllReduceExtended, self).__init__(
+        container_strategy,
         devices=local_devices,
-        cross_tower_ops=cross_tower_ops_lib.CollectiveAllReduce(
+        cross_device_ops=cross_tower_ops_lib.CollectiveAllReduce(
             num_workers=1,
             num_gpus_per_worker=num_gpus_per_worker,
             collective_keys=self._collective_keys))
@@ -84,8 +95,8 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
     logging.info("CollectiveAllReduceStrategy with local_devices = %r",
                  local_devices)
 
-  def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec,
-                               task_type, task_id):
+  def _initialize_multi_worker(self, container_strategy, num_gpus_per_worker,
+                               cluster_spec, task_type, task_id):
     """Initializes the object for multi-worker training."""
     if task_type is None or task_id is None:
       raise ValueError("When `cluster_spec` is given, you must also specify "
@@ -113,9 +124,10 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
       local_devices = [worker_device]
 
     self._collective_keys = cross_tower_utils.CollectiveKeys()
-    super(CollectiveAllReduceStrategy, self).__init__(
+    super(CollectiveAllReduceExtended, self).__init__(
+        container_strategy,
         devices=local_devices,
-        cross_tower_ops=cross_tower_ops_lib.CollectiveAllReduce(
+        cross_device_ops=cross_tower_ops_lib.CollectiveAllReduce(
             num_workers=self._num_workers,
             num_gpus_per_worker=num_gpus_per_worker,
             collective_keys=self._collective_keys))
@@ -202,7 +214,7 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
     return mirrored_strategy._create_mirrored_variable(
         devices, _real_mirrored_creator, *args, **kwargs)
 
-  def distribute_dataset(self, dataset_fn):
+  def _distribute_dataset(self, dataset_fn):
     """Distributes the dataset to each local GPU."""
     # TODO(yuefengz): shard the dataset.
     return values.PerReplicaDataset(
@@ -221,15 +233,15 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
     input_context = distribute_lib.InputContext(
         num_input_pipelines=self._num_workers,
         input_pipeline_id=input_pipeline_id,
-        num_replicas_in_sync=self.num_replicas_in_sync)
+        num_replicas_in_sync=self._num_replicas_in_sync)
     return values.PerReplicaDataset(
         self._call_dataset_fn(input_fn, input_context), self._devices, True)
 
-  def configure(self,
-                session_config=None,
-                cluster_spec=None,
-                task_type=None,
-                task_id=None):
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
     """Configures the object.
 
     Args:
@@ -246,8 +258,9 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
       # If a `cluster_spec` is already passed in, do nothing here.
       # TODO(yuefengz): check `cluster_spec` is the same if this object has
       # already been initialized with a `cluster_spec`.
-      self._initialize_multi_worker(self._num_gpus_per_worker, cluster_spec,
-                                    task_type, task_id)
+      self._initialize_multi_worker(
+          self._container_strategy(), self._num_gpus_per_worker, cluster_spec,
+          task_type, task_id)
 
     if not session_config:
       return
@@ -288,11 +301,11 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
         "/job:%s/task:%d" % (self._task_type, self._task_id))
 
   @property
-  def between_graph(self):
+  def experimental_between_graph(self):
     return True
 
   @property
-  def should_init(self):
+  def experimental_should_init(self):
     return True
 
   @property
@@ -304,5 +317,5 @@ class CollectiveAllReduceStrategy(mirrored_strategy.MirroredStrategy):
     return self._is_chief
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     return len(self._devices) * self._num_workers
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 28e69d2a5f..ae7b88aa2d 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -80,8 +80,8 @@ class CollectiveAllReduceStrategyTestBase(
         CollectiveAllReduceStrategyTestBase.collective_key_base,
         instance_key_with_id_start=num_gpus * 10000 +
         CollectiveAllReduceStrategyTestBase.collective_key_base)
-    distribution._collective_keys = collective_keys
-    distribution._cross_tower_ops._collective_keys = collective_keys
+    distribution.extended._collective_keys = collective_keys
+    distribution.extended._cross_tower_ops._collective_keys = collective_keys
     if task_type and task_id is not None:
       return distribution, 'grpc://' + self._cluster_spec[task_type][
           task_id], session_config
@@ -95,7 +95,8 @@ class CollectiveAllReduceStrategyTestBase(
          self.cached_session(config=config,
                              target=master_target) as sess, \
          d.scope():
-      l = core.Dense(1, use_bias=False, name='gpu_%d' % d._num_gpus_per_worker)
+      l = core.Dense(1, use_bias=False,
+                     name='gpu_%d' % d.extended._num_gpus_per_worker)
 
       def loss_fn(x):
         y = array_ops.reshape(l(x), []) - constant_op.constant(1.)
@@ -138,7 +139,7 @@ class CollectiveAllReduceStrategyTestBase(
 
       before_out, after_out = step()
 
-      if context.num_gpus() < d._num_gpus_per_worker:
+      if context.num_gpus() < d.extended._num_gpus_per_worker:
         return True
 
       sess.run(
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index c16fc52c9a..abf9e78adc 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -521,7 +521,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       loss = 'mse'
       strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
                                                      '/device:CPU:0'])
-      strategy._require_static_shapes = True
+      strategy.extended._require_static_shapes = True
 
       model.compile(optimizer, loss, distribute=strategy)
       iterator = model._distribution_standardize_user_data(inputs,
diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 8289f7e212..a2fc118173 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -103,13 +103,14 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
           return distribution.group(update)
 
         ctx = distribution.run_steps_on_dataset(
-            step_fn, iterator, iterations=distribution.steps_per_run)
+            step_fn, iterator, iterations=distribution.extended.steps_per_run)
         update = ctx.run_op
         value = ctx.non_tensor_outputs["value"]
         # In each run, we run multiple steps, and each steps consumes as many
         # batches as number of replicas.
         batches_per_update = (
-            distribution.num_replicas_in_sync * distribution.steps_per_run)
+            distribution.num_replicas_in_sync *
+            distribution.extended.steps_per_run)
       else:
         value, update = distribution.call_for_each_replica(
             metric_fn, iterator.get_next())
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 990f36ce2e..960cd09696 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -105,10 +105,10 @@ def _call_for_each_replica(distribution, fn, args, kwargs):
   # TODO(isaprykin): Create these threads once instead of during every run()
   # call.
   threads = []
-  for index, d in enumerate(distribution.worker_devices):
+  for index, d in enumerate(distribution.extended.worker_devices):
     variable_creator_fn = shared_variable_creator.make_fn(
         shared_variable_store, index)
-    t = MirroredStrategy._MirroredReplicaThread(  # pylint: disable=protected-access
+    t = MirroredExtended._MirroredReplicaThread(  # pylint: disable=protected-access
         distribution, coord, d, variable_creator_fn, fn,
         *values.select_device(d, args), **values.select_device(d, kwargs))
     threads.append(t)
@@ -180,8 +180,7 @@ def _call_for_each_replica(distribution, fn, args, kwargs):
   return values.regroup({t.device: t.main_result for t in threads})
 
 
-def _reduce_non_distributed_value(distribution, reduce_op, value,
-                                  destinations):
+def _reduce_non_distributed_value(extended, reduce_op, value, destinations):
   """Reduce a non-DistributedValue `value` to `destinations`."""
   if isinstance(value, values.DistributedValues):
     raise ValueError("You are passing a `DistributedValue` to "
@@ -202,7 +201,7 @@ def _reduce_non_distributed_value(distribution, reduce_op, value,
   # We do not support a reduce op of SUM if the value is the same across
   # all replicas. We call this as part of assign functions for MirroredVariables
   # and summing up identical values across replicas is not clearly defined.
-  if (len(distribution.worker_devices) != 1 or
+  if (len(extended.worker_devices) != 1 or
       not cross_tower_ops_lib.check_destinations(destinations)):
     raise ValueError("A non-DistributedValues value %s cannot be reduced with "
                      "the given reduce op %s." % (value, reduce_op))
@@ -354,10 +353,26 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                cross_device_ops=None,
                auto_shard_dataset=False,
                cross_tower_ops=None):
-    super(MirroredStrategy, self).__init__()
-
     assert not (cross_device_ops and cross_tower_ops)
-    self._cross_tower_ops = cross_device_ops or cross_tower_ops
+    extended = MirroredExtended(
+        self, devices, num_gpus, num_gpus_per_worker,
+        cross_device_ops or cross_tower_ops, auto_shard_dataset)
+    super(MirroredStrategy, self).__init__(extended)
+
+
+class MirroredExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of MirroredStrategy."""
+
+  def __init__(self,
+               container_strategy,
+               devices=None,
+               num_gpus=None,
+               num_gpus_per_worker=None,
+               cross_device_ops=None,
+               auto_shard_dataset=False):
+    super(MirroredExtended, self).__init__(container_strategy)
+    # TODO(josh11b): Rename self._cross_tower_ops -> self._cross_device_ops
+    self._cross_tower_ops = cross_device_ops
     self._auto_shard_dataset = auto_shard_dataset
     # Remember num GPUs which might be needed by `configure` method.
     if num_gpus is not None and num_gpus_per_worker is not None:
@@ -478,7 +493,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     return _create_mirrored_variable(devices, _real_mirrored_creator, *args,
                                      **kwargs)
 
-  def distribute_dataset(self, dataset_fn):
+  def _distribute_dataset(self, dataset_fn):
     if self._cluster_spec:
       return values.MultiWorkerDataset(
           partial(self._call_dataset_fn, dataset_fn), self._worker_devices,
@@ -497,7 +512,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
         input_context = distribute_lib.InputContext(
             num_input_pipelines=len(self._worker_devices),
             input_pipeline_id=i,
-            num_replicas_in_sync=self.num_replicas_in_sync)
+            num_replicas_in_sync=self._num_replicas_in_sync)
         input_fns.append(
             partial(self._call_dataset_fn, input_fn, input_context))
 
@@ -507,13 +522,13 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
       input_context = distribute_lib.InputContext(
           num_input_pipelines=1,
           input_pipeline_id=0,
-          num_replicas_in_sync=self.num_replicas_in_sync)
+          num_replicas_in_sync=self._num_replicas_in_sync)
       return values.PerReplicaDataset(
           self._call_dataset_fn(input_fn, input_context), self._devices)
 
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
-  def _run_steps_on_dataset(self, fn, iterator, iterations,
-                            initial_loop_values=None):
+  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
+                                          initial_loop_values=None):
     if initial_loop_values is None:
       initial_loop_values = {}
     initial_loop_values = nest.flatten(initial_loop_values)
@@ -528,7 +543,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
       fn_result = fn(ctx, *fn_inputs)
       for (name, output) in ctx.last_step_outputs.items():
         # Convert all outputs to tensors, potentially from `DistributedValues`.
-        ctx.last_step_outputs[name] = self.unwrap(output)
+        ctx.last_step_outputs[name] = self._unwrap(output)
       flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
       with ops.control_dependencies([fn_result]):
         return [i + 1] + flat_last_step_outputs
@@ -571,19 +586,19 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
     return ctx
 
-  def _broadcast(self, tensor, destinations):
+  def _broadcast_to(self, tensor, destinations):
     # TODO(josh11b): In eager mode, use one thread per device, or async mode.
     return self._get_cross_tower_ops().broadcast(tensor, destinations or
                                                  self._devices)
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    return _call_for_each_replica(self, fn, args, kwargs)
+    return _call_for_each_replica(self._container_strategy(), fn, args, kwargs)
 
-  def configure(self,
-                session_config=None,
-                cluster_spec=None,
-                task_type=None,
-                task_id=None):
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
     del task_type, task_id
 
     if session_config:
@@ -613,7 +628,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
           cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps())
     return self._cross_tower_ops
 
-  def _reduce(self, reduce_op, value, destinations):
+  def _reduce_to(self, reduce_op, value, destinations):
     assert not isinstance(value, values.Mirrored)
     if not isinstance(value, values.DistributedValues):
       # This function handles reducing values that are not PerReplica or
@@ -626,13 +641,13 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
       value = value.get(self._devices[0])
       if isinstance(value, (int, float)):
         return value
-      return self.broadcast(value, destinations)
+      return self.broadcast_to(value, destinations)
     return self._get_cross_tower_ops().reduce(
         reduce_op, value, destinations=destinations)
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
     if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return [self.broadcast(v.get(self._devices[0]), d)
+      return [self.broadcast_to(v.get(self._devices[0]), d)
               for v, d in value_destination_pairs]
     return self._get_cross_tower_ops().batch_reduce(reduce_op,
                                                     value_destination_pairs)
@@ -680,7 +695,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     return values.value_container(val)
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     return len(self._devices)
 
   @property
@@ -693,11 +708,11 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     return list(self._devices)
 
   @property
-  def between_graph(self):
+  def experimental_between_graph(self):
     return False
 
   @property
-  def should_init(self):
+  def experimental_should_init(self):
     return True
 
   @property
@@ -723,7 +738,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
 
     def __init__(self, dist, coord, device, variable_creator_fn, fn, *args,
                  **kwargs):
-      super(MirroredStrategy._MirroredReplicaThread, self).__init__()  # pylint: disable=protected-access
+      super(MirroredExtended._MirroredReplicaThread, self).__init__()  # pylint: disable=protected-access
       self.coord = coord
       self.distribution = dist
       self.device = device
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 9c46ce5665..78d6876723 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -644,7 +644,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       self.evaluate([y for x in ret_ops for y in dist.unwrap(x)])
       expected_sum = 0.0
       expected_mean = 0.0
-      for i, d in enumerate(dist.worker_devices):
+      for i, d in enumerate(dist.extended.worker_devices):
         # Should see different values on different devices.
         v_sum_value = self.evaluate(ret_v_sum.get(d).read_value())
         v_mean_value = self.evaluate(ret_v_mean.get(d).read_value())
@@ -654,7 +654,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         expected = i * 6.0
         self.assertEqual(expected, v_mean_value)
         expected_mean += expected
-      expected_mean /= len(dist.worker_devices)
+      expected_mean /= len(dist.extended.worker_devices)
 
       # Without get(device), should return the value you get by
       # applying the reduction across all replicas (whether you use
@@ -829,7 +829,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         # Assert that the aggregated value of the replica local vars is the sum
         # of the individual values before running the update ops.
         self.assertEquals(1.0, self.evaluate(
-            ret_v_sum.get(dist._devices[0]).read_value()))
+            ret_v_sum.get(dist.extended.worker_devices[0]).read_value()))
         self.assertEquals(2.0, self.evaluate(ret_v_sum))
 
         # Apply updates.
@@ -837,7 +837,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         # Assert that the aggregated value of the replica local vars is the sum
         # of the individual values after running the update ops.
         self.assertEquals(5.0, self.evaluate(
-            ret_v_sum.get(dist._devices[0]).read_value()))
+            ret_v_sum.get(dist.extended.worker_devices[0]).read_value()))
         self.assertEquals(10.0, self.evaluate(ret_v_sum))
 
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 681aa31432..893f073f00 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -40,7 +40,14 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
   # implementations?
 
   def __init__(self, device):
-    super(OneDeviceStrategy, self).__init__()
+    super(OneDeviceStrategy, self).__init__(OneDeviceExtended(self, device))
+
+
+class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of OneDeviceStrategy."""
+
+  def __init__(self, container_strategy, device):
+    super(OneDeviceExtended, self).__init__(container_strategy)
     self._device = device
     self._default_device = device
 
@@ -59,12 +66,12 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     with ops.colocate_with(colocate_with):
       return next_creator(*args, **kwargs)
 
-  def make_dataset_iterator(self, dataset):
+  def _make_dataset_iterator(self, dataset):
     distributed_dataset = values.PerReplicaDataset(dataset, [self._device])
     # TODO(priyag): Return distribution strategy specific InputIterator
     return distributed_dataset.make_initializable_iterator()
 
-  def distribute_dataset(self, dataset_fn):
+  def _distribute_dataset(self, dataset_fn):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), [self._device])
 
@@ -76,13 +83,13 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
         self._call_dataset_fn(input_fn, distribute_lib.InputContext()),
         [self._device])
 
-  def _broadcast(self, tensor, destinations):
+  def _broadcast_to(self, tensor, destinations):
     del destinations
     return tensor
 
   # TODO(priyag): Deal with OutOfRange errors  once b/111349762 is fixed.
-  def _run_steps_on_dataset(self, fn, iterator, iterations,
-                            initial_loop_values=None):
+  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
+                                          initial_loop_values=None):
     if initial_loop_values is None:
       initial_loop_values = {}
     initial_loop_values = nest.flatten(initial_loop_values)
@@ -128,10 +135,11 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     return ctx
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    with ops.device(self._device), _OneDeviceReplicaContext(self):
+    strategy = self._container_strategy()
+    with ops.device(self._device), _OneDeviceReplicaContext(strategy):
       return fn(*args, **kwargs)
 
-  def _reduce(self, reduce_op, value, destinations):
+  def _reduce_to(self, reduce_op, value, destinations):
     del reduce_op, destinations
     return value
 
@@ -160,7 +168,7 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     return value
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     return 1
 
   @property
@@ -175,6 +183,18 @@ class OneDeviceStrategy(distribute_lib.DistributionStrategy):
     del var_list
     return [self._device]
 
+  @property
+  def experimental_should_init(self):
+    return True
+
+  @property
+  def should_checkpoint(self):
+    return True
+
+  @property
+  def should_save_summary(self):
+    return True
+
 
 class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
   """ReplicaContext for OneDeviceStrategy."""
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 11d6be9e86..58b6c87179 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -95,7 +95,15 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
       ValueError: if `cluster_spec` is given but `task_type` or `task_id` is
         not.
     """
-    super(ParameterServerStrategy, self).__init__()
+    super(ParameterServerStrategy, self).__init__(
+        ParameterServerExtended(self, num_gpus_per_worker))
+
+
+class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of ParameterServerStrategy."""
+
+  def __init__(self, container_strategy, num_gpus_per_worker):
+    super(ParameterServerExtended, self).__init__(container_strategy)
     self._num_gpus_per_worker = num_gpus_per_worker
     self._initialize_local(num_gpus_per_worker)
 
@@ -222,7 +230,7 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
         "ParameterServerStrategy with compute_devices = %r, "
         "variable_device = %r", self._compute_devices, self._variable_device)
 
-  def distribute_dataset(self, dataset_fn):
+  def _distribute_dataset(self, dataset_fn):
     """Distributes the dataset to each local GPU."""
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._compute_devices, True)
@@ -243,12 +251,12 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     input_context = distribute_lib.InputContext(
         num_input_pipelines=num_input_pipelines,
         input_pipeline_id=input_pipeline_id,
-        num_replicas_in_sync=self.num_replicas_in_sync)
+        num_replicas_in_sync=self._num_replicas_in_sync)
     return values.PerReplicaDataset(
         self._call_dataset_fn(input_fn, input_context), self._compute_devices,
         True)
 
-  def _broadcast(self, tensor, destinations):
+  def _broadcast_to(self, tensor, destinations):
     if not cross_tower_ops_lib.check_destinations(destinations):
       destinations = self._compute_devices
     return self._cross_tower_ops.broadcast(tensor, destinations)
@@ -259,7 +267,7 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   # TODO(yuefengz): not all ops in device_setter.STANDARD_PS_OPS will go through
   # this creator, such as "MutableHashTable".
   def _create_variable(self, next_creator, *args, **kwargs):
-    if self.num_replicas_in_sync > 1:
+    if self._num_replicas_in_sync > 1:
       aggregation = kwargs.pop("aggregation", vs.VariableAggregation.NONE)
       if aggregation not in (
           vs.VariableAggregation.NONE,
@@ -315,7 +323,8 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
 
   def _call_for_each_replica(self, fn, args, kwargs):
     # pylint: disable=protected-access
-    return mirrored_strategy._call_for_each_replica(self, fn, args, kwargs)
+    return mirrored_strategy._call_for_each_replica(
+        self._container_strategy(), fn, args, kwargs)
 
   def _verify_destinations_not_different_worker(self, destinations):
     if not self._cluster_spec:
@@ -329,20 +338,21 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
             "Cannot reduce to another worker: %r, current worker is %r" %
             (d, self._worker_device))
 
-  def _reduce(self, reduce_op, value, destinations):
+  def _reduce_to(self, reduce_op, value, destinations):
     self._verify_destinations_not_different_worker(destinations)
     if not isinstance(value, values.DistributedValues):
       # pylint: disable=protected-access
       return mirrored_strategy._reduce_non_distributed_value(
           self, reduce_op, value, destinations)
     if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return self.broadcast(value.get(self._compute_devices[0]), destinations)
+      return self.broadcast_to(
+          value.get(self._compute_devices[0]), destinations)
     return self._cross_tower_ops.reduce(
         reduce_op, value, destinations=destinations)
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
     if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return [self.broadcast(v.get(self._compute_devices[0]), d)
+      return [self.broadcast_to(v.get(self._compute_devices[0]), d)
               for v, d in value_destination_pairs]
     for _, destinations in value_destination_pairs:
       self._verify_destinations_not_different_worker(destinations)
@@ -416,11 +426,11 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     # variables.
     return array_ops.identity(var)
 
-  def configure(self,
-                session_config=None,
-                cluster_spec=None,
-                task_type=None,
-                task_id=None):
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
     """Configures the strategy class.
 
     The strategy object will be re-initialized if `cluster_spec` is given but
@@ -468,7 +478,7 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
         ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     return len(self._compute_devices)
 
   @property
@@ -484,11 +494,12 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     return min(var_list, key=lambda x: x.name)
 
   @property
-  def between_graph(self):
+  def experimental_between_graph(self):
+    # TODO(yuefengz): Should this return False in the local case?
     return True
 
   @property
-  def should_init(self):
+  def experimental_should_init(self):
     return self._is_chief
 
   @property
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 7561d7b3d5..091a358fea 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -361,9 +361,9 @@ class ParameterServerStrategyTestBase(
   def _test_simple_increment(self, task_type, task_id, num_gpus):
     d, master_target, sess_config = self._get_test_objects(
         task_type, task_id, num_gpus)
-    if hasattr(d, '_cluster_spec') and d._cluster_spec:
-      num_workers = len(d._cluster_spec.as_dict().get(WORKER))
-      if 'chief' in d._cluster_spec.as_dict():
+    if d.extended._cluster_spec:
+      num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER))
+      if 'chief' in d.extended._cluster_spec.as_dict():
         num_workers += 1
     else:
       num_workers = 1
@@ -396,7 +396,7 @@ class ParameterServerStrategyTestBase(
       x, y, z, train_op = d.call_for_each_replica(model_fn)
       train_op = d.group(train_op)
 
-      if context.num_gpus() < d._num_gpus_per_worker:
+      if context.num_gpus() < d.extended._num_gpus_per_worker:
         return True
 
       if task_id == 0:
@@ -433,9 +433,9 @@ class ParameterServerStrategyTestBase(
         task_type, task_id, num_gpus)
     if task_type:
       # Multi-worker
-      assert hasattr(d, '_cluster_spec') and d._cluster_spec
-      num_workers = len(d._cluster_spec.as_dict().get(WORKER))
-      if CHIEF in d._cluster_spec.as_dict():
+      assert hasattr(d.extended, '_cluster_spec') and d.extended._cluster_spec
+      num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER))
+      if CHIEF in d.extended._cluster_spec.as_dict():
         num_workers += 1
     else:
       # local
@@ -488,11 +488,12 @@ class ParameterServerStrategyTestBase(
 
       before_out, after_out = step()
 
-      if context.num_gpus() < d._num_gpus_per_worker:
+      if context.num_gpus() < d.extended._num_gpus_per_worker:
         return True
 
       if (not task_type or
-          multi_worker_util.is_chief(d._cluster_spec, task_type, task_id)):
+          multi_worker_util.is_chief(
+              d.extended._cluster_spec, task_type, task_id)):
         variables.global_variables_initializer().run()
 
       # Workers waiting for chief worker's initializing variables.
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index b917334790..6e42bf9d97 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -134,8 +134,21 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
       num_cores: Number of cores to use on the TPU. If None specified, then
           auto-detect the cores and topology of the TPU system.
     """
-    super(TPUStrategy, self).__init__()
+    super(TPUStrategy, self).__init__(TPUExtended(
+        self, tpu_cluster_resolver, steps_per_run, num_cores))
 
+  @property
+  def steps_per_run(self):
+    """DEPRECATED: use .extended.steps_per_run instead."""
+    return self._extended.steps_per_run
+
+
+class TPUExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of TPUStrategy."""
+
+  def __init__(self, container_strategy, tpu_cluster_resolver, steps_per_run,
+               num_cores=None):
+    super(TPUExtended, self).__init__(container_strategy)
     self._tpu_cluster_resolver = tpu_cluster_resolver
     self._tpu_metadata = get_tpu_system_metadata(self._tpu_cluster_resolver)
     # TODO(sourabhbajaj): Change this from num_cores to metadata_override
@@ -149,7 +162,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     self._host_device = self.get_host_cpu_device(0)
     self._tpu_devices = sorted(device_map.keys())
     # Only create variables for the number of replicas we're running.
-    self._tpu_devices = self._tpu_devices[:self.num_replicas_in_sync]
+    self._tpu_devices = self._tpu_devices[:self._num_replicas_in_sync]
 
     # TODO(sourabhbajaj): Remove this once performance of running one step
     # at a time is comparable to multiple steps.
@@ -218,7 +231,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
 
     return enqueue_op_per_host
 
-  def make_dataset_iterator(self, dataset):
+  def _make_dataset_iterator(self, dataset):
     """Make iterators for each of the TPU hosts.
 
     We first unbatch the users input dataset and then rebatch it with the
@@ -255,11 +268,11 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
           "The batch operations can be followed by a prefetch.")
 
     global_batch_size = _get_dataset_batch_size(dataset)
-    if global_batch_size % self.num_replicas_in_sync:
+    if global_batch_size % self._num_replicas_in_sync:
       raise ValueError(
           "Batch size %s cannot be sharded evenly across replicas %s" % (
               global_batch_size, self.num_replicas_in_sync))
-    per_replica_batch_size = global_batch_size // self.num_replicas_in_sync
+    per_replica_batch_size = global_batch_size // self._num_replicas_in_sync
     dataset = dataset.apply(batching.unbatch())
     dataset = dataset.batch(per_replica_batch_size, drop_remainder=True)
 
@@ -273,7 +286,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     # TODO(priyag): Return distribution strategy specific InputIterator
     return distributed_dataset.make_initializable_iterator()
 
-  def distribute_dataset(self, dataset_fn):
+  def _distribute_dataset(self, dataset_fn):
     worker_devices = [
         (self.get_host(hid), [self.get_host_cpu_device(hid)])
         for hid in range(self.num_hosts)
@@ -284,9 +297,8 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   # TODO(sourabhbajaj): Remove the initial_loop_values parameter when we have
   # a mechanism to infer the outputs of `fn`. Pending b/110550782.
-  def _run_steps_on_dataset(self, fn, multi_worker_iterator, iterations,
-                            initial_loop_values=None):
-
+  def _experimental_run_steps_on_iterator(
+      self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
     output_shapes = multi_worker_iterator.output_shapes
     shapes = nest.flatten(output_shapes)
     if any([not s.is_fully_defined() for s in shapes]):
@@ -338,7 +350,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     self._outer_control_flow_context = (
         ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
 
-    replicate_inputs = [[]] * self.num_replicas_in_sync
+    replicate_inputs = [[]] * self._num_replicas_in_sync
     replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs)
     del self._outer_control_flow_context
     ctx.run_op = control_flow_ops.group(replicate_outputs, enqueue_ops)
@@ -379,10 +391,10 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def _call_for_each_replica(self, fn, args, kwargs):
     # TODO(jhseu): Consider making it so call_for_each_replica implies that
     # we're in a tpu.rewrite(), and update TPUMirroredVariable accordingly.
-    with _TPUReplicaContext(self):
+    with _TPUReplicaContext(self._container_strategy()):
       return fn(*args, **kwargs)
 
-  def initialize(self):
+  def _initialize(self):
     if context.executing_eagerly():
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
       raise NotImplementedError("Eager mode not supported in TPUStrategy.")
@@ -397,7 +409,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
                               tpu.initialize_system())
       return graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
 
-  def finalize(self):
+  def _finalize(self):
     if context.executing_eagerly():
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
       raise NotImplementedError("Eager mode not supported in TPUStrategy.")
@@ -442,11 +454,11 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     return _create_tpu_mirrored_variable(devices, _real_mirrored_creator, *args,
                                          **kwargs)
 
-  def _reduce(self, reduce_op, value, destinations):
+  def _reduce_to(self, reduce_op, value, destinations):
     if values._enclosing_tpu_context() is not None:  # pylint: disable=protected-access
       if reduce_op == reduce_util.ReduceOp.MEAN:
         # TODO(jhseu):  Revisit once we support model-parallelism.
-        value *= (1. / self.num_replicas_in_sync)
+        value *= (1. / self._num_replicas_in_sync)
       elif reduce_op != reduce_util.ReduceOp.SUM:
         raise NotImplementedError(
             "Currently only support sum & mean in TPUStrategy.")
@@ -507,7 +519,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def value_container(self, value):
     return value
 
-  def _broadcast(self, tensor, destinations):
+  def _broadcast_to(self, tensor, destinations):
     del destinations
     return tensor
 
@@ -520,15 +532,15 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
     return self._tpu_metadata.num_of_cores_per_host
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     return self._num_cores_override or self._tpu_metadata.num_cores
 
   @property
-  def between_graph(self):
+  def experimental_between_graph(self):
     return False
 
   @property
-  def should_init(self):
+  def experimental_should_init(self):
     return True
 
   @property
@@ -569,11 +581,11 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def get_host_cpu_device(self, host_id):
     return self.get_host(host_id) + "/device:CPU:0"
 
-  def configure(self,
-                session_config=None,
-                cluster_spec=None,
-                task_type=None,
-                task_id=None):
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
     del cluster_spec, task_type, task_id
     if session_config:
       session_config.isolate_session_state = True
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 2f4ef9de89..86e647c4c6 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -1062,18 +1062,18 @@ def select_device_mirrored(device, structured):
   return nest.map_structure(_get_mirrored, structured)
 
 
-def update_regroup(strategy, updates, group):
+def update_regroup(extended, updates, group):
   """Regroup for an update, with dependencies to ensure all updates execute."""
   regrouped = regroup(updates, Mirrored)
   if not group:
-    return nest.map_structure(strategy.unwrap, regrouped)
+    return nest.map_structure(extended._unwrap, regrouped)  # pylint: disable=protected-access
   grouped_flat = []
   for u in nest.flatten(regrouped):
     if isinstance(u, DistributedValues):
-      g = strategy.group(u)
+      g = extended._group(u)  # pylint: disable=protected-access
       if u.is_tensor_like:
         # Make sure we run all updates. Without this, something like
-        # session.run(strategy.update(...)) may only update one replica.
+        # session.run(extended.update(...)) may only update one replica.
         index = {}
         for d in u.devices:
           with ops.device(d), ops.control_dependencies([g]):
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 3499bbca2f..e55b2a0e92 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3565,6 +3565,7 @@ py_library(
         "//tensorflow/python/data",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/ops/losses",
+        "//tensorflow/tools/docs:doc_controls",
     ],
 )
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 8dd9ef470b..b41febbbc7 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -338,7 +338,7 @@ def configure_and_create_session(distribution_strategy):
     # Coordinator is integrated with keras and we can create a session from
     # there.
     distribution_strategy.configure(session_config)
-    master = distribution_strategy._tpu_cluster_resolver.master()  # pylint: disable=protected-access
+    master = distribution_strategy.extended._tpu_cluster_resolver.master()  # pylint: disable=protected-access
     session = session_module.Session(config=session_config, target=master)
   else:
     worker_context = dc_context.get_current_worker_context()
@@ -508,11 +508,11 @@ def get_cpu_device(distribution_strategy):
     multiple hosts in the case of Cloud TPU pods.
   """
   if distribution_strategy.__class__.__name__ == 'TPUStrategy':
-    if distribution_strategy.num_hosts > 1:
+    if distribution_strategy.extended.num_hosts > 1:
       raise NotImplementedError('TPUDistributionStrategy does not '
                                 'support numpy inputs when running on Cloud'
                                 'TPU pods.')
-    return distribution_strategy.get_host_cpu_device(0)
+    return distribution_strategy.extended.get_host_cpu_device(0)
   else:
     # For all strategies except TPUDistributionStrategy
     # TODO(anjalisridhar): We may need to modify this when we add support for
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 3c64233215..02d0befe8e 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -317,7 +317,7 @@ def _experimental_fit_loop(
     raise ValueError('`steps_per_epoch` should be specified when calling '
                      '`fit` on the model.')
   steps_per_run = K.variable(
-      value=min(steps_per_epoch, current_strategy.steps_per_run),
+      value=min(steps_per_epoch, current_strategy.extended.steps_per_run),
       dtype='int32',
       name='steps_per_run')
 
@@ -348,10 +348,11 @@ def _experimental_fit_loop(
       verbose=verbose)
 
   # Calculate the steps each time on the device.
-  steps_to_run = [current_strategy.steps_per_run] * (
-      steps_per_epoch // current_strategy.steps_per_run)
-  if steps_per_epoch % current_strategy.steps_per_run:
-    steps_to_run.append(steps_per_epoch % current_strategy.steps_per_run)
+  steps_to_run = [current_strategy.extended.steps_per_run] * (
+      steps_per_epoch // current_strategy.extended.steps_per_run)
+  if steps_per_epoch % current_strategy.extended.steps_per_run:
+    steps_to_run.append(
+        steps_per_epoch % current_strategy.extended.steps_per_run)
 
   callbacks.on_train_begin()
   for epoch in range(initial_epoch, epochs):
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index e86a3b8536..328a5dcbd8 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -302,7 +302,7 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args):
   """Aggregate metric value across replicas."""
   def fn(distribution, *a):
     """Call `metric_value_fn` in the correct control flow context."""
-    if hasattr(distribution, '_outer_control_flow_context'):
+    if hasattr(distribution.extended, '_outer_control_flow_context'):
       # If there was an outer context captured before this method was called,
       # then we enter that context to create the metric value op. If the
       # caputred context is `None`, ops.control_dependencies(None) gives the
@@ -315,13 +315,13 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args):
       # once the update ops have been evaluted.
 
       # pylint: disable=protected-access
-      if distribution._outer_control_flow_context is None:
+      if distribution.extended._outer_control_flow_context is None:
         with ops.control_dependencies(None):
           metric_value = metric_value_fn(distribution, *a)
       else:
-        distribution._outer_control_flow_context.Enter()
+        distribution.extended._outer_control_flow_context.Enter()
         metric_value = metric_value_fn(distribution, *a)
-        distribution._outer_control_flow_context.Exit()
+        distribution.extended._outer_control_flow_context.Exit()
         # pylint: enable=protected-access
     else:
       metric_value = metric_value_fn(distribution, *a)
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 1cd3bc2e04..130eb6ad5b 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import copy
 import threading
+import weakref
 import enum
 
 from tensorflow.python.data.ops import dataset_ops
@@ -35,6 +37,7 @@ from tensorflow.python.platform import tf_logging
 from tensorflow.python.training import device_util
 from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util import nest
+from tensorflow.tools.docs import doc_controls
 
 
 # ------------------------------------------------------------------------------
@@ -85,25 +88,33 @@ def get_loss_reduction():
 # Internal API for validating the current thread mode
 
 
-def _require_cross_replica_context(distribution_strategy):
-  """Verify in cross-replica context for `distribution_strategy`."""
+def _require_cross_replica_context_extended(extended):
+  """Verify in cross-replica context."""
   context = _get_per_thread_mode()
-  if context.cross_replica_context is distribution_strategy: return
+  cross_replica = context.cross_replica_context
+  if cross_replica is not None and cross_replica.extended is extended:
+    return
+  distribution_strategy = extended._container_strategy()  # pylint: disable=protected-access
   # We have an error to report, figure out the right message.
   if context.distribution_strategy is not distribution_strategy:
-    if not distribution_strategy_context.has_distribution_strategy():
-      raise RuntimeError(
-          'Need to be inside "with distribution_strategy.scope()" for %s' %
-          (distribution_strategy,))
-    else:
-      raise RuntimeError(
-          "Mixing different DistributionStrategy objects: %s is not %s" %
-          (context.distribution_strategy, distribution_strategy))
-  assert context.cross_replica_context is None
+    _wrong_distribution_strategy_scope(distribution_strategy, context)
+  assert cross_replica is None
   raise RuntimeError("Method requires being in cross-replica context, use "
                      "get_replica_context().merge_call()")
 
 
+def _wrong_distribution_strategy_scope(distribution_strategy, context):
+  # Figure out the right error message.
+  if not distribution_strategy_context.has_distribution_strategy():
+    raise RuntimeError(
+        'Need to be inside "with distribution_strategy.scope()" for %s' %
+        (distribution_strategy,))
+  else:
+    raise RuntimeError(
+        "Mixing different DistributionStrategy objects: %s is not %s" %
+        (context.distribution_strategy, distribution_strategy))
+
+
 def require_replica_context(replica_ctx):
   """Verify in `replica_ctx` replica context."""
   context = _get_per_thread_mode()
@@ -113,25 +124,26 @@ def require_replica_context(replica_ctx):
     raise RuntimeError("Need to be inside `call_for_each_replica()`")
   if context.distribution_strategy is replica_ctx.distribution_strategy:
     # Two different ReplicaContexts with the same DistributionStrategy.
-    raise RuntimeError("Mismatching replica context.")
+    raise RuntimeError("Mismatching ReplicaContext.")
   raise RuntimeError(
       "Mismatching DistributionStrategy objects: %s is not %s." %
       (context.distribution_strategy, replica_ctx.distribution_strategy))
 
 
-def _require_distribution_strategy_scope(distribution_strategy):
+def _require_distribution_strategy_scope_strategy(distribution_strategy):
   """Verify in a `distribution_strategy.scope()` in this thread."""
   context = _get_per_thread_mode()
   if context.distribution_strategy is distribution_strategy: return
-  # We have an error to report, figure out the right message.
-  if not distribution_strategy_context.has_distribution_strategy():
-    raise RuntimeError(
-        'Need to be inside "with distribution_strategy.scope()" for %s' %
-        (distribution_strategy,))
-  else:
-    raise RuntimeError(
-        "Mixing different DistributionStrategy objects: %s is not %s" %
-        (context.distribution_strategy, distribution_strategy))
+  _wrong_distribution_strategy_scope(distribution_strategy, context)
+
+
+def _require_distribution_strategy_scope_extended(extended):
+  """Verify in a `distribution_strategy.scope()` in this thread."""
+  context = _get_per_thread_mode()
+  if context.distribution_strategy.extended is extended: return
+  # Report error.
+  distribution_strategy = extended._container_strategy()  # pylint: disable=protected-access
+  _wrong_distribution_strategy_scope(distribution_strategy, context)
 
 
 # ------------------------------------------------------------------------------
@@ -271,6 +283,362 @@ class DistributionStrategy(object):
   See [tensorflow/contrib/distribute/README.md](
   https://www.tensorflow.org/code/tensorflow/contrib/distribute/README.md)
   for overview and examples.
+  """
+
+  # TODO(josh11b): Raise an exception if variable partitioning requested before
+  #   we add support.
+  # TODO(josh11b): Also `parameter_device_index` property?
+  # TODO(josh11b): `map()`
+  # TODO(josh11b): ClusterSpec/ClusterResolver
+  # TODO(josh11b): Partitioned computations, state; sharding
+  # TODO(josh11b): Model parallelism: "replicas" with multiple devices; shuffling
+  # TODO(josh11b): List of replicas with their worker and parameter devices
+  #   (where the parameter devices may overlap in the ps case).
+
+  def __init__(self, extended):
+    self._extended = extended
+
+  @property
+  def extended(self):
+    """`tf.contrib.distribute.DistributionStrategyExtended` with new methods."""
+    return self._extended
+
+  def scope(self):
+    """Returns a context manager selecting this DistributionStrategy as current.
+
+    Inside a `with distribution_strategy.scope():` code block, this thread
+    will use a variable creator set by `distribution_strategy`, and will
+    enter its "cross-replica context".
+
+    Returns:
+      A context manager.
+    """
+    return self._extended._scope(self)  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def read_var(self, v):
+    """DEPRECATED: use extended.read_var() instead."""
+    return self._extended.read_var(v)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def colocate_vars_with(self, colocate_with_variable):
+    """DEPRECATED: use extended.colocate_vars_with() instead."""
+    return self._extended.colocate_vars_with(colocate_with_variable)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED
+  def distribute_dataset(self, dataset_fn):
+    """Return a `dataset` split across all replicas.  DEPRECATED.
+
+    DEPRECATED: Please use `make_dataset_iterator()` or
+    `make_input_fn_iterator()` instead.
+
+    Suitable for providing input to for `extended.call_for_each_replica()` by
+    creating an iterator:
+
+    ```
+    def dataset_fn():
+      return tf.data.Dataset.from_tensors([[1.]]).repeat()
+
+    with distribution_strategy.scope():
+      distributed_dataset = distribution_strategy.distribute_dataset(dataset_fn)
+      iterator = distributed_dataset.make_initializable_iterator()
+      replica_results = distribution_strategy.extended.call_for_each_replica(
+          replica_fn, args=(iterator.get_next(),))
+    ```
+
+    Args:
+      dataset_fn: A function that returns a `tf.data.Dataset`.
+
+    Returns:
+      A `PerReplicaDataset` that will produce data for each replica.
+    """
+    return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
+
+  def make_dataset_iterator(self, dataset):
+    """Makes an iterator for input provided via input_dataset.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    global batch size. With this assumption, we will make a best effort to
+    divide each batch across all the replicas (one or more workers).
+    If this effort fails, an error will be thrown, and the user should instead
+    use `make_input_fn_iterator` which provides more control to the user, and
+    does not try to divide a batch across replicas.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `InputIterator` which returns inputs for each step of the computation.
+      User should call `initialize` on the returned iterator.
+    """
+    return self._extended._make_dataset_iterator(dataset)  # pylint: disable=protected-access
+
+  def make_input_fn_iterator(self,
+                             input_fn,
+                             replication_mode=InputReplicationMode.PER_WORKER):
+    """Returns an iterator split across replicas created from an input function.
+
+    The `input_fn` should take an `InputContext` object where information about
+    input sharding can be accessed:
+
+    ```
+    def input_fn(input_context):
+      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
+      return d.shard(input_context.num_input_pipelines,
+                     input_context.input_pipeline_id)
+    with distribution_strategy.scope():
+      iterator = distribution_strategy.make_input_fn_iterator(
+          input_fn)
+      replica_results = distribution_strategy.call_for_each_replica(
+          replica_fn, iterator.get_next())
+    ```
+
+    Args:
+      input_fn: A function that returns a `tf.data.Dataset`. This function is
+        expected to take an `InputContext` object.
+      replication_mode: an enum value of `InputReplicationMode`. Only
+        `PER_WORKER` is supported currently.
+
+    Returns:
+      An iterator object that can be initialized and fetched next element.
+    """
+    if replication_mode != InputReplicationMode.PER_WORKER:
+      raise ValueError(
+          "Input replication mode not supported: %r" % replication_mode)
+    return self.extended._make_input_fn_iterator(  # pylint: disable=protected-access
+        input_fn, replication_mode=replication_mode)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def broadcast(self, tensor, destinations=None):
+    """DEPRECATED: use extended.broadcast_to() instead."""
+    return self._extended.broadcast_to(tensor, destinations)
+
+  @doc_controls.do_not_generate_docs  # Use experimental_initialize() instead.
+  def initialize(self):
+    """DEPRECATED: Use `experimental_initialize()` instead."""
+    return self._extended._initialize()  # pylint: disable=protected-access
+
+  def experimental_initialize(self):
+    """Any initialization to be done before running any computations.
+
+    In eager mode, it executes any initialization as a side effect.
+    In graph mode, it creates the initialization ops and returns them.
+
+    For example, TPU initialize_system ops.
+
+    Returns:
+      A list of ops to execute.
+    """
+    return self._extended._initialize()  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # Use experimental_finalize() instead.
+  def finalize(self):
+    """DEPRECATED: Use `experimental_finalize()` instead."""
+    return self._extended._finalize()  # pylint: disable=protected-access
+
+  def experimental_finalize(self):
+    """Any final actions to be done at the end of all computations.
+
+    In eager mode, it executes any finalize actions as a side effect.
+    In graph mode, it creates the finalize ops and returns them.
+
+    For example, TPU shutdown ops.
+
+    Returns:
+      A list of ops to execute.
+    """
+    return self._extended._finalize()  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def run_steps_on_dataset(self, fn, iterator, iterations=1,
+                           initial_loop_values=None):
+    """DEPRECATED: use extended.experimental_run_steps_on_iterator() instead."""
+    return self._extended.experimental_run_steps_on_iterator(
+        fn, iterator, iterations, initial_loop_values)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def call_for_each_replica(self, fn, *args, **kwargs):
+    """DEPRECATED: use extended.call_for_each_replica() instead."""
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to call_for_each_replica")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to call_for_each_replica")
+      kwargs = k
+    kwargs.pop("run_concurrently", None)  # Ignore old option.
+    return self._extended.call_for_each_replica(fn, args, kwargs)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def reduce(self, aggregation, value, destinations):
+    """DEPRECATED: use extended.reduce_to() instead."""
+    return self._extended.reduce_to(aggregation, value, destinations)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def batch_reduce(self, aggregation, value_destination_pairs):
+    """DEPRECATED: use extended.batch_reduce_to() instead."""
+    return self._extended.batch_reduce_to(aggregation, value_destination_pairs)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def update(self, var, fn, *args, **kwargs):
+    """DEPRECATED: use extended.update() instead."""
+    group = kwargs.pop("group", True)
+    # We temporarily support "grouped" in addition to "group" for backward-
+    # compatibility.
+    group = kwargs.pop("grouped", True) and group
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to update")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to update")
+      kwargs = k
+    return self._extended.update(var, fn, args, kwargs, group)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    """DEPRECATED: use extended.update_non_slot() instead."""
+    group = kwargs.pop("group", True)
+    # We temporarily support "grouped" in addition to "group" for backward-
+    # compatibility.
+    group = kwargs.pop("grouped", True) and group
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to update_non_slot")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to update_non_slot")
+      kwargs = k
+    return self._extended.update_non_slot(
+        colocate_with, fn, args, kwargs, group)
+
+  def unwrap(self, value):
+    """Returns the list of all per-replica values contained in `value`.
+
+    Args:
+      value: A value returned by `extended.call_for_each_replica()` or a
+        variable created in `scope()`.
+
+    Returns:
+      A list of values contained in `value`. If `value` represents a single
+      value, this returns `[value].`
+    """
+    return self._extended._unwrap(value)  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def value_container(self, value):
+    """DEPRECATED: use extended.value_container() instead."""
+    return self._extended.value_container(value)
+
+  def group(self, value, name=None):
+    """Shortcut for `tf.group(self.unwrap(value))`."""
+    return self._extended._group(value, name)  # pylint: disable=protected-access
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def require_static_shapes(self):
+    """DEPRECATED: use extended.require_static_shapes instead."""
+    return self._extended.experimental_require_static_shapes
+
+  @property
+  def num_replicas_in_sync(self):
+    """Returns number of replicas over which gradients are aggregated."""
+    return self._extended._num_replicas_in_sync  # pylint: disable=protected-access
+
+  @property
+  def worker_devices(self):
+    """DEPRECATED: use extended.worker_devices instead."""
+    return self._extended.worker_devices
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def parameter_devices(self):
+    """DEPRECATED: use extended.parameter_devices instead."""
+    return self._extended.parameter_devices
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def non_slot_devices(self, var_list):
+    """DEPRECATED: use extended.non_slot_devices instead."""
+    return self._extended.non_slot_devices(var_list)
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def between_graph(self):
+    """DEPRECATED: use extended.experimental_between_graph instead."""
+    return self._extended.experimental_between_graph
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, being replaced by a new API.
+  def configure(self,
+                session_config=None,
+                cluster_spec=None,
+                task_type=None,
+                task_id=None):
+    """Configures the strategy class."""
+    return self._extended._configure(  # pylint: disable=protected-access
+        session_config, cluster_spec, task_type, task_id)
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def should_init(self):
+    """DEPRECATED: use extended.should_init instead."""
+    return self._extended.experimental_should_init
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def should_checkpoint(self):
+    """DEPRECATED: use extended.should_checkpoint instead."""
+    return self._extended.should_checkpoint
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def should_save_summary(self):
+    """DEPRECATED: use extended.should_save_summary instead."""
+    return self._extended.should_save_summary
+
+  def __deepcopy__(self, memo):
+    # First do a regular deepcopy of `self`.
+    cls = self.__class__
+    result = cls.__new__(cls)
+    memo[id(self)] = result
+    for k, v in self.__dict__.items():
+      setattr(result, k, copy.deepcopy(v, memo))
+    # One little fix-up: we want `result._extended` to reference `result`
+    # instead of `self`.
+    result._extended._container_strategy_weakref = weakref.ref(result)  # pylint: disable=protected-access
+    return result
+
+  def __copy__(self):
+    raise RuntimeError("Must only deepcopy DistributionStrategy.")
+
+
+class DistributionStrategyExtended(object):
+  """Additional APIs for algorithms that need to be distribution-aware.
 
   The intent is that you can write an algorithm in a stylized way and
   it will be usable with a variety of different `DistributionStrategy`
@@ -339,7 +707,7 @@ class DistributionStrategy(object):
     with my_distribution.scope():
       iterator = my_distribution.distribute_dataset(
           dataset).make_one_shot_iterator()
-      replica_train_ops = my_distribution.call_for_each_replica(
+      replica_train_ops = my_distribution.extended.call_for_each_replica(
           replica_fn, args=(iterator.get_next(),))
       train_op = tf.group(my_distribution.unwrap(replica_train_ops))
     ```
@@ -371,28 +739,26 @@ class DistributionStrategy(object):
     wrap them in a "PerReplica" or "Mirrored" object that contains a map
     from device to values. "PerReplica" is used when the value may be
     different across replicas, and "Mirrored" when the value are the same.
-  * Unwrapping and merging: Consider calling a function `fn` on
-    multiple replicas, like `call_for_each_replica(fn, args=[w])` with an
-    argument `w` that is a wrapped value. This means `w` will have a
-    map taking replica device `d0` to `w0`, replica device `d1` to `w1`,
-    etc. `call_for_each_replica()` unwraps `w` before calling `fn`, so
-    it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc.  It then merges
-    the return values from `fn()`, which can possibly result in
-    wrapped values. For example, let's say `fn()` returns a tuple with
-    three components: `(x, a, v0)` from replica 0, `(x, b, v1)` on replica 1,
-    etc. If the first component is the same object `x` from every
-    replica, then the first component of the merged result will also be
-    `x`. If the second component is different (`a`, `b`, ...)  from
-    each replica, then the merged value will have a wrapped map from
-    replica device to the different values. If the third component is
-    the members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to
-    `v1`, etc.), then the merged result will be that mirrored variable
-    (`v`).
+  * Unwrapping and merging: Consider calling a function `fn` on multiple
+    replicas, like `extended.call_for_each_replica(fn, args=[w])` with an
+    argument `w` that is a wrapped value. This means `w` will have a map taking
+    replica device `d0` to `w0`, replica device `d1` to `w1`,
+    etc. `extended.call_for_each_replica()` unwraps `w` before calling `fn`, so
+    it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc.  It then merges the return
+    values from `fn()`, which can possibly result in wrapped values. For
+    example, let's say `fn()` returns a tuple with three components: `(x, a,
+    v0)` from replica 0, `(x, b, v1)` on replica 1, etc. If the first component
+    is the same object `x` from every replica, then the first component of the
+    merged result will also be `x`. If the second component is different (`a`,
+    `b`, ...)  from each replica, then the merged value will have a wrapped map
+    from replica device to the different values. If the third component is the
+    members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to `v1`, etc.),
+    then the merged result will be that mirrored variable (`v`).
   * Replica context vs. Cross-replica context: _replica context_ is when we
     are in some function that is being called once for each replica.
     Otherwise we are in cross-replica context, which is useful for
     calling `DistributionStrategy` methods which operate across the
-    replicas (like `reduce()`). By default you start in a replica context
+    replicas (like `reduce_to()`). By default you start in a replica context
     (the default "single replica context") and then some methods can
     switch you back and forth, as described below.
   * Worker devices vs. parameter devices: Most replica computations will
@@ -428,38 +794,40 @@ class DistributionStrategy(object):
 
   * `with d.scope()`: default single-replica context -> cross-replica context
     for `d`
-  * `with d.colocate_vars_with(v)`: in replica/cross-replica context, variables
-    will be created with locality V(`v`). That is, if we write
-    `with d.colocate_vars_with(v1): v2 = tf.get_variable(...)`, then
-    `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal
+  * `with d.extended.colocate_vars_with(v)`: in replica/cross-replica context,
+    variables will be created with locality V(`v`). That is, if we write
+    `with d.extended.colocate_vars_with(v1): v2 = tf.get_variable(...)`,
+    then `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal
     V(`v1`).
-  * `with d.colocate_vars_with(d.non_slot_devices(...))`: in
+  * `with d.extended.colocate_vars_with(d.extended.non_slot_devices(...))`: in
     replica/cross-replica context, variables will be created with locality N
   * `v = tf.get_variable(...)`: in replica/cross-replica context, creates
     a variable (which by definition will have locality V(`v`), though
     will match another locality if inside a `colocate_vars_with`
     scope).
-  * `d.distribute_dataset(dataset).make_one_shot_iterator()`: in cross-replica
+  * `d.make_dataset_iterator(dataset)` (or the deprecated
+    `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
     context, produces an iterator with locality T
-  * `d.broadcast(t)`: in cross-replica context, produces a value with locality M
-  * `d.broadcast(t, v)`: in cross-replica context, produces a value with
-    locality V(`v`)
-  * `d.call_for_each_replica(fn, ...)`: in cross-replica context, runs
+  * `d.extended.broadcast_to(t)`: in cross-replica context, produces a value
+    with locality M
+  * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
+    with locality V(`v`)
+  * `d.extended.call_for_each_replica(fn, ...)`: in cross-replica context, runs
     `fn()` in a replica context (and so may call `get_replica_context()` and
     use its API, including `merge_call()` to get back to cross-replica
     context), once for each replica. May use values with locality T or
     M, and any variable.
-  * `d.reduce(m, t, t)`: in cross-replica context, accepts t with locality T
-    and produces a value with locality M.
-  * `d.reduce(m, t, v)`: in cross-replica context, accepts t with
+  * `d.extended.reduce_to(m, t, t)`: in cross-replica context, accepts t with
+    locality T and produces a value with locality M.
+  * `d.extended.reduce_to(m, t, v)`: in cross-replica context, accepts t with
     locality T and produces a value with locality V(`v`).
-  * `d.batch_reduce(m, [(t, v)]): see `d.reduce()`
-  * `d.update(v, fn, ...)`: in cross-replica context, runs `fn()` once
+  * `d.extended.batch_reduce_to(m, [(t, v)]): see `d.extended.reduce_to()`
+  * `d.extended.update(v, fn, ...)`: in cross-replica context, runs `fn()` once
     for each device `v` is copied to, all inputs should have locality
     V(`v`), output will have locality V(`v`) as well.
-  * `d.update_non_slot(d.non_slot_devices(), fn)`: in cross-replica
-    context, like `d.update()` except with locality N.
-  * `d.read_var(v)`: Gets the (read-only) value of the variable `v` (on
+  * `d.extended.update_non_slot(d.extended.non_slot_devices(), fn)`: in
+    cross-replica context, like `d.extended.update()` except with locality N.
+  * `d.extended.read_var(v)`: Gets the (read-only) value of the variable `v` (on
     the device determined by the current device scope), aggregating
     across replicas for replica-local variables. Frequently, this will be
     done automatically when using `v` in an expression or fetching it in
@@ -469,65 +837,68 @@ class DistributionStrategy(object):
 
   The standard pattern for updating variables is to:
 
-  1. Wrap your input dataset in `d.distribute_dataset()` and create an iterator.
-  2. Define each replica `d.call_for_each_replica()` up to the point of
+  1. Create an input iterator with `d.make_dataset_iterator()`.
+  2. Define each replica `d.extended.call_for_each_replica()` up to the point of
      getting a list of gradient, variable pairs.
-  3. Call `d.reduce(VariableAggregation.SUM, t, v)` or `d.batch_reduce()` to sum
-     the gradients (with locality T) into values with locality V(`v`).
-  4. Call `d.update(v)` for each variable to update its value.
+  3. Call `d.extended.reduce_to(VariableAggregation.SUM, t, v)` or
+     `d.extended.batch_reduce_to()` to sum the gradients (with locality T)
+     into values with locality V(`v`).
+  4. Call `d.extended.update(v)` for each variable to update its value.
 
   Steps 3 and 4 are done automatically by class `Optimizer` if you call
   its `apply_gradients` method in a replica context. Otherwise you can
   manually call its `_distributed_apply` method in a cross-replica context.
 
-  Another thing you might want to do in the middle of your replica function
-  is an all-reduce of some intermediate value, using `d.reduce()` or
-  `d.batch_reduce()`. You simply provide the same tensor as the input and
-  destination.
+  Another thing you might want to do in the middle of your replica function is
+  an all-reduce of some intermediate value, using `d.extended.reduce_to()` or
+  `d.extended.batch_reduce_to()`. You simply provide the same tensor as the
+  input and destination.
 
   Layers should expect to be called in a replica context, and can use
   the `get_replica_context()` function to get a `ReplicaContext` object. The
   `ReplicaContext` object has a `merge_call()` method for entering
-  cross-replica context where you can use `reduce()` (or
-  `batch_reduce()`) and then optionally `update()` to update state.
+  cross-replica context where you can use `reduce_to()` (or
+  `batch_reduce_to()`) and then optionally `update()` to update state.
 
   You may use this API whether or not a `DistributionStrategy` is
   being used, since there is a default implementation of
   `ReplicaContext` and `DistributionStrategy`.
-  """
 
-  # TODO(josh11b): Raise an exception if variable partitioning requested before
-  #   we add support.
-  # TODO(josh11b): Also `parameter_device_index` property?
-  # TODO(josh11b): `map()`
-  # TODO(josh11b): ClusterSpec/ClusterResolver
-  # TODO(josh11b): Partitioned computations, state; sharding
-  # TODO(josh11b): Model parallelism: "replicas" with multiple devices; shuffling
-  # TODO(josh11b): List of replicas with their worker and parameter devices
-  #   (where the parameter devices may overlap in the ps case).
+  NOTE for new `DistributionStrategy` implementations: Please put all logic
+  in a subclass of `DistributionStrategyExtended`. The only code needed for
+  the `DistributionStrategy` subclass is for instantiating your subclass of
+  `DistributionStrategyExtended` in the `__init__` method.
+  """
 
-  def __init__(self):
+  def __init__(self, container_strategy):
+    self._container_strategy_weakref = weakref.ref(container_strategy)
     self._default_device = None
     # This property is used to determine if we should set drop_remainder=True
     # when creating Datasets from numpy array inputs.
     self._require_static_shapes = False
 
-  def scope(self):
-    """Returns a context manager selecting this DistributionStrategy as current.
+  def _container_strategy(self):
+    """Get the containing `DistributionStrategy`.
 
-    Inside a `with distribution_strategy.scope():` code block, this thread
-    will use a variable creator set by `distribution_strategy`, and will
-    enter its "cross-replica context".
+    This should not generally be needed except when creating a new
+    `ReplicaContext` and to validate that the caller is in the correct
+    `scope()`.
 
     Returns:
-      A context manager.
+      The `DistributionStrategy` such that `strategy.extended` is `self`.
     """
+    container_strategy = self._container_strategy_weakref()
+    assert container_strategy is not None
+    return container_strategy
+
+  def _scope(self, strategy):
+    """Implementation of DistributionStrategy.scope()."""
     if distribution_strategy_context.has_distribution_strategy():
-      _require_cross_replica_context(self)
-      return _SameScopeAgainContext(self)
+      _require_cross_replica_context_extended(self)
+      return _SameScopeAgainContext(strategy)
 
     def creator_with_resource_vars(*args, **kwargs):
-      _require_distribution_strategy_scope(self)
+      _require_distribution_strategy_scope_extended(self)
       kwargs["use_resource"] = True
       return self._create_variable(*args, **kwargs)
 
@@ -540,7 +911,8 @@ class DistributionStrategy(object):
       return getter(*args, **kwargs)
 
     return _CurrentDistributionContext(
-        self, variable_scope.variable_creator_scope(creator_with_resource_vars),
+        strategy,
+        variable_scope.variable_creator_scope(creator_with_resource_vars),
         variable_scope.variable_scope(
             variable_scope.get_variable_scope(),
             custom_getter=distributed_getter), self._default_device)
@@ -603,12 +975,12 @@ class DistributionStrategy(object):
       A context manager.
     """
     def create_colocated_variable(next_creator, *args, **kwargs):
-      _require_distribution_strategy_scope(self)
+      _require_distribution_strategy_scope_extended(self)
       kwargs["use_resource"] = True
       kwargs["colocate_with"] = colocate_with_variable
       return next_creator(*args, **kwargs)
 
-    _require_distribution_strategy_scope(self)
+    _require_distribution_strategy_scope_extended(self)
     return variable_scope.variable_creator_scope(create_colocated_variable)
 
   def _call_dataset_fn(self, dataset_fn, input_context=None):
@@ -626,144 +998,44 @@ class DistributionStrategy(object):
           "DistributionStrategy.")
     return result
 
-  def make_dataset_iterator(self, dataset):
-    """Makes an iterator for input provided via input_dataset.
-
-    Data from the given dataset will be distributed evenly across all the
-    compute replicas. We will assume that the input dataset is batched by the
-    global batch size. With this assumption, we will make a best effort to
-    divide each batch across all the replicas (one or more workers).
-    If this effort fails, an error will be thrown, and the user should instead
-    use `make_input_fn_iterator` which provides more control to the user, and
-    does not try to divide a batch across replicas.
-
-    The user could also use `make_input_fn_iterator` if they want to
-    customize which input is fed to which replica/worker etc.
-
-    Args:
-      dataset: `tf.data.Dataset` that will be distributed evenly across all
-        replicas.
-
-    Returns:
-      An `InputIterator` which returns inputs for each step of the computation.
-      User should call `initialize` on the returned iterator.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
   # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
   # Dataset API such as make_one_shot_iterator and make_initializable_iterator.
   # Extend to implement more functionality of datasets.
-  def distribute_dataset(self, dataset_fn):
-    """Return a `dataset` split across all replicas.
-
-    Suitable for providing input to for `call_for_each_replica()` by creating an
-    iterator:
-
-    ```
-    def dataset_fn():
-      return tf.data.Dataset.from_tensors([[1.]]).repeat()
-    with distribution_strategy.scope():
-      distributed_dataset = distribution_strategy.distribute_dataset(dataset_fn)
-      iterator = distributed_dataset.make_initializable_iterator()
-      replica_results = distribution_strategy.call_for_each_replica(
-          replica_fn, args=(iterator.get_next(),))
-    ```
-
-    Args:
-      dataset_fn: A function that returns a `tf.data.Dataset`.
-
-    Returns:
-      A `PerReplicaDataset` that will produce data for each replica.
-    """
+  def _distribute_dataset(self, dataset_fn):
     raise NotImplementedError("must be implemented in descendants")
 
-  def make_input_fn_iterator(self,
-                             input_fn,
-                             replication_mode=InputReplicationMode.PER_WORKER):
-    """Returns an iterator split across replicas created from an input function.
-
-    The `input_fn` should take an `InputContext` object where information about
-    input sharding can be accessed:
-
-    ```
-    def input_fn(input_context):
-      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
-      return d.shard(input_context.num_input_pipelines,
-                     input_context.input_pipeline_id)
-    with distribution_strategy.scope():
-      iterator = distribution_strategy.make_input_fn_iterator(
-          input_fn)
-      replica_results = distribution_strategy.call_for_each_replica(
-          replica_fn, iterator.get_next())
-    ```
-
-    Args:
-      input_fn: A function that returns a `tf.data.Dataset`. This function is
-        expected to take an `InputContext` object.
-      replication_mode: an enum value of `InputReplicationMode`. Only
-        `PER_WORKER` is supported currently.
-
-    Returns:
-      An iterator object that can be initialized and fetched next element.
-    """
-    if replication_mode != InputReplicationMode.PER_WORKER:
-      raise ValueError(
-          "Input replication mode not supported: %r" % replication_mode)
-    return self._make_input_fn_iterator(
-        input_fn, replication_mode=replication_mode)
+  def _make_dataset_iterator(self, dataset):
+    raise NotImplementedError("must be implemented in descendants")
 
-  def _make_input_fn_iterator(self,
-                              input_fn,
-                              replication_mode=InputReplicationMode.PER_WORKER):
+  def _make_input_fn_iterator(self, input_fn, replication_mode):
     raise NotImplementedError("must be implemented in descendants")
 
-  def broadcast(self, tensor, destinations=None):
+  def broadcast_to(self, tensor, destinations):
     """Mirror a tensor on one device to all worker devices.
 
     Args:
       tensor: A Tensor value to broadcast.
-      destinations: An optional mirrored variable, device string, or
-        list of device strings, specifying the destination devices
-        to copy `tensor` to. Defaults to `self.worker_devices`.
+      destinations: A mirrored variable, device string, or list of device
+        strings, specifying the destination devices to copy `tensor` to.
 
     Returns:
       A value mirrored to `destinations` devices.
     """
     # TODO(josh11b): More docstring
-    _require_cross_replica_context(self)
-    return self._broadcast(tensor, destinations)
+    _require_cross_replica_context_extended(self)
+    return self._broadcast_to(tensor, destinations)
 
-  def _broadcast(self, tensor, destinations):
+  def _broadcast_to(self, tensor, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
-  def initialize(self):
-    """Any initialization to be done before running any computations.
-
-    In eager mode, it executes any initialization as a side effect.
-    In graph mode, it creates the initialization ops and returns them.
-
-    For example, TPU initialize_system ops.
-
-    Returns:
-      A list of ops to execute.
-    """
+  def _initialize(self):
     return []
 
-  def finalize(self):
-    """Any final actions to be done at the end of all computations.
-
-    In eager mode, it executes any finalize actions as a side effect.
-    In graph mode, it creates the finalize ops and returns them.
-
-    For example, TPU shutdown ops.
-
-    Returns:
-      A list of ops to execute.
-    """
+  def _finalize(self):
     return []
 
-  def run_steps_on_dataset(self, fn, iterator, iterations=1,
-                           initial_loop_values=None):
+  def experimental_run_steps_on_iterator(self, fn, iterator, iterations=1,
+                                         initial_loop_values=None):
     """Run `fn` with input from `iterator` for `iterations` times.
 
     This method can be used to run a step function for training a number of
@@ -804,15 +1076,15 @@ class DistributionStrategy(object):
         - non_tensor_outputs: A dictionatry containing anything that was set by
           `fn` by calling `context.set_non_tensor_output`.
     """
-    _require_cross_replica_context(self)
-    return self._run_steps_on_dataset(fn, iterator, iterations,
-                                      initial_loop_values)
+    _require_cross_replica_context_extended(self)
+    return self._experimental_run_steps_on_iterator(
+        fn, iterator, iterations, initial_loop_values)
 
-  def _run_steps_on_dataset(self, fn, iterator, iterations,
-                            initial_loop_values):
+  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
+                                          initial_loop_values):
     raise NotImplementedError("must be implemented in descendants")
 
-  def call_for_each_replica(self, fn, *args, **kwargs):
+  def call_for_each_replica(self, fn, args, kwargs):
     """Run `fn` once per replica.
 
     `fn` may call `tf.get_replica_context()` to access methods such as
@@ -855,38 +1127,21 @@ class DistributionStrategy(object):
     Returns:
       Merged return value of `fn` across all replicas.
     """
-    _require_cross_replica_context(self)
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to call_for_each_replica")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to call_for_each_replica")
-      kwargs = k
-    kwargs.pop("run_concurrently", None)  # Ignore old option.
+    _require_cross_replica_context_extended(self)
     return self._call_for_each_replica(fn, args, kwargs)
 
   def _call_for_each_replica(self, fn, args, kwargs):
     raise NotImplementedError("must be implemented in descendants")
 
-  def reduce(self, aggregation, value, destinations):
+  def reduce_to(self, reduce_op, value, destinations):
     """Combine (via e.g. sum or mean) values across replicas.
 
     Args:
-      aggregation: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
         DEPRECATED but still accepted values:
         `tf.VariableAggregation.SUM`,
         `tf.VariableAggregation.MEAN`,
         `tf.VariableAggregation.ONLY_FIRST_REPLICA`.
-        # TODO(priyag): Rename this argument when moving the method to
-        # DSExtended.
       value: A per-replica value with one value per replica.
       destinations: A mirrored variable, a per-replica tensor, a device string,
         or list of device strings. The return value will be copied to all
@@ -899,60 +1154,56 @@ class DistributionStrategy(object):
     # TODO(josh11b): More docstring
     # TODO(josh11b): Return an unwrapped value if colocate_with is a
     # single device.
-    _require_cross_replica_context(self)
+    _require_cross_replica_context_extended(self)
 
     # TODO(priyag): Remove this when all callers have been updated.
-    reduce_op = aggregation
-    if isinstance(aggregation, variable_scope.VariableAggregation):
-      assert aggregation in [
+    if isinstance(reduce_op, variable_scope.VariableAggregation):
+      assert reduce_op in [
           variable_scope.VariableAggregation.SUM,
           variable_scope.VariableAggregation.MEAN,
           variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
       ]
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
-    return self._reduce(reduce_op, value, destinations)
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    return self._reduce_to(reduce_op, value, destinations)
 
-  def _reduce(self, reduce_op, value, destinations):
+  def _reduce_to(self, reduce_op, value, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
-  def batch_reduce(self, aggregation, value_destination_pairs):
-    """Combine multiple `reduce` calls into one for faster execution.
+  def batch_reduce_to(self, reduce_op, value_destination_pairs):
+    """Combine multiple `reduce_to` calls into one for faster execution.
 
     Args:
-      aggregation: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
         DEPRECATED but still accepted values:
         `tf.VariableAggregation.SUM`,
         `tf.VariableAggregation.MEAN`,
         `tf.VariableAggregation.ONLY_FIRST_REPLICA`.
-        # TODO(priyag): Rename this argument when moving the method to
-        # DSExtended.
       value_destination_pairs: A sequence of (value, destinations)
-        pairs. See `reduce()` for a description.
+        pairs. See `reduce_to()` for a description.
 
     Returns:
       A list of mirrored values, one per pair in `value_destination_pairs`.
     """
     # TODO(josh11b): More docstring
-    _require_cross_replica_context(self)
+    _require_cross_replica_context_extended(self)
 
     # TODO(priyag): Remove this when all callers have been updated.
-    reduce_op = aggregation
-    if isinstance(aggregation, variable_scope.VariableAggregation):
-      assert aggregation in [
+    if isinstance(reduce_op, variable_scope.VariableAggregation):
+      assert reduce_op in [
           variable_scope.VariableAggregation.SUM,
           variable_scope.VariableAggregation.MEAN,
           variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
       ]
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
-    return self._batch_reduce(reduce_op, value_destination_pairs)
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    return self._batch_reduce_to(reduce_op, value_destination_pairs)
 
-  def _batch_reduce(self, reduce_op, value_destination_pairs):
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
     return [
-        self.reduce(reduce_op, t, destinations=v)
+        self.reduce_to(reduce_op, t, destinations=v)
         for t, v in value_destination_pairs
     ]
 
-  def update(self, var, fn, *args, **kwargs):
+  def update(self, var, fn, args, kwargs, group):
     """Run `fn` to update `var` using inputs mirrored to the same devices.
 
     If `var` is mirrored across multiple devices, then this implements
@@ -969,7 +1220,7 @@ class DistributionStrategy(object):
 
     Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`.
 
-    Neither `*args` nor `**kwargs` may contain per-replica values.
+    Neither `args` nor `kwargs` may contain per-replica values.
     If they contain mirrored values, they will be unwrapped before
     calling `fn`.
 
@@ -989,31 +1240,13 @@ class DistributionStrategy(object):
       where each list has an element per replica, and the caller is responsible
       for ensuring all elements are executed.
     """
-    _require_cross_replica_context(self)
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update")
-      kwargs = k
+    _require_cross_replica_context_extended(self)
     return self._update(var, fn, args, kwargs, group)
 
   def _update(self, var, fn, args, kwargs, group):
     raise NotImplementedError("must be implemented in descendants")
 
-  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
+  def update_non_slot(self, colocate_with, fn, args, kwargs, group):
     """Runs `fn(*args, **kwargs)` on `colocate_with` devices.
 
     Args:
@@ -1027,42 +1260,14 @@ class DistributionStrategy(object):
     Returns:
       Return value of `fn`, possibly merged across devices.
     """
-    _require_cross_replica_context(self)
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update_non_slot")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update_non_slot")
-      kwargs = k
+    _require_cross_replica_context_extended(self)
     return self._update_non_slot(colocate_with, fn, args, kwargs, group)
 
   def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
     raise NotImplementedError("must be implemented in descendants")
 
-  def unwrap(self, value):
-    """Returns the list of all per-replica values contained in `value`.
-
-    Args:
-      value: A value returned by `call_for_each_replica()` or a variable
-        created in `scope()`.
-
-    Returns:
-      A list of values contained in `value`. If `value` represents a single
-      value, this returns `[value].`
-    """
-    return self._unwrap(value)
+  def _unwrap(self, distributed_value):
+    raise NotImplementedError("must be implemented in descendants")
 
   def value_container(self, value):
     """Returns the container that this per-replica `value` belongs to.
@@ -1079,12 +1284,9 @@ class DistributionStrategy(object):
     """
     raise NotImplementedError("must be implemented in descendants")
 
-  def _unwrap(self, distributed_value):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def group(self, value, name=None):
+  def _group(self, value, name=None):
     """Shortcut for `tf.group(distribution.unwrap(value))`."""
-    value = nest.flatten(self.unwrap(value))
+    value = nest.flatten(self._unwrap(value))
 
     if len(value) != 1 or name is not None:
       return control_flow_ops.group(value, name=name)
@@ -1095,11 +1297,11 @@ class DistributionStrategy(object):
     return v
 
   @property
-  def require_static_shapes(self):
+  def experimental_require_static_shapes(self):
     return self._require_static_shapes
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     """Returns number of replicas over which gradients are aggregated."""
     raise NotImplementedError("must be implemented in descendants")
 
@@ -1130,7 +1332,7 @@ class DistributionStrategy(object):
     raise NotImplementedError("must be implemented in descendants")
 
   @property
-  def between_graph(self):
+  def experimental_between_graph(self):
     """Whether the strategy uses between-graph replication or not.
 
       This is expected to return a constant value that will not be changed
@@ -1138,16 +1340,16 @@ class DistributionStrategy(object):
     """
     raise NotImplementedError("must be implemented in descendants")
 
-  def configure(self,
-                session_config=None,
-                cluster_spec=None,
-                task_type=None,
-                task_id=None):
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
     """Configures the strategy class."""
     del session_config, cluster_spec, task_type, task_id
 
   @property
-  def should_init(self):
+  def experimental_should_init(self):
     """Whether initialization is needed."""
     raise NotImplementedError("must be implemented in descendants")
 
@@ -1258,7 +1460,7 @@ class ReplicaContext(object):
 
   @property
   def replica_id_in_sync_group(self):
-    """Which replica is being defined, a number from 0 to `num_replicas - 1`."""
+    """Which replica is being defined, from 0 to `num_replicas_in_sync - 1`."""
     require_replica_context(self)
     return self._replica_id_in_sync_group
 
@@ -1281,7 +1483,7 @@ class ReplicaContext(object):
   #   in the background, not blocking until the result was needed.
   # * When constructing a graph, it could batch up all reduction requests up
   #   to that point that the first result is needed. Most likely this can be
-  #   implemented in terms of `merge_call()` and `batch_reduce()`.
+  #   implemented in terms of `merge_call()` and `batch_reduce_to()`.
 
 # ------------------------------------------------------------------------------
 
@@ -1289,47 +1491,55 @@ class ReplicaContext(object):
 class _DefaultDistributionStrategy(DistributionStrategy):
   """Default `DistributionStrategy` if none is explicitly selected."""
 
-  def scope(self):
+  def __init__(self):
+    super(_DefaultDistributionStrategy, self).__init__(
+        _DefaultDistributionExtended(self))
+
+
+class _DefaultDistributionExtended(DistributionStrategyExtended):
+  """Implementation of _DefaultDistributionStrategy."""
+
+  def _scope(self, strategy):
     """Context manager setting a variable creator and `self` as current."""
     if distribution_strategy_context.has_distribution_strategy():
       raise RuntimeError("Must not nest DistributionStrategy scopes.")
 
     def creator(next_creator, *args, **kwargs):
-      _require_distribution_strategy_scope(self)
+      _require_distribution_strategy_scope_strategy(strategy)
       return next_creator(*args, **kwargs)
 
     return _CurrentDistributionContext(
-        self, variable_scope.variable_creator_scope(creator))
+        strategy, variable_scope.variable_creator_scope(creator))
 
   def colocate_vars_with(self, colocate_with_variable):
     """Does not require `self.scope`."""
-    _require_distribution_strategy_scope(self)
+    _require_distribution_strategy_scope_extended(self)
     return ops.colocate_with(colocate_with_variable)
 
-  def make_dataset_iterator(self, dataset):
-    return dataset.make_initializable_iterator()
-
-  def distribute_dataset(self, dataset_fn):
+  def _distribute_dataset(self, dataset_fn):
     return self._call_dataset_fn(dataset_fn)
 
+  def _make_dataset_iterator(self, dataset):
+    return dataset.make_initializable_iterator()
+
   def _make_input_fn_iterator(self,
                               input_fn,
                               replication_mode=InputReplicationMode.PER_WORKER):
     return self._call_dataset_fn(input_fn, InputContext())
 
-  def _broadcast(self, tensor, destinations):
+  def _broadcast_to(self, tensor, destinations):
     if destinations is None:
       return tensor
     else:
       raise NotImplementedError("TODO")
 
   def _call_for_each_replica(self, fn, args, kwargs):
-    with ReplicaContext(self, \
-                        replica_id_in_sync_group= \
-                        constant_op.constant(0, dtypes.int32)):
+    with ReplicaContext(
+        self._container_strategy(),
+        replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
       return fn(*args, **kwargs)
 
-  def _reduce(self, reduce_op, value, destinations):
+  def _reduce_to(self, reduce_op, value, destinations):
     # TODO(josh11b): Use destinations?
     del reduce_op, destinations
     return value
@@ -1359,7 +1569,7 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     return value
 
   @property
-  def num_replicas_in_sync(self):
+  def _num_replicas_in_sync(self):
     return 1
 
   @property
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index d8c7d64088..10b8dead5f 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -42,9 +42,16 @@ def _get_test_variable(name, synchronization, aggregation):
 
 class _TestStrategy(distribute_lib.DistributionStrategy):
 
+  def __init__(self):
+    super(_TestStrategy, self).__init__(_TestExtended(self))
+
+
+class _TestExtended(distribute_lib.DistributionStrategyExtended):
+
   def _call_for_each_replica(self, fn, args, kwargs):
     with _TestReplicaContext(
-        self, replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
+        self._container_strategy(),
+        replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
       return fn(*args, **kwargs)
 
   def _create_variable(self, next_creator, *args, **kwargs):
-- 
GitLab


From c82941f88c2ff47f4df68f651069a45f365b4eeb Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Thu, 15 Nov 2018 16:00:04 -0800
Subject: [PATCH 0353/1554] Remove tf.load_file_system_library from TF 2.0

PiperOrigin-RevId: 221706408
---
 tensorflow/python/framework/load_library.py     |  4 +++-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt |  4 ----
 tensorflow/tools/compatibility/renames_v2.py    | 14 ++++++++++++++
 tensorflow/tools/compatibility/tf_upgrade_v2.py |  1 +
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py
index 908a5f521e..727f6aa44c 100644
--- a/tensorflow/python/framework/load_library.py
+++ b/tensorflow/python/framework/load_library.py
@@ -31,6 +31,7 @@ from tensorflow.core.lib.core import error_codes_pb2  # pylint: disable=unused-i
 from tensorflow.python import pywrap_tensorflow as py_tf
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.util import compat
+from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -83,7 +84,8 @@ def load_op_library(library_filename):
   return module
 
 
-@tf_export('load_file_system_library')
+@deprecation.deprecated(date=None, instructions='Use tf.load_library instead.')
+@tf_export(v1=['load_file_system_library'])
 def load_file_system_library(library_filename):
   """Loads a TensorFlow plugin, containing file system implementation.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a2ac908195..42b9397504 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -796,10 +796,6 @@ tf_module {
     name: "linspace"
     argspec: "args=[\'start\', \'stop\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "load_file_system_library"
-    argspec: "args=[\'library_filename\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "load_library"
     argspec: "args=[\'library_location\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index de87f405bc..a653623a27 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -101,9 +101,22 @@ renames = {
     'tf.convert_to_tensor': 'tf.compat.v1.convert_to_tensor',
     'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices',
     'tf.convert_to_tensor_or_sparse_tensor': 'tf.compat.v1.convert_to_tensor_or_sparse_tensor',
+    'tf.count_nonzero': 'tf.compat.v1.count_nonzero',
     'tf.count_up_to': 'tf.compat.v1.count_up_to',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
+    'tf.data.Dataset': 'tf.compat.v1.data.Dataset',
+    'tf.data.FixedLengthRecordDataset': 'tf.compat.v1.data.FixedLengthRecordDataset',
+    'tf.data.TFRecordDataset': 'tf.compat.v1.data.TFRecordDataset',
+    'tf.data.TextLineDataset': 'tf.compat.v1.data.TextLineDataset',
+    'tf.data.experimental.Counter': 'tf.compat.v1.data.experimental.Counter',
+    'tf.data.experimental.CsvDataset': 'tf.compat.v1.data.experimental.CsvDataset',
+    'tf.data.experimental.RandomDataset': 'tf.compat.v1.data.experimental.RandomDataset',
+    'tf.data.experimental.SqlDataset': 'tf.compat.v1.data.experimental.SqlDataset',
+    'tf.data.experimental.choose_from_datasets': 'tf.compat.v1.data.experimental.choose_from_datasets',
+    'tf.data.experimental.make_batched_features_dataset': 'tf.compat.v1.data.experimental.make_batched_features_dataset',
+    'tf.data.experimental.make_csv_dataset': 'tf.compat.v1.data.experimental.make_csv_dataset',
+    'tf.data.experimental.sample_from_datasets': 'tf.compat.v1.data.experimental.sample_from_datasets',
     'tf.decode_base64': 'tf.io.decode_base64',
     'tf.decode_compressed': 'tf.io.decode_compressed',
     'tf.decode_csv': 'tf.io.decode_csv',
@@ -254,6 +267,7 @@ renames = {
     'tf.layers.separable_conv2d': 'tf.compat.v1.layers.separable_conv2d',
     'tf.lbeta': 'tf.math.lbeta',
     'tf.lgamma': 'tf.math.lgamma',
+    'tf.load_file_system_library': 'tf.compat.v1.load_file_system_library',
     'tf.local_variables': 'tf.compat.v1.local_variables',
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log_sigmoid': 'tf.math.log_sigmoid',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 7e741fa1d9..3a8ff25937 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -59,6 +59,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Add additional renames not in renames_v2.py here.
     self.symbol_renames.update({
         "tf.sparse_concat": "tf.sparse.concat",
+        "tf.load_file_system_library": "tf.load_library",
     })
     # pylint: enable=line-too-long
 
-- 
GitLab


From d5ae07428338f9be3f664fa549a3df23a2c15bb2 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 15 Nov 2018 16:08:09 -0800
Subject: [PATCH 0354/1554] Proper test for two gpus in DeviceToDevice copy.

PiperOrigin-RevId: 221707799
---
 tensorflow/c/eager/BUILD         |  1 -
 tensorflow/c/eager/c_api_test.cc | 15 ++++++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index c25b78434f..ba3d8533db 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -133,7 +133,6 @@ tf_cuda_cc_test(
     tags = [
         "guitar",
         "multi_gpu",
-        "no_oss",
     ],
     deps = [
         ":c_api",
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 55331022b9..0045bb5622 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -589,9 +589,22 @@ void TensorHandleCopyBetweenTwoGPUDevices(bool async) {
   TF_DeviceList* devices = TFE_ContextListDevices(ctx, status.get());
   ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
   const int num_devices = TF_DeviceListCount(devices);
+  bool has_gpu0 = false;
+  bool has_gpu1 = false;
+  for (int i = 0; i < num_devices; ++i) {
+    const char* dev = TF_DeviceListName(devices, i, status.get());
+    ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+    string device_name(dev);
+    if (device_name.find("GPU:0") != string::npos) {
+      has_gpu0 = true;
+    }
+    if (device_name.find("GPU:1") != string::npos) {
+      has_gpu1 = true;
+    }
+  }
 
   const char* kCPUDevice = "CPU:0";
-  if (num_devices < 3) {
+  if (!has_gpu0 || !has_gpu1) {
     TF_DeleteDeviceList(devices);
     TF_DeleteTensor(t);
     TFE_DeleteTensorHandle(hcpu);
-- 
GitLab


From 8828ef8e020323d5990622b2c0467715e600d0d5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 16:10:38 -0800
Subject: [PATCH 0355/1554] Drop support for ONLY_FIRST_REPLICA reductions.
 Instead when using ONLY_FIRST_REPLICA variable aggregation, perform a
 broadcast instead.

PiperOrigin-RevId: 221708135
---
 .../distribute/python/mirrored_strategy.py    | 21 ++++--------
 .../python/mirrored_strategy_multigpu_test.py | 19 -----------
 .../python/parameter_server_strategy.py       |  7 ----
 .../contrib/distribute/python/tpu_strategy.py |  2 --
 .../contrib/distribute/python/values.py       | 32 ++++++++-----------
 tensorflow/python/distribute/reduce_util.py   | 10 ++----
 tensorflow/python/training/distribute.py      |  4 ---
 7 files changed, 23 insertions(+), 72 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 960cd09696..fb59a8ad91 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -191,10 +191,9 @@ def _reduce_non_distributed_value(extended, reduce_op, value, destinations):
   # and equal to 0.
   if value == 0:
     return 0
-  # If the reduce op is MEAN or ONLY_FIRST_REPLICA, then this
-  # essentially means that the same value should be on all destinations.
-  if reduce_op in (reduce_util.ReduceOp.MEAN,
-                   reduce_util.ReduceOp.ONLY_FIRST_REPLICA):
+  # If there is only a single value and the reduce op is MEAN,
+  # that value should be on all destinations.
+  if reduce_op == reduce_util.ReduceOp.MEAN:
     return value
 
   cross_tower_ops_lib.validate_destinations(destinations)
@@ -587,9 +586,11 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     return ctx
 
   def _broadcast_to(self, tensor, destinations):
+    if isinstance(tensor, (float, int)):  # Fast path for Python constants.
+      return tensor
     # TODO(josh11b): In eager mode, use one thread per device, or async mode.
-    return self._get_cross_tower_ops().broadcast(tensor, destinations or
-                                                 self._devices)
+    return self._get_cross_tower_ops().broadcast(
+        tensor, destinations or self._devices)
 
   def _call_for_each_replica(self, fn, args, kwargs):
     return _call_for_each_replica(self._container_strategy(), fn, args, kwargs)
@@ -637,18 +638,10 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
       # be 0.
       return _reduce_non_distributed_value(self, reduce_op, value,
                                            destinations)
-    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      value = value.get(self._devices[0])
-      if isinstance(value, (int, float)):
-        return value
-      return self.broadcast_to(value, destinations)
     return self._get_cross_tower_ops().reduce(
         reduce_op, value, destinations=destinations)
 
   def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return [self.broadcast_to(v.get(self._devices[0]), d)
-              for v, d in value_destination_pairs]
     return self._get_cross_tower_ops().batch_reduce(reduce_op,
                                                     value_destination_pairs)
 
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 78d6876723..d45eb46e6f 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -126,25 +126,6 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
       expected = sum(range(dist.num_replicas_in_sync))
       self.assertEqual(expected, self.evaluate(unwrapped[0]))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testReduceOnlyFirstReplicaUpdates(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-
-    def run_fn():
-      return 3 + 5 * _replica_id()
-
-    dist = self._get_distribution_strategy()
-    with dist.scope():
-      result = dist.call_for_each_replica(run_fn)
-      reduced = dist.reduce(
-          reduce_util.ReduceOp.ONLY_FIRST_REPLICA,
-          result,
-          destinations="/device:CPU:0")
-      unwrapped = dist.unwrap(reduced)
-      self.assertEqual(1, len(unwrapped))
-      self.assertEqual(3, self.evaluate(unwrapped[0]))
-
   @test_util.run_in_graph_and_eager_modes()
   def testReduceToMultipleDestinations(self):
     if not GPU_TEST:
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 58b6c87179..e87337baff 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -22,7 +22,6 @@ from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import values
 from tensorflow.python.distribute import multi_worker_util
-from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
@@ -344,16 +343,10 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
       # pylint: disable=protected-access
       return mirrored_strategy._reduce_non_distributed_value(
           self, reduce_op, value, destinations)
-    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return self.broadcast_to(
-          value.get(self._compute_devices[0]), destinations)
     return self._cross_tower_ops.reduce(
         reduce_op, value, destinations=destinations)
 
   def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return [self.broadcast_to(v.get(self._compute_devices[0]), d)
-              for v, d in value_destination_pairs]
     for _, destinations in value_destination_pairs:
       self._verify_destinations_not_different_worker(destinations)
     return self._cross_tower_ops.batch_reduce(reduce_op,
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 6e42bf9d97..cfe411af76 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -474,8 +474,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     else:
       raise ValueError("Multiple devices are not supported for TPUStrategy")
 
-    if reduce_op == reduce_util.ReduceOp.ONLY_FIRST_REPLICA:
-      return value[0]
     output = math_ops.add_n(value)
     if reduce_op == reduce_util.ReduceOp.MEAN:
       return output * (1. / len(value))
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 86e647c4c6..80b9b1f457 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -318,6 +318,14 @@ class DistributedVariable(DistributedDelegate):
 ops.register_dense_tensor_like_type(DistributedVariable)
 
 
+def _apply_aggregation(strategy, value, aggregation, destinations):
+  if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA:
+    return strategy.broadcast(strategy.unwrap(value)[0],
+                              destinations=destinations)
+  reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
+  return strategy.reduce(reduce_op, value=value, destinations=destinations)
+
+
 class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable):
   """Class for defining how to restore a MirroredVariable."""
 
@@ -373,14 +381,10 @@ class MirroredVariable(DistributedVariable, Mirrored,
       if self._aggregation == vs.VariableAggregation.NONE:
         raise ValueError("You must specify an aggregation method to update a "
                          "MirroredVariable in Replica Context.")
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(
-          self._aggregation)
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
-        return strategy.update(
-            self, f,
-            strategy.reduce(reduce_op, value=value, destinations=self),
-            *other_args, **other_kwargs)
+        v = _apply_aggregation(strategy, value, self._aggregation, self)
+        return strategy.update(self, f, v, *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, *args, **kwargs)
@@ -615,14 +619,10 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
       if self._aggregation == vs.VariableAggregation.NONE:
         raise ValueError("You must specify an aggregation method to update a "
                          "TPUMirroredVariable in Replica Context.")
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(
-          self._aggregation)
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
-        return strategy.update(
-            self, f,
-            strategy.reduce(reduce_op, value=value, destinations=self),
-            *other_args, **other_kwargs)
+        v = _apply_aggregation(strategy, value, self._aggregation, self)
+        return strategy.update(self, f, v, *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, *args, **kwargs)
@@ -1662,14 +1662,10 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
       if self._aggregation == vs.VariableAggregation.NONE:
         raise ValueError("You must specify an aggregation method to update a "
                          "a variable in Replica Context.")
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(
-          self._aggregation)
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
-        return strategy.update(
-            self, f,
-            strategy.reduce(reduce_op, value=value, destinations=self),
-            *other_args, **other_kwargs)
+        v = _apply_aggregation(strategy, value, self._aggregation, self)
+        return strategy.update(self, f, v, *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, *args, **kwargs)
diff --git a/tensorflow/python/distribute/reduce_util.py b/tensorflow/python/distribute/reduce_util.py
index 8df3923daf..eb027d8b2a 100644
--- a/tensorflow/python/distribute/reduce_util.py
+++ b/tensorflow/python/distribute/reduce_util.py
@@ -29,7 +29,6 @@ class ReduceOp(enum.Enum):
 
   * `SUM`: Add all the values.
   * `MEAN`: Take the arithmetic mean ("average") of the values.
-  * `ONLY_FIRST_REPLICA`: Return the value on the first replica.
 
   TODO(priyag): Add the following types:
   * `MIN`: Return the minimum of all values.
@@ -38,21 +37,16 @@ class ReduceOp(enum.Enum):
 
   SUM = 0
   MEAN = 1
-  ONLY_FIRST_REPLICA = 2
 
   @staticmethod
   def from_variable_aggregation(aggregation):
     mapping = {
         variable_scope.VariableAggregation.SUM: ReduceOp.SUM,
         variable_scope.VariableAggregation.MEAN: ReduceOp.MEAN,
-        variable_scope.VariableAggregation.ONLY_FIRST_REPLICA:
-            ReduceOp.ONLY_FIRST_REPLICA
     }
 
     reduce_op = mapping.get(aggregation)
     if not reduce_op:
-      raise ValueError("Could not convert from `tf.VariableAggregation` to"
-                       "`tf.distribute.ReduceOp` type")
+      raise ValueError("Could not convert from `tf.VariableAggregation` %s to"
+                       "`tf.distribute.ReduceOp` type" % aggregation)
     return reduce_op
-
-
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 130eb6ad5b..53148ed6f9 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1141,7 +1141,6 @@ class DistributionStrategyExtended(object):
         DEPRECATED but still accepted values:
         `tf.VariableAggregation.SUM`,
         `tf.VariableAggregation.MEAN`,
-        `tf.VariableAggregation.ONLY_FIRST_REPLICA`.
       value: A per-replica value with one value per replica.
       destinations: A mirrored variable, a per-replica tensor, a device string,
         or list of device strings. The return value will be copied to all
@@ -1161,7 +1160,6 @@ class DistributionStrategyExtended(object):
       assert reduce_op in [
           variable_scope.VariableAggregation.SUM,
           variable_scope.VariableAggregation.MEAN,
-          variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
       ]
       reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
     return self._reduce_to(reduce_op, value, destinations)
@@ -1177,7 +1175,6 @@ class DistributionStrategyExtended(object):
         DEPRECATED but still accepted values:
         `tf.VariableAggregation.SUM`,
         `tf.VariableAggregation.MEAN`,
-        `tf.VariableAggregation.ONLY_FIRST_REPLICA`.
       value_destination_pairs: A sequence of (value, destinations)
         pairs. See `reduce_to()` for a description.
 
@@ -1192,7 +1189,6 @@ class DistributionStrategyExtended(object):
       assert reduce_op in [
           variable_scope.VariableAggregation.SUM,
           variable_scope.VariableAggregation.MEAN,
-          variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
       ]
       reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
     return self._batch_reduce_to(reduce_op, value_destination_pairs)
-- 
GitLab


From 5d1b29c525250e5121c269ee1157bc300cc84d46 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Thu, 15 Nov 2018 16:15:58 -0800
Subject: [PATCH 0356/1554] Make tf-nightly pip depend on tf-estimator-nightly
 pip.

PiperOrigin-RevId: 221708873
---
 tensorflow/tools/pip_package/setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 07475cc0c4..e164853428 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -87,7 +87,8 @@ if 'tf_nightly' in project_name:
   for i, pkg in enumerate(REQUIRED_PACKAGES):
     if 'tensorboard' in pkg:
       REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.13.0a0, < 1.14.0a0'
-      break
+    if 'tensorflow_estimator' in pkg:
+      REQUIRED_PACKAGES[i] = 'tf-estimator-nightly'
 
 # weakref.finalize and enum were introduced in Python 3.4
 if sys.version_info < (3, 4):
-- 
GitLab


From 98a1c014d5f3b93666818ad7266052233387216b Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Thu, 15 Nov 2018 16:16:00 -0800
Subject: [PATCH 0357/1554] Update grappler plugin to support swap of
 forward/backward defun call. Also added a new unit test to show the e2e LSTM
 with swap of underlying functions with grappler.

PiperOrigin-RevId: 221708879
---
 tensorflow/core/framework/function.cc         |  24 +
 tensorflow/core/framework/function_test.cc    |  37 +-
 tensorflow/core/grappler/optimizers/BUILD     |   4 +-
 .../experimental_implementation_selector.cc   | 114 ++++-
 ...perimental_implementation_selector_test.cc |  95 ++++
 .../grappler/optimizers/function_api_info.cc  | 180 +++++--
 .../grappler/optimizers/function_api_info.h   |  38 +-
 .../optimizers/function_api_info_test.cc      | 118 +++--
 tensorflow/python/keras/BUILD                 |  11 +
 tensorflow/python/keras/backend.py            |   4 +-
 .../python/keras/layers/unified_rnn_test.py   | 469 ++++++++++++++++++
 11 files changed, 959 insertions(+), 135 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/unified_rnn_test.py

diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index cb894099b0..838f899911 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -1282,12 +1282,21 @@ GET_ATTR(bool)
 
 namespace {
 
+constexpr char kExperimentalApiImplements[] = "experimental_api_implements";
+
 absl::flat_hash_set<string> ReachableFunctions(
     const FunctionLibraryDefinition& flib,
     const protobuf::RepeatedPtrField<NodeDef>& nodes) {
   // Functions that are reachable from the graph.
   absl::flat_hash_set<string> reachable_funcs;
 
+  // For any functions, if it has attribute "experimental_api_implements" =
+  // "some_interface" and it is reachable, then it means any other
+  // function with same attribute name and value could also be potentially
+  // reachable, eg via experimental_implementation_selector swapping the
+  // nodedef.
+  absl::flat_hash_set<string> reachable_api_interface;
+
   // Functions might be reachable from the nested function calls, so we keep a
   // queue of functions that we have to check.
   gtl::InlinedVector<const FunctionDef*, 4> func_queue;
@@ -1334,6 +1343,11 @@ absl::flat_hash_set<string> ReachableFunctions(
     const string& func_name = func->signature().name();
     reachable_funcs.insert(func_name);
 
+    const auto attr_it = func->attr().find(kExperimentalApiImplements);
+    if (attr_it != func->attr().end()) {
+      reachable_api_interface.insert(attr_it->second.s());
+    }
+
     // Find all the functions called from the function body.
     const auto& func_body = func->node_def();
     std::for_each(func_body.begin(), func_body.end(), process_node);
@@ -1343,6 +1357,16 @@ absl::flat_hash_set<string> ReachableFunctions(
     if (!grad_func_name.empty()) add_to_func_queue(grad_func_name);
   }
 
+  const FunctionDefLibrary library_proto = flib.ToProto();
+  for (const auto& it : library_proto.function()) {
+    const auto attr_it = it.attr().find(kExperimentalApiImplements);
+    if (attr_it != it.attr().end()) {
+      if (reachable_api_interface.contains(attr_it->second.s())) {
+        reachable_funcs.insert(it.signature().name());
+      }
+    }
+  }
+
   return reachable_funcs;
 }
 
diff --git a/tensorflow/core/framework/function_test.cc b/tensorflow/core/framework/function_test.cc
index 28c2318c76..f57a79b167 100644
--- a/tensorflow/core/framework/function_test.cc
+++ b/tensorflow/core/framework/function_test.cc
@@ -1298,19 +1298,29 @@ TEST(FunctionLibraryDefinitionTest, ReachableDefinitions) {
   using ::tensorflow::test::function::NDef;
   using FDH = ::tensorflow::FunctionDefHelper;
 
-  const auto make_simple_fdef = [](const string& name) {
-    return FDH::Create(
+  const auto make_simple_fdef = [](const string& name,
+                                   const string& interface_name) {
+    auto func_def = FDH::Create(
         name, {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
         {{{"output"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
         /* Mapping between function returns and function node outputs. */
         {{"z", "output:z:0"}});
+
+    if (!interface_name.empty()) {
+      auto* attr = func_def.mutable_attr();
+      (*attr)["experimental_api_implements"].set_s(interface_name);
+    }
+    return func_def;
   };
 
-  FunctionDef func_1 = make_simple_fdef("Func1");
-  FunctionDef func_2 = make_simple_fdef("Func2");
-  FunctionDef func_3 = make_simple_fdef("Func3");
+  FunctionDef func_1 = make_simple_fdef("Func1", "");
+  FunctionDef func_2 = make_simple_fdef("Func2", "");
+  FunctionDef func_3 = make_simple_fdef("Func3", "");
+  FunctionDef func_4 = make_simple_fdef("Func4", "api_1");
+  FunctionDef func_5 = make_simple_fdef("Func5", "api_1");
+  FunctionDef func_6 = make_simple_fdef("Func6", "api_2");
 
-  FunctionDef func_2_grad = make_simple_fdef("Func2_grad");
+  FunctionDef func_2_grad = make_simple_fdef("Func2_grad", "");
 
   constexpr char kDevice[] = "/device:CPU:0";
 
@@ -1324,9 +1334,10 @@ TEST(FunctionLibraryDefinitionTest, ReachableDefinitions) {
                 {"Tout", DataTypeSlice{DT_FLOAT}},
                 {"f", FDH::FunctionRef("Func2", {{"T", DT_FLOAT}})}},
                kDevice),
+          NDef("z", "Func4", {"a", "b"}, {{"T", DT_FLOAT}}, kDevice),
       },
       // FunctionLib
-      {func_1, func_2, func_3, func_2_grad});
+      {func_1, func_2, func_3, func_2_grad, func_4, func_5, func_6});
 
   // Register custom function gradient after the graph was constructed.
   GradientDef* func3_grad_def = graph.mutable_library()->add_gradient();
@@ -1338,13 +1349,21 @@ TEST(FunctionLibraryDefinitionTest, ReachableDefinitions) {
   // - 'Func1' is called directly from the graph.
   // - 'Func2' is called indirectly via a PartitionedCall attribute, and it also
   //   has a custom gradient ('Func2_grad') that must remain in the library.
-  // - 'Func3' is unreachable and has to be removed from the library.
+  // - 'Func3' is unreachable and has to be removed from the library
+  // - 'Func4' is called directly from the graph
+  // - 'Func5' is not called directly, but it implements same interface as Func4
+  //   which is directly called.
+  // - 'Func6' is not called directly, and the interface it implements has not
+  //   not been called by another nodes in the graph.
   FunctionLibraryDefinition reachable_flib = flib.ReachableDefinitions(graph);
-  EXPECT_EQ(reachable_flib.num_functions(), 3);
+  EXPECT_EQ(reachable_flib.num_functions(), 5);
   EXPECT_TRUE(reachable_flib.Contains("Func1"));
   EXPECT_TRUE(reachable_flib.Contains("Func2"));
   EXPECT_TRUE(reachable_flib.Contains("Func2_grad"));
   EXPECT_FALSE(reachable_flib.Contains("Func3"));
+  EXPECT_TRUE(reachable_flib.Contains("Func4"));
+  EXPECT_TRUE(reachable_flib.Contains("Func5"));
+  EXPECT_FALSE(reachable_flib.Contains("Func6"));
 }
 
 // TODO(skyewm): this could be more thorough
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 904fd42844..b6f989f2c9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -849,6 +849,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
@@ -875,11 +876,10 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/experimental_implementation_selector.cc b/tensorflow/core/grappler/optimizers/experimental_implementation_selector.cc
index 2c36c9b7b3..75ad8bffef 100644
--- a/tensorflow/core/grappler/optimizers/experimental_implementation_selector.cc
+++ b/tensorflow/core/grappler/optimizers/experimental_implementation_selector.cc
@@ -17,6 +17,8 @@ limitations under the License.
 
 #include <string>
 
+#include "absl/strings/numbers.h"
+#include "absl/strings/str_split.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -32,6 +34,73 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+Status UpdateNodeDef(NodeDef* node_def, const string& funcName,
+                     const FunctionApiInfo& apiInfo) {
+  VLOG(3) << "Node def before swap is: " << node_def->DebugString();
+  auto tin = node_def->mutable_attr()->find("Tin");
+  tin->second.mutable_list()->clear_type();
+  for (const auto& tin_dtype : apiInfo.input_arg_dtypes()) {
+    tin->second.mutable_list()->add_type(tin_dtype);
+  }
+
+  auto tout = node_def->mutable_attr()->find("Tout");
+  tout->second.mutable_list()->clear_type();
+  for (const auto& tout_dtype : apiInfo.output_arg_dtypes()) {
+    tout->second.mutable_list()->add_type(tout_dtype);
+  }
+
+  if (apiInfo.function_type() == FunctionApiInfo::BACKWARD) {
+    // Update the inputs since for backward function, it might have different
+    // number of inputs due the different number output from forward function.
+    // The output of forward function are composed by two parts:
+    //   1. Real output tensors from defun.
+    //   2. Internal states that will be used for gradient calculation.
+    // Part 1 will be static, and part 2 could be different based on the
+    // different implementation.
+
+    const int prev_input_size = node_def->input_size();
+    const int diff = prev_input_size - apiInfo.input_arg_dtypes().size();
+    if (diff >= 0) {
+      for (int i = 0; i < diff; ++i) node_def->mutable_input()->RemoveLast();
+    } else {
+      // Adding new inputs for internal states, the name of the internal states
+      // should be in format "{forward_node_name}:{index}", where the newly
+      // added index should start from last index of the state.
+      // Eg:
+      // {
+      //   input: "gradients/unified_lstm/strided_slice_1_grad/StridedSliceGrad"
+      //   input: "gradients/zeros_like_1"
+      //   input: "gradients/zeros_like_2"
+      //   input: "unified_lstm/StatefulPartitionedCall:3"
+      //   input: "unified_lstm/StatefulPartitionedCall:4"
+      //   # New input should be "unified_lstm/StatefulPartitionedCall:5"
+      // }
+      const string last_input = node_def->input(prev_input_size - 1);
+      const std::vector<string> name_index = ::absl::StrSplit(last_input, ':');
+      if (name_index.size() != 2) {
+        return errors::InvalidArgument(
+            "Invalid format of input node name: ", last_input,
+            " Expected: {forward_node_name}:{index}");
+      }
+      const absl::string_view node_name = name_index[0];
+      int last_index;
+      if (!::absl::SimpleAtoi(name_index[1], &last_index)) {
+        return errors::InvalidArgument(
+            "The index of input node is expected to be number, got: ",
+            name_index[1]);
+      }
+      for (int i = 1; i <= -diff; ++i)
+        node_def->add_input(strings::StrCat(node_name, ":", i + last_index));
+    }
+  }
+
+  node_def->mutable_attr()->find("f")->second.mutable_func()->set_name(
+      funcName);
+
+  VLOG(3) << "Node def after swap is: " << node_def->DebugString();
+  return Status::OK();
+}
+
 Status ExperimentalImplementationSelector::LoadFunctions(
     const GraphDef& graph) {
   lib_info_.reset(new FunctionLibraryApiInfo);
@@ -43,8 +112,11 @@ Status ExperimentalImplementationSelector::MaybeOptimizeFunctionCall(
     NodeDef* node_def) const {
   // There are two ways of calling functions:
   //  1. By specifying an op name as a function name, or
-  //  2. Via the @defun functional interface, where the real function name
-  //     appear as the attribute with type func.
+  //  2. Via the @defun functional interface, where the real function call
+  //     happens with partitionedcall op, and the function name appear as the
+  //     attribute with name "f" and type func. In this use case, there are more
+  //     attributes need to be taken care, like Tin and Tout which take care of
+  //     the DTYPE of input/output.
   std::vector<string> function_attribute_names;
   for (const auto& attr : node_def->attr()) {
     if (attr.second.has_func() &&
@@ -70,22 +142,29 @@ Status ExperimentalImplementationSelector::MaybeOptimizeFunctionCall(
 
   for (const auto& attr_name : function_attribute_names) {
     string function_name = node_def->attr().at(attr_name).func().name();
-    string best_function_name;
-    lib_info_->GetBestImplementation(function_name, parsed_name.type,
-                                     &best_function_name);
-    if (function_name != best_function_name) {
-      node_def->mutable_attr()
-          ->find(attr_name)
-          ->second.mutable_func()
-          ->set_name(best_function_name);
+    std::vector<string> equiv_func_names;
+    TF_RETURN_IF_ERROR(lib_info_->GetEquivalentImplementations(
+        function_name, &equiv_func_names));
+    for (const auto& func_name : equiv_func_names) {
+      const auto& func_api_info = lib_info_->GetApiInfo(func_name);
+      if (func_api_info->preferred_device() == parsed_name.type) {
+        VLOG(2) << "Swapping: " << function_name << " TO: " << func_name;
+        TF_RETURN_IF_ERROR(UpdateNodeDef(node_def, func_name, *func_api_info));
+        break;
+      }
     }
   }
+
   if (lib_info_->GetApiInfo(node_def->op()) != nullptr) {
-    string best_function_name;
-    lib_info_->GetBestImplementation(node_def->op(), parsed_name.type,
-                                     &best_function_name);
-    if (node_def->op() != best_function_name) {
-      node_def->set_op(best_function_name);
+    std::vector<string> equiv_func_names;
+    TF_RETURN_IF_ERROR(lib_info_->GetEquivalentImplementations(
+        node_def->op(), &equiv_func_names));
+    for (const string& func_name : equiv_func_names) {
+      const auto func_api_info = lib_info_->GetApiInfo(func_name);
+      if (func_api_info->preferred_device() == parsed_name.type) {
+        node_def->set_op(func_name);
+        break;
+      }
     }
   }
   return Status::OK();
@@ -93,6 +172,11 @@ Status ExperimentalImplementationSelector::MaybeOptimizeFunctionCall(
 
 Status ExperimentalImplementationSelector::SelectImplementation(
     GraphDef* graph) const {
+  if (!graph->has_library()) {
+    VLOG(2) << "Skipping graph since it does not have function def";
+    return Status::OK();
+  }
+
   for (int k = 0; k < graph->node_size(); ++k)
     TF_RETURN_IF_ERROR(MaybeOptimizeFunctionCall(graph->mutable_node(k)));
 
diff --git a/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc b/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc
index 3f1ebefac6..e1ac7766d3 100644
--- a/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc
+++ b/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc
@@ -133,6 +133,101 @@ TEST_F(ExperimentalImplementationSelectorTest, SwapImplementationEval) {
                                  test::AsScalar<float>(2.0f));
 }
 
+TEST_F(ExperimentalImplementationSelectorTest, SwapImplementationWithGradient) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+  // boost_1 returns the doubled input and a const as the internal state, the
+  // state will be feed to gradient function to mimic the behavior of backward
+  // function of defun that use internal states as extra inputs.
+  FunctionDef boost_1 = FDH::Create(
+      "Boost1", {"x:float"}, {"z:float", "s:float"}, {},
+      {{{"boost"}, "Add", {"x", "x"}, {{"T", DT_FLOAT}}},
+       FDH::Const("one", 1.0f)},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "boost:z:0"}, {"s", "one:output:0"}});
+  auto* boost_1_attr = boost_1.mutable_attr();
+  (*boost_1_attr)["experimental_api_implements"].set_s("random_boost");
+  (*boost_1_attr)["experimental_api_preferred_device"].set_s("CPU");
+  (*boost_1_attr)["backward_function_name"].set_s("BoostCpuGradient");
+
+  FunctionDef boost_1_gradient = FDH::Create(
+      "Boost1Gradient", {"x:float", "s:float"}, {"dx:float"}, {},
+      {FDH::Const("two", 2.0f),
+       {{"grad"}, "Mul", {"x", "two:output:0"}, {{"T", DT_FLOAT}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"dx", "grad:z:0"}});
+  auto* boost_1_grad_attr = boost_1_gradient.mutable_attr();
+  (*boost_1_grad_attr)["experimental_api_implements"].set_s("random_boost");
+  (*boost_1_grad_attr)["experimental_api_preferred_device"].set_s("CPU");
+  (*boost_1_grad_attr)["forward_function_name"].set_s("BoostCpu");
+
+  // boost_2 return the input * 4, and with two extra internal states.
+  FunctionDef boost_2_func = FDH::Create(
+      "Boost2", {"x:float"}, {"z:float", "s1:float", "s2:float"}, {},
+      {FDH::Const("four", 4.0f),
+       {{"boost"}, "Mul", {"x", "four:output:0"}, {{"T", DT_FLOAT}}},
+       FDH::Const("one", 1.0f),
+       FDH::Const("two", 2.0f)},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "boost:z:0"}, {"s1", "one:output:0"}, {"s2", "two:output:0"}});
+  auto* boost_2_attr = boost_2_func.mutable_attr();
+  (*boost_2_attr)["experimental_api_implements"].set_s("random_boost");
+  (*boost_2_attr)["experimental_api_preferred_device"].set_s("GPU");
+  (*boost_2_attr)["backward_function_name"].set_s("BoostGpuGradient");
+
+  FunctionDef boost_2_gradient = FDH::Create(
+      "Boost2Gradient", {"x:float", "s1:float", "s2:float"}, {"dx:float"}, {},
+      {FDH::Const("four", 4.0f),
+       {{"grad"}, "Mul", {"x", "four:output:0"}, {{"T", DT_FLOAT}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"dx", "grad:z:0"}});
+  auto* boost_2_grad_attr = boost_2_gradient.mutable_attr();
+  (*boost_2_grad_attr)["experimental_api_implements"].set_s("random_boost");
+  (*boost_2_grad_attr)["experimental_api_preferred_device"].set_s("GPU");
+  (*boost_2_grad_attr)["forward_function_name"].set_s("BoostGpu");
+
+  // Define the forward function with f = boost2 function but with CPU device.
+  // Expect the grappler plugin to swap f and attributes to use the boost1.
+  const auto forward =
+      NDef("lstm/StatefulPartitionedCall", "StatefulPartitionedCall", {"input"},
+           {{"Tin", DataTypeSlice{DT_FLOAT}},
+            {"Tout", DataTypeSlice{DT_FLOAT, DT_FLOAT, DT_FLOAT}},
+            {"f", FDH::FunctionRef("Boost2")}},
+           CpuDevice);
+  const auto backward =
+      NDef("gradient/lstm/StatefulPartitionedCall", "StatefulPartitionedCall",
+           {"input", "lstm/StatefulPartitionedCall:1",
+            "lstm/StatefulPartitionedCall:2"},
+           {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT, DT_FLOAT}},
+            {"Tout", DataTypeSlice{DT_FLOAT}},
+            {"f", FDH::FunctionRef("Boost2Gradient")}},
+           CpuDevice);
+
+  ExperimentalImplementationSelector optimizer;
+  GraphDef output;
+  GrapplerItem item;
+  item.graph = test::function::GDef(
+      {NDef("input", "Placeholder", {}, {{"dtype", DT_FLOAT}}, CpuDevice),
+       forward, backward,
+       NDef("output", "Identity", {"lstm/StatefulPartitionedCall:0"},
+            {{"T", DT_FLOAT}}, CpuDevice)},
+      // FunctionLib
+      {boost_1, boost_1_gradient, boost_2_func, boost_2_gradient});
+
+  const Tensor input = test::AsScalar<float>(1.0f);
+  item.fetch = {"output"};
+  item.feed.emplace_back("input", input);
+
+  const auto four_times_boosted_tensor = EvaluateFetchNodes(item);
+  test::ExpectTensorEqual<float>(four_times_boosted_tensor[0],
+                                 test::AsScalar<float>(4.0f));
+
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+  GrapplerItem optimized(item, std::move(output));
+  const auto twice_boosted_tensor = EvaluateFetchNodes(optimized);
+  test::ExpectTensorEqual<float>(twice_boosted_tensor[0],
+                                 test::AsScalar<float>(2.0f));
+}
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/function_api_info.cc b/tensorflow/core/grappler/optimizers/function_api_info.cc
index 798e0f6fd5..497ad6032e 100644
--- a/tensorflow/core/grappler/optimizers/function_api_info.cc
+++ b/tensorflow/core/grappler/optimizers/function_api_info.cc
@@ -27,6 +27,7 @@ FunctionApiInfo::FunctionApiInfo() {}
 FunctionApiInfo::~FunctionApiInfo() {}
 
 Status FunctionApiInfo::Init(const FunctionDef& function_def) {
+  function_type_ = FunctionApiInfo::FunctionType::INFERENCE;
   for (const auto& attr : function_def.attr()) {
     if (attr.first == "experimental_api_preferred_device") {
       preferred_device_ = attr.second.s();
@@ -34,7 +35,25 @@ Status FunctionApiInfo::Init(const FunctionDef& function_def) {
     if (attr.first == "experimental_api_implements") {
       interface_name_ = attr.second.s();
     }
+    if (attr.first == "forward_function_name") {
+      function_type_ = FunctionApiInfo::FunctionType::BACKWARD;
+      pairing_function_name_ = attr.second.s();
+    }
+    if (attr.first == "backward_function_name") {
+      function_type_ = FunctionApiInfo::FunctionType::FORWARD;
+      pairing_function_name_ = attr.second.s();
+    }
+  }
+
+  input_arg_dtypes_.reserve(function_def.signature().input_arg_size());
+  for (const auto& input_arg : function_def.signature().input_arg()) {
+    input_arg_dtypes_.emplace_back(input_arg.type());
   }
+  output_arg_dtypes_.reserve(function_def.signature().output_arg_size());
+  for (const auto& output_arg : function_def.signature().output_arg()) {
+    output_arg_dtypes_.emplace_back(output_arg.type());
+  }
+
   if (interface_name_.empty() && !preferred_device_.empty()) {
     return errors::InvalidArgument(
         "Function '", function_def.signature().name(),
@@ -51,53 +70,94 @@ const string& FunctionApiInfo::interface_name() const {
   return interface_name_;
 }
 
+const FunctionApiInfo::FunctionType FunctionApiInfo::function_type() const {
+  return function_type_;
+}
+
+const string& FunctionApiInfo::pairing_function_name() const {
+  return pairing_function_name_;
+}
+
+const DataTypeVector& FunctionApiInfo::input_arg_dtypes() const {
+  return input_arg_dtypes_;
+}
+
+const DataTypeVector& FunctionApiInfo::output_arg_dtypes() const {
+  return output_arg_dtypes_;
+}
+
 FunctionLibraryApiInfo::FunctionLibraryApiInfo() {}
 FunctionLibraryApiInfo::~FunctionLibraryApiInfo() {}
 
 namespace {
-bool IsSameSignature(const FunctionDef& f1, const FunctionDef& f2) {
-  if (f1.ret().size() != f2.ret().size()) return false;
+bool IsSameArgDef(const OpDef::ArgDef& arg1, const OpDef::ArgDef& arg2) {
+  if (arg1.type() != arg2.type()) return false;
+  if (arg1.type_attr() != arg2.type_attr()) return false;
+  if (arg1.number_attr() != arg2.number_attr()) return false;
+  if (arg1.type_list_attr() != arg2.type_list_attr()) return false;
+  if (arg1.is_ref() != arg2.is_ref()) return false;
+  return true;
+}
+
+bool IsSameSignature(const FunctionDef& f1, const FunctionDef& f2,
+                     const bool check_inputs, const bool check_outputs) {
   const auto& sig1 = f1.signature();
   const auto& sig2 = f2.signature();
   // Functions have positional semantics, so we don't check for names.
-  if (sig1.input_arg_size() != sig2.input_arg_size()) return false;
-  for (int k = 0; k < sig1.input_arg_size(); ++k) {
-    const OpDef::ArgDef& arg1 = sig1.input_arg(k);
-    const OpDef::ArgDef& arg2 = sig2.input_arg(k);
-    if (arg1.type() != arg2.type()) return false;
-    if (arg1.type_attr() != arg2.type_attr()) return false;
-    if (arg1.number_attr() != arg2.number_attr()) return false;
-    if (arg1.type_list_attr() != arg2.type_list_attr()) return false;
-    if (arg1.is_ref() != arg2.is_ref()) return false;
+  if (check_inputs) {
+    if (sig1.input_arg_size() != sig2.input_arg_size()) return false;
+    for (int k = 0; k < sig1.input_arg_size(); ++k) {
+      if (!IsSameArgDef(sig1.input_arg(k), sig2.input_arg(k))) return false;
+    }
+  }
+  if (check_outputs) {
+    if (f1.ret().size() != f2.ret().size()) return false;
+    if (sig1.output_arg_size() != sig2.output_arg_size()) return false;
+    for (int k = 0; k < sig1.output_arg_size(); ++k) {
+      if (!IsSameArgDef(sig1.output_arg(k), sig2.output_arg(k))) return false;
+    }
   }
   return true;
 }
 
 Status ValidateSignature(const string& interface_name,
-                         const std::vector<const FunctionDef*>& equiv_funcs) {
+                         const std::vector<const FunctionDef*>& equiv_funcs,
+                         const FunctionApiInfo::FunctionType function_type) {
   if (equiv_funcs.size() < 2) return Status::OK();
   for (size_t k = 1; k < equiv_funcs.size(); ++k) {
-    if (!IsSameSignature(*equiv_funcs[0], *equiv_funcs[k]))
+    const bool check_input =
+        (function_type == FunctionApiInfo::FunctionType::INFERENCE ||
+         function_type == FunctionApiInfo::FunctionType::FORWARD);
+    const bool check_output =
+        (function_type == FunctionApiInfo::FunctionType::INFERENCE ||
+         function_type == FunctionApiInfo::FunctionType::BACKWARD);
+    if (!IsSameSignature(*equiv_funcs[0], *equiv_funcs[k], check_input,
+                         check_output)) {
       return errors::InvalidArgument(
           "Functions '", equiv_funcs[0]->signature().name(), "' and '",
           equiv_funcs[k]->signature().name(), "' both implement '",
           interface_name, "' but their signatures do not match.");
+    }
   }
   return Status::OK();
 }
 
 Status ValidateSignatures(
     const std::unordered_map<string, std::vector<const FunctionDef*>>&
-        intf_to_func) {
+        intf_to_func,
+    const FunctionApiInfo::FunctionType function_type) {
   for (const auto& item : intf_to_func)
-    TF_RETURN_IF_ERROR(ValidateSignature(item.first, item.second));
+    TF_RETURN_IF_ERROR(
+        ValidateSignature(item.first, item.second, function_type));
   return Status::OK();
 }
 }  // namespace
 
 Status FunctionLibraryApiInfo::Init(
     const FunctionDefLibrary& function_library) {
-  std::unordered_map<string, std::vector<const FunctionDef*>> intf_to_func;
+  std::unordered_map<string, std::vector<const FunctionDef*>> infer_funcs;
+  std::unordered_map<string, std::vector<const FunctionDef*>> fwd_funcs;
+  std::unordered_map<string, std::vector<const FunctionDef*>> bwd_funcs;
   for (const auto& function : function_library.function()) {
     std::unique_ptr<FunctionApiInfo> func_info(new FunctionApiInfo);
     TF_RETURN_IF_ERROR(func_info->Init(function));
@@ -106,54 +166,64 @@ Status FunctionLibraryApiInfo::Init(
 
     const string& function_name = function.signature().name();
     const string& interface_name = func_info->interface_name();
-    func_to_intf_[function_name] = interface_name;
-    intf_to_funcs_[interface_name].emplace_back(function_name);
-    intf_to_func[interface_name].emplace_back(&function);
+    VLOG(3) << "Got " << func_info->function_type()
+            << " function: " << function_name
+            << " with interface: " << interface_name;
+    switch (func_info->function_type()) {
+      case FunctionApiInfo::FunctionType::INFERENCE:
+        intf_to_inference_funcs_[interface_name].emplace_back(function_name);
+        infer_funcs[interface_name].emplace_back(&function);
+        break;
+      case FunctionApiInfo::FunctionType::FORWARD:
+        intf_to_forward_funcs_[interface_name].emplace_back(function_name);
+        fwd_funcs[interface_name].emplace_back(&function);
+        break;
+      case FunctionApiInfo::FunctionType::BACKWARD:
+        intf_to_backward_funcs_[interface_name].emplace_back(function_name);
+        bwd_funcs[interface_name].emplace_back(&function);
+        break;
+      default:
+        return errors::InvalidArgument("Unrecognized function type: ",
+                                       func_info->function_type());
+    }
     func_info_[function_name] = std::move(func_info);
   }
-  TF_RETURN_IF_ERROR(ValidateSignatures(intf_to_func));
+  TF_RETURN_IF_ERROR(ValidateSignatures(
+      infer_funcs, FunctionApiInfo::FunctionType::INFERENCE));
+  TF_RETURN_IF_ERROR(
+      ValidateSignatures(fwd_funcs, FunctionApiInfo::FunctionType::FORWARD));
+  TF_RETURN_IF_ERROR(
+      ValidateSignatures(bwd_funcs, FunctionApiInfo::FunctionType::BACKWARD));
   return Status::OK();
 }
 
-void FunctionLibraryApiInfo::GetEquivalentImplementations(
-    const string& function_name, std::vector<string>* other_names) const {
-  const auto intf_it = func_to_intf_.find(function_name);
-  // The function does not implement any interface.
-  if (intf_it == func_to_intf_.end()) return;
-  CHECK(!intf_it->second.empty()) << "Function " << function_name
-                                  << "should at least implement 1 interface.";
-  const auto it = intf_to_funcs_.find(intf_it->second);
-  CHECK(it != intf_to_funcs_.end())
-      << "Function " << function_name << " maps to " << intf_it->second
-      << " but no reverse mapping was found";
-  CHECK_GE(it->second.size(), 1) << "Class " << it->first << " is empty";
-  other_names->reserve(it->second.size() - 1);
-  for (const auto& other_name : it->second) {
-    if (other_name == function_name) continue;
-    other_names->emplace_back(other_name);
+Status FunctionLibraryApiInfo::GetEquivalentImplementations(
+    const string& function_name, std::vector<string>* other_functions) const {
+  const auto func_it = func_info_.find(function_name);
+  if (func_it == func_info_.end()) return Status::OK();
+  const FunctionApiInfo* func_info = func_it->second.get();
+
+  absl::flat_hash_map<string, std::vector<string>>::const_iterator it;
+  switch (func_info->function_type()) {
+    case FunctionApiInfo::FunctionType::INFERENCE:
+      it = intf_to_inference_funcs_.find(func_info->interface_name());
+      break;
+    case FunctionApiInfo::FunctionType::FORWARD:
+      it = intf_to_forward_funcs_.find(func_info->interface_name());
+      break;
+    case FunctionApiInfo::FunctionType::BACKWARD:
+      it = intf_to_backward_funcs_.find(func_info->interface_name());
+      break;
+    default:
+      return errors::InvalidArgument("Unrecognized function type: ",
+                                     func_info->function_type());
   }
-}
 
-void FunctionLibraryApiInfo::GetBestImplementation(
-    const string& function_name, const string& device,
-    string* best_func_name) const {
-  CHECK(best_func_name != nullptr);
-  const auto func_it = func_to_intf_.find(function_name);
-  if (func_it == func_to_intf_.end()) return;
-
-  const auto it = intf_to_funcs_.find(func_it->second);
-  // No function found for the given interface.
-  if (it == intf_to_funcs_.end()) return;
   for (const auto& func_name : it->second) {
-    const auto func_api_info = func_info_.find(func_name)->second.get();
-    if (func_api_info->preferred_device() == device) {
-      best_func_name->assign(func_name);
-      return;
-    }
+    if (func_name == function_name) continue;
+    other_functions->emplace_back(func_name);
   }
-  // Didn't find a function with the match device name, choose the first one
-  // among all the available functions.
-  best_func_name->assign(it->second.front());
+  return Status::OK();
 }
 
 const FunctionApiInfo* FunctionLibraryApiInfo::GetApiInfo(
diff --git a/tensorflow/core/grappler/optimizers/function_api_info.h b/tensorflow/core/grappler/optimizers/function_api_info.h
index 412687c58c..9a5f548951 100644
--- a/tensorflow/core/grappler/optimizers/function_api_info.h
+++ b/tensorflow/core/grappler/optimizers/function_api_info.h
@@ -20,7 +20,10 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/framework/function.pb.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
@@ -30,14 +33,32 @@ class FunctionApiInfo {
   FunctionApiInfo();
   virtual ~FunctionApiInfo();
 
+  enum FunctionType {
+    INFERENCE,  // Default type.
+    FORWARD,
+    BACKWARD,
+  };
+
   Status Init(const FunctionDef& function_def);
 
   const string& interface_name() const;
   const string& preferred_device() const;
+  const FunctionType function_type() const;
+  const string& pairing_function_name() const;
+  const DataTypeVector& input_arg_dtypes() const;
+  const DataTypeVector& output_arg_dtypes() const;
 
  private:
   string interface_name_;
   string preferred_device_;
+  FunctionType function_type_;
+  // The pairing function is used to pair between forward and backward function,
+  // which will be useful during function swapping. Inference function won't
+  // have pairing function.
+  string pairing_function_name_;
+  // The following two attributes are useful for forward and backward functions.
+  DataTypeVector input_arg_dtypes_;
+  DataTypeVector output_arg_dtypes_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(FunctionApiInfo);
 };
@@ -55,21 +76,22 @@ class FunctionLibraryApiInfo {
   // Populate the internal field for the functions within the function_library.
   Status Init(const FunctionDefLibrary& function_library);
 
-  void GetEquivalentImplementations(const string& function_name,
-                                    std::vector<string>* other_names) const;
-
-  void GetBestImplementation(const string& function_name, const string& device,
-                             string* best_func_name) const;
+  Status GetEquivalentImplementations(
+      const string& function_name, std::vector<string>* other_functions) const;
 
   const FunctionApiInfo* GetApiInfo(const string& function_name) const;
 
  private:
   // Map between function name to function details.
   std::unordered_map<string, std::unique_ptr<FunctionApiInfo>> func_info_;
-  // Map between function name to interface name.
-  std::unordered_map<string, string> func_to_intf_;
+
   // Map between interface name to function names.
-  std::unordered_map<string, std::vector<string>> intf_to_funcs_;
+  // Forward/backward function pair usually have different signatures between
+  // each other since forward function could produce extra internal state as
+  // output, and backward will take those extra state as inputs.
+  absl::flat_hash_map<string, std::vector<string>> intf_to_inference_funcs_;
+  absl::flat_hash_map<string, std::vector<string>> intf_to_forward_funcs_;
+  absl::flat_hash_map<string, std::vector<string>> intf_to_backward_funcs_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(FunctionLibraryApiInfo);
 };
diff --git a/tensorflow/core/grappler/optimizers/function_api_info_test.cc b/tensorflow/core/grappler/optimizers/function_api_info_test.cc
index 582890d3e3..b683d26b32 100644
--- a/tensorflow/core/grappler/optimizers/function_api_info_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_api_info_test.cc
@@ -36,28 +36,35 @@ void SetArg(const string& name, const string& type_name,
 
 typedef std::pair<string, string> ArgSpec;  // name, type.
 
-void SetArgs(const std::vector<ArgSpec>& args_spec, OpDef* sig) {
-  for (const auto& arg_spec : args_spec)
+void SetArgs(const std::vector<ArgSpec>& input_args_spec,
+             const std::vector<ArgSpec>& output_args_spec, OpDef* sig) {
+  for (const auto& arg_spec : input_args_spec)
     SetArg(arg_spec.first, arg_spec.second, sig->add_input_arg());
-  SetArg("output", "float32", sig->add_output_arg());
+  for (const auto& arg_spec : output_args_spec)
+    SetArg(arg_spec.first, arg_spec.second, sig->add_output_arg());
 }
 
 void PopulateFunction(const string& name, const string& api_interface_name,
                       const string& preferred_device,
                       const std::vector<ArgSpec>& input_args,
+                      const std::vector<ArgSpec>& output_args,
+                      const string& forward_function_name,
+                      const string& backward_function_name,
                       FunctionDef* func_def) {
   OpDef* sig = func_def->mutable_signature();
   sig->set_name(name);
 
-  SetArgs(input_args, sig);
-
-  if (!api_interface_name.empty() || !preferred_device.empty()) {
-    auto* func_attr = func_def->mutable_attr();
-    if (!api_interface_name.empty())
-      (*func_attr)["experimental_api_implements"].set_s(api_interface_name);
-    if (!preferred_device.empty())
-      (*func_attr)["experimental_api_preferred_device"].set_s(preferred_device);
-  }
+  SetArgs(input_args, output_args, sig);
+
+  auto* func_attr = func_def->mutable_attr();
+  if (!api_interface_name.empty())
+    (*func_attr)["experimental_api_implements"].set_s(api_interface_name);
+  if (!preferred_device.empty())
+    (*func_attr)["experimental_api_preferred_device"].set_s(preferred_device);
+  if (!forward_function_name.empty())
+    (*func_attr)["forward_function_name"].set_s(forward_function_name);
+  if (!backward_function_name.empty())
+    (*func_attr)["backward_function_name"].set_s(backward_function_name);
 }
 
 void PopulateSampleLibrary(const bool mismatch_args,
@@ -65,39 +72,50 @@ void PopulateSampleLibrary(const bool mismatch_args,
   const std::vector<ArgSpec> func_args{{"in1", "float32"}, {"in2", "int32"}};
   const std::vector<ArgSpec> func_wrong_args{{"in1", "int32"},
                                              {"in2", "int32"}};
-  PopulateFunction("DoStuffCpu", "DoStuff", "CPU", func_args,
-                   func_lib->add_function());
+  const std::vector<ArgSpec> output_args{{"out", "float32"}};
+  PopulateFunction("DoStuffCpu", "DoStuff", "CPU", func_args, output_args, "",
+                   "", func_lib->add_function());
   PopulateFunction("DoStuffGpu", "DoStuff", "GPU",
-                   mismatch_args ? func_wrong_args : func_args,
+                   mismatch_args ? func_wrong_args : func_args, output_args, "",
+                   "", func_lib->add_function());
+  PopulateFunction("DoThings", "DoThings", "", func_args, output_args, "", "",
                    func_lib->add_function());
-  PopulateFunction("DoThings", "DoThings", "", func_args,
+  PopulateFunction("OneOff", "", "", func_args, output_args, "", "",
                    func_lib->add_function());
-  PopulateFunction("OneOff", "", "", func_args, func_lib->add_function());
-  PopulateFunction("AnotherOneOff", "", "", func_args,
+  PopulateFunction("AnotherOneOff", "", "", func_args, output_args, "", "",
                    func_lib->add_function());
 }
 
+void PopulateComplexLibrary(FunctionDefLibrary* func_lib) {
+  const std::vector<ArgSpec> input_args{{"in1", "float32"}, {"in2", "int32"}};
+  const std::vector<ArgSpec> output_args{{"out", "float32"}};
+  const std::vector<ArgSpec> output_with_state{
+      {"out", "float32"}, {"state1", "int32"}, {"state2", "int32"}};
+
+  PopulateFunction("DoStuffCpu", "DoStuff", "CPU", input_args, output_args, "",
+                   "DoStuffCpu_gradient", func_lib->add_function());
+  PopulateFunction("DoStuffCpu_gradient", "DoStuff", "CPU", output_args,
+                   input_args, "DoStuffCpu", "", func_lib->add_function());
+  PopulateFunction("DoStuffGpu", "DoStuff", "GPU", input_args,
+                   output_with_state, "", "DoStuffGpu_gradient",
+                   func_lib->add_function());
+  PopulateFunction("DoStuffGpu_gradient", "DoStuff", "GPU", output_with_state,
+                   input_args, "DoStuffGpu", "", func_lib->add_function());
+}
+
 bool CheckEquivImpl(const FunctionLibraryApiInfo& lib_api_info,
                     const string& func_name,
                     const std::vector<string>& expected_other) {
   std::vector<string> other_impl;
-  lib_api_info.GetEquivalentImplementations(func_name, &other_impl);
+  Status status =
+      lib_api_info.GetEquivalentImplementations(func_name, &other_impl);
+  EXPECT_EQ(status, Status::OK());
   const std::unordered_set<string> actual(other_impl.begin(), other_impl.end());
   const std::unordered_set<string> expected(expected_other.begin(),
                                             expected_other.end());
   return actual == expected;
 }
 
-bool CheckGetBestImpl(const FunctionLibraryApiInfo& lib_api_info,
-                      const string& function_name, const string& device,
-                      const string& expected_function_name) {
-  string best_function_name;
-  lib_api_info.GetBestImplementation(function_name, device,
-                                     &best_function_name);
-
-  return best_function_name == expected_function_name;
-}
-
 string GetInterfaceName(const FunctionLibraryApiInfo& lib_api_info,
                         const string& func_name) {
   auto* info = lib_api_info.GetApiInfo(func_name);
@@ -117,34 +135,46 @@ TEST(FunctionApiInfoTest, ParseTags) {
   PopulateSampleLibrary(/* mismatch_args */ false, &func_lib);
   FunctionLibraryApiInfo lib_api_info;
   TF_ASSERT_OK(lib_api_info.Init(func_lib));
+
+  EXPECT_EQ("DoStuff", GetInterfaceName(lib_api_info, "DoStuffCpu"));
+  EXPECT_EQ("DoStuff", GetInterfaceName(lib_api_info, "DoStuffGpu"));
+  EXPECT_EQ("DoThings", GetInterfaceName(lib_api_info, "DoThings"));
+
+  EXPECT_EQ("CPU", GetPreferredDevice(lib_api_info, "DoStuffCpu"));
+  EXPECT_EQ("GPU", GetPreferredDevice(lib_api_info, "DoStuffGpu"));
+  EXPECT_EQ("", GetPreferredDevice(lib_api_info, "DoThings"));
+
   EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoStuffCpu", {"DoStuffGpu"}));
   EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoStuffGpu", {"DoStuffCpu"}));
   EXPECT_TRUE(CheckEquivImpl(lib_api_info, "Undefined", {}));
   EXPECT_TRUE(CheckEquivImpl(lib_api_info, "OneOff", {}));
   EXPECT_TRUE(CheckEquivImpl(lib_api_info, "AnotherOneOff", {}));
   EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoThings", {}));
+}
+
+TEST(FunctionApiInfoTest, ComplexFunctionLib) {
+  FunctionDefLibrary func_lib;
+  PopulateComplexLibrary(&func_lib);
+  FunctionLibraryApiInfo lib_api_info;
+  TF_ASSERT_OK(lib_api_info.Init(func_lib));
 
   EXPECT_EQ("DoStuff", GetInterfaceName(lib_api_info, "DoStuffCpu"));
+  EXPECT_EQ("DoStuff", GetInterfaceName(lib_api_info, "DoStuffCpu_gradient"));
   EXPECT_EQ("DoStuff", GetInterfaceName(lib_api_info, "DoStuffGpu"));
-  EXPECT_EQ("DoThings", GetInterfaceName(lib_api_info, "DoThings"));
+  EXPECT_EQ("DoStuff", GetInterfaceName(lib_api_info, "DoStuffGpu_gradient"));
 
   EXPECT_EQ("CPU", GetPreferredDevice(lib_api_info, "DoStuffCpu"));
+  EXPECT_EQ("CPU", GetPreferredDevice(lib_api_info, "DoStuffCpu_gradient"));
   EXPECT_EQ("GPU", GetPreferredDevice(lib_api_info, "DoStuffGpu"));
-  EXPECT_EQ("", GetPreferredDevice(lib_api_info, "DoThings"));
+  EXPECT_EQ("GPU", GetPreferredDevice(lib_api_info, "DoStuffGpu_gradient"));
 
-  EXPECT_TRUE(
-      CheckGetBestImpl(lib_api_info, "DoStuffCpu", "CPU", "DoStuffCpu"));
-  EXPECT_TRUE(
-      CheckGetBestImpl(lib_api_info, "DoStuffCpu", "GPU", "DoStuffGpu"));
-  EXPECT_TRUE(
-      CheckGetBestImpl(lib_api_info, "DoStuffGpu", "CPU", "DoStuffCpu"));
-  EXPECT_TRUE(
-      CheckGetBestImpl(lib_api_info, "DoStuffGpu", "GPU", "DoStuffGpu"));
-
-  EXPECT_TRUE(CheckGetBestImpl(lib_api_info, "DoThings", "GPU", "DoThings"));
-  // TPU impl is not available, choose the first one available which is the CPU.
-  EXPECT_TRUE(
-      CheckGetBestImpl(lib_api_info, "DoStuffGpu", "TPU", "DoStuffCpu"));
+  EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoStuffCpu", {"DoStuffGpu"}));
+  EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoStuffGpu", {"DoStuffCpu"}));
+  EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoStuffCpu_gradient",
+                             {"DoStuffGpu_gradient"}));
+  EXPECT_TRUE(CheckEquivImpl(lib_api_info, "DoStuffGpu_gradient",
+                             {"DoStuffCpu_gradient"}));
+  EXPECT_TRUE(CheckEquivImpl(lib_api_info, "Undefined", {}));
 }
 
 TEST(FunctionApiInfoTest, MismatchedArguments) {
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 365ae4746d..510fcc04b4 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -508,6 +508,17 @@ py_test(
     ],
 )
 
+cuda_py_test(
+    name = "unified_rnn_test",
+    size = "medium",
+    srcs = ["layers/unified_rnn_test.py"],
+    additional_deps = [
+        ":keras",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_test(
     name = "serialization_test",
     size = "small",
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index c67b7a5b0b..8ab8f2ae04 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3570,12 +3570,12 @@ def rnn(step_function,
           **while_loop_kwargs)
       new_states = final_outputs[2:]
 
-    last_time = final_outputs[0]
     output_ta = final_outputs[1]
 
     outputs = tuple(o.stack() for o in output_ta)
+    last_output = tuple(o[-1] for o in outputs)
+
     outputs = nest.pack_sequence_as(output_time_zero, outputs)
-    last_output = tuple(o.read(last_time - 1) for o in output_ta)
     last_output = nest.pack_sequence_as(output_time_zero, last_output)
 
   # static shape inference
diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
new file mode 100644
index 0000000000..015a079a0d
--- /dev/null
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -0,0 +1,469 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for UnifiedLSTM layer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python import keras
+from tensorflow.python.client import session
+from tensorflow.python.eager import function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras import activations
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import constraints
+from tensorflow.python.keras import initializers
+from tensorflow.python.keras import regularizers
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.engine.base_layer import \
+  InputSpec
+from tensorflow.python.keras.layers.recurrent import RNN
+from tensorflow.python.keras.utils import tf_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_cudnn_rnn_ops
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.losses import losses
+from tensorflow.python.platform import test
+from tensorflow.python.training import gradient_descent
+
+
+class RNNTest(test.TestCase):
+
+  def test_unifiedRNN(self):
+    rewrites = rewriter_config_pb2.RewriterConfig()
+    rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
+    customer_optimizer = rewrites.custom_optimizers.add()
+    customer_optimizer.name = 'ExperimentalImplementationSelector'
+    rewrites.min_graph_nodes = -1
+    graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
+    config = config_pb2.ConfigProto(graph_options=graph_options)
+
+    input_shape = 10
+    rnn_state_size = 8
+    output_shape = 8
+    timestep = 4
+    batch = 100
+    epoch = 1
+
+    with ops.Graph().as_default(), session.Session(config=config) as sess:
+      (x_train, y_train), _ = testing_utils.get_test_data(
+          train_samples=batch,
+          test_samples=0,
+          input_shape=(timestep, input_shape),
+          num_classes=output_shape)
+      y_train = keras.utils.to_categorical(y_train)
+
+      layer = UnifiedLSTM(rnn_state_size)
+
+      inputs = array_ops.placeholder(
+          dtypes.float32, shape=(None, timestep, input_shape), name='inputs')
+      predict = array_ops.placeholder(
+          dtypes.float32, shape=(None, output_shape), name='predict')
+
+      outputs, runtime = layer(inputs)
+      loss = losses.softmax_cross_entropy(predict, outputs)
+      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+      train_op = optimizer.minimize(loss)
+
+      sess.run([variables.global_variables_initializer()])
+      existing_loss = 0
+      for _ in range(epoch):
+        loss_value, _, runtime_value = sess.run([loss, train_op, runtime], {
+            inputs: x_train,
+            predict: y_train
+        })
+        if test.is_gpu_available():
+          self.assertEquals(runtime_value, b'cudnn')
+        else:
+          self.assertEquals(runtime_value, b'cpu')
+        # Make sure the loss is updated for every epoch
+        # (layer weights properly updated).
+        self.assertNotEqual(existing_loss, loss_value)
+        existing_loss = loss_value
+
+  def test_unifiedRNN_with_cond(self):
+    # This test is to demonstrate the graph rewrite of grappler plugin under
+    # the condition that the function returns different number of internal
+    # states.
+    rewrites = rewriter_config_pb2.RewriterConfig()
+    rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
+    customer_optimizer = rewrites.custom_optimizers.add()
+    customer_optimizer.name = 'ExperimentalImplementationSelector'
+    rewrites.min_graph_nodes = -1
+    graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
+    config = config_pb2.ConfigProto(graph_options=graph_options)
+
+    input_shape = 10
+    rnn_state_size = 8
+    output_shape = 8
+    timestep = 4
+    batch = 100
+    epoch = 1
+
+    with ops.Graph().as_default(), session.Session(config=config) as sess:
+      (x_train, y_train), _ = testing_utils.get_test_data(
+          train_samples=batch,
+          test_samples=0,
+          input_shape=(timestep, input_shape),
+          num_classes=output_shape)
+      y_train = keras.utils.to_categorical(y_train)
+
+      layer = UnifiedLSTM(rnn_state_size)
+
+      inputs = array_ops.placeholder(
+          dtypes.float32, shape=(None, timestep, input_shape), name='inputs')
+      predict = array_ops.placeholder(
+          dtypes.float32, shape=(None, output_shape), name='predict')
+
+      zeros = array_ops.zeros([batch, output_shape])
+      dummy_runtime = constant_op.constant(
+          'unknown', dtype=dtypes.string, name='runtime')
+      a = constant_op.constant(0)
+      b = constant_op.constant(1)
+      # Will always run the lstm layer.
+      outputs, runtime = control_flow_ops.cond(
+          gen_math_ops.less(a, b),
+          lambda: layer(inputs),
+          lambda: (zeros, dummy_runtime))
+      loss = losses.softmax_cross_entropy(predict, outputs)
+      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+      train_op = optimizer.minimize(loss)
+
+      sess.run([variables.global_variables_initializer()])
+      existing_loss = 0
+
+      for _ in range(epoch):
+        loss_value, _, runtime_value = sess.run([loss, train_op, runtime], {
+            inputs: x_train,
+            predict: y_train
+        })
+        if test.is_gpu_available():
+          self.assertEquals(runtime_value, b'cudnn')
+        else:
+          self.assertEquals(runtime_value, b'cpu')
+        # Make sure the loss is updated for every epoch
+        # (layer weights properly updated).
+        self.assertNotEqual(existing_loss, loss_value)
+        existing_loss = loss_value
+
+
+class UnifiedLSTM(RNN):
+
+  def __init__(self,
+               units,
+               kernel_initializer='glorot_uniform',
+               recurrent_initializer='orthogonal',
+               bias_initializer='zeros',
+               unit_forget_bias=True,
+               kernel_regularizer=None,
+               recurrent_regularizer=None,
+               bias_regularizer=None,
+               activity_regularizer=None,
+               kernel_constraint=None,
+               recurrent_constraint=None,
+               bias_constraint=None,
+               return_sequences=False,
+               return_state=False,
+               go_backwards=False,
+               stateful=False,
+               time_major=False,
+               **kwargs):
+    super(RNN, self).__init__(**kwargs)  # pylint: disable=bad-super-call
+    self.units = units
+    cell_spec = collections.namedtuple('cell', ['state_size', 'output_size'])
+    self.cell = cell_spec(
+        state_size=(self.units, self.units), output_size=self.units)
+
+    self.kernel_initializer = initializers.get(kernel_initializer)
+    self.recurrent_initializer = initializers.get(recurrent_initializer)
+    self.bias_initializer = initializers.get(bias_initializer)
+    self.unit_forget_bias = unit_forget_bias
+
+    self.kernel_regularizer = regularizers.get(kernel_regularizer)
+    self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
+    self.bias_regularizer = regularizers.get(bias_regularizer)
+    self.activity_regularizer = regularizers.get(activity_regularizer)
+
+    self.kernel_constraint = constraints.get(kernel_constraint)
+    self.recurrent_constraint = constraints.get(recurrent_constraint)
+    self.bias_constraint = constraints.get(bias_constraint)
+
+    self.return_sequences = return_sequences
+    self.return_state = return_state
+    self.go_backwards = go_backwards
+    self.stateful = stateful
+    self.time_major = time_major
+    self._num_constants = None
+    self._num_inputs = None
+    self._states = None
+    self.input_spec = [InputSpec(ndim=3)]
+    self.state_spec = [
+        InputSpec(shape=(None, dim)) for dim in (self.units, self.units)
+    ]
+
+  @tf_utils.shape_type_conversion
+  def build(self, input_shape):
+    super(UnifiedLSTM, self).build(input_shape)
+    if isinstance(input_shape, list):
+      input_shape = input_shape[0]
+    input_dim = int(input_shape[-1])
+
+    self.kernel = self.add_weight(
+        shape=(input_dim, self.units * 4),
+        name='kernel',
+        dtype=dtypes.float32,
+        use_resource=True,
+        initializer=self.kernel_initializer,
+        regularizer=self.kernel_regularizer,
+        constraint=self.kernel_constraint)
+    self.recurrent_kernel = self.add_weight(
+        shape=(self.units, self.units * 4),
+        name='recurrent_kernel',
+        dtype=dtypes.float32,
+        use_resource=True,
+        initializer=self.recurrent_initializer,
+        regularizer=self.recurrent_regularizer,
+        constraint=self.recurrent_constraint)
+
+    # Normal LSTM has 4 bias instead of 8.
+    if self.unit_forget_bias:
+
+      def bias_initializer(_, *args, **kwargs):
+        return array_ops.concat([
+            self.bias_initializer((self.units * 5,), *args, **kwargs),
+            initializers.Ones()((self.units,), *args, **kwargs),
+            self.bias_initializer((self.units * 2,), *args, **kwargs),
+        ],
+                                axis=0)
+    else:
+      bias_initializer = self.bias_initializer
+    self.bias = self.add_weight(
+        shape=(self.units * 8,),
+        name='bias',
+        dtype=dtypes.float32,
+        use_resource=True,
+        initializer=bias_initializer,
+        regularizer=self.bias_regularizer,
+        constraint=self.bias_constraint)
+    self.built = True
+
+  def call(self, inputs, mask=None, training=None, initial_state=None):
+    if isinstance(inputs, list):
+      initial_state = inputs[1:]
+      inputs = inputs[0]
+    elif initial_state is not None:
+      pass
+    elif self.stateful:
+      initial_state = self.states
+    else:
+      initial_state = self.get_initial_state(inputs)
+
+    if len(initial_state) != len(self.states):
+      raise ValueError('Layer has ' + str(len(self.states)) +
+                       ' states but was passed ' + str(len(initial_state)) +
+                       ' initial states.')
+
+    if self.go_backwards:
+      # Reverse time axis.
+      inputs = K.reverse(inputs, 1)
+
+    outputs, [new_h, new_c], runtime = normal_lstm(
+        inputs, initial_state[0], initial_state[1], self.kernel,
+        self.recurrent_kernel, self.bias, self.units)
+
+    function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1],
+                      self.kernel, self.recurrent_kernel, self.bias, self.units)
+
+    states = [new_h, new_c]
+
+    if self.stateful:
+      updates = []
+      for i in range(len(states)):
+        updates.append(state_ops.assign(self.states[i], states[i]))
+      self.add_update(updates, inputs)
+
+    if self.return_sequences:
+      output = outputs
+    else:
+      output = outputs[:, -1, :]
+
+    if self.return_state:
+      return [output] + states
+    else:
+      return output, runtime
+
+  @tf_utils.shape_type_conversion
+  def compute_output_shape(self, input_shape):
+    if isinstance(input_shape, list):
+      input_shape = input_shape[0]
+
+    if _is_multiple_state(self.cell.state_size):
+      state_size = self.cell.state_size
+    else:
+      state_size = [self.cell.state_size]
+
+    if getattr(self.cell, 'output_size', None) is not None:
+      output_dim = tensor_shape.as_shape(self.cell.output_size).as_list()
+    else:
+      # Note that state_size[0] could be a tensor_shape or int.
+      output_dim = tensor_shape.as_shape(state_size[0]).as_list()
+
+    if self.return_sequences:
+      output_shape = tuple([input_shape[0], input_shape[1]] + output_dim)
+    else:
+      output_shape = tuple([input_shape[0]] + output_dim)
+
+    if self.return_state:
+      state_shape = [
+          tuple([input_shape[0]] + tensor_shape.as_shape(dim).as_list())
+          for dim in state_size
+      ]
+      return [output_shape] + state_shape
+    else:
+      return output_shape
+
+  @property
+  def trainable_weights(self):
+    if self.trainable and self.built:
+      return [self.kernel, self.recurrent_kernel, self.bias]
+    return []
+
+  @property
+  def non_trainable_weights(self):
+    if not self.trainable and self.built:
+      return [self.kernel, self.recurrent_kernel, self.bias]
+    return []
+
+  @property
+  def losses(self):
+    return super(RNN, self).losses
+
+  def get_losses_for(self, inputs=None):
+    return super(RNN, self).get_losses_for(inputs=inputs)   # pylint: disable=bad-super-call
+
+  def get_weights(self):
+    return super(RNN, self).get_weights()  # pylint: disable=bad-super-call
+
+
+def _canonical_to_params(weights, biases, shape):
+  weights = [array_ops.reshape(x, shape) for x in weights]
+  biases = [array_ops.reshape(x, shape) for x in biases]
+  return array_ops.concat(weights + biases, axis=0)
+
+
+def _is_multiple_state(state_size):
+  """Check whether the state_size contains multiple states."""
+  return (hasattr(state_size, '__len__') and
+          not isinstance(state_size, tensor_shape.TensorShape))
+
+
+@function.defun_with_attributes(
+    attributes={
+        'experimental_api_implements': 'lstm',
+        'experimental_api_preferred_device': 'CPU'
+    })
+def normal_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias, units):
+  input_shape = K.int_shape(inputs)
+  timesteps = input_shape[1]
+
+  def step(cell_inputs, cell_states):
+    h_tm1 = cell_states[0]  # previous memory state
+    c_tm1 = cell_states[1]  # previous carry state
+
+    # Only use the second half of the bias weights.
+    _, real_bias = array_ops.split(bias, 2)
+
+    z = K.dot(cell_inputs, kernel)
+    z += K.dot(h_tm1, recurrent_kernel)
+    z = K.bias_add(z, real_bias)
+
+    z0 = z[:, :units]
+    z1 = z[:, units:2 * units]
+    z2 = z[:, 2 * units:3 * units]
+    z3 = z[:, 3 * units:]
+
+    i = activations.get('hard_sigmoid')(z0)
+    f = activations.get('hard_sigmoid')(z1)
+    c = f * c_tm1 + i * activations.get('tanh')(z2)
+    o = activations.get('hard_sigmoid')(z3)
+
+    h = o * activations.get('tanh')(c)
+    return h, [h, c]
+
+  _, outputs, new_states = K.rnn(
+      step,
+      inputs, [init_h, init_c],
+      constants=None,
+      unroll=False,
+      input_length=timesteps)
+  return outputs, new_states, constant_op.constant(
+      'cpu', dtype=dtypes.string, name='runtime')
+
+
+@function.defun_with_attributes(
+    attributes={
+        'experimental_api_implements': 'lstm',
+        'experimental_api_preferred_device': 'GPU'
+    })
+def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias, units):
+  inputs = array_ops.transpose(inputs, perm=(1, 0, 2))
+  input_h = array_ops.expand_dims(input_h, axis=0)
+  input_c = array_ops.expand_dims(input_c, axis=0)
+
+  params = _canonical_to_params(
+      weights=[
+          kernel[:, :units],
+          kernel[:, units:units * 2],
+          kernel[:, units * 2:units * 3],
+          kernel[:, units * 3:],
+          recurrent_kernel[:, :units],
+          recurrent_kernel[:, units:units * 2],
+          recurrent_kernel[:, units * 2:units * 3],
+          recurrent_kernel[:, units * 3:],
+      ],
+      biases=[
+          bias[:units],
+          bias[units:units * 2],
+          bias[units * 2:units * 3],
+          bias[units * 3:units * 4],
+          bias[units * 4:units * 5],
+          bias[units * 5:units * 6],
+          bias[units * 6:units * 7],
+          bias[units * 7:],
+      ],
+      shape=constant_op.constant([-1]))
+
+  outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn(
+      inputs, input_h=input_h, input_c=input_c, params=params)
+  outputs = array_ops.transpose(outputs, perm=[1, 0, 2])
+  h = h[0]
+  c = c[0]
+  return outputs, [h, c], constant_op.constant(
+      'cudnn', dtype=dtypes.string, name='runtime')
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 4a46b26914dbe4e366ad07da2c7897115e07c6b0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 16:18:51 -0800
Subject: [PATCH 0358/1554] Drop unused contrib deps.

PiperOrigin-RevId: 221709337
---
 tensorflow/contrib/distribute/python/BUILD | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4b96179077..bf5adb1a50 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -152,7 +152,6 @@ py_library(
     visibility = ["//tensorflow:internal"],
     deps = [
         ":values",
-        "//tensorflow/contrib/eager/python:datasets",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:distribute",
         "//tensorflow/python:dtypes",
@@ -715,8 +714,6 @@ cuda_py_test(
     additional_deps = [
         ":input_ops",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/contrib/data/python/ops:batching",
-        "//tensorflow/contrib/data/python/ops:interleave_ops",
         "//tensorflow/python:errors",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
@@ -769,7 +766,6 @@ py_library(
     srcs = ["metrics_v1_test.py"],
     deps = [
         ":combinations",
-        "//tensorflow/contrib/data/python/ops:batching",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
         "//tensorflow/python:variables",
-- 
GitLab


From dba6ada47754dd60990ed999b25b6659819b55c3 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 15 Nov 2018 16:22:52 -0800
Subject: [PATCH 0359/1554] [tf.data] Add deprecated `tf.contrib.data` symbols
 to the conversion script.

PiperOrigin-RevId: 221709955
---
 .../tools/compatibility/tf_upgrade_v2.py      | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3a8ff25937..7e87b7d764 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -58,6 +58,48 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # pylint: disable=line-too-long
     # Add additional renames not in renames_v2.py here.
     self.symbol_renames.update({
+        "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
+        "tf.contrib.data.Counter": "tf.data.experimental.Counter",
+        "tf.contrib.data.CheckpointInputPipelineHook": "tf.data.experimental.CheckpointInputPipelineHook",
+        "tf.contrib.data.CsvDataset": "tf.data.experimental.CsvDataset",
+        "tf.contrib.data.Optional": "tf.data.experimental.Optional",
+        "tf.contrib.data.RandomDataset": "tf.data.experimental.RandomDataset",
+        "tf.contrib.data.Reducer": "tf.data.experimental.Reducer",
+        "tf.contrib.data.SqlDataset": "tf.data.experimental.SqlDataset",
+        "tf.contrib.data.StatsAggregator": "tf.data.experimental.StatsAggregator",
+        "tf.contrib.data.TFRecordWriter": "tf.data.experimental.TFRecordWriter",
+        "tf.contrib.data.assert_element_shape": "tf.data.experimental.assert_element_shape",
+        "tf.contrib.data.batch_and_drop_remainder": "tf.compat.v1.contrib.data.batch_and_drop_remainder",
+        "tf.contrib.data.bucket_by_sequence_length": "tf.data.experimental.bucket_by_sequence_length",
+        "tf.contrib.data.choose_from_datasets": "tf.data.experimental.choose_from_datasets",
+        "tf.contrib.data.copy_to_device": "tf.data.experimental.copy_to_device",
+        "tf.contrib.data.dense_to_sparse_batch": "tf.data.experimental.dense_to_sparse_batch",
+        "tf.contrib.data.enumerate_dataset": "tf.data.experimental.enumerate_dataset",
+        "tf.contrib.data.get_next_as_optional": "tf.data.experimental.get_next_as_optional",
+        "tf.contrib.data.get_single_element": "tf.data.experimental.get_single_element",
+        "tf.contrib.data.group_by_reducer": "tf.data.experimental.group_by_reducer",
+        "tf.contrib.data.group_by_window": "tf.data.experimental.group_by_window",
+        "tf.contrib.data.ignore_errors": "tf.data.experimental.ignore_errors",
+        "tf.contrib.data.latency_stats": "tf.data.experimental.latency_stats",
+        "tf.contrib.data.make_batched_features_dataset": "tf.data.experimental.make_batched_features_dataset",
+        "tf.contrib.data.make_csv_dataset": "tf.data.experimental.make_csv_dataset",
+        "tf.contrib.data.make_saveable_from_iterator": "tf.data.experimental.make_saveable_from_iterator",
+        "tf.contrib.data.map_and_batch": "tf.data.experimental.map_and_batch",
+        "tf.contrib.data.padded_batch_and_drop_remainder": "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
+        "tf.contrib.data.parallel_interleave": "tf.data.experimental.parallel_interleave",
+        "tf.contrib.data.parse_example_dataset": "tf.data.experimental.parse_example_dataset",
+        "tf.contrib.data.prefetch_to_device": "tf.data.experimental.prefetch_to_device",
+        "tf.contrib.data.read_batch_features": "tf.compat.v1.contrib.data.read_batch_features",
+        "tf.contrib.data.reduce_dataset": "tf.compat.v1.contrib.data.reduce_dataset",
+        "tf.contrib.data.rejection_resample": "tf.data.experimental.rejection_resample",
+        "tf.contrib.data.sample_from_datasets": "tf.data.experimental.sample_from_datasets",
+        "tf.contrib.data.scan": "tf.data.experimental.scan",
+        "tf.contrib.data.set_stats_aggregator": "tf.data.experimental.set_stats_aggregator",
+        "tf.contrib.data.shuffle_and_repeat": "tf.data.experimental.shuffle_and_repeat",
+        "tf.contrib.data.sliding_window_batch": "tf.compat.v1.contrib.data.sliding_window_batch",
+        "tf.contrib.data.sloppy_interleave": "tf.compat.v1.contrib.data.sloppy_interleave",
+        "tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
+        "tf.contrib.data.unique": "tf.data.experimental.unique",
         "tf.sparse_concat": "tf.sparse.concat",
         "tf.load_file_system_library": "tf.load_library",
     })
-- 
GitLab


From 582672a1c206238252dc7703bc5928f054dd6c15 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 15 Nov 2018 16:25:08 -0800
Subject: [PATCH 0360/1554] Automated rollback of commit
 0cfa4c157d317ccb4ccd1f6c85773f973d628695

PiperOrigin-RevId: 221710313
---
 tensorflow/core/framework/tensor.h       | 87 +++++++++---------------
 tensorflow/core/framework/tensor_test.cc | 11 ++-
 2 files changed, 36 insertions(+), 62 deletions(-)

diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 797ce5f0be..8a5633983b 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -30,7 +30,6 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
-#include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -120,7 +119,7 @@ class Tensor {
 
   class HostScalarTensorBufferBase;
   template <typename T>
-  struct ValueAndTensorBuffer;
+  class HostScalarTensorBuffer;
 
   // Creates a tensor with the given scalar `value` in CPU memory.
   template <typename T>
@@ -877,62 +876,40 @@ class Tensor::HostScalarTensorBufferBase : public TensorBuffer {
   void FillAllocationDescription(AllocationDescription* proto) const final;
 };
 
+// `Tensor::HostScalarTensorBuffer<T>` is a specialized `TensorBuffer`
+// implementation for storing a single scalar value.
+//
+// TODO(mrry): Evaluate other compilers or approaches to aligning the value
+// so that it can be used directly as a tensor value. For example, in a C++17
+// future, we could use `alignas(EIGEN_MAX_ALIGN_BYTES)` to store the value
+// inline in this object to save an allocation. However, this is not currently
+// widely supported in our compilers.
 template <typename T>
-Tensor::Tensor(T value, host_scalar_tag tag) {
-  // A packed representation for a single scalar value of type `T`, and a
-  // `TensorBuffer` implementation that describes (and manages the lifetime of)
-  // that value.
-  struct ValueAndTensorBuffer {
-    class HostScalarTensorBuffer : public HostScalarTensorBufferBase {
-     public:
-      HostScalarTensorBuffer(void* data) : data_(data) {}
-      void* data() const final { return const_cast<void*>(data_); }
-      size_t size() const final { return sizeof(T); }
-      TensorBuffer* root_buffer() final { return this; }
-
-      // Override `operator delete` so that calling `delete this` in
-      // `core::Refcounted::Unref()` for an object of this type will free
-      // the enclosing `ValueAndTensorBuffer` for the tensor buffer.
-      static void operator delete(void* ptr) {
-        // Use a dummy object to compute to offset of
-        // `ValueAndTensorBuffer::tensor_buffer`, because `offsetof()` is not
-        // necessarily defined on this non-POD type (until C++17).
-        typename std::aligned_storage<sizeof(ValueAndTensorBuffer),
-                                      alignof(ValueAndTensorBuffer)>::type
-            dummy_storage_;
-        ValueAndTensorBuffer* dummy_object =
-            reinterpret_cast<ValueAndTensorBuffer*>(&dummy_storage_);
-        intptr_t offset =
-            reinterpret_cast<intptr_t>(&dummy_object->tensor_buffer) -
-            reinterpret_cast<intptr_t>(dummy_object);
-
-        port::AlignedFree(static_cast<char*>(ptr) - offset);
-      }
-
-      static void operator delete(void*, void*) {
-        // Some compilers require an overridden class-specific deallocation
-        // function, which will be called if placement `new` throws an
-        // exception.
-      }
-
-     private:
-      ~HostScalarTensorBuffer() override { static_cast<T*>(data_)->~T(); }
-      void* const data_;
-    };
-
-    T value;
-    HostScalarTensorBuffer tensor_buffer;
-  };
-
-  auto* value_and_buf = static_cast<ValueAndTensorBuffer*>(
-      port::AlignedMalloc(sizeof(ValueAndTensorBuffer), EIGEN_MAX_ALIGN_BYTES));
-  new (&value_and_buf->value) T(std::move(value));
-  new (&value_and_buf->tensor_buffer)
-      typename ValueAndTensorBuffer::HostScalarTensorBuffer(value_and_buf);
-  buf_ = &value_and_buf->tensor_buffer;
+class Tensor::HostScalarTensorBuffer : public HostScalarTensorBufferBase {
+ public:
+  HostScalarTensorBuffer(T&& value)
+      : data_(reinterpret_cast<T*>(cpu_allocator()->AllocateRaw(
+            EIGEN_MAX_ALIGN_BYTES, sizeof(value)))) {
+    if (is_simple_type<T>::value) {
+      *data_ = value;
+    } else {
+      new (data_) T(std::move(value));
+    }
+  }
+  ~HostScalarTensorBuffer() { cpu_allocator()->Deallocate(data_, 1); }
+  void* data() const final { return const_cast<T*>(data_); }
+  size_t size() const final { return sizeof(*data_); }
+  TensorBuffer* root_buffer() final { return this; }
+
+ private:
+  T* const data_;
+};
+
+template <typename T>
+Tensor::Tensor(T value, host_scalar_tag tag)
+    : buf_(new HostScalarTensorBuffer<T>(std::move(value))) {
   set_dtype(DataTypeToEnum<T>::value);
 }
-
 inline Tensor& Tensor::operator=(Tensor&& other) {
   // Avoid self-assignment, since we might destroy our underlying buffer.
   if (&other != this) {
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 4fa9d1df67..925ebc4945 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -854,18 +854,15 @@ TEST(Tensor_HostScalar, Basics) {
     EXPECT_FLOAT_EQ(42.0f, Tt());
   }
   {
-    // NOTE(mrry): Use long enough strings so that the contents are dynamically
-    // allocated, and the absence of a call to the string destructor would
-    // cause a memory leak.
-    Tensor t("fooooooooooooooooooooooooooooooooooooo");
+    Tensor t("foo");
     EXPECT_EQ(DT_STRING, t.dtype());
     EXPECT_EQ(1, t.NumElements());
     auto Tt = t.scalar<string>();
     EXPECT_EQ(1, Tt.size());
     EXPECT_EQ(0, Tt.rank());
-    EXPECT_EQ("fooooooooooooooooooooooooooooooooooooo", Tt());
-    Tt() = "baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar";
-    EXPECT_EQ("baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar", Tt());
+    EXPECT_EQ("foo", Tt());
+    Tt() = "bar";
+    EXPECT_EQ("bar", Tt());
   }
 }
 
-- 
GitLab


From 2583b9ff7ad9c24a544aa203029521a64f32a5a2 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 15 Nov 2018 16:26:10 -0800
Subject: [PATCH 0361/1554] Do not use assertBetween. AssertBetween is an
 absl-py function, TF still uses python unittest.

PiperOrigin-RevId: 221710480
---
 .../contrib/boosted_trees/estimator_batch/estimator_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index 1ede129925..ae6bb5767f 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -417,7 +417,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
 
     frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3)
     # +/- 3%
-    self.assertBetween(frac_below_upper, 0.92, 0.98)
+    self.assertTrue(frac_below_upper >= 0.92)
+    self.assertTrue(frac_below_upper <= 0.98)
 
     train_input_fn, test_input_fn, _ = _quantile_regression_input_fns()
     model_lower = estimator.GradientBoostedDecisionTreeQuantileRegressor(
@@ -435,7 +436,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
 
     frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3)
     # +/- 3%
-    self.assertBetween(frac_above_lower, 0.92, 0.98)
+    self.assertTrue(frac_below_upper >= 0.92)
+    self.assertTrue(frac_below_upper <= 0.98)
 
 
 class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
-- 
GitLab


From 0f7c2537af48bc84740474485dca3e936d5aef27 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 15 Nov 2018 16:27:15 -0800
Subject: [PATCH 0362/1554] Implement Adadelta optimization.

PiperOrigin-RevId: 221710645
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  20 +++
 .../python/keras/optimizer_v2/adadelta.py     | 128 +++++++++++++
 .../keras/optimizer_v2/adadelta_test.py       | 170 ++++++++++++++++++
 3 files changed, 318 insertions(+)
 create mode 100644 tensorflow/python/keras/optimizer_v2/adadelta.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/adadelta_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 2421decc32..7c2b1d7d35 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -13,6 +13,7 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 py_library(
     name = "optimizer_v2",
     srcs = [
+        "adadelta.py",
         "adam.py",
         "gradient_descent.py",
         "optimizer_v2.py",
@@ -51,6 +52,25 @@ cuda_py_test(
     shard_count = 4,
 )
 
+cuda_py_test(
+    name = "adadelta_test",
+    size = "medium",
+    srcs = ["adadelta_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+    shard_count = 4,
+)
+
 cuda_py_test(
     name = "gradient_descent_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
new file mode 100644
index 0000000000..29407484a9
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -0,0 +1,128 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Adadelta for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import math_ops
+from tensorflow.python.training import training_ops
+
+
+class Adadelta(optimizer_v2.OptimizerV2):
+  r"""Optimizer that implements the Adadelta algorithm.
+
+  Adadelta optimization is a stochastic gradient descent method that is based on
+  adaptive learning rate per dimension to address two drawbacks:
+    1) the continual decay of learning rates throughout training
+    2) the need for a manually selected global learning rate
+
+  Two accumulation steps are required:
+    1) the accumulation of gradients squared,
+    2) the accumulation of updates squared.
+
+  Initialization:
+
+  $$accum_g_0 := 0 \text{(Initialize gradient 2nd order moment vector)}$$
+  $$accum_x_0 := 0 \text{(Initialize variable update 2nd order moment vector)}$$
+
+  $$t := t + 1$$
+  $$accum_g_t := rho * accum_g_{t-1} + (1 - rho) * g * g$$
+  $$delta = -\sqrt{accum_x_{t-1}} / (\sqrt{accum_g_{t-1}} + \epsilon)$$
+  $$accum_x_t := rho * accum_x_{t-1} + (1 - rho) * delta * delta$$
+
+  References
+    See [M. D. Zeiler](http://arxiv.org/abs/1212.5701)
+      ([pdf](http://arxiv.org/pdf/1212.5701v1.pdf))
+
+  """
+
+  def __init__(self,
+               learning_rate=0.001,
+               rho=0.95,
+               epsilon=1e-8,
+               name='Adadelta'):
+    """Construct a new Adadelta optimizer.
+
+    Adadelta is a more robust extension of Adagrad that adapts learning rates
+    based on a moving window of gradient updates, instead of accumulating all
+    past gradients. This way, Adadelta continues learning even when many updates
+    have been done. Compared to Adagrad, in the original version of Adadelta you
+    don't have to set an initial learning rate. In this version, initial
+    learning rate can be set, as in most other Keras optimizers.
+
+    Args:
+      learning_rate: A `Tensor` or a floating point value. The learning rate.
+        To match the exact form in the original paper use 1.0.
+      rho: A `Tensor` or a floating point value. The decay rate.
+      epsilon: A `Tensor` or a floating point value.  A constant epsilon used
+               to better conditioning the grad update.
+      name: Optional name prefix for the operations created when applying
+        gradients.  Defaults to "Adadelta".
+
+    @compatibility(eager)
+    When eager execution is enabled, `learning_rate`, `rho`, and `epsilon` can
+    each be a callable that takes no arguments and returns the actual value to
+    use. This can be useful for changing these values across different
+    invocations of optimizer functions.
+    @end_compatibility
+    """
+    super(Adadelta, self).__init__(name)
+    self._set_hyper('learning_rate', learning_rate)
+    self._set_hyper('rho', rho)
+    self._set_hyper('epsilon', epsilon)
+
+  def _create_slots(self, var_list):
+    for v in var_list:
+      self.add_slot(v, 'accum_grad')
+      self.add_slot(v, 'accum_var')
+
+  def _resource_apply_dense(self, grad, var):
+    accum_grad = self.get_slot(var, 'accum_grad')
+    accum_var = self.get_slot(var, 'accum_var')
+    return training_ops.resource_apply_adadelta(
+        var.handle,
+        accum_grad.handle,
+        accum_var.handle,
+        math_ops.cast(self._get_hyper('learning_rate'), grad.dtype.base_dtype),
+        math_ops.cast(self._get_hyper('rho'), grad.dtype.base_dtype),
+        math_ops.cast(self._get_hyper('epsilon'), grad.dtype.base_dtype),
+        grad,
+        use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices):
+    accum_grad = self.get_slot(var, 'accum_grad')
+    accum_var = self.get_slot(var, 'accum_var')
+    return training_ops.resource_sparse_apply_adadelta(
+        var.handle,
+        accum_grad.handle,
+        accum_var.handle,
+        math_ops.cast(self._get_hyper('learning_rate'), grad.dtype.base_dtype),
+        math_ops.cast(self._get_hyper('rho'), grad.dtype.base_dtype),
+        math_ops.cast(self._get_hyper('epsilon'), grad.dtype.base_dtype),
+        grad,
+        indices,
+        use_locking=self._use_locking)
+
+  def get_config(self):
+    config = super(Adadelta, self).get_config()
+    config.update({
+        'learning_rate': self._serialize_hyperparameter('learning_rate'),
+        'rho': self._serialize_hyperparameter('rho'),
+        'epsilon': self._serialize_hyperparameter('epsilon'),
+    })
+    return config
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
new file mode 100644
index 0000000000..f169fa6b86
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -0,0 +1,170 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Adadelta Optimizer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.optimizer_v2 import adadelta
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class AdadeltaOptimizerTest(test.TestCase):
+
+  def doTestBasic(self, use_resource=False, use_callable_params=False):
+    num_updates = 4  # number of ADADELTA steps to perform
+    for dtype in [dtypes.half, dtypes.float32]:
+      for grad in [0.2, 0.1, 0.01]:
+        for lr in [1.0, 0.5, 0.1]:
+          var0_init = [1.0, 2.0]
+          var1_init = [3.0, 4.0]
+          if use_resource:
+            var0 = resource_variable_ops.ResourceVariable(
+                var0_init, dtype=dtype)
+            var1 = resource_variable_ops.ResourceVariable(
+                var1_init, dtype=dtype)
+          else:
+            var0 = variables.Variable(var0_init, dtype=dtype)
+            var1 = variables.Variable(var1_init, dtype=dtype)
+
+          grads = constant_op.constant([grad, grad], dtype=dtype)
+
+          accum = 0.0
+          accum_update = 0.0
+
+          # ADADELTA gradient optimizer
+          rho = 0.95
+          epsilon = 1e-8
+          if use_callable_params:
+            adadelta_opt = adadelta.Adadelta(
+                learning_rate=lambda: lr,  # pylint: disable=cell-var-from-loop
+                rho=lambda: rho,  # pylint: disable=cell-var-from-loop
+                epsilon=lambda: epsilon)  # pylint: disable=cell-var-from-loop
+          else:
+            adadelta_opt = adadelta.Adadelta(
+                learning_rate=lr, rho=rho, epsilon=epsilon)
+          if not context.executing_eagerly():
+            adadelta_update = adadelta_opt.apply_gradients(
+                zip([grads, grads], [var0, var1]))
+            self.evaluate(variables.global_variables_initializer())
+
+            # Assign slots
+            slot = [None] * 2
+            slot_update = [None] * 2
+            slot[0] = adadelta_opt.get_slot(var0, "accum_grad")
+            self.assertEquals(slot[0].get_shape(), var0.get_shape())
+
+            slot_update[0] = adadelta_opt.get_slot(var0, "accum_var")
+            self.assertEquals(slot_update[0].get_shape(), var0.get_shape())
+
+            slot[1] = adadelta_opt.get_slot(var1, "accum_grad")
+            self.assertEquals(slot[1].get_shape(), var1.get_shape())
+
+            slot_update[1] = adadelta_opt.get_slot(var1, "accum_var")
+            self.assertEquals(slot_update[1].get_shape(), var1.get_shape())
+
+          # Fetch params to validate initial values
+          self.assertAllClose(var0_init, self.evaluate(var0))
+          self.assertAllClose(var1_init, self.evaluate(var1))
+
+          update = [None] * num_updates
+          tot_update = 0
+          for step in range(num_updates):
+            # Run adadelta update for comparison
+            if not context.executing_eagerly():
+              self.evaluate(adadelta_update)
+            else:
+              adadelta_opt.apply_gradients(zip([grads, grads], [var0, var1]))
+
+            # Perform initial update without previous accum values
+            accum = accum * rho + (grad**2) * (1 - rho)
+            update[step] = (
+                np.sqrt(accum_update + epsilon) *
+                (1. / np.sqrt(accum + epsilon)) * grad)
+            accum_update = (
+                accum_update * rho + (update[step]**2) * (1.0 - rho))
+            tot_update += update[step] * lr
+
+            if not context.executing_eagerly():
+              # Check that the accumulators have been updated
+              # TODO(lxuechen): This is hard to test in eager mode
+              for slot_idx in range(2):
+                self.assertAllCloseAccordingToType(
+                    np.array([accum, accum], dtype=dtype.as_numpy_dtype()),
+                    self.evaluate(slot[slot_idx]),
+                    rtol=1e-5)
+
+                self.assertAllCloseAccordingToType(
+                    np.array(
+                        [accum_update, accum_update],
+                        dtype=dtype.as_numpy_dtype()),
+                    self.evaluate(slot_update[slot_idx]),
+                    rtol=1e-5)
+
+              # Check that the parameters have been updated
+              self.assertAllCloseAccordingToType(
+                  np.array(
+                      [var0_init[0] - tot_update, var0_init[1] - tot_update],
+                      dtype=dtype.as_numpy_dtype()),
+                  self.evaluate(var0),
+                  rtol=1e-5)
+
+              self.assertAllCloseAccordingToType(
+                  np.array(
+                      [var1_init[0] - tot_update, var1_init[1] - tot_update],
+                      dtype=dtype.as_numpy_dtype()),
+                  self.evaluate(var1),
+                  rtol=1e-5)
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testResourceBasic(self):
+    self.doTestBasic(use_resource=True)
+
+  def testBasicCallableParams(self):
+    with context.eager_mode():
+      self.doTestBasic(use_resource=True, use_callable_params=True)
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = adadelta.Adadelta(1.0, 1.0, 1.0).minimize(
+            loss, var_list=[var0])
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[-111, -138]], var0.eval())
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 1612ea9fd2ef6896a339b00e8619e8a19e0efe67 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 15 Nov 2018 16:31:14 -0800
Subject: [PATCH 0363/1554] [TF:XLA] Bump open source abseil revision to
 a06c4a1d9093137b7217a5aaba8920d62e835dc0

PiperOrigin-RevId: 221711321
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 435ea2d733..3618f14650 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "f8536445cd480be7ec89bac19deaf766a1038330470fb0b469a98bce09e5c5ce",
-        strip_prefix = "abseil-cpp-7b46e1d31a6b08b1c6da2a13e7b151a20446fa07",
+        sha256 = "042badbfa529555ed63d57c39bf58410aa16e518ce98299599d85729feb25070",
+        strip_prefix = "abseil-cpp-a06c4a1d9093137b7217a5aaba8920d62e835dc0",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/7b46e1d31a6b08b1c6da2a13e7b151a20446fa07.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/7b46e1d31a6b08b1c6da2a13e7b151a20446fa07.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/a06c4a1d9093137b7217a5aaba8920d62e835dc0.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/a06c4a1d9093137b7217a5aaba8920d62e835dc0.tar.gz",
         ],
     )
 
-- 
GitLab


From 7a74b83897804d54789d240b224947d9c3185061 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Thu, 15 Nov 2018 16:52:18 -0800
Subject: [PATCH 0364/1554] Add initial symbol name validation check to
 tf_export.py

Add validation check to ensure @estimator_export only exports
public symbols under estimator.*. If TF and Estimator root
package names aren't disjoint, it can cause a bunch of issues
when trying to tie together the APIs. Adding initial check to
ensure this.

PiperOrigin-RevId: 221714350
---
 tensorflow/python/util/tf_export.py      | 40 ++++++++++++++++++++++++
 tensorflow/python/util/tf_export_test.py | 20 ++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py
index 0924b36ade..ec70cae7d2 100644
--- a/tensorflow/python/util/tf_export.py
+++ b/tensorflow/python/util/tf_export.py
@@ -50,6 +50,10 @@ from tensorflow.python.util import tf_decorator
 ESTIMATOR_API_NAME = 'estimator'
 TENSORFLOW_API_NAME = 'tensorflow'
 
+# List of subpackage names used by TensorFlow components. Have to check that
+# TensorFlow core repo does not export any symbols under these names.
+SUBPACKAGE_NAMESPACES = [ESTIMATOR_API_NAME]
+
 _Attributes = collections.namedtuple(
     'ExportedApiAttributes', ['names', 'constants'])
 
@@ -78,6 +82,11 @@ class SymbolAlreadyExposedError(Exception):
   pass
 
 
+class InvalidSymbolNameError(Exception):
+  """Raised when trying to export symbol as an invalid or unallowed name."""
+  pass
+
+
 def get_canonical_name_for_symbol(
     symbol, api_name=TENSORFLOW_API_NAME,
     add_prefix_to_v1_names=False):
@@ -163,6 +172,37 @@ class api_export(object):  # pylint: disable=invalid-name
     self._overrides = kwargs.get('overrides', [])
     self._allow_multiple_exports = kwargs.get('allow_multiple_exports', False)
 
+    self._validate_symbol_names()
+
+  def _validate_symbol_names(self):
+    """Validate you are exporting symbols under an allowed package.
+
+    We need to ensure things exported by tf_export, estimator_export, etc.
+    export symbols under disjoint top-level package names.
+
+    For TensorFlow, we check that it does not export anything under subpackage
+    names used by components (estimator, keras, etc.).
+
+    For each component, we check that it exports everything under its own
+    subpackage.
+
+    Raises:
+      InvalidSymbolNameError: If you try to export symbol under disallowed name.
+    """
+    all_symbol_names = set(self._names) | set(self._names_v1)
+    if self._api_name == TENSORFLOW_API_NAME:
+      for subpackage in SUBPACKAGE_NAMESPACES:
+        if any(n.startswith(subpackage) for n in all_symbol_names):
+          raise InvalidSymbolNameError(
+              '@tf_export is not allowed to export symbols under %s.*' % (
+                  subpackage))
+    else:
+      if not all(n.startswith(self._api_name) for n in all_symbol_names):
+        raise InvalidSymbolNameError(
+            'Can only export symbols under package name of component. '
+            'e.g. tensorflow_estimator must export all symbols under '
+            'tf.estimator')
+
   def __call__(self, func):
     """Calls this decorator.
 
diff --git a/tensorflow/python/util/tf_export_test.py b/tensorflow/python/util/tf_export_test.py
index 4ae1dc55e0..a0fac8bf36 100644
--- a/tensorflow/python/util/tf_export_test.py
+++ b/tensorflow/python/util/tf_export_test.py
@@ -130,6 +130,26 @@ class ValidateExportTest(test.TestCase):
     with self.assertRaises(tf_export.SymbolAlreadyExposedError):
       export_decorator(_test_function)
 
+  def testRaisesExceptionIfInvalidSymbolName(self):
+    # TensorFlow code is not allowed to export symbols under package
+    # tf.estimator
+    with self.assertRaises(tf_export.InvalidSymbolNameError):
+      tf_export.tf_export('estimator.invalid')
+
+    # All symbols exported by Estimator must be under tf.estimator package.
+    with self.assertRaises(tf_export.InvalidSymbolNameError):
+      tf_export.estimator_export('invalid')
+    with self.assertRaises(tf_export.InvalidSymbolNameError):
+      tf_export.estimator_export('Estimator.invalid')
+    with self.assertRaises(tf_export.InvalidSymbolNameError):
+      tf_export.estimator_export('invalid.estimator')
+
+  def testRaisesExceptionIfInvalidV1SymbolName(self):
+    with self.assertRaises(tf_export.InvalidSymbolNameError):
+      tf_export.tf_export('valid', v1=['estimator.invalid'])
+    with self.assertRaises(tf_export.InvalidSymbolNameError):
+      tf_export.estimator_export('estimator.valid', v1=['invalid'])
+
   def testOverridesFunction(self):
     _test_function2._tf_api_names = ['abc']
 
-- 
GitLab


From 1c0ff2f48f4bbfb8a91ef31e1fe8bf837d9fc4bd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 17:03:20 -0800
Subject: [PATCH 0365/1554] Add support for FunctionalizeControlFlow to
 optimize a GraphDef.

PiperOrigin-RevId: 221715957
---
 .../tf2xla/functionalize_control_flow.cc      |   19 +
 .../tf2xla/functionalize_control_flow.h       |    6 +
 .../tf2xla/functionalize_control_flow_test.cc | 1013 +++++++++--------
 3 files changed, 574 insertions(+), 464 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
index 9ef9f49f42..3dfd3f854c 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
@@ -75,6 +75,25 @@ Status FunctionalizeControlFlow(Graph* graph,
   return FunctionalizeControlFlow(/*lookup_library=*/nullptr, graph, library);
 }
 
+Status FunctionalizeControlFlowForGraphDef(GraphDef* graph_def,
+                                           FunctionLibraryDefinition* library) {
+  return FunctionalizeControlFlowForGraphDef(/*lookup_library=*/nullptr,
+                                             graph_def, library);
+}
+
+Status FunctionalizeControlFlowForGraphDef(
+    const FunctionLibraryDefinition* lookup_library, GraphDef* graph_def,
+    FunctionLibraryDefinition* library) {
+  FunctionDefLibrary function_lib = graph_def->library();
+  Graph graph(OpRegistry::Global());
+
+  TF_RETURN_IF_ERROR(ConvertGraphDefToGraph({}, *graph_def, &graph));
+  TF_RETURN_IF_ERROR(FunctionalizeControlFlow(lookup_library, &graph, library));
+  graph.ToGraphDef(graph_def);
+  std::swap(*graph_def->mutable_library(), function_lib);
+  return Status::OK();
+}
+
 Status FunctionalizeControlFlowForFunction(
     const string& func_name, const string& new_func_name,
     const protobuf::Map<string, tensorflow::AttrValue>& attrs,
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.h b/tensorflow/compiler/tf2xla/functionalize_control_flow.h
index ba99205640..91d33fa405 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.h
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.h
@@ -33,6 +33,12 @@ Status FunctionalizeControlFlow(const FunctionLibraryDefinition* lookup_library,
                                 Graph* graph,
                                 FunctionLibraryDefinition* library);
 
+Status FunctionalizeControlFlowForGraphDef(GraphDef* graph_def,
+                                           FunctionLibraryDefinition* library);
+Status FunctionalizeControlFlowForGraphDef(
+    const FunctionLibraryDefinition* lookup_library, GraphDef* graph_def,
+    FunctionLibraryDefinition* library);
+
 // This pass looks at the graph and all associated FunctionDefs, and turns
 // traditional control flow structure (Switch/Merge/etc.) into functional
 // control flow structure (If/While).
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
index c3841f996f..9784985af8 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
@@ -95,77 +95,87 @@ TEST(FunctionalizeControlFlow, Conditional) {
   }
 
   FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  GraphDef optimized_graph_def;
+  graph.ToGraphDef(&optimized_graph_def);
+  TF_ASSERT_OK(
+      FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library));
   TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+  GraphDef converted_graph_def;
+  graph.ToGraphDef(&converted_graph_def);
+
+  for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) {
+    string op_name;
+    NameAttrList then_fn;
+    NameAttrList else_fn;
+    TF_EXPECT_OK(FindIfThenAndElse(graph_def, &op_name, &then_fn, &else_fn));
+    InstantiationResultForTest else_result;
+    TF_EXPECT_OK(
+        InstantiateFunctionForTest(else_fn.name(), library, &else_result));
+
+    // Outer graph
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto y = ops::Placeholder(scope.WithOpName("y"), DT_INT32);
+      auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
+      auto less = ops::Less(scope.WithOpName("cond/Less"), y, x);
+      auto if_op = ops::If(scope.WithOpName(op_name), less,
+                           std::initializer_list<Input>{less, y, x}, {DT_INT32},
+                           then_fn, else_fn);
+      auto id = ops::Identity(scope.WithOpName("cond/Merge"), if_op.output[0]);
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+      TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    }
 
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
-  string op_name;
-  NameAttrList then_fn;
-  NameAttrList else_fn;
-  TF_EXPECT_OK(FindIfThenAndElse(graph_def, &op_name, &then_fn, &else_fn));
-  InstantiationResultForTest else_result;
-  TF_EXPECT_OK(
-      InstantiateFunctionForTest(else_fn.name(), library, &else_result));
-
-  // Outer graph
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto y = ops::Placeholder(scope.WithOpName("y"), DT_INT32);
-    auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
-    auto less = ops::Less(scope.WithOpName("cond/Less"), y, x);
-    auto if_op = ops::If(scope.WithOpName(op_name), less,
-                         std::initializer_list<Input>{less, y, x}, {DT_INT32},
-                         then_fn, else_fn);
-    auto id = ops::Identity(scope.WithOpName("cond/Merge"), if_op.output[0]);
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-    TF_EXPECT_GRAPH_EQ(expected, graph_def);
-  }
-
-  // then body.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0);
-    auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
-    auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0);
-    auto cond = ops::Const(
-        scope.WithOpName("cond").WithControlDependencies(identity), 17);
-    auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond);
-    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), mul, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(then_fn.name(), library, &result));
-
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
+    // then body.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0);
+      auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+      auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0);
+      auto cond = ops::Const(
+          scope.WithOpName("cond").WithControlDependencies(identity), 17);
+      auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond);
+      auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), mul, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(then_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+      EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}),
+                result.arg_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-  // else body.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0);
-    auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
-    auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0);
-    auto cond_1 = ops::Const(
-        scope.WithOpName("cond_1").WithControlDependencies(identity), 23);
-    auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1);
-    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(else_fn.name(), library, &result));
-
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    // else body.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0);
+      auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+      auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0);
+      auto cond_1 = ops::Const(
+          scope.WithOpName("cond_1").WithControlDependencies(identity), 23);
+      auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1);
+      auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(else_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+      EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}),
+                result.arg_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
   }
 }
 
@@ -239,75 +249,77 @@ TEST(FunctionalizeControlFlow, OneLoopVar) {
   }
 
   FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  GraphDef optimized_graph_def;
+  graph.ToGraphDef(&optimized_graph_def);
+  TF_ASSERT_OK(
+      FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library));
   TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+  GraphDef converted_graph_def;
+  graph.ToGraphDef(&converted_graph_def);
+
+  for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) {
+    NameAttrList cond_fn, body_fn;
+    TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
+
+    // Outer graph
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
+      auto while_op =
+          ops::While(scope.WithOpName("while/LoopCond"),
+                     std::initializer_list<Input>{source}, cond_fn, body_fn);
+      auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]);
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+      TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    }
 
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
-
-  NameAttrList cond_fn, body_fn;
-  TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
-
-  // Outer graph
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
-    auto while_op =
-        ops::While(scope.WithOpName("while/LoopCond"),
-                   std::initializer_list<Input>{source}, cond_fn, body_fn);
-    auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]);
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-    TF_EXPECT_GRAPH_EQ(expected, graph_def);
-  }
-
-  // Condition graph
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto ten = ops::Const<int32>(
-        scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10);
-    auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result));
-
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
-
-  // Body graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
-    auto one = ops::Const<int32>(
-        scope.WithOpName("while/add/y").WithControlDependencies(identity), 1);
-    auto add = ops::Add(scope.WithOpName("while/add"), identity, one);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result));
+    // Condition graph
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto ten = ops::Const<int32>(
+          scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10);
+      auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten);
+      auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(cond_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    // Body graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
+      auto one = ops::Const<int32>(
+          scope.WithOpName("while/add/y").WithControlDependencies(identity), 1);
+      auto add = ops::Add(scope.WithOpName("while/add"), identity, one);
+      auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(body_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
   }
 }
 
-// @function.Defun(noinline=True)
-// def increment_fn(x):
-//   return [x + 1]
-// Define the above function, and add it to the given graph. It's used as the
-// while loop body in NoinlineLoopBody test.
-Status AddNoinlineFunctionToGraph(const string& node_name, Graph* graph) {
+FunctionDef GetNoinlineFunctionDef() {
   FunctionDef fdef = FunctionDefHelper::Create(
       "increment_fn", {"x:int32"}, {"add:int32"}, {},
       {
@@ -316,8 +328,17 @@ Status AddNoinlineFunctionToGraph(const string& node_name, Graph* graph) {
       },
       {{"add", "add_0:z:0"}});
   (*fdef.mutable_attr())["_noinline"].set_b(true);
+  return fdef;
+}
+
+// @function.Defun(noinline=True)
+// def increment_fn(x):
+//   return [x + 1]
+// Define the above function, and add it to the given graph. It's used as the
+// while loop body in NoinlineLoopBody test.
+Status AddNoinlineFunctionToGraph(const string& node_name, Graph* graph) {
   FunctionDefLibrary fdef_lib;
-  *(fdef_lib.add_function()) = fdef;
+  *(fdef_lib.add_function()) = GetNoinlineFunctionDef();
   TF_RETURN_IF_ERROR(graph->AddFunctionLibrary(fdef_lib));
   NodeDef increment_fn;
   increment_fn.set_name(node_name);
@@ -376,55 +397,88 @@ TEST(FunctionalizeControlFlow, NoinlineLoopBody) {
   FunctionLibraryDefinition lookup_lib(graph.flib_def());
   FunctionLibraryDefinition library(OpRegistry::Global(), {});
   // Function increment_fn will be copied from lookup_lib to library.
-  TF_ASSERT_OK(FunctionalizeControlFlow(&lookup_lib, &graph, &library));
+  GraphDef optimized_graph_def;
+  graph.ToGraphDef(&optimized_graph_def);
 
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
+  *(optimized_graph_def.mutable_library()->add_function()) =
+      GetNoinlineFunctionDef();
 
-  NameAttrList cond_fn, body_fn;
-  TF_ASSERT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
+  TF_ASSERT_OK(FunctionalizeControlFlowForGraphDef(
+      &lookup_lib, &optimized_graph_def, &library));
+  TF_ASSERT_OK(FunctionalizeControlFlow(&lookup_lib, &graph, &library));
+  GraphDef converted_graph_def;
+  graph.ToGraphDef(&converted_graph_def);
+
+  for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) {
+    NameAttrList cond_fn, body_fn;
+    TF_ASSERT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
+
+    // Outer graph
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
+      auto while_op =
+          ops::While(scope.WithOpName("while/LoopCond"),
+                     std::initializer_list<Input>{source}, cond_fn, body_fn);
+      GraphDef expected;
+      TF_ASSERT_OK(scope.ToGraphDef(&expected));
+      TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    }
 
-  // Outer graph
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
-    auto while_op =
-        ops::While(scope.WithOpName("while/LoopCond"),
-                   std::initializer_list<Input>{source}, cond_fn, body_fn);
-    GraphDef expected;
-    TF_ASSERT_OK(scope.ToGraphDef(&expected));
-    TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    // Body graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      TF_ASSERT_OK(
+          AddNoinlineFunctionToGraph(noinline_node_name, scope.graph()));
+      auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
+      NodeDef retval;
+      retval.set_name("_retval0_RetVal");
+      retval.set_op(FunctionLibraryDefinition::kRetOp);
+      *retval.add_input() = noinline_node_name;
+      (*retval.mutable_attr())["T"].set_type(DT_INT32);
+      (*retval.mutable_attr())["index"].set_i(0);
+      Status status;
+      scope.graph()->AddNode(retval, &status);
+      TF_ASSERT_OK(status);
+
+      GraphDef expected;
+      TF_ASSERT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      // Verify that increment_fn has been copied to library.
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(body_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+      // Ignore the function library when comparing the graphs.
+      expected.clear_library();
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
   }
+}
 
-  // Body graph.
+TEST(FunctionalizeControlFlow, MissingFunctionDefInLibrary) {
+  const string& noinline_node_name = "while/increment_fn";
+  Graph graph(OpRegistry::Global());
   {
     Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
+    auto identity = ops::Identity(scope.WithOpName("while/Identity"), source);
     TF_ASSERT_OK(AddNoinlineFunctionToGraph(noinline_node_name, scope.graph()));
-    auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
-    NodeDef retval;
-    retval.set_name("_retval0_RetVal");
-    retval.set_op(FunctionLibraryDefinition::kRetOp);
-    *retval.add_input() = noinline_node_name;
-    (*retval.mutable_attr())["T"].set_type(DT_INT32);
-    (*retval.mutable_attr())["index"].set_i(0);
-    Status status;
-    scope.graph()->AddNode(retval, &status);
-    TF_ASSERT_OK(status);
-
-    GraphDef expected;
-    TF_ASSERT_OK(scope.ToGraphDef(&expected));
+    TF_ASSERT_OK(scope.ToGraph(&graph));
+  }
 
-    InstantiationResultForTest result;
-    // Verify that increment_fn has been copied to library.
-    TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result));
+  FunctionLibraryDefinition lookup_lib(graph.flib_def());
+  FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  GraphDef graph_def;
+  graph.ToGraphDef(&graph_def);
+  graph_def.clear_library();
 
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    // Ignore the function library when comparing the graphs.
-    expected.clear_library();
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
+  Status status =
+      FunctionalizeControlFlowForGraphDef(&lookup_lib, &graph_def, &library);
+  EXPECT_EQ(tensorflow::error::NOT_FOUND, status.code());
 }
 
 // Tests functionalizing OneLoopVar where the loop value is not used post the
@@ -467,65 +521,72 @@ TEST(FunctionalizeControlFlow, OneLoopVarWithoutExit) {
   }
 
   FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  GraphDef optimized_graph_def;
+  graph.ToGraphDef(&optimized_graph_def);
+  TF_ASSERT_OK(
+      FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library));
   TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+  GraphDef converted_graph_def;
+  graph.ToGraphDef(&converted_graph_def);
+
+  for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) {
+    NameAttrList cond_fn, body_fn;
+    TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
+
+    // Outer graph
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
+      auto while_op =
+          ops::While(scope.WithOpName("while/LoopCond"),
+                     std::initializer_list<Input>{source}, cond_fn, body_fn);
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+      TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    }
 
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
-
-  NameAttrList cond_fn, body_fn;
-  TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
-
-  // Outer graph
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
-    auto while_op =
-        ops::While(scope.WithOpName("while/LoopCond"),
-                   std::initializer_list<Input>{source}, cond_fn, body_fn);
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-    TF_EXPECT_GRAPH_EQ(expected, graph_def);
-  }
-
-  // Condition graph
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto ten = ops::Const<int32>(
-        scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10);
-    auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result));
-
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
-
-  // Body graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
-    auto one = ops::Const<int32>(
-        scope.WithOpName("while/add/y").WithControlDependencies(identity), 1);
-    auto add = ops::Add(scope.WithOpName("while/add"), identity, one);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result));
+    // Condition graph
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto ten = ops::Const<int32>(
+          scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10);
+      auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten);
+      auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(cond_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    // Body graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
+      auto one = ops::Const<int32>(
+          scope.WithOpName("while/add/y").WithControlDependencies(identity), 1);
+      auto add = ops::Add(scope.WithOpName("while/add"), identity, one);
+      auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(body_fn.name(), library, &result));
+
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
   }
 }
 
@@ -608,86 +669,95 @@ TEST(FunctionalizeControlFlow, TwoLoopVars) {
   }
 
   FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  GraphDef optimized_graph_def;
+  graph.ToGraphDef(&optimized_graph_def);
+  TF_ASSERT_OK(
+      FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library));
   TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+  GraphDef converted_graph_def;
+  graph.ToGraphDef(&converted_graph_def);
+
+  for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) {
+    NameAttrList cond_fn, body_fn;
+    TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
+
+    // Outer graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto x = ops::Placeholder(scope.WithOpName("Placeholder/x"), DT_INT32);
+      auto y = ops::Placeholder(scope.WithOpName("Placeholder/y"), DT_INT32);
+      auto while_op =
+          ops::While(scope.WithOpName("while/LoopCond"),
+                     std::initializer_list<Input>{x, y}, cond_fn, body_fn);
+      auto sink_x = ops::Identity(scope.WithOpName("sink_x"), while_op[0]);
+      auto sink_y = ops::Identity(scope.WithOpName("sink_y"), while_op[1]);
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+      TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    }
 
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
-
-  NameAttrList cond_fn, body_fn;
-  TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
-
-  // Outer graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto x = ops::Placeholder(scope.WithOpName("Placeholder/x"), DT_INT32);
-    auto y = ops::Placeholder(scope.WithOpName("Placeholder/y"), DT_INT32);
-    auto while_op =
-        ops::While(scope.WithOpName("while/LoopCond"),
-                   std::initializer_list<Input>{x, y}, cond_fn, body_fn);
-    auto sink_x = ops::Identity(scope.WithOpName("sink_x"), while_op[0]);
-    auto sink_y = ops::Identity(scope.WithOpName("sink_y"), while_op[1]);
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-    TF_EXPECT_GRAPH_EQ(expected, graph_def);
-  }
-
-  // Condition graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto three = ops::Const<int32>(scope.WithOpName("while/cond/three")
+    // Condition graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto three = ops::Const<int32>(scope.WithOpName("while/cond/three")
+                                         .WithControlDependencies(arg0.output),
+                                     3);
+      auto cond_add =
+          ops::Add(scope.WithOpName("while/cond/Add"), arg0.output, three);
+      auto ten = ops::Const<int32>(scope.WithOpName("while/cond/ten")
                                        .WithControlDependencies(arg0.output),
-                                   3);
-    auto cond_add =
-        ops::Add(scope.WithOpName("while/cond/Add"), arg0.output, three);
-    auto ten = ops::Const<int32>(
-        scope.WithOpName("while/cond/ten").WithControlDependencies(arg0.output),
-        10);
-    auto less = ops::Less(scope.WithOpName("while/cond/Less"), cond_add, ten);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result));
-
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
-
-  // Body graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-
-    auto identity_x = ops::Identity(scope.WithOpName("while/Identity/x"), arg0);
-    auto identity_y = ops::Identity(scope.WithOpName("while/Identity/y"), arg1);
-
-    auto one = ops::Const<int32>(
-        scope.WithOpName("while/add/one").WithControlDependencies(identity_x),
-        1);
-    auto two = ops::Const<int32>(
-        scope.WithOpName("while/mul/two").WithControlDependencies(identity_x),
-        2);
+                                   10);
+      auto less = ops::Less(scope.WithOpName("while/cond/Less"), cond_add, ten);
+      auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
 
-    auto add = ops::Add(scope.WithOpName("while/add"), identity_x, one);
-    auto mul = ops::Add(scope.WithOpName("while/mul"), identity_y, two);
-    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
-    auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), mul, 1);
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
 
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(cond_fn.name(), library, &result));
 
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result));
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types);
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    // Body graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+
+      auto identity_x =
+          ops::Identity(scope.WithOpName("while/Identity/x"), arg0);
+      auto identity_y =
+          ops::Identity(scope.WithOpName("while/Identity/y"), arg1);
+
+      auto one = ops::Const<int32>(
+          scope.WithOpName("while/add/one").WithControlDependencies(identity_x),
+          1);
+      auto two = ops::Const<int32>(
+          scope.WithOpName("while/mul/two").WithControlDependencies(identity_x),
+          2);
+
+      auto add = ops::Add(scope.WithOpName("while/add"), identity_x, one);
+      auto mul = ops::Add(scope.WithOpName("while/mul"), identity_y, two);
+      auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
+      auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), mul, 1);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(body_fn.name(), library, &result));
+
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.arg_types);
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32}), result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
   }
 }
 
@@ -841,177 +911,192 @@ TEST(FunctionalizeControlFlow, Complex) {
   }
 
   FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  GraphDef optimized_graph_def;
+  graph.ToGraphDef(&optimized_graph_def);
+  TF_ASSERT_OK(
+      FunctionalizeControlFlowForGraphDef(&optimized_graph_def, &library));
   TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+  GraphDef converted_graph_def;
+  graph.ToGraphDef(&converted_graph_def);
 
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
-
-  NameAttrList outer_cond_fn, outer_body_fn;
-  TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &outer_cond_fn, &outer_body_fn));
-
-  // Outer graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
-    auto three = ops::Const<int32>(scope.WithOpName("three"), 3);
-    auto y = ops::Add(scope.WithOpName("y"), x, three);
-
-    auto var = ops::VarHandleOp(scope.WithOpName("Variable"), DT_INT32,
-                                TensorShape({}));
-
-    auto zero = ops::Const<int32>(scope.WithOpName("outer/Const"), 0);
-
-    auto while_op = ops::While(scope.WithOpName("outer/LoopCond"),
-                               std::initializer_list<Input>{zero, y, x, var},
-                               outer_cond_fn, outer_body_fn);
-    auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]);
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-    TF_EXPECT_GRAPH_EQ(expected, graph_def);
-  }
-
-  // Outer condition graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
-    auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
-
-    auto ten = ops::Const<int32>(
-        scope.WithOpName("outer/Less/y").WithControlDependencies(arg0.output),
-        10);
-    auto less = ops::Less(scope.WithOpName("outer/Less_i"), arg0, ten);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(
-        InstantiateFunctionForTest(outer_cond_fn.name(), library, &result));
-
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
-              result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
-
-  // Outer body graph.
-  NameAttrList inner_cond_fn, inner_body_fn;
-  {
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(
-        InstantiateFunctionForTest(outer_body_fn.name(), library, &result));
-
-    // Find the inner condition and body names.
-    TF_EXPECT_OK(
-        FindWhileCondAndBody(result.gdef, &inner_cond_fn, &inner_body_fn));
-
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
-    auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
-
-    auto identity_i = ops::Identity(scope.WithOpName("outer/Identity"), arg0);
-    auto one_j = ops::Const<int32>(
-        scope.WithOpName("outer/j").WithControlDependencies(identity_i), 1);
-    auto while_op =
-        ops::While(scope.WithOpName("outer/LoopCond_1"),
-                   std::initializer_list<Input>{one_j, arg1, arg2, arg3},
-                   inner_cond_fn, inner_body_fn);
-
-    auto one_outer = ops::Const<int32>(
-        scope.WithOpName("outer/add/y").WithControlDependencies(identity_i), 1);
-    auto add_i =
-        ops::Add(scope.WithOpName("outer/add")
-                     .WithControlDependencies(absl::Span<const Operation>{
-                         while_op[0].op(), while_op[1].op()}),
-                 identity_i, one_outer);
-
-    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_i, 0);
-    auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), arg1, 1);
-    auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
-              result.arg_types);
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}), result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
-
-  // Inner condition graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
-    auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
-
-    auto five = ops::Const<int32>(
-        scope.WithOpName("outer/inner/Five").WithControlDependencies(arg0), 5);
-    auto less_j = ops::Less(scope.WithOpName("outer/inner/Less_j"), arg0, five);
-    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less_j, 0);
-
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
-
-    InstantiationResultForTest result;
+  for (const GraphDef& graph_def : {optimized_graph_def, converted_graph_def}) {
+    NameAttrList outer_cond_fn, outer_body_fn;
     TF_EXPECT_OK(
-        InstantiateFunctionForTest(inner_cond_fn.name(), library, &result));
-
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
-              result.arg_types);
-    EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
-  }
-
-  // Inner body graph.
-  {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
-    auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
-    auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
-
-    auto identity_j =
-        ops::Identity(scope.WithOpName("outer/inner/Identity_j"), arg0);
-    auto identity_k =
-        ops::Identity(scope.WithOpName("outer/inner/Identity_k"), arg1);
-
-    auto mul_jk =
-        ops::Mul(scope.WithOpName("outer/inner/mul"), identity_j, identity_k);
-    auto add_jkx = ops::Add(scope.WithOpName("outer/inner/add"), mul_jk, arg2);
-    auto assign = ops::AssignAddVariableOp(
-        scope.WithOpName("outer/inner/assign_add"), arg3, add_jkx);
-
-    auto one = ops::Const<int32>(
-        scope.WithOpName("outer/inner/One")
-            .WithControlDependencies(
-                absl::Span<const Operation>{assign.operation}),
-        1);
-    auto add_j =
-        ops::Add(scope.WithOpName("outer/inner/add_j"), identity_j, one);
+        FindWhileCondAndBody(graph_def, &outer_cond_fn, &outer_body_fn));
+
+    // Outer graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
+      auto three = ops::Const<int32>(scope.WithOpName("three"), 3);
+      auto y = ops::Add(scope.WithOpName("y"), x, three);
+
+      auto var = ops::VarHandleOp(scope.WithOpName("Variable"), DT_INT32,
+                                  TensorShape({}));
+
+      auto zero = ops::Const<int32>(scope.WithOpName("outer/Const"), 0);
+
+      auto while_op = ops::While(scope.WithOpName("outer/LoopCond"),
+                                 std::initializer_list<Input>{zero, y, x, var},
+                                 outer_cond_fn, outer_body_fn);
+      auto sink = ops::Identity(scope.WithOpName("sink"), while_op[0]);
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+      TF_EXPECT_GRAPH_EQ(expected, graph_def);
+    }
 
-    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_j, 0);
-    auto retval1 =
-        ops::_Retval(scope.WithOpName("_retval1_RetVal"), identity_k, 1);
-    auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2);
+    // Outer condition graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+      auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
+
+      auto ten = ops::Const<int32>(
+          scope.WithOpName("outer/Less/y").WithControlDependencies(arg0.output),
+          10);
+      auto less = ops::Less(scope.WithOpName("outer/Less_i"), arg0, ten);
+      auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(outer_cond_fn.name(), library, &result));
+
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
+                result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-    GraphDef expected;
-    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+    // Outer body graph.
+    NameAttrList inner_cond_fn, inner_body_fn;
+    {
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(outer_body_fn.name(), library, &result));
+
+      // Find the inner condition and body names.
+      TF_EXPECT_OK(
+          FindWhileCondAndBody(result.gdef, &inner_cond_fn, &inner_body_fn));
+
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+      auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
+
+      auto identity_i = ops::Identity(scope.WithOpName("outer/Identity"), arg0);
+      auto one_j = ops::Const<int32>(
+          scope.WithOpName("outer/j").WithControlDependencies(identity_i), 1);
+      auto while_op =
+          ops::While(scope.WithOpName("outer/LoopCond_1"),
+                     std::initializer_list<Input>{one_j, arg1, arg2, arg3},
+                     inner_cond_fn, inner_body_fn);
+
+      auto one_outer = ops::Const<int32>(
+          scope.WithOpName("outer/add/y").WithControlDependencies(identity_i),
+          1);
+      auto add_i =
+          ops::Add(scope.WithOpName("outer/add")
+                       .WithControlDependencies(absl::Span<const Operation>{
+                           while_op[0].op(), while_op[1].op()}),
+                   identity_i, one_outer);
+
+      auto retval0 =
+          ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_i, 0);
+      auto retval1 = ops::_Retval(scope.WithOpName("_retval1_RetVal"), arg1, 1);
+      auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
+                result.arg_types);
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}),
+                result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-    InstantiationResultForTest result;
-    TF_EXPECT_OK(
-        InstantiateFunctionForTest(inner_body_fn.name(), library, &result));
+    // Inner condition graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+      auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
+
+      auto five = ops::Const<int32>(
+          scope.WithOpName("outer/inner/Five").WithControlDependencies(arg0),
+          5);
+      auto less_j =
+          ops::Less(scope.WithOpName("outer/inner/Less_j"), arg0, five);
+      auto retval =
+          ops::_Retval(scope.WithOpName("_retval0_RetVal"), less_j, 0);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(inner_cond_fn.name(), library, &result));
+
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
+                result.arg_types);
+      EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
 
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
-              result.arg_types);
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}), result.ret_types);
-    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    // Inner body graph.
+    {
+      Scope scope = Scope::NewRootScope().ExitOnError();
+      auto arg0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+      auto arg1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+      auto arg2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+      auto arg3 = ops::_Arg(scope.WithOpName("_arg3"), DT_RESOURCE, 3);
+
+      auto identity_j =
+          ops::Identity(scope.WithOpName("outer/inner/Identity_j"), arg0);
+      auto identity_k =
+          ops::Identity(scope.WithOpName("outer/inner/Identity_k"), arg1);
+
+      auto mul_jk =
+          ops::Mul(scope.WithOpName("outer/inner/mul"), identity_j, identity_k);
+      auto add_jkx =
+          ops::Add(scope.WithOpName("outer/inner/add"), mul_jk, arg2);
+      auto assign = ops::AssignAddVariableOp(
+          scope.WithOpName("outer/inner/assign_add"), arg3, add_jkx);
+
+      auto one = ops::Const<int32>(
+          scope.WithOpName("outer/inner/One")
+              .WithControlDependencies(
+                  absl::Span<const Operation>{assign.operation}),
+          1);
+      auto add_j =
+          ops::Add(scope.WithOpName("outer/inner/add_j"), identity_j, one);
+
+      auto retval0 =
+          ops::_Retval(scope.WithOpName("_retval0_RetVal"), add_j, 0);
+      auto retval1 =
+          ops::_Retval(scope.WithOpName("_retval1_RetVal"), identity_k, 1);
+      auto retval2 = ops::_Retval(scope.WithOpName("_retval2_RetVal"), arg2, 2);
+
+      GraphDef expected;
+      TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+      InstantiationResultForTest result;
+      TF_EXPECT_OK(
+          InstantiateFunctionForTest(inner_body_fn.name(), library, &result));
+
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32, DT_RESOURCE}),
+                result.arg_types);
+      EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_INT32}),
+                result.ret_types);
+      TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+    }
   }
 }
 
-- 
GitLab


From 2226a4d599224d4759844db5c80460fafd87145f Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Thu, 15 Nov 2018 17:04:23 -0800
Subject: [PATCH 0366/1554] LinearOperatorInversion added as a meta-class. 
 Computes the naive inverse of a non-singular operator by swapping .solve and
 .matmul, and other appropriate modifications.

This is hidden from the public API, with an upcoming method, .inverse, the preferred way to get an inverse, since it has more operator specific information.

PiperOrigin-RevId: 221716123
---
 tensorflow/python/kernel_tests/linalg/BUILD   |  22 ++
 .../linalg/linear_operator_inversion_test.py  | 130 +++++++++++
 .../ops/linalg/linear_operator_inversion.py   | 207 ++++++++++++++++++
 3 files changed, 359 insertions(+)
 create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py
 create mode 100644 tensorflow/python/ops/linalg/linear_operator_inversion.py

diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index c2f14a119a..d90e65526a 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -167,6 +167,28 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "linear_operator_inversion_test",
+    size = "medium",
+    srcs = ["linear_operator_inversion_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+    shard_count = 5,
+    tags = [
+        "noasan",  # times out, b/63678675
+        "optonly",  # times out
+    ],
+)
+
 cuda_py_test(
     name = "linear_operator_full_matrix_test",
     size = "medium",
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py
new file mode 100644
index 0000000000..9344c526ee
--- /dev/null
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_inversion_test.py
@@ -0,0 +1,130 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_inversion
+from tensorflow.python.ops.linalg import linear_operator_test_util
+from tensorflow.python.platform import test
+
+linalg = linalg_lib
+
+LinearOperatorInversion = linear_operator_inversion.LinearOperatorInversion  # pylint: disable=invalid-name
+
+
+class LinearOperatorInversionTest(
+    linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
+  """Most tests done in the base class LinearOperatorDerivedClassTest."""
+
+  def setUp(self):
+    self._atol[dtypes.complex64] = 1e-5
+    self._rtol[dtypes.complex64] = 1e-5
+
+  def _operator_and_matrix(self,
+                           build_info,
+                           dtype,
+                           use_placeholder,
+                           ensure_self_adjoint_and_pd=False):
+    shape = list(build_info.shape)
+
+    if ensure_self_adjoint_and_pd:
+      matrix = linear_operator_test_util.random_positive_definite_matrix(
+          shape, dtype, force_well_conditioned=True)
+    else:
+      matrix = linear_operator_test_util.random_tril_matrix(
+          shape, dtype, force_well_conditioned=True, remove_upper=True)
+
+    lin_op_matrix = matrix
+
+    if use_placeholder:
+      lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None)
+
+    if ensure_self_adjoint_and_pd:
+      operator = LinearOperatorInversion(
+          linalg.LinearOperatorFullMatrix(
+              lin_op_matrix, is_positive_definite=True, is_self_adjoint=True))
+    else:
+      operator = LinearOperatorInversion(
+          linalg.LinearOperatorLowerTriangular(lin_op_matrix))
+
+    return operator, linalg.inv(matrix)
+
+  def test_base_operator_hint_used(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 0.], [1., 1.]]
+    operator = linalg.LinearOperatorFullMatrix(
+        matrix,
+        is_positive_definite=True,
+        is_non_singular=True,
+        is_self_adjoint=False)
+    operator_inv = LinearOperatorInversion(operator)
+    self.assertTrue(operator_inv.is_positive_definite)
+    self.assertTrue(operator_inv.is_non_singular)
+    self.assertFalse(operator_inv.is_self_adjoint)
+
+  def test_supplied_hint_used(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 0.], [1., 1.]]
+    operator = linalg.LinearOperatorFullMatrix(matrix)
+    operator_inv = LinearOperatorInversion(
+        operator,
+        is_positive_definite=True,
+        is_non_singular=True,
+        is_self_adjoint=False)
+    self.assertTrue(operator_inv.is_positive_definite)
+    self.assertTrue(operator_inv.is_non_singular)
+    self.assertFalse(operator_inv.is_self_adjoint)
+
+  def test_contradicting_hints_raise(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 0.], [1., 1.]]
+    operator = linalg.LinearOperatorFullMatrix(
+        matrix, is_positive_definite=False)
+    with self.assertRaisesRegexp(ValueError, "positive-definite"):
+      LinearOperatorInversion(operator, is_positive_definite=True)
+
+    operator = linalg.LinearOperatorFullMatrix(matrix, is_self_adjoint=False)
+    with self.assertRaisesRegexp(ValueError, "self-adjoint"):
+      LinearOperatorInversion(operator, is_self_adjoint=True)
+
+  def test_singular_raises(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 1.], [1., 1.]]
+
+    operator = linalg.LinearOperatorFullMatrix(matrix, is_non_singular=False)
+    with self.assertRaisesRegexp(ValueError, "is_non_singular"):
+      LinearOperatorInversion(operator)
+
+    operator = linalg.LinearOperatorFullMatrix(matrix)
+    with self.assertRaisesRegexp(ValueError, "is_non_singular"):
+      LinearOperatorInversion(operator, is_non_singular=False)
+
+  def test_name(self):
+    matrix = [[11., 0.], [1., 8.]]
+    operator = linalg.LinearOperatorFullMatrix(
+        matrix, name="my_operator", is_non_singular=True)
+
+    operator = LinearOperatorInversion(operator)
+
+    self.assertEqual("my_operator_inv", operator.name)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/linalg/linear_operator_inversion.py b/tensorflow/python/ops/linalg/linear_operator_inversion.py
new file mode 100644
index 0000000000..7aa4b40e16
--- /dev/null
+++ b/tensorflow/python/ops/linalg/linear_operator_inversion.py
@@ -0,0 +1,207 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Inverts a non-singular `LinearOperator`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.util.tf_export import tf_export
+
+__all__ = []
+
+
+@tf_export("linalg.LinearOperatorInversion")
+class LinearOperatorInversion(linear_operator.LinearOperator):
+  """`LinearOperator` representing the inverse of another operator.
+
+  This operator represents the inverse of another operator.
+
+  ```python
+  # Create a 2 x 2 linear operator.
+  operator = LinearOperatorFullMatrix([[1., 0.], [0., 2.]])
+  operator_inv = LinearOperatorInversion(operator)
+
+  operator_inv.to_dense()
+  ==> [[1., 0.]
+       [0., 0.5]]
+
+  operator_inv.shape
+  ==> [2, 2]
+
+  operator_inv.log_abs_determinant()
+  ==> - log(2)
+
+  x = ... Shape [2, 4] Tensor
+  operator_inv.matmul(x)
+  ==> Shape [2, 4] Tensor, equal to operator.solve(x)
+  ```
+
+  #### Performance
+
+  The performance of `LinearOperatorInversion` depends on the underlying
+  operators performance:  `solve` and `matmul` are swapped, and determinant is
+  inverted.
+
+  #### Matrix property hints
+
+  This `LinearOperator` is initialized with boolean flags of the form `is_X`,
+  for `X = non_singular, self_adjoint, positive_definite, square`.
+  These have the following meaning:
+
+  * If `is_X == True`, callers should expect the operator to have the
+    property `X`.  This is a promise that should be fulfilled, but is *not* a
+    runtime assert.  For example, finite floating point precision may result
+    in these promises being violated.
+  * If `is_X == False`, callers should expect the operator to not have `X`.
+  * If `is_X == None` (the default), callers should have no expectation either
+    way.
+  """
+
+  def __init__(self,
+               operator,
+               is_non_singular=None,
+               is_self_adjoint=None,
+               is_positive_definite=None,
+               is_square=None,
+               name=None):
+    r"""Initialize a `LinearOperatorInversion`.
+
+    `LinearOperatorInversion` is initialized with an operator `A`.  The `solve`
+    and `matmul` methods are effectively swapped.  E.g.
+
+    ```
+    A = MyLinearOperator(...)
+    B = LinearOperatorInversion(A)
+    x = [....]  # a vector
+
+    assert A.matvec(x) == B.solvevec(x)
+    ```
+
+    Args:
+      operator: `LinearOperator` object. If `operator.is_non_singular == False`,
+        an exception is raised.  We do allow `operator.is_non_singular == None`,
+        in which case this operator will have `is_non_singular == None`.
+        Similarly for `is_self_adjoint` and `is_positive_definite`.
+      is_non_singular:  Expect that this operator is non-singular.
+      is_self_adjoint:  Expect that this operator is equal to its hermitian
+        transpose.
+      is_positive_definite:  Expect that this operator is positive definite,
+        meaning the quadratic form `x^H A x` has positive real part for all
+        nonzero `x`.  Note that we do not require the operator to be
+        self-adjoint to be positive-definite.  See:
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
+      is_square:  Expect that this operator acts like square [batch] matrices.
+      name: A name for this `LinearOperator`. Default is `operator.name +
+        "_inv"`.
+
+    Raises:
+      ValueError:  If `operator.is_non_singular` is False.
+    """
+
+    self._operator = operator
+
+    # Auto-set and check hints.
+    if operator.is_non_singular is False or is_non_singular is False:
+      raise ValueError(
+          "operator and supplied hints must have `is_non_singular` equal to "
+          "`True` or `None`.  Found %s, %s" % (operator.is_non_singular,
+                                               is_non_singular))
+    if operator.is_square is False or is_square is False:
+      raise ValueError(
+          "operator and supplied hints must have `is_square` equal to "
+          "`True` or `None`.  Found %s, %s" % (operator.is_square, is_square))
+
+    # The congruency of is_non_singular and is_self_adjoint was checked in the
+    # base operator.  Other hints are, in this special case of inversion, ones
+    # that must be the same for base/derived operator.
+    def _combined_hint(hint_str, provided_hint_value, message):
+      """Get combined hint in the case where operator.hint should equal hint."""
+      op_hint = getattr(operator, hint_str)
+      if op_hint is False and provided_hint_value:
+        raise ValueError(message)
+      if op_hint and provided_hint_value is False:
+        raise ValueError(message)
+      return (op_hint or provided_hint_value) or None
+
+    is_square = _combined_hint(
+        "is_square", is_square,
+        "An operator is square if and only if its inverse is square.")
+
+    is_non_singular = _combined_hint(
+        "is_non_singular", is_non_singular,
+        "An operator is non-singular if and only if its inverse is "
+        "non-singular.")
+
+    is_self_adjoint = _combined_hint(
+        "is_self_adjoint", is_self_adjoint,
+        "An operator is self-adjoint if and only if its inverse is "
+        "self-adjoint.")
+
+    is_positive_definite = _combined_hint(
+        "is_positive_definite", is_positive_definite,
+        "An operator is positive-definite if and only if its inverse is "
+        "positive-definite.")
+
+    is_square = _combined_hint(
+        "is_square", is_square,
+        "An operator is square if and only if its inverse is square.")
+
+    # Initialization.
+    if name is None:
+      name = operator.name + "_inv"
+    with ops.name_scope(name, values=operator.graph_parents):
+      super(LinearOperatorInversion, self).__init__(
+          dtype=operator.dtype,
+          graph_parents=operator.graph_parents,
+          is_non_singular=is_non_singular,
+          is_self_adjoint=is_self_adjoint,
+          is_positive_definite=is_positive_definite,
+          is_square=is_square,
+          name=name)
+
+  @property
+  def operator(self):
+    """The operator before inversion."""
+    return self._operator
+
+  def _assert_non_singular(self):
+    return self.operator.assert_non_singular()
+
+  def _assert_positive_definite(self):
+    return self.operator.assert_positive_definite()
+
+  def _assert_self_adjoint(self):
+    return self.operator.assert_self_adjoint()
+
+  def _shape(self):
+    return self.operator.shape
+
+  def _shape_tensor(self):
+    return self.operator.shape_tensor()
+
+  def _matmul(self, x, adjoint=False, adjoint_arg=False):
+    return self.operator.solve(x, adjoint=adjoint, adjoint_arg=adjoint_arg)
+
+  def _determinant(self):
+    return 1. / self.operator.determinant()
+
+  def _log_abs_determinant(self):
+    return -1. * self.operator.log_abs_determinant()
+
+  def _solve(self, rhs, adjoint=False, adjoint_arg=False):
+    return self.operator.matmul(rhs, adjoint=adjoint, adjoint_arg=adjoint_arg)
-- 
GitLab


From 073e29df4b2b2b9b9d0bea5191229deb1462ebac Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Thu, 15 Nov 2018 17:51:48 -0800
Subject: [PATCH 0367/1554] Update the name for `tf.sets.set_*` to `tf.sets.*`
 and exporting to v2 and v1.

PiperOrigin-RevId: 221722355
---
 tensorflow/python/ops/sets_impl.py               | 11 +++++++----
 .../tools/api/golden/v1/tensorflow.sets.pbtxt    | 16 ++++++++++++++++
 .../tools/api/golden/v2/tensorflow.sets.pbtxt    |  8 ++++----
 tensorflow/tools/compatibility/renames_v2.py     |  4 ++++
 4 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/sets_impl.py b/tensorflow/python/ops/sets_impl.py
index 21e08d03d2..ee9c9b6bc0 100644
--- a/tensorflow/python/ops/sets_impl.py
+++ b/tensorflow/python/ops/sets_impl.py
@@ -31,7 +31,7 @@ _VALID_DTYPES = set([
     dtypes.uint8, dtypes.uint16, dtypes.string])
 
 
-@tf_export("sets.set_size")
+@tf_export("sets.size", v1=["sets.size", "sets.set_size"])
 def set_size(a, validate_indices=True):
   """Compute number of unique elements along last dimension of `a`.
 
@@ -133,7 +133,8 @@ def _set_operation(a, b, set_operation, validate_indices=True):
   return sparse_tensor.SparseTensor(indices, values, shape)
 
 
-@tf_export("sets.set_intersection")
+@tf_export(
+    "sets.intersection", v1=["sets.intersection", "sets.set_intersection"])
 def set_intersection(a, b, validate_indices=True):
   """Compute set intersection of elements in last dimension of `a` and `b`.
 
@@ -200,7 +201,8 @@ def set_intersection(a, b, validate_indices=True):
   return _set_operation(a, b, "intersection", validate_indices)
 
 
-@tf_export("sets.set_difference")
+@tf_export(
+	   "sets.difference", v1=["sets.difference", "sets.set_difference"])
 def set_difference(a, b, aminusb=True, validate_indices=True):
   """Compute set difference of elements in last dimension of `a` and `b`.
 
@@ -271,7 +273,8 @@ def set_difference(a, b, aminusb=True, validate_indices=True):
   return _set_operation(a, b, "a-b" if aminusb else "b-a", validate_indices)
 
 
-@tf_export("sets.set_union")
+@tf_export(
+	   "sets.union", v1=["sets.union", "sets.set_union"])
 def set_union(a, b, validate_indices=True):
   """Compute set union of elements in last dimension of `a` and `b`.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt
index 8a196b1a55..09d6f1424b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.sets.pbtxt
@@ -1,5 +1,13 @@
 path: "tensorflow.sets"
 tf_module {
+  member_method {
+    name: "difference"
+    argspec: "args=[\'a\', \'b\', \'aminusb\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], "
+  }
+  member_method {
+    name: "intersection"
+    argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
+  }
   member_method {
     name: "set_difference"
     argspec: "args=[\'a\', \'b\', \'aminusb\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], "
@@ -16,4 +24,12 @@ tf_module {
     name: "set_union"
     argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
   }
+  member_method {
+    name: "size"
+    argspec: "args=[\'a\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
+  }
+  member_method {
+    name: "union"
+    argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt
index 8a196b1a55..900d08ff47 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sets.pbtxt
@@ -1,19 +1,19 @@
 path: "tensorflow.sets"
 tf_module {
   member_method {
-    name: "set_difference"
+    name: "difference"
     argspec: "args=[\'a\', \'b\', \'aminusb\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\', \'True\'], "
   }
   member_method {
-    name: "set_intersection"
+    name: "intersection"
     argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
   }
   member_method {
-    name: "set_size"
+    name: "size"
     argspec: "args=[\'a\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
   }
   member_method {
-    name: "set_union"
+    name: "union"
     argspec: "args=[\'a\', \'b\', \'validate_indices\'], varargs=None, keywords=None, defaults=[\'True\'], "
   }
 }
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index a653623a27..7ba6982f97 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -419,6 +419,10 @@ renames = {
     'tf.serialize_sparse': 'tf.io.serialize_sparse',
     'tf.serialize_tensor': 'tf.io.serialize_tensor',
     'tf.setdiff1d': 'tf.compat.v1.setdiff1d',
+    'tf.sets.set_difference': 'tf.sets.difference',
+    'tf.sets.set_intersection': 'tf.sets.intersection',
+    'tf.sets.set_size': 'tf.sets.size',
+    'tf.sets.set_union': 'tf.sets.union',
     'tf.space_to_batch': 'tf.nn.space_to_batch',
     'tf.space_to_depth': 'tf.nn.space_to_depth',
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
-- 
GitLab


From 668afe69dba9106c76682e6e169e2334a50ac87a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 17:57:07 -0800
Subject: [PATCH 0368/1554] BREAKING: Make the `gain` argument of convolutional
 orthogonal initializers (`convolutional_delta_orthogonal`,
 `convolutional_orthogonal_1D`, `convolutional_orthogonal_2D`,
 `convolutional_orthogonal_3D`) have a consistent behavior with the
 `tf.initializers.orthogonal` initializer, i.e. scale the output l2-norm by
 `gain` and NOT by `sqrt(gain)`. PiperOrigin-RevId: 221723104

---
 .../python/kernel_tests/init_ops_test.py      | 19 +++---
 tensorflow/python/ops/init_ops.py             | 59 ++++++++-----------
 2 files changed, 33 insertions(+), 45 deletions(-)

diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 70bfbf8544..3f3fcfd2ff 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -616,7 +616,7 @@ class OrthogonalInitializerTest(test.TestCase):
       with self.session(graph=ops.Graph(), use_gpu=True):
         t1 = init1(shape).eval()
         t2 = init2(shape).eval()
-      return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15)
+      self.assertAllClose(t1, t2 / 3.14)
 
   def testShapesValues(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -674,7 +674,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
       with self.session(graph=ops.Graph(), use_gpu=True):
         t1 = init1(shape).eval()
         t2 = init2(shape).eval()
-      return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15)
+      self.assertAllClose(t1, t2 / 3.14)
 
   def testShapesValues(self):
     gain = 3.14
@@ -709,8 +709,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
           t = outputs.eval()
           self.assertAllEqual(t.shape, outputs_shape)
           # Check isometry of the delta-orthogonal kernel.
-          self.assertAllClose(sess.run(ratio), np.sqrt(gain),
-                              rtol=tol, atol=tol)
+          self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
   def testNonuniformity(self):
     value = 0
@@ -774,7 +773,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
       with self.session(graph=ops.Graph(), use_gpu=True):
         t1 = init1(shape).eval()
         t2 = init2(shape).eval()
-      return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15)
+      self.assertAllClose(t1, t2 / 3.14)
 
   def testNonuniformity(self):
     value = 0
@@ -848,7 +847,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
         t = outputs.eval()
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), np.sqrt(gain), rtol=tol, atol=tol)
+        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
@@ -888,7 +887,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
       with self.session(graph=ops.Graph(), use_gpu=True):
         t1 = init1(shape).eval()
         t2 = init2(shape).eval()
-      return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15)
+      self.assertAllClose(t1, t2 / 3.14)
 
   def testShapesValues(self):
     def circular_pad(input_, width, kernel_size):
@@ -943,7 +942,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
         t = outputs.eval()
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), np.sqrt(gain), rtol=tol, atol=tol)
+        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
@@ -983,7 +982,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
       with self.session(graph=ops.Graph(), use_gpu=True):
         t1 = init1(shape).eval()
         t2 = init2(shape).eval()
-      return np.allclose(t1, t2 / 3.14, rtol=1e-15, atol=1e-15)
+      self.assertAllClose(t1, t2 / 3.14)
 
   def testNonuniformity(self):
     value = 0
@@ -1068,7 +1067,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
         t = outputs.eval()
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), np.sqrt(gain), rtol=tol, atol=tol)
+        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
 
 class IdentityInitializerTest(test.TestCase):
diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py
index 4fe6d05620..5a1ac675db 100644
--- a/tensorflow/python/ops/init_ops.py
+++ b/tensorflow/python/ops/init_ops.py
@@ -580,9 +580,9 @@ class ConvolutionDeltaOrthogonal(Initializer):
 
 
   Args:
-    gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
-      The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after
-      applying this convolution.
+    gain: Multiplicative factor to apply to the orthogonal
+      matrix. Default is 1. The 2-norm of an input is multiplied by a factor of
+      `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
@@ -613,7 +613,7 @@ class ConvolutionDeltaOrthogonal(Initializer):
     d = array_ops.diag_part(r)
     q *= math_ops.sign(d)
     q = q[:shape[-2], :]
-    q *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype))
+    q *= math_ops.cast(self.gain, dtype=dtype)
     if len(shape) == 3:
       weight = array_ops.scatter_nd([[(shape[0]-1)//2]],
                                     array_ops.expand_dims(q, 0), shape)
@@ -636,9 +636,9 @@ class ConvolutionOrthogonal(Initializer):
   Base class used to construct 1D, 2D and 3D orthogonal kernels for convolution.
 
   Args:
-    gain: multiplicative factor to apply to the orthogonal matrix. Default is 1.
-      The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after
-      applying this convolution.
+    gain: multiplicative factor to apply to the orthogonal
+      matrix. Default is 1. The 2-norm of an input is multiplied by a factor of
+      `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
@@ -701,9 +701,9 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal):
   See algorithm 1 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393
 
   Args:
-    gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
-      This has the effect of scaling the output 2-norm by a factor of
-      `sqrt(gain)`.
+    gain: Multiplicative factor to apply to the orthogonal
+      matrix. Default is 1. This has the effect of scaling the output 2-norm by
+      a factor of `gain`.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
@@ -722,7 +722,7 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal):
       raise ValueError("Kernel sizes must be equal.")
 
     kernel = self._orthogonal_kernel(shape[0], shape[2], shape[3])
-    kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype))
+    kernel *= math_ops.cast(self.gain, dtype=dtype)
     return kernel
 
   def _dict_to_tensor(self, x, k1, k2):
@@ -837,9 +837,9 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal):
   See algorithm 1 in [Xiao et al., 2018]: https://arxiv.org/abs/1806.05393
 
   Args:
-    gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
-      The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after
-      applying this convolution.
+    gain: Multiplicative factor to apply to the orthogonal
+      matrix. Default is 1. The 2-norm of an input is multiplied by a factor of
+      `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
@@ -856,7 +856,7 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal):
       raise ValueError("In_filters cannot be greater than out_filters.")
 
     kernel = self._orthogonal_kernel(shape[0], shape[-2], shape[-1])
-    kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype))
+    kernel *= math_ops.cast(self.gain, dtype=dtype)
     return kernel
 
   def _dict_to_tensor(self, x, k):
@@ -954,9 +954,9 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal):
   See algorithm 1 [Xiao et al., 2018] in: https://arxiv.org/abs/1806.05393
 
   Args:
-    gain: Multiplicative factor to apply to the orthogonal matrix. Default is 1.
-      The 2-norm of an input is multiplied by a factor of 'sqrt(gain)' after
-      applying this convolution.
+    gain: Multiplicative factor to apply to the orthogonal
+      matrix. Default is 1. The 2-norm of an input is multiplied by a factor of
+      `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
     dtype: The data type.
@@ -975,7 +975,7 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal):
       raise ValueError("Kernel sizes must be equal.")
 
     kernel = self._orthogonal_kernel(shape[0], shape[-2], shape[-1])
-    kernel *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype))
+    kernel *= math_ops.cast(self.gain, dtype=dtype)
     return kernel
 
   def _dict_to_tensor(self, x, k1, k2, k3):
@@ -1148,9 +1148,7 @@ class GlorotUniform(VarianceScaling):
     dtype: The data type. Only floating point types are supported.
   """
 
-  def __init__(self,
-               seed=None,
-               dtype=dtypes.float32):
+  def __init__(self, seed=None, dtype=dtypes.float32):
     super(GlorotUniform, self).__init__(
         scale=1.0,
         mode="fan_avg",
@@ -1159,10 +1157,7 @@ class GlorotUniform(VarianceScaling):
         dtype=dtype)
 
   def get_config(self):
-    return {
-        "seed": self.seed,
-        "dtype": self.dtype.name
-    }
+    return {"seed": self.seed, "dtype": self.dtype.name}
 
 
 @tf_export(
@@ -1185,14 +1180,11 @@ class GlorotNormal(VarianceScaling):
 
   Args:
     seed: A Python integer. Used to create random seeds. See
-      `tf.set_random_seed`
-      for behavior.
+      `tf.set_random_seed` for behavior.
     dtype: The data type. Only floating point types are supported.
   """
 
-  def __init__(self,
-               seed=None,
-               dtype=dtypes.float32):
+  def __init__(self, seed=None, dtype=dtypes.float32):
     super(GlorotNormal, self).__init__(
         scale=1.0,
         mode="fan_avg",
@@ -1201,10 +1193,7 @@ class GlorotNormal(VarianceScaling):
         dtype=dtype)
 
   def get_config(self):
-    return {
-        "seed": self.seed,
-        "dtype": self.dtype.name
-    }
+    return {"seed": self.seed, "dtype": self.dtype.name}
 
 
 # Aliases.
-- 
GitLab


From 7830dce1d574eadfa2c62782acee8904db0aaaff Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Thu, 15 Nov 2018 18:21:08 -0800
Subject: [PATCH 0369/1554] Dynamic parameter binding.

Dynamic parameter binding is the minimum API to start working on
dynamic padder and unblock the bridge to start integrating dynamic
shapes features.

PiperOrigin-RevId: 221726042
---
 tensorflow/compiler/xla/client/xla_builder.cc |  16 ++
 tensorflow/compiler/xla/client/xla_builder.h  |  16 ++
 tensorflow/compiler/xla/service/BUILD         |  21 +++
 .../xla/service/dynamic_parameter_binding.cc  | 138 ++++++++++++++++
 .../xla/service/dynamic_parameter_binding.h   | 125 ++++++++++++++
 .../service/dynamic_parameter_binding_test.cc | 153 ++++++++++++++++++
 tensorflow/compiler/xla/service/hlo.proto     |  37 +++++
 tensorflow/compiler/xla/service/hlo_module.cc |   6 +
 tensorflow/compiler/xla/service/hlo_module.h  |  14 ++
 .../compiler/xla/service/hlo_verifier.cc      |   2 +
 tensorflow/compiler/xla/shape_util.h          |   5 +
 11 files changed, 533 insertions(+)
 create mode 100644 tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
 create mode 100644 tensorflow/compiler/xla/service/dynamic_parameter_binding.h
 create mode 100644 tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 0a587725d2..f508ffb9c9 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -239,6 +239,19 @@ void XlaBuilder::IsConstantVisitor(const int64 op_handle,
   visited->insert(op_handle);
 }
 
+Status XlaBuilder::SetDynamicBinding(int64 dynamic_size_param_num,
+                                     ShapeIndex dynamic_size_param_index,
+                                     int64 target_param_num,
+                                     ShapeIndex target_param_index,
+                                     int64 target_dim_num) {
+  TF_RETURN_IF_ERROR(dynamic_parameter_binding_.Bind(
+      DynamicParameterBinding::DynamicParameter{dynamic_size_param_num,
+                                                dynamic_size_param_index},
+      DynamicParameterBinding::DynamicDimension{
+          target_param_num, target_param_index, target_dim_num}));
+  return Status::OK();
+}
+
 XlaComputation XlaBuilder::BuildAndNoteError() {
   DCHECK(parent_builder_ != nullptr);
   auto build_status = Build();
@@ -297,6 +310,9 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
   }
   module->add_computations()->Swap(&entry);
 
+  *(module->mutable_dynamic_parameter_binding()) =
+      dynamic_parameter_binding_.ToProto();
+
   // Clear data held by this builder.
   this->instructions_.clear();
   this->handle_to_index_.clear();
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 68314a026e..e6ee41b300 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -263,6 +264,18 @@ class XlaBuilder {
   // evaluating the computation.
   StatusOr<bool> IsConstant(const XlaOp& operand) const;
 
+  // Sets up binding which indicates that the `target_dim_num` in the subshape
+  // `target_param_index` of parameter `target_param_num` is a dynamic dimension
+  // and its real dynamic size is represented by `dynamic_param_index` in
+  // parameter `dynamic_param_num`.
+  //
+  // TODO(b/119520625): Remove this API once we have more dynamic shape infra
+  // ready.
+  Status SetDynamicBinding(int64 dynamic_size_param_num,
+                           ShapeIndex dynamic_size_param_index,
+                           int64 target_param_num,
+                           ShapeIndex target_param_index, int64 target_dim_num);
+
  private:
   // Build helper which takes the id of the root operation..
   StatusOr<XlaComputation> Build(int64 root_id);
@@ -1019,6 +1032,9 @@ class XlaBuilder {
   // The instructions of this computation.
   std::vector<HloInstructionProto> instructions_;
 
+  // Dynamic parameter configuration of this computation.
+  DynamicParameterBinding dynamic_parameter_binding_;
+
   // A map from XlaOp::Handle to the index in the instructions_ vector where the
   // instruction is held.
   absl::flat_hash_map<int64, int64> handle_to_index_;
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 779a700099..ccbe83bd96 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -292,6 +292,7 @@ cc_library(
     name = "hlo",
     srcs = [
         "dfs_hlo_visitor.cc",
+        "dynamic_parameter_binding.cc",
         "hlo_computation.cc",
         "hlo_input_output_alias_config.cc",
         "hlo_instruction.cc",
@@ -305,6 +306,7 @@ cc_library(
     hdrs = [
         "dfs_hlo_visitor.h",
         "dfs_hlo_visitor_with_default.h",
+        "dynamic_parameter_binding.h",
         "hlo_clone_context.h",
         "hlo_computation.h",
         "hlo_domain_metadata.h",
@@ -350,6 +352,25 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "dynamic_parameter_binding_test",
+    srcs = ["dynamic_parameter_binding_test.cc"],
+    deps = [
+        ":hlo",
+        ":hlo_dce",
+        ":hlo_memory_scheduler",
+        ":hlo_ordering",
+        ":hlo_parser",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+)
+
 tf_cc_test(
     name = "dfs_hlo_visitor_with_default_test",
     srcs = ["dfs_hlo_visitor_with_default_test.cc"],
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
new file mode 100644
index 0000000000..c8bfc89050
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
@@ -0,0 +1,138 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+
+namespace xla {
+
+Status DynamicParameterBinding::Bind(
+    const DynamicParameter& dynamic_parameter,
+    const DynamicDimension& dynamic_dimension) {
+  auto result = bindings_.emplace(dynamic_dimension, dynamic_parameter);
+  TF_RET_CHECK(result.second);
+  return Status::OK();
+}
+
+absl::optional<DynamicParameterBinding::DynamicParameter>
+DynamicParameterBinding::GetBinding(const DynamicDimension& dynamic_dimension) {
+  auto param_iter = bindings_.find(dynamic_dimension);
+  if (param_iter == bindings_.end()) {
+    return absl::nullopt;
+  }
+  return param_iter->second;
+}
+
+DynamicParameterBindingProto DynamicParameterBinding::ToProto() const {
+  DynamicParameterBindingProto result;
+  for (const auto& binding : bindings_) {
+    const DynamicDimension& dynamic_dimension = binding.first;
+    const DynamicParameter& dynamic_param = binding.second;
+    DynamicParameterBindingProto::Binding binding_proto;
+    binding_proto.set_dynamic_param_num(dynamic_param.parameter_num);
+    for (int64 i : dynamic_param.parameter_index) {
+      binding_proto.add_dynamic_param_index(i);
+    }
+
+    binding_proto.set_target_param_num(dynamic_dimension.parameter_num);
+
+    for (int64 i : dynamic_dimension.parameter_index) {
+      binding_proto.add_target_param_index(i);
+    }
+
+    binding_proto.set_target_param_dim_num(dynamic_dimension.dimension);
+    result.add_entries()->Swap(&binding_proto);
+  }
+  return result;
+}
+
+StatusOr<DynamicParameterBinding> DynamicParameterBinding::CreateFromProto(
+    const DynamicParameterBindingProto& proto) {
+  DynamicParameterBinding result;
+  for (const DynamicParameterBindingProto::Binding& binding : proto.entries()) {
+    int64 dynamic_param_num = binding.dynamic_param_num();
+    ShapeIndex dynamic_param_index(binding.dynamic_param_index().begin(),
+                                   binding.dynamic_param_index().end());
+    int64 target_param_num = binding.target_param_num();
+    ShapeIndex target_param_index(binding.target_param_index().begin(),
+                                  binding.target_param_index().end());
+    int64 target_dim_num = binding.target_param_num();
+
+    TF_RETURN_IF_ERROR(
+        result.Bind(DynamicParameter{dynamic_param_num, dynamic_param_index},
+                    DynamicDimension{target_param_num, target_param_index,
+                                     target_dim_num}));
+  }
+
+  return result;
+}
+
+string DynamicParameterBinding::ToString() const {
+  std::vector<string> pieces;
+  pieces.push_back("DynamicParameterBinding: ");
+  for (const auto& binding : bindings_) {
+    const DynamicDimension& dynamic_dimension = binding.first;
+    const DynamicParameter& dynamic_param = binding.second;
+    pieces.push_back(absl::StrFormat(
+        " -- Input param number %lld at %s has dim %lld as dynamic"
+        " dimension, which is represented by param number %lld at "
+        "%s",
+        dynamic_dimension.parameter_num,
+        dynamic_dimension.parameter_index.ToString(),
+        dynamic_dimension.dimension, dynamic_param.parameter_num,
+        dynamic_param.parameter_index.ToString()));
+  }
+  return absl::StrJoin(pieces, "\n");
+}
+
+Status DynamicParameterBinding::ForEachBinding(BindingFn fn) const {
+  for (const auto& binding : bindings_) {
+    TF_RETURN_IF_ERROR(fn(binding.second, binding.first));
+  }
+  return Status::OK();
+}
+
+Status DynamicParameterBinding::Verify(const HloModule& module) const {
+  const HloComputation* entry = module.entry_computation();
+  return ForEachBinding([&](const DynamicParameter& dynamic_parameter,
+                            const DynamicDimension& dynamic_dimension)
+                            -> Status {
+    TF_RET_CHECK(dynamic_parameter.parameter_num < entry->num_parameters());
+    TF_RET_CHECK(dynamic_dimension.parameter_num < entry->num_parameters());
+    TF_RET_CHECK(ShapeUtil::IndexIsValid(
+        entry->parameter_instruction(dynamic_parameter.parameter_num)->shape(),
+        dynamic_parameter.parameter_index));
+    TF_RET_CHECK(ShapeUtil::IndexIsValid(
+        entry->parameter_instruction(dynamic_dimension.parameter_num)->shape(),
+        dynamic_dimension.parameter_index));
+    TF_RET_CHECK(
+        dynamic_dimension.dimension <
+        ShapeUtil::Rank(ShapeUtil::GetSubshape(
+            entry->parameter_instruction(dynamic_dimension.parameter_num)
+                ->shape(),
+            dynamic_dimension.parameter_index)));
+    return Status::OK();
+  });
+}
+
+std::ostream& operator<<(std::ostream& out,
+                         const DynamicParameterBinding& binding) {
+  out << binding.ToString();
+  return out;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.h b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
new file mode 100644
index 0000000000..dd474d8eed
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
@@ -0,0 +1,125 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PARAMETER_BINDING_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PARAMETER_BINDING_H_
+
+#include <utility>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/service/hlo.pb.h"
+#include "tensorflow/compiler/xla/shape_tree.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+
+namespace xla {
+
+class HloModule;
+// We currently use an explicit API that takes an extra parameter to indicate
+// the runtime size of a dynamic dimension. DynamicParameterBinding indicates
+// the relationship between parameter: We can have a dynamic parameter that
+// points to another target parameter to indicate that the target parameter is
+// dynamic.
+//
+//
+// TODO(b/119520625): Remove this API once we have more dynamic shape infra
+// ready.
+class DynamicParameterBinding {
+ public:
+  // DynamicParameter represents a special parameter that is used to represent
+  // the runtime size of a dimension of another parameter. A dynamic parameter
+  // has to be a scalar value.
+  struct DynamicParameter {
+    // The parameter number of dynamic parameter.
+    int64 parameter_num;
+    // The index of the parameter.
+    ShapeIndex parameter_index;
+  };
+
+  // DynamicDimension represents a dimension whose size is determined at
+  // runtime. A DynamicDimension's runtime size is determined by the binded
+  // DynamicParameter using `DynamicParameterBinding::Bind` method.
+  struct DynamicDimension {
+    // The parameter number of dynamic dimension.
+    int64 parameter_num;
+    // The subshape index of the parameter.
+    ShapeIndex parameter_index;
+    // The dimension number in the subshape.
+    int64 dimension;
+
+    // "friend" keyword are added so these functions can be found by ADL.
+    template <typename H>
+    friend H AbslHashValue(H h, const DynamicDimension& m) {
+      return H::combine(std::move(h), m.parameter_num, m.parameter_index,
+                        m.dimension);
+    }
+
+    friend bool operator==(const DynamicDimension& lhs,
+                           const DynamicDimension& rhs) {
+      return lhs.parameter_num == rhs.parameter_num &&
+             lhs.parameter_index == rhs.parameter_index &&
+             lhs.dimension == rhs.dimension;
+    }
+  };
+
+  DynamicParameterBinding() = default;
+
+  virtual ~DynamicParameterBinding() = default;
+
+  // Adds binding which indicates that the dimension indicated by
+  // `dynamic_dimension` is dynamic, and its runtime size is represented by
+  // `dynamic_parameter`.
+  Status Bind(const DynamicParameter& dynamic_parameter,
+              const DynamicDimension& dynamic_dimension);
+
+  // Returns the parameter and the index representing the runtime size of
+  // dimension `dim_num` of parameter `param_num` at `param_index`.
+  //
+  // Returns nullopt if the binding is not set.
+  absl::optional<DynamicParameter> GetBinding(
+      const DynamicDimension& dynamic_dimension);
+
+  using BindingFn =
+      std::function<Status(const DynamicParameter& dynamic_parameter,
+                           const DynamicDimension& dynamic_dimension)>;
+
+  // Iterate through each binding.
+  Status ForEachBinding(BindingFn fn) const;
+
+  DynamicParameterBindingProto ToProto() const;
+
+  static StatusOr<DynamicParameterBinding> CreateFromProto(
+      const DynamicParameterBindingProto& proto);
+
+  string ToString() const;
+
+  // Verifies that the given binding is valid for the given module.
+  // Specifically, the binding's parameter and parameter size should be valid.
+  Status Verify(const HloModule& module) const;
+
+ private:
+  // Keeps track of mappings from DynamicDimension to DynamicParameter. The
+  // direction of is chosen so that we can easily query if a dimension is
+  // dynamic and which dynamic parameter represents the real size of that
+  // dimension.
+  absl::flat_hash_map<DynamicDimension, DynamicParameter> bindings_;
+};
+
+std::ostream& operator<<(std::ostream& out,
+                         const DynamicParameterBinding& binding);
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PARAMETER_BINDING_H_
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
new file mode 100644
index 0000000000..83a6d83dff
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
@@ -0,0 +1,153 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h"
+
+#include <memory>
+#include <string>
+
+#include "absl/algorithm/container.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_dce.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_memory_scheduler.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_ordering.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace xla {
+namespace {
+class DynamicParameterBindingTest : public HloTestBase {};
+
+TEST_F(DynamicParameterBindingTest, SimpleBinding) {
+  // 'b' is a dynamic shape; 'a' represents the real size of b's first
+  // dimension.
+  const string module_str = R"(
+HloModule TEST
+
+ENTRY main {
+  a = f32[] parameter(0)
+  b = f32[10] parameter(1)
+  ROOT root = (f32[], f32[10]) tuple(%a, %b)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(module_str));
+
+  DynamicParameterBinding binding;
+
+  TF_EXPECT_OK(
+      binding.Bind(DynamicParameterBinding::DynamicParameter{0, {}},
+                   DynamicParameterBinding::DynamicDimension{1, {}, 0}));
+
+  absl::optional<DynamicParameterBinding::DynamicParameter> param =
+      binding.GetBinding(
+          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/1,
+                                                    /*parameter_index=*/{},
+                                                    /*dimension=*/0});
+  EXPECT_TRUE(param);
+  EXPECT_EQ(param->parameter_num, 0);
+  EXPECT_EQ(param->parameter_index, ShapeIndex({}));
+  TF_EXPECT_OK(binding.Verify(*module));
+}
+
+TEST_F(DynamicParameterBindingTest, TupleBinding) {
+  // 'gte2' is a dynamic shape; 'gte1' represents the real size of gte2's first
+  // dimension.
+  const string module_str = R"(
+HloModule TEST
+
+ENTRY main {
+  param = (f32[], f32[10]) parameter(0)
+  gte1 = f32[] get-tuple-element(%param), index=0
+  gte2 = f32[10] get-tuple-element(%param), index=1
+  ROOT root = (f32[], f32[10]) tuple(%gte1, %gte2)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(module_str));
+
+  DynamicParameterBinding binding;
+
+  TF_EXPECT_OK(
+      binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
+                   DynamicParameterBinding::DynamicDimension{0, {1}, 0}));
+
+  absl::optional<DynamicParameterBinding::DynamicParameter> param =
+      binding.GetBinding(
+          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                    /*parameter_index=*/{1},
+                                                    /*dimension=*/0});
+
+  EXPECT_TRUE(param);
+  EXPECT_EQ(param->parameter_num, 0);
+  EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
+  TF_EXPECT_OK(binding.Verify(*module));
+}
+
+TEST_F(DynamicParameterBindingTest, TupleBindingWithMultiDimension) {
+  // 'gte2' is a dynamic shape; 'gte1' represents the real size of gte2's both
+  // dimensions.
+  const string module_str = R"(
+HloModule TEST
+
+ENTRY main {
+  param = (f32[], f32[10, 10]) parameter(0)
+  gte1 = f32[] get-tuple-element(%param), index=0
+  gte2 = f32[10, 10] get-tuple-element(%param), index=1
+  ROOT root = (f32[], f32[10, 10]) tuple(%gte1, %gte2)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(module_str));
+
+  DynamicParameterBinding binding;
+
+  TF_EXPECT_OK(
+      binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
+                   DynamicParameterBinding::DynamicDimension{0, {1}, 0}));
+
+  TF_EXPECT_OK(
+      binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
+                   DynamicParameterBinding::DynamicDimension{0, {1}, 1}));
+
+  absl::optional<DynamicParameterBinding::DynamicParameter> param =
+      binding.GetBinding(
+          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                    /*parameter_index=*/{1},
+                                                    /*dimension=*/0});
+
+  EXPECT_TRUE(param);
+  EXPECT_EQ(param->parameter_num, 0);
+  EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
+
+  absl::optional<DynamicParameterBinding::DynamicParameter> param2 =
+      binding.GetBinding(
+          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                    /*parameter_index=*/{1},
+                                                    /*dimension=*/0});
+  EXPECT_TRUE(param2);
+  EXPECT_EQ(param2->parameter_num, 0);
+  EXPECT_EQ(param2->parameter_index, ShapeIndex({0}));
+
+  TF_EXPECT_OK(binding.Verify(*module));
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index dbab62f847..913d4c34b4 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -251,6 +251,41 @@ message HloInputOutputAliasProto {
   repeated AliasEntryProto entries = 1;
 }
 
+message DynamicParameterBindingProto {
+  // A list of bindings which indicates that the `target_dim_num` in
+  // the subshape `target_param_index` of parameter `target_param_num`
+  // is a dynamic dimension and its real dynamic size is represented
+  // by `dynamic_param_index` in parameter `dynamic_param_num`.
+  //
+  // As an example, imagine we have a program:
+  //
+  // ENTRY main {
+  //   a = f32[] parameter(0)
+  //   b = f32[10] parameter(1)
+  //   ROOT root = (f32[], f32[10]) tuple(%a, %b)
+  // }
+  //
+  // Let's say 'b' (param index 1) is a dynamic shape whose input has
+  // an upperbound of 10 and real size is determined at runtime.'a'
+  // represents the real size of b's first dimension.
+  //
+  // In this case, the fields are set in the following way:
+  // dynamic_param_num = 1
+  // dynamic_param_index = {}
+  // target_param_num = 0
+  // target_param_index = {}
+  // target_param_dim = 0
+  message Binding {
+    int64 dynamic_param_num = 1;
+    repeated int64 dynamic_param_index = 2;
+    int64 target_param_num = 3;
+    repeated int64 target_param_index = 4;
+    int64 target_param_dim_num = 5;
+  }
+
+  repeated Binding entries = 1;
+}
+
 // Serialization of HloModule.
 message HloModuleProto {
   string name = 1;
@@ -272,6 +307,8 @@ message HloModuleProto {
 
   // Describes alias information between inputs and outputs.
   HloInputOutputAliasProto input_output_alias = 8;
+
+  DynamicParameterBindingProto dynamic_parameter_binding = 9;
 }
 
 // Serialization of LogicalBuffer.
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 14bf17f4be..59f44475df 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -242,6 +242,8 @@ HloModuleProto HloModule::ToProto() const {
   *proto.mutable_host_program_shape() =
       entry_computation_layout().ComputeProgramShape();
   *proto.mutable_input_output_alias() = input_output_alias_config().ToProto();
+  *proto.mutable_dynamic_parameter_binding() =
+      dynamic_parameter_binding().ToProto();
   return proto;
 }
 
@@ -325,6 +327,10 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
 
   // Because we didn't uniquify the names or the ids, double-check that the
   // instruction and computation names and ids are unique from the proto.
+  TF_ASSIGN_OR_RETURN(module->dynamic_parameter_binding_,
+                      DynamicParameterBinding::CreateFromProto(
+                          proto.dynamic_parameter_binding()));
+
   absl::flat_hash_set<string> computation_names;
   absl::flat_hash_set<string> instruction_names;
   absl::flat_hash_set<int> computation_ids;
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 8a1f999e3a..7974999e07 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/iterator_util.h"
+#include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_clone_context.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
@@ -232,6 +233,16 @@ class HloModule {
     return input_output_alias_config_;
   }
 
+  // DynamicParameterBinding holds the list of bindings that indicates which
+  // parameter dimensions are dynamic and which parameters represent their
+  // runtime value.
+  DynamicParameterBinding& dynamic_parameter_binding() {
+    return dynamic_parameter_binding_;
+  }
+  const DynamicParameterBinding& dynamic_parameter_binding() const {
+    return dynamic_parameter_binding_;
+  }
+
   // Returns an id that is unique to this module across all modules created over
   // the lifetime of this process.
   int unique_id() const { return unique_id_; }
@@ -285,6 +296,9 @@ class HloModule {
   // alias_config indicates the alias information of input/output buffers that
   // are expected from the module.
   HloInputOutputAliasConfig input_output_alias_config_;
+
+  // Bindings for dynamic parameter mapping.
+  DynamicParameterBinding dynamic_parameter_binding_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index c5688a445a..60d8a511b5 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -1426,6 +1426,8 @@ StatusOr<bool> HloVerifier::Run(HloModule* module) {
         return target_metadata_->ShapeSize(shape);
       }));
 
+  TF_RETURN_IF_ERROR(module->dynamic_parameter_binding().Verify(*module));
+
   return false;
 }
 
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index a7a3026cf3..bb347bf0de 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -100,6 +100,11 @@ class ShapeIndex {
 
   string ToString() const;
 
+  template <typename H>
+  friend H AbslHashValue(H h, const ShapeIndex& index) {
+    return H::combine(std::move(h), index.indices_);
+  }
+
  private:
   container_type indices_;
 };
-- 
GitLab


From beac73a40fe38d6ee0fc09327eeeb74d2838558d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 18:34:28 -0800
Subject: [PATCH 0370/1554] Add `in_cross_replica_context` and deprecate
 `get_cross_replica_context` in `tf.contrib.distribute`.

PiperOrigin-RevId: 221727283
---
 tensorflow/contrib/distribute/__init__.py     |  1 +
 tensorflow/python/training/distribute_test.py |  4 ++
 .../training/distribution_strategy_context.py | 53 ++++++++++++++-----
 3 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 69784c737a..2853bbdb11 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -63,6 +63,7 @@ _allowed_symbols = [
     'get_loss_reduction',
     'get_replica_context',
     'has_distribution_strategy',
+    'in_cross_replica_context',
     'require_replica_context',
     'run_standard_tensorflow_server',
     'UpdateContext',
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index 10b8dead5f..be7c80d1fa 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -63,6 +63,7 @@ def _assert_in_default_state(t):
   t.assertIs(distribution_strategy_context._get_default_replica_context(),
              distribution_strategy_context.get_replica_context())
   t.assertIs(None, distribution_strategy_context.get_cross_replica_context())
+  t.assertFalse(distribution_strategy_context.in_cross_replica_context())
   t.assertIs(distribution_strategy_context._get_default_distribution_strategy(),
              distribution_strategy_context.get_distribution_strategy())
   t.assertFalse(distribution_strategy_context.has_distribution_strategy())
@@ -79,6 +80,7 @@ class TestStrategyTest(test.TestCase):
       self.assertTrue(replica_context is not None)
       self.assertIs(None,
                     distribution_strategy_context.get_cross_replica_context())
+      self.assertFalse(distribution_strategy_context.in_cross_replica_context())
       self.assertTrue(distribution_strategy_context.has_distribution_strategy())
       self.assertIs(dist,
                     distribution_strategy_context.get_distribution_strategy())
@@ -102,6 +104,7 @@ class TestStrategyTest(test.TestCase):
       self.assertIs(None, distribution_strategy_context.get_replica_context())
       self.assertIs(dist,
                     distribution_strategy_context.get_cross_replica_context())
+      self.assertTrue(distribution_strategy_context.in_cross_replica_context())
       self.assertTrue(distribution_strategy_context.has_distribution_strategy())
       self.assertIs(dist,
                     distribution_strategy_context.get_distribution_strategy())
@@ -141,6 +144,7 @@ class DefaultDistributionStrategyTest(test.TestCase):
       self.assertIs(None, distribution_strategy_context.get_replica_context())
       self.assertIs(dist,
                     distribution_strategy_context.get_cross_replica_context())
+      self.assertTrue(distribution_strategy_context.in_cross_replica_context())
       self.assertIs(dist,
                     distribution_strategy_context.get_distribution_strategy())
       self.assertFalse(
diff --git a/tensorflow/python/training/distribution_strategy_context.py b/tensorflow/python/training/distribution_strategy_context.py
index 9a33f5f416..c73d65d330 100644
--- a/tensorflow/python/training/distribution_strategy_context.py
+++ b/tensorflow/python/training/distribution_strategy_context.py
@@ -86,24 +86,26 @@ def _get_per_thread_mode():
 
 
 def get_replica_context():
-  """Returns the current ReplicaContext or None if in a cross-replica context.
+  """Returns the current `tf.distribute.ReplicaContext` or `None`.
+
+  Returns `None` if in a cross-replica context.
 
   Note that execution:
 
   1. starts in the default (single-replica) replica context (this function
-     will return the default ReplicaContext object);
+     will return the default `ReplicaContext` object);
   2. switches to cross-replica context (in which case this will return
-     None) when entering a `with DistributionStrategy.scope():` block;
+     `None`) when entering a `with DistributionStrategy.scope():` block;
   3. switches to a (non-default) replica context inside
-     `call_for_each_replica(fn, ...)`;
-  4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then
+     `extended.call_for_each_replica(fn, ...)`;
+  4. if `fn` calls `get_replica_context().merge_call(merge_fn, ...)`, then
      inside `merge_fn` you are back in the cross-replica context (and again
-     this function will return None).
+     this function will return `None`).
 
   Note that you can also go directly from step 1 to 4 to switch to a
   cross-replica context for the default `DistributionStrategy`. You may
   also switch from the cross-replica context of 4 to a replica context by
-  calling `call_for_each_replica()`, jumping back to step 3.
+  calling `extended.call_for_each_replica()`, jumping back to step 3.
 
   Most `DistributionStrategy` methods may only be executed in
   a cross-replica context, in a replica context you should use the
@@ -111,10 +113,12 @@ def get_replica_context():
 
   Returns:
     The current `ReplicaContext` object when in a replica context scope,
-    else None.
+    else `None`.
 
-    Exactly one of `get_replica_context()` and `get_cross_replica_context()`
-    will return None in a particular block.
+    Within a particular block, exactly one of these two things will be true:
+
+    * `get_replica_context()` returns non-`None`, or
+    * `tf.distribute.is_cross_replica_context()` returns True.
   """
   return _get_per_thread_mode().replica_context
 
@@ -122,6 +126,9 @@ def get_replica_context():
 def get_cross_replica_context():
   """Returns the current DistributionStrategy if in a cross-replica context.
 
+  DEPRECATED: Please use `in_cross_replica_context()` and
+  `get_distribution_strategy()` instead.
+
   Note that execution:
 
   1. starts in the default (single-replica) replica context;
@@ -142,19 +149,37 @@ def get_cross_replica_context():
 
   Returns:
     Returns the current `DistributionStrategy` object in a cross-replica
-    context, or None.
+    context, or `None`.
 
     Exactly one of `get_replica_context()` and `get_cross_replica_context()`
-    will return None in a particular block.
+    will return `None` in a particular block.
   """
   return _get_per_thread_mode().cross_replica_context
 
 
+def in_cross_replica_context():
+  """Returns True if in a cross-replica context.
+
+  See `tf.distribute.get_replica_context` for details.
+
+  Returns:
+    True if in a cross-replica context (`get_replica_context()` returns
+    `None`), or False if in a replica context (`get_replica_context()` returns
+    non-`None`).
+  """
+  return _get_per_thread_mode().cross_replica_context is not None
+
+
 def get_distribution_strategy():
   """Returns the current `DistributionStrategy` object.
 
-  Prefer to use `get_replica_context()` or `get_cross_replica_context()`
-  instead when possible.
+  Typically only used in a cross-replica context:
+
+  ```
+  if tf.distribute.in_cross_replica_context():
+    strategy = tf.distribute.get_distribution_strategy()
+    ...
+  ```
 
   Returns:
     A `DistributionStrategy` object. Inside a
-- 
GitLab


From ea45713a672cfd1e4df1f0a3f4b9ca9b503f6e69 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 19:02:07 -0800
Subject: [PATCH 0371/1554] [XLA] Fix britle CPU test.

Relies on a specific ordering of two constants in the IR.

Also add HLO verification to the test's modules while we're here.

PiperOrigin-RevId: 221729688
---
 .../xla/service/cpu/tests/cpu_literal_caching_test.cc     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc
index 3b87683fff..fa0e09ff6b 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_literal_caching_test.cc
@@ -63,7 +63,7 @@ CHECK-NOT: private constant [48 x i8]
 )";
 
   TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          ParseHloString(hlo_text));
+                          ParseAndReturnVerifiedModule(hlo_text));
 
   CpuAotCompilationOptions options{
       /*triple=*/"x86_64-pc-linux", /*cpu_name=*/"", /*features=*/"",
@@ -104,14 +104,14 @@ ENTRY main {
 )";
 
   string filecheck_pattern = R"(
-CHECK: private constant [4 x i8]
-CHECK: private constant [8 x i8]
+CHECK-DAG: private constant [4 x i8]
+CHECK-DAG: private constant [8 x i8]
 CHECK-NOT: private constant [4 x i8]
 CHECK-NOT: private constant [8 x i8]
 )";
 
   TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          ParseHloString(hlo_text));
+                          ParseAndReturnVerifiedModule(hlo_text));
 
   CpuAotCompilationOptions options{
       /*triple=*/"x86_64-pc-linux", /*cpu_name=*/"", /*features=*/"",
-- 
GitLab


From b8c1442d124402830464b67d210761a9b5a4db35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Nov 2018 19:05:47 -0800
Subject: [PATCH 0372/1554] Fork MirroredStrategy in anticipation of moving it
 out of contrib.

PiperOrigin-RevId: 221730172
---
 .../python/checkpoint_utils_test.py           |   4 +-
 .../contrib/distribute/python/combinations.py |  22 +++-
 .../distribute/python/cross_tower_ops_test.py |   7 +-
 .../python/estimator_integration_test.py      |   4 +-
 .../python/keras_optimizer_v2_test.py         |   4 +-
 .../contrib/distribute/python/keras_test.py   |   6 +-
 .../distribute/python/metrics_v1_test.py      |   4 +-
 .../distribute/python/minimize_loss_test.py   |   4 +-
 .../distribute/python/mirrored_strategy.py    | 105 ++++++++++++++++--
 .../distribute/python/moving_averages_test.py |   3 +-
 .../python/warm_starting_util_test.py         |   4 +-
 11 files changed, 141 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
index d38bdb592a..31bd0e996a 100644
--- a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
+++ b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
@@ -43,7 +43,9 @@ class CheckpointUtilsWithDistributionStrategyTest(
       distribution=[combinations.default_strategy,
                     combinations.one_device_strategy,
                     combinations.mirrored_strategy_with_gpu_and_cpu,
-                    combinations.mirrored_strategy_with_two_gpus],
+                    combinations.mirrored_strategy_with_two_gpus,
+                    combinations.core_mirrored_strategy_with_gpu_and_cpu,
+                    combinations.core_mirrored_strategy_with_two_gpus],
       in_replica_mode=[True, False],
       mode=["graph"]))
   def testInitFromCheckpoint(self, distribution, in_replica_mode):
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index a513716540..0fcabb1afe 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -343,6 +343,14 @@ mirrored_strategy_with_two_gpus = NamedDistribution(
     "Mirrored2GPUs",
     lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/gpu:1"]),
     required_gpus=2)
+core_mirrored_strategy_with_gpu_and_cpu = NamedDistribution(
+    "CoreMirroredCPUAndGPU",
+    lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0", "/cpu:0"]),
+    required_gpus=1)
+core_mirrored_strategy_with_two_gpus = NamedDistribution(
+    "CoreMirrored2GPUs",
+    lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0", "/gpu:1"]),
+    required_gpus=2)
 
 
 gradient_descent_optimizer_v1_fn = NamedObject(
@@ -373,8 +381,11 @@ def distributions_and_v1_optimizers():
   """A common set of combination with DistributionStrategies and Optimizers."""
   return combine(
       distribution=[
-          one_device_strategy, mirrored_strategy_with_gpu_and_cpu,
-          mirrored_strategy_with_two_gpus
+          one_device_strategy,
+          mirrored_strategy_with_gpu_and_cpu,
+          mirrored_strategy_with_two_gpus,
+          core_mirrored_strategy_with_gpu_and_cpu,
+          core_mirrored_strategy_with_two_gpus,
       ],
       optimizer_fn=optimizers_v1)
 
@@ -383,7 +394,10 @@ def distributions_and_v2_optimizers():
   """DistributionStrategies and V2 Optimizers."""
   return combine(
       distribution=[
-          one_device_strategy, mirrored_strategy_with_gpu_and_cpu,
-          mirrored_strategy_with_two_gpus
+          one_device_strategy,
+          mirrored_strategy_with_gpu_and_cpu,
+          mirrored_strategy_with_two_gpus,
+          core_mirrored_strategy_with_gpu_and_cpu,
+          core_mirrored_strategy_with_two_gpus,
       ],
       optimizer_fn=optimizers_v2)
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
index 2e352360a4..100328efba 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
@@ -212,7 +212,9 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
       distribution=[
           combinations.one_device_strategy,
           combinations.mirrored_strategy_with_gpu_and_cpu,
-          combinations.mirrored_strategy_with_two_gpus
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_two_gpus
       ],
       mode=["graph", "eager"])
   allreduce_combinations = combinations.combine(
@@ -232,7 +234,8 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
               cross_tower_ops_lib.AllReduceCrossDeviceOps(
                   "hierarchical_copy", 0, 100, 10))
       ],
-      distribution=[combinations.mirrored_strategy_with_two_gpus],
+      distribution=[combinations.mirrored_strategy_with_two_gpus,
+                    combinations.core_mirrored_strategy_with_two_gpus],
       mode=["graph", "eager"])
 
   @combinations.generate(reduction_to_one_combinations + allreduce_combinations)
diff --git a/tensorflow/contrib/distribute/python/estimator_integration_test.py b/tensorflow/contrib/distribute/python/estimator_integration_test.py
index ecda5337bb..264dca6f38 100644
--- a/tensorflow/contrib/distribute/python/estimator_integration_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_integration_test.py
@@ -63,7 +63,9 @@ class DNNLinearCombinedClassifierIntegrationTest(test.TestCase,
           distribution=[
               combinations.one_device_strategy,
               combinations.mirrored_strategy_with_gpu_and_cpu,
-              combinations.mirrored_strategy_with_two_gpus
+              combinations.mirrored_strategy_with_two_gpus,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_two_gpus
           ],
           use_train_and_evaluate=[True, False]))
   def test_complete_flow_with_mode(self, distribution, use_train_and_evaluate):
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 6a4cbb113a..40f10922b1 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -71,7 +71,9 @@ class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase):
           distribution=[
               combinations.one_device_strategy,
               combinations.mirrored_strategy_with_gpu_and_cpu,
-              combinations.mirrored_strategy_with_two_gpus
+              combinations.mirrored_strategy_with_two_gpus,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_two_gpus
           ],
           use_train_and_evaluate=[True, False]))
   def test_complete_flow_with_mode(self, distribution, use_train_and_evaluate):
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index abf9e78adc..4adc380b83 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -279,6 +279,8 @@ strategies = [combinations.default_strategy,
               combinations.one_device_strategy,
               combinations.mirrored_strategy_with_gpu_and_cpu,
               combinations.mirrored_strategy_with_two_gpus,
+              combinations.core_mirrored_strategy_with_gpu_and_cpu,
+              combinations.core_mirrored_strategy_with_two_gpus,
               combinations.tpu_strategy,  # steps_per_run=2
               combinations.tpu_strategy_one_step]
 
@@ -288,7 +290,9 @@ def strategy_minus_tpu_combinations():
       distribution=[combinations.default_strategy,
                     combinations.one_device_strategy,
                     combinations.mirrored_strategy_with_gpu_and_cpu,
-                    combinations.mirrored_strategy_with_two_gpus],
+                    combinations.mirrored_strategy_with_two_gpus,
+                    combinations.core_mirrored_strategy_with_gpu_and_cpu,
+                    combinations.core_mirrored_strategy_with_two_gpus],
       mode=['graph'])
 
 
diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index a2fc118173..e622e1df23 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -77,7 +77,9 @@ def all_combinations():
       distribution=[combinations.default_strategy,
                     combinations.one_device_strategy,
                     combinations.mirrored_strategy_with_gpu_and_cpu,
-                    combinations.mirrored_strategy_with_two_gpus],
+                    combinations.mirrored_strategy_with_two_gpus,
+                    combinations.core_mirrored_strategy_with_gpu_and_cpu,
+                    combinations.core_mirrored_strategy_with_two_gpus],
       mode=["graph"])
 
 
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 1f57dd1754..195d8a90c2 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -286,7 +286,9 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
                   distribution=[
                       combinations.one_device_strategy,
                       combinations.mirrored_strategy_with_gpu_and_cpu,
-                      combinations.mirrored_strategy_with_two_gpus
+                      combinations.mirrored_strategy_with_two_gpus,
+                      combinations.core_mirrored_strategy_with_gpu_and_cpu,
+                      combinations.core_mirrored_strategy_with_two_gpus
                   ]),
               combinations.combine(
                   mode=["graph"], use_callable_loss=[True, False]) +
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index fb59a8ad91..34061696ed 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -295,9 +295,11 @@ def _create_mirrored_variable(devices, real_mirrored_creator, *args, **kwargs):
   return result
 
 
-class MirroredStrategy(distribute_lib.DistributionStrategy):
+class CoreMirroredStrategy(distribute_lib.DistributionStrategy):
   """Mirrors vars to distribute across multiple devices and machines.
 
+  *** core version ***
+
   This strategy uses one replica per device and sync replication for its
   multi-GPU version.
 
@@ -342,7 +344,6 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
       set, the `configure` method will try to find the best one.
     auto_shard_dataset: whether to auto-shard the dataset when there are
       multiple workers.
-    cross_tower_ops: Deprecated alias for `cross_device_ops`.
   """
 
   def __init__(self,
@@ -350,17 +351,15 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                num_gpus=None,
                num_gpus_per_worker=None,
                cross_device_ops=None,
-               auto_shard_dataset=False,
-               cross_tower_ops=None):
-    assert not (cross_device_ops and cross_tower_ops)
-    extended = MirroredExtended(
+               auto_shard_dataset=False):
+    extended = CoreMirroredExtended(
         self, devices, num_gpus, num_gpus_per_worker,
-        cross_device_ops or cross_tower_ops, auto_shard_dataset)
-    super(MirroredStrategy, self).__init__(extended)
+        cross_device_ops, auto_shard_dataset)
+    super(CoreMirroredStrategy, self).__init__(extended)
 
 
-class MirroredExtended(distribute_lib.DistributionStrategyExtended):
-  """Implementation of MirroredStrategy."""
+class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of CoreMirroredStrategy."""
 
   def __init__(self,
                container_strategy,
@@ -369,7 +368,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
                num_gpus_per_worker=None,
                cross_device_ops=None,
                auto_shard_dataset=False):
-    super(MirroredExtended, self).__init__(container_strategy)
+    super(CoreMirroredExtended, self).__init__(container_strategy)
     # TODO(josh11b): Rename self._cross_tower_ops -> self._cross_device_ops
     self._cross_tower_ops = cross_device_ops
     self._auto_shard_dataset = auto_shard_dataset
@@ -731,7 +730,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
 
     def __init__(self, dist, coord, device, variable_creator_fn, fn, *args,
                  **kwargs):
-      super(MirroredExtended._MirroredReplicaThread, self).__init__()  # pylint: disable=protected-access
+      super(CoreMirroredExtended._MirroredReplicaThread, self).__init__()  # pylint: disable=protected-access
       self.coord = coord
       self.distribution = dist
       self.device = device
@@ -806,6 +805,88 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
         self.has_paused.set()
 
 
+class MirroredStrategy(distribute_lib.DistributionStrategy):
+  """Mirrors vars to distribute across multiple devices and machines.
+
+  *** contrib version ***
+
+  This strategy uses one replica per device and sync replication for its
+  multi-GPU version.
+
+  When `cluster_spec` is given by the `configure` method., it turns into the
+  mulit-worker version that works on multiple workers with in-graph replication.
+  Note: `configure` will be called by higher-level APIs if running in
+  distributed environment.
+
+  There are several important concepts for distributed TensorFlow, e.g.
+  `client`, `job`, 'task', `cluster`, `in-graph replication` and
+  'synchronous training' and they have already been defined in the
+  [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed).
+  The distribution strategy inherits these concepts as well and in addition to
+  that we also clarify several more concepts:
+
+  * **In-graph replication**: the `client` creates a single `tf.Graph` that
+    specifies tasks for devices on all workers. The `client` then creates a
+    client session which will talk to the `master` service of a `worker`. Then
+    the `master` will partition the graph and distribute the work to all
+    participating workers.
+  * **Worker**: A `worker` is a TensorFlow `task` that usually maps to one
+    physical machine. We will have multiple `worker`s with different `task`
+    index. They all do similar things except for one worker checkpointing model
+    variables, writing summaries, etc. in addition to its ordinary work.
+
+  The multi-worker version of this class maps one replica to one device on a
+  worker. It mirrors all model variables on all replicas. For example, if you
+  have two `worker`s and each `worker` has 4 GPUs, it will create 8 copies of
+  the model variables on these 8 GPUs. Then like in MirroredStrategy, each
+  replica performs their computation with their own copy of variables unless in
+  cross-replica model where variable or tensor reduction happens.
+
+  Args:
+    devices: a list of device strings.
+    num_gpus: number of GPUs. For local training, either specify `devices` or
+      `num_gpus`. In distributed training, this must be specified as number of
+      GPUs on each worker.
+    num_gpus_per_worker: number of GPUs per worker. This is the same as
+      `num_gpus` and only one of `num_gpus` and `num_gpus_per_worker` can be
+      specified.
+    cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not
+      set, the `configure` method will try to find the best one.
+    auto_shard_dataset: whether to auto-shard the dataset when there are
+      multiple workers.
+    cross_tower_ops: Deprecated alias for `cross_device_ops`.
+  """
+
+  def __init__(self,
+               devices=None,
+               num_gpus=None,
+               num_gpus_per_worker=None,
+               cross_device_ops=None,
+               auto_shard_dataset=False,
+               cross_tower_ops=None):
+    assert not (cross_device_ops and cross_tower_ops)
+    extended = MirroredExtended(
+        self, devices, num_gpus, num_gpus_per_worker,
+        cross_device_ops or cross_tower_ops, auto_shard_dataset)
+    super(MirroredStrategy, self).__init__(extended)
+
+
+class MirroredExtended(CoreMirroredExtended):
+  """Implementation of (contrib) MirroredStrategy."""
+
+  # pylint: disable=useless-super-delegation
+  def __init__(self,
+               container_strategy,
+               devices=None,
+               num_gpus=None,
+               num_gpus_per_worker=None,
+               cross_device_ops=None,
+               auto_shard_dataset=False):
+    super(MirroredExtended, self).__init__(
+        container_strategy, devices, num_gpus, num_gpus_per_worker,
+        cross_device_ops, auto_shard_dataset)
+
+
 class MirroredReplicaContext(distribute_lib.ReplicaContext):
   """ReplicaContext used in MirroredStrategy.call_for_each_replica().
 
diff --git a/tensorflow/contrib/distribute/python/moving_averages_test.py b/tensorflow/contrib/distribute/python/moving_averages_test.py
index 7ecc852d20..c492d8bafc 100644
--- a/tensorflow/contrib/distribute/python/moving_averages_test.py
+++ b/tensorflow/contrib/distribute/python/moving_averages_test.py
@@ -32,7 +32,8 @@ from tensorflow.python.training import moving_averages
 all_combinations = combinations.combine(
     distribution=[combinations.default_strategy,
                   combinations.one_device_strategy,
-                  combinations.mirrored_strategy_with_gpu_and_cpu],
+                  combinations.mirrored_strategy_with_gpu_and_cpu,
+                  combinations.core_mirrored_strategy_with_gpu_and_cpu],
     mode=["graph"])
 
 
diff --git a/tensorflow/contrib/distribute/python/warm_starting_util_test.py b/tensorflow/contrib/distribute/python/warm_starting_util_test.py
index 5d57d144c1..b0bcf9b174 100644
--- a/tensorflow/contrib/distribute/python/warm_starting_util_test.py
+++ b/tensorflow/contrib/distribute/python/warm_starting_util_test.py
@@ -44,7 +44,9 @@ class WarmStartingUtilWithDistributionStrategyTest(
       distribution=[combinations.default_strategy,
                     combinations.one_device_strategy,
                     combinations.mirrored_strategy_with_gpu_and_cpu,
-                    combinations.mirrored_strategy_with_two_gpus],
+                    combinations.mirrored_strategy_with_two_gpus,
+                    combinations.core_mirrored_strategy_with_gpu_and_cpu,
+                    combinations.core_mirrored_strategy_with_two_gpus],
       save_with_distribution=[True, False],
       restore_with_distribution=[True, False],
       mode=["graph"]))
-- 
GitLab


From e3a4aa36027b779ca1011c6331f173cc15d16135 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 15 Nov 2018 19:38:57 -0800
Subject: [PATCH 0373/1554] Remove xw_plus_b from the API for TF 2.0.

PiperOrigin-RevId: 221732670
---
 tensorflow/python/ops/nn_ops.py                    | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 651bd32e5f..9103253874 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2299,7 +2299,7 @@ def _calc_bias_add_flops(graph, node):
   return ops.OpStats("flops", input_count)
 
 
-@tf_export("nn.xw_plus_b")
+@tf_export(v1=["nn.xw_plus_b"])
 def xw_plus_b(x, weights, biases, name=None):  # pylint: disable=invalid-name
   """Computes matmul(x, weights) + biases.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 881f566fe2..481365be24 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -336,10 +336,6 @@ tf_module {
     name: "with_space_to_batch"
     argspec: "args=[\'input\', \'dilation_rate\', \'padding\', \'op\', \'filter_shape\', \'spatial_dims\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "xw_plus_b"
-    argspec: "args=[\'x\', \'weights\', \'biases\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "zero_fraction"
     argspec: "args=[\'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 59beb2ca929a0492a59a76d51bed774176156798 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Thu, 15 Nov 2018 20:01:33 -0800
Subject: [PATCH 0374/1554] Remove tf.quantize_v2 in the v2 API.

PiperOrigin-RevId: 221734249
---
 tensorflow/python/ops/array_ops.py              | 4 ++--
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 5 +++++
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 5a81442f3a..b7823ccc2f 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3005,7 +3005,7 @@ def batch_gather(params, indices, name=None):
 
 # Define quantize_v2 here in order to make name the second-to-last attribute,
 # because round_mode was added later.
-@tf_export("quantize_v2")
+@tf_export(v1=["quantize_v2"])
 @deprecation.deprecated(
     "2017-10-25",
     "`tf.quantize_v2` is deprecated, please use `tf.quantization.quantize` "
@@ -3026,7 +3026,7 @@ def quantize_v2(input,  # pylint: disable=redefined-builtin
                                    round_mode=round_mode)
 
 
-quantize_v2.__doc__ = """Please use `tf.quantize` instead."""
+quantize_v2.__doc__ = """Please use `tf.quantization.quantize` instead."""
 
 
 # We want to expose tf.quantize instead of tf.quantize_v2; we can deprecate
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 42b9397504..d37f83efdc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -924,10 +924,6 @@ tf_module {
     name: "py_function"
     argspec: "args=[\'func\', \'inp\', \'Tout\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "quantize_v2"
-    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], "
-  }
   member_method {
     name: "range"
     argspec: "args=[\'start\', \'limit\', \'delta\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'range\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 7e87b7d764..41948441ec 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -100,6 +100,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.contrib.data.sloppy_interleave": "tf.compat.v1.contrib.data.sloppy_interleave",
         "tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
         "tf.contrib.data.unique": "tf.data.experimental.unique",
+        "tf.quantize_v2": "tf.quantization.quantize",
         "tf.sparse_concat": "tf.sparse.concat",
         "tf.load_file_system_library": "tf.load_library",
     })
@@ -132,6 +133,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "padding", "data_format", "dilations", "name"
         ],
         "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
+        "tf.quantize_v2": [
+            "input", "min_range", "max_range", "T", "mode", "name",
+            "round_mode"
+        ],
         "tf.shape": ["input", "name", "out_type"],
         "tf.size": ["input", "name", "out_type"],
         "tf.sparse.concat": [
-- 
GitLab


From 5b57bcabe03428d6b89c88618f4ddf71517e0456 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 15 Nov 2018 20:16:04 -0800
Subject: [PATCH 0375/1554] Remove learning phase placeholder from inputs to
 the test & predict graph functions (since it has a default value of False).

PiperOrigin-RevId: 221735591
---
 .../contrib/tpu/python/tpu/keras_support.py   |  5 +----
 tensorflow/python/keras/engine/training.py    | 21 ++++---------------
 .../python/keras/engine/training_arrays.py    |  7 +++----
 .../keras/engine/training_distributed.py      | 12 ++---------
 4 files changed, 10 insertions(+), 35 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index 08f58a5f5b..e1cc17b301 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -1184,12 +1184,9 @@ class TPUFunction(object):
       # pipelined loop.
       return None, None
 
-    if not isinstance(K.learning_phase(), int):
+    if isinstance(inputs[-1], int):
       # Remove the learning_phase flag at the end. We currently hard code the
       # learning_phase in TPUFunction.
-      assert isinstance(inputs[-1], int), (
-          'Expect the final element be learning_phase flag. Got {}'.format(
-              inputs[-1]))
       inputs = inputs[:-1]
 
     if (self.execution_mode == model_fn_lib.ModeKeys.TRAIN or
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 9fd12c4b56..4928098901 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -802,8 +802,6 @@ class Model(Network):
       inputs = (self._feed_inputs +
                 self._feed_targets +
                 self._feed_sample_weights)
-      if not isinstance(K.symbolic_learning_phase(), int):
-        inputs += [K.symbolic_learning_phase()]
 
       with K.name_scope('evaluation'):
         updates = self.state_updates
@@ -832,10 +830,7 @@ class Model(Network):
     if not hasattr(self, 'predict_function'):
       self.predict_function = None
     if self.predict_function is None:
-      if not isinstance(K.symbolic_learning_phase(), int):
-        inputs = self._feed_inputs + [K.symbolic_learning_phase()]
-      else:
-        inputs = self._feed_inputs
+      inputs = self._feed_inputs
       # Gets network outputs. Does not update weights.
       # Does update the network states.
       kwargs = getattr(self, '_function_kwargs', {})
@@ -2049,12 +2044,9 @@ class Model(Network):
       outputs = training_eager.test_on_batch(
           self, x, y, sample_weights=sample_weights)
     else:
-      if not isinstance(K.symbolic_learning_phase(), int):
-        ins = x + y + sample_weights + [False]
-      else:
-        ins = x + y + sample_weights
+      inputs = x + y + sample_weights
       self._make_test_function()
-      outputs = self.test_function(ins)  # pylint: disable=not-callable
+      outputs = self.test_function(inputs)  # pylint: disable=not-callable
 
     if len(outputs) == 1:
       return outputs[0]
@@ -2093,13 +2085,8 @@ class Model(Network):
         ]
       return self(inputs)  # pylint: disable=not-callable
 
-    if not isinstance(K.symbolic_learning_phase(), int):
-      ins = inputs + [False]
-    else:
-      ins = inputs
-
     self._make_predict_function()
-    outputs = self.predict_function(ins)
+    outputs = self.predict_function(inputs)
 
     if len(outputs) == 1:
       return outputs[0]
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 7c603f4e01..7f4cfc729b 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -247,10 +247,9 @@ def model_iteration(model,
   inputs = training_utils.ModelInputs(inputs).as_list()
   targets = targets or []
   sample_weights = sample_weights or []
-  learning_phase_input = []
-  if not isinstance(K.symbolic_learning_phase(), int):
-    learning_phase_input = [True] if mode == 'train' else [False]
-  ins = inputs + targets + sample_weights + learning_phase_input
+  ins = inputs + targets + sample_weights
+  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
+    ins += [True]
   num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                    steps_per_epoch)
 
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 02d0befe8e..a614614c9b 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -477,10 +477,7 @@ def test_loop(model, iterator, verbose=0, steps=None):
     # placeholders that are created with default values.
     sample_weights = [None for _ in range(
         len(model.outputs) * current_strategy.num_replicas_in_sync)]
-    if not isinstance(K.learning_phase(), int):
-      ins = dataset_inputs + dataset_targets + sample_weights + [0]
-    else:
-      ins = dataset_inputs + dataset_targets
+    ins = dataset_inputs + dataset_targets + sample_weights
 
     for m in model.stateful_metric_functions:
       m.reset_states()
@@ -685,11 +682,6 @@ def predict_loop(model, iterator, verbose=0, steps=None):
         name='distributed_predict_function',
         **all_session_args)
 
-    if not isinstance(K.learning_phase(), int):
-      ins = dataset_inputs + [0]
-    else:
-      ins = dataset_inputs
-
     if verbose == 1:
       progbar = Progbar(target=steps)
 
@@ -706,7 +698,7 @@ def predict_loop(model, iterator, verbose=0, steps=None):
     unconcatenated_outs = []
     assert steps is not None
     for step in range(steps):
-      batch_outs = distributed_predict_function(ins)
+      batch_outs = distributed_predict_function(dataset_inputs)
       if not isinstance(batch_outs, list):
         batch_outs = [batch_outs]
       if step == 0:
-- 
GitLab


From 186744c73b9d0e64933bbc9e15db9ce2938e9e6b Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 15 Nov 2018 20:16:41 -0800
Subject: [PATCH 0376/1554] Change API for tf.zeros_like/tf.ones_like in TF
 2.0.

PiperOrigin-RevId: 221735639
---
 tensorflow/python/ops/array_ops.py            | 80 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  8 +-
 3 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index b7823ccc2f..3845e66b07 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -1740,8 +1740,8 @@ def zeros(shape, dtype=dtypes.float32, name=None):
   return output
 
 
-@tf_export("zeros_like")
-def zeros_like(tensor, dtype=None, name=None, optimize=True):
+@tf_export(v1=["zeros_like"])
+def zeros_like_v1(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to zero.
 
   Given a single tensor (`tensor`), this operation returns a tensor of the
@@ -1767,6 +1767,43 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True):
   Returns:
     A `Tensor` with all elements set to zero.
   """
+  zeros_like_impl(tensor, dtype, name, optimize)
+
+
+@tf_export("zeros_like", v1=[])
+def zeros_like_v2(
+    input,  # pylint: disable=redefined-builtin
+    dtype=None,
+    name=None):
+  """Creates a tensor with all elements set to zero.
+
+  Given a single tensor (`tensor`), this operation returns a tensor of the
+  same type and shape as `tensor` with all elements set to zero. Optionally,
+  you can use `dtype` to specify a new type for the returned tensor.
+
+  For example:
+
+  ```python
+  tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
+  tf.zeros_like(tensor)  # [[0, 0, 0], [0, 0, 0]]
+  ```
+
+  Args:
+    input: A `Tensor`.
+    dtype: A type for the returned `Tensor`. Must be `float16`, `float32`,
+      `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`,
+      `complex64`, `complex128`, `bool` or `string`.
+    name: A name for the operation (optional).
+    and encode it as a constant.
+
+  Returns:
+    A `Tensor` with all elements set to zero.
+  """
+  zeros_like_impl(input, dtype, name, optimize=True)
+
+
+def zeros_like_impl(tensor, dtype, name, optimize=True):
+  """Internal implementation for the v1/v2 zeros_like API calls."""
   with ops.name_scope(name, "zeros_like", [tensor]) as name:
     tensor = ops.convert_to_tensor(tensor, name="tensor")
 
@@ -1793,7 +1830,7 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True):
       return gen_array_ops.zeros_like(tensor, name=name)
 
 
-@tf_export("ones_like")
+@tf_export(v1=["ones_like"])
 def ones_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to 1.
 
@@ -1820,6 +1857,43 @@ def ones_like(tensor, dtype=None, name=None, optimize=True):
   Returns:
     A `Tensor` with all elements set to 1.
   """
+  return ones_like_impl(tensor, dtype, name, optimize)
+
+
+@tf_export("ones_like", v1=[])
+def ones_like_v2(
+    input,  # pylint: disable=redefined-builtin
+    dtype=None,
+    name=None):
+  """Creates a tensor with all elements set to zero.
+
+  Given a single tensor (`tensor`), this operation returns a tensor of the
+  same type and shape as `tensor` with all elements set to zero. Optionally,
+  you can use `dtype` to specify a new type for the returned tensor.
+
+  For example:
+
+  ```python
+  tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
+  tf.ones_like(tensor)  # [[1, 1, 1], [1, 1, 1]]
+  ```
+
+  Args:
+    input: A `Tensor`.
+    dtype: A type for the returned `Tensor`. Must be `float16`, `float32`,
+      `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`,
+      `complex64`, `complex128`, `bool` or `string`.
+    name: A name for the operation (optional).
+    and encode it as a constant.
+
+  Returns:
+    A `Tensor` with all elements set to zero.
+  """
+  return ones_like_impl(input, dtype, name, optimize=True)
+
+
+def ones_like_impl(tensor, dtype, name, optimize=True):
+  """Internal implementation for the v1/v2 ones_like API calls."""
   with ops.name_scope(name, "ones_like", [tensor]) as name:
     tensor = ops.convert_to_tensor(tensor, name="tensor")
     ones_shape = shape_internal(tensor, optimize=optimize)
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index d37f83efdc..51210e77d4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -902,7 +902,7 @@ tf_module {
   }
   member_method {
     name: "ones_like"
-    argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
+    argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "pad"
@@ -1206,6 +1206,6 @@ tf_module {
   }
   member_method {
     name: "zeros_like"
-    argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
+    argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
 }
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 41948441ec..d7428a2417 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -50,7 +50,13 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         },
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
-        }
+        },
+        "tf.zeros_like": {
+            "tensor": "input",
+        },
+        "tf.ones_like": {
+            "tensor": "input",
+        },
     }
 
     # Mapping from function to the new name of the function
-- 
GitLab


From bec2d2d4b58d6d52014c6d0fb52268f6fac12fd4 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Nov 2018 21:26:33 -0800
Subject: [PATCH 0377/1554] Automated rollback of commit
 186744c73b9d0e64933bbc9e15db9ce2938e9e6b

PiperOrigin-RevId: 221740619
---
 tensorflow/python/ops/array_ops.py            | 80 +------------------
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  8 +-
 3 files changed, 6 insertions(+), 86 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 3845e66b07..b7823ccc2f 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -1740,8 +1740,8 @@ def zeros(shape, dtype=dtypes.float32, name=None):
   return output
 
 
-@tf_export(v1=["zeros_like"])
-def zeros_like_v1(tensor, dtype=None, name=None, optimize=True):
+@tf_export("zeros_like")
+def zeros_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to zero.
 
   Given a single tensor (`tensor`), this operation returns a tensor of the
@@ -1767,43 +1767,6 @@ def zeros_like_v1(tensor, dtype=None, name=None, optimize=True):
   Returns:
     A `Tensor` with all elements set to zero.
   """
-  zeros_like_impl(tensor, dtype, name, optimize)
-
-
-@tf_export("zeros_like", v1=[])
-def zeros_like_v2(
-    input,  # pylint: disable=redefined-builtin
-    dtype=None,
-    name=None):
-  """Creates a tensor with all elements set to zero.
-
-  Given a single tensor (`tensor`), this operation returns a tensor of the
-  same type and shape as `tensor` with all elements set to zero. Optionally,
-  you can use `dtype` to specify a new type for the returned tensor.
-
-  For example:
-
-  ```python
-  tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
-  tf.zeros_like(tensor)  # [[0, 0, 0], [0, 0, 0]]
-  ```
-
-  Args:
-    input: A `Tensor`.
-    dtype: A type for the returned `Tensor`. Must be `float16`, `float32`,
-      `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`,
-      `complex64`, `complex128`, `bool` or `string`.
-    name: A name for the operation (optional).
-    and encode it as a constant.
-
-  Returns:
-    A `Tensor` with all elements set to zero.
-  """
-  zeros_like_impl(input, dtype, name, optimize=True)
-
-
-def zeros_like_impl(tensor, dtype, name, optimize=True):
-  """Internal implementation for the v1/v2 zeros_like API calls."""
   with ops.name_scope(name, "zeros_like", [tensor]) as name:
     tensor = ops.convert_to_tensor(tensor, name="tensor")
 
@@ -1830,7 +1793,7 @@ def zeros_like_impl(tensor, dtype, name, optimize=True):
       return gen_array_ops.zeros_like(tensor, name=name)
 
 
-@tf_export(v1=["ones_like"])
+@tf_export("ones_like")
 def ones_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to 1.
 
@@ -1857,43 +1820,6 @@ def ones_like(tensor, dtype=None, name=None, optimize=True):
   Returns:
     A `Tensor` with all elements set to 1.
   """
-  return ones_like_impl(tensor, dtype, name, optimize)
-
-
-@tf_export("ones_like", v1=[])
-def ones_like_v2(
-    input,  # pylint: disable=redefined-builtin
-    dtype=None,
-    name=None):
-  """Creates a tensor with all elements set to zero.
-
-  Given a single tensor (`tensor`), this operation returns a tensor of the
-  same type and shape as `tensor` with all elements set to zero. Optionally,
-  you can use `dtype` to specify a new type for the returned tensor.
-
-  For example:
-
-  ```python
-  tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
-  tf.ones_like(tensor)  # [[1, 1, 1], [1, 1, 1]]
-  ```
-
-  Args:
-    input: A `Tensor`.
-    dtype: A type for the returned `Tensor`. Must be `float16`, `float32`,
-      `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`,
-      `complex64`, `complex128`, `bool` or `string`.
-    name: A name for the operation (optional).
-    and encode it as a constant.
-
-  Returns:
-    A `Tensor` with all elements set to zero.
-  """
-  return ones_like_impl(input, dtype, name, optimize=True)
-
-
-def ones_like_impl(tensor, dtype, name, optimize=True):
-  """Internal implementation for the v1/v2 ones_like API calls."""
   with ops.name_scope(name, "ones_like", [tensor]) as name:
     tensor = ops.convert_to_tensor(tensor, name="tensor")
     ones_shape = shape_internal(tensor, optimize=optimize)
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 51210e77d4..d37f83efdc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -902,7 +902,7 @@ tf_module {
   }
   member_method {
     name: "ones_like"
-    argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "pad"
@@ -1206,6 +1206,6 @@ tf_module {
   }
   member_method {
     name: "zeros_like"
-    argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index d7428a2417..41948441ec 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -50,13 +50,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         },
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
-        },
-        "tf.zeros_like": {
-            "tensor": "input",
-        },
-        "tf.ones_like": {
-            "tensor": "input",
-        },
+        }
     }
 
     # Mapping from function to the new name of the function
-- 
GitLab


From 1cd869a9b52f5bea0041f27544dc741a5016d7d5 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Nov 2018 21:46:38 -0800
Subject: [PATCH 0378/1554] Fix the way we check if a different function is
 exported with the same name in TF 2.0. In this case, we don't want to add the
 function name to renames_v2.py since it is still available.

PiperOrigin-RevId: 221742137
---
 tensorflow/tools/compatibility/renames_v2.py  | 49 +------------------
 .../update/generate_v2_renames_map.py         | 24 +++++++--
 2 files changed, 23 insertions(+), 50 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 7ba6982f97..46d1bf445c 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -53,12 +53,9 @@ renames = {
     'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator',
     'tf.SparseFeature': 'tf.io.SparseFeature',
     'tf.TFRecordReader': 'tf.compat.v1.TFRecordReader',
-    'tf.TensorShape': 'tf.compat.v1.TensorShape',
     'tf.TextLineReader': 'tf.compat.v1.TextLineReader',
     'tf.VERSION': 'tf.version.VERSION',
     'tf.VarLenFeature': 'tf.io.VarLenFeature',
-    'tf.Variable': 'tf.compat.v1.Variable',
-    'tf.VariableAggregation': 'tf.compat.v1.VariableAggregation',
     'tf.VariableScope': 'tf.compat.v1.VariableScope',
     'tf.WholeFileReader': 'tf.compat.v1.WholeFileReader',
     'tf.accumulate_n': 'tf.math.accumulate_n',
@@ -67,8 +64,6 @@ renames = {
     'tf.add_to_collections': 'tf.compat.v1.add_to_collections',
     'tf.all_variables': 'tf.compat.v1.all_variables',
     'tf.angle': 'tf.math.angle',
-    'tf.argmax': 'tf.compat.v1.argmax',
-    'tf.argmin': 'tf.compat.v1.argmin',
     'tf.assert_greater_equal': 'tf.debugging.assert_greater_equal',
     'tf.assert_integer': 'tf.debugging.assert_integer',
     'tf.assert_less_equal': 'tf.debugging.assert_less_equal',
@@ -98,25 +93,12 @@ renames = {
     'tf.confusion_matrix': 'tf.math.confusion_matrix',
     'tf.conj': 'tf.math.conj',
     'tf.container': 'tf.compat.v1.container',
-    'tf.convert_to_tensor': 'tf.compat.v1.convert_to_tensor',
     'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices',
     'tf.convert_to_tensor_or_sparse_tensor': 'tf.compat.v1.convert_to_tensor_or_sparse_tensor',
     'tf.count_nonzero': 'tf.compat.v1.count_nonzero',
     'tf.count_up_to': 'tf.compat.v1.count_up_to',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
-    'tf.data.Dataset': 'tf.compat.v1.data.Dataset',
-    'tf.data.FixedLengthRecordDataset': 'tf.compat.v1.data.FixedLengthRecordDataset',
-    'tf.data.TFRecordDataset': 'tf.compat.v1.data.TFRecordDataset',
-    'tf.data.TextLineDataset': 'tf.compat.v1.data.TextLineDataset',
-    'tf.data.experimental.Counter': 'tf.compat.v1.data.experimental.Counter',
-    'tf.data.experimental.CsvDataset': 'tf.compat.v1.data.experimental.CsvDataset',
-    'tf.data.experimental.RandomDataset': 'tf.compat.v1.data.experimental.RandomDataset',
-    'tf.data.experimental.SqlDataset': 'tf.compat.v1.data.experimental.SqlDataset',
-    'tf.data.experimental.choose_from_datasets': 'tf.compat.v1.data.experimental.choose_from_datasets',
-    'tf.data.experimental.make_batched_features_dataset': 'tf.compat.v1.data.experimental.make_batched_features_dataset',
-    'tf.data.experimental.make_csv_dataset': 'tf.compat.v1.data.experimental.make_csv_dataset',
-    'tf.data.experimental.sample_from_datasets': 'tf.compat.v1.data.experimental.sample_from_datasets',
     'tf.decode_base64': 'tf.io.decode_base64',
     'tf.decode_compressed': 'tf.io.decode_compressed',
     'tf.decode_csv': 'tf.io.decode_csv',
@@ -126,7 +108,6 @@ renames = {
     'tf.depth_to_space': 'tf.nn.depth_to_space',
     'tf.dequantize': 'tf.quantization.dequantize',
     'tf.deserialize_many_sparse': 'tf.io.deserialize_many_sparse',
-    'tf.device': 'tf.compat.v1.device',
     'tf.diag': 'tf.linalg.tensor_diag',
     'tf.diag_part': 'tf.linalg.tensor_diag_part',
     'tf.digamma': 'tf.math.digamma',
@@ -158,7 +139,6 @@ renames = {
     'tf.encode_base64': 'tf.io.encode_base64',
     'tf.erf': 'tf.math.erf',
     'tf.erfc': 'tf.math.erfc',
-    'tf.expand_dims': 'tf.compat.v1.expand_dims',
     'tf.expm1': 'tf.math.expm1',
     'tf.extract_image_patches': 'tf.image.extract_image_patches',
     'tf.fake_quant_with_min_max_args': 'tf.quantization.fake_quant_with_min_max_args',
@@ -167,20 +147,8 @@ renames = {
     'tf.fake_quant_with_min_max_vars_gradient': 'tf.quantization.fake_quant_with_min_max_vars_gradient',
     'tf.fake_quant_with_min_max_vars_per_channel': 'tf.quantization.fake_quant_with_min_max_vars_per_channel',
     'tf.fake_quant_with_min_max_vars_per_channel_gradient': 'tf.quantization.fake_quant_with_min_max_vars_per_channel_gradient',
-    'tf.feature_column.bucketized_column': 'tf.compat.v1.feature_column.bucketized_column',
-    'tf.feature_column.categorical_column_with_hash_bucket': 'tf.compat.v1.feature_column.categorical_column_with_hash_bucket',
-    'tf.feature_column.categorical_column_with_identity': 'tf.compat.v1.feature_column.categorical_column_with_identity',
-    'tf.feature_column.categorical_column_with_vocabulary_file': 'tf.compat.v1.feature_column.categorical_column_with_vocabulary_file',
-    'tf.feature_column.categorical_column_with_vocabulary_list': 'tf.compat.v1.feature_column.categorical_column_with_vocabulary_list',
-    'tf.feature_column.crossed_column': 'tf.compat.v1.feature_column.crossed_column',
-    'tf.feature_column.embedding_column': 'tf.compat.v1.feature_column.embedding_column',
-    'tf.feature_column.indicator_column': 'tf.compat.v1.feature_column.indicator_column',
     'tf.feature_column.input_layer': 'tf.compat.v1.feature_column.input_layer',
     'tf.feature_column.linear_model': 'tf.compat.v1.feature_column.linear_model',
-    'tf.feature_column.make_parse_example_spec': 'tf.compat.v1.feature_column.make_parse_example_spec',
-    'tf.feature_column.numeric_column': 'tf.compat.v1.feature_column.numeric_column',
-    'tf.feature_column.shared_embedding_columns': 'tf.compat.v1.feature_column.shared_embedding_columns',
-    'tf.feature_column.weighted_categorical_column': 'tf.compat.v1.feature_column.weighted_categorical_column',
     'tf.fft': 'tf.signal.fft',
     'tf.fft2d': 'tf.signal.fft2d',
     'tf.fft3d': 'tf.signal.fft3d',
@@ -272,7 +240,6 @@ renames = {
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log_sigmoid': 'tf.math.log_sigmoid',
     'tf.logical_xor': 'tf.math.logical_xor',
-    'tf.losses.Reduction': 'tf.compat.v1.losses.Reduction',
     'tf.make_template': 'tf.compat.v1.make_template',
     'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto',
     'tf.manip.batch_to_space_nd': 'tf.batch_to_space_nd',
@@ -284,8 +251,6 @@ renames = {
     'tf.manip.space_to_batch_nd': 'tf.space_to_batch_nd',
     'tf.manip.tile': 'tf.tile',
     'tf.matching_files': 'tf.io.matching_files',
-    'tf.math.argmax': 'tf.compat.v1.math.argmax',
-    'tf.math.argmin': 'tf.compat.v1.math.argmin',
     'tf.matrix_band_part': 'tf.linalg.band_part',
     'tf.matrix_determinant': 'tf.linalg.det',
     'tf.matrix_diag': 'tf.linalg.diag',
@@ -309,9 +274,9 @@ renames = {
     'tf.nn.softmax_cross_entropy_with_logits_v2': 'tf.nn.softmax_cross_entropy_with_logits',
     'tf.nn.static_rnn': 'tf.compat.v1.nn.static_rnn',
     'tf.nn.uniform_candidate_sampler': 'tf.random.uniform_candidate_sampler',
+    'tf.nn.xw_plus_b': 'tf.compat.v1.nn.xw_plus_b',
     'tf.op_scope': 'tf.compat.v1.op_scope',
     'tf.orthogonal_initializer': 'tf.keras.initializers.Orthogonal',
-    'tf.pad': 'tf.compat.v1.pad',
     'tf.parse_example': 'tf.io.parse_example',
     'tf.parse_single_example': 'tf.io.parse_single_example',
     'tf.parse_single_sequence_example': 'tf.io.parse_single_sequence_example',
@@ -418,6 +383,7 @@ renames = {
     'tf.serialize_many_sparse': 'tf.io.serialize_many_sparse',
     'tf.serialize_sparse': 'tf.io.serialize_sparse',
     'tf.serialize_tensor': 'tf.io.serialize_tensor',
+    'tf.set_random_seed': 'tf.random.set_random_seed',
     'tf.setdiff1d': 'tf.compat.v1.setdiff1d',
     'tf.sets.set_difference': 'tf.sets.difference',
     'tf.sets.set_intersection': 'tf.sets.intersection',
@@ -475,7 +441,6 @@ renames = {
     'tf.string_to_number': 'tf.strings.to_number',
     'tf.svd': 'tf.linalg.svd',
     'tf.tables_initializer': 'tf.compat.v1.tables_initializer',
-    'tf.test.assert_equal_graph_def': 'tf.compat.v1.test.assert_equal_graph_def',
     'tf.test.compute_gradient': 'tf.compat.v1.test.compute_gradient',
     'tf.test.compute_gradient_error': 'tf.compat.v1.test.compute_gradient_error',
     'tf.test.get_temp_dir': 'tf.compat.v1.test.get_temp_dir',
@@ -502,11 +467,8 @@ renames = {
     'tf.train.batch': 'tf.compat.v1.train.batch',
     'tf.train.batch_join': 'tf.compat.v1.train.batch_join',
     'tf.train.checkpoint_exists': 'tf.compat.v1.train.checkpoint_exists',
-    'tf.train.cosine_decay': 'tf.compat.v1.train.cosine_decay',
-    'tf.train.cosine_decay_restarts': 'tf.compat.v1.train.cosine_decay_restarts',
     'tf.train.create_global_step': 'tf.compat.v1.train.create_global_step',
     'tf.train.do_quantize_training_on_graphdef': 'tf.compat.v1.train.do_quantize_training_on_graphdef',
-    'tf.train.exponential_decay': 'tf.compat.v1.train.exponential_decay',
     'tf.train.export_meta_graph': 'tf.compat.v1.train.export_meta_graph',
     'tf.train.generate_checkpoint_state_proto': 'tf.compat.v1.train.generate_checkpoint_state_proto',
     'tf.train.get_checkpoint_mtimes': 'tf.compat.v1.train.get_checkpoint_mtimes',
@@ -516,18 +478,12 @@ renames = {
     'tf.train.import_meta_graph': 'tf.compat.v1.train.import_meta_graph',
     'tf.train.init_from_checkpoint': 'tf.compat.v1.train.init_from_checkpoint',
     'tf.train.input_producer': 'tf.compat.v1.train.input_producer',
-    'tf.train.inverse_time_decay': 'tf.compat.v1.train.inverse_time_decay',
     'tf.train.limit_epochs': 'tf.compat.v1.train.limit_epochs',
-    'tf.train.linear_cosine_decay': 'tf.compat.v1.train.linear_cosine_decay',
     'tf.train.match_filenames_once': 'tf.io.match_filenames_once',
     'tf.train.maybe_batch': 'tf.compat.v1.train.maybe_batch',
     'tf.train.maybe_batch_join': 'tf.compat.v1.train.maybe_batch_join',
     'tf.train.maybe_shuffle_batch': 'tf.compat.v1.train.maybe_shuffle_batch',
     'tf.train.maybe_shuffle_batch_join': 'tf.compat.v1.train.maybe_shuffle_batch_join',
-    'tf.train.natural_exp_decay': 'tf.compat.v1.train.natural_exp_decay',
-    'tf.train.noisy_linear_cosine_decay': 'tf.compat.v1.train.noisy_linear_cosine_decay',
-    'tf.train.piecewise_constant': 'tf.compat.v1.train.piecewise_constant',
-    'tf.train.polynomial_decay': 'tf.compat.v1.train.polynomial_decay',
     'tf.train.queue_runner.QueueRunner': 'tf.compat.v1.train.queue_runner.QueueRunner',
     'tf.train.queue_runner.add_queue_runner': 'tf.compat.v1.train.queue_runner.add_queue_runner',
     'tf.train.queue_runner.start_queue_runners': 'tf.compat.v1.train.queue_runner.start_queue_runners',
@@ -549,7 +505,6 @@ renames = {
     'tf.unsorted_segment_prod': 'tf.math.unsorted_segment_prod',
     'tf.unsorted_segment_sqrt_n': 'tf.math.unsorted_segment_sqrt_n',
     'tf.unsorted_segment_sum': 'tf.math.unsorted_segment_sum',
-    'tf.variable_creator_scope': 'tf.compat.v1.variable_creator_scope',
     'tf.variable_op_scope': 'tf.compat.v1.variable_op_scope',
     'tf.variable_scope': 'tf.compat.v1.variable_scope',
     'tf.variables_initializer': 'tf.compat.v1.variables_initializer',
diff --git a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
index 9a7bae0da3..949946c827 100644
--- a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
+++ b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
@@ -78,6 +78,26 @@ def get_canonical_name(v2_names, v1_name):
   return 'compat.v1.%s' % v1_name
 
 
+def get_all_v2_names():
+  """Get a set of function/class names available in TensorFlow 2.0."""
+  v2_names = set()  # All op names in TensorFlow 2.0
+
+  def visit(unused_path, unused_parent, children):
+    """Visitor that collects TF 2.0 names."""
+    for child in children:
+      _, attr = tf_decorator.unwrap(child[1])
+      if not hasattr(attr, '__dict__'):
+        continue
+      api_names_v2 = attr.__dict__.get(_TENSORFLOW_API_ATTR, [])
+      for name in api_names_v2:
+        v2_names.add(name)
+
+  visitor = public_api.PublicAPIVisitor(visit)
+  visitor.do_not_descend_map['tf'].append('contrib')
+  traverse.traverse(tf.compat.v2, visitor)
+  return v2_names
+
+
 def collect_constant_renames():
   """Looks for constants that need to be renamed in TF 2.0.
 
@@ -120,7 +140,6 @@ def collect_function_renames():
   # Set of rename lines to write to output file in the form:
   #   'tf.deprecated_name': 'tf.canonical_name'
   renames = set()
-  v2_names = set()  # All op names in TensorFlow 2.0
 
   def visit(unused_path, unused_parent, children):
     """Visitor that collects rename strings to add to rename_line_set."""
@@ -133,8 +152,6 @@ def collect_function_renames():
       deprecated_api_names = set(api_names_v1) - set(api_names_v2)
       for name in deprecated_api_names:
         renames.add((name, get_canonical_name(api_names_v2, name)))
-      for name in api_names_v2:
-        v2_names.add(name)
 
   visitor = public_api.PublicAPIVisitor(visit)
   visitor.do_not_descend_map['tf'].append('contrib')
@@ -144,6 +161,7 @@ def collect_function_renames():
   # It is possible that a different function is exported with the
   # same name. For e.g. when creating a different function to
   # rename arguments. Exclude it from renames in this case.
+  v2_names = get_all_v2_names()
   renames = set((name, new_name) for name, new_name in renames
                 if name not in v2_names)
   return renames
-- 
GitLab


From 489e181be77b83b2b631f48968aaf40897001838 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Fri, 16 Nov 2018 11:56:50 +0530
Subject: [PATCH 0379/1554] Update graph_transformations.h

---
 .../lite/toco/graph_transformations/graph_transformations.h     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
index 73a90c8239..187b584b69 100644
--- a/tensorflow/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
@@ -139,7 +139,7 @@ DECLARE_GRAPH_TRANSFORMATION(MakeInitialDequantizeOperator)
 DECLARE_GRAPH_TRANSFORMATION(MoveBinaryOperatorBeforeReshape)
 DECLARE_GRAPH_TRANSFORMATION(PropagateActivationFunctionIntoConstants)
 DECLARE_GRAPH_TRANSFORMATION(PropagateArrayDataTypes)
-DECLARE_GRAPH_TRANSFORMATION(PropagateFakeQuantNumBits);
+DECLARE_GRAPH_TRANSFORMATION(PropagateFakeQuantNumBits)
 DECLARE_GRAPH_TRANSFORMATION(PropagateFixedSizes)
 DECLARE_GRAPH_TRANSFORMATION(HardcodeMinMax)
 DECLARE_GRAPH_TRANSFORMATION(Quantize)
-- 
GitLab


From 49c91cd9e3ead217eb31d2b06e51b6463d5f82f5 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 22:22:53 -0800
Subject: [PATCH 0380/1554] [XLA] Print the contents of VerifiedHloModules that
 fail verification.

Aids in debugging when this fails.

PiperOrigin-RevId: 221744981
---
 tensorflow/compiler/xla/tests/hlo_test_base.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index 2f467a26b0..989a7c705a 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -99,6 +99,8 @@ void VerifiedHloModule::VerifyOrAddFailure(const string& message) {
     ADD_FAILURE() << "HloVerifier failed on module " << name()
                   << (message.empty() ? "" : absl::StrCat(" (", message, ")"))
                   << ": " << status;
+    LOG(ERROR) << "Contents of bad module:";
+    XLA_LOG_LINES(tensorflow::ERROR, ToString());
   }
 }
 
-- 
GitLab


From c4ba167d474e354b5a4749e608e85d192bf27f11 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 22:32:02 -0800
Subject: [PATCH 0381/1554] [TF:XLA] Remove legacy_flags:: namespace now that
 it no longer matches the directory structure.

PiperOrigin-RevId: 221745541
---
 tensorflow/c/c_api_experimental.cc                   |  8 ++++----
 tensorflow/compiler/jit/build_xla_ops_pass.cc        |  8 ++++----
 tensorflow/compiler/jit/flags.cc                     |  2 --
 tensorflow/compiler/jit/flags.h                      |  2 --
 .../jit/increase_dynamism_for_auto_jit_pass.cc       |  3 +--
 tensorflow/compiler/jit/kernels/xla_ops.cc           |  2 +-
 tensorflow/compiler/jit/mark_for_compilation_pass.cc | 12 ++++--------
 tensorflow/compiler/jit/xla_cpu_device.cc            |  2 +-
 tensorflow/compiler/tf2xla/dump_graph.cc             |  3 +--
 tensorflow/compiler/tf2xla/xla_op_registry.cc        |  3 +--
 10 files changed, 17 insertions(+), 28 deletions(-)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index bc434e7593..7fe74b5db3 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -51,8 +51,8 @@ void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) {
     // These XLA flags are needed to trigger XLA properly from C (more generally
     // non-Python) clients. If this API is called again with `enable` set to
     // false, it is safe to keep these flag values as is.
-    tensorflow::legacy_flags::MarkForCompilationPassFlags* flags =
-        tensorflow::legacy_flags::GetMarkForCompilationPassFlags();
+    tensorflow::MarkForCompilationPassFlags* flags =
+        tensorflow::GetMarkForCompilationPassFlags();
     flags->tf_xla_cpu_global_jit = true;
     flags->tf_xla_min_cluster_size = 1;
   } else {
@@ -71,8 +71,8 @@ TF_Buffer* TF_CreateConfig(unsigned char enable_xla_compilation,
     // These XLA flags are needed to trigger XLA properly from C (more generally
     // non-Python) clients. If this API is called again with `enable` set to
     // false, it is safe to keep these flag values as is.
-    tensorflow::legacy_flags::MarkForCompilationPassFlags* flags =
-        tensorflow::legacy_flags::GetMarkForCompilationPassFlags();
+    tensorflow::MarkForCompilationPassFlags* flags =
+        tensorflow::GetMarkForCompilationPassFlags();
     flags->tf_xla_cpu_global_jit = true;
     flags->tf_xla_min_cluster_size = 1;
   } else {
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass.cc b/tensorflow/compiler/jit/build_xla_ops_pass.cc
index 01bea34af9..9f4042630e 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass.cc
@@ -320,10 +320,10 @@ Status BuildXlaOpsPass::Run(const GraphOptimizationPassOptions& options) {
                     return IsXlaCompiledKernel(*n);
                   });
 
-  bool lazy_compilation_enabled = enable_lazy_compilation_
-                                      ? *enable_lazy_compilation_
-                                      : legacy_flags::GetBuildXlaOpsPassFlags()
-                                            .tf_xla_enable_lazy_compilation;
+  bool lazy_compilation_enabled =
+      enable_lazy_compilation_
+          ? *enable_lazy_compilation_
+          : GetBuildXlaOpsPassFlags().tf_xla_enable_lazy_compilation;
 
   for (Node* n : xla_compiled_kernels) {
     TF_RETURN_IF_ERROR(ReplaceNodeWithXlaCompileAndXlaRun(
diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc
index 702197e1b1..98e344b3a0 100644
--- a/tensorflow/compiler/jit/flags.cc
+++ b/tensorflow/compiler/jit/flags.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/core/util/command_line_flags.h"
 
 namespace tensorflow {
-namespace legacy_flags {
 namespace {
 
 BuildXlaOpsPassFlags* build_ops_flags;
@@ -150,5 +149,4 @@ void AppendDumpGraphFlags(std::vector<Flag>* flag_list) {
   AppendDumpGraphFlagsInternal(flag_list);
 }
 
-}  // namespace legacy_flags
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/flags.h b/tensorflow/compiler/jit/flags.h
index 50f4f35836..5ddea588ee 100644
--- a/tensorflow/compiler/jit/flags.h
+++ b/tensorflow/compiler/jit/flags.h
@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow/core/util/command_line_flags.h"
 
 namespace tensorflow {
-namespace legacy_flags {
 
 // Flags associated with the XLA bridge's mark_for_compilation_pass module.
 struct MarkForCompilationPassFlags {
@@ -99,7 +98,6 @@ void AppendMarkForCompilationPassFlags(
     std::vector<tensorflow::Flag>* flag_list);
 void AppendDumpGraphFlags(std::vector<tensorflow::Flag>* flag_list);
 
-}  // namespace legacy_flags
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_COMPILER_JIT_FLAGS_H_
diff --git a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
index a8295bb77b..ce53f70b79 100644
--- a/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
+++ b/tensorflow/compiler/jit/increase_dynamism_for_auto_jit_pass.cc
@@ -345,8 +345,7 @@ Status FindAndRewriteSlices(Graph* g, bool* changed) {
 
 Status IncreaseDynamismForAutoJitPass::Run(
     const GraphOptimizationPassOptions& options) {
-  legacy_flags::MarkForCompilationPassFlags* flags =
-      legacy_flags::GetMarkForCompilationPassFlags();
+  MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags();
   if (flags->tf_xla_clustering_debug) {
     dump_graph::DumpGraphToFile("before_increase_dynamism_for_auto_jit_pass",
                                 **options.graph, options.flib_def);
diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc
index 2dea0c5330..ad71df5a69 100644
--- a/tensorflow/compiler/jit/kernels/xla_ops.cc
+++ b/tensorflow/compiler/jit/kernels/xla_ops.cc
@@ -418,7 +418,7 @@ void XlaCompileOp::Compute(OpKernelContext* ctx) {
     cannot_compile_cluster = cannot_compile_cluster_;
   }
 
-  if (legacy_flags::GetXlaOpsCommonFlags().tf_xla_always_defer_compilation ||
+  if (GetXlaOpsCommonFlags().tf_xla_always_defer_compilation ||
       cannot_compile_cluster) {
     executable = nullptr;
   } else {
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 9b02b0c3f9..60b962d2e8 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -427,8 +427,7 @@ Status FindCompilationCandidates(
       BackwardsConstAnalysis(graph, /*compile_time_const_arg_indices=*/nullptr,
                              &compile_time_const_nodes));
 
-  int64& fuel =
-      legacy_flags::GetMarkForCompilationPassFlags()->tf_xla_clustering_fuel;
+  int64& fuel = GetMarkForCompilationPassFlags()->tf_xla_clustering_fuel;
 
   // Iterate over nodes in sorted order so that compiler fuel is deterministic.
   // We can't simply pass op_nodes().begin() and op_nodes().end to the
@@ -607,8 +606,7 @@ OptimizerOptions::GlobalJitLevel GetGlobalJitLevel(
     // To set compilation to be on by default, change the following line.
     global_jit_level = OptimizerOptions::OFF;
   }
-  legacy_flags::MarkForCompilationPassFlags* flags =
-      legacy_flags::GetMarkForCompilationPassFlags();
+  MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags();
   if (flags->tf_xla_auto_jit == -1 ||
       (1 <= flags->tf_xla_auto_jit && flags->tf_xla_auto_jit <= 2)) {
     // If the flag tf_xla_auto_jit is a valid, non-zero setting, it overrides
@@ -651,8 +649,7 @@ Status MarkForCompilationPass::Run(
   // device ahead of time.
   OptimizerOptions::GlobalJitLevel global_jit_level =
       GetGlobalJitLevel(options);
-  legacy_flags::MarkForCompilationPassFlags* flags =
-      legacy_flags::GetMarkForCompilationPassFlags();
+  MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags();
   bool fusion_only = flags->tf_xla_fusion_only;
 
   VLOG(1) << "flags->tf_xla_fusion_only = " << flags->tf_xla_fusion_only;
@@ -953,8 +950,7 @@ Status MarkForCompilationPass::RunImpl(
 
   OptimizerOptions::GlobalJitLevel global_jit_level =
       GetGlobalJitLevel(options);
-  legacy_flags::MarkForCompilationPassFlags* flags =
-      legacy_flags::GetMarkForCompilationPassFlags();
+  MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags();
 
   // Repeatedly contract edges between clusters that are on the same device,
   // provided the contraction would not create a cycle.
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 13126e0878..9006dd514b 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -37,7 +37,7 @@ class XlaCpuDeviceFactory : public DeviceFactory {
 Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
                                           const string& name_prefix,
                                           std::vector<Device*>* devices) {
-  legacy_flags::XlaDeviceFlags* flags = legacy_flags::GetXlaDeviceFlags();
+  XlaDeviceFlags* flags = GetXlaDeviceFlags();
   bool compile_on_demand = flags->tf_xla_compile_on_demand;
 
   XlaOpRegistry::DeviceRegistration registration;
diff --git a/tensorflow/compiler/tf2xla/dump_graph.cc b/tensorflow/compiler/tf2xla/dump_graph.cc
index 34f891890a..1de85004a5 100644
--- a/tensorflow/compiler/tf2xla/dump_graph.cc
+++ b/tensorflow/compiler/tf2xla/dump_graph.cc
@@ -61,8 +61,7 @@ string MakeUniqueFilename(string name) {
 string WriteTextProtoToUniqueFile(
     Env* env, const string& name, const char* proto_type,
     const ::tensorflow::protobuf::Message& proto) {
-  const string& dirname =
-      legacy_flags::GetDumpGraphFlags()->tf_dump_graph_prefix;
+  const string& dirname = GetDumpGraphFlags()->tf_dump_graph_prefix;
   Status status = env->RecursivelyCreateDir(dirname);
   if (!status.ok()) {
     LOG(WARNING) << "Failed to create " << dirname << " for dumping "
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.cc b/tensorflow/compiler/tf2xla/xla_op_registry.cc
index 49a6ebbbde..14237df690 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.cc
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.cc
@@ -130,8 +130,7 @@ XlaOpRegistry::~XlaOpRegistry() = default;
   // Lazily register the CPU and GPU JIT devices the first time
   // GetCompilationDevice is called.
   static void* registration_init = [&registry]() {
-    legacy_flags::MarkForCompilationPassFlags* flags =
-        legacy_flags::GetMarkForCompilationPassFlags();
+    MarkForCompilationPassFlags* flags = GetMarkForCompilationPassFlags();
     bool cpu_global_jit = flags->tf_xla_cpu_global_jit;
 
     mutex_lock lock(registry.mutex_);
-- 
GitLab


From 18d8ac67307fbb66b6f7d7abaede4fac5beed9f7 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 22:35:42 -0800
Subject: [PATCH 0382/1554] [XLA] Remove ShapeUtil::IsNil in favor of
 ShapeUtil::IsEmptyTuple.

No need to have two functions that do the same thing.

PiperOrigin-RevId: 221745815
---
 tensorflow/compiler/xla/literal_test.cc    |  6 +++---
 tensorflow/compiler/xla/shape_tree_test.cc |  4 ++--
 tensorflow/compiler/xla/shape_util.cc      |  4 ----
 tensorflow/compiler/xla/shape_util.h       |  3 ---
 tensorflow/compiler/xla/shape_util_test.cc | 10 +++++-----
 5 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index dac6067861..bd93517728 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -1587,9 +1587,9 @@ TEST_F(LiteralUtilTest, DecomposeTuple) {
   Literal nested_tuple = LiteralUtil::MakeTuple(
       {&tuple_elements[0], &tuple_elements[1], &nil_literal});
 
-  EXPECT_FALSE(ShapeUtil::IsNil(nested_tuple.shape()));
+  EXPECT_FALSE(ShapeUtil::IsEmptyTuple(nested_tuple.shape()));
   std::vector<Literal> elements = nested_tuple.DecomposeTuple();
-  EXPECT_TRUE(ShapeUtil::IsNil(nested_tuple.shape()));
+  EXPECT_TRUE(ShapeUtil::IsEmptyTuple(nested_tuple.shape()));
 
   ASSERT_EQ(elements.size(), 3);
 
@@ -1640,7 +1640,7 @@ TEST_F(LiteralUtilTest, MoveIntoTuple) {
   EXPECT_EQ(literal.Get<double>({1}, /*shape_index=*/{2, 1}), 44.0);
 
   for (const Literal& element : elements) {
-    EXPECT_TRUE(ShapeUtil::IsNil(element.shape()));
+    EXPECT_TRUE(ShapeUtil::IsEmptyTuple(element.shape()));
   }
 }
 
diff --git a/tensorflow/compiler/xla/shape_tree_test.cc b/tensorflow/compiler/xla/shape_tree_test.cc
index c8ff55e784..2b6c484bc4 100644
--- a/tensorflow/compiler/xla/shape_tree_test.cc
+++ b/tensorflow/compiler/xla/shape_tree_test.cc
@@ -52,10 +52,10 @@ class ShapeTreeTest : public ::testing::Test {
 
 TEST_F(ShapeTreeTest, DefaultConstructor) {
   ShapeTree<int> int_tree;
-  EXPECT_TRUE(ShapeUtil::IsNil(int_tree.shape()));
+  EXPECT_TRUE(ShapeUtil::IsEmptyTuple(int_tree.shape()));
 
   ShapeTree<bool> bool_tree;
-  EXPECT_TRUE(ShapeUtil::IsNil(bool_tree.shape()));
+  EXPECT_TRUE(ShapeUtil::IsEmptyTuple(bool_tree.shape()));
 }
 
 void ShapeTreeTest::TestShapeConstructor(const Shape& shape,
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index d0c35d8dee..7d011bfc65 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -372,10 +372,6 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   return IsTuple(shape) && TupleElementCount(shape) == 0;
 }
 
-/* static */ bool ShapeUtil::IsNil(const Shape& shape) {
-  return IsEmptyTuple(shape);
-}
-
 /* static */ int64 ShapeUtil::TupleElementCount(const Shape& shape) {
   CHECK(IsTuple(shape)) << HumanString(shape);
   return shape.tuple_shapes_size();
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index bb347bf0de..f9f1347460 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -473,9 +473,6 @@ class ShapeUtil {
   // Returns true if shape is an empty tuple.
   static bool IsEmptyTuple(const Shape& shape);
 
-  // Returns true if shape is the nil shape (an empty tuple).
-  static bool IsNil(const Shape& shape);
-
   // Returns the number of elements in the given tuple shape.
   // Precondition: IsTuple(shape)
   static int64 TupleElementCount(const Shape& shape);
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 0c647369a3..11b493323c 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -376,12 +376,12 @@ TEST(ShapeUtilTest, ByteSizeOfWithoutPadding) {
 }
 
 TEST(ShapeUtilTest, NilShape) {
-  EXPECT_TRUE(ShapeUtil::IsNil(ShapeUtil::MakeNil()));
-  EXPECT_FALSE(ShapeUtil::IsNil(ShapeUtil::MakeShape(F32, {1, 2, 3})));
-  EXPECT_FALSE(ShapeUtil::IsNil(ShapeUtil::MakeShape(F32, {0, 1})));
-  EXPECT_FALSE(ShapeUtil::IsNil(
+  EXPECT_TRUE(ShapeUtil::IsEmptyTuple(ShapeUtil::MakeNil()));
+  EXPECT_FALSE(ShapeUtil::IsEmptyTuple(ShapeUtil::MakeShape(F32, {1, 2, 3})));
+  EXPECT_FALSE(ShapeUtil::IsEmptyTuple(ShapeUtil::MakeShape(F32, {0, 1})));
+  EXPECT_FALSE(ShapeUtil::IsEmptyTuple(
       ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(S32, {})})));
-  EXPECT_FALSE(ShapeUtil::IsNil(
+  EXPECT_FALSE(ShapeUtil::IsEmptyTuple(
       ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {0})})));
 }
 
-- 
GitLab


From d17a4fcecdde116b9eae39ab21ae72e9b70a111e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 22:35:48 -0800
Subject: [PATCH 0383/1554] Internal change

PiperOrigin-RevId: 221745821
---
 tensorflow/contrib/distribute/python/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index bf5adb1a50..b8d6208150 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -752,10 +752,13 @@ cuda_py_test(
     ],
     shard_count = 16,
     tags = [
+        "manual",  # TODO(b/119646053): Failing when run.
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "noguitar",  # TODO(b/119646053): Failing when run.
+        "notap",  # TODO(b/119646053): Failing when run.
         "notsan",
     ],
 )
-- 
GitLab


From eec9c3f9551c3e414090ed25bb6756538e81d3ae Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Nov 2018 22:40:20 -0800
Subject: [PATCH 0384/1554] [XLA] Simplify away constants in while loop
 parameters.

If a while loop param is a constant and is the same in the loop's init and
body, then this parameter can be eliminated from the loop carry.

PiperOrigin-RevId: 221746124
---
 tensorflow/compiler/xla/service/BUILD         |   1 +
 .../xla/service/while_loop_simplifier.cc      | 172 ++++++++++++++++--
 .../xla/service/while_loop_simplifier_test.cc |  96 ++++++++++
 3 files changed, 257 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ccbe83bd96..3cbf0d0c8c 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1746,6 +1746,7 @@ tf_cc_test(
         ":hlo",
         ":hlo_dce",
         ":hlo_matchers",
+        ":tuple_simplifier",
         ":while_loop_simplifier",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index 6f924a29d8..646f75ca2a 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -302,6 +302,147 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
   return true;
 }
 
+// Removes each loop parameter (i.e. member of the while loop tuple) that is a
+// constant and is the same in the while loop body and the while loop init.
+static StatusOr<bool> TryRemoveConstantParams(HloInstruction* while_op) {
+  HloModule* module = while_op->GetModule();
+  HloComputation* computation = while_op->parent();
+  auto* while_init = while_op->mutable_operand(0);
+  auto* while_body = while_op->while_body();
+  auto* while_cond = while_op->while_condition();
+  auto* while_body_root = while_body->root_instruction();
+  if (while_init->opcode() != HloOpcode::kTuple ||
+      while_body_root->opcode() != HloOpcode::kTuple) {
+    return false;
+  }
+
+  TF_RET_CHECK(while_cond->num_parameters() == 1);
+  TF_RET_CHECK(while_body->num_parameters() == 1);
+  TF_RET_CHECK(
+      ShapeUtil::Compatible(while_init->shape(), while_body_root->shape()));
+
+  absl::flat_hash_set<int64> constant_tuple_indices;
+  const auto& while_shape = while_init->shape();
+  for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+    auto* init_elem = while_init->operand(i);
+    auto* body_elem = while_body_root->operand(i);
+    if (init_elem->opcode() == HloOpcode::kConstant &&
+        body_elem->opcode() == HloOpcode::kConstant &&
+        init_elem->literal() == body_elem->literal()) {
+      constant_tuple_indices.insert(i);
+    }
+  }
+
+  if (constant_tuple_indices.empty()) {
+    return false;
+  }
+
+  // OK, we found some constant elements of the while parameter!  Eliminate
+  // them.
+  std::vector<Shape> new_while_shape_elems;
+  for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+    if (!constant_tuple_indices.count(i)) {
+      new_while_shape_elems.push_back(while_shape.tuple_shapes(i));
+    }
+  }
+  Shape new_while_shape = ShapeUtil::MakeTupleShape(new_while_shape_elems);
+
+  // `new_instrs` holds instructions created outside of a computation for
+  // cloning.  Elements added here just need to live until the end of the
+  // relevant CloneWithReplacement call.
+  std::vector<std::unique_ptr<HloInstruction>> new_instrs;
+  auto add_new_instr = [&](std::unique_ptr<HloInstruction> instr) {
+    new_instrs.push_back(std::move(instr));
+    return new_instrs.back().get();
+  };
+
+  // Returns a new tuple without the elements of constant_tuple_indices.
+  auto remove_constant_elems = [&](HloInstruction* instr) {
+    CHECK(ShapeUtil::Compatible(instr->shape(), while_shape));
+
+    std::vector<HloInstruction*> tuple_elems;
+    for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+      if (!constant_tuple_indices.count(i)) {
+        tuple_elems.push_back(
+            add_new_instr(HloInstruction::CreateGetTupleElement(
+                while_shape.tuple_shapes(i), instr, i)));
+      }
+    }
+    return HloInstruction::CreateTuple(tuple_elems);
+  };
+
+  auto add_constant_elems = [&](HloInstruction* instr) {
+    CHECK(ShapeUtil::Compatible(instr->shape(), new_while_shape));
+
+    std::vector<HloInstruction*> tuple_elems;
+    int64 j = 0;
+    for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+      if (constant_tuple_indices.count(i)) {
+        tuple_elems.push_back(while_init->mutable_operand(i));
+      } else {
+        tuple_elems.push_back(
+            add_new_instr(HloInstruction::CreateGetTupleElement(
+                while_shape.tuple_shapes(i), instr, j)));
+        ++j;
+      }
+    }
+    return HloInstruction::CreateTuple(tuple_elems);
+  };
+
+  // Special case: constant_tuple_indices covers the whole while parameter, so
+  // the new while shape is the empty tuple.  In this case, the value of the
+  // while loop is simply equal to the value of `init`.
+  //
+  // It's unfortunate to special-case this, but it's simpler than the
+  // alternative.  The problem is that if our while parameter has no
+  // non-constant elems, the tuple returned by `add_constant_elems` won't depend
+  // on instr (the loop body/cond parameter), and therefore
+  // CloneWithReplacementPairs will *leave the parameter out entirely*, creating
+  // invalid HLO.
+  if (ShapeUtil::IsEmptyTuple(new_while_shape)) {
+    TF_RETURN_IF_ERROR(computation->ReplaceInstruction(while_op, while_init));
+    return true;
+  }
+
+  std::unique_ptr<HloComputation> new_while_cond =
+      while_cond->CloneWithReplacementPairs({
+          while_cond->parameter_instruction(0),
+          add_constant_elems(add_new_instr(HloInstruction::CreateParameter(
+              0, new_while_shape,
+              while_cond->parameter_instruction(0)->name()))),
+      });
+
+  std::unique_ptr<HloComputation> new_while_body =
+      while_body->CloneWithReplacementPairs(
+          {
+              while_body->parameter_instruction(0),
+              add_constant_elems(add_new_instr(HloInstruction::CreateParameter(
+                  0, new_while_shape,
+                  while_cond->parameter_instruction(0)->name()))),
+          },
+          {
+              while_body->root_instruction(),
+              remove_constant_elems(
+                  add_new_instr(while_body->root_instruction()->Clone())),
+          });
+
+  // Create the final while loop, and add any new instructions created to
+  // `computation`.
+  new_instrs.clear();
+  TF_RETURN_IF_ERROR(computation->ReplaceWithNewInstruction(
+      while_op,
+      add_constant_elems(
+          computation->AddInstruction(HloInstruction::CreateWhile(
+              new_while_shape,
+              module->AddEmbeddedComputation(std::move(new_while_cond)),
+              module->AddEmbeddedComputation(std::move(new_while_body)),
+              add_new_instr(remove_constant_elems(while_init)))))));
+  for (auto& instr : new_instrs) {
+    computation->AddInstruction(std::move(instr));
+  }
+  return true;
+}
+
 // Tries to remove a while loop from the graph.
 //
 //  - Loops with trip count of 0 can be replaced by the loop's "init" value.
@@ -519,14 +660,6 @@ static StatusOr<bool> TryFlattenNestedTuples(HloInstruction* while_op) {
     return false;
   }
 
-  // Cowardly refuse to perform this optimization in the presence of kDomain
-  // instructions, which may reference other instructions in the loop and
-  // therefore make this complicated.
-  if (ContainsInstrWithOpcode(while_body, {HloOpcode::kDomain}) ||
-      ContainsInstrWithOpcode(while_cond, {HloOpcode::kDomain})) {
-    return false;
-  }
-
   std::vector<Shape> flattened_shape_elems;
   ShapeUtil::ForEachSubshape(while_shape,
                              [&](const Shape& s, const ShapeIndex& /*index*/) {
@@ -650,19 +783,34 @@ StatusOr<bool> WhileLoopSimplifier::Run(HloModule* module) {
       continue;
     }
 
+    // TODO(b/119281462): Cowardly refuse to perform any of the following
+    // optimizations in the presence of kDomain instructions.  It seems that
+    // modifying a while loop's tuple doesn't work when kDomain is present.
+    if (ContainsInstrWithOpcode(while_op->while_body(), {HloOpcode::kDomain}) ||
+        ContainsInstrWithOpcode(while_op->while_condition(),
+                                {HloOpcode::kDomain})) {
+      continue;
+    }
+
+    // Each of the optimizations below modifies the while loop itself if it's
+    // successful, meaning that `while_op` is no longer valid after one of these
+    // transformations returns true.
+
     TF_ASSIGN_OR_RETURN(result, TryFlattenNestedTuples(while_op));
     changed |= result;
     if (result) {
-      // Successfully flattening nested tuples results in us cloning and
-      // replacing the while loop, meaning that `while_op` is no longer valid.
       continue;
     }
 
     TF_ASSIGN_OR_RETURN(result, TryRemoveDeadWhileParams(while_op));
     changed |= result;
     if (result) {
-      // Successfully removing dead while params results in us cloning and
-      // replacing the while loop, meaning that `while_op` is no longer valid.
+      continue;
+    }
+
+    TF_ASSIGN_OR_RETURN(result, TryRemoveConstantParams(while_op));
+    changed |= result;
+    if (result) {
       continue;
     }
   }
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
index 05005e0b26..e036c2aabc 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_dce.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/tuple_simplifier.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
@@ -27,6 +28,7 @@ limitations under the License.
 namespace xla {
 namespace {
 
+using ::testing::_;
 namespace op = xla::testing::opcode_matchers;
 
 class WhileLoopSimplifierTest : public HloTestBase {
@@ -563,5 +565,99 @@ TEST_F(WhileLoopSimplifierTest, FlattenNestedTuple) {
           .ValueOrDie()));
 }
 
+// Edge-case: All elements of the loop carry are constants which can be removed,
+// leaving us with a nullary loop.  This is a special case, we just replace the
+// loop with its init.
+TEST_F(WhileLoopSimplifierTest, OnlyConstantsInLoopCarry) {
+  const string hlo_string = R"(
+  HloModule Test
+  Body {
+    param = (s32[1]) parameter(0)
+    a = s32[1] constant({0})
+    ROOT tuple = (s32[1]) tuple(a)
+  }
+  Cond {
+    param = (s32[1]) parameter(0)
+    ROOT cond = pred[] constant(true)
+  }
+  ENTRY Loop {
+    a = s32[1] constant({0})
+    init = (s32[1]) tuple(a)
+    ROOT while = (s32[1]) while(init), condition=Cond, body=Body
+  })";
+
+  auto m = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
+  EXPECT_TRUE(WhileLoopSimplifier().Run(m.get()).ValueOrDie());
+  EXPECT_TRUE(HloDCE().Run(m.get()).ok());
+  EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              op::Tuple(op::Constant()));
+}
+
+TEST_F(WhileLoopSimplifierTest, RemoveConstantFromLoopCarry) {
+  const string hlo_string = R"(
+  HloModule Test
+  Body {
+    param = (s32[1], s32[2], s32[3]) parameter(0)
+    a = s32[1] get-tuple-element(param), index=0
+    a.1 = s32[1] add(a, a)
+    b = s32[2] constant({1,1})
+    c = s32[3] constant({10,10,10})
+    ROOT tuple = (s32[1], s32[2], s32[3]) tuple(a.1, b, c)
+  }
+  Cond {
+    param = (s32[1], s32[2], s32[3]) parameter(0)
+    /* Use each tuple element.  The verifier will then ensure that if any of
+     * these get modified, they're replaced with values of the correct shape. */
+    a = s32[1] get-tuple-element(param), index=0
+    b = s32[2] get-tuple-element(param), index=1
+    c = s32[3] get-tuple-element(param), index=2
+    ROOT cond = pred[] constant(true)
+  }
+  ENTRY Loop {
+    /* Only `b` should be simplified away.  `a` is not a constant within the
+     * loop, and `c`'s value changes depending on whether we run 0 or 1
+     * iterations of the loop. */
+    a = s32[1] constant({0})
+    b = s32[2] constant({1,1})
+    c = s32[3] constant({2,2,2})
+    init = (s32[1], s32[2], s32[3]) tuple(a,b,c)
+    ROOT while = (s32[1], s32[2], s32[3]) while(init),
+      condition=Cond, body=Body
+  })";
+
+  auto m = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
+  EXPECT_TRUE(WhileLoopSimplifier().Run(m.get()).ValueOrDie());
+  // DCE away the old loop so there's just one while loop in the module, making
+  // it easy to find.
+  EXPECT_TRUE(HloDCE().Run(m.get()).ok());
+  // Run the tuple simplifier to make the resulting HLO a bit easier to check.
+  EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok());
+
+  const auto& instrs = m->entry_computation()->instructions();
+  HloInstruction* new_while =
+      *absl::c_find_if(instrs, [](const HloInstruction* instr) {
+        return instr->opcode() == HloOpcode::kWhile;
+      });
+
+  Shape new_while_shape =
+      ShapeUtil::ParseShapeString("(s32[1], s32[3])").ValueOrDie();
+  SCOPED_TRACE(m->ToString());
+  EXPECT_TRUE(ShapeUtil::Equal(new_while->shape(), new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      new_while->while_body()->root_instruction()->shape(), new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      new_while->while_body()->parameter_instruction(0)->shape(),
+      new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      new_while->while_condition()->parameter_instruction(0)->shape(),
+      new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      m->entry_computation()->root_instruction()->shape(),
+      ShapeUtil::ParseShapeString("(s32[1], s32[2], s32[3])").ValueOrDie()));
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              op::Tuple(_, op::Constant(), _));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 3b45c882ff23a1e5bd157fbb8a37dd665d8892c8 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Nov 2018 23:08:43 -0800
Subject: [PATCH 0385/1554] Deprecate logging endpoints.

PiperOrigin-RevId: 221748184
---
 tensorflow/python/platform/tf_logging.py      | 40 ++++-----
 .../tools/api/generator/api_init_files.bzl    |  1 -
 .../api/golden/v2/tensorflow.logging.pbtxt    | 83 -------------------
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 -
 tensorflow/tools/compatibility/renames_v2.py  | 21 +++++
 5 files changed, 41 insertions(+), 108 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt

diff --git a/tensorflow/python/platform/tf_logging.py b/tensorflow/python/platform/tf_logging.py
index 59e60856ae..9f00abb201 100644
--- a/tensorflow/python/platform/tf_logging.py
+++ b/tensorflow/python/platform/tf_logging.py
@@ -130,37 +130,37 @@ def _get_logger():
     _logger_lock.release()
 
 
-@tf_export('logging.log')
+@tf_export(v1=['logging.log'])
 def log(level, msg, *args, **kwargs):
   _get_logger().log(level, msg, *args, **kwargs)
 
 
-@tf_export('logging.debug')
+@tf_export(v1=['logging.debug'])
 def debug(msg, *args, **kwargs):
   _get_logger().debug(msg, *args, **kwargs)
 
 
-@tf_export('logging.error')
+@tf_export(v1=['logging.error'])
 def error(msg, *args, **kwargs):
   _get_logger().error(msg, *args, **kwargs)
 
 
-@tf_export('logging.fatal')
+@tf_export(v1=['logging.fatal'])
 def fatal(msg, *args, **kwargs):
   _get_logger().fatal(msg, *args, **kwargs)
 
 
-@tf_export('logging.info')
+@tf_export(v1=['logging.info'])
 def info(msg, *args, **kwargs):
   _get_logger().info(msg, *args, **kwargs)
 
 
-@tf_export('logging.warn')
+@tf_export(v1=['logging.warn'])
 def warn(msg, *args, **kwargs):
   _get_logger().warn(msg, *args, **kwargs)
 
 
-@tf_export('logging.warning')
+@tf_export(v1=['logging.warning'])
 def warning(msg, *args, **kwargs):
   _get_logger().warning(msg, *args, **kwargs)
 
@@ -183,18 +183,18 @@ _log_prefix = None  # later set to google2_log_prefix
 _log_counter_per_token = {}
 
 
-@tf_export('logging.TaskLevelStatusMessage')
+@tf_export(v1=['logging.TaskLevelStatusMessage'])
 def TaskLevelStatusMessage(msg):
   error(msg)
 
 
-@tf_export('logging.flush')
+@tf_export(v1=['logging.flush'])
 def flush():
   raise NotImplementedError()
 
 
 # Code below is taken from pyglib/logging
-@tf_export('logging.vlog')
+@tf_export(v1=['logging.vlog'])
 def vlog(level, msg, *args, **kwargs):
   _get_logger().log(level, msg, *args, **kwargs)
 
@@ -214,7 +214,7 @@ def _GetNextLogCountPerToken(token):
   return _log_counter_per_token[token]
 
 
-@tf_export('logging.log_every_n')
+@tf_export(v1=['logging.log_every_n'])
 def log_every_n(level, msg, n, *args):
   """Log 'msg % args' at level 'level' once per 'n' times.
 
@@ -231,7 +231,7 @@ def log_every_n(level, msg, n, *args):
   log_if(level, msg, not (count % n), *args)
 
 
-@tf_export('logging.log_first_n')
+@tf_export(v1=['logging.log_first_n'])
 def log_first_n(level, msg, n, *args):  # pylint: disable=g-bad-name
   """Log 'msg % args' at level 'level' only first 'n' times.
 
@@ -247,7 +247,7 @@ def log_first_n(level, msg, n, *args):  # pylint: disable=g-bad-name
   log_if(level, msg, count < n, *args)
 
 
-@tf_export('logging.log_if')
+@tf_export(v1=['logging.log_if'])
 def log_if(level, msg, condition, *args):
   """Log 'msg % args' at level 'level' only if condition is fulfilled."""
   if condition:
@@ -296,13 +296,13 @@ def google2_log_prefix(level, timestamp=None, file_and_line=None):
   return s
 
 
-@tf_export('logging.get_verbosity')
+@tf_export(v1=['logging.get_verbosity'])
 def get_verbosity():
   """Return how much logging output will be produced."""
   return _get_logger().getEffectiveLevel()
 
 
-@tf_export('logging.set_verbosity')
+@tf_export(v1=['logging.set_verbosity'])
 def set_verbosity(v):
   """Sets the threshold for what messages will be logged."""
   _get_logger().setLevel(v)
@@ -318,8 +318,8 @@ def _get_thread_id():
 
 _log_prefix = google2_log_prefix
 
-tf_export('logging.DEBUG').export_constant(__name__, 'DEBUG')
-tf_export('logging.ERROR').export_constant(__name__, 'ERROR')
-tf_export('logging.FATAL').export_constant(__name__, 'FATAL')
-tf_export('logging.INFO').export_constant(__name__, 'INFO')
-tf_export('logging.WARN').export_constant(__name__, 'WARN')
+tf_export(v1=['logging.DEBUG']).export_constant(__name__, 'DEBUG')
+tf_export(v1=['logging.ERROR']).export_constant(__name__, 'ERROR')
+tf_export(v1=['logging.FATAL']).export_constant(__name__, 'FATAL')
+tf_export(v1=['logging.INFO']).export_constant(__name__, 'INFO')
+tf_export(v1=['logging.WARN']).export_constant(__name__, 'WARN')
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index a9e8ed4578..1097455734 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -62,7 +62,6 @@ TENSORFLOW_API_INIT_FILES = [
     "linalg/__init__.py",
     "lite/__init__.py",
     "lite/constants/__init__.py",
-    "logging/__init__.py",
     "losses/__init__.py",
     "math/__init__.py",
     "metrics/__init__.py",
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt
deleted file mode 100644
index 85bb15455d..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.logging.pbtxt
+++ /dev/null
@@ -1,83 +0,0 @@
-path: "tensorflow.logging"
-tf_module {
-  member {
-    name: "DEBUG"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "ERROR"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "FATAL"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "INFO"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "WARN"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "TaskLevelStatusMessage"
-    argspec: "args=[\'msg\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "debug"
-    argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "error"
-    argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "fatal"
-    argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "flush"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_verbosity"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "info"
-    argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "log"
-    argspec: "args=[\'level\', \'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "log_every_n"
-    argspec: "args=[\'level\', \'msg\', \'n\'], varargs=args, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "log_first_n"
-    argspec: "args=[\'level\', \'msg\', \'n\'], varargs=args, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "log_if"
-    argspec: "args=[\'level\', \'msg\', \'condition\'], varargs=args, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_verbosity"
-    argspec: "args=[\'v\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "vlog"
-    argspec: "args=[\'level\', \'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "warn"
-    argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "warning"
-    argspec: "args=[\'msg\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index d37f83efdc..08d3cb5072 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -296,10 +296,6 @@ tf_module {
     name: "lite"
     mtype: "<type \'module\'>"
   }
-  member {
-    name: "logging"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "losses"
     mtype: "<type \'module\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 46d1bf445c..2cbb82a9ab 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -239,6 +239,26 @@ renames = {
     'tf.local_variables': 'tf.compat.v1.local_variables',
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log_sigmoid': 'tf.math.log_sigmoid',
+    'tf.logging.DEBUG': 'tf.compat.v1.logging.DEBUG',
+    'tf.logging.ERROR': 'tf.compat.v1.logging.ERROR',
+    'tf.logging.FATAL': 'tf.compat.v1.logging.FATAL',
+    'tf.logging.INFO': 'tf.compat.v1.logging.INFO',
+    'tf.logging.TaskLevelStatusMessage': 'tf.compat.v1.logging.TaskLevelStatusMessage',
+    'tf.logging.WARN': 'tf.compat.v1.logging.WARN',
+    'tf.logging.debug': 'tf.compat.v1.logging.debug',
+    'tf.logging.error': 'tf.compat.v1.logging.error',
+    'tf.logging.fatal': 'tf.compat.v1.logging.fatal',
+    'tf.logging.flush': 'tf.compat.v1.logging.flush',
+    'tf.logging.get_verbosity': 'tf.compat.v1.logging.get_verbosity',
+    'tf.logging.info': 'tf.compat.v1.logging.info',
+    'tf.logging.log': 'tf.compat.v1.logging.log',
+    'tf.logging.log_every_n': 'tf.compat.v1.logging.log_every_n',
+    'tf.logging.log_first_n': 'tf.compat.v1.logging.log_first_n',
+    'tf.logging.log_if': 'tf.compat.v1.logging.log_if',
+    'tf.logging.set_verbosity': 'tf.compat.v1.logging.set_verbosity',
+    'tf.logging.vlog': 'tf.compat.v1.logging.vlog',
+    'tf.logging.warn': 'tf.compat.v1.logging.warn',
+    'tf.logging.warning': 'tf.compat.v1.logging.warning',
     'tf.logical_xor': 'tf.math.logical_xor',
     'tf.make_template': 'tf.compat.v1.make_template',
     'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto',
@@ -300,6 +320,7 @@ renames = {
     'tf.python_io.tf_record_iterator': 'tf.io.tf_record_iterator',
     'tf.qr': 'tf.linalg.qr',
     'tf.quantize': 'tf.quantization.quantize',
+    'tf.quantize_v2': 'tf.compat.v1.quantize_v2',
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
     'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
     'tf.random_crop': 'tf.image.random_crop',
-- 
GitLab


From 5e2be9addaa7bd384cbe7d55ad0cb0cca2dc5ed6 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 15 Nov 2018 23:15:41 -0800
Subject: [PATCH 0386/1554] Adding `false_positive` metric and confusion matrix
 calculation logic in the new metrics module.

PiperOrigin-RevId: 221748567
---
 tensorflow/python/keras/metrics.py      | 202 ++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py |  64 ++++++++
 2 files changed, 266 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 2ea6405597..5f63192475 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -24,6 +24,7 @@ import functools
 import sys
 import types
 import weakref
+from enum import Enum
 import six
 
 from tensorflow.python.compat import compat
@@ -49,6 +50,7 @@ from tensorflow.python.keras.losses import squared_hinge
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
@@ -267,6 +269,153 @@ def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
   return y_pred, y_true, sample_weight
 
 
+class _ConfusionMatrix(Enum):
+  TRUE_POSITIVES = 'tp'
+  FALSE_POSITIVES = 'fp'
+  TRUE_NEGATIVES = 'tn'
+  FALSE_NEGATIVES = 'fn'
+
+
+def _update_confusion_matrix_variables(variables_to_update,
+                                       y_true,
+                                       y_pred,
+                                       thresholds,
+                                       sample_weight=None):
+  """Returns op to update the given confusion matrix variables.
+
+  For every pair of values in y_true and y_pred:
+
+  true_positive: y_true == True and y_pred > thresholds
+  false_negatives: y_true == True and y_pred <= thresholds
+  true_negatives: y_true == False and y_pred <= thresholds
+  false_positive: y_true == False and y_pred > thresholds
+
+  The results will be weighted and added together. When multiple thresholds are
+  provided, we will repeat the same for every threshold.
+
+  For estimation of these metrics over a stream of data, the function creates an
+  `update_op` operation that updates the given variables.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use weights of 0 to mask values.
+
+  Args:
+    variables_to_update: Dictionary with 'tp', 'fn', 'tn', 'fp' as valid keys
+      and corresponding variables to update as values.
+    y_true: A `Tensor` whose shape matches `y_pred`. Will be cast to `bool`.
+    y_pred: A floating point `Tensor` of arbitrary shape and whose values are in
+      the range `[0, 1]`.
+    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
+    sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
+      `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `y_true` dimension).
+
+  Returns:
+    Update op.
+
+  Raises:
+    ValueError: If `y_pred` and `y_true` have mismatched shapes, or if
+      `sample_weight` is not `None` and its shape doesn't match `y_pred`, or if
+      `variables_to_update` contains invalid keys.
+  """
+  if variables_to_update is None:
+    return
+  y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
+
+  if not any(
+      key for key in variables_to_update if key in list(_ConfusionMatrix)):
+    raise ValueError(
+        'Please provide at least one valid confusion matrix '
+        'variable to update. Valid variable key options are: "{}". '
+        'Received: "{}"'.format(
+            list(_ConfusionMatrix), variables_to_update.keys()))
+
+  invalid_keys = [
+      key for key in variables_to_update if key not in list(_ConfusionMatrix)
+  ]
+  if invalid_keys:
+    raise ValueError(
+        'Invalid keys: {}. Valid variable key options are: "{}"'.format(
+            invalid_keys, list(_ConfusionMatrix)))
+
+  with ops.control_dependencies([
+      check_ops.assert_greater_equal(
+          y_pred,
+          math_ops.cast(0.0, dtype=y_pred.dtype),
+          message='predictions must be >= 0'),
+      check_ops.assert_less_equal(
+          y_pred,
+          math_ops.cast(1.0, dtype=y_pred.dtype),
+          message='predictions must be <= 1')
+  ]):
+    y_pred, y_true, sample_weight = squeeze_or_expand_dimensions(
+        math_ops.cast(y_pred, dtype=dtypes.float32),
+        math_ops.cast(y_true, dtype=dtypes.bool), sample_weight)
+
+  num_thresholds = len(thresholds)
+  num_predictions = array_ops.size(y_pred)
+
+  # Reshape predictions and labels.
+  predictions_2d = array_ops.reshape(y_pred, [1, -1])
+  labels_2d = array_ops.reshape(
+      math_ops.cast(y_true, dtype=dtypes.bool), [1, -1])
+
+  # Tile the thresholds for every prediction.
+  thresh_tiled = array_ops.tile(
+      array_ops.expand_dims(array_ops.constant(thresholds), 1),
+      array_ops.stack([1, num_predictions]))
+
+  # Tile the predictions for every threshold.
+  preds_tiled = array_ops.tile(predictions_2d, [num_thresholds, 1])
+
+  # Compare predictions and threshold.
+  pred_is_pos = math_ops.greater(preds_tiled, thresh_tiled)
+
+  # Tile labels by number of thresholds
+  label_is_pos = array_ops.tile(labels_2d, [num_thresholds, 1])
+
+  if sample_weight is not None:
+    weights = weights_broadcast_ops.broadcast_weights(
+        math_ops.cast(sample_weight, dtype=dtypes.float32), y_pred)
+    weights_tiled = array_ops.tile(
+        array_ops.reshape(weights, [1, -1]), [num_thresholds, 1])
+  else:
+    weights_tiled = None
+
+  update_ops = []
+
+  def weighted_assign_add(label, pred, weights, var):
+    label_and_pred = math_ops.cast(
+        math_ops.logical_and(label, pred), dtype=dtypes.float32)
+    if weights is not None:
+      label_and_pred *= weights
+    return state_ops.assign_add(var, math_ops.reduce_sum(label_and_pred, 1))
+
+  loop_vars = {
+      _ConfusionMatrix.TRUE_POSITIVES: (label_is_pos, pred_is_pos),
+  }
+  update_tn = _ConfusionMatrix.TRUE_NEGATIVES in variables_to_update
+  update_fp = _ConfusionMatrix.FALSE_POSITIVES in variables_to_update
+  update_fn = _ConfusionMatrix.FALSE_NEGATIVES in variables_to_update
+
+  if update_fn or update_tn:
+    pred_is_neg = math_ops.logical_not(pred_is_pos)
+    loop_vars[_ConfusionMatrix.FALSE_NEGATIVES] = (label_is_pos, pred_is_neg)
+
+  if update_fp or update_tn:
+    label_is_neg = math_ops.logical_not(label_is_pos)
+    loop_vars[_ConfusionMatrix.FALSE_POSITIVES] = (label_is_neg, pred_is_pos)
+    if update_tn:
+      loop_vars[_ConfusionMatrix.TRUE_NEGATIVES] = (label_is_neg, pred_is_neg)
+
+  for matrix_cond, (label, pred) in loop_vars.items():
+    if matrix_cond in variables_to_update:
+      update_ops.append(
+          weighted_assign_add(label, pred, weights_tiled,
+                              variables_to_update[matrix_cond]))
+  return control_flow_ops.group(update_ops)
+
+
 @six.add_metaclass(abc.ABCMeta)
 class Metric(Layer):
   """Encapsulates metric logic and state.
@@ -644,6 +793,59 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
         sparse_categorical_accuracy, name, dtype=dtype)
 
 
+class FalsePositives(Metric):
+  """Calculates the number of false positives.
+
+  If `sample_weight` is given, calculates the sum of the weights of
+  false positives. This metric creates one local variable, `false_positives`
+  that is used to keep track of the number of false positives.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `FalsePositives` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(FalsePositives, self).__init__(name=name, dtype=dtype)
+    self.thresholds = [0.5] if thresholds is None else thresholds
+    self.fp = self.add_weight(
+        'false_positives',
+        shape=(len(self.thresholds),),
+        initializer=init_ops.zeros_initializer)
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates false positive statistics.
+
+    `y_true` and `y_pred` should have the same shape.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    return _update_confusion_matrix_variables({
+        _ConfusionMatrix.FALSE_POSITIVES: self.fp
+    }, y_true, y_pred, self.thresholds, sample_weight)
+
+  def result(self):
+    return ops.convert_to_tensor(self.fp)
+
+
 @tf_export('keras.metrics.binary_accuracy')
 def binary_accuracy(y_true, y_pred, threshold=0.5):
   threshold = math_ops.cast(threshold, y_pred.dtype)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 5f5565d4d5..f506b3041b 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -22,6 +22,7 @@ import os
 import numpy as np
 
 from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -478,5 +479,68 @@ class KerasMetricsTest(test.TestCase):
       invalid_update_obj.update_state()
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class FalsePositivesTest(test.TestCase):
+
+  def test_config(self):
+    fp_obj = metrics.FalsePositives(name='my_fp', thresholds=[0.4, 0.9])
+    self.assertEqual(fp_obj.name, 'my_fp')
+    self.assertEqual(len(fp_obj.variables), 1)
+    self.assertEqual(fp_obj.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    fp_obj = metrics.FalsePositives()
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = fp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fp_obj.result()
+    self.assertAllClose([7.], result)
+
+  def test_weighted(self):
+    fp_obj = metrics.FalsePositives()
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([14.], self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = fp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fp_obj.result()
+    self.assertAllClose([7., 4., 2.], result)
+
+  def test_weighted_with_thresholds(self):
+    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),
+                     (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))
+
+    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([125., 42., 12.], self.evaluate(result))
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 62389b3a5a656164e00e3affeea6f5a342965ce2 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Nov 2018 23:34:51 -0800
Subject: [PATCH 0387/1554] Updating tf.nn.conv1d, tf.nn.conv2d,
 tf.nn.conv2d_backprop_filter, tf.nn.conv2d_packprop_input,
 tf.nn.conv2d_transpose, tf.nn.conv3d, tf.nn.conv3d_backprop_filter_v2,
 tf.nn.conv3d_transpose, tf.nn.convolution, tf.nn.crelu for TensorFlow 2.0
 API.

PiperOrigin-RevId: 221749850
---
 .../api_def/python_api/api_def_Conv2D.pbtxt   |   4 +-
 .../api_def_Conv2DBackpropFilter.pbtxt        |   4 +-
 .../api_def_Conv2DBackpropInput.pbtxt         |   4 +-
 .../api_def/python_api/api_def_Conv3D.pbtxt   |   4 +-
 .../api_def_Conv3DBackpropFilterV2.pbtxt      |   4 +
 tensorflow/python/ops/nn_ops.py               | 371 +++++++++++++++++-
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |   4 +
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  20 +-
 tensorflow/tools/compatibility/renames_v2.py  |   1 +
 .../tools/compatibility/tf_upgrade_v2.py      |  26 ++
 .../tools/compatibility/tf_upgrade_v2_test.py |  13 +
 11 files changed, 428 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_Conv2D.pbtxt b/tensorflow/core/api_def/python_api/api_def_Conv2D.pbtxt
index 2ae75d6da2..1f4bc6d22e 100644
--- a/tensorflow/core/api_def/python_api/api_def_Conv2D.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Conv2D.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "Conv2D"
-  endpoint {
-    name: "nn.conv2d"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropFilter.pbtxt b/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropFilter.pbtxt
index 6f21d8c880..1a9d96f3ab 100644
--- a/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropFilter.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropFilter.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "Conv2DBackpropFilter"
-  endpoint {
-    name: "nn.conv2d_backprop_filter"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropInput.pbtxt b/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropInput.pbtxt
index ea976799cb..1505a30765 100644
--- a/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropInput.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Conv2DBackpropInput.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "Conv2DBackpropInput"
-  endpoint {
-    name: "nn.conv2d_backprop_input"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Conv3D.pbtxt b/tensorflow/core/api_def/python_api/api_def_Conv3D.pbtxt
index ba8d178263..cb463dd0d8 100644
--- a/tensorflow/core/api_def/python_api/api_def_Conv3D.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Conv3D.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "Conv3D"
-  endpoint {
-    name: "nn.conv3d"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Conv3DBackpropFilterV2.pbtxt b/tensorflow/core/api_def/python_api/api_def_Conv3DBackpropFilterV2.pbtxt
index 1da8ee3a25..590b37c95f 100644
--- a/tensorflow/core/api_def/python_api/api_def_Conv3DBackpropFilterV2.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Conv3DBackpropFilterV2.pbtxt
@@ -1,6 +1,10 @@
 op {
   graph_op_name: "Conv3DBackpropFilterV2"
+  endpoint {
+    name: "nn.conv3d_backprop_filter"
+  }
   endpoint {
     name: "nn.conv3d_backprop_filter_v2"
+    deprecation_version: 2
   }
 }
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 9103253874..c733d90ab5 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -645,7 +645,7 @@ def _get_strides_and_dilation_rate(num_spatial_dims, strides, dilation_rate):
   return strides, dilation_rate
 
 
-@tf_export("nn.convolution")
+@tf_export(v1=["nn.convolution"])
 def convolution(
     input,  # pylint: disable=redefined-builtin
     filter,  # pylint: disable=redefined-builtin
@@ -783,6 +783,30 @@ def convolution(
     return op(input, filter)
 
 
+@tf_export("nn.convolution", v1=[])
+def convolution_v2(
+    input,  # pylint: disable=redefined-builtin
+    filters,
+    strides=None,
+    padding="VALID",
+    data_format=None,
+    dilations=None,
+    name=None):
+  return convolution(
+      input,  # pylint: disable=redefined-builtin
+      filters,
+      padding=padding,
+      strides=strides,
+      dilation_rate=dilations,
+      name=name,
+      data_format=data_format)
+
+convolution_v2.__doc__ = deprecation.rewrite_argument_docstring(
+    deprecation.rewrite_argument_docstring(
+        convolution.__doc__, "dilation_rate", "dilations"),
+    "filter", "filters")
+
+
 class Convolution(object):
   """Helper class for convolution.
 
@@ -1281,7 +1305,208 @@ def atrous_conv2d(value, filters, rate, padding, name=None):
       name=name)
 
 
-@tf_export("nn.conv2d_transpose")
+@tf_export("nn.conv2d", v1=[])
+def conv2d_v2(input,  # pylint: disable=redefined-builtin
+              filters,
+              strides,
+              padding,
+              data_format="NHWC",
+              dilations=None,
+              name=None):
+  # pylint: disable=line-too-long
+  r"""Computes a 2-D convolution given 4-D `input` and `filters` tensors.
+
+  Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+  and a filter / kernel tensor of shape
+  `[filter_height, filter_width, in_channels, out_channels]`, this op
+  performs the following:
+
+  1. Flattens the filter to a 2-D matrix with shape
+     `[filter_height * filter_width * in_channels, output_channels]`.
+  2. Extracts image patches from the input tensor to form a *virtual*
+     tensor of shape `[batch, out_height, out_width,
+     filter_height * filter_width * in_channels]`.
+  3. For each patch, right-multiplies the filter matrix and the image patch
+     vector.
+
+  In detail, with the default NHWC format,
+
+      output[b, i, j, k] =
+          sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q] *
+                          filter[di, dj, q, k]
+
+  Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+  horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      A 4-D tensor. The dimension order is interpreted according to the value
+      of `data_format`, see below for details.
+    filters: A `Tensor`. Must have the same type as `input`.
+      A 4-D tensor of shape
+      `[filter_height, filter_width, in_channels, out_channels]`
+    strides: A list of `ints`.
+      1-D tensor of length 4.  The stride of the sliding window for each
+      dimension of `input`. The dimension order is determined by the value of
+      `data_format`, see below for details.
+    padding: A `string` from: `"SAME", "VALID"`.
+      The type of padding algorithm to use.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, height, width, channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, channels, height, width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by the
+      value of `data_format`, see above for details. Dilations in the batch and
+      depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  # pylint: enable=line-too-long
+  if dilations is None:
+    dilations = [1, 1, 1, 1]
+  return gen_nn_ops.conv2d(input,  # pylint: disable=redefined-builtin
+                           filters,
+                           strides,
+                           padding,
+                           use_cudnn_on_gpu=True,
+                           data_format=data_format,
+                           dilations=dilations,
+                           name=name)
+tf_export(v1=["nn.conv2d"])(gen_nn_ops.conv2d)
+
+
+@tf_export("nn.conv2d_backprop_filter", v1=[])
+def conv2d_backprop_filter_v2(input,  # pylint: disable=redefined-builtin
+                              filter_sizes,
+                              out_backprop,
+                              strides,
+                              padding,
+                              data_format="NHWC",
+                              dilations=None,
+                              name=None):
+  r"""Computes the gradients of convolution with respect to the filter.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      4-D with shape `[batch, in_height, in_width, in_channels]`.
+    filter_sizes: A `Tensor` of type `int32`.
+      An integer vector representing the tensor shape of `filter`,
+      where `filter` is a 4-D
+      `[filter_height, filter_width, in_channels, out_channels]` tensor.
+    out_backprop: A `Tensor`. Must have the same type as `input`.
+      4-D with shape `[batch, out_height, out_width, out_channels]`.
+      Gradients w.r.t. the output of the convolution.
+    strides: A list of `ints`.
+      The stride of the sliding window for each dimension of the input
+      of the convolution. Must be in the same order as the dimension specified
+      with format.
+    padding: A `string` from: `"SAME", "VALID"`.
+      The type of padding algorithm to use.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, in_height, in_width, in_channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, in_channels, in_height, in_width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by
+      the value of `data_format`, see above for details. Dilations in the batch
+      and depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  if dilations is None:
+    dilations = [1, 1, 1, 1]
+  return gen_nn_ops.conv2d_backprop_filter(input,  # pylint: disable=redefined-builtin
+                                           filter_sizes,
+                                           out_backprop,
+                                           strides,
+                                           padding,
+                                           use_cudnn_on_gpu=True,
+                                           data_format=data_format,
+                                           dilations=dilations,
+                                           name=name)
+tf_export(v1=["nn.conv2d_backprop_filter"])(
+    gen_nn_ops.conv2d_backprop_filter)
+
+
+@tf_export("nn.conv2d_backprop_input", v1=[])
+def conv2d_backprop_input_v2(input_sizes,
+                             filters,
+                             out_backprop,
+                             strides,
+                             padding,
+                             data_format="NHWC",
+                             dilations=None,
+                             name=None):
+  r"""Computes the gradients of convolution with respect to the input.
+
+  Args:
+    input_sizes: A `Tensor` of type `int32`.
+      An integer vector representing the shape of `input`,
+      where `input` is a 4-D `[batch, height, width, channels]` tensor.
+    filters: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      4-D with shape
+      `[filter_height, filter_width, in_channels, out_channels]`.
+    out_backprop: A `Tensor`. Must have the same type as `filters`.
+      4-D with shape `[batch, out_height, out_width, out_channels]`.
+      Gradients w.r.t. the output of the convolution.
+    strides: A list of `ints`.
+      The stride of the sliding window for each dimension of the input
+      of the convolution. Must be in the same order as the dimension specified
+      with format.
+    padding: A `string` from: `"SAME", "VALID"`.
+      The type of padding algorithm to use.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, in_height, in_width, in_channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, in_channels, in_height, in_width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by
+      the value of `data_format`, see above for details. Dilations in the batch
+      and depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `filters`.
+  """
+  if dilations is None:
+    dilations = [1, 1, 1, 1]
+  return gen_nn_ops.conv2d_backprop_input(input_sizes,
+                                          filters,
+                                          out_backprop,
+                                          strides,
+                                          padding,
+                                          use_cudnn_on_gpu=True,
+                                          data_format=data_format,
+                                          dilations=dilations,
+                                          name=name)
+tf_export(v1=["nn.conv2d_backprop_input"])(
+    gen_nn_ops.conv2d_backprop_input)
+
+
+@tf_export(v1=["nn.conv2d_transpose"])
 def conv2d_transpose(
     value,
     filter,  # pylint: disable=redefined-builtin
@@ -1361,6 +1586,31 @@ def conv2d_transpose(
         name=name)
 
 
+# pylint: disable=redefined-builtin
+@tf_export("nn.conv2d_transpose", v1=[])
+def conv2d_transpose_v2(
+    input,
+    filters,  # pylint: disable=redefined-builtin
+    output_shape,
+    strides,
+    padding="SAME",
+    data_format="NHWC",
+    name=None):
+  return conv2d_transpose(
+      input,
+      filters,
+      output_shape,
+      strides,
+      padding=padding,
+      data_format=data_format,
+      name=name)
+# pylint: enable=redefined-builtin
+conv2d_transpose_v2.__doc__ = deprecation.rewrite_argument_docstring(
+    deprecation.rewrite_argument_docstring(
+        conv2d_transpose.__doc__, "filter", "filters"),
+    "value", "input")
+
+
 @tf_export("nn.atrous_conv2d_transpose")
 def atrous_conv2d_transpose(value,
                             filters,
@@ -1509,7 +1759,29 @@ def atrous_conv2d_transpose(value,
         input=value, crops=batch_to_space_crop, block_size=rate)
 
 
-@tf_export("nn.conv3d_transpose")
+@tf_export("nn.conv3d", v1=[])
+def conv3d_v2(input,  # pylint: disable=redefined-builtin,missing-docstring
+              filters,
+              strides,
+              padding,
+              data_format="NDHWC",
+              dilations=None,
+              name=None):
+  if dilations is None:
+    dilations = [1, 1, 1, 1, 1]
+  return gen_nn_ops.conv3d(input,  # pylint: disable=redefined-builtin
+                           filters,
+                           strides,
+                           padding,
+                           data_format=data_format,
+                           dilations=dilations,
+                           name=name)
+tf_export(v1=["nn.conv3d"])(gen_nn_ops.conv3d)
+conv3d_v2.__doc__ = deprecation.rewrite_argument_docstring(
+    gen_nn_ops.conv3d.__doc__, "filter", "filters")
+
+
+@tf_export(v1=["nn.conv3d_transpose"])
 def conv3d_transpose(
     value,
     filter,  # pylint: disable=redefined-builtin
@@ -1587,6 +1859,31 @@ def conv3d_transpose(
         name=name)
 
 
+# pylint: disable=redefined-builtin
+@tf_export("nn.conv3d_transpose", v1=[])
+def conv3d_transpose_v2(
+    input,
+    filters,
+    output_shape,
+    strides,
+    padding="SAME",
+    data_format="NDHWC",
+    name=None):
+  return conv3d_transpose(
+      input,
+      filters,
+      output_shape,
+      strides,
+      padding=padding,
+      data_format=data_format,
+      name=name)
+# pylint: enable=redefined-builtin
+conv3d_transpose_v2.__doc__ = deprecation.rewrite_argument_docstring(
+    deprecation.rewrite_argument_docstring(
+        conv3d_transpose.__doc__, "filter", "filters"),
+    "value", "input")
+
+
 @tf_export("nn.bias_add")
 def bias_add(value, bias, data_format=None, name=None):
   """Adds `bias` to `value`.
@@ -1642,7 +1939,7 @@ def bias_add_v1(value, bias, name=None):
     return gen_nn_ops.bias_add_v1(value, bias, name=name)
 
 
-@tf_export("nn.crelu")
+@tf_export(v1=["nn.crelu"])
 def crelu(features, name=None, axis=-1):
   """Computes Concatenated ReLU.
 
@@ -1668,6 +1965,12 @@ def crelu(features, name=None, axis=-1):
     return gen_nn_ops.relu(c)
 
 
+@tf_export("nn.crelu", v1=[])
+def crelu_v2(features, axis=-1, name=None):
+  return crelu(features, name=name, axis=axis)
+crelu_v2.__doc__ = crelu.__doc__
+
+
 @tf_export("nn.relu6")
 def relu6(features, name=None):
   """Computes Rectified Linear 6: `min(max(features, 0), 6)`.
@@ -2788,7 +3091,7 @@ def fractional_avg_pool_v2(value,
                                           seed=seed1, seed2=seed2, name=name)
 
 
-@tf_export("nn.conv1d")
+@tf_export(v1=["nn.conv1d"])
 @deprecation.deprecated_arg_values(
     None,
     "`NCHW` for data_format is deprecated, use `NCW` instead",
@@ -2873,6 +3176,64 @@ def conv1d(value,
     return array_ops.squeeze(result, [spatial_start_dim])
 
 
+@tf_export("nn.conv1d", v1=[])
+def conv1d_v2(input,  # pylint: disable=redefined-builtin
+              filters,
+              stride,
+              padding,
+              data_format=None,
+              name=None):
+  r"""Computes a 1-D convolution given 3-D input and filter tensors.
+
+  Given an input tensor of shape
+    [batch, in_width, in_channels]
+  if data_format is "NWC", or
+    [batch, in_channels, in_width]
+  if data_format is "NCW",
+  and a filter / kernel tensor of shape
+  [filter_width, in_channels, out_channels], this op reshapes
+  the arguments to pass them to conv2d to perform the equivalent
+  convolution operation.
+
+  Internally, this op reshapes the input tensors and invokes `tf.nn.conv2d`.
+  For example, if `data_format` does not start with "NC", a tensor of shape
+    [batch, in_width, in_channels]
+  is reshaped to
+    [batch, 1, in_width, in_channels],
+  and the filter is reshaped to
+    [1, filter_width, in_channels, out_channels].
+  The result is then reshaped back to
+    [batch, out_width, out_channels]
+  \(where out_width is a function of the stride and padding as in conv2d\) and
+  returned to the caller.
+
+  Args:
+    input: A 3D `Tensor`.  Must be of type `float16`, `float32`, or `float64`.
+    filters: A 3D `Tensor`.  Must have the same type as `input`.
+    stride: An `integer`.  The number of entries by which
+      the filter is moved right at each step.
+    padding: 'SAME' or 'VALID'
+    data_format: An optional `string` from `"NWC", "NCW"`.  Defaults
+      to `"NWC"`, the data is stored in the order of
+      [batch, in_width, in_channels].  The `"NCW"` format stores
+      data as [batch, in_channels, in_width].
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`.  Has the same type as input.
+
+  Raises:
+    ValueError: if `data_format` is invalid.
+  """
+  return conv1d(input,  # pylint: disable=redefined-builtin
+                filters,
+                stride,
+                padding,
+                use_cudnn_on_gpu=True,
+                data_format=data_format,
+                name=name)
+
+
 def conv1d_transpose(
     value,
     filter,  # pylint: disable=redefined-builtin
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 93f2fda2ac..cb28ea8424 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -72,6 +72,10 @@ tf_module {
     name: "conv3d"
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], "
   }
+  member_method {
+    name: "conv3d_backprop_filter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], "
+  }
   member_method {
     name: "conv3d_backprop_filter_v2"
     argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 481365be24..36aaf7565c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -50,43 +50,43 @@ tf_module {
   }
   member_method {
     name: "conv1d"
-    argspec: "args=[\'value\', \'filters\', \'stride\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'stride\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "conv2d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d_backprop_filter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d_backprop_input"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input_sizes\', \'filters\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv2d_transpose"
-    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NHWC\', \'None\'], "
   }
   member_method {
     name: "conv3d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\', \'None\'], "
   }
   member_method {
-    name: "conv3d_backprop_filter_v2"
+    name: "conv3d_backprop_filter"
     argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], "
   }
   member_method {
     name: "conv3d_transpose"
-    argspec: "args=[\'value\', \'filter\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'output_shape\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'SAME\', \'NDHWC\', \'None\'], "
   }
   member_method {
     name: "convolution"
-    argspec: "args=[\'input\', \'filter\', \'padding\', \'strides\', \'dilation_rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'VALID\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "crelu"
-    argspec: "args=[\'features\', \'name\', \'axis\'], varargs=None, keywords=None, defaults=[\'None\', \'-1\'], "
+    argspec: "args=[\'features\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], "
   }
   member_method {
     name: "ctc_beam_search_decoder"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 2cbb82a9ab..5f9d33228b 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -283,6 +283,7 @@ renames = {
     'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
+    'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 41948441ec..6f46c13710 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -50,6 +50,17 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         },
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
+        },
+        "tf.nn.conv3d": {
+            "filter": "filters"
+        },
+        "tf.nn.conv3d_transpose": {
+            "value": "input",
+            "filter": "filters",
+        },
+        "tf.nn.convolution": {
+            "filter": "filters",
+            "dilation_rate": "dilations",
         }
     }
 
@@ -124,6 +135,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.argmin": ["input", "axis", "name", "dimension", "output_type"],
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
         "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
+        "tf.nn.convolution": [
+            "input", "filter", "padding", "strides", "dilation_rate", "name",
+            "data_format"],
+        "tf.nn.crelu": ["features", "name", "axis"],
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
@@ -198,6 +213,17 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             default_loss_reduction_changed,
         "tf.estimator.BaselineRegressor":
             default_loss_reduction_changed,
+        "tf.nn.conv1d":
+        "WARNING: use_cudnn_on_gpu argument has been removed and \"value\" was "
+        "renamed to \"input\"",
+        "tf.nn.conv2d":
+        "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" "
+        "was renamed to \"filters\"",
+        "tf.nn.conv2d_backprop_filter":
+        "WARNING: use_cudnn_on_gpu argument has been removed",
+        "tf.nn.conv2d_backprop_input":
+        "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" "
+        "was renamed to \"filters\"",
     }
     # Right now we can't have both a rename and a warning.
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 4b83d50036..b5e3f1907e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -128,6 +128,19 @@ class TestUpgrade(test_util.TensorFlowTestCase):
         )
     self.assertEqual(new_text, expected_text)
 
+  def testConvolutionOpUpdate(self):
+    text = (
+        "tf.nn.convolution(input, filter, padding, strides, dilation_rate, "
+        "name, data_format)"
+    )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    expected_text = (
+        "tf.nn.convolution(input=input, filters=filter, padding=padding, "
+        "strides=strides, dilations=dilation_rate, name=name, "
+        "data_format=data_format)"
+    )
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From eaaaa8497d0164fa266b42728b553dce4d3a26f2 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Thu, 15 Nov 2018 23:35:52 -0800
Subject: [PATCH 0388/1554] Remove some class symbols from tf 2.0. -
 ConditionalAccumulator - ConditionalAccumulatorBase - DeviceSpec - Dimension
 - gfile.FastGFile - gfile.GFile  -> move to io.gfile.GFile

PiperOrigin-RevId: 221749919
---
 tensorflow/python/framework/device.py         |  2 +-
 tensorflow/python/framework/tensor_shape.py   |  2 +-
 tensorflow/python/ops/data_flow_ops.py        |  4 +-
 tensorflow/python/platform/gfile.py           |  4 +-
 ...orflow.-conditional-accumulator-base.pbtxt | 29 ----------
 .../tensorflow.-conditional-accumulator.pbtxt | 38 ------------
 .../golden/v2/tensorflow.-device-spec.pbtxt   | 37 ------------
 .../api/golden/v2/tensorflow.-dimension.pbtxt | 25 --------
 .../v2/tensorflow.gfile.-fast-g-file.pbtxt    | 58 -------------------
 .../golden/v2/tensorflow.gfile.-g-file.pbtxt  | 58 -------------------
 .../golden/v2/tensorflow.gfile.-open.pbtxt    | 58 -------------------
 .../api/golden/v2/tensorflow.gfile.pbtxt      | 12 ----
 .../tools/api/golden/v2/tensorflow.pbtxt      | 16 -----
 13 files changed, 6 insertions(+), 337 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt

diff --git a/tensorflow/python/framework/device.py b/tensorflow/python/framework/device.py
index 7f6e0a75a5..e7ac6444a4 100644
--- a/tensorflow/python/framework/device.py
+++ b/tensorflow/python/framework/device.py
@@ -23,7 +23,7 @@ import threading
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("DeviceSpec")
+@tf_export(v1=["DeviceSpec"])
 class DeviceSpec(object):
   """Represents a (possibly partial) specification for a TensorFlow device.
 
diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py
index 5a58d27148..960a3dad73 100644
--- a/tensorflow/python/framework/tensor_shape.py
+++ b/tensorflow/python/framework/tensor_shape.py
@@ -169,7 +169,7 @@ def dimension_at_index(shape, index):
     return shape.dims[index]
 
 
-@tf_export("Dimension")
+@tf_export(v1=["Dimension"])
 class Dimension(object):
   """Represents the value of one dimension in a TensorShape."""
 
diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py
index cca8e12b43..0fac7994cb 100644
--- a/tensorflow/python/ops/data_flow_ops.py
+++ b/tensorflow/python/ops/data_flow_ops.py
@@ -1148,7 +1148,7 @@ class Barrier(object):
         self._barrier_ref, name=name)
 
 
-@tf_export("ConditionalAccumulatorBase")
+@tf_export(v1=["ConditionalAccumulatorBase"])
 class ConditionalAccumulatorBase(object):
   """A conditional accumulator for aggregating gradients.
 
@@ -1227,7 +1227,7 @@ class ConditionalAccumulatorBase(object):
         name=name)
 
 
-@tf_export("ConditionalAccumulator")
+@tf_export(v1=["ConditionalAccumulator"])
 class ConditionalAccumulator(ConditionalAccumulatorBase):
   """A conditional accumulator for aggregating gradients.
 
diff --git a/tensorflow/python/platform/gfile.py b/tensorflow/python/platform/gfile.py
index 5927bc2409..d0159e9e98 100644
--- a/tensorflow/python/platform/gfile.py
+++ b/tensorflow/python/platform/gfile.py
@@ -37,7 +37,7 @@ from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('gfile.GFile', 'gfile.Open')
+@tf_export(v1=['gfile.GFile', 'gfile.Open'], v2=['io.gfile.GFile'])
 class GFile(_FileIO):
   """File I/O wrappers without thread locking.
 
@@ -52,7 +52,7 @@ class GFile(_FileIO):
     super(GFile, self).__init__(name=name, mode=mode)
 
 
-@tf_export('gfile.FastGFile')
+@tf_export(v1=['gfile.FastGFile'])
 class FastGFile(_FileIO):
   """File I/O wrappers without thread locking.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt
deleted file mode 100644
index c9a32c16b3..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator-base.pbtxt
+++ /dev/null
@@ -1,29 +0,0 @@
-path: "tensorflow.ConditionalAccumulatorBase"
-tf_class {
-  is_instance: "<class \'tensorflow.python.ops.data_flow_ops.ConditionalAccumulatorBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "accumulator_ref"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'dtype\', \'shape\', \'accumulator_ref\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "num_accumulated"
-    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "set_global_step"
-    argspec: "args=[\'self\', \'new_global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt
deleted file mode 100644
index 15e0ab76b6..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-conditional-accumulator.pbtxt
+++ /dev/null
@@ -1,38 +0,0 @@
-path: "tensorflow.ConditionalAccumulator"
-tf_class {
-  is_instance: "<class \'tensorflow.python.ops.data_flow_ops.ConditionalAccumulator\'>"
-  is_instance: "<class \'tensorflow.python.ops.data_flow_ops.ConditionalAccumulatorBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "accumulator_ref"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'dtype\', \'shape\', \'shared_name\', \'name\', \'reduction_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'conditional_accumulator\', \'MEAN\'], "
-  }
-  member_method {
-    name: "apply_grad"
-    argspec: "args=[\'self\', \'grad\', \'local_step\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], "
-  }
-  member_method {
-    name: "num_accumulated"
-    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "set_global_step"
-    argspec: "args=[\'self\', \'new_global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "take_grad"
-    argspec: "args=[\'self\', \'num_required\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt
deleted file mode 100644
index 92e535c341..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-device-spec.pbtxt
+++ /dev/null
@@ -1,37 +0,0 @@
-path: "tensorflow.DeviceSpec"
-tf_class {
-  is_instance: "<class \'tensorflow.python.framework.device.DeviceSpec\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "job"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "replica"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "task"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'job\', \'replica\', \'task\', \'device_type\', \'device_index\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "from_string"
-    argspec: "args=[\'spec\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "merge_from"
-    argspec: "args=[\'self\', \'dev\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "parse_from_string"
-    argspec: "args=[\'self\', \'spec\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "to_string"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt
deleted file mode 100644
index a9ab27719b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-dimension.pbtxt
+++ /dev/null
@@ -1,25 +0,0 @@
-path: "tensorflow.Dimension"
-tf_class {
-  is_instance: "<class \'tensorflow.python.framework.tensor_shape.Dimension\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "value"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "assert_is_compatible_with"
-    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "is_compatible_with"
-    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "merge_with"
-    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt
deleted file mode 100644
index eecfaffd0a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-fast-g-file.pbtxt
+++ /dev/null
@@ -1,58 +0,0 @@
-path: "tensorflow.gfile.FastGFile"
-tf_class {
-  is_instance: "<class \'tensorflow.python.platform.gfile.FastGFile\'>"
-  is_instance: "<class \'tensorflow.python.lib.io.file_io.FileIO\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "mode"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'mode\'], varargs=None, keywords=None, defaults=[\'r\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "flush"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "next"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "read"
-    argspec: "args=[\'self\', \'n\'], varargs=None, keywords=None, defaults=[\'-1\'], "
-  }
-  member_method {
-    name: "readline"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "readlines"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "seek"
-    argspec: "args=[\'self\', \'offset\', \'whence\', \'position\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], "
-  }
-  member_method {
-    name: "size"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "tell"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "write"
-    argspec: "args=[\'self\', \'file_content\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt
deleted file mode 100644
index 305251059d..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-g-file.pbtxt
+++ /dev/null
@@ -1,58 +0,0 @@
-path: "tensorflow.gfile.GFile"
-tf_class {
-  is_instance: "<class \'tensorflow.python.platform.gfile.GFile\'>"
-  is_instance: "<class \'tensorflow.python.lib.io.file_io.FileIO\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "mode"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'mode\'], varargs=None, keywords=None, defaults=[\'r\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "flush"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "next"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "read"
-    argspec: "args=[\'self\', \'n\'], varargs=None, keywords=None, defaults=[\'-1\'], "
-  }
-  member_method {
-    name: "readline"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "readlines"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "seek"
-    argspec: "args=[\'self\', \'offset\', \'whence\', \'position\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], "
-  }
-  member_method {
-    name: "size"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "tell"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "write"
-    argspec: "args=[\'self\', \'file_content\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt
deleted file mode 100644
index 6e8894180a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.-open.pbtxt
+++ /dev/null
@@ -1,58 +0,0 @@
-path: "tensorflow.gfile.Open"
-tf_class {
-  is_instance: "<class \'tensorflow.python.platform.gfile.GFile\'>"
-  is_instance: "<class \'tensorflow.python.lib.io.file_io.FileIO\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "mode"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'mode\'], varargs=None, keywords=None, defaults=[\'r\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "flush"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "next"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "read"
-    argspec: "args=[\'self\', \'n\'], varargs=None, keywords=None, defaults=[\'-1\'], "
-  }
-  member_method {
-    name: "readline"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "readlines"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "seek"
-    argspec: "args=[\'self\', \'offset\', \'whence\', \'position\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], "
-  }
-  member_method {
-    name: "size"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "tell"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "write"
-    argspec: "args=[\'self\', \'file_content\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
index 65b55a8b7c..c5ef944a97 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
@@ -1,17 +1,5 @@
 path: "tensorflow.gfile"
 tf_module {
-  member {
-    name: "FastGFile"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "GFile"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Open"
-    mtype: "<type \'type\'>"
-  }
   member_method {
     name: "Copy"
     argspec: "args=[\'oldpath\', \'newpath\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 08d3cb5072..8aa3f8e0f3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -8,14 +8,6 @@ tf_module {
     name: "AttrValue"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "ConditionalAccumulator"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "ConditionalAccumulatorBase"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ConfigProto"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -24,14 +16,6 @@ tf_module {
     name: "DType"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "DeviceSpec"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Dimension"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Event"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-- 
GitLab


From b474a89d530f9b0e7a15eb0a37f130e6751320b0 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 16 Nov 2018 00:16:44 -0800
Subject: [PATCH 0389/1554] Change assert_* endpoints for 2.0. Adds docstring
 to assert_scalar.

This removes the data argument everywhere, except Assert, and reorders the arguments to be consistent. It also removes the return values from assert_* functions.

PiperOrigin-RevId: 221753314
---
 tensorflow/python/kernel_tests/BUILD          |   1 +
 .../python/kernel_tests/check_ops_test.py     |  64 +++
 tensorflow/python/ops/check_ops.py            | 523 ++++++++++++++++--
 .../api/golden/v1/tensorflow.debugging.pbtxt  |   2 +-
 .../tools/api/golden/v1/tensorflow.pbtxt      |   2 +-
 .../api/golden/v2/tensorflow.debugging.pbtxt  |  30 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |   8 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  29 +
 8 files changed, 593 insertions(+), 66 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 43353f7a51..5f3b8a2449 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1461,6 +1461,7 @@ cuda_py_test(
     additional_deps = [
         "//third_party/py/numpy",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:def_function",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:math_ops",
diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py
index 88f5cd6f22..15124a19a2 100644
--- a/tensorflow/python/kernel_tests/check_ops_test.py
+++ b/tensorflow/python/kernel_tests/check_ops_test.py
@@ -25,6 +25,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -39,6 +40,69 @@ from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 
 
+class AssertV2Asserts(test.TestCase):
+
+  def test_passes_when_it_should(self):
+    # This is a v2 test and need to run eagerly
+    with context.eager_mode():
+      c1 = constant_op.constant(-1, name="minus_one", dtype=dtypes.int32)
+      c2 = constant_op.constant(2, name="two", dtype=dtypes.int32)
+      c3 = constant_op.constant([3., 3.], name="three", dtype=dtypes.float32)
+      c4 = constant_op.constant([3., 3.5], name="three_and_a_half",
+                                dtype=dtypes.float32)
+      scalar = c1
+      non_scalar = c3
+      integer = c1
+      non_integer = c3
+      positive = c2
+      negative = c1
+      cases = [
+          (check_ops.assert_equal_v2, (c1, c1), (c1, c2)),
+          (check_ops.assert_less_v2, (c1, c2), (c1, c1)),
+          (check_ops.assert_near_v2, (c3, c3), (c3, c4)),
+          (check_ops.assert_greater_v2, (c2, c1), (c1, c1)),
+          (check_ops.assert_negative_v2, (negative,), (positive,)),
+          (check_ops.assert_positive_v2, (positive,), (negative,)),
+          (check_ops.assert_less_equal_v2, (c1, c1), (c2, c1)),
+          (check_ops.assert_none_equal_v2, (c1, c2), (c3, c4)),
+          (check_ops.assert_non_negative_v2, (positive,), (negative,)),
+          (check_ops.assert_non_positive_v2, (negative,), (positive,)),
+          (check_ops.assert_greater_equal_v2, (c1, c1), (c1, c2)),
+          (check_ops.assert_type_v2, (c1, dtypes.int32), (c1, dtypes.float32),
+           TypeError),
+          (check_ops.assert_integer_v2, (integer,), (non_integer,),
+           TypeError),
+          (check_ops.assert_scalar_v2, (scalar,), (non_scalar,),
+           ValueError),
+          (check_ops.assert_rank_v2, (c1, 0), (c3, 2), ValueError),
+          (check_ops.assert_rank_in_v2, (c1, [0, 1]), (c1, [1, 2]),
+           ValueError),
+          (check_ops.assert_rank_at_least_v2, (non_scalar, 1), (scalar, 1),
+           ValueError),
+      ]
+
+      for case in cases:
+        fn = case[0]
+        passing_args = case[1]
+        failing_args = case[2]
+        error = errors.InvalidArgumentError if len(case) < 4 else case[3]
+
+        print("Testing %s passing properly." % fn)
+
+        fn(*passing_args)
+
+        print("Testing %s failing properly." % fn)
+
+        @def_function.function
+        def failing_fn():
+          fn(*failing_args, message="fail")  # pylint: disable=cell-var-from-loop
+
+        with self.assertRaisesRegexp(error, "fail"):
+          failing_fn()
+
+        del failing_fn
+
+
 class AssertProperIterableTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py
index 5589bbc848..75dad172fc 100644
--- a/tensorflow/python/ops/check_ops.py
+++ b/tensorflow/python/ops/check_ops.py
@@ -119,9 +119,31 @@ def assert_proper_iterable(values):
         'Expected argument "values" to be iterable.  Found: %s' % type(values))
 
 
-@tf_export(
-    'debugging.assert_negative',
-    v1=['debugging.assert_negative', 'assert_negative'])
+@tf_export('debugging.assert_negative', v1=[])
+def assert_negative_v2(x, message=None, summarize=None, name=None):
+  """Assert the condition `x < 0` holds element-wise.
+
+  This Op checks that `x[i] < 0` holds for every element of `x`. If `x` is
+  empty, this is trivially satisfied.
+
+  If `x` is not negative everywhere, `message`, as well as the first `summarize`
+  entries of `x` are printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to "assert_negative".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x[i] < 0` is False. The check can be performed immediately during eager
+      execution or if `x` is statically known.
+  """
+  assert_negative(x=x, message=message, summarize=summarize, name=name)
+
+
+@tf_export(v1=['debugging.assert_negative', 'assert_negative'])
 @deprecation.deprecated_endpoints('assert_negative')
 def assert_negative(x, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x < 0` holds element-wise.
@@ -163,9 +185,31 @@ def assert_negative(x, data=None, summarize=None, message=None, name=None):
     return assert_less(x, zero, data=data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_positive',
-    v1=['debugging.assert_positive', 'assert_positive'])
+@tf_export('debugging.assert_positive', v1=[])
+def assert_positive_v2(x, message=None, summarize=None, name=None):
+  """Assert the condition `x > 0` holds element-wise.
+
+  This Op checks that `x[i] > 0` holds for every element of `x`. If `x` is
+  empty, this is trivially satisfied.
+
+  If `x` is not positive everywhere, `message`, as well as the first `summarize`
+  entries of `x` are printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional). Defaults to "assert_positive".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x[i] > 0` is False. The check can be performed immediately during eager
+      execution or if `x` is statically known.
+  """
+  assert_positive(x=x, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_positive', 'assert_positive'])
 @deprecation.deprecated_endpoints('assert_positive')
 def assert_positive(x, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x > 0` holds element-wise.
@@ -206,9 +250,32 @@ def assert_positive(x, data=None, summarize=None, message=None, name=None):
     return assert_less(zero, x, data=data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_non_negative',
-    v1=['debugging.assert_non_negative', 'assert_non_negative'])
+@tf_export('debugging.assert_non_negative', v1=[])
+def assert_non_negative_v2(x, message=None, summarize=None, name=None):
+  """Assert the condition `x >= 0` holds element-wise.
+
+  This Op checks that `x[i] >= 0` holds for every element of `x`. If `x` is
+  empty, this is trivially satisfied.
+
+  If `x` is not >= 0 everywhere, `message`, as well as the first `summarize`
+  entries of `x` are printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to
+      "assert_non_negative".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x[i] >= 0` is False. The check can be performed immediately during eager
+      execution or if `x` is statically known.
+  """
+  assert_non_negative(x=x, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_non_negative', 'assert_non_negative'])
 @deprecation.deprecated_endpoints('assert_non_negative')
 def assert_non_negative(x, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x >= 0` holds element-wise.
@@ -251,9 +318,32 @@ def assert_non_negative(x, data=None, summarize=None, message=None, name=None):
     return assert_less_equal(zero, x, data=data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_non_positive',
-    v1=['debugging.assert_non_positive', 'assert_non_positive'])
+@tf_export('debugging.assert_non_positive', v1=[])
+def assert_non_positive_v2(x, message=None, summarize=None, name=None):
+  """Assert the condition `x <= 0` holds element-wise.
+
+  This Op checks that `x[i] <= 0` holds for every element of `x`. If `x` is
+  empty, this is trivially satisfied.
+
+  If `x` is not <= 0 everywhere, `message`, as well as the first `summarize`
+  entries of `x` are printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to
+      "assert_non_positive".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x[i] <= 0` is False. The check can be performed immediately during eager
+      execution or if `x` is statically known.
+  """
+  assert_non_positive(x=x, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_non_positive', 'assert_non_positive'])
 @deprecation.deprecated_endpoints('assert_non_positive')
 def assert_non_positive(x, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x <= 0` holds element-wise.
@@ -296,7 +386,33 @@ def assert_non_positive(x, data=None, summarize=None, message=None, name=None):
     return assert_less_equal(x, zero, data=data, summarize=summarize)
 
 
-@tf_export('debugging.assert_equal', 'assert_equal')
+@tf_export('debugging.assert_equal', 'assert_equal', v1=[])
+def assert_equal_v2(x, y, message=None, summarize=None, name=None):
+  """Assert the condition `x == y` holds element-wise.
+
+  This Op checks that `x[i] == y[i]` holds for every pair of (possibly
+  broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is
+  trivially satisfied.
+
+  If `x` and `y` are not equal, `message`, as well as the first `summarize`
+  entries of `x` and `y` are printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    y:  Numeric `Tensor`, same dtype as and broadcastable to `x`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to "assert_equal".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x == y` is False. The check can be performed immediately during eager
+      execution or if `x` and `y` are statically known.
+  """
+  assert_equal(x=x, y=y, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_equal', 'assert_equal'])
 def assert_equal(x, y, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x == y` holds element-wise.
 
@@ -396,9 +512,36 @@ def assert_equal(x, y, data=None, summarize=None, message=None, name=None):
     return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_none_equal',
-    v1=['debugging.assert_none_equal', 'assert_none_equal'])
+@tf_export('debugging.assert_none_equal', v1=[])
+def assert_none_equal_v2(x, y, summarize=None, message=None, name=None):
+  """Assert the condition `x != y` holds for all elements.
+
+  This Op checks that `x[i] != y[i]` holds for every pair of (possibly
+  broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is
+  trivially satisfied.
+
+  If any elements of `x` and `y` are equal, `message`, as well as the first
+  `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError`
+  is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    y:  Numeric `Tensor`, same dtype as and broadcastable to `x`.
+    summarize: Print this many entries of each tensor.
+    message: A string to prefix to the default message.
+    name: A name for this operation (optional).  Defaults to
+    "assert_none_equal".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x != y` is False for any pair of elements in `x` and `y`. The check can
+      be performed immediately during eager execution or if `x` and `y` are
+      statically known.
+  """
+  assert_none_equal(x=x, y=y, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_none_equal', 'assert_none_equal'])
 @deprecation.deprecated_endpoints('assert_none_equal')
 def assert_none_equal(
     x, y, data=None, summarize=None, message=None, name=None):
@@ -450,7 +593,52 @@ def assert_none_equal(
     return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export('debugging.assert_near', v1=['debugging.assert_near', 'assert_near'])
+@tf_export('debugging.assert_near', v1=[])
+def assert_near_v2(x, y, rtol=None, atol=None, message=None, summarize=None,
+                   name=None):
+  """Assert the condition `x` and `y` are close element-wise.
+
+  This Op checks that `x[i] - y[i] < atol + rtol * tf.abs(y[i])` holds for every
+  pair of (possibly broadcast) elements of `x` and `y`. If both `x` and `y` are
+  empty, this is trivially satisfied.
+
+  If any elements of `x` and `y` are not close, `message`, as well as the first
+  `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError`
+  is raised.
+
+  The default `atol` and `rtol` is `10 * eps`, where `eps` is the smallest
+  representable positive number such that `1 + eps != 1`.  This is about
+  `1.2e-6` in `32bit`, `2.22e-15` in `64bit`, and `0.00977` in `16bit`.
+  See `numpy.finfo`.
+
+  Args:
+    x: Float or complex `Tensor`.
+    y: Float or complex `Tensor`, same dtype as and broadcastable to `x`.
+    rtol:  `Tensor`.  Same `dtype` as, and broadcastable to, `x`.
+      The relative tolerance.  Default is `10 * eps`.
+    atol:  `Tensor`.  Same `dtype` as, and broadcastable to, `x`.
+      The absolute tolerance.  Default is `10 * eps`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to "assert_near".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x != y` is False for any pair of elements in `x` and `y`. The check can
+      be performed immediately during eager execution or if `x` and `y` are
+      statically known.
+
+  @compatibility(numpy)
+  Similar to `numpy.assert_allclose`, except tolerance depends on data type.
+  This is due to the fact that `TensorFlow` is often used with `32bit`, `64bit`,
+  and even `16bit` data.
+  @end_compatibility
+  """
+  assert_near(x=x, y=y, rtol=rtol, atol=atol, summarize=summarize,
+              message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_near', 'assert_near'])
 @deprecation.deprecated_endpoints('assert_near')
 def assert_near(
     x, y, rtol=None, atol=None, data=None, summarize=None, message=None,
@@ -529,7 +717,34 @@ def assert_near(
     return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export('debugging.assert_less', 'assert_less')
+@tf_export('debugging.assert_less', 'assert_less', v1=[])
+def assert_less_v2(x, y, message=None, summarize=None, name=None):
+  """Assert the condition `x < y` holds element-wise.
+
+  This Op checks that `x[i] < y[i]` holds for every pair of (possibly
+  broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is
+  trivially satisfied.
+
+  If `x` is not less than `y` element-wise, `message`, as well as the first
+  `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError` is
+  raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    y:  Numeric `Tensor`, same dtype as and broadcastable to `x`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to "assert_less".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x < y` is False. The check can be performed immediately during eager
+      execution or if `x` and `y` are statically known.
+  """
+  assert_less(x=x, y=y, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_less', 'assert_less'])
 def assert_less(x, y, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x < y` holds element-wise.
 
@@ -577,9 +792,34 @@ def assert_less(x, y, data=None, summarize=None, message=None, name=None):
     return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_less_equal',
-    v1=['debugging.assert_less_equal', 'assert_less_equal'])
+@tf_export('debugging.assert_less_equal', v1=[])
+def assert_less_equal_v2(x, y, message=None, summarize=None, name=None):
+  """Assert the condition `x <= y` holds element-wise.
+
+  This Op checks that `x[i] <= y[i]` holds for every pair of (possibly
+  broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is
+  trivially satisfied.
+
+  If `x` is not less or equal than `y` element-wise, `message`, as well as the
+  first `summarize` entries of `x` and `y` are printed, and
+  `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    y:  Numeric `Tensor`, same dtype as and broadcastable to `x`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional). Defaults to "assert_less_equal".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x <= y` is False. The check can be performed immediately during eager
+      execution or if `x` and `y` are statically known.
+  """
+  assert_less_equal(x=x, y=y, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_less_equal', 'assert_less_equal'])
 @deprecation.deprecated_endpoints('assert_less_equal')
 def assert_less_equal(x, y, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x <= y` holds element-wise.
@@ -628,7 +868,34 @@ def assert_less_equal(x, y, data=None, summarize=None, message=None, name=None):
     return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export('debugging.assert_greater', 'assert_greater')
+@tf_export('debugging.assert_greater', 'assert_greater', v1=[])
+def assert_greater_v2(x, y, message=None, summarize=None, name=None):
+  """Assert the condition `x > y` holds element-wise.
+
+  This Op checks that `x[i] > y[i]` holds for every pair of (possibly
+  broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is
+  trivially satisfied.
+
+  If `x` is not greater than `y` element-wise, `message`, as well as the first
+  `summarize` entries of `x` and `y` are printed, and `InvalidArgumentError` is
+  raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    y:  Numeric `Tensor`, same dtype as and broadcastable to `x`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to "assert_greater".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x > y` is False. The check can be performed immediately during eager
+      execution or if `x` and `y` are statically known.
+  """
+  assert_greater(x=x, y=y, summarize=summarize, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_greater', 'assert_greater'])
 def assert_greater(x, y, data=None, summarize=None, message=None, name=None):
   """Assert the condition `x > y` holds element-wise.
 
@@ -676,9 +943,36 @@ def assert_greater(x, y, data=None, summarize=None, message=None, name=None):
     return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_greater_equal',
-    v1=['debugging.assert_greater_equal', 'assert_greater_equal'])
+@tf_export('debugging.assert_greater_equal', v1=[])
+def assert_greater_equal_v2(x, y, message=None, summarize=None, name=None):
+  """Assert the condition `x >= y` holds element-wise.
+
+  This Op checks that `x[i] >= y[i]` holds for every pair of (possibly
+  broadcast) elements of `x` and `y`. If both `x` and `y` are empty, this is
+  trivially satisfied.
+
+  If `x` is not greater or equal to `y` element-wise, `message`, as well as the
+  first `summarize` entries of `x` and `y` are printed, and
+  `InvalidArgumentError` is raised.
+
+  Args:
+    x:  Numeric `Tensor`.
+    y:  Numeric `Tensor`, same dtype as and broadcastable to `x`.
+    message: A string to prefix to the default message.
+    summarize: Print this many entries of each tensor.
+    name: A name for this operation (optional).  Defaults to
+    "assert_greater_equal".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x >= y` is False. The check can be performed immediately during eager
+      execution or if `x` and `y` are statically known.
+  """
+  assert_greater_equal(x=x, y=y, summarize=summarize, message=message,
+                       name=name)
+
+
+@tf_export(v1=['debugging.assert_greater_equal', 'assert_greater_equal'])
 @deprecation.deprecated_endpoints('assert_greater_equal')
 def assert_greater_equal(x, y, data=None, summarize=None, message=None,
                          name=None):
@@ -777,7 +1071,31 @@ def _assert_rank_condition(
   return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export('debugging.assert_rank', 'assert_rank')
+@tf_export('debugging.assert_rank', 'assert_rank', v1=[])
+def assert_rank_v2(x, rank, message=None, name=None):
+  """Assert that `x` has rank equal to `rank`.
+
+  This Op checks that the rank of `x` is equal to `rank`.
+
+  If `x` has a different rank, `message`, as well as the shape of `x` are
+  printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x: `Tensor`.
+    rank: Scalar integer `Tensor`.
+    message: A string to prefix to the default message.
+    name: A name for this operation (optional). Defaults to
+      "assert_rank".
+
+  Raises:
+    InvalidArgumentError: if the check can be performed immediately and
+      `x` does not have rank `rank`. The check can be performed immediately
+      during eager execution or if the shape of `x` is statically known.
+  """
+  assert_rank(x=x, rank=rank, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_rank', 'assert_rank'])
 def assert_rank(x, rank, data=None, summarize=None, message=None, name=None):
   """Assert `x` has rank equal to `rank`.
 
@@ -792,7 +1110,7 @@ def assert_rank(x, rank, data=None, summarize=None, message=None, name=None):
     x:  Numeric `Tensor`.
     rank:  Scalar integer `Tensor`.
     data:  The tensors to print out if the condition is False.  Defaults to
-      error message and first few entries of `x`.
+      error message and the shape of `x`.
     summarize: Print this many entries of each tensor.
     message: A string to prefix to the default message.
     name: A name for this operation (optional).  Defaults to "assert_rank".
@@ -839,9 +1157,31 @@ def assert_rank(x, rank, data=None, summarize=None, message=None, name=None):
   return assert_op
 
 
-@tf_export(
-    'debugging.assert_rank_at_least',
-    v1=['debugging.assert_rank_at_least', 'assert_rank_at_least'])
+@tf_export('debugging.assert_rank_at_least', v1=[])
+def assert_rank_at_least_v2(x, rank, message=None, name=None):
+  """Assert that `x` has rank of at least `rank`.
+
+  This Op checks that the rank of `x` is greater or equal to `rank`.
+
+  If `x` has a rank lower than `rank`, `message`, as well as the shape of `x`
+  are printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x: `Tensor`.
+    rank: Scalar integer `Tensor`.
+    message: A string to prefix to the default message.
+    name: A name for this operation (optional).  Defaults to
+      "assert_rank_at_least".
+
+  Raises:
+    InvalidArgumentError: `x` does not have rank at least `rank`, but the rank
+      cannot be statically determined.
+    ValueError: If static checks determine `x` has mismatched rank.
+  """
+  assert_rank_at_least(x=x, rank=rank, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_rank_at_least', 'assert_rank_at_least'])
 @deprecation.deprecated_endpoints('assert_rank_at_least')
 def assert_rank_at_least(
     x, rank, data=None, summarize=None, message=None, name=None):
@@ -973,9 +1313,30 @@ def _assert_ranks_condition(
   return control_flow_ops.Assert(condition, data, summarize=summarize)
 
 
-@tf_export(
-    'debugging.assert_rank_in',
-    v1=['debugging.assert_rank_in', 'assert_rank_in'])
+@tf_export('debugging.assert_rank_in', v1=[])
+def assert_rank_in_v2(x, ranks, message=None, name=None):
+  """Assert that `x` has a rank in `ranks`.
+
+  This Op checks that the rank of `x` is in `ranks`.
+
+  If `x` has a different rank, `message`, as well as the shape of `x` are
+  printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x: `Tensor`.
+    ranks: `Iterable` of scalar `Tensor` objects.
+    message: A string to prefix to the default message.
+    name: A name for this operation (optional). Defaults to "assert_rank_in".
+
+  Raises:
+    InvalidArgumentError: `x` does not have rank in `ranks`, but the rank cannot
+      be statically determined.
+    ValueError: If static checks determine `x` has mismatched rank.
+  """
+  assert_rank_in(x=x, ranks=ranks, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_rank_in', 'assert_rank_in'])
 @deprecation.deprecated_endpoints('assert_rank_in')
 def assert_rank_in(
     x, ranks, data=None, summarize=None, message=None, name=None):
@@ -1038,9 +1399,25 @@ def assert_rank_in(
   return assert_op
 
 
-@tf_export(
-    'debugging.assert_integer',
-    v1=['debugging.assert_integer', 'assert_integer'])
+@tf_export('debugging.assert_integer', v1=[])
+def assert_integer_v2(x, message=None, name=None):
+  """Assert that `x` is of integer dtype.
+
+  If `x` has a non-integer type, `message`, as well as the dtype of `x` are
+  printed, and `InvalidArgumentError` is raised.
+
+  Args:
+    x: A `Tensor`.
+    message: A string to prefix to the default message.
+    name: A name for this operation (optional). Defaults to "assert_integer".
+
+  Raises:
+    TypeError:  If `x.dtype` is not a non-quantized integer type.
+  """
+  assert_integer(x=x, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_integer', 'assert_integer'])
 @deprecation.deprecated_endpoints('assert_integer')
 def assert_integer(x, message=None, name=None):
   """Assert that `x` is of integer dtype.
@@ -1079,13 +1456,30 @@ def assert_integer(x, message=None, name=None):
     return control_flow_ops.no_op('statically_determined_was_integer')
 
 
-@tf_export('debugging.assert_type', v1=['debugging.assert_type', 'assert_type'])
+@tf_export('debugging.assert_type', v1=[])
+def assert_type_v2(tensor, tf_type, message=None, name=None):
+  """Asserts that the given `Tensor` is of the specified type.
+
+  Args:
+    tensor: A `Tensor`.
+    tf_type: A tensorflow type (`dtypes.float32`, `tf.int64`, `dtypes.bool`,
+      etc).
+    message: A string to prefix to the default message.
+    name:  A name for this operation. Defaults to "assert_type"
+
+  Raises:
+    TypeError: If the tensor's data type doesn't match `tf_type`.
+  """
+  assert_type(tensor=tensor, tf_type=tf_type, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_type', 'assert_type'])
 @deprecation.deprecated_endpoints('assert_type')
 def assert_type(tensor, tf_type, message=None, name=None):
   """Statically asserts that the given `Tensor` is of the specified type.
 
   Args:
-    tensor: A tensorflow `Tensor`.
+    tensor: A `Tensor`.
     tf_type: A tensorflow type (`dtypes.float32`, `tf.int64`, `dtypes.bool`,
       etc).
     message: A string to prefix to the default message.
@@ -1260,8 +1654,10 @@ def assert_same_float_dtype(tensors=None, dtype=None):
     tensors: Tensors of input values. Can include `None` elements, which will be
         ignored.
     dtype: Expected type.
+
   Returns:
     Validated type.
+
   Raises:
     ValueError: if neither `tensors` nor `dtype` is supplied, or result is not
         float, or the common type of the inputs is not a floating point type.
@@ -1275,20 +1671,57 @@ def assert_same_float_dtype(tensors=None, dtype=None):
   return dtype
 
 
-@tf_export(
-    'debugging.assert_scalar', v1=['debugging.assert_scalar', 'assert_scalar'])
+@tf_export('debugging.assert_scalar', v1=[])
+def assert_scalar_v2(tensor, message=None, name=None):
+  """Asserts that the given `tensor` is a scalar.
+
+  This function raises `ValueError` unless it can be certain that the given
+  `tensor` is a scalar. `ValueError` is also raised if the shape of `tensor` is
+  unknown.
+
+  Args:
+    tensor: A `Tensor`.
+    message: A string to prefix to the default message.
+    name:  A name for this operation. Defaults to "assert_scalar"
+
+  Raises:
+    ValueError: If the tensor is not scalar (rank 0), or if its shape is
+      unknown.
+  """
+  assert_scalar(tensor=tensor, message=message, name=name)
+
+
+@tf_export(v1=['debugging.assert_scalar', 'assert_scalar'])
 @deprecation.deprecated_endpoints('assert_scalar')
-def assert_scalar(tensor, name=None):
+def assert_scalar(tensor, name=None, message=None):
+  """Asserts that the given `tensor` is a scalar.
+
+  This function raises `ValueError` unless it can be certain that the given
+  `tensor` is a scalar. `ValueError` is also raised if the shape of `tensor` is
+  unknown.
+
+  Args:
+    tensor: A `Tensor`.
+    name:  A name for this operation. Defaults to "assert_scalar"
+    message: A string to prefix to the default message.
+
+  Returns:
+    The input tensor (potentially converted to a `Tensor`).
+
+  Raises:
+    ValueError: If the tensor is not scalar (rank 0), or if its shape is
+      unknown.
+  """
   with ops.name_scope(name, 'assert_scalar', [tensor]) as name_scope:
     tensor = ops.convert_to_tensor(tensor, name=name_scope)
     shape = tensor.get_shape()
     if shape.ndims != 0:
       if context.executing_eagerly():
-        raise ValueError('Expected scalar shape, saw shape: %s.'
-                         % (shape,))
+        raise ValueError('%sExpected scalar shape, saw shape: %s.'
+                         % (message or '', shape,))
       else:
-        raise ValueError('Expected scalar shape for %s, saw shape: %s.'
-                         % (tensor.name, shape))
+        raise ValueError('%sExpected scalar shape for %s, saw shape: %s.'
+                         % (message or '', tensor.name, shape))
     return tensor
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt
index ab6287f8cd..8a7f1e9363 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.debugging.pbtxt
@@ -78,7 +78,7 @@ tf_module {
   }
   member_method {
     name: "assert_scalar"
-    argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'tensor\', \'name\', \'message\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "assert_type"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 9904e264eb..707337ebfa 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -778,7 +778,7 @@ tf_module {
   }
   member_method {
     name: "assert_scalar"
-    argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'tensor\', \'name\', \'message\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "assert_type"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
index ab6287f8cd..5d8afcd8b2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
@@ -10,15 +10,15 @@ tf_module {
   }
   member_method {
     name: "assert_equal"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_greater"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_greater_equal"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_integer"
@@ -26,35 +26,35 @@ tf_module {
   }
   member_method {
     name: "assert_less"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_less_equal"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_near"
-    argspec: "args=[\'x\', \'y\', \'rtol\', \'atol\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'rtol\', \'atol\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_negative"
-    argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_non_negative"
-    argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_non_positive"
-    argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_none_equal"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_positive"
-    argspec: "args=[\'x\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_proper_iterable"
@@ -62,15 +62,15 @@ tf_module {
   }
   member_method {
     name: "assert_rank"
-    argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'rank\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "assert_rank_at_least"
-    argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'rank\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "assert_rank_in"
-    argspec: "args=[\'x\', \'ranks\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'ranks\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "assert_same_float_dtype"
@@ -78,7 +78,7 @@ tf_module {
   }
   member_method {
     name: "assert_scalar"
-    argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'tensor\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "assert_type"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 8aa3f8e0f3..a1e7b3231f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -482,19 +482,19 @@ tf_module {
   }
   member_method {
     name: "assert_equal"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_greater"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_less"
-    argspec: "args=[\'x\', \'y\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'y\', \'message\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "assert_rank"
-    argspec: "args=[\'x\', \'rank\', \'data\', \'summarize\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'rank\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "atan"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 6f46c13710..e83e771001 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -176,9 +176,38 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "SUM_OVER_BATCH_SIZE.\n"
     )
 
+    assert_return_type_comment = (
+        "WARNING: assert_* functions have been changed to return None, the "
+        "data argument has been removed, and arguments have been reordered."
+    )
+
+    assert_rank_comment = (
+        "WARNING: assert_rank_* functions have been changed to return None, and"
+        " the data and summarize arguments have been removed."
+    )
+
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
     self.function_warnings = {
+        "tf.assert_greater": assert_return_type_comment,
+        "tf.assert_equal": assert_return_type_comment,
+        "tf.assert_less": assert_return_type_comment,
+        "tf.assert_rank": assert_rank_comment,
+        "tf.debugging.assert_equal": assert_return_type_comment,
+        "tf.debugging.assert_greater": assert_return_type_comment,
+        "tf.debugging.assert_greater_equal": assert_return_type_comment,
+        "tf.debugging.assert_integer": assert_return_type_comment,
+        "tf.debugging.assert_less": assert_return_type_comment,
+        "tf.debugging.assert_less_equal": assert_return_type_comment,
+        "tf.debugging.assert_near": assert_return_type_comment,
+        "tf.debugging.assert_negative": assert_return_type_comment,
+        "tf.debugging.assert_non_negative": assert_return_type_comment,
+        "tf.debugging.assert_non_positive": assert_return_type_comment,
+        "tf.debugging.assert_none_equal": assert_return_type_comment,
+        "tf.debugging.assert_positive": assert_return_type_comment,
+        "tf.debugging.assert_rank": assert_rank_comment,
+        "tf.debugging.assert_rank_at_least": assert_rank_comment,
+        "tf.debugging.assert_rank_in": assert_rank_comment,
         "tf.train.exponential_decay":
             decay_function_comment,
         "tf.train.piecewise_constant":
-- 
GitLab


From 2a0e004358f13d6ebe936ceab1b5e7d147606583 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 01:02:23 -0800
Subject: [PATCH 0390/1554] compat: Update forward compatibility horizon to
 2018-11-16

PiperOrigin-RevId: 221756964
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index eea6567774..a12f798d8f 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 15)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 16)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 25c6011c3725461fa507ea5abbbb7f382a279742 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 01:20:26 -0800
Subject: [PATCH 0391/1554] Add -fno-canonical-system-headers to gcc's command
 line so we can correctly validate system includes with bazel.

PiperOrigin-RevId: 221759265
---
 third_party/gpus/crosstool/CROSSTOOL.tpl | 3 ++-
 third_party/gpus/cuda_configure.bzl      | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/third_party/gpus/crosstool/CROSSTOOL.tpl b/third_party/gpus/crosstool/CROSSTOOL.tpl
index 3189cf8e31..921188cbb4 100644
--- a/third_party/gpus/crosstool/CROSSTOOL.tpl
+++ b/third_party/gpus/crosstool/CROSSTOOL.tpl
@@ -184,7 +184,8 @@ toolchain {
       action: "c++-link-dynamic-library"
       action: "c++-link-nodeps-dynamic-library"
       flag_group {
-        flag:"-no-canonical-prefixes"
+        flag: "-no-canonical-prefixes"
+        %{extra_no_canonical_prefixes_flags}
       }
     }
   }
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 831a3067b2..03c67bcb3d 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1418,6 +1418,7 @@ def _create_local_cuda_repository(repository_ctx):
         flag: "-Wno-invalid-partial-specialization"
     """
     cuda_defines["%{host_compiler_includes}"] = host_compiler_includes
+    cuda_defines["%{extra_no_canonical_prefixes_flags}"] = ""
     _tpl(repository_ctx, "crosstool:BUILD", {
         "%{linker_files}": ":empty",
         "%{win_linker_files}": ":empty"
@@ -1439,6 +1440,14 @@ def _create_local_cuda_repository(repository_ctx):
             repository_ctx, cuda_config) +
         "\n  cxx_builtin_include_directory: \"%s\"" % cupti_header_dir +
         "\n  cxx_builtin_include_directory: \"%s\"" % cudnn_header_dir)
+
+    # For gcc, do not canonicalize system header paths; some versions of gcc
+    # pick the shortest possible path for system includes when creating the
+    # .d file - given that includes that are prefixed with "../" multiple
+    # time quickly grow longer than the root of the tree, this can lead to
+    # bazel's header check failing.
+    cuda_defines["%{extra_no_canonical_prefixes_flags}"] = (
+        "flag: \"-fno-canonical-system-headers\"")
     nvcc_path = str(
         repository_ctx.path("%s/bin/nvcc%s" % (
             cuda_config.cuda_toolkit_path,
-- 
GitLab


From 54588d50cbde7fefe3dd67c7e8338c68917f5e33 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 16 Nov 2018 02:48:36 -0800
Subject: [PATCH 0392/1554] Allowed creating partitioned variables in eager

PiperOrigin-RevId: 221768491
---
 tensorflow/python/ops/variable_scope.py | 13 -------------
 tensorflow/python/ops/variables.py      |  4 ----
 2 files changed, 17 deletions(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index fe93bfb61f..077bb647ef 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -646,10 +646,6 @@ class _VariableStore(object):
         when violating reuse during variable creation, or if an existing
         sharded variable exists for the given name but with different sharding.
     """
-    if context.executing_eagerly():
-      raise NotImplementedError("Partitioned variables are not yet supported "
-                                "when eager execution is enabled.")
-
     initializing_from_value = initializer is not None and isinstance(
         initializer, ops.Tensor)
     reuse_without_partition = reuse and not partitioner
@@ -1080,9 +1076,6 @@ class VariableScope(object):
       if self._caching_device is not None:
         raise NotImplementedError("Caching devices is not yet supported "
                                   "when eager execution is enabled.")
-      if self._partitioner is not None:
-        raise NotImplementedError("Partitioned variables are not yet supported "
-                                  "when eager execution is enabled.")
       self._reuse = AUTO_REUSE
       self._use_resource = True
 
@@ -1162,9 +1155,6 @@ class VariableScope(object):
 
   def set_partitioner(self, partitioner):
     """Set partitioner for this scope."""
-    if partitioner and context.executing_eagerly():
-      raise NotImplementedError("Partitioned variables are not yet supported "
-                                "when eager execution is enabled.")
     self._partitioner = partitioner
 
   def set_custom_getter(self, custom_getter):
@@ -1277,9 +1267,6 @@ class VariableScope(object):
                                 synchronization=VariableSynchronization.AUTO,
                                 aggregation=VariableAggregation.NONE):
     """Gets an existing variable with this name or create a new one."""
-    if context.executing_eagerly():
-      raise NotImplementedError("Partitioned variables are not yet supported "
-                                "when eager execution is enabled.")
     if initializer is None:
       initializer = self._initializer
     if regularizer is None:
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index e43736069e..8da740d265 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -2504,11 +2504,7 @@ class PartitionedVariable(object):
         `partitions` is not a list.
       ValueError: If `variable_list` is empty, or the `Variable` shape
         information does not match `shape`, or `partitions` has invalid values.
-      RuntimeError: If eager execution is enabled
     """
-    if context.executing_eagerly():
-      raise RuntimeError(
-          "tf.PartitionedVariable not supported with eager execution enabled.")
     if not isinstance(variable_list, (list, tuple)):
       raise TypeError(
           "variable_list is not a list or tuple: %s" % variable_list)
-- 
GitLab


From 1d3d92dfdecd38daf068583b39aef7811a604601 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 06:39:12 -0800
Subject: [PATCH 0393/1554] No-op refactor and comment fix.

PiperOrigin-RevId: 221786311
---
 tensorflow/contrib/gan/python/train.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index 73185c79fc..cf5b9d9476 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -1071,8 +1071,19 @@ def get_sequential_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
   return get_hooks
 
 
+def _num_joint_steps(train_steps):
+  g_steps = train_steps.generator_train_steps
+  d_steps = train_steps.discriminator_train_steps
+  # Get the number of each type of step that should be run.
+  num_d_and_g_steps = min(g_steps, d_steps)
+  num_g_steps = g_steps - num_d_and_g_steps
+  num_d_steps = d_steps - num_d_and_g_steps
+
+  return num_d_and_g_steps, num_g_steps, num_d_steps
+
+
 def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
-  """Returns a hooks function for sequential GAN training.
+  """Returns a hooks function for joint GAN training.
 
   When using these train hooks, IT IS RECOMMENDED TO USE `use_locking=True` ON
   ALL OPTIMIZERS TO AVOID RACE CONDITIONS.
@@ -1105,12 +1116,7 @@ def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
   Returns:
     A function that takes a GANTrainOps tuple and returns a list of hooks.
   """
-  g_steps = train_steps.generator_train_steps
-  d_steps = train_steps.discriminator_train_steps
-  # Get the number of each type of step that should be run.
-  num_d_and_g_steps = min(g_steps, d_steps)
-  num_g_steps = g_steps - num_d_and_g_steps
-  num_d_steps = d_steps - num_d_and_g_steps
+  num_d_and_g_steps, num_g_steps, num_d_steps = _num_joint_steps(train_steps)
 
   def get_hooks(train_ops):
     g_op = train_ops.generator_train_op
-- 
GitLab


From a5ad58c7e9f8cb2b6830f8e6ecf506ad8373f3f4 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 16 Nov 2018 06:48:38 -0800
Subject: [PATCH 0394/1554] [XLA] Add a less aggressive mode to LICM.

LICM extends the live-range of the output of the hoisted instruction to be the entire while loop, which may be problematic on platforms where memory is limited. This adds a less aggressive mode to LICM, which will not hoist instructions if their output is significantly larger than their inputs, mitigating the most egregious cases of this problem.

PiperOrigin-RevId: 221787120
---
 tensorflow/compiler/xla/service/BUILD         |  2 +-
 .../while_loop_invariant_code_motion.cc       | 32 +++++++++++
 .../while_loop_invariant_code_motion.h        | 11 +++-
 .../while_loop_invariant_code_motion_test.cc  | 54 +++++++++++++++++++
 4 files changed, 96 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 3cbf0d0c8c..da185db2e3 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3343,9 +3343,9 @@ cc_library(
         ":tuple_util",
         ":while_loop_analysis",
         ":while_util",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
-        "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
index b7c28bfac7..41011176ff 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/tuple_util.h"
 #include "tensorflow/compiler/xla/service/while_loop_analysis.h"
 #include "tensorflow/compiler/xla/service/while_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/util.h"
 
 namespace xla {
@@ -207,6 +208,37 @@ WhileLoopInvariantCodeMotion::TryHoistingInvariantInstructionsFromWhileBody(
       continue;
     }
 
+    if (!hoist_size_inflating_ops_) {
+      // Check that hoisting the instruction doesn't cause a significant memory
+      // blow-up. LICM extends the live-range of the output of the hoisted
+      // instruction to be the entire while loop, which may be problematic on
+      // platforms where memory is limited. This can be especially harmful if
+      // the instruction has a significantly larger output than its input, e.g.
+      // kIota, kBroadcast or kConstant.
+      int64 input_size = 0, output_size = 0;
+
+      for (auto* operand : instruction->operands()) {
+        ShapeUtil::ForEachSubshape(
+            operand->shape(),
+            [&input_size](const Shape& subshape, const ShapeIndex& /*index*/) {
+              if (ShapeUtil::IsArray(subshape)) {
+                input_size += ShapeUtil::ByteSizeOfElements(subshape);
+              }
+            });
+      }
+      ShapeUtil::ForEachSubshape(
+          instruction->shape(),
+          [&output_size](const Shape& subshape, const ShapeIndex& /*index*/) {
+            if (ShapeUtil::IsArray(subshape)) {
+              output_size += ShapeUtil::ByteSizeOfElements(subshape);
+            }
+          });
+
+      if (output_size > input_size) {
+        continue;
+      }
+    }
+
     auto is_invariant = [&](HloInstruction* op) {
       return hoisted_instructions.find(op) != hoisted_instructions.end() ||
              unhoisted_invariant_instructions.count(op) ||
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h
index 3031899f71..bd6232dc0a 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h
@@ -34,8 +34,14 @@ class WhileLoopInvariantCodeMotion : public HloModulePass {
   // Setting `hoist_constants` to false can be help if LICM is run in the mid
   // level HLO pipeline because hoisting constants out of while loop bodies can
   // break optimizations like constant folding.
-  explicit WhileLoopInvariantCodeMotion(bool hoist_constants = false)
-      : hoist_constants_(hoist_constants) {}
+  // Setting `hoist_size_inflating_ops` to false will forbid hoisting
+  // instructions where the size of the output(s) is larger than the size of the
+  // input(s). This is useful on platforms on which it's important to prevent
+  // blow-ups in memory size.
+  explicit WhileLoopInvariantCodeMotion(bool hoist_constants = false,
+                                        bool hoist_size_inflating_ops = true)
+      : hoist_constants_(hoist_constants),
+        hoist_size_inflating_ops_(hoist_size_inflating_ops) {}
   ~WhileLoopInvariantCodeMotion() override = default;
 
   absl::string_view name() const override {
@@ -49,6 +55,7 @@ class WhileLoopInvariantCodeMotion : public HloModulePass {
       HloInstruction* while_instr);
 
   bool hoist_constants_;
+  bool hoist_size_inflating_ops_;
 };
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
index 046ccb2d3f..8e7c4bc882 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
@@ -570,5 +570,59 @@ TEST_F(WhileLoopInvariantCodeMotionTest, DoNotHoistOutOfSingleIteration) {
   EXPECT_FALSE(simplified_loop);
 }
 
+const char* const kInflatingTestCase = R"(
+HloModule ModuleWithWhile
+
+mul {
+  lhs = f32[] parameter(0)
+  rhs = f32[] parameter(1)
+  ROOT mul = f32[] multiply(lhs, rhs)
+}
+
+body {
+  p_body = (f32[]) parameter(0)
+  iota = f32[1024, 1024] iota(), iota_dimension=0
+  add = f32[1024, 1024] add(iota, iota)
+  constant = f32[] constant(1.0)
+  reduce = f32[] reduce(f32[1024, 1024] add, f32[] constant), dimensions={0,1}, to_apply=mul
+  ROOT root = (f32[]) tuple(reduce)
+}
+
+condition {
+  p_cond = (f32[]) parameter(0)
+  ROOT result = pred[] constant(true)
+}
+
+ENTRY entry {
+  param = f32[] parameter(0)
+  while_init = (f32[]) tuple(param)
+  ROOT while = (f32[]) while(while_init), condition=condition, body=body
+}
+)";
+
+TEST_F(WhileLoopInvariantCodeMotionTest, HoistsInflatingByDefault) {
+  auto m = ParseAndReturnVerifiedModule(kInflatingTestCase).ValueOrDie();
+
+  TF_ASSERT_OK_AND_ASSIGN(
+      bool simplified_loop,
+      WhileLoopInvariantCodeMotion(/*hoist_constants=*/true).Run(m.get()));
+  EXPECT_TRUE(simplified_loop);
+
+  HloComputation* while_body = m->GetComputationWithName("wide.body");
+  ASSERT_NE(while_body, nullptr);
+  EXPECT_THAT(while_body->instructions(), Not(Contains(op::Iota())));
+}
+
+TEST_F(WhileLoopInvariantCodeMotionTest, NoHoistInflating) {
+  auto m = ParseAndReturnVerifiedModule(kInflatingTestCase).ValueOrDie();
+
+  TF_ASSERT_OK_AND_ASSIGN(
+      bool simplified_loop,
+      WhileLoopInvariantCodeMotion(/*hoist_constants=*/true,
+                                   /*hoist_size_inflating_ops=*/false)
+          .Run(m.get()));
+  EXPECT_FALSE(simplified_loop);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 5822f97659aa3a19231f43fecb90546d3b83f680 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 16 Nov 2018 07:31:27 -0800
Subject: [PATCH 0395/1554] Automated rollback of commit
 582672a1c206238252dc7703bc5928f054dd6c15

PiperOrigin-RevId: 221791370
---
 tensorflow/core/framework/tensor.h       | 91 +++++++++++++++++-------
 tensorflow/core/framework/tensor_test.cc | 11 +--
 2 files changed, 71 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 8a5633983b..3177bbe7e9 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -119,7 +120,7 @@ class Tensor {
 
   class HostScalarTensorBufferBase;
   template <typename T>
-  class HostScalarTensorBuffer;
+  struct ValueAndTensorBuffer;
 
   // Creates a tensor with the given scalar `value` in CPU memory.
   template <typename T>
@@ -876,40 +877,76 @@ class Tensor::HostScalarTensorBufferBase : public TensorBuffer {
   void FillAllocationDescription(AllocationDescription* proto) const final;
 };
 
-// `Tensor::HostScalarTensorBuffer<T>` is a specialized `TensorBuffer`
-// implementation for storing a single scalar value.
-//
-// TODO(mrry): Evaluate other compilers or approaches to aligning the value
-// so that it can be used directly as a tensor value. For example, in a C++17
-// future, we could use `alignas(EIGEN_MAX_ALIGN_BYTES)` to store the value
-// inline in this object to save an allocation. However, this is not currently
-// widely supported in our compilers.
+// A packed representation for a single scalar value of type `T`, and a
+// `TensorBuffer` implementation that describes (and manages the lifetime of)
+// that value.
 template <typename T>
-class Tensor::HostScalarTensorBuffer : public HostScalarTensorBufferBase {
- public:
-  HostScalarTensorBuffer(T&& value)
-      : data_(reinterpret_cast<T*>(cpu_allocator()->AllocateRaw(
-            EIGEN_MAX_ALIGN_BYTES, sizeof(value)))) {
-    if (is_simple_type<T>::value) {
-      *data_ = value;
-    } else {
-      new (data_) T(std::move(value));
+struct Tensor::ValueAndTensorBuffer {
+  class HostScalarTensorBuffer : public Tensor::HostScalarTensorBufferBase {
+   public:
+    HostScalarTensorBuffer(void* data) : data_(data) {}
+    void* data() const final { return const_cast<void*>(data_); }
+    size_t size() const final { return sizeof(T); }
+    TensorBuffer* root_buffer() final { return this; }
+
+    // Override `operator delete` so that calling `delete this` in
+    // `core::Refcounted::Unref()` for an object of this type will free
+    // the enclosing `ValueAndTensorBuffer` for the tensor buffer.
+    //
+    // NOTE(mrry): The definition of this method must be outside the class
+    // definition in order to satisfy some compilers.
+    static void operator delete(void* ptr);
+
+    static void operator delete(void*, void*) {
+      // Some compilers require an overridden class-specific deallocation
+      // function, which will be called if placement `new` throws an
+      // exception.
     }
-  }
-  ~HostScalarTensorBuffer() { cpu_allocator()->Deallocate(data_, 1); }
-  void* data() const final { return const_cast<T*>(data_); }
-  size_t size() const final { return sizeof(*data_); }
-  TensorBuffer* root_buffer() final { return this; }
 
- private:
-  T* const data_;
+   private:
+    ~HostScalarTensorBuffer() override { static_cast<T*>(data_)->~T(); }
+    void* const data_;
+  };
+
+  T value;
+  HostScalarTensorBuffer tensor_buffer;
 };
 
+/* static */
 template <typename T>
-Tensor::Tensor(T value, host_scalar_tag tag)
-    : buf_(new HostScalarTensorBuffer<T>(std::move(value))) {
+void Tensor::ValueAndTensorBuffer<T>::HostScalarTensorBuffer::operator delete(
+    void* ptr) {
+  // Use a dummy object to compute to offset of
+  // `ValueAndTensorBuffer::tensor_buffer`, because `offsetof()` is not
+  // necessarily defined on this non-POD type (until C++17).
+  //
+  // NOTE(mrry): Using `sizeof(Tensor::ValueAndTensorBuffer<T>)` here requires
+  // us to define this method outside the class definition, so that it is not
+  // considered an incomplete type.
+  typename std::aligned_storage<sizeof(Tensor::ValueAndTensorBuffer<T>),
+                                alignof(Tensor::ValueAndTensorBuffer<T>)>::type
+      dummy_storage_;
+  Tensor::ValueAndTensorBuffer<T>* dummy_object =
+      reinterpret_cast<Tensor::ValueAndTensorBuffer<T>*>(&dummy_storage_);
+  intptr_t offset = reinterpret_cast<intptr_t>(&dummy_object->tensor_buffer) -
+                    reinterpret_cast<intptr_t>(dummy_object);
+
+  port::AlignedFree(static_cast<char*>(ptr) - offset);
+}
+
+template <typename T>
+Tensor::Tensor(T value, host_scalar_tag tag) {
+  auto* value_and_buf = static_cast<Tensor::ValueAndTensorBuffer<T>*>(
+      port::AlignedMalloc(sizeof(typename Tensor::ValueAndTensorBuffer<T>),
+                          EIGEN_MAX_ALIGN_BYTES));
+  new (&value_and_buf->value) T(std::move(value));
+  new (&value_and_buf->tensor_buffer)
+      typename Tensor::ValueAndTensorBuffer<T>::HostScalarTensorBuffer(
+          value_and_buf);
+  buf_ = &value_and_buf->tensor_buffer;
   set_dtype(DataTypeToEnum<T>::value);
 }
+
 inline Tensor& Tensor::operator=(Tensor&& other) {
   // Avoid self-assignment, since we might destroy our underlying buffer.
   if (&other != this) {
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 925ebc4945..4fa9d1df67 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -854,15 +854,18 @@ TEST(Tensor_HostScalar, Basics) {
     EXPECT_FLOAT_EQ(42.0f, Tt());
   }
   {
-    Tensor t("foo");
+    // NOTE(mrry): Use long enough strings so that the contents are dynamically
+    // allocated, and the absence of a call to the string destructor would
+    // cause a memory leak.
+    Tensor t("fooooooooooooooooooooooooooooooooooooo");
     EXPECT_EQ(DT_STRING, t.dtype());
     EXPECT_EQ(1, t.NumElements());
     auto Tt = t.scalar<string>();
     EXPECT_EQ(1, Tt.size());
     EXPECT_EQ(0, Tt.rank());
-    EXPECT_EQ("foo", Tt());
-    Tt() = "bar";
-    EXPECT_EQ("bar", Tt());
+    EXPECT_EQ("fooooooooooooooooooooooooooooooooooooo", Tt());
+    Tt() = "baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar";
+    EXPECT_EQ("baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar", Tt());
   }
 }
 
-- 
GitLab


From 1480288eecc3ff13af5951c0bb79d56aca2f83f7 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 16 Nov 2018 07:55:34 -0800
Subject: [PATCH 0396/1554] Remove the tf.app module in v2. Users interested in
 command line flag parsing should use the standard argparse module.

PiperOrigin-RevId: 221793880
---
 tensorflow/api_template.__init__.py                      | 3 ++-
 tensorflow/python/platform/app.py                        | 2 +-
 tensorflow/python/tools/api/generator/api_init_files.bzl | 1 -
 tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt      | 7 -------
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt          | 4 ----
 5 files changed, 3 insertions(+), 14 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt

diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index 0d49756838..2efb8846c6 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -34,7 +34,8 @@ from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-
 
 # Make sure directory containing top level submodules is in
 # the __path__ so that "from tensorflow.foo import bar" works.
-_tf_api_dir = _os.path.dirname(_os.path.dirname(app.__file__))  # pylint: disable=undefined-variable
+# We're using bitwise, but there's nothing special about that.
+_tf_api_dir = _os.path.dirname(_os.path.dirname(bitwise.__file__))  # pylint: disable=undefined-variable
 if _tf_api_dir not in __path__:
   __path__.append(_tf_api_dir)
 
diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py
index 4c91bc3652..7b917235c0 100644
--- a/tensorflow/python/platform/app.py
+++ b/tensorflow/python/platform/app.py
@@ -108,7 +108,7 @@ def _define_help_flags():
     _define_help_flags_called = True
 
 
-@tf_export('app.run')
+@tf_export(v1=['app.run'])
 def run(main=None, argv=None):
   """Runs the program with an optional 'main' function and 'argv' list."""
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 1097455734..8ce9038d3a 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -4,7 +4,6 @@
 TENSORFLOW_API_INIT_FILES = [
     # BEGIN GENERATED FILES
     "__init__.py",
-    "app/__init__.py",
     "bitwise/__init__.py",
     "compat/__init__.py",
     "data/__init__.py",
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt
deleted file mode 100644
index 67e1b76cab..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.app.pbtxt
+++ /dev/null
@@ -1,7 +0,0 @@
-path: "tensorflow.app"
-tf_module {
-  member_method {
-    name: "run"
-    argspec: "args=[\'main\', \'argv\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a1e7b3231f..c5181ac1ac 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -144,10 +144,6 @@ tf_module {
     name: "VariableSynchronization"
     mtype: "<class \'enum.EnumMeta\'>"
   }
-  member {
-    name: "app"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "bfloat16"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
-- 
GitLab


From 4f8c575890a63def2bf667b61d85f1c2ff2ef91b Mon Sep 17 00:00:00 2001
From: Greg Billock <gbillock@google.com>
Date: Fri, 16 Nov 2018 08:05:28 -0800
Subject: [PATCH 0397/1554] Update unicode_decode_with_splits type to int32

PiperOrigin-RevId: 221795193
---
 tensorflow/core/kernels/unicode_ops.cc        |  4 +--
 .../core/ops/compat/ops_history.v1.pbtxt      |  2 +-
 tensorflow/core/ops/string_ops.cc             |  2 +-
 .../kernel_tests/unicode_decode_op_test.py    | 33 ++++++++-----------
 4 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc
index fde579dd9f..eb0c1d1228 100644
--- a/tensorflow/core/kernels/unicode_ops.cc
+++ b/tensorflow/core/kernels/unicode_ops.cc
@@ -425,12 +425,12 @@ class UnicodeDecodeWithOffsetsOp : public OpKernel {
         ctx, ctx->allocate_output("char_to_byte_starts",
                                   {static_cast<int64>(offset_values.size())},
                                   &output_offset_values));
-    auto out_char_values = output_char_values->vec<uint32>();
+    auto out_char_values = output_char_values->vec<int32>();
     auto out_offset_values = output_offset_values->vec<int64>();
 
     // Load output tensors from intermediate value arrays.
     for (int i = 0; i < char_values.size(); ++i) {
-      out_char_values(i) = static_cast<uint32>(char_values[i]);
+      out_char_values(i) = static_cast<int32>(char_values[i]);
       out_offset_values(i) = offset_values[i];
     }
   }
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 13416c179a..803e4b434b 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77022,7 +77022,7 @@ op {
   }
   output_arg {
     name: "char_values"
-    type: DT_UINT32
+    type: DT_INT32
   }
   output_arg {
     name: "char_to_byte_starts"
diff --git a/tensorflow/core/ops/string_ops.cc b/tensorflow/core/ops/string_ops.cc
index 8fe8e5db52..fbecff11df 100644
--- a/tensorflow/core/ops/string_ops.cc
+++ b/tensorflow/core/ops/string_ops.cc
@@ -263,7 +263,7 @@ REGISTER_OP("UnicodeTranscode")
 REGISTER_OP("UnicodeDecodeWithOffsets")
     .Input("input: string")
     .Output("row_splits: int64")
-    .Output("char_values: uint32")
+    .Output("char_values: int32")
     .Output("char_to_byte_starts: int64")
     .Attr("input_encoding: string")
     .Attr("errors: {'strict', 'replace', 'ignore'} = 'replace'")
diff --git a/tensorflow/python/kernel_tests/unicode_decode_op_test.py b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
index 9401e77adf..34dae9b731 100644
--- a/tensorflow/python/kernel_tests/unicode_decode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
@@ -37,7 +37,7 @@ class UnicodeDecodeTest(test.TestCase):
 
   def testBatchDecode(self):
     text = constant_op.constant(
-        ["仅今年前", "中国进出口银行与中国银行加强合作"])
+        ["仅今年前", "分享介面終於迎來更新"])
     row_splits, utf8_text, offsets = gen_string_ops.unicode_decode_with_offsets(
         text, "utf-8")
 
@@ -47,27 +47,20 @@ class UnicodeDecodeTest(test.TestCase):
           codepoint("今"),
           codepoint("年"),
           codepoint("前"),
-          codepoint("中"),
-          codepoint("国"),
-          codepoint("进"),
-          codepoint("出"),
-          codepoint("口"),
-          codepoint("银"),
-          codepoint("行"),
-          codepoint("与"),
-          codepoint("中"),
-          codepoint("国"),
-          codepoint("银"),
-          codepoint("行"),
-          codepoint("加"),
-          codepoint("强"),
-          codepoint("合"),
-          codepoint("作")
+          codepoint("分"),
+          codepoint("享"),
+          codepoint("介"),
+          codepoint("面"),
+          codepoint("終"),
+          codepoint("於"),
+          codepoint("迎"),
+          codepoint("來"),
+          codepoint("更"),
+          codepoint("新")
       ],
                           utf8_text.eval().tolist())
-      self.assertAllEqual([0, 4, 20], row_splits.eval().tolist())
-      self.assertAllEqual([0, 3, 6, 9, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30,
-                           33, 36, 39, 42, 45],
+      self.assertAllEqual([0, 4, 14], row_splits.eval().tolist())
+      self.assertAllEqual([0, 3, 6, 9, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27],
                           offsets.eval().tolist())
 
   def testBasicDecodeWithOffset(self):
-- 
GitLab


From f0713a4169cfeb22d96ccb119f8200573d4360ca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 08:19:48 -0800
Subject: [PATCH 0398/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 221796820
---
 tensorflow/core/ops/ops.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index ac83059aec..f9474d4a16 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36955,7 +36955,7 @@ op {
   }
   output_arg {
     name: "char_values"
-    type: DT_UINT32
+    type: DT_INT32
   }
   output_arg {
     name: "char_to_byte_starts"
-- 
GitLab


From f91d1fc2dd43ce022d550d47f6639fdb2d3f9f30 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 08:59:38 -0800
Subject: [PATCH 0399/1554] Make the error thrown by PassThroughAllStages more
 informative by including the node name.

PiperOrigin-RevId: 221801636
---
 tensorflow/core/grappler/optimizers/graph_optimizer_stage.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
index f31a30ec0e..99fcb31523 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
@@ -239,7 +239,8 @@ class GraphOptimizerStagePipeline {
         // case of any error it must leave optimized graph unmodified.
         if (!stage_status.ok()) {
           LOG(WARNING) << "Failed to run optimizer " << stage->optimizer_name()
-                       << ", stage " << stage->stage_name()
+                       << ", stage " << stage->stage_name() << " node "
+                       << node->name()
                        << ". Error: " << stage_status.error_message();
         }
         if (break_predicate_(*result)) return true;
-- 
GitLab


From b6a23fd293902065b020d75baf2606c1c6f00e08 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 16 Nov 2018 09:09:28 -0800
Subject: [PATCH 0400/1554] Change API for tf.random.multinomial in TF2.0.

PiperOrigin-RevId: 221803100
---
 tensorflow/python/ops/random_ops.py           | 49 ++++++++++++++++---
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 --
 .../api/golden/v2/tensorflow.random.pbtxt     |  8 +--
 .../tools/compatibility/tf_upgrade_v2.py      | 19 +++++--
 .../tools/compatibility/tf_upgrade_v2_test.py | 21 ++++++++
 5 files changed, 82 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py
index 1f7db0af61..9c33ef3407 100644
--- a/tensorflow/python/ops/random_ops.py
+++ b/tensorflow/python/ops/random_ops.py
@@ -325,7 +325,9 @@ def random_crop(value, size, seed=None, name=None):
     return array_ops.slice(value, offset, size, name=name)
 
 
-@tf_export("random.multinomial", "multinomial")
+@tf_export(v1=["random.multinomial", "multinomial"])
+@deprecation.deprecated(
+    date=None, instructions="Use tf.random.categorical instead.")
 def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None):
   """Draws samples from a multinomial distribution.
 
@@ -342,9 +344,7 @@ def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None):
       `[i, :]` represents the unnormalized log-probabilities for all classes.
     num_samples: 0-D.  Number of independent samples to draw for each row slice.
     seed: A Python integer. Used to create a random seed for the distribution.
-      See
-      `tf.set_random_seed`
-      for behavior.
+      See `tf.set_random_seed` for behavior.
     name: Optional name for the operation.
     output_dtype: integer type to use for the output. Defaults to int64.
 
@@ -352,10 +352,43 @@ def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None):
     The drawn samples of shape `[batch_size, num_samples]`.
   """
   with ops.name_scope(name, "multinomial", [logits]):
-    logits = ops.convert_to_tensor(logits, name="logits")
-    seed1, seed2 = random_seed.get_seed(seed)
-    return gen_random_ops.multinomial(
-        logits, num_samples, seed=seed1, seed2=seed2, output_dtype=output_dtype)
+    return multinomial_categorical_impl(logits, num_samples, output_dtype, seed)
+
+
+@tf_export("random.categorical", v1=[])
+def categorical(logits, num_samples, dtype=None, seed=None, name=None):
+  """Draws samples from a multinomial distribution.
+
+  Example:
+
+  ```python
+  # samples has shape [1, 5], where each value is either 0 or 1 with equal
+  # probability.
+  samples = tf.random.categorical(tf.log([[10., 10.]]), 5)
+  ```
+
+  Args:
+    logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice
+      `[i, :]` represents the unnormalized log-probabilities for all classes.
+    num_samples: 0-D.  Number of independent samples to draw for each row slice.
+    dtype: integer type to use for the output. Defaults to int64.
+    seed: A Python integer. Used to create a random seed for the distribution.
+      See `tf.set_random_seed` for behavior.
+    name: Optional name for the operation.
+
+  Returns:
+    The drawn samples of shape `[batch_size, num_samples]`.
+  """
+  with ops.name_scope(name, "categorical", [logits]):
+    return multinomial_categorical_impl(logits, num_samples, dtype, seed)
+
+
+def multinomial_categorical_impl(logits, num_samples, dtype, seed):
+  """Implementation for random.multinomial (v1) and random.categorical (v2)."""
+  logits = ops.convert_to_tensor(logits, name="logits")
+  seed1, seed2 = random_seed.get_seed(seed)
+  return gen_random_ops.multinomial(
+      logits, num_samples, seed=seed1, seed2=seed2, output_dtype=dtype)
 
 
 ops.NotDifferentiable("Multinomial")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index c5181ac1ac..eba7b39561 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -836,10 +836,6 @@ tf_module {
     name: "mod"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "multinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "multiply"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index bfc38fbf3c..a9dd64383d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.random"
 tf_module {
+  member_method {
+    name: "categorical"
+    argspec: "args=[\'logits\', \'num_samples\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
   member_method {
     name: "gamma"
     argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
@@ -8,10 +12,6 @@ tf_module {
     name: "log_uniform_candidate_sampler"
     argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "multinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "normal"
     argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index e83e771001..9cd00e6994 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -51,6 +51,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
         },
+        "tf.multinomial": {
+            "output_dtype": "dtype",
+        },
+        "tf.random.multinomial": {
+            "output_dtype": "dtype",
+        },
         "tf.nn.conv3d": {
             "filter": "filters"
         },
@@ -68,6 +74,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.symbol_renames = renames_v2.renames
     # pylint: disable=line-too-long
     # Add additional renames not in renames_v2.py here.
+    # IMPORTANT: For the renames in here, if you also need to add to
+    # function_reorders or function_keyword_renames, use the OLD function name.
+    # These renames happen after the arguments have been processed.
     self.symbol_renames.update({
         "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
         "tf.contrib.data.Counter": "tf.data.experimental.Counter",
@@ -113,6 +122,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.contrib.data.unique": "tf.data.experimental.unique",
         "tf.quantize_v2": "tf.quantization.quantize",
         "tf.sparse_concat": "tf.sparse.concat",
+        "tf.multinomial": "tf.random.categorical",
+        "tf.random.multinomial": "tf.random.categorical",
         "tf.load_file_system_library": "tf.load_library",
     })
     # pylint: enable=line-too-long
@@ -143,9 +154,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
         ],
-        "tf.nn.separable_conv2d": [
-            "input", "depthwise_filter", "pointwise_filter", "strides",
-            "padding", "data_format", "dilations", "name"
+        "tf.multinomial": [
+            "logits", "num_samples", "seed", "name", "output_dtype"
+        ],
+        "tf.random.multinomial": [
+            "logits", "num_samples", "seed", "name", "output_dtype"
         ],
         "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
         "tf.quantize_v2": [
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b5e3f1907e..7baa1cafdd 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -128,6 +128,27 @@ class TestUpgrade(test_util.TensorFlowTestCase):
         )
     self.assertEqual(new_text, expected_text)
 
+  def testRandomMultinomialToRandomCategorical(self):
+    text = (
+        "tf.random.multinomial(logits, samples, seed, name, output_dtype)\n"
+        )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    expected_text = (
+        "tf.random.categorical(logits=logits, num_samples=samples, seed=seed, "
+        "name=name, dtype=output_dtype)\n"
+        )
+    self.assertEqual(new_text, expected_text)
+
+    text = (
+        "tf.multinomial(logits, samples, seed, name, output_dtype)\n"
+        )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    expected_text = (
+        "tf.random.categorical(logits=logits, num_samples=samples, seed=seed, "
+        "name=name, dtype=output_dtype)\n"
+        )
+    self.assertEqual(new_text, expected_text)
+
   def testConvolutionOpUpdate(self):
     text = (
         "tf.nn.convolution(input, filter, padding, strides, dilation_rate, "
-- 
GitLab


From e510289890c19b4ebab990e101d918b4c37029a7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 09:19:48 -0800
Subject: [PATCH 0401/1554] import [0] shapes as they are, instead of mapping
 to [].

PiperOrigin-RevId: 221804264
---
 tensorflow/lite/toco/import_tensorflow.cc      | 5 +----
 tensorflow/lite/toco/import_tensorflow_test.cc | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 96f3c6a6ab..535aef05b1 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -219,10 +219,7 @@ tensorflow::Status ImportShape(
   // allocation code gets a bit confused. It seems that the code expects an
   // empty shape for zero-sized shapes, so we will do just that, except for the
   // [0] case.
-  // TODO(b/119325030): In order to correctly import the "scalar" shapes the
-  // following test must include "&& input_dims_only_sizes.size() > 1", but
-  // that seems to slow everything down a lot.
-  if (zero_sized_shape) {
+  if (zero_sized_shape && input_dims_only_sizes.size() > 1) {
     shape->mutable_dims()->clear();
     if (input_flat_size != nullptr) *input_flat_size = 0;
     return tensorflow::Status::OK();
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index 0be358b1f7..07b52d3970 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -190,9 +190,7 @@ TEST_P(ShapeImportTest, ShapeIsOneDimZero) {
   EXPECT_TRUE(ImportNode(node, &model).ok());
 
   const auto& array = model.GetArray("Node1");
-  // We would like to have [0] shapes actually import correctly, but
-  // for some reason that slows everything down.
-  EXPECT_THAT(array.shape().dims(), ::testing::ElementsAre());
+  EXPECT_THAT(array.shape().dims(), ::testing::ElementsAre(0));
 }
 
 TEST_P(ShapeImportTest, ShapeElementTooLarge) {
-- 
GitLab


From 5c4fe7b6680df500cf129306e08b86ff3128c5f4 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Fri, 16 Nov 2018 09:48:36 -0800
Subject: [PATCH 0402/1554] Remove dead code.

USE_C_SHAPES has been forced True for a while now
(June: commit 1d74a69443f741e69f9f52cb6bc2940b4d4ae3b7
August: commit a473f435cf7345cb9dc2efacb471a7f318141a9b)
PiperOrigin-RevId: 221808148
---
 tensorflow/python/BUILD                  |   1 -
 tensorflow/python/framework/importer.py  |  15 ++-
 tensorflow/python/framework/ops.py       | 115 ++---------------------
 tensorflow/python/framework/test_util.py |  47 +--------
 4 files changed, 17 insertions(+), 161 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e55b2a0e92..e81ed2f73a 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -853,7 +853,6 @@ py_library(
     deps = [
         ":c_api_util",
         ":control_flow_util",
-        ":cpp_shape_inference_proto_py",
         ":device",
         ":dtypes",
         ":error_interpolation",
diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index c9ac27e788..71ebfd6ceb 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -431,17 +431,16 @@ def import_graph_def(graph_def,
     #
     # TODO(skyewm): fetch the TF_Functions directly from the TF_Graph
     # TODO(skyewm): avoid sending serialized FunctionDefs back to the TF_Graph
-    # TODO(b/74620627): move this after _ProcessNewOps outside the lock once
-    # _USE_C_SHAPES is removed.
-    if graph_def.library and graph_def.library.function:
-      # pylint: disable=protected-access
-      functions = function._from_library(graph_def.library)
-      for f in functions:
-        f.add_to_graph(graph)
-      # pylint: enable=protected-access
 
     _ProcessNewOps(graph)
 
+  if graph_def.library and graph_def.library.function:
+    # pylint: disable=protected-access
+    functions = function._from_library(graph_def.library)
+    for f in functions:
+      f.add_to_graph(graph)
+    # pylint: enable=protected-access
+
   # Treat input mappings that don't appear in the graph as an error, because
   # they are likely to be due to a typo.
   missing_unused_input_keys = (
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 11a73eb11d..ee7323fc9d 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -40,7 +40,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
-from tensorflow.python.framework import cpp_shape_inference_pb2
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import error_interpolation
@@ -318,22 +317,13 @@ class Tensor(_TensorLike):
     self._op = op
     self._value_index = value_index
     self._dtype = dtypes.as_dtype(dtype)
-
     # This will be set by self._as_tf_output().
     self._tf_output = None
-
     # This will be set by self.shape().
     self._shape_val = None
-
     # List of operations that use this Tensor as input.  We maintain this list
     # to easily navigate a computation graph.
     self._consumers = []
-
-    if not _USE_C_SHAPES:
-      # Attributes used for C++ shape inference. Not inspected, only forwarded.
-      # If set, will be a HandleData object from cpp_shape_inference.proto.
-      self._handle_data = None
-
     self._id = uid()
 
   @property
@@ -408,17 +398,7 @@ class Tensor(_TensorLike):
 
     """
     if self._shape_val is None:
-      if _USE_C_SHAPES:
-        self._shape_val = self._c_api_shape()
-      else:
-        # Call set_shape_and_handle_data_for_outputs in topological order on all
-        # ops that are needed to compute self.op's shape. We do this instead of
-        # having set_shape_and_handle_data_for_outputs recursively call
-        # Operation.shape on self.op.inputs to overflowing the call stack.
-        need_shapes = self._get_input_ops_without_shapes(self.op)
-        need_shapes.sort(key=lambda op: op._id)
-        for op in need_shapes:
-          set_shape_and_handle_data_for_outputs(op)
+      self._shape_val = self._c_api_shape()
     return self._shape_val
 
   def _get_input_ops_without_shapes(self, target_op):
@@ -533,14 +513,10 @@ class Tensor(_TensorLike):
       ValueError: If `shape` is not compatible with the current shape of
         this tensor.
     """
-    if _USE_C_SHAPES:  # pylint: disable=protected-access
-      # Reset cached shape.
-      self._shape_val = None
-    else:
-      self._shape_val = self.shape.merge_with(shape)
+    # Reset cached shape.
+    self._shape_val = None
 
-    # Update C shape even if _USE_C_SHAPES = False, since we still want
-    # set_shape to be reflected in the C API graph for when we run it.
+    # We want set_shape to be reflected in the C API graph for when we run it.
     if not isinstance(shape, tensor_shape.TensorShape):
       shape = tensor_shape.TensorShape(shape)
     dim_list = []
@@ -634,10 +610,7 @@ class Tensor(_TensorLike):
     return id(self) == id(other)
 
   def __copy__(self):
-    # Make sure _shape_val is computed before we copy.
     # TODO(b/77597810): get rid of Tensor copies.
-    if self._shape_val is None:
-      set_shape_and_handle_data_for_outputs(self.op)
     cls = self.__class__
     result = cls.__new__(cls)
     result.__dict__.update(self.__dict__)
@@ -2135,12 +2108,6 @@ class Operation(object):
       raise TypeError("tensor must be a Tensor: %s" % tensor)
     _assert_same_graph(self, tensor)
 
-    # Make sure output shapes are already computed for this op in case we create
-    # a cycle (we cannot compute shapes for cycles). Usually shapes are computed
-    # lazily upon request.
-    if not _USE_C_SHAPES:
-      set_shape_and_handle_data_for_outputs(self)
-
     # Reset cached inputs.
     self._inputs_val = None
     c_api.UpdateEdge(
@@ -2614,72 +2581,9 @@ class RegisterShape(object):
     return f
 
 
-# TODO(b/74620627): remove when _USE_C_SHAPES is removed
-def _set_shape_and_handle_data_for_outputs_c_api(op):
-  """Set shapes and resource handle data using info from the C API."""
-  assert not _USE_C_SHAPES
-  for output in op.outputs:
-    output._shape_val = output._c_api_shape()
-    # Set the resource handle data for compatibility with the Python shape
-    # inference code.
-    serialized = c_api.GetHandleShapeAndType(op._graph._c_graph,  # pylint: disable=protected-access
-                                             output._as_tf_output())
-    if serialized:
-      output._handle_data = (
-          cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData
-          .FromString(compat.as_bytes(serialized)))
-    else:
-      output._handle_data = None
-
-
-# TODO(b/74620627): remove when _USE_C_SHAPES is removed
-def set_shape_and_handle_data_for_outputs(op):
-  """Set the shapes and resource handle data for op's outputs.
-
-  When _USE_C_SHAPES = False, this is lazily called when a tensor's shape is
-  first requested. Usually this should work automatically, but some edge cases
-  may require manually calling this first to make sure Tensor._shape_val and
-  Tensor._handle_data are set (e.g. manually overriding _handle_data, copying a
-  Tensor).
-  """
-  if _USE_C_SHAPES: return
-
-  if op.graph._is_function(op.type):
-    for output in op.outputs:
-      output._shape_val = tensor_shape.unknown_shape()
-    return
-
-  try:
-    shape_func = _shape_registry.lookup(op.type)
-  except LookupError:
-    try:
-      shape_func = _default_shape_function_registry.lookup(op.type)
-    except LookupError:
-      shape_func = _call_cpp_shape_fn_and_require_op
-
-  shapes = shape_func(op)
-  if shapes is None:
-    raise RuntimeError(
-        "Shape function for op %s did not return any shapes" % op)
-  elif isinstance(shapes, dict):
-    # Returned by call_cpp_shape_fn
-    shapes_dict = shapes
-    shapes = shapes_dict["shapes"]
-    handle_datas = shapes_dict["handle_data"]
-    for output, handle_data in zip(op.outputs, handle_datas):
-      # Don't override any existing handle data that may have been manually set.
-      # pylint: disable=protected-access
-      if output._handle_data is None:
-        output._handle_data = handle_data
-      # pylint: enable=protected-access
-
-  if len(op.outputs) != len(shapes):
-    raise RuntimeError(
-        "Shape function for op %s returned %d shapes but expected %d %s %s" %
-        (op, len(shapes), len(op.outputs), shape_func.__name__, str(shapes)))
-  for output, s in zip(op.outputs, shapes):
-    output._shape_val = tensor_shape.unknown_shape()
-    output._shape_val = output._shape_val.merge_with(s)
+def set_shape_and_handle_data_for_outputs(_):
+  """No op. TODO(b/74620627): Remove this."""
+  pass
 
 
 class OpStats(object):
@@ -3532,11 +3436,6 @@ class Graph(object):
 
     # pylint: disable=protected-access
     for op in new_ops:
-      # Operations created by the C API always retrieve shapes from the C API so
-      # we preserve the shapes of ops created in import_graph_def (from the
-      # "_output_shapes" attr of the imported NodeDef).
-      if not _USE_C_SHAPES:
-        _set_shape_and_handle_data_for_outputs_c_api(op)
       new_control_inputs = self._control_dependencies_for_inputs(op.inputs)
       op._add_control_inputs(new_control_inputs)
       op._control_flow_post_processing()
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 0a08f3b5bd..39c8ba152d 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -378,53 +378,12 @@ def skip_if(condition):
 
 
 def enable_c_shapes(fn):
-  """Decorator for enabling C shapes on a test.
-
-  Note this enables the C shapes after running the test class's setup/teardown
-  methods.
-
-  Args:
-    fn: the function to be wrapped
-
-  Returns:
-    The wrapped function
-  """
-
-  # pylint: disable=protected-access
-  def wrapper(*args, **kwargs):
-    prev_value = ops._USE_C_SHAPES
-    ops._USE_C_SHAPES = True
-    try:
-      fn(*args, **kwargs)
-    finally:
-      ops._USE_C_SHAPES = prev_value
-
-  # pylint: enable=protected-access
-
-  return wrapper
+  """No-op. TODO(b/74620627): Remove this."""
+  return fn
 
 
 def with_c_shapes(cls):
-  """Adds methods that call original methods but with C API shapes enabled.
-
-  Note this enables C shapes in new methods after running the test class's
-  setup method.
-
-  Args:
-    cls: class to decorate
-
-  Returns:
-    cls with new test methods added
-  """
-  # If C shapes are already enabled, don't do anything. Some tests break if the
-  # same test is run twice, so this allows us to turn on the C shapes by default
-  # without breaking these tests.
-  if ops._USE_C_SHAPES:
-    return cls
-
-  for name, value in cls.__dict__.copy().items():
-    if callable(value) and name.startswith("test"):
-      setattr(cls, name + "WithCShapes", enable_c_shapes(value))
+  """No-op. TODO(b/74620627): Remove this."""
   return cls
 
 
-- 
GitLab


From 28bb75b696bbfea014c2c9c62a09cad987720d8f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 09:55:23 -0800
Subject: [PATCH 0403/1554] Renamed maybe_saved_model_directory to
 contains_saved_model for TF 2.0

PiperOrigin-RevId: 221809031
---
 tensorflow/python/saved_model/loader_impl.py  | 28 ++++++++++++---
 .../golden/v1/tensorflow.saved_model.pbtxt    |  4 +++
 .../golden/v2/tensorflow.saved_model.pbtxt    |  8 ++---
 tensorflow/tools/compatibility/renames_v2.py  | 36 +++++++++++--------
 4 files changed, 53 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index 8c8eaf038a..601133ce90 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -145,12 +145,11 @@ def _get_main_op_tensor(
   return main_op_tensor
 
 
-@tf_export(
+@tf_export(v1=[
+    "saved_model.contains_saved_model",
     "saved_model.maybe_saved_model_directory",
-    v1=[
-        "saved_model.maybe_saved_model_directory",
-        "saved_model.loader.maybe_saved_model_directory"
-    ])
+    "saved_model.loader.maybe_saved_model_directory"
+])
 @deprecation.deprecated_endpoints(
     "saved_model.loader.maybe_saved_model_directory")
 def maybe_saved_model_directory(export_dir):
@@ -173,6 +172,25 @@ def maybe_saved_model_directory(export_dir):
   return file_io.file_exists(txt_path) or file_io.file_exists(pb_path)
 
 
+@tf_export("saved_model.contains_saved_model", v1=[])
+def contains_saved_model(export_dir):
+  """Checks whether the provided export directory could contain a SavedModel.
+
+  Note that the method does not load any data by itself. If the method returns
+  `false`, the export directory definitely does not contain a SavedModel. If the
+  method returns `true`, the export directory may contain a SavedModel but
+  provides no guarantee that it can be loaded.
+
+  Args:
+    export_dir: Absolute string path to possible export location. For example,
+                '/my/foo/model'.
+
+  Returns:
+    True if the export directory contains SavedModel files, False otherwise.
+  """
+  return maybe_saved_model_directory(export_dir)
+
+
 @tf_export(v1=["saved_model.load", "saved_model.loader.load"])
 @deprecation.deprecated(
     None,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt
index 2055bfbf06..3929003fa1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt
@@ -148,6 +148,10 @@ tf_module {
     name: "classification_signature_def"
     argspec: "args=[\'examples\', \'classes\', \'scores\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "contains_saved_model"
+    argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_tensor_from_tensor_info"
     argspec: "args=[\'tensor_info\', \'graph\', \'import_scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
index c83b569176..d946c666c2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
@@ -101,12 +101,12 @@ tf_module {
     argspec: "args=[\'examples\', \'classes\', \'scores\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
-    name: "is_valid_signature"
-    argspec: "args=[\'signature_def\'], varargs=None, keywords=None, defaults=None"
+    name: "contains_saved_model"
+    argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
-    name: "maybe_saved_model_directory"
-    argspec: "args=[\'export_dir\'], varargs=None, keywords=None, defaults=None"
+    name: "is_valid_signature"
+    argspec: "args=[\'signature_def\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "predict_signature_def"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5f9d33228b..c1a550ff3e 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -28,6 +28,10 @@ renames = {
     'tf.AUTO_REUSE': 'tf.compat.v1.AUTO_REUSE',
     'tf.COMPILER_VERSION': 'tf.version.COMPILER_VERSION',
     'tf.CXX11_ABI_FLAG': 'tf.sysconfig.CXX11_ABI_FLAG',
+    'tf.ConditionalAccumulator': 'tf.compat.v1.ConditionalAccumulator',
+    'tf.ConditionalAccumulatorBase': 'tf.compat.v1.ConditionalAccumulatorBase',
+    'tf.DeviceSpec': 'tf.compat.v1.DeviceSpec',
+    'tf.Dimension': 'tf.compat.v1.Dimension',
     'tf.FixedLenFeature': 'tf.io.FixedLenFeature',
     'tf.FixedLenSequenceFeature': 'tf.io.FixedLenSequenceFeature',
     'tf.FixedLengthRecordReader': 'tf.compat.v1.FixedLengthRecordReader',
@@ -64,21 +68,21 @@ renames = {
     'tf.add_to_collections': 'tf.compat.v1.add_to_collections',
     'tf.all_variables': 'tf.compat.v1.all_variables',
     'tf.angle': 'tf.math.angle',
-    'tf.assert_greater_equal': 'tf.debugging.assert_greater_equal',
-    'tf.assert_integer': 'tf.debugging.assert_integer',
-    'tf.assert_less_equal': 'tf.debugging.assert_less_equal',
-    'tf.assert_near': 'tf.debugging.assert_near',
-    'tf.assert_negative': 'tf.debugging.assert_negative',
-    'tf.assert_non_negative': 'tf.debugging.assert_non_negative',
-    'tf.assert_non_positive': 'tf.debugging.assert_non_positive',
-    'tf.assert_none_equal': 'tf.debugging.assert_none_equal',
-    'tf.assert_positive': 'tf.debugging.assert_positive',
+    'tf.assert_greater_equal': 'tf.compat.v1.assert_greater_equal',
+    'tf.assert_integer': 'tf.compat.v1.assert_integer',
+    'tf.assert_less_equal': 'tf.compat.v1.assert_less_equal',
+    'tf.assert_near': 'tf.compat.v1.assert_near',
+    'tf.assert_negative': 'tf.compat.v1.assert_negative',
+    'tf.assert_non_negative': 'tf.compat.v1.assert_non_negative',
+    'tf.assert_non_positive': 'tf.compat.v1.assert_non_positive',
+    'tf.assert_none_equal': 'tf.compat.v1.assert_none_equal',
+    'tf.assert_positive': 'tf.compat.v1.assert_positive',
     'tf.assert_proper_iterable': 'tf.debugging.assert_proper_iterable',
-    'tf.assert_rank_at_least': 'tf.debugging.assert_rank_at_least',
-    'tf.assert_rank_in': 'tf.debugging.assert_rank_in',
+    'tf.assert_rank_at_least': 'tf.compat.v1.assert_rank_at_least',
+    'tf.assert_rank_in': 'tf.compat.v1.assert_rank_in',
     'tf.assert_same_float_dtype': 'tf.debugging.assert_same_float_dtype',
-    'tf.assert_scalar': 'tf.debugging.assert_scalar',
-    'tf.assert_type': 'tf.debugging.assert_type',
+    'tf.assert_scalar': 'tf.compat.v1.assert_scalar',
+    'tf.assert_type': 'tf.compat.v1.assert_type',
     'tf.assert_variables_initialized': 'tf.compat.v1.assert_variables_initialized',
     'tf.assign': 'tf.compat.v1.assign',
     'tf.assign_add': 'tf.compat.v1.assign_add',
@@ -163,6 +167,9 @@ renames = {
     'tf.get_session_tensor': 'tf.compat.v1.get_session_tensor',
     'tf.get_variable': 'tf.compat.v1.get_variable',
     'tf.get_variable_scope': 'tf.compat.v1.get_variable_scope',
+    'tf.gfile.FastGFile': 'tf.compat.v1.gfile.FastGFile',
+    'tf.gfile.GFile': 'tf.compat.v1.gfile.GFile',
+    'tf.gfile.Open': 'tf.compat.v1.gfile.Open',
     'tf.global_norm': 'tf.linalg.global_norm',
     'tf.global_variables': 'tf.compat.v1.global_variables',
     'tf.global_variables_initializer': 'tf.compat.v1.global_variables_initializer',
@@ -362,10 +369,11 @@ renames = {
     'tf.saved_model.get_tensor_from_tensor_info': 'tf.compat.v1.saved_model.get_tensor_from_tensor_info',
     'tf.saved_model.load': 'tf.compat.v1.saved_model.load',
     'tf.saved_model.loader.load': 'tf.compat.v1.saved_model.loader.load',
-    'tf.saved_model.loader.maybe_saved_model_directory': 'tf.saved_model.maybe_saved_model_directory',
+    'tf.saved_model.loader.maybe_saved_model_directory': 'tf.compat.v1.saved_model.loader.maybe_saved_model_directory',
     'tf.saved_model.main_op.main_op': 'tf.compat.v1.saved_model.main_op.main_op',
     'tf.saved_model.main_op.main_op_with_restore': 'tf.compat.v1.saved_model.main_op.main_op_with_restore',
     'tf.saved_model.main_op_with_restore': 'tf.compat.v1.saved_model.main_op_with_restore',
+    'tf.saved_model.maybe_saved_model_directory': 'tf.compat.v1.saved_model.maybe_saved_model_directory',
     'tf.saved_model.signature_constants.CLASSIFY_INPUTS': 'tf.saved_model.CLASSIFY_INPUTS',
     'tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME': 'tf.saved_model.CLASSIFY_METHOD_NAME',
     'tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES': 'tf.saved_model.CLASSIFY_OUTPUT_CLASSES',
-- 
GitLab


From 1023a54b03751d859468905403e2ae7f4faf4e60 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 16 Nov 2018 10:13:49 -0800
Subject: [PATCH 0404/1554] Remove the input flattening as required in the new
 API

PiperOrigin-RevId: 221811903
---
 .../contrib/distribute/python/metrics_v1_test.py     |  2 +-
 .../contrib/distribute/python/minimize_loss_test.py  | 12 ++++++------
 .../contrib/distribute/python/mirrored_strategy.py   |  2 +-
 .../contrib/distribute/python/one_device_strategy.py |  2 +-
 tensorflow/contrib/distribute/python/step_fn.py      |  2 +-
 tensorflow/contrib/distribute/python/tpu_strategy.py |  2 +-
 .../python/keras/engine/training_distributed.py      |  8 +++++---
 tensorflow/python/training/distribute.py             |  9 ++-------
 8 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index e622e1df23..8ac659abe9 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -100,7 +100,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
       if isinstance(distribution, tpu_strategy.TPUStrategy):
         def step_fn(ctx, inputs):
           value, update = distribution.call_for_each_replica(
-              metric_fn, args=[inputs])
+              metric_fn, args=inputs)
           ctx.set_non_tensor_output(name="value", output=value)
           return distribution.group(update)
 
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 195d8a90c2..e77d3d455b 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -64,7 +64,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       model_fn, dataset_fn, layer = minimize_loss_example(
           optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss)
 
-      def step_fn(ctx, *inputs):
+      def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
             distribution.call_for_each_replica(model_fn, args=inputs))
@@ -158,7 +158,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
           use_callable_loss=True,
           create_optimizer_inside_model_fn=True)
 
-      def step_fn(ctx, *inputs):
+      def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
             distribution.call_for_each_replica(model_fn, args=inputs))
@@ -227,7 +227,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
           renorm=renorm,
           update_ops_in_replica_mode=not update_ops_in_cross_replica_mode)
 
-      def step_fn(ctx, *inputs):
+      def step_fn(ctx, inputs):
         del ctx  # Unused
         fetches = distribution.unwrap(
             distribution.call_for_each_replica(model_fn, args=inputs))
@@ -324,10 +324,10 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         labels = dataset_ops.Dataset.from_tensors([[6.], [21.]])
         return dataset_ops.Dataset.zip((features, labels)).repeat()
 
-      def step_fn(ctx, x, y):
+      def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(x, y)))
+            distribution.call_for_each_replica(model_fn, args=inputs))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
@@ -411,7 +411,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         output_context.set_non_tensor_output(key1, value1)
         return (train_op, loss)
 
-      def step_fn(output_context, *inputs):
+      def step_fn(output_context, inputs):
         (train_op, loss) = distribution.call_for_each_replica(
             model_fn, args=(output_context,) + inputs)
         output_context.set_last_step_output(
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 34061696ed..3b24f1b03d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -538,7 +538,7 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
       fn_inputs = iterator.get_next()
       if not isinstance(fn_inputs, tuple):
         fn_inputs = (fn_inputs,)
-      fn_result = fn(ctx, *fn_inputs)
+      fn_result = fn(ctx, fn_inputs)
       for (name, output) in ctx.last_step_outputs.items():
         # Convert all outputs to tensors, potentially from `DistributedValues`.
         ctx.last_step_outputs[name] = self._unwrap(output)
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 893f073f00..6df3b3a92b 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -101,7 +101,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
       fn_inputs = iterator.get_next()
       if not isinstance(fn_inputs, tuple):
         fn_inputs = (fn_inputs,)
-      fn_result = fn(ctx, *fn_inputs)
+      fn_result = fn(ctx, fn_inputs)
       flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
       with ops.control_dependencies([fn_result]):
         return [i + 1] + flat_last_step_outputs
diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py
index 3dc815f037..c928b6d9f1 100644
--- a/tensorflow/contrib/distribute/python/step_fn.py
+++ b/tensorflow/contrib/distribute/python/step_fn.py
@@ -94,7 +94,7 @@ class StandardSingleLossStep(StandardInputStep):
 
   def __call__(self):
     with self._distribution.scope():
-      def step_fn(ctx, *inputs):
+      def step_fn(ctx, inputs):
         """Function to run one iteration with one input."""
         gradients_fn = backprop.implicit_grad(self._loss_fn)
         gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn)
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index cfe411af76..f36171e4c2 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -328,7 +328,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       fn_inputs = dequeue_fn()
       if not isinstance(fn_inputs, tuple):
         fn_inputs = (fn_inputs,)
-      fn_result = fn(ctx, *fn_inputs)
+      fn_result = fn(ctx, fn_inputs)
       flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
       if flat_last_step_outputs:
         with ops.control_dependencies([fn_result]):
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index a614614c9b..0d6f4d772f 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -266,11 +266,12 @@ def _experimental_fit_loop(
   K.set_learning_phase(1)
   out_labels = model.metrics_names or []
 
-  def step_fn(ctx, inputs, targets):
+  def step_fn(ctx, inputs):
     """Clones the model and calls make_fit_function."""
     # TODO(priyag, sourabhbajaj): The model gets cloned every time
     # fit/test/predict is called. We should look into caching this keyed on
     # input shapes.
+    inputs, targets = inputs
     clone_model_on_replicas(
         model,
         current_strategy,
@@ -546,11 +547,12 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None,
   # TODO(priyag, sourabhbajaj): This should likely not be hardcoded here.
   K.set_learning_phase(0)
 
-  def step_fn(ctx, inputs, targets):
+  def step_fn(ctx, inputs):
     """Clones the model and calls make_eval_function."""
     # TODO(priyag, sourabhbajaj): The model gets cloned every time
     # fit/test/predict is called. We should look into caching this keyed on
     # input shapes.
+    inputs, targets = inputs
     clone_model_on_replicas(
         model,
         current_strategy,
@@ -750,7 +752,7 @@ def _experimental_predict_loop(model, iterator, verbose=0, steps=None):
             model.predict_function.updates_op,
             model.predict_function.session_kwargs)
 
-  def step_fn(ctx, *inputs):
+  def step_fn(ctx, inputs):
     """Clones the model and calls make_predict_function."""
 
     # TODO(priyag, sourabhbajaj): The model gets cloned every time
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 53148ed6f9..b68d62a0a5 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1043,18 +1043,13 @@ class DistributionStrategyExtended(object):
 
     Args:
       fn: function to run using this distribution strategy. The function must
-        have the following signature: `def fn(context, *inputs)`.
+        have the following signature: `def fn(context, inputs)`.
         `context` is an instance of `MultiStepContext` that will be passed when
         `fn` is run. `context` can be used to specify the outputs to be returned
         from `fn` by calling `context.set_last_step_output`. It can also be used
         to capture non tensor outputs by `context.set_non_tensor_output`.
         See `MultiStepContext` documentation for more information.
-        `inputs` will have same type/structure as `iterator.get_next()`. If the
-        `iterator.get_next()` returns a tuple say `return x, y` then whose will
-        be unpacked and passed to the `step_fn`; and step_fn signature would
-        look like `def step_fn(context, x, y)`. If the iterator returns a single
-        value say `return x` then the value is passed as is; the step_fn
-        signature would look like `def step_fn(context, x)`.
+        `inputs` will have same type/structure as `iterator.get_next()`.
         Typically, `fn` will use `call_for_each_replica` method of the strategy
         to distribute the computation over multiple replicas.
       iterator: Iterator of a dataset that represents the input for `fn`. The
-- 
GitLab


From 5bdfc37e3fa3cd3bfee670b0e9705a8188d0f83d Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 16 Nov 2018 10:29:06 -0800
Subject: [PATCH 0405/1554] [tf.data] Relax argument type checks to allow
 `DatasetV2`.

In each case, we are only relying on the V2 subset of methods, so this will smooth the transition to the V2 API.

PiperOrigin-RevId: 221814307
---
 .../kernel_tests/get_single_element_test.py           | 11 +++++++++++
 .../data/experimental/ops/get_single_element.py       |  2 +-
 .../python/data/experimental/ops/interleave_ops.py    |  2 +-
 tensorflow/python/data/experimental/ops/writers.py    |  2 +-
 tensorflow/python/data/ops/readers.py                 |  2 +-
 5 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py b/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py
index 8c07afbac5..0147988c59 100644
--- a/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py
@@ -67,6 +67,17 @@ class GetSingleElementTest(test_base.DatasetTestBase, parameterized.TestCase):
         with self.assertRaisesRegexp(error, error_msg):
           sess.run(element, feed_dict={skip_t: skip, take_t: take})
 
+  def testWindow(self):
+    """Test that `get_single_element()` can consume a nested dataset."""
+    def flat_map_func(ds):
+      batched = ds.batch(2)
+      element = get_single_element.get_single_element(batched)
+      return dataset_ops.Dataset.from_tensors(element)
+
+    dataset = dataset_ops.Dataset.range(10).window(2).flat_map(flat_map_func)
+    self.assertDatasetProduces(
+        dataset, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]])
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/ops/get_single_element.py b/tensorflow/python/data/experimental/ops/get_single_element.py
index 132526166c..73116edf12 100644
--- a/tensorflow/python/data/experimental/ops/get_single_element.py
+++ b/tensorflow/python/data/experimental/ops/get_single_element.py
@@ -60,7 +60,7 @@ def get_single_element(dataset):
     InvalidArgumentError (at runtime): if `dataset` does not contain exactly
       one element.
   """
-  if not isinstance(dataset, dataset_ops.Dataset):
+  if not isinstance(dataset, dataset_ops.DatasetV2):
     raise TypeError("`dataset` must be a `tf.data.Dataset` object.")
 
   nested_ret = nest.pack_sequence_as(
diff --git a/tensorflow/python/data/experimental/ops/interleave_ops.py b/tensorflow/python/data/experimental/ops/interleave_ops.py
index 5605e1d60e..8b0fdfce11 100644
--- a/tensorflow/python/data/experimental/ops/interleave_ops.py
+++ b/tensorflow/python/data/experimental/ops/interleave_ops.py
@@ -158,7 +158,7 @@ def sample_from_datasets_v2(datasets, weights=None, seed=None):
       length of the `datasets` element.
   """
   num_datasets = len(datasets)
-  if not isinstance(weights, dataset_ops.Dataset):
+  if not isinstance(weights, dataset_ops.DatasetV2):
     if weights is None:
       # Select inputs with uniform probability.
       logits = [[1.0] * num_datasets]
diff --git a/tensorflow/python/data/experimental/ops/writers.py b/tensorflow/python/data/experimental/ops/writers.py
index 994447cb4d..cc0a80336c 100644
--- a/tensorflow/python/data/experimental/ops/writers.py
+++ b/tensorflow/python/data/experimental/ops/writers.py
@@ -48,7 +48,7 @@ class TFRecordWriter(object):
     Returns:
       A `tf.Operation` that, when run, writes contents of `dataset` to a file.
     """
-    if not isinstance(dataset, dataset_ops.Dataset):
+    if not isinstance(dataset, dataset_ops.DatasetV2):
       raise TypeError("`dataset` must be a `tf.data.Dataset` object.")
     if (dataset.output_types != dtypes.string or
         dataset.output_shapes != tensor_shape.scalar()):
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index 8a563122a0..880e005653 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -201,7 +201,7 @@ class TFRecordDatasetV2(dataset_ops.DatasetV2):
       ValueError: If any argument does not have the expected shape.
     """
     super(TFRecordDatasetV2, self).__init__()
-    if isinstance(filenames, dataset_ops.Dataset):
+    if isinstance(filenames, dataset_ops.DatasetV2):
       if filenames.output_types != dtypes.string:
         raise TypeError(
             "`filenames` must be a `tf.data.Dataset` of `tf.string` elements.")
-- 
GitLab


From 5fb82fc0184b199d302757bfb4f0a57921a8acf0 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Fri, 16 Nov 2018 10:33:45 -0800
Subject: [PATCH 0406/1554] Propagate int8 type through toco/tflite.

PiperOrigin-RevId: 221815026
---
 tensorflow/lite/c/c_api_internal.c                         | 2 ++
 tensorflow/lite/c/c_api_internal.h                         | 2 ++
 tensorflow/lite/c/c_api_internal_test.cc                   | 1 +
 tensorflow/lite/core/api/flatbuffer_conversions.cc         | 3 +++
 tensorflow/lite/delegates/flex/util.cc                     | 4 ++++
 tensorflow/lite/interpreter.cc                             | 7 +++++--
 tensorflow/lite/interpreter.h                              | 4 ++++
 tensorflow/lite/kernels/internal/tensor_ctypes.h           | 5 +++++
 tensorflow/lite/kernels/test_util.h                        | 1 +
 tensorflow/lite/optional_debug_tools.cc                    | 2 ++
 .../lite/python/interpreter_wrapper/interpreter_wrapper.cc | 4 ++++
 tensorflow/lite/toco/tooling_util.cc                       | 2 ++
 tensorflow/lite/toco/types.proto                           | 3 +++
 13 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c
index b131f06774..7f67b1c272 100644
--- a/tensorflow/lite/c/c_api_internal.c
+++ b/tensorflow/lite/c/c_api_internal.c
@@ -125,6 +125,8 @@ const char* TfLiteTypeGetName(TfLiteType type) {
       return "INT32";
     case kTfLiteUInt8:
       return "UINT8";
+    case kTfLiteInt8:
+      return "INT8";
     case kTfLiteInt64:
       return "INT64";
     case kTfLiteBool:
diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index e05fd19936..d7bf06442b 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -179,6 +179,7 @@ typedef enum {
   kTfLiteBool = 6,
   kTfLiteInt16 = 7,
   kTfLiteComplex64 = 8,
+  kTfLiteInt8 = 9,
 } TfLiteType;
 
 // Return the name of a given type, for error reporting purposes.
@@ -203,6 +204,7 @@ typedef union {
   bool* b;
   int16_t* i16;
   TfLiteComplex64* c64;
+  int8_t* int8;
 } TfLitePtrUnion;
 
 // Memory allocation strategies. kTfLiteMmapRo is for read-only memory-mapped
diff --git a/tensorflow/lite/c/c_api_internal_test.cc b/tensorflow/lite/c/c_api_internal_test.cc
index e21823c41f..acf0dfc5be 100644
--- a/tensorflow/lite/c/c_api_internal_test.cc
+++ b/tensorflow/lite/c/c_api_internal_test.cc
@@ -74,6 +74,7 @@ TEST(Types, TestTypeNames) {
   EXPECT_EQ(type_name(kTfLiteInt16), "INT16");
   EXPECT_EQ(type_name(kTfLiteInt32), "INT32");
   EXPECT_EQ(type_name(kTfLiteUInt8), "UINT8");
+  EXPECT_EQ(type_name(kTfLiteInt8), "INT8");
   EXPECT_EQ(type_name(kTfLiteInt64), "INT64");
   EXPECT_EQ(type_name(kTfLiteBool), "BOOL");
   EXPECT_EQ(type_name(kTfLiteComplex64), "COMPLEX64");
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index e63a3ec5d6..3e67badfe4 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -61,6 +61,9 @@ TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type,
     case TensorType_UINT8:
       *type = kTfLiteUInt8;
       break;
+    case TensorType_INT8:
+      *type = kTfLiteInt8;
+      break;
     case TensorType_INT64:
       *type = kTfLiteInt64;
       break;
diff --git a/tensorflow/lite/delegates/flex/util.cc b/tensorflow/lite/delegates/flex/util.cc
index c786ffa1a2..c995b360f9 100644
--- a/tensorflow/lite/delegates/flex/util.cc
+++ b/tensorflow/lite/delegates/flex/util.cc
@@ -66,6 +66,8 @@ TF_DataType GetTensorFlowDataType(TfLiteType type) {
       return TF_INT32;
     case kTfLiteUInt8:
       return TF_UINT8;
+    case kTfLiteInt8:
+      return TF_INT8;
     case kTfLiteInt64:
       return TF_INT64;
     case kTfLiteComplex64:
@@ -87,6 +89,8 @@ TfLiteType GetTensorFlowLiteType(TF_DataType type) {
       return kTfLiteInt32;
     case TF_UINT8:
       return kTfLiteUInt8;
+    case TF_INT8:
+      return kTfLiteInt8;
     case TF_INT64:
       return kTfLiteInt64;
     case TF_COMPLEX64:
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 39b105042d..04db91ffd8 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -407,6 +407,9 @@ TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims,
     case kTfLiteUInt8:
       *bytes = sizeof(uint8_t) * count;
       break;
+    case kTfLiteInt8:
+      *bytes = sizeof(int8_t) * count;
+      break;
     case kTfLiteInt64:
       *bytes = sizeof(int64_t) * count;
       break;
@@ -418,8 +421,8 @@ TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims,
       break;
     default:
       ReportError(&context_,
-                  "Only float32, int16, int32, int64, uint8, bool, complex64 "
-                  "supported currently.");
+                  "Only float32, int16, int32, int64, uint8, int8, bool, "
+                  "complex64 supported currently.");
       return kTfLiteError;
   }
   return kTfLiteOk;
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index 1cadb5eb5d..eb7d2e47f1 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -58,6 +58,10 @@ constexpr TfLiteType typeToTfLiteType<unsigned char>() {
   return kTfLiteUInt8;
 }
 template <>
+constexpr TfLiteType typeToTfLiteType<int8_t>() {
+  return kTfLiteInt8;
+}
+template <>
 constexpr TfLiteType typeToTfLiteType<bool>() {
   return kTfLiteBool;
 }
diff --git a/tensorflow/lite/kernels/internal/tensor_ctypes.h b/tensorflow/lite/kernels/internal/tensor_ctypes.h
index d24dca9bfb..b4822d5701 100644
--- a/tensorflow/lite/kernels/internal/tensor_ctypes.h
+++ b/tensorflow/lite/kernels/internal/tensor_ctypes.h
@@ -66,6 +66,11 @@ inline const uint8_t* GetTensorData(const TfLiteTensor* tensor) {
   return tensor != nullptr ? tensor->data.uint8 : nullptr;
 }
 
+template <>
+inline const int8_t* GetTensorData(const TfLiteTensor* tensor) {
+  return tensor != nullptr ? tensor->data.int8 : nullptr;
+}
+
 template <>
 inline const int16_t* GetTensorData(const TfLiteTensor* tensor) {
   return tensor != nullptr ? tensor->data.i16 : nullptr;
diff --git a/tensorflow/lite/kernels/test_util.h b/tensorflow/lite/kernels/test_util.h
index 43a5137a94..a49d6c2cae 100644
--- a/tensorflow/lite/kernels/test_util.h
+++ b/tensorflow/lite/kernels/test_util.h
@@ -307,6 +307,7 @@ class SingleOpModel {
 
     if (is_quantized) {
       if (t.min != 0 || t.max != 0) {
+        // TODO(b/119422369): Handle signed int8 here.
         if (t.type == TensorType_UINT8) {
           std::tie(t.scale, t.zero_point) =
               QuantizationParams<uint8_t>(t.min, t.max);
diff --git a/tensorflow/lite/optional_debug_tools.cc b/tensorflow/lite/optional_debug_tools.cc
index 5ee1cf6d33..1113bf01b1 100644
--- a/tensorflow/lite/optional_debug_tools.cc
+++ b/tensorflow/lite/optional_debug_tools.cc
@@ -44,6 +44,8 @@ const char* TensorTypeName(TfLiteType type) {
       return "kTfLiteInt32";
     case kTfLiteUInt8:
       return "kTfLiteUInt8";
+    case kTfLiteInt8:
+      return "kTfLiteInt8";
     case kTfLiteInt64:
       return "kTfLiteInt64";
     case kTfLiteString:
diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index e71752fe63..d14af439ec 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -124,6 +124,8 @@ int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
       return NPY_INT16;
     case kTfLiteUInt8:
       return NPY_UINT8;
+    case kTfLiteInt8:
+      return NPY_INT8;
     case kTfLiteInt64:
       return NPY_INT64;
     case kTfLiteString:
@@ -150,6 +152,8 @@ TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
       return kTfLiteInt16;
     case NPY_UINT8:
       return kTfLiteUInt8;
+    case NPY_INT8:
+      return kTfLiteInt8;
     case NPY_INT64:
       return kTfLiteInt64;
     case NPY_BOOL:
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index e33f7c8452..4a96450e07 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -2207,6 +2207,8 @@ ArrayDataType ConvertIODataTypeToArrayDataType(IODataType type) {
       return ArrayDataType::kFloat;
     case QUANTIZED_UINT8:
       return ArrayDataType::kUint8;
+    case INT8:
+      return ArrayDataType::kInt8;
     case QUANTIZED_INT16:
       return ArrayDataType::kInt16;
     case INT32:
diff --git a/tensorflow/lite/toco/types.proto b/tensorflow/lite/toco/types.proto
index 12f711fd8a..fa911b8a4c 100644
--- a/tensorflow/lite/toco/types.proto
+++ b/tensorflow/lite/toco/types.proto
@@ -43,4 +43,7 @@ enum IODataType {
 
   // Complex64, not quantized
   COMPLEX64 = 8;
+
+  // Int8, quantized based on QuantizationParameters in schema.
+  INT8 = 9;
 }
-- 
GitLab


From 2c1edcbacf5070d96690f8e091cc2bcfe7fda4cb Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 16 Nov 2018 10:35:00 -0800
Subject: [PATCH 0407/1554] [tf.data] Fix the default thread pool size use for
 auto-tuning.

Prior to this fix, the default was using the size of the intra-op threadpool not the inter-op threadpool.

PiperOrigin-RevId: 221815222
---
 tensorflow/core/framework/dataset.h | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index 5960c105c8..9b11449b30 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -30,8 +30,10 @@ limitations under the License.
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/framework/variant_encode_decode.h"
 #include "tensorflow/core/framework/variant_tensor_data.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/tracing.h"
 
 // Polymorphic datasets should support all primitive TensorFlow
@@ -285,15 +287,20 @@ class IteratorContext {
     explicit Params(OpKernelContext* ctx)
         : env(ctx->env()),
           lib(ctx->function_library()),
-          runner(*(ctx->runner())),
-          runner_threadpool_size(
-              ctx->device()->tensorflow_cpu_worker_threads()->num_threads) {
+          runner(*(ctx->runner())) {
       // NOTE: need reinterpret_cast because function.h forward-declares Device.
       DeviceBase* device =
           reinterpret_cast<DeviceBase*>(ctx->function_library()->device());
       allocator_getter = [device](AllocatorAttributes attrs) {
         return device->GetAllocator(attrs);
       };
+      thread::ThreadPool* thread_pool =
+          ctx->device()->tensorflow_device_thread_pool();
+      if (thread_pool) {
+        runner_threadpool_size = thread_pool->NumThreads();
+      } else {
+        runner_threadpool_size = port::NumSchedulableCPUs();
+      }
     }
 
     // The Allocator to be used to allocate the output of an iterator.
-- 
GitLab


From 5e1ab9651b3490f0b3e7a9f50bc28631be30bce4 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Fri, 16 Nov 2018 10:38:33 -0800
Subject: [PATCH 0408/1554] Reduce tolerance for CoreMirroredStrategy when
 running the correctness test.

PiperOrigin-RevId: 221815733
---
 tensorflow/contrib/distribute/python/BUILD         | 3 ---
 tensorflow/contrib/distribute/python/keras_test.py | 3 ++-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index b8d6208150..bf5adb1a50 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -752,13 +752,10 @@ cuda_py_test(
     ],
     shard_count = 16,
     tags = [
-        "manual",  # TODO(b/119646053): Failing when run.
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
-        "noguitar",  # TODO(b/119646053): Failing when run.
-        "notap",  # TODO(b/119646053): Failing when run.
         "notsan",
     ],
 )
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 4adc380b83..e7dc57b15b 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1102,7 +1102,8 @@ class TestDistributionStrategyCorrectness(test.TestCase,
     with self.cached_session():
       tolerance = 1e-5
 
-      if isinstance(distribution, mirrored_strategy.MirroredStrategy):
+      if isinstance(distribution, (mirrored_strategy.MirroredStrategy,
+                                   mirrored_strategy.CoreMirroredStrategy)):
         # TODO(b/119257215): use the default one once the flakyness is fixed.
         tolerance = 1e-4
 
-- 
GitLab


From 0bae83b92ba0d8b62eae966022017237f599221e Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 16 Nov 2018 10:40:54 -0800
Subject: [PATCH 0409/1554] Support returning TensorArrays and SparseTensors
 from functions (IndexedSlices was already supported). Support returning
 TensorArrays, SparseTensors and IndexedSlices from cond_v2. Pack the outputs
 of cond_v2 to have the same shape as the function outputs and check that the
 structure of true_graph.outputs and false_graph.outputs matches.

PiperOrigin-RevId: 221816081
---
 tensorflow/python/BUILD                       |  5 ++
 .../python/framework/auto_control_deps.py     | 13 +++
 tensorflow/python/framework/func_graph.py     | 66 +++++++++++++-
 tensorflow/python/kernel_tests/BUILD          |  2 +
 .../python/kernel_tests/cond_v2_test.py       | 86 +++++++++++++++++++
 .../kernel_tests/control_flow_ops_py_test.py  |  7 +-
 .../python/kernel_tests/while_v2_test.py      |  6 +-
 tensorflow/python/ops/cond_v2.py              | 20 ++---
 tensorflow/python/ops/tensor_array_ops.py     | 12 +++
 tensorflow/python/ops/while_v2.py             | 11 ++-
 10 files changed, 203 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e81ed2f73a..83b7bc9403 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -878,6 +878,8 @@ py_library(
     deps = [
         ":auto_control_deps",
         ":framework_ops",
+        ":sparse_tensor",
+        ":tensor_array_ops",
         "//tensorflow/python/autograph",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:graph_only_ops",
@@ -892,6 +894,8 @@ py_library(
     deps = [
         ":control_flow_ops",
         ":framework_ops",
+        ":sparse_tensor",
+        ":tensor_array_ops",
         ":util",
     ],
 )
@@ -2137,6 +2141,7 @@ py_library(
         ":control_flow_util_v2",
         ":dtypes",
         ":framework_ops",
+        ":framework_test_lib",
         ":function_def_to_graph",
         ":functional_ops_gen",
         ":gradients_impl",
diff --git a/tensorflow/python/framework/auto_control_deps.py b/tensorflow/python/framework/auto_control_deps.py
index 9a9ee46aab..30dc959e9a 100644
--- a/tensorflow/python/framework/auto_control_deps.py
+++ b/tensorflow/python/framework/auto_control_deps.py
@@ -21,9 +21,11 @@ from __future__ import print_function
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes as dtypes_module
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import control_flow_util
+from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 
@@ -70,6 +72,17 @@ class AutomaticControlDependencies(object):
       self._returned_tensors.add(indices)
       self._returned_tensors.add(values)
       return ops.IndexedSlices(values, indices, dense_shape=tensor.dense_shape)
+    elif isinstance(tensor, sparse_tensor.SparseTensor):
+      values = array_ops.identity(tensor.values)
+      indices = array_ops.identity(tensor.indices)
+      self._returned_tensors.add(indices)
+      self._returned_tensors.add(values)
+      return sparse_tensor.SparseTensor(
+          indices, values, dense_shape=tensor.dense_shape)
+    elif isinstance(tensor, tensor_array_ops.TensorArray):
+      flow = array_ops.identity(tensor.flow)
+      self._returned_tensors.add(flow)
+      return tensor_array_ops.build_ta_with_new_flow(tensor, flow)
     # We want to make the return values depend on the stateful operations, but
     # we don't want to introduce a cycle, so we make the return value the result
     # of a new identity operation that the stateful operations definitely don't
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index c7a5d1ee20..f1e508c365 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -26,11 +26,13 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.eager.graph_only_ops import graph_placeholder
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework.auto_control_deps import AutomaticControlDependencies
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
@@ -372,7 +374,7 @@ def func_graph_from_py_func(name,
         # captured Operations).
         with ops.control_dependencies([x]):
           x = array_ops.identity(op_return_value)
-      else:
+      elif not isinstance(x, tensor_array_ops.TensorArray):
         try:
           x = ops.convert_to_tensor_or_indexed_slices(x)
         except (ValueError, TypeError):
@@ -408,7 +410,8 @@ def func_graph_from_py_func(name,
 
       func_outputs = python_func(*func_args, **func_kwargs)
 
-      # invariant: `func_outputs` contains only Tensors and `None`s.
+      # invariant: `func_outputs` contains only Tensors, IndexedSlices,
+      # SparseTensors, TensorArrays and `None`s.
       func_outputs = nest.map_structure(convert, func_outputs)
 
       check_mutation(func_args_before, func_args)
@@ -495,7 +498,17 @@ def check_mutation(n1, n2):
 
 
 def flatten(sequence):
-  """A wrapper around `nest.flatten` that also unpacks `IndexedSlices`."""
+  """Like `nest.flatten` but also unpacks other Tensor-like objects.
+
+  Flattens non-tensor objects into their constituent tensors.
+
+  Args:
+    sequence: A nested structure of Tensors, IndexedSlices, SparseTensors and
+      TensorArrays.
+
+  Returns:
+    A list of tensors.
+  """
   # TODO(akshayka): Support `SparseTensor` in a similar fashion.
   flat_sequence = nest.flatten(sequence)
   outputs = []
@@ -505,11 +518,58 @@ def flatten(sequence):
         outputs.extend([item.values, item.indices, item.dense_shape])
       else:
         outputs.extend([item.values, item.indices])
+    elif isinstance(item, sparse_tensor.SparseTensor):
+      outputs.extend([item.indices, item.values, item.dense_shape])
+    elif isinstance(item, tensor_array_ops.TensorArray):
+      outputs.append(item.flow)
     else:
       outputs.append(item)
   return outputs
 
 
+def pack_sequence_as(structure, flat_sequence):
+  """Like `nest.pack_sequence_as` but also packs other Tensor-like objects.
+
+  Args:
+    structure: The structure to pack into. May contain Tensors, IndexedSlices,
+      TensorArrays or SparseTensors.
+    flat_sequence: An iterable containing tensors.
+
+  Returns:
+    A nested structure.
+
+  Raises:
+    AssertionError if `structure` and `flat_sequence` are not compatible.
+  """
+  flattened_structure = nest.flatten(structure)
+  flat_sequence_with_slices_and_tas = []
+  index = 0
+  for t in flattened_structure:
+    if isinstance(t, ops.IndexedSlices):
+      if t.dense_shape is not None:
+        flat_sequence_with_slices_and_tas.append(
+            ops.IndexedSlices(*flat_sequence[index:index + 3]))
+        index += 3
+      else:
+        flat_sequence_with_slices_and_tas.append(
+            ops.IndexedSlices(*flat_sequence[index:index + 2]))
+        index += 2
+    elif isinstance(t, sparse_tensor.SparseTensor):
+      flat_sequence_with_slices_and_tas.append(
+          sparse_tensor.SparseTensor(*flat_sequence[index:index + 3]))
+      index += 3
+    elif isinstance(t, tensor_array_ops.TensorArray):
+      flow = flat_sequence[index]
+      ta = tensor_array_ops.build_ta_with_new_flow(t, flow)
+      flat_sequence_with_slices_and_tas.append(ta)
+      index += 1
+    else:
+      flat_sequence_with_slices_and_tas.append(flat_sequence[index])
+      index += 1
+  assert len(flattened_structure) == len(flat_sequence_with_slices_and_tas)
+  return nest.pack_sequence_as(structure, flat_sequence_with_slices_and_tas)
+
+
 def _create_substitute_placeholder(value, name=None, dtype=None):
   """Creates a placeholder for `value` and propagates shape info to it."""
   # Note: setting ops.control_dependencies(None) ensures we always put
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 5f3b8a2449..975b603006 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -3315,7 +3315,9 @@ cuda_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:gradients",
+        "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:training",
+        "//tensorflow/python:while_v2",
     ],
     grpc_enabled = True,
 )
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index cf41134e7c..70b66bc52a 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import saver
@@ -684,6 +685,91 @@ class CondV2Test(test.TestCase):
       # the single threaded executor does not support cond v1 ops.
       sess.run(out_cond, feed_dict={x: 1.0})
 
+  @test_util.enable_control_flow_v2
+  def testStructuredOutputs(self):
+    x = constant_op.constant(1.0, name="x")
+    y = constant_op.constant(3.0, name="y")
+
+    def true_fn():
+      return ((x * y,), y)
+
+    def false_fn():
+      return ((x,), y * 3.0)
+
+    output = control_flow_ops.cond(
+        constant_op.constant(False), true_fn, false_fn)
+    self.assertEqual(self.evaluate(output[0][0]), 1.)
+    self.assertEqual(self.evaluate(output[1]), 9.)
+
+  @test_util.enable_control_flow_v2
+  def testRaisesOutputStructuresMismatch(self):
+    x = constant_op.constant(1.0, name="x")
+    y = constant_op.constant(3.0, name="y")
+
+    def true_fn():
+      return x * y, y
+
+    def false_fn():
+      return ((x,), y * 3.0)
+
+    with self.assertRaisesRegexp(
+        ValueError, "Outputs of true_fn and false_fn must"
+        " have the same structure"):
+      control_flow_ops.cond(constant_op.constant(False), true_fn, false_fn)
+
+  @test_util.enable_control_flow_v2
+  def testCondAndTensorArray(self):
+    x = math_ops.range(-5, 5)
+    output = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=x.shape[0])
+
+    def loop_body(i, output):
+
+      def if_true():
+        return output.write(i, x[i]**2)
+
+      def if_false():
+        return output.write(i, x[i])
+
+      output = control_flow_ops.cond(x[i] > 0, if_true, if_false)
+      return i + 1, output
+
+    _, output = control_flow_ops.while_loop(
+        lambda i, arr: i < x.shape[0],
+        loop_body,
+        loop_vars=(constant_op.constant(0), output))
+    output_t = output.stack()
+    self.assertAllEqual(
+        self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
+
+  @test_util.enable_control_flow_v2
+  def testCondAndTensorArrayInDefun(self):
+
+    @function.defun
+    def f():
+      x = math_ops.range(-5, 5)
+      output = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=x.shape[0])
+
+      def loop_body(i, output):
+
+        def if_true():
+          return output.write(i, x[i]**2)
+
+        def if_false():
+          return output.write(i, x[i])
+
+        output = control_flow_ops.cond(x[i] > 0, if_true, if_false)
+        return i + 1, output
+
+      _, output = control_flow_ops.while_loop(
+          lambda i, arr: i < x.shape[0],
+          loop_body,
+          loop_vars=(constant_op.constant(0), output))
+      return output.stack()
+
+    output_t = f()
+    self.assertAllEqual(
+        self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
+
 
 class CondV2CollectionTest(test.TestCase):
 
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 3f088266cd..e5b45d2ed8 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -389,7 +389,6 @@ class ControlFlowTest(test.TestCase):
             with self.assertRaisesRegexp(ValueError, "may not be fed"):
               sess.run(r, feed_dict={t: 3})
 
-  @test_util.disable_control_flow_v2("b/113296180 (IndexedSlices)")
   def testCondIndexedSlices(self):
     with self.cached_session():
       values = constant_op.constant(10)
@@ -405,7 +404,6 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(11, val)
     self.assertAllEqual(0, ind)
 
-  @test_util.disable_control_flow_v2("b/113296161 (SparseTensors)")
   def testCondSparseTensor(self):
     with self.cached_session():
       values = constant_op.constant([2.0, 4.0], name="values")
@@ -660,9 +658,10 @@ class ControlFlowTest(test.TestCase):
       pred = math_ops.less(1, 2)
       fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
       fn2 = lambda: {"c": y, "d": y}
+      v1_msg = "The two structures don't have the same nested structure"
+      v2_msg = "Outputs of true_fn and false_fn must have the same structure"
       with self.assertRaisesRegexp(
-          ValueError,
-          "The two structures don't have the same nested structure"):
+          ValueError, v2_msg if control_flow_ops.ENABLE_COND_V2 else v1_msg):
         r = control_flow_ops.cond(pred, fn1, fn2)
         test_result = sess.run(r)
 
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 5868b0d226..0634dfa2d8 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -26,9 +26,9 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
@@ -308,9 +308,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
         self.assertRegexpMatches(
             while2_op.get_attr("body").name, r"foo_while_1_body_\d*")
 
+  @test_util.enable_control_flow_v2
   def testWhileAndTensorArray(self):
-    old_enable_while_v2 = control_flow_ops.ENABLE_WHILE_V2
-    control_flow_ops.ENABLE_WHILE_V2 = True
     with self.cached_session() as sess:
       param = constant_op.constant(2.0)
       y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
@@ -319,7 +318,6 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], sess.run(r))
       r = gradients_impl.gradients(r, param)[0]
       self.assertAllClose(21.0, sess.run(r))
-    control_flow_ops.ENABLE_WHILE_V2 = old_enable_while_v2
 
   def testNestedWhile(self):
     # Compute sum of geometric progression: n^0 + n^1 + ... + n^m
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 255df4e582..0f08c611bc 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -121,12 +121,8 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
     # correct output structure
     tensors = tuple(array_ops.identity(t) for t in tensors)
 
-    # Packing output tensors in the same nested structure as the true and false
-    # functions return
-    result = nest.pack_sequence_as(
-        structure=true_graph.structured_outputs,
-        flat_sequence=tensors[:num_cond_outputs])
-    return result
+    return func_graph_module.pack_sequence_as(true_graph.structured_outputs,
+                                              tensors[:num_cond_outputs])
 
 
 @ops.RegisterGradient("If")
@@ -453,15 +449,19 @@ def _check_same_outputs(true_graph, false_graph):
   false_output_types = [t.dtype for t in false_graph.outputs]
   if (len(true_graph.outputs) != len(false_graph.outputs) or
       true_output_types != false_output_types):
-    raise ValueError(
+    raise TypeError(
         "true_fn() and false_fn() must return the same number and type of "
         "arguments, got:\n"
         "  true_fn: %s\n"
         "  false_fn: %s" % (true_output_types, false_output_types))
 
-  # Make sure both structured outputs for both graphs have the same structure
-  nest.assert_same_structure(true_graph.structured_outputs,
-                             false_graph.structured_outputs)
+  # Make sure `structured_outputs` for both graphs have the same structure.
+  try:
+    nest.assert_same_structure(true_graph.structured_outputs,
+                               false_graph.structured_outputs)
+  except (ValueError, TypeError) as e:
+    raise ValueError("Outputs of true_fn and false_fn must have the same "
+                     "structure: %s" % str(e))
 
 
 def _get_output_shapes(true_graph_outputs, false_graph_outputs):
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index f86dfb3527..c08a2b0499 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -953,4 +953,16 @@ class TensorArray(object):
     """Close the current TensorArray."""
     return self._implementation.close(name=name)
 
+
+def build_ta_with_new_flow(old_ta, flow):
+  ta = TensorArray(
+      dtype=old_ta.dtype,
+      handle=old_ta.handle,
+      flow=flow,
+      infer_shape=old_ta._infer_shape,
+      colocate_with_first_write_call=old_ta._colocate_with_first_write_call)
+  ta._colocate_with = old_ta._colocate_with
+  ta._element_shape = old_ta._element_shape
+  return ta
+
 # pylint: enable=protected-access
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index c35431154e..568531cc5c 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -256,11 +256,14 @@ def while_loop(cond,
     outputs = tuple(array_ops.identity(t) for t in outputs)
 
   # First var is loop counter.
-  if num_flattened_outputs == 1:
-    return outputs[1]
+  outputs = _pack_sequence_as(orig_loop_vars,
+                              outputs[1:1 + num_flattened_outputs])
+
+  flattened_outputs = nest.flatten(outputs)
+  if len(flattened_outputs) == 1:
+    return flattened_outputs[0]
   else:
-    return _pack_sequence_as(orig_loop_vars,
-                             outputs[1:1 + num_flattened_outputs])
+    return outputs
 
 
 @ops.RegisterGradient("While")
-- 
GitLab


From 67c42feaac2e5594ec190441fe5f00c9735488b2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 10:41:29 -0800
Subject: [PATCH 0410/1554] Remove split_dim parameter from sparse.split for TF
 API 2.0

PiperOrigin-RevId: 221816187
---
 tensorflow/python/ops/sparse_ops.py           | 47 ++++++++++++++++++-
 .../api/golden/v2/tensorflow.sparse.pbtxt     |  2 +-
 tensorflow/tools/compatibility/renames_v2.py  |  5 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  4 ++
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 8a7cfd45b2..6beceb83d5 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -711,7 +711,7 @@ class KeywordRequired(object):
     return "KeywordRequired()"
 
 
-@tf_export("sparse.split", v1=["sparse.split", "sparse_split"])
+@tf_export(v1=["sparse.split", "sparse_split"])
 @deprecation.deprecated_endpoints("sparse_split")
 @deprecation.deprecated_args(
     None, "split_dim is deprecated, use axis instead", "split_dim")
@@ -785,6 +785,51 @@ def sparse_split(keyword_required=KeywordRequired(),
   return sparse_tensors
 
 
+@tf_export("sparse.split", v1=[])
+def sparse_split_v2(sp_input=None,
+                    num_split=None,
+                    axis=None,
+                    name=None):
+  """Split a `SparseTensor` into `num_split` tensors along `axis`.
+
+  If the `sp_input.dense_shape[axis]` is not an integer multiple of `num_split`
+  each slice starting from 0:`shape[axis] % num_split` gets extra one
+  dimension. For example, if `axis = 1` and `num_split = 2` and the
+  input is:
+
+      input_tensor = shape = [2, 7]
+      [    a   d e  ]
+      [b c          ]
+
+  Graphically the output tensors are:
+
+      output_tensor[0] =
+      [    a ]
+      [b c   ]
+
+      output_tensor[1] =
+      [ d e  ]
+      [      ]
+
+  Args:
+    sp_input: The `SparseTensor` to split.
+    num_split: A Python integer. The number of ways to split.
+    axis: A 0-D `int32` `Tensor`. The dimension along which to split.
+    name: A name for the operation (optional).
+
+  Returns:
+    `num_split` `SparseTensor` objects resulting from splitting `value`.
+
+  Raises:
+    TypeError: If `sp_input` is not a `SparseTensor`.
+  """
+  return sparse_split(sp_input=sp_input,
+                      num_split=num_split,
+                      axis=axis,
+                      name=name,
+                      split_dim=None)
+
+
 @tf_export("sparse.slice", v1=["sparse.slice", "sparse_slice"])
 @deprecation.deprecated_endpoints("sparse_slice")
 def sparse_slice(sp_input, start, size, name=None):
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index cf63924bb1..bb86f07f1e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -110,7 +110,7 @@ tf_module {
   }
   member_method {
     name: "split"
-    argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'sp_input\', \'num_split\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "to_dense"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index c1a550ff3e..5f66d2925e 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -68,6 +68,7 @@ renames = {
     'tf.add_to_collections': 'tf.compat.v1.add_to_collections',
     'tf.all_variables': 'tf.compat.v1.all_variables',
     'tf.angle': 'tf.math.angle',
+    'tf.app.run': 'tf.compat.v1.app.run',
     'tf.assert_greater_equal': 'tf.compat.v1.assert_greater_equal',
     'tf.assert_integer': 'tf.compat.v1.assert_integer',
     'tf.assert_less_equal': 'tf.compat.v1.assert_less_equal',
@@ -290,6 +291,7 @@ renames = {
     'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
+    'tf.multinomial': 'tf.compat.v1.multinomial',
     'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
@@ -331,6 +333,7 @@ renames = {
     'tf.quantize_v2': 'tf.compat.v1.quantize_v2',
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
     'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
+    'tf.random.multinomial': 'tf.compat.v1.random.multinomial',
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
@@ -443,7 +446,7 @@ renames = {
     'tf.sparse_segment_sum': 'tf.sparse.segment_sum',
     'tf.sparse_slice': 'tf.sparse.slice',
     'tf.sparse_softmax': 'tf.sparse.softmax',
-    'tf.sparse_split': 'tf.sparse.split',
+    'tf.sparse_split': 'tf.compat.v1.sparse_split',
     'tf.sparse_tensor_dense_matmul': 'tf.sparse.matmul',
     'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense',
     'tf.sparse_to_indicator': 'tf.sparse.to_indicator',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9cd00e6994..3bfefd3623 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -51,6 +51,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
         },
+        "tf.sparse.split": {
+            "split_dim": "axis",
+        },
         "tf.multinomial": {
             "output_dtype": "dtype",
         },
@@ -122,6 +125,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.contrib.data.unique": "tf.data.experimental.unique",
         "tf.quantize_v2": "tf.quantization.quantize",
         "tf.sparse_concat": "tf.sparse.concat",
+        "tf.sparse_split": "tf.sparse.split",
         "tf.multinomial": "tf.random.categorical",
         "tf.random.multinomial": "tf.random.categorical",
         "tf.load_file_system_library": "tf.load_library",
-- 
GitLab


From 32d2f5199942a5ec52002341665238ba9e73c855 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 16 Nov 2018 10:45:59 -0800
Subject: [PATCH 0411/1554] Treat LegacyFedInput as a placeholder in full flex
 mode

PiperOrigin-RevId: 221816858
---
 tensorflow/lite/toco/import_tensorflow.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 535aef05b1..c13517566f 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2230,6 +2230,7 @@ ConverterMapType GetTensorFlowNodeConverterMapForFlex() {
   return std::unordered_map<std::string, ConverterType>({
       // We need to let TCO convert Placeholder information into
       // array data, so that the data types are correct.
+      {"LegacyFedInput", ConvertPlaceholderOperator},
       {"Placeholder", ConvertPlaceholderOperator},
   });
 }
-- 
GitLab


From 540788794fdd9811267fa1392c966ea1312a6c84 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 16 Nov 2018 10:50:17 -0800
Subject: [PATCH 0412/1554] Add aliases random.all_unigram_canditate_sampler
 and random.all_candidate_sampler in TF2.0.

PiperOrigin-RevId: 221817469
---
 tensorflow/python/ops/candidate_sampling_ops.py        | 7 +++++--
 tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt | 8 ++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/candidate_sampling_ops.py b/tensorflow/python/ops/candidate_sampling_ops.py
index f0bfdb2b7a..c64000b65d 100644
--- a/tensorflow/python/ops/candidate_sampling_ops.py
+++ b/tensorflow/python/ops/candidate_sampling_ops.py
@@ -208,7 +208,9 @@ def learned_unigram_candidate_sampler(true_classes, num_true, num_sampled,
       seed2=seed2, name=name)
 
 
-@tf_export('nn.fixed_unigram_candidate_sampler')
+@tf_export('random.fixed_unigram_candidate_sampler',
+           'nn.fixed_unigram_candidate_sampler',
+           v1=['nn.fixed_unigram_candidate_sampler'])
 def fixed_unigram_candidate_sampler(true_classes,
                                     num_true,
                                     num_sampled,
@@ -300,7 +302,8 @@ def fixed_unigram_candidate_sampler(true_classes,
       unigrams=unigrams, seed=seed1, seed2=seed2, name=name)
 
 
-@tf_export('nn.all_candidate_sampler')
+@tf_export('random.all_candidate_sampler', 'nn.all_candidate_sampler',
+           v1=['nn.all_candidate_sampler'])
 def all_candidate_sampler(true_classes, num_true, num_sampled, unique,
                           seed=None, name=None):
   """Generate the set of all classes.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index a9dd64383d..cace49a5ce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -1,9 +1,17 @@
 path: "tensorflow.random"
 tf_module {
+  member_method {
+    name: "all_candidate_sampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "categorical"
     argspec: "args=[\'logits\', \'num_samples\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "fixed_unigram_candidate_sampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'1.0\', \'0\', \'1\', \'0\', \'()\', \'None\', \'None\'], "
+  }
   member_method {
     name: "gamma"
     argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
-- 
GitLab


From a767a02ca976d00b9e8e06042bdc2a2bb33b00eb Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 16 Nov 2018 10:54:59 -0800
Subject: [PATCH 0413/1554] [Grappler] Remapper optimizer for Conv2D + Squeeze
 + BiasAdd.

 Replace: Conv2D + Squeeze + BiasAdd
    with: _FusedConv2D + Squeeze

PiperOrigin-RevId: 221818332
---
 .../core/grappler/optimizers/remapper.cc      | 110 ++++++++++++++++--
 .../core/grappler/optimizers/remapper_test.cc |  70 +++++++++++
 .../python/grappler/tf_optimizer_test.py      |  13 ++-
 3 files changed, 179 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index 8b93411358..5282b08066 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -37,7 +37,7 @@ constexpr char kDataFormat[] = "data_format";
 constexpr char kIsTraining[] = "is_training";
 
 struct RemapperContext {
-  RemapperContext(const GrapplerItem& item)
+  explicit RemapperContext(const GrapplerItem& item)
       : nodes_to_preserve(item.NodesToPreserve()),
         graph_view(&item.graph),
         graph_properties(item),
@@ -67,6 +67,13 @@ struct Conv2DWithBiasAddAndRelu {
   const NodeDef* relu = nullptr;
 };
 
+// Conv2D node followed by a Squeeze and BiasAdd.
+struct Conv2DWithSqueezeAndBiasAdd {
+  const NodeDef* conv2d = nullptr;
+  const NodeDef* squeeze = nullptr;
+  const NodeDef* bias_add = nullptr;
+};
+
 // Conv2D node followed by a FusedBatchNorm.
 struct Conv2DWithBatchNorm {
   const NodeDef* conv2d = nullptr;
@@ -162,6 +169,54 @@ bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* node,
   return true;
 }
 
+bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
+                                  const NodeDef* node,
+                                  Conv2DWithSqueezeAndBiasAdd* matched) {
+  // Root of the pattern must be a BiasAdd.
+  if (node == nullptr) return false;
+  if (node->op() != "BiasAdd") return false;
+  if (!NodeIsOnCpu(node)) return false;
+  if (!IsFloatOrDoubleDataType(node)) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+
+  // Input to the BiasAdd must be a Squeeze.
+  const auto bias_input_port = GraphView::InputPort(node, 0);
+  const auto squeeze = ctx.graph_view.GetRegularFanin(bias_input_port);
+  if (squeeze.node == nullptr) return false;
+  if (squeeze.node->op() != "Squeeze") return false;
+  if (!NodeIsOnCpu(squeeze.node)) return false;
+  if (!HaveSameDataType(node, squeeze.node, "T")) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, squeeze.node)) return false;
+  if (!HasSingleFanoutNode(ctx.graph_view, squeeze.node)) return false;
+  if (IsInPreserveSet(ctx, squeeze.node)) return false;
+
+  // Squeeze must not squeeze output channel dimension.
+  std::vector<int32> dims;
+  if (!GetNodeAttr(*squeeze.node, "squeeze_dims", &dims).ok()) return false;
+  for (auto dim : dims) {
+    if (dim == 3) return false;
+  }
+
+  // Input to the Squeeze must be a Conv2D in NHWC format.
+  const auto squeeze_input_port = GraphView::InputPort(squeeze.node, 0);
+  const auto conv2d = ctx.graph_view.GetRegularFanin(squeeze_input_port);
+  if (conv2d.node == nullptr) return false;
+  if (conv2d.node->op() != "Conv2D") return false;
+  if (conv2d.node->attr().at("data_format").s() != "NHWC") return false;
+  if (!NodeIsOnCpu(conv2d.node)) return false;
+  if (!HaveSameDataType(node, conv2d.node, "T")) return false;
+  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
+  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
+  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  // We successfully found a Conv2D+Squeeze+BiasAdd pattern.
+  matched->conv2d = conv2d.node;
+  matched->squeeze = squeeze.node;
+  matched->bias_add = node;
+
+  return true;
+}
+
 bool FindConv2DWithBatchNorm(const RemapperContext& ctx, const NodeDef* node,
                              Conv2DWithBatchNorm* matched) {
   // Root of the pattern must be a FusedBatchNorm or a FusedBatchNormV2.
@@ -281,8 +336,6 @@ bool FindFusedBatchNorm(const RemapperContext& ctx, const NodeDef* node,
   return true;
 }
 
-#undef REMAPPER_REQUIRES
-
 void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
                           const std::vector<string>& fused_ops = {},
                           int num_args = 1, float epsilon = 0.0) {
@@ -347,6 +400,37 @@ void AddFusedConv2DNode(
   invalidated_nodes->insert(matched.conv2d);
 }
 
+void AddFusedConv2DNode(
+    const Conv2DWithSqueezeAndBiasAdd& matched, GraphDef* optimized_graph,
+    absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
+  VLOG(2) << "Fuse Conv2D with Squeeze and BiasAdd: "
+          << " bias_add=" << matched.bias_add->name()
+          << " squeeze=" << matched.squeeze->name()
+          << " conv2d=" << matched.conv2d->name();
+
+  // Replace Conv2D node with a fused Conv2D. Matched pattern guarantees that it
+  // has single consumer (only the squeeze node).
+  NodeDef* fused_conv2d = optimized_graph->add_node();
+  fused_conv2d->set_name(matched.conv2d->name());
+  fused_conv2d->set_op("_FusedConv2D");
+  fused_conv2d->set_device(matched.conv2d->device());
+  fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
+  fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
+  fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
+
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd"});
+
+  // Replace BiasAdd node with a Squeeze.
+  NodeDef* remapped_squeeze = optimized_graph->add_node();
+  *remapped_squeeze = *matched.squeeze;
+  remapped_squeeze->set_name(matched.bias_add->name());
+  remapped_squeeze->set_input(0, fused_conv2d->name());
+
+  invalidated_nodes->insert(matched.squeeze);
+  invalidated_nodes->insert(matched.bias_add);
+  invalidated_nodes->insert(matched.conv2d);
+}
+
 void AddFusedConv2DNode(
     const Conv2DWithBatchNorm& matched, GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
@@ -552,6 +636,7 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
   Conv2DWithBiasAddAndRelu    conv2d_with_bias_and_relu;
   Conv2DWithBatchNorm         conv2d_with_batch_norm;
   Conv2DWithBatchNormAndRelu  conv2d_with_batch_norm_and_relu;
+  Conv2DWithSqueezeAndBiasAdd conv2d_with_squeeze_and_bias;
   // clang-format on
 
   // Processing graph in reverse-topological sorted order allows to remap
@@ -561,14 +646,15 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
   std::reverse(topo_sorted_graph.mutable_node()->begin(),
                topo_sorted_graph.mutable_node()->end());
 
-  RemapperContext ctx(item);
+  GrapplerItem topo_sorted_item(item, std::move(topo_sorted_graph));
+  RemapperContext ctx(topo_sorted_item);
 
   // Skip nodes that were invalidated by a remapper, e.g. do not process BiasAdd
   // and Relu nodes that were fused into a Conv2D node.
   absl::flat_hash_set<const NodeDef*> invalidated_nodes;
 
-  optimized_graph->mutable_node()->Reserve(item.graph.node_size());
-  for (const NodeDef& node : item.graph.node()) {
+  optimized_graph->mutable_node()->Reserve(topo_sorted_item.graph.node_size());
+  for (const NodeDef& node : topo_sorted_item.graph.node()) {
     // Check if node was invalidated by one of the previous remaps.
     if (invalidated_nodes.count(&node) > 0) continue;
 
@@ -585,6 +671,14 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
       continue;
     }
 
+    // Remap Conv2D+Squeeze+BiasAdd into the _FusedConv2D+Squeeze.
+    if (FindConv2DWithSqueezeAndBias(ctx, &node,
+                                     &conv2d_with_squeeze_and_bias)) {
+      AddFusedConv2DNode(conv2d_with_squeeze_and_bias, optimized_graph,
+                         &invalidated_nodes);
+      continue;
+    }
+
     // Remap Conv2D+FusedBatchNorm into the _FusedConv2D;
     if (FindConv2DWithBatchNorm(ctx, &node, &conv2d_with_batch_norm)) {
       AddFusedConv2DNode(conv2d_with_batch_norm, optimized_graph,
@@ -617,8 +711,8 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
     *optimized_graph->add_node() = node;
   }
 
-  *optimized_graph->mutable_library() = item.graph.library();
-  *optimized_graph->mutable_versions() = item.graph.versions();
+  *optimized_graph->mutable_library() = topo_sorted_item.graph.library();
+  *optimized_graph->mutable_versions() = topo_sorted_item.graph.versions();
 
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/optimizers/remapper_test.cc b/tensorflow/core/grappler/optimizers/remapper_test.cc
index 1536c91ef7..1711c3dbcd 100644
--- a/tensorflow/core/grappler/optimizers/remapper_test.cc
+++ b/tensorflow/core/grappler/optimizers/remapper_test.cc
@@ -367,5 +367,75 @@ TEST_F(RemapperTest, FuseConv2DWithBatchNormAndRelu) {
   test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
+TEST_F(RemapperTest, FuseConv2DWithSqueezeAndBias) {
+  using ops::Placeholder;
+
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  auto input_shape = ops::Placeholder::Shape({8, 32, 1, 3});
+  auto filter_shape = ops::Placeholder::Shape({1, 1, 3, 128});
+  auto bias_shape = ops::Placeholder::Shape({128});
+
+  auto input = Placeholder(s.WithOpName("input"), DT_FLOAT, input_shape);
+  auto filter = Placeholder(s.WithOpName("filter"), DT_FLOAT, filter_shape);
+  auto bias = Placeholder(s.WithOpName("bias"), DT_FLOAT, bias_shape);
+
+  std::vector<int> strides = {1, 1, 1, 1};
+  auto conv = ops::Conv2D(s.WithOpName("conv"), input, filter, strides, "SAME");
+
+  ops::Squeeze::Attrs attrs;
+  attrs = attrs.Axis({2});
+  auto squeeze = ops::Squeeze(s.WithOpName("squeeze"), conv, attrs);
+
+  auto bias_add = ops::BiasAdd(s.WithOpName("bias_add"), squeeze, bias);
+  auto fetch = ops::Identity(s.WithOpName("fetch"), bias_add);
+
+  auto input_t = GenerateRandomTensor<DT_FLOAT>({8, 32, 1, 3});
+  auto filter_t = GenerateRandomTensor<DT_FLOAT>({1, 1, 3, 128});
+  auto bias_t = GenerateRandomTensor<DT_FLOAT>({128});
+
+  GrapplerItem item;
+  item.fetch = {"fetch"};
+  item.feed = {{"input", input_t}, {"filter", filter_t}, {"bias", bias_t}};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  // Place all nodes on CPU.
+  for (int i = 0; i < item.graph.node_size(); ++i) {
+    item.graph.mutable_node(i)->set_device("/device:CPU:0");
+  }
+
+  Remapper optimizer(RewriterConfig::ON);
+  GraphDef output;
+  TF_CHECK_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int found = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "conv") {
+      EXPECT_EQ("_FusedConv2D", node.op());
+      EXPECT_EQ("input", node.input(0));
+      EXPECT_EQ("filter", node.input(1));
+
+      EXPECT_EQ(1, node.attr().at("num_args").i());
+      EXPECT_EQ("bias", node.input(2));
+
+      const auto fused_ops = node.attr().at("fused_ops").list().s();
+      ASSERT_EQ(1, fused_ops.size());
+      EXPECT_EQ("BiasAdd", fused_ops[0]);
+      found++;
+    } else if (node.name() == "bias_add") {
+      EXPECT_EQ("Squeeze", node.op());
+      EXPECT_EQ("conv", node.input(0));
+      found++;
+    }
+  }
+  EXPECT_EQ(2, found);
+
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed);
+  auto tensors = EvaluateNodes(output, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors_expected.size());
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
+}
+
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/python/grappler/tf_optimizer_test.py b/tensorflow/python/grappler/tf_optimizer_test.py
index 1321e1cb0d..0a4d4cbe2d 100644
--- a/tensorflow/python/grappler/tf_optimizer_test.py
+++ b/tensorflow/python/grappler/tf_optimizer_test.py
@@ -75,12 +75,13 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     optimized_graph = tf_optimizer.OptimizeGraph(config, mg)
 
     # Check that the nodes referenced in various collections have been preserved
-    self.assertEqual(len(optimized_graph.node), 5)
-    self.assertEqual(d.op.name, optimized_graph.node[0].name)
-    self.assertEqual(a1.op.name, optimized_graph.node[1].name)
-    self.assertEqual('Variable/initial_value', optimized_graph.node[2].name)
-    self.assertEqual(a2.op.name, optimized_graph.node[3].name)
-    self.assertEqual('Variable/Assign', optimized_graph.node[4].name)
+    optimized_graph_nodes = [node.name for node in optimized_graph.node]
+    expected_nodes = [
+        d.op.name, a1.op.name, a2.op.name, 'Variable/initial_value',
+        'Variable/Assign'
+    ]
+    self.assertEqual(len(optimized_graph_nodes), len(expected_nodes))
+    self.assertAllInSet(optimized_graph_nodes, expected_nodes)
 
   def testLoops(self):
     g = ops.Graph()
-- 
GitLab


From 68ad707cfedf1e93dc14e431b5417c77856214b6 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 16 Nov 2018 10:57:59 -0800
Subject: [PATCH 0414/1554] RL cartpole benchmark.

PiperOrigin-RevId: 221818834
---
 .../autograph/examples/benchmarks/BUILD       |  34 ++
 .../examples/benchmarks/benchmark_base.py     |  62 +++
 .../examples/benchmarks/cartpole_benchmark.py | 492 ++++++++++++++++++
 3 files changed, 588 insertions(+)
 create mode 100644 tensorflow/contrib/autograph/examples/benchmarks/BUILD
 create mode 100644 tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py
 create mode 100644 tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py

diff --git a/tensorflow/contrib/autograph/examples/benchmarks/BUILD b/tensorflow/contrib/autograph/examples/benchmarks/BUILD
new file mode 100644
index 0000000000..ed3b054774
--- /dev/null
+++ b/tensorflow/contrib/autograph/examples/benchmarks/BUILD
@@ -0,0 +1,34 @@
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow/tools/test:performance.bzl", "tf_py_logged_benchmark")
+
+py_library(
+    name = "benchmark_base",
+    srcs = [
+        "benchmark_base.py",
+    ],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_test(
+    name = "cartpole_benchmark",
+    size = "enormous",
+    srcs = ["cartpole_benchmark.py"],
+    tags = [
+        "local",
+        "notap",
+        "nozapfhahn",
+    ],
+    deps = [
+        ":benchmark_base",
+        # Note: required gym dependency may need to be added here.
+    ],
+)
+
+tf_py_logged_benchmark(
+    name = "cartpole_logged_benchmark",
+    target = "//tensorflow/contrib/autograph/examples/benchmarks:cartpole_benchmark",
+)
diff --git a/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py b/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py
new file mode 100644
index 0000000000..93c694849c
--- /dev/null
+++ b/tensorflow/contrib/autograph/examples/benchmarks/benchmark_base.py
@@ -0,0 +1,62 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Common benchmarking code.
+
+See https://www.tensorflow.org/community/benchmarks for usage.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+import tensorflow as tf
+
+
+class ReportingBenchmark(tf.test.Benchmark):
+  """Base class for a benchmark that reports general performance metrics.
+
+  Subclasses only need to call one of the _profile methods, and optionally
+  report_results.
+  """
+
+  def time_execution(self, name, target, iters, warm_up_iters=5):
+    for _ in range(warm_up_iters):
+      target()
+
+    all_times = []
+    for _ in range(iters):
+      iter_time = time.time()
+      target()
+      all_times.append(time.time() - iter_time)
+
+    avg_time = np.average(all_times)
+
+    extras = dict()
+    extras['all_times'] = all_times
+
+    if isinstance(name, tuple):
+      extras['name'] = name
+      name = '_'.join(str(piece) for piece in name)
+
+    self.report_benchmark(
+        iters=iters, wall_time=avg_time, name=name, extras=extras)
+
+
+if __name__ == '__main__':
+  tf.test.main()
diff --git a/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py b/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py
new file mode 100644
index 0000000000..4f553be58e
--- /dev/null
+++ b/tensorflow/contrib/autograph/examples/benchmarks/cartpole_benchmark.py
@@ -0,0 +1,492 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A basic RL cartpole benchmark.
+
+The RL model uses the OpenAI Gym environment to train a simple network using
+the policy gradients method. The training scales the gradients for each step
+by the episode's cumulative discounted reward and averages these gradients over
+a fixed number of games before applying the optimization step.
+
+For benchmarking purposes, we replace the OpenAI Gym environment to a fake
+that returns random actions and rewards and never ends the episode. This way
+the benchmarks compare the same amount of computation at each step.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gym
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.contrib import eager
+from tensorflow.contrib.autograph.examples.benchmarks import benchmark_base
+from tensorflow.python import autograph as ag
+from tensorflow.python.eager import context
+
+#
+# AutoGraph implementation
+#
+
+
+@ag.convert()
+def graph_append_discounted_rewards(destination, rewards, discount_rate):
+  """Discounts episode rewards and appends them to destination."""
+  ag.set_element_type(rewards, tf.float32)
+
+  cdr = 0.0
+  reverse_discounted = []
+  ag.set_element_type(reverse_discounted, tf.float32)
+
+  for i in range(len(rewards) - 1, -1, -1):
+    cdr = cdr * discount_rate + rewards[i]
+    cdr.set_shape(())
+    reverse_discounted.append(cdr)
+
+  retval = destination
+  # Note: AutoGraph doesn't yet support reversed() so we use a loop instead.
+  for i in range(len(reverse_discounted) - 1, -1, -1):
+    retval.append(reverse_discounted[i])
+
+  return retval
+
+
+class GraphPolicyNetwork(tf.keras.Model):
+  """Policy network for the cart-pole reinforcement learning problem.
+
+  The forward path of the network takes an observation from the cart-pole
+  environment (length-4 vector) and outputs an action.
+  """
+
+  def __init__(self, hidden_size):
+    super(GraphPolicyNetwork, self).__init__()
+    self._hidden_layer = tf.keras.layers.Dense(
+        hidden_size, activation=tf.nn.elu)
+    self._output_layer = tf.keras.layers.Dense(1)
+
+  def call(self, inputs):
+    """Calculates logits and action.
+
+    Args:
+      inputs: Observations from a step in the cart-pole environment, of shape
+        `(batch_size, input_size)`
+
+    Returns:
+      logits: the logits output by the output layer. This can be viewed as the
+        likelihood vales of choosing the left (0) action. Shape:
+        `(batch_size, 1)`.
+      actions: randomly selected actions ({0, 1}) based on the logits. Shape:
+        `(batch_size, 1)`.
+    """
+    hidden = self._hidden_layer(inputs)
+    logits = self._output_layer(hidden)
+
+    left_prob = tf.nn.sigmoid(logits)
+    action_probs = tf.concat([left_prob, 1.0 - left_prob], 1)
+
+    actions = tf.multinomial(tf.log(action_probs), 1)
+    return logits, actions
+
+  # TODO(mdan): Move this method out of the class.
+  @ag.convert()
+  def train(self, cart_pole_env, optimizer, discount_rate, num_games,
+            max_steps_per_game):
+    var_list = tf.trainable_variables()
+    grad_list = [
+        tf.TensorArray(tf.float32, 0, dynamic_size=True) for _ in var_list
+    ]
+
+    step_counts = []
+    discounted_rewards = []
+    ag.set_element_type(discounted_rewards, tf.float32)
+    ag.set_element_type(step_counts, tf.int32)
+
+    # Note: we use a shared object, cart_pole_env here. Because calls to the
+    # object's method are made through py_func, TensorFlow cannot detect its
+    # data dependencies. Hence we must manually synchronize access to it
+    # and ensure the control dependencies are set in such a way that
+    # calls to reset(), take_one_step, etc. are made in the correct order.
+    sync_counter = tf.constant(0)
+
+    for _ in tf.range(num_games):
+      with tf.control_dependencies([sync_counter]):
+        obs = cart_pole_env.reset()
+        with tf.control_dependencies([obs]):
+          sync_counter += 1
+
+        game_rewards = []
+        ag.set_element_type(game_rewards, tf.float32)
+
+        for step in tf.range(max_steps_per_game):
+          logits, actions = self(obs)  # pylint:disable=not-callable
+          logits = tf.reshape(logits, ())
+          actions = tf.reshape(actions, ())
+
+          labels = 1.0 - tf.cast(actions, tf.float32)
+          loss = tf.nn.sigmoid_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+          grads = tf.gradients(loss, var_list)
+
+          for i in range(len(grads)):
+            grad_list[i].append(grads[i])
+
+          with tf.control_dependencies([sync_counter]):
+            obs, reward, done = cart_pole_env.step(actions)
+            with tf.control_dependencies([obs]):
+              sync_counter += 1
+            obs = tf.reshape(obs, (1, 4))
+
+          game_rewards.append(reward)
+          if reward < 0.1 or done:
+            step_counts.append(step + 1)
+            break
+
+        discounted_rewards = graph_append_discounted_rewards(
+            discounted_rewards, game_rewards, discount_rate)
+
+    discounted_rewards = ag.stack(discounted_rewards)
+    discounted_rewards.set_shape((None,))
+    mean, variance = tf.nn.moments(discounted_rewards, [0])
+    normalized_rewards = (discounted_rewards - mean) / tf.sqrt(variance)
+
+    for i in range(len(grad_list)):
+      g = ag.stack(grad_list[i])
+
+      # This block just adjusts the shapes to match for multiplication.
+      r = normalized_rewards
+      if r.shape.ndims < g.shape.ndims:
+        r = tf.expand_dims(r, -1)
+      if r.shape.ndims < g.shape.ndims:
+        r = tf.expand_dims(r, -1)
+
+      grad_list[i] = tf.reduce_mean(g * r, axis=0)
+
+    optimizer.apply_gradients(
+        zip(grad_list, var_list), global_step=tf.train.get_global_step())
+
+    return ag.stack(step_counts)
+
+
+@ag.convert()
+def graph_train_model(policy_network, cart_pole_env, optimizer, iterations):
+  """Trains the policy network for a given number of iterations."""
+  i = tf.constant(0)
+  mean_steps_per_iteration = []
+  ag.set_element_type(mean_steps_per_iteration, tf.int32)
+
+  while i < iterations:
+    steps_per_game = policy_network.train(
+        cart_pole_env,
+        optimizer,
+        discount_rate=0.95,
+        num_games=20,
+        max_steps_per_game=200)
+    mean_steps_per_iteration.append(tf.reduce_mean(steps_per_game))
+    i += 1
+
+  return ag.stack(mean_steps_per_iteration)
+
+
+class GraphGymCartpoleEnv(object):
+  """An env backed by OpenAI Gym's CartPole environment.
+
+  Used to confirm a functional model only.
+  """
+
+  def __init__(self):
+    cart_pole_env = gym.make('CartPole-v1')
+    cart_pole_env.seed(0)
+    cart_pole_env.reset()
+    self.env = cart_pole_env
+
+  def reset(self):
+    obs = ag.utils.wrap_py_func(self.env.reset, tf.float64, ())
+    obs = tf.reshape(obs, (1, 4))
+    obs = tf.cast(obs, tf.float32)
+    return obs
+
+  def step(self, actions):
+
+    def take_one_step(actions):
+      obs, reward, done, _ = self.env.step(actions)
+      obs = obs.astype(np.float32)
+      reward = np.float32(reward)
+      return obs, reward, done
+
+    return ag.utils.wrap_py_func(take_one_step,
+                                 (tf.float32, tf.float32, tf.bool), (actions,))
+
+
+class GraphRandomCartpoleEnv(object):
+  """An environment that returns random actions and never finishes.
+
+  Used during benchmarking, it will cause training to run a constant number of
+  steps.
+  """
+
+  def reset(self):
+    return tf.random.normal((1, 4))
+
+  def step(self, actions):
+    with tf.control_dependencies([actions]):
+      random_obs = tf.random.normal((1, 4))
+      fixed_reward = tf.constant(0.001)
+      done = tf.constant(False)
+      return random_obs, fixed_reward, done
+
+
+#
+# Eager implementation
+#
+
+
+def eager_append_discounted_rewards(discounted_rewards, rewards, discount_rate):
+  cdr = 0.0
+  reverse_discounted = []
+
+  for i in range(len(rewards) - 1, -1, -1):
+    cdr = cdr * discount_rate + rewards[i]
+    reverse_discounted.append(cdr)
+
+  discounted_rewards.extend(reversed(reverse_discounted))
+  return discounted_rewards
+
+
+class EagerPolicyNetwork(tf.keras.Model):
+  """Policy network for the cart-pole reinforcement learning problem.
+
+  The forward path of the network takes an observation from the cart-pole
+  environment (length-4 vector) and outputs an action.
+  """
+
+  def __init__(self, hidden_size):
+    super(EagerPolicyNetwork, self).__init__()
+    self._hidden_layer = tf.keras.layers.Dense(
+        hidden_size, activation=tf.nn.elu)
+    self._output_layer = tf.keras.layers.Dense(1)
+
+  def call(self, inputs):
+    """Calculates logits and action.
+
+    Args:
+      inputs: Observations from a step in the cart-pole environment, of shape
+        `(batch_size, input_size)`
+
+    Returns:
+      logits: the logits output by the output layer. This can be viewed as the
+        likelihood vales of choosing the left (0) action. Shape:
+        `(batch_size, 1)`.
+      actions: randomly selected actions ({0, 1}) based on the logits. Shape:
+        `(batch_size, 1)`.
+    """
+    hidden = self._hidden_layer(inputs)
+    logits = self._output_layer(hidden)
+
+    left_prob = tf.nn.sigmoid(logits)
+    action_probs = tf.concat([left_prob, 1.0 - left_prob], 1)
+
+    self._grad_fn = eager.implicit_gradients(
+        self._get_cross_entropy_and_save_actions)
+
+    actions = tf.multinomial(tf.log(action_probs), 1)
+    return logits, actions
+
+  def _get_cross_entropy_and_save_actions(self, inputs):
+    logits, actions = self(inputs)  # pylint:disable=not-callable
+    self._current_actions = actions
+    labels = 1.0 - tf.cast(actions, tf.float32)
+    return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)
+
+  def train(self, cart_pole_env, optimizer, discount_rate, num_games,
+            max_steps_per_game):
+    grad_list = None
+
+    step_counts = []
+    discounted_rewards = []
+
+    for _ in range(num_games):
+      obs = cart_pole_env.reset()
+
+      game_rewards = []
+
+      for step in range(max_steps_per_game):
+        grads_and_vars = self._grad_fn(tf.constant([obs], dtype=tf.float32))
+        grads, var_list = zip(*grads_and_vars)
+        actions = self._current_actions.numpy()[0][0]
+
+        if grad_list is None:
+          grad_list = [[g] for g in grads]
+        else:
+          for i in range(len(grads)):
+            grad_list[i].append(grads[i])
+
+        obs, reward, done = cart_pole_env.step(actions)
+
+        game_rewards.append(reward)
+        if reward < 0.1 or done:
+          step_counts.append(step + 1)
+          break
+
+      discounted_rewards = eager_append_discounted_rewards(
+          discounted_rewards, game_rewards, discount_rate)
+
+    discounted_rewards = tf.stack(discounted_rewards)
+    mean, variance = tf.nn.moments(discounted_rewards, [0])
+    normalized_rewards = (discounted_rewards - mean) / tf.sqrt(variance)
+
+    for i in range(len(grad_list)):
+      g = tf.stack(grad_list[i])
+
+      r = normalized_rewards
+      while r.shape.ndims < g.shape.ndims:
+        r = tf.expand_dims(r, -1)
+
+      grad_list[i] = tf.reduce_mean(g * r, axis=0)
+
+    optimizer.apply_gradients(
+        zip(grad_list, var_list), global_step=tf.train.get_global_step())
+
+    return tf.stack(step_counts)
+
+
+def eager_train_model(policy_network, cart_pole_env, optimizer, iterations):
+  """Trains the policy network for a given number of iterations."""
+  mean_steps_per_iteration = []
+
+  for _ in range(iterations):
+    steps_per_game = policy_network.train(
+        cart_pole_env,
+        optimizer,
+        discount_rate=0.95,
+        num_games=20,
+        max_steps_per_game=200)
+    mean_steps_per_iteration.append(tf.reduce_mean(steps_per_game))
+
+  return mean_steps_per_iteration
+
+
+class EagerGymCartpoleEnv(object):
+  """An env backed by OpenAI Gym's CartPole environment.
+
+  Used to confirm a functional model only.
+  """
+
+  def __init__(self):
+    cart_pole_env = gym.make('CartPole-v1')
+    cart_pole_env.seed(0)
+    cart_pole_env.reset()
+    self.env = cart_pole_env
+
+  def reset(self):
+    return self.env.reset()
+
+  def step(self, actions):
+    obs, reward, done, _ = self.env.step(actions)
+    return obs, reward, done
+
+
+class EagerRandomCartpoleEnv(object):
+  """An environment that returns random actions and never finishes.
+
+  Used during benchmarking, it will cause training to run a constant number of
+  steps.
+  """
+
+  def reset(self):
+    return np.random.normal(size=(4,))
+
+  def step(self, actions):
+    with tf.control_dependencies([actions]):
+      random_obs = np.random.normal(size=(4,))
+      fixed_reward = 0.001
+      done = False
+      return random_obs, fixed_reward, done
+
+
+def graph_demo_training():
+  """Not used in the benchmark. Used to confirm a functional model."""
+  with tf.Graph().as_default():
+    tf.set_random_seed(0)
+
+    network = GraphPolicyNetwork(hidden_size=5)
+    network.build((1, 4))
+    env = GraphGymCartpoleEnv()
+    opt = tf.train.AdamOptimizer(0.05)
+
+    train_ops = graph_train_model(network, env, opt, iterations=5)
+
+    with tf.Session() as sess:
+      sess.run(tf.global_variables_initializer())
+      sess.run(tf.local_variables_initializer())
+      steps_per_iteration = sess.run(train_ops)
+      for i, steps in enumerate(steps_per_iteration):
+        print('Step {} iterations: {}'.format(i, steps))
+
+
+def eager_demo_training():
+  with context.eager_mode():
+    network = EagerPolicyNetwork(hidden_size=5)
+    network.build((1, 4))
+    env = EagerGymCartpoleEnv()
+    opt = tf.train.AdamOptimizer(0.05)
+
+    steps_per_iteration = eager_train_model(network, env, opt, iterations=5)
+    for i, steps in enumerate(steps_per_iteration):
+      print('Step {} iterations: {}'.format(i, steps))
+
+
+class RLCartPoleBenchmark(benchmark_base.ReportingBenchmark):
+  """Actual benchmark.
+
+  Trains the RL agent a fixed number of times, on random environments that
+  result in constant number of steps.
+  """
+
+  def benchmark_cartpole(self):
+
+    def train_session(sess, ops):
+      return lambda: sess.run(ops)
+
+    def train_eager(network, env, opt):
+      return lambda: eager_train_model(network, env, opt, iterations=10)
+
+    for model_size in (10, 100, 1000):
+      with tf.Graph().as_default():
+        network = GraphPolicyNetwork(hidden_size=model_size)
+        network.build((1, 4))
+        env = GraphRandomCartpoleEnv()
+        opt = tf.train.AdamOptimizer(0.05)
+        train_ops = graph_train_model(network, env, opt, iterations=10)
+
+        with tf.Session() as sess:
+          sess.run(tf.global_variables_initializer())
+          sess.run(tf.local_variables_initializer())
+
+          self.time_execution(('cartpole', 'autograph', model_size),
+                              train_session(sess, train_ops), 20)
+
+      with context.eager_mode():
+        network = EagerPolicyNetwork(hidden_size=model_size)
+        network.build((1, 4))
+        env = EagerRandomCartpoleEnv()
+        opt = tf.train.AdamOptimizer(0.05)
+
+        self.time_execution(('cartpole', 'eager', model_size),
+                            train_eager(network, env, opt), 20)
+
+
+if __name__ == '__main__':
+  tf.test.main()
-- 
GitLab


From 61e236f99f8c7ec6a3236179e7277e7b5183c77e Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Fri, 16 Nov 2018 11:11:49 -0800
Subject: [PATCH 0415/1554] Update the argument order for
 `tf.nn.sparse.segment_*` and exporting in v2.

PiperOrigin-RevId: 221821515
---
 tensorflow/python/ops/math_ops.py             | 97 ++++++++++++++++---
 .../api/golden/v2/tensorflow.sparse.pbtxt     |  6 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  9 ++
 3 files changed, 96 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index a769c0dff4..679ce07f3f 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2984,8 +2984,7 @@ def unsorted_segment_sqrt_n(data, segment_ids, num_segments, name=None):
     return summed / gen_math_ops.sqrt(N)
 
 
-@tf_export(
-    "sparse.segment_sum", v1=["sparse.segment_sum", "sparse_segment_sum"])
+@tf_export(v1=["sparse.segment_sum", "sparse_segment_sum"])
 @deprecation.deprecated_endpoints("sparse_segment_sum")
 def sparse_segment_sum(data, indices, segment_ids, name=None,
                        num_segments=None):
@@ -3059,8 +3058,17 @@ def sparse_segment_sum(data, indices, segment_ids, name=None,
         data=data, indices=indices, segment_ids=segment_ids, name=name)
 
 
-@tf_export(
-    "sparse.segment_mean", v1=["sparse.segment_mean", "sparse_segment_mean"])
+@tf_export("sparse.segment_sum", v1=[])
+def sparse_segment_sum_v2(data,
+                          indices,
+                          segment_ids,
+                          num_segments=None,
+                          name=None):
+  return sparse_segment_mean(
+      data, indices, segment_ids, name=name, num_segments=num_segments)
+
+
+@tf_export(v1=["sparse.segment_mean", "sparse_segment_mean"])
 @deprecation.deprecated_endpoints("sparse_segment_mean")
 def sparse_segment_mean(data,
                         indices,
@@ -3106,9 +3114,44 @@ def sparse_segment_mean(data,
         data=data, indices=indices, segment_ids=segment_ids, name=name)
 
 
-@tf_export(
-    "sparse.segment_sqrt_n",
-    v1=["sparse.segment_sqrt_n", "sparse_segment_sqrt_n"])
+@tf_export("sparse.segment_mean", v1=[])
+def sparse_segment_mean_v2(data,
+                           indices,
+                           segment_ids,
+                           num_segments=None,
+                           name=None):
+  r"""Computes the mean along sparse segments of a tensor.
+
+  Read [the section on
+  segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+  for an explanation of segments.
+
+  Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
+  dimension, selecting a subset of dimension 0, specified by `indices`.
+  `segment_ids` is allowed to have missing ids, in which case the output will
+  be zeros at those indices. In those cases `num_segments` is used to determine
+  the size of the output.
+
+  Args:
+    data: A `Tensor` with data that will be assembled in the output.
+    indices: A 1-D `Tensor` with indices into `data`. Has same rank as
+      `segment_ids`.
+    segment_ids: A 1-D `Tensor` with indices into the output `Tensor`. Values
+      should be sorted and can be repeated.
+    num_segments: An optional int32 scalar. Indicates the size of the output
+      `Tensor`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `tensor` of the shape as data, except for dimension 0 which
+    has size `k`, the number of segments specified via `num_segments` or
+    inferred for the last element in `segments_ids`.
+  """
+  return sparse_segment_mean(
+      data, indices, segment_ids, name=name, num_segments=num_segments)
+
+
+@tf_export(v1=["sparse.segment_sqrt_n", "sparse_segment_sqrt_n"])
 @deprecation.deprecated_endpoints("sparse_segment_sqrt_n")
 def sparse_segment_sqrt_n(data,
                           indices,
@@ -3146,6 +3189,35 @@ def sparse_segment_sqrt_n(data,
         data=data, indices=indices, segment_ids=segment_ids, name=name)
 
 
+@tf_export("sparse.segment_sqrt_n", v1=[])
+def sparse_segment_sqrt_n_v2(data,
+                             indices,
+                             segment_ids,
+                             num_segments=None,
+                             name=None):
+  r"""Computes the sum along sparse segments of a tensor divided by the sqrt(N).
+
+  `N` is the size of the segment being reduced.
+
+  Args:
+    data: A `Tensor` with data that will be assembled in the output.
+    indices: A 1-D `Tensor` with indices into `data`. Has same rank as
+      `segment_ids`.
+    segment_ids: A 1-D `Tensor` with indices into the output `Tensor`. Values
+      should be sorted and can be repeated.
+    num_segments: An optional int32 scalar. Indicates the size of the output
+      `Tensor`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `tensor` of the shape as data, except for dimension 0 which
+    has size `k`, the number of segments specified via `num_segments` or
+    inferred for the last element in `segments_ids`.
+  """
+  return sparse_segment_sqrt_n(
+      data, indices, segment_ids, name=name, num_segments=num_segments)
+
+
 @tf_export("tensordot", "linalg.tensordot")
 def tensordot(a, b, axes, name=None):
   r"""Tensor contraction of a and b along specified axes.
@@ -3179,12 +3251,11 @@ def tensordot(a, b, axes, name=None):
     a: `Tensor` of type `float32` or `float64`.
     b: `Tensor` with the same type as `a`.
     axes: Either a scalar `N`, or a list or an `int32` `Tensor` of shape [2, k].
-     If axes is a scalar, sum over the last N axes of a and the first N axes
-     of b in order.
-     If axes is a list or `Tensor` the first and second row contain the set of
-     unique integers specifying axes along which the contraction is computed,
-     for `a` and `b`, respectively. The number of axes for `a` and `b` must
-     be equal.
+      If axes is a scalar, sum over the last N axes of a and the first N axes of
+      b in order. If axes is a list or `Tensor` the first and second row contain
+      the set of unique integers specifying axes along which the contraction is
+      computed, for `a` and `b`, respectively. The number of axes for `a` and
+      `b` must be equal.
     name: A name for the operation (optional).
 
   Returns:
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index bb86f07f1e..48aa1f7d8d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -90,15 +90,15 @@ tf_module {
   }
   member_method {
     name: "segment_mean"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "segment_sqrt_n"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "segment_sum"
-    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'name\', \'num_segments\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'data\', \'indices\', \'segment_ids\', \'num_segments\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "slice"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3bfefd3623..be6dbc1ff8 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -174,6 +174,15 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
+        "tf.sparse.segment_mean": [
+            "data", "indices", "segment_ids", "name", "num_segments"
+        ],
+        "tf.sparse.segment_sqrt_n": [
+            "data", "indices", "segment_ids", "name", "num_segments"
+        ],
+        "tf.sparse.segment_sum": [
+            "data", "indices", "segment_ids", "name", "num_segments"
+        ],
     }
 
     # Specially handled functions.
-- 
GitLab


From 4070b1650c68fb88a5f22e910e193650fb2be515 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 11:15:47 -0800
Subject: [PATCH 0416/1554] Changed remaining tests in
 BatchMatrixTransposeTest.

PiperOrigin-RevId: 221822243
---
 tensorflow/python/kernel_tests/BUILD          |  1 +
 .../python/kernel_tests/array_ops_test.py     | 44 ++++++++++---------
 2 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 975b603006..307330d1cb 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1350,6 +1350,7 @@ cuda_py_test(
         "//tensorflow/python:test_ops",
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:def_function",
     ],
     shard_count = 10,
     tags = [
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 5caed54c4d..da29a070cd 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -25,6 +25,7 @@ import numpy as np
 
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -32,6 +33,7 @@ from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
@@ -46,9 +48,9 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test as test_lib
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
 
-  @test_util.run_in_graph_and_eager_modes
   def testNonBatchMatrix(self):
     matrix = [[1, 2, 3], [4, 5, 6]]  # Shape (2, 3)
     expected_transposed = [[1, 4], [2, 5], [3, 6]]  # Shape (3, 2)
@@ -56,7 +58,6 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
     self.assertEqual((3, 2), transposed.get_shape())
     self.assertAllEqual(expected_transposed, transposed)
 
-  @test_util.run_in_graph_and_eager_modes
   def testConjugate(self):
     m = [[1 + 1j, 2 + 2j, 3 + 3j], [4 + 4j, 5 + 5j, 6 + 6j]]
     expected_transposed = [[1 - 1j, 4 - 4j], [2 - 2j, 5 - 5j], [3 - 3j, 6 - 6j]]
@@ -65,7 +66,6 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
     self.assertEqual((3, 2), transposed.get_shape())
     self.assertAllEqual(expected_transposed, transposed)
 
-  @test_util.run_in_graph_and_eager_modes
   def testBatchMatrix(self):
     matrix_0 = [[1, 2, 3], [4, 5, 6]]
     matrix_0_t = [[1, 4], [2, 5], [3, 6]]
@@ -78,33 +78,35 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_transposed, transposed)
 
   def testNonBatchMatrixDynamicallyDefined(self):
-    matrix = [[1, 2, 3], [4, 5, 6]]  # Shape (2, 3)
+    # needs explicit `constant` because lists are not automatically
+    # converted to sensors when applying `transpose` below
+    matrix = constant_op.constant([[1, 2, 3], [4, 5, 6]])  # Shape (2, 3)
     expected_transposed = [[1, 4], [2, 5], [3, 6]]  # Shape (3, 2)
-    with self.cached_session():
-      matrix_ph = array_ops.placeholder(dtypes.int32)
-      transposed = array_ops.matrix_transpose(matrix_ph)
-      self.assertAllEqual(
-          expected_transposed, transposed.eval(feed_dict={
-              matrix_ph: matrix
-          }))
+    @def_function.function(input_signature=
+                           [tensor_spec.TensorSpec
+                            (shape=None, dtype=dtypes.int32)])
+    def transpose(matrix):
+      self.assertIs(matrix.shape.ndims, None)
+      return array_ops.matrix_transpose(matrix)
+    self.assertAllEqual(expected_transposed, transpose(matrix))
 
   def testBatchMatrixDynamicallyDefined(self):
     matrix_0 = [[1, 2, 3], [4, 5, 6]]
     matrix_0_t = [[1, 4], [2, 5], [3, 6]]
     matrix_1 = [[11, 22, 33], [44, 55, 66]]
     matrix_1_t = [[11, 44], [22, 55], [33, 66]]
-    batch_matrix = [matrix_0, matrix_1]  # Shape (2, 2, 3)
+    # needs explicit `constant` because lists are not automatically
+    # converted to sensors when applying `transpose` below
+    batch_matrix = constant_op.constant([matrix_0, matrix_1])  # Shape (2, 2, 3)
     expected_transposed = [matrix_0_t, matrix_1_t]  # Shape (2, 3, 2)
-    with self.cached_session():
-      batch_matrix_ph = array_ops.placeholder(dtypes.int32)
-      transposed = array_ops.matrix_transpose(batch_matrix_ph)
-      self.assertAllEqual(
-          expected_transposed,
-          transposed.eval(feed_dict={
-              batch_matrix_ph: batch_matrix
-          }))
+    @def_function.function(input_signature=
+                           [tensor_spec.TensorSpec
+                            (shape=None, dtype=dtypes.int32)])
+    def transpose(matrix):
+      self.assertIs(matrix.shape.ndims, None)
+      return array_ops.matrix_transpose(matrix)
+    self.assertAllEqual(expected_transposed, transpose(batch_matrix))
 
-  @test_util.run_in_graph_and_eager_modes
   def testTensorWithStaticRankLessThanTwoRaisesBecauseNotAMatrix(self):
     vector = [1, 2, 3]
     with self.assertRaisesRegexp(ValueError, "should be a "):
-- 
GitLab


From 74660a6db35c909a49c073b9f992c1f8fa0d20c3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 11:53:30 -0800
Subject: [PATCH 0417/1554] Add broadcasting support for
 tf.ragged.to_tensor()'s default_value argument.

PiperOrigin-RevId: 221828849
---
 .../ops/ragged/ragged_conversion_ops.py       | 18 +++++--
 .../ops/ragged/ragged_to_tensor_op_test.py    | 47 +++++++++++++------
 2 files changed, 47 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/ops/ragged/ragged_conversion_ops.py b/tensorflow/python/ops/ragged/ragged_conversion_ops.py
index 3ec246ccaf..0385be02d4 100644
--- a/tensorflow/python/ops/ragged/ragged_conversion_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_conversion_ops.py
@@ -196,7 +196,7 @@ def to_tensor(rt_input, default_value=None, name=None):
   Args:
     rt_input: The input `RaggedTensor`.
     default_value: Value to set for indices not specified in `rt_input`.
-      Defaults to zero.  `default_value.shape` must be equal to
+      Defaults to zero.  `default_value` must be broadcastable to
       `rt_input.shape[rt_input.ragged_rank + 1:]`.
     name: A name prefix for the returned tensors (optional).
 
@@ -210,6 +210,9 @@ def to_tensor(rt_input, default_value=None, name=None):
         rt_input, name='rt_input')
     if not ragged_tensor.is_ragged(rt_input):
       return rt_input  # already dense
+    if default_value is not None:
+      default_value = ops.convert_to_tensor(
+          default_value, name='default_value', dtype=rt_input.dtype)
 
     # If ragged_rank > 1, then recursively convert the ragged values into a
     # `Tensor` before we proceed.
@@ -217,6 +220,16 @@ def to_tensor(rt_input, default_value=None, name=None):
     if ragged_tensor.is_ragged(values):
       values = to_tensor(values, default_value)
 
+    # Tile the default value, if necessary.
+    if default_value is not None:
+      if values.shape.ndims is not None:
+        default_value.shape.with_rank_at_most(values.shape.ndims - 1)
+      if (values.shape.ndims is None or default_value.shape.ndims is None or
+          values.shape.ndims != default_value.shape.ndims + 1):
+        value_shape = array_ops.shape(values)[1:]
+        default_value = array_ops.broadcast_to(default_value, value_shape)
+      default_value.shape.assert_is_compatible_with(values.shape[1:])
+
     # Get the expected dense shape ([nrows, ncols] + value_shape).
     rt_row_lengths = [rt_input.row_splits[1:] - rt_input.row_splits[:-1]]
     nrows = array_ops.shape(rt_input.row_splits, out_type=dtypes.int64)[0] - 1
@@ -228,9 +241,6 @@ def to_tensor(rt_input, default_value=None, name=None):
     # Build a default value if none was supplied.
     if default_value is None:
       default_value = array_ops.zeros(value_shape, dtype=values.dtype)
-    else:
-      default_value = ops.convert_to_tensor(
-          default_value, name='default_value', dtype=values.dtype)
     default_value.shape.assert_is_compatible_with(values.shape[1:])
     default_value.set_shape(values.shape[1:])
 
diff --git a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
index 0ccc214a9c..688676e46c 100644
--- a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
@@ -71,8 +71,30 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
               [[1, 2], [0, 0], [3, 4]],  #
               [[0, 0], [0, 0], [0, 0]],  #
               [[5, 0], [0, 0], [0, 0]],  #
-              [[6, 7], [8, 0], [0, 0]]
-          ]  #
+              [[6, 7], [8, 0], [0, 0]],  #
+          ]
+      },
+      {
+          'rt_input': [[[1, 2], [], [3, 4]], [], [[5]], [[6, 7], [8]]],
+          'default':
+              9,
+          'expected': [
+              [[1, 2], [9, 9], [3, 4]],  #
+              [[9, 9], [9, 9], [9, 9]],  #
+              [[5, 9], [9, 9], [9, 9]],  #
+              [[6, 7], [8, 9], [9, 9]],  #
+          ]
+      },
+      {
+          'rt_input': [[[1], [2], [3]]],
+          'ragged_rank': 1,
+          'default': 0,
+          'expected': [[[1], [2], [3]]],
+      },
+      {
+          'rt_input': [[[[1], [2]], [], [[3]]]],
+          'default': 9,
+          'expected': [[[[1], [2]], [[9], [9]], [[3], [9]]]],
       },
   )
   def testRaggedTensorToTensor(self,
@@ -96,17 +118,13 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
       {
           'rt_input': [[1, 2, 3]],
           'default': [0],
-          'error': (ValueError, r'Shapes \(1,\) and \(\) are incompatible'),
-      },
-      {
-          'rt_input': [[[1], [2], [3]]],
-          'default': 0,
-          'error': (ValueError, r'Shapes \(\) and \(1,\) are incompatible'),
+          'error': (ValueError, r'Shape \(1,\) must have rank at most 0'),
       },
       {
-          'rt_input': [[[[1], [2]], [], [[3]]]],
-          'default': 0,
-          'error': (ValueError, r'Shapes \(\) and \(1,\) are incompatible'),
+          'rt_input': [[[1, 2], [3, 4]], [[5, 6]]],
+          'ragged_rank': 1,
+          'default': [7, 8, 9],
+          'error': (ValueError, r'Shapes \(3,\) and \(2,\) are incompatible'),
       },
       {
           'rt_input': [[1, 2, 3]],
@@ -114,9 +132,10 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
           'error': (TypeError, "Expected int32, got 'a' of type 'str' instead"),
       },
   )
-  def testError(self, rt_input, default, error):
-    rt = ragged.constant(rt_input)
-    self.assertRaisesRegexp(error[0], error[1], ragged.to_tensor, rt, default)
+  def testError(self, rt_input, default, error, ragged_rank=None):
+    rt = ragged.constant(rt_input, ragged_rank=ragged_rank)
+    with self.assertRaisesRegexp(error[0], error[1]):
+      ragged.to_tensor(rt, default)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 36035b92300e2183357c70b18b5400e656bf958b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 11:54:10 -0800
Subject: [PATCH 0418/1554] A round of moving some DistributionStrategy
 libraries from contrib to core (just implementations, not public APIs):   *
 cross_tower_ops -> cross_device_ops   * cross_tower_utils ->
 cross_device_utils   * input_ops (and test)   * shared_variable_creator (and
 test)   * values

Also:
  * BUILD clean up
  * renaming cross_tower -> cross_device in a number of places.
PiperOrigin-RevId: 221828945
---
 tensorflow/contrib/distribute/BUILD           |   2 +-
 tensorflow/contrib/distribute/__init__.py     |   2 +-
 tensorflow/contrib/distribute/python/BUILD    | 163 ++++--------------
 .../python/collective_all_reduce_strategy.py  |  14 +-
 .../collective_all_reduce_strategy_test.py    |   6 +-
 ...r_ops_test.py => cross_device_ops_test.py} |  98 +++++------
 ...ils_test.py => cross_device_utils_test.py} |  26 +--
 .../contrib/distribute/python/keras_test.py   |   2 +-
 .../distribute/python/mirrored_strategy.py    |  44 ++---
 .../python/mirrored_strategy_multigpu_test.py |   2 +-
 .../distribute/python/one_device_strategy.py  |   2 +-
 .../python/parameter_server_strategy.py       |  20 +--
 .../python/parameter_server_strategy_test.py  |   2 +-
 .../contrib/distribute/python/tpu_strategy.py |   6 +-
 .../contrib/distribute/python/values_test.py  |   2 +-
 tensorflow/python/distribute/BUILD            | 108 +++++++++++-
 .../distribute/cross_device_ops.py}           |  40 ++---
 .../distribute/cross_device_utils.py}         |   4 +-
 .../python => python/distribute}/input_ops.py |   0
 .../distribute}/input_ops_test.py             |   2 +-
 .../distribute}/shared_variable_creator.py    |   0
 .../shared_variable_creator_test.py           |   2 +-
 .../python => python/distribute}/values.py    |   2 +-
 23 files changed, 268 insertions(+), 281 deletions(-)
 rename tensorflow/contrib/distribute/python/{cross_tower_ops_test.py => cross_device_ops_test.py} (87%)
 rename tensorflow/contrib/distribute/python/{cross_tower_utils_test.py => cross_device_utils_test.py} (84%)
 rename tensorflow/{contrib/distribute/python/cross_tower_ops.py => python/distribute/cross_device_ops.py} (96%)
 rename tensorflow/{contrib/distribute/python/cross_tower_utils.py => python/distribute/cross_device_utils.py} (99%)
 rename tensorflow/{contrib/distribute/python => python/distribute}/input_ops.py (100%)
 rename tensorflow/{contrib/distribute/python => python/distribute}/input_ops_test.py (99%)
 rename tensorflow/{contrib/distribute/python => python/distribute}/shared_variable_creator.py (100%)
 rename tensorflow/{contrib/distribute/python => python/distribute}/shared_variable_creator_test.py (97%)
 rename tensorflow/{contrib/distribute/python => python/distribute}/values.py (99%)

diff --git a/tensorflow/contrib/distribute/BUILD b/tensorflow/contrib/distribute/BUILD
index a87a5624c8..3ecd755d86 100644
--- a/tensorflow/contrib/distribute/BUILD
+++ b/tensorflow/contrib/distribute/BUILD
@@ -26,7 +26,6 @@ py_library(
     visibility = ["//tensorflow:internal"],
     deps = [
         "//tensorflow/contrib/distribute/python:collective_all_reduce_strategy",
-        "//tensorflow/contrib/distribute/python:cross_tower_ops",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
         "//tensorflow/contrib/distribute/python:monitor",
         "//tensorflow/contrib/distribute/python:one_device_strategy",
@@ -35,6 +34,7 @@ py_library(
         "//tensorflow/contrib/distribute/python:tpu_strategy",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
+        "//tensorflow/python/distribute:cross_device_ops",
         "//tensorflow/python/distribute:distribute_config",
         "//tensorflow/python/distribute:distribute_coordinator",
     ],
diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 2853bbdb11..8ec73654e3 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -25,13 +25,13 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,wildcard-import
 from tensorflow.contrib.distribute.python.collective_all_reduce_strategy import CollectiveAllReduceStrategy
-from tensorflow.contrib.distribute.python.cross_tower_ops import *
 from tensorflow.contrib.distribute.python.mirrored_strategy import MirroredStrategy
 from tensorflow.contrib.distribute.python.monitor import Monitor
 from tensorflow.contrib.distribute.python.one_device_strategy import OneDeviceStrategy
 from tensorflow.contrib.distribute.python.parameter_server_strategy import ParameterServerStrategy
 from tensorflow.contrib.distribute.python.step_fn import *
 from tensorflow.contrib.distribute.python.tpu_strategy import TPUStrategy
+from tensorflow.python.distribute.cross_device_ops import *
 from tensorflow.python.distribute.distribute_config import DistributeConfig
 from tensorflow.python.distribute.distribute_coordinator import run_standard_tensorflow_server
 from tensorflow.python.training.distribute import *
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index bf5adb1a50..80d102249b 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -16,45 +16,24 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 # TODO(priyag): Figure out testonly issues that are preventing us from
 # including our tests in pip for now.
 
-py_library(
-    name = "values",
-    srcs = ["values.py"],
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        ":input_ops",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:device_util",
-        "//tensorflow/python:distribute",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:multi_device_iterator_ops",
-        "//tensorflow/python/eager:context",
-        "//tensorflow/python/training/checkpointable:base",
-        "@six_archive//:six",
-    ],
-)
-
 cuda_py_test(
     name = "values_test",
     srcs = ["values_test.py"],
     additional_deps = [
         ":mirrored_strategy",
         ":multi_worker_test_base",
-        ":values",
         "//tensorflow/core:protos_all_py",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python:errors",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
+        "//tensorflow/python:device_util",
+        "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
-        "//tensorflow/python:device_util",
         "//tensorflow/python/eager:test",
         "//tensorflow/python/estimator:estimator_py",
     ],
@@ -68,9 +47,6 @@ py_library(
     srcs = ["mirrored_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":cross_tower_ops",
-        ":shared_variable_creator",
-        ":values",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -86,8 +62,11 @@ py_library(
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:cross_device_ops",
         "//tensorflow/python/distribute:multi_worker_util",
         "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:shared_variable_creator",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:tape",
     ],
@@ -98,17 +77,17 @@ py_library(
     srcs = ["parameter_server_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":cross_tower_ops",
         ":mirrored_strategy",
-        ":values",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
+        "//tensorflow/python/distribute:cross_device_ops",
         "//tensorflow/python/distribute:multi_worker_util",
         "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
     ],
 )
@@ -121,7 +100,6 @@ cuda_py_test(
         ":multi_worker_test_base",
         ":parameter_server_strategy",
         ":strategy_test_lib",
-        ":values",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
@@ -137,6 +115,7 @@ cuda_py_test(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
         "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/estimator:estimator_py",
     ],
@@ -151,13 +130,13 @@ py_library(
     srcs = ["one_device_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":values",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:distribute",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "@six_archive//:six",
     ],
@@ -168,16 +147,16 @@ py_library(
     srcs = ["collective_all_reduce_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":cross_tower_ops",
-        ":cross_tower_utils",
         ":mirrored_strategy",
-        ":values",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:collective_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:training",
+        "//tensorflow/python/distribute:cross_device_ops",
+        "//tensorflow/python/distribute:cross_device_utils",
         "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
     ],
 )
@@ -283,16 +262,16 @@ cuda_py_test(
     additional_deps = [
         ":mirrored_strategy",
         ":multi_worker_test_base",
-        ":values",
         ":strategy_test_lib",
-        "//tensorflow/python:distribute",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
+        "//tensorflow/python:distribute",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:layers",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
@@ -344,7 +323,6 @@ py_library(
     visibility = ["//tensorflow:internal"],
     deps = [
         ":one_device_strategy",
-        ":values",
         "//tensorflow/contrib/tpu:tpu_lib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
@@ -353,6 +331,7 @@ py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
     ],
 )
 
@@ -362,7 +341,6 @@ cuda_py_test(
     additional_deps = [
         ":collective_all_reduce_strategy",
         ":combinations",
-        ":cross_tower_utils",
         ":multi_worker_test_base",
         ":strategy_test_lib",
         "@absl_py//absl/testing:parameterized",
@@ -378,6 +356,7 @@ cuda_py_test(
         "//tensorflow/python:layers",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:cross_device_utils",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/estimator:estimator_py",
     ],
@@ -508,7 +487,9 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/contrib/optimizer_v2:training",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/distribute",
+        "//tensorflow/python/distribute:distribute_config",
+        "//tensorflow/python/distribute:distribute_coordinator",
+        "//tensorflow/python/distribute:distribute_coordinator_context",
         "//tensorflow/python/eager:test",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/feature_column",
@@ -600,52 +581,16 @@ cuda_py_test(
     ],
 )
 
-py_library(
-    name = "shared_variable_creator",
-    srcs = ["shared_variable_creator.py"],
-    visibility = ["//tensorflow:internal"],
-)
-
-py_test(
-    name = "shared_variable_creator_test",
-    srcs = ["shared_variable_creator_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":shared_variable_creator",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python/eager:test",
-    ],
-)
-
-py_library(
-    name = "cross_tower_utils",
-    srcs = ["cross_tower_utils.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":values",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:collective_ops",
-        "//tensorflow/python:device",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:gradients",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:nccl_ops",
-        "//tensorflow/python/distribute:all_reduce",
-    ],
-)
-
 cuda_py_test(
-    name = "cross_tower_utils_test",
-    srcs = ["cross_tower_utils_test.py"],
+    name = "cross_device_utils_test",
+    srcs = ["cross_device_utils_test.py"],
     additional_deps = [
         ":combinations",
-        ":cross_tower_utils",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/distribute:cross_device_utils",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
@@ -654,41 +599,20 @@ cuda_py_test(
     ],
 )
 
-py_library(
-    name = "cross_tower_ops",
-    srcs = ["cross_tower_ops.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":cross_tower_utils",
-        ":values",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:device_lib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python/distribute:reduce_util",
-        "//tensorflow/python/eager:context",
-        "@six_archive//:six",
-    ],
-)
-
 cuda_py_test(
-    name = "cross_tower_ops_test",
-    srcs = ["cross_tower_ops_test.py"],
+    name = "cross_device_ops_test",
+    srcs = ["cross_device_ops_test.py"],
     additional_deps = [
         ":combinations",
-        ":cross_tower_ops",
         ":multi_worker_test_base",
         ":mirrored_strategy",
-        ":values",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/distribute:cross_device_ops",
+        "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
@@ -698,35 +622,6 @@ cuda_py_test(
     ],
 )
 
-py_library(
-    name = "input_ops",
-    srcs = ["input_ops.py"],
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python/data/util:nest",
-    ],
-)
-
-cuda_py_test(
-    name = "input_ops_test",
-    srcs = ["input_ops_test.py"],
-    additional_deps = [
-        ":input_ops",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python/data/ops:readers",
-        "//tensorflow/python:util",
-    ],
-    tags = [
-        "no_pip",
-    ],
-)
-
 py_library(
     name = "keras_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index dd7d29ac2b..2c9c901ece 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -18,12 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
-from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import cross_device_utils
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -79,11 +79,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     else:
       local_devices = ["/device:CPU:0"]
 
-    self._collective_keys = cross_tower_utils.CollectiveKeys()
+    self._collective_keys = cross_device_utils.CollectiveKeys()
     super(CollectiveAllReduceExtended, self).__init__(
         container_strategy,
         devices=local_devices,
-        cross_device_ops=cross_tower_ops_lib.CollectiveAllReduce(
+        cross_device_ops=cross_device_ops_lib.CollectiveAllReduce(
             num_workers=1,
             num_gpus_per_worker=num_gpus_per_worker,
             collective_keys=self._collective_keys))
@@ -123,11 +123,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     else:
       local_devices = [worker_device]
 
-    self._collective_keys = cross_tower_utils.CollectiveKeys()
+    self._collective_keys = cross_device_utils.CollectiveKeys()
     super(CollectiveAllReduceExtended, self).__init__(
         container_strategy,
         devices=local_devices,
-        cross_device_ops=cross_tower_ops_lib.CollectiveAllReduce(
+        cross_device_ops=cross_device_ops_lib.CollectiveAllReduce(
             num_workers=self._num_workers,
             num_gpus_per_worker=num_gpus_per_worker,
             collective_keys=self._collective_keys))
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index ae7b88aa2d..27b9306c85 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -23,11 +23,11 @@ import numpy as np
 
 from tensorflow.contrib.distribute.python import collective_all_reduce_strategy
 from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import keras
+from tensorflow.python.distribute import cross_device_utils
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
@@ -73,7 +73,7 @@ class CollectiveAllReduceStrategyTestBase(
           cluster_spec=self._cluster_spec,
           task_type=task_type,
           task_id=task_id)
-    collective_keys = cross_tower_utils.CollectiveKeys(
+    collective_keys = cross_device_utils.CollectiveKeys(
         group_key_start=10 * num_gpus +
         CollectiveAllReduceStrategyTestBase.collective_key_base,
         instance_key_start=num_gpus * 100 +
@@ -81,7 +81,7 @@ class CollectiveAllReduceStrategyTestBase(
         instance_key_with_id_start=num_gpus * 10000 +
         CollectiveAllReduceStrategyTestBase.collective_key_base)
     distribution.extended._collective_keys = collective_keys
-    distribution.extended._cross_tower_ops._collective_keys = collective_keys
+    distribution.extended._cross_device_ops._collective_keys = collective_keys
     if task_type and task_id is not None:
       return distribution, 'grpc://' + self._cluster_spec[task_type][
           task_id], session_config
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
similarity index 87%
rename from tensorflow/contrib/distribute/python/cross_tower_ops_test.py
rename to tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 100328efba..36610ce3a9 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -24,13 +24,13 @@ from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
-from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
-from tensorflow.contrib.distribute.python import values as value_lib
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import cross_device_utils
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -41,7 +41,7 @@ from tensorflow.python.training import device_util
 
 
 def _make_per_replica(values, devices, regroup=False):
-  devices = cross_tower_ops_lib.get_devices_from(devices)
+  devices = cross_device_ops_lib.get_devices_from(devices)
   assert len(values) == len(devices)
 
   # We simulate the result of regroup called on PerReplica which strips the
@@ -66,7 +66,7 @@ def _fake_mirrored(value, devices):
   All components of the returned Mirrored have the same objects, which is not
   true in reality.
   """
-  devices = cross_tower_ops_lib.get_devices_from(devices)
+  devices = cross_device_ops_lib.get_devices_from(devices)
   return value_lib.Mirrored(
       {d: v for d, v in zip(devices, [value] * len(devices))})
 
@@ -118,7 +118,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
           self.assertEqual(
               sess.run(list(left._index.values())), list(right._index.values()))
 
-  def _testReductionAndBroadcast(self, cross_tower_ops, distribution):
+  def _testReductionAndBroadcast(self, cross_device_ops, distribution):
     devices = distribution.worker_devices
 
     values = [constant_op.constant(float(d)) for d in range(len(devices))]
@@ -142,24 +142,24 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
     # test reduce()
     for destinations in all_destinations:
       self._assert_values_equal(
-          cross_tower_ops.reduce(
+          cross_device_ops.reduce(
               reduce_util.ReduceOp.MEAN,
               per_replica,
               destinations=destinations),
           _fake_mirrored(mean, destinations))
       self._assert_values_equal(
-          cross_tower_ops.reduce(
+          cross_device_ops.reduce(
               reduce_util.ReduceOp.MEAN,
               per_replica_2,
               destinations=destinations),
           _fake_mirrored(mean_2, destinations))
       self._assert_values_equal(
-          cross_tower_ops.reduce(
+          cross_device_ops.reduce(
               reduce_util.ReduceOp.SUM, per_replica,
               destinations=destinations),
           _fake_mirrored(mean * len(devices), destinations))
       self._assert_values_equal(
-          cross_tower_ops.reduce(
+          cross_device_ops.reduce(
               reduce_util.ReduceOp.SUM,
               per_replica_2,
               destinations=destinations),
@@ -168,7 +168,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
     # test batch_reduce()
     for d1, d2 in itertools.product(all_destinations, all_destinations):
       self._assert_values_equal(
-          cross_tower_ops.batch_reduce(
+          cross_device_ops.batch_reduce(
               reduce_util.ReduceOp.MEAN,
               [(per_replica, d1), (per_replica_2, d2)]),
           [
@@ -176,7 +176,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
               _fake_mirrored(mean_2, d2)
           ])
       self._assert_values_equal(
-          cross_tower_ops.batch_reduce(
+          cross_device_ops.batch_reduce(
               reduce_util.ReduceOp.SUM,
               [(per_replica, d1), (per_replica_2, d2)]),
           [
@@ -187,7 +187,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
     # test broadcast()
     for destinations in all_destinations:
       self._assert_values_equal(
-          cross_tower_ops.broadcast(constant_op.constant(1.), destinations),
+          cross_device_ops.broadcast(constant_op.constant(1.), destinations),
           _fake_mirrored(1., destinations))
 
 
@@ -196,17 +196,17 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
   # combinations module so that we can pass in devices instead of a distribution
   # strategy.
   reduction_to_one_combinations = combinations.combine(
-      cross_tower_ops=[
+      cross_device_ops=[
           combinations.NamedObject(
               "DefaultReductionToOneDeviceCrossDeviceOps",
-              cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps()),
+              cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps()),
           combinations.NamedObject(
               "ReductionToCPUDeviceCrossDeviceOps",
-              cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+              cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
                   reduce_to_device=_cpu_device)),
           combinations.NamedObject(
               "AccumulateNCrossDeviceOp",
-              cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+              cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
                   accumulation_fn=math_ops.accumulate_n)),
       ],
       distribution=[
@@ -218,20 +218,20 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
       ],
       mode=["graph", "eager"])
   allreduce_combinations = combinations.combine(
-      cross_tower_ops=[
+      cross_device_ops=[
           combinations.NamedObject(
               "AllReduce",
-              cross_tower_ops_lib.AllReduceCrossDeviceOps("nccl", 1, 0, 0)),
+              cross_device_ops_lib.AllReduceCrossDeviceOps("nccl", 1, 0, 0)),
           combinations.NamedObject(
               "HierarchicalCopy",
-              cross_tower_ops_lib.AllReduceCrossDeviceOps(
+              cross_device_ops_lib.AllReduceCrossDeviceOps(
                   "hierarchical_copy", 8, 0, 0)),
           combinations.NamedObject(
               "AllReduceNoGradientRepacking",
-              cross_tower_ops_lib.AllReduceCrossDeviceOps("nccl", 0, 0, 0)),
+              cross_device_ops_lib.AllReduceCrossDeviceOps("nccl", 0, 0, 0)),
           combinations.NamedObject(
               "HierarchicalCopyAggregateSmallTensors",
-              cross_tower_ops_lib.AllReduceCrossDeviceOps(
+              cross_device_ops_lib.AllReduceCrossDeviceOps(
                   "hierarchical_copy", 0, 100, 10))
       ],
       distribution=[combinations.mirrored_strategy_with_two_gpus,
@@ -239,22 +239,22 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
       mode=["graph", "eager"])
 
   @combinations.generate(reduction_to_one_combinations + allreduce_combinations)
-  def testReductionAndBroadcast(self, cross_tower_ops, distribution):
+  def testReductionAndBroadcast(self, cross_device_ops, distribution):
     with distribution.scope():
-      self._testReductionAndBroadcast(cross_tower_ops, distribution)
+      self._testReductionAndBroadcast(cross_device_ops, distribution)
 
   def testChooseAlgorithm(self):
     device_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7],
                     [0, 5, 6, 7], [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6]]
-    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
-    self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps)
+    result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps)
     self.assertEqual(result._all_reduce_alg, "hierarchical_copy")
     self.assertEqual(result._num_packs, 8)
 
     # if there are only 4 devices
     device_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7]]
-    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
-    self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps)
+    result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps)
     self.assertEqual(result._all_reduce_alg, "nccl")
     self.assertEqual(result._num_packs, 1)
 
@@ -262,16 +262,16 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
     device_links = [[0, 1, 2, 3, 4], [0, 1, 2, 3, 5], [0, 1, 2, 3, 6],
                     [0, 1, 2, 3, 7], [0, 4, 5, 6, 7], [1, 4, 5, 6, 7],
                     [2, 4, 5, 6, 7], [3, 4, 5, 6, 7]]
-    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
-    self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps)
+    result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps)
     self.assertEqual(result._all_reduce_alg, "hierarchical_copy")
     self.assertEqual(result._num_packs, 8)
 
     # if not dgx1-like links
     device_links = [[0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7], [0, 5, 6, 7],
                     [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6], [1, 2, 3, 4]]
-    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
-    self.assertIsInstance(result, cross_tower_ops_lib.AllReduceCrossDeviceOps)
+    result = cross_device_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertIsInstance(result, cross_device_ops_lib.AllReduceCrossDeviceOps)
     self.assertEqual(result._all_reduce_alg, "nccl")
     self.assertEqual(result._num_packs, 1)
 
@@ -283,7 +283,7 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
     t0 = _make_indexed_slices([[1., 2.]], [1], [5, 2], devices[0])
     t1 = _make_indexed_slices([[3., 4.], [5., 6.]], [1, 3], [5, 2], devices[1])
     per_replica = value_lib.PerReplica({devices[0]: t0, devices[1]: t1})
-    result = cross_tower_ops_lib._simple_reduce(
+    result = cross_device_ops_lib._simple_reduce(
         per_replica, devices[0], math_ops.add_n, reduce_util.ReduceOp.SUM)
 
     # Test that the result is semantically equal to both the concatenated
@@ -297,19 +297,19 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
 
   @combinations.generate(
       combinations.combine(
-          cross_tower_ops_instance=[
+          cross_device_ops_instance=[
               combinations.NamedObject(
                   "ReductionToOneDeviceCrossDeviceOps",
-                  cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps()),
+                  cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps()),
               combinations.NamedObject(
                   "AllReduceCrossDeviceOps",
-                  cross_tower_ops_lib.AllReduceCrossDeviceOps())
+                  cross_device_ops_lib.AllReduceCrossDeviceOps())
           ],
           reduce_op=[reduce_util.ReduceOp.SUM, reduce_util.ReduceOp.MEAN],
           batch_reduce=[True, False],
           mode=["graph", "eager"],
           required_gpus=1))
-  def testIndexedSlicesAllReduce(self, cross_tower_ops_instance, reduce_op,
+  def testIndexedSlicesAllReduce(self, cross_device_ops_instance, reduce_op,
                                  batch_reduce):
     devices = ["/cpu:0", "/gpu:0"]
     dense_shape = [5, 2]
@@ -319,10 +319,10 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
     per_replica = value_lib.PerReplica({devices[0]: t0, devices[1]: t1})
 
     if batch_reduce:
-      result = cross_tower_ops_instance.batch_reduce(
+      result = cross_device_ops_instance.batch_reduce(
           reduce_op, [(per_replica, devices)])
     else:
-      result = cross_tower_ops_instance.reduce(
+      result = cross_device_ops_instance.reduce(
           reduce_op, per_replica, devices)
 
     total_indices_with_dups = [1, 1, 3]
@@ -359,22 +359,22 @@ class MultiWorkerCrossDeviceOpsTest(multi_worker_test_base.MultiWorkerTestBase,
       "/job:worker/replica:0/task:0", "/job:worker/replica:0/task:1"
   ]
   multi_worker_allreduce_combinations = combinations.combine(
-      cross_tower_ops=[
+      cross_device_ops=[
           combinations.NamedObject(
               "MultiWorkerAllReduce",
-              cross_tower_ops_lib.MultiWorkerAllReduce(
+              cross_device_ops_lib.MultiWorkerAllReduce(
                   worker_devices, 2, ("pscpu/pscpu", 2, -1), 0, 0, 0)),
           combinations.NamedObject(
               "MultiWorkerAllReducePack",
-              cross_tower_ops_lib.MultiWorkerAllReduce(
+              cross_device_ops_lib.MultiWorkerAllReduce(
                   worker_devices, 2, ("pscpu/pscpu", 2, -1), 1, 0, 0)),
           combinations.NamedObject(
               "MultiWorkerAllReduceAggregation",
-              cross_tower_ops_lib.MultiWorkerAllReduce(
+              cross_device_ops_lib.MultiWorkerAllReduce(
                   worker_devices, 2, ("pscpu/pscpu", 2, -1), 0, 100, 10)),
           combinations.NamedObject(
               "MultiWorkerAllReduceMultipleSpecs",
-              cross_tower_ops_lib.MultiWorkerAllReduce(
+              cross_device_ops_lib.MultiWorkerAllReduce(
                   worker_devices, 2, [("pscpu/pscpu", 2, 100),
                                       ("xring", 2, -1)], 0, 0, 0)),
       ],
@@ -395,13 +395,13 @@ class MultiWorkerCrossDeviceOpsTest(multi_worker_test_base.MultiWorkerTestBase,
       mode=["graph"])
 
   @combinations.generate(multi_worker_allreduce_combinations)
-  def testReductionAndBroadcast(self, cross_tower_ops, distribution):
+  def testReductionAndBroadcast(self, cross_device_ops, distribution):
     distribution.configure(cluster_spec={
         "worker":
             ["/job:worker/replica:0/task:0", "/job:worker/replica:0/task:1"]
     })
     with distribution.scope():
-      self._testReductionAndBroadcast(cross_tower_ops, distribution)
+      self._testReductionAndBroadcast(cross_device_ops, distribution)
 
 
 class MultiWorkerCollectiveAllReduceTest(
@@ -422,7 +422,7 @@ class MultiWorkerCollectiveAllReduceTest(
     MultiWorkerCollectiveAllReduceTest.collective_key_base += 100000
 
   def _get_test_objects(self, task_type, task_id, num_gpus=0, local_mode=False):
-    collective_keys = cross_tower_utils.CollectiveKeys(
+    collective_keys = cross_device_utils.CollectiveKeys(
         group_key_start=10 * num_gpus +
         MultiWorkerCollectiveAllReduceTest.collective_key_base,
         instance_key_start=num_gpus * 100 +
@@ -430,7 +430,7 @@ class MultiWorkerCollectiveAllReduceTest(
         instance_key_with_id_start=num_gpus * 10000 +
         MultiWorkerCollectiveAllReduceTest.collective_key_base)
     if local_mode:
-      collective_all_reduce_ops = cross_tower_ops_lib.CollectiveAllReduce(
+      collective_all_reduce_ops = cross_device_ops_lib.CollectiveAllReduce(
           1, num_gpus, collective_keys=collective_keys)
       if num_gpus:
         devices = ["/device:GPU:%d" % i for i in range(num_gpus)]
@@ -438,7 +438,7 @@ class MultiWorkerCollectiveAllReduceTest(
         devices = ["/device:CPU:0"]
       return collective_all_reduce_ops, devices, ""
     else:
-      collective_all_reduce_ops = cross_tower_ops_lib.CollectiveAllReduce(
+      collective_all_reduce_ops = cross_device_ops_lib.CollectiveAllReduce(
           3, num_gpus, collective_keys=collective_keys)
       if num_gpus:
         devices = [
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils_test.py b/tensorflow/contrib/distribute/python/cross_device_utils_test.py
similarity index 84%
rename from tensorflow/contrib/distribute/python/cross_tower_utils_test.py
rename to tensorflow/contrib/distribute/python/cross_device_utils_test.py
index e46240abbf..6086eba098 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_utils_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for cross_tower_utils."""
+"""Tests for cross_device_utils."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -21,8 +21,8 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import cross_tower_utils
-from tensorflow.contrib.distribute.python import values as value_lib
+from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
@@ -43,7 +43,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
     t0 = constant_op.constant([[1., 2.], [0, 0], [3., 4.]])
     t1 = constant_op.constant([[0., 0.], [5, 6], [7., 8.]])
     total = constant_op.constant([[1., 2.], [5, 6], [10., 12.]])
-    result = cross_tower_utils.aggregate_tensors_or_indexed_slices([t0, t1])
+    result = cross_device_utils.aggregate_tensors_or_indexed_slices([t0, t1])
     self._assert_values_equal(total, result)
 
   @test_util.run_in_graph_and_eager_modes
@@ -53,7 +53,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
     t1 = math_ops._as_indexed_slices(
         constant_op.constant([[0., 0.], [5, 6], [7., 8.]]))
     total = constant_op.constant([[1., 2.], [5, 6], [10., 12.]])
-    result = cross_tower_utils.aggregate_tensors_or_indexed_slices([t0, t1])
+    result = cross_device_utils.aggregate_tensors_or_indexed_slices([t0, t1])
     self.assertIsInstance(result, ops.IndexedSlices)
     self._assert_values_equal(total, result)
 
@@ -62,7 +62,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
     t = constant_op.constant([[1., 2.], [0, 0], [3., 4.]])
     n = 2
     expected = constant_op.constant([[0.5, 1.], [0, 0], [1.5, 2.]])
-    result = cross_tower_utils.divide_by_n_tensors_or_indexed_slices(t, n)
+    result = cross_device_utils.divide_by_n_tensors_or_indexed_slices(t, n)
     self._assert_values_equal(expected, result)
 
   @test_util.run_in_graph_and_eager_modes
@@ -71,7 +71,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
         constant_op.constant([[1., 2.], [0, 0], [3., 4.]]))
     n = 2
     expected = constant_op.constant([[0.5, 1.], [0, 0], [1.5, 2.]])
-    result = cross_tower_utils.divide_by_n_tensors_or_indexed_slices(t, n)
+    result = cross_device_utils.divide_by_n_tensors_or_indexed_slices(t, n)
     self.assertIsInstance(result, ops.IndexedSlices)
     self._assert_values_equal(expected, result)
 
@@ -79,7 +79,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
   def testIsIndexedSlices(self):
     t = math_ops._as_indexed_slices(
         constant_op.constant([[1., 2.], [0, 0], [3., 4.]]))
-    self.assertTrue(cross_tower_utils.contains_indexed_slices(t))
+    self.assertTrue(cross_device_utils.contains_indexed_slices(t))
 
   @test_util.run_in_graph_and_eager_modes
   def testContainsIndexedSlices_List(self):
@@ -87,7 +87,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
         constant_op.constant([[1., 2.], [0, 0], [3., 4.]]))
     t1 = math_ops._as_indexed_slices(
         constant_op.constant([[0., 0.], [5, 6], [7., 8.]]))
-    self.assertTrue(cross_tower_utils.contains_indexed_slices([t0, t1]))
+    self.assertTrue(cross_device_utils.contains_indexed_slices([t0, t1]))
 
   @test_util.run_in_graph_and_eager_modes
   def testContainsIndexedSlices_Tuple(self):
@@ -95,7 +95,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
         constant_op.constant([[1., 2.], [0, 0], [3., 4.]]))
     t1 = math_ops._as_indexed_slices(
         constant_op.constant([[0., 0.], [5, 6], [7., 8.]]))
-    self.assertTrue(cross_tower_utils.contains_indexed_slices((t0, t1)))
+    self.assertTrue(cross_device_utils.contains_indexed_slices((t0, t1)))
 
   @test_util.run_in_graph_and_eager_modes
   def testContainsIndexedSlices_PerReplica(self):
@@ -104,7 +104,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
     t1 = math_ops._as_indexed_slices(
         constant_op.constant([[0., 0.], [5, 6], [7., 8.]]))
     per_replica = value_lib.PerReplica({"/gpu:0": t0, "/cpu:0": t1})
-    self.assertTrue(cross_tower_utils.contains_indexed_slices(per_replica))
+    self.assertTrue(cross_device_utils.contains_indexed_slices(per_replica))
 
   @combinations.generate(combinations.combine(
       mode=["graph", "eager"],
@@ -113,7 +113,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
     with ops.device("/cpu:0"):
       t = constant_op.constant([[1., 2.], [0, 0], [3., 4.]])
     destination = "/gpu:0"
-    result = cross_tower_utils.copy_tensor_or_indexed_slices_to_device(
+    result = cross_device_utils.copy_tensor_or_indexed_slices_to_device(
         t, destination)
 
     self._assert_values_equal(t, result)
@@ -128,7 +128,7 @@ class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
       t = math_ops._as_indexed_slices(
           constant_op.constant([[1., 2.], [0, 0], [3., 4.]]))
     destination = "/gpu:0"
-    result = cross_tower_utils.copy_tensor_or_indexed_slices_to_device(
+    result = cross_device_utils.copy_tensor_or_indexed_slices_to_device(
         t, destination)
 
     self.assertIsInstance(result, ops.IndexedSlices)
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index e7dc57b15b..9c2c05c928 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -24,9 +24,9 @@ import numpy as np
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import tpu_strategy
-from tensorflow.contrib.distribute.python import values
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import values
 from tensorflow.python.estimator import keras as keras_lib
 from tensorflow.python.estimator import run_config as run_config_lib
 from tensorflow.python.framework import constant_op
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 3b24f1b03d..f7d209cc3d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -22,12 +22,12 @@ import contextlib
 from functools import partial
 import threading
 
-from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
-from tensorflow.contrib.distribute.python import shared_variable_creator
-from tensorflow.contrib.distribute.python import values
 from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import shared_variable_creator
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
@@ -196,16 +196,16 @@ def _reduce_non_distributed_value(extended, reduce_op, value, destinations):
   if reduce_op == reduce_util.ReduceOp.MEAN:
     return value
 
-  cross_tower_ops_lib.validate_destinations(destinations)
+  cross_device_ops_lib.validate_destinations(destinations)
   # We do not support a reduce op of SUM if the value is the same across
   # all replicas. We call this as part of assign functions for MirroredVariables
   # and summing up identical values across replicas is not clearly defined.
   if (len(extended.worker_devices) != 1 or
-      not cross_tower_ops_lib.check_destinations(destinations)):
+      not cross_device_ops_lib.check_destinations(destinations)):
     raise ValueError("A non-DistributedValues value %s cannot be reduced with "
                      "the given reduce op %s." % (value, reduce_op))
   # TODO(anjalisridhar): Moves these methods to a device utility file?
-  devices = cross_tower_ops_lib.get_devices_from(destinations)
+  devices = cross_device_ops_lib.get_devices_from(destinations)
   if len(devices) == 1:
     with ops.device(devices[0]):
       return array_ops.identity(value)
@@ -369,8 +369,7 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
                cross_device_ops=None,
                auto_shard_dataset=False):
     super(CoreMirroredExtended, self).__init__(container_strategy)
-    # TODO(josh11b): Rename self._cross_tower_ops -> self._cross_device_ops
-    self._cross_tower_ops = cross_device_ops
+    self._cross_device_ops = cross_device_ops
     self._auto_shard_dataset = auto_shard_dataset
     # Remember num GPUs which might be needed by `configure` method.
     if num_gpus is not None and num_gpus_per_worker is not None:
@@ -588,7 +587,7 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
     if isinstance(tensor, (float, int)):  # Fast path for Python constants.
       return tensor
     # TODO(josh11b): In eager mode, use one thread per device, or async mode.
-    return self._get_cross_tower_ops().broadcast(
+    return self._get_cross_device_ops().broadcast(
         tensor, destinations or self._devices)
 
   def _call_for_each_replica(self, fn, args, kwargs):
@@ -607,26 +606,27 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
     if cluster_spec:
       self._initialize_multi_worker(self._num_gpus, cluster_spec)
 
-    if self._cross_tower_ops is None:
+    if self._cross_device_ops is None:
       if self._cluster_spec:
         # It currently cannot detect the toplogy of remote workers. So we
         # hard-code the multi-worker all-reduce algorithm for now.
         if len(self._workers) == 1:
           # The default is "nccl".
-          self._cross_tower_ops = cross_tower_ops_lib.AllReduceCrossDeviceOps()
+          self._cross_device_ops = (
+              cross_device_ops_lib.AllReduceCrossDeviceOps())
         else:
           # The default is hierarchical reduce and broadcast.
-          self._cross_tower_ops = cross_tower_ops_lib.MultiWorkerAllReduce(
+          self._cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce(
               self._workers, self._num_gpus)
       else:
-        self._cross_tower_ops = cross_tower_ops_lib.choose_the_best(
+        self._cross_device_ops = cross_device_ops_lib.choose_the_best(
             self._devices, session_config=session_config)
 
-  def _get_cross_tower_ops(self):
-    if self._cross_tower_ops is None:
-      self._cross_tower_ops = (
-          cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps())
-    return self._cross_tower_ops
+  def _get_cross_device_ops(self):
+    if self._cross_device_ops is None:
+      self._cross_device_ops = (
+          cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps())
+    return self._cross_device_ops
 
   def _reduce_to(self, reduce_op, value, destinations):
     assert not isinstance(value, values.Mirrored)
@@ -637,12 +637,12 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
       # be 0.
       return _reduce_non_distributed_value(self, reduce_op, value,
                                            destinations)
-    return self._get_cross_tower_ops().reduce(
+    return self._get_cross_device_ops().reduce(
         reduce_op, value, destinations=destinations)
 
   def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    return self._get_cross_tower_ops().batch_reduce(reduce_op,
-                                                    value_destination_pairs)
+    return self._get_cross_device_ops().batch_reduce(reduce_op,
+                                                     value_destination_pairs)
 
   def _update(self, var, fn, args, kwargs, group):
     # TODO(josh11b): In eager mode, use one thread per device.
@@ -723,7 +723,7 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
     if colocate_with is None:
       return self._devices
     else:
-      return cross_tower_ops_lib.get_devices_from(colocate_with)
+      return cross_device_ops_lib.get_devices_from(colocate_with)
 
   class _MirroredReplicaThread(threading.Thread):
     """A thread that runs() a function on a device."""
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index d45eb46e6f..4977946afe 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -25,10 +25,10 @@ import numpy as np
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
-from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 6df3b3a92b..b1dda84d4e 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import six
 
-from tensorflow.contrib.distribute.python import values
+from tensorflow.python.distribute import values
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index e87337baff..86d1d56a82 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -18,10 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
 from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.contrib.distribute.python import values
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
@@ -107,8 +107,8 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
     self._initialize_local(num_gpus_per_worker)
 
     # We typically don't need to do all-reduce in this strategy.
-    self._cross_tower_ops = (
-        cross_tower_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+    self._cross_device_ops = (
+        cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
             reduce_to_device=_LOCAL_CPU))
 
   def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec,
@@ -256,9 +256,9 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
         True)
 
   def _broadcast_to(self, tensor, destinations):
-    if not cross_tower_ops_lib.check_destinations(destinations):
+    if not cross_device_ops_lib.check_destinations(destinations):
       destinations = self._compute_devices
-    return self._cross_tower_ops.broadcast(tensor, destinations)
+    return self._cross_device_ops.broadcast(tensor, destinations)
 
   def _allow_variable_partition(self):
     return not context.executing_eagerly()
@@ -330,7 +330,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
       return
     if destinations is None:
       return
-    for d in cross_tower_ops_lib.get_devices_from(destinations):
+    for d in cross_device_ops_lib.get_devices_from(destinations):
       d_spec = tf_device.DeviceSpec.from_string(d)
       if d_spec.job == self._task_type and d_spec.task != self._task_id:
         raise ValueError(
@@ -343,14 +343,14 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
       # pylint: disable=protected-access
       return mirrored_strategy._reduce_non_distributed_value(
           self, reduce_op, value, destinations)
-    return self._cross_tower_ops.reduce(
+    return self._cross_device_ops.reduce(
         reduce_op, value, destinations=destinations)
 
   def _batch_reduce_to(self, reduce_op, value_destination_pairs):
     for _, destinations in value_destination_pairs:
       self._verify_destinations_not_different_worker(destinations)
-    return self._cross_tower_ops.batch_reduce(reduce_op,
-                                              value_destination_pairs)
+    return self._cross_device_ops.batch_reduce(reduce_op,
+                                               value_destination_pairs)
 
   def _select_single_value(self, structured):
     """Select any single values in `structured`."""
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 091a358fea..dc84d1788f 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -26,10 +26,10 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import parameter_server_strategy
 from tensorflow.contrib.distribute.python import strategy_test_lib
-from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f36171e4c2..d159b3e419 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -23,15 +23,15 @@ from __future__ import print_function
 
 import functools
 
-from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
-from tensorflow.contrib.distribute.python import values
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
@@ -467,7 +467,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     # Validate that the destination is same as the host device
     # Note we don't do this when in replicate context as the reduction is
     # performed on the TPU device itself.
-    devices = cross_tower_ops_lib.get_devices_from(destinations)
+    devices = cross_device_ops_lib.get_devices_from(destinations)
     if len(devices) == 1:
       assert device_util.canonicalize(devices[0]) == device_util.canonicalize(
           self._host_device)
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index b4b56eb4e4..62cef18ad7 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -22,9 +22,9 @@ import os
 
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
-from tensorflow.contrib.distribute.python import values
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.estimator import model_fn as model_fn_lib
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 13d797d987..78ad8e7e15 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -8,6 +8,7 @@ exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "tf_py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 py_library(
     name = "all_reduce",
@@ -44,13 +45,41 @@ tf_py_test(
 )
 
 py_library(
-    name = "distribute",
+    name = "cross_device_ops",
+    srcs = ["cross_device_ops.py"],
     srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
     deps = [
-        ":distribute_config",
-        ":distribute_coordinator",
-        ":distribute_coordinator_context",
+        ":cross_device_utils",
+        ":reduce_util",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:device_lib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:context",
+        "@six_archive//:six",
+    ],
+)
+
+py_library(
+    name = "cross_device_utils",
+    srcs = ["cross_device_utils.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":all_reduce",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:collective_ops",
+        "//tensorflow/python:device",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nccl_ops",
     ],
 )
 
@@ -123,6 +152,34 @@ py_library(
     ],
 )
 
+py_library(
+    name = "input_ops",
+    srcs = ["input_ops.py"],
+    deps = [
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python/data/util:nest",
+    ],
+)
+
+cuda_py_test(
+    name = "input_ops_test",
+    srcs = ["input_ops_test.py"],
+    additional_deps = [
+        ":input_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python:util",
+    ],
+    tags = [
+        "no_pip",
+    ],
+)
+
 py_test(
     name = "multi_worker_util_test",
     srcs = ["multi_worker_util_test.py"],
@@ -158,8 +215,43 @@ py_library(
 
 py_library(
     name = "reduce_util",
-    srcs = [
-        "reduce_util.py",
-    ],
+    srcs = ["reduce_util.py"],
     deps = [],
 )
+
+py_library(
+    name = "shared_variable_creator",
+    srcs = ["shared_variable_creator.py"],
+)
+
+py_test(
+    name = "shared_variable_creator_test",
+    srcs = ["shared_variable_creator_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":shared_variable_creator",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+py_library(
+    name = "values",
+    srcs = ["values.py"],
+    deps = [
+        ":input_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:device_util",
+        "//tensorflow/python:distribute",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/ops:multi_device_iterator_ops",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/training/checkpointable:base",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/python/distribute/cross_device_ops.py
similarity index 96%
rename from tensorflow/contrib/distribute/python/cross_tower_ops.py
rename to tensorflow/python/distribute/cross_device_ops.py
index 994ed345d8..f55385eddc 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -21,10 +21,10 @@ from __future__ import print_function
 import collections
 import six
 
-from tensorflow.contrib.distribute.python import cross_tower_utils
-from tensorflow.contrib.distribute.python import values as value_lib
 from tensorflow.python.client import device_lib
+from tensorflow.python.distribute import cross_device_utils
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -144,7 +144,7 @@ def _simple_broadcast(value, destinations):
   index = {}
   devices = get_devices_from(destinations)
   for d in devices:
-    index[d] = cross_tower_utils.copy_tensor_or_indexed_slices_to_device(
+    index[d] = cross_device_utils.copy_tensor_or_indexed_slices_to_device(
         value, d)
   return value_lib.Mirrored(index)
 
@@ -162,10 +162,10 @@ def _simple_reduce(per_replica_value, reduce_to_device, accumulation_fn,
 
   with ops.device(reduce_to_device):
     with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
-      reduced = cross_tower_utils.aggregate_tensors_or_indexed_slices(
+      reduced = cross_device_utils.aggregate_tensors_or_indexed_slices(
           all_values, accumulation_fn)
       if reduce_op == reduce_util.ReduceOp.MEAN:
-        reduced = cross_tower_utils.divide_by_n_tensors_or_indexed_slices(
+        reduced = cross_device_utils.divide_by_n_tensors_or_indexed_slices(
             reduced, count)
       elif reduce_op != reduce_util.ReduceOp.SUM:
         raise ValueError("`reduce_op` must be Reduce.SUM or Reduce.MEAN.")
@@ -332,7 +332,7 @@ def _ungroup_and_make_mirrored(grouped_reduced,
   Args:
     grouped_reduced: a list of lists, each sublist has components for each
       device, paired with a None. It is the result from
-      cross_tower_utils.aggregate_gradients_using*.
+      cross_device_utils.aggregate_gradients_using*.
     destinations: a list of device strings for returned Mirrored objects.
     reduce_op: Indicates how values will be aggregated. Accepted values
       are `tf.distribute.ReduceOp.SUM`, `tf.distribute.ReduceOp.MEAN`.
@@ -485,7 +485,7 @@ class AggregateSmallTensorPacker(object):
     """Aggregate small tensors."""
     if (self.agg_small_grads_max_bytes > 0 and
         self.agg_small_grads_max_group > 0):
-      device_grads, self.packing = cross_tower_utils.pack_small_tensors(
+      device_grads, self.packing = cross_device_utils.pack_small_tensors(
           grouped_grads_and_vars,
           max_bytes=self.agg_small_grads_max_bytes,
           max_group=self.agg_small_grads_max_group)
@@ -493,8 +493,8 @@ class AggregateSmallTensorPacker(object):
 
   def unpack(self, summed_device_grad_packs):
     """Reverse the aggregation process."""
-    return cross_tower_utils.unpack_small_tensors(summed_device_grad_packs,
-                                                  self.packing)
+    return cross_device_utils.unpack_small_tensors(summed_device_grad_packs,
+                                                   self.packing)
 
 
 def _pack_tensors(device_grads,
@@ -557,7 +557,7 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
     super(AllReduceCrossDeviceOps, self).__init__()
 
   def _reduce(self, reduce_op, per_replica_value, destinations):
-    contains_indexed_slices = cross_tower_utils.contains_indexed_slices(
+    contains_indexed_slices = cross_device_utils.contains_indexed_slices(
         per_replica_value)
     if (_devices_match(per_replica_value, destinations)
         and not context.executing_eagerly()
@@ -580,7 +580,7 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
 
   def _batch_reduce(self, reduce_op, value_destination_pairs):
     all_devices_match = _all_devices_match(value_destination_pairs)
-    contains_indexed_slices = cross_tower_utils.contains_indexed_slices(
+    contains_indexed_slices = cross_device_utils.contains_indexed_slices(
         value_destination_pairs)
     if (all_devices_match and not context.executing_eagerly()
         and not contains_indexed_slices):
@@ -618,13 +618,13 @@ class AllReduceCrossDeviceOps(CrossDeviceOps):
     # the balance on num_splits.
     if self._all_reduce_alg == "nccl":
       # TODO(yuefengz): merge this into the all-reduce library.
-      reduced = cross_tower_utils.aggregate_gradients_using_nccl(
+      reduced = cross_device_utils.aggregate_gradients_using_nccl(
           device_grad_packs)
     else:
       # TODO(yuefengz): check that gpu ids in `destinations` are in ascending
       # order.
       reduced = (
-          cross_tower_utils.aggregate_gradients_using_hierarchical_copy(
+          cross_device_utils.aggregate_gradients_using_hierarchical_copy(
               destinations, device_grad_packs))
 
     reduced = _unpack_tensors(reduced, tensor_packer)
@@ -740,13 +740,13 @@ class MultiWorkerAllReduce(AllReduceCrossDeviceOps):
         this_grads = remaining_grads
         remaining_grads = []
       else:
-        (this_grads, remaining_grads) = cross_tower_utils.split_grads_by_size(
+        (this_grads, remaining_grads) = cross_device_utils.split_grads_by_size(
             spec_tuple.limit, remaining_grads)
       if this_grads:
         device_grad_packs, tensor_packer = _pack_tensors(
             this_grads, self._num_packs, self._agg_small_grads_max_bytes,
             self._agg_small_grads_max_group)
-        range_agg_grads = cross_tower_utils.sum_gradients_all_reduce(
+        range_agg_grads = cross_device_utils.sum_gradients_all_reduce(
             self._worker_devices, device_grad_packs, len(self._worker_devices),
             spec_tuple.alg, spec_tuple.shards, range(self._num_gpus_per_worker))
         range_agg_grads = _unpack_tensors(range_agg_grads, tensor_packer)
@@ -789,13 +789,13 @@ class CollectiveAllReduce(CrossDeviceOps):
     self._num_workers = num_workers
     self._num_gpus_per_worker = num_gpus_per_worker
     self._all_reduce_merge_scope = all_reduce_merge_scope
-    self._collective_keys = collective_keys or cross_tower_utils.CollectiveKeys(
-    )
+    self._collective_keys = (collective_keys or
+                             cross_device_utils.CollectiveKeys())
     super(CollectiveAllReduce, self).__init__()
 
   # TODO(yuefengz, tucker): is indexed slices supported by collective ops?
   def _reduce(self, reduce_op, per_replica_value, destinations):
-    if cross_tower_utils.contains_indexed_slices(per_replica_value):
+    if cross_device_utils.contains_indexed_slices(per_replica_value):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
     if context.executing_eagerly():
@@ -819,7 +819,7 @@ class CollectiveAllReduce(CrossDeviceOps):
       return value_lib.Mirrored(index)
 
   def _batch_reduce(self, reduce_op, value_destination_pairs):
-    if cross_tower_utils.contains_indexed_slices(value_destination_pairs):
+    if cross_device_utils.contains_indexed_slices(value_destination_pairs):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
     if context.executing_eagerly():
@@ -870,7 +870,7 @@ class CollectiveAllReduce(CrossDeviceOps):
       with ops.name_scope("allreduce"):
         for grad_and_vars in chunk:
           scaled_grads = [g for g, _ in grad_and_vars]
-          collective_reduced = cross_tower_utils.build_collective_reduce(
+          collective_reduced = cross_device_utils.build_collective_reduce(
               scaled_grads, self._num_workers, self._collective_keys, "Add",
               "Id")
           result = []
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/python/distribute/cross_device_utils.py
similarity index 99%
rename from tensorflow/contrib/distribute/python/cross_tower_utils.py
rename to tensorflow/python/distribute/cross_device_utils.py
index d408f77cc9..7903992ac7 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils.py
+++ b/tensorflow/python/distribute/cross_device_utils.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utilities for cross_tower_ops."""
+"""Utilities for cross_device_ops."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -21,8 +21,8 @@ from __future__ import print_function
 import collections as pycoll
 import threading
 
-from tensorflow.contrib.distribute.python import values as value_lib
 from tensorflow.python.distribute import all_reduce
+from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/distribute/python/input_ops.py b/tensorflow/python/distribute/input_ops.py
similarity index 100%
rename from tensorflow/contrib/distribute/python/input_ops.py
rename to tensorflow/python/distribute/input_ops.py
diff --git a/tensorflow/contrib/distribute/python/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py
similarity index 99%
rename from tensorflow/contrib/distribute/python/input_ops_test.py
rename to tensorflow/python/distribute/input_ops_test.py
index 559de97bb1..cbb93e8995 100644
--- a/tensorflow/contrib/distribute/python/input_ops_test.py
+++ b/tensorflow/python/distribute/input_ops_test.py
@@ -20,9 +20,9 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.contrib.distribute.python import input_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
+from tensorflow.python.distribute import input_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.lib.io import python_io
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator.py b/tensorflow/python/distribute/shared_variable_creator.py
similarity index 100%
rename from tensorflow/contrib/distribute/python/shared_variable_creator.py
rename to tensorflow/python/distribute/shared_variable_creator.py
diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator_test.py b/tensorflow/python/distribute/shared_variable_creator_test.py
similarity index 97%
rename from tensorflow/contrib/distribute/python/shared_variable_creator_test.py
rename to tensorflow/python/distribute/shared_variable_creator_test.py
index 2a9ab51fcf..4ddc29f256 100644
--- a/tensorflow/contrib/distribute/python/shared_variable_creator_test.py
+++ b/tensorflow/python/distribute/shared_variable_creator_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.distribute.python import shared_variable_creator
+from tensorflow.python.distribute import shared_variable_creator
 from tensorflow.python.eager import test
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variable_scope
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/python/distribute/values.py
similarity index 99%
rename from tensorflow/contrib/distribute/python/values.py
rename to tensorflow/python/distribute/values.py
index 80b9b1f457..70293aa3be 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -27,9 +27,9 @@ import operator
 import weakref
 import six
 
-from tensorflow.contrib.distribute.python import input_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import multi_device_iterator_ops
+from tensorflow.python.distribute import input_ops
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
-- 
GitLab


From b0510ee05bf0687f92bb209b8b70f45fb94d2e84 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 16 Nov 2018 11:55:29 -0800
Subject: [PATCH 0419/1554] Remove deprecated params for tf.reverse_sequence in
 v2.

PiperOrigin-RevId: 221829135
---
 tensorflow/python/ops/array_ops.py            | 23 ++++++++++++++++---
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index b7823ccc2f..3ce1a0cf97 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2876,7 +2876,7 @@ def where(condition, x=None, y=None, name=None):
 
 
 # pylint: disable=redefined-builtin
-@tf_export("reverse_sequence")
+@tf_export(v1=["reverse_sequence"])
 @deprecation.deprecated_args(
     None, "seq_dim is deprecated, use seq_axis instead", "seq_dim")
 @deprecation.deprecated_args(
@@ -2900,14 +2900,31 @@ def reverse_sequence(input,
       name=name)
 
 
-# pylint: enable=redefined-builtin
-
 reverse_sequence.__doc__ = deprecation.rewrite_argument_docstring(
     deprecation.rewrite_argument_docstring(
         gen_array_ops.reverse_sequence.__doc__, "batch_dim", "batch_axis"),
     "seq_dim", "seq_axis")
 
 
+@tf_export("reverse_sequence", v1=[])
+def reverse_sequence_v2(
+    input, seq_lengths, seq_axis=None, batch_axis=None, name=None):
+  return gen_array_ops.reverse_sequence(
+      input=input,
+      seq_lengths=seq_lengths,
+      seq_dim=seq_axis,
+      batch_dim=batch_axis,
+      name=name)
+
+
+reverse_sequence_v2.__doc__ = deprecation.rewrite_argument_docstring(
+    deprecation.rewrite_argument_docstring(
+        gen_array_ops.reverse_sequence.__doc__, "batch_dim", "batch_axis"),
+    "seq_dim", "seq_axis")
+
+# pylint: enable=redefined-builtin
+
+
 @tf_export("gather")
 def gather(params, indices, validate_indices=None, name=None, axis=0):
   del validate_indices
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index eba7b39561..8fcee46107 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -962,7 +962,7 @@ tf_module {
   }
   member_method {
     name: "reverse_sequence"
-    argspec: "args=[\'input\', \'seq_lengths\', \'seq_axis\', \'batch_axis\', \'name\', \'seq_dim\', \'batch_dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'seq_lengths\', \'seq_axis\', \'batch_axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "roll"
-- 
GitLab


From 9bbcd38048b9265c77e1000d571c449304170fbc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 12:08:41 -0800
Subject: [PATCH 0420/1554] Reorder strings.length parameters for TF 2.0 API

PiperOrigin-RevId: 221831392
---
 tensorflow/python/ops/string_ops.py                     | 6 +++++-
 tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt | 2 +-
 tensorflow/tools/compatibility/tf_upgrade_v2.py         | 1 +
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py
index 25e86cadeb..a20eec20b8 100644
--- a/tensorflow/python/ops/string_ops.py
+++ b/tensorflow/python/ops/string_ops.py
@@ -337,10 +337,14 @@ reduce_join.__doc__ = reduce_join.__doc__.replace("tf.reduce_join(",
 
 # This wrapper provides backwards compatibility for code that predates the
 # unit argument and that passed 'name' as a positional argument.
-@tf_export("strings.length")
+@tf_export(v1=["strings.length"])
 def string_length(input, name=None, unit="BYTE"):
   return gen_string_ops.string_length(input, unit=unit, name=name)
 
+@tf_export("strings.length", v1=[])
+def string_length_v2(input, unit="BYTE", name=None):
+  return string_length(input, name, unit)
+
 
 string_length.__doc__ = gen_string_ops.string_length.__doc__
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
index 03144cbe70..16b7f14ab2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
@@ -10,7 +10,7 @@ tf_module {
   }
   member_method {
     name: "length"
-    argspec: "args=[\'input\', \'name\', \'unit\'], varargs=None, keywords=None, defaults=[\'None\', \'BYTE\'], "
+    argspec: "args=[\'input\', \'unit\', \'name\'], varargs=None, keywords=None, defaults=[\'BYTE\', \'None\'], "
   }
   member_method {
     name: "reduce_join"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index be6dbc1ff8..1d29b61d67 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -183,6 +183,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.segment_sum": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
+        "tf.strings.length": ["input", "name", "unit"],
     }
 
     # Specially handled functions.
-- 
GitLab


From 9c7ae97de576f560596e64de8aee9db4748fa0e5 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 16 Nov 2018 12:13:26 -0800
Subject: [PATCH 0421/1554] Clear more uses of assertBetween, and fix typo.

PiperOrigin-RevId: 221832096
---
 .../boosted_trees/estimator_batch/estimator_test.py    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index ae6bb5767f..7863b5a4f8 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -436,8 +436,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
 
     frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3)
     # +/- 3%
-    self.assertTrue(frac_below_upper >= 0.92)
-    self.assertTrue(frac_below_upper <= 0.98)
+    self.assertTrue(frac_above_lower >= 0.92)
+    self.assertTrue(frac_above_lower <= 0.98)
 
 
 class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
@@ -663,7 +663,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
 
     frac_below_upper = round(1. * np.count_nonzero(upper > y) / len(y), 3)
     # +/- 3%
-    self.assertBetween(frac_below_upper, 0.92, 0.98)
+    self.assertTrue(frac_below_upper >= 0.92)
+    self.assertTrue(frac_below_upper <= 0.98)
 
     train_input_fn, test_input_fn, _ = _quantile_regression_input_fns()
     model_lower = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor(
@@ -681,7 +682,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
 
     frac_above_lower = round(1. * np.count_nonzero(lower < y) / len(y), 3)
     # +/- 3%
-    self.assertBetween(frac_above_lower, 0.92, 0.98)
+    self.assertTrue(frac_above_lower >= 0.92)
+    self.assertTrue(frac_above_lower <= 0.98)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 4531e552baaf648319b8e004a56b25aad5b8e1c7 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 16 Nov 2018 12:25:25 -0800
Subject: [PATCH 0422/1554] Change API for tf.random.stateless_multinomial in
 TF2.0.

PiperOrigin-RevId: 221833786
---
 tensorflow/python/ops/stateless_random_ops.py |  58 ++++++-
 .../api/golden/v1/tensorflow.random.pbtxt     |   4 +
 .../api/golden/v2/tensorflow.random.pbtxt     |   4 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 151 ++++++++++++------
 4 files changed, 161 insertions(+), 56 deletions(-)

diff --git a/tensorflow/python/ops/stateless_random_ops.py b/tensorflow/python/ops/stateless_random_ops.py
index c6defabacd..b119049b16 100644
--- a/tensorflow/python/ops/stateless_random_ops.py
+++ b/tensorflow/python/ops/stateless_random_ops.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 ops.NotDifferentiable("StatelessMultinomial")
@@ -179,7 +180,9 @@ def stateless_truncated_normal(shape,
     return math_ops.add(rnd * stddev, mean, name=name)
 
 
-@tf_export("random.stateless_multinomial")
+@tf_export(v1=["random.stateless_multinomial"])
+@deprecation.deprecated(
+    date=None, instructions="Use tf.random.stateless_categorical instead.")
 def stateless_multinomial(logits,
                           num_samples,
                           seed,
@@ -207,13 +210,58 @@ def stateless_multinomial(logits,
       `[i, :]` represents the unnormalized log-probabilities for all classes.
     num_samples: 0-D.  Number of independent samples to draw for each row slice.
     seed: A shape [2] integer Tensor of seeds to the random number generator.
-    name: Optional name for the operation.
     output_dtype: integer type to use for the output. Defaults to int64.
+    name: Optional name for the operation.
 
   Returns:
     The drawn samples of shape `[batch_size, num_samples]`.
   """
   with ops.name_scope(name, "stateless_multinomial", [logits, seed]):
-    logits = ops.convert_to_tensor(logits, name="logits")
-    return gen_stateless_random_ops.stateless_multinomial(
-        logits, num_samples, seed, output_dtype=output_dtype)
+    return stateless_multinomial_categorical_impl(logits, num_samples,
+                                                  output_dtype, seed)
+
+
+@tf_export("random.stateless_categorical")
+def stateless_categorical(logits,
+                          num_samples,
+                          seed,
+                          dtype=dtypes.int64,
+                          name=None):
+  """Draws deterministic pseudorandom samples from a categorical distribution.
+
+  This is a stateless version of `tf.categorical`: if run twice with the
+  same seeds, it will produce the same pseudorandom numbers.  The output is
+  consistent across multiple runs on the same hardware (and between CPU
+  and GPU), but may change between versions of TensorFlow or on non-CPU/GPU
+  hardware.
+
+  Example:
+
+  ```python
+  # samples has shape [1, 5], where each value is either 0 or 1 with equal
+  # probability.
+  samples = tf.random.stateless_categorical(
+      tf.log([[10., 10.]]), 5, seed=[7, 17])
+  ```
+
+  Args:
+    logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice
+      `[i, :]` represents the unnormalized log-probabilities for all classes.
+    num_samples: 0-D.  Number of independent samples to draw for each row slice.
+    seed: A shape [2] integer Tensor of seeds to the random number generator.
+    dtype: integer type to use for the output. Defaults to int64.
+    name: Optional name for the operation.
+
+  Returns:
+    The drawn samples of shape `[batch_size, num_samples]`.
+  """
+  with ops.name_scope(name, "stateless_categorical", [logits, seed]):
+    return stateless_multinomial_categorical_impl(logits, num_samples, dtype,
+                                                  seed)
+
+
+def stateless_multinomial_categorical_impl(logits, num_samples, dtype, seed):
+  """Implementation for stateless multinomial/categorical ops (v1/v2)."""
+  logits = ops.convert_to_tensor(logits, name="logits")
+  return gen_stateless_random_ops.stateless_multinomial(
+      logits, num_samples, seed, output_dtype=dtype)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
index 160c09798d..d788f6dfca 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "shuffle"
     argspec: "args=[\'value\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "stateless_categorical"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
+  }
   member_method {
     name: "stateless_multinomial"
     argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index cace49a5ce..9dfa3c091d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -37,8 +37,8 @@ tf_module {
     argspec: "args=[\'value\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
-    name: "stateless_multinomial"
-    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'output_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
+    name: "stateless_categorical"
+    argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
   }
   member_method {
     name: "stateless_normal"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 1d29b61d67..77f50ab21f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -70,7 +70,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.convolution": {
             "filter": "filters",
             "dilation_rate": "dilations",
-        }
+        },
+        "tf.random.stateless_multinomial": {
+            "output_dtype": "dtype",
+        },
     }
 
     # Mapping from function to the new name of the function
@@ -81,54 +84,104 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # function_reorders or function_keyword_renames, use the OLD function name.
     # These renames happen after the arguments have been processed.
     self.symbol_renames.update({
-        "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
-        "tf.contrib.data.Counter": "tf.data.experimental.Counter",
-        "tf.contrib.data.CheckpointInputPipelineHook": "tf.data.experimental.CheckpointInputPipelineHook",
-        "tf.contrib.data.CsvDataset": "tf.data.experimental.CsvDataset",
-        "tf.contrib.data.Optional": "tf.data.experimental.Optional",
-        "tf.contrib.data.RandomDataset": "tf.data.experimental.RandomDataset",
-        "tf.contrib.data.Reducer": "tf.data.experimental.Reducer",
-        "tf.contrib.data.SqlDataset": "tf.data.experimental.SqlDataset",
-        "tf.contrib.data.StatsAggregator": "tf.data.experimental.StatsAggregator",
-        "tf.contrib.data.TFRecordWriter": "tf.data.experimental.TFRecordWriter",
-        "tf.contrib.data.assert_element_shape": "tf.data.experimental.assert_element_shape",
-        "tf.contrib.data.batch_and_drop_remainder": "tf.compat.v1.contrib.data.batch_and_drop_remainder",
-        "tf.contrib.data.bucket_by_sequence_length": "tf.data.experimental.bucket_by_sequence_length",
-        "tf.contrib.data.choose_from_datasets": "tf.data.experimental.choose_from_datasets",
-        "tf.contrib.data.copy_to_device": "tf.data.experimental.copy_to_device",
-        "tf.contrib.data.dense_to_sparse_batch": "tf.data.experimental.dense_to_sparse_batch",
-        "tf.contrib.data.enumerate_dataset": "tf.data.experimental.enumerate_dataset",
-        "tf.contrib.data.get_next_as_optional": "tf.data.experimental.get_next_as_optional",
-        "tf.contrib.data.get_single_element": "tf.data.experimental.get_single_element",
-        "tf.contrib.data.group_by_reducer": "tf.data.experimental.group_by_reducer",
-        "tf.contrib.data.group_by_window": "tf.data.experimental.group_by_window",
-        "tf.contrib.data.ignore_errors": "tf.data.experimental.ignore_errors",
-        "tf.contrib.data.latency_stats": "tf.data.experimental.latency_stats",
-        "tf.contrib.data.make_batched_features_dataset": "tf.data.experimental.make_batched_features_dataset",
-        "tf.contrib.data.make_csv_dataset": "tf.data.experimental.make_csv_dataset",
-        "tf.contrib.data.make_saveable_from_iterator": "tf.data.experimental.make_saveable_from_iterator",
-        "tf.contrib.data.map_and_batch": "tf.data.experimental.map_and_batch",
-        "tf.contrib.data.padded_batch_and_drop_remainder": "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
-        "tf.contrib.data.parallel_interleave": "tf.data.experimental.parallel_interleave",
-        "tf.contrib.data.parse_example_dataset": "tf.data.experimental.parse_example_dataset",
-        "tf.contrib.data.prefetch_to_device": "tf.data.experimental.prefetch_to_device",
-        "tf.contrib.data.read_batch_features": "tf.compat.v1.contrib.data.read_batch_features",
-        "tf.contrib.data.reduce_dataset": "tf.compat.v1.contrib.data.reduce_dataset",
-        "tf.contrib.data.rejection_resample": "tf.data.experimental.rejection_resample",
-        "tf.contrib.data.sample_from_datasets": "tf.data.experimental.sample_from_datasets",
-        "tf.contrib.data.scan": "tf.data.experimental.scan",
-        "tf.contrib.data.set_stats_aggregator": "tf.data.experimental.set_stats_aggregator",
-        "tf.contrib.data.shuffle_and_repeat": "tf.data.experimental.shuffle_and_repeat",
-        "tf.contrib.data.sliding_window_batch": "tf.compat.v1.contrib.data.sliding_window_batch",
-        "tf.contrib.data.sloppy_interleave": "tf.compat.v1.contrib.data.sloppy_interleave",
-        "tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
-        "tf.contrib.data.unique": "tf.data.experimental.unique",
-        "tf.quantize_v2": "tf.quantization.quantize",
-        "tf.sparse_concat": "tf.sparse.concat",
-        "tf.sparse_split": "tf.sparse.split",
-        "tf.multinomial": "tf.random.categorical",
-        "tf.random.multinomial": "tf.random.categorical",
-        "tf.load_file_system_library": "tf.load_library",
+        "tf.contrib.data.AUTOTUNE":
+            "tf.data.experimental.AUTOTUNE",
+        "tf.contrib.data.Counter":
+            "tf.data.experimental.Counter",
+        "tf.contrib.data.CheckpointInputPipelineHook":
+            "tf.data.experimental.CheckpointInputPipelineHook",
+        "tf.contrib.data.CsvDataset":
+            "tf.data.experimental.CsvDataset",
+        "tf.contrib.data.Optional":
+            "tf.data.experimental.Optional",
+        "tf.contrib.data.RandomDataset":
+            "tf.data.experimental.RandomDataset",
+        "tf.contrib.data.Reducer":
+            "tf.data.experimental.Reducer",
+        "tf.contrib.data.SqlDataset":
+            "tf.data.experimental.SqlDataset",
+        "tf.contrib.data.StatsAggregator":
+            "tf.data.experimental.StatsAggregator",
+        "tf.contrib.data.TFRecordWriter":
+            "tf.data.experimental.TFRecordWriter",
+        "tf.contrib.data.assert_element_shape":
+            "tf.data.experimental.assert_element_shape",
+        "tf.contrib.data.batch_and_drop_remainder":
+            "tf.compat.v1.contrib.data.batch_and_drop_remainder",
+        "tf.contrib.data.bucket_by_sequence_length":
+            "tf.data.experimental.bucket_by_sequence_length",
+        "tf.contrib.data.choose_from_datasets":
+            "tf.data.experimental.choose_from_datasets",
+        "tf.contrib.data.copy_to_device":
+            "tf.data.experimental.copy_to_device",
+        "tf.contrib.data.dense_to_sparse_batch":
+            "tf.data.experimental.dense_to_sparse_batch",
+        "tf.contrib.data.enumerate_dataset":
+            "tf.data.experimental.enumerate_dataset",
+        "tf.contrib.data.get_next_as_optional":
+            "tf.data.experimental.get_next_as_optional",
+        "tf.contrib.data.get_single_element":
+            "tf.data.experimental.get_single_element",
+        "tf.contrib.data.group_by_reducer":
+            "tf.data.experimental.group_by_reducer",
+        "tf.contrib.data.group_by_window":
+            "tf.data.experimental.group_by_window",
+        "tf.contrib.data.ignore_errors":
+            "tf.data.experimental.ignore_errors",
+        "tf.contrib.data.latency_stats":
+            "tf.data.experimental.latency_stats",
+        "tf.contrib.data.make_batched_features_dataset":
+            "tf.data.experimental.make_batched_features_dataset",
+        "tf.contrib.data.make_csv_dataset":
+            "tf.data.experimental.make_csv_dataset",
+        "tf.contrib.data.make_saveable_from_iterator":
+            "tf.data.experimental.make_saveable_from_iterator",
+        "tf.contrib.data.map_and_batch":
+            "tf.data.experimental.map_and_batch",
+        "tf.contrib.data.padded_batch_and_drop_remainder":
+            "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
+        "tf.contrib.data.parallel_interleave":
+            "tf.data.experimental.parallel_interleave",
+        "tf.contrib.data.parse_example_dataset":
+            "tf.data.experimental.parse_example_dataset",
+        "tf.contrib.data.prefetch_to_device":
+            "tf.data.experimental.prefetch_to_device",
+        "tf.contrib.data.read_batch_features":
+            "tf.compat.v1.contrib.data.read_batch_features",
+        "tf.contrib.data.reduce_dataset":
+            "tf.compat.v1.contrib.data.reduce_dataset",
+        "tf.contrib.data.rejection_resample":
+            "tf.data.experimental.rejection_resample",
+        "tf.contrib.data.sample_from_datasets":
+            "tf.data.experimental.sample_from_datasets",
+        "tf.contrib.data.scan":
+            "tf.data.experimental.scan",
+        "tf.contrib.data.set_stats_aggregator":
+            "tf.data.experimental.set_stats_aggregator",
+        "tf.contrib.data.shuffle_and_repeat":
+            "tf.data.experimental.shuffle_and_repeat",
+        "tf.contrib.data.sliding_window_batch":
+            "tf.compat.v1.contrib.data.sliding_window_batch",
+        "tf.contrib.data.sloppy_interleave":
+            "tf.compat.v1.contrib.data.sloppy_interleave",
+        "tf.contrib.data.unbatch":
+            "tf.data.experimental.unbatch",
+        "tf.contrib.data.unique":
+            "tf.data.experimental.unique",
+        "tf.quantize_v2":
+            "tf.quantization.quantize",
+        "tf.sparse_concat":
+            "tf.sparse.concat",
+        "tf.sparse_split":
+            "tf.sparse.split",
+        "tf.multinomial":
+            "tf.random.categorical",
+        "tf.random.multinomial":
+            "tf.random.categorical",
+        "tf.load_file_system_library":
+            "tf.load_library",
+        "tf.random.stateless_multinomial":
+            "tf.random.stateless_categorical",
     })
     # pylint: enable=line-too-long
 
-- 
GitLab


From ad2c5c378b246d7bfa93e945f0d2f57108826e50 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 12:32:18 -0800
Subject: [PATCH 0423/1554] Add TF_InitPlatform which indirects to calling
 tensorflow::Port::InitMain so that users can properly initialize tensorflow
 directly through publically exposed symbols without calling any C++ apis.

PiperOrigin-RevId: 221834794
---
 tensorflow/c/c_api_experimental.cc | 5 +++++
 tensorflow/c/c_api_experimental.h  | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 7fe74b5db3..d84bdcae58 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/platform.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/tensorflow_server.pb.h"
@@ -8800,3 +8801,7 @@ const char* TF_GetNumberAttrForOpListInput(const char* op_name, int input_index,
   // The returned string is owned by OpRegistry, so liveness is not a concern.
   return input_arg.number_attr().c_str();
 }
+
+void TF_InitMain(const char* usage, int* argc, char*** argv) {
+  tensorflow::port::InitMain(usage, argc, argv);
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 6639b0be72..f1dedbdb49 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -209,6 +209,10 @@ TF_CAPI_EXPORT extern void TF_AttrBuilderCheckCanRunOnDevice(
 TF_CAPI_EXPORT extern const char* TF_GetNumberAttrForOpListInput(
     const char* op_name, int input_index, TF_Status* status);
 
+// Platform specific initialization routine. Very few platforms actually require
+// this to be called.
+TF_CAPI_EXPORT void TF_InitMain(const char* usage, int* argc, char*** argv);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
-- 
GitLab


From ff9bd9f31ac4883c3eef31cc74df8bc2d9e9ea8d Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 16 Nov 2018 12:32:23 -0800
Subject: [PATCH 0424/1554] Automated rollback of commit
 f15948957d31f6632c343ec6b7261c1d01acc79e

PiperOrigin-RevId: 221834805
---
 tensorflow/lite/BUILD            |  2 -
 tensorflow/lite/core/subgraph.cc | 32 -----------
 tensorflow/lite/core/subgraph.h  | 97 --------------------------------
 tensorflow/lite/interpreter.cc   | 90 ++++++++++++++---------------
 tensorflow/lite/interpreter.h    | 62 ++++++++++----------
 5 files changed, 71 insertions(+), 212 deletions(-)
 delete mode 100644 tensorflow/lite/core/subgraph.cc
 delete mode 100644 tensorflow/lite/core/subgraph.h

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index bb2c53b8c9..be84fc5db1 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -131,7 +131,6 @@ cc_library(
     name = "framework",
     srcs = [
         "allocation.cc",
-        "core/subgraph.cc",
         "graph_info.cc",
         "interpreter.cc",
         "model.cc",
@@ -156,7 +155,6 @@ cc_library(
         "allocation.h",
         "context.h",
         "context_util.h",
-        "core/subgraph.h",
         "error_reporter.h",
         "graph_info.h",
         "interpreter.h",
diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
deleted file mode 100644
index 1033fecbb5..0000000000
--- a/tensorflow/lite/core/subgraph.cc
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/core/subgraph.h"
-
-namespace tflite {
-
-Subgraph::~Subgraph() {
-  for (auto& node_and_reg : nodes_and_registration_) {
-    TfLiteNode& node = node_and_reg.first;
-    TfLiteIntArrayFree(node.inputs);
-    TfLiteIntArrayFree(node.outputs);
-    TfLiteIntArrayFree(node.temporaries);
-    if (node.builtin_data) free(node.builtin_data);
-    OpFree(node_and_reg.second, node.user_data);
-    node.builtin_data = nullptr;
-  }
-}
-
-}  // namespace tflite
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
deleted file mode 100644
index e53f752631..0000000000
--- a/tensorflow/lite/core/subgraph.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_LITE_CORE_SUBGRAPH_H_
-#define TENSORFLOW_LITE_CORE_SUBGRAPH_H_
-
-#include <cstdlib>
-#include <vector>
-
-#include "tensorflow/lite/allocation.h"
-#include "tensorflow/lite/c/c_api_internal.h"
-
-namespace tflite {
-
-class Subgraph {
- public:
-  Subgraph(TfLiteContext* context) : context_(context) {}
-
-  virtual ~Subgraph();
-
-  // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return inputs_; }
-
-  // Read only access to list of outputs.
-  const std::vector<int>& outputs() const { return outputs_; }
-
-  // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const { return variables_; }
-
-  // Read only access to list of inputs.
-  std::vector<int>& inputs() { return inputs_; }
-
-  // Read only access to list of outputs.
-  std::vector<int>& outputs() { return outputs_; }
-
-  // Read only access to list of variable tensors.
-  std::vector<int>& variables() { return variables_; }
-
-  // Mutable form of tensors (TEMPORARY for refactor).
-  // TODO(b/119495520): remove when refactoring complete.
-  std::vector<TfLiteTensor>& tensors() { return tensors_; }
-  // Mutable form of tensors (TEMPORARY for refactor).
-  // TODO(b/119495520): remove when refactoring complete.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
-  nodes_and_registration() {
-    return nodes_and_registration_;
-  }
-
-  const std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
-  nodes_and_registration() const {
-    return nodes_and_registration_;
-  }
-
- private:
-  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
-  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
-    if (op_reg.free == nullptr) return;
-    if (buffer) {
-      op_reg.free(context_, buffer);
-    }
-  }
-
-  // TODO(b/119495520): Make this be the authoritative copy.
-  TfLiteContext* context_;
-
-  std::vector<TfLiteTensor> tensors_;
-
-  // Array of indices representing the tensors that are inputs to the
-  // interpreter.
-  std::vector<int> inputs_;
-
-  // Array of indices representing the tensors that are outputs to the
-  // interpreter.
-  std::vector<int> outputs_;
-
-  // Array of indices representing the tensors that are variable tensors.
-  std::vector<int> variables_;
-
-  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
-  // function pointers to actual implementation.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
-      nodes_and_registration_;
-};
-
-}  // namespace tflite
-#endif  // TENSORFLOW_LITE_CORE_SUBGRAPH_H_
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 04db91ffd8..0bac5b07b2 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -117,7 +117,6 @@ class InterpreterInfo : public GraphInfo {
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  subgraphs_.emplace_back(&context_);
   context_.impl_ = static_cast<void*>(this);
   context_.ResizeTensor = ResizeTensor;
   context_.ReportError = ReportError;
@@ -133,9 +132,8 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
   SwitchToKernelContext();
 
   // Reserve some space for the tensors to avoid excessive resizing.
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-  tensors.reserve(kTensorsReservedCapacity);
-  primary_subgraph().nodes_and_registration().reserve(kTensorsReservedCapacity);
+  tensors_.reserve(kTensorsReservedCapacity);
+  nodes_and_registration_.reserve(kTensorsReservedCapacity);
   next_execution_plan_index_to_prepare_ = 0;
 
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
@@ -146,6 +144,16 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
 }
 
 Interpreter::~Interpreter() {
+  for (auto& nodeAndReg : nodes_and_registration_) {
+    TfLiteNode& node = nodeAndReg.first;
+    TfLiteIntArrayFree(node.inputs);
+    TfLiteIntArrayFree(node.outputs);
+    TfLiteIntArrayFree(node.temporaries);
+    if (node.builtin_data) free(node.builtin_data);
+    OpFree(nodeAndReg.second, node.user_data);
+    node.builtin_data = nullptr;
+  }
+
   for (size_t i = 0; i < context_.tensors_size; i++) {
     TfLiteTensor* tensor = &context_.tensors[i];
     if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
@@ -253,8 +261,6 @@ TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
                                            &node_subsets);
 
   execution_plan_.clear();
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-
   for (auto& node_subset : node_subsets) {
     // Subsets calimed by the delegate should have a "macro" op created, the
     // other node_subsets (kTfNonPartition) just have their nodes added back to
@@ -277,15 +283,14 @@ TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
 
         // Initialize the output tensors's delegate-related fields.
         for (int tensor_index : node_subset.output_tensors) {
-          TfLiteTensor* tensor = &tensors[tensor_index];
+          TfLiteTensor* tensor = &tensors_[tensor_index];
           TF_LITE_ENSURE(&context_, tensor->delegate == nullptr ||
                                         tensor->delegate == delegate);
           tensor->delegate = delegate;
         }
 
         // Associate the node with the delegate.
-        TfLiteNode* node =
-            &primary_subgraph().nodes_and_registration()[node_index].first;
+        TfLiteNode* node = &nodes_and_registration_[node_index].first;
         node->delegate = delegate;
       } break;
       case NodeSubset::kTfUnexplored:
@@ -348,21 +353,21 @@ TfLiteStatus Interpreter::GetExecutionPlan(struct TfLiteContext* context,
 TfLiteStatus Interpreter::SetInputs(std::vector<int> inputs) {
   TF_LITE_ENSURE_OK(&context_,
                     CheckTensorIndices("inputs", inputs.data(), inputs.size()));
-  primary_subgraph().inputs() = std::move(inputs);
+  inputs_ = std::move(inputs);
   return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetOutputs(std::vector<int> outputs) {
   TF_LITE_ENSURE_OK(
       &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
-  primary_subgraph().outputs() = std::move(outputs);
+  outputs_ = std::move(outputs);
   return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetVariables(std::vector<int> variables) {
   TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
                                                   variables.size()));
-  primary_subgraph().variables() = std::move(variables);
+  variables_ = std::move(variables);
   return kTfLiteOk;
 }
 
@@ -437,8 +442,7 @@ TfLiteStatus Interpreter::AllocateTensors() {
   // Explicit (re)allocation is necessary if nodes have been changed or tensors
   // have been resized. For inputs marked as dynamic, we can't short-circuit the
   // allocation as the client may have done the resize manually.
-  if (state_ != kStateUninvokable &&
-      !HasDynamicTensorImpl(context_, inputs())) {
+  if (state_ != kStateUninvokable && !HasDynamicTensorImpl(context_, inputs_)) {
     return kTfLiteOk;
   }
 
@@ -462,8 +466,7 @@ TfLiteStatus Interpreter::AllocateTensors() {
 
 // TODO(ycling): Support non-zero default values.
 TfLiteStatus Interpreter::ResetVariableTensors() {
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-  for (auto& tensor : tensors) {
+  for (auto& tensor : tensors_) {
     if (!tensor.is_variable) {
       continue;
     }
@@ -480,7 +483,7 @@ TfLiteStatus Interpreter::ResetVariableTensors() {
 }
 
 void Interpreter::ReserveNodes(int count) {
-  primary_subgraph().nodes_and_registration().reserve(count);
+  nodes_and_registration_.reserve(count);
 }
 
 TfLiteStatus Interpreter::AddNodeWithParameters(
@@ -503,11 +506,10 @@ TfLiteStatus Interpreter::AddNodeWithParameters(
       &context_,
       CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
 
-  int new_node_index = primary_subgraph().nodes_and_registration().size();
+  int new_node_index = nodes_and_registration_.size();
   if (node_index) *node_index = new_node_index;
-  primary_subgraph().nodes_and_registration().resize(
-      primary_subgraph().nodes_and_registration().size() + 1);
-  auto& node_and_reg = primary_subgraph().nodes_and_registration().back();
+  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
+  auto& node_and_reg = nodes_and_registration_.back();
   TfLiteNode& node = node_and_reg.first;
   if (node.inputs) TfLiteIntArrayFree(node.inputs);
   if (node.outputs) TfLiteIntArrayFree(node.outputs);
@@ -578,13 +580,12 @@ bool HasDynamicTensor(const TfLiteContext& context,
 
 TfLiteStatus Interpreter::PrepareOpsStartingAt(
     int first_execution_plan_index, int* last_execution_plan_index_prepared) {
-  auto& subgraph = primary_subgraph();
   for (int execution_plan_index = first_execution_plan_index;
        execution_plan_index < execution_plan_.size(); execution_plan_index++) {
     int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = subgraph.nodes_and_registration()[node_index].first;
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
     const TfLiteRegistration& registration =
-        subgraph.nodes_and_registration()[node_index].second;
+        nodes_and_registration_[node_index].second;
     EnsureTensorsVectorCapacity();
     if (OpPrepare(registration, &node) == kTfLiteError) {
       return ReportOpError(&context_, node, registration, node_index,
@@ -623,8 +624,6 @@ TfLiteStatus Interpreter::PrepareOpsAndTensors() {
 }
 
 TfLiteStatus Interpreter::Invoke() {
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-
   if (!consistent_) {
     ReportError(&context_, "Invoke called on model that is not consistent.");
     return kTfLiteError;
@@ -661,10 +660,9 @@ TfLiteStatus Interpreter::Invoke() {
                                     execution_plan_index);
     }
     int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node =
-        primary_subgraph().nodes_and_registration()[node_index].first;
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
     const TfLiteRegistration& registration =
-        primary_subgraph().nodes_and_registration()[node_index].second;
+        nodes_and_registration_[node_index].second;
     SCOPED_OPERATOR_PROFILE(profiler_, node_index);
 
     // TODO(ycling): This is an extra loop through inputs to check if the data
@@ -676,7 +674,7 @@ TfLiteStatus Interpreter::Invoke() {
       if (tensor_index == kOptionalTensor) {
         continue;
       }
-      TfLiteTensor* tensor = &tensors[tensor_index];
+      TfLiteTensor* tensor = &tensors_[tensor_index];
       if (tensor->delegate && tensor->delegate != node.delegate &&
           tensor->data_is_stale) {
         EnsureTensorDataIsReadable(tensor_index);
@@ -699,7 +697,7 @@ TfLiteStatus Interpreter::Invoke() {
   }
 
   if (!allow_buffer_handle_output_) {
-    for (int tensor_index : outputs()) {
+    for (int tensor_index : outputs_) {
       EnsureTensorDataIsReadable(tensor_index);
     }
   }
@@ -734,17 +732,15 @@ void Interpreter::ReportError(TfLiteContext* context, const char* format, ...) {
 
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
                                      int* first_new_tensor_index) {
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-
-  const size_t base_index = tensors.size();
+  const size_t base_index = tensors_.size();
   if (first_new_tensor_index) *first_new_tensor_index = base_index;
-  tensors.resize(tensors.size() + tensors_to_add);
-  for (size_t i = base_index; i < tensors.size(); i++) {
-    memset(&tensors[i], 0, sizeof(tensors[i]));
-    tensors[i].buffer_handle = kTfLiteNullBufferHandle;
+  tensors_.resize(tensors_.size() + tensors_to_add);
+  for (size_t i = base_index; i < tensors_.size(); i++) {
+    memset(&tensors_[i], 0, sizeof(tensors_[i]));
+    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
   }
-  context_.tensors = tensors.data();
-  context_.tensors_size = tensors.size();
+  context_.tensors = tensors_.data();
+  context_.tensors_size = tensors_.size();
   return kTfLiteOk;
 }
 
@@ -762,9 +758,8 @@ TfLiteStatus Interpreter::GetNodeAndRegistration(
   TF_LITE_ENSURE(&context_, node_index >= 0);
   TF_LITE_ENSURE(&context_, static_cast<size_t>(node_index) < nodes_size());
   TF_LITE_ENSURE(&context_, node != nullptr && registration != nullptr);
-  auto& node_and_reg = primary_subgraph().nodes_and_registration()[node_index];
-  *node = &node_and_reg.first;
-  *registration = &node_and_reg.second;
+  *node = &nodes_and_registration_[node_index].first;
+  *registration = &nodes_and_registration_[node_index].second;
   return kTfLiteOk;
 }
 
@@ -954,8 +949,7 @@ TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
       // If all the nodes can be prepared, check if the last node has dynamic
       // tensors.
       int node_index = execution_plan_[last_execution_plan_index_prepared];
-      TfLiteNode& node =
-          primary_subgraph().nodes_and_registration()[node_index].first;
+      TfLiteNode& node = nodes_and_registration_[node_index].first;
       if (!HasDynamicTensor(context_, node.outputs)) {
         has_dynamic_tensors = false;
       }
@@ -997,8 +991,7 @@ TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle buffer_handle,
                                           TfLiteDelegate* delegate) {
   TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-  TfLiteTensor* tensor = &tensors[tensor_index];
+  TfLiteTensor* tensor = &tensors_[tensor_index];
 
   TF_LITE_ENSURE(&context_,
                  tensor->delegate == nullptr || tensor->delegate == delegate);
@@ -1017,8 +1010,7 @@ TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle* buffer_handle,
                                           TfLiteDelegate** delegate) {
   TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-  TfLiteTensor* tensor = &tensors[tensor_index];
+  TfLiteTensor* tensor = &tensors_[tensor_index];
 
   *delegate = tensor->delegate;
   *buffer_handle = tensor->buffer_handle;
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index eb7d2e47f1..de6c42bfcc 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
-#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/memory_planner.h"
 #include "tensorflow/lite/profiling/profiler.h"
 #include "tensorflow/lite/stderr_reporter.h"
@@ -202,37 +201,31 @@ class Interpreter {
   // Functions to access tensor data
 
   // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return primary_subgraph().inputs(); }
+  const std::vector<int>& inputs() const { return inputs_; }
 
   // Return the name of a given input. The given index must be between 0 and
   // inputs().size().
   const char* GetInputName(int index) const {
-    return context_.tensors[inputs()[index]].name;
+    return context_.tensors[inputs_[index]].name;
   }
 
   // Read only access to list of outputs.
-  const std::vector<int>& outputs() const {
-    return primary_subgraph().outputs();
-  }
+  const std::vector<int>& outputs() const { return outputs_; }
 
   // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const {
-    return primary_subgraph().variables();
-  }
+  const std::vector<int>& variables() const { return variables_; }
 
   // Return the name of a given output. The given index must be between 0 and
   // outputs().size().
   const char* GetOutputName(int index) const {
-    return context_.tensors[outputs()[index]].name;
+    return context_.tensors[outputs_[index]].name;
   }
 
   // Return the number of tensors in the model.
   size_t tensors_size() const { return context_.tensors_size; }
 
   // Return the number of ops in the model.
-  size_t nodes_size() const {
-    return primary_subgraph().nodes_and_registration().size();
-  }
+  size_t nodes_size() const { return nodes_and_registration_.size(); }
 
   // WARNING: Experimental interface, subject to change
   const std::vector<int>& execution_plan() const { return execution_plan_; }
@@ -262,9 +255,10 @@ class Interpreter {
   // Get a pointer to an operation and registration data structure if in bounds.
   const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
       int node_index) const {
-    if (node_index < 0 || static_cast<size_t>(node_index) >= nodes_size())
+    if (node_index < 0 ||
+        static_cast<size_t>(node_index) >= nodes_and_registration_.size())
       return nullptr;
-    return &primary_subgraph().nodes_and_registration()[node_index];
+    return &nodes_and_registration_[node_index];
   }
 
   // Perform a checked cast to the appropriate tensor type (mutable pointer
@@ -295,28 +289,28 @@ class Interpreter {
   // index must be between 0 and inputs().size().
   template <class T>
   T* typed_input_tensor(int index) {
-    return typed_tensor<T>(inputs()[index]);
+    return typed_tensor<T>(inputs_[index]);
   }
 
   // Return an immutable pointer into the data of a given input tensor. The
   // given index must be between 0 and inputs().size().
   template <class T>
   const T* typed_input_tensor(int index) const {
-    return typed_tensor<T>(inputs()[index]);
+    return typed_tensor<T>(inputs_[index]);
   }
 
   // Return a mutable pointer into the data of a given output tensor. The given
   // index must be between 0 and outputs().size().
   template <class T>
   T* typed_output_tensor(int index) {
-    return typed_tensor<T>(outputs()[index]);
+    return typed_tensor<T>(outputs_[index]);
   }
 
   // Return an immutable pointer into the data of a given output tensor. The
   // given index must be between 0 and outputs().size().
   template <class T>
   const T* typed_output_tensor(int index) const {
-    return typed_tensor<T>(outputs()[index]);
+    return typed_tensor<T>(outputs_[index]);
   }
 
   // Change the dimensionality of a given tensor. Note, this is only acceptable
@@ -456,14 +450,6 @@ class Interpreter {
   friend class InterpreterBuilder;
   friend class InterpreterTest;
 
-  Subgraph& primary_subgraph() {
-    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
-  }
-
-  const Subgraph& primary_subgraph() const {
-    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
-  }
-
   // Prevent 'context_' from accessing functions that are only available to
   // delegated kernels.
   void SwitchToKernelContext();
@@ -516,7 +502,7 @@ class Interpreter {
   // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
   // `context_` whenever this std::vector is reallocated. Currently this
   // only happens in `AddTensors()`.
-  // std::vector<TfLiteTensor> tensors_;
+  std::vector<TfLiteTensor> tensors_;
 
   // Check if an array of tensor indices are valid with respect to the Tensor
   // array.
@@ -610,7 +596,6 @@ class Interpreter {
   // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
   // more tensors won't invalidate the pointer to existing tensors.
   void EnsureTensorsVectorCapacity() {
-    std::vector<TfLiteTensor>& tensors_ = primary_subgraph().tensors();
     const size_t required_capacity = tensors_size() + kTensorsCapacityHeadroom;
     if (required_capacity > tensors_.capacity()) {
       tensors_.reserve(required_capacity);
@@ -637,11 +622,27 @@ class Interpreter {
   // structure to store tensors.
   TfLiteContext context_;
 
+  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
+  // function pointers to actual implementation.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
+      nodes_and_registration_;
+
   // Whether the model is consistent. That is to say if the inputs and outputs
   // of every node and the global inputs and outputs are valid indexes into
   // the tensor array.
   bool consistent_ = true;
 
+  // Array of indices representing the tensors that are inputs to the
+  // interpreter.
+  std::vector<int> inputs_;
+
+  // Array of indices representing the tensors that are outputs to the
+  // interpreter.
+  std::vector<int> outputs_;
+
+  // Array of indices representing the tensors that are variable tensors.
+  std::vector<int> variables_;
+
   // The error reporter delegate that tflite will forward queries errors to.
   ErrorReporter* error_reporter_;
 
@@ -689,9 +690,6 @@ class Interpreter {
 
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
-
-  // Subgraphs
-  std::vector<Subgraph> subgraphs_;
 };
 
 }  // namespace tflite
-- 
GitLab


From 8efc032d609b04686cc47b4c799cb417618991d0 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Fri, 16 Nov 2018 12:35:10 -0800
Subject: [PATCH 0425/1554] Remove
 'nn.quantized_(avg_pool|conv2d|max_pool|relux)' from TF 2.0

Remove the following from TF 2.0:
 - nn.quantized_avg_pool
 - nn.quantized_conv2d
 - nn.quantized_max_pool
 - nn.quantized_relux

PiperOrigin-RevId: 221835284
---
 .../python_api/api_def_QuantizedAvgPool.pbtxt    |  2 ++
 .../python_api/api_def_QuantizedConv2D.pbtxt     |  2 ++
 .../python_api/api_def_QuantizedMaxPool.pbtxt    |  2 ++
 .../python_api/api_def_QuantizedReluX.pbtxt      |  2 ++
 tensorflow/python/ops/nn_ops.py                  |  7 +++++++
 .../tools/api/golden/v2/tensorflow.nn.pbtxt      | 16 ----------------
 tensorflow/tools/compatibility/renames_v2.py     |  4 ++++
 7 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_QuantizedAvgPool.pbtxt b/tensorflow/core/api_def/python_api/api_def_QuantizedAvgPool.pbtxt
index dfa793a16e..6aceba3b11 100644
--- a/tensorflow/core/api_def/python_api/api_def_QuantizedAvgPool.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_QuantizedAvgPool.pbtxt
@@ -2,5 +2,7 @@ op {
   graph_op_name: "QuantizedAvgPool"
   endpoint {
     name: "nn.quantized_avg_pool"
+    deprecation_version: 2
   }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_QuantizedConv2D.pbtxt b/tensorflow/core/api_def/python_api/api_def_QuantizedConv2D.pbtxt
index 2409d12abe..4b5a04f45e 100644
--- a/tensorflow/core/api_def/python_api/api_def_QuantizedConv2D.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_QuantizedConv2D.pbtxt
@@ -2,5 +2,7 @@ op {
   graph_op_name: "QuantizedConv2D"
   endpoint {
     name: "nn.quantized_conv2d"
+    deprecation_version: 2
   }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_QuantizedMaxPool.pbtxt b/tensorflow/core/api_def/python_api/api_def_QuantizedMaxPool.pbtxt
index 3a58590f57..cd1c7fdbf2 100644
--- a/tensorflow/core/api_def/python_api/api_def_QuantizedMaxPool.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_QuantizedMaxPool.pbtxt
@@ -2,5 +2,7 @@ op {
   graph_op_name: "QuantizedMaxPool"
   endpoint {
     name: "nn.quantized_max_pool"
+    deprecation_version: 2
   }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_QuantizedReluX.pbtxt b/tensorflow/core/api_def/python_api/api_def_QuantizedReluX.pbtxt
index 926ec98eeb..d83d71c65c 100644
--- a/tensorflow/core/api_def/python_api/api_def_QuantizedReluX.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_QuantizedReluX.pbtxt
@@ -2,5 +2,7 @@ op {
   graph_op_name: "QuantizedReluX"
   endpoint {
     name: "nn.quantized_relu_x"
+    deprecation_version: 2
   }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index c733d90ab5..21008fc392 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3440,3 +3440,10 @@ def in_top_k(predictions, targets, k, name=None):
   """
   with ops.name_scope(name, "in_top_k"):
     return gen_nn_ops.in_top_kv2(predictions, targets, k, name=name)
+
+
+tf_export(v1=["nn.quantized_avg_pool"])(gen_nn_ops.quantized_avg_pool)
+tf_export(v1=["nn.quantized_conv2d"])(gen_nn_ops.quantized_conv2d)
+tf_export(v1=["nn.quantized_relu_x"])(gen_nn_ops.quantized_relu_x)
+tf_export(v1=["nn.quantized_max_pool"])(gen_nn_ops.quantized_max_pool)
+
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 36aaf7565c..b0d9c5c905 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -224,22 +224,6 @@ tf_module {
     name: "pool"
     argspec: "args=[\'input\', \'window_shape\', \'pooling_type\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'VALID\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "quantized_avg_pool"
-    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "quantized_conv2d"
-    argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'strides\', \'padding\', \'out_type\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'qint32\'>\", \'[1, 1, 1, 1]\', \'None\'], "
-  }
-  member_method {
-    name: "quantized_max_pool"
-    argspec: "args=[\'input\', \'min_input\', \'max_input\', \'ksize\', \'strides\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "quantized_relu_x"
-    argspec: "args=[\'features\', \'max_value\', \'min_features\', \'max_features\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'quint8\'>\", \'None\'], "
-  }
   member_method {
     name: "relu"
     argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5f66d2925e..dde076bf92 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -296,6 +296,10 @@ renames = {
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
+    'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool',
+    'tf.nn.quantized_conv2d': 'tf.compat.v1.nn.quantized_conv2d',
+    'tf.nn.quantized_max_pool': 'tf.compat.v1.nn.quantized_max_pool',
+    'tf.nn.quantized_relu_x': 'tf.compat.v1.nn.quantized_relu_x',
     'tf.nn.raw_rnn': 'tf.compat.v1.nn.raw_rnn',
     'tf.nn.rnn_cell.BasicLSTMCell': 'tf.compat.v1.nn.rnn_cell.BasicLSTMCell',
     'tf.nn.rnn_cell.BasicRNNCell': 'tf.compat.v1.nn.rnn_cell.BasicRNNCell',
-- 
GitLab


From 4a5126674c9c3086a2c38c78126d0e190cb93a61 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Fri, 16 Nov 2018 12:35:39 -0800
Subject: [PATCH 0426/1554] Add IsImmutableConst helper for ImmutableConst
 check.

PiperOrigin-RevId: 221835364
---
 tensorflow/core/grappler/op_types.cc | 4 ++++
 tensorflow/core/grappler/op_types.h  | 1 +
 2 files changed, 5 insertions(+)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 08d4e39ec2..06248393ba 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -253,6 +253,10 @@ bool IsIgammac(const NodeDef& node) { return node.op() == "Igammac"; }
 
 bool IsImag(const NodeDef& node) { return node.op() == "Imag"; }
 
+bool IsImmutableConst(const NodeDef& node) {
+  return node.op() == "ImmutableConst";
+}
+
 bool IsInvGrad(const NodeDef& node) { return node.op() == "InvGrad"; }
 
 bool IsLess(const NodeDef& node) { return node.op() == "Less"; }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 067d4e774f..bd286f2c72 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -77,6 +77,7 @@ bool IsIdentityNSingleInput(const NodeDef& node);
 bool IsIgamma(const NodeDef& node);
 bool IsIgammac(const NodeDef& node);
 bool IsImag(const NodeDef& node);
+bool IsImmutableConst(const NodeDef& node);
 bool IsInvGrad(const NodeDef& node);
 bool IsLess(const NodeDef& node);
 bool IsLessEqual(const NodeDef& node);
-- 
GitLab


From e0cf7147023b31b4d42a39ddf375f323d5207365 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 12:40:48 -0800
Subject: [PATCH 0427/1554] Add support for `add_weight`, `add_variable`, and
 `add_update` for Keras subclassed models.

PiperOrigin-RevId: 221836228
---
 tensorflow/python/keras/engine/network.py     | 56 ++++------------
 .../python/keras/model_subclassing_test.py    | 64 +++++++++++++++++++
 .../golden/v1/tensorflow.keras.-model.pbtxt   |  2 +-
 .../v1/tensorflow.keras.-sequential.pbtxt     |  2 +-
 .../v1/tensorflow.keras.models.-model.pbtxt   |  2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |  2 +-
 .../golden/v2/tensorflow.keras.-model.pbtxt   |  2 +-
 .../v2/tensorflow.keras.-sequential.pbtxt     |  2 +-
 ...ensorflow.keras.layers.-linear-model.pbtxt |  2 +-
 .../v2/tensorflow.keras.models.-model.pbtxt   |  2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |  2 +-
 11 files changed, 87 insertions(+), 51 deletions(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index febc224689..c016020e06 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -112,11 +112,6 @@ class Network(base_layer.Layer):
     self.trainable = True
     self._is_compiled = False
     self._expects_training_arg = False
-    # A list of "extra" variables assigned to attributes of this class, included
-    # in self.weights and self.variables. Always empty for graph networks (but
-    # included in base_init to avoid excessive special casing when retrieving
-    # the value).
-    self._extra_variables = []
     # In many internal cases one needs to compute both the model's output
     # and its output mask without relying on `__call__` (which would do both and
     # set mask metadata), but for models, computing the mask requires to
@@ -134,6 +129,8 @@ class Network(base_layer.Layer):
       self.optimizer = None
 
     # Private attributes to implement compatibility with Layer.
+    self._trainable_weights = []
+    self._non_trainable_weights = []
     self._updates = []  # Used in symbolic mode only.
     self._losses = []
     self._eager_losses = []
@@ -408,40 +405,15 @@ class Network(base_layer.Layer):
             # simply by assigning them to attributes.
           not self._is_graph_network
           and isinstance(value, variables.Variable)):
-        self._extra_variables.append(value)
+        if value.trainable:
+          # Could already be added via `add_weight`.
+          if value not in self._trainable_weights:
+            self._trainable_weights.append(value)
+        else:
+          if value not in self._non_trainable_weights:
+            self._non_trainable_weights.append(value)
     super(Network, self).__setattr__(name, value)
 
-  def add_variable(self, name, shape, dtype=None, initializer=None,
-                   regularizer=None, trainable=True, constraint=None):
-    if self._is_graph_network:
-      raise NotImplementedError('`add_variable` is not supported on Networks.')
-    else:
-      raise NotImplementedError(
-          '`add_variable` is not supported on Networks. However, you may '
-          'assign variables to attributes and they will show up in the weights '
-          'and variables properties.')
-
-  def add_weight(self,
-                 name,
-                 shape,
-                 dtype=None,
-                 initializer=None,
-                 regularizer=None,
-                 trainable=None,
-                 constraint=None,
-                 partitioner=None,
-                 use_resource=None,
-                 synchronization=variables.VariableSynchronization.AUTO,
-                 aggregation=variables.VariableAggregation.NONE,
-                 **kwargs):
-    if self._is_graph_network:
-      raise NotImplementedError('`add_weight` is not supported on Networks.')
-    else:
-      raise NotImplementedError(
-          '`add_weight` is not supported on Networks. However, you may '
-          'assign variables to attributes and they will show up in the weights '
-          'and variables properties.')
-
   @property
   def stateful(self):
     return any([(hasattr(layer, 'stateful') and layer.stateful)
@@ -556,6 +528,7 @@ class Network(base_layer.Layer):
         updates += layer._unfiltered_updates
       else:
         updates += layer.updates
+    updates += self._updates
     return updates
 
   @property
@@ -647,7 +620,7 @@ class Network(base_layer.Layer):
       else:
         relevant_inputs.append(inputs)
     if not relevant_inputs:
-      return updates
+      return list(set(updates))
 
     reachable = tf_utils.get_reachable_from_inputs(relevant_inputs, updates)
     relevant_conditional_updates = [x for x in updates if x in reachable]
@@ -655,8 +628,7 @@ class Network(base_layer.Layer):
         x for x in updates if x._unconditional_update]  # pylint: disable=protected-access
     # A layer could be used multiple times in a nested structure,
     # so the updates list must be de-duped.
-    return list(set(
-        relevant_conditional_updates + unconditional_updates + self._updates))
+    return list(set(relevant_conditional_updates + unconditional_updates))
 
   @property
   def losses(self):
@@ -716,14 +688,14 @@ class Network(base_layer.Layer):
     return checkpointable_layer_utils.gather_trainable_weights(
         trainable=self.trainable,
         sub_layers=self._layers,
-        extra_variables=self._extra_variables)
+        extra_variables=self._trainable_weights)
 
   @property
   def non_trainable_weights(self):
     return checkpointable_layer_utils.gather_non_trainable_weights(
         trainable=self.trainable,
         sub_layers=self._layers,
-        extra_variables=self._extra_variables)
+        extra_variables=self._non_trainable_weights + self._trainable_weights)
 
   @property
   def input_spec(self):
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index aca058b111..87802d8df0 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -25,6 +25,7 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
@@ -819,6 +820,69 @@ class ModelSubclassingTest(test.TestCase):
     self.assertEqual([m.dense.kernel, m.dense.bias, m.not_trainable_var],
                      m.non_trainable_variables)
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_add_weight_in_model(self):
+
+    class MyModel(keras.Model):
+
+      def __init__(self):
+        super(MyModel, self).__init__()
+        self.b = self.add_weight('bias', (10,))
+        self.c = self.add_weight('bias2', (10,), trainable=False)
+
+      def call(self, inputs):
+        return inputs + self.b + self.c
+
+    x = ops.convert_to_tensor(np.ones((10, 10), 'float32'))
+    model = MyModel()
+    model(x)
+    self.assertEqual(1, len(model.trainable_weights))
+    self.assertEqual(1, len(model.non_trainable_weights))
+    self.assertEqual(2, len(model.weights))
+
+    class MyModelCustomBuild(keras.Model):
+
+      def build(self, input_shape):
+        self.b = self.add_weight('bias', (10,))
+        self.c = self.add_weight('bias2', (10,), trainable=False)
+
+      def call(self, inputs):
+        return inputs + self.b + self.c
+
+    x = ops.convert_to_tensor(np.ones((10, 10), 'float32'))
+    model = MyModelCustomBuild()
+    model(x)
+    self.assertEqual(1, len(model.trainable_weights))
+    self.assertEqual(1, len(model.non_trainable_weights))
+    self.assertEqual(2, len(model.weights))
+
+  def test_add_update_in_model(self):
+
+    class MyModel(keras.Model):
+
+      def __init__(self):
+        super(MyModel, self).__init__()
+        self.b = self.add_weight('bias', (10,))
+        self.c = self.add_weight('bias2', (10,))
+
+      def call(self, inputs):
+        # Unconditional
+        self.add_update(self.b.assign(self.b * 2))
+        # Conditional
+        self.add_update(self.c.assign(inputs[1, :]), inputs)
+        return inputs + self.b + self.c
+
+    x = ops.convert_to_tensor(np.ones((10, 10), 'float32'))
+    model = MyModel()
+    model(x)
+
+    if context.executing_eagerly():
+      self.assertEqual(0, len(model.updates))
+    else:
+      self.assertEqual(2, len(model.updates))
+      self.assertEqual(1, len(model.get_updates_for(None)))
+      self.assertEqual(1, len(model.get_updates_for(x)))
+
 
 class GraphSpecificModelSubclassingTests(test.TestCase):
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index e90629407b..75944663e4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -115,7 +115,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 4f734ffa98..9e3e862b79 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -120,7 +120,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 7a6a18b501..5b88db51bf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -115,7 +115,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index f8dacfc886..68e2dbeedd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -120,7 +120,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index e90629407b..75944663e4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -115,7 +115,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 4f734ffa98..9e3e862b79 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -120,7 +120,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index 8e0f0e3826..93372ff0f9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -120,7 +120,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 7a6a18b501..5b88db51bf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -115,7 +115,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index f8dacfc886..68e2dbeedd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -120,7 +120,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
   }
   member_method {
     name: "add_weight"
-- 
GitLab


From 704961fe7298a769bbead3940ac94a82d3daa3eb Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 16 Nov 2018 12:43:46 -0800
Subject: [PATCH 0428/1554] Add documentation for using select TensorFlow ops
 in TensorFlow Lite

This experimental feature allows the use of certain TensorFlow ops from
within the TensorFlow Lite runtime. Using these ops requires opting in to
TF op usage during model conversion, as well as adding an additional
dependency to the client's target.

See `lite/g3doc/using_select_tf_ops.md` for more details. Note that this
feature is under active development and is still in the experimental
stage.

BUG=113614898
PiperOrigin-RevId: 221836702
---
 tensorflow/lite/g3doc/using_select_tf_ops.md | 249 +++++++++++++++++++
 1 file changed, 249 insertions(+)
 create mode 100644 tensorflow/lite/g3doc/using_select_tf_ops.md

diff --git a/tensorflow/lite/g3doc/using_select_tf_ops.md b/tensorflow/lite/g3doc/using_select_tf_ops.md
new file mode 100644
index 0000000000..aa51f58baa
--- /dev/null
+++ b/tensorflow/lite/g3doc/using_select_tf_ops.md
@@ -0,0 +1,249 @@
+# [Experimental] Using TensorFlow Lite with select TensorFlow ops
+
+The TensorFlow Lite builtin op library has grown rapidly, and will continue to
+grow, but there remains a long tail of TensorFlow ops that are not yet natively
+supported by TensorFlow Lite . These unsupported ops can be a point of friction
+in the TensorFlow Lite model conversion process. To that end, the team has
+recently been working on an experimental mechanism for reducing this friction.
+
+This document outlines how to use TensorFlow Lite with select TensorFlow ops.
+*Note that this feature is experimental and is under active development.* As you
+use this feature, keep in mind the [known limitations](#known-limitations), and
+please send feedback about models that work and issues you are facing to
+tflite@tensorflow.org.
+
+TensorFlow Lite will continue to have
+[TensorFlow Lite builtin ops](tf_ops_compatibility.md) optimized for mobile and
+embedded devices. However, TensorFlow Lite models can now use a subset of
+TensorFlow ops when TFLite builtin ops are not sufficient.
+
+Models converted with TensorFlow ops will require a TensorFlow Lite interpreter
+that has a larger binary size than the interpreter with only TFLite builtin ops.
+Additionally, performance optimizations will not be available for any TensorFlow
+ops in the TensorFlow Lite model.
+
+This document outlines how to [convert](#converting-the-model) and
+[run](#running-the-model) a TFLite model with TensorFlow ops on your platform of
+choice. It also discusses some [known limitations](#known-limitations), the
+[future plans](#future-plans) for this feature, and basic
+[performance and size metrics](#metrics).
+
+## Converting the model
+
+To convert a TensorFlow model to a TensorFlow Lite model with TensorFlow ops,
+use the `target_ops` argument in the
+[TensorFlow Lite converter](https://www.tensorflow.org/lite/convert/). The
+following values are valid options for `target_ops`:
+
+*   `TFLITE_BUILTINS` - Converts models using TensorFlow Lite builtin ops.
+*   `SELECT_TF_OPS` - Converts models using TensorFlow ops. The exact subset of
+    supported ops can be found in the whitelist at
+    `lite/toco/tflite/whitelisted_flex_ops.cc`.
+
+The recommended approach is to convert the model with `TFLITE_BUILTINS`, then
+with both `TFLITE_BUILTINS,SELECT_TF_OPS`, and finally with only
+`SELECT_TF_OPS`. Using both options (i.e. `TFLITE_BUILTINS,SELECT_TF_OPS`)
+creates models with TensorFlow Lite ops where possible. Using only
+`SELECT_TF_OPS` is useful when the model contains TensorFlow ops that are only
+partially supported by TensorFlow Lite, and one would like to avoid those
+limitations.
+
+The following example shows how to use `target_ops` in the
+[`TFLiteConverter`](https://www.tensorflow.org/lite/convert/python_api) Python
+API.
+
+```
+import tensorflow as tf
+
+converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir)
+converter.target_ops = [tf.lite.OpsSet.TFLITE_BUILTINS,
+                        tf.lite.OpsSet.SELECT_TF_OPS]
+tflite_model = converter.convert()
+open("converted_model.tflite", "wb").write(tflite_model)
+```
+
+The following example shows how to use `target_ops` in the
+[`tflite_convert`](https://www.tensorflow.org/lite/convert/cmdline_examples)
+command line tool.
+
+```
+tflite_convert \
+  --output_file=/tmp/foo.tflite \
+  --graph_def_file=/tmp/foo.pb \
+  --input_arrays=input \
+  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
+  --target_ops=TFLITE_BUILTINS,SELECT_TF_OPS
+```
+
+When building and running `tflite_convert` directly with `bazel`, please pass
+`--define=with_select_tf_ops=true` as an additional argument.
+
+```
+bazel run --define=with_select_tf_ops=true tflite_convert -- \
+  --output_file=/tmp/foo.tflite \
+  --graph_def_file=/tmp/foo.pb \
+  --input_arrays=input \
+  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
+  --target_ops=TFLITE_BUILTINS,SELECT_TF_OPS
+```
+
+## Running the model
+
+When using a TensorFlow Lite model that has been converted with support for
+select TensorFlow ops, the client must also use a TensorFlow Lite runtime that
+includes the necessary library of TensorFlow ops.
+
+### Android AAR
+
+A new Android AAR target with select TensorFlow ops has been added for
+convenience. Assuming a <a href="./demo_android.md">working TensorFlow Lite
+build environment</a>, build the Android AAR with select TensorFlow ops as
+follows:
+
+```sh
+bazel build --cxxopt='--std=c++11' -c opt             \
+  --config=android_arm --config=monolithic          \
+  //tensorflow/lite/java:tensorflow-lite-with-select-tf-ops
+```
+
+This will generate an AAR file in `bazel-genfiles/tensorflow/lite/java/`. From
+there, you can either import the AAR directly into your project, or publish the
+custom AAR to your local Maven repository:
+
+```sh
+mvn install:install-file \
+  -Dfile=bazel-genfiles/tensorflow/lite/java/tensorflow-lite-with-select-tf-ops.aar \
+  -DgroupId=org.tensorflow \
+  -DartifactId=tensorflow-lite-with-select-tf-ops -Dversion=0.1.100 -Dpackaging=aar
+```
+
+Finally, in your app's `build.gradle`, ensure you have the `mavenLocal()`
+dependency and replace the standard TensorFlow Lite dependency with the one that
+has support for select TensorFlow ops:
+
+```
+allprojects {
+    repositories {
+        jcenter()
+        mavenLocal()
+    }
+}
+
+dependencies {
+    compile 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100'
+}
+```
+
+### iOS
+
+With XCode Command Line Tools installed, TensorFlow Lite with select TensorFlow
+ops support can be built with the following command:
+
+```sh
+tensorflow/contrib/makefile/build_all_ios_with_tflite.sh
+```
+
+This will generate the required static linking libraries in the
+`tensorflow/contrib/makefile/gen/lib/` directory.
+
+The TensorFlow Lite camera example app can be used to test this. A new
+TensorFlow Lite XCode project with support for select TensorFlow ops has been
+added to
+`tensorflow/lite/examples/ios/camera/tflite_camera_example_with_select_tf_ops.xcodeproj`.
+
+To use this feature in a your own project, either clone the example project or
+set the project settings for a new or existing project to the following:
+
+*   In Build Phases -> Link Binary With Libraries, add the static libraries
+    under `tensorflow/contrib/makefile/gen/lib/` directory:
+    *   `libtensorflow-lite.a`
+    *   `libprotobuf.a`
+    *   `nsync.a`
+*   In Build Settings -> Header Search Paths, add the following directories:
+    *   `tensorflow/lite/`
+    *   `tensorflow/contrib/makefile/downloads/flatbuffer/include`
+    *   `tensorflow/contrib/makefile/downloads/eigen`
+*   In Build Settings -> Other Linker Flags, add `-force_load
+    tensorflow/contrib/makefile/gen/lib/libtensorflow-lite.a`.
+
+A CocoaPod with support for select TensorFlow ops will also be released in the
+future.
+
+### C++
+
+When building TensorFlow Lite libraries using the bazel pipeline, the additional
+TensorFlow ops library can be included and enabled as follows:
+
+*   Enable monolithic builds if necessary by adding the `--config=monolithic`
+    build flag.
+*   Do one of the following:
+    *   Include the `--define=with_select_tf_ops=true` build flag in the `bazel
+        build` invocation when building TensorFlow Lite.
+    *   Add the TensorFlow ops delegate library dependency to the build
+        dependencies: `tensorflow/lite/delegates/flex:delegate`.
+
+Note that the necessary `TfLiteDelegate` will be installed automatically when
+creating the interpreter at runtime as long as the delegate is linked into the
+client library. It is not necessary to explicitly install the delegate instance
+as is typically required with other delegate types.
+
+### Python pip Package
+
+Python support is actively under development.
+
+## Metrics
+
+### Performance
+
+When using a mixture of both builtin and select TensorFlow ops, all of the same
+TensorFlow Lite optimizations and optimized builtin kernels will be be available
+and usable with the converted model. For the TensorFlow ops, performance should
+generally be comparable to that of
+[TensorFlow Mobile](https://www.tensorflow.org/lite/tfmobile/).
+
+The following table describes the average time taken to run inference on
+MobileNet on a Pixel 2. The listed times are an average of 100 runs. These
+targets were built for Android using the flags: `--config=android_arm64 -c opt`.
+
+Build                                | Time (milliseconds)
+------------------------------------ | -------------------
+Only built-in ops (`TFLITE_BUILTIN`) | 260.7
+Using only TF ops (`SELECT_TF_OPS`)  | 264.5
+
+### Binary Size
+
+The following table describes the binary size of TensorFlow Lite for each build.
+These targets were built for Android using `--config=android_arm -c opt`.
+
+Build                 | C++ Binary Size | Android APK Size
+--------------------- | --------------- | ----------------
+Only built-in ops     | 796 KB          | 561 KB
+Built-in ops + TF ops | 23.0 MB         | 8.0 MB
+
+## Known Limitations
+
+The following is a list of some of the known limitations:
+
+*   Control flow ops are not yet supported.
+*   The
+    [`post_training_quantization`](https://www.tensorflow.org/performance/post_training_quantization)
+    flag is currently not supported for TensorFlow ops so it will not quantize
+    weights for any TensorFlow ops. In models with both TensorFlow Lite builtin
+    ops and TensorFlow ops, the weights for the builtin ops will be quantized.
+*   Ops that require explicit initialization from resources, like HashTableV2,
+    are not yet supported.
+*   Certain TensorFlow ops may not support the full set of input/output types
+    that are typically available on stock TensorFlow.
+
+## Future Plans
+
+The following is a list of improvements to this pipeline that are in progress:
+
+*   *Selective registration* - There is work being done to make it simple to
+    generate TFLite interpreter binaries that only contain the TensorFlow ops
+    required for a particular set of models.
+*   *Improved usability* - The conversion process will be simplified to only
+    require a single pass through the converter. Additionally, pre-built Android
+    AAR and iOS CocoaPod binaries will be provided.
+*   *Improved performance* - There is work being done to ensure TensorFlow Lite
+    with TensorFlow ops has performance parity to TensorFlow Mobile.
-- 
GitLab


From 4fe22bc9da311b4c3c5d0ab2cd30ae43709a7772 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 16 Nov 2018 12:44:53 -0800
Subject: [PATCH 0429/1554] Replace many calls to eval() with self.evaluate()

In order to get tests running in eager mode we need to remove invalid
functions calls such as eval(). This change is simply a search and
replace for tests where this was safe. As a result, a few more tests now
work in eager mode.

PiperOrigin-RevId: 221836866
---
 tensorflow/compiler/tests/adagrad_da_test.py  |  32 +-
 tensorflow/compiler/tests/adagrad_test.py     |  30 +-
 tensorflow/compiler/tests/adam_test.py        |  39 +-
 tensorflow/compiler/tests/adamax_test.py      |  24 +-
 tensorflow/compiler/tests/addsign_test.py     |   8 +-
 tensorflow/compiler/tests/clustering_test.py  |   4 +-
 tensorflow/compiler/tests/concat_ops_test.py  |  12 +-
 tensorflow/compiler/tests/conv3d_test.py      |   6 +-
 tensorflow/compiler/tests/fifo_queue_test.py  |   6 +-
 tensorflow/compiler/tests/ftrl_test.py        |  84 +--
 tensorflow/compiler/tests/lrn_ops_test.py     |   4 +-
 tensorflow/compiler/tests/momentum_test.py    |  64 +-
 tensorflow/compiler/tests/powersign_test.py   |   8 +-
 .../compiler/tests/proximal_adagrad_test.py   |  40 +-
 .../tests/proximal_gradient_descent_test.py   |  38 +-
 tensorflow/compiler/tests/qr_op_test.py       |   2 +-
 tensorflow/compiler/tests/rmsprop_test.py     |  20 +-
 .../compiler/tests/tensor_array_ops_test.py   |  25 +-
 .../examples/adding_an_op/zero_out_3_test.py  |   4 +-
 .../audio_microfrontend_op_test.py            |   4 +-
 tensorflow/python/data/util/sparse_test.py    |   6 +-
 .../python/debug/wrappers/framework_test.py   |   2 +-
 .../python/distribute/all_reduce_test.py      |   2 +-
 tensorflow/python/eager/backprop_test.py      |   2 +-
 .../python/eager/graph_only_ops_test.py       |   2 +-
 tensorflow/python/eager/tape_test.py          |   8 +-
 .../feature_column/feature_column_test.py     | 500 +++++++--------
 .../feature_column/feature_column_v2_test.py  | 581 +++++++++---------
 tensorflow/python/framework/function_test.py  |   4 +-
 tensorflow/python/framework/importer_test.py  |   6 +-
 tensorflow/python/framework/ops_test.py       |  22 +-
 .../python/framework/sparse_tensor_test.py    |   4 +-
 .../keras/optimizer_v2/adadelta_test.py       |   5 +-
 .../python/keras/optimizer_v2/adam_test.py    |  43 +-
 .../optimizer_v2/gradient_descent_test.py     |  96 +--
 .../python/keras/optimizer_v2/rmsprop_test.py |  54 +-
 .../python/kernel_tests/accumulate_n_test.py  |   4 +-
 .../python/kernel_tests/argmax_op_test.py     |   8 +-
 .../python/kernel_tests/atrous_conv2d_test.py |  12 +-
 .../python/kernel_tests/attention_ops_test.py |   3 +-
 .../python/kernel_tests/basic_gpu_test.py     |   2 +-
 .../kernel_tests/batch_gather_op_test.py      |   6 +-
 .../kernel_tests/batch_matmul_op_test.py      |   2 +-
 .../kernel_tests/batch_scatter_ops_test.py    |   2 +-
 .../python/kernel_tests/betainc_op_test.py    |   2 +-
 .../python/kernel_tests/bitcast_op_test.py    |   2 +-
 .../boosted_trees/resource_ops_test.py        |  42 +-
 .../boosted_trees/stats_ops_test.py           |   6 +-
 .../candidate_sampler_ops_test.py             |   8 +-
 .../python/kernel_tests/cast_op_test.py       |   4 +-
 .../kernel_tests/checkpoint_ops_test.py       |  43 +-
 .../python/kernel_tests/clip_ops_test.py      |  48 +-
 .../compare_and_bitpack_op_test.py            |   4 +-
 .../python/kernel_tests/concat_op_test.py     |  20 +-
 .../conditional_accumulator_test.py           |  28 +-
 .../kernel_tests/confusion_matrix_test.py     |  34 +-
 .../python/kernel_tests/constant_op_test.py   |  52 +-
 .../kernel_tests/control_flow_ops_py_test.py  | 231 +++----
 tensorflow/python/kernel_tests/conv1d_test.py |   4 +-
 .../kernel_tests/conv2d_transpose_test.py     |  12 +-
 .../kernel_tests/conv3d_transpose_test.py     |   8 +-
 .../kernel_tests/cwise_ops_binary_test.py     |   8 +-
 .../python/kernel_tests/cwise_ops_test.py     |  42 +-
 .../kernel_tests/cwise_ops_unary_test.py      |   4 +-
 .../python/kernel_tests/decode_bmp_op_test.py |   4 +-
 .../kernel_tests/decode_image_op_test.py      |   6 +-
 .../python/kernel_tests/decode_png_op_test.py |   2 +-
 .../dense_update_ops_no_tsan_test.py          |   8 +-
 .../kernel_tests/dense_update_ops_test.py     |  16 +-
 .../kernel_tests/depthtospace_op_test.py      |  12 +-
 .../kernel_tests/depthwise_conv_op_test.py    |   8 +-
 .../kernel_tests/determinant_op_test.py       |   6 +-
 .../python/kernel_tests/diag_op_test.py       |  18 +-
 .../distributions/categorical_test.py         |  18 +-
 .../dirichlet_multinomial_test.py             |  30 +-
 .../distributions/kullback_leibler_test.py    |   8 +-
 .../distributions/multinomial_test.py         |  12 +-
 .../distributions/special_math_test.py        |   2 +-
 .../kernel_tests/dynamic_stitch_op_test.py    |  26 +-
 .../kernel_tests/edit_distance_op_test.py     |   4 +-
 .../python/kernel_tests/embedding_ops_test.py |   6 +-
 .../extract_image_patches_op_test.py          |   2 +-
 .../extract_volume_patches_op_test.py         |   2 +-
 .../python/kernel_tests/fifo_queue_test.py    |  56 +-
 .../fractional_avg_pool_op_test.py            |  16 +-
 .../fractional_max_pool_op_test.py            |  20 +-
 .../kernel_tests/functional_ops_test.py       |   2 +-
 .../python/kernel_tests/gather_nd_op_test.py  |  46 +-
 .../python/kernel_tests/gather_op_test.py     |  13 +-
 .../python/kernel_tests/huge_slice_op_test.py |   4 +-
 .../python/kernel_tests/in_topk_op_test.py    |   4 +-
 .../python/kernel_tests/init_ops_test.py      |  16 +-
 .../kernel_tests/large_concat_op_test.py      |   2 +-
 .../linalg/linear_operator_circulant_test.py  |  13 +-
 .../linalg/linear_operator_identity_test.py   |   4 +-
 .../linalg/linear_operator_kronecker_test.py  |   4 +-
 .../linalg/linear_operator_test.py            |   6 +-
 .../linalg/linear_operator_util_test.py       |  24 +-
 .../python/kernel_tests/linalg_grad_test.py   |   4 +-
 .../python/kernel_tests/linalg_ops_test.py    |  16 +-
 .../python/kernel_tests/logging_ops_test.py   |   8 +-
 .../python/kernel_tests/lookup_ops_test.py    | 142 ++---
 tensorflow/python/kernel_tests/losses_test.py | 153 ++---
 .../python/kernel_tests/matmul_op_test.py     |   6 +-
 .../kernel_tests/matrix_band_part_op_test.py  |   2 +-
 .../matrix_exponential_op_test.py             |   2 +-
 .../kernel_tests/matrix_inverse_op_test.py    |   2 +-
 .../kernel_tests/matrix_logarithm_op_test.py  |   2 +-
 .../kernel_tests/matrix_solve_op_test.py      |   2 +-
 .../matrix_triangular_solve_op_test.py        |   2 +-
 .../kernel_tests/morphological_ops_test.py    |   8 +-
 .../python/kernel_tests/numerics_test.py      |  14 +-
 .../python/kernel_tests/one_hot_op_test.py    |   4 +-
 tensorflow/python/kernel_tests/pad_op_test.py |  29 +-
 .../kernel_tests/padding_fifo_queue_test.py   |  58 +-
 .../python/kernel_tests/parsing_ops_test.py   |   4 +-
 .../partitioned_variables_test.py             |  16 +-
 tensorflow/python/kernel_tests/pool_test.py   |   2 +-
 .../python/kernel_tests/pooling_ops_test.py   |  30 +-
 .../python/kernel_tests/py_func_test.py       |   6 +-
 tensorflow/python/kernel_tests/qr_op_test.py  |   2 +-
 .../random/multinomial_op_test.py             |   2 +-
 .../kernel_tests/random/random_crop_test.py   |   4 +-
 .../random/random_poisson_test.py             |   2 +-
 .../random/random_shuffle_queue_test.py       |  32 +-
 .../random/stateless_random_ops_test.py       |   2 +-
 .../python/kernel_tests/reader_ops_test.py    |  42 +-
 .../kernel_tests/reduce_join_op_test.py       |  14 +-
 .../python/kernel_tests/reduction_ops_test.py |   8 +-
 .../kernel_tests/regex_full_match_op_test.py  |   2 +-
 .../kernel_tests/regex_replace_op_test.py     |   2 +-
 .../python/kernel_tests/reshape_op_test.py    |   4 +-
 .../resource_variable_ops_test.py             |   2 +-
 .../kernel_tests/reverse_sequence_op_test.py  |   4 +-
 .../kernel_tests/scatter_nd_ops_test.py       |  24 +-
 .../python/kernel_tests/scatter_ops_test.py   |   2 +-
 .../segment_reduction_ops_test.py             |  71 +--
 .../kernel_tests/self_adjoint_eig_op_test.py  |   4 +-
 .../python/kernel_tests/shape_ops_test.py     |  52 +-
 .../kernel_tests/signal/mel_ops_test.py       |   2 +-
 .../kernel_tests/signal/shape_ops_test.py     |   8 +-
 .../kernel_tests/signal/spectral_ops_test.py  |  10 +-
 .../python/kernel_tests/slice_op_test.py      |  40 +-
 .../python/kernel_tests/softmax_op_test.py    |   4 +-
 .../python/kernel_tests/softplus_op_test.py   |   2 +-
 .../python/kernel_tests/softsign_op_test.py   |   2 +-
 .../kernel_tests/spacetodepth_op_test.py      |  12 +-
 .../kernel_tests/sparse_cross_op_test.py      |   2 +-
 .../kernel_tests/sparse_matmul_op_test.py     |   2 +-
 .../python/kernel_tests/sparse_ops_test.py    |  14 +-
 .../sparse_tensor_dense_matmul_op_test.py     |   2 +-
 .../sparse_to_dense_op_py_test.py             |  18 +-
 .../kernel_tests/sparse_xent_op_test.py       |   2 +-
 .../python/kernel_tests/split_op_test.py      |   2 +-
 .../python/kernel_tests/stack_op_test.py      |  12 +-
 .../python/kernel_tests/stack_ops_test.py     |  18 +-
 .../string_to_hash_bucket_op_test.py          |   4 +-
 .../python/kernel_tests/substr_op_test.py     |  36 +-
 tensorflow/python/kernel_tests/svd_op_test.py |   2 +-
 .../kernel_tests/tensor_array_ops_test.py     |  10 +-
 .../python/kernel_tests/transpose_op_test.py  |  18 +-
 .../kernel_tests/unicode_decode_op_test.py    |  24 +-
 .../python/kernel_tests/variable_ops_test.py  |  16 +-
 .../kernel_tests/variable_scope_test.py       |   2 +-
 .../python/kernel_tests/variables_test.py     | 165 ++---
 .../kernel_tests/weights_broadcast_test.py    |   2 +-
 .../python/kernel_tests/where_op_test.py      |   4 +-
 .../python/kernel_tests/zero_division_test.py |   2 +-
 .../python/ops/control_flow_ops_test.py       |  14 +-
 tensorflow/python/ops/dequantize_op_test.py   |   2 +-
 tensorflow/python/ops/gradients_test.py       |  18 +-
 tensorflow/python/ops/histogram_ops_test.py   |  18 +-
 tensorflow/python/ops/image_ops_test.py       | 155 ++---
 tensorflow/python/ops/math_ops_test.py        |  13 +-
 tensorflow/python/ops/nn_batchnorm_test.py    |   4 +-
 .../python/ops/nn_fused_batchnorm_test.py     |   4 +-
 tensorflow/python/ops/nn_test.py              |  26 +-
 tensorflow/python/ops/nn_xent_test.py         |   8 +-
 ...vert_to_tensor_or_ragged_tensor_op_test.py |   2 +-
 .../python/ops/ragged/ragged_const_op_test.py |   2 +-
 .../ops/ragged/ragged_from_tensor_op_test.py  |   2 +-
 .../ops/ragged/ragged_map_fn_op_test.py       |   6 +-
 .../ops/ragged/ragged_segment_op_test.py      |   2 +-
 .../python/ops/ragged/ragged_where_op_test.py |   2 +-
 tensorflow/python/saved_model/loader_test.py  |   2 +-
 .../python/saved_model/saved_model_test.py    |   6 +-
 .../python/saved_model/simple_save_test.py    |   2 +-
 .../python/summary/writer/writer_test.py      |   7 +-
 tensorflow/python/training/adadelta_test.py   |   5 +-
 tensorflow/python/training/adagrad_da_test.py |   7 +-
 tensorflow/python/training/adagrad_test.py    |  49 +-
 tensorflow/python/training/adam_test.py       |  43 +-
 .../training/checkpoint_management_test.py    |   4 +-
 .../python/training/checkpoint_ops_test.py    |   3 +-
 tensorflow/python/training/ftrl_test.py       |   6 +-
 .../python/training/gradient_descent_test.py  |  68 +-
 tensorflow/python/training/input_test.py      |  48 +-
 tensorflow/python/training/momentum_test.py   | 120 ++--
 .../python/training/moving_averages_test.py   |  36 +-
 tensorflow/python/training/optimizer_test.py  |  24 +-
 .../python/training/proximal_adagrad_test.py  |   7 +-
 .../proximal_gradient_descent_test.py         |   7 +-
 .../python/training/queue_runner_test.py      |  18 +-
 tensorflow/python/training/rmsprop_test.py    | 108 ++--
 tensorflow/python/training/server_lib_test.py |   2 +-
 .../python/training/slot_creator_test.py      |  12 +-
 tensorflow/python/training/supervisor_test.py |   4 +-
 .../python/training/training_ops_test.py      |  49 +-
 tensorflow/python/util/tf_should_use_test.py  |   2 +-
 209 files changed, 2590 insertions(+), 2403 deletions(-)

diff --git a/tensorflow/compiler/tests/adagrad_da_test.py b/tensorflow/compiler/tests/adagrad_da_test.py
index 69fb3ec296..e9c2d363ac 100644
--- a/tensorflow/compiler/tests/adagrad_da_test.py
+++ b/tensorflow/compiler/tests/adagrad_da_test.py
@@ -50,8 +50,8 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        self.assertAllClose([0.0, 0.0], var0.eval())
-        self.assertAllClose([0.0, 0.0], var1.eval())
+        self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+        self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
         # Run a step of AdagradDA
         update.run()
@@ -63,9 +63,9 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase):
         # For -0.1*3.0*(0.1 - 0)/(0 + sqrt(0.1 + 0.1*0.1)) = -0.904534
         # similarly for others.
         self.assertAllCloseAccordingToType(
-            np.array([-0.904534, -1.603567]), var0.eval())
+            np.array([-0.904534, -1.603567]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([-0.094821, -0.189358]), var1.eval())
+            np.array([-0.094821, -0.189358]), self.evaluate(var1))
 
   def testAdagradDAwithoutRegularizationBasic2(self):
     for dtype in self.float_types:
@@ -87,16 +87,16 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1))
 
         # Run a step of AdagradDA
         update.run()
 
         self.assertAllCloseAccordingToType(
-            np.array([-0.904534, -1.603567]), var0.eval())
+            np.array([-0.904534, -1.603567]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([-0.094821, -0.189358]), var1.eval())
+            np.array([-0.094821, -0.189358]), self.evaluate(var1))
 
   def testAdagradDAWithL1(self):
     for dtype in self.float_types:
@@ -118,16 +118,16 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1))
 
         # Run a step of AdagradDA
         update.run()
 
         self.assertAllCloseAccordingToType(
-            np.array([-0.895489, -1.59555]), var0.eval())
+            np.array([-0.895489, -1.59555]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([-0.085339, -0.17989]), var1.eval())
+            np.array([-0.085339, -0.17989]), self.evaluate(var1))
 
   def testAdagradDAWithL1_L2(self):
     for dtype in self.float_types:
@@ -149,16 +149,16 @@ class AdagradDAOptimizerTest(xla_test.XLATestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1))
 
         # Run a step of AdagradDA
         update.run()
 
         self.assertAllCloseAccordingToType(
-            np.array([-0.046907, -0.093659]), var0.eval())
+            np.array([-0.046907, -0.093659]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([-0.004275, -0.009023]), var1.eval())
+            np.array([-0.004275, -0.009023]), self.evaluate(var1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tests/adagrad_test.py b/tensorflow/compiler/tests/adagrad_test.py
index ab69319c59..e26483303c 100644
--- a/tensorflow/compiler/tests/adagrad_test.py
+++ b/tensorflow/compiler/tests/adagrad_test.py
@@ -42,17 +42,19 @@ class AdagradOptimizerTest(xla_test.XLATestCase):
             zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Run 3 steps of adagrad
         for _ in range(3):
           ada_update.run()
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(),
+            np.array([-1.6026098728179932, -0.6026098728179932]),
+            self.evaluate(var0),
             float_rtol=1e-5)
         self.assertAllCloseAccordingToType(
-            np.array([2.715679168701172, 3.715679168701172]), var1.eval(),
+            np.array([2.715679168701172, 3.715679168701172]),
+            self.evaluate(var1),
             float_rtol=1e-5)
 
   def testTensorLearningRate(self):
@@ -68,17 +70,19 @@ class AdagradOptimizerTest(xla_test.XLATestCase):
             zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Run 3 steps of adagrad
         for _ in range(3):
           ada_update.run()
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(),
+            np.array([-1.6026098728179932, -0.6026098728179932]),
+            self.evaluate(var0),
             float_rtol=1e-5)
         self.assertAllCloseAccordingToType(
-            np.array([2.715679168701172, 3.715679168701172]), var1.eval(),
+            np.array([2.715679168701172, 3.715679168701172]),
+            self.evaluate(var1),
             float_rtol=1e-5)
 
   def testSharing(self):
@@ -103,18 +107,20 @@ class AdagradOptimizerTest(xla_test.XLATestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values.
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Mix the first and the second adagrad for 3 steps.
         ada_update1.run()
         ada_update2.run()
         ada_update1.run()
         # Validate updated params (the same as with only 1 Adagrad).
         self.assertAllCloseAccordingToType(
-            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(),
+            np.array([-1.6026098728179932, -0.6026098728179932]),
+            self.evaluate(var0),
             float_rtol=1e-5)
         self.assertAllCloseAccordingToType(
-            np.array([2.715679168701172, 3.715679168701172]), var1.eval(),
+            np.array([2.715679168701172, 3.715679168701172]),
+            self.evaluate(var1),
             float_rtol=1e-5)
 
 
diff --git a/tensorflow/compiler/tests/adam_test.py b/tensorflow/compiler/tests/adam_test.py
index 058576b3d4..8bcff9d379 100644
--- a/tensorflow/compiler/tests/adam_test.py
+++ b/tensorflow/compiler/tests/adam_test.py
@@ -75,23 +75,24 @@ class AdamOptimizerTest(xla_test.XLATestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
         # Run 3 steps of Adam
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           update.run(feed_dict={grads0: grads0_np, grads1: grads1_np})
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testTensorLearningRate(self):
     for dtype in self.float_types:
@@ -117,23 +118,24 @@ class AdamOptimizerTest(xla_test.XLATestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
         # Run 3 steps of Adam
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           update.run(feed_dict={grads0: grads0_np, grads1: grads1_np})
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testSharing(self):
     for dtype in self.float_types:
@@ -162,13 +164,14 @@ class AdamOptimizerTest(xla_test.XLATestCase):
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 3 steps of intertwined Adam1 and Adam2.
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           if t % 2 == 0:
             update1.run(feed_dict={grads0: grads0_np, grads1: grads1_np})
           else:
@@ -178,8 +181,8 @@ class AdamOptimizerTest(xla_test.XLATestCase):
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tests/adamax_test.py b/tensorflow/compiler/tests/adamax_test.py
index 3ed1d41b71..961b46375c 100644
--- a/tensorflow/compiler/tests/adamax_test.py
+++ b/tensorflow/compiler/tests/adamax_test.py
@@ -78,8 +78,8 @@ class AdaMaxOptimizerTest(xla_test.XLATestCase):
 
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power = opt._get_beta_accumulators()
 
@@ -87,14 +87,17 @@ class AdaMaxOptimizerTest(xla_test.XLATestCase):
         for t in range(1, 4):
           update.run()
 
-          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta1_power))
 
           var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval(), rtol=1e-2)
-          self.assertAllCloseAccordingToType(var1_np, var1.eval(), rtol=1e-2)
+          self.assertAllCloseAccordingToType(
+              var0_np, self.evaluate(var0), rtol=1e-2)
+          self.assertAllCloseAccordingToType(
+              var1_np, self.evaluate(var1), rtol=1e-2)
           self.assertEqual("var0_%d/AdaMax:0" % (i,),
                            opt.get_slot(var=var0, name="m").name)
 
@@ -118,22 +121,23 @@ class AdaMaxOptimizerTest(xla_test.XLATestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power = opt._get_beta_accumulators()
 
         # Run 3 steps of AdaMax
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
           update.run()
 
           var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/compiler/tests/addsign_test.py b/tensorflow/compiler/tests/addsign_test.py
index 1bc07ace23..a37c97e6d3 100644
--- a/tensorflow/compiler/tests/addsign_test.py
+++ b/tensorflow/compiler/tests/addsign_test.py
@@ -90,8 +90,8 @@ class AddSignTest(xla_test.XLATestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 7 steps of AddSign
         # first 4 steps with positive gradient
@@ -125,8 +125,8 @@ class AddSignTest(xla_test.XLATestCase):
 
           # Validate updated params
           self.assertAllCloseAccordingToType(
-              var0_np, var0.eval(), half_rtol=1e-2)
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+              var0_np, self.evaluate(var0), half_rtol=1e-2)
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testDense(self):
     decay_steps = 10
diff --git a/tensorflow/compiler/tests/clustering_test.py b/tensorflow/compiler/tests/clustering_test.py
index 88bd58b2da..ef2d7af69d 100644
--- a/tensorflow/compiler/tests/clustering_test.py
+++ b/tensorflow/compiler/tests/clustering_test.py
@@ -43,7 +43,7 @@ class ClusteringTest(xla_test.XLATestCase):
         input1 = constant_op.constant(val1, name="const1")
         input2 = constant_op.constant(val2, name="const2")
         output = math_ops.add(input1, input2)
-      result = output.eval()
+      result = self.evaluate(output)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testAddFromCpuMultiple(self):
@@ -57,7 +57,7 @@ class ClusteringTest(xla_test.XLATestCase):
       with self.test_scope():
         output = math_ops.add(input1, input2)
       for _ in xrange(10):
-        result = output.eval()
+        result = self.evaluate(output)
         self.assertAllClose(result, expected, rtol=1e-3)
 
   def testDeadlock(self):
diff --git a/tensorflow/compiler/tests/concat_ops_test.py b/tensorflow/compiler/tests/concat_ops_test.py
index 2d225ad226..30fbe6f701 100644
--- a/tensorflow/compiler/tests/concat_ops_test.py
+++ b/tensorflow/compiler/tests/concat_ops_test.py
@@ -72,7 +72,7 @@ class ConcatTest(xla_test.XLATestCase):
       x2 = constant_op.constant(p2)
       with self.test_scope():
         c = array_ops.concat([x1, x2], 0)
-      result = c.eval()
+      result = self.evaluate(c)
     self.assertAllEqual(result[:2, :], p1)
     self.assertAllEqual(result[2:, :], p2)
 
@@ -150,7 +150,7 @@ class ConcatTest(xla_test.XLATestCase):
             [float(x) for x in grad_inp.flatten()], shape=output_shape)
         grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
         concated_grad = array_ops.concat(grad, 1)
-      result = concated_grad.eval()
+      result = self.evaluate(concated_grad)
     self.assertAllEqual(result, grad_inp)
 
   def testGradientsSimpleAll(self):
@@ -177,7 +177,7 @@ class ConcatTest(xla_test.XLATestCase):
             [float(x) for x in grad_inp.flatten()], shape=output_shape)
         grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
         concated_grad = array_ops.concat(grad, 0)
-        result = concated_grad.eval()
+        result = self.evaluate(concated_grad)
 
     self.assertAllEqual(result, grad_inp)
 
@@ -205,7 +205,7 @@ class ConcatTest(xla_test.XLATestCase):
             [float(x) for x in grad_inp.flatten()], shape=output_shape)
         grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
         concated_grad = array_ops.concat(grad, 2)
-        result = concated_grad.eval()
+        result = self.evaluate(concated_grad)
 
     self.assertAllEqual(result, grad_inp)
 
@@ -242,7 +242,7 @@ class ConcatTest(xla_test.XLATestCase):
             [float(x) for x in grad_inp.flatten()], shape=output_shape)
         grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
         concated_grad = array_ops.concat(grad, concat_dim)
-        result = concated_grad.eval()
+        result = self.evaluate(concated_grad)
 
     self.assertAllEqual(result, grad_inp)
 
@@ -280,7 +280,7 @@ class ConcatTest(xla_test.XLATestCase):
       with self.test_scope():
         concat_list_t = array_ops.concat([c1, c2], 0)
         concat_tuple_t = array_ops.concat((c1, c2), 0)
-      self.assertAllEqual(concat_list_t.eval(), concat_tuple_t.eval())
+      self.assertAllEqual(concat_list_t.eval(), self.evaluate(concat_tuple_t))
 
   def testConcatNoScalars(self):
     with self.cached_session():
diff --git a/tensorflow/compiler/tests/conv3d_test.py b/tensorflow/compiler/tests/conv3d_test.py
index d59fd0236f..01cc1b6392 100644
--- a/tensorflow/compiler/tests/conv3d_test.py
+++ b/tensorflow/compiler/tests/conv3d_test.py
@@ -85,7 +85,7 @@ class Conv3DTransposeTest(xla_test.XLATestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv3d_transpose(
           x, f, y_shape, strides=strides, padding="SAME")
-      value = output.eval()
+      value = self.evaluate(output)
 
       # We count the number of cells being added at the locations in the output.
       # At the center, #cells = kernel_depth * kernel_height * kernel_width
@@ -135,7 +135,7 @@ class Conv3DTransposeTest(xla_test.XLATestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv3d_transpose(
           x, f, y_shape, strides=strides, padding="SAME")
-      value = output.eval()
+      value = self.evaluate(output)
 
       for n in xrange(x_shape[0]):
         for k in xrange(f_shape[3]):
@@ -173,7 +173,7 @@ class Conv3DTransposeTest(xla_test.XLATestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv3d_transpose(
           x, f, y_shape, strides=strides, padding="VALID")
-      value = output.eval()
+      value = self.evaluate(output)
 
       cache_values = np.zeros(y_shape, dtype=np.float32)
 
diff --git a/tensorflow/compiler/tests/fifo_queue_test.py b/tensorflow/compiler/tests/fifo_queue_test.py
index 8c7edfd277..91d77d2f79 100644
--- a/tensorflow/compiler/tests/fifo_queue_test.py
+++ b/tensorflow/compiler/tests/fifo_queue_test.py
@@ -129,7 +129,7 @@ class FIFOQueueTest(xla_test.XLATestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        vals = dequeued_t.eval()
+        vals = self.evaluate(dequeued_t)
         self.assertEqual([elems[i]], vals)
 
   def testEnqueueAndBlockingDequeue(self):
@@ -192,9 +192,9 @@ class FIFOQueueTest(xla_test.XLATestCase):
       self.assertEqual([], size.get_shape())
 
       enqueue_op.run()
-      self.assertEqual(1, size.eval())
+      self.assertEqual(1, self.evaluate(size))
       dequeued_t.op.run()
-      self.assertEqual(0, size.eval())
+      self.assertEqual(0, self.evaluate(size))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tests/ftrl_test.py b/tensorflow/compiler/tests/ftrl_test.py
index 5b197afd65..b078053cdb 100644
--- a/tensorflow/compiler/tests/ftrl_test.py
+++ b/tensorflow/compiler/tests/ftrl_test.py
@@ -50,14 +50,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
     ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
     variables.global_variables_initializer().run()
     # Fetch params to validate initial values
-    self.assertAllClose([0.0, 0.0], var0.eval())
-    self.assertAllClose([0.0, 0.0], var1.eval())
+    self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+    self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
     # Run Ftrl for a few steps
     for _ in range(steps):
       ftrl_update.run()
 
-    return var0.eval(), var1.eval()
+    return self.evaluate(var0), self.evaluate(var1)
 
   def equivAdagradTest_AdagradPart(self, steps, dtype):
     var0, var1, grads0, grads1 = self.initVariableAndGradient(dtype)
@@ -65,14 +65,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
     adagrad_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
     variables.global_variables_initializer().run()
     # Fetch params to validate initial values
-    self.assertAllClose([0.0, 0.0], var0.eval())
-    self.assertAllClose([0.0, 0.0], var1.eval())
+    self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+    self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
     # Run Adagrad for a few steps
     for _ in range(steps):
       adagrad_update.run()
 
-    return var0.eval(), var1.eval()
+    return self.evaluate(var0), self.evaluate(var1)
 
   def equivGradientDescentTest_FtrlPart(self, steps, dtype):
     var0, var1, grads0, grads1 = self.initVariableAndGradient(dtype)
@@ -85,14 +85,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
     ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
     variables.global_variables_initializer().run()
     # Fetch params to validate initial values
-    self.assertAllClose([0.0, 0.0], var0.eval())
-    self.assertAllClose([0.0, 0.0], var1.eval())
+    self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+    self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
     # Run Ftrl for a few steps
     for _ in range(steps):
       ftrl_update.run()
 
-    return var0.eval(), var1.eval()
+    return self.evaluate(var0), self.evaluate(var1)
 
   def equivGradientDescentTest_GradientDescentPart(self, steps, dtype):
     var0, var1, grads0, grads1 = self.initVariableAndGradient(dtype)
@@ -100,14 +100,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
     sgd_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
     variables.global_variables_initializer().run()
     # Fetch params to validate initial values
-    self.assertAllClose([0.0, 0.0], var0.eval())
-    self.assertAllClose([0.0, 0.0], var1.eval())
+    self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+    self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
     # Run GradientDescent for a few steps
     for _ in range(steps):
       sgd_update.run()
 
-    return var0.eval(), var1.eval()
+    return self.evaluate(var0), self.evaluate(var1)
 
   def testFtrlwithoutRegularization(self):
     for dtype in self.float_types:
@@ -124,8 +124,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([0.0, 0.0], var0.eval())
-        self.assertAllClose([0.0, 0.0], var1.eval())
+        self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+        self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
         # Run 3 steps FTRL
         for _ in range(3):
@@ -134,12 +134,12 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         # Validate updated params
         self.assertAllCloseAccordingToType(
             np.array([-2.60260963, -4.29698515]),
-            var0.eval(),
+            self.evaluate(var0),
             float_rtol=1e-4,
             half_rtol=1e-2)
         self.assertAllCloseAccordingToType(
             np.array([-0.28432083, -0.56694895]),
-            var1.eval(),
+            self.evaluate(var1),
             float_rtol=1e-5,
             half_rtol=1e-2)
 
@@ -158,8 +158,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([4.0, 3.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
         # Run 3 steps FTRL
         for _ in range(3):
@@ -167,10 +167,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
 
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([-2.55607247, -3.98729396]), var0.eval(), 1e-5, 1e-5,
+            np.array([-2.55607247, -3.98729396]),
+            self.evaluate(var0),
+            1e-5,
+            1e-5,
             float_rtol=1e-4)
         self.assertAllCloseAccordingToType(
-            np.array([-0.28232238, -0.56096673]), var1.eval(), 1e-5, 1e-5)
+            np.array([-0.28232238, -0.56096673]), self.evaluate(var1), 1e-5,
+            1e-5)
 
   def testFtrlWithL1(self):
     for dtype in self.float_types:
@@ -187,8 +191,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([4.0, 3.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
         # Run 10 steps FTRL
         for _ in range(10):
@@ -197,12 +201,14 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         # Validate updated params
         self.assertAllCloseAccordingToType(
             np.array([-7.66718769, -10.91273689]),
-            var0.eval(),
+            self.evaluate(var0),
             rtol=1e-4,
             bfloat16_rtol=1e-1,
             bfloat16_atol=1e-1)
         self.assertAllCloseAccordingToType(
-            np.array([-0.93460727, -1.86147261]), var1.eval(), rtol=1e-4)
+            np.array([-0.93460727, -1.86147261]),
+            self.evaluate(var1),
+            rtol=1e-4)
 
   def testFtrlWithL1_L2(self):
     for dtype in self.float_types:
@@ -219,8 +225,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([4.0, 3.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
         # Run 10 steps FTRL
         for _ in range(10):
@@ -228,9 +234,13 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
 
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([-0.24059935, -0.46829352]), var0.eval(), rtol=1e-5)
+            np.array([-0.24059935, -0.46829352]),
+            self.evaluate(var0),
+            rtol=1e-5)
         self.assertAllCloseAccordingToType(
-            np.array([-0.02406147, -0.04830509]), var1.eval(), rtol=1e-5)
+            np.array([-0.02406147, -0.04830509]),
+            self.evaluate(var1),
+            rtol=1e-5)
 
   def testFtrlWithL1_L2_L2Shrinkage(self):
     """Test the new FTRL op with support for l2 shrinkage.
@@ -254,8 +264,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([4.0, 3.0], self.evaluate(var1))
 
         # Run 10 steps FTRL
         for _ in range(10):
@@ -263,9 +273,13 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
 
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([-0.22578996, -0.44345799]), var0.eval(), rtol=1e-4)
+            np.array([-0.22578996, -0.44345799]),
+            self.evaluate(var0),
+            rtol=1e-4)
         self.assertAllCloseAccordingToType(
-            np.array([-0.14378493, -0.13229476]), var1.eval(), rtol=1e-4)
+            np.array([-0.14378493, -0.13229476]),
+            self.evaluate(var1),
+            rtol=1e-4)
 
   def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self):
     """Verifies that l2 shrinkage in FTRL does not change lr schedule."""
@@ -291,8 +305,8 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
         update1 = opt1.apply_gradients([(grads1, var1)])
         variables.global_variables_initializer().run()
 
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([1.0, 2.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var1))
 
         # Run 10 steps FTRL
         for _ in range(10):
@@ -301,7 +315,7 @@ class FtrlOptimizerTest(xla_test.XLATestCase):
 
         # var0 is experiencing L2 shrinkage so it should be smaller than var1
         # in magnitude.
-        self.assertTrue((var0.eval()**2 < var1.eval()**2).all())
+        self.assertTrue((var0.eval()**2 < self.evaluate(var1)**2).all())
         accum0 = list(opt0._slots["accum"].values())[0].eval()
         accum1 = list(opt1._slots["accum"].values())[0].eval()
         # L2 shrinkage should not change how we update grad accumulator.
diff --git a/tensorflow/compiler/tests/lrn_ops_test.py b/tensorflow/compiler/tests/lrn_ops_test.py
index c6ad67993e..5dddf6ae4e 100644
--- a/tensorflow/compiler/tests/lrn_ops_test.py
+++ b/tensorflow/compiler/tests/lrn_ops_test.py
@@ -120,8 +120,8 @@ class LRNTest(xla_test.XLATestCase):
       with self.test_scope():
         actual = gen_nn_ops.lrn_grad(out_grads, in_image, out_image,
                                      depth_radius, bias, alpha, beta)
-      expected_val = expected.eval()
-      actual_val = actual.eval()
+      expected_val = self.evaluate(expected)
+      actual_val = self.evaluate(actual)
     self.assertAllClose(actual_val, expected_val, rtol=1e-3)
 
 
diff --git a/tensorflow/compiler/tests/momentum_test.py b/tensorflow/compiler/tests/momentum_test.py
index f77521a7c4..3416f7dbd6 100644
--- a/tensorflow/compiler/tests/momentum_test.py
+++ b/tensorflow/compiler/tests/momentum_test.py
@@ -61,37 +61,43 @@ class MomentumOptimizerTest(xla_test.XLATestCase):
         self.assertFalse(slot1 in variables.trainable_variables())
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
-        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.1, 0.1]), self.evaluate(slot0))
+        self.assertAllCloseAccordingToType(
+            np.array([0.01, 0.01]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+            self.evaluate(var1))
         # Step 2: the momentum accumulators contain the previous update.
         mom_update.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
-                2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
-                    (0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval())
+                2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
+                3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
+            ]), self.evaluate(var1))
 
   def testNesterovMomentum(self):
     for dtype in self.float_types:
@@ -115,8 +121,8 @@ class MomentumOptimizerTest(xla_test.XLATestCase):
               var0_np, accum0_np, var0_np * 0.8, 0.1, 0.9)
           var1_np, accum1_np = self._update_nesterov_momentum_numpy(
               var1_np, accum1_np, 0.9, 0.1, 0.9)
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testTensorLearningRateAndMomentum(self):
     for dtype in self.float_types:
@@ -141,37 +147,43 @@ class MomentumOptimizerTest(xla_test.XLATestCase):
         self.assertFalse(slot1 in variables.trainable_variables())
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
-        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.1, 0.1]), self.evaluate(slot0))
+        self.assertAllCloseAccordingToType(
+            np.array([0.01, 0.01]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+            self.evaluate(var1))
         # Step 2: the momentum accumulators contain the previous update.
         mom_update.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
-                2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
-                    (0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval())
+                2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
+                3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
+            ]), self.evaluate(var1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tests/powersign_test.py b/tensorflow/compiler/tests/powersign_test.py
index 86536da7fe..5b35c20027 100644
--- a/tensorflow/compiler/tests/powersign_test.py
+++ b/tensorflow/compiler/tests/powersign_test.py
@@ -91,8 +91,8 @@ class PowerSignTest(xla_test.XLATestCase):
 
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 7 steps of powersign
         # first 4 steps with positive gradient
@@ -125,8 +125,8 @@ class PowerSignTest(xla_test.XLATestCase):
           )
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testDense(self):
     decay_steps = 10
diff --git a/tensorflow/compiler/tests/proximal_adagrad_test.py b/tensorflow/compiler/tests/proximal_adagrad_test.py
index c41b4171e2..63cc51a470 100644
--- a/tensorflow/compiler/tests/proximal_adagrad_test.py
+++ b/tensorflow/compiler/tests/proximal_adagrad_test.py
@@ -45,15 +45,17 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([0.0, 0.0], var0.eval())
-      self.assertAllClose([0.0, 0.0], var1.eval())
+      self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+      self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
       # Run 3 steps Proximal Adagrad.
       for _ in range(3):
         update.run()
 
-      self.assertAllClose(np.array([-2.60260963, -4.29698515]), var0.eval())
-      self.assertAllClose(np.array([-0.28432083, -0.56694895]), var1.eval())
+      self.assertAllClose(
+          np.array([-2.60260963, -4.29698515]), self.evaluate(var0))
+      self.assertAllClose(
+          np.array([-0.28432083, -0.56694895]), self.evaluate(var1))
       opt_vars = opt.variables()
       self.assertStartsWith(opt_vars[0].name, var0._shared_name)
       self.assertStartsWith(opt_vars[1].name, var1._shared_name)
@@ -74,14 +76,14 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([4.0, 3.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
       # Run 3 steps Proximal Adagrad.
       for _ in range(3):
         update.run()
-      self.assertAllClose(np.array([-1.60261, -2.296985]), var0.eval())
-      self.assertAllClose(np.array([3.715679, 2.433051]), var1.eval())
+      self.assertAllClose(np.array([-1.60261, -2.296985]), self.evaluate(var0))
+      self.assertAllClose(np.array([3.715679, 2.433051]), self.evaluate(var1))
 
   def testProximalAdagradWithL1(self):
     with self.cached_session(), self.test_scope():
@@ -98,14 +100,14 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([4.0, 3.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
       # Run 10 steps Proximal Adagrad
       for _ in range(10):
         update.run()
-      self.assertAllClose(np.array([-6.663634, -9.190331]), var0.eval())
-      self.assertAllClose(np.array([2.959304, 1.029232]), var1.eval())
+      self.assertAllClose(np.array([-6.663634, -9.190331]), self.evaluate(var0))
+      self.assertAllClose(np.array([2.959304, 1.029232]), self.evaluate(var1))
 
   def testProximalAdagradWithL1_L2(self):
     with self.cached_session(), self.test_scope():
@@ -122,15 +124,15 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([4.0, 3.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
       # Run 10 steps Proximal Adagrad.
       for _ in range(10):
         update.run()
 
-      self.assertAllClose(np.array([-0.0495, -0.0995]), var0.eval())
-      self.assertAllClose(np.array([-0.0045, -0.0095]), var1.eval())
+      self.assertAllClose(np.array([-0.0495, -0.0995]), self.evaluate(var0))
+      self.assertAllClose(np.array([-0.0045, -0.0095]), self.evaluate(var1))
 
   def applyOptimizer(self, opt, steps=5):
     var0 = resource_variable_ops.ResourceVariable([1.0, 2.0])
@@ -141,14 +143,14 @@ class ProximalAdagradOptimizerTest(xla_test.XLATestCase):
     update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
     variables.global_variables_initializer().run()
 
-    self.assertAllClose([1.0, 2.0], var0.eval())
-    self.assertAllClose([3.0, 4.0], var1.eval())
+    self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+    self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
     # Run ProximalAdagrad for a few steps
     for _ in range(steps):
       update.run()
 
-    return var0.eval(), var1.eval()
+    return self.evaluate(var0), self.evaluate(var1)
 
   def testEquivAdagradwithoutRegularization(self):
     with self.cached_session(), self.test_scope():
diff --git a/tensorflow/compiler/tests/proximal_gradient_descent_test.py b/tensorflow/compiler/tests/proximal_gradient_descent_test.py
index 3d808e6b8a..5aec433be7 100644
--- a/tensorflow/compiler/tests/proximal_gradient_descent_test.py
+++ b/tensorflow/compiler/tests/proximal_gradient_descent_test.py
@@ -42,15 +42,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([0.0, 0.0], var0.eval())
-      self.assertAllClose([0.0, 0.0], var1.eval())
+      self.assertAllClose([0.0, 0.0], self.evaluate(var0))
+      self.assertAllClose([0.0, 0.0], self.evaluate(var1))
 
       # Run 3 steps Proximal Gradient Descent.
       for _ in range(3):
         update.run()
 
-      self.assertAllClose(np.array([-0.9, -1.8]), var0.eval())
-      self.assertAllClose(np.array([-0.09, -0.18]), var1.eval())
+      self.assertAllClose(np.array([-0.9, -1.8]), self.evaluate(var0))
+      self.assertAllClose(np.array([-0.09, -0.18]), self.evaluate(var1))
 
   def testProximalGradientDescentwithoutRegularization2(self):
     with self.cached_session(), self.test_scope():
@@ -64,15 +64,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([4.0, 3.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
       # Run 3 steps Proximal Gradient Descent
       for _ in range(3):
         update.run()
 
-      self.assertAllClose(np.array([0.1, 0.2]), var0.eval())
-      self.assertAllClose(np.array([3.91, 2.82]), var1.eval())
+      self.assertAllClose(np.array([0.1, 0.2]), self.evaluate(var0))
+      self.assertAllClose(np.array([3.91, 2.82]), self.evaluate(var1))
 
   def testProximalGradientDescentWithL1(self):
     with self.cached_session(), self.test_scope():
@@ -86,15 +86,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([4.0, 3.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
       # Run 10 steps proximal gradient descent.
       for _ in range(10):
         update.run()
 
-      self.assertAllClose(np.array([-1.988, -3.988001]), var0.eval())
-      self.assertAllClose(np.array([3.67, 2.37]), var1.eval())
+      self.assertAllClose(np.array([-1.988, -3.988001]), self.evaluate(var0))
+      self.assertAllClose(np.array([3.67, 2.37]), self.evaluate(var1))
 
   def testProximalGradientDescentWithL1_L2(self):
     with self.cached_session(), self.test_scope():
@@ -108,15 +108,15 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([4.0, 3.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([4.0, 3.0], self.evaluate(var1))
 
       # Run 10 steps Proximal Gradient Descent
       for _ in range(10):
         update.run()
 
-      self.assertAllClose(np.array([-0.0495, -0.0995]), var0.eval())
-      self.assertAllClose(np.array([-0.0045, -0.0095]), var1.eval())
+      self.assertAllClose(np.array([-0.0495, -0.0995]), self.evaluate(var0))
+      self.assertAllClose(np.array([-0.0045, -0.0095]), self.evaluate(var1))
 
   def applyOptimizer(self, opt, steps=5):
     var0 = resource_variable_ops.ResourceVariable([1.0, 2.0])
@@ -127,14 +127,14 @@ class ProximalGradientDescentOptimizerTest(xla_test.XLATestCase):
     update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
     variables.global_variables_initializer().run()
 
-    self.assertAllClose([1.0, 2.0], var0.eval())
-    self.assertAllClose([3.0, 4.0], var1.eval())
+    self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+    self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
     # Run ProximalAdagrad for a few steps
     for _ in range(steps):
       update.run()
 
-    return var0.eval(), var1.eval()
+    return self.evaluate(var0), self.evaluate(var1)
 
   def testEquivGradientDescentwithoutRegularization(self):
     with self.cached_session(), self.test_scope():
diff --git a/tensorflow/compiler/tests/qr_op_test.py b/tensorflow/compiler/tests/qr_op_test.py
index 236b1b881d..b4d4193e35 100644
--- a/tensorflow/compiler/tests/qr_op_test.py
+++ b/tensorflow/compiler/tests/qr_op_test.py
@@ -63,7 +63,7 @@ class QrOpTest(xla_test.XLATestCase, parameterized.TestCase):
     # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity.
     xx = math_ops.matmul(x, x, adjoint_a=True)
     identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0)
-    precision = self.AdjustedNorm(xx.eval() - identity.eval())
+    precision = self.AdjustedNorm(xx.eval() - self.evaluate(identity))
     self.assertTrue(np.all(precision < 5.0))
 
   def _test(self, dtype, shape, full_matrices):
diff --git a/tensorflow/compiler/tests/rmsprop_test.py b/tensorflow/compiler/tests/rmsprop_test.py
index 8840a1329a..5138a4a2a9 100644
--- a/tensorflow/compiler/tests/rmsprop_test.py
+++ b/tensorflow/compiler/tests/rmsprop_test.py
@@ -92,8 +92,8 @@ class RmspropTest(xla_test.XLATestCase):
           self.assertTrue(mom1 is not None)
 
           # Fetch params to validate initial values
-          self.assertAllClose([1.0, 2.0], var0.eval())
-          self.assertAllClose([3.0, 4.0], var1.eval())
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
           # Run 3 steps of RMSProp
           for _ in range(3):
@@ -118,14 +118,14 @@ class RmspropTest(xla_test.XLATestCase):
 
             # Validate updated params
             if centered:
-              self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-              self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-            self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-            self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-            self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-            self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-            self.assertAllCloseAccordingToType(var0_np, var0.eval())
-            self.assertAllCloseAccordingToType(var1_np, var1.eval())
+              self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
+              self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+            self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+            self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+            self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+            self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+            self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+            self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tests/tensor_array_ops_test.py b/tensorflow/compiler/tests/tensor_array_ops_test.py
index 46ca371c8a..c8208adb58 100644
--- a/tensorflow/compiler/tests/tensor_array_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_array_ops_test.py
@@ -79,7 +79,8 @@ class TensorArrayTest(xla_test.XLATestCase):
       c0 = w2.stack()
 
       self.assertAllEqual(
-          convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0.eval())
+          convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]),
+          self.evaluate(c0))
 
   def testTensorArrayWritePack(self):
     for dtype in self.numeric_tf_types:
@@ -97,7 +98,7 @@ class TensorArrayTest(xla_test.XLATestCase):
 
       c0 = w2.stack()
 
-      self.assertAllEqual([3, 0, 1], c0.eval().shape)
+      self.assertAllEqual([3, 0, 1], self.evaluate(c0).shape)
 
   def _testTensorArrayWriteConcat(self, tf_dtype):
     with self.cached_session(), self.test_scope():
@@ -113,8 +114,8 @@ class TensorArrayTest(xla_test.XLATestCase):
       c0 = w2.concat()
 
       self.assertAllEqual(
-          convert([[4.0, 5.0], [104.0, 105.0], [6.0, 7.0],
-                   [106.0, 107.0], [8.0, 9.0], [204.0, 205.0]]), c0.eval())
+          convert([[4.0, 5.0], [104.0, 105.0], [6.0, 7.0], [106.0, 107.0],
+                   [8.0, 9.0], [204.0, 205.0]]), self.evaluate(c0))
 
   def testTensorArrayWriteConcat(self):
     for dtype in self.numeric_tf_types:
@@ -341,7 +342,7 @@ class TensorArrayTest(xla_test.XLATestCase):
         r0_bad = gen_data_flow_ops.tensor_array_read_v3(
             handle=w0.handle, index=0, dtype=dtype2, flow_in=w0.flow)
         with self.assertRaisesOpError("TensorArray dtype is "):
-          r0_bad.eval()
+          self.evaluate(r0_bad)
 
         # Test reading from a different index than the one we wrote to
         w0.read(1)
@@ -422,7 +423,7 @@ class TensorArrayTest(xla_test.XLATestCase):
       w2 = h2.write(0, 5.0)
       r2 = w2.read(0)
       r = r1 + r2
-      self.assertAllClose(9.0, r.eval())
+      self.assertAllClose(9.0, self.evaluate(r))
 
   def _testTensorArrayGradientWriteReadType(self, dtype):
     with self.cached_session() as session, self.test_scope():
@@ -526,7 +527,7 @@ class TensorArrayTest(xla_test.XLATestCase):
       with ops.control_dependencies([r0_readtwice]):
         r1_readtwice = w_readtwice.read(0)
 
-      self.assertAllEqual([1.0, -1.0], r1_readtwice.eval())
+      self.assertAllEqual([1.0, -1.0], self.evaluate(r1_readtwice))
 
   def _testTensorArrayGradientUnpackRead(self):
     with self.cached_session() as session, self.test_scope():
@@ -592,7 +593,7 @@ class TensorArrayTest(xla_test.XLATestCase):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
       s = ta.size()
-      self.assertAllEqual(3, s.eval())
+      self.assertAllEqual(3, self.evaluate(s))
 
   def testWriteCloseTensorArray(self):
     with self.cached_session(), self.test_scope():
@@ -722,7 +723,7 @@ class TensorArrayTest(xla_test.XLATestCase):
 
   #     r = acc2.stack()
   #     grad = gradients_impl.gradients(r, [x])[0]
-  #     self.assertAllClose(31.0, grad.eval())
+  #     self.assertAllClose(31.0, self.evaluate(grad))
 
   def testSumOfTwoReadVariablesWithoutRepeatGrad(self):
     with self.cached_session() as session, self.test_scope():
@@ -912,7 +913,7 @@ class TensorArrayTest(xla_test.XLATestCase):
       self.assertEqual(0, ta.size().eval())
       ta = ta.unstack(array_ops.zeros([0, 3, 5]))
       packed = ta.stack()
-      self.assertAllEqual([0, 3, 5], packed.eval().shape)
+      self.assertAllEqual([0, 3, 5], self.evaluate(packed).shape)
       # Concatenating zero tensors along their first dimension gives a
       # first dimension of zero
       self.assertAllEqual([0, 5], ta.concat().eval().shape)
@@ -1041,8 +1042,8 @@ class TensorArrayTest(xla_test.XLATestCase):
           (read0, read1, size0, size1))
 
       # Tests that the control dependencies was added and executed.
-      self.assertEqual(1, v0.eval())
-      self.assertEqual(1, v1.eval())
+      self.assertEqual(1, self.evaluate(v0))
+      self.assertEqual(1, self.evaluate(v1))
 
       # Tests correct TensorArray.
       self.assertEqual(read0_v, 0)
diff --git a/tensorflow/examples/adding_an_op/zero_out_3_test.py b/tensorflow/examples/adding_an_op/zero_out_3_test.py
index 15d62495aa..2327e7cd8f 100644
--- a/tensorflow/examples/adding_an_op/zero_out_3_test.py
+++ b/tensorflow/examples/adding_an_op/zero_out_3_test.py
@@ -39,13 +39,13 @@ class ZeroOut3Test(tf.test.TestCase):
     with self.cached_session():
       result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=-1)
       with self.assertRaisesOpError("Need preserve_index >= 0, got -1"):
-        result.eval()
+        self.evaluate(result)
 
   def testLarge(self):
     with self.cached_session():
       result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=17)
       with self.assertRaisesOpError("preserve_index out of range"):
-        result.eval()
+        self.evaluate(result)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py b/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py
index 020d40bc13..561f5f7a50 100644
--- a/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py
+++ b/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py
@@ -110,7 +110,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase):
           left_context=1,
           right_context=1)
       self.assertAllEqual(
-          filterbanks.eval(),
+          self.evaluate(filterbanks),
           [[479, 425, 479, 425, 436, 378], [479, 425, 436, 378, 410, 350],
            [436, 378, 410, 350, 391, 325], [410, 350, 391, 325, 391, 325]])
 
@@ -153,7 +153,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase):
           frame_stride=3,
           zero_padding=True)
       self.assertAllEqual(
-          filterbanks.eval(),
+          self.evaluate(filterbanks),
           [[0, 0, 0, 0, 479, 425], [436, 378, 410, 350, 391, 325],
            [374, 308, 362, 292, 352, 275]])
 
diff --git a/tensorflow/python/data/util/sparse_test.py b/tensorflow/python/data/util/sparse_test.py
index 056b32480f..4ba314f06a 100644
--- a/tensorflow/python/data/util/sparse_test.py
+++ b/tensorflow/python/data/util/sparse_test.py
@@ -292,9 +292,9 @@ class SparseTest(test.TestCase):
       return
     self.assertTrue(isinstance(b, sparse_tensor.SparseTensor))
     with self.cached_session():
-      self.assertAllEqual(a.eval().indices, b.eval().indices)
-      self.assertAllEqual(a.eval().values, b.eval().values)
-      self.assertAllEqual(a.eval().dense_shape, b.eval().dense_shape)
+      self.assertAllEqual(a.eval().indices, self.evaluate(b).indices)
+      self.assertAllEqual(a.eval().values, self.evaluate(b).values)
+      self.assertAllEqual(a.eval().dense_shape, self.evaluate(b).dense_shape)
 
   def testSerializeDeserialize(self):
     test_cases = (
diff --git a/tensorflow/python/debug/wrappers/framework_test.py b/tensorflow/python/debug/wrappers/framework_test.py
index 73e08ce7d5..68584b4ede 100644
--- a/tensorflow/python/debug/wrappers/framework_test.py
+++ b/tensorflow/python/debug/wrappers/framework_test.py
@@ -339,7 +339,7 @@ class DebugWrapperSessionTest(test_util.TensorFlowTestCase):
 
     with wrapper.as_default():
       foo = constant_op.constant(42, name="foo")
-      self.assertEqual(42, foo.eval())
+      self.assertEqual(42, self.evaluate(foo))
       self.assertEqual(foo, self._observer["run_fetches"])
 
   def testWrapperShouldSupportSessionClose(self):
diff --git a/tensorflow/python/distribute/all_reduce_test.py b/tensorflow/python/distribute/all_reduce_test.py
index b1956c75b7..5bf983a1b2 100644
--- a/tensorflow/python/distribute/all_reduce_test.py
+++ b/tensorflow/python/distribute/all_reduce_test.py
@@ -159,7 +159,7 @@ class AllReduceTest(test_util.TensorFlowTestCase):
       output_tensors = build_f(input_tensors, un_op)
       sum_reduced = math_ops.add_n(output_tensors)
       sum_reduced.op.run()
-      self.assertAllClose(sum_reduced.eval(), simple_sum.eval())
+      self.assertAllClose(sum_reduced.eval(), self.evaluate(simple_sum))
 
   def _testRingAllReduce(self, num_workers, num_gpus, shape, subdiv):
     start_time = time.time()
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 274d5320df..d3458741bd 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -215,7 +215,7 @@ class BackpropTest(test.TestCase):
       self.assertAllClose(tf_grad.values.eval(), grad.values)
 
       tf_opt.apply_gradients([(tf_grad, tf_embedding)]).run()
-      expected = tf_embedding.eval()
+      expected = self.evaluate(tf_embedding)
     opt.apply_gradients([(grad, embedding)])
     self.assertAllClose(expected, embedding.read_value())
 
diff --git a/tensorflow/python/eager/graph_only_ops_test.py b/tensorflow/python/eager/graph_only_ops_test.py
index 3cf3a61a62..3aedf5fee1 100644
--- a/tensorflow/python/eager/graph_only_ops_test.py
+++ b/tensorflow/python/eager/graph_only_ops_test.py
@@ -33,7 +33,7 @@ class GraphOnlyOpsTest(test_util.TensorFlowTestCase):
     x = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32)
     z_tf = graph_only_ops.graph_zeros_like(x)
     with self.cached_session():
-      self.assertAllClose(np.zeros((2, 3)), z_tf.eval())
+      self.assertAllClose(np.zeros((2, 3)), self.evaluate(z_tf))
 
   def testGraphPlaceholder(self):
     x_tf = graph_only_ops.graph_placeholder(dtypes.int32, shape=(1,))
diff --git a/tensorflow/python/eager/tape_test.py b/tensorflow/python/eager/tape_test.py
index acd0e569f1..48d3b8ac6e 100644
--- a/tensorflow/python/eager/tape_test.py
+++ b/tensorflow/python/eager/tape_test.py
@@ -80,8 +80,8 @@ class TapeTest(test.TestCase):
       tf_e = tf_d + tf_f
       tf_da, tf_db = gradients_impl.gradients(tf_e, [tf_a, tf_b])
 
-      self.assertAllEqual(da, tf_da.eval())
-      self.assertAllEqual(db, tf_db.eval())
+      self.assertAllEqual(da, self.evaluate(tf_da))
+      self.assertAllEqual(db, self.evaluate(tf_db))
 
   def testBasicFunctional(self):
 
@@ -142,8 +142,8 @@ class TapeTest(test.TestCase):
       tf_rr = 2 * math_ops.reduce_sum(tf_mm)
       tf_da, tf_db = gradients_impl.gradients(tf_rr, [tf_a, tf_b])
 
-      self.assertAllEqual(da, tf_da.eval())
-      self.assertAllEqual(db, tf_db.eval())
+      self.assertAllEqual(da, self.evaluate(tf_da))
+      self.assertAllEqual(db, self.evaluate(tf_db))
 
   def testGcTwoOutputs(self):
 
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 7f4b1902e1..73df7d9eb8 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -352,11 +352,11 @@ class NumericColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[10.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     price = fc.numeric_column('price')
@@ -366,11 +366,11 @@ class NumericColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[10.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
 
 class BucketizedColumnTest(test.TestCase):
@@ -469,11 +469,9 @@ class BucketizedColumnTest(test.TestCase):
         bucketized_price_tensor = bucketized_price._get_dense_tensor(builder)
         self.assertAllClose(
             # One-hot tensor.
-            [[[1., 0., 0., 0., 0.]],
-             [[0., 1., 0., 0., 0.]],
-             [[0., 0., 0., 1., 0.]],
-             [[0., 0., 0., 0., 1.]]],
-            bucketized_price_tensor.eval())
+            [[[1., 0., 0., 0., 0.]], [[0., 1., 0., 0., 0.]],
+             [[0., 0., 0., 1., 0.]], [[0., 0., 0., 0., 1.]]],
+            self.evaluate(bucketized_price_tensor))
 
   def test_get_dense_tensor_two_input_values(self):
     """Tests _get_dense_tensor() for input with shape=[2]."""
@@ -487,7 +485,7 @@ class BucketizedColumnTest(test.TestCase):
             # One-hot tensor.
             [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]],
              [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]],
-            bucketized_price_tensor.eval())
+            self.evaluate(bucketized_price_tensor))
 
   def test_get_sparse_tensors_one_input_value(self):
     """Tests _get_sparse_tensors() for input with shape=[1]."""
@@ -550,20 +548,23 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight variable per bucket, all initialized to zero.
-        self.assertAllClose(
-            [[0.], [0.], [0.], [0.], [0.]], bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.], [0.]],
+                            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(predictions))
         sess.run(bucketized_price_var.assign(
             [[10.], [20.], [30.], [40.], [50.]]))
         # price -1. is in the 0th bucket, whose weight is 10.
         # price 1. is in the 1st bucket, whose weight is 20.
         # price 5. is in the 3rd bucket, whose weight is 40.
         # price 6. is in the 4th bucket, whose weight is 50.
-        self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [20.], [40.], [50.]],
+                            self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
+        self.assertAllClose([[11.], [21.], [41.], [51.]],
+                            self.evaluate(predictions))
 
   def test_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
@@ -575,12 +576,12 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight per bucket per input column, all initialized to zero.
         self.assertAllClose(
             [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]],
-            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(bucketized_price_var.assign(
             [[10.], [20.], [30.], [40.], [50.],
              [60.], [70.], [80.], [90.], [100.]]))
@@ -590,9 +591,9 @@ class BucketizedColumnTest(test.TestCase):
         # 2nd example:
         #   price 5. is in the 3rd bucket, whose weight is 40.
         #   price 6. is in the 9th bucket, whose weight is 100.
-        self.assertAllClose([[80.], [140.]], predictions.eval())
+        self.assertAllClose([[80.], [140.]], self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[81.], [141.]], predictions.eval())
+        self.assertAllClose([[81.], [141.]], self.evaluate(predictions))
 
   def test_keras_linear_model_one_input_value(self):
     """Tests _LinearModel for input with shape=[1]."""
@@ -605,20 +606,23 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight variable per bucket, all initialized to zero.
         self.assertAllClose([[0.], [0.], [0.], [0.], [0.]],
-                            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval())
+                            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(predictions))
         sess.run(
             bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]]))
         # price -1. is in the 0th bucket, whose weight is 10.
         # price 1. is in the 1st bucket, whose weight is 20.
         # price 5. is in the 3rd bucket, whose weight is 40.
         # price 6. is in the 4th bucket, whose weight is 50.
-        self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [20.], [40.], [50.]],
+                            self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
+        self.assertAllClose([[11.], [21.], [41.], [51.]],
+                            self.evaluate(predictions))
 
   def test_keras_linear_model_two_input_values(self):
     """Tests _LinearModel for input with shape=[2]."""
@@ -631,12 +635,12 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight per bucket per input column, all initialized to zero.
         self.assertAllClose(
             [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]],
-            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(
             bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.],
                                          [60.], [70.], [80.], [90.], [100.]]))
@@ -646,9 +650,9 @@ class BucketizedColumnTest(test.TestCase):
         # 2nd example:
         #   price 5. is in the 3rd bucket, whose weight is 40.
         #   price 6. is in the 9th bucket, whose weight is 100.
-        self.assertAllClose([[80.], [140.]], predictions.eval())
+        self.assertAllClose([[80.], [140.]], self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[81.], [141.]], predictions.eval())
+        self.assertAllClose([[81.], [141.]], self.evaluate(predictions))
 
 
 class HashedCategoricalColumnTest(test.TestCase):
@@ -852,13 +856,14 @@ class HashedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 3: wire_var[3] = 4
         # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
-        self.assertAllClose(((4.,), (6.,)), predictions.eval())
+        self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
@@ -874,13 +879,14 @@ class HashedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 3: wire_var[3] = 4
         # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
-        self.assertAllClose(((4.,), (6.,)), predictions.eval())
+        self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
 
 class CrossedColumnTest(test.TestCase):
@@ -1113,15 +1119,15 @@ class CrossedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       crossed_var = get_linear_model_column_var(crossed)
       with _initialized_session() as sess:
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(
-            ((0.,), (0.,), (0.,), (0.,), (0.,)), crossed_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(crossed_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,))))
         # Expected ids after cross = (1, 0, 1, 3, 4, 2)
-        self.assertAllClose(((3.,), (14.,)), predictions.eval())
+        self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions))
         sess.run(bias.assign((.1,)))
-        self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
+        self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
   def test_linear_model_with_weights(self):
     class _TestColumnWithWeights(_CategoricalColumn):
@@ -1196,15 +1202,15 @@ class CrossedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       crossed_var = get_linear_model_column_var(crossed)
       with _initialized_session() as sess:
-        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
         self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)),
-                            crossed_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+                            self.evaluate(crossed_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,))))
         # Expected ids after cross = (1, 0, 1, 3, 4, 2)
-        self.assertAllClose(((3.,), (14.,)), predictions.eval())
+        self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions))
         sess.run(bias.assign((.1,)))
-        self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
+        self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
   def test_keras_linear_model_with_weights(self):
 
@@ -1349,10 +1355,10 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         sess.run(price_var.assign([[10.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -1366,11 +1372,12 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(wire_cast_var))
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -1389,7 +1396,7 @@ class LinearModelTest(test.TestCase):
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[1015.], [10065.]], predictions.eval())
+        self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_column(self):
     """When the column is both dense and sparse, uses sparse tensors."""
@@ -1442,7 +1449,7 @@ class LinearModelTest(test.TestCase):
         sess.run(dense_and_sparse_column_var.assign(
             [[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
     price = fc.numeric_column('price')
@@ -1452,12 +1459,12 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((1, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[10., 100., 1000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -1471,15 +1478,15 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var))
         sess.run(
             wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [
                 1000., 1100., 1200.
             ], [10000., 11000., 12000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
     price = fc.numeric_column('price', shape=2)
@@ -1488,9 +1495,9 @@ class LinearModelTest(test.TestCase):
       predictions = fc.linear_model(features, [price])
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([[0.], [0.]], price_var.eval())
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -1504,7 +1511,7 @@ class LinearModelTest(test.TestCase):
       predictions = fc.linear_model(features, [wire_cast])
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var))
         self.assertAllClose(
             np.zeros((2, 1)),
             predictions.eval(feed_dict={wire_tensor: wire_value}))
@@ -1528,7 +1535,7 @@ class LinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [5010.]], predictions.eval())
+        self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_sparse_combiner_with_negative_weights(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -1550,7 +1557,7 @@ class LinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [-9985.]], predictions.eval())
+        self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
     price = fc.numeric_column('price', shape=2)
@@ -1560,12 +1567,12 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((2, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]]))
         sess.run(bias.assign([2., 3., 4.]))
         self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column('price', shape=2)
@@ -1584,11 +1591,11 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -1603,14 +1610,14 @@ class LinearModelTest(test.TestCase):
       price1_var = get_linear_model_column_var(price1)
       price2_var = get_linear_model_column_var(price2)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price1_var.eval())
-        self.assertAllClose([[0.]], price2_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price1_var))
+        self.assertAllClose([[0.]], self.evaluate(price2_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price1_var.assign([[10.], [100.]]))
         sess.run(price2_var.assign([[1000.]]))
         sess.run(bias.assign([7.]))
-        self.assertAllClose([[3217.], [4657.]], predictions.eval())
+        self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_fills_cols_to_vars(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -1950,14 +1957,14 @@ class LinearModelTest(test.TestCase):
       price_var1 = get_linear_model_column_var(price, name='linear_model')
       price_var2 = get_linear_model_column_var(price, name='linear_model_1')
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias1.eval())
+        self.assertAllClose([0.], self.evaluate(bias1))
         sess.run(price_var1.assign([[10.]]))
         sess.run(bias1.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions1.eval())
-        self.assertAllClose([0.], bias2.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions1))
+        self.assertAllClose([0.], self.evaluate(bias2))
         sess.run(price_var2.assign([[10.]]))
         sess.run(bias2.assign([5.]))
-        self.assertAllClose([[25.], [105.]], predictions2.eval())
+        self.assertAllClose([[25.], [105.]], self.evaluate(predictions2))
 
 
 class _LinearModelTest(test.TestCase):
@@ -2014,10 +2021,10 @@ class _LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         sess.run(price_var.assign([[10.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -2031,11 +2038,12 @@ class _LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(wire_cast_var))
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -2055,7 +2063,7 @@ class _LinearModelTest(test.TestCase):
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[1015.], [10065.]], predictions.eval())
+        self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_column(self):
     """When the column is both dense and sparse, uses sparse tensors."""
@@ -2114,7 +2122,7 @@ class _LinearModelTest(test.TestCase):
             dense_and_sparse_column_var.assign([[10.], [100.], [1000.],
                                                 [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
     price = fc.numeric_column('price')
@@ -2125,12 +2133,12 @@ class _LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((1, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[10., 100., 1000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -2145,15 +2153,15 @@ class _LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var))
         sess.run(
             wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.],
                                   [1000., 1100.,
                                    1200.], [10000., 11000., 12000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
     price = fc.numeric_column('price', shape=2)
@@ -2162,9 +2170,9 @@ class _LinearModelTest(test.TestCase):
       predictions = get_keras_linear_model_predictions(features, [price])
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([[0.], [0.]], price_var.eval())
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
     wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
@@ -2178,7 +2186,7 @@ class _LinearModelTest(test.TestCase):
       predictions = get_keras_linear_model_predictions(features, [wire_cast])
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var))
         self.assertAllClose(
             np.zeros((2, 1)),
             predictions.eval(feed_dict={wire_tensor: wire_value}))
@@ -2202,7 +2210,7 @@ class _LinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [5010.]], predictions.eval())
+        self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
     price = fc.numeric_column('price', shape=2)
@@ -2213,12 +2221,12 @@ class _LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((2, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]]))
         sess.run(bias.assign([2., 3., 4.]))
         self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column('price', shape=2)
@@ -2237,11 +2245,11 @@ class _LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -2254,14 +2262,14 @@ class _LinearModelTest(test.TestCase):
       price1_var = get_linear_model_column_var(price1)
       price2_var = get_linear_model_column_var(price2)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price1_var.eval())
-        self.assertAllClose([[0.]], price2_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price1_var))
+        self.assertAllClose([[0.]], self.evaluate(price2_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price1_var.assign([[10.], [100.]]))
         sess.run(price2_var.assign([[1000.]]))
         sess.run(bias.assign([7.]))
-        self.assertAllClose([[3217.], [4657.]], predictions.eval())
+        self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_fills_cols_to_vars(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -2701,7 +2709,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = features = {'a': [0.]}
       net = fc.input_layer(features, fc.numeric_column('a'))
       with _initialized_session():
-        self.assertAllClose([[0.]], net.eval())
+        self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
@@ -2709,7 +2717,7 @@ class FunctionalInputLayerTest(test.TestCase):
       columns = (fc.numeric_column(key) for key in features)
       net = fc.input_layer(features, columns)
       with _initialized_session():
-        self.assertAllClose([[0., 1.]], net.eval())
+        self.assertAllClose([[0., 1.]], self.evaluate(net))
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
@@ -2725,7 +2733,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price': [[1.], [5.]]}
       net = fc.input_layer(features, [price])
       with _initialized_session():
-        self.assertAllClose([[1.], [5.]], net.eval())
+        self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
     price = fc.numeric_column('price', shape=2)
@@ -2733,7 +2741,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc.input_layer(features, [price])
       with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
+        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column('price', shape=2)
@@ -2750,7 +2758,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc.input_layer(features, [price])
       with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
+        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -2762,7 +2770,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       net = fc.input_layer(features, [price1, price2])
       with _initialized_session():
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
+        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_fills_cols_to_vars(self):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
@@ -2893,8 +2901,8 @@ class FunctionalInputLayerTest(test.TestCase):
       net1 = fc.input_layer(features, [price_a, price_b])
       net2 = fc.input_layer(features, [price_b, price_a])
       with _initialized_session():
-        self.assertAllClose([[1., 3.]], net1.eval())
-        self.assertAllClose([[1., 3.]], net2.eval())
+        self.assertAllClose([[1., 3.]], self.evaluate(net1))
+        self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
     animal = fc.categorical_column_with_identity('animal', num_buckets=4)
@@ -3514,13 +3522,12 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         dense_shape=(2, 2))
     id_tensor = _transform_features({'aaa': inputs}, [column])[column]
     with _initialized_session():
-      _assert_sparse_tensor_value(self,
-                                  sparse_tensor.SparseTensorValue(
-                                      indices=inputs.indices,
-                                      values=np.array(
-                                          (2, -1, 0), dtype=np.int64),
-                                      dense_shape=inputs.dense_shape),
-                                  id_tensor.eval())
+      _assert_sparse_tensor_value(
+          self,
+          sparse_tensor.SparseTensorValue(
+              indices=inputs.indices,
+              values=np.array((2, -1, 0), dtype=np.int64),
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_weight_collections(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -3706,13 +3713,14 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_file(
@@ -3732,13 +3740,14 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
 
 class VocabularyListCategoricalColumnTest(test.TestCase):
@@ -3946,8 +3955,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_weight_collections(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -4104,13 +4112,14 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_list(
@@ -4129,13 +4138,14 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
 
 class IdentityCategoricalColumnTest(test.TestCase):
@@ -4241,8 +4251,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_weight_collections(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
@@ -4357,13 +4366,13 @@ class IdentityCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] = 1
         # weight_var[2] + weight_var[1] = 3+2 = 5
-        self.assertAllClose(((1.,), (5.,)), predictions.eval())
+        self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
@@ -4379,13 +4388,13 @@ class IdentityCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] = 1
         # weight_var[2] + weight_var[1] = 3+2 = 5
-        self.assertAllClose(((1.,), (5.,)), predictions.eval())
+        self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
 
 class TransformFeaturesTest(test.TestCase):
@@ -4472,7 +4481,8 @@ class IndicatorColumnTest(test.TestCase):
     builder = _LazyBuilder({'animal': ['fox', 'fox']})
     output = builder.get(animal)
     with self.cached_session():
-      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval())
+      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
+                          self.evaluate(output))
 
   def test_2D_shape_succeeds(self):
     # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready.
@@ -4487,7 +4497,8 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = builder.get(animal)
     with self.cached_session():
-      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval())
+      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
+                          self.evaluate(output))
 
   def test_multi_hot(self):
     animal = fc.indicator_column(
@@ -4500,7 +4511,7 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = builder.get(animal)
     with self.cached_session():
-      self.assertAllEqual([[0., 2., 0., 0.]], output.eval())
+      self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output))
 
   def test_multi_hot2(self):
     animal = fc.indicator_column(
@@ -4512,7 +4523,7 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = builder.get(animal)
     with self.cached_session():
-      self.assertAllEqual([[0., 1., 1., 0.]], output.eval())
+      self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
   def test_deep_copy(self):
     a = fc.categorical_column_with_hash_bucket('a', 4)
@@ -4557,7 +4568,8 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = _transform_features(features, [a_indicator])[a_indicator]
     with _initialized_session():
-      self.assertAllEqual([[0, 0, 1], [1, 0, 0]], indicator_tensor.eval())
+      self.assertAllEqual([[0, 0, 1], [1, 0, 0]],
+                          self.evaluate(indicator_tensor))
 
   def test_transform_with_weighted_column(self):
     # Github issue 12557
@@ -4571,7 +4583,7 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = _transform_features(features, [indicator])[indicator]
     with _initialized_session():
-      self.assertAllEqual([[6., 4., 3.]], indicator_tensor.eval())
+      self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
@@ -4585,7 +4597,7 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = _transform_features(features, [indicator])[indicator]
     with _initialized_session():
-      self.assertAllEqual([[0., 4., 2.]], indicator_tensor.eval())
+      self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
@@ -4597,7 +4609,7 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = _transform_features(features, [indicator])[indicator]
     with _initialized_session():
-      self.assertAllEqual([[0., 1., 1.]], indicator_tensor.eval())
+      self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
   def test_linear_model(self):
     animal = fc.indicator_column(
@@ -4613,10 +4625,10 @@ class IndicatorColumnTest(test.TestCase):
       weight_var = get_linear_model_column_var(animal)
       with _initialized_session():
         # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval())
-        self.assertAllClose([[0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+        self.assertAllClose([[0.]], self.evaluate(predictions))
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], predictions.eval())
+        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     animal = fc.indicator_column(
@@ -4632,10 +4644,10 @@ class IndicatorColumnTest(test.TestCase):
       weight_var = get_linear_model_column_var(animal)
       with _initialized_session():
         # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval())
-        self.assertAllClose([[0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+        self.assertAllClose([[0.]], self.evaluate(predictions))
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], predictions.eval())
+        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_input_layer(self):
     animal = fc.indicator_column(
@@ -4648,7 +4660,7 @@ class IndicatorColumnTest(test.TestCase):
       }
       net = fc.input_layer(features, [animal])
       with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], net.eval())
+        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
 
 class EmbeddingColumnTest(test.TestCase):
@@ -4769,8 +4781,8 @@ class EmbeddingColumnTest(test.TestCase):
     output_a = outputs[a]
     output_embedded = outputs[a_embedded]
     with _initialized_session():
-      _assert_sparse_tensor_value(
-          self, output_a.eval(), output_embedded.eval())
+      _assert_sparse_tensor_value(self, self.evaluate(output_a),
+                                  self.evaluate(output_embedded))
 
   def test_get_dense_tensor(self):
     # Inputs.
@@ -4828,7 +4840,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_3d(self):
     # Inputs.
@@ -4888,7 +4900,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_weight_collections(self):
     sparse_input = sparse_tensor.SparseTensorValue(
@@ -5044,7 +5056,7 @@ class EmbeddingColumnTest(test.TestCase):
         ('embedding_weights:0',), tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_linear_model(self):
     # Inputs.
@@ -5100,11 +5112,13 @@ class EmbeddingColumnTest(test.TestCase):
           'linear_model/aaa_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
+        self.assertAllClose(
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -5119,7 +5133,8 @@ class EmbeddingColumnTest(test.TestCase):
         # example 3, ids [1], embedding[3] = [3, 5]
         # sum(embeddings * linear_weights)
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
+        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                            self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     # Inputs.
@@ -5176,11 +5191,13 @@ class EmbeddingColumnTest(test.TestCase):
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -5195,7 +5212,8 @@ class EmbeddingColumnTest(test.TestCase):
         # example 3, ids [1], embedding[3] = [3, 5]
         # sum(embeddings * linear_weights)
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
+        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                            self.evaluate(predictions))
 
   def test_input_layer(self):
     # Inputs.
@@ -5255,7 +5273,7 @@ class EmbeddingColumnTest(test.TestCase):
         tuple([v.name for v in trainable_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, input_layer.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(input_layer))
 
   def test_input_layer_not_trainable(self):
     # Inputs.
@@ -5313,7 +5331,7 @@ class EmbeddingColumnTest(test.TestCase):
         [], ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, input_layer.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(input_layer))
 
 
 class SharedEmbeddingColumnTest(test.TestCase):
@@ -5550,10 +5568,10 @@ class SharedEmbeddingColumnTest(test.TestCase):
     output_b = outputs[b]
     output_b_embedded = outputs[b_embedded]
     with _initialized_session():
-      _assert_sparse_tensor_value(
-          self, output_a.eval(), output_a_embedded.eval())
-      _assert_sparse_tensor_value(
-          self, output_b.eval(), output_b_embedded.eval())
+      _assert_sparse_tensor_value(self, self.evaluate(output_a),
+                                  self.evaluate(output_a_embedded))
+      _assert_sparse_tensor_value(self, self.evaluate(output_b),
+                                  self.evaluate(output_b_embedded))
 
   def test_get_dense_tensor(self):
     # Inputs.
@@ -5618,9 +5636,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     embedding_var = global_vars[0]
     with _initialized_session():
-      self.assertAllEqual(embedding_values, embedding_var.eval())
-      self.assertAllEqual(expected_lookups_a, embedding_lookup_a.eval())
-      self.assertAllEqual(expected_lookups_b, embedding_lookup_b.eval())
+      self.assertAllEqual(embedding_values, self.evaluate(embedding_var))
+      self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a))
+      self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b))
 
   def test_get_dense_tensor_weight_collections(self):
     # Inputs.
@@ -5790,13 +5808,15 @@ class SharedEmbeddingColumnTest(test.TestCase):
           'linear_model/aaa_bbb_shared_embedding_1/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights_a.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights_b.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -5814,7 +5834,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         # example 1, ids [], embedding[1] = 0, 0]
         # sum(embeddings * linear_weights)
         # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
-        self.assertAllClose([[94. + 13.], [29.]], predictions.eval())
+        self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions))
 
   def test_keras_linear_model(self):
     # Inputs.
@@ -5881,13 +5901,15 @@ class SharedEmbeddingColumnTest(test.TestCase):
           'linear_model/aaa_bbb_shared_embedding_1/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights_a.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights_b.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b))
+        self.assertAllClose(
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -5905,7 +5927,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         # example 1, ids [], embedding[1] = 0, 0]
         # sum(embeddings * linear_weights)
         # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
-        self.assertAllClose([[94. + 13.], [29.]], predictions.eval())
+        self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions))
 
   def _test_input_layer(self, trainable=True):
     # Inputs.
@@ -5978,7 +6000,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     shared_embedding_vars = global_vars
     with _initialized_session():
       self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval())
-      self.assertAllEqual(expected_lookups, input_layer.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(input_layer))
 
   def test_input_layer(self):
     self._test_input_layer()
@@ -6121,15 +6143,13 @@ class WeightedCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array(inputs.values, dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
       _assert_sparse_tensor_value(
           self,
           sparse_tensor.SparseTensorValue(
               indices=weights.indices,
               values=np.array(weights.values, dtype=np.float32),
-              dense_shape=weights.dense_shape),
-          weight_tensor.eval())
+              dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_input(self):
     column = fc.weighted_categorical_column(
@@ -6150,15 +6170,13 @@ class WeightedCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=((0, 0), (1, 0), (1, 1)),
               values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=(2, 2)),
-          id_tensor.eval())
+              dense_shape=(2, 2)), self.evaluate(id_tensor))
       _assert_sparse_tensor_value(
           self,
           sparse_tensor.SparseTensorValue(
               indices=weights.indices,
               values=np.array(weights.values, dtype=np.float32),
-              dense_shape=weights.dense_shape),
-          weight_tensor.eval())
+              dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_weights(self):
     column = fc.weighted_categorical_column(
@@ -6179,15 +6197,13 @@ class WeightedCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array(inputs.values, dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
       _assert_sparse_tensor_value(
           self,
           sparse_tensor.SparseTensorValue(
               indices=((0, 0), (1, 0), (1, 1)),
               values=np.array((.5, 1., .1), dtype=np.float32),
-              dense_shape=(2, 2)),
-          weight_tensor.eval())
+              dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
   def test_keras_linear_model(self):
     column = fc.weighted_categorical_column(
@@ -6210,14 +6226,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_keras_linear_model_mismatched_shape(self):
     column = fc.weighted_categorical_column(
@@ -6263,7 +6279,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
           rewriter_config_pb2.RewriterConfig.OFF)
       with _initialized_session(config):
         with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'):
-          predictions.eval()
+          self.evaluate(predictions)
 
   def test_keras_linear_model_mismatched_dense_shape(self):
     column = fc.weighted_categorical_column(
@@ -6282,14 +6298,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model(self):
     column = fc.weighted_categorical_column(
@@ -6310,14 +6326,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model_mismatched_shape(self):
     column = fc.weighted_categorical_column(
@@ -6361,7 +6377,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
           rewriter_config_pb2.RewriterConfig.OFF)
       with _initialized_session(config):
         with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'):
-          predictions.eval()
+          self.evaluate(predictions)
 
   def test_linear_model_mismatched_dense_shape(self):
     column = fc.weighted_categorical_column(
@@ -6379,14 +6395,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   # TODO(ptucker): Add test with embedding of weighted categorical.
 
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 2a133e2892..78321ee096 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -413,11 +413,11 @@ class NumericColumnTest(test.TestCase):
       predictions = model(features)
       price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[10.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
   def test_old_linear_model(self):
     price = fc.numeric_column_v2('price')
@@ -427,11 +427,11 @@ class NumericColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[10.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
   def test_serialization(self):
 
@@ -557,11 +557,9 @@ class BucketizedColumnTest(test.TestCase):
             transformation_cache, None)
         self.assertAllClose(
             # One-hot tensor.
-            [[[1., 0., 0., 0., 0.]],
-             [[0., 1., 0., 0., 0.]],
-             [[0., 0., 0., 1., 0.]],
-             [[0., 0., 0., 0., 1.]]],
-            bucketized_price_tensor.eval())
+            [[[1., 0., 0., 0., 0.]], [[0., 1., 0., 0., 0.]],
+             [[0., 0., 0., 1., 0.]], [[0., 0., 0., 0., 1.]]],
+            self.evaluate(bucketized_price_tensor))
 
   def test_get_dense_tensor_two_input_values(self):
     """Tests _get_dense_tensor() for input with shape=[2]."""
@@ -578,7 +576,7 @@ class BucketizedColumnTest(test.TestCase):
             # One-hot tensor.
             [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]],
              [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]],
-            bucketized_price_tensor.eval())
+            self.evaluate(bucketized_price_tensor))
 
   def test_get_sparse_tensors_one_input_value(self):
     """Tests _get_sparse_tensors() for input with shape=[1]."""
@@ -647,20 +645,23 @@ class BucketizedColumnTest(test.TestCase):
       predictions = model(features)
       bucketized_price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight variable per bucket, all initialized to zero.
-        self.assertAllClose(
-            [[0.], [0.], [0.], [0.], [0.]], bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.], [0.]],
+                            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(predictions))
         sess.run(bucketized_price_var.assign(
             [[10.], [20.], [30.], [40.], [50.]]))
         # price -1. is in the 0th bucket, whose weight is 10.
         # price 1. is in the 1st bucket, whose weight is 20.
         # price 5. is in the 3rd bucket, whose weight is 40.
         # price 6. is in the 4th bucket, whose weight is 50.
-        self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [20.], [40.], [50.]],
+                            self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
+        self.assertAllClose([[11.], [21.], [41.], [51.]],
+                            self.evaluate(predictions))
 
   def test_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
@@ -672,12 +673,12 @@ class BucketizedColumnTest(test.TestCase):
       predictions = model(features)
       bucketized_price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight per bucket per input column, all initialized to zero.
         self.assertAllClose(
             [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]],
-            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(bucketized_price_var.assign(
             [[10.], [20.], [30.], [40.], [50.],
              [60.], [70.], [80.], [90.], [100.]]))
@@ -687,9 +688,9 @@ class BucketizedColumnTest(test.TestCase):
         # 2nd example:
         #   price 5. is in the 3rd bucket, whose weight is 40.
         #   price 6. is in the 9th bucket, whose weight is 100.
-        self.assertAllClose([[80.], [140.]], predictions.eval())
+        self.assertAllClose([[80.], [140.]], self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[81.], [141.]], predictions.eval())
+        self.assertAllClose([[81.], [141.]], self.evaluate(predictions))
 
   def test_old_linear_model_one_input_value(self):
     """Tests linear_model() for input with shape=[1]."""
@@ -701,20 +702,23 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight variable per bucket, all initialized to zero.
         self.assertAllClose([[0.], [0.], [0.], [0.], [0.]],
-                            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval())
+                            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(predictions))
         sess.run(
             bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]]))
         # price -1. is in the 0th bucket, whose weight is 10.
         # price 1. is in the 1st bucket, whose weight is 20.
         # price 5. is in the 3rd bucket, whose weight is 40.
         # price 6. is in the 4th bucket, whose weight is 50.
-        self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [20.], [40.], [50.]],
+                            self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
+        self.assertAllClose([[11.], [21.], [41.], [51.]],
+                            self.evaluate(predictions))
 
   def test_old_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
@@ -726,12 +730,12 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight per bucket per input column, all initialized to zero.
         self.assertAllClose(
             [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]],
-            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(
             bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.],
                                          [60.], [70.], [80.], [90.], [100.]]))
@@ -741,9 +745,9 @@ class BucketizedColumnTest(test.TestCase):
         # 2nd example:
         #   price 5. is in the 3rd bucket, whose weight is 40.
         #   price 6. is in the 9th bucket, whose weight is 100.
-        self.assertAllClose([[80.], [140.]], predictions.eval())
+        self.assertAllClose([[80.], [140.]], self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[81.], [141.]], predictions.eval())
+        self.assertAllClose([[81.], [141.]], self.evaluate(predictions))
 
   def test_old_linear_model_one_input_value_old_numeric(self):
     """Tests linear_model() for input with shape=[1]."""
@@ -755,20 +759,23 @@ class BucketizedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       bucketized_price_var = get_linear_model_column_var(bucketized_price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         # One weight variable per bucket, all initialized to zero.
         self.assertAllClose([[0.], [0.], [0.], [0.], [0.]],
-                            bucketized_price_var.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval())
+                            self.evaluate(bucketized_price_var))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(predictions))
         sess.run(
             bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]]))
         # price -1. is in the 0th bucket, whose weight is 10.
         # price 1. is in the 1st bucket, whose weight is 20.
         # price 5. is in the 3rd bucket, whose weight is 40.
         # price 6. is in the 4th bucket, whose weight is 50.
-        self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval())
+        self.assertAllClose([[10.], [20.], [40.], [50.]],
+                            self.evaluate(predictions))
         sess.run(bias.assign([1.]))
-        self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
+        self.assertAllClose([[11.], [21.], [41.], [51.]],
+                            self.evaluate(predictions))
 
   def test_serialization(self):
     price = fc.numeric_column_v2('price', shape=[2])
@@ -995,13 +1002,14 @@ class HashedCategoricalColumnTest(test.TestCase):
       })
       wire_var, bias = model.variables
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 3: wire_var[3] = 4
         # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
-        self.assertAllClose(((4.,), (6.,)), predictions.eval())
+        self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
@@ -1017,13 +1025,14 @@ class HashedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 3: wire_var[3] = 4
         # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
-        self.assertAllClose(((4.,), (6.,)), predictions.eval())
+        self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
@@ -1276,15 +1285,15 @@ class CrossedColumnTest(test.TestCase):
       })
       crossed_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(
-            ((0.,), (0.,), (0.,), (0.,), (0.,)), crossed_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(crossed_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,))))
         # Expected ids after cross = (1, 0, 1, 3, 4, 2)
-        self.assertAllClose(((3.,), (14.,)), predictions.eval())
+        self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions))
         sess.run(bias.assign((.1,)))
-        self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
+        self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
   def test_linear_model_with_weights(self):
 
@@ -1369,15 +1378,15 @@ class CrossedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       crossed_var = get_linear_model_column_var(crossed)
       with _initialized_session() as sess:
-        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
         self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)),
-                            crossed_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+                            self.evaluate(crossed_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,))))
         # Expected ids after cross = (1, 0, 1, 3, 4, 2)
-        self.assertAllClose(((3.,), (14.,)), predictions.eval())
+        self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions))
         sess.run(bias.assign((.1,)))
-        self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
+        self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
   def test_old_linear_model_with_weights(self):
 
@@ -1479,15 +1488,15 @@ class CrossedColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       crossed_var = get_linear_model_column_var(crossed)
       with _initialized_session() as sess:
-        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
         self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)),
-                            crossed_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+                            self.evaluate(crossed_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,))))
         # Expected ids after cross = (1, 0, 1, 3, 4, 2)
-        self.assertAllClose(((3.,), (14.,)), predictions.eval())
+        self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions))
         sess.run(bias.assign((.1,)))
-        self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
+        self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
   def test_serialization(self):
     a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
@@ -1594,10 +1603,10 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         sess.run(price_var.assign([[10.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -1611,11 +1620,12 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       wire_cast_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(wire_cast_var))
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -1633,7 +1643,7 @@ class LinearModelTest(test.TestCase):
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[1015.], [10065.]], predictions.eval())
+        self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_column(self):
     """When the column is both dense and sparse, uses sparse tensors."""
@@ -1688,7 +1698,7 @@ class LinearModelTest(test.TestCase):
         sess.run(dense_and_sparse_column_var.assign(
             [[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
     price = fc.numeric_column_v2('price')
@@ -1698,12 +1708,12 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((1, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[10., 100., 1000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -1717,15 +1727,15 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       wire_cast_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var))
         sess.run(
             wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [
                 1000., 1100., 1200.
             ], [10000., 11000., 12000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -1735,9 +1745,9 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       price_var, _ = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([[0.], [0.]], price_var.eval())
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -1752,7 +1762,7 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       wire_cast_var, _ = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var))
         self.assertAllClose(
             np.zeros((2, 1)),
             predictions.eval(feed_dict={wire_tensor: wire_value}))
@@ -1775,7 +1785,7 @@ class LinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [5010.]], predictions.eval())
+        self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_sparse_combiner_with_negative_weights(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -1796,7 +1806,7 @@ class LinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [-9985.]], predictions.eval())
+        self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -1806,12 +1816,12 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((2, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]]))
         sess.run(bias.assign([2., 3., 4.]))
         self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -1831,11 +1841,11 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       price_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -1849,14 +1859,14 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       price1_var, price2_var, bias = model.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price1_var.eval())
-        self.assertAllClose([[0.]], price2_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price1_var))
+        self.assertAllClose([[0.]], self.evaluate(price2_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price1_var.assign([[10.], [100.]]))
         sess.run(price2_var.assign([[1000.]]))
         sess.run(bias.assign([7.]))
-        self.assertAllClose([[3217.], [4657.]], predictions.eval())
+        self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_dense_trainable_default(self):
     price = fc.numeric_column_v2('price')
@@ -2200,14 +2210,14 @@ class LinearModelTest(test.TestCase):
       price_var1, bias1 = model1.variables
       price_var2, bias2 = model2.variables
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias1.eval())
+        self.assertAllClose([0.], self.evaluate(bias1))
         sess.run(price_var1.assign([[10.]]))
         sess.run(bias1.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions1.eval())
-        self.assertAllClose([0.], bias2.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions1))
+        self.assertAllClose([0.], self.evaluate(bias2))
         sess.run(price_var2.assign([[10.]]))
         sess.run(bias2.assign([5.]))
-        self.assertAllClose([[25.], [105.]], predictions2.eval())
+        self.assertAllClose([[25.], [105.]], self.evaluate(predictions2))
 
 
 class OldLinearModelTest(test.TestCase):
@@ -2278,10 +2288,10 @@ class OldLinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
         sess.run(price_var.assign([[10.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -2295,11 +2305,12 @@ class OldLinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.], [0.], [0.]],
+                            self.evaluate(wire_cast_var))
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -2318,7 +2329,7 @@ class OldLinearModelTest(test.TestCase):
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
         sess.run(price_var.assign([[10.]]))
-        self.assertAllClose([[1015.], [10065.]], predictions.eval())
+        self.assertAllClose([[1015.], [10065.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_column(self):
     """When the column is both dense and sparse, uses sparse tensors."""
@@ -2400,7 +2411,7 @@ class OldLinearModelTest(test.TestCase):
             dense_and_sparse_column_var.assign([[10.], [100.], [1000.],
                                                 [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+        self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
     price = fc.numeric_column_v2('price')
@@ -2410,12 +2421,12 @@ class OldLinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((1, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((1, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[10., 100., 1000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -2429,15 +2440,15 @@ class OldLinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var))
         sess.run(
             wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.],
                                   [1000., 1100., 1200.],
                                   [10000., 11000., 12000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -2446,9 +2457,9 @@ class OldLinearModelTest(test.TestCase):
       predictions = fc_old.linear_model(features, [price])
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([[0.], [0.]], price_var.eval())
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -2462,7 +2473,7 @@ class OldLinearModelTest(test.TestCase):
       predictions = fc_old.linear_model(features, [wire_cast])
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval())
+        self.assertAllClose(np.zeros((4, 1)), self.evaluate(wire_cast_var))
         self.assertAllClose(
             np.zeros((2, 1)),
             predictions.eval(feed_dict={wire_tensor: wire_value}))
@@ -2486,7 +2497,7 @@ class OldLinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [5010.]], predictions.eval())
+        self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_sparse_combiner_with_negative_weights(self):
     wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
@@ -2508,7 +2519,7 @@ class OldLinearModelTest(test.TestCase):
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
         sess.run(bias.assign([5.]))
-        self.assertAllClose([[1005.], [-9985.]], predictions.eval())
+        self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -2518,12 +2529,12 @@ class OldLinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose(np.zeros((3,)), bias.eval())
-        self.assertAllClose(np.zeros((2, 3)), price_var.eval())
+        self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
+        self.assertAllClose(np.zeros((2, 3)), self.evaluate(price_var))
         sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]]))
         sess.run(bias.assign([2., 3., 4.]))
         self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]],
-                            predictions.eval())
+                            self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -2542,11 +2553,11 @@ class OldLinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price_var.assign([[10.], [100.]]))
-        self.assertAllClose([[210.], [650.]], predictions.eval())
+        self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -2558,14 +2569,14 @@ class OldLinearModelTest(test.TestCase):
       price1_var = get_linear_model_column_var(price1)
       price2_var = get_linear_model_column_var(price2)
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias.eval())
-        self.assertAllClose([[0.], [0.]], price1_var.eval())
-        self.assertAllClose([[0.]], price2_var.eval())
-        self.assertAllClose([[0.], [0.]], predictions.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
+        self.assertAllClose([[0.], [0.]], self.evaluate(price1_var))
+        self.assertAllClose([[0.]], self.evaluate(price2_var))
+        self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
         sess.run(price1_var.assign([[10.], [100.]]))
         sess.run(price2_var.assign([[1000.]]))
         sess.run(bias.assign([7.]))
-        self.assertAllClose([[3217.], [4657.]], predictions.eval())
+        self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_fills_cols_to_vars(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -2918,14 +2929,14 @@ class OldLinearModelTest(test.TestCase):
       price_var1 = get_linear_model_column_var(price, name='linear_model')
       price_var2 = get_linear_model_column_var(price, name='linear_model_1')
       with _initialized_session() as sess:
-        self.assertAllClose([0.], bias1.eval())
+        self.assertAllClose([0.], self.evaluate(bias1))
         sess.run(price_var1.assign([[10.]]))
         sess.run(bias1.assign([5.]))
-        self.assertAllClose([[15.], [55.]], predictions1.eval())
-        self.assertAllClose([0.], bias2.eval())
+        self.assertAllClose([[15.], [55.]], self.evaluate(predictions1))
+        self.assertAllClose([0.], self.evaluate(bias2))
         sess.run(price_var2.assign([[10.]]))
         sess.run(bias2.assign([5.]))
-        self.assertAllClose([[25.], [105.]], predictions2.eval())
+        self.assertAllClose([[25.], [105.]], self.evaluate(predictions2))
 
   def test_linear_model_v1_shared_embedding_all_other_v2(self):
     price = fc.numeric_column_v2('price')  # v2
@@ -2961,7 +2972,7 @@ class OldLinearModelTest(test.TestCase):
       fc_old.linear_model(features, all_cols)
       bias = get_linear_model_bias()
       with _initialized_session():
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v1_shared_embedding_with_v2_cat_all_other_v2(self):
     price = fc.numeric_column_v2('price')  # v2
@@ -2997,7 +3008,7 @@ class OldLinearModelTest(test.TestCase):
       fc_old.linear_model(features, all_cols)
       bias = get_linear_model_bias()
       with _initialized_session():
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v1_v2_mix(self):
     price = fc.numeric_column_v2('price')  # v2
@@ -3033,7 +3044,7 @@ class OldLinearModelTest(test.TestCase):
       fc_old.linear_model(features, all_cols)
       bias = get_linear_model_bias()
       with _initialized_session():
-        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v2_shared_embedding_all_other_v1(self):
     price = fc_old.numeric_column('price')  # v1
@@ -3197,7 +3208,7 @@ class DenseFeaturesTest(test.TestCase):
       features = features = {'a': [0.]}
       net = fc.DenseFeatures(fc.numeric_column_v2('a'))(features)
       with _initialized_session():
-        self.assertAllClose([[0.]], net.eval())
+        self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
@@ -3205,7 +3216,7 @@ class DenseFeaturesTest(test.TestCase):
       columns = (fc.numeric_column_v2(key) for key in features)
       net = fc.DenseFeatures(columns)(features)
       with _initialized_session():
-        self.assertAllClose([[0., 1.]], net.eval())
+        self.assertAllClose([[0., 1.]], self.evaluate(net))
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
@@ -3224,7 +3235,7 @@ class DenseFeaturesTest(test.TestCase):
       features = {'price': [[1.], [5.]]}
       net = fc.DenseFeatures([price])(features)
       with _initialized_session():
-        self.assertAllClose([[1.], [5.]], net.eval())
+        self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -3232,7 +3243,7 @@ class DenseFeaturesTest(test.TestCase):
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc.DenseFeatures([price])(features)
       with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
+        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_compute_output_shape(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -3247,7 +3258,8 @@ class DenseFeaturesTest(test.TestCase):
       net = dense_features(features)
       with _initialized_session():
         self.assertAllClose(
-            [[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]], net.eval())
+            [[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]],
+            self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -3264,7 +3276,7 @@ class DenseFeaturesTest(test.TestCase):
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc.DenseFeatures([price])(features)
       with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
+        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -3276,7 +3288,7 @@ class DenseFeaturesTest(test.TestCase):
       }
       net = fc.DenseFeatures([price1, price2])(features)
       with _initialized_session():
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
+        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_cols_to_output_tensors(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -3289,7 +3301,7 @@ class DenseFeaturesTest(test.TestCase):
       with _initialized_session():
         self.assertAllClose([[1., 2.], [5., 6.]], cols_dict[price1].eval())
         self.assertAllClose([[3.], [4.]], cols_dict[price2].eval())
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
+        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_column_order(self):
     price_a = fc.numeric_column_v2('price_a')
@@ -3302,8 +3314,8 @@ class DenseFeaturesTest(test.TestCase):
       net1 = fc.DenseFeatures([price_a, price_b])(features)
       net2 = fc.DenseFeatures([price_b, price_a])(features)
       with _initialized_session():
-        self.assertAllClose([[1., 3.]], net1.eval())
-        self.assertAllClose([[1., 3.]], net2.eval())
+        self.assertAllClose([[1., 3.]], self.evaluate(net1))
+        self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
     animal = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
@@ -3788,7 +3800,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = features = {'a': [0.]}
       net = fc_old.input_layer(features, fc.numeric_column_v2('a'))
       with _initialized_session():
-        self.assertAllClose([[0.]], net.eval())
+        self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
@@ -3796,7 +3808,7 @@ class FunctionalInputLayerTest(test.TestCase):
       columns = (fc.numeric_column_v2(key) for key in features)
       net = fc_old.input_layer(features, columns)
       with _initialized_session():
-        self.assertAllClose([[0., 1.]], net.eval())
+        self.assertAllClose([[0., 1.]], self.evaluate(net))
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
@@ -3814,7 +3826,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price': [[1.], [5.]]}
       net = fc_old.input_layer(features, [price])
       with _initialized_session():
-        self.assertAllClose([[1.], [5.]], net.eval())
+        self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -3822,7 +3834,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc_old.input_layer(features, [price])
       with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
+        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column_v2('price', shape=2)
@@ -3839,7 +3851,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc_old.input_layer(features, [price])
       with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], net.eval())
+        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
     price1 = fc.numeric_column_v2('price1', shape=2)
@@ -3848,7 +3860,7 @@ class FunctionalInputLayerTest(test.TestCase):
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       net = fc_old.input_layer(features, [price1, price2])
       with _initialized_session():
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
+        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_fills_cols_to_vars(self):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
@@ -3979,8 +3991,8 @@ class FunctionalInputLayerTest(test.TestCase):
       net1 = fc_old.input_layer(features, [price_a, price_b])
       net2 = fc_old.input_layer(features, [price_b, price_a])
       with _initialized_session():
-        self.assertAllClose([[1., 3.]], net1.eval())
-        self.assertAllClose([[1., 3.]], net2.eval())
+        self.assertAllClose([[1., 3.]], self.evaluate(net1))
+        self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
     animal = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
@@ -4573,13 +4585,12 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         dense_shape=(2, 2))
     id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column]
     with _initialized_session():
-      _assert_sparse_tensor_value(self,
-                                  sparse_tensor.SparseTensorValue(
-                                      indices=inputs.indices,
-                                      values=np.array(
-                                          (2, -1, 0), dtype=np.int64),
-                                      dense_shape=inputs.dense_shape),
-                                  id_tensor.eval())
+      _assert_sparse_tensor_value(
+          self,
+          sparse_tensor.SparseTensorValue(
+              indices=inputs.indices,
+              values=np.array((2, -1, 0), dtype=np.int64),
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_vocabulary_file_v2(
@@ -4763,13 +4774,14 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       })
       wire_var, bias = model.variables
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_file_v2(
@@ -4789,13 +4801,14 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     wire_column = fc.categorical_column_with_vocabulary_file_v2(
@@ -5032,8 +5045,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_vocabulary_list_v2(
@@ -5185,13 +5197,14 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       })
       wire_var, bias = model.variables
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_list_v2(
@@ -5210,13 +5223,14 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
+                            self.evaluate(wire_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
         # 'marlo' -> 2: wire_var[2] = 3
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     wire_column = fc.categorical_column_with_vocabulary_list_v2(
@@ -5349,8 +5363,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
@@ -5463,13 +5476,13 @@ class IdentityCategoricalColumnTest(test.TestCase):
       })
       weight_var, bias = model.variables
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] = 1
         # weight_var[2] + weight_var[1] = 3+2 = 5
-        self.assertAllClose(((1.,), (5.,)), predictions.eval())
+        self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
@@ -5485,13 +5498,13 @@ class IdentityCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] = 1
         # weight_var[2] + weight_var[1] = 3+2 = 5
-        self.assertAllClose(((1.,), (5.,)), predictions.eval())
+        self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
@@ -5597,7 +5610,8 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = transformation_cache.get(animal, None)
     with self.cached_session():
-      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval())
+      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
+                          self.evaluate(output))
 
   def test_2D_shape_succeeds(self):
     # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready.
@@ -5612,7 +5626,8 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = transformation_cache.get(animal, None)
     with self.cached_session():
-      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]], output.eval())
+      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
+                          self.evaluate(output))
 
   def test_multi_hot(self):
     animal = fc.indicator_column_v2(
@@ -5625,7 +5640,7 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = transformation_cache.get(animal, None)
     with self.cached_session():
-      self.assertAllEqual([[0., 2., 0., 0.]], output.eval())
+      self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output))
 
   def test_multi_hot2(self):
     animal = fc.indicator_column_v2(
@@ -5637,7 +5652,7 @@ class IndicatorColumnTest(test.TestCase):
     })
     output = transformation_cache.get(animal, None)
     with self.cached_session():
-      self.assertAllEqual([[0., 1., 1., 0.]], output.eval())
+      self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
   def test_deep_copy(self):
     a = fc.categorical_column_with_hash_bucket_v2('a', 4)
@@ -5683,7 +5698,8 @@ class IndicatorColumnTest(test.TestCase):
     indicator_tensor = fc._transform_features(features, [a_indicator],
                                               None)[a_indicator]
     with _initialized_session():
-      self.assertAllEqual([[0, 0, 1], [1, 0, 0]], indicator_tensor.eval())
+      self.assertAllEqual([[0, 0, 1], [1, 0, 0]],
+                          self.evaluate(indicator_tensor))
 
   def test_transform_with_weighted_column(self):
     # Github issue 12557
@@ -5698,7 +5714,7 @@ class IndicatorColumnTest(test.TestCase):
     indicator_tensor = fc._transform_features(features, [indicator],
                                               None)[indicator]
     with _initialized_session():
-      self.assertAllEqual([[6., 4., 2.]], indicator_tensor.eval())
+      self.assertAllEqual([[6., 4., 2.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
@@ -5713,7 +5729,7 @@ class IndicatorColumnTest(test.TestCase):
     indicator_tensor = fc._transform_features(features, [indicator],
                                               None)[indicator]
     with _initialized_session():
-      self.assertAllEqual([[0., 4., 2.]], indicator_tensor.eval())
+      self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
@@ -5726,7 +5742,7 @@ class IndicatorColumnTest(test.TestCase):
     indicator_tensor = fc._transform_features(features, [indicator],
                                               None)[indicator]
     with _initialized_session():
-      self.assertAllEqual([[0., 1., 1.]], indicator_tensor.eval())
+      self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
   def test_linear_model(self):
     animal = fc.indicator_column_v2(
@@ -5743,10 +5759,10 @@ class IndicatorColumnTest(test.TestCase):
       weight_var, _ = model.variables
       with _initialized_session():
         # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval())
-        self.assertAllClose([[0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+        self.assertAllClose([[0.]], self.evaluate(predictions))
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], predictions.eval())
+        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_old_linear_model(self):
     animal = fc.indicator_column_v2(
@@ -5762,10 +5778,10 @@ class IndicatorColumnTest(test.TestCase):
       weight_var = get_linear_model_column_var(animal)
       with _initialized_session():
         # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval())
-        self.assertAllClose([[0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+        self.assertAllClose([[0.]], self.evaluate(predictions))
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], predictions.eval())
+        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
     animal = fc.indicator_column_v2(
@@ -5781,10 +5797,10 @@ class IndicatorColumnTest(test.TestCase):
       weight_var = get_linear_model_column_var(animal)
       with _initialized_session():
         # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval())
-        self.assertAllClose([[0.]], predictions.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+        self.assertAllClose([[0.]], self.evaluate(predictions))
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], predictions.eval())
+        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_dense_features(self):
     animal = fc.indicator_column_v2(
@@ -5797,7 +5813,7 @@ class IndicatorColumnTest(test.TestCase):
       }
       net = fc.DenseFeatures([animal])(features)
       with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], net.eval())
+        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_input_layer(self):
     animal = fc.indicator_column_v2(
@@ -5810,7 +5826,7 @@ class IndicatorColumnTest(test.TestCase):
       }
       net = fc_old.input_layer(features, [animal])
       with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], net.eval())
+        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_input_layer_old_categorical(self):
     animal = fc.indicator_column_v2(
@@ -5823,7 +5839,7 @@ class IndicatorColumnTest(test.TestCase):
       }
       net = fc_old.input_layer(features, [animal])
       with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], net.eval())
+        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_serialization(self):
     parent = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
@@ -6024,8 +6040,8 @@ class EmbeddingColumnTest(test.TestCase):
     output_a = outputs[a]
     output_embedded = outputs[a_embedded]
     with _initialized_session():
-      _assert_sparse_tensor_value(
-          self, output_a.eval(), output_embedded.eval())
+      _assert_sparse_tensor_value(self, self.evaluate(output_a),
+                                  self.evaluate(output_embedded))
 
   def test_get_dense_tensor(self):
     # Inputs.
@@ -6086,7 +6102,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_old_categorical(self):
     # Inputs.
@@ -6146,7 +6162,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_3d(self):
     # Inputs.
@@ -6209,7 +6225,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
@@ -6342,7 +6358,7 @@ class EmbeddingColumnTest(test.TestCase):
         ('embedding_weights:0',), tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_linear_model(self):
     # Inputs.
@@ -6397,11 +6413,13 @@ class EmbeddingColumnTest(test.TestCase):
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
+        self.assertAllClose(
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -6416,7 +6434,8 @@ class EmbeddingColumnTest(test.TestCase):
         # example 3, ids [1], embedding[3] = [3, 5]
         # sum(embeddings * linear_weights)
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
+        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                            self.evaluate(predictions))
 
   def test_dense_features(self):
     # Inputs.
@@ -6478,7 +6497,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in trainable_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, dense_features.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_dense_features_not_trainable(self):
     # Inputs.
@@ -6539,7 +6558,7 @@ class EmbeddingColumnTest(test.TestCase):
         [], ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, dense_features.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_input_layer(self):
     # Inputs.
@@ -6601,7 +6620,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in trainable_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, dense_features.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_old_linear_model(self):
     # Inputs.
@@ -6658,11 +6677,13 @@ class EmbeddingColumnTest(test.TestCase):
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -6677,7 +6698,8 @@ class EmbeddingColumnTest(test.TestCase):
         # example 3, ids [1], embedding[3] = [3, 5]
         # sum(embeddings * linear_weights)
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
+        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                            self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
     # Inputs.
@@ -6734,11 +6756,13 @@ class EmbeddingColumnTest(test.TestCase):
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -6753,7 +6777,8 @@ class EmbeddingColumnTest(test.TestCase):
         # example 3, ids [1], embedding[3] = [3, 5]
         # sum(embeddings * linear_weights)
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
+        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                            self.evaluate(predictions))
 
   def test_serialization(self):
 
@@ -6993,10 +7018,10 @@ class SharedEmbeddingColumnTest(test.TestCase):
     output_b = outputs[b]
     output_b_embedded = outputs[b_embedded]
     with _initialized_session():
-      _assert_sparse_tensor_value(
-          self, output_a.eval(), output_a_embedded.eval())
-      _assert_sparse_tensor_value(
-          self, output_b.eval(), output_b_embedded.eval())
+      _assert_sparse_tensor_value(self, self.evaluate(output_a),
+                                  self.evaluate(output_a_embedded))
+      _assert_sparse_tensor_value(self, self.evaluate(output_b),
+                                  self.evaluate(output_b_embedded))
 
   def test_get_dense_tensor(self):
     # Inputs.
@@ -7062,9 +7087,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in global_vars]))
     embedding_var = global_vars[0]
     with _initialized_session():
-      self.assertAllEqual(embedding_values, embedding_var.eval())
-      self.assertAllEqual(expected_lookups_a, embedding_lookup_a.eval())
-      self.assertAllEqual(expected_lookups_b, embedding_lookup_b.eval())
+      self.assertAllEqual(embedding_values, self.evaluate(embedding_var))
+      self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a))
+      self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b))
 
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
@@ -7185,13 +7210,15 @@ class SharedEmbeddingColumnTest(test.TestCase):
           'linear_model/bbb_shared_embedding/weights:0']
       with _initialized_session():
         # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), bias.eval())
-        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+        self.assertAllClose(zeros_embedding_values,
+                            self.evaluate(embedding_weights))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights_a.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a))
         self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), linear_weights_b.eval())
-        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b))
+        self.assertAllClose(
+            np.zeros((batch_size, 1)), self.evaluate(predictions))
 
         # Predictions with all non-zero weights.
         embedding_weights.assign((
@@ -7209,7 +7236,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         # example 1, ids [], embedding[1] = 0, 0]
         # sum(embeddings * linear_weights)
         # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
-        self.assertAllClose([[94. + 13.], [29.]], predictions.eval())
+        self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions))
 
   def _test_dense_features(self, trainable=True):
     # Inputs.
@@ -7319,7 +7346,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     shared_embedding_vars = global_vars
     with _initialized_session():
       self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval())
-      self.assertAllEqual(expected_lookups, dense_features.eval())
+      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_dense_features(self):
     self._test_dense_features()
@@ -7493,15 +7520,13 @@ class WeightedCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array(inputs.values, dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
       _assert_sparse_tensor_value(
           self,
           sparse_tensor.SparseTensorValue(
               indices=weights.indices,
               values=np.array(weights.values, dtype=np.float32),
-              dense_shape=weights.dense_shape),
-          weight_tensor.eval())
+              dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_input(self):
     column = fc.weighted_categorical_column_v2(
@@ -7522,15 +7547,13 @@ class WeightedCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=((0, 0), (1, 0), (1, 1)),
               values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=(2, 2)),
-          id_tensor.eval())
+              dense_shape=(2, 2)), self.evaluate(id_tensor))
       _assert_sparse_tensor_value(
           self,
           sparse_tensor.SparseTensorValue(
               indices=weights.indices,
               values=np.array(weights.values, dtype=np.float32),
-              dense_shape=weights.dense_shape),
-          weight_tensor.eval())
+              dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_weights(self):
     column = fc.weighted_categorical_column_v2(
@@ -7551,15 +7574,13 @@ class WeightedCategoricalColumnTest(test.TestCase):
           sparse_tensor.SparseTensorValue(
               indices=inputs.indices,
               values=np.array(inputs.values, dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_tensor.eval())
+              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
       _assert_sparse_tensor_value(
           self,
           sparse_tensor.SparseTensorValue(
               indices=((0, 0), (1, 0), (1, 1)),
               values=np.array((.5, 1., .1), dtype=np.float32),
-              dense_shape=(2, 2)),
-          weight_tensor.eval())
+              dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
   def test_linear_model(self):
     column = fc.weighted_categorical_column_v2(
@@ -7582,14 +7603,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       })
       weight_var, bias = model.variables
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model_mismatched_shape(self):
     column = fc.weighted_categorical_column_v2(
@@ -7635,7 +7656,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
           rewriter_config_pb2.RewriterConfig.OFF)
       with _initialized_session(config):
         with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'):
-          predictions.eval()
+          self.evaluate(predictions)
 
   def test_linear_model_mismatched_dense_shape(self):
     column = fc.weighted_categorical_column_v2(
@@ -7654,14 +7675,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       })
       weight_var, bias = model.variables
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     column = fc.weighted_categorical_column_v2(
@@ -7684,14 +7705,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model_mismatched_shape(self):
     column = fc.weighted_categorical_column_v2(
@@ -7736,7 +7757,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
           rewriter_config_pb2.RewriterConfig.OFF)
       with _initialized_session(config):
         with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'):
-          predictions.eval()
+          self.evaluate(predictions)
 
   def test_old_linear_model_mismatched_dense_shape(self):
     column = fc.weighted_categorical_column_v2(
@@ -7755,14 +7776,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
     column = fc.weighted_categorical_column_v2(
@@ -7785,14 +7806,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
-        self.assertAllClose((0.,), bias.eval())
-        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
-        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        self.assertAllClose((0.,), self.evaluate(bias))
+        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
         weight_var.assign(((1.,), (2.,), (3.,))).eval()
         # weight_var[0] * weights[0, 0] = 1 * .5 = .5
         # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
         # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   # TODO(ptucker): Add test with embedding of weighted categorical.
 
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 13ee6c5d2d..971219d5b0 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -552,8 +552,8 @@ class FunctionTest(test.TestCase):
 
     with self.session(graph=g):
       v.initializer.run()
-      self.assertAllEqual(expected_val.eval(), actual_val.eval())
-      self.assertAllEqual(expected_shape, actual_shape.eval())
+      self.assertAllEqual(expected_val.eval(), self.evaluate(actual_val))
+      self.assertAllEqual(expected_shape, self.evaluate(actual_shape))
 
   def testDefineErrors(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index 2b4d8e7299..fc7367649e 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -930,7 +930,7 @@ class ImportGraphDefTest(test.TestCase):
           name="",
           return_elements=["id:0"])
       with self.cached_session():
-        self.assertEqual(5.0, t.eval())
+        self.assertEqual(5.0, self.evaluate(t))
 
   def testInvalidInputForReturnOperations(self):
     with ops.Graph().as_default():
@@ -1071,7 +1071,7 @@ class ImportGraphDefTest(test.TestCase):
       tensor_input = np.ones(input_shape, dtype=np.float32)
       t = constant_op.constant(tensor_input, shape=input_shape)
       g = array_ops.identity(t)
-      g.eval()
+      self.evaluate(g)
 
   def testVersion(self):
     v0 = versions.GRAPH_DEF_VERSION_MIN_CONSUMER
@@ -1255,7 +1255,7 @@ class ImportGraphDefTest(test.TestCase):
     z = TestFunc()
 
     with self.cached_session():
-      z_val = z.eval()
+      z_val = self.evaluate(z)
       self.assertEqual(z_val, -2.0)
 
   def testImportGraphWithFunctionTwice(self):
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index d7c5a1a6e5..3957d1de53 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -317,7 +317,7 @@ class OperationTest(test_util.TensorFlowTestCase):
       values = [[2], [3], [5], [7]]
       tensor = ops.convert_to_tensor(values)
       self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, tensor.eval())
+      self.assertAllEqual(values, self.evaluate(tensor))
 
   def testShapeTuple(self):
     with self.cached_session():
@@ -346,18 +346,18 @@ class OperationTest(test_util.TensorFlowTestCase):
       tensor = ops.convert_to_tensor(
           [constant_op.constant(row) for row in values])
       self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, tensor.eval())
+      self.assertAllEqual(values, self.evaluate(tensor))
       tensor = ops.convert_to_tensor(
           [[constant_op.constant(v) for v in row] for row in values])
       self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, tensor.eval())
+      self.assertAllEqual(values, self.evaluate(tensor))
 
   def testConvertToTensorNestedMix(self):
     with self.cached_session():
       values = ([2], (3,), [constant_op.constant(5)], constant_op.constant([7]))
       tensor = ops.convert_to_tensor(values)
       self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(((2,), (3,), (5,), (7,)), tensor.eval())
+      self.assertAllEqual(((2,), (3,), (5,), (7,)), self.evaluate(tensor))
 
   def testConvertToTensorPreferred(self):
     with self.cached_session():
@@ -2490,12 +2490,14 @@ class KernelLabelTest(test_util.TensorFlowTestCase):
       # pylint: enable=protected-access
       default_3 = test_ops.kernel_label()
 
-      self.assertAllEqual(b"My label is: default", default_1.eval())
-      self.assertAllEqual(b"My label is: default", default_2.eval())
-      self.assertAllEqual(b"My label is: default", default_3.eval())
-      self.assertAllEqual(b"My label is: overload_1", overload_1_1.eval())
-      self.assertAllEqual(b"My label is: overload_1", overload_1_2.eval())
-      self.assertAllEqual(b"My label is: overload_2", overload_2.eval())
+      self.assertAllEqual(b"My label is: default", self.evaluate(default_1))
+      self.assertAllEqual(b"My label is: default", self.evaluate(default_2))
+      self.assertAllEqual(b"My label is: default", self.evaluate(default_3))
+      self.assertAllEqual(b"My label is: overload_1",
+                          self.evaluate(overload_1_1))
+      self.assertAllEqual(b"My label is: overload_1",
+                          self.evaluate(overload_1_2))
+      self.assertAllEqual(b"My label is: overload_2", self.evaluate(overload_2))
 
 
 class AsGraphDefTest(test_util.TensorFlowTestCase):
diff --git a/tensorflow/python/framework/sparse_tensor_test.py b/tensorflow/python/framework/sparse_tensor_test.py
index 22423c4f58..2f7591abbd 100644
--- a/tensorflow/python/framework/sparse_tensor_test.py
+++ b/tensorflow/python/framework/sparse_tensor_test.py
@@ -46,7 +46,7 @@ class SparseTensorTest(test_util.TensorFlowTestCase):
       self.assertEqual(sp.get_shape(), (4, 5))
 
       with self.cached_session() as sess:
-        value = sp.eval()
+        value = self.evaluate(sp)
         self.assertAllEqual(indices, value.indices)
         self.assertAllEqual(values, value.values)
         self.assertAllEqual(shape, value.dense_shape)
@@ -85,7 +85,7 @@ class ConvertToTensorOrSparseTensorTest(test_util.TensorFlowTestCase):
       value = [42, 43]
       from_value = sparse_tensor.convert_to_tensor_or_sparse_tensor(
           value)
-      self.assertAllEqual(value, from_value.eval())
+      self.assertAllEqual(value, self.evaluate(from_value))
 
   def test_convert_sparse(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index f169fa6b86..ef95d27abf 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -158,12 +158,11 @@ class AdadeltaOptimizerTest(test.TestCase):
             loss, var_list=[var0])
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-111, -138]], var0.eval())
+        self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 69b601ffc7..70bf127dd8 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -87,23 +87,24 @@ class AdamOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
 
         # Run 3 steps of Adam
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           update.run()
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testSparseDevicePlacement(self):
     for index_dtype in [dtypes.int32, dtypes.int64]:
@@ -141,12 +142,12 @@ class AdamOptimizerTest(test.TestCase):
             [(grad_aggregated, aggregated_update_var)])
         variables.global_variables_initializer().run()
         self.assertAllClose(aggregated_update_var.eval(),
-                            repeated_index_update_var.eval())
+                            self.evaluate(repeated_index_update_var))
         for _ in range(3):
           repeated_update.run()
           aggregated_update.run()
           self.assertAllClose(aggregated_update_var.eval(),
-                              repeated_index_update_var.eval())
+                              self.evaluate(repeated_index_update_var))
 
   def doTestBasic(self, use_callable_params=False):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
@@ -226,23 +227,24 @@ class AdamOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
 
         # Run 3 steps of Adam
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           update.run()
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -266,13 +268,14 @@ class AdamOptimizerTest(test.TestCase):
         beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 3 steps of intertwined Adam1 and Adam2.
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           if t % 2 == 0:
             update1.run()
           else:
@@ -282,8 +285,8 @@ class AdamOptimizerTest(test.TestCase):
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testSlotsUniqueEager(self):
     with context.eager_mode():
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index b84bf1a6ec..fa7cca1420 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -289,8 +289,8 @@ class MomentumOptimizerTest(test.TestCase):
               var0_np, accum0_np, var0_np * 10, 2.0, 0.9)
           var1_np, accum1_np = self._update_nesterov_momentum_numpy(
               var1_np, accum1_np, 3, 2.0, 0.9)
-          self.assertAllClose(var0_np, var0.eval())
-          self.assertAllClose(var1_np, var1.eval())
+          self.assertAllClose(var0_np, self.evaluate(var0))
+          self.assertAllClose(var1_np, self.evaluate(var1))
 
   def testSparseNesterovMomentum(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -329,8 +329,8 @@ class MomentumOptimizerTest(test.TestCase):
               var0_np, accum0_np, var0_np * 10, 2.0, 0.9)
           var1_np, accum1_np = self._update_nesterov_momentum_numpy(
               var1_np, accum1_np, 3, 2.0, 0.9)
-          self.assertAllClose(var0_np, var0.eval())
-          self.assertAllClose(var1_np, var1.eval())
+          self.assertAllClose(var0_np, self.evaluate(var0))
+          self.assertAllClose(var1_np, self.evaluate(var1))
 
   @test_util.run_in_graph_and_eager_modes
   def testMinimizeSparseResourceVariable(self):
@@ -406,37 +406,43 @@ class MomentumOptimizerTest(test.TestCase):
         self.assertEquals(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
-        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.1, 0.1]), self.evaluate(slot0))
+        self.assertAllCloseAccordingToType(
+            np.array([0.01, 0.01]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+            self.evaluate(var1))
         # Step 2: the momentum accumulators contain the previous update.
         mom_update.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
                 2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
                 3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval())
+            ]), self.evaluate(var1))
 
   def testSparse(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -461,51 +467,57 @@ class MomentumOptimizerTest(test.TestCase):
         self.assertEquals(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
-        self.assertAllClose([0, 0], var0.eval()[0])
-        self.assertAllClose([0, 0], var0.eval()[1])
-        self.assertAllClose([1, 1], var1.eval()[2])
+        self.assertAllClose([0, 0], self.evaluate(var0)[0])
+        self.assertAllClose([0, 0], self.evaluate(var0)[1])
+        self.assertAllClose([1, 1], self.evaluate(var1)[2])
 
         # Step 1: the momentum accumulators are 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0, 0]), slot0.eval()[0])
-        self.assertAllCloseAccordingToType(np.array([.1, .1]), slot0.eval()[1])
+        self.assertAllCloseAccordingToType(
+            np.array([0, 0]),
+            self.evaluate(slot0)[0])
+        self.assertAllCloseAccordingToType(
+            np.array([.1, .1]),
+            self.evaluate(slot0)[1])
         self.assertAllCloseAccordingToType(
             np.array([.01, .01]),
-            slot1.eval()[2])
+            self.evaluate(slot1)[2])
         # Check that the parameters have been updated.
-        self.assertAllCloseAccordingToType(np.array([0, 0]), var0.eval()[0])
+        self.assertAllCloseAccordingToType(
+            np.array([0, 0]),
+            self.evaluate(var0)[0])
         self.assertAllCloseAccordingToType(
             np.array([-(0.1 * 2.0), -(0.1 * 2.0)]),
-            var0.eval()[1])
+            self.evaluate(var0)[1])
         self.assertAllCloseAccordingToType(
             np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]),
-            var1.eval()[2])
+            self.evaluate(var1)[2])
         # Step 2: the momentum accumulators contain the previous update.
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllClose(np.array([0, 0]), slot0.eval()[0])
+        self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0])
         self.assertAllCloseAccordingToType(
             np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
-            slot0.eval()[1])
+            self.evaluate(slot0)[1])
         self.assertAllCloseAccordingToType(
             np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
-            slot1.eval()[2])
+            self.evaluate(slot1)[2])
         # Check that the parameters have been updated.
-        self.assertAllClose(np.array([0, 0]), var0.eval()[0])
+        self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0])
         self.assertAllCloseAccordingToType(
             np.array([
                 -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
             ]),
-            var0.eval()[1])
+            self.evaluate(var0)[1])
         self.assertAllCloseAccordingToType(
             np.array([
                 0.98 - ((0.9 * 0.01 + 0.01) * 2.0),
                 0.98 - ((0.9 * 0.01 + 0.01) * 2.0)
             ]),
-            var1.eval()[2])
+            self.evaluate(var1)[2])
 
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -527,37 +539,43 @@ class MomentumOptimizerTest(test.TestCase):
         self.assertEquals(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update1.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
-        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.1, 0.1]), self.evaluate(slot0))
+        self.assertAllCloseAccordingToType(
+            np.array([0.01, 0.01]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+            self.evaluate(var1))
         # Step 2: the second momentum accumulators contain the previous update.
         mom_update2.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
                 2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
                 3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval())
+            ]), self.evaluate(var1))
 
   @test_util.run_in_graph_and_eager_modes
   def testConfig(self):
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 57a2c7789f..8d7afa54cc 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -132,8 +132,8 @@ class RMSPropOptimizerTest(test.TestCase):
         mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 4 steps of RMSProp
         for _ in range(1, 5):
@@ -148,14 +148,14 @@ class RMSPropOptimizerTest(test.TestCase):
 
           # Validate updated params
           if centered:
-            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+            self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
+            self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+          self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+          self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -173,12 +173,13 @@ class RMSPropOptimizerTest(test.TestCase):
                 loss, var_list=[var0])
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[0., 1.]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[0., 1.]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testMinimizeSparseResourceVariableCentered(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -196,12 +197,13 @@ class RMSPropOptimizerTest(test.TestCase):
                 loss, var_list=[var0])
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-111, -138]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[-111, -138]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testSparse(self):
     for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
@@ -256,8 +258,8 @@ class RMSPropOptimizerTest(test.TestCase):
         mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 4 steps of RMSProp
         for _ in range(1, 5):
@@ -272,14 +274,14 @@ class RMSPropOptimizerTest(test.TestCase):
 
           # Validate updated params
           if centered:
-            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+            self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
+            self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+          self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+          self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testCallableParams(self):
     with context.eager_mode():
diff --git a/tensorflow/python/kernel_tests/accumulate_n_test.py b/tensorflow/python/kernel_tests/accumulate_n_test.py
index 7889edc198..ae24cf8f14 100644
--- a/tensorflow/python/kernel_tests/accumulate_n_test.py
+++ b/tensorflow/python/kernel_tests/accumulate_n_test.py
@@ -88,13 +88,13 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase):
       np_val = random_arrays[0]
       for random_array in random_arrays[1:]:
         np_val += random_array
-      self.assertAllClose(np_val, tf_val.eval())
+      self.assertAllClose(np_val, self.evaluate(tf_val))
 
   def testZeroArgs(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
         tf_val = math_ops.accumulate_n([])
-        tf_val.eval()
+        self.evaluate(tf_val)
 
   def testWrongShape(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/argmax_op_test.py b/tensorflow/python/kernel_tests/argmax_op_test.py
index fa370c17b4..d34a1dc9b2 100644
--- a/tensorflow/python/kernel_tests/argmax_op_test.py
+++ b/tensorflow/python/kernel_tests/argmax_op_test.py
@@ -37,14 +37,14 @@ class ArgMaxTest(test.TestCase):
     with self.session(use_gpu=use_gpu):
       ans = method(x, axis=axis)
       if expected_err_re is None:
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
         # Defaults to int64 output.
         self.assertEqual(np.int64, tf_ans.dtype)
         self.assertAllEqual(tf_ans, expected_values)
         self.assertShapeEqual(expected_values, ans)
       else:
         with self.assertRaisesOpError(expected_err_re):
-          ans.eval()
+          self.evaluate(ans)
 
   def _testBothArg(self,
                    method,
@@ -79,7 +79,7 @@ class ArgMaxTest(test.TestCase):
     expected_values = x.argmax()
     with self.session(use_gpu=True):
       ans = math_ops.argmax(x, axis=0, output_type=dtypes.int32)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
       self.assertEqual(np.int32, tf_ans.dtype)
       # The values are equal when comparing int32 to int64 because
       # the values don't have a range that exceeds 32-bit integers.
@@ -87,7 +87,7 @@ class ArgMaxTest(test.TestCase):
     expected_values = x.argmin()
     with self.session(use_gpu=True):
       ans = math_ops.argmin(x, axis=0, output_type=dtypes.int32)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
       self.assertEqual(np.int32, tf_ans.dtype)
       self.assertAllEqual(tf_ans, expected_values)
 
diff --git a/tensorflow/python/kernel_tests/atrous_conv2d_test.py b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
index 1d82b3d058..fefb797995 100644
--- a/tensorflow/python/kernel_tests/atrous_conv2d_test.py
+++ b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
@@ -79,7 +79,8 @@ class AtrousConv2DTest(test.TestCase):
                 y1 = nn_ops.atrous_conv2d(x, f, rate, padding=padding)
                 y2 = nn_ops.conv2d(
                     x, f_up, strides=[1, 1, 1, 1], padding=padding)
-                self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-3, atol=1e-3)
+                self.assertAllClose(
+                    y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3)
 
   def testAtrousSequence(self):
     """Tests optimization of sequence of atrous convolutions.
@@ -131,7 +132,8 @@ class AtrousConv2DTest(test.TestCase):
               y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding)
               y2 = nn_ops.conv2d(y2, f, strides=[1, 1, 1, 1], padding=padding)
               y2 = array_ops.batch_to_space(y2, crops=pad, block_size=rate)
-              self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-2, atol=1e-2)
+              self.assertAllClose(
+                  y1.eval(), self.evaluate(y2), rtol=1e-2, atol=1e-2)
 
   def testGradient(self):
     with self.session(use_gpu=True):
@@ -193,7 +195,8 @@ class AtrousConv2DTransposeTest(test.TestCase):
                                                     padding)
                 y2 = nn_ops.conv2d_transpose(
                     x, f_up, y_shape, strides=[1, 1, 1, 1], padding=padding)
-                self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-3, atol=1e-3)
+                self.assertAllClose(
+                    y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3)
 
 
 class AtrousDepthwiseConv2DTest(test.TestCase):
@@ -220,7 +223,8 @@ class AtrousDepthwiseConv2DTest(test.TestCase):
                 y1 = nn_impl.depthwise_conv2d(
                     x, f, strides, padding, rate=[rate, rate])
                 y2 = nn_impl.depthwise_conv2d(x, f_up, strides, padding)
-                self.assertAllClose(y1.eval(), y2.eval(), rtol=1e-3, atol=1e-3)
+                self.assertAllClose(
+                    y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py
index 1e09ba5b65..14db06b783 100644
--- a/tensorflow/python/kernel_tests/attention_ops_test.py
+++ b/tensorflow/python/kernel_tests/attention_ops_test.py
@@ -121,8 +121,7 @@ class ExtractGlimpseTest(test.TestCase):
     with self.cached_session():
       result = image_ops.extract_glimpse(empty_image, [1, 1], offsets)
       self.assertAllEqual(
-          np.zeros(
-              (0, 1, 1, 0), dtype=np.float32), result.eval())
+          np.zeros((0, 1, 1, 0), dtype=np.float32), self.evaluate(result))
 
   def testLargeCenterGlimpse(self):
     self._VerifyValues(
diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py
index 225c1b35ae..ac5cbc810a 100644
--- a/tensorflow/python/kernel_tests/basic_gpu_test.py
+++ b/tensorflow/python/kernel_tests/basic_gpu_test.py
@@ -214,7 +214,7 @@ class BroadcastSimpleTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_gpu = out.eval()
+      tf_gpu = self.evaluate(out)
     self.assertAllClose(np_ans, tf_gpu)
     self.assertShapeEqual(np_ans, out)
     # TODO(zhifengc/ke): make gradient checker work on GPU.
diff --git a/tensorflow/python/kernel_tests/batch_gather_op_test.py b/tensorflow/python/kernel_tests/batch_gather_op_test.py
index 547506d844..ad4e879131 100644
--- a/tensorflow/python/kernel_tests/batch_gather_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_gather_op_test.py
@@ -52,7 +52,7 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         gather_t = array_ops.batch_gather(params, indices_tf)
         expected_result = np.array([3, 7])
         np_val = self._buildParams(expected_result, dtype)
-        gather_val = gather_t.eval()
+        gather_val = self.evaluate(gather_t)
         self.assertAllEqual(np_val, gather_val)
         self.assertEqual(np_val.shape, gather_t.get_shape())
 
@@ -68,7 +68,7 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         gather_t = array_ops.batch_gather(params, indices_tf)
         expected_result = np.array([[3], [15]])
         np_val = self._buildParams(expected_result, dtype)
-        gather_val = gather_t.eval()
+        gather_val = self.evaluate(gather_t)
         self.assertAllEqual(np_val, gather_val)
         self.assertEqual(np_val.shape, gather_t.get_shape())
 
@@ -81,7 +81,7 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         params = constant_op.constant(params_np)
         indices_tf = constant_op.constant(indices)
         gather_t = array_ops.batch_gather(params, indices_tf)
-        gather_val = gather_t.eval()
+        gather_val = self.evaluate(gather_t)
         expected_result = np.array([[[2, 0], [7, 5]], [[10, 8], [11, 15]]])
         np_val = self._buildParams(expected_result, dtype)
         self.assertAllEqual(np_val, gather_val)
diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
index 8f6c089b42..a0ad8151b2 100644
--- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
@@ -86,7 +86,7 @@ class BatchMatmulOpTest(test.TestCase):
     with self.cached_session(use_gpu=is_floating) as sess:
       if static_shape:
         z0 = math_ops.matmul(x, y, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
-        z0_val = z0.eval()
+        z0_val = self.evaluate(z0)
       else:
         x_ph = array_ops.placeholder(x.dtype)
         y_ph = array_ops.placeholder(y.dtype)
diff --git a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
index 742a204883..a4b461bc87 100644
--- a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
@@ -91,7 +91,7 @@ class ScatterTest(test.TestCase):
 
       session.run([update0, update1])
 
-      self.assertAllEqual([False, True], var.eval())
+      self.assertAllEqual([False, True], self.evaluate(var))
 
   def testScatterOutOfRange(self):
     params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32)
diff --git a/tensorflow/python/kernel_tests/betainc_op_test.py b/tensorflow/python/kernel_tests/betainc_op_test.py
index 92d21462d5..5d7446042e 100644
--- a/tensorflow/python/kernel_tests/betainc_op_test.py
+++ b/tensorflow/python/kernel_tests/betainc_op_test.py
@@ -48,7 +48,7 @@ class BetaincTest(test.TestCase):
       tf_x_s = constant_op.constant(x_s, dtype=dtype)
       tf_out_t = math_ops.betainc(tf_a_s, tf_b_s, tf_x_s)
       with self.cached_session():
-        tf_out = tf_out_t.eval()
+        tf_out = self.evaluate(tf_out_t)
       scipy_out = special.betainc(a_s, b_s, x_s).astype(np_dt)
 
       # the scipy version of betainc uses a double-only implementation.
diff --git a/tensorflow/python/kernel_tests/bitcast_op_test.py b/tensorflow/python/kernel_tests/bitcast_op_test.py
index 79e0f36d24..4dcf218d7c 100644
--- a/tensorflow/python/kernel_tests/bitcast_op_test.py
+++ b/tensorflow/python/kernel_tests/bitcast_op_test.py
@@ -30,7 +30,7 @@ class BitcastTest(test.TestCase):
   def _testBitcast(self, x, datatype, shape):
     with self.session(use_gpu=True):
       tf_ans = array_ops.bitcast(x, datatype)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       buff_after = memoryview(out).tobytes()
       buff_before = memoryview(x).tobytes()
       self.assertEqual(buff_before, buff_after)
diff --git a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
index 65bb9ab55f..493cad80f3 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
@@ -35,13 +35,13 @@ class ResourceOpsTest(test_util.TensorFlowTestCase):
       ensemble = boosted_trees_ops.TreeEnsemble('ensemble')
       resources.initialize_resources(resources.shared_resources()).run()
       stamp_token = ensemble.get_stamp_token()
-      self.assertEqual(0, stamp_token.eval())
+      self.assertEqual(0, self.evaluate(stamp_token))
       (_, num_trees, num_finalized_trees, num_attempted_layers,
        nodes_range) = ensemble.get_states()
-      self.assertEqual(0, num_trees.eval())
-      self.assertEqual(0, num_finalized_trees.eval())
-      self.assertEqual(0, num_attempted_layers.eval())
-      self.assertAllEqual([0, 1], nodes_range.eval())
+      self.assertEqual(0, self.evaluate(num_trees))
+      self.assertEqual(0, self.evaluate(num_finalized_trees))
+      self.assertEqual(0, self.evaluate(num_attempted_layers))
+      self.assertAllEqual([0, 1], self.evaluate(nodes_range))
 
   def testCreateWithProto(self):
     with self.cached_session():
@@ -154,11 +154,11 @@ class ResourceOpsTest(test_util.TensorFlowTestCase):
       resources.initialize_resources(resources.shared_resources()).run()
       (stamp_token, num_trees, num_finalized_trees, num_attempted_layers,
        nodes_range) = ensemble.get_states()
-      self.assertEqual(7, stamp_token.eval())
-      self.assertEqual(2, num_trees.eval())
-      self.assertEqual(1, num_finalized_trees.eval())
-      self.assertEqual(6, num_attempted_layers.eval())
-      self.assertAllEqual([16, 19], nodes_range.eval())
+      self.assertEqual(7, self.evaluate(stamp_token))
+      self.assertEqual(2, self.evaluate(num_trees))
+      self.assertEqual(1, self.evaluate(num_finalized_trees))
+      self.assertEqual(6, self.evaluate(num_attempted_layers))
+      self.assertAllEqual([16, 19], self.evaluate(nodes_range))
 
   def testSerializeDeserialize(self):
     with self.cached_session():
@@ -167,11 +167,11 @@ class ResourceOpsTest(test_util.TensorFlowTestCase):
       resources.initialize_resources(resources.shared_resources()).run()
       (stamp_token, num_trees, num_finalized_trees, num_attempted_layers,
        nodes_range) = ensemble.get_states()
-      self.assertEqual(5, stamp_token.eval())
-      self.assertEqual(0, num_trees.eval())
-      self.assertEqual(0, num_finalized_trees.eval())
-      self.assertEqual(0, num_attempted_layers.eval())
-      self.assertAllEqual([0, 1], nodes_range.eval())
+      self.assertEqual(5, self.evaluate(stamp_token))
+      self.assertEqual(0, self.evaluate(num_trees))
+      self.assertEqual(0, self.evaluate(num_finalized_trees))
+      self.assertEqual(0, self.evaluate(num_attempted_layers))
+      self.assertAllEqual([0, 1], self.evaluate(nodes_range))
 
       # Deserialize.
       ensemble_proto = boosted_trees_pb2.TreeEnsemble()
@@ -219,18 +219,18 @@ class ResourceOpsTest(test_util.TensorFlowTestCase):
       ]):
         (stamp_token, num_trees, num_finalized_trees, num_attempted_layers,
          nodes_range) = ensemble.get_states()
-      self.assertEqual(3, stamp_token.eval())
-      self.assertEqual(1, num_trees.eval())
+      self.assertEqual(3, self.evaluate(stamp_token))
+      self.assertEqual(1, self.evaluate(num_trees))
       # This reads from metadata, not really counting the layers.
-      self.assertEqual(5, num_attempted_layers.eval())
-      self.assertEqual(0, num_finalized_trees.eval())
-      self.assertAllEqual([3, 7], nodes_range.eval())
+      self.assertEqual(5, self.evaluate(num_attempted_layers))
+      self.assertEqual(0, self.evaluate(num_finalized_trees))
+      self.assertAllEqual([3, 7], self.evaluate(nodes_range))
 
 
       # Serialize.
       new_ensemble_proto = boosted_trees_pb2.TreeEnsemble()
       new_stamp_token, new_serialized = ensemble.serialize()
-      self.assertEqual(3, new_stamp_token.eval())
+      self.assertEqual(3, self.evaluate(new_stamp_token))
       new_ensemble_proto.ParseFromString(new_serialized.eval())
       self.assertProtoEquals(ensemble_proto, new_ensemble_proto)
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
index 09e9cfa3af..cc3984015d 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
@@ -359,7 +359,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
               [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
               [[-.33, .58], [0., 0.], [.3, .4], [0., 0.]],  # node 2
           ]],
-          result.eval())
+          self.evaluate(result))
 
   def testMakeStatsSummaryMultipleFeatures(self):
     """Tests that MakeStatsSummary works for multiple features."""
@@ -389,7 +389,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
                   [[.3, .4], [0., 0.], [-.4, .5], [.07, .08]],  # node 2
               ],  # feature 1
           ],
-          result.eval())
+          self.evaluate(result))
 
   def _verify_precision(self, length):
     with self.cached_session():
@@ -408,7 +408,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           node_ids, gradients, hessians, [bucketized_features], max_splits,
           num_buckets)  # shape=[max_splits, num_buckets, num_features, 2]
 
-      self.assertAllClose([[[[2., 0.2]]]], result.eval())
+      self.assertAllClose([[[[2., 0.2]]]], self.evaluate(result))
 
   def testMakeStatsSummaryNumericalPrecisionSmallBatch(self):
     """Tests numeric precision."""
diff --git a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
index b19077db56..46ab71537f 100644
--- a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
+++ b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
@@ -55,7 +55,7 @@ class RangeSamplerOpsTest(test.TestCase):
           [[1, 2], [0, 4], [3, 3]], dtype=dtypes.int64)
       sampled_candidates, _, _ = candidate_sampling_ops.all_candidate_sampler(
           true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True)
-      result = sampled_candidates.eval()
+      result = self.evaluate(sampled_candidates)
 
     expected_ids = [0, 1, 2, 3, 4]
     self.assertAllEqual(result, expected_ids)
@@ -68,7 +68,7 @@ class RangeSamplerOpsTest(test.TestCase):
       _, true_expected_count, _ = candidate_sampling_ops.all_candidate_sampler(
           true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True)
       true_log_expected_count = math_ops.log(true_expected_count)
-      result = true_log_expected_count.eval()
+      result = self.evaluate(true_log_expected_count)
 
     self.assertAllEqual(result, [[0.0] * self.NUM_TRUE] * self.BATCH_SIZE)
     self.assertEqual(true_expected_count.get_shape(),
@@ -83,7 +83,7 @@ class RangeSamplerOpsTest(test.TestCase):
       _, _, sampled_expected_count = candidate_sampling_ops.all_candidate_sampler(  # pylint: disable=line-too-long
           true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True)
       sampled_log_expected_count = math_ops.log(sampled_expected_count)
-      result = sampled_log_expected_count.eval()
+      result = self.evaluate(sampled_log_expected_count)
 
     self.assertAllEqual(result, [0.0] * self.NUM_SAMPLED)
     self.assertEqual(sampled_expected_count.get_shape(), [self.NUM_SAMPLED])
@@ -114,7 +114,7 @@ class RangeSamplerOpsTest(test.TestCase):
             [[1, 2], [0, 4], [3, 3]], dtype=dtypes.int64)
         sampled, _, _ = candidate_sampling_ops.log_uniform_candidate_sampler(
             true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True, 5, seed=seed)
-        return sampled.eval()
+        return self.evaluate(sampled)
 
     # Non-zero seed. Repeatable.
     for seed in [1, 12, 123, 1234]:
diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py
index a5dff5df62..bc49cd5a04 100644
--- a/tensorflow/python/kernel_tests/cast_op_test.py
+++ b/tensorflow/python/kernel_tests/cast_op_test.py
@@ -107,10 +107,10 @@ class CastOpTest(test.TestCase):
     a = np.random.uniform(-100, 100, 100).astype(np.float32)
     with self.cached_session(use_gpu=False):
       b = math_ops.cast(math_ops.cast(a, dtypes.bfloat16), dtypes.float32)
-      self.assertAllClose(a, b.eval(), rtol=1 / 128.)
+      self.assertAllClose(a, self.evaluate(b), rtol=1 / 128.)
     with self.cached_session(use_gpu=True):
       b = math_ops.cast(math_ops.cast(a, dtypes.bfloat16), dtypes.float32)
-      self.assertAllClose(a, b.eval(), rtol=1 / 128.)
+      self.assertAllClose(a, self.evaluate(b), rtol=1 / 128.)
 
   def testRandom(self):
     self._testAll(np.random.normal(0, 10, 210).reshape([2, 3, 5, 7]))
diff --git a/tensorflow/python/kernel_tests/checkpoint_ops_test.py b/tensorflow/python/kernel_tests/checkpoint_ops_test.py
index 51611b75af..213ac292d3 100644
--- a/tensorflow/python/kernel_tests/checkpoint_ops_test.py
+++ b/tensorflow/python/kernel_tests/checkpoint_ops_test.py
@@ -58,8 +58,8 @@ class GenerateVocabRemappingTest(test.TestCase):
     expected_remapping = range(0, 3)
     expected_num_present = 3
     with self.cached_session():
-      self.assertAllEqual(expected_remapping, remapping.eval())
-      self.assertAllEqual(expected_num_present, num_present.eval())
+      self.assertAllEqual(expected_remapping, self.evaluate(remapping))
+      self.assertAllEqual(expected_num_present, self.evaluate(num_present))
 
   def test_generate_remapping_with_shifted_vocab(self):
     """Tests where vocab is the same, but shifted / ordered differently."""
@@ -71,8 +71,8 @@ class GenerateVocabRemappingTest(test.TestCase):
     expected_remapping = [2, 0, 1]
     expected_num_present = 3
     with self.cached_session():
-      self.assertAllEqual(expected_remapping, remapping.eval())
-      self.assertAllEqual(expected_num_present, num_present.eval())
+      self.assertAllEqual(expected_remapping, self.evaluate(remapping))
+      self.assertAllEqual(expected_num_present, self.evaluate(num_present))
 
   def test_generate_remapping_with_offset(self):
     """Tests offset and num_new_vocab logic."""
@@ -84,8 +84,8 @@ class GenerateVocabRemappingTest(test.TestCase):
     expected_remapping = [0]
     expected_num_present = 1
     with self.cached_session():
-      self.assertAllEqual(expected_remapping, remapping.eval())
-      self.assertAllEqual(expected_num_present, num_present.eval())
+      self.assertAllEqual(expected_remapping, self.evaluate(remapping))
+      self.assertAllEqual(expected_num_present, self.evaluate(num_present))
 
   def test_generate_remapping_with_old_vocab_size(self):
     """Tests where old_vocab_size is specified."""
@@ -99,8 +99,8 @@ class GenerateVocabRemappingTest(test.TestCase):
     expected_remapping = [-1, 0, 1]
     expected_num_present = 2
     with self.cached_session():
-      self.assertAllEqual(expected_remapping, remapping.eval())
-      self.assertAllEqual(expected_num_present, num_present.eval())
+      self.assertAllEqual(expected_remapping, self.evaluate(remapping))
+      self.assertAllEqual(expected_num_present, self.evaluate(num_present))
 
 
 class LoadAndRemapMatrixTest(test.TestCase):
@@ -142,7 +142,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_cols=self.old_num_cols)
     with self.cached_session():
       self.assertAllClose(self.matrix_value[row_remapping],
-                          remapped_matrix.eval())
+                          self.evaluate(remapped_matrix))
 
     # No row remapping, new weight matrix has third col, then first col.
     row_remapping = list(range(self.old_num_rows))
@@ -157,7 +157,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_cols=len(col_remapping))
     with self.cached_session():
       self.assertAllClose(self.matrix_value[row_remapping][:, col_remapping],
-                          remapped_matrix.eval())
+                          self.evaluate(remapped_matrix))
 
     # Both row and column remappings.
     row_remapping = [1, 0, 4]
@@ -172,7 +172,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_cols=len(col_remapping))
     with self.cached_session():
       self.assertAllClose(self.matrix_value[row_remapping][:, col_remapping],
-                          remapped_matrix.eval())
+                          self.evaluate(remapped_matrix))
 
   def test_load_and_remap_with_init(self):
     """Tests the op's load and remap where there are missing entries."""
@@ -190,7 +190,8 @@ class LoadAndRemapMatrixTest(test.TestCase):
         [33, init_val, init_val, init_val, 1, init_val], [3, 2])
 
     with self.cached_session():
-      self.assertAllClose(expected_remapped_matrix, remapped_matrix.eval())
+      self.assertAllClose(expected_remapped_matrix,
+                          self.evaluate(remapped_matrix))
 
   def test_load_and_remap_all_missing_rows(self):
     """Tests when all the rows are missing and need to be initialized."""
@@ -207,7 +208,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
     with self.cached_session():
       self.assertAllClose(
           np.reshape(initializing_values, (num_rows, self.old_num_cols)),
-          remapped_matrix.eval())
+          self.evaluate(remapped_matrix))
 
   def test_load_and_remap_all_missing_rows_and_cols(self):
     """Tests when all the rows & cols are missing and need to be initialized."""
@@ -225,7 +226,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
     with self.cached_session():
       self.assertAllClose(
           np.reshape(initializing_values, (num_rows, num_cols)),
-          remapped_matrix.eval())
+          self.evaluate(remapped_matrix))
 
   def test_load_and_remap_invalid_remapping(self):
     """Tests that errors are raised when an ID maps to multiple new IDs.
@@ -244,7 +245,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_rows=len(invalid_remapping),
         num_cols=self.old_num_cols)
     with self.cached_session(), self.assertRaises(errors.UnimplementedError):
-      remapped_matrix.eval()
+      self.evaluate(remapped_matrix)
 
     # Invalid column remapping.
     remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
@@ -256,7 +257,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_rows=self.old_num_rows,
         num_cols=len(invalid_remapping))
     with self.cached_session(), self.assertRaises(errors.UnimplementedError):
-      remapped_matrix.eval()
+      self.evaluate(remapped_matrix)
 
   def test_load_and_remap_incorrect_initializing_values(self):
     """Tests that errors are raised with incorrect number of init values."""
@@ -273,7 +274,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_rows=3,
         num_cols=2)
     with self.cached_session(), self.assertRaises(errors.InvalidArgumentError):
-      remapped_matrix.eval()
+      self.evaluate(remapped_matrix)
 
     remapped_matrix = gen_checkpoint_ops.load_and_remap_matrix(
         ckpt_path=[self.bundle_file],
@@ -285,7 +286,7 @@ class LoadAndRemapMatrixTest(test.TestCase):
         num_rows=3,
         num_cols=2)
     with self.cached_session(), self.assertRaises(errors.InvalidArgumentError):
-      remapped_matrix.eval()
+      self.evaluate(remapped_matrix)
 
 
 class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
@@ -324,7 +325,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
           num_rows=num_rows,
           num_cols=num_cols,
           max_rows_in_memory=max_rows_in_memory)
-      self.assertAllClose(np_value[::-1], remapped_matrix.eval())
+      self.assertAllClose(np_value[::-1], self.evaluate(remapped_matrix))
 
       # Tests loading the tensor (except for the first and last rows), with
       # uninitialized values. Requires num_rows to be at least 3 since we're
@@ -348,7 +349,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
           np.vstack([
               np.tile(42, [prefix_rows, num_cols]), np_value[1:-1],
               np.tile(42, [suffix_rows, num_cols])
-          ]), remapped_matrix.eval())
+          ]), self.evaluate(remapped_matrix))
 
       # Tests when everything is taken from initializing_values.
       new_rows = 7
@@ -365,7 +366,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
           max_rows_in_memory=max_rows_in_memory)
       self.assertAllClose(
           np.reshape(initializing_values, (new_rows, num_cols)),
-          remapped_matrix.eval())
+          self.evaluate(remapped_matrix))
 
   def test_loading_rows_divisible_by_max_rows(self):
     """Tests loading normal var when rows are evenly divisible by max_rows."""
diff --git a/tensorflow/python/kernel_tests/clip_ops_test.py b/tensorflow/python/kernel_tests/clip_ops_test.py
index efd7eee847..d0cd7eb302 100644
--- a/tensorflow/python/kernel_tests/clip_ops_test.py
+++ b/tensorflow/python/kernel_tests/clip_ops_test.py
@@ -55,7 +55,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-4.4, 2.0, 3.0], [4.0, 4.4, 4.4]]
       clip_value = 4.4
       ans = clip_ops.clip_by_value(x, -clip_value, clip_value)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -71,7 +71,7 @@ class ClipTest(test.TestCase):
         clip_value_min = 2
         clip_value_max = 4
         ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max)
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
 
       self.assertAllClose(np_ans, tf_ans)
 
@@ -88,7 +88,7 @@ class ClipTest(test.TestCase):
             [2, 2, 2, 3, 3, 3], shape=[2, 3], dtype=dtype)
         clip_value_max = 4
         ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max)
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
 
       self.assertAllClose(np_ans, tf_ans)
 
@@ -105,7 +105,7 @@ class ClipTest(test.TestCase):
         clip_value_max = constant_op.constant(
             [6, 6, 6, 6, 6, 6], shape=[2, 3], dtype=dtype)
         ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max)
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
 
       self.assertAllClose(np_ans, tf_ans)
 
@@ -123,7 +123,7 @@ class ClipTest(test.TestCase):
         clip_value_max = constant_op.constant(
             [5, 5, 5, 7, 7, 7], shape=[2, 3], dtype=dtype)
         ans = clip_ops.clip_by_value(x, clip_value_min, clip_value_max)
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
 
       self.assertAllClose(np_ans, tf_ans)
 
@@ -144,7 +144,7 @@ class ClipTest(test.TestCase):
       np_ans = [float('NaN'), 4.0, -4.0]
       clip_value = 4.0
       ans = clip_ops.clip_by_value(x, -clip_value, clip_value)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -157,10 +157,10 @@ class ClipTest(test.TestCase):
       np_ans = [[-2.4, 0.0, 0.0], [3.2, 0.0, 0.0]]
       clip_norm = 4.0
       ans = clip_ops.clip_by_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
       ans = clip_ops.clip_by_norm(x, clip_norm)
-      tf_ans_tensor = ans.eval()
+      tf_ans_tensor = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
     self.assertAllClose(np_ans, tf_ans_tensor)
@@ -188,7 +188,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]]
       clip_norm = 6.0
       ans = clip_ops.clip_by_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -200,7 +200,7 @@ class ClipTest(test.TestCase):
       np_ans = [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
       clip_norm = 6.0
       ans = clip_ops.clip_by_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -212,7 +212,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-2.4, 0.0, 0.0], [3.2, 0.0, 3.0]]
       clip_norm = 4.0
       ans = clip_ops.clip_by_norm(x, clip_norm, [0])
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -224,7 +224,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-3.0, 0.0, 0.0], [3.2, 0.0, 2.4]]
       clip_norm = 4.0
       ans = clip_ops.clip_by_norm(x, clip_norm, [1])
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -236,7 +236,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-3.0, 0.0, 0.0], [4.0, 0.0, 3.0]]
       clip_norm = 6.0
       ans = clip_ops.clip_by_norm(x, clip_norm, [1])
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -256,7 +256,7 @@ class ClipTest(test.TestCase):
       ans, norm = clip_ops.clip_by_global_norm((x0, x1), clip_norm)
       tf_ans_1 = ans[0].eval()
       tf_ans_2 = ans[1].eval()
-      tf_norm = norm.eval()
+      tf_norm = self.evaluate(norm)
 
     self.assertAllClose(tf_norm, 5.0)
     self.assertAllClose(np_ans_0, tf_ans_1)
@@ -277,7 +277,7 @@ class ClipTest(test.TestCase):
       ans, norm = clip_ops.clip_by_global_norm((x0, x1), clip_norm)
       tf_ans_1 = ans[0].eval()
       tf_ans_2 = ans[1].eval()
-      tf_norm = norm.eval()
+      tf_norm = self.evaluate(norm)
 
     self.assertAllClose(tf_norm, 5.0)
     self.assertAllClose(np_ans_0, tf_ans_1)
@@ -300,7 +300,7 @@ class ClipTest(test.TestCase):
       self.assertTrue(ans[3] is None)
       tf_ans_1 = ans[0].eval()
       tf_ans_2 = ans[2].eval()
-      tf_norm = norm.eval()
+      tf_norm = self.evaluate(norm)
 
     self.assertAllClose(tf_norm, 5.0)
     self.assertAllClose(np_ans_0, tf_ans_1)
@@ -322,7 +322,7 @@ class ClipTest(test.TestCase):
       ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm)
       tf_ans_1 = ans[0].eval()
       tf_ans_2 = ans[1].values.eval()
-      tf_norm = norm.eval()
+      tf_norm = self.evaluate(norm)
 
     self.assertAllClose(tf_norm, 5.0)
     self.assertAllClose(np_ans_0, tf_ans_1)
@@ -352,7 +352,7 @@ class ClipTest(test.TestCase):
       ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm)
       tf_ans_1 = ans[0].eval()
       tf_ans_2 = ans[1].eval()
-      tf_norm = norm.eval()
+      tf_norm = self.evaluate(norm)
 
     self.assertAllClose(tf_norm, 5.0)
     self.assertAllClose(np_ans_0, tf_ans_1)
@@ -371,7 +371,7 @@ class ClipTest(test.TestCase):
       ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm)
       tf_ans_1 = ans[0].eval()
       tf_ans_2 = ans[1].eval()
-      tf_norm = norm.eval()
+      tf_norm = self.evaluate(norm)
 
     self.assertAllClose(tf_norm, 0.0)
     self.assertAllClose(np_ans_0, tf_ans_1)
@@ -386,7 +386,7 @@ class ClipTest(test.TestCase):
 
       ans, norm = clip_ops.clip_by_global_norm([x0, x1], clip_norm)
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "global norm"):
-        norm.eval()
+        self.evaluate(norm)
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "global norm"):
         ans[0].eval()
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "global norm"):
@@ -400,7 +400,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-2.88, 0.0, 0.0], [3.84, 0.0, 0.0]]
       clip_norm = 0.8
       ans = clip_ops.clip_by_average_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -412,7 +412,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-2.88, 0.0, 0.0], [3.84, 0.0, 0.0]]
       clip_norm = constant_op.constant(0.8)
       ans = clip_ops.clip_by_average_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -424,7 +424,7 @@ class ClipTest(test.TestCase):
       np_ans = [[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]]
       clip_norm = 0.9
       ans = clip_ops.clip_by_average_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
@@ -436,7 +436,7 @@ class ClipTest(test.TestCase):
       np_ans = [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
       clip_norm = 0.9
       ans = clip_ops.clip_by_average_norm(x, clip_norm)
-      tf_ans = ans.eval()
+      tf_ans = self.evaluate(ans)
 
     self.assertAllClose(np_ans, tf_ans)
 
diff --git a/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py b/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py
index f27a0fc472..e1928c5a1c 100644
--- a/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py
+++ b/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py
@@ -33,12 +33,12 @@ class CompareAndBitpackTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ans = math_ops.compare_and_bitpack(x, threshold)
       if expected_err_re is None:
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
         self.assertShapeEqual(truth, ans)
         self.assertAllEqual(tf_ans, truth)
       else:
         with self.assertRaisesOpError(expected_err_re):
-          ans.eval()
+          self.evaluate(ans)
 
   def _testBasic(self, dtype):
     rows = 371
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 92d09986e6..149302831b 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -71,7 +71,7 @@ class ConcatOpTest(test.TestCase):
       x1 = constant_op.constant(p1)
       x2 = constant_op.constant(p2)
       c = array_ops.concat([x1, x2], 0)
-      result = c.eval()
+      result = self.evaluate(c)
     self.assertAllEqual(result[:2, :], p1)
     self.assertAllEqual(result[2:, :], p2)
 
@@ -83,7 +83,7 @@ class ConcatOpTest(test.TestCase):
       v2 = variables.Variable(p2)
       c = array_ops.concat([v1, v2], 0)
       variables.global_variables_initializer().run()
-      result = c.eval()
+      result = self.evaluate(c)
 
     self.assertEqual(result.shape, c.get_shape())
     self.assertAllEqual(result[:4, :], p1)
@@ -195,7 +195,7 @@ class ConcatOpTest(test.TestCase):
             grad_inp.flatten(), shape=output_shape)
         grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
         concated_grad = array_ops.concat(grad, axis)
-        result = concated_grad.eval()
+        result = self.evaluate(concated_grad)
     self.assertAllEqual(result, grad_inp)
 
   def testGradientsSimple(self):
@@ -222,7 +222,7 @@ class ConcatOpTest(test.TestCase):
           grad_inp.flatten(), shape=output_shape)
       grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
       concated_grad = array_ops.concat(grad, 0)
-      result = concated_grad.eval()
+      result = self.evaluate(concated_grad)
 
     self.assertAllEqual(result, grad_inp)
 
@@ -249,7 +249,7 @@ class ConcatOpTest(test.TestCase):
             grad_inp.flatten(), shape=output_shape)
         grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
         concated_grad = array_ops.concat(grad, axis)
-        result = concated_grad.eval()
+        result = self.evaluate(concated_grad)
 
     self.assertAllEqual(result, grad_inp)
 
@@ -279,7 +279,7 @@ class ConcatOpTest(test.TestCase):
       grad_tensor = constant_op.constant(grad_inp.flatten(), shape=output_shape)
       grad = gradients_impl.gradients([c], inp_tensors, [grad_tensor])
       concated_grad = array_ops.concat(grad, concat_dim)
-      result = concated_grad.eval()
+      result = self.evaluate(concated_grad)
 
     self.assertAllEqual(result, grad_inp)
 
@@ -476,7 +476,7 @@ class ConcatOpTest(test.TestCase):
     with self.cached_session():
       concat_list_t = array_ops.concat([c1, c2], 0)
       concat_tuple_t = array_ops.concat((c1, c2), 0)
-      self.assertAllEqual(concat_list_t.eval(), concat_tuple_t.eval())
+      self.assertAllEqual(concat_list_t.eval(), self.evaluate(concat_tuple_t))
 
   def testConcatNoScalars(self):
     with self.cached_session():
@@ -543,13 +543,13 @@ class ConcatOpTest(test.TestCase):
 
       c = gen_array_ops.concat_v2([t1, t2], -2)
       self.assertEqual([4, 3], c.get_shape().as_list())
-      output = c.eval()
+      output = self.evaluate(c)
       self.assertAllEqual([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]],
                           output)
 
       c = gen_array_ops.concat_v2([t1, t2], -1)
       self.assertEqual([2, 6], c.get_shape().as_list())
-      output = c.eval()
+      output = self.evaluate(c)
       self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
   def _testGradientsForAxis(
@@ -615,7 +615,7 @@ class ConcatOpTest(test.TestCase):
         c = gen_array_ops.concat_v2([t1, t2],
                                     constant_op.constant(1, dtype=dtype))
         self.assertEqual([2, 6], c.get_shape().as_list())
-        output = c.eval()
+        output = self.evaluate(c)
         self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
 class ConcatOffsetTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
index 97ab23fe49..893cb7cce3 100644
--- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
@@ -149,7 +149,7 @@ class ConditionalAccumulatorTest(test.TestCase):
         accum_op.run()
 
       is_all_equal = True
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       for i in range(len(val)):
         for j in range(len(val[i])):
           is_all_equal &= (val[i][j] == elems_ave[i][j])
@@ -184,7 +184,7 @@ class ConditionalAccumulatorTest(test.TestCase):
         sess.run(accum_op, feed_dict={x: elem})
 
       is_all_equal = True
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       for i in range(len(val)):
         for j in range(len(val[i])):
           is_all_equal &= (val[i][j] == elems_ave[i][j])
@@ -259,7 +259,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(15.0, val)
 
       accum_ops = [q.apply_grad((x,), local_step=1) for x in elems]
@@ -268,7 +268,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(15.0, val)
 
   def testAccumulatorTakeGradSum(self):
@@ -286,7 +286,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(30.0, val)
 
       accum_ops = [q.apply_grad((x,), local_step=1) for x in elems]
@@ -295,7 +295,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(30.0, val)
 
   def testAccumulatorTakeGradInvalidReductionType(self):
@@ -319,7 +319,7 @@ class ConditionalAccumulatorTest(test.TestCase):
         accum_op.run()
 
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        takeg_t.eval()
+        self.evaluate(takeg_t)
 
   def testAccumulatorRepeatedTakeGradMean(self):
     with self.cached_session():
@@ -334,7 +334,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(elems_ave, val)
 
       elems = [20.0, 30.0]
@@ -345,7 +345,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(elems_ave + 0.0, val)
 
   def testAccumulatorRepeatedTakeGradSum(self):
@@ -364,7 +364,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(elems_sum, val)
 
       elems = [20.0, 30.0]
@@ -375,7 +375,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for accum_op in accum_ops:
         accum_op.run()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
       self.assertEqual(elems_sum, val)
 
   def testAccumulatorIncrementGlobalStep(self):
@@ -392,7 +392,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       variables.global_variables_initializer().run()
       for _ in range(3):
         set_global_step_op.run()
-        inc_global_step.eval()
+        self.evaluate(inc_global_step)
 
   def testAccumulatorSetGlobalStepPreventsAccumulation(self):
     with self.cached_session():
@@ -410,7 +410,7 @@ class ConditionalAccumulatorTest(test.TestCase):
           accum_op.run()
         takeg_t = q.take_grad(1)
 
-        val = takeg_t.eval()
+        val = self.evaluate(takeg_t)
         self.assertEqual(0.0 + sum(x for x in local_steps
                                    if x >= ls) / sum(1 for x in local_steps
                                                      if x >= ls), val)
@@ -436,7 +436,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = takeg_t.eval()
+      val = self.evaluate(takeg_t)
 
       self.assertEqual(val, sum(elems) / len(elems))
 
diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py
index bc24345261..b001341c03 100644
--- a/tensorflow/python/kernel_tests/confusion_matrix_test.py
+++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py
@@ -232,7 +232,7 @@ class ConfusionMatrixTest(test.TestCase):
     with self.cached_session():
       cm = confusion_matrix.confusion_matrix(
           labels, predictions, dtype=dtypes.int32)
-      tf_cm = cm.eval()
+      tf_cm = self.evaluate(cm)
     self.assertEqual(tf_cm.dtype, np.int32)
 
   def testOutputIsInt64(self):
@@ -241,7 +241,7 @@ class ConfusionMatrixTest(test.TestCase):
     with self.cached_session():
       cm = confusion_matrix.confusion_matrix(
           labels, predictions, dtype=dtypes.int64)
-      tf_cm = cm.eval()
+      tf_cm = self.evaluate(cm)
     self.assertEqual(tf_cm.dtype, np.int64)
 
 
@@ -261,8 +261,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
             labels_placeholder, predictions_placeholder))
 
     with self.cached_session():
-      self.assertAllEqual(label_values, static_labels.eval())
-      self.assertAllEqual(prediction_values, static_predictions.eval())
+      self.assertAllEqual(label_values, self.evaluate(static_labels))
+      self.assertAllEqual(prediction_values, self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
@@ -286,8 +286,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
             labels_placeholder, predictions_placeholder))
 
     with self.cached_session():
-      self.assertAllEqual(label_values, static_labels.eval())
-      self.assertAllEqual(prediction_values, static_predictions.eval())
+      self.assertAllEqual(label_values, self.evaluate(static_labels))
+      self.assertAllEqual(prediction_values, self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
@@ -311,8 +311,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
             labels_placeholder, predictions_placeholder, expected_rank_diff=0))
 
     with self.cached_session():
-      self.assertAllEqual(label_values, static_labels.eval())
-      self.assertAllEqual(prediction_values, static_predictions.eval())
+      self.assertAllEqual(label_values, self.evaluate(static_labels))
+      self.assertAllEqual(prediction_values, self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
@@ -337,8 +337,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
 
     expected_label_values = np.reshape(label_values, newshape=(2, 3))
     with self.cached_session():
-      self.assertAllEqual(expected_label_values, static_labels.eval())
-      self.assertAllEqual(prediction_values, static_predictions.eval())
+      self.assertAllEqual(expected_label_values, self.evaluate(static_labels))
+      self.assertAllEqual(prediction_values, self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
@@ -363,8 +363,8 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
 
     expected_label_values = np.reshape(label_values, newshape=(2, 3))
     with self.cached_session():
-      self.assertAllEqual(expected_label_values, static_labels.eval())
-      self.assertAllEqual(prediction_values, static_predictions.eval())
+      self.assertAllEqual(expected_label_values, self.evaluate(static_labels))
+      self.assertAllEqual(prediction_values, self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
@@ -389,8 +389,9 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
 
     expected_prediction_values = np.reshape(prediction_values, newshape=(2, 3))
     with self.cached_session():
-      self.assertAllEqual(label_values, static_labels.eval())
-      self.assertAllEqual(expected_prediction_values, static_predictions.eval())
+      self.assertAllEqual(label_values, self.evaluate(static_labels))
+      self.assertAllEqual(expected_prediction_values,
+                          self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
@@ -416,8 +417,9 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
 
     expected_prediction_values = np.reshape(prediction_values, newshape=(2, 3))
     with self.cached_session():
-      self.assertAllEqual(label_values, static_labels.eval())
-      self.assertAllEqual(expected_prediction_values, static_predictions.eval())
+      self.assertAllEqual(label_values, self.evaluate(static_labels))
+      self.assertAllEqual(expected_prediction_values,
+                          self.evaluate(static_predictions))
       feed_dict = {
           labels_placeholder: label_values,
           predictions_placeholder: prediction_values
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 38b8c0c146..112e201c88 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -282,29 +282,29 @@ class AsTensorTest(test.TestCase):
     with self.cached_session():
       x = ops.convert_to_tensor(tensor_shape.TensorShape([]))
       self.assertEqual(dtypes_lib.int32, x.dtype)
-      self.assertAllEqual([], x.eval())
+      self.assertAllEqual([], self.evaluate(x))
 
       x = ops.convert_to_tensor(tensor_shape.TensorShape([1, 2, 3]))
       self.assertEqual(dtypes_lib.int32, x.dtype)
-      self.assertAllEqual([1, 2, 3], x.eval())
+      self.assertAllEqual([1, 2, 3], self.evaluate(x))
 
       x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31-1, 2, 3]))
       self.assertEqual(dtypes_lib.int32, x.dtype)
-      self.assertAllEqual([2**31-1, 2, 3], x.eval())
+      self.assertAllEqual([2**31 - 1, 2, 3], self.evaluate(x))
 
       x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31-1, 2, 3]),
                                 dtype=dtypes_lib.int32)
       self.assertEqual(dtypes_lib.int32, x.dtype)
-      self.assertAllEqual([2**31-1, 2, 3], x.eval())
+      self.assertAllEqual([2**31 - 1, 2, 3], self.evaluate(x))
 
       x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31, 2, 3]))
       self.assertEqual(dtypes_lib.int64, x.dtype)
-      self.assertAllEqual([2**31, 2, 3], x.eval())
+      self.assertAllEqual([2**31, 2, 3], self.evaluate(x))
 
       x = ops.convert_to_tensor(tensor_shape.TensorShape([2**31, 2, 3]),
                                 dtype=dtypes_lib.int64)
       self.assertEqual(dtypes_lib.int64, x.dtype)
-      self.assertAllEqual([2**31, 2, 3], x.eval())
+      self.assertAllEqual([2**31, 2, 3], self.evaluate(x))
 
       with self.assertRaisesRegexp(
           ValueError, "a dimension is too large .2147483648."):
@@ -314,11 +314,11 @@ class AsTensorTest(test.TestCase):
       x = ops.convert_to_tensor(
           tensor_shape.TensorShape([1, 2, 3]), dtype=dtypes_lib.int64)
       self.assertEqual(dtypes_lib.int64, x.dtype)
-      self.assertAllEqual([1, 2, 3], x.eval())
+      self.assertAllEqual([1, 2, 3], self.evaluate(x))
 
       x = array_ops.reshape(
           array_ops.zeros([6]), tensor_shape.TensorShape([2, 3]))
-      self.assertAllEqual([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], x.eval())
+      self.assertAllEqual([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], self.evaluate(x))
 
     with self.assertRaisesRegexp(ValueError, "partially known"):
       ops.convert_to_tensor(tensor_shape.TensorShape(None))
@@ -334,12 +334,12 @@ class AsTensorTest(test.TestCase):
     with self.cached_session():
       x = ops.convert_to_tensor(tensor_shape.TensorShape([1, 2, 3])[1])
       self.assertEqual(dtypes_lib.int32, x.dtype)
-      self.assertAllEqual(2, x.eval())
+      self.assertAllEqual(2, self.evaluate(x))
 
       x = ops.convert_to_tensor(
           tensor_shape.TensorShape([1, 2, 3])[1], dtype=dtypes_lib.int64)
       self.assertEqual(dtypes_lib.int64, x.dtype)
-      self.assertAllEqual(2, x.eval())
+      self.assertAllEqual(2, self.evaluate(x))
 
     shape = tensor_shape.TensorShape(None)
     if shape._v2_behavior:
@@ -372,7 +372,7 @@ class ZerosTest(test.TestCase):
     with self.cached_session():
       ret = array_ops.zeros(shape)
       self.assertEqual(shape, ret.get_shape())
-      return ret.eval()
+      return self.evaluate(ret)
 
   def testConst(self):
     self.assertTrue(
@@ -383,7 +383,7 @@ class ZerosTest(test.TestCase):
     self.assertEqual(0, self._Zeros(()))
     with self.cached_session():
       scalar = array_ops.zeros(constant_op.constant([], dtype=dtypes_lib.int32))
-      self.assertEqual(0, scalar.eval())
+      self.assertEqual(0, self.evaluate(scalar))
 
   def testDynamicSizes(self):
     np_ans = np.array([[0] * 3] * 2)
@@ -392,7 +392,7 @@ class ZerosTest(test.TestCase):
       d = array_ops.fill([2, 3], 12., name="fill")
       # Constructs a tensor of zeros of the same dimensions as "d".
       z = array_ops.zeros(array_ops.shape(d))
-      out = z.eval()
+      out = self.evaluate(z)
     self.assertAllEqual(np_ans, out)
     self.assertShapeEqual(np_ans, d)
     self.assertShapeEqual(np_ans, z)
@@ -420,13 +420,13 @@ class ZerosTest(test.TestCase):
         z = array_ops.zeros([2, 3], dtype=dtype)
         self.assertEqual(z.dtype, dtype)
         self.assertEqual([2, 3], z.get_shape())
-        z_value = z.eval()
+        z_value = self.evaluate(z)
         self.assertFalse(np.any(z_value))
         self.assertEqual((2, 3), z_value.shape)
         z = array_ops.zeros(array_ops.shape(d), dtype=dtype)
         self.assertEqual(z.dtype, dtype)
         self.assertEqual([2, 3], z.get_shape())
-        z_value = z.eval()
+        z_value = self.evaluate(z)
         self.assertFalse(np.any(z_value))
         self.assertEqual((2, 3), z_value.shape)
 
@@ -538,7 +538,7 @@ class OnesTest(test.TestCase):
     with self.cached_session():
       ret = array_ops.ones(shape)
       self.assertEqual(shape, ret.get_shape())
-      return ret.eval()
+      return self.evaluate(ret)
 
   def testConst(self):
     self.assertTrue(np.array_equal(self._Ones([2, 3]), np.array([[1] * 3] * 2)))
@@ -548,7 +548,7 @@ class OnesTest(test.TestCase):
     self.assertEqual(1, self._Ones(()))
     with self.cached_session():
       scalar = array_ops.ones(constant_op.constant([], dtype=dtypes_lib.int32))
-      self.assertEqual(1, scalar.eval())
+      self.assertEqual(1, self.evaluate(scalar))
 
   def testDynamicSizes(self):
     np_ans = np.array([[1] * 3] * 2)
@@ -557,7 +557,7 @@ class OnesTest(test.TestCase):
       d = array_ops.fill([2, 3], 12., name="fill")
       # Constructs a tensor of ones of the same dimensions as "d".
       z = array_ops.ones(array_ops.shape(d))
-      out = z.eval()
+      out = self.evaluate(z)
     self.assertAllEqual(np_ans, out)
     self.assertShapeEqual(np_ans, d)
     self.assertShapeEqual(np_ans, z)
@@ -617,7 +617,7 @@ class OnesLikeTest(test.TestCase):
         z_var = array_ops.ones_like(d)
         # Test that the type is correct
         self.assertEqual(z_var.dtype, dtype)
-        z_value = z_var.eval()
+        z_value = self.evaluate(z_var)
 
       # Test that the value is correct
       self.assertTrue(np.array_equal(z_value, np.array([[1] * 3] * 2)))
@@ -634,7 +634,7 @@ class FillTest(test.TestCase):
   def _compare(self, dims, val, np_ans, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.fill(dims, val, name="fill")
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
     self.assertAllClose(np_ans, out)
     # Fill does not set the shape.
     # self.assertShapeEqual(np_ans, tf_ans)
@@ -726,7 +726,7 @@ class PlaceholderTest(test.TestCase):
 
       with self.assertRaisesOpError(
           "must feed a value for placeholder tensor 'p' with dtype float"):
-        p_identity.eval()
+        self.evaluate(p_identity)
 
   def testShape(self):
     with self.cached_session():
@@ -739,7 +739,7 @@ class PlaceholderTest(test.TestCase):
       with self.assertRaisesOpError(
           "must feed a value for placeholder tensor 'p' with dtype float and "
           r"shape \[10,10\]"):
-        p_identity.eval()
+        self.evaluate(p_identity)
 
       with self.assertRaisesWithPredicateMatch(
           ValueError, lambda e: "Cannot feed value of shape" in str(e)):
@@ -783,7 +783,7 @@ class PlaceholderTest(test.TestCase):
       # Should trigger an operator error, not a shape error.
       with self.assertRaisesOpError(
           "must feed a value for placeholder tensor 'p' with dtype float"):
-        p_identity.eval()
+        self.evaluate(p_identity)
 
   def testControlDependency(self):
     with self.cached_session():
@@ -896,7 +896,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
     with self.session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([[2, 2], [2, 2]], shape=[2, 2])
       a = array_ops.identity(p)
-      self.assertAllEqual([[2, 2], [2, 2]], a.eval())
+      self.assertAllEqual([[2, 2], [2, 2]], self.evaluate(a))
       self.assertAllEqual(
           [[3, 3], [3, 3]], a.eval(feed_dict={p: [[3, 3], [3, 3]]}))
 
@@ -907,7 +907,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
     with self.session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([1, 2, 3], shape=[None])
       a = array_ops.identity(p)
-      self.assertAllEqual([1, 2, 3], a.eval())
+      self.assertAllEqual([1, 2, 3], self.evaluate(a))
       self.assertAllEqual([3, 37], a.eval(feed_dict={p: [3, 37]}))
 
       with self.assertRaises(ValueError):
@@ -917,7 +917,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
     with self.session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([17], shape=None)
       a = array_ops.identity(p)
-      self.assertAllEqual([17], a.eval())
+      self.assertAllEqual([17], self.evaluate(a))
       self.assertAllEqual([3, 37], a.eval(feed_dict={p: [3, 37]}))
       self.assertAllEqual(
           [[3, 3], [3, 3]], a.eval(feed_dict={p: [[3, 3], [3, 3]]}))
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index e5b45d2ed8..c0068c2681 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -139,7 +139,7 @@ class ControlFlowTest(test.TestCase):
 
       self.assertTrue(isinstance(v2, ops.Tensor))
       variables.global_variables_initializer().run()
-      self.assertEqual(9, v2.eval())
+      self.assertEqual(9, self.evaluate(v2))
 
   def testRefEnter(self):
     with self.cached_session():
@@ -152,7 +152,7 @@ class ControlFlowTest(test.TestCase):
       v2 = control_flow_ops.with_dependencies([op], enter_v)
       v3 = control_flow_ops.exit(v2)
       variables.global_variables_initializer().run()
-      self.assertEqual(9, v3.eval())
+      self.assertEqual(9, self.evaluate(v3))
 
   def testRefSwitch(self):
     with self.cached_session():
@@ -162,7 +162,7 @@ class ControlFlowTest(test.TestCase):
       v1 = control_flow_ops._SwitchRefOrTensor(v._ref(), p)  # pylint: disable=protected-access
       v2 = state_ops.assign(v1[1], 9)
       variables.global_variables_initializer().run()
-      self.assertEqual(9, v2.eval())
+      self.assertEqual(9, self.evaluate(v2))
 
   def testEnterMulExit(self):
     with self.cached_session():
@@ -173,7 +173,7 @@ class ControlFlowTest(test.TestCase):
       mul_op = math_ops.multiply(enter_data, enter_five)
       exit_op = control_flow_ops.exit(mul_op)
 
-      result = exit_op.eval()
+      result = self.evaluate(exit_op)
     self.assertAllEqual(np.array([x * 5 for x in [1, 2, 3, 4, 5, 6]]), result)
 
   def testEnterShapePropagation(self):
@@ -214,7 +214,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaisesWithPredicateMatch(
           errors_impl.InvalidArgumentError,
           lambda e: "Retval[0] does not have value" in str(e)):
-        dead_branch.eval()
+        self.evaluate(dead_branch)
 
   def testSwitchMergeLess(self):
     with self.cached_session():
@@ -225,7 +225,7 @@ class ControlFlowTest(test.TestCase):
       switch_op = control_flow_ops.switch(data, less_op)
       merge_op = control_flow_ops.merge(switch_op)[0]
 
-      result = merge_op.eval()
+      result = self.evaluate(merge_op)
     self.assertAllEqual(np.arange(1, 7), result)
 
   def testSwitchMergeAddIdentity(self):
@@ -238,7 +238,7 @@ class ControlFlowTest(test.TestCase):
       id_op = array_ops.identity(switch_op[1])
       merge_op = control_flow_ops.merge([add_op, id_op])[0]
 
-      result = merge_op.eval()
+      result = self.evaluate(merge_op)
     self.assertAllEqual(np.array([x + 1 for x in [1, 2, 3, 4, 5, 6]]), result)
 
   def testSwitchMergeAddMul(self):
@@ -252,7 +252,7 @@ class ControlFlowTest(test.TestCase):
       mul_op = math_ops.multiply(switch_op[1], five)
       merge_op = control_flow_ops.merge([add_op, mul_op])[0]
 
-      result = merge_op.eval()
+      result = self.evaluate(merge_op)
     self.assertAllEqual(np.array([x * 5 for x in [1, 2, 3, 4, 5, 6]]), result)
 
   def testLoop_false(self):
@@ -269,7 +269,7 @@ class ControlFlowTest(test.TestCase):
       next_n = control_flow_ops.next_iteration(switch_n[0])
       merge_n.op._update_input(1, next_n)
 
-      result = exit_n.eval()
+      result = self.evaluate(exit_n)
     self.assertAllEqual(10, result)
 
   def testLoop_1(self):
@@ -295,7 +295,7 @@ class ControlFlowTest(test.TestCase):
       merge_i.op._update_input(1, next_i)
 
       exit_i = control_flow_ops.exit(switch_i[0])
-      result = exit_i.eval()
+      result = self.evaluate(exit_i)
     self.assertAllEqual(10, result)
 
   def testLoop_2(self):
@@ -321,7 +321,7 @@ class ControlFlowTest(test.TestCase):
       merge_i.op._update_input(1, next_i)
 
       exit_i = control_flow_ops.exit(switch_i[0])
-      result = exit_i.eval()
+      result = self.evaluate(exit_i)
     self.assertAllEqual(10, result)
 
   def testDifferentFrame(self):
@@ -476,7 +476,7 @@ class ControlFlowTest(test.TestCase):
       fn2 = lambda: math_ops.subtract(x, 1)
       r = control_flow_ops.cond(pred, fn1, fn2)
 
-      result = r.eval()
+      result = self.evaluate(r)
     self.assertAllEqual(11, result)
 
   def testCond_1(self):
@@ -492,7 +492,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(
           math_ops.less(1, 0), lambda: math_ops.add(x, 1),
           lambda: math_ops.subtract(x, 1))
-      result = r.eval()
+      result = self.evaluate(r)
     self.assertAllEqual(9, result)
 
   def testCond_3(self):
@@ -505,7 +505,7 @@ class ControlFlowTest(test.TestCase):
       fn3 = lambda: math_ops.add(control_flow_ops.cond(pred, fn1, fn2), 1)
       r = control_flow_ops.cond(pred, fn3, fn2)
 
-      result = r.eval()
+      result = self.evaluate(r)
     self.assertAllEqual(12, result)
 
   @test_util.run_in_graph_and_eager_modes
@@ -532,9 +532,9 @@ class ControlFlowTest(test.TestCase):
         result = f().eval()
         self.assertEqual(True, result)
         # Only second cond result was fetched, so v1 assign shouldn't run.
-        self.assertEqual(7, v1.eval())
-        self.assertEqual(2, v2.eval())
-        self.assertEqual(7, v3.eval())
+        self.assertEqual(7, self.evaluate(v1))
+        self.assertEqual(2, self.evaluate(v2))
+        self.assertEqual(7, self.evaluate(v3))
 
     result = f_defun()
     self.assertEqual(True, self.evaluate(result))
@@ -555,7 +555,7 @@ class ControlFlowTest(test.TestCase):
 
       for i in range(10):
         alive, count = body(i)
-      self.assertAllEqual(4, count.eval())
+      self.assertAllEqual(4, self.evaluate(count))
 
   def testCond_6(self):
     with self.cached_session():
@@ -568,7 +568,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
 
       variables.global_variables_initializer().run()
-      result = r.eval()
+      result = self.evaluate(r)
       self.assertAllEqual(np.array([7]), result)
 
   def testCond_7(self):
@@ -677,7 +677,7 @@ class ControlFlowTest(test.TestCase):
       true_fn = lambda: x
       false_fn = lambda: constant_op.constant([2.0])
       r = control_flow_ops.cond(constant_op.constant(False), true_fn, false_fn)
-      self.assertAllEqual([2.0], r.eval())
+      self.assertAllEqual([2.0], self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/79881896 (control deps)")
   def testCondWithControl(self):
@@ -693,7 +693,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(
           constant_op.constant(True), true_branch,
           lambda: constant_op.constant(1))
-      self.assertEqual(5, r.eval())
+      self.assertEqual(5, self.evaluate(r))
 
   def testUninitializedRefIdentity(self):
     with self.cached_session() as sess:
@@ -758,7 +758,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
 
       grad = gradients_impl.gradients(r, [x])[0]
-      self.assertAllEqual(1.0, grad.eval())
+      self.assertAllEqual(1.0, self.evaluate(grad))
 
   def testCondGrad_2(self):
     with self.cached_session():
@@ -827,13 +827,13 @@ class ControlFlowTest(test.TestCase):
           constant_op.constant(True), lambda: x,
           lambda: control_flow_ops.cond(x < 1., lambda: x, lambda: x))
       result = gradients_impl.gradients(y, x)[0]
-      self.assertEqual(1.0, result.eval())
+      self.assertEqual(1.0, self.evaluate(result))
 
       z = control_flow_ops.cond(
           constant_op.constant(False), lambda: x,
           lambda: control_flow_ops.cond(x < 1., lambda: x, lambda: x))
       result = gradients_impl.gradients(z, x)[0]
-      self.assertEqual(1.0, result.eval())
+      self.assertEqual(1.0, self.evaluate(result))
 
   @test_util.disable_control_flow_v2("b/113327884")
   def testCondGrad_Gather(self):
@@ -982,7 +982,7 @@ class ControlFlowTest(test.TestCase):
       c = lambda x: math_ops.less(x, 10000)
       b = lambda x: math_ops.add(x, 1)
       r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20)
-      self.assertEqual(10000, r.eval())
+      self.assertEqual(10000, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/79881896 (control deps)")
   def testWhileExternalControlDependencies(self):
@@ -1013,7 +1013,7 @@ class ControlFlowTest(test.TestCase):
 
       result = control_flow_ops.while_loop(cond=lambda i: i < 5,
                                            body=body_fn, loop_vars=[0])
-      result.eval()
+      self.evaluate(result)
       self.assertAllEqual(v.eval(), 1.0)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
@@ -1045,19 +1045,19 @@ class ControlFlowTest(test.TestCase):
     with self.cached_session():
       s = constant_op.constant(0)
       r = isum(s)
-      self.assertAllEqual(45, r.eval())
+      self.assertAllEqual(45, self.evaluate(r))
 
   def testWhileWithMaximumIterations(self):
     with self.cached_session():
       s = constant_op.constant([1, 2, 3, 4, 5])
       r = isum(s, maximum_iterations=3)
-      self.assertAllEqual([1 + 3, 2 + 3, 3 + 3, 4 + 3, 5 + 3], r.eval())
+      self.assertAllEqual([1 + 3, 2 + 3, 3 + 3, 4 + 3, 5 + 3], self.evaluate(r))
 
   def testWhileWithMaximumIterationsAndSingleArgument(self):
     with self.cached_session():
       r = control_flow_ops.while_loop(
           lambda i: i < 3, lambda i: i + 1, [0], maximum_iterations=1)
-      self.assertEqual(1, r.eval())
+      self.assertEqual(1, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/115776323 (max_iters)")
   def testSingleNestedMaximumIterationsWhileLoopGradientInXLAContext(self):
@@ -1350,7 +1350,7 @@ class ControlFlowTest(test.TestCase):
       c = lambda x: math_ops.less(x, 10.0)
       b = lambda x: math_ops.add(x, 1.0)
       r = control_flow_ops.while_loop(c, b, [n])
-      self.assertAllClose(10.0, r.eval())
+      self.assertAllClose(10.0, self.evaluate(r))
 
   def testWhile_Gpu_1(self):
     self._testWhile_Gpu_1(use_gpu=False)
@@ -1366,7 +1366,7 @@ class ControlFlowTest(test.TestCase):
           return math_ops.add(x, 1.0)
 
       r = control_flow_ops.while_loop(c, b, [n])
-      self.assertAllClose(10.0, r.eval())
+      self.assertAllClose(10.0, self.evaluate(r))
 
   def testWhile_Gpu_2(self):
     self._testWhile_Gpu_2(use_gpu=False)
@@ -1387,7 +1387,7 @@ class ControlFlowTest(test.TestCase):
           c, _b, [i, m],
           [i.get_shape(), tensor_shape.unknown_shape()])
       r = r[1] * array_ops.ones([8, 8])
-      self.assertAllEqual(np.ones((8, 8)), r.eval())
+      self.assertAllEqual(np.ones((8, 8)), self.evaluate(r))
 
   def testWhileWithNonTensorInput_Scalar(self):
     with self.cached_session():
@@ -1395,7 +1395,7 @@ class ControlFlowTest(test.TestCase):
       c = lambda x: x < 10000
       b = lambda x: x + 1
       r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20)
-      self.assertEqual(10000, r.eval())
+      self.assertEqual(10000, self.evaluate(r))
 
   def testWhileWithNonTensorInput_Vector(self):
     with self.cached_session():
@@ -1403,7 +1403,7 @@ class ControlFlowTest(test.TestCase):
       c = lambda x: x[0] < 10000
       b = lambda x: array_ops.stack([x[0] + 1])
       r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20)
-      self.assertEqual([10000], r.eval())
+      self.assertEqual([10000], self.evaluate(r))
 
   def testWhileShapeInference(self):
     with self.cached_session():
@@ -1515,7 +1515,7 @@ class ControlFlowTest(test.TestCase):
       c = lambda x: math_ops.less(x, 200)
       b = lambda x: math_ops.add(x, cpu_sum(n))
       r = control_flow_ops.while_loop(c, b, [n])
-      self.assertEqual(225, r.eval())
+      self.assertEqual(225, self.evaluate(r))
 
   def testNestedWhile_1(self):
     self._testNestedWhile_1(use_gpu=False)
@@ -1547,7 +1547,7 @@ class ControlFlowTest(test.TestCase):
 
       r = control_flow_ops.while_loop(
           outer_c, outer_b, [s0], parallel_iterations=1)
-      self.assertEqual(1048576.0, r.eval())
+      self.assertEqual(1048576.0, self.evaluate(r))
 
   def testNestedWhile_2(self):
     self._testNestedWhile_2(use_gpu=False)
@@ -1581,7 +1581,7 @@ class ControlFlowTest(test.TestCase):
 
       res = control_flow_ops.while_loop(
           condition, body, [r], parallel_iterations=1)
-      self.assertAllEqual(12, res.eval())
+      self.assertAllEqual(12, self.evaluate(res))
 
   def testWhileWithControl_3(self):
     with self.cached_session() as sess:
@@ -1650,8 +1650,8 @@ class ControlFlowTest(test.TestCase):
 
       r = control_flow_ops.while_loop(loop_condition, loop_body, (i0,))
       variables.global_variables_initializer().run()
-      self.assertEqual(4, r.eval())
-      self.assertAllClose(65536.0, v.eval())
+      self.assertEqual(4, self.evaluate(r))
+      self.assertAllClose(65536.0, self.evaluate(v))
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
   def testWhileCondExitControl(self):
@@ -1675,8 +1675,8 @@ class ControlFlowTest(test.TestCase):
           constant_op.constant(False), lambda: constant_op.constant(1.0),
           false_branch)
       variables.global_variables_initializer().run()
-      self.assertEqual(6.0, r.eval())
-      self.assertEqual(99, v.eval())
+      self.assertEqual(6.0, self.evaluate(r))
+      self.assertEqual(99, self.evaluate(v))
 
   def testCondWhile_1(self):
 
@@ -1687,7 +1687,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(
           math_ops.less(0, 1), lambda: control_flow_ops.while_loop(c, b, [n]),
           lambda: n)
-      self.assertAllEqual(10, r.eval())
+      self.assertAllEqual(10, self.evaluate(r))
 
   def testCondWhile_2(self):
 
@@ -1698,7 +1698,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(
           math_ops.less(1, 0), lambda: math_ops.add(n, 1),
           lambda: control_flow_ops.while_loop(c, b, [n]))
-      self.assertAllEqual(10, r.eval())
+      self.assertAllEqual(10, self.evaluate(r))
 
   def _testCondWhile_3(self, use_gpu):
     with self.cached_session(use_gpu=use_gpu) as sess:
@@ -1741,7 +1741,7 @@ class ControlFlowTest(test.TestCase):
           lambda: math_ops.add(x, one), lambda: math_ops.subtract(x, one))
       # pylint: enable=undefined-variable
       r = control_flow_ops.while_loop(c, b, [i])
-      self.assertAllEqual(10, r.eval())
+      self.assertAllEqual(10, self.evaluate(r))
 
   def testWhileCond_2(self):
 
@@ -1750,7 +1750,7 @@ class ControlFlowTest(test.TestCase):
       c = lambda x: math_ops.less(x, 10)
       b = lambda x: control_flow_ops.cond(constant_op.constant(True), lambda: math_ops.add(x, 1), lambda: n)
       r = control_flow_ops.while_loop(c, b, [n])
-      self.assertAllEqual(10, r.eval())
+      self.assertAllEqual(10, self.evaluate(r))
 
   def testWhileCond_3(self):
 
@@ -1764,7 +1764,7 @@ class ControlFlowTest(test.TestCase):
                                           lambda: math_ops.subtract(x, 1))
       # pylint: enable=undefined-variable
       r = control_flow_ops.while_loop(c, b, [n])
-      self.assertAllEqual(10, r.eval())
+      self.assertAllEqual(10, self.evaluate(r))
 
   def testWhileCondGradMultiDevice(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 2},
@@ -1815,8 +1815,8 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(
           loop_iterator, loop_body, [n], parallel_iterations=1)
       variables.global_variables_initializer().run()
-      self.assertEqual(3, r.eval())
-      result = select.eval()
+      self.assertEqual(3, self.evaluate(r))
+      result = self.evaluate(select)
       self.assertAllClose(np.array([10.0, 10.0, 10.0]), result)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
@@ -1840,10 +1840,10 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(
           loop_iterator, loop_body, [n], parallel_iterations=1)
       variables.global_variables_initializer().run()
-      self.assertEqual(3, r.eval())
-      result1 = select1.eval()
+      self.assertEqual(3, self.evaluate(r))
+      result1 = self.evaluate(select1)
       self.assertAllClose(np.array([10.0, 10.0, 10.0]), result1)
-      result2 = select2.eval()
+      result2 = self.evaluate(select2)
       self.assertAllClose(np.array([10.0, 10.0, 10.0]), result2)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
@@ -1892,9 +1892,9 @@ class ControlFlowTest(test.TestCase):
       lpa = control_flow_ops.while_loop(
           pred, loop_body, [c], parallel_iterations=1)
 
-      self.assertEqual(0, var_b.eval())
-      lpa.eval()  # Run the loop
-      self.assertEqual(10, var_b.eval())
+      self.assertEqual(0, self.evaluate(var_b))
+      self.evaluate(lpa)  # Run the loop
+      self.assertEqual(10, self.evaluate(var_b))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileUpdateVariable_5(self):
@@ -1921,10 +1921,10 @@ class ControlFlowTest(test.TestCase):
       lpa = control_flow_ops.while_loop(
           pred, loop_body, [var_b], parallel_iterations=1, name="loop")
 
-      self.assertEqual(0, var_b.eval())
-      lpa.eval()  # Run the loop
-      self.assertEqual(10, var_a.eval())
-      self.assertEqual(10, var_b.eval())
+      self.assertEqual(0, self.evaluate(var_b))
+      self.evaluate(lpa)  # Run the loop
+      self.assertEqual(10, self.evaluate(var_a))
+      self.assertEqual(10, self.evaluate(var_b))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileUpdateVariable_6(self):
@@ -1951,10 +1951,10 @@ class ControlFlowTest(test.TestCase):
       lpa = control_flow_ops.while_loop(
           pred, loop_body, [c], parallel_iterations=1, name="loop")
 
-      self.assertEqual(0, var_b.eval())
-      lpa.eval()  # Run the loop
-      self.assertEqual(55, var_b.eval())
-      self.assertEqual(10, var_a.eval())
+      self.assertEqual(0, self.evaluate(var_b))
+      self.evaluate(lpa)  # Run the loop
+      self.assertEqual(55, self.evaluate(var_b))
+      self.assertEqual(10, self.evaluate(var_a))
 
   def testWhileQueue_1(self):
     with self.cached_session():
@@ -1970,7 +1970,7 @@ class ControlFlowTest(test.TestCase):
         return ni
 
       r = control_flow_ops.while_loop(c, b, [i], parallel_iterations=1)
-      self.assertEqual([10], r.eval())
+      self.assertEqual([10], self.evaluate(r))
       for i in xrange(10):
         self.assertEqual([i], q.dequeue().eval())
 
@@ -2006,7 +2006,7 @@ class ControlFlowTest(test.TestCase):
           b1, [r, x],
           [r.get_shape(), tensor_shape.unknown_shape()],
           parallel_iterations=1)
-      self.assertEqual(45, rx.eval())
+      self.assertEqual(45, self.evaluate(rx))
 
   def _testWhileGrad_ColocateGradients(self, colocate):
     gpu_dev_name = test.gpu_device_name() if test.is_gpu_available(
@@ -2057,7 +2057,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(math_ops.less(1, 2), lambda: r, lambda: v)
 
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(1024.0, r.eval())
+      self.assertAllClose(1024.0, self.evaluate(r))
 
   def testWhileGrad_Shape(self):
     with self.cached_session():
@@ -2097,7 +2097,7 @@ class ControlFlowTest(test.TestCase):
       r = math_ops.multiply(r, r)
 
       r = gradients_impl.gradients(r, v)[0]
-      self.assertEqual(524288.0, r.eval())
+      self.assertEqual(524288.0, self.evaluate(r))
 
   def testWhileGrad_LoopAdd(self):
     with self.cached_session():
@@ -2108,7 +2108,7 @@ class ControlFlowTest(test.TestCase):
       r = math_ops.add(r, r)
 
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(2048.0, r.eval())
+      self.assertAllClose(2048.0, self.evaluate(r))
 
   def _testWhileGrad_Mul(self, use_gpu, p_iters):
     with self.cached_session(use_gpu=use_gpu) as sess:
@@ -2151,7 +2151,7 @@ class ControlFlowTest(test.TestCase):
 
       r = control_flow_ops.while_loop(c, b, [v])
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(512.0, r.eval())
+      self.assertAllClose(512.0, self.evaluate(r))
 
   def testNestedWhileCondWhileGrad(self):
     if control_flow_ops.ENABLE_WHILE_V2 and test_util.is_gpu_available():
@@ -2443,10 +2443,11 @@ class ControlFlowTest(test.TestCase):
       with ops.control_dependencies([x_f]):
         y_f_d = array_ops.identity(y_f, name="y_f_d")
 
-      self.assertAllClose(2.0, y_f_d.eval())  # y_f_d = 1.0 + 1.0
+      self.assertAllClose(2.0, self.evaluate(y_f_d))  # y_f_d = 1.0 + 1.0
       g = gradients_impl.gradients([y_f_d], [x])[0]
       self.assertTrue(g is not None)
-      self.assertAllClose(1.0, g.eval())  # y_f_d = x + 1.0, dy_f_d/dx = 1.0
+      self.assertAllClose(1.0,
+                          self.evaluate(g))  # y_f_d = x + 1.0, dy_f_d/dx = 1.0
 
   def _testNestedWhileGrad_Simple(self, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
@@ -2462,7 +2463,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [v])
 
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(8.0, r.eval())
+      self.assertAllClose(8.0, self.evaluate(r))
 
   def testNestedWhileGrad_Simple(self):
     self._testNestedWhileGrad_Simple(use_gpu=False)
@@ -2489,7 +2490,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [v])
 
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(256.0, r.eval())
+      self.assertAllClose(256.0, self.evaluate(r))
 
   def testNestedWhileGrad_ParallelInner(self):
     with self.cached_session():
@@ -2512,7 +2513,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [v])
 
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(512.0, r.eval())
+      self.assertAllClose(512.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("unsupported: resource creation in body. "
                                      "Enable with new TAs b/117675481")
@@ -2536,7 +2537,7 @@ class ControlFlowTest(test.TestCase):
       train_op = optimizer.minimize(math_ops.reduce_mean(math_ops.square(res)))
       sess.run(variables.global_variables_initializer())
       sess.run(train_op)
-      self.assertAllClose(2.999, var.eval())
+      self.assertAllClose(2.999, self.evaluate(var))
 
   def _testWhileCondGrad_Simple(self, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
@@ -2552,7 +2553,7 @@ class ControlFlowTest(test.TestCase):
       # pylint: enable=undefined-variable
       r = control_flow_ops.while_loop(c, b, [v])
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllClose(1024.0, r.eval())
+      self.assertAllClose(1024.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/117519152")
   def testWhileCondGrad_Simple(self):
@@ -2649,7 +2650,7 @@ class ControlFlowTest(test.TestCase):
 
       _, r = control_flow_ops.while_loop(c, b, [i, x])
       r = gradients_impl.gradients(r.values, values)[0]
-      self.assertAllClose(np.array([1024.0, 1024.0]), r.eval())
+      self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)")
   def testWhileGrad_SparseTensor(self):
@@ -2672,7 +2673,7 @@ class ControlFlowTest(test.TestCase):
 
       _, r = control_flow_ops.while_loop(c, b, [i, x])
       r = gradients_impl.gradients(r.values, values)[0]
-      self.assertAllClose(np.array([1024.0, 1024.0]), r.eval())
+      self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   def testCallGradInLoop(self):
@@ -2730,9 +2731,9 @@ class ControlFlowTest(test.TestCase):
       rx, ry = control_flow_ops.while_loop(c, b, [x, y])
 
       r = gradients_impl.gradients(rx, y)[0]
-      self.assertEqual(136.0, r.eval())
+      self.assertEqual(136.0, self.evaluate(r))
       r = gradients_impl.gradients(ry, y)[0]
-      self.assertEqual(32.0, r.eval())
+      self.assertEqual(32.0, self.evaluate(r))
 
       r = gradients_impl.gradients(array_ops.stop_gradient(rx), y)[0]
       self.assertEqual(r, None)
@@ -2750,13 +2751,13 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(r, None)
 
       r = gradients_impl.gradients(math_ops.add(rx, ry), y)[0]
-      self.assertEqual(168.0, r.eval())
+      self.assertEqual(168.0, self.evaluate(r))
       r = gradients_impl.gradients(
           math_ops.add(rx, array_ops.stop_gradient(ry)), y)[0]
-      self.assertEqual(136.0, r.eval())
+      self.assertEqual(136.0, self.evaluate(r))
       r = gradients_impl.gradients(
           math_ops.add(array_ops.stop_gradient(rx), ry), y)[0]
-      self.assertEqual(32.0, r.eval())
+      self.assertEqual(32.0, self.evaluate(r))
 
   def testWhileGrad_StopGradInside(self):
     with self.cached_session():
@@ -2773,9 +2774,9 @@ class ControlFlowTest(test.TestCase):
       rx, _ = control_flow_ops.while_loop(c, b, [x, y])
 
       r = gradients_impl.gradients(rx, y)[0]
-      self.assertAllClose(0.0, r.eval())
+      self.assertAllClose(0.0, self.evaluate(r))
       r = gradients_impl.gradients(rx, x)[0]
-      self.assertAllClose(156.0, r.eval())
+      self.assertAllClose(156.0, self.evaluate(r))
 
   def testWhileGrad_StopGradInsideNoShape(self):
     with self.cached_session() as sess:
@@ -2829,7 +2830,7 @@ class ControlFlowTest(test.TestCase):
       r = math_ops.add(math_ops.square(y), rx)
       r = math_ops.add(r, rg)
       r = gradients_impl.gradients(r, y)[0]
-      self.assertEqual(388.0, r.eval())
+      self.assertEqual(388.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileGradientWithNontrainablePath1(self):
@@ -2915,7 +2916,7 @@ class ControlFlowTest(test.TestCase):
       z = math_ops.add(r, array_ops.stop_gradient(math_ops.reduce_sum(grads)))
       result = gradients_impl.gradients(z, vars_)[0]
       variables.global_variables_initializer().run()
-      self.assertEqual(5.0, result.eval())
+      self.assertEqual(5.0, self.evaluate(result))
 
   def testOneValueCond(self):
 
@@ -2976,7 +2977,7 @@ class ControlFlowTest(test.TestCase):
       r4 = control_flow_ops.case(
           [(x < y, f1), (x < y, f2)], default=f3, exclusive=True)
       with self.assertRaisesOpError("Input error:"):
-        r4.eval()
+        self.evaluate(r4)
 
       # Check that the default is called if none of the others are
       r5 = control_flow_ops.case({x > y: f1}, default=f3)
@@ -3023,17 +3024,17 @@ class ControlFlowTest(test.TestCase):
 
       variables.global_variables_initializer().run()
       self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3)
-      self.assertEqual(2, r2.eval())
+      self.assertEqual(2, self.evaluate(r2))
       self.assertAllEqual(sess.run([v0, v1, v2]), [-1, -1, 2])
 
       variables.global_variables_initializer().run()
       self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3)
-      self.assertEqual(1, r1.eval())
+      self.assertEqual(1, self.evaluate(r1))
       self.assertAllEqual(sess.run([v0, v1, v2]), [-1, 1, -1])
 
       variables.global_variables_initializer().run()
       self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3)
-      self.assertEqual(0, r0.eval())
+      self.assertEqual(0, self.evaluate(r0))
       self.assertAllEqual(sess.run([v0, v1, v2]), [0, -1, -1])
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
@@ -3055,15 +3056,15 @@ class ControlFlowTest(test.TestCase):
       self.assertTrue(isinstance(i, ops.Tensor))
       variables.global_variables_initializer().run()
 
-      self.assertEqual(0, v.eval())
+      self.assertEqual(0, self.evaluate(v))
 
       # True case: c = 2 is >= 1, v is set to 1.
       self.assertEqual(1, i.eval(feed_dict={c.name: 2}))
-      self.assertEqual(1, v.eval())
+      self.assertEqual(1, self.evaluate(v))
 
       # False case: c = 0 is not >= 1, v is set to 2.
       self.assertEqual(2, i.eval(feed_dict={c.name: 0}))
-      self.assertEqual(2, v.eval())
+      self.assertEqual(2, self.evaluate(v))
 
   def testWithOpsDependencies(self):
     with self.cached_session() as sess:
@@ -3105,14 +3106,14 @@ class ControlFlowTest(test.TestCase):
 
       # Fetching v directly will result in an uninitialized error
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        v.eval()
+        self.evaluate(v)
 
       # Get the value of 'c2_with_c1_dep', which should cause 'v'
       # to be initialized.
-      self.assertAllEqual(20, c2_with_c1_dep.eval())
+      self.assertAllEqual(20, self.evaluate(c2_with_c1_dep))
 
       # Ensure that 'v' is initialized
-      self.assertAllClose(0.0, v.eval())
+      self.assertAllClose(0.0, self.evaluate(v))
 
   def testWithIndexedSlicesDependencies(self):
     with self.cached_session():
@@ -3127,13 +3128,15 @@ class ControlFlowTest(test.TestCase):
 
       # Fetching gather_v_at_1 will result in an uninitialized error
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        gather_v_at_1.eval()
+        self.evaluate(gather_v_at_1)
 
       # Getting gather_v_at_1_after_init will work, and initialize v.
-      self.assertAllEqual([[10.0, 11.0]], gather_v_at_1_after_init.eval())
+      self.assertAllEqual([[10.0, 11.0]],
+                          self.evaluate(gather_v_at_1_after_init))
 
       # Double check that 'v' is initialized
-      self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]], v.eval())
+      self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]],
+                          self.evaluate(v))
 
   def testDependenciesDevice(self):
     with ops.Graph().as_default():
@@ -3167,7 +3170,7 @@ class ControlFlowTest(test.TestCase):
       init = control_flow_ops.group(v1.initializer, v2.initializer)
       # Fetching v1 directly will result in an uninitialized error
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        v1.eval()
+        self.evaluate(v1)
 
       # Runs "init" before fetching v1 and v2.
       init.run()
@@ -3495,20 +3498,20 @@ class TupleTest(test.TestCase):
 
         # v1 is not initialized.
         with self.assertRaisesOpError("Attempting to use uninitialized value"):
-          v1.eval()
+          self.evaluate(v1)
 
         # v2 is not initialized.
         with self.assertRaisesOpError("Attempting to use uninitialized value"):
-          v2.eval()
+          self.evaluate(v2)
 
         if v1_first:
           # Getting t1 initializes v2.
-          self.assertAllClose([3.0], t1.eval())
-          self.assertAllClose([10.0], v2.eval())
+          self.assertAllClose([3.0], self.evaluate(t1))
+          self.assertAllClose([10.0], self.evaluate(v2))
         else:
           # Getting t2 initializes v1.
-          self.assertAllClose([30.0], t2.eval())
-          self.assertAllClose([1.0], v1.eval())
+          self.assertAllClose([30.0], self.evaluate(t2))
+          self.assertAllClose([1.0], self.evaluate(v1))
 
   def testIndexedSlices(self):
     for v1_first in [True, False]:
@@ -3533,22 +3536,22 @@ class TupleTest(test.TestCase):
 
         # v1 is not initialized.
         with self.assertRaisesOpError("Attempting to use uninitialized value"):
-          v1.eval()
+          self.evaluate(v1)
 
         # v2 is not initialized.
         with self.assertRaisesOpError("Attempting to use uninitialized value"):
-          v2.eval()
+          self.evaluate(v2)
 
         if v1_first:
           # Getting g1 initializes v2.
-          self.assertAllClose([[10.0, 11.0]], g1.eval())
+          self.assertAllClose([[10.0, 11.0]], self.evaluate(g1))
           self.assertAllClose([[0.1, 1.1], [10.1, 11.1], [20.1, 21.1]],
-                              v2.eval())
+                              self.evaluate(v2))
         else:
           # Getting g2 initializes v1.
-          self.assertAllClose([[10.1, 11.1]], g2.eval())
+          self.assertAllClose([[10.1, 11.1]], self.evaluate(g2))
           self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]],
-                              v1.eval())
+                              self.evaluate(v1))
 
   def testAcceptTensorsAsControlInputs(self):
     with self.cached_session():
@@ -3558,9 +3561,9 @@ class TupleTest(test.TestCase):
           [constant_op.constant(0)], control_inputs=[assign])
 
       # Should trigger the assign.
-      t.eval()
+      self.evaluate(t)
 
-      self.assertEquals(1, var.eval())
+      self.assertEquals(1, self.evaluate(var))
 
 
 class AssertTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/conv1d_test.py b/tensorflow/python/kernel_tests/conv1d_test.py
index 8540875d75..e8463323df 100644
--- a/tensorflow/python/kernel_tests/conv1d_test.py
+++ b/tensorflow/python/kernel_tests/conv1d_test.py
@@ -43,7 +43,7 @@ class Conv1DTest(test.TestCase):
         with self.cached_session(use_gpu=test.is_gpu_available()):
           c = nn_ops.conv1d(x, filters, stride, padding="VALID")
           reduced = array_ops.squeeze(c)
-          output = reduced.eval()
+          output = self.evaluate(reduced)
           if stride == 1:
             self.assertEqual(len(output), 3)
             self.assertAllClose(output,
@@ -69,7 +69,7 @@ class Conv1DTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv1d_transpose(
           x, f, y_shape, stride=stride, padding="VALID")
-      value = output.eval()
+      value = self.evaluate(output)
 
       cache_values = np.zeros(y_shape, dtype=np.float32)
 
diff --git a/tensorflow/python/kernel_tests/conv2d_transpose_test.py b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
index 95a17cfb44..d9aa4ab967 100644
--- a/tensorflow/python/kernel_tests/conv2d_transpose_test.py
+++ b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
@@ -52,7 +52,7 @@ class Conv2DTransposeTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv2d_transpose(
           x, f, y_shape, strides=strides, padding="SAME")
-      value = output.eval()
+      value = self.evaluate(output)
 
       # We count the number of cells being added at the locations in the output.
       # At the center, #cells=kernel_height * kernel_width
@@ -90,7 +90,7 @@ class Conv2DTransposeTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv2d_transpose(
           x, f, y_shape, strides=strides, padding="SAME")
-      value = output.eval()
+      value = self.evaluate(output)
 
       for n in xrange(x_shape[0]):
         for k in xrange(f_shape[2]):
@@ -123,7 +123,7 @@ class Conv2DTransposeTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv2d_transpose(
           x, f, y_shape, strides=strides, padding="VALID")
-      value = output.eval()
+      value = self.evaluate(output)
 
       cache_values = np.zeros(y_shape, dtype=np.float32)
 
@@ -194,7 +194,7 @@ class Conv2DTransposeTest(test.TestCase):
         output = nn_ops.conv2d_transpose(
             x, f, y_shape, strides=strides, padding="SAME", data_format="NCHW")
 
-        value = output.eval()
+        value = self.evaluate(output)
         for n in xrange(x_shape[0]):
           for k in xrange(f_shape[2]):
             for w in xrange(y_shape[3]):
@@ -229,7 +229,7 @@ class Conv2DTransposeTest(test.TestCase):
         output = nn_ops.conv2d_transpose(
             x, f, y_shape, strides=strides, padding="SAME", data_format="NCHW")
 
-        value = output.eval()
+        value = self.evaluate(output)
         for n in xrange(x_shape[0]):
           for k in xrange(f_shape[2]):
             for w in xrange(y_shape[3]):
@@ -264,7 +264,7 @@ class Conv2DTransposeTest(test.TestCase):
         output = nn_ops.conv2d_transpose(
             x, f, y_shape, strides=strides, padding="VALID", data_format="NCHW")
 
-        value = output.eval()
+        value = self.evaluate(output)
         cache_values = np.zeros(y_shape, dtype=np.float32)
         # The amount of padding added
         pad = 1
diff --git a/tensorflow/python/kernel_tests/conv3d_transpose_test.py b/tensorflow/python/kernel_tests/conv3d_transpose_test.py
index 2527b83769..d4e7ec14da 100644
--- a/tensorflow/python/kernel_tests/conv3d_transpose_test.py
+++ b/tensorflow/python/kernel_tests/conv3d_transpose_test.py
@@ -48,7 +48,7 @@ class Conv3DTransposeTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv3d_transpose(
           x, f, y_shape, strides=strides, padding="SAME")
-      value = output.eval()
+      value = self.evaluate(output)
 
       # We count the number of cells being added at the locations in the output.
       # At the center, #cells = kernel_depth * kernel_height * kernel_width
@@ -98,7 +98,7 @@ class Conv3DTransposeTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv3d_transpose(
           x, f, y_shape, strides=strides, padding="SAME")
-      value = output.eval()
+      value = self.evaluate(output)
 
       for n in xrange(x_shape[0]):
         for k in xrange(f_shape[3]):
@@ -146,7 +146,7 @@ class Conv3DTransposeTest(test.TestCase):
         output = nn_ops.conv3d_transpose(
             x_value, f_value, constant_op.constant(y_shape, dtype=dtype),
             strides=strides, padding="SAME")
-        output.eval()
+        self.evaluate(output)
 
   def testConv3DTransposeValid(self):
     with self.cached_session():
@@ -165,7 +165,7 @@ class Conv3DTransposeTest(test.TestCase):
           1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
       output = nn_ops.conv3d_transpose(
           x, f, y_shape, strides=strides, padding="VALID")
-      value = output.eval()
+      value = self.evaluate(output)
 
       cache_values = np.zeros(y_shape, dtype=np.float32)
 
diff --git a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
index 8028f93a8c..df166b6101 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
@@ -81,7 +81,7 @@ class BinaryOpTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_cpu = out.eval()
+      tf_cpu = self.evaluate(out)
       # Test that the op takes precedence over numpy operators.
       np_left = tf_func(x, iny).eval()
       np_right = tf_func(inx, y).eval()
@@ -178,7 +178,7 @@ class BinaryOpTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_gpu = out.eval()
+      tf_gpu = self.evaluate(out)
     self.assertAllClose(np_ans, tf_gpu)
     self.assertShapeEqual(np_ans, out)
     # TODO(zhifengc/ke): make gradient checker work on GPU.
@@ -748,7 +748,7 @@ class ComparisonOpTest(test.TestCase):
       out = func(
           ops.convert_to_tensor(np.array([x]).astype(dtype)),
           ops.convert_to_tensor(np.array([y]).astype(dtype)))
-      ret = out.eval()
+      ret = self.evaluate(out)
     return ret[0]
 
   def testScalarCompareScalar(self):
@@ -779,7 +779,7 @@ class ComparisonOpTest(test.TestCase):
     np_ans = np_func(x, y)
     with self.test_session(force_gpu=test_util.is_gpu_available()):
       out = tf_func(ops.convert_to_tensor(x), ops.convert_to_tensor(y))
-      tf_ans = out.eval()
+      tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
 
   def testTensorCompareTensor(self):
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index c5311ad834..d7dbf5ab9a 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -88,7 +88,7 @@ class ComparisonOpTest(test.TestCase):
       out = func(
           ops.convert_to_tensor(np.array([x]).astype(dtype)),
           ops.convert_to_tensor(np.array([y]).astype(dtype)))
-      ret = out.eval()
+      ret = self.evaluate(out)
     return ret[0]
 
   def testScalarCompareScalar(self):
@@ -119,7 +119,7 @@ class ComparisonOpTest(test.TestCase):
     np_ans = np_func(x, y)
     with self.test_session(force_gpu=test_util.is_gpu_available()):
       out = tf_func(ops.convert_to_tensor(x), ops.convert_to_tensor(y))
-      tf_ans = out.eval()
+      tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
 
   def testTensorCompareTensor(self):
@@ -223,7 +223,7 @@ class LogicalOpTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_val = out.eval()
+      tf_val = self.evaluate(out)
     self.assertEqual(out.dtype, dtypes_lib.bool)
     self.assertAllEqual(np_ans, tf_val)
     self.assertShapeEqual(np_ans, out)
@@ -233,7 +233,7 @@ class LogicalOpTest(test.TestCase):
     with self.test_session(use_gpu=use_gpu,
                            force_gpu=use_gpu and test_util.is_gpu_available()):
       out = math_ops.logical_not(ops.convert_to_tensor(x))
-      tf_val = out.eval()
+      tf_val = self.evaluate(out)
     self.assertEqual(out.dtype, dtypes_lib.bool)
     self.assertAllEqual(np_ans, tf_val)
     self.assertShapeEqual(np_ans, out)
@@ -319,7 +319,7 @@ class SelectOpTest(test.TestCase):
     with self.test_session(use_gpu=use_gpu,
                            force_gpu=use_gpu and test_util.is_gpu_available()):
       out = array_ops.where(c, x, y)
-      tf_ans = out.eval()
+      tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
     self.assertShapeEqual(np_ans, out)
 
@@ -463,7 +463,7 @@ class BatchSelectOpTest(test.TestCase):
     with self.test_session(use_gpu=use_gpu,
                            force_gpu=use_gpu and test_util.is_gpu_available()):
       out = array_ops.where(c, x, y)
-      tf_ans = out.eval()
+      tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
     self.assertShapeEqual(np_ans, out)
 
@@ -644,13 +644,13 @@ class MathOpsOverloadTest(test.TestCase):
     with self.test_session(use_gpu=False):
       inx = ops.convert_to_tensor(x, dtype=dtype)
       z = func(inx, y)  # Should use __add__, __sub__, etc.
-      return z.eval()
+      return self.evaluate(z)
 
   def _computeLiteralAndTensor(self, x, y, dtype, func):
     with self.test_session(use_gpu=False):
       iny = ops.convert_to_tensor(y, dtype=dtype)
       z = func(x, iny)  # Should use __radd__, __rsub__, etc.
-      return z.eval()
+      return self.evaluate(z)
 
   def _compareBinary(self, x, y, dtype, np_func, tf_func):
     np_ans = np_func(x, y).astype(dtype.as_numpy_dtype)
@@ -777,9 +777,9 @@ class IsFiniteInfNanTest(test.TestCase):
             tf_y = math_ops.sqrt(x)
             tf_nan = math_ops.is_nan(tf_y)
             if value < 0:
-              self.assertAllEqual(np_nan, tf_nan.eval())
+              self.assertAllEqual(np_nan, self.evaluate(tf_nan))
             else:
-              self.assertAllCloseAccordingToType(np_y, tf_y.eval())
+              self.assertAllCloseAccordingToType(np_y, self.evaluate(tf_y))
 
 
 class RoundingTest(test.TestCase):
@@ -833,7 +833,7 @@ class ComplexMakeRealImagTest(test.TestCase):
       real = ops.convert_to_tensor(real)
       imag = ops.convert_to_tensor(imag)
       tf_ans = math_ops.complex(real, imag)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, out)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -855,10 +855,10 @@ class ComplexMakeRealImagTest(test.TestCase):
       tf_imag = math_ops.imag(inx)
       tf_real_real = math_ops.real(tf_real)
       tf_imag_real = math_ops.imag(tf_real)
-      self.assertAllEqual(np_real, tf_real.eval())
-      self.assertAllEqual(np_imag, tf_imag.eval())
-      self.assertAllEqual(np_real, tf_real_real.eval())
-      self.assertAllEqual(np_zeros, tf_imag_real.eval())
+      self.assertAllEqual(np_real, self.evaluate(tf_real))
+      self.assertAllEqual(np_imag, self.evaluate(tf_imag))
+      self.assertAllEqual(np_real, self.evaluate(tf_real_real))
+      self.assertAllEqual(np_zeros, self.evaluate(tf_imag_real))
 
   def testRealImag64(self):
     real = (np.arange(-3, 3) / 4.).reshape([1, 3, 2]).astype(np.float32)
@@ -916,7 +916,7 @@ class ComplexMakeRealImagTest(test.TestCase):
                            force_gpu=use_gpu and test_util.is_gpu_available()):
       inx = ops.convert_to_tensor(cplx)
       tf_conj = math_ops.conj(inx)
-      tf_ans = tf_conj.eval()
+      tf_ans = self.evaluate(tf_conj)
     self.assertAllEqual(np_ans, tf_ans)
     self.assertShapeEqual(np_ans, tf_conj)
 
@@ -1032,13 +1032,13 @@ class AccumulateTest(test.TestCase):
       np_val = random_arrays[0]
       for random_array in random_arrays[1:]:
         np_val += random_array
-      self.assertAllClose(np_val, tf_val.eval())
+      self.assertAllClose(np_val, self.evaluate(tf_val))
 
   def testZeroArgs(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
         tf_val = math_ops.accumulate_n([])
-        tf_val.eval()
+        self.evaluate(tf_val)
 
   def testWrongShape(self):
     with self.cached_session():
@@ -1070,7 +1070,7 @@ class PolyvalTest(test.TestCase):
     np_val = np.polyval(coeffs, x)
     with self.cached_session():
       tf_val = math_ops.polyval(coeffs, x)
-      self.assertAllClose(np_val, tf_val.eval())
+      self.assertAllClose(np_val, self.evaluate(tf_val))
 
   def testSimple(self):
     for dtype in [
@@ -1093,7 +1093,7 @@ class PolyvalTest(test.TestCase):
         np_val = np.polyval(coeffs, x)
         with self.cached_session():
           tf_val = math_ops.polyval(coeffs, x)
-          self.assertAllClose(np_val, tf_val.eval())
+          self.assertAllClose(np_val, self.evaluate(tf_val))
 
   def testEmpty(self):
     x = np.random.rand(2, 2).astype(np.float32)
@@ -1101,7 +1101,7 @@ class PolyvalTest(test.TestCase):
     np_val = np.polyval(coeffs, x)
     with self.cached_session():
       tf_val = math_ops.polyval(coeffs, x)
-      self.assertAllClose(np_val, tf_val.eval())
+      self.assertAllClose(np_val, self.evaluate(tf_val))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
index 77f182784e..7096083a1f 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
@@ -84,7 +84,7 @@ class UnaryOpTest(test.TestCase):
         np_ans *= 1.1
       else:
         y = tf_func(inx)
-      tf_cpu = y.eval()
+      tf_cpu = self.evaluate(y)
       self.assertShapeEqual(np_ans, y)
       if x.dtype == np.float16:
         self.assertAllClose(np_ans, tf_cpu, rtol=1e-3, atol=1e-3)
@@ -140,7 +140,7 @@ class UnaryOpTest(test.TestCase):
     np_ans = np_func(x)
     with self.test_session(force_gpu=test_util.is_gpu_available()):
       result = tf_func(ops.convert_to_tensor(x))
-      tf_gpu = result.eval()
+      tf_gpu = self.evaluate(result)
     if x.dtype == np.float16:
       self.assertAllClose(np_ans, tf_gpu, rtol=1e-3, atol=1e-3)
     else:
diff --git a/tensorflow/python/kernel_tests/decode_bmp_op_test.py b/tensorflow/python/kernel_tests/decode_bmp_op_test.py
index eebaffbe13..5e7991382e 100644
--- a/tensorflow/python/kernel_tests/decode_bmp_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_bmp_op_test.py
@@ -61,7 +61,7 @@ class DecodeBmpOpTest(test.TestCase):
     decode = array_ops.squeeze(image_ops.decode_bmp(img_in))
 
     with self.cached_session():
-      decoded = decode.eval()
+      decoded = self.evaluate(decode)
       self.assertAllEqual(decoded, img_bytes)
 
   def testGrayscale(self):
@@ -136,7 +136,7 @@ class DecodeBmpOpTest(test.TestCase):
     decode = image_ops.decode_bmp(img_in)
 
     with self.cached_session():
-      decoded = decode.eval()
+      decoded = self.evaluate(decode)
       self.assertAllEqual(decoded, img_bytes)
 
 
diff --git a/tensorflow/python/kernel_tests/decode_image_op_test.py b/tensorflow/python/kernel_tests/decode_image_op_test.py
index 0975f964b5..7a8743e11f 100644
--- a/tensorflow/python/kernel_tests/decode_image_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_image_op_test.py
@@ -76,7 +76,7 @@ class DecodeImageOpTest(test.TestCase):
 
         bad_channels = image_ops.decode_image(gif0, channels=1)
         with self.assertRaises(errors_impl.InvalidArgumentError):
-          bad_channels.eval()
+          self.evaluate(bad_channels)
 
   def testJpeg(self):
     # Read a real jpeg and verify shape
@@ -92,7 +92,7 @@ class DecodeImageOpTest(test.TestCase):
 
       bad_channels = image_ops.decode_image(jpeg0, channels=4)
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        bad_channels.eval()
+        self.evaluate(bad_channels)
 
   def testPng(self):
     # Read some real PNGs, converting to different channel numbers
@@ -113,7 +113,7 @@ class DecodeImageOpTest(test.TestCase):
     decode = image_ops.decode_image(image_bytes)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        decode.eval()
+        self.evaluate(decode)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/decode_png_op_test.py b/tensorflow/python/kernel_tests/decode_png_op_test.py
index 8f36343667..5a0b742a6a 100644
--- a/tensorflow/python/kernel_tests/decode_png_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_png_op_test.py
@@ -47,7 +47,7 @@ class DecodePngOpTest(test.TestCase):
             img_in, dtype=dtypes.uint16))
 
     with self.cached_session():
-      decoded = decode.eval()
+      decoded = self.evaluate(decode)
       self.assertAllEqual(decoded, img_bytes)
 
 
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
index affbaf159d..3ed7dba966 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
@@ -54,7 +54,7 @@ class AssignOpTest(test.TestCase):
       for t in threads:
         t.join()
 
-      vals = p.eval()
+      vals = self.evaluate(p)
       ones = np.ones((1024, 1024)).astype(np.float32)
       self.assertTrue((vals >= ones).all())
       self.assertTrue((vals <= ones * 20).all())
@@ -81,7 +81,7 @@ class AssignOpTest(test.TestCase):
       for t in threads:
         t.join()
 
-      vals = p.eval()
+      vals = self.evaluate(p)
 
       # Assert every element is taken from one of the assignments.
       self.assertTrue((vals > 0).all())
@@ -114,7 +114,7 @@ class AssignOpTest(test.TestCase):
       for t in threads:
         t.join()
 
-      vals = p.eval()
+      vals = self.evaluate(p)
       ones = np.ones((1024, 1024)).astype(np.float32)
       self.assertAllEqual(vals, ones * 20)
 
@@ -142,7 +142,7 @@ class AssignOpTest(test.TestCase):
       for t in threads:
         t.join()
 
-      vals = p.eval()
+      vals = self.evaluate(p)
 
       # Assert every element is the same, and taken from one of the assignments.
       self.assertTrue(vals[0, 0] > 0)
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_test.py b/tensorflow/python/kernel_tests/dense_update_ops_test.py
index 3e0a03d634..a4766fed72 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_test.py
@@ -36,8 +36,8 @@ class AssignOpTest(test.TestCase):
       p = variables.Variable(x)
       assign = state_ops.assign(p, y)
       p.initializer.run()
-      new_value = assign.eval()
-      return p.eval(), new_value
+      new_value = self.evaluate(assign)
+      return self.evaluate(p), new_value
 
   def _initAssignAddFetch(self, x, y, use_gpu=False):
     """Initialize a param to init, and compute param += y."""
@@ -45,8 +45,8 @@ class AssignOpTest(test.TestCase):
       p = variables.Variable(x)
       add = state_ops.assign_add(p, y)
       p.initializer.run()
-      new_value = add.eval()
-      return p.eval(), new_value
+      new_value = self.evaluate(add)
+      return self.evaluate(p), new_value
 
   def _initAssignSubFetch(self, x, y, use_gpu=False):
     """Initialize a param to init, and compute param -= y."""
@@ -54,8 +54,8 @@ class AssignOpTest(test.TestCase):
       p = variables.Variable(x)
       sub = state_ops.assign_sub(p, y)
       p.initializer.run()
-      new_value = sub.eval()
-      return p.eval(), new_value
+      new_value = self.evaluate(sub)
+      return self.evaluate(p), new_value
 
   def _testTypes(self, vals):
     for dtype in [np.float32, np.float64, np.int32, np.int64]:
@@ -90,13 +90,13 @@ class AssignOpTest(test.TestCase):
       p = variables.VariableV1([1])
       a = state_ops.assign(p, data, validate_shape=False)
       a.op.run()
-      self.assertAllEqual(p.eval(), data.eval())
+      self.assertAllEqual(p.eval(), self.evaluate(data))
 
       # Assign to yet another shape
       data2 = array_ops.fill([10, 10], 1)
       a2 = state_ops.assign(p, data2, validate_shape=False)
       a2.op.run()
-      self.assertAllEqual(p.eval(), data2.eval())
+      self.assertAllEqual(p.eval(), self.evaluate(data2))
 
   def testInitRequiredAssignAdd(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 13a28caf1f..c4bed11080 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -106,13 +106,13 @@ class DepthToSpaceTest(test.TestCase):
       # test NHWC (default) on CPU
       x_tf = array_ops.depth_to_space(input_nhwc, block_size)
       self.assertAllEqual(x_tf.shape, x_out.shape)
-      x_tf.eval()
+      self.evaluate(x_tf)
     if test.is_gpu_available():
       with self.cached_session(use_gpu=True):
         # test NHWC (default) on GPU
         x_tf = array_ops.depth_to_space(input_nhwc, block_size)
         self.assertAllEqual(x_tf.shape, x_out.shape)
-        x_tf.eval()
+        self.evaluate(x_tf)
 
   # Tests for different width and height.
   def testNonSquare(self):
@@ -185,7 +185,7 @@ class DepthToSpaceTest(test.TestCase):
     # divisible by 16.
     with self.assertRaises(ValueError):
       out_tf = array_ops.depth_to_space(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   # Test when the block size is 0.
   def testBlockSize0(self):
@@ -194,7 +194,7 @@ class DepthToSpaceTest(test.TestCase):
     block_size = 0
     with self.assertRaises(ValueError):
       out_tf = array_ops.depth_to_space(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   # Test when the block size is 1. The block size should be > 1.
   def testBlockSizeOne(self):
@@ -205,7 +205,7 @@ class DepthToSpaceTest(test.TestCase):
     block_size = 1
     with self.assertRaises(ValueError):
       out_tf = array_ops.depth_to_space(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   def testBlockSizeLargerThanInput(self):
     # The block size is too large for this input.
@@ -214,7 +214,7 @@ class DepthToSpaceTest(test.TestCase):
     block_size = 10
     with self.assertRaises(ValueError):
       out_tf = array_ops.space_to_depth(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   def testBlockSizeNotDivisibleDepth(self):
     # The depth is not divisible by the square of the block size.
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index 77b27c6c7e..f65d0be367 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -528,7 +528,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=output_sizes)
         backprop = nn_ops.depthwise_conv2d_native_backprop_input(
             t0, t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        ret = backprop.eval()
+        ret = self.evaluate(backprop)
         self.assertShapeEqual(ret, backprop)
         return ret
 
@@ -548,7 +548,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=output_sizes)
         backprop = nn_ops.depthwise_conv2d_native_backprop_input(
             t0, t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        ret = backprop.eval()
+        ret = self.evaluate(backprop)
         self.assertShapeEqual(ret, backprop)
         return ret
 
@@ -580,7 +580,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=output_sizes)
         backprop = nn_ops.depthwise_conv2d_native_backprop_filter(
             t0, t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        ret = backprop.eval()
+        ret = self.evaluate(backprop)
         self.assertShapeEqual(ret, backprop)
         return ret
 
@@ -600,7 +600,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=output_sizes)
         backprop = nn_ops.depthwise_conv2d_native_backprop_filter(
             t0, t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        ret = backprop.eval()
+        ret = self.evaluate(backprop)
         self.assertShapeEqual(ret, backprop)
         return ret
 
diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py
index da33b2848b..78c1d74da0 100644
--- a/tensorflow/python/kernel_tests/determinant_op_test.py
+++ b/tensorflow/python/kernel_tests/determinant_op_test.py
@@ -35,7 +35,7 @@ from tensorflow.python.platform import test
 class DeterminantOpTest(test.TestCase):
 
   def _compareDeterminantBase(self, matrix_x, tf_ans):
-    out = tf_ans.eval()
+    out = self.evaluate(tf_ans)
     shape = matrix_x.shape
     if shape[-1] == 0 and shape[-2] == 0:
       np_ans = np.ones(shape[:-2]).astype(matrix_x.dtype)
@@ -54,8 +54,8 @@ class DeterminantOpTest(test.TestCase):
       np_ans = np_ans.astype(matrix_x.dtype)
 
     self.assertShapeEqual(np_ans, abs_log_det_tf)
-    sign_tf_val = sign_tf.eval()
-    abs_log_det_tf_val = abs_log_det_tf.eval()
+    sign_tf_val = self.evaluate(sign_tf)
+    abs_log_det_tf_val = self.evaluate(abs_log_det_tf)
     self.assertAllClose(
         sign_tf_val * np.exp(abs_log_det_tf_val),
         np_sign * np.exp(np_ans),
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index 9e43258fa2..f7a9cd8d6e 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -89,7 +89,7 @@ class MatrixSetDiagTest(test.TestCase):
                                [1.0, 1.0, 3.0]])
       output = array_ops.matrix_set_diag(mat, v)
       self.assertEqual((3, 3), output.get_shape())
-      self.assertAllEqual(mat_set_diag, output.eval())
+      self.assertAllEqual(mat_set_diag, self.evaluate(output))
 
   def testRectangular(self):
     with self.session(use_gpu=True):
@@ -98,14 +98,14 @@ class MatrixSetDiagTest(test.TestCase):
       expected = np.array([[3.0, 1.0, 0.0], [1.0, 4.0, 1.0]])
       output = array_ops.matrix_set_diag(mat, v)
       self.assertEqual((2, 3), output.get_shape())
-      self.assertAllEqual(expected, output.eval())
+      self.assertAllEqual(expected, self.evaluate(output))
 
       v = np.array([3.0, 4.0])
       mat = np.array([[0.0, 1.0], [1.0, 0.0], [1.0, 1.0]])
       expected = np.array([[3.0, 1.0], [1.0, 4.0], [1.0, 1.0]])
       output = array_ops.matrix_set_diag(mat, v)
       self.assertEqual((3, 2), output.get_shape())
-      self.assertAllEqual(expected, output.eval())
+      self.assertAllEqual(expected, self.evaluate(output))
 
   def _testSquareBatch(self, dtype):
     with self.cached_session(use_gpu=True):
@@ -121,7 +121,7 @@ class MatrixSetDiagTest(test.TestCase):
 
       output = array_ops.matrix_set_diag(mat_batch, v_batch)
       self.assertEqual((2, 3, 3), output.get_shape())
-      self.assertAllEqual(mat_set_diag_batch, output.eval())
+      self.assertAllEqual(mat_set_diag_batch, self.evaluate(output))
 
   def testSquareBatch(self):
     self._testSquareBatch(np.float32)
@@ -140,7 +140,7 @@ class MatrixSetDiagTest(test.TestCase):
                                      [[-4.0, 0.0, 4.0], [0.0, -5.0, 0.0]]])
       output = array_ops.matrix_set_diag(mat_batch, v_batch)
       self.assertEqual((2, 2, 3), output.get_shape())
-      self.assertAllEqual(mat_set_diag_batch, output.eval())
+      self.assertAllEqual(mat_set_diag_batch, self.evaluate(output))
 
   def testInvalidShape(self):
     with self.assertRaisesRegexp(ValueError, "must be at least rank 2"):
@@ -273,9 +273,9 @@ class DiagTest(test.TestCase):
   def _diagOp(self, diag, dtype, expected_ans, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype)))
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       tf_ans_inv = array_ops.diag_part(expected_ans)
-      inv_out = tf_ans_inv.eval()
+      inv_out = self.evaluate(tf_ans_inv)
     self.assertAllClose(out, expected_ans)
     self.assertAllClose(inv_out, diag)
     self.assertShapeEqual(expected_ans, tf_ans)
@@ -421,7 +421,7 @@ class DiagPartOpTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu):
       tensor = ops.convert_to_tensor(tensor.astype(dtype))
       tf_ans_inv = array_ops.diag_part(tensor)
-      inv_out = tf_ans_inv.eval()
+      inv_out = self.evaluate(tf_ans_inv)
     self.assertAllClose(inv_out, expected_ans)
     self.assertShapeEqual(expected_ans, tf_ans_inv)
 
@@ -445,7 +445,7 @@ class DiagPartOpTest(test.TestCase):
         t = ops.convert_to_tensor(x.astype(np.float32))
         t.set_shape(shape)
         tf_ans = array_ops.diag_part(t)
-        out = tf_ans.eval()
+        out = self.evaluate(tf_ans)
       self.assertAllClose(out, expected_ans)
       self.assertShapeEqual(expected_ans, tf_ans)
 
diff --git a/tensorflow/python/kernel_tests/distributions/categorical_test.py b/tensorflow/python/kernel_tests/distributions/categorical_test.py
index c6bb06eab3..f116c54bd1 100644
--- a/tensorflow/python/kernel_tests/distributions/categorical_test.py
+++ b/tensorflow/python/kernel_tests/distributions/categorical_test.py
@@ -355,7 +355,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       samples = dist.sample(n, seed=123)
       samples.set_shape([n, 1, 2])
       self.assertEqual(samples.dtype, dtypes.int32)
-      sample_values = samples.eval()
+      sample_values = self.evaluate(samples)
       self.assertFalse(np.any(sample_values < 0))
       self.assertFalse(np.any(sample_values > 1))
       self.assertAllClose(
@@ -371,7 +371,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       dist = categorical.Categorical(math_ops.log(histograms) - 50.)
       samples = dist.sample((100, 100), seed=123)
       prob = dist.prob(samples)
-      prob_val = prob.eval()
+      prob_val = self.evaluate(prob)
       self.assertAllClose(
           [0.2**2 + 0.8**2], [prob_val[:, :, :, 0].mean()], atol=1e-2)
       self.assertAllClose(
@@ -393,26 +393,26 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       dist = categorical.Categorical(math_ops.log(histograms) - 50.)
 
       prob = dist.prob(1)
-      self.assertAllClose([[0.8, 0.6]], prob.eval())
+      self.assertAllClose([[0.8, 0.6]], self.evaluate(prob))
 
       prob = dist.prob([1])
-      self.assertAllClose([[0.8, 0.6]], prob.eval())
+      self.assertAllClose([[0.8, 0.6]], self.evaluate(prob))
 
       prob = dist.prob([0, 1])
-      self.assertAllClose([[0.2, 0.6]], prob.eval())
+      self.assertAllClose([[0.2, 0.6]], self.evaluate(prob))
 
       prob = dist.prob([[0, 1]])
-      self.assertAllClose([[0.2, 0.6]], prob.eval())
+      self.assertAllClose([[0.2, 0.6]], self.evaluate(prob))
 
       prob = dist.prob([[[0, 1]]])
-      self.assertAllClose([[[0.2, 0.6]]], prob.eval())
+      self.assertAllClose([[[0.2, 0.6]]], self.evaluate(prob))
 
       prob = dist.prob([[1, 0], [0, 1]])
-      self.assertAllClose([[0.8, 0.4], [0.2, 0.6]], prob.eval())
+      self.assertAllClose([[0.8, 0.4], [0.2, 0.6]], self.evaluate(prob))
 
       prob = dist.prob([[[1, 1], [1, 0]], [[1, 0], [0, 1]]])
       self.assertAllClose([[[0.8, 0.6], [0.8, 0.4]], [[0.8, 0.4], [0.2, 0.6]]],
-                          prob.eval())
+                          self.evaluate(prob))
 
   def testLogPMFShape(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py
index d558ca09cc..3662ca1ad1 100644
--- a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py
+++ b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py
@@ -110,7 +110,7 @@ class DirichletMultinomialTest(test.TestCase):
       counts = [1., 0]
       dist = ds.DirichletMultinomial(1., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(1 / 3., pmf.eval())
+      self.assertAllClose(1 / 3., self.evaluate(pmf))
       self.assertEqual((), pmf.get_shape())
 
   def testPmfBothZeroBatchesNontrivialN(self):
@@ -122,7 +122,7 @@ class DirichletMultinomialTest(test.TestCase):
       counts = [3., 2]
       dist = ds.DirichletMultinomial(5., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(1 / 7., pmf.eval())
+      self.assertAllClose(1 / 7., self.evaluate(pmf))
       self.assertEqual((), pmf.get_shape())
 
   def testPmfBothZeroBatchesMultidimensionalN(self):
@@ -134,7 +134,7 @@ class DirichletMultinomialTest(test.TestCase):
       n = np.full([4, 3], 5., dtype=np.float32)
       dist = ds.DirichletMultinomial(n, alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose([[1 / 7., 1 / 7., 1 / 7.]] * 4, pmf.eval())
+      self.assertAllClose([[1 / 7., 1 / 7., 1 / 7.]] * 4, self.evaluate(pmf))
       self.assertEqual((4, 3), pmf.get_shape())
 
   def testPmfAlphaStretchedInBroadcastWhenSameRank(self):
@@ -145,7 +145,7 @@ class DirichletMultinomialTest(test.TestCase):
       counts = [[1., 0], [0., 1]]
       dist = ds.DirichletMultinomial([1.], alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose([1 / 3., 2 / 3.], pmf.eval())
+      self.assertAllClose([1 / 3., 2 / 3.], self.evaluate(pmf))
       self.assertAllEqual([2], pmf.get_shape())
 
   def testPmfAlphaStretchedInBroadcastWhenLowerRank(self):
@@ -155,7 +155,7 @@ class DirichletMultinomialTest(test.TestCase):
       alpha = [1., 2]
       counts = [[1., 0], [0., 1]]
       pmf = ds.DirichletMultinomial(1., alpha).prob(counts)
-      self.assertAllClose([1 / 3., 2 / 3.], pmf.eval())
+      self.assertAllClose([1 / 3., 2 / 3.], self.evaluate(pmf))
       self.assertAllEqual([2], pmf.get_shape())
 
   def testPmfCountsStretchedInBroadcastWhenSameRank(self):
@@ -165,7 +165,7 @@ class DirichletMultinomialTest(test.TestCase):
       alpha = [[1., 2], [2., 3]]
       counts = [[1., 0]]
       pmf = ds.DirichletMultinomial([1., 1.], alpha).prob(counts)
-      self.assertAllClose([1 / 3., 2 / 5.], pmf.eval())
+      self.assertAllClose([1 / 3., 2 / 5.], self.evaluate(pmf))
       self.assertAllEqual([2], pmf.get_shape())
 
   def testPmfCountsStretchedInBroadcastWhenLowerRank(self):
@@ -175,7 +175,7 @@ class DirichletMultinomialTest(test.TestCase):
       alpha = [[1., 2], [2., 3]]
       counts = [1., 0]
       pmf = ds.DirichletMultinomial(1., alpha).prob(counts)
-      self.assertAllClose([1 / 3., 2 / 5.], pmf.eval())
+      self.assertAllClose([1 / 3., 2 / 5.], self.evaluate(pmf))
       self.assertAllEqual([2], pmf.get_shape())
 
   def testPmfForOneVoteIsTheMeanWithOneRecordInput(self):
@@ -289,7 +289,7 @@ class DirichletMultinomialTest(test.TestCase):
         expected_covariance = n * (n + alpha_0) / (1 + alpha_0) * shared_matrix
 
         self.assertEqual([2, 2], covariance.get_shape())
-        self.assertAllClose(expected_covariance, covariance.eval())
+        self.assertAllClose(expected_covariance, self.evaluate(covariance))
 
   def testCovarianceNAlphaBroadcast(self):
     alpha_v = [1., 2, 3]
@@ -327,7 +327,7 @@ class DirichletMultinomialTest(test.TestCase):
           ns * (ns + alpha_0) / (1 + alpha_0))[..., array_ops.newaxis]
 
       self.assertEqual([4, 3, 3], covariance.get_shape())
-      self.assertAllClose(expected_covariance, covariance.eval())
+      self.assertAllClose(expected_covariance, self.evaluate(covariance))
 
   def testCovarianceMultidimensional(self):
     alpha = np.random.rand(3, 5, 4).astype(np.float32)
@@ -353,7 +353,7 @@ class DirichletMultinomialTest(test.TestCase):
     with self.cached_session():
       dist = ds.DirichletMultinomial(0., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(1.0, pmf.eval())
+      self.assertAllClose(1.0, self.evaluate(pmf))
       self.assertEqual((), pmf.get_shape())
 
   def testLargeTauGivesPreciseProbabilities(self):
@@ -368,7 +368,7 @@ class DirichletMultinomialTest(test.TestCase):
     with self.cached_session():
       dist = ds.DirichletMultinomial(1., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(0.8, pmf.eval(), atol=1e-4)
+      self.assertAllClose(0.8, self.evaluate(pmf), atol=1e-4)
       self.assertEqual((), pmf.get_shape())
 
     # Two (three sided) coin flips.  Prob[coin 3] = 0.8.
@@ -376,7 +376,7 @@ class DirichletMultinomialTest(test.TestCase):
     with self.cached_session():
       dist = ds.DirichletMultinomial(2., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(0.8**2, pmf.eval(), atol=1e-2)
+      self.assertAllClose(0.8**2, self.evaluate(pmf), atol=1e-2)
       self.assertEqual((), pmf.get_shape())
 
     # Three (three sided) coin flips.
@@ -384,7 +384,7 @@ class DirichletMultinomialTest(test.TestCase):
     with self.cached_session():
       dist = ds.DirichletMultinomial(3., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(3 * 0.1 * 0.8 * 0.8, pmf.eval(), atol=1e-2)
+      self.assertAllClose(3 * 0.1 * 0.8 * 0.8, self.evaluate(pmf), atol=1e-2)
       self.assertEqual((), pmf.get_shape())
 
   def testSmallTauPrefersCorrelatedResults(self):
@@ -399,7 +399,7 @@ class DirichletMultinomialTest(test.TestCase):
     with self.cached_session():
       dist = ds.DirichletMultinomial(1., alpha)
       pmf = dist.prob(counts)
-      self.assertAllClose(0.5, pmf.eval())
+      self.assertAllClose(0.5, self.evaluate(pmf))
       self.assertEqual((), pmf.get_shape())
 
     # If there are two draws, it is much more likely that they are the same.
@@ -409,7 +409,7 @@ class DirichletMultinomialTest(test.TestCase):
       dist = ds.DirichletMultinomial(2., alpha)
       pmf_same = dist.prob(counts_same)
       pmf_different = dist.prob(counts_different)
-      self.assertLess(5 * pmf_different.eval(), pmf_same.eval())
+      self.assertLess(5 * self.evaluate(pmf_different), self.evaluate(pmf_same))
       self.assertEqual((), pmf_same.get_shape())
 
   def testNonStrictTurnsOffAllChecks(self):
diff --git a/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py b/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py
index e77e1117d4..b8bc2e55cf 100644
--- a/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py
+++ b/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py
@@ -63,17 +63,17 @@ class KLTest(test.TestCase):
       kl = kullback_leibler.kl_divergence(a, a, allow_nan_stats=False)
       with self.assertRaisesOpError(
           "KL calculation between .* and .* returned NaN values"):
-        kl.eval()
+        self.evaluate(kl)
       with self.assertRaisesOpError(
           "KL calculation between .* and .* returned NaN values"):
         a.kl_divergence(a).eval()
       a = MyDistException(loc=0.0, scale=1.0, allow_nan_stats=True)
       kl_ok = kullback_leibler.kl_divergence(a, a)
-      self.assertAllEqual([float("nan")], kl_ok.eval())
+      self.assertAllEqual([float("nan")], self.evaluate(kl_ok))
       self_kl_ok = a.kl_divergence(a)
-      self.assertAllEqual([float("nan")], self_kl_ok.eval())
+      self.assertAllEqual([float("nan")], self.evaluate(self_kl_ok))
       cross_ok = a.cross_entropy(a)
-      self.assertAllEqual([float("nan")], cross_ok.eval())
+      self.assertAllEqual([float("nan")], self.evaluate(cross_ok))
 
   def testRegistrationFailures(self):
 
diff --git a/tensorflow/python/kernel_tests/distributions/multinomial_test.py b/tensorflow/python/kernel_tests/distributions/multinomial_test.py
index 3840d7331c..b3f3416a52 100644
--- a/tensorflow/python/kernel_tests/distributions/multinomial_test.py
+++ b/tensorflow/python/kernel_tests/distributions/multinomial_test.py
@@ -127,7 +127,7 @@ class MultinomialTest(test.TestCase):
       p = [0.5, 0.5]
       counts = [1., 0]
       pmf = multinomial.Multinomial(total_count=1., probs=p).prob(counts)
-      self.assertAllClose(0.5, pmf.eval())
+      self.assertAllClose(0.5, self.evaluate(pmf))
       self.assertEqual((), pmf.get_shape())
 
   def testPmfBothZeroBatchesNontrivialN(self):
@@ -138,7 +138,7 @@ class MultinomialTest(test.TestCase):
       dist = multinomial.Multinomial(total_count=5., probs=p)
       pmf = dist.prob(counts)
       # 5 choose 3 = 5 choose 2 = 10. 10 * (.9)^2 * (.1)^3 = 81/10000.
-      self.assertAllClose(81. / 10000, pmf.eval())
+      self.assertAllClose(81. / 10000, self.evaluate(pmf))
       self.assertEqual((), pmf.get_shape())
 
   def testPmfPStretchedInBroadcastWhenSameRank(self):
@@ -146,7 +146,7 @@ class MultinomialTest(test.TestCase):
       p = [[0.1, 0.9]]
       counts = [[1., 0], [0, 1]]
       pmf = multinomial.Multinomial(total_count=1., probs=p).prob(counts)
-      self.assertAllClose([0.1, 0.9], pmf.eval())
+      self.assertAllClose([0.1, 0.9], self.evaluate(pmf))
       self.assertEqual((2), pmf.get_shape())
 
   def testPmfPStretchedInBroadcastWhenLowerRank(self):
@@ -154,7 +154,7 @@ class MultinomialTest(test.TestCase):
       p = [0.1, 0.9]
       counts = [[1., 0], [0, 1]]
       pmf = multinomial.Multinomial(total_count=1., probs=p).prob(counts)
-      self.assertAllClose([0.1, 0.9], pmf.eval())
+      self.assertAllClose([0.1, 0.9], self.evaluate(pmf))
       self.assertEqual((2), pmf.get_shape())
 
   def testPmfCountsStretchedInBroadcastWhenSameRank(self):
@@ -182,7 +182,7 @@ class MultinomialTest(test.TestCase):
       # [2]
       counts = [2., 1]
       pmf = multinomial.Multinomial(total_count=n, probs=p).prob(counts)
-      pmf.eval()
+      self.evaluate(pmf)
       self.assertEqual(pmf.get_shape(), (2, 2))
 
   def testPmfShapeCountsPStretchedN(self):
@@ -191,7 +191,7 @@ class MultinomialTest(test.TestCase):
       counts = [3., 2]
       n = np.full([4, 3], 5., dtype=np.float32)
       pmf = multinomial.Multinomial(total_count=n, probs=p).prob(counts)
-      pmf.eval()
+      self.evaluate(pmf)
       self.assertEqual((4, 3), pmf.get_shape())
 
   def testMultinomialMean(self):
diff --git a/tensorflow/python/kernel_tests/distributions/special_math_test.py b/tensorflow/python/kernel_tests/distributions/special_math_test.py
index cc43e12168..6b6de8b139 100644
--- a/tensorflow/python/kernel_tests/distributions/special_math_test.py
+++ b/tensorflow/python/kernel_tests/distributions/special_math_test.py
@@ -362,7 +362,7 @@ class ErfInvTest(test.TestCase):
 
       expected_x = special.erfinv(x)
       x = special_math.erfinv(x)
-      self.assertAllClose(expected_x, x.eval(), atol=0.)
+      self.assertAllClose(expected_x, self.evaluate(x), atol=0.)
 
   def testErfInvIntegerInput(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
index c3f67d29aa..c0b0e3f193 100644
--- a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
@@ -41,7 +41,7 @@ class DynamicStitchTestBase(object):
       data = [constant_op.constant(40), constant_op.constant(60)]
       for step in -1, 1:
         stitched_t = self.stitch_op(indices[::step], data)
-        stitched_val = stitched_t.eval()
+        stitched_val = self.evaluate(stitched_t)
         self.assertAllEqual([40, 60][::step], stitched_val)
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([2], stitched_t.get_shape().as_list())
@@ -78,7 +78,7 @@ class DynamicStitchTestBase(object):
                 constant_op.constant([10, 60, 20, 30, 50]), dtype=dtype)
         ]
         stitched_t = self.stitch_op(indices, data)
-        stitched_val = stitched_t.eval()
+        stitched_val = self.evaluate(stitched_t)
         self.assertAllEqual([0, 10, 20, 30, 40, 50, 60, 70], stitched_val)
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([8], stitched_t.get_shape().as_list())
@@ -88,7 +88,7 @@ class DynamicStitchTestBase(object):
       indices = [constant_op.constant([1, 6, 2, 3, 5, 0, 4, 7])]
       data = [constant_op.constant([10, 60, 20, 30, 50, 0, 40, 70])]
       stitched_t = self.stitch_op(indices, data)
-      stitched_val = stitched_t.eval()
+      stitched_val = self.evaluate(stitched_t)
       self.assertAllEqual([0, 10, 20, 30, 40, 50, 60, 70], stitched_val)
       # Dimension 0 is max(flatten(indices))+1.
       self.assertEqual([8], stitched_t.get_shape().as_list())
@@ -106,7 +106,7 @@ class DynamicStitchTestBase(object):
           constant_op.constant([[20, 21], [30, 31], [50, 51]])
       ]
       stitched_t = self.stitch_op(indices, data)
-      stitched_val = stitched_t.eval()
+      stitched_val = self.evaluate(stitched_t)
       self.assertAllEqual([[0, 1], [10, 11], [20, 21], [30, 31], [40, 41],
                            [50, 51], [60, 61], [70, 71]], stitched_val)
       # Dimension 0 is max(flatten(indices))+1.
@@ -127,7 +127,7 @@ class DynamicStitchTestBase(object):
           array_ops.zeros([0, 2], dtype=dtypes.int32)
       ]
       stitched_t = self.stitch_op(indices, data)
-      stitched_val = stitched_t.eval()
+      stitched_val = self.evaluate(stitched_t)
       self.assertAllEqual([[0, 1], [10, 11], [20, 21], [30, 31], [40, 41],
                            [50, 51], [60, 61], [70, 71]], stitched_val)
       # Dimension 0 is max(flatten(indices))+1.
@@ -147,7 +147,7 @@ class DynamicStitchTestBase(object):
                                 [[1., 2.], [31., 32.]]])
       ]
       stitched_t = self.stitch_op(indices, data)
-      stitched_val = stitched_t.eval()
+      stitched_val = self.evaluate(stitched_t)
       correct = 10. * np.arange(7)[:, None] + [1., 2.]
       self.assertAllEqual(correct, stitched_val)
       self.assertEqual([7, 2], stitched_t.get_shape().as_list())
@@ -157,7 +157,7 @@ class DynamicStitchTestBase(object):
                                        stitched_grad)
       self.assertEqual(grads[:3], [None] * 3)  # Indices have no gradients
       for datum, grad in zip(data, sess.run(grads[3:])):
-        self.assertAllEqual(7. * datum.eval(), grad)
+        self.assertAllEqual(7. * self.evaluate(datum), grad)
 
   def testErrorIndicesMultiDimensional(self):
     indices = [
@@ -227,7 +227,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
       data = [constant_op.constant(40.0), constant_op.constant(60.0)]
       for step in -1, 1:
         stitched_t = data_flow_ops.dynamic_stitch(indices[::step], data)
-        stitched_val = stitched_t.eval()
+        stitched_val = self.evaluate(stitched_t)
         self.assertAllEqual([40.0, 60.0][::step], stitched_val)
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([2], stitched_t.get_shape().as_list())
@@ -246,7 +246,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
               [[[51, 52], [21, 22]], [[1, 2], [31, 32]]], dtype=dtypes.float32)
       ]
       stitched_t = data_flow_ops.dynamic_stitch(indices, data)
-      stitched_val = stitched_t.eval()
+      stitched_val = self.evaluate(stitched_t)
       correct = 10 * np.arange(7)[:, None] + [1.0, 2.0]
       self.assertAllEqual(correct, stitched_val)
       self.assertEqual([7, 2], stitched_t.get_shape().as_list())
@@ -256,7 +256,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
                                        stitched_grad)
       self.assertEqual(grads[:3], [None] * 3)  # Indices have no gradients
       for datum, grad in zip(data, sess.run(grads[3:])):
-        self.assertAllEqual(7.0 * datum.eval(), grad)
+        self.assertAllEqual(7.0 * self.evaluate(datum), grad)
 
   # GPU version unit tests
   def testScalarGPU(self):
@@ -265,7 +265,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
       data = [constant_op.constant(40.0), constant_op.constant(60.0)]
       for step in -1, 1:
         stitched_t = data_flow_ops.dynamic_stitch(indices[::step], data)
-        stitched_val = stitched_t.eval()
+        stitched_val = self.evaluate(stitched_t)
         self.assertAllEqual([40.0, 60.0][::step], stitched_val)
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([2], stitched_t.get_shape().as_list())
@@ -284,7 +284,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
               [[[51, 52], [21, 22]], [[1, 2], [31, 32]]], dtype=dtypes.float32)
       ]
       stitched_t = data_flow_ops.dynamic_stitch(indices, data)
-      stitched_val = stitched_t.eval()
+      stitched_val = self.evaluate(stitched_t)
       correct = 10 * np.arange(7)[:, None] + [1.0, 2.0]
       self.assertAllEqual(correct, stitched_val)
       self.assertEqual([7, 2], stitched_t.get_shape().as_list())
@@ -294,7 +294,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
                                        stitched_grad)
       self.assertEqual(grads[:3], [None] * 3)  # Indices have no gradients
       for datum, grad in zip(data, sess.run(grads[3:])):
-        self.assertAllEqual(7.0 * datum.eval(), grad)
+        self.assertAllEqual(7.0 * self.evaluate(datum), grad)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/edit_distance_op_test.py b/tensorflow/python/kernel_tests/edit_distance_op_test.py
index dab5eee7f5..4a06ab770a 100644
--- a/tensorflow/python/kernel_tests/edit_distance_op_test.py
+++ b/tensorflow/python/kernel_tests/edit_distance_op_test.py
@@ -49,11 +49,11 @@ class EditDistanceTest(test.TestCase):
 
     if expected_err_re is None:
       self.assertEqual(edit_distance.get_shape(), expected_shape)
-      output = edit_distance.eval()
+      output = self.evaluate(edit_distance)
       self.assertAllClose(output, expected_output)
     else:
       with self.assertRaisesOpError(expected_err_re):
-        edit_distance.eval()
+        self.evaluate(edit_distance)
 
   def _testEditDistance(self,
                         hypothesis,
diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py
index 008d6fbf57..443f54a958 100644
--- a/tensorflow/python/kernel_tests/embedding_ops_test.py
+++ b/tensorflow/python/kernel_tests/embedding_ops_test.py
@@ -76,7 +76,7 @@ class ScatterAddSubTest(test.TestCase):
       # p = init
       variables.global_variables_initializer().run()
       # p += vals
-      result = p2.eval()
+      result = self.evaluate(p2)
     # Compute the expected 'p' using numpy operations.
     for i, ind in enumerate(indices):
       if scatter_op == state_ops.scatter_add:
@@ -278,7 +278,7 @@ class EmbeddingLookupTest(test.TestCase):
       norms = math_ops.sqrt(
           math_ops.reduce_sum(embeddings * embeddings, axis=1))
       normalized = embeddings / array_ops.stack([norms, norms], axis=1)
-      self.assertAllEqual(embedding.eval(), 2 * normalized.eval())
+      self.assertAllEqual(embedding.eval(), 2 * self.evaluate(normalized))
 
   def testSimpleShardedPartitionedVariable(self):
     with self.cached_session() as sess:
@@ -319,7 +319,7 @@ class EmbeddingLookupTest(test.TestCase):
       p_var_val = sess.run(list(p_variable))
       # Actual test
       print(ops.get_default_graph().as_graph_def())
-      tf_result = embedding.eval()
+      tf_result = self.evaluate(embedding)
     np_result, _, _ = _EmbeddingResult(params, id_vals, num_shards, vocab_size)
     self.assertAllEqual(params_values, p_var_val)
     self.assertAllEqual(np_result, tf_result)
diff --git a/tensorflow/python/kernel_tests/extract_image_patches_op_test.py b/tensorflow/python/kernel_tests/extract_image_patches_op_test.py
index 61436f24cf..4fe51e94e1 100644
--- a/tensorflow/python/kernel_tests/extract_image_patches_op_test.py
+++ b/tensorflow/python/kernel_tests/extract_image_patches_op_test.py
@@ -51,7 +51,7 @@ class ExtractImagePatches(test.TestCase):
           rates=rates,
           padding=padding,
           name="im2col")
-      self.assertAllClose(patches, out_tensor.eval())
+      self.assertAllClose(patches, self.evaluate(out_tensor))
 
   def testKsize1x1Stride1x1Rate1x1(self):
     """Verifies that for 1x1 kernel the output equals the input."""
diff --git a/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py b/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py
index bbb3fef85b..d99823d517 100644
--- a/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py
+++ b/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py
@@ -52,7 +52,7 @@ class ExtractVolumePatches(test.TestCase):
           strides=strides,
           padding=padding,
           name="im2col_3d")
-      self.assertAllClose(patches, out_tensor.eval())
+      self.assertAllClose(patches, self.evaluate(out_tensor))
 
   # pylint: disable=bad-whitespace
   def testKsize1x1x1Stride1x1x1(self):
diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py
index 8961c4b13c..e3742f2e72 100644
--- a/tensorflow/python/kernel_tests/fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/fifo_queue_test.py
@@ -211,7 +211,7 @@ class FIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        vals = dequeued_t.eval()
+        vals = self.evaluate(dequeued_t)
         self.assertEqual([elems[i]], vals)
 
   def testDequeueHalf(self):
@@ -225,7 +225,7 @@ class FIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        vals = dequeued_t.eval()
+        vals = self.evaluate(dequeued_t)
         self.assertEqual([elems[i]], vals)
 
   def testEnqueueAndBlockingDequeue(self):
@@ -288,9 +288,9 @@ class FIFOQueueTest(test.TestCase):
       self.assertEqual([], size.get_shape())
 
       enqueue_op.run()
-      self.assertEqual(1, size.eval())
+      self.assertEqual(1, self.evaluate(size))
       dequeued_t.op.run()
-      self.assertEqual(0, size.eval())
+      self.assertEqual(0, self.evaluate(size))
 
   def testEnqueueMany(self):
     with self.cached_session():
@@ -302,7 +302,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        vals = dequeued_t.eval()
+        vals = self.evaluate(dequeued_t)
         self.assertEqual([elems[i % 4]], vals)
 
   def testEmptyEnqueueMany(self):
@@ -313,9 +313,9 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue_many((empty_t,))
       size_t = q.size()
 
-      self.assertEqual([0], size_t.eval())
+      self.assertEqual([0], self.evaluate(size_t))
       enqueue_op.run()
-      self.assertEqual([0], size_t.eval())
+      self.assertEqual([0], self.evaluate(size_t))
 
   def testEmptyDequeueMany(self):
     with self.cached_session():
@@ -323,9 +323,9 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue((10.0,))
       dequeued_t = q.dequeue_many(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testEmptyDequeueUpTo(self):
     with self.cached_session():
@@ -333,9 +333,9 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue((10.0,))
       dequeued_t = q.dequeue_up_to(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testEmptyDequeueManyWithNoShape(self):
     with self.cached_session():
@@ -369,8 +369,8 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      self.assertAllEqual(elems[0:4], dequeued_t.eval())
-      self.assertAllEqual(elems[4:8], dequeued_t.eval())
+      self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t))
+      self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t))
 
   def testDequeueUpToNoBlocking(self):
     with self.cached_session():
@@ -381,8 +381,8 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      self.assertAllEqual(elems[0:4], dequeued_t.eval())
-      self.assertAllEqual(elems[4:8], dequeued_t.eval())
+      self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t))
+      self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t))
 
   def testMultiDequeueMany(self):
     with self.cached_session() as sess:
@@ -518,7 +518,7 @@ class FIFOQueueTest(test.TestCase):
                                    r"Expected \[2,3,3\], got \[2,3,4\]"):
         sess.run([enqueue_op],
                  feed_dict={elems_bad: np.array([1] * 24).reshape((2, 3, 4))})
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testParallelEnqueueMany(self):
     with self.cached_session() as sess:
@@ -672,7 +672,7 @@ class FIFOQueueTest(test.TestCase):
       while elements_dequeued < 250:
         # With equal probability, run Dequeue or dequeue_many.
         if random.random() > 0.5:
-          self.assertEqual(elements_dequeued, dequeued_t.eval())
+          self.assertEqual(elements_dequeued, self.evaluate(dequeued_t))
           elements_dequeued += 1
         else:
           count = random.randint(0, min(20, 250 - elements_dequeued))
@@ -778,12 +778,12 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
       close_op.run()
       for elem in elems:
-        self.assertEqual([elem], dequeued_t.eval())
+        self.assertEqual([elem], self.evaluate(dequeued_t))
 
       # Expect the operation to fail due to the queue being closed.
       with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                    "is closed and has insufficient"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testBlockingDequeueFromClosedQueue(self):
     with self.cached_session() as sess:
@@ -1059,8 +1059,8 @@ class FIFOQueueTest(test.TestCase):
       # TODO(mrry): Figure out how to do this without sleeping.
       time.sleep(0.1)
       for elem in elems:
-        self.assertEqual([elem], dequeued_t.eval())
-      self.assertEqual([50.0], dequeued_t.eval())
+        self.assertEqual([elem], self.evaluate(dequeued_t))
+      self.assertEqual([50.0], self.evaluate(dequeued_t))
       thread.join()
 
   def testBlockingEnqueueManyToFullQueue(self):
@@ -1082,10 +1082,10 @@ class FIFOQueueTest(test.TestCase):
       # TODO(mrry): Figure out how to do this without sleeping.
       time.sleep(0.1)
       for elem in elems:
-        self.assertEqual([elem], dequeued_t.eval())
+        self.assertEqual([elem], self.evaluate(dequeued_t))
         time.sleep(0.01)
-      self.assertEqual([50.0], dequeued_t.eval())
-      self.assertEqual([60.0], dequeued_t.eval())
+      self.assertEqual([50.0], self.evaluate(dequeued_t))
+      self.assertEqual([60.0], self.evaluate(dequeued_t))
 
       # Make sure the thread finishes before exiting.
       thread.join()
@@ -1119,12 +1119,12 @@ class FIFOQueueTest(test.TestCase):
       close_thread.start()
 
       # The dequeue will unblock both threads.
-      self.assertEqual(10.0, dequeued_t.eval())
+      self.assertEqual(10.0, self.evaluate(dequeued_t))
       enqueue_thread.join()
       close_thread.join()
 
       for elem in [20.0, 30.0, 40.0, 50.0]:
-        self.assertEqual(elem, dequeued_t.eval())
+        self.assertEqual(elem, self.evaluate(dequeued_t))
       self.assertEqual(0, q.size().eval())
 
   def testBlockingEnqueueManyBeforeClose(self):
@@ -1154,11 +1154,11 @@ class FIFOQueueTest(test.TestCase):
       close_thread.start()
 
       # The dequeue will unblock both threads.
-      self.assertEqual(10.0, dequeued_t.eval())
+      self.assertEqual(10.0, self.evaluate(dequeued_t))
       enqueue_thread.join()
       close_thread.join()
       for elem in [20.0, 30.0, 50.0, 60.0]:
-        self.assertEqual(elem, dequeued_t.eval())
+        self.assertEqual(elem, self.evaluate(dequeued_t))
 
   def testDoesNotLoseValue(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
index 114240fd85..cb7659a89a 100644
--- a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
@@ -364,7 +364,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               padding = "VALID"
               output_tensor = nn_ops.avg_pool(input_tensor, window_size,
                                               stride_size, padding)
-              output_data = output_tensor.eval()
+              output_data = self.evaluate(output_tensor)
               num_elements = 1
               for dim_size in output_data.shape:
                 num_elements *= dim_size
@@ -373,7 +373,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               input_backprop_tensor = gen_nn_ops.avg_pool_grad(
                   input_tensor.get_shape(), output_backprop, window_size,
                   stride_size, padding)
-              input_backprop = input_backprop_tensor.eval()
+              input_backprop = self.evaluate(input_backprop_tensor)
               row_seq = list(range(0, num_rows + 1, row_window_size))
               col_seq = list(range(0, num_cols + 1, col_window_size))
               fap_input_backprop_tensor = gen_nn_ops.fractional_avg_pool_grad(
@@ -382,7 +382,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
                   row_seq,
                   col_seq,
                   overlapping=False)
-              fap_input_backprop = fap_input_backprop_tensor.eval()
+              fap_input_backprop = self.evaluate(fap_input_backprop_tensor)
               self.assertShapeEqual(input_backprop, fap_input_backprop_tensor)
               self.assertAllClose(input_backprop, fap_input_backprop)
 
@@ -403,7 +403,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               padding = "VALID"
               output_tensor = nn_ops.avg_pool(input_tensor, window_size,
                                               stride_size, padding)
-              output_data = output_tensor.eval()
+              output_data = self.evaluate(output_tensor)
               num_elements = 1
               for dim_size in output_data.shape:
                 num_elements *= dim_size
@@ -412,7 +412,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               input_backprop_tensor = gen_nn_ops.avg_pool_grad(
                   input_tensor.get_shape(), output_backprop, window_size,
                   stride_size, padding)
-              input_backprop = input_backprop_tensor.eval()
+              input_backprop = self.evaluate(input_backprop_tensor)
               row_seq = list(range(0, num_rows, row_window_size - 1))
               col_seq = list(range(0, num_cols, col_window_size - 1))
               row_seq[-1] += 1
@@ -423,7 +423,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
                   row_seq,
                   col_seq,
                   overlapping=True)
-              fap_input_backprop = fap_input_backprop_tensor.eval()
+              fap_input_backprop = self.evaluate(fap_input_backprop_tensor)
               self.assertShapeEqual(input_backprop, fap_input_backprop_tensor)
               self.assertAllClose(input_backprop, fap_input_backprop)
 
@@ -442,7 +442,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               pseudo_random=pseudo_random,
               overlapping=overlapping,
               seed=self._SEED)
-          output_data = output_tensor.eval()
+          output_data = self.evaluate(output_tensor)
           output_shape = output_data.shape
           # error_margin and delta setting is similar to avg_pool_grad.
           error_margin = 1e-4
@@ -473,7 +473,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
                   pseudo_random=pseudo_random,
                   overlapping=overlapping,
                   seed=self._SEED)
-              output_data = output_tensor.eval()
+              output_data = self.evaluate(output_tensor)
               output_shape = output_data.shape
               # error_margin and delta setting is similar to avg_pool_grad.
               error_margin = 1e-4
diff --git a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
index 1b536ba28a..0427e34fc1 100644
--- a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
@@ -374,12 +374,12 @@ class FractionalMaxPoolGradTest(test.TestCase):
               padding = "VALID"
               output_tensor = nn_ops.max_pool(input_tensor, window_size,
                                               stride_size, padding)
-              output_data = output_tensor.eval()
+              output_data = self.evaluate(output_tensor)
               output_backprop = self._PRNG.randint(100, size=output_data.shape)
               input_backprop_tensor = gen_nn_ops.max_pool_grad(
                   input_tensor, output_tensor, output_backprop, window_size,
                   stride_size, padding)
-              input_backprop = input_backprop_tensor.eval()
+              input_backprop = self.evaluate(input_backprop_tensor)
               row_seq = list(range(0, num_rows + 1, row_window_size))
               col_seq = list(range(0, num_cols + 1, col_window_size))
               fmp_input_backprop_tensor = gen_nn_ops.fractional_max_pool_grad(
@@ -389,7 +389,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
                   row_seq,
                   col_seq,
                   overlapping=False)
-              fmp_input_backprop = fmp_input_backprop_tensor.eval()
+              fmp_input_backprop = self.evaluate(fmp_input_backprop_tensor)
               self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor)
               self.assertAllClose(input_backprop, fmp_input_backprop)
 
@@ -409,12 +409,12 @@ class FractionalMaxPoolGradTest(test.TestCase):
               padding = "VALID"
               output_tensor = nn_ops.max_pool(input_tensor, window_size,
                                               stride_size, padding)
-              output_data = output_tensor.eval()
+              output_data = self.evaluate(output_tensor)
               output_backprop = self._PRNG.randint(100, size=output_data.shape)
               input_backprop_tensor = gen_nn_ops.max_pool_grad(
                   input_tensor, output_tensor, output_backprop, window_size,
                   stride_size, padding)
-              input_backprop = input_backprop_tensor.eval()
+              input_backprop = self.evaluate(input_backprop_tensor)
               row_seq = list(range(0, num_rows, row_window_size - 1))
               col_seq = list(range(0, num_cols, col_window_size - 1))
               row_seq[-1] += 1
@@ -426,7 +426,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
                   row_seq,
                   col_seq,
                   overlapping=True)
-              fmp_input_backprop = fmp_input_backprop_tensor.eval()
+              fmp_input_backprop = self.evaluate(fmp_input_backprop_tensor)
               self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor)
               self.assertAllClose(input_backprop, fmp_input_backprop)
 
@@ -447,7 +447,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
               pseudo_random=pseudo_random,
               overlapping=overlapping,
               seed=self._SEED)
-          output_data = output_tensor.eval()
+          output_data = self.evaluate(output_tensor)
           output_shape = output_data.shape
           # error_margin and delta setting is similar to max_pool_grad.
           error_margin = 1e-3
@@ -480,7 +480,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
                   pseudo_random=pseudo_random,
                   overlapping=overlapping,
                   seed=self._SEED)
-              output_data = output_tensor.eval()
+              output_data = self.evaluate(output_tensor)
               output_shape = output_data.shape
               # error_margin and delta setting is similar to max_pool_grad.
               error_margin = 1e-3
@@ -578,7 +578,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
           row_seq,
           col_seq,
           overlapping=False)
-      input_backprop_not_overlapping = r.eval()
+      input_backprop_not_overlapping = self.evaluate(r)
       self.assertShapeEqual(
           np.reshape(expected_input_backprop_not_overlapping, input_size), r)
       self.assertAllClose(expected_input_backprop_not_overlapping,
@@ -588,7 +588,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
           output_data_overlapping, shape=output_size)
       r = gen_nn_ops.fractional_max_pool_grad(
           input_tensor, output_tensor, grad, row_seq, col_seq, overlapping=True)
-      input_backprop_overlapping = r.eval()
+      input_backprop_overlapping = self.evaluate(r)
       self.assertShapeEqual(
           np.reshape(expected_input_backprop_overlapping, input_size), r)
       self.assertAllClose(expected_input_backprop_overlapping,
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 04c1032722..bebd98558f 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -974,7 +974,7 @@ class FunctionalOpsTest(test.TestCase):
           MLP,
           rewrite_with_while=rewrite_with_while)[0]
 
-      return ret.eval()
+      return self.evaluate(ret)
 
   def _npMLP(self, xval, wsval, bsval):
     for i in range(wsval.shape[0]):
diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py
index ee761435d8..532d8903ee 100644
--- a/tensorflow/python/kernel_tests/gather_nd_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py
@@ -40,7 +40,7 @@ class GatherNdTest(test.TestCase):
       params = constant_op.constant(np.array([8, 1, 2, 3, 7, 5], dtype=dtype))
       indices = constant_op.constant([[4], [4], [0]])
       gather_nd_t = array_ops.gather_nd(params, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     self.assertAllEqual(np.array([7, 7, 8], dtype=dtype), gather_nd_val)
     self.assertEqual([3], gather_nd_t.get_shape())
@@ -60,20 +60,20 @@ class GatherNdTest(test.TestCase):
 
       indices_empty = np.empty((0, 2), dtype=np.int32)
       gather_nd_ok_t = array_ops.gather_nd(params, indices_empty)
-      gather_nd_ok_val = gather_nd_ok_t.eval()
+      gather_nd_ok_val = self.evaluate(gather_nd_ok_t)
       self.assertEqual([0], gather_nd_ok_t.get_shape())
       self.assertAllClose(np.empty((0,), dtype=np.float32), gather_nd_ok_val)
 
       indices_empty = np.empty((0, 1), dtype=np.int32)
       gather_nd_ok_t = array_ops.gather_nd(params, indices_empty)
-      gather_nd_ok_val = gather_nd_ok_t.eval()
+      gather_nd_ok_val = self.evaluate(gather_nd_ok_t)
       self.assertEqual([0, 3], gather_nd_ok_t.get_shape())
       self.assertAllClose(np.empty((0, 3), dtype=np.float32), gather_nd_ok_val)
 
       params_empty = np.empty((0, 3), dtype=np.float32)
       indices_empty = np.empty((0, 2), dtype=np.int32)
       gather_nd_ok_t = array_ops.gather_nd(params_empty, indices_empty)
-      gather_nd_ok_val = gather_nd_ok_t.eval()
+      gather_nd_ok_val = self.evaluate(gather_nd_ok_t)
       self.assertEqual([0], gather_nd_ok_t.get_shape())
       self.assertAllClose(np.empty((0,), dtype=np.float32), gather_nd_ok_val)
 
@@ -82,7 +82,7 @@ class GatherNdTest(test.TestCase):
       gather_nd_break_t = array_ops.gather_nd(params_empty, indices_nonempty)
       with self.assertRaisesOpError(
           r"Requested more than 0 entries, but params is empty."):
-        gather_nd_break_t.eval()
+        self.evaluate(gather_nd_break_t)
       self.assertAllClose(np.empty((0,), dtype=np.float32), gather_nd_ok_val)
 
   def testIndexScalar(self):
@@ -91,7 +91,7 @@ class GatherNdTest(test.TestCase):
           [[-8, -1, -2, -3, -7, -5], [8, 1, 2, 3, 7, 5]], dtype=np.float32).T
       indices = constant_op.constant([4, 1])
       gather_nd_t = array_ops.gather_nd(params, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
       self.assertEqual([], gather_nd_t.get_shape())
       self.assertAllEqual(np.array(7), gather_nd_val)
 
@@ -101,7 +101,7 @@ class GatherNdTest(test.TestCase):
           [[-8, -1, -2, -3, -7, -5], [8, 1, 2, 3, 7, 5]], dtype=np.float32).T
       indices = constant_op.constant([4])
       gather_nd_t = array_ops.gather_nd(params, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
       self.assertEqual([2], gather_nd_t.get_shape())
       self.assertAllEqual(np.array([-7, 7]), gather_nd_val)
 
@@ -111,7 +111,7 @@ class GatherNdTest(test.TestCase):
           [[-8, -1, -2, -3, -7, -5], [8, 1, 2, 3, 7, 5]], dtype=np.float32).T
       indices = constant_op.constant([[4], [4], [0]])
       gather_nd_t = array_ops.gather_nd(params, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     self.assertEqual([3, 2], gather_nd_t.get_shape())
     self.assertAllEqual(np.array([[-7, 7], [-7, 7], [-8, 8]]), gather_nd_val)
@@ -125,7 +125,7 @@ class GatherNdTest(test.TestCase):
       params_t = constant_op.constant(params)
       indices = constant_op.constant([[4], [4], [0]])
       gather_nd_t = array_ops.gather_nd(params_t, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     self.assertEqual([3, 2, 2], gather_nd_t.get_shape())
     self.assertAllEqual(params[[4, 4, 0]], gather_nd_val)
@@ -140,7 +140,7 @@ class GatherNdTest(test.TestCase):
       indices = constant_op.constant(
           [[], []], dtype=dtypes.int32)  # Size (2, 0)
       gather_nd_t = array_ops.gather_nd(params_t, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     self.assertEqual([2, 6, 2, 2], gather_nd_t.get_shape())
     self.assertAllEqual(
@@ -156,7 +156,7 @@ class GatherNdTest(test.TestCase):
       params_t = constant_op.constant(params)
       indices = constant_op.constant([[[3], [2], [1]], [[4], [4], [0]]])
       gather_nd_t = array_ops.gather_nd(params_t, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     self.assertEqual([2, 3, 2, 2], gather_nd_t.get_shape())
     self.assertAllEqual(params[[3, 2, 1, 4, 4, 0]].reshape(2, 3, 2, 2),
@@ -168,7 +168,7 @@ class GatherNdTest(test.TestCase):
       params = np.random.rand(*shape)
       indices = np.vstack([np.random.randint(0, s, size=2000) for s in shape]).T
       gather_nd_t = array_ops.gather_nd(params, indices)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     expected = params[tuple(indices.T)]
     self.assertAllEqual(expected, gather_nd_val)
@@ -181,7 +181,7 @@ class GatherNdTest(test.TestCase):
       indices = np.vstack([np.random.randint(0, s, size=2000) for s in shape]).T
       indices_reshaped = indices.reshape([10, 10, 20, 5])
       gather_nd_t = array_ops.gather_nd(params, indices_reshaped)
-      gather_nd_val = gather_nd_t.eval()
+      gather_nd_val = self.evaluate(gather_nd_t)
 
     expected = params[tuple(indices.T)]
     self.assertAllEqual(expected.reshape([10, 10, 20]), gather_nd_val)
@@ -205,7 +205,7 @@ class GatherNdTest(test.TestCase):
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
           r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"):
-        gather_nd.eval()
+        self.evaluate(gather_nd)
 
   def _disabledTestBadIndicesGPU(self):
     # TODO disabled due to different behavior on GPU and CPU
@@ -218,7 +218,7 @@ class GatherNdTest(test.TestCase):
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
           r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"):
-        gather_nd.eval()
+        self.evaluate(gather_nd)
 
   def testBadIndicesWithSlicesCPU(self):
     with self.session(use_gpu=False):
@@ -227,7 +227,7 @@ class GatherNdTest(test.TestCase):
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
           r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"):
-        gather_nd.eval()
+        self.evaluate(gather_nd)
 
   def _disabledTestBadIndicesWithSlicesGPU(self):
     # TODO disabled due to different behavior on GPU and CPU
@@ -240,7 +240,7 @@ class GatherNdTest(test.TestCase):
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
           r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"):
-        gather_nd.eval()
+        self.evaluate(gather_nd)
 
   def testGradientsRank2Elements(self):
     indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32)
@@ -251,7 +251,7 @@ class GatherNdTest(test.TestCase):
     grads = gradients_impl.gradients([outputs], [inputs], [grad_vals])[0]
     expected_grads = np.array([[1, 0], [0, 2]], dtype=np.float64)
     with self.session(use_gpu=True):
-      assert np.array_equal(expected_grads, grads.eval())
+      assert np.array_equal(expected_grads, self.evaluate(grads))
 
   def testGradientsRank2Slices(self):
     indices = constant_op.constant([[1], [0]], dtype=dtypes.int32)
@@ -278,7 +278,7 @@ class GatherNdTest(test.TestCase):
     expected_grads = np.array(
         [[[5, 6], [1, 2]], [[3, 4], [7, 8]]], dtype=np.float64)
     with self.session(use_gpu=True):
-      self.assertAllEqual(expected_grads, grads.eval())
+      self.assertAllEqual(expected_grads, self.evaluate(grads))
 
   def testGradientsRank7Elements(self):
     # Shape [1,1,2,1,1,2,2]
@@ -307,7 +307,7 @@ class GatherNdTest(test.TestCase):
             [[[[3, 4], [7, 8]]]]
         ]]], dtype=np.float64)
     with self.session(use_gpu=True):
-      self.assertAllEqual(expected_grads, grads.eval())
+      self.assertAllEqual(expected_grads, self.evaluate(grads))
 
   def testGradientsInt64Indices(self):
     indices = constant_op.constant(
@@ -322,7 +322,7 @@ class GatherNdTest(test.TestCase):
     expected_grads = np.array(
         [[[5, 6], [1, 2]], [[3, 4], [7, 8]]], dtype=np.float64)
     with self.session(use_gpu=True):
-      self.assertAllEqual(expected_grads, grads.eval())
+      self.assertAllEqual(expected_grads, self.evaluate(grads))
 
   def testGradientsRank2SlicesWithEmptySpace(self):
     indices = constant_op.constant([[2], [0], [5]], dtype=dtypes.int32)
@@ -361,10 +361,10 @@ class GatherNdOpBenchmark(test.Benchmark):
       gather_op = array_ops.gather_nd(t_params, t_indices)
       variables.global_variables_initializer().run()
       for _ in range(10):
-        gather_op.eval()
+        self.evaluate(gather_op)
       t1 = time.time()
       for _ in range(1000):
-        gather_op.eval()
+        self.evaluate(gather_op)
       t2 = time.time()
       self.report_benchmark(iters=1000, wall_time=(t2 - t1) / 1000.0)
 
diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py
index bdafc52ab5..326e4aacd2 100644
--- a/tensorflow/python/kernel_tests/gather_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_op_test.py
@@ -50,7 +50,7 @@ class GatherTest(test.TestCase):
           params = constant_op.constant(params_np)
           indices_tf = constant_op.constant(indices)
           gather_t = array_ops.gather(params, indices_tf)
-          gather_val = gather_t.eval()
+          gather_val = self.evaluate(gather_t)
           np_val = params_np[indices]
           self.assertAllEqual(np_val, gather_val)
           self.assertEqual(np_val.shape, gather_t.get_shape())
@@ -65,7 +65,7 @@ class GatherTest(test.TestCase):
           params = constant_op.constant(params_np)
           indices = constant_op.constant(2)
           gather_t = array_ops.gather(params, indices, axis=axis)
-          gather_val = gather_t.eval()
+          gather_val = self.evaluate(gather_t)
           self.assertAllEqual(np.take(params_np, 2, axis=axis), gather_val)
           expected_shape = data.shape[:axis] + data.shape[axis + 1:]
           self.assertEqual(expected_shape, gather_t.get_shape())
@@ -81,7 +81,7 @@ class GatherTest(test.TestCase):
           # The indices must be in bounds for any axis.
           indices = constant_op.constant([0, 1, 0, 2])
           gather_t = array_ops.gather(params, indices, axis=axis)
-          gather_val = gather_t.eval()
+          gather_val = self.evaluate(gather_t)
           self.assertAllEqual(np.take(params_np, [0, 1, 0, 2], axis=axis),
                               gather_val)
           expected_shape = data.shape[:axis] + (4,) + data.shape[axis + 1:]
@@ -142,8 +142,11 @@ class GatherTest(test.TestCase):
               source_slice = ((slice(None),) * outer_dims + (source_index,) +
                               (slice(None),) * inner_dims)
               correct_params_grad[dest_slice] += gather_grad[source_slice]
-            self.assertAllClose(correct_params_grad, params_grad.eval(),
-                                atol=2e-6, rtol=2e-6)
+            self.assertAllClose(
+                correct_params_grad,
+                self.evaluate(params_grad),
+                atol=2e-6,
+                rtol=2e-6)
 
   def testString(self):
     params = np.array([[b"asdf", b"zxcv"], [b"qwer", b"uiop"]])
diff --git a/tensorflow/python/kernel_tests/huge_slice_op_test.py b/tensorflow/python/kernel_tests/huge_slice_op_test.py
index 8646d74c96..4074946350 100644
--- a/tensorflow/python/kernel_tests/huge_slice_op_test.py
+++ b/tensorflow/python/kernel_tests/huge_slice_op_test.py
@@ -33,11 +33,11 @@ class SliceTest(test.TestCase):
       a_large = array_ops.tile(
           constant_op.constant(np.array([False, True] * 4)), [2**29 + 3])
       slice_t = array_ops.slice(a_large, np.asarray([3]).astype(np.int64), [3])
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([True, False, True], slice_val)
 
       slice_t = array_ops.slice(
           a_large, constant_op.constant([long(2)**32 + 3], dtype=dtypes.int64),
           [3])
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([True, False, True], slice_val)
diff --git a/tensorflow/python/kernel_tests/in_topk_op_test.py b/tensorflow/python/kernel_tests/in_topk_op_test.py
index 6fdb497bc6..507822b314 100644
--- a/tensorflow/python/kernel_tests/in_topk_op_test.py
+++ b/tensorflow/python/kernel_tests/in_topk_op_test.py
@@ -32,7 +32,7 @@ class InTopKTest(test.TestCase):
     np_ans = np.array(expected)
     with self.cached_session():
       precision = nn_ops.in_top_k(predictions, target, k)
-      out = precision.eval()
+      out = self.evaluate(precision)
       self.assertAllClose(np_ans, out)
       self.assertShapeEqual(np_ans, precision)
 
@@ -77,7 +77,7 @@ class InTopKTest(test.TestCase):
     np_ans = np.array([False, True])
     with self.cached_session():
       precision = nn_ops.in_top_k(predictions, target, k)
-      out = precision.eval()
+      out = self.evaluate(precision)
       self.assertAllClose(np_ans, out)
       self.assertShapeEqual(np_ans, precision)
 
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 3f3fcfd2ff..a3f2c0ddd7 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -349,7 +349,7 @@ class UniformUnitScalingInitializationTest(test.TestCase):
           shape=shape,
           initializer=init_ops.uniform_unit_scaling_initializer())
       variables.global_variables_initializer().run()
-      self.assertAllEqual(shape, x.eval().shape)
+      self.assertAllEqual(shape, self.evaluate(x).shape)
 
   def testDuplicatedInitializer(self):
     init = init_ops.uniform_unit_scaling_initializer()
@@ -435,7 +435,7 @@ class RangeTest(test.TestCase):
       tf_ans = math_ops.range(start, limit, delta, name="range")
       self.assertEqual([len(np.arange(start, limit, delta))],
                        tf_ans.get_shape())
-      return tf_ans.eval()
+      return self.evaluate(tf_ans)
 
   def testBasic(self):
     self.assertTrue(
@@ -524,7 +524,7 @@ class LinSpaceTest(test.TestCase):
       with self.session(graph=graph, force_gpu=self.force_gpu):
         tf_ans = math_ops.linspace(start, stop, num, name="linspace")
         self.assertEqual([num], tf_ans.get_shape())
-        return tf_ans.eval()
+        return self.evaluate(tf_ans)
 
   def testPositive(self):
     for self.force_gpu in self._gpu_modes():
@@ -706,7 +706,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
         with self.session(use_gpu=True) as sess:
           sess.run(my_ops)
           # Check the shape of the outputs
-          t = outputs.eval()
+          t = self.evaluate(outputs)
           self.assertAllEqual(t.shape, outputs_shape)
           # Check isometry of the delta-orthogonal kernel.
           self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
@@ -723,7 +723,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
                                         initializer=
                                         init_ops.convolutional_delta_orthogonal)
         x.initializer.run()
-        y = x.eval()[1, 1, :, :]
+        y = self.evaluate(x)[1, 1, :, :]
         determinant = np.linalg.det(y)
         value += determinant
         abs_value += np.abs(determinant)
@@ -844,7 +844,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         sess.run(my_ops)
         # Check the shape of the outputs
-        t = outputs.eval()
+        t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
         self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
@@ -939,7 +939,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         sess.run(my_ops)
         # Check the shape of the outputs
-        t = outputs.eval()
+        t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
         self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
@@ -1064,7 +1064,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
       with self.cached_session(use_gpu=True) as sess:
         sess.run(my_ops)
         # Check the shape of the outputs
-        t = outputs.eval()
+        t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
         self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
diff --git a/tensorflow/python/kernel_tests/large_concat_op_test.py b/tensorflow/python/kernel_tests/large_concat_op_test.py
index 1b23e74776..bf6fa9ea71 100644
--- a/tensorflow/python/kernel_tests/large_concat_op_test.py
+++ b/tensorflow/python/kernel_tests/large_concat_op_test.py
@@ -35,7 +35,7 @@ class LargeConcatOpTest(test.TestCase):
     with self.session(use_gpu=False):
       # TODO(dga):  Add more depth to this test to validate correctness,
       # not just non-crashingness, once other large tensor fixes have gone in.
-      _ = onezeros.eval()
+      _ = self.evaluate(onezeros)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
index bd666c0db3..d5580d0e88 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
@@ -137,7 +137,8 @@ class LinearOperatorCirculantTestSelfAdjointOperator(
       matrix = operator.to_dense()
       imag_matrix = math_ops.imag(matrix)
       eps = np.finfo(np.float32).eps
-      np.testing.assert_allclose(0, imag_matrix.eval(), rtol=0, atol=eps * 3)
+      np.testing.assert_allclose(
+          0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3)
 
 
 class LinearOperatorCirculantTestHermitianSpectrum(
@@ -203,7 +204,8 @@ class LinearOperatorCirculantTestHermitianSpectrum(
       matrix = operator.to_dense()
       imag_matrix = math_ops.imag(matrix)
       eps = np.finfo(np.float32).eps
-      np.testing.assert_allclose(0, imag_matrix.eval(), rtol=0, atol=eps * 3)
+      np.testing.assert_allclose(
+          0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3)
 
 
 class LinearOperatorCirculantTestNonHermitianSpectrum(
@@ -257,7 +259,8 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       matrix = operator.to_dense()
       imag_matrix = math_ops.imag(matrix)
       eps = np.finfo(np.float32).eps
-      np.testing.assert_allclose(0, imag_matrix.eval(), rtol=0, atol=eps * 3)
+      np.testing.assert_allclose(
+          0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3)
 
   def test_simple_positive_real_spectrum_gives_self_adjoint_pos_def_oper(self):
     with self.cached_session() as sess:
@@ -299,7 +302,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       imag_matrix = math_ops.imag(matrix)
       eps = np.finfo(np.float32).eps
       np.testing.assert_allclose(
-          0, imag_matrix.eval(), rtol=0, atol=eps * 3 * 4)
+          0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3 * 4)
 
   def test_convolution_kernel_same_as_first_row_of_to_dense(self):
     spectrum = [[3., 2., 1.], [2., 1.5, 1.]]
@@ -310,7 +313,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
 
       self.assertAllEqual((2, 3), h.get_shape())
       self.assertAllEqual((2, 3, 3), c.get_shape())
-      self.assertAllClose(h.eval(), c.eval()[:, :, 0])
+      self.assertAllClose(h.eval(), self.evaluate(c)[:, :, 0])
 
   def test_assert_non_singular_fails_for_singular_operator(self):
     spectrum = math_ops.cast([0, 4, 2j + 2], dtypes.complex64)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index d47a21c1c3..45cd14ce04 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -83,7 +83,7 @@ class LinearOperatorIdentityTest(
           num_rows=2, dtype=dtypes.float16)
       x = rng.randn(2, 3).astype(np.float16)
       y = operator.matmul(x)
-      self.assertAllClose(x, y.eval())
+      self.assertAllClose(x, self.evaluate(y))
 
   def test_non_scalar_num_rows_raises_static(self):
     with self.assertRaisesRegexp(ValueError, "must be a 0-D Tensor"):
@@ -357,7 +357,7 @@ class LinearOperatorScaledIdentityTest(
           num_rows=2, multiplier=multiplier)
       x = rng.randn(2, 3).astype(np.float16)
       y = operator.matmul(x)
-      self.assertAllClose(multiplier[..., None, None] * x, y.eval())
+      self.assertAllClose(multiplier[..., None, None] * x, self.evaluate(y))
 
   def test_non_scalar_num_rows_raises_static(self):
     # Many "test_...num_rows" tests are performed in LinearOperatorIdentity.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
index df6954f520..2b1ae6e1f5 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
@@ -70,8 +70,8 @@ class KroneckerDenseTest(test.TestCase):
         [5., 10., -1., -2.]], dtype=dtypes.float32)
 
     with self.cached_session():
-      self.assertAllClose(_kronecker_dense([x, y]).eval(), z.eval())
-      self.assertAllClose(_kronecker_dense([y, x]).eval(), w.eval())
+      self.assertAllClose(_kronecker_dense([x, y]).eval(), self.evaluate(z))
+      self.assertAllClose(_kronecker_dense([y, x]).eval(), self.evaluate(w))
 
 
 class SquareLinearOperatorKroneckerTest(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
index 819347343b..4cb60dac98 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
@@ -134,7 +134,7 @@ class LinearOperatorTest(test.TestCase):
     with self.cached_session():
       operator_dense = operator.to_dense()
       self.assertAllEqual((2, 3, 4), operator_dense.get_shape())
-      self.assertAllClose(matrix, operator_dense.eval())
+      self.assertAllClose(matrix, self.evaluate(operator_dense))
 
   def test_generic_to_dense_method_non_square_matrix_tensor(self):
     matrix = rng.randn(2, 3, 4)
@@ -152,7 +152,7 @@ class LinearOperatorTest(test.TestCase):
     with self.cached_session():
       y = operator.matvec(x)
       self.assertAllEqual((2,), y.get_shape())
-      self.assertAllClose([1., 2.], y.eval())
+      self.assertAllClose([1., 2.], self.evaluate(y))
 
   def test_solvevec(self):
     matrix = [[1., 0], [0., 2.]]
@@ -161,7 +161,7 @@ class LinearOperatorTest(test.TestCase):
     with self.cached_session():
       x = operator.solvevec(y)
       self.assertAllEqual((2,), x.get_shape())
-      self.assertAllClose([1., 1 / 2.], x.eval())
+      self.assertAllClose([1., 1 / 2.], self.evaluate(x))
 
   def test_is_square_set_to_true_for_square_static_shapes(self):
     operator = LinearOperatorShape(shape=(2, 4, 4))
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
index 31fb19e4a6..5ce2616972 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
@@ -102,7 +102,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
     self.assertTrue(isinstance(tensor, ops.Tensor))
 
     with self.cached_session():
-      self.assertAllClose(arr, tensor.eval())
+      self.assertAllClose(arr, self.evaluate(tensor))
 
   def test_static_dims_broadcast(self):
     # x.batch_shape = [3, 1, 2]
@@ -205,7 +205,7 @@ class CholeskySolveWithBroadcastTest(test.TestCase):
       result = linear_operator_util.cholesky_solve_with_broadcast(chol, rhs)
       self.assertAllEqual((2, 3, 7), result.get_shape())
       expected = linalg_ops.cholesky_solve(chol_broadcast, rhs)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2, 2]
@@ -244,7 +244,7 @@ class MatmulWithBroadcastTest(test.TestCase):
       result = linear_operator_util.matmul_with_broadcast(x, y)
       self.assertAllEqual((2, 1, 7), result.get_shape())
       expected = math_ops.matmul(x, y_broadcast)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_y_has_extra_dims(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -261,7 +261,7 @@ class MatmulWithBroadcastTest(test.TestCase):
       result = linear_operator_util.matmul_with_broadcast(x, y)
       self.assertAllEqual((2, 3, 5, 5), result.get_shape())
       expected = math_ops.matmul(x_broadcast, y)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_y_has_extra_dims_transpose_a_and_b(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -280,7 +280,7 @@ class MatmulWithBroadcastTest(test.TestCase):
       self.assertAllEqual((2, 3, 5, 1), result.get_shape())
       expected = math_ops.matmul(
           x_broadcast, y, transpose_a=True, transpose_b=True)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_y_has_extra_dims_transpose_dynamic(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -344,7 +344,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
           matrix, rhs)
       self.assertAllEqual((2, 3, 7), result.get_shape())
       expected = linalg_ops.matrix_solve(matrix, rhs_broadcast)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_rhs_has_extra_dims(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -362,7 +362,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
       result = linear_operator_util.matrix_solve_with_broadcast(matrix, rhs)
       self.assertAllEqual((2, 3, 2), result.get_shape())
       expected = linalg_ops.matrix_solve(matrix_broadcast, rhs)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_rhs_has_extra_dims_dynamic(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -385,7 +385,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
       self.assertAllEqual(3, result.shape.ndims)
       expected = linalg_ops.matrix_solve(matrix_broadcast, rhs)
       self.assertAllClose(
-          expected.eval(),
+          self.evaluate(expected),
           result.eval(feed_dict={
               matrix_ph: matrix,
               rhs_ph: rhs
@@ -408,7 +408,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
           matrix, rhs, adjoint=True)
       self.assertAllEqual((2, 3, 2), result.get_shape())
       expected = linalg_ops.matrix_solve(matrix_broadcast, rhs, adjoint=True)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2, 2]
@@ -447,7 +447,7 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
           matrix, rhs)
       self.assertAllEqual((2, 3, 7), result.get_shape())
       expected = linalg_ops.matrix_triangular_solve(matrix, rhs_broadcast)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_rhs_has_extra_dims(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -466,7 +466,7 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
           matrix, rhs)
       self.assertAllEqual((2, 3, 2), result.get_shape())
       expected = linalg_ops.matrix_triangular_solve(matrix_broadcast, rhs)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_static_dims_broadcast_rhs_has_extra_dims_and_adjoint(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
@@ -486,7 +486,7 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
       self.assertAllEqual((2, 3, 2), result.get_shape())
       expected = linalg_ops.matrix_triangular_solve(
           matrix_broadcast, rhs, adjoint=True)
-      self.assertAllClose(expected.eval(), result.eval())
+      self.assertAllClose(expected.eval(), self.evaluate(result))
 
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2]
diff --git a/tensorflow/python/kernel_tests/linalg_grad_test.py b/tensorflow/python/kernel_tests/linalg_grad_test.py
index 03b640a85a..709ecbfc35 100644
--- a/tensorflow/python/kernel_tests/linalg_grad_test.py
+++ b/tensorflow/python/kernel_tests/linalg_grad_test.py
@@ -50,7 +50,7 @@ class ShapeTest(test_lib.TestCase):
       determinants = linalg_ops.matrix_determinant(batch_identity)
       reduced = math_ops.reduce_sum(determinants)
       sum_grad = gradients_impl.gradients(reduced, batch_identity)[0]
-      self.assertAllClose(batch_identity.eval(), sum_grad.eval())
+      self.assertAllClose(batch_identity.eval(), self.evaluate(sum_grad))
 
 
 class MatrixUnaryFunctorGradientTest(test_lib.TestCase):
@@ -69,7 +69,7 @@ def _GetMatrixUnaryFunctorGradientTest(functor_, dtype_, shape_, **kwargs_):
       if functor_.__name__ == 'matrix_square_root':
         # Square the input matrix to ensure that its matrix square root exists
         a = math_ops.matmul(a, a)
-        a_np = a.eval()
+        a_np = self.evaluate(a)
       b = functor_(a, **kwargs_)
 
       # Optimal stepsize for central difference is O(epsilon^{1/3}).
diff --git a/tensorflow/python/kernel_tests/linalg_ops_test.py b/tensorflow/python/kernel_tests/linalg_ops_test.py
index 28391aaa87..b5eeee0998 100644
--- a/tensorflow/python/kernel_tests/linalg_ops_test.py
+++ b/tensorflow/python/kernel_tests/linalg_ops_test.py
@@ -85,7 +85,7 @@ class LogdetTest(test.TestCase):
           #     [_RandomPDMatrix(n, self.rng, np_dtype),
           #      _RandomPDMatrix(n, self.rng, np_dtype)]).astype(np_dtype)
           logdet_tf = linalg.logdet(matrix)
-          self.assertAllClose(logdet_np, logdet_tf.eval(), atol=atol)
+          self.assertAllClose(logdet_np, self.evaluate(logdet_tf), atol=atol)
 
   def test_works_with_underflow_case(self):
     for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5),
@@ -94,7 +94,7 @@ class LogdetTest(test.TestCase):
       _, logdet_np = np.linalg.slogdet(matrix)
       with self.session(use_gpu=True):
         logdet_tf = linalg.logdet(matrix)
-        self.assertAllClose(logdet_np, logdet_tf.eval(), atol=atol)
+        self.assertAllClose(logdet_np, self.evaluate(logdet_tf), atol=atol)
 
 
 class SlogdetTest(test.TestCase):
@@ -110,8 +110,9 @@ class SlogdetTest(test.TestCase):
         sign_np, log_abs_det_np = np.linalg.slogdet(matrix)
         with self.session(use_gpu=True):
           sign_tf, log_abs_det_tf = linalg.slogdet(matrix)
-          self.assertAllClose(log_abs_det_np, log_abs_det_tf.eval(), atol=atol)
-          self.assertAllClose(sign_np, sign_tf.eval(), atol=atol)
+          self.assertAllClose(
+              log_abs_det_np, self.evaluate(log_abs_det_tf), atol=atol)
+          self.assertAllClose(sign_np, self.evaluate(sign_tf), atol=atol)
 
   def test_works_with_underflow_case(self):
     for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5),
@@ -120,8 +121,9 @@ class SlogdetTest(test.TestCase):
       sign_np, log_abs_det_np = np.linalg.slogdet(matrix)
       with self.session(use_gpu=True):
         sign_tf, log_abs_det_tf = linalg.slogdet(matrix)
-        self.assertAllClose(log_abs_det_np, log_abs_det_tf.eval(), atol=atol)
-        self.assertAllClose(sign_np, sign_tf.eval(), atol=atol)
+        self.assertAllClose(
+            log_abs_det_np, self.evaluate(log_abs_det_tf), atol=atol)
+        self.assertAllClose(sign_np, self.evaluate(sign_tf), atol=atol)
 
 
 class AdjointTest(test.TestCase):
@@ -135,7 +137,7 @@ class AdjointTest(test.TestCase):
         matrix = ops.convert_to_tensor(matrix_np)
         transposed = linalg.adjoint(matrix)
         self.assertEqual((3, 2), transposed.get_shape())
-        self.assertAllEqual(expected_transposed, transposed.eval())
+        self.assertAllEqual(expected_transposed, self.evaluate(transposed))
 
 
 class EyeTest(parameterized.TestCase, test.TestCase):
diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py
index 8e9b87f651..e8fa1cfa28 100644
--- a/tensorflow/python/kernel_tests/logging_ops_test.py
+++ b/tensorflow/python/kernel_tests/logging_ops_test.py
@@ -52,7 +52,7 @@ class LoggingOpsTest(test.TestCase):
               math_ops.less(epsilon, y), ["Divide-by-zero"])
       ]):
         out = math_ops.div(z, y)
-      self.assertAllEqual(2.0, out.eval())
+      self.assertAllEqual(2.0, self.evaluate(out))
       # assert(epsilon < x)
       # z / x
       #
@@ -63,7 +63,7 @@ class LoggingOpsTest(test.TestCase):
       ]):
         out = math_ops.div(z, x)
       with self.assertRaisesOpError("less than x"):
-        out.eval()
+        self.evaluate(out)
 
 
 class PrintV2Test(test.TestCase):
@@ -387,8 +387,8 @@ class PrintGradientTest(test.TestCase):
       wx_print = logging_ops.Print(wx, [w, w, w])
       wx_grad = gradients_impl.gradients(wx, w)[0]
       wx_print_grad = gradients_impl.gradients(wx_print, w)[0]
-      wxg = wx_grad.eval()
-      wxpg = wx_print_grad.eval()
+      wxg = self.evaluate(wx_grad)
+      wxpg = self.evaluate(wx_print_grad)
     self.assertAllEqual(wxg, wxpg)
 
 
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index bd93942efb..3efad4ea11 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -52,14 +52,14 @@ class HashTableOpTest(test.TestCase):
       output = table.lookup(input_string)
       self.assertAllEqual([3], output.get_shape())
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
       exported_keys_tensor, exported_values_tensor = table.export()
 
       self.assertItemsEqual([b"brain", b"salad", b"surgery"],
-                            exported_keys_tensor.eval())
-      self.assertItemsEqual([0, 1, 2], exported_values_tensor.eval())
+                            self.evaluate(exported_keys_tensor))
+      self.assertItemsEqual([0, 1, 2], self.evaluate(exported_values_tensor))
 
   def testHashTableFindHighRank(self):
     with self.cached_session():
@@ -76,7 +76,7 @@ class HashTableOpTest(test.TestCase):
           [["brain", "salad"], ["tank", "tarkus"]])
       output = table.lookup(input_string)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([[0, 1], [-1, -1]], result)
 
   def testHashTableInitWithPythonArrays(self):
@@ -94,7 +94,7 @@ class HashTableOpTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
   def testHashTableInitWithNumPyArrays(self):
@@ -111,7 +111,7 @@ class HashTableOpTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
   def testMultipleHashTables(self):
@@ -154,7 +154,7 @@ class HashTableOpTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
   def testHashTableWithSparseTensorInput(self):
@@ -221,7 +221,7 @@ class HashTableOpTest(test.TestCase):
       output = table.lookup(input_string)
 
       with self.assertRaisesOpError("Table not initialized"):
-        output.eval()
+        self.evaluate(output)
 
   def testInitializeTwice(self):
     with self.cached_session():
@@ -286,7 +286,7 @@ class HashTableOpTest(test.TestCase):
       input_tensor = constant_op.constant([0, 1, -1])
       output = table.lookup(input_tensor)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([b"brain", b"salad", b"n/a"], result)
 
 
@@ -306,9 +306,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_string_index_table_from_multicolumn_file(self):
     vocabulary_file = self._createVocabFile(
@@ -322,9 +322,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_string_index_table_from_multicolumn_file_custom_delimiter(self):
     vocabulary_file = self._createVocabFile(
@@ -339,9 +339,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_string_index_table_from_file_tensor_filename(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
@@ -352,9 +352,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
       self.assertEqual(1,
                        len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
 
@@ -367,11 +367,11 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
 
       feed_dict = {vocabulary_placeholder.name: vocabulary_file}
       lookup_ops.tables_initializer().run(feed_dict=feed_dict)
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
       self.assertEqual(0,
                        len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
 
@@ -387,9 +387,9 @@ class IndexTableFromFile(test.TestCase):
           constant_op.constant((1, -1000, 11), dtype=dtypes.int32))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_int64_index_table_from_file(self):
     vocabulary_file = self._createVocabFile(
@@ -403,9 +403,9 @@ class IndexTableFromFile(test.TestCase):
           constant_op.constant((1, -1000, 11), dtype=dtypes.int64))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_index_table_from_file_with_default_value(self):
     default_value = -42
@@ -416,9 +416,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, default_value), ids.eval())
+      self.assertAllEqual((1, 2, default_value), self.evaluate(ids))
 
   def test_index_table_from_file_with_oov_buckets(self):
     vocabulary_file = self._createVocabFile("f2i_vocab5.txt")
@@ -429,7 +429,7 @@ class IndexTableFromFile(test.TestCase):
           constant_op.constant(["salad", "surgery", "tarkus", "toccata"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
       self.assertAllEqual(
           (
@@ -437,7 +437,7 @@ class IndexTableFromFile(test.TestCase):
               2,  # From vocabulary file.
               867,  # 3 + fingerprint("tarkus") mod 300.
               860),  # 3 + fingerprint("toccata") mod 300.
-          ids.eval())
+          self.evaluate(ids))
 
   def test_index_table_from_file_fails_with_empty_vocabulary_file_name(self):
     self.assertRaises(
@@ -476,9 +476,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, -1, -1), ids.eval())
+      self.assertAllEqual((1, -1, -1), self.evaluate(ids))
       self.assertEqual(2, table.size().eval())
 
   def test_index_table_from_file_with_vocab_size_too_large(self):
@@ -504,9 +504,9 @@ class IndexTableFromFile(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, -1), ids.eval())
+      self.assertAllEqual((1, 2, -1), self.evaluate(ids))
       self.assertEqual(3, table.size().eval())
 
   def test_index_table_from_file_with_invalid_hashers(self):
@@ -614,9 +614,9 @@ class IndexTableFromTensor(test.TestCase):
           constant_op.constant((1, -1000, 11), dtype=dtypes.int32))
 
       with self.assertRaises(errors_impl.FailedPreconditionError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_int64_index_table_from_tensor_with_tensor_init(self):
     with self.cached_session():
@@ -626,9 +626,9 @@ class IndexTableFromTensor(test.TestCase):
           constant_op.constant((1, -1000, 11), dtype=dtypes.int64))
 
       with self.assertRaises(errors_impl.FailedPreconditionError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, 3), ids.eval())
+      self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
   def test_index_table_from_tensor_with_default_value(self):
     default_value = -42
@@ -639,9 +639,9 @@ class IndexTableFromTensor(test.TestCase):
       ids = table.lookup(constant_op.constant(["salad", "surgery", "tarkus"]))
 
       with self.assertRaises(errors_impl.FailedPreconditionError):
-        ids.eval()
+        self.evaluate(ids)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((1, 2, default_value), ids.eval())
+      self.assertAllEqual((1, 2, default_value), self.evaluate(ids))
 
   def test_index_table_from_tensor_missing_vocabulary_list(self):
     with self.cached_session():
@@ -656,7 +656,7 @@ class IndexTableFromTensor(test.TestCase):
           vocabulary_list=np.array([], dtype=np.str_), num_oov_buckets=1)
       ids = table.lookup(constant_op.constant(["salad", "surgery", "brain"]))
       with self.assertRaises(errors_impl.OpError):
-        ids.eval()
+        self.evaluate(ids)
       with self.assertRaisesRegexp(
           errors_impl.OpError, "keys and values cannot be empty"):
         lookup_ops.tables_initializer().run()
@@ -698,10 +698,10 @@ class IndexToStringTableFromFileTest(test.TestCase):
         features = table.lookup(
             constant_op.constant([0, 1, 2, 3], dtypes.int64))
         with self.assertRaises(errors_impl.OpError):
-          features.eval()
+          self.evaluate(features)
         lookup_ops.tables_initializer().run()
         self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
-                            features.eval())
+                            self.evaluate(features))
 
   def test_index_to_string_table_from_multicolumn_file(self):
     vocabulary_file = self._createVocabFile(
@@ -713,10 +713,10 @@ class IndexToStringTableFromFileTest(test.TestCase):
           value_column_index=0)
       features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64))
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
-                          features.eval())
+                          self.evaluate(features))
 
   def test_index_to_string_table_from_multicolumn_file_custom_delimiter(self):
     vocabulary_file = self._createVocabFile(
@@ -729,10 +729,10 @@ class IndexToStringTableFromFileTest(test.TestCase):
           delimiter=" ")
       features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64))
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
-                          features.eval())
+                          self.evaluate(features))
 
   def test_index_to_string_table_with_default_value(self):
     default_value = b"NONE"
@@ -742,10 +742,10 @@ class IndexToStringTableFromFileTest(test.TestCase):
           vocabulary_file=vocabulary_file, default_value=default_value)
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((b"salad", b"surgery", default_value),
-                          features.eval())
+                          self.evaluate(features))
 
   def test_index_to_string_table_with_vocab_size_too_small(self):
     default_value = b"NONE"
@@ -757,10 +757,10 @@ class IndexToStringTableFromFileTest(test.TestCase):
           default_value=default_value)
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((b"salad", default_value, default_value),
-                          features.eval())
+                          self.evaluate(features))
 
   def test_index_to_string_table_with_vocab_size_too_large(self):
     vocabulary_file = self._createVocabFile("f2i_vocab6.txt")
@@ -770,7 +770,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
 
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       init = lookup_ops.tables_initializer()
       self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                               "Invalid vocab_size", init.run)
@@ -783,9 +783,10 @@ class IndexToStringTableFromFileTest(test.TestCase):
       features = table.lookup(constant_op.constant([1, 2, 4], dtypes.int64))
 
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((b"salad", b"surgery", b"UNK"), features.eval())
+      self.assertAllEqual((b"salad", b"surgery", b"UNK"),
+                          self.evaluate(features))
 
 
 class IndexToStringTableFromTensorTest(test.TestCase):
@@ -799,11 +800,11 @@ class IndexToStringTableFromTensorTest(test.TestCase):
       indices = constant_op.constant([0, 1, 2, 3], dtypes.int64)
       features = table.lookup(indices)
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
       lookup_ops.tables_initializer().run()
 
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
-                          features.eval())
+                          self.evaluate(features))
 
   def test_duplicate_entries(self):
     with self.cached_session():
@@ -813,7 +814,7 @@ class IndexToStringTableFromTensorTest(test.TestCase):
       indices = constant_op.constant([0, 1, 4], dtypes.int64)
       features = table.lookup(indices)
       lookup_ops.tables_initializer().run()
-      self.assertAllEqual((b"hello", b"hello", b"UNK"), features.eval())
+      self.assertAllEqual((b"hello", b"hello", b"UNK"), self.evaluate(features))
 
   def test_index_to_string_with_default_value(self):
     default_value = b"NONE"
@@ -824,11 +825,11 @@ class IndexToStringTableFromTensorTest(test.TestCase):
       indices = constant_op.constant([1, 2, 4], dtypes.int64)
       features = table.lookup(indices)
       with self.assertRaises(errors_impl.OpError):
-        features.eval()
+        self.evaluate(features)
 
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((b"salad", b"surgery", default_value),
-                          features.eval())
+                          self.evaluate(features))
 
 
 class InitializeTableFromFileOpTest(test.TestCase):
@@ -870,7 +871,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       output = table.lookup(
           constant_op.constant((42, 1, 11), dtype=dtypes.int64))
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
   def testInitializeIndexTable(self):
@@ -889,7 +890,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
       output = table.lookup(input_values)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], result)
 
   def testMultiColumn(self):
@@ -911,7 +912,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "surgery"])
       output = table.lookup(input_string)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([1, 5, 6], result)
 
   def testInvalidDataTypeInMultiColumn(self):
@@ -1078,7 +1079,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "tank"])
       output = table.lookup(input_string)
 
-      result = output.eval()
+      result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
   def testInvalidFilenames(self):
@@ -1119,7 +1120,8 @@ class InitializeTableFromFileOpTest(test.TestCase):
       input_values = constant_op.constant([0, 1, 2, 3], dtypes.int64)
 
       out = table.lookup(input_values)
-      self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], out.eval())
+      self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"],
+                          self.evaluate(out))
       self.assertEquals(vocab_size, table.size().eval())
 
   def testStringToIdTable(self):
@@ -1135,7 +1137,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"])
 
       out = table.lookup(input_string)
-      self.assertAllEqual([0, 1, 2, -1], out.eval())
+      self.assertAllEqual([0, 1, 2, -1], self.evaluate(out))
       self.assertEquals(vocab_size, table.size().eval())
 
   def testInt64ToIdTable(self):
@@ -1152,7 +1154,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
 
       out = table.lookup(
           constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64))
-      self.assertAllEqual((0, 1, 2, -1), out.eval())
+      self.assertAllEqual((0, 1, 2, -1), self.evaluate(out))
       self.assertEquals(vocab_size, table.size().eval())
 
 
@@ -1181,7 +1183,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       input_string = constant_op.constant(["brain", "salad", "surgery", "UNK"])
 
       out = table.lookup(input_string)
-      self.assertAllEqual([0, 1, 2, 3], out.eval())
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(out))
       self.assertEquals(vocab_size + oov_buckets, table.size().eval())
 
   def testInt32IdTableWithHashBuckets(self):
@@ -1203,7 +1205,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       values = constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int32)
 
       out = table.lookup(values)
-      self.assertAllEqual([0, 1, 2, 3], out.eval())
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(out))
       self.assertEquals(vocab_size + oov_buckets, table.size().eval())
 
   def testInt64IdTableWithHashBuckets(self):
@@ -1223,7 +1225,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       values = constant_op.constant((42, 1, -1000, 11), dtype=dtypes.int64)
 
       out = table.lookup(values)
-      self.assertAllEqual([0, 1, 2, 3], out.eval())
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(out))
       self.assertEquals(vocab_size + oov_buckets, table.size().eval())
 
   def testStringIdTableWithOnlyHashBucket(self):
@@ -1244,7 +1246,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
               1,  # fingerprint("salad") mod 5.
               4  # fingerprint("surgery") mod 5
           ],
-          out.eval())
+          self.evaluate(out))
       self.assertEquals(oov_buckets, table.size().eval())
 
   def testInt32IdTableWithOnlyHashBucket(self):
@@ -1266,7 +1268,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
               4,  # fingerprint("1") mod 5.
               2  # fingerprint("-1000") mod 5
           ],
-          out.eval())
+          self.evaluate(out))
       self.assertEquals(oov_buckets, table.size().eval())
 
   def testFloat64IdTableWithOnlyHashBucket(self):
@@ -1342,7 +1344,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
 
       out1 = table1.lookup(input_string_1)
 
-      self.assertAllEqual([0, 1, 2, 3], out1.eval())
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(out1))
       self.assertEquals(vocab_size + oov_buckets, table1.size().eval())
 
     with self.cached_session():
@@ -1363,7 +1365,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
 
       out2 = table2.lookup(input_string_2)
 
-      self.assertAllEqual([3, 1, 3], out2.eval())
+      self.assertAllEqual([3, 1, 3], self.evaluate(out2))
       self.assertEquals(vocab_size + oov_buckets, table2.size().eval())
 
   def testIdTableWithHashBucketsWithMultipleInitializersDifferentDefault(self):
diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py
index b04996f788..d3a907852a 100644
--- a/tensorflow/python/kernel_tests/losses_test.py
+++ b/tensorflow/python/kernel_tests/losses_test.py
@@ -54,55 +54,55 @@ class AbsoluteDifferenceLossTest(test.TestCase):
   def testAllCorrectNoLossWeight(self):
     loss = losses.absolute_difference(self._predictions, self._predictions)
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testNonZeroLoss(self):
     loss = losses.absolute_difference(self._labels, self._predictions)
     with self.cached_session():
-      self.assertAlmostEqual(5.5, loss.eval(), 3)
+      self.assertAlmostEqual(5.5, self.evaluate(loss), 3)
 
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(5.5 * weights, loss.eval(), 3)
+      self.assertAlmostEqual(5.5 * weights, self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.absolute_difference(self._labels, self._predictions,
                                       constant_op.constant(weights))
     with self.cached_session():
-      self.assertAlmostEqual(5.5 * weights, loss.eval(), 3)
+      self.assertAlmostEqual(5.5 * weights, self.evaluate(loss), 3)
 
   def testNonZeroLossWithOneDimBatchSpecificWeights(self):
     weights = constant_op.constant((1.2, 0.0), shape=(2, 1))
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(5.6, loss.eval(), 3)
+      self.assertAlmostEqual(5.6, self.evaluate(loss), 3)
 
   def testNonZeroLossWithTwoDimBatchSpecificWeights(self):
     weights = constant_op.constant([1.2, 0.0], shape=[2, 1])
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(5.6, loss.eval(), 3)
+      self.assertAlmostEqual(5.6, self.evaluate(loss), 3)
 
   def testNonZeroLossWithSampleSpecificWeights(self):
     weights = constant_op.constant([3, 6, 5, 0, 4, 2], shape=[2, 3])
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(16.6, loss.eval(), 3)
+      self.assertAlmostEqual(16.6, self.evaluate(loss), 3)
 
   def testNonZeroLossWithSampleSpecificWeightsMostZero(self):
     weights = constant_op.constant([0, 0, 0, 0, 0, 2], shape=[2, 3])
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(6.0, loss.eval(), 3)
+      self.assertAlmostEqual(6.0, self.evaluate(loss), 3)
 
   def testLossWithSampleSpecificWeightsAllZero(self):
     weights = array_ops.zeros((2, 3))
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   @test_util.assert_no_new_pyobjects_executing_eagerly
   def testEagerNoMemoryLeaked(self):
@@ -149,7 +149,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
     weights = 2.3
     with self.cached_session():
       loss = losses.softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual(weights * 10.0, loss.eval(), 3)
+      self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -159,7 +159,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
     with self.cached_session():
       loss = losses.softmax_cross_entropy(labels, logits,
                                           constant_op.constant(weights))
-      self.assertAlmostEqual(weights * 10.0, loss.eval(), 3)
+      self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithOneDimBatchSpecificWeights(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -168,7 +168,8 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant((1.2, 3.4, 5.6))
     with self.cached_session():
       loss = losses.softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3)
+      self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0,
+                             self.evaluate(loss), 3)
 
   def testAllWrongAllWeightsMissing(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -177,7 +178,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant([0, 0, 0], shape=[3])
     with self.cached_session():
       loss = losses.softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testSomeWeightsMissing(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -186,7 +187,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant([1.2, 0, 0], shape=[3])
     with self.cached_session():
       loss = losses.softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual(12.0, loss.eval(), 3)
+      self.assertAlmostEqual(12.0, self.evaluate(loss), 3)
 
   def testSoftmaxWithMeasurementSpecificWeightsRaisesException(self):
     with self.cached_session():
@@ -302,7 +303,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     weights = 2.3
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual(weights * 10.0, loss.eval(), 3)
+      self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -312,7 +313,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(labels, logits,
                                                  constant_op.constant(weights))
-      self.assertAlmostEqual(weights * 10.0, loss.eval(), 3)
+      self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWith1DTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -322,7 +323,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(
           labels, logits, constant_op.constant((weights,)))
-      self.assertAlmostEqual(weights * 10.0, loss.eval(), 3)
+      self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithPlaceholderForWeights(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0],
@@ -374,7 +375,8 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant([1.2, 3.4, 5.6], shape=(3, 1))
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3)
+      self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0,
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithColumnWeights(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -383,7 +385,8 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant([[1.2], [3.4], [5.6]])
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss.eval(), 3)
+      self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0,
+                             self.evaluate(loss), 3)
 
   def testAllWrongAllWeightsMissing(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -392,7 +395,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant([0, 0, 0], shape=(3, 1))
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testSomeWeightsMissing(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -401,7 +404,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     weights = constant_op.constant([1.2, 0, 0], shape=(3, 1))
     with self.cached_session():
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
-      self.assertAlmostEqual(12.0, loss.eval(), 3)
+      self.assertAlmostEqual(12.0, self.evaluate(loss), 3)
 
   def testMeasurementSpecificWeightsRaisesException(self):
     with self.cached_session():
@@ -481,7 +484,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       loss = losses.sigmoid_cross_entropy(labels, logits)
       self.assertEquals(logits.dtype, loss.dtype)
       self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name)
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testLossWithSingleDimPlaceholderForLogitsAndWeights1(self):
     logits = array_ops.placeholder(dtypes.float32, shape=(None, 1))
@@ -536,7 +539,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       loss = losses.sigmoid_cross_entropy(labels, logits, weights)
       self.assertEquals(logits.dtype, loss.dtype)
       self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name)
-      self.assertAlmostEqual(1700.0 / 7.0, loss.eval(), 3)
+      self.assertAlmostEqual(1700.0 / 7.0, self.evaluate(loss), 3)
 
   def testMultiCorrectSigmoid(self):
     logits = constant_op.constant([[100.0, -100.0, 100.0],
@@ -548,7 +551,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
     self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name)
 
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testSigmoidFloat64(self):
     logits = constant_op.constant((
@@ -563,7 +566,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
     self.assertEquals(logits.dtype, loss.dtype)
 
     with self.cached_session():
-      self.assertAlmostEqual(44.444, loss.eval(), 3)
+      self.assertAlmostEqual(44.444, self.evaluate(loss), 3)
 
   def testSigmoidNoReduction(self):
     logits = constant_op.constant((
@@ -576,11 +579,8 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
     self.assertEquals(logits.dtype, loss.dtype)
 
     with self.cached_session():
-      self.assertAllClose((
-          (0., 0., 0.),
-          (0., 100., 100.),
-          (100., 0., 100.)
-      ), loss.eval(), 3)
+      self.assertAllClose(((0., 0., 0.), (0., 100., 100.), (100., 0., 100.)),
+                          self.evaluate(loss), 3)
 
   def testSigmoidLabelSmoothingCorrect(self):
     with self.cached_session():
@@ -619,7 +619,8 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       softmax_labels = constant_op.constant([[0, 1], [1, 0], [0, 1]])
       softmax_loss = losses.softmax_cross_entropy(
           softmax_labels, softmax_logits, label_smoothing=label_smoothing)
-      self.assertAlmostEqual(sigmoid_loss.eval(), softmax_loss.eval(), 3)
+      self.assertAlmostEqual(sigmoid_loss.eval(), self.evaluate(softmax_loss),
+                             3)
 
 
 class LogLossTest(test.TestCase):
@@ -648,7 +649,7 @@ class LogLossTest(test.TestCase):
   def testAllCorrectNoLossWeight(self):
     loss = losses.log_loss(self._labels, self._labels)
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testAllCorrectNoLossWeightWithPlaceholder(self):
     tf_predictions = array_ops.placeholder(
@@ -662,14 +663,14 @@ class LogLossTest(test.TestCase):
     loss = losses.log_loss(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(-np.sum(self._expected_losses) / 6.0,
-                             loss.eval(), 3)
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.log_loss(self._labels, self._predictions, weights)
     with self.cached_session():
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
-                             loss.eval(), 3)
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
@@ -677,7 +678,7 @@ class LogLossTest(test.TestCase):
                            constant_op.constant(weights))
     with self.cached_session():
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
-                             loss.eval(), 3)
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarTensorWeightAndPlaceholder(self):
     tf_predictions = array_ops.placeholder(
@@ -707,7 +708,8 @@ class LogLossTest(test.TestCase):
         np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
     loss = losses.log_loss(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(-np.sum(expected_losses) / 6.0, loss.eval(), 3)
+      self.assertAlmostEqual(-np.sum(expected_losses) / 6.0,
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithOneDimBatchSpecificWeightsSomeZero(self):
     weights = constant_op.constant((1.2, 0), shape=(2, 1))
@@ -716,7 +718,8 @@ class LogLossTest(test.TestCase):
                                       (2, 3)))
     loss = losses.log_loss(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, loss.eval(), 3)
+      self.assertAlmostEqual(-np.sum(expected_losses) / 3.0,
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithTwoDimBatchSpecificWeightsSomeZero(self):
     weights = constant_op.constant([1.2, 0], shape=[2, 1])
@@ -725,7 +728,8 @@ class LogLossTest(test.TestCase):
                                       (2, 3)))
     loss = losses.log_loss(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(-np.sum(expected_losses) / 3.0, loss.eval(), 3)
+      self.assertAlmostEqual(-np.sum(expected_losses) / 3.0,
+                             self.evaluate(loss), 3)
 
   def testWeightsWithSameNumDimsButWrongShapeThrowsException(self):
     weights = constant_op.constant(np.random.normal(size=(2, 4)), shape=[2, 4])
@@ -743,7 +747,8 @@ class LogLossTest(test.TestCase):
         constant_op.constant(
             weights, shape=(2, 3)))
     with self.cached_session():
-      self.assertAlmostEqual(-np.sum(expected_losses) / 5.0, loss.eval(), 3)
+      self.assertAlmostEqual(-np.sum(expected_losses) / 5.0,
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithMeasurementSpecificWeightsWithPlaceholder(self):
     weights = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3))
@@ -770,7 +775,7 @@ class LogLossTest(test.TestCase):
         constant_op.constant(
             weights, shape=(2, 3)))
     with self.cached_session():
-      self.assertAlmostEqual(-np.sum(expected_losses), loss.eval(), 3)
+      self.assertAlmostEqual(-np.sum(expected_losses), self.evaluate(loss), 3)
 
   def testNonZeroLossWithSampleSpecificWeightsMostZeroWithPlaceholder(self):
     weights = np.array([0, 0, 0, 0, 0, 2]).reshape((2, 3))
@@ -788,7 +793,7 @@ class LogLossTest(test.TestCase):
     tf_weights = array_ops.zeros(shape=(2, 3))
     loss = losses.log_loss(self._labels, self._predictions, tf_weights)
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
 
 class HingeLossTest(test.TestCase):
@@ -870,7 +875,7 @@ class HuberLossTest(test.TestCase):
       labels = constant_op.constant([1.0, -1.0, 0.0, 0.5])
       expected = 0.5 * np.array([0.5**2, 0.4**2, 0.5**2, 0.5**2]).mean()
       loss = losses.huber_loss(labels, predictions, delta=delta)
-      self.assertAllClose(expected, loss.eval(), atol=1e-5)
+      self.assertAllClose(expected, self.evaluate(loss), atol=1e-5)
 
   def testAllLinearDelta(self):
     delta = 0.5
@@ -880,7 +885,7 @@ class HuberLossTest(test.TestCase):
     expected -= 0.5 * delta**2
     loss = losses.huber_loss(labels, predictions, delta=delta)
     with self.cached_session():
-      self.assertAllClose(expected, loss.eval(), atol=1e-5)
+      self.assertAllClose(expected, self.evaluate(loss), atol=1e-5)
 
 
 class MeanSquaredErrorTest(test.TestCase):
@@ -906,55 +911,55 @@ class MeanSquaredErrorTest(test.TestCase):
   def testAllCorrectNoLossWeight(self):
     loss = losses.mean_squared_error(self._predictions, self._predictions)
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
   def testNonZeroLoss(self):
     loss = losses.mean_squared_error(self._labels, self._predictions)
     with self.cached_session():
-      self.assertAlmostEqual(49.5, loss.eval(), 3)
+      self.assertAlmostEqual(49.5, self.evaluate(loss), 3)
 
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(49.5 * weights, loss.eval(), 3)
+      self.assertAlmostEqual(49.5 * weights, self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.mean_squared_error(self._labels, self._predictions,
                                      constant_op.constant(weights))
     with self.cached_session():
-      self.assertAlmostEqual(49.5 * weights, loss.eval(), 3)
+      self.assertAlmostEqual(49.5 * weights, self.evaluate(loss), 3)
 
   def testNonZeroLossWithOneDimBatchSpecificWeights(self):
     weights = constant_op.constant([1.2, 3.4], shape=(2, 1))
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(767.8 / 6.0, loss.eval(), 3)
+      self.assertAlmostEqual(767.8 / 6.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithTwoDimBatchSpecificWeights(self):
     weights = constant_op.constant([1.2, 3.4], shape=[2, 1])
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(767.8 / 6.0, loss.eval(), 3)
+      self.assertAlmostEqual(767.8 / 6.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithSampleSpecificWeights(self):
     weights = constant_op.constant([3, 6, 5, 0, 4, 2], shape=[2, 3])
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(587 / 5.0, loss.eval(), 3)
+      self.assertAlmostEqual(587 / 5.0, self.evaluate(loss), 3)
 
   def testNonZeroLossWithSampleSpecificWeightsMostZero(self):
     weights = constant_op.constant([0, 0, 0, 0, 0, 2], shape=[2, 3])
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(18.0, loss.eval(), 3)
+      self.assertAlmostEqual(18.0, self.evaluate(loss), 3)
 
   def testLossWithSampleSpecificWeightsAllZero(self):
     weights = array_ops.zeros((2, 3))
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
-      self.assertAlmostEqual(0.0, loss.eval(), 3)
+      self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
 
 class MeanPairwiseSquaredErrorTest(test.TestCase):
@@ -991,7 +996,8 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
     with self.cached_session():
       static_inputs_op = losses.mean_pairwise_squared_error(
           predictions=predictions, labels=labels, weights=weights)
-      self.assertAlmostEqual(expected_loss, static_inputs_op.eval(), places=3)
+      self.assertAlmostEqual(
+          expected_loss, self.evaluate(static_inputs_op), places=3)
 
       predictions_placeholder = array_ops.placeholder(
           dtypes.float32, shape=np.asarray(predictions.shape))
@@ -1060,7 +1066,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
         weights=constant_op.constant(weights))
     with self.cached_session():
       self.assertAlmostEqual(weights * np.sum(self._expected_losses),
-                             loss.eval(), 3)
+                             self.evaluate(loss), 3)
 
   def testNonZeroLossWithScalarZeroWeight(self):
     self._test_valid_weights(
@@ -1215,7 +1221,7 @@ class CosineDistanceLossTest(test.TestCase):
         labels=constant_op.constant(self._labels),
         dim=2)
     with self.cached_session():
-      self.assertAlmostEqual(0, loss.eval(), 5)
+      self.assertAlmostEqual(0, self.evaluate(loss), 5)
 
   def testPartiallyCorrectWithIntegerValues(self):
     loss = losses.cosine_distance(
@@ -1223,7 +1229,7 @@ class CosineDistanceLossTest(test.TestCase):
         labels=constant_op.constant(self._labels),
         dim=2)
     with self.cached_session():
-      self.assertAlmostEqual(1, loss.eval(), 5)
+      self.assertAlmostEqual(1, self.evaluate(loss), 5)
 
   def testPartiallyCorrectFloatingPointValues(self):
     predictions = np.matrix(
@@ -1241,7 +1247,7 @@ class CosineDistanceLossTest(test.TestCase):
     loss = losses.cosine_distance(tf_labels, tf_preds, dim=2)
 
     with self.cached_session():
-      self.assertAlmostEqual(1.0, loss.eval(), 5)
+      self.assertAlmostEqual(1.0, self.evaluate(loss), 5)
 
   def testSampleSpecificWeights(self):
     loss = losses.cosine_distance(
@@ -1250,7 +1256,7 @@ class CosineDistanceLossTest(test.TestCase):
         dim=2,
         weights=np.asarray((1, 0, 0)).reshape((3, 1, 1)))
     with self.cached_session():
-      self.assertEqual(1.0, loss.eval())
+      self.assertEqual(1.0, self.evaluate(loss))
 
   def testMeasurementSpecificWeights(self):
     loss = losses.cosine_distance(
@@ -1260,7 +1266,7 @@ class CosineDistanceLossTest(test.TestCase):
         weights=constant_op.constant(
             [1, 0, 0, 1, 1, 1], shape=(3, 2, 1)))
     with self.cached_session():
-      self.assertEqual(3.0 / 4.0, loss.eval())
+      self.assertEqual(3.0 / 4.0, self.evaluate(loss))
 
   def testMeasurementSpecificWeightsWithPlaceholderWithShape(self):
     tf_predictions = array_ops.placeholder(
@@ -1282,7 +1288,7 @@ class CosineDistanceLossTest(test.TestCase):
         dim=2,
         weights=array_ops.zeros((3, 1, 1)))
     with self.cached_session():
-      self.assertEqual(0, loss.eval())
+      self.assertEqual(0, self.evaluate(loss))
 
   def testZeroLossWhenAllMeasurementSpecificWeightsAreZero(self):
     loss = losses.cosine_distance(
@@ -1291,7 +1297,7 @@ class CosineDistanceLossTest(test.TestCase):
         dim=2,
         weights=array_ops.zeros((3, 2, 1)))
     with self.cached_session():
-      self.assertEqual(0, loss.eval())
+      self.assertEqual(0, self.evaluate(loss))
 
 
 class AddLossTest(test.TestCase):
@@ -1351,15 +1357,16 @@ class ComputeWeightedLossTest(test.TestCase):
         with self.session(g):
           for unweighted_loss in unweighted_losses:
             if reduction == losses.Reduction.NONE:
-              self.assertAllClose(self._raw_losses, unweighted_loss.eval())
+              self.assertAllClose(self._raw_losses,
+                                  self.evaluate(unweighted_loss))
             elif reduction == losses.Reduction.SUM:
               self.assertAllClose(
-                  np.sum(self._raw_losses), unweighted_loss.eval())
+                  np.sum(self._raw_losses), self.evaluate(unweighted_loss))
             else:
               # reduction one of MEAN, SUM_OVER_NONZERO_WEIGHTS,
               # SUM_BY_NONZERO_WEIGHTS or SUM_OVER_BATCH_SIZE.
               self.assertAllClose(
-                  np.mean(self._raw_losses), unweighted_loss.eval())
+                  np.mean(self._raw_losses), self.evaluate(unweighted_loss))
 
   def testUnweightedFromPlaceholder(self):
     for reduction in losses.Reduction.all():
@@ -1398,7 +1405,7 @@ class ComputeWeightedLossTest(test.TestCase):
       self.assertEqual(1, len(util.get_losses()))
       with self.cached_session():
         self.assertAllClose(
-            np.mean(weight * self._raw_losses), weighted_loss.eval())
+            np.mean(weight * self._raw_losses), self.evaluate(weighted_loss))
 
   def _test_invalid_weights(self, weights):
     with ops.Graph().as_default():
@@ -1470,24 +1477,22 @@ class ComputeWeightedLossTest(test.TestCase):
           weighted_losses = weights * self._raw_losses
           weighted_sum = np.sum(weighted_losses)
           if reduction == losses.Reduction.NONE:
-            self.assertAllClose(weighted_losses, weighted_loss.eval())
+            self.assertAllClose(weighted_losses, self.evaluate(weighted_loss))
           elif reduction == losses.Reduction.SUM:
-            self.assertAllClose(weighted_sum, weighted_loss.eval())
+            self.assertAllClose(weighted_sum, self.evaluate(weighted_loss))
           else:
             broadcast_weights = weights * np.ones_like(self._raw_losses)
             if reduction == losses.Reduction.MEAN:
-              self.assertAllClose(
-                  weighted_sum / np.sum(broadcast_weights),
-                  weighted_loss.eval())
+              self.assertAllClose(weighted_sum / np.sum(broadcast_weights),
+                                  self.evaluate(weighted_loss))
             elif (reduction == losses.Reduction.SUM_OVER_NONZERO_WEIGHTS or
                   reduction == losses.Reduction.SUM_BY_NONZERO_WEIGHTS):
               self.assertAllClose(
                   weighted_sum / np.count_nonzero(broadcast_weights),
-                  weighted_loss.eval())
+                  self.evaluate(weighted_loss))
             elif reduction == losses.Reduction.SUM_OVER_BATCH_SIZE:
-              self.assertAllClose(
-                  weighted_sum / self._raw_losses.size,
-                  weighted_loss.eval())
+              self.assertAllClose(weighted_sum / self._raw_losses.size,
+                                  self.evaluate(weighted_loss))
 
   def test1x1x1Weight(self):
     self._test_valid_weights((((17.0,),),))
diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py
index 1c2822180a..6167e01864 100644
--- a/tensorflow/python/kernel_tests/matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/matmul_op_test.py
@@ -44,7 +44,7 @@ class MatVecTest(test_lib.TestCase):
     with self.cached_session():
       c = math_ops.matvec(a, b)
       self.assertAllEqual((2,), c.shape)
-      c_ = c.eval()
+      c_ = self.evaluate(c)
     self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c_)
 
 
@@ -90,7 +90,7 @@ def _GetMatMulTest(a_np_, b_np_, use_static_shape_, **kwargs_):
         a = constant_op.constant(effective_a_np)
         b = constant_op.constant(effective_b_np)
         res = math_ops.matmul(a, b, **kwargs_)
-        tf_val = res.eval()
+        tf_val = self.evaluate(res)
       else:
         a = array_ops.placeholder(a_np_.dtype)
         b = array_ops.placeholder(b_np_.dtype)
@@ -220,7 +220,7 @@ class MatMulInfixOperatorTest(test_lib.TestCase):
     c = infix_matmul(a, b)
     d = math_ops.matmul(a, b)
     with self.cached_session():
-      self.assertAllEqual(c.eval(), d.eval())
+      self.assertAllEqual(c.eval(), self.evaluate(d))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/matrix_band_part_op_test.py b/tensorflow/python/kernel_tests/matrix_band_part_op_test.py
index 93a668f125..129ea40dfe 100644
--- a/tensorflow/python/kernel_tests/matrix_band_part_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_band_part_op_test.py
@@ -62,7 +62,7 @@ def _GetMatrixBandPartTest(dtype_, batch_shape_, shape_):
                 batch_mat,
                 constant_op.constant(lower, index_dtype),
                 constant_op.constant(upper, index_dtype))
-            self.assertAllEqual(band_np, band.eval())
+            self.assertAllEqual(band_np, self.evaluate(band))
 
   return Test
 
diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
index 3abdf50ece..7fe6cd4141 100644
--- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
@@ -61,7 +61,7 @@ class ExponentialOpTest(test.TestCase):
             np_ans[i] = np_expm(inp[i])
         else:
           np_ans = np_expm(inp)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-3)
 
   def _verifyExponentialReal(self, x):
diff --git a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
index 2247f1541e..434458721c 100644
--- a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
@@ -46,7 +46,7 @@ class InverseOpTest(test.TestCase):
           tiling = list(y.shape)
           tiling[-2:] = [1, 1]
           np_ans = np.tile(np_ans, tiling)
-        out = tf_ans.eval()
+        out = self.evaluate(tf_ans)
         self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-3)
         self.assertShapeEqual(y, tf_ans)
 
diff --git a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
index 2010a4b2a8..102502ae0d 100644
--- a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
@@ -43,7 +43,7 @@ class LogarithmOpTest(test.TestCase):
       # Verify that expm(logm(A)) == A.
       tf_ans = linalg_impl.matrix_exponential(
           gen_linalg_ops.matrix_logarithm(inp))
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       self.assertAllClose(inp, out, rtol=1e-4, atol=1e-3)
 
   def _verifyLogarithmComplex(self, x):
diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
index 9e30ae1628..1334d0c4ce 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
@@ -63,7 +63,7 @@ class MatrixSolveOpTest(test.TestCase):
               out = sess.run(tf_ans, {a_ph: a, b_ph: b})
             else:
               tf_ans = linalg_ops.matrix_solve(a, b, adjoint=adjoint)
-              out = tf_ans.eval()
+              out = self.evaluate(tf_ans)
               self.assertEqual(tf_ans.get_shape(), out.shape)
             self.assertEqual(np_ans.shape, out.shape)
             self.assertAllClose(np_ans, out, atol=tol, rtol=tol)
diff --git a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py
index 445faca3ee..317b8f8716 100644
--- a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py
@@ -87,7 +87,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
           b_tf = constant_op.constant(b)
           tf_ans = linalg_ops.matrix_triangular_solve(
               a_tf, b_tf, lower=lower, adjoint=adjoint)
-          tf_val = tf_ans.eval()
+          tf_val = self.evaluate(tf_ans)
           np_ans = np.linalg.solve(a_np, b)
           self.assertEqual(np_ans.shape, tf_ans.get_shape())
         self.assertEqual(np_ans.shape, tf_val.shape)
diff --git a/tensorflow/python/kernel_tests/morphological_ops_test.py b/tensorflow/python/kernel_tests/morphological_ops_test.py
index 6d601554b8..4ee04209cc 100644
--- a/tensorflow/python/kernel_tests/morphological_ops_test.py
+++ b/tensorflow/python/kernel_tests/morphological_ops_test.py
@@ -52,7 +52,7 @@ class DilationTest(test.TestCase):
           rates=rates,
           padding=padding,
           name="dilation2d")
-      self.assertAllClose(out, out_tensor.eval())
+      self.assertAllClose(out, self.evaluate(out_tensor))
 
   def _testDilationValidPadding(self, use_gpu):
     # [1, 2, 2, 1]
@@ -216,7 +216,7 @@ class DilationTest(test.TestCase):
           rates=rates,
           padding=padding,
           name="dilation2d")
-      out_shape = out_tensor.eval().shape
+      out_shape = self.evaluate(out_tensor).shape
 
       # Small delta is necessary for argmax to remain the same.
       err = gradient_checker.compute_gradient_error(
@@ -327,7 +327,7 @@ class ErosionTest(test.TestCase):
           rates=rates,
           padding=padding,
           name="erosion2d")
-      self.assertAllClose(out, out_tensor.eval())
+      self.assertAllClose(out, self.evaluate(out_tensor))
 
   def _testErosionValidPadding(self, use_gpu):
     # [1, 2, 2, 1]
@@ -491,7 +491,7 @@ class ErosionTest(test.TestCase):
           rates=rates,
           padding=padding,
           name="erosion2d")
-      out_shape = out_tensor.eval().shape
+      out_shape = self.evaluate(out_tensor).shape
 
       # Small delta is necessary for argmax to remain the same.
       err = gradient_checker.compute_gradient_error(
diff --git a/tensorflow/python/kernel_tests/numerics_test.py b/tensorflow/python/kernel_tests/numerics_test.py
index 5db591ed30..d25d97349d 100644
--- a/tensorflow/python/kernel_tests/numerics_test.py
+++ b/tensorflow/python/kernel_tests/numerics_test.py
@@ -39,7 +39,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
       t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32)
       t_verified = numerics.verify_tensor_all_finite(t,
                                                      "Input is not a number.")
-      self.assertAllClose(x, t_verified.eval())
+      self.assertAllClose(x, self.evaluate(t_verified))
 
   def testVerifyTensorAllFiniteFails(self):
     x_shape = [5, 4]
@@ -52,7 +52,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
       with self.assertRaisesOpError(my_msg):
         t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32)
         t_verified = numerics.verify_tensor_all_finite(t, my_msg)
-        t_verified.eval()
+        self.evaluate(t_verified)
 
     # Test Inf.
     x[0] = np.inf
@@ -60,7 +60,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
       with self.assertRaisesOpError(my_msg):
         t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32)
         t_verified = numerics.verify_tensor_all_finite(t, my_msg)
-        t_verified.eval()
+        self.evaluate(t_verified)
 
 
 class NumericsTest(test.TestCase):
@@ -73,7 +73,7 @@ class NumericsTest(test.TestCase):
       check = numerics.add_check_numerics_ops()
       a = control_flow_ops.with_dependencies([check], a)
       with self.assertRaisesOpError("Inf"):
-        a.eval()
+        self.evaluate(a)
 
   def testNaN(self):
     with self.session(graph=ops.Graph()):
@@ -83,7 +83,7 @@ class NumericsTest(test.TestCase):
       check = numerics.add_check_numerics_ops()
       a = control_flow_ops.with_dependencies([check], a)
       with self.assertRaisesOpError("NaN"):
-        a.eval()
+        self.evaluate(a)
 
   def testBoth(self):
     with self.session(graph=ops.Graph()):
@@ -93,13 +93,13 @@ class NumericsTest(test.TestCase):
       check = numerics.add_check_numerics_ops()
       a = control_flow_ops.with_dependencies([check], a)
       with self.assertRaisesOpError("Inf and NaN"):
-        a.eval()
+        self.evaluate(a)
 
   def testPassThrough(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3])
       checked = array_ops.check_numerics(t1, message="pass through test")
-      value = checked.eval()
+      value = self.evaluate(checked)
       self.assertAllEqual(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), value)
       self.assertEqual([2, 3], checked.get_shape())
 
diff --git a/tensorflow/python/kernel_tests/one_hot_op_test.py b/tensorflow/python/kernel_tests/one_hot_op_test.py
index 377d545c9c..856ba7bb7f 100644
--- a/tensorflow/python/kernel_tests/one_hot_op_test.py
+++ b/tensorflow/python/kernel_tests/one_hot_op_test.py
@@ -41,12 +41,12 @@ class OneHotTest(test.TestCase):
       else:
         ans = array_ops.one_hot(**inputs)
         if expected_err_re is None:
-          tf_ans = ans.eval()
+          tf_ans = self.evaluate(ans)
           self.assertAllEqual(tf_ans, truth)
           self.assertEqual(tf_ans.shape, ans.get_shape())
         else:
           with self.assertRaisesOpError(expected_err_re):
-            ans.eval()
+            self.evaluate(ans)
 
   def _testBothOneHot(self, truth, expected_err_re=None, raises=None, **inputs):
     self._testOneHot(truth, True, expected_err_re, raises, **inputs)
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index fc302c4141..6fe98d2559 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -88,7 +88,7 @@ class PadOpTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       tf_val = array_ops.pad(np_inputs, paddings, mode=mode,
                              constant_values=constant_values)
-      out = tf_val.eval()
+      out = self.evaluate(tf_val)
     self.assertAllEqual(np_val, out)
     self.assertShapeEqual(np_val, tf_val)
 
@@ -208,7 +208,7 @@ class PadOpTest(test.TestCase):
                                  constant_op.constant(paddings, padding_dtype),
                                  mode=mode,
                                  constant_values=0)
-          out = tf_val.eval()
+          out = self.evaluate(tf_val)
         self.assertAllEqual(np_val, out)
         self.assertShapeEqual(np_val, tf_val)
 
@@ -250,16 +250,16 @@ class PadOpTest(test.TestCase):
     symmetric = array_ops.pad(x, [[1, 0], [0, 1]], mode="SYMMETRIC",
                               constant_values="PAD")
     with self.session(use_gpu=True):
-      self.assertAllEqual([[b"PAD", b"PAD", b"PAD"],
-                           [b"Hello", b"World", b"PAD"],
-                           [b"Goodnight", b"Moon", b"PAD"]], constant.eval())
+      self.assertAllEqual(
+          [[b"PAD", b"PAD", b"PAD"], [b"Hello", b"World", b"PAD"],
+           [b"Goodnight", b"Moon", b"PAD"]], self.evaluate(constant))
       self.assertAllEqual([[b"Goodnight", b"Moon", b"Goodnight"],
                            [b"Hello", b"World", b"Hello"],
                            [b"Goodnight", b"Moon", b"Goodnight"]],
-                          reflect.eval())
-      self.assertAllEqual([[b"Hello", b"World", b"World"],
-                           [b"Hello", b"World", b"World"],
-                           [b"Goodnight", b"Moon", b"Moon"]], symmetric.eval())
+                          self.evaluate(reflect))
+      self.assertAllEqual(
+          [[b"Hello", b"World", b"World"], [b"Hello", b"World", b"World"],
+           [b"Goodnight", b"Moon", b"Moon"]], self.evaluate(symmetric))
 
   def testShapeFunctionEdgeCases(self):
     # Unknown paddings shape.
@@ -327,7 +327,7 @@ class PadOpTest(test.TestCase):
     inp = np.asarray(7)
     with self.session(use_gpu=True):
       tf_val = array_ops.pad(inp, paddings)
-      out = tf_val.eval()
+      out = self.evaluate(tf_val)
     self.assertAllEqual(inp, out)
     self.assertShapeEqual(inp, tf_val)
 
@@ -337,7 +337,7 @@ class PadOpTest(test.TestCase):
       inp = np.asarray(7)
       with self.cached_session(use_gpu=True):
         tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype))
-        out = tf_val.eval()
+        out = self.evaluate(tf_val)
       self.assertAllEqual(inp, out)
       self.assertShapeEqual(inp, tf_val)
 
@@ -361,11 +361,12 @@ class PadOpTest(test.TestCase):
             [paddings_value[i][0] + inp.shape.dims[i].value for i in range(4)],
             [-1, -1, -1, -1])
         with self.cached_session(use_gpu=True):
-          self.assertAllEqual(inp.eval(), middle.eval())
+          self.assertAllEqual(inp.eval(), self.evaluate(middle))
           self.assertAllEqual(
-              np.zeros([row[0] for row in paddings_value]), left.eval())
+              np.zeros([row[0] for row in paddings_value]), self.evaluate(left))
           self.assertAllEqual(
-              np.zeros([row[1] for row in paddings_value]), right.eval())
+              np.zeros([row[1] for row in paddings_value]),
+              self.evaluate(right))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
index 95f3dcceea..520b663375 100644
--- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
@@ -178,7 +178,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        vals = dequeued_t.eval()
+        vals = self.evaluate(dequeued_t)
         self.assertEqual([elems[i]], vals)
 
   def testEnqueueAndBlockingDequeue(self):
@@ -243,9 +243,9 @@ class PaddingFIFOQueueTest(test.TestCase):
       self.assertEqual([], size.get_shape())
 
       enqueue_op.run()
-      self.assertEqual(1, size.eval())
+      self.assertEqual(1, self.evaluate(size))
       dequeued_t.op.run()
-      self.assertEqual(0, size.eval())
+      self.assertEqual(0, self.evaluate(size))
 
   def testEnqueueMany(self):
     with self.cached_session():
@@ -257,7 +257,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        vals = dequeued_t.eval()
+        vals = self.evaluate(dequeued_t)
         self.assertEqual([elems[i % 4]], vals)
 
   def testEmptyEnqueueMany(self):
@@ -269,9 +269,9 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue_many((empty_t,))
       size_t = q.size()
 
-      self.assertEqual([0], size_t.eval())
+      self.assertEqual([0], self.evaluate(size_t))
       enqueue_op.run()
-      self.assertEqual([0], size_t.eval())
+      self.assertEqual([0], self.evaluate(size_t))
 
   def testEmptyDequeueMany(self):
     with self.cached_session():
@@ -279,9 +279,9 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue((10.0,))
       dequeued_t = q.dequeue_many(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testEmptyDequeueManyWithDynamicShape(self):
     with self.cached_session():
@@ -290,9 +290,9 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue(([10.0],))
       dequeued_t = q.dequeue_many(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testEmptyDequeueUpToWithDynamicShape(self):
     with self.cached_session():
@@ -301,9 +301,9 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op = q.enqueue(([10.0],))
       dequeued_t = q.dequeue_up_to(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testConstructPaddingFIFOQueueWithNoShape(self):
     with self.cached_session():
@@ -357,8 +357,8 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      self.assertAllEqual(elems[0:4], dequeued_t.eval())
-      self.assertAllEqual(elems[4:8], dequeued_t.eval())
+      self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t))
+      self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t))
 
   def testDequeueUpToNoBlocking(self):
     with self.cached_session():
@@ -369,8 +369,8 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      self.assertAllEqual(elems[0:4], dequeued_t.eval())
-      self.assertAllEqual(elems[4:8], dequeued_t.eval())
+      self.assertAllEqual(elems[0:4], self.evaluate(dequeued_t))
+      self.assertAllEqual(elems[4:8], self.evaluate(dequeued_t))
 
   def testMultiDequeueMany(self):
     with self.cached_session() as sess:
@@ -622,7 +622,7 @@ class PaddingFIFOQueueTest(test.TestCase):
                                    r"Expected \[2,\?,3\], got \[2,3,4\]"):
         sess.run([enqueue_op],
                  feed_dict={elems_bad: np.array([1] * 24).reshape((2, 3, 4))})
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testParallelEnqueueMany(self):
     with self.cached_session() as sess:
@@ -776,7 +776,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       while elements_dequeued < 250:
         # With equal probability, run Dequeue or dequeue_many.
         if random.random() > 0.5:
-          self.assertEqual(elements_dequeued, dequeued_t.eval())
+          self.assertEqual(elements_dequeued, self.evaluate(dequeued_t))
           elements_dequeued += 1
         else:
           count = random.randint(0, min(20, 250 - elements_dequeued))
@@ -882,12 +882,12 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
       close_op.run()
       for elem in elems:
-        self.assertEqual([elem], dequeued_t.eval())
+        self.assertEqual([elem], self.evaluate(dequeued_t))
 
       # Expect the operation to fail due to the queue being closed.
       with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                    "is closed and has insufficient"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testBlockingDequeueFromClosedQueue(self):
     with self.cached_session() as sess:
@@ -1163,8 +1163,8 @@ class PaddingFIFOQueueTest(test.TestCase):
       # TODO(mrry): Figure out how to do this without sleeping.
       time.sleep(0.1)
       for elem in elems:
-        self.assertEqual([elem], dequeued_t.eval())
-      self.assertEqual([50.0], dequeued_t.eval())
+        self.assertEqual([elem], self.evaluate(dequeued_t))
+      self.assertEqual([50.0], self.evaluate(dequeued_t))
       thread.join()
 
   def testBlockingEnqueueManyToFullQueue(self):
@@ -1186,10 +1186,10 @@ class PaddingFIFOQueueTest(test.TestCase):
       # TODO(mrry): Figure out how to do this without sleeping.
       time.sleep(0.1)
       for elem in elems:
-        self.assertEqual([elem], dequeued_t.eval())
+        self.assertEqual([elem], self.evaluate(dequeued_t))
         time.sleep(0.01)
-      self.assertEqual([50.0], dequeued_t.eval())
-      self.assertEqual([60.0], dequeued_t.eval())
+      self.assertEqual([50.0], self.evaluate(dequeued_t))
+      self.assertEqual([60.0], self.evaluate(dequeued_t))
 
       # Make sure the thread finishes before exiting.
       thread.join()
@@ -1223,12 +1223,12 @@ class PaddingFIFOQueueTest(test.TestCase):
       close_thread.start()
 
       # The dequeue will unblock both threads.
-      self.assertEqual(10.0, dequeued_t.eval())
+      self.assertEqual(10.0, self.evaluate(dequeued_t))
       enqueue_thread.join()
       close_thread.join()
 
       for elem in [20.0, 30.0, 40.0, 50.0]:
-        self.assertEqual(elem, dequeued_t.eval())
+        self.assertEqual(elem, self.evaluate(dequeued_t))
       self.assertEqual(0, q.size().eval())
 
   def testBlockingEnqueueManyBeforeClose(self):
@@ -1258,11 +1258,11 @@ class PaddingFIFOQueueTest(test.TestCase):
       close_thread.start()
 
       # The dequeue will unblock both threads.
-      self.assertEqual(10.0, dequeued_t.eval())
+      self.assertEqual(10.0, self.evaluate(dequeued_t))
       enqueue_thread.join()
       close_thread.join()
       for elem in [20.0, 30.0, 50.0, 60.0]:
-        self.assertEqual(elem, dequeued_t.eval())
+        self.assertEqual(elem, self.evaluate(dequeued_t))
 
   def testDoesNotLoseValue(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/parsing_ops_test.py b/tensorflow/python/kernel_tests/parsing_ops_test.py
index 71d8b60d3c..8f359bd32c 100644
--- a/tensorflow/python/kernel_tests/parsing_ops_test.py
+++ b/tensorflow/python/kernel_tests/parsing_ops_test.py
@@ -108,8 +108,8 @@ class ParseExampleTest(test.TestCase):
       # properly check.
       serialized = kwargs["serialized"]
       batch_size = (
-          serialized.eval().size if isinstance(serialized, ops.Tensor) else
-          np.asarray(serialized).size)
+          self.evaluate(serialized).size if isinstance(serialized, ops.Tensor)
+          else np.asarray(serialized).size)
       for k, f in kwargs["features"].items():
         if isinstance(f, parsing_ops.FixedLenFeature) and f.shape is not None:
           self.assertEqual(
diff --git a/tensorflow/python/kernel_tests/partitioned_variables_test.py b/tensorflow/python/kernel_tests/partitioned_variables_test.py
index d1f0c6c2a0..0c04656196 100644
--- a/tensorflow/python/kernel_tests/partitioned_variables_test.py
+++ b/tensorflow/python/kernel_tests/partitioned_variables_test.py
@@ -328,7 +328,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       vs = partitioned_variables.create_partitioned_variables([4], [4], rnd_par)
       variables.global_variables_initializer().run()
       val = array_ops.concat(vs, 0).eval()
-      rnd = rnd_par.eval()
+      rnd = self.evaluate(rnd_par)
       self.assertAllClose(rnd, val)
       self.assertEqual([dtypes.int32] * 4, [v.dtype.base_dtype for v in vs])
       self._TestSaveSpec(vs, ["4 0,1", "4 1,1", "4 2,1", "4 3,1"])
@@ -340,7 +340,7 @@ class PartitionedVariablesTestCase(test.TestCase):
                                                               rnd_par)
       variables.global_variables_initializer().run()
       val = array_ops.concat(vs, 1).eval()
-      rnd = rnd_par.eval()
+      rnd = self.evaluate(rnd_par)
       self.assertAllClose(rnd, val)
       self.assertEqual([dtypes.int32] * 2, [v.dtype.base_dtype for v in vs])
       self._TestSaveSpec(vs, ["2 4 0,2:0,2", "2 4 0,2:2,2"])
@@ -414,7 +414,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           rnd.get_shape(), [1, 10], rnd.initialized_value())
       variables.global_variables_initializer().run()
       val = array_ops.concat(vs, 1).eval()
-      rnd = rnd.eval()
+      rnd = self.evaluate(rnd)
       self.assertAllClose(rnd, val)
       self.assertEqual([dtypes.float32] * 10, [v.dtype.base_dtype for v in vs])
       self._TestSaveSpec(vs, [
@@ -434,7 +434,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           for i in xrange(1, 10)
       ]
       variables.global_variables_initializer().run()
-      rnd_val = rnd.eval()
+      rnd_val = self.evaluate(rnd)
       # Only check the slice save specs for the first 5 tf.
       save_specs = [
           # One slice
@@ -469,7 +469,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           rnd.get_shape(), [1, 1], rnd.initialized_value())
       variables.global_variables_initializer().run()
       val = array_ops.concat(vs, 0).eval()
-      rnd = rnd.eval()
+      rnd = self.evaluate(rnd)
       self.assertAllClose(rnd, val)
       self._TestSaveSpec(vs, ["10 43 0,10:0,43"])
 
@@ -480,7 +480,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           rnd.get_shape(), [10, 1], rnd.initialized_value())
       variables.global_variables_initializer().run()
       val = array_ops.concat(vs, 0).eval()
-      rnd = rnd.eval()
+      rnd = self.evaluate(rnd)
       self.assertAllClose(rnd, val)
       self._TestSaveSpec(vs, [
           "10 43 0,1:0,43", "10 43 1,1:0,43", "10 43 2,1:0,43",
@@ -510,7 +510,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       var0, var1 = partitioned_variables.create_partitioned_variables(
           [20, 12], [1, 2], init_ops.random_uniform_initializer())
       variables.global_variables_initializer().run()
-      val0, val1 = var0.eval().flatten(), var1.eval().flatten()
+      val0, val1 = self.evaluate(var0).flatten(), self.evaluate(var1).flatten()
       self.assertTrue(np.linalg.norm(val0 - val1) > 1e-6)
     # Negative test that proves that slices have the same values if
     # the random initializer uses a seed.
@@ -518,7 +518,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       var0, var1 = partitioned_variables.create_partitioned_variables(
           [20, 12], [1, 2], init_ops.random_uniform_initializer(seed=201))
       variables.global_variables_initializer().run()
-      val0, val1 = var0.eval().flatten(), var1.eval().flatten()
+      val0, val1 = self.evaluate(var0).flatten(), self.evaluate(var1).flatten()
       self.assertAllClose(val0, val1)
 
   def testSomeErrors(self):
diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py
index 372861297f..92016a49a2 100644
--- a/tensorflow/python/kernel_tests/pool_test.py
+++ b/tensorflow/python/kernel_tests/pool_test.py
@@ -151,7 +151,7 @@ class PoolingTest(test.TestCase):
         np.prod(input_shape), dtype=np.float32).reshape(input_shape) - 1
     y1 = pool_direct(input=x, **kwargs)
     y2 = nn_ops.pool(input=x, **kwargs)
-    self.assertAllClose(y1, y2.eval(), rtol=1e-2, atol=1e-2)
+    self.assertAllClose(y1, self.evaluate(y2), rtol=1e-2, atol=1e-2)
 
   def testPoolSimple(self):
     with self.session(use_gpu=test.is_gpu_available()):
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 53003a7f28..61628c4756 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -166,7 +166,7 @@ class PoolingTest(test.TestCase):
             strides_placeholder: strides
         })
       else:
-        actual = t.eval()
+        actual = self.evaluate(t)
         self.assertShapeEqual(actual, t)
       self.assertAllCloseAccordingToType(expected, actual.flatten())
 
@@ -750,11 +750,11 @@ class PoolingTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         t = constant_op.constant(tensor_input, shape=input_shape)
         out_op, _ = nn_ops.max_pool_with_argmax(t, ksize, strides, padding)
-        gpu_val = out_op.eval()
+        gpu_val = self.evaluate(out_op)
       with self.cached_session(use_gpu=False):
         t = constant_op.constant(tensor_input, shape=input_shape)
         out_op = nn_ops.max_pool(t, ksize, strides, padding)
-        cpu_val = out_op.eval()
+        cpu_val = self.evaluate(out_op)
       self.assertAllCloseAccordingToType(cpu_val, gpu_val)
 
   def _CompareMaxPoolingBk(self, input_shape, output_shape, ksize, strides,
@@ -767,20 +767,20 @@ class PoolingTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         t = constant_op.constant(tensor_input, shape=input_shape)
         _, argmax_op = nn_ops.max_pool_with_argmax(t, ksize, strides, padding)
-        argmax = argmax_op.eval()
+        argmax = self.evaluate(argmax_op)
         grad_in = constant_op.constant(tensor_output, shape=output_shape)
         out_op = gen_nn_ops.max_pool_grad_with_argmax(t, grad_in, argmax, ksize,
                                                       strides, padding)
-        gpu_val = out_op.eval()
+        gpu_val = self.evaluate(out_op)
         self.assertShapeEqual(gpu_val, out_op)
       with self.cached_session(use_gpu=False):
         t = constant_op.constant(tensor_input, shape=input_shape)
         out_op = nn_ops.max_pool(t, ksize, strides, padding)
-        orig_out = out_op.eval()
+        orig_out = self.evaluate(out_op)
         grad_in = constant_op.constant(tensor_output, shape=output_shape)
         out_op = gen_nn_ops.max_pool_grad(t, orig_out, grad_in, ksize, strides,
                                           padding)
-        cpu_val = out_op.eval()
+        cpu_val = self.evaluate(out_op)
         self.assertShapeEqual(cpu_val, out_op)
       # The CPU version accumulates its gradient on fp16, so it's less
       # accurate than the GPU version that does the accumulation on fp32
@@ -796,20 +796,20 @@ class PoolingTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         t = constant_op.constant(tensor_input, shape=input_shape)
         _, argmax_op = nn_ops.max_pool_with_argmax(t, ksize, strides, padding)
-        argmax = argmax_op.eval()
+        argmax = self.evaluate(argmax_op)
         grad_in = constant_op.constant(tensor_input, shape=input_shape)
         out_op = gen_nn_ops.max_pool_grad_grad_with_argmax(
             t, grad_in, argmax, ksize, strides, padding)
-        gpu_val = out_op.eval()
+        gpu_val = self.evaluate(out_op)
         self.assertShapeEqual(gpu_val, out_op)
       with self.cached_session(use_gpu=False):
         t = constant_op.constant(tensor_input, shape=input_shape)
         out_op = nn_ops.max_pool(t, ksize, strides, padding)
-        orig_out = out_op.eval()
+        orig_out = self.evaluate(out_op)
         grad_in = constant_op.constant(tensor_input, shape=input_shape)
         out_op = gen_nn_ops.max_pool_grad_grad(t, orig_out, grad_in, ksize,
                                                strides, padding)
-        cpu_val = out_op.eval()
+        cpu_val = self.evaluate(out_op)
         self.assertShapeEqual(cpu_val, out_op)
       # The CPU version accumulates its gradient on fp16, so it's less
       # accurate than the GPU version that does the accumulation on fp32
@@ -848,7 +848,7 @@ class PoolingTest(test.TestCase):
           ksize=[1, 2, 2, 1],
           strides=[1, 1, 1, 1],
           padding="VALID")
-      out = out_op.eval().flatten()
+      out = self.evaluate(out_op).flatten()
       self.assertAllClose(out,
                           [11.0, 12.0, 0.0, 13.0, 0.0, 14.0, 0.0, 0.0, 0.0])
 
@@ -871,7 +871,7 @@ class PoolingTest(test.TestCase):
           ksize=[1, 2, 2, 1],
           strides=[1, 1, 1, 1],
           padding="VALID")
-      out = out_op.eval().flatten()
+      out = self.evaluate(out_op).flatten()
       self.assertAllClose(out, [11.0, 12.0, 14.0, 16.0])
 
   def _ConstructAndTestGradient(self,
@@ -1221,12 +1221,12 @@ class PoolingTest(test.TestCase):
           input_tensor, output_tensor, output_backprop_tensor, window_rows,
           window_cols, row_stride, col_stride, padding, v2)
 
-      actual_input_backprop = input_backprop_tensor.eval()
+      actual_input_backprop = self.evaluate(input_backprop_tensor)
       self.assertShapeEqual(actual_input_backprop, input_backprop_tensor)
       actual_input_backprop = actual_input_backprop.flatten()
       actual_input_backprop = self._GetNdArray(actual_input_backprop)
 
-      actual_output = output_tensor.eval().flatten()
+      actual_output = self.evaluate(output_tensor).flatten()
       actual_output = self._GetNdArray(actual_output)
 
       self.assertAllClose(
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index 837f1ec054..b101da036e 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -272,7 +272,7 @@ class PyFuncTest(test.TestCase):
 
       with self.assertRaisesRegexp(errors.UnimplementedError,
                                    "Unsupported numpy type"):
-        y.eval()
+        self.evaluate(y)
 
   def testBadReturnType(self):
     with self.cached_session():
@@ -285,7 +285,7 @@ class PyFuncTest(test.TestCase):
 
       with self.assertRaisesRegexp(errors.UnimplementedError,
                                    "Unsupported object type"):
-        z.eval()
+        self.evaluate(z)
 
   def testReturnInput(self):
     with self.cached_session():
@@ -335,7 +335,7 @@ class PyFuncTest(test.TestCase):
       val = [[1, 2], [3, 4]]
       x, = script_ops.py_func(lambda: np.array(val, order="F"), [],
                               [dtypes.int64])
-      self.assertAllEqual(val, x.eval())
+      self.assertAllEqual(val, self.evaluate(x))
 
   def testParallel(self):
     # Tests that tf.py_func's can run in parallel if they release the GIL.
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index a60237fb25..617b724204 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -110,7 +110,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
       tol = 1e-5
     else:
       tol = 1e-14
-    self.assertAllClose(identity.eval(), xx.eval(), atol=tol)
+    self.assertAllClose(identity.eval(), self.evaluate(xx), atol=tol)
 
   def Test(self):
     np.random.seed(1)
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
index bd64d61af8..cfec4d08fb 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
@@ -197,7 +197,7 @@ class MultinomialTest(test.TestCase):
     with self.test_session(use_gpu=True):
       x = random_ops.multinomial(array_ops.zeros([5, 0]), 7)
       with self.assertRaisesOpError("num_classes should be positive"):
-        x.eval()
+        self.evaluate(x)
 
   def testNegativeMinLogits(self):
     random_seed.set_random_seed(78844)
diff --git a/tensorflow/python/kernel_tests/random/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py
index 8ded522320..491d19d6a0 100644
--- a/tensorflow/python/kernel_tests/random/random_crop_test.py
+++ b/tensorflow/python/kernel_tests/random/random_crop_test.py
@@ -44,7 +44,7 @@ class RandomCropTest(test.TestCase):
           for i in range(2) for j in range(3) for k in range(4))
       crop = random_ops.random_crop(value, size=target)
       for _ in range(20):
-        y = crop.eval()
+        y = self.evaluate(crop)
         self.assertAllEqual(y.shape, target)
         self.assertTrue(tuple(y.ravel()) in value_set)
 
@@ -61,7 +61,7 @@ class RandomCropTest(test.TestCase):
       crop = random_ops.random_crop(value, single, seed=7)
       counts = np.zeros(size, dtype=np.int32)
       for _ in range(num_samples):
-        y = crop.eval()
+        y = self.evaluate(crop)
         self.assertAllEqual(y.shape, single)
         counts[y] += 1
 
diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
index 417588f8a3..95e48101f6 100644
--- a/tensorflow/python/kernel_tests/random/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py
@@ -140,7 +140,7 @@ class RandomPoissonTest(test.TestCase):
     with self.cached_session():
       rnd = random_ops.random_poisson([], [], seed=12345)
       self.assertEqual([0], rnd.get_shape().as_list())
-      self.assertAllClose(np.array([], dtype=np.float32), rnd.eval())
+      self.assertAllClose(np.array([], dtype=np.float32), self.evaluate(rnd))
 
   def testShape(self):
     # Fully known shape
diff --git a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
index 0d85a072d4..f3fcf1eff7 100644
--- a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
@@ -215,9 +215,9 @@ class RandomShuffleQueueTest(test.TestCase):
       self.assertEqual([], size.get_shape())
 
       enqueue_op.run()
-      self.assertEqual([1], size.eval())
+      self.assertEqual([1], self.evaluate(size))
       dequeued_t.op.run()
-      self.assertEqual([0], size.eval())
+      self.assertEqual([0], self.evaluate(size))
 
   def testEnqueueMany(self):
     with self.cached_session():
@@ -241,9 +241,9 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op = q.enqueue_many((empty_t,))
       size_t = q.size()
 
-      self.assertEqual(0, size_t.eval())
+      self.assertEqual(0, self.evaluate(size_t))
       enqueue_op.run()
-      self.assertEqual(0, size_t.eval())
+      self.assertEqual(0, self.evaluate(size_t))
 
   def testEmptyDequeueMany(self):
     with self.cached_session():
@@ -251,9 +251,9 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op = q.enqueue((10.0,))
       dequeued_t = q.dequeue_many(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testEmptyDequeueUpTo(self):
     with self.cached_session():
@@ -261,9 +261,9 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op = q.enqueue((10.0,))
       dequeued_t = q.dequeue_up_to(0)
 
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
       enqueue_op.run()
-      self.assertEqual([], dequeued_t.eval().tolist())
+      self.assertEqual([], self.evaluate(dequeued_t).tolist())
 
   def testEmptyDequeueManyWithNoShape(self):
     with self.cached_session():
@@ -275,7 +275,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # Expect the operation to fail due to the shape not being constrained.
       with self.assertRaisesOpError(
           "require the components to have specified shapes"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
       enqueue_op.run()
 
@@ -284,7 +284,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # elements enqueued.
       with self.assertRaisesOpError(
           "require the components to have specified shapes"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testEmptyDequeueUpToWithNoShape(self):
     with self.cached_session():
@@ -296,7 +296,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # Expect the operation to fail due to the shape not being constrained.
       with self.assertRaisesOpError(
           "require the components to have specified shapes"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
       enqueue_op.run()
 
@@ -305,7 +305,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # elements enqueued.
       with self.assertRaisesOpError(
           "require the components to have specified shapes"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testMultiEnqueueMany(self):
     with self.cached_session() as sess:
@@ -335,7 +335,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      results = dequeued_t.eval().tolist()
+      results = self.evaluate(dequeued_t).tolist()
       results.extend(dequeued_t.eval())
       self.assertItemsEqual(elems, results)
 
@@ -348,7 +348,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      results = dequeued_t.eval().tolist()
+      results = self.evaluate(dequeued_t).tolist()
       results.extend(dequeued_t.eval())
       self.assertItemsEqual(elems, results)
 
@@ -649,7 +649,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # Expect the operation to fail due to the queue being closed.
       with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                    "is closed and has insufficient"):
-        dequeued_t.eval()
+        self.evaluate(dequeued_t)
 
   def testBlockingDequeueFromClosedQueue(self):
     with self.cached_session() as sess:
@@ -1040,7 +1040,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # First blocking_enqueue_op of blocking_enqueue has enqueued 1 of 2
       # elements, and is blocked waiting for one more element to be dequeue.
       for i in range(50):
-        queue_size = size_t.eval()
+        queue_size = self.evaluate(size_t)
         if queue_size == 4:
           break
         elif i == 49:
diff --git a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
index d57db3c512..13f97a9367 100644
--- a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
@@ -62,7 +62,7 @@ class StatelessOpsTest(test.TestCase):
         for stateless_op, stateful_op in cases:
           stateful = stateful_op(seed=seed[1])
           pure = stateless_op(seed=preseed)
-          self.assertAllEqual(stateful.eval(), pure.eval())
+          self.assertAllEqual(stateful.eval(), self.evaluate(pure))
 
   def _test_determinism(self, cases):
     # Stateless values should be equal iff the seeds are equal (roughly)
diff --git a/tensorflow/python/kernel_tests/reader_ops_test.py b/tensorflow/python/kernel_tests/reader_ops_test.py
index ac9be56d63..18a8a3d547 100644
--- a/tensorflow/python/kernel_tests/reader_ops_test.py
+++ b/tensorflow/python/kernel_tests/reader_ops_test.py
@@ -154,30 +154,30 @@ class IdentityReaderTest(test.TestCase):
       queued_length = queue.size()
       key, value = reader.read(queue)
 
-      self.assertAllEqual(0, work_completed.eval())
-      self.assertAllEqual(0, produced.eval())
-      self.assertAllEqual(0, queued_length.eval())
+      self.assertAllEqual(0, self.evaluate(work_completed))
+      self.assertAllEqual(0, self.evaluate(produced))
+      self.assertAllEqual(0, self.evaluate(queued_length))
 
       queue.enqueue_many([["A", "B", "C"]]).run()
       queue.close().run()
-      self.assertAllEqual(3, queued_length.eval())
+      self.assertAllEqual(3, self.evaluate(queued_length))
 
       self._ExpectRead(sess, key, value, b"A")
-      self.assertAllEqual(1, produced.eval())
+      self.assertAllEqual(1, self.evaluate(produced))
 
       self._ExpectRead(sess, key, value, b"B")
 
       self._ExpectRead(sess, key, value, b"C")
-      self.assertAllEqual(3, produced.eval())
-      self.assertAllEqual(0, queued_length.eval())
+      self.assertAllEqual(3, self.evaluate(produced))
+      self.assertAllEqual(0, self.evaluate(queued_length))
 
       with self.assertRaisesOpError("is closed and has insufficient elements "
                                     "\\(requested 1, current size 0\\)"):
         sess.run([key, value])
 
-      self.assertAllEqual(3, work_completed.eval())
-      self.assertAllEqual(3, produced.eval())
-      self.assertAllEqual(0, queued_length.eval())
+      self.assertAllEqual(3, self.evaluate(work_completed))
+      self.assertAllEqual(3, self.evaluate(produced))
+      self.assertAllEqual(0, self.evaluate(queued_length))
 
   def testMultipleEpochs(self):
     with self.cached_session() as sess:
@@ -209,23 +209,23 @@ class IdentityReaderTest(test.TestCase):
       key, value = reader.read(queue)
 
       self._ExpectRead(sess, key, value, b"X")
-      self.assertAllEqual(1, produced.eval())
+      self.assertAllEqual(1, self.evaluate(produced))
       state = reader.serialize_state().eval()
 
       self._ExpectRead(sess, key, value, b"Y")
       self._ExpectRead(sess, key, value, b"Z")
-      self.assertAllEqual(3, produced.eval())
+      self.assertAllEqual(3, self.evaluate(produced))
 
       queue.enqueue_many([["Y", "Z"]]).run()
       queue.close().run()
       reader.restore_state(state).run()
-      self.assertAllEqual(1, produced.eval())
+      self.assertAllEqual(1, self.evaluate(produced))
       self._ExpectRead(sess, key, value, b"Y")
       self._ExpectRead(sess, key, value, b"Z")
       with self.assertRaisesOpError("is closed and has insufficient elements "
                                     "\\(requested 1, current size 0\\)"):
         sess.run([key, value])
-      self.assertAllEqual(3, produced.eval())
+      self.assertAllEqual(3, self.evaluate(produced))
 
       self.assertEqual(bytes, type(state))
 
@@ -266,17 +266,17 @@ class IdentityReaderTest(test.TestCase):
 
       queue.enqueue_many([["X", "Y", "Z"]]).run()
       self._ExpectRead(sess, key, value, b"X")
-      self.assertLess(0, queued_length.eval())
-      self.assertAllEqual(1, produced.eval())
+      self.assertLess(0, self.evaluate(queued_length))
+      self.assertAllEqual(1, self.evaluate(produced))
 
       self._ExpectRead(sess, key, value, b"Y")
-      self.assertLess(0, work_completed.eval())
-      self.assertAllEqual(2, produced.eval())
+      self.assertLess(0, self.evaluate(work_completed))
+      self.assertAllEqual(2, self.evaluate(produced))
 
       reader.reset().run()
-      self.assertAllEqual(0, work_completed.eval())
-      self.assertAllEqual(0, produced.eval())
-      self.assertAllEqual(1, queued_length.eval())
+      self.assertAllEqual(0, self.evaluate(work_completed))
+      self.assertAllEqual(0, self.evaluate(produced))
+      self.assertAllEqual(1, self.evaluate(queued_length))
       self._ExpectRead(sess, key, value, b"Z")
 
       queue.enqueue_many([["K", "L"]]).run()
diff --git a/tensorflow/python/kernel_tests/reduce_join_op_test.py b/tensorflow/python/kernel_tests/reduce_join_op_test.py
index 3bb4986313..c26e62738c 100644
--- a/tensorflow/python/kernel_tests/reduce_join_op_test.py
+++ b/tensorflow/python/kernel_tests/reduce_join_op_test.py
@@ -119,7 +119,7 @@ class ReduceJoinTest(UnicodeTestCase):
           axis=axis,
           keep_dims=keep_dims,
           separator=separator)
-      output_array = output.eval()
+      output_array = self.evaluate(output)
 
     self.assertAllEqualUnicode(truth, output_array)
     self.assertAllEqual(truth_shape, output.get_shape())
@@ -149,10 +149,10 @@ class ReduceJoinTest(UnicodeTestCase):
       if not axis:
         truth = constant_op.constant(truth)
       truth_squeezed = array_ops.squeeze(truth, axis=axis)
-      output_array = output.eval()
-      output_keep_dims_array = output_keep_dims.eval()
-      truth_array = truth.eval()
-      truth_squeezed_array = truth_squeezed.eval()
+      output_array = self.evaluate(output)
+      output_keep_dims_array = self.evaluate(output_keep_dims)
+      truth_array = self.evaluate(truth)
+      truth_squeezed_array = self.evaluate(truth_squeezed)
     self.assertAllEqualUnicode(truth_array, output_keep_dims_array)
     self.assertAllEqualUnicode(truth_squeezed_array, output_array)
     self.assertAllEqual(truth.get_shape(), output_keep_dims.get_shape())
@@ -318,11 +318,11 @@ class ReduceJoinTest(UnicodeTestCase):
 
       # Reduction that drops the dim of size 0.
       output = string_ops.reduce_join(inputs=inputs, axis=0)
-      self.assertAllEqualUnicode([""], output.eval())
+      self.assertAllEqualUnicode([""], self.evaluate(output))
 
       # Reduction that keeps the dim of size 0.
       output = string_ops.reduce_join(inputs=inputs, axis=1)
-      output_shape = output.eval().shape
+      output_shape = self.evaluate(output).shape
       self.assertAllEqual([0], output_shape)
 
   def testInvalidArgsUnknownShape(self):
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index 2ac3996e25..d1a295f42b 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -562,7 +562,7 @@ class MinReductionTest(test.TestCase):
       if reduction_axes is not None:
         reduction_axes = np.array(reduction_axes).astype(np.int32)
       tf_ans = math_ops.reduce_min(x, reduction_axes, keepdims)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
     self.assertAllClose(np_ans, out)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -675,7 +675,7 @@ class MaxReductionTest(test.TestCase):
       if reduction_axes is not None:
         reduction_axes = np.array(reduction_axes).astype(np.int32)
       tf_ans = math_ops.reduce_max(x, reduction_axes, keepdims)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
     self.assertAllClose(np_ans, out)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -802,7 +802,7 @@ class AllReductionTest(test.TestCase):
       if reduction_axes is not None:
         reduction_axes = np.array(reduction_axes).astype(np.int32)
       tf_ans = math_ops.reduce_all(x, reduction_axes, keepdims)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, out)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -851,7 +851,7 @@ class AnyReductionTest(test.TestCase):
       if reduction_axes is not None:
         reduction_axes = np.array(reduction_axes).astype(np.int32)
       tf_ans = math_ops.reduce_any(x, reduction_axes, keepdims)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, out)
     self.assertShapeEqual(np_ans, tf_ans)
 
diff --git a/tensorflow/python/kernel_tests/regex_full_match_op_test.py b/tensorflow/python/kernel_tests/regex_full_match_op_test.py
index 98746e7d9b..4edd3e98d9 100644
--- a/tensorflow/python/kernel_tests/regex_full_match_op_test.py
+++ b/tensorflow/python/kernel_tests/regex_full_match_op_test.py
@@ -61,7 +61,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
       invalid_pattern = "A["
       matched = op(input_tensor, invalid_pattern)
       with self.assertRaisesOpError("Invalid pattern"):
-        matched.eval()
+        self.evaluate(matched)
 
 
 class RegexFullMatchOpTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/regex_replace_op_test.py b/tensorflow/python/kernel_tests/regex_replace_op_test.py
index d9b7ed28d2..ce9a1b5279 100644
--- a/tensorflow/python/kernel_tests/regex_replace_op_test.py
+++ b/tensorflow/python/kernel_tests/regex_replace_op_test.py
@@ -74,7 +74,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
       invalid_pattern = "A["
       replace = op(input_vector, invalid_pattern, "x")
       with self.assertRaisesOpError("Invalid pattern"):
-        replace.eval()
+        self.evaluate(replace)
 
   def testGlobal(self, op):
     values = ["ababababab", "abcabcabc", ""]
diff --git a/tensorflow/python/kernel_tests/reshape_op_test.py b/tensorflow/python/kernel_tests/reshape_op_test.py
index 14cdae1837..84539c2b02 100644
--- a/tensorflow/python/kernel_tests/reshape_op_test.py
+++ b/tensorflow/python/kernel_tests/reshape_op_test.py
@@ -33,14 +33,14 @@ class ReshapeTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu):
       np_ans = x.reshape(y)
       tf_ans = array_ops.reshape(x, y)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       self.assertEqual(tf_ans.get_shape(), out.shape)
       self.assertShapeEqual(np_ans, tf_ans)
 
       # Repeat with an int64 shape tensor.
       y64 = constant_op.constant(y, dtype=dtypes.int64)
       tf_ans = array_ops.reshape(x, y64)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       self.assertEqual(tf_ans.get_shape(), out.shape)
       self.assertShapeEqual(np_ans, tf_ans)
 
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index c8227dc117..45b9ede813 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -736,7 +736,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
           # Needed in Eager since we get a unique container name by default.
           container=ops.get_default_graph()._container)
       w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype)
-      self.assertEqual(300.0, w_read.eval())
+      self.assertEqual(300.0, self.evaluate(w_read))
 
       x = resource_variable_ops.var_handle_op(
           dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5",
diff --git a/tensorflow/python/kernel_tests/reverse_sequence_op_test.py b/tensorflow/python/kernel_tests/reverse_sequence_op_test.py
index 56609bd0a5..91d054ad9a 100644
--- a/tensorflow/python/kernel_tests/reverse_sequence_op_test.py
+++ b/tensorflow/python/kernel_tests/reverse_sequence_op_test.py
@@ -42,12 +42,12 @@ class ReverseSequenceTest(test.TestCase):
       ans = array_ops.reverse_sequence(
           x, batch_axis=batch_axis, seq_axis=seq_axis, seq_lengths=seq_lengths)
       if expected_err_re is None:
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
         self.assertAllClose(tf_ans, truth, atol=1e-10)
         self.assertShapeEqual(truth, ans)
       else:
         with self.assertRaisesOpError(expected_err_re):
-          ans.eval()
+          self.evaluate(ans)
 
   def _testBothReverseSequence(self,
                                x,
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index 0ed508b9fe..952ef34456 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -144,7 +144,7 @@ class StatefulScatterNdTest(test.TestCase):
         tf_scatter(ref_var, indices, updates).eval()
 
         # Compare
-        self.assertAllClose(new, ref_var.eval())
+        self.assertAllClose(new, self.evaluate(ref_var))
 
   def _VariableRankTests(self, np_scatter, tf_scatter):
     for vtype in (np.int32, np.float16, np.float32, np.float64, np.complex64,
@@ -249,7 +249,7 @@ class StatefulScatterNdTest(test.TestCase):
   #             [[0]], dtype=tf.int64), [False])
   #     var.initializer.run()
   #     session.run([update0, update1])
-  #     self.assertAllEqual([False, True], var.eval())
+  #     self.assertAllEqual([False, True], self.evaluate(var))
 
   def testScatterOutOfRangeCpu(self):
     # TODO(simister): Re-enable once binary size increase due to
@@ -307,7 +307,7 @@ class StatefulScatterNdTest(test.TestCase):
     expected_result = np.zeros([2, 2], dtype=np.int32)
     with self.cached_session():
       ref.initializer.run()
-      self.assertAllEqual(expected_result, scatter_update.eval())
+      self.assertAllEqual(expected_result, self.evaluate(scatter_update))
 
   def testRank3InvalidShape1(self):
     indices = array_ops.zeros([3, 2, 2], dtypes.int32)
@@ -463,7 +463,7 @@ class ScatterNdTest(test.TestCase):
     self.assertAllEqual(scatter.get_shape().as_list(), shape)
     expected_result = np.zeros([2, 2], dtype=np.int32)
     with self.cached_session():
-      self.assertAllEqual(expected_result, scatter.eval())
+      self.assertAllEqual(expected_result, self.evaluate(scatter))
 
   def testUndefinedIndicesShape(self):
     indices = array_ops.placeholder(dtypes.int32, shape=None)
@@ -545,9 +545,9 @@ class ScatterNdTest(test.TestCase):
       expected_input_grad = np.array([[1, 2], [3, 4]],
                                      dtype=dtype.as_numpy_dtype())
       with self.cached_session():
-        self.assertAllEqual(expected_updates_grad, updates_grad.eval())
+        self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad))
         if self.non_aliasing_add_test:
-          self.assertAllEqual(expected_input_grad, input_grad.eval())
+          self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
   def testGradientsRank2SliceUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
@@ -565,9 +565,9 @@ class ScatterNdTest(test.TestCase):
       expected_input_grad = np.array([[3, 4], [1, 2]],
                                      dtype=dtype.as_numpy_dtype())
       with self.cached_session():
-        self.assertAllEqual(expected_updates_grad, updates_grad.eval())
+        self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad))
         if self.non_aliasing_add_test:
-          self.assertAllEqual(expected_input_grad, input_grad.eval())
+          self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
   def testGradientsRank3SliceUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
@@ -588,9 +588,9 @@ class ScatterNdTest(test.TestCase):
       expected_input_grad = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]],
                                      dtype=dtype.as_numpy_dtype())
       with self.cached_session():
-        self.assertAllEqual(expected_updates_grad, updates_grad.eval())
+        self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad))
         if self.non_aliasing_add_test:
-          self.assertAllEqual(expected_input_grad, input_grad.eval())
+          self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
   def testGradientsRank7SliceUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
@@ -615,9 +615,9 @@ class ScatterNdTest(test.TestCase):
           [[[[[[[1, 2], [3, 4]]]], [[[[5, 6], [7, 8]]]]]]],
           dtype=dtype.as_numpy_dtype())
       with self.cached_session():
-        self.assertAllEqual(expected_updates_grad, updates_grad.eval())
+        self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad))
         if self.non_aliasing_add_test:
-          self.assertAllEqual(expected_input_grad, input_grad.eval())
+          self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
   def testScatterNdRepatedIndicesAdd(self):
     indices = array_ops.zeros([100000, 1], dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/scatter_ops_test.py b/tensorflow/python/kernel_tests/scatter_ops_test.py
index 87c345245c..1c7006ac0b 100644
--- a/tensorflow/python/kernel_tests/scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_ops_test.py
@@ -286,7 +286,7 @@ class ScatterTest(test.TestCase):
 
         session.run([update0, update1])
 
-        self.assertAllEqual([False, True], var.eval())
+        self.assertAllEqual([False, True], self.evaluate(var))
 
   def testScatterOutOfRangeCpu(self):
     for op, _ in _TF_OPS_TO_NUMPY.items():
diff --git a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py
index 3f7e43b533..5ab889895e 100644
--- a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py
@@ -118,7 +118,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
           for np_op1, np_op2, tf_op in curr_ops_list:
             np_ans = self._segmentReduce(indices, np_x, np_op1, np_op2)
             s = tf_op(data=tf_x, segment_ids=indices)
-            tf_ans = s.eval()
+            tf_ans = self.evaluate(s)
             self.assertAllClose(np_ans, tf_ans)
             # NOTE(mrry): The static shape inference that computes
             # `tf_ans.shape` can only infer that sizes from dimension 1
@@ -141,7 +141,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         indices = [0, 1]
         s = math_ops.segment_sum(data=tf_x, segment_ids=indices)
         with self.assertRaisesOpError("segment_ids should be the same size"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentIdsValid(self):
     # This is a baseline for the following SegmentIdsInvalid* tests.
@@ -161,7 +161,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         indices = [1, 1, 2, 2]
         np_ans = self._segmentReduce(indices, np_x, np.add)
         s = math_ops.segment_sum(data=tf_x, segment_ids=indices)
-        tf_ans = s.eval()
+        tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
 
   def testSegmentIdsHole(self):
@@ -172,7 +172,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         indices = [0, 0, 3, 3]
         np_ans = self._segmentReduce(indices, np_x, np.add)
         s = math_ops.segment_sum(data=tf_x, segment_ids=indices)
-        tf_ans = s.eval()
+        tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
 
   def testSegmentIdsInvalid1(self):
@@ -184,7 +184,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
       with self.assertRaisesOpError(
           r"Segment id -1 out of range \[0, 1\), possibly because "
           "'segment_ids' input is not sorted."):
-        s.eval()
+        self.evaluate(s)
 
   def testSegmentIdsInvalid2(self):
     shape = [4, 4]
@@ -193,7 +193,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
       indices = [0, 1, 0, 1]
       s = math_ops.segment_sum(data=tf_x, segment_ids=indices)
       with self.assertRaisesOpError("segment ids are not increasing"):
-        s.eval()
+        self.evaluate(s)
 
   def testSegmentIdsInvalid3(self):
     shape = [4, 4]
@@ -204,7 +204,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
       with self.assertRaisesOpError(
           r"Segment id 1 out of range \[0, 1\), possibly "
           "because 'segment_ids' input is not sorted."):
-        s.eval()
+        self.evaluate(s)
 
   def testSegmentIdsInvalid4(self):
     shape = [4, 4]
@@ -214,7 +214,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         indices = [0, 0, 0, -1]
         s = math_ops.segment_sum(data=tf_x, segment_ids=indices)
         with self.assertRaisesOpError("segment ids must be >= 0"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentIdsInvalid5(self):
     shape = [4, 4]
@@ -224,7 +224,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         indices = [0, 0, 0, -2]
         s = math_ops.segment_sum(data=tf_x, segment_ids=indices)
         with self.assertRaisesOpError("segment ids must be >= 0"):
-          s.eval()
+          self.evaluate(s)
 
   def testGradient(self):
     shape = [4, 4]
@@ -297,7 +297,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
                   indices, np_x, np_op1, np_op2, num_segments=num_segments,
                   initial_value=init_op(dtype))
               s = tf_op(tf_x, segment_ids=indices, num_segments=num_segments)
-              tf_ans = s.eval()
+              tf_ans = self.evaluate(s)
               if dtype is dtypes_lib.bfloat16:
                 tf_ans = tf_ans.astype(np.float32)
               self.assertAllCloseAccordingToType(np_ans, tf_ans)
@@ -320,7 +320,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
               data=tf_x,
               segment_ids=indices,
               num_segments=num_segments_constant)
-          tf_ans = s.eval()
+          tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
         self.assertShapeEqual(np_ans, s)
 
@@ -412,7 +412,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
         unsorted = math_ops.unsorted_segment_sum([[17]], bad, num_segments=2)
         with self.assertRaisesOpError(
             r"segment_ids\[0,0\] = %d is out of range \[0, 2\)" % bad[0][0]):
-          unsorted.eval()
+          self.evaluate(unsorted)
 
   def testEmptySecondDimension(self):
     dtypes = [np.float16, np.float32, np.float64, np.int64, np.int32,
@@ -443,7 +443,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
           np.place(indices, indices == 8, [-1])
           s = math_ops.unsorted_segment_sum(
               data=tf_x, segment_ids=indices, num_segments=num_segments)
-          tf_ans = s.eval()
+          tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
         self.assertShapeEqual(np_ans, s)
 
@@ -499,7 +499,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
           np_ans = self._sparseSegmentReduce(np_x, np_indices, segment_indices,
                                              np_op1, np_op2)
           s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
-          tf_ans = s.eval()
+          tf_ans = self.evaluate(s)
           self.assertAllClose(np_ans, tf_ans)
           # NOTE(mrry): The static shape inference that computes
           # `tf_ans.shape` can only infer that sizes from dimension 1
@@ -518,7 +518,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         np_ans = self._sparseSegmentReduce(np_x, tf_indices, segment_indices,
                                            np_op1, np_op2)
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
-        tf_ans = s.eval()
+        tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
 
   def testWithNumSegments(self):
@@ -543,7 +543,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             indices=tf_indices,
             segment_ids=segment_indices,
             num_segments=num_segments)
-        tf_ans = s.eval()
+        tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
 
   def testWithEmptySegments(self):
@@ -562,7 +562,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             indices=tf_indices,
             segment_ids=segment_indices,
             num_segments=num_segments)
-        tf_ans = s.eval()
+        tf_ans = self.evaluate(s)
         self.assertAllClose(np.zeros([5, 4]), tf_ans)
 
   def testSegmentIdsGreaterThanZero(self):
@@ -576,7 +576,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         np_ans = self._sparseSegmentReduce(np_x, tf_indices, segment_indices,
                                            np_op1, np_op2)
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
-        tf_ans = s.eval()
+        tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
 
   def testValid(self):
@@ -588,7 +588,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
     with self.session(use_gpu=False):
       for tf_op in ops_list:
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
-        s.eval()
+        self.evaluate(s)
 
   def testIndicesInvalid1(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -600,7 +600,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
         with self.assertRaisesOpError(
             r"indices\[1\] == -1 out of range \[0, 10\)"):
-          s.eval()
+          self.evaluate(s)
 
   def testIndicesInvalid2(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -612,7 +612,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
         with self.assertRaisesOpError(
             r"indices\[3\] == 10 out of range \[0, 10\)"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentsInvalid2(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -623,7 +623,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
         with self.assertRaisesOpError("segment ids are not increasing"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentsInvalid3(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -636,7 +636,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError(
             r"Segment id 1 out of range \[0, 1\), possibly because "
             "'segment_ids' input is not sorted"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentsInvalid4(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -649,7 +649,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError(
             r"Segment id -1 out of range \[0, 2\), possibly because "
             "'segment_ids' input is not sorted"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentsInvalid6(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -660,7 +660,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
         with self.assertRaisesOpError("segment ids must be >= 0"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentsInvalid7(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -671,7 +671,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
         with self.assertRaisesOpError("segment ids must be >= 0"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentWithNumSegmentsValid(self):
     # Baseline for the test*WithNumSegmentsInvalid* methods below.
@@ -690,7 +690,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             indices=tf_indices,
             segment_ids=segment_indices,
             num_segments=num_segments)
-        s.eval()
+        self.evaluate(s)
 
   def testSegmentWithNumSegmentsInvalid1(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -709,7 +709,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             segment_ids=segment_indices,
             num_segments=num_segments)
         with self.assertRaisesOpError("segment ids must be < num_segments"):
-          s.eval()
+          self.evaluate(s)
 
   def testSegmentWithNumSegmentsInvalid2(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
@@ -785,7 +785,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
     with self.session(use_gpu=False):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
-        s.eval()
+        self.evaluate(s)
 
   def testGradientIndicesInvalid1(self):
     tf_x, _ = self._input([3, 4], dtype=dtypes_lib.float32)
@@ -798,7 +798,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         with self.assertRaisesOpError(r"Index 10 out of range \[0, 10\)"):
-          s.eval()
+          self.evaluate(s)
 
   def testGradientIndicesInvalid2(self):
     tf_x, _ = self._input([3, 4], dtype=dtypes_lib.float32)
@@ -811,7 +811,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         with self.assertRaisesOpError(r"Index -1 out of range \[0, 10\)"):
-          s.eval()
+          self.evaluate(s)
 
   def testGradientSegmentsInvalid1(self):
     tf_x, _ = self._input(
@@ -825,7 +825,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         with self.assertRaisesOpError("Invalid number of segments"):
-          s.eval()
+          self.evaluate(s)
 
   def testGradientSegmentsInvalid2(self):
     tf_x, _ = self._input([1, 4], dtype=dtypes_lib.float32)
@@ -838,7 +838,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         with self.assertRaisesOpError(r"Segment id 1 out of range \[0, 1\)"):
-          s.eval()
+          self.evaluate(s)
 
   def testGradientSegmentsInvalid3(self):
     tf_x, _ = self._input([2, 4], dtype=dtypes_lib.float32)
@@ -851,7 +851,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         with self.assertRaisesOpError(r"Segment id -1 out of range \[0, 2\)"):
-          s.eval()
+          self.evaluate(s)
 
   def testGradientSegmentsInvalid4(self):
     tf_x, _ = self._input([0, 4], dtype=dtypes_lib.float32)
@@ -864,7 +864,8 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
       for tf_op in ops_list:
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         with self.assertRaisesOpError(r"Segment id 0 out of range \[0, 0\)"):
-          s.eval()
+          self.evaluate(s)
+
 
 class SegmentReductionOpBenchmark(test.Benchmark):
   outer_dim_options = [2**x for x in range(9, 14, 2)]
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 1b4aff8c9c..85756b769d 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -164,8 +164,8 @@ def _GetSelfAdjointEigTest(dtype_, shape_, compute_v_):
         self.assertAllClose(a_ev.eval(), a, atol=atol)
 
         # Compare to numpy.linalg.eigh.
-        CompareEigenDecompositions(self, np_e, np_v,
-                                   tf_e.eval(), tf_v.eval(), atol)
+        CompareEigenDecompositions(self, np_e, np_v, self.evaluate(tf_e),
+                                   self.evaluate(tf_v), atol)
       else:
         tf_e = linalg_ops.self_adjoint_eigvals(constant_op.constant(a))
         self.assertAllClose(
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index ee813e5ffd..3e0eae326b 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -53,8 +53,8 @@ class ShapeOpsTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.shape(x)
       tf_ans_64 = array_ops.shape(x, out_type=dtypes.int64)
-      result = tf_ans.eval()
-      result_64 = tf_ans_64.eval()
+      result = self.evaluate(tf_ans)
+      result_64 = self.evaluate(tf_ans_64)
     self.assertAllEqual(np_ans, result)
     self.assertAllEqual(np_ans, result_64)
     self.assertShapeEqual(np_ans, tf_ans)
@@ -64,7 +64,7 @@ class ShapeOpsTest(test.TestCase):
     x_tf, unused_nnz = _sparsify(x_np)
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.shape(x_tf)
-      result = tf_ans.eval()
+      result = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, result)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -84,7 +84,7 @@ class ShapeOpsTest(test.TestCase):
     np_ans = np.asarray(np.ndim(x))
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.rank(x)
-      result = tf_ans.eval()
+      result = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, result)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -93,7 +93,7 @@ class ShapeOpsTest(test.TestCase):
     x_tf, unused_nnz = _sparsify(x_np)
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.rank(x_tf)
-      result = tf_ans.eval()
+      result = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, result)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -101,9 +101,9 @@ class ShapeOpsTest(test.TestCase):
     np_ans = np.asarray(np.size(x))
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.size(x)
-      result = tf_ans.eval()
+      result = self.evaluate(tf_ans)
       tf_ans_64 = array_ops.size(x, out_type=dtypes.int64)
-      result_64 = tf_ans_64.eval()
+      result_64 = self.evaluate(tf_ans_64)
     self.assertAllEqual(np_ans, result)
     self.assertAllEqual(np_ans, result_64)
     self.assertShapeEqual(np_ans, tf_ans)
@@ -113,7 +113,7 @@ class ShapeOpsTest(test.TestCase):
     x_tf, unused_nnz = _sparsify(x_np)
     with self.cached_session(use_gpu=use_gpu):
       tf_ans = array_ops.size(x_tf)
-      result = tf_ans.eval()
+      result = self.evaluate(tf_ans)
     self.assertAllEqual(np_ans, result)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -162,7 +162,7 @@ class ShapeOpsTest(test.TestCase):
       inp = array_ops.zeros([2**31])
       num_elements = array_ops.size_internal(
           inp, optimize=False, out_type=dtypes.int64)
-      self.assertEqual(2**31, num_elements.eval())
+      self.assertEqual(2**31, self.evaluate(num_elements))
 
     # Too large for tf.int32 output.
     with self.assertRaises(errors_impl.InvalidArgumentError):
@@ -170,13 +170,13 @@ class ShapeOpsTest(test.TestCase):
         inp = array_ops.zeros([2**31])
         num_elements = array_ops.size_internal(
             inp, optimize=False, out_type=dtypes.int32)
-        self.assertEqual(2**31, num_elements.eval())
+        self.assertEqual(2**31, self.evaluate(num_elements))
 
   def _compareExpandDims(self, x, dim, use_gpu):
     np_ans = np.expand_dims(x, axis=dim)
     with self.cached_session(use_gpu=use_gpu):
       tensor = array_ops.expand_dims(x, dim)
-      tf_ans = tensor.eval()
+      tf_ans = self.evaluate(tensor)
     self.assertShapeEqual(np_ans, tensor)
     self.assertAllEqual(np_ans, tf_ans)
 
@@ -264,7 +264,7 @@ class ShapeOpsTest(test.TestCase):
       np_ans = np.expand_dims(x, axis=0)
       with self.cached_session(use_gpu=True):
         tensor = array_ops.expand_dims(x, constant_op.constant(0, dtype))
-        tf_ans = tensor.eval()
+        tf_ans = self.evaluate(tensor)
       self.assertShapeEqual(np_ans, tensor)
       self.assertAllEqual(np_ans, tf_ans)
 
@@ -273,11 +273,11 @@ class ShapeOpsTest(test.TestCase):
       if squeeze_dims:
         np_ans = np.squeeze(x, axis=tuple(squeeze_dims))
         tensor = array_ops.squeeze(x, squeeze_dims)
-        tf_ans = tensor.eval()
+        tf_ans = self.evaluate(tensor)
       else:
         np_ans = np.squeeze(x)
         tensor = array_ops.squeeze(x)
-        tf_ans = tensor.eval()
+        tf_ans = self.evaluate(tensor)
     self.assertShapeEqual(np_ans, tensor)
     self.assertAllEqual(np_ans, tf_ans)
 
@@ -340,7 +340,7 @@ class ShapeOpsTest(test.TestCase):
       with self.cached_session(use_gpu=use_gpu):
         tensor = array_ops.squeeze(np.zeros([1, 1, 1]), [])
         self.assertEqual(np.shape(1), tensor.get_shape())
-        tf_ans = tensor.eval()
+        tf_ans = self.evaluate(tensor)
         self.assertEqual(np.shape(1), tf_ans.shape)
 
   def testSqueezeAllOnesBool(self):
@@ -350,7 +350,7 @@ class ShapeOpsTest(test.TestCase):
       with self.cached_session(use_gpu=use_gpu):
         tensor = array_ops.squeeze([[[False]]], [])
         self.assertEqual(np.shape(1), tensor.get_shape())
-        tf_ans = tensor.eval()
+        tf_ans = self.evaluate(tensor)
         self.assertEqual(np.shape(1), tf_ans.shape)
 
   def testSqueezeOnlyOnes(self):
@@ -415,7 +415,7 @@ class TileTest(test.TestCase):
       with self.cached_session(use_gpu=use_gpu):
         a = constant_op.constant(7, shape=[], dtype=dtypes.float32)
         tiled = array_ops.tile(a, [])
-        result = tiled.eval()
+        result = self.evaluate(tiled)
       self.assertEqual(result.shape, ())
       self.assertEqual([], tiled.get_shape())
       self.assertEqual(7, result)
@@ -427,7 +427,7 @@ class TileTest(test.TestCase):
         inp = np.random.rand(4, 1).astype(np.float32)
         a = constant_op.constant(inp)
         tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype))
-        result = tiled.eval()
+        result = self.evaluate(tiled)
       self.assertEqual(result.shape, (4, 4))
       self.assertEqual([4, 4], tiled.get_shape())
       self.assertTrue((result == np.tile(inp, (1, 4))).all())
@@ -437,7 +437,7 @@ class TileTest(test.TestCase):
       inp = np.random.rand(4, 1).astype(np.float32)
       a = constant_op.constant(inp)
       tiled = array_ops.tile(a, [1, 1])
-      result = tiled.eval()
+      result = self.evaluate(tiled)
     self.assertEqual(result.shape, (4, 1))
     self.assertEqual([4, 1], tiled.get_shape())
     self.assertTrue((result == np.tile(inp, (1, 1))).all())
@@ -447,7 +447,7 @@ class TileTest(test.TestCase):
       inp = np.random.rand(2, 3).astype(np.float32)
       a = constant_op.constant(inp)
       tiled = array_ops.tile(a, [5, 0])
-      result = tiled.eval()
+      result = self.evaluate(tiled)
     self.assertEqual(result.shape, (10, 0))
     self.assertEqual([10, 0], tiled.get_shape())
 
@@ -497,7 +497,7 @@ class TileTest(test.TestCase):
             shape=[4, 1],
             dtype=dtype_tf)
         tiled = array_ops.tile(a, [1, 4])
-        result = tiled.eval()
+        result = self.evaluate(tiled)
       self.assertEqual(result.shape, (4, 4))
       self.assertEqual([4, 4], tiled.get_shape())
       self.assertAllEqual(result, np.tile(inp, (1, 4)))
@@ -527,7 +527,7 @@ class TileTest(test.TestCase):
           dtype=dtypes.float32)
       multiples = np.random.randint(1, 4, size=rank).astype(np.int32)
       tiled = array_ops.tile(a, multiples)
-      result = tiled.eval()
+      result = self.evaluate(tiled)
     self.assertTrue((np.array(multiples) * np.array(inp.shape) == np.array(
         result.shape)).all())
     self.assertAllEqual(result, np.tile(inp, tuple(multiples)))
@@ -557,7 +557,7 @@ class TileTest(test.TestCase):
           [float(x) for x in grad_inp.flatten()], shape=grad_shape)
       grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0]
       self.assertShapeEqual(inp, grad)
-      result = grad.eval()
+      result = self.evaluate(grad)
     self.assertAllClose(np.sum(grad_inp, axis=1).reshape(4, 1), result, 1e-3)
 
   def testGradientStridedReduction(self):
@@ -572,7 +572,7 @@ class TileTest(test.TestCase):
           [float(x) for x in grad_inp.flatten()], shape=grad_shape)
       grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0]
       self.assertShapeEqual(inp, grad)
-      result = grad.eval()
+      result = self.evaluate(grad)
     expected_shape = [4, 2]
     expected = np.zeros(expected_shape)
     expected[:, 0] = grad_inp[:, 0] + grad_inp[:, 2]
@@ -590,7 +590,7 @@ class TileTest(test.TestCase):
       grad_tensor = constant_op.constant(
           [float(x) for x in grad_inp.flatten()], shape=grad_shape)
       grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0]
-      result = grad.eval()
+      result = self.evaluate(grad)
     self.assertAllClose(np.sum(grad_inp, axis=1).reshape(4, 1), result, 1e-3)
 
   def testGradientStridedReductionOnGPU(self):
@@ -604,7 +604,7 @@ class TileTest(test.TestCase):
       grad_tensor = constant_op.constant(
           [float(x) for x in grad_inp.flatten()], shape=grad_shape)
       grad = gradients_impl.gradients([tiled], [a], [grad_tensor])[0]
-      result = grad.eval()
+      result = self.evaluate(grad)
     expected_shape = [4, 2]
     expected = np.zeros(expected_shape)
     expected[:, 0] = grad_inp[:, 0] + grad_inp[:, 2]
diff --git a/tensorflow/python/kernel_tests/signal/mel_ops_test.py b/tensorflow/python/kernel_tests/signal/mel_ops_test.py
index 1ed4429b42..2b3dde30f3 100644
--- a/tensorflow/python/kernel_tests/signal/mel_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/mel_ops_test.py
@@ -141,7 +141,7 @@ class LinearToMelTest(test.TestCase):
       for config in configs:
         mel_matrix_np = spectrogram_to_mel_matrix(*config)
         mel_matrix = mel_ops.linear_to_mel_weight_matrix(*config)
-        self.assertAllClose(mel_matrix_np, mel_matrix.eval(), atol=3e-6)
+        self.assertAllClose(mel_matrix_np, self.evaluate(mel_matrix), atol=3e-6)
 
   def test_dtypes(self):
     # LinSpace is not supported for tf.float16.
diff --git a/tensorflow/python/kernel_tests/signal/shape_ops_test.py b/tensorflow/python/kernel_tests/signal/shape_ops_test.py
index 398fba8b6d..21a6b23b30 100644
--- a/tensorflow/python/kernel_tests/signal/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/shape_ops_test.py
@@ -150,7 +150,7 @@ class FrameTest(test.TestCase):
           op = shape_ops.frame(signal, frame_length, frame_step,
                                pad_end=pad_end, pad_value=99)
           with self.cached_session(use_gpu=True):
-            result = op.eval()
+            result = self.evaluate(op)
           self.assertEqual(op.shape.as_list(), list(result.shape))
 
   def test_basic_mono(self):
@@ -248,7 +248,7 @@ class FrameTest(test.TestCase):
       result = shape_ops.frame(signal, frame_length=2, frame_step=2,
                                pad_end=True, axis=1)
       expected = np.reshape(np.arange(16), (2, 2, 2, 2))
-      self.assertAllEqual(expected, result.eval())
+      self.assertAllEqual(expected, self.evaluate(result))
 
       result = shape_ops.frame(signal, frame_length=2, frame_step=1,
                                pad_end=True, axis=1)
@@ -260,7 +260,7 @@ class FrameTest(test.TestCase):
                    [[10, 11], [12, 13]],
                    [[12, 13], [14, 15]],
                    [[14, 15], [0, 0]]]]
-      self.assertAllEqual(expected, result.eval())
+      self.assertAllEqual(expected, self.evaluate(result))
 
       result = shape_ops.frame(signal, frame_length=3, frame_step=1,
                                pad_end=True, axis=1)
@@ -272,7 +272,7 @@ class FrameTest(test.TestCase):
                    [[10, 11], [12, 13], [14, 15]],
                    [[12, 13], [14, 15], [0, 0]],
                    [[14, 15], [0, 0], [0, 0]]]]
-      self.assertAllEqual(expected, result.eval())
+      self.assertAllEqual(expected, self.evaluate(result))
 
   def test_window_larger_than_signal(self):
     signal = constant_op.constant([[1, 2], [11, 12]], dtype=dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
index 26cb127063..7583c4d8fc 100644
--- a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
@@ -125,22 +125,22 @@ class SpectralOpsTest(test.TestCase):
       stft = spectral_ops.stft(signal, frame_length=7, frame_step=8,
                                pad_end=True)
       self.assertAllEqual([64, 5], stft.shape.as_list())
-      self.assertAllEqual([64, 5], stft.eval().shape)
+      self.assertAllEqual([64, 5], self.evaluate(stft).shape)
 
       stft = spectral_ops.stft(signal, frame_length=8, frame_step=8,
                                pad_end=True)
       self.assertAllEqual([64, 5], stft.shape.as_list())
-      self.assertAllEqual([64, 5], stft.eval().shape)
+      self.assertAllEqual([64, 5], self.evaluate(stft).shape)
 
       stft = spectral_ops.stft(signal, frame_length=8, frame_step=8,
                                fft_length=16, pad_end=True)
       self.assertAllEqual([64, 9], stft.shape.as_list())
-      self.assertAllEqual([64, 9], stft.eval().shape)
+      self.assertAllEqual([64, 9], self.evaluate(stft).shape)
 
       stft = spectral_ops.stft(signal, frame_length=16, frame_step=8,
                                fft_length=8, pad_end=True)
       self.assertAllEqual([64, 5], stft.shape.as_list())
-      self.assertAllEqual([64, 5], stft.eval().shape)
+      self.assertAllEqual([64, 5], self.evaluate(stft).shape)
 
       stft = np.zeros((32, 9)).astype(np.complex64)
 
@@ -148,7 +148,7 @@ class SpectralOpsTest(test.TestCase):
                                                fft_length=16, frame_step=8)
       expected_length = (stft.shape[0] - 1) * 8 + 8
       self.assertAllEqual([256], inverse_stft.shape.as_list())
-      self.assertAllEqual([expected_length], inverse_stft.eval().shape)
+      self.assertAllEqual([expected_length], self.evaluate(inverse_stft).shape)
 
   def test_stft_and_inverse_stft(self):
     """Test that spectral_ops.stft/inverse_stft match a NumPy implementation."""
diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py
index 41f040ab73..5bb34a632d 100644
--- a/tensorflow/python/kernel_tests/slice_op_test.py
+++ b/tensorflow/python/kernel_tests/slice_op_test.py
@@ -38,7 +38,7 @@ class SliceTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         a = constant_op.constant(inp, shape=[4, 4], dtype=dtypes.float32)
         slice_t = a[2, k:k]
-        slice_val = slice_t.eval()
+        slice_val = self.evaluate(slice_t)
       self.assertAllEqual(slice_val, inp[2, k:k])
 
   def testInt32(self):
@@ -47,7 +47,7 @@ class SliceTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         a = constant_op.constant(inp, shape=[4, 4], dtype=dtypes.int32)
         slice_t = a[2, k:k]
-        slice_val = slice_t.eval()
+        slice_val = self.evaluate(slice_t)
       self.assertAllEqual(slice_val, inp[2, k:k])
 
   def testSlicingWithInt64Index(self):
@@ -57,33 +57,33 @@ class SliceTest(test.TestCase):
       # Slice using int64 Tensor.
       i = constant_op.constant(1, dtype=dtypes.int64)
       slice_t = a[i]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual(1, slice_val)
       slice_t = a[i:i+1]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1], slice_val)
 
       # Slice using int64 integer.
       i = np.asarray(1).astype(np.int64)
       slice_t = a[i]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual(1, slice_val)
       slice_t = a[i:i+1]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1], slice_val)
 
       a_int32 = constant_op.constant([0, 1, 2], dtype=dtypes.int32)
       slice_t = array_ops.slice(a_int32,
                                 np.asarray([1]).astype(np.int64),
                                 np.asarray([2]).astype(np.int64))
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1, 2], slice_val)
 
       a_float32 = constant_op.constant([0, 1, 2], dtype=dtypes.float32)
       slice_t = array_ops.slice(a_float32,
                                 np.asarray([1]).astype(np.int64),
                                 np.asarray([2]).astype(np.int64))
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1, 2], slice_val)
 
   def testSlicingInt64Tensor(self):
@@ -93,23 +93,23 @@ class SliceTest(test.TestCase):
       # Slice using int32 Tensor.
       i = constant_op.constant(1, dtype=dtypes.int32)
       slice_t = a[i]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual(1, slice_val)
       slice_t = a[i:i + 1]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1], slice_val)
 
       # Slice using int32 integer.
       i = np.asarray(1).astype(np.int32)
       slice_t = a[i]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual(1, slice_val)
       slice_t = a[i:i + 1]
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1], slice_val)
 
       slice_t = array_ops.slice(a, [1], [2])
-      slice_val = slice_t.eval()
+      slice_val = self.evaluate(slice_t)
       self.assertAllEqual([1, 2], slice_val)
 
   def testSelectAll(self):
@@ -121,8 +121,8 @@ class SliceTest(test.TestCase):
         slice_explicit_t = array_ops.slice(a, [0, 0, 0, 0], [-1, -1, -1, -1])
         slice_implicit_t = a[:, :, :, :]
 
-        self.assertAllEqual(inp, slice_explicit_t.eval())
-        self.assertAllEqual(inp, slice_implicit_t.eval())
+        self.assertAllEqual(inp, self.evaluate(slice_explicit_t))
+        self.assertAllEqual(inp, self.evaluate(slice_implicit_t))
         self.assertEqual(inp.shape, slice_explicit_t.get_shape())
         self.assertEqual(inp.shape, slice_implicit_t.get_shape())
 
@@ -134,7 +134,7 @@ class SliceTest(test.TestCase):
 
         hi = np.random.randint(0, 9)
         scalar_t = a[hi]
-        scalar_val = scalar_t.eval()
+        scalar_val = self.evaluate(scalar_t)
         self.assertAllEqual(scalar_val, inp[hi])
 
         if hi > 0:
@@ -142,7 +142,7 @@ class SliceTest(test.TestCase):
         else:
           lo = 0
         slice_t = a[lo:hi]
-        slice_val = slice_t.eval()
+        slice_val = self.evaluate(slice_t)
         self.assertAllEqual(slice_val, inp[lo:hi])
 
   def testScalarInput(self):
@@ -195,7 +195,7 @@ class SliceTest(test.TestCase):
 
         x, y = np.random.randint(0, 3, size=2).tolist()
         slice_t = a[x, 0:y]
-        slice_val = slice_t.eval()
+        slice_val = self.evaluate(slice_t)
       self.assertAllEqual(slice_val, inp[x, 0:y])
 
   def testSimple(self):
@@ -282,7 +282,7 @@ class SliceTest(test.TestCase):
       grads = np.random.rand(num_grads).astype("f").reshape(slice_size)
       grad_tensor = constant_op.constant(grads)
       grad = gradients_impl.gradients(slice_t, [a], grad_tensor)[0]
-      result = grad.eval()
+      result = self.evaluate(grad)
 
     # Create a zero tensor of the input shape ane place
     # the grads into the right location to compare against TensorFlow.
@@ -368,7 +368,7 @@ class SliceTest(test.TestCase):
       c = b[:-1, :]
       d = c[1, :]
       res = 2 * d - c[1, :] + a[2, :] - 2 * b[-2, :]
-      self.assertAllEqual([0, 0, 0], res.eval())
+      self.assertAllEqual([0, 0, 0], self.evaluate(res))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/softmax_op_test.py b/tensorflow/python/kernel_tests/softmax_op_test.py
index ef9301d4e3..8b1a2e4c4e 100644
--- a/tensorflow/python/kernel_tests/softmax_op_test.py
+++ b/tensorflow/python/kernel_tests/softmax_op_test.py
@@ -64,7 +64,7 @@ class SoftmaxTest(test.TestCase):
         tf_softmax = nn_ops.log_softmax(np_features, axis=dim, name=name)
       else:
         tf_softmax = nn_ops.softmax(np_features, axis=dim, name=name)
-      out = tf_softmax.eval()
+      out = self.evaluate(tf_softmax)
     self.assertAllCloseAccordingToType(np_softmax, out)
     self.assertShapeEqual(np_softmax, tf_softmax)
     if not log:
@@ -113,7 +113,7 @@ class SoftmaxTest(test.TestCase):
     features = np.array([[1., 1., 1., 1.], [max, 1., 2., 3.]]).astype(type)
     with self.cached_session(use_gpu=use_gpu):
       tf_log_softmax = nn_ops.log_softmax(features)
-      out = tf_log_softmax.eval()
+      out = self.evaluate(tf_log_softmax)
     self.assertAllClose(
         np.array([[-1.386294, -1.386294, -1.386294, -1.386294],
                   [0, -max, -max, -max]]),
diff --git a/tensorflow/python/kernel_tests/softplus_op_test.py b/tensorflow/python/kernel_tests/softplus_op_test.py
index 50a8291ea8..48445a7380 100644
--- a/tensorflow/python/kernel_tests/softplus_op_test.py
+++ b/tensorflow/python/kernel_tests/softplus_op_test.py
@@ -39,7 +39,7 @@ class SoftplusTest(test.TestCase):
     np_softplus = self._npSoftplus(np_features)
     with self.cached_session(use_gpu=use_gpu):
       softplus = nn_ops.softplus(np_features)
-      tf_softplus = softplus.eval()
+      tf_softplus = self.evaluate(softplus)
     self.assertAllCloseAccordingToType(np_softplus, tf_softplus)
     self.assertTrue(np.all(tf_softplus > 0))
     self.assertShapeEqual(np_softplus, softplus)
diff --git a/tensorflow/python/kernel_tests/softsign_op_test.py b/tensorflow/python/kernel_tests/softsign_op_test.py
index ee2e2e0303..71aac7e48e 100644
--- a/tensorflow/python/kernel_tests/softsign_op_test.py
+++ b/tensorflow/python/kernel_tests/softsign_op_test.py
@@ -36,7 +36,7 @@ class SoftsignTest(test.TestCase):
     np_softsign = self._npSoftsign(np_features)
     with self.cached_session(use_gpu=use_gpu):
       softsign = nn_ops.softsign(np_features)
-      tf_softsign = softsign.eval()
+      tf_softsign = self.evaluate(softsign)
     self.assertAllClose(np_softsign, tf_softsign)
     self.assertShapeEqual(np_softsign, softsign)
 
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index b05f14f738..c32b4ff42d 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -138,13 +138,13 @@ class SpaceToDepthTest(test.TestCase):
       # test NHWC (default) on CPU
       x_tf = array_ops.space_to_depth(input_nhwc, block_size)
       self.assertAllEqual(x_tf.shape, x_out.shape)
-      x_tf.eval()
+      self.evaluate(x_tf)
     if test.is_gpu_available():
       with self.session(use_gpu=True):
         # test NHWC (default) on GPU
         x_tf = array_ops.space_to_depth(input_nhwc, block_size)
         self.assertAllEqual(x_tf.shape, x_out.shape)
-        x_tf.eval()
+        self.evaluate(x_tf)
 
   # Tests for different width and height.
   def testNonSquare(self):
@@ -163,7 +163,7 @@ class SpaceToDepthTest(test.TestCase):
     block_size = 2
     with self.assertRaises(ValueError):
       out_tf = array_ops.space_to_depth(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   def testInputWrongDimMissingBatch(self):
     # The input is missing the first dimension ("batch")
@@ -178,7 +178,7 @@ class SpaceToDepthTest(test.TestCase):
     block_size = 0
     with self.assertRaises(ValueError):
       out_tf = array_ops.space_to_depth(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   def testBlockSizeOne(self):
     # The block size is 1. The block size needs to be > 1.
@@ -186,7 +186,7 @@ class SpaceToDepthTest(test.TestCase):
     block_size = 1
     with self.assertRaises(ValueError):
       out_tf = array_ops.space_to_depth(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   def testBlockSizeLarger(self):
     # The block size is too large for this input.
@@ -194,7 +194,7 @@ class SpaceToDepthTest(test.TestCase):
     block_size = 10
     with self.assertRaises(ValueError):
       out_tf = array_ops.space_to_depth(x_np, block_size)
-      out_tf.eval()
+      self.evaluate(out_tf)
 
   def testBlockSizeNotDivisibleWidth(self):
     # The block size divides width but not height.
diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
index 6e0714da70..17e867439a 100644
--- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
@@ -331,7 +331,7 @@ class SparseCrossOpTest(test.TestCase):
         [t2, t1], num_buckets=1024, hash_key=sparse_ops._DEFAULT_HASH_KEY + 1)
     cross_dense = sparse_ops.sparse_tensor_to_dense(cross)
     with session.Session():
-      values = cross_dense.eval()
+      values = self.evaluate(cross_dense)
       self.assertTrue(numpy.not_equal(values[0], values[1]).all())
 
   def test_hashed_3x1x2(self):
diff --git a/tensorflow/python/kernel_tests/sparse_matmul_op_test.py b/tensorflow/python/kernel_tests/sparse_matmul_op_test.py
index 541463e76b..4de69a26e3 100644
--- a/tensorflow/python/kernel_tests/sparse_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_matmul_op_test.py
@@ -58,7 +58,7 @@ class SparseMatMulTest(test.TestCase):
           transpose_b=tr_b,
           a_is_sparse=sp_a,
           b_is_sparse=sp_b)
-      out = tf_ans.eval()
+      out = self.evaluate(tf_ans)
       np_x = math_ops.cast(tf_x, dtypes.float32).eval()
       np_y = math_ops.cast(tf_y, dtypes.float32).eval()
 
diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index a45ce2e13b..db3f6c44e2 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -635,7 +635,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
       else:
         tf_dense_ans = sparse_ops.sparse_reduce_max(sp_t, reduction_axes,
                                                     keep_dims)
-      out_dense = tf_dense_ans.eval()
+      out_dense = self.evaluate(tf_dense_ans)
 
       if do_sum:
         tf_sparse_ans = sparse_ops.sparse_reduce_sum_sparse(sp_t,
@@ -710,16 +710,16 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
           axes = np.random.choice(len(dims), size=d, replace=False).tolist()
           reduced = sparse_ops.sparse_reduce_sum(sp_t, axes)
 
-          err = gradient_checker.compute_gradient_error(sp_t.values, (nnz,),
-                                                        reduced,
-                                                        reduced.eval().shape)
+          err = gradient_checker.compute_gradient_error(
+              sp_t.values, (nnz,), reduced,
+              self.evaluate(reduced).shape)
           self.assertLess(err, 1e-3)
 
         # Tests for negative axes.
         reduced = sparse_ops.sparse_reduce_sum(sp_t, -1)
-        err = gradient_checker.compute_gradient_error(sp_t.values, (nnz,),
-                                                      reduced,
-                                                      reduced.eval().shape)
+        err = gradient_checker.compute_gradient_error(
+            sp_t.values, (nnz,), reduced,
+            self.evaluate(reduced).shape)
         self.assertLess(err, 1e-3)
 
 
diff --git a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py
index fe334045af..e605cb1c35 100644
--- a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py
@@ -80,7 +80,7 @@ class SparseTensorDenseMatMulTest(test.TestCase):
       self.assertEqual(tf_value_ans.get_shape()[1], np_ans.shape[1])
       self.assertEqual(tf_tensor_ans.get_shape()[1], np_ans.shape[1])
 
-      for out in (tf_value_ans.eval(), tf_tensor_ans.eval()):
+      for out in (tf_value_ans.eval(), self.evaluate(tf_tensor_ans)):
         if x.dtype == np.float32:
           self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-4)
         elif x.dtype == np.float64:
diff --git a/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py b/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py
index 7f63532e10..fa6cb13432 100644
--- a/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py
+++ b/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py
@@ -104,20 +104,20 @@ class SparseToDenseTest(test.TestCase):
       with self.assertRaisesOpError(
           r"sparse_values has incorrect shape \[2,1\], "
           r"should be \[\] or \[2\]"):
-        dense.eval()
+        self.evaluate(dense)
 
   def testBadNumValues(self):
     with self.cached_session():
       dense = _SparseToDense([1, 3], [5], [1, 2, 3], -1)
       with self.assertRaisesOpError(
           r"sparse_values has incorrect shape \[3\], should be \[\] or \[2\]"):
-        dense.eval()
+        self.evaluate(dense)
 
   def testBadDefault(self):
     with self.cached_session():
       dense = _SparseToDense([1, 3], [5], [1, 2], [0])
       with self.assertRaisesOpError("default_value should be a scalar"):
-        dense.eval()
+        self.evaluate(dense)
 
   def testOutOfBoundsIndicesWithWithoutValidation(self):
     with self.cached_session():
@@ -128,7 +128,7 @@ class SparseToDenseTest(test.TestCase):
           default_value=0.0)
       with self.assertRaisesOpError(
           r"indices\[1\] = \[10\] is out of bounds: need 0 <= index < \[5\]"):
-        dense.eval()
+        self.evaluate(dense)
       # Disable checks, the allocation should still fail.
       with self.assertRaisesOpError("out of bounds"):
         dense_without_validation = _SparseToDense(
@@ -137,7 +137,7 @@ class SparseToDenseTest(test.TestCase):
             sparse_values=[-1.0, 1.0],
             default_value=0.0,
             validate_indices=False)
-        dense_without_validation.eval()
+        self.evaluate(dense_without_validation)
 
   def testRepeatingIndicesWithWithoutValidation(self):
     with self.cached_session():
@@ -147,7 +147,7 @@ class SparseToDenseTest(test.TestCase):
           sparse_values=[-1.0, 1.0],
           default_value=0.0)
       with self.assertRaisesOpError(r"indices\[1\] = \[1\] is repeated"):
-        dense.eval()
+        self.evaluate(dense)
       # Disable checks
       dense_without_validation = _SparseToDense(
           sparse_indices=[[1], [1]],
@@ -155,7 +155,7 @@ class SparseToDenseTest(test.TestCase):
           sparse_values=[-1.0, 1.0],
           default_value=0.0,
           validate_indices=False)
-      dense_without_validation.eval()
+      self.evaluate(dense_without_validation)
 
   def testUnsortedIndicesWithWithoutValidation(self):
     with self.cached_session():
@@ -165,7 +165,7 @@ class SparseToDenseTest(test.TestCase):
           sparse_values=[-1.0, 1.0],
           default_value=0.0)
       with self.assertRaisesOpError(r"indices\[1\] = \[1\] is out of order"):
-        dense.eval()
+        self.evaluate(dense)
       # Disable checks
       dense_without_validation = _SparseToDense(
           sparse_indices=[[2], [1]],
@@ -173,7 +173,7 @@ class SparseToDenseTest(test.TestCase):
           sparse_values=[-1.0, 1.0],
           default_value=0.0,
           validate_indices=False)
-      dense_without_validation.eval()
+      self.evaluate(dense_without_validation)
 
   def testShapeInferenceKnownShape(self):
     with self.session(use_gpu=False):
diff --git a/tensorflow/python/kernel_tests/sparse_xent_op_test.py b/tensorflow/python/kernel_tests/sparse_xent_op_test.py
index 0510bc5321..3f91131dab 100644
--- a/tensorflow/python/kernel_tests/sparse_xent_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_xent_op_test.py
@@ -164,7 +164,7 @@ class SparseXentTest(test.TestCase):
     with self.session(use_gpu=True):
       loss = nn_ops.sparse_softmax_cross_entropy_with_logits(
           labels=constant_op.constant(0), logits=constant_op.constant([1.0]))
-      self.assertAllClose(0.0, loss.eval())
+      self.assertAllClose(0.0, self.evaluate(loss))
 
   def testFloat(self):
     for label_dtype in np.int32, np.int64:
diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py
index 944b0e59b1..af90e03966 100644
--- a/tensorflow/python/kernel_tests/split_op_test.py
+++ b/tensorflow/python/kernel_tests/split_op_test.py
@@ -318,7 +318,7 @@ class SplitOpTest(test.TestCase):
       inp_grads = [self._makeData((4, 1), dtype)for _ in range(4)]
       grad_tensors = [constant_op.constant(x) for x in inp_grads]
       grad = gradients_impl.gradients(s, [inp_tensor], grad_tensors)[0]
-      result = grad.eval()
+      result = self.evaluate(grad)
     for i in range(4):
       self.assertAllEqual(result[:, i:i + 1], inp_grads[i])
 
diff --git a/tensorflow/python/kernel_tests/stack_op_test.py b/tensorflow/python/kernel_tests/stack_op_test.py
index 4b355620bf..0f1fa97c38 100644
--- a/tensorflow/python/kernel_tests/stack_op_test.py
+++ b/tensorflow/python/kernel_tests/stack_op_test.py
@@ -204,11 +204,11 @@ class StackOpTest(test.TestCase):
         with self.cached_session(use_gpu=True):
           actual_pack = array_ops.stack(test_arrays, axis=j)
           self.assertEqual(expected.shape, actual_pack.get_shape())
-          actual_pack = actual_pack.eval()
+          actual_pack = self.evaluate(actual_pack)
 
           actual_stack = array_ops.stack(test_arrays, axis=j)
           self.assertEqual(expected.shape, actual_stack.get_shape())
-          actual_stack = actual_stack.eval()
+          actual_stack = self.evaluate(actual_stack)
 
         self.assertNDArrayNear(expected, actual_stack, 1e-6)
 
@@ -253,17 +253,19 @@ class AutomaticStackingTest(test.TestCase):
                                           [[2., 2.], [3., 3.]],
                                           dtype=np.float32)])
       self.assertAllEqual([[[0., 0.], [1., 1.]], [[2., 2.], [3., 3.]]],
-                          result.eval())
+                          self.evaluate(result))
 
   def testVariable(self):
     with self.session(use_gpu=True):
       v = variables.Variable(17)
       result = ops.convert_to_tensor([[0, 0, 0], [0, v, 0], [0, 0, 0]])
       v.initializer.run()
-      self.assertAllEqual([[0, 0, 0], [0, 17, 0], [0, 0, 0]], result.eval())
+      self.assertAllEqual([[0, 0, 0], [0, 17, 0], [0, 0, 0]],
+                          self.evaluate(result))
 
       v.assign(38).op.run()
-      self.assertAllEqual([[0, 0, 0], [0, 38, 0], [0, 0, 0]], result.eval())
+      self.assertAllEqual([[0, 0, 0], [0, 38, 0], [0, 0, 0]],
+                          self.evaluate(result))
 
   def testDtype(self):
     t_0 = ops.convert_to_tensor([[0., 0., 0.], [0., 0., 0.], [0., 0., 0.]])
diff --git a/tensorflow/python/kernel_tests/stack_ops_test.py b/tensorflow/python/kernel_tests/stack_ops_test.py
index 1aa12009ea..6c6fe8aba4 100644
--- a/tensorflow/python/kernel_tests/stack_ops_test.py
+++ b/tensorflow/python/kernel_tests/stack_ops_test.py
@@ -39,7 +39,7 @@ class StackOpTest(test.TestCase):
       c = gen_data_flow_ops.stack_push_v2(h, [[4.0, 5.0]])
       with ops.control_dependencies([c]):
         c1 = gen_data_flow_ops.stack_pop_v2(h, dtypes.float32)
-      self.assertAllClose([[4.0, 5.0]], c1.eval())
+      self.assertAllClose([[4.0, 5.0]], self.evaluate(c1))
 
   def testStackPushPop(self):
     self._testStackPushPop(use_gpu=False)
@@ -54,7 +54,7 @@ class StackOpTest(test.TestCase):
       c = gen_data_flow_ops.stack_push_v2(h, x, swap_memory=True)
       with ops.control_dependencies([c]):
         c1 = gen_data_flow_ops.stack_pop_v2(h, dtypes.float32)
-      self.assertAllClose(a, c1.eval())
+      self.assertAllClose(a, self.evaluate(c1))
 
   def testStackPushPopSwap(self):
     self._testStackPushPopSwap(use_gpu=False)
@@ -91,7 +91,7 @@ class StackOpTest(test.TestCase):
 
       _, ry = control_flow_ops.while_loop(
           c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()])
-      self.assertAllClose(np.ones(2000) * 10.0, ry.eval())
+      self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry))
 
   def testStackWhileSwap(self):
     self._testStackWhileSwap(use_gpu=False)
@@ -110,7 +110,7 @@ class StackOpTest(test.TestCase):
       with ops.control_dependencies([c2]):
         c2 = gen_data_flow_ops.stack_pop_v2(h2, dtypes.float32)
       r = c1 + c2
-      self.assertAllClose(9.0, r.eval())
+      self.assertAllClose(9.0, self.evaluate(r))
 
   def testMultiStack(self):
     self._testMultiStack(use_gpu=False)
@@ -173,7 +173,7 @@ class StackOpRefTest(test.TestCase):
       c = gen_data_flow_ops.stack_push(h, [[4.0, 5.0]])
       with ops.control_dependencies([c]):
         c1 = gen_data_flow_ops.stack_pop(h, dtypes.float32)
-      self.assertAllClose([[4.0, 5.0]], c1.eval())
+      self.assertAllClose([[4.0, 5.0]], self.evaluate(c1))
 
   def testStackPushPop(self):
     self._testStackPushPop(use_gpu=False)
@@ -187,7 +187,7 @@ class StackOpRefTest(test.TestCase):
       c = gen_data_flow_ops.stack_push(h, x, swap_memory=True)
       with ops.control_dependencies([c]):
         c1 = gen_data_flow_ops.stack_pop(h, dtypes.float32)
-      self.assertAllClose(a, c1.eval())
+      self.assertAllClose(a, self.evaluate(c1))
 
   def testStackPushPopSwap(self):
     self._testStackPushPopSwap(use_gpu=False)
@@ -204,7 +204,7 @@ class StackOpRefTest(test.TestCase):
       with ops.control_dependencies([c2]):
         c2 = gen_data_flow_ops.stack_pop(h2, dtypes.float32)
       r = c1 + c2
-      self.assertAllClose(9.0, r.eval())
+      self.assertAllClose(9.0, self.evaluate(r))
 
   def _testStackWhileSwap(self, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
@@ -236,7 +236,7 @@ class StackOpRefTest(test.TestCase):
 
       _, ry = control_flow_ops.while_loop(
           c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()])
-      self.assertAllClose(np.ones(2000) * 10.0, ry.eval())
+      self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry))
 
   def testStackWhileSwap(self):
     self._testStackWhileSwap(use_gpu=False)
@@ -253,7 +253,7 @@ class StackOpRefTest(test.TestCase):
       h2 = gen_data_flow_ops._stack(dtypes.float32, stack_name="foo")
       c2 = gen_data_flow_ops.stack_push(h2, 5.0)
       _ = c1 + c2
-      self.assertNotEqual(h1.eval()[1], h2.eval()[1])
+      self.assertNotEqual(h1.eval()[1], self.evaluate(h2)[1])
 
   def testSameNameStacks(self):
     self._testSameNameStacks(use_gpu=False)
diff --git a/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py b/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py
index 9cb0c9d18f..2cc87008da 100644
--- a/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py
+++ b/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py
@@ -70,7 +70,7 @@ class StringToHashBucketOpTest(test.TestCase):
       input_string = constant_op.constant(['a', 'b', 'c'])
       output = string_ops.string_to_hash_bucket_strong(
           input_string, 1, key=[123, 345])
-      self.assertAllEqual([0, 0, 0], output.eval())
+      self.assertAllEqual([0, 0, 0], self.evaluate(output))
 
   def testStringToHashBucketsStrong(self):
     with self.cached_session():
@@ -81,7 +81,7 @@ class StringToHashBucketOpTest(test.TestCase):
       # StrongKeyedHash(key, 'a') -> 7157389809176466784 -> mod 10 -> 4
       # StrongKeyedHash(key, 'b') -> 15805638358933211562 -> mod 10 -> 2
       # StrongKeyedHash(key, 'c') -> 18100027895074076528 -> mod 10 -> 8
-      self.assertAllEqual([4, 2, 8], output.eval())
+      self.assertAllEqual([4, 2, 8], self.evaluate(output))
 
   def testStringToHashBucketsStrongInvalidKey(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/substr_op_test.py b/tensorflow/python/kernel_tests/substr_op_test.py
index 37aa624b07..bb2d4a7913 100644
--- a/tensorflow/python/kernel_tests/substr_op_test.py
+++ b/tensorflow/python/kernel_tests/substr_op_test.py
@@ -51,7 +51,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     length = np.array(3, dtype)
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
   @parameterized.parameters(
@@ -71,7 +71,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     length = np.array(3, dtype)
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
     # Full string
@@ -83,7 +83,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     length = np.array(5, dtype)
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, test_string)
 
     # Full string (Negative)
@@ -95,7 +95,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     length = np.array(5, dtype)
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, test_string)
 
     # Length is larger in magnitude than a negative position
@@ -111,7 +111,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     length = np.array(5, dtype)
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_string)
 
   @parameterized.parameters(
@@ -138,7 +138,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     length = np.array(3, dtype)
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
   @parameterized.parameters(
@@ -173,7 +173,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     }[unit]
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
     position = np.array(-3, dtype)
@@ -188,7 +188,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     }[unit]
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
   @parameterized.parameters(
@@ -229,7 +229,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     }[unit]
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
   @parameterized.parameters(
@@ -271,7 +271,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     }[unit]
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
     # Broadcast input string onto pos/len
@@ -294,7 +294,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     }[unit]
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
     # Test 1D broadcast
@@ -310,7 +310,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     }[unit]
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
-      substr = substr_op.eval()
+      substr = self.evaluate(substr_op)
       self.assertAllEqual(substr, expected_value)
 
   @parameterized.parameters(
@@ -349,7 +349,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        substr_op.eval()
+        self.evaluate(substr_op)
 
   @parameterized.parameters(
       (np.int32, 4, "BYTE"),
@@ -373,7 +373,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        substr_op.eval()
+        self.evaluate(substr_op)
 
   @parameterized.parameters(
       (np.int32, "BYTE"),
@@ -398,7 +398,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        substr_op.eval()
+        self.evaluate(substr_op)
 
     # Matrix/Matrix (with negative)
     position = np.array([[1, 2, -3], [1, 2, -4], [1, 2, -3]], dtype)
@@ -406,7 +406,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        substr_op.eval()
+        self.evaluate(substr_op)
 
   @parameterized.parameters(
       (np.int32, "BYTE"),
@@ -428,7 +428,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        substr_op.eval()
+        self.evaluate(substr_op)
 
     # Broadcast (with negative)
     position = np.array([-1, -2, -4], dtype)
@@ -436,7 +436,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     substr_op = string_ops.substr(test_string, position, length, unit=unit)
     with self.cached_session():
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        substr_op.eval()
+        self.evaluate(substr_op)
 
   @parameterized.parameters(
       (np.int32, "BYTE"),
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index 57298c0fec..32c97a7b19 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -123,7 +123,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
     # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity.
     xx = math_ops.matmul(x, x, adjoint_a=True)
     identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0)
-    self.assertAllClose(identity.eval(), xx.eval(), atol=tol)
+    self.assertAllClose(identity.eval(), self.evaluate(xx), atol=tol)
 
   def Test(self):
     is_complex = dtype_ in (np.complex64, np.complex128)
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 0188eb246f..08578c0c49 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -429,7 +429,7 @@ class TensorArrayTest(test.TestCase):
             handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow)
         with self.assertRaisesOpError(
             "TensorArray dtype is float but Op requested dtype double."):
-          r0_bad.eval()
+          self.evaluate(r0_bad)
 
       # Test reading from a negative index, which is not allowed
       with self.assertRaisesOpError("index -1"):
@@ -1212,7 +1212,7 @@ class TensorArrayTest(test.TestCase):
       w0 = ta.unstack(x)
       w1 = w0.write(3, 4.0)
       r = w1.stack()
-      self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), r.eval())
+      self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
       self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
 
@@ -1227,7 +1227,7 @@ class TensorArrayTest(test.TestCase):
       w0 = ta.split(x, [1, 1, 1])
       w1 = w0.write(3, [4.0])
       r = w1.concat()
-      self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), r.eval())
+      self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
       self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
 
@@ -1255,10 +1255,10 @@ class TensorArrayTest(test.TestCase):
       ta.unstack(array_ops.zeros([0, 3, 5])).mark_used()
       packed = ta.stack()
       concatenated = ta.concat()
-      self.assertAllEqual([0, 3, 5], packed.eval().shape)
+      self.assertAllEqual([0, 3, 5], self.evaluate(packed).shape)
       # Concatenating zero tensors along their first dimension gives a
       # first dimension of zero
-      self.assertAllEqual([0, 5], concatenated.eval().shape)
+      self.assertAllEqual([0, 5], self.evaluate(concatenated).shape)
 
   def testTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py
index 8c11c20709..76e1002ee1 100644
--- a/tensorflow/python/kernel_tests/transpose_op_test.py
+++ b/tensorflow/python/kernel_tests/transpose_op_test.py
@@ -50,7 +50,7 @@ class TransposeTest(test.TestCase):
     with self.cached_session(use_gpu=False):
       inx = ops.convert_to_tensor(x)
       y = array_ops.transpose(inx, p, conjugate=conjugate)
-      tf_ans = y.eval()
+      tf_ans = self.evaluate(y)
       self.assertShapeEqual(np_ans, y)
       self.assertAllEqual(np_ans, tf_ans)
 
@@ -81,7 +81,7 @@ class TransposeTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       inx = ops.convert_to_tensor(x)
       y = array_ops.transpose(inx, p, conjugate=conjugate)
-      tf_ans = y.eval()
+      tf_ans = self.evaluate(y)
 
       self.assertAllEqual(np_ans, tf_ans)
       self.assertShapeEqual(np_ans, y)
@@ -168,7 +168,7 @@ class TransposeTest(test.TestCase):
         with self.cached_session(use_gpu=True):
           inx = ops.convert_to_tensor(inp)
           y = array_ops.transpose(inx, perm)
-          tf_ans = y.eval()
+          tf_ans = self.evaluate(y)
         self.assertAllEqual(np_ans, tf_ans)
         self.assertShapeEqual(np_ans, y)
 
@@ -189,7 +189,7 @@ class TransposeTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         inx = ops.convert_to_tensor(inp)
         y = array_ops.transpose(inx, perm)
-        tf_ans = y.eval()
+        tf_ans = self.evaluate(y)
       self.assertAllEqual(np_ans, tf_ans)
       self.assertShapeEqual(np_ans, y)
 
@@ -224,7 +224,7 @@ class TransposeTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         inx = ops.convert_to_tensor(inp)
         y = array_ops.transpose(inx, perm)
-        tf_ans = y.eval()
+        tf_ans = self.evaluate(y)
       self.assertAllEqual(np_ans, tf_ans)
       self.assertShapeEqual(np_ans, y)
 
@@ -246,7 +246,7 @@ class TransposeTest(test.TestCase):
         with self.cached_session(use_gpu=True):
           inx = ops.convert_to_tensor(inp)
           y = array_ops.transpose(inx, perm)
-          tf_ans = y.eval()
+          tf_ans = self.evaluate(y)
         self.assertAllEqual(np_ans, tf_ans)
         self.assertShapeEqual(np_ans, y)
 
@@ -267,7 +267,7 @@ class TransposeTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         inx = ops.convert_to_tensor(inp)
         y = array_ops.transpose(inx, perm)
-        tf_ans = y.eval()
+        tf_ans = self.evaluate(y)
       self.assertAllEqual(np_ans, tf_ans)
       self.assertShapeEqual(np_ans, y)
 
@@ -319,7 +319,7 @@ class TransposeTest(test.TestCase):
       with self.cached_session(use_gpu=True):
         inx = ops.convert_to_tensor(inp)
         y = array_ops.transpose(inx, perm)
-        tf_ans = y.eval()
+        tf_ans = self.evaluate(y)
       self.assertAllEqual(np_ans, tf_ans)
       self.assertShapeEqual(np_ans, y)
       self._ClearCachedSession()
@@ -341,7 +341,7 @@ class TransposeTest(test.TestCase):
         inx = ops.convert_to_tensor(x)
         inp = constant_op.constant(p)
         y = array_ops.transpose(inx, inp)
-        tf_ans = y.eval()
+        tf_ans = self.evaluate(y)
         self.assertShapeEqual(np_ans, y)
         self.assertAllEqual(np_ans, tf_ans)
 
diff --git a/tensorflow/python/kernel_tests/unicode_decode_op_test.py b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
index 34dae9b731..c34145bff1 100644
--- a/tensorflow/python/kernel_tests/unicode_decode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
@@ -58,10 +58,10 @@ class UnicodeDecodeTest(test.TestCase):
           codepoint("更"),
           codepoint("新")
       ],
-                          utf8_text.eval().tolist())
-      self.assertAllEqual([0, 4, 14], row_splits.eval().tolist())
+                          self.evaluate(utf8_text).tolist())
+      self.assertAllEqual([0, 4, 14], self.evaluate(row_splits).tolist())
       self.assertAllEqual([0, 3, 6, 9, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27],
-                          offsets.eval().tolist())
+                          self.evaluate(offsets).tolist())
 
   def testBasicDecodeWithOffset(self):
     text = constant_op.constant(["仅今年前"])
@@ -75,9 +75,9 @@ class UnicodeDecodeTest(test.TestCase):
           codepoint("年"),
           codepoint("前"),
       ],
-                          utf8_text.eval().tolist())
-      self.assertAllEqual(row_splits.eval().tolist(), [0, 4])
-      self.assertAllEqual(starts.eval().tolist(), [0, 3, 6, 9])
+                          self.evaluate(utf8_text).tolist())
+      self.assertAllEqual(self.evaluate(row_splits).tolist(), [0, 4])
+      self.assertAllEqual(self.evaluate(starts).tolist(), [0, 3, 6, 9])
 
   def testStrictError(self):
     text = constant_op.constant([b"\xFEED"])
@@ -86,7 +86,7 @@ class UnicodeDecodeTest(test.TestCase):
 
     with self.assertRaises(errors.InvalidArgumentError):
       with self.test_session():
-        error.eval()
+        self.evaluate(error)
 
   def testReplaceOnError(self):
     text = constant_op.constant([b"\xFE"])
@@ -95,7 +95,7 @@ class UnicodeDecodeTest(test.TestCase):
         text, "utf-8", errors="replace")
 
     with self.test_session():
-      self.assertAllEqual(utf8_text.eval().tolist(), [65533])
+      self.assertAllEqual(self.evaluate(utf8_text).tolist(), [65533])
 
   def testBadReplacementChar(self):
     text = constant_op.constant([b"\xFE"])
@@ -104,7 +104,7 @@ class UnicodeDecodeTest(test.TestCase):
 
     with self.assertRaises(errors.InvalidArgumentError):
       with self.test_session():
-        error.eval()
+        self.evaluate(error)
 
   def testIgnoreOnError(self):
     text = constant_op.constant([b"\xFEhello"])
@@ -113,7 +113,7 @@ class UnicodeDecodeTest(test.TestCase):
         text, "utf-8", errors="ignore")
 
     with self.test_session():
-      self.assertAllEqual(utf8_text.eval().tolist(), [
+      self.assertAllEqual(self.evaluate(utf8_text).tolist(), [
           codepoint("h"),
           codepoint("e"),
           codepoint("l"),
@@ -141,8 +141,8 @@ class UnicodeDecodeTest(test.TestCase):
           codepoint("年"),
           codepoint("前"),
       ],
-                          utf8_text.eval().tolist())
-      self.assertAllEqual([0, 5], row_splits.eval().tolist())
+                          self.evaluate(utf8_text).tolist())
+      self.assertAllEqual([0, 5], self.evaluate(row_splits).tolist())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/variable_ops_test.py b/tensorflow/python/kernel_tests/variable_ops_test.py
index 3d2f8b6155..769bbba47b 100644
--- a/tensorflow/python/kernel_tests/variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/variable_ops_test.py
@@ -46,7 +46,7 @@ class VariableOpTest(test.TestCase):
       p = state_ops.variable_op(x.shape, tftype)
       op = state_ops.assign(p, x)
       op.op.run()
-      return p.eval()
+      return self.evaluate(p)
 
   def _testTypes(self, vals):
     for dtype in [np.float32, np.float64, np.int32, np.int64]:
@@ -170,14 +170,14 @@ class VariableOpTest(test.TestCase):
       var = state_ops.assign(var, [[4.0, 5.0]])
       var = state_ops.assign_add(var, [[6.0, 7.0]])
       final = gen_state_ops.destroy_temporary_variable(var, var_name="foo")
-      self.assertAllClose([[10.0, 12.0]], final.eval())
+      self.assertAllClose([[10.0, 12.0]], self.evaluate(final))
 
   def testDestroyNonexistentTemporaryVariable(self):
     with self.test_session(use_gpu=True):
       var = gen_state_ops.temporary_variable([1, 2], dtypes.float32)
       final = gen_state_ops.destroy_temporary_variable(var, var_name="bad")
       with self.assertRaises(errors.NotFoundError):
-        final.eval()
+        self.evaluate(final)
 
   def testDuplicateTemporaryVariable(self):
     with self.test_session(use_gpu=True):
@@ -189,7 +189,7 @@ class VariableOpTest(test.TestCase):
       var2 = state_ops.assign(var2, [[3.0, 4.0]])
       final = var1 + var2
       with self.assertRaises(errors.AlreadyExistsError):
-        final.eval()
+        self.evaluate(final)
 
   def testDestroyTemporaryVariableTwice(self):
     with self.test_session(use_gpu=True):
@@ -198,14 +198,14 @@ class VariableOpTest(test.TestCase):
       val2 = gen_state_ops.destroy_temporary_variable(var, var_name="dup")
       final = val1 + val2
       with self.assertRaises(errors.NotFoundError):
-        final.eval()
+        self.evaluate(final)
 
   def testTemporaryVariableNoLeak(self):
     with self.test_session(use_gpu=True):
       var = gen_state_ops.temporary_variable(
           [1, 2], dtypes.float32, var_name="bar")
       final = array_ops.identity(var)
-      final.eval()
+      self.evaluate(final)
 
   def testTwoTemporaryVariablesNoLeaks(self):
     with self.test_session(use_gpu=True):
@@ -214,7 +214,7 @@ class VariableOpTest(test.TestCase):
       var2 = gen_state_ops.temporary_variable(
           [1, 2], dtypes.float32, var_name="var2")
       final = var1 + var2
-      final.eval()
+      self.evaluate(final)
 
   def testAssignDependencyAcrossDevices(self):
     with self.test_session(use_gpu=True):
@@ -229,7 +229,7 @@ class VariableOpTest(test.TestCase):
           # honored, i.e., the Send and Recv from GPU to CPU should take place
           # only after the increment.
           result = math_ops.multiply(var, var)
-      self.assertAllClose([4.0], result.eval())
+      self.assertAllClose([4.0], self.evaluate(result))
 
   def testIsVariableInitialized(self):
     for use_gpu in [True, False]:
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 2ba064f8a5..0aac4adfa6 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -649,7 +649,7 @@ class VariableScopeTest(test.TestCase):
             "testVarScopeGetOrCreateReuse_bar",
             reuse=variable_scope.AUTO_REUSE):
           _ = variable_scope.get_variable("var", [])
-        self.assertEqual(value, x.eval())
+        self.assertEqual(value, self.evaluate(x))
 
       test_value(42.)  # Variable is created.
       test_value(13.)  # Variable is reused hereafter.
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index b3eebf8316..6213f86272 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -58,15 +58,15 @@ class VariablesTestCase(test.TestCase):
       self.assertEqual([], var1.shape)
 
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        var0.eval()
+        self.evaluate(var0)
 
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        var1.eval()
+        self.evaluate(var1)
 
       variables.global_variables_initializer().run()
 
-      self.assertAllClose(0.0, var0.eval())
-      self.assertAllClose(1.1, var1.eval())
+      self.assertAllClose(0.0, self.evaluate(var0))
+      self.assertAllClose(1.1, self.evaluate(var1))
 
   def testInitializationOrder(self):
     with self.cached_session():
@@ -94,8 +94,9 @@ class VariablesTestCase(test.TestCase):
 
       variables.global_variables_initializer().run()
 
-      self.assertAllClose(rnd.eval(), dep.eval())
-      self.assertAllClose(rnd.eval() + dep.eval() + 2.0, depdep.eval())
+      self.assertAllClose(rnd.eval(), self.evaluate(dep))
+      self.assertAllClose(rnd.eval() + self.evaluate(dep) + 2.0,
+                          self.evaluate(depdep))
 
   def testIterable(self):
     with self.assertRaisesRegexp(TypeError, "not iterable"):
@@ -112,16 +113,16 @@ class VariablesTestCase(test.TestCase):
       minus_one = var.assign_sub(2.0)
       four = var.assign(4.0)
       variables.global_variables_initializer().run()
-      self.assertAllClose(0.0, var.eval())
+      self.assertAllClose(0.0, self.evaluate(var))
 
-      self.assertAllClose(1.0, plus_one.eval())
-      self.assertAllClose(1.0, var.eval())
+      self.assertAllClose(1.0, self.evaluate(plus_one))
+      self.assertAllClose(1.0, self.evaluate(var))
 
-      self.assertAllClose(-1.0, minus_one.eval())
-      self.assertAllClose(-1.0, var.eval())
+      self.assertAllClose(-1.0, self.evaluate(minus_one))
+      self.assertAllClose(-1.0, self.evaluate(var))
 
-      self.assertAllClose(4.0, four.eval())
-      self.assertAllClose(4.0, var.eval())
+      self.assertAllClose(4.0, self.evaluate(four))
+      self.assertAllClose(4.0, self.evaluate(var))
 
   def testResourceAssignments(self):
     with self.session(use_gpu=True):
@@ -130,16 +131,16 @@ class VariablesTestCase(test.TestCase):
       minus_one = var.assign_sub(2.0)
       four = var.assign(4.0)
       variables.global_variables_initializer().run()
-      self.assertAllClose(0.0, var.eval())
+      self.assertAllClose(0.0, self.evaluate(var))
 
-      plus_one.eval()
-      self.assertAllClose(1.0, var.eval())
+      self.evaluate(plus_one)
+      self.assertAllClose(1.0, self.evaluate(var))
 
-      minus_one.eval()
-      self.assertAllClose(-1.0, var.eval())
+      self.evaluate(minus_one)
+      self.assertAllClose(-1.0, self.evaluate(var))
 
-      four.eval()
-      self.assertAllClose(4.0, var.eval())
+      self.evaluate(four)
+      self.assertAllClose(4.0, self.evaluate(var))
 
   def testZeroSizeStringAssign(self):
     with self.cached_session() as sess:
@@ -160,24 +161,24 @@ class VariablesTestCase(test.TestCase):
       count_up_to = var.count_up_to(3)
 
       variables.global_variables_initializer().run()
-      self.assertEqual(0, var.eval())
+      self.assertEqual(0, self.evaluate(var))
 
-      self.assertEqual(0, count_up_to.eval())
-      self.assertEqual(1, var.eval())
+      self.assertEqual(0, self.evaluate(count_up_to))
+      self.assertEqual(1, self.evaluate(var))
 
-      self.assertEqual(1, count_up_to.eval())
-      self.assertEqual(2, var.eval())
+      self.assertEqual(1, self.evaluate(count_up_to))
+      self.assertEqual(2, self.evaluate(var))
 
-      self.assertEqual(2, count_up_to.eval())
-      self.assertEqual(3, var.eval())
+      self.assertEqual(2, self.evaluate(count_up_to))
+      self.assertEqual(3, self.evaluate(var))
 
       with self.assertRaisesOpError("Reached limit of 3"):
-        count_up_to.eval()
-      self.assertEqual(3, var.eval())
+        self.evaluate(count_up_to)
+      self.assertEqual(3, self.evaluate(var))
 
       with self.assertRaisesOpError("Reached limit of 3"):
-        count_up_to.eval()
-      self.assertEqual(3, var.eval())
+        self.evaluate(count_up_to)
+      self.assertEqual(3, self.evaluate(var))
 
   def testCountUpToInt32(self):
     self._countUpToTest(dtypes.int32)
@@ -252,8 +253,8 @@ class VariablesTestCase(test.TestCase):
       var_x = variables.Variable(2.0)
       var_y = variables.Variable(3.0)
       variables.global_variables_initializer().run()
-      self.assertAllClose(2.0, var_x.eval())
-      self.assertAllClose(3.0, var_y.eval())
+      self.assertAllClose(2.0, self.evaluate(var_x))
+      self.assertAllClose(3.0, self.evaluate(var_y))
       self.assertAllClose(5.0, math_ops.add(var_x, var_y).eval())
 
   def testZeroSizeVarSameAsConst(self):
@@ -264,7 +265,7 @@ class VariablesTestCase(test.TestCase):
       const_mul = math_ops.matmul(
           zero_size_const, zero_size_const, transpose_b=True)
       variables.global_variables_initializer().run()
-      variable_output = variable_mul.eval()
+      variable_output = self.evaluate(variable_mul)
       self.assertAllClose(const_mul.eval(), variable_output)
       self.assertAllClose([[0., 0.], [0., 0.]], variable_output)
 
@@ -349,37 +350,37 @@ class VariablesTestCase(test.TestCase):
       rmatmul = var_m.__rmatmul__([[10.0], [20.0]])
 
       variables.global_variables_initializer().run()
-      self.assertAllClose([2.0], add.eval())
-      self.assertAllClose([3.0], radd.eval())
-      self.assertAllClose([1.0], sub.eval())
-      self.assertAllClose([-1.0], rsub.eval())
-      self.assertAllClose([20.0], mul.eval())
-      self.assertAllClose([20.0], rmul.eval())
-      self.assertAllClose([0.2], div.eval())
-      self.assertAllClose([5.0], rdiv.eval())
-      self.assertAllClose([-2.0], neg.eval())
-      self.assertAllClose([2.0], abs_v.eval())
-      self.assertAllClose([True], lt.eval())
-      self.assertAllClose([False], rlt.eval())
-      self.assertAllClose([True], le.eval())
-      self.assertAllClose([True], rle.eval())
-      self.assertAllClose([False], gt.eval())
-      self.assertAllClose([True], rgt.eval())
-      self.assertAllClose([True], ge.eval())
-      self.assertAllClose([True], rge.eval())
-
-      self.assertAllClose([6], mod.eval())
-      self.assertAllClose([3], rmod.eval())
-
-      self.assertAllClose([True, False], and_v.eval())
-      self.assertAllClose([True, True], or_v.eval())
-      self.assertAllClose([True, False], xor_v.eval())
-      self.assertAllClose([False, True], invert_v.eval())
-
-      self.assertAllClose(rnd[2, 0:0], slice_v.eval())
-
-      self.assertAllClose([[80.0]], matmul.eval())
-      self.assertAllClose([[20.0, 30.0], [40.0, 60.0]], rmatmul.eval())
+      self.assertAllClose([2.0], self.evaluate(add))
+      self.assertAllClose([3.0], self.evaluate(radd))
+      self.assertAllClose([1.0], self.evaluate(sub))
+      self.assertAllClose([-1.0], self.evaluate(rsub))
+      self.assertAllClose([20.0], self.evaluate(mul))
+      self.assertAllClose([20.0], self.evaluate(rmul))
+      self.assertAllClose([0.2], self.evaluate(div))
+      self.assertAllClose([5.0], self.evaluate(rdiv))
+      self.assertAllClose([-2.0], self.evaluate(neg))
+      self.assertAllClose([2.0], self.evaluate(abs_v))
+      self.assertAllClose([True], self.evaluate(lt))
+      self.assertAllClose([False], self.evaluate(rlt))
+      self.assertAllClose([True], self.evaluate(le))
+      self.assertAllClose([True], self.evaluate(rle))
+      self.assertAllClose([False], self.evaluate(gt))
+      self.assertAllClose([True], self.evaluate(rgt))
+      self.assertAllClose([True], self.evaluate(ge))
+      self.assertAllClose([True], self.evaluate(rge))
+
+      self.assertAllClose([6], self.evaluate(mod))
+      self.assertAllClose([3], self.evaluate(rmod))
+
+      self.assertAllClose([True, False], self.evaluate(and_v))
+      self.assertAllClose([True, True], self.evaluate(or_v))
+      self.assertAllClose([True, False], self.evaluate(xor_v))
+      self.assertAllClose([False, True], self.evaluate(invert_v))
+
+      self.assertAllClose(rnd[2, 0:0], self.evaluate(slice_v))
+
+      self.assertAllClose([[80.0]], self.evaluate(matmul))
+      self.assertAllClose([[20.0, 30.0], [40.0, 60.0]], self.evaluate(rmatmul))
 
   def testSession(self):
     with self.cached_session() as sess:
@@ -416,7 +417,7 @@ class VariablesTestCase(test.TestCase):
       self.assertEqual(shape, v1.shape)
       self.assertAllClose(value, v1.initial_value.eval())
       with self.assertRaises(errors_impl.FailedPreconditionError):
-        v1.eval()
+        self.evaluate(v1)
 
       v2 = variables.Variable(
           math_ops.negative(v1.initialized_value()), dtype=dtypes.float32)
@@ -425,9 +426,9 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(np.negative(value), v2.initial_value.eval())
 
       with self.assertRaises(errors_impl.FailedPreconditionError):
-        v2.eval()
+        self.evaluate(v2)
       variables.global_variables_initializer().run()
-      self.assertAllClose(np.negative(value), v2.eval())
+      self.assertAllClose(np.negative(value), self.evaluate(v2))
 
   def testConstraintArg(self):
     constraint = lambda x: x
@@ -519,7 +520,7 @@ class VariablesTestCase(test.TestCase):
       variables.global_variables_initializer().run()
       var.load(np.ones((5, 5), np.float32))
 
-      self.assertAllClose(np.ones((5, 5), np.float32), var.eval())
+      self.assertAllClose(np.ones((5, 5), np.float32), self.evaluate(var))
 
   def testRepr(self):
     var = variables.VariableV1(np.zeros((5, 5), np.float32), name="noop")
@@ -582,7 +583,7 @@ class IsInitializedTest(test.TestCase):
       do_opt = gradient_descent.GradientDescentOptimizer(0.1).minimize(
           objective)
       sess.run([do_opt])
-      self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], b.eval())
+      self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], self.evaluate(b))
 
 
 class ObsoleteIsInitializedTest(test.TestCase):
@@ -744,34 +745,34 @@ class PartitionedVariableTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       self.assertEqual([1.0], plus_delta[0].eval())
-      self.assertEqual([1.0], v0.eval())
+      self.assertEqual([1.0], self.evaluate(v0))
       self.assertEqual([3.0], plus_delta[1].eval())
-      self.assertEqual([3.0], v1.eval())
+      self.assertEqual([3.0], self.evaluate(v1))
 
       self.assertEqual([-2.0], minus_delta[0].eval())
-      self.assertEqual([-2.0], v0.eval())
+      self.assertEqual([-2.0], self.evaluate(v0))
       self.assertEqual([-1.0], minus_delta[1].eval())
-      self.assertEqual([-1.0], v1.eval())
+      self.assertEqual([-1.0], self.evaluate(v1))
 
       self.assertEqual([1.0], assign_ones[0].eval())
-      self.assertEqual([1.0], v0.eval())
+      self.assertEqual([1.0], self.evaluate(v0))
       self.assertEqual([1.0], assign_ones[1].eval())
-      self.assertEqual([1.0], v1.eval())
+      self.assertEqual([1.0], self.evaluate(v1))
 
       self.assertEqual([2.0], assign_list[0].eval())
-      self.assertEqual([2.0], v2.eval())
+      self.assertEqual([2.0], self.evaluate(v2))
       self.assertEqual([3.0], assign_list[1].eval())
-      self.assertEqual([3.0], v3.eval())
+      self.assertEqual([3.0], self.evaluate(v3))
 
       self.assertEqual([3.0], assign_part_value[0].eval())
-      self.assertEqual([3.0], v2.eval())
+      self.assertEqual([3.0], self.evaluate(v2))
       self.assertEqual([4.0], assign_part_value[1].eval())
-      self.assertEqual([4.0], v3.eval())
+      self.assertEqual([4.0], self.evaluate(v3))
 
       self.assertEqual([2.0], assign_part_var[0].eval())
-      self.assertEqual([2.0], v2.eval())
+      self.assertEqual([2.0], self.evaluate(v2))
       self.assertEqual([3.0], assign_part_var[1].eval())
-      self.assertEqual([3.0], v3.eval())
+      self.assertEqual([3.0], self.evaluate(v3))
 
 
 class VariableContainerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/weights_broadcast_test.py b/tensorflow/python/kernel_tests/weights_broadcast_test.py
index 85f9abc69f..c476004b89 100644
--- a/tensorflow/python/kernel_tests/weights_broadcast_test.py
+++ b/tensorflow/python/kernel_tests/weights_broadcast_test.py
@@ -158,7 +158,7 @@ class BroadcastWeightsTest(test.TestCase):
     dynamic_op = weights_broadcast_ops.broadcast_weights(
         weights=weights_placeholder, values=values_placeholder)
     with self.cached_session():
-      self.assertAllEqual(expected, static_op.eval())
+      self.assertAllEqual(expected, self.evaluate(static_op))
       self.assertAllEqual(expected, dynamic_op.eval(feed_dict={
           weights_placeholder: weights,
           values_placeholder: values,
diff --git a/tensorflow/python/kernel_tests/where_op_test.py b/tensorflow/python/kernel_tests/where_op_test.py
index fca45c3ece..9e074b2304 100644
--- a/tensorflow/python/kernel_tests/where_op_test.py
+++ b/tensorflow/python/kernel_tests/where_op_test.py
@@ -41,11 +41,11 @@ class WhereOpTest(test.TestCase):
       ans = array_ops.where(x)
       self.assertEqual([None, x.ndim], ans.get_shape().as_list())
       if expected_err_re is None:
-        tf_ans = ans.eval()
+        tf_ans = self.evaluate(ans)
         self.assertAllClose(tf_ans, truth, atol=1e-10)
       else:
         with self.assertRaisesOpError(expected_err_re):
-          ans.eval()
+          self.evaluate(ans)
 
   def testWrongNumbers(self):
     with self.session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/zero_division_test.py b/tensorflow/python/kernel_tests/zero_division_test.py
index e68b96e670..73ab382e53 100644
--- a/tensorflow/python/kernel_tests/zero_division_test.py
+++ b/tensorflow/python/kernel_tests/zero_division_test.py
@@ -36,7 +36,7 @@ class ZeroDivisionTest(test.TestCase):
           bads.append(one % zero)
         for bad in bads:
           try:
-            result = bad.eval()
+            result = self.evaluate(bad)
           except errors_impl.OpError as e:
             # Ideally, we'd get a nice exception.  In theory, this should only
             # happen on CPU, but 32 bit integer GPU division is actually on
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index c3514c183c..47675d3f34 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -155,9 +155,9 @@ class WithDependenciesTestCase(test_util.TensorFlowTestCase):
           constant_op.constant(7))
       with self.cached_session():
         variables.global_variables_initializer().run()
-        self.assertEquals(0, counter.eval())
-        self.assertEquals(7, const_with_dep.eval())
-        self.assertEquals(1, counter.eval())
+        self.assertEquals(0, self.evaluate(counter))
+        self.assertEquals(7, self.evaluate(const_with_dep))
+        self.assertEquals(1, self.evaluate(counter))
 
   def testListDependencies(self):
     with ops.Graph().as_default():
@@ -169,9 +169,9 @@ class WithDependenciesTestCase(test_util.TensorFlowTestCase):
           constant_op.constant(7))
       with self.cached_session():
         variables.global_variables_initializer().run()
-        self.assertEquals(0, counter.eval())
-        self.assertEquals(7, const_with_dep.eval())
-        self.assertEquals(1, counter.eval())
+        self.assertEquals(0, self.evaluate(counter))
+        self.assertEquals(7, self.evaluate(const_with_dep))
+        self.assertEquals(1, self.evaluate(counter))
 
 
 class SwitchTestCase(test_util.TensorFlowTestCase):
@@ -233,7 +233,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
                        constant_op.constant(0.0)])
       with self.cached_session() as sess:
         sess.run(variables.global_variables_initializer())
-        self.assertAllEqual(10.0, cost.eval())
+        self.assertAllEqual(10.0, self.evaluate(cost))
 
   def doTestIndexedSlicesGradientInCondInWhileLoop(self, use_resource=False):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/ops/dequantize_op_test.py b/tensorflow/python/ops/dequantize_op_test.py
index 13e50273d8..794985b2db 100644
--- a/tensorflow/python/ops/dequantize_op_test.py
+++ b/tensorflow/python/ops/dequantize_op_test.py
@@ -35,7 +35,7 @@ class DequantizeOpTest(test.TestCase):
     with self.cached_session():
       input_op = constant_op.constant(inputs, shape=[len(inputs)], dtype=dtype)
       dequantized = array_ops.dequantize(input_op, min_range, max_range)
-      tf_ans = dequantized.eval()
+      tf_ans = self.evaluate(dequantized)
 
     # TODO(vrv): Add support for DT_QINT32 quantization if needed.
     type_dict = {
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index 103e3902b6..262b62e013 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -144,7 +144,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
                                  gate_gradients=True)[0]
       with session.Session():
         # Make sure the placer doesn't complain.
-        gz_x.eval()
+        self.evaluate(gz_x)
 
   def testBoundaryStop(self):
     # Test that we don't differentiate 'x'. The gradient function for 'x' is
@@ -628,7 +628,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase):
         mat_x = math_ops.matmul(mat, x, name="Ax")
         x_mat_x = math_ops.matmul(array_ops.transpose(x), mat_x, name="xAx")
         hess_v = gradients_impl._hessian_vector_product(x_mat_x, [x], [v])[0]
-        hess_v_actual = hess_v.eval()
+        hess_v_actual = self.evaluate(hess_v)
       self.assertAllClose(hess_v_value, hess_v_actual)
 
 
@@ -648,7 +648,7 @@ class HessianTest(test_util.TensorFlowTestCase):
       x = constant_op.constant(x_value)
       x_mat_x = math_ops.reduce_sum(x[:, None] * mat * x[None, :])
       hess = gradients.hessians(x_mat_x, x)[0]
-      hess_actual = hess.eval()
+      hess_actual = self.evaluate(hess)
     self.assertAllClose(hess_value, hess_actual)
 
   def testHessian1D_multi(self):
@@ -692,7 +692,7 @@ class HessianTest(test_util.TensorFlowTestCase):
           math_ops.matmul(array_ops.transpose(x), x) * 0.5
       )
       hess = gradients.hessians(x_square, x)[0]
-      hess_actual = hess.eval()
+      hess_actual = self.evaluate(hess)
     hess_value = np.bmat([
         [elem*np.ones((m, m)) for elem in vec]
         for vec in np.eye(m)
@@ -711,7 +711,7 @@ class HessianTest(test_util.TensorFlowTestCase):
           math_ops.matmul(array_ops.transpose(x), x) * 0.5
       )
       hess = gradients.hessians(x_square, x)[0]
-      hess_actual = hess.eval()
+      hess_actual = self.evaluate(hess)
     hess_value = np.bmat([
         [elem*np.ones((n, n)) for elem in vec]
         for vec in np.eye(m)
@@ -729,7 +729,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
       c_sparse = math_ops._as_indexed_slices(c)
       self.assertAllEqual(np_val.shape, c_sparse.dense_shape.eval())
       c_dense = math_ops.multiply(c_sparse, 1.0)
-      self.assertAllClose(np_val, c_dense.eval())
+      self.assertAllClose(np_val, self.evaluate(c_dense))
 
   def testIndexedSlicesToTensorList(self):
     with self.cached_session():
@@ -745,7 +745,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
         sparse_list.append(c_sparse)
       packed_dense = array_ops.stack(dense_list)
       packed_sparse = array_ops.stack(sparse_list)
-      self.assertAllClose(packed_dense.eval(), packed_sparse.eval())
+      self.assertAllClose(packed_dense.eval(), self.evaluate(packed_sparse))
 
   def testInt64Indices(self):
     with self.cached_session():
@@ -757,7 +757,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
           math_ops.cast(c_sparse.indices, dtypes.int64), c_sparse.dense_shape)
       self.assertAllEqual(np_val.shape, c_sparse.dense_shape.eval())
       c_dense = math_ops.multiply(c_sparse, 1.0)
-      self.assertAllClose(np_val, c_dense.eval())
+      self.assertAllClose(np_val, self.evaluate(c_dense))
 
   def testWarnings(self):
     # TODO(gunan) Reenable after this issue is fixed:
@@ -853,7 +853,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       y = MyIdentity(MyIdentity(x))
       dy = gradients.gradients(y, x)[0]
       with session.Session():
-        self.assertEqual(9., dy.eval())
+        self.assertEqual(9., self.evaluate(dy))
 
   def testCustomGradient(self):
 
diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py
index e7fe0efba4..cd1abbfc13 100644
--- a/tensorflow/python/ops/histogram_ops_test.py
+++ b/tensorflow/python/ops/histogram_ops_test.py
@@ -39,7 +39,7 @@ class BinValuesFixedWidth(test.TestCase):
       bins = histogram_ops.histogram_fixed_width_bins(
           values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, bins.dtype)
-      self.assertAllClose(expected_bins, bins.eval())
+      self.assertAllClose(expected_bins, self.evaluate(bins))
 
   def test_1d_values_int32_output(self):
     # Bins will be:
@@ -51,7 +51,7 @@ class BinValuesFixedWidth(test.TestCase):
       bins = histogram_ops.histogram_fixed_width_bins(
           values, value_range, nbins=5, dtype=dtypes.int64)
       self.assertEqual(dtypes.int32, bins.dtype)
-      self.assertAllClose(expected_bins, bins.eval())
+      self.assertAllClose(expected_bins, self.evaluate(bins))
 
   def test_1d_float64_values_int32_output(self):
     # Bins will be:
@@ -63,7 +63,7 @@ class BinValuesFixedWidth(test.TestCase):
       bins = histogram_ops.histogram_fixed_width_bins(
           values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, bins.dtype)
-      self.assertAllClose(expected_bins, bins.eval())
+      self.assertAllClose(expected_bins, self.evaluate(bins))
 
   def test_2d_values(self):
     # Bins will be:
@@ -76,7 +76,7 @@ class BinValuesFixedWidth(test.TestCase):
       bins = histogram_ops.histogram_fixed_width_bins(
           values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, bins.dtype)
-      self.assertAllClose(expected_bins, bins.eval())
+      self.assertAllClose(expected_bins, self.evaluate(bins))
 
 
 class HistogramFixedWidthTest(test.TestCase):
@@ -110,7 +110,7 @@ class HistogramFixedWidthTest(test.TestCase):
     with self.session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
-      self.assertAllClose(expected_bin_counts, hist.eval())
+      self.assertAllClose(expected_bin_counts, self.evaluate(hist))
 
   def test_1d_values_int64_output(self):
     # Bins will be:
@@ -122,7 +122,7 @@ class HistogramFixedWidthTest(test.TestCase):
       hist = histogram_ops.histogram_fixed_width(
           values, value_range, nbins=5, dtype=dtypes.int64)
       self.assertEqual(dtypes.int64, hist.dtype)
-      self.assertAllClose(expected_bin_counts, hist.eval())
+      self.assertAllClose(expected_bin_counts, self.evaluate(hist))
 
   def test_1d_float64_values(self):
     # Bins will be:
@@ -133,7 +133,7 @@ class HistogramFixedWidthTest(test.TestCase):
     with self.session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
-      self.assertAllClose(expected_bin_counts, hist.eval())
+      self.assertAllClose(expected_bin_counts, self.evaluate(hist))
 
   def test_2d_values(self):
     # Bins will be:
@@ -144,7 +144,7 @@ class HistogramFixedWidthTest(test.TestCase):
     with self.session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
-      self.assertAllClose(expected_bin_counts, hist.eval())
+      self.assertAllClose(expected_bin_counts, self.evaluate(hist))
 
   def test_shape_inference(self):
     value_range = [0.0, 5.0]
@@ -155,7 +155,7 @@ class HistogramFixedWidthTest(test.TestCase):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertAllEqual(hist.shape.as_list(), (5,))
       self.assertEqual(dtypes.int32, hist.dtype)
-      self.assertAllClose(expected_bin_counts, hist.eval())
+      self.assertAllClose(expected_bin_counts, self.evaluate(hist))
 
       hist = histogram_ops.histogram_fixed_width(
           values, value_range, nbins=placeholder)
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index a3aeb79586..07406982c5 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -84,7 +84,7 @@ class RGBToHSVTest(test_util.TensorFlowTestCase):
       with self.test_session(use_gpu=True):
         hsv = image_ops.rgb_to_hsv(rgb_np)
         rgb = image_ops.hsv_to_rgb(hsv)
-        rgb_tf = rgb.eval()
+        rgb_tf = self.evaluate(rgb)
       self.assertAllClose(rgb_tf, rgb_np)
 
 
@@ -173,7 +173,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.rgb_to_grayscale(x_tf)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testBasicRGBToGrayscale(self):
@@ -195,7 +195,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.grayscale_to_rgb(x_tf)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
     # 3-D input with no batch dimension.
@@ -205,7 +205,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.grayscale_to_rgb(x_tf)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testShapeInference(self):
@@ -245,7 +245,7 @@ class AdjustGamma(test_util.TensorFlowTestCase):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_gamma(x, gamma=1)
 
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       y_np = x_np
 
       self.assertAllClose(y_tf, y_np, 1e-6)
@@ -281,7 +281,7 @@ class AdjustGamma(test_util.TensorFlowTestCase):
 
       err_msg = "Gamma should be a non-negative real number."
       try:
-        image.eval()
+        self.evaluate(image)
       except Exception as e:
         if err_msg not in str(e):
           raise
@@ -297,7 +297,7 @@ class AdjustGamma(test_util.TensorFlowTestCase):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_gamma(x, gamma=0)
 
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
 
       dtype = x.dtype.as_numpy_dtype
       y_np = np.array([dtypes.dtype_range[dtype][1]] * x_np.size)
@@ -360,7 +360,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_hue(x, delta)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testAdjustPositiveHue(self):
@@ -375,7 +375,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_hue(x, delta)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testBatchAdjustHue(self):
@@ -390,7 +390,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_hue(x, delta)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def _adjustHueNp(self, x_np, delta_h):
@@ -415,7 +415,7 @@ class AdjustHueTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np)
       y = image_ops.adjust_hue(x, delta_h)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
     return y_tf
 
   def testAdjustRandomHue(self):
@@ -846,7 +846,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_saturation(x, saturation_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testTwiceSaturation(self):
@@ -861,7 +861,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_saturation(x, saturation_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testBatchSaturation(self):
@@ -876,7 +876,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.adjust_saturation(x, saturation_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def _adjust_saturation(self, image, saturation_factor):
@@ -899,7 +899,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = self._adjust_saturation(x, saturation_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testTwiceSaturationFused(self):
@@ -914,7 +914,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = self._adjust_saturation(x, saturation_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def _adjustSaturationNp(self, x_np, scale):
@@ -980,7 +980,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(image_ops.flip_left_right(x_tf))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
   def testInvolutionLeftRightWithBatch(self):
@@ -990,7 +990,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(image_ops.flip_left_right(x_tf))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
   def testLeftRight(self):
@@ -1001,7 +1001,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(x_tf)
       self.assertTrue(y.op.name.startswith("flip_left_right"))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testLeftRightWithBatch(self):
@@ -1015,7 +1015,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_left_right(x_tf)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testRandomFlipLeftRight(self):
@@ -1031,7 +1031,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       count_flipped = 0
       count_unflipped = 0
       for _ in range(100):
-        y_tf = y.eval()
+        y_tf = self.evaluate(y)
         if y_tf[0][0] == 1:
           self.assertAllEqual(y_tf, x_np)
           count_unflipped += 1
@@ -1070,7 +1070,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       count_flipped = 0
       count_unflipped = 0
       for _ in range(100):
-        y_tf = y.eval()
+        y_tf = self.evaluate(y)
 
         # check every element of the batch
         for i in range(batch_size):
@@ -1096,7 +1096,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(image_ops.flip_up_down(x_tf))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
   def testInvolutionUpDownWithBatch(self):
@@ -1107,7 +1107,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(image_ops.flip_up_down(x_tf))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
   def testUpDown(self):
@@ -1118,7 +1118,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(x_tf)
       self.assertTrue(y.op.name.startswith("flip_up_down"))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testUpDownWithBatch(self):
@@ -1132,7 +1132,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.flip_up_down(x_tf)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testRandomFlipUpDown(self):
@@ -1148,7 +1148,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       count_flipped = 0
       count_unflipped = 0
       for _ in range(100):
-        y_tf = y.eval()
+        y_tf = self.evaluate(y)
         if y_tf[0][0] == 1:
           self.assertAllEqual(y_tf, x_np)
           count_unflipped += 1
@@ -1187,7 +1187,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       count_flipped = 0
       count_unflipped = 0
       for _ in range(100):
-        y_tf = y.eval()
+        y_tf = self.evaluate(y)
 
         # check every element of the batch
         for i in range(batch_size):
@@ -1213,7 +1213,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(image_ops.transpose_image(x_tf))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
   def testInvolutionTransposeWithBatch(self):
@@ -1224,7 +1224,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(image_ops.transpose_image(x_tf))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
   def testTranspose(self):
@@ -1235,7 +1235,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(x_tf)
       self.assertTrue(y.op.name.startswith("transpose_image"))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testTransposeWithBatch(self):
@@ -1250,7 +1250,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(x_tf)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
   def testPartialShapes(self):
@@ -1301,7 +1301,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       rotated = image
       for _ in xrange(4):
         rotated = image_ops.rot90(rotated)
-      self.assertAllEqual(image, rotated.eval())
+      self.assertAllEqual(image, self.evaluate(rotated))
 
   def testRot90GroupOrderWithBatch(self):
     image = np.arange(48, dtype=np.uint8).reshape([2, 2, 4, 3])
@@ -1309,7 +1309,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       rotated = image
       for _ in xrange(4):
         rotated = image_ops.rot90(rotated)
-      self.assertAllEqual(image, rotated.eval())
+      self.assertAllEqual(image, self.evaluate(rotated))
 
   def testRot90NumpyEquivalence(self):
     image = np.arange(24, dtype=np.uint8).reshape([2, 4, 3])
@@ -1335,7 +1335,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_contrast(x, contrast_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllClose(y_tf, y_np, 1e-6)
 
   def testDoubleContrastUint8(self):
@@ -1390,7 +1390,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np)
       y = image_ops.adjust_contrast(x, contrast_factor)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
     return y_tf
 
   def testRandomContrast(self):
@@ -1423,7 +1423,7 @@ class AdjustBrightnessTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_brightness(x, delta)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllClose(y_tf, y_np, 1e-6)
 
   def testPositiveDeltaUint8(self):
@@ -1480,7 +1480,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.per_image_standardization(x)
       self.assertTrue(y.op.name.startswith("per_image_standardization"))
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllClose(y_tf, y_np, atol=1e-4)
 
   def testUniformImage(self):
@@ -1488,7 +1488,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase):
     im = constant_op.constant(im_np)
     whiten = image_ops.per_image_standardization(im)
     with self.test_session(use_gpu=True):
-      whiten_np = whiten.eval()
+      whiten_np = self.evaluate(whiten)
       self.assertFalse(np.any(np.isnan(whiten_np)))
 
   def testBatchWhitening(self):
@@ -1497,7 +1497,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       imgs = constant_op.constant(imgs_np)
       whiten = image_ops.per_image_standardization(imgs)
-      whiten_tf = whiten.eval()
+      whiten_tf = self.evaluate(whiten)
       for w_tf, w_np in zip(whiten_tf, whiten_np):
         self.assertAllClose(w_tf, w_np, atol=1e-4)
 
@@ -1696,7 +1696,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
         with self.test_session(use_gpu=use_gpu):
           x = constant_op.constant(x_np, shape=x_shape)
           y = image_ops.central_crop(x, 1.0)
-          y_tf = y.eval()
+          y_tf = self.evaluate(y)
           self.assertAllEqual(y_tf, x_np)
           self.assertEqual(y.op.name, x.op.name)
 
@@ -1711,7 +1711,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
       with self.test_session(use_gpu=use_gpu):
         x = constant_op.constant(x_np, shape=x_shape)
         y = image_ops.central_crop(x, 0.5)
-        y_tf = y.eval()
+        y_tf = self.evaluate(y)
         self.assertAllEqual(y_tf, y_np)
         self.assertAllEqual(y_tf.shape, y_np.shape)
 
@@ -1727,7 +1727,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_shape)
       y = image_ops.central_crop(x, 0.5)
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
       self.assertAllEqual(y_tf.shape, y_np.shape)
 
@@ -1897,7 +1897,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64)
     y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3])
     with self.test_session(use_gpu=True):
-      self.assertAllClose(y, y_tf.eval())
+      self.assertAllClose(y, self.evaluate(y_tf))
 
   def testNoOp(self):
     x_shape = [10, 10, 10]
@@ -2040,7 +2040,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
       y = array_ops.strided_slice(image_tf, begin, begin + size)
 
       for _ in xrange(num_iter):
-        y_tf = y.eval()
+        y_tf = self.evaluate(y)
         crop_height = y_tf.shape[0]
         crop_width = y_tf.shape[1]
         aspect_ratio = float(crop_width) / float(crop_height)
@@ -2171,9 +2171,9 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
       self.assertAllEqual([3], end.get_shape().as_list())
       self.assertAllEqual([1, 1, 4], bbox_for_drawing.get_shape().as_list())
       # Actual run to make sure shape is correct inside Compute().
-      begin = begin.eval()
-      end = end.eval()
-      bbox_for_drawing = bbox_for_drawing.eval()
+      begin = self.evaluate(begin)
+      end = self.evaluate(end)
+      bbox_for_drawing = self.evaluate(bbox_for_drawing)
 
       begin, end, bbox_for_drawing = image_ops.sample_distorted_bounding_box(
           image_size=image_size,
@@ -2207,9 +2207,9 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
       self.assertAllEqual([3], end.get_shape().as_list())
       self.assertAllEqual([1, 1, 4], bbox_for_drawing.get_shape().as_list())
       # Actual run to make sure shape is correct inside Compute().
-      begin = begin.eval()
-      end = end.eval()
-      bbox_for_drawing = bbox_for_drawing.eval()
+      begin = self.evaluate(begin)
+      end = self.evaluate(end)
+      bbox_for_drawing = self.evaluate(bbox_for_drawing)
 
 
 class ResizeImagesTest(test_util.TensorFlowTestCase):
@@ -2276,7 +2276,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         y = image_ops.resize_images(image, [target_height, target_width],
                                     self.OPTIONS[0])
         yshape = array_ops.shape(y)
-        newshape = yshape.eval()
+        newshape = self.evaluate(yshape)
         self.assertAllEqual(single_shape, newshape)
 
   def testTensorArguments(self):
@@ -2411,7 +2411,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
               y = image_ops.resize_images(image, [target_height, target_width],
                                           opt)
               expected = np.array(expected_data).reshape(target_shape)
-              resized = y.eval()
+              resized = self.evaluate(y)
               self.assertAllClose(resized, expected, atol=1e-5)
 
   def testResizeUpAlignCornersFalse(self):
@@ -2446,7 +2446,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
           image = constant_op.constant(img_np, shape=img_shape)
           y = image_ops.resize_images(
               image, [target_height, target_width], opt, align_corners=False)
-          resized = y.eval()
+          resized = self.evaluate(y)
           expected = np.array(expected_data[opt]).reshape(
               [1, target_height, target_width, 1])
           self.assertAllClose(resized, expected, atol=1e-05)
@@ -2482,7 +2482,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
           image = constant_op.constant(img_np, shape=img_shape)
           y = image_ops.resize_images(
               image, [target_height, target_width], opt, align_corners=True)
-          resized = y.eval()
+          resized = self.evaluate(y)
           expected = np.array(expected_data[opt]).reshape(
               [1, target_height, target_width, 1])
           self.assertAllClose(resized, expected, atol=1e-05)
@@ -2509,7 +2509,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
       image = constant_op.constant(img_np, shape=img_shape)
       y = image_ops.resize_images(image, [target_height, target_width],
                                   image_ops.ResizeMethod.BICUBIC)
-      resized = y.eval()
+      resized = self.evaluate(y)
       expected = np.array(expected_data).reshape(
           [1, target_height, target_width, 1])
       self.assertAllClose(resized, expected, atol=1)
@@ -2534,7 +2534,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
                                   image_ops.ResizeMethod.AREA)
       expected = np.array(expected_data).reshape(
           [1, target_height, target_width, 1])
-      resized = y.eval()
+      resized = self.evaluate(y)
       self.assertAllClose(resized, expected, atol=1)
 
   def testCompareNearestNeighbor(self):
@@ -2554,7 +2554,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
                 new_size,
                 image_ops.ResizeMethod.NEAREST_NEIGHBOR,
                 align_corners=align_corners)
-            gpu_val = out_op.eval()
+            gpu_val = self.evaluate(out_op)
           with self.test_session(use_gpu=False):
             image = constant_op.constant(img_np, shape=input_shape)
             new_size = constant_op.constant([target_height, target_width])
@@ -2563,7 +2563,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
                 new_size,
                 image_ops.ResizeMethod.NEAREST_NEIGHBOR,
                 align_corners=align_corners)
-            cpu_val = out_op.eval()
+            cpu_val = self.evaluate(out_op)
           self.assertAllClose(cpu_val, gpu_val, rtol=1e-5, atol=1e-5)
 
   def testCompareBilinear(self):
@@ -2585,7 +2585,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
                   new_size,
                   image_ops.ResizeMethod.BILINEAR,
                   align_corners=align_corners)
-              value[use_gpu] = out_op.eval()
+              value[use_gpu] = self.evaluate(out_op)
           self.assertAllClose(value[True], value[False], rtol=1e-5, atol=1e-5)
 
   def testShapeInference(self):
@@ -3234,7 +3234,7 @@ class PngTest(test_util.TensorFlowTestCase):
           self.assertEqual(image0.shape, (26, 51, channels or channels_in))
           if channels == channels_in:
             image1 = image_ops.decode_png(image_ops.encode_png(image0))
-            self.assertAllEqual(image0, image1.eval())
+            self.assertAllEqual(image0, self.evaluate(image1))
 
   def testSynthetic(self):
     with self.test_session(use_gpu=True) as sess:
@@ -3431,7 +3431,7 @@ class TotalVariationTest(test_util.TensorFlowTestCase):
       y = image_ops.total_variation(images=x_tf)
 
       # Run the TensorFlow session to calculate the result.
-      y_tf = y.eval()
+      y_tf = self.evaluate(y)
 
       # Assert that the results are as expected within
       # some small error-bound in case they are float-values.
@@ -3709,7 +3709,7 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase):
         iou_threshold = constant_op.constant(iou_threshold_np)
         selected_indices, _ = gen_image_ops.non_max_suppression_v4(
             boxes, scores, max_output_size, iou_threshold, score_threshold)
-        selected_indices = selected_indices.eval()
+        selected_indices = self.evaluate(selected_indices)
         self.assertAllClose(selected_indices, [3, 0, 5])
 
 
@@ -3916,7 +3916,8 @@ class PSNRTest(test_util.TensorFlowTestCase):
     img2 = image_ops.convert_image_dtype(img2, dtypes.float32)
     psnr_float32 = image_ops.psnr(img1, img2, 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(psnr_uint8.eval(), psnr_float32.eval(), atol=0.001)
+      self.assertAllClose(
+          psnr_uint8.eval(), self.evaluate(psnr_float32), atol=0.001)
 
 
 class SSIMTest(test_util.TensorFlowTestCase):
@@ -3969,7 +3970,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     ssim = image_ops.ssim(constant_op.constant(img1),
                           constant_op.constant(img2), 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(expected, ssim.eval(), atol=1e-4)
+      self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4)
 
   def testBroadcast(self):
     img = self._LoadTestImages()[:2]
@@ -3981,7 +3982,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
 
     ssim = image_ops.ssim(img1, img2, 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(expected, ssim.eval(), atol=1e-4)
+      self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4)
 
   def testNegative(self):
     """Tests against negative SSIM index."""
@@ -4007,7 +4008,8 @@ class SSIMTest(test_util.TensorFlowTestCase):
     img2 = image_ops.convert_image_dtype(img2, dtypes.float32)
     ssim_float32 = image_ops.ssim(img1, img2, 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(ssim_uint8.eval(), ssim_float32.eval(), atol=0.001)
+      self.assertAllClose(
+          ssim_uint8.eval(), self.evaluate(ssim_float32), atol=0.001)
 
 
 class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
@@ -4077,7 +4079,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     msssim = image_ops.ssim_multiscale(constant_op.constant(img1),
                                        constant_op.constant(img2), 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(expected, msssim.eval(), 1e-4)
+      self.assertAllClose(expected, self.evaluate(msssim), 1e-4)
 
   def testBroadcast(self):
     """Tests MS-SSIM broadcasting."""
@@ -4090,7 +4092,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
 
     score_tensor = image_ops.ssim_multiscale(img1, img2, 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(expected, score_tensor.eval(), 1e-4)
+      self.assertAllClose(expected, self.evaluate(score_tensor), 1e-4)
 
   def testRange(self):
     """Tests against low MS-SSIM score.
@@ -4124,7 +4126,8 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     img2 = image_ops.convert_image_dtype(img2, dtypes.float32)
     ssim_float32 = image_ops.ssim_multiscale(img1, img2, 1.0)
     with self.test_session(use_gpu=True):
-      self.assertAllClose(ssim_uint8.eval(), ssim_float32.eval(), atol=0.001)
+      self.assertAllClose(
+          ssim_uint8.eval(), self.evaluate(ssim_float32), atol=0.001)
 
 
 class ImageGradientsTest(test_util.TensorFlowTestCase):
@@ -4139,8 +4142,8 @@ class ImageGradientsTest(test_util.TensorFlowTestCase):
 
     dy, dx = image_ops.image_gradients(img)
     with self.cached_session():
-      actual_dy = dy.eval()
-      actual_dx = dx.eval()
+      actual_dy = self.evaluate(dy)
+      actual_dx = self.evaluate(dx)
       self.assertAllClose(expected_dy, actual_dy)
       self.assertAllClose(expected_dx, actual_dx)
 
@@ -4164,8 +4167,8 @@ class ImageGradientsTest(test_util.TensorFlowTestCase):
     assert batch.get_shape().as_list() == [2, 2, 3, 2]
     dy, dx = image_ops.image_gradients(batch)
     with self.test_session(use_gpu=True):
-      actual_dy = dy.eval()
-      actual_dx = dx.eval()
+      actual_dy = self.evaluate(dy)
+      actual_dx = self.evaluate(dx)
       self.assertAllClose(expected_dy, actual_dy)
       self.assertAllClose(expected_dx, actual_dx)
 
@@ -4185,7 +4188,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase):
                            [[0, 0], [0, 12], [0, 0]]], [1, 2, 3, 1, 2])
     sobel = image_ops.sobel_edges(img)
     with self.test_session(use_gpu=True):
-      actual_sobel = sobel.eval()
+      actual_sobel = self.evaluate(sobel)
       self.assertAllClose(expected, actual_sobel)
 
   def testSobelEdges5x3x4x2(self):
@@ -4207,7 +4210,7 @@ class SobelEdgesTest(test_util.TensorFlowTestCase):
 
     sobel = image_ops.sobel_edges(img)
     with self.test_session(use_gpu=True):
-      actual_sobel = sobel.eval()
+      actual_sobel = self.evaluate(sobel)
       self.assertAllClose(expected_batch, actual_sobel)
 
 
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index a4da0c6c33..dc46f1cbd1 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -107,7 +107,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
         y_tf = math_ops.reduce_logsumexp(x_np, reduction_indices=[0])
         y_np = log(np.sum(exp(x_np), axis=0))
         self.assertShapeEqual(y_np, y_tf)
-        y_tf_np = y_tf.eval()
+        y_tf_np = self.evaluate(y_tf)
         self.assertAllClose(y_tf_np, y_np)
 
   def testReductionIndices2(self):
@@ -117,7 +117,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
         y_tf = math_ops.reduce_logsumexp(x_np, reduction_indices=0)
         y_np = log(np.sum(exp(x_np), axis=0))
         self.assertShapeEqual(y_np, y_tf)
-        y_tf_np = y_tf.eval()
+        y_tf_np = self.evaluate(y_tf)
         self.assertAllClose(y_tf_np, y_np)
 
   def testKeepDims(self):
@@ -195,7 +195,7 @@ class ModTest(test_util.TensorFlowTestCase):
         with self.cached_session(use_gpu=True):
           x_tf = constant_op.constant(x_np, shape=x_np.shape)
           y_tf = math_ops.mod(x_tf, denom)
-          y_tf_np = y_tf.eval()
+          y_tf_np = self.evaluate(y_tf)
           y_np = np.fmod(x_np, denom)
         self.assertAllClose(y_tf_np, y_np, atol=1e-2)
 
@@ -208,7 +208,7 @@ class ModTest(test_util.TensorFlowTestCase):
         with self.cached_session(use_gpu=True):
           x_tf = constant_op.constant(x_np, shape=x_np.shape)
           y_tf = math_ops.mod(x_tf, denom)
-          y_tf_np = y_tf.eval()
+          y_tf_np = self.evaluate(y_tf)
           y_np = np.mod(x_np, denom)
         self.assertAllClose(y_tf_np, y_np)
 
@@ -467,8 +467,9 @@ class DivAndModTest(test_util.TensorFlowTestCase):
         c_grad = gradients.gradients(math_ops.div(a, b), [a, b])
         self.assertAllEqual([x.eval() for x in c_grad], [.25, -.125])
         c_grad = gradients.gradients(math_ops.floordiv(a, b), [a, b])
-        self.assertAllEqual([None if x is None else x.eval()
-                             for x in c_grad], [None, None])
+        self.assertAllEqual(
+            [None if x is None else self.evaluate(x) for x in c_grad],
+            [None, None])
 
   def testConsistent(self):
     nums, divs = self.intTestData()
diff --git a/tensorflow/python/ops/nn_batchnorm_test.py b/tensorflow/python/ops/nn_batchnorm_test.py
index c8a5b58e45..b50bccfde2 100644
--- a/tensorflow/python/ops/nn_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_batchnorm_test.py
@@ -507,8 +507,8 @@ class MomentsTest(test.TestCase):
       expected_variance = expected_x_squared - expected_mean_squared
 
       # Check that the moments are correct.
-      self.assertAllCloseAccordingToType(expected_mean, mean.eval())
-      self.assertAllCloseAccordingToType(expected_variance, var.eval())
+      self.assertAllCloseAccordingToType(expected_mean, self.evaluate(mean))
+      self.assertAllCloseAccordingToType(expected_variance, self.evaluate(var))
 
   def testBasic(self):
     for keep_dims in [False, True]:
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index 5ac8eba6f7..a6c582fcac 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -50,7 +50,7 @@ class BatchNormalizationTest(test.TestCase):
     y = self._batch_norm(x, mean, var, offset, scale, epsilon)
     if data_format == 'NCHW':
       y = array_ops.transpose(y, [0, 3, 1, 2])
-    return y.eval()
+    return self.evaluate(y)
 
   def _test_inference(self,
                       x_shape,
@@ -102,7 +102,7 @@ class BatchNormalizationTest(test.TestCase):
     y = self._batch_norm(x, mean, var, offset, scale, epsilon)
     if data_format == 'NCHW':
       y = array_ops.transpose(y, [0, 3, 1, 2])
-    return y.eval(), mean.eval(), var.eval()
+    return self.evaluate(y), self.evaluate(mean), self.evaluate(var)
 
   def _test_training(self,
                      x_shape,
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 152b2020eb..af6c728694 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -57,7 +57,7 @@ class ZeroFractionTest(test_lib.TestCase):
       x_tf = constant_op.constant(x_np)
       x_tf.set_shape(x_shape)
       y_tf = nn_impl.zero_fraction(x_tf)
-      y_tf_np = y_tf.eval()
+      y_tf_np = self.evaluate(y_tf)
     eps = 1e-8
     self.assertAllClose(y_tf_np, y_np, eps)
 
@@ -71,13 +71,13 @@ class ZeroFractionTest(test_lib.TestCase):
     sparsity = nn_impl.zero_fraction(
         array_ops.zeros([int(2**27 * 1.01)], dtype=dtypes.int8))
     with self.cached_session():
-      self.assertAllClose(1.0, sparsity.eval())
+      self.assertAllClose(1.0, self.evaluate(sparsity))
 
   def testZeroFraction2_27Ones(self):
     sparsity = nn_impl.zero_fraction(
         array_ops.ones([int(2**27 * 1.01)], dtype=dtypes.int8))
     with self.cached_session():
-      self.assertAllClose(0.0, sparsity.eval())
+      self.assertAllClose(0.0, self.evaluate(sparsity))
 
   def testUnknownSize(self):
     value = array_ops.placeholder(dtype=dtypes.float32)
@@ -309,7 +309,7 @@ class DropoutTest(test_lib.TestCase):
         final_count = 0
         self.assertEqual([x_dim, y_dim], dropout.get_shape())
         for _ in xrange(0, num_iter):
-          value = dropout.eval()
+          value = self.evaluate(dropout)
           final_count += np.count_nonzero(value)
           # Verifies that there are only two values: 0 and 1/keep_prob.
           sorted_value = np.unique(np.sort(value))
@@ -337,7 +337,7 @@ class DropoutTest(test_lib.TestCase):
         self.assertEqual([x_dim, y_dim], dropout.get_shape())
         final_count = 0
         for _ in xrange(0, num_iter):
-          value = dropout.eval()
+          value = self.evaluate(dropout)
           final_count += np.count_nonzero(value)
           # Verifies that there are only two values: 0 and 1/keep_prob.
           sorted_value = np.unique(np.sort(value))
@@ -361,7 +361,7 @@ class DropoutTest(test_lib.TestCase):
         dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
         self.assertEqual([x_dim, y_dim], dropout.get_shape())
         for _ in xrange(0, num_iter):
-          value = dropout.eval()
+          value = self.evaluate(dropout)
           # Verifies that each y column as only one type of activation.
           for i in xrange(x_dim):
             sorted_value = np.unique(np.sort(value[i, :]))
@@ -417,7 +417,7 @@ class DropoutTest(test_lib.TestCase):
         self.assertEqual([x_dim, y_dim], dropout.get_shape())
         final_count = 0
         for _ in xrange(0, num_iter):
-          value = dropout.eval()
+          value = self.evaluate(dropout)
           final_count += np.count_nonzero(value)
           # Verifies that there are only two values: 0 and 1/keep_prob.
           sorted_value = np.unique(np.sort(value))
@@ -672,7 +672,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
         # Test that the exponentiated logits of accidental hits are near 0.
         # First we need to find the hits in this random test run:
         labels_reshape = labels.reshape((batch_size, num_true))
-        got_logits = logits_tensor.eval()
+        got_logits = self.evaluate(logits_tensor)
         for row in xrange(batch_size):
           row_labels = labels_reshape[row, :]
           for col in xrange(len(sampled)):
@@ -794,7 +794,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
           sampled_values=sampled_vals,
           partition_strategy="div")
 
-      self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4)
+      self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
 
       # Test with sharded weights and sharded biases.
       weight_shards, bias_shards = self._ShardTestEmbeddings(
@@ -810,7 +810,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
           sampled_values=sampled_vals,
           partition_strategy="div")
 
-      self.assertAllClose(exp_nce_loss, got_nce_loss.eval(), 1e-4)
+      self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
 
   def testSampledSoftmaxLoss(self):
     # A simple test to verify the numerics.
@@ -853,7 +853,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
           partition_strategy="div")
 
       self.assertAllClose(exp_sampled_softmax_loss,
-                          got_sampled_softmax_loss.eval(), 1e-4)
+                          self.evaluate(got_sampled_softmax_loss), 1e-4)
 
       # Test with sharded weights and sharded biases.
       weight_shards, bias_shards = self._ShardTestEmbeddings(
@@ -871,7 +871,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
           partition_strategy="div")
 
       self.assertAllClose(exp_sampled_softmax_loss,
-                          got_sampled_softmax_loss.eval(), 1e-4)
+                          self.evaluate(got_sampled_softmax_loss), 1e-4)
 
   def testSampledSoftmaxLossBf16(self):
     # A simple test to verify the numerics for bfloat16.
@@ -922,7 +922,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
               partition_strategy="div"), dtypes.float32)
 
       self.assertAllClose(exp_sampled_softmax_loss,
-                          got_sampled_softmax_loss.eval(), 1e-1)
+                          self.evaluate(got_sampled_softmax_loss), 1e-1)
 
 
 class CReluTest(test_lib.TestCase):
diff --git a/tensorflow/python/ops/nn_xent_test.py b/tensorflow/python/ops/nn_xent_test.py
index 57ce4fd0a9..7bf18c47fe 100644
--- a/tensorflow/python/ops/nn_xent_test.py
+++ b/tensorflow/python/ops/nn_xent_test.py
@@ -68,7 +68,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase):
           loss = nn_impl.sigmoid_cross_entropy_with_logits(
               labels=targets, logits=logits)
           np_loss = np.array(losses).astype(np.float32)
-          tf_loss = loss.eval()
+          tf_loss = self.evaluate(loss)
         self.assertAllClose(np_loss, tf_loss, atol=0.001)
 
   def testLogisticOutputMultiDim(self):
@@ -79,7 +79,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase):
           loss = nn_impl.sigmoid_cross_entropy_with_logits(
               labels=targets, logits=logits)
           np_loss = np.array(losses).astype(np.float32)
-          tf_loss = loss.eval()
+          tf_loss = self.evaluate(loss)
         self.assertAllClose(np_loss, tf_loss, atol=0.001)
 
   def testGradient(self):
@@ -143,7 +143,7 @@ class WeightedCrossEntropyTest(test.TestCase):
         loss = nn_impl.weighted_cross_entropy_with_logits(
             targets=targets, logits=logits, pos_weight=pos_weight)
         np_loss = np.array(losses).astype(np.float32)
-        tf_loss = loss.eval()
+        tf_loss = self.evaluate(loss)
       self.assertAllClose(np_loss, tf_loss, atol=0.001)
 
   def testOutputMultiDim(self):
@@ -154,7 +154,7 @@ class WeightedCrossEntropyTest(test.TestCase):
         loss = nn_impl.weighted_cross_entropy_with_logits(
             targets=targets, logits=logits, pos_weight=pos_weight)
         np_loss = np.array(losses).astype(np.float32)
-        tf_loss = loss.eval()
+        tf_loss = self.evaluate(loss)
       self.assertAllClose(np_loss, tf_loss, atol=0.001)
 
   def testGradient(self):
diff --git a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
index b43470dfa1..243fa34c4b 100644
--- a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
@@ -102,7 +102,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
     self.assertEqual(value.ragged_rank, converted.ragged_rank)
     self.assertEqual(dtypes.as_dtype(expected_dtype), converted.dtype)
     with self.test_session():
-      self.assertEqual(value.tolist(), converted.eval().tolist())
+      self.assertEqual(value.tolist(), self.evaluate(converted).tolist())
 
   @parameterized.parameters([
       dict(
diff --git a/tensorflow/python/ops/ragged/ragged_const_op_test.py b/tensorflow/python/ops/ragged/ragged_const_op_test.py
index 13f79c5729..66c39475fa 100644
--- a/tensorflow/python/ops/ragged/ragged_const_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_const_op_test.py
@@ -183,7 +183,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertEqual(tuple(rt.shape.as_list()), expected_shape)
 
     with self.test_session():
-      result = rt.eval()
+      result = self.evaluate(rt)
       if rt.shape.ndims > 0:
         self.assertEqual(result.tolist(), pylist)
         if expected_shape is not None:
diff --git a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
index eb237f4c95..7c59cd0b77 100644
--- a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
@@ -291,7 +291,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
           dt.shape.is_compatible_with(rt.shape),
           '%s is incompatible with %s' % (dt.shape, rt.shape))
       with self.test_session():
-        self.assertEqual(rt.eval().tolist(), dt.eval().tolist())
+        self.assertEqual(rt.eval().tolist(), self.evaluate(dt).tolist())
 
   @parameterized.parameters(
       # With no padding or lengths
diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
index 6f3f33b444..dac86310b9 100644
--- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
@@ -161,7 +161,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.test_session():
       if ragged.is_ragged(expected_output):
         self.assertEqual(output.ragged_rank, expected_rt.ragged_rank)
-      output_values = output.eval()
+      output_values = self.evaluate(output)
       self.assertAllEqual(expected_output, output_values.tolist())
 
   def testRaggedMapOnStructure(self):
@@ -232,7 +232,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         infer_shape=False)
 
     with self.test_session():
-      result = output.eval().tolist()
+      result = self.evaluate(output).tolist()
       self.assertAllEqual(
           result, [[[0, 10], [0, 20]], [[1, 30], [1, 40]], [[2, 50], [2, 60]],
                    [[3, 70]], [[4, 80], [4, 90], [4, 100]]])
@@ -255,7 +255,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     with self.test_session():
       self.assertAllEqual(
-          out.eval().tolist(),
+          self.evaluate(out).tolist(),
           [[b'hello', b'there'], [b'merhaba'], [b'bonjour', b'ca va']])
 
   def testMismatchRaggedRank(self):
diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
index 7d41eb7f75..373a332f13 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
@@ -157,7 +157,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
     segmented = segment_op(rt, segment_ids, num_segments)
     with self.test_session():
       self.assertNestedListAmostEqual(
-          segmented.eval().tolist(), expected, places=5)
+          self.evaluate(segmented).tolist(), expected, places=5)
 
   def testRaggedRankTwo(self):
     rt = ragged.constant([
diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py
index 755333de39..03672e4521 100644
--- a/tensorflow/python/ops/ragged/ragged_where_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py
@@ -170,7 +170,7 @@ class RaggedWhereOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertEqual(
         getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0))
     with self.test_session():
-      result_value = result.eval()
+      result_value = self.evaluate(result)
       if hasattr(result_value, 'tolist'):
         result_value = result_value.tolist()
       if hasattr(expected, 'tolist'):
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 924b2e7c06..648c1c5928 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -145,7 +145,7 @@ class SavedModelLoaderTest(test.TestCase):
         loader.restore_variables(sess, None)
 
       loader.restore_variables(sess, tf_saver.Saver())
-      self.assertEqual(55, z.eval())
+      self.assertEqual(55, self.evaluate(z))
 
   def test_run_init_op(self):
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index 5d6167ab38..ab43e93853 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -62,7 +62,7 @@ class SavedModelTest(test.TestCase):
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.VariableV1(variable_value, name=variable_name)
     sess.run(variables.global_variables_initializer())
-    self.assertEqual(variable_value, v.eval())
+    self.assertEqual(variable_value, self.evaluate(v))
 
   def _build_asset_collection(self, asset_file_name, asset_file_contents,
                               asset_file_tensor_name, asset_subdir=""):
@@ -461,7 +461,7 @@ class SavedModelTest(test.TestCase):
       v = variables.VariableV1(42, name="v")
       ops.add_to_collection("foo_vars", v)
       sess.run(variables.global_variables_initializer())
-      self.assertEqual(42, v.eval())
+      self.assertEqual(42, self.evaluate(v))
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
     # Graph with the same single variable added to a different collection.
@@ -471,7 +471,7 @@ class SavedModelTest(test.TestCase):
       v = variables.VariableV1(43, name="v")
       ops.add_to_collection("bar_vars", v)
       sess.run(variables.global_variables_initializer())
-      self.assertEqual(43, v.eval())
+      self.assertEqual(43, self.evaluate(v))
       builder.add_meta_graph(["bar"])
 
     # Save the SavedModel to disk.
diff --git a/tensorflow/python/saved_model/simple_save_test.py b/tensorflow/python/saved_model/simple_save_test.py
index 18f82daada..2d404dcea4 100644
--- a/tensorflow/python/saved_model/simple_save_test.py
+++ b/tensorflow/python/saved_model/simple_save_test.py
@@ -34,7 +34,7 @@ class SimpleSaveTest(test.TestCase):
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.Variable(variable_value, name=variable_name)
     sess.run(variables.global_variables_initializer())
-    self.assertEqual(variable_value, v.eval())
+    self.assertEqual(variable_value, self.evaluate(v))
     return v
 
   def _check_variable_info(self, actual_variable, expected_variable):
diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py
index 09d4b63fbb..20b62e5016 100644
--- a/tensorflow/python/summary/writer/writer_test.py
+++ b/tensorflow/python/summary/writer/writer_test.py
@@ -309,12 +309,11 @@ class FileWriterTestCase(test.TestCase):
       summ = summary_pb2.Summary(
           value=[summary_pb2.Summary.Value(
               tag="i", simple_value=1.0)])
-      sw.add_summary(summ.SerializeToString(), i.eval())
+      sw.add_summary(summ.SerializeToString(), self.evaluate(i))
       sw.add_summary(
           summary_pb2.Summary(
-              value=[summary_pb2.Summary.Value(
-                  tag="l", simple_value=2.0)]),
-          l.eval())
+              value=[summary_pb2.Summary.Value(tag="l", simple_value=2.0)]),
+          self.evaluate(l))
       sw.close()
 
     rr = self._EventsReader(test_dir)
diff --git a/tensorflow/python/training/adadelta_test.py b/tensorflow/python/training/adadelta_test.py
index a14ac895ac..7cbaf1039f 100644
--- a/tensorflow/python/training/adadelta_test.py
+++ b/tensorflow/python/training/adadelta_test.py
@@ -177,12 +177,11 @@ class AdadeltaOptimizerTest(test.TestCase):
             1.0, 1.0, 1.0).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-111, -138]], var0.eval())
+        self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/adagrad_da_test.py b/tensorflow/python/training/adagrad_da_test.py
index 00801be3b4..761f703cb5 100644
--- a/tensorflow/python/training/adagrad_da_test.py
+++ b/tensorflow/python/training/adagrad_da_test.py
@@ -92,12 +92,13 @@ class AdagradDAOptimizerTest(test.TestCase):
             1.0, global_step).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-1, -1]], var0.eval(), rtol=0.01)
+        self.assertAllCloseAccordingToType([[-1, -1]],
+                                           self.evaluate(var0),
+                                           rtol=0.01)
 
   def testAdagradDAwithoutRegularizationBasic2(self):
     for dtype in [dtypes.float64, dtypes.float32]:
diff --git a/tensorflow/python/training/adagrad_test.py b/tensorflow/python/training/adagrad_test.py
index 7caf01f64d..962e65c41f 100644
--- a/tensorflow/python/training/adagrad_test.py
+++ b/tensorflow/python/training/adagrad_test.py
@@ -107,13 +107,14 @@ class AdagradOptimizerTest(test.TestCase):
         sgd_op = adagrad.AdagradOptimizer(1.0).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType(
-            [[1.0, 2.0], [3.0, 4.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0], [3.0, 4.0]],
+                                           self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[0, 1], [3, 4]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[0, 1], [3, 4]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -128,16 +129,18 @@ class AdagradOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Run 3 steps of adagrad
         for _ in range(3):
           ada_update.run()
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval())
+            np.array([-1.6026098728179932, -0.6026098728179932]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([2.715679168701172, 3.715679168701172]), var1.eval())
+            np.array([2.715679168701172, 3.715679168701172]),
+            self.evaluate(var1))
 
   def testSparseBasic(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -159,16 +162,16 @@ class AdagradOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([[1.0], [2.0]], var0.eval())
-        self.assertAllClose([[3.0], [4.0]], var1.eval())
+        self.assertAllClose([[1.0], [2.0]], self.evaluate(var0))
+        self.assertAllClose([[3.0], [4.0]], self.evaluate(var1))
         # Run 3 step of sgd
         for _ in range(3):
           ada_update.run()
         # Validate updated params
         self.assertAllCloseAccordingToType(
-            np.array([[-1.6026098728179932], [2.0]]), var0.eval())
+            np.array([[-1.6026098728179932], [2.0]]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([[3.0], [3.715679168701172]]), var1.eval())
+            np.array([[3.0], [3.715679168701172]]), self.evaluate(var1))
 
   def testSparseRepeatedIndices(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -193,12 +196,12 @@ class AdagradOptimizerTest(test.TestCase):
             [(grad_aggregated, aggregated_update_var)])
         variables.global_variables_initializer().run()
         self.assertAllClose(aggregated_update_var.eval(),
-                            repeated_index_update_var.eval())
+                            self.evaluate(repeated_index_update_var))
         for _ in range(3):
           repeated_update.run()
           aggregated_update.run()
           self.assertAllClose(aggregated_update_var.eval(),
-                              repeated_index_update_var.eval())
+                              self.evaluate(repeated_index_update_var))
 
   def testSparseRepeatedIndicesResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -217,12 +220,12 @@ class AdagradOptimizerTest(test.TestCase):
             2.0).minimize(loss_aggregated)
         variables.global_variables_initializer().run()
         self.assertAllCloseAccordingToType(
-            var_repeated.eval(), var_aggregated.eval())
+            self.evaluate(var_repeated), self.evaluate(var_aggregated))
         for _ in range(3):
           update_op_repeated.run()
           update_op_aggregated.run()
           self.assertAllCloseAccordingToType(
-              var_repeated.eval(), var_aggregated.eval())
+              self.evaluate(var_repeated), self.evaluate(var_aggregated))
 
   def testSparseStability(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -253,12 +256,12 @@ class AdagradOptimizerTest(test.TestCase):
           init.run()
           ada_update.run()
           self.assertAllCloseAccordingToType(
-              np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval())
+              np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), self.evaluate(slot0))
           self.assertAllCloseAccordingToType(
               np.array([[
                   0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573,
                   -0.01029443
-              ]]), var0.eval())
+              ]]), self.evaluate(var0))
 
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -282,17 +285,19 @@ class AdagradOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values.
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Mix the first and the second adagrad for 3 steps.
         ada_update1.run()
         ada_update2.run()
         ada_update1.run()
         # Validate updated params (the same as with only 1 Adagrad).
         self.assertAllCloseAccordingToType(
-            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval())
+            np.array([-1.6026098728179932, -0.6026098728179932]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([2.715679168701172, 3.715679168701172]), var1.eval())
+            np.array([2.715679168701172, 3.715679168701172]),
+            self.evaluate(var1))
 
   def testDynamicShapeVariable_Ok(self):
     with self.cached_session():
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index 0d42cc7b9c..87dad0a8a6 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -83,23 +83,24 @@ class AdamOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
         # Run 3 steps of Adam
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           update.run()
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testSparse(self):
     self.doTestSparse(use_resource=False)
@@ -143,12 +144,12 @@ class AdamOptimizerTest(test.TestCase):
             [(grad_aggregated, aggregated_update_var)])
         variables.global_variables_initializer().run()
         self.assertAllClose(aggregated_update_var.eval(),
-                            repeated_index_update_var.eval())
+                            self.evaluate(repeated_index_update_var))
         for _ in range(3):
           repeated_update.run()
           aggregated_update.run()
           self.assertAllClose(aggregated_update_var.eval(),
-                              repeated_index_update_var.eval())
+                              self.evaluate(repeated_index_update_var))
 
   def doTestBasic(self, use_resource=False, use_callable_params=False):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
@@ -254,23 +255,24 @@ class AdamOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
         # Run 3 steps of Adam
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           update.run()
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -294,13 +296,14 @@ class AdamOptimizerTest(test.TestCase):
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 3 steps of intertwined Adam1 and Adam2.
         for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**t,
+                                             self.evaluate(beta2_power))
           if t % 2 == 0:
             update1.run()
           else:
@@ -310,8 +313,8 @@ class AdamOptimizerTest(test.TestCase):
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
           # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testTwoSessions(self):
     optimizer = adam.AdamOptimizer()
diff --git a/tensorflow/python/training/checkpoint_management_test.py b/tensorflow/python/training/checkpoint_management_test.py
index 3a061bcb35..b61ed17531 100644
--- a/tensorflow/python/training/checkpoint_management_test.py
+++ b/tensorflow/python/training/checkpoint_management_test.py
@@ -123,9 +123,9 @@ class LatestCheckpointWithRelativePaths(test.TestCase):
           # Record a short training history.
           variables.global_variables_initializer().run()
           save.save(sess, filepath, global_step=0)
-          inc.eval()
+          self.evaluate(inc)
           save.save(sess, filepath, global_step=1)
-          inc.eval()
+          self.evaluate(inc)
           save.save(sess, filepath, global_step=2)
 
         with self.cached_session() as sess:
diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py
index dde8431497..38d4acf85f 100644
--- a/tensorflow/python/training/checkpoint_ops_test.py
+++ b/tensorflow/python/training/checkpoint_ops_test.py
@@ -115,7 +115,8 @@ class LoadAndRemapWrappersTest(test.TestCase):
         axis=1)
 
     with self.cached_session():
-      self.assertAllClose(expected_remapped_matrix, remapped_matrix.eval())
+      self.assertAllClose(expected_remapped_matrix,
+                          self.evaluate(remapped_matrix))
 
   def test_load_and_remap_output_layer_weight_initializer_linear(self):
     """Tests for the output layer initializer in the linear multi-class case."""
diff --git a/tensorflow/python/training/ftrl_test.py b/tensorflow/python/training/ftrl_test.py
index 15c50bc878..a61132a966 100644
--- a/tensorflow/python/training/ftrl_test.py
+++ b/tensorflow/python/training/ftrl_test.py
@@ -113,11 +113,13 @@ class FtrlOptimizerTest(test.TestCase):
         sgd_op = ftrl.FtrlOptimizer(1.0).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType([[0, 1]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[0, 1]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testFtrlWithL1(self):
     for dtype in [dtypes.half, dtypes.float32]:
diff --git a/tensorflow/python/training/gradient_descent_test.py b/tensorflow/python/training/gradient_descent_test.py
index 1ddea598e5..2028e7b4b0 100644
--- a/tensorflow/python/training/gradient_descent_test.py
+++ b/tensorflow/python/training/gradient_descent_test.py
@@ -47,15 +47,15 @@ class GradientDescentOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
-                                           var0.eval())
+                                           self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
-                                           var1.eval())
+                                           self.evaluate(var1))
         self.assertEqual(0, len(optimizer.variables()))
 
   def testBasicResourceVariable(self):
@@ -73,15 +73,15 @@ class GradientDescentOptimizerTest(test.TestCase):
         # a long-term solution for this.
         resources.initialize_resources([var0, var1]).run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
-                                           var0.eval())
+                                           self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
-                                           var1.eval())
+                                           self.evaluate(var1))
 
   def testBasicCallableParams(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -99,15 +99,15 @@ class GradientDescentOptimizerTest(test.TestCase):
         # a long-term solution for this.
         resources.initialize_resources([var0, var1]).run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
-                                           var0.eval())
+                                           self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
-                                           var1.eval())
+                                           self.evaluate(var1))
 
   def testMinimizeResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -124,16 +124,16 @@ class GradientDescentOptimizerTest(test.TestCase):
         # a long-term solution for this.
         resources.initialize_resources([var0, var1]).run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
-        self.assertAllCloseAccordingToType([3.0], var1.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0
         np_grad = 2 * np_pred
         self.assertAllCloseAccordingToType(
-            [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval())
-        self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval())
+            [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1))
 
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -151,16 +151,16 @@ class GradientDescentOptimizerTest(test.TestCase):
         # a long-term solution for this.
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
-        self.assertAllCloseAccordingToType([3.0], var1.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0
         np_grad = 2 * np_pred
         self.assertAllCloseAccordingToType(
-            [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval())
-        self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval())
+            [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1))
 
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -174,15 +174,15 @@ class GradientDescentOptimizerTest(test.TestCase):
             lrate).apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
-                                           var0.eval())
+                                           self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
-                                           var1.eval())
+                                           self.evaluate(var1))
 
   def testGradWrtRef(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -193,7 +193,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         grads_and_vars = opt.compute_gradients(vars_[0] + vars_[1], vars_)
         variables.global_variables_initializer().run()
         for grad, _ in grads_and_vars:
-          self.assertAllCloseAccordingToType([1.0], grad.eval())
+          self.assertAllCloseAccordingToType([1.0], self.evaluate(grad))
 
   def testWithGlobalStep(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -207,16 +207,16 @@ class GradientDescentOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
-        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        self.assertAllCloseAccordingToType([1.0, 2.0], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params and global_step
         self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
-                                           var0.eval())
+                                           self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
-                                           var1.eval())
-        self.assertAllCloseAccordingToType(1, global_step.eval())
+                                           self.evaluate(var1))
+        self.assertAllCloseAccordingToType(1, self.evaluate(global_step))
 
   def testSparseBasic(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -237,15 +237,15 @@ class GradientDescentOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0], [2.0]], var0.eval())
-        self.assertAllCloseAccordingToType([[3.0], [4.0]], var1.eval())
+        self.assertAllCloseAccordingToType([[1.0], [2.0]], self.evaluate(var0))
+        self.assertAllCloseAccordingToType([[3.0], [4.0]], self.evaluate(var1))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
         self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]],
-                                           var0.eval())
+                                           self.evaluate(var0))
         self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]],
-                                           var1.eval())
+                                           self.evaluate(var1))
 
   def testCapturingInDefunWhileExecutingEagerly(self):
     with context.eager_mode():
diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py
index 085b77d1d6..e5aac5da18 100644
--- a/tensorflow/python/training/input_test.py
+++ b/tensorflow/python/training/input_test.py
@@ -58,9 +58,12 @@ class MatchFilenamesOnceTest(test_lib.TestCase):
       one = inp.match_filenames_once(additional[1])
       variables.global_variables_initializer().run()
       variables.local_variables_initializer().run()
-      self.assertItemsEqual(map(compat.as_bytes, filenames), star.eval())
-      self.assertItemsEqual(map(compat.as_bytes, additional), question.eval())
-      self.assertItemsEqual([compat.as_bytes(additional[1])], one.eval())
+      self.assertItemsEqual(
+          map(compat.as_bytes, filenames), self.evaluate(star))
+      self.assertItemsEqual(
+          map(compat.as_bytes, additional), self.evaluate(question))
+      self.assertItemsEqual([compat.as_bytes(additional[1])],
+                            self.evaluate(one))
 
 
 class LimitEpochsTest(test_lib.TestCase):
@@ -71,7 +74,7 @@ class LimitEpochsTest(test_lib.TestCase):
       seven_forever = inp.limit_epochs(seven)
       variables.local_variables_initializer().run()
       for _ in range(100):
-        self.assertEqual(7, seven_forever.eval())
+        self.assertEqual(7, self.evaluate(seven_forever))
 
   def testLimit(self):
     with self.cached_session():
@@ -79,10 +82,10 @@ class LimitEpochsTest(test_lib.TestCase):
       love_me_two_times = inp.limit_epochs(love_me, num_epochs=2)
       variables.global_variables_initializer().run()
       variables.local_variables_initializer().run()
-      self.assertEqual(b"Love Me", love_me_two_times.eval())
-      self.assertEqual(b"Love Me", love_me_two_times.eval())
+      self.assertEqual(b"Love Me", self.evaluate(love_me_two_times))
+      self.assertEqual(b"Love Me", self.evaluate(love_me_two_times))
       with self.assertRaises(errors_impl.OutOfRangeError):
-        love_me_two_times.eval()
+        self.evaluate(love_me_two_times)
 
 
 class InputProducerTest(test_lib.TestCase):
@@ -102,11 +105,12 @@ class InputProducerTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       # No randomness, so just see repeated copies of the input.
-      self.assertAllEqual(input_tensor * num_epochs, dequeue_many.eval())
+      self.assertAllEqual(input_tensor * num_epochs,
+                          self.evaluate(dequeue_many))
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       for thread in threads:
         thread.join()
 
@@ -127,11 +131,11 @@ class InputProducerTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       # No randomness, so just see repeated copies of the input.
-      self.assertAllEqual(input_value * num_epochs, dequeue_many.eval())
+      self.assertAllEqual(input_value * num_epochs, self.evaluate(dequeue_many))
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       for thread in threads:
         thread.join()
 
@@ -156,12 +160,12 @@ class StringInputProducerTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       # No randomness, so just see repeated copies of the input.
-      output = dequeue_many.eval()
+      output = self.evaluate(dequeue_many)
       self.assertAllEqual(strings * num_epochs, output)
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       for thread in threads:
         thread.join()
 
@@ -184,7 +188,7 @@ class StringInputProducerTest(test_lib.TestCase):
       for e in expected:
         frequency[e] = 0
       for _ in range(num_epochs):
-        output = dequeue_many.eval()
+        output = self.evaluate(dequeue_many)
         key = b"".join(output)
         self.assertIn(key, expected)
         frequency[key] += 1
@@ -200,7 +204,7 @@ class StringInputProducerTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       for thread in threads:
         thread.join()
 
@@ -224,7 +228,7 @@ class StringInputProducerTest(test_lib.TestCase):
       variables.local_variables_initializer().run()
       threads = queue_runner_impl.start_queue_runners(coord=coord)
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       coord.request_stop()
       for thread in threads:
         thread.join()
@@ -272,12 +276,12 @@ class RangeInputProducerTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       # No randomness, so just see repeated copies of the input.
-      output = dequeue_many.eval()
+      output = self.evaluate(dequeue_many)
       self.assertAllEqual(list(xrange(range_size)) * num_epochs, output)
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       for thread in threads:
         thread.join()
 
@@ -300,7 +304,7 @@ class RangeInputProducerTest(test_lib.TestCase):
       for e in expected:
         frequency[e] = 0
       for _ in range(num_epochs):
-        output = dequeue_many.eval()
+        output = self.evaluate(dequeue_many)
         key = 10 * (output[0] + 1) + (output[1] + 1)
         self.assertIn(key, expected)
         frequency[key] += 1
@@ -316,7 +320,7 @@ class RangeInputProducerTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        dequeue.eval()
+        self.evaluate(dequeue)
       for thread in threads:
         thread.join()
 
@@ -938,7 +942,7 @@ class BatchTest(test_lib.TestCase):
       coord = coordinator.Coordinator()
       threads = queue_runner_impl.start_queue_runners(coord=coord)
 
-      batched_np = batched.eval()
+      batched_np = self.evaluate(batched)
 
       coord.request_stop()
       for thread in threads:
@@ -1525,7 +1529,7 @@ class BatchJoinTest(test_lib.TestCase):
       coord = coordinator.Coordinator()
       threads = queue_runner_impl.start_queue_runners(coord=coord)
 
-      batched_np = batched.eval()
+      batched_np = self.evaluate(batched)
 
       coord.request_stop()
       for thread in threads:
diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py
index 8a21c39d32..b6cac6addf 100644
--- a/tensorflow/python/training/momentum_test.py
+++ b/tensorflow/python/training/momentum_test.py
@@ -183,8 +183,8 @@ class MomentumOptimizerTest(test.TestCase):
           var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np,
                                                                     accum1_np,
                                                                     3, 2.0, 0.9)
-          self.assertAllClose(var0_np, var0.eval())
-          self.assertAllClose(var1_np, var1.eval())
+          self.assertAllClose(var0_np, self.evaluate(var0))
+          self.assertAllClose(var1_np, self.evaluate(var1))
 
   def testSparseNesterovMomentum(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -224,8 +224,8 @@ class MomentumOptimizerTest(test.TestCase):
           var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np,
                                                                     accum1_np,
                                                                     3, 2.0, 0.9)
-          self.assertAllClose(var0_np, var0.eval())
-          self.assertAllClose(var1_np, var1.eval())
+          self.assertAllClose(var0_np, self.evaluate(var0))
+          self.assertAllClose(var1_np, self.evaluate(var1))
 
   @test_util.run_in_graph_and_eager_modes(reset_test=True)
   def testMinimizeSparseResourceVariable(self):
@@ -303,37 +303,43 @@ class MomentumOptimizerTest(test.TestCase):
         self.assertFalse(slot1 in variables.trainable_variables())
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
-        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.1, 0.1]), self.evaluate(slot0))
+        self.assertAllCloseAccordingToType(
+            np.array([0.01, 0.01]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+            self.evaluate(var1))
         # Step 2: the momentum accumulators contain the previous update.
         mom_update.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
-                2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
-                    (0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval())
+                2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
+                3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
+            ]), self.evaluate(var1))
 
   def _dbParamsMom01(self):
     """Return dist-belief momentum values.
@@ -445,7 +451,7 @@ class MomentumOptimizerTest(test.TestCase):
       variables.global_variables_initializer().run()
       for i in xrange(num_samples):
         mom_update.run(feed_dict={grads0: db_grad[i]})
-        self.assertAllClose(np.array(db_out[i]), var0.eval())
+        self.assertAllClose(np.array(db_out[i]), self.evaluate(var0))
 
   def testSparse(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -476,45 +482,57 @@ class MomentumOptimizerTest(test.TestCase):
         self.assertEquals(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
-        self.assertAllClose([0, 0], var0.eval()[0])
-        self.assertAllClose([0, 0], var0.eval()[1])
-        self.assertAllClose([1, 1], var1.eval()[2])
+        self.assertAllClose([0, 0], self.evaluate(var0)[0])
+        self.assertAllClose([0, 0], self.evaluate(var0)[1])
+        self.assertAllClose([1, 1], self.evaluate(var1)[2])
 
         # Step 1: the momentum accumulators are 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0, 0]), slot0.eval()[0])
-        self.assertAllCloseAccordingToType(np.array([.1, .1]), slot0.eval()[1])
         self.assertAllCloseAccordingToType(
-            np.array([.01, .01]), slot1.eval()[2])
+            np.array([0, 0]),
+            self.evaluate(slot0)[0])
+        self.assertAllCloseAccordingToType(
+            np.array([.1, .1]),
+            self.evaluate(slot0)[1])
+        self.assertAllCloseAccordingToType(
+            np.array([.01, .01]),
+            self.evaluate(slot1)[2])
         # Check that the parameters have been updated.
-        self.assertAllCloseAccordingToType(np.array([0, 0]), var0.eval()[0])
         self.assertAllCloseAccordingToType(
-            np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), var0.eval()[1])
+            np.array([0, 0]),
+            self.evaluate(var0)[0])
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), var1.eval()[2])
+            np.array([-(0.1 * 2.0), -(0.1 * 2.0)]),
+            self.evaluate(var0)[1])
+        self.assertAllCloseAccordingToType(
+            np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]),
+            self.evaluate(var1)[2])
         # Step 2: the momentum accumulators contain the previous update.
         mom_update.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllClose(np.array([0, 0]), slot0.eval()[0])
+        self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0])
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()[1])
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0)[1])
         self.assertAllCloseAccordingToType(
             np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
-            slot1.eval()[2])
+            self.evaluate(slot1)[2])
         # Check that the parameters have been updated.
-        self.assertAllClose(np.array([0, 0]), var0.eval()[0])
+        self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0])
         self.assertAllCloseAccordingToType(
             np.array([
-                -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), -(0.1 * 2.0) - (
-                    (0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval()[1])
+                -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
+                -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
+            ]),
+            self.evaluate(var0)[1])
         self.assertAllCloseAccordingToType(
             np.array([
-                0.98 - ((0.9 * 0.01 + 0.01) * 2.0), 0.98 - (
-                    (0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval()[2])
+                0.98 - ((0.9 * 0.01 + 0.01) * 2.0),
+                0.98 - ((0.9 * 0.01 + 0.01) * 2.0)
+            ]),
+            self.evaluate(var1)[2])
 
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -538,37 +556,43 @@ class MomentumOptimizerTest(test.TestCase):
         self.assertEquals(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
         mom_update1.run()
         # Check that the momentum accumulators have been updated.
-        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
-        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.1, 0.1]), self.evaluate(slot0))
+        self.assertAllCloseAccordingToType(
+            np.array([0.01, 0.01]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+            self.evaluate(var0))
         self.assertAllCloseAccordingToType(
-            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+            self.evaluate(var1))
         # Step 2: the second momentum accumulators contain the previous update.
         mom_update2.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
                 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
-                2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
-                    (0.9 * 0.01 + 0.01) * 2.0)
-            ]), var1.eval())
+                2.98 - ((0.9 * 0.01 + 0.01) * 2.0),
+                3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
+            ]), self.evaluate(var1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py
index bb2fca66e3..8009e3c24e 100644
--- a/tensorflow/python/training/moving_averages_test.py
+++ b/tensorflow/python/training/moving_averages_test.py
@@ -43,11 +43,11 @@ class MovingAveragesTest(test.TestCase):
       assign = moving_averages.assign_moving_average(
           var, val, decay, zero_debias=False)
       variables.global_variables_initializer().run()
-      self.assertAllClose([10.0, 11.0], var.eval())
+      self.assertAllClose([10.0, 11.0], self.evaluate(var))
       assign.op.run()
       self.assertAllClose(
           [10.0 * 0.25 + 1.0 * (1.0 - 0.25), 11.0 * 0.25 + 2.0 * (1.0 - 0.25)],
-          var.eval())
+          self.evaluate(var))
 
   def testAssignMovingAverage(self):
     with self.cached_session():
@@ -56,11 +56,11 @@ class MovingAveragesTest(test.TestCase):
       decay = 0.25
       assign = moving_averages.assign_moving_average(var, val, decay)
       variables.global_variables_initializer().run()
-      self.assertAllClose([0.0, 0.0], var.eval())
+      self.assertAllClose([0.0, 0.0], self.evaluate(var))
       assign.op.run()
-      self.assertAllClose([
-          1.0 * (1.0 - 0.25) / (1 - 0.25), 2.0 * (1.0 - 0.25) / (1 - 0.25)
-      ], var.eval())
+      self.assertAllClose(
+          [1.0 * (1.0 - 0.25) / (1 - 0.25), 2.0 * (1.0 - 0.25) / (1 - 0.25)],
+          self.evaluate(var))
 
   def testAssignMovingAverageNewNamingMultipleCalls(self):
     with variable_scope.variable_scope("scope1") as vs1:
@@ -179,39 +179,39 @@ class ExponentialMovingAverageTest(test.TestCase):
     self.assertEqual("add/ExponentialMovingAverage:0", avg2.name)
 
     # Check initial values.
-    self.assertAllClose(tens, var0.eval())
-    self.assertAllClose(thirties, var1.eval())
-    self.assertAllClose(_Repeat(10.0 + 30.0, dim), tensor2.eval())
+    self.assertAllClose(tens, self.evaluate(var0))
+    self.assertAllClose(thirties, self.evaluate(var1))
+    self.assertAllClose(_Repeat(10.0 + 30.0, dim), self.evaluate(tensor2))
 
     # Check that averages are initialized correctly.
-    self.assertAllClose(tens, avg0.eval())
-    self.assertAllClose(thirties, avg1.eval())
+    self.assertAllClose(tens, self.evaluate(avg0))
+    self.assertAllClose(thirties, self.evaluate(avg1))
     # Note that averages of Tensor's initialize to zeros_like since no value
     # of the Tensor is known because the Op has not been run (yet).
-    self.assertAllClose(_Repeat(0.0, dim), avg2.eval())
+    self.assertAllClose(_Repeat(0.0, dim), self.evaluate(avg2))
 
     # Update the averages and check.
     update.run()
     dk = actual_decay
 
     expected = _Repeat(10.0 * dk + 10.0 * (1 - dk), dim)
-    self.assertAllClose(expected, avg0.eval())
+    self.assertAllClose(expected, self.evaluate(avg0))
     expected = _Repeat(30.0 * dk + 30.0 * (1 - dk), dim)
-    self.assertAllClose(expected, avg1.eval())
+    self.assertAllClose(expected, self.evaluate(avg1))
     expected = _Repeat(0.0 * dk + (10.0 + 30.0) * (1 - dk) / _Scale(dk, 1), dim)
-    self.assertAllClose(expected, avg2.eval())
+    self.assertAllClose(expected, self.evaluate(avg2))
 
     # Again, update the averages and check.
     update.run()
     expected = _Repeat((10.0 * dk + 10.0 * (1 - dk)) * dk + 10.0 * (1 - dk),
                        dim)
-    self.assertAllClose(expected, avg0.eval())
+    self.assertAllClose(expected, self.evaluate(avg0))
     expected = _Repeat((30.0 * dk + 30.0 * (1 - dk)) * dk + 30.0 * (1 - dk),
                        dim)
-    self.assertAllClose(expected, avg1.eval())
+    self.assertAllClose(expected, self.evaluate(avg1))
     expected = _Repeat(((0.0 * dk + (10.0 + 30.0) * (1 - dk)) * dk +
                         (10.0 + 30.0) * (1 - dk)) / _Scale(dk, 2), dim)
-    self.assertAllClose(expected, avg2.eval())
+    self.assertAllClose(expected, self.evaluate(avg2))
 
   def testAverageVariablesNoNumUpdates_Scalar(self):
     with self.cached_session():
diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py
index 7a7d01d50e..5ed0a30285 100644
--- a/tensorflow/python/training/optimizer_test.py
+++ b/tensorflow/python/training/optimizer_test.py
@@ -79,13 +79,13 @@ class OptimizerTest(test.TestCase):
 
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd through optimizer
         opt_op.run()
         # Validate updated params
-        self.assertAllClose([-14., -13.], var0.eval())
-        self.assertAllClose([-6., -5.], var1.eval())
+        self.assertAllClose([-14., -13.], self.evaluate(var0))
+        self.assertAllClose([-6., -5.], self.evaluate(var1))
 
   def testPrecomputedGradient(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -102,15 +102,15 @@ class OptimizerTest(test.TestCase):
 
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Run 1 step of sgd through optimizer
         opt_op.run()
         # Validate updated params
         self.assertAllClose([1.0 - 3 * 5 * 42.0, 2.0 - 3 * 5 * (-42.0)],
-                            var0.eval())
+                            self.evaluate(var0))
         self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)],
-                            var1.eval())
+                            self.evaluate(var1))
 
   @test_util.run_in_graph_and_eager_modes
   def testNoVariables(self):
@@ -257,13 +257,13 @@ class OptimizerTest(test.TestCase):
 
       variables.global_variables_initializer().run()
       # Fetch params to validate initial values
-      self.assertAllClose([1.0, 2.0], var0.eval())
-      self.assertAllClose([3.0, 4.0], var1.eval())
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([3.0, 4.0], self.evaluate(var1))
       # Run 1 step of sgd through optimizer
       opt_op.run()
       # Validate updated params
-      self.assertAllClose([-0.1, -0.1], var0.eval())
-      self.assertAllClose([0., 0.], var1.eval())
+      self.assertAllClose([-0.1, -0.1], self.evaluate(var0))
+      self.assertAllClose([0., 0.], self.evaluate(var1))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/training/proximal_adagrad_test.py b/tensorflow/python/training/proximal_adagrad_test.py
index 74e06a5e2e..272f9019e7 100644
--- a/tensorflow/python/training/proximal_adagrad_test.py
+++ b/tensorflow/python/training/proximal_adagrad_test.py
@@ -106,12 +106,13 @@ class ProximalAdagradOptimizerTest(test.TestCase):
         sgd_op = proximal_adagrad.ProximalAdagradOptimizer(1.0).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[0, 1]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[0, 1]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testProximalAdagradWithL1(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/training/proximal_gradient_descent_test.py b/tensorflow/python/training/proximal_gradient_descent_test.py
index f77f68b234..a9355f4824 100644
--- a/tensorflow/python/training/proximal_gradient_descent_test.py
+++ b/tensorflow/python/training/proximal_gradient_descent_test.py
@@ -103,12 +103,13 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
             1.0).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-111, -138]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[-111, -138]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testProximalGradientDescentWithL1_L2(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/training/queue_runner_test.py b/tensorflow/python/training/queue_runner_test.py
index 15fe42bbd8..65c2c13d8b 100644
--- a/tensorflow/python/training/queue_runner_test.py
+++ b/tensorflow/python/training/queue_runner_test.py
@@ -58,7 +58,7 @@ class QueueRunnerTest(test.TestCase):
         t.join()
       self.assertEqual(0, len(qr.exceptions_raised))
       # The variable should be 3.
-      self.assertEqual(3, var.eval())
+      self.assertEqual(3, self.evaluate(var))
 
   def testTwoOps(self):
     with self.cached_session() as sess:
@@ -80,8 +80,8 @@ class QueueRunnerTest(test.TestCase):
       for t in threads:
         t.join()
       self.assertEqual(0, len(qr.exceptions_raised))
-      self.assertEqual(3, var0.eval())
-      self.assertEqual(30, var1.eval())
+      self.assertEqual(3, self.evaluate(var0))
+      self.assertEqual(30, self.evaluate(var1))
 
   def testExceptionsCaptured(self):
     with self.cached_session() as sess:
@@ -121,11 +121,11 @@ class QueueRunnerTest(test.TestCase):
       # It should have terminated cleanly.
       self.assertEqual(0, len(qr.exceptions_raised))
       # The 2 values should be in queue1.
-      self.assertEqual(10.0, dequeue1.eval())
-      self.assertEqual(10.0, dequeue1.eval())
+      self.assertEqual(10.0, self.evaluate(dequeue1))
+      self.assertEqual(10.0, self.evaluate(dequeue1))
       # And queue1 should now be closed.
       with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed"):
-        dequeue1.eval()
+        self.evaluate(dequeue1)
 
   def testRespectCoordShouldStop(self):
     with self.cached_session() as sess:
@@ -149,7 +149,7 @@ class QueueRunnerTest(test.TestCase):
       coord.join()
       self.assertEqual(0, len(qr.exceptions_raised))
       # The variable should be 0.
-      self.assertEqual(0, var.eval())
+      self.assertEqual(0, self.evaluate(var))
 
   def testRequestStopOnException(self):
     with self.cached_session() as sess:
@@ -263,7 +263,7 @@ class QueueRunnerTest(test.TestCase):
         t.join()
       self.assertEqual(0, len(qr.exceptions_raised))
       # The variable should be 3.
-      self.assertEqual(3, var.eval())
+      self.assertEqual(3, self.evaluate(var))
 
   def testStartQueueRunnersRaisesIfNotASession(self):
     zero64 = constant_op.constant(0, dtype=dtypes.int64)
@@ -310,7 +310,7 @@ class QueueRunnerTest(test.TestCase):
         t.join()
       self.assertEqual(0, len(qr.exceptions_raised))
       # The variable should be 3.
-      self.assertEqual(3, var.eval())
+      self.assertEqual(3, self.evaluate(var))
 
   def testQueueRunnerSerializationRoundTrip(self):
     graph = ops.Graph()
diff --git a/tensorflow/python/training/rmsprop_test.py b/tensorflow/python/training/rmsprop_test.py
index b63abe0529..9ec315f62d 100644
--- a/tensorflow/python/training/rmsprop_test.py
+++ b/tensorflow/python/training/rmsprop_test.py
@@ -138,8 +138,8 @@ class RMSPropOptimizerTest(test.TestCase):
         mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 4 steps of RMSProp
         for _ in range(1, 5):
@@ -154,14 +154,14 @@ class RMSPropOptimizerTest(test.TestCase):
 
           # Validate updated params
           if centered:
-            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+            self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
+            self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+          self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+          self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -178,12 +178,13 @@ class RMSPropOptimizerTest(test.TestCase):
             centered=False).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[0., 1.]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[0., 1.]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testMinimizeSparseResourceVariableCentered(self):
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -200,12 +201,13 @@ class RMSPropOptimizerTest(test.TestCase):
             centered=True).minimize(loss)
         variables.global_variables_initializer().run()
         # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
         sgd_op.run()
         # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-111, -138]], var0.eval(), atol=0.01)
+        self.assertAllCloseAccordingToType([[-111, -138]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
 
   def testSparse(self):
     # TODO(yori): Use ParameterizedTest when available
@@ -258,8 +260,8 @@ class RMSPropOptimizerTest(test.TestCase):
         mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 4 steps of RMSProp
         for _ in range(1, 5):
@@ -274,14 +276,14 @@ class RMSPropOptimizerTest(test.TestCase):
 
           # Validate updated params
           if centered:
-            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+            self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
+            self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+          self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+          self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testWithoutMomentum(self):
     for dtype in [dtypes.half, dtypes.float32]:
@@ -305,34 +307,36 @@ class RMSPropOptimizerTest(test.TestCase):
         self.assertTrue(mom1 is not None)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the rms accumulators where 1. So we should see a normal
         # update: v -= grad * learning_rate
         update.run()
         # Check the root mean square accumulators.
         self.assertAllCloseAccordingToType(
-            np.array([0.901, 0.901]), rms0.eval())
+            np.array([0.901, 0.901]), self.evaluate(rms0))
         self.assertAllCloseAccordingToType(
-            np.array([0.90001, 0.90001]), rms1.eval())
+            np.array([0.90001, 0.90001]), self.evaluate(rms1))
         # Check the parameters.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)),
                 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0))
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
                 3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)),
                 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0))
-            ]), var1.eval())
+            ]), self.evaluate(var1))
         # Step 2: the root mean square accumulators contain the previous update.
         update.run()
         # Check the rms accumulators.
         self.assertAllCloseAccordingToType(
-            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
+            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]),
+            self.evaluate(rms0))
         self.assertAllCloseAccordingToType(
-            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
+            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]),
+            self.evaluate(rms1))
         # Check the parameters.
         self.assertAllCloseAccordingToType(
             np.array([
@@ -340,14 +344,14 @@ class RMSPropOptimizerTest(test.TestCase):
                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0)),
                 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0))
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
                 3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0)),
                 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0))
-            ]), var1.eval())
+            ]), self.evaluate(var1))
 
   def testWithMomentum(self):
     for dtype in [dtypes.half, dtypes.float32]:
@@ -372,57 +376,61 @@ class RMSPropOptimizerTest(test.TestCase):
         self.assertTrue(mom1 is not None)
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: rms = 1, mom = 0. So we should see a normal
         # update: v -= grad * learning_rate
         update.run()
         # Check the root mean square accumulators.
         self.assertAllCloseAccordingToType(
-            np.array([0.901, 0.901]), rms0.eval())
+            np.array([0.901, 0.901]), self.evaluate(rms0))
         self.assertAllCloseAccordingToType(
-            np.array([0.90001, 0.90001]), rms1.eval())
+            np.array([0.90001, 0.90001]), self.evaluate(rms1))
         # Check the momentum accumulators
         self.assertAllCloseAccordingToType(
             np.array([(0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
-                      (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]), mom0.eval())
+                      (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]),
+            self.evaluate(mom0))
         self.assertAllCloseAccordingToType(
             np.array([(0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
-                      (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]), mom1.eval())
+                      (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]),
+            self.evaluate(mom1))
 
         # Check that the parameters.
         self.assertAllCloseAccordingToType(
             np.array([
                 1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
                 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))
-            ]), var0.eval())
+            ]), self.evaluate(var0))
         self.assertAllCloseAccordingToType(
             np.array([
                 3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
                 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))
-            ]), var1.eval())
+            ]), self.evaluate(var1))
 
         # Step 2: the root mean square accumulators contain the previous update.
         update.run()
         # Check the rms accumulators.
         self.assertAllCloseAccordingToType(
-            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
+            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]),
+            self.evaluate(rms0))
         self.assertAllCloseAccordingToType(
-            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
+            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]),
+            self.evaluate(rms1))
         self.assertAllCloseAccordingToType(
             np.array([
                 0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)),
                 0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))
-            ]), mom0.eval())
+            ]), self.evaluate(mom0))
         self.assertAllCloseAccordingToType(
             np.array([
                 0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)),
                 0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))
-            ]), mom1.eval())
+            ]), self.evaluate(mom1))
 
         # Check the parameters.
         self.assertAllCloseAccordingToType(
@@ -433,7 +441,7 @@ class RMSPropOptimizerTest(test.TestCase):
                 2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
                 (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
                  (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)))
-            ]), var0.eval())
+            ]), self.evaluate(var0))
 
         self.assertAllCloseAccordingToType(
             np.array([
@@ -443,7 +451,7 @@ class RMSPropOptimizerTest(test.TestCase):
                 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
                 (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
                  (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)))
-            ]), var1.eval())
+            ]), self.evaluate(var1))
 
   def testCallableParams(self):
     with context.eager_mode():
diff --git a/tensorflow/python/training/server_lib_test.py b/tensorflow/python/training/server_lib_test.py
index cf995707fc..323e94c257 100644
--- a/tensorflow/python/training/server_lib_test.py
+++ b/tensorflow/python/training/server_lib_test.py
@@ -174,7 +174,7 @@ class GrpcServerTest(test.TestCase):
     # is not supported, but it should successfully ignore it.
     sess = session.InteractiveSession(server.target)
     c = constant_op.constant(42.0)
-    self.assertEqual(42.0, c.eval())
+    self.assertEqual(42.0, self.evaluate(c))
     sess.close()
 
   def testSetConfiguration(self):
diff --git a/tensorflow/python/training/slot_creator_test.py b/tensorflow/python/training/slot_creator_test.py
index 6d6364169f..382c15bb55 100644
--- a/tensorflow/python/training/slot_creator_test.py
+++ b/tensorflow/python/training/slot_creator_test.py
@@ -41,7 +41,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual("var/slot", slot.op.name)
       self.assertEqual([2], slot.get_shape().as_list())
       self.assertEqual(dtypes.float32, slot.dtype.base_dtype)
-      self.assertAllEqual([1.0, 2.5], slot.eval())
+      self.assertAllEqual([1.0, 2.5], self.evaluate(slot))
 
   def testCreateSlotFromTensor(self):
     with self.cached_session():
@@ -53,7 +53,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual("const/slot", slot.op.name)
       self.assertEqual([2], slot.get_shape().as_list())
       self.assertEqual(dtypes.float32, slot.dtype.base_dtype)
-      self.assertAllEqual([2.0, 5.0], slot.eval())
+      self.assertAllEqual([2.0, 5.0], self.evaluate(slot))
 
   def testCreateZerosSlotFromVariable(self):
     with self.cached_session():
@@ -67,7 +67,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual("var/slot", slot.op.name)
       self.assertEqual([2], slot.get_shape().as_list())
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
-      self.assertAllEqual([0.0, 0.0], slot.eval())
+      self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
   def testCreateZerosSlotFromDynamicShapedVariable(self):
     with self.cached_session():
@@ -88,7 +88,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual("var/slot", slot.op.name)
       self.assertEqual([2], array_ops.shape(slot).eval())
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
-      self.assertAllEqual([0.0, 0.0], slot.eval())
+      self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
   def testCreateZerosSlotFromTensor(self):
     with self.cached_session():
@@ -101,7 +101,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual("const/slot", slot.op.name)
       self.assertEqual([2], slot.get_shape().as_list())
       self.assertEqual(dtypes.float32, slot.dtype.base_dtype)
-      self.assertAllEqual([0.0, 0.0], slot.eval())
+      self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
   def testCreateZerosSlotFromDynamicShapedTensor(self):
     with self.cached_session():
@@ -116,7 +116,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual("const/slot", slot.op.name)
       self.assertEqual([2], array_ops.shape(slot).eval())
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
-      self.assertAllEqual([0.0, 0.0], slot.eval())
+      self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
   def testCreateSlotFromVariableRespectsScope(self):
     # See discussion on #2740.
diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py
index 7cd99d8680..b734e9653e 100644
--- a/tensorflow/python/training/supervisor_test.py
+++ b/tensorflow/python/training/supervisor_test.py
@@ -799,7 +799,7 @@ class SupervisorTest(test.TestCase):
       v = variables.VariableV1([10.10], name="foo")
       sav = saver_lib.Saver([v])
       sav.restore(sess, save_path)
-      self.assertEqual(1.0, v.eval()[0])
+      self.assertEqual(1.0, self.evaluate(v)[0])
 
   # Same as testStandardServicesNoGlobalStep but with a global step.
   # We should get a summary about the step time.
@@ -863,7 +863,7 @@ class SupervisorTest(test.TestCase):
       v = variables.VariableV1([-12], name="global_step")
       sav = saver_lib.Saver([v])
       sav.restore(sess, save_path)
-      self.assertEqual(123, v.eval()[0])
+      self.assertEqual(123, self.evaluate(v)[0])
 
   def testNoQueueRunners(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
diff --git a/tensorflow/python/training/training_ops_test.py b/tensorflow/python/training/training_ops_test.py
index 0216482825..929dd74ac6 100644
--- a/tensorflow/python/training/training_ops_test.py
+++ b/tensorflow/python/training/training_ops_test.py
@@ -53,9 +53,9 @@ class TrainingOpsTest(TensorFlowTestCase):
     with self.session(use_gpu=use_gpu):
       var = variables.VariableV1(x)
       variables.global_variables_initializer().run()
-      self.assertAllCloseAccordingToType(x, var.eval())
+      self.assertAllCloseAccordingToType(x, self.evaluate(var))
       apply_sgd = training_ops.apply_gradient_descent(var, alpha, delta)
-      out = apply_sgd.eval()
+      out = self.evaluate(apply_sgd)
       self.assertShapeEqual(out, apply_sgd)
       self.assertAllCloseAccordingToType(x - alpha * delta, out)
 
@@ -74,13 +74,13 @@ class TrainingOpsTest(TensorFlowTestCase):
       accum = variables.VariableV1(y)
       variables.global_variables_initializer().run()
 
-      self.assertAllCloseAccordingToType(x, var.eval())
+      self.assertAllCloseAccordingToType(x, self.evaluate(var))
       apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad)
-      out = apply_adagrad.eval()
+      out = self.evaluate(apply_adagrad)
       self.assertShapeEqual(out, apply_adagrad)
       self.assertAllCloseAccordingToType(x - lr * grad * (y + grad * grad)**
                                          (-0.5), out)
-      self.assertAllCloseAccordingToType(y + grad * grad, accum.eval())
+      self.assertAllCloseAccordingToType(y + grad * grad, self.evaluate(accum))
 
   def _testTypesForFtrl(self,
                         x,
@@ -99,10 +99,10 @@ class TrainingOpsTest(TensorFlowTestCase):
       linear = variables.VariableV1(z)
       variables.global_variables_initializer().run()
 
-      self.assertAllCloseAccordingToType(x, var.eval())
+      self.assertAllCloseAccordingToType(x, self.evaluate(var))
       apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2,
                                            lr_power)
-      out = apply_ftrl.eval()
+      out = self.evaluate(apply_ftrl)
       self.assertShapeEqual(out, apply_ftrl)
       accum_update = y + grad * grad
       linear_update = z + grad - (accum_update**(-lr_power) - y**
@@ -112,17 +112,19 @@ class TrainingOpsTest(TensorFlowTestCase):
           np.sign(linear_update[i]) * l1 - linear_update[i]) / (quadratic[i]) if
                                np.abs(linear_update[i]) > l1 else 0.0
                                for i in range(linear_update.size)])
-      self.assertAllCloseAccordingToType(accum_update, accum.eval())
+      self.assertAllCloseAccordingToType(accum_update, self.evaluate(accum))
       if x.dtype == np.float16:
         # The calculations here really are not very precise in float16.
-        self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2)
+        self.assertAllClose(
+            linear_update, self.evaluate(linear), rtol=2e-2, atol=2e-2)
         self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2)
       elif x.dtype == np.float32:
         # The calculations here not sufficiently precise in float32.
-        self.assertAllClose(linear_update, linear.eval(), rtol=1e-5, atol=1e-5)
+        self.assertAllClose(
+            linear_update, self.evaluate(linear), rtol=1e-5, atol=1e-5)
         self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5)
       else:
-        self.assertAllClose(linear_update, linear.eval())
+        self.assertAllClose(linear_update, self.evaluate(linear))
         self.assertAllClose(expected_out, out)
 
   def testApplyAdagrad(self):
@@ -152,19 +154,19 @@ class TrainingOpsTest(TensorFlowTestCase):
       accum = variables.VariableV1(y)
       variables.global_variables_initializer().run()
 
-      self.assertAllCloseAccordingToType(x, var.eval())
+      self.assertAllCloseAccordingToType(x, self.evaluate(var))
       sparse_apply_adagrad = training_ops.sparse_apply_adagrad(
           var, accum, lr, grad,
           constant_op.constant(indices, self._toType(indices.dtype)))
-      out = sparse_apply_adagrad.eval()
+      out = self.evaluate(sparse_apply_adagrad)
       self.assertShapeEqual(out, sparse_apply_adagrad)
 
       for (i, index) in enumerate(indices):
         self.assertAllCloseAccordingToType(
             x[index] - lr * grad[i] * (y[index] + grad[i] * grad[i])**(-0.5),
-            var.eval()[index])
+            self.evaluate(var)[index])
         self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i],
-                                           accum.eval()[index])
+                                           self.evaluate(accum)[index])
 
   def _testTypesForSparseFtrl(self,
                               x,
@@ -183,7 +185,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       linear = variables.VariableV1(z)
       variables.global_variables_initializer().run()
 
-      self.assertAllCloseAccordingToType(x, var.eval())
+      self.assertAllCloseAccordingToType(x, self.evaluate(var))
       sparse_apply_ftrl = training_ops.sparse_apply_ftrl(
           var,
           accum,
@@ -194,15 +196,16 @@ class TrainingOpsTest(TensorFlowTestCase):
           l1,
           l2,
           lr_power=lr_power)
-      out = sparse_apply_ftrl.eval()
+      out = self.evaluate(sparse_apply_ftrl)
       self.assertShapeEqual(out, sparse_apply_ftrl)
 
       for (i, index) in enumerate(indices):
-        self.assertAllCloseAccordingToType(x[index] - lr * grad[i] *
-                                           (y[index] + grad[i] * grad[i])**
-                                           (lr_power), var.eval()[index])
+        self.assertAllCloseAccordingToType(
+            x[index] - lr * grad[i] * (y[index] + grad[i] * grad[i])**
+            (lr_power),
+            self.evaluate(var)[index])
         self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i],
-                                           accum.eval()[index])
+                                           self.evaluate(accum)[index])
 
   def testSparseApplyAdagrad(self):
     for (dtype, index_type) in itertools.product(
@@ -276,13 +279,13 @@ class TrainingOpsTest(TensorFlowTestCase):
       epsilon_t = constant_op.constant(epsilon, self._toType(var.dtype), [])
       variables.global_variables_initializer().run()
 
-      self.assertAllCloseAccordingToType(var, var_t.eval())
+      self.assertAllCloseAccordingToType(var, self.evaluate(var_t))
       new_var, _, _ = self._adamUpdateNumpy(var, grad, t, m, v, lr, beta1,
                                             beta2, epsilon)
       apply_adam = training_ops.apply_adam(var_t, m_t, v_t, beta1_power_t,
                                            beta2_power_t, lr_t, beta1_t,
                                            beta2_t, epsilon_t, grad)
-      out = apply_adam.eval()
+      out = self.evaluate(apply_adam)
       self.assertShapeEqual(out, apply_adam)
       self.assertAllCloseAccordingToType(new_var, out)
 
diff --git a/tensorflow/python/util/tf_should_use_test.py b/tensorflow/python/util/tf_should_use_test.py
index fedbe1dff6..cde67c4e4f 100644
--- a/tensorflow/python/util/tf_should_use_test.py
+++ b/tensorflow/python/util/tf_should_use_test.py
@@ -111,7 +111,7 @@ class TfShouldUseTest(test.TestCase):
         # Creating another op and executing it does not mark the
         # unused op as being "used".
         v = constant_op.constant(1.0, name='meh')
-        v.eval()
+        self.evaluate(v)
     msg = '\n'.join(error.call_args[0])
     self.assertIn('Object was never used', msg)
     self.assertIn('blah3:0', msg)
-- 
GitLab


From 22ff24b6578097b23ad451e0e6c3e8dc0b02a3b6 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 16 Nov 2018 13:08:43 -0800
Subject: [PATCH 0430/1554] [tf.data] Enable nested dataset support in core
 `tf.data` transformations.

This allows (e.g.) using `tf.data.Dataset.window()` (which produces nested datasets) and composing it with `tf.data.Dataset.map(f)`, where `f` is a function that takes a `Dataset` as argument, and can now return a `Dataset`.

PiperOrigin-RevId: 221840464
---
 .../python/data/experimental/ops/grouping.py  |  8 ++----
 .../data/kernel_tests/map_dataset_op_test.py  | 14 +++++-----
 tensorflow/python/data/ops/dataset_ops.py     | 26 +------------------
 3 files changed, 11 insertions(+), 37 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index 026867d405..80ca7104d8 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -454,8 +454,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
         self._transformation_name(),
         input_classes=(ops.Tensor, nested_dataset),
         input_shapes=(tensor_shape.scalar(), nested_dataset),
-        input_types=(dtypes.int64, nested_dataset),
-        experimental_nested_dataset_support=True)
+        input_types=(dtypes.int64, nested_dataset))
     if not isinstance(
         wrapped_func.output_classes, dataset_ops._NestedDatasetComponent):  # pylint: disable=protected-access
       raise TypeError("`reduce_func` must return a `Dataset` object.")
@@ -528,10 +527,7 @@ class _MapXDataset(dataset_ops.UnaryDataset):
     self._input_dataset = input_dataset
 
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
-        map_func,
-        self._transformation_name(),
-        dataset=input_dataset,
-        experimental_nested_dataset_support=True)
+        map_func, self._transformation_name(), dataset=input_dataset)
     self._output_classes = wrapped_func.output_classes
     self._output_shapes = wrapped_func.output_shapes
     self._output_types = wrapped_func.output_types
diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
index 81ef7d16be..187b9da14c 100644
--- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
@@ -901,12 +901,14 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         break
     self.assertTrue(found_warning)
 
-  def testNestedDatasetError(self):
-    dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0])
-    with self.assertRaisesRegexp(
-        NotImplementedError, r"The Dataset.map\(\) transformation does not "
-        "currently support nested datasets as outputs."):
-      _ = dataset.map(dataset_ops.Dataset.from_tensor_slices)
+  def testNestedDatasetMap(self):
+    # TODO(b/110122868): When iterators can yield a `tf.data.Dataset`, remove
+    # the `get_single_element()` call.
+    dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0]).map(
+        dataset_ops.Dataset.from_tensor_slices).map(
+            lambda ds: ds.batch(3)).flat_map(lambda x: x)
+
+    self.assertDatasetProduces(dataset, [[1.0, 2.0, 3.0]])
 
   def testReturnValueError(self):
     dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0])
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index cf51fdffdd..4a11619112 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1876,11 +1876,6 @@ class _NestedDatasetComponent(object):
   corresponding position in the `output_classes`, `output_shapes`, and
   `output_types` properties.
 
-  NOTE(mrry): This class is not currently exposed via the public API. Support
-  for nested datasets can be enabled on a function-by-function basis by setting
-  `experimental_nested_dataset_support=True` in the `StructuredFunctionWrapper`
-  initializer.
-
   TODO(b/110122868): Add this class, or something equivalent, to the public API.
   We are considering revising the public API for accessing Dataset structure
   (`output_classes` etc.) based on experience with nested datasets and other
@@ -1963,7 +1958,6 @@ class StructuredFunctionWrapper(object):
                input_shapes=None,
                input_types=None,
                add_to_graph=True,
-               experimental_nested_dataset_support=False,
                defun_kwargs=None):
     """Creates a new `StructuredFunctionWrapper` for the given function.
 
@@ -1983,8 +1977,6 @@ class StructuredFunctionWrapper(object):
         argument defines the element types and structure for `func` arguments.
       add_to_graph: (Optional.) If `True`, the function will be added to the
         default graph.
-      experimental_nested_dataset_support: (Optional.) If `True`, the function
-        will support `tf.data.Dataset` objects as arguments and return values.
       defun_kwargs: (Optional.) A dictionary mapping string argument names to
         values. If supplied, will be passed to `function.Defun()` as keyword
         arguments.
@@ -2019,9 +2011,6 @@ class StructuredFunctionWrapper(object):
 
     ])
 
-    # TODO(b/110122868): Enable this support for all `tf.data` functions.
-    self._nested_dataset_support = experimental_nested_dataset_support
-
     if defun_kwargs is None:
       defun_kwargs = {}
 
@@ -2043,7 +2032,6 @@ class StructuredFunctionWrapper(object):
           arg.indices.set_shape([None, arg_shape.ndims])
           arg.dense_shape.set_shape([arg_shape.ndims])
         elif isinstance(arg_class, _NestedDatasetComponent):
-          assert self._nested_dataset_support
           arg = _VariantDataset(arg, arg_class)
         else:
           arg.set_shape(arg_shape)
@@ -2082,11 +2070,6 @@ class StructuredFunctionWrapper(object):
           flat_shapes.append(t.get_shape())
           flat_types.append(t.dtype)
         elif isinstance(t, DatasetV2):
-          if not self._nested_dataset_support:
-            raise NotImplementedError(
-                "The %s transformation does not currently support nested "
-                "datasets as outputs." % self._transformation_name)
-
           flat_ret.append(t._as_variant_tensor())  # pylint: disable=protected-access
           component = _NestedDatasetComponent(t)
           flat_classes.append(component)
@@ -2134,10 +2117,6 @@ class StructuredFunctionWrapper(object):
       if input_class is sparse_tensor_lib.SparseTensor:
         ret.append(dtypes.variant)
       elif isinstance(input_class, _NestedDatasetComponent):
-        if not self._nested_dataset_support:
-          raise NotImplementedError(
-              "The %s transformation does not currently support nested "
-              "datasets as inputs." % self._transformation_name)
         ret.append(dtypes.variant)
       else:
         assert isinstance(input_type, dtypes.DType)
@@ -2922,10 +2901,7 @@ class FlatMapDataset(UnaryDataset):
     self._input_dataset = input_dataset
 
     wrapped_func = StructuredFunctionWrapper(
-        map_func,
-        self._transformation_name(),
-        dataset=input_dataset,
-        experimental_nested_dataset_support=True)
+        map_func, self._transformation_name(), dataset=input_dataset)
     if not isinstance(wrapped_func.output_classes, _NestedDatasetComponent):
       raise TypeError("`map_func` must return a `Dataset` object.")
     self._output_classes = wrapped_func.output_classes.output_classes
-- 
GitLab


From 9abb61a4db2697a841ba1e27a1b0b09bd952ac15 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 16 Nov 2018 13:12:09 -0800
Subject: [PATCH 0431/1554] Distribution Strategies: Use newly created
 `InputIterator` in the make_input_fn_iterator API.

PiperOrigin-RevId: 221840949
---
 .../python/collective_all_reduce_strategy.py  |   5 +-
 .../collective_all_reduce_strategy_test.py    | 105 +++++++++++++-----
 .../distribute/python/mirrored_strategy.py    |  29 ++---
 .../python/mirrored_strategy_multigpu_test.py |  40 +++++++
 .../python/mirrored_strategy_test.py          |  22 ----
 .../distribute/python/one_device_strategy.py  |   6 +-
 .../python/one_device_strategy_test.py        |  10 +-
 .../python/parameter_server_strategy.py       |   7 +-
 .../python/parameter_server_strategy_test.py  |  92 +++++++++++----
 .../distribute/python/strategy_test_lib.py    |  34 +++++-
 .../contrib/distribute/python/values_test.py  |  20 ++--
 tensorflow/python/distribute/values.py        |  19 +++-
 tensorflow/python/training/distribute.py      |  12 +-
 13 files changed, 275 insertions(+), 126 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 2c9c901ece..f13cf26d36 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -234,8 +234,9 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
         num_input_pipelines=self._num_workers,
         input_pipeline_id=input_pipeline_id,
         num_replicas_in_sync=self._num_replicas_in_sync)
-    return values.PerReplicaDataset(
-        self._call_dataset_fn(input_fn, input_context), self._devices, True)
+
+    return values.InputFunctionIterator(
+        input_fn, [(self._default_device, self._devices)], [input_context])
 
   def _configure(self,
                  session_config=None,
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 27b9306c85..a47eef94e9 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -27,11 +27,14 @@ from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import cross_device_utils
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
@@ -242,9 +245,42 @@ class CollectiveAllReduceStrategyTestBase(
                                                        reduced_x_value)))
     return np.allclose(x_value, reduced_x_value, atol=1e-5)
 
+  def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn,
+                              expected_values):
+    distribution, master_target, config = self._get_test_object(
+        task_type, task_id, num_gpus)
+    devices = distribution.worker_devices
+
+    with ops.Graph().as_default(), \
+         self.cached_session(config=config,
+                             target=master_target) as sess:
+      iterator = distribution.make_input_fn_iterator(input_fn)
+      sess.run(iterator.initialize())
+
+      for expected_value in expected_values:
+        next_element = iterator.get_next()
+        computed_value = sess.run(
+            [values.select_device(d, next_element) for d in devices])
+        self.assertEqual(expected_value, computed_value)
+
+      with self.assertRaises(errors.OutOfRangeError):
+        next_element = iterator.get_next()
+        sess.run([values.select_device(d, next_element) for d in devices])
+
+      # After re-initializing the iterator, should be able to iterate again.
+      sess.run(iterator.initialize())
+
+      for expected_value in expected_values:
+        next_element = iterator.get_next()
+        computed_value = sess.run(
+            [values.select_device(d, next_element) for d in devices])
+        self.assertEqual(expected_value, computed_value)
+
 
 class DistributedCollectiveAllReduceStrategyTest(
-    CollectiveAllReduceStrategyTestBase, parameterized.TestCase):
+    CollectiveAllReduceStrategyTestBase,
+    strategy_test_lib.DistributionTestBase,
+    parameterized.TestCase):
 
   @classmethod
   def setUpClass(cls):
@@ -272,7 +308,7 @@ class DistributedCollectiveAllReduceStrategyTest(
       combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
   def testVariableInitialization(self, num_gpus):
     if context.num_gpus() < num_gpus:
-      return
+      self.skipTest('Not enough GPUs')
     self._run_between_graph_clients(
         self._test_variable_initialization,
         self._cluster_spec,
@@ -282,10 +318,30 @@ class DistributedCollectiveAllReduceStrategyTest(
       combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
   def testComplexModel(self, num_gpus):
     if context.num_gpus() < num_gpus:
-      return
+      self.skipTest('Not enough GPUs')
     self._run_between_graph_clients(
         self._test_complex_model, self._cluster_spec, num_gpus=num_gpus)
 
+  # TODO(yuefengz): Update how we use num_gpus and required_gpus
+  @combinations.generate(
+      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
+  def testMakeInputFnIterator(self, num_gpus):
+    if context.num_gpus() < num_gpus:
+      self.skipTest('Not enough GPUs')
+    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    # We use CPU as the device when num_gpus = 0
+    devices_per_worker = max(1, num_gpus)
+    expected_values = [[i+j for j in range(devices_per_worker)]
+                       for i in range(0, 100, devices_per_worker)]
+
+    input_fn = self._input_fn_to_test_input_context(
+        dataset_fn,
+        expected_num_replicas_in_sync=3*devices_per_worker,
+        expected_num_input_pipelines=3,
+        expected_input_pipeline_id=1)  # because task_id = 1
+    self._test_input_fn_iterator('worker', 1, num_gpus,
+                                 input_fn, expected_values)
+
 
 class DistributedCollectiveAllReduceStrategyTestWithChief(
     CollectiveAllReduceStrategyTestBase, parameterized.TestCase):
@@ -326,44 +382,35 @@ class DistributedCollectiveAllReduceStrategyTestWithChief(
 
 
 class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase,
+                                       strategy_test_lib.DistributionTestBase,
                                        parameterized.TestCase):
 
   def testMinimizeLossGraph(self, num_gpus=2):
     # Collective ops doesn't support strategy with one device.
     if context.num_gpus() < num_gpus:
-      return
+      self.skipTest('Not enough GPUs')
     self._test_minimize_loss_graph(None, None, num_gpus)
 
   def testComplexModel(self, num_gpus=2):
     # Collective ops doesn't support strategy with one device.
     if context.num_gpus() < num_gpus:
-      return
+      self.skipTest('Not enough GPUs')
     self._test_complex_model(None, None, num_gpus)
 
-
-class InputContextTest(strategy_test_lib.DistributionTestBase):
-
-  def testInputContextPropertyLocal(self):
-    d = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
-        num_gpus_per_worker=2)
-    with context.graph_mode():
-      input_fn = self._input_fn_to_test_input_context(
-          expected_num_replicas_in_sync=2,
-          expected_num_input_pipelines=1,
-          expected_input_pipeline_id=0)
-      d.make_input_fn_iterator(input_fn)
-
-  def testInputContextPropertyMultiWorker(self):
-    d = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
-        num_gpus_per_worker=2)
-    cluster_spec = {'worker': ['worker1', 'worker2', 'worker3'], 'ps': ['ps1']}
-    d.configure(cluster_spec=cluster_spec, task_type='worker', task_id=1)
-    with context.graph_mode():
-      input_fn = self._input_fn_to_test_input_context(
-          expected_num_replicas_in_sync=6,
-          expected_num_input_pipelines=3,
-          expected_input_pipeline_id=1)  # because task_id = 1
-      d.make_input_fn_iterator(input_fn)
+  def testMakeInputFnIterator(self, num_gpus=2):
+    # Collective ops doesn't support strategy with one device.
+    if context.num_gpus() < num_gpus:
+      self.skipTest('Not enough GPUs')
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+
+    input_fn = self._input_fn_to_test_input_context(
+        dataset_fn,
+        expected_num_replicas_in_sync=num_gpus,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)
+    self._test_input_fn_iterator(None, None, num_gpus,
+                                 input_fn, expected_values)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index f7d209cc3d..6a4fb07bb3 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -503,25 +503,20 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
       self,
       input_fn,
       replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    input_contexts = []
     if self._cluster_spec:
-      input_fns = []
-      for i in range(len(self._worker_devices)):
-        input_context = distribute_lib.InputContext(
-            num_input_pipelines=len(self._worker_devices),
-            input_pipeline_id=i,
-            num_replicas_in_sync=self._num_replicas_in_sync)
-        input_fns.append(
-            partial(self._call_dataset_fn, input_fn, input_context))
-
-      return values.MultiWorkerDataset(input_fns, self._worker_devices,
-                                       self._auto_shard_dataset)
+      num_workers = len(self._worker_devices)
+      worker_device_pairs = self._worker_devices
     else:
-      input_context = distribute_lib.InputContext(
-          num_input_pipelines=1,
-          input_pipeline_id=0,
-          num_replicas_in_sync=self._num_replicas_in_sync)
-      return values.PerReplicaDataset(
-          self._call_dataset_fn(input_fn, input_context), self._devices)
+      num_workers = 1
+      worker_device_pairs = [("/job:localhost", self._devices)]
+    for i in range(num_workers):
+      input_contexts.append(distribute_lib.InputContext(
+          num_input_pipelines=num_workers,
+          input_pipeline_id=i,
+          num_replicas_in_sync=self._num_replicas_in_sync))
+    return values.InputFunctionIterator(
+        input_fn, worker_device_pairs, input_contexts)
 
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 4977946afe..76fdc6f762 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -146,6 +146,22 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
       self.assertEqual(2, len(unwrapped))
       self.assertEqual(1.0, self.evaluate(unwrapped[0]))
 
+  @test_util.run_in_graph_and_eager_modes
+  def testMakeInputFnIterator(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    d = self._get_distribution_strategy()
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+
+    input_fn = self._input_fn_to_test_input_context(
+        dataset_fn,
+        expected_num_replicas_in_sync=2,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)
+    iterator = d.make_input_fn_iterator(input_fn)
+    self._test_input_fn_iterator(iterator, d.worker_devices, expected_values)
+
 
 class MirroredStrategyVariableCreationTest(test.TestCase):
 
@@ -1410,6 +1426,30 @@ class MultiWorkerMirroredStrategyTest(
     self._test_minimize_loss_graph(self._get_distribution_strategy(),
                                    learning_rate=0.05)
 
+  def testMakeInputFnIterator(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+
+    d = self._get_distribution_strategy()
+    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    num_gpus = context.num_gpus()
+    num_workers = 2
+
+    expected_values = [[i+j for j in range(num_gpus)] * num_workers
+                       for i in range(0, 100, num_gpus)]
+
+    with context.graph_mode(), self.cached_session() as sess:
+      # `expected_input_pipeline_id` is None because the input_fn will be called
+      # multiple times, each with a different input_pipeline_id.
+      input_fn = self._input_fn_to_test_input_context(
+          dataset_fn,
+          expected_num_replicas_in_sync=num_workers*num_gpus,
+          expected_num_input_pipelines=num_workers,
+          expected_input_pipeline_id=None)
+      iterator = d.make_input_fn_iterator(input_fn)
+      self._test_input_fn_iterator(
+          iterator, d.worker_devices, expected_values, sess)
+
 
 class MultiWorkerMirroredStrategyTestWithChief(
     multi_worker_test_base.MultiWorkerTestBase,
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
index 0886d0ef34..2809ecb5d0 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -47,28 +47,6 @@ class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
   def testCallAndMergeExceptions(self):
     self._test_call_and_merge_exceptions(self._get_distribution_strategy())
 
-  @test_util.run_in_graph_and_eager_modes
-  def testInputContextPropertyLocal(self):
-    d = mirrored_strategy.MirroredStrategy(num_gpus_per_worker=2)
-    input_fn = self._input_fn_to_test_input_context(
-        expected_num_replicas_in_sync=2,
-        expected_num_input_pipelines=1,
-        expected_input_pipeline_id=0)
-    d.make_input_fn_iterator(input_fn)
-
-  def testInputContextPropertyMultiWorker(self):
-    d = mirrored_strategy.MirroredStrategy(num_gpus_per_worker=2)
-    cluster_spec = {"worker": ["worker1", "worker2", "worker3"]}
-    d.configure(cluster_spec=cluster_spec)
-    with context.graph_mode():
-      # `expected_input_pipeline_id` is None because the input_fn will be called
-      # multiple times, each with a different input_pipeline_id.
-      input_fn = self._input_fn_to_test_input_context(
-          expected_num_replicas_in_sync=6,
-          expected_num_input_pipelines=3,
-          expected_input_pipeline_id=None)
-      d.make_input_fn_iterator(input_fn)
-
 
 class VariableCreatorStackTest(test.TestCase):
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index b1dda84d4e..2ef60bf912 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -79,9 +79,9 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
       self,
       input_fn,
       replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    return values.PerReplicaDataset(
-        self._call_dataset_fn(input_fn, distribute_lib.InputContext()),
-        [self._device])
+    return values.InputFunctionIterator(
+        input_fn, [("/job:localhost", [self._device])],
+        [distribute_lib.InputContext()])
 
   def _broadcast_to(self, tensor, destinations):
     del destinations
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index cf6d890390..b0a2ba3415 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.contrib.distribute.python import one_device_strategy
 from tensorflow.contrib.distribute.python import strategy_test_lib
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import test
 from tensorflow.python.framework import test_util
 
@@ -43,13 +44,18 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
     self._test_call_and_merge_exceptions(self._get_distribution_strategy())
 
   @test_util.run_in_graph_and_eager_modes
-  def testInputContextPropertyLocal(self):
+  def testMakeInputFnIterator(self):
     d = one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
+    expected_values = [[i] for i in range(10)]
     input_fn = self._input_fn_to_test_input_context(
+        dataset_fn,
         expected_num_replicas_in_sync=1,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
-    d.make_input_fn_iterator(input_fn)
+    iterator = d.make_input_fn_iterator(input_fn)
+    self._test_input_fn_iterator(
+        iterator, d.worker_devices, expected_values)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 86d1d56a82..4419d4afe1 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -197,6 +197,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
 
   def _initialize_local(self, num_gpus_per_worker):
     """Initialize internal devices for local training."""
+    self._worker_device = "/job:localhost"
     # Define compute devices which is a list of device strings and one for each
     # replica. When there are GPUs, replicate operations on these GPUs.
     # Otherwise, place operations on CPU.
@@ -251,9 +252,9 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
         num_input_pipelines=num_input_pipelines,
         input_pipeline_id=input_pipeline_id,
         num_replicas_in_sync=self._num_replicas_in_sync)
-    return values.PerReplicaDataset(
-        self._call_dataset_fn(input_fn, input_context), self._compute_devices,
-        True)
+    worker_device_pairs = [(self._worker_device, self._compute_devices)]
+    return values.InputFunctionIterator(
+        input_fn, worker_device_pairs, [input_context])
 
   def _broadcast_to(self, tensor, destinations):
     if not cross_device_ops_lib.check_destinations(destinations):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index dc84d1788f..5d5e65600c 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import parameter_server_strategy
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -34,6 +35,7 @@ from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.layers import core
@@ -516,8 +518,40 @@ class ParameterServerStrategyTestBase(
       self.assertLess(error_after, error_before)
       return error_after < error_before
 
+  def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn,
+                              expected_values):
+    distribution, master_target, config = self._get_test_objects(
+        task_type, task_id, num_gpus)
+    devices = distribution.worker_devices
+
+    with ops.Graph().as_default(), \
+         self.cached_session(config=config,
+                             target=master_target) as sess:
+      iterator = distribution.make_input_fn_iterator(input_fn)
+      sess.run(iterator.initialize())
+
+      for expected_value in expected_values:
+        next_element = iterator.get_next()
+        computed_value = sess.run(
+            [values.select_device(d, next_element) for d in devices])
+        self.assertEqual(expected_value, computed_value)
+
+      with self.assertRaises(errors.OutOfRangeError):
+        next_element = iterator.get_next()
+        sess.run([values.select_device(d, next_element) for d in devices])
+
+      # After re-initializing the iterator, should be able to iterate again.
+      sess.run(iterator.initialize())
+
+      for expected_value in expected_values:
+        next_element = iterator.get_next()
+        computed_value = sess.run(
+            [values.select_device(d, next_element) for d in devices])
+        self.assertEqual(expected_value, computed_value)
+
 
 class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
+                                  strategy_test_lib.DistributionTestBase,
                                   parameterized.TestCase):
 
   @classmethod
@@ -582,6 +616,41 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
   def testMinimizeLossGraphLocal(self, num_gpus):
     self._test_minimize_loss_graph(None, None, num_gpus)
 
+  # TODO(priyag): Refactor this and other multi worker tests.
+  @combinations.generate(
+      combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1))
+  def testMakeInputFnIteratorDistributed(self, num_gpus):
+    if context.num_gpus() < num_gpus:
+      self.skipTest('Not enough GPUs')
+    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    expected_values = [[i+j for j in range(num_gpus)]
+                       for i in range(0, 100, num_gpus)]
+
+    input_fn = self._input_fn_to_test_input_context(
+        dataset_fn,
+        expected_num_replicas_in_sync=num_gpus,
+        expected_num_input_pipelines=3,
+        expected_input_pipeline_id=1)  # because task_id = 1
+    self._test_input_fn_iterator('worker', 1, num_gpus,
+                                 input_fn, expected_values)
+
+  @combinations.generate(
+      combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1))
+  def testMakeInputFnIteratorLocal(self, num_gpus):
+    if context.num_gpus() < num_gpus:
+      self.skipTest('Not enough GPUs')
+    dataset_fn = lambda: dataset_ops.Dataset.range(100)
+    expected_values = [[i+j for j in range(num_gpus)]
+                       for i in range(0, 100, num_gpus)]
+
+    input_fn = self._input_fn_to_test_input_context(
+        dataset_fn,
+        expected_num_replicas_in_sync=num_gpus,
+        expected_num_input_pipelines=1,
+        expected_input_pipeline_id=0)  # only one worker and pipeline for local.
+    self._test_input_fn_iterator(None, None, num_gpus,
+                                 input_fn, expected_values)
+
 
 class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                                            parameterized.TestCase):
@@ -629,28 +698,5 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
       distribution.call_for_each_replica(f)
 
 
-class InputContextTest(strategy_test_lib.DistributionTestBase):
-
-  def testInputContextPropertyLocal(self):
-    d = parameter_server_strategy.ParameterServerStrategy(num_gpus_per_worker=2)
-    with context.graph_mode():
-      input_fn = self._input_fn_to_test_input_context(
-          expected_num_replicas_in_sync=2,
-          expected_num_input_pipelines=1,
-          expected_input_pipeline_id=0)
-      d.make_input_fn_iterator(input_fn)
-
-  def testInputContextPropertyMultiWorker(self):
-    d = parameter_server_strategy.ParameterServerStrategy(num_gpus_per_worker=2)
-    cluster_spec = {'worker': ['worker1', 'worker2', 'worker3'], 'ps': ['ps1']}
-    d.configure(cluster_spec=cluster_spec, task_type='worker', task_id=1)
-    with context.graph_mode():
-      input_fn = self._input_fn_to_test_input_context(
-          expected_num_replicas_in_sync=2,
-          expected_num_input_pipelines=3,
-          expected_input_pipeline_id=1)  # because task_id =1
-      d.make_input_fn_iterator(input_fn)
-
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 7c6f7123da..b9e3761738 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -19,12 +19,13 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import values
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
@@ -209,7 +210,9 @@ class DistributionTestBase(test.TestCase):
       with self.assertRaises(_TestException):
         dist.call_for_each_replica(_merge_call_merge_raises_fn)
 
-  def _input_fn_to_test_input_context(self, expected_num_replicas_in_sync,
+  def _input_fn_to_test_input_context(self,
+                                      dataset_fn,
+                                      expected_num_replicas_in_sync,
                                       expected_num_input_pipelines,
                                       expected_input_pipeline_id):
     # Use a list of one element as counter so that it can be captured by the
@@ -232,6 +235,31 @@ class DistributionTestBase(test.TestCase):
         self.assertEqual(worker_id_counter[0], input_context.input_pipeline_id)
         worker_id_counter[0] += 1
 
-      return dataset_ops.Dataset.from_tensors([[1.]]).repeat()
+      return dataset_fn()
 
     return _input_fn
+
+  def _test_input_fn_iterator(self, iterator, devices, expected_values,
+                              sess=None):
+    evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
+    evaluate(iterator.initialize())
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      computed_value = evaluate(
+          [values.select_device(d, next_element) for d in devices])
+      self.assertEqual(expected_value, computed_value)
+
+    with self.assertRaises(errors.OutOfRangeError):
+      next_element = iterator.get_next()
+      evaluate([values.select_device(d, next_element) for d in devices])
+
+    # After re-initializing the iterator, should be able to iterate again.
+    evaluate(iterator.initialize())
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      computed_value = evaluate(
+          [values.select_device(d, next_element) for d in devices])
+      self.assertEqual(expected_value, computed_value)
+
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 62cef18ad7..9d51129cae 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -38,6 +38,7 @@ from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.training import device_util
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.util import nest
 
@@ -573,7 +574,10 @@ class InputFunctionIteratorTestBase(test.TestCase):
   def _test_iterator(self, input_fn, worker_device_pairs, expected_values,
                      sess=None):
     devices = nest.flatten([ds for _, ds in worker_device_pairs])
-    iterator = values.InputFunctionIterator(input_fn, worker_device_pairs)
+    input_contexts = [
+        distribute_lib.InputContext() for _ in worker_device_pairs]
+    iterator = values.InputFunctionIterator(
+        input_fn, worker_device_pairs, input_contexts)
 
     evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
 
@@ -604,7 +608,7 @@ class InputFunctionIteratorSingleWorkerTest(InputFunctionIteratorTestBase):
   @test_util.run_in_graph_and_eager_modes
   def testOneDeviceCPU(self):
     worker_device_pairs = [("", ["/device:CPU:0"])]
-    input_fn = lambda: dataset_ops.Dataset.range(10)
+    input_fn = lambda _: dataset_ops.Dataset.range(10)
 
     expected_values = [[i] for i in range(10)]
 
@@ -616,7 +620,7 @@ class InputFunctionIteratorSingleWorkerTest(InputFunctionIteratorTestBase):
       self.skipTest("A GPU is not available for this test.")
 
     worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    input_fn = lambda: dataset_ops.Dataset.range(10)
+    input_fn = lambda _: dataset_ops.Dataset.range(10)
 
     expected_values = [[i, i+1] for i in range(0, 10, 2)]
 
@@ -628,7 +632,7 @@ class InputFunctionIteratorSingleWorkerTest(InputFunctionIteratorTestBase):
       self.skipTest("A GPU is not available for this test.")
 
     worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    def input_fn():
+    def input_fn(_):
       dataset1 = dataset_ops.Dataset.range(10)
       dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
       return dataset_ops.Dataset.zip((dataset1, dataset2))
@@ -643,7 +647,7 @@ class InputFunctionIteratorSingleWorkerTest(InputFunctionIteratorTestBase):
       self.skipTest("A GPU is not available for this test.")
 
     worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    input_fn = lambda: dataset_ops.Dataset.range(11)
+    input_fn = lambda _: dataset_ops.Dataset.range(11)
 
     expected_values = [[i, i+1] for i in range(0, 10, 2)]
     self._test_iterator(input_fn, worker_device_pairs, expected_values)
@@ -675,7 +679,7 @@ class InputFunctionIteratorMultiWorkerTest(
   def testOneDevicePerWorker(self):
     worker_devices = self._cpu_devices()
     with context.graph_mode(), self.cached_session() as sess:
-      input_fn = lambda: dataset_ops.Dataset.range(4)
+      input_fn = lambda _: dataset_ops.Dataset.range(4)
       self._test_iterator(input_fn, worker_devices,
                           [[0, 0], [1, 1], [2, 2], [3, 3]], sess)
 
@@ -684,14 +688,14 @@ class InputFunctionIteratorMultiWorkerTest(
       self.skipTest("A GPU is not available for this test.")
     worker_devices = self._cpu_and_one_gpu_devices()
     with context.graph_mode(), self.cached_session() as sess:
-      input_fn = lambda: dataset_ops.Dataset.range(4)
+      input_fn = lambda _: dataset_ops.Dataset.range(4)
       self._test_iterator(input_fn, worker_devices,
                           [[0, 1, 0, 1], [2, 3, 2, 3]], sess)
 
   def testTupleDataset(self):
     worker_devices = self._cpu_devices()
     with context.graph_mode(), self.cached_session() as sess:
-      def input_fn():
+      def input_fn(_):
         dataset1 = dataset_ops.Dataset.range(4)
         dataset2 = dataset_ops.Dataset.range(4).map(lambda x: x**2)
         return dataset_ops.Dataset.zip((dataset1, dataset2))
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 70293aa3be..df7e4083bb 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -1330,13 +1330,12 @@ class InputIterator(object):
 class InputFunctionIterator(InputIterator):
   """Iterator created from input function."""
 
-  def __init__(self, input_fn, worker_device_pairs):
+  def __init__(self, input_fn, worker_device_pairs, input_contexts):
     """Make an iterator for input provided via an input function.
 
     Currently implements PER_WORKER mode, in which the `input_fn` is called
     once on each worker.
 
-    TODO(priyag): Integrate with `InputContext` when it is submitted.
     TODO(priyag): Add other replication modes.
     TODO(priyag): Allow taking input function that returns a callable that
     returns nest of tensors.
@@ -1345,20 +1344,30 @@ class InputFunctionIterator(InputIterator):
       input_fn: Input function that returns a `tf.data.Dataset` object.
       worker_device_pairs: A list of (worker, list of devices on that worker)
         pairs.
+      input_contexts: A list of `InputContext` instances to be passed to call(s)
+        to `input_fn`. Length and order should match worker order in
+        `worker_device_pairs`.
     """
     if not worker_device_pairs:
       raise ValueError("Cannot create iterator when no devices given.")
 
+    if len(worker_device_pairs) != len(input_contexts):
+      raise ValueError(
+          "Number of worker_device_pairs (%d) is not same as number of"
+          "input_contexts (%d)" % (
+              len(worker_device_pairs), len(input_contexts)))
+
     self._worker_device_pairs = worker_device_pairs
     self._is_eager = context.executing_eagerly()
     self._iterators = []
 
-    for worker, worker_devices in worker_device_pairs:
+    for (worker, devices), ctx in zip(worker_device_pairs, input_contexts):
+      # TODO(priyag): We should probably explicitly specify CPU device on worker.
       with ops.device(worker):
-        result = input_fn()
+        result = input_fn(ctx)
         if not isinstance(result, dataset_ops.Dataset):
           raise ValueError("input_fn must return a tf.data.Dataset.")
-        iterator = _DatasetIterator(result, worker, worker_devices)
+        iterator = _DatasetIterator(result, worker, devices)
         self._iterators.append(iterator)
 
   def get_next(self, name=None):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index b68d62a0a5..325a13ca4f 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -983,15 +983,9 @@ class DistributionStrategyExtended(object):
     _require_distribution_strategy_scope_extended(self)
     return variable_scope.variable_creator_scope(create_colocated_variable)
 
-  def _call_dataset_fn(self, dataset_fn, input_context=None):
+  def _call_dataset_fn(self, dataset_fn):
     """Call the `dataset_fn` with `input_context` as argument."""
-    # This method is invoked by both `make_input_fn_iterator` and
-    # `distribute_dataset`. The `dataset_fn` for the former one accepts an
-    # input_context while the latter one doesn't.
-    if input_context:
-      result = dataset_fn(input_context)
-    else:
-      result = dataset_fn()
+    result = dataset_fn()
     if not isinstance(result, dataset_ops.Dataset):
       raise ValueError(
           "dataset_fn() must return a tf.data.Dataset when using a "
@@ -1516,7 +1510,7 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def _make_input_fn_iterator(self,
                               input_fn,
                               replication_mode=InputReplicationMode.PER_WORKER):
-    return self._call_dataset_fn(input_fn, InputContext())
+    return input_fn(InputContext()).make_initializable_iterator()
 
   def _broadcast_to(self, tensor, destinations):
     if destinations is None:
-- 
GitLab


From fd73f953a5baa3ac440845e519bb4a33510dee2c Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Fri, 16 Nov 2018 13:14:09 -0800
Subject: [PATCH 0432/1554] Add missing build dependencies

PiperOrigin-RevId: 221841237
---
 tensorflow/compiler/xla/tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index db34d34f96..42f44513a5 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -135,6 +135,7 @@ cc_library(
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/core:test",
         "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
-- 
GitLab


From 290dd72d2e0ee11da5c6d212d2c92235f699c7d4 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 16 Nov 2018 13:22:34 -0800
Subject: [PATCH 0433/1554] Utility to check whether an op is stateful.

PiperOrigin-RevId: 221842425
---
 tensorflow/c/c_api_experimental.cc      | 10 ++++++++++
 tensorflow/c/c_api_experimental.h       |  5 +++++
 tensorflow/c/c_api_experimental_test.cc | 11 +++++++++++
 3 files changed, 26 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index d84bdcae58..f160f204de 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -8802,6 +8802,16 @@ const char* TF_GetNumberAttrForOpListInput(const char* op_name, int input_index,
   return input_arg.number_attr().c_str();
 }
 
+int TF_OpIsStateful(const char* op_type, TF_Status* status) {
+  const tensorflow::OpRegistrationData* op_reg_data;
+  status->status =
+      tensorflow::OpRegistry::Global()->LookUp(op_type, &op_reg_data);
+  if (!status->status.ok()) {
+    return 0;
+  }
+  return op_reg_data->op_def.is_stateful();
+}
+
 void TF_InitMain(const char* usage, int* argc, char*** argv) {
   tensorflow::port::InitMain(usage, argc, argv);
 }
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index f1dedbdb49..25c03df518 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -209,6 +209,11 @@ TF_CAPI_EXPORT extern void TF_AttrBuilderCheckCanRunOnDevice(
 TF_CAPI_EXPORT extern const char* TF_GetNumberAttrForOpListInput(
     const char* op_name, int input_index, TF_Status* status);
 
+// Returns 1 if the op is stateful, 0 otherwise. The return value is undefined
+// if the status is not ok.
+TF_CAPI_EXPORT extern int TF_OpIsStateful(const char* op_type,
+                                          TF_Status* status);
+
 // Platform specific initialization routine. Very few platforms actually require
 // this to be called.
 TF_CAPI_EXPORT void TF_InitMain(const char* usage, int* argc, char*** argv);
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index c6effd3969..881dbaf35a 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -162,5 +162,16 @@ protocol: "grpc"
   TF_DeleteStatus(status);
 }
 
+TEST(CAPI_EXPERIMENTAL, IsStateful) {
+  std::unique_ptr<TF_Status, decltype(&TF_DeleteStatus)> status(
+      TF_NewStatus(), TF_DeleteStatus);
+  int assign = TF_OpIsStateful("AssignAddVariableOp", status.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+  EXPECT_EQ(assign, 1);
+  int id = TF_OpIsStateful("Identity", status.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+  EXPECT_EQ(id, 0);
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From b52b171ef5b3550136872c9271193dc4eed7c185 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 16 Nov 2018 13:23:31 -0800
Subject: [PATCH 0434/1554] TFTS: Soften collocation constraints

A device placement hint should be strong enough, and collocation constraints are error-prone.

PiperOrigin-RevId: 221842596
---
 .../timeseries/python/timeseries/math_utils.py     | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index 43c5267e63..aab3306438 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -802,7 +802,7 @@ class InputStatisticsFromMiniBatch(object):
             array_ops.shape(times)[1] - 1, self._dtype))
     # Co-locate updates with their variables to minimize race conditions when
     # updating statistics.
-    with ops.colocate_with(auxiliary_variables.max_time_seen):
+    with ops.device(auxiliary_variables.max_time_seen.device):
       # There is a race condition if this value is being updated from multiple
       # workers. However, it should eventually reach the correct value if the
       # last chunk is presented enough times.
@@ -810,16 +810,16 @@ class InputStatisticsFromMiniBatch(object):
           auxiliary_variables.max_time_seen,
           gen_math_ops.maximum(auxiliary_variables.max_time_seen,
                                math_ops.reduce_max(times)))
-    with ops.colocate_with(auxiliary_variables.chunk_count):
+    with ops.device(auxiliary_variables.chunk_count.device):
       chunk_count_assign = state_ops.assign_add(auxiliary_variables.chunk_count,
                                                 array_ops.shape(
                                                     times,
                                                     out_type=dtypes.int64)[0])
-    with ops.colocate_with(auxiliary_variables.inter_observation_duration_sum):
+    with ops.device(auxiliary_variables.inter_observation_duration_sum.device):
       inter_observation_duration_assign = state_ops.assign_add(
           auxiliary_variables.inter_observation_duration_sum,
           math_ops.reduce_sum(batch_inter_observation_duration))
-    with ops.colocate_with(auxiliary_variables.example_count):
+    with ops.device(auxiliary_variables.example_count.device):
       example_count_assign = state_ops.assign_add(
           auxiliary_variables.example_count,
           array_ops.size(times, out_type=dtypes.int64))
@@ -829,11 +829,11 @@ class InputStatisticsFromMiniBatch(object):
     # the series are then members of fewer chunks. For series which are much
     # longer than the chunk size (the usual/expected case), this effect becomes
     # irrelevant.
-    with ops.colocate_with(auxiliary_variables.overall_feature_sum):
+    with ops.device(auxiliary_variables.overall_feature_sum.device):
       overall_feature_sum_assign = state_ops.assign_add(
           auxiliary_variables.overall_feature_sum,
           math_ops.reduce_sum(values, axis=[0, 1]))
-    with ops.colocate_with(auxiliary_variables.overall_feature_sum_of_squares):
+    with ops.device(auxiliary_variables.overall_feature_sum_of_squares.device):
       overall_feature_sum_of_squares_assign = state_ops.assign_add(
           auxiliary_variables.overall_feature_sum_of_squares,
           math_ops.reduce_sum(values**2, axis=[0, 1]))
@@ -869,7 +869,7 @@ class InputStatisticsFromMiniBatch(object):
             state_ops.assign(statistics.series_start_moments.mean, mean),
             state_ops.assign(statistics.series_start_moments.variance,
                              variance))
-      with ops.colocate_with(statistics.start_time):
+      with ops.device(statistics.start_time.device):
         series_start_update = control_flow_ops.cond(
             # Update moments whenever we even match the lowest time seen so far,
             # to ensure that series start statistics are eventually updated to
-- 
GitLab


From fbb08168d67a0982893bfb2b32dd9d4ba79f606c Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 16 Nov 2018 13:25:52 -0800
Subject: [PATCH 0435/1554] Change API for tf.random.poisson in TF2.0.

PiperOrigin-RevId: 221842976
---
 tensorflow/python/ops/random_ops.py           | 41 ++++++++++++++++++-
 .../api/golden/v2/tensorflow.random.pbtxt     |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  1 +
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py
index 9c33ef3407..c893ef011b 100644
--- a/tensorflow/python/ops/random_ops.py
+++ b/tensorflow/python/ops/random_ops.py
@@ -478,7 +478,7 @@ def random_gamma(shape,
             shape, alpha_broadcast, seed=seed1, seed2=seed2) / beta)
 
 
-@tf_export("random.poisson", v1=["random.poisson", "random_poisson"])
+@tf_export(v1=["random.poisson", "random_poisson"])
 @deprecation.deprecated_endpoints("random_poisson")
 def random_poisson(lam, shape, dtype=dtypes.float32, seed=None, name=None):
   """Draws `shape` samples from each of the given Poisson distribution(s).
@@ -511,6 +511,45 @@ def random_poisson(lam, shape, dtype=dtypes.float32, seed=None, name=None):
       for behavior.
     name: Optional name for the operation.
 
+  Returns:
+    samples: a `Tensor` of shape `tf.concat([shape, tf.shape(lam)], axis=0)`
+      with values of type `dtype`.
+  """
+  return random_poisson_v2(shape, lam, dtype, seed, name)
+
+
+@tf_export("random.poisson", v1=[])
+def random_poisson_v2(shape, lam, dtype=dtypes.float32, seed=None, name=None):
+  """Draws `shape` samples from each of the given Poisson distribution(s).
+
+  `lam` is the rate parameter describing the distribution(s).
+
+  Example:
+
+  ```python
+  samples = tf.random_poisson([10], [0.5, 1.5])
+  # samples has shape [10, 2], where each slice [:, 0] and [:, 1] represents
+  # the samples drawn from each distribution
+
+  samples = tf.random_poisson([7, 5], [12.2, 3.3])
+  # samples has shape [7, 5, 2], where each slice [:, :, 0] and [:, :, 1]
+  # represents the 7x5 samples drawn from each of the two distributions
+  ```
+
+  Args:
+    shape: A 1-D integer Tensor or Python array. The shape of the output samples
+      to be drawn per "rate"-parameterized distribution.
+    lam: A Tensor or Python value or N-D array of type `dtype`.
+      `lam` provides the rate parameter(s) describing the poisson
+      distribution(s) to sample.
+    dtype: The type of the output: `float16`, `float32`, `float64`, `int32` or
+      `int64`.
+    seed: A Python integer. Used to create a random seed for the distributions.
+      See
+      `tf.set_random_seed`
+      for behavior.
+    name: Optional name for the operation.
+
   Returns:
     samples: a `Tensor` of shape `tf.concat([shape, tf.shape(lam)], axis=0)`
       with values of type `dtype`.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index 9dfa3c091d..ce8d277ec8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -26,7 +26,7 @@ tf_module {
   }
   member_method {
     name: "poisson"
-    argspec: "args=[\'lam\', \'shape\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\', \'None\'], "
+    argspec: "args=[\'shape\', \'lam\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\', \'None\'], "
   }
   member_method {
     name: "set_random_seed"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 77f50ab21f..19ea541a3f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -227,6 +227,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
+        "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
         "tf.sparse.segment_mean": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
-- 
GitLab


From 9bd1f90058a42fb4464f5ef3fbbee204dad7c10a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 16 Nov 2018 13:46:00 -0800
Subject: [PATCH 0436/1554] [TF:XLA] Bump open source llvm revision to r347046

PiperOrigin-RevId: 221846011
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 3618f14650..fd6afeb400 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "9d4ceba32fc6cedea2e299c32005f36e273f6174d4f2a7ad808231bd4ab43396",
-        strip_prefix = "llvm-45c215a28e72a6a6b301d0ee3466d999bd1e39d3",
+        sha256 = "286465fc41ade5c1c44e4a6dce9681106664fcdd12264dc9be63fc22bbee3c9c",
+        strip_prefix = "llvm-0478924a3727c74fd482d07eed45a8347540576e",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/45c215a28e72a6a6b301d0ee3466d999bd1e39d3.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/45c215a28e72a6a6b301d0ee3466d999bd1e39d3.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0478924a3727c74fd482d07eed45a8347540576e.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/0478924a3727c74fd482d07eed45a8347540576e.tar.gz",
         ],
     )
 
-- 
GitLab


From e07269873f7d53e6799b3dbd64d48513ab0f2cea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 14:02:59 -0800
Subject: [PATCH 0437/1554] Add additional tests for depthwise convolutions

PiperOrigin-RevId: 221848806
---
 tensorflow/compiler/xla/tests/BUILD           |  25 ++
 .../compiler/xla/tests/conv_depthwise_test.cc | 234 ++++++++++++++++++
 2 files changed, 259 insertions(+)
 create mode 100644 tensorflow/compiler/xla/tests/conv_depthwise_test.cc

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 42f44513a5..20493a354c 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -298,6 +298,31 @@ xla_test(
     ],
 )
 
+xla_test(
+    name = "conv_depthwise_test",
+    timeout = "long",
+    srcs = ["conv_depthwise_test.cc"],
+    blacklisted_backends = [
+        # disabled because of a break b/119590850.
+        "cpu",
+        "gpu",
+    ],
+    shard_count = 50,
+    deps = [
+        "//tensorflow/compiler/xla:execution_options_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/service:bfloat16_normalization",
+        "//tensorflow/compiler/xla/service:despecializer",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
 xla_test(
     name = "check_execution_arity_test",
     srcs = ["check_execution_arity_test.cc"],
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
new file mode 100644
index 0000000000..60ce576ceb
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
@@ -0,0 +1,234 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/execution_options_util.h"
+#include "tensorflow/compiler/xla/service/bfloat16_normalization.h"
+#include "tensorflow/compiler/xla/service/despecializer.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+
+namespace xla {
+namespace {
+
+string GetFloatDataType(bool use_bfloat16) {
+  return use_bfloat16 ? "bf16" : "f32";
+}
+
+struct DepthwiseConvolution2DSpec {
+  int64 output_feature, window, stride, pad, lhs_dilate;
+  std::vector<int64> activation_dims;
+  std::vector<int64> activation_layout;
+  std::vector<int64> kernel_dims;
+  std::vector<int64> kernel_layout;
+  std::vector<int64> output_dims;
+  std::vector<int64> output_layout;
+};
+
+class DepthwiseConvolution2DTest
+    : public HloTestBase,
+      public ::testing::WithParamInterface<
+          ::testing::tuple<DepthwiseConvolution2DSpec, bool>> {};
+
+static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
+  std::vector<DepthwiseConvolution2DSpec> config_set;
+  std::vector<std::vector<int64>> config_options = {
+      {128, 6, 3, 64},  {256, 5, 3, 256},  {256, 5, 2, 144}, {144, 5, 3, 64},
+      {144, 5, 2, 256}, {8, 48, 17, 8},    {128, 20, 6, 64}, {128, 1, 2, 144},
+      {256, 1, 2, 64},  {64, 14, 12, 172}, {16, 9, 4, 16}};
+
+  for (auto option : config_options) {
+    int64 feature = option[0];
+    int64 activation_size = option[1];
+    int64 kernel_size = option[2];
+    int64 batch = option[3];
+
+    std::vector<int64> kernel_layout = {3, 2, 1, 0};
+    DepthwiseConvolution2DSpec config;
+    config.output_feature = feature;
+    config.window = kernel_size;
+
+    config.activation_dims = {batch, activation_size, activation_size, feature};
+    config.activation_layout = {3, 0, 2, 1};
+
+    config.kernel_dims = {kernel_size, kernel_size, 1, feature};
+    config.kernel_layout = {3, 2, 1, 0};
+
+    if (activation_size == 1 && kernel_size == 2) {
+      // Test for outer dim.
+      config.output_dims = {batch, activation_size + kernel_size - 1,
+                            activation_size + kernel_size, feature};
+    } else if (feature == 256) {
+      // Restrict dilation-based tests only to one feature configuration.
+      config.stride = activation_size - 1;
+      config.pad = 0;
+      config.lhs_dilate = feature / 32;
+      config.output_dims = {batch, feature / 32,
+                            activation_size - kernel_size + 1, feature};
+    } else {
+      config.stride = config.pad = config.lhs_dilate = -1;
+      config.output_dims = {batch, activation_size - kernel_size + 1,
+                            activation_size - kernel_size + 1, feature};
+    }
+
+    // Try this layout for all kernel shapes.
+    config.output_layout = {3, 0, 2, 1};
+    config_set.push_back(config);
+
+    // Try other layouts only for certain kernel shapes.
+    if (kernel_size % 2 == 0) {
+      config.activation_layout = {0, 3, 2, 1};
+      config_set.push_back(config);
+
+      config.output_layout = {0, 3, 2, 1};
+      config_set.push_back(config);
+
+      config.activation_layout = {3, 0, 2, 1};
+      config_set.push_back(config);
+    }
+  }
+
+  return config_set;
+}
+
+string DepthwiseConvolution2DTestDataToString(
+    const ::testing::TestParamInfo<
+        ::testing::tuple<DepthwiseConvolution2DSpec, bool>>& data) {
+  const auto& spec = ::testing::get<0>(data.param);
+  const string data_type = GetFloatDataType(::testing::get<1>(data.param));
+  string str = absl::StrCat(
+      "activation_dims_", absl::StrJoin(spec.activation_dims, "x"),
+      "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"),
+      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_",
+      absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_",
+      absl::StrJoin(spec.output_dims, "x"), "_output_layout_",
+      absl::StrJoin(spec.output_layout, "_"), data_type);
+  // -1 indicates non-existence.
+  if (spec.stride != -1) {
+    absl::StrAppend(&str, "_lhs_dilation_", spec.lhs_dilate, "x1");
+  }
+
+  // Test names are not allowed to contain the '-' character.
+  absl::c_replace(str, '-', 'n');
+  return str;
+}
+
+string BuildHloTextDepthwiseConvolution2D(
+    const DepthwiseConvolution2DSpec& spec, bool use_bfloat16) {
+  const string data_type = GetFloatDataType(use_bfloat16);
+  if (spec.activation_dims[1] == 1 && spec.kernel_dims[1] == 2) {
+    return absl::StrFormat(
+        R"(
+    HloModule TensorFlowDepthwiseConv, is_scheduled=true
+
+    ENTRY main {
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+          window={size=%dx%d  pad=1_1x%d_%d rhs_dilate=1x%d}, dim_labels=b01f_01io->b01f,
+          feature_group_count=%d
+    }
+    )",
+        data_type, absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), data_type,
+        absl::StrJoin(spec.output_dims, ","),
+        absl::StrJoin(spec.output_layout, ","), data_type,
+        absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+        spec.window, spec.window, spec.window, spec.output_feature);
+
+  } else if (spec.stride == -1) {
+    return absl::StrFormat(
+        R"(
+      HloModule TensorFlowDepthwiseConv, is_scheduled=true
+
+      ENTRY main {
+        activation = %s[%s]{%s} parameter(0)
+        kernel = %s[%s]{%s} parameter(1)
+        ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+            window={size=%dx%d}, dim_labels=b01f_01io->b01f,
+            feature_group_count=%d
+      }
+      )",
+        data_type, absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), data_type,
+        absl::StrJoin(spec.output_dims, ","),
+        absl::StrJoin(spec.output_layout, ","), data_type,
+        absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+        spec.output_feature);
+  } else {
+    return absl::StrFormat(
+        R"(
+    HloModule TensorFlowDepthwiseConv, is_scheduled=true
+
+    ENTRY main {
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+          window={size=%dx%d stride=%dx1 pad=%d_%dx0_0 lhs_dilate=%dx1}, 
+          dim_labels=b01f_01io->b01f, feature_group_count=%d
+    }
+    )",
+        data_type, absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), data_type,
+        absl::StrJoin(spec.output_dims, ","),
+        absl::StrJoin(spec.output_layout, ","), data_type,
+        absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+        spec.stride, 0, 0, spec.lhs_dilate, spec.output_feature);
+  }
+}
+
+XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
+  const DepthwiseConvolution2DSpec& spec = ::testing::get<0>(GetParam());
+  bool use_bfloat16 = ::testing::get<1>(GetParam());
+  const string hlo_text =
+      BuildHloTextDepthwiseConvolution2D(spec, use_bfloat16);
+
+  EXPECT_TRUE(RunAndCompareNoHloPasses(
+      hlo_text, ErrorSpec{0.01, 0.01}, [](HloModule* module) -> Status {
+        BFloat16MixedPrecisionRemoval remover;
+        TF_RETURN_IF_ERROR(remover.Run(module).status());
+        Despecializer despecializer;
+        return despecializer.Run(module).status();
+      }));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    DepthwiseConvolution2DTestWithRandomIndices, DepthwiseConvolution2DTest,
+    ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()),
+                       ::testing::Bool()),
+    DepthwiseConvolution2DTestDataToString);
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From b51ad2408b5050ee497a3af602d4bbd950f89973 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Fri, 16 Nov 2018 14:16:23 -0800
Subject: [PATCH 0438/1554] Add option to disable nccl

---
 configure.py            |   1 +
 tensorflow/BUILD        |   6 +++
 tensorflow/core/BUILD   | 111 +++++++++++++++++++++-------------------
 tensorflow/python/BUILD |   1 +
 tools/bazel.rc          |   2 +
 5 files changed, 67 insertions(+), 54 deletions(-)

diff --git a/configure.py b/configure.py
index 234561d94a..f663f6df6f 100644
--- a/configure.py
+++ b/configure.py
@@ -1694,6 +1694,7 @@ def main():
   config_info_line('nohdfs', 'Disable HDFS support.')
   config_info_line('noignite', 'Disable Apacha Ignite support.')
   config_info_line('nokafka', 'Disable Apache Kafka support.')
+  config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 2dc70c359c..ab70486002 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -246,6 +246,12 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "no_nccl_support",
+    define_values = {"no_nccl_support": "true"},
+    visibility = ["//visibility:public"],
+)
+
 # Crosses between platforms and file system libraries not supported on those
 # platforms due to limitations in nested select() statements.
 config_setting(
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 73e8db58a8..3237c34a99 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1349,60 +1349,63 @@ cc_library(
     name = "all_kernels_statically_linked",
     visibility = ["//visibility:private"],
     deps = [
-        "//tensorflow/core/kernels:array",
-        "//tensorflow/core/kernels:audio",
-        "//tensorflow/core/kernels:batch_kernels",
-        "//tensorflow/core/kernels:bincount_op",
-        "//tensorflow/core/kernels:boosted_trees_ops",
-        "//tensorflow/core/kernels:candidate_sampler_ops",
-        "//tensorflow/core/kernels:checkpoint_ops",
-        "//tensorflow/core/kernels:collective_ops",
-        "//tensorflow/core/kernels:control_flow_ops",
-        "//tensorflow/core/kernels:ctc_ops",
-        "//tensorflow/core/kernels:cudnn_rnn_kernels",
-        "//tensorflow/core/kernels:data_flow",
-        "//tensorflow/core/kernels:dataset_ops",
-        "//tensorflow/core/kernels:decode_proto_op",
-        "//tensorflow/core/kernels:encode_proto_op",
-        "//tensorflow/core/kernels:fake_quant_ops",
-        "//tensorflow/core/kernels:function_ops",
-        "//tensorflow/core/kernels:functional_ops",
-        "//tensorflow/core/kernels:grappler",
-        "//tensorflow/core/kernels:histogram_op",
-        "//tensorflow/core/kernels:image",
-        "//tensorflow/core/kernels:io",
-        "//tensorflow/core/kernels:linalg",
-        "//tensorflow/core/kernels:list_kernels",
-        "//tensorflow/core/kernels:lookup",
-        "//tensorflow/core/kernels:logging",
-        "//tensorflow/core/kernels:manip",
-        "//tensorflow/core/kernels:math",
-        "//tensorflow/core/kernels:multinomial_op",
-        "//tensorflow/core/kernels:nn",
-        "//tensorflow/core/kernels:parameterized_truncated_normal_op",
-        "//tensorflow/core/kernels:parsing",
-        "//tensorflow/core/kernels:partitioned_function_ops",
-        "//tensorflow/core/kernels:ragged_ops",
-        "//tensorflow/core/kernels:random_ops",
-        "//tensorflow/core/kernels:random_poisson_op",
-        "//tensorflow/core/kernels:remote_fused_graph_ops",
-        "//tensorflow/core/kernels:required",
-        "//tensorflow/core/kernels:resource_variable_ops",
-        "//tensorflow/core/kernels:rpc_op",
-        "//tensorflow/core/kernels:scoped_allocator_ops",
-        "//tensorflow/core/kernels:sdca_ops",
-        "//tensorflow/core/kernels:searchsorted_op",
-        "//tensorflow/core/kernels:set_kernels",
-        "//tensorflow/core/kernels:sparse",
-        "//tensorflow/core/kernels:state",
-        "//tensorflow/core/kernels:stateless_random_ops",
-        "//tensorflow/core/kernels:string",
-        "//tensorflow/core/kernels:summary_kernels",
-        "//tensorflow/core/kernels:training_ops",
-        "//tensorflow/core/kernels:word2vec_kernels",
-    ] + tf_additional_cloud_kernel_deps() + if_not_tx2_llvm_or_windows_cuda([
-        "//tensorflow/core/kernels:nccl_kernels",
-    ]) + if_not_windows([
+               "//tensorflow/core/kernels:array",
+               "//tensorflow/core/kernels:audio",
+               "//tensorflow/core/kernels:batch_kernels",
+               "//tensorflow/core/kernels:bincount_op",
+               "//tensorflow/core/kernels:boosted_trees_ops",
+               "//tensorflow/core/kernels:candidate_sampler_ops",
+               "//tensorflow/core/kernels:checkpoint_ops",
+               "//tensorflow/core/kernels:collective_ops",
+               "//tensorflow/core/kernels:control_flow_ops",
+               "//tensorflow/core/kernels:ctc_ops",
+               "//tensorflow/core/kernels:cudnn_rnn_kernels",
+               "//tensorflow/core/kernels:data_flow",
+               "//tensorflow/core/kernels:dataset_ops",
+               "//tensorflow/core/kernels:decode_proto_op",
+               "//tensorflow/core/kernels:encode_proto_op",
+               "//tensorflow/core/kernels:fake_quant_ops",
+               "//tensorflow/core/kernels:function_ops",
+               "//tensorflow/core/kernels:functional_ops",
+               "//tensorflow/core/kernels:grappler",
+               "//tensorflow/core/kernels:histogram_op",
+               "//tensorflow/core/kernels:image",
+               "//tensorflow/core/kernels:io",
+               "//tensorflow/core/kernels:linalg",
+               "//tensorflow/core/kernels:list_kernels",
+               "//tensorflow/core/kernels:lookup",
+               "//tensorflow/core/kernels:logging",
+               "//tensorflow/core/kernels:manip",
+               "//tensorflow/core/kernels:math",
+               "//tensorflow/core/kernels:multinomial_op",
+               "//tensorflow/core/kernels:nn",
+               "//tensorflow/core/kernels:parameterized_truncated_normal_op",
+               "//tensorflow/core/kernels:parsing",
+               "//tensorflow/core/kernels:partitioned_function_ops",
+               "//tensorflow/core/kernels:ragged_ops",
+               "//tensorflow/core/kernels:random_ops",
+               "//tensorflow/core/kernels:random_poisson_op",
+               "//tensorflow/core/kernels:remote_fused_graph_ops",
+               "//tensorflow/core/kernels:required",
+               "//tensorflow/core/kernels:resource_variable_ops",
+               "//tensorflow/core/kernels:rpc_op",
+               "//tensorflow/core/kernels:scoped_allocator_ops",
+               "//tensorflow/core/kernels:sdca_ops",
+               "//tensorflow/core/kernels:searchsorted_op",
+               "//tensorflow/core/kernels:set_kernels",
+               "//tensorflow/core/kernels:sparse",
+               "//tensorflow/core/kernels:state",
+               "//tensorflow/core/kernels:stateless_random_ops",
+               "//tensorflow/core/kernels:string",
+               "//tensorflow/core/kernels:summary_kernels",
+               "//tensorflow/core/kernels:training_ops",
+               "//tensorflow/core/kernels:word2vec_kernels",
+           ] + tf_additional_cloud_kernel_deps() +
+           select({
+               "//tensorflow:no_nccl_support": [],
+               "//tensorflow:with_cuda_support_windows_override": [],
+               "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"],
+           }) + if_not_windows([
         "//tensorflow/core/kernels:fact_op",
         "//tensorflow/core/kernels:array_not_windows",
         "//tensorflow/core/kernels:math_not_windows",
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e55b2a0e92..4d9ed5eea4 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -5778,6 +5778,7 @@ cuda_py_test(
         "no_oss",
         "noguitar",
         "notap",
+        "no_nccl"
     ],
 )
 
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 8c2052ee8a..76da3470dc 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -72,6 +72,8 @@ build:nogcp --define=no_gcp_support=true
 build:nohdfs --define=no_hdfs_support=true
 build:nokafka --define=no_kafka_support=true
 build:noignite --define=no_ignite_support=true
+build:nonccl --define=no_nccl_support=true
+test:nonccl --define=no_nccl_support=true --test_tag_filters=-no_nccl
 
 build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
-- 
GitLab


From a59a88d9e462b03e434bd8305afeab945cee06b6 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Fri, 16 Nov 2018 14:14:51 -0800
Subject: [PATCH 0439/1554] TensorSpec and TensorInfo

Export tf.TensorSpec.
This is needed to specify signatures to tf.function() calls
in TensorFlow 2.x (and tf.contrib.eager.function() calls in 1.x)

Hide tf.TensorInfo in TF 2.0
Not sure why users of the TensorFlow library should need it, so remove it.

PiperOrigin-RevId: 221850858
---
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/__init__.py                 |  2 +-
 tensorflow/python/framework/tensor_spec.py    | 40 +++++--------
 .../python/framework/tensor_spec_test.py      | 21 -------
 .../golden/v1/tensorflow.-tensor-spec.pbtxt   | 33 +++++++++++
 .../tools/api/golden/v1/tensorflow.pbtxt      |  4 ++
 .../tensorflow.-tensor-info.-coo-sparse.pbtxt | 24 --------
 .../golden/v2/tensorflow.-tensor-info.pbtxt   | 59 -------------------
 .../golden/v2/tensorflow.-tensor-spec.pbtxt   | 33 +++++++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  6 +-
 10 files changed, 90 insertions(+), 133 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 83b7bc9403..c21356e676 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -996,6 +996,7 @@ py_library(
         ":common_shapes",
         ":dtypes",
         ":tensor_shape",
+        ":util",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index c93ba4dcee..3b462c7de8 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -162,7 +162,7 @@ tf_export('Summary', 'summary.Summary')(Summary)
 tf_export('summary.SummaryDescription')(SummaryDescription)
 tf_export('SummaryMetadata')(SummaryMetadata)
 tf_export('summary.TaggedRunMetadata')(TaggedRunMetadata)
-tf_export('TensorInfo')(TensorInfo)
+tf_export(v1=['TensorInfo'])(TensorInfo)
 # pylint: enable=undefined-variable
 
 # Special dunders that we choose to export:
diff --git a/tensorflow/python/framework/tensor_spec.py b/tensorflow/python/framework/tensor_spec.py
index fbea930fe0..c44636edc4 100644
--- a/tensorflow/python/framework/tensor_spec.py
+++ b/tensorflow/python/framework/tensor_spec.py
@@ -24,14 +24,15 @@ from tensorflow.python.framework import common_shapes
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.util.tf_export import tf_export
 
 
+@tf_export("TensorSpec")
 class TensorSpec(object):
   """Describes a tf.Tensor.
 
-  A TensorSpec allows an API to describe the Tensors that it accepts or
-  returns, before that Tensor exists. This allows dynamic and flexible graph
-  construction and configuration.
+  Metadata for describing the `tf.Tensor` objects accepted or returned
+  by some TensorFlow APIs.
   """
 
   __slots__ = ["_shape", "_shape_tuple", "_dtype", "_name"]
@@ -69,11 +70,6 @@ class TensorSpec(object):
     else:
       raise ValueError("`tensor` should be a tf.Tensor")
 
-  @classmethod
-  def is_bounded(cls):
-    del cls
-    return False
-
   @property
   def shape(self):
     """Returns the `TensorShape` that represents the shape of the tensor."""
@@ -86,21 +82,21 @@ class TensorSpec(object):
 
   @property
   def name(self):
-    """Returns the name of the described tensor."""
+    """Returns the (optionally provided) name of the described tensor."""
     return self._name
 
-  @property
-  def is_discrete(self):
-    """Whether spec is discrete."""
-    return self.dtype.is_integer
+  def is_compatible_with(self, spec_or_tensor):
+    """Returns True if spec_or_tensor is compatible with this TensorSpec.
 
-  @property
-  def is_continuous(self):
-    """Whether spec is continuous."""
-    return self.dtype.is_floating
+    Two tensors are considered compatible if they have the same dtype
+    and their shapes are compatible (see `tf.TensorShape.is_compatible_with`).
 
-  def is_compatible_with(self, spec_or_tensor):
-    """True if the shape and dtype of `spec_or_tensor` are compatible."""
+    Args:
+      spec_or_tensor: A tf.TensorSpec or a tf.Tensor
+
+    Returns:
+      True if spec_or_tensor is compatible with self.
+    """
     return (self._dtype.is_compatible_with(spec_or_tensor.dtype) and
             self._shape.is_compatible_with(spec_or_tensor.shape))
 
@@ -188,11 +184,6 @@ class BoundedTensorSpec(TensorSpec):
     self._maximum = np.array(maximum, dtype=self.dtype.as_numpy_dtype())
     self._maximum.setflags(write=False)
 
-  @classmethod
-  def is_bounded(cls):
-    del cls
-    return True
-
   @classmethod
   def from_spec(cls, spec):
     dtype = dtypes.as_dtype(spec.dtype)
@@ -223,4 +214,3 @@ class BoundedTensorSpec(TensorSpec):
   def __reduce__(self):
     return BoundedTensorSpec, (self._shape, self._dtype, self._minimum,
                                self._maximum, self._name)
-
diff --git a/tensorflow/python/framework/tensor_spec_test.py b/tensorflow/python/framework/tensor_spec_test.py
index 40611e5f84..e3aad7cc23 100644
--- a/tensorflow/python/framework/tensor_spec_test.py
+++ b/tensorflow/python/framework/tensor_spec_test.py
@@ -134,22 +134,6 @@ class TensorSpecTest(test_util.TensorFlowTestCase):
     self.assertEqual(bounded_spec.dtype, spec.dtype)
     self.assertEqual(bounded_spec.name, spec.name)
 
-  def testIsDiscrete(self):
-    discrete_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32)
-    continuous_spec = tensor_spec.TensorSpec((1, 2), dtypes.float32)
-    self.assertTrue(discrete_spec.is_discrete)
-    self.assertFalse(continuous_spec.is_discrete)
-
-  def testIsContinuous(self):
-    discrete_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32)
-    continuous_spec = tensor_spec.TensorSpec((1, 2), dtypes.float32)
-    self.assertFalse(discrete_spec.is_continuous)
-    self.assertTrue(continuous_spec.is_continuous)
-
-  def testIsBounded(self):
-    unbounded_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32)
-    self.assertFalse(unbounded_spec.is_bounded())
-
   def testSerialization(self):
     desc = tensor_spec.TensorSpec([1, 5], dtypes.float32, "test")
     self.assertEqual(pickle.loads(pickle.dumps(desc)), desc)
@@ -165,11 +149,6 @@ class BoundedTensorSpecTest(test_util.TensorFlowTestCase):
     with self.assertRaisesRegexp(ValueError, "not compatible"):
       tensor_spec.BoundedTensorSpec((3, 5), dtypes.uint8, 0, (1, 1, 1))
 
-  def testIsBounded(self):
-    bounded_spec = tensor_spec.BoundedTensorSpec(
-        (1, 2), dtypes.int32, minimum=0, maximum=1)
-    self.assertTrue(bounded_spec.is_bounded())
-
   def testMinimumMaximumAttributes(self):
     spec = tensor_spec.BoundedTensorSpec(
         (1, 2, 3), dtypes.float32, 0, (5, 5, 5))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt
new file mode 100644
index 0000000000..493dcba892
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-tensor-spec.pbtxt
@@ -0,0 +1,33 @@
+path: "tensorflow.TensorSpec"
+tf_class {
+  is_instance: "<class \'tensorflow.python.framework.tensor_spec.TensorSpec\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'shape\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_spec"
+    argspec: "args=[\'cls\', \'spec\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_tensor"
+    argspec: "args=[\'cls\', \'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'spec_or_tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 707337ebfa..1bb0152f19 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -244,6 +244,10 @@ tf_module {
     name: "TensorShape"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TensorSpec"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TextLineReader"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt
deleted file mode 100644
index 0064c8460c..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.-coo-sparse.pbtxt
+++ /dev/null
@@ -1,24 +0,0 @@
-path: "tensorflow.TensorInfo.CooSparse"
-tf_proto {
-  descriptor {
-    name: "CooSparse"
-    field {
-      name: "values_tensor_name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "indices_tensor_name"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "dense_shape_tensor_name"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt
deleted file mode 100644
index 63566c808e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-info.pbtxt
+++ /dev/null
@@ -1,59 +0,0 @@
-path: "tensorflow.TensorInfo"
-tf_proto {
-  descriptor {
-    name: "TensorInfo"
-    field {
-      name: "name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      oneof_index: 0
-    }
-    field {
-      name: "coo_sparse"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorInfo.CooSparse"
-      oneof_index: 0
-    }
-    field {
-      name: "dtype"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.DataType"
-    }
-    field {
-      name: "tensor_shape"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorShapeProto"
-    }
-    nested_type {
-      name: "CooSparse"
-      field {
-        name: "values_tensor_name"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "indices_tensor_name"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "dense_shape_tensor_name"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-    }
-    oneof_decl {
-      name: "encoding"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt
new file mode 100644
index 0000000000..493dcba892
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-tensor-spec.pbtxt
@@ -0,0 +1,33 @@
+path: "tensorflow.TensorSpec"
+tf_class {
+  is_instance: "<class \'tensorflow.python.framework.tensor_spec.TensorSpec\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'shape\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_spec"
+    argspec: "args=[\'cls\', \'spec\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "from_tensor"
+    argspec: "args=[\'cls\', \'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'spec_or_tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 8fcee46107..fa4a538122 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -121,11 +121,11 @@ tf_module {
     mtype: "<type \'type\'>"
   }
   member {
-    name: "TensorInfo"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
+    name: "TensorShape"
+    mtype: "<type \'type\'>"
   }
   member {
-    name: "TensorShape"
+    name: "TensorSpec"
     mtype: "<type \'type\'>"
   }
   member {
-- 
GitLab


From aaff777c6d90f1717c970f3651fc254d20a95146 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 16 Nov 2018 14:15:09 -0800
Subject: [PATCH 0440/1554] Remove rewriter_config parameter from
 create_inference_graph, that existed before because Grappler's
 OptimizeGraph() needs one. Now OptimizeGraph() is changed to accept a
 ConfigProto so we can provide the RewriterConfig inside the 'session_config'
 parameter.

PiperOrigin-RevId: 221850911
---
 .../contrib/tensorrt/python/trt_convert.py    | 64 +++++++++++--------
 .../tensorrt/python/trt_convert_test.py       |  6 +-
 .../test/tf_trt_integration_test_base.py      | 21 +++---
 3 files changed, 52 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index d149dcbd56..0e59fdd1fe 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -63,19 +63,19 @@ class TrtPrecisionMode(object):
     return [TrtPrecisionMode.FP32, TrtPrecisionMode.FP16, TrtPrecisionMode.INT8]
 
 
-def tensorrt_rewriter_config(rewriter_config=None,
-                             max_batch_size=1,
-                             max_workspace_size_bytes=2 << 20,
-                             precision_mode=TrtPrecisionMode.FP32,
-                             minimum_segment_size=3,
-                             is_dynamic_op=False,
-                             maximum_cached_engines=1,
-                             cached_engine_batch_sizes=None):
+def get_tensorrt_rewriter_config(rewriter_config=None,
+                                 max_batch_size=1,
+                                 max_workspace_size_bytes=2 << 20,
+                                 precision_mode=TrtPrecisionMode.FP32,
+                                 minimum_segment_size=3,
+                                 is_dynamic_op=False,
+                                 maximum_cached_engines=1,
+                                 cached_engine_batch_sizes=None):
   """Returns a RewriterConfig proto for TRT transformation.
 
   Args:
-    rewriter_config: a RewriterConfig proto to append the TensorRTOptimizer to.
-      If None, it will create one with default settings.
+    rewriter_config: a template RewriterConfig proto used to create a
+      TRT-enabled RewriterConfig. If None, it will use a default one.
     max_batch_size: max size for the input batch
     max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
       engine can use at execution time. This corresponds to the 'workspaceSize'
@@ -107,13 +107,16 @@ def tensorrt_rewriter_config(rewriter_config=None,
       rewriter_config, rewriter_config_pb2.RewriterConfig):
     raise TypeError("rewriter_config should be a RewriterConfig proto.")
 
+  rewriter_config_with_trt = rewriter_config_pb2.RewriterConfig()
   if rewriter_config is None:
-    rewriter_config = rewriter_config_pb2.RewriterConfig()
     # Layout optimizer may add Const nodes followed by Reshape nodes, thus we
     # need to run constant folding again.
-    rewriter_config.optimizers.extend(["constfold", "layout", "constfold"])
-    rewriter_config.meta_optimizer_iterations = (
+    rewriter_config_with_trt.optimizers.extend(
+        ["constfold", "layout", "constfold"])
+    rewriter_config_with_trt.meta_optimizer_iterations = (
         rewriter_config_pb2.RewriterConfig.ONE)
+  else:
+    rewriter_config_with_trt.CopyFrom(rewriter_config)
 
   if precision_mode.upper() not in TrtPrecisionMode.supported_precision_modes():
     raise ValueError(("precision mode '{}' is not supported."
@@ -121,7 +124,7 @@ def tensorrt_rewriter_config(rewriter_config=None,
                           precision_mode,
                           TrtPrecisionMode.supported_precision_modes))
 
-  optimizer = rewriter_config.custom_optimizers.add()
+  optimizer = rewriter_config_with_trt.custom_optimizers.add()
   optimizer.name = "TensorRTOptimizer"
   optimizer.parameter_map["minimum_segment_size"].i = minimum_segment_size
   optimizer.parameter_map["max_batch_size"].i = max_batch_size
@@ -138,7 +141,7 @@ def tensorrt_rewriter_config(rewriter_config=None,
                        "maximum_cached_engines items.")
     optimizer.parameter_map["cached_engine_batches"].list.i.extend(
         cached_engine_batch_sizes)
-  return rewriter_config
+  return rewriter_config_with_trt
 
 
 def create_inference_graph(input_graph_def,
@@ -150,7 +153,6 @@ def create_inference_graph(input_graph_def,
                            is_dynamic_op=False,
                            maximum_cached_engines=1,
                            cached_engine_batch_sizes=None,
-                           rewriter_config=None,
                            input_saved_model_dir=None,
                            input_saved_model_tags=None,
                            output_saved_model_dir=None,
@@ -182,8 +184,6 @@ def create_inference_graph(input_graph_def,
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
-    rewriter_config: a RewriterConfig proto to append the TensorRTOptimizer to.
-      If None, it will create one with default settings.
     input_saved_model_dir: the directory to load the SavedModel which contains
       the input graph to transforms. Used only when input_graph_def is None.
     input_saved_model_tags: list of tags to load the SavedModel.
@@ -191,8 +191,9 @@ def create_inference_graph(input_graph_def,
       returned GraphDef and save it to the specified directory. This option only
       works when the input graph is loaded from a SavedModel, i.e. when
       input_saved_model_dir is specified and input_graph_def is None.
-    session_config: the ConfigProto used to create a Session. If not specified,
-      a default ConfigProto will be used.
+    session_config: the ConfigProto used to create a Session. It's also used as
+      a template to create a TRT-enabled ConfigProto for conversion. If not
+      specified, a default ConfigProto will be used.
 
   Returns:
     A GraphDef transformed from input_graph_def (or the SavedModel graph def
@@ -322,23 +323,30 @@ def create_inference_graph(input_graph_def,
       grappler_meta_graph_def.collection_def["train_op"].CopyFrom(
           output_collection)
 
-  # Create RewriterConfig.
-  config = config_pb2.ConfigProto()
-  config.graph_options.rewrite_options.CopyFrom(
-      tensorrt_rewriter_config(
-          rewriter_config, max_batch_size, max_workspace_size_bytes,
-          precision_mode, minimum_segment_size, is_dynamic_op,
-          maximum_cached_engines, cached_engine_batch_sizes))
+  # Create TRT-enabled ConfigProto.
+  session_config_with_trt = config_pb2.ConfigProto()
+  session_config_with_trt.CopyFrom(session_config)
+  rewriter_config = None
+  if (session_config_with_trt.HasField("graph_options") and
+      session_config_with_trt.graph_options.HasField("rewrite_options")):
+    rewriter_config = session_config_with_trt.graph_options.rewrite_options
+  rewriter_config_with_trt = get_tensorrt_rewriter_config(
+      rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
+      minimum_segment_size, is_dynamic_op, maximum_cached_engines,
+      cached_engine_batch_sizes)
+  session_config_with_trt.graph_options.rewrite_options.CopyFrom(
+      rewriter_config_with_trt)
 
   # Run Grappler.
   transformed_graph_def = tf_optimizer.OptimizeGraph(
-      config, grappler_meta_graph_def, graph_id=b"tf_graph")
+      session_config_with_trt, grappler_meta_graph_def, graph_id=b"tf_graph")
 
   # Optionally write the transformed graphdef as SavedModel.
   if output_saved_model_dir is not None:
     saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir)
     with ops.Graph().as_default():
       importer.import_graph_def(transformed_graph_def, name="")
+      # We don't use TRT here.
       with session.Session(config=session_config) as sess:
         saved_model_builder.add_meta_graph_and_variables(
             sess,
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 9f2eeac990..aa82f4207f 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -47,9 +47,9 @@ from tensorflow.python.tools import saved_model_utils
 class TrtConvertTest(test_util.TensorFlowTestCase):
   """Class to test Tensorflow-TensorRT integration python API."""
 
-  def testTensorrtRewriterConfig(self):
-    """Test case for trt_convert.tensorrt_rewriter_config()."""
-    rewriter_cfg = trt_convert.tensorrt_rewriter_config(
+  def testGetTensorrtRewriterConfig(self):
+    """Test case for trt_convert.get_tensorrt_rewriter_config()."""
+    rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
         rewriter_config=None,
         max_batch_size=128,
         max_workspace_size_bytes=1234,
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index a725d0651c..1763f3b22c 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -198,11 +198,16 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     trt_convert.clear_test_values("my_trt_op_.*:ExecuteCalibration")
     trt_convert.clear_test_values("my_trt_op_.*:ExecuteNativeSegment")
 
+  def _GetGPUOptions(self):
+    gpu_options = config_pb2.GPUOptions()
+    gpu_options.allow_growth = True
+    return gpu_options
+
   def _GetConfigProto(self, run_params, graph_state):
     """Get config proto based on specific settings."""
     if graph_state != GraphState.ORIGINAL and run_params.use_optimizer:
       conversion_params = self.GetConversionParams(run_params)
-      rewriter_cfg = trt_convert.tensorrt_rewriter_config(
+      rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
           conversion_params.rewriter_config, conversion_params.max_batch_size,
           conversion_params.max_workspace_size_bytes,
           conversion_params.precision_mode,
@@ -215,13 +220,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     else:
       graph_options = config_pb2.GraphOptions()
 
-    gpu_options = config_pb2.GPUOptions()
-    gpu_options.allow_growth = True
-    if trt_convert.get_linked_tensorrt_version()[0] == 3:
-      gpu_options.per_process_gpu_memory_fraction = 0.50
-
     config = config_pb2.ConfigProto(
-        gpu_options=gpu_options, graph_options=graph_options)
+        gpu_options=self._GetGPUOptions(), graph_options=graph_options)
     return config
 
   def _ExpectTestValue(self, engine_name, method, expected_value):
@@ -291,6 +291,11 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     params = self._GetParamsCached()
     conversion_params = self.GetConversionParams(run_params)
     logging.info(conversion_params)
+
+    config_for_trt = config_pb2.ConfigProto(gpu_options=self._GetGPUOptions())
+    if conversion_params.rewriter_config is not None:
+      config_for_trt.graph_options.rewrite_options.CopyFrom(
+          conversion_params.rewriter_config)
     return trt_convert.create_inference_graph(
         input_graph_def=gdef,
         outputs=params.input_names + params.output_names,
@@ -301,7 +306,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_op=conversion_params.is_dynamic_op,
         maximum_cached_engines=conversion_params.maximum_cached_engines,
         cached_engine_batch_sizes=conversion_params.cached_engine_batch_sizes,
-        rewriter_config=conversion_params.rewriter_config)
+        session_config=config_for_trt)
 
   def _WriteGraph(self, run_params, gdef, graph_state):
     if graph_state == GraphState.ORIGINAL:
-- 
GitLab


From 32d82226827272b458f551078352c26005bb1b46 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 16 Nov 2018 14:15:26 -0800
Subject: [PATCH 0441/1554] PR #21365: Fix nadam optimizer

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

Imported from GitHub PR #21365

Resolves issues #15035 and #13980.

Copybara import of the project:

  - 4d389e6cee2e0b355ee27a529864c8843438487e fix nadam optimizer by Emma Strubell <strubell@cs.umass.edu>
  - 1f2816b3063a6a1c635db48cc52604acbbc63682 update nadam optimizer test and fix nadam to work with Re... by Emma Strubell <strubell@cs.umass.edu>
  - 3110cb6d13a24bb0434f4bc4944931fc0cd27d16 fix whitespace and elaborate on comment by Emma Strubell <strubell@cs.umass.edu>
  - 98deea7f207b226a152509dd663af6192147bb30 actually fix whitespace by Emma Strubell <strubell@cs.umass.edu>
  - 42292b7a430460b30d12bc4c03b30eeeec1e9900 fix line length by Emma Strubell <strubell@cs.umass.edu>
  - e05efc155344b9aec22b037894d92eb4a65be4e6 Merge 42292b7a430460b30d12bc4c03b30eeeec1e9900 into a767a... by Emma Strubell <emma.strubell@gmail.com>

PiperOrigin-RevId: 221850946
---
 .../opt/python/training/nadam_optimizer.py    |  9 +++--
 .../python/training/nadam_optimizer_test.py   | 37 +++++++++++--------
 2 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer.py b/tensorflow/contrib/opt/python/training/nadam_optimizer.py
index 155ff5b3f4..960826407b 100644
--- a/tensorflow/contrib/opt/python/training/nadam_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/nadam_optimizer.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
@@ -83,14 +84,14 @@ class NadamOptimizer(adam.AdamOptimizer):
     with ops.control_dependencies([m_t]):
       m_t = scatter_add(m, indices, m_scaled_g_values)
       # m_bar = (1 - beta1) * g_t + beta1 * m_t
-      m_bar = m_scaled_g_values + beta1_t * m_t
+      m_bar = m_scaled_g_values + beta1_t * array_ops.gather(m_t, indices)
     # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
     v = self.get_slot(var, "v")
     v_scaled_g_values = (grad * grad) * (1 - beta2_t)
     v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking)
     with ops.control_dependencies([v_t]):
       v_t = scatter_add(v, indices, v_scaled_g_values)
-    v_sqrt = math_ops.sqrt(v_t)
-    var_update = state_ops.assign_sub(
-        var, lr * m_bar / (v_sqrt + epsilon_t), use_locking=self._use_locking)
+    v_t_slice = array_ops.gather(v_t, indices)
+    v_sqrt = math_ops.sqrt(v_t_slice)
+    var_update = scatter_add(var, indices, -lr * m_bar / (v_sqrt + epsilon_t))
     return control_flow_ops.group(*[var_update, m_bar, v_t])
diff --git a/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py b/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py
index 85e05ce71c..a4372f6487 100644
--- a/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py
+++ b/tensorflow/contrib/opt/python/training/nadam_optimizer_test.py
@@ -52,14 +52,19 @@ def nadam_update_numpy(param,
 class NadamOptimizerTest(test.TestCase):
 
   def doTestSparse(self, use_resource=False):
+    # need to use a larger value of epsilon here so that
+    # np.sqrt(v_t) + epsilon doesn't get rounded to 0 when
+    # the dtype is half and np.sqrt(v_t) = 0, as is the case
+    # when the gradient is 0
+    sparse_epsilon = 1e-7
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
         # Initialize variables for numpy implementation.
         m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
-        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
-        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+        var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype)
 
         if use_resource:
           var0 = resource_variable_ops.ResourceVariable(var0_np)
@@ -67,21 +72,21 @@ class NadamOptimizerTest(test.TestCase):
         else:
           var0 = variables.Variable(var0_np)
           var1 = variables.Variable(var1_np)
-        grads0_np_indices = np.array([0, 1], dtype=np.int32)
+        grads0_np_indices = np.array([0, 2], dtype=np.int32)
         grads0 = ops.IndexedSlices(
-            constant_op.constant(grads0_np),
-            constant_op.constant(grads0_np_indices), constant_op.constant([2]))
-        grads1_np_indices = np.array([0, 1], dtype=np.int32)
+            constant_op.constant(grads0_np[grads0_np_indices]),
+            constant_op.constant(grads0_np_indices), constant_op.constant([3]))
+        grads1_np_indices = np.array([0, 2], dtype=np.int32)
         grads1 = ops.IndexedSlices(
-            constant_op.constant(grads1_np),
-            constant_op.constant(grads1_np_indices), constant_op.constant([2]))
-        opt = nadam_optimizer.NadamOptimizer()
+            constant_op.constant(grads1_np[grads1_np_indices]),
+            constant_op.constant(grads1_np_indices), constant_op.constant([3]))
+        opt = nadam_optimizer.NadamOptimizer(epsilon=sparse_epsilon)
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
+        self.assertAllClose([1.0, 1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 3.0, 4.0], var1.eval())
 
         beta1_power, beta2_power = opt._get_beta_accumulators()
 
@@ -91,8 +96,10 @@ class NadamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
           update.run()
 
-          var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0)
-          var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1)
+          var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0,
+                                               epsilon=sparse_epsilon)
+          var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1,
+                                               epsilon=sparse_epsilon)
 
           # Validate updated params
           self.assertAllCloseAccordingToType(var0_np, var0.eval())
-- 
GitLab


From 20df63d2f364088f8f2992f15214538d42d07b41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 14:26:50 -0800
Subject: [PATCH 0442/1554] This CL fixes a bug where GradientTapes return an
 unconnected gradient for no-ops on watched tensors instead of returning a
 ones tensor.

PiperOrigin-RevId: 221852847
---
 tensorflow/c/eager/tape.h                 | 23 ++++++---
 tensorflow/python/eager/backprop_test.py  | 62 +++++++++++++++++++++++
 tensorflow/python/eager/pywrap_tfe_src.cc | 26 +++++++++-
 3 files changed, 103 insertions(+), 8 deletions(-)

diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h
index 5ba55a203f..5c11f51e87 100644
--- a/tensorflow/c/eager/tape.h
+++ b/tensorflow/c/eager/tape.h
@@ -141,8 +141,9 @@ class GradientTape {
   // null. The result is populated with one tensor per target element.
   Status ComputeGradient(
       const VSpace<Gradient, BackwardFunction, TapeTensor>& vspace,
-      gtl::ArraySlice<int64> target_tensor_ids,
-      gtl::ArraySlice<int64> source_tensor_id,
+      const gtl::ArraySlice<int64> target_tensor_ids,
+      const gtl::ArraySlice<int64> source_tensor_ids,
+      const gtl::FlatMap<int64, TapeTensor> sources_that_are_targets,
       gtl::ArraySlice<Gradient*> output_gradients,
       std::vector<Gradient*>* result);
 
@@ -396,6 +397,7 @@ template <typename Gradient, typename BackwardFunction, typename TapeTensor>
 Status InitialGradients(
     const VSpace<Gradient, BackwardFunction, TapeTensor>& vspace,
     gtl::ArraySlice<int64> target_tensor_ids,
+    gtl::FlatMap<int64, TapeTensor> sources_that_are_targets,
     gtl::ArraySlice<Gradient*> output_gradients, const TensorTape& tensor_tape,
     const OpTape<BackwardFunction, TapeTensor>& op_tape,
     gtl::FlatMap<int64, std::vector<Gradient*>>* result) {
@@ -425,8 +427,13 @@ Status InitialGradients(
               "none of operations outputs match expected tensor");
         }
       } else {
-        // No record of the target tensor found on the tape, so no gradient
-        // needs to be computed from it. Do nothing.
+        // This target tensor was not generated by any operation recorded on
+        // the tape, so no gradient needs to be computed from it unless this
+        // target is also a source.
+        auto source_tensor = sources_that_are_targets.find(id);
+        if (source_tensor != sources_that_are_targets.end()) {
+          (*result)[id].push_back(vspace.Ones(source_tensor->second));
+        }
       }
     } else {
       (*result)[id].push_back(output_gradients[i]);
@@ -467,8 +474,9 @@ constexpr int kMinAggregateBytes = 128 * 1024 * 1024;
 template <typename Gradient, typename BackwardFunction, typename TapeTensor>
 Status GradientTape<Gradient, BackwardFunction, TapeTensor>::ComputeGradient(
     const VSpace<Gradient, BackwardFunction, TapeTensor>& vspace,
-    gtl::ArraySlice<int64> target_tensor_ids,
-    gtl::ArraySlice<int64> source_tensor_ids,
+    const gtl::ArraySlice<int64> target_tensor_ids,
+    const gtl::ArraySlice<int64> source_tensor_ids,
+    const gtl::FlatMap<int64, TapeTensor> sources_that_are_targets,
     gtl::ArraySlice<Gradient*> output_gradients,
     std::vector<Gradient*>* result) {
   gtl::FlatSet<int64> sources_set(source_tensor_ids.begin(),
@@ -478,7 +486,8 @@ Status GradientTape<Gradient, BackwardFunction, TapeTensor>::ComputeGradient(
   std::vector<int64> op_stack =
       InitialStack(state.op_tape, state.op_missing_tensor);
   gtl::FlatMap<int64, std::vector<Gradient*>> gradients;
-  Status s = InitialGradients(vspace, target_tensor_ids, output_gradients,
+  Status s = InitialGradients(vspace, target_tensor_ids,
+                              sources_that_are_targets, output_gradients,
                               tensor_tape_, state.op_tape, &gradients);
   auto cleanup = [this, &state]() {
     if (!persistent_) {
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index d3458741bd..d9f2a95828 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -233,6 +233,68 @@ class BackpropTest(test.TestCase):
     self.assertTrue(ordered_variables[0] is v0)
     self.assertTrue(ordered_variables[1] is v1)
 
+  def testTapeNoOpGradient(self):
+    x = constant_op.constant(3.0)
+    with backprop.GradientTape() as t:
+      t.watch(x)
+      y = x
+    self.assertEqual(t.gradient(y, x).numpy(), 1.0)
+
+  def testTapeIdentityGradientIsIdentity(self):
+    x = constant_op.constant(3.0)
+    with backprop.GradientTape() as t:
+      t.watch(x)
+      y = array_ops.identity(x)
+    self.assertEqual(t.gradient(y, x).numpy(), 1.0)
+
+  def testTapeGradientMultiTargetOneIsSource(self):
+    x = constant_op.constant(2.0)
+    with backprop.GradientTape() as t:
+      t.watch(x)
+      y = x*x
+    self.assertEqual(t.gradient([x, y], x).numpy(), 5.0)
+
+  def testTapeNoOpGradientWithMultiTargetAllSource(self):
+    x = constant_op.constant(3.0)
+    with backprop.GradientTape() as t:
+      t.watch(x)
+      y = x
+    self.assertEqual(t.gradient([y, y], x).numpy(), 2.0)
+
+  def testTapeNoOpGradientWithMultiTargetMultiSource(self):
+    x = constant_op.constant(3.0)
+    y = constant_op.constant(5.0)
+    with backprop.GradientTape() as t:
+      t.watch(x)
+      t.watch(y)
+      z = y * y
+    self.assertAllEqual(t.gradient([x, y, z], [x, y]), [1.0, 11.0])
+
+  def testTapeNoOpOnVariableIsIdentity(self):
+    v0 = resource_variable_ops.ResourceVariable(1.0)
+    with backprop.GradientTape() as t:
+      y = v0.read_value()
+    self.assertEqual(t.gradient(y, v0).numpy(), 1.0)
+
+  @test_util.assert_no_new_tensors
+  @test_util.assert_no_garbage_created
+  def testTapeNoOpGradient2By2(self):
+    a_2_by_2 = constant_op.constant(2.0, shape=[2, 2])
+    with backprop.GradientTape(persistent=True) as tape:
+      tape.watch(a_2_by_2)
+    dy_dy = tape.gradient(a_2_by_2, [a_2_by_2])[0]
+    self.assertAllEqual(dy_dy.numpy(),
+                        constant_op.constant(1.0, shape=[2, 2]).numpy())
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
+  def testTapeNoOpGradientMultiTarget2By2(self):
+    a_2_by_2 = constant_op.constant(2.0, shape=[2, 2])
+    with backprop.GradientTape(persistent=True) as tape:
+      tape.watch(a_2_by_2)
+    dy_dy = tape.gradient([a_2_by_2, a_2_by_2], [a_2_by_2])[0]
+    self.assertAllEqual(dy_dy.numpy(),
+                        constant_op.constant(2.0, shape=[2, 2]).numpy())
+
   def testTapeStopRecording(self):
     with backprop.GradientTape() as t:
       x = resource_variable_ops.ResourceVariable(1.0)
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 6ca8eadbde..f074b73a9f 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1645,6 +1645,29 @@ PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* target,
   if (PyErr_Occurred()) {
     return nullptr;
   }
+  tensorflow::gtl::FlatSet<tensorflow::int64> sources_set(sources_vec.begin(),
+                                                          sources_vec.end());
+
+  tensorflow::Safe_PyObjectPtr seq =
+      tensorflow::make_safe(PySequence_Fast(target, "expected a sequence"));
+  int len = PySequence_Fast_GET_SIZE(seq.get());
+  tensorflow::gtl::FlatMap<tensorflow::int64, PyTapeTensor>
+      source_tensors_that_are_targets;
+  for (int i = 0; i < len; ++i) {
+    tensorflow::int64 target_id = target_vec[i];
+    if (sources_set.find(target_id) != sources_set.end()) {
+      auto tensor = PySequence_Fast_GET_ITEM(seq.get(), i);
+      source_tensors_that_are_targets.insert(
+          std::make_pair(target_id, TapeTensorFromTensor(tensor)));
+    }
+    if (PyErr_Occurred()) {
+      return nullptr;
+    }
+  }
+  if (PyErr_Occurred()) {
+    return nullptr;
+  }
+
   std::vector<PyObject*> outgrad_vec;
   if (output_gradients != Py_None) {
     outgrad_vec = MakeTensorList(output_gradients);
@@ -1659,7 +1682,8 @@ PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* target,
   }
   std::vector<PyObject*> result;
   status->status = tape_obj->tape->ComputeGradient(
-      *py_vspace, target_vec, sources_vec, outgrad_vec, &result);
+      *py_vspace, target_vec, sources_vec, source_tensors_that_are_targets,
+      outgrad_vec, &result);
   if (!status->status.ok()) {
     if (PyErr_Occurred()) {
       // Do not propagate the erroneous status as that would swallow the
-- 
GitLab


From 4bb3a95413b94791e4ee3ba8fd40d7a11b9824b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 14:26:55 -0800
Subject: [PATCH 0443/1554] Update all callers of ReplicaContext.merge_call()
 to use the new args/kwargs parameters instead of *args/**kwargs.  Change
 signature of args/kwargs methods to make them optional.

PiperOrigin-RevId: 221852856
---
 .../metrics/python/metrics/classification.py  |  2 +-
 .../contrib/optimizer_v2/optimizer_v2.py      |  3 +-
 tensorflow/python/distribute/values.py        | 10 +++---
 tensorflow/python/keras/metrics.py            |  3 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |  4 +--
 tensorflow/python/ops/metrics_impl.py         |  2 +-
 .../python/training/checkpoint_utils.py       |  2 +-
 tensorflow/python/training/distribute.py      | 31 ++++++++-----------
 tensorflow/python/training/distribute_test.py |  2 +-
 tensorflow/python/training/moving_averages.py |  2 +-
 tensorflow/python/training/optimizer.py       |  2 +-
 11 files changed, 30 insertions(+), 33 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/metrics/classification.py b/tensorflow/contrib/metrics/python/metrics/classification.py
index ac12360865..062deb74b1 100644
--- a/tensorflow/contrib/metrics/python/metrics/classification.py
+++ b/tensorflow/contrib/metrics/python/metrics/classification.py
@@ -175,7 +175,7 @@ def f1_score(labels, predictions, weights=None, num_thresholds=200,
       return best_f1
 
     best_f1 = distribution_strategy_context.get_replica_context().merge_call(
-        f1_across_replicas, values)
+        f1_across_replicas, args=(values,))
 
     update_op = compute_best_f1_score(tp=update_ops['tp'], fp=update_ops['fp'],
                                       fn=update_ops['fn'], name='update')
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 1b6e70d3a0..d6dedc2774 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -892,7 +892,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
       raise ValueError("No gradients provided for any variable: %s." %
                        ([str(v) for _, v in grads_and_vars],))
     return distribute_ctx.get_replica_context().merge_call(
-        self._distributed_apply, filtered, global_step=global_step, name=name)
+        self._distributed_apply, args=(filtered,),
+        kwargs={"global_step": global_step, "name": name})
 
   def _get_or_create_state(self, var_list=None):
     """Either looks up or creates `_OptimizerV2State`.
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index df7e4083bb..27057bb4a6 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -387,7 +387,7 @@ class MirroredVariable(DistributedVariable, Mirrored,
         return strategy.update(self, f, v, *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, *args, **kwargs)
+          merge_fn, args=args, kwargs=kwargs)
 
   def assign_sub(self, *args, **kwargs):
     assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw)
@@ -625,7 +625,7 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
         return strategy.update(self, f, v, *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, *args, **kwargs)
+          merge_fn, args=args, kwargs=kwargs)
 
   @contextlib.contextmanager
   def _handle_graph(self, handle):
@@ -1591,7 +1591,7 @@ class MultiStepContext(object):
         self._last_step_outputs_reduce_ops[name] = reduce_op
 
       distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, output)
+          merge_fn, args=(output,))
 
   @property
   def non_tensor_outputs(self):
@@ -1608,7 +1608,7 @@ class MultiStepContext(object):
         # in a list as reduction doesn't make sense on non tensors.
         self._non_tensor_outputs[name] = distribution.unwrap(value)
       distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, output)
+          merge_fn, args=(output,))
 
 
 def value_container(val):
@@ -1677,7 +1677,7 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
         return strategy.update(self, f, v, *other_args, **other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, *args, **kwargs)
+          merge_fn, args=args, kwargs=kwargs)
 
   def assign_sub(self, *args, **kwargs):
     assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw)
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 5f63192475..7e165a5b5a 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -149,7 +149,8 @@ def result_wrapper(result_fn):
 
       # Wrapping result in merge_call. merge_call is used when we want to leave
       # replica mode and compute a value in cross replica mode.
-      result_t = replica_context.merge_call(merge_fn_wrapper, result_fn, *args)
+      result_t = replica_context.merge_call(
+          merge_fn_wrapper, args=(result_fn,) + args)
     check_is_tensor_or_operation(result_t,
                                  'Metric {0}\'s result'.format(metric_obj.name))
     return result_t
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 8e3aea2846..99902ff9a9 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -572,7 +572,7 @@ def merge_update_step(update_ops, local_step):
     return incre_op
 
   return distribution_strategy_context.get_replica_context().merge_call(
-      merge_update_step_fn, update_ops, local_step)
+      merge_update_step_fn, args=(update_ops, local_step))
 
 
 def merge_grads(grads_and_vars):
@@ -584,7 +584,7 @@ def merge_grads(grads_and_vars):
     return reduced_grads
 
   return distribution_strategy_context.get_replica_context().merge_call(
-      merge_grad_fn, grads_and_vars)
+      merge_grad_fn, args=(grads_and_vars,))
 
 
 def _var_key(var):
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 328a5dcbd8..601b75fe06 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -330,7 +330,7 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args):
     return metric_value
 
   return distribution_strategy_context.get_replica_context().merge_call(
-      fn, *args)
+      fn, args=args)
 
 
 @tf_export('metrics.mean')
diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 2ed7d7f29c..58166dbb68 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -187,7 +187,7 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
     _init_from_checkpoint(None, ckpt_dir_or_file, assignment_map)
   else:
     distribution_strategy_context.get_replica_context().merge_call(
-        _init_from_checkpoint, ckpt_dir_or_file, assignment_map)
+        _init_from_checkpoint, args=(ckpt_dir_or_file, assignment_map))
 
 
 def _init_from_checkpoint(_, ckpt_dir_or_file, assignment_map):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 325a13ca4f..51da8a7eec 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1073,7 +1073,7 @@ class DistributionStrategyExtended(object):
                                           initial_loop_values):
     raise NotImplementedError("must be implemented in descendants")
 
-  def call_for_each_replica(self, fn, args, kwargs):
+  def call_for_each_replica(self, fn, args=(), kwargs=None):
     """Run `fn` once per replica.
 
     `fn` may call `tf.get_replica_context()` to access methods such as
@@ -1117,6 +1117,8 @@ class DistributionStrategyExtended(object):
       Merged return value of `fn` across all replicas.
     """
     _require_cross_replica_context_extended(self)
+    if kwargs is None:
+      kwargs = {}
     return self._call_for_each_replica(fn, args, kwargs)
 
   def _call_for_each_replica(self, fn, args, kwargs):
@@ -1188,7 +1190,7 @@ class DistributionStrategyExtended(object):
         for t, v in value_destination_pairs
     ]
 
-  def update(self, var, fn, args, kwargs, group):
+  def update(self, var, fn, args=(), kwargs=None, group=True):
     """Run `fn` to update `var` using inputs mirrored to the same devices.
 
     If `var` is mirrored across multiple devices, then this implements
@@ -1226,12 +1228,15 @@ class DistributionStrategyExtended(object):
       for ensuring all elements are executed.
     """
     _require_cross_replica_context_extended(self)
+    if kwargs is None:
+      kwargs = {}
     return self._update(var, fn, args, kwargs, group)
 
   def _update(self, var, fn, args, kwargs, group):
     raise NotImplementedError("must be implemented in descendants")
 
-  def update_non_slot(self, colocate_with, fn, args, kwargs, group):
+  def update_non_slot(
+      self, colocate_with, fn, args=(), kwargs=None, group=True):
     """Runs `fn(*args, **kwargs)` on `colocate_with` devices.
 
     Args:
@@ -1246,6 +1251,8 @@ class DistributionStrategyExtended(object):
       Return value of `fn`, possibly merged across devices.
     """
     _require_cross_replica_context_extended(self)
+    if kwargs is None:
+      kwargs = {}
     return self._update_non_slot(colocate_with, fn, args, kwargs, group)
 
   def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
@@ -1383,7 +1390,7 @@ class ReplicaContext(object):
   def __exit__(self, exception_type, exception_value, traceback):
     _pop_per_thread_mode()
 
-  def merge_call(self, merge_fn, *args, **kwargs):
+  def merge_call(self, merge_fn, args=(), kwargs=None):
     """Merge args across replicas and run `merge_fn` in a cross-replica context.
 
     This allows communication and coordination when there are multiple calls
@@ -1412,20 +1419,8 @@ class ReplicaContext(object):
       unpacked.
     """
     require_replica_context(self)
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to merge_call")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to merge_call")
-      kwargs = k
+    if kwargs is None:
+      kwargs = {}
     return self._merge_call(merge_fn, args, kwargs)
 
   def _merge_call(self, merge_fn, args, kwargs):
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index be7c80d1fa..ad4d50c548 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -154,7 +154,7 @@ class DefaultDistributionStrategyTest(test.TestCase):
     replica_ctx = distribution_strategy_context.get_replica_context()
     self.assertIs(distribution_strategy_context._get_default_replica_context(),
                   replica_ctx)
-    self.assertEqual("foo_bar", replica_ctx.merge_call(merge_fn, "bar"))
+    self.assertEqual("foo_bar", replica_ctx.merge_call(merge_fn, args=("bar",)))
     _assert_in_default_state(self)
 
 
diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 957c8810ac..9b5449498b 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -99,7 +99,7 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None):
         value = strategy.reduce(ds_reduce_util.ReduceOp.MEAN, value, v)
         return strategy.update(v, update_fn, value)
 
-      return replica_context.merge_call(merge_fn, variable, value)
+      return replica_context.merge_call(merge_fn, args=(variable, value))
     else:
       strategy = distribution_strategy_context.get_cross_replica_context()
       return strategy.update(variable, update_fn, value)
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index ada8a7d616..6fca4ca7d4 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -565,7 +565,7 @@ class Optimizer(
     if distribute_ctx.has_distribution_strategy():
       grads_and_vars = get_filtered_grad_fn(lambda: grads_and_vars)()
       return distribute_ctx.get_replica_context().merge_call(
-          self._distributed_apply, grads_and_vars, global_step, name)
+          self._distributed_apply, args=(grads_and_vars, global_step, name))
 
     # No DistributionStrategy case.
     grads_and_vars = tuple(grads_and_vars)  # Make sure repeat iteration works.
-- 
GitLab


From 77c683f3d915bc80ff72818bbc5b9305291fb8d1 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 16 Nov 2018 14:59:52 -0800
Subject: [PATCH 0444/1554] Change signature of tf.cond for TF 2.0.

PiperOrigin-RevId: 221858556
---
 tensorflow/python/ops/control_flow_ops.py     | 73 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 0d04f0697d..eab9b3f993 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -1976,7 +1976,7 @@ def _UnpackIfSingleton(res):
 
 # pylint: disable=redefined-outer-name
 # pylint: disable=g-doc-args
-@tf_export("cond")
+@tf_export(v1=["cond"])
 @deprecation.deprecated_args(
     None, "fn1/fn2 are deprecated in favor of the true_fn/false_fn arguments.",
     "fn1", "fn2")
@@ -2173,6 +2173,77 @@ def cond(pred,
 # pylint: enable=redefined-outer-name
 
 
+@tf_export("cond", v1=[])
+def cond_for_tf_v2(pred,
+                   true_fn=None,
+                   false_fn=None,
+                   name=None):
+  """Return `true_fn()` if the predicate `pred` is true else `false_fn()`.
+
+  `true_fn` and `false_fn` both return lists of output tensors. `true_fn` and
+  `false_fn` must have the same non-zero number and type of outputs.
+
+  **WARNING**: Any Tensors or Operations created outside of `true_fn` and
+  `false_fn` will be executed regardless of which branch is selected at runtime.
+
+  Although this behavior is consistent with the dataflow model of TensorFlow,
+  it has frequently surprised users who expected a lazier semantics.
+  Consider the following simple program:
+
+  ```python
+  z = tf.multiply(a, b)
+  result = tf.cond(x < y, lambda: tf.add(x, z), lambda: tf.square(y))
+  ```
+
+  If `x < y`, the `tf.add` operation will be executed and `tf.square`
+  operation will not be executed. Since `z` is needed for at least one
+  branch of the `cond`, the `tf.multiply` operation is always executed,
+  unconditionally.
+
+  Note that `cond` calls `true_fn` and `false_fn` *exactly once* (inside the
+  call to `cond`, and not at all during `Session.run()`). `cond`
+  stitches together the graph fragments created during the `true_fn` and
+  `false_fn` calls with some additional graph nodes to ensure that the right
+  branch gets executed depending on the value of `pred`.
+
+  `tf.cond` supports nested structures as implemented in
+  `tensorflow.python.util.nest`. Both `true_fn` and `false_fn` must return the
+  same (possibly nested) value structure of lists, tuples, and/or named tuples.
+  Singleton lists and tuples form the only exceptions to this: when returned by
+  `true_fn` and/or `false_fn`, they are implicitly unpacked to single values.
+
+  Args:
+    pred: A scalar determining whether to return the result of `true_fn` or
+      `false_fn`.
+    true_fn: The callable to be performed if pred is true.
+    false_fn: The callable to be performed if pred is false.
+    name: Optional name prefix for the returned tensors.
+
+  Returns:
+    Tensors returned by the call to either `true_fn` or `false_fn`. If the
+    callables return a singleton list, the element is extracted from the list.
+
+  Raises:
+    TypeError: if `true_fn` or `false_fn` is not callable.
+    ValueError: if `true_fn` and `false_fn` do not return the same number of
+      tensors, or return tensors of different types.
+
+  Example:
+
+  ```python
+  x = tf.constant(2)
+  y = tf.constant(5)
+  def f1(): return tf.multiply(x, 17)
+  def f2(): return tf.add(y, 23)
+  r = tf.cond(tf.less(x, y), f1, f2)
+  # r is set to f1().
+  # Operations in f2 (e.g., tf.add) are not executed.
+  ```
+
+  """
+  return cond(pred, true_fn=true_fn, false_fn=false_fn, strict=True, name=name)
+
+
 def _resource_safe_shape(t):
   """Returns the shape of t or the variable it points to."""
   if t.dtype == dtypes.resource:
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index fa4a538122..2d2ab7d0d3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -574,7 +574,7 @@ tf_module {
   }
   member_method {
     name: "cond"
-    argspec: "args=[\'pred\', \'true_fn\', \'false_fn\', \'strict\', \'name\', \'fn1\', \'fn2\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'pred\', \'true_fn\', \'false_fn\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "constant"
-- 
GitLab


From 1af863a0fcb18574d77f20ff9b2d440a27efb3fe Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 16 Nov 2018 15:02:18 -0800
Subject: [PATCH 0445/1554] Add true positives, true negatives, false negatives
 metrics to the new metrics module.

PiperOrigin-RevId: 221859053
---
 tensorflow/python/keras/metrics.py      | 164 +++++++++++++++++---
 tensorflow/python/keras/metrics_test.py | 191 ++++++++++++++++++++++++
 2 files changed, 335 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 7e165a5b5a..6916241007 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -277,6 +277,13 @@ class _ConfusionMatrix(Enum):
   FALSE_NEGATIVES = 'fn'
 
 
+def _assert_thresholds_range(thresholds):
+  invalid_thresholds = [t for t in thresholds if t < 0 or t > 1]
+  if any(invalid_thresholds):
+    raise ValueError('Threshold values must be in [0, 1]. Invalid values: {}'
+                     .format(invalid_thresholds))
+
+
 def _update_confusion_matrix_variables(variables_to_update,
                                        y_true,
                                        y_pred,
@@ -794,21 +801,18 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
         sparse_categorical_accuracy, name, dtype=dtype)
 
 
-class FalsePositives(Metric):
-  """Calculates the number of false positives.
+class _ConfusionMatrixConditionCount(Metric):
+  """Calculates the number of the given confusion matrix condition."""
 
-  If `sample_weight` is given, calculates the sum of the weights of
-  false positives. This metric creates one local variable, `false_positives`
-  that is used to keep track of the number of false positives.
-
-  If `sample_weight` is `None`, weights default to 1.
-  Use `sample_weight` of 0 to mask values.
-  """
-
-  def __init__(self, thresholds=None, name=None, dtype=None):
-    """Creates a `FalsePositives` instance.
+  def __init__(self,
+               confusion_matrix_cond,
+               thresholds=None,
+               name=None,
+               dtype=None):
+    """Creates a `_ConfusionMatrixConditionCount` instance.
 
     Args:
+      confusion_matrix_cond: One of `_ConfusionMatrix` conditions.
       thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
         threshold values in [0, 1]. A threshold is compared with prediction
         values to determine the truth value of predictions (i.e., above the
@@ -817,17 +821,17 @@ class FalsePositives(Metric):
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
-    super(FalsePositives, self).__init__(name=name, dtype=dtype)
+    super(_ConfusionMatrixConditionCount, self).__init__(name=name, dtype=dtype)
+    self._confusion_matrix_cond = confusion_matrix_cond
     self.thresholds = [0.5] if thresholds is None else thresholds
-    self.fp = self.add_weight(
-        'false_positives',
+    _assert_thresholds_range(self.thresholds)
+    self.accumulator = self.add_weight(
+        'accumulator',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
 
   def update_state(self, y_true, y_pred, sample_weight=None):
-    """Accumulates false positive statistics.
-
-    `y_true` and `y_pred` should have the same shape.
+    """Accumulates the given confusion matrix condition statistics.
 
     Args:
       y_true: The ground truth values.
@@ -840,11 +844,131 @@ class FalsePositives(Metric):
       Update op.
     """
     return _update_confusion_matrix_variables({
-        _ConfusionMatrix.FALSE_POSITIVES: self.fp
+        self._confusion_matrix_cond: self.accumulator
     }, y_true, y_pred, self.thresholds, sample_weight)
 
   def result(self):
-    return ops.convert_to_tensor(self.fp)
+    return ops.convert_to_tensor(self.accumulator)
+
+
+class FalsePositives(_ConfusionMatrixConditionCount):
+  """Calculates the number of false positives.
+
+  If `sample_weight` is given, calculates the sum of the weights of
+  false positives. This metric creates one local variable, `accumulator`
+  that is used to keep track of the number of false positives.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `FalsePositives` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(FalsePositives, self).__init__(
+        confusion_matrix_cond=_ConfusionMatrix.FALSE_POSITIVES,
+        thresholds=thresholds,
+        name=name,
+        dtype=dtype)
+
+
+class FalseNegatives(_ConfusionMatrixConditionCount):
+  """Calculates the number of false negatives.
+
+  If `sample_weight` is given, calculates the sum of the weights of
+  false negatives. This metric creates one local variable, `accumulator`
+  that is used to keep track of the number of false negatives.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `FalseNegatives` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(FalseNegatives, self).__init__(
+        confusion_matrix_cond=_ConfusionMatrix.FALSE_NEGATIVES,
+        thresholds=thresholds,
+        name=name,
+        dtype=dtype)
+
+
+class TrueNegatives(_ConfusionMatrixConditionCount):
+  """Calculates the number of true negatives.
+
+  If `sample_weight` is given, calculates the sum of the weights of
+  true negatives. This metric creates one local variable, `accumulator`
+  that is used to keep track of the number of true negatives.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `TrueNegatives` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(TrueNegatives, self).__init__(
+        confusion_matrix_cond=_ConfusionMatrix.TRUE_NEGATIVES,
+        thresholds=thresholds,
+        name=name,
+        dtype=dtype)
+
+
+class TruePositives(_ConfusionMatrixConditionCount):
+  """Calculates the number of true positives.
+
+  If `sample_weight` is given, calculates the sum of the weights of
+  true positives. This metric creates one local variable, `true_positives`
+  that is used to keep track of the number of true positives.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `TruePositives` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(TruePositives, self).__init__(
+        confusion_matrix_cond=_ConfusionMatrix.TRUE_POSITIVES,
+        thresholds=thresholds,
+        name=name,
+        dtype=dtype)
 
 
 @tf_export('keras.metrics.binary_accuracy')
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index f506b3041b..c6a49c3c3e 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -541,6 +541,197 @@ class FalsePositivesTest(test.TestCase):
     result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
     self.assertAllClose([125., 42., 12.], self.evaluate(result))
 
+  def test_threshold_limit(self):
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):
+      metrics.FalsePositives(thresholds=[-1, 0.5, 2])
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FalseNegativesTest(test.TestCase):
+
+  def test_config(self):
+    fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9])
+    self.assertEqual(fn_obj.name, 'my_fn')
+    self.assertEqual(len(fn_obj.variables), 1)
+    self.assertEqual(fn_obj.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    fn_obj = metrics.FalseNegatives()
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = fn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fn_obj.result()
+    self.assertAllClose([3.], result)
+
+  def test_weighted(self):
+    fn_obj = metrics.FalseNegatives()
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([5.], self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = fn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fn_obj.result()
+    self.assertAllClose([1., 4., 6.], result)
+
+  def test_weighted_with_thresholds(self):
+    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,))
+
+    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([4., 16., 23.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TrueNegativesTest(test.TestCase):
+
+  def test_config(self):
+    tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9])
+    self.assertEqual(tn_obj.name, 'my_tn')
+    self.assertEqual(len(tn_obj.variables), 1)
+    self.assertEqual(tn_obj.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    tn_obj = metrics.TrueNegatives()
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = tn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tn_obj.result()
+    self.assertAllClose([3.], result)
+
+  def test_weighted(self):
+    tn_obj = metrics.TrueNegatives()
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([4.], self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = tn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tn_obj.result()
+    self.assertAllClose([2., 5., 7.], result)
+
+  def test_weighted_with_thresholds(self):
+    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((0.0, 2.0, 3.0, 5.0),)
+
+    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([5., 15., 23.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TruePositivesTest(test.TestCase):
+
+  def test_config(self):
+    tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9])
+    self.assertEqual(tp_obj.name, 'my_tp')
+    self.assertEqual(len(tp_obj.variables), 1)
+    self.assertEqual(tp_obj.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    tp_obj = metrics.TruePositives()
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = tp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tp_obj.result()
+    self.assertAllClose([7.], result)
+
+  def test_weighted(self):
+    tp_obj = metrics.TruePositives()
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([12.], self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = tp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tp_obj.result()
+    self.assertAllClose([6., 3., 1.], result)
+
+  def test_weighted_with_thresholds(self):
+    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    result = tp_obj(y_true, y_pred, sample_weight=37.)
+    self.assertAllClose([222., 111., 37.], self.evaluate(result))
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 1a90c933a806812228a7b290cf670996cf1f674d Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 16 Nov 2018 15:16:24 -0800
Subject: [PATCH 0446/1554] Rename gfile.Exists to io.gfile.exists for TF 2.0
 API.

PiperOrigin-RevId: 221861380
---
 tensorflow/python/lib/io/file_io.py           |  21 ++-
 .../tools/api/generator/api_init_files.bzl    |   1 +
 .../api/golden/v2/tensorflow.gfile.pbtxt      |   4 -
 .../api/golden/v2/tensorflow.io.gfile.pbtxt   |   7 +
 .../tools/api/golden/v2/tensorflow.io.pbtxt   |   4 +
 .../tools/compatibility/tf_upgrade_v2.py      | 149 ++++++------------
 6 files changed, 82 insertions(+), 104 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt

diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py
index f22fb253e4..c8aa5311d9 100644
--- a/tensorflow/python/lib/io/file_io.py
+++ b/tensorflow/python/lib/io/file_io.py
@@ -241,7 +241,7 @@ class FileIO(object):
     self._writable_file = None
 
 
-@tf_export("gfile.Exists")
+@tf_export(v1=["gfile.Exists"])
 def file_exists(filename):
   """Determines whether a path exists or not.
 
@@ -252,12 +252,29 @@ def file_exists(filename):
     True if the path exists, whether its a file or a directory.
     False if the path does not exist and there are no filesystem errors.
 
+  Raises:
+    errors.OpError: Propagates any errors reported by the FileSystem API.
+  """
+  return file_exists_v2(filename)
+
+
+@tf_export("io.gfile.exists", v1=[])
+def file_exists_v2(path):
+  """Determines whether a path exists or not.
+
+  Args:
+    path: string, a path
+
+  Returns:
+    True if the path exists, whether its a file or a directory.
+    False if the path does not exist and there are no filesystem errors.
+
   Raises:
     errors.OpError: Propagates any errors reported by the FileSystem API.
   """
   try:
     with errors.raise_exception_on_not_ok_status() as status:
-      pywrap_tensorflow.FileExists(compat.as_bytes(filename), status)
+      pywrap_tensorflow.FileExists(compat.as_bytes(path), status)
   except errors.NotFoundError:
     return False
   return True
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 8ce9038d3a..2e3d1d93c4 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -14,6 +14,7 @@ TENSORFLOW_API_INIT_FILES = [
     "experimental/__init__.py",
     "feature_column/__init__.py",
     "gfile/__init__.py",
+    "io/gfile/__init__.py",
     "graph_util/__init__.py",
     "image/__init__.py",
     "io/__init__.py",
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
index c5ef944a97..74d0a0579e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
@@ -8,10 +8,6 @@ tf_module {
     name: "DeleteRecursively"
     argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "Exists"
-    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "Glob"
     argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
new file mode 100644
index 0000000000..59652cb063
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.io.gfile"
+tf_module {
+  member_method {
+    name: "exists"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
index 64b63ed1a4..b27df178b5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
@@ -44,6 +44,10 @@ tf_module {
     name: "VarLenFeature"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "gfile"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "decode_base64"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 19ea541a3f..c0ebfccd9e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -71,6 +71,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "filter": "filters",
             "dilation_rate": "dilations",
         },
+        "tf.gfile.Exists": {
+            "filename": "path",
+        },
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
@@ -84,104 +87,54 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # function_reorders or function_keyword_renames, use the OLD function name.
     # These renames happen after the arguments have been processed.
     self.symbol_renames.update({
-        "tf.contrib.data.AUTOTUNE":
-            "tf.data.experimental.AUTOTUNE",
-        "tf.contrib.data.Counter":
-            "tf.data.experimental.Counter",
-        "tf.contrib.data.CheckpointInputPipelineHook":
-            "tf.data.experimental.CheckpointInputPipelineHook",
-        "tf.contrib.data.CsvDataset":
-            "tf.data.experimental.CsvDataset",
-        "tf.contrib.data.Optional":
-            "tf.data.experimental.Optional",
-        "tf.contrib.data.RandomDataset":
-            "tf.data.experimental.RandomDataset",
-        "tf.contrib.data.Reducer":
-            "tf.data.experimental.Reducer",
-        "tf.contrib.data.SqlDataset":
-            "tf.data.experimental.SqlDataset",
-        "tf.contrib.data.StatsAggregator":
-            "tf.data.experimental.StatsAggregator",
-        "tf.contrib.data.TFRecordWriter":
-            "tf.data.experimental.TFRecordWriter",
-        "tf.contrib.data.assert_element_shape":
-            "tf.data.experimental.assert_element_shape",
-        "tf.contrib.data.batch_and_drop_remainder":
-            "tf.compat.v1.contrib.data.batch_and_drop_remainder",
-        "tf.contrib.data.bucket_by_sequence_length":
-            "tf.data.experimental.bucket_by_sequence_length",
-        "tf.contrib.data.choose_from_datasets":
-            "tf.data.experimental.choose_from_datasets",
-        "tf.contrib.data.copy_to_device":
-            "tf.data.experimental.copy_to_device",
-        "tf.contrib.data.dense_to_sparse_batch":
-            "tf.data.experimental.dense_to_sparse_batch",
-        "tf.contrib.data.enumerate_dataset":
-            "tf.data.experimental.enumerate_dataset",
-        "tf.contrib.data.get_next_as_optional":
-            "tf.data.experimental.get_next_as_optional",
-        "tf.contrib.data.get_single_element":
-            "tf.data.experimental.get_single_element",
-        "tf.contrib.data.group_by_reducer":
-            "tf.data.experimental.group_by_reducer",
-        "tf.contrib.data.group_by_window":
-            "tf.data.experimental.group_by_window",
-        "tf.contrib.data.ignore_errors":
-            "tf.data.experimental.ignore_errors",
-        "tf.contrib.data.latency_stats":
-            "tf.data.experimental.latency_stats",
-        "tf.contrib.data.make_batched_features_dataset":
-            "tf.data.experimental.make_batched_features_dataset",
-        "tf.contrib.data.make_csv_dataset":
-            "tf.data.experimental.make_csv_dataset",
-        "tf.contrib.data.make_saveable_from_iterator":
-            "tf.data.experimental.make_saveable_from_iterator",
-        "tf.contrib.data.map_and_batch":
-            "tf.data.experimental.map_and_batch",
-        "tf.contrib.data.padded_batch_and_drop_remainder":
-            "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
-        "tf.contrib.data.parallel_interleave":
-            "tf.data.experimental.parallel_interleave",
-        "tf.contrib.data.parse_example_dataset":
-            "tf.data.experimental.parse_example_dataset",
-        "tf.contrib.data.prefetch_to_device":
-            "tf.data.experimental.prefetch_to_device",
-        "tf.contrib.data.read_batch_features":
-            "tf.compat.v1.contrib.data.read_batch_features",
-        "tf.contrib.data.reduce_dataset":
-            "tf.compat.v1.contrib.data.reduce_dataset",
-        "tf.contrib.data.rejection_resample":
-            "tf.data.experimental.rejection_resample",
-        "tf.contrib.data.sample_from_datasets":
-            "tf.data.experimental.sample_from_datasets",
-        "tf.contrib.data.scan":
-            "tf.data.experimental.scan",
-        "tf.contrib.data.set_stats_aggregator":
-            "tf.data.experimental.set_stats_aggregator",
-        "tf.contrib.data.shuffle_and_repeat":
-            "tf.data.experimental.shuffle_and_repeat",
-        "tf.contrib.data.sliding_window_batch":
-            "tf.compat.v1.contrib.data.sliding_window_batch",
-        "tf.contrib.data.sloppy_interleave":
-            "tf.compat.v1.contrib.data.sloppy_interleave",
-        "tf.contrib.data.unbatch":
-            "tf.data.experimental.unbatch",
-        "tf.contrib.data.unique":
-            "tf.data.experimental.unique",
-        "tf.quantize_v2":
-            "tf.quantization.quantize",
-        "tf.sparse_concat":
-            "tf.sparse.concat",
-        "tf.sparse_split":
-            "tf.sparse.split",
-        "tf.multinomial":
-            "tf.random.categorical",
-        "tf.random.multinomial":
-            "tf.random.categorical",
-        "tf.load_file_system_library":
-            "tf.load_library",
-        "tf.random.stateless_multinomial":
-            "tf.random.stateless_categorical",
+        "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
+        "tf.contrib.data.Counter": "tf.data.experimental.Counter",
+        "tf.contrib.data.CheckpointInputPipelineHook": "tf.data.experimental.CheckpointInputPipelineHook",
+        "tf.contrib.data.CsvDataset": "tf.data.experimental.CsvDataset",
+        "tf.contrib.data.Optional": "tf.data.experimental.Optional",
+        "tf.contrib.data.RandomDataset": "tf.data.experimental.RandomDataset",
+        "tf.contrib.data.Reducer": "tf.data.experimental.Reducer",
+        "tf.contrib.data.SqlDataset": "tf.data.experimental.SqlDataset",
+        "tf.contrib.data.StatsAggregator": "tf.data.experimental.StatsAggregator",
+        "tf.contrib.data.TFRecordWriter": "tf.data.experimental.TFRecordWriter",
+        "tf.contrib.data.assert_element_shape": "tf.data.experimental.assert_element_shape",
+        "tf.contrib.data.batch_and_drop_remainder": "tf.compat.v1.contrib.data.batch_and_drop_remainder",
+        "tf.contrib.data.bucket_by_sequence_length": "tf.data.experimental.bucket_by_sequence_length",
+        "tf.contrib.data.choose_from_datasets": "tf.data.experimental.choose_from_datasets",
+        "tf.contrib.data.copy_to_device": "tf.data.experimental.copy_to_device",
+        "tf.contrib.data.dense_to_sparse_batch": "tf.data.experimental.dense_to_sparse_batch",
+        "tf.contrib.data.enumerate_dataset": "tf.data.experimental.enumerate_dataset",
+        "tf.contrib.data.get_next_as_optional": "tf.data.experimental.get_next_as_optional",
+        "tf.contrib.data.get_single_element": "tf.data.experimental.get_single_element",
+        "tf.contrib.data.group_by_reducer": "tf.data.experimental.group_by_reducer",
+        "tf.contrib.data.group_by_window": "tf.data.experimental.group_by_window",
+        "tf.contrib.data.ignore_errors": "tf.data.experimental.ignore_errors",
+        "tf.contrib.data.latency_stats": "tf.data.experimental.latency_stats",
+        "tf.contrib.data.make_batched_features_dataset": "tf.data.experimental.make_batched_features_dataset",
+        "tf.contrib.data.make_csv_dataset": "tf.data.experimental.make_csv_dataset",
+        "tf.contrib.data.make_saveable_from_iterator": "tf.data.experimental.make_saveable_from_iterator",
+        "tf.contrib.data.map_and_batch": "tf.data.experimental.map_and_batch",
+        "tf.contrib.data.padded_batch_and_drop_remainder": "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
+        "tf.contrib.data.parallel_interleave": "tf.data.experimental.parallel_interleave",
+        "tf.contrib.data.parse_example_dataset": "tf.data.experimental.parse_example_dataset",
+        "tf.contrib.data.prefetch_to_device": "tf.data.experimental.prefetch_to_device",
+        "tf.contrib.data.read_batch_features": "tf.compat.v1.contrib.data.read_batch_features",
+        "tf.contrib.data.reduce_dataset": "tf.compat.v1.contrib.data.reduce_dataset",
+        "tf.contrib.data.rejection_resample": "tf.data.experimental.rejection_resample",
+        "tf.contrib.data.sample_from_datasets": "tf.data.experimental.sample_from_datasets",
+        "tf.contrib.data.scan": "tf.data.experimental.scan",
+        "tf.contrib.data.set_stats_aggregator": "tf.data.experimental.set_stats_aggregator",
+        "tf.contrib.data.shuffle_and_repeat": "tf.data.experimental.shuffle_and_repeat",
+        "tf.contrib.data.sliding_window_batch": "tf.compat.v1.contrib.data.sliding_window_batch",
+        "tf.contrib.data.sloppy_interleave": "tf.compat.v1.contrib.data.sloppy_interleave",
+        "tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
+        "tf.contrib.data.unique": "tf.data.experimental.unique",
+        "tf.quantize_v2": "tf.quantization.quantize",
+        "tf.sparse_concat": "tf.sparse.concat",
+        "tf.sparse_split": "tf.sparse.split",
+        "tf.multinomial": "tf.random.categorical",
+        "tf.random.multinomial": "tf.random.categorical",
+        "tf.load_file_system_library": "tf.load_library",
     })
     # pylint: enable=line-too-long
 
-- 
GitLab


From dd18b138ffcbf319e0496370bbe7fcb673d71da8 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 16 Nov 2018 15:28:17 -0800
Subject: [PATCH 0447/1554] Implement Keras V2 Adamax optimizer.

PiperOrigin-RevId: 221863075
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  20 ++
 tensorflow/python/keras/optimizer_v2/adam.py  |  15 +-
 .../python/keras/optimizer_v2/adam_test.py    |  24 +-
 .../python/keras/optimizer_v2/adamax.py       | 149 +++++++++
 .../python/keras/optimizer_v2/adamax_test.py  | 309 ++++++++++++++++++
 5 files changed, 497 insertions(+), 20 deletions(-)
 create mode 100644 tensorflow/python/keras/optimizer_v2/adamax.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/adamax_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 7c2b1d7d35..f77f0460eb 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -15,6 +15,7 @@ py_library(
     srcs = [
         "adadelta.py",
         "adam.py",
+        "adamax.py",
         "gradient_descent.py",
         "optimizer_v2.py",
         "rmsprop.py",
@@ -52,6 +53,25 @@ cuda_py_test(
     shard_count = 4,
 )
 
+cuda_py_test(
+    name = "adamax_test",
+    size = "medium",
+    srcs = ["adamax_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+    shard_count = 4,
+)
+
 cuda_py_test(
     name = "adadelta_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 4b3f8563ff..1cab03eec7 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -132,12 +132,6 @@ class Adam(optimizer_v2.OptimizerV2):
         use_locking=self._use_locking)
 
   def _resource_apply_sparse(self, grad, var, indices):
-
-    def _resource_scatter_add(x, i, v):
-      with ops.control_dependencies(
-          [resource_variable_ops.resource_scatter_add(x.handle, i, v)]):
-        return x.value()
-
     var_dtype = var.dtype.base_dtype
     local_step = math_ops.cast(self.iterations + 1, var_dtype)
     beta_1_t = math_ops.cast(self._get_hyper('beta_1'), var_dtype)
@@ -153,20 +147,25 @@ class Adam(optimizer_v2.OptimizerV2):
     m_scaled_g_values = grad * (1 - beta_1_t)
     m_t = state_ops.assign(m, m * beta_1_t, use_locking=self._use_locking)
     with ops.control_dependencies([m_t]):
-      m_t = _resource_scatter_add(m, indices, m_scaled_g_values)
+      m_t = self._resource_scatter_add(m, indices, m_scaled_g_values)
 
     # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
     v = self.get_slot(var, 'v')
     v_scaled_g_values = (grad * grad) * (1 - beta_2_t)
     v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking)
     with ops.control_dependencies([v_t]):
-      v_t = _resource_scatter_add(v, indices, v_scaled_g_values)
+      v_t = self._resource_scatter_add(v, indices, v_scaled_g_values)
 
     v_sqrt = math_ops.sqrt(v_t)
     var_update = state_ops.assign_sub(
         var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
     return control_flow_ops.group(*[var_update, m_t, v_t])
 
+  def _resource_scatter_add(self, x, i, v):
+    with ops.control_dependencies(
+        [resource_variable_ops.resource_scatter_add(x.handle, i, v)]):
+      return x.value()
+
   def get_config(self):
     config = super(Adam, self).get_config()
     config.update({
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 70bf127dd8..76d3307e5f 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -67,28 +67,28 @@ class AdamOptimizerTest(test.TestCase):
       with self.cached_session():
         # Initialize variables for numpy implementation.
         m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
-        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
-        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+        var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.0, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.0, 0.01], dtype=dtype.as_numpy_dtype)
 
         var0 = resource_variable_ops.ResourceVariable(var0_np)
         var1 = resource_variable_ops.ResourceVariable(var1_np)
-        grads0_np_indices = np.array([0, 1], dtype=np.int32)
+        grads0_np_indices = np.array([0, 2], dtype=np.int32)
         grads0 = ops.IndexedSlices(
-            constant_op.constant(grads0_np),
-            constant_op.constant(grads0_np_indices), constant_op.constant([2]))
-        grads1_np_indices = np.array([0, 1], dtype=np.int32)
+            constant_op.constant(grads0_np[grads0_np_indices]),
+            constant_op.constant(grads0_np_indices), constant_op.constant([3]))
+        grads1_np_indices = np.array([0, 2], dtype=np.int32)
         grads1 = ops.IndexedSlices(
-            constant_op.constant(grads1_np),
-            constant_op.constant(grads1_np_indices), constant_op.constant([2]))
+            constant_op.constant(grads1_np[grads1_np_indices]),
+            constant_op.constant(grads1_np_indices), constant_op.constant([3]))
         opt = adam.Adam()
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
-        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+        self.assertAllClose([1.0, 1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 3.0, 4.0], self.evaluate(var1))
 
         beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
 
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
new file mode 100644
index 0000000000..539654580e
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -0,0 +1,149 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""AdaMax for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.training import training_ops
+
+
+class AdaMax(adam.Adam):
+  """Optimizer that implements the AdaMax algorithm.
+
+  It is a variant of Adam based on the infinity norm.
+  Default parameters follow those provided in the paper.
+  Adamax is sometimes superior to adam, specially in models with embeddings.
+
+  References
+    see Section 7 of [Kingma et al., 2014](http://arxiv.org/abs/1412.6980)
+      ([pdf](http://arxiv.org/pdf/1412.6980.pdf)).
+  """
+
+  def __init__(self,
+               learning_rate=0.001,
+               beta_1=0.9,
+               beta_2=0.999,
+               epsilon=1e-8,
+               name='AdaMax'):
+    """Construct a new AdaMax optimizer.
+
+    Initialization:
+
+    ```
+    m_0 <- 0 (Initialize initial 1st moment vector)
+    v_0 <- 0 (Initialize the exponentially weighted infinity norm)
+    t <- 0 (Initialize timestep)
+    ```
+
+    The update rule for `variable` with gradient `g` uses an optimization
+    described at the end of section 7.1 of the paper:
+
+    ```
+    t <- t + 1
+
+    m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+    v_t <- max(beta2 * v_{t-1}, abs(g))
+    variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon)
+    ```
+
+    Similar to AdamOptimizer, the epsilon is added for numerical stability
+    (especially to get rid of division by zero when v_t = 0).
+
+    Contrast to AdamOptimizer, the sparse implementation of this algorithm
+    (used when the gradient is an IndexedSlices object, typically because of
+    `tf.gather` or an embedding lookup in the forward pass) only updates
+    variable slices and corresponding `m_t`, `v_t` terms when that part of
+    the variable was used in the forward pass. This means that the sparse
+    behavior is contrast to the dense behavior (similar to some momentum
+    implementations which ignore momentum unless a variable slice was actually
+    used).
+
+    Args:
+      learning_rate: A Tensor or a floating point value.  The learning rate.
+      beta_1: A float value or a constant float tensor. The exponential decay
+        rate for the 1st moment estimates.
+      beta_2: A float value or a constant float tensor. The exponential decay
+        rate for the exponentially weighted infinity norm.
+      epsilon: A small constant for numerical stability.
+      name: Optional name for the operations created when applying gradients.
+        Defaults to "AdaMax".
+    """
+    # pylint: disable=useless-super-delegation
+    super(AdaMax, self).__init__(learning_rate, beta_1, beta_2, epsilon, name)
+    # pylint: enable=useless-super-delegation
+
+  def _resource_apply_dense(self, grad, var):
+    grad_dtype = grad.dtype.base_dtype
+    m = self.get_slot(var, 'm')
+    v = self.get_slot(var, 'v')
+    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
+    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
+    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_power = math_ops.pow(beta_1_t, local_step)
+    return training_ops.resource_apply_ada_max(
+        var.handle,
+        m.handle,
+        v.handle,
+        beta_1_power,
+        math_ops.cast(self._get_hyper('learning_rate'), grad_dtype),
+        beta_1_t,
+        beta_2_t,
+        math_ops.cast(self._get_hyper('epsilon'), grad_dtype),
+        grad,
+        use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices):
+    grad_dtype = grad.dtype.base_dtype
+
+    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
+    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
+    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_power = math_ops.pow(beta_1_t, local_step)
+    lr_t = math_ops.cast(self._get_hyper('learning_rate'), grad_dtype)
+    epsilon_t = math_ops.cast(self._get_hyper('epsilon'), grad_dtype)
+
+    # m_t = beta1 * m + (1 - beta1) * g_t
+    m = self.get_slot(var, 'm')
+    m_slice = array_ops.gather(m, indices)
+    m_t_slice = m_slice * beta_1_t + grad * (1 - beta_1_t)
+    with ops.control_dependencies([m_t_slice]):
+      m_t = self._resource_scatter_update(m, indices, m_t_slice)
+
+    # u_t = max(beta2 * u, abs(g_t))
+    v = self.get_slot(var, 'v')
+    v_slice = array_ops.gather(v, indices)
+    v_t_slice = math_ops.maximum(v_slice * beta_2_t, math_ops.abs(grad))
+    with ops.control_dependencies([v_t_slice]):
+      v_t = self._resource_scatter_update(v, indices, v_t_slice)
+    # theta_t = theta - lr / (1 - beta1^t) * m_t / u_t
+    var_slice = -lr_t / (1 - beta_1_power) * (
+        m_t_slice / (v_t_slice + epsilon_t))
+    with ops.control_dependencies([var_slice]):
+      var_update = self._resource_scatter_add(var, indices, var_slice)
+    return control_flow_ops.group(*[var_update, m_t, v_t])
+
+  def _resource_scatter_update(self, x, i, v):
+    with ops.control_dependencies(
+        [resource_variable_ops.resource_scatter_update(
+            x.handle, i, v)]):
+      return x.value()
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
new file mode 100644
index 0000000000..23eb718429
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -0,0 +1,309 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for AdaMax."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.optimizer_v2 import adamax
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+def adamax_update_numpy(param,
+                        g_t,
+                        t,
+                        m,
+                        v,
+                        alpha=0.001,
+                        beta1=0.9,
+                        beta2=0.999,
+                        epsilon=1e-8):
+  m_t = beta1 * m + (1 - beta1) * g_t
+  v_t = np.maximum(beta2 * v, np.abs(g_t))
+  param_t = param - (alpha / (1 - beta1**t)) * (m_t / (v_t + epsilon))
+  return param_t, m_t, v_t
+
+
+def adamax_sparse_update_numpy(param,
+                               indices,
+                               g_t,
+                               t,
+                               m,
+                               v,
+                               alpha=0.001,
+                               beta1=0.9,
+                               beta2=0.999,
+                               epsilon=1e-8):
+  m_t, v_t, param_t = np.copy(m), np.copy(v), np.copy(param)
+  m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t
+  v_t_slice = np.maximum(beta2 * v[indices], np.abs(g_t))
+  param_t_slice = param[indices] - ((alpha / (1 - beta1**t)) *
+                                    (m_t_slice / (v_t_slice + epsilon)))
+  m_t[indices] = m_t_slice
+  v_t[indices] = v_t_slice
+  param_t[indices] = param_t_slice
+  return param_t, m_t, v_t
+
+
+def get_beta_accumulators(opt, dtype):
+  local_step = math_ops.cast(opt.iterations + 1, dtype)
+  beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype)
+  beta_1_power = math_ops.pow(beta_1_t, local_step)
+  return beta_1_power
+
+
+class AdaMaxOptimizerTest(test.TestCase):
+
+  def doTestSparse(self, use_resource=False):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        zero_slots = lambda: np.zeros((3), dtype=dtype.as_numpy_dtype)  # pylint: disable=cell-var-from-loop
+        m0, v0, m1, v1 = zero_slots(), zero_slots(), zero_slots(), zero_slots()
+        var0_np = np.array([1.0, 2.0, 3.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([4.0, 5.0, 6.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+
+        grads0_np_indices = np.array([0, 1], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np),
+            constant_op.constant(grads0_np_indices), constant_op.constant([3]))
+        grads1_np_indices = np.array([2, 1], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np),
+            constant_op.constant(grads1_np_indices), constant_op.constant([3]))
+        opt = adamax.AdaMax()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0, 3.0], var0.eval())
+        self.assertAllClose([4.0, 5.0, 6.0], var1.eval())
+
+        beta1_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of AdaMax
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = adamax_sparse_update_numpy(
+              var0_np, grads0_np_indices, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adamax_sparse_update_numpy(
+              var1_np, grads1_np_indices, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testResourceSparse(self):
+    self.doTestSparse(use_resource=True)
+
+  def testSparseDevicePlacement(self):
+    for index_dtype in [dtypes.int32, dtypes.int64]:
+      with self.cached_session(force_gpu=test.is_gpu_available()):
+        # If a GPU is available, tests that all optimizer ops can be placed on
+        # it (i.e. they have GPU kernels).
+        var = variables.Variable([[1.0], [2.0]])
+        indices = constant_op.constant([0, 1], dtype=index_dtype)
+        gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices))
+        optimizer = adamax.AdaMax(3.0)
+        minimize_op = optimizer.minimize(gathered_sum, var_list=[var])
+        variables.global_variables_initializer().run()
+        minimize_op.run()
+
+  def testSparseRepeatedIndices(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        repeated_index_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        aggregated_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        grad_repeated_index = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1, 0.1], shape=[2, 1], dtype=dtype),
+            constant_op.constant([1, 1]),
+            constant_op.constant([2, 1]))
+        grad_aggregated = ops.IndexedSlices(
+            constant_op.constant(
+                [0.2], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        repeated_update = adamax.AdaMax().apply_gradients(
+            [(grad_repeated_index, repeated_index_update_var)])
+        aggregated_update = adamax.AdaMax().apply_gradients(
+            [(grad_aggregated, aggregated_update_var)])
+        variables.global_variables_initializer().run()
+        self.assertAllClose(aggregated_update_var.eval(),
+                            repeated_index_update_var.eval())
+        for _ in range(3):
+          repeated_update.run()
+          aggregated_update.run()
+          self.assertAllClose(aggregated_update_var.eval(),
+                              repeated_index_update_var.eval())
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testBasic(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        opt = adamax.AdaMax()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+        if not context.executing_eagerly():
+          self.evaluate(variables.global_variables_initializer())
+          # Fetch params to validate initial values
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+        # Run 3 steps of AdaMax
+        for t in range(1, 4):
+          if not context.executing_eagerly():
+            self.evaluate(update)
+          elif t > 1:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          beta_1_power = get_beta_accumulators(opt, dtype)
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
+
+          var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0),
+                                             rtol=1e-2)
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1),
+                                             rtol=1e-2)
+
+  def testTensorLearningRate(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = adamax.AdaMax(constant_op.constant(0.001))
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of AdaMax
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSharing(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = adamax.AdaMax()
+        update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        beta1_power = get_beta_accumulators(opt, dtype)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 3 steps of intertwined AdaMax1 and AdaMax2.
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          if t % 2 == 0:
+            update1.run()
+          else:
+            update2.run()
+
+          var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSlotsUniqueEager(self):
+    with context.eager_mode():
+      v1 = resource_variable_ops.ResourceVariable(1.)
+      v2 = resource_variable_ops.ResourceVariable(1.)
+      opt = adamax.AdaMax(1.)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, hyper variables, and two unique slot
+      # variables for v1 and v2 respectively.
+      self.assertEqual(9, len(set(opt.variables())))
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 8930d5aff5b3cfcbfdbf2b2bc63414908fc180aa Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 16 Nov 2018 15:46:27 -0800
Subject: [PATCH 0448/1554] Adding `add_metric` API for layers/models in Keras.

Supports
- adding metric tensor (both aggregated and not aggregated) on model/layer in graph mode.
- adding metric tensor (both aggregated and not aggregated) in model/layer call in graph and eager mode.

PiperOrigin-RevId: 221865769
---
 tensorflow/contrib/saved_model/BUILD          |   5 +-
 .../saved_model/keras_saved_model_test.py     |  12 +-
 .../contrib/tpu/python/tpu/keras_support.py   |  26 +-
 tensorflow/python/keras/callbacks.py          |   2 +-
 tensorflow/python/keras/engine/base_layer.py  |  93 +++++
 tensorflow/python/keras/engine/network.py     |  36 ++
 tensorflow/python/keras/engine/saving.py      |   8 +-
 tensorflow/python/keras/engine/training.py    | 169 +++++---
 .../python/keras/engine/training_arrays.py    |   4 +-
 .../keras/engine/training_distributed.py      |  15 +-
 .../python/keras/engine/training_eager.py     |  33 +-
 .../python/keras/engine/training_generator.py |   6 +-
 .../python/keras/engine/training_test.py      | 361 +++++++++++++++++-
 tensorflow/python/keras/metrics.py            |  36 +-
 tensorflow/python/keras/models.py             |  13 +-
 tensorflow/python/keras/models_test.py        |   3 +-
 .../golden/v1/tensorflow.keras.-model.pbtxt   |  12 +
 .../v1/tensorflow.keras.-sequential.pbtxt     |  12 +
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |   4 +
 .../tensorflow.keras.layers.-activation.pbtxt |   4 +
 ...eras.layers.-activity-regularization.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-add.pbtxt     |   4 +
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   4 +
 ...low.keras.layers.-average-pooling1-d.pbtxt |   4 +
 ...low.keras.layers.-average-pooling2-d.pbtxt |   4 +
 ...low.keras.layers.-average-pooling3-d.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-average.pbtxt |   4 +
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   4 +
 ...ow.keras.layers.-batch-normalization.pbtxt |   4 +
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   4 +
 ...tensorflow.keras.layers.-concatenate.pbtxt |   4 +
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-conv1-d.pbtxt |   4 +
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-conv2-d.pbtxt |   4 +
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-conv3-d.pbtxt |   4 +
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   4 +
 ...ras.layers.-convolution2-d-transpose.pbtxt |   4 +
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   4 +
 ...ras.layers.-convolution3-d-transpose.pbtxt |   4 +
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   4 +
 ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt |   4 +
 ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-dense.pbtxt   |   4 +
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-dot.pbtxt     |   4 +
 .../v1/tensorflow.keras.layers.-dropout.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-e-l-u.pbtxt   |   4 +
 .../tensorflow.keras.layers.-embedding.pbtxt  |   4 +
 .../v1/tensorflow.keras.layers.-flatten.pbtxt |   4 +
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-g-r-u.pbtxt   |   4 +
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   4 +
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   4 +
 ...as.layers.-global-average-pooling1-d.pbtxt |   4 +
 ...as.layers.-global-average-pooling2-d.pbtxt |   4 +
 ...as.layers.-global-average-pooling3-d.pbtxt |   4 +
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   4 +
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   4 +
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   4 +
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   4 +
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   4 +
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   4 +
 ....keras.layers.-global-max-pooling1-d.pbtxt |   4 +
 ....keras.layers.-global-max-pooling2-d.pbtxt |   4 +
 ....keras.layers.-global-max-pooling3-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-input-layer.pbtxt |   4 +
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-l-s-t-m.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-lambda.pbtxt  |   4 +
 .../v1/tensorflow.keras.layers.-layer.pbtxt   |   4 +
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   4 +
 ...w.keras.layers.-locally-connected1-d.pbtxt |   4 +
 ...w.keras.layers.-locally-connected2-d.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-masking.pbtxt |   4 +
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   4 +
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   4 +
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   4 +
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-maximum.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-minimum.pbtxt |   4 +
 .../tensorflow.keras.layers.-multiply.pbtxt   |   4 +
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   4 +
 .../v1/tensorflow.keras.layers.-permute.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-r-n-n.pbtxt   |   4 +
 .../v1/tensorflow.keras.layers.-re-l-u.pbtxt  |   4 +
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-reshape.pbtxt |   4 +
 ...flow.keras.layers.-separable-conv1-d.pbtxt |   4 +
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   4 +
 ...ras.layers.-separable-convolution1-d.pbtxt |   4 +
 ...ras.layers.-separable-convolution2-d.pbtxt |   4 +
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |   4 +
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-softmax.pbtxt |   4 +
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   4 +
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   4 +
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   4 +
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |   4 +
 .../tensorflow.keras.layers.-subtract.pbtxt   |   4 +
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   4 +
 ...rflow.keras.layers.-time-distributed.pbtxt |   4 +
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   4 +
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   4 +
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   4 +
 .../v1/tensorflow.keras.layers.-wrapper.pbtxt |   4 +
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   4 +
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   4 +
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   4 +
 .../v1/tensorflow.keras.models.-model.pbtxt   |  12 +
 .../tensorflow.keras.models.-sequential.pbtxt |  12 +
 ...ensorflow.layers.-average-pooling1-d.pbtxt |   4 +
 ...ensorflow.layers.-average-pooling2-d.pbtxt |   4 +
 ...ensorflow.layers.-average-pooling3-d.pbtxt |   4 +
 ...nsorflow.layers.-batch-normalization.pbtxt |   4 +
 .../v1/tensorflow.layers.-conv1-d.pbtxt       |   4 +
 ...tensorflow.layers.-conv2-d-transpose.pbtxt |   4 +
 .../v1/tensorflow.layers.-conv2-d.pbtxt       |   4 +
 ...tensorflow.layers.-conv3-d-transpose.pbtxt |   4 +
 .../v1/tensorflow.layers.-conv3-d.pbtxt       |   4 +
 .../golden/v1/tensorflow.layers.-dense.pbtxt  |   4 +
 .../v1/tensorflow.layers.-dropout.pbtxt       |   4 +
 .../v1/tensorflow.layers.-flatten.pbtxt       |   4 +
 .../golden/v1/tensorflow.layers.-layer.pbtxt  |   4 +
 .../tensorflow.layers.-max-pooling1-d.pbtxt   |   4 +
 .../tensorflow.layers.-max-pooling2-d.pbtxt   |   4 +
 .../tensorflow.layers.-max-pooling3-d.pbtxt   |   4 +
 ...tensorflow.layers.-separable-conv1-d.pbtxt |   4 +
 ...tensorflow.layers.-separable-conv2-d.pbtxt |   4 +
 ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt |   4 +
 ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt |   4 +
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   4 +
 ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt |   4 +
 .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt  |   4 +
 ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt |   4 +
 ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt |   4 +
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   4 +
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   4 +
 .../golden/v2/tensorflow.keras.-model.pbtxt   |  12 +
 .../v2/tensorflow.keras.-sequential.pbtxt     |  12 +
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |   4 +
 .../tensorflow.keras.layers.-activation.pbtxt |   4 +
 ...eras.layers.-activity-regularization.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-add.pbtxt     |   4 +
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   4 +
 ...low.keras.layers.-average-pooling1-d.pbtxt |   4 +
 ...low.keras.layers.-average-pooling2-d.pbtxt |   4 +
 ...low.keras.layers.-average-pooling3-d.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-average.pbtxt |   4 +
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   4 +
 ...ow.keras.layers.-batch-normalization.pbtxt |   4 +
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   4 +
 ...tensorflow.keras.layers.-concatenate.pbtxt |   4 +
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-conv1-d.pbtxt |   4 +
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-conv2-d.pbtxt |   4 +
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-conv3-d.pbtxt |   4 +
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   4 +
 ...ras.layers.-convolution2-d-transpose.pbtxt |   4 +
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   4 +
 ...ras.layers.-convolution3-d-transpose.pbtxt |   4 +
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   4 +
 ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt |   4 +
 ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt |   4 +
 ...sorflow.keras.layers.-dense-features.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-dense.pbtxt   |   4 +
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-dot.pbtxt     |   4 +
 .../v2/tensorflow.keras.layers.-dropout.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-e-l-u.pbtxt   |   4 +
 .../tensorflow.keras.layers.-embedding.pbtxt  |   4 +
 .../v2/tensorflow.keras.layers.-flatten.pbtxt |   4 +
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-g-r-u.pbtxt   |   4 +
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   4 +
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   4 +
 ...as.layers.-global-average-pooling1-d.pbtxt |   4 +
 ...as.layers.-global-average-pooling2-d.pbtxt |   4 +
 ...as.layers.-global-average-pooling3-d.pbtxt |   4 +
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   4 +
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   4 +
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   4 +
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   4 +
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   4 +
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   4 +
 ....keras.layers.-global-max-pooling1-d.pbtxt |   4 +
 ....keras.layers.-global-max-pooling2-d.pbtxt |   4 +
 ....keras.layers.-global-max-pooling3-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-input-layer.pbtxt |   4 +
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-l-s-t-m.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-lambda.pbtxt  |   4 +
 .../v2/tensorflow.keras.layers.-layer.pbtxt   |   4 +
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   4 +
 ...ensorflow.keras.layers.-linear-model.pbtxt |  12 +
 ...w.keras.layers.-locally-connected1-d.pbtxt |   4 +
 ...w.keras.layers.-locally-connected2-d.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-masking.pbtxt |   4 +
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   4 +
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   4 +
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   4 +
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   4 +
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-maximum.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-minimum.pbtxt |   4 +
 .../tensorflow.keras.layers.-multiply.pbtxt   |   4 +
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   4 +
 .../v2/tensorflow.keras.layers.-permute.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-r-n-n.pbtxt   |   4 +
 .../v2/tensorflow.keras.layers.-re-l-u.pbtxt  |   4 +
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-reshape.pbtxt |   4 +
 ...flow.keras.layers.-separable-conv1-d.pbtxt |   4 +
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   4 +
 ...ras.layers.-separable-convolution1-d.pbtxt |   4 +
 ...ras.layers.-separable-convolution2-d.pbtxt |   4 +
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |   4 +
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-softmax.pbtxt |   4 +
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   4 +
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   4 +
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   4 +
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |   4 +
 .../tensorflow.keras.layers.-subtract.pbtxt   |   4 +
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   4 +
 ...rflow.keras.layers.-time-distributed.pbtxt |   4 +
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   4 +
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   4 +
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   4 +
 .../v2/tensorflow.keras.layers.-wrapper.pbtxt |   4 +
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   4 +
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   4 +
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   4 +
 .../v2/tensorflow.keras.models.-model.pbtxt   |  12 +
 .../tensorflow.keras.models.-sequential.pbtxt |  12 +
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   4 +
 ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt |   4 +
 ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt |   4 +
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   4 +
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   4 +
 256 files changed, 1731 insertions(+), 123 deletions(-)

diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index f0947fe423..269443b2c6 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -102,7 +102,10 @@ py_test(
     size = "medium",
     srcs = ["python/saved_model/keras_saved_model_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
+    tags = [
+        "no_oss",  # TODO(b/119349471): Re-enable
+        "no_windows",
+    ],
     deps = [
         ":keras_saved_model",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
index 0619c4ca5a..d8637effe2 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
@@ -348,12 +348,14 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
         # feeding in the inputs and targets.
         loss, predictions, _ = sess.run(
             (outputs['loss'], outputs['predictions/' + output_name],
-             outputs['metrics/mae/update_op']),
-            {inputs[input_name]: input_arr, inputs[target_name]: target_arr})
+             outputs['metrics/mean_absolute_error/update_op']), {
+                 inputs[input_name]: input_arr,
+                 inputs[target_name]: target_arr
+             })
 
         # The metric value should be run after the update op, to ensure that it
         # reflects the correct value.
-        metric_value = sess.run(outputs['metrics/mae/value'])
+        metric_value = sess.run(outputs['metrics/mean_absolute_error/value'])
 
         self.assertEqual(int(train_before_export),
                          sess.run(training_module.get_global_step()))
@@ -368,8 +370,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
         self.assertEqual(int(train_before_export),
                          sess.run(training_module.get_global_step()))
         self.assertIn('loss', outputs)
-        self.assertIn('metrics/mae/update_op', outputs)
-        self.assertIn('metrics/mae/value', outputs)
+        self.assertIn('metrics/mean_absolute_error/update_op', outputs)
+        self.assertIn('metrics/mean_absolute_error/value', outputs)
         self.assertIn('predictions/' + output_name, outputs)
 
         # Train for a step
diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index e1cc17b301..73753cd918 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -1012,9 +1012,10 @@ class TPUFunction(object):
                   optimizer=_replicated_optimizer(self._cloned_optimizer),
                   loss=self.model.loss,
                   loss_weights=self.model.loss_weights,
-                  metrics=metrics_module.clone_metrics(self.model.metrics),
+                  metrics=metrics_module.clone_metrics(
+                      self.model._compile_metrics),
                   weighted_metrics=metrics_module.clone_metrics(
-                      self.model.weighted_metrics),
+                      self.model._compile_weighted_metrics),
                   target_tensors=tpu_targets,
               )
 
@@ -1376,6 +1377,7 @@ class KerasTPUModel(models.Model):
     self.train_function = None
     self._fit_function = None
     self._eval_function = None
+    self._stateful_metric_functions = []
 
     cluster_resolver = strategy._tpu_cluster_resolver
     self._tpu_name_or_address = cluster_resolver.get_master()
@@ -1390,10 +1392,10 @@ class KerasTPUModel(models.Model):
       self.compile(
           self._cpu_model.optimizer,
           self._cpu_model.loss,
-          self._cpu_model.metrics,
+          self._cpu_model._compile_metrics,
           self._cpu_model.loss_weights,
           self._cpu_model.sample_weight_mode,
-          self._cpu_model.weighted_metrics,
+          self._cpu_model._compile_weighted_metrics,
           self._cpu_model.target_tensors,
       )
 
@@ -1697,7 +1699,7 @@ class KerasTPUModel(models.Model):
     callbacks.on_train_begin()
     for epoch in range(initial_epoch, epochs):
       # Reset stateful metrics
-      for m in self.stateful_metric_functions:
+      for m in self.metrics:
         m.reset_states()
       # Update callbacks
       callbacks.on_epoch_begin(epoch)
@@ -1995,14 +1997,14 @@ class KerasTPUModel(models.Model):
     self._optimizer = optimizer
 
   @property
-  def stateful_metric_functions(self):
+  def metrics(self):
     if self._tpu_model:
-      return self._tpu_model.stateful_metric_functions
+      return self._tpu_model.metrics
     return self._stateful_metric_functions
 
-  @stateful_metric_functions.setter
-  def stateful_metric_functions(self, stateful_metric_functions):
-    self._stateful_metric_functions = stateful_metric_functions
+  @metrics.setter
+  def metrics(self, metrics):
+    self._stateful_metric_functions = metrics
 
   def _make_train_function(self):
     if not self.train_function:
@@ -2227,10 +2229,10 @@ def tpu_model(model, strategy=None):
     cpu_model.compile(
         _clone_optimizer(model.optimizer, optimizer_config),
         model.loss,
-        metrics_module.clone_metrics(model.metrics),
+        metrics_module.clone_metrics(model._compile_metrics),
         model.loss_weights,
         model.sample_weight_mode,
-        metrics_module.clone_metrics(model.weighted_metrics),
+        metrics_module.clone_metrics(model._compile_weighted_metrics),
     )
 
   if model_weights:
diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index fde17cb6bc..d90523121b 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -126,7 +126,7 @@ def configure_callbacks(callbacks,
   callback_metrics = []
   # When we have deferred build scenario with iterator input, we will compile
   # when we standardize first batch of data.
-  if mode != 'predict' and model._is_compiled:  # pylint: disable=protected-access
+  if mode != 'predict' and hasattr(model, 'metrics_names'):
     callback_metrics = copy.copy(model.metrics_names)
     if do_validation:
       callback_metrics += ['val_' + n for n in model.metrics_names]
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 9f875cdc3c..5dcbc4d04b 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -68,6 +68,14 @@ class CallConvention(enum.Enum):
   POSITIONAL_ARGUMENTS_ARE_INPUTS = 3
 
 
+def _create_mean_metric(value, name=None):
+  # TODO(psv): Remove this import when b/110718070 is fixed.
+  from tensorflow.python.keras import metrics as metrics_module  # pylint: disable=g-import-not-at-top
+  metric_obj = metrics_module.Mean(name=name)
+  result = metric_obj(value)
+  return metric_obj, result
+
+
 @tf_export('keras.layers.Layer')
 class Layer(checkpointable.CheckpointableBase):
   """Base layer class.
@@ -170,6 +178,13 @@ class Layer(checkpointable.CheckpointableBase):
     # in eager mode or graph mode alternatively, we need to keep track of
     # eager losses and symbolic losses via separate attributes.
     self._eager_losses = []
+    # A list of metric instances corresponding to the symbolic metric tensors
+    # added using the `add_metric` API.
+    self._metrics = []
+    # TODO(psv): Remove this property.
+    # A dictionary that maps metric names to metric result tensors. The results
+    # are the running averages of metric values over an epoch.
+    self._metrics_tensors = {}
     self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name
     self._call_fn_args = function_utils.fn_args(self.call)
     self._compute_previous_mask = ('mask' in self._call_fn_args or
@@ -433,6 +448,84 @@ class Layer(checkpointable.CheckpointableBase):
         else:
           self._losses.append(_tag_unconditional(loss))
 
+  @doc_controls.for_subclass_implementers
+  def add_metric(self, value, aggregation=None, name=None):
+    """Adds metric tensor to the layer.
+
+    Args:
+      value: Metric tensor.
+      aggregation: Sample-wise metric reduction function. If `aggregation=None`,
+        it indicates that the metric tensor provided has been aggregated
+        already. eg, `model.add_metric(BinaryAccuracy(name='acc')(y_true,
+        y_pred))`. If aggregation='mean', the given metric tensor will be
+        sample-wise reduced using `mean` function. eg, `model.add_metric(
+        tf.reduce_mean(outputs), name='output_mean', aggregation='mean')`.
+      name: String metric name.
+
+    Raises:
+      ValueError: If `aggregation` is anything other than None or `mean`.
+    """
+    if aggregation is not None and aggregation != 'mean':
+      raise ValueError(
+          'We currently support only `mean` sample-wise metric aggregation. '
+          'You provided aggregation=`%s`' % aggregation)
+
+    if tf_utils.is_symbolic_tensor(value):
+      self._symbolic_add_metric(value, aggregation, name)
+    else:
+      self._eager_add_metric(value, aggregation, name)
+
+  def _get_existing_metric(self, name=None):
+    match = [m for m in self._metrics if m.name == name]
+    if not match:
+      return
+    if len(match) > 1:
+      raise ValueError(
+          'Please provide different names for the metrics you have added. '
+          'We found {} metrics with the name: "{}"'.format(len(match), name))
+    return match[0]
+
+  def _eager_add_metric(self, value, aggregation=None, name=None):
+    # If the given metric is available in `metrics` list we just update state
+    # on it, otherwise we create a new metric instance and
+    # add it to the `metrics` list.
+    match = self._get_existing_metric(name)
+    if match:
+      match(value)  # Update the metric state.
+      return
+    else:
+      if aggregation is None:
+        raise ValueError('We do not support adding an aggregated metric tensor '
+                         'in `call` in eager execution.')
+      metric_obj, _ = _create_mean_metric(value, name)
+      self._metrics.append(metric_obj)
+
+  def _symbolic_add_metric(self, value, aggregation=None, name=None):
+    if aggregation is None:
+      # Iterate over the metrics and check if the given metric exists already.
+      # This can happen when a metric instance is created in subclassed model
+      # layer `__init__` and we have tracked that instance already in
+      # model.__setattr__.
+      match = self._get_existing_metric(name)
+      if match:
+        result_tensor = value
+        if match.name not in self._metrics_tensors:
+          self._metrics_tensors[match.name] = result_tensor
+          return
+        else:
+          raise ValueError(
+              'We currently do not support reusing a metric instance.')
+      else:
+        # We track the instance using the metadata on the result tensor.
+        result_tensor = value
+        metric_obj = result_tensor._metric_obj
+    else:
+      # If a non-aggregated tensor is given as input (ie. `aggregation` is
+      # explicitly set to `mean`), we wrap the tensor in `Mean` metric.
+      metric_obj, result_tensor = _create_mean_metric(value, name)
+    self._metrics.append(metric_obj)
+    self._metrics_tensors[metric_obj.name] = result_tensor
+
   def get_losses_for(self, inputs):
     """Retrieves losses relevant to a specific set of inputs.
 
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index c016020e06..4163176483 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -134,6 +134,11 @@ class Network(base_layer.Layer):
     self._updates = []  # Used in symbolic mode only.
     self._losses = []
     self._eager_losses = []
+    # A list of metric instances corresponding to the symbolic metric tensors
+    # added using the `add_metric` API.
+    self._metrics = []
+    # A dictionary that maps metric names to metric result tensors.
+    self._metrics_tensors = {}
     self._scope = None  # Never used.
     self._reuse = None  # Never used.
     self._call_is_graph_friendly = True
@@ -412,6 +417,13 @@ class Network(base_layer.Layer):
         else:
           if value not in self._non_trainable_weights:
             self._non_trainable_weights.append(value)
+
+    # Keeping track of metric instance created in subclassed model/layer.
+    # We do this so that we can maintain the correct order of metrics by adding
+    # the instance to the `metrics` list as soon as it is created.
+    from tensorflow.python.keras import metrics as metrics_module  # pylint: disable=g-import-not-at-top
+    if isinstance(value, metrics_module.Metric):
+      self._metrics.append(value)
     super(Network, self).__setattr__(name, value)
 
   @property
@@ -697,6 +709,30 @@ class Network(base_layer.Layer):
         sub_layers=self._layers,
         extra_variables=self._non_trainable_weights + self._trainable_weights)
 
+  @property
+  def metrics(self):
+    """Returns the network's symbolic metrics.
+
+    Model overrides this function to include the metrics from `compile` API.
+    """
+    metrics = []
+    for layer in self.layers:
+      metrics += layer._metrics  # pylint: disable=protected-access
+    return metrics + self._metrics
+
+  @property
+  def _all_metrics_tensors(self):
+    """Returns the network's symbolic metric tensors."""
+    # TODO(psv): Remove this property.
+    metrics_tensors = {}
+    for layer in self.layers:
+      if isinstance(layer, Network):
+        metrics_tensors.update(layer._all_metrics_tensors)
+      else:
+        metrics_tensors.update(layer._metrics_tensors)
+    metrics_tensors.update(self._metrics_tensors)
+    return metrics_tensors
+
   @property
   def input_spec(self):
     """Gets the network's input specs.
diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 61bff7fff2..22c48e3f13 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -79,6 +79,10 @@ def save_model(model, filepath, overwrite=True, include_optimizer=True):
 
   from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
 
+  # TODO(psv) Add warning when we save models that contain non-serializable
+  # entities like metrics added using `add_metric` and losses added using
+  # `add_loss.`
+
   if not isinstance(filepath, h5py.File):
     # If file exists and should not be overwritten.
     if not overwrite and os.path.isfile(filepath):
@@ -126,8 +130,8 @@ def save_model(model, filepath, overwrite=True, include_optimizer=True):
                     'config': model.optimizer.get_config()
                 },
                 'loss': model.loss,
-                'metrics': model.metrics,
-                'weighted_metrics': model.weighted_metrics,
+                'metrics': model._compile_metrics,
+                'weighted_metrics': model._compile_weighted_metrics,
                 'sample_weight_mode': model.sample_weight_mode,
                 'loss_weights': model.loss_weights,
             },
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 4928098901..c2a953f383 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import weakref
 import numpy as np
 
@@ -174,25 +175,66 @@ class Model(Network):
       metric_name = '%s_%s' % (self.output_names[output_index], metric_name)
     j = 1
     base_metric_name = metric_name
-    while metric_name in self.metrics_names:
+    while metric_name in self._compile_metrics_names:
       metric_name = '%s_%d' % (base_metric_name, j)
       j += 1
 
     return metric_name
 
+  @property
+  def metrics(self):
+    """Returns the model's metrics added using `compile`, `add_metric` APIs."""
+    metrics = []
+    if self._is_compiled:
+      metrics += self._compile_stateful_metric_functions
+    return metrics + super(Model, self).metrics
+
+  @property
+  def metrics_names(self):
+    """Returns the model's display labels for all outputs."""
+    metrics_names = []
+    if self._is_compiled:
+      metrics_names += self._compile_metrics_names  # Includes names of losses.
+
+    # Add metric names from layers.
+    for layer in self.layers:
+      metrics_names += [m.name for m in layer._metrics]  # pylint: disable=protected-access
+    metrics_names += [m.name for m in self._metrics]
+    return metrics_names
+
+  @property
+  def _all_metrics_tensors(self):
+    """Returns the network's symbolic metric tensors."""
+    metrics_tensors = {}
+    if self._is_compiled:
+      metrics_tensors.update(self._compile_metrics_tensors)
+    metrics_tensors.update(super(Model, self)._all_metrics_tensors)
+    return metrics_tensors
+
+  @property
+  def _all_stateful_metrics_tensors(self):
+    """Returns the network's symbolic metric tensors."""
+    metrics_tensors = {}
+    if self._is_compiled:
+      metrics_tensors.update(self._compile_stateful_metrics_tensors)
+    metrics_tensors.update(super(Model, self)._all_metrics_tensors)
+    return metrics_tensors
+
   def _init_metric_attributes(self):
     """Initialized model metric attributes."""
     # List of all metric names in the model.
-    self.metrics_names = ['loss']
-    # List of all aggregated metric result tensors. This includes aggregated
-    # loss result tensors.
-    self._stateful_metrics_tensors = []
-    # List of all metric result tensors (aggregated or not - based on the
-    # values given in compile.)
-    self.metrics_tensors = []
+    self._compile_metrics_names = ['loss']
     # List of stateful metric functions. Used for resetting metric state during
-    # training/eval. This includes loss functions.
-    self.stateful_metric_functions = []
+    # training/eval.
+    # This includes loss functions when there are multiple outputs.
+    self._compile_stateful_metric_functions = []
+    # Dict of all aggregated metric result tensors. This includes aggregated
+    # loss result tensors when there are multiple outputs.
+    self._compile_stateful_metrics_tensors = {}
+    # Dict of all metric result tensors (aggregated or not - based on the
+    # values given in compile.). This includes aggregated loss result tensors
+    # when there are multiple outputs.
+    self._compile_metrics_tensors = {}
 
   def _set_per_output_metric_attributes(self, metrics_dict, output_index):
     """Sets the metric attributes on the model for the given output.
@@ -201,24 +243,39 @@ class Model(Network):
       metrics_dict: A dict with metric names as keys and metric fns as values.
       output_index: The index of the model output for which the metric
         attributes are added.
+
+    Returns:
+      Metrics dict updated with unique metric names as keys.
     """
-    for metric_name, (_, stateful_metric_fn) in metrics_dict.items():
+    updated_metrics_dict = collections.OrderedDict()
+    for metric_name, (metric_fn, stateful_metric_fn) in metrics_dict.items():
       metric_name = self._add_unique_metric_name(metric_name, output_index)
-      # Keep track of metric name.
-      self.metrics_names.append(metric_name)
-
-      # Keep track of stateful metric function.
-      self.stateful_metric_functions.append(stateful_metric_fn)
+      updated_metrics_dict[metric_name] = (metric_fn, stateful_metric_fn)
+      # Keep track of metric name, function and stateful function.
+      self._compile_metrics_names.append(metric_name)
+      self._compile_stateful_metric_functions.append(stateful_metric_fn)
+    return updated_metrics_dict
 
   def _set_metric_attributes(self, outputs, skip_target_indices=None):
     """Sets the metric attributes on the model for all the model outputs."""
     skip_target_indices = skip_target_indices or []
+    updated_per_output_metrics = []
+    updated_per_output_weighted_metrics = []
     for i in range(len(outputs)):
       if i in skip_target_indices:
+        updated_per_output_metrics.append(self._per_output_metrics[i])
+        updated_per_output_weighted_metrics.append(
+            self._per_output_weighted_metrics[i])
         continue
-      self._set_per_output_metric_attributes(self._per_output_metrics[i], i)
-      self._set_per_output_metric_attributes(
-          self._per_output_weighted_metrics[i], i)
+      updated_per_output_metrics.append(
+          self._set_per_output_metric_attributes(self._per_output_metrics[i],
+                                                 i))
+      updated_per_output_weighted_metrics.append(
+          self._set_per_output_metric_attributes(
+              self._per_output_weighted_metrics[i], i))
+
+    self._per_output_metrics = updated_per_output_metrics
+    self._per_output_weighted_metrics = updated_per_output_weighted_metrics
 
   def _handle_per_output_metrics(self,
                                  metrics_dict,
@@ -253,16 +310,16 @@ class Model(Network):
           weighted_metric_fn = training_utils.weighted_masked_objective(fn)
           return weighted_metric_fn(y_true, y_pred, weights=weights, mask=mask)
 
-        def _track_metric_tensors(stateless_result, stateful_result):
-          self.metrics_tensors.append(stateless_result)
-          self._stateful_metrics_tensors.append(stateful_result)
+        def _track_metric_tensors(name, stateless_result, stateful_result):
+          self._compile_metrics_tensors[name] = stateless_result
+          self._compile_stateful_metrics_tensors[name] = stateful_result
 
         if isinstance(metric_fn, metrics_module.Metric):
           # If the given metric fn is stateful, call the fn and return result.
           metric_result = _call_stateful_fn(metric_fn)
           metric_results.append(metric_result)
           if not self.run_eagerly:
-            _track_metric_tensors(metric_result, metric_result)
+            _track_metric_tensors(metric_name, metric_result, metric_result)
         elif self.run_eagerly:
           # In eager mode, if the given metric fn is not stateful, we invoke the
           # given fn or its stateful version based on the given flag.
@@ -276,7 +333,8 @@ class Model(Network):
           # stateless fns.
           stateful_metric_result = _call_stateful_fn(stateful_fn)
           metric_result = _call_stateless_fn(metric_fn)
-          _track_metric_tensors(metric_result, stateful_metric_result)
+          _track_metric_tensors(metric_name, metric_result,
+                                stateful_metric_result)
 
     return metric_results
 
@@ -304,6 +362,7 @@ class Model(Network):
     skip_target_indices = skip_target_indices or []
     metric_results = []
     with K.name_scope('metrics'):
+      # Invoke all metrics added using `compile`.
       for i in range(len(outputs)):
         if i in skip_target_indices:
           continue
@@ -325,6 +384,12 @@ class Model(Network):
                 output_mask,
                 weights=sample_weights[i],
                 return_stateful_result=return_stateful_result))
+
+    # Add metric results from the `add_metric` metrics in eager mode.
+    if context.executing_eagerly():
+      for m in self.metrics:
+        if m not in self._compile_stateful_metric_functions:
+          metric_results.append(m.result())
     return metric_results
 
   @property
@@ -461,10 +526,10 @@ class Model(Network):
       self._track_checkpointable(
           self.optimizer, name='optimizer', overwrite=True)
     self.loss = loss
-    self.metrics = metrics or []
+    self._compile_metrics = metrics or []
     self.loss_weights = loss_weights
     self.sample_weight_mode = sample_weight_mode
-    self.weighted_metrics = weighted_metrics
+    self._compile_weighted_metrics = weighted_metrics
     if self.run_eagerly and target_tensors is not None:
       raise ValueError(
           'target_tensors argument is not supported when '
@@ -573,7 +638,7 @@ class Model(Network):
       self.total_loss = None
       for i in range(len(self.outputs)):
         if len(self.outputs) > 1:
-          self.metrics_names.append(self.output_names[i] + '_loss')
+          self._compile_metrics_names.append(self.output_names[i] + '_loss')
 
       # Set metric attributes on model.
       self._set_metric_attributes(
@@ -666,7 +731,8 @@ class Model(Network):
 
           if len(self.outputs) > 1:
             # Keep track of the un-aggregated loss result tensor.
-            self.metrics_tensors.append(output_loss)
+            self._compile_metrics_tensors[self.output_names[i] +
+                                          '_loss'] = output_loss
 
             # Keep track of stateful result tensor and function for the loss.
             mean_wrapped_loss = metrics_module.MeanMetricWrapper(
@@ -677,10 +743,11 @@ class Model(Network):
                 y_pred,
                 weights=sample_weight,
                 mask=mask)
-            self._stateful_metrics_tensors.append(result_tensor)
-            self.stateful_metric_functions.append(mean_wrapped_loss)
+            self._compile_stateful_metrics_tensors[self.output_names[i] +
+                                                   '_loss'] = result_tensor
+            self._compile_stateful_metric_functions.append(mean_wrapped_loss)
 
-            self.metrics_names.append(self.output_names[i] + '_loss')
+            self._compile_metrics_names.append(self.output_names[i] + '_loss')
           if total_loss is None:
             total_loss = loss_weight * output_loss
           else:
@@ -782,18 +849,24 @@ class Model(Network):
         setattr(self, fn_name, fn)
 
   def _make_train_function(self):
+    metrics_tensors = [
+        self._all_metrics_tensors[m] for m in self.metrics_names[1:]
+    ]
     self._make_train_function_helper('train_function',
-                                     [self.total_loss] + self.metrics_tensors)
+                                     [self.total_loss] + metrics_tensors)
 
   def _make_fit_function(self):
     # TODO(psv/anjalisridhar): Remove updates after we fix b/118841692
     # Stateful metrics updates
     metric_updates = []
-    for m in self.stateful_metric_functions:
+    for m in self.metrics:
       metric_updates += m.updates
+
+    metrics_tensors = [
+        self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:]
+    ]
     self._make_train_function_helper(
-        '_fit_function', [self.total_loss] + self._stateful_metrics_tensors,
-        metric_updates)
+        '_fit_function', [self.total_loss] + metrics_tensors, metric_updates)
 
   def _make_test_function_helper(self, fn_name, outputs, metric_updates=None):
     if not hasattr(self, fn_name):
@@ -819,12 +892,18 @@ class Model(Network):
         setattr(self, fn_name, fn)
 
   def _make_test_function(self):
+    metrics_tensors = [
+        self._all_metrics_tensors[m] for m in self.metrics_names[1:]
+    ]
     self._make_test_function_helper('test_function',
-                                    [self.total_loss] + self.metrics_tensors)
+                                    [self.total_loss] + metrics_tensors)
 
   def _make_eval_function(self):
-    self._make_test_function_helper(
-        '_eval_function', [self.total_loss] + self._stateful_metrics_tensors)
+    metrics_tensors = [
+        self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:]
+    ]
+    self._make_test_function_helper('_eval_function',
+                                    [self.total_loss] + metrics_tensors)
 
   def _make_predict_function(self):
     if not hasattr(self, 'predict_function'):
@@ -1202,12 +1281,14 @@ class Model(Network):
             y = [y]
           target_tensors = [v for v in y if tensor_util.is_tensor(v)]
         is_compile_called = True
-        self.compile(optimizer=self.optimizer,
-                     loss=self.loss,
-                     metrics=self.metrics,
-                     loss_weights=self.loss_weights,
-                     target_tensors=target_tensors,
-                     run_eagerly=self.run_eagerly)
+        self.compile(
+            optimizer=self.optimizer,
+            loss=self.loss,
+            metrics=self._compile_metrics,
+            weighted_metrics=self._compile_weighted_metrics,
+            loss_weights=self.loss_weights,
+            target_tensors=target_tensors,
+            run_eagerly=self.run_eagerly)
 
     # In graph mode, if we had just set inputs and targets as symbolic tensors
     # by invoking build and compile on the model respectively, we do not have to
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 7f4cfc729b..48d6af7ec4 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -299,8 +299,8 @@ def model_iteration(model,
     # Setup work for each epoch
     results = []
     epoch_logs = {}
-    if hasattr(model, 'stateful_metric_functions'):
-      for m in model.stateful_metric_functions:
+    if hasattr(model, 'metrics'):
+      for m in model.metrics:
         m.reset_states()
     callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode)
     progbar.on_epoch_begin(epoch, epoch_logs)
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 0d6f4d772f..9bf41734c3 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -171,7 +171,7 @@ def fit_loop(
 
     for epoch in range(initial_epoch, epochs):
       # Reset stateful metrics
-      for m in model.stateful_metric_functions:
+      for m in model.metrics:
         m.reset_states()
       callbacks.on_epoch_begin(epoch)
       epoch_logs = {}
@@ -311,7 +311,8 @@ def _experimental_fit_loop(
   # Add initial dummy values for loss and other metric tensors.
   initial_loop_values = {}
   initial_loop_values['loss'] = constant_op.constant(1e7)
-  for name, tensor in zip(model.metrics_names[1:], model.metrics_tensors):
+  for name in model.metrics_names[1:]:
+    tensor = model._all_stateful_metrics_tensors[name]
     initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype)
 
   if steps_per_epoch is None:
@@ -480,7 +481,7 @@ def test_loop(model, iterator, verbose=0, steps=None):
         len(model.outputs) * current_strategy.num_replicas_in_sync)]
     ins = dataset_inputs + dataset_targets + sample_weights
 
-    for m in model.stateful_metric_functions:
+    for m in model.metrics:
       m.reset_states()
 
     outs = []
@@ -590,7 +591,8 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None,
   # Add initial dummy values for loss and other metric tensors.
   initial_loop_values = {}
   initial_loop_values['loss'] = constant_op.constant(1e7)
-  for name, tensor in zip(model.metrics_names[1:], model.metrics_tensors):
+  for name in model.metrics_names[1:]:
+    tensor = model._all_stateful_metrics_tensors[name]
     initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype)
 
   with current_strategy.scope():
@@ -867,10 +869,11 @@ def _clone_and_build_model(model, inputs=None, targets=None):
   cloned_model.compile(
       optimizer,
       model.loss,
-      metrics=metrics_module.clone_metrics(model.metrics),
+      metrics=metrics_module.clone_metrics(model._compile_metrics),
       loss_weights=model.loss_weights,
       sample_weight_mode=model.sample_weight_mode,
-      weighted_metrics=metrics_module.clone_metrics(model.weighted_metrics),
+      weighted_metrics=metrics_module.clone_metrics(
+          model._compile_weighted_metrics),
       target_tensors=targets)
   return cloned_model
 
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index 9cb752a5c6..b0f40dcb4f 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -254,16 +254,25 @@ def iterator_fit_loop(model,
           if val is not None else None for val in sample_weights
       ]
 
-    # Set stateful_metrics in callbacks. We do not do this before the
-    # `steps_per_epoch` loop because model will be compiled only in the first
-    # iteration of this loop in the deferred build scenario.
+    # Train model.
+    outs, loss, _, aggregated_loss_metrics, masks = _process_single_batch(
+        model,
+        x,
+        y,
+        output_loss_metrics=output_loss_metrics,
+        sample_weights=sample_weights,
+        training=True)
+    outs = generic_utils.to_list(outs)
+
     if step_index == 0:
+      # Set stateful_metrics in callbacks. We do not do this before the
+      # `steps_per_epoch` loop because model will be compiled only in the first
+      # iteration of this loop in the deferred build scenario.
       for cbk in callbacks:
         if (isinstance(cbk, cbks.BaseLogger) or
             isinstance(cbk, cbks.ProgbarLogger)):
           cbk.stateful_metrics = model.metrics_names[1:]  # Exclude `loss`
 
-    if step_index == 0 and not callbacks.params['metrics']:
       callback_metrics = copy.copy(model.metrics_names)
       if do_validation:
         callback_metrics += ['val_' + n for n in model.metrics_names]
@@ -277,16 +286,6 @@ def iterator_fit_loop(model,
           'validation_steps': validation_steps
       })
 
-    # Train model.
-    outs, loss, _, aggregated_loss_metrics, masks = _process_single_batch(
-        model,
-        x,
-        y,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=True)
-    outs = generic_utils.to_list(outs)
-
     # Calculate metrics.
     for l, o in zip(model.metrics_names, outs):
       batch_logs[l] = o
@@ -392,8 +391,8 @@ def iterator_test_loop(model, inputs, steps, verbose=0):
       # Get stateful metrics indices. We do not do this before the `steps` loop
       # because model will be compiled only in the first iteration of this loop
       # in the deferred build scenario.
-      if hasattr(model, 'metrics'):
-        for m in model.stateful_metric_functions:
+      if hasattr(model, '_compile_metrics'):
+        for m in model.metrics:
           m.reset_states()
       for m in output_loss_metrics:
         m.reset_states()
@@ -750,7 +749,7 @@ def fit_loop(model,
     for epoch in range(initial_epoch, epochs):
       if model._is_compiled:  # Model may not be compiled the first time.
         # Reset stateful metrics
-        for m in model.stateful_metric_functions:
+        for m in model.metrics:
           m.reset_states()
 
       for m in output_loss_metrics:
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index b5e3a03976..21f2b9dfe3 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -137,7 +137,7 @@ def fit_generator(model,
     # Construct epoch logs.
     epoch_logs = {}
     while epoch < epochs:
-      for m in model.stateful_metric_functions:
+      for m in model.metrics:
         m.reset_states()
       callbacks.on_epoch_begin(epoch)
       steps_done = 0
@@ -240,8 +240,8 @@ def evaluate_generator(model,
   if not context.executing_eagerly():
     model._make_test_function()
 
-  if hasattr(model, 'metrics'):
-    for m in model.stateful_metric_functions:
+  if hasattr(model, '_compile_metrics'):
+    for m in model.metrics:
       m.reset_states()
 
   steps_done = 0
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 359b4da0db..cd25707f51 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -37,6 +37,7 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.callbacks import Callback
 from tensorflow.python.keras.engine.training_utils import weighted_masked_objective
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
@@ -574,6 +575,31 @@ class TrainingTest(test.TestCase):
       # Test with eager execution and iterator
       model.fit(iterator, epochs=1, steps_per_epoch=2)
 
+  def test_losses_in_defun(self):
+    with context.eager_mode():
+      layer = keras.layers.Dense(1, kernel_regularizer='l1')
+      layer(array_ops.ones([1, 10]))
+
+      @function.defun
+      def get_losses():
+        return layer.losses
+
+      self.assertAllEqual(
+          self.evaluate(layer.losses), self.evaluate(get_losses()))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_logging(self):
+    mock_stdout = io.BytesIO() if six.PY2 else io.StringIO()
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(10, activation='relu'))
+    model.add(keras.layers.Dense(1, activation='sigmoid'))
+    model.compile(
+        RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy')
+    with test.mock.patch.object(sys, 'stdout', mock_stdout):
+      model.fit(
+          np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10)
+    self.assertTrue('Epoch 5/10' in mock_stdout.getvalue())
+
 
 class TestExceptionsAndWarnings(test.TestCase):
 
@@ -2248,30 +2274,327 @@ class TestTrainingWithMetrics(test.TestCase):
       scores = model.train_on_batch(x, y, sample_weight=w)
       self.assertArrayNear(scores, [0.2, 0.8], 0.1)
 
+  def test_add_metric_with_tensor_on_model_in_graph_mode(self):
+    with self.cached_session():
+      x = keras.layers.Input(shape=(1,))
+      y = keras.layers.Dense(1, kernel_initializer='ones')(x)
+      model = keras.models.Model(x, y)
+      model.add_metric(
+          math_ops.reduce_sum(y), name='metric_1', aggregation='mean')
+
+      # test with a metric which does not have the standard signature:
+      # (y_true, y_pred, sample_Weight)
+      model.add_metric(metrics_module.Mean(name='metric_2')(y))
+      model.compile('sgd', loss='mse')
+
+      inputs = np.ones(shape=(10, 1))
+      targets = np.ones(shape=(10, 1))
+      history = model.fit(
+          inputs,
+          targets,
+          epochs=2,
+          batch_size=5,
+          validation_data=(inputs, targets))
+      self.assertEqual(history.history['metric_1'][-1], 5)
+      self.assertEqual(history.history['metric_2'][-1], 1)
+      self.assertEqual(history.history['val_metric_1'][-1], 5)
+      self.assertEqual(history.history['val_metric_2'][-1], 1)
+
+      eval_results = model.evaluate(inputs, targets, batch_size=5)
+      self.assertEqual(eval_results[-1], 1)
+      self.assertEqual(eval_results[-2], 5)
+
+      model.predict(inputs, batch_size=5)
+      model.train_on_batch(inputs, targets)
+      model.test_on_batch(inputs, targets)
+
   @tf_test_util.run_in_graph_and_eager_modes
-  def test_logging(self):
-    mock_stdout = io.BytesIO() if six.PY2 else io.StringIO()
-    model = keras.models.Sequential()
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(1, activation='sigmoid'))
-    model.compile(
-        RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy')
-    with test.mock.patch.object(sys, 'stdout', mock_stdout):
-      model.fit(
-          np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10)
-    self.assertTrue('Epoch 5/10' in mock_stdout.getvalue())
+  def test_add_metric_in_model_call(self):
 
-  def test_losses_in_defun(self):
+    class TestModel(keras.Model):
+
+      def __init__(self):
+        super(TestModel, self).__init__(name='test_model')
+        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
+        self.mean = metrics_module.Mean(name='metric_1')
+
+      def call(self, x):
+        self.add_metric(
+            math_ops.reduce_sum(x), name='metric_2', aggregation='mean')
+        # Provide same name as in the instance created in __init__
+        # for eager mode
+        self.add_metric(self.mean(x), name='metric_1')
+        return self.dense1(x)
+
+    model = TestModel()
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+
+    x = np.ones(shape=(10, 1))
+    y = np.ones(shape=(10, 2))
+    history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+    self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
+    self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0)
+    self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0)
+    self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0)
+
+    eval_results = model.evaluate(x, y, batch_size=5)
+    self.assertAlmostEqual(eval_results[1], 1, 0)
+    self.assertAlmostEqual(eval_results[2], 5, 0)
+
+    model.predict(x, batch_size=5)
+    model.train_on_batch(x, y)
+    model.test_on_batch(x, y)
+
+  def test_add_metric_in_model_call_run_eagerly(self):
     with context.eager_mode():
-      layer = keras.layers.Dense(1, kernel_regularizer='l1')
-      layer(array_ops.ones([1, 10]))
 
-      @function.defun
-      def get_losses():
-        return layer.losses
+      class TestModel(keras.Model):
+
+        def __init__(self):
+          super(TestModel, self).__init__(name='test_model')
+          self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
+          self.mean = metrics_module.Mean(name='metric_1')
+
+        def call(self, x):
+          self.add_metric(
+              math_ops.reduce_sum(x), name='metric_2', aggregation='mean')
+          # Provide same name as in the instance created in __init__
+          # for eager mode
+          self.add_metric(self.mean(x), name='metric_1')
+          return self.dense1(x)
+
+      model = TestModel()
+      model.compile(
+          loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True)
+
+      x = np.ones(shape=(10, 1))
+      y = np.ones(shape=(10, 2))
+      history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+      self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
+      self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0)
+      self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0)
+      self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0)
+
+      eval_results = model.evaluate(x, y, batch_size=5)
+      self.assertAlmostEqual(eval_results[1], 1, 0)
+      self.assertAlmostEqual(eval_results[2], 5, 0)
+
+      model.predict(x, batch_size=5)
+      model.train_on_batch(x, y)
+      model.test_on_batch(x, y)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_add_metric_in_layer_call(self):
+
+    class TestLayer(keras.layers.Layer):
+
+      def build(self, input_shape):
+        self.a = self.add_variable(
+            'a', (1, 1), initializer='ones', trainable=False)
+        self.built = True
+
+      def call(self, inputs):
+        self.add_metric(
+            math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean')
+        return inputs + 1
+
+    model = keras.Sequential()
+    model.add(TestLayer(input_shape=(1,)))
+    model.add(keras.layers.Dense(2, kernel_initializer='ones'))
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+
+    x = np.ones(shape=(10, 1))
+    y = np.ones(shape=(10, 2))
+    history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+    self.assertEqual(history.history['metric_1'][-1], 5)
+    self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
+
+  def test_add_metric_in_layer_call_run_eagerly(self):
+    with context.eager_mode():
+
+      class TestLayer(keras.layers.Layer):
+
+        def build(self, input_shape):
+          self.a = self.add_variable(
+              'a', (1, 1), initializer='ones', trainable=False)
+          self.built = True
+
+        def call(self, inputs):
+          self.add_metric(
+              math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean')
+          return inputs + 1
+
+      model = keras.Sequential()
+      model.add(TestLayer(input_shape=(1,)))
+      model.add(keras.layers.Dense(2, kernel_initializer='ones'))
+      model.compile(
+          loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True)
+
+      x = np.ones(shape=(10, 1))
+      y = np.ones(shape=(10, 2))
+      history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+      self.assertEqual(history.history['metric_1'][-1], 5)
+      self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
+
+  def test_model_metrics_list(self):
+    with self.cached_session():
+      x = keras.layers.Input(shape=(1,))
+      y = keras.layers.Dense(1, kernel_initializer='ones')(x)
+      model = keras.models.Model(x, y)
+      model.add_metric(
+          math_ops.reduce_sum(y), name='metric_1', aggregation='mean')
+      model.add_metric(metrics_module.Mean(name='metric_2')(y))
+      model.compile('sgd', loss='mse', metrics=['acc'])
+
+      # Verify that the metrics added using `compile` and `add_metric` API are
+      # included
+      self.assertEqual(model._compile_metrics, ['acc'])
+      names = []
+      for m in model.metrics:
+        if isinstance(m, metrics_module.Metric):
+          names.append(m.name)
+        else:
+          names.append(m.__name__)
+      self.assertEqual(names, ['binary_accuracy', 'metric_1', 'metric_2'])
+
+  def test_model_eager_metrics_list(self):
+    with context.eager_mode():
+
+      class TestModel(keras.Model):
+
+        def __init__(self):
+          super(TestModel, self).__init__(name='test_model')
+          self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
+
+        def call(self, x):
+          self.add_metric(
+              math_ops.reduce_sum(x), name='metric_1', aggregation='mean')
+          return self.dense1(x)
+
+      model = TestModel()
+      model.compile(
+          loss='mse',
+          optimizer=RMSPropOptimizer(0.01),
+          metrics=['acc'],
+          run_eagerly=True)
+      x = np.ones(shape=(10, 1))
+      y = np.ones(shape=(10, 2))
+      model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+
+      self.assertEqual(model._compile_metrics, ['acc'])
+      names = []
+      for m in model.metrics:
+        if isinstance(m, metrics_module.Metric):
+          names.append(m.name)
+        else:
+          names.append(m.__name__)
+      self.assertEqual(names, ['categorical_accuracy', 'metric_1'])
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_multiple_add_metric_calls(self):
+
+    class TestModel(keras.Model):
+
+      def __init__(self):
+        super(TestModel, self).__init__(name='test_model')
+        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
+        self.mean1 = metrics_module.Mean(name='metric_1')
+        self.mean2 = metrics_module.Mean(name='metric_2')
+
+      def call(self, x):
+        self.add_metric(self.mean2(x), name='metric_2')
+        self.add_metric(self.mean1(x), name='metric_1')
+        self.add_metric(
+            math_ops.reduce_sum(x), name='metric_3', aggregation='mean')
+        return self.dense1(x)
+
+    model = TestModel()
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+
+    x = np.ones(shape=(10, 1))
+    y = np.ones(shape=(10, 2))
+    history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+    self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
+    self.assertAlmostEqual(history.history['metric_2'][-1], 1, 0)
+    self.assertAlmostEqual(history.history['metric_3'][-1], 5, 0)
+
+    eval_results = model.evaluate(x, y, batch_size=5)
+    self.assertArrayNear(eval_results[1:4], [1, 1, 5], 0.1)
+
+    model.predict(x, batch_size=5)
+    model.train_on_batch(x, y)
+    model.test_on_batch(x, y)
+
+  def test_invalid_metric_tensor_in_call(self):
+    with context.eager_mode():
+
+      class TestLayer(keras.layers.Layer):
+
+        def call(self, inputs):
+          self.add_metric(metrics_module.Mean(name='metric_1')(inputs))
+          return inputs + 1
+
+      model = keras.Sequential()
+      model.add(TestLayer(input_shape=(1,)))
+      model.add(keras.layers.Dense(2, kernel_initializer='ones'))
+      model.compile(
+          loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True)
+
+      x = np.ones(shape=(10, 1))
+      y = np.ones(shape=(10, 2))
+      with self.assertRaisesRegexp(
+          ValueError,
+          'We do not support adding an aggregated metric tensor in `call` in '
+          'eager execution.'):
+        model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_duplicate_metric_name_in_add_metric(self):
+
+    class TestModel(keras.Model):
+
+      def __init__(self):
+        super(TestModel, self).__init__(name='test_model')
+        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
+        self.mean = metrics_module.Mean(name='metric_1')
+        self.mean2 = metrics_module.Mean(name='metric_1')
+
+      def call(self, x):
+        self.add_metric(self.mean(x), name='metric_1')
+        return self.dense1(x)
+
+    model = TestModel()
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+
+    x = np.ones(shape=(10, 1))
+    y = np.ones(shape=(10, 2))
+    with self.assertRaisesRegexp(
+        ValueError,
+        'Please provide different names for the metrics you have added. '
+        'We found 2 metrics with the name: "metric_1"'):
+      model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_multiple_no_name_input_to_add_metric(self):
+
+    class TestModel(keras.Model):
+
+      def __init__(self):
+        super(TestModel, self).__init__(name='test_model')
+        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
+
+      def call(self, x):
+        self.add_metric(math_ops.reduce_sum(x), aggregation='mean')
+        self.add_metric(math_ops.reduce_sum(x), aggregation='mean')
+        return self.dense1(x)
+
+    model = TestModel()
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+    x = np.ones(shape=(10, 1))
+    y = np.ones(shape=(10, 2))
+    model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
+    self.assertEqual([m.name for m in model.metrics], ['mean', 'mean_1'])
 
-      self.assertAllEqual(self.evaluate(layer.losses),
-                          self.evaluate(get_losses()))
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 6916241007..2c3822e38d 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -545,9 +545,20 @@ class Metric(Layer):
     Returns:
       The metric value tensor.
     """
-    update_op = self.update_state(*args, **kwargs)  # pylint: disable=not-callable
+    update_op = self.update_state(*args, **kwargs)
     with ops.control_dependencies([update_op]):
-      return self.result()  # pylint: disable=not-callable
+      result_t = self.result()
+
+      # We are adding the metric object as metadata on the result tensor.
+      # This is required when we want to use a metric with `add_metric` API on
+      # a Model/Layer in graph mode. This metric instance will later be used
+      # to reset variable state after each epoch of training.
+      # Example:
+      #   model = Model()
+      #   model.add_metric(Mean()(values), name='mean')
+      if not context.executing_eagerly():
+        result_t._metric_obj = self  # pylint: disable=protected-access
+      return result_t
 
   def reset_states(self):
     """Resets all of the metric state variables.
@@ -731,7 +742,8 @@ class MeanMetricWrapper(Mean):
         matches, sample_weight=sample_weight)
 
   def get_config(self):
-    config = self._fn_kwargs
+    config = {'fn': self._fn}
+    config.update(self._fn_kwargs)
     base_config = super(MeanMetricWrapper, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -760,6 +772,12 @@ class BinaryAccuracy(MeanMetricWrapper):
     super(BinaryAccuracy, self).__init__(
         binary_accuracy, name, dtype=dtype, threshold=threshold)
 
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(BinaryAccuracy, cls).from_config(config)
+
 
 class CategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
@@ -783,6 +801,12 @@ class CategoricalAccuracy(MeanMetricWrapper):
     super(CategoricalAccuracy, self).__init__(
         categorical_accuracy, name, dtype=dtype)
 
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(CategoricalAccuracy, cls).from_config(config)
+
 
 class SparseCategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches integer labels.
@@ -800,6 +824,12 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
     super(SparseCategoricalAccuracy, self).__init__(
         sparse_categorical_accuracy, name, dtype=dtype)
 
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(SparseCategoricalAccuracy, cls).from_config(config)
+
 
 class _ConfusionMatrixConditionCount(Metric):
   """Calculates the number of the given confusion matrix condition."""
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 225c6c6af8..3a0c51b497 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -304,8 +304,9 @@ def _in_place_subclassed_model_reset(model):
       attributes_cache[name] = value
       assert value in model._layers
     elif isinstance(
-        value, (list, tuple)) and name not in ('layers', '_layers',
-                                               'stateful_metric_functions'):
+        value,
+        (list, tuple)) and name not in ('layers', '_layers', 'metrics',
+                                        '_compile_stateful_metric_functions'):
       # Handle case: list/tuple of layers (also tracked by the Network API).
       if value and all(isinstance(val, Layer) for val in value):
         raise ValueError('We do not support the use of list-of-layers '
@@ -345,9 +346,6 @@ def _in_place_subclassed_model_reset(model):
           'targets',
           '_feed_targets',
           'sample_weight_modes',
-          'weighted_metrics',
-          'metrics_names',
-          'metrics_tensors',
           'total_loss',
           'sample_weights',
           '_feed_sample_weights',
@@ -495,10 +493,11 @@ def clone_and_build_model(
     clone.compile(
         optimizer,
         model.loss,
-        metrics=metrics_module.clone_metrics(model.metrics),
+        metrics=metrics_module.clone_metrics(model._compile_metrics),
         loss_weights=model.loss_weights,
         sample_weight_mode=model.sample_weight_mode,
-        weighted_metrics=metrics_module.clone_metrics(model.weighted_metrics),
+        weighted_metrics=metrics_module.clone_metrics(
+            model._compile_weighted_metrics),
         target_tensors=target_tensors)
 
   return clone
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index bf778f1497..4b6bb74ef9 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -331,7 +331,8 @@ class TestCloneAndBuildModel(test.TestCase):
     self.assertEqual('mse', model.loss)
     self.assertTrue(
         isinstance(model.optimizer, keras.optimizers.RMSprop))
-    self.assertEqual(['acc', metrics.categorical_accuracy], model.metrics)
+    self.assertEqual(['acc', metrics.categorical_accuracy],
+                     model._compile_metrics)
 
   def _clone_and_build_test_helper(self, model, is_subclassed=False):
     inp = np.random.random((10, 4))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index 75944663e4..9dc8daea5c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -41,6 +41,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -109,6 +117,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 9e3e862b79..a357a82515 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -42,6 +42,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -114,6 +122,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index db69e25c5b..1d814b2c8b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
index 5510465d7b..b84629540e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
index 38ec8a0aff..5918a13ad8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
index 41cb8e30bf..599da06427 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 9a7aaa8e96..f9ff1538c8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index 014f5828fa..723fc9cdb0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index cc303bf7b9..957ce2f0ce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 628447ce35..a52c0af681 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
index f03c986c22..a004db62dd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index a6e4856de9..44f83d1387 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index a01eaf8a12..8378faf718 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index 0d6698f2ef..9d5655c964 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
index f1b23be48f..820034564f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
index 0672cd5b7b..d37a6b4710 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -97,6 +97,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
index b25ae1e82e..1ad7a91be0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index bb1918eba6..cb9abc2539 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -178,6 +178,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
index 16e0fd5a31..47dba1d81f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index 381839d6de..fd64941896 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
index 543bae6fa9..1b1425d531 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index 2933f9f4b3..1741063fe8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
index 072943dc2c..50feb4f458 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
index 222a1ef4fc..faaa535df9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index 9c9c7461c8..4079329d1e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
index f939067178..32e56696e1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index 44ca598724..381abe7340 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
index 471b18ef85..b3e4bf9689 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
index 0f250a09b7..7aeff8003c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
index f52128483c..a1728d9d4f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
index 98daf3bab1..8d8fd142cc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
index b207c68000..7758209adf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
index 2d7a09ceda..7c463ff125 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
index 3ac3825759..4960d0264e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index 280ec8c25f..8fad7535f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
index 560f66f9c7..5b425f2d4d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
index c0543529c3..f6c4d0a438 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
index 04eb2824b9..82b761fc17 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
index f400432915..c9ff323877 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
index ab176b441a..9b4165d4cb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index c3895a0ac1..f225f7c430 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
index 9e24bb8ae6..855d001700 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -161,6 +161,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index 55e0d7ef02..2c404c99cd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 38fbff5e4a..6f109d59d0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index a8094c0bde..69f8a9031d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index 929f48df23..4299f765e5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index 2e6d59337f..9153a1a240 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 3ebe162f57..625e81fd23 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 4e3e258430..2fc769742c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index fb9166316f..e307a65c7c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index c0a53b847b..4394ad0364 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 87b7f6797a..050ed39fe9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 98bf96fa0c..436191821e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index ff6c6f3ec4..4ba540aa6a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index c9d4158d1c..a2e9322cb3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index 9953102ff9..5d16a57fc1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
index 2617f5a95f..9dd29c1251 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index e9f6ef45aa..0045d5775e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 1b1ccbe118..529c750f98 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -161,6 +161,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
index 2e0b6bac24..d4d1bc6b6b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
index 1e93d1118a..e1f5491180 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
@@ -87,6 +87,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index bfd36012a7..9b69d9a944 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index 5ad5990d7e..fd52259432 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index 40d03369a5..5fc8af0d03 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
index 86666b51bb..7f8932270e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
index d26da270e7..4723b99cb0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 85f23df671..173c5d4a8b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
index 235806b965..14e1899e14 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index 524c5fd69e..a708e652bf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index fda2562fc8..e6706b5cf9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index 71d2d09a8d..a73c082d1b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
index 12949b39a6..f3f195554b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
index ab16d0021e..f345d1d67b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
index 61ccbf5962..31cb8bc177 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
index ce2320d703..44cccc92bd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
index 69848af8cf..b55e191ff1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
index 3358f26aeb..e9575436e5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
index 413f45f018..98223b207f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
index 9c61ff6027..2df918b16b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
index baa91804c4..ce5f9e2129 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 15a5d6ac9e..a0bb917775 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index be43bd5b3c..d7942f201b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index 6105992c7a..f7ac9042d4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 1b6cf1e9ec..e5a9268822 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 29488a37f8..0fe2c974a7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 3d70cf8b65..2ee5873f0f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -149,6 +149,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
index d29731ecf9..5b8f64aa35 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index a6d7494ca7..240cb6e562 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index c36e802693..6226c469f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index 9c46cfe40f..34dabce6d8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index 8982f78794..0ddf628ace 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
index ec2cc50298..12eb35ad15 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index d7bc1980f3..c41020c2b4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
index fec2de6b49..479f89cf6a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -93,6 +93,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 3d285e7f17..233363ce02 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index b05e5ec84d..cb6228ac44 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index 728eca415a..03bad3ccb6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
index da64e77c39..158996792a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index 2f505f9293..63a56cd3ee 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index f82c77072e..965a4cca04 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 54e01a9917..1a62430887 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 5b88db51bf..41d8b2fc95 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -41,6 +41,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -109,6 +117,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index 68e2dbeedd..2cf107a5cd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -42,6 +42,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -114,6 +122,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
index c82e67526b..059c91f724 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
index 1d031cb5f8..d06c8e81ee 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
index a8dda6655d..6be8e7c210 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
index 97f65ed894..b132bd43c4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
index ccd9578f0d..21c695935c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
index 9cbb58d721..f24d030720 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
@@ -100,6 +100,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
index c75ea3911e..0a510ece35 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
index 5dc834e514..d0ee44bed3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
@@ -100,6 +100,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
index 96ab209874..546de3cdab 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
index 7e9656b352..3ad311581e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
index e9a2269a6e..9b83271350 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
index 7d2eaaab2a..87a7fb3d84 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
index 8bc3eb26e9..32b17e90ad 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
index 6a0dcce56a..643c469717 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
index b6c84edf2a..434e25adc1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
index 062a02fa59..089fc6f924 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
@@ -99,6 +99,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
index eaad0fb23e..bc3d58b9ca 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
@@ -100,6 +100,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
index ece28a8ce9..fe7d71af3a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
@@ -100,6 +100,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
index 88b8f37c4f..f7f9978c06 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
@@ -107,6 +107,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
index a4483fefa2..f9e898484b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
@@ -107,6 +107,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 381c4975d7..9e52a42526 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -106,6 +106,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
index 912365a28b..9836433d08 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
@@ -110,6 +110,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
index a4bb3219c7..5fd9b329bd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
@@ -107,6 +107,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
index 715bfd5fc7..76c8cff22b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
@@ -107,6 +107,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
index b66c0f89cc..f53567af52 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
@@ -106,6 +106,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index faeb4f3513..d3b68e4f29 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -105,6 +105,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index caa2e60080..1f7840ab91 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -106,6 +106,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index 75944663e4..9dc8daea5c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -41,6 +41,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -109,6 +117,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 9e3e862b79..a357a82515 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -42,6 +42,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -114,6 +122,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index db69e25c5b..1d814b2c8b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
index 5510465d7b..b84629540e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
index 38ec8a0aff..5918a13ad8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
index 41cb8e30bf..599da06427 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 9a7aaa8e96..f9ff1538c8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index 014f5828fa..723fc9cdb0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index cc303bf7b9..957ce2f0ce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 628447ce35..a52c0af681 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
index f03c986c22..a004db62dd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index a6e4856de9..44f83d1387 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index a01eaf8a12..8378faf718 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index 0d6698f2ef..9d5655c964 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
index f1b23be48f..820034564f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
index 0672cd5b7b..d37a6b4710 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -97,6 +97,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
index b25ae1e82e..1ad7a91be0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index bb1918eba6..cb9abc2539 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -178,6 +178,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
index 16e0fd5a31..47dba1d81f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index 381839d6de..fd64941896 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
index 543bae6fa9..1b1425d531 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index 2933f9f4b3..1741063fe8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
index 072943dc2c..50feb4f458 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
index 222a1ef4fc..faaa535df9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index 9c9c7461c8..4079329d1e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
index f939067178..32e56696e1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index 44ca598724..381abe7340 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
index 471b18ef85..b3e4bf9689 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
index 0f250a09b7..7aeff8003c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
index f52128483c..a1728d9d4f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
index 98daf3bab1..8d8fd142cc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
index b207c68000..7758209adf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
index 2d7a09ceda..7c463ff125 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
@@ -98,6 +98,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
index 35b31bd3ee..0781a93bd5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
index 3ac3825759..4960d0264e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index 280ec8c25f..8fad7535f8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
index 560f66f9c7..5b425f2d4d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
index c0543529c3..f6c4d0a438 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
index 04eb2824b9..82b761fc17 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
index f400432915..c9ff323877 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
index ab176b441a..9b4165d4cb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index c3895a0ac1..f225f7c430 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
index 9e24bb8ae6..855d001700 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -161,6 +161,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index 55e0d7ef02..2c404c99cd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 38fbff5e4a..6f109d59d0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index a8094c0bde..69f8a9031d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index 929f48df23..4299f765e5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index 2e6d59337f..9153a1a240 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 3ebe162f57..625e81fd23 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 4e3e258430..2fc769742c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index fb9166316f..e307a65c7c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index c0a53b847b..4394ad0364 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 87b7f6797a..050ed39fe9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 98bf96fa0c..436191821e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index ff6c6f3ec4..4ba540aa6a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index c9d4158d1c..a2e9322cb3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index 9953102ff9..5d16a57fc1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
index 2617f5a95f..9dd29c1251 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index e9f6ef45aa..0045d5775e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 1b1ccbe118..529c750f98 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -161,6 +161,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
index 2e0b6bac24..d4d1bc6b6b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
index 1e93d1118a..e1f5491180 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
@@ -87,6 +87,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index bfd36012a7..9b69d9a944 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index 93372ff0f9..a94bd5930f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -46,6 +46,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -114,6 +122,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index 5ad5990d7e..fd52259432 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index 40d03369a5..5fc8af0d03 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
index 86666b51bb..7f8932270e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
index d26da270e7..4723b99cb0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 85f23df671..173c5d4a8b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
index 235806b965..14e1899e14 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index 524c5fd69e..a708e652bf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index fda2562fc8..e6706b5cf9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index 71d2d09a8d..a73c082d1b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
index 12949b39a6..f3f195554b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
index ab16d0021e..f345d1d67b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
index 61ccbf5962..31cb8bc177 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
index ce2320d703..44cccc92bd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
index 69848af8cf..b55e191ff1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
index 3358f26aeb..e9575436e5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
index 413f45f018..98223b207f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
index 9c61ff6027..2df918b16b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
index baa91804c4..ce5f9e2129 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 15a5d6ac9e..a0bb917775 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index be43bd5b3c..d7942f201b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index 6105992c7a..f7ac9042d4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 1b6cf1e9ec..e5a9268822 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 29488a37f8..0fe2c974a7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 3d70cf8b65..2ee5873f0f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -149,6 +149,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
index d29731ecf9..5b8f64aa35 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index a6d7494ca7..240cb6e562 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index c36e802693..6226c469f8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index 9c46cfe40f..34dabce6d8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index 8982f78794..0ddf628ace 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
index ec2cc50298..12eb35ad15 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
@@ -89,6 +89,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index d7bc1980f3..c41020c2b4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
index fec2de6b49..479f89cf6a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -93,6 +93,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 3d285e7f17..233363ce02 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index b05e5ec84d..cb6228ac44 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index 728eca415a..03bad3ccb6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
index da64e77c39..158996792a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index 2f505f9293..63a56cd3ee 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index f82c77072e..965a4cca04 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 54e01a9917..1a62430887 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -88,6 +88,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 5b88db51bf..41d8b2fc95 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -41,6 +41,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -109,6 +117,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index 68e2dbeedd..2cf107a5cd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -42,6 +42,14 @@ tf_class {
     name: "losses"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "metrics"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "metrics_names"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "name"
     mtype: "<type \'property\'>"
@@ -114,6 +122,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 381c4975d7..9e52a42526 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -106,6 +106,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
index 912365a28b..9836433d08 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
@@ -110,6 +110,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
index b66c0f89cc..f53567af52 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
@@ -106,6 +106,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index faeb4f3513..d3b68e4f29 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -105,6 +105,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index caa2e60080..1f7840ab91 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -106,6 +106,10 @@ tf_class {
     name: "add_loss"
     argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "add_update"
     argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 2121365e745361889ec9b4862f67b7f0b92e9607 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 16:06:20 -0800
Subject: [PATCH 0449/1554] [TF:XLA] Remove a const_cast from HLO
 rematerialization.

This required making HloSchedule and related classes a sequence of HloInstruction* (instead of const HloInstruction*)

Also, use HloInstructionSequence in a few more places (hlo_rematerialization.cc)

No functional change.

PiperOrigin-RevId: 221868639
---
 .../xla/service/buffer_assignment_test.cc     | 18 ++--
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  8 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    |  2 +-
 .../compiler/xla/service/cpu/ir_emitter.h     |  2 +-
 .../xla/service/gpu/gpu_hlo_schedule.cc       | 18 ++--
 .../xla/service/gpu/gpu_hlo_schedule.h        |  4 +-
 .../xla/service/gpu/gpu_hlo_schedule_test.cc  |  2 +-
 .../xla/service/gpu/thunk_schedule.cc         |  4 +-
 .../compiler/xla/service/gpu/thunk_schedule.h |  2 +-
 .../xla/service/heap_simulator_test.cc        |  6 +-
 .../compiler/xla/service/hlo_computation.cc   |  6 +-
 .../compiler/xla/service/hlo_computation.h    |  2 +-
 .../xla/service/hlo_memory_scheduler.cc       | 82 +++++++++----------
 .../xla/service/hlo_memory_scheduler.h        | 14 ++--
 .../xla/service/hlo_memory_scheduler_test.cc  | 33 ++++----
 tensorflow/compiler/xla/service/hlo_module.h  |  6 +-
 .../compiler/xla/service/hlo_ordering.cc      |  3 +-
 tensorflow/compiler/xla/service/hlo_parser.cc |  6 +-
 .../xla/service/hlo_rematerialization.cc      | 19 ++---
 .../xla/service/hlo_rematerialization.h       |  5 +-
 .../compiler/xla/service/hlo_schedule.cc      | 25 +++---
 .../compiler/xla/service/hlo_schedule.h       | 12 +--
 .../compiler/xla/service/hlo_schedule_test.cc | 14 ++--
 23 files changed, 142 insertions(+), 151 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index b1fc50cb18..8f482e6ba8 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -137,8 +137,7 @@ class BufferAssignmentTest : public HloTestBase {
   }
 
   std::unique_ptr<BufferAssignment> RunBufferAssignmentWithInstructionSequence(
-      HloModule* module,
-      absl::Span<const HloInstruction* const> instruction_sequence,
+      HloModule* module, absl::Span<HloInstruction* const> instruction_sequence,
       int64 alignment = 1) {
     HloSchedule schedule(module);
     schedule.set_sequence(module->entry_computation(), instruction_sequence);
@@ -1853,7 +1852,7 @@ class WhileBufferAssignmentTest : public HloTestBase {
   std::unique_ptr<BufferAssignment> RunBufferAssignment(HloModule* module,
                                                         int64 alignment = 1) {
     HloSchedule schedule =
-        ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie();
+        ScheduleModule(module, ByteSizeOf).ConsumeValueOrDie();
     return BufferAssigner::Run(
                module, absl::make_unique<SequentialHloOrdering>(schedule),
                ByteSizeOf,
@@ -2162,7 +2161,7 @@ TEST_F(WhileBufferAssignmentTest, ColocatedBuffers) {
   // nodes are traversed during BufferAssignment.
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
@@ -2391,15 +2390,16 @@ TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) {
   RunCopyInsertion(module.get());
 
   HloSchedule schedule =
-      ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie();
+      ScheduleModule(module.get(), ByteSizeOf).ConsumeValueOrDie();
 
   // To trigger b/38494731, we want a specific Hlo schedule for the
   // root computation, so we overwrite that entry with a manually
   // crafted sequence.
-  schedule.set_sequence(module->entry_computation(),
-                        {input1, weights1, one, output1, while1->operand(0),
-                         while1, input0, weights0, zero, output0,
-                         while0->operand(0), while0, gte0, gte1, root_add});
+  schedule.set_sequence(
+      module->entry_computation(),
+      {input1, weights1, one, output1, while1->mutable_operand(0), while1,
+       input0, weights0, zero, output0, while0->mutable_operand(0), while0,
+       gte0, gte1, root_add});
 
   // If this ASSERT fails, we constructed a bogus sequence above and this test
   // itself is buggy.
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 4ce5a8a292..b529a6e8da 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -587,9 +587,9 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::RunBackend(
   // Select an order for emitting the HLO instructions for each
   // computation. Using this sequence enables tighter buffer liveness analysis
   // and reduced memory usage (as compared to using DependencyHloOrdering).
-  TF_ASSIGN_OR_RETURN(
-      HloSchedule schedule,
-      ScheduleModule(*module, BufferSizeBytesFunction(), DFSMemoryScheduler));
+  TF_ASSIGN_OR_RETURN(HloSchedule schedule,
+                      ScheduleModule(module.get(), BufferSizeBytesFunction(),
+                                     DFSMemoryScheduler));
 
   // Run buffer allocation on the HLO graph.
   TF_ASSIGN_OR_RETURN(
@@ -779,7 +779,7 @@ CpuCompiler::CompileAheadOfTime(std::unique_ptr<HloModuleGroup> module_group,
     XLA_VLOG_LINES(2, module->ToString());
 
     TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                        ScheduleModule(*module, BufferSizeBytesFunction()));
+                        ScheduleModule(module, BufferSizeBytesFunction()));
 
     // Run buffer analysis on the HLO graph. This analysis figures out which
     // temporary buffers are required to run the computation.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 620c45fa39..9b731b0a6a 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -111,7 +111,7 @@ IrEmitter::IrEmitter(
 StatusOr<llvm::Function*> IrEmitter::EmitComputation(
     HloComputation* computation, const string& function_name_prefix,
     bool is_top_level_computation,
-    const std::vector<const HloInstruction*>* instruction_order) {
+    const std::vector<HloInstruction*>* instruction_order) {
   string function_name = name_uniquer_.GetUniqueName(function_name_prefix);
   VLOG(2) << "Emitting IR for CPU function [" << function_name_prefix
           << "]; ordered? " << (instruction_order != nullptr);
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 136b88ff75..e03ba42b0d 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -101,7 +101,7 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   StatusOr<llvm::Function*> EmitComputation(
       HloComputation* computation, const string& function_name_prefix,
       bool is_top_level_computation,
-      const std::vector<const HloInstruction*>* instruction_order);
+      const std::vector<HloInstruction*>* instruction_order);
 
   llvm::IRBuilder<>* b() { return &b_; }
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
index 91609c730b..1126943624 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
@@ -37,7 +37,7 @@ class GpuHloOrdering : public PredecessorHloOrdering {
  public:
   GpuHloOrdering(const HloModule* module,
                  const StreamAssignment& stream_assignment,
-                 const std::vector<const HloInstruction*>& thunk_launch_order);
+                 const std::vector<HloInstruction*>& thunk_launch_order);
   ~GpuHloOrdering() override = default;
 
   // Only the entry computation can possibly be sequentially ordered, and only
@@ -56,7 +56,7 @@ class GpuHloOrdering : public PredecessorHloOrdering {
 
 GpuHloOrdering::GpuHloOrdering(
     const HloModule* module, const StreamAssignment& stream_assignment,
-    const std::vector<const HloInstruction*>& thunk_launch_order)
+    const std::vector<HloInstruction*>& thunk_launch_order)
     : PredecessorHloOrdering(module) {
   // The entry computation has a total order when there's only one stream.
   if (stream_assignment.StreamCount() == 1) {
@@ -150,7 +150,7 @@ GpuHloOrdering::GpuHloOrdering(
 // However, if the total order is A,B,D,C,E, then C and E can run
 // concurrently.
 void BFSLaunchOrder(const HloComputation* computation,
-                    std::vector<const HloInstruction*>* launch_order) {
+                    std::vector<HloInstruction*>* launch_order) {
   // This topological sort uses two data structures:
   // 1. `incoming_edge_count` which keeps track of the number of incoming
   // edges to each HLO;
@@ -158,9 +158,9 @@ void BFSLaunchOrder(const HloComputation* computation,
   //
   // The sorting algorithm repeatedly pops the top from the queue and deletes
   // that HLO from the graph, making more HLOs incoming-edge free.
-  std::deque<const HloInstruction*> queue;
+  std::deque<HloInstruction*> queue;
   std::unordered_map<const HloInstruction*, int64> incoming_edge_count;
-  for (const auto& hlo : computation->instructions()) {
+  for (auto* hlo : computation->instructions()) {
     if (hlo->operand_count() == 0) {
       queue.push_back(hlo);
     } else {
@@ -172,10 +172,10 @@ void BFSLaunchOrder(const HloComputation* computation,
   }
 
   while (!queue.empty()) {
-    const HloInstruction* x = queue.front();
+    HloInstruction* x = queue.front();
     queue.pop_front();
     launch_order->push_back(x);
-    for (const HloInstruction* y : x->users()) {
+    for (HloInstruction* y : x->users()) {
       --incoming_edge_count[y];
       if (incoming_edge_count[y] == 0) {
         queue.push_back(y);
@@ -195,14 +195,14 @@ StatusOr<std::unique_ptr<GpuHloSchedule>> GpuHloSchedule::Build(
   std::unique_ptr<GpuHloSchedule> schedule(new GpuHloSchedule);
 
   // Initialize thunk_launch_order_, the total order of thunk launches.
-  const HloComputation* entry_computation = module.entry_computation();
+  HloComputation* entry_computation = module.entry_computation();
   if (stream_assignment.StreamCount() == 1) {
     // All kernels are launched on a single stream, so there's no loss of
     // concurrency by optimizing for minimal memory usage.
     TF_ASSIGN_OR_RETURN(
         HloInstructionSequence sequence,
         ScheduleComputation(
-            *entry_computation, [pointer_size](const BufferValue& buffer) {
+            entry_computation, [pointer_size](const BufferValue& buffer) {
               return ShapeUtil::ByteSizeOf(buffer.shape(), pointer_size);
             }));
     schedule->thunk_launch_order_ = sequence.instructions();
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
index 07a7fc67aa..7f224ffe4f 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
@@ -46,7 +46,7 @@ class GpuHloSchedule {
 
   // Returns the total order of thunk launches, represented in terms of HLO
   // instructions.
-  const std::vector<const HloInstruction*>& ThunkLaunchOrder() const {
+  const std::vector<HloInstruction*>& ThunkLaunchOrder() const {
     return thunk_launch_order_;
   }
 
@@ -60,7 +60,7 @@ class GpuHloSchedule {
  private:
   GpuHloSchedule();
 
-  std::vector<const HloInstruction*> thunk_launch_order_;
+  std::vector<HloInstruction*> thunk_launch_order_;
   std::unique_ptr<HloOrdering> hlo_ordering_;
 };
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
index 5f857a1a54..91db7151f2 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
@@ -33,7 +33,7 @@ namespace gpu {
 
 class GpuHloScheduleTest : public HloTestBase {
  protected:
-  using HloVec = std::vector<const HloInstruction*>;
+  using HloVec = std::vector<HloInstruction*>;
 
   // Pre-canned shapes.
   Shape f32_2x2_ = ShapeUtil::MakeShape(F32, {2, 2});
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
index 141f321938..6b2d76764a 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
@@ -45,7 +45,7 @@ void ThunkSchedule::AddDependenciesOnTransitiveOperands(
 ThunkSchedule::ThunkSchedule(
     std::unique_ptr<ThunkSequence> thunks,
     std::unique_ptr<StreamAssignment> stream_assignment,
-    const std::vector<const HloInstruction*>& hlo_total_order)
+    const std::vector<HloInstruction*>& hlo_total_order)
     : thunks_(std::move(thunks)),
       stream_assignment_(std::move(stream_assignment)) {
   std::unordered_map<const HloInstruction*, Thunk*> hlo_to_thunk;
@@ -53,7 +53,7 @@ ThunkSchedule::ThunkSchedule(
     InsertOrDie(&hlo_to_thunk, thunk->hlo_instruction(), thunk.get());
   }
 
-  for (const HloInstruction* hlo : hlo_total_order) {
+  for (HloInstruction* hlo : hlo_total_order) {
     if (hlo_to_thunk.count(hlo)) {
       thunk_total_order_.push_back(FindOrDie(hlo_to_thunk, hlo));
     }
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
index d3352994f8..43b628a1ba 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
@@ -46,7 +46,7 @@ class ThunkSchedule {
  public:
   ThunkSchedule(std::unique_ptr<ThunkSequence> thunks,
                 std::unique_ptr<StreamAssignment> stream_assignment,
-                const std::vector<const HloInstruction*>& hlo_total_order);
+                const std::vector<HloInstruction*>& hlo_total_order);
 
   // Returns the total order of executing all the thunks.
   const std::vector<Thunk*>& TotalOrder() const { return thunk_total_order_; }
diff --git a/tensorflow/compiler/xla/service/heap_simulator_test.cc b/tensorflow/compiler/xla/service/heap_simulator_test.cc
index fad3215fc8..dc40b9446a 100644
--- a/tensorflow/compiler/xla/service/heap_simulator_test.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator_test.cc
@@ -258,7 +258,7 @@ class HeapSimulatorTracker {
   // Constructor for testing a single entry computation.
   HeapSimulatorTracker(
       const string& name, std::unique_ptr<HloComputation> computation,
-      const std::vector<const HloInstruction*>& instruction_sequence) {
+      const std::vector<HloInstruction*>& instruction_sequence) {
     HloModuleConfig config;
     module_ = absl::make_unique<HloModule>(name, config);
     module_->AddEntryComputation(std::move(computation));
@@ -286,7 +286,7 @@ class HeapSimulatorTracker {
   // Similar to the single entry computation constructor above, but runs the
   // simulation over the entire module.
   void RunWholeModule(
-      const std::vector<const HloInstruction*>& full_module_sequence) {
+      const std::vector<HloInstruction*>& full_module_sequence) {
     points_to_analysis_ =
         TuplePointsToAnalysis::Run(module_.get()).ConsumeValueOrDie();
 
@@ -294,7 +294,7 @@ class HeapSimulatorTracker {
     HloSchedule schedule(module_.get());
     absl::flat_hash_map<const HloInstruction*, int> reverse_position;
     for (int i = 0; i < full_module_sequence.size(); ++i) {
-      const HloInstruction* instruction = full_module_sequence[i];
+      HloInstruction* instruction = full_module_sequence[i];
       schedule.GetOrCreateSequence(instruction->parent())
           .push_back(instruction);
       reverse_position[instruction] = full_module_sequence.size() - i;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 0c20d207dd..65bd251dd8 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -795,7 +795,7 @@ Status HloComputation::AcceptWithOperandOrder(
 template <typename HloInstructionPtr>
 Status HloComputation::AcceptOrdered(
     DfsHloVisitorBase<HloInstructionPtr>* visitor,
-    const std::vector<const HloInstruction*>& order) const {
+    const std::vector<HloInstruction*>& order) const {
   VLOG(3) << "Accepting visitor with order.";
   for (HloInstruction* root : CollectUnreachableRoots()) {
     TF_RET_CHECK(std::find(order.begin(), order.end(), root) != order.end())
@@ -825,9 +825,9 @@ Status HloComputation::AcceptOrdered(
 
 // Explicit instantiations.
 template Status HloComputation::AcceptOrdered(
-    DfsHloVisitor*, const std::vector<const HloInstruction*>&) const;
+    DfsHloVisitor*, const std::vector<HloInstruction*>&) const;
 template Status HloComputation::AcceptOrdered(
-    ConstDfsHloVisitor*, const std::vector<const HloInstruction*>&) const;
+    ConstDfsHloVisitor*, const std::vector<HloInstruction*>&) const;
 
 Status HloComputation::Accept(
     const std::function<Status(HloInstruction*)>& visitor_func) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index fc7d2035e5..be1ce33696 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -301,7 +301,7 @@ class HloComputation {
   // be a topological sort of all instructions in the computation.
   template <typename HloInstructionPtr>
   Status AcceptOrdered(DfsHloVisitorBase<HloInstructionPtr>* visitor,
-                       const std::vector<const HloInstruction*>& order) const;
+                       const std::vector<HloInstruction*>& order) const;
 
   // Same as Accept() above, but the visitor is given as a function.
   Status Accept(const std::function<Status(HloInstruction*)>& visitor_func);
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
index 234fcd266a..d2740bcce2 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
@@ -73,7 +73,7 @@ class ListScheduler {
   // Construct and return a memory-minimizing sequence of HLO instructions
   // containing the given HLO computation.
   static StatusOr<HloInstructionSequence> Run(
-      const HloComputation& computation,
+      HloComputation* computation,
       const TuplePointsToAnalysis& points_to_analysis,
       const LogicalBuffer::SizeFunction& size_function,
       const absl::flat_hash_map<const HloComputation*, int64>&
@@ -98,7 +98,7 @@ class ListScheduler {
   // comparison operators.
   using Priority = std::pair<int64, int64>;
 
-  ListScheduler(const HloComputation& computation,
+  ListScheduler(HloComputation* computation,
                 const TuplePointsToAnalysis& points_to_analysis,
                 const LogicalBuffer::SizeFunction& size_function,
                 const absl::flat_hash_map<const HloComputation*, int64>&
@@ -111,7 +111,7 @@ class ListScheduler {
     // instruction. An HLO instruction "uses" a LogicalBuffer if the
     // LogicalBuffer is in an operand of the instruction as indicated by
     // points-to analysis.
-    for (auto* instruction : computation.instructions()) {
+    for (auto* instruction : computation->instructions()) {
       absl::flat_hash_set<const LogicalBuffer*> instr_uses;
       for (auto* operand : instruction->operands()) {
         points_to_analysis.GetPointsToSet(operand).ForEachElement(
@@ -126,13 +126,13 @@ class ListScheduler {
 
     // Create map containing the number of unscheduled uses (hlo instructions)
     // of each logical buffer.
-    for (auto* instruction : computation.instructions()) {
+    for (auto* instruction : computation->instructions()) {
       for (auto* buffer :
            points_to_analysis.GetBuffersDefinedByInstruction(instruction)) {
         unscheduled_use_count_[buffer] = 0;
       }
     }
-    for (auto* instruction : computation.instructions()) {
+    for (auto* instruction : computation->instructions()) {
       for (const LogicalBuffer* buffer : buffer_uses_.at(instruction)) {
         ++unscheduled_use_count_[buffer];
       }
@@ -141,7 +141,7 @@ class ListScheduler {
     // Buffers live out of the computation have an implicit use at the end of
     // the computation.
     for (const LogicalBuffer* live_out_buffer :
-         points_to_analysis.GetPointsToSet(computation.root_instruction())
+         points_to_analysis.GetPointsToSet(computation->root_instruction())
              .CreateFlattenedSet()) {
       ++unscheduled_use_count_[live_out_buffer];
     }
@@ -157,7 +157,7 @@ class ListScheduler {
   // HloInstruction, plus some cached metadata, saved for the purposes of making
   // BytesFreedIfScheduled fast.
   struct ReadyListEntry {
-    const HloInstruction* instruction;
+    HloInstruction* instruction;
 
     // The total size of all buffers defined by this instruction.
     int64 bytes_defined;
@@ -171,7 +171,7 @@ class ListScheduler {
   };
 
   // Creates a ReadyListEntry for the given instruction.
-  ReadyListEntry MakeReadyListEntry(const HloInstruction* instruction) {
+  ReadyListEntry MakeReadyListEntry(HloInstruction* instruction) {
     ReadyListEntry entry;
     entry.instruction = instruction;
 
@@ -250,13 +250,13 @@ class ListScheduler {
     // Populate the ready list with instructions which have no operands or
     // control predecessors.
     absl::flat_hash_map<const HloInstruction*, int64> unscheduled_pred_count;
-    for (auto* instruction : computation_.instructions()) {
+    for (auto* instruction : computation_->instructions()) {
       // TODO(b/34466113): Replace this and above with successors() or
       // predecessors() when these methods are added to HloInstruction.
-      for (const HloInstruction* user : instruction->users()) {
+      for (HloInstruction* user : instruction->users()) {
         unscheduled_pred_count[user]++;
       }
-      for (const HloInstruction* succ : instruction->control_successors()) {
+      for (HloInstruction* succ : instruction->control_successors()) {
         unscheduled_pred_count[succ]++;
       }
     }
@@ -275,7 +275,7 @@ class ListScheduler {
       ready_instructions[inst] = it;
     };
 
-    for (auto* instruction : computation_.instructions()) {
+    for (auto* instruction : computation_->instructions()) {
       if (instruction->operands().empty() &&
           instruction->control_predecessors().empty()) {
         add_to_ready_queue(instruction);
@@ -287,7 +287,7 @@ class ListScheduler {
       // schedule.
       auto best_it = ready_queue.end();
       --best_it;
-      const HloInstruction* best = best_it->second.instruction;
+      HloInstruction* best = best_it->second.instruction;
       VLOG(2) << "Schedule instruction: " << best->ToShortString()
               << " Bytes freed: " << best_it->first.first;
       ready_queue.erase(best_it);
@@ -348,13 +348,13 @@ class ListScheduler {
         }
       }
     }
-    CHECK_EQ(schedule.size(), computation_.instruction_count());
-    CHECK_EQ(scheduled_instructions_.size(), computation_.instruction_count());
+    CHECK_EQ(schedule.size(), computation_->instruction_count());
+    CHECK_EQ(scheduled_instructions_.size(), computation_->instruction_count());
 
     return schedule;
   }
 
-  const HloComputation& computation_;
+  HloComputation* computation_;
   const TuplePointsToAnalysis& points_to_analysis_;
   const LogicalBuffer::SizeFunction& size_function_;
   // Computations are analyzed in post-order. When scheduling an instruction
@@ -386,13 +386,13 @@ int64 SumLogicalBufferSizes(
 }
 
 StatusOr<HloInstructionSequence> ScheduleComputationHelper(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
-  VLOG(2) << "Computation: " << computation.name();
+  VLOG(2) << "Computation: " << computation->name();
   if (algorithm) {
     return algorithm(computation, points_to_analysis, size_function,
                      memory_by_computation);
@@ -404,17 +404,17 @@ StatusOr<HloInstructionSequence> ScheduleComputationHelper(
 }  // namespace
 
 StatusOr<HloInstructionSequence> DFSMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
   // These variables are a hack to prevent overflows.
   int64 cumulative_total_size = 0;
-  int64 total_hlos = computation.parent()->instruction_count();
+  int64 total_hlos = computation->parent()->instruction_count();
   absl::flat_hash_map<const HloInstruction*, int64> extra_users;
   absl::flat_hash_map<const HloInstruction*, int64> total_sizes;
-  for (const HloInstruction* hlo : computation.MakeInstructionPostOrder()) {
+  for (const HloInstruction* hlo : computation->MakeInstructionPostOrder()) {
     if (ListScheduler::IgnoreInstruction(*hlo)) {
       extra_users[hlo] = 0;
       total_sizes[hlo] = 0;
@@ -448,8 +448,8 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
     total_sizes[hlo] = std::min(total_sizes[hlo], cumulative_total_size);
     extra_users[hlo] = std::min(extra_users[hlo], total_hlos);
   }
-  CHECK_EQ(extra_users.size(), computation.instruction_count());
-  CHECK_EQ(total_sizes.size(), computation.instruction_count());
+  CHECK_EQ(extra_users.size(), computation->instruction_count());
+  CHECK_EQ(total_sizes.size(), computation->instruction_count());
 
   // Construct a total order based on DFS post-order, visiting operands in
   // decreasing cumulative extra user order, and next by cumulative size, with a
@@ -459,7 +459,7 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
     sequence.push_back(hlo);
     return Status::OK();
   });
-  TF_RETURN_IF_ERROR(computation.AcceptWithOperandOrder(
+  TF_RETURN_IF_ERROR(computation->AcceptWithOperandOrder(
       &visitor, [&extra_users, &total_sizes](const HloInstruction* a,
                                              const HloInstruction* b) {
         if (extra_users[a] != extra_users[b]) {
@@ -470,12 +470,12 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
         }
         return a->name() < b->name();
       }));
-  CHECK_EQ(sequence.size(), computation.instruction_count());
+  CHECK_EQ(sequence.size(), computation->instruction_count());
   return sequence;
 }  // namespace xla
 
 StatusOr<HloInstructionSequence> ListMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -485,16 +485,16 @@ StatusOr<HloInstructionSequence> ListMemoryScheduler(
 }
 
 StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
-  return HloInstructionSequence(computation.MakeInstructionPostOrder());
+  return HloInstructionSequence(computation->MakeInstructionPostOrder());
 }
 
 StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -513,7 +513,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                           memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 list_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          computation, list_sequence, points_to_analysis,
+                          *computation, list_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory list sequence: " << HumanReadableNumBytes(list_memory);
 
@@ -522,7 +522,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                                          size_function, memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 dfs_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          computation, dfs_sequence, points_to_analysis,
+                          *computation, dfs_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory dfs sequence: " << HumanReadableNumBytes(dfs_memory);
 
@@ -532,7 +532,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                                memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 post_order_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          computation, post_order_sequence, points_to_analysis,
+                          *computation, post_order_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory post order sequence: "
           << HumanReadableNumBytes(post_order_memory);
@@ -555,17 +555,17 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
 }
 
 StatusOr<HloSchedule> ScheduleModule(
-    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
+    HloModule* module, const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm) {
-  HloSchedule schedule(&module);
+  HloSchedule schedule(module);
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
-                      TuplePointsToAnalysis::Run(&module));
+                      TuplePointsToAnalysis::Run(module));
   absl::flat_hash_map<const HloComputation*, int64> memory_by_computation;
-  for (const auto* computation : module.MakeComputationPostOrder()) {
+  for (auto* computation : module->MakeComputationPostOrder()) {
     if (!computation->IsFusionComputation()) {
       TF_ASSIGN_OR_RETURN(HloInstructionSequence computation_sequence,
                           ScheduleComputationHelper(
-                              *computation, *points_to_analysis, size_function,
+                              computation, *points_to_analysis, size_function,
                               algorithm, memory_by_computation));
       memory_by_computation[computation] =
           HeapSimulator::MinimumMemoryForComputation(
@@ -583,11 +583,11 @@ StatusOr<HloSchedule> ScheduleModule(
 }
 
 StatusOr<HloInstructionSequence> ScheduleComputation(
-    const HloComputation& computation,
+    HloComputation* computation,
     const LogicalBuffer::SizeFunction& size_function) {
-  CHECK(!computation.IsFusionComputation());
+  CHECK(!computation->IsFusionComputation());
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
-                      TuplePointsToAnalysis::Run(computation.parent()));
+                      TuplePointsToAnalysis::Run(computation->parent()));
   absl::flat_hash_map<const HloComputation*, int64> empty_map;
   return ScheduleComputationHelper(computation, *points_to_analysis,
                                    size_function, nullptr, empty_map);
@@ -600,7 +600,7 @@ HloMemoryScheduler::HloMemoryScheduler(
 
 StatusOr<bool> HloMemoryScheduler::Run(HloModule* module) {
   TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                      ScheduleModule(*module, size_function_, algorithm_));
+                      ScheduleModule(module, size_function_, algorithm_));
   TF_RETURN_IF_ERROR(module->set_schedule(std::move(schedule)));
   return true;
 }
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
index cca5dc4939..7227bfb27c 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
@@ -36,14 +36,14 @@ namespace xla {
 // that describes buffer aliasing, together with a target-specific size function
 // that maps a tensor's logical size to its padded size.
 typedef std::function<StatusOr<HloInstructionSequence>(
-    const HloComputation&, const TuplePointsToAnalysis&,
+    HloComputation*, const TuplePointsToAnalysis&,
     const LogicalBuffer::SizeFunction&,
     const absl::flat_hash_map<const HloComputation*, int64>&)>
     MemorySchedulerAlgorithm;
 
 // List scheduler
 StatusOr<HloInstructionSequence> ListMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -51,7 +51,7 @@ StatusOr<HloInstructionSequence> ListMemoryScheduler(
 
 // DFS-order scheduler
 StatusOr<HloInstructionSequence> DFSMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -59,7 +59,7 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
 
 // Naive Post Order scheduler
 StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -69,7 +69,7 @@ StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
 // and the DFS scheduler, and chooses whichever returns a lower min-memory,
 // not accounting for fragmentation.
 StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -79,13 +79,13 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
 // the computation. size_function is the function returning the number of bytes
 // required for a LogicalBuffer.
 StatusOr<HloSchedule> ScheduleModule(
-    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
+    HloModule* module, const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm = {});
 
 // Computes the schedule for a single computation.
 // Currently only used by the GPU backend.
 StatusOr<HloInstructionSequence> ScheduleComputation(
-    const HloComputation& computation,
+    HloComputation* computation,
     const LogicalBuffer::SizeFunction& size_function);
 
 // A pass which schedules the HLO instructions in a module. The HloModule's
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
index 3d8482065a..bc0d7e2bc0 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
@@ -78,7 +78,7 @@ TEST_F(HloSchedulingTest, LastUseScheduledFirst) {
   TF_ASSERT_OK(module->schedule().Verify());
 
   // Verify that all instructions are in the sequence.
-  const std::vector<const HloInstruction*>& sequence =
+  const std::vector<HloInstruction*>& sequence =
       module->schedule().sequence(module->entry_computation()).instructions();
   EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size());
 
@@ -124,9 +124,9 @@ ENTRY root {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, size_fn, ListMemoryScheduler));
+      ScheduleModule(module.get(), size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
-  const std::vector<const HloInstruction*>& sequence =
+  const std::vector<HloInstruction*>& sequence =
       schedule.sequence(module->entry_computation()).instructions();
   EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size());
 
@@ -175,12 +175,13 @@ TEST_F(HloSchedulingTest, TuplesAreAccountedCorrectly) {
   auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
-                          ScheduleModule(*module,
-                                         [](const BufferValue& buffer) {
-                                           return ShapeUtil::ByteSizeOf(
-                                               buffer.shape(), TUPLE_SIZE);
-                                         },
-                                         ListMemoryScheduler));
+                          ScheduleModule(
+                              module.get(),
+                              [](const BufferValue& buffer) {
+                                return ShapeUtil::ByteSizeOf(buffer.shape(),
+                                                             TUPLE_SIZE);
+                              },
+                              ListMemoryScheduler));
 
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
@@ -225,12 +226,12 @@ TEST_F(HloSchedulingTest, MultiOutputFusionAccountedCorrectly) {
       {tuple, mul, add}, HloInstruction::FusionKind::kLoop);
 
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
-                          ScheduleModule(*module,
-                                         [](const BufferValue& buffer) {
-                                           return ShapeUtil::ByteSizeOf(
-                                               buffer.shape(), 2);
-                                         },
-                                         ListMemoryScheduler));
+                          ScheduleModule(
+                              module.get(),
+                              [](const BufferValue& buffer) {
+                                return ShapeUtil::ByteSizeOf(buffer.shape(), 2);
+                              },
+                              ListMemoryScheduler));
 
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
@@ -284,7 +285,7 @@ TEST_F(HloSchedulingTest, HeapSimulatorAccountsForSubcomputations) {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, size_fn, ListMemoryScheduler));
+      ScheduleModule(module.get(), size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
   auto entry_computation = module->entry_computation();
   EXPECT_EQ(module->entry_computation()->instruction_count(),
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 7974999e07..66622a1d26 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -104,11 +104,7 @@ class HloModule {
                                        HloCloneContext* context = nullptr);
 
   // Return a pointer to the entry computation of the module.
-  const HloComputation* entry_computation() const {
-    CHECK_NE(nullptr, entry_computation_);
-    return entry_computation_;
-  }
-  HloComputation* entry_computation() {
+  HloComputation* entry_computation() const {
     CHECK_NE(nullptr, entry_computation_);
     return entry_computation_;
   }
diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc
index f5f99bece1..ca6a154809 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering.cc
@@ -356,8 +356,7 @@ void SequentialHloOrdering::Initialize() {
   // Create a map from instruction to its order position.
   TF_DCHECK_OK(schedule_.Verify());
   for (const auto& computation_sequence : schedule_.sequences()) {
-    const std::vector<const HloInstruction*>& order =
-        computation_sequence.second.instructions();
+    const auto& order = computation_sequence.second.instructions();
     for (int i = 0; i < order.size(); ++i) {
       InsertOrDie(&order_position_, order[i], i);
     }
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 4390145c6b..4bf287a9ed 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -47,11 +47,11 @@ const double kF16max = 65504;
 
 // Creates and returns a schedule created using the order of the instructions in
 // the HloComputation::instructions() vectors in the module.
-HloSchedule ScheduleFromInstructionOrder(const HloModule* module) {
+HloSchedule ScheduleFromInstructionOrder(HloModule* module) {
   HloSchedule schedule(module);
-  for (const HloComputation* computation : module->computations()) {
+  for (HloComputation* computation : module->computations()) {
     if (!computation->IsFusionComputation()) {
-      for (const HloInstruction* instruction : computation->instructions()) {
+      for (HloInstruction* instruction : computation->instructions()) {
         schedule.GetOrCreateSequence(computation).push_back(instruction);
       }
     }
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 49e46ecd00..48add75523 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -130,10 +130,10 @@ using ItemList = absl::InlinedVector<Item*, 3>;
 // before arbitrary elements.
 class InstructionList {
  public:
-  explicit InstructionList(const std::vector<const HloInstruction*>& order) {
+  explicit InstructionList(const HloInstructionSequence& order) {
     int64 position = 0;
     Item* last = nullptr;
-    for (const HloInstruction* inst : order) {
+    for (HloInstruction* inst : order.instructions()) {
       // Add a new item to the linked list.
       Item* item = new Item;
       item->next = nullptr;
@@ -151,7 +151,7 @@ class InstructionList {
       // to be monotonically increasing through the list, and so is still useful
       // for quickly(-ish) determining the order of arbitrary instructions in
       // the list.
-      item->instruction = const_cast<HloInstruction*>(inst);
+      item->instruction = inst;
       item->position = position;
       position++;
 
@@ -927,7 +927,7 @@ Item* PickRematerializationCandidate(
 
 StatusOr<int64> HloRematerialization::ComputePeakMemory(
     const HloComputation* computation,
-    const std::vector<const HloInstruction*>& order) const {
+    const HloInstructionSequence& order) const {
   InstructionList instruction_list(order);
   MemoryUsageTracker tracker(computation, size_function_, *points_to_analysis_,
                              instruction_list);
@@ -971,8 +971,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
           << HumanReadableNumBytes(computation_peak_memory_.at(computation));
   CHECK(!ContainsKey(rematerialized_computations_, computation));
 
-  InstructionList instruction_list(
-      schedule->sequence(computation).instructions());
+  InstructionList instruction_list(schedule->sequence(computation));
   MemoryUsageTracker memory_tracker(computation, size_function_,
                                     *points_to_analysis_, instruction_list);
   bool changed = false;
@@ -1184,7 +1183,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
   sequence.clear();
   for (auto* item = instruction_list.first(); item != nullptr;
        item = instruction_list.next(item)) {
-    const HloInstruction* instruction = item->instruction;
+    HloInstruction* instruction = item->instruction;
     sequence.push_back(instruction);
   }
   rematerialized_computations_.insert(computation);
@@ -1235,10 +1234,8 @@ StatusOr<bool> HloRematerialization::Run(HloModule* module) {
         if (node.context() == CallContext::kSequential) {
           TF_ASSIGN_OR_RETURN(
               computation_peak_memory_[node.computation()],
-              ComputePeakMemory(node.computation(),
-                                module->schedule()
-                                    .sequence(node.computation())
-                                    .instructions()));
+              ComputePeakMemory(node.computation(), module->schedule().sequence(
+                                                        node.computation())));
         }
         return Status::OK();
       },
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h
index 70d83c04f0..a07d348041 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.h
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h
@@ -87,9 +87,8 @@ class HloRematerialization : public HloModulePass {
   // peak memory is the maximum total size of all live HLO instruction values at
   // any program point. 'order' is the order in which the HLO instructions will
   // be emitted which is used to determine lifespans of HLO values.
-  StatusOr<int64> ComputePeakMemory(
-      const HloComputation* computation,
-      const std::vector<const HloInstruction*>& order) const;
+  StatusOr<int64> ComputePeakMemory(const HloComputation* computation,
+                                    const HloInstructionSequence& order) const;
 
   // Returns the peak memory usage of the called computations for the given
   // instruction. Zero is returned if the instruction calls no computations.
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.cc b/tensorflow/compiler/xla/service/hlo_schedule.cc
index a5780b7551..8f6eb974c5 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule.cc
@@ -46,8 +46,8 @@ namespace xla {
         << "No computation exists in HLO module with id " << computation_id;
     const HloComputation* computation = comp_it->second;
 
-    absl::flat_hash_map<int64, const HloInstruction*> id_to_instruction;
-    for (const HloInstruction* instruction : computation->instructions()) {
+    absl::flat_hash_map<int64, HloInstruction*> id_to_instruction;
+    for (HloInstruction* instruction : computation->instructions()) {
       id_to_instruction[instruction->unique_id()] = instruction;
     }
 
@@ -81,9 +81,8 @@ StatusOr<HloScheduleProto> HloSchedule::ToProto() const {
   return std::move(proto);
 }
 
-void HloSchedule::set_sequence(
-    const HloComputation* computation,
-    absl::Span<const HloInstruction* const> sequence) {
+void HloSchedule::set_sequence(const HloComputation* computation,
+                               absl::Span<HloInstruction* const> sequence) {
   set_sequence(computation, HloInstructionSequence(sequence));
 }
 
@@ -114,8 +113,8 @@ Status HloSchedule::UpdateComputationSchedule(
     const HloComputation* computation) {
   // Map from unique ID to HloInstruction pointer for instructions in the
   // computation.
-  absl::flat_hash_map<int, const HloInstruction*> id_to_instruction;
-  for (const HloInstruction* instruction : computation->instructions()) {
+  absl::flat_hash_map<int, HloInstruction*> id_to_instruction;
+  for (HloInstruction* instruction : computation->instructions()) {
     InsertOrDie(&id_to_instruction, instruction->unique_id(), instruction);
   }
 
@@ -128,7 +127,7 @@ Status HloSchedule::UpdateComputationSchedule(
   // Map from HloInstruction X to newly added instructions (instruction is in
   // computation, but not in schedule) which use X. If an instruction is not in
   // the map, then it has no users which are newly added instructions.
-  absl::flat_hash_map<const HloInstruction*, std::vector<const HloInstruction*>>
+  absl::flat_hash_map<const HloInstruction*, std::vector<HloInstruction*>>
       new_instruction_uses;
 
   // For each newly added instruction, this is the count of the instruction's
@@ -138,9 +137,9 @@ Status HloSchedule::UpdateComputationSchedule(
 
   // Create a worklist of newly added instructions which are ready to be added
   // to the schedule. Initialize worklist with those that have zero operands.
-  std::queue<const HloInstruction*> worklist;
+  std::queue<HloInstruction*> worklist;
 
-  for (const HloInstruction* instruction : computation->instructions()) {
+  for (HloInstruction* instruction : computation->instructions()) {
     if (ids_in_schedule.count(instruction->unique_id()) == 0) {
       // This is a newly added instruction which is not in the schedule.
       if (instruction->operands().empty()) {
@@ -161,17 +160,17 @@ Status HloSchedule::UpdateComputationSchedule(
   // Lambda which schedules all instructions on the worklist.
   auto schedule_worklist = [&]() {
     while (!worklist.empty()) {
-      const HloInstruction* instruction = worklist.front();
+      HloInstruction* instruction = worklist.front();
       worklist.pop();
       new_sequence.push_back(instruction);
-      std::vector<const HloInstruction*>* new_users =
+      std::vector<HloInstruction*>* new_users =
           tensorflow::gtl::FindOrNull(new_instruction_uses, instruction);
       if (new_users != nullptr) {
         // This just-scheduled instruction has users which are newly added to
         // the module. Update the number of unscheduled operands and push the
         // newly added instruction to the worklist if it is ready to
         // schedule.
-        for (const HloInstruction* new_user : *new_users) {
+        for (HloInstruction* new_user : *new_users) {
           unscheduled_operand_count.at(new_user)--;
           CHECK_GE(unscheduled_operand_count.at(new_user), 0);
           if (unscheduled_operand_count.at(new_user) == 0) {
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.h b/tensorflow/compiler/xla/service/hlo_schedule.h
index 0a714101ee..486ddbf499 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/hlo_schedule.h
@@ -35,14 +35,14 @@ class HloInstructionSequence {
  public:
   HloInstructionSequence() = default;
   explicit HloInstructionSequence(
-      absl::Span<const HloInstruction* const> instructions) {
-    for (const HloInstruction* instruction : instructions) {
+      absl::Span<HloInstruction* const> instructions) {
+    for (HloInstruction* instruction : instructions) {
       push_back(instruction);
     }
   }
 
   // Adds the instruction to the end of the sequence.
-  void push_back(const HloInstruction* instruction) {
+  void push_back(HloInstruction* instruction) {
     instruction_sequence_.push_back(instruction);
     id_sequence_.push_back(instruction->unique_id());
   }
@@ -56,7 +56,7 @@ class HloInstructionSequence {
   int64 size() const { return instruction_sequence_.size(); }
 
   // Returns the sequence of HLO instructions.
-  const std::vector<const HloInstruction*>& instructions() const {
+  const std::vector<HloInstruction*>& instructions() const {
     return instruction_sequence_;
   }
 
@@ -65,7 +65,7 @@ class HloInstructionSequence {
 
  private:
   // The sequence as HloInstructions.
-  std::vector<const HloInstruction*> instruction_sequence_;
+  std::vector<HloInstruction*> instruction_sequence_;
 
   // The sequence of HLO instructions, represented by their unique IDs. The
   // sequence is stored as both HloInstructions and unique IDs because the
@@ -98,7 +98,7 @@ class HloSchedule {
 
   // Sets the sequence for the given computation to the given sequence.
   void set_sequence(const HloComputation* computation,
-                    absl::Span<const HloInstruction* const> sequence);
+                    absl::Span<HloInstruction* const> sequence);
   void set_sequence(const HloComputation* computation,
                     HloInstructionSequence sequence);
 
diff --git a/tensorflow/compiler/xla/service/hlo_schedule_test.cc b/tensorflow/compiler/xla/service/hlo_schedule_test.cc
index 1424569ac1..0e56e6f760 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule_test.cc
@@ -56,10 +56,10 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
-  const std::vector<const HloInstruction*>& entry_schedule =
+  const auto& entry_schedule =
       schedule.sequence(module->entry_computation()).instructions();
 
   EXPECT_EQ(entry_schedule.size(), 6);
@@ -90,7 +90,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -139,7 +139,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -183,7 +183,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -244,7 +244,7 @@ ENTRY %WhileLoop () -> s32[] {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
@@ -313,7 +313,7 @@ ENTRY %WhileLoop () -> s32[] {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
-- 
GitLab


From de8ebdb12fe26f7f0ed76330d6a705de4126cd5f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 16:16:00 -0800
Subject: [PATCH 0450/1554] LinearOperatorAdjoint added as a meta-class. 
 Computes the adjoint of an operator by flipping the adjoint arg in .matmul
 and .solve, and other appropriate modifications.

This is hidden from the public API, with an upcoming method, .adjoint, the preferred way to get an adjoint, since it has more operator specific information.

PiperOrigin-RevId: 221869871
---
 tensorflow/python/kernel_tests/linalg/BUILD   |  22 ++
 .../linalg/linear_operator_adjoint_test.py    | 118 ++++++++++
 .../ops/linalg/linear_operator_adjoint.py     | 207 ++++++++++++++++++
 3 files changed, 347 insertions(+)
 create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py
 create mode 100644 tensorflow/python/ops/linalg/linear_operator_adjoint.py

diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index d90e65526a..ba9e64979a 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -40,6 +40,28 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "linear_operator_adjoint_test",
+    size = "medium",
+    srcs = ["linear_operator_adjoint_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+    shard_count = 5,
+    tags = [
+        "noasan",  # times out, b/63678675
+        "optonly",  # times out
+    ],
+)
+
 cuda_py_test(
     name = "linear_operator_algebra_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py
new file mode 100644
index 0000000000..1bed4b5268
--- /dev/null
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py
@@ -0,0 +1,118 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_adjoint
+from tensorflow.python.ops.linalg import linear_operator_test_util
+from tensorflow.python.platform import test
+
+linalg = linalg_lib
+
+LinearOperatorAdjoint = linear_operator_adjoint.LinearOperatorAdjoint  # pylint: disable=invalid-name
+
+
+class LinearOperatorAdjointTest(
+    linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
+  """Most tests done in the base class LinearOperatorDerivedClassTest."""
+
+  def setUp(self):
+    self._atol[dtypes.complex64] = 1e-5
+    self._rtol[dtypes.complex64] = 1e-5
+
+  def _operator_and_matrix(self,
+                           build_info,
+                           dtype,
+                           use_placeholder,
+                           ensure_self_adjoint_and_pd=False):
+    shape = list(build_info.shape)
+
+    if ensure_self_adjoint_and_pd:
+      matrix = linear_operator_test_util.random_positive_definite_matrix(
+          shape, dtype, force_well_conditioned=True)
+    else:
+      matrix = linear_operator_test_util.random_tril_matrix(
+          shape, dtype, force_well_conditioned=True, remove_upper=True)
+
+    lin_op_matrix = matrix
+
+    if use_placeholder:
+      lin_op_matrix = array_ops.placeholder_with_default(matrix, shape=None)
+
+    if ensure_self_adjoint_and_pd:
+      operator = LinearOperatorAdjoint(
+          linalg.LinearOperatorFullMatrix(
+              lin_op_matrix, is_positive_definite=True, is_self_adjoint=True))
+    else:
+      operator = LinearOperatorAdjoint(
+          linalg.LinearOperatorLowerTriangular(lin_op_matrix))
+
+    return operator, linalg.adjoint(matrix)
+
+  def test_base_operator_hint_used(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 0.], [1., 1.]]
+    operator = linalg.LinearOperatorFullMatrix(
+        matrix,
+        is_positive_definite=True,
+        is_non_singular=True,
+        is_self_adjoint=False)
+    operator_adjoint = LinearOperatorAdjoint(operator)
+    self.assertTrue(operator_adjoint.is_positive_definite)
+    self.assertTrue(operator_adjoint.is_non_singular)
+    self.assertFalse(operator_adjoint.is_self_adjoint)
+
+  def test_supplied_hint_used(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 0.], [1., 1.]]
+    operator = linalg.LinearOperatorFullMatrix(matrix)
+    operator_adjoint = LinearOperatorAdjoint(
+        operator,
+        is_positive_definite=True,
+        is_non_singular=True,
+        is_self_adjoint=False)
+    self.assertTrue(operator_adjoint.is_positive_definite)
+    self.assertTrue(operator_adjoint.is_non_singular)
+    self.assertFalse(operator_adjoint.is_self_adjoint)
+
+  def test_contradicting_hints_raise(self):
+    # The matrix values do not effect auto-setting of the flags.
+    matrix = [[1., 0.], [1., 1.]]
+    operator = linalg.LinearOperatorFullMatrix(
+        matrix, is_positive_definite=False)
+    with self.assertRaisesRegexp(ValueError, "positive-definite"):
+      LinearOperatorAdjoint(operator, is_positive_definite=True)
+
+    operator = linalg.LinearOperatorFullMatrix(matrix, is_self_adjoint=False)
+    with self.assertRaisesRegexp(ValueError, "self-adjoint"):
+      LinearOperatorAdjoint(operator, is_self_adjoint=True)
+
+  def test_name(self):
+    matrix = [[11., 0.], [1., 8.]]
+    operator = linalg.LinearOperatorFullMatrix(
+        matrix, name="my_operator", is_non_singular=True)
+
+    operator = LinearOperatorAdjoint(operator)
+
+    self.assertEqual("my_operator_adjoint", operator.name)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/linalg/linear_operator_adjoint.py b/tensorflow/python/ops/linalg/linear_operator_adjoint.py
new file mode 100644
index 0000000000..858e224b9a
--- /dev/null
+++ b/tensorflow/python/ops/linalg/linear_operator_adjoint.py
@@ -0,0 +1,207 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Takes the adjoint of a `LinearOperator`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as linalg
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.util.tf_export import tf_export
+
+__all__ = []
+
+
+@tf_export("linalg.LinearOperatorAdjoint")
+class LinearOperatorAdjoint(linear_operator.LinearOperator):
+  """`LinearOperator` representing the adjoint of another operator.
+
+  This operator represents the adjoint of another operator.
+
+  ```python
+  # Create a 2 x 2 linear operator.
+  operator = LinearOperatorFullMatrix([[1 - i., 3.], [0., 1. + i]])
+  operator_adjoint = LinearOperatorAdjoint(operator)
+
+  operator_adjoint.to_dense()
+  ==> [[1. + i, 0.]
+       [3., 1 - i]]
+
+  operator_adjoint.shape
+  ==> [2, 2]
+
+  operator_adjoint.log_abs_determinant()
+  ==> - log(2)
+
+  x = ... Shape [2, 4] Tensor
+  operator_adjoint.matmul(x)
+  ==> Shape [2, 4] Tensor, equal to operator.matmul(x, adjoint=True)
+  ```
+
+  #### Performance
+
+  The performance of `LinearOperatorAdjoint` depends on the underlying
+  operators performance.
+
+  #### Matrix property hints
+
+  This `LinearOperator` is initialized with boolean flags of the form `is_X`,
+  for `X = non_singular, self_adjoint, positive_definite, square`.
+  These have the following meaning:
+
+  * If `is_X == True`, callers should expect the operator to have the
+    property `X`.  This is a promise that should be fulfilled, but is *not* a
+    runtime assert.  For example, finite floating point precision may result
+    in these promises being violated.
+  * If `is_X == False`, callers should expect the operator to not have `X`.
+  * If `is_X == None` (the default), callers should have no expectation either
+    way.
+  """
+
+  def __init__(self,
+               operator,
+               is_non_singular=None,
+               is_self_adjoint=None,
+               is_positive_definite=None,
+               is_square=None,
+               name=None):
+    r"""Initialize a `LinearOperatorAdjoint`.
+
+    `LinearOperatorAdjoint` is initialized with an operator `A`.  The `solve`
+    and `matmul` methods effectively flip the `adjoint` argument.  E.g.
+
+    ```
+    A = MyLinearOperator(...)
+    B = LinearOperatorAdjoint(A)
+    x = [....]  # a vector
+
+    assert A.matvec(x, adjoint=True) == B.matvec(x, adjoint=False)
+    ```
+
+    Args:
+      operator: `LinearOperator` object.
+      is_non_singular:  Expect that this operator is non-singular.
+      is_self_adjoint:  Expect that this operator is equal to its hermitian
+        transpose.
+      is_positive_definite:  Expect that this operator is positive definite,
+        meaning the quadratic form `x^H A x` has positive real part for all
+        nonzero `x`.  Note that we do not require the operator to be
+        self-adjoint to be positive-definite.  See:
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
+      is_square:  Expect that this operator acts like square [batch] matrices.
+      name: A name for this `LinearOperator`. Default is `operator.name +
+        "_adjoint"`.
+
+    Raises:
+      ValueError:  If `operator.is_non_singular` is False.
+    """
+
+    self._operator = operator
+
+    # The congruency of is_non_singular and is_self_adjoint was checked in the
+    # base operator.
+    def _combined_hint(hint_str, provided_hint_value, message):
+      """Get combined hint in the case where operator.hint should equal hint."""
+      op_hint = getattr(operator, hint_str)
+      if op_hint is False and provided_hint_value:
+        raise ValueError(message)
+      if op_hint and provided_hint_value is False:
+        raise ValueError(message)
+      return (op_hint or provided_hint_value) or None
+
+    is_square = _combined_hint(
+        "is_square", is_square,
+        "An operator is square if and only if its adjoint is square.")
+
+    is_non_singular = _combined_hint(
+        "is_non_singular", is_non_singular,
+        "An operator is non-singular if and only if its adjoint is "
+        "non-singular.")
+
+    is_self_adjoint = _combined_hint(
+        "is_self_adjoint", is_self_adjoint,
+        "An operator is self-adjoint if and only if its adjoint is "
+        "self-adjoint.")
+
+    is_positive_definite = _combined_hint(
+        "is_positive_definite", is_positive_definite,
+        "An operator is positive-definite if and only if its adjoint is "
+        "positive-definite.")
+
+    is_square = _combined_hint(
+        "is_square", is_square,
+        "An operator is square if and only if its adjoint is square.")
+
+    # Initialization.
+    if name is None:
+      name = operator.name + "_adjoint"
+    with ops.name_scope(name, values=operator.graph_parents):
+      super(LinearOperatorAdjoint, self).__init__(
+          dtype=operator.dtype,
+          graph_parents=operator.graph_parents,
+          is_non_singular=is_non_singular,
+          is_self_adjoint=is_self_adjoint,
+          is_positive_definite=is_positive_definite,
+          is_square=is_square,
+          name=name)
+
+  @property
+  def operator(self):
+    """The operator before taking the adjoint."""
+    return self._operator
+
+  def _assert_non_singular(self):
+    return self.operator.assert_non_singular()
+
+  def _assert_positive_definite(self):
+    return self.operator.assert_positive_definite()
+
+  def _assert_self_adjoint(self):
+    return self.operator.assert_self_adjoint()
+
+  def _shape(self):
+    return self.operator.shape
+
+  def _shape_tensor(self):
+    return self.operator.shape_tensor()
+
+  def _matmul(self, x, adjoint=False, adjoint_arg=False):
+    return self.operator.matmul(
+        x, adjoint=(not adjoint), adjoint_arg=adjoint_arg)
+
+  def _determinant(self):
+    if self.is_self_adjoint:
+      return self.operator.determinant()
+    return math_ops.conj(self.operator.determinant())
+
+  def _log_abs_determinant(self):
+    return self.operator.log_abs_determinant()
+
+  def _trace(self):
+    if self.is_self_adjoint:
+      return self.operator.trace()
+    return math_ops.conj(self.operator.trace())
+
+  def _solve(self, rhs, adjoint=False, adjoint_arg=False):
+    return self.operator.solve(
+        rhs, adjoint=(not adjoint), adjoint_arg=adjoint_arg)
+
+  def _to_dense(self):
+    if self.is_self_adjoint:
+      return self.operator.to_dense()
+    return linalg.adjoint(self.operator.to_dense())
-- 
GitLab


From db1b3aafc08d9208e427f3a9103a60dbba79fa06 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 16:25:04 -0800
Subject: [PATCH 0451/1554] Including attributes in the error message in case
 of attributes mismatch.

PiperOrigin-RevId: 221870891
---
 tensorflow/core/framework/node_def_util.cc | 4 ++++
 tensorflow/core/framework/node_def_util.h  | 1 +
 tensorflow/core/framework/op_kernel.cc     | 6 ++++--
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/framework/node_def_util.cc b/tensorflow/core/framework/node_def_util.cc
index 578ec7f2e4..95a787b2df 100644
--- a/tensorflow/core/framework/node_def_util.cc
+++ b/tensorflow/core/framework/node_def_util.cc
@@ -102,6 +102,10 @@ string SummarizeNodeDef(const NodeDef& node_def) {
   return ret;
 }
 
+string SummarizeAttrs(const NodeDef& node_def) {
+  return SummarizeAttrsHelper(node_def, node_def.device());
+}
+
 string FormatNodeForError(const Node& node) {
   return FormatNodeDefForError(node.def());
 }
diff --git a/tensorflow/core/framework/node_def_util.h b/tensorflow/core/framework/node_def_util.h
index 0ff67554eb..f682bb1535 100644
--- a/tensorflow/core/framework/node_def_util.h
+++ b/tensorflow/core/framework/node_def_util.h
@@ -48,6 +48,7 @@ extern const char* const kColocationGroupPrefix;
 // than a text-format proto.
 string SummarizeNode(const Node& node);
 string SummarizeNodeDef(const NodeDef& node_def);
+string SummarizeAttrs(const NodeDef& node_def);
 
 // Produces a formatted string pattern from the node which can uniquely identify
 // this node upstream to produce an informative error message. The pattern
diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index 1213e25116..e3cb4a40ec 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -1127,7 +1127,8 @@ Status FindKernelDef(const DeviceType& device_type, const NodeDef& node_def,
         FormatNodeDefForError(node_def));
     if (was_attr_mismatch) {
       errors::AppendToMessage(
-          &s, " (OpKernel was found, but attributes didn't match)");
+          &s, " (OpKernel was found, but attributes didn't match) ",
+          "Requested Attributes: ", SummarizeAttrs(node_def));
     }
     errors::AppendToMessage(
         &s, ".  Registered:", KernelsRegisteredForOp(node_def.op()));
@@ -1262,7 +1263,8 @@ Status CreateOpKernel(DeviceType device_type, DeviceBase* device,
                               FormatNodeDefForError(node_def)));
     if (was_attr_mismatch) {
       errors::AppendToMessage(
-          &s, " (OpKernel was found, but attributes didn't match)");
+          &s, " (OpKernel was found, but attributes didn't match) ",
+          "Requested Attributes: ", SummarizeAttrs(node_def));
     }
     errors::AppendToMessage(
         &s, ".  Registered:", KernelsRegisteredForOp(node_def.op()));
-- 
GitLab


From a4602189610551435fbff5f99059450163d012de Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 16 Nov 2018 16:27:38 -0800
Subject: [PATCH 0452/1554] [TF:XLA] Bump open source abseil revision to
 f6ae816808cd913e0e2b3e2af14f328fa1071af0

PiperOrigin-RevId: 221871178
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index fd6afeb400..fd800cf67d 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "042badbfa529555ed63d57c39bf58410aa16e518ce98299599d85729feb25070",
-        strip_prefix = "abseil-cpp-a06c4a1d9093137b7217a5aaba8920d62e835dc0",
+        sha256 = "28a6cb644dcebe7d3e0ee347706fec2e6975fae2bceb0add834c77140c7b6632",
+        strip_prefix = "abseil-cpp-f6ae816808cd913e0e2b3e2af14f328fa1071af0",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/a06c4a1d9093137b7217a5aaba8920d62e835dc0.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/a06c4a1d9093137b7217a5aaba8920d62e835dc0.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f6ae816808cd913e0e2b3e2af14f328fa1071af0.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/f6ae816808cd913e0e2b3e2af14f328fa1071af0.tar.gz",
         ],
     )
 
-- 
GitLab


From 6b5bef9216ae89067e2a600772ac17a0ca4a5010 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 16 Nov 2018 16:46:40 -0800
Subject: [PATCH 0453/1554] Return an error if the user enabled XLA without
 linking it in.

PiperOrigin-RevId: 221873554
---
 tensorflow/compiler/jit/BUILD                 | 26 ++++++
 .../compiler/jit/register_xla_cpu_jit.cc      | 22 +++++
 .../compiler/jit/register_xla_gpu_jit.cc      | 22 +++++
 tensorflow/core/BUILD                         | 14 ++++
 .../common_runtime/graph_execution_state.cc   |  3 +
 tensorflow/core/common_runtime/tf_xla_stub.cc | 82 +++++++++++++++++++
 tensorflow/core/common_runtime/tf_xla_stub.h  | 50 +++++++++++
 7 files changed, 219 insertions(+)
 create mode 100644 tensorflow/compiler/jit/register_xla_cpu_jit.cc
 create mode 100644 tensorflow/compiler/jit/register_xla_gpu_jit.cc
 create mode 100644 tensorflow/core/common_runtime/tf_xla_stub.cc
 create mode 100644 tensorflow/core/common_runtime/tf_xla_stub.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 682c0f0cb0..bfc0b0070e 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -45,15 +45,37 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "register_xla_cpu_jit",
+    srcs = ["register_xla_cpu_jit.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:tf_xla_stub",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "xla_cpu_jit",
     visibility = ["//visibility:public"],
     deps = [
         ":jit_compilation_passes",
+        ":register_xla_cpu_jit",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/xla/service:cpu_plugin",
+        "//tensorflow/core:tf_xla_stub",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "register_xla_gpu_jit",
+    srcs = ["register_xla_gpu_jit.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:tf_xla_stub",
     ],
     alwayslink = 1,
 )
@@ -63,6 +85,8 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = if_cuda([
         ":jit_compilation_passes",
+        ":register_xla_gpu_jit",
+        "//tensorflow/core:tf_xla_stub",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops",
@@ -78,6 +102,7 @@ cc_library(
     deps = [
         ":flags",
         ":jit_compilation_passes",
+        ":register_xla_cpu_jit",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla:xla_compiler",
@@ -96,6 +121,7 @@ cc_library(
     visibility = [":friends"],
     deps = [
         ":jit_compilation_passes",
+        ":register_xla_gpu_jit",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla:xla_compiler",
diff --git a/tensorflow/compiler/jit/register_xla_cpu_jit.cc b/tensorflow/compiler/jit/register_xla_cpu_jit.cc
new file mode 100644
index 0000000000..9cbef63127
--- /dev/null
+++ b/tensorflow/compiler/jit/register_xla_cpu_jit.cc
@@ -0,0 +1,22 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/tf_xla_stub.h"
+
+namespace tensorflow {
+namespace {
+XlaCpuJitIsLinkedIn register_xla_cpu_jit;
+}
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/register_xla_gpu_jit.cc b/tensorflow/compiler/jit/register_xla_gpu_jit.cc
new file mode 100644
index 0000000000..7399a41d25
--- /dev/null
+++ b/tensorflow/compiler/jit/register_xla_gpu_jit.cc
@@ -0,0 +1,22 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/tf_xla_stub.h"
+
+namespace tensorflow {
+namespace {
+XlaGpuJitIsLinkedIn register_xla_gpu_jit;
+}
+}  // namespace tensorflow
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 73e8db58a8..1162c47e6b 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2976,6 +2976,19 @@ tf_cuda_library(
     ] + if_static([":core_cpu_impl"]) + tf_protos_all() + tf_protos_grappler(),
 )
 
+tf_cuda_library(
+    name = "tf_xla_stub",
+    srcs = ["common_runtime/tf_xla_stub.cc"],
+    hdrs = ["common_runtime/tf_xla_stub.h"],
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":lib",
+        ":proto_text",
+        ":session_options",
+    ],
+)
+
 tf_cuda_library(
     name = "core_cpu_internal",
     srcs = [
@@ -2989,6 +3002,7 @@ tf_cuda_library(
         ":framework",
         ":graph",
         ":lib",
+        ":tf_xla_stub",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 0d36930324..c18268ad7b 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
+#include "tensorflow/core/common_runtime/tf_xla_stub.h"
 #include "tensorflow/core/framework/graph.pb_text.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -720,6 +721,8 @@ Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
   CHECK_EQ(options.callable_options.fetch_size(),
            rewrite_metadata.fetch_types.size());
 
+  TF_RETURN_IF_ERROR(CheckXlaJitOptimizerOptions(session_options_));
+
   // TODO(andydavis): Clarify optimization pass requirements around CostModel.
   GraphOptimizationPassOptions optimization_options;
   optimization_options.session_options = session_options_;
diff --git a/tensorflow/core/common_runtime/tf_xla_stub.cc b/tensorflow/core/common_runtime/tf_xla_stub.cc
new file mode 100644
index 0000000000..d463693669
--- /dev/null
+++ b/tensorflow/core/common_runtime/tf_xla_stub.cc
@@ -0,0 +1,82 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <cstdlib>
+
+#include "tensorflow/core/common_runtime/tf_xla_stub.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace tensorflow {
+namespace {
+
+bool is_xla_gpu_jit_registered = false;
+bool is_xla_cpu_jit_registered = false;
+
+struct XlaEnvVars {
+  bool xla_flags_env_var_present;
+  bool tf_xla_flags_env_var_present;
+};
+
+XlaEnvVars ComputeEnvVarHasXlaFlags() {
+  XlaEnvVars env_vars;
+  env_vars.xla_flags_env_var_present = getenv("XLA_FLAGS") != nullptr;
+  env_vars.tf_xla_flags_env_var_present = getenv("TF_XLA_FLAGS") != nullptr;
+  return env_vars;
+}
+
+}  // namespace
+
+XlaGpuJitIsLinkedIn::XlaGpuJitIsLinkedIn() { is_xla_gpu_jit_registered = true; }
+XlaCpuJitIsLinkedIn::XlaCpuJitIsLinkedIn() { is_xla_cpu_jit_registered = true; }
+
+Status CheckXlaJitOptimizerOptions(const SessionOptions* session_options) {
+  static XlaEnvVars env_vars = ComputeEnvVarHasXlaFlags();
+
+  if (is_xla_cpu_jit_registered || is_xla_gpu_jit_registered) {
+    return Status::OK();
+  }
+
+  if (env_vars.xla_flags_env_var_present) {
+    return errors::InvalidArgument(
+        "The XLA JIT is not linked in but the \"XLA_FLAGS\" environment "
+        "variable is set.  Please either link in XLA or remove \"XLA_FLAGS\" "
+        "from the environment.");
+  }
+
+  if (env_vars.tf_xla_flags_env_var_present) {
+    return errors::InvalidArgument(
+        "The XLA JIT is not linked in but the \"TF_XLA_FLAGS\" environment "
+        "variable is set.  Please either link in XLA or remove "
+        "\"TF_XLA_FLAGS\" from the environment.");
+  }
+
+  if (session_options) {
+    OptimizerOptions::GlobalJitLevel jit_level =
+        session_options->config.graph_options()
+            .optimizer_options()
+            .global_jit_level();
+
+    if (jit_level == OptimizerOptions::ON_1 ||
+        jit_level == OptimizerOptions::ON_2) {
+      return errors::InvalidArgument(
+          "The XLA JIT is enabled in the session options but XLA is not linked "
+          "in.  Plesae either link in XLA or disable the JIT in the session "
+          "options.");
+    }
+  }
+
+  return Status::OK();
+}
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/tf_xla_stub.h b/tensorflow/core/common_runtime/tf_xla_stub.h
new file mode 100644
index 0000000000..723b2b5cd2
--- /dev/null
+++ b/tensorflow/core/common_runtime/tf_xla_stub.h
@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_TF_XLA_STUB_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_TF_XLA_STUB_H_
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+// Returns an error if the XLA JIT is enabled via `session_options` or if the
+// TF_XLA_FLAGS or XLA_FLAGS environment variables are set, but neither the
+// XLA CPU JIT nor the XLA GPU JIT are linked in.
+//
+// If `session_options` is null then only the environment variables are checked.
+Status CheckXlaJitOptimizerOptions(const SessionOptions* session_options);
+
+// The XLA CPU JIT creates a static instance of this class to notify
+// `CheckXlaJitOptimizerOptions` that the XLA CPU JIT is linked in.
+//
+// NB!  The constructor of this class (if run at all) needs to be ordered (via
+// happens before) before any call to `CheckXlaJitOptimizerOptions`.
+class XlaCpuJitIsLinkedIn {
+ public:
+  XlaCpuJitIsLinkedIn();
+};
+
+// The XLA GPU JIT creates a static instance of this class to notify
+// `CheckXlaJitOptimizerOptions` that the XLA GPU JIT is linked in.
+//
+// NB!  The constructor of this class (if run at all) needs to be ordered (via
+// happens before) before any call to `CheckXlaJitOptimizerOptions`.
+class XlaGpuJitIsLinkedIn {
+ public:
+  XlaGpuJitIsLinkedIn();
+};
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_TF_XLA_STUB_H_
-- 
GitLab


From 55a2cc9bcf6dfd3820bb2321090d0b89f18e9d7a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 26 Aug 2018 19:16:00 +0000
Subject: [PATCH 0454/1554] Fix incorrect link to version compatibility

While looking into `tensorflow/java/README.md` the
`API stability guarantees` retured 404. This
fix made the change `version_semantics` -> `version_compat`
to fix the issue.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/java/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/java/README.md b/tensorflow/java/README.md
index 2fa81ed88f..6415290745 100644
--- a/tensorflow/java/README.md
+++ b/tensorflow/java/README.md
@@ -1,7 +1,7 @@
 # TensorFlow for Java
 
 > *WARNING*: The TensorFlow Java API is not currently covered by the TensorFlow
-> [API stability guarantees](https://www.tensorflow.org/guide/version_semantics).
+> [API stability guarantees](https://www.tensorflow.org/guide/version_compat).
 >
 > For using TensorFlow on Android refer instead to
 > [contrib/android](https://www.tensorflow.org/code/tensorflow/contrib/android),
-- 
GitLab


From b8d1b1f1d2705e18b3f336741c675c61beb585d7 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 16 Nov 2018 17:04:18 -0800
Subject: [PATCH 0455/1554] Add multinomial op to TFMobile

PiperOrigin-RevId: 221875797
---
 tensorflow/contrib/makefile/tf_op_files.txt         | 2 ++
 tensorflow/core/kernels/BUILD                       | 4 ++++
 tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc | 1 +
 3 files changed, 7 insertions(+)

diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index e779eff689..655c7eefcb 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -157,6 +157,7 @@ tensorflow/core/kernels/mirror_pad_op_cpu_impl_2.cc
 tensorflow/core/kernels/mirror_pad_op_cpu_impl_3.cc
 tensorflow/core/kernels/mirror_pad_op_cpu_impl_4.cc
 tensorflow/core/kernels/mirror_pad_op_cpu_impl_5.cc
+tensorflow/core/kernels/multinomial_op.cc
 tensorflow/core/kernels/no_op.cc
 tensorflow/core/kernels/non_max_suppression_op.cc
 tensorflow/core/kernels/one_hot_op.cc
@@ -252,6 +253,7 @@ tensorflow/core/kernels/split_op.cc
 tensorflow/core/kernels/split_v_op.cc
 tensorflow/core/kernels/stack.cc
 tensorflow/core/kernels/stack_ops.cc
+tensorflow/core/kernels/stateless_random_ops.cc
 tensorflow/core/kernels/strided_slice_op.cc
 tensorflow/core/kernels/strided_slice_op_inst_0.cc
 tensorflow/core/kernels/strided_slice_op_inst_1.cc
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 9065989657..b523e3e718 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5396,6 +5396,7 @@ filegroup(
         "mfcc_mel_filterbank.h",
         "mirror_pad_op.h",
         "mirror_pad_op_cpu_impl.h",
+        "multinomial_op.h",
         "pad_op.h",
         "pooling_ops_3d.h",
         "random_op.h",
@@ -5414,6 +5415,7 @@ filegroup(
         "spacetobatch_functor.h",
         "spacetodepth_op.h",
         "spectrogram.h",
+        "stateless_random_ops.h",
         "string_util.h",
         "tensor_array.h",
         "tile_functor.h",
@@ -5553,6 +5555,7 @@ filegroup(
         "mirror_pad_op_cpu_impl_3.cc",
         "mirror_pad_op_cpu_impl_4.cc",
         "mirror_pad_op_cpu_impl_5.cc",
+        "multinomial_op.cc",
         "pad_op.cc",
         "padding_fifo_queue.cc",
         "padding_fifo_queue_op.cc",
@@ -5593,6 +5596,7 @@ filegroup(
         "stack.cc",
         "stack.h",
         "stack_ops.cc",
+        "stateless_random_ops.cc",
         "string_join_op.cc",
         "string_util.cc",
         "summary_op.cc",
diff --git a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
index d251589b48..039a918af1 100644
--- a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
+++ b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
@@ -187,6 +187,7 @@ bool IsWhitelistedFlexOp(const std::string& tensorflow_op_name) {
           "MirrorPad",
           "MirrorPadGrad",
           "Mul",
+          "Multinomial",
           "Neg",
           "NextIteration",
           "NonMaxSuppression",
-- 
GitLab


From 6fee93a639ec51d1b1cfbe1da7450d99e0e85e7f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 17:11:43 -0800
Subject: [PATCH 0456/1554] Reorder nn.depthwise_conv2d parameters for TF 2.0
 API

PiperOrigin-RevId: 221876586
---
 tensorflow/python/ops/nn_impl.py              | 59 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 tensorflow/tools/compatibility/renames_v2.py  |  7 ++-
 .../tools/compatibility/tf_upgrade_v2.py      |  4 ++
 4 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 245aa0e52f..d697834206 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -424,7 +424,7 @@ def zero_fraction(value, name=None):
 
 
 # pylint: disable=redefined-builtin
-@tf_export("nn.depthwise_conv2d")
+@tf_export(v1=["nn.depthwise_conv2d"])
 def depthwise_conv2d(input,
                      filter,
                      strides,
@@ -497,6 +497,63 @@ def depthwise_conv2d(input,
         op=op)
 
 
+@tf_export("nn.depthwise_conv2d", v1=[])
+def depthwise_conv2d_v2(input,
+                        filter,
+                        strides,
+                        padding,
+                        data_format=None,
+                        dilations=None,
+                        name=None):
+  """Depthwise 2-D convolution.
+
+  Given a 4D input tensor ('NHWC' or 'NCHW' data formats)
+  and a filter tensor of shape
+  `[filter_height, filter_width, in_channels, channel_multiplier]`
+  containing `in_channels` convolutional filters of depth 1, `depthwise_conv2d`
+  applies a different filter to each input channel (expanding from 1 channel
+  to `channel_multiplier` channels for each), then concatenates the results
+  together.  The output has `in_channels * channel_multiplier` channels.
+
+  In detail,
+
+      output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}
+           filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,
+                                           strides[2] * j + rate[1] * dj, k]
+
+  Must have `strides[0] = strides[3] = 1`.  For the most common case of the
+  same horizontal and vertical strides, `strides = [1, stride, stride, 1]`.
+  If any value in `rate` is greater than 1, we perform atrous depthwise
+  convolution, in which case all values in the `strides` tensor must be equal
+  to 1.
+
+  Args:
+    input: 4-D with shape according to `data_format`.
+    filter: 4-D with shape
+      `[filter_height, filter_width, in_channels, channel_multiplier]`.
+    strides: 1-D of size 4.  The stride of the sliding window for each
+      dimension of `input`.
+    padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.
+      See the "returns" section of `tf.nn.convolution` for details.
+    data_format: The data format for input. Either "NHWC" (default) or "NCHW".
+    dilations: 1-D of size 2. The dilation rate in which we sample input values
+      across the `height` and `width` dimensions in atrous convolution. If it is
+      greater than 1, then all values of strides must be 1.
+    name: A name for this operation (optional).
+
+  Returns:
+    A 4-D `Tensor` with shape according to `data_format`.  E.g., for
+    "NHWC" format, shape is
+    `[batch, out_height, out_width, in_channels * channel_multiplier].`
+  """
+  return depthwise_conv2d(input=input,
+                          filter=filter,
+                          strides=strides,
+                          padding=padding,
+                          rate=dilations,
+                          name=name,
+                          data_format=data_format)
+
 # pylint: enable=redefined-builtin
 
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index b0d9c5c905..b252a91f11 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -106,7 +106,7 @@ tf_module {
   }
   member_method {
     name: "depthwise_conv2d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "depthwise_conv2d_native"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index dde076bf92..09ffd77fab 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -338,6 +338,7 @@ renames = {
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
     'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
     'tf.random.multinomial': 'tf.compat.v1.random.multinomial',
+    'tf.random.stateless_multinomial': 'tf.compat.v1.random.stateless_multinomial',
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
@@ -445,9 +446,9 @@ renames = {
     'tf.sparse_reset_shape': 'tf.sparse.reset_shape',
     'tf.sparse_reshape': 'tf.sparse.reshape',
     'tf.sparse_retain': 'tf.sparse.retain',
-    'tf.sparse_segment_mean': 'tf.sparse.segment_mean',
-    'tf.sparse_segment_sqrt_n': 'tf.sparse.segment_sqrt_n',
-    'tf.sparse_segment_sum': 'tf.sparse.segment_sum',
+    'tf.sparse_segment_mean': 'tf.compat.v1.sparse_segment_mean',
+    'tf.sparse_segment_sqrt_n': 'tf.compat.v1.sparse_segment_sqrt_n',
+    'tf.sparse_segment_sum': 'tf.compat.v1.sparse_segment_sum',
     'tf.sparse_slice': 'tf.sparse.slice',
     'tf.sparse_softmax': 'tf.sparse.softmax',
     'tf.sparse_split': 'tf.compat.v1.sparse_split',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index c0ebfccd9e..c10f4eed39 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -164,6 +164,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
         ],
+        "tf.nn.depthwise_conv2d": [
+            "input", "filter", "strides", "padding", "rate", "name",
+            "data_format"
+        ],
         "tf.multinomial": [
             "logits", "num_samples", "seed", "name", "output_dtype"
         ],
-- 
GitLab


From f28e49eac192620f78cc4ce9d9b4c4177839f520 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 16 Nov 2018 17:19:19 -0800
Subject: [PATCH 0457/1554] Create common utilities for `make_dataset_iterator`
 and use that in both TPUStrategy.

PiperOrigin-RevId: 221877298
---
 tensorflow/contrib/distribute/python/BUILD    |   2 +
 .../contrib/distribute/python/tpu_strategy.py |  53 +-----
 .../contrib/distribute/python/values_test.py  | 142 +++++++++-----
 tensorflow/python/distribute/values.py        | 180 +++++++++++++-----
 tensorflow/python/keras/engine/training.py    |   3 +-
 5 files changed, 237 insertions(+), 143 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 80d102249b..99a1c05ec8 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -21,8 +21,10 @@ cuda_py_test(
     srcs = ["values_test.py"],
     additional_deps = [
         ":mirrored_strategy",
+        ":combinations",
         ":multi_worker_test_base",
         "//tensorflow/core:protos_all_py",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:device_util",
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index d159b3e419..314dcc5e01 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -27,8 +27,6 @@ from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
-from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -232,59 +230,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     return enqueue_op_per_host
 
   def _make_dataset_iterator(self, dataset):
-    """Make iterators for each of the TPU hosts.
-
-    We first unbatch the users input dataset and then rebatch it with the
-    per replica batch size that is calculated using
-    `global_batch_size // num_replicas_in_sync`. The currently supported cases
-    are as follows:
-    `dataset.batch()` is the last operation on the dataset.
-    `dataset.apply(map_and_batch)` is the last operation on the dataset.
-    `dataset.batch().prefetch()` are the last 2 operations on the dataset.
-    `dataset.apply(map_and_batch).prefetch()` are the last 2 operations.
-
-    Args:
-      dataset: The `tf.data` dataset passed by the user.
-
-    Returns:
-      iterator: InputIterator created for each of the host machines.
-    """
-    # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
-    def _get_dataset_batch_size(dataset):
-      """Get the global batch size from the dataset object."""
-      # pylint: disable=protected-access
-      if isinstance(dataset, dataset_ops.DatasetV1Adapter):
-        dataset = dataset._dataset
-      if isinstance(dataset, dataset_ops.BatchDataset):
-        return tensor_util.constant_value(dataset._batch_size)
-      elif isinstance(dataset, batching._MapAndBatchDataset):
-        return dataset._batch_size
-      elif isinstance(dataset, dataset_ops.PrefetchDataset):
-        return _get_dataset_batch_size(dataset._input_dataset)
-      # pylint: enable=protected-access
-      raise ValueError(
-          "Unable to fetch the batch size from the input dataset. `batch` "
-          "`map_and_batch` need to be the last operations on the dataset. "
-          "The batch operations can be followed by a prefetch.")
-
-    global_batch_size = _get_dataset_batch_size(dataset)
-    if global_batch_size % self._num_replicas_in_sync:
-      raise ValueError(
-          "Batch size %s cannot be sharded evenly across replicas %s" % (
-              global_batch_size, self.num_replicas_in_sync))
-    per_replica_batch_size = global_batch_size // self._num_replicas_in_sync
-    dataset = dataset.apply(batching.unbatch())
-    dataset = dataset.batch(per_replica_batch_size, drop_remainder=True)
+    """Make iterators for each of the TPU hosts."""
 
     worker_devices = [
         (self.get_host(hid), [self.get_host_cpu_device(hid)])
         for hid in range(self.num_hosts)
     ]
-    distributed_dataset = values.MultiWorkerDataset(
-        functools.partial(self._call_dataset_fn, lambda: dataset),
-        worker_devices)
-    # TODO(priyag): Return distribution strategy specific InputIterator
-    return distributed_dataset.make_initializable_iterator()
+    return values.DatasetIterator(dataset, worker_devices,
+                                  self._num_replicas_in_sync)
 
   def _distribute_dataset(self, dataset_fn):
     worker_devices = [
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 9d51129cae..d3ff14e087 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -19,7 +19,9 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+from absl.testing import parameterized
 
+from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.core.protobuf import config_pb2
@@ -569,15 +571,21 @@ class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase):
         multi_worker_iterator.get_next()
 
 
-class InputFunctionIteratorTestBase(test.TestCase):
+class InputIteratorTestBase(test.TestCase):
 
-  def _test_iterator(self, input_fn, worker_device_pairs, expected_values,
-                     sess=None):
+  def _test_iterator(self, input_type, dataset_fn, worker_device_pairs,
+                     expected_values, sess=None, split_batch_by=None):
     devices = nest.flatten([ds for _, ds in worker_device_pairs])
-    input_contexts = [
-        distribute_lib.InputContext() for _ in worker_device_pairs]
-    iterator = values.InputFunctionIterator(
-        input_fn, worker_device_pairs, input_contexts)
+
+    if input_type == "input_fn":
+      input_contexts = [
+          distribute_lib.InputContext() for _ in worker_device_pairs]
+      input_fn = lambda _: dataset_fn()
+      iterator = values.InputFunctionIterator(input_fn, worker_device_pairs,
+                                              input_contexts)
+    else:
+      iterator = values.DatasetIterator(dataset_fn(), worker_device_pairs,
+                                        split_batch_by)
 
     evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
 
@@ -587,7 +595,7 @@ class InputFunctionIteratorTestBase(test.TestCase):
       next_element = iterator.get_next()
       computed_value = evaluate(
           [values.select_device(d, next_element) for d in devices])
-      self.assertEqual(expected_value, computed_value)
+      self.assertAllEqual(expected_value, computed_value)
 
     with self.assertRaises(errors.OutOfRangeError):
       next_element = iterator.get_next()
@@ -600,62 +608,89 @@ class InputFunctionIteratorTestBase(test.TestCase):
       next_element = iterator.get_next()
       computed_value = evaluate(
           [values.select_device(d, next_element) for d in devices])
-      self.assertEqual(expected_value, computed_value)
+      self.assertAllEqual(expected_value, computed_value)
 
 
-class InputFunctionIteratorSingleWorkerTest(InputFunctionIteratorTestBase):
+class InputIteratorSingleWorkerTest(InputIteratorTestBase,
+                                    parameterized.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes
-  def testOneDeviceCPU(self):
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"]))
+  def testOneDeviceCPU(self, input_type):
     worker_device_pairs = [("", ["/device:CPU:0"])]
-    input_fn = lambda _: dataset_ops.Dataset.range(10)
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
 
     expected_values = [[i] for i in range(10)]
 
-    self._test_iterator(input_fn, worker_device_pairs, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testTwoDevicesOneGPUOneCPU(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
 
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testTwoDevicesOneGPUOneCPU(self, input_type):
     worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    input_fn = lambda _: dataset_ops.Dataset.range(10)
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
 
     expected_values = [[i, i+1] for i in range(0, 10, 2)]
 
-    self._test_iterator(input_fn, worker_device_pairs, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testTupleDataset(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
 
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testTupleDataset(self, input_type):
     worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    def input_fn(_):
+    def dataset_fn():
       dataset1 = dataset_ops.Dataset.range(10)
       dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
       return dataset_ops.Dataset.zip((dataset1, dataset2))
 
     expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
 
-    self._test_iterator(input_fn, worker_device_pairs, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testUnevenDatasetBatches(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
 
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testUnevenDatasetBatches(self, input_type):
     worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    input_fn = lambda _: dataset_ops.Dataset.range(11)
+    dataset_fn = lambda: dataset_ops.Dataset.range(11)
 
     expected_values = [[i, i+1] for i in range(0, 10, 2)]
-    self._test_iterator(input_fn, worker_device_pairs, expected_values)
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["dataset"],
+      split_batch_by=[None, 2],
+      required_gpus=1))
+  def testBatchSplitting(self, input_type, split_batch_by):
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    batch_size = 10
+    dataset_fn = lambda: dataset_ops.Dataset.range(100).batch(batch_size)
+
+    updated_batch_size = (
+        batch_size // split_batch_by if split_batch_by else batch_size)
+    expected_values = [[range(i, i+updated_batch_size),
+                        range(i+updated_batch_size, i+2*updated_batch_size)]
+                       for i in range(0, 100, updated_batch_size*2)]
+
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values, sess=None,
+                        split_batch_by=split_batch_by)
 
 
-class InputFunctionIteratorMultiWorkerTest(
-    multi_worker_test_base.MultiWorkerTestBase,
-    InputFunctionIteratorTestBase):
+class InputIteratorMultiWorkerTest(
+    multi_worker_test_base.MultiWorkerTestBase, InputIteratorTestBase,
+    parameterized.TestCase):
 
   def _cpu_devices(self):
     return [
@@ -676,32 +711,41 @@ class InputFunctionIteratorMultiWorkerTest(
         ])
     ]
 
-  def testOneDevicePerWorker(self):
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"]))
+  def testOneDevicePerWorker(self, input_type):
     worker_devices = self._cpu_devices()
     with context.graph_mode(), self.cached_session() as sess:
-      input_fn = lambda _: dataset_ops.Dataset.range(4)
-      self._test_iterator(input_fn, worker_devices,
+      dataset_fn = lambda: dataset_ops.Dataset.range(4)
+      self._test_iterator(input_type, dataset_fn, worker_devices,
                           [[0, 0], [1, 1], [2, 2], [3, 3]], sess)
 
-  def testTwoDevicesPerWorker(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testTwoDevicesPerWorker(self, input_type):
     worker_devices = self._cpu_and_one_gpu_devices()
     with context.graph_mode(), self.cached_session() as sess:
-      input_fn = lambda _: dataset_ops.Dataset.range(4)
-      self._test_iterator(input_fn, worker_devices,
+      dataset_fn = lambda: dataset_ops.Dataset.range(4)
+      self._test_iterator(input_type, dataset_fn, worker_devices,
                           [[0, 1, 0, 1], [2, 3, 2, 3]], sess)
 
-  def testTupleDataset(self):
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"]))
+  def testTupleDataset(self, input_type):
     worker_devices = self._cpu_devices()
     with context.graph_mode(), self.cached_session() as sess:
-      def input_fn(_):
+      def dataset_fn():
         dataset1 = dataset_ops.Dataset.range(4)
         dataset2 = dataset_ops.Dataset.range(4).map(lambda x: x**2)
         return dataset_ops.Dataset.zip((dataset1, dataset2))
 
       expected_values = [[(i, i**2), (i, i**2)] for i in range(0, 4)]
-      self._test_iterator(input_fn, worker_devices, expected_values, sess)
+      self._test_iterator(input_type, dataset_fn, worker_devices,
+                          expected_values, sess)
 
 
 class MirroredVariableTest(test.TestCase):
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 27057bb4a6..33ca27c63b 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -27,6 +27,7 @@ import operator
 import weakref
 import six
 
+from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import multi_device_iterator_ops
 from tensorflow.python.distribute import input_ops
@@ -1327,48 +1328,16 @@ class InputIterator(object):
     raise NotImplementedError("must be implemented in descendants")
 
 
-class InputFunctionIterator(InputIterator):
-  """Iterator created from input function."""
-
-  def __init__(self, input_fn, worker_device_pairs, input_contexts):
-    """Make an iterator for input provided via an input function.
-
-    Currently implements PER_WORKER mode, in which the `input_fn` is called
-    once on each worker.
+class InputIteratorImpl(InputIterator):
+  """Common implementation for all input iterators."""
 
-    TODO(priyag): Add other replication modes.
-    TODO(priyag): Allow taking input function that returns a callable that
-    returns nest of tensors.
-
-    Args:
-      input_fn: Input function that returns a `tf.data.Dataset` object.
-      worker_device_pairs: A list of (worker, list of devices on that worker)
-        pairs.
-      input_contexts: A list of `InputContext` instances to be passed to call(s)
-        to `input_fn`. Length and order should match worker order in
-        `worker_device_pairs`.
-    """
+  def __init__(self, worker_device_pairs, iterators):
     if not worker_device_pairs:
-      raise ValueError("Cannot create iterator when no devices given.")
-
-    if len(worker_device_pairs) != len(input_contexts):
-      raise ValueError(
-          "Number of worker_device_pairs (%d) is not same as number of"
-          "input_contexts (%d)" % (
-              len(worker_device_pairs), len(input_contexts)))
+      raise ValueError("Should have at least one worker for input iterator.")
 
+    self._iterators = iterators
     self._worker_device_pairs = worker_device_pairs
     self._is_eager = context.executing_eagerly()
-    self._iterators = []
-
-    for (worker, devices), ctx in zip(worker_device_pairs, input_contexts):
-      # TODO(priyag): We should probably explicitly specify CPU device on worker.
-      with ops.device(worker):
-        result = input_fn(ctx)
-        if not isinstance(result, dataset_ops.Dataset):
-          raise ValueError("input_fn must return a tf.data.Dataset.")
-        iterator = _DatasetIterator(result, worker, devices)
-        self._iterators.append(iterator)
 
   def get_next(self, name=None):
     """Returns the next input from the iterator for all replicas."""
@@ -1410,30 +1379,116 @@ class InputFunctionIterator(InputIterator):
     return init_ops
 
   # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  def _output_classes(self):
+  @property
+  def output_classes(self):
     return self._iterators[0].output_classes
 
   # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  def _output_shapes(self):
+  @property
+  def output_shapes(self):
     return self._iterators[0].output_shapes
 
   # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  def _output_types(self):
+  @property
+  def output_types(self):
     return self._iterators[0].output_types
 
   # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  def _get_iterator(self, worker):
+  def get_iterator(self, worker):
     for i, (w, _) in enumerate(self._worker_device_pairs):
       if worker == w:
         return self._iterators[i]
     return None
 
 
-class _DatasetIterator(object):
+class InputFunctionIterator(InputIteratorImpl):
+  """Iterator created from input function."""
+
+  def __init__(self, input_fn, worker_device_pairs, input_contexts):
+    """Make an iterator for input provided via an input function.
+
+    Currently implements PER_WORKER mode, in which the `input_fn` is called
+    once on each worker.
+
+    TODO(priyag): Add other replication modes.
+    TODO(priyag): Allow taking input function that returns a callable that
+    returns nest of tensors.
+
+    Args:
+      input_fn: Input function that returns a `tf.data.Dataset` object.
+      worker_device_pairs: A list of (worker, list of devices on that worker)
+        pairs.
+      input_contexts: A list of `InputContext` instances to be passed to call(s)
+        to `input_fn`. Length and order should match worker order in
+        `worker_device_pairs`.
+    """
+    if len(worker_device_pairs) != len(input_contexts):
+      raise ValueError(
+          "Number of worker_device_pairs (%d) is not same as number of"
+          "input_contexts (%d)" % (
+              len(worker_device_pairs), len(input_contexts)))
+
+    iterators = []
+    for (worker, devices), ctx in zip(worker_device_pairs, input_contexts):
+      # TODO(priyag): We should probably explicitly specify CPU device on worker.
+      with ops.device(worker):
+        result = input_fn(ctx)
+        if not isinstance(result, dataset_ops.Dataset):
+          raise ValueError("input_fn must return a tf.data.Dataset.")
+        iterator = _SingleWorkerDatasetIterator(result, worker, devices)
+        iterators.append(iterator)
+
+    super(InputFunctionIterator, self).__init__(
+        worker_device_pairs, iterators)
+
+
+class DatasetIterator(InputIteratorImpl):
+  """Iterator created from input dataset."""
+
+  def __init__(self, dataset, worker_device_pairs, split_batch_by=None):
+    """Make an iterator for the dataset on given devices.
+
+    If `split_batch_by` is not None, we "split" each batch of the
+    dataset by `split_batch_by` value. To achieve this, we first unbatch the
+    input dataset and then rebatch it with the per replica batch size that is
+    calculated using `global_batch_size // split_batch_by`.
+    The currently supported datasets are as follows:
+    `dataset.batch()` is the last operation on the dataset OR
+    `dataset.apply(map_and_batch)` is the last operation on the dataset OR
+    `dataset.batch().prefetch()` are the last 2 operations on the dataset OR
+    `dataset.apply(map_and_batch).prefetch()` are the last 2 operations.
+
+    TODO(priyag): Support multi worker / host cases properly by cloning
+    and sharding the dataset on each worker. Current setup will only work in
+    some cases, such as in-graph multi worker GPU case. If the input pipeline
+    has random shuffling (with a different seed on each worker), each worker
+    will see random input from the same overall dataset in each step. Otherwise,
+    each worker will see the same input in each step.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be used as the input source.
+      worker_device_pairs: A list of (worker, list of devices on that worker)
+        pairs.
+      split_batch_by: Optional integer. If present, we "split" each batch of the
+        dataset by `split_batch_by` value.
+    """
+    if split_batch_by:
+      dataset = _split_dataset_batch(dataset, split_batch_by)
+
+    iterators = []
+    for worker, worker_devices in worker_device_pairs:
+      with ops.device(worker):
+        iterator = _SingleWorkerDatasetIterator(dataset, worker, worker_devices)
+        iterators.append(iterator)
+
+    super(DatasetIterator, self).__init__(worker_device_pairs, iterators)
+
+
+class _SingleWorkerDatasetIterator(object):
   """Iterator for a single `tf.data.Dataset`."""
 
   def __init__(self, dataset, worker, devices):
-    """Create iterator for the given dataset.
+    """Create iterator for the `dataset` to fetch data to worker's `devices` .
 
     `MultiDeviceIterator` is used to prefetch input to the devices on the
     given worker. `MultiDeviceIterator` doesn't work in eager mode yet.
@@ -1510,6 +1565,45 @@ class _DatasetIterator(object):
     return self._iterator.output_types
 
 
+def _split_dataset_batch(dataset, split_batch_by):
+  """Divide a batch-ed dataset's batches into smaller batches."""
+  # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
+  # pylint: disable=protected-access
+  def _get_batch_dataset(d):
+    """Get the underlying batch dataset from the dataset object."""
+    if isinstance(d, dataset_ops.DatasetV1Adapter):
+      d = d._dataset
+
+    if isinstance(d, (dataset_ops.BatchDataset, batching._MapAndBatchDataset)):
+      return d
+    elif isinstance(d, dataset_ops.PrefetchDataset):
+      return _get_batch_dataset(d._input_dataset)
+    raise ValueError(
+        "Unable to get batched dataset from the input dataset. `batch` "
+        "`map_and_batch` need to be the last operations on the dataset. "
+        "The batch operations can be followed by a prefetch.")
+
+  batched_dataset = _get_batch_dataset(dataset)
+  batch_size = batched_dataset._batch_size
+  drop_remainder = batched_dataset._drop_remainder
+  # pylint: enable=protected-access
+
+  if tensor_util.is_tensor(batch_size):
+    batch_size = tensor_util.constant_value(batch_size)
+
+  if tensor_util.is_tensor(drop_remainder):
+    drop_remainder = tensor_util.constant_value(drop_remainder)
+
+  if batch_size % split_batch_by:
+    raise ValueError(
+        "Batch size %s cannot be sharded evenly across replicas %s" % (
+            batch_size, split_batch_by))
+  new_batch_size = batch_size // split_batch_by
+
+  dataset = dataset.apply(batching.unbatch())
+  return dataset.batch(new_batch_size, drop_remainder=drop_remainder)
+
+
 class MultiStepContext(object):
   """A context object that can be used to capture things when running steps.
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index c2a953f383..5d5af12805 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1049,10 +1049,11 @@ class Model(Network):
     with self._distribution_strategy.scope():
       if type(self._distribution_strategy).__name__ == 'TPUStrategy':
         iterator = self._distribution_strategy.make_dataset_iterator(x)
+        K.get_session().run(iterator.initialize())
       else:
         dataset = self._distribution_strategy.distribute_dataset(lambda: x)
         iterator = dataset.make_initializable_iterator()
-      K.get_session().run(iterator.initializer)
+        K.get_session().run(iterator.initializer)
 
     training_utils.validate_iterator_input(x, y, sample_weight,
                                            validation_split)
-- 
GitLab


From d33634f616c278d37e9d57d20a3666b7b5ee67be Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 16 Nov 2018 17:21:02 -0800
Subject: [PATCH 0458/1554] Implement Keras V2 NAdam optimization.

PiperOrigin-RevId: 221877440
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  20 +++
 tensorflow/python/keras/optimizer_v2/nadam.py | 110 ++++++++++++
 .../python/keras/optimizer_v2/nadam_test.py   | 169 ++++++++++++++++++
 3 files changed, 299 insertions(+)
 create mode 100644 tensorflow/python/keras/optimizer_v2/nadam.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/nadam_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index f77f0460eb..4913eaf9fc 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -17,6 +17,7 @@ py_library(
         "adam.py",
         "adamax.py",
         "gradient_descent.py",
+        "nadam.py",
         "optimizer_v2.py",
         "rmsprop.py",
     ],
@@ -110,6 +111,25 @@ cuda_py_test(
     shard_count = 4,
 )
 
+cuda_py_test(
+    name = "nadam_test",
+    size = "medium",
+    srcs = ["nadam_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+    shard_count = 4,
+)
+
 py_test(
     name = "optimizer_v2_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
new file mode 100644
index 0000000000..4be421a73f
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -0,0 +1,110 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Nadam for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.training import training_ops
+
+
+class Nadam(adam.Adam):
+  r"""Optimizer that implements the NAdam algorithm.
+
+  Much like Adam is essentially RMSprop with momentum, Nadam is Adam with
+  Nesterov momentum.
+
+  Initialization:
+
+  $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$
+  $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
+  $$t := 0 \text{(Initialize timestep)}$$
+
+  Computes:
+  $$t := t + 1$$
+  $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+  $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+  $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+  $$m_bar_t := beta_1 * v_t + (1 - beta_1) * g$$
+  $$theta_t := theta_{t-1} - lr_t * m_bar_t / (\sqrt{v_t} + \epsilon)$$
+
+  gradient is evaluated at theta(t) + momentum * v(t), and the variables always
+  store theta + beta_1 * m / sqrt(v) instead of theta.
+
+  References
+    See [Dozat, T., 2015](http://cs229.stanford.edu/proj2015/054_report.pdf).
+  """
+
+  def _resource_apply_dense(self, grad, var):
+    grad_dtype = grad.dtype.base_dtype
+    m = self.get_slot(var, 'm')
+    v = self.get_slot(var, 'v')
+    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
+    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
+    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_power = math_ops.pow(beta_1_t, local_step)
+    beta_2_power = math_ops.pow(beta_2_t, local_step)
+    return training_ops.resource_apply_adam(
+        var.handle,
+        m.handle,
+        v.handle,
+        beta_1_power,
+        beta_2_power,
+        math_ops.cast(self._get_hyper('learning_rate'), grad_dtype),
+        beta_1_t,
+        beta_2_t,
+        math_ops.cast(self._get_hyper('epsilon'), grad_dtype),
+        grad,
+        use_locking=self._use_locking,
+        use_nesterov=True)
+
+  def _resource_apply_sparse(self, grad, var, indices):
+    var_dtype = var.dtype.base_dtype
+    local_step = math_ops.cast(self.iterations + 1, var_dtype)
+    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), var_dtype)
+    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), var_dtype)
+    beta_1_power = math_ops.pow(beta_1_t, local_step)
+    beta_2_power = math_ops.pow(beta_2_t, local_step)
+    lr_t = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
+    epsilon_t = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    lr = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power))
+
+    # m_t = beta1 * m + (1 - beta1) * g_t
+    m = self.get_slot(var, 'm')
+    m_scaled_g_values = grad * (1 - beta_1_t)
+    m_t = state_ops.assign(m, m * beta_1_t, use_locking=self._use_locking)
+    with ops.control_dependencies([m_t]):
+      m_t = self._resource_scatter_add(m, indices, m_scaled_g_values)
+      # m_bar = (1 - beta1) * g_t + beta1 * m_t
+      m_bar = m_scaled_g_values + beta_1_t * array_ops.gather(m_t, indices)
+
+    # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
+    v = self.get_slot(var, 'v')
+    v_scaled_g_values = (grad * grad) * (1 - beta_2_t)
+    v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking)
+    with ops.control_dependencies([v_t]):
+      v_t = self._resource_scatter_add(v, indices, v_scaled_g_values)
+
+    v_t_slice = array_ops.gather(v_t, indices)
+    v_sqrt = math_ops.sqrt(v_t_slice)
+    var_update = self._resource_scatter_add(var, indices,
+                                            -lr * m_bar / (v_sqrt + epsilon_t))
+    return control_flow_ops.group(*[var_update, m_bar, v_t])
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
new file mode 100644
index 0000000000..9cc81b1d11
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -0,0 +1,169 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Nadam."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import nadam
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+def get_beta_accumulators(opt, dtype):
+  local_step = math_ops.cast(opt.iterations + 1, dtype)
+  beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype)
+  beta_1_power = math_ops.pow(beta_1_t, local_step)
+  beta_2_t = math_ops.cast(opt._get_hyper("beta_2"), dtype)
+  beta_2_power = math_ops.pow(beta_2_t, local_step)
+  return (beta_1_power, beta_2_power)
+
+
+def nadam_update_numpy(param,
+                       g_t,
+                       t,
+                       m,
+                       v,
+                       alpha=0.001,
+                       beta1=0.9,
+                       beta2=0.999,
+                       epsilon=1e-8):
+  alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
+
+  m_t = beta1 * m + (1 - beta1) * g_t
+  v_t = beta2 * v + (1 - beta2) * g_t * g_t
+
+  m_bar = (1 - beta1) * g_t + beta1 * m_t
+
+  param_t = param - alpha_t * m_bar / (np.sqrt(v_t) + epsilon)
+  return param_t, m_t, v_t
+
+
+class NadamOptimizerTest(test.TestCase):
+
+  def doTestSparse(self, use_resource=False):
+    sparse_epsilon = 1e-7
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype)
+
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(var0_np)
+          var1 = resource_variable_ops.ResourceVariable(var1_np)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0_np_indices = np.array([0, 2], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np[grads0_np_indices]),
+            constant_op.constant(grads0_np_indices), constant_op.constant([3]))
+        grads1_np_indices = np.array([0, 2], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np[grads1_np_indices]),
+            constant_op.constant(grads1_np_indices), constant_op.constant([3]))
+        opt = nadam.Nadam(epsilon=sparse_epsilon)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of Nadam
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = nadam_update_numpy(
+              var0_np, grads0_np, t, m0, v0, epsilon=sparse_epsilon)
+          var1_np, m1, v1 = nadam_update_numpy(
+              var1_np, grads1_np, t, m1, v1, epsilon=sparse_epsilon)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSparse(self):
+    self.doTestSparse(use_resource=False)
+
+  def testResourceSparse(self):
+    self.doTestSparse(use_resource=True)
+
+  def doTestBasic(self, use_resource=False):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(var0_np)
+          var1 = resource_variable_ops.ResourceVariable(var1_np)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = nadam.Nadam()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of Nadam
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = nadam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testBasic(self):
+    self.doTestBasic(use_resource=False)
+
+  def testResourceBasic(self):
+    self.doTestBasic(use_resource=True)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 2a1c305916b3ad31711fa507a9ef99380491821c Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 16 Nov 2018 17:32:19 -0800
Subject: [PATCH 0459/1554] New TensorArray implementation that uses
 TensorLists instead of Resources. Not feature complete yet but mostly works
 except for split and concat. This enables creating TensorArrays inside
 while_v2 and tf.function e.g. using tf.map_fn inside tf.function.

PiperOrigin-RevId: 221878566
---
 tensorflow/python/BUILD                       |   4 +
 .../operators/data_structures_test.py         |   3 +-
 tensorflow/python/framework/test_util.py      |  28 ++
 tensorflow/python/kernel_tests/BUILD          |   8 +-
 .../python/kernel_tests/cond_v2_test.py       |  13 +
 .../kernel_tests/control_flow_ops_py_test.py  |   4 -
 .../kernel_tests/functional_ops_test.py       |   3 +
 .../python/kernel_tests/list_ops_test.py      | 146 ++++++----
 .../kernel_tests/tensor_array_ops_test.py     | 224 ++++++++++-----
 tensorflow/python/ops/list_ops.py             |  60 +++-
 tensorflow/python/ops/tensor_array_ops.py     | 256 +++++++++++++++++-
 tensorflow/python/ops/while_v2.py             |  25 +-
 12 files changed, 610 insertions(+), 164 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c21356e676..603eca2c85 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1055,6 +1055,7 @@ py_library(
         ":random_seed",
         ":resource_variable_ops",
         ":session",
+        ":tensor_array_ops",
         ":training",
         ":util",
         ":variables",
@@ -3042,13 +3043,16 @@ py_library(
     deps = [
         ":array_ops",
         ":constant_op",
+        ":control_flow_ops_gen",
         ":data_flow_ops_gen",
         ":dtypes",
         ":errors",
         ":framework_ops",
+        ":list_ops",
         ":math_ops",
         ":tensor_shape",
         ":tensor_util",
+        ":tf2",
         ":tf_should_use",
         "//tensorflow/python/eager:context",
     ],
diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py
index 6039b07982..72476ccb6b 100644
--- a/tensorflow/python/autograph/operators/data_structures_test.py
+++ b/tensorflow/python/autograph/operators/data_structures_test.py
@@ -156,8 +156,7 @@ class ListTest(test.TestCase):
 
   def test_stack_tensor_list_empty(self):
     l = list_ops.empty_tensor_list(
-        element_shape=-1,
-        element_dtype=dtypes.variant)
+        element_shape=None, element_dtype=dtypes.variant)
 
     opts = data_structures.ListStackOpts(
         element_dtype=dtypes.int32, original_call=None)
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 39c8ba152d..897122746b 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -66,6 +66,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import versions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
 from tensorflow.python.platform import tf_logging as logging
@@ -406,13 +407,40 @@ def enable_control_flow_v2(fn):
   def wrapper(*args, **kwargs):
     enable_cond_v2_old = control_flow_ops.ENABLE_COND_V2
     enable_while_v2_old = control_flow_ops.ENABLE_WHILE_V2
+    enable_tensor_array_v2_old = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2
     control_flow_ops.ENABLE_COND_V2 = True
     control_flow_ops.ENABLE_WHILE_V2 = True
+    tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = True
     try:
       fn(*args, **kwargs)
     finally:
       control_flow_ops.ENABLE_COND_V2 = enable_cond_v2_old
       control_flow_ops.ENABLE_WHILE_V2 = enable_while_v2_old
+      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = enable_tensor_array_v2_old
+
+  return wrapper
+
+
+def enable_tensor_array_v2(fn):
+  """Decorator for enabling _GraphTensorArrayV2 on a test.
+
+  Note this enables _GraphTensorArrayV2 after running the test class's
+  setup/teardown methods.
+
+  Args:
+    fn: the function to be wrapped
+
+  Returns:
+    The wrapped function
+  """
+
+  def wrapper(*args, **kwargs):
+    enable_tensor_array_v2_old = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2
+    tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = True
+    try:
+      fn(*args, **kwargs)
+    finally:
+      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = enable_tensor_array_v2_old
 
   return wrapper
 
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 307330d1cb..41099ba2e8 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -121,8 +121,10 @@ cuda_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:math_ops",
+        "//tensorflow/python:gradients_impl",
         "//tensorflow/python:list_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/eager:context",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:framework_test_lib",
@@ -1750,9 +1752,11 @@ cuda_py_test(
         "//tensorflow/python:tensor_array_grad",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python:while_v2",
         "//tensorflow/python/data/ops:iterator_ops",
     ],
     grpc_enabled = True,
+    shard_count = 2,
     tags = ["no_windows"],
 )
 
@@ -2385,6 +2389,8 @@ cuda_py_test(
         "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:variables",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python:cond_v2",
+        "//tensorflow/python:while_v2",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
     ],
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 70b66bc52a..ace18dbc44 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -719,6 +719,10 @@ class CondV2Test(test.TestCase):
 
   @test_util.enable_control_flow_v2
   def testCondAndTensorArray(self):
+    if test_util.is_gpu_available():
+      old_enable_tensor_array_v2 = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2
+      # TODO(b/119689663): Enable this.
+      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = False
     x = math_ops.range(-5, 5)
     output = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=x.shape[0])
 
@@ -740,9 +744,15 @@ class CondV2Test(test.TestCase):
     output_t = output.stack()
     self.assertAllEqual(
         self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
+    if test_util.is_gpu_available():
+      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = old_enable_tensor_array_v2
 
   @test_util.enable_control_flow_v2
   def testCondAndTensorArrayInDefun(self):
+    if test_util.is_gpu_available():
+      old_enable_tensor_array_v2 = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2
+      # TODO(b/119689663): Enable this.
+      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = False
 
     @function.defun
     def f():
@@ -770,6 +780,9 @@ class CondV2Test(test.TestCase):
     self.assertAllEqual(
         self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
 
+    if test_util.is_gpu_available():
+      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = old_enable_tensor_array_v2
+
 
 class CondV2CollectionTest(test.TestCase):
 
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index c0068c2681..55654174d3 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2515,8 +2515,6 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(512.0, self.evaluate(r))
 
-  @test_util.disable_control_flow_v2("unsupported: resource creation in body. "
-                                     "Enable with new TAs b/117675481")
   def testNestedWhileGrad_ParallelIterations(self):
     # Make sure the stack pushes and pops of an inner loop are executed in
     # the sequential order of the iterations of its outer loop.
@@ -2695,8 +2693,6 @@ class ControlFlowTest(test.TestCase):
           c, b, [i0, constant_op.constant(0.0)])
       self.assertAllClose(600.0, sess.run(output_grad)[1])
 
-  @test_util.disable_control_flow_v2("unsupported: resource creation in body. "
-                                     "Enable with new TAs b/117675481")
   def testWhileAndTensorArray(self):
     with self.cached_session() as sess:
       param = constant_op.constant(2.0)
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index bebd98558f..503569f3b1 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -56,6 +56,7 @@ def simple_scoped_fn(a, x):
     return math_ops.multiply(math_ops.add(a, x), two)
 
 
+@test_util.with_control_flow_v2
 class FunctionalOpsTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
@@ -481,6 +482,7 @@ class FunctionalOpsTest(test.TestCase):
     y = functional_ops.map_fn(lambda e: e, x)
     self.assertIs(None, y.get_shape().dims)
 
+  @test_util.disable_control_flow_v2("b/119323354")
   @test_util.run_in_graph_and_eager_modes
   def testMapEmptyScalar(self):
     map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([]))
@@ -489,6 +491,7 @@ class FunctionalOpsTest(test.TestCase):
 
   # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
   # so the body of the while loop never executes
+  @test_util.disable_control_flow_v2("b/119323354")
   def testMapEmptyTensor(self):
     with self.cached_session():
       map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]),
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 92552854aa..2bc8ba463b 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -30,8 +30,10 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
@@ -39,17 +41,13 @@ from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.platform import test
 
 
-def scalar_shape():
-  return ops.convert_to_tensor([], dtype=dtypes.int32)
-
-
 @test_util.run_all_in_graph_and_eager_modes
 class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def _testPushPop(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=scalar_shape(),
+        element_shape=[],
         max_num_elements=max_num_elements)
     l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
     l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
@@ -70,9 +68,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testPushInFullListFails(self):
     l = list_ops.empty_tensor_list(
-        element_dtype=dtypes.float32,
-        element_shape=scalar_shape(),
-        max_num_elements=1)
+        element_dtype=dtypes.float32, element_shape=[], max_num_elements=1)
     l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  "Tried to push item into a full list"):
@@ -84,7 +80,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testPopFromEmptyTensorListFails(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=scalar_shape(),
+        element_shape=[],
         max_num_elements=max_num_elements)
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  "Trying to pop from an empty list"):
@@ -94,11 +90,13 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def _testStack(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=scalar_shape(),
+        element_shape=[],
         max_num_elements=max_num_elements)
     l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
     l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    if not context.executing_eagerly():
+      self.assertAllEqual(t.shape.as_list(), [None])
     self.assertAllEqual(self.evaluate(t), [1.0, 2.0])
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
@@ -119,7 +117,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testStackWithUnknownElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=-1,
+        element_shape=None,
         max_num_elements=max_num_elements)
     l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
     l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
@@ -139,7 +137,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testStackWithPartiallyDefinedElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=[-1],
+        element_shape=[None],
         max_num_elements=max_num_elements)
     l = list_ops.tensor_list_push_back(l, constant_op.constant([1.0]))
     l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0]))
@@ -171,7 +169,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                                  "non-fully-defined"):
       l = list_ops.empty_tensor_list(
           element_dtype=dtypes.float32,
-          element_shape=[-1, 2],
+          element_shape=[None, 2],
           max_num_elements=max_num_elements)
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -181,7 +179,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                                  "non-fully-defined"):
       l = list_ops.empty_tensor_list(
           element_dtype=dtypes.float32,
-          element_shape=-1,
+          element_shape=None,
           max_num_elements=max_num_elements)
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -192,7 +190,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with backprop.GradientTape() as tape:
       l = list_ops.empty_tensor_list(
           element_dtype=dtypes.float32,
-          element_shape=scalar_shape(),
+          element_shape=[],
           max_num_elements=max_num_elements)
       c0 = constant_op.constant(1.0)
       tape.watch(c0)
@@ -209,7 +207,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testGatherWithUnknownElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=-1,
+        element_shape=None,
         max_num_elements=max_num_elements)
     l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
     l = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
@@ -232,7 +230,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testGatherWithPartiallyDefinedElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
-        element_shape=[-1],
+        element_shape=[None],
         max_num_elements=max_num_elements)
     l = list_ops.tensor_list_push_back(l, constant_op.constant([1.0]))
     l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0]))
@@ -268,7 +266,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                                  "non-fully-defined"):
       l = list_ops.empty_tensor_list(
           element_dtype=dtypes.float32,
-          element_shape=[-1, 2],
+          element_shape=[None, 2],
           max_num_elements=max_num_elements)
       t = list_ops.tensor_list_gather(l, [], element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -279,7 +277,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                                  "non-fully-defined"):
       l = list_ops.empty_tensor_list(
           element_dtype=dtypes.float32,
-          element_shape=-1,
+          element_shape=None,
           max_num_elements=max_num_elements)
       t = list_ops.tensor_list_gather(l, [], element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -300,7 +298,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testTensorListFromTensor(self):
     t = constant_op.constant([1.0, 2.0])
-    l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape())
+    l = list_ops.tensor_list_from_tensor(t, element_shape=[])
     l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
     self.assertAllEqual(self.evaluate(e), 2.0)
     l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
@@ -315,7 +313,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testGetSetItem(self):
     t = constant_op.constant([1.0, 2.0])
-    l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape())
+    l = list_ops.tensor_list_from_tensor(t, element_shape=[])
     e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
     self.assertAllEqual(self.evaluate(e0), 1.0)
     l = list_ops.tensor_list_set_item(l, 0, 3.0)
@@ -333,9 +331,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = constant_op.constant(5.)
       tape.watch(t)
       l = list_ops.tensor_list_reserve(
-          element_dtype=dtypes.float32,
-          element_shape=scalar_shape(),
-          num_elements=3)
+          element_dtype=dtypes.float32, element_shape=[], num_elements=3)
       l = list_ops.tensor_list_set_item(l, 1, 2. * t)
       e = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
       self.assertAllEqual(self.evaluate(e), 10.0)
@@ -343,9 +339,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testSetOnEmptyListWithMaxNumElementsFails(self):
     l = list_ops.empty_tensor_list(
-        element_dtype=dtypes.float32,
-        element_shape=scalar_shape(),
-        max_num_elements=3)
+        element_dtype=dtypes.float32, element_shape=[], max_num_elements=3)
     with self.assertRaisesRegexp(
         errors.InvalidArgumentError,
         "Trying to modify element 0 in a list with 0 elements."):
@@ -354,7 +348,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testUnknownShape(self):
     l = list_ops.empty_tensor_list(
-        element_dtype=dtypes.float32, element_shape=-1)
+        element_dtype=dtypes.float32, element_shape=None)
     l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
     l = list_ops.tensor_list_push_back(l, constant_op.constant([1.0, 2.0]))
     l, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
@@ -366,7 +360,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     if not context.num_gpus():
       return
     t = constant_op.constant([1.0, 2.0])
-    l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape())
+    l = list_ops.tensor_list_from_tensor(t, element_shape=[])
     with context.device("gpu:0"):
       l_gpu = array_ops.identity(l)
       self.assertAllEqual(
@@ -383,7 +377,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     if not context.num_gpus():
       return
     t = constant_op.constant([1.0, 2.0])
-    child_l = list_ops.tensor_list_from_tensor(t, element_shape=scalar_shape())
+    child_l = list_ops.tensor_list_from_tensor(t, element_shape=[])
     l = list_ops.empty_tensor_list(
         element_shape=constant_op.constant([], dtype=dtypes.int32),
         element_dtype=dtypes.variant)
@@ -495,9 +489,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with ops.Graph().as_default(), session.Session(target=worker.target):
       with ops.device("/job:worker"):
         l = list_ops.tensor_list_reserve(
-            element_dtype=dtypes.float32,
-            element_shape=scalar_shape(),
-            num_elements=2)
+            element_dtype=dtypes.float32, element_shape=[], num_elements=2)
         l = list_ops.tensor_list_set_item(l, 0, 1.)
       with ops.device("/job:ps"):
         l_ps = array_ops.identity(l)
@@ -512,7 +504,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with ops.Graph().as_default(), session.Session(target=worker.target):
       with ops.device("/job:worker"):
         t = constant_op.constant([[1.0], [2.0]])
-        l = list_ops.tensor_list_from_tensor(t, element_shape=-1)
+        l = list_ops.tensor_list_from_tensor(t, element_shape=None)
       with ops.device("/job:ps"):
         l_ps = array_ops.identity(l)
         element_shape = list_ops.tensor_list_element_shape(
@@ -529,7 +521,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with ops.Graph().as_default(), session.Session(target=worker.target):
       with ops.device("/job:worker"):
         l = list_ops.empty_tensor_list(
-            element_shape=-1, element_dtype=dtypes.float32, max_num_elements=2)
+            element_shape=None,
+            element_dtype=dtypes.float32,
+            max_num_elements=2)
         l = list_ops.tensor_list_push_back(l, 1.)
       with ops.device("/job:ps"):
         l_ps = array_ops.identity(l)
@@ -543,8 +537,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testPushPopGradients(self):
     with backprop.GradientTape() as tape:
-      l = list_ops.empty_tensor_list(element_dtype=dtypes.float32,
-                                     element_shape=scalar_shape())
+      l = list_ops.empty_tensor_list(
+          element_dtype=dtypes.float32, element_shape=[])
       c = constant_op.constant(1.0)
       tape.watch(c)
       l = list_ops.tensor_list_push_back(l, c)
@@ -556,7 +550,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with backprop.GradientTape() as tape:
       c = constant_op.constant([1.0, 2.0])
       tape.watch(c)
-      l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
+      l = list_ops.tensor_list_from_tensor(c, element_shape=[])
       c2 = list_ops.tensor_list_stack(
           l, element_dtype=dtypes.float32, num_elements=2)
       result = c2 * 2.0
@@ -567,7 +561,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with backprop.GradientTape() as tape:
       c = constant_op.constant([1.0, 2.0])
       tape.watch(c)
-      l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
+      l = list_ops.tensor_list_from_tensor(c, element_shape=[])
       c2 = constant_op.constant(3.0)
       tape.watch(c2)
       l = list_ops.tensor_list_set_item(l, 0, c2)
@@ -580,7 +574,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testSetOutOfBounds(self):
     c = constant_op.constant([1.0, 2.0])
-    l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
+    l = list_ops.tensor_list_from_tensor(c, element_shape=[])
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(list_ops.tensor_list_set_item(l, 20, 3.0))
 
@@ -588,7 +582,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       ph = array_ops.placeholder(dtypes.float32)
       c = constant_op.constant([1.0, 2.0])
-      l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
+      l = list_ops.tensor_list_from_tensor(c, element_shape=[])
       # Set a placeholder with unknown shape to satisfy the shape inference
       # at graph building time.
       l = list_ops.tensor_list_set_item(l, 0, ph)
@@ -599,7 +593,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testResourceVariableScatterGather(self):
     c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32)
-    l = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
+    l = list_ops.tensor_list_from_tensor(c, element_shape=[])
     v = vs.get_variable("var", initializer=[l] * 10, use_resource=True)
     v_r_0_stacked = list_ops.tensor_list_stack(v[0], dtypes.float32)
     self.evaluate(v.initializer)
@@ -607,10 +601,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     v_r_sparse_stacked = list_ops.tensor_list_stack(
         v.sparse_read(0), dtypes.float32)
     self.assertAllEqual([1.0, 2.0], self.evaluate(v_r_sparse_stacked))
-    l_new_0 = list_ops.tensor_list_from_tensor(
-        [3.0, 4.0], element_shape=scalar_shape())
-    l_new_1 = list_ops.tensor_list_from_tensor(
-        [5.0, 6.0], element_shape=scalar_shape())
+    l_new_0 = list_ops.tensor_list_from_tensor([3.0, 4.0], element_shape=[])
+    l_new_1 = list_ops.tensor_list_from_tensor([5.0, 6.0], element_shape=[])
     updated_v = state_ops.scatter_update(v, [3, 5], [l_new_0, l_new_1])
     updated_v_elems = array_ops.unstack(updated_v)
     updated_v_stacked = [
@@ -622,8 +614,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testConcat(self):
     c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32)
-    l0 = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
-    l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=scalar_shape())
+    l0 = list_ops.tensor_list_from_tensor(c, element_shape=[])
+    l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=[])
     l_batch_0 = array_ops.stack([l0, l1])
     l_batch_1 = array_ops.stack([l1, l0])
 
@@ -659,7 +651,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.evaluate(
           list_ops.tensor_list_concat_lists(
               l_batch_0,
-              list_ops.empty_tensor_list(scalar_shape(), dtypes.float32),
+              list_ops.empty_tensor_list([], dtypes.float32),
               element_dtype=dtypes.float32))
 
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
@@ -673,16 +665,15 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r"input_b\[0\].dtype != element_dtype."):
       l_batch_of_int_tls = array_ops.stack(
-          [list_ops.tensor_list_from_tensor([1], element_shape=scalar_shape())]
-          * 2)
+          [list_ops.tensor_list_from_tensor([1], element_shape=[])] * 2)
       self.evaluate(
           list_ops.tensor_list_concat_lists(l_batch_0, l_batch_of_int_tls,
                                             element_dtype=dtypes.float32))
 
   def testPushBackBatch(self):
     c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32)
-    l0 = list_ops.tensor_list_from_tensor(c, element_shape=scalar_shape())
-    l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=scalar_shape())
+    l0 = list_ops.tensor_list_from_tensor(c, element_shape=[])
+    l1 = list_ops.tensor_list_from_tensor([-1.0], element_shape=[])
     l_batch = array_ops.stack([l0, l1])
     l_push = list_ops.tensor_list_push_back_batch(l_batch, [3.0, 4.0])
     l_unstack = array_ops.unstack(l_push)
@@ -726,7 +717,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                   dtypes.float64, dtypes.complex64, dtypes.complex128,
                   dtypes.bool):
       l_empty = list_ops.empty_tensor_list(
-          element_dtype=dtype, element_shape=scalar_shape())
+          element_dtype=dtype, element_shape=[])
       l_empty_zeros = array_ops.zeros_like(l_empty)
       t_empty_zeros = list_ops.tensor_list_stack(
           l_empty_zeros, element_dtype=dtype)
@@ -750,10 +741,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                   dtypes.float64, dtypes.complex64, dtypes.complex128,
                   dtypes.bool):
       l = list_ops.empty_tensor_list(
-          element_dtype=dtypes.variant, element_shape=scalar_shape())
+          element_dtype=dtypes.variant, element_shape=[])
 
-      sub_l = list_ops.empty_tensor_list(
-          element_dtype=dtype, element_shape=scalar_shape())
+      sub_l = list_ops.empty_tensor_list(element_dtype=dtype, element_shape=[])
       l = list_ops.tensor_list_push_back(l, sub_l)
       sub_l = list_ops.tensor_list_push_back(sub_l, math_ops.cast(
           1, dtype=dtype))
@@ -786,13 +776,12 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testElementShape(self):
     l = list_ops.empty_tensor_list(
-        element_dtype=dtypes.float32, element_shape=-1)
+        element_dtype=dtypes.float32, element_shape=None)
     shape = list_ops.tensor_list_element_shape(l, shape_type=dtypes.int32)
     self.assertEqual(self.evaluate(shape), -1)
 
   def testZerosLikeUninitialized(self):
-    l0 = list_ops.tensor_list_reserve(
-        scalar_shape(), 3, element_dtype=dtypes.float32)
+    l0 = list_ops.tensor_list_reserve([], 3, element_dtype=dtypes.float32)
     l1 = list_ops.tensor_list_set_item(l0, 0, 1.)  # [1., _, _]
     zeros_1 = array_ops.zeros_like(l1)  # [0., _, _]
     l2 = list_ops.tensor_list_set_item(l1, 2, 2.)  # [1., _, 2.]
@@ -808,6 +797,43 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(self.evaluate(res_1), [0.])
     self.assertAllEqual(self.evaluate(res_2), [0., 0.])
 
+  def testSkipEagerTensorListGetItemGradAggregation(self):
+    l = list_ops.tensor_list_reserve(
+        element_shape=[], num_elements=1, element_dtype=dtypes.float32)
+    x = constant_op.constant(1.0)
+    l = list_ops.tensor_list_set_item(l, 0, x)
+    l_read1 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+    l_read2 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+    grad = gradients_impl.gradients([l_read1, l_read2], [x])
+    with self.cached_session() as sess:
+      self.assertSequenceEqual(sess.run(grad), [2.])
+
+  def testSkipEagerBuildElementShape(self):
+    fn = list_ops._build_element_shape
+    # Unknown shape -> -1.
+    self.assertEqual(fn(None), -1)
+    self.assertEqual(fn(tensor_shape.unknown_shape()), -1)
+    # Scalar shape -> [] with type int32.
+    self.assertEqual(fn([]).dtype, dtypes.int32)
+    self.assertEqual(fn(tensor_shape.scalar()).dtype, dtypes.int32)
+    self.assertAllEqual(self.evaluate(fn([])), np.array([], np.int32))
+    self.assertAllEqual(
+        self.evaluate(fn(tensor_shape.scalar())), np.array([], np.int32))
+    # Tensor -> Tensor
+    shape = constant_op.constant(1)
+    self.assertIs(fn(shape), shape)
+    # Shape with unknown dims -> shape list with -1's.
+    shape = [None, 5]
+    self.assertAllEqual(fn(shape), [-1, 5])
+    self.assertAllEqual(fn(tensor_shape.TensorShape(shape)), [-1, 5])
+    # Shape with unknown dims and tensor dims -> shape list with -1's and tensor
+    # dims.
+    t = array_ops.placeholder(dtypes.int32)
+    shape = [None, 5, t]
+    result = fn(shape)
+    self.assertAllEqual(result[:2], [-1, 5])
+    self.assertIs(result[2], t)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 08578c0c49..7e8db8947b 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -63,6 +63,8 @@ def _make_ta(size, name, dtype=dtypes.float32, infer_shape=False):
       dtype=dtype, tensor_array_name=name, size=size, infer_shape=infer_shape)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+@test_util.with_control_flow_v2
 class TensorArrayTest(test.TestCase):
 
   @classmethod
@@ -121,13 +123,14 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayWritePack(dtypes.int64)
     self._testTensorArrayWritePack(dtypes.complex64)
     self._testTensorArrayWritePack(dtypes.complex128)
-    self._testTensorArrayWritePack(dtypes.string)
+    if not (test.is_gpu_available() and
+            tensor_array_ops.ENABLE_TENSOR_ARRAY_V2):
+      # TODO(b/119684648): Enable this.
+      self._testTensorArrayWritePack(dtypes.string)
 
-  @test_util.run_in_graph_and_eager_modes
   def testTensorArrayWritePack(self):
     self._testTensorArrayWritePackMaybeLegacy()
 
-  @test_util.run_in_graph_and_eager_modes
   def testEmptyTensorArrayPack(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -161,7 +164,7 @@ class TensorArrayTest(test.TestCase):
           convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0],
                    [106.0, 107.0], [8.0, 9.0]]), c0)
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
   def testTensorArrayWriteConcat(self):
     self._testTensorArrayWriteConcat(dtypes.float32)
     self._testTensorArrayWriteConcat(dtypes.float64)
@@ -184,7 +187,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                           self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118890905")
   def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros()
 
@@ -200,7 +203,7 @@ class TensorArrayTest(test.TestCase):
     self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                         self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118890905")
   def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros()
 
@@ -249,9 +252,11 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayUnpackRead(dtypes.int64)
     self._testTensorArrayUnpackRead(dtypes.complex64)
     self._testTensorArrayUnpackRead(dtypes.complex128)
-    self._testTensorArrayUnpackRead(dtypes.string)
+    if not (test.is_gpu_available() and
+            tensor_array_ops.ENABLE_TENSOR_ARRAY_V2):
+      # TODO(b/119684648): Enable this.
+      self._testTensorArrayUnpackRead(dtypes.string)
 
-  @test_util.run_in_graph_and_eager_modes
   def testTensorArrayUnpackRead(self):
     self._testTensorArrayUnpackReadMaybeLegacy()
 
@@ -297,7 +302,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(convert([]).reshape(0, 2), d1)
       self.assertAllEqual(convert([[3.0, 301.0]]), d2)
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   def testTensorArraySplitRead(self):
     self._testTensorArraySplitRead(dtypes.float32)
     self._testTensorArraySplitRead(dtypes.float64)
@@ -307,7 +312,8 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArraySplitRead(dtypes.complex128)
     self._testTensorArraySplitRead(dtypes.string)
 
-  def testTensorGradArrayWriteRead(self):
+  @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  def testSkipEagerTensorGradArrayWriteRead(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
@@ -340,7 +346,27 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[2.0]], g_d1)
       self.assertAllEqual(-2.0, g_d2)
 
-  def testTensorGradArrayDynamicWriteRead(self):
+  def testSkipEagerTensorArrayGradGrad(self):
+    if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2:
+      self.skipTest("Legacy TensorArray does not support double derivatives.")
+    with self.test_session(use_gpu=True) as session:
+      x = constant_op.constant(4.0)
+
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32,
+          tensor_array_name="foo",
+          size=1,
+          infer_shape=False)
+      w0 = ta.write(0, x)
+      r0 = w0.read(0)
+      y = r0 * r0
+
+      g1 = gradients_impl.gradients(ys=[y], xs=[x])
+      g2 = gradients_impl.gradients(ys=[g1], xs=[x])
+      self.assertAllEqual([2.0], session.run(g2))
+
+  @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  def testSkipEagerTensorGradArrayDynamicWriteRead(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
@@ -381,7 +407,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(3, vs)
       self.assertAllEqual(3, g_vs)
 
-  def testTensorGradAccessTwiceReceiveSameObject(self):
+  @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  def testSkipEagerTensorGradAccessTwiceReceiveSameObject(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
@@ -397,26 +424,39 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(t_g_ta_0, t_g_ta_1)
       self.assertAllEqual([[4.0, 5.0]], d_r1_0)
 
-  @test_util.run_in_graph_and_eager_modes
   def testTensorArrayWriteWrongIndexOrDataTypeFails(self):
     with self.session(use_gpu=True):
       ta = _make_ta(3, "foo", dtype=dtypes.float32)
       # Test writing the wrong datatype
-      with self.assertRaisesOpError(
-          "TensorArray dtype is (float|float32) but Op is trying to write "
-          "dtype string"):
+      if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and
+          not context.executing_eagerly()):
+        error_msg = ("Invalid data types; op elements string but list elements "
+                     "float")
+      else:
+        error_msg = (
+            "TensorArray dtype is (float|float32) but Op is trying to write "
+            "dtype string")
+      with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.write(0, "wrong_type_scalar").flow)
 
-      with self.assertRaisesOpError("index -1"):
+      if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and
+          not context.executing_eagerly()):
+        error_msg = "Trying to modify element -1 in a list with 3 elements."
+      else:
+        error_msg = "index -1"
+      with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.write(-1, 3.0).flow)
 
+      if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and
+          not context.executing_eagerly()):
+        error_msg = "Trying to modify element 3 in a list with 3 elements"
+      else:
+        error_msg = ("Tried to write to index 3 but array is not "
+                     "resizeable and size is: 3")
       # Test reading from too large an index
-      with self.assertRaisesOpError(
-          "Tried to write to index 3 but array is not "
-          "resizeable and size is: 3"):
+      with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.write(3, 3.0).flow)
 
-  @test_util.run_in_graph_and_eager_modes
   def testTensorArrayReadWrongIndexOrDataTypeFails(self):
     with self.session(use_gpu=True):
       ta = _make_ta(3, "foo", dtype=dtypes.float32)
@@ -424,23 +464,34 @@ class TensorArrayTest(test.TestCase):
       w0 = ta.write(0, [[4.0, 5.0]])
 
       # Test reading wrong datatype (only possible when constructing graphs).
-      if not context.executing_eagerly():
+      if (not context.executing_eagerly() and
+          not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2):
         r0_bad = gen_data_flow_ops.tensor_array_read_v3(
             handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow)
         with self.assertRaisesOpError(
             "TensorArray dtype is float but Op requested dtype double."):
           self.evaluate(r0_bad)
 
+      if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and
+          not context.executing_eagerly()):
+        error_msg = "Trying to access element -1 in a list with 3 elements."
+      else:
+        error_msg = "index -1"
       # Test reading from a negative index, which is not allowed
-      with self.assertRaisesOpError("index -1"):
+      with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.read(-1))
 
+      if (tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and
+          not context.executing_eagerly()):
+        error_msg = "Trying to access element 3 in a list with 3 elements."
+      else:
+        error_msg = "Tried to read from index 3 but array size is: 3"
       # Test reading from too large an index
-      with self.assertRaisesOpError(
-          "Tried to read from index 3 but array size is: 3"):
+      with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.read(3))
 
-  def testTensorArrayWriteMultipleFails(self):
+  @test_util.disable_control_flow_v2("v2 allows multiple writes.")
+  def testSkipEagerTensorArrayWriteMultipleFails(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
@@ -450,7 +501,7 @@ class TensorArrayTest(test.TestCase):
           "it has already been written to."):
         self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow)
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
   def testTensorArrayConcatIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -482,7 +533,7 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaisesOpError("shape"):
         self.evaluate(w3.concat())
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   def testTensorArraySplitIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
       in_eager_mode = context.executing_eagerly()
@@ -546,12 +597,14 @@ class TensorArrayTest(test.TestCase):
           r"existing shape is \[\] but the new input shape is \[1\]"):
         wb1_grad.flow.eval()
 
-  def testTensorArrayWriteGradientAddMultipleAdds(self):
+  @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  def testSkipEagerTensorArrayWriteGradientAddMultipleAdds(self):
     for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
                   dtypes.complex64, dtypes.complex128):
       self._testTensorArrayWriteGradientAddMultipleAdds(dtype)
 
-  def testTensorArrayGradWithShapeKnownElementShape(self):
+  @test_util.disable_control_flow_v2("Low level legacy TA op test.")
+  def testSkipEagerTensorArrayGradWithShapeKnownElementShape(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
           size=3,
@@ -580,7 +633,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllClose(fed_value,
                           sess.run(read_value, feed_dict={value: fed_value}))
 
-  def testTensorArrayGradWithShapeUnknownElementShape(self):
+  @test_util.disable_control_flow_v2("Low level legacy TA op test.")
+  def testSkipEagerTensorArrayGradWithShapeUnknownElementShape(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
           size=3, dtype=dtypes.float32,
@@ -603,7 +657,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllClose(fed_value,
                           sess.run(read_value, feed_dict={value: fed_value}))
 
-  @test_util.run_in_graph_and_eager_modes
   def testMultiTensorArray(self):
     with self.session(use_gpu=True):
       h1 = tensor_array_ops.TensorArray(
@@ -667,7 +720,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(c([[3.0, 2.0]]), grad_vals[0])
       self.assertAllEqual(c(-2.0), grad_vals[1])
 
-  def testTensorArrayGradientWriteRead(self):
+  def testSkipEagerTensorArrayGradientWriteRead(self):
     for dtype in (np.float32, np.float64, np.complex64, np.complex128):
       self._testTensorArrayGradientWriteReadType(dtype)
 
@@ -703,10 +756,11 @@ class TensorArrayTest(test.TestCase):
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
 
-  def testTensorArrayGradientWritePackConcatAndRead(self):
+  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  def testSkipEagerTensorArrayGradientWritePackConcatAndRead(self):
     self._testTensorArrayGradientWritePackConcatAndRead()
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("v2 does not support clear_after_read.")
   def testTensorArrayReadTwice(self):
     with self.session(use_gpu=True):
       value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
@@ -760,10 +814,11 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(len(grad_vals), 1)
       self.assertAllEqual([[2.0 - 1.5, 3.0 + 1.5], [4.0, 5.0]], grad_vals[0])
 
-  def testTensorArrayGradientUnpackRead(self):
+  def testSkipEagerTensorArrayGradientUnpackRead(self):
     self._testTensorArrayGradientUnpackRead()
 
-  def testTensorArrayGradientSplitConcat(self):
+  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  def testSkipEagerTensorArrayGradientSplitConcat(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=2,
@@ -808,17 +863,15 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(len(grad_vals), 1)
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  def testTensorArrayGradientDynamicUnpackRead(self):
+  def testSkipEagerTensorArrayGradientDynamicUnpackRead(self):
     self._testTensorArrayGradientDynamicUnpackRead()
 
-  @test_util.run_in_graph_and_eager_modes
   def testCloseTensorArray(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
       self.evaluate(ta.close())
 
-  @test_util.run_in_graph_and_eager_modes
   def testSizeTensorArray(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -826,7 +879,6 @@ class TensorArrayTest(test.TestCase):
       s = ta.size()
       self.assertAllEqual(3, self.evaluate(s))
 
-  @test_util.run_in_graph_and_eager_modes
   def testWriteCloseTensorArray(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -924,7 +976,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllClose(grad_val.sum(axis=0), var_grad_t)
       self.assertAllClose(grad_val.sum(axis=0), state0_grad_t)
 
-  @test_util.run_in_graph_and_eager_modes
   def testWhileLoopWritePackGradients(self):
     self._testWhileLoopWritePackGradients(
         dynamic_size=False, dtype=dtypes.float32)
@@ -932,11 +983,27 @@ class TensorArrayTest(test.TestCase):
     # self._testWhileLoopWritePackGradients(
     #     dynamic_size=False, dtype=tf.int64)
 
-  def testWhileLoopDynamicWritePackGradients(self):
+  @test_util.disable_control_flow_v2("Testing v1 while_loop with v2 TA")
+  @test_util.enable_tensor_array_v2
+  def testWhileLoopV1WithTensorArrayV2(self):
+    size = 3
+    ta = tensor_array_ops.TensorArray(
+        dtype=dtypes.int32, size=size, element_shape=tensor_shape.scalar())
+
+    def Body(counter, ta):
+      return counter + 1, ta.write(counter, counter)
+
+    _, ta = control_flow_ops.while_loop(lambda i, _: i < size, Body, [0, ta])
+
+    for i in range(size):
+      self.assertEqual(self.evaluate(ta.read(i)), i)
+
+  @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
+  def testSkipEagerWhileLoopDynamicWritePackGradients(self):
     self._testWhileLoopWritePackGradients(
         dynamic_size=True, dtype=dtypes.float32)
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/119323158")
   def testGradSerialTwoLoops(self):
     with self.session(use_gpu=True):
       def loop(x):
@@ -976,7 +1043,7 @@ class TensorArrayTest(test.TestCase):
         grad = gradients_impl.gradients(loop(x), [x])[0]
       self.assertAllClose(31.0, self.evaluate(grad))
 
-  def testSumOfTwoReadVariablesWithoutRepeatGrad(self):
+  def testSkipEagerSumOfTwoReadVariablesWithoutRepeatGrad(self):
     with self.session(use_gpu=True) as session:
       a = array_ops.identity(
           np.arange(
@@ -1011,7 +1078,7 @@ class TensorArrayTest(test.TestCase):
   def _grad_source_for_name(self, name):
     return tensor_array_grad._GetGradSource(constant_op.constant(0, name=name))
 
-  def testGetGradSource_Invalid(self):
+  def testSkipEagerGetGradSource_Invalid(self):
     with self.assertRaises(ValueError):
       self._grad_source_for_name("")
     with self.assertRaises(ValueError):
@@ -1019,7 +1086,7 @@ class TensorArrayTest(test.TestCase):
     with self.assertRaises(ValueError):
       self._grad_source_for_name("foo/bar")
 
-  def testGetGradSource_NoEnclosingScope(self):
+  def testSkipEagerGetGradSource_NoEnclosingScope(self):
     self.assertEqual("gradients:0", self._grad_source_for_name("gradients"))
     self.assertEqual("gradients_0:0", self._grad_source_for_name("gradients_0"))
     self.assertEqual("gradients", self._grad_source_for_name("gradients/foo"))
@@ -1030,7 +1097,7 @@ class TensorArrayTest(test.TestCase):
     self.assertEqual("gradients_0",
                      self._grad_source_for_name("gradients_0/foo/bar"))
 
-  def testGetGradSource_EnclosingScope(self):
+  def testSkipEagerGetGradSource_EnclosingScope(self):
     self.assertEqual("foo/gradients:0",
                      self._grad_source_for_name("foo/gradients"))
     self.assertEqual("foo/gradients_0:0",
@@ -1044,12 +1111,12 @@ class TensorArrayTest(test.TestCase):
     self.assertEqual("foo/bar/gradients_0",
                      self._grad_source_for_name("foo/bar/gradients_0/baz"))
 
-  def testGetGradSource_NestedUsesInnermost(self):
+  def testSkipEagerGetGradSource_NestedUsesInnermost(self):
     self.assertEqual(
         "foo/gradients/bar/gradients_0",
         self._grad_source_for_name("foo/gradients/bar/gradients_0/baz"))
 
-  def testWriteShape(self):
+  def testSkipEagerWriteShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
@@ -1073,7 +1140,8 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaises(ValueError):
         w0.write(0, c2)
 
-  def testPartlyUnknownShape(self):
+  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  def testSkipEagerPartlyUnknownShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=6)
@@ -1113,7 +1181,6 @@ class TensorArrayTest(test.TestCase):
       r5 = w5.read(0)
       self.assertAllEqual([5, 4, 2, 3], r5.get_shape().as_list())
 
-  @test_util.run_in_graph_and_eager_modes
   def _testUnpackShape(self):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1144,10 +1211,11 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaises(ValueError):
         w1.write(4, c2)
 
+  @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
   def testUnpackShape(self):
     self._testUnpackShape()
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   def testSplitShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1178,7 +1246,7 @@ class TensorArrayTest(test.TestCase):
             tensor_shape.TensorShape(
                 ta1.handle.op.get_attr("element_shape")).ndims, None)
 
-  def testWriteUnknownShape(self):
+  def testSkipEagerWriteUnknownShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
@@ -1201,7 +1269,11 @@ class TensorArrayTest(test.TestCase):
       grad_r0_vals = session.run(grad_r0)[0]
       self.assertAllEqual(grad_r0_vals, [1.0, 0.0])
 
-  def testGradientWhenNotAllComponentsRead(self):
+  # TODO(srbs): Figure out how to enable this. This is probably failing
+  # because we are trying to stack a TensorList with invalid tensors.
+  # That is because we do not receive gradients for all list indices.
+  # Figure out how TensorArray handles this.
+  def disabletestGradientWhenNotAllComponentsRead(self):
     self._testGradientWhenNotAllComponentsRead()
 
   def _testTensorArrayUnpackDynamic(self):
@@ -1216,10 +1288,12 @@ class TensorArrayTest(test.TestCase):
       grad = gradients_impl.gradients(ys=[r], xs=[x])
       self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
 
-  def testTensorArrayUnpackDynamic(self):
+  @test_util.disable_control_flow_v2("b/117943489")
+  def testSkipEagerTensorArrayUnpackDynamic(self):
     self._testTensorArrayUnpackDynamic()
 
-  def testTensorArraySplitDynamic(self):
+  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  def testSkipEagerTensorArraySplitDynamic(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=3, dynamic_size=True)
@@ -1235,13 +1309,17 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      with self.assertRaisesOpError(
-          "TensorArray has size zero, but element shape <unknown> is not fully "
-          "defined. Currently only static shapes are supported when packing "
-          "zero-size TensorArrays."):
+      v2_msg = ("Tried to stack elements of a empty list with "
+                "non-fully-defined shape")
+      v1_msg = (
+          "TensorArray has size zero, but element shape <unknown> is not "
+          "fully defined. Currently only static shapes are supported when "
+          "packing zero-size TensorArrays.")
+      with self.assertRaisesOpError(v2_msg if tensor_array_ops
+                                    .ENABLE_TENSOR_ARRAY_V2 else v1_msg):
         ta.stack().eval()
 
-  def testTensorArrayEvalEmpty(self):
+  def testSkipEagerTensorArrayEvalEmpty(self):
     self._testTensorArrayEvalEmpty()
 
   # this test is ill-defined for Eager mode --- unpacking an empty tensor
@@ -1260,10 +1338,12 @@ class TensorArrayTest(test.TestCase):
       # first dimension of zero
       self.assertAllEqual([0, 5], self.evaluate(concatenated).shape)
 
-  def testTensorArrayEvalEmptyWithDefault(self):
+  @test_util.disable_control_flow_v2("b/117943489")
+  def testSkipEagerTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
 
-  def testTensorArrayScatterReadAndGradients(self):
+  @test_util.disable_control_flow_v2("b/117943489")
+  def testSkipEagerTensorArrayScatterReadAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
@@ -1289,7 +1369,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_control_flow_v2("b/117943286")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -1326,7 +1406,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[1.0, -1.0], [8.0, -8.0]], g_vals[0])
       self.assertAllEqual(expected_grad, grad_vals[0])
 
-  def testTensorArrayGetsDeviceFromFirstWrite(self):
+  @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
+  def testSkipEagerTensorArrayGetsDeviceFromFirstWrite(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       # this initial device will be ignored.
       ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2)
@@ -1374,7 +1455,8 @@ class TensorArrayTest(test.TestCase):
         self.assertFalse(
             [s for s in dev_stats[d] if "/TensorArray" in s.node_name])
 
-  def testTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self):
+  @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
+  def testSkipEagerTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2)
 
@@ -1403,7 +1485,8 @@ class TensorArrayTest(test.TestCase):
         self.assertFalse(
             [s for s in dev_stats[d] if "TensorArray" == s.node_name])
 
-  def testTensorArrayDisabledColocateWithFirstWriteCall(self):
+  @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
+  def testSkipEagerTensorArrayDisabledColocateWithFirstWriteCall(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=2, colocate_with_first_write_call=False)
@@ -1433,7 +1516,6 @@ class TensorArrayTest(test.TestCase):
         self.assertFalse(
             [s for s in dev_stats[d] if "TensorArray" == s.node_name])
 
-  @test_util.run_in_graph_and_eager_modes
   def testTensorArrayIdentity(self):
     with self.session(use_gpu=True):
       ta0 = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2,
@@ -1486,7 +1568,7 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(size0_v, 2)
       self.assertEqual(size1_v, 4)
 
-  def testTensorArrayGradYsInCorrectScope(self):
+  def testSkipEagerTensorArrayGradYsInCorrectScope(self):
     n_time = 1
     n_dim = 1
     x = constant_op.constant([[1.42]])
@@ -1504,7 +1586,7 @@ class TensorArrayTest(test.TestCase):
         vdx, vdy = sess.run([dx, dy])
       self.assertAllClose(vdx, vdy)
 
-  def testTensorArrayInt64GPU(self):
+  def testSkipEagerTensorArrayInt64GPU(self):
     if not test.is_gpu_available():
       return
     with self.session(use_gpu=True, force_gpu=True) as sess:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index b4a1fc6af6..515926002d 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_list_ops
 # go/tf-wildcard-import
@@ -41,12 +42,27 @@ def empty_tensor_list(element_shape,
     max_num_elements = -1
 
   return gen_list_ops.empty_tensor_list(
-      element_shape=element_shape,
+      element_shape=_build_element_shape(element_shape),
       element_dtype=element_dtype,
       max_num_elements=max_num_elements,
       name=name)
 
 
+def tensor_list_reserve(element_shape, num_elements, element_dtype, name=None):
+  return gen_list_ops.tensor_list_reserve(
+      element_shape=_build_element_shape(element_shape),
+      num_elements=num_elements,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_from_tensor(tensor, element_shape, name=None):
+  return gen_list_ops.tensor_list_from_tensor(
+      tensor=tensor,
+      element_shape=_build_element_shape(element_shape),
+      name=name)
+
+
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
@@ -65,14 +81,13 @@ def _PopBackGrad(op, dlist, delement):
 
 @ops.RegisterGradient("TensorListStack")
 def _TensorListStackGrad(unused_op, dtensor):
-  return gen_list_ops.tensor_list_from_tensor(dtensor,
-                                              element_shape=dtensor.shape[1:])
+  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:])
 
 
 @ops.RegisterGradient("TensorListFromTensor")
 def _TensorListFromTensorGrad(op, dlist):
   """Gradient for TensorListFromTensor."""
-  if op.inputs[0].shape.dims[0].value is not None:
+  if op.inputs[0].shape.dims and op.inputs[0].shape.dims[0].value is not None:
     num_elements = op.inputs[0].shape.dims[0].value
   else:
     num_elements = None
@@ -126,3 +141,40 @@ def _TensorListScatterGrad(op, dlist):
   t, indices, _ = op.inputs
   return gen_list_ops.tensor_list_gather(
       dlist, indices, element_dtype=t.dtype), None
+
+
+def _build_element_shape(shape):
+  """Converts shape to a format understood by list_ops for element_shape.
+
+  If `shape` is already a `Tensor` it is returned as-is. We do not perform a
+  type check here.
+
+  If shape is None or a TensorShape with unknown rank, -1 is returned.
+
+  If shape is a scalar, an int32 tensor with empty list is returned. Note we
+  do directly return an empty list since ops.convert_to_tensor would conver it
+  to a float32 which is not a valid type for element_shape.
+
+  If shape is a sequence of dims, None's in the list are replaced with -1. We
+  do not check the dtype of the other dims.
+
+  Args:
+    shape: Could be None, Tensor, TensorShape or a list of dims (each dim could
+      be a None, scalar or Tensor).
+
+  Returns:
+    A None-free shape that can be converted to a tensor.
+  """
+  if isinstance(shape, ops.Tensor):
+    return shape
+  if isinstance(shape, tensor_shape.TensorShape):
+    # `TensorShape.as_list` requires rank to be known.
+    shape = shape.as_list() if shape else None
+  # Shape is unknown.
+  if shape is None:
+    return -1
+  # Shape is a scalar.
+  if not shape:
+    return ops.convert_to_tensor(shape, dtype=dtypes.int32)
+  # Shape is a sequence of dimensions. Convert None dims to -1.
+  return [d if d is not None else -1 for d in shape]
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index c08a2b0499..e3375ad0ab 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -20,8 +20,10 @@ from __future__ import division
 from __future__ import print_function
 
 import contextlib
+import os
 import weakref
 
+from tensorflow.python import tf2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -30,12 +32,18 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_control_flow_ops
 from tensorflow.python.ops import gen_data_flow_ops
+from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.util import tf_should_use
 from tensorflow.python.util.tf_export import tf_export
 
 
+ENABLE_TENSOR_ARRAY_V2 = (
+    tf2.enabled() or os.getenv("TF_ENABLE_TENSOR_ARRAY_V2") is not None)
+
+
 # _GraphTensorArray accesses many of the hidden generated ops, but is in
 # fact built to wrap these methods.
 # pylint: disable=protected-access
@@ -393,6 +401,246 @@ class _GraphTensorArray(object):
     return gen_data_flow_ops.tensor_array_close_v3(
         handle=self._handle, name=name)
 
+
+class _GraphTensorArrayV2(object):
+  """Graph-mode implementation of TensorArray backed by TensorLists.
+
+  The backing tensor of this TensorArray is a TensorList variant tensor which is
+  stored in the `flow`. The `handle` is always none here. The reason we use the
+  `flow` field and not the `handle` field is to ensure backwards compatibility
+  with legacy control flow.
+  """
+
+  def __init__(self,
+               dtype,
+               size=None,
+               dynamic_size=None,
+               clear_after_read=None,
+               tensor_array_name=None,
+               handle=None,
+               flow=None,
+               infer_shape=True,
+               element_shape=None,
+               colocate_with_first_write_call=True,
+               name=None):
+    """Constructs a graph mode TensorArray.
+
+    Args:
+      dtype: (required) data type of the TensorArray.
+      size: (optional) int32 scalar `Tensor`: the size of the TensorArray.
+        Required if flow is not provided.
+      dynamic_size: (optional) Python bool: If true, writes to the TensorArray
+        can grow the TensorArray past its initial size.  Default: False.
+      clear_after_read: (optional) unused. Not supported in TensorLists.
+      tensor_array_name: (optional) unused.
+      handle: (optional) Must always be None.
+      flow: (optional) A variant `Tensor` scalar for a TensorList.
+      infer_shape: (optional, default: True) If True, shape inference is
+        enabled.  In this case, all elements must have the same shape.
+      element_shape: (optional, default: None) A `TensorShape` object specifying
+        the shape constraints of each of the elements of the TensorArray. Need
+        not be fully defined.
+      colocate_with_first_write_call: (optional). unused.
+      name: (optional) A name for the operation.
+
+    Raises:
+      ValueError: if both handle and tensor_array_name are provided.
+      TypeError: if handle is provided but is not a Tensor.
+    """
+    assert handle is None
+    del handle
+    del clear_after_read
+    del tensor_array_name
+    del colocate_with_first_write_call
+
+    del dynamic_size  # TODO(b/117943489): Unused for now.
+
+    if (flow is not None and
+        (not isinstance(flow, ops.Tensor) or flow.dtype != dtypes.variant)):
+      raise TypeError("flow must be a variant tensor")
+    if flow is None and size is None:
+      raise ValueError("Size must be provided if flow is not provided")
+    if flow is not None and size is not None:
+      raise ValueError("Cannot provide both a flow and size "
+                       "at the same time")
+    if flow is not None and element_shape is not None:
+      raise ValueError("Cannot provide both a flow and element_shape "
+                       "at the same time")
+
+    self._dtype = dtype
+
+    # Record the current static shape for the array elements. The element
+    # shape is defined either by `element_shape` or the shape of the tensor
+    # of the first write. If `infer_shape` is true, all writes checks for
+    # shape equality.
+    if element_shape is None:
+      self._infer_shape = infer_shape
+      self._element_shape = []
+    else:
+      self._infer_shape = True
+      self._element_shape = [tensor_shape.TensorShape(element_shape)]
+    with ops.name_scope(name, "TensorArrayV2", [size, flow]) as scope:
+      if flow is None:
+        self._flow = list_ops.tensor_list_reserve(
+            element_shape=element_shape,
+            num_elements=size,
+            element_dtype=dtype,
+            name=scope)
+      else:
+        self._flow = flow
+
+    # For backwards compatibility.
+    self._colocate_with_first_write_call = None
+    self._colocate_with = None
+
+  @property
+  def flow(self):
+    return self._flow
+
+  @property
+  def dtype(self):
+    return self._dtype
+
+  @property
+  def handle(self):
+    # We intentionally do not raise an error so that legacy while_loop does not
+    # complain.
+    return None
+
+  def _merge_element_shape(self, shape):
+    """Changes the element shape of the array given a shape to merge with.
+
+    Args:
+      shape: A `TensorShape` object to merge with.
+
+    Raises:
+      ValueError: if the provided shape is incompatible with the current
+          element shape of the `TensorArray`.
+    """
+
+    if self._element_shape:
+      if not shape.is_compatible_with(self._element_shape[0]):
+        raise ValueError(
+            "Inconsistent shapes: saw %s but expected %s "
+            "(and infer_shape=True)" % (shape, self._element_shape[0]))
+      self._element_shape[0] = self._element_shape[0].merge_with(shape)
+    else:
+      self._element_shape.append(shape)
+
+  def identity(self):
+    """See TensorArray."""
+    flow = array_ops.identity(self._flow)
+    ta = TensorArray(
+        dtype=self._dtype, flow=flow, infer_shape=self._infer_shape)
+    ta._element_shape = self._element_shape
+    return ta
+
+  def grad(self, source, flow=None, name=None):
+    """Not supported."""
+    raise NotImplementedError()
+
+  def read(self, index, name=None):
+    """See TensorArray."""
+    value = list_ops.tensor_list_get_item(
+        input_handle=self._flow,
+        index=index,
+        element_dtype=self._dtype,
+        name=name)
+    if self._element_shape:
+      value.set_shape(self._element_shape[0].dims)
+    return value
+
+  @tf_should_use.should_use_result
+  def write(self, index, value, name=None):
+    """See TensorArray."""
+    with ops.name_scope(name, "TensorArrayV2Write", [self._flow, index, value]):
+      value = ops.convert_to_tensor(value, name="value")
+      if self._infer_shape:
+        self._merge_element_shape(value.shape)
+      flow_out = list_ops.tensor_list_set_item(
+          input_handle=self._flow, index=index, item=value, name=name)
+      ta = TensorArray(dtype=self._dtype, handle=None, flow=flow_out)
+      ta._infer_shape = self._infer_shape
+      ta._element_shape = self._element_shape
+      return ta
+
+  def stack(self, name=None):
+    """See TensorArray."""
+    with ops.name_scope(name, "TensorArrayV2Stack", [self._flow]):
+      value = list_ops.tensor_list_stack(
+          input_handle=self._flow, element_dtype=self._dtype)
+      if self._element_shape and self._element_shape[0].dims is not None:
+        value.set_shape([None] + self._element_shape[0].dims)
+      return value
+
+  def gather(self, indices, name=None):
+    """See TensorArray."""
+    value = list_ops.tensor_list_gather(
+        input_handle=self._flow,
+        indices=indices,
+        element_dtype=self._dtype,
+        name=name)
+    if self._element_shape and self._element_shape[0].dims is not None:
+      value.set_shape([None] + self._element_shape[0].dims)
+    return value
+
+  def concat(self, name=None):
+    """See TensorArray."""
+    raise NotImplementedError("TensorArray.concat")
+
+  @tf_should_use.should_use_result
+  def unstack(self, value, name=None):
+    """See TensorArray."""
+    with ops.name_scope(name, "TensorArrayUnstack", [self._flow, value]):
+      value = ops.convert_to_tensor(value, name="value")
+      if self._infer_shape and not context.executing_eagerly():
+        self._merge_element_shape(value.shape[1:])
+      flow_out = list_ops.tensor_list_from_tensor(
+          tensor=value, element_shape=value.shape[1:])
+      ta = TensorArray(
+          dtype=self._dtype,
+          handle=self.handle,
+          flow=flow_out,
+          colocate_with_first_write_call=self._colocate_with_first_write_call)
+      ta._infer_shape = self._infer_shape
+      ta._element_shape = self._element_shape
+      ta._colocate_with = self._colocate_with
+      return ta
+
+  @tf_should_use.should_use_result
+  def scatter(self, indices, value, name=None):
+    """See TensorArray."""
+    with ops.name_scope(name, "TensorArrayScatter",
+                        [self._flow, value, indices]):
+      value = ops.convert_to_tensor(value, name="value")
+      if self._infer_shape and not context.executing_eagerly():
+        self._merge_element_shape(value.shape[1:])
+      flow_out = list_ops.tensor_list_scatter(
+          tensor=value, indices=indices, element_shape=-1)
+      ta = TensorArray(
+          dtype=self._dtype,
+          handle=self.handle,
+          flow=flow_out,
+          colocate_with_first_write_call=self._colocate_with_first_write_call)
+      ta._infer_shape = self._infer_shape
+      ta._element_shape = self._element_shape
+      ta._colocate_with = self._colocate_with
+      return ta
+
+  @tf_should_use.should_use_result
+  def split(self, value, lengths, name=None):
+    """See TensorArray."""
+    raise NotImplementedError("TensorArray.split")
+
+  def size(self, name=None):
+    """See TensorArray."""
+    return list_ops.tensor_list_length(input_handle=self._flow, name=name)
+
+  @tf_should_use.should_use_result
+  def close(self, name=None):
+    """See TensorArray."""
+    return gen_control_flow_ops.no_op(name=name)
+
 # pylint: enable=protected-access
 
 
@@ -738,8 +986,10 @@ class TensorArray(object):
     if context.executing_eagerly():
       implementation = _EagerTensorArray
     else:
-      implementation = _GraphTensorArray
-
+      if ENABLE_TENSOR_ARRAY_V2:
+        implementation = _GraphTensorArrayV2
+      else:
+        implementation = _GraphTensorArray
     self._implementation = implementation(
         dtype,
         size=size,
@@ -768,7 +1018,7 @@ class TensorArray(object):
   @property
   def handle(self):
     """The reference to the TensorArray."""
-    return self._implementation._handle
+    return self._implementation.handle
 
   @property
   def _infer_shape(self):
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 568531cc5c..5ab7bffedc 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -99,7 +99,8 @@ def while_loop(cond,
     # Add loop counter needed for computing gradients.
     loop_vars = [loop_counter] + loop_vars
 
-    shape_invariants = [tensor_shape.scalar()] + shape_invariants
+    shape_invariants = type(shape_invariants)([tensor_shape.scalar()
+                                              ]) + shape_invariants
 
     # Automatic control dependencies are added in defuns, but not in v1
     # graphs. Propagate that behavior here.
@@ -132,9 +133,8 @@ def while_loop(cond,
     # the value of that tensor in each iteration is the same as it was at the
     # beginning of the loop execution.
     loop_vars = loop_vars + cond_graph.external_captures
-    shape_invariants = shape_invariants + [
-        t.shape for t in cond_graph.external_captures
-    ]
+    shape_invariants = shape_invariants + type(shape_invariants)(
+        [t.shape for t in cond_graph.external_captures])
 
     def wrapped_body(loop_counter, *args):
       """Loop body augmented with counter update.
@@ -207,8 +207,7 @@ def while_loop(cond,
     for intermediate_tensor in intermediate_tensors:
       tensor_list = list_ops.empty_tensor_list(
           element_dtype=intermediate_tensor.dtype,
-          element_shape=_get_tensor_convertible_shape(
-              intermediate_tensor.shape),
+          element_shape=intermediate_tensor.shape,
           max_num_elements=maximum_iterations)
       loop_vars.append(tensor_list)
       with cond_graph.as_default():
@@ -315,7 +314,7 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   for intermediate_tensor in intermediate_tensors:
     tensor_list = list_ops.empty_tensor_list(
         element_dtype=intermediate_tensor.dtype,
-        element_shape=_get_tensor_convertible_shape(intermediate_tensor.shape),
+        element_shape=intermediate_tensor.shape,
         max_num_elements=maximum_iterations)
 
     with body_grad_graph.as_default():
@@ -819,18 +818,6 @@ def _is_in_xla_context():
   return control_flow_util.GetContainingXLAContext(cur_ctxt) is not None
 
 
-def _get_tensor_convertible_shape(shape):
-  assert isinstance(shape, tensor_shape.TensorShape)
-  if shape.is_fully_defined():
-    return shape
-  if not shape:  # Unknown shape.
-    return -1
-  # Partially defined shape.
-  shape_list = shape.as_list()
-  shape_list = [s if s is not None else -1 for s in shape_list]
-  return ops.convert_to_tensor(shape_list)
-
-
 def _graph_name(graph):
   if isinstance(graph, func_graph_module.FuncGraph):
     return graph.name
-- 
GitLab


From 19b23cb3a88b3d83bcbeb8c0cc0e4714181da4d8 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 16 Nov 2018 17:40:17 -0800
Subject: [PATCH 0460/1554] Change sparse.reduce_max(sp_input, axis, keepdims,
 reduction_axes, keep_dims) to sparse.reduce_max(sp_input, axis, keepdims,
 output_is_sparse, name)

and unify with reduce_max_sparse via dispatching (see new output_is_sparse_arg).

PiperOrigin-RevId: 221879420
---
 tensorflow/python/ops/sparse_ops.py           | 85 ++++++++++++++++++-
 .../api/golden/v2/tensorflow.sparse.pbtxt     |  6 +-
 2 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 6beceb83d5..a65778b9aa 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -939,7 +939,87 @@ def sparse_to_dense(sparse_indices,
       name=name)
 
 
-@tf_export("sparse.reduce_max", v1=["sparse.reduce_max", "sparse_reduce_max"])
+@tf_export("sparse.reduce_max", v1=[])
+def sparse_reduce_max_v2(
+    sp_input, axis=None, keepdims=None, output_is_sparse=False, name=None):
+  """Computes the max of elements across dimensions of a SparseTensor.
+
+  This Op takes a SparseTensor and is the sparse counterpart to
+  `tf.reduce_max()`.  In particular, this Op also returns a dense `Tensor`
+  if `output_is_sparse` is `False`, or a `SparseTensor` if `output_is_sparse`
+  is `True`.
+
+  Note: A gradient is not defined for this function, so it can't be used
+  in training models that need gradient descent.
+
+  Reduces `sp_input` along the dimensions given in `axis`.  Unless
+  `keepdims` is true, the rank of the tensor is reduced by 1 for each entry in
+  `axis`. If `keepdims` is true, the reduced dimensions are retained
+  with length 1.
+
+  If `axis` has no entries, all dimensions are reduced, and a tensor
+  with a single element is returned.  Additionally, the axes can be negative,
+  similar to the indexing rules in Python.
+
+  The values not defined in `sp_input` don't participate in the reduce max,
+  as opposed to be implicitly assumed 0 -- hence it can return negative values
+  for sparse `axis`. But, in case there are no values in
+  `axis`, it will reduce to 0. See second example below.
+
+  For example:
+
+  ```python
+  # 'x' represents [[1, ?, 2]
+  #                 [?, 3, ?]]
+  # where ? is implicitly-zero.
+  tf.sparse.reduce_max(x) ==> 3
+  tf.sparse.reduce_max(x, 0) ==> [1, 3, 2]
+  tf.sparse.reduce_max(x, 1) ==> [2, 3]  # Can also use -1 as the axis.
+  tf.sparse.reduce_max(x, 1, keepdims=True) ==> [[2], [3]]
+  tf.sparse.reduce_max(x, [0, 1]) ==> 3
+
+  # 'y' represents [[-7, ?]
+  #                 [ 4, 3]
+  #                 [ ?, ?]
+  tf.sparse.reduce_max(x, 1) ==> [-7, 4, 0]
+  ```
+
+  Args:
+    sp_input: The SparseTensor to reduce. Should have numeric type.
+    axis: The dimensions to reduce; list or scalar. If `None` (the
+      default), reduces all dimensions.
+    keepdims: If true, retain reduced dimensions with length 1.
+    output_is_sparse: If true, returns a `SparseTensor` instead of a dense
+      `Tensor` (the default).
+    name: A name for the operation (optional).
+
+
+  Returns:
+    The reduced Tensor or the reduced SparseTensor if `output_is_sparse` is
+    True.
+  """
+  if keepdims is None:
+    keepdims = False
+
+  # reduction_axes is the deprecated name for axis.
+  reduction_axes = None
+
+  if output_is_sparse:
+    output_ind, output_val, output_shape = (
+        gen_sparse_ops.sparse_reduce_max_sparse(
+            sp_input.indices, sp_input.values, sp_input.dense_shape,
+            math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims,
+            name=name))
+
+    return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
+
+  return gen_sparse_ops.sparse_reduce_max(
+      sp_input.indices, sp_input.values, sp_input.dense_shape,
+      math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims,
+      name=name)
+
+
+@tf_export(v1=["sparse.reduce_max", "sparse_reduce_max"])
 @deprecation.deprecated_endpoints("sparse_reduce_max")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
@@ -1007,8 +1087,7 @@ def sparse_reduce_max(sp_input, axis=None, keepdims=None,
       math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims)
 
 
-@tf_export("sparse.reduce_max_sparse",
-           v1=["sparse.reduce_max_sparse", "sparse_reduce_max_sparse"])
+@tf_export(v1=["sparse.reduce_max_sparse", "sparse_reduce_max_sparse"])
 @deprecation.deprecated_endpoints("sparse_reduce_max_sparse")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index 48aa1f7d8d..161744a914 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -58,11 +58,7 @@ tf_module {
   }
   member_method {
     name: "reduce_max"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "reduce_max_sparse"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'output_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_sum"
-- 
GitLab


From 8a36a1ef8102a4a04651820331342d4fd84071ad Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 16 Nov 2018 17:42:52 -0800
Subject: [PATCH 0461/1554] Disable cartpole benchmark in opensource.

PiperOrigin-RevId: 221879651
---
 tensorflow/contrib/autograph/examples/benchmarks/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/autograph/examples/benchmarks/BUILD b/tensorflow/contrib/autograph/examples/benchmarks/BUILD
index ed3b054774..6d2d70c99b 100644
--- a/tensorflow/contrib/autograph/examples/benchmarks/BUILD
+++ b/tensorflow/contrib/autograph/examples/benchmarks/BUILD
@@ -19,6 +19,8 @@ py_test(
     srcs = ["cartpole_benchmark.py"],
     tags = [
         "local",
+        "manual",
+        "no_oss",
         "notap",
         "nozapfhahn",
     ],
-- 
GitLab


From 6badd1f1569590e679a4f177f8d3dad0d88a11c4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 17:54:36 -0800
Subject: [PATCH 0462/1554] Bring Keras Distribution Strategies loops into
 training_arrays.py

PiperOrigin-RevId: 221880671
---
 tensorflow/python/keras/callbacks.py          |  31 +-
 tensorflow/python/keras/engine/training.py    |  20 +-
 .../python/keras/engine/training_arrays.py    |  61 ++-
 .../keras/engine/training_distributed.py      | 500 +++++-------------
 4 files changed, 208 insertions(+), 404 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index d90523121b..8223e795bc 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -54,6 +54,7 @@ except ImportError:
   requests = None
 
 
+# pylint: disable=protected-access
 def configure_callbacks(callbacks,
                         model,
                         do_validation=False,
@@ -114,12 +115,7 @@ def configure_callbacks(callbacks,
   callback_list = CallbackList(callbacks)
 
   # Set callback model
-  callback_model = model._get_callback_model()  # pylint: disable=protected-access
-  if do_validation and val_inputs and not context.executing_eagerly():
-    # Need to create the eval_function before start of the first epoch
-    # because TensorBoard callback on_epoch_begin adds summary to the
-    # list of fetches of the eval_function
-    callback_model._make_eval_function()  # pylint: disable=protected-access
+  callback_model = model._get_callback_model()
   callback_list.set_model(callback_model)
 
   # Set callback parameters
@@ -146,21 +142,28 @@ def configure_callbacks(callbacks,
 
   # Pass validation data to callbacks
   # TODO(omalleyt): remove this once val hooks are ready.
-  if not val_inputs:
+  if model._distribution_strategy or not val_inputs:
     val_data = []
-  elif _is_generator_like(val_inputs):
-    val_data = val_inputs
   else:
-    val_data = val_inputs + val_targets
-    if val_sample_weights:
-      val_data += val_sample_weights
-    if not isinstance(K.learning_phase(), int):
-      val_data += [0.]
+    if not model.run_eagerly:
+      # Need to create the eval_function before start of the first epoch
+      # because TensorBoard callback on_epoch_begin adds summary to the
+      # list of fetches of the eval_function
+      callback_model._make_eval_function()
+    if _is_generator_like(val_inputs):
+      val_data = val_inputs
+    else:
+      val_data = val_inputs + val_targets
+      if val_sample_weights:
+        val_data += val_sample_weights
+      if not isinstance(K.symbolic_learning_phase(), int):
+        val_data += [False]
   for cbk in callbacks:
     cbk.validation_data = val_data
 
   callback_list.model.stop_training = False
   return callback_list
+# pylint: enable=protected-access
 
 
 def _is_generator_like(data):
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 5d5af12805..610c8e625a 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1760,9 +1760,10 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps)
-    elif self._distribution_strategy:
-      return training_distributed.fit_loop(
-          self, x,
+    elif training_distributed.should_run_experimental_loop(self):
+      return training_distributed.experimental_fit_loop(
+          self,
+          x,
           epochs=epochs,
           verbose=verbose,
           callbacks=callbacks,
@@ -1909,12 +1910,9 @@ class Model(Network):
           batch_size=batch_size,
           verbose=verbose,
           steps=steps)
-    elif self._distribution_strategy:
-      return training_distributed.test_loop(
-          self,
-          iterator=x,
-          verbose=verbose,
-          steps=steps)
+    elif training_distributed.should_run_experimental_loop(self):
+      return training_distributed.experimental_test_loop(
+          self, iterator=x, verbose=verbose, steps=steps)
     else:
       return training_arrays.test_loop(
           self,
@@ -2009,8 +2007,8 @@ class Model(Network):
     if self.run_eagerly:
       return training_eager.predict_loop(
           self, x, batch_size=batch_size, verbose=verbose, steps=steps)
-    elif self._distribution_strategy:
-      results = training_distributed.predict_loop(
+    elif training_distributed.should_run_experimental_loop(self):
+      results = training_distributed.experimental_predict_loop(
           self, x, verbose=verbose, steps=steps)
       return results
     else:
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 48d6af7ec4..390357303e 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -26,6 +26,7 @@ import numpy as np
 from tensorflow.python.framework import errors
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
+from tensorflow.python.keras.engine import training_distributed
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.generic_utils import make_batches
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
@@ -178,6 +179,38 @@ def _make_logs(model, outputs, mode, prefix=''):
   return logs
 
 
+def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
+  """Prepare feed values to the model execution function.
+
+  Arguments:
+    model: Model to prepare feed values for.
+    inputs: List or dict of model inputs.
+    targets: Optional list of model targets.
+    sample_weights: Optional list of sample weight arrays.
+    mode: One of 'train'/'test'/'predict'.
+
+  Returns:
+    Feed values for the model in the given mode.
+  """
+  if model._distribution_strategy:
+    return training_distributed._prepare_feed_values(model, inputs, targets,
+                                                     sample_weights, mode)
+  inputs = training_utils.ModelInputs(inputs).as_list()
+  targets = targets or []
+  sample_weights = sample_weights or []
+  ins = inputs + targets + sample_weights
+  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
+    ins += [True]
+  return ins
+
+
+def _get_execution_function(model, mode):
+  """Get function to run one step of model execution."""
+  if model._distribution_strategy:
+    return training_distributed._get_execution_function(model, mode)
+  return model._get_execution_function(mode)
+
+
 def model_iteration(model,
                     inputs,
                     targets=None,
@@ -238,18 +271,18 @@ def model_iteration(model,
   if mode == 'train':
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
 
+  # Enter DistributionStrategy scope.
+  if model._distribution_strategy:
+    scope = model._distribution_strategy.scope()
+    scope.__enter__()
+
   # Get step function and loop type.
-  f = model._get_execution_function(mode)
+  f = _get_execution_function(model, mode)
   use_steps = steps_per_epoch is not None
   do_validation = val_inputs is not None
 
   # Prepare input data.
-  inputs = training_utils.ModelInputs(inputs).as_list()
-  targets = targets or []
-  sample_weights = sample_weights or []
-  ins = inputs + targets + sample_weights
-  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
-    ins += [True]
+  ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                    steps_per_epoch)
 
@@ -289,6 +322,9 @@ def model_iteration(model,
   else:
     aggregator = MetricsAggregator(use_steps, num_samples_or_steps)
 
+  if model._distribution_strategy:
+    training_distributed._copy_weights_to_distributed_model(model)
+
   callbacks.model.stop_training = False
   callbacks._call_begin_hook(mode)
   progbar.on_train_begin()
@@ -321,12 +357,15 @@ def model_iteration(model,
                           'can generate at least `steps_per_epoch * epochs` '
                           'batches (in this case, %d batches). You may need to'
                           'use the repeat() function when building your '
-                          'dataset.' %
-                          steps_per_epoch * epochs)
+                          'dataset.' % steps_per_epoch * epochs)
           break
         if not isinstance(batch_outs, list):
           batch_outs = [batch_outs]
 
+        if model._distribution_strategy:
+          batch_outs = training_distributed._per_device_aggregate_batch(
+              batch_outs, model, mode)
+
         # Aggregate results.
         if step == 0:
           aggregator.create(batch_outs)
@@ -417,6 +456,10 @@ def model_iteration(model,
     progbar.on_epoch_end(epoch, epoch_logs)
   callbacks._call_end_hook(mode)
 
+  if model._distribution_strategy:
+    training_distributed._copy_weights_to_original_model(model, mode)
+    scope.__exit__(None, None, None)
+
   if mode == 'train':
     return model.history
   return results
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 9bf41734c3..49ef44e4fd 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -48,187 +48,15 @@ class _Mode(enum.Enum):
 # TODO(priyag, sourabhbajaj): Refactor this file to address code duplication.
 
 
-def fit_loop(
-    model,
-    iterator,
-    epochs=100,
-    verbose=1,
-    callbacks=None,
-    val_iterator=None,
-    initial_epoch=0,
-    steps_per_epoch=None,
-    validation_steps=None):
-  """Fit loop for training with DistributionStrategy.
-
-  Arguments:
-      model: Keras Model instance.
-      iterator: Iterator for input data.
-      epochs: Number of times to iterate over the data
-      verbose: Integer, Verbosity mode, 0, 1 or 2
-      callbacks: List of callbacks to be called during training
-      val_iterator: Iterator for validation data.
-      initial_epoch: Epoch at which to start training
-          (useful for resuming a previous training run)
-      steps_per_epoch: Total number of steps (batches of samples)
-          before declaring one epoch finished and starting the
-          next epoch. Ignored with the default value of `None`.
-      validation_steps: Number of steps to run validation for
-          (only if doing validation from data tensors).
-          Ignored with the default value of `None`.
-
-  Returns:
-      `History` object.
-
-  Raises:
-      ValueError: in case of invalid arguments.
-  """
-  current_strategy = model._distribution_strategy
-
-  # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged.
-  if current_strategy.__class__.__name__ == 'TPUStrategy':
-    return _experimental_fit_loop(
-        model, iterator, epochs, verbose, callbacks, initial_epoch,
-        steps_per_epoch, val_iterator, validation_steps)
-
-  if not model._grouped_model:
-    clone_model_on_replicas(model, current_strategy, make_callback_model=True)
-
-  def _per_device_fit_function(model):
-    model._make_fit_function()
-    return (model._fit_function.inputs, model._fit_function.outputs,
-            model._fit_function.updates_op, model._fit_function.session_kwargs)
-
-  inputs, targets, sample_weights = _get_input_from_iterator(iterator, model)
-  with current_strategy.scope():
-    # Create train ops on each of the devices when we call
-    # `_per_device_fit_function`.
-    (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = current_strategy.call_for_each_replica(
-         _per_device_fit_function, args=(model._grouped_model,))
-
-    # Initialize the variables in the replicated model. This is necessary for
-    # multi-worker training because on some workers, initialization is not
-    # needed. This method does initialization or waiting for initialization
-    # according to the context object of distribute coordinator.
-    distributed_training_utils.init_restore_or_wait_for_variables()
-
-    # Unwrap all the per device values returned from `call_for_each_replica`.
-    # Unwrapping per device values gives you a list of values that can be
-    # used to construct a new train function that is composed of update ops on
-    # all the devices over which the model is distributed.
-    (all_inputs, all_outputs, all_updates,
-     all_session_args) = distributed_training_utils.unwrap_values(
-         current_strategy, grouped_inputs, grouped_outputs,
-         grouped_updates, grouped_session_args, with_loss_tensor=True)
-
-    # Dataset inputs and targets are also per devices values that need to be
-    # unwrapped.
-    dataset_inputs = distributed_training_utils.flatten_perdevice_values(
-        current_strategy, inputs)
-    dataset_targets = distributed_training_utils.flatten_perdevice_values(
-        current_strategy, targets)
-
-    # Create a train function that is composed of all the parameters above.
-    distributed_fit_function = K.function(
-        all_inputs,
-        all_outputs,
-        updates=all_updates,
-        name='distributed_fit_function',
-        **all_session_args)
-
-    # We need to set sample_weights to None since there are sample weight
-    # placeholders that are created with default values.
-    sample_weights = [None for _ in range(
-        len(model.outputs) * current_strategy.num_replicas_in_sync)]
-    if not isinstance(K.learning_phase(), int):
-      ins = dataset_inputs + dataset_targets + sample_weights + [1]
-    else:
-      ins = dataset_inputs + dataset_targets
-
-    do_validation = False
-    if validation_steps:
-      do_validation = True
-
-    # Copy the weights from the original model to each of the replicated models.
-    orig_model_weights = model.get_weights()
-    distributed_model = current_strategy.unwrap(model._grouped_model)[0]
-    distributed_training_utils.set_weights(
-        current_strategy, distributed_model, orig_model_weights)
-
-    callbacks = cbks.configure_callbacks(
-        callbacks,
-        model,
-        do_validation=do_validation,
-        val_inputs=None,
-        val_targets=None,
-        epochs=epochs,
-        steps_per_epoch=steps_per_epoch,
-        verbose=verbose)
-    out_labels = model.metrics_names or []
-    callbacks.on_train_begin()
-
-    assert steps_per_epoch is not None
-
-    for epoch in range(initial_epoch, epochs):
-      # Reset stateful metrics
-      for m in model.metrics:
-        m.reset_states()
-      callbacks.on_epoch_begin(epoch)
-      epoch_logs = {}
-      for step_index in range(steps_per_epoch):
-        batch_logs = {'batch': step_index, 'size': 1}
-        callbacks.on_batch_begin(step_index, batch_logs)
-        try:
-          outs = distributed_fit_function(ins)
-        except errors.OutOfRangeError:
-          logging.warning('Your dataset iterator ran out of data; '
-                          'interrupting training. Make sure that your dataset '
-                          'can generate at least `steps_per_epoch * epochs` '
-                          'batches (in this case, %d batches).' %
-                          steps_per_epoch * epochs)
-          break
-
-        if not isinstance(outs, list):
-          outs = [outs]
-        for l, o in zip(out_labels, outs):
-          batch_logs[l] = o
-        callbacks.on_batch_end(step_index, batch_logs)
-        if callbacks.model.stop_training:
-          break
-      if do_validation:
-        val_outs = test_loop(
-            model,
-            val_iterator,
-            steps=validation_steps,
-            verbose=0)
-        if not isinstance(val_outs, list):
-          val_outs = [val_outs]
-        # Same labels assumed.
-        for l, o in zip(out_labels, val_outs):
-          epoch_logs['val_' + l] = o
-
-      callbacks.on_epoch_end(epoch, epoch_logs)
-      if callbacks.model.stop_training:
-        break
-    callbacks.on_train_end()
-
-    # Copy the weights back from the replicated model to the original model.
-    updated_weights = current_strategy.unwrap(
-        model._grouped_model)[0].get_weights()
-    model.set_weights(updated_weights)
-    return model.history
-
-
-def _experimental_fit_loop(
-    model,
-    iterator,
-    epochs=100,
-    verbose=1,
-    callbacks=None,
-    initial_epoch=0,
-    steps_per_epoch=None,
-    val_iterator=None,
-    validation_steps=None):
+def experimental_fit_loop(model,
+                          iterator,
+                          epochs=100,
+                          verbose=1,
+                          callbacks=None,
+                          initial_epoch=0,
+                          steps_per_epoch=None,
+                          val_iterator=None,
+                          validation_steps=None):
   """Fit loop for training with TPU DistributionStrategy.
 
   Arguments:
@@ -394,7 +222,7 @@ def _experimental_fit_loop(
             model._grouped_model_train)[0].get_weights()
         model.set_weights(updated_weights)
 
-      val_outs = _experimental_test_loop(
+      val_outs = experimental_test_loop(  # pylint: disable=undefined-variable
           model,
           val_iterator,
           steps=validation_steps,
@@ -421,102 +249,11 @@ def _experimental_fit_loop(
   return model.history
 
 
-def test_loop(model, iterator, verbose=0, steps=None):
-  """Test loop for evaluating with DistributionStrategy.
-
-  Arguments:
-      model: Keras Model instance.
-      iterator: Iterator for input data.
-      verbose: Integer, Verbosity mode 0 or 1.
-      steps: Total number of steps (batches of samples)
-          before declaring predictions finished.
-          Ignored with the default value of `None`.
-
-  Returns:
-      Scalar loss (if the model has a single output and no metrics)
-      or list of scalars (if the model has multiple outputs
-      and/or metrics). The attribute `model.metrics_names` will give you
-      the display labels for the outputs.
-  """
-  current_strategy = model._distribution_strategy
-
-  # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged.
-  if current_strategy.__class__.__name__ == 'TPUStrategy':
-    return _experimental_test_loop(model, iterator, verbose, steps)
-
-  if not model._grouped_model:
-    clone_model_on_replicas(model, current_strategy)
-
-  def _per_device_eval_function(model):
-    model._make_eval_function()
-    return (model._eval_function.inputs, model._eval_function.outputs,
-            model._eval_function.updates_op,
-            model._eval_function.session_kwargs)
-
-  inputs, targets, sample_weights = _get_input_from_iterator(iterator, model)
-  with current_strategy.scope():
-    (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = current_strategy.call_for_each_replica(
-         _per_device_eval_function, args=(model._grouped_model,))
-
-    (all_inputs, all_outputs, all_updates,
-     all_session_args) = distributed_training_utils.unwrap_values(
-         current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
-         grouped_session_args, with_loss_tensor=True)
-
-    dataset_inputs = distributed_training_utils.flatten_perdevice_values(
-        current_strategy, inputs)
-    dataset_targets = distributed_training_utils.flatten_perdevice_values(
-        current_strategy, targets)
-
-    distributed_test_function = K.function(
-        all_inputs, all_outputs,
-        updates=all_updates,
-        name='distributed_test_function',
-        **all_session_args)
-
-    # We need to set sample_weights to None since there are sample weight
-    # placeholders that are created with default values.
-    sample_weights = [None for _ in range(
-        len(model.outputs) * current_strategy.num_replicas_in_sync)]
-    ins = dataset_inputs + dataset_targets + sample_weights
-
-    for m in model.metrics:
-      m.reset_states()
-
-    outs = []
-    if verbose == 1:
-      progbar = Progbar(target=steps)
-
-    # Copy the weights from the original model to each of the replicated models.
-    orig_model_weights = model.get_weights()
-    distributed_model = current_strategy.unwrap(model._grouped_model)[0]
-    distributed_training_utils.set_weights(
-        current_strategy, distributed_model, orig_model_weights)
-
-    assert steps is not None
-    for step in range(steps):
-      batch_outs = distributed_test_function(ins)
-      if isinstance(batch_outs, list):
-        if step == 0:
-          outs = [0.] * len(batch_outs)
-        outs[0] += batch_outs[0]  # index 0 = 'loss'
-        outs[1:] = batch_outs[1:]
-      else:
-        if step == 0:
-          outs.append(0.)
-        outs[0] += batch_outs  # index 0 = 'loss'
-      if verbose >= 1:
-        progbar.update(step + 1)
-    outs[0] /= steps  # index 0 = 'loss'
-
-    if len(outs) == 1:
-      return outs[0]
-    return outs
-
-
-def _experimental_test_loop(model, iterator, verbose=0, steps=None,
-                            initialize_finalize_strategy=True):
+def experimental_test_loop(model,
+                           iterator,
+                           verbose=0,
+                           steps=None,
+                           initialize_finalize_strategy=True):
   """Test loop for evaluating with TPU DistributionStrategy.
 
   Arguments:
@@ -634,98 +371,7 @@ def _experimental_test_loop(model, iterator, verbose=0, steps=None,
   return outs
 
 
-def predict_loop(model, iterator, verbose=0, steps=None):
-  """Predict loop for predicting with DistributionStrategy.
-
-  Arguments:
-      model: Keras Model instance.
-      iterator: Iterator for input data.
-      verbose: Integer, Verbosity mode 0 or 1.
-      steps: Total number of steps (batches of samples)
-          before declaring `_predict_loop` finished.
-          Ignored with the default value of `None`.
-
-  Returns:
-      Array of predictions (if the model has a single output)
-      or list of arrays of predictions
-      (if the model has multiple outputs).
-  """
-  current_strategy = model._distribution_strategy
-
-  # TODO(priyag, sourabhbajaj): Remove this when the codepaths are merged.
-  if current_strategy.__class__.__name__ == 'TPUStrategy':
-    return _experimental_predict_loop(model, iterator, verbose, steps)
-
-  if not model._grouped_model:
-    clone_model_on_replicas(model, current_strategy)
-
-  def _per_device_predict_function(model):
-    model._make_predict_function()
-    return (model.predict_function.inputs,
-            model.predict_function.outputs,
-            model.predict_function.updates_op,
-            model.predict_function.session_kwargs)
-
-  inputs, _, _ = _get_input_from_iterator(iterator, model)
-  with current_strategy.scope():
-    (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = current_strategy.call_for_each_replica(
-         _per_device_predict_function, args=(model._grouped_model,))
-
-    (all_inputs, all_outputs, all_updates,
-     all_session_args) = distributed_training_utils.unwrap_values(
-         current_strategy, grouped_inputs, grouped_outputs, grouped_updates,
-         grouped_session_args)
-
-    dataset_inputs = distributed_training_utils.flatten_perdevice_values(
-        current_strategy, inputs)
-
-    distributed_predict_function = K.function(
-        all_inputs, all_outputs,
-        updates=all_updates,
-        name='distributed_predict_function',
-        **all_session_args)
-
-    if verbose == 1:
-      progbar = Progbar(target=steps)
-
-    # Copy the weights from the original model to each of the replicated models.
-    orig_model_weights = model.get_weights()
-    distributed_model = current_strategy.unwrap(model._grouped_model)[0]
-    distributed_training_utils.set_weights(
-        current_strategy, distributed_model, orig_model_weights)
-
-    num_replicas = current_strategy.num_replicas_in_sync
-    # Since we do not know how many samples we will see, we cannot
-    # pre-allocate the returned Numpy arrays. Instead, we store one array per
-    # batch seen and concatenate them upon returning.
-    unconcatenated_outs = []
-    assert steps is not None
-    for step in range(steps):
-      batch_outs = distributed_predict_function(dataset_inputs)
-      if not isinstance(batch_outs, list):
-        batch_outs = [batch_outs]
-      if step == 0:
-        # batch_outs gives you the number of model outputs. In the distributed
-        # case this will be number of model_outputs * num_replicas.
-        for _ in range(len(model.outputs)):
-          unconcatenated_outs.append([])
-      for i in range(len(model.outputs)):
-        nested_outs = batch_outs[i * num_replicas:
-                                 i * num_replicas + num_replicas]
-        outs = nest.flatten(nested_outs)
-        unconcatenated_outs[i].extend(outs)
-      if verbose >= 1:
-        progbar.update(step + 1)
-    if len(unconcatenated_outs) == 1:
-      return np.concatenate(unconcatenated_outs[0], axis=0)
-    return [
-        np.concatenate(unconcatenated_outs[i], axis=0)
-        for i in range(len(unconcatenated_outs))
-    ]
-
-
-def _experimental_predict_loop(model, iterator, verbose=0, steps=None):
+def experimental_predict_loop(model, iterator, verbose=0, steps=None):
   """Predict loop for predicting with TPU DistributionStrategy.
 
   Arguments:
@@ -920,3 +566,117 @@ def _get_input_from_iterator(iterator, model):
   model._standardize_weights(x_values, y_values,
                              sample_weight=sample_weights_values)
   return x, y, sample_weights
+
+
+def _get_execution_function(model, mode):
+  """Get function to run one step of distributed model execution."""
+  strategy = model._distribution_strategy
+  if not model._grouped_model:
+    clone_model_on_replicas(
+        model, strategy, make_callback_model=(mode == 'train'))
+
+  def _per_device_function(model):
+    f = model._get_execution_function(mode)
+    return (f.inputs, f.outputs, f.updates_op, f.session_kwargs)
+
+  with strategy.scope():
+    # Create train ops on each of the devices when we call
+    # `_per_device_fit_function`.
+    (grouped_inputs, grouped_outputs, grouped_updates,
+     grouped_session_args) = strategy.call_for_each_replica(
+         _per_device_function, args=(model._grouped_model,))
+
+    if mode == 'train':
+      # Initialize the variables in the replicated model. This is necessary for
+      # multi-worker training because on some workers, initialization is not
+      # needed. This method does initialization or waiting for initialization
+      # according to the context object of distribute coordinator.
+      distributed_training_utils.init_restore_or_wait_for_variables()
+
+    # Unwrap all the per device values returned from `call_for_each_replica`.
+    # Unwrapping per device values gives you a list of values that can be
+    # used to construct a new train function that is composed of update ops on
+    # all the devices over which the model is distributed.
+    (all_inputs, all_outputs, all_updates,
+     all_session_args) = distributed_training_utils.unwrap_values(
+         strategy,
+         grouped_inputs,
+         grouped_outputs,
+         grouped_updates,
+         grouped_session_args,
+         with_loss_tensor=(mode != 'predict'))
+
+    return K.function(
+        all_inputs,
+        all_outputs,
+        updates=all_updates,
+        name='distributed_{}_function'.format(mode),
+        **all_session_args)
+
+
+def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
+  """Prepare feed values to the model execution function.
+
+  Arguments:
+    model: Model to prepare feed values for.
+    inputs: List or dict of model inputs.
+    targets: Optional list of model targets.
+    sample_weights: Optional list of sample weight arrays.
+    mode: One of 'train'/'test'/'predict'.
+
+  Returns:
+    Feed values for the model in the given mode.
+  """
+  strategy = model._distribution_strategy
+  inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)
+  inputs = distributed_training_utils.flatten_perdevice_values(strategy, inputs)
+  targets = distributed_training_utils.flatten_perdevice_values(
+      strategy, targets)
+  if mode == 'predict':
+    sample_weights = []
+    targets = []
+  else:
+    sample_weights = [
+        None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
+    ]
+  ins = inputs + targets + sample_weights
+  if mode == 'train' and not isinstance(K.learning_phase(), int):
+    ins += [True]
+  return ins
+
+
+def _copy_weights_to_distributed_model(model):
+  """Copies weights from original model to distributed models."""
+  if model._distribution_strategy:
+    # Copy the weights from the original model to each of the replicated models.
+    orig_model_weights = model.get_weights()
+    distributed_model = model._distribution_strategy.unwrap(
+        model._grouped_model)[0]
+    distributed_training_utils.set_weights(
+        model._distribution_strategy, distributed_model, orig_model_weights)
+
+
+def _copy_weights_to_original_model(model, mode):
+  """Copies weights from first distributed model back to original model."""
+  if model._distribution_strategy and mode == 'train':
+    updated_weights = model._distribution_strategy.unwrap(
+        model._grouped_model)[0].get_weights()
+    model.set_weights(updated_weights)
+
+
+def _per_device_aggregate_batch(batch_outs, model, mode):
+  """Aggregates the per-device batch-level outputs from a distributed step."""
+  if model._distribution_strategy is not None and mode == 'predict':
+    total_batch_outs = []
+    for i in range(len(model.outputs)):
+      num_replicas = model._distribution_strategy.num_replicas_in_sync
+      nested_outs = batch_outs[i * num_replicas:i * num_replicas + num_replicas]
+      total_batch_outs.append(np.concatenate(nest.flatten(nested_outs)))
+    return total_batch_outs
+  return batch_outs
+
+
+def should_run_experimental_loop(model):
+  """Whether to run the experimental loops in this file."""
+  return (hasattr(model, '_distribution_strategy') and
+          model._distribution_strategy.__class__.__name__ == 'TPUStrategy')
-- 
GitLab


From 1ca6373defadd0203df8c1bab82e2ea92e30683b Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 16 Nov 2018 17:59:18 -0800
Subject: [PATCH 0463/1554] Disable failing tests thaat use autograph on
 windows.

PiperOrigin-RevId: 221880994
---
 tensorflow/python/eager/BUILD              | 6 ++++++
 tensorflow/python/keras/optimizer_v2/BUILD | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index db5c6b2eb7..55728b19ab 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -189,6 +189,9 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
     ],
     shard_count = 5,
+    tags = [
+        "no_windows",
+    ],
 )
 
 cuda_py_test(
@@ -211,6 +214,9 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
     ],
     shard_count = 15,
+    tags = [
+        "no_windows",
+    ],
 )
 
 py_library(
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 4913eaf9fc..2296a9562b 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -134,6 +134,9 @@ py_test(
     name = "optimizer_v2_test",
     size = "medium",
     srcs = ["optimizer_v2_test.py"],
+    tags = [
+        "no_windows",
+    ],
     deps = [
         ":optimizer_v2",
         "//tensorflow/python:array_ops",
-- 
GitLab


From 8aeb9686e85ca7e4a8586841ce0de41acfe8a769 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 18:23:03 -0800
Subject: [PATCH 0464/1554] Automated rollback of commit
 2121365e745361889ec9b4862f67b7f0b92e9607

PiperOrigin-RevId: 221883009
---
 .../xla/service/buffer_assignment_test.cc     | 18 ++--
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  8 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    |  2 +-
 .../compiler/xla/service/cpu/ir_emitter.h     |  2 +-
 .../xla/service/gpu/gpu_hlo_schedule.cc       | 18 ++--
 .../xla/service/gpu/gpu_hlo_schedule.h        |  4 +-
 .../xla/service/gpu/gpu_hlo_schedule_test.cc  |  2 +-
 .../xla/service/gpu/thunk_schedule.cc         |  4 +-
 .../compiler/xla/service/gpu/thunk_schedule.h |  2 +-
 .../xla/service/heap_simulator_test.cc        |  6 +-
 .../compiler/xla/service/hlo_computation.cc   |  6 +-
 .../compiler/xla/service/hlo_computation.h    |  2 +-
 .../xla/service/hlo_memory_scheduler.cc       | 82 +++++++++----------
 .../xla/service/hlo_memory_scheduler.h        | 14 ++--
 .../xla/service/hlo_memory_scheduler_test.cc  | 33 ++++----
 tensorflow/compiler/xla/service/hlo_module.h  |  6 +-
 .../compiler/xla/service/hlo_ordering.cc      |  3 +-
 tensorflow/compiler/xla/service/hlo_parser.cc |  6 +-
 .../xla/service/hlo_rematerialization.cc      | 19 +++--
 .../xla/service/hlo_rematerialization.h       |  5 +-
 .../compiler/xla/service/hlo_schedule.cc      | 25 +++---
 .../compiler/xla/service/hlo_schedule.h       | 12 +--
 .../compiler/xla/service/hlo_schedule_test.cc | 14 ++--
 23 files changed, 151 insertions(+), 142 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index 8f482e6ba8..b1fc50cb18 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -137,7 +137,8 @@ class BufferAssignmentTest : public HloTestBase {
   }
 
   std::unique_ptr<BufferAssignment> RunBufferAssignmentWithInstructionSequence(
-      HloModule* module, absl::Span<HloInstruction* const> instruction_sequence,
+      HloModule* module,
+      absl::Span<const HloInstruction* const> instruction_sequence,
       int64 alignment = 1) {
     HloSchedule schedule(module);
     schedule.set_sequence(module->entry_computation(), instruction_sequence);
@@ -1852,7 +1853,7 @@ class WhileBufferAssignmentTest : public HloTestBase {
   std::unique_ptr<BufferAssignment> RunBufferAssignment(HloModule* module,
                                                         int64 alignment = 1) {
     HloSchedule schedule =
-        ScheduleModule(module, ByteSizeOf).ConsumeValueOrDie();
+        ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie();
     return BufferAssigner::Run(
                module, absl::make_unique<SequentialHloOrdering>(schedule),
                ByteSizeOf,
@@ -2161,7 +2162,7 @@ TEST_F(WhileBufferAssignmentTest, ColocatedBuffers) {
   // nodes are traversed during BufferAssignment.
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
@@ -2390,16 +2391,15 @@ TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) {
   RunCopyInsertion(module.get());
 
   HloSchedule schedule =
-      ScheduleModule(module.get(), ByteSizeOf).ConsumeValueOrDie();
+      ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie();
 
   // To trigger b/38494731, we want a specific Hlo schedule for the
   // root computation, so we overwrite that entry with a manually
   // crafted sequence.
-  schedule.set_sequence(
-      module->entry_computation(),
-      {input1, weights1, one, output1, while1->mutable_operand(0), while1,
-       input0, weights0, zero, output0, while0->mutable_operand(0), while0,
-       gte0, gte1, root_add});
+  schedule.set_sequence(module->entry_computation(),
+                        {input1, weights1, one, output1, while1->operand(0),
+                         while1, input0, weights0, zero, output0,
+                         while0->operand(0), while0, gte0, gte1, root_add});
 
   // If this ASSERT fails, we constructed a bogus sequence above and this test
   // itself is buggy.
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index b529a6e8da..4ce5a8a292 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -587,9 +587,9 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::RunBackend(
   // Select an order for emitting the HLO instructions for each
   // computation. Using this sequence enables tighter buffer liveness analysis
   // and reduced memory usage (as compared to using DependencyHloOrdering).
-  TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                      ScheduleModule(module.get(), BufferSizeBytesFunction(),
-                                     DFSMemoryScheduler));
+  TF_ASSIGN_OR_RETURN(
+      HloSchedule schedule,
+      ScheduleModule(*module, BufferSizeBytesFunction(), DFSMemoryScheduler));
 
   // Run buffer allocation on the HLO graph.
   TF_ASSIGN_OR_RETURN(
@@ -779,7 +779,7 @@ CpuCompiler::CompileAheadOfTime(std::unique_ptr<HloModuleGroup> module_group,
     XLA_VLOG_LINES(2, module->ToString());
 
     TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                        ScheduleModule(module, BufferSizeBytesFunction()));
+                        ScheduleModule(*module, BufferSizeBytesFunction()));
 
     // Run buffer analysis on the HLO graph. This analysis figures out which
     // temporary buffers are required to run the computation.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 9b731b0a6a..620c45fa39 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -111,7 +111,7 @@ IrEmitter::IrEmitter(
 StatusOr<llvm::Function*> IrEmitter::EmitComputation(
     HloComputation* computation, const string& function_name_prefix,
     bool is_top_level_computation,
-    const std::vector<HloInstruction*>* instruction_order) {
+    const std::vector<const HloInstruction*>* instruction_order) {
   string function_name = name_uniquer_.GetUniqueName(function_name_prefix);
   VLOG(2) << "Emitting IR for CPU function [" << function_name_prefix
           << "]; ordered? " << (instruction_order != nullptr);
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index e03ba42b0d..136b88ff75 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -101,7 +101,7 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   StatusOr<llvm::Function*> EmitComputation(
       HloComputation* computation, const string& function_name_prefix,
       bool is_top_level_computation,
-      const std::vector<HloInstruction*>* instruction_order);
+      const std::vector<const HloInstruction*>* instruction_order);
 
   llvm::IRBuilder<>* b() { return &b_; }
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
index 1126943624..91609c730b 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
@@ -37,7 +37,7 @@ class GpuHloOrdering : public PredecessorHloOrdering {
  public:
   GpuHloOrdering(const HloModule* module,
                  const StreamAssignment& stream_assignment,
-                 const std::vector<HloInstruction*>& thunk_launch_order);
+                 const std::vector<const HloInstruction*>& thunk_launch_order);
   ~GpuHloOrdering() override = default;
 
   // Only the entry computation can possibly be sequentially ordered, and only
@@ -56,7 +56,7 @@ class GpuHloOrdering : public PredecessorHloOrdering {
 
 GpuHloOrdering::GpuHloOrdering(
     const HloModule* module, const StreamAssignment& stream_assignment,
-    const std::vector<HloInstruction*>& thunk_launch_order)
+    const std::vector<const HloInstruction*>& thunk_launch_order)
     : PredecessorHloOrdering(module) {
   // The entry computation has a total order when there's only one stream.
   if (stream_assignment.StreamCount() == 1) {
@@ -150,7 +150,7 @@ GpuHloOrdering::GpuHloOrdering(
 // However, if the total order is A,B,D,C,E, then C and E can run
 // concurrently.
 void BFSLaunchOrder(const HloComputation* computation,
-                    std::vector<HloInstruction*>* launch_order) {
+                    std::vector<const HloInstruction*>* launch_order) {
   // This topological sort uses two data structures:
   // 1. `incoming_edge_count` which keeps track of the number of incoming
   // edges to each HLO;
@@ -158,9 +158,9 @@ void BFSLaunchOrder(const HloComputation* computation,
   //
   // The sorting algorithm repeatedly pops the top from the queue and deletes
   // that HLO from the graph, making more HLOs incoming-edge free.
-  std::deque<HloInstruction*> queue;
+  std::deque<const HloInstruction*> queue;
   std::unordered_map<const HloInstruction*, int64> incoming_edge_count;
-  for (auto* hlo : computation->instructions()) {
+  for (const auto& hlo : computation->instructions()) {
     if (hlo->operand_count() == 0) {
       queue.push_back(hlo);
     } else {
@@ -172,10 +172,10 @@ void BFSLaunchOrder(const HloComputation* computation,
   }
 
   while (!queue.empty()) {
-    HloInstruction* x = queue.front();
+    const HloInstruction* x = queue.front();
     queue.pop_front();
     launch_order->push_back(x);
-    for (HloInstruction* y : x->users()) {
+    for (const HloInstruction* y : x->users()) {
       --incoming_edge_count[y];
       if (incoming_edge_count[y] == 0) {
         queue.push_back(y);
@@ -195,14 +195,14 @@ StatusOr<std::unique_ptr<GpuHloSchedule>> GpuHloSchedule::Build(
   std::unique_ptr<GpuHloSchedule> schedule(new GpuHloSchedule);
 
   // Initialize thunk_launch_order_, the total order of thunk launches.
-  HloComputation* entry_computation = module.entry_computation();
+  const HloComputation* entry_computation = module.entry_computation();
   if (stream_assignment.StreamCount() == 1) {
     // All kernels are launched on a single stream, so there's no loss of
     // concurrency by optimizing for minimal memory usage.
     TF_ASSIGN_OR_RETURN(
         HloInstructionSequence sequence,
         ScheduleComputation(
-            entry_computation, [pointer_size](const BufferValue& buffer) {
+            *entry_computation, [pointer_size](const BufferValue& buffer) {
               return ShapeUtil::ByteSizeOf(buffer.shape(), pointer_size);
             }));
     schedule->thunk_launch_order_ = sequence.instructions();
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
index 7f224ffe4f..07a7fc67aa 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
@@ -46,7 +46,7 @@ class GpuHloSchedule {
 
   // Returns the total order of thunk launches, represented in terms of HLO
   // instructions.
-  const std::vector<HloInstruction*>& ThunkLaunchOrder() const {
+  const std::vector<const HloInstruction*>& ThunkLaunchOrder() const {
     return thunk_launch_order_;
   }
 
@@ -60,7 +60,7 @@ class GpuHloSchedule {
  private:
   GpuHloSchedule();
 
-  std::vector<HloInstruction*> thunk_launch_order_;
+  std::vector<const HloInstruction*> thunk_launch_order_;
   std::unique_ptr<HloOrdering> hlo_ordering_;
 };
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
index 91db7151f2..5f857a1a54 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
@@ -33,7 +33,7 @@ namespace gpu {
 
 class GpuHloScheduleTest : public HloTestBase {
  protected:
-  using HloVec = std::vector<HloInstruction*>;
+  using HloVec = std::vector<const HloInstruction*>;
 
   // Pre-canned shapes.
   Shape f32_2x2_ = ShapeUtil::MakeShape(F32, {2, 2});
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
index 6b2d76764a..141f321938 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
@@ -45,7 +45,7 @@ void ThunkSchedule::AddDependenciesOnTransitiveOperands(
 ThunkSchedule::ThunkSchedule(
     std::unique_ptr<ThunkSequence> thunks,
     std::unique_ptr<StreamAssignment> stream_assignment,
-    const std::vector<HloInstruction*>& hlo_total_order)
+    const std::vector<const HloInstruction*>& hlo_total_order)
     : thunks_(std::move(thunks)),
       stream_assignment_(std::move(stream_assignment)) {
   std::unordered_map<const HloInstruction*, Thunk*> hlo_to_thunk;
@@ -53,7 +53,7 @@ ThunkSchedule::ThunkSchedule(
     InsertOrDie(&hlo_to_thunk, thunk->hlo_instruction(), thunk.get());
   }
 
-  for (HloInstruction* hlo : hlo_total_order) {
+  for (const HloInstruction* hlo : hlo_total_order) {
     if (hlo_to_thunk.count(hlo)) {
       thunk_total_order_.push_back(FindOrDie(hlo_to_thunk, hlo));
     }
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
index 43b628a1ba..d3352994f8 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
@@ -46,7 +46,7 @@ class ThunkSchedule {
  public:
   ThunkSchedule(std::unique_ptr<ThunkSequence> thunks,
                 std::unique_ptr<StreamAssignment> stream_assignment,
-                const std::vector<HloInstruction*>& hlo_total_order);
+                const std::vector<const HloInstruction*>& hlo_total_order);
 
   // Returns the total order of executing all the thunks.
   const std::vector<Thunk*>& TotalOrder() const { return thunk_total_order_; }
diff --git a/tensorflow/compiler/xla/service/heap_simulator_test.cc b/tensorflow/compiler/xla/service/heap_simulator_test.cc
index dc40b9446a..fad3215fc8 100644
--- a/tensorflow/compiler/xla/service/heap_simulator_test.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator_test.cc
@@ -258,7 +258,7 @@ class HeapSimulatorTracker {
   // Constructor for testing a single entry computation.
   HeapSimulatorTracker(
       const string& name, std::unique_ptr<HloComputation> computation,
-      const std::vector<HloInstruction*>& instruction_sequence) {
+      const std::vector<const HloInstruction*>& instruction_sequence) {
     HloModuleConfig config;
     module_ = absl::make_unique<HloModule>(name, config);
     module_->AddEntryComputation(std::move(computation));
@@ -286,7 +286,7 @@ class HeapSimulatorTracker {
   // Similar to the single entry computation constructor above, but runs the
   // simulation over the entire module.
   void RunWholeModule(
-      const std::vector<HloInstruction*>& full_module_sequence) {
+      const std::vector<const HloInstruction*>& full_module_sequence) {
     points_to_analysis_ =
         TuplePointsToAnalysis::Run(module_.get()).ConsumeValueOrDie();
 
@@ -294,7 +294,7 @@ class HeapSimulatorTracker {
     HloSchedule schedule(module_.get());
     absl::flat_hash_map<const HloInstruction*, int> reverse_position;
     for (int i = 0; i < full_module_sequence.size(); ++i) {
-      HloInstruction* instruction = full_module_sequence[i];
+      const HloInstruction* instruction = full_module_sequence[i];
       schedule.GetOrCreateSequence(instruction->parent())
           .push_back(instruction);
       reverse_position[instruction] = full_module_sequence.size() - i;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 65bd251dd8..0c20d207dd 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -795,7 +795,7 @@ Status HloComputation::AcceptWithOperandOrder(
 template <typename HloInstructionPtr>
 Status HloComputation::AcceptOrdered(
     DfsHloVisitorBase<HloInstructionPtr>* visitor,
-    const std::vector<HloInstruction*>& order) const {
+    const std::vector<const HloInstruction*>& order) const {
   VLOG(3) << "Accepting visitor with order.";
   for (HloInstruction* root : CollectUnreachableRoots()) {
     TF_RET_CHECK(std::find(order.begin(), order.end(), root) != order.end())
@@ -825,9 +825,9 @@ Status HloComputation::AcceptOrdered(
 
 // Explicit instantiations.
 template Status HloComputation::AcceptOrdered(
-    DfsHloVisitor*, const std::vector<HloInstruction*>&) const;
+    DfsHloVisitor*, const std::vector<const HloInstruction*>&) const;
 template Status HloComputation::AcceptOrdered(
-    ConstDfsHloVisitor*, const std::vector<HloInstruction*>&) const;
+    ConstDfsHloVisitor*, const std::vector<const HloInstruction*>&) const;
 
 Status HloComputation::Accept(
     const std::function<Status(HloInstruction*)>& visitor_func) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index be1ce33696..fc7d2035e5 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -301,7 +301,7 @@ class HloComputation {
   // be a topological sort of all instructions in the computation.
   template <typename HloInstructionPtr>
   Status AcceptOrdered(DfsHloVisitorBase<HloInstructionPtr>* visitor,
-                       const std::vector<HloInstruction*>& order) const;
+                       const std::vector<const HloInstruction*>& order) const;
 
   // Same as Accept() above, but the visitor is given as a function.
   Status Accept(const std::function<Status(HloInstruction*)>& visitor_func);
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
index d2740bcce2..234fcd266a 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
@@ -73,7 +73,7 @@ class ListScheduler {
   // Construct and return a memory-minimizing sequence of HLO instructions
   // containing the given HLO computation.
   static StatusOr<HloInstructionSequence> Run(
-      HloComputation* computation,
+      const HloComputation& computation,
       const TuplePointsToAnalysis& points_to_analysis,
       const LogicalBuffer::SizeFunction& size_function,
       const absl::flat_hash_map<const HloComputation*, int64>&
@@ -98,7 +98,7 @@ class ListScheduler {
   // comparison operators.
   using Priority = std::pair<int64, int64>;
 
-  ListScheduler(HloComputation* computation,
+  ListScheduler(const HloComputation& computation,
                 const TuplePointsToAnalysis& points_to_analysis,
                 const LogicalBuffer::SizeFunction& size_function,
                 const absl::flat_hash_map<const HloComputation*, int64>&
@@ -111,7 +111,7 @@ class ListScheduler {
     // instruction. An HLO instruction "uses" a LogicalBuffer if the
     // LogicalBuffer is in an operand of the instruction as indicated by
     // points-to analysis.
-    for (auto* instruction : computation->instructions()) {
+    for (auto* instruction : computation.instructions()) {
       absl::flat_hash_set<const LogicalBuffer*> instr_uses;
       for (auto* operand : instruction->operands()) {
         points_to_analysis.GetPointsToSet(operand).ForEachElement(
@@ -126,13 +126,13 @@ class ListScheduler {
 
     // Create map containing the number of unscheduled uses (hlo instructions)
     // of each logical buffer.
-    for (auto* instruction : computation->instructions()) {
+    for (auto* instruction : computation.instructions()) {
       for (auto* buffer :
            points_to_analysis.GetBuffersDefinedByInstruction(instruction)) {
         unscheduled_use_count_[buffer] = 0;
       }
     }
-    for (auto* instruction : computation->instructions()) {
+    for (auto* instruction : computation.instructions()) {
       for (const LogicalBuffer* buffer : buffer_uses_.at(instruction)) {
         ++unscheduled_use_count_[buffer];
       }
@@ -141,7 +141,7 @@ class ListScheduler {
     // Buffers live out of the computation have an implicit use at the end of
     // the computation.
     for (const LogicalBuffer* live_out_buffer :
-         points_to_analysis.GetPointsToSet(computation->root_instruction())
+         points_to_analysis.GetPointsToSet(computation.root_instruction())
              .CreateFlattenedSet()) {
       ++unscheduled_use_count_[live_out_buffer];
     }
@@ -157,7 +157,7 @@ class ListScheduler {
   // HloInstruction, plus some cached metadata, saved for the purposes of making
   // BytesFreedIfScheduled fast.
   struct ReadyListEntry {
-    HloInstruction* instruction;
+    const HloInstruction* instruction;
 
     // The total size of all buffers defined by this instruction.
     int64 bytes_defined;
@@ -171,7 +171,7 @@ class ListScheduler {
   };
 
   // Creates a ReadyListEntry for the given instruction.
-  ReadyListEntry MakeReadyListEntry(HloInstruction* instruction) {
+  ReadyListEntry MakeReadyListEntry(const HloInstruction* instruction) {
     ReadyListEntry entry;
     entry.instruction = instruction;
 
@@ -250,13 +250,13 @@ class ListScheduler {
     // Populate the ready list with instructions which have no operands or
     // control predecessors.
     absl::flat_hash_map<const HloInstruction*, int64> unscheduled_pred_count;
-    for (auto* instruction : computation_->instructions()) {
+    for (auto* instruction : computation_.instructions()) {
       // TODO(b/34466113): Replace this and above with successors() or
       // predecessors() when these methods are added to HloInstruction.
-      for (HloInstruction* user : instruction->users()) {
+      for (const HloInstruction* user : instruction->users()) {
         unscheduled_pred_count[user]++;
       }
-      for (HloInstruction* succ : instruction->control_successors()) {
+      for (const HloInstruction* succ : instruction->control_successors()) {
         unscheduled_pred_count[succ]++;
       }
     }
@@ -275,7 +275,7 @@ class ListScheduler {
       ready_instructions[inst] = it;
     };
 
-    for (auto* instruction : computation_->instructions()) {
+    for (auto* instruction : computation_.instructions()) {
       if (instruction->operands().empty() &&
           instruction->control_predecessors().empty()) {
         add_to_ready_queue(instruction);
@@ -287,7 +287,7 @@ class ListScheduler {
       // schedule.
       auto best_it = ready_queue.end();
       --best_it;
-      HloInstruction* best = best_it->second.instruction;
+      const HloInstruction* best = best_it->second.instruction;
       VLOG(2) << "Schedule instruction: " << best->ToShortString()
               << " Bytes freed: " << best_it->first.first;
       ready_queue.erase(best_it);
@@ -348,13 +348,13 @@ class ListScheduler {
         }
       }
     }
-    CHECK_EQ(schedule.size(), computation_->instruction_count());
-    CHECK_EQ(scheduled_instructions_.size(), computation_->instruction_count());
+    CHECK_EQ(schedule.size(), computation_.instruction_count());
+    CHECK_EQ(scheduled_instructions_.size(), computation_.instruction_count());
 
     return schedule;
   }
 
-  HloComputation* computation_;
+  const HloComputation& computation_;
   const TuplePointsToAnalysis& points_to_analysis_;
   const LogicalBuffer::SizeFunction& size_function_;
   // Computations are analyzed in post-order. When scheduling an instruction
@@ -386,13 +386,13 @@ int64 SumLogicalBufferSizes(
 }
 
 StatusOr<HloInstructionSequence> ScheduleComputationHelper(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
-  VLOG(2) << "Computation: " << computation->name();
+  VLOG(2) << "Computation: " << computation.name();
   if (algorithm) {
     return algorithm(computation, points_to_analysis, size_function,
                      memory_by_computation);
@@ -404,17 +404,17 @@ StatusOr<HloInstructionSequence> ScheduleComputationHelper(
 }  // namespace
 
 StatusOr<HloInstructionSequence> DFSMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
   // These variables are a hack to prevent overflows.
   int64 cumulative_total_size = 0;
-  int64 total_hlos = computation->parent()->instruction_count();
+  int64 total_hlos = computation.parent()->instruction_count();
   absl::flat_hash_map<const HloInstruction*, int64> extra_users;
   absl::flat_hash_map<const HloInstruction*, int64> total_sizes;
-  for (const HloInstruction* hlo : computation->MakeInstructionPostOrder()) {
+  for (const HloInstruction* hlo : computation.MakeInstructionPostOrder()) {
     if (ListScheduler::IgnoreInstruction(*hlo)) {
       extra_users[hlo] = 0;
       total_sizes[hlo] = 0;
@@ -448,8 +448,8 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
     total_sizes[hlo] = std::min(total_sizes[hlo], cumulative_total_size);
     extra_users[hlo] = std::min(extra_users[hlo], total_hlos);
   }
-  CHECK_EQ(extra_users.size(), computation->instruction_count());
-  CHECK_EQ(total_sizes.size(), computation->instruction_count());
+  CHECK_EQ(extra_users.size(), computation.instruction_count());
+  CHECK_EQ(total_sizes.size(), computation.instruction_count());
 
   // Construct a total order based on DFS post-order, visiting operands in
   // decreasing cumulative extra user order, and next by cumulative size, with a
@@ -459,7 +459,7 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
     sequence.push_back(hlo);
     return Status::OK();
   });
-  TF_RETURN_IF_ERROR(computation->AcceptWithOperandOrder(
+  TF_RETURN_IF_ERROR(computation.AcceptWithOperandOrder(
       &visitor, [&extra_users, &total_sizes](const HloInstruction* a,
                                              const HloInstruction* b) {
         if (extra_users[a] != extra_users[b]) {
@@ -470,12 +470,12 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
         }
         return a->name() < b->name();
       }));
-  CHECK_EQ(sequence.size(), computation->instruction_count());
+  CHECK_EQ(sequence.size(), computation.instruction_count());
   return sequence;
 }  // namespace xla
 
 StatusOr<HloInstructionSequence> ListMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -485,16 +485,16 @@ StatusOr<HloInstructionSequence> ListMemoryScheduler(
 }
 
 StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
-  return HloInstructionSequence(computation->MakeInstructionPostOrder());
+  return HloInstructionSequence(computation.MakeInstructionPostOrder());
 }
 
 StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -513,7 +513,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                           memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 list_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          *computation, list_sequence, points_to_analysis,
+                          computation, list_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory list sequence: " << HumanReadableNumBytes(list_memory);
 
@@ -522,7 +522,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                                          size_function, memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 dfs_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          *computation, dfs_sequence, points_to_analysis,
+                          computation, dfs_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory dfs sequence: " << HumanReadableNumBytes(dfs_memory);
 
@@ -532,7 +532,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                                memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 post_order_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          *computation, post_order_sequence, points_to_analysis,
+                          computation, post_order_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory post order sequence: "
           << HumanReadableNumBytes(post_order_memory);
@@ -555,17 +555,17 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
 }
 
 StatusOr<HloSchedule> ScheduleModule(
-    HloModule* module, const LogicalBuffer::SizeFunction& size_function,
+    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm) {
-  HloSchedule schedule(module);
+  HloSchedule schedule(&module);
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
-                      TuplePointsToAnalysis::Run(module));
+                      TuplePointsToAnalysis::Run(&module));
   absl::flat_hash_map<const HloComputation*, int64> memory_by_computation;
-  for (auto* computation : module->MakeComputationPostOrder()) {
+  for (const auto* computation : module.MakeComputationPostOrder()) {
     if (!computation->IsFusionComputation()) {
       TF_ASSIGN_OR_RETURN(HloInstructionSequence computation_sequence,
                           ScheduleComputationHelper(
-                              computation, *points_to_analysis, size_function,
+                              *computation, *points_to_analysis, size_function,
                               algorithm, memory_by_computation));
       memory_by_computation[computation] =
           HeapSimulator::MinimumMemoryForComputation(
@@ -583,11 +583,11 @@ StatusOr<HloSchedule> ScheduleModule(
 }
 
 StatusOr<HloInstructionSequence> ScheduleComputation(
-    HloComputation* computation,
+    const HloComputation& computation,
     const LogicalBuffer::SizeFunction& size_function) {
-  CHECK(!computation->IsFusionComputation());
+  CHECK(!computation.IsFusionComputation());
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
-                      TuplePointsToAnalysis::Run(computation->parent()));
+                      TuplePointsToAnalysis::Run(computation.parent()));
   absl::flat_hash_map<const HloComputation*, int64> empty_map;
   return ScheduleComputationHelper(computation, *points_to_analysis,
                                    size_function, nullptr, empty_map);
@@ -600,7 +600,7 @@ HloMemoryScheduler::HloMemoryScheduler(
 
 StatusOr<bool> HloMemoryScheduler::Run(HloModule* module) {
   TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                      ScheduleModule(module, size_function_, algorithm_));
+                      ScheduleModule(*module, size_function_, algorithm_));
   TF_RETURN_IF_ERROR(module->set_schedule(std::move(schedule)));
   return true;
 }
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
index 7227bfb27c..cca5dc4939 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
@@ -36,14 +36,14 @@ namespace xla {
 // that describes buffer aliasing, together with a target-specific size function
 // that maps a tensor's logical size to its padded size.
 typedef std::function<StatusOr<HloInstructionSequence>(
-    HloComputation*, const TuplePointsToAnalysis&,
+    const HloComputation&, const TuplePointsToAnalysis&,
     const LogicalBuffer::SizeFunction&,
     const absl::flat_hash_map<const HloComputation*, int64>&)>
     MemorySchedulerAlgorithm;
 
 // List scheduler
 StatusOr<HloInstructionSequence> ListMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -51,7 +51,7 @@ StatusOr<HloInstructionSequence> ListMemoryScheduler(
 
 // DFS-order scheduler
 StatusOr<HloInstructionSequence> DFSMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -59,7 +59,7 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
 
 // Naive Post Order scheduler
 StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -69,7 +69,7 @@ StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
 // and the DFS scheduler, and chooses whichever returns a lower min-memory,
 // not accounting for fragmentation.
 StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
-    HloComputation* computation,
+    const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -79,13 +79,13 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
 // the computation. size_function is the function returning the number of bytes
 // required for a LogicalBuffer.
 StatusOr<HloSchedule> ScheduleModule(
-    HloModule* module, const LogicalBuffer::SizeFunction& size_function,
+    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm = {});
 
 // Computes the schedule for a single computation.
 // Currently only used by the GPU backend.
 StatusOr<HloInstructionSequence> ScheduleComputation(
-    HloComputation* computation,
+    const HloComputation& computation,
     const LogicalBuffer::SizeFunction& size_function);
 
 // A pass which schedules the HLO instructions in a module. The HloModule's
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
index bc0d7e2bc0..3d8482065a 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
@@ -78,7 +78,7 @@ TEST_F(HloSchedulingTest, LastUseScheduledFirst) {
   TF_ASSERT_OK(module->schedule().Verify());
 
   // Verify that all instructions are in the sequence.
-  const std::vector<HloInstruction*>& sequence =
+  const std::vector<const HloInstruction*>& sequence =
       module->schedule().sequence(module->entry_computation()).instructions();
   EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size());
 
@@ -124,9 +124,9 @@ ENTRY root {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), size_fn, ListMemoryScheduler));
+      ScheduleModule(*module, size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
-  const std::vector<HloInstruction*>& sequence =
+  const std::vector<const HloInstruction*>& sequence =
       schedule.sequence(module->entry_computation()).instructions();
   EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size());
 
@@ -175,13 +175,12 @@ TEST_F(HloSchedulingTest, TuplesAreAccountedCorrectly) {
   auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
-                          ScheduleModule(
-                              module.get(),
-                              [](const BufferValue& buffer) {
-                                return ShapeUtil::ByteSizeOf(buffer.shape(),
-                                                             TUPLE_SIZE);
-                              },
-                              ListMemoryScheduler));
+                          ScheduleModule(*module,
+                                         [](const BufferValue& buffer) {
+                                           return ShapeUtil::ByteSizeOf(
+                                               buffer.shape(), TUPLE_SIZE);
+                                         },
+                                         ListMemoryScheduler));
 
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
@@ -226,12 +225,12 @@ TEST_F(HloSchedulingTest, MultiOutputFusionAccountedCorrectly) {
       {tuple, mul, add}, HloInstruction::FusionKind::kLoop);
 
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
-                          ScheduleModule(
-                              module.get(),
-                              [](const BufferValue& buffer) {
-                                return ShapeUtil::ByteSizeOf(buffer.shape(), 2);
-                              },
-                              ListMemoryScheduler));
+                          ScheduleModule(*module,
+                                         [](const BufferValue& buffer) {
+                                           return ShapeUtil::ByteSizeOf(
+                                               buffer.shape(), 2);
+                                         },
+                                         ListMemoryScheduler));
 
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
@@ -285,7 +284,7 @@ TEST_F(HloSchedulingTest, HeapSimulatorAccountsForSubcomputations) {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), size_fn, ListMemoryScheduler));
+      ScheduleModule(*module, size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
   auto entry_computation = module->entry_computation();
   EXPECT_EQ(module->entry_computation()->instruction_count(),
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 66622a1d26..7974999e07 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -104,7 +104,11 @@ class HloModule {
                                        HloCloneContext* context = nullptr);
 
   // Return a pointer to the entry computation of the module.
-  HloComputation* entry_computation() const {
+  const HloComputation* entry_computation() const {
+    CHECK_NE(nullptr, entry_computation_);
+    return entry_computation_;
+  }
+  HloComputation* entry_computation() {
     CHECK_NE(nullptr, entry_computation_);
     return entry_computation_;
   }
diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc
index ca6a154809..f5f99bece1 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering.cc
@@ -356,7 +356,8 @@ void SequentialHloOrdering::Initialize() {
   // Create a map from instruction to its order position.
   TF_DCHECK_OK(schedule_.Verify());
   for (const auto& computation_sequence : schedule_.sequences()) {
-    const auto& order = computation_sequence.second.instructions();
+    const std::vector<const HloInstruction*>& order =
+        computation_sequence.second.instructions();
     for (int i = 0; i < order.size(); ++i) {
       InsertOrDie(&order_position_, order[i], i);
     }
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 4bf287a9ed..4390145c6b 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -47,11 +47,11 @@ const double kF16max = 65504;
 
 // Creates and returns a schedule created using the order of the instructions in
 // the HloComputation::instructions() vectors in the module.
-HloSchedule ScheduleFromInstructionOrder(HloModule* module) {
+HloSchedule ScheduleFromInstructionOrder(const HloModule* module) {
   HloSchedule schedule(module);
-  for (HloComputation* computation : module->computations()) {
+  for (const HloComputation* computation : module->computations()) {
     if (!computation->IsFusionComputation()) {
-      for (HloInstruction* instruction : computation->instructions()) {
+      for (const HloInstruction* instruction : computation->instructions()) {
         schedule.GetOrCreateSequence(computation).push_back(instruction);
       }
     }
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 48add75523..49e46ecd00 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -130,10 +130,10 @@ using ItemList = absl::InlinedVector<Item*, 3>;
 // before arbitrary elements.
 class InstructionList {
  public:
-  explicit InstructionList(const HloInstructionSequence& order) {
+  explicit InstructionList(const std::vector<const HloInstruction*>& order) {
     int64 position = 0;
     Item* last = nullptr;
-    for (HloInstruction* inst : order.instructions()) {
+    for (const HloInstruction* inst : order) {
       // Add a new item to the linked list.
       Item* item = new Item;
       item->next = nullptr;
@@ -151,7 +151,7 @@ class InstructionList {
       // to be monotonically increasing through the list, and so is still useful
       // for quickly(-ish) determining the order of arbitrary instructions in
       // the list.
-      item->instruction = inst;
+      item->instruction = const_cast<HloInstruction*>(inst);
       item->position = position;
       position++;
 
@@ -927,7 +927,7 @@ Item* PickRematerializationCandidate(
 
 StatusOr<int64> HloRematerialization::ComputePeakMemory(
     const HloComputation* computation,
-    const HloInstructionSequence& order) const {
+    const std::vector<const HloInstruction*>& order) const {
   InstructionList instruction_list(order);
   MemoryUsageTracker tracker(computation, size_function_, *points_to_analysis_,
                              instruction_list);
@@ -971,7 +971,8 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
           << HumanReadableNumBytes(computation_peak_memory_.at(computation));
   CHECK(!ContainsKey(rematerialized_computations_, computation));
 
-  InstructionList instruction_list(schedule->sequence(computation));
+  InstructionList instruction_list(
+      schedule->sequence(computation).instructions());
   MemoryUsageTracker memory_tracker(computation, size_function_,
                                     *points_to_analysis_, instruction_list);
   bool changed = false;
@@ -1183,7 +1184,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
   sequence.clear();
   for (auto* item = instruction_list.first(); item != nullptr;
        item = instruction_list.next(item)) {
-    HloInstruction* instruction = item->instruction;
+    const HloInstruction* instruction = item->instruction;
     sequence.push_back(instruction);
   }
   rematerialized_computations_.insert(computation);
@@ -1234,8 +1235,10 @@ StatusOr<bool> HloRematerialization::Run(HloModule* module) {
         if (node.context() == CallContext::kSequential) {
           TF_ASSIGN_OR_RETURN(
               computation_peak_memory_[node.computation()],
-              ComputePeakMemory(node.computation(), module->schedule().sequence(
-                                                        node.computation())));
+              ComputePeakMemory(node.computation(),
+                                module->schedule()
+                                    .sequence(node.computation())
+                                    .instructions()));
         }
         return Status::OK();
       },
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h
index a07d348041..70d83c04f0 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.h
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h
@@ -87,8 +87,9 @@ class HloRematerialization : public HloModulePass {
   // peak memory is the maximum total size of all live HLO instruction values at
   // any program point. 'order' is the order in which the HLO instructions will
   // be emitted which is used to determine lifespans of HLO values.
-  StatusOr<int64> ComputePeakMemory(const HloComputation* computation,
-                                    const HloInstructionSequence& order) const;
+  StatusOr<int64> ComputePeakMemory(
+      const HloComputation* computation,
+      const std::vector<const HloInstruction*>& order) const;
 
   // Returns the peak memory usage of the called computations for the given
   // instruction. Zero is returned if the instruction calls no computations.
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.cc b/tensorflow/compiler/xla/service/hlo_schedule.cc
index 8f6eb974c5..a5780b7551 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule.cc
@@ -46,8 +46,8 @@ namespace xla {
         << "No computation exists in HLO module with id " << computation_id;
     const HloComputation* computation = comp_it->second;
 
-    absl::flat_hash_map<int64, HloInstruction*> id_to_instruction;
-    for (HloInstruction* instruction : computation->instructions()) {
+    absl::flat_hash_map<int64, const HloInstruction*> id_to_instruction;
+    for (const HloInstruction* instruction : computation->instructions()) {
       id_to_instruction[instruction->unique_id()] = instruction;
     }
 
@@ -81,8 +81,9 @@ StatusOr<HloScheduleProto> HloSchedule::ToProto() const {
   return std::move(proto);
 }
 
-void HloSchedule::set_sequence(const HloComputation* computation,
-                               absl::Span<HloInstruction* const> sequence) {
+void HloSchedule::set_sequence(
+    const HloComputation* computation,
+    absl::Span<const HloInstruction* const> sequence) {
   set_sequence(computation, HloInstructionSequence(sequence));
 }
 
@@ -113,8 +114,8 @@ Status HloSchedule::UpdateComputationSchedule(
     const HloComputation* computation) {
   // Map from unique ID to HloInstruction pointer for instructions in the
   // computation.
-  absl::flat_hash_map<int, HloInstruction*> id_to_instruction;
-  for (HloInstruction* instruction : computation->instructions()) {
+  absl::flat_hash_map<int, const HloInstruction*> id_to_instruction;
+  for (const HloInstruction* instruction : computation->instructions()) {
     InsertOrDie(&id_to_instruction, instruction->unique_id(), instruction);
   }
 
@@ -127,7 +128,7 @@ Status HloSchedule::UpdateComputationSchedule(
   // Map from HloInstruction X to newly added instructions (instruction is in
   // computation, but not in schedule) which use X. If an instruction is not in
   // the map, then it has no users which are newly added instructions.
-  absl::flat_hash_map<const HloInstruction*, std::vector<HloInstruction*>>
+  absl::flat_hash_map<const HloInstruction*, std::vector<const HloInstruction*>>
       new_instruction_uses;
 
   // For each newly added instruction, this is the count of the instruction's
@@ -137,9 +138,9 @@ Status HloSchedule::UpdateComputationSchedule(
 
   // Create a worklist of newly added instructions which are ready to be added
   // to the schedule. Initialize worklist with those that have zero operands.
-  std::queue<HloInstruction*> worklist;
+  std::queue<const HloInstruction*> worklist;
 
-  for (HloInstruction* instruction : computation->instructions()) {
+  for (const HloInstruction* instruction : computation->instructions()) {
     if (ids_in_schedule.count(instruction->unique_id()) == 0) {
       // This is a newly added instruction which is not in the schedule.
       if (instruction->operands().empty()) {
@@ -160,17 +161,17 @@ Status HloSchedule::UpdateComputationSchedule(
   // Lambda which schedules all instructions on the worklist.
   auto schedule_worklist = [&]() {
     while (!worklist.empty()) {
-      HloInstruction* instruction = worklist.front();
+      const HloInstruction* instruction = worklist.front();
       worklist.pop();
       new_sequence.push_back(instruction);
-      std::vector<HloInstruction*>* new_users =
+      std::vector<const HloInstruction*>* new_users =
           tensorflow::gtl::FindOrNull(new_instruction_uses, instruction);
       if (new_users != nullptr) {
         // This just-scheduled instruction has users which are newly added to
         // the module. Update the number of unscheduled operands and push the
         // newly added instruction to the worklist if it is ready to
         // schedule.
-        for (HloInstruction* new_user : *new_users) {
+        for (const HloInstruction* new_user : *new_users) {
           unscheduled_operand_count.at(new_user)--;
           CHECK_GE(unscheduled_operand_count.at(new_user), 0);
           if (unscheduled_operand_count.at(new_user) == 0) {
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.h b/tensorflow/compiler/xla/service/hlo_schedule.h
index 486ddbf499..0a714101ee 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/hlo_schedule.h
@@ -35,14 +35,14 @@ class HloInstructionSequence {
  public:
   HloInstructionSequence() = default;
   explicit HloInstructionSequence(
-      absl::Span<HloInstruction* const> instructions) {
-    for (HloInstruction* instruction : instructions) {
+      absl::Span<const HloInstruction* const> instructions) {
+    for (const HloInstruction* instruction : instructions) {
       push_back(instruction);
     }
   }
 
   // Adds the instruction to the end of the sequence.
-  void push_back(HloInstruction* instruction) {
+  void push_back(const HloInstruction* instruction) {
     instruction_sequence_.push_back(instruction);
     id_sequence_.push_back(instruction->unique_id());
   }
@@ -56,7 +56,7 @@ class HloInstructionSequence {
   int64 size() const { return instruction_sequence_.size(); }
 
   // Returns the sequence of HLO instructions.
-  const std::vector<HloInstruction*>& instructions() const {
+  const std::vector<const HloInstruction*>& instructions() const {
     return instruction_sequence_;
   }
 
@@ -65,7 +65,7 @@ class HloInstructionSequence {
 
  private:
   // The sequence as HloInstructions.
-  std::vector<HloInstruction*> instruction_sequence_;
+  std::vector<const HloInstruction*> instruction_sequence_;
 
   // The sequence of HLO instructions, represented by their unique IDs. The
   // sequence is stored as both HloInstructions and unique IDs because the
@@ -98,7 +98,7 @@ class HloSchedule {
 
   // Sets the sequence for the given computation to the given sequence.
   void set_sequence(const HloComputation* computation,
-                    absl::Span<HloInstruction* const> sequence);
+                    absl::Span<const HloInstruction* const> sequence);
   void set_sequence(const HloComputation* computation,
                     HloInstructionSequence sequence);
 
diff --git a/tensorflow/compiler/xla/service/hlo_schedule_test.cc b/tensorflow/compiler/xla/service/hlo_schedule_test.cc
index 0e56e6f760..1424569ac1 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule_test.cc
@@ -56,10 +56,10 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
-  const auto& entry_schedule =
+  const std::vector<const HloInstruction*>& entry_schedule =
       schedule.sequence(module->entry_computation()).instructions();
 
   EXPECT_EQ(entry_schedule.size(), 6);
@@ -90,7 +90,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -139,7 +139,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -183,7 +183,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -244,7 +244,7 @@ ENTRY %WhileLoop () -> s32[] {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
@@ -313,7 +313,7 @@ ENTRY %WhileLoop () -> s32[] {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(module.get(), [](const BufferValue& buffer) {
+      ScheduleModule(*module, [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
-- 
GitLab


From 6bb7813672ca7f843212932f9000adbb52740e06 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 16 Nov 2018 18:31:24 -0800
Subject: [PATCH 0465/1554] Disable broken timeseries tests on macos.

PiperOrigin-RevId: 221883555
---
 tensorflow/contrib/timeseries/python/timeseries/BUILD        | 1 +
 .../timeseries/python/timeseries/state_space_models/BUILD    | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index c230919168..ae7db35b47 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -106,6 +106,7 @@ py_test(
     ],
     srcs_version = "PY2AND3",
     tags = [
+        "no_mac",
         "no_pip_gpu",  # b/63391119
         "nomsan",  # Takes too long to run.
         "notsan",  # b/67865658
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index 3c07a74ed8..125750e763 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -40,7 +40,10 @@ py_test(
     timeout = "long",  # Moderate but for asan
     srcs = ["state_space_model_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
+    tags = [
+        "no_mac",
+        "no_windows",  # TODO: needs investigation on Windows
+    ],
     deps = [
         ":state_space_model",
         "//tensorflow/contrib/layers:layers_py",
-- 
GitLab


From 32a54c9f4fc2e66c7982291be2eab7a321c6234e Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 16 Nov 2018 18:37:48 -0800
Subject: [PATCH 0466/1554] Disable flaky values_test

PiperOrigin-RevId: 221883979
---
 tensorflow/contrib/distribute/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 99a1c05ec8..d1dabf3f35 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -40,6 +40,7 @@ cuda_py_test(
         "//tensorflow/python/estimator:estimator_py",
     ],
     tags = [
+        "no_oss",
         "no_pip",
     ],
 )
-- 
GitLab


From 3c3a1b6d4d9a2d754e10eca724b3a0bb3d15b8f3 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 16 Nov 2018 18:38:45 -0800
Subject: [PATCH 0467/1554] Implement make_dataset_iterator in MirroredStrategy
 and use it in Keras.

PiperOrigin-RevId: 221884049
---
 .../contrib/distribute/python/keras_test.py   | 14 +++++-----
 .../distribute/python/mirrored_strategy.py    | 26 +++++++++++++++++++
 .../distribute/python/one_device_strategy.py  |  5 ++--
 .../engine/distributed_training_utils.py      | 24 +++++++++++------
 tensorflow/python/keras/engine/training.py    |  9 ++-----
 tensorflow/python/training/distribute.py      | 25 +++++++++++++++++-
 6 files changed, 76 insertions(+), 27 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 9c2c05c928..0776d1772c 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -220,7 +220,8 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
   # TODO(b/118776054): Use global batch size for Keras/DS support.
   use_per_core_batch_size = (
       with_distribution and
-      not isinstance(with_distribution, tpu_strategy.TPUStrategy))
+      not isinstance(with_distribution, (
+          tpu_strategy.TPUStrategy, mirrored_strategy.CoreMirroredStrategy)))
   if use_per_core_batch_size:
     batch_size //= with_distribution.num_replicas_in_sync
 
@@ -541,7 +542,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # to the number of replicas(2).
       # The global batch size and batch size are rounded integer values.
       self.assertEqual(10, distributed_training_utils.get_batch_dimension(
-          iterator._iterator))
+          iterator))
 
   @combinations.generate(strategy_combinations())
   def test_calling_model_with_numpy_arrays(self, distribution):
@@ -615,9 +616,9 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
     loss = 'mse'
     model.compile(optimizer, loss, distribute=distribution)
 
-    inputs = np.zeros((10, 3), np.float32)
-    targets = np.zeros((10, 4), np.float32)
-    sample_weights = np.ones((10), np.float32)
+    inputs = np.zeros((20, 3), np.float32)
+    targets = np.zeros((20, 4), np.float32)
+    sample_weights = np.ones((20), np.float32)
 
     model.fit(inputs, targets, sample_weight=sample_weights, epochs=1,
               steps_per_epoch=2, verbose=1)
@@ -1168,8 +1169,5 @@ class TestDistributionStrategyCorrectness(test.TestCase,
           predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance)
 
 
-# TODO(priyag): Add a test for TPUStrategy with steps_per_run > 1.
-
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 6a4fb07bb3..370c96ac4f 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -499,6 +499,14 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
       return values.PerReplicaDataset(
           self._call_dataset_fn(dataset_fn), self._devices)
 
+  def _make_dataset_iterator(self, dataset):
+    if self._cluster_spec:
+      worker_device_pairs = self._worker_devices
+    else:
+      worker_device_pairs = [("/job:localhost", self._devices)]
+    return values.DatasetIterator(dataset, worker_device_pairs,
+                                  self._num_replicas_in_sync)
+
   def _make_input_fn_iterator(
       self,
       input_fn,
@@ -881,6 +889,24 @@ class MirroredExtended(CoreMirroredExtended):
         container_strategy, devices, num_gpus, num_gpus_per_worker,
         cross_device_ops, auto_shard_dataset)
 
+  def _make_dataset_iterator(self, dataset):
+    """Make iterator from dataset without splitting the batch.
+
+    This implementation is different than the one in
+    `tf.distribute.MirroredStrategy` for purposes of backward compatibility.
+    We treat the incoming dataset's batch size as per replica batch size.
+
+    Args:
+      dataset: `tf.data.Dataset` for input.
+    Returns:
+      An `InputIterator` which returns inputs for each step of the computation.
+    """
+    if self._cluster_spec:
+      worker_device_pairs = self._worker_devices
+    else:
+      worker_device_pairs = [("/job:localhost", self._devices)]
+    return values.DatasetIterator(dataset, worker_device_pairs)
+
 
 class MirroredReplicaContext(distribute_lib.ReplicaContext):
   """ReplicaContext used in MirroredStrategy.call_for_each_replica().
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 2ef60bf912..2f6d38547c 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -67,9 +67,8 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
       return next_creator(*args, **kwargs)
 
   def _make_dataset_iterator(self, dataset):
-    distributed_dataset = values.PerReplicaDataset(dataset, [self._device])
-    # TODO(priyag): Return distribution strategy specific InputIterator
-    return distributed_dataset.make_initializable_iterator()
+    """Make iterator from dataset without splitting the batch."""
+    return values.DatasetIterator(dataset, [("/job:localhost", [self._device])])
 
   def _distribute_dataset(self, dataset_fn):
     return values.PerReplicaDataset(
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index b41febbbc7..8e0e2adca0 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -391,6 +391,16 @@ def validate_inputs(x, y, distribution_strategy):
               'Found unknown shape {} in input {}.'.format(s, i))
 
 
+# TODO(b/118776054): Currently we support global batch size for TPUStrategy
+# and CoreMirroredStrategy only. Remove this check when contrib MirroredStrategy
+# is no longer needed.
+def global_batch_size_supported(distribution_strategy):
+  strategy_name = distribution_strategy.__class__.__name__
+  # TODO(priyag): Change this to whatever condition makes sense when
+  # CoreMirroredStrategy is moved to core and renamed.
+  return strategy_name in ('TPUStrategy', 'CoreMirroredStrategy')
+
+
 def get_input_batch_params(first_x_value, batch_size, distribution_strategy):
   """Calculate the number of batches and steps/steps_per_epoch.
 
@@ -413,7 +423,7 @@ def get_input_batch_params(first_x_value, batch_size, distribution_strategy):
     # Default the global batch size to the minimum of 32 and the size of
     # the numpy array. 32 is chosen to guarantee backward compatibility.
     batch_size = min(first_x_value.shape[0], 32)
-    if distribution_strategy.__class__.__name__ != 'TPUStrategy':
+    if not global_batch_size_supported(distribution_strategy):
       if batch_size % distribution_strategy.num_replicas_in_sync:
         raise ValueError(
             'The batch size (%s) could not be sharded evenly across the sync '
@@ -428,12 +438,10 @@ def get_input_batch_params(first_x_value, batch_size, distribution_strategy):
   if not num_batches:
     raise ValueError('Please specify a batch_size that is smaller than '
                      'the number of input samples %d.' % first_x_value.shape[0])
-  # TODO(b/118776054): Use global batch size for Keras/DS support.
-  # The Keras API supports using the global batch size which is currently only
-  # supported in TPU Strategy. For other strategies we use a per_replica
-  # batch size so the number of steps required to run needs to be divide by
-  # the number of replicas.
-  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
+
+  # Number of steps required to run needs to be divide by the number of replicas
+  # if global batch size is not supported.
+  if global_batch_size_supported(distribution_strategy):
     steps = num_batches
   else:
     steps = num_batches // distribution_strategy.num_replicas_in_sync
@@ -479,7 +487,7 @@ def get_batch_size(distribution_strategy, num_samples, steps):
   # The Keras API supports using the global batch size which is currently only
   # supported in TPU Strategy. For other strategies we use a per_replica
   # batch size so we need to divide it by the number of replicas.
-  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
+  if global_batch_size_supported(distribution_strategy):
     return global_batch_size
 
   num_replicas = distribution_strategy.num_replicas_in_sync
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 610c8e625a..c3ec23c6b4 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1047,13 +1047,8 @@ class Model(Network):
     assert isinstance(x, dataset_ops.Dataset)
 
     with self._distribution_strategy.scope():
-      if type(self._distribution_strategy).__name__ == 'TPUStrategy':
-        iterator = self._distribution_strategy.make_dataset_iterator(x)
-        K.get_session().run(iterator.initialize())
-      else:
-        dataset = self._distribution_strategy.distribute_dataset(lambda: x)
-        iterator = dataset.make_initializable_iterator()
-        K.get_session().run(iterator.initializer)
+      iterator = self._distribution_strategy.make_dataset_iterator(x)
+      K.get_session().run(iterator.initialize())
 
     training_utils.validate_iterator_input(x, y, sample_weight,
                                            validation_split)
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 51da8a7eec..b7869e0330 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -25,6 +25,7 @@ import enum
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
+from tensorflow.python.eager import context as eager_context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -1500,7 +1501,7 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
     return self._call_dataset_fn(dataset_fn)
 
   def _make_dataset_iterator(self, dataset):
-    return dataset.make_initializable_iterator()
+    return _DefaultDistributionExtended.DefaultInputIterator(dataset)
 
   def _make_input_fn_iterator(self,
                               input_fn,
@@ -1565,6 +1566,28 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def non_slot_devices(self, var_list):
     return min(var_list, key=lambda x: x.name)
 
+  # TODO(priyag): This should inherit from `InputIterator`, once dependency
+  # issues have been resolved.
+  class DefaultInputIterator(object):
+    """Default implementation of `InputIterator` for default strategy."""
+
+    def __init__(self, dataset):
+      self._dataset = dataset
+      if eager_context.executing_eagerly():
+        self._iterator = dataset.make_one_shot_iterator()
+      else:
+        self._iterator = dataset.make_initializable_iterator()
+
+    def get_next(self):
+      return self._iterator.get_next()
+
+    def initialize(self):
+      if eager_context.executing_eagerly():
+        self._iterator = self._dataset.make_one_shot_iterator()
+        return []
+      else:
+        return [self._iterator.initializer]
+
 
 # ------------------------------------------------------------------------------
 # We haven't yet implemented deserialization for DistributedVariables.
-- 
GitLab


From 5476ec23618c2da528436cec74de0e301fcff643 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 18:46:37 -0800
Subject: [PATCH 0468/1554] Allow LinearOperator.matmul to operate on other
 LinearOperators.

PiperOrigin-RevId: 221884512
---
 .../linalg/linear_operator_algebra_test.py    |  61 ++++-
 .../linalg/linear_operator_diag_test.py       |  29 ++
 .../linalg/linear_operator_identity_test.py   |  25 +-
 .../linear_operator_lower_triangular_test.py  |  25 ++
 .../linalg/linear_operator_test.py            |  71 +++++
 .../linalg/linear_operator_zeros_test.py      |  11 +
 tensorflow/python/ops/linalg/linalg.py        |   1 +
 .../python/ops/linalg/linear_operator.py      |  19 +-
 .../ops/linalg/linear_operator_algebra.py     | 111 +++++++-
 .../python/ops/linalg/matmul_registrations.py | 252 ++++++++++++++++++
 10 files changed, 583 insertions(+), 22 deletions(-)
 create mode 100644 tensorflow/python/ops/linalg/matmul_registrations.py

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
index 98091f9758..8e296c026c 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
@@ -18,14 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops.linalg import cholesky_registrations  # pylint: disable=unused-import
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.ops.linalg import matmul_registrations  # pylint: disable=unused-import
 from tensorflow.python.platform import test
 
 # pylint: disable=protected-access
 _CHOLESKY_DECOMPS = linear_operator_algebra._CHOLESKY_DECOMPS
+_MATMUL = linear_operator_algebra._MATMUL
 _registered_cholesky = linear_operator_algebra._registered_cholesky
+_registered_matmul = linear_operator_algebra._registered_matmul
 # pylint: enable=protected-access
 
 
@@ -39,7 +43,7 @@ class CholeskyTest(test.TestCase):
         pass
 
       def _shape(self):
-        pass
+        return tensor_shape.TensorShape([1, 1])
 
       def _shape_tensor(self):
         pass
@@ -55,8 +59,9 @@ class CholeskyTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "self adjoint"):
       CustomLinOp(dtype=None, is_positive_definite=True).cholesky()
 
-    self.assertEqual("OK", CustomLinOp(
-        dtype=None, is_self_adjoint=True, is_positive_definite=True).cholesky())
+    custom_linop = CustomLinOp(
+        dtype=None, is_self_adjoint=True, is_positive_definite=True)
+    self.assertEqual("OK", custom_linop.cholesky())
 
   def testRegistrationFailures(self):
 
@@ -73,9 +78,55 @@ class CholeskyTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "has already been registered"):
       linear_operator_algebra.RegisterCholesky(CustomLinOp)(lambda a: None)
 
-  def testExactRegistrationsAllMatch(self):
+  def testExactCholeskyRegistrationsAllMatch(self):
     for (k, v) in _CHOLESKY_DECOMPS.items():
-      self.assertEqual(v, _registered_cholesky(k))
+      self.assertEqual(v, _registered_cholesky(k[0]))
+
+
+class MatmulTest(test.TestCase):
+
+  def testRegistration(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+
+      def _matmul(self, a):
+        pass
+
+      def _shape(self):
+        return tensor_shape.TensorShape([1, 1])
+
+      def _shape_tensor(self):
+        pass
+
+    # Register Matmul to a lambda that spits out the name parameter
+    @linear_operator_algebra.RegisterMatmul(CustomLinOp, CustomLinOp)
+    def _matmul(a, b):  # pylint: disable=unused-argument,unused-variable
+      return "OK"
+
+    custom_linop = CustomLinOp(
+        dtype=None, is_self_adjoint=True, is_positive_definite=True)
+    self.assertEqual("OK", custom_linop.matmul(custom_linop))
+
+  def testRegistrationFailures(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+      pass
+
+    with self.assertRaisesRegexp(TypeError, "must be callable"):
+      linear_operator_algebra.RegisterMatmul(CustomLinOp, CustomLinOp)("blah")
+
+    # First registration is OK
+    linear_operator_algebra.RegisterMatmul(
+        CustomLinOp, CustomLinOp)(lambda a: None)
+
+    # Second registration fails
+    with self.assertRaisesRegexp(ValueError, "has already been registered"):
+      linear_operator_algebra.RegisterMatmul(
+          CustomLinOp, CustomLinOp)(lambda a: None)
+
+  def testExactMatmulRegistrationsAllMatch(self):
+    for (k, v) in _MATMUL.items():
+      self.assertEqual(v, _registered_matmul(k[0], k[1]))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index d8e53fdcf5..91f4097438 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -154,6 +154,35 @@ class LinearOperatorDiagTest(
       self.assertAllEqual(operator_solve.get_shape(), mat_solve.get_shape())
       self.assertAllClose(*sess.run([operator_solve, mat_solve]))
 
+  def test_diag_matmul(self):
+    operator1 = linalg_lib.LinearOperatorDiag([2., 3.])
+    operator2 = linalg_lib.LinearOperatorDiag([1., 2.])
+    operator3 = linalg_lib.LinearOperatorScaledIdentity(
+        num_rows=2, multiplier=3.)
+    operator_matmul = operator1.matmul(operator2)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorDiag))
+    self.assertAllClose([2., 6.], self.evaluate(operator_matmul.diag))
+
+    operator_matmul = operator2.matmul(operator1)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorDiag))
+    self.assertAllClose([2., 6.], self.evaluate(operator_matmul.diag))
+
+    operator_matmul = operator1.matmul(operator3)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorDiag))
+    self.assertAllClose([6., 9.], self.evaluate(operator_matmul.diag))
+
+    operator_matmul = operator3.matmul(operator1)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorDiag))
+    self.assertAllClose([6., 9.], self.evaluate(operator_matmul.diag))
+
   def test_diag_cholesky_type(self):
     diag = [1., 3., 5., 8.]
     operator = linalg.LinearOperatorDiag(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 45cd14ce04..522213e26b 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -445,6 +445,30 @@ class LinearOperatorScaledIdentityTest(
     self.assertTrue(operator.is_non_singular)
     self.assertTrue(operator.is_self_adjoint is None)
 
+  def test_identity_matmul(self):
+    operator1 = linalg_lib.LinearOperatorIdentity(num_rows=2)
+    operator2 = linalg_lib.LinearOperatorScaledIdentity(
+        num_rows=2, multiplier=3.)
+    self.assertTrue(isinstance(
+        operator1.matmul(operator1),
+        linalg_lib.LinearOperatorIdentity))
+
+    self.assertTrue(isinstance(
+        operator1.matmul(operator1),
+        linalg_lib.LinearOperatorIdentity))
+
+    operator_matmul = operator1.matmul(operator2)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorScaledIdentity))
+    self.assertAllClose(3., self.evaluate(operator_matmul.multiplier))
+
+    operator_matmul = operator2.matmul(operator1)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorScaledIdentity))
+    self.assertAllClose(3., self.evaluate(operator_matmul.multiplier))
+
   def test_scaled_identity_cholesky_type(self):
     operator = linalg_lib.LinearOperatorScaledIdentity(
         num_rows=2,
@@ -457,6 +481,5 @@ class LinearOperatorScaledIdentityTest(
         linalg_lib.LinearOperatorScaledIdentity))
 
 
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
index 70a3e36c41..bd41f9ed9d 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
@@ -76,6 +77,30 @@ class LinearOperatorLowerTriangularTest(
     with self.assertRaisesRegexp(ValueError, "at least 2 dimensions"):
       linalg.LinearOperatorLowerTriangular([1.])
 
+  def test_triangular_diag_matmul(self):
+    operator1 = linalg_lib.LinearOperatorLowerTriangular(
+        [[1., 0., 0.], [2., 1., 0.], [2., 3., 3.]])
+    operator2 = linalg_lib.LinearOperatorDiag([2., 2., 3.])
+    operator_matmul = operator1.matmul(operator2)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorLowerTriangular))
+    self.assertAllClose(
+        math_ops.matmul(
+            operator1.to_dense(),
+            operator2.to_dense()),
+        self.evaluate(operator_matmul.to_dense()))
+
+    operator_matmul = operator2.matmul(operator1)
+    self.assertTrue(isinstance(
+        operator_matmul,
+        linalg_lib.LinearOperatorLowerTriangular))
+    self.assertAllClose(
+        math_ops.matmul(
+            operator2.to_dense(),
+            operator1.to_dense()),
+        self.evaluate(operator_matmul.to_dense()))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
index 4cb60dac98..2f67df408c 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
@@ -208,6 +208,77 @@ class LinearOperatorTest(test.TestCase):
     operator = LinearOperatorMatmulSolve(matrix, is_square=True)
     self.assertTrue(operator.is_square)
 
+  def test_linear_operator_matmul_hints_closed(self):
+    matrix = array_ops.placeholder(dtypes.float32)
+    operator1 = LinearOperatorMatmulSolve(matrix)
+
+    operator_matmul = operator1.matmul(operator1)
+
+    self.assertEqual(None, operator_matmul.is_square)
+    self.assertEqual(None, operator_matmul.is_non_singular)
+    self.assertEqual(None, operator_matmul.is_self_adjoint)
+    self.assertEqual(None, operator_matmul.is_positive_definite)
+
+    operator2 = LinearOperatorMatmulSolve(
+        matrix,
+        is_non_singular=True,
+        is_self_adjoint=True,
+        is_positive_definite=True,
+        is_square=True,
+    )
+
+    operator_matmul = operator2.matmul(operator2)
+
+    self.assertTrue(operator_matmul.is_square)
+    self.assertTrue(operator_matmul.is_non_singular)
+    self.assertTrue(operator_matmul.is_self_adjoint)
+    self.assertEqual(None, operator_matmul.is_positive_definite)
+
+  def test_linear_operator_matmul_hints_false(self):
+    matrix = array_ops.placeholder(dtypes.float32)
+    operator1 = LinearOperatorMatmulSolve(
+        matrix,
+        is_non_singular=False,
+        is_self_adjoint=False,
+        is_positive_definite=False,
+        is_square=True,
+    )
+
+    operator_matmul = operator1.matmul(operator1)
+
+    self.assertTrue(operator_matmul.is_square)
+    self.assertFalse(operator_matmul.is_non_singular)
+    self.assertEqual(None, operator_matmul.is_self_adjoint)
+    self.assertEqual(None, operator_matmul.is_positive_definite)
+
+    operator2 = LinearOperatorMatmulSolve(
+        matrix,
+        is_non_singular=False,
+        is_self_adjoint=False,
+        is_positive_definite=False,
+        is_square=False,
+    )
+
+    operator_matmul = operator2.matmul(operator2)
+
+    self.assertEqual(None, operator_matmul.is_square)
+    self.assertEqual(None, operator_matmul.is_non_singular)
+    self.assertEqual(None, operator_matmul.is_self_adjoint)
+    self.assertEqual(None, operator_matmul.is_positive_definite)
+
+  def test_linear_operator_matmul_hint_infer_square(self):
+    matrix1 = array_ops.placeholder(shape=[2, 3], dtype=dtypes.float32)
+    matrix2 = array_ops.placeholder(shape=[3, 2], dtype=dtypes.float32)
+    matrix3 = array_ops.placeholder(shape=[3, 4], dtype=dtypes.float32)
+
+    operator1 = LinearOperatorMatmulSolve(matrix1, is_square=False)
+    operator2 = LinearOperatorMatmulSolve(matrix2, is_square=False)
+    operator3 = LinearOperatorMatmulSolve(matrix3, is_square=False)
+
+    self.assertTrue(operator1.matmul(operator2).is_square)
+    self.assertTrue(operator2.matmul(operator1).is_square)
+    self.assertFalse(operator1.matmul(operator3).is_square)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
index d1bcb89994..e875579a7a 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
@@ -169,6 +169,17 @@ class LinearOperatorZerosTest(
     self.assertFalse(operator.is_non_singular)
     self.assertTrue(operator.is_self_adjoint)
 
+  def test_zeros_matmul(self):
+    operator1 = linalg_lib.LinearOperatorIdentity(num_rows=2)
+    operator2 = linalg_lib.LinearOperatorZeros(num_rows=2)
+    self.assertTrue(isinstance(
+        operator1.matmul(operator2),
+        linalg_lib.LinearOperatorZeros))
+
+    self.assertTrue(isinstance(
+        operator2.matmul(operator1),
+        linalg_lib.LinearOperatorZeros))
+
 
 class LinearOperatorZerosNotSquareTest(
     linear_operator_test_util.NonSquareLinearOperatorDerivedClassTest):
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index e29c45c5f5..ac4fd4ebc6 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 # pylint: disable=wildcard-import,unused-import
 from tensorflow.python.ops.linalg import cholesky_registrations as _cholesky_registrations
 from tensorflow.python.ops.linalg import linear_operator_algebra as _linear_operator_algebra
+from tensorflow.python.ops.linalg import matmul_registrations as _matmul_registrations
 from tensorflow.python.ops.linalg.linalg_impl import *
 from tensorflow.python.ops.linalg.linear_operator import *
 from tensorflow.python.ops.linalg.linear_operator_block_diag import *
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index b8683a24fc..6fb7a57e4d 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -582,16 +582,29 @@ class LinearOperator(object):
     ```
 
     Args:
-      x: `Tensor` with compatible shape and same `dtype` as `self`.
-        See class docstring for definition of compatibility.
+      x: `LinearOperator` or `Tensor` with compatible shape and same `dtype` as
+        `self`. See class docstring for definition of compatibility.
       adjoint: Python `bool`.  If `True`, left multiply by the adjoint: `A^H x`.
       adjoint_arg:  Python `bool`.  If `True`, compute `A x^H` where `x^H` is
         the hermitian transpose (transposition and complex conjugation).
       name:  A name for this `Op`.
 
     Returns:
-      A `Tensor` with shape `[..., M, R]` and same `dtype` as `self`.
+      A `LinearOperator` or `Tensor` with shape `[..., M, R]` and same `dtype`
+        as `self`.
     """
+    if isinstance(x, LinearOperator):
+      if adjoint or adjoint_arg:
+        raise ValueError(".matmul not supported with adjoints.")
+      if (x.range_dimension is not None and
+          self.domain_dimension is not None and
+          x.range_dimension != self.domain_dimension):
+        raise ValueError(
+            "Operators are incompatible. Expected `x` to have dimension"
+            " {} but got {}.".format(self.domain_dimension, x.range_dimension))
+      with self._name_scope(name):
+        return linear_operator_algebra.matmul(self, x)
+
     with self._name_scope(name, values=[x]):
       x = ops.convert_to_tensor(x, name="x")
       self._check_input_dtype(x)
diff --git a/tensorflow/python/ops/linalg/linear_operator_algebra.py b/tensorflow/python/ops/linalg/linear_operator_algebra.py
index ba9b732797..7b99066e4c 100644
--- a/tensorflow/python/ops/linalg/linear_operator_algebra.py
+++ b/tensorflow/python/ops/linalg/linear_operator_algebra.py
@@ -19,26 +19,40 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import itertools
+
 from tensorflow.python.framework import ops
 from tensorflow.python.util import tf_inspect
 
 
 _CHOLESKY_DECOMPS = {}
+_MATMUL = {}
+
+
+def _registered_function(type_list, registry):
+  """Given a list of classes, finds the most specific function registered."""
+  enumerated_hierarchies = [enumerate(tf_inspect.getmro(t)) for t in type_list]
+  # Get all possible combinations of hierarchies.
+  cls_combinations = list(itertools.product(*enumerated_hierarchies))
+
+  def hierarchy_distance(cls_combination):
+    candidate_distance = sum(c[0] for c in cls_combination)
+    if tuple(c[1] for c in cls_combination) in registry:
+      return candidate_distance
+    return 10000
+
+  registered_combination = min(cls_combinations, key=hierarchy_distance)
+  return registry.get(tuple(r[1] for r in registered_combination), None)
 
 
 def _registered_cholesky(type_a):
   """Get the Cholesky function registered for class a."""
-  hierarchy_a = tf_inspect.getmro(type_a)
-  distance_to_children = None
-  cholesky_fn = None
-  for mro_to_a, parent_a in enumerate(hierarchy_a):
-    candidate_dist = mro_to_a
-    candidate_cholesky_fn = _CHOLESKY_DECOMPS.get(parent_a, None)
-    if not cholesky_fn or (
-        candidate_cholesky_fn and candidate_dist < distance_to_children):
-      distance_to_children = candidate_dist
-      cholesky_fn = candidate_cholesky_fn
-  return cholesky_fn
+  return _registered_function([type_a], _CHOLESKY_DECOMPS)
+
+
+def _registered_matmul(type_a, type_b):
+  """Get the Matmul function registered for classes a and b."""
+  return _registered_function([type_a, type_b], _MATMUL)
 
 
 def cholesky(lin_op_a, name=None):
@@ -64,6 +78,31 @@ def cholesky(lin_op_a, name=None):
     return cholesky_fn(lin_op_a)
 
 
+def matmul(lin_op_a, lin_op_b, name=None):
+  """Compute lin_op_a.matmul(lin_op_b).
+
+  Args:
+    lin_op_a: The LinearOperator on the left.
+    lin_op_b: The LinearOperator on the right.
+    name: Name to use for this operation.
+
+  Returns:
+    A LinearOperator that represents the matmul between `lin_op_a` and
+      `lin_op_b`.
+
+  Raises:
+    NotImplementedError: If no matmul method is defined between types of
+      `lin_op_a` and `lin_op_b`.
+  """
+  matmul_fn = _registered_matmul(type(lin_op_a), type(lin_op_b))
+  if matmul_fn is None:
+    raise ValueError("No matmul registered for {}.matmul({})".format(
+        type(lin_op_a), type(lin_op_b)))
+
+  with ops.name_scope(name, "Matmul"):
+    return matmul_fn(lin_op_a, lin_op_b)
+
+
 class RegisterCholesky(object):
   """Decorator to register a Cholesky implementation function.
 
@@ -80,7 +119,7 @@ class RegisterCholesky(object):
     Args:
       lin_op_cls_a: the class of the LinearOperator to decompose.
     """
-    self._key = lin_op_cls_a
+    self._key = (lin_op_cls_a,)
 
   def __call__(self, cholesky_fn):
     """Perform the Cholesky registration.
@@ -101,6 +140,52 @@ class RegisterCholesky(object):
           "cholesky_fn must be callable, received: {}".format(cholesky_fn))
     if self._key in _CHOLESKY_DECOMPS:
       raise ValueError("Cholesky({}) has already been registered to: {}".format(
-          self._key.__name__, _CHOLESKY_DECOMPS[self._key]))
+          self._key[0].__name__, _CHOLESKY_DECOMPS[self._key]))
     _CHOLESKY_DECOMPS[self._key] = cholesky_fn
     return cholesky_fn
+
+
+class RegisterMatmul(object):
+  """Decorator to register a Matmul implementation function.
+
+  Usage:
+
+  @linear_operator_algebra.RegisterMatmul(
+    lin_op.LinearOperatorIdentity,
+    lin_op.LinearOperatorIdentity)
+  def _matmul_identity(a, b):
+    # Return the identity matrix.
+  """
+
+  def __init__(self, lin_op_cls_a, lin_op_cls_b):
+    """Initialize the LinearOperator registrar.
+
+    Args:
+      lin_op_cls_a: the class of the LinearOperator to multiply.
+      lin_op_cls_b: the class of the second LinearOperator to multiply.
+    """
+    self._key = (lin_op_cls_a, lin_op_cls_b)
+
+  def __call__(self, matmul_fn):
+    """Perform the Matmul registration.
+
+    Args:
+      matmul_fn: The function to use for the Matmul.
+
+    Returns:
+      matmul_fn
+
+    Raises:
+      TypeError: if matmul_fn is not a callable.
+      ValueError: if a Matmul function has already been registered for
+        the given argument classes.
+    """
+    if not callable(matmul_fn):
+      raise TypeError(
+          "matmul_fn must be callable, received: {}".format(matmul_fn))
+    if self._key in _MATMUL:
+      raise ValueError("Matmul({}, {}) has already been registered.".format(
+          self._key[0].__name__,
+          self._key[1].__name__))
+    _MATMUL[self._key] = matmul_fn
+    return matmul_fn
diff --git a/tensorflow/python/ops/linalg/matmul_registrations.py b/tensorflow/python/ops/linalg/matmul_registrations.py
new file mode 100644
index 0000000000..e0ac988ba2
--- /dev/null
+++ b/tensorflow/python/ops/linalg/matmul_registrations.py
@@ -0,0 +1,252 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Registrations for LinearOperator.matmul."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.ops.linalg import linear_operator_circulant
+from tensorflow.python.ops.linalg import linear_operator_composition
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_identity
+from tensorflow.python.ops.linalg import linear_operator_lower_triangular
+from tensorflow.python.ops.linalg import linear_operator_zeros
+
+
+def _combined_self_adjoint_hint(operator_a, operator_b):
+  """Get combined hint for self-adjoint-ness."""
+  # Note: only use this method in the commuting case.
+  # The property is preserved under composition when the operators commute.
+  if operator_a.is_self_adjoint and operator_b.is_self_adjoint:
+    return True
+
+  # The property is not preserved when an operator with the property is composed
+  # with an operator without the property.
+  if ((operator_a.is_self_adjoint is True and
+       operator_b.is_self_adjoint is False) or
+      (operator_a.is_self_adjoint is False and
+       operator_b.is_self_adjoint is True)):
+    return False
+
+  # The property is not known when operators are not known to have the property
+  # or both operators don't have the property (the property for the complement
+  # class is not closed under composition).
+  return None
+
+
+def _is_square(operator_a, operator_b):
+  """Return a hint to whether the composition is square."""
+  if operator_a.is_square and operator_b.is_square:
+    return True
+  if operator_a.is_square is False and operator_b.is_square is False:
+    # Let A have shape [B, M, N], B have shape [B, N, L].
+    m = operator_a.range_dimension
+    l = operator_b.domain_dimension
+    if m is not None and l is not None:
+      return m == l
+
+    return None
+
+
+def _combined_positive_definite_hint(operator_a, operator_b):
+  """Get combined PD hint for compositions."""
+  # Note: Positive definiteness is only guaranteed to be preserved
+  # when the operators commute and are symmetric. Only use this method in
+  # commuting cases.
+
+  if (operator_a.is_positive_definite is True and
+      operator_a.is_self_adjoint is True and
+      operator_b.is_positive_definite is True and
+      operator_b.is_self_adjoint is True):
+    return True
+
+  return None
+
+
+def _combined_non_singular_hint(operator_a, operator_b):
+  """Get combined hint for when ."""
+  # If either operator is not-invertible the composition isn't.
+  if (operator_a.is_non_singular is False or
+      operator_b.is_non_singular is False):
+    return False
+
+  return operator_a.is_non_singular and operator_b.is_non_singular
+
+
+# By default, use a LinearOperatorComposition to delay the computation.
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator.LinearOperator, linear_operator.LinearOperator)
+def _matmul_linear_operator(linop_a, linop_b):
+  """Generic matmul of two `LinearOperator`s."""
+  is_square = _is_square(linop_a, linop_b)
+  is_non_singular = None
+  is_self_adjoint = None
+  is_positive_definite = None
+
+  if is_square:
+    is_non_singular = _combined_non_singular_hint(linop_a, linop_b)
+    is_self_adjoint = _combined_self_adjoint_hint(linop_a, linop_b)
+  elif is_square is False:
+    is_non_singular = False
+    is_self_adjoint = False
+    is_positive_definite = False
+
+  return linear_operator_composition.LinearOperatorComposition(
+      operators=[linop_a, linop_b],
+      is_non_singular=is_non_singular,
+      is_self_adjoint=is_self_adjoint,
+      is_positive_definite=is_positive_definite,
+      is_square=is_square,
+  )
+
+# Identity
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_identity.LinearOperatorIdentity,
+    linear_operator.LinearOperator)
+def _matmul_linear_operator_identity_left(identity, linop):
+  del identity
+  return linop
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator.LinearOperator,
+    linear_operator_identity.LinearOperatorIdentity)
+def _matmul_linear_operator_identity_right(linop, identity):
+  del identity
+  return linop
+
+
+# Zeros
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator.LinearOperator,
+    linear_operator_zeros.LinearOperatorZeros)
+def _matmul_linear_operator_zeros_right(linop, zeros):
+  if not zeros.is_square or not linop.is_square:
+    raise ValueError("Matmul with non-square `LinearOperator`s or non-square "
+                     "`LinearOperatorZeros` not supported at this time.")
+  return zeros
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_zeros.LinearOperatorZeros,
+    linear_operator.LinearOperator)
+def _matmul_linear_operator_zeros_left(zeros, linop):
+  if not zeros.is_square or not linop.is_square:
+    raise ValueError("Matmul with non-square `LinearOperator`s or non-square "
+                     "`LinearOperatorZeros` not supported at this time.")
+  return zeros
+
+
+# Diag.
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_diag.LinearOperatorDiag,
+    linear_operator_diag.LinearOperatorDiag)
+def _matmul_linear_operator_diag(linop_a, linop_b):
+  return linear_operator_diag.LinearOperatorDiag(
+      diag=linop_a.diag * linop_b.diag,
+      is_non_singular=_combined_non_singular_hint(linop_a, linop_b),
+      is_self_adjoint=_combined_self_adjoint_hint(
+          linop_a, linop_b),
+      is_positive_definite=_combined_positive_definite_hint(
+          linop_a, linop_b),
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_diag.LinearOperatorDiag,
+    linear_operator_identity.LinearOperatorScaledIdentity)
+def _matmul_linear_operator_diag_scaled_identity_right(
+    linop_diag, linop_scaled_identity):
+  return linear_operator_diag.LinearOperatorDiag(
+      diag=linop_diag.diag * linop_scaled_identity.multiplier,
+      is_non_singular=_combined_non_singular_hint(
+          linop_diag, linop_scaled_identity),
+      is_self_adjoint=_combined_self_adjoint_hint(
+          linop_diag, linop_scaled_identity),
+      is_positive_definite=_combined_positive_definite_hint(
+          linop_diag, linop_scaled_identity),
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_identity.LinearOperatorScaledIdentity,
+    linear_operator_diag.LinearOperatorDiag)
+def _matmul_linear_operator_diag_scaled_identity_left(
+    linop_scaled_identity, linop_diag):
+  return linear_operator_diag.LinearOperatorDiag(
+      diag=linop_diag.diag * linop_scaled_identity.multiplier,
+      is_non_singular=_combined_non_singular_hint(
+          linop_diag, linop_scaled_identity),
+      is_self_adjoint=_combined_self_adjoint_hint(
+          linop_diag, linop_scaled_identity),
+      is_positive_definite=_combined_positive_definite_hint(
+          linop_diag, linop_scaled_identity),
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_diag.LinearOperatorDiag,
+    linear_operator_lower_triangular.LinearOperatorLowerTriangular)
+def _matmul_linear_operator_diag_tril(linop_diag, linop_triangular):
+  return linear_operator_lower_triangular.LinearOperatorLowerTriangular(
+      tril=linop_diag.diag[..., None] * linop_triangular.to_dense(),
+      is_non_singular=_combined_non_singular_hint(
+          linop_diag, linop_triangular),
+      # This is safe to do since the Triangular matrix is only self-adjoint
+      # when it is a diagonal matrix, and hence commutes.
+      is_self_adjoint=_combined_self_adjoint_hint(
+          linop_diag, linop_triangular),
+      is_positive_definite=None,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_lower_triangular.LinearOperatorLowerTriangular,
+    linear_operator_diag.LinearOperatorDiag)
+def _matmul_linear_operator_tril_diag(linop_triangular, linop_diag):
+  return linear_operator_lower_triangular.LinearOperatorLowerTriangular(
+      tril=linop_triangular.to_dense() * linop_diag.diag,
+      is_non_singular=_combined_non_singular_hint(
+          linop_diag, linop_triangular),
+      # This is safe to do since the Triangular matrix is only self-adjoint
+      # when it is a diagonal matrix, and hence commutes.
+      is_self_adjoint=_combined_self_adjoint_hint(
+          linop_diag, linop_triangular),
+      is_positive_definite=None,
+      is_square=True)
+
+# Circulant.
+
+
+@linear_operator_algebra.RegisterMatmul(
+    linear_operator_circulant.LinearOperatorCirculant,
+    linear_operator_circulant.LinearOperatorCirculant)
+def _matmul_linear_operator_circulant_circulant(linop_a, linop_b):
+  return linear_operator_circulant.LinearOperatorCirculant(
+      spectrum=linop_a.spectrum * linop_b.spectrum,
+      is_non_singular=_combined_non_singular_hint(linop_a, linop_b),
+      is_self_adjoint=_combined_self_adjoint_hint(linop_a, linop_b),
+      is_positive_definite=_combined_positive_definite_hint(
+          linop_a, linop_b),
+      is_square=True)
-- 
GitLab


From 496499706417ffbbd2a935cbf35116369d9c2e6f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 18:50:09 -0800
Subject: [PATCH 0469/1554] Remove nn.depthwise_conv_2d_native from TF 2.0 API

Users should call nn.depthwise_conv_2d instead.

PiperOrigin-RevId: 221884704
---
 .../api_def/python_api/api_def_DepthwiseConv2dNative.pbtxt    | 2 ++
 tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt            | 4 ----
 tensorflow/tools/compatibility/renames_v2.py                  | 2 ++
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNative.pbtxt b/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNative.pbtxt
index 1bb17e548d..e26d029212 100644
--- a/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNative.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNative.pbtxt
@@ -1,6 +1,8 @@
 op {
   graph_op_name: "DepthwiseConv2dNative"
+  deprecation_message: "Use nn.depthwise_conv2d instead"
   endpoint {
     name: "nn.depthwise_conv2d_native"
+    deprecation_version: 2
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index b252a91f11..a38fda6584 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -108,10 +108,6 @@ tf_module {
     name: "depthwise_conv2d"
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "depthwise_conv2d_native"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
-  }
   member_method {
     name: "depthwise_conv2d_native_backprop_filter"
     argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 09ffd77fab..423ff9cdc3 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -111,6 +111,7 @@ renames = {
     'tf.decode_raw': 'tf.io.decode_raw',
     'tf.delete_session_tensor': 'tf.compat.v1.delete_session_tensor',
     'tf.depth_to_space': 'tf.nn.depth_to_space',
+    'tf.depthwise_conv2d_native': 'tf.compat.v1.depthwise_conv2d_native',
     'tf.dequantize': 'tf.quantization.dequantize',
     'tf.deserialize_many_sparse': 'tf.io.deserialize_many_sparse',
     'tf.diag': 'tf.linalg.tensor_diag',
@@ -294,6 +295,7 @@ renames = {
     'tf.multinomial': 'tf.compat.v1.multinomial',
     'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
+    'tf.nn.depthwise_conv2d_native': 'tf.compat.v1.nn.depthwise_conv2d_native',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
     'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool',
-- 
GitLab


From fc06b8f468859d597c9303a557557dbf4ef108b3 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 16 Nov 2018 20:38:54 -0800
Subject: [PATCH 0470/1554] Change enums to use strings.

PiperOrigin-RevId: 221890462
---
 tensorflow/python/distribute/reduce_util.py | 4 ++--
 tensorflow/python/training/distribute.py    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/distribute/reduce_util.py b/tensorflow/python/distribute/reduce_util.py
index eb027d8b2a..857c7993df 100644
--- a/tensorflow/python/distribute/reduce_util.py
+++ b/tensorflow/python/distribute/reduce_util.py
@@ -35,8 +35,8 @@ class ReduceOp(enum.Enum):
   * `MAX`: Return the maximum of all values.
   """
 
-  SUM = 0
-  MEAN = 1
+  SUM = "SUM"
+  MEAN = "MEAN"
 
   @staticmethod
   def from_variable_aggregation(aggregation):
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index b7869e0330..7d68dd904f 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -208,7 +208,7 @@ class InputReplicationMode(enum.Enum):
   # many input pipelines as number of workers. Replicas will dequeue from the
   # local Dataset on their worker. Distribution Strategy doesn't manage any
   # state sharing between such separate input pipelines.
-  PER_WORKER = 0
+  PER_WORKER = "PER_WORKER"
 
 
 class InputContext(object):
-- 
GitLab


From a562627db116a63607ea42da7a13d6802b4c7bf3 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 16 Nov 2018 22:05:58 -0800
Subject: [PATCH 0471/1554] BEGIN_PUBLIC Move most tflite implementation from
 interpreter.cc to subgraph.cc

This prepares for adding control flow by allowing multiple subgraphs
Automated rollback of commit ff9bd9f31ac4883c3eef31cc74df8bc2d9e9ea8d

PiperOrigin-RevId: 221894816
---
 tensorflow/lite/BUILD                      |   2 +
 tensorflow/lite/core/subgraph.cc           | 976 +++++++++++++++++++++
 tensorflow/lite/core/subgraph.h            | 476 ++++++++++
 tensorflow/lite/interpreter.cc             | 916 +------------------
 tensorflow/lite/interpreter.h              | 289 +-----
 tensorflow/lite/interpreter_test.cc        |   2 +-
 tensorflow/lite/nnapi_delegate.cc          |  87 +-
 tensorflow/lite/nnapi_delegate.h           |   5 +-
 tensorflow/lite/nnapi_delegate_disabled.cc |   6 +-
 tensorflow/lite/util.h                     |   6 +
 10 files changed, 1587 insertions(+), 1178 deletions(-)
 create mode 100644 tensorflow/lite/core/subgraph.cc
 create mode 100644 tensorflow/lite/core/subgraph.h

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index be84fc5db1..bb2c53b8c9 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -131,6 +131,7 @@ cc_library(
     name = "framework",
     srcs = [
         "allocation.cc",
+        "core/subgraph.cc",
         "graph_info.cc",
         "interpreter.cc",
         "model.cc",
@@ -155,6 +156,7 @@ cc_library(
         "allocation.h",
         "context.h",
         "context_util.h",
+        "core/subgraph.h",
         "error_reporter.h",
         "graph_info.h",
         "interpreter.h",
diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
new file mode 100644
index 0000000000..05a60962b1
--- /dev/null
+++ b/tensorflow/lite/core/subgraph.cc
@@ -0,0 +1,976 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/core/subgraph.h"
+#include "tensorflow/lite/arena_planner.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/context_util.h"
+#include "tensorflow/lite/graph_info.h"
+#include "tensorflow/lite/nnapi_delegate.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+
+namespace {
+TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
+                           const TfLiteRegistration& registration,
+                           int node_index, const char* message) {
+  context->ReportError(
+      context, "Node number %d (%s) %s.\n", node_index,
+      registration.custom_name
+          ? registration.custom_name
+          : EnumNameBuiltinOperator(
+                static_cast<BuiltinOperator>(registration.builtin_code)),
+      message);
+  return kTfLiteError;
+}
+
+// Stub method which returns kTfLiteError when the function is forbidden.
+// We're registrating this function to several different function to save
+// compiled binary size. Please note the restrictions:
+// * The type of first parameter have to be `TfLiteContext*`.
+// * All paramteters must be trivailly destructible. (E.g. No C++ class)
+TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) {
+  context->ReportError(context,
+                       "The function is forbidden if not calling in delegate.");
+  return kTfLiteError;
+}
+
+// Set the ForbiddenContextFunction to a compatible function pointer.
+template <typename FunctionType>
+void SetForbiddenContextFunction(FunctionType* func) {
+  *func = reinterpret_cast<FunctionType>(ForbiddenContextFunction);
+}
+
+// Returns true if at least one tensor in the given list is kTfLiteDynamic.
+template <typename TensorIntArray>
+bool HasDynamicTensorImpl(const TfLiteContext& context,
+                          const TensorIntArray& int_array) {
+  for (int i : int_array) {
+    const TfLiteTensor& tensor = context.tensors[i];
+    if (tensor.allocation_type == kTfLiteDynamic) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool HasDynamicTensor(const TfLiteContext& context,
+                      const TfLiteIntArray* int_array) {
+  return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
+}
+
+}  // namespace
+
+// A trivial implementation of GraphInfo around the Interpreter.
+// NOTE: this interpreter info represents the subset of the
+// graph that is executed according to execution plan. Thus,
+// the indices are execution plan indices rather than raw node
+// indices.
+class InterpreterInfo : public GraphInfo {
+ public:
+  explicit InterpreterInfo(Subgraph* subgraph) : subgraph_(subgraph) {}
+
+  size_t num_tensors() const override { return subgraph_->tensors().size(); }
+  TfLiteTensor* tensor(size_t index) override {
+    return &subgraph_->tensors()[index];
+  }
+  size_t num_nodes() const override {
+    return subgraph_->execution_plan().size();
+  }
+  const TfLiteNode& node(size_t index) const override {
+    int node_index = subgraph_->execution_plan()[index];
+    return subgraph_->nodes_and_registration()[node_index].first;
+  }
+  const std::vector<int>& inputs() const override {
+    return subgraph_->inputs();
+  }
+  const std::vector<int>& outputs() const override {
+    return subgraph_->outputs();
+  }
+  const std::vector<int>& variables() const override {
+    return subgraph_->variables();
+  }
+
+ public:
+  Subgraph* subgraph_;
+};
+
+Subgraph::Subgraph(ErrorReporter* error_reporter,
+                   TfLiteExternalContext** external_contexts)
+    : context_(&owned_context_),
+      error_reporter_(error_reporter),
+      next_execution_plan_index_to_prepare_(0),
+      external_contexts_(external_contexts) {
+  context_->impl_ = static_cast<void*>(this);
+  context_->ResizeTensor = ResizeTensor;
+  context_->ReportError = ReportErrorC;
+  context_->AddTensors = AddTensors;
+  context_->tensors = nullptr;
+  context_->tensors_size = 0;
+  context_->allow_fp32_relax_to_fp16 = false;
+  context_->recommended_num_threads = -1;
+  context_->GetExternalContext = GetExternalContext;
+  context_->SetExternalContext = SetExternalContext;
+
+  // Reserve some space for the tensors to avoid excessive resizing.
+  tensors_.reserve(kTensorsReservedCapacity);
+  nodes_and_registration().reserve(kTensorsReservedCapacity);
+  // Invalid to call these these except from TfLiteDelegate
+  SwitchToKernelContext();
+}
+
+Subgraph::~Subgraph() {
+  for (auto& node_and_reg : nodes_and_registration_) {
+    TfLiteNode& node = node_and_reg.first;
+    TfLiteIntArrayFree(node.inputs);
+    TfLiteIntArrayFree(node.outputs);
+    TfLiteIntArrayFree(node.temporaries);
+    if (node.builtin_data) free(node.builtin_data);
+    OpFree(node_and_reg.second, node.user_data);
+    node.builtin_data = nullptr;
+  }
+
+  for (size_t i = 0; i < context_->tensors_size; i++) {
+    TfLiteTensor* tensor = &context_->tensors[i];
+    if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
+        tensor->delegate->FreeBufferHandle != nullptr) {
+      tensor->delegate->FreeBufferHandle(context_, tensor->delegate,
+                                         &tensor->buffer_handle);
+    }
+    TfLiteTensorFree(tensor);
+  }
+}
+
+TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels(
+    TfLiteContext* context, TfLiteRegistration registration,
+    const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
+  return static_cast<Subgraph*>(context->impl_)
+      ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace,
+                                              delegate);
+}
+
+namespace {
+
+// Copy a std::vector<int> to an existing TfLiteIntArray.
+// This is a low-level data manipulation function, and it's caller's
+// responsibility to ensure TfLiteIntArray has enough size.
+void CopyVectorToTfLiteIntArray(const std::vector<int>& vec,
+                                TfLiteIntArray* arr) {
+  arr->size = vec.size();
+  memcpy(arr->data, vec.data(), sizeof(int) * arr->size);
+}
+
+// This function allocates a continuous memory space that contains a
+// TfLiteDelegateParams followed by a several TfLiteIntArray.
+// When calling `free` at TfLiteDelegateParams*, all the allocated space
+// will be freed together.
+//
+// +-----------------------------------+
+// | TfLiteDelegateParams              |
+// | TfLiteDelegate* delegate;         |
+// | TfLiteIntArray* nodes_to_replace; |--\
+// | TfLiteIntArray* input_tensors;    |--+--\
+// | TfLiteIntArray* output_tensors;   |--+--+--\
+// +-----------------------------------+  |  |  |
+// | TfLiteIntArray (variable size)    |<-/  |  |
+// +-----------------------------------+     |  |
+// | TfLiteIntArray (variable size)    |<----/  |
+// +-----------------------------------+        |
+// | TfLiteIntArray (variable size)    |<-------/
+// +-----------------------------------+
+TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate,
+                                           const NodeSubset& node_subset) {
+  // Step 1: Calculate the allocation size.
+  int allocation_size = sizeof(TfLiteDelegateParams);
+
+  int nodes_to_replace_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size());
+  allocation_size += nodes_to_replace_size;
+
+  int input_tensors_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size());
+  allocation_size += input_tensors_size;
+
+  int output_tensors_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size());
+  allocation_size += output_tensors_size;
+
+  // Step 2: Allocate the memory.
+  // Use `char*` for conveniently step through the allocated space by bytes.
+  char* allocation = reinterpret_cast<char*>(malloc(allocation_size));
+
+  // Step 3: Fill all data structures structures.
+  TfLiteDelegateParams* params =
+      reinterpret_cast<TfLiteDelegateParams*>(allocation);
+  params->delegate = delegate;
+  allocation += sizeof(TfLiteDelegateParams);
+
+  params->nodes_to_replace = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace);
+  allocation += nodes_to_replace_size;
+
+  params->input_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors);
+  allocation += input_tensors_size;
+
+  params->output_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.output_tensors,
+                             params->output_tensors);
+  allocation += output_tensors_size;
+
+  return params;
+}
+
+}  // namespace
+
+TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels(
+    TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
+    TfLiteDelegate* delegate) {
+  // Annotate the registration as DELEGATE op.
+  registration.builtin_code = BuiltinOperator_DELEGATE;
+
+  // Analyze the graph to find all independent node_subsets that are either
+  // fully not-this-delegate or this-delegate computation.
+  InterpreterInfo info(this);
+  std::vector<NodeSubset> node_subsets;
+  PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace,
+                                           &node_subsets);
+
+  execution_plan_.clear();
+
+  for (auto& node_subset : node_subsets) {
+    // Subsets calimed by the delegate should have a "macro" op created, the
+    // other node_subsets (kTfNonPartition) just have their nodes added back to
+    // the execution plan.
+    switch (node_subset.type) {
+      case NodeSubset::kTfNonPartition:
+        for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end();
+             ++it) {
+          execution_plan_.push_back(*it);
+        }
+        break;
+      case NodeSubset::kTfPartition: {
+        int node_index;
+
+        TfLiteDelegateParams* params =
+            CreateDelegateParams(delegate, node_subset);
+        TF_LITE_ENSURE_STATUS(AddNodeWithParameters(
+            node_subset.input_tensors, node_subset.output_tensors, nullptr, 0,
+            params, &registration, &node_index));
+
+        // Initialize the output tensors's delegate-related fields.
+        for (int tensor_index : node_subset.output_tensors) {
+          TfLiteTensor* tensor = &tensors_[tensor_index];
+          TF_LITE_ENSURE(context_, tensor->delegate == nullptr ||
+                                       tensor->delegate == delegate);
+          tensor->delegate = delegate;
+        }
+
+        // Associate the node with the delegate.
+        TfLiteNode* node = &nodes_and_registration_[node_index].first;
+        node->delegate = delegate;
+      } break;
+      case NodeSubset::kTfUnexplored:
+        return kTfLiteError;
+        break;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteExternalContext* Subgraph::GetExternalContext(
+    TfLiteExternalContextType type) {
+  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
+    return external_contexts_[type];
+  }
+  return nullptr;
+}
+
+TfLiteExternalContext* Subgraph::GetExternalContext(
+    struct TfLiteContext* context, TfLiteExternalContextType type) {
+  return static_cast<Subgraph*>(context->impl_)->GetExternalContext(type);
+}
+
+void Subgraph::SetExternalContext(TfLiteExternalContextType type,
+                                  TfLiteExternalContext* ctx) {
+  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
+    external_contexts_[type] = ctx;
+  }
+}
+
+void Subgraph::SetExternalContext(struct TfLiteContext* context,
+                                  TfLiteExternalContextType type,
+                                  TfLiteExternalContext* ctx) {
+  return static_cast<Subgraph*>(context->impl_)->SetExternalContext(type, ctx);
+}
+
+// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns
+// this memory and it is only guaranteed to exist during the invocation of the
+// delegate prepare.
+TfLiteStatus Subgraph::GetExecutionPlan(TfLiteIntArray** execution_plan) {
+  // TODO(aselle): Do not make a copy here
+  plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size()));
+  *execution_plan = plan_cache_.get();
+  static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]),
+                "TfLiteIntArray and execution_plan do not contain same type.");
+  std::memcpy(plan_cache_->data, execution_plan_.data(),
+              sizeof(plan_cache_->data[0]) * execution_plan_.size());
+  return kTfLiteOk;
+}
+
+// WARNING: This is an experimental interface that is subject to change.
+// Entry point for C node plugin API to get the execution plan
+TfLiteStatus Subgraph::GetExecutionPlan(struct TfLiteContext* context,
+                                        TfLiteIntArray** execution_plan) {
+  return static_cast<Subgraph*>(context->impl_)
+      ->GetExecutionPlan(execution_plan);
+}
+
+TfLiteStatus Subgraph::SetInputs(std::vector<int> inputs) {
+  TF_LITE_ENSURE_OK(&context_,
+                    CheckTensorIndices("inputs", inputs.data(), inputs.size()));
+  inputs_ = std::move(inputs);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::SetOutputs(std::vector<int> outputs) {
+  TF_LITE_ENSURE_OK(
+      &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
+  outputs_ = std::move(outputs);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::SetVariables(std::vector<int> variables) {
+  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
+                                                  variables.size()));
+  variables_ = std::move(variables);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices,
+                                          int length) {
+  // Making sure kOptionalTensor is not re-defined to something other than -1.
+  static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1");
+
+  for (int i = 0; i < length; i++) {
+    int index = indices[i];
+    // Continue if index == kOptionalTensor before additional comparisons below,
+    // size_t(-1) is always >= context_tensors_size.
+    if (index == kOptionalTensor) {
+      continue;
+    }
+    if (index < 0 || static_cast<size_t>(index) >= context_->tensors_size) {
+      ReportError("Invalid tensor index %d in %s\n", index, label);
+      consistent_ = false;
+      return kTfLiteError;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims,
+                                     size_t dims_size, size_t* bytes) {
+  // TODO(aselle): Check for overflow here using overflow.h in TensorFlow
+  // MultiplyWithoutOverflow.
+  TF_LITE_ENSURE(context_, bytes != nullptr);
+  size_t count = 1;
+  for (int k = 0; k < dims_size; k++) count *= dims[k];
+  switch (type) {
+    case kTfLiteFloat32:
+      *bytes = sizeof(float) * count;
+      break;
+    case kTfLiteInt16:
+      *bytes = sizeof(int16_t) * count;
+      break;
+    case kTfLiteInt32:
+      *bytes = sizeof(int32_t) * count;
+      break;
+    case kTfLiteUInt8:
+      *bytes = sizeof(uint8_t) * count;
+      break;
+    case kTfLiteInt64:
+      *bytes = sizeof(int64_t) * count;
+      break;
+    case kTfLiteBool:
+      *bytes = sizeof(bool) * count;
+      break;
+    case kTfLiteComplex64:
+      *bytes = sizeof(std::complex<float>) * count;
+      break;
+    case kTfLiteInt8:
+      *bytes = sizeof(int8_t) * count;
+      break;
+    default:
+      ReportError(
+          "Only float32, int8, int16, int32, int64, uint8, bool, complex64 "
+          "supported currently.");
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::AllocateTensors() {
+  if (!consistent_) {
+    ReportError("AllocateTensors() called on inconsistent model.");
+    return kTfLiteError;
+  }
+
+  // Explicit (re)allocation is necessary if nodes have been changed or tensors
+  // have been resized. For inputs marked as dynamic, we can't short-circuit the
+  // allocation as the client may have done the resize manually.
+  if (state_ != kStateUninvokable &&
+      !HasDynamicTensorImpl(*context_, inputs())) {
+    return kTfLiteOk;
+  }
+
+  next_execution_plan_index_to_prepare_ = 0;
+  if (memory_planner_) {
+    TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations());
+  }
+
+  TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
+
+  state_ = kStateInvokable;
+
+  // Reset the variable tensors to zero after (re)allocating the tensors.
+  // Developers shouldn't rely on the side effect of this function to reset
+  // variable tesnsors. They should call `ResetVariableTensors` directly
+  // instead.
+  ResetVariableTensors();
+
+  return kTfLiteOk;
+}
+
+// TODO(ycling): Support non-zero default values.
+TfLiteStatus Subgraph::ResetVariableTensors() {
+  for (auto& tensor : tensors_) {
+    if (!tensor.is_variable) {
+      continue;
+    }
+
+    // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be
+    // allocated after the initial `PrepareOpsAndTensors()` is called.
+    TF_LITE_ENSURE_EQ(context_, tensor.allocation_type,
+                      kTfLiteArenaRwPersistent);
+    TF_LITE_ENSURE(context_, tensor.data.raw != nullptr);
+
+    memset(tensor.data.raw, 0, tensor.bytes);
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::AddNodeWithParameters(
+    const std::vector<int>& inputs, const std::vector<int>& outputs,
+    const char* init_data, size_t init_data_size, void* builtin_data,
+    const TfLiteRegistration* registration, int* node_index) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError("AddNodeWithParameters is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  state_ = kStateUninvokable;
+
+  std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
+                                                              free);
+
+  TF_LITE_ENSURE_OK(context_, CheckTensorIndices("node inputs", inputs.data(),
+                                                 inputs.size()));
+  TF_LITE_ENSURE_OK(
+      &context_,
+      CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
+
+  int new_node_index = nodes_and_registration_.size();
+  if (node_index) *node_index = new_node_index;
+  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
+  auto& node_and_reg = nodes_and_registration_.back();
+  TfLiteNode& node = node_and_reg.first;
+  if (node.inputs) TfLiteIntArrayFree(node.inputs);
+  if (node.outputs) TfLiteIntArrayFree(node.outputs);
+  if (node.temporaries) TfLiteIntArrayFree(node.temporaries);
+
+  // NOTE, here we are not using move semantics yet, since our internal
+  // representation isn't std::vector, but in the future we would like to avoid
+  // copies, so we want the interface to take r-value references now.
+  node.inputs = ConvertVectorToTfLiteIntArray(inputs);
+  node.outputs = ConvertVectorToTfLiteIntArray(outputs);
+  node.temporaries = TfLiteIntArrayCreate(0);
+  if (init_data) {
+    node.user_data = OpInit(*registration, init_data, init_data_size);
+  } else {
+    node.user_data =
+        OpInit(*registration,
+               reinterpret_cast<const char*>(builtin_data_deleter.get()), 0);
+  }
+
+  node.builtin_data = builtin_data_deleter.release();
+  // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size`
+  // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels.
+
+  if (registration->builtin_code == BuiltinOperator_CUSTOM) {
+    // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer
+    // `Operator` table is passed in.
+    node.custom_initial_data = init_data;
+    node.custom_initial_data_size = init_data_size;
+  } else {
+    node.custom_initial_data = nullptr;
+    node.custom_initial_data_size = 0;
+  }
+
+  node.delegate = nullptr;
+  node_and_reg.second = *registration;
+  execution_plan_.push_back(new_node_index);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::ResizeInputTensor(int tensor_index,
+                                         const std::vector<int>& dims) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError("ResizeInputTensor is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
+  // TODO(aselle): All bounds checks can be implemented as one-sided bounds
+  // checks by casting to unsigned for efficiency. Profile before doing this.
+  TF_LITE_ENSURE(context_,
+                 tensor_index < context_->tensors_size && tensor_index >= 0);
+  TfLiteTensor* tensor = &context_->tensors[tensor_index];
+
+  // Short-circuit the state change if the dimensions don't change, avoiding
+  // unnecessary (re)allocations.
+  if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) {
+    return kTfLiteOk;
+  }
+
+  state_ = kStateUninvokable;
+  return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims));
+}
+
+TfLiteStatus Subgraph::PrepareOpsStartingAt(
+    int first_execution_plan_index, int* last_execution_plan_index_prepared) {
+  for (int execution_plan_index = first_execution_plan_index;
+       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
+    int node_index = execution_plan_[execution_plan_index];
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    const TfLiteRegistration& registration =
+        nodes_and_registration_[node_index].second;
+    EnsureTensorsVectorCapacity();
+    if (OpPrepare(registration, &node) == kTfLiteError) {
+      return ReportOpError(context_, node, registration, node_index,
+                           "failed to prepare");
+    }
+
+    *last_execution_plan_index_prepared = execution_plan_index;
+
+    // Discontinue if the node has dynamic outputs. Note that we don't
+    // stop for dynamic temporary tensors since they won't affect the
+    // sizes of other tensors in the graph.
+    if (HasDynamicTensor(*context_, node.outputs)) {
+      break;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::PrepareOpsAndTensors() {
+  if (!memory_planner_) {
+    memory_planner_.reset(new ArenaPlanner(
+        context_, std::unique_ptr<GraphInfo>(new InterpreterInfo(this)),
+        /*preserve_inputs=*/true, /*preserve_intermediates*/ false));
+    memory_planner_->PlanAllocations();
+  }
+
+  int last_exec_plan_index_prepared = 0;
+
+  TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt(
+      next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared));
+  TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations(
+      next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared));
+
+  next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::Invoke() {
+  if (!consistent_) {
+    ReportError("Invoke called on model that is not consistent.");
+    return kTfLiteError;
+  }
+
+  TfLiteStatus status = kTfLiteOk;
+  if (state_ == kStateUninvokable) {
+    ReportError("Invoke called on model that is not ready.");
+    return kTfLiteError;
+  }
+
+  if (nnapi_delegate_) {
+    if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) {
+      TF_LITE_ENSURE_OK(context_, nnapi_delegate_->Invoke(this));
+      return kTfLiteOk;
+    } else {
+      // TODO(aselle): In the future, we would like this to be an
+      // automatic tflite CPU fallback.
+      ReportError(
+          "NNAPI was requested, but dependent sized tensors "
+          "being used.\n");
+      return kTfLiteError;
+    }
+  }
+
+  // Invocations are always done in node order.
+  // Note that calling Invoke repeatedly will cause the original memory plan to
+  // be reused, unless either ResizeInputTensor() or AllocateTensors() has been
+  // called.
+  for (int execution_plan_index = 0;
+       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
+    if (execution_plan_index == next_execution_plan_index_to_prepare_) {
+      TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
+      TF_LITE_ENSURE(context_, next_execution_plan_index_to_prepare_ >=
+                                   execution_plan_index);
+    }
+    int node_index = execution_plan_[execution_plan_index];
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    const TfLiteRegistration& registration =
+        nodes_and_registration_[node_index].second;
+    SCOPED_OPERATOR_PROFILE(profiler_, node_index);
+
+    // TODO(ycling): This is an extra loop through inputs to check if the data
+    // need to be copied from Delegate buffer to raw memory, which is often not
+    // needed. We may want to cache this in prepare to know if this needs to be
+    // done for a node or not.
+    for (int i = 0; i < node.inputs->size; ++i) {
+      int tensor_index = node.inputs->data[i];
+      if (tensor_index == kOptionalTensor) {
+        continue;
+      }
+      TfLiteTensor* tensor = &tensors_[tensor_index];
+      if (tensor->delegate && tensor->delegate != node.delegate &&
+          tensor->data_is_stale) {
+        EnsureTensorDataIsReadable(tensor_index);
+      }
+    }
+
+    EnsureTensorsVectorCapacity();
+    tensor_resized_since_op_invoke_ = false;
+    if (OpInvoke(registration, &node) == kTfLiteError) {
+      status = ReportOpError(context_, node, registration, node_index,
+                             "failed to invoke");
+    }
+
+    // Force execution prep for downstream ops if the latest op triggered the
+    // resize of a dynamic tensor.
+    if (tensor_resized_since_op_invoke_ &&
+        HasDynamicTensor(*context_, node.outputs)) {
+      next_execution_plan_index_to_prepare_ = execution_plan_index + 1;
+    }
+  }
+
+  return status;
+}
+
+TfLiteStatus Subgraph::ResizeTensor(TfLiteContext* context,
+                                    TfLiteTensor* tensor,
+                                    TfLiteIntArray* new_size) {
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function ResizeTensorImpl
+  // (this function is static).
+  return static_cast<Subgraph*>(context->impl_)
+      ->ResizeTensorImpl(tensor, new_size);
+}
+
+void Subgraph::ReportErrorImpl(const char* format, va_list args) {
+  error_reporter_->Report(format, args);
+}
+
+void Subgraph::ReportErrorC(TfLiteContext* context, const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  auto* f = static_cast<Subgraph*>(context->impl_);
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Subgraph to call into the member function ReportErrorImpl
+  // (this function is static).
+  f->ReportErrorImpl(format, args);
+  va_end(args);
+}
+
+// Entry point for C node plugin API to report an error.
+void Subgraph::ReportError(const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  auto* f = static_cast<Subgraph*>(context_->impl_);
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Subgraph to call into the member function ReportErrorImpl
+  // (this function is static).
+  f->ReportErrorImpl(format, args);
+  va_end(args);
+}
+
+TfLiteStatus Subgraph::AddTensors(int tensors_to_add,
+                                  int* first_new_tensor_index) {
+  const size_t base_index = tensors_.size();
+  if (first_new_tensor_index) *first_new_tensor_index = base_index;
+  tensors_.resize(tensors_.size() + tensors_to_add);
+  for (size_t i = base_index; i < tensors_.size(); i++) {
+    memset(&tensors_[i], 0, sizeof(tensors_[i]));
+    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
+  }
+  context_->tensors = tensors_.data();
+  context_->tensors_size = tensors_.size();
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::AddTensors(TfLiteContext* context, int tensors_to_add,
+                                  int* first_new_tensor_index) {
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function AddTensors
+  // (this function is static).
+  return static_cast<Subgraph*>(context->impl_)
+      ->AddTensors(tensors_to_add, first_new_tensor_index);
+}
+
+TfLiteStatus Subgraph::GetNodeAndRegistration(
+    int node_index, TfLiteNode** node, TfLiteRegistration** registration) {
+  TF_LITE_ENSURE(context_, node_index >= 0);
+  auto nodes_size = nodes_and_registration_.size();
+  TF_LITE_ENSURE(context_, static_cast<size_t>(node_index) < nodes_size);
+  TF_LITE_ENSURE(context_, node != nullptr && registration != nullptr);
+  auto& node_and_reg = nodes_and_registration_[node_index];
+  *node = &node_and_reg.first;
+  *registration = &node_and_reg.second;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::GetNodeAndRegistration(
+    struct TfLiteContext* context, int node_index, TfLiteNode** node,
+    TfLiteRegistration** registration) {
+  return static_cast<Subgraph*>(context->impl_)
+      ->GetNodeAndRegistration(node_index, node, registration);
+}
+
+TfLiteStatus Subgraph::SetTensorParametersReadOnly(
+    int tensor_index, TfLiteType type, const char* name, const size_t rank,
+    const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
+    size_t bytes, const Allocation* allocation) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
+  TF_LITE_ENSURE(context_,
+                 tensor_index < context_->tensors_size && tensor_index >= 0);
+  // For most tensors we know exactly how much memory is necessary so we can
+  // ensure the buffer is large enough. However, we need to skip string tensors
+  // because their sizes change with the contents of the individual strings.
+  if (type != kTfLiteString) {
+    size_t required_bytes;
+    TF_LITE_ENSURE_OK(context_,
+                      BytesRequired(type, dims, rank, &required_bytes));
+    TF_LITE_ENSURE_EQ(context_, required_bytes, bytes);
+  }
+
+  TfLiteTensor& tensor = context_->tensors[tensor_index];
+  if (type == tensor.type &&
+      EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
+    // Fast path which does not invalidate the invokable property.
+    TfLiteTensorDataFree(&tensor);
+    tensor.data.raw = const_cast<char*>(buffer);
+    if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
+    tensor.params = quantization;
+    tensor.allocation_type = kTfLiteMmapRo;
+    tensor.allocation = allocation;
+  } else {
+    state_ = kStateUninvokable;
+    TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
+                      quantization, const_cast<char*>(buffer), bytes,
+                      kTfLiteMmapRo, allocation, false, &tensor);
+  }
+  return kTfLiteOk;
+}
+
+// Set description of inputs/outputs/data/fptrs for node `node_index`.
+// This variant assumes an external buffer has been allocated of size
+// bytes. The lifetime of buffer must be ensured to be greater or equal
+// to Interpreter.
+TfLiteStatus Subgraph::SetTensorParametersReadWrite(
+    int tensor_index, TfLiteType type, const char* name, const size_t rank,
+    const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  TF_LITE_ENSURE(context_,
+                 tensor_index < context_->tensors_size && tensor_index >= 0);
+  size_t required_bytes = 0;
+  if (type != kTfLiteString) {
+    // These types will be allocated in our arena so we need to record how
+    // many bytes we will need based on the dimensions. String tensors are
+    // allocated dynamically and we can't know ahead of time how much space
+    // they will require.
+    TF_LITE_ENSURE_OK(context_,
+                      BytesRequired(type, dims, rank, &required_bytes));
+  }
+
+  TfLiteAllocationType allocation_type = kTfLiteArenaRw;
+  if (type == kTfLiteString) {
+    if (is_variable) {
+      // We don't have a real use case for string variable tensor.
+      ReportError("String variable tensor isn't supported.");
+      return kTfLiteError;
+    }
+    allocation_type = kTfLiteDynamic;
+  } else if (is_variable) {
+    allocation_type = kTfLiteArenaRwPersistent;
+  }
+
+  TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
+                    quantization,
+                    /*buffer=*/nullptr, required_bytes, allocation_type,
+                    nullptr, is_variable, &context_->tensors[tensor_index]);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::SetExecutionPlan(const std::vector<int>& new_plan) {
+  for (int node_index : new_plan) {
+    TF_LITE_ENSURE(context_, node_index >= 0 &&
+                                 node_index < nodes_and_registration_.size());
+  }
+  execution_plan_ = new_plan;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::ResizeTensorImpl(TfLiteTensor* tensor,
+                                        TfLiteIntArray* new_size) {
+  // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too.
+  if (tensor->allocation_type == kTfLiteArenaRw ||
+      tensor->allocation_type == kTfLiteDynamic ||
+      tensor->allocation_type == kTfLiteArenaRwPersistent) {
+    tensor_resized_since_op_invoke_ |=
+        TfLiteIntArrayEqual(tensor->dims, new_size) == 0;
+    if (tensor->type != kTfLiteString) {
+      size_t bytesRequired;
+      TfLiteStatus status = BytesRequired(tensor->type, new_size->data,
+                                          new_size->size, &bytesRequired);
+      if (status != kTfLiteOk) {
+        TfLiteIntArrayFree(new_size);
+        return kTfLiteError;
+      }
+
+      // Realloc space for kTfLiteDynamic tensors.
+      TfLiteTensorRealloc(bytesRequired, tensor);
+      tensor->bytes = bytesRequired;
+    }
+    if (tensor->dims) TfLiteIntArrayFree(tensor->dims);
+    tensor->dims = new_size;
+
+    if (tensor->allocation_type != kTfLiteDynamic) {
+      tensor->data.raw = nullptr;
+    }
+  } else {
+    // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore
+    // of fixed size.
+    TfLiteIntArrayFree(new_size);
+    ReportError("Attempting to resize a fixed-size tensor.");
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+void Subgraph::UseNNAPI(bool enable) {
+  // TODO(aselle): This is a workaround for finding if NNAPI exists.
+  // We also need to make sure getLibraryHandle() is renamed to be NNAPI
+  // prefixed.
+  if (!NNAPIDelegate::IsSupported()) enable = false;
+  if (!enable) {
+    nnapi_delegate_.reset();
+  } else if (!nnapi_delegate_) {
+    nnapi_delegate_.reset(new NNAPIDelegate);
+  }
+}
+
+void Subgraph::SwitchToDelegateContext() {
+  context_->GetNodeAndRegistration = GetNodeAndRegistration;
+  context_->ReplaceNodeSubsetsWithDelegateKernels =
+      ReplaceNodeSubsetsWithDelegateKernels;
+  context_->GetExecutionPlan = GetExecutionPlan;
+}
+
+void Subgraph::SwitchToKernelContext() {
+  context_->GetNodeAndRegistration = [](struct TfLiteContext* context,
+                                        int node_index, TfLiteNode** node,
+                                        TfLiteRegistration** registration) {
+    return ForbiddenContextFunction(context);
+  };
+  context_->ReplaceNodeSubsetsWithDelegateKernels =
+      [](TfLiteContext* context, TfLiteRegistration registration,
+         const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
+        return ForbiddenContextFunction(context);
+      };
+  context_->GetExecutionPlan = [](struct TfLiteContext* context,
+                                  TfLiteIntArray**) {
+    return ForbiddenContextFunction(context);
+  };
+}
+
+TfLiteStatus Subgraph::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
+  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
+    int last_execution_plan_index_prepared;
+    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
+                                     0, &last_execution_plan_index_prepared));
+
+    bool has_dynamic_tensors = true;
+    // Dynamic tensors exist if not all nodes can be prepared.
+    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
+      // If all the nodes can be prepared, check if the last node has dynamic
+      // tensors.
+      int node_index = execution_plan_[last_execution_plan_index_prepared];
+      TfLiteNode& node = nodes_and_registration_[node_index].first;
+      if (!HasDynamicTensor(*context_, node.outputs)) {
+        has_dynamic_tensors = false;
+      }
+    }
+    if (has_dynamic_tensors) {
+      ReportError(
+          "Attempting to use a delegate that only supports static-sized "
+          "tensors with a graph that has dynamic-sized tensors.");
+      return kTfLiteError;
+    }
+  }
+
+  // TODO(aselle): Consider if it is worth storing pointers to delegates.
+  // Setup additional context interface.
+  SwitchToDelegateContext();
+
+  TfLiteStatus status = delegate->Prepare(context_, delegate);
+
+  // Remove additional context info.
+  SwitchToKernelContext();
+
+  TF_LITE_ENSURE_OK(context_, status);
+
+  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
+    // Reset the state to force tensor/op reallocation.
+    state_ = kStateUninvokable;
+    TF_LITE_ENSURE_OK(context_, AllocateTensors());
+    TF_LITE_ENSURE_EQ(context_, state_, kStateInvokable);
+    // After using a delegate which doesn't support dynamic tensors, make the
+    // entire graph immutable.
+    state_ = kStateInvokableAndImmutable;
+  }
+
+  return status;
+}
+
+}  // namespace tflite
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
new file mode 100644
index 0000000000..9783747ff6
--- /dev/null
+++ b/tensorflow/lite/core/subgraph.h
@@ -0,0 +1,476 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_CORE_SUBGRAPH_H_
+#define TENSORFLOW_LITE_CORE_SUBGRAPH_H_
+
+#include <cstdlib>
+#include <vector>
+
+#include "tensorflow/lite/allocation.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/memory_planner.h"
+#include "tensorflow/lite/util.h"
+
+namespace tflite {
+
+// Forward declare since NNAPIDelegate uses Interpreter.
+class NNAPIDelegate;
+
+class Subgraph {
+ public:
+  friend class Interpreter;
+
+  Subgraph(ErrorReporter* error_reporter,
+           TfLiteExternalContext** external_contexts);
+  Subgraph(const Subgraph&) = delete;
+
+  // Subgraphs should be movable but not copyable.
+  Subgraph(Subgraph&&) = default;
+  Subgraph& operator=(const Subgraph&) = delete;
+  virtual ~Subgraph();
+
+  // Provide a list of tensor indexes that are inputs to the model.
+  // Each index is bound check and this modifies the consistent_ flag of the
+  // interpreter.
+  TfLiteStatus SetInputs(std::vector<int> inputs);
+
+  // Provide a list of tensor indexes that are outputs to the model
+  // Each index is bound check and this modifies the consistent_ flag of the
+  // interpreter.
+  TfLiteStatus SetOutputs(std::vector<int> outputs);
+
+  // Provide a list of tensor indexes that are variable tensors.
+  // Each index is bound check and this modifies the consistent_ flag of the
+  // interpreter.
+  TfLiteStatus SetVariables(std::vector<int> variables);
+
+
+  // Adds a node with the given parameters and returns the index of the new
+  // node in `node_index` (optionally). Interpreter will take ownership of
+  // `builtin_data` and destroy it with `free`. Ownership of 'init_data'
+  // remains with the caller.
+  TfLiteStatus AddNodeWithParameters(const std::vector<int>& inputs,
+                                     const std::vector<int>& outputs,
+                                     const char* init_data,
+                                     size_t init_data_size, void* builtin_data,
+                                     const TfLiteRegistration* registration,
+                                     int* node_index);
+
+  // Adds `tensors_to_add` tensors, preserving pre-existing Tensor entries.
+  // The value pointed to by `first_new_tensor_index` will be set to the
+  // index of the first new tensor if `first_new_tensor_index` is non-null.
+  TfLiteStatus AddTensors(int tensors_to_add, int* first_new_tensor_index);
+
+  // Set description of inputs/outputs/data/fptrs for node `node_index`.
+  // This variant assumes an external buffer has been allocated of size
+  // bytes. The lifetime of buffer must be ensured to be greater or equal
+  // to Interpreter.
+  TfLiteStatus SetTensorParametersReadOnly(
+      int tensor_index, TfLiteType type, const char* name, const size_t rank,
+      const int* dims, TfLiteQuantizationParams quantization,
+      const char* buffer, size_t bytes, const Allocation* allocation);
+
+  // Set description of inputs/outputs/data/fptrs for node `node_index`.
+  // This variant assumes an external buffer has been allocated of size
+  // bytes. The lifetime of buffer must be ensured to be greater or equal
+  // to Interpreter.
+  TfLiteStatus SetTensorParametersReadWrite(
+      int tensor_index, TfLiteType type, const char* name, const size_t rank,
+      const int* dims, TfLiteQuantizationParams quantization, bool is_variable);
+
+  // WARNING: Experimental interface, subject to change
+  // Overrides execution plan. This bounds checks indices sent in.
+  TfLiteStatus SetExecutionPlan(const std::vector<int>& new_plan);
+
+  // Get a mutable tensor data structure.
+  // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this
+  // read/write access to structure
+  TfLiteTensor* tensor(int tensor_index) {
+    if (tensor_index < 0 ||
+        static_cast<size_t>(tensor_index) >= context_->tensors_size) {
+      return nullptr;
+    }
+    return &context_->tensors[tensor_index];
+  }
+
+  // Get an immutable tensor data structure.
+  const TfLiteTensor* tensor(int tensor_index) const {
+    if (tensor_index < 0 ||
+        static_cast<size_t>(tensor_index) >= context_->tensors_size) {
+      return nullptr;
+    }
+    return &context_->tensors[tensor_index];
+  }
+
+  // Read only access to list of inputs.
+  std::vector<int>& inputs() { return inputs_; }
+
+  // Read only access to list of inputs.
+  const std::vector<int>& inputs() const { return inputs_; }
+
+  // Read only access to list of outputs.
+  std::vector<int>& outputs() { return outputs_; }
+
+  // Read only access to list of outputs.
+  const std::vector<int>& outputs() const { return outputs_; }
+
+  // Read only access to list of variable tensors.
+  std::vector<int>& variables() { return variables_; }
+
+  // Read only access to list of variable tensors.
+  const std::vector<int>& variables() const { return variables_; }
+
+  size_t tensors_size() const { return tensors_.size(); }
+
+  // Return the number of ops in the model.
+  size_t nodes_size() const { return nodes_and_registration_.size(); }
+
+  // Read only access to list of variable tensors.
+  std::vector<int>& execution_plan() { return execution_plan_; }
+
+  // Read only access to list of variable tensors.
+  const std::vector<int>& execution_plan() const { return execution_plan_; }
+
+  // Mutable form of tensors (TEMPORARY for refactor).
+  // TODO(b/119495520): remove when refactoring complete.
+  std::vector<TfLiteTensor>& tensors() { return tensors_; }
+  // Mutable form of tensors (TEMPORARY for refactor).
+  // TODO(b/119495520): remove when refactoring complete.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
+  nodes_and_registration() {
+    return nodes_and_registration_;
+  }
+
+  const std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
+  nodes_and_registration() const {
+    return nodes_and_registration_;
+  }
+
+  // Get a pointer to an operation and registration data structure if in bounds.
+  const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
+      int node_index) const {
+    if (node_index < 0 || static_cast<size_t>(node_index) >= nodes_size())
+      return nullptr;
+    return &nodes_and_registration_[node_index];
+  }
+
+
+  // Change the dimensionality of a given tensor. Note, this is only acceptable
+  // for tensor indices that are inputs.
+  // Returns status of failure or success.
+  // TODO(aselle): Consider implementing ArraySlice equivalent to make this
+  //   more adept at accepting data without an extra copy. Use absl::ArraySlice
+  //   if our partners determine that dependency is acceptable.
+  TfLiteStatus ResizeInputTensor(int tensor_index,
+                                 const std::vector<int>& dims);
+
+  // Update allocations for all tensors. This will redim dependent tensors using
+  // the input tensor dimensionality as given. This is relatively expensive.
+  // If you know that your sizes are not changing, you need not call this.
+  // Returns status of success or failure.
+  TfLiteStatus AllocateTensors();
+
+  // Invoke the subgraph (run the whole graph in dependency order).
+  //
+  // NOTE: It is possible that the interpreter is not in a ready state
+  // to evaluate (i.e. if a ResizeTensor() has been performed without an
+  // AllocateTensors().
+  // Returns status of success or failure.
+  TfLiteStatus Invoke();
+
+  // Entry point for C node plugin API to report an error.
+  void ReportError(const char* format, ...);
+
+  void UseNNAPI(bool enable);
+
+  // Return the subgraph specific context.
+  TfLiteContext* context() { return context_; }
+
+  // Set the value of an external context.
+  void SetExternalContext(TfLiteExternalContextType type,
+                          TfLiteExternalContext* ctx);
+  // Get the half precision flag.
+  // WARNING: This is an experimental API and subject to change.
+  bool GetAllowFp16PrecisionForFp32() const {
+    return context_->allow_fp32_relax_to_fp16;
+  }
+
+  // Ensure the data in `tensor.data` is readable. In case delegate is used,
+  // it might require to copy the data from delegate buffer to raw memory.
+  // WARNING: This is an experimental API and subject to change.
+  // TODO(b/119495520): make this private when refactoring complete.
+  TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
+    TfLiteTensor* t = &tensors_[tensor_index];
+    TF_LITE_ENSURE(context_, t != nullptr);
+    if (t->data_is_stale) {
+      TF_LITE_ENSURE(context_, t->delegate != nullptr);
+      TF_LITE_ENSURE(context_, t->buffer_handle != kTfLiteNullBufferHandle);
+      // This can be null if the delegate doesn't use its own buffer.
+      TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle != nullptr);
+      t->delegate->CopyFromBufferHandle(context_, t->delegate, t->buffer_handle,
+                                        t->data.raw, t->bytes);
+      t->data_is_stale = false;
+    }
+    return kTfLiteOk;
+  }
+
+
+  // The default capacity of `tensors_` vector.
+  static constexpr int kTensorsReservedCapacity = 128;
+  // The capacity headroom of `tensors_` vector before calling ops'
+  // `prepare` and `invoke` function. In these functions, it's guaranteed
+  // allocating up to `kTensorsCapacityHeadroom` more tensors won't invalidate
+  // pointers to existing tensors.
+  static constexpr int kTensorsCapacityHeadroom = 16;
+
+  // Reset all variable tensors to the default value.
+  // If a variable tensor doesn't have a buffer, reset it to zero.
+  // TODO(b/115961645): Implement - If a variable tensor has a buffer, reset it
+  // to the value of the buffer.
+  // WARNING: This is an experimental API and subject to change.
+  TfLiteStatus ResetVariableTensors();
+
+ private:
+  // Prevent 'context_' from accessing functions that are only available to
+  // delegated kernels.
+  void SwitchToKernelContext();
+
+  // Add delegate-only functions to 'context_'.
+  void SwitchToDelegateContext();
+
+  // Give 'op_reg' a chance to initialize itself using the contents of
+  // 'buffer'.
+  void* OpInit(const TfLiteRegistration& op_reg, const char* buffer,
+               size_t length) {
+    if (op_reg.init == nullptr) return nullptr;
+    return op_reg.init(context_, buffer, length);
+  }
+
+  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
+  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
+    if (op_reg.free == nullptr) return;
+    if (buffer) {
+      op_reg.free(context_, buffer);
+    }
+  }
+
+  // Prepare the given 'node' for execution.
+  TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) {
+    if (op_reg.prepare == nullptr) return kTfLiteOk;
+    return op_reg.prepare(context_, node);
+  }
+
+  // Invoke the operator represented by 'node'.
+  TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) {
+    if (op_reg.invoke == nullptr) return kTfLiteError;
+    return op_reg.invoke(context_, node);
+  }
+
+  // Call OpPrepare() for as many ops as possible, allocating memory for their
+  // tensors. If an op containing dynamic tensors is found, preparation will be
+  // postponed until this function is called again. This allows the interpreter
+  // to wait until Invoke() to resolve the sizes of dynamic tensors.
+  TfLiteStatus PrepareOpsAndTensors();
+
+  // Call OpPrepare() for all ops starting at 'first_node'. Stop when a
+  // dynamic tensors is found or all ops have been prepared. Fill
+  // 'last_node_prepared' with the id of the op containing dynamic tensors, or
+  // the last in the graph.
+  TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index,
+                                    int* last_execution_plan_index_prepared);
+
+  // Tensors needed by the interpreter. Use `AddTensors` to add more blank
+  // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
+  // `context_` whenever this std::vector is reallocated. Currently this
+  // only happens in `AddTensors()`.
+  std::vector<TfLiteTensor> tensors_;
+
+  // Check if an array of tensor indices are valid with respect to the Tensor
+  // array.
+  // NOTE: this changes consistent_ to be false if indices are out of bounds.
+  TfLiteStatus CheckTensorIndices(const char* label, const int* indices,
+                                  int length);
+
+  // Compute the number of bytes required to represent a tensor with dimensions
+  // specified by the array dims (of length dims_size). Returns the status code
+  // and bytes.
+  TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size,
+                             size_t* bytes);
+
+  // Request an tensor be resized implementation. If the given tensor is of
+  // type kTfLiteDynamic it will also be allocated new memory.
+  TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size);
+
+  // Report a detailed error string (will be printed to stderr).
+  // TODO(aselle): allow user of class to provide alternative destinations.
+  void ReportErrorImpl(const char* format, va_list args);
+
+  // Entry point for C node plugin API to request an tensor be resized.
+  static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor,
+                                   TfLiteIntArray* new_size);
+  // Entry point for C node plugin API to report an error.
+  static void ReportErrorC(TfLiteContext* context, const char* format, ...);
+
+  // Entry point for C node plugin API to add new tensors.
+  static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add,
+                                 int* first_new_tensor_index);
+
+  // WARNING: This is an experimental API and subject to change.
+  // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels
+  static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
+      TfLiteContext* context, TfLiteRegistration registration,
+      const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate);
+
+  // Update the execution graph to replace some of the nodes with stub
+  // nodes. Specifically any node index that has `nodes[index]==1` will be
+  // slated for replacement with a delegate kernel specified by registration.
+  // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller.
+  // WARNING: This is an experimental interface that is subject to change.
+  TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
+      TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
+      TfLiteDelegate* delegate);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Gets the internal pointer to a TensorFlow lite node by node_index.
+  TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node,
+                                      TfLiteRegistration** registration);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Entry point for C node plugin API to get a node by index.
+  static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*,
+                                             int node_index, TfLiteNode** node,
+                                             TfLiteRegistration** registration);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Gets an TfLiteIntArray* representing the execution plan. The interpreter
+  // owns this memory and it is only guaranteed to exist during the invocation
+  // of the delegate prepare.
+  TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Entry point for C node plugin API to get the execution plan.
+  static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context,
+                                       TfLiteIntArray** execution_plan);
+
+  // Retrieve an existing external context by type.
+  TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type);
+  static TfLiteExternalContext* GetExternalContext(
+      struct TfLiteContext* context, TfLiteExternalContextType type);
+
+  // Set the value of an external context.
+  static void SetExternalContext(struct TfLiteContext* context,
+                                 TfLiteExternalContextType type,
+                                 TfLiteExternalContext* ctx);
+
+  // Allow a delegate to look at the graph and modify the graph to handle
+  // parts of the graph themselves. After this is called, the graph may
+  // contain new nodes that replace 1 more nodes.
+  // WARNING: This is an experimental API and subject to change.
+  TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
+
+  // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra
+  // capacity. Calling this function may invalidate existing pointers to
+  // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
+  // more tensors won't invalidate the pointer to existing tensors.
+  void EnsureTensorsVectorCapacity() {
+    const size_t required_capacity = tensors_.size() + kTensorsCapacityHeadroom;
+    if (required_capacity > tensors_.capacity()) {
+      tensors_.reserve(required_capacity);
+      context_->tensors = tensors_.data();
+    }
+  }
+
+  // The state of the Interpreter.
+  enum State {
+    // The interpreter isn't ready to be invoked.
+    // `AllocateTensor` need to be called to enter an invokable state.
+    kStateUninvokable = 0,
+    // The interpreter is ready to be invoked.
+    kStateInvokable,
+    // The interpreter is ready to be invoked, and graph can't be further
+    // modified. The interpreter will enter this state when calling
+    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
+    kStateInvokableAndImmutable,
+  };
+  State state_ = kStateUninvokable;
+
+  // A pure C data structure used to communicate with the pure C plugin
+  // interface. To avoid copying tensor metadata, this is also the definitive
+  // structure to store tensors.
+  // TODO(b/119495520): Get rid of owned and just make context_ a instance.
+  TfLiteContext owned_context_;
+  TfLiteContext* context_;
+
+  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
+  // function pointers to actual implementation.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
+      nodes_and_registration_;
+
+  // Whether the model is consistent. That is to say if the inputs and outputs
+  // of every node and the global inputs and outputs are valid indexes into
+  // the tensor array.
+  bool consistent_ = true;
+
+  // Array of indices representing the tensors that are inputs to the
+  // interpreter.
+  std::vector<int> inputs_;
+
+  // Array of indices representing the tensors that are outputs to the
+  // interpreter.
+  std::vector<int> outputs_;
+
+  // Array of indices representing the tensors that are variable tensors.
+  std::vector<int> variables_;
+
+  // The error reporter delegate that tflite will forward queries errors to.
+  ErrorReporter* error_reporter_;
+
+  // Index of the next node to prepare.
+  // During Invoke(), Interpreter will allocate input tensors first, which are
+  // known to be fixed size. Then it will allocate outputs from nodes as many
+  // as possible. When there is a node that produces dynamic sized tensor.
+  // Interpreter will stop allocating tensors, set the value of next allocate
+  // node id, and execute the node to generate the output tensor before continue
+  // to allocate successors. This process repeats until all nodes are executed.
+  // NOTE: this relies on the order of nodes that is in topological order.
+  int next_execution_plan_index_to_prepare_;
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // This is a list of node indices (to index into nodes_and_registration).
+  // This represents a valid topological sort (dependency ordered) execution
+  // plan. In particular, it is valid for this ordering to contain only a
+  // subset of the node indices.
+  std::vector<int> execution_plan_;
+
+  // In the future, we'd like a TfLiteIntArray compatible representation.
+  // TODO(aselle): replace execution_plan_ with this.
+  std::unique_ptr<TfLiteIntArray, TfLiteIntArrayDeleter> plan_cache_;
+
+  // Whether to delegate to NN API
+  std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
+
+  std::unique_ptr<MemoryPlanner> memory_planner_;
+
+  // Tracking bit for whether a tensor was resized in the course of an op
+  // invocation. This is a useful hint to ensure that dynamic tensor outputs
+  // trigger downstream reallocation after op invocation.
+  bool tensor_resized_since_op_invoke_ = false;
+
+  // External contexts (kTfLiteMaxExternalContexts).
+  TfLiteExternalContext** external_contexts_;
+};
+
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_CORE_SUBGRAPH_H_
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 0bac5b07b2..4f4a999210 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include <cstdint>
 #include <cstring>
 
-#include "tensorflow/lite/arena_planner.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/context_util.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
@@ -32,110 +31,14 @@ limitations under the License.
 #include "tensorflow/lite/util.h"
 
 namespace tflite {
-namespace {
-
-TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
-                           const TfLiteRegistration& registration,
-                           int node_index, const char* message) {
-  context->ReportError(
-      context, "Node number %d (%s) %s.\n", node_index,
-      registration.custom_name
-          ? registration.custom_name
-          : EnumNameBuiltinOperator(
-                static_cast<BuiltinOperator>(registration.builtin_code)),
-      message);
-  return kTfLiteError;
-}
-
-// Stub method which returns kTfLiteError when the function is forbidden.
-// We're registrating this function to several different function to save
-// compiled binary size. Please note the restrictions:
-// * The type of first parameter have to be `TfLiteContext*`.
-// * All paramteters must be trivailly destructible. (E.g. No C++ class)
-TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) {
-  context->ReportError(context,
-                       "The function is forbidden if not calling in delegate.");
-  return kTfLiteError;
-}
-
-// Set the ForbiddenContextFunction to a compatible function pointer.
-template <typename FunctionType>
-void SetForbiddenContextFunction(FunctionType* func) {
-  *func = reinterpret_cast<FunctionType>(ForbiddenContextFunction);
-}
-
-// Returns true if at least one tensor in the given list is kTfLiteDynamic.
-template <typename TensorIntArray>
-bool HasDynamicTensorImpl(const TfLiteContext& context,
-                          const TensorIntArray& int_array) {
-  for (int i : int_array) {
-    const TfLiteTensor& tensor = context.tensors[i];
-    if (tensor.allocation_type == kTfLiteDynamic) {
-      return true;
-    }
-  }
-  return false;
-}
-
-}  // namespace
-
-// A trivial implementation of GraphInfo around the Interpreter.
-// NOTE: this interpreter info represents the subset of the
-// graph that is executed according to execution plan. Thus,
-// the indices are execution plan indices rather than raw node
-// indices.
-class InterpreterInfo : public GraphInfo {
- public:
-  explicit InterpreterInfo(Interpreter* interpreter)
-      : interpreter_(interpreter) {}
-
-  size_t num_tensors() const override { return interpreter_->tensors_size(); }
-  TfLiteTensor* tensor(size_t index) override {
-    return interpreter_->tensor(index);
-  }
-  size_t num_nodes() const override {
-    return interpreter_->execution_plan().size();
-  }
-  const TfLiteNode& node(size_t index) const override {
-    int node_index = interpreter_->execution_plan()[index];
-    return interpreter_->node_and_registration(node_index)->first;
-  }
-  const std::vector<int>& inputs() const override {
-    return interpreter_->inputs();
-  }
-  const std::vector<int>& outputs() const override {
-    return interpreter_->outputs();
-  }
-  const std::vector<int>& variables() const override {
-    return interpreter_->variables();
-  }
-
- public:
-  Interpreter* interpreter_;
-};
 
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  context_.impl_ = static_cast<void*>(this);
-  context_.ResizeTensor = ResizeTensor;
-  context_.ReportError = ReportError;
-  context_.AddTensors = AddTensors;
-  context_.tensors = nullptr;
-  context_.tensors_size = 0;
-  context_.allow_fp32_relax_to_fp16 = false;
-  context_.recommended_num_threads = -1;
-  context_.GetExternalContext = GetExternalContext;
-  context_.SetExternalContext = SetExternalContext;
-
-  // Invalid to call these these except from TfLiteDelegate
-  SwitchToKernelContext();
+  subgraphs_.emplace_back(error_reporter_, external_contexts_);
+  context_ = primary_subgraph().context();
 
   // Reserve some space for the tensors to avoid excessive resizing.
-  tensors_.reserve(kTensorsReservedCapacity);
-  nodes_and_registration_.reserve(kTensorsReservedCapacity);
-  next_execution_plan_index_to_prepare_ = 0;
-
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
     external_contexts_[i] = nullptr;
   }
@@ -143,673 +46,75 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
   UseNNAPI(false);
 }
 
-Interpreter::~Interpreter() {
-  for (auto& nodeAndReg : nodes_and_registration_) {
-    TfLiteNode& node = nodeAndReg.first;
-    TfLiteIntArrayFree(node.inputs);
-    TfLiteIntArrayFree(node.outputs);
-    TfLiteIntArrayFree(node.temporaries);
-    if (node.builtin_data) free(node.builtin_data);
-    OpFree(nodeAndReg.second, node.user_data);
-    node.builtin_data = nullptr;
-  }
-
-  for (size_t i = 0; i < context_.tensors_size; i++) {
-    TfLiteTensor* tensor = &context_.tensors[i];
-    if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
-        tensor->delegate->FreeBufferHandle != nullptr) {
-      tensor->delegate->FreeBufferHandle(&context_, tensor->delegate,
-                                         &tensor->buffer_handle);
-    }
-    TfLiteTensorFree(tensor);
-  }
-}
-
-TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
-    TfLiteContext* context, TfLiteRegistration registration,
-    const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace,
-                                              delegate);
-}
-
-namespace {
-
-// Copy a std::vector<int> to an existing TfLiteIntArray.
-// This is a low-level data manipulation function, and it's caller's
-// responsibility to ensure TfLiteIntArray has enough size.
-void CopyVectorToTfLiteIntArray(const std::vector<int>& vec,
-                                TfLiteIntArray* arr) {
-  arr->size = vec.size();
-  memcpy(arr->data, vec.data(), sizeof(int) * arr->size);
-}
-
-// This function allocates a continuous memory space that contains a
-// TfLiteDelegateParams followed by a several TfLiteIntArray.
-// When calling `free` at TfLiteDelegateParams*, all the allocated space
-// will be freed together.
-//
-// +-----------------------------------+
-// | TfLiteDelegateParams              |
-// | TfLiteDelegate* delegate;         |
-// | TfLiteIntArray* nodes_to_replace; |--\
-// | TfLiteIntArray* input_tensors;    |--+--\
-// | TfLiteIntArray* output_tensors;   |--+--+--\
-// +-----------------------------------+  |  |  |
-// | TfLiteIntArray (variable size)    |<-/  |  |
-// +-----------------------------------+     |  |
-// | TfLiteIntArray (variable size)    |<----/  |
-// +-----------------------------------+        |
-// | TfLiteIntArray (variable size)    |<-------/
-// +-----------------------------------+
-TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate,
-                                           const NodeSubset& node_subset) {
-  // Step 1: Calculate the allocation size.
-  int allocation_size = sizeof(TfLiteDelegateParams);
-
-  int nodes_to_replace_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size());
-  allocation_size += nodes_to_replace_size;
-
-  int input_tensors_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size());
-  allocation_size += input_tensors_size;
-
-  int output_tensors_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size());
-  allocation_size += output_tensors_size;
-
-  // Step 2: Allocate the memory.
-  // Use `char*` for conveniently step through the allocated space by bytes.
-  char* allocation = reinterpret_cast<char*>(malloc(allocation_size));
-
-  // Step 3: Fill all data structures structures.
-  TfLiteDelegateParams* params =
-      reinterpret_cast<TfLiteDelegateParams*>(allocation);
-  params->delegate = delegate;
-  allocation += sizeof(TfLiteDelegateParams);
-
-  params->nodes_to_replace = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace);
-  allocation += nodes_to_replace_size;
-
-  params->input_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors);
-  allocation += input_tensors_size;
-
-  params->output_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.output_tensors,
-                             params->output_tensors);
-  allocation += output_tensors_size;
-
-  return params;
-}
-
-}  // namespace
-
-TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
-    TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
-    TfLiteDelegate* delegate) {
-  // Annotate the registration as DELEGATE op.
-  registration.builtin_code = BuiltinOperator_DELEGATE;
-
-  // Analyze the graph to find all independent node_subsets that are either
-  // fully not-this-delegate or this-delegate computation.
-  InterpreterInfo info(this);
-  std::vector<NodeSubset> node_subsets;
-  PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace,
-                                           &node_subsets);
-
-  execution_plan_.clear();
-  for (auto& node_subset : node_subsets) {
-    // Subsets calimed by the delegate should have a "macro" op created, the
-    // other node_subsets (kTfNonPartition) just have their nodes added back to
-    // the execution plan.
-    switch (node_subset.type) {
-      case NodeSubset::kTfNonPartition:
-        for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end();
-             ++it) {
-          execution_plan_.push_back(*it);
-        }
-        break;
-      case NodeSubset::kTfPartition: {
-        int node_index;
-
-        TfLiteDelegateParams* params =
-            CreateDelegateParams(delegate, node_subset);
-        TF_LITE_ENSURE_STATUS(AddNodeWithParameters(
-            node_subset.input_tensors, node_subset.output_tensors, nullptr, 0,
-            params, &registration, &node_index));
-
-        // Initialize the output tensors's delegate-related fields.
-        for (int tensor_index : node_subset.output_tensors) {
-          TfLiteTensor* tensor = &tensors_[tensor_index];
-          TF_LITE_ENSURE(&context_, tensor->delegate == nullptr ||
-                                        tensor->delegate == delegate);
-          tensor->delegate = delegate;
-        }
-
-        // Associate the node with the delegate.
-        TfLiteNode* node = &nodes_and_registration_[node_index].first;
-        node->delegate = delegate;
-      } break;
-      case NodeSubset::kTfUnexplored:
-        return kTfLiteError;
-        break;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteExternalContext* Interpreter::GetExternalContext(
-    TfLiteExternalContextType type) {
-  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
-    return external_contexts_[type];
-  }
-  return nullptr;
-}
-
-TfLiteExternalContext* Interpreter::GetExternalContext(
-    struct TfLiteContext* context, TfLiteExternalContextType type) {
-  return static_cast<Interpreter*>(context->impl_)->GetExternalContext(type);
-}
+Interpreter::~Interpreter() {}
 
 void Interpreter::SetExternalContext(TfLiteExternalContextType type,
                                      TfLiteExternalContext* ctx) {
-  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
-    external_contexts_[type] = ctx;
-  }
-}
-
-void Interpreter::SetExternalContext(struct TfLiteContext* context,
-                                     TfLiteExternalContextType type,
-                                     TfLiteExternalContext* ctx) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->SetExternalContext(type, ctx);
-}
-
-// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns
-// this memory and it is only guaranteed to exist during the invocation of the
-// delegate prepare.
-TfLiteStatus Interpreter::GetExecutionPlan(TfLiteIntArray** execution_plan) {
-  // TODO(aselle): Do not make a copy here
-  plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size()));
-  *execution_plan = plan_cache_.get();
-  static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]),
-                "TfLiteIntArray and execution_plan do not contain same type.");
-  std::memcpy(plan_cache_->data, execution_plan_.data(),
-              sizeof(plan_cache_->data[0]) * execution_plan_.size());
-  return kTfLiteOk;
-}
-
-// WARNING: This is an experimental interface that is subject to change.
-// Entry point for C node plugin API to get the execution plan
-TfLiteStatus Interpreter::GetExecutionPlan(struct TfLiteContext* context,
-                                           TfLiteIntArray** execution_plan) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->GetExecutionPlan(execution_plan);
+  primary_subgraph().SetExternalContext(type, ctx);
 }
 
 TfLiteStatus Interpreter::SetInputs(std::vector<int> inputs) {
-  TF_LITE_ENSURE_OK(&context_,
-                    CheckTensorIndices("inputs", inputs.data(), inputs.size()));
-  inputs_ = std::move(inputs);
-  return kTfLiteOk;
+  return primary_subgraph().SetInputs(inputs);
 }
 
 TfLiteStatus Interpreter::SetOutputs(std::vector<int> outputs) {
-  TF_LITE_ENSURE_OK(
-      &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
-  outputs_ = std::move(outputs);
-  return kTfLiteOk;
+  return primary_subgraph().SetOutputs(outputs);
 }
 
 TfLiteStatus Interpreter::SetVariables(std::vector<int> variables) {
-  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
-                                                  variables.size()));
-  variables_ = std::move(variables);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::CheckTensorIndices(const char* label,
-                                             const int* indices, int length) {
-  // Making sure kOptionalTensor is not re-defined to something other than -1.
-  static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1");
-
-  for (int i = 0; i < length; i++) {
-    int index = indices[i];
-    // Continue if index == kOptionalTensor before additional comparisons below,
-    // size_t(-1) is always >= context_tensors_size.
-    if (index == kOptionalTensor) {
-      continue;
-    }
-    if (index < 0 || static_cast<size_t>(index) >= context_.tensors_size) {
-      ReportError(&context_, "Invalid tensor index %d in %s\n", index, label);
-      consistent_ = false;
-      return kTfLiteError;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims,
-                                        size_t dims_size, size_t* bytes) {
-  // TODO(aselle): Check for overflow here using overflow.h in TensorFlow
-  // MultiplyWithoutOverflow.
-  TF_LITE_ENSURE(&context_, bytes != nullptr);
-  size_t count = 1;
-  for (int k = 0; k < dims_size; k++) count *= dims[k];
-  switch (type) {
-    case kTfLiteFloat32:
-      *bytes = sizeof(float) * count;
-      break;
-    case kTfLiteInt16:
-      *bytes = sizeof(int16_t) * count;
-      break;
-    case kTfLiteInt32:
-      *bytes = sizeof(int32_t) * count;
-      break;
-    case kTfLiteUInt8:
-      *bytes = sizeof(uint8_t) * count;
-      break;
-    case kTfLiteInt8:
-      *bytes = sizeof(int8_t) * count;
-      break;
-    case kTfLiteInt64:
-      *bytes = sizeof(int64_t) * count;
-      break;
-    case kTfLiteBool:
-      *bytes = sizeof(bool) * count;
-      break;
-    case kTfLiteComplex64:
-      *bytes = sizeof(std::complex<float>) * count;
-      break;
-    default:
-      ReportError(&context_,
-                  "Only float32, int16, int32, int64, uint8, int8, bool, "
-                  "complex64 supported currently.");
-      return kTfLiteError;
-  }
-  return kTfLiteOk;
+  return primary_subgraph().SetVariables(variables);
 }
 
 TfLiteStatus Interpreter::AllocateTensors() {
-  if (!consistent_) {
-    ReportError(&context_, "AllocateTensors() called on inconsistent model.");
-    return kTfLiteError;
-  }
-
-  // Explicit (re)allocation is necessary if nodes have been changed or tensors
-  // have been resized. For inputs marked as dynamic, we can't short-circuit the
-  // allocation as the client may have done the resize manually.
-  if (state_ != kStateUninvokable && !HasDynamicTensorImpl(context_, inputs_)) {
-    return kTfLiteOk;
-  }
-
-  next_execution_plan_index_to_prepare_ = 0;
-  if (memory_planner_) {
-    TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations());
-  }
-
-  TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-
-  state_ = kStateInvokable;
-
-  // Reset the variable tensors to zero after (re)allocating the tensors.
-  // Developers shouldn't rely on the side effect of this function to reset
-  // variable tesnsors. They should call `ResetVariableTensors` directly
-  // instead.
-  ResetVariableTensors();
-
-  return kTfLiteOk;
-}
-
-// TODO(ycling): Support non-zero default values.
-TfLiteStatus Interpreter::ResetVariableTensors() {
-  for (auto& tensor : tensors_) {
-    if (!tensor.is_variable) {
-      continue;
-    }
-
-    // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be
-    // allocated after the initial `PrepareOpsAndTensors()` is called.
-    TF_LITE_ENSURE_EQ(&context_, tensor.allocation_type,
-                      kTfLiteArenaRwPersistent);
-    TF_LITE_ENSURE(&context_, tensor.data.raw != nullptr);
-
-    memset(tensor.data.raw, 0, tensor.bytes);
-  }
-  return kTfLiteOk;
+  return primary_subgraph().AllocateTensors();
 }
 
 void Interpreter::ReserveNodes(int count) {
-  nodes_and_registration_.reserve(count);
+  primary_subgraph().nodes_and_registration().reserve(count);
 }
 
 TfLiteStatus Interpreter::AddNodeWithParameters(
     const std::vector<int>& inputs, const std::vector<int>& outputs,
     const char* init_data, size_t init_data_size, void* builtin_data,
     const TfLiteRegistration* registration, int* node_index) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(&context_,
-                "AddNodeWithParameters is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-  state_ = kStateUninvokable;
-
-  std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
-                                                              free);
-
-  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("node inputs", inputs.data(),
-                                                  inputs.size()));
-  TF_LITE_ENSURE_OK(
-      &context_,
-      CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
-
-  int new_node_index = nodes_and_registration_.size();
-  if (node_index) *node_index = new_node_index;
-  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
-  auto& node_and_reg = nodes_and_registration_.back();
-  TfLiteNode& node = node_and_reg.first;
-  if (node.inputs) TfLiteIntArrayFree(node.inputs);
-  if (node.outputs) TfLiteIntArrayFree(node.outputs);
-  if (node.temporaries) TfLiteIntArrayFree(node.temporaries);
-
-  // NOTE, here we are not using move semantics yet, since our internal
-  // representation isn't std::vector, but in the future we would like to avoid
-  // copies, so we want the interface to take r-value references now.
-  node.inputs = ConvertVectorToTfLiteIntArray(inputs);
-  node.outputs = ConvertVectorToTfLiteIntArray(outputs);
-  node.temporaries = TfLiteIntArrayCreate(0);
-  if (init_data) {
-    node.user_data = OpInit(*registration, init_data, init_data_size);
-  } else {
-    node.user_data =
-        OpInit(*registration,
-               reinterpret_cast<const char*>(builtin_data_deleter.get()), 0);
-  }
-
-  node.builtin_data = builtin_data_deleter.release();
-  // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size`
-  // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels.
-
-  if (registration->builtin_code == BuiltinOperator_CUSTOM) {
-    // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer
-    // `Operator` table is passed in.
-    node.custom_initial_data = init_data;
-    node.custom_initial_data_size = init_data_size;
-  } else {
-    node.custom_initial_data = nullptr;
-    node.custom_initial_data_size = 0;
-  }
-
-  node.delegate = nullptr;
-  node_and_reg.second = *registration;
-  execution_plan_.push_back(new_node_index);
-  return kTfLiteOk;
+  return primary_subgraph().AddNodeWithParameters(inputs, outputs, init_data,
+                                                  init_data_size, builtin_data,
+                                                  registration, node_index);
 }
 
 TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index,
                                             const std::vector<int>& dims) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(&context_,
-                "ResizeInputTensor is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-
-  // TODO(aselle): All bounds checks can be implemented as one-sided bounds
-  // checks by casting to unsigned for efficiency. Profile before doing this.
-  TF_LITE_ENSURE(&context_,
-                 tensor_index < context_.tensors_size && tensor_index >= 0);
-  TfLiteTensor* tensor = &context_.tensors[tensor_index];
-
-  // Short-circuit the state change if the dimensions don't change, avoiding
-  // unnecessary (re)allocations.
-  if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) {
-    return kTfLiteOk;
-  }
-
-  state_ = kStateUninvokable;
-  return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims));
-}
-
-bool HasDynamicTensor(const TfLiteContext& context,
-                      const TfLiteIntArray* int_array) {
-  return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
-}
-
-TfLiteStatus Interpreter::PrepareOpsStartingAt(
-    int first_execution_plan_index, int* last_execution_plan_index_prepared) {
-  for (int execution_plan_index = first_execution_plan_index;
-       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
-    int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
-    const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
-    EnsureTensorsVectorCapacity();
-    if (OpPrepare(registration, &node) == kTfLiteError) {
-      return ReportOpError(&context_, node, registration, node_index,
-                           "failed to prepare");
-    }
-
-    *last_execution_plan_index_prepared = execution_plan_index;
-
-    // Discontinue if the node has dynamic outputs. Note that we don't
-    // stop for dynamic temporary tensors since they won't affect the
-    // sizes of other tensors in the graph.
-    if (HasDynamicTensor(context_, node.outputs)) {
-      break;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::PrepareOpsAndTensors() {
-  if (!memory_planner_) {
-    memory_planner_.reset(new ArenaPlanner(
-        &context_, std::unique_ptr<GraphInfo>(new InterpreterInfo(this)),
-        /*preserve_inputs=*/true, /*preserve_intermediates*/ false));
-    memory_planner_->PlanAllocations();
-  }
-
-  int last_exec_plan_index_prepared = 0;
-
-  TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt(
-      next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared));
-  TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations(
-      next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared));
-
-  next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1;
-  return kTfLiteOk;
+  return primary_subgraph().ResizeInputTensor(tensor_index, dims);
 }
 
 TfLiteStatus Interpreter::Invoke() {
-  if (!consistent_) {
-    ReportError(&context_, "Invoke called on model that is not consistent.");
-    return kTfLiteError;
-  }
-  if (state_ == kStateUninvokable) {
-    ReportError(&context_, "Invoke called on model that is not ready.");
-    return kTfLiteError;
-  }
-
-  TfLiteStatus status = kTfLiteOk;
-  if (nnapi_delegate_) {
-    if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) {
-      TF_LITE_ENSURE_OK(&context_, nnapi_delegate_->Invoke(this));
-      return kTfLiteOk;
-    } else {
-      // TODO(aselle): In the future, we would like this to be an
-      // automatic tflite CPU fallback.
-      ReportError(&context_,
-                  "NNAPI was requested, but dependent sized tensors "
-                  "being used.\n");
-      return kTfLiteError;
-    }
-  }
-
-  // Invocations are always done in node order.
-  // Note that calling Invoke repeatedly will cause the original memory plan to
-  // be reused, unless either ResizeInputTensor() or AllocateTensors() has been
-  // called.
-  for (int execution_plan_index = 0;
-       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
-    if (execution_plan_index == next_execution_plan_index_to_prepare_) {
-      TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-      TF_LITE_ENSURE(&context_, next_execution_plan_index_to_prepare_ >=
-                                    execution_plan_index);
-    }
-    int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
-    const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
-    SCOPED_OPERATOR_PROFILE(profiler_, node_index);
-
-    // TODO(ycling): This is an extra loop through inputs to check if the data
-    // need to be copied from Delegate buffer to raw memory, which is often not
-    // needed. We may want to cache this in prepare to know if this needs to be
-    // done for a node or not.
-    for (int i = 0; i < node.inputs->size; ++i) {
-      int tensor_index = node.inputs->data[i];
-      if (tensor_index == kOptionalTensor) {
-        continue;
-      }
-      TfLiteTensor* tensor = &tensors_[tensor_index];
-      if (tensor->delegate && tensor->delegate != node.delegate &&
-          tensor->data_is_stale) {
-        EnsureTensorDataIsReadable(tensor_index);
-      }
-    }
-
-    EnsureTensorsVectorCapacity();
-    tensor_resized_since_op_invoke_ = false;
-    if (OpInvoke(registration, &node) == kTfLiteError) {
-      status = ReportOpError(&context_, node, registration, node_index,
-                             "failed to invoke");
-    }
-
-    // Force execution prep for downstream ops if the latest op triggered the
-    // resize of a dynamic tensor.
-    if (tensor_resized_since_op_invoke_ &&
-        HasDynamicTensor(context_, node.outputs)) {
-      next_execution_plan_index_to_prepare_ = execution_plan_index + 1;
-    }
-  }
+  TfLiteStatus status = primary_subgraph().Invoke();
 
   if (!allow_buffer_handle_output_) {
-    for (int tensor_index : outputs_) {
-      EnsureTensorDataIsReadable(tensor_index);
+    for (int tensor_index : outputs()) {
+      primary_subgraph().EnsureTensorDataIsReadable(tensor_index);
     }
   }
 
   return status;
 }
 
-TfLiteStatus Interpreter::ResizeTensor(TfLiteContext* context,
-                                       TfLiteTensor* tensor,
-                                       TfLiteIntArray* new_size) {
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function ResizeTensorImpl
-  // (this function is static).
-  return static_cast<Interpreter*>(context->impl_)
-      ->ResizeTensorImpl(tensor, new_size);
-}
-
-void Interpreter::ReportErrorImpl(const char* format, va_list args) {
-  error_reporter_->Report(format, args);
-}
-
-void Interpreter::ReportError(TfLiteContext* context, const char* format, ...) {
-  va_list args;
-  va_start(args, format);
-  auto* f = static_cast<Interpreter*>(context->impl_);
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function ReportErrorImpl
-  // (this function is static).
-  f->ReportErrorImpl(format, args);
-  va_end(args);
-}
-
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
                                      int* first_new_tensor_index) {
-  const size_t base_index = tensors_.size();
-  if (first_new_tensor_index) *first_new_tensor_index = base_index;
-  tensors_.resize(tensors_.size() + tensors_to_add);
-  for (size_t i = base_index; i < tensors_.size(); i++) {
-    memset(&tensors_[i], 0, sizeof(tensors_[i]));
-    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
-  }
-  context_.tensors = tensors_.data();
-  context_.tensors_size = tensors_.size();
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::AddTensors(TfLiteContext* context, int tensors_to_add,
-                                     int* first_new_tensor_index) {
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function AddTensors
-  // (this function is static).
-  return static_cast<Interpreter*>(context->impl_)
-      ->AddTensors(tensors_to_add, first_new_tensor_index);
+  return primary_subgraph().AddTensors(tensors_to_add, first_new_tensor_index);
 }
 
-TfLiteStatus Interpreter::GetNodeAndRegistration(
-    int node_index, TfLiteNode** node, TfLiteRegistration** registration) {
-  TF_LITE_ENSURE(&context_, node_index >= 0);
-  TF_LITE_ENSURE(&context_, static_cast<size_t>(node_index) < nodes_size());
-  TF_LITE_ENSURE(&context_, node != nullptr && registration != nullptr);
-  *node = &nodes_and_registration_[node_index].first;
-  *registration = &nodes_and_registration_[node_index].second;
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::GetNodeAndRegistration(
-    struct TfLiteContext* context, int node_index, TfLiteNode** node,
-    TfLiteRegistration** registration) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->GetNodeAndRegistration(node_index, node, registration);
+TfLiteStatus Interpreter::ResetVariableTensors() {
+  return primary_subgraph().ResetVariableTensors();
 }
 
 TfLiteStatus Interpreter::SetTensorParametersReadOnly(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
     size_t bytes, const Allocation* allocation) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(
-        &context_,
-        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-
-  TF_LITE_ENSURE(&context_,
-                 tensor_index < context_.tensors_size && tensor_index >= 0);
-  // For most tensors we know exactly how much memory is necessary so we can
-  // ensure the buffer is large enough. However, we need to skip string tensors
-  // because their sizes change with the contents of the individual strings.
-  if (type != kTfLiteString) {
-    size_t required_bytes;
-    TF_LITE_ENSURE_OK(&context_,
-                      BytesRequired(type, dims, rank, &required_bytes));
-    TF_LITE_ENSURE_EQ(&context_, required_bytes, bytes);
-  }
-
-  TfLiteTensor& tensor = context_.tensors[tensor_index];
-  if (type == tensor.type &&
-      EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
-    // Fast path which does not invalidate the invokable property.
-    TfLiteTensorDataFree(&tensor);
-    tensor.data.raw = const_cast<char*>(buffer);
-    if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
-    tensor.params = quantization;
-    tensor.allocation_type = kTfLiteMmapRo;
-    tensor.allocation = allocation;
-  } else {
-    state_ = kStateUninvokable;
-    TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                      quantization, const_cast<char*>(buffer), bytes,
-                      kTfLiteMmapRo, allocation, false, &tensor);
-  }
-  return kTfLiteOk;
+  return primary_subgraph().SetTensorParametersReadOnly(
+      tensor_index, type, name, rank, dims, quantization, buffer, bytes,
+      allocation);
 }
 
 // Set description of inputs/outputs/data/fptrs for node `node_index`.
@@ -819,186 +124,52 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly(
 TfLiteStatus Interpreter::SetTensorParametersReadWrite(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(
-        &context_,
-        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-  TF_LITE_ENSURE(&context_,
-                 tensor_index < context_.tensors_size && tensor_index >= 0);
-  size_t required_bytes = 0;
-  if (type != kTfLiteString) {
-    // These types will be allocated in our arena so we need to record how
-    // many bytes we will need based on the dimensions. String tensors are
-    // allocated dynamically and we can't know ahead of time how much space
-    // they will require.
-    TF_LITE_ENSURE_OK(&context_,
-                      BytesRequired(type, dims, rank, &required_bytes));
-  }
-
-  TfLiteAllocationType allocation_type = kTfLiteArenaRw;
-  if (type == kTfLiteString) {
-    if (is_variable) {
-      // We don't have a real use case for string variable tensor.
-      ReportError(&context_, "String variable tensor isn't supported.");
-      return kTfLiteError;
-    }
-    allocation_type = kTfLiteDynamic;
-  } else if (is_variable) {
-    allocation_type = kTfLiteArenaRwPersistent;
-  }
-
-  TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                    quantization,
-                    /*buffer=*/nullptr, required_bytes, allocation_type,
-                    nullptr, is_variable, &context_.tensors[tensor_index]);
-  return kTfLiteOk;
+  return primary_subgraph().SetTensorParametersReadWrite(
+      tensor_index, type, name, rank, dims, quantization, is_variable);
 }
 
 TfLiteStatus Interpreter::SetExecutionPlan(const std::vector<int>& new_plan) {
-  for (int node_index : new_plan) {
-    TF_LITE_ENSURE(&context_, node_index >= 0 && node_index < nodes_size());
-  }
-  execution_plan_ = new_plan;
-  return kTfLiteOk;
+  return primary_subgraph().SetExecutionPlan(new_plan);
 }
 
-TfLiteStatus Interpreter::ResizeTensorImpl(TfLiteTensor* tensor,
-                                           TfLiteIntArray* new_size) {
-  // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too.
-  if (tensor->allocation_type == kTfLiteArenaRw ||
-      tensor->allocation_type == kTfLiteDynamic ||
-      tensor->allocation_type == kTfLiteArenaRwPersistent) {
-    tensor_resized_since_op_invoke_ |=
-        TfLiteIntArrayEqual(tensor->dims, new_size) == 0;
-    if (tensor->type != kTfLiteString) {
-      size_t bytesRequired;
-      TfLiteStatus status = BytesRequired(tensor->type, new_size->data,
-                                          new_size->size, &bytesRequired);
-      if (status != kTfLiteOk) {
-        TfLiteIntArrayFree(new_size);
-        return kTfLiteError;
-      }
-
-      // Realloc space for kTfLiteDynamic tensors.
-      TfLiteTensorRealloc(bytesRequired, tensor);
-      tensor->bytes = bytesRequired;
-    }
-    if (tensor->dims) TfLiteIntArrayFree(tensor->dims);
-    tensor->dims = new_size;
-
-    if (tensor->allocation_type != kTfLiteDynamic) {
-      tensor->data.raw = nullptr;
-    }
-  } else {
-    // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore
-    // of fixed size.
-    TfLiteIntArrayFree(new_size);
-    ReportError(&context_, "Attempting to resize a fixed-size tensor.");
-    return kTfLiteError;
-  }
-  return kTfLiteOk;
-}
-
-void Interpreter::UseNNAPI(bool enable) {
-  // TODO(aselle): This is a workaround for finding if NNAPI exists.
-  // We also need to make sure getLibraryHandle() is renamed to be NNAPI
-  // prefixed.
-  if (!NNAPIDelegate::IsSupported()) enable = false;
-  if (!enable) {
-    nnapi_delegate_.reset();
-  } else if (!nnapi_delegate_) {
-    nnapi_delegate_.reset(new NNAPIDelegate);
-  }
-}
+void Interpreter::UseNNAPI(bool enable) { primary_subgraph().UseNNAPI(enable); }
 
 void Interpreter::SetNumThreads(int num_threads) {
-  context_.recommended_num_threads = num_threads;
+  for (auto& subgraph : subgraphs_) {
+    subgraph.context()->recommended_num_threads = num_threads;
+  }
 
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
     auto* c = external_contexts_[i];
     if (c && c->Refresh) {
-      c->Refresh(&context_);
+      c->Refresh(context_);
     }
   }
 }
 
-void Interpreter::SwitchToDelegateContext() {
-  context_.GetNodeAndRegistration = GetNodeAndRegistration;
-  context_.ReplaceNodeSubsetsWithDelegateKernels =
-      ReplaceNodeSubsetsWithDelegateKernels;
-  context_.GetExecutionPlan = GetExecutionPlan;
-}
-
-void Interpreter::SwitchToKernelContext() {
-  SetForbiddenContextFunction(&context_.GetNodeAndRegistration);
-  SetForbiddenContextFunction(&context_.ReplaceNodeSubsetsWithDelegateKernels);
-  SetForbiddenContextFunction(&context_.GetExecutionPlan);
+void Interpreter::SetAllowFp16PrecisionForFp32(bool allow) {
+  for (auto& subgraph : subgraphs_) {
+    subgraph.context()->allow_fp32_relax_to_fp16 = allow;
+  }
 }
 
 TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
-  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
-    int last_execution_plan_index_prepared;
-    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
-                                     0, &last_execution_plan_index_prepared));
-
-    bool has_dynamic_tensors = true;
-    // Dynamic tensors exist if not all nodes can be prepared.
-    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
-      // If all the nodes can be prepared, check if the last node has dynamic
-      // tensors.
-      int node_index = execution_plan_[last_execution_plan_index_prepared];
-      TfLiteNode& node = nodes_and_registration_[node_index].first;
-      if (!HasDynamicTensor(context_, node.outputs)) {
-        has_dynamic_tensors = false;
-      }
-    }
-    if (has_dynamic_tensors) {
-      ReportError(
-          &context_,
-          "Attempting to use a delegate that only supports static-sized "
-          "tensors with a graph that has dynamic-sized tensors.");
-      return kTfLiteError;
-    }
-  }
-
-  // TODO(aselle): Consider if it is worth storing pointers to delegates.
-  // Setup additional context interface.
-  SwitchToDelegateContext();
-
-  TfLiteStatus status = delegate->Prepare(&context_, delegate);
-
-  // Remove additional context info.
-  SwitchToKernelContext();
-
-  TF_LITE_ENSURE_OK(&context_, status);
-
-  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
-    // Reset the state to force tensor/op reallocation.
-    state_ = kStateUninvokable;
-    TF_LITE_ENSURE_OK(&context_, AllocateTensors());
-    TF_LITE_ENSURE_EQ(&context_, state_, kStateInvokable);
-    // After using a delegate which doesn't support dynamic tensors, make the
-    // entire graph immutable.
-    state_ = kStateInvokableAndImmutable;
-  }
-
-  return status;
+  return primary_subgraph().ModifyGraphWithDelegate(delegate);
 }
 
 TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle buffer_handle,
                                           TfLiteDelegate* delegate) {
-  TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  TfLiteTensor* tensor = &tensors_[tensor_index];
+  TF_LITE_ENSURE(context_, tensor_index < tensors_size());
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  TfLiteTensor* tensor = &tensors[tensor_index];
 
-  TF_LITE_ENSURE(&context_,
+  TF_LITE_ENSURE(context_,
                  tensor->delegate == nullptr || tensor->delegate == delegate);
   tensor->delegate = delegate;
   if (tensor->buffer_handle != kTfLiteNullBufferHandle) {
-    TF_LITE_ENSURE(&context_, tensor->delegate->FreeBufferHandle != nullptr);
-    tensor->delegate->FreeBufferHandle(&context_, tensor->delegate,
+    TF_LITE_ENSURE(context_, tensor->delegate->FreeBufferHandle != nullptr);
+    tensor->delegate->FreeBufferHandle(context_, tensor->delegate,
                                        &tensor->buffer_handle);
   }
   tensor->buffer_handle = buffer_handle;
@@ -1009,8 +180,9 @@ TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
 TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle* buffer_handle,
                                           TfLiteDelegate** delegate) {
-  TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  TfLiteTensor* tensor = &tensors_[tensor_index];
+  TF_LITE_ENSURE(context_, tensor_index < tensors_size());
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  TfLiteTensor* tensor = &tensors[tensor_index];
 
   *delegate = tensor->delegate;
   *buffer_handle = tensor->buffer_handle;
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index de6c42bfcc..a6ad5d9137 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/memory_planner.h"
 #include "tensorflow/lite/profiling/profiler.h"
 #include "tensorflow/lite/stderr_reporter.h"
@@ -73,9 +74,6 @@ constexpr TfLiteType typeToTfLiteType<string>() {
   return kTfLiteString;
 }
 
-// Forward declare since NNAPIDelegate uses Interpreter.
-class NNAPIDelegate;
-
 // An interpreter for a graph of nodes that input and output from tensors.
 // Each node of the graph processes a set of input tensors and produces a
 // set of output Tensors. All inputs/output tensors are referenced by index.
@@ -104,12 +102,6 @@ class NNAPIDelegate;
 // foo.Invoke();
 //
 
-struct TfLiteIntArrayDeleter {
-  void operator()(TfLiteIntArray* a) {
-    if (a) TfLiteIntArrayFree(a);
-  }
-};
-
 class Interpreter {
  public:
   // Instantiate an interpreter. All errors associated with reading and
@@ -121,6 +113,7 @@ class Interpreter {
 
   ~Interpreter();
 
+  // Interpreters are not copyable as they have non-trivial memory semantics.
   Interpreter(const Interpreter&) = delete;
   Interpreter& operator=(const Interpreter&) = delete;
 
@@ -201,34 +194,40 @@ class Interpreter {
   // Functions to access tensor data
 
   // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return inputs_; }
+  const std::vector<int>& inputs() const { return primary_subgraph().inputs(); }
 
   // Return the name of a given input. The given index must be between 0 and
   // inputs().size().
   const char* GetInputName(int index) const {
-    return context_.tensors[inputs_[index]].name;
+    return context_->tensors[inputs()[index]].name;
   }
 
   // Read only access to list of outputs.
-  const std::vector<int>& outputs() const { return outputs_; }
+  const std::vector<int>& outputs() const {
+    return primary_subgraph().outputs();
+  }
 
   // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const { return variables_; }
+  const std::vector<int>& variables() const {
+    return primary_subgraph().variables();
+  }
 
   // Return the name of a given output. The given index must be between 0 and
   // outputs().size().
   const char* GetOutputName(int index) const {
-    return context_.tensors[outputs_[index]].name;
+    return context_->tensors[outputs()[index]].name;
   }
 
   // Return the number of tensors in the model.
-  size_t tensors_size() const { return context_.tensors_size; }
+  size_t tensors_size() const { return context_->tensors_size; }
 
   // Return the number of ops in the model.
-  size_t nodes_size() const { return nodes_and_registration_.size(); }
+  size_t nodes_size() const { return primary_subgraph().nodes_size(); }
 
   // WARNING: Experimental interface, subject to change
-  const std::vector<int>& execution_plan() const { return execution_plan_; }
+  const std::vector<int>& execution_plan() const {
+    return primary_subgraph().execution_plan();
+  }
 
   // WARNING: Experimental interface, subject to change
   // Overrides execution plan. This bounds checks indices sent in.
@@ -238,27 +237,18 @@ class Interpreter {
   // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this
   // read/write access to structure
   TfLiteTensor* tensor(int tensor_index) {
-    if (tensor_index < 0 ||
-        static_cast<size_t>(tensor_index) >= context_.tensors_size)
-      return nullptr;
-    return &context_.tensors[tensor_index];
+    return primary_subgraph().tensor(tensor_index);
   }
 
   // Get an immutable tensor data structure.
   const TfLiteTensor* tensor(int tensor_index) const {
-    if (tensor_index < 0 ||
-        static_cast<size_t>(tensor_index) >= context_.tensors_size)
-      return nullptr;
-    return &context_.tensors[tensor_index];
+    return primary_subgraph().tensor(tensor_index);
   }
 
   // Get a pointer to an operation and registration data structure if in bounds.
   const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
       int node_index) const {
-    if (node_index < 0 ||
-        static_cast<size_t>(node_index) >= nodes_and_registration_.size())
-      return nullptr;
-    return &nodes_and_registration_[node_index];
+    return primary_subgraph().node_and_registration(node_index);
   }
 
   // Perform a checked cast to the appropriate tensor type (mutable pointer
@@ -289,28 +279,28 @@ class Interpreter {
   // index must be between 0 and inputs().size().
   template <class T>
   T* typed_input_tensor(int index) {
-    return typed_tensor<T>(inputs_[index]);
+    return typed_tensor<T>(inputs()[index]);
   }
 
   // Return an immutable pointer into the data of a given input tensor. The
   // given index must be between 0 and inputs().size().
   template <class T>
   const T* typed_input_tensor(int index) const {
-    return typed_tensor<T>(inputs_[index]);
+    return typed_tensor<T>(inputs()[index]);
   }
 
   // Return a mutable pointer into the data of a given output tensor. The given
   // index must be between 0 and outputs().size().
   template <class T>
   T* typed_output_tensor(int index) {
-    return typed_tensor<T>(outputs_[index]);
+    return typed_tensor<T>(outputs()[index]);
   }
 
   // Return an immutable pointer into the data of a given output tensor. The
   // given index must be between 0 and outputs().size().
   template <class T>
   const T* typed_output_tensor(int index) const {
-    return typed_tensor<T>(outputs_[index]);
+    return typed_tensor<T>(outputs()[index]);
   }
 
   // Change the dimensionality of a given tensor. Note, this is only acceptable
@@ -325,7 +315,6 @@ class Interpreter {
   // Update allocations for all tensors. This will redim dependent tensors using
   // the input tensor dimensionality as given. This is relatively expensive.
   // If you know that your sizes are not changing, you need not call this.
-
   // Returns status of success or failure.
   TfLiteStatus AllocateTensors();
 
@@ -346,14 +335,12 @@ class Interpreter {
   // Allow float16 precision for FP32 calculation when possible.
   // default: not allow.
   // WARNING: This is an experimental API and subject to change.
-  void SetAllowFp16PrecisionForFp32(bool allow) {
-    context_.allow_fp32_relax_to_fp16 = allow;
-  }
+  void SetAllowFp16PrecisionForFp32(bool allow);
 
   // Get the half precision flag.
   // WARNING: This is an experimental API and subject to change.
   bool GetAllowFp16PrecisionForFp32() const {
-    return context_.allow_fp32_relax_to_fp16;
+    return context_->allow_fp32_relax_to_fp16;
   }
 
   // Owning handle to a TfLiteDelegate instance.
@@ -366,24 +353,6 @@ class Interpreter {
   // WARNING: This is an experimental API and subject to change.
   TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
 
-  // Ensure the data in `tensor.data` is readable. In case delegate is used,
-  // it might require to copy the data from delegate buffer to raw memory.
-  // WARNING: This is an experimental API and subject to change.
-  TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
-    TfLiteTensor* t = tensor(tensor_index);
-    TF_LITE_ENSURE(&context_, t != nullptr);
-    if (t->data_is_stale) {
-      TF_LITE_ENSURE(&context_, t->delegate != nullptr);
-      TF_LITE_ENSURE(&context_, t->buffer_handle != kTfLiteNullBufferHandle);
-      // This can be null if the delegate doesn't use its own buffer.
-      TF_LITE_ENSURE(&context_, t->delegate->CopyFromBufferHandle != nullptr);
-      t->delegate->CopyFromBufferHandle(
-          &context_, t->delegate, t->buffer_handle, t->data.raw, t->bytes);
-      t->data_is_stale = false;
-    }
-    return kTfLiteOk;
-  }
-
   // Set the delegate buffer handle to a tensor. It can be called in the
   // following cases:
   // 1. Set the buffer handle to a tensor that's not being written by a
@@ -439,7 +408,7 @@ class Interpreter {
   const char* OpProfilingString(const TfLiteRegistration& op_reg,
                                 const TfLiteNode* node) const {
     if (op_reg.profiling_string == nullptr) return nullptr;
-    return op_reg.profiling_string(&context_, node);
+    return op_reg.profiling_string(context_, node);
   }
 
   // Set the value of an external context.
@@ -450,131 +419,19 @@ class Interpreter {
   friend class InterpreterBuilder;
   friend class InterpreterTest;
 
-  // Prevent 'context_' from accessing functions that are only available to
-  // delegated kernels.
-  void SwitchToKernelContext();
-
-  // Add delegate-only functions to 'context_'.
-  void SwitchToDelegateContext();
-
-  // Give 'op_reg' a chance to initialize itself using the contents of
-  // 'buffer'.
-  void* OpInit(const TfLiteRegistration& op_reg, const char* buffer,
-               size_t length) {
-    if (op_reg.init == nullptr) return nullptr;
-    return op_reg.init(&context_, buffer, length);
-  }
-
-  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
-  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
-    if (op_reg.free == nullptr) return;
-    if (buffer) {
-      op_reg.free(&context_, buffer);
-    }
-  }
-
-  // Prepare the given 'node' for execution.
-  TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) {
-    if (op_reg.prepare == nullptr) return kTfLiteOk;
-    return op_reg.prepare(&context_, node);
+  Subgraph& primary_subgraph() {
+    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
-  // Invoke the operator represented by 'node'.
-  TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) {
-    if (op_reg.invoke == nullptr) return kTfLiteError;
-    return op_reg.invoke(&context_, node);
+  const Subgraph& primary_subgraph() const {
+    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
-  // Call OpPrepare() for as many ops as possible, allocating memory for their
-  // tensors. If an op containing dynamic tensors is found, preparation will be
-  // postponed until this function is called again. This allows the interpreter
-  // to wait until Invoke() to resolve the sizes of dynamic tensors.
-  TfLiteStatus PrepareOpsAndTensors();
-
-  // Call OpPrepare() for all ops starting at 'first_node'. Stop when a
-  // dynamic tensors is found or all ops have been prepared. Fill
-  // 'last_node_prepared' with the id of the op containing dynamic tensors, or
-  // the last in the graph.
-  TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index,
-                                    int* last_execution_plan_index_prepared);
-
   // Tensors needed by the interpreter. Use `AddTensors` to add more blank
   // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
   // `context_` whenever this std::vector is reallocated. Currently this
   // only happens in `AddTensors()`.
-  std::vector<TfLiteTensor> tensors_;
-
-  // Check if an array of tensor indices are valid with respect to the Tensor
-  // array.
-  // NOTE: this changes consistent_ to be false if indices are out of bounds.
-  TfLiteStatus CheckTensorIndices(const char* label, const int* indices,
-                                  int length);
-
-  // Compute the number of bytes required to represent a tensor with dimensions
-  // specified by the array dims (of length dims_size). Returns the status code
-  // and bytes.
-  TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size,
-                             size_t* bytes);
-
-  // Request an tensor be resized implementation. If the given tensor is of
-  // type kTfLiteDynamic it will also be allocated new memory.
-  TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size);
-
-  // Report a detailed error string (will be printed to stderr).
-  // TODO(aselle): allow user of class to provide alternative destinations.
-  void ReportErrorImpl(const char* format, va_list args);
-
-  // Entry point for C node plugin API to request an tensor be resized.
-  static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor,
-                                   TfLiteIntArray* new_size);
-  // Entry point for C node plugin API to report an error.
-  static void ReportError(TfLiteContext* context, const char* format, ...);
-
-  // Entry point for C node plugin API to add new tensors.
-  static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add,
-                                 int* first_new_tensor_index);
-
-  // WARNING: This is an experimental API and subject to change.
-  // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels
-  static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
-      TfLiteContext* context, TfLiteRegistration registration,
-      const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate);
-
-  // Update the execution graph to replace some of the nodes with stub
-  // nodes. Specifically any node index that has `nodes[index]==1` will be
-  // slated for replacement with a delegate kernel specified by registration.
-  // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller.
-  // WARNING: This is an experimental interface that is subject to change.
-  TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
-      TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
-      TfLiteDelegate* delegate);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Gets the internal pointer to a TensorFlow lite node by node_index.
-  TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node,
-                                      TfLiteRegistration** registration);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Entry point for C node plugin API to get a node by index.
-  static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*,
-                                             int node_index, TfLiteNode** node,
-                                             TfLiteRegistration** registration);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Gets an TfLiteIntArray* representing the execution plan. The interpreter
-  // owns this memory and it is only guaranteed to exist during the invocation
-  // of the delegate prepare.
-  TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Entry point for C node plugin API to get the execution plan.
-  static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context,
-                                       TfLiteIntArray** execution_plan);
-
-  // Retrieve an existing external context by type.
-  TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type);
-  static TfLiteExternalContext* GetExternalContext(
-      struct TfLiteContext* context, TfLiteExternalContextType type);
+  // std::vector<TfLiteTensor> tensors_;
 
   // Set the value of an external context.
   static void SetExternalContext(struct TfLiteContext* context,
@@ -591,105 +448,31 @@ class Interpreter {
     return ModifyGraphWithDelegate(owned_delegates_.back().get());
   }
 
-  // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra
-  // capacity. Calling this function may invalidate existing pointers to
-  // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
-  // more tensors won't invalidate the pointer to existing tensors.
-  void EnsureTensorsVectorCapacity() {
-    const size_t required_capacity = tensors_size() + kTensorsCapacityHeadroom;
-    if (required_capacity > tensors_.capacity()) {
-      tensors_.reserve(required_capacity);
-      context_.tensors = tensors_.data();
-    }
-  }
-
-  // The state of the Interpreter.
-  enum State {
-    // The interpreter isn't ready to be invoked.
-    // `AllocateTensor` need to be called to enter an invokable state.
-    kStateUninvokable = 0,
-    // The interpreter is ready to be invoked.
-    kStateInvokable,
-    // The interpreter is ready to be invoked, and graph can't be further
-    // modified. The interpreter will enter this state when calling
-    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
-    kStateInvokableAndImmutable,
-  };
-  State state_ = kStateUninvokable;
-
   // A pure C data structure used to communicate with the pure C plugin
   // interface. To avoid copying tensor metadata, this is also the definitive
   // structure to store tensors.
-  TfLiteContext context_;
-
-  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
-  // function pointers to actual implementation.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
-      nodes_and_registration_;
-
-  // Whether the model is consistent. That is to say if the inputs and outputs
-  // of every node and the global inputs and outputs are valid indexes into
-  // the tensor array.
-  bool consistent_ = true;
-
-  // Array of indices representing the tensors that are inputs to the
-  // interpreter.
-  std::vector<int> inputs_;
-
-  // Array of indices representing the tensors that are outputs to the
-  // interpreter.
-  std::vector<int> outputs_;
-
-  // Array of indices representing the tensors that are variable tensors.
-  std::vector<int> variables_;
+  // This is the primary subgraph context.
+  TfLiteContext* context_;
 
   // The error reporter delegate that tflite will forward queries errors to.
   ErrorReporter* error_reporter_;
 
-  // Index of the next node to prepare.
-  // During Invoke(), Interpreter will allocate input tensors first, which are
-  // known to be fixed size. Then it will allocate outputs from nodes as many
-  // as possible. When there is a node that produces dynamic sized tensor.
-  // Interpreter will stop allocating tensors, set the value of next allocate
-  // node id, and execute the node to generate the output tensor before continue
-  // to allocate successors. This process repeats until all nodes are executed.
-  // NOTE: this relies on the order of nodes that is in topological order.
-  int next_execution_plan_index_to_prepare_;
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // This is a list of node indices (to index into nodes_and_registration).
-  // This represents a valid topological sort (dependency ordered) execution
-  // plan. In particular, it is valid for this ordering to contain only a
-  // subset of the node indices.
-  std::vector<int> execution_plan_;
-
-  // In the future, we'd like a TfLiteIntArray compatible representation.
-  // TODO(aselle): replace execution_plan_ with this.
-  std::unique_ptr<TfLiteIntArray, TfLiteIntArrayDeleter> plan_cache_;
-
-  // Whether to delegate to NN API
-  std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
-
   // List of delegates that have been installed and are owned by this
   // interpreter instance. Useful if client delegate ownership is burdensome.
   // WARNING: This is an experimental API and subject to change.
   // TODO(b/116667551): Use TfLiteExternalContext for storing state.
   std::vector<TfLiteDelegatePtr> owned_delegates_;
 
-  std::unique_ptr<MemoryPlanner> memory_planner_;
-
   bool allow_buffer_handle_output_ = false;
 
-  // Tracking bit for whether a tensor was resized in the course of an op
-  // invocation. This is a useful hint to ensure that dynamic tensor outputs
-  // trigger downstream reallocation after op invocation.
-  bool tensor_resized_since_op_invoke_ = false;
-
   // Profiler for this interpreter instance.
   profiling::Profiler* profiler_ = nullptr;
 
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
+
+  // Subgraphs
+  std::vector<Subgraph> subgraphs_;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index 7f03c3ceba..2e0dc77dcd 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -38,7 +38,7 @@ class InterpreterTest : public ::testing::Test {
   }
 
  protected:
-  TfLiteContext* GetInterpreterContext() { return &interpreter_.context_; }
+  TfLiteContext* GetInterpreterContext() { return interpreter_.context_; }
 
   Interpreter interpreter_;
 };
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index a1be2a5abc..983665b7da 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -140,13 +140,13 @@ NNAPIDelegate::~NNAPIDelegate() {
   // ANeuralNetworksShutdown();
 }
 
-// Adds the tensors of the interpreter to the NN API model.
-TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter,
+// Adds the tensors of the subgraph to the NN API model.
+TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
   uint32_t next_id = 0;
-  for (size_t i = 0; i < interpreter->tensors_size(); i++) {
+  for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
     if ((*nnapi_ids)[i] == kOperandNotNeeded) continue;
 
@@ -156,7 +156,7 @@ TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter,
     // NNAPI requires 32-bit float scale to be zero, tflite doesn't care
     float scale = 0.0f;
     int32_t zeroPoint = 0;
-    TfLiteTensor* tensor = interpreter->tensor(i);
+    TfLiteTensor* tensor = subgraph->tensor(i);
     switch (tensor->type) {
       case kTfLiteNoType:
         // Tensors added during initialization of Ops don't have a type yet and
@@ -240,12 +240,12 @@ void MapAndAddTensorIds(const int* from_ids_buf, size_t from_ids_count,
 // Adds the operations and their parameters to the NN API model.
 // 'next-id' is the operand ID of the next operand of the model.
 TfLiteStatus AddOpsAndParams(
-    tflite::Interpreter* interpreter, ANeuralNetworksModel* nn_model,
+    tflite::Subgraph* subgraph, ANeuralNetworksModel* nn_model,
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
-  for (size_t i = 0; i < interpreter->nodes_size(); i++) {
-    const auto* node_and_registration = interpreter->node_and_registration(i);
+  for (size_t i = 0; i < subgraph->nodes_size(); i++) {
+    const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
     const TfLiteRegistration& registration = node_and_registration->second;
     tflite::BuiltinOperator builtin =
@@ -291,9 +291,9 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [interpreter, &nn_model, &next_id, &augmented_inputs,
-         &model_state_inputs, &model_state_outputs](int tensor_id) {
-          const TfLiteTensor* tensor = interpreter->tensor(tensor_id);
+        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
+         &model_state_outputs](int tensor_id) {
+          const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
@@ -388,11 +388,11 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [interpreter, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
-      const TfLiteTensor* tensor = interpreter->tensor(scratch_buffer_index);
+      const TfLiteTensor* tensor = subgraph->tensor(scratch_buffer_index);
       ANeuralNetworksOperandType operand_type{
           ANEURALNETWORKS_TENSOR_FLOAT32,
           static_cast<uint32_t>(tensor->dims->size),
@@ -584,7 +584,7 @@ TfLiteStatus AddOpsAndParams(
         // The permutation input tensor value dictates the output dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((node.inputs->size > 1) &&
-            (interpreter->tensor(node.inputs->data[1])->allocation_type !=
+            (subgraph->tensor(node.inputs->data[1])->allocation_type !=
              kTfLiteMmapRo)) {
           logError("NNAPI does not yet support dynamic tensors.");
           return kTfLiteError;
@@ -601,14 +601,13 @@ TfLiteStatus AddOpsAndParams(
           return kTfLiteError;
         }
         if ((node.inputs->size > 0) &&
-            (interpreter->tensor(node.inputs->data[0])->dims->size != 4)) {
+            (subgraph->tensor(node.inputs->data[0])->dims->size != 4)) {
           logError("NNAPI only supports input rank 4 for L2Normalization");
           return kTfLiteError;
         }
         break;
       case tflite::BuiltinOperator_HASHTABLE_LOOKUP:
-        if (interpreter->tensor(node.outputs->data[0])->type !=
-            kTfLiteFloat32) {
+        if (subgraph->tensor(node.outputs->data[0])->type != kTfLiteFloat32) {
           logError("NNAPI only support HASHTABLE_LOOKUP with float32 output",
                    builtin);
           return kTfLiteError;
@@ -707,7 +706,7 @@ TfLiteStatus AddOpsAndParams(
   return kTfLiteOk;
 }
 
-TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
+TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
   // TODO(aselle): This is not correct. need to handle resize invalidation.
@@ -719,7 +718,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
     // inputs and outputs and mark the mapping in tensor_id_to_nnapi_id with
     // kOperandIdNotSet. addTensorOperands will replace those with the
     // corresponding NNAPI operand ids and skip kOperandNotNeeded entries.
-    std::vector<int64_t> tensor_id_to_nnapi_id(interpreter->tensors_size(),
+    std::vector<int64_t> tensor_id_to_nnapi_id(subgraph->tensors_size(),
                                                kOperandNotNeeded);
     auto set_ids_to_not_set = [&tensor_id_to_nnapi_id](const int* buf,
                                                        size_t count) {
@@ -730,35 +729,31 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
         }
       }
     };
-    for (size_t i = 0; i < interpreter->nodes_size(); i++) {
-      const auto* node_and_registration = interpreter->node_and_registration(i);
+    for (size_t i = 0; i < subgraph->nodes_size(); i++) {
+      const auto* node_and_registration = subgraph->node_and_registration(i);
       const TfLiteNode& node = node_and_registration->first;
       set_ids_to_not_set(node.inputs->data, node.inputs->size);
       set_ids_to_not_set(node.outputs->data, node.outputs->size);
     }
-    set_ids_to_not_set(interpreter->inputs().data(),
-                       interpreter->inputs().size());
-    set_ids_to_not_set(interpreter->outputs().data(),
-                       interpreter->outputs().size());
+    set_ids_to_not_set(subgraph->inputs().data(), subgraph->inputs().size());
+    set_ids_to_not_set(subgraph->outputs().data(), subgraph->outputs().size());
 
     uint32_t next_id = 0;
     RETURN_ERROR_IF_TFLITE_FAILED(addTensorOperands(
-        interpreter, nn_model_, &next_id, &tensor_id_to_nnapi_id));
+        subgraph, nn_model_, &next_id, &tensor_id_to_nnapi_id));
     RETURN_ERROR_IF_TFLITE_FAILED(
-        AddOpsAndParams(interpreter, nn_model_, next_id, &model_states_inputs_,
+        AddOpsAndParams(subgraph, nn_model_, next_id, &model_states_inputs_,
                         &model_states_outputs_, tensor_id_to_nnapi_id));
 
     std::vector<uint32_t> augmented_inputs;
-    MapAndAddTensorIds(interpreter->inputs().data(),
-                       interpreter->inputs().size(), &augmented_inputs,
-                       tensor_id_to_nnapi_id);
+    MapAndAddTensorIds(subgraph->inputs().data(), subgraph->inputs().size(),
+                       &augmented_inputs, tensor_id_to_nnapi_id);
     augmented_inputs.insert(augmented_inputs.end(),
                             model_states_inputs_.begin(),
                             model_states_inputs_.end());
     std::vector<uint32_t> augmented_outputs;
-    MapAndAddTensorIds(interpreter->outputs().data(),
-                       interpreter->outputs().size(), &augmented_outputs,
-                       tensor_id_to_nnapi_id);
+    MapAndAddTensorIds(subgraph->outputs().data(), subgraph->outputs().size(),
+                       &augmented_outputs, tensor_id_to_nnapi_id);
     MapAndAddTensorIds(model_states_outputs_.data(),
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
@@ -771,7 +766,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
 
     if (GetAndroidSdkVersionCached() >= 28) {
       CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-          nn_model_, interpreter->GetAllowFp16PrecisionForFp32()));
+          nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
     CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
   }
@@ -782,9 +777,9 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
   return kTfLiteOk;
 }
 
-TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
+TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   if (!nn_model_) {
-    model_status_ = BuildGraph(interpreter);
+    model_status_ = BuildGraph(subgraph);
     if (model_status_ != kTfLiteOk) {
       logError("Failed to build graph for NNAPI");
     }
@@ -797,19 +792,19 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
   CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
-  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
-    int input = interpreter->inputs()[i];
+  for (size_t i = 0; i < subgraph->inputs().size(); i++) {
+    int input = subgraph->inputs()[i];
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
-    TfLiteTensor* tensor = interpreter->tensor(input);
+    TfLiteTensor* tensor = subgraph->tensor(input);
     CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
   // Tell nn api where to place final data.
-  for (size_t i = 0; i < interpreter->outputs().size(); i++) {
-    int output = interpreter->outputs()[i];
-    TfLiteTensor* tensor = interpreter->tensor(output);
+  for (size_t i = 0; i < subgraph->outputs().size(); i++) {
+    int output = subgraph->outputs()[i];
+    TfLiteTensor* tensor = subgraph->tensor(output);
     CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
@@ -818,16 +813,16 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
   // current invocation.
   for (size_t i = 0; i < model_states_outputs_.size(); i++) {
     int state_tensor_idx = model_states_outputs_[i];
-    TfLiteTensor* tensor = interpreter->tensor(state_tensor_idx);
+    TfLiteTensor* tensor = subgraph->tensor(state_tensor_idx);
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
     CHECK_NN(ANeuralNetworksExecution_setInput(
-        execution, i + interpreter->inputs().size(), nullptr, tensor->data.raw,
+        execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
     CHECK_NN(ANeuralNetworksExecution_setOutput(
-        execution, i + interpreter->outputs().size(), nullptr, tensor->data.raw,
+        execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
@@ -840,9 +835,9 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
 
 #if 0
   printf("From the NN API:\n");
-  TfLiteTensor* tensor = interpreter->tensor(interpreter->outputs()[0]);
+  TfLiteTensor* tensor = subgraph->tensor(subgraph->outputs()[0]);
   if (float* data =
-          interpreter->typed_tensor<float>(interpreter->outputs()[0])) {
+          subgraph->typed_tensor<float>(subgraph->outputs()[0])) {
     size_t num = tensor->bytes / sizeof(float);
     for (float* p = data; p < data + num; p++) {
       printf(" %f", *p);
diff --git a/tensorflow/lite/nnapi_delegate.h b/tensorflow/lite/nnapi_delegate.h
index 63b408c141..b4f8e4ecf3 100644
--- a/tensorflow/lite/nnapi_delegate.h
+++ b/tensorflow/lite/nnapi_delegate.h
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/interpreter.h"
 
 class ANeuralNetworksModel;
@@ -50,10 +51,10 @@ class NNAPIDelegate {
   ~NNAPIDelegate();
 
   // Convert a tflite graph to NNAPI
-  TfLiteStatus BuildGraph(Interpreter* interpreter);
+  TfLiteStatus BuildGraph(Subgraph* subgraph);
 
   // Run
-  TfLiteStatus Invoke(Interpreter* interpreter);
+  TfLiteStatus Invoke(Subgraph* subgraph);
 
   // Whether the current platform supports NNAPI delegation.
   static bool IsSupported();
diff --git a/tensorflow/lite/nnapi_delegate_disabled.cc b/tensorflow/lite/nnapi_delegate_disabled.cc
index 44dc21f1b6..a8f2c0bfe3 100644
--- a/tensorflow/lite/nnapi_delegate_disabled.cc
+++ b/tensorflow/lite/nnapi_delegate_disabled.cc
@@ -35,13 +35,11 @@ NNAPIDelegate::~NNAPIDelegate() {
 #undef UNUSED_MEMBER
 }
 
-TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
+TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   return kTfLiteError;
 }
 
-TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
-  return kTfLiteError;
-}
+TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) { return kTfLiteError; }
 
 bool NNAPIDelegate::IsSupported() { return false; }
 
diff --git a/tensorflow/lite/util.h b/tensorflow/lite/util.h
index 64a5b52e2f..dbb87528d0 100644
--- a/tensorflow/lite/util.h
+++ b/tensorflow/lite/util.h
@@ -52,6 +52,12 @@ bool EqualArrayAndTfLiteIntArray(const TfLiteIntArray* a, const int b_size,
 
 size_t CombineHashes(std::initializer_list<size_t> hashes);
 
+struct TfLiteIntArrayDeleter {
+  void operator()(TfLiteIntArray* a) {
+    if (a) TfLiteIntArrayFree(a);
+  }
+};
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_LITE_UTIL_H_
-- 
GitLab


From 41d29682e9c35802825b69996d512825990522a3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Nov 2018 22:15:30 -0800
Subject: [PATCH 0472/1554] Automated rollback of commit
 a562627db116a63607ea42da7a13d6802b4c7bf3

PiperOrigin-RevId: 221895215
---
 tensorflow/lite/BUILD                      |   2 -
 tensorflow/lite/core/subgraph.cc           | 976 ---------------------
 tensorflow/lite/core/subgraph.h            | 476 ----------
 tensorflow/lite/interpreter.cc             | 916 ++++++++++++++++++-
 tensorflow/lite/interpreter.h              | 289 +++++-
 tensorflow/lite/interpreter_test.cc        |   2 +-
 tensorflow/lite/nnapi_delegate.cc          |  87 +-
 tensorflow/lite/nnapi_delegate.h           |   5 +-
 tensorflow/lite/nnapi_delegate_disabled.cc |   6 +-
 tensorflow/lite/util.h                     |   6 -
 10 files changed, 1178 insertions(+), 1587 deletions(-)
 delete mode 100644 tensorflow/lite/core/subgraph.cc
 delete mode 100644 tensorflow/lite/core/subgraph.h

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index bb2c53b8c9..be84fc5db1 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -131,7 +131,6 @@ cc_library(
     name = "framework",
     srcs = [
         "allocation.cc",
-        "core/subgraph.cc",
         "graph_info.cc",
         "interpreter.cc",
         "model.cc",
@@ -156,7 +155,6 @@ cc_library(
         "allocation.h",
         "context.h",
         "context_util.h",
-        "core/subgraph.h",
         "error_reporter.h",
         "graph_info.h",
         "interpreter.h",
diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
deleted file mode 100644
index 05a60962b1..0000000000
--- a/tensorflow/lite/core/subgraph.cc
+++ /dev/null
@@ -1,976 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/core/subgraph.h"
-#include "tensorflow/lite/arena_planner.h"
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/context_util.h"
-#include "tensorflow/lite/graph_info.h"
-#include "tensorflow/lite/nnapi_delegate.h"
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-
-namespace {
-TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
-                           const TfLiteRegistration& registration,
-                           int node_index, const char* message) {
-  context->ReportError(
-      context, "Node number %d (%s) %s.\n", node_index,
-      registration.custom_name
-          ? registration.custom_name
-          : EnumNameBuiltinOperator(
-                static_cast<BuiltinOperator>(registration.builtin_code)),
-      message);
-  return kTfLiteError;
-}
-
-// Stub method which returns kTfLiteError when the function is forbidden.
-// We're registrating this function to several different function to save
-// compiled binary size. Please note the restrictions:
-// * The type of first parameter have to be `TfLiteContext*`.
-// * All paramteters must be trivailly destructible. (E.g. No C++ class)
-TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) {
-  context->ReportError(context,
-                       "The function is forbidden if not calling in delegate.");
-  return kTfLiteError;
-}
-
-// Set the ForbiddenContextFunction to a compatible function pointer.
-template <typename FunctionType>
-void SetForbiddenContextFunction(FunctionType* func) {
-  *func = reinterpret_cast<FunctionType>(ForbiddenContextFunction);
-}
-
-// Returns true if at least one tensor in the given list is kTfLiteDynamic.
-template <typename TensorIntArray>
-bool HasDynamicTensorImpl(const TfLiteContext& context,
-                          const TensorIntArray& int_array) {
-  for (int i : int_array) {
-    const TfLiteTensor& tensor = context.tensors[i];
-    if (tensor.allocation_type == kTfLiteDynamic) {
-      return true;
-    }
-  }
-  return false;
-}
-
-bool HasDynamicTensor(const TfLiteContext& context,
-                      const TfLiteIntArray* int_array) {
-  return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
-}
-
-}  // namespace
-
-// A trivial implementation of GraphInfo around the Interpreter.
-// NOTE: this interpreter info represents the subset of the
-// graph that is executed according to execution plan. Thus,
-// the indices are execution plan indices rather than raw node
-// indices.
-class InterpreterInfo : public GraphInfo {
- public:
-  explicit InterpreterInfo(Subgraph* subgraph) : subgraph_(subgraph) {}
-
-  size_t num_tensors() const override { return subgraph_->tensors().size(); }
-  TfLiteTensor* tensor(size_t index) override {
-    return &subgraph_->tensors()[index];
-  }
-  size_t num_nodes() const override {
-    return subgraph_->execution_plan().size();
-  }
-  const TfLiteNode& node(size_t index) const override {
-    int node_index = subgraph_->execution_plan()[index];
-    return subgraph_->nodes_and_registration()[node_index].first;
-  }
-  const std::vector<int>& inputs() const override {
-    return subgraph_->inputs();
-  }
-  const std::vector<int>& outputs() const override {
-    return subgraph_->outputs();
-  }
-  const std::vector<int>& variables() const override {
-    return subgraph_->variables();
-  }
-
- public:
-  Subgraph* subgraph_;
-};
-
-Subgraph::Subgraph(ErrorReporter* error_reporter,
-                   TfLiteExternalContext** external_contexts)
-    : context_(&owned_context_),
-      error_reporter_(error_reporter),
-      next_execution_plan_index_to_prepare_(0),
-      external_contexts_(external_contexts) {
-  context_->impl_ = static_cast<void*>(this);
-  context_->ResizeTensor = ResizeTensor;
-  context_->ReportError = ReportErrorC;
-  context_->AddTensors = AddTensors;
-  context_->tensors = nullptr;
-  context_->tensors_size = 0;
-  context_->allow_fp32_relax_to_fp16 = false;
-  context_->recommended_num_threads = -1;
-  context_->GetExternalContext = GetExternalContext;
-  context_->SetExternalContext = SetExternalContext;
-
-  // Reserve some space for the tensors to avoid excessive resizing.
-  tensors_.reserve(kTensorsReservedCapacity);
-  nodes_and_registration().reserve(kTensorsReservedCapacity);
-  // Invalid to call these these except from TfLiteDelegate
-  SwitchToKernelContext();
-}
-
-Subgraph::~Subgraph() {
-  for (auto& node_and_reg : nodes_and_registration_) {
-    TfLiteNode& node = node_and_reg.first;
-    TfLiteIntArrayFree(node.inputs);
-    TfLiteIntArrayFree(node.outputs);
-    TfLiteIntArrayFree(node.temporaries);
-    if (node.builtin_data) free(node.builtin_data);
-    OpFree(node_and_reg.second, node.user_data);
-    node.builtin_data = nullptr;
-  }
-
-  for (size_t i = 0; i < context_->tensors_size; i++) {
-    TfLiteTensor* tensor = &context_->tensors[i];
-    if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
-        tensor->delegate->FreeBufferHandle != nullptr) {
-      tensor->delegate->FreeBufferHandle(context_, tensor->delegate,
-                                         &tensor->buffer_handle);
-    }
-    TfLiteTensorFree(tensor);
-  }
-}
-
-TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels(
-    TfLiteContext* context, TfLiteRegistration registration,
-    const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
-  return static_cast<Subgraph*>(context->impl_)
-      ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace,
-                                              delegate);
-}
-
-namespace {
-
-// Copy a std::vector<int> to an existing TfLiteIntArray.
-// This is a low-level data manipulation function, and it's caller's
-// responsibility to ensure TfLiteIntArray has enough size.
-void CopyVectorToTfLiteIntArray(const std::vector<int>& vec,
-                                TfLiteIntArray* arr) {
-  arr->size = vec.size();
-  memcpy(arr->data, vec.data(), sizeof(int) * arr->size);
-}
-
-// This function allocates a continuous memory space that contains a
-// TfLiteDelegateParams followed by a several TfLiteIntArray.
-// When calling `free` at TfLiteDelegateParams*, all the allocated space
-// will be freed together.
-//
-// +-----------------------------------+
-// | TfLiteDelegateParams              |
-// | TfLiteDelegate* delegate;         |
-// | TfLiteIntArray* nodes_to_replace; |--\
-// | TfLiteIntArray* input_tensors;    |--+--\
-// | TfLiteIntArray* output_tensors;   |--+--+--\
-// +-----------------------------------+  |  |  |
-// | TfLiteIntArray (variable size)    |<-/  |  |
-// +-----------------------------------+     |  |
-// | TfLiteIntArray (variable size)    |<----/  |
-// +-----------------------------------+        |
-// | TfLiteIntArray (variable size)    |<-------/
-// +-----------------------------------+
-TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate,
-                                           const NodeSubset& node_subset) {
-  // Step 1: Calculate the allocation size.
-  int allocation_size = sizeof(TfLiteDelegateParams);
-
-  int nodes_to_replace_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size());
-  allocation_size += nodes_to_replace_size;
-
-  int input_tensors_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size());
-  allocation_size += input_tensors_size;
-
-  int output_tensors_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size());
-  allocation_size += output_tensors_size;
-
-  // Step 2: Allocate the memory.
-  // Use `char*` for conveniently step through the allocated space by bytes.
-  char* allocation = reinterpret_cast<char*>(malloc(allocation_size));
-
-  // Step 3: Fill all data structures structures.
-  TfLiteDelegateParams* params =
-      reinterpret_cast<TfLiteDelegateParams*>(allocation);
-  params->delegate = delegate;
-  allocation += sizeof(TfLiteDelegateParams);
-
-  params->nodes_to_replace = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace);
-  allocation += nodes_to_replace_size;
-
-  params->input_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors);
-  allocation += input_tensors_size;
-
-  params->output_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.output_tensors,
-                             params->output_tensors);
-  allocation += output_tensors_size;
-
-  return params;
-}
-
-}  // namespace
-
-TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels(
-    TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
-    TfLiteDelegate* delegate) {
-  // Annotate the registration as DELEGATE op.
-  registration.builtin_code = BuiltinOperator_DELEGATE;
-
-  // Analyze the graph to find all independent node_subsets that are either
-  // fully not-this-delegate or this-delegate computation.
-  InterpreterInfo info(this);
-  std::vector<NodeSubset> node_subsets;
-  PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace,
-                                           &node_subsets);
-
-  execution_plan_.clear();
-
-  for (auto& node_subset : node_subsets) {
-    // Subsets calimed by the delegate should have a "macro" op created, the
-    // other node_subsets (kTfNonPartition) just have their nodes added back to
-    // the execution plan.
-    switch (node_subset.type) {
-      case NodeSubset::kTfNonPartition:
-        for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end();
-             ++it) {
-          execution_plan_.push_back(*it);
-        }
-        break;
-      case NodeSubset::kTfPartition: {
-        int node_index;
-
-        TfLiteDelegateParams* params =
-            CreateDelegateParams(delegate, node_subset);
-        TF_LITE_ENSURE_STATUS(AddNodeWithParameters(
-            node_subset.input_tensors, node_subset.output_tensors, nullptr, 0,
-            params, &registration, &node_index));
-
-        // Initialize the output tensors's delegate-related fields.
-        for (int tensor_index : node_subset.output_tensors) {
-          TfLiteTensor* tensor = &tensors_[tensor_index];
-          TF_LITE_ENSURE(context_, tensor->delegate == nullptr ||
-                                       tensor->delegate == delegate);
-          tensor->delegate = delegate;
-        }
-
-        // Associate the node with the delegate.
-        TfLiteNode* node = &nodes_and_registration_[node_index].first;
-        node->delegate = delegate;
-      } break;
-      case NodeSubset::kTfUnexplored:
-        return kTfLiteError;
-        break;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteExternalContext* Subgraph::GetExternalContext(
-    TfLiteExternalContextType type) {
-  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
-    return external_contexts_[type];
-  }
-  return nullptr;
-}
-
-TfLiteExternalContext* Subgraph::GetExternalContext(
-    struct TfLiteContext* context, TfLiteExternalContextType type) {
-  return static_cast<Subgraph*>(context->impl_)->GetExternalContext(type);
-}
-
-void Subgraph::SetExternalContext(TfLiteExternalContextType type,
-                                  TfLiteExternalContext* ctx) {
-  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
-    external_contexts_[type] = ctx;
-  }
-}
-
-void Subgraph::SetExternalContext(struct TfLiteContext* context,
-                                  TfLiteExternalContextType type,
-                                  TfLiteExternalContext* ctx) {
-  return static_cast<Subgraph*>(context->impl_)->SetExternalContext(type, ctx);
-}
-
-// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns
-// this memory and it is only guaranteed to exist during the invocation of the
-// delegate prepare.
-TfLiteStatus Subgraph::GetExecutionPlan(TfLiteIntArray** execution_plan) {
-  // TODO(aselle): Do not make a copy here
-  plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size()));
-  *execution_plan = plan_cache_.get();
-  static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]),
-                "TfLiteIntArray and execution_plan do not contain same type.");
-  std::memcpy(plan_cache_->data, execution_plan_.data(),
-              sizeof(plan_cache_->data[0]) * execution_plan_.size());
-  return kTfLiteOk;
-}
-
-// WARNING: This is an experimental interface that is subject to change.
-// Entry point for C node plugin API to get the execution plan
-TfLiteStatus Subgraph::GetExecutionPlan(struct TfLiteContext* context,
-                                        TfLiteIntArray** execution_plan) {
-  return static_cast<Subgraph*>(context->impl_)
-      ->GetExecutionPlan(execution_plan);
-}
-
-TfLiteStatus Subgraph::SetInputs(std::vector<int> inputs) {
-  TF_LITE_ENSURE_OK(&context_,
-                    CheckTensorIndices("inputs", inputs.data(), inputs.size()));
-  inputs_ = std::move(inputs);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::SetOutputs(std::vector<int> outputs) {
-  TF_LITE_ENSURE_OK(
-      &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
-  outputs_ = std::move(outputs);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::SetVariables(std::vector<int> variables) {
-  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
-                                                  variables.size()));
-  variables_ = std::move(variables);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices,
-                                          int length) {
-  // Making sure kOptionalTensor is not re-defined to something other than -1.
-  static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1");
-
-  for (int i = 0; i < length; i++) {
-    int index = indices[i];
-    // Continue if index == kOptionalTensor before additional comparisons below,
-    // size_t(-1) is always >= context_tensors_size.
-    if (index == kOptionalTensor) {
-      continue;
-    }
-    if (index < 0 || static_cast<size_t>(index) >= context_->tensors_size) {
-      ReportError("Invalid tensor index %d in %s\n", index, label);
-      consistent_ = false;
-      return kTfLiteError;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims,
-                                     size_t dims_size, size_t* bytes) {
-  // TODO(aselle): Check for overflow here using overflow.h in TensorFlow
-  // MultiplyWithoutOverflow.
-  TF_LITE_ENSURE(context_, bytes != nullptr);
-  size_t count = 1;
-  for (int k = 0; k < dims_size; k++) count *= dims[k];
-  switch (type) {
-    case kTfLiteFloat32:
-      *bytes = sizeof(float) * count;
-      break;
-    case kTfLiteInt16:
-      *bytes = sizeof(int16_t) * count;
-      break;
-    case kTfLiteInt32:
-      *bytes = sizeof(int32_t) * count;
-      break;
-    case kTfLiteUInt8:
-      *bytes = sizeof(uint8_t) * count;
-      break;
-    case kTfLiteInt64:
-      *bytes = sizeof(int64_t) * count;
-      break;
-    case kTfLiteBool:
-      *bytes = sizeof(bool) * count;
-      break;
-    case kTfLiteComplex64:
-      *bytes = sizeof(std::complex<float>) * count;
-      break;
-    case kTfLiteInt8:
-      *bytes = sizeof(int8_t) * count;
-      break;
-    default:
-      ReportError(
-          "Only float32, int8, int16, int32, int64, uint8, bool, complex64 "
-          "supported currently.");
-      return kTfLiteError;
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::AllocateTensors() {
-  if (!consistent_) {
-    ReportError("AllocateTensors() called on inconsistent model.");
-    return kTfLiteError;
-  }
-
-  // Explicit (re)allocation is necessary if nodes have been changed or tensors
-  // have been resized. For inputs marked as dynamic, we can't short-circuit the
-  // allocation as the client may have done the resize manually.
-  if (state_ != kStateUninvokable &&
-      !HasDynamicTensorImpl(*context_, inputs())) {
-    return kTfLiteOk;
-  }
-
-  next_execution_plan_index_to_prepare_ = 0;
-  if (memory_planner_) {
-    TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations());
-  }
-
-  TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-
-  state_ = kStateInvokable;
-
-  // Reset the variable tensors to zero after (re)allocating the tensors.
-  // Developers shouldn't rely on the side effect of this function to reset
-  // variable tesnsors. They should call `ResetVariableTensors` directly
-  // instead.
-  ResetVariableTensors();
-
-  return kTfLiteOk;
-}
-
-// TODO(ycling): Support non-zero default values.
-TfLiteStatus Subgraph::ResetVariableTensors() {
-  for (auto& tensor : tensors_) {
-    if (!tensor.is_variable) {
-      continue;
-    }
-
-    // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be
-    // allocated after the initial `PrepareOpsAndTensors()` is called.
-    TF_LITE_ENSURE_EQ(context_, tensor.allocation_type,
-                      kTfLiteArenaRwPersistent);
-    TF_LITE_ENSURE(context_, tensor.data.raw != nullptr);
-
-    memset(tensor.data.raw, 0, tensor.bytes);
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::AddNodeWithParameters(
-    const std::vector<int>& inputs, const std::vector<int>& outputs,
-    const char* init_data, size_t init_data_size, void* builtin_data,
-    const TfLiteRegistration* registration, int* node_index) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError("AddNodeWithParameters is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-  state_ = kStateUninvokable;
-
-  std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
-                                                              free);
-
-  TF_LITE_ENSURE_OK(context_, CheckTensorIndices("node inputs", inputs.data(),
-                                                 inputs.size()));
-  TF_LITE_ENSURE_OK(
-      &context_,
-      CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
-
-  int new_node_index = nodes_and_registration_.size();
-  if (node_index) *node_index = new_node_index;
-  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
-  auto& node_and_reg = nodes_and_registration_.back();
-  TfLiteNode& node = node_and_reg.first;
-  if (node.inputs) TfLiteIntArrayFree(node.inputs);
-  if (node.outputs) TfLiteIntArrayFree(node.outputs);
-  if (node.temporaries) TfLiteIntArrayFree(node.temporaries);
-
-  // NOTE, here we are not using move semantics yet, since our internal
-  // representation isn't std::vector, but in the future we would like to avoid
-  // copies, so we want the interface to take r-value references now.
-  node.inputs = ConvertVectorToTfLiteIntArray(inputs);
-  node.outputs = ConvertVectorToTfLiteIntArray(outputs);
-  node.temporaries = TfLiteIntArrayCreate(0);
-  if (init_data) {
-    node.user_data = OpInit(*registration, init_data, init_data_size);
-  } else {
-    node.user_data =
-        OpInit(*registration,
-               reinterpret_cast<const char*>(builtin_data_deleter.get()), 0);
-  }
-
-  node.builtin_data = builtin_data_deleter.release();
-  // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size`
-  // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels.
-
-  if (registration->builtin_code == BuiltinOperator_CUSTOM) {
-    // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer
-    // `Operator` table is passed in.
-    node.custom_initial_data = init_data;
-    node.custom_initial_data_size = init_data_size;
-  } else {
-    node.custom_initial_data = nullptr;
-    node.custom_initial_data_size = 0;
-  }
-
-  node.delegate = nullptr;
-  node_and_reg.second = *registration;
-  execution_plan_.push_back(new_node_index);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::ResizeInputTensor(int tensor_index,
-                                         const std::vector<int>& dims) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError("ResizeInputTensor is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-
-  // TODO(aselle): All bounds checks can be implemented as one-sided bounds
-  // checks by casting to unsigned for efficiency. Profile before doing this.
-  TF_LITE_ENSURE(context_,
-                 tensor_index < context_->tensors_size && tensor_index >= 0);
-  TfLiteTensor* tensor = &context_->tensors[tensor_index];
-
-  // Short-circuit the state change if the dimensions don't change, avoiding
-  // unnecessary (re)allocations.
-  if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) {
-    return kTfLiteOk;
-  }
-
-  state_ = kStateUninvokable;
-  return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims));
-}
-
-TfLiteStatus Subgraph::PrepareOpsStartingAt(
-    int first_execution_plan_index, int* last_execution_plan_index_prepared) {
-  for (int execution_plan_index = first_execution_plan_index;
-       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
-    int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
-    const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
-    EnsureTensorsVectorCapacity();
-    if (OpPrepare(registration, &node) == kTfLiteError) {
-      return ReportOpError(context_, node, registration, node_index,
-                           "failed to prepare");
-    }
-
-    *last_execution_plan_index_prepared = execution_plan_index;
-
-    // Discontinue if the node has dynamic outputs. Note that we don't
-    // stop for dynamic temporary tensors since they won't affect the
-    // sizes of other tensors in the graph.
-    if (HasDynamicTensor(*context_, node.outputs)) {
-      break;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::PrepareOpsAndTensors() {
-  if (!memory_planner_) {
-    memory_planner_.reset(new ArenaPlanner(
-        context_, std::unique_ptr<GraphInfo>(new InterpreterInfo(this)),
-        /*preserve_inputs=*/true, /*preserve_intermediates*/ false));
-    memory_planner_->PlanAllocations();
-  }
-
-  int last_exec_plan_index_prepared = 0;
-
-  TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt(
-      next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared));
-  TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations(
-      next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared));
-
-  next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1;
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::Invoke() {
-  if (!consistent_) {
-    ReportError("Invoke called on model that is not consistent.");
-    return kTfLiteError;
-  }
-
-  TfLiteStatus status = kTfLiteOk;
-  if (state_ == kStateUninvokable) {
-    ReportError("Invoke called on model that is not ready.");
-    return kTfLiteError;
-  }
-
-  if (nnapi_delegate_) {
-    if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) {
-      TF_LITE_ENSURE_OK(context_, nnapi_delegate_->Invoke(this));
-      return kTfLiteOk;
-    } else {
-      // TODO(aselle): In the future, we would like this to be an
-      // automatic tflite CPU fallback.
-      ReportError(
-          "NNAPI was requested, but dependent sized tensors "
-          "being used.\n");
-      return kTfLiteError;
-    }
-  }
-
-  // Invocations are always done in node order.
-  // Note that calling Invoke repeatedly will cause the original memory plan to
-  // be reused, unless either ResizeInputTensor() or AllocateTensors() has been
-  // called.
-  for (int execution_plan_index = 0;
-       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
-    if (execution_plan_index == next_execution_plan_index_to_prepare_) {
-      TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-      TF_LITE_ENSURE(context_, next_execution_plan_index_to_prepare_ >=
-                                   execution_plan_index);
-    }
-    int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
-    const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
-    SCOPED_OPERATOR_PROFILE(profiler_, node_index);
-
-    // TODO(ycling): This is an extra loop through inputs to check if the data
-    // need to be copied from Delegate buffer to raw memory, which is often not
-    // needed. We may want to cache this in prepare to know if this needs to be
-    // done for a node or not.
-    for (int i = 0; i < node.inputs->size; ++i) {
-      int tensor_index = node.inputs->data[i];
-      if (tensor_index == kOptionalTensor) {
-        continue;
-      }
-      TfLiteTensor* tensor = &tensors_[tensor_index];
-      if (tensor->delegate && tensor->delegate != node.delegate &&
-          tensor->data_is_stale) {
-        EnsureTensorDataIsReadable(tensor_index);
-      }
-    }
-
-    EnsureTensorsVectorCapacity();
-    tensor_resized_since_op_invoke_ = false;
-    if (OpInvoke(registration, &node) == kTfLiteError) {
-      status = ReportOpError(context_, node, registration, node_index,
-                             "failed to invoke");
-    }
-
-    // Force execution prep for downstream ops if the latest op triggered the
-    // resize of a dynamic tensor.
-    if (tensor_resized_since_op_invoke_ &&
-        HasDynamicTensor(*context_, node.outputs)) {
-      next_execution_plan_index_to_prepare_ = execution_plan_index + 1;
-    }
-  }
-
-  return status;
-}
-
-TfLiteStatus Subgraph::ResizeTensor(TfLiteContext* context,
-                                    TfLiteTensor* tensor,
-                                    TfLiteIntArray* new_size) {
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function ResizeTensorImpl
-  // (this function is static).
-  return static_cast<Subgraph*>(context->impl_)
-      ->ResizeTensorImpl(tensor, new_size);
-}
-
-void Subgraph::ReportErrorImpl(const char* format, va_list args) {
-  error_reporter_->Report(format, args);
-}
-
-void Subgraph::ReportErrorC(TfLiteContext* context, const char* format, ...) {
-  va_list args;
-  va_start(args, format);
-  auto* f = static_cast<Subgraph*>(context->impl_);
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Subgraph to call into the member function ReportErrorImpl
-  // (this function is static).
-  f->ReportErrorImpl(format, args);
-  va_end(args);
-}
-
-// Entry point for C node plugin API to report an error.
-void Subgraph::ReportError(const char* format, ...) {
-  va_list args;
-  va_start(args, format);
-  auto* f = static_cast<Subgraph*>(context_->impl_);
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Subgraph to call into the member function ReportErrorImpl
-  // (this function is static).
-  f->ReportErrorImpl(format, args);
-  va_end(args);
-}
-
-TfLiteStatus Subgraph::AddTensors(int tensors_to_add,
-                                  int* first_new_tensor_index) {
-  const size_t base_index = tensors_.size();
-  if (first_new_tensor_index) *first_new_tensor_index = base_index;
-  tensors_.resize(tensors_.size() + tensors_to_add);
-  for (size_t i = base_index; i < tensors_.size(); i++) {
-    memset(&tensors_[i], 0, sizeof(tensors_[i]));
-    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
-  }
-  context_->tensors = tensors_.data();
-  context_->tensors_size = tensors_.size();
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::AddTensors(TfLiteContext* context, int tensors_to_add,
-                                  int* first_new_tensor_index) {
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function AddTensors
-  // (this function is static).
-  return static_cast<Subgraph*>(context->impl_)
-      ->AddTensors(tensors_to_add, first_new_tensor_index);
-}
-
-TfLiteStatus Subgraph::GetNodeAndRegistration(
-    int node_index, TfLiteNode** node, TfLiteRegistration** registration) {
-  TF_LITE_ENSURE(context_, node_index >= 0);
-  auto nodes_size = nodes_and_registration_.size();
-  TF_LITE_ENSURE(context_, static_cast<size_t>(node_index) < nodes_size);
-  TF_LITE_ENSURE(context_, node != nullptr && registration != nullptr);
-  auto& node_and_reg = nodes_and_registration_[node_index];
-  *node = &node_and_reg.first;
-  *registration = &node_and_reg.second;
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::GetNodeAndRegistration(
-    struct TfLiteContext* context, int node_index, TfLiteNode** node,
-    TfLiteRegistration** registration) {
-  return static_cast<Subgraph*>(context->impl_)
-      ->GetNodeAndRegistration(node_index, node, registration);
-}
-
-TfLiteStatus Subgraph::SetTensorParametersReadOnly(
-    int tensor_index, TfLiteType type, const char* name, const size_t rank,
-    const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
-    size_t bytes, const Allocation* allocation) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(
-        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-
-  TF_LITE_ENSURE(context_,
-                 tensor_index < context_->tensors_size && tensor_index >= 0);
-  // For most tensors we know exactly how much memory is necessary so we can
-  // ensure the buffer is large enough. However, we need to skip string tensors
-  // because their sizes change with the contents of the individual strings.
-  if (type != kTfLiteString) {
-    size_t required_bytes;
-    TF_LITE_ENSURE_OK(context_,
-                      BytesRequired(type, dims, rank, &required_bytes));
-    TF_LITE_ENSURE_EQ(context_, required_bytes, bytes);
-  }
-
-  TfLiteTensor& tensor = context_->tensors[tensor_index];
-  if (type == tensor.type &&
-      EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
-    // Fast path which does not invalidate the invokable property.
-    TfLiteTensorDataFree(&tensor);
-    tensor.data.raw = const_cast<char*>(buffer);
-    if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
-    tensor.params = quantization;
-    tensor.allocation_type = kTfLiteMmapRo;
-    tensor.allocation = allocation;
-  } else {
-    state_ = kStateUninvokable;
-    TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                      quantization, const_cast<char*>(buffer), bytes,
-                      kTfLiteMmapRo, allocation, false, &tensor);
-  }
-  return kTfLiteOk;
-}
-
-// Set description of inputs/outputs/data/fptrs for node `node_index`.
-// This variant assumes an external buffer has been allocated of size
-// bytes. The lifetime of buffer must be ensured to be greater or equal
-// to Interpreter.
-TfLiteStatus Subgraph::SetTensorParametersReadWrite(
-    int tensor_index, TfLiteType type, const char* name, const size_t rank,
-    const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(
-        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-  TF_LITE_ENSURE(context_,
-                 tensor_index < context_->tensors_size && tensor_index >= 0);
-  size_t required_bytes = 0;
-  if (type != kTfLiteString) {
-    // These types will be allocated in our arena so we need to record how
-    // many bytes we will need based on the dimensions. String tensors are
-    // allocated dynamically and we can't know ahead of time how much space
-    // they will require.
-    TF_LITE_ENSURE_OK(context_,
-                      BytesRequired(type, dims, rank, &required_bytes));
-  }
-
-  TfLiteAllocationType allocation_type = kTfLiteArenaRw;
-  if (type == kTfLiteString) {
-    if (is_variable) {
-      // We don't have a real use case for string variable tensor.
-      ReportError("String variable tensor isn't supported.");
-      return kTfLiteError;
-    }
-    allocation_type = kTfLiteDynamic;
-  } else if (is_variable) {
-    allocation_type = kTfLiteArenaRwPersistent;
-  }
-
-  TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                    quantization,
-                    /*buffer=*/nullptr, required_bytes, allocation_type,
-                    nullptr, is_variable, &context_->tensors[tensor_index]);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::SetExecutionPlan(const std::vector<int>& new_plan) {
-  for (int node_index : new_plan) {
-    TF_LITE_ENSURE(context_, node_index >= 0 &&
-                                 node_index < nodes_and_registration_.size());
-  }
-  execution_plan_ = new_plan;
-  return kTfLiteOk;
-}
-
-TfLiteStatus Subgraph::ResizeTensorImpl(TfLiteTensor* tensor,
-                                        TfLiteIntArray* new_size) {
-  // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too.
-  if (tensor->allocation_type == kTfLiteArenaRw ||
-      tensor->allocation_type == kTfLiteDynamic ||
-      tensor->allocation_type == kTfLiteArenaRwPersistent) {
-    tensor_resized_since_op_invoke_ |=
-        TfLiteIntArrayEqual(tensor->dims, new_size) == 0;
-    if (tensor->type != kTfLiteString) {
-      size_t bytesRequired;
-      TfLiteStatus status = BytesRequired(tensor->type, new_size->data,
-                                          new_size->size, &bytesRequired);
-      if (status != kTfLiteOk) {
-        TfLiteIntArrayFree(new_size);
-        return kTfLiteError;
-      }
-
-      // Realloc space for kTfLiteDynamic tensors.
-      TfLiteTensorRealloc(bytesRequired, tensor);
-      tensor->bytes = bytesRequired;
-    }
-    if (tensor->dims) TfLiteIntArrayFree(tensor->dims);
-    tensor->dims = new_size;
-
-    if (tensor->allocation_type != kTfLiteDynamic) {
-      tensor->data.raw = nullptr;
-    }
-  } else {
-    // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore
-    // of fixed size.
-    TfLiteIntArrayFree(new_size);
-    ReportError("Attempting to resize a fixed-size tensor.");
-    return kTfLiteError;
-  }
-  return kTfLiteOk;
-}
-
-void Subgraph::UseNNAPI(bool enable) {
-  // TODO(aselle): This is a workaround for finding if NNAPI exists.
-  // We also need to make sure getLibraryHandle() is renamed to be NNAPI
-  // prefixed.
-  if (!NNAPIDelegate::IsSupported()) enable = false;
-  if (!enable) {
-    nnapi_delegate_.reset();
-  } else if (!nnapi_delegate_) {
-    nnapi_delegate_.reset(new NNAPIDelegate);
-  }
-}
-
-void Subgraph::SwitchToDelegateContext() {
-  context_->GetNodeAndRegistration = GetNodeAndRegistration;
-  context_->ReplaceNodeSubsetsWithDelegateKernels =
-      ReplaceNodeSubsetsWithDelegateKernels;
-  context_->GetExecutionPlan = GetExecutionPlan;
-}
-
-void Subgraph::SwitchToKernelContext() {
-  context_->GetNodeAndRegistration = [](struct TfLiteContext* context,
-                                        int node_index, TfLiteNode** node,
-                                        TfLiteRegistration** registration) {
-    return ForbiddenContextFunction(context);
-  };
-  context_->ReplaceNodeSubsetsWithDelegateKernels =
-      [](TfLiteContext* context, TfLiteRegistration registration,
-         const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
-        return ForbiddenContextFunction(context);
-      };
-  context_->GetExecutionPlan = [](struct TfLiteContext* context,
-                                  TfLiteIntArray**) {
-    return ForbiddenContextFunction(context);
-  };
-}
-
-TfLiteStatus Subgraph::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
-  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
-    int last_execution_plan_index_prepared;
-    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
-                                     0, &last_execution_plan_index_prepared));
-
-    bool has_dynamic_tensors = true;
-    // Dynamic tensors exist if not all nodes can be prepared.
-    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
-      // If all the nodes can be prepared, check if the last node has dynamic
-      // tensors.
-      int node_index = execution_plan_[last_execution_plan_index_prepared];
-      TfLiteNode& node = nodes_and_registration_[node_index].first;
-      if (!HasDynamicTensor(*context_, node.outputs)) {
-        has_dynamic_tensors = false;
-      }
-    }
-    if (has_dynamic_tensors) {
-      ReportError(
-          "Attempting to use a delegate that only supports static-sized "
-          "tensors with a graph that has dynamic-sized tensors.");
-      return kTfLiteError;
-    }
-  }
-
-  // TODO(aselle): Consider if it is worth storing pointers to delegates.
-  // Setup additional context interface.
-  SwitchToDelegateContext();
-
-  TfLiteStatus status = delegate->Prepare(context_, delegate);
-
-  // Remove additional context info.
-  SwitchToKernelContext();
-
-  TF_LITE_ENSURE_OK(context_, status);
-
-  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
-    // Reset the state to force tensor/op reallocation.
-    state_ = kStateUninvokable;
-    TF_LITE_ENSURE_OK(context_, AllocateTensors());
-    TF_LITE_ENSURE_EQ(context_, state_, kStateInvokable);
-    // After using a delegate which doesn't support dynamic tensors, make the
-    // entire graph immutable.
-    state_ = kStateInvokableAndImmutable;
-  }
-
-  return status;
-}
-
-}  // namespace tflite
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
deleted file mode 100644
index 9783747ff6..0000000000
--- a/tensorflow/lite/core/subgraph.h
+++ /dev/null
@@ -1,476 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_LITE_CORE_SUBGRAPH_H_
-#define TENSORFLOW_LITE_CORE_SUBGRAPH_H_
-
-#include <cstdlib>
-#include <vector>
-
-#include "tensorflow/lite/allocation.h"
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/memory_planner.h"
-#include "tensorflow/lite/util.h"
-
-namespace tflite {
-
-// Forward declare since NNAPIDelegate uses Interpreter.
-class NNAPIDelegate;
-
-class Subgraph {
- public:
-  friend class Interpreter;
-
-  Subgraph(ErrorReporter* error_reporter,
-           TfLiteExternalContext** external_contexts);
-  Subgraph(const Subgraph&) = delete;
-
-  // Subgraphs should be movable but not copyable.
-  Subgraph(Subgraph&&) = default;
-  Subgraph& operator=(const Subgraph&) = delete;
-  virtual ~Subgraph();
-
-  // Provide a list of tensor indexes that are inputs to the model.
-  // Each index is bound check and this modifies the consistent_ flag of the
-  // interpreter.
-  TfLiteStatus SetInputs(std::vector<int> inputs);
-
-  // Provide a list of tensor indexes that are outputs to the model
-  // Each index is bound check and this modifies the consistent_ flag of the
-  // interpreter.
-  TfLiteStatus SetOutputs(std::vector<int> outputs);
-
-  // Provide a list of tensor indexes that are variable tensors.
-  // Each index is bound check and this modifies the consistent_ flag of the
-  // interpreter.
-  TfLiteStatus SetVariables(std::vector<int> variables);
-
-
-  // Adds a node with the given parameters and returns the index of the new
-  // node in `node_index` (optionally). Interpreter will take ownership of
-  // `builtin_data` and destroy it with `free`. Ownership of 'init_data'
-  // remains with the caller.
-  TfLiteStatus AddNodeWithParameters(const std::vector<int>& inputs,
-                                     const std::vector<int>& outputs,
-                                     const char* init_data,
-                                     size_t init_data_size, void* builtin_data,
-                                     const TfLiteRegistration* registration,
-                                     int* node_index);
-
-  // Adds `tensors_to_add` tensors, preserving pre-existing Tensor entries.
-  // The value pointed to by `first_new_tensor_index` will be set to the
-  // index of the first new tensor if `first_new_tensor_index` is non-null.
-  TfLiteStatus AddTensors(int tensors_to_add, int* first_new_tensor_index);
-
-  // Set description of inputs/outputs/data/fptrs for node `node_index`.
-  // This variant assumes an external buffer has been allocated of size
-  // bytes. The lifetime of buffer must be ensured to be greater or equal
-  // to Interpreter.
-  TfLiteStatus SetTensorParametersReadOnly(
-      int tensor_index, TfLiteType type, const char* name, const size_t rank,
-      const int* dims, TfLiteQuantizationParams quantization,
-      const char* buffer, size_t bytes, const Allocation* allocation);
-
-  // Set description of inputs/outputs/data/fptrs for node `node_index`.
-  // This variant assumes an external buffer has been allocated of size
-  // bytes. The lifetime of buffer must be ensured to be greater or equal
-  // to Interpreter.
-  TfLiteStatus SetTensorParametersReadWrite(
-      int tensor_index, TfLiteType type, const char* name, const size_t rank,
-      const int* dims, TfLiteQuantizationParams quantization, bool is_variable);
-
-  // WARNING: Experimental interface, subject to change
-  // Overrides execution plan. This bounds checks indices sent in.
-  TfLiteStatus SetExecutionPlan(const std::vector<int>& new_plan);
-
-  // Get a mutable tensor data structure.
-  // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this
-  // read/write access to structure
-  TfLiteTensor* tensor(int tensor_index) {
-    if (tensor_index < 0 ||
-        static_cast<size_t>(tensor_index) >= context_->tensors_size) {
-      return nullptr;
-    }
-    return &context_->tensors[tensor_index];
-  }
-
-  // Get an immutable tensor data structure.
-  const TfLiteTensor* tensor(int tensor_index) const {
-    if (tensor_index < 0 ||
-        static_cast<size_t>(tensor_index) >= context_->tensors_size) {
-      return nullptr;
-    }
-    return &context_->tensors[tensor_index];
-  }
-
-  // Read only access to list of inputs.
-  std::vector<int>& inputs() { return inputs_; }
-
-  // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return inputs_; }
-
-  // Read only access to list of outputs.
-  std::vector<int>& outputs() { return outputs_; }
-
-  // Read only access to list of outputs.
-  const std::vector<int>& outputs() const { return outputs_; }
-
-  // Read only access to list of variable tensors.
-  std::vector<int>& variables() { return variables_; }
-
-  // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const { return variables_; }
-
-  size_t tensors_size() const { return tensors_.size(); }
-
-  // Return the number of ops in the model.
-  size_t nodes_size() const { return nodes_and_registration_.size(); }
-
-  // Read only access to list of variable tensors.
-  std::vector<int>& execution_plan() { return execution_plan_; }
-
-  // Read only access to list of variable tensors.
-  const std::vector<int>& execution_plan() const { return execution_plan_; }
-
-  // Mutable form of tensors (TEMPORARY for refactor).
-  // TODO(b/119495520): remove when refactoring complete.
-  std::vector<TfLiteTensor>& tensors() { return tensors_; }
-  // Mutable form of tensors (TEMPORARY for refactor).
-  // TODO(b/119495520): remove when refactoring complete.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
-  nodes_and_registration() {
-    return nodes_and_registration_;
-  }
-
-  const std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
-  nodes_and_registration() const {
-    return nodes_and_registration_;
-  }
-
-  // Get a pointer to an operation and registration data structure if in bounds.
-  const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
-      int node_index) const {
-    if (node_index < 0 || static_cast<size_t>(node_index) >= nodes_size())
-      return nullptr;
-    return &nodes_and_registration_[node_index];
-  }
-
-
-  // Change the dimensionality of a given tensor. Note, this is only acceptable
-  // for tensor indices that are inputs.
-  // Returns status of failure or success.
-  // TODO(aselle): Consider implementing ArraySlice equivalent to make this
-  //   more adept at accepting data without an extra copy. Use absl::ArraySlice
-  //   if our partners determine that dependency is acceptable.
-  TfLiteStatus ResizeInputTensor(int tensor_index,
-                                 const std::vector<int>& dims);
-
-  // Update allocations for all tensors. This will redim dependent tensors using
-  // the input tensor dimensionality as given. This is relatively expensive.
-  // If you know that your sizes are not changing, you need not call this.
-  // Returns status of success or failure.
-  TfLiteStatus AllocateTensors();
-
-  // Invoke the subgraph (run the whole graph in dependency order).
-  //
-  // NOTE: It is possible that the interpreter is not in a ready state
-  // to evaluate (i.e. if a ResizeTensor() has been performed without an
-  // AllocateTensors().
-  // Returns status of success or failure.
-  TfLiteStatus Invoke();
-
-  // Entry point for C node plugin API to report an error.
-  void ReportError(const char* format, ...);
-
-  void UseNNAPI(bool enable);
-
-  // Return the subgraph specific context.
-  TfLiteContext* context() { return context_; }
-
-  // Set the value of an external context.
-  void SetExternalContext(TfLiteExternalContextType type,
-                          TfLiteExternalContext* ctx);
-  // Get the half precision flag.
-  // WARNING: This is an experimental API and subject to change.
-  bool GetAllowFp16PrecisionForFp32() const {
-    return context_->allow_fp32_relax_to_fp16;
-  }
-
-  // Ensure the data in `tensor.data` is readable. In case delegate is used,
-  // it might require to copy the data from delegate buffer to raw memory.
-  // WARNING: This is an experimental API and subject to change.
-  // TODO(b/119495520): make this private when refactoring complete.
-  TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
-    TfLiteTensor* t = &tensors_[tensor_index];
-    TF_LITE_ENSURE(context_, t != nullptr);
-    if (t->data_is_stale) {
-      TF_LITE_ENSURE(context_, t->delegate != nullptr);
-      TF_LITE_ENSURE(context_, t->buffer_handle != kTfLiteNullBufferHandle);
-      // This can be null if the delegate doesn't use its own buffer.
-      TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle != nullptr);
-      t->delegate->CopyFromBufferHandle(context_, t->delegate, t->buffer_handle,
-                                        t->data.raw, t->bytes);
-      t->data_is_stale = false;
-    }
-    return kTfLiteOk;
-  }
-
-
-  // The default capacity of `tensors_` vector.
-  static constexpr int kTensorsReservedCapacity = 128;
-  // The capacity headroom of `tensors_` vector before calling ops'
-  // `prepare` and `invoke` function. In these functions, it's guaranteed
-  // allocating up to `kTensorsCapacityHeadroom` more tensors won't invalidate
-  // pointers to existing tensors.
-  static constexpr int kTensorsCapacityHeadroom = 16;
-
-  // Reset all variable tensors to the default value.
-  // If a variable tensor doesn't have a buffer, reset it to zero.
-  // TODO(b/115961645): Implement - If a variable tensor has a buffer, reset it
-  // to the value of the buffer.
-  // WARNING: This is an experimental API and subject to change.
-  TfLiteStatus ResetVariableTensors();
-
- private:
-  // Prevent 'context_' from accessing functions that are only available to
-  // delegated kernels.
-  void SwitchToKernelContext();
-
-  // Add delegate-only functions to 'context_'.
-  void SwitchToDelegateContext();
-
-  // Give 'op_reg' a chance to initialize itself using the contents of
-  // 'buffer'.
-  void* OpInit(const TfLiteRegistration& op_reg, const char* buffer,
-               size_t length) {
-    if (op_reg.init == nullptr) return nullptr;
-    return op_reg.init(context_, buffer, length);
-  }
-
-  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
-  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
-    if (op_reg.free == nullptr) return;
-    if (buffer) {
-      op_reg.free(context_, buffer);
-    }
-  }
-
-  // Prepare the given 'node' for execution.
-  TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) {
-    if (op_reg.prepare == nullptr) return kTfLiteOk;
-    return op_reg.prepare(context_, node);
-  }
-
-  // Invoke the operator represented by 'node'.
-  TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) {
-    if (op_reg.invoke == nullptr) return kTfLiteError;
-    return op_reg.invoke(context_, node);
-  }
-
-  // Call OpPrepare() for as many ops as possible, allocating memory for their
-  // tensors. If an op containing dynamic tensors is found, preparation will be
-  // postponed until this function is called again. This allows the interpreter
-  // to wait until Invoke() to resolve the sizes of dynamic tensors.
-  TfLiteStatus PrepareOpsAndTensors();
-
-  // Call OpPrepare() for all ops starting at 'first_node'. Stop when a
-  // dynamic tensors is found or all ops have been prepared. Fill
-  // 'last_node_prepared' with the id of the op containing dynamic tensors, or
-  // the last in the graph.
-  TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index,
-                                    int* last_execution_plan_index_prepared);
-
-  // Tensors needed by the interpreter. Use `AddTensors` to add more blank
-  // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
-  // `context_` whenever this std::vector is reallocated. Currently this
-  // only happens in `AddTensors()`.
-  std::vector<TfLiteTensor> tensors_;
-
-  // Check if an array of tensor indices are valid with respect to the Tensor
-  // array.
-  // NOTE: this changes consistent_ to be false if indices are out of bounds.
-  TfLiteStatus CheckTensorIndices(const char* label, const int* indices,
-                                  int length);
-
-  // Compute the number of bytes required to represent a tensor with dimensions
-  // specified by the array dims (of length dims_size). Returns the status code
-  // and bytes.
-  TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size,
-                             size_t* bytes);
-
-  // Request an tensor be resized implementation. If the given tensor is of
-  // type kTfLiteDynamic it will also be allocated new memory.
-  TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size);
-
-  // Report a detailed error string (will be printed to stderr).
-  // TODO(aselle): allow user of class to provide alternative destinations.
-  void ReportErrorImpl(const char* format, va_list args);
-
-  // Entry point for C node plugin API to request an tensor be resized.
-  static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor,
-                                   TfLiteIntArray* new_size);
-  // Entry point for C node plugin API to report an error.
-  static void ReportErrorC(TfLiteContext* context, const char* format, ...);
-
-  // Entry point for C node plugin API to add new tensors.
-  static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add,
-                                 int* first_new_tensor_index);
-
-  // WARNING: This is an experimental API and subject to change.
-  // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels
-  static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
-      TfLiteContext* context, TfLiteRegistration registration,
-      const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate);
-
-  // Update the execution graph to replace some of the nodes with stub
-  // nodes. Specifically any node index that has `nodes[index]==1` will be
-  // slated for replacement with a delegate kernel specified by registration.
-  // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller.
-  // WARNING: This is an experimental interface that is subject to change.
-  TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
-      TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
-      TfLiteDelegate* delegate);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Gets the internal pointer to a TensorFlow lite node by node_index.
-  TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node,
-                                      TfLiteRegistration** registration);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Entry point for C node plugin API to get a node by index.
-  static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*,
-                                             int node_index, TfLiteNode** node,
-                                             TfLiteRegistration** registration);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Gets an TfLiteIntArray* representing the execution plan. The interpreter
-  // owns this memory and it is only guaranteed to exist during the invocation
-  // of the delegate prepare.
-  TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Entry point for C node plugin API to get the execution plan.
-  static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context,
-                                       TfLiteIntArray** execution_plan);
-
-  // Retrieve an existing external context by type.
-  TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type);
-  static TfLiteExternalContext* GetExternalContext(
-      struct TfLiteContext* context, TfLiteExternalContextType type);
-
-  // Set the value of an external context.
-  static void SetExternalContext(struct TfLiteContext* context,
-                                 TfLiteExternalContextType type,
-                                 TfLiteExternalContext* ctx);
-
-  // Allow a delegate to look at the graph and modify the graph to handle
-  // parts of the graph themselves. After this is called, the graph may
-  // contain new nodes that replace 1 more nodes.
-  // WARNING: This is an experimental API and subject to change.
-  TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
-
-  // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra
-  // capacity. Calling this function may invalidate existing pointers to
-  // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
-  // more tensors won't invalidate the pointer to existing tensors.
-  void EnsureTensorsVectorCapacity() {
-    const size_t required_capacity = tensors_.size() + kTensorsCapacityHeadroom;
-    if (required_capacity > tensors_.capacity()) {
-      tensors_.reserve(required_capacity);
-      context_->tensors = tensors_.data();
-    }
-  }
-
-  // The state of the Interpreter.
-  enum State {
-    // The interpreter isn't ready to be invoked.
-    // `AllocateTensor` need to be called to enter an invokable state.
-    kStateUninvokable = 0,
-    // The interpreter is ready to be invoked.
-    kStateInvokable,
-    // The interpreter is ready to be invoked, and graph can't be further
-    // modified. The interpreter will enter this state when calling
-    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
-    kStateInvokableAndImmutable,
-  };
-  State state_ = kStateUninvokable;
-
-  // A pure C data structure used to communicate with the pure C plugin
-  // interface. To avoid copying tensor metadata, this is also the definitive
-  // structure to store tensors.
-  // TODO(b/119495520): Get rid of owned and just make context_ a instance.
-  TfLiteContext owned_context_;
-  TfLiteContext* context_;
-
-  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
-  // function pointers to actual implementation.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
-      nodes_and_registration_;
-
-  // Whether the model is consistent. That is to say if the inputs and outputs
-  // of every node and the global inputs and outputs are valid indexes into
-  // the tensor array.
-  bool consistent_ = true;
-
-  // Array of indices representing the tensors that are inputs to the
-  // interpreter.
-  std::vector<int> inputs_;
-
-  // Array of indices representing the tensors that are outputs to the
-  // interpreter.
-  std::vector<int> outputs_;
-
-  // Array of indices representing the tensors that are variable tensors.
-  std::vector<int> variables_;
-
-  // The error reporter delegate that tflite will forward queries errors to.
-  ErrorReporter* error_reporter_;
-
-  // Index of the next node to prepare.
-  // During Invoke(), Interpreter will allocate input tensors first, which are
-  // known to be fixed size. Then it will allocate outputs from nodes as many
-  // as possible. When there is a node that produces dynamic sized tensor.
-  // Interpreter will stop allocating tensors, set the value of next allocate
-  // node id, and execute the node to generate the output tensor before continue
-  // to allocate successors. This process repeats until all nodes are executed.
-  // NOTE: this relies on the order of nodes that is in topological order.
-  int next_execution_plan_index_to_prepare_;
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // This is a list of node indices (to index into nodes_and_registration).
-  // This represents a valid topological sort (dependency ordered) execution
-  // plan. In particular, it is valid for this ordering to contain only a
-  // subset of the node indices.
-  std::vector<int> execution_plan_;
-
-  // In the future, we'd like a TfLiteIntArray compatible representation.
-  // TODO(aselle): replace execution_plan_ with this.
-  std::unique_ptr<TfLiteIntArray, TfLiteIntArrayDeleter> plan_cache_;
-
-  // Whether to delegate to NN API
-  std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
-
-  std::unique_ptr<MemoryPlanner> memory_planner_;
-
-  // Tracking bit for whether a tensor was resized in the course of an op
-  // invocation. This is a useful hint to ensure that dynamic tensor outputs
-  // trigger downstream reallocation after op invocation.
-  bool tensor_resized_since_op_invoke_ = false;
-
-  // External contexts (kTfLiteMaxExternalContexts).
-  TfLiteExternalContext** external_contexts_;
-};
-
-}  // namespace tflite
-#endif  // TENSORFLOW_LITE_CORE_SUBGRAPH_H_
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 4f4a999210..0bac5b07b2 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <cstdint>
 #include <cstring>
 
+#include "tensorflow/lite/arena_planner.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/context_util.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
@@ -31,14 +32,110 @@ limitations under the License.
 #include "tensorflow/lite/util.h"
 
 namespace tflite {
+namespace {
+
+TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
+                           const TfLiteRegistration& registration,
+                           int node_index, const char* message) {
+  context->ReportError(
+      context, "Node number %d (%s) %s.\n", node_index,
+      registration.custom_name
+          ? registration.custom_name
+          : EnumNameBuiltinOperator(
+                static_cast<BuiltinOperator>(registration.builtin_code)),
+      message);
+  return kTfLiteError;
+}
+
+// Stub method which returns kTfLiteError when the function is forbidden.
+// We're registrating this function to several different function to save
+// compiled binary size. Please note the restrictions:
+// * The type of first parameter have to be `TfLiteContext*`.
+// * All paramteters must be trivailly destructible. (E.g. No C++ class)
+TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) {
+  context->ReportError(context,
+                       "The function is forbidden if not calling in delegate.");
+  return kTfLiteError;
+}
+
+// Set the ForbiddenContextFunction to a compatible function pointer.
+template <typename FunctionType>
+void SetForbiddenContextFunction(FunctionType* func) {
+  *func = reinterpret_cast<FunctionType>(ForbiddenContextFunction);
+}
+
+// Returns true if at least one tensor in the given list is kTfLiteDynamic.
+template <typename TensorIntArray>
+bool HasDynamicTensorImpl(const TfLiteContext& context,
+                          const TensorIntArray& int_array) {
+  for (int i : int_array) {
+    const TfLiteTensor& tensor = context.tensors[i];
+    if (tensor.allocation_type == kTfLiteDynamic) {
+      return true;
+    }
+  }
+  return false;
+}
+
+}  // namespace
+
+// A trivial implementation of GraphInfo around the Interpreter.
+// NOTE: this interpreter info represents the subset of the
+// graph that is executed according to execution plan. Thus,
+// the indices are execution plan indices rather than raw node
+// indices.
+class InterpreterInfo : public GraphInfo {
+ public:
+  explicit InterpreterInfo(Interpreter* interpreter)
+      : interpreter_(interpreter) {}
+
+  size_t num_tensors() const override { return interpreter_->tensors_size(); }
+  TfLiteTensor* tensor(size_t index) override {
+    return interpreter_->tensor(index);
+  }
+  size_t num_nodes() const override {
+    return interpreter_->execution_plan().size();
+  }
+  const TfLiteNode& node(size_t index) const override {
+    int node_index = interpreter_->execution_plan()[index];
+    return interpreter_->node_and_registration(node_index)->first;
+  }
+  const std::vector<int>& inputs() const override {
+    return interpreter_->inputs();
+  }
+  const std::vector<int>& outputs() const override {
+    return interpreter_->outputs();
+  }
+  const std::vector<int>& variables() const override {
+    return interpreter_->variables();
+  }
+
+ public:
+  Interpreter* interpreter_;
+};
 
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  subgraphs_.emplace_back(error_reporter_, external_contexts_);
-  context_ = primary_subgraph().context();
+  context_.impl_ = static_cast<void*>(this);
+  context_.ResizeTensor = ResizeTensor;
+  context_.ReportError = ReportError;
+  context_.AddTensors = AddTensors;
+  context_.tensors = nullptr;
+  context_.tensors_size = 0;
+  context_.allow_fp32_relax_to_fp16 = false;
+  context_.recommended_num_threads = -1;
+  context_.GetExternalContext = GetExternalContext;
+  context_.SetExternalContext = SetExternalContext;
+
+  // Invalid to call these these except from TfLiteDelegate
+  SwitchToKernelContext();
 
   // Reserve some space for the tensors to avoid excessive resizing.
+  tensors_.reserve(kTensorsReservedCapacity);
+  nodes_and_registration_.reserve(kTensorsReservedCapacity);
+  next_execution_plan_index_to_prepare_ = 0;
+
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
     external_contexts_[i] = nullptr;
   }
@@ -46,75 +143,673 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
   UseNNAPI(false);
 }
 
-Interpreter::~Interpreter() {}
+Interpreter::~Interpreter() {
+  for (auto& nodeAndReg : nodes_and_registration_) {
+    TfLiteNode& node = nodeAndReg.first;
+    TfLiteIntArrayFree(node.inputs);
+    TfLiteIntArrayFree(node.outputs);
+    TfLiteIntArrayFree(node.temporaries);
+    if (node.builtin_data) free(node.builtin_data);
+    OpFree(nodeAndReg.second, node.user_data);
+    node.builtin_data = nullptr;
+  }
+
+  for (size_t i = 0; i < context_.tensors_size; i++) {
+    TfLiteTensor* tensor = &context_.tensors[i];
+    if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
+        tensor->delegate->FreeBufferHandle != nullptr) {
+      tensor->delegate->FreeBufferHandle(&context_, tensor->delegate,
+                                         &tensor->buffer_handle);
+    }
+    TfLiteTensorFree(tensor);
+  }
+}
+
+TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
+    TfLiteContext* context, TfLiteRegistration registration,
+    const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
+  return static_cast<Interpreter*>(context->impl_)
+      ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace,
+                                              delegate);
+}
+
+namespace {
+
+// Copy a std::vector<int> to an existing TfLiteIntArray.
+// This is a low-level data manipulation function, and it's caller's
+// responsibility to ensure TfLiteIntArray has enough size.
+void CopyVectorToTfLiteIntArray(const std::vector<int>& vec,
+                                TfLiteIntArray* arr) {
+  arr->size = vec.size();
+  memcpy(arr->data, vec.data(), sizeof(int) * arr->size);
+}
+
+// This function allocates a continuous memory space that contains a
+// TfLiteDelegateParams followed by a several TfLiteIntArray.
+// When calling `free` at TfLiteDelegateParams*, all the allocated space
+// will be freed together.
+//
+// +-----------------------------------+
+// | TfLiteDelegateParams              |
+// | TfLiteDelegate* delegate;         |
+// | TfLiteIntArray* nodes_to_replace; |--\
+// | TfLiteIntArray* input_tensors;    |--+--\
+// | TfLiteIntArray* output_tensors;   |--+--+--\
+// +-----------------------------------+  |  |  |
+// | TfLiteIntArray (variable size)    |<-/  |  |
+// +-----------------------------------+     |  |
+// | TfLiteIntArray (variable size)    |<----/  |
+// +-----------------------------------+        |
+// | TfLiteIntArray (variable size)    |<-------/
+// +-----------------------------------+
+TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate,
+                                           const NodeSubset& node_subset) {
+  // Step 1: Calculate the allocation size.
+  int allocation_size = sizeof(TfLiteDelegateParams);
+
+  int nodes_to_replace_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size());
+  allocation_size += nodes_to_replace_size;
+
+  int input_tensors_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size());
+  allocation_size += input_tensors_size;
+
+  int output_tensors_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size());
+  allocation_size += output_tensors_size;
+
+  // Step 2: Allocate the memory.
+  // Use `char*` for conveniently step through the allocated space by bytes.
+  char* allocation = reinterpret_cast<char*>(malloc(allocation_size));
+
+  // Step 3: Fill all data structures structures.
+  TfLiteDelegateParams* params =
+      reinterpret_cast<TfLiteDelegateParams*>(allocation);
+  params->delegate = delegate;
+  allocation += sizeof(TfLiteDelegateParams);
+
+  params->nodes_to_replace = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace);
+  allocation += nodes_to_replace_size;
+
+  params->input_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors);
+  allocation += input_tensors_size;
+
+  params->output_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.output_tensors,
+                             params->output_tensors);
+  allocation += output_tensors_size;
+
+  return params;
+}
+
+}  // namespace
+
+TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
+    TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
+    TfLiteDelegate* delegate) {
+  // Annotate the registration as DELEGATE op.
+  registration.builtin_code = BuiltinOperator_DELEGATE;
+
+  // Analyze the graph to find all independent node_subsets that are either
+  // fully not-this-delegate or this-delegate computation.
+  InterpreterInfo info(this);
+  std::vector<NodeSubset> node_subsets;
+  PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace,
+                                           &node_subsets);
+
+  execution_plan_.clear();
+  for (auto& node_subset : node_subsets) {
+    // Subsets calimed by the delegate should have a "macro" op created, the
+    // other node_subsets (kTfNonPartition) just have their nodes added back to
+    // the execution plan.
+    switch (node_subset.type) {
+      case NodeSubset::kTfNonPartition:
+        for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end();
+             ++it) {
+          execution_plan_.push_back(*it);
+        }
+        break;
+      case NodeSubset::kTfPartition: {
+        int node_index;
+
+        TfLiteDelegateParams* params =
+            CreateDelegateParams(delegate, node_subset);
+        TF_LITE_ENSURE_STATUS(AddNodeWithParameters(
+            node_subset.input_tensors, node_subset.output_tensors, nullptr, 0,
+            params, &registration, &node_index));
+
+        // Initialize the output tensors's delegate-related fields.
+        for (int tensor_index : node_subset.output_tensors) {
+          TfLiteTensor* tensor = &tensors_[tensor_index];
+          TF_LITE_ENSURE(&context_, tensor->delegate == nullptr ||
+                                        tensor->delegate == delegate);
+          tensor->delegate = delegate;
+        }
+
+        // Associate the node with the delegate.
+        TfLiteNode* node = &nodes_and_registration_[node_index].first;
+        node->delegate = delegate;
+      } break;
+      case NodeSubset::kTfUnexplored:
+        return kTfLiteError;
+        break;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteExternalContext* Interpreter::GetExternalContext(
+    TfLiteExternalContextType type) {
+  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
+    return external_contexts_[type];
+  }
+  return nullptr;
+}
+
+TfLiteExternalContext* Interpreter::GetExternalContext(
+    struct TfLiteContext* context, TfLiteExternalContextType type) {
+  return static_cast<Interpreter*>(context->impl_)->GetExternalContext(type);
+}
 
 void Interpreter::SetExternalContext(TfLiteExternalContextType type,
                                      TfLiteExternalContext* ctx) {
-  primary_subgraph().SetExternalContext(type, ctx);
+  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
+    external_contexts_[type] = ctx;
+  }
+}
+
+void Interpreter::SetExternalContext(struct TfLiteContext* context,
+                                     TfLiteExternalContextType type,
+                                     TfLiteExternalContext* ctx) {
+  return static_cast<Interpreter*>(context->impl_)
+      ->SetExternalContext(type, ctx);
+}
+
+// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns
+// this memory and it is only guaranteed to exist during the invocation of the
+// delegate prepare.
+TfLiteStatus Interpreter::GetExecutionPlan(TfLiteIntArray** execution_plan) {
+  // TODO(aselle): Do not make a copy here
+  plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size()));
+  *execution_plan = plan_cache_.get();
+  static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]),
+                "TfLiteIntArray and execution_plan do not contain same type.");
+  std::memcpy(plan_cache_->data, execution_plan_.data(),
+              sizeof(plan_cache_->data[0]) * execution_plan_.size());
+  return kTfLiteOk;
+}
+
+// WARNING: This is an experimental interface that is subject to change.
+// Entry point for C node plugin API to get the execution plan
+TfLiteStatus Interpreter::GetExecutionPlan(struct TfLiteContext* context,
+                                           TfLiteIntArray** execution_plan) {
+  return static_cast<Interpreter*>(context->impl_)
+      ->GetExecutionPlan(execution_plan);
 }
 
 TfLiteStatus Interpreter::SetInputs(std::vector<int> inputs) {
-  return primary_subgraph().SetInputs(inputs);
+  TF_LITE_ENSURE_OK(&context_,
+                    CheckTensorIndices("inputs", inputs.data(), inputs.size()));
+  inputs_ = std::move(inputs);
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetOutputs(std::vector<int> outputs) {
-  return primary_subgraph().SetOutputs(outputs);
+  TF_LITE_ENSURE_OK(
+      &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
+  outputs_ = std::move(outputs);
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetVariables(std::vector<int> variables) {
-  return primary_subgraph().SetVariables(variables);
+  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
+                                                  variables.size()));
+  variables_ = std::move(variables);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Interpreter::CheckTensorIndices(const char* label,
+                                             const int* indices, int length) {
+  // Making sure kOptionalTensor is not re-defined to something other than -1.
+  static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1");
+
+  for (int i = 0; i < length; i++) {
+    int index = indices[i];
+    // Continue if index == kOptionalTensor before additional comparisons below,
+    // size_t(-1) is always >= context_tensors_size.
+    if (index == kOptionalTensor) {
+      continue;
+    }
+    if (index < 0 || static_cast<size_t>(index) >= context_.tensors_size) {
+      ReportError(&context_, "Invalid tensor index %d in %s\n", index, label);
+      consistent_ = false;
+      return kTfLiteError;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims,
+                                        size_t dims_size, size_t* bytes) {
+  // TODO(aselle): Check for overflow here using overflow.h in TensorFlow
+  // MultiplyWithoutOverflow.
+  TF_LITE_ENSURE(&context_, bytes != nullptr);
+  size_t count = 1;
+  for (int k = 0; k < dims_size; k++) count *= dims[k];
+  switch (type) {
+    case kTfLiteFloat32:
+      *bytes = sizeof(float) * count;
+      break;
+    case kTfLiteInt16:
+      *bytes = sizeof(int16_t) * count;
+      break;
+    case kTfLiteInt32:
+      *bytes = sizeof(int32_t) * count;
+      break;
+    case kTfLiteUInt8:
+      *bytes = sizeof(uint8_t) * count;
+      break;
+    case kTfLiteInt8:
+      *bytes = sizeof(int8_t) * count;
+      break;
+    case kTfLiteInt64:
+      *bytes = sizeof(int64_t) * count;
+      break;
+    case kTfLiteBool:
+      *bytes = sizeof(bool) * count;
+      break;
+    case kTfLiteComplex64:
+      *bytes = sizeof(std::complex<float>) * count;
+      break;
+    default:
+      ReportError(&context_,
+                  "Only float32, int16, int32, int64, uint8, int8, bool, "
+                  "complex64 supported currently.");
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::AllocateTensors() {
-  return primary_subgraph().AllocateTensors();
+  if (!consistent_) {
+    ReportError(&context_, "AllocateTensors() called on inconsistent model.");
+    return kTfLiteError;
+  }
+
+  // Explicit (re)allocation is necessary if nodes have been changed or tensors
+  // have been resized. For inputs marked as dynamic, we can't short-circuit the
+  // allocation as the client may have done the resize manually.
+  if (state_ != kStateUninvokable && !HasDynamicTensorImpl(context_, inputs_)) {
+    return kTfLiteOk;
+  }
+
+  next_execution_plan_index_to_prepare_ = 0;
+  if (memory_planner_) {
+    TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations());
+  }
+
+  TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
+
+  state_ = kStateInvokable;
+
+  // Reset the variable tensors to zero after (re)allocating the tensors.
+  // Developers shouldn't rely on the side effect of this function to reset
+  // variable tesnsors. They should call `ResetVariableTensors` directly
+  // instead.
+  ResetVariableTensors();
+
+  return kTfLiteOk;
+}
+
+// TODO(ycling): Support non-zero default values.
+TfLiteStatus Interpreter::ResetVariableTensors() {
+  for (auto& tensor : tensors_) {
+    if (!tensor.is_variable) {
+      continue;
+    }
+
+    // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be
+    // allocated after the initial `PrepareOpsAndTensors()` is called.
+    TF_LITE_ENSURE_EQ(&context_, tensor.allocation_type,
+                      kTfLiteArenaRwPersistent);
+    TF_LITE_ENSURE(&context_, tensor.data.raw != nullptr);
+
+    memset(tensor.data.raw, 0, tensor.bytes);
+  }
+  return kTfLiteOk;
 }
 
 void Interpreter::ReserveNodes(int count) {
-  primary_subgraph().nodes_and_registration().reserve(count);
+  nodes_and_registration_.reserve(count);
 }
 
 TfLiteStatus Interpreter::AddNodeWithParameters(
     const std::vector<int>& inputs, const std::vector<int>& outputs,
     const char* init_data, size_t init_data_size, void* builtin_data,
     const TfLiteRegistration* registration, int* node_index) {
-  return primary_subgraph().AddNodeWithParameters(inputs, outputs, init_data,
-                                                  init_data_size, builtin_data,
-                                                  registration, node_index);
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(&context_,
+                "AddNodeWithParameters is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  state_ = kStateUninvokable;
+
+  std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
+                                                              free);
+
+  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("node inputs", inputs.data(),
+                                                  inputs.size()));
+  TF_LITE_ENSURE_OK(
+      &context_,
+      CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
+
+  int new_node_index = nodes_and_registration_.size();
+  if (node_index) *node_index = new_node_index;
+  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
+  auto& node_and_reg = nodes_and_registration_.back();
+  TfLiteNode& node = node_and_reg.first;
+  if (node.inputs) TfLiteIntArrayFree(node.inputs);
+  if (node.outputs) TfLiteIntArrayFree(node.outputs);
+  if (node.temporaries) TfLiteIntArrayFree(node.temporaries);
+
+  // NOTE, here we are not using move semantics yet, since our internal
+  // representation isn't std::vector, but in the future we would like to avoid
+  // copies, so we want the interface to take r-value references now.
+  node.inputs = ConvertVectorToTfLiteIntArray(inputs);
+  node.outputs = ConvertVectorToTfLiteIntArray(outputs);
+  node.temporaries = TfLiteIntArrayCreate(0);
+  if (init_data) {
+    node.user_data = OpInit(*registration, init_data, init_data_size);
+  } else {
+    node.user_data =
+        OpInit(*registration,
+               reinterpret_cast<const char*>(builtin_data_deleter.get()), 0);
+  }
+
+  node.builtin_data = builtin_data_deleter.release();
+  // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size`
+  // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels.
+
+  if (registration->builtin_code == BuiltinOperator_CUSTOM) {
+    // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer
+    // `Operator` table is passed in.
+    node.custom_initial_data = init_data;
+    node.custom_initial_data_size = init_data_size;
+  } else {
+    node.custom_initial_data = nullptr;
+    node.custom_initial_data_size = 0;
+  }
+
+  node.delegate = nullptr;
+  node_and_reg.second = *registration;
+  execution_plan_.push_back(new_node_index);
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index,
                                             const std::vector<int>& dims) {
-  return primary_subgraph().ResizeInputTensor(tensor_index, dims);
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(&context_,
+                "ResizeInputTensor is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
+  // TODO(aselle): All bounds checks can be implemented as one-sided bounds
+  // checks by casting to unsigned for efficiency. Profile before doing this.
+  TF_LITE_ENSURE(&context_,
+                 tensor_index < context_.tensors_size && tensor_index >= 0);
+  TfLiteTensor* tensor = &context_.tensors[tensor_index];
+
+  // Short-circuit the state change if the dimensions don't change, avoiding
+  // unnecessary (re)allocations.
+  if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) {
+    return kTfLiteOk;
+  }
+
+  state_ = kStateUninvokable;
+  return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims));
+}
+
+bool HasDynamicTensor(const TfLiteContext& context,
+                      const TfLiteIntArray* int_array) {
+  return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
+}
+
+TfLiteStatus Interpreter::PrepareOpsStartingAt(
+    int first_execution_plan_index, int* last_execution_plan_index_prepared) {
+  for (int execution_plan_index = first_execution_plan_index;
+       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
+    int node_index = execution_plan_[execution_plan_index];
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    const TfLiteRegistration& registration =
+        nodes_and_registration_[node_index].second;
+    EnsureTensorsVectorCapacity();
+    if (OpPrepare(registration, &node) == kTfLiteError) {
+      return ReportOpError(&context_, node, registration, node_index,
+                           "failed to prepare");
+    }
+
+    *last_execution_plan_index_prepared = execution_plan_index;
+
+    // Discontinue if the node has dynamic outputs. Note that we don't
+    // stop for dynamic temporary tensors since they won't affect the
+    // sizes of other tensors in the graph.
+    if (HasDynamicTensor(context_, node.outputs)) {
+      break;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Interpreter::PrepareOpsAndTensors() {
+  if (!memory_planner_) {
+    memory_planner_.reset(new ArenaPlanner(
+        &context_, std::unique_ptr<GraphInfo>(new InterpreterInfo(this)),
+        /*preserve_inputs=*/true, /*preserve_intermediates*/ false));
+    memory_planner_->PlanAllocations();
+  }
+
+  int last_exec_plan_index_prepared = 0;
+
+  TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt(
+      next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared));
+  TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations(
+      next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared));
+
+  next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1;
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::Invoke() {
-  TfLiteStatus status = primary_subgraph().Invoke();
+  if (!consistent_) {
+    ReportError(&context_, "Invoke called on model that is not consistent.");
+    return kTfLiteError;
+  }
+  if (state_ == kStateUninvokable) {
+    ReportError(&context_, "Invoke called on model that is not ready.");
+    return kTfLiteError;
+  }
+
+  TfLiteStatus status = kTfLiteOk;
+  if (nnapi_delegate_) {
+    if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) {
+      TF_LITE_ENSURE_OK(&context_, nnapi_delegate_->Invoke(this));
+      return kTfLiteOk;
+    } else {
+      // TODO(aselle): In the future, we would like this to be an
+      // automatic tflite CPU fallback.
+      ReportError(&context_,
+                  "NNAPI was requested, but dependent sized tensors "
+                  "being used.\n");
+      return kTfLiteError;
+    }
+  }
+
+  // Invocations are always done in node order.
+  // Note that calling Invoke repeatedly will cause the original memory plan to
+  // be reused, unless either ResizeInputTensor() or AllocateTensors() has been
+  // called.
+  for (int execution_plan_index = 0;
+       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
+    if (execution_plan_index == next_execution_plan_index_to_prepare_) {
+      TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
+      TF_LITE_ENSURE(&context_, next_execution_plan_index_to_prepare_ >=
+                                    execution_plan_index);
+    }
+    int node_index = execution_plan_[execution_plan_index];
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    const TfLiteRegistration& registration =
+        nodes_and_registration_[node_index].second;
+    SCOPED_OPERATOR_PROFILE(profiler_, node_index);
+
+    // TODO(ycling): This is an extra loop through inputs to check if the data
+    // need to be copied from Delegate buffer to raw memory, which is often not
+    // needed. We may want to cache this in prepare to know if this needs to be
+    // done for a node or not.
+    for (int i = 0; i < node.inputs->size; ++i) {
+      int tensor_index = node.inputs->data[i];
+      if (tensor_index == kOptionalTensor) {
+        continue;
+      }
+      TfLiteTensor* tensor = &tensors_[tensor_index];
+      if (tensor->delegate && tensor->delegate != node.delegate &&
+          tensor->data_is_stale) {
+        EnsureTensorDataIsReadable(tensor_index);
+      }
+    }
+
+    EnsureTensorsVectorCapacity();
+    tensor_resized_since_op_invoke_ = false;
+    if (OpInvoke(registration, &node) == kTfLiteError) {
+      status = ReportOpError(&context_, node, registration, node_index,
+                             "failed to invoke");
+    }
+
+    // Force execution prep for downstream ops if the latest op triggered the
+    // resize of a dynamic tensor.
+    if (tensor_resized_since_op_invoke_ &&
+        HasDynamicTensor(context_, node.outputs)) {
+      next_execution_plan_index_to_prepare_ = execution_plan_index + 1;
+    }
+  }
 
   if (!allow_buffer_handle_output_) {
-    for (int tensor_index : outputs()) {
-      primary_subgraph().EnsureTensorDataIsReadable(tensor_index);
+    for (int tensor_index : outputs_) {
+      EnsureTensorDataIsReadable(tensor_index);
     }
   }
 
   return status;
 }
 
+TfLiteStatus Interpreter::ResizeTensor(TfLiteContext* context,
+                                       TfLiteTensor* tensor,
+                                       TfLiteIntArray* new_size) {
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function ResizeTensorImpl
+  // (this function is static).
+  return static_cast<Interpreter*>(context->impl_)
+      ->ResizeTensorImpl(tensor, new_size);
+}
+
+void Interpreter::ReportErrorImpl(const char* format, va_list args) {
+  error_reporter_->Report(format, args);
+}
+
+void Interpreter::ReportError(TfLiteContext* context, const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  auto* f = static_cast<Interpreter*>(context->impl_);
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function ReportErrorImpl
+  // (this function is static).
+  f->ReportErrorImpl(format, args);
+  va_end(args);
+}
+
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
                                      int* first_new_tensor_index) {
-  return primary_subgraph().AddTensors(tensors_to_add, first_new_tensor_index);
+  const size_t base_index = tensors_.size();
+  if (first_new_tensor_index) *first_new_tensor_index = base_index;
+  tensors_.resize(tensors_.size() + tensors_to_add);
+  for (size_t i = base_index; i < tensors_.size(); i++) {
+    memset(&tensors_[i], 0, sizeof(tensors_[i]));
+    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
+  }
+  context_.tensors = tensors_.data();
+  context_.tensors_size = tensors_.size();
+  return kTfLiteOk;
 }
 
-TfLiteStatus Interpreter::ResetVariableTensors() {
-  return primary_subgraph().ResetVariableTensors();
+TfLiteStatus Interpreter::AddTensors(TfLiteContext* context, int tensors_to_add,
+                                     int* first_new_tensor_index) {
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function AddTensors
+  // (this function is static).
+  return static_cast<Interpreter*>(context->impl_)
+      ->AddTensors(tensors_to_add, first_new_tensor_index);
+}
+
+TfLiteStatus Interpreter::GetNodeAndRegistration(
+    int node_index, TfLiteNode** node, TfLiteRegistration** registration) {
+  TF_LITE_ENSURE(&context_, node_index >= 0);
+  TF_LITE_ENSURE(&context_, static_cast<size_t>(node_index) < nodes_size());
+  TF_LITE_ENSURE(&context_, node != nullptr && registration != nullptr);
+  *node = &nodes_and_registration_[node_index].first;
+  *registration = &nodes_and_registration_[node_index].second;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Interpreter::GetNodeAndRegistration(
+    struct TfLiteContext* context, int node_index, TfLiteNode** node,
+    TfLiteRegistration** registration) {
+  return static_cast<Interpreter*>(context->impl_)
+      ->GetNodeAndRegistration(node_index, node, registration);
 }
 
 TfLiteStatus Interpreter::SetTensorParametersReadOnly(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
     size_t bytes, const Allocation* allocation) {
-  return primary_subgraph().SetTensorParametersReadOnly(
-      tensor_index, type, name, rank, dims, quantization, buffer, bytes,
-      allocation);
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        &context_,
+        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
+  TF_LITE_ENSURE(&context_,
+                 tensor_index < context_.tensors_size && tensor_index >= 0);
+  // For most tensors we know exactly how much memory is necessary so we can
+  // ensure the buffer is large enough. However, we need to skip string tensors
+  // because their sizes change with the contents of the individual strings.
+  if (type != kTfLiteString) {
+    size_t required_bytes;
+    TF_LITE_ENSURE_OK(&context_,
+                      BytesRequired(type, dims, rank, &required_bytes));
+    TF_LITE_ENSURE_EQ(&context_, required_bytes, bytes);
+  }
+
+  TfLiteTensor& tensor = context_.tensors[tensor_index];
+  if (type == tensor.type &&
+      EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
+    // Fast path which does not invalidate the invokable property.
+    TfLiteTensorDataFree(&tensor);
+    tensor.data.raw = const_cast<char*>(buffer);
+    if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
+    tensor.params = quantization;
+    tensor.allocation_type = kTfLiteMmapRo;
+    tensor.allocation = allocation;
+  } else {
+    state_ = kStateUninvokable;
+    TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
+                      quantization, const_cast<char*>(buffer), bytes,
+                      kTfLiteMmapRo, allocation, false, &tensor);
+  }
+  return kTfLiteOk;
 }
 
 // Set description of inputs/outputs/data/fptrs for node `node_index`.
@@ -124,52 +819,186 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly(
 TfLiteStatus Interpreter::SetTensorParametersReadWrite(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
-  return primary_subgraph().SetTensorParametersReadWrite(
-      tensor_index, type, name, rank, dims, quantization, is_variable);
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        &context_,
+        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  TF_LITE_ENSURE(&context_,
+                 tensor_index < context_.tensors_size && tensor_index >= 0);
+  size_t required_bytes = 0;
+  if (type != kTfLiteString) {
+    // These types will be allocated in our arena so we need to record how
+    // many bytes we will need based on the dimensions. String tensors are
+    // allocated dynamically and we can't know ahead of time how much space
+    // they will require.
+    TF_LITE_ENSURE_OK(&context_,
+                      BytesRequired(type, dims, rank, &required_bytes));
+  }
+
+  TfLiteAllocationType allocation_type = kTfLiteArenaRw;
+  if (type == kTfLiteString) {
+    if (is_variable) {
+      // We don't have a real use case for string variable tensor.
+      ReportError(&context_, "String variable tensor isn't supported.");
+      return kTfLiteError;
+    }
+    allocation_type = kTfLiteDynamic;
+  } else if (is_variable) {
+    allocation_type = kTfLiteArenaRwPersistent;
+  }
+
+  TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
+                    quantization,
+                    /*buffer=*/nullptr, required_bytes, allocation_type,
+                    nullptr, is_variable, &context_.tensors[tensor_index]);
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::SetExecutionPlan(const std::vector<int>& new_plan) {
-  return primary_subgraph().SetExecutionPlan(new_plan);
+  for (int node_index : new_plan) {
+    TF_LITE_ENSURE(&context_, node_index >= 0 && node_index < nodes_size());
+  }
+  execution_plan_ = new_plan;
+  return kTfLiteOk;
 }
 
-void Interpreter::UseNNAPI(bool enable) { primary_subgraph().UseNNAPI(enable); }
+TfLiteStatus Interpreter::ResizeTensorImpl(TfLiteTensor* tensor,
+                                           TfLiteIntArray* new_size) {
+  // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too.
+  if (tensor->allocation_type == kTfLiteArenaRw ||
+      tensor->allocation_type == kTfLiteDynamic ||
+      tensor->allocation_type == kTfLiteArenaRwPersistent) {
+    tensor_resized_since_op_invoke_ |=
+        TfLiteIntArrayEqual(tensor->dims, new_size) == 0;
+    if (tensor->type != kTfLiteString) {
+      size_t bytesRequired;
+      TfLiteStatus status = BytesRequired(tensor->type, new_size->data,
+                                          new_size->size, &bytesRequired);
+      if (status != kTfLiteOk) {
+        TfLiteIntArrayFree(new_size);
+        return kTfLiteError;
+      }
+
+      // Realloc space for kTfLiteDynamic tensors.
+      TfLiteTensorRealloc(bytesRequired, tensor);
+      tensor->bytes = bytesRequired;
+    }
+    if (tensor->dims) TfLiteIntArrayFree(tensor->dims);
+    tensor->dims = new_size;
 
-void Interpreter::SetNumThreads(int num_threads) {
-  for (auto& subgraph : subgraphs_) {
-    subgraph.context()->recommended_num_threads = num_threads;
+    if (tensor->allocation_type != kTfLiteDynamic) {
+      tensor->data.raw = nullptr;
+    }
+  } else {
+    // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore
+    // of fixed size.
+    TfLiteIntArrayFree(new_size);
+    ReportError(&context_, "Attempting to resize a fixed-size tensor.");
+    return kTfLiteError;
   }
+  return kTfLiteOk;
+}
+
+void Interpreter::UseNNAPI(bool enable) {
+  // TODO(aselle): This is a workaround for finding if NNAPI exists.
+  // We also need to make sure getLibraryHandle() is renamed to be NNAPI
+  // prefixed.
+  if (!NNAPIDelegate::IsSupported()) enable = false;
+  if (!enable) {
+    nnapi_delegate_.reset();
+  } else if (!nnapi_delegate_) {
+    nnapi_delegate_.reset(new NNAPIDelegate);
+  }
+}
+
+void Interpreter::SetNumThreads(int num_threads) {
+  context_.recommended_num_threads = num_threads;
 
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
     auto* c = external_contexts_[i];
     if (c && c->Refresh) {
-      c->Refresh(context_);
+      c->Refresh(&context_);
     }
   }
 }
 
-void Interpreter::SetAllowFp16PrecisionForFp32(bool allow) {
-  for (auto& subgraph : subgraphs_) {
-    subgraph.context()->allow_fp32_relax_to_fp16 = allow;
-  }
+void Interpreter::SwitchToDelegateContext() {
+  context_.GetNodeAndRegistration = GetNodeAndRegistration;
+  context_.ReplaceNodeSubsetsWithDelegateKernels =
+      ReplaceNodeSubsetsWithDelegateKernels;
+  context_.GetExecutionPlan = GetExecutionPlan;
+}
+
+void Interpreter::SwitchToKernelContext() {
+  SetForbiddenContextFunction(&context_.GetNodeAndRegistration);
+  SetForbiddenContextFunction(&context_.ReplaceNodeSubsetsWithDelegateKernels);
+  SetForbiddenContextFunction(&context_.GetExecutionPlan);
 }
 
 TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
-  return primary_subgraph().ModifyGraphWithDelegate(delegate);
+  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
+    int last_execution_plan_index_prepared;
+    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
+                                     0, &last_execution_plan_index_prepared));
+
+    bool has_dynamic_tensors = true;
+    // Dynamic tensors exist if not all nodes can be prepared.
+    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
+      // If all the nodes can be prepared, check if the last node has dynamic
+      // tensors.
+      int node_index = execution_plan_[last_execution_plan_index_prepared];
+      TfLiteNode& node = nodes_and_registration_[node_index].first;
+      if (!HasDynamicTensor(context_, node.outputs)) {
+        has_dynamic_tensors = false;
+      }
+    }
+    if (has_dynamic_tensors) {
+      ReportError(
+          &context_,
+          "Attempting to use a delegate that only supports static-sized "
+          "tensors with a graph that has dynamic-sized tensors.");
+      return kTfLiteError;
+    }
+  }
+
+  // TODO(aselle): Consider if it is worth storing pointers to delegates.
+  // Setup additional context interface.
+  SwitchToDelegateContext();
+
+  TfLiteStatus status = delegate->Prepare(&context_, delegate);
+
+  // Remove additional context info.
+  SwitchToKernelContext();
+
+  TF_LITE_ENSURE_OK(&context_, status);
+
+  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
+    // Reset the state to force tensor/op reallocation.
+    state_ = kStateUninvokable;
+    TF_LITE_ENSURE_OK(&context_, AllocateTensors());
+    TF_LITE_ENSURE_EQ(&context_, state_, kStateInvokable);
+    // After using a delegate which doesn't support dynamic tensors, make the
+    // entire graph immutable.
+    state_ = kStateInvokableAndImmutable;
+  }
+
+  return status;
 }
 
 TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle buffer_handle,
                                           TfLiteDelegate* delegate) {
-  TF_LITE_ENSURE(context_, tensor_index < tensors_size());
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-  TfLiteTensor* tensor = &tensors[tensor_index];
+  TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
+  TfLiteTensor* tensor = &tensors_[tensor_index];
 
-  TF_LITE_ENSURE(context_,
+  TF_LITE_ENSURE(&context_,
                  tensor->delegate == nullptr || tensor->delegate == delegate);
   tensor->delegate = delegate;
   if (tensor->buffer_handle != kTfLiteNullBufferHandle) {
-    TF_LITE_ENSURE(context_, tensor->delegate->FreeBufferHandle != nullptr);
-    tensor->delegate->FreeBufferHandle(context_, tensor->delegate,
+    TF_LITE_ENSURE(&context_, tensor->delegate->FreeBufferHandle != nullptr);
+    tensor->delegate->FreeBufferHandle(&context_, tensor->delegate,
                                        &tensor->buffer_handle);
   }
   tensor->buffer_handle = buffer_handle;
@@ -180,9 +1009,8 @@ TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
 TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle* buffer_handle,
                                           TfLiteDelegate** delegate) {
-  TF_LITE_ENSURE(context_, tensor_index < tensors_size());
-  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
-  TfLiteTensor* tensor = &tensors[tensor_index];
+  TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
+  TfLiteTensor* tensor = &tensors_[tensor_index];
 
   *delegate = tensor->delegate;
   *buffer_handle = tensor->buffer_handle;
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index a6ad5d9137..de6c42bfcc 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
-#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/memory_planner.h"
 #include "tensorflow/lite/profiling/profiler.h"
 #include "tensorflow/lite/stderr_reporter.h"
@@ -74,6 +73,9 @@ constexpr TfLiteType typeToTfLiteType<string>() {
   return kTfLiteString;
 }
 
+// Forward declare since NNAPIDelegate uses Interpreter.
+class NNAPIDelegate;
+
 // An interpreter for a graph of nodes that input and output from tensors.
 // Each node of the graph processes a set of input tensors and produces a
 // set of output Tensors. All inputs/output tensors are referenced by index.
@@ -102,6 +104,12 @@ constexpr TfLiteType typeToTfLiteType<string>() {
 // foo.Invoke();
 //
 
+struct TfLiteIntArrayDeleter {
+  void operator()(TfLiteIntArray* a) {
+    if (a) TfLiteIntArrayFree(a);
+  }
+};
+
 class Interpreter {
  public:
   // Instantiate an interpreter. All errors associated with reading and
@@ -113,7 +121,6 @@ class Interpreter {
 
   ~Interpreter();
 
-  // Interpreters are not copyable as they have non-trivial memory semantics.
   Interpreter(const Interpreter&) = delete;
   Interpreter& operator=(const Interpreter&) = delete;
 
@@ -194,40 +201,34 @@ class Interpreter {
   // Functions to access tensor data
 
   // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return primary_subgraph().inputs(); }
+  const std::vector<int>& inputs() const { return inputs_; }
 
   // Return the name of a given input. The given index must be between 0 and
   // inputs().size().
   const char* GetInputName(int index) const {
-    return context_->tensors[inputs()[index]].name;
+    return context_.tensors[inputs_[index]].name;
   }
 
   // Read only access to list of outputs.
-  const std::vector<int>& outputs() const {
-    return primary_subgraph().outputs();
-  }
+  const std::vector<int>& outputs() const { return outputs_; }
 
   // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const {
-    return primary_subgraph().variables();
-  }
+  const std::vector<int>& variables() const { return variables_; }
 
   // Return the name of a given output. The given index must be between 0 and
   // outputs().size().
   const char* GetOutputName(int index) const {
-    return context_->tensors[outputs()[index]].name;
+    return context_.tensors[outputs_[index]].name;
   }
 
   // Return the number of tensors in the model.
-  size_t tensors_size() const { return context_->tensors_size; }
+  size_t tensors_size() const { return context_.tensors_size; }
 
   // Return the number of ops in the model.
-  size_t nodes_size() const { return primary_subgraph().nodes_size(); }
+  size_t nodes_size() const { return nodes_and_registration_.size(); }
 
   // WARNING: Experimental interface, subject to change
-  const std::vector<int>& execution_plan() const {
-    return primary_subgraph().execution_plan();
-  }
+  const std::vector<int>& execution_plan() const { return execution_plan_; }
 
   // WARNING: Experimental interface, subject to change
   // Overrides execution plan. This bounds checks indices sent in.
@@ -237,18 +238,27 @@ class Interpreter {
   // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this
   // read/write access to structure
   TfLiteTensor* tensor(int tensor_index) {
-    return primary_subgraph().tensor(tensor_index);
+    if (tensor_index < 0 ||
+        static_cast<size_t>(tensor_index) >= context_.tensors_size)
+      return nullptr;
+    return &context_.tensors[tensor_index];
   }
 
   // Get an immutable tensor data structure.
   const TfLiteTensor* tensor(int tensor_index) const {
-    return primary_subgraph().tensor(tensor_index);
+    if (tensor_index < 0 ||
+        static_cast<size_t>(tensor_index) >= context_.tensors_size)
+      return nullptr;
+    return &context_.tensors[tensor_index];
   }
 
   // Get a pointer to an operation and registration data structure if in bounds.
   const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
       int node_index) const {
-    return primary_subgraph().node_and_registration(node_index);
+    if (node_index < 0 ||
+        static_cast<size_t>(node_index) >= nodes_and_registration_.size())
+      return nullptr;
+    return &nodes_and_registration_[node_index];
   }
 
   // Perform a checked cast to the appropriate tensor type (mutable pointer
@@ -279,28 +289,28 @@ class Interpreter {
   // index must be between 0 and inputs().size().
   template <class T>
   T* typed_input_tensor(int index) {
-    return typed_tensor<T>(inputs()[index]);
+    return typed_tensor<T>(inputs_[index]);
   }
 
   // Return an immutable pointer into the data of a given input tensor. The
   // given index must be between 0 and inputs().size().
   template <class T>
   const T* typed_input_tensor(int index) const {
-    return typed_tensor<T>(inputs()[index]);
+    return typed_tensor<T>(inputs_[index]);
   }
 
   // Return a mutable pointer into the data of a given output tensor. The given
   // index must be between 0 and outputs().size().
   template <class T>
   T* typed_output_tensor(int index) {
-    return typed_tensor<T>(outputs()[index]);
+    return typed_tensor<T>(outputs_[index]);
   }
 
   // Return an immutable pointer into the data of a given output tensor. The
   // given index must be between 0 and outputs().size().
   template <class T>
   const T* typed_output_tensor(int index) const {
-    return typed_tensor<T>(outputs()[index]);
+    return typed_tensor<T>(outputs_[index]);
   }
 
   // Change the dimensionality of a given tensor. Note, this is only acceptable
@@ -315,6 +325,7 @@ class Interpreter {
   // Update allocations for all tensors. This will redim dependent tensors using
   // the input tensor dimensionality as given. This is relatively expensive.
   // If you know that your sizes are not changing, you need not call this.
+
   // Returns status of success or failure.
   TfLiteStatus AllocateTensors();
 
@@ -335,12 +346,14 @@ class Interpreter {
   // Allow float16 precision for FP32 calculation when possible.
   // default: not allow.
   // WARNING: This is an experimental API and subject to change.
-  void SetAllowFp16PrecisionForFp32(bool allow);
+  void SetAllowFp16PrecisionForFp32(bool allow) {
+    context_.allow_fp32_relax_to_fp16 = allow;
+  }
 
   // Get the half precision flag.
   // WARNING: This is an experimental API and subject to change.
   bool GetAllowFp16PrecisionForFp32() const {
-    return context_->allow_fp32_relax_to_fp16;
+    return context_.allow_fp32_relax_to_fp16;
   }
 
   // Owning handle to a TfLiteDelegate instance.
@@ -353,6 +366,24 @@ class Interpreter {
   // WARNING: This is an experimental API and subject to change.
   TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
 
+  // Ensure the data in `tensor.data` is readable. In case delegate is used,
+  // it might require to copy the data from delegate buffer to raw memory.
+  // WARNING: This is an experimental API and subject to change.
+  TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
+    TfLiteTensor* t = tensor(tensor_index);
+    TF_LITE_ENSURE(&context_, t != nullptr);
+    if (t->data_is_stale) {
+      TF_LITE_ENSURE(&context_, t->delegate != nullptr);
+      TF_LITE_ENSURE(&context_, t->buffer_handle != kTfLiteNullBufferHandle);
+      // This can be null if the delegate doesn't use its own buffer.
+      TF_LITE_ENSURE(&context_, t->delegate->CopyFromBufferHandle != nullptr);
+      t->delegate->CopyFromBufferHandle(
+          &context_, t->delegate, t->buffer_handle, t->data.raw, t->bytes);
+      t->data_is_stale = false;
+    }
+    return kTfLiteOk;
+  }
+
   // Set the delegate buffer handle to a tensor. It can be called in the
   // following cases:
   // 1. Set the buffer handle to a tensor that's not being written by a
@@ -408,7 +439,7 @@ class Interpreter {
   const char* OpProfilingString(const TfLiteRegistration& op_reg,
                                 const TfLiteNode* node) const {
     if (op_reg.profiling_string == nullptr) return nullptr;
-    return op_reg.profiling_string(context_, node);
+    return op_reg.profiling_string(&context_, node);
   }
 
   // Set the value of an external context.
@@ -419,19 +450,131 @@ class Interpreter {
   friend class InterpreterBuilder;
   friend class InterpreterTest;
 
-  Subgraph& primary_subgraph() {
-    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
+  // Prevent 'context_' from accessing functions that are only available to
+  // delegated kernels.
+  void SwitchToKernelContext();
+
+  // Add delegate-only functions to 'context_'.
+  void SwitchToDelegateContext();
+
+  // Give 'op_reg' a chance to initialize itself using the contents of
+  // 'buffer'.
+  void* OpInit(const TfLiteRegistration& op_reg, const char* buffer,
+               size_t length) {
+    if (op_reg.init == nullptr) return nullptr;
+    return op_reg.init(&context_, buffer, length);
+  }
+
+  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
+  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
+    if (op_reg.free == nullptr) return;
+    if (buffer) {
+      op_reg.free(&context_, buffer);
+    }
+  }
+
+  // Prepare the given 'node' for execution.
+  TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) {
+    if (op_reg.prepare == nullptr) return kTfLiteOk;
+    return op_reg.prepare(&context_, node);
   }
 
-  const Subgraph& primary_subgraph() const {
-    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
+  // Invoke the operator represented by 'node'.
+  TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) {
+    if (op_reg.invoke == nullptr) return kTfLiteError;
+    return op_reg.invoke(&context_, node);
   }
 
+  // Call OpPrepare() for as many ops as possible, allocating memory for their
+  // tensors. If an op containing dynamic tensors is found, preparation will be
+  // postponed until this function is called again. This allows the interpreter
+  // to wait until Invoke() to resolve the sizes of dynamic tensors.
+  TfLiteStatus PrepareOpsAndTensors();
+
+  // Call OpPrepare() for all ops starting at 'first_node'. Stop when a
+  // dynamic tensors is found or all ops have been prepared. Fill
+  // 'last_node_prepared' with the id of the op containing dynamic tensors, or
+  // the last in the graph.
+  TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index,
+                                    int* last_execution_plan_index_prepared);
+
   // Tensors needed by the interpreter. Use `AddTensors` to add more blank
   // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
   // `context_` whenever this std::vector is reallocated. Currently this
   // only happens in `AddTensors()`.
-  // std::vector<TfLiteTensor> tensors_;
+  std::vector<TfLiteTensor> tensors_;
+
+  // Check if an array of tensor indices are valid with respect to the Tensor
+  // array.
+  // NOTE: this changes consistent_ to be false if indices are out of bounds.
+  TfLiteStatus CheckTensorIndices(const char* label, const int* indices,
+                                  int length);
+
+  // Compute the number of bytes required to represent a tensor with dimensions
+  // specified by the array dims (of length dims_size). Returns the status code
+  // and bytes.
+  TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size,
+                             size_t* bytes);
+
+  // Request an tensor be resized implementation. If the given tensor is of
+  // type kTfLiteDynamic it will also be allocated new memory.
+  TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size);
+
+  // Report a detailed error string (will be printed to stderr).
+  // TODO(aselle): allow user of class to provide alternative destinations.
+  void ReportErrorImpl(const char* format, va_list args);
+
+  // Entry point for C node plugin API to request an tensor be resized.
+  static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor,
+                                   TfLiteIntArray* new_size);
+  // Entry point for C node plugin API to report an error.
+  static void ReportError(TfLiteContext* context, const char* format, ...);
+
+  // Entry point for C node plugin API to add new tensors.
+  static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add,
+                                 int* first_new_tensor_index);
+
+  // WARNING: This is an experimental API and subject to change.
+  // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels
+  static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
+      TfLiteContext* context, TfLiteRegistration registration,
+      const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate);
+
+  // Update the execution graph to replace some of the nodes with stub
+  // nodes. Specifically any node index that has `nodes[index]==1` will be
+  // slated for replacement with a delegate kernel specified by registration.
+  // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller.
+  // WARNING: This is an experimental interface that is subject to change.
+  TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
+      TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
+      TfLiteDelegate* delegate);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Gets the internal pointer to a TensorFlow lite node by node_index.
+  TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node,
+                                      TfLiteRegistration** registration);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Entry point for C node plugin API to get a node by index.
+  static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*,
+                                             int node_index, TfLiteNode** node,
+                                             TfLiteRegistration** registration);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Gets an TfLiteIntArray* representing the execution plan. The interpreter
+  // owns this memory and it is only guaranteed to exist during the invocation
+  // of the delegate prepare.
+  TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Entry point for C node plugin API to get the execution plan.
+  static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context,
+                                       TfLiteIntArray** execution_plan);
+
+  // Retrieve an existing external context by type.
+  TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type);
+  static TfLiteExternalContext* GetExternalContext(
+      struct TfLiteContext* context, TfLiteExternalContextType type);
 
   // Set the value of an external context.
   static void SetExternalContext(struct TfLiteContext* context,
@@ -448,31 +591,105 @@ class Interpreter {
     return ModifyGraphWithDelegate(owned_delegates_.back().get());
   }
 
+  // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra
+  // capacity. Calling this function may invalidate existing pointers to
+  // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
+  // more tensors won't invalidate the pointer to existing tensors.
+  void EnsureTensorsVectorCapacity() {
+    const size_t required_capacity = tensors_size() + kTensorsCapacityHeadroom;
+    if (required_capacity > tensors_.capacity()) {
+      tensors_.reserve(required_capacity);
+      context_.tensors = tensors_.data();
+    }
+  }
+
+  // The state of the Interpreter.
+  enum State {
+    // The interpreter isn't ready to be invoked.
+    // `AllocateTensor` need to be called to enter an invokable state.
+    kStateUninvokable = 0,
+    // The interpreter is ready to be invoked.
+    kStateInvokable,
+    // The interpreter is ready to be invoked, and graph can't be further
+    // modified. The interpreter will enter this state when calling
+    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
+    kStateInvokableAndImmutable,
+  };
+  State state_ = kStateUninvokable;
+
   // A pure C data structure used to communicate with the pure C plugin
   // interface. To avoid copying tensor metadata, this is also the definitive
   // structure to store tensors.
-  // This is the primary subgraph context.
-  TfLiteContext* context_;
+  TfLiteContext context_;
+
+  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
+  // function pointers to actual implementation.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
+      nodes_and_registration_;
+
+  // Whether the model is consistent. That is to say if the inputs and outputs
+  // of every node and the global inputs and outputs are valid indexes into
+  // the tensor array.
+  bool consistent_ = true;
+
+  // Array of indices representing the tensors that are inputs to the
+  // interpreter.
+  std::vector<int> inputs_;
+
+  // Array of indices representing the tensors that are outputs to the
+  // interpreter.
+  std::vector<int> outputs_;
+
+  // Array of indices representing the tensors that are variable tensors.
+  std::vector<int> variables_;
 
   // The error reporter delegate that tflite will forward queries errors to.
   ErrorReporter* error_reporter_;
 
+  // Index of the next node to prepare.
+  // During Invoke(), Interpreter will allocate input tensors first, which are
+  // known to be fixed size. Then it will allocate outputs from nodes as many
+  // as possible. When there is a node that produces dynamic sized tensor.
+  // Interpreter will stop allocating tensors, set the value of next allocate
+  // node id, and execute the node to generate the output tensor before continue
+  // to allocate successors. This process repeats until all nodes are executed.
+  // NOTE: this relies on the order of nodes that is in topological order.
+  int next_execution_plan_index_to_prepare_;
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // This is a list of node indices (to index into nodes_and_registration).
+  // This represents a valid topological sort (dependency ordered) execution
+  // plan. In particular, it is valid for this ordering to contain only a
+  // subset of the node indices.
+  std::vector<int> execution_plan_;
+
+  // In the future, we'd like a TfLiteIntArray compatible representation.
+  // TODO(aselle): replace execution_plan_ with this.
+  std::unique_ptr<TfLiteIntArray, TfLiteIntArrayDeleter> plan_cache_;
+
+  // Whether to delegate to NN API
+  std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
+
   // List of delegates that have been installed and are owned by this
   // interpreter instance. Useful if client delegate ownership is burdensome.
   // WARNING: This is an experimental API and subject to change.
   // TODO(b/116667551): Use TfLiteExternalContext for storing state.
   std::vector<TfLiteDelegatePtr> owned_delegates_;
 
+  std::unique_ptr<MemoryPlanner> memory_planner_;
+
   bool allow_buffer_handle_output_ = false;
 
+  // Tracking bit for whether a tensor was resized in the course of an op
+  // invocation. This is a useful hint to ensure that dynamic tensor outputs
+  // trigger downstream reallocation after op invocation.
+  bool tensor_resized_since_op_invoke_ = false;
+
   // Profiler for this interpreter instance.
   profiling::Profiler* profiler_ = nullptr;
 
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
-
-  // Subgraphs
-  std::vector<Subgraph> subgraphs_;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index 2e0dc77dcd..7f03c3ceba 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -38,7 +38,7 @@ class InterpreterTest : public ::testing::Test {
   }
 
  protected:
-  TfLiteContext* GetInterpreterContext() { return interpreter_.context_; }
+  TfLiteContext* GetInterpreterContext() { return &interpreter_.context_; }
 
   Interpreter interpreter_;
 };
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 983665b7da..a1be2a5abc 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -140,13 +140,13 @@ NNAPIDelegate::~NNAPIDelegate() {
   // ANeuralNetworksShutdown();
 }
 
-// Adds the tensors of the subgraph to the NN API model.
-TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
+// Adds the tensors of the interpreter to the NN API model.
+TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
   uint32_t next_id = 0;
-  for (size_t i = 0; i < subgraph->tensors_size(); i++) {
+  for (size_t i = 0; i < interpreter->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
     if ((*nnapi_ids)[i] == kOperandNotNeeded) continue;
 
@@ -156,7 +156,7 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
     // NNAPI requires 32-bit float scale to be zero, tflite doesn't care
     float scale = 0.0f;
     int32_t zeroPoint = 0;
-    TfLiteTensor* tensor = subgraph->tensor(i);
+    TfLiteTensor* tensor = interpreter->tensor(i);
     switch (tensor->type) {
       case kTfLiteNoType:
         // Tensors added during initialization of Ops don't have a type yet and
@@ -240,12 +240,12 @@ void MapAndAddTensorIds(const int* from_ids_buf, size_t from_ids_count,
 // Adds the operations and their parameters to the NN API model.
 // 'next-id' is the operand ID of the next operand of the model.
 TfLiteStatus AddOpsAndParams(
-    tflite::Subgraph* subgraph, ANeuralNetworksModel* nn_model,
+    tflite::Interpreter* interpreter, ANeuralNetworksModel* nn_model,
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
-  for (size_t i = 0; i < subgraph->nodes_size(); i++) {
-    const auto* node_and_registration = subgraph->node_and_registration(i);
+  for (size_t i = 0; i < interpreter->nodes_size(); i++) {
+    const auto* node_and_registration = interpreter->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
     const TfLiteRegistration& registration = node_and_registration->second;
     tflite::BuiltinOperator builtin =
@@ -291,9 +291,9 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
-         &model_state_outputs](int tensor_id) {
-          const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
+        [interpreter, &nn_model, &next_id, &augmented_inputs,
+         &model_state_inputs, &model_state_outputs](int tensor_id) {
+          const TfLiteTensor* tensor = interpreter->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
@@ -388,11 +388,11 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [interpreter, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
-      const TfLiteTensor* tensor = subgraph->tensor(scratch_buffer_index);
+      const TfLiteTensor* tensor = interpreter->tensor(scratch_buffer_index);
       ANeuralNetworksOperandType operand_type{
           ANEURALNETWORKS_TENSOR_FLOAT32,
           static_cast<uint32_t>(tensor->dims->size),
@@ -584,7 +584,7 @@ TfLiteStatus AddOpsAndParams(
         // The permutation input tensor value dictates the output dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((node.inputs->size > 1) &&
-            (subgraph->tensor(node.inputs->data[1])->allocation_type !=
+            (interpreter->tensor(node.inputs->data[1])->allocation_type !=
              kTfLiteMmapRo)) {
           logError("NNAPI does not yet support dynamic tensors.");
           return kTfLiteError;
@@ -601,13 +601,14 @@ TfLiteStatus AddOpsAndParams(
           return kTfLiteError;
         }
         if ((node.inputs->size > 0) &&
-            (subgraph->tensor(node.inputs->data[0])->dims->size != 4)) {
+            (interpreter->tensor(node.inputs->data[0])->dims->size != 4)) {
           logError("NNAPI only supports input rank 4 for L2Normalization");
           return kTfLiteError;
         }
         break;
       case tflite::BuiltinOperator_HASHTABLE_LOOKUP:
-        if (subgraph->tensor(node.outputs->data[0])->type != kTfLiteFloat32) {
+        if (interpreter->tensor(node.outputs->data[0])->type !=
+            kTfLiteFloat32) {
           logError("NNAPI only support HASHTABLE_LOOKUP with float32 output",
                    builtin);
           return kTfLiteError;
@@ -706,7 +707,7 @@ TfLiteStatus AddOpsAndParams(
   return kTfLiteOk;
 }
 
-TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
+TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
   // TODO(aselle): This is not correct. need to handle resize invalidation.
@@ -718,7 +719,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
     // inputs and outputs and mark the mapping in tensor_id_to_nnapi_id with
     // kOperandIdNotSet. addTensorOperands will replace those with the
     // corresponding NNAPI operand ids and skip kOperandNotNeeded entries.
-    std::vector<int64_t> tensor_id_to_nnapi_id(subgraph->tensors_size(),
+    std::vector<int64_t> tensor_id_to_nnapi_id(interpreter->tensors_size(),
                                                kOperandNotNeeded);
     auto set_ids_to_not_set = [&tensor_id_to_nnapi_id](const int* buf,
                                                        size_t count) {
@@ -729,31 +730,35 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
         }
       }
     };
-    for (size_t i = 0; i < subgraph->nodes_size(); i++) {
-      const auto* node_and_registration = subgraph->node_and_registration(i);
+    for (size_t i = 0; i < interpreter->nodes_size(); i++) {
+      const auto* node_and_registration = interpreter->node_and_registration(i);
       const TfLiteNode& node = node_and_registration->first;
       set_ids_to_not_set(node.inputs->data, node.inputs->size);
       set_ids_to_not_set(node.outputs->data, node.outputs->size);
     }
-    set_ids_to_not_set(subgraph->inputs().data(), subgraph->inputs().size());
-    set_ids_to_not_set(subgraph->outputs().data(), subgraph->outputs().size());
+    set_ids_to_not_set(interpreter->inputs().data(),
+                       interpreter->inputs().size());
+    set_ids_to_not_set(interpreter->outputs().data(),
+                       interpreter->outputs().size());
 
     uint32_t next_id = 0;
     RETURN_ERROR_IF_TFLITE_FAILED(addTensorOperands(
-        subgraph, nn_model_, &next_id, &tensor_id_to_nnapi_id));
+        interpreter, nn_model_, &next_id, &tensor_id_to_nnapi_id));
     RETURN_ERROR_IF_TFLITE_FAILED(
-        AddOpsAndParams(subgraph, nn_model_, next_id, &model_states_inputs_,
+        AddOpsAndParams(interpreter, nn_model_, next_id, &model_states_inputs_,
                         &model_states_outputs_, tensor_id_to_nnapi_id));
 
     std::vector<uint32_t> augmented_inputs;
-    MapAndAddTensorIds(subgraph->inputs().data(), subgraph->inputs().size(),
-                       &augmented_inputs, tensor_id_to_nnapi_id);
+    MapAndAddTensorIds(interpreter->inputs().data(),
+                       interpreter->inputs().size(), &augmented_inputs,
+                       tensor_id_to_nnapi_id);
     augmented_inputs.insert(augmented_inputs.end(),
                             model_states_inputs_.begin(),
                             model_states_inputs_.end());
     std::vector<uint32_t> augmented_outputs;
-    MapAndAddTensorIds(subgraph->outputs().data(), subgraph->outputs().size(),
-                       &augmented_outputs, tensor_id_to_nnapi_id);
+    MapAndAddTensorIds(interpreter->outputs().data(),
+                       interpreter->outputs().size(), &augmented_outputs,
+                       tensor_id_to_nnapi_id);
     MapAndAddTensorIds(model_states_outputs_.data(),
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
@@ -766,7 +771,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
 
     if (GetAndroidSdkVersionCached() >= 28) {
       CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-          nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
+          nn_model_, interpreter->GetAllowFp16PrecisionForFp32()));
     }
     CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
   }
@@ -777,9 +782,9 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   return kTfLiteOk;
 }
 
-TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
+TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
   if (!nn_model_) {
-    model_status_ = BuildGraph(subgraph);
+    model_status_ = BuildGraph(interpreter);
     if (model_status_ != kTfLiteOk) {
       logError("Failed to build graph for NNAPI");
     }
@@ -792,19 +797,19 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
-  for (size_t i = 0; i < subgraph->inputs().size(); i++) {
-    int input = subgraph->inputs()[i];
+  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
+    int input = interpreter->inputs()[i];
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
-    TfLiteTensor* tensor = subgraph->tensor(input);
+    TfLiteTensor* tensor = interpreter->tensor(input);
     CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
   // Tell nn api where to place final data.
-  for (size_t i = 0; i < subgraph->outputs().size(); i++) {
-    int output = subgraph->outputs()[i];
-    TfLiteTensor* tensor = subgraph->tensor(output);
+  for (size_t i = 0; i < interpreter->outputs().size(); i++) {
+    int output = interpreter->outputs()[i];
+    TfLiteTensor* tensor = interpreter->tensor(output);
     CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
@@ -813,16 +818,16 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   // current invocation.
   for (size_t i = 0; i < model_states_outputs_.size(); i++) {
     int state_tensor_idx = model_states_outputs_[i];
-    TfLiteTensor* tensor = subgraph->tensor(state_tensor_idx);
+    TfLiteTensor* tensor = interpreter->tensor(state_tensor_idx);
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
     CHECK_NN(ANeuralNetworksExecution_setInput(
-        execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
+        execution, i + interpreter->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
     CHECK_NN(ANeuralNetworksExecution_setOutput(
-        execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
+        execution, i + interpreter->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
@@ -835,9 +840,9 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
 
 #if 0
   printf("From the NN API:\n");
-  TfLiteTensor* tensor = subgraph->tensor(subgraph->outputs()[0]);
+  TfLiteTensor* tensor = interpreter->tensor(interpreter->outputs()[0]);
   if (float* data =
-          subgraph->typed_tensor<float>(subgraph->outputs()[0])) {
+          interpreter->typed_tensor<float>(interpreter->outputs()[0])) {
     size_t num = tensor->bytes / sizeof(float);
     for (float* p = data; p < data + num; p++) {
       printf(" %f", *p);
diff --git a/tensorflow/lite/nnapi_delegate.h b/tensorflow/lite/nnapi_delegate.h
index b4f8e4ecf3..63b408c141 100644
--- a/tensorflow/lite/nnapi_delegate.h
+++ b/tensorflow/lite/nnapi_delegate.h
@@ -18,7 +18,6 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
-#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/interpreter.h"
 
 class ANeuralNetworksModel;
@@ -51,10 +50,10 @@ class NNAPIDelegate {
   ~NNAPIDelegate();
 
   // Convert a tflite graph to NNAPI
-  TfLiteStatus BuildGraph(Subgraph* subgraph);
+  TfLiteStatus BuildGraph(Interpreter* interpreter);
 
   // Run
-  TfLiteStatus Invoke(Subgraph* subgraph);
+  TfLiteStatus Invoke(Interpreter* interpreter);
 
   // Whether the current platform supports NNAPI delegation.
   static bool IsSupported();
diff --git a/tensorflow/lite/nnapi_delegate_disabled.cc b/tensorflow/lite/nnapi_delegate_disabled.cc
index a8f2c0bfe3..44dc21f1b6 100644
--- a/tensorflow/lite/nnapi_delegate_disabled.cc
+++ b/tensorflow/lite/nnapi_delegate_disabled.cc
@@ -35,11 +35,13 @@ NNAPIDelegate::~NNAPIDelegate() {
 #undef UNUSED_MEMBER
 }
 
-TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
+TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
   return kTfLiteError;
 }
 
-TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) { return kTfLiteError; }
+TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
+  return kTfLiteError;
+}
 
 bool NNAPIDelegate::IsSupported() { return false; }
 
diff --git a/tensorflow/lite/util.h b/tensorflow/lite/util.h
index dbb87528d0..64a5b52e2f 100644
--- a/tensorflow/lite/util.h
+++ b/tensorflow/lite/util.h
@@ -52,12 +52,6 @@ bool EqualArrayAndTfLiteIntArray(const TfLiteIntArray* a, const int b_size,
 
 size_t CombineHashes(std::initializer_list<size_t> hashes);
 
-struct TfLiteIntArrayDeleter {
-  void operator()(TfLiteIntArray* a) {
-    if (a) TfLiteIntArrayFree(a);
-  }
-};
-
 }  // namespace tflite
 
 #endif  // TENSORFLOW_LITE_UTIL_H_
-- 
GitLab


From d64fcd6a55e51ddf892a799ebc877325ae2de978 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Fri, 16 Nov 2018 22:39:21 -0800
Subject: [PATCH 0473/1554] TensorFlow distribute: Multi worker iterator tests
 don't work in eager mode.

PiperOrigin-RevId: 221896160
---
 tensorflow/contrib/distribute/python/values_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index d3ff14e087..27077656e4 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -712,7 +712,7 @@ class InputIteratorMultiWorkerTest(
     ]
 
   @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
+      mode=["graph"],
       input_type=["input_fn", "dataset"]))
   def testOneDevicePerWorker(self, input_type):
     worker_devices = self._cpu_devices()
@@ -722,7 +722,7 @@ class InputIteratorMultiWorkerTest(
                           [[0, 0], [1, 1], [2, 2], [3, 3]], sess)
 
   @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
+      mode=["graph"],
       input_type=["input_fn", "dataset"],
       required_gpus=1))
   def testTwoDevicesPerWorker(self, input_type):
@@ -733,7 +733,7 @@ class InputIteratorMultiWorkerTest(
                           [[0, 1, 0, 1], [2, 3, 2, 3]], sess)
 
   @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
+      mode=["graph"],
       input_type=["input_fn", "dataset"]))
   def testTupleDataset(self, input_type):
     worker_devices = self._cpu_devices()
-- 
GitLab


From 12d86380c897be13c16bfebf09ddd6593c464655 Mon Sep 17 00:00:00 2001
From: "Li, Guizi" <guizi.li@intel.com>
Date: Sat, 17 Nov 2018 15:21:59 +0800
Subject: [PATCH 0474/1554] update mklml version to make it consistent with
 mkldnn.

---
 tensorflow/workspace.bzl | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index fd800cf67d..b3c4938dd0 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -77,31 +77,31 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     mkl_repository(
         name = "mkl_linux",
         build_file = clean_dep("//third_party/mkl:mkl.BUILD"),
-        sha256 = "e2233534a9d15c387e22260997af4312a39e9f86f791768409be273b5453c4e6",
-        strip_prefix = "mklml_lnx_2019.0.20180710",
+        sha256 = "f00dc3b142a5be399bdeebd7e7ea369545a35d4fb84c86f98b6b048d72685295",
+        strip_prefix = "mklml_lnx_2019.0.1.20180928",
         urls = [
-            "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.16/mklml_lnx_2019.0.20180710.tgz",
-            "https://github.com/intel/mkl-dnn/releases/download/v0.16/mklml_lnx_2019.0.20180710.tgz",
+            "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz",
+            "https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz",
         ],
     )
     mkl_repository(
         name = "mkl_windows",
         build_file = clean_dep("//third_party/mkl:mkl.BUILD"),
-        sha256 = "3fdcff17b018a0082491adf3ba143358265336a801646e46e0191ec8d58d24a2",
-        strip_prefix = "mklml_win_2019.0.20180710",
+        sha256 = "efef90b7b9613fab10f44c8ac4ff28db613a112c64ed94826d7e44df09c44b0b",
+        strip_prefix = "mklml_win_2019.0.1.20180928",
         urls = [
-            "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.16/mklml_win_2019.0.20180710.zip",
-            "https://github.com/intel/mkl-dnn/releases/download/v0.16/mklml_win_2019.0.20180710.zip",
+            "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_win_2019.0.1.20180928.zip",
+            "https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_win_2019.0.1.20180928.zip",
         ],
     )
     mkl_repository(
         name = "mkl_darwin",
         build_file = clean_dep("//third_party/mkl:mkl.BUILD"),
-        sha256 = "411a30014a938eb83fb9f37b3dbe8e371b106fc1dd621fc23123cadc72737ce6",
-        strip_prefix = "mklml_mac_2019.0.20180710",
+        sha256 = "83f02938a0c095274db7b8b7b694157abafa3837c5cbaef740440d466c86a477",
+        strip_prefix = "mklml_mac_2019.0.1.20180928",
         urls = [
-            "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.16/mklml_mac_2019.0.20180710.tgz",
-            "https://github.com/intel/mkl-dnn/releases/download/v0.16/mklml_mac_2019.0.20180710.tgz",
+            "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_mac_2019.0.1.20180928.tgz",
+            "https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_mac_2019.0.1.20180928.tgz",
         ],
     )
 
-- 
GitLab


From 495a3a3dd3ca24fa1c93da28420322ed66844791 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 17 Nov 2018 01:02:07 -0800
Subject: [PATCH 0475/1554] compat: Update forward compatibility horizon to
 2018-11-17

PiperOrigin-RevId: 221904112
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index a12f798d8f..b3f98a12fb 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 16)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 17)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 67d114bfceef73c9e75badcbe34484a35513e492 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Sat, 17 Nov 2018 11:29:27 -0800
Subject: [PATCH 0476/1554] Deprecate tf.debugging. APIs for TF 2.0

Deprecate
tf.debugging.(is_finite|is_inf|is_nan|is_non_decreasing|is_strictly_increasing)
and rename to tf.math. variants.

PiperOrigin-RevId: 221931279
---
 .../api_def/python_api/api_def_IsFinite.pbtxt |  4 ++++
 .../api_def/python_api/api_def_IsInf.pbtxt    |  4 ++++
 .../api_def/python_api/api_def_IsNan.pbtxt    |  4 ++++
 tensorflow/python/ops/check_ops.py            | 20 +++++++++++++------
 .../tools/api/golden/v1/tensorflow.math.pbtxt | 20 +++++++++++++++++++
 .../api/golden/v2/tensorflow.debugging.pbtxt  | 20 -------------------
 .../tools/api/golden/v2/tensorflow.math.pbtxt | 20 +++++++++++++++++++
 tensorflow/tools/compatibility/renames_v2.py  | 15 +++++++++-----
 8 files changed, 76 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_IsFinite.pbtxt b/tensorflow/core/api_def/python_api/api_def_IsFinite.pbtxt
index 91160bd8bf..ccd736a483 100644
--- a/tensorflow/core/api_def/python_api/api_def_IsFinite.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_IsFinite.pbtxt
@@ -1,7 +1,11 @@
 op {
   graph_op_name: "IsFinite"
+  endpoint {
+    name: "math.is_finite"
+  }
   endpoint {
     name: "debugging.is_finite"
+    deprecation_version: 2
   }
   endpoint {
     name: "is_finite"
diff --git a/tensorflow/core/api_def/python_api/api_def_IsInf.pbtxt b/tensorflow/core/api_def/python_api/api_def_IsInf.pbtxt
index 7f029ee8cf..3cbfb7317c 100644
--- a/tensorflow/core/api_def/python_api/api_def_IsInf.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_IsInf.pbtxt
@@ -1,7 +1,11 @@
 op {
   graph_op_name: "IsInf"
+  endpoint {
+    name: "math.is_inf"
+  }
   endpoint {
     name: "debugging.is_inf"
+    deprecation_version: 2
   }
   endpoint {
     name: "is_inf"
diff --git a/tensorflow/core/api_def/python_api/api_def_IsNan.pbtxt b/tensorflow/core/api_def/python_api/api_def_IsNan.pbtxt
index f2b8862c28..b01536664e 100644
--- a/tensorflow/core/api_def/python_api/api_def_IsNan.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_IsNan.pbtxt
@@ -1,7 +1,11 @@
 op {
   graph_op_name: "IsNan"
+  endpoint {
+    name: "math.is_nan"
+  }
   endpoint {
     name: "debugging.is_nan"
+    deprecation_version: 2
   }
   endpoint {
     name: "is_nan"
diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py
index 75dad172fc..f1f36269cf 100644
--- a/tensorflow/python/ops/check_ops.py
+++ b/tensorflow/python/ops/check_ops.py
@@ -1530,9 +1530,13 @@ def is_numeric_tensor(tensor):
 
 
 @tf_export(
-    'debugging.is_non_decreasing',
-    v1=['debugging.is_non_decreasing', 'is_non_decreasing'])
-@deprecation.deprecated_endpoints('is_non_decreasing')
+    'math.is_non_decreasing',
+    v1=[
+        'math.is_non_decreasing', 'debugging.is_non_decreasing',
+        'is_non_decreasing'
+    ])
+@deprecation.deprecated_endpoints('debugging.is_non_decreasing',
+                                  'is_non_decreasing')
 def is_non_decreasing(x, name=None):
   """Returns `True` if `x` is non-decreasing.
 
@@ -1560,9 +1564,13 @@ def is_non_decreasing(x, name=None):
 
 
 @tf_export(
-    'debugging.is_strictly_increasing',
-    v1=['debugging.is_strictly_increasing', 'is_strictly_increasing'])
-@deprecation.deprecated_endpoints('is_strictly_increasing')
+    'math.is_strictly_increasing',
+    v1=[
+        'math.is_strictly_increasing', 'debugging.is_strictly_increasing',
+        'is_strictly_increasing'
+    ])
+@deprecation.deprecated_endpoints('debugging.is_strictly_increasing',
+                                  'is_strictly_increasing')
 def is_strictly_increasing(x, name=None):
   """Returns `True` if `x` is strictly increasing.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
index a8334fdd1d..67f348be21 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
@@ -176,6 +176,26 @@ tf_module {
     name: "invert_permutation"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "is_finite"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_inf"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_nan"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_non_decreasing"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_strictly_increasing"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "l2_normalize"
     argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
index 5d8afcd8b2..fbc5cd2df0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
@@ -88,28 +88,8 @@ tf_module {
     name: "check_numerics"
     argspec: "args=[\'tensor\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "is_finite"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "is_inf"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "is_nan"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "is_non_decreasing"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "is_numeric_tensor"
     argspec: "args=[\'tensor\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "is_strictly_increasing"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index a7638ffde6..e6b8fd225d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -176,6 +176,26 @@ tf_module {
     name: "invert_permutation"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "is_finite"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_inf"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_nan"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_non_decreasing"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "is_strictly_increasing"
+    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "l2_normalize"
     argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 423ff9cdc3..e2bf6bf3a7 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -104,6 +104,11 @@ renames = {
     'tf.count_up_to': 'tf.compat.v1.count_up_to',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
+    'tf.debugging.is_finite': 'tf.math.is_finite',
+    'tf.debugging.is_inf': 'tf.math.is_inf',
+    'tf.debugging.is_nan': 'tf.math.is_nan',
+    'tf.debugging.is_non_decreasing': 'tf.math.is_non_decreasing',
+    'tf.debugging.is_strictly_increasing': 'tf.math.is_strictly_increasing',
     'tf.decode_base64': 'tf.io.decode_base64',
     'tf.decode_compressed': 'tf.io.decode_compressed',
     'tf.decode_csv': 'tf.io.decode_csv',
@@ -196,12 +201,12 @@ renames = {
     'tf.initializers.tables_initializer': 'tf.compat.v1.initializers.tables_initializer',
     'tf.initializers.variables': 'tf.compat.v1.initializers.variables',
     'tf.invert_permutation': 'tf.math.invert_permutation',
-    'tf.is_finite': 'tf.debugging.is_finite',
-    'tf.is_inf': 'tf.debugging.is_inf',
-    'tf.is_nan': 'tf.debugging.is_nan',
-    'tf.is_non_decreasing': 'tf.debugging.is_non_decreasing',
+    'tf.is_finite': 'tf.math.is_finite',
+    'tf.is_inf': 'tf.math.is_inf',
+    'tf.is_nan': 'tf.math.is_nan',
+    'tf.is_non_decreasing': 'tf.math.is_non_decreasing',
     'tf.is_numeric_tensor': 'tf.debugging.is_numeric_tensor',
-    'tf.is_strictly_increasing': 'tf.debugging.is_strictly_increasing',
+    'tf.is_strictly_increasing': 'tf.math.is_strictly_increasing',
     'tf.is_variable_initialized': 'tf.compat.v1.is_variable_initialized',
     'tf.keras.backend.get_session': 'tf.compat.v1.keras.backend.get_session',
     'tf.layers.AveragePooling1D': 'tf.compat.v1.layers.AveragePooling1D',
-- 
GitLab


From 1f33c8d2ed08d672881d871e0b1476012a8bfaed Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 17 Nov 2018 12:23:04 -0800
Subject: [PATCH 0477/1554] Remove "_native_" from
 depthwise_conv2d_native_backprop_input in TF 2.0 API

New name (depthwise_conv2d_backprop_input), same semantics.

PiperOrigin-RevId: 221933568
---
 .../api_def_DepthwiseConv2dNativeBackpropInput.pbtxt  |  5 +++++
 tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt    |  4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt    |  8 ++++----
 tensorflow/tools/compatibility/renames_v2.py          | 11 +++++++----
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt b/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt
index 0bd72539e9..f32aa8a69f 100644
--- a/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt
@@ -2,5 +2,10 @@ op {
   graph_op_name: "DepthwiseConv2dNativeBackpropInput"
   endpoint {
     name: "nn.depthwise_conv2d_native_backprop_input"
+    deprecated: true
+    deprecation_version: 2
+  }
+  endpoint {
+    name: "nn.depthwise_conv2d_backprop_input"
   }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index cb28ea8424..233567d600 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -116,6 +116,10 @@ tf_module {
     name: "depthwise_conv2d"
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "depthwise_conv2d_backprop_input"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+  }
   member_method {
     name: "depthwise_conv2d_native"
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index a38fda6584..0d2d6d682e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -109,12 +109,12 @@ tf_module {
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
-    name: "depthwise_conv2d_native_backprop_filter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    name: "depthwise_conv2d_backprop_input"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
   }
   member_method {
-    name: "depthwise_conv2d_native_backprop_input"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    name: "depthwise_conv2d_native_backprop_filter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
   }
   member_method {
     name: "dilation2d"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index e2bf6bf3a7..294cd66fdc 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -57,6 +57,7 @@ renames = {
     'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator',
     'tf.SparseFeature': 'tf.io.SparseFeature',
     'tf.TFRecordReader': 'tf.compat.v1.TFRecordReader',
+    'tf.TensorInfo': 'tf.compat.v1.TensorInfo',
     'tf.TextLineReader': 'tf.compat.v1.TextLineReader',
     'tf.VERSION': 'tf.version.VERSION',
     'tf.VarLenFeature': 'tf.io.VarLenFeature',
@@ -116,7 +117,6 @@ renames = {
     'tf.decode_raw': 'tf.io.decode_raw',
     'tf.delete_session_tensor': 'tf.compat.v1.delete_session_tensor',
     'tf.depth_to_space': 'tf.nn.depth_to_space',
-    'tf.depthwise_conv2d_native': 'tf.compat.v1.depthwise_conv2d_native',
     'tf.dequantize': 'tf.quantization.dequantize',
     'tf.deserialize_many_sparse': 'tf.io.deserialize_many_sparse',
     'tf.diag': 'tf.linalg.tensor_diag',
@@ -174,6 +174,7 @@ renames = {
     'tf.get_session_tensor': 'tf.compat.v1.get_session_tensor',
     'tf.get_variable': 'tf.compat.v1.get_variable',
     'tf.get_variable_scope': 'tf.compat.v1.get_variable_scope',
+    'tf.gfile.Exists': 'tf.compat.v1.gfile.Exists',
     'tf.gfile.FastGFile': 'tf.compat.v1.gfile.FastGFile',
     'tf.gfile.GFile': 'tf.compat.v1.gfile.GFile',
     'tf.gfile.Open': 'tf.compat.v1.gfile.Open',
@@ -301,6 +302,7 @@ renames = {
     'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
     'tf.nn.depthwise_conv2d_native': 'tf.compat.v1.nn.depthwise_conv2d_native',
+    'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
     'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool',
@@ -349,7 +351,7 @@ renames = {
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
-    'tf.random_poisson': 'tf.random.poisson',
+    'tf.random_poisson': 'tf.compat.v1.random_poisson',
     'tf.random_shuffle': 'tf.random.shuffle',
     'tf.random_uniform': 'tf.random.uniform',
     'tf.read_file': 'tf.io.read_file',
@@ -437,6 +439,7 @@ renames = {
     'tf.space_to_batch': 'tf.nn.space_to_batch',
     'tf.space_to_depth': 'tf.nn.space_to_depth',
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
+    'tf.sparse.reduce_max_sparse': 'tf.compat.v1.sparse.reduce_max_sparse',
     'tf.sparse_add': 'tf.sparse.add',
     'tf.sparse_fill_empty_rows': 'tf.sparse.fill_empty_rows',
     'tf.sparse_mask': 'tf.sparse.mask',
@@ -445,8 +448,8 @@ renames = {
     'tf.sparse_merge': 'tf.sparse.merge',
     'tf.sparse_minimum': 'tf.sparse.minimum',
     'tf.sparse_placeholder': 'tf.compat.v1.sparse_placeholder',
-    'tf.sparse_reduce_max': 'tf.sparse.reduce_max',
-    'tf.sparse_reduce_max_sparse': 'tf.sparse.reduce_max_sparse',
+    'tf.sparse_reduce_max': 'tf.compat.v1.sparse_reduce_max',
+    'tf.sparse_reduce_max_sparse': 'tf.compat.v1.sparse_reduce_max_sparse',
     'tf.sparse_reduce_sum': 'tf.sparse.reduce_sum',
     'tf.sparse_reduce_sum_sparse': 'tf.sparse.reduce_sum_sparse',
     'tf.sparse_reorder': 'tf.sparse.reorder',
-- 
GitLab


From 8697d63b374acfc2fa85913a557f52f67d616dde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 17 Nov 2018 14:23:58 -0800
Subject: [PATCH 0478/1554] Fix bug with `global_step.assign_add(1)` when using
 multiple GPUs per machine and `distribute.ParameterServerStrategy`. Add some
 testing for this case with `MirroredStrategy` as well.

PiperOrigin-RevId: 221938703
---
 .../distribute/python/mirrored_strategy.py    |  7 +++++-
 .../python/mirrored_strategy_multigpu_test.py |  6 +++++
 .../python/parameter_server_strategy.py       |  7 ++++++
 .../python/parameter_server_strategy_test.py  |  5 ++++
 .../distribute/python/strategy_test_lib.py    | 24 +++++++++++++++++++
 5 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 370c96ac4f..f7432162cb 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -587,7 +587,12 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
     return ctx
 
   def _broadcast_to(self, tensor, destinations):
-    if isinstance(tensor, (float, int)):  # Fast path for Python constants.
+    # This is both a fast path for Python constants, and a way to delay
+    # converting Python values to a tensor until we know what type it
+    # should be converted to. Otherwise we have trouble with:
+    #   global_step.assign_add(1)
+    # since the `1` gets broadcast as an int32 but global_step is int64.
+    if isinstance(tensor, (float, int)):
       return tensor
     # TODO(josh11b): In eager mode, use one thread per device, or async mode.
     return self._get_cross_device_ops().broadcast(
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 76fdc6f762..2cc56565d3 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -162,6 +162,12 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
     iterator = d.make_input_fn_iterator(input_fn)
     self._test_input_fn_iterator(iterator, d.worker_devices, expected_values)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testGlobalStepUpdate(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self._test_global_step_update(self._get_distribution_strategy())
+
 
 class MirroredStrategyVariableCreationTest(test.TestCase):
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 4419d4afe1..6fc81a1e57 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -257,6 +257,13 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
         input_fn, worker_device_pairs, [input_context])
 
   def _broadcast_to(self, tensor, destinations):
+    # This is both a fast path for Python constants, and a way to delay
+    # converting Python values to a tensor until we know what type it
+    # should be converted to. Otherwise we have trouble with:
+    #   global_step.assign_add(1)
+    # since the `1` gets broadcast as an int32 but global_step is int64.
+    if isinstance(tensor, (float, int)):
+      return tensor
     if not cross_device_ops_lib.check_destinations(destinations):
       destinations = self._compute_devices
     return self._cross_device_ops.broadcast(tensor, destinations)
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 5d5e65600c..b4c098aa57 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -651,6 +651,11 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
     self._test_input_fn_iterator(None, None, num_gpus,
                                  input_fn, expected_values)
 
+  def testGlobalStepUpdate(self):
+    strategy = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=context.num_gpus())
+    self._test_global_step_update(strategy)
+
 
 class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                                            parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index b9e3761738..d94ebdc06b 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -25,10 +25,13 @@ from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import optimizer
@@ -263,3 +266,24 @@ class DistributionTestBase(test.TestCase):
           [values.select_device(d, next_element) for d in devices])
       self.assertEqual(expected_value, computed_value)
 
+  def _test_global_step_update(self, strategy):
+    with strategy.scope():
+      global_step = variable_scope.get_variable(
+          "global_step",
+          shape=[],
+          dtype=dtypes.int64,
+          initializer=init_ops.zeros_initializer(),
+          trainable=False,
+          aggregation=variables.VariableAggregation.ONLY_FIRST_REPLICA)
+      self.evaluate(variables.global_variables_initializer())
+
+      def model_fn():
+        train_op = global_step.assign_add(1)
+        value = global_step.read_value()
+        return train_op, value
+
+      train_ops, value = strategy.call_for_each_replica(model_fn)
+      self.evaluate(strategy.group(train_ops))
+      global_step_tensors = strategy.unwrap(value)
+      global_step_values = self.evaluate(global_step_tensors)
+      self.assertEqual([1] * len(global_step_tensors), global_step_values)
-- 
GitLab


From 27171a09e5812d3c8d237c69aa5d53250e7f1696 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 18 Nov 2018 01:28:50 +0000
Subject: [PATCH 0479/1554] Fix deprecated div

While running test I noticed the following warning:
```
WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn_ops.py:2744: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
```

This fix fixes the deprecated warning.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/nn_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 21008fc392..223a37c87e 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2741,7 +2741,7 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):  # pylint: di
         noise_shape, seed=seed, dtype=x.dtype)
     # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
     binary_tensor = math_ops.floor(random_tensor)
-    ret = math_ops.div(x, keep_prob) * binary_tensor
+    ret = math_ops.divide(x, keep_prob) * binary_tensor
     if not context.executing_eagerly():
       ret.set_shape(x.get_shape())
     return ret
-- 
GitLab


From 0c610a2be55f6d5e929f554d736b6bfa4ae35e81 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 17 Nov 2018 22:53:18 -0800
Subject: [PATCH 0480/1554] Automated rollback of commit
 6b5bef9216ae89067e2a600772ac17a0ca4a5010

PiperOrigin-RevId: 221958705
---
 tensorflow/compiler/jit/BUILD                 | 26 ------
 .../compiler/jit/register_xla_cpu_jit.cc      | 22 -----
 .../compiler/jit/register_xla_gpu_jit.cc      | 22 -----
 tensorflow/core/BUILD                         | 14 ----
 .../common_runtime/graph_execution_state.cc   |  3 -
 tensorflow/core/common_runtime/tf_xla_stub.cc | 82 -------------------
 tensorflow/core/common_runtime/tf_xla_stub.h  | 50 -----------
 7 files changed, 219 deletions(-)
 delete mode 100644 tensorflow/compiler/jit/register_xla_cpu_jit.cc
 delete mode 100644 tensorflow/compiler/jit/register_xla_gpu_jit.cc
 delete mode 100644 tensorflow/core/common_runtime/tf_xla_stub.cc
 delete mode 100644 tensorflow/core/common_runtime/tf_xla_stub.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index bfc0b0070e..682c0f0cb0 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -45,37 +45,15 @@ cc_library(
     alwayslink = 1,
 )
 
-cc_library(
-    name = "register_xla_cpu_jit",
-    srcs = ["register_xla_cpu_jit.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/core:tf_xla_stub",
-    ],
-    alwayslink = 1,
-)
-
 cc_library(
     name = "xla_cpu_jit",
     visibility = ["//visibility:public"],
     deps = [
         ":jit_compilation_passes",
-        ":register_xla_cpu_jit",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/xla/service:cpu_plugin",
-        "//tensorflow/core:tf_xla_stub",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "register_xla_gpu_jit",
-    srcs = ["register_xla_gpu_jit.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/core:tf_xla_stub",
     ],
     alwayslink = 1,
 )
@@ -85,8 +63,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = if_cuda([
         ":jit_compilation_passes",
-        ":register_xla_gpu_jit",
-        "//tensorflow/core:tf_xla_stub",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops",
@@ -102,7 +78,6 @@ cc_library(
     deps = [
         ":flags",
         ":jit_compilation_passes",
-        ":register_xla_cpu_jit",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla:xla_compiler",
@@ -121,7 +96,6 @@ cc_library(
     visibility = [":friends"],
     deps = [
         ":jit_compilation_passes",
-        ":register_xla_gpu_jit",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_ops",
         "//tensorflow/compiler/tf2xla:xla_compiler",
diff --git a/tensorflow/compiler/jit/register_xla_cpu_jit.cc b/tensorflow/compiler/jit/register_xla_cpu_jit.cc
deleted file mode 100644
index 9cbef63127..0000000000
--- a/tensorflow/compiler/jit/register_xla_cpu_jit.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/common_runtime/tf_xla_stub.h"
-
-namespace tensorflow {
-namespace {
-XlaCpuJitIsLinkedIn register_xla_cpu_jit;
-}
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/register_xla_gpu_jit.cc b/tensorflow/compiler/jit/register_xla_gpu_jit.cc
deleted file mode 100644
index 7399a41d25..0000000000
--- a/tensorflow/compiler/jit/register_xla_gpu_jit.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/common_runtime/tf_xla_stub.h"
-
-namespace tensorflow {
-namespace {
-XlaGpuJitIsLinkedIn register_xla_gpu_jit;
-}
-}  // namespace tensorflow
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1162c47e6b..73e8db58a8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2976,19 +2976,6 @@ tf_cuda_library(
     ] + if_static([":core_cpu_impl"]) + tf_protos_all() + tf_protos_grappler(),
 )
 
-tf_cuda_library(
-    name = "tf_xla_stub",
-    srcs = ["common_runtime/tf_xla_stub.cc"],
-    hdrs = ["common_runtime/tf_xla_stub.h"],
-    copts = tf_copts(),
-    visibility = ["//visibility:public"],
-    deps = [
-        ":lib",
-        ":proto_text",
-        ":session_options",
-    ],
-)
-
 tf_cuda_library(
     name = "core_cpu_internal",
     srcs = [
@@ -3002,7 +2989,6 @@ tf_cuda_library(
         ":framework",
         ":graph",
         ":lib",
-        ":tf_xla_stub",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index c18268ad7b..0d36930324 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
-#include "tensorflow/core/common_runtime/tf_xla_stub.h"
 #include "tensorflow/core/framework/graph.pb_text.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -721,8 +720,6 @@ Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
   CHECK_EQ(options.callable_options.fetch_size(),
            rewrite_metadata.fetch_types.size());
 
-  TF_RETURN_IF_ERROR(CheckXlaJitOptimizerOptions(session_options_));
-
   // TODO(andydavis): Clarify optimization pass requirements around CostModel.
   GraphOptimizationPassOptions optimization_options;
   optimization_options.session_options = session_options_;
diff --git a/tensorflow/core/common_runtime/tf_xla_stub.cc b/tensorflow/core/common_runtime/tf_xla_stub.cc
deleted file mode 100644
index d463693669..0000000000
--- a/tensorflow/core/common_runtime/tf_xla_stub.cc
+++ /dev/null
@@ -1,82 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <cstdlib>
-
-#include "tensorflow/core/common_runtime/tf_xla_stub.h"
-#include "tensorflow/core/lib/core/errors.h"
-
-namespace tensorflow {
-namespace {
-
-bool is_xla_gpu_jit_registered = false;
-bool is_xla_cpu_jit_registered = false;
-
-struct XlaEnvVars {
-  bool xla_flags_env_var_present;
-  bool tf_xla_flags_env_var_present;
-};
-
-XlaEnvVars ComputeEnvVarHasXlaFlags() {
-  XlaEnvVars env_vars;
-  env_vars.xla_flags_env_var_present = getenv("XLA_FLAGS") != nullptr;
-  env_vars.tf_xla_flags_env_var_present = getenv("TF_XLA_FLAGS") != nullptr;
-  return env_vars;
-}
-
-}  // namespace
-
-XlaGpuJitIsLinkedIn::XlaGpuJitIsLinkedIn() { is_xla_gpu_jit_registered = true; }
-XlaCpuJitIsLinkedIn::XlaCpuJitIsLinkedIn() { is_xla_cpu_jit_registered = true; }
-
-Status CheckXlaJitOptimizerOptions(const SessionOptions* session_options) {
-  static XlaEnvVars env_vars = ComputeEnvVarHasXlaFlags();
-
-  if (is_xla_cpu_jit_registered || is_xla_gpu_jit_registered) {
-    return Status::OK();
-  }
-
-  if (env_vars.xla_flags_env_var_present) {
-    return errors::InvalidArgument(
-        "The XLA JIT is not linked in but the \"XLA_FLAGS\" environment "
-        "variable is set.  Please either link in XLA or remove \"XLA_FLAGS\" "
-        "from the environment.");
-  }
-
-  if (env_vars.tf_xla_flags_env_var_present) {
-    return errors::InvalidArgument(
-        "The XLA JIT is not linked in but the \"TF_XLA_FLAGS\" environment "
-        "variable is set.  Please either link in XLA or remove "
-        "\"TF_XLA_FLAGS\" from the environment.");
-  }
-
-  if (session_options) {
-    OptimizerOptions::GlobalJitLevel jit_level =
-        session_options->config.graph_options()
-            .optimizer_options()
-            .global_jit_level();
-
-    if (jit_level == OptimizerOptions::ON_1 ||
-        jit_level == OptimizerOptions::ON_2) {
-      return errors::InvalidArgument(
-          "The XLA JIT is enabled in the session options but XLA is not linked "
-          "in.  Plesae either link in XLA or disable the JIT in the session "
-          "options.");
-    }
-  }
-
-  return Status::OK();
-}
-}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/tf_xla_stub.h b/tensorflow/core/common_runtime/tf_xla_stub.h
deleted file mode 100644
index 723b2b5cd2..0000000000
--- a/tensorflow/core/common_runtime/tf_xla_stub.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_TF_XLA_STUB_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_TF_XLA_STUB_H_
-
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/public/session_options.h"
-
-namespace tensorflow {
-// Returns an error if the XLA JIT is enabled via `session_options` or if the
-// TF_XLA_FLAGS or XLA_FLAGS environment variables are set, but neither the
-// XLA CPU JIT nor the XLA GPU JIT are linked in.
-//
-// If `session_options` is null then only the environment variables are checked.
-Status CheckXlaJitOptimizerOptions(const SessionOptions* session_options);
-
-// The XLA CPU JIT creates a static instance of this class to notify
-// `CheckXlaJitOptimizerOptions` that the XLA CPU JIT is linked in.
-//
-// NB!  The constructor of this class (if run at all) needs to be ordered (via
-// happens before) before any call to `CheckXlaJitOptimizerOptions`.
-class XlaCpuJitIsLinkedIn {
- public:
-  XlaCpuJitIsLinkedIn();
-};
-
-// The XLA GPU JIT creates a static instance of this class to notify
-// `CheckXlaJitOptimizerOptions` that the XLA GPU JIT is linked in.
-//
-// NB!  The constructor of this class (if run at all) needs to be ordered (via
-// happens before) before any call to `CheckXlaJitOptimizerOptions`.
-class XlaGpuJitIsLinkedIn {
- public:
-  XlaGpuJitIsLinkedIn();
-};
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_TF_XLA_STUB_H_
-- 
GitLab


From 1aaa68d93c6b2f4151446eb211399b4330c96a09 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 18 Nov 2018 01:02:28 -0800
Subject: [PATCH 0481/1554] compat: Update forward compatibility horizon to
 2018-11-18

PiperOrigin-RevId: 221964754
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index b3f98a12fb..321a059d41 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 17)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 18)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 1fdd7c7408aa1cb37729e76a6e9fbfe8daa0b1f5 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Sun, 18 Nov 2018 20:00:30 -0800
Subject: [PATCH 0482/1554] Replace a few calls of Session `run` with
 `evaluate`

In order to support tests running in eager mode we need to avoid
unnecessary use of Sessions in tests. This moves to remove some
of the uses of the `run` function in favor of `evaluate`.

PiperOrigin-RevId: 222013881
---
 .../compiler/tests/categorical_op_test.py     |  10 +-
 tensorflow/compiler/tests/concat_ops_test.py  |   8 +-
 tensorflow/compiler/tests/eager_test.py       |   2 +-
 tensorflow/compiler/tests/function_test.py    |   6 +-
 tensorflow/compiler/tests/lstm_test.py        |   4 +-
 tensorflow/compiler/tests/placeholder_test.py |   2 +-
 tensorflow/compiler/tests/random_ops_test.py  |  14 +-
 .../compiler/tests/tensor_array_ops_test.py   |   2 +-
 .../compiler/tests/variable_ops_test.py       |  30 +-
 .../autograph/integration_tests/keras_test.py |   2 +-
 .../integration_tests/list_literals_test.py   |   2 +-
 .../speech_commands/input_data_test.py        |   2 +-
 .../speech_commands/label_wav_test.py         |   2 +-
 .../speech_commands/wav_to_features_test.py   |   2 +-
 .../autograph/converters/call_trees_test.py   |   2 +-
 .../python/autograph/converters/lists_test.py |   6 +-
 .../converters/side_effect_guards_test.py     |  20 +-
 .../autograph/converters/slices_test.py       |   2 +-
 tensorflow/python/autograph/impl/api_test.py  |  46 +-
 .../autograph/lang/special_functions_test.py  |  12 +-
 .../autograph/operators/control_flow_test.py  |  14 +-
 .../operators/data_structures_test.py         |  16 +-
 .../autograph/operators/logical_test.py       |  14 +-
 .../autograph/operators/py_builtins_test.py   |  28 +-
 .../python/autograph/operators/slices_test.py |   8 +-
 .../python/autograph/utils/misc_test.py       |   4 +-
 .../python/autograph/utils/py_func_test.py    |  18 +-
 .../autograph/utils/tensor_list_test.py       |   4 +-
 .../client/session_clusterspec_prop_test.py   |   6 +-
 tensorflow/python/client/timeline_test.py     |   4 +-
 tensorflow/python/client/virtual_gpu_test.py  |   2 +-
 .../kernel_tests/batch_dataset_op_test.py     |  54 +--
 .../bucket_by_sequence_length_test.py         |   2 +-
 .../kernel_tests/copy_to_device_test.py       |  84 ++--
 .../experimental/kernel_tests/counter_test.py |  12 +-
 .../dense_to_sparse_batch_test.py             |   8 +-
 .../directed_interleave_dataset_test.py       |   6 +-
 .../kernel_tests/enumerate_dataset_test.py    |   6 +-
 .../function_buffering_resource_test.py       |  70 +--
 .../kernel_tests/group_by_reducer_test.py     |   6 +-
 .../kernel_tests/group_by_window_test.py      |  52 +--
 .../kernel_tests/ignore_errors_test.py        |  16 +-
 .../kernel_tests/indexed_dataset_ops_test.py  |   6 +-
 .../make_batched_features_dataset_test.py     |   4 +-
 .../kernel_tests/make_csv_dataset_test.py     |   2 +-
 .../make_tf_record_dataset_test.py            |   6 +-
 .../kernel_tests/map_and_batch_test.py        |  38 +-
 .../kernel_tests/map_defun_op_test.py         |   2 +-
 .../kernel_tests/override_threadpool_test.py  |   2 +-
 .../kernel_tests/parallel_interleave_test.py  |   6 +-
 .../kernel_tests/prefetch_to_device_test.py   |  26 +-
 .../experimental/kernel_tests/scan_test.py    |   6 +-
 .../range_dataset_serialization_test.py       |  26 +-
 .../serialization_integration_test.py         |   4 +-
 .../kernel_tests/shuffle_and_repeat_test.py   |   2 +-
 .../experimental/kernel_tests/sleep_test.py   |   4 +-
 .../kernel_tests/sql_dataset_test.py          |  97 ++--
 .../kernel_tests/stats_dataset_ops_test.py    |  58 +--
 .../experimental/kernel_tests/unbatch_test.py |  12 +-
 .../experimental/kernel_tests/unique_test.py  |   4 +-
 .../kernel_tests/batch_dataset_op_test.py     |  26 +-
 .../kernel_tests/cache_dataset_op_test.py     |  18 +-
 .../concatenate_dataset_op_test.py            |   8 +-
 .../dataset_constructor_op_test.py            |  44 +-
 .../dataset_from_generator_op_test.py         |  78 ++--
 .../kernel_tests/filter_dataset_op_test.py    |  26 +-
 .../kernel_tests/flat_map_dataset_op_test.py  |  20 +-
 .../interleave_dataset_op_test.py             |   8 +-
 .../kernel_tests/iterator_ops_cluster_test.py |  12 +-
 .../data/kernel_tests/iterator_ops_test.py    |  44 +-
 .../list_files_dataset_op_test.py             |   2 +-
 .../data/kernel_tests/map_dataset_op_test.py  | 126 +++---
 .../multi_device_iterator_test.py             |  68 +--
 .../data/kernel_tests/optional_ops_test.py    |   4 +-
 .../kernel_tests/prefetch_dataset_op_test.py  |   2 +-
 .../kernel_tests/range_dataset_op_test.py     | 116 ++---
 .../kernel_tests/reader_dataset_ops_test.py   | 112 ++---
 .../kernel_tests/reduce_dataset_op_test.py    |  11 +-
 .../kernel_tests/sequence_dataset_op_test.py  |  20 +-
 .../kernel_tests/shuffle_dataset_op_test.py   |  12 +-
 .../kernel_tests/window_dataset_op_test.py    |  24 +-
 .../data/kernel_tests/zip_dataset_op_test.py  |   6 +-
 tensorflow/python/data/util/convert_test.py   |   8 +-
 .../python/debug/cli/analyzer_cli_test.py     |   2 +-
 .../lib/debug_graph_reconstruction_test.py    |  14 +-
 .../debug/lib/dist_session_debug_grpc_test.py |   8 +-
 .../debug/lib/session_debug_multi_gpu_test.py |   2 +-
 .../python/debug/lib/source_utils_test.py     |   4 +-
 .../distribute/distribute_coordinator_test.py |   6 +-
 .../python/distribute/input_ops_test.py       |   7 +-
 tensorflow/python/eager/def_function_test.py  |   8 +-
 .../python/eager/function_gradients_test.py   |   2 +-
 tensorflow/python/eager/function_test.py      |   4 +-
 .../feature_column/feature_column_test.py     |   8 +-
 .../feature_column/feature_column_v2_test.py  |  11 +-
 tensorflow/python/framework/function_test.py  |  50 +--
 .../python/framework/graph_util_test.py       |  10 +-
 tensorflow/python/framework/importer_test.py  |  12 +-
 .../python/framework/meta_graph_test.py       |  12 +-
 tensorflow/python/framework/ops_test.py       |   6 +-
 .../python/framework/smart_cond_test.py       |   8 +-
 .../python/framework/sparse_tensor_test.py    |   2 +-
 .../python/framework/tensor_util_test.py      |   2 +-
 .../python/grappler/constant_folding_test.py  |   2 +-
 .../python/grappler/layout_optimizer_test.py  |  48 +-
 .../python/grappler/memory_optimizer_test.py  |  12 +-
 tensorflow/python/keras/backend_test.py       |   2 +-
 .../python/keras/layers/recurrent_test.py     |   4 +-
 .../python/kernel_tests/accumulate_n_test.py  |   2 +-
 .../python/kernel_tests/array_ops_test.py     |  14 +-
 .../python/kernel_tests/basic_gpu_test.py     |   8 +-
 .../boosted_trees/quantile_ops_test.py        |   4 +-
 .../boosted_trees/stats_ops_test.py           |  30 +-
 .../python/kernel_tests/bucketize_op_test.py  |   6 +-
 .../candidate_sampler_ops_test.py             |   2 +-
 .../python/kernel_tests/cast_op_test.py       |   2 +-
 .../python/kernel_tests/concat_op_test.py     |   6 +-
 .../conditional_accumulator_test.py           |   8 +-
 .../kernel_tests/control_flow_ops_py_test.py  |  48 +-
 .../python/kernel_tests/conv_ops_3d_test.py   |  10 +-
 .../python/kernel_tests/conv_ops_test.py      |  12 +-
 .../python/kernel_tests/cwise_ops_test.py     |   4 +-
 .../kernel_tests/decode_jpeg_op_test.py       |   2 +-
 .../dense_update_ops_no_tsan_test.py          |   8 +-
 .../kernel_tests/depthwise_conv_op_test.py    |   6 +-
 .../distributions/categorical_test.py         |   4 +-
 .../kernel_tests/dynamic_partition_op_test.py |  28 +-
 .../python/kernel_tests/fifo_queue_test.py    |  84 ++--
 .../kernel_tests/functional_ops_test.py       |  46 +-
 .../kernel_tests/gradient_correctness_test.py |   8 +-
 .../python/kernel_tests/init_ops_test.py      |   8 +-
 .../python/kernel_tests/lookup_ops_test.py    |   2 +-
 tensorflow/python/kernel_tests/losses_test.py |   4 +-
 .../python/kernel_tests/map_stage_op_test.py  |  22 +-
 .../kernel_tests/matrix_inverse_op_test.py    |   2 +-
 .../kernel_tests/matrix_solve_op_test.py      |   2 +-
 .../matrix_square_root_op_test.py             |   2 +-
 .../python/kernel_tests/metrics_test.py       | 419 +++++++++---------
 .../neon_depthwise_conv_op_test.py            |   6 +-
 .../python/kernel_tests/norm_op_test.py       |   2 +-
 .../kernel_tests/nth_element_op_test.py       |   2 +-
 .../kernel_tests/padding_fifo_queue_test.py   |  88 ++--
 .../parse_single_example_op_test.py           |   2 +-
 .../python/kernel_tests/parsing_ops_test.py   |   4 +-
 .../kernel_tests/pooling_ops_3d_test.py       |   2 +-
 .../kernel_tests/priority_queue_test.py       |  20 +-
 .../python/kernel_tests/py_func_test.py       |  14 +-
 tensorflow/python/kernel_tests/qr_op_test.py  |   2 +-
 .../random/multinomial_op_big_test.py         |   6 +-
 .../random/multinomial_op_test.py             |  12 +-
 .../kernel_tests/random/random_gamma_test.py  |   2 +-
 .../kernel_tests/random/random_ops_test.py    |  12 +-
 .../random/random_poisson_test.py             |   2 +-
 .../random/random_shuffle_queue_test.py       |  66 +--
 .../python/kernel_tests/reader_ops_test.py    |   2 +-
 .../python/kernel_tests/record_input_test.py  |  16 +-
 .../python/kernel_tests/reduction_ops_test.py |  18 +-
 .../resource_variable_ops_test.py             |   2 +-
 .../kernel_tests/scatter_nd_ops_test.py       |  14 +-
 .../kernel_tests/self_adjoint_eig_op_test.py  |   2 +-
 .../python/kernel_tests/session_ops_test.py   |  22 +-
 tensorflow/python/kernel_tests/sets_test.py   |   2 +-
 .../python/kernel_tests/shape_ops_test.py     |   4 +-
 .../signal/reconstruction_ops_test.py         |   8 +-
 .../python/kernel_tests/sparse_add_op_test.py |   8 +-
 .../kernel_tests/sparse_concat_op_test.py     |  14 +-
 .../sparse_conditional_accumulator_test.py    |  24 +-
 .../kernel_tests/sparse_cross_op_test.py      |  34 +-
 .../python/kernel_tests/sparse_ops_test.py    |  34 +-
 .../kernel_tests/sparse_reorder_op_test.py    |   4 +-
 .../kernel_tests/sparse_reshape_op_test.py    |   4 +-
 .../sparse_serialization_ops_test.py          |   2 +-
 .../sparse_tensors_map_ops_test.py            |  11 +-
 .../python/kernel_tests/stage_op_test.py      |  18 +-
 .../kernel_tests/string_length_op_test.py     |   2 +-
 .../kernel_tests/string_split_op_test.py      |  28 +-
 .../kernel_tests/string_strip_op_test.py      |   6 +-
 .../kernel_tests/summary_v1_audio_op_test.py  |   2 +-
 .../kernel_tests/summary_v1_image_op_test.py  |   4 +-
 .../kernel_tests/summary_v1_ops_test.py       |   6 +-
 .../kernel_tests/summary_v1_tensor_op_test.py |  12 +-
 tensorflow/python/kernel_tests/svd_op_test.py |   4 +-
 .../python/kernel_tests/template_test.py      |   8 +-
 .../kernel_tests/tensor_array_ops_test.py     |   6 +-
 .../kernel_tests/unicode_transcode_op_test.py |  46 +-
 .../kernel_tests/variable_scope_test.py       |  16 +-
 .../python/kernel_tests/variables_test.py     |  36 +-
 .../python/kernel_tests/while_v2_test.py      |  56 +--
 .../python/kernel_tests/xent_op_test.py       |   4 +-
 .../python/layers/convolutional_test.py       |  16 +-
 tensorflow/python/layers/core_test.py         |   2 +-
 .../python/layers/normalization_test.py       |  74 ++--
 .../python/ops/control_flow_ops_test.py       |  12 +-
 tensorflow/python/ops/gradients_test.py       |  28 +-
 tensorflow/python/ops/image_grad_test.py      |   8 +-
 tensorflow/python/ops/image_ops_test.py       |  30 +-
 tensorflow/python/ops/init_ops_test.py        |   4 +-
 tensorflow/python/ops/math_ops_test.py        |   4 +-
 .../python/ops/nn_fused_batchnorm_test.py     |   2 +-
 tensorflow/python/ops/nn_test.py              |  28 +-
 .../python/ops/parallel_for/gradients_test.py |   2 +-
 .../python/ops/quantized_conv_ops_test.py     |   2 +-
 tensorflow/python/ops/quantized_ops_test.py   |   4 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |   2 +-
 .../python/profiler/model_analyzer_test.py    |  30 +-
 .../python/profiler/profile_context_test.py   |   8 +-
 tensorflow/python/saved_model/loader_test.py  |   4 +-
 .../python/saved_model/saved_model_test.py    |  42 +-
 .../python/saved_model/simple_save_test.py    |   2 +-
 tensorflow/python/tools/strip_unused_test.py  |   4 +-
 .../training/basic_session_run_hooks_test.py  |  32 +-
 .../python/training/checkpoint_ops_test.py    |   2 +-
 tensorflow/python/training/input_test.py      |  54 +--
 .../python/training/monitored_session_test.py |  16 +-
 .../python/training/moving_averages_test.py   |  12 +-
 tensorflow/python/training/saver_test.py      |  62 +--
 .../training/server_lib_sparse_job_test.py    |   2 +-
 tensorflow/python/training/supervisor_test.py |  10 +-
 .../training/warm_starting_util_test.py       |  76 ++--
 219 files changed, 2039 insertions(+), 2015 deletions(-)

diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py
index a57d1dc81e..532e2b5748 100644
--- a/tensorflow/compiler/tests/categorical_op_test.py
+++ b/tensorflow/compiler/tests/categorical_op_test.py
@@ -60,7 +60,7 @@ class CategoricalTest(xla_test.XLATestCase):
       random_seed.set_random_seed(1618)
       op = random_ops.multinomial(logits, num_samples,
                                   output_dtype=dtypes.int32)
-      d = sess.run(op)
+      d = self.evaluate(op)
 
     batch_size, num_classes = logits.shape
     freqs_mat = []
@@ -85,9 +85,9 @@ class CategoricalTest(xla_test.XLATestCase):
 
       # The random-number generator, if working correctly, should produce the
       # same output multiple times with low probability.
-      y = sess.run(x)
-      z = sess.run(x)
-      w = sess.run(x)
+      y = self.evaluate(x)
+      z = self.evaluate(x)
+      w = self.evaluate(x)
 
       # We use exact equality here. If the random-number generator is producing
       # deterministic output, all three outputs will be bitwise identical.
@@ -112,7 +112,7 @@ class CategoricalTest(xla_test.XLATestCase):
             x = random_ops.multinomial(
                 array_ops.ones(shape=[1, 20], dtype=dtype), 1000,
                 output_dtype=output_dtype)
-          y = sess.run(x)
+          y = self.evaluate(x)
           self.assertTrue((y >= 0).sum() == 1000)
           self.assertTrue((y < 20).sum() == 1000)
 
diff --git a/tensorflow/compiler/tests/concat_ops_test.py b/tensorflow/compiler/tests/concat_ops_test.py
index 30fbe6f701..deb9ac186e 100644
--- a/tensorflow/compiler/tests/concat_ops_test.py
+++ b/tensorflow/compiler/tests/concat_ops_test.py
@@ -337,7 +337,7 @@ class ConcatOffsetTest(xla_test.XLATestCase):
         s1 = constant_op.constant([2, 7, 5], dtypes.int32)
         s2 = constant_op.constant([2, 20, 5], dtypes.int32)
         off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-        ans = sess.run(off)
+        ans = self.evaluate(off)
         self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
 
@@ -350,7 +350,7 @@ class PackTest(xla_test.XLATestCase):
         s1 = constant_op.constant([2, 7, 5], dtypes.int32)
         s2 = constant_op.constant([2, 20, 5], dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = sess.run(packed)
+        ans = self.evaluate(packed)
         self.assertAllEqual(ans, [[2, 3, 5], [2, 7, 5], [2, 20, 5]])
 
   def testScalars(self):
@@ -360,7 +360,7 @@ class PackTest(xla_test.XLATestCase):
         s1 = constant_op.constant(3, dtypes.int32)
         s2 = constant_op.constant(5, dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = sess.run(packed)
+        ans = self.evaluate(packed)
         self.assertAllEqual(ans, [2, 3, 5])
 
   def testEmpty(self):
@@ -370,7 +370,7 @@ class PackTest(xla_test.XLATestCase):
         s1 = constant_op.constant([[]], dtypes.int32)
         s2 = constant_op.constant([[]], dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = sess.run(packed)
+        ans = self.evaluate(packed)
         self.assertAllEqual(ans, [[[]], [[]], [[]]])
 
 
diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index 63cee550fd..76706ad40a 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -106,7 +106,7 @@ class EagerTest(xla_test.XLATestCase):
         three = constant_op.constant(3)
         five = constant_op.constant(5)
         product = three * five
-        self.assertAllEqual(15, sess.run(product))
+        self.assertAllEqual(15, self.evaluate(product))
 
   def testDegenerateSlices(self):
     with self.test_scope():
diff --git a/tensorflow/compiler/tests/function_test.py b/tensorflow/compiler/tests/function_test.py
index b1891b918c..dd9b7f30ef 100644
--- a/tensorflow/compiler/tests/function_test.py
+++ b/tensorflow/compiler/tests/function_test.py
@@ -50,7 +50,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_f = Foo(a, b)
-      result = sess.run(call_f)
+      result = self.evaluate(call_f)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testNestedFunctions(self):
@@ -76,7 +76,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_g = Foo(a, b)
-      result = sess.run(call_g)
+      result = self.evaluate(call_g)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testFunctionMultipleRetvals(self):
@@ -100,7 +100,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_f = Foo(a, b)
-      result = sess.run(call_f)
+      result = self.evaluate(call_f)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testCompileTimeConstantsInDefun(self):
diff --git a/tensorflow/compiler/tests/lstm_test.py b/tensorflow/compiler/tests/lstm_test.py
index 265c0b6d14..fd02a50aff 100644
--- a/tensorflow/compiler/tests/lstm_test.py
+++ b/tensorflow/compiler/tests/lstm_test.py
@@ -88,7 +88,7 @@ class LSTMTest(test.TestCase):
                  (basename, m_prev_scalar, c_prev_scalar, pad_scalar))
 
       # Initialize variables and run the unrolled LSTM step.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       return sess.run([m, c])
 
   def testLSTMCell(self):
@@ -173,7 +173,7 @@ class LSTMTest(test.TestCase):
                  (basename, m_init_scalar, c_init_scalar, pad_scalar))
 
       # Initialize variables and run the unrolled LSTM layer.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       return sess.run(out_seq)
 
   def testLSTMLayer(self):
diff --git a/tensorflow/compiler/tests/placeholder_test.py b/tensorflow/compiler/tests/placeholder_test.py
index 77bb839409..9671ae0ae9 100644
--- a/tensorflow/compiler/tests/placeholder_test.py
+++ b/tensorflow/compiler/tests/placeholder_test.py
@@ -33,7 +33,7 @@ class PlaceholderTest(xla_test.XLATestCase):
       ph = array_ops.placeholder_with_default(v, shape=[])
       out = ph * 2
       sess.run(variables.variables_initializer([v]))
-      self.assertEqual(8.0, sess.run(out))
+      self.assertEqual(8.0, self.evaluate(out))
 
   def test_placeholder_with_default_fed(self):
     with self.cached_session() as sess, self.test_scope():
diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py
index 36ef6ed5fe..1e91390945 100644
--- a/tensorflow/compiler/tests/random_ops_test.py
+++ b/tensorflow/compiler/tests/random_ops_test.py
@@ -46,9 +46,9 @@ class RandomOpsTest(xla_test.XLATestCase):
 
       # The random-number generator, if working correctly, should produce the
       # same output multiple times with low probability.
-      y = sess.run(x)
-      z = sess.run(x)
-      w = sess.run(x)
+      y = self.evaluate(x)
+      z = self.evaluate(x)
+      w = self.evaluate(x)
 
       # We use exact equality here. If the random-number generator is producing
       # deterministic output, all three outputs will be bitwise identical.
@@ -83,7 +83,7 @@ class RandomOpsTest(xla_test.XLATestCase):
         with self.test_scope():
           x = random_ops.random_uniform(
               shape=[1000], dtype=dtype, minval=-2, maxval=33)
-        y = sess.run(x)
+        y = self.evaluate(x)
         self.assertTrue((y >= -2).sum() == 1000)
         self.assertTrue((y < 33).sum() == 1000)
 
@@ -102,7 +102,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.cached_session() as sess:
         with self.test_scope():
           x = random_ops.truncated_normal(shape=[count], dtype=dtype)
-        y = sess.run(x)
+        y = self.evaluate(x)
 
         def normal_cdf(x):
           return .5 * math.erfc(-x / math.sqrt(2))
@@ -148,7 +148,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.test_scope():
         x = math_ops.range(1 << 16)
         shuffle = random_ops.random_shuffle(x)
-      result = sess.run(shuffle)
+      result = self.evaluate(shuffle)
       expected = range(1 << 16)
       # Compare sets to avoid randomness behavior changes but make sure still
       # have all the values.
@@ -159,7 +159,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.test_scope():
         x = array_ops.diag(math_ops.range(20))
         shuffle = random_ops.random_shuffle(x)
-      result = sess.run(shuffle)
+      result = self.evaluate(shuffle)
       expected = np.diag(range(20)).flatten()
       # Compare sets to avoid randomness behavior changes but make sure still
       # have all the values.
diff --git a/tensorflow/compiler/tests/tensor_array_ops_test.py b/tensorflow/compiler/tests/tensor_array_ops_test.py
index c8208adb58..d7e26d79c4 100644
--- a/tensorflow/compiler/tests/tensor_array_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_array_ops_test.py
@@ -505,7 +505,7 @@ class TensorArrayTest(xla_test.XLATestCase):
                 [-0.5, 1.5],  # read(0) gradient
                 [20.0, 30.0, 40.0, 50.0],  # concat gradient
             ])
-      grad_vals = sess.run(grad_r)  # 2 + 2 entries
+      grad_vals = self.evaluate(grad_r)  # 2 + 2 entries
 
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py
index 77cdeac816..e776c8a951 100644
--- a/tensorflow/compiler/tests/variable_ops_test.py
+++ b/tensorflow/compiler/tests/variable_ops_test.py
@@ -229,7 +229,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_add(
               handle, [0], constant_op.constant([[2]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(sess.run(read), [[3], [7]])
+      self.assertAllEqual(self.evaluate(read), [[3], [7]])
 
   def testScatterSub(self):
     with self.test_session() as sess, self.test_scope():
@@ -242,7 +242,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_sub(
               handle, [1], constant_op.constant([[2]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(sess.run(read), [[4], [-1]])
+      self.assertAllEqual(self.evaluate(read), [[4], [-1]])
 
   def testScatterMul(self):
     with self.test_session() as sess, self.test_scope():
@@ -255,7 +255,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_mul(
               handle, [0], constant_op.constant([[5]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[5]])
+      self.assertEqual(self.evaluate(read), [[5]])
 
   def testScatterDiv(self):
     with self.test_session() as sess, self.test_scope():
@@ -268,7 +268,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_div(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(sess.run(read), [[2]])
+      self.assertAllEqual(self.evaluate(read), [[2]])
 
   def testScatterMin(self):
     with self.test_session() as sess, self.test_scope():
@@ -281,7 +281,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_min(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterMax(self):
     with self.test_session() as sess, self.test_scope():
@@ -294,7 +294,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_max(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[6]])
+      self.assertEqual(self.evaluate(read), [[6]])
 
   def testScatterUpdate(self):
     with self.test_session() as sess, self.test_scope():
@@ -307,7 +307,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_update(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterAddScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -320,7 +320,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_add(
               handle, [0], constant_op.constant(2, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterSubScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -333,7 +333,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_sub(
               handle, [0], constant_op.constant(2, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[-1]])
+      self.assertEqual(self.evaluate(read), [[-1]])
 
   def testScatterMulScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -346,7 +346,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_mul(
               handle, [0], constant_op.constant(5, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[5]])
+      self.assertEqual(self.evaluate(read), [[5]])
 
   def testScatterDivScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -359,7 +359,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_div(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[2]])
+      self.assertEqual(self.evaluate(read), [[2]])
 
   def testScatterMinScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -372,7 +372,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_min(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterMaxScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -385,7 +385,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_max(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[6]])
+      self.assertEqual(self.evaluate(read), [[6]])
 
   def testScatterNdAddOps(self):
     with self.test_session() as sess, self.test_scope():
@@ -400,7 +400,7 @@ class VariableOpsTest(xla_test.XLATestCase):
       sess.run(gen_state_ops.resource_scatter_nd_add(handle, indices, updates))
       read = resource_variable_ops.read_variable_op(
           handle, dtype=dtypes.float32)
-      self.assertAllClose(expected, sess.run(read))
+      self.assertAllClose(expected, self.evaluate(read))
 
   def testScatterNdUpdateAddOps(self):
     with self.test_session() as sess, self.test_scope():
@@ -416,7 +416,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           gen_state_ops.resource_scatter_nd_update(handle, indices, updates))
       read = resource_variable_ops.read_variable_op(
           handle, dtype=dtypes.float32)
-      self.assertAllClose(expected, sess.run(read))
+      self.assertAllClose(expected, self.evaluate(read))
 
 
 class StridedSliceAssignChecker(object):
diff --git a/tensorflow/examples/autograph/integration_tests/keras_test.py b/tensorflow/examples/autograph/integration_tests/keras_test.py
index dca7c07b47..9828ac34dc 100644
--- a/tensorflow/examples/autograph/integration_tests/keras_test.py
+++ b/tensorflow/examples/autograph/integration_tests/keras_test.py
@@ -96,7 +96,7 @@ class KerasTest(tf.test.TestCase):
           sess.run(init)
           sample_input = tf.random_uniform((1, 10, 10, 1))
           output = model(sample_input)  # pylint: disable=not-callable
-          self.assertEqual(sess.run(output).shape, (1, 3))
+          self.assertEqual(self.evaluate(output).shape, (1, 3))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/examples/autograph/integration_tests/list_literals_test.py b/tensorflow/examples/autograph/integration_tests/list_literals_test.py
index 917f5ff9d8..e85d4abcfc 100644
--- a/tensorflow/examples/autograph/integration_tests/list_literals_test.py
+++ b/tensorflow/examples/autograph/integration_tests/list_literals_test.py
@@ -34,7 +34,7 @@ class ListLiteralsTest(tf.test.TestCase):
     result = converted()
 
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(result), [1, 2, 3])
+      self.assertAllEqual(self.evaluate(result), [1, 2, 3])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/examples/speech_commands/input_data_test.py b/tensorflow/examples/speech_commands/input_data_test.py
index b766ba6de0..33b58b9d09 100644
--- a/tensorflow/examples/speech_commands/input_data_test.py
+++ b/tensorflow/examples/speech_commands/input_data_test.py
@@ -35,7 +35,7 @@ class InputDataTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([32000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = sess.run(wav_encoder)
+      wav_data = self.evaluate(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/examples/speech_commands/label_wav_test.py b/tensorflow/examples/speech_commands/label_wav_test.py
index f0af2a4798..77a88f98e1 100644
--- a/tensorflow/examples/speech_commands/label_wav_test.py
+++ b/tensorflow/examples/speech_commands/label_wav_test.py
@@ -33,7 +33,7 @@ class LabelWavTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([1000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = sess.run(wav_encoder)
+      wav_data = self.evaluate(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/examples/speech_commands/wav_to_features_test.py b/tensorflow/examples/speech_commands/wav_to_features_test.py
index 87f2987693..cb8ea912fa 100644
--- a/tensorflow/examples/speech_commands/wav_to_features_test.py
+++ b/tensorflow/examples/speech_commands/wav_to_features_test.py
@@ -33,7 +33,7 @@ class WavToFeaturesTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([32000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = sess.run(wav_encoder)
+      wav_data = self.evaluate(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 916c736fb4..892f90e350 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -113,7 +113,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.compiled(node, ns) as result:
       with self.cached_session() as sess:
         result_tensor = result.test_fn(constant_op.constant(1))
-        self.assertEquals(sess.run(result_tensor), 3)
+        self.assertEquals(self.evaluate(result_tensor), 3)
 
   def test_call_to_decorated_function(self):
 
diff --git a/tensorflow/python/autograph/converters/lists_test.py b/tensorflow/python/autograph/converters/lists_test.py
index f6da845fcc..8c8135acef 100644
--- a/tensorflow/python/autograph/converters/lists_test.py
+++ b/tensorflow/python/autograph/converters/lists_test.py
@@ -68,7 +68,7 @@ class ListTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(sess.run(r), [1, 2, 3])
+        self.assertAllEqual(self.evaluate(r), [1, 2, 3])
 
   def test_list_pop(self):
 
@@ -91,8 +91,8 @@ class ListTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         ts, tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(sess.run(r), [1, 2])
-        self.assertAllEqual(sess.run(ts), 3)
+        self.assertAllEqual(self.evaluate(r), [1, 2])
+        self.assertAllEqual(self.evaluate(ts), 3)
 
   def test_double_list_pop(self):
 
diff --git a/tensorflow/python/autograph/converters/side_effect_guards_test.py b/tensorflow/python/autograph/converters/side_effect_guards_test.py
index cef3199169..e72b5eac32 100644
--- a/tensorflow/python/autograph/converters/side_effect_guards_test.py
+++ b/tensorflow/python/autograph/converters/side_effect_guards_test.py
@@ -48,12 +48,12 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
+        self.evaluate(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Add support for this use case.
         # Right now the variable `a` is not conditioned on the `assign` because
         # there's no way to add control dependencies to a variable object.
-        self.assertEqual(2, sess.run(v))
+        self.assertEqual(2, self.evaluate(v))
 
   def test_side_effect_on_used_variable(self):
 
@@ -69,11 +69,11 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
+        self.evaluate(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
         # Right now it's 3 or 4 based on whether the read is synchronized.
-        self.assertEqual(3, sess.run(v))
+        self.assertEqual(3, self.evaluate(v))
 
   def test_side_effect_on_tensor(self):
 
@@ -109,10 +109,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign_add) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
+        self.evaluate(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(4, sess.run(v))
+        self.assertEqual(4, self.evaluate(v))
 
   def test_multiline_nested_block(self):
 
@@ -130,10 +130,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign, ops.name_scope) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
+        self.evaluate(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(3, sess.run(v))
+        self.assertEqual(3, self.evaluate(v))
 
   def test_multiline_block_unsafe(self):
 
@@ -153,10 +153,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
                        state_ops.assign_add) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
+        self.evaluate(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(4, sess.run(v))
+        self.assertEqual(4, self.evaluate(v))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py
index e190a7cfe8..bd049afdfc 100644
--- a/tensorflow/python/autograph/converters/slices_test.py
+++ b/tensorflow/python/autograph/converters/slices_test.py
@@ -49,7 +49,7 @@ class SliceTest(converter_testing.TestCase):
         tl = list_ops.tensor_list_from_tensor(
             [1, 2], element_shape=constant_op.constant([], dtype=dtypes.int32))
         y = result.test_fn(tl)
-        self.assertEqual(2, sess.run(y))
+        self.assertEqual(2, self.evaluate(y))
 
   def test_index_access_multiple_definitions(self):
 
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index ef577568c4..44cb99d657 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -63,7 +63,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_does_not_recurse(self):
 
@@ -83,7 +83,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_calls_unconverted_graph(self):
 
@@ -104,7 +104,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_calls_unconverted_py_func(self):
 
@@ -130,7 +130,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_calls_decorated(self):
 
@@ -153,7 +153,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_preserves_argspec(self):
 
@@ -192,7 +192,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_converted_call_builtin(self):
     x = api.converted_call(range, None, converter.ConversionOptions(), 3)
@@ -208,7 +208,7 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       x = api.converted_call(test_fn, None, converter.ConversionOptions(),
                              constant_op.constant(-1))
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_explicit_owner(self):
     # TODO(mdan): Implement.
@@ -234,7 +234,7 @@ class ApiTest(test.TestCase):
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(tc.test_method, None,
                              converter.ConversionOptions(), tc)
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_by_class(self):
 
@@ -252,7 +252,7 @@ class ApiTest(test.TestCase):
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(TestClass.test_method, None,
                              converter.ConversionOptions(), tc)
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_callable_object(self):
 
@@ -269,7 +269,7 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(tc, None, converter.ConversionOptions())
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_constructor(self):
 
@@ -288,7 +288,7 @@ class ApiTest(test.TestCase):
                               constant_op.constant(-1))
       # tc is now a converted object.
       x = tc.test_method()
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_already_converted(self):
 
@@ -298,12 +298,12 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       x = api.converted_call(f, None, converter.ConversionOptions(),
                              constant_op.constant(0))
-      self.assertTrue(sess.run(x))
+      self.assertTrue(self.evaluate(x))
 
       converted_f = api.to_graph(f)
       x = api.converted_call(converted_f, None, converter.ConversionOptions(),
                              constant_op.constant(0))
-      self.assertTrue(sess.run(x))
+      self.assertTrue(self.evaluate(x))
 
   def test_converted_call_no_user_code(self):
 
@@ -334,8 +334,8 @@ class ApiTest(test.TestCase):
                            constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_extra_self(self):
 
@@ -349,8 +349,8 @@ class ApiTest(test.TestCase):
                            model, constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_via_owner(self):
 
@@ -364,8 +364,8 @@ class ApiTest(test.TestCase):
                            constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_lambda(self):
 
@@ -376,8 +376,8 @@ class ApiTest(test.TestCase):
     x = api.converted_call(l, None, opts, constant_op.constant(0))
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual(True, sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual(True, self.evaluate(x))
 
   def test_to_graph_basic(self):
 
@@ -390,7 +390,7 @@ class ApiTest(test.TestCase):
 
     with self.cached_session() as sess:
       x = compiled_fn(constant_op.constant([4, 8]), 4)
-      self.assertListEqual([1, 2], sess.run(x).tolist())
+      self.assertListEqual([1, 2], self.evaluate(x).tolist())
 
   def test_to_graph_with_defaults(self):
 
@@ -405,7 +405,7 @@ class ApiTest(test.TestCase):
 
     with self.cached_session() as sess:
       x = compiled_fn(constant_op.constant([4, 8]))
-      self.assertListEqual([1, 2], sess.run(x).tolist())
+      self.assertListEqual([1, 2], self.evaluate(x).tolist())
 
   def test_to_code_basic(self):
 
diff --git a/tensorflow/python/autograph/lang/special_functions_test.py b/tensorflow/python/autograph/lang/special_functions_test.py
index 123ee65b32..8d40f4036c 100644
--- a/tensorflow/python/autograph/lang/special_functions_test.py
+++ b/tensorflow/python/autograph/lang/special_functions_test.py
@@ -36,7 +36,7 @@ class SpecialFunctionsTest(test.TestCase):
     python_one = special_functions.match_staging_level(1, 1)
     with self.cached_session() as sess:
       self.assertTrue(tensor_util.is_tensor(tensor_one))
-      self.assertAllEqual(sess.run(tensor_one), 1)
+      self.assertAllEqual(self.evaluate(tensor_one), 1)
       self.assertEqual(python_one, 1)
 
   def test_tensor_list_empty_list(self):
@@ -45,21 +45,21 @@ class SpecialFunctionsTest(test.TestCase):
                                       element_shape=())
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [])
+      self.assertAllEqual(self.evaluate(sl), [])
 
     l = special_functions.tensor_list((),
                                       element_dtype=dtypes.int32,
                                       element_shape=())
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [])
+      self.assertAllEqual(self.evaluate(sl), [])
 
   def test_tensor_list_tensor(self):
     l = special_functions.tensor_list(
         constant_op.constant([], dtype=dtypes.int32))
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [])
+      self.assertAllEqual(self.evaluate(sl), [])
 
   def test_tensor_list_unsupported_initializer(self):
     with self.assertRaisesRegexp(ValueError, 'unknown type'):
@@ -76,7 +76,7 @@ class SpecialFunctionsTest(test.TestCase):
     l = special_functions.tensor_list(elements)
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
+      self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]])
 
   def test_tensor_list_array_from_elements(self):
     elements = [constant_op.constant([1, 2]), constant_op.constant([3, 4])]
@@ -84,7 +84,7 @@ class SpecialFunctionsTest(test.TestCase):
     l = special_functions.tensor_list(elements, use_tensor_array=True)
     sl = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
+      self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]])
 
   def test_stack(self):
     self.assertEqual(special_functions.stack(1, strict=False), 1)
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 2dea18dc5f..05b5660941 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -35,7 +35,7 @@ class ForLoopTest(test.TestCase):
         body=lambda i, s: (s + i,),
         init_state=(0,))
     with self.cached_session() as sess:
-      self.assertEqual((10,), sess.run(s))
+      self.assertEqual((10,), self.evaluate(s))
 
   def test_python(self):
     s = control_flow.for_stmt(
@@ -53,7 +53,7 @@ class ForLoopTest(test.TestCase):
         body=lambda i, s: (s + i,),
         init_state=(0,))
     with self.cached_session() as sess:
-      self.assertEqual((10,), sess.run(s))
+      self.assertEqual((10,), self.evaluate(s))
 
 
 class WhileLoopTest(test.TestCase):
@@ -66,7 +66,7 @@ class WhileLoopTest(test.TestCase):
         init_state=(0, 0),
         extra_deps=(n,))
     with self.cached_session() as sess:
-      self.assertEqual((5, 10), sess.run(results))
+      self.assertEqual((5, 10), self.evaluate(results))
 
   def test_python(self):
     n = 5
@@ -90,9 +90,9 @@ class IfStmtTest(test.TestCase):
   def test_tensor(self):
     with self.cached_session() as sess:
       t = self.single_return_if_stmt(constant_op.constant(True))
-      self.assertEqual(1, sess.run(t))
+      self.assertEqual(1, self.evaluate(t))
       t = self.single_return_if_stmt(constant_op.constant(False))
-      self.assertEqual(-1, sess.run(t))
+      self.assertEqual(-1, self.evaluate(t))
 
   def test_python(self):
     self.assertEqual(1, self.single_return_if_stmt(True))
@@ -101,9 +101,9 @@ class IfStmtTest(test.TestCase):
   def test_tensor_multiple_returns(self):
     with self.cached_session() as sess:
       t = self.multi_return_if_stmt(constant_op.constant(True))
-      self.assertAllEqual([1, 2], sess.run(t))
+      self.assertAllEqual([1, 2], self.evaluate(t))
       t = self.multi_return_if_stmt(constant_op.constant(False))
-      self.assertAllEqual([-1, -2], sess.run(t))
+      self.assertAllEqual([-1, -2], self.evaluate(t))
 
   def test_python_multiple_returns(self):
     self.assertEqual((1, 2), self.multi_return_if_stmt(True))
diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py
index 72476ccb6b..dc50edb4c9 100644
--- a/tensorflow/python/autograph/operators/data_structures_test.py
+++ b/tensorflow/python/autograph/operators/data_structures_test.py
@@ -43,7 +43,7 @@ class ListTest(test.TestCase):
     l = data_structures.tf_tensor_list_new([3, 4, 5])
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4, 5])
+      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
   def test_tf_tensor_list_new_empty(self):
     l = data_structures.tf_tensor_list_new([],
@@ -51,13 +51,13 @@ class ListTest(test.TestCase):
                                            element_shape=())
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [])
+      self.assertAllEqual(self.evaluate(t), [])
 
   def test_tf_tensor_list_new_from_tensor(self):
     l = data_structures.tf_tensor_list_new(constant_op.constant([3, 4, 5]))
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4, 5])
+      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
   def test_tf_tensor_list_new_illegal_input(self):
     with self.assertRaises(ValueError):
@@ -77,7 +77,7 @@ class ListTest(test.TestCase):
     l = data_structures.tf_tensor_array_new([3, 4, 5])
     t = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4, 5])
+      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
   def test_tf_tensor_array_new_illegal_input(self):
     with self.assertRaises(ValueError):
@@ -102,7 +102,7 @@ class ListTest(test.TestCase):
 
     t = list_ops.tensor_list_stack(l, element_dtype=x.dtype)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [[1, 2, 3]])
+      self.assertAllEqual(self.evaluate(t), [[1, 2, 3]])
 
   def test_append_tensorarray(self):
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
@@ -131,10 +131,10 @@ class ListTest(test.TestCase):
 
     with self.cached_session() as sess:
       l, x = data_structures.list_pop(l, None, opts)
-      self.assertAllEqual(sess.run(x), [3, 4])
+      self.assertAllEqual(self.evaluate(x), [3, 4])
 
       t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
-      self.assertAllEqual(sess.run(t), [[1, 2]])
+      self.assertAllEqual(self.evaluate(t), [[1, 2]])
 
   def test_pop_python(self):
     l = [1, 2, 3]
@@ -152,7 +152,7 @@ class ListTest(test.TestCase):
 
     with self.cached_session() as sess:
       t = data_structures.list_stack(l, opts)
-      self.assertAllEqual(sess.run(t), sess.run(initial_list))
+      self.assertAllEqual(sess.run(t), self.evaluate(initial_list))
 
   def test_stack_tensor_list_empty(self):
     l = list_ops.empty_tensor_list(
diff --git a/tensorflow/python/autograph/operators/logical_test.py b/tensorflow/python/autograph/operators/logical_test.py
index d6649f7b2b..ebf6458f01 100644
--- a/tensorflow/python/autograph/operators/logical_test.py
+++ b/tensorflow/python/autograph/operators/logical_test.py
@@ -45,11 +45,11 @@ class LogicalOperatorsTest(test.TestCase):
   def test_and_tf(self):
     with self.cached_session() as sess:
       t = logical.and_(self._tf_true, self._tf_true)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.and_(self._tf_true, lambda: True)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.and_(self._tf_false, lambda: True)
-      self.assertEqual(sess.run(t), False)
+      self.assertEqual(self.evaluate(t), False)
       # TODO(mdan): Add a test for ops with side effects.
 
   def test_or_python(self):
@@ -63,11 +63,11 @@ class LogicalOperatorsTest(test.TestCase):
   def test_or_tf(self):
     with self.cached_session() as sess:
       t = logical.or_(self._tf_false, self._tf_true)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.or_(self._tf_false, lambda: True)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.or_(self._tf_true, lambda: True)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       # TODO(mdan): Add a test for ops with side effects.
 
   def test_not_python(self):
@@ -78,7 +78,7 @@ class LogicalOperatorsTest(test.TestCase):
   def test_not_tf(self):
     with self.cached_session() as sess:
       t = logical.not_(self._tf_false())
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/operators/py_builtins_test.py b/tensorflow/python/autograph/operators/py_builtins_test.py
index 443e30a475..4d9eec77c3 100644
--- a/tensorflow/python/autograph/operators/py_builtins_test.py
+++ b/tensorflow/python/autograph/operators/py_builtins_test.py
@@ -38,29 +38,29 @@ class PyBuiltinsTest(test.TestCase):
     self.assertEqual(py_builtins.abs_(-1), 1)
     with self.cached_session() as sess:
       t = py_builtins.abs_(constant_op.constant(-1))
-      self.assertEqual(sess.run(t), 1)
+      self.assertEqual(self.evaluate(t), 1)
       t = py_builtins.abs_(constant_op.constant([-1, 2, -3]))
-      self.assertAllEqual(sess.run(t), [1, 2, 3])
+      self.assertAllEqual(self.evaluate(t), [1, 2, 3])
 
   def test_float(self):
     self.assertEqual(py_builtins.float_(10), 10.0)
     self.assertEqual(py_builtins.float_('10.0'), 10.0)
     with self.cached_session() as sess:
       t = py_builtins.float_(constant_op.constant(1, dtype=dtypes.int64))
-      self.assertEqual(sess.run(t), 1.0)
+      self.assertEqual(self.evaluate(t), 1.0)
       st = py_builtins.float_(constant_op.constant('1.0'))
-      self.assertEqual(sess.run(st), 1.0)
+      self.assertEqual(self.evaluate(st), 1.0)
 
   def test_int(self):
     self.assertEqual(py_builtins.int_(10.0), 10)
     self.assertEqual(py_builtins.int_('11', 2), 3)
     with self.cached_session() as sess:
       t = py_builtins.int_(constant_op.constant(1, dtype=dtypes.float64))
-      self.assertEqual(sess.run(t), 1)
+      self.assertEqual(self.evaluate(t), 1)
       st = py_builtins.int_(constant_op.constant('1'))
-      self.assertEqual(sess.run(st), 1)
+      self.assertEqual(self.evaluate(st), 1)
       st = py_builtins.int_(constant_op.constant('1'), 10)
-      self.assertEqual(sess.run(st), 1)
+      self.assertEqual(self.evaluate(st), 1)
 
   def test_int_unsupported_base(self):
     t = constant_op.constant(1, dtype=dtypes.float64)
@@ -73,9 +73,9 @@ class PyBuiltinsTest(test.TestCase):
       t = py_builtins.len_(constant_op.constant([[1], [2], [3]]))
       self.assertEqual(t, 3)
       ta = py_builtins.len_(tensor_array_ops.TensorArray(dtypes.int32, size=5))
-      self.assertEqual(sess.run(ta), 5)
+      self.assertEqual(self.evaluate(ta), 5)
       tl = py_builtins.len_(data_structures.tf_tensor_list_new([3, 4, 5]))
-      self.assertEqual(sess.run(tl), 3)
+      self.assertEqual(self.evaluate(tl), 3)
 
   def test_len_scalar(self):
     with self.assertRaises(ValueError):
@@ -120,18 +120,18 @@ class PyBuiltinsTest(test.TestCase):
   def test_range_tensor(self):
     with self.cached_session() as sess:
       r = py_builtins.range_(constant_op.constant(3))
-      self.assertAllEqual(sess.run(r), [0, 1, 2])
+      self.assertAllEqual(self.evaluate(r), [0, 1, 2])
       r = py_builtins.range_(1, constant_op.constant(3))
-      self.assertAllEqual(sess.run(r), [1, 2])
+      self.assertAllEqual(self.evaluate(r), [1, 2])
       r = py_builtins.range_(2, 0, constant_op.constant(-1))
-      self.assertAllEqual(sess.run(r), [2, 1])
+      self.assertAllEqual(self.evaluate(r), [2, 1])
 
   def test_range_tensor_empty_range(self):
     with self.session() as sess:
       r = py_builtins.range_(constant_op.constant(-3))
-      self.assertAllEqual(sess.run(r), [])
+      self.assertAllEqual(self.evaluate(r), [])
       r = py_builtins.range_(5, constant_op.constant(2))
-      self.assertAllEqual(sess.run(r), [])
+      self.assertAllEqual(self.evaluate(r), [])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/operators/slices_test.py b/tensorflow/python/autograph/operators/slices_test.py
index 9e4865b3c6..d444054fd7 100644
--- a/tensorflow/python/autograph/operators/slices_test.py
+++ b/tensorflow/python/autograph/operators/slices_test.py
@@ -34,7 +34,7 @@ class SlicesTest(test.TestCase):
 
     with self.cached_session() as sess:
       t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
-      self.assertAllEqual(sess.run(t), [[5, 6], [3, 4]])
+      self.assertAllEqual(self.evaluate(t), [[5, 6], [3, 4]])
 
   def test_get_item_tensor_list(self):
     initial_list = constant_op.constant([[1, 2], [3, 4]])
@@ -44,7 +44,7 @@ class SlicesTest(test.TestCase):
         l, 1, slices.GetItemOpts(element_dtype=initial_list.dtype))
 
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4])
+      self.assertAllEqual(self.evaluate(t), [3, 4])
 
   def test_get_item_tensor_string(self):
     initial_str = constant_op.constant('abcd')
@@ -52,14 +52,14 @@ class SlicesTest(test.TestCase):
                         slices.GetItemOpts(element_dtype=initial_str.dtype))
 
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(t), b'b')
+      self.assertEqual(self.evaluate(t), b'b')
 
     initial_list_str = constant_op.constant(['abcd', 'bcde'])
     t = slices.get_item(initial_list_str, 1,
                         slices.GetItemOpts(element_dtype=initial_str.dtype))
 
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(t), b'bcde')
+      self.assertEqual(self.evaluate(t), b'bcde')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py
index 8d2b0d6e13..c813e0f5c9 100644
--- a/tensorflow/python/autograph/utils/misc_test.py
+++ b/tensorflow/python/autograph/utils/misc_test.py
@@ -32,7 +32,7 @@ class MiscTest(test.TestCase):
     new_a = alias_tensors(a)
     self.assertFalse(new_a is a)
     with self.cached_session() as sess:
-      self.assertEqual(1, sess.run(new_a))
+      self.assertEqual(1, self.evaluate(new_a))
 
   def test_alias_tensors(self):
     a = constant(1)
@@ -47,7 +47,7 @@ class MiscTest(test.TestCase):
     self.assertTrue(new_s is s)
     self.assertTrue(new_l is l)
     with self.cached_session() as sess:
-      self.assertEqual(1, sess.run(new_a))
+      self.assertEqual(1, self.evaluate(new_a))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/utils/py_func_test.py b/tensorflow/python/autograph/utils/py_func_test.py
index 1c220d9492..28cefd8c3e 100644
--- a/tensorflow/python/autograph/utils/py_func_test.py
+++ b/tensorflow/python/autograph/utils/py_func_test.py
@@ -34,13 +34,13 @@ class PyFuncTest(test.TestCase):
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (1, constant_op.constant(1), 1))
-      self.assertEqual(3, sess.run(result))
+      self.assertEqual(3, self.evaluate(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, 1, 1))
-      self.assertEqual(3, sess.run(result))
+      self.assertEqual(3, self.evaluate(result))
       result = py_func.wrap_py_func(
           test_fn, dtypes.int64,
           (constant_op.constant(1), 1, constant_op.constant(1)))
-      self.assertEqual(3, sess.run(result))
+      self.assertEqual(3, self.evaluate(result))
 
   def test_wrap_py_func_complex_args(self):
 
@@ -54,10 +54,10 @@ class PyFuncTest(test.TestCase):
 
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass()))
-      self.assertEqual(35, sess.run(result))
+      self.assertEqual(35, self.evaluate(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (constant_op.constant(7), TestClass()))
-      self.assertEqual(35, sess.run(result))
+      self.assertEqual(35, self.evaluate(result))
 
   def test_wrap_py_func_kwargs(self):
 
@@ -74,13 +74,13 @@ class PyFuncTest(test.TestCase):
           'c': 11,
           'd': TestClass(13)
       })
-      self.assertEqual(178, sess.run(result))
+      self.assertEqual(178, self.evaluate(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (constant_op.constant(7), TestClass(5)), {
                                         'c': constant_op.constant(11),
                                         'd': TestClass(13)
                                     })
-      self.assertEqual(178, sess.run(result))
+      self.assertEqual(178, self.evaluate(result))
 
   def test_wrap_py_func_dummy_return(self):
 
@@ -91,11 +91,11 @@ class PyFuncTest(test.TestCase):
 
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, None, (5,), use_dummy_return=True)
-      self.assertEqual(1, sess.run(result))
+      self.assertEqual(1, self.evaluate(result))
       self.assertEqual([1], side_counter)
       result = py_func.wrap_py_func(
           test_fn, None, (constant_op.constant(5),), use_dummy_return=True)
-      self.assertEqual(1, sess.run(result))
+      self.assertEqual(1, self.evaluate(result))
       self.assertEqual([2], side_counter)
 
 
diff --git a/tensorflow/python/autograph/utils/tensor_list_test.py b/tensorflow/python/autograph/utils/tensor_list_test.py
index 697c166eb1..a5bbd97cf9 100644
--- a/tensorflow/python/autograph/utils/tensor_list_test.py
+++ b/tensorflow/python/autograph/utils/tensor_list_test.py
@@ -43,13 +43,13 @@ class TensorListTest(test.TestCase):
     l = tl.dynamic_list_append(l, 1)
     s = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(s), [1])
+      self.assertAllEqual(self.evaluate(s), [1])
 
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l = tl.dynamic_list_append(l, 1)
     s = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(s), [1])
+      self.assertAllEqual(self.evaluate(s), [1])
 
     l = tl.TensorList(self._shape(()), dtypes.int32)
     l = tl.dynamic_list_append(l, 1)
diff --git a/tensorflow/python/client/session_clusterspec_prop_test.py b/tensorflow/python/client/session_clusterspec_prop_test.py
index df020f88a8..224f880ed1 100644
--- a/tensorflow/python/client/session_clusterspec_prop_test.py
+++ b/tensorflow/python/client/session_clusterspec_prop_test.py
@@ -62,7 +62,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
 
     const = constant_op.constant(17)
     sess = session.Session(server1.target, config=config)
-    output = sess.run(const)
+    output = self.evaluate(const)
     self.assertEqual(17, output)
 
   def testClusterSpecPropagationWorker2Placement(self):
@@ -106,7 +106,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
     with ops.Graph().as_default() as g, ops.device('/job:worker/task:0'):
       const = constant_op.constant(17)
     sess = session.Session(server1.target, config=config, graph=g)
-    output = sess.run(const)
+    output = self.evaluate(const)
     self.assertEqual(17, output)
 
   def testCanonicalDeviceNames(self):
@@ -208,7 +208,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
       with ops.device('/job:worker/task:0/cpu:0'):
         sum3 = sum1 + sum2
     sess = session.Session(server1.target, config=config, graph=g)
-    output = sess.run(sum3)
+    output = self.evaluate(sum3)
     self.assertEqual(40, output)
 
   def testLegacyDeviceNames(self):
diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index dfd0147643..f9bd50957a 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -147,7 +147,7 @@ class TimelineTest(test.TestCase):
         num2 = variables.Variable(2.0, name='num2')
       with ops.device('/cpu:2'):
         result = num1 + num2 + num1 * num2
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(result, options=run_options, run_metadata=run_metadata)
 
     self.assertTrue(run_metadata.HasField('step_stats'))
@@ -176,7 +176,7 @@ class TimelineTest(test.TestCase):
         num2 = variables.Variable(2.0, name='num2')
       with ops.device('/cpu:2'):
         result = num1 + num2 + num1 * num2
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(result, options=run_options, run_metadata=run_metadata)
     self.assertTrue(run_metadata.HasField('step_stats'))
     step_stats = run_metadata.step_stats
diff --git a/tensorflow/python/client/virtual_gpu_test.py b/tensorflow/python/client/virtual_gpu_test.py
index 5892e0fc84..e82ee0666c 100644
--- a/tensorflow/python/client/virtual_gpu_test.py
+++ b/tensorflow/python/client/virtual_gpu_test.py
@@ -216,7 +216,7 @@ class VirtualGpuTest(test_util.TensorFlowTestCase):
       for d in self._util.devices:
         with ops.device(d):
           var = variables.Variable(random_ops.random_uniform(mat_shape))
-          sess.run(var.initializer)
+          self.evaluate(var.initializer)
           data.append(var)
       s = data[0]
       for i in range(1, len(data)):
diff --git a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
index e896752a26..dbb780c47d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
@@ -53,10 +53,10 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       for start in range(0, len(components), 4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual([[i, j]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)], results.indices)
@@ -81,10 +81,10 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       for start in range(0, len(components), 4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual([[i, j, z]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)
@@ -141,7 +141,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]})
       for i in range(4):
-        self.assertEqual(i, sess.run(next_elem))
+        self.assertEqual(i, self.evaluate(next_elem))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_elem)
 
@@ -159,7 +159,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i,) * 3, sess.run(op))
+        self.assertEqual((i,) * 3, self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -179,7 +179,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op))
+        self.assertEqual((i, compat.as_bytes(str(i)), i), self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -198,7 +198,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        st_row = sess.run(next_element)
+        st_row = self.evaluate(next_element)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
         self.assertEqual([10], st_row.dense_shape)
@@ -219,7 +219,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        dense_elem, st_row = sess.run(next_element)
+        dense_elem, st_row = self.evaluate(next_element)
         self.assertEqual(i, dense_elem)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
@@ -241,7 +241,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(((i,),) * 3, sess.run(op))
+        self.assertEqual(((i,),) * 3, self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -354,7 +354,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 28, batch_size: 14})
       num_batches = (28 * 7) // 14
       for i in range(num_batches):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(14):
             self.assertAllEqual(component[(i * 14 + j) % 7]**2,
@@ -369,12 +369,12 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       # We expect (num_batches - 1) full-sized batches.
       num_batches = int(math.ceil((14 * 7) / 8))
       for i in range(num_batches - 1):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(8):
             self.assertAllEqual(component[(i * 8 + j) % 7]**2,
                                 result_component[j])
-      result = sess.run(get_next)
+      result = self.evaluate(get_next)
       for component, result_component in zip(components, result):
         for j in range((14 * 7) % 8):
           self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2,
@@ -408,10 +408,10 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
       if not drop_remainder:
-        self.assertAllEqual([[64], [81]], sess.run(next_element))
+        self.assertAllEqual([[64], [81]], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -423,9 +423,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
+      self.assertAllEqual([[64], [81]], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -439,7 +439,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(5):
-        got = sess.run(elements)
+        got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -459,7 +459,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(4):
-        got = sess.run(elements)
+        got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -480,9 +480,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(2):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -524,7 +524,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "number of elements does not match"):
         sess.run(get_next)
@@ -576,7 +576,8 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(threshold // 10):
-        self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next))
+        self.assertAllEqual([i * 10 + j for j in range(10)],
+                            self.evaluate(get_next))
       if threshold % 10 != 0:
         self.assertAllEqual(
             [threshold // 10 * 10 + j for j in range(threshold % 10)],
@@ -609,7 +610,8 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(10):
-        self.assertAllEqual([element for _ in range(10)], sess.run(get_next))
+        self.assertAllEqual([element for _ in range(10)],
+                            self.evaluate(get_next))
 
 
 class UnbatchDatasetBenchmark(test.Benchmark):
diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 3903ec49b9..4263a90f4c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -300,7 +300,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       with self.cached_session() as sess:
         with self.assertRaises(errors.OutOfRangeError):
           while True:
-            output = sess.run(batch)
+            output = self.evaluate(batch)
             sprs_tensor = (tuple([tuple(idx) for idx in output.indices]),
                            tuple(output.values))
             all_sparse_tensors.add(sprs_tensor)
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index cea8bd6f0b..6d063ac9c8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -57,7 +57,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -82,7 +82,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -108,7 +108,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -134,7 +134,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -160,7 +160,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, sess.run(next_element))
+        self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -186,7 +186,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, sess.run(next_element))
+        self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -217,7 +217,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = sess.run(next_element)
+        actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
@@ -251,7 +251,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = sess.run(next_element)
+        actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
@@ -271,9 +271,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -290,9 +290,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -323,9 +323,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        x, y, z = sess.run(next_element)
+        x, y, z = self.evaluate(next_element)
         self.assertEqual(i**2, x)
         self.assertEqual(float(i**2), y)
         self.assertEqual(util_compat.as_bytes(str(i)), z)
@@ -345,8 +345,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -363,8 +363,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -381,8 +381,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -399,8 +399,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -420,9 +420,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         next_element = iterator.get_next()
 
       with self.cached_session() as sess:
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         for i in range(10):
-          self.assertEqual(i, sess.run(next_element))
+          self.assertEqual(i, self.evaluate(next_element))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
@@ -447,12 +447,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -477,12 +477,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -499,12 +499,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -521,12 +521,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -553,7 +553,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
       # For each element of the dataset, assert that the optional evaluates to
       # the expected value.
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(3):
         elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t])
         self.assertTrue(elem_has_value)
@@ -562,7 +562,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       # After exhausting the iterator, `next_elem.has_value()` will evaluate to
       # false, and attempting to get the value will fail.
       for _ in range(2):
-        self.assertFalse(sess.run(elem_has_value_t))
+        self.assertFalse(self.evaluate(elem_has_value_t))
         with self.assertRaises(errors.InvalidArgumentError):
           sess.run(elem_value_t)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/counter_test.py b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
index 4e114ac479..d1dd07a879 100644
--- a/tensorflow/python/data/experimental/kernel_tests/counter_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
@@ -38,13 +38,13 @@ class CounterTest(test_base.DatasetTestBase):
     negative_get_next = negative_iterator.get_next()
 
     with self.cached_session() as sess:
-      self.assertEqual(3, sess.run(get_next))
-      self.assertEqual(3 + 4, sess.run(get_next))
-      self.assertEqual(3 + 2 * 4, sess.run(get_next))
+      self.assertEqual(3, self.evaluate(get_next))
+      self.assertEqual(3 + 4, self.evaluate(get_next))
+      self.assertEqual(3 + 2 * 4, self.evaluate(get_next))
 
-      self.assertEqual(0, sess.run(negative_get_next))
-      self.assertEqual(-1, sess.run(negative_get_next))
-      self.assertEqual(-2, sess.run(negative_get_next))
+      self.assertEqual(0, self.evaluate(negative_get_next))
+      self.assertEqual(-1, self.evaluate(negative_get_next))
+      self.assertEqual(-2, self.evaluate(negative_get_next))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
index 73be6cbcca..9fe2ee43ed 100644
--- a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
@@ -41,10 +41,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       for start in range(0, len(components), 4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual([[i, j]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)], results.indices)
@@ -69,10 +69,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       for start in range(0, len(components), 4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual([[i, j, z]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)
diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
index 796a692c56..234fd86bdd 100644
--- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
@@ -40,10 +40,10 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for _ in range(100):
         for i in range(10):
-          self.assertEqual(i, sess.run(next_element))
+          self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -107,7 +107,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in choice_array:
-        self.assertEqual(words[i], sess.run(next_element))
+        self.assertEqual(words[i], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
index e54235d9f8..78805bb801 100644
--- a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
@@ -44,9 +44,9 @@ class EnumerateDatasetTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next[1]])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertEqual((20, (b"a", 1, 37.0)), sess.run(get_next))
-      self.assertEqual((21, (b"b", 2, 38.0)), sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertEqual((20, (b"a", 1, 37.0)), self.evaluate(get_next))
+      self.assertEqual((21, (b"b", 2, 38.0)), self.evaluate(get_next))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
diff --git a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
index d38452e265..860442571e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
@@ -94,18 +94,18 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
                                                   device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [5.0])
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
   def testSameDeviceCPU(self):
     self._prefetch_fn_helper_one_shot("same_device_cpu",
@@ -135,35 +135,35 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
         ds, ds_iterator, "reinit", device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      sess.run(ds_iterator.initializer)
-      elem = sess.run(prefetch_op)
+      self.evaluate(ds_iterator.initializer)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [5.0])
       # Lets reset the function buffering resource and reinitialize the
       # iterator. Should be able to go through this again.
       self._event.clear()
-      sess.run(reset_op)
-      sess.run(ds_iterator.initializer)
-      elem = sess.run(prefetch_op)
+      self.evaluate(reset_op)
+      self.evaluate(ds_iterator.initializer)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [5.0])
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
   def testReinitializationOutOfRange(self):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
@@ -175,30 +175,30 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
         ds, ds_iterator, "reinit", device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      sess.run(ds_iterator.initializer)
+      self.evaluate(ds_iterator.initializer)
       for i in range(1, 10):
-        elem = sess.run(prefetch_op)
+        elem = self.evaluate(prefetch_op)
         self.assertEqual(elem, [float(i)])
       # Try fetching after its over twice to test out end of sequence.
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
 
       # Now reset everything and try it out again.
       self._event.clear()
-      sess.run(reset_op)
-      sess.run(ds_iterator.initializer)
+      self.evaluate(reset_op)
+      self.evaluate(ds_iterator.initializer)
       for i in range(1, 10):
-        elem = sess.run(prefetch_op)
+        elem = self.evaluate(prefetch_op)
         self.assertEqual(elem, [float(i)])
       # Try fetching after its over twice to test out end of sequence.
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
 
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
   def testStringsGPU(self):
     if not test_util.is_gpu_available():
@@ -235,13 +235,13 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
           buffer_resource_handle, ignore_lookup_error=True)
 
     with self.cached_session() as sess:
-      self.assertEqual([b"a"], sess.run(prefetch_op))
-      self.assertEqual([b"b"], sess.run(prefetch_op))
-      self.assertEqual([b"c"], sess.run(prefetch_op))
+      self.assertEqual([b"a"], self.evaluate(prefetch_op))
+      self.assertEqual([b"b"], self.evaluate(prefetch_op))
+      self.assertEqual([b"c"], self.evaluate(prefetch_op))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
 
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
index 9030328593..15396f329d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
@@ -39,7 +39,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
     get_next = dataset.make_one_shot_iterator().get_next()
     with self.cached_session() as sess:
       for expected in values:
-        got = sess.run(get_next)
+        got = self.evaluate(get_next)
         self.assertEqual(got, expected)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -127,7 +127,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       iterator = dataset.make_one_shot_iterator()
       get_next = iterator.get_next()
       with self.cached_session() as sess:
-        x, y = sess.run(get_next)
+        x, y = self.evaluate(get_next)
         self.assertAllEqual([0] * (2**i), x)
         self.assertAllEqual(np.array(1, ndmin=i), y)
         with self.assertRaises(errors.OutOfRangeError):
@@ -190,7 +190,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
             grouping.group_by_reducer(lambda x, y: np.int64(0), reducer))
     get_next = dataset.make_one_shot_iterator().get_next()
     with self.cached_session() as sess:
-      x, y = sess.run(get_next)
+      x, y = self.evaluate(get_next)
       self.assertAllEqual(x, np.asarray([x for x in range(10)]))
       self.assertEqual(y, 45)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
index 557d56e8b9..cfc357ba13 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
@@ -68,9 +68,9 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
-      which_bucket, bucketed_values = sess.run(get_next)
+      which_bucket, bucketed_values = self.evaluate(get_next)
 
       self.assertEqual(0, which_bucket)
 
@@ -103,11 +103,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       # Get two minibatches (one containing even values, one containing odds)
-      which_bucket_even, bucketed_values_even = sess.run(get_next)
-      which_bucket_odd, bucketed_values_odd = sess.run(get_next)
+      which_bucket_even, bucketed_values_even = self.evaluate(get_next)
+      which_bucket_odd, bucketed_values_odd = self.evaluate(get_next)
 
       # Count number of bucket_tensors.
       self.assertEqual(3, len(bucketed_values_even))
@@ -174,11 +174,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       # Get two minibatches ([0, 2, ...] and [64, 66, ...])
-      which_bucket0, bucketed_values_even0 = sess.run(get_next)
-      which_bucket1, bucketed_values_even1 = sess.run(get_next)
+      which_bucket0, bucketed_values_even0 = self.evaluate(get_next)
+      which_bucket1, bucketed_values_even1 = self.evaluate(get_next)
 
       # Ensure that bucket 1 was completely filtered out
       self.assertAllEqual(0, which_bucket0)
@@ -207,11 +207,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaises(errors.OutOfRangeError):
         batches = 0
         while True:
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           is_even = all(x % 2 == 0 for x in result)
           is_odd = all(x % 2 == 1 for x in result)
           self.assertTrue(is_even or is_odd)
@@ -232,11 +232,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       counts = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           self.assertTrue(
               all(x % 2 == 0
                   for x in result) or all(x % 2 == 1)
@@ -259,16 +259,16 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       # The input is infinite, so this test demonstrates that:
       # 1. We produce output without having to consume the entire input,
       # 2. Different buckets can produce output at different rates, and
       # 3. For deterministic input, the output is deterministic.
       for _ in range(3):
-        self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
-        self.assertAllEqual([1, 1, 1, 1], sess.run(get_next))
-        self.assertAllEqual([2, 2, 2, 2], sess.run(get_next))
-        self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
+        self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
+        self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next))
+        self.assertAllEqual([2, 2, 2, 2], self.evaluate(get_next))
+        self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
 
   def testSmallGroups(self):
     components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64)
@@ -280,13 +280,13 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
-      self.assertAllEqual([1, 1, 1, 1], sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
+      self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next))
       # The small outputs at the end are deterministically produced in key
       # order.
-      self.assertAllEqual([0, 0, 0], sess.run(get_next))
-      self.assertAllEqual([1], sess.run(get_next))
+      self.assertAllEqual([0, 0, 0], self.evaluate(get_next))
+      self.assertAllEqual([1], self.evaluate(get_next))
 
   def testEmpty(self):
     iterator = (
@@ -297,7 +297,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           "Window size must be greater than zero, but got 0."):
@@ -323,7 +323,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(get_next)
 
@@ -351,11 +351,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       counts = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-          tight_result, multiple_of_10_result = sess.run(get_next)
+          tight_result, multiple_of_10_result = self.evaluate(get_next)
           self.assertEqual(0, multiple_of_10_result.shape[1] % 10)
           self.assertAllEqual(tight_result,
                               multiple_of_10_result[:, :tight_result.shape[1]])
diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
index c0ec1486ab..cb0fc13914 100644
--- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
@@ -47,9 +47,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for x in [1., 2., 3., 5.]:
-        self.assertEqual(x, sess.run(get_next))
+        self.assertEqual(x, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -65,9 +65,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for x in [1., 2., 3., 5.]:
-        self.assertEqual(x, sess.run(get_next))
+        self.assertEqual(x, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -93,9 +93,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       # All of the files are present.
-      sess.run(init_op)
+      self.evaluate(init_op)
       for filename in filenames:
-        self.assertEqual(compat.as_bytes(filename), sess.run(get_next))
+        self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -104,9 +104,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
 
       # Attempting to read filenames[0] will fail, but ignore_errors()
       # will catch the error.
-      sess.run(init_op)
+      self.evaluate(init_op)
       for filename in filenames[1:]:
-        self.assertEqual(compat.as_bytes(filename), sess.run(get_next))
+        self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
index c93a8353ce..c4076daef2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
@@ -53,7 +53,7 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
     ds = indexed_dataset_ops.IdentityIndexedDataset(16)
     materialized = ds.materialize()
     with self.cached_session() as sess:
-      sess.run(materialized.initializer)
+      self.evaluate(materialized.initializer)
       placeholder = array_ops.placeholder(dtypes.uint64, shape=[])
       for i in range(16):
         output = sess.run(
@@ -68,9 +68,9 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
     itr = ds.make_initializable_iterator()
     n = itr.get_next()
     with self.cached_session() as sess:
-      sess.run(itr.initializer)
+      self.evaluate(itr.initializer)
       for i in range(16):
-        output = sess.run(n)
+        output = self.evaluate(n)
         self.assertEqual(i, output)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(n)
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
index 91ae8cb1bd..c6cefa7034 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
@@ -112,10 +112,10 @@ class MakeBatchedFeaturesDatasetTest(
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for file_batch, _, _, _, record_batch, _ in self._next_expected_batch(
           range(self._num_files), 2, 10):
-        actual_batch = sess.run(next_element)
+        actual_batch = self.evaluate(next_element)
         self.assertAllEqual(file_batch, actual_batch["file"])
         self.assertAllEqual(record_batch, actual_batch["record"])
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index e4bf089184..5486369462 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -90,7 +90,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         batch_size,
         num_epochs,
     ):
-      actual_features = sess.run(nxt)
+      actual_features = self.evaluate(nxt)
 
       if label_name is not None:
         expected_labels = expected_features.pop(label_name)
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
index 657cf3c00e..404edf2fda 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
@@ -105,7 +105,7 @@ class MakeTFRecordDatasetTest(
     for expected_batch in self._next_expected_batch(
         file_indices, batch_size, num_epochs, interleave_cycle_length,
         drop_final_batch, use_parser_fn):
-      actual_batch = sess.run(outputs)
+      actual_batch = self.evaluate(outputs)
       self.assertAllEqual(expected_batch, actual_batch)
 
   def _read_test(self, batch_size, num_epochs, file_index=None,
@@ -188,7 +188,7 @@ class MakeTFRecordDatasetTest(
         iterator = dataset.make_initializable_iterator()
         next_element = iterator.get_next()
 
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         first_batches = []
         try:
           while True:
@@ -196,7 +196,7 @@ class MakeTFRecordDatasetTest(
         except errors.OutOfRangeError:
           pass
 
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         second_batches = []
         try:
           while True:
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index 5ead6d1c75..b4bc4a617f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -89,7 +89,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 28, batch_size: 14})
       num_batches = (28 * 7) // 14
       for i in range(num_batches):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(14):
             self.assertAllEqual(component[(i * 14 + j) % 7]**2,
@@ -104,12 +104,12 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       # We expect (num_batches - 1) full-sized batches.
       num_batches = int(math.ceil((14 * 7) / 8))
       for i in range(num_batches - 1):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(8):
             self.assertAllEqual(component[(i * 8 + j) % 7]**2,
                                 result_component[j])
-      result = sess.run(get_next)
+      result = self.evaluate(get_next)
       for component, result_component in zip(components, result):
         for j in range((14 * 7) % 8):
           self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2,
@@ -152,10 +152,10 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
       if not drop_remainder:
-        self.assertAllEqual([[64], [81]], sess.run(next_element))
+        self.assertAllEqual([[64], [81]], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -177,9 +177,9 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
+      self.assertAllEqual([[64], [81]], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -201,7 +201,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(5):
-        got = sess.run(elements)
+        got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -230,7 +230,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(4):
-        got = sess.run(elements)
+        got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -261,9 +261,9 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(2):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -321,7 +321,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "number of elements does not match"):
         sess.run(get_next)
@@ -393,7 +393,8 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(threshold // 10):
-        self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next))
+        self.assertAllEqual([i * 10 + j for j in range(10)],
+                            self.evaluate(get_next))
       if threshold % 10 != 0:
         self.assertAllEqual(
             [threshold // 10 * 10 + j for j in range(threshold % 10)],
@@ -442,7 +443,8 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(10):
-        self.assertAllEqual([element for _ in range(10)], sess.run(get_next))
+        self.assertAllEqual([element for _ in range(10)],
+                            self.evaluate(get_next))
 
   @parameterized.named_parameters(
       ("Identity", None, lambda x: x, None),
@@ -462,7 +464,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       else:
         expected = map_fn(
             sess.run(self.structuredElement(structure, shape=[10])))
-      self.assertAllEqual(expected, sess.run(get_next))
+      self.assertAllEqual(expected, self.evaluate(get_next))
 
   def testShortCircuitCapturedInput(self):
     captured_t = array_ops.placeholder(dtypes.int64, shape=[])
@@ -473,7 +475,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={captured_t: 42})
-      self.assertAllEqual([42] * 10, sess.run(get_next))
+      self.assertAllEqual([42] * 10, self.evaluate(get_next))
 
   @parameterized.named_parameters(
       ("Normal", False),
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
index 11694540fa..3cf3b89c3f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
@@ -218,7 +218,7 @@ class MapDefunTest(test_base.DatasetTestBase):
 
   def _assert_op_cancelled(self, sess, map_defun_op):
     with self.assertRaisesRegexp(errors.CancelledError, "was cancelled"):
-      sess.run(map_defun_op)
+      self.evaluate(map_defun_op)
 
   def testMapDefunWithParentCancellation(self):
     # Checks that a cancellation of the parent graph is threaded through to
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index 5e419a9b2f..ca8bc5ff97 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -72,7 +72,7 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       thread_ids = []
       try:
         while True:
diff --git a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
index 90ac250df7..91908f5582 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
@@ -637,11 +637,11 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, sess.run(get_next))
+          self.assertAllEqual(expected, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -796,7 +796,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for _ in range(2):
         elements = []
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         try:
           while True:
             elements.extend(sess.run(next_element))
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index f73725366c..60c3741d32 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -57,7 +57,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -87,7 +87,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -117,7 +117,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, sess.run(next_element))
+        self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -150,7 +150,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = sess.run(next_element)
+        actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
@@ -170,7 +170,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -199,12 +199,12 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -220,12 +220,12 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index 0730455431..0e9bb462f3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -60,7 +60,7 @@ class ScanTest(test_base.DatasetTestBase):
                  feed_dict={start: start_val, step: step_val, take: take_val})
         for expected, _ in zip(
             itertools.count(start_val, step_val), range(take_val)):
-          self.assertEqual(expected, sess.run(next_element))
+          self.assertEqual(expected, self.evaluate(next_element))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
@@ -110,7 +110,7 @@ class ScanTest(test_base.DatasetTestBase):
                  feed_dict={start: start_val, step: step_val, take: take_val})
         for expected, _ in zip(
             itertools.count(start_val, step_val), range(take_val)):
-          self.assertEqual(expected, sess.run(next_element).values[0])
+          self.assertEqual(expected, self.evaluate(next_element).values[0])
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
@@ -136,7 +136,7 @@ class ScanTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(5):
-        (longer_vector_val, larger_rank_val), _ = sess.run(next_element)
+        (longer_vector_val, larger_rank_val), _ = self.evaluate(next_element)
         self.assertAllEqual([0] * (2**i), longer_vector_val)
         self.assertAllEqual(np.array(1, ndmin=i), larger_rank_val)
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
index ef99d01c73..704a40721f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
@@ -71,19 +71,19 @@ class RangeDatasetSerializationTest(
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
+        self.evaluate(init_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -91,14 +91,14 @@ class RangeDatasetSerializationTest(
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-        sess.run(restore_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
index 88d5c896c9..496fd45947 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
@@ -62,7 +62,7 @@ class SerializationIntegrationTest(test.TestCase):
       with self.session(graph=g) as sess:
         sess.run(init_ops)
         for _ in range(break_point):
-          output = sess.run(get_next_ops)
+          output = self.evaluate(get_next_ops)
           for i in range(num_pipelines):
             all_outputs[i].append(output[i])
         saver.save(sess, self._ckpt_path())
@@ -73,7 +73,7 @@ class SerializationIntegrationTest(test.TestCase):
       with self.session(graph=g) as sess:
         saver.restore(sess, self._ckpt_path())
         for _ in range(num_outputs - break_point):
-          output = sess.run(get_next_ops)
+          output = self.evaluate(get_next_ops)
           for i in range(num_pipelines):
             all_outputs[i].append(output[i])
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
index c208963a86..5f7d9051ec 100644
--- a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
@@ -108,7 +108,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
           shuffle_ops.shuffle_and_repeat(buffer_size=21))
       get_next_op = ds.make_one_shot_iterator().get_next()
       with self.session(graph=g) as sess:
-        sess.run(get_next_op)
+        self.evaluate(get_next_op)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
index bf53acc82a..f7d42bc5b3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
@@ -38,10 +38,10 @@ class SleepTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       start_time = time.time()
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       end_time = time.time()
       self.assertGreater(end_time - start_time, (10 * sleep_microseconds) / 1e6)
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
index a2c1169638..e11bad7969 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
@@ -39,8 +39,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                             "ORDER BY first_name DESC"
             })
         for _ in range(2):  # Dataset is repeated. See setUp.
-          self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next))
-          self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next))
+          self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next))
+          self.assertEqual((b"Jane", b"Moe", b"Hi again!"),
+                           self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -58,7 +59,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ON students.first_name = people.first_name "
                   "AND students.last_name = people.last_name"
           })
-      self.assertEqual((b"John", b"California", b"Hi!"), sess.run(get_next))
+      self.assertEqual((b"John", b"California", b"Hi!"),
+                       self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -75,8 +77,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, last_name, favorite_nonsense_word "
                   "FROM students ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"n\0nsense"), sess.run(get_next))
-      self.assertEqual((b"Jane", b"Moe", b"nonsense\0"), sess.run(get_next))
+      self.assertEqual((b"John", b"Doe", b"n\0nsense"), self.evaluate(get_next))
+      self.assertEqual((b"Jane", b"Moe", b"nonsense\0"),
+                       self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -93,8 +96,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, last_name, motto FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next))
-      self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next))
+      self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next))
+      self.assertEqual((b"Jane", b"Moe", b"Hi again!"), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
       sess.run(
@@ -103,7 +106,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, last_name, state FROM people "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"California"), sess.run(get_next))
+      self.assertEqual((b"John", b"Doe", b"California"),
+                       self.evaluate(get_next))
       self.assertEqual((b"Benjamin", b"Franklin", b"Pennsylvania"),
                        sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
@@ -212,8 +216,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -230,7 +234,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students "
                           "WHERE first_name = 'John' ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0, -2), sess.run(get_next))
+      self.assertEqual((b"John", 0, -2), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -246,9 +250,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT desk_number, favorite_negative_number FROM students "
                   "ORDER BY first_name DESC"
           })
-      self.assertEqual((9, -2), sess.run(get_next))
+      self.assertEqual((9, -2), self.evaluate(get_next))
       # Max and min values of int8
-      self.assertEqual((127, -128), sess.run(get_next))
+      self.assertEqual((127, -128), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -263,8 +267,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -281,7 +285,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students "
                           "WHERE first_name = 'John' ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0, -2), sess.run(get_next))
+      self.assertEqual((b"John", 0, -2), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -297,9 +301,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students ORDER BY first_name DESC"
           })
       # Max value of int16
-      self.assertEqual((b"John", 32767), sess.run(get_next))
+      self.assertEqual((b"John", 32767), self.evaluate(get_next))
       # Min value of int16
-      self.assertEqual((b"Jane", -32768), sess.run(get_next))
+      self.assertEqual((b"Jane", -32768), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -314,8 +318,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
 
   # Test that `SqlDataset` can read a negative or 0-valued integer from a
   # SQLite database table and place it in an `int32` tensor.
@@ -328,8 +332,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, income FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0), sess.run(get_next))
-      self.assertEqual((b"Jane", -20000), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -20000), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -345,9 +349,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Max value of int32
-      self.assertEqual((b"John", 2147483647), sess.run(get_next))
+      self.assertEqual((b"John", 2147483647), self.evaluate(get_next))
       # Min value of int32
-      self.assertEqual((b"Jane", -2147483648), sess.run(get_next))
+      self.assertEqual((b"Jane", -2147483648), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -362,8 +366,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, school_id FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 123), sess.run(get_next))
-      self.assertEqual((b"Jane", 1000), sess.run(get_next))
+      self.assertEqual((b"John", 123), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 1000), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -378,8 +382,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -394,8 +398,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, income FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0), sess.run(get_next))
-      self.assertEqual((b"Jane", -20000), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -20000), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -412,9 +416,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ORDER BY first_name DESC"
           })
       # Max value of int64
-      self.assertEqual((b"John", 9223372036854775807), sess.run(get_next))
+      self.assertEqual((b"John", 9223372036854775807), self.evaluate(get_next))
       # Min value of int64
-      self.assertEqual((b"Jane", -9223372036854775808), sess.run(get_next))
+      self.assertEqual((b"Jane", -9223372036854775808), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -429,8 +433,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -446,9 +450,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Min value of uint8
-      self.assertEqual((b"John", 0), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
       # Max value of uint8
-      self.assertEqual((b"Jane", 255), sess.run(get_next))
+      self.assertEqual((b"Jane", 255), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -463,8 +467,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -480,9 +484,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Min value of uint16
-      self.assertEqual((b"John", 0), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
       # Max value of uint16
-      self.assertEqual((b"Jane", 65535), sess.run(get_next))
+      self.assertEqual((b"Jane", 65535), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -499,8 +503,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, registration_complete FROM students "
                   "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", True), sess.run(get_next))
-      self.assertEqual((b"Jane", False), sess.run(get_next))
+      self.assertEqual((b"John", True), self.evaluate(get_next))
+      self.assertEqual((b"Jane", False), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -515,8 +519,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, favorite_medium_sized_number "
                           "FROM students ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", True), sess.run(get_next))
-      self.assertEqual((b"Jane", True), sess.run(get_next))
+      self.assertEqual((b"John", True), self.evaluate(get_next))
+      self.assertEqual((b"Jane", True), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -533,8 +537,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, last_name, victories FROM townspeople "
                   "ORDER BY first_name"
           })
-      self.assertEqual((b"George", b"Washington", 20.0), sess.run(get_next))
-      self.assertEqual((b"John", b"Adams", -19.95), sess.run(get_next))
+      self.assertEqual((b"George", b"Washington", 20.0),
+                       self.evaluate(get_next))
+      self.assertEqual((b"John", b"Adams", -19.95), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index 83028937d3..958c3f0038 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -74,18 +74,18 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       expected_sum = 0.0
       for i in range(100):
         self.assertAllEqual(
             np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = sess.run(summary_t)
+        summary_str = self.evaluate(summary_t)
         self._assertSummaryHasCount(summary_str, "bytes_produced", float(i + 1))
         expected_sum += i * 8.0
         self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      summary_str = sess.run(summary_t)
+      summary_str = self.evaluate(summary_t)
       self._assertSummaryHasCount(summary_str, "bytes_produced", 100.0)
       self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
 
@@ -99,14 +99,15 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 100.0)
 
   def testPrefetchBufferUtilization(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -118,11 +119,11 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
         self.assertAllEqual(
             np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = sess.run(summary_t)
+        summary_str = self.evaluate(summary_t)
         self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                     float(i + 1))
         self._assertSummaryContains(summary_str, "Prefetch::buffer_capacity")
@@ -131,7 +132,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
                                     0, 1)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      summary_str = sess.run(summary_t)
+      summary_str = self.evaluate(summary_t)
       self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                   100)
 
@@ -145,11 +146,11 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
         self.assertAllEqual(
             np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = sess.run(summary_t)
+        summary_str = self.evaluate(summary_t)
         self._assertSummaryHasScalarValue(summary_str,
                                           "Prefetch::buffer_capacity", 0)
         self._assertSummaryHasScalarValue(summary_str, "Prefetch::buffer_size",
@@ -167,9 +168,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(34):
-        self.assertEqual(i * 3, sess.run(next_element))
+        self.assertEqual(i * 3, self.evaluate(next_element))
         if i is not 0:
           self._assertSummaryHasScalarValue(
               sess.run(summary_t), "Filter::dropped_elements", float(i * 2))
@@ -261,9 +262,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
     with self.cached_session() as sess:
       for j in range(5):
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         for i in range(100):
-          self.assertEqual(i, sess.run(next_element))
+          self.assertEqual(i, self.evaluate(next_element))
           self._assertSummaryHasCount(
               sess.run(summary_t), "record_latency", float((j * 100) + i + 1))
         with self.assertRaises(errors.OutOfRangeError):
@@ -278,9 +279,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -295,16 +296,17 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(i + 1))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency_2", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 100.0)
       self._assertSummaryHasCount(
           sess.run(summary_t), "record_latency_2", 100.0)
 
@@ -319,14 +321,15 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(2 * (i + 1)))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 200.0)
 
   def testMultipleIteratorsSameAggregator(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -341,12 +344,13 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     with self.cached_session() as sess:
       sess.run([iterator_0.initializer, iterator_1.initializer])
       for i in range(100):
-        self.assertEqual(i * 2, sess.run(next_element))
+        self.assertEqual(i * 2, self.evaluate(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(2 * (i + 1)))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 200.0)
 
   def testMultipleDatasetWithPrefixes(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -364,7 +368,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     with self.test_session() as sess:
       sess.run([iterator_0.initializer, iterator_1.initializer])
       for i in range(100):
-        self.assertEqual(i * 2, sess.run(next_element))
+        self.assertEqual(i * 2, self.evaluate(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "dataset1_record_latency", float(i + 1))
         self._assertSummaryHasCount(
@@ -421,7 +425,7 @@ class FeatureStatsDatasetTest(
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for _ in range(num_output):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index 0278a208cb..755294ac45 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -50,7 +50,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]})
       for i in range(4):
-        self.assertEqual(i, sess.run(next_elem))
+        self.assertEqual(i, self.evaluate(next_elem))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_elem)
 
@@ -68,7 +68,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i,) * 3, sess.run(op))
+        self.assertEqual((i,) * 3, self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -88,7 +88,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op))
+        self.assertEqual((i, compat.as_bytes(str(i)), i), self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -107,7 +107,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        st_row = sess.run(next_element)
+        st_row = self.evaluate(next_element)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
         self.assertEqual([10], st_row.dense_shape)
@@ -128,7 +128,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        dense_elem, st_row = sess.run(next_element)
+        dense_elem, st_row = self.evaluate(next_element)
         self.assertEqual(i, dense_elem)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
@@ -150,7 +150,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(((i,),) * 3, sess.run(op))
+        self.assertEqual(((i,),) * 3, self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
diff --git a/tensorflow/python/data/experimental/kernel_tests/unique_test.py b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
index 847cff26b0..4b14a7e963 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unique_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
@@ -49,11 +49,11 @@ class UniqueTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for test_case, expected in test_cases:
         current_test_case = test_case
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         for element in expected:
           if dtype == dtypes.string:
             element = compat.as_bytes(element)
-          self.assertAllEqual(element, sess.run(next_element))
+          self.assertAllEqual(element, self.evaluate(next_element))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
diff --git a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
index e8decb9ad0..10a0427c7f 100644
--- a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
@@ -93,13 +93,13 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           })
       num_full_batches = (count * 7) // batch_size
       for i in range(num_full_batches):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(batch_size):
             self.assertAllEqual(component[(i * batch_size + j) % 7]**2,
                                 result_component[j])
       if not drop_remainder and (count * 7) % batch_size > 0:
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range((count * 7) % batch_size):
             self.assertAllEqual(
@@ -128,9 +128,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(2):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -155,9 +155,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(2):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected_indices = []
         expected_values = []
         for j in range(5):
@@ -185,8 +185,8 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      actual = sess.run(get_next)
+      self.evaluate(init_op)
+      actual = self.evaluate(get_next)
       expected = sparse_tensor.SparseTensorValue(
           indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0],
                    [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]],
@@ -211,7 +211,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           r'Cannot batch tensors with different shapes in component 0. '
@@ -271,7 +271,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       num_full_batches = len(seq_lens) // batch_size
 
       for i in range(num_full_batches):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         padded_len = padded_shapes[0]
         if padded_len is None or padded_len == -1:
           padded_len = np.max(result) if result.size > 0 else 0
@@ -283,7 +283,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
                               [0] * (padded_len - seq_len))
 
       if not drop_remainder and len(seq_lens) % batch_size > 0:
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         padded_len = np.max(result) if result.size > 0 else 0
         self.assertEqual((len(seq_lens) % batch_size, padded_len),
                          result.shape)
@@ -315,7 +315,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      result = sess.run(get_next)
+      result = self.evaluate(get_next)
       self.assertAllEqual([[], [], [], []], result)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -347,7 +347,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
               seq_lens: random_seq_lens
           })
       for i in range(8):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         padded_len = np.max(result[0])
         self.assertEqual((4, padded_len), result[0].shape)
         self.assertEqual((4, padded_len), result[1].shape)
diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
index 63625fac03..1f351279c6 100644
--- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
@@ -71,7 +71,7 @@ class FileCacheDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       # First run without caching to collect the "ground truth".
-      sess.run(init_fifo_op)
+      self.evaluate(init_fifo_op)
       elements = []
       for _ in range(20):
         elements.append(sess.run(get_next))
@@ -220,14 +220,14 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
 
       with self.cached_session() as sess:
 
-        sess.run(repeat_count.initializer)
-        sess.run(cached_iterator.initializer)
-        sess.run(uncached_iterator.initializer)
+        self.evaluate(repeat_count.initializer)
+        self.evaluate(cached_iterator.initializer)
+        self.evaluate(uncached_iterator.initializer)
 
         for i in range(3):
           for _ in range(10):
-            self.assertEqual(sess.run(cached_next), i)
-            self.assertEqual(sess.run(uncached_next), i)
+            self.assertEqual(self.evaluate(cached_next), i)
+            self.assertEqual(self.evaluate(uncached_next), i)
 
         sess.run(repeat_count.assign(0))
 
@@ -238,7 +238,7 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
         # The cached iterator replays from cache.
         for i in range(3):
           for _ in range(10):
-            self.assertEqual(sess.run(cached_next), i)
+            self.assertEqual(self.evaluate(cached_next), i)
 
         # The cached iterator should now be empty.
         with self.assertRaises(errors.OutOfRangeError):
@@ -280,7 +280,7 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
     i2 = d2.make_initializable_iterator()
 
     with self.cached_session() as sess:
-      sess.run(i1.initializer)
+      self.evaluate(i1.initializer)
 
       self.assertEqual(1, sess.run(i1.get_next()))
       self.assertEqual(2, sess.run(i1.get_next()))
@@ -307,7 +307,7 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i, expected in enumerate(expected_values):
-        self.assertEqual(expected, sess.run(n),
+        self.assertEqual(expected, self.evaluate(n),
                          "Unexpected value at index %s" % i)
 
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
index 83af31f380..a0ef69f082 100644
--- a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
@@ -51,9 +51,9 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(9):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         if i < 4:
           for component, result_component in zip(input_components, result):
             self.assertAllEqual(component[i], result_component)
@@ -85,9 +85,9 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(9):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         if i < 4:
           for component, result_component in zip(input_components, result):
             self.assertAllEqual(component[i], result_component)
diff --git a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
index bc6b36285a..f7b500881c 100644
--- a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
@@ -52,8 +52,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      results = sess.run(get_next)
+      self.evaluate(init_op)
+      results = self.evaluate(get_next)
       for component, result_component in zip(components, results):
         self.assertAllEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -81,8 +81,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
         [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      results = sess.run(get_next)
+      self.evaluate(init_op)
+      results = self.evaluate(get_next)
       for component, result_component in zip(components, results):
         self.assertSparseValuesEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -112,8 +112,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     ], [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      results = sess.run(get_next)
+      self.evaluate(init_op)
+      results = self.evaluate(get_next)
       for component, result_component in zip(components, results):
         if sparse_tensor.is_sparse(component):
           self.assertSparseValuesEqual(component, result_component)
@@ -139,9 +139,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component[i], result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -169,7 +169,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
         [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       expected = [
           (sparse_tensor.SparseTensorValue(
               indices=np.array([[0]]),
@@ -197,7 +197,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                dense_shape=np.array([3]))),
       ]
       for i in range(3):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(expected[i], results):
           self.assertSparseValuesEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -229,7 +229,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     ], [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       expected = [
           (sparse_tensor.SparseTensorValue(
               indices=np.array([[0]]),
@@ -257,7 +257,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                dense_shape=np.array([3]))),
       ]
       for i in range(3):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(
             (list(zip(*components[:3]))[i] + expected[i]), results):
           if sparse_tensor.is_sparse(component):
@@ -280,9 +280,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     self.assertEqual((1,), iterator.output_shapes["bar"])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(3):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertEqual(components["foo"][i], results["foo"])
         self.assertEqual(components["bar"][i], results["bar"])
       with self.assertRaises(errors.OutOfRangeError):
@@ -308,7 +308,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                                                     dense_shape)
       sess.run(init_op, feed_dict={st: sparse_feed})
       for i, s in enumerate(slices):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual(s, results.values)
         expected_indices = np.array(
             [[j] for j in range(len(slices[i]))]).reshape([-1, 1])
@@ -474,15 +474,15 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
       with ops.device("/cpu:0"):
         var_0 = resource_variable_ops.ResourceVariable(initial_value=0)
         dataset = dataset.map(lambda x: x + var_0.read_value())
-      sess.run(var_0.initializer)
+      self.evaluate(var_0.initializer)
 
       with ops.device("/cpu:1"):
         var_1 = resource_variable_ops.ResourceVariable(initial_value=0)
         dataset = dataset.map(lambda x: x + var_1.read_value())
-      sess.run(var_1.initializer)
+      self.evaluate(var_1.initializer)
 
       iterator = dataset.make_initializable_iterator()
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
 
       with self.assertRaisesRegexp(
           errors.FailedPreconditionError,
@@ -506,7 +506,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
         sess.run(next_element)
@@ -543,7 +543,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       get_next_element = sess.make_callable(next_element)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
@@ -582,7 +582,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       get_next_element = sess.make_callable(next_element)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
@@ -620,7 +620,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       get_next_element = sess.make_callable(next_element)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
diff --git a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
index cb8cb9a77d..7087b4dd57 100644
--- a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
@@ -47,10 +47,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for _ in range(2):  # Run twice to test reinitialization.
-        sess.run(init_op)
+        self.evaluate(init_op)
         for _ in range(num_repeats):
           for elem in elem_sequence:
-            self.assertAllEqual(elem, sess.run(get_next))
+            self.assertAllEqual(elem, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -65,7 +65,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for _ in range(num_repeats):
         for elem in elem_sequence:
-          self.assertAllEqual(elem, sess.run(get_next))
+          self.assertAllEqual(elem, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -133,10 +133,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for _ in range(num_inner_repeats * num_outer_repeats):
         for elem in input_list:
-          val0, val1 = sess.run(get_next)
+          val0, val1 = self.evaluate(get_next)
           self.assertAllEqual(elem[0], val0)
           self.assertAllEqual(elem[1], val1)
       with self.assertRaises(errors.OutOfRangeError):
@@ -192,10 +192,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for elem in [0, 1]:
         for _ in range(num_parallel_iterators):
-          self.assertAllEqual(elem, sess.run(get_next))
+          self.assertAllEqual(elem, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -215,9 +215,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
       self.assertEqual(dtype, get_next.dtype)
 
       with self.cached_session() as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         for expected in [[1], [2], [3]]:
-          next_val = sess.run(get_next)
+          next_val = self.evaluate(get_next)
           self.assertEqual(dtype.as_numpy_dtype, next_val.dtype)
           self.assertAllEqual(expected, next_val)
         with self.assertRaises(errors.OutOfRangeError):
@@ -236,9 +236,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for expected in [b"foo", b"bar", b"baz"]:
-        next_val = sess.run(get_next)
+        next_val = self.evaluate(get_next)
         self.assertAllEqual(expected, next_val)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -257,12 +257,12 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual([1, 2, 3], sess.run(get_next))
-      self.assertAllEqual([4, 5, 6], sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
+      self.assertAllEqual([4, 5, 6], self.evaluate(get_next))
       with self.assertRaisesOpError("The expected type was int64"):
         sess.run(get_next)
-      self.assertAllEqual([7, 8, 9], sess.run(get_next))
+      self.assertAllEqual([7, 8, 9], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -280,12 +280,12 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual([1, 2, 3], sess.run(get_next))
-      self.assertAllEqual([4, 5, 6], sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
+      self.assertAllEqual([4, 5, 6], self.evaluate(get_next))
       with self.assertRaisesOpError(r"element of shape \(3,\) was expected"):
         sess.run(get_next)
-      self.assertAllEqual([11, 12, 13], sess.run(get_next))
+      self.assertAllEqual([11, 12, 13], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -304,16 +304,16 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertEqual((1, 2), sess.run(get_next))
-      self.assertEqual((3, 4), sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertEqual((1, 2), self.evaluate(get_next))
+      self.assertEqual((3, 4), self.evaluate(get_next))
       with self.assertRaisesOpError(
           r"The expected structure was \(tf\.int64, tf\.int64\)"):
         sess.run(get_next)
       with self.assertRaisesOpError(
           r"The expected structure was \(tf\.int64, tf\.int64\)"):
         sess.run(get_next)
-      self.assertEqual((9, 10), sess.run(get_next))
+      self.assertEqual((9, 10), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -329,9 +329,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(1, sess.run(get_next))
-      self.assertAllEqual([2, 3], sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual(1, self.evaluate(get_next))
+      self.assertAllEqual([2, 3], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -349,9 +349,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(0, sess.run(get_next))
-      self.assertAllEqual(1, sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual(0, self.evaluate(get_next))
+      self.assertAllEqual(1, self.evaluate(get_next))
 
   def testFromGeneratorDestructorCalled(self):
     # Use an `Event` to signal that the generator has been deleted.
@@ -378,9 +378,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with session.Session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(42, sess.run(get_next))
-      self.assertAllEqual(42, sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual(42, self.evaluate(get_next))
+      self.assertAllEqual(42, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
       # Test that `GeneratorWrapper` object is destroyed when the
@@ -407,10 +407,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       expected = [1, 2, 2, 3, 3, 3, 4, 4, 4, 4]
       for x in expected:
-        self.assertEqual(x, sess.run(get_next))
+        self.assertEqual(x, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -436,13 +436,13 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       expected = [(0, b"Hi!"),
                   (0, b"Hi!"), (1, b"Hi!"),
                   (0, b"Hi!"), (1, b"Hi!"), (2, b"Hi!"),
                   (0, b"Hi!"), (1, b"Hi!"), (2, b"Hi!"), (3, b"Hi!")]
       for x in expected:
-        self.assertEqual(x, sess.run(get_next))
+        self.assertEqual(x, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -470,9 +470,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(37, sess.run(get_next))
-      self.assertAllEqual(37, sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual(37, self.evaluate(get_next))
+      self.assertAllEqual(37, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
         self.assertTrue(event.is_set())
diff --git a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
index a0c6b37a6d..5ddb22285f 100644
--- a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
@@ -67,7 +67,7 @@ class FilterDatasetTest(test_base.DatasetTestBase):
         sess.run(init_op, feed_dict={count: count_val, modulus: modulus_val})
         for _ in range(count_val):
           for i in [x for x in range(7) if x**2 % modulus_val == 0]:
-            result = sess.run(get_next)
+            result = self.evaluate(get_next)
             for component, result_component in zip(components, result):
               self.assertAllEqual(component[i]**2, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -86,9 +86,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(get_next))
-      self.assertEqual(1, sess.run(get_next))
-      self.assertEqual(3, sess.run(get_next))
+      self.assertEqual(0, self.evaluate(get_next))
+      self.assertEqual(1, self.evaluate(get_next))
+      self.assertEqual(3, self.evaluate(get_next))
 
   def testFilterDict(self):
     iterator = (dataset_ops.Dataset.range(10)
@@ -100,10 +100,10 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
         if (i ** 2) % 2 == 0:
-          self.assertEqual(i * 2 + i ** 2, sess.run(get_next))
+          self.assertEqual(i * 2 + i**2, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -125,8 +125,8 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(input_data[0], sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual(input_data[0], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -148,9 +148,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(5):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         self.assertTrue(isinstance(actual, sparse_tensor.SparseTensorValue))
         self.assertSparseValuesEqual(actual, _map_fn(i * 2)[0])
       with self.assertRaises(errors.OutOfRangeError):
@@ -166,9 +166,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual((i, True), sess.run(get_next))
+        self.assertEqual((i, True), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -178,7 +178,7 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     iterators = [dataset.make_one_shot_iterator() for _ in range(10)]
     next_elements = [iterator.get_next() for iterator in iterators]
     with self.cached_session() as sess:
-      self.assertEqual([0 for _ in range(10)], sess.run(next_elements))
+      self.assertEqual([0 for _ in range(10)], self.evaluate(next_elements))
 
 
 class FilterDatasetBenchmark(test.Benchmark):
diff --git a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
index 68038f9cfc..02979fc2c4 100644
--- a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
@@ -45,10 +45,10 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in repeats:
         for _ in range(i):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -64,11 +64,11 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for row in repeats:
         for i in row:
           for _ in range(i):
-            self.assertEqual(i, sess.run(get_next))
+            self.assertEqual(i, self.evaluate(get_next))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -94,12 +94,12 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
       with session.Session(server.target) as sess2:
         for _ in range(3):
           sess = random.choice([sess1, sess2])
-          sess.run(init_op)
+          self.evaluate(init_op)
           for row in repeats:
             for i in row:
               for _ in range(i):
                 sess = random.choice([sess1, sess2])
-                self.assertEqual(i, sess.run(get_next))
+                self.assertEqual(i, self.evaluate(get_next))
 
         with self.assertRaises(errors.OutOfRangeError):
           sess = random.choice([sess1, sess2])
@@ -115,10 +115,10 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
         for _ in range(i ** 2):
-          self.assertEqual(i * 2, sess.run(get_next))
+          self.assertEqual(i * 2, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
   # pylint: enable=g-long-lambda
@@ -139,11 +139,11 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, sess.run(get_next))
+          self.assertAllEqual(expected, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py b/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
index b911c249ce..56434d6e4c 100644
--- a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
@@ -196,7 +196,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       for expected_element in _interleave(
           _repeat(input_values, count), cycle_length, block_length):
-        self.assertEqual(expected_element, sess.run(get_next))
+        self.assertEqual(expected_element, self.evaluate(get_next))
 
       for _ in range(2):
         with self.assertRaises(errors.OutOfRangeError):
@@ -231,7 +231,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           with self.assertRaises(errors.InvalidArgumentError):
             sess.run(get_next)
         else:
-          self.assertEqual(value, sess.run(get_next))
+          self.assertEqual(value, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -254,7 +254,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, sess.run(get_next))
+          self.assertAllEqual(expected, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -308,7 +308,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
       for element in elements:
         coordination_events[element].set()
-        self.assertEqual(element * element, sess.run(get_next))
+        self.assertEqual(element * element, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
index bf5fd781d6..cb38728f23 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
@@ -57,7 +57,7 @@ class IteratorClusterTest(test.TestCase):
 
     with session.Session(worker[0].target) as sess:
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(get_next_op)
+        self.evaluate(get_next_op)
 
   def _testRemoteIteratorHelper(self, device0, device1, target):
     with ops.device(device1):
@@ -134,12 +134,12 @@ class IteratorClusterTest(test.TestCase):
     get_next = iterator.get_next()
 
     with session.Session(worker[0].target) as sess:
-      sess.run(table.initializer)
-      sess.run(init_op)
-      self.assertAllEqual([0, 0, -1, 1, 2], sess.run(get_next))
+      self.evaluate(table.initializer)
+      self.evaluate(init_op)
+      self.assertAllEqual([0, 0, -1, 1, 2], self.evaluate(get_next))
 
     with session.Session(worker[0].target) as sess:
-      self.assertAllEqual([2, 0], sess.run(get_next))
+      self.assertAllEqual([2, 0], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -166,7 +166,7 @@ class IteratorClusterTest(test.TestCase):
     get_next = iterator.get_next()
 
     with session.Session(worker[0].target) as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for _ in range(3):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
index 490ca813dc..405d94d956 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
@@ -97,7 +97,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       for _ in range(14):
         for i in range(7):
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           for component, result_component in zip(components, result):
             self.assertAllEqual(component[i]**2, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -123,7 +123,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       for _ in range(14):
         for i in range(7):
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           for component, result_component in zip(components, result):
             self.assertAllEqual(component[i]**2, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -159,7 +159,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
         for _ in range(14):
           for i in range(7):
-            result = sess.run(get_next)
+            result = self.evaluate(get_next)
             for component, result_component in zip(components, result):
               self.assertAllEqual(component[i]**2, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -175,7 +175,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     config = config_pb2.ConfigProto(
         inter_op_parallelism_threads=1, use_per_session_threads=True)
     with session.Session(config=config) as sess:
-      self.assertAllEqual([1, 4, 9], sess.run(next_element))
+      self.assertAllEqual([1, 4, 9], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -254,15 +254,15 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       get_next = iterator.get_next()
 
       with session.Session(server.target) as sess:
-        sess.run(init_op)
-        results = sess.run(get_next)
+        self.evaluate(init_op)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
         # Re-initialize the iterator in the first session.
-        sess.run(init_op)
+        self.evaluate(init_op)
 
     with ops.Graph().as_default():
       # Re-define the iterator manually, without defining any of the
@@ -277,7 +277,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
       with session.Session(server.target) as sess:
         # Use the iterator without re-initializing in the second session.
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -317,20 +317,20 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         sess.run(get_next)
 
       # Initialize with one dataset.
-      sess.run(dataset_3_init_op)
-      self.assertAllEqual([1, 2, 3], sess.run(get_next))
+      self.evaluate(dataset_3_init_op)
+      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
       # Initialize with a different dataset.
-      sess.run(dataset_4_init_op)
-      self.assertAllEqual([4, 5, 6, 7], sess.run(get_next))
+      self.evaluate(dataset_4_init_op)
+      self.assertAllEqual([4, 5, 6, 7], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
       # Reinitialize with the first dataset.
-      sess.run(dataset_3_init_op)
-      self.assertAllEqual([1, 2, 3], sess.run(get_next))
+      self.evaluate(dataset_3_init_op)
+      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -348,7 +348,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
           g, output_types=dtypes.int64)
       sess.run(iterator.make_initializer(dataset_1))
       for expected in range(10):
-        self.assertEqual(expected, sess.run(next_element))
+        self.assertEqual(expected, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -356,7 +356,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
           g, output_types=dtypes.int64)
       sess.run(iterator.make_initializer(dataset_2))
       for expected in range(10):
-        self.assertEqual(expected, sess.run(next_element))
+        self.assertEqual(expected, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -679,10 +679,10 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       n = itr.get_next()
 
     with session.Session(s3.target, config=config) as sess:
-      sess.run(itr.initializer)
+      self.evaluate(itr.initializer)
       expected_values = worker_devices
       for expected in expected_values:
-        self.assertEqual((compat.as_bytes(expected),), sess.run(n))
+        self.assertEqual((compat.as_bytes(expected),), self.evaluate(n))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(n)
@@ -786,8 +786,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with ops.Graph().as_default() as g:
       init_op, _, save_op, _ = _build_range_dataset_graph()
       with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(save_op)
+        self.evaluate(init_op)
+        self.evaluate(save_op)
 
     # Attempt to restore the saved iterator into an IteratorResource of
     # incompatible type. An iterator of RangeDataset has output type int64,
@@ -798,7 +798,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       _, _, _, restore_op = _build_reader_dataset_graph()
       with self.session(graph=g) as sess:
         with self.assertRaises(errors.InvalidArgumentError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
 
   def testRepeatedGetNextWarning(self):
     iterator = dataset_ops.Dataset.range(10).make_one_shot_iterator()
@@ -949,7 +949,7 @@ class IteratorCheckpointingTest(test.TestCase):
         checkpoint.restore(checkpoint_management.latest_checkpoint(
             checkpoint_directory)).initialize_or_restore(sess)
         for j in range(2):
-          self.assertEqual(i * 2 + j, sess.run(get_next))
+          self.assertEqual(i * 2 + j, self.evaluate(get_next))
         checkpoint.save(file_prefix=checkpoint_prefix)
 
 
diff --git a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
index b58c1444da..ac6fbabcd5 100644
--- a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
@@ -102,7 +102,7 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
       all_produced_filenames = []
       for _ in range(3):
         produced_filenames = []
-        sess.run(itr.initializer)
+        self.evaluate(itr.initializer)
         try:
           while True:
             produced_filenames.append(sess.run(next_element))
diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
index 187b9da14c..8f7a19d7e1 100644
--- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
@@ -114,7 +114,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 14})
       for _ in range(14):
         for i in range(7):
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           for component, result_component in zip(components, result):
             self.assertAllEqual(component[i]**2, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -185,7 +185,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
             output_buffer_size: output_buffer_size_val})
         for _ in range(14):
           for i in range(7):
-            result = sess.run(get_next)
+            result = self.evaluate(get_next)
             for component, result_component in zip(components, result):
               self.assertAllEqual(component[i]**2, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -242,7 +242,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for _ in range(3):
         sess.run(get_next)
 
@@ -257,7 +257,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for _ in range(3):
         sess.run(get_next)
 
@@ -272,7 +272,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for _ in range(3):
         sess.run(get_next)
       # The 4th element is NaN, so `array_ops.check_numerics()` should fail.
@@ -293,7 +293,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for _ in range(3):
         sess.run(get_next)
       # The 4th element is NaN, so `array_ops.check_numerics()` should fail.
@@ -325,10 +325,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with ops.Graph().as_default() as g:
       captured_init_op, init_op, get_next = _build_graph()
       with self.session(graph=g) as sess:
-        sess.run(captured_init_op)
-        sess.run(init_op)
+        self.evaluate(captured_init_op)
+        self.evaluate(init_op)
         for i in range(10):
-          self.assertEqual(i * i, sess.run(get_next))
+          self.assertEqual(i * i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -353,8 +353,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(table.initializer)
-      sess.run(init_op)
+      self.evaluate(table.initializer)
+      self.evaluate(init_op)
       sess.run(get_next)
       sess.run(get_next)
       with self.assertRaises(errors.OutOfRangeError):
@@ -371,11 +371,11 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(enqueue_op)
-      sess.run(close_op)
-      sess.run(init_op)
+      self.evaluate(enqueue_op)
+      self.evaluate(close_op)
+      self.evaluate(init_op)
       for element in elements:
-        self.assertEqual(element, sess.run(get_next))
+        self.assertEqual(element, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -396,9 +396,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(enqueue_op)
-      sess.run(close_op)
-      sess.run(init_op)
+      self.evaluate(enqueue_op)
+      self.evaluate(close_op)
+      self.evaluate(init_op)
       for i in range(100):
         self.assertEqual(sorted([elements[i * 2], elements[i * 2 + 1]]),
                          sorted(sess.run(get_next)))
@@ -415,15 +415,15 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(counter_var.initializer)
-      sess.run(init_op)
+      self.evaluate(counter_var.initializer)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual(i, sess.run(counter_var))
-        self.assertEqual(i + 1, sess.run(get_next))
-      self.assertEqual(10, sess.run(counter_var))
+        self.assertEqual(i, self.evaluate(counter_var))
+        self.assertEqual(i + 1, self.evaluate(get_next))
+      self.assertEqual(10, self.evaluate(counter_var))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
-      self.assertEqual(10, sess.run(counter_var))
+      self.assertEqual(10, self.evaluate(counter_var))
 
   def testCaptureUninitializedVariableError(self):
     counter_var = variable_scope.get_variable(
@@ -435,7 +435,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaises(errors.NotFoundError):
         sess.run(get_next)
 
@@ -447,14 +447,14 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       random_values = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
           random_values.extend(sess.run(get_next))
       self.assertEqual(10, len(random_values))
       self.assertGreater(np.abs(np.diff(random_values)).max(), 1e-6)
-      sess.run(init_op)
+      self.evaluate(init_op)
       random_values_2 = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
@@ -473,8 +473,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      random_values = sess.run(get_next)
+      self.evaluate(init_op)
+      random_values = self.evaluate(get_next)
 
       # Assert that one of the next 99 batches yielded by the iterator is
       # different from the first.
@@ -500,15 +500,15 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(counter_var.initializer)
-      sess.run(init_op)
+      self.evaluate(counter_var.initializer)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual(i, sess.run(counter_var))
-        self.assertEqual(i, sess.run(get_next))
-      self.assertEqual(10, sess.run(counter_var))
+        self.assertEqual(i, self.evaluate(counter_var))
+        self.assertEqual(i, self.evaluate(get_next))
+      self.assertEqual(10, self.evaluate(counter_var))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
-      self.assertEqual(10, sess.run(counter_var))
+      self.assertEqual(10, self.evaluate(counter_var))
 
   def testMapDict(self):
     iterator = (dataset_ops.Dataset.range(10)
@@ -519,9 +519,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual(i * 2 + i ** 2, sess.run(get_next))
+        self.assertEqual(i * 2 + i**2, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -569,8 +569,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(row ** 2, sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual(row**2, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -611,7 +611,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       row = np.arange(6)
       for num in [2, 3, 4]:
         init_op, get_next = build_dataset(row, num)
-        sess.run(init_op)
+        self.evaluate(init_op)
         for i in range(6):
           self.assertEqual(
               (i // 2 if i % 2 else i * 2) if (num == 2 or num == 3) else i * 2,
@@ -652,7 +652,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       row = np.arange(6)
       for num in [2, 3, 4]:
         init_op, get_next = build_dataset(row, num)
-        sess.run(init_op)
+        self.evaluate(init_op)
         self.assertAllEqual(
             [x // 2 if (num == 2 or num == 3) else x * 2 for x in row],
             sess.run(get_next))
@@ -697,7 +697,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       self.assertAllEqual([(x // 2 if x % 2 else x * 2) if
                            (num == 2 or num == 3) else x * 2 for x in row],
                           sess.run(get_next))
@@ -735,7 +735,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       for buffer_size in [1, 10, 100, 1000]:
         sess.run(init_op, feed_dict={buffer_size_placeholder: buffer_size})
         for i in range(100):
-          self.assertEqual(i * i, sess.run(get_next))
+          self.assertEqual(i * i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -753,10 +753,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(init_op, feed_dict={buffer_size_placeholder: buffer_size})
         for i in range(event_will_be_set_after_consuming):
           self.assertFalse(ev.is_set())
-          self.assertEqual(i * i, sess.run(get_next))
+          self.assertEqual(i * i, self.evaluate(get_next))
         ev.wait()
         for i in range(event_will_be_set_after_consuming, 100):
-          self.assertEqual(i * i, sess.run(get_next))
+          self.assertEqual(i * i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -768,9 +768,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual((i, 37.0), sess.run(get_next))
+        self.assertEqual((i, 37.0), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -789,9 +789,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual((i, 37.0), sess.run(get_next))
+        self.assertEqual((i, 37.0), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -810,9 +810,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         self.assertIsInstance(actual, sparse_tensor.SparseTensorValue)
         self.assertSparseValuesEqual(actual, _sparse(i))
       with self.assertRaises(errors.OutOfRangeError):
@@ -837,9 +837,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         self.assertIsInstance(actual, sparse_tensor.SparseTensorValue)
         self.assertSparseValuesEqual(actual, _check(_sparse(i)).eval())
       with self.assertRaises(errors.OutOfRangeError):
@@ -861,9 +861,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(100):
-        self.assertEqual(i, sess.run(get_next))
+        self.assertEqual(i, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -875,9 +875,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
-        self.assertEqual((i, b"hello", 10), sess.run(get_next))
+        self.assertEqual((i, b"hello", 10), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -945,7 +945,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "BrokenConst"):
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
 
 # pylint: disable=g-long-lambda
   @parameterized.named_parameters(
@@ -972,7 +972,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      tids = sess.run(get_next)
+      tids = self.evaluate(get_next)
       self.assertTrue(all(tids[0] == tid for tid in tids))
 # pylint: enable=g-long-lambda
 
@@ -996,7 +996,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         expected = map_fn(*sess.run(self.structuredElement(structure)))
       else:
         expected = map_fn(sess.run(self.structuredElement(structure)))
-      self.assertEqual(expected, sess.run(get_next))
+      self.assertEqual(expected, self.evaluate(get_next))
 
   @parameterized.named_parameters(
       ("Sequential", None),
@@ -1011,7 +1011,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={captured_t: 42})
-      self.assertEqual(42, sess.run(get_next))
+      self.assertEqual(42, self.evaluate(get_next))
 
   @parameterized.named_parameters(
       ("1", 1, 1),
@@ -1030,7 +1030,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session(config=config) as sess:
       for i in range(num_elements):
         coordination_events[i].set()
-        self.assertEqual(i * i, sess.run(get_next))
+        self.assertEqual(i * i, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -1052,7 +1052,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
       for element in elements:
         coordination_events[element].set()
-        self.assertEqual(element * element, sess.run(get_next))
+        self.assertEqual(element * element, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 42ee1e2186..ea6828e575 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -40,7 +40,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
 
   def testBasic(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -50,10 +50,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -67,10 +67,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -85,12 +85,12 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 20, 4):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
-        self.assertEqual(i + 2, sess.run(elem_on_3))
-        self.assertEqual(i + 3, sess.run(elem_on_4))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i + 2, self.evaluate(elem_on_3))
+        self.assertEqual(i + 3, self.evaluate(elem_on_4))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -105,11 +105,11 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
-      self.assertEqual(8, sess.run(elem_on_1))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+      self.assertEqual(8, self.evaluate(elem_on_1))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -126,7 +126,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
         elem_on_1_has_value, elem_on_1_value = sess.run(
             [elem_on_1_has_value_t, elem_on_1_t])
@@ -140,8 +140,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
           [elem_on_1_has_value_t, elem_on_1_t])
       self.assertTrue(elem_on_1_has_value)
       self.assertEqual(8, elem_on_1_value)
-      self.assertFalse(sess.run(elem_on_1_has_value_t))
-      self.assertFalse(sess.run(elem_on_2_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_1_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_2_has_value_t))
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(elem_on_1_t)
       with self.assertRaises(errors.InvalidArgumentError):
@@ -155,11 +155,11 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i, self.evaluate(elem_on_1))
       for i in range(0, 10, 2):
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -192,10 +192,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -211,11 +211,11 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i, self.evaluate(elem_on_1))
       for i in range(0, 10, 2):
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -235,7 +235,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
         elem_on_1_has_value, elem_on_1_value = sess.run(
             [elem_on_1_has_value_t, elem_on_1_t])
@@ -249,8 +249,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
           [elem_on_1_has_value_t, elem_on_1_t])
       self.assertTrue(elem_on_1_has_value)
       self.assertEqual(8, elem_on_1_value)
-      self.assertFalse(sess.run(elem_on_1_has_value_t))
-      self.assertFalse(sess.run(elem_on_2_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_1_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_2_has_value_t))
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(elem_on_1_t)
       with self.assertRaises(errors.InvalidArgumentError):
@@ -272,10 +272,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
diff --git a/tensorflow/python/data/kernel_tests/optional_ops_test.py b/tensorflow/python/data/kernel_tests/optional_ops_test.py
index 604e3ad88e..0981ff9651 100644
--- a/tensorflow/python/data/kernel_tests/optional_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_ops_test.py
@@ -227,7 +227,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
 
       # For each element of the dataset, assert that the optional evaluates to
       # the expected value.
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for _ in range(3):
         elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t])
         self.assertTrue(elem_has_value)
@@ -236,7 +236,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       # After exhausting the iterator, `next_elem.has_value()` will evaluate to
       # false, and attempting to get the value will fail.
       for _ in range(2):
-        self.assertFalse(sess.run(elem_has_value_t))
+        self.assertFalse(self.evaluate(elem_has_value_t))
         with self.assertRaises(errors.InvalidArgumentError):
           sess.run(elem_value_t)
 
diff --git a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
index 76e2697b29..af326ec210 100644
--- a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
@@ -40,7 +40,7 @@ class PrefetchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(init_op, feed_dict={buffer_size_t: buffer_size})
       for m in range(10):
-        self.assertEqual(m, sess.run(get_next))
+        self.assertEqual(m, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
index 9fc79707d0..fcb025c8b8 100644
--- a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
@@ -124,19 +124,19 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
+        self.evaluate(init_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -144,14 +144,14 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-        sess.run(restore_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -175,14 +175,14 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for _ in range(break_epoch):
           for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
+            self.assertEqual(i, self.evaluate(get_next))
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       # Create an empty IteratorResource and restore the Iterator into it.
@@ -193,12 +193,12 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
       restore_op = self._restore_op(iterator._iterator_resource)
       get_next = iterator.get_next()
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         for _ in range(break_epoch + 1, num_epochs):
           for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
+            self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -221,20 +221,20 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       # Intentionally build a graph with a different value for stop to make sure
       # the original dataset graph is actually getting loaded.
       init_op, get_next, _, restore_op = _build_graph(start, stop_1)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -259,19 +259,19 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
+        self.evaluate(init_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -294,27 +294,27 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point1):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for i in range(break_point1, break_point2):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     break_point2 = 7
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for i in range(break_point2, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -338,28 +338,28 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
       init_op, get_next, save_op, restore_op = _build_graph(
           start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for _ in range(break_epoch - 1):
           for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
+            self.assertEqual(i, self.evaluate(get_next))
         for i in range(start, break_range):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for i in range(break_range, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         for _ in range(break_epoch, num_epochs):
           for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
+            self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -381,23 +381,23 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
       init_op, get_next, save_op, restore_op = _build_graph(
           start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for _ in range(num_epochs):
           for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
+            self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
-        sess.run(save_op)
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
index 4fef4f30bf..e26381e902 100644
--- a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
@@ -107,7 +107,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[0]],
                               num_epochs: 1})
       for i in range(5):
-        self.assertEqual(self._lineText(0, i), sess.run(get_next))
+        self.assertEqual(self._lineText(0, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -116,7 +116,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[1]],
                               num_epochs: 1})
       for i in range(5):
-        self.assertEqual(self._lineText(1, i), sess.run(get_next))
+        self.assertEqual(self._lineText(1, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -124,7 +124,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1})
       for j in range(2):
         for i in range(5):
-          self.assertEqual(self._lineText(j, i), sess.run(get_next))
+          self.assertEqual(self._lineText(j, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -133,7 +133,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
       for _ in range(10):
         for j in range(2):
           for i in range(5):
-            self.assertEqual(self._lineText(j, i), sess.run(get_next))
+            self.assertEqual(self._lineText(j, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -267,7 +267,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[0]],
                               num_epochs: 1})
       for i in range(self._num_records):
-        self.assertEqual(self._record(0, i), sess.run(get_next))
+        self.assertEqual(self._record(0, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -276,7 +276,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[1]],
                               num_epochs: 1})
       for i in range(self._num_records):
-        self.assertEqual(self._record(1, i), sess.run(get_next))
+        self.assertEqual(self._record(1, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -284,7 +284,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1})
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertEqual(self._record(j, i), sess.run(get_next))
+          self.assertEqual(self._record(j, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -293,7 +293,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       for _ in range(10):
         for j in range(self._num_files):
           for i in range(self._num_records):
-            self.assertEqual(self._record(j, i), sess.run(get_next))
+            self.assertEqual(self._record(j, i), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -405,19 +405,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                sess.run(save_op)
+                self.evaluate(save_op)
                 break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
             else:
               continue
             break
@@ -426,13 +426,13 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
+            self.evaluate(get_next_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -441,9 +441,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
 
   def testInitThenRestore(self):
     # Note: Calling init_op before restore_op is redundant. This test just makes
@@ -458,19 +458,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                sess.run(save_op)
+                self.evaluate(save_op)
                 break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
             else:
               continue
             break
@@ -479,14 +479,14 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
+            self.evaluate(get_next_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
+        self.evaluate(init_op)
+        self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -495,9 +495,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
 
   def testRestoreInModifiedGraph(self):
     num_epochs = 10
@@ -510,19 +510,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                sess.run(save_op)
+                self.evaluate(save_op)
                 break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
             else:
               continue
             break
@@ -531,13 +531,13 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
+            self.evaluate(get_next_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs_1)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -546,9 +546,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
 
   def testRestoreWithoutBuildingDatasetGraph(self):
     num_epochs = 10
@@ -560,19 +560,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                sess.run(save_op)
+                self.evaluate(save_op)
                 break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
             else:
               continue
             break
@@ -581,12 +581,12 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
+            self.evaluate(get_next_op)
 
     with ops.Graph().as_default() as g:
       restore_op, get_next_op = self._restore_iterator()
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -595,9 +595,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
 
   def testRestoreUnusedIterator(self):
     num_epochs = 10
@@ -605,22 +605,22 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         # Save unused iterator.
-        sess.run(save_op)
+        self.evaluate(save_op)
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         for _ in range(num_epochs * self._num_files * self._num_records):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
 
   def testRestoreExhaustedIterator(self):
     num_epochs = 10
@@ -629,26 +629,26 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
+          self.evaluate(restore_op)
         for _ in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
+              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-        sess.run(save_op)
+          self.evaluate(get_next_op)
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        sess.run(restore_op)
+        self.evaluate(restore_op)
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+          self.evaluate(get_next_op)
 
 
 class TFRecordDatasetTest(test_base.DatasetTestBase):
@@ -807,7 +807,7 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(next_element))
+          self.assertAllEqual(self._record(j, i), self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -819,7 +819,7 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(next_element))
+          self.assertAllEqual(self._record(j, i), self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py b/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
index 11e07300b9..d7f3988b1a 100644
--- a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
@@ -36,7 +36,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.range(1, i + 1)
       result = ds.reduce(np.int64(0), lambda x, y: x + y)
       with self.cached_session() as sess:
-        self.assertEqual(((i + 1) * i) // 2, sess.run(result))
+        self.assertEqual(((i + 1) * i) // 2, self.evaluate(result))
 
   def testSumTuple(self):
 
@@ -49,7 +49,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.zip((ds, ds))
       result = ds.reduce(np.int64(0), reduce_fn)
       with self.cached_session() as sess:
-        self.assertEqual(((i + 1) * i), sess.run(result))
+        self.assertEqual(((i + 1) * i), self.evaluate(result))
 
   def testSumAndCount(self):
 
@@ -61,7 +61,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.range(1, i + 1)
       result = ds.reduce((np.int64(0), np.int64(0)), reduce_fn)
       with self.cached_session() as sess:
-        s, c = sess.run(result)
+        s, c = self.evaluate(result)
         self.assertEqual(((i + 1) * i) // 2, s)
         self.assertEqual(i, c)
 
@@ -93,7 +93,8 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.from_tensors(make_sparse_fn(i+1))
       result = ds.reduce(make_sparse_fn(0), reduce_fn)
       with self.cached_session() as sess:
-        self.assertSparseValuesEqual(make_sparse_fn(i+1), sess.run(result))
+        self.assertSparseValuesEqual(
+            make_sparse_fn(i + 1), self.evaluate(result))
 
   def testNested(self):
 
@@ -116,7 +117,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.range(1, i + 1).map(map_fn)
       result = ds.reduce(map_fn(0), reduce_fn)
       with self.cached_session() as sess:
-        result = sess.run(result)
+        result = self.evaluate(result)
         self.assertEqual(((i + 1) * i) // 2, result["dense"])
         self.assertSparseValuesEqual(make_sparse_fn(i), result["sparse"])
 
diff --git a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
index e86356dee7..946aa01f73 100644
--- a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
@@ -49,7 +49,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Test a finite repetition.
       sess.run(init_op, feed_dict={count_placeholder: 3})
       for _ in range(3):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
 
@@ -59,7 +59,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Test a different finite repetition.
       sess.run(init_op, feed_dict={count_placeholder: 7})
       for _ in range(7):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -75,7 +75,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # actually is infinite.
       sess.run(init_op, feed_dict={count_placeholder: -1})
       for _ in range(17):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
 
@@ -95,7 +95,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Take fewer than input size
       sess.run(init_op, feed_dict={count_placeholder: 4})
       for i in range(4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
 
       with self.assertRaises(errors.OutOfRangeError):
@@ -104,7 +104,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Take more than input size
       sess.run(init_op, feed_dict={count_placeholder: 25})
       for i in range(10):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
 
       with self.assertRaises(errors.OutOfRangeError):
@@ -113,7 +113,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Take all of input
       sess.run(init_op, feed_dict={count_placeholder: -1})
       for i in range(10):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
 
       with self.assertRaises(errors.OutOfRangeError):
@@ -142,7 +142,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # the first 4 elements and then read the rest.
       sess.run(init_op, feed_dict={count_placeholder: 4})
       for i in range(4, 10):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -165,7 +165,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Skip nothing
       sess.run(init_op, feed_dict={count_placeholder: 0})
       for i in range(0, 10):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -187,7 +187,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       sess.run(init_op, feed_dict={inner_count: 7, outer_count: 14})
       for _ in range(7 * 14):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -201,7 +201,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
index cad28f860e..990f4f212b 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
@@ -66,7 +66,7 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       # First run without shuffling to collect the "ground truth".
-      sess.run(init_fifo_op)
+      self.evaluate(init_fifo_op)
       unshuffled_elements = []
       for _ in range(20):
         unshuffled_elements.append(sess.run(get_next))
@@ -159,7 +159,7 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={seed_placeholder: 0})
       for elem in elems:
-        self.assertEqual(elem, sess.run(get_next))
+        self.assertEqual(elem, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -188,9 +188,9 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      initial_permutation = sess.run(next_element)
-      self.assertAllEqual(initial_permutation, sess.run(next_element))
-      self.assertAllEqual(initial_permutation, sess.run(next_element))
+      initial_permutation = self.evaluate(next_element)
+      self.assertAllEqual(initial_permutation, self.evaluate(next_element))
+      self.assertAllEqual(initial_permutation, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -261,7 +261,7 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.session(graph=g) as sess:
         for iterator in iterators:
           if initializable:
-            sess.run(iterator.initializer)
+            self.evaluate(iterator.initializer)
           next_element = iterator.get_next()
           run_results = []
           for _ in range(300):
diff --git a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
index 9d06781094..35adcddfe7 100644
--- a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
@@ -102,7 +102,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       num_full_batches = max(
           0, (count * 7 - ((size - 1) * stride + 1)) // shift + 1)
       for i in range(num_full_batches):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(size):
             self.assertAllEqual(component[(i * shift + j * stride) % 7]**2,
@@ -111,7 +111,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         num_partial_batches = (count * 7) // shift + (
             (count * 7) % shift > 0) - num_full_batches
         for i in range(num_partial_batches):
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           for component, result_component in zip(components, result):
             remaining = (count * 7) - ((num_full_batches + i) * shift)
             num_elements = remaining // stride + ((remaining % stride) > 0)
@@ -164,10 +164,10 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       num_batches = (10 - 5) // 3 + 1
       for i in range(num_batches):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4],
@@ -193,10 +193,10 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       num_batches = (10 - 5) // 3 + 1
       for i in range(num_batches):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected_indices = []
         expected_values = []
         for j in range(5):
@@ -227,9 +227,9 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       # Slide: 1st batch.
-      actual = sess.run(get_next)
+      actual = self.evaluate(get_next)
       expected = sparse_tensor.SparseTensorValue(
           indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
                    [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
@@ -239,7 +239,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertTrue(sparse_tensor.is_sparse(actual))
       self.assertSparseValuesEqual(actual, expected)
       # Slide: 2nd batch.
-      actual = sess.run(get_next)
+      actual = self.evaluate(get_next)
       expected = sparse_tensor.SparseTensorValue(
           indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
                    [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
@@ -265,7 +265,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           r"Cannot batch tensors with different shapes in component 0. "
@@ -281,8 +281,8 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = dataset.make_one_shot_iterator().get_next()
 
     with self.cached_session() as sess:
-      self.assertAllEqual(np.float32([1., 2.]), sess.run(get_next))
-      self.assertAllEqual(np.float32([2., 3.]), sess.run(get_next))
+      self.assertAllEqual(np.float32([1., 2.]), self.evaluate(get_next))
+      self.assertAllEqual(np.float32([2., 3.]), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
index 9d76387a34..b60ec4ecce 100644
--- a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
@@ -55,7 +55,7 @@ class ZipDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={ph: value for ph, value in zip(
           component_placeholders, equal_length_components)})
       for i in range(4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(
             equal_length_components, results):
           self.assertAllEqual(component[i], result_component)
@@ -66,7 +66,7 @@ class ZipDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={ph: value for ph, value in zip(
           component_placeholders, variable_length_components)})
       for i in range(2):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         for component, result_component in zip(
             variable_length_components, results):
           self.assertAllEqual(component[i], result_component)
@@ -103,7 +103,7 @@ class ZipDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={ph: value for ph, value in zip(
           component_placeholders, equal_length_components)})
       for i in range(4):
-        result1, (result2, result3) = sess.run(get_next)
+        result1, (result2, result3) = self.evaluate(get_next)
         self.assertAllEqual(equal_length_components[0][i], result1)
         self.assertAllEqual(equal_length_components[1][i], result2)
         self.assertAllEqual(equal_length_components[2][i], result3)
diff --git a/tensorflow/python/data/util/convert_test.py b/tensorflow/python/data/util/convert_test.py
index 89c3afb296..4a5b730381 100644
--- a/tensorflow/python/data/util/convert_test.py
+++ b/tensorflow/python/data/util/convert_test.py
@@ -31,24 +31,24 @@ class ConvertTest(test.TestCase):
   def testInteger(self):
     resp = convert.optional_param_to_tensor("foo", 3)
     with self.cached_session() as sess:
-      self.assertEqual(3, sess.run(resp))
+      self.assertEqual(3, self.evaluate(resp))
 
   def testIntegerDefault(self):
     resp = convert.optional_param_to_tensor("foo", None)
     with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(resp))
+      self.assertEqual(0, self.evaluate(resp))
 
   def testStringDefault(self):
     resp = convert.optional_param_to_tensor("bar", None, "default",
                                             dtypes.string)
     with self.cached_session() as sess:
-      self.assertEqual(compat.as_bytes("default"), sess.run(resp))
+      self.assertEqual(compat.as_bytes("default"), self.evaluate(resp))
 
   def testString(self):
     resp = convert.optional_param_to_tensor("bar", "value", "default",
                                             dtypes.string)
     with self.cached_session() as sess:
-      self.assertEqual(compat.as_bytes("value"), sess.run(resp))
+      self.assertEqual(compat.as_bytes("value"), self.evaluate(resp))
 
   def testPartialShapeToTensorKnownDimension(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index f197a9e4dc..5aa7d1bb4c 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -1583,7 +1583,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
       x = variables.VariableV1([1, 3, 3, 7], name="x")
       _, idx = array_ops.unique(x, name="x_unique")
       idx_times_two = math_ops.multiply(idx, 2, name="idx_times_two")
-      sess.run(x.initializer)
+      self.evaluate(x.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(
diff --git a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
index 1f67f8a0d4..34030c0adc 100644
--- a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
+++ b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
@@ -126,8 +126,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       u = variables.Variable([12.0], name="u")
       v = variables.Variable([30.0], name="v")
       w = math_ops.add(u, v, name="w")
-      sess.run(u.initializer)
-      sess.run(v.initializer)
+      self.evaluate(u.initializer)
+      self.evaluate(v.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, w, expected_output=[42.0])
@@ -139,7 +139,7 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
         b = math_ops.add(a, a, name="b")
       with ops.control_dependencies([a, b]):
         c = math_ops.multiply(b, b, name="c")
-      sess.run(a.initializer)
+      self.evaluate(a.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, c, expected_output=400.0)
@@ -150,8 +150,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       y = variables.Variable(20.0, name="y")
       cond = control_flow_ops.cond(
           x > y, lambda: math_ops.add(x, 1), lambda: math_ops.add(y, 1))
-      sess.run(x.initializer)
-      sess.run(y.initializer)
+      self.evaluate(x.initializer)
+      self.evaluate(y.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, cond, expected_output=21.0)
@@ -173,8 +173,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       toy_loss = x * (u - v)
       train_op = gradient_descent.GradientDescentOptimizer(
           learning_rate=0.1).minimize(toy_loss, name="train_op")
-      sess.run(u.initializer)
-      sess.run(v.initializer)
+      self.evaluate(u.initializer)
+      self.evaluate(v.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(sess, train_op)
 
diff --git a/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py b/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
index 74498c8ea3..b78c3d16d4 100644
--- a/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
+++ b/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
@@ -131,8 +131,8 @@ class DistributedSessionDebugTest(test_util.TensorFlowTestCase):
     with session.Session(
         config=self.session_config, graph=graph,
         target=self.server_target) as sess:
-      sess.run(self.a.initializer)
-      sess.run(self.b.initializer)
+      self.evaluate(self.a.initializer)
+      self.evaluate(self.b.initializer)
 
       run_options = config_pb2.RunOptions()
       debug_utils.watch_graph(
@@ -198,8 +198,8 @@ class DistributedSessionDebugTest(test_util.TensorFlowTestCase):
     with session.Session(
         config=self.session_config, graph=graph,
         target=self.server_target) as sess:
-      sess.run(self.a.initializer)
-      sess.run(self.b.initializer)
+      self.evaluate(self.a.initializer)
+      self.evaluate(self.b.initializer)
 
       def watch_fn(feeds, fetch_keys):
         del feeds, fetch_keys
diff --git a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
index b0dc25851c..8eef45392f 100644
--- a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
+++ b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
@@ -67,7 +67,7 @@ class SessionDebugMultiGPUTest(test_util.TensorFlowTestCase):
         u1 = math_ops.multiply(v, v, name="u1")
       w = math_ops.subtract(u1, u0, name="w")
 
-      sess.run(v.initializer)
+      self.evaluate(v.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(run_options, sess.graph,
diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py
index 4a8d4eaa99..a16d68329a 100644
--- a/tensorflow/python/debug/lib/source_utils_test.py
+++ b/tensorflow/python/debug/lib/source_utils_test.py
@@ -109,8 +109,8 @@ class SourceHelperTest(test_util.TensorFlowTestCase):
       self.w = math_ops.matmul(self.u, self.v, name="w")
       self.w_line_number = line_number_above()
 
-      sess.run(self.u.initializer)
-      sess.run(self.v.initializer)
+      self.evaluate(self.u.initializer)
+      self.evaluate(self.v.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(
diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py
index 5d336648ce..bf81ac0455 100644
--- a/tensorflow/python/distribute/distribute_coordinator_test.py
+++ b/tensorflow/python/distribute/distribute_coordinator_test.py
@@ -235,7 +235,7 @@ class DistributeCoordinatorTestBase(test.TestCase):
         result = math_ops.add_n(xs)
 
       variables.global_variables_initializer().run()
-      result_value = sess.run(result)
+      result_value = self.evaluate(result)
     self.assertEqual(result_value, expected)
     if result_value == expected:
       self._result_correct += 1
@@ -294,7 +294,7 @@ class DistributeCoordinatorTestBase(test.TestCase):
           if len(uninit_vars) == 0:
             break
 
-      sess.run(train_op)
+      self.evaluate(train_op)
 
       # Synchronize workers after one step to make sure they all have finished
       # training.
@@ -327,7 +327,7 @@ class DistributeCoordinatorTestBase(test.TestCase):
 
     # The monitored session will run init or ready ops.
     with monitored_session.MonitoredSession() as sess:
-      sess.run(train_op)
+      self.evaluate(train_op)
 
       # Synchronize workers after one step to make sure they all have finished
       # training.
diff --git a/tensorflow/python/distribute/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py
index cbb93e8995..54f7c5d012 100644
--- a/tensorflow/python/distribute/input_ops_test.py
+++ b/tensorflow/python/distribute/input_ops_test.py
@@ -92,7 +92,7 @@ class AutoShardDatasetTest(test.TestCase):
     with self.cached_session() as sess:
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(record_fn(r, f), sess.run(next_element))
+          self.assertAllEqual(record_fn(r, f), self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -205,10 +205,11 @@ class AutoShardDatasetTest(test.TestCase):
     with self.cached_session() as sess:
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(self._record(r, f), sess.run(next_element))
+          self.assertAllEqual(self._record(r, f), self.evaluate(next_element))
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(self._text_line(r, f), sess.run(next_element))
+          self.assertAllEqual(
+              self._text_line(r, f), self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index f0f71a219e..54991344b7 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -149,9 +149,9 @@ class DefFunctionTest(test.TestCase):
 
       result = fn(3.0)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual(sess.run(state[0]), 2.0)
-      self.assertAllEqual(sess.run(result), 6.0)
+      self.assertAllEqual(self.evaluate(result), 6.0)
 
   def testLegacyGraphModeVariablesNonTrivialInitializer(self):
     with ops.Graph().as_default(), self.test_session() as sess:
@@ -168,9 +168,9 @@ class DefFunctionTest(test.TestCase):
 
       result = fn(3.0)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual(sess.run(state[0]), 6.0)
-      self.assertAllEqual(sess.run(result), 18.0)
+      self.assertAllEqual(self.evaluate(result), 18.0)
 
   def testLegacyGraphModeInputDependentInitializerFails(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py
index d4f8aaa7e3..1ba596573f 100644
--- a/tensorflow/python/eager/function_gradients_test.py
+++ b/tensorflow/python/eager/function_gradients_test.py
@@ -78,7 +78,7 @@ class FunctionGradientsTest(test.TestCase, parameterized.TestCase):
       c = constant_op.constant([[2.]])
       f_c = f(c)
       g, = gradients_impl.gradients(f_c, c)
-      self.assertAllEqual(sess.run(g).values, [[1.0]])
+      self.assertAllEqual(self.evaluate(g).values, [[1.0]])
 
   def testNoSymGradNestedDefun(self):
 
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 98040dc68c..fe5f0ef57f 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -564,7 +564,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       variables.global_variables_initializer().run()
       call = def_function.function(o.call)
       op = call()
-      self.assertAllEqual(sess.run(op), 2.0)
+      self.assertAllEqual(self.evaluate(op), 2.0)
 
   def testGraphModeManyFunctions(self):
     with ops.Graph().as_default(), self.cached_session():
@@ -1733,7 +1733,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
       function.register(cpu_boost, x)
       y = gpu_boost(x)
-      y_value = sess.run(y)
+      y_value = self.evaluate(y)
 
       if test.is_gpu_available():
         self.assertEqual(y_value, 5.0)
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 73df7d9eb8..e069b96bb9 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -1026,7 +1026,7 @@ class CrossedColumnTest(test.TestCase):
     outputs = _transform_features(features, [price_cross_wire])
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
-      output_val = sess.run(output)
+      output_val = self.evaluate(output)
       self.assertAllEqual(
           [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
       for val in output_val.values:
@@ -1880,7 +1880,8 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
@@ -2514,7 +2515,8 @@ class _LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 78321ee096..251ede925c 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -1190,7 +1190,7 @@ class CrossedColumnTest(test.TestCase):
     outputs = fc._transform_features(features, [price_cross_wire], None)
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
-      output_val = sess.run(output)
+      output_val = self.evaluate(output)
       self.assertAllEqual(
           [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
       for val in output_val.values:
@@ -2091,7 +2091,8 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]],
+                          self.evaluate(net))
 
       coord.request_stop()
       coord.join(threads)
@@ -2127,7 +2128,8 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column_v2('price')
@@ -2849,7 +2851,8 @@ class OldLinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column_v2('price')
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 971219d5b0..90deb9765f 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -102,7 +102,7 @@ class FunctionTest(test.TestCase):
       call = MyIdentityFunc([18.0])
       self.assertEqual("MyIdentity", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([18.0], sess.run(call))
+        self.assertAllEqual([18.0], self.evaluate(call))
 
   def testIdentityImplicitDeref(self):
 
@@ -116,8 +116,8 @@ class FunctionTest(test.TestCase):
       self.assertEqual("MyIdentity", call.op.name)
       for cfg in _OptimizerOptions():
         with session.Session(config=cfg) as sess:
-          sess.run(var.initializer)
-          self.assertAllEqual([18.0], sess.run(call))
+          self.evaluate(var.initializer)
+          self.assertAllEqual([18.0], self.evaluate(call))
 
   def testIdentityOutputName(self):
 
@@ -130,7 +130,7 @@ class FunctionTest(test.TestCase):
       call = MyIdentityFunc([18.0])
       self.assertEqual("MyIdentity", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([18.0], sess.run(call))
+        self.assertAllEqual([18.0], self.evaluate(call))
 
   def testTooManyOutputNames(self):
 
@@ -158,7 +158,7 @@ class FunctionTest(test.TestCase):
       call = APlus2B([1.0], [2.0])
       self.assertEqual("APlus2B", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([5.0], sess.run(call))
+        self.assertAllEqual([5.0], self.evaluate(call))
 
   def testFunctionWithNoOutput(self):
 
@@ -187,7 +187,7 @@ class FunctionTest(test.TestCase):
       call = APlus2B([1.0], [2.0])
       self.assertEqual("APlus2B", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([5.0], sess.run(call))
+        self.assertAllEqual([5.0], self.evaluate(call))
 
   def testDefineFunctionDuplicateOutputs(self):
 
@@ -224,8 +224,8 @@ class FunctionTest(test.TestCase):
       call_g = XSquarePlusOneGrad([2.0], [0.1])
 
       with session.Session() as sess:
-        self.assertAllClose([5.0], sess.run(call_f))
-        self.assertAllClose([0.4], sess.run(call_g))
+        self.assertAllClose([5.0], self.evaluate(call_f))
+        self.assertAllClose([0.4], self.evaluate(call_g))
 
   def testTanhSymGrad(self):
 
@@ -387,7 +387,7 @@ class FunctionTest(test.TestCase):
       call = AConstant()
       self.assertEqual("AConstant", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([42], sess.run(call))
+        self.assertAllEqual([42], self.evaluate(call))
 
   def testDefineFunctionNames(self):
 
@@ -468,7 +468,7 @@ class FunctionTest(test.TestCase):
 
       loop = control_flow_ops.while_loop(lambda x: x < 1e5, Body, [1.0])
 
-      ans = sess.run(loop)
+      ans = self.evaluate(loop)
       self.assertAllClose(ans, 131072.)
 
   def testControlFlowStrictness(self):
@@ -650,8 +650,8 @@ class FunctionTest(test.TestCase):
       # pylint: enable=unexpected-keyword-arg
       self.assertEqual("next", call2.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([1], sess.run(call1))
-        self.assertAllEqual([0], sess.run(call2))
+        self.assertAllEqual([1], self.evaluate(call1))
+        self.assertAllEqual([0], self.evaluate(call2))
 
   def testNestedFunction(self):
 
@@ -794,7 +794,7 @@ class FunctionTest(test.TestCase):
       y = Foo()
 
     with self.session(graph=g) as sess:
-      self.assertEqual(sess.run(y), 10)
+      self.assertEqual(self.evaluate(y), 10)
 
   def testCaptureInCond(self):
     g = ops.Graph()
@@ -809,8 +809,8 @@ class FunctionTest(test.TestCase):
       z = Foo(False)
 
     with self.session(graph=g) as sess:
-      self.assertEqual(sess.run(y), 1)
-      self.assertEqual(sess.run(z), 2)
+      self.assertEqual(self.evaluate(y), 1)
+      self.assertEqual(self.evaluate(z), 2)
 
   def testStableName(self):
 
@@ -900,7 +900,7 @@ class FunctionTest(test.TestCase):
     self.assertEqual(global_vars[0].name, "linear/w:0")
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       output_val = sess.run(
           output_op, feed_dict={input_op: np.random.rand(32, 100)})
       self.assertEqual(output_val.shape, (32, 100))
@@ -928,7 +928,7 @@ class FunctionTest(test.TestCase):
     self.assertEqual(global_vars[0].name, "vs1/var:0")
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       out1, out2 = sess.run(
           [out1_op, out2_op], feed_dict={input_op: np.linspace(1, 10, 10)})
       self.assertAllEqual(out1, np.linspace(2, 11, 10))
@@ -991,8 +991,8 @@ class FunctionTest(test.TestCase):
     result_2 = Bar(constant_op.constant(100, dtype=dtypes.int64))
 
     with session.Session() as sess:
-      self.assertEqual(4.0, sess.run(result_1))
-      self.assertEqual(100, sess.run(result_2))
+      self.assertEqual(4.0, self.evaluate(result_1))
+      self.assertEqual(100, self.evaluate(result_2))
       self.assertEqual((4.0, 100), sess.run((result_1, result_2)))
 
   def testStatefulFunction(self):
@@ -1052,8 +1052,8 @@ class FunctionTest(test.TestCase):
     for config in _OptimizerOptions():
       config.device_count["CPU"] = 2
       with session.Session(config=config) as sess:
-        self.assertEqual(42.0, sess.run(f_0))
-        self.assertEqual(44.0, sess.run(f_1))
+        self.assertEqual(42.0, self.evaluate(f_0))
+        self.assertEqual(44.0, self.evaluate(f_1))
         self.assertEqual((42.0, 44.0), sess.run((f_0, f_1)))
 
   def testGuaranteedConstsAreCaptured(self):
@@ -1076,7 +1076,7 @@ class FunctionTest(test.TestCase):
       return output
 
     with self.session(use_gpu=False) as sess:
-      sess.run(var.initializer)
+      self.evaluate(var.initializer)
       _ = sess.run(CapturesGuaranteedConst(), {also_not_const: 1.0})
 
   def testSameFunctionDifferentGrads(self):
@@ -1651,8 +1651,8 @@ class ModuleFunctionTest(test.TestCase):
       y = LinearWithCApi(a, b, c)
       z = Linear2WithCApi(a, b, c, d, e)
       with session.Session() as sess:
-        self.assertAllEqual([[1]], sess.run(y))
-        self.assertAllEqual([[5]], sess.run(z))
+        self.assertAllEqual([[1]], self.evaluate(y))
+        self.assertAllEqual([[5]], self.evaluate(z))
 
 
 class VariableHoistingTest(test.TestCase):
@@ -1704,7 +1704,7 @@ class VariableHoistingTest(test.TestCase):
     self.assertEqual("Foo/b", b.op.name)
 
     with self.session(graph=g) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       w, b, x, y0, loss, dw, db = sess.run([w, b, x, y0, loss, dw, db])
 
     self.assertAllEqual(w.shape, (64, 64))
diff --git a/tensorflow/python/framework/graph_util_test.py b/tensorflow/python/framework/graph_util_test.py
index 563a177dd0..7a9f2e8d86 100644
--- a/tensorflow/python/framework/graph_util_test.py
+++ b/tensorflow/python/framework/graph_util_test.py
@@ -211,7 +211,7 @@ class DeviceFunctionsTest(test.TestCase):
       with session.Session() as sess:
         init = variables.variables_initializer([variable_node])
         sess.run(init)
-        output = sess.run(output_node)
+        output = self.evaluate(output_node)
         self.assertNear(4.0, output, 0.00001)
         variable_graph_def = sess.graph.as_graph_def()
 
@@ -242,8 +242,8 @@ class DeviceFunctionsTest(test.TestCase):
         output_node = math_ops_lib.multiply(
             variable_node, 2.0, name="output_node")
         with session.Session() as sess:
-          sess.run(variable_node.initializer)
-          output = sess.run(output_node)
+          self.evaluate(variable_node.initializer)
+          output = self.evaluate(output_node)
           self.assertNear(2.0, output, 0.00001)
           variable_graph_def = sess.graph.as_graph_def()
           # First get the constant_graph_def when variable_names_whitelist is
@@ -256,7 +256,7 @@ class DeviceFunctionsTest(test.TestCase):
 
           # Then initialize the unused variable, and get another
           # constant_graph_def when variable_names_whitelist is not set.
-          sess.run(another_variable.initializer)
+          self.evaluate(another_variable.initializer)
           constant_graph_def_without_variable_whitelist = (
               graph_util.convert_variables_to_constants(
                   sess, variable_graph_def, ["output_node"]))
@@ -295,7 +295,7 @@ class DeviceFunctionsTest(test.TestCase):
             ["Variable", "VariableV2", "VarHandleOp", "ReadVariableOp"])
       with session.Session() as sess:
         output_node = sess.graph.get_tensor_by_name("output_node:0")
-        output = sess.run(output_node)
+        output = self.evaluate(output_node)
         self.assertNear(2.0, output, 0.00001)
 
   def create_node_def(self, op, name, inputs):
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index fc7367649e..a57f0b3654 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -398,10 +398,10 @@ class ImportGraphDefTest(test.TestCase):
       # TODO(b/76173421): make this work (currently DCHECKS)
       # with self.cached_session() as sess:
       #   sess.run(imported_init)
-      #   self.assertEqual(sess.run(imported_var), 1.0)
-      #   self.assertEqual(sess.run(imported_assign), 2.0)
-      #   self.assertEqual(list(sess.run(imported_shape)), [])
-      #   self.assertEqual(list(sess.run(new_var_shape)), [])
+      #   self.assertEqual(self.evaluate(imported_var), 1.0)
+      #   self.assertEqual(self.evaluate(imported_assign), 2.0)
+      #   self.assertEqual(list(self.evaluate(imported_shape)), [])
+      #   self.assertEqual(list(self.evaluate(new_var_shape)), [])
 
   def testWhileLoop(self):
     # Produce GraphDef containing while loop.
@@ -418,7 +418,7 @@ class ImportGraphDefTest(test.TestCase):
                                               return_elements=[r.name])
       self.assertEqual(imported_r.name, "import/" + r.name)
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(imported_r), 10)
+        self.assertEqual(self.evaluate(imported_r), 10)
 
   def testImportWhileLoopInCond(self):
     # Produce GraphDef containing while loop.
@@ -458,7 +458,7 @@ class ImportGraphDefTest(test.TestCase):
           lambda i: i < 2, ImportFn, [0],
           shape_invariants=[tensor_shape.TensorShape(None)])
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(out), 10)
+        self.assertEqual(self.evaluate(out), 10)
 
   def testTypeMismatchInGraphDef(self):
     # TODO(skyewm): improve error message
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index fc98b91a01..3605ed7fa2 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -492,8 +492,8 @@ class ScopedMetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
       grad = gradients_impl.gradients([output], [var])
       with session.Session() as sess:
-        sess.run(init_op)
-        expected_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        expected_grad_value = self.evaluate(grad)
 
     # Restore the MetaGraphDef into a new Graph with an import scope.
     with ops.Graph().as_default():
@@ -518,8 +518,8 @@ class ScopedMetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with session.Session() as sess:
-        sess.run(init_op)
-        actual_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        actual_grad_value = self.evaluate(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
   def testImportWhileLoopInWhileLoop(self):
@@ -544,7 +544,7 @@ class ScopedMetaGraphTest(test.TestCase):
       _, x = control_flow_ops.while_loop(lambda i, x: i < 2, body, [0, 0.0],
                                          name="")
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         sess.run(x)
 
   def testScopedImportUnderNameScope(self):
@@ -869,7 +869,7 @@ class MetaGraphWithVariableScopeTest(test.TestCase):
 
       initializer = variables.local_variables_initializer()
       sess.run(initializer)
-      sess.run(update_op)
+      self.evaluate(update_op)
 
     meta_graph.export_scoped_meta_graph(
         filename=meta_graph_filename, graph=graph)
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 3957d1de53..b9c690849d 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -517,21 +517,21 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEquals(x.consumers(), [])
     self.assertEquals(y.consumers(), [z.op, z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(sess.run(z), 4)
+      self.assertEquals(self.evaluate(z), 4)
 
     z.op._update_input(0, x)  # pylint: disable=protected-access
     self.assertEquals(list(z.op.inputs), [x, y])
     self.assertEquals(x.consumers(), [z.op])
     self.assertEquals(y.consumers(), [z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(sess.run(z), 3)
+      self.assertEquals(self.evaluate(z), 3)
 
     z.op._update_input(1, y)  # pylint: disable=protected-access
     self.assertEquals(list(z.op.inputs), [x, y])
     self.assertEquals(x.consumers(), [z.op])
     self.assertEquals(y.consumers(), [z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(sess.run(z), 3)
+      self.assertEquals(self.evaluate(z), 3)
 
   def testUpdateInputGraphError(self):
     g_0 = ops.Graph()
diff --git a/tensorflow/python/framework/smart_cond_test.py b/tensorflow/python/framework/smart_cond_test.py
index b8a9672b06..174ada9fe1 100644
--- a/tensorflow/python/framework/smart_cond_test.py
+++ b/tensorflow/python/framework/smart_cond_test.py
@@ -109,8 +109,8 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
                               exclusive=True)
     with session.Session() as sess:
       # No feed_dict necessary
-      self.assertEqual(sess.run(y), 1)
-      self.assertEqual(sess.run(z), 1)
+      self.assertEqual(self.evaluate(y), 1)
+      self.assertEqual(self.evaluate(z), 1)
 
   def testFalse(self):
     conditions = [(False, raise_exception)]
@@ -121,8 +121,8 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
                               default=lambda: constant_op.constant(1),
                               exclusive=True)
     with session.Session() as sess:
-      self.assertEqual(sess.run(y), 1)
-      self.assertEqual(sess.run(z), 1)
+      self.assertEqual(self.evaluate(y), 1)
+      self.assertEqual(self.evaluate(z), 1)
 
   def testMix(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
diff --git a/tensorflow/python/framework/sparse_tensor_test.py b/tensorflow/python/framework/sparse_tensor_test.py
index 2f7591abbd..9ee1bd75a5 100644
--- a/tensorflow/python/framework/sparse_tensor_test.py
+++ b/tensorflow/python/framework/sparse_tensor_test.py
@@ -50,7 +50,7 @@ class SparseTensorTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(indices, value.indices)
         self.assertAllEqual(values, value.values)
         self.assertAllEqual(shape, value.dense_shape)
-        sess_run_value = sess.run(sp)
+        sess_run_value = self.evaluate(sp)
         self.assertAllEqual(sess_run_value.indices, value.indices)
         self.assertAllEqual(sess_run_value.values, value.values)
         self.assertAllEqual(sess_run_value.dense_shape, value.dense_shape)
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index bdf759f220..87d65c8c46 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -771,7 +771,7 @@ class TensorUtilTest(test.TestCase):
     with self.cached_session() as sess:
       ma = MockArray(np.array([10, 20, 30]))
       t = ops.convert_to_tensor(ma)
-      a = sess.run(t)
+      a = self.evaluate(t)
       self.assertEquals(np.int64, a.dtype)
       self.assertAllClose(np.array([10, 20, 30], dtype=np.int64), a)
 
diff --git a/tensorflow/python/grappler/constant_folding_test.py b/tensorflow/python/grappler/constant_folding_test.py
index ab1d0ed25b..30c1e14681 100644
--- a/tensorflow/python/grappler/constant_folding_test.py
+++ b/tensorflow/python/grappler/constant_folding_test.py
@@ -61,7 +61,7 @@ class ConstantFoldingTest(test.TestCase):
           back_prop=False,
           parallel_iterations=1)
       with session.Session() as sess:
-        y_v = sess.run(y)
+        y_v = self.evaluate(y)
         self.assertAllEqual(np.zeros([10, 20, 30]), y_v)
 
 
diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 7b68d5e80d..55ccfbb93c 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -241,7 +241,7 @@ class LayoutOptimizerTest(test.TestCase):
       if restore:
         saver.restore(sess, checkpoint_path)
       else:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
 
       np.random.seed(0)
       for _ in range(2):
@@ -262,7 +262,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _two_layer_model(x)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -365,7 +365,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(pad)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -396,7 +396,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -425,7 +425,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(cast)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -456,7 +456,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -486,7 +486,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -516,7 +516,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -545,7 +545,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -574,7 +574,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -603,7 +603,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -632,7 +632,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -662,7 +662,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -691,7 +691,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -724,7 +724,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(concat)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -835,7 +835,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reverse)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -905,7 +905,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(select)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -966,7 +966,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(select)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1179,7 +1179,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(s)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1214,7 +1214,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(s)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1347,7 +1347,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1374,7 +1374,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop_with_branch()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1398,7 +1398,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop_with_vec_and_4d()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1422,7 +1422,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _model_with_second_port()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py
index 98cbb1a4b6..d233629cbb 100644
--- a/tensorflow/python/grappler/memory_optimizer_test.py
+++ b/tensorflow/python/grappler/memory_optimizer_test.py
@@ -231,10 +231,10 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
       train_op = graph.get_operation_by_name(train_op_name)
       loss_op = graph.get_tensor_by_name(loss_op_name)
       with session.Session(config=config, graph=graph) as sess:
-        sess.run(init_op)
-        sess.run(train_op)
-        sess.run(train_op)
-        return sess.run(loss_op)
+        self.evaluate(init_op)
+        self.evaluate(train_op)
+        self.evaluate(train_op)
+        return self.evaluate(loss_op)
 
   def testRecomputationRewritingNoErrors(self):
     """Tests that graph output is not significantly different with rewriting."""
@@ -295,8 +295,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
           rewrite_options=manual_memory_config)
       session_config = config_pb2.ConfigProto(graph_options=graph_options)
       with session.Session(config=session_config) as sess:
-        sess.run(init_op)
-        sess.run(train_op)
+        self.evaluate(init_op)
+        self.evaluate(train_op)
 
   def testHintDoesRewrite(self):
     graph = self._annotated_graph()[0]
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index d8aa3e9b52..0ab651b59e 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -136,7 +136,7 @@ class BackendUtilsTest(test.TestCase):
       x = keras.Input((3,))
       y = keras.layers.BatchNormalization()(x)
       if not context.executing_eagerly():
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
   def test_learning_phase_scope(self):
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index 7172571f7c..b1449069e3 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -1013,8 +1013,8 @@ class RNNTest(test.TestCase):
         inputs, _ = cell(inputs, initial_state)
         output = inputs
         if not context.executing_eagerly():
-          sess.run(variables_lib.global_variables_initializer())
-          output = sess.run(output)
+          self.evaluate(variables_lib.global_variables_initializer())
+          output = self.evaluate(output)
         return output
 
     random_seed.set_random_seed(12345)
diff --git a/tensorflow/python/kernel_tests/accumulate_n_test.py b/tensorflow/python/kernel_tests/accumulate_n_test.py
index ae24cf8f14..c7f11f854d 100644
--- a/tensorflow/python/kernel_tests/accumulate_n_test.py
+++ b/tensorflow/python/kernel_tests/accumulate_n_test.py
@@ -65,7 +65,7 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase):
             for _ in range(0, num_inputs)
         ]
         accum_n = math_ops.accumulate_n(input_vars)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         accum_n_grad = gradients.gradients(accum_n, input_vars)
         self.assertAllEqual(
             np.repeat(1.0, num_inputs),  # d/dx (x + y + ...) = 1
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index da29a070cd..b9d9d54c98 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -832,7 +832,7 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase):
       index = constant_op.constant(1, dtype=dtypes.int64)
       b = 2. * a[index]
       grad, = gradients_impl.gradients(b, a)
-      self.assertAllEqual(sess.run(grad), [0., 2., 0.])
+      self.assertAllEqual(self.evaluate(grad), [0., 2., 0.])
 
 
 class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
@@ -845,7 +845,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
               math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
               shape=(4, 1, 1)))
       varshape = variables.Variable([6, 4, 4], dtype=dtypes.int32)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0])
       end = constant_op.constant([4, 1, 1])
       strides = constant_op.constant([1, 1, 1])
@@ -858,7 +858,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
           math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
           shape=(4, 1, 1))
       original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0], dtype=dtypes.int64)
       end = constant_op.constant([4, 1, 1], dtype=dtypes.int64)
       strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64)
@@ -872,7 +872,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
           math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
           shape=(4, 1, 1))
       original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0], dtype=dtypes.int32)
       end = constant_op.constant([4, 1, 1], dtype=dtypes.int64)
       strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64)
@@ -1041,7 +1041,7 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     too_large_val = constant_op.constant([3, 4], dtype=dtypes.int64)
     v = resource_variable_ops.ResourceVariable(init_val)
     with self.cached_session() as sess:
-      sess.run(v.initializer)
+      self.evaluate(v.initializer)
       with self.assertRaises(ValueError):
         sess.run(v[:].assign(too_large_val))
       with self.assertRaises(ValueError):
@@ -1268,7 +1268,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
             initializer=init_ops.constant_initializer(10.0),
             use_resource=use_resource)
         guarantee_a = array_ops.guarantee_const(a)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertEqual(10.0, guarantee_a.eval())
 
   def testResourceRejection(self):
@@ -1278,7 +1278,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
           initializer=init_ops.constant_initializer(10.0),
           use_resource=True)
       guarantee_a = array_ops.guarantee_const(a.handle)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError,
                                                "cannot be a resource variable"):
         guarantee_a.eval()
diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py
index ac5cbc810a..cd33048121 100644
--- a/tensorflow/python/kernel_tests/basic_gpu_test.py
+++ b/tensorflow/python/kernel_tests/basic_gpu_test.py
@@ -44,13 +44,13 @@ class GPUBinaryOpsTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_gpu = sess.run(out)
+      tf_gpu = self.evaluate(out)
 
     with self.cached_session(use_gpu=False) as sess:
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_cpu = sess.run(out)
+      tf_cpu = self.evaluate(out)
 
     self.assertAllClose(tf_cpu, tf_gpu)
 
@@ -96,7 +96,7 @@ class MathBuiltinUnaryTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       inx = ops.convert_to_tensor(x)
       ofunc = tf_func(inx)
-      tf_out = sess.run(ofunc)
+      tf_out = self.evaluate(ofunc)
     self.assertAllClose(np_out, tf_out)
 
   def _inv(self, x):
@@ -148,7 +148,7 @@ class MathBuiltinUnaryTest(test.TestCase):
       iny = ops.convert_to_tensor(y + 0.1)
       ofunc = inx / iny
       out_func2 = math_ops.floor(ofunc)
-      tf_out = sess.run(out_func2)
+      tf_out = self.evaluate(out_func2)
 
     self.assertAllClose(np_out, tf_out)
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index 12afb6a2ad..adfb094971 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -98,8 +98,8 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
           quantile_accumulator_handle, num_features=2)
       quantiles = boosted_trees_ops.boosted_trees_bucketize(
           [self._feature_0, self._feature_1], buckets)
-      sess.run(summary_op)
-      sess.run(flush_op)
+      self.evaluate(summary_op)
+      self.evaluate(flush_op)
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
index cc3984015d..e4c5431c26 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
@@ -65,10 +65,10 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
       self.assertAllClose([[0.004775, 0.41184], [0.02823, 0.41184]],
                           sess.run(gains_list))
-      self.assertAllEqual([[1, 1], [1, 1]], sess.run(thresholds_list))
+      self.assertAllEqual([[1, 1], [1, 1]], self.evaluate(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[-.416667], [.568966]], [[-.6], [-.75]]],
@@ -113,10 +113,10 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
       self.assertAllClose([[0., 0.33931375], [0.01879096, 0.33931375]],
                           sess.run(gains_list))
-      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
@@ -162,9 +162,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
       self.assertAllClose([[[0.0], [0.3965517]], [[-0.4], [-0.5]]],
                           sess.run(left_node_contribs_list))
 
@@ -214,12 +214,12 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
 
       self.assertAllClose([[-3., -2.66068625], [-2.98120904, -2.66068625]],
                           sess.run(gains_list))
 
-      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
@@ -266,9 +266,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
 
       # We can't split node 1 on feature 1 and node 2 on feature 2 because of
       # the min node weight.
-      self.assertAllEqual([[2], [1]], sess.run(node_ids_list))
-      self.assertAllClose([[0.384314], [0.098013]], sess.run(gains_list))
-      self.assertAllEqual([[1], [1]], sess.run(thresholds_list))
+      self.assertAllEqual([[2], [1]], self.evaluate(node_ids_list))
+      self.assertAllClose([[0.384314], [0.098013]], self.evaluate(gains_list))
+      self.assertAllEqual([[1], [1]], self.evaluate(thresholds_list))
       self.assertAllClose([[[0.4852941]], [[-.6]]],
                           sess.run(left_node_contribs_list))
       self.assertAllClose([[[-0.75]], [[-0.014925]]],
@@ -311,9 +311,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            max_splits=max_splits)
 
       # We can't split either of the nodes on the first feature
-      self.assertEqual(2, len(sess.run(node_ids_list)))
-      self.assertAllEqual([], sess.run(node_ids_list)[0])
-      self.assertAllEqual([1], sess.run(node_ids_list)[1])
+      self.assertEqual(2, len(self.evaluate(node_ids_list)))
+      self.assertAllEqual([], self.evaluate(node_ids_list)[0])
+      self.assertAllEqual([1], self.evaluate(node_ids_list)[1])
 
       # Now check when we can't split on any feature
       (node_ids_list, _, _, _,
@@ -325,7 +325,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            tree_complexity=0.0,
            min_node_weight=10,
            max_splits=max_splits)
-      self.assertAllEqual([[], []], sess.run(node_ids_list))
+      self.assertAllEqual([[], []], self.evaluate(node_ids_list))
 
   def testMakeStatsSummarySimple(self):
     """Simple test for MakeStatsSummary."""
diff --git a/tensorflow/python/kernel_tests/bucketize_op_test.py b/tensorflow/python/kernel_tests/bucketize_op_test.py
index 57413e6af5..9575b28899 100644
--- a/tensorflow/python/kernel_tests/bucketize_op_test.py
+++ b/tensorflow/python/kernel_tests/bucketize_op_test.py
@@ -32,7 +32,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0, 3, 8, 11])
     expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, sess.run(op))
+      self.assertAllEqual(expected_out, self.evaluate(op))
 
   def testFloat(self):
     op = math_ops._bucketize(
@@ -40,7 +40,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0., 3., 8., 11.])
     expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, sess.run(op))
+      self.assertAllEqual(expected_out, self.evaluate(op))
 
   def test2DInput(self):
     op = math_ops._bucketize(
@@ -48,7 +48,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0, 3, 8, 11])
     expected_out = [[0, 1, 1, 2, 2], [3, 3, 4, 4, 1]]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, sess.run(op))
+      self.assertAllEqual(expected_out, self.evaluate(op))
 
   def testInvalidBoundariesOrder(self):
     op = math_ops._bucketize(
diff --git a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
index 46ab71537f..031accee55 100644
--- a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
+++ b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
@@ -97,7 +97,7 @@ class RangeSamplerOpsTest(test.TestCase):
           true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True)
       accidental_hits = candidate_sampling_ops.compute_accidental_hits(
           true_classes, sampled_candidates, self.NUM_TRUE)
-      indices, ids, weights = sess.run(accidental_hits)
+      indices, ids, weights = self.evaluate(accidental_hits)
 
     self.assertEqual(1, accidental_hits[0].get_shape().ndims)
     self.assertEqual(1, accidental_hits[1].get_shape().ndims)
diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py
index bc49cd5a04..cdeaf7b696 100644
--- a/tensorflow/python/kernel_tests/cast_op_test.py
+++ b/tensorflow/python/kernel_tests/cast_op_test.py
@@ -187,7 +187,7 @@ class CastOpTest(test.TestCase):
       y = variables.Variable(True, dtype=dtypes.bool)
       cast = math_ops.cast(y, x.dtype)
       variables.global_variables_initializer().run()
-      self.assertEqual(1.0, sess.run(cast))
+      self.assertEqual(1.0, self.evaluate(cast))
 
   def testGradients(self):
     t = [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128]
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 149302831b..6944d73c5f 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -627,7 +627,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([2, 7, 5], dtypes.int32)
       s2 = constant_op.constant([2, 20, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = sess.run(off)
+      ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
   def testNotVector(self):
@@ -679,7 +679,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([2, 7, 5], dtypes.int32)
       s2 = constant_op.constant([2, 20, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = sess.run(off)
+      ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
       cdim = constant_op.constant(-3, dtypes.int32)
@@ -687,7 +687,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([1, 3, 5], dtypes.int32)
       s2 = constant_op.constant([3, 3, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = sess.run(off)
+      ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [2, 0, 0], [3, 0, 0]])
 
 
diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
index 893cb7cce3..8388070c63 100644
--- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
@@ -424,7 +424,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_grad(1)
 
       def apply_grad(accum_op):
-        sess.run(accum_op)
+        self.evaluate(accum_op)
 
       threads = [
           self.checkedThread(
@@ -451,7 +451,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       def apply_grad():
         for accum_op in accum_ops:
           time.sleep(1.0)
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       apply_grad_thread = self.checkedThread(target=apply_grad)
 
@@ -485,7 +485,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       def apply_grad():
         time.sleep(1.0)
         for accum_op in accum_ops:
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       return_array = []
 
@@ -503,7 +503,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
   def _blocking_takeg(self, sess, takeg_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(takeg_op)
+      self.evaluate(takeg_op)
 
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 55654174d3..0d6d2cc6da 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -579,7 +579,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: [math_ops.add(x, 1), math_ops.add(x, 2)]
       fn2 = lambda: [y, y]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      self.assertAllEqual([11, 12], sess.run(r))
+      self.assertAllEqual([11, 12], self.evaluate(r))
 
   def testCondListOutput(self):
     with self.cached_session() as sess:
@@ -589,7 +589,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: [math_ops.add(x, y), math_ops.add(x, y)]
       fn2 = lambda: [y, y]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertListEqual([210, 210], test_result)
 
   def testTupleOutput(self):
@@ -600,7 +600,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: (math_ops.add(x, y), math_ops.add(x, y))
       fn2 = lambda: (y, y)
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertTupleEqual((210, 210), test_result)
 
   def testDictOutput(self):
@@ -611,7 +611,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
       fn2 = lambda: {"a": y, "b": y}
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertDictEqual({"a": 210, "b": 210}, test_result)
 
   def testEmbeddedListOutput(self):
@@ -624,7 +624,7 @@ class ControlFlowTest(test.TestCase):
       # Pass strict=True flag as cond_v2 allows for tensors to be
       # in nested output structures as singletons
       r = control_flow_ops.cond(pred, fn1, fn2, strict=True)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertListEqual([[210, 210]], test_result)
 
   def testEmbeddedTupleOutput(self):
@@ -635,7 +635,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: ((math_ops.add(x, y), math_ops.add(x, y)))
       fn2 = lambda: ((y, y))
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertTupleEqual(((210, 210)), test_result)
 
   def testEmbeddedDictOutput(self):
@@ -648,7 +648,7 @@ class ControlFlowTest(test.TestCase):
       fn2 = lambda: {"a": {"c": y},
                      "b": {"d": y}}
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertDictEqual({"a": {"c": 210}, "b": {"d": 210}}, test_result)
 
   def testCheckNestedOutputStruct(self):
@@ -663,7 +663,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError, v2_msg if control_flow_ops.ENABLE_COND_V2 else v1_msg):
         r = control_flow_ops.cond(pred, fn1, fn2)
-        test_result = sess.run(r)
+        self.evaluate(r)
 
   def testCondRef(self):
 
@@ -1036,7 +1036,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(r[0].dtype, dtypes.int32)
       self.assertEqual(r[1].dtype, dtypes.int32_ref)
 
-      value_i, value_x = sess.run(r)
+      value_i, value_x = self.evaluate(r)
 
     self.assertEqual(100, value_i)
     self.assertEqual(0, value_x)
@@ -1628,7 +1628,7 @@ class ControlFlowTest(test.TestCase):
       with ops.control_dependencies([control_flow_ops.no_op()]):
         loop = control_flow_ops.while_loop(cond, body,
                                            (constant_op.constant(5),))
-      self.assertEqual(0, sess.run(loop))
+      self.assertEqual(0, self.evaluate(loop))
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
   def testWhileCondWithControl_1(self):
@@ -2041,7 +2041,7 @@ class ControlFlowTest(test.TestCase):
         self.assertFalse(gpu_dev_name in dev)
 
     with self.session(graph=graph) as sess:
-      self.assertAllClose(1024.0, sess.run(r))
+      self.assertAllClose(1024.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/116351701 (colocation)")
   def testWhileGrad_ColocateGradients(self):
@@ -2533,8 +2533,8 @@ class ControlFlowTest(test.TestCase):
       res = outer_loop(inp)
       optimizer = adam.AdamOptimizer(learning_rate=0.001)
       train_op = optimizer.minimize(math_ops.reduce_mean(math_ops.square(res)))
-      sess.run(variables.global_variables_initializer())
-      sess.run(train_op)
+      self.evaluate(variables.global_variables_initializer())
+      self.evaluate(train_op)
       self.assertAllClose(2.999, self.evaluate(var))
 
   def _testWhileCondGrad_Simple(self, use_gpu):
@@ -2593,11 +2593,11 @@ class ControlFlowTest(test.TestCase):
           [i0.get_shape(), tensor_shape.TensorShape([None, 2])])
       s = math_ops.reduce_sum(h)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       optimizer = gradient_descent.GradientDescentOptimizer(0.01)
       op = optimizer.minimize(s)
       sess.run(op)
-      self.assertAllClose([[0.98000002, 1.98000002]], sess.run(x))
+      self.assertAllClose([[0.98000002, 1.98000002]], self.evaluate(x))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileWithRefsWithGradients_1(self):
@@ -2691,7 +2691,7 @@ class ControlFlowTest(test.TestCase):
 
       output_grad = control_flow_ops.while_loop(
           c, b, [i0, constant_op.constant(0.0)])
-      self.assertAllClose(600.0, sess.run(output_grad)[1])
+      self.assertAllClose(600.0, self.evaluate(output_grad)[1])
 
   def testWhileAndTensorArray(self):
     with self.cached_session() as sess:
@@ -2710,7 +2710,7 @@ class ControlFlowTest(test.TestCase):
 
       r = control_flow_ops.while_loop(c, b, [n0, y0], parallel_iterations=1)
       r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(107520.0, sess.run(r))
+      self.assertAllClose(107520.0, self.evaluate(r))
 
   def testWhileGrad_StopGrad(self):
     with self.cached_session():
@@ -2843,8 +2843,8 @@ class ControlFlowTest(test.TestCase):
     dy_dq, = gradients_impl.gradients(y, q)
     self.assertIsNotNone(dy_dq)
     with self.cached_session() as sess:
-      sess.run(q.initializer)
-      self.assertAllClose([0., 0.], sess.run(dy_dq))
+      self.evaluate(q.initializer)
+      self.assertAllClose([0., 0.], self.evaluate(dy_dq))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileGradientWithNontrainablePath2(self):
@@ -2861,8 +2861,8 @@ class ControlFlowTest(test.TestCase):
     dy_dq, = gradients_impl.gradients(y, q)
     self.assertIsNotNone(dy_dq)
     with self.cached_session() as sess:
-      sess.run(q.initializer)
-      self.assertAllClose([1., 1.], sess.run(dy_dq))
+      self.evaluate(q.initializer)
+      self.assertAllClose([1., 1.], self.evaluate(dy_dq))
 
   @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   def testIssue16504(self):
@@ -3281,7 +3281,7 @@ class ControlFlowTest(test.TestCase):
 
       result = control_flow_ops.while_loop(condition, body,
                                            [constant_op.constant(4)])
-      self.assertEqual(10, sess.run(result))
+      self.assertEqual(10, self.evaluate(result))
 
       # Ensure that we cannot run a tensor that escapes the loop body
       # accidentally.
@@ -3339,7 +3339,7 @@ class ControlFlowTest(test.TestCase):
           shape=[1], dtype=dtypes.qint8, name="v", container="", shared_name="")
       assign_op = state_ops.assign(
           var_qint, constant_op.constant(np.array([42]), dtypes.qint8))
-      sess.run(assign_op)
+      self.evaluate(assign_op)
 
       cond = constant_op.constant(True, dtypes.bool)
       v_f, v_t = control_flow_ops.ref_switch(var_qint, cond)
@@ -3660,7 +3660,7 @@ class WhileOpBenchmark(test.Benchmark):
     with session.Session() as sess, ops.device(default_device):
       # Get the initial id i, input x, and kernel.
       i, x, kernel = self._getInitVariables()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       if static_unroll:
         for _ in xrange(steps):
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index 3924e13575..3ec5c29df7 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -109,7 +109,7 @@ class Conv3DTest(test.TestCase):
         results.append(result)
 
       with self.cached_session() as sess:
-        values = sess.run(results)
+        values = self.evaluate(results)
         for value in values:
           print("expected = ", expected)
           print("actual = ", value)
@@ -184,8 +184,8 @@ class Conv3DTest(test.TestCase):
         computed_results.append(computed)
         tolerance = 1e-2 if use_gpu else 1e-5
         with self.cached_session() as sess:
-          expected_values = sess.run(expected_results)
-          computed_values = sess.run(computed_results)
+          expected_values = self.evaluate(expected_results)
+          computed_values = self.evaluate(computed_results)
           for e_value, c_value in zip(expected_values, computed_values):
             print("expected = ", e_value)
             print("actual = ", c_value)
@@ -715,8 +715,8 @@ class Conv3DTest(test.TestCase):
         expected_grad = gradients_impl.gradients(expected, t1
                                                  if mode == "input" else t2)[0]
         # "values" consists of two tensors for two backprops
-        actual_value = sess.run(actual_grad)
-        expected_value = sess.run(expected_grad)
+        actual_value = self.evaluate(actual_grad)
+        expected_value = self.evaluate(expected_grad)
         self.assertShapeEqual(actual_value, actual_grad)
         self.assertShapeEqual(expected_value, expected_grad)
       print("expected = ", expected_value)
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 835cc1504d..2d21f6f4ae 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -908,8 +908,8 @@ class Conv2DTest(test.TestCase):
         conv = gradients_impl.gradients(conv_forward, t1)[0]
         conv_2 = gradients_impl.gradients(conv_forward_2, t1)[0]
         # "values" consists of two tensors for two backprops
-        value = sess.run(conv)
-        value_2 = sess.run(conv_2)
+        value = self.evaluate(conv)
+        value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
       tf_logging.info("expected = ", value_2)
@@ -961,8 +961,8 @@ class Conv2DTest(test.TestCase):
           conv_forward_2 = test_util.NCHWToNHWC(conv_forward_2)
         conv = gradients_impl.gradients(conv_forward, t2)[0]
         conv_2 = gradients_impl.gradients(conv_forward, t2)[0]
-        value = sess.run(conv)
-        value_2 = sess.run(conv_2)
+        value = self.evaluate(conv)
+        value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
       tf_logging.info("expected = ", value_2)
@@ -1545,7 +1545,7 @@ class DepthwiseConv2DTest(test.TestCase):
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
       conv = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     tf_logging.info("value = ", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
@@ -1667,7 +1667,7 @@ class SeparableConv2DTest(test.TestCase):
       if data_format == "NCHW":
         conv = array_ops.transpose(conv, [0, 2, 3, 1])
 
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     tf_logging.info("value = ", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index d7dbf5ab9a..87248bf9c8 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -788,7 +788,7 @@ class RoundingTest(test.TestCase):
     y = np.rint(x) if y is None else np.asarray(y)
     with self.cached_session() as sess:
       tf_rint = math_ops.rint(x)
-      np_rint = sess.run(tf_rint)
+      np_rint = self.evaluate(tf_rint)
     self.assertAllEqual(y, np_rint)
     self.assertShapeEqual(y, tf_rint)
 
@@ -881,7 +881,7 @@ class ComplexMakeRealImagTest(test.TestCase):
         force_gpu=use_gpu and test_util.is_gpu_available()) as sess:
       inx = ops.convert_to_tensor(cplx)
       tf_angle = math_ops.angle(inx)
-      tf_angle_val = sess.run(tf_angle)
+      tf_angle_val = self.evaluate(tf_angle)
     self.assertAllEqual(np_angle, tf_angle_val)
     self.assertShapeEqual(np_angle, tf_angle)
 
diff --git a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
index 66b3e0f22f..8c4ccbd88e 100644
--- a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
@@ -80,7 +80,7 @@ class DecodeJpegBenchmark(test.Benchmark):
           initializer=image_ops.encode_jpeg(tiled_image))
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       images = []
       for _ in xrange(parallelism):
         if crop_window is None:
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
index 3ed7dba966..0676664685 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
@@ -43,7 +43,7 @@ class AssignOpTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       def run_add(add_op):
-        sess.run(add_op)
+        self.evaluate(add_op)
 
       threads = [
           self.checkedThread(
@@ -70,7 +70,7 @@ class AssignOpTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       def run_assign(assign_op):
-        sess.run(assign_op)
+        self.evaluate(assign_op)
 
       threads = [
           self.checkedThread(
@@ -103,7 +103,7 @@ class AssignOpTest(test.TestCase):
       p.initializer.run()
 
       def run_add(add_op):
-        sess.run(add_op)
+        self.evaluate(add_op)
 
       threads = [
           self.checkedThread(
@@ -131,7 +131,7 @@ class AssignOpTest(test.TestCase):
       p.initializer.run()
 
       def run_assign(assign_op):
-        sess.run(assign_op)
+        self.evaluate(assign_op)
 
       threads = [
           self.checkedThread(
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index f65d0be367..f6d834c2f8 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -162,7 +162,7 @@ class DepthwiseConv2DTest(test.TestCase):
         conv_native = array_ops.transpose(conv_native, [0, 2, 3, 1])
 
       try:
-        native_result = sess.run(conv_native)
+        native_result = self.evaluate(conv_native)
       except errors.InvalidArgumentError as e:
         # Grouped convolution kernel is only registered for cuDNN 7. Silently
         # return when we are running on an earlier version or without GPU.
@@ -174,7 +174,7 @@ class DepthwiseConv2DTest(test.TestCase):
 
       conv_interface = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      interface_result = sess.run(conv_interface)
+      interface_result = self.evaluate(conv_interface)
 
     tf_logging.info(
         "data_type: %r, use_gpu: %r, grouped_conv: %r, max diff = %f",
@@ -269,7 +269,7 @@ class DepthwiseConv2DTest(test.TestCase):
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
       conv = nn_ops.depthwise_conv2d_native(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     tf_logging.info("value = %r", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/distributions/categorical_test.py b/tensorflow/python/kernel_tests/distributions/categorical_test.py
index f116c54bd1..9c593d2737 100644
--- a/tensorflow/python/kernel_tests/distributions/categorical_test.py
+++ b/tensorflow/python/kernel_tests/distributions/categorical_test.py
@@ -287,7 +287,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     }
 
     with self.cached_session() as sess:
-      run_result = sess.run(to_run)
+      run_result = self.evaluate(to_run)
 
     self.assertAllEqual(run_result["cat_prob"].shape,
                         run_result["norm_prob"].shape)
@@ -462,7 +462,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
           b = categorical.Categorical(logits=b_logits)
 
           kl = kullback_leibler.kl_divergence(a, b)
-          kl_val = sess.run(kl)
+          kl_val = self.evaluate(kl)
           # Make sure KL(a||a) is 0
           kl_same = sess.run(kullback_leibler.kl_divergence(a, a))
 
diff --git a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
index 07da855a01..80da39dfde 100644
--- a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
@@ -40,7 +40,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant([0, 0, 2, 3, 2, 1])
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([0, 13], partition_vals[0])
@@ -62,7 +62,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant([0, 0, 2, 3, 2, 1])
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([[0, 1, 2], [3, 4, 5]], partition_vals[0])
@@ -87,7 +87,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual(part1, partition_vals[0])
@@ -109,7 +109,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=num_partitions)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(num_partitions, len(partition_vals))
     for i in range(num_partitions):
@@ -125,7 +125,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([3 + 4j, 7 + 8j], partition_vals[0])
@@ -138,7 +138,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = 3
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual(np.array([], dtype=np.float64).reshape(-1, 4),
@@ -164,7 +164,7 @@ class DynamicPartitionTest(test.TestCase):
             outputs = data_flow_ops.dynamic_partition(
                 data_t, partitions_t, num_partitions=n)
             self.assertEqual(n, len(outputs))
-            outputs_val = sess.run(outputs)
+            outputs_val = self.evaluate(outputs)
             for i, output in enumerate(outputs_val):
               self.assertAllEqual(output, data[partitions == i])
 
@@ -183,7 +183,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([], partition_vals[0])
@@ -199,7 +199,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=3)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(3, len(partition_vals))
     self.assertAllEqual([[]], partition_vals[0])
@@ -215,7 +215,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([], partition_vals[0])
@@ -236,7 +236,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([6], partition_vals[0])
@@ -257,7 +257,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=5)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(5, len(partition_vals))
     self.assertAllEqual([5], partition_vals[0])
@@ -281,7 +281,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=40)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(40, len(partition_vals))
     for i in range(40):
@@ -335,7 +335,7 @@ class DynamicPartitionTest(test.TestCase):
     self.assertEqual(len(inds), x.shape[0])
     partitioned = data_flow_ops.dynamic_partition(x, inds, 16)
     with self.cached_session() as sess:
-      res = sess.run(partitioned)
+      res = self.evaluate(partitioned)
     self.assertEqual(res[-1].shape[0], 192)
 
 
diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py
index e3742f2e72..c184b93c80 100644
--- a/tensorflow/python/kernel_tests/fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/fifo_queue_test.py
@@ -159,7 +159,7 @@ class FIFOQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -240,7 +240,7 @@ class FIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       results = []
 
@@ -269,7 +269,7 @@ class FIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        x_val, y_val = sess.run(dequeued_t)
+        x_val, y_val = self.evaluate(dequeued_t)
         x, y = elems[i]
         self.assertEqual([x], x_val)
         self.assertEqual([y], y_val)
@@ -356,7 +356,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -399,17 +399,17 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
@@ -429,13 +429,13 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual([None], dequeued_t[0].get_shape().as_list())
       self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list())
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
@@ -529,7 +529,7 @@ class FIFOQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -596,11 +596,11 @@ class FIFOQueueTest(test.TestCase):
 
       def enqueue():
         for _ in xrange(100):
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       def dequeue():
         for _ in xrange(100):
-          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))
+          self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0))
 
       enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
@@ -632,7 +632,7 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         for i in xrange(250):
-          self.assertEqual(i, sess.run(dequeued_t))
+          self.assertEqual(i, self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -663,7 +663,7 @@ class FIFOQueueTest(test.TestCase):
       dequeuemany_t = q.dequeue_many(count_placeholder)
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -701,7 +701,7 @@ class FIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -728,7 +728,7 @@ class FIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -797,7 +797,7 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         for elem in elems:
-          self.assertEqual([elem], sess.run(dequeued_t))
+          self.assertEqual([elem], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -842,7 +842,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems, sess.run(dequeued_t))
+        self.assertAllEqual(elems, self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -867,7 +867,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -892,8 +892,8 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
-        self.assertAllEqual(elems[3:], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[3:], self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -913,16 +913,16 @@ class FIFOQueueTest(test.TestCase):
       cleanup_dequeue_t = q.dequeue()
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        self.assertAllEqual(elems[0:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t))
         with self.assertRaises(errors_impl.OutOfRangeError):
           sess.run(dequeued_t)
-        self.assertEqual(elems[3], sess.run(cleanup_dequeue_t))
+        self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t))
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -1051,7 +1051,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1074,7 +1074,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1103,7 +1103,7 @@ class FIFOQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # Expect the operation to succeed once the dequeue op runs.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1113,7 +1113,7 @@ class FIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1138,7 +1138,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1148,7 +1148,7 @@ class FIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1266,19 +1266,19 @@ class FIFOQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_op)
+      self.evaluate(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_many_op)
+      self.evaluate(dequeue_many_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_many_op)
+      self.evaluate(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1331,14 +1331,14 @@ class FIFOQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1405,7 +1405,7 @@ class FIFOQueueTest(test.TestCase):
       q.enqueue_many(input_tuple).run()
 
       output_tuple_t = q.dequeue_many(32)
-      output_tuple = sess.run(output_tuple_t)
+      output_tuple = self.evaluate(output_tuple_t)
 
       for (input_elem, output_elem) in zip(input_tuple, output_tuple):
         self.assertAllEqual(input_elem, output_elem)
@@ -1507,7 +1507,7 @@ class FIFOQueueDictTest(test.TestCase):
       enqueue_op4 = q.enqueue_many({"f": [40.0, 50.0]})
       dequeue = q.dequeue()
       dequeue_2 = q.dequeue_many(2)
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
       sess.run(enqueue_op2)
       sess.run(enqueue_op3)
       sess.run(enqueue_op4)
@@ -1565,7 +1565,7 @@ class FIFOQueueDictTest(test.TestCase):
       })
       dequeue = q.dequeue()
       dequeue_2 = q.dequeue_many(2)
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
       sess.run(enqueue_op2)
       sess.run(enqueue_op3)
       sess.run(enqueue_op4)
@@ -1613,8 +1613,8 @@ class FIFOQueueWithTimeoutTest(test.TestCase):
                                    "Timed out waiting for notification"):
         sess.run(dequeued_t, options=config_pb2.RunOptions(timeout_in_ms=10))
 
-      sess.run(enqueue_op)
-      self.assertEqual(37, sess.run(dequeued_t))
+      self.evaluate(enqueue_op)
+      self.assertEqual(37, self.evaluate(dequeued_t))
 
 
 class QueueContainerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 503569f3b1..0af32b048e 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -567,8 +567,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:worker/replica:0/task:0/cpu:1")
 
     with session.Session(worker[0].target) as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionDirectSession(self):
@@ -591,8 +591,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/cpu:1")
 
     with self.test_session(config=worker_config) as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionSameDeviceDirectSession(self):
@@ -610,8 +610,8 @@ class FunctionalOpsTest(test.TestCase):
           args=[a, b], Tout=[dtypes.int32], f=_remote_fn, target="/cpu:0")
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionCPUGPU(self):
@@ -634,8 +634,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/device:GPU:0")[0] + 3.0
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9.0)
 
   def testRemoteFunctionGPUCPU(self):
@@ -658,8 +658,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/cpu:0")[0] + 3.0
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9.0)
 
   def testRemoteFunctionGPUCPUStrings(self):
@@ -677,7 +677,7 @@ class FunctionalOpsTest(test.TestCase):
           args=[a], Tout=[dtypes.string], f=_remote_fn, target="/cpu:0")
 
     with self.cached_session() as sess:
-      ret = sess.run(remote_op)
+      ret = self.evaluate(remote_op)
       self.assertAllEqual(ret, [b"a"])
 
   def testRemoteFunctionCrossProcess(self):
@@ -699,8 +699,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:worker/replica:0/task:1/cpu:0")[0] + 3.0
 
     with session.Session(workers[0].target) as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9)
 
   def testIf(self):
@@ -857,11 +857,11 @@ class FunctionalOpsTest(test.TestCase):
           result_binary = functional_ops.While(
               [1.0, 0., 0.],
               function.Defun(*[dtypes.float32] * 3)(TestCond), TestBinary)
-          sess.run(variables.global_variables_initializer())
+          self.evaluate(variables.global_variables_initializer())
           assert len(result_unary) == 2
-          self.assertEqual([10.0, 54.0], sess.run(result_unary))
+          self.assertEqual([10.0, 54.0], self.evaluate(result_unary))
           assert len(result_binary) == 3
-          self.assertEqual([10.0, 54.0, 9.0], sess.run(result_binary))
+          self.assertEqual([10.0, 54.0, 9.0], self.evaluate(result_binary))
 
           def TestCondCapture(n, *args):
             del args
@@ -892,7 +892,7 @@ class FunctionalOpsTest(test.TestCase):
                 100, 0, -1, [0.], Body, rewrite_with_while=rewrite_with_while)
             [0],
         ]
-        xvals = sess.run(xs)
+        xvals = self.evaluate(xs)
       self.assertAllEqual(210, xvals[0])
       self.assertAllEqual(5050, xvals[1])
 
@@ -949,16 +949,16 @@ class FunctionalOpsTest(test.TestCase):
         result_binary = functional_ops.For(
             1, 10, 1, [0., 0.], TestBinary,
             rewrite_with_while=rewrite_with_while)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         assert not result_nullary
         # The nullary variant doesn't return anything so we can't easily run it.
         # As a total hack, fetch the operation by name and run it.
         sess.run(ops.get_default_graph().get_operation_by_name(
             "While" if rewrite_with_while else "For"))
         assert len(result_unary) == 1
-        self.assertEqual([54.0], sess.run(result_unary))
+        self.assertEqual([54.0], self.evaluate(result_unary))
         assert len(result_binary) == 2
-        self.assertEqual([54.0, 9.0], sess.run(result_binary))
+        self.assertEqual([54.0, 9.0], self.evaluate(result_binary))
 
   def _tfMLP(self, xval, wsval, bsval, rewrite_with_while):
     # On GPU, don't rewrite using a while loop.
@@ -1041,8 +1041,8 @@ class FunctionalOpsTest(test.TestCase):
       avals = [Poly(a), Grad(a)]
       b = constant_op.constant(1.)
       bvals = [Poly(b), Grad(b)]
-      self.assertAllEqual(sess.run(avals), [8., 4.])
-      self.assertAllEqual(sess.run(bvals), [17., 16.])
+      self.assertAllEqual(self.evaluate(avals), [8., 4.])
+      self.assertAllEqual(self.evaluate(bvals), [17., 16.])
 
 
 # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the
@@ -1193,7 +1193,7 @@ class PartitionedCallTest(test.TestCase):
             allow_soft_placement=False,
             log_device_placement=True,
             device_count={"CPU": 2})) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       expected = sess.run(sum_gather())
       result = sess.run(
           functional_ops.partitioned_call(
diff --git a/tensorflow/python/kernel_tests/gradient_correctness_test.py b/tensorflow/python/kernel_tests/gradient_correctness_test.py
index 291a69ebac..12b8a4c8e3 100644
--- a/tensorflow/python/kernel_tests/gradient_correctness_test.py
+++ b/tensorflow/python/kernel_tests/gradient_correctness_test.py
@@ -35,7 +35,7 @@ class GradientCorrectnessTest(test.TestCase):
       yexp = math_ops.exp(x)
       yexplog = math_ops.log(yexp)
       grads = gradients_impl.gradients([yexp, yexplog], [x])
-      grad_vals = sess.run(grads)
+      grad_vals = self.evaluate(grads)
       exp1_plus_one = (1.0 + np.exp(1.0)).astype(np.float32)
       # [dexp(x)/dx + d(log(exp(x)))/dx] @ x=1 == exp(1) + 1
       self.assertAllClose(grad_vals[0], exp1_plus_one)
@@ -44,13 +44,13 @@ class GradientCorrectnessTest(test.TestCase):
     x = constant_op.constant(3.)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
-      self.assertAllClose(1., sess.run(dx_dx))
+      self.assertAllClose(1., self.evaluate(dx_dx))
 
   def testIntegerIdentityGradient(self):
     x = constant_op.constant(3)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
-      self.assertAllClose(1, sess.run(dx_dx))
+      self.assertAllClose(1, self.evaluate(dx_dx))
 
   def testGradientWithIntegerPath(self):
     x = constant_op.constant([3.9, 4.1])
@@ -58,7 +58,7 @@ class GradientCorrectnessTest(test.TestCase):
     y = x * k
     dy_dx, = gradients_impl.gradients(y, x)
     with self.cached_session() as sess:
-      self.assertAllClose([3., 4.], sess.run(dy_dx))
+      self.assertAllClose([3., 4.], self.evaluate(dy_dx))
 
   def testNoIntegerGradient1(self):
     x = constant_op.constant([3.9, 4.1])
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index a3f2c0ddd7..074985dd93 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -709,7 +709,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
           t = self.evaluate(outputs)
           self.assertAllEqual(t.shape, outputs_shape)
           # Check isometry of the delta-orthogonal kernel.
-          self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+          self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
   def testNonuniformity(self):
     value = 0
@@ -847,7 +847,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
@@ -942,7 +942,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
@@ -1067,7 +1067,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
 
 class IdentityInitializerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 3efad4ea11..ab4c9c730b 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -174,7 +174,7 @@ class HashTableOpTest(test.TestCase):
           constant_op.constant(sp_shape, dtypes.int64))
       output = table.lookup(input_tensor)
 
-      out_indices, out_values, out_shape = sess.run(output)
+      out_indices, out_values, out_shape = self.evaluate(output)
 
       self.assertAllEqual([0, 1, -1], out_values)
       self.assertAllEqual(sp_indices, out_indices)
diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py
index d3a907852a..bda63bcaa9 100644
--- a/tensorflow/python/kernel_tests/losses_test.py
+++ b/tensorflow/python/kernel_tests/losses_test.py
@@ -1046,9 +1046,9 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with self.cached_session() as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         for grad, _ in gradients_to_variables:
-          np_grad = sess.run(grad)
+          np_grad = self.evaluate(grad)
           self.assertFalse(np.isnan(np_grad).any())
 
   def testNonZeroLossWithPythonScalarWeight(self):
diff --git a/tensorflow/python/kernel_tests/map_stage_op_test.py b/tensorflow/python/kernel_tests/map_stage_op_test.py
index d503f3d7c9..4b5bd4059f 100644
--- a/tensorflow/python/kernel_tests/map_stage_op_test.py
+++ b/tensorflow/python/kernel_tests/map_stage_op_test.py
@@ -148,7 +148,7 @@ class MapStageTest(test.TestCase):
       for i in range(n):
         self.assertTrue(sess.run(peek, feed_dict={gi: i})[0] == i)
 
-      self.assertTrue(sess.run(size) == 10)
+      self.assertTrue(self.evaluate(size) == 10)
 
   def testSizeAndClear(self):
     with ops.Graph().as_default() as G:
@@ -170,11 +170,11 @@ class MapStageTest(test.TestCase):
 
     with self.session(use_gpu=True, graph=G) as sess:
       sess.run(stage, feed_dict={x: -1, pi: 3})
-      self.assertEqual(sess.run(size), 1)
+      self.assertEqual(self.evaluate(size), 1)
       sess.run(stage, feed_dict={x: -1, pi: 1})
-      self.assertEqual(sess.run(size), 2)
+      self.assertEqual(self.evaluate(size), 2)
       sess.run(clear)
-      self.assertEqual(sess.run(size), 0)
+      self.assertEqual(self.evaluate(size), 0)
 
   def testCapacity(self):
     capacity = 3
@@ -231,13 +231,13 @@ class MapStageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(sess.run(size) == capacity)
+      self.assertTrue(self.evaluate(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
         sess.run(get)
 
-      self.assertTrue(sess.run(size) == 0)
+      self.assertTrue(self.evaluate(size) == 0)
 
   def testMemoryLimit(self):
     memory_limit = 512 * 1024  # 512K
@@ -295,13 +295,13 @@ class MapStageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(sess.run(size) == capacity)
+      self.assertTrue(self.evaluate(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
         sess.run(get)
 
-      self.assertTrue(sess.run(size) == 0)
+      self.assertTrue(self.evaluate(size) == 0)
 
   def testOrdering(self):
     import six
@@ -332,14 +332,14 @@ class MapStageTest(test.TestCase):
       for i in keys:
         sess.run(stage, feed_dict={pi: i, x: i})
 
-      self.assertTrue(sess.run(size) == n)
+      self.assertTrue(self.evaluate(size) == n)
 
       # Check that key, values come out in ascending order
       for i, k in enumerate(reversed(keys)):
-        get_key, values = sess.run(get)
+        get_key, values = self.evaluate(get)
         self.assertTrue(i == k == get_key == values)
 
-      self.assertTrue(sess.run(size) == 0)
+      self.assertTrue(self.evaluate(size) == 0)
 
   def testPartialDictInsert(self):
     with ops.Graph().as_default() as G:
diff --git a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
index 434458721c..5cef4b79a3 100644
--- a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
@@ -146,7 +146,7 @@ class InverseOpTest(test.TestCase):
         inv1 = linalg_ops.matrix_inverse(matrix1, adjoint=adjoint_)
         inv2 = linalg_ops.matrix_inverse(matrix2, adjoint=adjoint_)
         all_ops += [inv1, inv2]
-      inv = sess.run(all_ops)
+      inv = self.evaluate(all_ops)
       self.assertAllEqual(inv[0], inv[1])
       self.assertAllEqual(inv[2], inv[3])
 
diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
index 1334d0c4ce..80badee896 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
@@ -126,7 +126,7 @@ class MatrixSolveOpTest(test.TestCase):
         s1 = linalg_ops.matrix_solve(lhs1, rhs1, adjoint=adjoint_)
         s2 = linalg_ops.matrix_solve(lhs2, rhs2, adjoint=adjoint_)
         all_ops += [s1, s2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       self.assertAllEqual(val[0], val[1])
       self.assertAllEqual(val[2], val[3])
 
diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 9212580313..1f2144bdee 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -108,7 +108,7 @@ class SquareRootOpTest(test.TestCase):
       sqrt1 = gen_linalg_ops.matrix_square_root(matrix1)
       sqrt2 = gen_linalg_ops.matrix_square_root(matrix2)
       all_ops = [sqrt1, sqrt2]
-      sqrt = sess.run(all_ops)
+      sqrt = self.evaluate(all_ops)
       self.assertAllEqual(sqrt[0], sqrt[1])
 
 
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index 5dcdb9e420..b68327105a 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -203,10 +203,10 @@ class MeanTest(test.TestCase):
 
       mean, update_op = metrics.mean(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAlmostEqual(1.65, sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
 
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
@@ -220,14 +220,14 @@ class MeanTest(test.TestCase):
 
       mean, update_op = metrics.mean(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(0.5, sess.run(update_op), 5)
-      self.assertAlmostEqual(1.475, sess.run(update_op), 5)
-      self.assertAlmostEqual(12.4 / 6.0, sess.run(update_op), 5)
-      self.assertAlmostEqual(1.65, sess.run(update_op), 5)
+      self.assertAlmostEqual(0.5, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(1.475, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(12.4 / 6.0, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(1.65, self.evaluate(update_op), 5)
 
-      self.assertAlmostEqual(1.65, sess.run(mean), 5)
+      self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
 
   def testUnweighted(self):
     values = _test_values((3, 2, 4, 1))
@@ -370,10 +370,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean))
+        self.evaluate(update_op)
+      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean))
 
   def testMultiDimensional(self):
     with self.cached_session() as sess:
@@ -391,10 +391,11 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(2):
-        sess.run(update_op)
-      self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]], sess.run(mean))
+        self.evaluate(update_op)
+      self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]],
+                          self.evaluate(mean))
 
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
@@ -408,14 +409,14 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAllClose([[0, 1]], sess.run(update_op), 5)
-      self.assertAllClose([[-2.1, 5.05]], sess.run(update_op), 5)
-      self.assertAllClose([[2.3 / 3., 10.1 / 3.]], sess.run(update_op), 5)
-      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(update_op), 5)
+      self.assertAllClose([[0, 1]], self.evaluate(update_op), 5)
+      self.assertAllClose([[-2.1, 5.05]], self.evaluate(update_op), 5)
+      self.assertAllClose([[2.3 / 3., 10.1 / 3.]], self.evaluate(update_op), 5)
+      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(update_op), 5)
 
-      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean), 5)
+      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean), 5)
 
   def testBinaryWeighted1d(self):
     with self.cached_session() as sess:
@@ -439,10 +440,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[3.25, 0.5]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[3.25, 0.5]], self.evaluate(mean), 5)
 
   def testWeighted1d(self):
     with self.cached_session() as sess:
@@ -466,10 +467,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[0.8, 3.52]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[0.8, 3.52]], self.evaluate(mean), 5)
 
   def testWeighted2d_1(self):
     with self.cached_session() as sess:
@@ -493,10 +494,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[-2.1, 0.5]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[-2.1, 0.5]], self.evaluate(mean), 5)
 
   def testWeighted2d_2(self):
     with self.cached_session() as sess:
@@ -520,10 +521,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[0, 0.5]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[0, 0.5]], self.evaluate(mean), 5)
 
 
 class AccuracyTest(test.TestCase):
@@ -576,11 +577,11 @@ class AccuracyTest(test.TestCase):
     accuracy, update_op = metrics.accuracy(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_accuracy = accuracy.eval()
@@ -609,10 +610,10 @@ class AccuracyTest(test.TestCase):
 
       accuracy, update_op = metrics.accuracy(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in xrange(3):
-        sess.run(update_op)
-      self.assertEqual(0.5, sess.run(update_op))
+        self.evaluate(update_op)
+      self.assertEqual(0.5, self.evaluate(update_op))
       self.assertEqual(0.5, accuracy.eval())
 
   def testEffectivelyEquivalentSizes(self):
@@ -621,7 +622,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
@@ -631,7 +632,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions, weights=2.0)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
@@ -645,7 +646,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # if streaming_accuracy does not flatten the weight, accuracy would be
       # 0.33333334 due to an intended broadcast of weight. Due to flattening,
       # it will be higher than .95
@@ -666,7 +667,7 @@ class AccuracyTest(test.TestCase):
       accuracy, update_op = metrics.accuracy(labels, predictions,
                                              weights_placeholder)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # if streaming_accuracy does not flatten the weight, accuracy would be
       # 0.33333334 due to an intended broadcast of weight. Due to flattening,
       # it will be higher than .95
@@ -704,10 +705,10 @@ class AccuracyTest(test.TestCase):
 
       accuracy, update_op = metrics.accuracy(labels, predictions, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in xrange(3):
-        sess.run(update_op)
-      self.assertEqual(1.0, sess.run(update_op))
+        self.evaluate(update_op)
+      self.assertEqual(1.0, self.evaluate(update_op))
       self.assertEqual(1.0, accuracy.eval())
 
 
@@ -747,11 +748,11 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_precision = precision.eval()
@@ -766,8 +767,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op))
       self.assertAlmostEqual(1, precision.eval())
 
   def testSomeCorrect_multipleInputDtypes(self):
@@ -779,7 +780,7 @@ class PrecisionTest(test.TestCase):
       precision, update_op = metrics.precision(labels, predictions)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, precision.eval())
 
@@ -882,8 +883,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertAlmostEqual(0, precision.eval())
 
   def testZeroTrueAndFalsePositivesGivesZeroPrecision(self):
@@ -892,8 +893,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(0.0, precision.eval())
 
 
@@ -934,11 +935,11 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_recall = recall.eval()
@@ -953,8 +954,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(1, recall.eval())
 
   def testSomeCorrect_multipleInputDtypes(self):
@@ -966,7 +967,7 @@ class RecallTest(test.TestCase):
       recall, update_op = metrics.recall(labels, predictions)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, recall.eval())
 
@@ -977,7 +978,7 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       weighted_tp = 2.0 + 5.0
       weighted_t = (2.0 + 2.0) + (5.0 + 5.0)
       expected_precision = weighted_tp / weighted_t
@@ -991,7 +992,7 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       weighted_tp = 3.0 + 1.0
       weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
       expected_precision = weighted_tp / weighted_t
@@ -1006,8 +1007,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(0, recall.eval())
 
   def testZeroTruePositivesAndFalseNegativesGivesZeroRecall(self):
@@ -1016,8 +1017,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(0, recall.eval())
 
 
@@ -1056,11 +1057,11 @@ class AUCTest(test.TestCase):
     auc, update_op = metrics.auc(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_auc = auc.eval()
@@ -1078,8 +1079,8 @@ class AUCTest(test.TestCase):
       labels = constant_op.constant(inputs)
       auc, update_op = metrics.auc(labels, predictions, curve=curve)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(1, self.evaluate(update_op))
 
       self.assertEqual(1, auc.eval())
 
@@ -1093,8 +1094,8 @@ class AUCTest(test.TestCase):
             constant_op.constant([0, 1, 1, 0], shape=(1, 4)), dtype=label_dtype)
         auc, update_op = metrics.auc(labels, predictions)
 
-        sess.run(variables.local_variables_initializer())
-        self.assertAlmostEqual(0.5, sess.run(update_op))
+        self.evaluate(variables.local_variables_initializer())
+        self.assertAlmostEqual(0.5, self.evaluate(update_op))
 
         self.assertAlmostEqual(0.5, auc.eval())
 
@@ -1106,8 +1107,8 @@ class AUCTest(test.TestCase):
       weights = constant_op.constant([2], shape=(1, 1))
       auc, update_op = metrics.auc(labels, predictions, weights=weights)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.5, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, self.evaluate(update_op), 5)
 
       self.assertAlmostEqual(0.5, auc.eval(), 5)
 
@@ -1119,8 +1120,8 @@ class AUCTest(test.TestCase):
       weights = constant_op.constant([1, 2, 3, 4], shape=(1, 4))
       auc, update_op = metrics.auc(labels, predictions, weights=weights)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.7, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.7, self.evaluate(update_op), 5)
 
       self.assertAlmostEqual(0.7, auc.eval(), 5)
 
@@ -1134,10 +1135,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # expected ~= 0.79726744594
       expected = 1 - math.log(1.5) / 2
-      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testCorrectAnotherAUCPRSpecialCase(self):
@@ -1150,10 +1151,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # expected ~= 0.61350593198
       expected = (2.5 - 2 * math.log(4./3) - 0.25 * math.log(7./5)) / 3
-      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testThirdCorrectAUCPRSpecialCase(self):
@@ -1166,10 +1167,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # expected ~= 0.90410597584
       expected = 1 - math.log(4./3) / 3
-      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testIncorrectAUCPRSpecialCase(self):
@@ -1180,8 +1181,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.79166, sess.run(update_op), delta=1e-3)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.79166, self.evaluate(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3)
 
@@ -1195,8 +1196,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.610317, sess.run(update_op), delta=1e-3)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.610317, self.evaluate(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3)
 
@@ -1210,8 +1211,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.90277, sess.run(update_op), delta=1e-3)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.90277, self.evaluate(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-3)
 
@@ -1223,8 +1224,8 @@ class AUCTest(test.TestCase):
       labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
       auc, update_op = metrics.auc(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0, self.evaluate(update_op))
 
       self.assertAlmostEqual(0, auc.eval())
 
@@ -1234,8 +1235,8 @@ class AUCTest(test.TestCase):
       labels = array_ops.zeros([4])
       auc, update_op = metrics.auc(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op), 6)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op), 6)
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
@@ -1245,8 +1246,8 @@ class AUCTest(test.TestCase):
       labels = array_ops.ones([4])
       auc, update_op = metrics.auc(labels, predictions, curve='PR')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op), 6)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op), 6)
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
@@ -1317,9 +1318,9 @@ class AUCTest(test.TestCase):
                                      num_thresholds=500,
                                      weights=tf_weights)
 
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         for i in range(num_batches):
-          sess.run(update_op)
+          self.evaluate(update_op)
 
         # Since this is only approximate, we can't expect a 6 digits match.
         # Although with higher number of samples/thresholds we should see the
@@ -1371,11 +1372,11 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_specificity = specificity.eval()
@@ -1391,8 +1392,8 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(1, self.evaluate(update_op))
       self.assertEqual(1, specificity.eval())
 
   def testSomeCorrectHighSensitivity(self):
@@ -1406,8 +1407,8 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.8)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1.0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1.0, self.evaluate(update_op))
       self.assertAlmostEqual(1.0, specificity.eval())
 
   def testSomeCorrectLowSensitivity(self):
@@ -1421,9 +1422,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.4)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(0.6, sess.run(update_op))
+      self.assertAlmostEqual(0.6, self.evaluate(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted1d_multipleLabelDtypes(self):
@@ -1440,9 +1441,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
           labels, predictions, weights=weights, sensitivity=0.4)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
 
-        self.assertAlmostEqual(0.6, sess.run(update_op))
+        self.assertAlmostEqual(0.6, self.evaluate(update_op))
         self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted2d(self):
@@ -1458,9 +1459,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, weights=weights, sensitivity=0.4)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(8.0 / 15.0, sess.run(update_op))
+      self.assertAlmostEqual(8.0 / 15.0, self.evaluate(update_op))
       self.assertAlmostEqual(8.0 / 15.0, specificity.eval())
 
 
@@ -1508,11 +1509,11 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_sensitivity = sensitivity.eval()
@@ -1528,8 +1529,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(1, self.evaluate(update_op))
       self.assertEqual(1, specificity.eval())
 
   def testSomeCorrectHighSpecificity(self):
@@ -1543,8 +1544,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.8)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.8, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.8, self.evaluate(update_op))
       self.assertAlmostEqual(0.8, specificity.eval())
 
   def testSomeCorrectLowSpecificity(self):
@@ -1558,8 +1559,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.4)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.6, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.6, self.evaluate(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted_multipleLabelDtypes(self):
@@ -1577,8 +1578,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
           labels, predictions, weights=weights, specificity=0.4)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
-        self.assertAlmostEqual(0.675, sess.run(update_op))
+        self.evaluate(variables.local_variables_initializer())
+        self.assertAlmostEqual(0.675, self.evaluate(update_op))
         self.assertAlmostEqual(0.675, specificity.eval())
 
 
@@ -1639,7 +1640,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
     rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates, then verify idempotency.
       sess.run([prec_op, rec_op])
@@ -1663,7 +1664,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertEqual(1, prec.eval())
@@ -1683,7 +1684,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                    thresholds)
 
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         sess.run([prec_op, rec_op])
 
         self.assertAlmostEqual(0.5, prec.eval())
@@ -1701,7 +1702,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(0, prec.eval())
@@ -1729,7 +1730,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec_low = array_ops.reshape(rec_low, shape=())
       rec_high = array_ops.reshape(rec_high, shape=())
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(1.0, prec_low.eval(), places=5)
@@ -1759,7 +1760,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec_low = array_ops.reshape(rec_low, shape=())
       rec_high = array_ops.reshape(rec_high, shape=())
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(1.0, prec_low.eval(), places=5)
@@ -1783,7 +1784,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       [rec_low, rec_high] = array_ops.split(
           value=rec, num_or_size_splits=2, axis=0)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(0.75, prec_low.eval())
@@ -1801,7 +1802,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(0, prec.eval(), 6)
@@ -1869,7 +1870,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(tf_labels, tf_predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(int(num_samples / batch_size)):
         sess.run([prec_op, rec_op])
       # Since this is only approximate, we can't expect a 6 digits match.
@@ -2802,11 +2803,11 @@ class MeanAbsoluteErrorTest(test.TestCase):
     error, update_op = metrics.mean_absolute_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2823,8 +2824,8 @@ class MeanAbsoluteErrorTest(test.TestCase):
     error, update_op = metrics.mean_absolute_error(labels, predictions, weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(3, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(3, self.evaluate(update_op))
       self.assertEqual(3, error.eval())
 
 
@@ -2867,11 +2868,11 @@ class MeanRelativeErrorTest(test.TestCase):
                                                    normalizer)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2892,8 +2893,8 @@ class MeanRelativeErrorTest(test.TestCase):
         labels, predictions, normalizer=labels)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(expected_error, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(expected_error, self.evaluate(update_op))
       self.assertEqual(expected_error, error.eval())
 
   def testSingleUpdateNormalizedByZeros(self):
@@ -2908,8 +2909,8 @@ class MeanRelativeErrorTest(test.TestCase):
         labels, predictions, normalizer=array_ops.zeros_like(labels))
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0.0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0.0, self.evaluate(update_op))
       self.assertEqual(0.0, error.eval())
 
 
@@ -2946,11 +2947,11 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2964,8 +2965,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithError(self):
@@ -2977,8 +2978,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(6, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(6, self.evaluate(update_op))
       self.assertEqual(6, error.eval())
 
   def testSingleUpdateWithErrorAndWeights(self):
@@ -2991,8 +2992,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions, weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(13, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(13, self.evaluate(update_op))
       self.assertEqual(13, error.eval())
 
   def testMultipleBatchesOfSizeOne(self):
@@ -3013,9 +3014,9 @@ class MeanSquaredErrorTest(test.TestCase):
 
       error, update_op = metrics.mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertAlmostEqual(208.0 / 6, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
+      self.assertAlmostEqual(208.0 / 6, self.evaluate(update_op), 5)
 
       self.assertAlmostEqual(208.0 / 6, error.eval(), 5)
 
@@ -3054,7 +3055,7 @@ class MeanSquaredErrorTest(test.TestCase):
       mse1, update_op1 = metrics.mean_squared_error(
           labels1, predictions1, name='msd1')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([update_op0, update_op1])
       sess.run([update_op0, update_op1])
 
@@ -3081,7 +3082,7 @@ class MeanSquaredErrorTest(test.TestCase):
       mae, ma_update_op = metrics.mean_absolute_error(labels, predictions)
       mse, ms_update_op = metrics.mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([ma_update_op, ms_update_op])
       sess.run([ma_update_op, ms_update_op])
 
@@ -3123,11 +3124,11 @@ class RootMeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.root_mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -3142,8 +3143,8 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
 
       self.assertEqual(0, rmse.eval())
 
@@ -3156,7 +3157,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAlmostEqual(math.sqrt(6), update_op.eval(), 5)
       self.assertAlmostEqual(math.sqrt(6), rmse.eval(), 5)
 
@@ -3171,8 +3172,8 @@ class RootMeanSquaredErrorTest(test.TestCase):
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions,
                                                         weights)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(math.sqrt(13), sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(math.sqrt(13), self.evaluate(update_op))
 
       self.assertAlmostEqual(math.sqrt(13), rmse.eval(), 5)
 
@@ -3221,11 +3222,11 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=1)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -3243,8 +3244,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithError1(self):
@@ -3259,8 +3260,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op), 5)
       self.assertAlmostEqual(1, error.eval(), 5)
 
   def testSingleUpdateWithError2(self):
@@ -3280,8 +3281,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1.0, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1.0, self.evaluate(update_op), 5)
       self.assertAlmostEqual(1.0, error.eval(), 5)
 
   def testSingleUpdateWithErrorAndWeights1(self):
@@ -3299,8 +3300,8 @@ class MeanCosineDistanceTest(test.TestCase):
         labels, predictions, dim=2, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithErrorAndWeights2(self):
@@ -3318,7 +3319,7 @@ class MeanCosineDistanceTest(test.TestCase):
         labels, predictions, dim=2, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.5, update_op.eval())
       self.assertEqual(1.5, error.eval())
 
@@ -3360,7 +3361,7 @@ class PcntBelowThreshTest(test.TestCase):
       pcnt1, update_op1 = metrics.percentage_below(values, 7, name='medium')
       pcnt2, update_op2 = metrics.percentage_below(values, 1, name='low')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       sess.run([update_op0, update_op1, update_op2])
 
       pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2])
@@ -3382,7 +3383,7 @@ class PcntBelowThreshTest(test.TestCase):
       pcnt2, update_op2 = metrics.percentage_below(
           values, 1, weights=weights, name='low')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertListEqual([1.0, 0.5, 0.0],
                            sess.run([update_op0, update_op1, update_op2]))
 
@@ -3446,11 +3447,11 @@ class MeanIOUTest(test.TestCase):
         labels, predictions, num_classes=num_classes)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_mean_iou = mean_iou.eval()
@@ -3482,9 +3483,9 @@ class MeanIOUTest(test.TestCase):
 
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0 / 2.0, 1.0 / 4.0, 0.])
       self.assertEqual(desired_output, miou.eval())
 
@@ -3529,7 +3530,7 @@ class MeanIOUTest(test.TestCase):
 
       variables.local_variables_initializer().run()
       for _ in range(6):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([2.0 / 3.0, 1.0 / 2.0])
       self.assertAlmostEqual(desired_output, mean_iou.eval())
 
@@ -3563,9 +3564,9 @@ class MeanIOUTest(test.TestCase):
 
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
@@ -3587,7 +3588,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       confusion_matrix = update_op.eval()
       self.assertAllEqual([[3, 0], [2, 5]], confusion_matrix)
       desired_miou = np.mean([3. / 5., 5. / 7.])
@@ -3599,7 +3600,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 1
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(40, update_op.eval()[0])
       self.assertEqual(1.0, miou.eval())
 
@@ -3609,7 +3610,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[0, 0], [40, 0]], update_op.eval())
       self.assertEqual(0., miou.eval())
 
@@ -3640,7 +3641,7 @@ class MeanIOUTest(test.TestCase):
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(
           labels, predictions, num_classes, weights=weights)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[2, 0], [2, 4]], update_op.eval())
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
@@ -3659,7 +3660,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 3
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
       self.assertAlmostEqual(
           1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
@@ -3671,7 +3672,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
       self.assertAlmostEqual(1, miou.eval())
 
@@ -3689,7 +3690,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 3
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
       self.assertAlmostEqual(
           1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
@@ -3752,11 +3753,11 @@ class MeanPerClassAccuracyTest(test.TestCase):
         labels, predictions, num_classes=num_classes)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_mean_accuracy = mean_accuracy.eval()
@@ -3788,9 +3789,9 @@ class MeanPerClassAccuracyTest(test.TestCase):
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0, 1.0 / 3.0, 0.0])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3835,7 +3836,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
 
       variables.local_variables_initializer().run()
       for _ in range(6):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([2.0 / 2.0, 0.5 / 1.5])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3870,9 +3871,9 @@ class MeanPerClassAccuracyTest(test.TestCase):
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0 / 2.0, 2.0 / 3.0, 0.])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3883,7 +3884,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval()[0])
       self.assertEqual(1.0, mean_accuracy.eval())
 
@@ -3894,7 +3895,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([0.0, 0.0], update_op.eval())
       self.assertEqual(0., mean_accuracy.eval())
 
@@ -3913,7 +3914,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes, weights=weights)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       desired_accuracy = np.array([2. / 2., 4. / 6.], dtype=np.float32)
       self.assertAllEqual(desired_accuracy, update_op.eval())
       desired_mean_accuracy = np.mean(desired_accuracy)
@@ -3945,7 +3946,7 @@ class FalseNegativesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
@@ -3964,7 +3965,7 @@ class FalseNegativesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(5., tn_update_op.eval())
       self.assertAllClose(5., tn.eval())
@@ -3994,7 +3995,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), fn.eval())
       self.assertAllEqual((0, 2, 3), fn_update_op.eval())
       self.assertAllEqual((0, 2, 3), fn.eval())
@@ -4013,7 +4014,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), fn.eval())
       self.assertAllEqual((0.0, 8.0, 11.0), fn_update_op.eval())
       self.assertAllEqual((0.0, 8.0, 11.0), fn.eval())
@@ -4044,7 +4045,7 @@ class FalsePositivesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
@@ -4063,7 +4064,7 @@ class FalsePositivesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(14., tn_update_op.eval())
       self.assertAllClose(14., tn.eval())
@@ -4093,7 +4094,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), fp.eval())
       self.assertAllEqual((7, 4, 2), fp_update_op.eval())
       self.assertAllEqual((7, 4, 2), fp.eval())
@@ -4114,7 +4115,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), fp.eval())
       self.assertAllEqual((125.0, 42.0, 12.0), fp_update_op.eval())
       self.assertAllEqual((125.0, 42.0, 12.0), fp.eval())
@@ -4145,7 +4146,7 @@ class TrueNegativesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
@@ -4164,7 +4165,7 @@ class TrueNegativesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(4., tn_update_op.eval())
       self.assertAllClose(4., tn.eval())
@@ -4194,7 +4195,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), tn.eval())
       self.assertAllEqual((2, 5, 7), tn_update_op.eval())
       self.assertAllEqual((2, 5, 7), tn.eval())
@@ -4213,7 +4214,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), tn.eval())
       self.assertAllEqual((5.0, 15.0, 23.0), tn_update_op.eval())
       self.assertAllEqual((5.0, 15.0, 23.0), tn.eval())
@@ -4244,7 +4245,7 @@ class TruePositivesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
@@ -4263,7 +4264,7 @@ class TruePositivesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(12., tn_update_op.eval())
       self.assertAllClose(12., tn.eval())
@@ -4293,7 +4294,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), tp.eval())
       self.assertAllEqual((3, 1, 0), tp_update_op.eval())
       self.assertAllEqual((3, 1, 0), tp.eval())
@@ -4310,7 +4311,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), tp.eval())
       self.assertAllEqual((111.0, 37.0, 0.0), tp_update_op.eval())
       self.assertAllEqual((111.0, 37.0, 0.0), tp.eval())
diff --git a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
index 15e3826542..87f1991aa7 100644
--- a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
@@ -142,8 +142,8 @@ class DepthwiseConv2DTest(test.TestCase):
       conv_interface = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
 
-      native_result = sess.run(conv_native)
-      interface_result = sess.run(conv_interface)
+      native_result = self.evaluate(conv_native)
+      interface_result = self.evaluate(conv_interface)
 
     print("depthwise conv_2d: ", tensor_in_sizes, "*", filter_in_sizes,
           ", stride:", stride, ", padding: ", padding, ", max diff: ",
@@ -211,7 +211,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=filter_in_sizes)
         conv = nn_ops.depthwise_conv2d_native(
             t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        value = sess.run(conv)
+        value = self.evaluate(conv)
     print("value = ", value)
     self.assertAllClose(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py
index e202b6e8a4..5ff0c58bf1 100644
--- a/tensorflow/python/kernel_tests/norm_op_test.py
+++ b/tensorflow/python/kernel_tests/norm_op_test.py
@@ -70,7 +70,7 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_):
         tf_matrix = constant_op.constant(matrix)
         tf_norm = linalg_ops.norm(
             tf_matrix, ord=ord_, axis=axis_, keepdims=keep_dims_)
-        tf_norm_val = sess.run(tf_norm)
+        tf_norm_val = self.evaluate(tf_norm)
       else:
         tf_matrix = array_ops.placeholder(dtype_)
         tf_norm = linalg_ops.norm(
diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py
index 338b6cec01..6cd4974671 100644
--- a/tensorflow/python/kernel_tests/nth_element_op_test.py
+++ b/tensorflow/python/kernel_tests/nth_element_op_test.py
@@ -35,7 +35,7 @@ class NthElementTest(test.TestCase):
     with self.cached_session(use_gpu=False) as sess:
       inputs_op = ops.convert_to_tensor(inputs, dtype=dtype)
       values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse)
-      values = sess.run(values_op)
+      values = self.evaluate(values_op)
 
       self.assertShapeEqual(np_expected_values, values_op)
       self.assertAllClose(np_expected_values, values)
diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
index 520b663375..3696298132 100644
--- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
@@ -126,7 +126,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -193,7 +193,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       results = []
 
@@ -224,7 +224,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        x_val, y_val = sess.run(dequeued_t)
+        x_val, y_val = self.evaluate(dequeued_t)
         x, y = elems[i]
         self.assertEqual([x], x_val)
         self.assertEqual([y], y_val)
@@ -327,7 +327,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -344,7 +344,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -387,17 +387,17 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
@@ -418,7 +418,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertTrue(
@@ -428,11 +428,11 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertTrue(
@@ -459,7 +459,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       for enqueue_op in enqueue_ops:
         enqueue_op.run()
-      string_val, int_val = sess.run(dequeued_t)
+      string_val, int_val = self.evaluate(dequeued_t)
 
       self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""],
                            [b"abc", b"", b""], [b"abc", b"d", b""],
@@ -473,7 +473,7 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      string_val, int_val = sess.run(dequeued_single_t)
+      string_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val)
       self.assertAllEqual([[1, 2, 3, 4]], int_val)
       self.assertTrue(
@@ -500,7 +500,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       for enqueue_op in enqueue_ops:
         enqueue_op.run()
-      string_val, int_val = sess.run(dequeued_t)
+      string_val, int_val = self.evaluate(dequeued_t)
 
       self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""],
                            [b"abc", b"", b""], [b"abc", b"d", b""],
@@ -514,7 +514,7 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      string_val, int_val = sess.run(dequeued_single_t)
+      string_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val)
       self.assertAllEqual([[1, 2, 3, 4]], int_val)
       self.assertTrue(
@@ -633,7 +633,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -700,11 +700,11 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def enqueue():
         for _ in xrange(100):
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       def dequeue():
         for _ in xrange(100):
-          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))
+          self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0))
 
       enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
@@ -736,7 +736,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         for i in xrange(250):
-          self.assertEqual(i, sess.run(dequeued_t))
+          self.assertEqual(i, self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -767,7 +767,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       dequeuemany_t = q.dequeue_many(count_placeholder)
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -805,7 +805,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -832,7 +832,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -901,7 +901,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         for elem in elems:
-          self.assertEqual([elem], sess.run(dequeued_t))
+          self.assertEqual([elem], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -926,8 +926,8 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
-        self.assertAllEqual(elems[3:], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[3:], self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -968,7 +968,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems, sess.run(dequeued_t))
+        self.assertAllEqual(elems, self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -993,7 +993,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -1017,16 +1017,16 @@ class PaddingFIFOQueueTest(test.TestCase):
       cleanup_dequeue_t = q.dequeue()
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        self.assertAllEqual(elems[0:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t))
         with self.assertRaises(errors_impl.OutOfRangeError):
           sess.run(dequeued_t)
-        self.assertEqual(elems[3], sess.run(cleanup_dequeue_t))
+        self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t))
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -1155,7 +1155,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1178,7 +1178,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1207,7 +1207,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # Expect the operation to succeed once the dequeue op runs.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1217,7 +1217,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1242,7 +1242,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1252,7 +1252,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1379,19 +1379,19 @@ class PaddingFIFOQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_op)
+      self.evaluate(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_many_op)
+      self.evaluate(dequeue_many_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_many_op)
+      self.evaluate(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1444,14 +1444,14 @@ class PaddingFIFOQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1517,7 +1517,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       q.enqueue_many(input_tuple).run()
 
       output_tuple_t = q.dequeue_many(32)
-      output_tuple = sess.run(output_tuple_t)
+      output_tuple = self.evaluate(output_tuple_t)
 
       for (input_elem, output_elem) in zip(input_tuple, output_tuple):
         self.assertAllEqual(input_elem, output_elem)
diff --git a/tensorflow/python/kernel_tests/parse_single_example_op_test.py b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
index a84895a287..3f50087282 100644
--- a/tensorflow/python/kernel_tests/parse_single_example_op_test.py
+++ b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
@@ -107,7 +107,7 @@ class ParseExampleTest(test.TestCase):
         for result_dict in [out, out_with_example_name]:
           result = flatten_values_tensors_or_sparse(result_dict.values())
           # Check values.
-          tf_result = sess.run(result)
+          tf_result = self.evaluate(result)
           _compare_output_to_expected(self, result_dict, expected_values,
                                       tf_result)
 
diff --git a/tensorflow/python/kernel_tests/parsing_ops_test.py b/tensorflow/python/kernel_tests/parsing_ops_test.py
index 8f359bd32c..d87adbfc2e 100644
--- a/tensorflow/python/kernel_tests/parsing_ops_test.py
+++ b/tensorflow/python/kernel_tests/parsing_ops_test.py
@@ -101,7 +101,7 @@ class ParseExampleTest(test.TestCase):
         out = parsing_ops.parse_example(**kwargs)
         result = flatten_values_tensors_or_sparse(out.values())
         # Check values.
-        tf_result = sess.run(result)
+        tf_result = self.evaluate(result)
         _compare_output_to_expected(self, out, expected_values, tf_result)
 
       # Check shapes; if serialized is a Tensor we need its size to
@@ -1614,7 +1614,7 @@ class DecodeJSONExampleTest(test.TestCase):
           shape=examples.shape,
           dtype=dtypes.string)
       binary_tensor = parsing_ops.decode_json_example(json_tensor)
-      binary_val = sess.run(binary_tensor)
+      binary_val = self.evaluate(binary_tensor)
 
       if examples.shape:
         self.assertShapeEqual(binary_val, json_tensor)
diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
index e393c7a022..a8e962bc3a 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
@@ -81,7 +81,7 @@ class PoolingTest(test.TestCase):
           data_format=data_format)
       if data_format == "NCDHW":
         t = test_util.NCHWToNHWC(t)
-      vals = sess.run(t)
+      vals = self.evaluate(t)
     # Verifies values.
     actual = vals.flatten()
     self.assertAllClose(expected, actual)
diff --git a/tensorflow/python/kernel_tests/priority_queue_test.py b/tensorflow/python/kernel_tests/priority_queue_test.py
index 73a9c81638..a510fccaaa 100644
--- a/tensorflow/python/kernel_tests/priority_queue_test.py
+++ b/tensorflow/python/kernel_tests/priority_queue_test.py
@@ -50,7 +50,7 @@ class PriorityQueueTest(test.TestCase):
         enq.run()
 
       deq = q.dequeue_many(100)
-      deq_elem, deq_value_0, deq_value_1 = sess.run(deq)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq)
 
       allowed = {}
       missed = set()
@@ -81,7 +81,7 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       dequeue_op = q.dequeue_many(100)
 
@@ -93,7 +93,7 @@ class PriorityQueueTest(test.TestCase):
       for t in enqueue_threads:
         t.start()
 
-      deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op)
 
       for t in enqueue_threads:
         t.join()
@@ -132,12 +132,12 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       dequeued = []
 
       def dequeue(dequeue_op):
-        (dequeue_indices, dequeue_values) = sess.run(dequeue_op)
+        (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op)
         self.assertAllEqual(dequeue_indices, dequeue_values)
         dequeued.extend(dequeue_indices)
 
@@ -184,10 +184,10 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue(dequeue_op, dequeued):
-        (dequeue_indices, dequeue_values) = sess.run(dequeue_op)
+        (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op)
         self.assertAllEqual(dequeue_indices, dequeue_values)
         dequeue_wait.acquire()
         dequeued.extend(dequeue_indices)
@@ -236,7 +236,7 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       dequeue_op = q.dequeue_many(100)
 
@@ -248,7 +248,7 @@ class PriorityQueueTest(test.TestCase):
       for t in enqueue_threads:
         t.start()
 
-      deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op)
 
       for t in enqueue_threads:
         t.join()
@@ -276,7 +276,7 @@ class PriorityQueueTest(test.TestCase):
       side_value_1 = np.random.rand(1000).astype(bytes)
       q.enqueue_many((elem, side_value_0, side_value_1)).run()
       deq = q.dequeue_many(1000)
-      deq_elem, deq_value_0, deq_value_1 = sess.run(deq)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq)
 
       allowed = {}
       for e, v0, v1 in zip(elem, side_value_0, side_value_1):
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index b101da036e..c9cbe44a7f 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -307,9 +307,9 @@ class PyFuncTest(test.TestCase):
     with session_lib.Session() as sess:
       producer = iter(range(3))
       x, = script_ops.py_func(lambda: next(producer), [], [dtypes.int64])
-      self.assertEqual(sess.run(x), 0)
-      self.assertEqual(sess.run(x), 1)
-      self.assertEqual(sess.run(x), 2)
+      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(self.evaluate(x), 1)
+      self.assertEqual(self.evaluate(x), 2)
 
   def testStateless(self):
     # Not using self.cached_session(), which disables optimization.
@@ -317,9 +317,9 @@ class PyFuncTest(test.TestCase):
       producer = iter(range(3))
       x, = script_ops.py_func(
           lambda: next(producer), [], [dtypes.int64], stateful=False)
-      self.assertEqual(sess.run(x), 0)
-      self.assertEqual(sess.run(x), 0)
-      self.assertEqual(sess.run(x), 0)
+      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(self.evaluate(x), 0)
 
   def testGradientFunction(self):
     # Input to tf.py_func is necessary, otherwise get_gradient_function()
@@ -390,7 +390,7 @@ class PyFuncTest(test.TestCase):
     f = script_ops.py_func(
         do_nothing, [constant_op.constant(3, dtypes.int64)], [], stateful=False)
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(f), [])
+      self.assertEqual(self.evaluate(f), [])
 
   def _testExceptionHandling(self, py_exp, tf_exp, eager=False):
 
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 617b724204..114481ed6a 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -60,7 +60,7 @@ class QrOpTest(test.TestCase):
             q1, r1 = linalg_ops.qr(matrix1, full_matrices=full_matrices_)
             q2, r2 = linalg_ops.qr(matrix2, full_matrices=full_matrices_)
             all_ops += [q1, r1, q2, r2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       for i in range(8):
         q = 4 * i
         self.assertAllEqual(val[q], val[q + 2])  # q1 == q2
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
index 0023506b77..cab841741e 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
@@ -39,7 +39,7 @@ class MultinomialTest(test.TestCase):
           num_samples=1000000,
           seed=15)
       for _ in range(100):
-        x = sess.run(samples)
+        x = self.evaluate(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
@@ -57,7 +57,7 @@ class MultinomialTest(test.TestCase):
           num_samples=1000000,
           seed=15)
       for _ in range(100):
-        x = sess.run(samples)
+        x = self.evaluate(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
@@ -79,7 +79,7 @@ class MultinomialTest(test.TestCase):
       # we'll run out of memory if we try to draw 1e9 samples directly
       # really should fit in 12GB of memory...
       for _ in range(100):
-        x = sess.run(samples)
+        x = self.evaluate(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
index cfec4d08fb..8d2718c6d5 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
@@ -70,8 +70,8 @@ class MultinomialTest(test.TestCase):
     with self.test_session(use_gpu=True) as sess:
       sample_op1, _ = self._make_ops(10)
       # Consecutive runs shouldn't yield identical output.
-      sample1a = sess.run(sample_op1)
-      sample1b = sess.run(sample_op1)
+      sample1a = self.evaluate(sample_op1)
+      sample1b = self.evaluate(sample_op1)
       self.assertFalse(np.equal(sample1a, sample1b).all())
 
   def testEagerOneOpMultipleStepsIndependent(self):
@@ -160,7 +160,7 @@ class MultinomialTest(test.TestCase):
     with self.test_session(use_gpu=True) as sess:
       random_seed.set_random_seed(1618)
       op = sampler(constant_op.constant(logits), num_samples)
-      d = sess.run(op)
+      d = self.evaluate(op)
 
     batch_size, num_classes = logits.shape
     freqs_mat = []
@@ -225,8 +225,10 @@ def native_op_vs_composed_ops(batch_size, num_classes, num_samples, num_iters):
     native_op = control_flow_ops.group(native_sampler(logits, num_samples))
     composed_op = control_flow_ops.group(composed_sampler(logits, num_samples))
 
-    native_dt = timeit.timeit(lambda: sess.run(native_op), number=num_iters)
-    composed_dt = timeit.timeit(lambda: sess.run(composed_op), number=num_iters)
+    native_dt = timeit.timeit(
+        lambda: sess.run(native_op), number=num_iters)
+    composed_dt = timeit.timeit(
+        lambda: sess.run(composed_op), number=num_iters)
     return native_dt, composed_dt
 
 
diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py
index 606e8862c4..d18e3feb04 100644
--- a/tensorflow/python/kernel_tests/random/random_gamma_test.py
+++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py
@@ -48,7 +48,7 @@ class RandomGammaTest(test.TestCase):
             [num], alpha, beta=beta, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
index 6de894846b..76618316b2 100644
--- a/tensorflow/python/kernel_tests/random/random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/random_ops_test.py
@@ -49,9 +49,9 @@ class RandomOpTestCommon(test.TestCase):
         random_seed.set_random_seed(graph_seed)
       x = rng_func([num], min_or_mean, max_or_stddev, dtype=dtype, seed=op_seed)
 
-      y = sess.run(x)
-      z = sess.run(x)
-      w = sess.run(x)
+      y = self.evaluate(x)
+      z = self.evaluate(x)
+      w = self.evaluate(x)
 
       # We use exact equality here. If the random-number generator is producing
       # the same output, all three outputs will be bitwise identical.
@@ -69,7 +69,7 @@ class RandomNormalTest(RandomOpTestCommon):
             [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
@@ -160,7 +160,7 @@ class TruncatedNormalTest(test.TestCase):
             [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
@@ -256,7 +256,7 @@ class RandomUniformTest(RandomOpTestCommon):
             [num], minval=minv, maxval=maxv, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
index 95e48101f6..47c0858db7 100644
--- a/tensorflow/python/kernel_tests/random/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py
@@ -43,7 +43,7 @@ class RandomPoissonTest(test.TestCase):
         rng = random_ops.random_poisson(lam, [num], dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
index f3fcf1eff7..5601b9864b 100644
--- a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
@@ -84,7 +84,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeue_t = q.dequeue()
       results = []
       for _ in range(2):
-        a, b = sess.run(dequeue_t)
+        a, b = self.evaluate(dequeue_t)
         results.append((a, b))
       a, b = sess.run(q.dequeue_many(3))
       for i in range(3):
@@ -101,7 +101,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -167,7 +167,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       results = []
 
@@ -197,7 +197,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
       for _ in xrange(len(elems)):
-        x, y = sess.run(dequeued_t)
+        x, y = self.evaluate(dequeued_t)
         results.append((x, y))
       self.assertItemsEqual(elems, results)
 
@@ -321,7 +321,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
       for _ in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         results.append((float_val, [int_val[0], int_val[1]]))
       expected = list(zip(float_elems, int_elems)) * 2
       self.assertItemsEqual(expected, results)
@@ -368,20 +368,20 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       results = []
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape())
       results.append((float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       results.append((float_val, int_val.tolist()))
 
       self.assertItemsEqual(zip(float_elems, int_elems), results)
@@ -402,21 +402,21 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       results = []
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       # dequeue_up_to has undefined shape.
       self.assertEqual([None], dequeued_t[0].get_shape().as_list())
       self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list())
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape())
       results.append((float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       results.append((float_val, int_val.tolist()))
 
       self.assertItemsEqual(zip(float_elems, int_elems), results)
@@ -442,7 +442,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -515,7 +515,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue(dequeue_op):
-        dequeued_elems.extend(sess.run(dequeue_op))
+        dequeued_elems.extend(self.evaluate(dequeue_op))
 
       threads = []
       for dequeue_op in dequeue_ops:
@@ -539,7 +539,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -566,7 +566,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -727,7 +727,7 @@ class RandomShuffleQueueTest(test.TestCase):
       progress = []  # Must be mutable
 
       def dequeue():
-        self.assertItemsEqual(elems, sess.run(dequeued_t))
+        self.assertItemsEqual(elems, self.evaluate(dequeued_t))
         progress.append(1)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
@@ -922,7 +922,7 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -950,7 +950,7 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -987,11 +987,11 @@ class RandomShuffleQueueTest(test.TestCase):
       def blocking_enqueue():
         # Expect the operation to succeed since it will complete
         # before the queue is closed.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"):
-          sess.run(blocking_enqueue_op)
+          self.evaluate(blocking_enqueue_op)
 
       thread1 = self.checkedThread(target=blocking_enqueue)
       thread1.start()
@@ -1001,7 +1001,7 @@ class RandomShuffleQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def blocking_close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       thread2 = self.checkedThread(target=blocking_close)
       thread2.start()
@@ -1032,7 +1032,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # This will block until the dequeue after the close.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread1 = self.checkedThread(target=blocking_enqueue)
       thread1.start()
@@ -1050,7 +1050,7 @@ class RandomShuffleQueueTest(test.TestCase):
         time.sleep(0.1)
 
       def blocking_close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       thread2 = self.checkedThread(target=blocking_close)
       thread2.start()
@@ -1064,7 +1064,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # At this point the close operation will complete, so the next enqueue
       # will fail.
       with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"):
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
   def testSharedQueueSameSession(self):
     with self.cached_session():
@@ -1216,23 +1216,23 @@ class RandomShuffleQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_op)
+      self.evaluate(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_many_op)
+      self.evaluate(dequeue_many_op)
 
   def _blockingDequeueUpTo(self, sess, dequeue_up_to_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_up_to_op)
+      self.evaluate(dequeue_up_to_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_many_op)
+      self.evaluate(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1393,14 +1393,14 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
diff --git a/tensorflow/python/kernel_tests/reader_ops_test.py b/tensorflow/python/kernel_tests/reader_ops_test.py
index 18a8a3d547..4d9b26f4eb 100644
--- a/tensorflow/python/kernel_tests/reader_ops_test.py
+++ b/tensorflow/python/kernel_tests/reader_ops_test.py
@@ -724,7 +724,7 @@ class AsyncReaderTest(test.TestCase):
         thread_data.append(thread_data_t(t, queue, output))
 
       # Start all readers. They are all blocked waiting for queue entries.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for d in thread_data:
         d.thread.start()
 
diff --git a/tensorflow/python/kernel_tests/record_input_test.py b/tensorflow/python/kernel_tests/record_input_test.py
index ebb9872f22..74020667d9 100644
--- a/tensorflow/python/kernel_tests/record_input_test.py
+++ b/tensorflow/python/kernel_tests/record_input_test.py
@@ -54,7 +54,7 @@ class RecordInputOpTest(test.TestCase):
           batch_size=1,
           name="record_input").get_yield_op()
 
-      self.assertEqual(sess.run(yield_op), b"0000000000")
+      self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
   def testRecordInputSimpleGzip(self):
     with self.cached_session() as sess:
@@ -73,7 +73,7 @@ class RecordInputOpTest(test.TestCase):
           compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op(
           )
 
-      self.assertEqual(sess.run(yield_op), b"0000000000")
+      self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
   def testRecordInputSimpleZlib(self):
     with self.cached_session() as sess:
@@ -92,7 +92,7 @@ class RecordInputOpTest(test.TestCase):
           compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op(
           )
 
-      self.assertEqual(sess.run(yield_op), b"0000000000")
+      self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
   def testRecordInputEpochs(self):
     files = 100
@@ -117,7 +117,7 @@ class RecordInputOpTest(test.TestCase):
       for _ in range(3):
         epoch_set = set()
         for _ in range(int(files * records_per_file / batches)):
-          op_list = sess.run(yield_op)
+          op_list = self.evaluate(yield_op)
           self.assertTrue(len(op_list) is batches)
           for r in op_list:
             self.assertTrue(r[0] not in epoch_set)
@@ -138,15 +138,15 @@ class RecordInputOpTest(test.TestCase):
 
         yield_op = records.get_yield_op()
         for _ in range(50):
-          sess.run(yield_op)
+          self.evaluate(yield_op)
 
   def testEmptyGlob(self):
     with self.cached_session() as sess:
       record_input = data_flow_ops.RecordInput(file_pattern="foo")
       yield_op = record_input.get_yield_op()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       with self.assertRaises(NotFoundError):
-        sess.run(yield_op)
+        self.evaluate(yield_op)
 
   def testBufferTooSmall(self):
     files = 10
@@ -171,7 +171,7 @@ class RecordInputOpTest(test.TestCase):
       for _ in range(3):
         epoch_set = set()
         for _ in range(int(files * records_per_file / batches)):
-          op_list = sess.run(yield_op)
+          op_list = self.evaluate(yield_op)
           self.assertTrue(len(op_list) is batches)
           for r in op_list:
             self.assertTrue(r[0] not in epoch_set)
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index d1a295f42b..612b2c56a5 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -185,7 +185,7 @@ class SumReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -216,7 +216,7 @@ class SumReductionTest(BaseReductionTest):
       tf_arr = variables.Variable(arr)
       variables.global_variables_initializer().run()
       tf_mean = math_ops.reduce_mean(tf_arr, 0, False)
-      tf_out_mean = sess.run(tf_mean)
+      tf_out_mean = self.evaluate(tf_mean)
     self.assertAllClose(tf_out_mean, 1.)
 
   def testFloat32(self):
@@ -400,7 +400,7 @@ class MeanReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -473,7 +473,7 @@ class ProdReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -576,7 +576,7 @@ class MinReductionTest(test.TestCase):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -689,7 +689,7 @@ class MaxReductionTest(test.TestCase):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -817,7 +817,7 @@ class AllReductionTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         v = math_ops.reduce_all([True, True],
                                 constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, True)
 
   def testAll3D(self):
@@ -866,7 +866,7 @@ class AnyReductionTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         v = math_ops.reduce_any([True, True],
                                 constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, True)
 
   def testAll3D(self):
@@ -962,7 +962,7 @@ class CountNonzeroReductionTest(test.TestCase):
     # Test case for GitHub issue 18712
     with self.cached_session() as sess:
       v = math_ops.count_nonzero(constant_op.constant(["test"]))
-      self.assertAllClose(sess.run(v), 1)
+      self.assertAllClose(self.evaluate(v), 1)
 
   def testStringReduce1D(self):
     # Create a 1D array of strings
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 45b9ede813..eedc2d263d 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -153,7 +153,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
   def testCachedValueReadBeforeWrite(self):
     with self.cached_session() as sess:
       v = resource_variable_ops.ResourceVariable(0.0, caching_device="cpu:0")
-      sess.run(v.initializer)
+      self.evaluate(v.initializer)
       value, _ = sess.run([v, v.assign_add(1.0)])
       self.assertAllEqual(value, 0.0)
 
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index 952ef34456..1f1249727c 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -162,7 +162,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with self.session(use_gpu=True) as sess:
       sess.run(init)
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
   def testSimpleResource(self):
@@ -190,7 +190,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with self.session(use_gpu=True) as sess:
       sess.run(init)
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
   def testSimple3(self):
@@ -204,7 +204,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with self.session(use_gpu=True) as sess:
       sess.run(init)
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
   def testVariableRankUpdate(self):
@@ -342,7 +342,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with session.Session() as sess:
       sess.run(init)
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       assert np.allclose(result, expected_result)
 
   # TODO(fpmc): Re-enable this test when gpu_pip test actually runs on a GPU.
@@ -421,7 +421,7 @@ class ScatterNdTest(test.TestCase):
                          b"", b"", b"seven"])
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllEqual(expected, result)
 
     # Same indice is updated twice by same value.
@@ -432,7 +432,7 @@ class ScatterNdTest(test.TestCase):
     expected = np.array([b"", b"", b"", b"bb", b"a", b"", b"", b"c"])
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllEqual(expected, result)
 
     # Same indice is updated twice by different value.
@@ -444,7 +444,7 @@ class ScatterNdTest(test.TestCase):
                 np.array([b"", b"", b"", b"cb", b"a", b"", b"", b"d"])]
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertTrue(np.array_equal(result, expected[0]) or
                       np.array_equal(result, expected[1]))
 
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 85756b769d..8ca8e9dddf 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -63,7 +63,7 @@ class SelfAdjointEigTest(test.TestCase):
           e1 = linalg_ops.self_adjoint_eigvals(matrix1)
           e2 = linalg_ops.self_adjoint_eigvals(matrix2)
           all_ops += [e1, e2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       self.assertAllEqual(val[0], val[2])
       # The algorithm is slightly different for compute_v being True and False,
       # so require approximate equality only here.
diff --git a/tensorflow/python/kernel_tests/session_ops_test.py b/tensorflow/python/kernel_tests/session_ops_test.py
index 03e1ae852f..73d85ddc07 100644
--- a/tensorflow/python/kernel_tests/session_ops_test.py
+++ b/tensorflow/python/kernel_tests/session_ops_test.py
@@ -37,7 +37,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Feed a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -51,7 +51,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Get the tensor from its handle.
       self.assertEqual(50, h.eval())
@@ -94,7 +94,7 @@ class SessionOpsTest(test.TestCase):
       # Initialize a handle.
       a = constant_op.constant(0)
       h = session_ops.get_session_handle(a)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Do some computation.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -111,7 +111,7 @@ class SessionOpsTest(test.TestCase):
       # Initialize a handle.
       a = constant_op.constant(0)
       h = session_ops.get_session_handle(a)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Do some computation.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -133,7 +133,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Feed a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -144,7 +144,7 @@ class SessionOpsTest(test.TestCase):
       with ops.device(test.gpu_device_name()):
         a = constant_op.constant(10)
         h = session_ops.get_session_handle(a)
-        h = sess.run(h)
+        h = self.evaluate(h)
         self.assertEqual(100, sess.run(y, feed_dict={f: h.handle}))
 
   def testHandleDelete(self):
@@ -163,7 +163,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Delete using a raw tensor handle.
       raw_h = h.get_raw_handle()
@@ -219,8 +219,8 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(2.0)
       b_handle_op = session_ops.get_session_handle(b)
 
-      a_handle = sess.run(a_handle_op)
-      b_handle = sess.run(b_handle_op)
+      a_handle = self.evaluate(a_handle_op)
+      b_handle = self.evaluate(b_handle_op)
 
       a_p, a_t = session_ops.get_session_tensor(a_handle.handle, dtypes.float32)
       b_p, b_t = session_ops.get_session_tensor(b_handle.handle, dtypes.float32)
@@ -288,10 +288,10 @@ class SessionOpsTest(test.TestCase):
       a = variables.Variable(12.0)
       inc_a = state_ops.assign_add(a, 2.0)
       b = math_ops.add(a, 5.0)
-      sess.run(a.initializer)
+      self.evaluate(a.initializer)
 
       h_a_read = sess.run(session_ops.get_session_handle(a.read_value()))
-      self.assertAllClose(12.0, sess.run(a))
+      self.assertAllClose(12.0, self.evaluate(a))
 
       self.assertAllClose(17.0, sess.run(b, feed_dict={a: h_a_read}))
       sess.run(inc_a)
diff --git a/tensorflow/python/kernel_tests/sets_test.py b/tensorflow/python/kernel_tests/sets_test.py
index 8335e9c139..e037f51e0f 100644
--- a/tensorflow/python/kernel_tests/sets_test.py
+++ b/tensorflow/python/kernel_tests/sets_test.py
@@ -159,7 +159,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(None, op.get_shape().dims)
       self.assertEqual(dtypes.int32, op.dtype)
     with self.cached_session() as sess:
-      results = sess.run(ops)
+      results = self.evaluate(ops)
     self.assertAllEqual(results[0], results[1])
     return results[0]
 
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index 3e0eae326b..a0506fbfc5 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -73,8 +73,8 @@ class ShapeOpsTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       tf_ans = array_ops.shape_n([x, x, x])
       tf_ans_64 = array_ops.shape_n([x, x, x], out_type=dtypes.int64)
-      result = sess.run(tf_ans)
-      result_64 = sess.run(tf_ans_64)
+      result = self.evaluate(tf_ans)
+      result_64 = self.evaluate(tf_ans_64)
     for i in range(3):
       self.assertAllEqual(np_ans, result[i])
       self.assertAllEqual(np_ans, result_64[i])
diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index c4e5b6f674..de3351e543 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -56,7 +56,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, 2)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
 
       expected_output = np.array([1, 1, 2, 2, 3, 2, 2, 1, 1])
 
@@ -99,7 +99,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
       string_output = [np.base_repr(x, self.bases[0]) for x in output]
 
       self.assertEqual(string_output, self.expected_string)
@@ -109,7 +109,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
 
       accumulator = True
       for i in range(self.batch_size):
@@ -125,7 +125,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
 
       string_output = [np.base_repr(int(x), self.bases[0]) for x in
                        np.squeeze(output)]
diff --git a/tensorflow/python/kernel_tests/sparse_add_op_test.py b/tensorflow/python/kernel_tests/sparse_add_op_test.py
index a746830afb..845950bca7 100644
--- a/tensorflow/python/kernel_tests/sparse_add_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_add_op_test.py
@@ -91,7 +91,7 @@ class SparseAddTest(test.TestCase):
           sp_sum = sparse_ops.sparse_add(sp_a, sp_b)
           self.assertAllEqual((3, 3), sp_sum.get_shape())
 
-          sum_out = sess.run(sp_sum)
+          sum_out = self.evaluate(sp_sum)
 
           self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
           self.assertAllEqual(sum_out.indices, [[0, 1], [1, 0], [2, 0], [2, 1]])
@@ -104,7 +104,7 @@ class SparseAddTest(test.TestCase):
       sp_b = self._SparseTensor_3x3(negate=True)
 
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, 0.1)
-      sum_out = sess.run(sp_sum)
+      sum_out = self.evaluate(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, np.empty([0, 2]))
@@ -123,7 +123,7 @@ class SparseAddTest(test.TestCase):
 
       # two values should vanish: |.1| < .21, and |-.2| < .21
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.21)
-      sum_out = sess.run(sp_sum)
+      sum_out = self.evaluate(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0]])
@@ -132,7 +132,7 @@ class SparseAddTest(test.TestCase):
 
       # only .1 vanishes
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.11)
-      sum_out = sess.run(sp_sum)
+      sum_out = self.evaluate(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0], [2, 1]])
diff --git a/tensorflow/python/kernel_tests/sparse_concat_op_test.py b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
index 402c5eb4ea..a3d136c8d5 100644
--- a/tensorflow/python/kernel_tests/sparse_concat_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
@@ -147,7 +147,7 @@ class SparseConcatTest(test.TestCase):
           self.assertEqual(sp_concat.values.get_shape(), [4])
           self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-          concat_out = sess.run(sp_concat)
+          concat_out = self.evaluate(sp_concat)
 
           self.assertAllEqual(concat_out.indices,
                               [[0, 2], [1, 0], [2, 0], [2, 2]])
@@ -169,7 +169,7 @@ class SparseConcatTest(test.TestCase):
             self.assertEqual(sp_concat.values.get_shape(), [8])
             self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-            concat_out = sess.run(sp_concat)
+            concat_out = self.evaluate(sp_concat)
 
             self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4],
                                                      [2, 0], [2, 2], [2, 3],
@@ -195,7 +195,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [7])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = sess.run(sp_concat)
+        concat_out = self.evaluate(sp_concat)
 
         self.assertAllEqual(
             concat_out.indices,
@@ -220,7 +220,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [10])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = sess.run(sp_concat)
+        concat_out = self.evaluate(sp_concat)
 
         self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [1, 8],
                                                  [2, 0], [2, 2], [2, 3], [2, 6],
@@ -244,7 +244,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [8])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = sess.run(sp_concat)
+        concat_out = self.evaluate(sp_concat)
 
         self.assertAllEqual(
             concat_out.indices,
@@ -302,8 +302,8 @@ class SparseConcatTest(test.TestCase):
           sp_concat_dim1 = sparse_ops.sparse_concat(
               concat_dim1, [sp_a, sp_b, sp_c, sp_d], expand_nonconcat_dim=True)
 
-          sp_concat_dim0_out = sess.run(sp_concat_dim0)
-          sp_concat_dim1_out = sess.run(sp_concat_dim1)
+          sp_concat_dim0_out = self.evaluate(sp_concat_dim0)
+          sp_concat_dim1_out = self.evaluate(sp_concat_dim1)
 
           self.assertAllEqual(sp_concat_dim0_out.indices,
                               [[0, 2], [1, 0], [2, 0], [2, 2], [4, 1], [5, 0],
diff --git a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
index a824d5c826..267275e771 100644
--- a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
@@ -189,7 +189,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual([0, 1, 2], val.indices)
       self.assertAllEqual([[0.5, 0.5], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
@@ -209,7 +209,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual([0, 1, 2], val.indices)
       self.assertAllEqual([[1, 1], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
@@ -235,7 +235,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual(val.indices, [0, 1, 2])
       self.assertAllEqual(val.values, [[0.5, 0.5], [0, 2], [3, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
@@ -252,7 +252,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual(val.indices, [0, 1, 2])
       self.assertAllEqual(val.values, [[5, 5], [0, 20], [30, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
@@ -269,7 +269,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_indexed_slices_grad(1)
 
       def apply_indexed_slices_grad(accum_op):
-        sess.run(accum_op)
+        self.evaluate(accum_op)
 
       threads = [
           self.checkedThread(
@@ -281,7 +281,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
 
       expected_val = sum(elems) / len(elems)
       self._assertEqual_nparray(
@@ -303,7 +303,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_indexed_slices_grad(1)
 
       def apply_indexed_slices_grad(accum_op):
-        sess.run(accum_op)
+        self.evaluate(accum_op)
 
       threads = [
           self.checkedThread(target=apply_indexed_slices_grad, args=(o,))
@@ -315,7 +315,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
 
       expected_val = 550.0
       self._assertEqual_nparray(
@@ -338,13 +338,13 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       def apply_indexed_slices_grad():
         for accum_op in accum_ops:
           time.sleep(1.0)
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       apply_indexed_slices_grad_thread = self.checkedThread(
           target=apply_indexed_slices_grad)
 
       def take_grad():
-        t = sess.run(takeg_t)
+        t = self.evaluate(takeg_t)
         results.append(t)
 
       threads = [self.checkedThread(target=take_grad) for _ in range(10)]
@@ -378,7 +378,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
       def apply_indexed_slices_grad():
         for accum_op in accum_ops:
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       def take_grad():
         results.append(sess.run(takeg_t))
@@ -394,7 +394,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
   def _blocking_takeg(self, sess, takeg_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(takeg_op)
+      self.evaluate(takeg_op)
 
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
index 17e867439a..8451b96c56 100644
--- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
@@ -43,7 +43,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_dense(self):
     """Tests only dense inputs."""
@@ -63,7 +63,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_integer_mixed_string_sparse(self):
     """Tests mixed type."""
@@ -77,7 +77,7 @@ class SparseCrossOpTest(test.TestCase):
         '55555_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_integer_mixed_string_dense(self):
     """Tests mixed dense inputs."""
@@ -95,7 +95,7 @@ class SparseCrossOpTest(test.TestCase):
         '999999_X_batch2-FC2-F1', '999999_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_sparse_cross_dense(self):
     """Tests sparse and dense inputs."""
@@ -112,7 +112,7 @@ class SparseCrossOpTest(test.TestCase):
             'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
         ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_integer_sparse_input(self):
     """Tests mixed type sparse and dense inputs."""
@@ -128,7 +128,7 @@ class SparseCrossOpTest(test.TestCase):
             '5555_X_batch2-FC2-F1', '5555_X_batch2-FC2-F2'
         ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_permutation_3x3x3(self):
     """Tests 3x3x3 permutation."""
@@ -170,7 +170,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F3'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_permutation_3x1x2(self):
     """Tests 3x1x2 permutation."""
@@ -189,7 +189,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_large_batch(self):
     """Tests with large batch size to force multithreading."""
@@ -222,7 +222,7 @@ class SparseCrossOpTest(test.TestCase):
 
     expected_out = self._sparse_tensor(col_out)
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_one_column_empty(self):
     """Tests when one column is empty.
@@ -235,7 +235,7 @@ class SparseCrossOpTest(test.TestCase):
         self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']])
     ])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_empty(sess.run(op))
+      self._assert_sparse_tensor_empty(self.evaluate(op))
 
   def test_some_columns_empty(self):
     """Tests when more than one columns are empty.
@@ -254,7 +254,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2'
     ]], 2)
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_all_columns_empty(self):
     """Tests when all columns are empty.
@@ -267,7 +267,7 @@ class SparseCrossOpTest(test.TestCase):
         self._sparse_tensor([])
     ])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_empty(sess.run(op))
+      self._assert_sparse_tensor_empty(self.evaluate(op))
 
   def test_hashed_zero_bucket_no_hash_key(self):
     op = sparse_ops.sparse_cross_hashed([
@@ -278,7 +278,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[1971693436396284976]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_hashed_zero_bucket(self):
     op = sparse_ops.sparse_cross_hashed(
@@ -291,7 +291,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[4847552627144134031]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   # TODO(sibyl-Aix6ihai): Add benchmark to compare Hashed vs Non-hashed.
   def test_hashed_no_hash_key(self):
@@ -305,7 +305,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[83]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_hashed_output(self):
     op = sparse_ops.sparse_cross_hashed(
@@ -319,7 +319,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[31]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_hashed__has_no_collision(self):
     """Tests that fingerprint concatenation has no collisions."""
@@ -345,7 +345,7 @@ class SparseCrossOpTest(test.TestCase):
         ],
         num_buckets=1000)
     with self.cached_session() as sess:
-      out = sess.run(op)
+      out = self.evaluate(op)
       self.assertEqual(6, len(out.values))
       self.assertAllEqual([[0, i] for i in range(6)], out.indices)
       self.assertTrue(all(x < 1000 and x >= 0 for x in out.values))
diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index db3f6c44e2..ad253595d2 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -154,7 +154,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
                        sparse_tensor.SparseTensor.from_value(values_v)):
           sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-          output = sess.run(sp_output)
+          output = self.evaluate(sp_output)
           self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat32(self):
@@ -163,7 +163,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float32)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64(self):
@@ -172,7 +172,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt32AndFloat32NonCanonicalOrder(self):
@@ -182,7 +182,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size, already_sorted=True)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testInt64AndFloat32NonCanonicalOrder(self):
@@ -192,7 +192,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size, already_sorted=True)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testInt64AndFloat64NonCanonicalOrder(self):
@@ -203,7 +203,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size_tensor, already_sorted=True)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testShouldSetLastDimensionInDynamicShape(self):
@@ -261,7 +261,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float32)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64(self):
@@ -270,7 +270,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64Shape(self):
@@ -279,7 +279,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
 
@@ -302,7 +302,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
         to_retain = np.array([1, 0, 0, 1, 1, 0], dtype=np.bool)
         sp_output = sparse_ops.sparse_retain(sp_input, to_retain)
 
-        output = sess.run(sp_output)
+        output = self.evaluate(sp_output)
 
         self.assertAllEqual(output.indices, [[0, 0], [1, 4], [3, 2]])
         self.assertAllEqual(output.values, [0, 14, 32])
@@ -314,7 +314,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
       to_retain = np.zeros((6,), dtype=np.bool)
       sp_output = sparse_ops.sparse_retain(sp_input, to_retain)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, np.array([]).reshape((0, 2)))
       self.assertAllEqual(output.values, [])
@@ -365,7 +365,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       new_shape = np.array([3, 6, 7], dtype=np.int64)
       sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -378,7 +378,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       new_shape = np.array([3, 6, 7], dtype=np.int64)
       sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -404,7 +404,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       sp_input = self._SparseTensor_2x5x6()
       sp_output = sparse_ops.sparse_reset_shape(sp_input)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -416,7 +416,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       sp_input = self._SparseTensor_2x5x6_Empty()
       sp_output = sparse_ops.sparse_reset_shape(sp_input)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices.shape, [0, 3])
       self.assertAllEqual(output.values.shape, [0])
@@ -591,8 +591,8 @@ class SparseAddTest(test_util.TensorFlowTestCase):
     sp_output = sparse_ops.sparse_add(sp_input, sp_input)
 
     with self.session(use_gpu=False) as sess:
-      sess.run(variables.global_variables_initializer())
-      output = sess.run(sp_output)
+      self.evaluate(variables.global_variables_initializer())
+      output = self.evaluate(sp_output)
       self.assertAllEqual(output.values, [2])
 
 
diff --git a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
index 7b83ae5177..bbf2f39202 100644
--- a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
@@ -60,7 +60,7 @@ class SparseReorderTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6(np.arange(6))
       sp_output = sparse_ops.sparse_reorder(input_val)
 
-      output_val = sess.run(sp_output)
+      output_val = self.evaluate(sp_output)
       self.assertAllEqual(output_val.indices, input_val.indices)
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
@@ -83,7 +83,7 @@ class SparseReorderTest(test.TestCase):
         input_val = self._SparseTensorValue_5x6(np.random.permutation(6))
         sp_output = sparse_ops.sparse_reorder(input_val)
 
-        output_val = sess.run(sp_output)
+        output_val = self.evaluate(sp_output)
         self.assertAllEqual(output_val.indices, expected_output_val.indices)
         self.assertAllEqual(output_val.values, expected_output_val.values)
         self.assertAllEqual(output_val.dense_shape,
diff --git a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
index f7be397c33..918af27091 100644
--- a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
@@ -81,7 +81,7 @@ class SparseReshapeTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6()
       sp_output = sparse_ops.sparse_reshape(input_val, [5, 6])
 
-      output_val = sess.run(sp_output)
+      output_val = self.evaluate(sp_output)
       self.assertAllEqual(output_val.indices, input_val.indices)
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
@@ -151,7 +151,7 @@ class SparseReshapeTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6()
       sp_output = sparse_ops.sparse_reshape(input_val, [2, 3, 5])
 
-      output_val = sess.run(sp_output)
+      output_val = self.evaluate(sp_output)
       self.assertAllEqual(output_val.indices,
                           np.array([[0, 0, 0], [0, 1, 1], [0, 1, 4], [0, 2, 0],
                                     [1, 1, 0], [1, 1, 1]]))
diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
index b24a086969..39a9ab9b49 100644
--- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
@@ -73,7 +73,7 @@ class SerializeSparseTest(test.TestCase):
       serialized = serialize_fn(sp_input, out_type=out_type)
       sp_deserialized = deserialize_fn(serialized, dtype=dtypes.int32)
 
-      indices, values, shape = sess.run(sp_deserialized)
+      indices, values, shape = self.evaluate(sp_deserialized)
 
       self.assertAllEqual(indices, sp_input[0])
       self.assertAllEqual(values, sp_input[1])
diff --git a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
index e08464a701..e63ba8f697 100644
--- a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
@@ -88,7 +88,7 @@ class SparseTensorsMapTest(test.TestCase):
       sp_out = take_many_sparse_from_tensors_map(
           sparse_map_op=handle0.op, sparse_handles=handles_concat)
 
-      combined_indices, combined_values, combined_shape = sess.run(sp_out)
+      combined_indices, combined_values, combined_shape = self.evaluate(sp_out)
 
       self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
       self.assertAllEqual(combined_indices[:6, 1:], sp_input0[0])
@@ -114,7 +114,8 @@ class SparseTensorsMapTest(test.TestCase):
       sp_roundtrip = take_many_sparse_from_tensors_map(
           sparse_map_op=handle.op, sparse_handles=sparse_handles)
 
-      combined_indices, combined_values, combined_shape = sess.run(sp_roundtrip)
+      combined_indices, combined_values, combined_shape = self.evaluate(
+          sp_roundtrip)
 
       self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
       self.assertAllEqual(combined_indices[:6, 1:], input0_val[0])
@@ -171,7 +172,7 @@ class SparseTensorsMapTest(test.TestCase):
     with self.session(use_gpu=False) as sess:
       input_val = self._SparseTensorValue_5x6(np.arange(6))
       handle = add_sparse_to_tensors_map(input_val)
-      handle_value = sess.run(handle)
+      handle_value = self.evaluate(handle)
       bad_handle = handle_value + 10
       sp_roundtrip = take_many_sparse_from_tensors_map(
           sparse_map_op=handle.op, sparse_handles=[handle_value, bad_handle])
@@ -212,8 +213,8 @@ class BenchmarkSparseTensorsMapVsSerialization(test.Benchmark):
 
         variables.global_variables_initializer().run()
 
-        st_roundtrip_values = sess.run(st_roundtrip)
-        st_deserialized_values = sess.run(st_deserialized)
+        st_roundtrip_values = self.evaluate(st_roundtrip)
+        st_deserialized_values = self.evaluate(st_deserialized)
         np.testing.assert_equal(st_roundtrip_values.values,
                                 st_deserialized_values.values)
         np.testing.assert_equal(st_roundtrip_values.indices,
diff --git a/tensorflow/python/kernel_tests/stage_op_test.py b/tensorflow/python/kernel_tests/stage_op_test.py
index b814843b86..b1e7ce5d62 100644
--- a/tensorflow/python/kernel_tests/stage_op_test.py
+++ b/tensorflow/python/kernel_tests/stage_op_test.py
@@ -152,11 +152,11 @@ class StageTest(test.TestCase):
 
     with self.session(use_gpu=True, graph=G) as sess:
       sess.run(stage, feed_dict={x: -1})
-      self.assertEqual(sess.run(size), 1)
+      self.assertEqual(self.evaluate(size), 1)
       sess.run(stage, feed_dict={x: -1})
-      self.assertEqual(sess.run(size), 2)
+      self.assertEqual(self.evaluate(size), 2)
       sess.run(clear)
-      self.assertEqual(sess.run(size), 0)
+      self.assertEqual(self.evaluate(size), 0)
 
   def testCapacity(self):
     capacity = 3
@@ -210,14 +210,14 @@ class StageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(sess.run(size) == capacity)
+      self.assertTrue(self.evaluate(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
-        self.assertTrue(sess.run(ret) == [i])
+        self.assertTrue(self.evaluate(ret) == [i])
 
       # It should now be empty
-      self.assertTrue(sess.run(size) == 0)
+      self.assertTrue(self.evaluate(size) == 0)
 
   def testMemoryLimit(self):
     memory_limit = 512 * 1024  # 512K
@@ -274,13 +274,13 @@ class StageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(sess.run(size) == capacity)
+      self.assertTrue(self.evaluate(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
-        self.assertTrue(np.all(sess.run(ret)[0] == i))
+        self.assertTrue(np.all(self.evaluate(ret)[0] == i))
 
-      self.assertTrue(sess.run(size) == 0)
+      self.assertTrue(self.evaluate(size) == 0)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/kernel_tests/string_length_op_test.py b/tensorflow/python/kernel_tests/string_length_op_test.py
index 57db7302b1..0c68f0cadd 100644
--- a/tensorflow/python/kernel_tests/string_length_op_test.py
+++ b/tensorflow/python/kernel_tests/string_length_op_test.py
@@ -29,7 +29,7 @@ class StringLengthOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       lengths = string_ops.string_length(strings)
-      values = sess.run(lengths)
+      values = self.evaluate(lengths)
       self.assertAllEqual(values, [[[1, 2], [3, 4], [5, 6]]])
 
   def testUnit(self):
diff --git a/tensorflow/python/kernel_tests/string_split_op_test.py b/tensorflow/python/kernel_tests/string_split_op_test.py
index b968e885ed..92e13db0f7 100644
--- a/tensorflow/python/kernel_tests/string_split_op_test.py
+++ b/tensorflow/python/kernel_tests/string_split_op_test.py
@@ -34,7 +34,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
@@ -44,7 +44,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, delimiter="")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4],
                                     [1, 0], [1, 1], [1, 2], [1, 3], [2, 0],
                                     [2, 1], [2, 2], [2, 3]])
@@ -62,7 +62,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(
           indices,
           [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
@@ -74,7 +74,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, delimiter=" .")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(
           indices,
           [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
@@ -92,13 +92,13 @@ class StringSplitOpTest(test.TestCase):
           ValueError, string_ops.string_split, strings, delimiter=["a"])
 
       tokens = string_ops.string_split(strings, delimiter="|")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
       self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
       self.assertAllEqual(shape, [2, 2])
 
       tokens = string_ops.string_split(strings, delimiter="| ")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"])
       self.assertAllEqual(shape, [2, 2])
@@ -145,7 +145,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, "#", skip_empty=False)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1],
                                     [2, 0], [2, 1], [2, 2]])
@@ -154,7 +154,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, "#")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(values, [b"a", b"b", b"c"])
       self.assertAllEqual(indices, [[0, 0], [1, 0], [2, 0]])
       self.assertAllEqual(shape, [3, 1])
@@ -167,7 +167,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
@@ -182,7 +182,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep="<>")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(
           indices, [[0, 0], [0, 1], [0, 2],
                     [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6]])
@@ -200,7 +200,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep=',')
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2],
                                     [1, 0], [1, 1], [1, 2], [1, 3], [1, 4]])
       self.assertAllEqual(values, [b"1", b"2", b"3",
@@ -217,7 +217,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2],
                                     [1, 0], [1, 1], [1, 2]])
       self.assertAllEqual(values, [b"1", b"2", b"3", b"4", b"5", b"6"])
@@ -233,7 +233,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep=',', maxsplit=1)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"1", b"2,3", b"4", b"5,,6,"])
@@ -249,7 +249,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, maxsplit=1)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"1", b"2 3", b"4", b"5    6  "])
diff --git a/tensorflow/python/kernel_tests/string_strip_op_test.py b/tensorflow/python/kernel_tests/string_strip_op_test.py
index 1e404b7146..edff3862ff 100644
--- a/tensorflow/python/kernel_tests/string_strip_op_test.py
+++ b/tensorflow/python/kernel_tests/string_strip_op_test.py
@@ -30,7 +30,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = sess.run(output)
+      output = self.evaluate(output)
       self.assertAllEqual(output, [b"pigs on the wing", b"animals"])
 
   def test_string_strip_2d(self):
@@ -39,7 +39,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = sess.run(output)
+      output = self.evaluate(output)
       self.assertAllEqual(output, [[b"pigs on the wing", b"animals"],
                                    [b"hello", b"world"]])
 
@@ -48,7 +48,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = sess.run(output)
+      output = self.evaluate(output)
       self.assertAllEqual(output, [b"hello", b"", b"world", b""])
 
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
index 63ce77b9d5..1547c55f8b 100644
--- a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
@@ -60,7 +60,7 @@ class SummaryV1AudioOpTest(test.TestCase):
         sample_rate = 8000
         summ = summary.audio(
             "snd", const, max_outputs=3, sample_rate=sample_rate)
-        value = sess.run(summ)
+        value = self.evaluate(summ)
         self.assertEqual([], summ.get_shape())
         audio_summ = self._AsSummary(value)
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
index 094606944f..e1b24756f3 100644
--- a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
@@ -70,7 +70,7 @@ class SummaryV1ImageOpTest(test.TestCase):
 
           # Summarize
           summ = summary.image("img", const)
-          value = sess.run(summ)
+          value = self.evaluate(summ)
           self.assertEqual([], summ.get_shape())
           image_summ = self._AsSummary(value)
 
@@ -97,7 +97,7 @@ class SummaryV1ImageOpTest(test.TestCase):
 
         # Summarize
         summ = summary.image("img", tf_images)
-        value = sess.run(summ)
+        value = self.evaluate(summ)
         self.assertEqual([], summ.get_shape())
         image_summ = self._AsSummary(value)
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_ops_test.py b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
index 6c4e106b11..1206cb7013 100644
--- a/tensorflow/python/kernel_tests/summary_v1_ops_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
@@ -42,7 +42,7 @@ class SummaryV1OpsTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant([10.0, 20.0])
       summ = logging_ops.scalar_summary(["c1", "c2"], const, name="mysumm")
-      value = sess.run(summ)
+      value = self.evaluate(summ)
     self.assertEqual([], summ.get_shape())
     self.assertProtoEquals("""
       value { tag: "c1" simple_value: 10.0 }
@@ -53,7 +53,7 @@ class SummaryV1OpsTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant([10.0, 20.0])
       summ = logging_ops.scalar_summary(["c1", "c2"], const)
-      value = sess.run(summ)
+      value = self.evaluate(summ)
     self.assertEqual([], summ.get_shape())
     self.assertProtoEquals("""
       value { tag: "c1" simple_value: 10.0 }
@@ -66,7 +66,7 @@ class SummaryV1OpsTest(test.TestCase):
       summ1 = summary.histogram("h", const)
       summ2 = logging_ops.scalar_summary("c", const)
       merge = summary.merge([summ1, summ2])
-      value = sess.run(merge)
+      value = self.evaluate(merge)
     self.assertEqual([], merge.get_shape())
     self.assertProtoEquals("""
       value {
diff --git a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
index 34f771679a..71251f5602 100644
--- a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
@@ -68,7 +68,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(10.0)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -79,7 +79,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(s)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -89,7 +89,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = array_ops.ones([5, 5, 5])
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
     self._AssertNumpyEq(n, np.ones([5, 5, 5]))
@@ -99,7 +99,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(strings)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
     self._AssertNumpyEq(n, strings)
@@ -109,7 +109,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(bools)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -119,7 +119,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
 
       def get_description(summary_op):
-        summ_str = sess.run(summary_op)
+        summ_str = self.evaluate(summary_op)
         summ = summary_pb2.Summary()
         summ.ParseFromString(summ_str)
         return summ.value[0].metadata
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index 32c97a7b19..589172e4b7 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -68,7 +68,7 @@ class SvdOpTest(test.TestCase):
             s2 = linalg_ops.svd(
                 matrix2, compute_uv=compute_uv_, full_matrices=full_matrices_)
             all_ops += [s1, s2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       for i in range(2):
         s = 6 * i
         self.assertAllEqual(val[s], val[s + 3])  # s1 == s2
@@ -158,7 +158,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
         s_tf = linalg_ops.svd(
             x_tf, compute_uv=compute_uv_, full_matrices=full_matrices_)
         if use_static_shape_:
-          s_tf_val = sess.run(s_tf)
+          s_tf_val = self.evaluate(s_tf)
         else:
           s_tf_val = sess.run(s_tf, feed_dict={x_tf: x_np})
 
diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py
index 9dcdaa61ed..a187fa115c 100644
--- a/tensorflow/python/kernel_tests/template_test.py
+++ b/tensorflow/python/kernel_tests/template_test.py
@@ -104,10 +104,10 @@ class TemplateTest(test.TestCase):
     train_op = optimizer.minimize(train_loss)
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
-      initial_test_loss = sess.run(test_loss)
-      sess.run(train_op)
-      final_test_loss = sess.run(test_loss)
+      self.evaluate(variables.global_variables_initializer())
+      initial_test_loss = self.evaluate(test_loss)
+      self.evaluate(train_op)
+      final_test_loss = self.evaluate(test_loss)
 
     # Parameters are tied, so the loss should have gone down when we trained it.
     self.assertLess(final_test_loss, initial_test_loss)
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 7e8db8947b..4ee1c27a87 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -751,7 +751,7 @@ class TensorArrayTest(test.TestCase):
                 [-0.5, 1.5],  # read(0) gradient
                 [20.0, 30.0, 40.0, 50.0]
             ])  # concat gradient
-      grad_vals = sess.run(grad_r)  # 2 + 2 entries
+      grad_vals = self.evaluate(grad_r)  # 2 + 2 entries
 
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
@@ -1286,7 +1286,7 @@ class TensorArrayTest(test.TestCase):
       r = w1.stack()
       self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
-      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
+      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
   @test_util.disable_control_flow_v2("b/117943489")
   def testSkipEagerTensorArrayUnpackDynamic(self):
@@ -1303,7 +1303,7 @@ class TensorArrayTest(test.TestCase):
       r = w1.concat()
       self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
-      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
+      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
   def _testTensorArrayEvalEmpty(self):
     with self.cached_session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
index 4ad5ee4103..d1c7b41c7b 100644
--- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
@@ -42,7 +42,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
       outputs = string_ops.unicode_transcode(
@@ -52,7 +52,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
       outputs = string_ops.unicode_transcode(
@@ -62,7 +62,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
   def test_transcode_utf16_to_utf8(self):
@@ -77,7 +77,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, expected)
 
   def test_transcode_bad_utf8(self):
@@ -90,7 +90,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=True)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"  ")
 
       outputs = string_ops.unicode_transcode(
@@ -100,7 +100,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
   def test_transcode_bad_utf8_with_some_good(self):
@@ -113,7 +113,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"abc abcdefg")
 
   def test_transcode_bad_utf8_with_defaults(self):
@@ -121,7 +121,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00\xef\xbf\xbd")
 
   def test_transcode_bad_utf8_with_space_replacement(self):
@@ -130,7 +130,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8",
           replacement_char=ord(" "))
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
   def test_transcode_bad_utf8_with_strict_errors(self):
@@ -165,7 +165,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           input_encoding="UTF-8",
           output_encoding="UTF-8",
           errors="ignore")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00")
 
   def test_transcode_bad_utf8_with_elision_including_control_chars(self):
@@ -177,7 +177,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           errors="ignore",
           replace_control_characters=True)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"")
 
   def test_transcode_bad_utf8_termination_with_defaults(self):
@@ -185,7 +185,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"a\xef\xbf\xbd")   # 0xFFFD
 
   def test_transcode_utf8_with_replacement_char(self):
@@ -194,13 +194,13 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       outputs = string_ops.unicode_transcode(
           strings, input_encoding="UTF-8", output_encoding="UTF-8",
           errors="strict")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
 
       outputs = string_ops.unicode_transcode(
           strings, input_encoding="UTF-8", output_encoding="UTF-8",
           errors="replace", replacement_char=ord("?"))
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
 
   def test_transcode_utf8_to_utf16(self):
@@ -214,7 +214,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-16-BE",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       print("values=", values)
       self.assertAllEqual(values, expected)
 
@@ -230,7 +230,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, expected)
 
   def test_transcode_utf8_to_utf32(self):
@@ -243,7 +243,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-32-BE",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, expected)
 
   # Documentation in ICU suggests that getNextUChar may produce a different
@@ -258,7 +258,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
   def test_transcode_utf8_with_bom(self):
@@ -266,12 +266,12 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\xef\xbb\xbfabcdefg")  # BOM preserved
 
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-8", output_encoding="UTF-16-BE")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       utf16expected = bom_string.decode("UTF-8").encode("UTF-16-BE")
       self.assertAllEqual(values, utf16expected)
 
@@ -280,20 +280,20 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-BE", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       # BOM is preserved in output
       self.assertAllEqual(values, b"\xef\xbb\xbfa")
 
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       # mangled BOM and value from (incorrect) LE encoding
       self.assertAllEqual(values, b"\xef\xbf\xbe\xe6\x84\x80")
 
       bom_string = b"\xff\xfe\x61\x00"  # Little-endian BOM with 'a' encoded
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\xef\xbb\xbfa")
 
   @parameterized.parameters(
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 0aac4adfa6..838838e0ac 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -438,15 +438,15 @@ class VariableScopeTest(test.TestCase):
         sess.run(v0)
       # We should be able to initialize and run v1 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      sess.run(v1.initializer)
-      self.assertEqual(1, sess.run(v1))
+      self.evaluate(v1.initializer)
+      self.assertEqual(1, self.evaluate(v1))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(v0)
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(add)
       # If we initialize v0 we should be able to run 'add'.
-      sess.run(v0.initializer)
+      self.evaluate(v0.initializer)
       sess.run(add)
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
@@ -490,10 +490,10 @@ class VariableScopeTest(test.TestCase):
       v2 = var_dict["v2"]
       # We should be able to initialize and run v1 and v2 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      sess.run(v1.initializer)
-      self.assertEqual([1], sess.run(v1))
-      sess.run(v2.initializer)
-      self.assertEqual([2], sess.run(v2))
+      self.evaluate(v1.initializer)
+      self.assertEqual([1], self.evaluate(v1))
+      self.evaluate(v2.initializer)
+      self.assertEqual([2], self.evaluate(v2))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(v0)
@@ -501,7 +501,7 @@ class VariableScopeTest(test.TestCase):
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(add)
       # If we initialize v0 we should be able to run 'add'.
-      sess.run(v0.initializer)
+      self.evaluate(v0.initializer)
       sess.run(add)
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 6213f86272..d15801f31b 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -149,10 +149,10 @@ class VariablesTestCase(test.TestCase):
           name="foo",
           trainable=False,
           collections=[ops.GraphKeys.LOCAL_VARIABLES])
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       old_value = array.value()
       copy_op = array.assign(old_value)
-      self.assertEqual([], list(sess.run(copy_op)))
+      self.assertEqual([], list(self.evaluate(copy_op)))
 
   def _countUpToTest(self, dtype):
     with self.cached_session():
@@ -221,10 +221,10 @@ class VariablesTestCase(test.TestCase):
       v2 = var_dict["v2"]
       # We should be able to initialize and run v1 and v2 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      sess.run(v1.initializer)
-      self.assertEqual([1], sess.run(v1))
-      sess.run(v2.initializer)
-      self.assertEqual([2], sess.run(v2))
+      self.evaluate(v1.initializer)
+      self.assertEqual([1], self.evaluate(v1))
+      self.evaluate(v2.initializer)
+      self.assertEqual([2], self.evaluate(v2))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors_impl.OpError, "uninitialized"):
         sess.run(v0)
@@ -232,7 +232,7 @@ class VariablesTestCase(test.TestCase):
       with self.assertRaisesRegexp(errors_impl.OpError, "uninitialized"):
         sess.run(add)
       # If we initialize v0 we should be able to run 'add'.
-      sess.run(v0.initializer)
+      self.evaluate(v0.initializer)
       sess.run(add)
 
   def testControlFlowInitialization(self):
@@ -386,7 +386,7 @@ class VariablesTestCase(test.TestCase):
     with self.cached_session() as sess:
       var = variables.Variable([1, 12])
       variables.global_variables_initializer().run()
-      self.assertAllClose([1, 12], sess.run(var))
+      self.assertAllClose([1, 12], self.evaluate(var))
 
   def testDevicePlacement(self):
     with self.cached_session() as sess:
@@ -396,7 +396,7 @@ class VariablesTestCase(test.TestCase):
       init_op = variables.global_variables_initializer()
       self.assertEqual(var.op.device, init_value.device)
       self.assertEqual(var.op.device, init_op.device)
-      sess.run(init_op)
+      self.evaluate(init_op)
 
   def testColocation(self):
     with ops.device("/job:ps"):
@@ -543,7 +543,7 @@ class IsInitializedTest(test.TestCase):
   def testNoVars(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       uninited = variables.report_uninitialized_variables()
-      self.assertEqual(0, sess.run(uninited).size)
+      self.assertEqual(0, self.evaluate(uninited).size)
 
   def testAssertVariablesInitialized(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
@@ -551,27 +551,27 @@ class IsInitializedTest(test.TestCase):
       w = variables.Variable([3, 4], name="w")
       _ = v, w
       uninited = variables.report_uninitialized_variables()
-      self.assertAllEqual(np.array([b"v", b"w"]), sess.run(uninited))
+      self.assertAllEqual(np.array([b"v", b"w"]), self.evaluate(uninited))
       variables.global_variables_initializer().run()
-      self.assertEqual(0, sess.run(uninited).size)
+      self.assertEqual(0, self.evaluate(uninited).size)
 
   def testVariableList(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2], name="v")
       w = variables.VariableV1([3, 4], name="w")
       uninited = variables.report_uninitialized_variables()
-      self.assertAllEqual(np.array([b"v", b"w"]), sess.run(uninited))
-      sess.run(w.initializer)
-      self.assertAllEqual(np.array([b"v"]), sess.run(uninited))
+      self.assertAllEqual(np.array([b"v", b"w"]), self.evaluate(uninited))
+      self.evaluate(w.initializer)
+      self.assertAllEqual(np.array([b"v"]), self.evaluate(uninited))
       v.initializer.run()
-      self.assertEqual(0, sess.run(uninited).size)
+      self.assertEqual(0, self.evaluate(uninited).size)
 
   def testZeroSizeVarInitialized(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.Variable(array_ops.zeros([0, 2]), name="v")
       uninited = variables.report_uninitialized_variables()
       v.initializer.run()  # not strictly necessary
-      self.assertEqual(0, sess.run(uninited).size)
+      self.assertEqual(0, self.evaluate(uninited).size)
 
   def testTrainingWithZeroSizeVar(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
@@ -610,7 +610,7 @@ class ObsoleteIsInitializedTest(test.TestCase):
       inited = variables.assert_variables_initialized([v])
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
         inited.op.run()
-      sess.run(w.initializer)
+      self.evaluate(w.initializer)
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
         inited.op.run()
       v.initializer.run()
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 0634dfa2d8..48b32f06aa 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -48,8 +48,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., lambda v: v * v, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertEqual(self.evaluate(ret), 16.)
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
@@ -65,8 +65,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # Note: This is simply d_ret[0]/d_x since d_ret[1]/d_x is 0.
     grad = gradients_impl.gradients(ret, [x])  # [2*x*y]
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(ret), [45., 3.])
-      self.assertSequenceEqual(sess.run(grad), [9.])
+      self.assertSequenceEqual(self.evaluate(ret), [45., 3.])
+      self.assertSequenceEqual(self.evaluate(grad), [9.])
 
   def testMultipleLoopVars(self):
     x = constant_op.constant(5.)
@@ -88,13 +88,13 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grady_1 = gradients_impl.gradients(ret[1], [y])  # [x + 1]
     grady_2 = gradients_impl.gradients(ret, [y])  # [2*x*y + x**2 + x + 1]
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(ret), [120., 23.])
-      self.assertSequenceEqual(sess.run(gradx_0), [39.])
-      self.assertSequenceEqual(sess.run(gradx_1), [4.])
-      self.assertSequenceEqual(sess.run(gradx_2), [43.])
-      self.assertSequenceEqual(sess.run(grady_0), [55.])
-      self.assertSequenceEqual(sess.run(grady_1), [6.])
-      self.assertSequenceEqual(sess.run(grady_2), [61.])
+      self.assertSequenceEqual(self.evaluate(ret), [120., 23.])
+      self.assertSequenceEqual(self.evaluate(gradx_0), [39.])
+      self.assertSequenceEqual(self.evaluate(gradx_1), [4.])
+      self.assertSequenceEqual(self.evaluate(gradx_2), [43.])
+      self.assertSequenceEqual(self.evaluate(grady_0), [55.])
+      self.assertSequenceEqual(self.evaluate(grady_1), [6.])
+      self.assertSequenceEqual(self.evaluate(grady_2), [61.])
 
   def testMultipleWhileLoops(self):
     x = constant_op.constant(2.)
@@ -103,8 +103,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret2, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(grad), [32.])
-      self.assertSequenceEqual(sess.run(grad_grad), [48.])
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
   def testDoubleDerivative(self):
     x = constant_op.constant(2.)
@@ -112,9 +112,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
-      self.assertSequenceEqual(sess.run(grad_grad), [48.])
+      self.assertEqual(self.evaluate(ret), 16.)
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
   def testPruning(self):
     x = constant_op.constant(1)
@@ -157,8 +157,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v + y < 9., lambda v: v * 3., [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 18.)
-      self.assertSequenceEqual(sess.run(grad), [9.])
+      self.assertEqual(self.evaluate(ret), 18.)
+      self.assertSequenceEqual(self.evaluate(grad), [9.])
 
   def testCaptureExternalTensorInBody(self):
     x = constant_op.constant(2.)
@@ -166,8 +166,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., lambda v: v * y, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 18.)
-      self.assertSequenceEqual(sess.run(grad), [9.])
+      self.assertEqual(self.evaluate(ret), 18.)
+      self.assertSequenceEqual(self.evaluate(grad), [9.])
 
   def testLoopWithTensorListPushBack(self):
     x = constant_op.constant(2.)
@@ -188,7 +188,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   def testDuplicateAccumulator(self):
     x = constant_op.constant(2.)
@@ -222,7 +222,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   @parameterized.named_parameters(
       ("UnknownShape", None),
@@ -315,9 +315,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
       # map_fn uses TensorArray internally.
       r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0)
-      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], sess.run(r))
+      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r))
       r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(21.0, sess.run(r))
+      self.assertAllClose(21.0, self.evaluate(r))
 
   def testNestedWhile(self):
     # Compute sum of geometric progression: n^0 + n^1 + ... + n^m
@@ -334,8 +334,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     result = while_loop_v2(lambda i, _: i >= 0, Body, [m, sum_of_powers])[1]
     grad = gradients_impl.gradients(result, [n])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(result), 364.)
-      self.assertSequenceEqual(sess.run(grad), [547.])
+      self.assertEqual(self.evaluate(result), 364.)
+      self.assertSequenceEqual(self.evaluate(grad), [547.])
 
   def testIdentityNodeInBody(self):
 
@@ -348,8 +348,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., Body, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertEqual(self.evaluate(ret), 16.)
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   def testNestedWhileAndTensorArray(self):
     n = constant_op.constant(3.0)
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index c3c7f867a1..bd3142132c 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -65,7 +65,7 @@ class XentTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       loss = nn_ops.softmax_cross_entropy_with_logits(
           labels=np_labels, logits=np_features, dim=dim)
-      tf_loss = sess.run(loss)
+      tf_loss = self.evaluate(loss)
     print("np_loss:", np_loss)
     print("tf_loss:", tf_loss)
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
@@ -280,7 +280,7 @@ class XentTest(test.TestCase):
     with self.session(use_gpu=True) as sess:
       loss = nn_ops.softmax_cross_entropy_with_logits(
           labels=labels, logits=features)
-      tf_loss = sess.run(loss)
+      tf_loss = self.evaluate(loss)
     self.assertAllEqual(np_loss, tf_loss)
 
 
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index 257fa27156..d3200fa5b5 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -276,8 +276,8 @@ class ConvTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 32)))
         # Check that the bias still got initialized to zeros.
@@ -663,8 +663,8 @@ class SeparableConv2DTest(test.TestCase):
         self.assertTrue('depthwise_kernel' in weights[0].name)
         self.assertTrue('pointwise_kernel' in weights[1].name)
         self.assertTrue('bias' in weights[2].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 1)))
         self.assertAllClose(weights[1], np.ones((1, 1, 3, 32)))
@@ -902,8 +902,8 @@ class Conv2DTransposeTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 32, 3)))
         # Check that the bias still got initialized to zeros.
@@ -1084,8 +1084,8 @@ class Conv3DTransposeTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 4, 32)))
         # Check that the bias still got initialized to zeros.
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index 0343bfa8bd..a61639b2db 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -443,7 +443,7 @@ class DropoutTest(test.TestCase):
       dp = core_layers.Dropout(rate, name='dropout')
       inputs = array_ops.ones((5, 5))
       dropped = dp.apply(inputs, training=True)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_output = sess.run(dropped, feed_dict={rate: 0.5})
       self.assertAlmostEqual(0., np_output.min())
       np_output = sess.run(dropped, feed_dict={rate: 0.0})
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index ba2bf10cf3..febc3587fe 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -78,7 +78,7 @@ class BNTest(test.TestCase):
       if restore:
         saver.restore(sess, checkpoint_path)
       else:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
       np.random.seed(0)
       for _ in range(2):
         image_val = np.random.rand(*shape).astype(dtype.as_numpy_dtype)
@@ -321,7 +321,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 4, 1))
@@ -337,7 +337,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 2))
       std = np.std(np_inputs, axis=(0, 2))
       variance = np.square(std)
@@ -363,7 +363,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 3))
       np_beta = np.reshape(np_beta, (1, 1, 3))
@@ -377,7 +377,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1))
       std = np.std(np_inputs, axis=(0, 1))
       variance = np.square(std)
@@ -404,7 +404,7 @@ class BNTest(test.TestCase):
 
       with self.session(use_gpu=True) as sess:
         # Test training with placeholder learning phase.
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
         np_gamma = np.reshape(np_gamma, (1, 4, 1, 1))
         np_beta = np.reshape(np_beta, (1, 4, 1, 1))
@@ -418,7 +418,7 @@ class BNTest(test.TestCase):
 
         # Verify that the statistics are updated during training.
         moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-        np_inputs = sess.run(inputs)
+        np_inputs = self.evaluate(inputs)
         mean = np.mean(np_inputs, axis=(0, 2, 3))
         std = np.std(np_inputs, axis=(0, 2, 3))
         variance = np.square(std)
@@ -444,7 +444,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 3, 1))
       np_beta = np.reshape(np_beta, (1, 1, 3, 1))
@@ -458,7 +458,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 3))
       std = np.std(np_inputs, axis=(0, 1, 3))
       variance = np.square(std)
@@ -484,7 +484,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -498,7 +498,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -524,7 +524,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -538,7 +538,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -565,7 +565,7 @@ class BNTest(test.TestCase):
 
       with self.cached_session() as sess:
         # Test training with placeholder learning phase.
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
         np_gamma = np.reshape(np_gamma, (1, 4, 1, 1))
         np_beta = np.reshape(np_beta, (1, 4, 1, 1))
@@ -579,7 +579,7 @@ class BNTest(test.TestCase):
 
         # Verify that the statistics are updated during training.
         moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-        np_inputs = sess.run(inputs)
+        np_inputs = self.evaluate(inputs)
         mean = np.mean(np_inputs, axis=(0, 2, 3))
         std = np.std(np_inputs, axis=(0, 2, 3))
         variance = np.square(std)
@@ -605,7 +605,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -620,7 +620,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -646,7 +646,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -659,7 +659,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -667,7 +667,7 @@ class BNTest(test.TestCase):
       self.assertAllClose(variance, moving_var, atol=1e-2)
 
       # Test inference with placeholder learning phase.
-      np_output = sess.run(outputs_infer)
+      np_output = self.evaluate(outputs_infer)
 
       # Verify that the axis is normalized during inference.
       normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
@@ -696,7 +696,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([gamma, beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -710,7 +710,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       np_mean = np.mean(np_inputs, axis=(0, 1, 2))
       np_std = np.std(np_inputs, axis=(0, 1, 2))
       np_variance = np.square(np_std)
@@ -758,14 +758,14 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(100):
         np_output, _, _ = sess.run([outputs2] + updates,
                                    feed_dict={training: True})
 
       # Verify that the statistics are updated during training.
       np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance])
-      np_inputs = sess.run(inputs2)
+      np_inputs = self.evaluate(inputs2)
       np_mean = np.mean(np_inputs, axis=(0, 1, 2))
       np_std = np.std(np_inputs, axis=(0, 1, 2))
       np_variance = np.square(np_std)
@@ -885,7 +885,7 @@ class BNTest(test.TestCase):
     renorm_mean = renorm_stddev = 0.
     renorm_weight = 0.
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -937,7 +937,7 @@ class BNTest(test.TestCase):
     moving_mean = 0.
     moving_variance = 1.
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
         yt_val_train, adj_scale_val, adj_bias_val = sess.run(
@@ -990,7 +990,7 @@ class BNTest(test.TestCase):
     renorm_mean = renorm_stddev = 0.
     renorm_weight = 0.
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
         yt_val_train, adj_scale_val, adj_bias_val = sess.run(
@@ -1040,7 +1040,7 @@ class BNTest(test.TestCase):
         out1.shape.as_list(), out2.shape.as_list())
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       x = np.random.random(shape)
       y1, y2 = sess.run([out1, out2], feed_dict={inp: x})
@@ -1062,7 +1062,7 @@ class BNTest(test.TestCase):
         inp, virtual_batch_size=2)
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       x = np.random.random(np_shape)
       y = sess.run(out, feed_dict={inp: x})
@@ -1093,7 +1093,7 @@ class BNTest(test.TestCase):
                     shape[1]])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1146,7 +1146,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1200,7 +1200,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1256,7 +1256,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
 
@@ -1270,7 +1270,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=0, keepdims=True)
       std = np.std(np_inputs, axis=0, keepdims=True)
       variance = np.square(std)
@@ -1296,7 +1296,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
 
@@ -1310,7 +1310,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 4), keepdims=True)
       std = np.std(np_inputs, axis=(0, 4), keepdims=True)
       variance = np.square(std)
@@ -1350,7 +1350,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index 47675d3f34..260af95a3b 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -209,7 +209,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
       optimizer = momentum.MomentumOptimizer(0.1, 0.9)
       train_op = optimizer.minimize(cost)
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
           sess.run([train_op])
 
@@ -232,7 +232,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
           cond, body, [constant_op.constant(0),
                        constant_op.constant(0.0)])
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllEqual(10.0, self.evaluate(cost))
 
   def doTestIndexedSlicesGradientInCondInWhileLoop(self, use_resource=False):
@@ -269,7 +269,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
                                           static_grads.indices)
 
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllEqual(*sess.run([static_grads, dynamic_grads]))
 
   def testIndexedSlicesGradientInCondInWhileLoop(self):
@@ -398,9 +398,9 @@ class CondTest(test_util.TensorFlowTestCase):
             pred=bool_var,
             true_fn=lambda: state_ops.assign(bool_var, False),
             false_fn=lambda: True)
-        sess.run(bool_var.initializer)
-        self.assertEquals(sess.run(cond_on_bool_var), False)
-        self.assertEquals(sess.run(cond_on_bool_var), True)
+        self.evaluate(bool_var.initializer)
+        self.assertEquals(self.evaluate(cond_on_bool_var), False)
+        self.assertEquals(self.evaluate(cond_on_bool_var), True)
 
   def testCondMissingArg1(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index 262b62e013..a9058c4a34 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -365,7 +365,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(
           [y], [x], unconnected_gradients="zero")
       with self.cached_session() as sess:
-        self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], sess.run(grads)[0])
+        self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], self.evaluate(grads)[0])
 
   def testUnconnectedGradientsZeroConnectedGradients(self):
     with ops.Graph().as_default():
@@ -374,7 +374,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grad = gradients.gradients(
           [y], [x], unconnected_gradients="zero")
       with self.cached_session() as sess:
-        self.assertEquals(3.0, sess.run(grad)[0])
+        self.assertEquals(3.0, self.evaluate(grad)[0])
 
   def testUnknownUnconnectedGradientsValueGiven(self):
     with ops.Graph().as_default():
@@ -438,8 +438,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(y, [x, b1])
 
       with self.cached_session() as sess:
-        self.assertAllEqual([40.0], sess.run(grads)[0])
-        self.assertAllEqual([10.0], sess.run(grads)[1])
+        self.assertAllEqual([40.0], self.evaluate(grads)[0])
+        self.assertAllEqual([10.0], self.evaluate(grads)[1])
 
   def testFunctionGradientsWithGradFunc(self):
     g = ops.Graph()
@@ -487,7 +487,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(f), 2.0)
+        self.assertEqual(self.evaluate(f), 2.0)
 
   def testGradientOfCaptured(self):
     with ops.Graph().as_default():
@@ -501,7 +501,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(f), 2.0)
+        self.assertEqual(self.evaluate(f), 2.0)
 
   def testCapturedResourceVariable(self):
     with ops.Graph().as_default():
@@ -515,8 +515,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
-        self.assertEqual(sess.run(f), 2.0)
+        self.evaluate(variables.global_variables_initializer())
+        self.assertEqual(self.evaluate(f), 2.0)
 
   def testCapturedNested(self):
     with ops.Graph().as_default():
@@ -541,9 +541,9 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
       x1_grad, x2_grad = Outer()
       with self.cached_session() as sess:
         # 1.0 + None + 2.0 + 1.0 = 4.0
-        self.assertEqual(sess.run(x1_grad), 4.0)
+        self.assertEqual(self.evaluate(x1_grad), 4.0)
         # None + 1.0 + 1.0 + None = 2.0
-        self.assertEqual(sess.run(x2_grad), 2.0)
+        self.assertEqual(self.evaluate(x2_grad), 2.0)
 
   def testCapturedFromFunction(self):
     with ops.Graph().as_default():
@@ -563,7 +563,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       z_grad = Outer()
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(z_grad), 3.0)
+        self.assertEqual(self.evaluate(z_grad), 3.0)
 
   def testCapturedEagerTensors(self):
     # Test that we can handle captured eager tensors unrelated to the gradient
@@ -873,7 +873,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       y = MyMultiply(x1, x2)
       dy = gradients.gradients(y, [x1, x2])
       with session.Session() as sess:
-        self.assertAllEqual([3., 5.], sess.run(dy))
+        self.assertAllEqual([3., 5.], self.evaluate(dy))
 
   def testCustomGradientErrors(self):
 
@@ -914,7 +914,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       for g in grads:
         self.assertTrue(g is not None)
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         dw = sess.run(math_ops.reduce_sum(grads[1]))
         self.assertEqual(12., dw)
 
@@ -1074,7 +1074,7 @@ class TensorListGradientsTest(test_util.TensorFlowTestCase):
 
       grad = gradients.gradients(tl, a, grad_ys=grad_tl)[0]
       with self.cached_session() as sess:
-        self.assertEquals(sess.run(grad), 5.)
+        self.assertEquals(self.evaluate(grad), 5.)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py
index 32c2f37c0b..0ea15b0d23 100644
--- a/tensorflow/python/ops/image_grad_test.py
+++ b/tensorflow/python/ops/image_grad_test.py
@@ -44,7 +44,7 @@ class ResizeNearestNeighborOpTest(test.TestCase):
                                                        out_shape[1:3])
         self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-        resize_out = sess.run(resize_out)
+        resize_out = self.evaluate(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -113,7 +113,7 @@ class ResizeBilinearOpTest(test.TestCase):
       resize_out = image_ops.resize_bilinear(input_tensor, out_shape[1:3])
       self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-      resize_out = sess.run(resize_out)
+      resize_out = self.evaluate(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -196,7 +196,7 @@ class ResizeBicubicOpTest(test.TestCase):
                                               align_corners=align_corners)
         self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-        resize_out = sess.run(resize_out)
+        resize_out = self.evaluate(resize_out)
         self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -273,7 +273,7 @@ class CropAndResizeOpTest(test.TestCase):
           constant_op.constant(
               crop_size, shape=[2]))
       self.assertEqual(crops_shape, list(crops.get_shape()))
-      crops = sess.run(crops)
+      crops = self.evaluate(crops)
       self.assertEqual(crops_shape, list(crops.shape))
 
   def _randomUniformAvoidAnchors(self, low, high, anchors, radius, num_samples):
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index 07406982c5..7eee70c0e3 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -488,11 +488,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.flip_left_right(inputs)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          sess.run(run_op)
+          self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -518,11 +518,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.random_flip_left_right(inputs)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          sess.run(run_op)
+          self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -548,11 +548,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.random_flip_left_right(inputs)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          sess.run(run_op)
+          self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -610,11 +610,11 @@ class AdjustHueBenchmark(test.Benchmark):
       delta = constant_op.constant(0.1, dtype=dtypes.float32)
       outputs = image_ops.adjust_hue(inputs, delta)
       run_op = control_flow_ops.group(outputs)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for i in xrange(warmup_rounds + benchmark_rounds):
         if i == warmup_rounds:
           start = time.time()
-        sess.run(run_op)
+        self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -653,12 +653,12 @@ class AdjustSaturationBenchmark(test.Benchmark):
       delta = constant_op.constant(0.1, dtype=dtypes.float32)
       outputs = image_ops.adjust_saturation(inputs, delta)
       run_op = control_flow_ops.group(outputs)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in xrange(warmup_rounds):
-        sess.run(run_op)
+        self.evaluate(run_op)
       start = time.time()
       for _ in xrange(benchmark_rounds):
-        sess.run(run_op)
+        self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -698,7 +698,7 @@ class ResizeBilinearBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -746,7 +746,7 @@ class ResizeBicubicBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -803,7 +803,7 @@ class ResizeAreaBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -4110,7 +4110,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
       images = [ops.convert_to_tensor(x, dtype=dtypes.float32) for x in images]
       msssim_ops = [image_ops.ssim_multiscale(x, y, 1.0)
                     for x, y in itertools.combinations(images, 2)]
-      msssim = sess.run(msssim_ops)
+      msssim = self.evaluate(msssim_ops)
       msssim = np.squeeze(msssim)
 
     self.assertTrue(np.all(msssim >= 0.0))
diff --git a/tensorflow/python/ops/init_ops_test.py b/tensorflow/python/ops/init_ops_test.py
index 5693c3caaf..1f22248004 100644
--- a/tensorflow/python/ops/init_ops_test.py
+++ b/tensorflow/python/ops/init_ops_test.py
@@ -45,8 +45,8 @@ class InitializersTest(test.TestCase):
       output = variable.numpy()
     else:
       sess = ops.get_default_session()
-      sess.run(variable.initializer)
-      output = sess.run(variable)
+      self.evaluate(variable.initializer)
+      output = self.evaluate(variable)
     lim = 3e-2
     if target_std is not None:
       self.assertGreater(lim, abs(output.std() - target_std))
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index dc46f1cbd1..e0329f66ff 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -373,7 +373,7 @@ class AddNTest(test_util.TensorFlowTestCase):
             for i in range(0, num_inputs)
         ]
         addn = math_ops.add_n(input_vars)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         add_n_grad = gradients.gradients(addn, input_vars)
         self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
                             [g.eval() for g in add_n_grad])
@@ -461,7 +461,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       a = variables.Variable(2.)
       b = variables.Variable(4.)
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         c_grad = gradients.gradients(math_ops.divide(a, b), [a, b])
         self.assertAllEqual([x.eval() for x in c_grad], [.25, -.125])
         c_grad = gradients.gradients(math_ops.div(a, b), [a, b])
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index a6c582fcac..552b274b83 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -82,7 +82,7 @@ class BatchNormalizationTest(test.TestCase):
           epsilon=epsilon,
           data_format=data_format,
           is_training=False)
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       y_ref = self._inference_ref(x, scale, offset, mean, var, epsilon,
                                   data_format)
     # An atol value of 1e-3 is too small for float16's, because some adjacent
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index af6c728694..14cc1c6b5a 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -978,7 +978,7 @@ class LeakyReluTest(test_lib.TestCase):
       np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype)
       outputs = nn_ops.leaky_relu(constant_op.constant(np_values))
       with self.cached_session() as sess:
-        outputs = sess.run(outputs)
+        outputs = self.evaluate(outputs)
       tol = 2e-3 if dtype == np.float16 else 1e-6
       self.assertAllClose(
           outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol)
@@ -1095,7 +1095,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x)
     with self.cached_session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
   def test(self):
@@ -1118,7 +1118,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="NCHW")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
   def testNHWCtoHWNC(self):
@@ -1127,7 +1127,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="HWNC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
   def testNHWCtoWHCN(self):
@@ -1136,7 +1136,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="WHCN")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
   def testArbitraryASCII(self):
@@ -1145,7 +1145,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="qwer", dst_format="rewq")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
 
@@ -1156,7 +1156,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x)
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [7, 3, 4, 9])
 
   def testNCHWToNHWC(self):
@@ -1164,7 +1164,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [7, 9, 3, 4])
 
   def testNHWCToHWNC(self):
@@ -1172,7 +1172,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [4, 9, 7, 3])
 
   def testHWNCToNHWC(self):
@@ -1180,7 +1180,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [9, 7, 4, 3])
 
   def testNHWCToNCHW2D(self):
@@ -1188,7 +1188,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x)
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[7, 4], [5, 1], [9, 3], [4, 5]])
 
   def testNHWCToHWNC2D(self):
@@ -1196,7 +1196,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[9, 3], [4, 5], [7, 4], [5, 1]])
 
   def testHWNCToNHWC2D(self):
@@ -1204,7 +1204,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[4, 5], [7, 4], [9, 3], [5, 1]])
 
   def testNCHWToNHWC2D(self):
@@ -1212,7 +1212,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC")
     with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]])
 
 
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 5a058bae82..b2be24e110 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -472,7 +472,7 @@ class GradientsTest(test.TestCase):
     with session.Session() as sess:
       init = variables.global_variables_initializer()
       sess.run(init)
-      pfor = sess.run(pfor_jacobian)
+      pfor = self.evaluate(pfor_jacobian)
       for i in range(4):
         while_i = sess.run(while_gradients[i])
         self.assertAllClose(while_i, pfor[:, i, ...])
diff --git a/tensorflow/python/ops/quantized_conv_ops_test.py b/tensorflow/python/ops/quantized_conv_ops_test.py
index f7fa264461..6b469a954f 100644
--- a/tensorflow/python/ops/quantized_conv_ops_test.py
+++ b/tensorflow/python/ops/quantized_conv_ops_test.py
@@ -73,7 +73,7 @@ class Conv2DTest(test.TestCase):
           max_input=x1_max,
           min_filter=x2_min,
           max_filter=x2_max)
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     quantized_output = value[0]
     output_min = value[1]
     output_max = value[2]
diff --git a/tensorflow/python/ops/quantized_ops_test.py b/tensorflow/python/ops/quantized_ops_test.py
index 0f3b04e4ad..b81843d174 100644
--- a/tensorflow/python/ops/quantized_ops_test.py
+++ b/tensorflow/python/ops/quantized_ops_test.py
@@ -41,7 +41,7 @@ class QuantizedOpsTest(test.TestCase):
       x_min = 0.0
       x_max = 255.0
       op = array_ops.quantize(x, x_min, x_max, dtypes.quint8, mode="MIN_FIRST")
-      value = sess.run(op)
+      value = self.evaluate(op)
       self.assertArrayNear(expected_output, value.output, 0.1)
 
   def testDequantizeOp(self):
@@ -52,7 +52,7 @@ class QuantizedOpsTest(test.TestCase):
       x_min = 0.0
       x_max = 255.0
       op = array_ops.dequantize(x, x_min, x_max, mode="MIN_FIRST")
-      value = sess.run(op)
+      value = self.evaluate(op)
       self.assertArrayNear(expected_output, value, 0.1)
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index dcf1feaa69..c52db9e2a1 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -190,7 +190,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
     with self.test_session() as sess:
       if hasattr(expected, 'tolist'):
         expected = expected.tolist()
-      self.assertEqual(sess.run(result).tolist(), expected)
+      self.assertEqual(self.evaluate(result).tolist(), expected)
 
   def testRaggedGatherNdUnknownRankError(self):
     params = ragged.constant([['a', 'b'], ['c', 'd']])
diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py
index 94c685274a..8648f0b514 100644
--- a/tensorflow/python/profiler/model_analyzer_test.py
+++ b/tensorflow/python/profiler/model_analyzer_test.py
@@ -93,10 +93,10 @@ class PrintModelAnalysisTest(test.TestCase):
           config=self._no_rewrite_session_config()) as sess, ops.device(dev):
         x = lib.BuildSmallModel()
 
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         pctx.trace_next_step()
         pctx.dump_next_step()
-        _ = sess.run(x)
+        _ = self.evaluate(x)
 
         pctx.profiler.profile_name_scope(options=opts)
 
@@ -160,7 +160,7 @@ class PrintModelAnalysisTest(test.TestCase):
                         ) as sess, ops.device('/device:CPU:0'):
       x = lib.BuildSmallModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -186,7 +186,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -220,9 +220,9 @@ class PrintModelAnalysisTest(test.TestCase):
       with session.Session(config=self._no_rewrite_session_config()) as sess:
         x = lib.BuildFullModel()
 
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         pctx.trace_next_step()
-        _ = sess.run(x)
+        _ = self.evaluate(x)
         tfprof_node = pctx.profiler.profile_python(options=opts)
 
         # pylint: disable=line-too-long
@@ -281,7 +281,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -309,7 +309,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -345,7 +345,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -391,7 +391,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -424,7 +424,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -490,7 +490,7 @@ class PrintModelAnalysisTest(test.TestCase):
 
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -555,7 +555,7 @@ class PrintModelAnalysisTest(test.TestCase):
 
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -587,10 +587,10 @@ class PrintModelAnalysisTest(test.TestCase):
   def _trainLoop(self, train_op, train_steps, time_dir, time_step,
                  memory_dir, memory_step, profile_dir, dump_step):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       # start from 1 because variable_initializer took one step.
       for i in range(1, train_steps + 1):
-        _ = sess.run(train_op)
+        _ = self.evaluate(train_op)
         if i in time_step:
           ret = gfile.ListDirectory(time_dir)
           self.assertEqual(len(ret), 1)
diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py
index 107ad443c3..abbeb8bedf 100644
--- a/tensorflow/python/profiler/profile_context_test.py
+++ b/tensorflow/python/profiler/profile_context_test.py
@@ -48,7 +48,7 @@ class ProfilerContextTest(test.TestCase):
     with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
       pctx.add_auto_profiling("op", options=opts, profile_steps=[15, 50, 100])
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         total_steps = 101
         for i in range(total_steps):
           sess.run(x)
@@ -75,7 +75,7 @@ class ProfilerContextTest(test.TestCase):
 
     with profile_context.ProfileContext(test.get_temp_dir(), debug=True):
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
           sess.run(x)
           for f in gfile.ListDirectory(test.get_temp_dir()):
@@ -96,7 +96,7 @@ class ProfilerContextTest(test.TestCase):
     with profile_context.ProfileContext(test.get_temp_dir(),
                                         enabled=False) as pctx:
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
           sess.run(x)
       self.assertTrue(pctx.profiler is None)
@@ -105,7 +105,7 @@ class ProfilerContextTest(test.TestCase):
 
     with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
           sess.run(x)
       self.assertFalse(pctx.profiler is None)
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 648c1c5928..0b97a73441 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -50,7 +50,7 @@ class SavedModelLoaderTest(test.TestCase):
       x = variables.VariableV1(5, name="x")
       y = variables.VariableV1(11, name="y")
       z = x + y
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       foo_sig_def = signature_def_utils.build_signature_def(
           {"foo_input": utils.build_tensor_info(x)},
@@ -138,7 +138,7 @@ class SavedModelLoaderTest(test.TestCase):
       y = variables.VariableV1(0, name="y")
       z = x * y
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       # There are variables to restore, so a saver must be created.
       with self.assertRaises(ValueError):
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index ab43e93853..a04f4dd099 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -61,7 +61,7 @@ class SavedModelTest(test.TestCase):
 
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.VariableV1(variable_value, name=variable_name)
-    sess.run(variables.global_variables_initializer())
+    self.evaluate(variables.global_variables_initializer())
     self.assertEqual(variable_value, self.evaluate(v))
 
   def _build_asset_collection(self, asset_file_name, asset_file_contents,
@@ -385,7 +385,7 @@ class SavedModelTest(test.TestCase):
       a = ops.get_default_graph().get_tensor_by_name(constant_5_name)
       b = constant_op.constant(6.0)
       c = a * b
-      self.assertEqual(30.0, sess.run(c))
+      self.assertEqual(30.0, self.evaluate(c))
 
     # Restore the graph with tag "bar".
     with self.session(graph=ops.Graph()) as sess:
@@ -394,7 +394,7 @@ class SavedModelTest(test.TestCase):
       a = ops.get_default_graph().get_tensor_by_name(constant_6_name)
       b = constant_op.constant(5.0)
       c = a * b
-      self.assertEqual(30.0, sess.run(c))
+      self.assertEqual(30.0, self.evaluate(c))
 
   def testNoOverwrite(self):
     export_dir = self._get_export_dir("test_no_overwrite")
@@ -460,7 +460,7 @@ class SavedModelTest(test.TestCase):
     with self.session(graph=ops.Graph()) as sess:
       v = variables.VariableV1(42, name="v")
       ops.add_to_collection("foo_vars", v)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertEqual(42, self.evaluate(v))
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
@@ -470,7 +470,7 @@ class SavedModelTest(test.TestCase):
     with self.session(graph=ops.Graph()) as sess:
       v = variables.VariableV1(43, name="v")
       ops.add_to_collection("bar_vars", v)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertEqual(43, self.evaluate(v))
       builder.add_meta_graph(["bar"])
 
@@ -794,7 +794,7 @@ class SavedModelTest(test.TestCase):
         add_v1_v2 = math_ops.add(v1._ref(), v2._ref())
         custom_main_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2))
 
-      sess.run(custom_main_op)
+      self.evaluate(custom_main_op)
       builder.add_meta_graph_and_variables(
           sess, ["foo"], main_op=custom_main_op)
 
@@ -828,7 +828,7 @@ class SavedModelTest(test.TestCase):
       assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2))
       legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op")
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(
           sess, ["foo"], legacy_init_op=legacy_init_op)
 
@@ -871,7 +871,7 @@ class SavedModelTest(test.TestCase):
       assign_v2 = state_ops.assign(v2, v1)
       init_op = control_flow_ops.group(assign_v2, name="init_op")
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       ops.add_to_collection(key, control_flow_ops.no_op())
       # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
@@ -894,10 +894,10 @@ class SavedModelTest(test.TestCase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       train_op = state_ops.assign_add(v1, v2)
 
-      sess.run(train_op)
+      self.evaluate(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph_and_variables(sess, ["foo"])
@@ -923,10 +923,10 @@ class SavedModelTest(test.TestCase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       train_op = control_flow_ops.group()
 
-      sess.run(train_op)
+      self.evaluate(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph_and_variables(sess, ["foo"])
@@ -952,11 +952,11 @@ class SavedModelTest(test.TestCase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["pre_foo"])
 
       train_op = state_ops.assign_add(v1, v2)
-      sess.run(train_op)
+      self.evaluate(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph(["foo"])
@@ -1086,7 +1086,7 @@ class SavedModelTest(test.TestCase):
       ops.add_to_collection("v", v3)
       ops.add_to_collection("init_op", init_op)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1, ops.get_collection("v")[0].eval())
       self.assertEqual(2, ops.get_collection("v")[1].eval())
 
@@ -1141,7 +1141,7 @@ class SavedModelTest(test.TestCase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       custom_saver = training.Saver(name="my_saver")
       builder.add_meta_graph_and_variables(sess, ["tag"], saver=custom_saver)
 
@@ -1163,7 +1163,7 @@ class SavedModelTest(test.TestCase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       training.Saver(name="my_saver")
       builder.add_meta_graph_and_variables(sess, ["tag"])
 
@@ -1185,7 +1185,7 @@ class SavedModelTest(test.TestCase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["tag_0"])
 
       saver_1 = training.Saver()
@@ -1296,7 +1296,7 @@ class SavedModelTest(test.TestCase):
       real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
       imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
       math_ops.complex(real_num, imag_num, name="complex")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(
           sess, ["foo"], strip_default_attrs=True)
 
@@ -1306,7 +1306,7 @@ class SavedModelTest(test.TestCase):
       real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
       imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
       math_ops.complex(real_num, imag_num, name="complex")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph(["bar"], strip_default_attrs=False)
 
     # Save the SavedModel to disk in text format.
@@ -1368,7 +1368,7 @@ class SavedModelTest(test.TestCase):
     with session.Session(graph=ops.Graph()) as sess:
       variables.VariableV1(1.0, dtype=dtypes.float64, name="var")
       test_ops.test_attr(T=dtypes.float32, name="test_attr")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
     # Save the SavedModel to disk in text format.
diff --git a/tensorflow/python/saved_model/simple_save_test.py b/tensorflow/python/saved_model/simple_save_test.py
index 2d404dcea4..0d0665072a 100644
--- a/tensorflow/python/saved_model/simple_save_test.py
+++ b/tensorflow/python/saved_model/simple_save_test.py
@@ -33,7 +33,7 @@ class SimpleSaveTest(test.TestCase):
 
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.Variable(variable_value, name=variable_name)
-    sess.run(variables.global_variables_initializer())
+    self.evaluate(variables.global_variables_initializer())
     self.assertEqual(variable_value, self.evaluate(v))
     return v
 
diff --git a/tensorflow/python/tools/strip_unused_test.py b/tensorflow/python/tools/strip_unused_test.py
index 7cf0c3e3ed..e906ff94ba 100644
--- a/tensorflow/python/tools/strip_unused_test.py
+++ b/tensorflow/python/tools/strip_unused_test.py
@@ -50,7 +50,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase):
           wanted_input_node, 2.0, name="output_node")
       math_ops.add(output_node, 2.0, name="later_node")
       sess = session.Session()
-      output = sess.run(output_node)
+      output = self.evaluate(output_node)
       self.assertNear(-4.0, output, 0.00001)
       graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
 
@@ -113,7 +113,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase):
           input_node1, input_node2, name="output_node")
       math_ops.add(output_node, 2.0, name="later_node")
       sess = session.Session()
-      output = sess.run(output_node)
+      output = self.evaluate(output_node)
       self.assertNear(6.0, output, 0.00001)
       graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
 
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 2d469634e0..13c9e9aa67 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -243,7 +243,7 @@ class LoggingTensorHookTest(test.TestCase):
           tensors=[t.name], at_end=True)
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       self.logged_message = ''
       for _ in range(3):
         mon_sess.run(train_op)
@@ -261,7 +261,7 @@ class LoggingTensorHookTest(test.TestCase):
         tensors=[t.name], every_n_iter=10, at_end=at_end)
     hook.begin()
     mon_sess = monitored_session._HookedSession(sess, [hook])
-    sess.run(variables_lib.global_variables_initializer())
+    self.evaluate(variables_lib.global_variables_initializer())
     mon_sess.run(train_op)
     self.assertRegexpMatches(str(self.logged_message), t.name)
     for _ in range(3):
@@ -308,7 +308,7 @@ class LoggingTensorHookTest(test.TestCase):
           tensors={'foo': t}, every_n_iter=1)
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess.run(train_op)
       self.assertRegexpMatches(str(self.logged_message), 'foo')
       # in first run, elapsed time is None.
@@ -322,7 +322,7 @@ class LoggingTensorHookTest(test.TestCase):
         tensors=[t.name], every_n_secs=1.0, at_end=at_end)
     hook.begin()
     mon_sess = monitored_session._HookedSession(sess, [hook])
-    sess.run(variables_lib.global_variables_initializer())
+    self.evaluate(variables_lib.global_variables_initializer())
 
     mon_sess.run(train_op)
     self.assertRegexpMatches(str(self.logged_message), t.name)
@@ -366,7 +366,7 @@ class LoggingTensorHookTest(test.TestCase):
           formatter=lambda items: 'qqq=%s' % items[t.name])
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess.run(train_op)
       self.assertEqual(self.logged_message[0], 'qqq=42.0')
 
@@ -921,7 +921,7 @@ class StepCounterHookTest(test.TestCase):
       hook = basic_session_run_hooks.StepCounterHook(
           summary_writer=summary_writer, every_n_steps=10)
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       with test.mock.patch.object(tf_logging, 'warning') as mock_log:
         for _ in range(30):
@@ -950,7 +950,7 @@ class StepCounterHookTest(test.TestCase):
           summary_writer=summary_writer, every_n_steps=None, every_n_secs=0.1)
 
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
       time.sleep(0.2)
@@ -987,7 +987,7 @@ class StepCounterHookTest(test.TestCase):
           summary_writer=summary_writer, every_n_steps=1, every_n_secs=None)
 
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
       mon_sess.run(train_op)
@@ -1007,7 +1007,7 @@ class StepCounterHookTest(test.TestCase):
     with ops.Graph().as_default(), session_lib.Session() as sess:
       variables.get_or_create_global_step()
       train_op = training_util._increment_global_step(0)  # keep same.
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       hook = basic_session_run_hooks.StepCounterHook(
           every_n_steps=1, every_n_secs=None)
       hook.begin()
@@ -1034,7 +1034,7 @@ class StepCounterHookTest(test.TestCase):
         summary_writer=self.summary_writer, every_n_steps=every_n_steps)
     self.hook._set_steps_per_run(steps_per_run)
     self.hook.begin()
-    sess.run(variables_lib.global_variables_initializer())
+    self.evaluate(variables_lib.global_variables_initializer())
     self.mon_sess = monitored_session._HookedSession(sess, [self.hook])
 
   def test_steps_per_run_less_than_every_n_steps(self):
@@ -1147,7 +1147,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(30):
         mon_sess.run(self.train_op)
@@ -1179,7 +1179,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(10):
         mon_sess.run(self.train_op)
@@ -1207,7 +1207,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(4):
         mon_sess.run(self.train_op)
@@ -1242,7 +1242,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(8):
         mon_sess.run(self.train_op)
@@ -1285,7 +1285,7 @@ class GlobalStepWaiterHookTest(test.TestCase):
       hook = basic_session_run_hooks.GlobalStepWaiterHook(wait_until_step=1000)
       hook.begin()
       with session_lib.Session() as sess:
-        sess.run(variables_lib.global_variables_initializer())
+        self.evaluate(variables_lib.global_variables_initializer())
         waiter = threading.Thread(
             target=hook.before_run,
             args=(session_run_hook.SessionRunContext(
@@ -1390,7 +1390,7 @@ class ResourceSummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(30):
         mon_sess.run(self.train_op)
diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py
index 38d4acf85f..21ad3df1c8 100644
--- a/tensorflow/python/training/checkpoint_ops_test.py
+++ b/tensorflow/python/training/checkpoint_ops_test.py
@@ -47,7 +47,7 @@ class LoadAndRemapWrappersTest(test.TestCase):
       with variable_scope.variable_scope('some_scope'):
         variable_scope.get_variable(name='embeddings', shape=[5, 16],
                                     initializer=initializer)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver = saver_lib.Saver()
       saver.save(sess, checkpoint_prefix, global_step=5)
     self.checkpoint_file = '{}-5'.format(checkpoint_prefix)
diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py
index e5aac5da18..31c2cc56c0 100644
--- a/tensorflow/python/training/input_test.py
+++ b/tensorflow/python/training/input_test.py
@@ -474,7 +474,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -539,7 +539,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         expected_results = np.arange(i * batch_size, (i + 1) * batch_size)
         max_len = expected_results[-1]
         self.assertAllEqual(results[0], expected_results)
@@ -571,7 +571,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -610,7 +610,7 @@ class BatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertAllEqual(results[0], results[1].values)
@@ -651,7 +651,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -667,7 +667,7 @@ class BatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertAllEqual(results[0],
                           np.arange(num_batches * batch_size,
                                     num_batches * batch_size + extra_elements))
@@ -709,7 +709,7 @@ class BatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertAllEqual(results[0], results[1].values)
@@ -721,7 +721,7 @@ class BatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), extra_elements)
       self.assertAllEqual(results[0], results[1].values)
@@ -827,7 +827,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
@@ -1020,7 +1020,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(3, len(results))
         self.assertEqual(batch_size, len(results[0]))
         self.assertEqual(batch_size, len(results[2]))
@@ -1116,7 +1116,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertEqual(2, len(results))
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[1]), batch_size)
@@ -1201,7 +1201,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -1221,7 +1221,7 @@ class BatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached the final batch with 2 * extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertEqual(len(results[2]), 2 * extra_elements)
@@ -1296,7 +1296,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[1]), batch_size)
@@ -1316,7 +1316,7 @@ class BatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached the final batch with 2 * extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertEqual(len(results[1]), 2 * extra_elements)
@@ -1410,7 +1410,7 @@ class BatchJoinTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual(
             [0] * batch_size,
             np.mod(results[0], 2),)
@@ -1579,7 +1579,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
         self.assertAllEqual(
@@ -1634,7 +1634,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for _ in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
         self.assertAllEqual(
@@ -1645,7 +1645,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertAllEqual(results[1].dense_shape, [extra_elements, 1])
       self.assertAllEqual(results[2], [b"string"] * extra_elements)
       all_counts.extend(results[0])
@@ -1687,7 +1687,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
@@ -1737,7 +1737,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
@@ -1749,7 +1749,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertAllEqual(results[0].shape, [extra_elements])
       self.assertAllEqual(results[1].dense_shape, [extra_elements, 1])
       self.assertAllEqual(results[2], [b"string"] * extra_elements)
@@ -1817,7 +1817,7 @@ class ShuffleBatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
@@ -1990,7 +1990,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(3, len(results))
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -2082,7 +2082,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -2102,7 +2102,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached end with 2 * extra_elements left
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertAllEqual(results[1].dense_shape, [2 * extra_elements, 1])
       self.assertEqual(len(results[2]), 2 * extra_elements)
@@ -2203,7 +2203,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index c870d99de9..ebe2f15a55 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -1178,7 +1178,7 @@ class HookedSessionTest(test.TestCase):
       mock_run = FakeSession(sess)
       mon_sess = monitored_session._HookedSession(sess=mock_run, hooks=[])
       a_tensor = constant_op.constant([0], name='a_tensor')
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       output = mon_sess.run(fetches=a_tensor,
                             feed_dict='a_feed',
                             options='an_option',
@@ -1197,7 +1197,7 @@ class HookedSessionTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(
           sess=sess, hooks=[mock_hook, mock_hook2])
       a_tensor = constant_op.constant([0], name='a_tensor')
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       mon_sess.run(a_tensor)
 
       for hook in [mock_hook, mock_hook2]:
@@ -1222,7 +1222,7 @@ class HookedSessionTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(
           sess=sess, hooks=[mock_hook, mock_hook2])
       constant_op.constant([0], name='a_tensor')
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       mon_sess.run(fetches='a_tensor')
       self.assertFalse(mon_sess.should_stop())
@@ -1242,7 +1242,7 @@ class HookedSessionTest(test.TestCase):
       third_tensor = constant_op.constant([10], name='third_tensor')
       mock_hook.request = session_run_hook.SessionRunArgs([another_tensor])
       mock_hook2.request = session_run_hook.SessionRunArgs([third_tensor])
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       output = mon_sess.run(fetches=a_tensor)
       self.assertEqual(output, [0])
@@ -1262,7 +1262,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       self.assertEqual(mon_sess.run(fetches=add_tensor), [15])
 
@@ -1280,7 +1280,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       feed_dict = {c_tensor: [20]}
       self.assertEqual(
@@ -1301,7 +1301,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={a_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'):
         mon_sess.run(fetches=add_tensor)
@@ -1319,7 +1319,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'):
         mon_sess.run(fetches=add_tensor, feed_dict={b_tensor: [10]})
diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py
index 8009e3c24e..41e9dcea84 100644
--- a/tensorflow/python/training/moving_averages_test.py
+++ b/tensorflow/python/training/moving_averages_test.py
@@ -274,14 +274,14 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual([], v1_avg.value().op.control_inputs)
       self.assertEqual([], v1_avg.value().op.control_inputs)
       # We should be able to initialize v1_avg before v0.
-      sess.run(v1_avg.initializer)
-      sess.run(v0.initializer)
-      self.assertEqual([10.0], sess.run(v1_avg))
+      self.evaluate(v1_avg.initializer)
+      self.evaluate(v0.initializer)
+      self.assertEqual([10.0], self.evaluate(v1_avg))
       # running ema_op should add to v0 (in addition to updating v1_avg)
       sess.run(assign_to_v1)
-      sess.run(ema_op)
-      self.assertEqual(1, sess.run(v0))
-      self.assertEqual([17.5], sess.run(v1_avg))
+      self.evaluate(ema_op)
+      self.assertEqual(1, self.evaluate(v0))
+      self.assertEqual([17.5], self.evaluate(v1_avg))
 
   @test_util.run_in_graph_and_eager_modes
   def testBasicEager(self):
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index eb2690985d..7bc0a178a4 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -227,7 +227,7 @@ class SaverTest(test.TestCase):
         w1 = resource_variable_ops.ResourceVariable(1.0, name="w1")
         w2 = resource_variable_ops.ResourceVariable(2.0, name="w2")
         graph_saver = saver_module.Saver([w1, w2])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         graph_saver.save(sess, graph_ckpt_prefix)
 
     with context.eager_mode():
@@ -260,7 +260,7 @@ class SaverTest(test.TestCase):
         w3 = resource_variable_ops.ResourceVariable(0.0, name="w3")
         w4 = resource_variable_ops.ResourceVariable(0.0, name="w4")
         graph_saver = saver_module.Saver([w3, w4])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         graph_saver.restore(sess, eager_ckpt_prefix)
         self.assertAllEqual(w3.eval(), 3.0)
         self.assertAllEqual(w4.eval(), 4.0)
@@ -326,7 +326,7 @@ class SaverTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initialize all variables
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
 
       # Check that the parameter nodes have been initialized.
       self.assertEqual(10.0, v0.eval())
@@ -376,7 +376,7 @@ class SaverTest(test.TestCase):
     with self.cached_session() as sess:
       tensor = sess.graph.get_tensor_by_name(
           save.saver_def.filename_tensor_name)
-      self.assertEqual(sess.run(tensor), filename)
+      self.assertEqual(self.evaluate(tensor), filename)
 
   def testInvalidPath(self):
     v0 = variables.VariableV1(0, name="v0")
@@ -742,7 +742,7 @@ class SaverTest(test.TestCase):
       try:
         with self.cached_session() as sess:
           # Initialize all variables
-          sess.run(init_all_op)
+          self.evaluate(init_all_op)
 
           # Check that the parameter nodes have been initialized.
           self.assertEqual(10.0, v0.eval())
@@ -777,7 +777,7 @@ class SaverTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initialize all variables
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
 
       # Check that the parameter nodes have been initialized.
       self.assertEqual(10.0, v0.eval())
@@ -824,11 +824,11 @@ class SaverTest(test.TestCase):
     save_graph = ops_lib.Graph()
     with save_graph.as_default(), self.session(graph=save_graph) as sess:
       orig_vars = _model()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       save = saver_module.Saver(max_to_keep=1)
       variables.global_variables_initializer().run()
       save.save(sess, save_dir)
-      orig_vals = sess.run(orig_vars)
+      orig_vals = self.evaluate(orig_vars)
 
     restore_graph = ops_lib.Graph()
     with restore_graph.as_default(), self.session(
@@ -836,7 +836,7 @@ class SaverTest(test.TestCase):
       restored_vars = _model()
       save = saver_module.Saver(max_to_keep=1)
       save.restore(sess, save_dir)
-      restored_vals = sess.run(restored_vars)
+      restored_vals = self.evaluate(restored_vars)
 
     for orig, restored in zip(orig_vals, restored_vals):
       self.assertAllEqual(orig, restored)
@@ -1949,7 +1949,7 @@ class MetaGraphTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initializes all the variables.
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
       # Runs to logit.
       sess.run(logits)
       # Creates a saver.
@@ -1991,7 +1991,7 @@ class MetaGraphTest(test.TestCase):
       ops_lib.add_to_collection("train_op", train_op)
 
       # Runs train_op.
-      sess.run(train_op)
+      self.evaluate(train_op)
 
       # Generates MetaGraphDef.
       saver_module.export_meta_graph(train_filename)
@@ -2005,7 +2005,7 @@ class MetaGraphTest(test.TestCase):
       # Restores from checkpoint.
       new_saver.restore(sess, saver0_ckpt)
       train_op = ops_lib.get_collection("train_op")[0]
-      sess.run(train_op)
+      self.evaluate(train_op)
 
   def testGraphExtension(self):
     test_dir = self._get_test_dir("graph_extension")
@@ -2037,7 +2037,7 @@ class MetaGraphTest(test.TestCase):
 
       # Generate a MetaGraphDef containing the while loop.
       with session.Session() as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         sess.run(output)
         saver = saver_module.Saver()
         saver.save(sess, saver_ckpt)
@@ -2053,8 +2053,8 @@ class MetaGraphTest(test.TestCase):
       no_constfold_config.graph_options.rewrite_options.constant_folding = (
           rewriter_config_pb2.RewriterConfig.OFF)
       with session.Session(config=no_constfold_config) as sess:
-        sess.run(init_op)
-        expected_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        expected_grad_value = self.evaluate(grad)
 
     # Restore the MetaGraphDef into a new Graph.
     with ops_lib.Graph().as_default():
@@ -2070,8 +2070,8 @@ class MetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with session.Session(config=no_constfold_config) as sess:
-        sess.run(init_op)
-        actual_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        actual_grad_value = self.evaluate(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
   def _testWhileLoopAndGradientSerDes(self, outer_body_fn):
@@ -2209,7 +2209,7 @@ class MetaGraphTest(test.TestCase):
                                                       logits=logit, name="cost")
       adam.AdamOptimizer().minimize(cost, name="optimize")
       saver = saver_module.Saver()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver.save(sess, filename)
 
     graph = ops_lib.Graph()
@@ -2246,7 +2246,7 @@ class MetaGraphTest(test.TestCase):
 
       # Create a variable in graph_2 under scope "my_scope".
       variables.VariableV1(array_ops.zeros([10]), name="my_scope/my_var")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       # Restore the checkpoint into a different scope "subgraph_2".
       new_saver_2 = saver_module.import_meta_graph(
           filename + ".meta", graph=graph_2, import_scope="subgraph_2")
@@ -2279,7 +2279,7 @@ class MetaGraphTest(test.TestCase):
                                                       logits=logit, name="cost")
       adam.AdamOptimizer().minimize(cost, name="optimize")
       saver = saver_module.Saver()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver.save(sess, filename)
 
     graph = ops_lib.Graph()
@@ -2316,12 +2316,12 @@ class MetaGraphTest(test.TestCase):
           meta_graph_def, clear_devices=False, import_scope="new_model")
       # Device refers to GPU, which is not available here.
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
 
     with session.Session(graph=ops_lib.Graph()) as sess:
       saver_module.import_meta_graph(
           meta_graph_def, clear_devices=True, import_scope="new_model")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(["new_model/optimize"], {
           "new_model/image:0": np.random.random([1, 784]),
           "new_model/label:0": np.random.randint(
@@ -2348,7 +2348,7 @@ class MetaGraphTest(test.TestCase):
 
     with session.Session(graph=ops_lib.Graph()) as sess:
       saver_module.import_meta_graph(meta_graph_def, import_scope="new_model")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(["new_model/optimize"], {
           "new_model/image:0": np.random.random([1, 784]),
           "new_model/label:0": np.random.randint(
@@ -2374,7 +2374,7 @@ class MetaGraphTest(test.TestCase):
                            meta_graph_def_from_graph_def]:
       with session.Session(graph=ops_lib.Graph()) as sess:
         saver_module.import_meta_graph(meta_graph_def, import_scope="new_model")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in range(10):
           self.assertEqual(i * i, sess.run("new_model/output:0"))
         with self.assertRaises(errors.OutOfRangeError):
@@ -2400,7 +2400,7 @@ class CheckpointReaderTest(test.TestCase):
     save_path = os.path.join(self.get_temp_dir(),
                              "ckpt_for_debug_string" + str(self._WRITE_VERSION))
     with self.cached_session() as sess:
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
       # Saves a checkpoint.
       save.save(sess, save_path)
 
@@ -2546,7 +2546,7 @@ class ScopedGraphTest(test.TestCase):
       self.assertEqual(["biases:0", "weights:0"], sorted(var_list.keys()))
 
     with self.session(graph=graph) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver = saver_module.Saver(var_list=var_list, max_to_keep=1)
       saver.save(sess, os.path.join(test_dir, ckpt_filename), write_state=False)
 
@@ -2611,7 +2611,7 @@ class ScopedGraphTest(test.TestCase):
       # Verify that we have restored weights1 and biases1.
       sess.run([weights1, biases1])
       # Initialize the rest of the variables and run logits.
-      sess.run(init_rest_op)
+      self.evaluate(init_rest_op)
       sess.run(logits)
 
   # Verifies that we can save the subgraph under "hidden1" and restore it
@@ -2640,7 +2640,7 @@ class ScopedGraphTest(test.TestCase):
 
     # Run the graph and save scoped checkpoint.
     with self.session(graph=graph1) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       _, var_list_1 = meta_graph.export_scoped_meta_graph(
           export_scope="hidden1")
       saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1)
@@ -2696,7 +2696,7 @@ class ScopedGraphTest(test.TestCase):
 
     # Run the graph and save scoped checkpoint.
     with self.session(graph=graph1) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       _, var_list_1 = meta_graph.export_scoped_meta_graph(
           graph_def=graph1.as_graph_def(), export_scope="hidden1")
       saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1)
@@ -2964,7 +2964,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
     a_saver = saver_module.Saver([a])
     b_saver = saver_module.Saver([b])
     with self.cached_session() as sess:
-      sess.run(a.initializer)
+      self.evaluate(a.initializer)
       save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix)
       with self.assertRaisesRegexp(
           errors.NotFoundError, "Key b not found in checkpoint"):
@@ -2986,7 +2986,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
       a_saver = saver_module.Saver([a])
 
       with self.session(graph=g) as sess:
-        sess.run(a.initializer)
+        self.evaluate(a.initializer)
         save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix)
 
     with ops_lib.Graph().as_default() as g:
diff --git a/tensorflow/python/training/server_lib_sparse_job_test.py b/tensorflow/python/training/server_lib_sparse_job_test.py
index 1a6b44b90e..8c2745b51a 100644
--- a/tensorflow/python/training/server_lib_sparse_job_test.py
+++ b/tensorflow/python/training/server_lib_sparse_job_test.py
@@ -36,7 +36,7 @@ class SparseJobTest(test.TestCase):
       a = constant_op.constant(1.0)
 
     with session.Session(server.target) as sess:
-      self.assertEqual(1.0, sess.run(a))
+      self.assertEqual(1.0, self.evaluate(a))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py
index b734e9653e..9dc88d78cc 100644
--- a/tensorflow/python/training/supervisor_test.py
+++ b/tensorflow/python/training/supervisor_test.py
@@ -100,7 +100,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir=logdir)
       sess = sv.prepare_or_wait_for_session("")
       for _ in xrange(10):
-        sess.run(my_op)
+        self.evaluate(my_op)
       sess.close()
       sv.stop()
 
@@ -111,7 +111,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir=logdir)
       with sv.managed_session("") as sess:
         for _ in xrange(10):
-          sess.run(my_op)
+          self.evaluate(my_op)
       # Supervisor has been stopped.
       self.assertTrue(sv.should_stop())
 
@@ -128,7 +128,7 @@ class SupervisorTest(test.TestCase):
             if step == 1:
               raise RuntimeError("failing here")
             else:
-              sess.run(my_op)
+              self.evaluate(my_op)
       # Supervisor has been stopped.
       self.assertTrue(sv.should_stop())
       self.assertEqual(1, last_step)
@@ -146,7 +146,7 @@ class SupervisorTest(test.TestCase):
             raise errors_impl.OutOfRangeError(my_op.op.node_def, my_op.op,
                                               "all done")
           else:
-            sess.run(my_op)
+            self.evaluate(my_op)
       # Supervisor has been stopped.  OutOfRangeError was not thrown.
       self.assertTrue(sv.should_stop())
       self.assertEqual(3, last_step)
@@ -335,7 +335,7 @@ class SupervisorTest(test.TestCase):
       sess = sv.prepare_or_wait_for_session(
           "", config=config_pb2.ConfigProto(device_count={"CPU": 2}))
       for _ in xrange(10):
-        sess.run(my_op)
+        self.evaluate(my_op)
       sess.close()
       sv.stop()
 
diff --git a/tensorflow/python/training/warm_starting_util_test.py b/tensorflow/python/training/warm_starting_util_test.py
index b575b8d364..f1e719e6db 100644
--- a/tensorflow/python/training/warm_starting_util_test.py
+++ b/tensorflow/python/training/warm_starting_util_test.py
@@ -49,7 +49,7 @@ class WarmStartingUtilTest(test.TestCase):
     return vocab_file
 
   def _write_checkpoint(self, sess):
-    sess.run(variables.global_variables_initializer())
+    self.evaluate(variables.global_variables_initializer())
     saver = saver_lib.Saver()
     ckpt_prefix = os.path.join(self.get_temp_dir(), "model")
     saver.save(sess, ckpt_prefix, global_step=0)
@@ -125,7 +125,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose(prev_val, fruit_weights.eval(sess))
 
   def testWarmStartVarPrevVarPartitioned(self):
@@ -143,7 +143,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose(prev_val, fruit_weights.eval(sess))
 
   def testWarmStartVarCurrentVarPartitioned(self):
@@ -162,7 +162,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         fruit_weights = fruit_weights._get_variable_list()
         new_val = np.concatenate(
             [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
@@ -189,7 +189,7 @@ class WarmStartingUtilTest(test.TestCase):
             fruit_weights, prev_tensor_name="old_scope/fruit_weights")
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         fruit_weights = fruit_weights._get_variable_list()
         new_val = np.concatenate(
             [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
@@ -211,7 +211,7 @@ class WarmStartingUtilTest(test.TestCase):
             "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5,
                                            self.get_temp_dir(), prev_vocab_path)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
 
@@ -236,7 +236,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.],
                              [2.3, 2., 0.]], fruit_output_layer.eval(sess))
 
@@ -261,7 +261,7 @@ class WarmStartingUtilTest(test.TestCase):
             self.get_temp_dir(),
             prev_vocab_path,
             previous_vocab_size=2)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Old vocabulary limited to ['apple', 'banana'].
         self.assertAllClose([[0.], [0.], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
@@ -285,7 +285,7 @@ class WarmStartingUtilTest(test.TestCase):
             "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5,
                                            self.get_temp_dir(), prev_vocab_path)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
 
@@ -312,7 +312,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.],
                              [2.3, 2., 0.]], fruit_output_layer.eval(sess))
 
@@ -340,7 +340,7 @@ class WarmStartingUtilTest(test.TestCase):
             self.get_temp_dir(),
             prev_vocab_path,
             current_oov_buckets=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_weights, variables.PartitionedVariable))
         fruit_weights_vars = fruit_weights._get_variable_list()
@@ -372,7 +372,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_output_layer, variables.PartitionedVariable))
         fruit_output_layer_vars = fruit_output_layer._get_variable_list()
@@ -404,7 +404,7 @@ class WarmStartingUtilTest(test.TestCase):
             partitioner=lambda shape, dtype: [2, 1])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 6,
                                            self.get_temp_dir(), prev_vocab_path)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_weights, variables.PartitionedVariable))
         fruit_weights_vars = fruit_weights._get_variable_list()
@@ -438,7 +438,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_output_layer, variables.PartitionedVariable))
         fruit_output_layer_vars = fruit_output_layer._get_variable_list()
@@ -463,7 +463,7 @@ class WarmStartingUtilTest(test.TestCase):
             shape=[10, 1],
             initializer=zeros())
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=[var])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started (init overridden to ones).
         self.assertAllEqual(var.eval(), prev_int_val)
 
@@ -483,7 +483,7 @@ class WarmStartingUtilTest(test.TestCase):
             shape=[10, 1],
             initializer=zeros())
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=["v1"])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started (init overridden to ones).
         self.assertAllEqual(var.eval(), prev_int_val)
 
@@ -519,7 +519,7 @@ class WarmStartingUtilTest(test.TestCase):
                            # This warm-starts both v1 and v1/Momentum, but only
                            # v2 (and not v2/Momentum).
                            vars_to_warm_start=["v1", "v2[^/]"])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify the selection of weights were correctly warm-started (init
         # overridden to ones).
         self.assertAllEqual(v1.eval(), prev_v1_val)
@@ -542,7 +542,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_int], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_int: [np.zeros([10, 1])]},
@@ -553,7 +553,7 @@ class WarmStartingUtilTest(test.TestCase):
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_int], partitioner)
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=".*sc_int.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_int: [prev_int_val]}, sess)
 
@@ -571,7 +571,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_hash], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_hash: [np.zeros([15, 1])]},
@@ -583,7 +583,7 @@ class WarmStartingUtilTest(test.TestCase):
         cols_to_vars = self._create_linear_model([sc_hash], partitioner)
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*sc_hash.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_hash: [prev_hash_val]},
                                   sess)
@@ -605,7 +605,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
@@ -619,7 +619,7 @@ class WarmStartingUtilTest(test.TestCase):
         # vocab is assumed to be same as new vocab.
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*sc_vocab.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
                                   sess)
@@ -641,7 +641,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
@@ -657,7 +657,7 @@ class WarmStartingUtilTest(test.TestCase):
             # Explicitly provide the file prefix instead of just the dir.
             os.path.join(self.get_temp_dir(), "model-0"),
             vars_to_warm_start=".*sc_vocab.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
                                   sess)
@@ -686,7 +686,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([2, 1])]},
@@ -708,7 +708,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab/weights": vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  'banana' isn't in the
         # first two entries of the old vocabulary, so it's newly initialized.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [[[1], [0]]]}, sess)
@@ -729,7 +729,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([real_bucket], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars,
@@ -741,7 +741,7 @@ class WarmStartingUtilTest(test.TestCase):
         cols_to_vars = self._create_linear_model([real_bucket], partitioner)
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*real_bucketized.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars,
                                   {real_bucket: [prev_bucket_val]}, sess)
@@ -800,7 +800,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model(all_linear_cols, partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, all weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {
@@ -826,7 +826,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab/weights": vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {
             sc_int: [prev_int_val],
@@ -865,7 +865,7 @@ class WarmStartingUtilTest(test.TestCase):
             "linear_model/sc_vocab/weights",
             initializer=[[0.5], [1.], [2.], [3.]])
         self._write_checkpoint(sess)
-        prev_keys_val = sess.run(sc_keys_weights)
+        prev_keys_val = self.evaluate(sc_keys_weights)
 
     def _partitioner(shape, dtype):  # pylint:disable=unused-argument
       # Partition each var into 2 equal slices.
@@ -892,7 +892,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_hash should not be warm-started.  Var corresponding to sc_vocab
         # should be correctly warm-started after vocab remapping.
@@ -933,7 +933,7 @@ class WarmStartingUtilTest(test.TestCase):
             "linear_model/sc_vocab/weights",
             initializer=[[0.5], [1.], [2.], [3.]])
         self._write_checkpoint(sess)
-        prev_keys_val = sess.run(sc_keys_weights)
+        prev_keys_val = self.evaluate(sc_keys_weights)
 
     # New graph, new session with warm-starting.
     with ops.Graph().as_default() as g:
@@ -955,7 +955,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_hash should not be warm-started.  Var corresponding to sc_vocab
         # should be correctly warm-started after vocab remapping.
@@ -1024,7 +1024,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_vocab should be correctly warm-started after vocab remapping,
         # and neither of the other two should be warm-started..
@@ -1091,7 +1091,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[emb_vocab_column]):
                     vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started. Var corresponding to
         # emb_vocab_column should be correctly warm-started after vocab
         # remapping. Missing values are filled in with the EmbeddingColumn's
@@ -1163,7 +1163,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab_embedding/embedding_weights": vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started. Var corresponding to
         # emb_vocab should be correctly warm-started after vocab remapping.
         # Missing values are filled in with the EmbeddingColumn's initializer.
-- 
GitLab


From d63e3ea9a26fc049c654a966d0ebc56bc2747729 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 18 Nov 2018 21:05:54 -0800
Subject: [PATCH 0483/1554] Rollback of "Replace a few calls of Session `run`
 with `evaluate`" for distribute_coordinator_test to fix breakage.

PiperOrigin-RevId: 222017627
---
 tensorflow/python/distribute/distribute_coordinator_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py
index bf81ac0455..5d336648ce 100644
--- a/tensorflow/python/distribute/distribute_coordinator_test.py
+++ b/tensorflow/python/distribute/distribute_coordinator_test.py
@@ -235,7 +235,7 @@ class DistributeCoordinatorTestBase(test.TestCase):
         result = math_ops.add_n(xs)
 
       variables.global_variables_initializer().run()
-      result_value = self.evaluate(result)
+      result_value = sess.run(result)
     self.assertEqual(result_value, expected)
     if result_value == expected:
       self._result_correct += 1
@@ -294,7 +294,7 @@ class DistributeCoordinatorTestBase(test.TestCase):
           if len(uninit_vars) == 0:
             break
 
-      self.evaluate(train_op)
+      sess.run(train_op)
 
       # Synchronize workers after one step to make sure they all have finished
       # training.
@@ -327,7 +327,7 @@ class DistributeCoordinatorTestBase(test.TestCase):
 
     # The monitored session will run init or ready ops.
     with monitored_session.MonitoredSession() as sess:
-      self.evaluate(train_op)
+      sess.run(train_op)
 
       # Synchronize workers after one step to make sure they all have finished
       # training.
-- 
GitLab


From 3b4b45dd4436d007dc954d1a6c8061f2d276a6d4 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Sun, 18 Nov 2018 23:47:00 -0800
Subject: [PATCH 0484/1554] DistributionStrategy: Increase test coverage for
 CoreMirroredStrategy. Also, merged mirrored_strategy and
 mirrored_strategy_multigpu test targets into one.

PiperOrigin-RevId: 222026737
---
 tensorflow/contrib/distribute/python/BUILD    |   37 +-
 .../contrib/distribute/python/combinations.py |   16 +
 .../python/cross_device_ops_test.py           |   12 +
 .../python/estimator_training_test.py         |   47 +-
 .../python/keras_optimizer_v2_test.py         |   52 +-
 .../contrib/distribute/python/keras_test.py   |  202 +--
 .../python/mirrored_strategy_multigpu_test.py | 1165 ++++++++---------
 .../python/mirrored_strategy_test.py          |  109 --
 .../distribute/python/strategy_test_lib.py    |    3 +-
 .../contrib/distribute/python/values_test.py  |  195 ++-
 10 files changed, 846 insertions(+), 992 deletions(-)
 delete mode 100644 tensorflow/contrib/distribute/python/mirrored_strategy_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index d1dabf3f35..b5318dcfe6 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -20,11 +20,11 @@ cuda_py_test(
     name = "values_test",
     srcs = ["values_test.py"],
     additional_deps = [
-        ":mirrored_strategy",
         ":combinations",
+        ":mirrored_strategy",
         ":multi_worker_test_base",
-        "//tensorflow/core:protos_all_py",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:device_util",
@@ -222,28 +222,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "mirrored_strategy_test",
-    srcs = ["mirrored_strategy_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_pip",
-    ],
-    deps = [
-        ":mirrored_strategy",
-        ":multi_worker_test_base",
-        ":strategy_test_lib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:distribute",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python/eager:context",
-        "//tensorflow/python/eager:test",
-    ],
-)
-
 py_test(
     name = "one_device_strategy_test",
     srcs = ["one_device_strategy_test.py"],
@@ -259,10 +237,12 @@ py_test(
     ],
 )
 
+# TODO(priyag): Rename this test to mirrored_strategy_test
 cuda_py_test(
     name = "mirrored_strategy_multigpu_test",
     srcs = ["mirrored_strategy_multigpu_test.py"],
     additional_deps = [
+        ":combinations",
         ":mirrored_strategy",
         ":multi_worker_test_base",
         ":strategy_test_lib",
@@ -278,16 +258,11 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
+    shard_count = 5,
     tags = [
         "guitar",
-        "no_pip",
         "multi_and_single_gpu",
-        # Do not perform the extra analysis on this test, because it is already
-        # performed for the `:mirrored_strategy_test` target.
-        "no_oss",
-        "noasan",
-        "notap",
-        "notsan",
+        "no_pip",
     ],
 )
 
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 0fcabb1afe..f3ce547f4d 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -168,6 +168,8 @@ def _augment_with_special_arguments(test_method):
       if GPU_TEST:
         self.skipTest("Test that doesn't require GPUs.")
     elif context.num_gpus() < required_gpus:
+      # TODO(priyag): Consider allowing tests in graph mode using soft
+      # placement.
       self.skipTest(
           "{} GPUs are not available for this test. {} GPUs are available".
           format(required_gpus, context.num_gpus()))
@@ -335,6 +337,13 @@ tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", lambda: tpu_lib.TPUStrategy(
         TPUClusterResolver(""), steps_per_run=1),
     required_tpu=True)
+mirrored_strategy_with_one_cpu = NamedDistribution(
+    "Mirrored1CPU",
+    lambda: mirrored_lib.MirroredStrategy(["/cpu:0"]))
+mirrored_strategy_with_one_gpu = NamedDistribution(
+    "Mirrored1GPU",
+    lambda: mirrored_lib.MirroredStrategy(["/gpu:0"]),
+    required_gpus=1)
 mirrored_strategy_with_gpu_and_cpu = NamedDistribution(
     "MirroredCPUAndGPU",
     lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/cpu:0"]),
@@ -343,6 +352,13 @@ mirrored_strategy_with_two_gpus = NamedDistribution(
     "Mirrored2GPUs",
     lambda: mirrored_lib.MirroredStrategy(["/gpu:0", "/gpu:1"]),
     required_gpus=2)
+core_mirrored_strategy_with_one_cpu = NamedDistribution(
+    "CoreMirrored1CPU",
+    lambda: mirrored_lib.CoreMirroredStrategy(["/cpu:0"]))
+core_mirrored_strategy_with_one_gpu = NamedDistribution(
+    "CoreMirrored1GPU",
+    lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0"]),
+    required_gpus=1)
 core_mirrored_strategy_with_gpu_and_cpu = NamedDistribution(
     "CoreMirroredCPUAndGPU",
     lambda: mirrored_lib.CoreMirroredStrategy(["/gpu:0", "/cpu:0"]),
diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 36610ce3a9..00672a4401 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -391,6 +391,18 @@ class MultiWorkerCrossDeviceOpsTest(multi_worker_test_base.MultiWorkerTestBase,
               "Mirrored2GPUs",
               lambda: mirrored_strategy.MirroredStrategy(num_gpus=2),
               required_gpus=2),
+          combinations.NamedDistribution(
+              "CoreMirroredCPU",
+              lambda: mirrored_strategy.CoreMirroredStrategy(num_gpus=0),
+              required_gpus=0),
+          combinations.NamedDistribution(
+              "CoreMirrored1GPU",
+              lambda: mirrored_strategy.CoreMirroredStrategy(num_gpus=1),
+              required_gpus=1),
+          combinations.NamedDistribution(
+              "CoreMirrored2GPUs",
+              lambda: mirrored_strategy.CoreMirroredStrategy(num_gpus=2),
+              required_gpus=2),
       ],
       mode=["graph"])
 
diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index cc15fc80a9..9d8bc89b33 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -291,10 +291,13 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
           train_distribute_cls=[
               collective_all_reduce_strategy.CollectiveAllReduceStrategy,
               mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy,
               parameter_server_strategy.ParameterServerStrategy
           ],
           eval_distribute_cls=[
-              None, mirrored_strategy.MirroredStrategy,
+              None,
+              mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy,
               parameter_server_strategy.ParameterServerStrategy,
           ],
           required_gpus=[0, 1]))
@@ -322,10 +325,12 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
           mode=["graph"],
           train_distribute_cls=[
               mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy,
           ],
           eval_distribute_cls=[
               None,
               mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy,
           ],
           required_gpus=[0, 1]))
   def test_estimator_standalone_client(self, train_distribute_cls,
@@ -405,6 +410,7 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
           ],
           eval_distribute_cls=[
               None, mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy,
               parameter_server_strategy.ParameterServerStrategy,
           ],
           required_gpus=[0, 1]))
@@ -449,8 +455,15 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
   @combinations.generate(
       combinations.combine(
           mode=["graph"],
-          train_distribute_cls=[mirrored_strategy.MirroredStrategy],
-          eval_distribute_cls=[None, mirrored_strategy.MirroredStrategy],
+          train_distribute_cls=[
+              mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy
+          ],
+          eval_distribute_cls=[
+              None,
+              mirrored_strategy.MirroredStrategy,
+              mirrored_strategy.CoreMirroredStrategy
+          ],
           required_gpus=[0, 1]))
   def test_complete_flow_indepedent_worker_in_graph(self, train_distribute_cls,
                                                     eval_distribute_cls):
@@ -506,7 +519,8 @@ class RunConfigTest(test.TestCase):
         "os.environ", {"TF_CONFIG": json.dumps(TF_CONFIG_WITHOUT_TASK)}):
       run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
-              train_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2)))
+              train_distribute=mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus=2)))
 
   def test_should_run_distribute_coordinator(self):
     """Tests that should_run_distribute_coordinator return a correct value."""
@@ -529,10 +543,12 @@ class RunConfigTest(test.TestCase):
                               {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_CHIEF)}):
       config_with_train_distribute = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
-              train_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2)))
+              train_distribute=mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus=2)))
       config_with_eval_distribute = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
-              eval_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2)))
+              eval_distribute=mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus=2)))
     self.assertTrue(
         dc_training.should_run_distribute_coordinator(
             config_with_train_distribute))
@@ -545,26 +561,27 @@ class RunConfigTest(test.TestCase):
                               {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_MASTER)}):
       config = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
-              train_distribute=mirrored_strategy.MirroredStrategy(num_gpus=2)))
+              train_distribute=mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus=2)))
     self.assertFalse(dc_training.should_run_distribute_coordinator(config))
 
   def test_init_run_config_duplicate_distribute(self):
     with self.assertRaises(ValueError):
       run_config_lib.RunConfig(
-          train_distribute=mirrored_strategy.MirroredStrategy(),
+          train_distribute=mirrored_strategy.CoreMirroredStrategy(),
           experimental_distribute=DistributeConfig(
-              train_distribute=mirrored_strategy.MirroredStrategy()))
+              train_distribute=mirrored_strategy.CoreMirroredStrategy()))
 
     with self.assertRaises(ValueError):
       run_config_lib.RunConfig(
-          eval_distribute=mirrored_strategy.MirroredStrategy(),
+          eval_distribute=mirrored_strategy.CoreMirroredStrategy(),
           experimental_distribute=DistributeConfig(
-              eval_distribute=mirrored_strategy.MirroredStrategy()))
+              eval_distribute=mirrored_strategy.CoreMirroredStrategy()))
 
   def test_init_run_config_none_distribute_coordinator_mode(self):
     # We don't use distribute coordinator for local training.
     config = run_config_lib.RunConfig(
-        train_distribute=mirrored_strategy.MirroredStrategy())
+        train_distribute=mirrored_strategy.CoreMirroredStrategy())
     dc_training.init_run_config(config, {})
     self.assertIsNone(config._distribute_coordinator_mode)
 
@@ -572,7 +589,7 @@ class RunConfigTest(test.TestCase):
     with test.mock.patch.dict("os.environ",
                               {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_MASTER)}):
       config = run_config_lib.RunConfig(
-          train_distribute=mirrored_strategy.MirroredStrategy())
+          train_distribute=mirrored_strategy.CoreMirroredStrategy())
       self.assertIsNone(config._distribute_coordinator_mode)
 
     # When `train_distribute` is not specified, don't use distribute
@@ -588,7 +605,7 @@ class RunConfigTest(test.TestCase):
     with test.mock.patch.dict("os.environ",
                               {"TF_CONFIG": json.dumps(TF_CONFIG_WITH_CHIEF)}):
       config = run_config_lib.RunConfig(
-          train_distribute=mirrored_strategy.MirroredStrategy())
+          train_distribute=mirrored_strategy.CoreMirroredStrategy())
     self.assertEqual(config._distribute_coordinator_mode,
                      dc.CoordinatorMode.INDEPENDENT_WORKER)
 
@@ -597,7 +614,7 @@ class RunConfigTest(test.TestCase):
     # `experimental.remote_cluster` is set use distribute coordinator with
     # STANDALONE_CLIENT mode.
     config = run_config_lib.RunConfig(
-        train_distribute=mirrored_strategy.MirroredStrategy(),
+        train_distribute=mirrored_strategy.CoreMirroredStrategy(),
         experimental_distribute=DistributeConfig(
             remote_cluster={"chief": ["fake_worker"]}))
     self.assertEqual(config._distribute_coordinator_mode,
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 40f10922b1..0d7e11c3b6 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -25,11 +25,9 @@ import numpy as np
 import six
 
 from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
 from tensorflow.python.estimator import training
 from tensorflow.python.estimator.canned import dnn_linear_combined
@@ -152,12 +150,14 @@ def get_model():
   return model
 
 
-class MirroredStrategyOptimizerV2Test(test.TestCase):
-
-  def testKerasOptimizerWithUnequalInput(self):
-    if context.num_gpus() < 1:
-      self.skipTest('Not enough GPUs.')
+class MirroredStrategyOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
+      mode=['graph']))
+  def testKerasOptimizerWithUnequalInput(self, distribution):
     def create_fn():
       var = variables.Variable(
           2.0, name='var', aggregation=variable_scope.VariableAggregation.SUM)
@@ -170,25 +170,24 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       return (var, m, v, train_op, optimizer.iterations)
 
     devices = ['/device:GPU:0', '/device:CPU:0']
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
-      (var, m, v, op, counter) = dist.call_for_each_replica(create_fn)
+    with distribution.scope():
+      (var, m, v, op, counter) = distribution.call_for_each_replica(create_fn)
       self.evaluate(variables.global_variables_initializer())
       var_val = [2.0, 2.0, 2.0]
       self.assertAllClose(
           var_val,
           self.evaluate(
-              [dist.read_var(var),
+              [distribution.read_var(var),
                var.get(devices[0]),
                var.get(devices[1])]))
       self.assertAllClose([0, 0, 0],
                           self.evaluate([
-                              dist.read_var(counter),
+                              distribution.read_var(counter),
                               counter.get(devices[0]),
                               counter.get(devices[1])
                           ]))
 
-      train_op = dist.unwrap(op)
+      train_op = distribution.unwrap(op)
       self.evaluate(train_op)
       # m(1) = beta1 * m(0) + (1-beta1) * grad = 0.2 * 0 + 0.8 * (1 + 2) / 2
       m_val = [1.2, 1.2, 1.2]
@@ -196,7 +195,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       self.assertAllClose(
           m_val,
           self.evaluate(
-              [dist.read_var(m),
+              [distribution.read_var(m),
                m.get(devices[0]),
                m.get(devices[1])]))
       # v(1) = beta2 * v(0) + (1-beta2) * grad^2 = 0.2 * 0 + 0.8 * 2.25
@@ -204,7 +203,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       self.assertAllClose(
           v_val,
           self.evaluate(
-              [dist.read_var(v),
+              [distribution.read_var(v),
                v.get(devices[0]),
                v.get(devices[1])]))
       # var(1) = var(0) - lr * m(1) * sqrt(1 - beta2) / sqrt(v(1)) / (1 - beta1)
@@ -213,12 +212,12 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       self.assertAllClose(
           var_val,
           self.evaluate(
-              [dist.read_var(var),
+              [distribution.read_var(var),
                var.get(devices[0]),
                var.get(devices[1])]))
       self.assertAllClose([1, 1, 1],
                           self.evaluate([
-                              dist.read_var(counter),
+                              distribution.read_var(counter),
                               counter.get(devices[0]),
                               counter.get(devices[1])
                           ]))
@@ -229,7 +228,7 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       self.assertAllClose(
           m_val,
           self.evaluate(
-              [dist.read_var(m),
+              [distribution.read_var(m),
                m.get(devices[0]),
                m.get(devices[1])]))
       # v(2) = beta2 * v(1) + (1-beta2) * grad^2 = 0.2 * 1.8 + 0.8 * 2.25
@@ -237,28 +236,29 @@ class MirroredStrategyOptimizerV2Test(test.TestCase):
       self.assertAllClose(
           v_val,
           self.evaluate(
-              [dist.read_var(v),
+              [distribution.read_var(v),
                v.get(devices[0]),
                v.get(devices[1])]))
       self.assertAllClose([2, 2, 2],
                           self.evaluate([
-                              dist.read_var(counter),
+                              distribution.read_var(counter),
                               counter.get(devices[0]),
                               counter.get(devices[1])
                           ]))
 
-  def testOptimizerWithKerasModelAndNumpyArrays(self):
-    if context.num_gpus() < 1:
-      self.skipTest('Not enough GPUs.')
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
+      mode=['graph']))
+  def testOptimizerWithKerasModelAndNumpyArrays(self, distribution):
 
     with self.cached_session():
       model = get_model()
       optimizer = gradient_descent.SGD(0.001)
       loss = 'mse'
       metrics = ['mae']
-      devices = ['/device:GPU:0', '/device:CPU:0']
-      dist = mirrored_strategy.MirroredStrategy(devices)
-      model.compile(optimizer, loss, metrics=metrics, distribute=dist)
+      model.compile(optimizer, loss, metrics=metrics, distribute=distribution)
 
       inputs = np.zeros((64, 3), dtype=np.float32)
       targets = np.zeros((64, 4), dtype=np.float32)
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 0776d1772c..2df8d7a66e 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -320,7 +320,8 @@ def strategy_and_inputs():
       mode=['graph'])
 
 
-class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
+class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
+                                        parameterized.TestCase):
 
   def setUp(self):
     self._base_dir = os.path.join(self.get_temp_dir(),
@@ -328,17 +329,18 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
     gfile.MakeDirs(self._base_dir)
     self._config = run_config_lib.RunConfig(
         tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir)
-    self._dist = mirrored_strategy.MirroredStrategy(
-        devices=['/device:GPU:0', '/device:GPU:1'])
 
   def tearDown(self):
     writer_cache.FileWriterCache.clear()
     if os.path.isdir(self._base_dir):
       gfile.DeleteRecursively(self._base_dir)
 
-  def test_train_functional_with_distribution_strategy(self):
-    dist = mirrored_strategy.MirroredStrategy(
-        devices=['/device:GPU:0', '/device:GPU:1'])
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_train_functional_with_distribution_strategy(self, distribution):
     keras_model = simple_functional_model()
     keras_model.compile(
         loss='categorical_crossentropy',
@@ -346,8 +348,8 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
         optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01))
     config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED,
                                       model_dir=self._base_dir,
-                                      train_distribute=dist,
-                                      eval_distribute=dist)
+                                      train_distribute=distribution,
+                                      eval_distribute=distribution)
     with self.cached_session():
       est_keras = keras_lib.model_to_estimator(
           keras_model=keras_model, config=config)
@@ -361,9 +363,12 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
     writer_cache.FileWriterCache.clear()
     gfile.DeleteRecursively(self._config.model_dir)
 
-  def test_train_sequential_with_distribution_strategy(self):
-    dist = mirrored_strategy.MirroredStrategy(
-        devices=['/device:GPU:0', '/device:GPU:1'])
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_train_sequential_with_distribution_strategy(self, distribution):
     keras_model = simple_sequential_model()
     keras_model.compile(
         loss='categorical_crossentropy',
@@ -371,7 +376,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
         optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.01))
     config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED,
                                       model_dir=self._base_dir,
-                                      train_distribute=dist)
+                                      train_distribute=distribution)
     with self.cached_session():
       est_keras = keras_lib.model_to_estimator(
           keras_model=keras_model, config=config)
@@ -385,7 +390,12 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
     writer_cache.FileWriterCache.clear()
     gfile.DeleteRecursively(self._config.model_dir)
 
-  def test_multi_inputs_multi_outputs_with_input_fn_as_dict(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_multi_inputs_multi_outputs_with_input_fn_as_dict(self, distribution):
     train_data, test_data = get_multi_inputs_multi_outputs_data()
 
     def train_input_fn():
@@ -415,14 +425,14 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
                                                      output_dict)).batch(16)
 
     self.do_test_multi_inputs_multi_outputs_with_input_fn(
-        train_input_fn, eval_input_fn)
+        distribution, train_input_fn, eval_input_fn)
 
-  def do_test_multi_inputs_multi_outputs_with_input_fn(self, train_input_fn,
-                                                       eval_input_fn):
+  def do_test_multi_inputs_multi_outputs_with_input_fn(
+      self, distribution, train_input_fn, eval_input_fn):
     config = run_config_lib.RunConfig(
         tf_random_seed=_RANDOM_SEED,
         model_dir=self._base_dir,
-        train_distribute=self._dist)
+        train_distribute=distribution)
     with self.cached_session():
       model = multi_inputs_multi_outputs_model()
       est_keras = keras_lib.model_to_estimator(keras_model=model, config=config)
@@ -432,9 +442,12 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
       eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1)
       self.assertLess(eval_results['loss'], baseline_eval_results['loss'])
 
-  def test_keras_optimizer_with_distribution_strategy(self):
-    dist = mirrored_strategy.MirroredStrategy(
-        devices=['/device:GPU:0', '/device:GPU:1'])
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_keras_optimizer_with_distribution_strategy(self, distribution):
     keras_model = simple_sequential_model()
     keras_model.compile(
         loss='categorical_crossentropy',
@@ -442,7 +455,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase):
 
     config = run_config_lib.RunConfig(tf_random_seed=_RANDOM_SEED,
                                       model_dir=self._base_dir,
-                                      train_distribute=dist)
+                                      train_distribute=distribution)
     with self.cached_session():
       est_keras = keras_lib.model_to_estimator(keras_model=keras_model,
                                                config=config)
@@ -467,6 +480,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # Verify that the numpy value is copied to the variable.
       self.assertAllEqual(x, val)
 
+  # TODO(sourabhbajaj): Test this with all strategies.
   def test_calculating_batch_params(self):
     # This verifies that we calculate the number of steps when the batch size
     # is specified.
@@ -515,6 +529,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # number of batches(4) to the number of replicas(2).
       self.assertEqual(steps, 2)
 
+  # TODO(sourabhbajaj): Test this with all strategies.
   def test_calculating_batch_size(self):
     with self.cached_session():
       # 64 is the number of input samples.
@@ -713,16 +728,20 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
   # TODO(priyag): Enable this test for TPU. Currently tuples/dict don't work
   # as clone_model's input_tensors argument only seems to accept list and not
   # tuples or dict.
-  def test_fit_with_tuple_and_dict_dataset_inputs(self):
+
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
+      mode=['graph']))
+  def test_fit_with_tuple_and_dict_dataset_inputs(self, distribution):
     with self.cached_session():
       model = multi_input_output_model()
 
       optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.001)
       loss = 'mse'
       metrics = ['mae', keras.metrics.CategoricalAccuracy()]
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
-                                                     '/device:CPU:0'])
-      model.compile(optimizer, loss, metrics=metrics, distribute=strategy)
+      model.compile(optimizer, loss, metrics=metrics, distribute=distribution)
 
       input_a_np = np.random.random((10, 3))
       input_b_np = np.random.random((10, 5))
@@ -795,35 +814,48 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
     model.evaluate(dataset, steps=2, verbose=1)
     model.predict(dataset, steps=2)
 
-  def test_dataset_input_shape_validation(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_dataset_wrong_input_shape(self, distribution):
     with self.cached_session():
       model = get_model()
 
       optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
       loss = 'mse'
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1',
-                                                     '/device:GPU:0'])
-
-      model.compile(optimizer, loss, distribute=strategy)
+      model.compile(optimizer, loss, distribute=distribution)
 
-      # User forgets to batch the dataset
-      inputs = np.zeros((10, 3), dtype=np.float32)
+      # Wrong input shape
+      inputs = np.zeros((10, 5), dtype=np.float32)
       targets = np.zeros((10, 4), dtype=np.float32)
       dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
       dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
 
-      with self.assertRaisesRegexp(ValueError, 'expected input to have shape'):
+      with self.assertRaisesRegexp(ValueError,
+                                   'expected input to have shape'):
         model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0)
 
-      # Wrong input shape
-      inputs = np.zeros((10, 5), dtype=np.float32)
+  @combinations.generate(combinations.combine(
+      distribution=[combinations.mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_dataset_no_batch_input_validation(self, distribution):
+    with self.cached_session():
+      model = get_model()
+
+      optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
+      loss = 'mse'
+      model.compile(optimizer, loss, distribute=distribution)
+
+      # User forgets to batch the dataset
+      inputs = np.zeros((10, 3), dtype=np.float32)
       targets = np.zeros((10, 4), dtype=np.float32)
       dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
       dataset = dataset.repeat(100)
-      dataset = dataset.batch(10)
 
-      with self.assertRaisesRegexp(ValueError,
-                                   'expected input to have shape'):
+      with self.assertRaisesRegexp(ValueError, 'expected input to have shape'):
         model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0)
 
   @combinations.generate(combinations.combine(
@@ -845,7 +877,12 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       with self.assertRaisesRegexp(ValueError, 'requires fully defined shapes'):
         model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0)
 
-  def test_learning_phase_value(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_learning_phase_value(self, distribution):
     # TODO(anjalisridhar): Modify this test to use Lambdas since we can compare
     # meaningful values. Currently we don't pass the learning phase if the
     # Lambda layer uses the learning phase.
@@ -859,15 +896,17 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       optimizer = gradient_descent.GradientDescentOptimizer(0.005)
       loss = 'mse'
       metrics = ['acc']
-      strategy = mirrored_strategy.MirroredStrategy(
-          ['/device:GPU:0', '/device:GPU:1'])
+      model.compile(optimizer, loss, metrics=metrics, distribute=distribution)
 
-      model.compile(optimizer, loss, metrics=metrics, distribute=strategy)
+      batch_size = 8
+      if isinstance(distribution, mirrored_strategy.CoreMirroredStrategy):
+        # CoreMirroredStrategy uses global batch size.
+        batch_size = 8 * distribution.num_replicas_in_sync
 
       inputs = np.ones((10, 1), dtype=np.float32)
       targets = np.ones((10, 1), dtype=np.float32)
       dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-      dataset = dataset.repeat().batch(8)
+      dataset = dataset.repeat().batch(batch_size)
       hist = model.fit(dataset, epochs=1, steps_per_epoch=20, verbose=1)
       self.assertAlmostEqual(hist.history['acc'][0], 0, 0)
 
@@ -878,24 +917,29 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
       inputs = np.ones((10, 1), dtype=np.float32)
       predict_dataset = dataset_ops.Dataset.from_tensor_slices(inputs)
-      predict_dataset = predict_dataset.repeat().batch(5)
+
+      predict_dataset = predict_dataset.repeat().batch(batch_size)
       output = model.predict(predict_dataset, steps=10)
-      # `predict` runs for 10 steps and in each step you process 10 samples.
-      ref_output = np.ones((100, 1), dtype=np.float32)
+      # `predict` runs for 10 steps
+      ref_output = np.ones((160, 1), dtype=np.float32)
       self.assertArrayNear(output, ref_output, 1e-1)
 
 
 class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
 
-  def test_validating_dataset_input_tensors_with_shape_mismatch(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
+      mode=['graph']))
+  def test_validating_dataset_input_tensors_with_shape_mismatch(self,
+                                                                distribution):
     with self.cached_session():
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
-                                                     '/device:CPU:0'])
       a = constant_op.constant([1, 2], shape=(1, 2))
       b = constant_op.constant([[1, 2], [1, 2]], shape=(2, 2))
       x = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': b})
       y = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': a})
-      with strategy.scope():
+      with distribution.scope():
         # Removed device and input tensor shape details from the error message
         # since the order of the device and the corresponding input tensor shape
         # is not deterministic over different runs.
@@ -904,17 +948,21 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
                                      'distributed tensor inputs '
                                      'DistributedValues:.+'):
           distributed_training_utils.validate_distributed_dataset_inputs(
-              strategy, x, y)
+              distribution, x, y)
 
-  def test_validating_dataset_input_tensors_with_dtype_mismatch(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
+      mode=['graph']))
+  def test_validating_dataset_input_tensors_with_dtype_mismatch(self,
+                                                                distribution):
     with self.cached_session():
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
-                                                     '/device:CPU:0'])
       a = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.int32)
       b = constant_op.constant([1, 2], shape=(1, 2), dtype=dtypes.float64)
       x = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': b})
       y = values.DistributedValues({'/device:CPU:0': a, '/device:GPU:0': a})
-      with strategy.scope():
+      with distribution.scope():
         # Removed device and input tensor dtype details from the error message
         # since the order of the device and the corresponding input tensor dtype
         # is not deterministic over different runs.
@@ -923,21 +971,23 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
                                      'distributed tensor inputs '
                                      'DistributedValues:.+'):
           distributed_training_utils.validate_distributed_dataset_inputs(
-              strategy, x, y)
+              distribution, x, y)
 
-  def test_unsupported_features(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_unsupported_features(self, distribution):
     with self.cached_session():
       model = get_model()
 
       optimizer = gradient_descent.GradientDescentOptimizer(0.001)
       loss = 'mse'
       metrics = ['mae']
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1',
-                                                     '/device:GPU:0'])
-
-      model.compile(optimizer, loss, metrics=metrics, distribute=strategy)
+      model.compile(optimizer, loss, metrics=metrics, distribute=distribution)
 
-      dataset = get_dataset(strategy)
+      dataset = get_dataset(distribution)
 
       # Test with validation split
       with self.assertRaisesRegexp(
@@ -972,18 +1022,21 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
                                    'you should specify the `steps` argument'):
         model.predict(dataset, verbose=0)
 
-  def test_calling_with_unsupported_predefined_callbacks(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_calling_with_unsupported_predefined_callbacks(self, distribution):
     with self.cached_session():
       model = get_model()
 
       optimizer = gradient_descent.GradientDescentOptimizer(0.001)
       loss = 'mse'
       metrics = ['mae']
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1',
-                                                     '/device:GPU:0'])
-      model.compile(optimizer, loss, metrics=metrics, distribute=strategy)
+      model.compile(optimizer, loss, metrics=metrics, distribute=distribution)
 
-      dataset = get_dataset(strategy)
+      dataset = get_dataset(distribution)
 
       def schedule(_):
         return 0.001
@@ -1006,11 +1059,17 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
                   callbacks=[keras.callbacks.TensorBoard(histogram_freq=10)])
 
 
-class TestDistributionStrategyWithLossMasking(test.TestCase):
+class TestDistributionStrategyWithLossMasking(test.TestCase,
+                                              parameterized.TestCase):
 
   # TODO(priyag): Enable all strategies for this test. Currently it does not
   # work for TPU due to some invalid datatype.
-  def test_masking(self):
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph']))
+  def test_masking(self, distribution):
     with self.cached_session():
       np.random.seed(1337)
       x = np.array([[[1], [1]], [[0], [0]]])
@@ -1019,12 +1078,9 @@ class TestDistributionStrategyWithLossMasking(test.TestCase):
       model.add(
           keras.layers.TimeDistributed(
               keras.layers.Dense(1, kernel_initializer='one')))
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:1',
-                                                     '/device:GPU:0'])
-
       model.compile(loss='mse',
                     optimizer=gradient_descent.GradientDescentOptimizer(0.01),
-                    distribute=strategy)
+                    distribute=distribution)
       y = np.array([[[1], [1]], [[1], [1]]])
       dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
       dataset = dataset.repeat(100)
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 2cc56565d3..9fd4cca319 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -20,12 +20,13 @@ from __future__ import print_function
 
 import sys
 
+from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -36,7 +37,6 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training as keras_training
 from tensorflow.python.keras.layers import core as keras_core
 from tensorflow.python.layers import core
@@ -57,100 +57,54 @@ from tensorflow.python.training import server_lib
 GPU_TEST = "test_gpu" in sys.argv[0]
 
 
-class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.mirrored_strategy_with_two_gpus,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_two_gpus],
+    mode=["graph", "eager"]))
+class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
+                                        parameterized.TestCase):
 
-  def _get_distribution_strategy(self):
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    if GPU_TEST:
-      self.assertGreater(context.num_gpus(), 0)
-      if context.num_gpus() > 1:
-        devices = ["/device:GPU:0", "/device:GPU:1"]
-    print(self.id().split(".")[-1], "devices:", ", ".join(devices))
-    return mirrored_strategy.MirroredStrategy(devices)
+  def testMinimizeLoss(self, distribution):
+    if context.executing_eagerly():
+      self._test_minimize_loss_eager(distribution)
+    else:
+      self._test_minimize_loss_graph(distribution)
 
-  def testMinimizeLossEager(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self._test_minimize_loss_eager(self._get_distribution_strategy())
+  def testReplicaId(self, distribution):
+    self._test_replica_id(distribution)
 
-  def testMinimizeLossGraph(self):
-    soft_placement = not GPU_TEST
-    print("testMinimizeLossGraph soft_placement:", soft_placement)
-    self._test_minimize_loss_graph(
-        self._get_distribution_strategy(), soft_placement=soft_placement)
-
-  def testReplicaId(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self._test_replica_id(self._get_distribution_strategy())
-
-  def testNumReplicasInSync(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self.assertEqual(2, self._get_distribution_strategy().
-                     num_replicas_in_sync)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testCallAndMergeExceptions(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self._test_call_and_merge_exceptions(self._get_distribution_strategy())
-
-  @test_util.run_in_graph_and_eager_modes
-  def testRunRegroupError(self):
+  def testNumReplicasInSync(self, distribution):
+    self.assertEqual(2, distribution.num_replicas_in_sync)
 
+  def testCallAndMergeExceptions(self, distribution):
+    self._test_call_and_merge_exceptions(distribution)
+
+  def testRunRegroupError(self, distribution):
     def run_fn():
       replica_id = int(self.evaluate(_replica_id()))
       # Generates a list with different lengths on different devices.
       # Will fail in _regroup() (if more than one device).
       return list(range(replica_id))
 
-    dist = self._get_distribution_strategy()
-    with dist.scope(), self.assertRaises(AssertionError):
-      dist.call_for_each_replica(run_fn)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testReduceToCpu(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
+    with distribution.scope(), self.assertRaises(AssertionError):
+      distribution.call_for_each_replica(run_fn)
 
-    dist = self._get_distribution_strategy()
-    with dist.scope():
-      result = dist.call_for_each_replica(_replica_id)
-      reduced = dist.reduce(
+  def testReduceToCpu(self, distribution):
+    with distribution.scope():
+      result = distribution.call_for_each_replica(_replica_id)
+      reduced = distribution.reduce(
           reduce_util.ReduceOp.SUM,
           result,
           destinations="/device:CPU:0")
-      unwrapped = dist.unwrap(reduced)
+      unwrapped = distribution.unwrap(reduced)
       self.assertEqual(1, len(unwrapped))
-      expected = sum(range(dist.num_replicas_in_sync))
+      expected = sum(range(distribution.num_replicas_in_sync))
       self.assertEqual(expected, self.evaluate(unwrapped[0]))
 
-  @test_util.run_in_graph_and_eager_modes()
-  def testReduceToMultipleDestinations(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-
-    devices = ["/device:GPU:0"]
-    if GPU_TEST:
-      self.assertGreater(context.num_gpus(), 0)
-    print(self.id().split(".")[-1], "devices:", ", ".join(devices))
-
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
-      reduced = dist.reduce(
-          reduce_util.ReduceOp.SUM,
-          1.0,
-          destinations=["/device:CPU:0", "/device:GPU:0"])
-      unwrapped = dist.unwrap(reduced)
-      self.assertEqual(2, len(unwrapped))
-      self.assertEqual(1.0, self.evaluate(unwrapped[0]))
-
-  @test_util.run_in_graph_and_eager_modes
-  def testMakeInputFnIterator(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    d = self._get_distribution_strategy()
+  def testMakeInputFnIterator(self, distribution):
     dataset_fn = lambda: dataset_ops.Dataset.range(10)
     expected_values = [[i, i+1] for i in range(0, 10, 2)]
 
@@ -159,66 +113,137 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
         expected_num_replicas_in_sync=2,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
-    iterator = d.make_input_fn_iterator(input_fn)
-    self._test_input_fn_iterator(iterator, d.worker_devices, expected_values)
+    iterator = distribution.make_input_fn_iterator(input_fn)
+    self._test_input_fn_iterator(iterator, distribution.worker_devices,
+                                 expected_values)
+
+  def testGlobalStepUpdate(self, distribution):
+    self._test_global_step_update(distribution)
+
+
+def one_device_combinations():
+  return combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_one_cpu,
+          combinations.mirrored_strategy_with_one_gpu,
+          combinations.core_mirrored_strategy_with_one_cpu,
+          combinations.core_mirrored_strategy_with_one_gpu],
+      mode=["graph", "eager"])
+
+
+class MirroredOneDeviceDistributionTest(
+    strategy_test_lib.DistributionTestBase,
+    parameterized.TestCase):
+
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.NamedDistribution(
+              "Mirrored1CPU",
+              lambda: mirrored_strategy.MirroredStrategy(["/device:CPU:0"]),
+              required_gpus=1),
+          combinations.mirrored_strategy_with_one_gpu,
+          combinations.NamedDistribution(
+              "CoreMirrored1CPU",
+              lambda: mirrored_strategy.CoreMirroredStrategy(["/device:CPU:0"]),
+              required_gpus=1),
+          combinations.core_mirrored_strategy_with_one_gpu],
+      mode=["graph", "eager"]))
+  def testReduceToMultipleDestinations(self, distribution):
+    with distribution.scope():
+      reduced = distribution.reduce(
+          reduce_util.ReduceOp.SUM,
+          1.0,
+          destinations=["/device:CPU:0", "/device:GPU:0"])
+      unwrapped = distribution.unwrap(reduced)
+      self.assertLen(unwrapped, 2)
+      self.assertEqual(1.0, self.evaluate(unwrapped[0]))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testGlobalStepUpdate(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-    self._test_global_step_update(self._get_distribution_strategy())
+  @combinations.generate(one_device_combinations())
+  def testMinimizeLoss(self, distribution):
+    if context.executing_eagerly():
+      self._test_minimize_loss_eager(distribution)
+    else:
+      self._test_minimize_loss_graph(distribution)
 
+  @combinations.generate(one_device_combinations())
+  def testReplicaId(self, distribution):
+    self._test_replica_id(distribution)
 
-class MirroredStrategyVariableCreationTest(test.TestCase):
+  @combinations.generate(one_device_combinations())
+  def testCallAndMergeExceptions(self, distribution):
+    self._test_call_and_merge_exceptions(distribution)
 
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
 
-  def _skip_eager_if_gpus_less_than(self, num_gpus):
-    if context.num_gpus() < num_gpus and context.executing_eagerly():
-      self.skipTest("Enough GPUs not available for this test in eager mode.")
+class MirroredStrategyVariableCreatorStackTest(
+    test.TestCase, parameterized.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSingleVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
+  @combinations.generate(combinations.combine(
+      distribution=[combinations.mirrored_strategy_with_gpu_and_cpu,
+                    combinations.core_mirrored_strategy_with_gpu_and_cpu],
+      mode=["graph"]))
+  def testCreatorStacksAreThreadLocal(self, distribution):
+    def model_fn():
+      replica_id_str = str(self.evaluate(_replica_id()))
 
+      def thread_creator_fn(next_creator, *args, **kwargs):
+        return next_creator(*args, **kwargs) + ":thread_" + replica_id_str
+
+      with variable_scope.variable_creator_scope(thread_creator_fn):
+        # Create a variable in this scope.
+        v = variable_scope.variable(1.0)
+
+        # This will pause the current thread, and execute the other thread.
+        ds_context.get_replica_context().merge_call(lambda _: _)
+      return v
+
+    def main_thread_creator(next_creator, *args, **kwargs):
+      # We are not using the underlying next_creator for test purposes.
+      del next_creator, args, kwargs
+      return "main_thread"
+
+    with context.graph_mode(), \
+        distribution.scope(), \
+        variable_scope.variable_creator_scope(main_thread_creator):
+      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.unwrap(result)
+      expected = ["main_thread:thread_0", "main_thread:thread_1"]
+      self.assertEqual(expected, result)
+
+
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph", "eager"]))
+class MirroredStrategyVariableCreationTest(test.TestCase):
+
+  def testSingleVariable(self, distribution):
     def model_fn():
       # This variable should be created only once across the threads because of
-      # special variable_creator functions used by `dist.call_for_each_replica`.
+      # special variable_creator functions used by
+      # `distribution.call_for_each_replica`.
       v = variable_scope.variable(1.0, name="foo")
       ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
-      self.assertEquals("foo:0", result.name)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testUnnamedVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
+      self.assertEqual("foo:0", result.name)
 
+  def testUnnamedVariable(self, distribution):
     def model_fn():
       v = variable_scope.variable(1.0)
       ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       # Default name of "Variable" will be used.
-      self.assertEquals("Variable:0", result.name)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testMultipleVariables(self):
-    self._skip_eager_if_gpus_less_than(1)
+      self.assertEqual("Variable:0", result.name)
 
+  def testMultipleVariables(self, distribution):
     def model_fn():
       vs = []
       for i in range(5):
@@ -226,19 +251,13 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       ds_context.get_replica_context().merge_call(lambda _: _)
       return vs
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
       for i, v in enumerate(result):
         self.assertIsInstance(v, values.MirroredVariable)
-        self.assertEquals("foo" + str(i) + ":0", v.name)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testMultipleVariablesWithSameCanonicalName(self):
-    self._skip_eager_if_gpus_less_than(1)
+        self.assertEqual("foo" + str(i) + ":0", v.name)
 
+  def testMultipleVariablesWithSameCanonicalName(self, distribution):
     def model_fn():
       vs = []
       vs.append(variable_scope.variable(1.0, name="foo/bar"))
@@ -248,41 +267,30 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       ds_context.get_replica_context().merge_call(lambda _: _)
       return vs
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
       for v in result:
         self.assertIsInstance(v, values.MirroredVariable)
-      self.assertEquals(4, len(result))
-      self.assertEquals("foo/bar:0", result[0].name)
-      self.assertEquals("foo_1/bar:0", result[1].name)
-      self.assertEquals("foo_1/bar_1:0", result[2].name)
-      self.assertEquals("foo/bar_1:0", result[3].name)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testVariableWithSameCanonicalNameAcrossThreads(self):
-    self._skip_eager_if_gpus_less_than(1)
+      self.assertEqual(4, len(result))
+      self.assertEqual("foo/bar:0", result[0].name)
+      self.assertEqual("foo_1/bar:0", result[1].name)
+      self.assertEqual("foo_1/bar_1:0", result[2].name)
+      self.assertEqual("foo/bar_1:0", result[3].name)
 
+  def testVariableWithSameCanonicalNameAcrossThreads(self, distribution):
     def model_fn():
       replica_id = self.evaluate(_replica_id())
       v = variable_scope.variable(1.0, name="foo_" + str(replica_id))
       ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       # The resulting mirrored variable will use the name from the first device.
-      self.assertEquals("foo_0:0", result.name)
+      self.assertEqual("foo_0:0", result.name)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testWithLayers(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testWithLayers(self, distribution):
     def model_fn(features):
       with variable_scope.variable_scope("common"):
         layer1 = core.Dense(1)
@@ -297,9 +305,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
                 (layer2.kernel, layer2.bias),
                 (layer3.kernel, layer3.bias)]
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-    ds = dist.distribute_dataset(
+    ds = distribution.distribute_dataset(
         lambda: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10))
     if context.executing_eagerly():
       iterator = ds.make_one_shot_iterator()
@@ -309,19 +315,16 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     features = iterator.get_next()
 
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn, args=(features,))
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn, args=(features,))
       suffixes = ["", "_1", "_2"]
       for (kernel, bias), suffix in zip(result, suffixes):
         self.assertIsInstance(kernel, values.MirroredVariable)
-        self.assertEquals("common/dense" + suffix + "/kernel:0", kernel.name)
+        self.assertEqual("common/dense" + suffix + "/kernel:0", kernel.name)
         self.assertIsInstance(bias, values.MirroredVariable)
-        self.assertEquals("common/dense" + suffix + "/bias:0", bias.name)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testWithVariableAndVariableScope(self):
-    self._skip_eager_if_gpus_less_than(1)
+        self.assertEqual("common/dense" + suffix + "/bias:0", bias.name)
 
+  def testWithVariableAndVariableScope(self, distribution):
     def model_fn():
       v0 = variable_scope.variable(1.0, name="var0", aggregation=None)
       with variable_scope.variable_scope("common"):
@@ -341,30 +344,25 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
       return v0, v1, v2, v3
 
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+    with distribution.scope():
       v = variable_scope.variable(1.0, name="var-main0")
-      self.assertEquals("var-main0:0", v.name)
+      self.assertEqual("var-main0:0", v.name)
 
-      result = dist.call_for_each_replica(model_fn)
-      self.assertEquals(4, len(result))
+      result = distribution.call_for_each_replica(model_fn)
+      self.assertEqual(4, len(result))
       v0, v1, v2, v3 = result
       self.assertIsInstance(v0, values.MirroredVariable)
-      self.assertEquals("var0:0", v0.name)
+      self.assertEqual("var0:0", v0.name)
       self.assertIsInstance(v1, values.MirroredVariable)
-      self.assertEquals("common/var1:0", v1.name)
+      self.assertEqual("common/var1:0", v1.name)
       self.assertIsInstance(v2, values.ReplicaLocalVariable)
-      self.assertEquals("common/var2:0", v2.name)
-      self.assertEquals(variable_scope.VariableAggregation.SUM, v2.aggregation)
+      self.assertEqual("common/var2:0", v2.name)
+      self.assertEqual(variable_scope.VariableAggregation.SUM, v2.aggregation)
       self.assertIsInstance(v3, values.MirroredVariable)
-      self.assertEquals("common/var3:0", v3.name)
-      self.assertEquals(variable_scope.VariableAggregation.MEAN, v3.aggregation)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testWithGetVariableAndVariableScope(self):
-    self._skip_eager_if_gpus_less_than(1)
+      self.assertEqual("common/var3:0", v3.name)
+      self.assertEqual(variable_scope.VariableAggregation.MEAN, v3.aggregation)
 
+  def testWithGetVariableAndVariableScope(self, distribution):
     def model_fn():
       v0 = variable_scope.get_variable("var0", [1])
       with variable_scope.variable_scope("common"):
@@ -382,33 +380,28 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
       return v0, v1, v2, v3
 
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+    with distribution.scope():
       with variable_scope.variable_scope("main"):
         v = variable_scope.get_variable("var-main0", [1])
-        self.assertEquals("main/var-main0:0", v.name)
+        self.assertEqual("main/var-main0:0", v.name)
 
-        result = dist.call_for_each_replica(model_fn)
-        self.assertEquals(4, len(result))
+        result = distribution.call_for_each_replica(model_fn)
+        self.assertEqual(4, len(result))
         v0, v1, v2, v3 = result
         self.assertIsInstance(v0, values.MirroredVariable)
-        self.assertEquals("main/var0:0", v0.name)
+        self.assertEqual("main/var0:0", v0.name)
         self.assertIsInstance(v1, values.MirroredVariable)
-        self.assertEquals("main/common/var1:0", v1.name)
+        self.assertEqual("main/common/var1:0", v1.name)
         self.assertIsInstance(v2, values.ReplicaLocalVariable)
-        self.assertEquals("main/common/var2:0", v2.name)
-        self.assertEquals(variable_scope.VariableAggregation.SUM,
-                          v2.aggregation)
+        self.assertEqual("main/common/var2:0", v2.name)
+        self.assertEqual(variable_scope.VariableAggregation.SUM,
+                         v2.aggregation)
         self.assertIsInstance(v3, values.MirroredVariable)
-        self.assertEquals("main/common/var3:0", v3.name)
-        self.assertEquals(variable_scope.VariableAggregation.MEAN,
-                          v3.aggregation)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testOnlyFirstReplicaUpdatesVariables(self):
-    self._skip_eager_if_gpus_less_than(1)
+        self.assertEqual("main/common/var3:0", v3.name)
+        self.assertEqual(variable_scope.VariableAggregation.MEAN,
+                         v3.aggregation)
 
+  def testOnlyFirstReplicaUpdatesVariables(self, distribution):
     def create_fn():
       aggregation = variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
       v0 = variable_scope.variable(
@@ -424,17 +417,16 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return v0, v1
 
     devices = ["/device:GPU:0", "/device:CPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
-      v0, v1 = dist.call_for_each_replica(create_fn)
+    with distribution.scope():
+      v0, v1 = distribution.call_for_each_replica(create_fn)
       self.evaluate(v0.initializer)
       self.assertEqual(2.0, self.evaluate(v0.get(devices[0])))
       self.assertEqual(2.0, self.evaluate(v0.get(devices[1])))
-      self.assertEqual(2.0, self.evaluate(dist.read_var(v0)))
+      self.assertEqual(2.0, self.evaluate(distribution.read_var(v0)))
       self.evaluate(v1.initializer)
       self.assertEqual(3.0, self.evaluate(v1.get(devices[0])))
       self.assertEqual(3.0, self.evaluate(v1.get(devices[1])))
-      self.assertEqual(3.0, self.evaluate(dist.read_var(v1)))
+      self.assertEqual(3.0, self.evaluate(distribution.read_var(v1)))
 
       def replica_id_plus_one():
         return math_ops.cast(_replica_id() + 1, dtype=dtypes.float32)
@@ -445,24 +437,24 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         update1 = v1.assign_add(7.0 * replica_id_plus_one())
         return update0, update1
 
-      update0a, update1a = dist.call_for_each_replica(update_member_fn)
+      update0a, update1a = distribution.call_for_each_replica(update_member_fn)
 
       # Update "sync on read" variable.
-      self.evaluate(dist.group(update0a))
+      self.evaluate(distribution.group(update0a))
       self.assertEqual(2.0 + 5.0, self.evaluate(v0.get(devices[0])))
       # Writes are not synchronized for "sync on read" variables,
       # so device[1] can end up with a different value.
       self.assertEqual(2.0 + 2*5.0, self.evaluate(v0.get(devices[1])))
       # Always reads from device 0.
-      self.assertEqual(2.0 + 5.0, self.evaluate(dist.read_var(v0)))
+      self.assertEqual(2.0 + 5.0, self.evaluate(distribution.read_var(v0)))
 
       # Update "sync on write" variable.
-      self.evaluate(dist.group(update1a))
+      self.evaluate(distribution.group(update1a))
       self.assertEqual(3.0 + 7.0, self.evaluate(v1.get(devices[0])))
       # Writes are synchronized for v1, only the argument to assign_add on
       # device[0] is used.
       self.assertEqual(3.0 + 7.0, self.evaluate(v1.get(devices[1])))
-      self.assertEqual(3.0 + 7.0, self.evaluate(dist.read_var(v1)))
+      self.assertEqual(3.0 + 7.0, self.evaluate(distribution.read_var(v1)))
 
       # Update using state_ops.assign_add global function.
       def update_state_ops_fn():
@@ -470,26 +462,25 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         update1 = state_ops.assign_add(v1, 13.0 * replica_id_plus_one())
         return update0, update1
 
-      update0b, update1b = dist.call_for_each_replica(update_state_ops_fn)
-      self.evaluate(dist.group(update0b))
+      update0b, update1b = distribution.call_for_each_replica(
+          update_state_ops_fn)
+      self.evaluate(distribution.group(update0b))
 
       # Update "sync on read" variable.
       self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(v0.get(devices[0])))
       self.assertEqual(2.0 + 2*5.0 + 2*11.0, self.evaluate(v0.get(devices[1])))
-      self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(dist.read_var(v0)))
+      self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(
+          distribution.read_var(v0)))
 
       # Update "sync on write" variable.
-      self.evaluate(dist.group(update1b))
+      self.evaluate(distribution.group(update1b))
       self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(v1.get(devices[0])))
       self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(v1.get(devices[1])))
-      self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(dist.read_var(v1)))
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testNoneSynchronizationWithGetVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+      self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(
+          distribution.read_var(v1)))
+
+  def testNoneSynchronizationWithGetVariable(self, distribution):
+    with distribution.scope():
       with self.assertRaisesRegexp(
           ValueError, "`NONE` variable synchronization mode is not "
           "supported with `Mirrored` distribution strategy. Please change "
@@ -498,12 +489,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
             "v", [1],
             synchronization=variable_scope.VariableSynchronization.NONE)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testNoneSynchronizationWithVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+  def testNoneSynchronizationWithVariable(self, distribution):
+    with distribution.scope():
       with self.assertRaisesRegexp(
           ValueError, "`NONE` variable synchronization mode is not "
           "supported with `Mirrored` distribution strategy. Please change "
@@ -513,23 +500,15 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
             name="v",
             synchronization=variable_scope.VariableSynchronization.NONE)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testInvalidSynchronizationWithVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+  def testInvalidSynchronizationWithVariable(self, distribution):
+    with distribution.scope():
       with self.assertRaisesRegexp(
           ValueError, "Invalid variable synchronization mode: Invalid for "
           "variable: v"):
         variable_scope.variable(1.0, name="v", synchronization="Invalid")
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testInvalidAggregationWithGetVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+  def testInvalidAggregationWithGetVariable(self, distribution):
+    with distribution.scope():
       with self.assertRaisesRegexp(
           ValueError, "Invalid variable aggregation mode: invalid for "
           "variable: v"):
@@ -538,12 +517,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
             synchronization=variable_scope.VariableSynchronization.ON_WRITE,
             aggregation="invalid")
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testInvalidAggregationWithVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-    with dist.scope():
+  def testInvalidAggregationWithVariable(self, distribution):
+    with distribution.scope():
       with self.assertRaisesRegexp(
           ValueError, "Invalid variable aggregation mode: invalid for "
           "variable: v"):
@@ -553,47 +528,21 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
             synchronization=variable_scope.VariableSynchronization.ON_WRITE,
             aggregation="invalid")
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testThreeDevices(self):
-    self._skip_eager_if_gpus_less_than(2)
-
-    def model_fn():
-      v = variable_scope.variable(1.0, name="foo")
-      ds_context.get_replica_context().merge_call(lambda _: _)
-      return v
-
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"])
-
-    with dist.scope():
-      result = dist.call_for_each_replica(model_fn)
-      self.assertIsInstance(result, values.MirroredVariable)
-      self.assertEquals("foo:0", result.name)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testNonMatchingVariableCreation(self):
-    self._skip_eager_if_gpus_less_than(1)
-
+  def testNonMatchingVariableCreation(self, distribution):
     def model_fn(name):
       v = variable_scope.variable(1.0, name=name)
       ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
+    with distribution.scope():
       names = values.DistributedValues({
           "/device:CPU:0": "foo",
           "/device:GPU:0": "bar"
       })
       with self.assertRaises(RuntimeError):
-        _ = dist.call_for_each_replica(model_fn, args=(names,))
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testReplicaLocalVariable(self):
-    self._skip_eager_if_gpus_less_than(1)
+        _ = distribution.call_for_each_replica(model_fn, args=(names,))
 
+  def testReplicaLocalVariable(self, distribution):
     all_v_sum = {}
     all_v_mean = {}
     components_sum = {}
@@ -623,13 +572,10 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       self.assertIsNot(v_mean, c_mean)
       return updates, v_sum, v_mean, c_sum, c_mean
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
+    with distribution.scope():
       # Create "sum" and "mean" versions of ReplicaLocalVariables.
       ret_ops, ret_v_sum, ret_v_mean, regrouped_sum, regrouped_mean = (
-          dist.call_for_each_replica(model_fn))
+          distribution.call_for_each_replica(model_fn))
       # Should see the same wrapping instance in all replicas.
       self.assertIs(all_v_sum[0], ret_v_sum)
       self.assertIs(all_v_mean[0], ret_v_mean)
@@ -644,10 +590,10 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
       # Apply updates
       self.evaluate(variables.global_variables_initializer())
-      self.evaluate([y for x in ret_ops for y in dist.unwrap(x)])
+      self.evaluate([y for x in ret_ops for y in distribution.unwrap(x)])
       expected_sum = 0.0
       expected_mean = 0.0
-      for i, d in enumerate(dist.extended.worker_devices):
+      for i, d in enumerate(distribution.extended.worker_devices):
         # Should see different values on different devices.
         v_sum_value = self.evaluate(ret_v_sum.get(d).read_value())
         v_mean_value = self.evaluate(ret_v_mean.get(d).read_value())
@@ -657,22 +603,85 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         expected = i * 6.0
         self.assertEqual(expected, v_mean_value)
         expected_mean += expected
-      expected_mean /= len(dist.extended.worker_devices)
+      expected_mean /= len(distribution.extended.worker_devices)
 
       # Without get(device), should return the value you get by
       # applying the reduction across all replicas (whether you use
       # read_var(), get(), or nothing).
-      self.assertEqual(expected_sum, self.evaluate(dist.read_var(ret_v_sum)))
-      self.assertEqual(expected_mean, self.evaluate(dist.read_var(ret_v_mean)))
+      self.assertEqual(expected_sum, self.evaluate(
+          distribution.read_var(ret_v_sum)))
+      self.assertEqual(expected_mean, self.evaluate(
+          distribution.read_var(ret_v_mean)))
       self.assertEqual(expected_sum, self.evaluate(ret_v_sum.get()))
       self.assertEqual(expected_mean, self.evaluate(ret_v_mean.get()))
       self.assertEqual(expected_sum, self.evaluate(ret_v_sum))
       self.assertEqual(expected_mean, self.evaluate(ret_v_mean))
 
+  # TODO(priyag): Update this test to work in eager mode as well.
+  def testDynamicRnnVariables(self, distribution):
+    def model_fn():
+      inputs = constant_op.constant(2 * [2 * [[0.0, 1.0, 2.0, 3.0, 4.0]]])
+      cell_fw = rnn_cell_impl.LSTMCell(300)
+      cell_bw = rnn_cell_impl.LSTMCell(300)
+      (outputs, _) = rnn.bidirectional_dynamic_rnn(
+          cell_fw,
+          cell_bw,
+          inputs,
+          dtype=dtypes.float32)
+      return outputs
+
+    with context.graph_mode(), distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
+      # Two variables are created by the RNN layer.
+      self.assertEqual(2, len(result))
+      for v in result:
+        self.assertIsInstance(v, values.DistributedValues)
+        _, v1 = distribution.unwrap(v)
+        self.assertStartsWith(v1._op.name, "replica_1/")
+
+  def testReplicaLocalVariableUpdate(self, distribution):
+    def model_fn():
+      v_sum = variable_scope.variable(
+          1.0,
+          synchronization=variable_scope.VariableSynchronization.ON_READ,
+          aggregation=variable_scope.VariableAggregation.SUM)
+      self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable))
+      return v_sum
+
+    def update(var, value):
+      return var.assign(value)
+
+    with distribution.scope():
+      ret_v_sum = distribution.call_for_each_replica(model_fn)
+
+      # Initialize variables.
+      self.evaluate(variables.global_variables_initializer())
+      # Assert that the aggregated value of the replica local vars is the sum
+      # of the individual values before running the update ops.
+      self.assertEqual(1.0, self.evaluate(ret_v_sum.get(
+          distribution.extended.worker_devices[0]).read_value()))
+      self.assertEqual(2.0, self.evaluate(ret_v_sum))
+
+      # Apply updates.
+      update_ops = distribution.update(ret_v_sum, update, 5.0, grouped=False)
+      self.evaluate(update_ops)
+      # Assert that the aggregated value of the replica local vars is the sum
+      # of the individual values after running the update ops.
+      self.assertEqual(5.0, self.evaluate(ret_v_sum.get(
+          distribution.extended.worker_devices[0]).read_value()))
+      self.assertEqual(10.0, self.evaluate(ret_v_sum))
+
+
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph"]))
+class MirroredStrategyNameScopeTest(test.TestCase):
   # NOTE(priyag): Names and name scopes are ignored in eager, hence we are not
   # testing this in eager mode.
 
-  def testNameScope(self):
+  def testNameScope(self, distribution):
     def model_fn():
       with ops.name_scope("foo"):
         a = constant_op.constant(1.0, name="a")
@@ -680,20 +689,17 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         b = constant_op.constant(1.0, name="b")
       return a, b
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with context.graph_mode(), dist.scope():
+    with context.graph_mode(), distribution.scope():
       with ops.name_scope("main"):
-        result = dist.call_for_each_replica(model_fn)
-        self.assertEquals(2, len(result))
+        result = distribution.call_for_each_replica(model_fn)
+        self.assertEqual(2, len(result))
         for v, name in zip(result, ["a", "b"]):
           self.assertIsInstance(v, values.DistributedValues)
-          v0, v1 = dist.unwrap(v)
-          self.assertEquals("main/foo/" + name + ":0", v0.name)
-          self.assertEquals("main/replica_1/foo/" + name + ":0", v1.name)
+          v0, v1 = distribution.unwrap(v)
+          self.assertEqual("main/foo/" + name + ":0", v0.name)
+          self.assertEqual("main/replica_1/foo/" + name + ":0", v1.name)
 
-  def testWithDefaultName(self):
+  def testWithDefaultName(self, distribution):
     def model_fn():
       with ops.name_scope(None, "foo"):
         a = constant_op.constant(1.0, name="a")
@@ -701,23 +707,20 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         b = constant_op.constant(2.0, name="b")
       return a, b
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with context.graph_mode(), dist.scope():
-      result = dist.call_for_each_replica(model_fn)
-      self.assertEquals(2, len(result))
+    with context.graph_mode(), distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
+      self.assertEqual(2, len(result))
       for v, name in zip(result, ["a", "b"]):
         self.assertIsInstance(v, values.DistributedValues)
-        v0, v1 = dist.unwrap(v)
-        self.assertEquals("foo/" + name + ":0", v0.name)
-        self.assertEquals("replica_1/foo/" + name + ":0", v1.name)
+        v0, v1 = distribution.unwrap(v)
+        self.assertEqual("foo/" + name + ":0", v0.name)
+        self.assertEqual("replica_1/foo/" + name + ":0", v1.name)
 
   # variable_scope.variable() respects name scopes when creating
   # variables. On the other hand variable_scope.get_variable() ignores name
   # scopes when creating variables. We test both methods of creating variables
   # to make sure that we have the same variable names in both cases.
-  def testNameScopeWithVariable(self):
+  def testNameScopeWithVariable(self, distribution):
     def in_cross_replica(_):
       c = variable_scope.variable(1.0, name="c")
       return c
@@ -728,28 +731,25 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         c = ds_context.get_replica_context().merge_call(in_cross_replica)
       return b, c
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with context.graph_mode(), dist.scope():
+    with context.graph_mode(), distribution.scope():
       with ops.name_scope("main"):
         a = variable_scope.variable(1.0, name="a")
-        result = dist.call_for_each_replica(model_fn)
+        result = distribution.call_for_each_replica(model_fn)
       result_b = result[0]
       result_c = result[1]
       self.assertIsInstance(result_b, values.DistributedValues)
       self.assertIsInstance(result_c, values.DistributedValues)
-      a0, a1 = dist.unwrap(a)
-      b0, b1 = dist.unwrap(result_b)
-      c0, c1 = dist.unwrap(result_c)
-      self.assertEquals("main/a:0", a0.name)
-      self.assertEquals("main/a/replica_1:0", a1.name)
-      self.assertEquals("main/b:0", b0.name)
-      self.assertEquals("main/b/replica_1:0", b1.name)
-      self.assertEquals("main/foo/c:0", c0.name)
-      self.assertEquals("main/foo/c/replica_1:0", c1.name)
-
-  def testNameScopeWithGetVariable(self):
+      a0, a1 = distribution.unwrap(a)
+      b0, b1 = distribution.unwrap(result_b)
+      c0, c1 = distribution.unwrap(result_c)
+      self.assertEqual("main/a:0", a0.name)
+      self.assertEqual("main/a/replica_1:0", a1.name)
+      self.assertEqual("main/b:0", b0.name)
+      self.assertEqual("main/b/replica_1:0", b1.name)
+      self.assertEqual("main/foo/c:0", c0.name)
+      self.assertEqual("main/foo/c/replica_1:0", c1.name)
+
+  def testNameScopeWithGetVariable(self, distribution):
     def in_cross_replica(_):
       c = variable_scope.get_variable("c", [1])
       return c
@@ -760,114 +760,75 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         c = ds_context.get_replica_context().merge_call(in_cross_replica)
       return b, c
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with context.graph_mode(), dist.scope():
+    with context.graph_mode(), distribution.scope():
       with ops.name_scope("main"):
         a = variable_scope.get_variable("a", [1])
-        result = dist.call_for_each_replica(model_fn)
+        result = distribution.call_for_each_replica(model_fn)
       result_b = result[0]
       result_c = result[1]
       self.assertIsInstance(result_b, values.DistributedValues)
       self.assertIsInstance(result_c, values.DistributedValues)
-      a0, a1 = dist.unwrap(a)
-      b0, b1 = dist.unwrap(result_b)
-      c0, c1 = dist.unwrap(result_c)
-      self.assertEquals("a:0", a0.name)
-      self.assertEquals("a/replica_1:0", a1.name)
-      self.assertEquals("b:0", b0.name)
-      self.assertEquals("b/replica_1:0", b1.name)
-      self.assertEquals("c:0", c0.name)
-      self.assertEquals("c/replica_1:0", c1.name)
-
-  def testDynamicRnnVariables(self):
+      a0, a1 = distribution.unwrap(a)
+      b0, b1 = distribution.unwrap(result_b)
+      c0, c1 = distribution.unwrap(result_c)
+      self.assertEqual("a:0", a0.name)
+      self.assertEqual("a/replica_1:0", a1.name)
+      self.assertEqual("b:0", b0.name)
+      self.assertEqual("b/replica_1:0", b1.name)
+      self.assertEqual("c:0", c0.name)
+      self.assertEqual("c/replica_1:0", c1.name)
+
+
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.NamedDistribution(
+            "Mirrored3Devices",
+            # pylint: disable=g-long-lambda
+            lambda: mirrored_strategy.MirroredStrategy(
+                ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]),
+            required_gpus=2),
+        combinations.NamedDistribution(
+            "CoreMirrored3Devices",
+            # pylint: disable=g-long-lambda
+            lambda: mirrored_strategy.CoreMirroredStrategy(
+                ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]),
+            required_gpus=2)],
+    mode=["graph", "eager"]))
+class MirroredThreeDeviceDistributionTest(
+    strategy_test_lib.DistributionTestBase,
+    parameterized.TestCase):
+
+  def testThreeDevices(self, distribution):
     def model_fn():
-      inputs = constant_op.constant(2 * [2 * [[0.0, 1.0, 2.0, 3.0, 4.0]]])
-      cell_fw = rnn_cell_impl.LSTMCell(300)
-      cell_bw = rnn_cell_impl.LSTMCell(300)
-      (outputs, _) = rnn.bidirectional_dynamic_rnn(
-          cell_fw,
-          cell_bw,
-          inputs,
-          dtype=dtypes.float32)
-      return outputs
-
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with context.graph_mode(), dist.scope():
-      result = dist.call_for_each_replica(model_fn)
-      # Two variables are created by the RNN layer.
-      self.assertEquals(2, len(result))
-      for v in result:
-        self.assertIsInstance(v, values.DistributedValues)
-        _, v1 = dist.unwrap(v)
-        self.assertStartsWith(v1.name, "replica_1/")
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testReplicaLocalVariableUpdate(self):
-    with context.graph_mode():
-
-      def model_fn():
-        v_sum = variable_scope.variable(
-            1.0,
-            synchronization=variable_scope.VariableSynchronization.ON_READ,
-            aggregation=variable_scope.VariableAggregation.SUM)
-        self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable))
-        return v_sum
-
-      dist = mirrored_strategy.MirroredStrategy(
-          ["/device:GPU:0", "/device:GPU:1"])
-
-      def update(var, value):
-        return var.assign(value)
-
-      with dist.scope():
-        ret_v_sum = dist.call_for_each_replica(model_fn)
-        update_ops = dist.update(ret_v_sum, update, 5.0, grouped=False)
-
-        # Initialize variables.
-        self.evaluate(variables.global_variables_initializer())
-        # Assert that the aggregated value of the replica local vars is the sum
-        # of the individual values before running the update ops.
-        self.assertEquals(1.0, self.evaluate(
-            ret_v_sum.get(dist.extended.worker_devices[0]).read_value()))
-        self.assertEquals(2.0, self.evaluate(ret_v_sum))
+      v = variable_scope.variable(1.0, name="foo")
+      ds_context.get_replica_context().merge_call(lambda _: _)
+      return v
 
-        # Apply updates.
-        self.evaluate(update_ops)
-        # Assert that the aggregated value of the replica local vars is the sum
-        # of the individual values after running the update ops.
-        self.assertEquals(5.0, self.evaluate(
-            ret_v_sum.get(dist.extended.worker_devices[0]).read_value()))
-        self.assertEquals(10.0, self.evaluate(ret_v_sum))
+    with distribution.scope():
+      result = distribution.call_for_each_replica(model_fn)
+      self.assertIsInstance(result, values.MirroredVariable)
+      self.assertEqual("foo:0", result.name)
 
 
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph", "eager"]))
 class MirroredVariableUpdateTest(test.TestCase):
   # The following tests check assign, assign_add and assign_sub on Mirrored
   # variables in replica and cross replica context.
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
-
-  def _skip_eager_if_gpus_less_than(self, num_gpus):
-    if context.num_gpus() < num_gpus and context.executing_eagerly():
-      self.skipTest("Enough GPUs not available for this test in eager mode.")
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignMirroredVarReplicaContextWithoutAggregationType(self):
+  def testAssignMirroredVarReplicaContextWithoutAggregationType(self,
+                                                                distribution):
     # Test that we always have an aggregation type set on the mirrored variable
     # if we assign to it in replica mode.
-    self._skip_eager_if_gpus_less_than(1)
     def var_fn():
       v = variable_scope.variable(1.0, name="foo")
       return v
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
 
@@ -877,23 +838,19 @@ class MirroredVariableUpdateTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError, "You must specify an aggregation method to update a "
                       "MirroredVariable in Replica Context."):
-        self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
+        self.evaluate(distribution.unwrap(
+            distribution.call_for_each_replica(model_fn)))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignMirroredVarReplicaContextWithSum(self):
+  def testAssignMirroredVarReplicaContextWithSum(self, distribution):
     # Test that we don't reduce a non-per-replica value with the "sum"
     # aggregation type.
-    self._skip_eager_if_gpus_less_than(1)
     def var_fn():
       v = variable_scope.variable(
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.SUM)
       return v
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
 
@@ -903,40 +860,31 @@ class MirroredVariableUpdateTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError, "A non-DistributedValues value 5.0 cannot be reduced "
           "with the given reduce op ReduceOp.SUM."):
-        self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
+        self.evaluate(distribution.unwrap(
+            distribution.call_for_each_replica(model_fn)))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignMirroredVarCrossDeviceContext(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignMirroredVarCrossDeviceContext(self, distribution):
     def var_fn():
       return variable_scope.variable(1.0, name="foo")
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(1.0, self.evaluate(mirrored_var))
+      self.assertEqual(1.0, self.evaluate(mirrored_var))
       mirrored_var_result = self.evaluate(mirrored_var.assign(6.0))
-      self.assertEquals(6.0, mirrored_var_result)
+      self.assertEqual(6.0, mirrored_var_result)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignMirroredVarReplicaContext(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignMirroredVarReplicaContext(self, distribution):
     def var_fn():
       return variable_scope.variable(
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(1.0, self.evaluate(mirrored_var))
+      self.assertEqual(1.0, self.evaluate(mirrored_var))
 
       def model_fn():
         value = math_ops.cast(
@@ -944,73 +892,60 @@ class MirroredVariableUpdateTest(test.TestCase):
             mirrored_var.dtype)
         return mirrored_var.assign(value)
 
-      self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
-      self.assertEquals(0.5, self.evaluate(mirrored_var))
+      self.evaluate(distribution.unwrap(
+          distribution.call_for_each_replica(model_fn)))
+      self.assertEqual(0.5, self.evaluate(mirrored_var))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignMirroredVarReplicaContextWithSingleValue(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignMirroredVarReplicaContextWithSingleValue(self, distribution):
     def var_fn():
       return variable_scope.variable(
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(1.0, self.evaluate(mirrored_var))
+      self.assertEqual(1.0, self.evaluate(mirrored_var))
 
       def model_fn():
         return mirrored_var.assign(5.0)
 
-      self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
-      self.assertEquals(5.0, self.evaluate(mirrored_var))
+      self.evaluate(distribution.unwrap(
+          distribution.call_for_each_replica(model_fn)))
+      self.assertEqual(5.0, self.evaluate(mirrored_var))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignAddMirroredVarCrossDeviceContext(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignAddMirroredVarCrossDeviceContext(self, distribution):
     def var_fn():
       return variable_scope.variable(1.0, name="foo")
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(1.0, self.evaluate(mirrored_var))
+      self.assertEqual(1.0, self.evaluate(mirrored_var))
 
       # read_value == True
       mirrored_var_result = self.evaluate(
           mirrored_var.assign_add(6.0, read_value=True))
-      self.assertEquals(7.0, mirrored_var_result)
-      self.assertEquals(7.0, self.evaluate(mirrored_var.get("/device:CPU:0")))
-      self.assertEquals(7.0, self.evaluate(mirrored_var.get("/device:GPU:0")))
+      self.assertEqual(7.0, mirrored_var_result)
+      self.assertEqual(7.0, self.evaluate(mirrored_var.get("/device:CPU:0")))
+      self.assertEqual(7.0, self.evaluate(mirrored_var.get("/device:GPU:0")))
 
       # read_value == False
       self.evaluate(mirrored_var.assign_add(2.0, read_value=False))
-      self.assertEquals(9.0, self.evaluate(mirrored_var.get("/device:CPU:0")))
-      self.assertEquals(9.0, self.evaluate(mirrored_var.get("/device:GPU:0")))
+      self.assertEqual(9.0, self.evaluate(mirrored_var.get("/device:CPU:0")))
+      self.assertEqual(9.0, self.evaluate(mirrored_var.get("/device:GPU:0")))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignAddMirroredVarReplicaContext(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignAddMirroredVarReplicaContext(self, distribution):
     def var_fn():
       return variable_scope.variable(
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(1.0, self.evaluate(mirrored_var))
+      self.assertEqual(1.0, self.evaluate(mirrored_var))
 
       def model_fn():
         value = math_ops.cast(
@@ -1018,65 +953,52 @@ class MirroredVariableUpdateTest(test.TestCase):
             mirrored_var.dtype)
         return mirrored_var.assign_add(value)
 
-      self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
-      self.assertEquals(1.5, self.evaluate(mirrored_var))
+      self.evaluate(distribution.unwrap(
+          distribution.call_for_each_replica(model_fn)))
+      self.assertEqual(1.5, self.evaluate(mirrored_var))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignAddMirroredVarReplicaContextWithSingleValue(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignAddMirroredVarReplicaContextWithSingleValue(self, distribution):
     def var_fn():
       return variable_scope.variable(
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(1.0, self.evaluate(mirrored_var))
+      self.assertEqual(1.0, self.evaluate(mirrored_var))
 
       def model_fn():
         return mirrored_var.assign_add(5.0)
 
-      self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
-      self.assertEquals(6.0, self.evaluate(mirrored_var))
+      self.evaluate(distribution.unwrap(
+          distribution.call_for_each_replica(model_fn)))
+      self.assertEqual(6.0, self.evaluate(mirrored_var))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignSubMirroredVarCrossDeviceContext(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignSubMirroredVarCrossDeviceContext(self, distribution):
     def var_fn():
       return variable_scope.variable(5.0, name="foo")
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(5.0, self.evaluate(mirrored_var))
+      self.assertEqual(5.0, self.evaluate(mirrored_var))
       mirrored_var_result = self.evaluate(mirrored_var.assign_sub(2.0))
-      self.assertEquals(3.0, mirrored_var_result)
-      self.assertEquals(3.0, self.evaluate(mirrored_var.get("/device:GPU:0")))
-      self.assertEquals(3.0, self.evaluate(mirrored_var.get("/device:CPU:0")))
+      self.assertEqual(3.0, mirrored_var_result)
+      self.assertEqual(3.0, self.evaluate(mirrored_var.get("/device:GPU:0")))
+      self.assertEqual(3.0, self.evaluate(mirrored_var.get("/device:CPU:0")))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignSubMirroredVarReplicaContext(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignSubMirroredVarReplicaContext(self, distribution):
     def var_fn():
       return variable_scope.variable(
           5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(5.0, self.evaluate(mirrored_var))
+      self.assertEqual(5.0, self.evaluate(mirrored_var))
 
       def model_fn():
         value = math_ops.cast(
@@ -1084,37 +1006,37 @@ class MirroredVariableUpdateTest(test.TestCase):
             mirrored_var.dtype)
         return mirrored_var.assign_sub(value)
 
-      self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
-      self.assertEquals(4.5, self.evaluate(mirrored_var))
+      self.evaluate(distribution.unwrap(
+          distribution.call_for_each_replica(model_fn)))
+      self.assertEqual(4.5, self.evaluate(mirrored_var))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignSubMirroredVarReplicaContextWithSingleValue(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignSubMirroredVarReplicaContextWithSingleValue(self, distribution):
     def var_fn():
       return variable_scope.variable(
           5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      mirrored_var = dist.call_for_each_replica(var_fn)
+    with distribution.scope():
+      mirrored_var = distribution.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
-      self.assertEquals(5.0, self.evaluate(mirrored_var))
+      self.assertEqual(5.0, self.evaluate(mirrored_var))
 
       def model_fn():
         return mirrored_var.assign_sub(1.0)
 
-      self.evaluate(dist.unwrap(dist.call_for_each_replica(model_fn)))
-      self.assertEquals(4.0, self.evaluate(mirrored_var))
+      self.evaluate(distribution.unwrap(
+          distribution.call_for_each_replica(model_fn)))
+      self.assertEqual(4.0, self.evaluate(mirrored_var))
 
 
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph", "eager"]))
 class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase):
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
 
-  def testAssignMirroredVarInitializer(self):
+  def testAssignMirroredVarInitializer(self, distribution):
     # This test is not eager compatible since in eager variables are initialized
     # upon construction instead of once the initialization op is run.
     with context.graph_mode():
@@ -1122,17 +1044,14 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase):
         v = variable_scope.variable(1.0, name="foo")
         return v
 
-      dist = mirrored_strategy.MirroredStrategy(
-          ["/device:GPU:0", "/device:CPU:0"])
-
-      with dist.scope():
-        mirrored_var = dist.call_for_each_replica(var_fn)
+      with distribution.scope():
+        mirrored_var = distribution.call_for_each_replica(var_fn)
         self.assertIsInstance(mirrored_var, values.MirroredVariable)
         self.assertFalse(self.evaluate(mirrored_var.is_initialized()))
         self.evaluate(mirrored_var.initializer)
         self.assertTrue(self.evaluate(mirrored_var.is_initialized()))
 
-  def testAssignReplicaLocalVarInitializer(self):
+  def testAssignReplicaLocalVarInitializer(self, distribution):
     # This test is not eager compatible since in eager variables are initialized
     # upon construction instead of once the initialization op is run.
     with context.graph_mode():
@@ -1144,11 +1063,8 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase):
         self.assertTrue(isinstance(v_sum, values.ReplicaLocalVariable))
         return v_sum
 
-      dist = mirrored_strategy.MirroredStrategy(
-          ["/device:GPU:0", "/device:CPU:0"])
-
-      with dist.scope():
-        replica_local_var = dist.call_for_each_replica(model_fn)
+      with distribution.scope():
+        replica_local_var = distribution.call_for_each_replica(model_fn)
         self.assertTrue(isinstance(replica_local_var,
                                    values.ReplicaLocalVariable))
         self.assertFalse(self.evaluate(replica_local_var.is_initialized()))
@@ -1156,17 +1072,14 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase):
         self.assertTrue(self.evaluate(replica_local_var.is_initialized()))
 
 
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph", "eager"]))
 class ReplicaLocalVariableAssignTest(test.TestCase):
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
 
-  def _skip_eager_if_gpus_less_than(self, num_gpus):
-    if context.num_gpus() < num_gpus and context.executing_eagerly():
-      self.skipTest("Not enough GPUs available for this test in eager mode.")
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignReplicaLocalVarSumAggregation(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignReplicaLocalVarSumAggregation(self, distribution):
     def model_fn():
       v_sum = variable_scope.variable(
           1.0,
@@ -1174,18 +1087,16 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
           aggregation=variable_scope.VariableAggregation.SUM)
       return v_sum
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      replica_local_var = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      replica_local_var = distribution.call_for_each_replica(model_fn)
       self.assertTrue(isinstance(replica_local_var,
                                  values.ReplicaLocalVariable))
       self.evaluate(variables.global_variables_initializer())
       # Each replica has a value of 1.0 assigned to it in replica context.
       # When we read the value using `read_var` we should see the SUM of each of
       # values on each of the replicas.
-      self.assertEqual(2.0, self.evaluate(dist.read_var(replica_local_var)))
+      self.assertEqual(2.0, self.evaluate(
+          distribution.read_var(replica_local_var)))
       # Assigning 6.0 in cross replica context will assign a value of
       # 6.0/num_replicas to each replica.
       tlv_ops = replica_local_var.assign(6.0)
@@ -1193,11 +1104,10 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
       # On reading the replica local var we should get the assigned value back.
       # The value on all the replicas are added before being returned by
       # `read_var`.
-      self.assertEqual(6.0, self.evaluate(dist.read_var(replica_local_var)))
+      self.assertEqual(6.0, self.evaluate(
+          distribution.read_var(replica_local_var)))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testAssignReplicaLocalVarMeanAggregation(self):
-    self._skip_eager_if_gpus_less_than(1)
+  def testAssignReplicaLocalVarMeanAggregation(self, distribution):
     def model_fn():
       v_sum = variable_scope.variable(
           1.0,
@@ -1205,23 +1115,22 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
           aggregation=variable_scope.VariableAggregation.MEAN)
       return v_sum
 
-    dist = mirrored_strategy.MirroredStrategy(
-        ["/device:GPU:0", "/device:CPU:0"])
-
-    with dist.scope():
-      replica_local_var = dist.call_for_each_replica(model_fn)
+    with distribution.scope():
+      replica_local_var = distribution.call_for_each_replica(model_fn)
       self.assertTrue(isinstance(replica_local_var,
                                  values.ReplicaLocalVariable))
       self.evaluate(variables.global_variables_initializer())
       # Each replica has a value of 1.0 assigned to it in replica context.
       # When we read the value using `read_var` we should see the MEAN of values
       # on all replicas which is the value assigned in replica context.
-      self.assertEqual(1.0, self.evaluate(dist.read_var(replica_local_var)))
+      self.assertEqual(1.0, self.evaluate(
+          distribution.read_var(replica_local_var)))
       tlv_ops = replica_local_var.assign(6.0)
       self.evaluate(tlv_ops)
       # On reading the replica local var we should get the MEAN of all values
       # which is equal to the value assigned.
-      self.assertEqual(6.0, self.evaluate(dist.read_var(replica_local_var)))
+      self.assertEqual(6.0, self.evaluate(
+          distribution.read_var(replica_local_var)))
 
 
 class MockModel(object):
@@ -1255,24 +1164,25 @@ class MiniModel(keras_training.Model):
     return self.fc(inputs)
 
 
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph", "eager"]))
 class MirroredStrategyDefunTest(test.TestCase):
 
-  def _skip_eager_if_gpus_less_than(self, num_gpus):
-    if context.num_gpus() < num_gpus and context.executing_eagerly():
-      self.skipTest("Not enough GPUs available for this test in eager mode.")
-
-  def _call_and_check(self, model_fn, inputs, expected_result, defuns,
-                      two_variables=False):
+  def _call_and_check(self, distribution, model_fn, inputs, expected_result,
+                      defuns, two_variables=False):
     cpu_dev = device_util.canonicalize("CPU:0")
     gpu_dev = device_util.canonicalize("GPU:0")
     devices = [cpu_dev, gpu_dev]
-    dist = mirrored_strategy.MirroredStrategy(devices)
 
-    with dist.scope():
+    with distribution.scope():
       mock_model = MockModel(two_variables)
       self.evaluate(variables.global_variables_initializer())
 
-      result = dist.call_for_each_replica(model_fn, args=[mock_model] + inputs)
+      result = distribution.call_for_each_replica(
+          model_fn, args=[mock_model] + inputs)
       for device in devices:
         device_result = values.select_device(device, result)
         device_expected_result = values.select_device(device, expected_result)
@@ -1284,17 +1194,14 @@ class MirroredStrategyDefunTest(test.TestCase):
         # call_for_each has one trace per device. To check that the expected set
         # of variables was accessed on each trace, we first retrieve each
         # device-specific graph function.
-        per_replica_graph_functions = dist.call_for_each_replica(
+        per_replica_graph_functions = distribution.call_for_each_replica(
             defun.get_concrete_function, args=[mock_model] + inputs)
         for device in devices:
           graph_function = per_replica_graph_functions.get(device=device)
           self.assertEqual(set(mock_model.variables),
                            set(graph_function.graph.variables))
 
-  @test_util.run_in_graph_and_eager_modes()
-  def testVariableInDefun(self):
-    self._skip_eager_if_gpus_less_than(1)
-
+  def testVariableInDefun(self, distribution):
     @function.defun
     def times_two(mock_model):
       return mock_model()
@@ -1302,12 +1209,9 @@ class MirroredStrategyDefunTest(test.TestCase):
     def model_fn(mock_model):
       return times_two(mock_model)
 
-    self._call_and_check(model_fn, [], 2.5, [times_two])
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testVariableInNestedDefun(self):
-    self._skip_eager_if_gpus_less_than(1)
+    self._call_and_check(distribution, model_fn, [], 2.5, [times_two])
 
+  def testVariableInNestedDefun(self, distribution):
     @function.defun
     def times_two(mock_model):
       return mock_model()
@@ -1319,12 +1223,10 @@ class MirroredStrategyDefunTest(test.TestCase):
     def model_fn(mock_model):
       return two_x_plus_one(mock_model)
 
-    self._call_and_check(model_fn, [], 3.5, [times_two, two_x_plus_one])
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testTwoVariablesInNestedDefun(self):
-    self._skip_eager_if_gpus_less_than(1)
+    self._call_and_check(distribution, model_fn, [], 3.5,
+                         [times_two, two_x_plus_one])
 
+  def testTwoVariablesInNestedDefun(self, distribution):
     @function.defun
     def fn1(mock_model):
       return mock_model()
@@ -1336,12 +1238,10 @@ class MirroredStrategyDefunTest(test.TestCase):
     def model_fn(mock_model):
       return fn2(mock_model)
 
-    self._call_and_check(model_fn, [], 5.5, [fn1, fn2], two_variables=True)
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testGradientTapeOverNestedDefuns(self):
-    self._skip_eager_if_gpus_less_than(1)
+    self._call_and_check(distribution, model_fn, [], 5.5, [fn1, fn2],
+                         two_variables=True)
 
+  def testGradientTapeOverNestedDefuns(self, distribution):
     @function.defun
     def fn1(mock_model):
       return mock_model()
@@ -1357,13 +1257,10 @@ class MirroredStrategyDefunTest(test.TestCase):
                              [v.get() for v in mock_model.variables])
       return grads
 
-    self._call_and_check(model_fn, [], [2.0, 1.0], [fn1, fn2],
+    self._call_and_check(distribution, model_fn, [], [2.0, 1.0], [fn1, fn2],
                          two_variables=True)
 
-  @test_util.run_in_graph_and_eager_modes()
-  def testPassPerReplica(self):
-    self._skip_eager_if_gpus_less_than(1)
-
+  def testPassPerReplica(self, distribution):
     @function.defun
     def fn1(mock_model, factor):
       return mock_model(factor)
@@ -1371,18 +1268,10 @@ class MirroredStrategyDefunTest(test.TestCase):
     factors = values.PerReplica({"CPU:0": 5.0, "GPU:0": 3.0})
     expected_result = values.PerReplica({"CPU:0": 5.0 * 1.25,
                                          "GPU:0": 3.0 * 1.25})
-    self._call_and_check(fn1, [factors], expected_result, [fn1])
-
-  @test_util.run_in_graph_and_eager_modes()
-  def testTrain(self):
-    self._skip_eager_if_gpus_less_than(1)
-
-    cpu_dev = device_util.canonicalize("CPU:0")
-    gpu_dev = device_util.canonicalize("GPU:0")
-    devices = [cpu_dev, gpu_dev]
-    dist = mirrored_strategy.MirroredStrategy(devices)
+    self._call_and_check(distribution, fn1, [factors], expected_result, [fn1])
 
-    with dist.scope():
+  def testTrain(self, distribution):
+    with distribution.scope():
       mock_model = MiniModel()
       mock_model.call = function.defun(mock_model.call)
 
@@ -1392,10 +1281,11 @@ class MirroredStrategyDefunTest(test.TestCase):
 
       gradients_fn = backprop.implicit_grad(loss_fn)
       gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn)
-      grads_and_vars = dist.call_for_each_replica(gradients_fn, args=(None,))
+      grads_and_vars = distribution.call_for_each_replica(
+          gradients_fn, args=(None,))
 
       optimizer = gradient_descent.GradientDescentOptimizer(0.25)
-      update_ops = optimizer._distributed_apply(dist, grads_and_vars)  # pylint: disable=protected-access
+      update_ops = optimizer._distributed_apply(distribution, grads_and_vars)  # pylint: disable=protected-access
 
       if not context.executing_eagerly():
         self.evaluate(variables.global_variables_initializer())
@@ -1407,36 +1297,53 @@ class MirroredStrategyDefunTest(test.TestCase):
       self.assertAllEqual([0.5], updated_var_values[1])
 
 
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.NamedDistribution(
+            "Mirrored",
+            # pylint: disable=g-long-lambda
+            lambda: mirrored_strategy.CoreMirroredStrategy(
+                num_gpus=context.num_gpus()),
+            required_gpus=1),
+        combinations.NamedDistribution(
+            "CoreMirrored",
+            # pylint: disable=g-long-lambda
+            lambda: mirrored_strategy.CoreMirroredStrategy(
+                num_gpus=context.num_gpus()),
+            required_gpus=1)],
+    mode=["graph"]))
 class MultiWorkerMirroredStrategyTest(
     multi_worker_test_base.MultiWorkerTestBase,
     strategy_test_lib.DistributionTestBase):
 
-  def _get_distribution_strategy(self):
+  def _configure_distribution_strategy(self, distribution):
     cluster_spec = server_lib.ClusterSpec({
         "worker": ["/job:worker/task:0", "/job:worker/task:1"]
     })
-    strategy = mirrored_strategy.MirroredStrategy(num_gpus=context.num_gpus())
-    strategy.configure(cluster_spec=cluster_spec)
-    return strategy
+    distribution.configure(cluster_spec=cluster_spec)
 
-  def test_num_replicas_in_sync(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-
-    strategy = self._get_distribution_strategy()
+  def test_num_replicas_in_sync(self, distribution):
+    self._configure_distribution_strategy(distribution)
     # We calculate the total number of gpus across the workers(2) specified in
     # the cluster spec.
-    self.assertEqual(context.num_gpus() * 2, strategy.num_replicas_in_sync)
-
-  def testMinimizeLossGraph(self):
-    self._test_minimize_loss_graph(self._get_distribution_strategy(),
-                                   learning_rate=0.05)
-
-  def testMakeInputFnIterator(self):
-    if not GPU_TEST:
-      self.skipTest("Not GPU test")
-
-    d = self._get_distribution_strategy()
+    self.assertEqual(context.num_gpus() * 2, distribution.num_replicas_in_sync)
+
+  def testMinimizeLossGraph(self, distribution):
+    self._configure_distribution_strategy(distribution)
+    self._test_minimize_loss_graph(distribution, learning_rate=0.05)
+
+  def testDeviceScope(self, distribution):
+    """Test the device scope of multi-worker MirroredStrategy."""
+    self._configure_distribution_strategy(distribution)
+    with distribution.scope():
+      a = constant_op.constant(1.)
+      with ops.device("/cpu:0"):
+        b = constant_op.constant(1.)
+      self.assertEqual(a.device, "/job:worker/task:0")
+      self.assertEqual(b.device, "/job:worker/task:0/device:CPU:0")
+
+  def testMakeInputFnIterator(self, distribution):
+    self._configure_distribution_strategy(distribution)
     dataset_fn = lambda: dataset_ops.Dataset.range(100)
     num_gpus = context.num_gpus()
     num_workers = 2
@@ -1452,9 +1359,9 @@ class MultiWorkerMirroredStrategyTest(
           expected_num_replicas_in_sync=num_workers*num_gpus,
           expected_num_input_pipelines=num_workers,
           expected_input_pipeline_id=None)
-      iterator = d.make_input_fn_iterator(input_fn)
+      iterator = distribution.make_input_fn_iterator(input_fn)
       self._test_input_fn_iterator(
-          iterator, d.worker_devices, expected_values, sess)
+          iterator, distribution.worker_devices, expected_values, sess)
 
 
 class MultiWorkerMirroredStrategyTestWithChief(
@@ -1474,6 +1381,12 @@ class MultiWorkerMirroredStrategyTestWithChief(
     strategy.configure(cluster_spec=self._cluster_spec)
     self._test_minimize_loss_graph(strategy, learning_rate=0.05)
 
+  def testMinimizeLossGraphCoreMirroredStrategy(self):
+    strategy = mirrored_strategy.CoreMirroredStrategy(
+        num_gpus_per_worker=context.num_gpus())
+    strategy.configure(cluster_spec=self._cluster_spec)
+    self._test_minimize_loss_graph(strategy, learning_rate=0.05)
+
 
 def _replica_id():
   replica_id = ds_context.get_replica_context().replica_id_in_sync_group
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
deleted file mode 100644
index 2809ecb5d0..0000000000
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for class MirroredStrategy."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.contrib.distribute.python import strategy_test_lib
-from tensorflow.python.eager import context
-from tensorflow.python.eager import test
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.training import distribution_strategy_context as ds_context
-
-
-class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
-
-  def _get_distribution_strategy(self):
-    return mirrored_strategy.MirroredStrategy(["/device:CPU:0"])
-
-  def testMinimizeLossEager(self):
-    self._test_minimize_loss_eager(self._get_distribution_strategy())
-
-  def testMinimizeLossGraph(self):
-    self._test_minimize_loss_graph(self._get_distribution_strategy())
-
-  def testReplicaId(self):
-    self._test_replica_id(self._get_distribution_strategy())
-
-  @test_util.run_in_graph_and_eager_modes
-  def testCallAndMergeExceptions(self):
-    self._test_call_and_merge_exceptions(self._get_distribution_strategy())
-
-
-class VariableCreatorStackTest(test.TestCase):
-
-  def testCreatorStacksAreThreadLocal(self):
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dist = mirrored_strategy.MirroredStrategy(devices)
-
-    def model_fn():
-      replica_id_str = str(self.evaluate(_replica_id()))
-
-      def thread_creator_fn(next_creator, *args, **kwargs):
-        return next_creator(*args, **kwargs) + ":thread_" + replica_id_str
-
-      with variable_scope.variable_creator_scope(thread_creator_fn):
-        # Create a variable in this scope.
-        v = variable_scope.variable(1.0)
-
-        # This will pause the current thread, and execute the other thread.
-        ds_context.get_replica_context().merge_call(lambda _: _)
-      return v
-
-    def main_thread_creator(next_creator, *args, **kwargs):
-      # We are not using the underlying next_creator for test purposes.
-      del next_creator, args, kwargs
-      return "main_thread"
-
-    with context.graph_mode(), \
-        dist.scope(), \
-        variable_scope.variable_creator_scope(main_thread_creator):
-      result = dist.call_for_each_replica(model_fn)
-      result = dist.unwrap(result)
-      expected = ["main_thread:thread_0", "main_thread:thread_1"]
-      self.assertEqual(expected, result)
-
-
-def _replica_id():
-  replica_id = ds_context.get_replica_context().replica_id_in_sync_group
-  if not isinstance(replica_id, ops.Tensor):
-    replica_id = constant_op.constant(replica_id)
-  return replica_id
-
-
-class MultiWorkerMirroredStrategyTest(test.TestCase):
-
-  def testDeviceScope(self):
-    """Test the device scope of multi-worker MirroredStrategy."""
-    with context.graph_mode():
-      strategy = mirrored_strategy.MirroredStrategy(num_gpus=context.num_gpus())
-      strategy.configure(
-          cluster_spec={"worker": ["/job:worker/task:0", "/job:worker/task:1"]})
-      with strategy.scope():
-        a = constant_op.constant(1.)
-        with ops.device("/cpu:0"):
-          b = constant_op.constant(1.)
-        self.assertEqual(a.device, "/job:worker/task:0")
-        self.assertEqual(b.device, "/job:worker/task:0/device:CPU:0")
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index d94ebdc06b..de0abc6f04 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -194,7 +194,8 @@ class DistributionTestBase(test.TestCase):
       expected_devices = [False] * len(d.worker_devices)
 
       def mark_devices_fn():
-        replica_id = ds_context.get_replica_context().replica_id_in_sync_group
+        replica_id = self.evaluate(
+            ds_context.get_replica_context().replica_id_in_sync_group)
         self.assertLess(replica_id, len(d.worker_devices))
         self.assertFalse(expected_devices[replica_id])
         expected_devices[replica_id] = True
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 27077656e4..fe9f688c8a 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -22,7 +22,6 @@ import os
 from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
@@ -327,20 +326,20 @@ class RegroupAndSelectDeviceTest(test.TestCase):
 
       self.assertTrue(
           isinstance(merged_estimator_spec, model_fn_lib.EstimatorSpec))
-      self.assertEquals(model_fn_lib.ModeKeys.TRAIN, merged_estimator_spec.mode)
+      self.assertEqual(model_fn_lib.ModeKeys.TRAIN, merged_estimator_spec.mode)
       for device_id in range(3):
         d = _device_str(device_id)
-        self.assertEquals(created_estimator_specs[device_id].loss,
-                          merged_estimator_spec.loss.get(d))
-        self.assertEquals(created_estimator_specs[device_id].train_op,
-                          merged_estimator_spec.train_op.get(d))
+        self.assertEqual(created_estimator_specs[device_id].loss,
+                         merged_estimator_spec.loss.get(d))
+        self.assertEqual(created_estimator_specs[device_id].train_op,
+                         merged_estimator_spec.train_op.get(d))
         # Scaffold is populated by `EstimatorSpec.__new__`.
-        self.assertEquals(created_estimator_specs[device_id].scaffold,
-                          merged_estimator_spec.scaffold.get(d))
+        self.assertEqual(created_estimator_specs[device_id].scaffold,
+                         merged_estimator_spec.scaffold.get(d))
         # Also test that we can undo the merge using select_device()
-        self.assertEquals(created_estimator_specs[device_id],
-                          values.select_device(_device_str(device_id),
-                                               merged_estimator_spec))
+        self.assertEqual(created_estimator_specs[device_id],
+                         values.select_device(_device_str(device_id),
+                                              merged_estimator_spec))
 
 
 class PerReplicaDatasetTest(test.TestCase):
@@ -748,7 +747,7 @@ class InputIteratorMultiWorkerTest(
                           expected_values, sess)
 
 
-class MirroredVariableTest(test.TestCase):
+class MirroredVariableTest(test.TestCase, parameterized.TestCase):
 
   config = config_pb2.ConfigProto()
   config.allow_soft_placement = True
@@ -760,9 +759,9 @@ class MirroredVariableTest(test.TestCase):
 
     v, _, mirrored = _make_mirrored()
 
-    self.assertEquals(v[0].name, mirrored.name)
-    self.assertEquals(v[0].dtype, mirrored.dtype)
-    self.assertEquals(v[0].shape, mirrored.shape)
+    self.assertEqual(v[0].name, mirrored.name)
+    self.assertEqual(v[0].dtype, mirrored.dtype)
+    self.assertEqual(v[0].shape, mirrored.shape)
 
   @test_util.run_in_graph_and_eager_modes(config=config)
   def testVariableOnAnotherDevice(self):
@@ -772,9 +771,9 @@ class MirroredVariableTest(test.TestCase):
     mirrored = values.MirroredVariable(index, v,
                                        variable_scope.VariableAggregation.MEAN)
 
-    self.assertEquals(v.name, mirrored.name)
-    self.assertEquals(v.dtype, mirrored.dtype)
-    self.assertEquals(v.shape, mirrored.shape)
+    self.assertEqual(v.name, mirrored.name)
+    self.assertEqual(v.dtype, mirrored.dtype)
+    self.assertEqual(v.shape, mirrored.shape)
 
   def _assign_mirrored(self, devices, v, new):
     for d, var, n in zip(devices, v, new):
@@ -894,14 +893,13 @@ class MirroredVariableTest(test.TestCase):
     save_path = self._save_normal()
     self._restore_mirrored(save_path)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testFetchAMirroredVariable(self):
-    if context.num_gpus() < 1 or context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test or it's eager mode.")
-
-    with self.session(
-        graph=ops.Graph()) as sess, mirrored_strategy.MirroredStrategy(
-            ["/device:GPU:0"]).scope():
+  @combinations.generate(combinations.combine(
+      distribution=[
+          combinations.mirrored_strategy_with_one_gpu,
+          combinations.core_mirrored_strategy_with_one_gpu],
+      mode=["graph"]))
+  def testFetchAMirroredVariable(self, distribution):
+    with self.session(graph=ops.Graph()) as sess, distribution.scope():
       with ops.device("/device:GPU:0"):
         v = variable_scope.get_variable(
             name="v", initializer=1., use_resource=True)
@@ -927,7 +925,7 @@ def _make_replica_local(method):
   return v, replica_local
 
 
-class ReplicaLocalVariableTest(test.TestCase):
+class ReplicaLocalVariablePropertiesTest(test.TestCase):
 
   config = config_pb2.ConfigProto()
   config.allow_soft_placement = True
@@ -936,15 +934,14 @@ class ReplicaLocalVariableTest(test.TestCase):
   def testProperties(self):
     if context.num_gpus() < 1 and context.executing_eagerly():
       self.skipTest("A GPU is not available for this test in eager mode.")
-
     v, replica_local = _make_replica_local(
         variable_scope.VariableAggregation.SUM)
 
-    self.assertEquals(v[0].name, replica_local.name)
-    self.assertEquals(v[0].dtype, replica_local.dtype)
-    self.assertEquals(v[0].shape, replica_local.shape)
-    self.assertEquals(variable_scope.VariableAggregation.SUM,
-                      replica_local.aggregation)
+    self.assertEqual(v[0].name, replica_local.name)
+    self.assertEqual(v[0].dtype, replica_local.dtype)
+    self.assertEqual(v[0].shape, replica_local.shape)
+    self.assertEqual(variable_scope.VariableAggregation.SUM,
+                     replica_local.aggregation)
 
   @test_util.run_in_graph_and_eager_modes(config=config)
   def testVariableOnAnotherDevice(self):
@@ -954,11 +951,32 @@ class ReplicaLocalVariableTest(test.TestCase):
     replica_local = values.ReplicaLocalVariable(
         index, v, variable_scope.VariableAggregation.MEAN)
 
-    self.assertEquals(v.name, replica_local.name)
-    self.assertEquals(v.dtype, replica_local.dtype)
-    self.assertEquals(v.shape, replica_local.shape)
-    self.assertEquals(variable_scope.VariableAggregation.MEAN,
-                      replica_local.aggregation)
+    self.assertEqual(v.name, replica_local.name)
+    self.assertEqual(v.dtype, replica_local.dtype)
+    self.assertEqual(v.shape, replica_local.shape)
+    self.assertEqual(variable_scope.VariableAggregation.MEAN,
+                     replica_local.aggregation)
+
+  def testTensorConversion(self):
+    with context.graph_mode():
+      _, replica_local = _make_replica_local(
+          variable_scope.VariableAggregation.SUM)
+      converted = ops.internal_convert_to_tensor(replica_local, as_ref=False)
+      self.assertIsInstance(converted, ops.Tensor)
+      self.assertEqual(converted.dtype, replica_local.dtype)
+
+      converted = ops.internal_convert_to_tensor(replica_local, as_ref=True)
+      # Resources variable are converted to tensors as well when as_ref is True.
+      self.assertIsInstance(converted, ops.Tensor)
+      self.assertEqual(converted.dtype, replica_local.dtype)
+
+
+@combinations.generate(combinations.combine(
+    distribution=[
+        combinations.mirrored_strategy_with_gpu_and_cpu,
+        combinations.core_mirrored_strategy_with_gpu_and_cpu],
+    mode=["graph", "eager"]))
+class ReplicaLocalVariableTest(test.TestCase, parameterized.TestCase):
 
   def _assign_replica_local(self, devices, v, new):
     for d, var, n in zip(devices, v, new):
@@ -975,22 +993,15 @@ class ReplicaLocalVariableTest(test.TestCase):
     save_path, _ = self._save_return_saver(sess, var)
     return save_path
 
-  def _dist_scope(self):
-    return mirrored_strategy.MirroredStrategy(_devices).scope()
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveAndRestoreReplicaLocalSumOneGraph(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    with self.cached_session(config=self.config) as sess:
+  def testSaveAndRestoreReplicaLocalSumOneGraph(self, distribution):
+    with self.cached_session() as sess:
       v, replica_local = _make_replica_local(
           variable_scope.VariableAggregation.SUM)
 
       # Overwrite the initial values.
       self._assign_replica_local(_devices, v, [3., 4.])
 
-      with self._dist_scope():
+      with distribution.scope():
         # Saves the current value of v[0] + v[1], 7.
         save_path, saver = self._save_return_saver(sess, replica_local)
 
@@ -1002,19 +1013,18 @@ class ReplicaLocalVariableTest(test.TestCase):
         saver.restore(sess, save_path)
         self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]]))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveAndRestoreReplicaLocalMeanOneGraph(self):
+  def testSaveAndRestoreReplicaLocalMeanOneGraph(self, distribution):
     if context.num_gpus() < 1 and context.executing_eagerly():
       self.skipTest("A GPU is not available for this test in eager mode.")
 
-    with self.cached_session(config=self.config) as sess:
+    with self.cached_session() as sess:
       v, replica_local = _make_replica_local(
           variable_scope.VariableAggregation.MEAN)
 
       # Overwrite the initial values.
       self._assign_replica_local(_devices, v, [3., 4.])
 
-      with self._dist_scope():
+      with distribution.scope():
         # Saves the current value of (v[0] + v[1])/2, 3.5.
         save_path, saver = self._save_return_saver(sess, replica_local)
 
@@ -1025,7 +1035,7 @@ class ReplicaLocalVariableTest(test.TestCase):
         saver.restore(sess, save_path)
         self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]]))
 
-  def _save_replica_local_mean(self):
+  def _save_replica_local_mean(self, distribution):
     """Save variables with mirroring, returns save_path."""
     with self.session(graph=ops.Graph()) as sess:
       v, replica_local = _make_replica_local(
@@ -1034,7 +1044,7 @@ class ReplicaLocalVariableTest(test.TestCase):
       # Overwrite the initial values.
       self._assign_replica_local(_devices, v, [3., 4.])
 
-      with self._dist_scope():
+      with distribution.scope():
         # Saves the current value of (v[0] + v[1])/2, 3.5
         save_path = self._save(sess, replica_local)
 
@@ -1042,7 +1052,7 @@ class ReplicaLocalVariableTest(test.TestCase):
         self._assign_replica_local(_devices, v, [5., 6.])
     return save_path
 
-  def _save_replica_local_sum(self):
+  def _save_replica_local_sum(self, distribution):
     """Save variables with mirroring, returns save_path."""
     with self.session(graph=ops.Graph()) as sess:
       v, replica_local = _make_replica_local("sum")
@@ -1050,7 +1060,7 @@ class ReplicaLocalVariableTest(test.TestCase):
       # Overwrite the initial values.
       self._assign_replica_local(_devices, v, [1.5, 2.])
 
-      with self._dist_scope():
+      with distribution.scope():
         # Saves the current value of v[0] + v[1], 3.5
         save_path = self._save(sess, replica_local)
 
@@ -1088,7 +1098,7 @@ class ReplicaLocalVariableTest(test.TestCase):
       saver.restore(sess, save_path)
       self.assertEqual(3.5, self.evaluate(var))
 
-  def _restore_replica_local_mean(self, save_path):
+  def _restore_replica_local_mean(self, save_path, distribution):
     """Restore to variables with mirroring in a fresh graph."""
     with self.session(graph=ops.Graph()) as sess:
       v, replica_local = _make_replica_local(
@@ -1097,13 +1107,13 @@ class ReplicaLocalVariableTest(test.TestCase):
       # Overwrite the initial values.
       self._assign_replica_local(_devices, v, [7., 8.])
 
-      with self._dist_scope():
+      with distribution.scope():
         # Restores the saved value of 3.5 to both variables.
         saver = saver_lib.Saver(var_list=[replica_local])
         saver.restore(sess, save_path)
         self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]]))
 
-  def _restore_replica_local_sum(self, save_path):
+  def _restore_replica_local_sum(self, save_path, distribution):
     """Restore to variables with mirroring in a fresh graph."""
     with self.session(graph=ops.Graph()) as sess:
       v, replica_local = _make_replica_local(
@@ -1112,72 +1122,35 @@ class ReplicaLocalVariableTest(test.TestCase):
       # Overwrite the initial values.
       self._assign_replica_local(_devices, v, [7., 8.])
 
-      with self._dist_scope():
+      with distribution.scope():
         # Restores the saved value of 3.5 to both variables.
         saver = saver_lib.Saver(var_list=[replica_local])
         saver.restore(sess, save_path)
         self.assertEqual([1.75, 1.75], self.evaluate([v[0], v[1]]))
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveReplicaLocalRestoreReplicaLocalMean(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    save_path = self._save_replica_local_mean()
-    self._restore_replica_local_mean(save_path)
+  def testSaveReplicaLocalRestoreReplicaLocalMean(self, distribution):
+    save_path = self._save_replica_local_mean(distribution)
+    self._restore_replica_local_mean(save_path, distribution)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveReplicaLocalRestoreReplicaLocalSum(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
+  def testSaveReplicaLocalRestoreReplicaLocalSum(self, distribution):
+    save_path = self._save_replica_local_sum(distribution)
+    self._restore_replica_local_sum(save_path, distribution)
 
-    save_path = self._save_replica_local_sum()
-    self._restore_replica_local_sum(save_path)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveReplicaLocalMeanRestoreNormal(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    save_path = self._save_replica_local_mean()
+  def testSaveReplicaLocalMeanRestoreNormal(self, distribution):
+    save_path = self._save_replica_local_mean(distribution)
     self._restore_normal(save_path)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveReplicaLocalSumRestoreNormal(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    save_path = self._save_replica_local_sum()
+  def testSaveReplicaLocalSumRestoreNormal(self, distribution):
+    save_path = self._save_replica_local_sum(distribution)
     self._restore_normal(save_path)
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveNormalRestoreReplicaLocalMean(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
+  def testSaveNormalRestoreReplicaLocalMean(self, distribution):
     save_path = self._save_normal()
-    self._restore_replica_local_mean(save_path)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testSaveNormalRestoreReplicaLocalSum(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
+    self._restore_replica_local_mean(save_path, distribution)
 
+  def testSaveNormalRestoreReplicaLocalSum(self, distribution):
     save_path = self._save_normal()
-    self._restore_replica_local_sum(save_path)
-
-  def testTensorConversion(self):
-    with context.graph_mode():
-      _, replica_local = _make_replica_local(
-          variable_scope.VariableAggregation.SUM)
-      converted = ops.internal_convert_to_tensor(replica_local, as_ref=False)
-      self.assertIsInstance(converted, ops.Tensor)
-      self.assertEqual(converted.dtype, replica_local.dtype)
-
-      converted = ops.internal_convert_to_tensor(replica_local, as_ref=True)
-      # Resources variable are converted to tensors as well when as_ref is True.
-      self.assertIsInstance(converted, ops.Tensor)
-      self.assertEqual(converted.dtype, replica_local.dtype)
+    self._restore_replica_local_sum(save_path, distribution)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 22b5de61e0ea458ae8a786886a5bdff4545b2f76 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 01:02:25 -0800
Subject: [PATCH 0485/1554] compat: Update forward compatibility horizon to
 2018-11-19

PiperOrigin-RevId: 222033412
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 321a059d41..43a5eb4e7f 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 18)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 19)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 90085327b25411c0f99135e711bbb54ae754cc0b Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 19 Nov 2018 01:17:13 -0800
Subject: [PATCH 0486/1554] Introduce an options class for configuring
 AlgebraicSimplifier.

We already have several knobs for configuring AlgebraicSimplifier.
With possibly more such options, it will become increasingly hard
to pass all of them around as parameters.

PiperOrigin-RevId: 222035225
---
 .../xla/service/algebraic_simplifier.cc       |  82 ++--
 .../xla/service/algebraic_simplifier.h        |  68 ++-
 .../xla/service/algebraic_simplifier_test.cc  | 425 +++++++-----------
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  17 +-
 .../xla/service/gpu/nvptx_compiler.cc         |   9 +-
 .../xla/service/layout_assignment_test.cc     |   9 +-
 6 files changed, 261 insertions(+), 349 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 89e62bd2f0..f0695131bc 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -84,7 +84,8 @@ bool TransposeIsBitcast(const HloInstruction* transpose) {
 // reshape may still be a bitcast. For example, a reshape from [28x28] to [784].
 bool ReshapeOrCopyIsBitcast(
     const HloInstruction* instr,
-    const AlgebraicSimplifier::ValidBitcastCallback& valid_bitcast_callback) {
+    const AlgebraicSimplifierOptions::ValidBitcastCallback&
+        valid_bitcast_callback) {
   CHECK(HloOpcode::kReshape == instr->opcode() ||
         HloOpcode::kCopy == instr->opcode());
 
@@ -180,21 +181,13 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   const bool changed() const { return changed_; }
 
   // Runs the visitor on a computation.
-  static bool Run(
-      HloComputation* computation, bool is_layout_sensitive,
-      AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback,
-      bool enable_dot_strength_reduction, bool enable_conv_simplification);
+  static bool Run(HloComputation* computation,
+                  const AlgebraicSimplifierOptions& options);
 
  private:
-  explicit AlgebraicSimplifierVisitor(
-      HloComputation* computation, bool is_layout_sensitive,
-      AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback,
-      bool enable_dot_strength_reduction, bool enable_conv_simplification)
-      : computation_(computation),
-        is_layout_sensitive_(is_layout_sensitive),
-        valid_bitcast_callback_(std::move(valid_bitcast_callback)),
-        enable_dot_strength_reduction_(enable_dot_strength_reduction),
-        enable_conv_simplification_(enable_conv_simplification) {}
+  explicit AlgebraicSimplifierVisitor(HloComputation* computation,
+                                      const AlgebraicSimplifierOptions& options)
+      : computation_(computation), options_(options) {}
 
   // Transforms Dots where at least one input is a vector or has a degenerate
   // dimension and converts it into a multiply and reduce. This should enable
@@ -233,10 +226,10 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
                                      HloInstruction* new_instruction);
 
   // Returns whether the shape of the output of the given instructions are the
-  // same for the purposes of simplification. If is_layout_sensitive_ is true,
-  // then this tests shape equality including layout (ShapeUtil::Equal). If
-  // is_layout_sensitive_ is false, then the tests shape compatibility
-  // (ShapeUtil::Compatible).
+  // same for the purposes of simplification. If options_.is_layout_sensitive()
+  // is true, then this tests shape equality including layout
+  // (ShapeUtil::Equal). If options_.is_layout_sensitive() is false, then the
+  // tests shape compatibility (ShapeUtil::Compatible).
   bool SameShape(const HloInstruction* lhs, const HloInstruction* rhs) const;
 
   // Returns whether it was possible to transform `root` to a clamp instruction.
@@ -325,22 +318,12 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   // traversing.
   HloComputation* computation_;
 
+  // The backend-specific options selected for the algebraic simplifier.
+  const AlgebraicSimplifierOptions& options_;
+
   // Whether algebraic simplification has occurred.
   bool changed_ = false;
 
-  // Whether layout is considered during transformation.
-  bool is_layout_sensitive_;
-
-  // Callback used to determine if a bitcast is possible.
-  AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback_;
-
-  // Disable dot strength reduction on platforms where it causes a slowdown.
-  bool enable_dot_strength_reduction_;
-
-  // Disable convolution -> dot simplification on platforms where it causes a
-  // slowdown.
-  bool enable_conv_simplification_;
-
   // Cached computation for adding two scalar F32.
   HloComputation* scalar_add_computation_ = nullptr;
 };
@@ -348,19 +331,15 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
 }  // namespace
 
 bool AlgebraicSimplifierVisitor::Run(
-    HloComputation* computation, bool is_layout_sensitive,
-    AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback,
-    bool enable_dot_strength_reduction, bool enable_conv_simplification) {
-  AlgebraicSimplifierVisitor visitor(
-      computation, is_layout_sensitive, std::move(valid_bitcast_callback),
-      enable_dot_strength_reduction, enable_conv_simplification);
+    HloComputation* computation, const AlgebraicSimplifierOptions& options) {
+  AlgebraicSimplifierVisitor visitor(computation, options);
   TF_CHECK_OK(computation->Accept(&visitor));
   return visitor.changed_;
 }
 
 bool AlgebraicSimplifierVisitor::SameShape(const HloInstruction* lhs,
                                            const HloInstruction* rhs) const {
-  if (is_layout_sensitive_) {
+  if (options_.is_layout_sensitive()) {
     return ShapeUtil::Equal(lhs->shape(), rhs->shape());
   } else {
     return ShapeUtil::Compatible(lhs->shape(), rhs->shape());
@@ -504,8 +483,8 @@ Status AlgebraicSimplifierVisitor::HandleCopy(HloInstruction* copy) {
     return Status::OK();
   }
 
-  if (is_layout_sensitive_ &&
-      ReshapeOrCopyIsBitcast(copy, valid_bitcast_callback_)) {
+  if (options_.is_layout_sensitive() &&
+      ReshapeOrCopyIsBitcast(copy, options_.valid_bitcast_callback())) {
     ReplaceWithBitcast(copy);
   }
 
@@ -1215,7 +1194,8 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) {
     return ReplaceInstruction(dot, dot_of_gather_optimized);
   }
 
-  if (enable_dot_strength_reduction_ && !is_layout_sensitive_) {
+  if (options_.enable_dot_strength_reduction() &&
+      !options_.is_layout_sensitive()) {
     TF_ASSIGN_OR_RETURN(bool did_strength_reduction,
                         HandleDotStrengthReduction(dot));
     if (did_strength_reduction) {
@@ -1910,8 +1890,8 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   }
 
   // Make this a bitcast if possible.
-  if (is_layout_sensitive_ &&
-      ReshapeOrCopyIsBitcast(reshape, valid_bitcast_callback_)) {
+  if (options_.is_layout_sensitive() &&
+      ReshapeOrCopyIsBitcast(reshape, options_.valid_bitcast_callback())) {
     ReplaceWithBitcast(reshape);
     return Status::OK();
   }
@@ -2525,7 +2505,7 @@ Status AlgebraicSimplifierVisitor::HandleTranspose(HloInstruction* transpose) {
     return ReplaceInstruction(transpose, operand);
   }
 
-  if (is_layout_sensitive_ && TransposeIsBitcast(transpose)) {
+  if (options_.is_layout_sensitive() && TransposeIsBitcast(transpose)) {
     ReplaceWithBitcast(transpose);
     return Status::OK();
   }
@@ -2674,13 +2654,13 @@ StatusOr<bool> AlgebraicSimplifierVisitor::SimplifyConvToDot(
   const ConvolutionDimensionNumbers& dnums =
       convolution->convolution_dimension_numbers();
 
-  if (!enable_conv_simplification_) {
+  if (!options_.enable_conv_simplification()) {
     return false;
   }
 
   // TODO(b/31337498): For now, we cowardly refuse to do this optimization in
   // layout-insensitive mode, for fear of adding nontrivial reshapes.
-  if (!is_layout_sensitive_) {
+  if (!options_.is_layout_sensitive()) {
     return false;
   }
 
@@ -2770,9 +2750,9 @@ StatusOr<bool> AlgebraicSimplifierVisitor::SimplifyConvToDot(
   // We cannot insert bitcasts if the layouts will not be compatible.
   // TODO(b/33178038): Consider inserting a transpose if a bitcast would be
   // invalid.
-  if (!valid_bitcast_callback_(input_shape, new_input_shape) ||
-      !valid_bitcast_callback_(filter_shape, new_filter_shape) ||
-      !valid_bitcast_callback_(dot_output_shape, convolution_shape)) {
+  if (!options_.valid_bitcast_callback()(input_shape, new_input_shape) ||
+      !options_.valid_bitcast_callback()(filter_shape, new_filter_shape) ||
+      !options_.valid_bitcast_callback()(dot_output_shape, convolution_shape)) {
     return false;
   }
 
@@ -2878,9 +2858,7 @@ StatusOr<bool> AlgebraicSimplifier::Run(HloModule* module) {
                  "AlgebraicSimplifier::Run(), before:\n" + module->ToString());
   bool changed = false;
   for (auto* comp : module->MakeNonfusionComputations()) {
-    if (AlgebraicSimplifierVisitor::Run(
-            comp, is_layout_sensitive_, valid_bitcast_callback_,
-            enable_dot_strength_reduction_, enable_conv_simplification_)) {
+    if (AlgebraicSimplifierVisitor::Run(comp, options_)) {
       changed = true;
     }
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index 9f8d0ee88b..e5c9a8424c 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -23,8 +23,7 @@ limitations under the License.
 
 namespace xla {
 
-// A pass which performs algebraic simplifications.
-class AlgebraicSimplifier : public HloModulePass {
+class AlgebraicSimplifierOptions {
  public:
   // Given shapes 'from_shape' and 'to_shape', determines if it is valid to
   // bitcast from 'from_shape' to 'to_shape' after considering platform
@@ -34,18 +33,52 @@ class AlgebraicSimplifier : public HloModulePass {
   using ValidBitcastCallback =
       std::function<bool(const Shape& from_shape, const Shape& to_shape)>;
 
+  explicit AlgebraicSimplifierOptions(
+      ValidBitcastCallback valid_bitcast_callback)
+      : valid_bitcast_callback_(std::move(valid_bitcast_callback)) {}
+  // If valid_bitcast_callback returns true, then the pass will replace reshapes
+  // and transposes with bitcasts.
+  const ValidBitcastCallback& valid_bitcast_callback() const {
+    return valid_bitcast_callback_;
+  }
+
   // If is_layout_sensitive is true, then the simplifier preserves layout during
-  // transformation. Otherwise, layout is ignored. If valid_bitcast_callback
-  // returns true, then the pass will replace reshapes and transposes with
-  // bitcasts.
-  AlgebraicSimplifier(bool is_layout_sensitive,
-                      ValidBitcastCallback valid_bitcast_callback,
-                      bool enable_dot_strength_reduction = true,
-                      bool enable_conv_simplification = true)
-      : is_layout_sensitive_(is_layout_sensitive),
-        valid_bitcast_callback_(std::move(valid_bitcast_callback)),
-        enable_dot_strength_reduction_(enable_dot_strength_reduction),
-        enable_conv_simplification_(enable_conv_simplification) {}
+  // transformation. Otherwise, layout is ignored.
+  void set_is_layout_sensitive(bool is_layout_sensitive) {
+    is_layout_sensitive_ = is_layout_sensitive;
+  }
+  bool is_layout_sensitive() const { return is_layout_sensitive_; }
+
+  // Enable dot simplification on platforms where it is profitable.
+  void set_enable_dot_strength_reduction(bool enable_dot_strength_reduction) {
+    enable_dot_strength_reduction_ = enable_dot_strength_reduction;
+  }
+  bool enable_dot_strength_reduction() const {
+    return enable_dot_strength_reduction_;
+  }
+
+  // Enable convolution simplification on platforms where it is profitable.
+  void set_enable_conv_simplification(bool enable_conv_simplification) {
+    enable_conv_simplification_ = enable_conv_simplification;
+  }
+  bool enable_conv_simplification() const {
+    return enable_conv_simplification_;
+  }
+
+ private:
+  ValidBitcastCallback valid_bitcast_callback_;
+  bool is_layout_sensitive_{false};
+  bool enable_dot_strength_reduction_{true};
+  bool enable_conv_simplification_{true};
+};
+
+// A pass which performs algebraic simplifications.
+class AlgebraicSimplifier : public HloModulePass {
+ public:
+  // If is_layout_sensitive is true, then the simplifier preserves layout during
+  // transformation. Otherwise, layout is ignored.
+  AlgebraicSimplifier(const AlgebraicSimplifierOptions& options)
+      : options_(options) {}
   ~AlgebraicSimplifier() override = default;
   absl::string_view name() const override { return "algsimp"; }
 
@@ -54,14 +87,7 @@ class AlgebraicSimplifier : public HloModulePass {
   StatusOr<bool> Run(HloModule* module) override;
 
  private:
-  bool is_layout_sensitive_;
-  ValidBitcastCallback valid_bitcast_callback_;
-
-  // Enable dot simplification on platforms where it is profitable.
-  bool enable_dot_strength_reduction_;
-
-  // Enable convolution simplification on platforms where it is profitable.
-  bool enable_conv_simplification_;
+  AlgebraicSimplifierOptions options_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index e4c4da1b0e..a5a4f760f6 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -45,15 +45,18 @@ using ::testing::ElementsAre;
 
 namespace op = xla::testing::opcode_matchers;
 
-AlgebraicSimplifier::ValidBitcastCallback bitcasting_callback() {
+AlgebraicSimplifierOptions::ValidBitcastCallback bitcasting_callback() {
   return [](const Shape&, const Shape&) { return true; };
 }
 
-AlgebraicSimplifier::ValidBitcastCallback non_bitcasting_callback() {
+AlgebraicSimplifierOptions::ValidBitcastCallback non_bitcasting_callback() {
   return [](const Shape&, const Shape&) { return false; };
 }
 
-class AlgebraicSimplifierTest : public HloTestBase {};
+class AlgebraicSimplifierTest : public HloTestBase {
+ protected:
+  AlgebraicSimplifierOptions default_options_{non_bitcasting_callback()};
+};
 
 // Test that A + 0 is simplified to A
 TEST_F(AlgebraicSimplifierTest, AddZero) {
@@ -70,8 +73,7 @@ TEST_F(AlgebraicSimplifierTest, AddZero) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAdd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -92,8 +94,7 @@ TEST_F(AlgebraicSimplifierTest, MulZero) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kMultiply);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_EQ(computation->root_instruction(), zero);
 }
@@ -115,8 +116,7 @@ TEST_F(AlgebraicSimplifierTest, SelectTrue) {
   auto computation = module->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kSelect);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   EXPECT_EQ(computation->root_instruction(), param0);
 }
@@ -138,8 +138,7 @@ TEST_F(AlgebraicSimplifierTest, SelectFalse) {
   auto computation = module->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kSelect);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   EXPECT_EQ(computation->root_instruction(), param1);
 }
@@ -159,8 +158,7 @@ TEST_F(AlgebraicSimplifierTest, SelectIdentical) {
   auto computation = module->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kSelect);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   EXPECT_EQ(computation->root_instruction(), param1);
 }
@@ -196,8 +194,7 @@ TEST_F(AlgebraicSimplifierTest, TwoReducesToOne) {
   builder.AddInstruction(HloInstruction::CreateReduce(r1f32, reduce0, zero,
                                                       dims1, add_computation));
   m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   HloInstruction* root = m->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Reduce(param, zero));
@@ -219,8 +216,7 @@ TEST_F(AlgebraicSimplifierTest, AddConstOnLHS) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAdd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Add(param0, op::Constant()));
@@ -246,8 +242,7 @@ TEST_F(AlgebraicSimplifierTest, AddReassociateMergeConstants) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAdd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Add(param0, op::Add(constant1, constant2)));
@@ -269,8 +264,7 @@ TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR0Operand) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAdd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -306,8 +300,7 @@ TEST_F(AlgebraicSimplifierTest, InlineTrivialMap) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kMap);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Add(param0, op::Broadcast(zero)));
@@ -329,8 +322,7 @@ TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR1Operand) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAdd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -345,8 +337,7 @@ TEST_F(AlgebraicSimplifierTest, ConstantToBroadcast) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_THAT(root, op::Constant());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Broadcast(op::Constant()));
@@ -362,8 +353,7 @@ TEST_F(AlgebraicSimplifierTest, ConstantNotToBroadcast) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_THAT(root, op::Constant());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_FALSE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Constant());
@@ -378,8 +368,7 @@ TEST_F(AlgebraicSimplifierTest, IotaToBroadcast) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_THAT(root, op::Constant());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Iota());
@@ -400,8 +389,7 @@ TEST_F(AlgebraicSimplifierTest, SubZero) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kSubtract);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -422,8 +410,7 @@ TEST_F(AlgebraicSimplifierTest, SubConstCanonicalization) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kSubtract);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Add(param0, op::Negate(constant)));
@@ -450,8 +437,7 @@ TEST_F(AlgebraicSimplifierTest, LhsDivOfDiv) {
   EXPECT_THAT(computation->root_instruction(),
               op::Divide(op::Divide(param0, param1), param2));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -479,8 +465,7 @@ TEST_F(AlgebraicSimplifierTest, RhsDivOfDiv) {
   EXPECT_THAT(computation->root_instruction(),
               op::Divide(param0, op::Divide(param1, param2)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -513,8 +498,7 @@ TEST_F(AlgebraicSimplifierTest, DivOfDivAndDiv) {
       computation->root_instruction(),
       op::Divide(op::Divide(param0, param1), op::Divide(param2, param3)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(
@@ -541,8 +525,7 @@ TEST_F(AlgebraicSimplifierTest, DivOfExp) {
   EXPECT_THAT(computation->root_instruction(),
               op::Divide(param0, op::Exp(param1)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -570,8 +553,7 @@ TEST_F(AlgebraicSimplifierTest, DivOfPower) {
   EXPECT_THAT(computation->root_instruction(),
               op::Divide(param0, op::Power(param1, param2)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -600,8 +582,7 @@ TEST_F(AlgebraicSimplifierTest, DivOfBroadcastingPower) {
   EXPECT_THAT(computation->root_instruction(),
               op::Divide(param0, op::Power(param1, param2)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   ASSERT_THAT(computation->root_instruction(),
@@ -623,8 +604,7 @@ TEST_F(AlgebraicSimplifierTest, DivideByConstant) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -648,8 +628,7 @@ TEST_F(AlgebraicSimplifierTest, PowerOfPower) {
                                                       inner_power, exp2));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
               op::Power(base, op::Multiply(exp1, exp2)));
@@ -673,8 +652,7 @@ TEST_F(AlgebraicSimplifierTest, PowerOfPowerComplex) {
                                                       inner_power, exp2));
 
   m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 }
 
@@ -693,8 +671,7 @@ TEST_F(AlgebraicSimplifierTest, DivOneScalar) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, div);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -715,8 +692,7 @@ TEST_F(AlgebraicSimplifierTest, DivOneArray) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, div);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -740,8 +716,7 @@ TEST_F(AlgebraicSimplifierTest, ComplexOfRealImagC) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, cplx);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -765,8 +740,7 @@ TEST_F(AlgebraicSimplifierTest, RealOfComplex) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, real);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -790,8 +764,7 @@ TEST_F(AlgebraicSimplifierTest, ImagOfComplex) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, imag);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param1);
@@ -818,8 +791,7 @@ TEST_F(AlgebraicSimplifierTest, SelectMakeTuple) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, add);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Add(param1, param2));
@@ -846,8 +818,7 @@ TEST_F(AlgebraicSimplifierTest, ExpDiv) {
   EXPECT_THAT(computation->root_instruction(),
               op::Divide(op::Exp(param0), op::Exp(param1)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -875,8 +846,7 @@ TEST_F(AlgebraicSimplifierTest, ExpMul) {
   EXPECT_THAT(computation->root_instruction(),
               op::Multiply(op::Exp(param0), op::Exp(param1)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -902,8 +872,7 @@ TEST_F(AlgebraicSimplifierTest, PowExp) {
   EXPECT_THAT(computation->root_instruction(),
               op::Power(op::Exp(param0), param1));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -929,8 +898,7 @@ TEST_F(AlgebraicSimplifierTest, LnPow) {
   EXPECT_THAT(computation->root_instruction(),
               op::Log(op::Power(param0, param1)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -953,8 +921,7 @@ TEST_F(AlgebraicSimplifierTest, LnExp) {
 
   EXPECT_THAT(computation->root_instruction(), op::Log(op::Exp(param0)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_EQ(computation->root_instruction(), param0);
@@ -983,8 +950,7 @@ TEST_F(AlgebraicSimplifierTest, LnExpDiv) {
   EXPECT_THAT(computation->root_instruction(),
               op::Log(op::Divide(op::Exp(param0), op::Exp(param1))));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Subtract(param0, param1));
@@ -1007,8 +973,7 @@ TEST_F(AlgebraicSimplifierTest, Pow0Scalar) {
 
   EXPECT_THAT(computation->root_instruction(), op::Power(param0, zero));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
@@ -1032,8 +997,7 @@ TEST_F(AlgebraicSimplifierTest, Pow0Vector) {
 
   EXPECT_THAT(computation->root_instruction(), op::Power(param0, zero));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
@@ -1061,8 +1025,7 @@ TEST_F(AlgebraicSimplifierTest, Pow1) {
 
   EXPECT_THAT(computation->root_instruction(), op::Power(param0, one));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_EQ(computation->root_instruction(), param0);
@@ -1084,8 +1047,7 @@ TEST_F(AlgebraicSimplifierTest, Pow2) {
 
   EXPECT_THAT(computation->root_instruction(), op::Power(param0, two));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Multiply(param0, param0));
@@ -1107,8 +1069,7 @@ TEST_F(AlgebraicSimplifierTest, PowNegative1) {
 
   EXPECT_THAT(computation->root_instruction(), op::Power(param0, negative_one));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
@@ -1153,8 +1114,7 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedConvolution) {
       ShapeUtil::MakeShape(F32, {3, 3, 3}), lhs, rhs, /*feature_group_count=*/1,
       window, dnums, DefaultPrecisionConfig(2)));
   m->AddEntryComputation(builder.Build());
-  HloPassFix<AlgebraicSimplifier> simplifier(/*is_layout_sensitive=*/false,
-                                             non_bitcasting_callback());
+  HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   EXPECT_THAT(m->entry_computation()->root_instruction(),
               op::Convolution(lhs, rhs));
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1196,8 +1156,7 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedReduceWindow) {
           HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0f))),
       window, add_computation));
   m->AddEntryComputation(builder.Build());
-  HloPassFix<AlgebraicSimplifier> simplifier(/*is_layout_sensitive=*/false,
-                                             non_bitcasting_callback());
+  HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   EXPECT_THAT(m->entry_computation()->root_instruction(),
               op::ReduceWindow(param, op::Constant()));
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1226,8 +1185,7 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedPad) {
   m->AddEntryComputation(builder.Build());
   EXPECT_THAT(m->entry_computation()->root_instruction(),
               op::Pad(param, op::Constant()));
-  HloPassFix<AlgebraicSimplifier> simplifier(/*is_layout_sensitive=*/false,
-                                             non_bitcasting_callback());
+  HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(m->entry_computation()->root_instruction(),
               op::Broadcast(op::Constant()));
@@ -1253,8 +1211,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeBroadcast) {
   EXPECT_THAT(m->entry_computation()->root_instruction(),
               op::Reshape(op::Broadcast(op::Reshape(op))));
 
-  HloPassFix<AlgebraicSimplifier> simplifier(/*is_layout_sensitive=*/false,
-                                             non_bitcasting_callback());
+  HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(m->entry_computation()->root_instruction(), op);
@@ -1273,8 +1230,7 @@ TEST_F(AlgebraicSimplifierTest, ConvertBetweenSameType) {
 
   EXPECT_THAT(computation->root_instruction(), op::Convert(input));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), input);
@@ -1294,8 +1250,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveCopy) {
 
   EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), param0);
@@ -1316,14 +1271,16 @@ TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
   auto computation = m->AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->root_instruction(), op::Copy(param));
 
-  AlgebraicSimplifier simplifier1(/*is_layout_sensitive=*/true,
-                                  non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier1(options);
   ASSERT_FALSE(simplifier1.Run(m.get()).ValueOrDie());
   // Verify that the copy is not replaced.
   EXPECT_THAT(computation->root_instruction(), op::Copy(param));
 
-  AlgebraicSimplifier simplifier2(/*is_layout_sensitive=*/true,
-                                  bitcasting_callback());
+  AlgebraicSimplifierOptions options2(bitcasting_callback());
+  options2.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier2(options2);
   ASSERT_TRUE(simplifier2.Run(m.get()).ValueOrDie());
   // Verify that the copy is replaced.
   EXPECT_THAT(computation->root_instruction(), op::Bitcast(param));
@@ -1343,8 +1300,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveUnaryConcatenate) {
 
   EXPECT_THAT(computation->root_instruction(), op::Concatenate(param0));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), param0);
@@ -1375,8 +1331,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveEmptyConcatenateOperands) {
       computation->root_instruction(),
       op::Concatenate(empty_literal, param0, param0, empty_slice, param1));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -1423,8 +1378,7 @@ TEST_F(AlgebraicSimplifierTest, SimplifyReduceOfConcat) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(
@@ -1455,8 +1409,7 @@ TEST_F(AlgebraicSimplifierTest, OnlyEmptyConcatenateOperands) {
   EXPECT_THAT(computation->root_instruction(),
               op::Concatenate(empty_literal, empty_slice));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_EQ(computation->root_instruction(), empty_literal);
@@ -1479,8 +1432,7 @@ TEST_F(AlgebraicSimplifierTest, ConcatenateOfBroadcastBecomesPad) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(), op::Pad(param0, param1));
 }
@@ -1504,8 +1456,9 @@ TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) {
 
   EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   // Copy has not been removed.
@@ -1531,8 +1484,9 @@ TEST_F(AlgebraicSimplifierTest, CopyWithSameLayout) {
 
   EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Copy has been removed.
@@ -1559,8 +1513,9 @@ TEST_F(AlgebraicSimplifierTest, NoBitcastAdded) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(param0));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   // Reshape is not replaced with a bitcast.
@@ -1588,8 +1543,8 @@ TEST_F(AlgebraicSimplifierTest, ReshapeOfTransposeOfRngToRng) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifier simplifier(
+      (AlgebraicSimplifierOptions(bitcasting_callback())));
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that that reshape(transpose(rng)) is replace by a single rng of the
@@ -1639,8 +1594,9 @@ TEST_F(AlgebraicSimplifierTest, ReshapeReplacedWithBitcast) {
               op::Tuple(transformable_reshape, dimensions_wrong_reshape,
                         layout_wrong_reshape));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   simplifier.Run(m.get()).ValueOrDie();
 
   // Verify that only the first reshape is replaced.
@@ -1667,8 +1623,8 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkReshapeDoesntAffectChangedBit) {
   builder.AddInstruction(
       HloInstruction::CreateReshape(ShapeUtil::MakeShape(F32, {4}), add));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifier simplifier(
+      (AlgebraicSimplifierOptions(bitcasting_callback())));
   m->AddEntryComputation(builder.Build());
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
@@ -1692,8 +1648,8 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkBroadcastDoesntAffectChangedBit) {
       HloInstruction::CreateBroadcast(ShapeUtil::MakeShape(F32, {2, 2, 2}), add,
                                       /*broadcast_dimensions=*/{0, 1}));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifier simplifier(
+      (AlgebraicSimplifierOptions(bitcasting_callback())));
   m->AddEntryComputation(builder.Build());
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
@@ -1717,8 +1673,9 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) {
 
   EXPECT_THAT(computation->root_instruction(), op::Transpose(param));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that the reshape is replaced.
@@ -1744,8 +1701,9 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) {
 
   EXPECT_THAT(computation->root_instruction(), op::Transpose(param));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that the reshape is replaced.
@@ -1771,8 +1729,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapesMerged) {
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Reshape(param0)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(param0));
@@ -1798,8 +1755,9 @@ TEST_F(AlgebraicSimplifierTest, CopiesMerged) {
 
   EXPECT_THAT(computation->root_instruction(), op::Copy(op::Copy(param0)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
@@ -1823,8 +1781,7 @@ TEST_F(AlgebraicSimplifierTest, TransposesMerged) {
 
   EXPECT_THAT(computation->root_instruction(), op::Transpose(transpose1));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Transpose(param0));
@@ -1848,8 +1805,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeAndBroadcastMerged) {
   EXPECT_THAT(computation->root_instruction(),
               op::Broadcast(op::Reshape(param0)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Broadcast(param0));
@@ -1871,8 +1827,7 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshapeMerged) {
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Broadcast(param0)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Broadcast(param0));
@@ -1893,8 +1848,7 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x1_3) {
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Broadcast(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -1916,8 +1870,7 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4_6x1x1x4) {
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Broadcast(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Broadcast(param));
@@ -1940,8 +1893,7 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x2x1_6x1x1x1) {
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Broadcast(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Broadcast(param));
@@ -1966,8 +1918,7 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4x2_6x8) {
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Broadcast(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
@@ -1986,8 +1937,7 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshapeMerged) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Iota());
@@ -2006,8 +1956,7 @@ TEST_F(AlgebraicSimplifierTest, IotaEffectiveScalar) {
 
   EXPECT_THAT(computation->root_instruction(), op::Iota());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   auto root = computation->root_instruction();
@@ -2029,8 +1978,7 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_1_3x2_6) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
@@ -2048,8 +1996,7 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4_6x1x1x4) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Iota());
@@ -2070,8 +2017,7 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_1_3x2x2_6x1x1x2) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Iota());
@@ -2093,8 +2039,7 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4x2_6x8) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
@@ -2122,8 +2067,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopPad) {
 
   EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), param);
@@ -2153,8 +2097,7 @@ TEST_F(AlgebraicSimplifierTest, NegativePadding) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
 
   auto has_negative_padding = [](const HloInstruction* pad) {
     for (auto& padding_dimension : pad->padding_config().dimensions()) {
@@ -2189,8 +2132,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopReshape) {
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(param));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), param);
@@ -2212,8 +2154,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSlice) {
 
   EXPECT_THAT(computation->root_instruction(), op::Slice(param));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), param);
@@ -2241,8 +2182,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfSliceToSlice) {
 
   EXPECT_THAT(computation->root_instruction(), op::Slice(op::Slice(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Slice(param));
@@ -2273,8 +2213,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeToReshapeOfSlice) {
 
   EXPECT_THAT(computation->root_instruction(), op::Slice(op::Reshape(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Slice(param)));
@@ -2298,8 +2237,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeUnchanged) {
 
   EXPECT_THAT(computation->root_instruction(), op::Slice(op::Reshape(param)));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
 
@@ -2312,8 +2250,7 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSort) {
   builder.AddInstruction(HloInstruction::CreateSort(keys_shape, 0, keys));
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(), keys);
 }
@@ -2334,8 +2271,7 @@ TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
       keys, {values0, values1}));
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
               op::Tuple(keys, values0, values1));
@@ -2356,8 +2292,7 @@ TEST_F(AlgebraicSimplifierTest, AndTrue) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAnd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -2378,8 +2313,7 @@ TEST_F(AlgebraicSimplifierTest, AndTrue2) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAnd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -2400,8 +2334,7 @@ TEST_F(AlgebraicSimplifierTest, AndFalse) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAnd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, const_false);
@@ -2422,8 +2355,7 @@ TEST_F(AlgebraicSimplifierTest, AndFalse2) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kAnd);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, const_false);
@@ -2444,8 +2376,7 @@ TEST_F(AlgebraicSimplifierTest, OrTrue) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kOr);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, const_true);
@@ -2466,8 +2397,7 @@ TEST_F(AlgebraicSimplifierTest, OrTrue2) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kOr);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, const_true);
@@ -2488,8 +2418,7 @@ TEST_F(AlgebraicSimplifierTest, OrFalse) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kOr);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -2510,8 +2439,7 @@ TEST_F(AlgebraicSimplifierTest, OrFalse2) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kOr);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_EQ(root, param0);
@@ -2641,8 +2569,7 @@ TEST_P(ConvInputPaddingTest, DoTest) {
   auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   if (testcase.expected_conv_window.empty()) {
     ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie());
   } else {
@@ -2759,8 +2686,7 @@ TEST_P(ConvFilterPaddingTest, DoIt) {
   auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   if (testcase.expected_conv_window.empty()) {
     ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie());
   } else {
@@ -2908,8 +2834,9 @@ TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) {
     auto module = CreateNewUnverifiedModule();
     auto* computation = module->AddEntryComputation(b.Build());
 
-    AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/true,
-                                   bitcasting_callback());
+    AlgebraicSimplifierOptions simplifier_options(bitcasting_callback());
+    simplifier_options.set_is_layout_sensitive(true);
+    AlgebraicSimplifier simplifier(simplifier_options);
     if (!simplifier.Run(module.get()).ValueOrDie()) {
       return "NO_CHANGE";
     }
@@ -3032,8 +2959,7 @@ TEST_F(AlgebraicSimplifierTest, ScalarBroadcastToSlice) {
   EXPECT_EQ(root, slice);
   EXPECT_TRUE(ShapeUtil::Equal(root->shape(), slice_shape));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
 
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
@@ -3071,8 +2997,7 @@ TEST_F(AlgebraicSimplifierTest, ScalarBroadcastToTransposeReshape) {
   EXPECT_EQ(root, reshape);
   EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reshape_shape));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   root = computation->root_instruction();
@@ -3138,8 +3063,7 @@ TEST_F(AlgebraicSimplifierTest, FoldPadIntoReduceWindow) {
   auto computation = module->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, reduce_window);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   // Running simplification again should not result in any further changes.
@@ -3224,8 +3148,7 @@ TEST_F(AlgebraicSimplifierTest, FoldConvertedPadIntoReduceWindow) {
   auto computation = module->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root, reduce_window);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   // Running simplification again should not result in any further changes.
@@ -3258,8 +3181,7 @@ TEST_F(AlgebraicSimplifierTest, ReversalOfTrivialDimensionsToBitcast) {
   auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
@@ -3295,8 +3217,7 @@ TEST_F(AlgebraicSimplifierTest, IteratorInvalidation) {
 
   m->AddEmbeddedComputation(std::move(dot_computation));
   m->AddEntryComputation(call_builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
 
@@ -3313,8 +3234,7 @@ TEST_F(AlgebraicSimplifierTest, ConstantTupleBecomesTupleOfConstants) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
               op::Tuple(op::Constant(), op::Constant()));
@@ -3337,8 +3257,7 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicSlice) {
       /*slice_sizes=*/{10, 100, 1000}));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(), op::Parameter());
 }
@@ -3371,8 +3290,7 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
           3, ShapeUtil::MakeShape(U32, {3}), "update_indices"))));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
               op::DynamicSlice(op::Parameter(), op::Parameter()));
@@ -3394,8 +3312,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcasts) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Broadcast(op::Constant()));
@@ -3421,8 +3338,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcasts2) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Broadcast(op::Parameter(0)));
@@ -3442,8 +3358,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcastAndIota) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Iota());
@@ -3464,8 +3379,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcastAndIota2) {
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
   EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
   EXPECT_THAT(root, op::Iota());
@@ -3486,8 +3400,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadLow) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Reshape(op::Constant()));
@@ -3507,8 +3421,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Reshape(op::Constant()));
@@ -3528,8 +3442,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidNonScalar) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
 
@@ -3547,8 +3461,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidScalar) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Parameter());
@@ -3569,8 +3483,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Parameter(1));
@@ -3591,8 +3505,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Slice(op::Parameter(2)));
@@ -3613,8 +3527,8 @@ TEST_F(AlgebraicSimplifierTest, NegateNegate) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Parameter(0));
@@ -3633,8 +3547,8 @@ TEST_F(AlgebraicSimplifierTest, NotNot) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 bitcasting_callback());
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root, op::Parameter(0));
@@ -3733,8 +3647,7 @@ TEST_P(PadReduceWindowEffectiveBroadcastTest, DoIt) {
       output_shape, pad, zero, window, add_computation));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get()));
   ASSERT_TRUE(run_successful);
 
@@ -3815,8 +3728,7 @@ TEST_P(DotStrengthReductionTest, DotStrengthReduction) {
   builder.AddInstruction(HloInstruction::CreateDot(
       dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2)));
   auto computation = module->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool changed, simplifier.Run(module.get()));
   const bool dot_should_be_transformed = m == 1 || k == 1 || n == 1;
   const bool computation_should_be_modified =
@@ -3845,7 +3757,7 @@ struct DotOfConcatTestSpec {
 };
 
 class DotOfConcatSimplificationTest
-    : public HloTestBase,
+    : public AlgebraicSimplifierTest,
       public ::testing::WithParamInterface<DotOfConcatTestSpec> {};
 
 // Test that we transform
@@ -3893,8 +3805,7 @@ TEST_P(DotOfConcatSimplificationTest, ConstantLHS) {
       dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2)));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get()));
   ASSERT_TRUE(run_successful);
 
@@ -3958,8 +3869,7 @@ TEST_P(DotOfConcatSimplificationTest, ConstantRHS) {
       dot_shape, lhs, rhs, dot_dnums, DefaultPrecisionConfig(2)));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get()));
   ASSERT_TRUE(run_successful);
   EXPECT_TRUE(
@@ -4000,8 +3910,7 @@ TEST_F(AlgebraicSimplifierTest, DynamicUpdateSliceZeroUpdate) {
   const HloComputation* const computation =
       m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(), operand);
 }
@@ -4021,7 +3930,7 @@ struct DotOfGatherTestSpec {
 };
 
 class DotOfGatherSimplificationTest
-    : public HloTestBase,
+    : public AlgebraicSimplifierTest,
       public ::testing::WithParamInterface<DotOfGatherTestSpec> {};
 
 // input: dot(DS(ctA), ctB))
@@ -4078,8 +3987,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) {
       dot_shape, ds, rhs, dot_dnums, DefaultPrecisionConfig(2)));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get()));
   ASSERT_TRUE(run_successful);
   EXPECT_TRUE(
@@ -4149,8 +4057,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) {
       dot_shape, lhs, ds, dot_dnums, DefaultPrecisionConfig(2)));
 
   auto computation = m->AddEntryComputation(builder.Build());
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
+  AlgebraicSimplifier simplifier(default_options_);
   TF_ASSERT_OK_AND_ASSIGN(bool run_successful, simplifier.Run(m.get()));
   ASSERT_TRUE(run_successful);
   EXPECT_TRUE(
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 4ce5a8a292..6fe76e18ca 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -268,10 +268,10 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
         /*rewrite_training_op=*/true,
         /*rewrite_inference_op=*/true,
         /*rewrite_grad_op=*/true);
-    pass.AddPass<AlgebraicSimplifier>(
-        /*is_layout_sensitive=*/false,
-        [](const Shape&, const Shape&) { return false; },
-        /*enable_dot_strength_reduction=*/false);
+    AlgebraicSimplifierOptions options(
+        [](const Shape&, const Shape&) { return false; });
+    options.set_enable_dot_strength_reduction(false);
+    pass.AddPass<AlgebraicSimplifier>(options);
     pass.AddPass<HloDCE>();
 
     // BatchNormExpander can create zero-sized ops, so zero-sized HLO
@@ -334,10 +334,11 @@ Status CpuCompiler::RunHloPassesAfterLayoutAssn(
     pass.AddInvariantChecker<HloVerifier>(
         /*layout_sensitive=*/true,
         /*allow_mixed_precision=*/false);
-    pass.AddPass<HloPassFix<AlgebraicSimplifier>>(
-        /*is_layout_sensitive=*/true,
-        [](const Shape&, const Shape&) { return true; },
-        /*enable_dot_strength_reduction=*/false);
+    AlgebraicSimplifierOptions options(
+        [](const Shape&, const Shape&) { return true; });
+    options.set_is_layout_sensitive(true);
+    options.set_enable_dot_strength_reduction(false);
+    pass.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
     pass.AddPass<HloDCE>();
     pass.AddPass<HloCSE>(/*is_layout_sensitive=*/true);
   }
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 48a860ba41..f9d13c8ec3 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -177,9 +177,9 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
       // elimination has to come after that pass.
       pipeline.AddPass<ZeroSizedHloElimination>();
 
-      pass.AddPass<AlgebraicSimplifier>(
-          /*is_layout_sensitive=*/false,
+      AlgebraicSimplifierOptions options(
           [](const Shape&, const Shape&) { return false; });
+      pass.AddPass<AlgebraicSimplifier>(options);
       pass.AddPass<TupleSimplifier>();
       pass.AddPass<WhileLoopConstantSinking>();
       pass.AddPass<WhileLoopSimplifier>();
@@ -248,11 +248,12 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
 
     // The LayoutAssignment pass may leave behind kCopy instructions which are
     // duplicate or NOPs, so remove them with algebraic simplification and CSE.
-    pipeline.AddPass<HloPassFix<AlgebraicSimplifier>>(
-        /*is_layout_sensitive=*/true,
+    AlgebraicSimplifierOptions options(
         /*valid_bitcast_callback=*/[](const Shape&, const Shape&) {
           return true;
         });
+    options.set_is_layout_sensitive(true);
+    pipeline.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
 
     // Choose the fastest algorithm for each conv.
     //
diff --git a/tensorflow/compiler/xla/service/layout_assignment_test.cc b/tensorflow/compiler/xla/service/layout_assignment_test.cc
index 2400b7bb7c..61d8a0a4e6 100644
--- a/tensorflow/compiler/xla/service/layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment_test.cc
@@ -328,11 +328,10 @@ TEST_F(LayoutAssignmentTest, ConflictingLayoutTuple) {
   //  %tuple.1 = Tuple(%copy) layout=({0,1})
   //  %tuple.2 = Tuple(%tuple.0, %tuple.1) layout=(({1,0}), ({0,1}))
   //
-  EXPECT_TRUE(
-      AlgebraicSimplifier(/*is_layout_sensitive=*/true,
-                          [](const Shape&, const Shape&) { return false; })
-          .Run(m.get())
-          .ValueOrDie());
+  AlgebraicSimplifierOptions options(
+      [](const Shape&, const Shape&) { return false; });
+  options.set_is_layout_sensitive(true);
+  EXPECT_TRUE(AlgebraicSimplifier(options).Run(m.get()).ValueOrDie());
   HloInstruction* root = m->entry_computation()->root_instruction();
   // Verify layout of the root and the root's operands.
   EXPECT_TRUE(ShapeUtil::Equal(result_shape, root->shape()));
-- 
GitLab


From 9d76b2efd203cf976d924b31b91ac758007fbd5b Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 19 Nov 2018 05:09:03 -0800
Subject: [PATCH 0487/1554] Replace sort with scatter when sorting
 permutations.

PiperOrigin-RevId: 222059361
---
 .../xla/service/algebraic_simplifier.cc       | 102 ++++++++++++++++++
 .../xla/service/algebraic_simplifier.h        |  13 ++-
 .../xla/service/algebraic_simplifier_test.cc  |  73 +++++++++++++
 .../xla/service/gpu/nvptx_compiler.cc         |   2 +
 4 files changed, 189 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index f0695131bc..56bf3a9f69 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -2481,6 +2481,108 @@ Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
     return ReplaceWithNewInstruction(
         sort, HloInstruction::CreateTuple(sort->operands()));
   }
+  if (!options_.enable_permutation_sort_replacement()) {
+    return Status::OK();
+  }
+  // Check if we are sorting a permutation. In that case, we know that the keys
+  // will be sorted to the identity permutation, and we can represent the
+  // changes to the 'values' parameter as a scatter.
+  if (sort->operand_count() == 2 &&
+      operand->opcode() == HloOpcode::kGetTupleElement) {
+    const HloInstruction* other_sort = operand->operand(0);
+    // Check whether the 'values' parameter is the result of another sort with
+    // the same sort dimension.
+    if (other_sort->opcode() == HloOpcode::kSort &&
+        other_sort->operand_count() >= 2 &&
+        other_sort->dimensions(0) == dimension_to_sort &&
+        other_sort->operand(operand->tuple_index())->opcode() ==
+            HloOpcode::kIota) {
+      auto* iota =
+          Cast<HloIotaInstruction>(other_sort->operand(operand->tuple_index()));
+      // The sort operand needs to be an integral iota, and the iota dimension
+      // needs to be the dimension that was sorted.
+      if (iota->iota_dimension() == dimension_to_sort &&
+          ShapeUtil::ElementIsIntegral(iota->shape())) {
+        // We use the following construction method for a Scatter that applies
+        // the permutation from 'keys' to the 'values' parameter.
+        // - Take the "keys" parameter of the second sort and reshape it to have
+        //   another "1" dimension at the end.
+        // - Concatenate it with iotas of the same extended shape with all
+        //   different iota_dimensions except the dimension_to_sort in the order
+        //   of iota_dimensions/dimension_to_sort, so e.g. with rank 3 and
+        //   dimension_to_sort = 1, we would have concatenate of (iota with
+        //   iota_dimension=0, keys, iota with iota_dimension = 2)
+        // - Use this as the indices parameter of scatter, and set updates
+        //   of the scatter to be a reshaped 'values' parameter of sort (adding
+        //   'rank' many 1 dimensions at the end).
+        int64 rank = ShapeUtil::Rank(operand->shape());
+        Shape extended_shape = operand->shape();
+        extended_shape.add_dimensions(1);
+        extended_shape.mutable_layout()->add_minor_to_major(rank);
+        auto reshaped_permutation = computation_->AddInstruction(
+            HloInstruction::CreateReshape(extended_shape, operand));
+        std::vector<HloInstruction*> concat_operands;
+        for (int64 i = 0; i < rank; ++i) {
+          if (i == dimension_to_sort) {
+            concat_operands.push_back(reshaped_permutation);
+          } else {
+            concat_operands.push_back(computation_->AddInstruction(
+                HloInstruction::CreateIota(extended_shape, i)));
+          }
+        }
+        Shape concat_shape = operand->shape();
+        concat_shape.add_dimensions(rank);
+        concat_shape.mutable_layout()->add_minor_to_major(rank);
+        auto scatter_indices =
+            rank > 1 ? computation_->AddInstruction(
+                           HloInstruction::CreateConcatenate(
+                               concat_shape, concat_operands, rank))
+                     : reshaped_permutation;
+
+        // We don't care about the operand, it will be completely overridden by
+        // the updates.
+        auto scatter_operand = computation_->AddInstruction(
+            HloInstruction::CreateIota(sort->operand(1)->shape(), 0));
+
+        // Construct the updates operand of scatter.
+        Shape update_shape = sort->operand(1)->shape();
+        for (int64 i = 0; i < rank; ++i) {
+          update_shape.add_dimensions(1);
+          update_shape.mutable_layout()->add_minor_to_major(rank + i);
+        }
+        auto scatter_updates =
+            computation_->AddInstruction(HloInstruction::CreateReshape(
+                update_shape, sort->mutable_operand(1)));
+
+        // Construct the updates computation, which simply replaces the operand
+        // values with the update values.
+        HloComputation::Builder b("update_replace_computation");
+        Shape scalar_shape = ShapeUtil::MakeShape(S32, {});
+        b.AddInstruction(
+            HloInstruction::CreateParameter(0, scalar_shape, "scalar_lhs"));
+        auto scalar_rhs = b.AddInstruction(
+            HloInstruction::CreateParameter(1, scalar_shape, "scalar_rhs"));
+        auto update_replace_computation =
+            computation_->parent()->AddEmbeddedComputation(b.Build(scalar_rhs));
+
+        ScatterDimensionNumbers dim_numbers;
+        dim_numbers.set_index_vector_dim(rank);
+        for (int64 i = 0; i < rank; ++i) {
+          dim_numbers.add_update_window_dims(rank + i);
+          dim_numbers.add_scatter_dims_to_operand_dims(i);
+        }
+        auto scatter =
+            computation_->AddInstruction(HloInstruction::CreateScatter(
+                sort->operand(1)->shape(), scatter_operand, scatter_indices,
+                scatter_updates, update_replace_computation, dim_numbers));
+        return ReplaceWithNewInstruction(
+            sort, HloInstruction::CreateTuple(
+                      {computation_->AddInstruction(HloInstruction::CreateIota(
+                           operand->shape(), dimension_to_sort)),
+                       scatter}));
+      }
+    }
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index e5c9a8424c..d2775b9faf 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -65,11 +65,22 @@ class AlgebraicSimplifierOptions {
     return enable_conv_simplification_;
   }
 
+  // If enable_permutation_sort_replacement is true, a sort op that is known to
+  // sort a permutation will be replaced with a scatter op.
+  void set_enable_permutation_sort_replacement(
+      bool enable_permutation_sort_replacement) {
+    enable_permutation_sort_replacement_ = enable_permutation_sort_replacement;
+  }
+  bool enable_permutation_sort_replacement() const {
+    return enable_permutation_sort_replacement_;
+  }
+
  private:
   ValidBitcastCallback valid_bitcast_callback_;
   bool is_layout_sensitive_{false};
   bool enable_dot_strength_reduction_{true};
   bool enable_conv_simplification_{true};
+  bool enable_permutation_sort_replacement_{false};
 };
 
 // A pass which performs algebraic simplifications.
@@ -77,7 +88,7 @@ class AlgebraicSimplifier : public HloModulePass {
  public:
   // If is_layout_sensitive is true, then the simplifier preserves layout during
   // transformation. Otherwise, layout is ignored.
-  AlgebraicSimplifier(const AlgebraicSimplifierOptions& options)
+  explicit AlgebraicSimplifier(const AlgebraicSimplifierOptions& options)
       : options_(options) {}
   ~AlgebraicSimplifier() override = default;
   absl::string_view name() const override { return "algsimp"; }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index a5a4f760f6..8b8ba2a77d 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2255,6 +2255,79 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSort) {
   EXPECT_THAT(computation->root_instruction(), keys);
 }
 
+TEST_F(AlgebraicSimplifierTest, ReplacePermutationSortWithScatter) {
+  const char* hlo_string = R"(
+    HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} iota(), iota_dimension=1
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
+      gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+      ROOT sort2 = (s32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(gte, values), dimensions={1}
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_enable_permutation_sort_replacement(true);
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root,
+              op::Tuple(op::Iota(),
+                        op::Scatter(op::Iota(),
+                                    op::Concatenate(op::Iota(), op::Reshape()),
+                                    op::Reshape())));
+}
+
+TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortIfNonIntegral) {
+  // Same as ReplacePermutationSortWithScatter except that the iota has F32
+  // type.
+  const char* hlo_string = R"(
+    HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = f32[64,8732]{1,0} iota(), iota_dimension=1
+      sort = (f32[64,8732]{1,0}, f32[64,8732]{1,0}) sort(keys, values), dimensions={1}
+      gte = f32[64,8732]{1,0} get-tuple-element(sort), index=1
+      ROOT sort2 = (f32[64,8732]{1,0}, f32[64,8732]{1,0}) sort(gte, values), dimensions={1}
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_enable_permutation_sort_replacement(true);
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
+}
+
+TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
+  // Same as ReplacePermutationSortWithScatter except that the sort dimensions
+  // don't match.
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} iota(), iota_dimension=1
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
+      gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+      ROOT sort2 = (s32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(gte, values), dimensions={0}
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  options.set_enable_permutation_sort_replacement(true);
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
+}
+
 TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   auto builder = HloComputation::Builder(TestName());
 
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index f9d13c8ec3..902e2b0502 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -179,6 +179,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
 
       AlgebraicSimplifierOptions options(
           [](const Shape&, const Shape&) { return false; });
+      options.set_enable_permutation_sort_replacement(true);
       pass.AddPass<AlgebraicSimplifier>(options);
       pass.AddPass<TupleSimplifier>();
       pass.AddPass<WhileLoopConstantSinking>();
@@ -253,6 +254,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
           return true;
         });
     options.set_is_layout_sensitive(true);
+    options.set_enable_permutation_sort_replacement(true);
     pipeline.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
 
     // Choose the fastest algorithm for each conv.
-- 
GitLab


From c698e2b0ad81218e3d6358059703defae7b30e7b Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 19 Nov 2018 09:01:22 -0800
Subject: [PATCH 0488/1554] Remove invalid/misleading test.

The test removed in this commit was invalid and only accidentally
passed because self.cached_session() forces placement of all operations
on CPU (since use_gpu defaults to False).

The implementation of Variables.initialized_value() creates a tf.cond
which is not forcefully colocated with the variable, hence running this
test with "self.cached_session(use_gpu=True)" would fail with:

AssertionError: '/device:CPU:0' != ''

This test was made invalid by
https://github.com/tensorflow/tensorflow/commit/b25d1c7d3e9f30925aa132ba62e79d281191a3dc
(see changes to tensorflow/python/ops/variables.py)

I don't think the test was actually useful, so removing it altogether.

PiperOrigin-RevId: 222084451
---
 tensorflow/python/kernel_tests/variables_test.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index d15801f31b..2bb75109b1 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -388,16 +388,6 @@ class VariablesTestCase(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllClose([1, 12], self.evaluate(var))
 
-  def testDevicePlacement(self):
-    with self.cached_session() as sess:
-      with ops.device("/cpu:0"):
-        var = variables.Variable([1, 12])
-      init_value = var.initialized_value()
-      init_op = variables.global_variables_initializer()
-      self.assertEqual(var.op.device, init_value.device)
-      self.assertEqual(var.op.device, init_op.device)
-      self.evaluate(init_op)
-
   def testColocation(self):
     with ops.device("/job:ps"):
       var = variables.VariableV1(0, name="v")
-- 
GitLab


From f96c50db51f9de3024186f93adb7faaa60536433 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 19 Nov 2018 09:20:10 -0800
Subject: [PATCH 0489/1554] Fix testing deprecation version.

PiperOrigin-RevId: 222087362
---
 tensorflow/core/api_def/api_test.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/api_def/api_test.cc b/tensorflow/core/api_def/api_test.cc
index 6f98856915..d38a8424eb 100644
--- a/tensorflow/core/api_def/api_test.cc
+++ b/tensorflow/core/api_def/api_test.cc
@@ -182,11 +182,14 @@ void TestDeprecationVersionSetCorrectly(
   for (const auto& name_and_api_def : api_defs_map) {
     const auto& name = name_and_api_def.first;
     const auto& api_def = name_and_api_def.second;
-    ASSERT_TRUE(api_def.deprecation_version() == 0 ||
-                api_def.deprecation_message().empty())
-        << "ApiDef that includes deprecation_version > 0 must also specify "
-        << "a deprecation_message. Op " << name
-        << " has deprecation_version > 0 but deprecation_message is not set.";
+    if (api_def.deprecation_version() != 0) {
+      ASSERT_TRUE(api_def.deprecation_version() > 0)
+          << "Found ApiDef with negative deprecation_version";
+      ASSERT_FALSE(api_def.deprecation_message().empty())
+          << "ApiDef that includes deprecation_version > 0 must also specify "
+          << "a deprecation_message. Op " << name
+          << " has deprecation_version > 0 but deprecation_message is not set.";
+    }
   }
 }
 }  // namespace
-- 
GitLab


From 6ee836b86245a8dc989a7e66bd39a96b2d5b26e4 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Mon, 19 Nov 2018 09:55:37 -0800
Subject: [PATCH 0490/1554] Review changes

---
 tensorflow/python/BUILD | 1 -
 tools/bazel.rc          | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4d9ed5eea4..e55b2a0e92 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -5778,7 +5778,6 @@ cuda_py_test(
         "no_oss",
         "noguitar",
         "notap",
-        "no_nccl"
     ],
 )
 
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 76da3470dc..1fdf51f53e 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -73,7 +73,6 @@ build:nohdfs --define=no_hdfs_support=true
 build:nokafka --define=no_kafka_support=true
 build:noignite --define=no_ignite_support=true
 build:nonccl --define=no_nccl_support=true
-test:nonccl --define=no_nccl_support=true --test_tag_filters=-no_nccl
 
 build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
-- 
GitLab


From eba747c55d7c02ad14d56802f0a6bda88ee2ec66 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 10:05:07 -0800
Subject: [PATCH 0491/1554] Incremental update of Eigen to commit
 https://bitbucket.org/eigen/eigen/commits/ea671884cc96a940c18f10f56c92913b3f8fefd7

PiperOrigin-RevId: 222094412
---
 tensorflow/workspace.bzl | 8 ++++----
 third_party/png.BUILD    | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index fd800cf67d..bb339f60f2 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "1e045bef75e9b17d459b60cc30b34408f3fdab300c5053d3919d1a5921f3c86a",
-        strip_prefix = "eigen-eigen-af2071407280",
+        sha256 = "8fa7ba1af23f0320be05f4658061138d6eb8dd1f320669cbf305b3a034f9d1c2",
+        strip_prefix = "eigen-eigen-ea671884cc96",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/af2071407280.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
         ],
     )
 
diff --git a/third_party/png.BUILD b/third_party/png.BUILD
index c26a289717..e82948648e 100644
--- a/third_party/png.BUILD
+++ b/third_party/png.BUILD
@@ -44,11 +44,11 @@ cc_library(
         "png.h",
         "pngconf.h",
     ],
-    includes = ["."],
     copts = select({
         ":windows": ["-DPNG_INTEL_SSE_OPT=1"],
         "//conditions:default": [],
     }),
+    includes = ["."],
     linkopts = select({
         ":windows": [],
         "//conditions:default": ["-lm"],
-- 
GitLab


From 5c60fb7e9b90d6641c2b5848773ef49956ed54e3 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 19 Nov 2018 10:18:20 -0800
Subject: [PATCH 0492/1554] Change API for debugging.assert_all_finite and
 debugging.check_numerics in TF 2.0.

PiperOrigin-RevId: 222096573
---
 .../python_api/api_def_CheckNumerics.pbtxt    |  2 ++
 tensorflow/python/ops/numerics.py             | 29 ++++++++++++++-----
 .../api/golden/v2/tensorflow.debugging.pbtxt  |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  4 +++
 4 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_CheckNumerics.pbtxt b/tensorflow/core/api_def/python_api/api_def_CheckNumerics.pbtxt
index 33110d8c9e..cf7a56ec78 100644
--- a/tensorflow/core/api_def/python_api/api_def_CheckNumerics.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_CheckNumerics.pbtxt
@@ -1,5 +1,7 @@
 op {
   graph_op_name: "CheckNumerics"
+  deprecation_version: 2
+  deprecation_message: "Use debugging.assert_all_finite instead"
   endpoint {
     name: "debugging.check_numerics"
   }
diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py
index 1a235de90c..0ab39ad0a8 100644
--- a/tensorflow/python/ops/numerics.py
+++ b/tensorflow/python/ops/numerics.py
@@ -28,9 +28,7 @@ from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export(
-    "debugging.assert_all_finite",
-    v1=["debugging.assert_all_finite", "verify_tensor_all_finite"])
+@tf_export(v1=["debugging.assert_all_finite", "verify_tensor_all_finite"])
 @deprecation.deprecated_endpoints("verify_tensor_all_finite")
 def verify_tensor_all_finite(t, msg, name=None):
   """Assert that the tensor does not contain any NaN's or Inf's.
@@ -43,11 +41,26 @@ def verify_tensor_all_finite(t, msg, name=None):
   Returns:
     Same tensor as `t`.
   """
-  with ops.name_scope(name, "VerifyFinite", [t]) as name:
-    t = ops.convert_to_tensor(t, name="t")
-    with ops.colocate_with(t):
-      verify_input = array_ops.check_numerics(t, message=msg)
-      out = control_flow_ops.with_dependencies([verify_input], t)
+  return verify_tensor_all_finite_v2(t, msg, name)
+
+
+@tf_export("debugging.assert_all_finite", v1=[])
+def verify_tensor_all_finite_v2(x, message, name=None):
+  """Assert that the tensor does not contain any NaN's or Inf's.
+
+  Args:
+    x: Tensor to check.
+    message: Message to log on failure.
+    name: A name for this operation (optional).
+
+  Returns:
+    Same tensor as `x`.
+  """
+  with ops.name_scope(name, "VerifyFinite", [x]) as name:
+    x = ops.convert_to_tensor(x, name="x")
+    with ops.colocate_with(x):
+      verify_input = array_ops.check_numerics(x, message=message)
+      out = control_flow_ops.with_dependencies([verify_input], x)
   return out
 
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
index fbc5cd2df0..314aedda90 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.debugging.pbtxt
@@ -6,7 +6,7 @@ tf_module {
   }
   member_method {
     name: "assert_all_finite"
-    argspec: "args=[\'t\', \'msg\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'x\', \'message\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "assert_equal"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index c10f4eed39..bd556c575c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -51,6 +51,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
         },
+        "tf.debugging.assert_all_finite": {
+            "t": "x",
+            "msg": "message",
+        },
         "tf.sparse.split": {
             "split_dim": "axis",
         },
-- 
GitLab


From a4bcedd676ea035eb73a2d27437a542f9683d59d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 10:21:43 -0800
Subject: [PATCH 0493/1554] Create tf.distribute namespace with the base
 classes and standard APIs for distribution strategies. Strategy
 implementations will be in a future change. Also rename DistributionStrategy
 to tf.distribute.Strategy, and other changes to match.

RELNOTES: Expose tf.distribute.Strategy as the new name for
tf.contrib.distribute.DistributionStrategy.
PiperOrigin-RevId: 222097121
---
 tensorflow/python/distribute/BUILD            |   5 +-
 tensorflow/python/distribute/reduce_util.py   |   3 +-
 .../tools/api/generator/api_init_files.bzl    |   1 +
 .../tools/api/generator/api_init_files_v1.bzl |   1 +
 .../python/tools/api/generator/doc_srcs.py    |   3 +-
 tensorflow/python/training/distribute.py      | 231 +++++++++---------
 .../training/distribution_strategy_context.py |  33 +--
 ...tensorflow.distribute.-input-context.pbtxt |  25 ++
 ...w.distribute.-input-replication-mode.pbtxt |   8 +
 .../v1/tensorflow.distribute.-reduce-op.pbtxt |  12 +
 ...nsorflow.distribute.-replica-context.pbtxt |  33 +++
 ...orflow.distribute.-strategy-extended.pbtxt |  81 ++++++
 .../v1/tensorflow.distribute.-strategy.pbtxt  | 133 ++++++++++
 .../api/golden/v1/tensorflow.distribute.pbtxt |  47 ++++
 .../tools/api/golden/v1/tensorflow.pbtxt      |   4 +
 ...tensorflow.distribute.-input-context.pbtxt |  25 ++
 ...w.distribute.-input-replication-mode.pbtxt |   8 +
 .../v2/tensorflow.distribute.-reduce-op.pbtxt |  12 +
 ...nsorflow.distribute.-replica-context.pbtxt |  33 +++
 ...orflow.distribute.-strategy-extended.pbtxt |  81 ++++++
 .../v2/tensorflow.distribute.-strategy.pbtxt  | 133 ++++++++++
 .../api/golden/v2/tensorflow.distribute.pbtxt |  47 ++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 +
 23 files changed, 837 insertions(+), 126 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt

diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 78ad8e7e15..04592ee6ed 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -216,7 +216,10 @@ py_library(
 py_library(
     name = "reduce_util",
     srcs = ["reduce_util.py"],
-    deps = [],
+    deps = [
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+    ],
 )
 
 py_library(
diff --git a/tensorflow/python/distribute/reduce_util.py b/tensorflow/python/distribute/reduce_util.py
index 857c7993df..2b2a4e9dba 100644
--- a/tensorflow/python/distribute/reduce_util.py
+++ b/tensorflow/python/distribute/reduce_util.py
@@ -21,9 +21,10 @@ from __future__ import print_function
 import enum
 
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.util.tf_export import tf_export
 
 
-# TODO(priyag): Add this to tf.distribute namespace when it exists.
+@tf_export("distribute.ReduceOp")
 class ReduceOp(enum.Enum):
   """Indicates how a set of values should be reduced.
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 2e3d1d93c4..b41a1bc8f6 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -9,6 +9,7 @@ TENSORFLOW_API_INIT_FILES = [
     "data/__init__.py",
     "data/experimental/__init__.py",
     "debugging/__init__.py",
+    "distribute/__init__.py",
     "dtypes/__init__.py",
     "errors/__init__.py",
     "experimental/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 89c817f609..0fadec00ab 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -10,6 +10,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "data/__init__.py",
     "data/experimental/__init__.py",
     "debugging/__init__.py",
+    "distribute/__init__.py",
     "distributions/__init__.py",
     "dtypes/__init__.py",
     "errors/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py
index 0524b5e35c..9e211d172e 100644
--- a/tensorflow/python/tools/api/generator/doc_srcs.py
+++ b/tensorflow/python/tools/api/generator/doc_srcs.py
@@ -35,10 +35,11 @@ DocSource.__new__.__defaults__ = (None,) * len(DocSource._fields)
 
 _TENSORFLOW_DOC_SOURCES = {
     'app': DocSource(docstring_module_name='platform.app'),
+    'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'),
     'compat': DocSource(docstring_module_name='util.compat'),
+    'distribute': DocSource(docstring_module_name='training.distribute'),
     'distributions': DocSource(
         docstring_module_name='ops.distributions.distributions'),
-    'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'),
     'errors': DocSource(docstring_module_name='framework.errors'),
     'gfile': DocSource(docstring_module_name='platform.gfile'),
     'graph_util': DocSource(docstring_module_name='framework.graph_util'),
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 7d68dd904f..646c352a71 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Class DistributionStrategy, ReplicaContext, and supporting APIs."""
+"""Library for running a computation across multiple devices."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -38,19 +38,19 @@ from tensorflow.python.platform import tf_logging
 from tensorflow.python.training import device_util
 from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
 from tensorflow.tools.docs import doc_controls
 
 
 # ------------------------------------------------------------------------------
-# Context tracking whether in a distribution.update() or .update_non_slot()
-# call.
+# Context tracking whether in a strategy.update() or .update_non_slot() call.
 
 
 _update_device = threading.local()
 
 
 def get_update_device():
-  """Get the current device if in a `DistributionStrategy.update()` call."""
+  """Get the current device if in a `tf.distribute.Strategy.update()` call."""
   try:
     return _update_device.current
   except AttributeError:
@@ -77,8 +77,9 @@ class UpdateContext(object):
 # Public utility functions.
 
 
+@tf_export("distribute.get_loss_reduction")
 def get_loss_reduction():
-  """Reduce op corresponding to the last loss reduction."""
+  """`tf.distribute.ReduceOp` corresponding to the last loss reduction."""
   loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
   if loss_reduction == losses_impl.Reduction.SUM:
     return reduce_util.ReduceOp.SUM
@@ -95,25 +96,25 @@ def _require_cross_replica_context_extended(extended):
   cross_replica = context.cross_replica_context
   if cross_replica is not None and cross_replica.extended is extended:
     return
-  distribution_strategy = extended._container_strategy()  # pylint: disable=protected-access
+  strategy = extended._container_strategy()  # pylint: disable=protected-access
   # We have an error to report, figure out the right message.
-  if context.distribution_strategy is not distribution_strategy:
-    _wrong_distribution_strategy_scope(distribution_strategy, context)
+  if context.distribution_strategy is not strategy:
+    _wrong_strategy_scope(strategy, context)
   assert cross_replica is None
   raise RuntimeError("Method requires being in cross-replica context, use "
                      "get_replica_context().merge_call()")
 
 
-def _wrong_distribution_strategy_scope(distribution_strategy, context):
+def _wrong_strategy_scope(strategy, context):
   # Figure out the right error message.
   if not distribution_strategy_context.has_distribution_strategy():
     raise RuntimeError(
-        'Need to be inside "with distribution_strategy.scope()" for %s' %
-        (distribution_strategy,))
+        'Need to be inside "with strategy.scope()" for %s' %
+        (strategy,))
   else:
     raise RuntimeError(
-        "Mixing different DistributionStrategy objects: %s is not %s" %
-        (context.distribution_strategy, distribution_strategy))
+        "Mixing different tf.distribute.Strategy objects: %s is not %s" %
+        (context.distribution_strategy, strategy))
 
 
 def require_replica_context(replica_ctx):
@@ -124,18 +125,18 @@ def require_replica_context(replica_ctx):
   if context.replica_context is None:
     raise RuntimeError("Need to be inside `call_for_each_replica()`")
   if context.distribution_strategy is replica_ctx.distribution_strategy:
-    # Two different ReplicaContexts with the same DistributionStrategy.
+    # Two different ReplicaContexts with the same tf.distribute.Strategy.
     raise RuntimeError("Mismatching ReplicaContext.")
   raise RuntimeError(
-      "Mismatching DistributionStrategy objects: %s is not %s." %
+      "Mismatching tf.distribute.Strategy objects: %s is not %s." %
       (context.distribution_strategy, replica_ctx.distribution_strategy))
 
 
-def _require_distribution_strategy_scope_strategy(distribution_strategy):
-  """Verify in a `distribution_strategy.scope()` in this thread."""
+def _require_distribution_strategy_scope_strategy(strategy):
+  """Verify in a `strategy.scope()` in this thread."""
   context = _get_per_thread_mode()
-  if context.distribution_strategy is distribution_strategy: return
-  _wrong_distribution_strategy_scope(distribution_strategy, context)
+  if context.distribution_strategy is strategy: return
+  _wrong_strategy_scope(strategy, context)
 
 
 def _require_distribution_strategy_scope_extended(extended):
@@ -143,8 +144,8 @@ def _require_distribution_strategy_scope_extended(extended):
   context = _get_per_thread_mode()
   if context.distribution_strategy.extended is extended: return
   # Report error.
-  distribution_strategy = extended._container_strategy()  # pylint: disable=protected-access
-  _wrong_distribution_strategy_scope(distribution_strategy, context)
+  strategy = extended._container_strategy()  # pylint: disable=protected-access
+  _wrong_strategy_scope(strategy, context)
 
 
 # ------------------------------------------------------------------------------
@@ -153,15 +154,18 @@ def _require_distribution_strategy_scope_extended(extended):
 
 
 class _CurrentDistributionContext(object):
-  """Context manager for setting the `DistributionStrategy` and var creator."""
+  """Context manager setting the current `tf.distribute.Strategy`.
+
+  Also: overrides the variable creator and optionally the current device.
+  """
 
   def __init__(self,
-               distribution_strategy,
+               strategy,
                var_creator_scope,
                var_scope=None,
                default_device=None):
     self._context = distribution_strategy_context._CrossReplicaThreadMode(  # pylint: disable=protected-access
-        distribution_strategy)
+        strategy)
     self._var_creator_scope = var_creator_scope
     self._var_scope = var_scope
     if default_device:
@@ -190,8 +194,8 @@ class _CurrentDistributionContext(object):
 class _SameScopeAgainContext(object):
   """Trivial context manager when you are already in `scope()`."""
 
-  def __init__(self, distribution_strategy):
-    self._distribution_strategy = distribution_strategy
+  def __init__(self, strategy):
+    self._distribution_strategy = strategy
 
   def __enter__(self):
     return self._distribution_strategy
@@ -201,6 +205,7 @@ class _SameScopeAgainContext(object):
 
 
 # TODO(yuefengz): add more replication modes.
+@tf_export("distribute.InputReplicationMode")
 class InputReplicationMode(enum.Enum):
   """Replication mode for input function."""
 
@@ -211,6 +216,7 @@ class InputReplicationMode(enum.Enum):
   PER_WORKER = "PER_WORKER"
 
 
+@tf_export("distribute.InputContext")
 class InputContext(object):
   """A class wrapping information needed by an input function.
 
@@ -278,6 +284,7 @@ class InputContext(object):
 # Base classes for all distribution strategies.
 
 
+@tf_export("distribute.Strategy")
 class DistributionStrategy(object):
   """A list of devices with a state & compute distribution policy.
 
@@ -301,14 +308,14 @@ class DistributionStrategy(object):
 
   @property
   def extended(self):
-    """`tf.contrib.distribute.DistributionStrategyExtended` with new methods."""
+    """`tf.distribute.StrategyExtended` with additional methods."""
     return self._extended
 
   def scope(self):
-    """Returns a context manager selecting this DistributionStrategy as current.
+    """Returns a context manager selecting this Strategy as current.
 
-    Inside a `with distribution_strategy.scope():` code block, this thread
-    will use a variable creator set by `distribution_strategy`, and will
+    Inside a `with strategy.scope():` code block, this thread
+    will use a variable creator set by `strategy`, and will
     enter its "cross-replica context".
 
     Returns:
@@ -330,20 +337,20 @@ class DistributionStrategy(object):
   def distribute_dataset(self, dataset_fn):
     """Return a `dataset` split across all replicas.  DEPRECATED.
 
-    DEPRECATED: Please use `make_dataset_iterator()` or
-    `make_input_fn_iterator()` instead.
+    DEPRECATED: Please use `make_dataset_iterator` or
+    `make_input_fn_iterator` instead.
 
-    Suitable for providing input to for `extended.call_for_each_replica()` by
+    Suitable for providing input to `extended.call_for_each_replica()` by
     creating an iterator:
 
     ```
     def dataset_fn():
       return tf.data.Dataset.from_tensors([[1.]]).repeat()
 
-    with distribution_strategy.scope():
-      distributed_dataset = distribution_strategy.distribute_dataset(dataset_fn)
+    with strategy.scope():
+      distributed_dataset = strategy.distribute_dataset(dataset_fn)
       iterator = distributed_dataset.make_initializable_iterator()
-      replica_results = distribution_strategy.extended.call_for_each_replica(
+      replica_results = strategy.extended.call_for_each_replica(
           replica_fn, args=(iterator.get_next(),))
     ```
 
@@ -374,8 +381,8 @@ class DistributionStrategy(object):
         replicas.
 
     Returns:
-      An `InputIterator` which returns inputs for each step of the computation.
-      User should call `initialize` on the returned iterator.
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
     """
     return self._extended._make_dataset_iterator(dataset)  # pylint: disable=protected-access
 
@@ -384,26 +391,26 @@ class DistributionStrategy(object):
                              replication_mode=InputReplicationMode.PER_WORKER):
     """Returns an iterator split across replicas created from an input function.
 
-    The `input_fn` should take an `InputContext` object where information about
-    input sharding can be accessed:
+    The `input_fn` should take an `tf.distribute.InputContext` object where
+    information about input sharding can be accessed:
 
     ```
     def input_fn(input_context):
       d = tf.data.Dataset.from_tensors([[1.]]).repeat()
       return d.shard(input_context.num_input_pipelines,
                      input_context.input_pipeline_id)
-    with distribution_strategy.scope():
-      iterator = distribution_strategy.make_input_fn_iterator(
+    with strategy.scope():
+      iterator = strategy.make_input_fn_iterator(
           input_fn)
-      replica_results = distribution_strategy.call_for_each_replica(
+      replica_results = strategy.extended.call_for_each_replica(
           replica_fn, iterator.get_next())
     ```
 
     Args:
       input_fn: A function that returns a `tf.data.Dataset`. This function is
-        expected to take an `InputContext` object.
-      replication_mode: an enum value of `InputReplicationMode`. Only
-        `PER_WORKER` is supported currently.
+        expected to take an `tf.distribute.InputContext` object.
+      replication_mode: an enum value of `tf.distribute.InputReplicationMode`.
+        Only `PER_WORKER` is supported currently.
 
     Returns:
       An iterator object that can be initialized and fetched next element.
@@ -544,7 +551,7 @@ class DistributionStrategy(object):
 
     Args:
       value: A value returned by `extended.call_for_each_replica()` or a
-        variable created in `scope()`.
+        variable created in `scope`.
 
     Returns:
       A list of values contained in `value`. If `value` represents a single
@@ -638,17 +645,19 @@ class DistributionStrategy(object):
     raise RuntimeError("Must only deepcopy DistributionStrategy.")
 
 
+@tf_export("distribute.StrategyExtended")
 class DistributionStrategyExtended(object):
   """Additional APIs for algorithms that need to be distribution-aware.
 
   The intent is that you can write an algorithm in a stylized way and
-  it will be usable with a variety of different `DistributionStrategy`
+  it will be usable with a variety of different
+  `tf.distribute.Strategy`
   implementations. Each descendant will implement a different strategy
   for distributing the algorithm across multiple devices/machines.
   Furthermore, these changes can be hidden inside the specific layers
   and other library classes that need special treatment to run in a
   distributed setting, so that most users' model definition code can
-  run unchanged. The `DistributionStrategy` API works the same way
+  run unchanged. The `tf.distribute.Strategy` API works the same way
   with eager and graph execution.
 
   First let's introduce a few high-level concepts:
@@ -696,42 +705,33 @@ class DistributionStrategyExtended(object):
 
   We have then a few approaches we want to support:
 
-  * Code written (as if) with no knowledge of class `DistributionStrategy`.
+  * Code written (as if) with no knowledge of class `tf.distribute.Strategy`.
     This code should work as before, even if some of the layers, etc.
     used by that code are written to be distribution-aware. This is done
-    by having a default `DistributionStrategy` that gives ordinary behavior,
+    by having a default `tf.distribute.Strategy` that gives ordinary behavior,
     and by default being in a single replica context.
   * Ordinary model code that you want to run using a specific
-    `DistributionStrategy`. This can be as simple as:
+    `tf.distribute.Strategy`. This can be as simple as:
 
     ```
-    with my_distribution.scope():
-      iterator = my_distribution.distribute_dataset(
-          dataset).make_one_shot_iterator()
-      replica_train_ops = my_distribution.extended.call_for_each_replica(
+    with my_strategy.scope():
+      iterator = my_strategy.make_dataset_iterator(dataset)
+      session.run(iterator.initialize())
+      replica_train_ops = my_strategy.extended.call_for_each_replica(
           replica_fn, args=(iterator.get_next(),))
-      train_op = tf.group(my_distribution.unwrap(replica_train_ops))
+      train_op = my_strategy.group(replica_train_ops)
     ```
 
     This takes an ordinary `dataset` and `replica_fn` and runs it
-    distributed using a particular `DistributionStrategy` in
-    `my_distribution`. Any variables created in `replica_fn` are created
-    using `my_distribution`'s policy, and library functions called by
+    distributed using a particular `tf.distribute.Strategy` in
+    `my_strategy`. Any variables created in `replica_fn` are created
+    using `my_strategy`'s policy, and library functions called by
     `replica_fn` can use the `get_replica_context()` API to get enhanced
     behavior in this case.
 
-    You can also create an initializable iterator instead of a one-shot
-    iterator. In that case, you will need to ensure that you initialize the
-    iterator before calling get_next.
-    ```
-    iterator = my_distribution.distribute_dataset(
-        dataset).make_initializable_iterator())
-    session.run(iterator.initializer)
-    ```
-
   * If you want to write a distributed algorithm, you may use any of
-    the `DistributionStrategy` APIs inside a
-    `with my_distribution.scope():` block of code.
+    the `tf.distribute.Strategy` APIs inside a
+    `with my_strategy.scope():` block of code.
 
   Lower-level concepts:
 
@@ -758,7 +758,7 @@ class DistributionStrategyExtended(object):
   * Replica context vs. Cross-replica context: _replica context_ is when we
     are in some function that is being called once for each replica.
     Otherwise we are in cross-replica context, which is useful for
-    calling `DistributionStrategy` methods which operate across the
+    calling `tf.distribute.Strategy` methods which operate across the
     replicas (like `reduce_to()`). By default you start in a replica context
     (the default "single replica context") and then some methods can
     switch you back and forth, as described below.
@@ -778,7 +778,7 @@ class DistributionStrategyExtended(object):
     pick a consistent set of devices to pass to both
     `colocate_vars_with()` and `update_non_slot()`.
 
-  When using a `DistributionStrategy`, we have a new type dimension
+  When using a `tf.distribute.Strategy`, we have a new type dimension
   called _locality_ that says what values are compatible with which
   APIs:
 
@@ -856,19 +856,20 @@ class DistributionStrategyExtended(object):
   input and destination.
 
   Layers should expect to be called in a replica context, and can use
-  the `get_replica_context()` function to get a `ReplicaContext` object. The
+  the `tf.distribute.get_replica_context` function to get a
+  `tf.distribute.ReplicaContext` object. The
   `ReplicaContext` object has a `merge_call()` method for entering
   cross-replica context where you can use `reduce_to()` (or
   `batch_reduce_to()`) and then optionally `update()` to update state.
 
-  You may use this API whether or not a `DistributionStrategy` is
+  You may use this API whether or not a `tf.distribute.Strategy` is
   being used, since there is a default implementation of
-  `ReplicaContext` and `DistributionStrategy`.
+  `ReplicaContext` and `tf.distribute.Strategy`.
 
-  NOTE for new `DistributionStrategy` implementations: Please put all logic
-  in a subclass of `DistributionStrategyExtended`. The only code needed for
-  the `DistributionStrategy` subclass is for instantiating your subclass of
-  `DistributionStrategyExtended` in the `__init__` method.
+  NOTE for new `tf.distribute.Strategy` implementations: Please put all logic
+  in a subclass of `tf.distribute.StrategyExtended`. The only code needed for
+  the `tf.distribute.Strategy` subclass is for instantiating your subclass of
+  `tf.distribute.StrategyExtended` in the `__init__` method.
   """
 
   def __init__(self, container_strategy):
@@ -908,7 +909,7 @@ class DistributionStrategyExtended(object):
         if kwargs.pop("partitioner", None) is not None:
           tf_logging.log_first_n(
               tf_logging.WARN, "Partitioned variables are disabled when using "
-              "current DistributionStrategy.", 1)
+              "current tf.distribute.Strategy.", 1)
       return getter(*args, **kwargs)
 
     return _CurrentDistributionContext(
@@ -932,7 +933,7 @@ class DistributionStrategyExtended(object):
     (read-only) value of any other variable.
 
     Args:
-      v: A variable allocated within the scope of this `DistributionStrategy`.
+      v: A variable allocated within the scope of this `tf.distribute.Strategy`.
 
     Returns:
       A tensor representing the value of `v`, aggregated across replicas if
@@ -953,9 +954,9 @@ class DistributionStrategyExtended(object):
     Example usage:
 
     ```
-    with distribution_strategy.scope():
+    with strategy.scope():
       var1 = tf.get_variable(...)
-      with distribution_strategy.colocate_vars_with(v1):
+      with strategy.extended.colocate_vars_with(v1):
         # var2 and var3 will be created on the same device(s) as var1
         var2 = tf.get_variable(...)
         var3 = tf.get_variable(...)
@@ -964,7 +965,7 @@ class DistributionStrategyExtended(object):
         # operates on v1 from var1, v2 from var2, and v3 from var3
 
       # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too.
-      distribution_strategy.update(v1, fn, args=(v2, v3))
+      strategy.extended.update(v1, fn, args=(v2, v3))
     ```
 
     Args:
@@ -990,7 +991,7 @@ class DistributionStrategyExtended(object):
     if not isinstance(result, dataset_ops.Dataset):
       raise ValueError(
           "dataset_fn() must return a tf.data.Dataset when using a "
-          "DistributionStrategy.")
+          "tf.distribute.Strategy.")
     return result
 
   # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
@@ -1320,7 +1321,7 @@ class DistributionStrategyExtended(object):
 
     Args:
       var_list: The list of variables being optimized, needed with the
-        default `DistributionStrategy`.
+        default `tf.distribute.Strategy`.
     """
     raise NotImplementedError("must be implemented in descendants")
 
@@ -1374,13 +1375,18 @@ class DistributionStrategyExtended(object):
 #   around their model creation and graph definition. There is no
 #   anticipated need to define descendants of _CurrentDistributionContext.
 #   It sets the current DistributionStrategy for purposes of
-#   `get_distribution_strategy()` and `has_distribution_strategy()`
+#   `get_strategy()` and `has_strategy()`
 #   and switches the thread mode to a "cross-replica context".
+@tf_export("distribute.ReplicaContext")
 class ReplicaContext(object):
-  """DistributionStrategy API inside a `call_for_each_replica()` call."""
+  """`tf.distribute.Strategy` API when in a replica context.
 
-  def __init__(self, distribution_strategy, replica_id_in_sync_group):
-    self._distribution_strategy = distribution_strategy
+  To be used inside your replicated step function, such as in a
+  `tf.distribute.StrategyExtended.call_for_each_replica` call.
+  """
+
+  def __init__(self, strategy, replica_id_in_sync_group):
+    self._distribution_strategy = strategy
     self._thread_context = distribution_strategy_context._InReplicaThreadMode(  # pylint: disable=protected-access
         self)
     self._replica_id_in_sync_group = replica_id_in_sync_group
@@ -1396,23 +1402,24 @@ class ReplicaContext(object):
 
     This allows communication and coordination when there are multiple calls
     to a model function triggered by a call to
-    `distribution.call_for_each_replica(model_fn, ...)`.
+    `strategy.extended.call_for_each_replica(model_fn, ...)`.
 
-    See `MirroredDistribution.call_for_each_replica()` for an explanation.
+    See `tf.distribute.StrategyExtended.call_for_each_replica` for an
+    explanation.
 
-    Otherwise, this is equivalent to:
+    If not inside a distributed scope, this is equivalent to:
 
     ```
-    distribution = get_distribution_strategy()
-    with cross-replica-context(distribution):
-      return merge_fn(distribution, *args, **kwargs)
+    strategy = tf.distribute.get_strategy()
+    with cross-replica-context(strategy):
+      return merge_fn(strategy, *args, **kwargs)
     ```
 
     Args:
       merge_fn: function that joins arguments from threads that are given as
-        PerReplica. It accepts `DistributionStrategy` object as the first
-        argument.
-      args: List or tuple with positional per-thread arguments for `merge_fn`
+        PerReplica. It accepts `tf.distribute.Strategy` object as
+        the first argument.
+      args: List or tuple with positional per-thread arguments for `merge_fn`.
       kwargs: Dict with keyword per-thread arguments for `merge_fn`.
 
     Returns:
@@ -1446,8 +1453,14 @@ class ReplicaContext(object):
     return self._replica_id_in_sync_group
 
   @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, use `strategy`
   def distribution_strategy(self):
-    """The current `DistributionStrategy` object."""
+    """DEPRECATED: use `self.stratgey` instead."""
+    return self._distribution_strategy
+
+  @property
+  def strategy(self):
+    """The current `tf.distribute.Strategy` object."""
     return self._distribution_strategy
 
   @property
@@ -1470,7 +1483,7 @@ class ReplicaContext(object):
 
 
 class _DefaultDistributionStrategy(DistributionStrategy):
-  """Default `DistributionStrategy` if none is explicitly selected."""
+  """Default `tf.distribute.Strategy` if none is explicitly selected."""
 
   def __init__(self):
     super(_DefaultDistributionStrategy, self).__init__(
@@ -1483,7 +1496,7 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def _scope(self, strategy):
     """Context manager setting a variable creator and `self` as current."""
     if distribution_strategy_context.has_distribution_strategy():
-      raise RuntimeError("Must not nest DistributionStrategy scopes.")
+      raise RuntimeError("Must not nest tf.distribute.Strategy scopes.")
 
     def creator(next_creator, *args, **kwargs):
       _require_distribution_strategy_scope_strategy(strategy)
@@ -1555,13 +1568,13 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
 
   @property
   def worker_devices(self):
-    raise RuntimeError(
-        "worker_devices() method unsupported by _DefaultDistributionStrategy.")
+    raise RuntimeError("worker_devices() method unsupported by default "
+                       "tf.distribute.Strategy.")
 
   @property
   def parameter_devices(self):
-    raise RuntimeError("parameter_devices() method unsupported by "
-                       "_DefaultDistributionStrategy.")
+    raise RuntimeError("parameter_devices() method unsupported by default "
+                       "tf.distribute.Strategy.")
 
   def non_slot_devices(self, var_list):
     return min(var_list, key=lambda x: x.name)
@@ -1600,8 +1613,8 @@ _original_from_proto = resource_variable_ops._from_proto_fn
 def _from_proto_fn(v, import_scope=None):
   if distribution_strategy_context.has_distribution_strategy():
     raise NotImplementedError(
-        "Deserialization of variables is not yet supported when using"
-        "distributed strategies.")
+        "Deserialization of variables is not yet supported when using a "
+        "tf.distribute.Strategy.")
   else:
     return _original_from_proto(v, import_scope=import_scope)
 
diff --git a/tensorflow/python/training/distribution_strategy_context.py b/tensorflow/python/training/distribution_strategy_context.py
index c73d65d330..0b3878de18 100644
--- a/tensorflow/python/training/distribution_strategy_context.py
+++ b/tensorflow/python/training/distribution_strategy_context.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import ops
 from tensorflow.python.util.lazy_loader import LazyLoader
+from tensorflow.python.util.tf_export import tf_export
 
 
 # There is a circular dependency between this and `distribute` module. So we
@@ -85,6 +86,7 @@ def _get_per_thread_mode():
 # Public API for accessing the current thread mode
 
 
+@tf_export("distribute.get_replica_context")
 def get_replica_context():
   """Returns the current `tf.distribute.ReplicaContext` or `None`.
 
@@ -95,7 +97,7 @@ def get_replica_context():
   1. starts in the default (single-replica) replica context (this function
      will return the default `ReplicaContext` object);
   2. switches to cross-replica context (in which case this will return
-     `None`) when entering a `with DistributionStrategy.scope():` block;
+     `None`) when entering a `with tf.distribute.Strategy.scope():` block;
   3. switches to a (non-default) replica context inside
      `extended.call_for_each_replica(fn, ...)`;
   4. if `fn` calls `get_replica_context().merge_call(merge_fn, ...)`, then
@@ -103,11 +105,11 @@ def get_replica_context():
      this function will return `None`).
 
   Note that you can also go directly from step 1 to 4 to switch to a
-  cross-replica context for the default `DistributionStrategy`. You may
+  cross-replica context for the default `tf.distribute.Strategy`. You may
   also switch from the cross-replica context of 4 to a replica context by
   calling `extended.call_for_each_replica()`, jumping back to step 3.
 
-  Most `DistributionStrategy` methods may only be executed in
+  Most `tf.distribute.Strategy` methods may only be executed in
   a cross-replica context, in a replica context you should use the
   `ReplicaContext` API instead.
 
@@ -124,7 +126,7 @@ def get_replica_context():
 
 
 def get_cross_replica_context():
-  """Returns the current DistributionStrategy if in a cross-replica context.
+  """Returns the current tf.distribute.Strategy if in a cross-replica context.
 
   DEPRECATED: Please use `in_cross_replica_context()` and
   `get_distribution_strategy()` instead.
@@ -133,22 +135,22 @@ def get_cross_replica_context():
 
   1. starts in the default (single-replica) replica context;
   2. switches to cross-replica context when entering a
-     `with DistributionStrategy.scope():` block;
+     `with tf.distribute.Strategy.scope():` block;
   3. switches to a (non-default) replica context inside
      `call_for_each_replica(fn, ...)`;
   4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then
      inside `merge_fn` you are back in the cross-replica context.
 
   Note that you can also go directly from step 1 to 4 to switch to a
-  cross-replica context for the default `DistributionStrategy`. You may
+  cross-replica context for the default `tf.distribute.Strategy`. You may
   also switch from the cross-replica context of 4 to a replica context by
   calling `call_for_each_replica()`, jumping back to step 3.
 
-  Most `DistributionStrategy` methods may only be executed in
+  Most `tf.distribute.Strategy` methods may only be executed in
   a cross-replica context.
 
   Returns:
-    Returns the current `DistributionStrategy` object in a cross-replica
+    Returns the current `tf.distribute.Strategy` object in a cross-replica
     context, or `None`.
 
     Exactly one of `get_replica_context()` and `get_cross_replica_context()`
@@ -157,6 +159,7 @@ def get_cross_replica_context():
   return _get_per_thread_mode().cross_replica_context
 
 
+@tf_export("distribute.in_cross_replica_context")
 def in_cross_replica_context():
   """Returns True if in a cross-replica context.
 
@@ -170,31 +173,33 @@ def in_cross_replica_context():
   return _get_per_thread_mode().cross_replica_context is not None
 
 
+@tf_export("distribute.get_strategy")
 def get_distribution_strategy():
-  """Returns the current `DistributionStrategy` object.
+  """Returns the current `tf.distribute.Strategy` object.
 
   Typically only used in a cross-replica context:
 
   ```
   if tf.distribute.in_cross_replica_context():
-    strategy = tf.distribute.get_distribution_strategy()
+    strategy = tf.distribute.get_strategy()
     ...
   ```
 
   Returns:
-    A `DistributionStrategy` object. Inside a
+    A `tf.distribute.Strategy` object. Inside a
     `with distribution_strategy.scope()` block, it returns
     `distribution_strategy`, otherwise it returns the default
-    (single-replica) `DistributionStrategy` object.
+    (single-replica) `tf.distribute.Strategy` object.
   """
   return _get_per_thread_mode().distribution_strategy
 
 
+@tf_export("distribute.has_strategy")
 def has_distribution_strategy():
-  """Return if there is a current non-default `DistributionStrategy`.
+  """Return if there is a current non-default `tf.distribute.Strategy`.
 
   Returns:
-    True if inside a `with distribution_strategy.scope():`.
+    True if inside a `with strategy.scope():`.
   """
   return get_distribution_strategy() is not _get_default_distribution_strategy()
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt
new file mode 100644
index 0000000000..c39ac5a20d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt
@@ -0,0 +1,25 @@
+path: "tensorflow.distribute.InputContext"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.InputContext\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "input_pipeline_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_input_pipelines"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_input_pipelines\', \'input_pipeline_id\', \'num_replicas_in_sync\'], varargs=None, keywords=None, defaults=[\'1\', \'0\', \'1\'], "
+  }
+  member_method {
+    name: "get_per_replica_batch_size"
+    argspec: "args=[\'self\', \'global_batch_size\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt
new file mode 100644
index 0000000000..6a7a3a97aa
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-replication-mode.pbtxt
@@ -0,0 +1,8 @@
+path: "tensorflow.distribute.InputReplicationMode"
+tf_class {
+  is_instance: "<enum \'InputReplicationMode\'>"
+  member {
+    name: "PER_WORKER"
+    mtype: "<enum \'InputReplicationMode\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt
new file mode 100644
index 0000000000..4899f38cad
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-reduce-op.pbtxt
@@ -0,0 +1,12 @@
+path: "tensorflow.distribute.ReduceOp"
+tf_class {
+  is_instance: "<enum \'ReduceOp\'>"
+  member {
+    name: "MEAN"
+    mtype: "<enum \'ReduceOp\'>"
+  }
+  member {
+    name: "SUM"
+    mtype: "<enum \'ReduceOp\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
new file mode 100644
index 0000000000..3eda6c6036
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
@@ -0,0 +1,33 @@
+path: "tensorflow.distribute.ReplicaContext"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.ReplicaContext\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "distribution_strategy"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "replica_id_in_sync_group"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "strategy"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "merge_call"
+    argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
new file mode 100644
index 0000000000..3b502b534b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
@@ -0,0 +1,81 @@
+path: "tensorflow.distribute.StrategyExtended"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategyExtended\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "experimental_between_graph"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "experimental_require_static_shapes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "experimental_should_init"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "parameter_devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_checkpoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_save_summary"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "worker_devices"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'container_strategy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_to"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_to"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call_for_each_replica"
+    argspec: "args=[\'self\', \'fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_run_steps_on_iterator"
+    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
+  }
+  member_method {
+    name: "non_slot_devices"
+    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "read_var"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_to"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update"
+    argspec: "args=[\'self\', \'var\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], "
+  }
+  member_method {
+    name: "update_non_slot"
+    argspec: "args=[\'self\', \'colocate_with\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], "
+  }
+  member_method {
+    name: "value_container"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
new file mode 100644
index 0000000000..4fe035b474
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -0,0 +1,133 @@
+path: "tensorflow.distribute.Strategy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategy\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "between_graph"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "extended"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "parameter_devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "require_static_shapes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_checkpoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_init"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_save_summary"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "worker_devices"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "call_for_each_replica"
+    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "configure"
+    argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "distribute_dataset"
+    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "group"
+    argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_dataset_iterator"
+    argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_input_fn_iterator"
+    argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
+  }
+  member_method {
+    name: "non_slot_devices"
+    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "read_var"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'aggregation\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_steps_on_dataset"
+    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
+  }
+  member_method {
+    name: "scope"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "unwrap"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update"
+    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "update_non_slot"
+    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "value_container"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
new file mode 100644
index 0000000000..4d833b54ba
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
@@ -0,0 +1,47 @@
+path: "tensorflow.distribute"
+tf_module {
+  member {
+    name: "InputContext"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "InputReplicationMode"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
+  member {
+    name: "ReduceOp"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
+  member {
+    name: "ReplicaContext"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Strategy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "StrategyExtended"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "get_loss_reduction"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_replica_context"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_strategy"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "has_strategy"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "in_cross_replica_context"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 1bb0152f19..6a45bc7b7f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -324,6 +324,10 @@ tf_module {
     name: "debugging"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "distribute"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "distributions"
     mtype: "<type \'module\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt
new file mode 100644
index 0000000000..c39ac5a20d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt
@@ -0,0 +1,25 @@
+path: "tensorflow.distribute.InputContext"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.InputContext\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "input_pipeline_id"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_input_pipelines"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_input_pipelines\', \'input_pipeline_id\', \'num_replicas_in_sync\'], varargs=None, keywords=None, defaults=[\'1\', \'0\', \'1\'], "
+  }
+  member_method {
+    name: "get_per_replica_batch_size"
+    argspec: "args=[\'self\', \'global_batch_size\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt
new file mode 100644
index 0000000000..6a7a3a97aa
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-replication-mode.pbtxt
@@ -0,0 +1,8 @@
+path: "tensorflow.distribute.InputReplicationMode"
+tf_class {
+  is_instance: "<enum \'InputReplicationMode\'>"
+  member {
+    name: "PER_WORKER"
+    mtype: "<enum \'InputReplicationMode\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt
new file mode 100644
index 0000000000..4899f38cad
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-reduce-op.pbtxt
@@ -0,0 +1,12 @@
+path: "tensorflow.distribute.ReduceOp"
+tf_class {
+  is_instance: "<enum \'ReduceOp\'>"
+  member {
+    name: "MEAN"
+    mtype: "<enum \'ReduceOp\'>"
+  }
+  member {
+    name: "SUM"
+    mtype: "<enum \'ReduceOp\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
new file mode 100644
index 0000000000..3eda6c6036
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
@@ -0,0 +1,33 @@
+path: "tensorflow.distribute.ReplicaContext"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.ReplicaContext\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "distribution_strategy"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "replica_id_in_sync_group"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "strategy"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "merge_call"
+    argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
new file mode 100644
index 0000000000..3b502b534b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
@@ -0,0 +1,81 @@
+path: "tensorflow.distribute.StrategyExtended"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategyExtended\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "experimental_between_graph"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "experimental_require_static_shapes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "experimental_should_init"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "parameter_devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_checkpoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_save_summary"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "worker_devices"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'container_strategy\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce_to"
+    argspec: "args=[\'self\', \'reduce_op\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast_to"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call_for_each_replica"
+    argspec: "args=[\'self\', \'fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_run_steps_on_iterator"
+    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
+  }
+  member_method {
+    name: "non_slot_devices"
+    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "read_var"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce_to"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update"
+    argspec: "args=[\'self\', \'var\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], "
+  }
+  member_method {
+    name: "update_non_slot"
+    argspec: "args=[\'self\', \'colocate_with\', \'fn\', \'args\', \'kwargs\', \'group\'], varargs=None, keywords=None, defaults=[\'()\', \'None\', \'True\'], "
+  }
+  member_method {
+    name: "value_container"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
new file mode 100644
index 0000000000..4fe035b474
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -0,0 +1,133 @@
+path: "tensorflow.distribute.Strategy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategy\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "between_graph"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "extended"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "parameter_devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "require_static_shapes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_checkpoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_init"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_save_summary"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "worker_devices"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "call_for_each_replica"
+    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "configure"
+    argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "distribute_dataset"
+    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "group"
+    argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_dataset_iterator"
+    argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_input_fn_iterator"
+    argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
+  }
+  member_method {
+    name: "non_slot_devices"
+    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "read_var"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'aggregation\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_steps_on_dataset"
+    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
+  }
+  member_method {
+    name: "scope"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "unwrap"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update"
+    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "update_non_slot"
+    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "value_container"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
new file mode 100644
index 0000000000..4d833b54ba
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
@@ -0,0 +1,47 @@
+path: "tensorflow.distribute"
+tf_module {
+  member {
+    name: "InputContext"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "InputReplicationMode"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
+  member {
+    name: "ReduceOp"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
+  member {
+    name: "ReplicaContext"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Strategy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "StrategyExtended"
+    mtype: "<type \'type\'>"
+  }
+  member_method {
+    name: "get_loss_reduction"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_replica_context"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_strategy"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "has_strategy"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "in_cross_replica_context"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 2d2ab7d0d3..10aa8b781f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "debugging"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "distribute"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "double"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
-- 
GitLab


From f96c3fb234c3411ec9b162661826ffaa0ef377a8 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Mon, 19 Nov 2018 10:46:50 -0800
Subject: [PATCH 0494/1554] Run buildifier on some files.

PiperOrigin-RevId: 222101413
---
 third_party/googleapis.BUILD       | 14 ++++++-----
 third_party/mkl_dnn/mkldnn.BUILD   |  2 +-
 third_party/nccl/archive.BUILD     | 26 +++++++++----------
 third_party/ngraph/ngraph.BUILD    | 26 +++++++++----------
 third_party/ngraph/ngraph_tf.BUILD | 40 +++++++++++++++---------------
 third_party/ngraph/tbb.BUILD       | 10 ++++----
 6 files changed, 60 insertions(+), 58 deletions(-)

diff --git a/third_party/googleapis.BUILD b/third_party/googleapis.BUILD
index 95e999af18..b8871eda72 100644
--- a/third_party/googleapis.BUILD
+++ b/third_party/googleapis.BUILD
@@ -13,7 +13,9 @@
 # limitations under the License.
 
 package(default_visibility = ["//visibility:public"])
+
 licenses(["notice"])  # Apache 2.0
+
 exports_files(["LICENSE"])
 
 load("@protobuf_archive//:protobuf.bzl", "cc_proto_library")
@@ -21,6 +23,9 @@ load("@protobuf_archive//:protobuf.bzl", "cc_proto_library")
 cc_proto_library(
     name = "bigtable_protos",
     srcs = [
+        "google/api/annotations.proto",
+        "google/api/auth.proto",
+        "google/api/http.proto",
         "google/bigtable/admin/v2/bigtable_instance_admin.proto",
         "google/bigtable/admin/v2/bigtable_table_admin.proto",
         "google/bigtable/admin/v2/common.proto",
@@ -31,15 +36,12 @@ cc_proto_library(
         "google/iam/v1/iam_policy.proto",
         "google/iam/v1/policy.proto",
         "google/longrunning/operations.proto",
-        "google/rpc/status.proto",
         "google/rpc/error_details.proto",
-        "google/api/annotations.proto",
-        "google/api/auth.proto",
-        "google/api/http.proto",
+        "google/rpc/status.proto",
     ],
     include = ".",
-    protoc = "@protobuf_archive//:protoc",
     default_runtime = "@protobuf_archive//:protobuf",
-    deps = ["@protobuf_archive//:cc_wkt_protos"],
+    protoc = "@protobuf_archive//:protoc",
     use_grpc_plugin = True,
+    deps = ["@protobuf_archive//:cc_wkt_protos"],
 )
diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
index 597ac69e2f..7a8ed3bf43 100644
--- a/third_party/mkl_dnn/mkldnn.BUILD
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -42,8 +42,8 @@ cc_library(
         "src",
         "src/common",
         "src/cpu",
-        "src/cpu/xbyak",
         "src/cpu/gemm",
+        "src/cpu/xbyak",
     ],
     nocopts = "-fno-exceptions",
     visibility = ["//visibility:public"],
diff --git a/third_party/nccl/archive.BUILD b/third_party/nccl/archive.BUILD
index c0833828a7..7a08f97ef3 100644
--- a/third_party/nccl/archive.BUILD
+++ b/third_party/nccl/archive.BUILD
@@ -64,13 +64,13 @@ nccl_library(
         ":device_srcs",
     ],
     copts = ["-DNCCL_OP=0"] + rdc_copts(),
+    linkstatic = True,
     prefix = "sum_",
     deps = [
-        ":src_hdrs",
         ":include_hdrs",
+        ":src_hdrs",
         "@local_config_cuda//cuda:cuda_headers",
     ],
-    linkstatic = True,
 )
 
 nccl_library(
@@ -80,13 +80,13 @@ nccl_library(
         ":device_srcs",
     ],
     copts = ["-DNCCL_OP=1"] + rdc_copts(),
+    linkstatic = True,
     prefix = "_prod",
     deps = [
-        ":src_hdrs",
         ":include_hdrs",
+        ":src_hdrs",
         "@local_config_cuda//cuda:cuda_headers",
     ],
-    linkstatic = True,
 )
 
 nccl_library(
@@ -96,13 +96,13 @@ nccl_library(
         ":device_srcs",
     ],
     copts = ["-DNCCL_OP=2"] + rdc_copts(),
+    linkstatic = True,
     prefix = "min_",
     deps = [
-        ":src_hdrs",
         ":include_hdrs",
+        ":src_hdrs",
         "@local_config_cuda//cuda:cuda_headers",
     ],
-    linkstatic = True,
 )
 
 nccl_library(
@@ -112,28 +112,28 @@ nccl_library(
         ":device_srcs",
     ],
     copts = ["-DNCCL_OP=3"] + rdc_copts(),
+    linkstatic = True,
     prefix = "max_",
     deps = [
-        ":src_hdrs",
         ":include_hdrs",
+        ":src_hdrs",
         "@local_config_cuda//cuda:cuda_headers",
     ],
-    linkstatic = True,
 )
 
 nccl_library(
     name = "functions",
     srcs = [
-        ":device_hdrs",
         "src/collectives/device/functions.cu",
+        ":device_hdrs",
     ],
     copts = rdc_copts(),
+    linkstatic = True,
     deps = [
-        ":src_hdrs",
         ":include_hdrs",
+        ":src_hdrs",
         "@local_config_cuda//cuda:cuda_headers",
     ],
-    linkstatic = True,
 )
 
 rdc_library(
@@ -162,13 +162,13 @@ nccl_library(
         "src/nccl.h",
     ],
     hdrs = ["src/nccl.h"],
+    copts = cuda_default_copts(),
     include_prefix = "third_party/nccl",
     strip_include_prefix = "src",
-    copts = cuda_default_copts(),
+    visibility = ["//visibility:public"],
     deps = [
         ":device_code",
         ":include_hdrs",
         ":src_hdrs",
     ],
-    visibility = ["//visibility:public"],
 )
diff --git a/third_party/ngraph/ngraph.BUILD b/third_party/ngraph/ngraph.BUILD
index f556c5279d..63e9548c53 100644
--- a/third_party/ngraph/ngraph.BUILD
+++ b/third_party/ngraph/ngraph.BUILD
@@ -97,13 +97,6 @@ cc_library(
         "src/ngraph/runtime/cpu/pass/cpu_workspace_insertion.cpp",
     ],
     hdrs = glob(["src/ngraph/runtime/cpu/**/*.hpp"]) + glob([]),
-    deps = [
-        ":ngraph_headers",
-        "@eigen_archive//:eigen",
-        "@nlohmann_json_lib",
-        "@tbb",
-        "@mkl_dnn//:mkl_dnn",
-    ],
     copts = [
         "-I external/ngraph/src",
         "-I external/nlohmann_json_lib/include/",
@@ -113,6 +106,13 @@ cc_library(
         '-D PROJECT_ROOT_DIR=\\"\\"',
     ],
     visibility = ["//visibility:public"],
+    deps = [
+        ":ngraph_headers",
+        "@eigen_archive//:eigen",
+        "@mkl_dnn",
+        "@nlohmann_json_lib",
+        "@tbb",
+    ],
     alwayslink = 1,
 )
 
@@ -138,12 +138,6 @@ cc_library(
         "src/ngraph/runtime/*.cpp",
         "src/ngraph/type/*.cpp",
     ]),
-    deps = [
-        ":ngraph_headers",
-        ":ngraph_cpu_backend",
-        "@eigen_archive//:eigen",
-        "@nlohmann_json_lib",
-    ],
     copts = [
         "-I external/ngraph/src",
         "-I external/nlohmann_json_lib/include/",
@@ -152,5 +146,11 @@ cc_library(
         '-D PROJECT_ROOT_DIR=\\"\\"',
     ],
     visibility = ["//visibility:public"],
+    deps = [
+        ":ngraph_cpu_backend",
+        ":ngraph_headers",
+        "@eigen_archive//:eigen",
+        "@nlohmann_json_lib",
+    ],
     alwayslink = 1,
 )
diff --git a/third_party/ngraph/ngraph_tf.BUILD b/third_party/ngraph/ngraph_tf.BUILD
index 068e411e81..db9a66f9b5 100644
--- a/third_party/ngraph/ngraph_tf.BUILD
+++ b/third_party/ngraph/ngraph_tf.BUILD
@@ -10,6 +10,10 @@ load(
 cc_library(
     name = "ngraph_tf",
     srcs = [
+        "logging/ngraph_log.cc",
+        "logging/ngraph_log.h",
+        "logging/tf_graph_writer.cc",
+        "logging/tf_graph_writer.h",
         "src/ngraph_api.cc",
         "src/ngraph_api.h",
         "src/ngraph_assign_clusters.cc",
@@ -41,27 +45,23 @@ cc_library(
         "src/tf_deadness_analysis.h",
         "src/tf_graphcycles.cc",
         "src/tf_graphcycles.h",
-        "logging/ngraph_log.h",
-        "logging/ngraph_log.cc",
-        "logging/tf_graph_writer.h",
-        "logging/tf_graph_writer.cc",
-    ],
-    deps = [
-        "@org_tensorflow//tensorflow/core:protos_all_proto_text",
-        "@org_tensorflow//tensorflow/core:framework_headers_lib",
-        "@org_tensorflow//tensorflow/core:core_cpu_headers_lib",
-        "@ngraph//:ngraph_core",
-        "@com_google_absl//absl/container:container_memory",
-        "@com_google_absl//absl/container:flat_hash_set",
-        "@com_google_absl//absl/types:variant",
     ],
     copts = [
         "-I external/ngraph_tf/src",
         "-I external/ngraph_tf/logging",
         "-I external/ngraph/src",
     ],
-    alwayslink = 1,
     visibility = ["//visibility:public"],
+    deps = [
+        "@com_google_absl//absl/container:container_memory",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/types:variant",
+        "@ngraph//:ngraph_core",
+        "@org_tensorflow//tensorflow/core:core_cpu_headers_lib",
+        "@org_tensorflow//tensorflow/core:framework_headers_lib",
+        "@org_tensorflow//tensorflow/core:protos_all_proto_text",
+    ],
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -82,6 +82,12 @@ tf_cc_test(
         "test/test_utilities.h",
         "test/tf_exec.cpp",
     ],
+    extra_copts = [
+        "-fexceptions ",
+        "-I external/ngraph_tf/src",
+        "-I external/ngraph_tf/logging",
+        "-I external/ngraph/src",
+    ],
     deps = [
         ":ngraph_tf",
         "@com_google_googletest//:gtest",
@@ -89,10 +95,4 @@ tf_cc_test(
         "@org_tensorflow//tensorflow/cc:client_session",
         "@org_tensorflow//tensorflow/core:tensorflow",
     ],
-    extra_copts = [
-        "-fexceptions ",
-        "-I external/ngraph_tf/src",
-        "-I external/ngraph_tf/logging",
-        "-I external/ngraph/src",
-    ],
 )
diff --git a/third_party/ngraph/tbb.BUILD b/third_party/ngraph/tbb.BUILD
index 04e6544ffb..c78a2d79dd 100644
--- a/third_party/ngraph/tbb.BUILD
+++ b/third_party/ngraph/tbb.BUILD
@@ -14,6 +14,10 @@ genrule(
     srcs = glob(["**"]) + [
         "@local_config_cc//:toolchain",
     ],
+    outs = [
+        "libtbb.a",
+        "libtbbmalloc.a",
+    ],
     cmd = """
 	    set -e
 	    WORK_DIR=$$PWD
@@ -45,19 +49,15 @@ genrule(
         cp build/build_{release,debug}/*.a $$DEST_DIR
 		cd $$WORK_DIR
 	""",
-    outs = [
-        "libtbb.a",
-        "libtbbmalloc.a",
-    ],
 )
 
 cc_library(
     name = "tbb",
+    srcs = ["libtbb.a"],
     hdrs = glob([
         "include/serial/**",
         "include/tbb/**/**",
     ]),
-    srcs = ["libtbb.a"],
     includes = ["include"],
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From 22c33cbffba73375f86101a207d2105f9c2b25a9 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 19 Nov 2018 10:58:17 -0800
Subject: [PATCH 0495/1554] Switching the TF default Feature Column bindings to
 the new implementation.

PiperOrigin-RevId: 222103424
---
 tensorflow/contrib/feature_column/BUILD       |   11 +-
 .../feature_column/sequence_feature_column.py |   16 +-
 ...equence_feature_column_integration_test.py |   17 +-
 .../sequence_feature_column_test.py           |   62 +-
 .../sequence_feature_column_v2.py             |   20 +-
 .../sequence_feature_column_v2_test.py        |   61 +-
 .../python/feature_column/feature_column.py   |  236 +--
 .../feature_column/feature_column_test.py     | 1241 ++++++++-------
 .../feature_column/feature_column_v2.py       |  419 ++---
 .../feature_column/feature_column_v2_test.py  | 1383 +++++++++--------
 .../feature_columns_integration_test.py       |   22 +-
 11 files changed, 1704 insertions(+), 1784 deletions(-)

diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD
index bbe335be3e..1cd83bdb5d 100644
--- a/tensorflow/contrib/feature_column/BUILD
+++ b/tensorflow/contrib/feature_column/BUILD
@@ -14,6 +14,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":sequence_feature_column",
+        ":sequence_feature_column_v2",
         "//tensorflow/python:util",
     ],
 )
@@ -32,7 +33,7 @@ py_library(
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/feature_column",
+        "//tensorflow/python/feature_column:feature_column_py",
     ],
 )
 
@@ -51,7 +52,7 @@ py_test(
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:training",
-        "//tensorflow/python/feature_column",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
@@ -69,7 +70,7 @@ py_test(
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
-        "//tensorflow/python/feature_column",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/keras:layers",
     ],
 )
@@ -89,7 +90,7 @@ py_library(
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/feature_column",
-        "//tensorflow/python/feature_column:feature_column_v2",
+        "//tensorflow/python/feature_column:feature_column_py",
     ],
 )
 
@@ -110,7 +111,7 @@ py_test(
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:training",
         "//tensorflow/python/feature_column",
-        "//tensorflow/python/feature_column:feature_column_v2",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py
index dd6da35ed0..9b3a5c58aa 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py
@@ -222,10 +222,8 @@ def sequence_categorical_column_with_identity(
     ValueError: if `default_value` is not in range `[0, num_buckets)`.
   """
   return fc._SequenceCategoricalColumn(
-      fc.categorical_column_with_identity(
-          key=key,
-          num_buckets=num_buckets,
-          default_value=default_value))
+      fc._categorical_column_with_identity(
+          key=key, num_buckets=num_buckets, default_value=default_value))
 
 
 def sequence_categorical_column_with_hash_bucket(
@@ -265,10 +263,8 @@ def sequence_categorical_column_with_hash_bucket(
     ValueError: `dtype` is neither string nor integer.
   """
   return fc._SequenceCategoricalColumn(
-      fc.categorical_column_with_hash_bucket(
-          key=key,
-          hash_bucket_size=hash_bucket_size,
-          dtype=dtype))
+      fc._categorical_column_with_hash_bucket(
+          key=key, hash_bucket_size=hash_bucket_size, dtype=dtype))
 
 
 def sequence_categorical_column_with_vocabulary_file(
@@ -324,7 +320,7 @@ def sequence_categorical_column_with_vocabulary_file(
     ValueError: `dtype` is neither string nor integer.
   """
   return fc._SequenceCategoricalColumn(
-      fc.categorical_column_with_vocabulary_file(
+      fc._categorical_column_with_vocabulary_file(
           key=key,
           vocabulary_file=vocabulary_file,
           vocabulary_size=vocabulary_size,
@@ -384,7 +380,7 @@ def sequence_categorical_column_with_vocabulary_list(
     ValueError: if `dtype` is not integer or string.
   """
   return fc._SequenceCategoricalColumn(
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key=key,
           vocabulary_list=vocabulary_list,
           dtype=dtype,
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py
index d8ca363627..bcc25b8de8 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_integration_test.py
@@ -53,19 +53,20 @@ class SequenceFeatureColumnIntegrationTest(test.TestCase):
     return example
 
   def _build_feature_columns(self):
-    col = fc.categorical_column_with_identity(
-        'int_ctx', num_buckets=100)
+    col = fc._categorical_column_with_identity('int_ctx', num_buckets=100)
     ctx_cols = [
-        fc.embedding_column(col, dimension=10),
-        fc.numeric_column('float_ctx')]
+        fc._embedding_column(col, dimension=10),
+        fc._numeric_column('float_ctx')
+    ]
 
     identity_col = sfc.sequence_categorical_column_with_identity(
         'int_list', num_buckets=10)
     bucket_col = sfc.sequence_categorical_column_with_hash_bucket(
         'bytes_list', hash_bucket_size=100)
     seq_cols = [
-        fc.embedding_column(identity_col, dimension=10),
-        fc.embedding_column(bucket_col, dimension=20)]
+        fc._embedding_column(identity_col, dimension=10),
+        fc._embedding_column(bucket_col, dimension=20)
+    ]
 
     return ctx_cols, seq_cols
 
@@ -148,8 +149,8 @@ class SequenceExampleParsingTest(test.TestCase):
     """
     example = _make_sequence_example()
     columns = [
-        fc.categorical_column_with_identity('int_ctx', num_buckets=100),
-        fc.numeric_column('float_ctx'),
+        fc._categorical_column_with_identity('int_ctx', num_buckets=100),
+        fc._numeric_column('float_ctx'),
         col_fn(col_name, col_arg)
     ]
     context, seq_features = parsing_ops.parse_single_sequence_example(
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py
index 2163af0b43..d5f7402829 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc
 from tensorflow.python.feature_column import feature_column as fc
+from tensorflow.python.feature_column import feature_column_lib as fc_lib
 from tensorflow.python.feature_column.feature_column import _LazyBuilder
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -109,13 +110,15 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc.embedding_column(
-        categorical_column_a, dimension=embedding_dimension_a,
+    embedding_column_a = fc._embedding_column(
+        categorical_column_a,
+        dimension=embedding_dimension_a,
         initializer=_get_initializer(embedding_dimension_a, embedding_values_a))
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_b = fc.embedding_column(
-        categorical_column_b, dimension=embedding_dimension_b,
+    embedding_column_b = fc._embedding_column(
+        categorical_column_b,
+        dimension=embedding_dimension_b,
         initializer=_get_initializer(embedding_dimension_b, embedding_values_b))
 
     input_layer, sequence_length = sfc.sequence_input_layer(
@@ -148,10 +151,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc.embedding_column(
-        categorical_column_a, dimension=2)
+    embedding_column_a = fc._embedding_column(categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -206,7 +208,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     # Test that columns are reordered alphabetically.
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc_lib.shared_embedding_columns(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension,
         initializer=_get_initializer(embedding_dimension, embedding_values))
@@ -244,11 +246,11 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc_lib.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     with self.assertRaisesRegexp(
@@ -315,10 +317,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size_a)
-    indicator_column_a = fc.indicator_column(categorical_column_a)
+    indicator_column_a = fc._indicator_column(categorical_column_a)
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size_b)
-    indicator_column_b = fc.indicator_column(categorical_column_b)
+    indicator_column_b = fc._indicator_column(categorical_column_b)
     input_layer, sequence_length = sfc.sequence_input_layer(
         features={
             'aaa': sparse_input_a,
@@ -342,9 +344,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc.indicator_column(categorical_column_a)
+    indicator_column_a = fc._indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -530,7 +532,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    indicator_column = fc.indicator_column(categorical_column)
+    indicator_column = fc._indicator_column(categorical_column)
 
     input_layer, _ = sfc.sequence_input_layer(
         features={'aaa': sparse_input}, feature_columns=[indicator_column])
@@ -616,8 +618,7 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc.embedding_column(
-        categorical_column_a, dimension=2)
+    embedding_column_a = fc._embedding_column(categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -639,7 +640,7 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc.indicator_column(categorical_column_a)
+    indicator_column_a = fc._indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -918,8 +919,9 @@ class SequenceEmbeddingColumnTest(
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     embedding_lookup, _ = embedding_column._get_sequence_dense_tensor(
@@ -956,8 +958,7 @@ class SequenceEmbeddingColumnTest(
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=2)
+    embedding_column = fc._embedding_column(categorical_column, dimension=2)
 
     _, sequence_length = embedding_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': inputs}))
@@ -984,8 +985,7 @@ class SequenceEmbeddingColumnTest(
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=2)
+    embedding_column = fc._embedding_column(categorical_column, dimension=2)
 
     _, sequence_length = embedding_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': sparse_input}))
@@ -1055,7 +1055,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase):
         key='aaa', num_buckets=vocabulary_size)
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc_lib.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -1101,7 +1101,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase):
     expected_sequence_length_b = [2, 1]
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc_lib.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor(
@@ -1152,7 +1152,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase):
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
 
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc_lib.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor(
@@ -1218,7 +1218,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc.indicator_column(categorical_column)
+    indicator_column = fc._indicator_column(categorical_column)
 
     indicator_tensor, _ = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': inputs}))
@@ -1250,7 +1250,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc.indicator_column(categorical_column)
+    indicator_column = fc._indicator_column(categorical_column)
 
     _, sequence_length = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': inputs}))
@@ -1277,7 +1277,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc.indicator_column(categorical_column)
+    indicator_column = fc._indicator_column(categorical_column)
 
     _, sequence_length = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': sparse_input}))
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
index 67ffb93966..0d34ad1618 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
@@ -26,7 +26,7 @@ import collections
 
 
 from tensorflow.python.feature_column import feature_column as fc_old
-from tensorflow.python.feature_column import feature_column_v2 as fc
+from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -226,10 +226,8 @@ def sequence_categorical_column_with_identity(
     ValueError: if `default_value` is not in range `[0, num_buckets)`.
   """
   return fc_old._SequenceCategoricalColumn(
-      fc_old.categorical_column_with_identity(
-          key=key,
-          num_buckets=num_buckets,
-          default_value=default_value))
+      fc_old._categorical_column_with_identity(
+          key=key, num_buckets=num_buckets, default_value=default_value))
 
 
 def sequence_categorical_column_with_hash_bucket(
@@ -269,10 +267,8 @@ def sequence_categorical_column_with_hash_bucket(
     ValueError: `dtype` is neither string nor integer.
   """
   return fc_old._SequenceCategoricalColumn(
-      fc_old.categorical_column_with_hash_bucket(
-          key=key,
-          hash_bucket_size=hash_bucket_size,
-          dtype=dtype))
+      fc_old._categorical_column_with_hash_bucket(
+          key=key, hash_bucket_size=hash_bucket_size, dtype=dtype))
 
 
 def sequence_categorical_column_with_vocabulary_file(
@@ -328,7 +324,7 @@ def sequence_categorical_column_with_vocabulary_file(
     ValueError: `dtype` is neither string nor integer.
   """
   return fc_old._SequenceCategoricalColumn(
-      fc_old.categorical_column_with_vocabulary_file(
+      fc_old._categorical_column_with_vocabulary_file(
           key=key,
           vocabulary_file=vocabulary_file,
           vocabulary_size=vocabulary_size,
@@ -388,7 +384,7 @@ def sequence_categorical_column_with_vocabulary_list(
     ValueError: if `dtype` is not integer or string.
   """
   return fc_old._SequenceCategoricalColumn(
-      fc_old.categorical_column_with_vocabulary_list(
+      fc_old._categorical_column_with_vocabulary_list(
           key=key,
           vocabulary_list=vocabulary_list,
           dtype=dtype,
@@ -441,7 +437,7 @@ def sequence_numeric_column(
     ValueError: if any dimension in shape is not a positive integer.
     ValueError: if `dtype` is not convertible to `tf.float32`.
   """
-  shape = fc._check_shape(shape=shape, key=key)
+  shape = fc_old._check_shape(shape=shape, key=key)
   if not (dtype.is_integer or dtype.is_floating):
     raise ValueError('dtype must be convertible to float. '
                      'dtype: {}, key: {}'.format(dtype, key))
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index 904d149fdb..ca4398a142 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -25,7 +25,7 @@ import numpy as np
 from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc_old
 from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column_v2 as sfc
 from tensorflow.python.feature_column import feature_column as fc_old
-from tensorflow.python.feature_column import feature_column_v2 as fc
+from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.feature_column.feature_column import _LazyBuilder
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -111,13 +111,15 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old.embedding_column(
-        categorical_column_a, dimension=embedding_dimension_a,
+    embedding_column_a = fc_old._embedding_column(
+        categorical_column_a,
+        dimension=embedding_dimension_a,
         initializer=_get_initializer(embedding_dimension_a, embedding_values_a))
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_b = fc_old.embedding_column(
-        categorical_column_b, dimension=embedding_dimension_b,
+    embedding_column_b = fc_old._embedding_column(
+        categorical_column_b,
+        dimension=embedding_dimension_b,
         initializer=_get_initializer(embedding_dimension_b, embedding_values_b))
 
     input_layer, sequence_length = sfc.sequence_input_layer(
@@ -150,9 +152,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old.categorical_column_with_identity(
+    categorical_column_a = fc_old._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old.embedding_column(
+    embedding_column_a = fc_old._embedding_column(
         categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
@@ -208,7 +210,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     # Test that columns are reordered alphabetically.
-    shared_embedding_columns = fc_old.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension,
         initializer=_get_initializer(embedding_dimension, embedding_values))
@@ -246,11 +248,11 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old.categorical_column_with_identity(
+    categorical_column_a = fc_old._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc_old.categorical_column_with_identity(
+    categorical_column_b = fc_old._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc_old.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     with self.assertRaisesRegexp(
@@ -317,10 +319,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size_a)
-    indicator_column_a = fc_old.indicator_column(categorical_column_a)
+    indicator_column_a = fc_old._indicator_column(categorical_column_a)
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size_b)
-    indicator_column_b = fc_old.indicator_column(categorical_column_b)
+    indicator_column_b = fc_old._indicator_column(categorical_column_b)
     input_layer, sequence_length = sfc.sequence_input_layer(
         features={
             'aaa': sparse_input_a,
@@ -344,9 +346,9 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old.categorical_column_with_identity(
+    categorical_column_a = fc_old._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc_old.indicator_column(categorical_column_a)
+    indicator_column_a = fc_old._indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -532,7 +534,7 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    indicator_column = fc_old.indicator_column(categorical_column)
+    indicator_column = fc_old._indicator_column(categorical_column)
 
     input_layer, _ = sfc.sequence_input_layer(
         features={'aaa': sparse_input}, feature_columns=[indicator_column])
@@ -618,7 +620,7 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old.embedding_column(
+    embedding_column_a = fc_old._embedding_column(
         categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
@@ -641,7 +643,7 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc_old.indicator_column(categorical_column_a)
+    indicator_column_a = fc_old._indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -920,8 +922,9 @@ class SequenceEmbeddingColumnTest(
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc_old.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc_old._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     embedding_lookup, _ = embedding_column._get_sequence_dense_tensor(
@@ -958,8 +961,7 @@ class SequenceEmbeddingColumnTest(
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc_old.embedding_column(
-        categorical_column, dimension=2)
+    embedding_column = fc_old._embedding_column(categorical_column, dimension=2)
 
     _, sequence_length = embedding_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': inputs}))
@@ -986,8 +988,7 @@ class SequenceEmbeddingColumnTest(
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc_old.embedding_column(
-        categorical_column, dimension=2)
+    embedding_column = fc_old._embedding_column(categorical_column, dimension=2)
 
     _, sequence_length = embedding_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': sparse_input}))
@@ -1057,7 +1058,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase):
         key='aaa', num_buckets=vocabulary_size)
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc_old.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -1103,7 +1104,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase):
     expected_sequence_length_b = [2, 1]
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc_old.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor(
@@ -1154,7 +1155,7 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase):
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
 
-    shared_embedding_columns = fc_old.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     sequence_length_a = shared_embedding_columns[0]._get_sequence_dense_tensor(
@@ -1220,7 +1221,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc_old.indicator_column(categorical_column)
+    indicator_column = fc_old._indicator_column(categorical_column)
 
     indicator_tensor, _ = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': inputs}))
@@ -1252,7 +1253,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc_old.indicator_column(categorical_column)
+    indicator_column = fc_old._indicator_column(categorical_column)
 
     _, sequence_length = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': inputs}))
@@ -1279,7 +1280,7 @@ class SequenceIndicatorColumnTest(test.TestCase, parameterized.TestCase):
 
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column = fc.indicator_column_v2(categorical_column)
+    indicator_column = fc.indicator_column(categorical_column)
 
     _, sequence_length = indicator_column._get_sequence_dense_tensor(
         _LazyBuilder({'aaa': sparse_input}))
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 92e308070e..a858d92608 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -807,11 +807,14 @@ def make_parse_example_spec(feature_columns):
   return result
 
 
-@tf_export(v1=['feature_column.embedding_column'])
-def embedding_column(
-    categorical_column, dimension, combiner='mean', initializer=None,
-    ckpt_to_load_from=None, tensor_name_in_ckpt=None, max_norm=None,
-    trainable=True):
+def _embedding_column(categorical_column,
+                      dimension,
+                      combiner='mean',
+                      initializer=None,
+                      ckpt_to_load_from=None,
+                      tensor_name_in_ckpt=None,
+                      max_norm=None,
+                      trainable=True):
   """`_DenseColumn` that converts from sparse, categorical input.
 
   Use this when your inputs are sparse, but you want to convert them to a dense
@@ -919,178 +922,11 @@ def embedding_column(
       trainable=trainable)
 
 
-@tf_export(v1=['feature_column.shared_embedding_columns'])
-def shared_embedding_columns(
-    categorical_columns, dimension, combiner='mean', initializer=None,
-    shared_embedding_collection_name=None, ckpt_to_load_from=None,
-    tensor_name_in_ckpt=None, max_norm=None, trainable=True):
-  """List of dense columns that convert from sparse, categorical input.
-
-  This is similar to `embedding_column`, except that it produces a list of
-  embedding columns that share the same embedding weights.
-
-  Use this when your inputs are sparse and of the same type (e.g. watched and
-  impression video IDs that share the same vocabulary), and you want to convert
-  them to a dense representation (e.g., to feed to a DNN).
-
-  Inputs must be a list of categorical columns created by any of the
-  `categorical_column_*` function. They must all be of the same type and have
-  the same arguments except `key`. E.g. they can be
-  categorical_column_with_vocabulary_file with the same vocabulary_file. Some or
-  all columns could also be weighted_categorical_column.
-
-  Here is an example embedding of two features for a DNNClassifier model:
-
-  ```python
-  watched_video_id = categorical_column_with_vocabulary_file(
-      'watched_video_id', video_vocabulary_file, video_vocabulary_size)
-  impression_video_id = categorical_column_with_vocabulary_file(
-      'impression_video_id', video_vocabulary_file, video_vocabulary_size)
-  columns = shared_embedding_columns(
-      [watched_video_id, impression_video_id], dimension=10)
-
-  estimator = tf.estimator.DNNClassifier(feature_columns=columns, ...)
-
-  label_column = ...
-  def input_fn():
-    features = tf.parse_example(
-        ..., features=make_parse_example_spec(columns + [label_column]))
-    labels = features.pop(label_column.name)
-    return features, labels
-
-  estimator.train(input_fn=input_fn, steps=100)
-  ```
-
-  Here is an example using `shared_embedding_columns` with model_fn:
-
-  ```python
-  def model_fn(features, ...):
-    watched_video_id = categorical_column_with_vocabulary_file(
-        'watched_video_id', video_vocabulary_file, video_vocabulary_size)
-    impression_video_id = categorical_column_with_vocabulary_file(
-        'impression_video_id', video_vocabulary_file, video_vocabulary_size)
-    columns = shared_embedding_columns(
-        [watched_video_id, impression_video_id], dimension=10)
-    dense_tensor = input_layer(features, columns)
-    # Form DNN layers, calculate loss, and return EstimatorSpec.
-    ...
-  ```
-
-  Args:
-    categorical_columns: List of categorical columns created by a
-      `categorical_column_with_*` function. These columns produce the sparse IDs
-      that are inputs to the embedding lookup. All columns must be of the same
-      type and have the same arguments except `key`. E.g. they can be
-      categorical_column_with_vocabulary_file with the same vocabulary_file.
-      Some or all columns could also be weighted_categorical_column.
-    dimension: An integer specifying dimension of the embedding, must be > 0.
-    combiner: A string specifying how to reduce if there are multiple entries
-      in a single row. Currently 'mean', 'sqrtn' and 'sum' are supported, with
-      'mean' the default. 'sqrtn' often achieves good accuracy, in particular
-      with bag-of-words columns. Each of this can be thought as example level
-      normalizations on the column. For more information, see
-      `tf.embedding_lookup_sparse`.
-    initializer: A variable initializer function to be used in embedding
-      variable initialization. If not specified, defaults to
-      `tf.truncated_normal_initializer` with mean `0.0` and standard deviation
-      `1/sqrt(dimension)`.
-    shared_embedding_collection_name: Optional name of the collection where
-      shared embedding weights are added. If not given, a reasonable name will
-      be chosen based on the names of `categorical_columns`. This is also used
-      in `variable_scope` when creating shared embedding weights.
-    ckpt_to_load_from: String representing checkpoint name/pattern from which to
-      restore column weights. Required if `tensor_name_in_ckpt` is not `None`.
-    tensor_name_in_ckpt: Name of the `Tensor` in `ckpt_to_load_from` from
-      which to restore the column weights. Required if `ckpt_to_load_from` is
-      not `None`.
-    max_norm: If not `None`, each embedding is clipped if its l2-norm is
-      larger than this value, before combining.
-    trainable: Whether or not the embedding is trainable. Default is True.
-
-  Returns:
-    A list of dense columns that converts from sparse input. The order of
-    results follows the ordering of `categorical_columns`.
-
-  Raises:
-    ValueError: if `dimension` not > 0.
-    ValueError: if any of the given `categorical_columns` is of different type
-      or has different arguments than the others.
-    ValueError: if exactly one of `ckpt_to_load_from` and `tensor_name_in_ckpt`
-      is specified.
-    ValueError: if `initializer` is specified and is not callable.
-    RuntimeError: if eager execution is enabled.
-  """
-  if context.executing_eagerly():
-    raise RuntimeError('shared_embedding_columns are not supported when eager '
-                       'execution is enabled.')
-
-  if (dimension is None) or (dimension < 1):
-    raise ValueError('Invalid dimension {}.'.format(dimension))
-  if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None):
-    raise ValueError('Must specify both `ckpt_to_load_from` and '
-                     '`tensor_name_in_ckpt` or none of them.')
-
-  if (initializer is not None) and (not callable(initializer)):
-    raise ValueError('initializer must be callable if specified.')
-  if initializer is None:
-    initializer = init_ops.truncated_normal_initializer(
-        mean=0.0, stddev=1. / math.sqrt(dimension))
-
-  # Sort the columns so the default collection name is deterministic even if the
-  # user passes columns from an unsorted collection, such as dict.values().
-  sorted_columns = sorted(categorical_columns, key=lambda x: x.name)
-
-  c0 = sorted_columns[0]
-  num_buckets = c0._num_buckets  # pylint: disable=protected-access
-  if not isinstance(c0, _CategoricalColumn):
-    raise ValueError(
-        'All categorical_columns must be subclasses of _CategoricalColumn. '
-        'Given: {}, of type: {}'.format(c0, type(c0)))
-  if isinstance(c0, _WeightedCategoricalColumn):
-    c0 = c0.categorical_column
-  for c in sorted_columns[1:]:
-    if isinstance(c, _WeightedCategoricalColumn):
-      c = c.categorical_column
-    if not isinstance(c, type(c0)):
-      raise ValueError(
-          'To use shared_embedding_column, all categorical_columns must have '
-          'the same type, or be weighted_categorical_column of the same type. '
-          'Given column: {} of type: {} does not match given column: {} of '
-          'type: {}'.format(c0, type(c0), c, type(c)))
-    if num_buckets != c._num_buckets:  # pylint: disable=protected-access
-      raise ValueError(
-          'To use shared_embedding_column, all categorical_columns must have '
-          'the same number of buckets. Given column: {} with buckets: {} does  '
-          'not match column: {} with buckets: {}'.format(
-              c0, num_buckets, c, c._num_buckets))  # pylint: disable=protected-access
-
-  if not shared_embedding_collection_name:
-    shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns)
-    shared_embedding_collection_name += '_shared_embedding'
-
-  result = []
-  for column in categorical_columns:
-    result.append(
-        _SharedEmbeddingColumn(
-            categorical_column=column,
-            initializer=initializer,
-            dimension=dimension,
-            combiner=combiner,
-            shared_embedding_collection_name=shared_embedding_collection_name,
-            ckpt_to_load_from=ckpt_to_load_from,
-            tensor_name_in_ckpt=tensor_name_in_ckpt,
-            max_norm=max_norm,
-            trainable=trainable))
-
-  return result
-
-
-@tf_export(v1=['feature_column.numeric_column'])
-def numeric_column(key,
-                   shape=(1,),
-                   default_value=None,
-                   dtype=dtypes.float32,
-                   normalizer_fn=None):
+def _numeric_column(key,
+                    shape=(1,),
+                    default_value=None,
+                    dtype=dtypes.float32,
+                    normalizer_fn=None):
   """Represents real valued or numerical features.
 
   Example:
@@ -1161,8 +997,7 @@ def numeric_column(key,
       normalizer_fn=normalizer_fn)
 
 
-@tf_export(v1=['feature_column.bucketized_column'])
-def bucketized_column(source_column, boundaries):
+def _bucketized_column(source_column, boundaries):
   """Represents discretized dense input.
 
   Buckets include the left boundary, and exclude the right boundary. Namely,
@@ -1258,10 +1093,9 @@ def _assert_key_is_string(key):
             type(key), key))
 
 
-@tf_export(v1=['feature_column.categorical_column_with_hash_bucket'])
-def categorical_column_with_hash_bucket(key,
-                                        hash_bucket_size,
-                                        dtype=dtypes.string):
+def _categorical_column_with_hash_bucket(key,
+                                         hash_bucket_size,
+                                         dtype=dtypes.string):
   """Represents sparse feature where ids are set by hashing.
 
   Use this when your sparse features are in string or integer format, and you
@@ -1317,13 +1151,12 @@ def categorical_column_with_hash_bucket(key,
   return _HashedCategoricalColumn(key, hash_bucket_size, dtype)
 
 
-@tf_export(v1=['feature_column.categorical_column_with_vocabulary_file'])
-def categorical_column_with_vocabulary_file(key,
-                                            vocabulary_file,
-                                            vocabulary_size=None,
-                                            num_oov_buckets=0,
-                                            default_value=None,
-                                            dtype=dtypes.string):
+def _categorical_column_with_vocabulary_file(key,
+                                             vocabulary_file,
+                                             vocabulary_size=None,
+                                             num_oov_buckets=0,
+                                             default_value=None,
+                                             dtype=dtypes.string):
   """A `_CategoricalColumn` with a vocabulary file.
 
   Use this when your inputs are in string or integer format, and you have a
@@ -1437,9 +1270,11 @@ def categorical_column_with_vocabulary_file(key,
       dtype=dtype)
 
 
-@tf_export(v1=['feature_column.categorical_column_with_vocabulary_list'])
-def categorical_column_with_vocabulary_list(
-    key, vocabulary_list, dtype=None, default_value=-1, num_oov_buckets=0):
+def _categorical_column_with_vocabulary_list(key,
+                                             vocabulary_list,
+                                             dtype=None,
+                                             default_value=-1,
+                                             num_oov_buckets=0):
   """A `_CategoricalColumn` with in-memory vocabulary.
 
   Use this when your inputs are in string or integer format, and you have an
@@ -1548,8 +1383,7 @@ def categorical_column_with_vocabulary_list(
       default_value=default_value, num_oov_buckets=num_oov_buckets)
 
 
-@tf_export(v1=['feature_column.categorical_column_with_identity'])
-def categorical_column_with_identity(key, num_buckets, default_value=None):
+def _categorical_column_with_identity(key, num_buckets, default_value=None):
   """A `_CategoricalColumn` that returns identity values.
 
   Use this when your inputs are integers in the range `[0, num_buckets)`, and
@@ -1616,8 +1450,7 @@ def categorical_column_with_identity(key, num_buckets, default_value=None):
       key=key, num_buckets=num_buckets, default_value=default_value)
 
 
-@tf_export(v1=['feature_column.indicator_column'])
-def indicator_column(categorical_column):
+def _indicator_column(categorical_column):
   """Represents multi-hot representation of given categorical column.
 
   - For DNN model, `indicator_column` can be used to wrap any
@@ -1651,9 +1484,9 @@ def indicator_column(categorical_column):
   return _IndicatorColumn(categorical_column)
 
 
-@tf_export(v1=['feature_column.weighted_categorical_column'])
-def weighted_categorical_column(
-    categorical_column, weight_feature_key, dtype=dtypes.float32):
+def _weighted_categorical_column(categorical_column,
+                                 weight_feature_key,
+                                 dtype=dtypes.float32):
   """Applies weight values to a `_CategoricalColumn`.
 
   Use this when each of your sparse inputs has both an ID and a value. For
@@ -1726,8 +1559,7 @@ def weighted_categorical_column(
       dtype=dtype)
 
 
-@tf_export(v1=['feature_column.crossed_column'])
-def crossed_column(keys, hash_bucket_size, hash_key=None):
+def _crossed_column(keys, hash_bucket_size, hash_key=None):
   """Returns a column for performing crosses of categorical features.
 
   Crossed features will be hashed according to `hash_bucket_size`. Conceptually,
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index e069b96bb9..2c70d66810 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.client import session
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.feature_column import feature_column as fc
+from tensorflow.python.feature_column import feature_column_v2 as fc_new
 from tensorflow.python.feature_column.feature_column import _CategoricalColumn
 from tensorflow.python.feature_column.feature_column import _DenseColumn
 from tensorflow.python.feature_column.feature_column import _FeatureColumn
@@ -185,7 +186,7 @@ class LazyColumnTest(test.TestCase):
 class NumericColumnTest(test.TestCase):
 
   def test_defaults(self):
-    a = fc.numeric_column('aaa')
+    a = fc._numeric_column('aaa')
     self.assertEqual('aaa', a.key)
     self.assertEqual('aaa', a.name)
     self.assertEqual('aaa', a._var_scope_name)
@@ -196,53 +197,53 @@ class NumericColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.numeric_column(key=('aaa',))
+      fc._numeric_column(key=('aaa',))
 
   def test_shape_saved_as_tuple(self):
-    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
+    a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
     self.assertEqual((1, 2), a.shape)
 
   def test_default_value_saved_as_tuple(self):
-    a = fc.numeric_column('aaa', default_value=4.)
+    a = fc._numeric_column('aaa', default_value=4.)
     self.assertEqual((4.,), a.default_value)
-    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
+    a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
     self.assertEqual(((3., 2.),), a.default_value)
 
   def test_shape_and_default_value_compatibility(self):
-    fc.numeric_column('aaa', shape=[2], default_value=[1, 2.])
+    fc._numeric_column('aaa', shape=[2], default_value=[1, 2.])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column('aaa', shape=[2], default_value=[1, 2, 3.])
-    fc.numeric_column(
+      fc._numeric_column('aaa', shape=[2], default_value=[1, 2, 3.])
+    fc._numeric_column(
         'aaa', shape=[3, 2], default_value=[[2, 3], [1, 2], [2, 3.]])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column(
+      fc._numeric_column(
           'aaa', shape=[3, 1], default_value=[[2, 3], [1, 2], [2, 3.]])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column(
+      fc._numeric_column(
           'aaa', shape=[3, 3], default_value=[[2, 3], [1, 2], [2, 3.]])
 
   def test_default_value_type_check(self):
-    fc.numeric_column(
+    fc._numeric_column(
         'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.float32)
-    fc.numeric_column(
+    fc._numeric_column(
         'aaa', shape=[2], default_value=[1, 2], dtype=dtypes.int32)
     with self.assertRaisesRegexp(TypeError, 'must be compatible with dtype'):
-      fc.numeric_column(
+      fc._numeric_column(
           'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.int32)
     with self.assertRaisesRegexp(TypeError,
                                  'default_value must be compatible with dtype'):
-      fc.numeric_column('aaa', default_value=['string'])
+      fc._numeric_column('aaa', default_value=['string'])
 
   def test_shape_must_be_positive_integer(self):
     with self.assertRaisesRegexp(TypeError, 'shape dimensions must be integer'):
-      fc.numeric_column(
+      fc._numeric_column(
           'aaa', shape=[
               1.0,
           ])
 
     with self.assertRaisesRegexp(ValueError,
                                  'shape dimensions must be greater than 0'):
-      fc.numeric_column(
+      fc._numeric_column(
           'aaa', shape=[
               0,
           ])
@@ -250,20 +251,20 @@ class NumericColumnTest(test.TestCase):
   def test_dtype_is_convertible_to_float(self):
     with self.assertRaisesRegexp(ValueError,
                                  'dtype must be convertible to float'):
-      fc.numeric_column('aaa', dtype=dtypes.string)
+      fc._numeric_column('aaa', dtype=dtypes.string)
 
   def test_scalar_default_value_fills_the_shape(self):
-    a = fc.numeric_column('aaa', shape=[2, 3], default_value=2.)
+    a = fc._numeric_column('aaa', shape=[2, 3], default_value=2.)
     self.assertEqual(((2., 2., 2.), (2., 2., 2.)), a.default_value)
 
   def test_parse_spec(self):
-    a = fc.numeric_column('aaa', shape=[2, 3], dtype=dtypes.int32)
+    a = fc._numeric_column('aaa', shape=[2, 3], dtype=dtypes.int32)
     self.assertEqual({
         'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32)
     }, a._parse_example_spec)
 
   def test_parse_example_no_default_value(self):
-    price = fc.numeric_column('price', shape=[2])
+    price = fc._numeric_column('price', shape=[2])
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -278,7 +279,7 @@ class NumericColumnTest(test.TestCase):
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
   def test_parse_example_with_default_value(self):
-    price = fc.numeric_column('price', shape=[2], default_value=11.)
+    price = fc._numeric_column('price', shape=[2], default_value=11.)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -301,14 +302,14 @@ class NumericColumnTest(test.TestCase):
 
   def test_normalizer_fn_must_be_callable(self):
     with self.assertRaisesRegexp(TypeError, 'must be a callable'):
-      fc.numeric_column('price', normalizer_fn='NotACallable')
+      fc._numeric_column('price', normalizer_fn='NotACallable')
 
   def test_normalizer_fn_transform_feature(self):
 
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two)
+    price = fc._numeric_column('price', shape=[2], normalizer_fn=_increment_two)
     output = _transform_features({'price': [[1., 2.], [5., 6.]]}, [price])
     with self.cached_session():
       self.assertAllEqual([[3., 4.], [7., 8.]], output[price].eval())
@@ -318,12 +319,12 @@ class NumericColumnTest(test.TestCase):
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two)
+    price = fc._numeric_column('price', shape=[2], normalizer_fn=_increment_two)
     builder = _LazyBuilder({'price': [[1., 2.], [5., 6.]]})
     self.assertEqual(builder.get(price), price._get_dense_tensor(builder))
 
   def test_sparse_tensor_not_supported(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     builder = _LazyBuilder({
         'price':
             sparse_tensor.SparseTensor(
@@ -333,19 +334,19 @@ class NumericColumnTest(test.TestCase):
       price._transform_feature(builder)
 
   def test_deep_copy(self):
-    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]])
+    a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]])
     a_copy = copy.deepcopy(a)
     self.assertEqual(a_copy.name, 'aaa')
     self.assertEqual(a_copy.shape, (1, 2))
     self.assertEqual(a_copy.default_value, ((3., 2.),))
 
   def test_numpy_default_value(self):
-    a = fc.numeric_column(
+    a = fc._numeric_column(
         'aaa', shape=[1, 2], default_value=np.array([[3., 2.]]))
     self.assertEqual(a.default_value, ((3., 2.),))
 
   def test_linear_model(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc.linear_model(features, [price])
@@ -359,7 +360,7 @@ class NumericColumnTest(test.TestCase):
         self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
   def test_keras_linear_model(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = get_keras_linear_model_predictions(features, [price])
@@ -376,65 +377,65 @@ class NumericColumnTest(test.TestCase):
 class BucketizedColumnTest(test.TestCase):
 
   def test_invalid_source_column_type(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', hash_bucket_size=10)
+    a = fc._categorical_column_with_hash_bucket('aaa', hash_bucket_size=10)
     with self.assertRaisesRegexp(
         ValueError,
         'source_column must be a column generated with numeric_column'):
-      fc.bucketized_column(a, boundaries=[0, 1])
+      fc._bucketized_column(a, boundaries=[0, 1])
 
   def test_invalid_source_column_shape(self):
-    a = fc.numeric_column('aaa', shape=[2, 3])
+    a = fc._numeric_column('aaa', shape=[2, 3])
     with self.assertRaisesRegexp(
         ValueError, 'source_column must be one-dimensional column'):
-      fc.bucketized_column(a, boundaries=[0, 1])
+      fc._bucketized_column(a, boundaries=[0, 1])
 
   def test_invalid_boundaries(self):
-    a = fc.numeric_column('aaa')
+    a = fc._numeric_column('aaa')
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=None)
+      fc._bucketized_column(a, boundaries=None)
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=1.)
+      fc._bucketized_column(a, boundaries=1.)
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=[1, 0])
+      fc._bucketized_column(a, boundaries=[1, 0])
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column(a, boundaries=[1, 1])
+      fc._bucketized_column(a, boundaries=[1, 1])
 
   def test_name(self):
-    a = fc.numeric_column('aaa', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc._numeric_column('aaa', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
     self.assertEqual('aaa_bucketized', b.name)
 
   def test_var_scope_name(self):
-    a = fc.numeric_column('aaa', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc._numeric_column('aaa', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
     self.assertEqual('aaa_bucketized', b._var_scope_name)
 
   def test_parse_spec(self):
-    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc._numeric_column('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
     self.assertEqual({
         'aaa': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32)
     }, b._parse_example_spec)
 
   def test_variable_shape(self):
-    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc._numeric_column('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
     # Column 'aaa` has shape [2] times three buckets -> variable_shape=[2, 3].
     self.assertAllEqual((2, 3), b._variable_shape)
 
   def test_num_buckets(self):
-    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc._numeric_column('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
     # Column 'aaa` has shape [2] times three buckets -> num_buckets=6.
     self.assertEqual(6, b._num_buckets)
 
   def test_parse_example(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 50])
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -449,8 +450,8 @@ class BucketizedColumnTest(test.TestCase):
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
   def test_transform_feature(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformed_tensor = _transform_features({
           'price': [[-1., 1.], [5., 6.]]
@@ -461,8 +462,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_dense_tensor_one_input_value(self):
     """Tests _get_dense_tensor() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[1])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       builder = _LazyBuilder({'price': [[-1.], [1.], [5.], [6.]]})
       with _initialized_session():
@@ -475,8 +476,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_dense_tensor_two_input_values(self):
     """Tests _get_dense_tensor() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       builder = _LazyBuilder({'price': [[-1., 1.], [5., 6.]]})
       with _initialized_session():
@@ -489,8 +490,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_one_input_value(self):
     """Tests _get_sparse_tensors() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[1])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       builder = _LazyBuilder({'price': [[-1.], [1.], [5.], [6.]]})
       with _initialized_session() as sess:
@@ -504,8 +505,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_two_input_values(self):
     """Tests _get_sparse_tensors() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       builder = _LazyBuilder({'price': [[-1., 1.], [5., 6.]]})
       with _initialized_session() as sess:
@@ -520,8 +521,8 @@ class BucketizedColumnTest(test.TestCase):
         self.assertAllEqual([2, 2], id_tensor_value.dense_shape)
 
   def test_sparse_tensor_input_not_supported(self):
-    price = fc.numeric_column('price')
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 1])
+    price = fc._numeric_column('price')
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 1])
     builder = _LazyBuilder({
         'price':
             sparse_tensor.SparseTensor(
@@ -531,8 +532,8 @@ class BucketizedColumnTest(test.TestCase):
       bucketized_price._transform_feature(builder)
 
   def test_deep_copy(self):
-    a = fc.numeric_column('aaa', shape=[2])
-    a_bucketized = fc.bucketized_column(a, boundaries=[0, 1])
+    a = fc._numeric_column('aaa', shape=[2])
+    a_bucketized = fc._bucketized_column(a, boundaries=[0, 1])
     a_bucketized_copy = copy.deepcopy(a_bucketized)
     self.assertEqual(a_bucketized_copy.name, 'aaa_bucketized')
     self.assertAllEqual(a_bucketized_copy._variable_shape, (2, 3))
@@ -540,8 +541,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_linear_model_one_input_value(self):
     """Tests linear_model() for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[1])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       predictions = fc.linear_model(features, [bucketized_price])
@@ -568,8 +569,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1., 1.], [5., 6.]]}
       predictions = fc.linear_model(features, [bucketized_price])
@@ -597,8 +598,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_keras_linear_model_one_input_value(self):
     """Tests _LinearModel for input with shape=[1]."""
-    price = fc.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[1])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       predictions = get_keras_linear_model_predictions(features,
@@ -626,8 +627,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_keras_linear_model_two_input_values(self):
     """Tests _LinearModel for input with shape=[2]."""
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1., 1.], [5., 6.]]}
       predictions = get_keras_linear_model_predictions(features,
@@ -658,7 +659,7 @@ class BucketizedColumnTest(test.TestCase):
 class HashedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10)
+    a = fc._categorical_column_with_hash_bucket('aaa', 10)
     self.assertEqual('aaa', a.name)
     self.assertEqual('aaa', a._var_scope_name)
     self.assertEqual('aaa', a.key)
@@ -667,25 +668,25 @@ class HashedCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_hash_bucket(('key',), 10)
+      fc._categorical_column_with_hash_bucket(('key',), 10)
 
   def test_bucket_size_should_be_given(self):
     with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be set.'):
-      fc.categorical_column_with_hash_bucket('aaa', None)
+      fc._categorical_column_with_hash_bucket('aaa', None)
 
   def test_bucket_size_should_be_positive(self):
     with self.assertRaisesRegexp(ValueError,
                                  'hash_bucket_size must be at least 1'):
-      fc.categorical_column_with_hash_bucket('aaa', 0)
+      fc._categorical_column_with_hash_bucket('aaa', 0)
 
   def test_dtype_should_be_string_or_integer(self):
-    fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string)
-    fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
+    fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string)
+    fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32)
+      fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_hash_bucket('aaa', 10)
+    original = fc._categorical_column_with_hash_bucket('aaa', 10)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(10, column.hash_bucket_size)
@@ -693,19 +694,19 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertEqual(dtypes.string, column.dtype)
 
   def test_parse_spec_string(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10)
+    a = fc._categorical_column_with_hash_bucket('aaa', 10)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.string)
     }, a._parse_example_spec)
 
   def test_parse_spec_int(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
+    a = fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, a._parse_example_spec)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_hash_bucket('aaa', 10)
+    a = fc._categorical_column_with_hash_bucket('aaa', 10)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -726,7 +727,7 @@ class HashedCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_strings_should_be_hashed(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     wire_tensor = sparse_tensor.SparseTensor(
         values=['omar', 'stringer', 'marlo'],
         indices=[[0, 0], [1, 0], [1, 1]],
@@ -743,11 +744,11 @@ class HashedCategoricalColumnTest(test.TestCase):
                           output.dense_shape.eval())
 
   def test_tensor_dtype_should_be_string_or_integer(self):
-    string_fc = fc.categorical_column_with_hash_bucket(
+    string_fc = fc._categorical_column_with_hash_bucket(
         'a_string', 10, dtype=dtypes.string)
-    int_fc = fc.categorical_column_with_hash_bucket(
+    int_fc = fc._categorical_column_with_hash_bucket(
         'a_int', 10, dtype=dtypes.int32)
-    float_fc = fc.categorical_column_with_hash_bucket(
+    float_fc = fc._categorical_column_with_hash_bucket(
         'a_float', 10, dtype=dtypes.string)
     int_tensor = sparse_tensor.SparseTensor(
         values=[101],
@@ -772,7 +773,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       builder.get(float_fc)
 
   def test_dtype_should_match_with_tensor(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket(
+    hashed_sparse = fc._categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -781,7 +782,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       builder.get(hashed_sparse)
 
   def test_ints_should_be_hashed(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket(
+    hashed_sparse = fc._categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=[101, 201, 301],
@@ -795,7 +796,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, output.values.eval())
 
   def test_int32_64_is_compatible(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket(
+    hashed_sparse = fc._categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=constant_op.constant([101, 201, 301], dtype=dtypes.int32),
@@ -809,7 +810,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, output.values.eval())
 
   def test_get_sparse_tensors(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     builder = _LazyBuilder({
         'wire':
             sparse_tensor.SparseTensor(
@@ -822,7 +823,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     self.assertEqual(builder.get(hashed_sparse), id_weight_pair.id_tensor)
 
   def test_get_sparse_tensors_weight_collections(self):
-    column = fc.categorical_column_with_hash_bucket('aaa', 10)
+    column = fc._categorical_column_with_hash_bucket('aaa', 10)
     inputs = sparse_tensor.SparseTensor(
         values=['omar', 'stringer', 'marlo'],
         indices=[[0, 0], [1, 0], [1, 1]],
@@ -837,14 +838,14 @@ class HashedCategoricalColumnTest(test.TestCase):
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
   def test_get_sparse_tensors_dense_input(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     builder = _LazyBuilder({'wire': (('omar', ''), ('stringer', 'marlo'))})
     id_weight_pair = hashed_sparse._get_sparse_tensors(builder)
     self.assertIsNone(id_weight_pair.weight_tensor)
     self.assertEqual(builder.get(hashed_sparse), id_weight_pair.id_tensor)
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
+    wire_column = fc._categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column._num_buckets)
     with ops.Graph().as_default():
       predictions = fc.linear_model({
@@ -866,7 +867,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
-    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
+    wire_column = fc._categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column._num_buckets)
     with ops.Graph().as_default():
       predictions = get_keras_linear_model_predictions({
@@ -894,100 +895,100 @@ class CrossedColumnTest(test.TestCase):
   def test_keys_empty(self):
     with self.assertRaisesRegexp(
         ValueError, 'keys must be a list with length > 1'):
-      fc.crossed_column([], 10)
+      fc._crossed_column([], 10)
 
   def test_keys_length_one(self):
     with self.assertRaisesRegexp(
         ValueError, 'keys must be a list with length > 1'):
-      fc.crossed_column(['a'], 10)
+      fc._crossed_column(['a'], 10)
 
   def test_key_type_unsupported(self):
     with self.assertRaisesRegexp(ValueError, 'Unsupported key type'):
-      fc.crossed_column(['a', fc.numeric_column('c')], 10)
+      fc._crossed_column(['a', fc._numeric_column('c')], 10)
 
     with self.assertRaisesRegexp(
         ValueError, 'categorical_column_with_hash_bucket is not supported'):
-      fc.crossed_column(
-          ['a', fc.categorical_column_with_hash_bucket('c', 10)], 10)
+      fc._crossed_column(
+          ['a', fc._categorical_column_with_hash_bucket('c', 10)], 10)
 
   def test_hash_bucket_size_negative(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column(['a', 'c'], -1)
+      fc._crossed_column(['a', 'c'], -1)
 
   def test_hash_bucket_size_zero(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column(['a', 'c'], 0)
+      fc._crossed_column(['a', 'c'], 0)
 
   def test_hash_bucket_size_none(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column(['a', 'c'], None)
+      fc._crossed_column(['a', 'c'], None)
 
   def test_name(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    a = fc._numeric_column('a', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc._crossed_column(['d1', 'd2'], 10)
 
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 10)
+    crossed2 = fc._crossed_column([b, 'c', crossed1], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_name_ordered_alphabetically(self):
     """Tests that the name does not depend on the order of given columns."""
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    a = fc._numeric_column('a', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc._crossed_column(['d1', 'd2'], 10)
 
-    crossed2 = fc.crossed_column([crossed1, 'c', b], 10)
+    crossed2 = fc._crossed_column([crossed1, 'c', b], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_name_leaf_keys_ordered_alphabetically(self):
     """Tests that the name does not depend on the order of given columns."""
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d2', 'c'], 10)
+    a = fc._numeric_column('a', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc._crossed_column(['d2', 'c'], 10)
 
-    crossed2 = fc.crossed_column([crossed1, 'd1', b], 10)
+    crossed2 = fc._crossed_column([crossed1, 'd1', b], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_var_scope_name(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    a = fc._numeric_column('a', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc._crossed_column(['d1', 'd2'], 10)
 
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 10)
+    crossed2 = fc._crossed_column([b, 'c', crossed1], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2._var_scope_name)
 
   def test_parse_spec(self):
-    a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed = fc.crossed_column([b, 'c'], 10)
+    a = fc._numeric_column('a', shape=[2], dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed = fc._crossed_column([b, 'c'], 10)
     self.assertEqual({
         'a': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32),
         'c': parsing_ops.VarLenFeature(dtypes.string),
     }, crossed._parse_example_spec)
 
   def test_num_buckets(self):
-    a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed = fc.crossed_column([b, 'c'], 15)
+    a = fc._numeric_column('a', shape=[2], dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed = fc._crossed_column([b, 'c'], 15)
     self.assertEqual(15, crossed._num_buckets)
 
   def test_deep_copy(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
+    a = fc._numeric_column('a', dtype=dtypes.int32)
+    b = fc._bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc._crossed_column(['d1', 'd2'], 10)
+    crossed2 = fc._crossed_column([b, 'c', crossed1], 15, hash_key=5)
     crossed2_copy = copy.deepcopy(crossed2)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2_copy.name,)
     self.assertEqual(15, crossed2_copy.hash_bucket_size)
     self.assertEqual(5, crossed2_copy.hash_key)
 
   def test_parse_example(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
-    price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], 10)
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 50])
+    price_cross_wire = fc._crossed_column([bucketized_price, 'wire'], 10)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -1011,11 +1012,11 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval())
 
   def test_transform_feature(self):
-    price = fc.numeric_column('price', shape=[2])
-    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
+    price = fc._numeric_column('price', shape=[2])
+    bucketized_price = fc._bucketized_column(price, boundaries=[0, 50])
     hash_bucket_size = 10
-    price_cross_wire = fc.crossed_column(
-        [bucketized_price, 'wire'], hash_bucket_size)
+    price_cross_wire = fc._crossed_column([bucketized_price, 'wire'],
+                                          hash_bucket_size)
     features = {
         'price': constant_op.constant([[1., 2.], [5., 6.]]),
         'wire': sparse_tensor.SparseTensor(
@@ -1034,10 +1035,10 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([2, 4], output_val.dense_shape)
 
   def test_get_sparse_tensors(self):
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
-    crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
+    a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc._bucketized_column(a, boundaries=(0, 1))
+    crossed1 = fc._crossed_column(['d1', 'd2'], 10)
+    crossed2 = fc._crossed_column([b, 'c', crossed1], 15, hash_key=5)
     with ops.Graph().as_default():
       builder = _LazyBuilder({
           'a':
@@ -1075,9 +1076,9 @@ class CrossedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_simple(self):
     """Same as test_get_sparse_tensors, but with simpler values."""
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc._bucketized_column(a, boundaries=(0, 1))
+    crossed = fc._crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       builder = _LazyBuilder({
           'a':
@@ -1105,9 +1106,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc._bucketized_column(a, boundaries=(0, 1))
+    crossed = fc._crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       predictions = fc.linear_model({
           'a': constant_op.constant(((-1., .5), (.5, 1.))),
@@ -1161,7 +1162,7 @@ class CrossedColumnTest(test.TestCase):
             id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
 
     t = _TestColumnWithWeights()
-    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
+    crossed = fc._crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
           ValueError,
@@ -1186,9 +1187,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column(a, boundaries=(0, 1))
-    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc._bucketized_column(a, boundaries=(0, 1))
+    crossed = fc._crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       predictions = get_keras_linear_model_predictions({
           'a':
@@ -1248,7 +1249,7 @@ class CrossedColumnTest(test.TestCase):
             id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
 
     t = _TestColumnWithWeights()
-    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
+    crossed = fc._crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
           ValueError,
@@ -1337,18 +1338,18 @@ class LinearModelTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc.linear_model(
-          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
+          features={'a': [[0]]}, feature_columns={'a': fc._numeric_column('a')})
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
       fc.linear_model(
           features={'a': [[0]]},
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])
+          feature_columns=[fc._numeric_column('a'),
+                           fc._numeric_column('a')])
 
   def test_dense_bias(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc.linear_model(features, [price])
@@ -1361,7 +1362,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1380,8 +1381,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    price = fc.numeric_column('price')
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1452,7 +1453,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc.linear_model(features, [price], units=3)
@@ -1467,7 +1468,7 @@ class LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1489,7 +1490,7 @@ class LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = fc.linear_model(features, [price])
@@ -1500,7 +1501,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = array_ops.sparse_placeholder(dtypes.string)
       wire_value = sparse_tensor.SparseTensorValue(
@@ -1521,7 +1522,7 @@ class LinearModelTest(test.TestCase):
             predictions.eval(feed_dict={wire_tensor: wire_value}))
 
   def test_sparse_combiner(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1538,8 +1539,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_sparse_combiner_with_negative_weights(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights')
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast_weights = fc._weighted_categorical_column(wire_cast, 'weights')
 
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
@@ -1560,7 +1561,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = fc.linear_model(features, [price], units=3)
@@ -1575,7 +1576,7 @@ class LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -1584,7 +1585,7 @@ class LinearModelTest(test.TestCase):
         fc.linear_model(features, [price])
 
   def test_dense_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc._numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       predictions = fc.linear_model(features, [price])
@@ -1598,8 +1599,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
@@ -1620,8 +1621,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_fills_cols_to_vars(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       cols_to_vars = {}
@@ -1634,8 +1635,8 @@ class LinearModelTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[price2], [price2_var])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2', shape=3)
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2', shape=3)
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [6., 7.]],
@@ -1660,13 +1661,13 @@ class LinearModelTest(test.TestCase):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
     # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
     # creates a Variable.
-    apple_numeric_column = fc.numeric_column('apple_numeric_column')
-    banana_dense_feature = fc.numeric_column('banana_dense_feature')
-    banana_dense_feature_bucketized = fc.bucketized_column(
+    apple_numeric_column = fc._numeric_column('apple_numeric_column')
+    banana_dense_feature = fc._numeric_column('banana_dense_feature')
+    banana_dense_feature_bucketized = fc._bucketized_column(
         banana_dense_feature, boundaries=[0.])
-    cherry_sparse_column = fc.categorical_column_with_hash_bucket(
+    cherry_sparse_column = fc._categorical_column_with_hash_bucket(
         'cherry_sparse_feature', hash_bucket_size=5)
-    dragonfruit_embedding_column = fc.embedding_column(
+    dragonfruit_embedding_column = fc._embedding_column(
         cherry_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -1691,7 +1692,7 @@ class LinearModelTest(test.TestCase):
       self.assertItemsEqual(input_layer_inputs, output_tensors)
 
   def test_dense_collection(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc.linear_model(features, [price], weight_collections=['my-vars'])
@@ -1702,7 +1703,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(price_var, my_vars)
 
   def test_sparse_collection(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1716,7 +1717,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, my_vars)
 
   def test_dense_trainable_default(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc.linear_model(features, [price])
@@ -1727,7 +1728,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(price_var, trainable_vars)
 
   def test_sparse_trainable_default(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1740,7 +1741,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, trainable_vars)
 
   def test_dense_trainable_false(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc.linear_model(features, [price], trainable=False)
@@ -1748,7 +1749,7 @@ class LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_sparse_trainable_false(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1758,9 +1759,9 @@ class LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price_a = fc._numeric_column('price_a')
+    price_b = fc._numeric_column('price_b')
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       features = {
           'price_a': [[1.]],
@@ -1794,8 +1795,8 @@ class LinearModelTest(test.TestCase):
       self.assertIn('wire_cast', my_vars[2].name)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -1807,9 +1808,9 @@ class LinearModelTest(test.TestCase):
       fc.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
+    price3 = fc._numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -1822,8 +1823,8 @@ class LinearModelTest(test.TestCase):
         fc.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -1837,8 +1838,8 @@ class LinearModelTest(test.TestCase):
               predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -1854,9 +1855,14 @@ class LinearModelTest(test.TestCase):
             })
 
   def test_with_1d_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    price = fc._numeric_column('price')
+    price_buckets = fc._bucketized_column(
+        price, boundaries=[
+            0.,
+            10.,
+            100.,
+        ])
+    body_style = fc._categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -1884,11 +1890,16 @@ class LinearModelTest(test.TestCase):
                           self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    price = fc._numeric_column('price')
+    price_buckets = fc._bucketized_column(
+        price, boundaries=[
+            0.,
+            10.,
+            100.,
+        ])
+    body_style = fc._categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc._categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -1926,7 +1937,7 @@ class LinearModelTest(test.TestCase):
                               }))
 
   def test_with_rank_0_feature(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -1947,7 +1958,7 @@ class LinearModelTest(test.TestCase):
         sess.run(net, feed_dict={features['price']: np.array(1)})
 
   def test_multiple_linear_models(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features1 = {'price': [[1.], [5.]]}
       features2 = {'price': [[2.], [10.]]}
@@ -2004,18 +2015,18 @@ class _LinearModelTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc.linear_model(
-          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
+          features={'a': [[0]]}, feature_columns={'a': fc._numeric_column('a')})
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
       get_keras_linear_model_predictions(
           features={'a': [[0]]},
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])
+          feature_columns=[fc._numeric_column('a'),
+                           fc._numeric_column('a')])
 
   def test_dense_bias(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = get_keras_linear_model_predictions(features, [price])
@@ -2028,7 +2039,7 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2047,8 +2058,8 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
-    price = fc.numeric_column('price')
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2126,7 +2137,7 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = get_keras_linear_model_predictions(
@@ -2142,7 +2153,7 @@ class _LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2165,7 +2176,7 @@ class _LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = get_keras_linear_model_predictions(features, [price])
@@ -2176,7 +2187,7 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = array_ops.sparse_placeholder(dtypes.string)
       wire_value = sparse_tensor.SparseTensorValue(
@@ -2197,7 +2208,7 @@ class _LinearModelTest(test.TestCase):
             predictions.eval(feed_dict={wire_tensor: wire_value}))
 
   def test_sparse_combiner(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2214,7 +2225,7 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = get_keras_linear_model_predictions(
@@ -2230,7 +2241,7 @@ class _LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -2239,7 +2250,7 @@ class _LinearModelTest(test.TestCase):
         get_keras_linear_model_predictions(features, [price])
 
   def test_dense_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc._numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       predictions = get_keras_linear_model_predictions(features, [price])
@@ -2253,8 +2264,8 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       predictions = get_keras_linear_model_predictions(features,
@@ -2273,8 +2284,8 @@ class _LinearModelTest(test.TestCase):
         self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_fills_cols_to_vars(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       cols_to_vars = {}
@@ -2288,8 +2299,8 @@ class _LinearModelTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[price2], [price2_var])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2', shape=3)
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2', shape=3)
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [6., 7.]],
@@ -2312,7 +2323,7 @@ class _LinearModelTest(test.TestCase):
         self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval())
 
   def test_dense_collection(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       get_keras_linear_model_predictions(
@@ -2324,7 +2335,7 @@ class _LinearModelTest(test.TestCase):
       self.assertIn(price_var, my_vars)
 
   def test_sparse_collection(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2338,7 +2349,7 @@ class _LinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, my_vars)
 
   def test_dense_trainable_default(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       get_keras_linear_model_predictions(features, [price])
@@ -2349,7 +2360,7 @@ class _LinearModelTest(test.TestCase):
       self.assertIn(price_var, trainable_vars)
 
   def test_sparse_trainable_default(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2362,7 +2373,7 @@ class _LinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, trainable_vars)
 
   def test_dense_trainable_false(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       get_keras_linear_model_predictions(features, [price], trainable=False)
@@ -2370,7 +2381,7 @@ class _LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_sparse_trainable_false(self):
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2380,9 +2391,9 @@ class _LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
-    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price_a = fc._numeric_column('price_a')
+    price_b = fc._numeric_column('price_b')
+    wire_cast = fc._categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       features = {
           'price_a': [[1.]],
@@ -2416,8 +2427,8 @@ class _LinearModelTest(test.TestCase):
       self.assertIn('wire_cast', my_vars[2].name)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -2429,9 +2440,9 @@ class _LinearModelTest(test.TestCase):
       get_keras_linear_model_predictions(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
+    price3 = fc._numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2444,8 +2455,8 @@ class _LinearModelTest(test.TestCase):
         get_keras_linear_model_predictions(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2460,8 +2471,8 @@ class _LinearModelTest(test.TestCase):
               predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -2478,14 +2489,14 @@ class _LinearModelTest(test.TestCase):
             })
 
   def test_with_1d_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc._numeric_column('price')
+    price_buckets = fc._bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc._categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2519,16 +2530,16 @@ class _LinearModelTest(test.TestCase):
                           self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
-    price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(
+    price = fc._numeric_column('price')
+    price_buckets = fc._bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc._categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc._categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2565,7 +2576,7 @@ class _LinearModelTest(test.TestCase):
                               }))
 
   def test_with_rank_0_feature(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -2591,7 +2602,7 @@ class InputLayerTest(test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def test_retrieving_input(self):
     features = {'a': [0.]}
-    input_layer = InputLayer(fc.numeric_column('a'))
+    input_layer = InputLayer(fc._numeric_column('a'))
     inputs = self.evaluate(input_layer(features))
     self.assertAllClose([[0.]], inputs)
 
@@ -2603,8 +2614,8 @@ class InputLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity(key='a',
-                                                               num_buckets=3)
+      categorical_column = fc._categorical_column_with_identity(
+          key='a', num_buckets=3)
       embedding_dimension = 2
       def _embedding_column_initializer(shape, dtype, partition_info):
         del shape  # unused
@@ -2615,7 +2626,8 @@ class InputLayerTest(test.TestCase):
             (0, 1),  # id 1
             (1, 1))  # id 2
         return embedding_values
-      embedding_column = fc.embedding_column(
+
+      embedding_column = fc._embedding_column(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -2646,8 +2658,8 @@ class InputLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity(key='a',
-                                                               num_buckets=3)
+      categorical_column = fc._categorical_column_with_identity(
+          key='a', num_buckets=3)
       embedding_dimension = 2
 
       def _embedding_column_initializer(shape, dtype, partition_info):
@@ -2660,7 +2672,7 @@ class InputLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column(
+      embedding_column = fc._embedding_column(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -2697,26 +2709,26 @@ class FunctionalInputLayerTest(test.TestCase):
       fc.input_layer(
           features={'a': [[0]]},
           feature_columns=[
-              fc.categorical_column_with_hash_bucket('wire_cast', 4)
+              fc._categorical_column_with_hash_bucket('wire_cast', 4)
           ])
 
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc.input_layer(
-          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
+          features={'a': [[0]]}, feature_columns={'a': fc._numeric_column('a')})
 
   def test_bare_column(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
-      net = fc.input_layer(features, fc.numeric_column('a'))
+      net = fc.input_layer(features, fc._numeric_column('a'))
       with _initialized_session():
         self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
-      columns = (fc.numeric_column(key) for key in features)
+      columns = (fc._numeric_column(key) for key in features)
       net = fc.input_layer(features, columns)
       with _initialized_session():
         self.assertAllClose([[0., 1.]], self.evaluate(net))
@@ -2726,11 +2738,11 @@ class FunctionalInputLayerTest(test.TestCase):
         ValueError, 'Duplicate feature column name found for columns'):
       fc.input_layer(
           features={'a': [[0]]},
-          feature_columns=[fc.numeric_column('a'),
-                           fc.numeric_column('a')])
+          feature_columns=[fc._numeric_column('a'),
+                           fc._numeric_column('a')])
 
   def test_one_column(self):
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       net = fc.input_layer(features, [price])
@@ -2738,7 +2750,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc.input_layer(features, [price])
@@ -2746,7 +2758,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column('price', shape=2)
+    price = fc._numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -2755,7 +2767,7 @@ class FunctionalInputLayerTest(test.TestCase):
         fc.input_layer(features, [price])
 
   def test_reshaping(self):
-    price = fc.numeric_column('price', shape=[1, 2])
+    price = fc._numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc.input_layer(features, [price])
@@ -2763,8 +2775,8 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1', shape=2)
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
@@ -2778,13 +2790,13 @@ class FunctionalInputLayerTest(test.TestCase):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
     # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
     # creates a Variable.
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc._numeric_column('price1')
+    dense_feature = fc._numeric_column('dense_feature')
+    dense_feature_bucketized = fc._bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc._categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc._embedding_column(
         some_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -2808,19 +2820,19 @@ class FunctionalInputLayerTest(test.TestCase):
     # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The
     # EmbeddingColumn creates a Variable and the two SharedEmbeddingColumns
     # shared one variable.
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc._numeric_column('price1')
+    dense_feature = fc._numeric_column('dense_feature')
+    dense_feature_bucketized = fc._bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc._categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc._embedding_column(
         some_sparse_column, dimension=10)
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
-    shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns(
+    shared_embedding_a, shared_embedding_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
     with ops.Graph().as_default():
       features = {
@@ -2860,13 +2872,13 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[shared_embedding_a][0].shape, [3, 2])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column('price1')
-    dense_feature = fc.numeric_column('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column(
+    price1 = fc._numeric_column('price1')
+    dense_feature = fc._numeric_column('dense_feature')
+    dense_feature_bucketized = fc._bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc._categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc._embedding_column(
         some_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -2893,8 +2905,8 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10])
 
   def test_column_order(self):
-    price_a = fc.numeric_column('price_a')
-    price_b = fc.numeric_column('price_b')
+    price_a = fc._numeric_column('price_a')
+    price_b = fc._numeric_column('price_b')
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
@@ -2907,7 +2919,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
-    animal = fc.categorical_column_with_identity('animal', num_buckets=4)
+    animal = fc._categorical_column_with_identity('animal', num_buckets=4)
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -2918,8 +2930,8 @@ class FunctionalInputLayerTest(test.TestCase):
         fc.input_layer(features, [animal])
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -2931,9 +2943,9 @@ class FunctionalInputLayerTest(test.TestCase):
         fc.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
-    price3 = fc.numeric_column('price3')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
+    price3 = fc._numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2946,8 +2958,8 @@ class FunctionalInputLayerTest(test.TestCase):
         fc.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2960,8 +2972,8 @@ class FunctionalInputLayerTest(test.TestCase):
           sess.run(net, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column('price1')
-    price2 = fc.numeric_column('price2')
+    price1 = fc._numeric_column('price1')
+    price2 = fc._numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -2977,9 +2989,9 @@ class FunctionalInputLayerTest(test.TestCase):
             })
 
   def test_multiple_layers_with_same_embedding_column(self):
-    some_sparse_column = fc.categorical_column_with_hash_bucket(
+    some_sparse_column = fc._categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column(
+    some_embedding_column = fc._embedding_column(
         some_sparse_column, dimension=10)
 
     with ops.Graph().as_default():
@@ -3001,12 +3013,12 @@ class FunctionalInputLayerTest(test.TestCase):
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
   def test_multiple_layers_with_same_shared_embedding_column(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
-    embedding_column_b, embedding_column_a = fc.shared_embedding_columns(
+    embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension)
 
@@ -3034,12 +3046,12 @@ class FunctionalInputLayerTest(test.TestCase):
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
   def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
-    embedding_column_b, embedding_column_a = fc.shared_embedding_columns(
+    embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension)
     all_cols = [embedding_column_a, embedding_column_b]
@@ -3095,18 +3107,18 @@ class FunctionalInputLayerTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in input_layer
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
 
     # one_hot_body_style has 3 dims in input_layer.
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc._categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column(body_style)
+    one_hot_body_style = fc._indicator_column(body_style)
 
     # embedded_body_style has 5 dims in input_layer.
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc._categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column(country, dimension=5,
-                                           initializer=_initializer)
+    embedded_country = fc._embedding_column(
+        country, dimension=5, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
     features = {
@@ -3145,17 +3157,17 @@ class FunctionalInputLayerTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in input_layer
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
 
     # one_hot_body_style has 3 dims in input_layer.
-    body_style = fc.categorical_column_with_vocabulary_list(
+    body_style = fc._categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column(body_style)
+    one_hot_body_style = fc._indicator_column(body_style)
 
     # embedded_body_style has 5 dims in input_layer.
-    country = fc.categorical_column_with_vocabulary_list(
+    country = fc._categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column(
+    embedded_country = fc._embedding_column(
         country, dimension=2, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -3195,7 +3207,7 @@ class FunctionalInputLayerTest(test.TestCase):
 
   def test_with_rank_0_feature(self):
     # price has 1 dimension in input_layer
-    price = fc.numeric_column('price')
+    price = fc._numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -3324,7 +3336,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     self._wire_vocabulary_size = 3
 
   def test_defaults(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column._var_scope_name)
@@ -3336,22 +3348,28 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_vocabulary_file(
+      fc._categorical_column_with_vocabulary_file(
           key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3)
 
   def test_all_constructor_args(self):
-    column = fc.categorical_column_with_vocabulary_file(
-        key='aaa', vocabulary_file='path_to_file', vocabulary_size=3,
-        num_oov_buckets=4, dtype=dtypes.int32)
+    column = fc._categorical_column_with_vocabulary_file(
+        key='aaa',
+        vocabulary_file='path_to_file',
+        vocabulary_size=3,
+        num_oov_buckets=4,
+        dtype=dtypes.int32)
     self.assertEqual(7, column._num_buckets)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, column._parse_example_spec)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_vocabulary_file(
-        key='aaa', vocabulary_file='path_to_file', vocabulary_size=3,
-        num_oov_buckets=4, dtype=dtypes.int32)
+    original = fc._categorical_column_with_vocabulary_file(
+        key='aaa',
+        vocabulary_file='path_to_file',
+        vocabulary_size=3,
+        num_oov_buckets=4,
+        dtype=dtypes.int32)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(7, column._num_buckets)
@@ -3361,16 +3379,16 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_vocabulary_file_none(self):
     with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'):
-      fc.categorical_column_with_vocabulary_file(
+      fc._categorical_column_with_vocabulary_file(
           key='aaa', vocabulary_file=None, vocabulary_size=3)
 
   def test_vocabulary_file_empty_string(self):
     with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'):
-      fc.categorical_column_with_vocabulary_file(
+      fc._categorical_column_with_vocabulary_file(
           key='aaa', vocabulary_file='', vocabulary_size=3)
 
   def test_invalid_vocabulary_file(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -3383,16 +3401,18 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_vocabulary_size(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file=self._wire_vocabulary_file_name,
+      fc._categorical_column_with_vocabulary_file(
+          key='aaa',
+          vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=-1)
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file=self._wire_vocabulary_file_name,
+      fc._categorical_column_with_vocabulary_file(
+          key='aaa',
+          vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=0)
 
   def test_too_large_vocabulary_size(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size + 1)
@@ -3407,20 +3427,24 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file='path', vocabulary_size=3,
+      fc._categorical_column_with_vocabulary_file(
+          key='aaa',
+          vocabulary_file='path',
+          vocabulary_size=3,
           num_oov_buckets=-1)
 
   def test_invalid_dtype(self):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_file(
-          key='aaa', vocabulary_file='path', vocabulary_size=3,
+      fc._categorical_column_with_vocabulary_file(
+          key='aaa',
+          vocabulary_file='path',
+          vocabulary_size=3,
           dtype=dtypes.float64)
 
   def test_invalid_buckets_and_default_value(self):
     with self.assertRaisesRegexp(
         ValueError, 'both num_oov_buckets and default_value'):
-      fc.categorical_column_with_vocabulary_file(
+      fc._categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=self._wire_vocabulary_size,
@@ -3428,7 +3452,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           default_value=2)
 
   def test_invalid_input_dtype_int32(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -3441,7 +3465,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
   def test_invalid_input_dtype_string(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -3454,7 +3478,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_file(
+    a = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -3476,7 +3500,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -3496,7 +3520,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_none_vocabulary_size(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file=self._wire_vocabulary_file_name)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -3514,7 +3538,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
                                   id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -3532,7 +3556,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_weight_collections(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -3550,7 +3574,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -3569,7 +3593,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -3590,7 +3614,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -3614,7 +3638,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     # 'marlo' is the last entry in our vocabulary file, so be setting
     # `vocabulary_size` to 1 less than number of entries in file, we take
     # 'marlo' out of the vocabulary.
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size - 1)
@@ -3634,7 +3658,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -3656,7 +3680,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -3677,7 +3701,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_file(
+    column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -3699,7 +3723,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_file(
+    wire_column = fc._categorical_column_with_vocabulary_file(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -3725,7 +3749,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_file(
+    wire_column = fc._categorical_column_with_vocabulary_file(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -3755,7 +3779,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_defaults_string(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -3767,11 +3791,11 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key=('aaa',), vocabulary_list=('omar', 'stringer', 'marlo'))
 
   def test_defaults_int(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36))
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -3782,8 +3806,10 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     }, column._parse_example_spec)
 
   def test_all_constructor_args(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32,
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa',
+        vocabulary_list=(12, 24, 36),
+        dtype=dtypes.int32,
         default_value=-99)
     self.assertEqual(3, column._num_buckets)
     self.assertEqual({
@@ -3791,7 +3817,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     }, column._parse_example_spec)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_vocabulary_list(
+    original = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
@@ -3802,65 +3828,65 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype(self):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_list(
-          key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'),
+      fc._categorical_column_with_vocabulary_list(
+          key='aaa',
+          vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.float32)
 
   def test_invalid_mapping_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12., 24., 36.))
 
   def test_mismatched_int_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'dtype.*and vocabulary dtype.*do not match'):
-      fc.categorical_column_with_vocabulary_list(
-          key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'),
+      fc._categorical_column_with_vocabulary_list(
+          key='aaa',
+          vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.int32)
 
   def test_mismatched_string_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'dtype.*and vocabulary dtype.*do not match'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.string)
 
   def test_none_mapping(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary_list.*must be non-empty'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=None)
 
   def test_empty_mapping(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary_list.*must be non-empty'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=tuple([]))
 
   def test_duplicate_mapping(self):
     with self.assertRaisesRegexp(ValueError, 'Duplicate keys'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12, 24, 12))
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
-      fc.categorical_column_with_vocabulary_list(
-          key='aaa', vocabulary_list=(12, 24, 36),
-          num_oov_buckets=-1)
+      fc._categorical_column_with_vocabulary_list(
+          key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=-1)
 
   def test_invalid_buckets_and_default_value(self):
     with self.assertRaisesRegexp(
         ValueError, 'both num_oov_buckets and default_value'):
-      fc.categorical_column_with_vocabulary_list(
+      fc._categorical_column_with_vocabulary_list(
           key='aaa',
           vocabulary_list=(12, 24, 36),
           num_oov_buckets=100,
           default_value=2)
 
   def test_invalid_input_dtype_int32(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(12, 24, 36),
@@ -3869,9 +3895,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
   def test_invalid_input_dtype_string(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=(12, 24, 36))
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa', vocabulary_list=(12, 24, 36))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('omar', 'stringer', 'marlo'),
@@ -3880,7 +3905,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
   def test_parse_example_string(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -3902,7 +3927,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_parse_example_int(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(11, 21, 31))
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -3924,9 +3949,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('marlo', 'skywalker', 'omar'),
@@ -3943,9 +3967,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('marlo', 'skywalker', 'omar'),
@@ -3960,9 +3983,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_weight_collections(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensor(
         values=['omar', 'stringer', 'marlo'],
         indices=[[0, 0], [1, 0], [1, 1]],
@@ -3977,9 +3999,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_vocabulary_list(
-        key='aaa',
-        vocabulary_list=('omar', 'stringer', 'marlo'))
+    column = fc._categorical_column_with_vocabulary_list(
+        key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     id_weight_pair = column._get_sparse_tensors(
         _LazyBuilder({
             'aaa': (('marlo', ''), ('skywalker', 'omar'))
@@ -3995,7 +4016,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         default_value=2)
@@ -4015,7 +4036,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=100)
@@ -4035,7 +4056,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32)
@@ -4056,7 +4077,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32,
@@ -4078,7 +4099,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_list(
+    column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32,
@@ -4099,7 +4120,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_list(
+    wire_column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -4124,7 +4145,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_list(
+    wire_column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -4153,7 +4174,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_constructor(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
     self.assertEqual('aaa', column._var_scope_name)
@@ -4164,10 +4185,10 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_identity(key=('aaa',), num_buckets=3)
+      fc._categorical_column_with_identity(key=('aaa',), num_buckets=3)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    original = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(3, column._num_buckets)
@@ -4177,24 +4198,24 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_invalid_num_buckets_zero(self):
     with self.assertRaisesRegexp(ValueError, 'num_buckets 0 < 1'):
-      fc.categorical_column_with_identity(key='aaa', num_buckets=0)
+      fc._categorical_column_with_identity(key='aaa', num_buckets=0)
 
   def test_invalid_num_buckets_negative(self):
     with self.assertRaisesRegexp(ValueError, 'num_buckets -1 < 1'):
-      fc.categorical_column_with_identity(key='aaa', num_buckets=-1)
+      fc._categorical_column_with_identity(key='aaa', num_buckets=-1)
 
   def test_invalid_default_value_too_small(self):
     with self.assertRaisesRegexp(ValueError, 'default_value -1 not in range'):
-      fc.categorical_column_with_identity(
+      fc._categorical_column_with_identity(
           key='aaa', num_buckets=3, default_value=-1)
 
   def test_invalid_default_value_too_big(self):
     with self.assertRaisesRegexp(ValueError, 'default_value 3 not in range'):
-      fc.categorical_column_with_identity(
+      fc._categorical_column_with_identity(
           key='aaa', num_buckets=3, default_value=3)
 
   def test_invalid_input_dtype(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('omar', 'stringer', 'marlo'),
@@ -4203,7 +4224,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_identity(key='aaa', num_buckets=30)
+    a = fc._categorical_column_with_identity(key='aaa', num_buckets=30)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -4224,7 +4245,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
@@ -4241,7 +4262,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
@@ -4256,7 +4277,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_weight_collections(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
@@ -4271,7 +4292,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     id_weight_pair = column._get_sparse_tensors(
         _LazyBuilder({
             'aaa': ((0, -1), (1, 0))
@@ -4287,7 +4308,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_inputs_too_small(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(1, -1, 0),
@@ -4300,7 +4321,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         id_weight_pair.id_tensor.eval()
 
   def test_get_sparse_tensors_with_inputs_too_big(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(1, 99, 0),
@@ -4313,7 +4334,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         id_weight_pair.id_tensor.eval()
 
   def test_get_sparse_tensors_with_default_value(self):
-    column = fc.categorical_column_with_identity(
+    column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4331,7 +4352,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self):
-    column = fc.categorical_column_with_identity(
+    column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
     input_indices = array_ops.placeholder(dtype=dtypes.int64)
     input_values = array_ops.placeholder(dtype=dtypes.int32)
@@ -4356,7 +4377,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           }))
 
   def test_linear_model(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column._num_buckets)
     with ops.Graph().as_default():
       predictions = fc.linear_model({
@@ -4377,7 +4398,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_keras_linear_model(self):
-    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column._num_buckets)
     with ops.Graph().as_default():
       predictions = get_keras_linear_model_predictions({
@@ -4404,9 +4425,9 @@ class TransformFeaturesTest(test.TestCase):
   # All transform tests are distributed in column test.
   # Here we only test multi column case and naming
   def transform_multi_column(self):
-    bucketized_price = fc.bucketized_column(
-        fc.numeric_column('price'), boundaries=[0, 2, 4, 6])
-    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
+    bucketized_price = fc._bucketized_column(
+        fc._numeric_column('price'), boundaries=[0, 2, 4, 6])
+    hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     with ops.Graph().as_default():
       features = {
           'price': [[-1.], [5.]],
@@ -4463,23 +4484,23 @@ class TransformFeaturesTest(test.TestCase):
 class IndicatorColumnTest(test.TestCase):
 
   def test_indicator_column(self):
-    a = fc.categorical_column_with_hash_bucket('a', 4)
-    indicator_a = fc.indicator_column(a)
+    a = fc._categorical_column_with_hash_bucket('a', 4)
+    indicator_a = fc._indicator_column(a)
     self.assertEqual(indicator_a.categorical_column.name, 'a')
     self.assertEqual(indicator_a.name, 'a_indicator')
     self.assertEqual(indicator_a._var_scope_name, 'a_indicator')
     self.assertEqual(indicator_a._variable_shape, [1, 4])
 
-    b = fc.categorical_column_with_hash_bucket('b', hash_bucket_size=100)
-    indicator_b = fc.indicator_column(b)
+    b = fc._categorical_column_with_hash_bucket('b', hash_bucket_size=100)
+    indicator_b = fc._indicator_column(b)
     self.assertEqual(indicator_b.categorical_column.name, 'b')
     self.assertEqual(indicator_b.name, 'b_indicator')
     self.assertEqual(indicator_b._var_scope_name, 'b_indicator')
     self.assertEqual(indicator_b._variable_shape, [1, 100])
 
   def test_1D_shape_succeeds(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_hash_bucket('animal', 4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_hash_bucket('animal', 4))
     builder = _LazyBuilder({'animal': ['fox', 'fox']})
     output = builder.get(animal)
     with self.cached_session():
@@ -4488,8 +4509,8 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_2D_shape_succeeds(self):
     # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready.
-    animal = fc.indicator_column(
-        fc.categorical_column_with_hash_bucket('animal', 4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_hash_bucket('animal', 4))
     builder = _LazyBuilder({
         'animal':
             sparse_tensor.SparseTensor(
@@ -4503,8 +4524,8 @@ class IndicatorColumnTest(test.TestCase):
                           self.evaluate(output))
 
   def test_multi_hot(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_identity('animal', num_buckets=4))
 
     builder = _LazyBuilder({
         'animal':
@@ -4516,8 +4537,8 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output))
 
   def test_multi_hot2(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_identity('animal', num_buckets=4))
     builder = _LazyBuilder({
         'animal':
             sparse_tensor.SparseTensor(
@@ -4528,17 +4549,17 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
   def test_deep_copy(self):
-    a = fc.categorical_column_with_hash_bucket('a', 4)
-    column = fc.indicator_column(a)
+    a = fc._categorical_column_with_hash_bucket('a', 4)
+    column = fc._indicator_column(a)
     column_copy = copy.deepcopy(column)
     self.assertEqual(column_copy.categorical_column.name, 'a')
     self.assertEqual(column.name, 'a_indicator')
     self.assertEqual(column._variable_shape, [1, 4])
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_indicator = fc.indicator_column(a)
+    a_indicator = fc._indicator_column(a)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -4559,9 +4580,9 @@ class IndicatorColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_transform(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_indicator = fc.indicator_column(a)
+    a_indicator = fc._indicator_column(a)
     features = {
         'aaa': sparse_tensor.SparseTensorValue(
             indices=((0, 0), (1, 0), (1, 1)),
@@ -4575,10 +4596,10 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_transform_with_weighted_column(self):
     # Github issue 12557
-    ids = fc.categorical_column_with_vocabulary_list(
+    ids = fc._categorical_column_with_vocabulary_list(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    weights = fc.weighted_categorical_column(ids, 'weights')
-    indicator = fc.indicator_column(weights)
+    weights = fc._weighted_categorical_column(ids, 'weights')
+    indicator = fc._indicator_column(weights)
     features = {
         'ids': constant_op.constant([['c', 'b', 'a', 'c']]),
         'weights': constant_op.constant([[2., 4., 6., 1.]])
@@ -4589,10 +4610,10 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
-    ids = fc.categorical_column_with_vocabulary_list(
+    ids = fc._categorical_column_with_vocabulary_list(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    weights = fc.weighted_categorical_column(ids, 'weights')
-    indicator = fc.indicator_column(weights)
+    weights = fc._weighted_categorical_column(ids, 'weights')
+    indicator = fc._indicator_column(weights)
     features = {
         'ids': constant_op.constant([['c', 'b', 'unknown']]),
         'weights': constant_op.constant([[2., 4., 6.]])
@@ -4603,9 +4624,9 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
-    ids = fc.categorical_column_with_vocabulary_list(
+    ids = fc._categorical_column_with_vocabulary_list(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    indicator = fc.indicator_column(ids)
+    indicator = fc._indicator_column(ids)
     features = {
         'ids': constant_op.constant([['c', 'b', 'unknown']]),
     }
@@ -4614,8 +4635,8 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
   def test_linear_model(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -4633,8 +4654,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_keras_linear_model(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -4652,8 +4673,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_input_layer(self):
-    animal = fc.indicator_column(
-        fc.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc._indicator_column(
+        fc._categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -4668,10 +4689,10 @@ class IndicatorColumnTest(test.TestCase):
 class EmbeddingColumnTest(test.TestCase):
 
   def test_defaults(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column(
+    embedding_column = fc._embedding_column(
         categorical_column, dimension=embedding_dimension)
     self.assertIs(categorical_column, embedding_column.categorical_column)
     self.assertEqual(embedding_dimension, embedding_column.dimension)
@@ -4689,14 +4710,18 @@ class EmbeddingColumnTest(test.TestCase):
     }, embedding_column._parse_example_spec)
 
   def test_all_constructor_args(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
-        combiner='my_combiner', initializer=lambda: 'my_initializer',
-        ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor',
-        max_norm=42., trainable=False)
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
+        combiner='my_combiner',
+        initializer=lambda: 'my_initializer',
+        ckpt_to_load_from='my_ckpt',
+        tensor_name_in_ckpt='my_ckpt_tensor',
+        max_norm=42.,
+        trainable=False)
     self.assertIs(categorical_column, embedding_column.categorical_column)
     self.assertEqual(embedding_dimension, embedding_column.dimension)
     self.assertEqual('my_combiner', embedding_column.combiner)
@@ -4713,14 +4738,18 @@ class EmbeddingColumnTest(test.TestCase):
     }, embedding_column._parse_example_spec)
 
   def test_deep_copy(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    original = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
-        combiner='my_combiner', initializer=lambda: 'my_initializer',
-        ckpt_to_load_from='my_ckpt', tensor_name_in_ckpt='my_ckpt_tensor',
-        max_norm=42., trainable=False)
+    original = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
+        combiner='my_combiner',
+        initializer=lambda: 'my_initializer',
+        ckpt_to_load_from='my_ckpt',
+        tensor_name_in_ckpt='my_ckpt_tensor',
+        max_norm=42.,
+        trainable=False)
     for embedding_column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', embedding_column.categorical_column.name)
       self.assertEqual(3, embedding_column.categorical_column._num_buckets)
@@ -4742,15 +4771,16 @@ class EmbeddingColumnTest(test.TestCase):
       }, embedding_column._parse_example_spec)
 
   def test_invalid_initializer(self):
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
-      fc.embedding_column(categorical_column, dimension=2, initializer='not_fn')
+      fc._embedding_column(
+          categorical_column, dimension=2, initializer='not_fn')
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_embedded = fc.embedding_column(a, dimension=2)
+    a_embedded = fc._embedding_column(a, dimension=2)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -4771,8 +4801,8 @@ class EmbeddingColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_transform_feature(self):
-    a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
-    a_embedded = fc.embedding_column(a, dimension=2)
+    a = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
+    a_embedded = fc._embedding_column(a, dimension=2)
     features = {
         'aaa': sparse_tensor.SparseTensor(
             indices=((0, 0), (1, 0), (1, 1)),
@@ -4824,10 +4854,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
@@ -4884,10 +4915,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
@@ -4915,9 +4947,9 @@ class EmbeddingColumnTest(test.TestCase):
         dense_shape=(4, 5))
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    embedding_column = fc.embedding_column(categorical_column, dimension=2)
+    embedding_column = fc._embedding_column(categorical_column, dimension=2)
 
     # Provide sparse input and get dense result.
     embedding_column._get_dense_tensor(
@@ -4971,10 +5003,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
@@ -5039,10 +5072,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         ckpt_to_load_from=ckpt_path,
         tensor_name_in_ckpt=ckpt_tensor)
 
@@ -5084,10 +5118,11 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     with ops.Graph().as_default():
@@ -5163,9 +5198,9 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
+    embedding_column = fc._embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -5255,10 +5290,11 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
@@ -5315,11 +5351,13 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity(
+    categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
-        initializer=_initializer, trainable=False)
+    embedding_column = fc._embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
+        initializer=_initializer,
+        trainable=False)
 
     # Provide sparse input and get dense result.
     input_layer = fc.input_layer({'aaa': sparse_input}, (embedding_column,))
@@ -5339,12 +5377,12 @@ class EmbeddingColumnTest(test.TestCase):
 class SharedEmbeddingColumnTest(test.TestCase):
 
   def test_defaults(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
-    embedding_column_b, embedding_column_a = fc.shared_embedding_columns(
+    embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension)
     self.assertIs(categorical_column_a, embedding_column_a.categorical_column)
@@ -5383,12 +5421,12 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }, embedding_column_b._parse_example_spec)
 
   def test_all_constructor_args(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
         dimension=embedding_dimension,
         combiner='my_combiner',
@@ -5434,12 +5472,12 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }, embedding_column_b._parse_example_spec)
 
   def test_deep_copy(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
-    original_a, _ = fc.shared_embedding_columns(
+    original_a, _ = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
         dimension=embedding_dimension,
         combiner='my_combiner',
@@ -5447,7 +5485,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
         shared_embedding_collection_name='shared_embedding_collection_name',
         ckpt_to_load_from='my_ckpt',
         tensor_name_in_ckpt='my_ckpt_tensor',
-        max_norm=42., trainable=False)
+        max_norm=42.,
+        trainable=False)
     for embedding_column_a in (original_a, copy.deepcopy(original_a)):
       self.assertEqual('aaa', embedding_column_a.categorical_column.name)
       self.assertEqual(3, embedding_column_a.categorical_column._num_buckets)
@@ -5471,54 +5510,55 @@ class SharedEmbeddingColumnTest(test.TestCase):
       }, embedding_column_a._parse_example_spec)
 
   def test_invalid_initializer(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
-      fc.shared_embedding_columns(
-          [categorical_column_a, categorical_column_b], dimension=2,
+      fc_new.shared_embedding_columns(
+          [categorical_column_a, categorical_column_b],
+          dimension=2,
           initializer='not_fn')
 
   def test_incompatible_column_type(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
-    categorical_column_c = fc.categorical_column_with_hash_bucket(
+    categorical_column_c = fc._categorical_column_with_hash_bucket(
         key='ccc', hash_bucket_size=3)
     with self.assertRaisesRegexp(
         ValueError,
         'all categorical_columns must have the same type.*'
         '_IdentityCategoricalColumn.*_HashedCategoricalColumn'):
-      fc.shared_embedding_columns(
+      fc_new.shared_embedding_columns(
           [categorical_column_a, categorical_column_b, categorical_column_c],
           dimension=2)
 
   def test_weighted_categorical_column_ok(self):
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    weighted_categorical_column_a = fc.weighted_categorical_column(
+    weighted_categorical_column_a = fc._weighted_categorical_column(
         categorical_column_a, weight_feature_key='aaa_weights')
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=3)
-    weighted_categorical_column_b = fc.weighted_categorical_column(
+    weighted_categorical_column_b = fc._weighted_categorical_column(
         categorical_column_b, weight_feature_key='bbb_weights')
-    fc.shared_embedding_columns(
+    fc_new.shared_embedding_columns(
         [weighted_categorical_column_a, categorical_column_b], dimension=2)
-    fc.shared_embedding_columns(
+    fc_new.shared_embedding_columns(
         [categorical_column_a, weighted_categorical_column_b], dimension=2)
-    fc.shared_embedding_columns(
+    fc_new.shared_embedding_columns(
         [weighted_categorical_column_a, weighted_categorical_column_b],
         dimension=2)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    b = fc.categorical_column_with_vocabulary_list(
+    b = fc._categorical_column_with_vocabulary_list(
         key='bbb', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_embedded, b_embedded = fc.shared_embedding_columns(
-        [a, b], dimension=2)
+    a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b],
+                                                             dimension=2)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -5550,10 +5590,10 @@ class SharedEmbeddingColumnTest(test.TestCase):
           features['bbb'].eval())
 
   def test_transform_feature(self):
-    a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
-    b = fc.categorical_column_with_identity(key='bbb', num_buckets=3)
-    a_embedded, b_embedded = fc.shared_embedding_columns(
-        [a, b], dimension=2)
+    a = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
+    b = fc._categorical_column_with_identity(key='bbb', num_buckets=3)
+    a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b],
+                                                             dimension=2)
     features = {
         'aaa': sparse_tensor.SparseTensor(
             indices=((0, 0), (1, 0), (1, 1)),
@@ -5618,13 +5658,14 @@ class SharedEmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
-        dimension=embedding_dimension, initializer=_initializer)
+        dimension=embedding_dimension,
+        initializer=_initializer)
 
     # Provide sparse input and get dense result.
     embedding_lookup_a = embedding_column_a._get_dense_tensor(
@@ -5671,11 +5712,11 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -5732,13 +5773,14 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
-        dimension=embedding_dimension, initializer=_initializer)
+        dimension=embedding_dimension,
+        initializer=_initializer)
 
     # Provide sparse input and get dense result.
     embedding_lookup_a = embedding_column_a._get_dense_tensor(
@@ -5772,13 +5814,14 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
-        dimension=embedding_dimension, initializer=_initializer)
+        dimension=embedding_dimension,
+        initializer=_initializer)
 
     with ops.Graph().as_default():
       predictions = fc.linear_model({
@@ -5864,11 +5907,11 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -5973,13 +6016,14 @@ class SharedEmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity(
+    categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity(
+    categorical_column_b = fc._categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+    embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
-        dimension=embedding_dimension, initializer=_initializer,
+        dimension=embedding_dimension,
+        initializer=_initializer,
         trainable=trainable)
 
     # Provide sparse input and get dense result.
@@ -6014,8 +6058,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
 class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     self.assertEqual('ids_weighted_by_values', column.name)
@@ -6028,8 +6072,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_deep_copy(self):
     """Tests deepcopy of categorical_column_with_hash_bucket."""
-    original = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    original = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     for column in (original, copy.deepcopy(original)):
@@ -6042,23 +6086,23 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype_none(self):
     with self.assertRaisesRegexp(ValueError, 'is not convertible to float'):
-      fc.weighted_categorical_column(
-          categorical_column=fc.categorical_column_with_identity(
+      fc._weighted_categorical_column(
+          categorical_column=fc._categorical_column_with_identity(
               key='ids', num_buckets=3),
           weight_feature_key='values',
           dtype=None)
 
   def test_invalid_dtype_string(self):
     with self.assertRaisesRegexp(ValueError, 'is not convertible to float'):
-      fc.weighted_categorical_column(
-          categorical_column=fc.categorical_column_with_identity(
+      fc._weighted_categorical_column(
+          categorical_column=fc._categorical_column_with_identity(
               key='ids', num_buckets=3),
           weight_feature_key='values',
           dtype=dtypes.string)
 
   def test_invalid_input_dtype(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     strings = sparse_tensor.SparseTensorValue(
@@ -6070,14 +6114,14 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_column_name_collision(self):
     with self.assertRaisesRegexp(ValueError, r'Parse config.*already exists'):
-      fc.weighted_categorical_column(
-          categorical_column=fc.categorical_column_with_identity(
+      fc._weighted_categorical_column(
+          categorical_column=fc._categorical_column_with_identity(
               key='aaa', num_buckets=3),
           weight_feature_key='aaa')._parse_example_spec()
 
   def test_missing_weights(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -6089,9 +6133,10 @@ class WeightedCategoricalColumnTest(test.TestCase):
       _transform_features({'ids': inputs}, (column,))
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list(
+    a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_weighted = fc.weighted_categorical_column(a, weight_feature_key='weights')
+    a_weighted = fc._weighted_categorical_column(
+        a, weight_feature_key='weights')
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -6123,8 +6168,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           features['weights'].eval())
 
   def test_transform_features(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -6154,8 +6199,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_input(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     weights = sparse_tensor.SparseTensorValue(
@@ -6181,8 +6226,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_weights(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -6208,8 +6253,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
   def test_keras_linear_model(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6238,8 +6283,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_keras_linear_model_mismatched_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6259,8 +6304,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         }, (column,))
 
   def test_keras_linear_model_mismatched_dense_values(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6284,8 +6329,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           self.evaluate(predictions)
 
   def test_keras_linear_model_mismatched_dense_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6310,8 +6355,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6338,8 +6383,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model_mismatched_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6357,8 +6402,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         }, (column,))
 
   def test_linear_model_mismatched_dense_values(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -6382,8 +6427,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           self.evaluate(predictions)
 
   def test_linear_model_mismatched_dense_shape(self):
-    column = fc.weighted_categorical_column(
-        categorical_column=fc.categorical_column_with_identity(
+    column = fc._weighted_categorical_column(
+        categorical_column=fc._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 8d27c96669..59828de840 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -654,7 +654,7 @@ class LinearModel(training.Model):
     return self.layer.bias
 
 
-def _transform_features(features, feature_columns, state_manager):
+def _transform_features_v2(features, feature_columns, state_manager):
   """Returns transformed features based on features columns passed in.
 
   Please note that most probably you would not need to use this function. Please
@@ -759,15 +759,15 @@ def make_parse_example_spec_v2(feature_columns):
   return result
 
 
-@tf_export('feature_column.embedding_column', v1=[])
-def embedding_column_v2(categorical_column,
-                        dimension,
-                        combiner='mean',
-                        initializer=None,
-                        ckpt_to_load_from=None,
-                        tensor_name_in_ckpt=None,
-                        max_norm=None,
-                        trainable=True):
+@tf_export('feature_column.embedding_column')
+def embedding_column(categorical_column,
+                     dimension,
+                     combiner='mean',
+                     initializer=None,
+                     ckpt_to_load_from=None,
+                     tensor_name_in_ckpt=None,
+                     max_norm=None,
+                     trainable=True):
   """`DenseColumn` that converts from sparse, categorical input.
 
   Use this when your inputs are sparse, but you want to convert them to a dense
@@ -864,6 +864,179 @@ def embedding_column_v2(categorical_column,
       trainable=trainable)
 
 
+@tf_export(v1=['feature_column.shared_embedding_columns'])
+def shared_embedding_columns(categorical_columns,
+                             dimension,
+                             combiner='mean',
+                             initializer=None,
+                             shared_embedding_collection_name=None,
+                             ckpt_to_load_from=None,
+                             tensor_name_in_ckpt=None,
+                             max_norm=None,
+                             trainable=True):
+  """List of dense columns that convert from sparse, categorical input.
+
+  This is similar to `embedding_column`, except that it produces a list of
+  embedding columns that share the same embedding weights.
+
+  Use this when your inputs are sparse and of the same type (e.g. watched and
+  impression video IDs that share the same vocabulary), and you want to convert
+  them to a dense representation (e.g., to feed to a DNN).
+
+  Inputs must be a list of categorical columns created by any of the
+  `categorical_column_*` function. They must all be of the same type and have
+  the same arguments except `key`. E.g. they can be
+  categorical_column_with_vocabulary_file with the same vocabulary_file. Some or
+  all columns could also be weighted_categorical_column.
+
+  Here is an example embedding of two features for a DNNClassifier model:
+
+  ```python
+  watched_video_id = categorical_column_with_vocabulary_file(
+      'watched_video_id', video_vocabulary_file, video_vocabulary_size)
+  impression_video_id = categorical_column_with_vocabulary_file(
+      'impression_video_id', video_vocabulary_file, video_vocabulary_size)
+  columns = shared_embedding_columns(
+      [watched_video_id, impression_video_id], dimension=10)
+
+  estimator = tf.estimator.DNNClassifier(feature_columns=columns, ...)
+
+  label_column = ...
+  def input_fn():
+    features = tf.parse_example(
+        ..., features=make_parse_example_spec(columns + [label_column]))
+    labels = features.pop(label_column.name)
+    return features, labels
+
+  estimator.train(input_fn=input_fn, steps=100)
+  ```
+
+  Here is an example using `shared_embedding_columns` with model_fn:
+
+  ```python
+  def model_fn(features, ...):
+    watched_video_id = categorical_column_with_vocabulary_file(
+        'watched_video_id', video_vocabulary_file, video_vocabulary_size)
+    impression_video_id = categorical_column_with_vocabulary_file(
+        'impression_video_id', video_vocabulary_file, video_vocabulary_size)
+    columns = shared_embedding_columns(
+        [watched_video_id, impression_video_id], dimension=10)
+    dense_tensor = input_layer(features, columns)
+    # Form DNN layers, calculate loss, and return EstimatorSpec.
+    ...
+  ```
+
+  Args:
+    categorical_columns: List of categorical columns created by a
+      `categorical_column_with_*` function. These columns produce the sparse IDs
+      that are inputs to the embedding lookup. All columns must be of the same
+      type and have the same arguments except `key`. E.g. they can be
+      categorical_column_with_vocabulary_file with the same vocabulary_file.
+      Some or all columns could also be weighted_categorical_column.
+    dimension: An integer specifying dimension of the embedding, must be > 0.
+    combiner: A string specifying how to reduce if there are multiple entries in
+      a single row. Currently 'mean', 'sqrtn' and 'sum' are supported, with
+      'mean' the default. 'sqrtn' often achieves good accuracy, in particular
+      with bag-of-words columns. Each of this can be thought as example level
+      normalizations on the column. For more information, see
+      `tf.embedding_lookup_sparse`.
+    initializer: A variable initializer function to be used in embedding
+      variable initialization. If not specified, defaults to
+      `tf.truncated_normal_initializer` with mean `0.0` and standard deviation
+      `1/sqrt(dimension)`.
+    shared_embedding_collection_name: Optional name of the collection where
+      shared embedding weights are added. If not given, a reasonable name will
+      be chosen based on the names of `categorical_columns`. This is also used
+      in `variable_scope` when creating shared embedding weights.
+    ckpt_to_load_from: String representing checkpoint name/pattern from which to
+      restore column weights. Required if `tensor_name_in_ckpt` is not `None`.
+    tensor_name_in_ckpt: Name of the `Tensor` in `ckpt_to_load_from` from which
+      to restore the column weights. Required if `ckpt_to_load_from` is not
+      `None`.
+    max_norm: If not `None`, each embedding is clipped if its l2-norm is larger
+      than this value, before combining.
+    trainable: Whether or not the embedding is trainable. Default is True.
+
+  Returns:
+    A list of dense columns that converts from sparse input. The order of
+    results follows the ordering of `categorical_columns`.
+
+  Raises:
+    ValueError: if `dimension` not > 0.
+    ValueError: if any of the given `categorical_columns` is of different type
+      or has different arguments than the others.
+    ValueError: if exactly one of `ckpt_to_load_from` and `tensor_name_in_ckpt`
+      is specified.
+    ValueError: if `initializer` is specified and is not callable.
+    RuntimeError: if eager execution is enabled.
+  """
+  if context.executing_eagerly():
+    raise RuntimeError('shared_embedding_columns are not supported when eager '
+                       'execution is enabled.')
+
+  if (dimension is None) or (dimension < 1):
+    raise ValueError('Invalid dimension {}.'.format(dimension))
+  if (ckpt_to_load_from is None) != (tensor_name_in_ckpt is None):
+    raise ValueError('Must specify both `ckpt_to_load_from` and '
+                     '`tensor_name_in_ckpt` or none of them.')
+
+  if (initializer is not None) and (not callable(initializer)):
+    raise ValueError('initializer must be callable if specified.')
+  if initializer is None:
+    initializer = init_ops.truncated_normal_initializer(
+        mean=0.0, stddev=1. / math.sqrt(dimension))
+
+  # Sort the columns so the default collection name is deterministic even if the
+  # user passes columns from an unsorted collection, such as dict.values().
+  sorted_columns = sorted(categorical_columns, key=lambda x: x.name)
+
+  c0 = sorted_columns[0]
+  num_buckets = c0._num_buckets  # pylint: disable=protected-access
+  if not isinstance(c0, fc_old._CategoricalColumn):  # pylint: disable=protected-access
+    raise ValueError(
+        'All categorical_columns must be subclasses of _CategoricalColumn. '
+        'Given: {}, of type: {}'.format(c0, type(c0)))
+  if isinstance(c0,
+                (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)):  # pylint: disable=protected-access
+    c0 = c0.categorical_column
+  for c in sorted_columns[1:]:
+    if isinstance(
+        c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)):  # pylint: disable=protected-access
+      c = c.categorical_column
+    if not isinstance(c, type(c0)):
+      raise ValueError(
+          'To use shared_embedding_column, all categorical_columns must have '
+          'the same type, or be weighted_categorical_column of the same type. '
+          'Given column: {} of type: {} does not match given column: {} of '
+          'type: {}'.format(c0, type(c0), c, type(c)))
+    if num_buckets != c._num_buckets:  # pylint: disable=protected-access
+      raise ValueError(
+          'To use shared_embedding_column, all categorical_columns must have '
+          'the same number of buckets. Given column: {} with buckets: {} does  '
+          'not match column: {} with buckets: {}'.format(
+              c0, num_buckets, c, c._num_buckets))  # pylint: disable=protected-access
+
+  if not shared_embedding_collection_name:
+    shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns)
+    shared_embedding_collection_name += '_shared_embedding'
+
+  result = []
+  for column in categorical_columns:
+    result.append(
+        fc_old._SharedEmbeddingColumn(  # pylint: disable=protected-access
+            categorical_column=column,
+            initializer=initializer,
+            dimension=dimension,
+            combiner=combiner,
+            shared_embedding_collection_name=shared_embedding_collection_name,
+            ckpt_to_load_from=ckpt_to_load_from,
+            tensor_name_in_ckpt=tensor_name_in_ckpt,
+            max_norm=max_norm,
+            trainable=trainable))
+
+  return result
+
+
 @tf_export('feature_column.shared_embedding_columns', v1=[])
 def shared_embedding_columns_v2(categorical_columns,
                                 dimension,
@@ -1030,12 +1203,12 @@ def shared_embedding_columns_v2(categorical_columns,
   return result
 
 
-@tf_export('feature_column.numeric_column', v1=[])
-def numeric_column_v2(key,
-                      shape=(1,),
-                      default_value=None,
-                      dtype=dtypes.float32,
-                      normalizer_fn=None):
+@tf_export('feature_column.numeric_column')
+def numeric_column(key,
+                   shape=(1,),
+                   default_value=None,
+                   dtype=dtypes.float32,
+                   normalizer_fn=None):
   """Represents real valued or numerical features.
 
   Example:
@@ -1106,8 +1279,8 @@ def numeric_column_v2(key,
       normalizer_fn=normalizer_fn)
 
 
-@tf_export('feature_column.bucketized_column', v1=[])
-def bucketized_column_v2(source_column, boundaries):
+@tf_export('feature_column.bucketized_column')
+def bucketized_column(source_column, boundaries):
   """Represents discretized dense input.
 
   Buckets include the left boundary, and exclude the right boundary. Namely,
@@ -1203,10 +1376,10 @@ def _assert_key_is_string(key):
             type(key), key))
 
 
-@tf_export('feature_column.categorical_column_with_hash_bucket', v1=[])
-def categorical_column_with_hash_bucket_v2(key,
-                                           hash_bucket_size,
-                                           dtype=dtypes.string):
+@tf_export('feature_column.categorical_column_with_hash_bucket')
+def categorical_column_with_hash_bucket(key,
+                                        hash_bucket_size,
+                                        dtype=dtypes.string):
   """Represents sparse feature where ids are set by hashing.
 
   Use this when your sparse features are in string or integer format, and you
@@ -1262,13 +1435,13 @@ def categorical_column_with_hash_bucket_v2(key,
   return HashedCategoricalColumn(key, hash_bucket_size, dtype)
 
 
-@tf_export('feature_column.categorical_column_with_vocabulary_file', v1=[])
-def categorical_column_with_vocabulary_file_v2(key,
-                                               vocabulary_file,
-                                               vocabulary_size=None,
-                                               num_oov_buckets=0,
-                                               default_value=None,
-                                               dtype=dtypes.string):
+@tf_export('feature_column.categorical_column_with_vocabulary_file')
+def categorical_column_with_vocabulary_file(key,
+                                            vocabulary_file,
+                                            vocabulary_size=None,
+                                            num_oov_buckets=0,
+                                            default_value=None,
+                                            dtype=dtypes.string):
   """A `CategoricalColumn` with a vocabulary file.
 
   Use this when your inputs are in string or integer format, and you have a
@@ -1382,12 +1555,12 @@ def categorical_column_with_vocabulary_file_v2(key,
       dtype=dtype)
 
 
-@tf_export('feature_column.categorical_column_with_vocabulary_list', v1=[])
-def categorical_column_with_vocabulary_list_v2(key,
-                                               vocabulary_list,
-                                               dtype=None,
-                                               default_value=-1,
-                                               num_oov_buckets=0):
+@tf_export('feature_column.categorical_column_with_vocabulary_list')
+def categorical_column_with_vocabulary_list(key,
+                                            vocabulary_list,
+                                            dtype=None,
+                                            default_value=-1,
+                                            num_oov_buckets=0):
   """A `CategoricalColumn` with in-memory vocabulary.
 
   Use this when your inputs are in string or integer format, and you have an
@@ -1499,8 +1672,8 @@ def categorical_column_with_vocabulary_list_v2(key,
       num_oov_buckets=num_oov_buckets)
 
 
-@tf_export('feature_column.categorical_column_with_identity', v1=[])
-def categorical_column_with_identity_v2(key, num_buckets, default_value=None):
+@tf_export('feature_column.categorical_column_with_identity')
+def categorical_column_with_identity(key, num_buckets, default_value=None):
   """A `CategoricalColumn` that returns identity values.
 
   Use this when your inputs are integers in the range `[0, num_buckets)`, and
@@ -1567,8 +1740,8 @@ def categorical_column_with_identity_v2(key, num_buckets, default_value=None):
       key=key, number_buckets=num_buckets, default_value=default_value)
 
 
-@tf_export('feature_column.indicator_column', v1=[])
-def indicator_column_v2(categorical_column):
+@tf_export('feature_column.indicator_column')
+def indicator_column(categorical_column):
   """Represents multi-hot representation of given categorical column.
 
   - For DNN model, `indicator_column` can be used to wrap any
@@ -1602,10 +1775,10 @@ def indicator_column_v2(categorical_column):
   return IndicatorColumn(categorical_column)
 
 
-@tf_export('feature_column.weighted_categorical_column', v1=[])
-def weighted_categorical_column_v2(categorical_column,
-                                   weight_feature_key,
-                                   dtype=dtypes.float32):
+@tf_export('feature_column.weighted_categorical_column')
+def weighted_categorical_column(categorical_column,
+                                weight_feature_key,
+                                dtype=dtypes.float32):
   """Applies weight values to a `CategoricalColumn`.
 
   Use this when each of your sparse inputs has both an ID and a value. For
@@ -1678,8 +1851,8 @@ def weighted_categorical_column_v2(categorical_column,
       dtype=dtype)
 
 
-@tf_export('feature_column.crossed_column', v1=[])
-def crossed_column_v2(keys, hash_bucket_size, hash_key=None):
+@tf_export('feature_column.crossed_column')
+def crossed_column(keys, hash_bucket_size, hash_key=None):
   """Returns a column for performing crosses of categorical features.
 
   Crossed features will be hashed according to `hash_bucket_size`. Conceptually,
@@ -2144,7 +2317,7 @@ def _create_categorical_column_weighted_sum(
     weight_tensor = sparse_ops.sparse_reshape(
         weight_tensor, [array_ops.shape(weight_tensor)[0], -1])
 
-  return _safe_embedding_lookup_sparse(
+  return embedding_ops.safe_embedding_lookup_sparse(
       weight_var,
       id_tensor,
       sparse_weights=weight_tensor,
@@ -2755,7 +2928,7 @@ class EmbeddingColumn(
       })
 
     # Return embedding lookup result.
-    return _safe_embedding_lookup_sparse(
+    return embedding_ops.safe_embedding_lookup_sparse(
         embedding_weights=embedding_weights,
         sparse_ids=sparse_ids,
         sparse_weights=sparse_weights,
@@ -3026,7 +3199,7 @@ class SharedEmbeddingColumn(
       embedding_weights = self.shared_embedding_column_creator.embedding_weights
 
       # Return embedding lookup result.
-      return _safe_embedding_lookup_sparse(
+      return embedding_ops.safe_embedding_lookup_sparse(
           embedding_weights=embedding_weights,
           sparse_ids=sparse_ids,
           sparse_weights=sparse_weights,
@@ -3711,9 +3884,13 @@ class WeightedCategoricalColumn(
 
   def transform_feature(self, transformation_cache, state_manager):
     """Applies weights to tensor generated from `categorical_column`'."""
+    print('WeightedCategoricalColumn.transform_feature: ', self.name)
+    print('Weight feature key: ', self.weight_feature_key)
     weight_tensor = transformation_cache.get(self.weight_feature_key,
                                              state_manager)
+    print('Weight tensor before: ', weight_tensor)
     weight_tensor = self._transform_weight_tensor(weight_tensor)
+    print('Weight tensor after: ', weight_tensor)
     return (transformation_cache.get(self.categorical_column, state_manager),
             weight_tensor)
 
@@ -3727,7 +3904,9 @@ class WeightedCategoricalColumn(
 
   def get_sparse_tensors(self, transformation_cache, state_manager):
     """See `CategoricalColumn` base class."""
+    print('WeightedCategoricalColumn.get_sparse_tensors: ', self.name)
     tensors = transformation_cache.get(self, state_manager)
+    print('tensors[1]: ', tensors[1])
     return CategoricalColumn.IdWeightPair(tensors[0], tensors[1])
 
   @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE,
@@ -3922,142 +4101,6 @@ def _collect_leaf_level_keys(cross):
   return leaf_level_keys
 
 
-# TODO(zakaria): Move this to embedding_ops and make it public.
-def _safe_embedding_lookup_sparse(embedding_weights,
-                                  sparse_ids,
-                                  sparse_weights=None,
-                                  combiner='mean',
-                                  default_id=None,
-                                  name=None,
-                                  partition_strategy='div',
-                                  max_norm=None):
-  """Lookup embedding results, accounting for invalid IDs and empty features.
-
-  The partitioned embedding in `embedding_weights` must all be the same shape
-  except for the first dimension. The first dimension is allowed to vary as the
-  vocabulary size is not necessarily a multiple of `P`.  `embedding_weights`
-  may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a
-  partitioner.
-
-  Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs
-  with non-positive weight. For an entry with no features, the embedding vector
-  for `default_id` is returned, or the 0-vector if `default_id` is not supplied.
-
-  The ids and weights may be multi-dimensional. Embeddings are always aggregated
-  along the last dimension.
-
-  Args:
-    embedding_weights:  A list of `P` float `Tensor`s or values representing
-        partitioned embedding `Tensor`s.  Alternatively, a `PartitionedVariable`
-        created by partitioning along dimension 0.  The total unpartitioned
-        shape should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the
-        vocab size and `e_1, ..., e_m` are the embedding dimensions.
-    sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the
-        ids. `d_0` is typically batch size.
-    sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing
-        float weights corresponding to `sparse_ids`, or `None` if all weights
-        are be assumed to be 1.0.
-    combiner: A string specifying how to combine embedding results for each
-        entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean"
-        the default.
-    default_id: The id to use for an entry with no features.
-    name: A name for this operation (optional).
-    partition_strategy: A string specifying the partitioning strategy.
-        Currently `"div"` and `"mod"` are supported. Default is `"div"`.
-    max_norm: If not `None`, all embeddings are l2-normalized to max_norm before
-        combining.
-
-
-  Returns:
-    Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`.
-
-  Raises:
-    ValueError: if `embedding_weights` is empty.
-  """
-  if embedding_weights is None:
-    raise ValueError('Missing embedding_weights %s.' % embedding_weights)
-  if isinstance(embedding_weights, variables.PartitionedVariable):
-    embedding_weights = list(embedding_weights)  # get underlying Variables.
-  if not isinstance(embedding_weights, list):
-    embedding_weights = [embedding_weights]
-  if len(embedding_weights) < 1:
-    raise ValueError('Missing embedding_weights %s.' % embedding_weights)
-
-  dtype = sparse_weights.dtype if sparse_weights is not None else None
-  # TODO(rohanj): Look into removing this convert_to_tensor call.
-  embedding_weights = [
-      ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights
-  ]
-
-  with ops.name_scope(name, 'embedding_lookup',
-                      embedding_weights + [sparse_ids,
-                                           sparse_weights]) as scope:
-    # Reshape higher-rank sparse ids and weights to linear segment ids.
-    original_shape = sparse_ids.dense_shape
-    original_rank_dim = tensor_shape.dimension_value(
-        sparse_ids.dense_shape.get_shape()[0])
-    original_rank = (
-        array_ops.size(original_shape)
-        if original_rank_dim is None
-        else original_rank_dim)
-    sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [
-        math_ops.reduce_prod(
-            array_ops.slice(original_shape, [0], [original_rank - 1])),
-        array_ops.gather(original_shape, original_rank - 1)])
-    if sparse_weights is not None:
-      sparse_weights = sparse_tensor_lib.SparseTensor(
-          sparse_ids.indices,
-          sparse_weights.values, sparse_ids.dense_shape)
-
-    # Prune invalid ids and weights.
-    sparse_ids, sparse_weights = _prune_invalid_ids(sparse_ids, sparse_weights)
-    if combiner != 'sum':
-      sparse_ids, sparse_weights = _prune_invalid_weights(
-          sparse_ids, sparse_weights)
-
-    # Fill in dummy values for empty features, if necessary.
-    sparse_ids, is_row_empty = sparse_ops.sparse_fill_empty_rows(sparse_ids,
-                                                                 default_id or
-                                                                 0)
-    if sparse_weights is not None:
-      sparse_weights, _ = sparse_ops.sparse_fill_empty_rows(sparse_weights, 1.0)
-
-    result = embedding_ops.embedding_lookup_sparse(
-        embedding_weights,
-        sparse_ids,
-        sparse_weights,
-        combiner=combiner,
-        partition_strategy=partition_strategy,
-        name=None if default_id is None else scope,
-        max_norm=max_norm)
-
-    if default_id is None:
-      # Broadcast is_row_empty to the same shape as embedding_lookup_result,
-      # for use in Select.
-      is_row_empty = array_ops.tile(
-          array_ops.reshape(is_row_empty, [-1, 1]),
-          array_ops.stack([1, array_ops.shape(result)[1]]))
-
-      result = array_ops.where(is_row_empty,
-                               array_ops.zeros_like(result),
-                               result,
-                               name=scope)
-
-    # Reshape back from linear ids back into higher-dimensional dense result.
-    final_result = array_ops.reshape(
-        result,
-        array_ops.concat([
-            array_ops.slice(
-                math_ops.cast(original_shape, dtypes.int32), [0],
-                [original_rank - 1]),
-            array_ops.slice(array_ops.shape(result), [1], [-1])
-        ], 0))
-    final_result.set_shape(tensor_shape.unknown_shape(
-        (tensor_shape.Dimension(original_rank_dim) - 1).value).concatenate(
-            result.get_shape()[1:]))
-    return final_result
-
-
 def _prune_invalid_ids(sparse_ids, sparse_weights):
   """Prune invalid IDs (< 0) from the input ids and weights."""
   is_id_valid = math_ops.greater_equal(sparse_ids.values, 0)
@@ -4113,10 +4156,14 @@ class IndicatorColumn(
           sp_ids=id_tensor,
           sp_values=weight_tensor,
           vocab_size=int(self._variable_shape[-1]))
-      # Remove (?, -1) index
+      # Remove (?, -1) index.
       weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0],
                                                 weighted_column.dense_shape)
-      return sparse_ops.sparse_tensor_to_dense(weighted_column)
+      # Use scatter_nd to merge duplicated indices if existed,
+      # instead of sparse_tensor_to_dense.
+      return array_ops.scatter_nd(weighted_column.indices,
+                                  weighted_column.values,
+                                  weighted_column.dense_shape)
 
     dense_id_tensor = sparse_ops.sparse_tensor_to_dense(
         id_tensor, default_value=-1)
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 251ede925c..23131e22ed 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -238,7 +238,7 @@ class LazyColumnTest(test.TestCase):
 class NumericColumnTest(test.TestCase):
 
   def test_defaults(self):
-    a = fc.numeric_column_v2('aaa')
+    a = fc.numeric_column('aaa')
     self.assertEqual('aaa', a.key)
     self.assertEqual('aaa', a.name)
     self.assertEqual((1,), a.shape)
@@ -249,53 +249,53 @@ class NumericColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.numeric_column_v2(key=('aaa',))
+      fc.numeric_column(key=('aaa',))
 
   def test_shape_saved_as_tuple(self):
-    a = fc.numeric_column_v2('aaa', shape=[1, 2], default_value=[[3, 2.]])
+    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
     self.assertEqual((1, 2), a.shape)
 
   def test_default_value_saved_as_tuple(self):
-    a = fc.numeric_column_v2('aaa', default_value=4.)
+    a = fc.numeric_column('aaa', default_value=4.)
     self.assertEqual((4.,), a.default_value)
-    a = fc.numeric_column_v2('aaa', shape=[1, 2], default_value=[[3, 2.]])
+    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3, 2.]])
     self.assertEqual(((3., 2.),), a.default_value)
 
   def test_shape_and_default_value_compatibility(self):
-    fc.numeric_column_v2('aaa', shape=[2], default_value=[1, 2.])
+    fc.numeric_column('aaa', shape=[2], default_value=[1, 2.])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column_v2('aaa', shape=[2], default_value=[1, 2, 3.])
-    fc.numeric_column_v2(
+      fc.numeric_column('aaa', shape=[2], default_value=[1, 2, 3.])
+    fc.numeric_column(
         'aaa', shape=[3, 2], default_value=[[2, 3], [1, 2], [2, 3.]])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column_v2(
+      fc.numeric_column(
           'aaa', shape=[3, 1], default_value=[[2, 3], [1, 2], [2, 3.]])
     with self.assertRaisesRegexp(ValueError, 'The shape of default_value'):
-      fc.numeric_column_v2(
+      fc.numeric_column(
           'aaa', shape=[3, 3], default_value=[[2, 3], [1, 2], [2, 3.]])
 
   def test_default_value_type_check(self):
-    fc.numeric_column_v2(
+    fc.numeric_column(
         'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.float32)
-    fc.numeric_column_v2(
+    fc.numeric_column(
         'aaa', shape=[2], default_value=[1, 2], dtype=dtypes.int32)
     with self.assertRaisesRegexp(TypeError, 'must be compatible with dtype'):
-      fc.numeric_column_v2(
+      fc.numeric_column(
           'aaa', shape=[2], default_value=[1, 2.], dtype=dtypes.int32)
     with self.assertRaisesRegexp(TypeError,
                                  'default_value must be compatible with dtype'):
-      fc.numeric_column_v2('aaa', default_value=['string'])
+      fc.numeric_column('aaa', default_value=['string'])
 
   def test_shape_must_be_positive_integer(self):
     with self.assertRaisesRegexp(TypeError, 'shape dimensions must be integer'):
-      fc.numeric_column_v2(
+      fc.numeric_column(
           'aaa', shape=[
               1.0,
           ])
 
     with self.assertRaisesRegexp(ValueError,
                                  'shape dimensions must be greater than 0'):
-      fc.numeric_column_v2(
+      fc.numeric_column(
           'aaa', shape=[
               0,
           ])
@@ -303,20 +303,20 @@ class NumericColumnTest(test.TestCase):
   def test_dtype_is_convertible_to_float(self):
     with self.assertRaisesRegexp(ValueError,
                                  'dtype must be convertible to float'):
-      fc.numeric_column_v2('aaa', dtype=dtypes.string)
+      fc.numeric_column('aaa', dtype=dtypes.string)
 
   def test_scalar_default_value_fills_the_shape(self):
-    a = fc.numeric_column_v2('aaa', shape=[2, 3], default_value=2.)
+    a = fc.numeric_column('aaa', shape=[2, 3], default_value=2.)
     self.assertEqual(((2., 2., 2.), (2., 2., 2.)), a.default_value)
 
   def test_parse_spec(self):
-    a = fc.numeric_column_v2('aaa', shape=[2, 3], dtype=dtypes.int32)
+    a = fc.numeric_column('aaa', shape=[2, 3], dtype=dtypes.int32)
     self.assertEqual({
         'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32)
     }, a.parse_example_spec)
 
   def test_parse_example_no_default_value(self):
-    price = fc.numeric_column_v2('price', shape=[2])
+    price = fc.numeric_column('price', shape=[2])
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -331,7 +331,7 @@ class NumericColumnTest(test.TestCase):
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
   def test_parse_example_with_default_value(self):
-    price = fc.numeric_column_v2('price', shape=[2], default_value=11.)
+    price = fc.numeric_column('price', shape=[2], default_value=11.)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -354,16 +354,15 @@ class NumericColumnTest(test.TestCase):
 
   def test_normalizer_fn_must_be_callable(self):
     with self.assertRaisesRegexp(TypeError, 'must be a callable'):
-      fc.numeric_column_v2('price', normalizer_fn='NotACallable')
+      fc.numeric_column('price', normalizer_fn='NotACallable')
 
   def test_normalizer_fn_transform_feature(self):
 
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column_v2(
-        'price', shape=[2], normalizer_fn=_increment_two)
-    output = fc._transform_features({
+    price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two)
+    output = fc._transform_features_v2({
         'price': [[1., 2.], [5., 6.]]
     }, [price], None)
     with self.cached_session():
@@ -374,8 +373,7 @@ class NumericColumnTest(test.TestCase):
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column_v2(
-        'price', shape=[2], normalizer_fn=_increment_two)
+    price = fc.numeric_column('price', shape=[2], normalizer_fn=_increment_two)
     transformation_cache = fc.FeatureTransformationCache({
         'price': [[1., 2.], [5., 6.]]
     })
@@ -384,7 +382,7 @@ class NumericColumnTest(test.TestCase):
         price.get_dense_tensor(transformation_cache, None))
 
   def test_sparse_tensor_not_supported(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     transformation_cache = fc.FeatureTransformationCache({
         'price':
             sparse_tensor.SparseTensor(
@@ -394,19 +392,19 @@ class NumericColumnTest(test.TestCase):
       price.transform_feature(transformation_cache, None)
 
   def test_deep_copy(self):
-    a = fc.numeric_column_v2('aaa', shape=[1, 2], default_value=[[3., 2.]])
+    a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]])
     a_copy = copy.deepcopy(a)
     self.assertEqual(a_copy.name, 'aaa')
     self.assertEqual(a_copy.shape, (1, 2))
     self.assertEqual(a_copy.default_value, ((3., 2.),))
 
   def test_numpy_default_value(self):
-    a = fc.numeric_column_v2(
+    a = fc.numeric_column(
         'aaa', shape=[1, 2], default_value=np.array([[3., 2.]]))
     self.assertEqual(a.default_value, ((3., 2.),))
 
   def test_linear_model(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price])
@@ -420,7 +418,7 @@ class NumericColumnTest(test.TestCase):
         self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc_old.linear_model(features, [price])
@@ -438,7 +436,7 @@ class NumericColumnTest(test.TestCase):
     def _increment_two(input_tensor):
       return input_tensor + 2.
 
-    price = fc.numeric_column_v2('price', normalizer_fn=_increment_two)
+    price = fc.numeric_column('price', normalizer_fn=_increment_two)
     self.assertEqual(['price'], price.parents)
 
     config = price._get_config()
@@ -459,67 +457,67 @@ class NumericColumnTest(test.TestCase):
 class BucketizedColumnTest(test.TestCase):
 
   def test_invalid_source_column_type(self):
-    a = fc.categorical_column_with_hash_bucket_v2('aaa', hash_bucket_size=10)
+    a = fc.categorical_column_with_hash_bucket('aaa', hash_bucket_size=10)
     with self.assertRaisesRegexp(
         ValueError,
         'source_column must be a column generated with numeric_column'):
-      fc.bucketized_column_v2(a, boundaries=[0, 1])
+      fc.bucketized_column(a, boundaries=[0, 1])
 
   def test_invalid_source_column_shape(self):
-    a = fc.numeric_column_v2('aaa', shape=[2, 3])
+    a = fc.numeric_column('aaa', shape=[2, 3])
     with self.assertRaisesRegexp(
         ValueError, 'source_column must be one-dimensional column'):
-      fc.bucketized_column_v2(a, boundaries=[0, 1])
+      fc.bucketized_column(a, boundaries=[0, 1])
 
   def test_invalid_boundaries(self):
-    a = fc.numeric_column_v2('aaa')
+    a = fc.numeric_column('aaa')
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column_v2(a, boundaries=None)
+      fc.bucketized_column(a, boundaries=None)
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column_v2(a, boundaries=1.)
+      fc.bucketized_column(a, boundaries=1.)
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column_v2(a, boundaries=[1, 0])
+      fc.bucketized_column(a, boundaries=[1, 0])
     with self.assertRaisesRegexp(
         ValueError, 'boundaries must be a sorted list'):
-      fc.bucketized_column_v2(a, boundaries=[1, 1])
+      fc.bucketized_column(a, boundaries=[1, 1])
 
   def test_name(self):
-    a = fc.numeric_column_v2('aaa', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    a = fc.numeric_column('aaa', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
     self.assertTrue(b._is_v2_column)
     self.assertEqual('aaa_bucketized', b.name)
 
   def test_is_v2_column_old_numeric(self):
-    a = fc_old.numeric_column('aaa', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    a = fc_old._numeric_column('aaa', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
     self.assertFalse(b._is_v2_column)
     self.assertEqual('aaa_bucketized', b.name)
 
   def test_parse_spec(self):
-    a = fc.numeric_column_v2('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
     self.assertEqual({
         'aaa': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32)
     }, b.parse_example_spec)
 
   def test_variable_shape(self):
-    a = fc.numeric_column_v2('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
     # Column 'aaa` has shape [2] times three buckets -> variable_shape=[2, 3].
     self.assertAllEqual((2, 3), b.variable_shape)
 
   def test_num_buckets(self):
-    a = fc.numeric_column_v2('aaa', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    a = fc.numeric_column('aaa', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
     # Column 'aaa` has shape [2] times three buckets -> num_buckets=6.
     self.assertEqual(6, b.num_buckets)
 
   def test_parse_example(self):
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 50])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -534,10 +532,10 @@ class BucketizedColumnTest(test.TestCase):
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
   def test_transform_feature(self):
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
-      transformed_tensor = fc._transform_features({
+      transformed_tensor = fc._transform_features_v2({
           'price': [[-1., 1.], [5., 6.]]
       }, [bucketized_price], None)
       with _initialized_session():
@@ -546,8 +544,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_dense_tensor_one_input_value(self):
     """Tests _get_dense_tensor() for input with shape=[1]."""
-    price = fc.numeric_column_v2('price', shape=[1])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[1])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1.], [1.], [5.], [6.]]
@@ -563,8 +561,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_dense_tensor_two_input_values(self):
     """Tests _get_dense_tensor() for input with shape=[2]."""
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1., 1.], [5., 6.]]
@@ -580,8 +578,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_one_input_value(self):
     """Tests _get_sparse_tensors() for input with shape=[1]."""
-    price = fc.numeric_column_v2('price', shape=[1])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[1])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1.], [1.], [5.], [6.]]
@@ -598,8 +596,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_two_input_values(self):
     """Tests _get_sparse_tensors() for input with shape=[2]."""
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1., 1.], [5., 6.]]
@@ -617,8 +615,8 @@ class BucketizedColumnTest(test.TestCase):
         self.assertAllEqual([2, 2], id_tensor_value.dense_shape)
 
   def test_sparse_tensor_input_not_supported(self):
-    price = fc.numeric_column_v2('price')
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 1])
+    price = fc.numeric_column('price')
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 1])
     transformation_cache = fc.FeatureTransformationCache({
         'price':
             sparse_tensor.SparseTensor(
@@ -628,8 +626,8 @@ class BucketizedColumnTest(test.TestCase):
       bucketized_price.transform_feature(transformation_cache, None)
 
   def test_deep_copy(self):
-    a = fc.numeric_column_v2('aaa', shape=[2])
-    a_bucketized = fc.bucketized_column_v2(a, boundaries=[0, 1])
+    a = fc.numeric_column('aaa', shape=[2])
+    a_bucketized = fc.bucketized_column(a, boundaries=[0, 1])
     a_bucketized_copy = copy.deepcopy(a_bucketized)
     self.assertEqual(a_bucketized_copy.name, 'aaa_bucketized')
     self.assertAllEqual(a_bucketized_copy.variable_shape, (2, 3))
@@ -637,8 +635,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_linear_model_one_input_value(self):
     """Tests linear_model() for input with shape=[1]."""
-    price = fc.numeric_column_v2('price', shape=[1])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[1])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       model = fc.LinearModel([bucketized_price])
@@ -665,8 +663,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1., 1.], [5., 6.]]}
       model = fc.LinearModel([bucketized_price])
@@ -694,8 +692,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_old_linear_model_one_input_value(self):
     """Tests linear_model() for input with shape=[1]."""
-    price = fc.numeric_column_v2('price', shape=[1])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[1])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       predictions = fc_old.linear_model(features, [bucketized_price])
@@ -722,8 +720,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_old_linear_model_two_input_values(self):
     """Tests linear_model() for input with shape=[2]."""
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1., 1.], [5., 6.]]}
       predictions = fc_old.linear_model(features, [bucketized_price])
@@ -751,8 +749,8 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_old_linear_model_one_input_value_old_numeric(self):
     """Tests linear_model() for input with shape=[1]."""
-    price = fc_old.numeric_column('price', shape=[1])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc_old._numeric_column('price', shape=[1])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     with ops.Graph().as_default():
       features = {'price': [[-1.], [1.], [5.], [6.]]}
       predictions = fc_old.linear_model(features, [bucketized_price])
@@ -778,8 +776,8 @@ class BucketizedColumnTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_serialization(self):
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 2, 4, 6])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
     self.assertEqual([price], bucketized_price.parents)
 
     config = bucketized_price._get_config()
@@ -810,7 +808,7 @@ class BucketizedColumnTest(test.TestCase):
 class HashedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
-    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
+    a = fc.categorical_column_with_hash_bucket('aaa', 10)
     self.assertEqual('aaa', a.name)
     self.assertEqual('aaa', a.key)
     self.assertEqual(10, a.hash_bucket_size)
@@ -819,25 +817,25 @@ class HashedCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_hash_bucket_v2(('key',), 10)
+      fc.categorical_column_with_hash_bucket(('key',), 10)
 
   def test_bucket_size_should_be_given(self):
     with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be set.'):
-      fc.categorical_column_with_hash_bucket_v2('aaa', None)
+      fc.categorical_column_with_hash_bucket('aaa', None)
 
   def test_bucket_size_should_be_positive(self):
     with self.assertRaisesRegexp(ValueError,
                                  'hash_bucket_size must be at least 1'):
-      fc.categorical_column_with_hash_bucket_v2('aaa', 0)
+      fc.categorical_column_with_hash_bucket('aaa', 0)
 
   def test_dtype_should_be_string_or_integer(self):
-    fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.string)
-    fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.int32)
+    fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string)
+    fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.float32)
+      fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
+    original = fc.categorical_column_with_hash_bucket('aaa', 10)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(10, column.hash_bucket_size)
@@ -845,19 +843,19 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertEqual(dtypes.string, column.dtype)
 
   def test_parse_spec_string(self):
-    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
+    a = fc.categorical_column_with_hash_bucket('aaa', 10)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.string)
     }, a.parse_example_spec)
 
   def test_parse_spec_int(self):
-    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10, dtype=dtypes.int32)
+    a = fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32)
     self.assertEqual({
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, a.parse_example_spec)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_hash_bucket_v2('aaa', 10)
+    a = fc.categorical_column_with_hash_bucket('aaa', 10)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -878,12 +876,12 @@ class HashedCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_strings_should_be_hashed(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
+    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     wire_tensor = sparse_tensor.SparseTensor(
         values=['omar', 'stringer', 'marlo'],
         indices=[[0, 0], [1, 0], [1, 1]],
         dense_shape=[2, 2])
-    outputs = fc._transform_features({
+    outputs = fc._transform_features_v2({
         'wire': wire_tensor
     }, [hashed_sparse], None)
     output = outputs[hashed_sparse]
@@ -897,11 +895,11 @@ class HashedCategoricalColumnTest(test.TestCase):
                           output.dense_shape.eval())
 
   def test_tensor_dtype_should_be_string_or_integer(self):
-    string_fc = fc.categorical_column_with_hash_bucket_v2(
+    string_fc = fc.categorical_column_with_hash_bucket(
         'a_string', 10, dtype=dtypes.string)
-    int_fc = fc.categorical_column_with_hash_bucket_v2(
+    int_fc = fc.categorical_column_with_hash_bucket(
         'a_int', 10, dtype=dtypes.int32)
-    float_fc = fc.categorical_column_with_hash_bucket_v2(
+    float_fc = fc.categorical_column_with_hash_bucket(
         'a_float', 10, dtype=dtypes.string)
     int_tensor = sparse_tensor.SparseTensor(
         values=[101],
@@ -926,7 +924,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       transformation_cache.get(float_fc, None)
 
   def test_dtype_should_match_with_tensor(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2(
+    hashed_sparse = fc.categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -935,7 +933,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       transformation_cache.get(hashed_sparse, None)
 
   def test_ints_should_be_hashed(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2(
+    hashed_sparse = fc.categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=[101, 201, 301],
@@ -949,7 +947,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, output.values.eval())
 
   def test_int32_64_is_compatible(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2(
+    hashed_sparse = fc.categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
     wire_tensor = sparse_tensor.SparseTensor(
         values=constant_op.constant([101, 201, 301], dtype=dtypes.int32),
@@ -963,7 +961,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, output.values.eval())
 
   def test_get_sparse_tensors(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
+    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     transformation_cache = fc.FeatureTransformationCache({
         'wire':
             sparse_tensor.SparseTensor(
@@ -978,7 +976,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor)
 
   def test_get_sparse_tensors_dense_input(self):
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
+    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     transformation_cache = fc.FeatureTransformationCache({
         'wire': (('omar', ''), ('stringer', 'marlo'))
     })
@@ -989,7 +987,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor)
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
+    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column.num_buckets)
     with ops.Graph().as_default():
       model = fc.LinearModel((wire_column,))
@@ -1012,7 +1010,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
+    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column.num_buckets)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
@@ -1035,7 +1033,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_serialization(self):
-    wire_column = fc.categorical_column_with_hash_bucket_v2('wire', 4)
+    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(['wire'], wire_column.parents)
 
     config = wire_column._get_config()
@@ -1054,104 +1052,104 @@ class CrossedColumnTest(test.TestCase):
   def test_keys_empty(self):
     with self.assertRaisesRegexp(
         ValueError, 'keys must be a list with length > 1'):
-      fc.crossed_column_v2([], 10)
+      fc.crossed_column([], 10)
 
   def test_keys_length_one(self):
     with self.assertRaisesRegexp(
         ValueError, 'keys must be a list with length > 1'):
-      fc.crossed_column_v2(['a'], 10)
+      fc.crossed_column(['a'], 10)
 
   def test_key_type_unsupported(self):
     with self.assertRaisesRegexp(ValueError, 'Unsupported key type'):
-      fc.crossed_column_v2(['a', fc.numeric_column_v2('c')], 10)
+      fc.crossed_column(['a', fc.numeric_column('c')], 10)
 
     with self.assertRaisesRegexp(
         ValueError, 'categorical_column_with_hash_bucket is not supported'):
-      fc.crossed_column_v2(
-          ['a', fc.categorical_column_with_hash_bucket_v2('c', 10)], 10)
+      fc.crossed_column(
+          ['a', fc.categorical_column_with_hash_bucket('c', 10)], 10)
 
   def test_hash_bucket_size_negative(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column_v2(['a', 'c'], -1)
+      fc.crossed_column(['a', 'c'], -1)
 
   def test_hash_bucket_size_zero(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column_v2(['a', 'c'], 0)
+      fc.crossed_column(['a', 'c'], 0)
 
   def test_hash_bucket_size_none(self):
     with self.assertRaisesRegexp(
         ValueError, 'hash_bucket_size must be > 1'):
-      fc.crossed_column_v2(['a', 'c'], None)
+      fc.crossed_column(['a', 'c'], None)
 
   def test_name(self):
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
+    a = fc.numeric_column('a', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
     self.assertTrue(crossed1._is_v2_column)
 
-    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 10)
+    crossed2 = fc.crossed_column([b, 'c', crossed1], 10)
     self.assertTrue(crossed2._is_v2_column)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_is_v2_column(self):
-    a = fc_old.numeric_column('a', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
+    a = fc_old._numeric_column('a', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
     self.assertTrue(crossed1._is_v2_column)
 
-    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 10)
+    crossed2 = fc.crossed_column([b, 'c', crossed1], 10)
     self.assertFalse(crossed2._is_v2_column)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_name_ordered_alphabetically(self):
     """Tests that the name does not depend on the order of given columns."""
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
+    a = fc.numeric_column('a', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
 
-    crossed2 = fc.crossed_column_v2([crossed1, 'c', b], 10)
+    crossed2 = fc.crossed_column([crossed1, 'c', b], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_name_leaf_keys_ordered_alphabetically(self):
     """Tests that the name does not depend on the order of given columns."""
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column_v2(['d2', 'c'], 10)
+    a = fc.numeric_column('a', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column(['d2', 'c'], 10)
 
-    crossed2 = fc.crossed_column_v2([crossed1, 'd1', b], 10)
+    crossed2 = fc.crossed_column([crossed1, 'd1', b], 10)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2.name)
 
   def test_parse_spec(self):
-    a = fc.numeric_column_v2('a', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed = fc.crossed_column_v2([b, 'c'], 10)
+    a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed = fc.crossed_column([b, 'c'], 10)
     self.assertEqual({
         'a': parsing_ops.FixedLenFeature((2,), dtype=dtypes.int32),
         'c': parsing_ops.VarLenFeature(dtypes.string),
     }, crossed.parse_example_spec)
 
   def test_num_buckets(self):
-    a = fc.numeric_column_v2('a', shape=[2], dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed = fc.crossed_column_v2([b, 'c'], 15)
+    a = fc.numeric_column('a', shape=[2], dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed = fc.crossed_column([b, 'c'], 15)
     self.assertEqual(15, crossed.num_buckets)
 
   def test_deep_copy(self):
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32)
-    b = fc.bucketized_column_v2(a, boundaries=[0, 1])
-    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
-    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 15, hash_key=5)
+    a = fc.numeric_column('a', dtype=dtypes.int32)
+    b = fc.bucketized_column(a, boundaries=[0, 1])
+    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
     crossed2_copy = copy.deepcopy(crossed2)
     self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2_copy.name,)
     self.assertEqual(15, crossed2_copy.hash_bucket_size)
     self.assertEqual(5, crossed2_copy.hash_key)
 
   def test_parse_example(self):
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 50])
-    price_cross_wire = fc.crossed_column_v2([bucketized_price, 'wire'], 10)
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
+    price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], 10)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'price':
@@ -1175,11 +1173,11 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval())
 
   def test_transform_feature(self):
-    price = fc.numeric_column_v2('price', shape=[2])
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 50])
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
     hash_bucket_size = 10
-    price_cross_wire = fc.crossed_column_v2([bucketized_price, 'wire'],
-                                            hash_bucket_size)
+    price_cross_wire = fc.crossed_column([bucketized_price, 'wire'],
+                                         hash_bucket_size)
     features = {
         'price': constant_op.constant([[1., 2.], [5., 6.]]),
         'wire': sparse_tensor.SparseTensor(
@@ -1187,7 +1185,7 @@ class CrossedColumnTest(test.TestCase):
             indices=[[0, 0], [1, 0], [1, 1]],
             dense_shape=[2, 2]),
     }
-    outputs = fc._transform_features(features, [price_cross_wire], None)
+    outputs = fc._transform_features_v2(features, [price_cross_wire], None)
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
       output_val = self.evaluate(output)
@@ -1198,10 +1196,10 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([2, 4], output_val.dense_shape)
 
   def test_get_sparse_tensors(self):
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
-    crossed1 = fc.crossed_column_v2(['d1', 'd2'], 10)
-    crossed2 = fc.crossed_column_v2([b, 'c', crossed1], 15, hash_key=5)
+    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed1 = fc.crossed_column(['d1', 'd2'], 10)
+    crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'a':
@@ -1239,9 +1237,9 @@ class CrossedColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_simple(self):
     """Same as test_get_sparse_tensors, but with simpler values."""
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
-    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       transformation_cache = fc.FeatureTransformationCache({
           'a':
@@ -1269,9 +1267,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
-    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       model = fc.LinearModel((crossed,))
       predictions = model({
@@ -1333,7 +1331,7 @@ class CrossedColumnTest(test.TestCase):
             id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
 
     t = _TestColumnWithWeights()
-    crossed = fc.crossed_column_v2([t, 'c'], hash_bucket_size=5, hash_key=5)
+    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
           ValueError,
@@ -1362,9 +1360,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
-    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
           'a':
@@ -1444,7 +1442,7 @@ class CrossedColumnTest(test.TestCase):
             id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
 
     t = _TestColumnWithWeights()
-    crossed = fc.crossed_column_v2([t, 'c'], hash_bucket_size=5, hash_key=5)
+    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
           ValueError,
@@ -1472,9 +1470,9 @@ class CrossedColumnTest(test.TestCase):
 
     Uses data from test_get_sparse_tesnsors_simple.
     """
-    a = fc_old.numeric_column('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
-    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc_old._numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
           'a':
@@ -1499,9 +1497,9 @@ class CrossedColumnTest(test.TestCase):
         self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
   def test_serialization(self):
-    a = fc.numeric_column_v2('a', dtype=dtypes.int32, shape=(2,))
-    b = fc.bucketized_column_v2(a, boundaries=(0, 1))
-    crossed = fc.crossed_column_v2([b, 'c'], hash_bucket_size=5, hash_key=5)
+    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
 
     self.assertEqual([b, 'c'], crossed.parents)
 
@@ -1577,18 +1575,17 @@ class LinearModelTest(test.TestCase):
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
-      fc.LinearModel(feature_columns={'a': fc.numeric_column_v2('a')})
+      fc.LinearModel(feature_columns={'a': fc.numeric_column('a')})
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
-      fc.LinearModel(feature_columns=[
-          fc.numeric_column_v2('a'),
-          fc.numeric_column_v2('a')
-      ])
+      fc.LinearModel(
+          feature_columns=[fc.numeric_column('a'),
+                           fc.numeric_column('a')])
 
   def test_not_dict_input_features(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = [[1.], [5.]]
       model = fc.LinearModel([price])
@@ -1596,7 +1593,7 @@ class LinearModelTest(test.TestCase):
         model(features)
 
   def test_dense_bias(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price])
@@ -1609,7 +1606,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1628,8 +1625,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
-    price = fc.numeric_column_v2('price')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1701,7 +1698,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price], units=3)
@@ -1716,7 +1713,7 @@ class LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1738,7 +1735,7 @@ class LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       model = fc.LinearModel([price])
@@ -1750,7 +1747,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = array_ops.sparse_placeholder(dtypes.string)
       wire_value = sparse_tensor.SparseTensorValue(
@@ -1772,7 +1769,7 @@ class LinearModelTest(test.TestCase):
             predictions.eval(feed_dict={wire_tensor: wire_value}))
 
   def test_sparse_combiner(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -1788,8 +1785,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_sparse_combiner_with_negative_weights(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
-    wire_cast_weights = fc.weighted_categorical_column_v2(wire_cast, 'weights')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights')
 
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
@@ -1809,7 +1806,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       model = fc.LinearModel([price], units=3)
@@ -1824,7 +1821,7 @@ class LinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -1834,7 +1831,7 @@ class LinearModelTest(test.TestCase):
         model(features)
 
   def test_dense_reshaping(self):
-    price = fc.numeric_column_v2('price', shape=[1, 2])
+    price = fc.numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       model = fc.LinearModel([price])
@@ -1848,8 +1845,8 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
@@ -1869,7 +1866,7 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_dense_trainable_default(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price])
@@ -1880,7 +1877,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(price_var, trainable_vars)
 
   def test_sparse_trainable_default(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1893,7 +1890,7 @@ class LinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, trainable_vars)
 
   def test_dense_trainable_false(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       model = fc.LinearModel([price], trainable=False)
@@ -1902,7 +1899,7 @@ class LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_sparse_trainable_false(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -1913,9 +1910,9 @@ class LinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_column_order(self):
-    price_a = fc.numeric_column_v2('price_a')
-    price_b = fc.numeric_column_v2('price_b')
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    price_a = fc.numeric_column('price_a')
+    price_b = fc.numeric_column('price_b')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
@@ -1949,13 +1946,13 @@ class LinearModelTest(test.TestCase):
       self.assertIn('wire_cast', my_vars[2].name)
 
   def test_variable_names(self):
-    price1 = fc.numeric_column_v2('price1')
-    dense_feature = fc.numeric_column_v2('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column_v2(
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)
     all_cols = [price1, dense_feature_bucketized, some_embedding_column]
 
@@ -1979,7 +1976,7 @@ class LinearModelTest(test.TestCase):
       ], variable_names)
 
   def test_fit_and_predict(self):
-    columns = [fc.numeric_column_v2('a')]
+    columns = [fc.numeric_column('a')]
 
     model = fc.LinearModel(columns)
     model.compile(
@@ -1996,8 +1993,8 @@ class LinearModelTest(test.TestCase):
     model.predict(x, batch_size=5)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -2010,9 +2007,9 @@ class LinearModelTest(test.TestCase):
       model(features)
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
-    price3 = fc.numeric_column_v2('price3')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    price3 = fc.numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2026,8 +2023,8 @@ class LinearModelTest(test.TestCase):
         model(features)
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2042,8 +2039,8 @@ class LinearModelTest(test.TestCase):
               predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -2060,14 +2057,14 @@ class LinearModelTest(test.TestCase):
             })
 
   def test_with_numpy_input_fn(self):
-    price = fc.numeric_column_v2('price')
-    price_buckets = fc.bucketized_column_v2(
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     input_fn = numpy_io.numpy_input_fn(
@@ -2098,14 +2095,14 @@ class LinearModelTest(test.TestCase):
       coord.join(threads)
 
   def test_with_1d_sparse_tensor(self):
-    price = fc.numeric_column_v2('price')
-    price_buckets = fc.bucketized_column_v2(
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2132,16 +2129,16 @@ class LinearModelTest(test.TestCase):
                           self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
-    price = fc.numeric_column_v2('price')
-    price_buckets = fc.bucketized_column_v2(
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    country = fc.categorical_column_with_vocabulary_list_v2(
+    country = fc.categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2178,7 +2175,7 @@ class LinearModelTest(test.TestCase):
                               }))
 
   def test_with_rank_0_feature(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -2201,7 +2198,7 @@ class LinearModelTest(test.TestCase):
         sess.run(net, feed_dict={features['price']: np.array(1)})
 
   def test_multiple_linear_models(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features1 = {'price': [[1.], [5.]]}
       features2 = {'price': [[2.], [10.]]}
@@ -2269,21 +2266,18 @@ class OldLinearModelTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc_old.linear_model(
-          features={'a': [[0]]},
-          feature_columns={'a': fc.numeric_column_v2('a')})
+          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
       fc_old.linear_model(
           features={'a': [[0]]},
-          feature_columns=[
-              fc.numeric_column_v2('a'),
-              fc.numeric_column_v2('a')
-          ])
+          feature_columns=[fc.numeric_column('a'),
+                           fc.numeric_column('a')])
 
   def test_dense_bias(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc_old.linear_model(features, [price])
@@ -2296,7 +2290,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[15.], [55.]], self.evaluate(predictions))
 
   def test_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2315,8 +2309,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_and_sparse_bias(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
-    price = fc.numeric_column_v2('price')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2416,7 +2410,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
   def test_dense_multi_output(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       predictions = fc_old.linear_model(features, [price], units=3)
@@ -2431,7 +2425,7 @@ class OldLinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_sparse_multi_output(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2453,7 +2447,7 @@ class OldLinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_dense_multi_dimension(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = fc_old.linear_model(features, [price])
@@ -2464,7 +2458,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_sparse_multi_rank(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = array_ops.sparse_placeholder(dtypes.string)
       wire_value = sparse_tensor.SparseTensorValue(
@@ -2485,7 +2479,7 @@ class OldLinearModelTest(test.TestCase):
             predictions.eval(feed_dict={wire_tensor: wire_value}))
 
   def test_sparse_combiner(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
@@ -2502,8 +2496,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [5010.]], self.evaluate(predictions))
 
   def test_sparse_combiner_with_negative_weights(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
-    wire_cast_weights = fc.weighted_categorical_column_v2(wire_cast, 'weights')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    wire_cast_weights = fc.weighted_categorical_column(wire_cast, 'weights')
 
     with ops.Graph().as_default():
       wire_tensor = sparse_tensor.SparseTensor(
@@ -2524,7 +2518,7 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[1005.], [-9985.]], self.evaluate(predictions))
 
   def test_dense_multi_dimension_multi_output(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       predictions = fc_old.linear_model(features, [price], units=3)
@@ -2539,7 +2533,7 @@ class OldLinearModelTest(test.TestCase):
                             self.evaluate(predictions))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -2548,7 +2542,7 @@ class OldLinearModelTest(test.TestCase):
         fc_old.linear_model(features, [price])
 
   def test_dense_reshaping(self):
-    price = fc.numeric_column_v2('price', shape=[1, 2])
+    price = fc.numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       predictions = fc_old.linear_model(features, [price])
@@ -2562,8 +2556,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[210.], [650.]], self.evaluate(predictions))
 
   def test_dense_multi_column(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       predictions = fc_old.linear_model(features, [price1, price2])
@@ -2581,8 +2575,8 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[3217.], [4657.]], self.evaluate(predictions))
 
   def test_fills_cols_to_vars(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       cols_to_vars = {}
@@ -2595,8 +2589,8 @@ class OldLinearModelTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[price2], [price2_var])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2', shape=3)
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2', shape=3)
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [6., 7.]],
@@ -2622,13 +2616,13 @@ class OldLinearModelTest(test.TestCase):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
     # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
     # creates a Variable.
-    apple_numeric_column = fc.numeric_column_v2('apple_numeric_column')
-    banana_dense_feature = fc.numeric_column_v2('banana_dense_feature')
-    banana_dense_feature_bucketized = fc.bucketized_column_v2(
+    apple_numeric_column = fc.numeric_column('apple_numeric_column')
+    banana_dense_feature = fc.numeric_column('banana_dense_feature')
+    banana_dense_feature_bucketized = fc.bucketized_column(
         banana_dense_feature, boundaries=[0.])
-    cherry_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    cherry_sparse_column = fc.categorical_column_with_hash_bucket(
         'cherry_sparse_feature', hash_bucket_size=5)
-    dragonfruit_embedding_column = fc.embedding_column_v2(
+    dragonfruit_embedding_column = fc.embedding_column(
         cherry_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -2653,7 +2647,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertItemsEqual(input_layer_inputs, output_tensors)
 
   def test_dense_collection(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc_old.linear_model(features, [price], weight_collections=['my-vars'])
@@ -2664,7 +2658,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(price_var, my_vars)
 
   def test_sparse_collection(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2677,7 +2671,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, my_vars)
 
   def test_dense_trainable_default(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc_old.linear_model(features, [price])
@@ -2688,7 +2682,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(price_var, trainable_vars)
 
   def test_sparse_trainable_default(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2701,7 +2695,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn(wire_cast_var, trainable_vars)
 
   def test_dense_trainable_false(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
       fc_old.linear_model(features, [price], trainable=False)
@@ -2709,7 +2703,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_sparse_trainable_false(self):
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
@@ -2719,9 +2713,9 @@ class OldLinearModelTest(test.TestCase):
       self.assertEqual([], trainable_vars)
 
   def test_column_order(self):
-    price_a = fc.numeric_column_v2('price_a')
-    price_b = fc.numeric_column_v2('price_b')
-    wire_cast = fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+    price_a = fc.numeric_column('price_a')
+    price_b = fc.numeric_column('price_b')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
     with ops.Graph().as_default() as g:
       features = {
           'price_a': [[1.]],
@@ -2755,8 +2749,8 @@ class OldLinearModelTest(test.TestCase):
       self.assertIn('wire_cast', my_vars[2].name)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -2768,9 +2762,9 @@ class OldLinearModelTest(test.TestCase):
       fc_old.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
-    price3 = fc.numeric_column_v2('price3')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    price3 = fc.numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2783,8 +2777,8 @@ class OldLinearModelTest(test.TestCase):
         fc_old.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -2798,8 +2792,8 @@ class OldLinearModelTest(test.TestCase):
               predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -2815,14 +2809,14 @@ class OldLinearModelTest(test.TestCase):
             })
 
   def test_with_1d_sparse_tensor(self):
-    price = fc.numeric_column_v2('price')
-    price_buckets = fc.bucketized_column_v2(
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2855,16 +2849,16 @@ class OldLinearModelTest(test.TestCase):
                           self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
-    price = fc.numeric_column_v2('price')
-    price_buckets = fc.bucketized_column_v2(
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(
         price, boundaries=[
             0.,
             10.,
             100.,
         ])
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    country = fc.categorical_column_with_vocabulary_list_v2(
+    country = fc.categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
 
     # Provides 1-dim tensor and dense tensor.
@@ -2900,7 +2894,7 @@ class OldLinearModelTest(test.TestCase):
                               }))
 
   def test_with_rank_0_feature(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -2921,7 +2915,7 @@ class OldLinearModelTest(test.TestCase):
         sess.run(net, feed_dict={features['price']: np.array(1)})
 
   def test_multiple_linear_models(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features1 = {'price': [[1.], [5.]]}
       features2 = {'price': [[2.], [10.]]}
@@ -2942,16 +2936,16 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([[25.], [105.]], self.evaluate(predictions2))
 
   def test_linear_model_v1_shared_embedding_all_other_v2(self):
-    price = fc.numeric_column_v2('price')  # v2
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    price = fc.numeric_column('price')  # v2
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)  # v2
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)  # v2
-    categorical_column_a = fc_old.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)  # v2
-    categorical_column_b = fc_old.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)  # v2
-    shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns(
+    shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)  # v1
     all_cols = [
         price, some_embedding_column, shared_embedding_a, shared_embedding_b
@@ -2978,16 +2972,16 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v1_shared_embedding_with_v2_cat_all_other_v2(self):
-    price = fc.numeric_column_v2('price')  # v2
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    price = fc.numeric_column('price')  # v2
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)  # v2
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)  # v2
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)  # v2
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)  # v2
-    shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns(
+    shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)  # v1
     all_cols = [
         price, some_embedding_column, shared_embedding_a, shared_embedding_b
@@ -3014,16 +3008,16 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v1_v2_mix(self):
-    price = fc.numeric_column_v2('price')  # v2
-    some_sparse_column = fc_old.categorical_column_with_hash_bucket(
+    price = fc.numeric_column('price')  # v2
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)  # v1
-    some_embedding_column = fc_old.embedding_column(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)  # v1
-    categorical_column_a = fc_old.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)  # v2
-    categorical_column_b = fc_old.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)  # v2
-    shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns(
+    shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)  # v1
     all_cols = [
         price, some_embedding_column, shared_embedding_a, shared_embedding_b
@@ -3050,14 +3044,14 @@ class OldLinearModelTest(test.TestCase):
         self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v2_shared_embedding_all_other_v1(self):
-    price = fc_old.numeric_column('price')  # v1
-    some_sparse_column = fc_old.categorical_column_with_hash_bucket(
+    price = fc.numeric_column('price')  # v1
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)  # v1
-    some_embedding_column = fc_old.embedding_column(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)  # v1
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)  # v2
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)  # v2
     shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b], dimension=2)  # v2
@@ -3090,7 +3084,7 @@ class DenseFeaturesTest(test.TestCase):
   @test_util.run_in_graph_and_eager_modes()
   def test_retrieving_input(self):
     features = {'a': [0.]}
-    dense_features = fc.DenseFeatures(fc.numeric_column_v2('a'))
+    dense_features = fc.DenseFeatures(fc.numeric_column('a'))
     inputs = self.evaluate(dense_features(features))
     self.assertAllClose([[0.]], inputs)
 
@@ -3102,7 +3096,7 @@ class DenseFeaturesTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity_v2(
+      categorical_column = fc.categorical_column_with_identity(
           key='a', num_buckets=3)
       embedding_dimension = 2
       def _embedding_column_initializer(shape, dtype, partition_info):
@@ -3115,7 +3109,7 @@ class DenseFeaturesTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column_v2(
+      embedding_column = fc.embedding_column(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -3146,7 +3140,7 @@ class DenseFeaturesTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity_v2(
+      categorical_column = fc.categorical_column_with_identity(
           key='a', num_buckets=3)
       embedding_dimension = 2
 
@@ -3160,7 +3154,7 @@ class DenseFeaturesTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column_v2(
+      embedding_column = fc.embedding_column(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -3192,7 +3186,7 @@ class DenseFeaturesTest(test.TestCase):
   def test_should_be_dense_column(self):
     with self.assertRaisesRegexp(ValueError, 'must be a DenseColumn'):
       fc.DenseFeatures(feature_columns=[
-          fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+          fc.categorical_column_with_hash_bucket('wire_cast', 4)
       ])(
           features={
               'a': [[0]]
@@ -3201,7 +3195,7 @@ class DenseFeaturesTest(test.TestCase):
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
-      fc.DenseFeatures(feature_columns={'a': fc.numeric_column_v2('a')})(
+      fc.DenseFeatures(feature_columns={'a': fc.numeric_column('a')})(
           features={
               'a': [[0]]
           })
@@ -3209,14 +3203,14 @@ class DenseFeaturesTest(test.TestCase):
   def test_bare_column(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
-      net = fc.DenseFeatures(fc.numeric_column_v2('a'))(features)
+      net = fc.DenseFeatures(fc.numeric_column('a'))(features)
       with _initialized_session():
         self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
-      columns = (fc.numeric_column_v2(key) for key in features)
+      columns = (fc.numeric_column(key) for key in features)
       net = fc.DenseFeatures(columns)(features)
       with _initialized_session():
         self.assertAllClose([[0., 1.]], self.evaluate(net))
@@ -3224,16 +3218,15 @@ class DenseFeaturesTest(test.TestCase):
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
-      fc.DenseFeatures(feature_columns=[
-          fc.numeric_column_v2('a'),
-          fc.numeric_column_v2('a')
-      ])(
-          features={
-              'a': [[0]]
-          })
+      fc.DenseFeatures(
+          feature_columns=[fc.numeric_column('a'),
+                           fc.numeric_column('a')])(
+                               features={
+                                   'a': [[0]]
+                               })
 
   def test_one_column(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       net = fc.DenseFeatures([price])(features)
@@ -3241,7 +3234,7 @@ class DenseFeaturesTest(test.TestCase):
         self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc.DenseFeatures([price])(features)
@@ -3249,8 +3242,8 @@ class DenseFeaturesTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_compute_output_shape(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2', shape=4)
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2', shape=4)
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
@@ -3265,7 +3258,7 @@ class DenseFeaturesTest(test.TestCase):
             self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -3274,7 +3267,7 @@ class DenseFeaturesTest(test.TestCase):
         fc.DenseFeatures([price])(features)
 
   def test_reshaping(self):
-    price = fc.numeric_column_v2('price', shape=[1, 2])
+    price = fc.numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc.DenseFeatures([price])(features)
@@ -3282,8 +3275,8 @@ class DenseFeaturesTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1., 2.], [5., 6.]],
@@ -3294,8 +3287,8 @@ class DenseFeaturesTest(test.TestCase):
         self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_cols_to_output_tensors(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       cols_dict = {}
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
@@ -3307,8 +3300,8 @@ class DenseFeaturesTest(test.TestCase):
         self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_column_order(self):
-    price_a = fc.numeric_column_v2('price_a')
-    price_b = fc.numeric_column_v2('price_b')
+    price_a = fc.numeric_column('price_a')
+    price_b = fc.numeric_column('price_b')
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
@@ -3321,7 +3314,7 @@ class DenseFeaturesTest(test.TestCase):
         self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
-    animal = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
+    animal = fc.categorical_column_with_identity('animal', num_buckets=4)
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -3332,8 +3325,8 @@ class DenseFeaturesTest(test.TestCase):
         fc.DenseFeatures([animal])(features)
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -3345,9 +3338,9 @@ class DenseFeaturesTest(test.TestCase):
         fc.DenseFeatures([price1, price2])(features)
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
-    price3 = fc.numeric_column_v2('price3')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    price3 = fc.numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -3360,8 +3353,8 @@ class DenseFeaturesTest(test.TestCase):
         fc.DenseFeatures([price1, price2, price3])(features)
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -3374,8 +3367,8 @@ class DenseFeaturesTest(test.TestCase):
           sess.run(net, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -3391,9 +3384,9 @@ class DenseFeaturesTest(test.TestCase):
             })
 
   def test_multiple_layers_with_same_embedding_column(self):
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)
 
     with ops.Graph().as_default():
@@ -3415,9 +3408,9 @@ class DenseFeaturesTest(test.TestCase):
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
   def test_multiple_layers_with_same_shared_embedding_column(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_b, embedding_column_a = fc.shared_embedding_columns_v2(
@@ -3448,9 +3441,9 @@ class DenseFeaturesTest(test.TestCase):
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
   def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_b, embedding_column_a = fc.shared_embedding_columns_v2(
@@ -3509,13 +3502,13 @@ class DenseFeaturesTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in dense_features
-    price = fc.numeric_column_v2('price')
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    price = fc.numeric_column('price')
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
     # one_hot_body_style has 3 dims in dense_features.
-    one_hot_body_style = fc.indicator_column_v2(body_style)
+    one_hot_body_style = fc.indicator_column(body_style)
     # embedded_body_style has 5 dims in dense_features.
-    embedded_body_style = fc.embedding_column_v2(
+    embedded_body_style = fc.embedding_column(
         body_style, dimension=5, initializer=_initializer)
 
     input_fn = numpy_io.numpy_input_fn(
@@ -3554,17 +3547,17 @@ class DenseFeaturesTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in dense_features
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
 
     # one_hot_body_style has 3 dims in dense_features.
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column_v2(body_style)
+    one_hot_body_style = fc.indicator_column(body_style)
 
     # embedded_body_style has 5 dims in dense_features.
-    country = fc.categorical_column_with_vocabulary_list_v2(
+    country = fc.categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column_v2(
+    embedded_country = fc.embedding_column(
         country, dimension=5, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -3604,17 +3597,17 @@ class DenseFeaturesTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in dense_features
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
 
     # one_hot_body_style has 3 dims in dense_features.
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column_v2(body_style)
+    one_hot_body_style = fc.indicator_column(body_style)
 
     # embedded_body_style has 5 dims in dense_features.
-    country = fc.categorical_column_with_vocabulary_list_v2(
+    country = fc.categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column_v2(
+    embedded_country = fc.embedding_column(
         country, dimension=2, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -3654,7 +3647,7 @@ class DenseFeaturesTest(test.TestCase):
 
   def test_with_rank_0_feature(self):
     # price has 1 dimension in dense_features
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -3680,7 +3673,7 @@ class InputLayerTest(test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def test_retrieving_input(self):
     features = {'a': [0.]}
-    input_layer = fc_old.InputLayer(fc.numeric_column_v2('a'))
+    input_layer = fc_old.InputLayer(fc.numeric_column('a'))
     inputs = self.evaluate(input_layer(features))
     self.assertAllClose([[0.]], inputs)
 
@@ -3692,7 +3685,7 @@ class InputLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity_v2(
+      categorical_column = fc.categorical_column_with_identity(
           key='a', num_buckets=3)
       embedding_dimension = 2
 
@@ -3706,7 +3699,7 @@ class InputLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column_v2(
+      embedding_column = fc.embedding_column(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -3737,7 +3730,7 @@ class InputLayerTest(test.TestCase):
           dense_shape=(3, 3))
 
       # Create feature columns (categorical and embedding).
-      categorical_column = fc.categorical_column_with_identity_v2(
+      categorical_column = fc.categorical_column_with_identity(
           key='a', num_buckets=3)
       embedding_dimension = 2
 
@@ -3751,7 +3744,7 @@ class InputLayerTest(test.TestCase):
             (1, 1))  # id 2
         return embedding_values
 
-      embedding_column = fc.embedding_column_v2(
+      embedding_column = fc.embedding_column(
           categorical_column,
           dimension=embedding_dimension,
           initializer=_embedding_column_initializer)
@@ -3788,27 +3781,26 @@ class FunctionalInputLayerTest(test.TestCase):
       fc_old.input_layer(
           features={'a': [[0]]},
           feature_columns=[
-              fc.categorical_column_with_hash_bucket_v2('wire_cast', 4)
+              fc.categorical_column_with_hash_bucket('wire_cast', 4)
           ])
 
   def test_does_not_support_dict_columns(self):
     with self.assertRaisesRegexp(
         ValueError, 'Expected feature_columns to be iterable, found dict.'):
       fc_old.input_layer(
-          features={'a': [[0]]},
-          feature_columns={'a': fc.numeric_column_v2('a')})
+          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
 
   def test_bare_column(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
-      net = fc_old.input_layer(features, fc.numeric_column_v2('a'))
+      net = fc_old.input_layer(features, fc.numeric_column('a'))
       with _initialized_session():
         self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
-      columns = (fc.numeric_column_v2(key) for key in features)
+      columns = (fc.numeric_column(key) for key in features)
       net = fc_old.input_layer(features, columns)
       with _initialized_session():
         self.assertAllClose([[0., 1.]], self.evaluate(net))
@@ -3818,13 +3810,11 @@ class FunctionalInputLayerTest(test.TestCase):
         ValueError, 'Duplicate feature column name found for columns'):
       fc_old.input_layer(
           features={'a': [[0]]},
-          feature_columns=[
-              fc.numeric_column_v2('a'),
-              fc.numeric_column_v2('a')
-          ])
+          feature_columns=[fc.numeric_column('a'),
+                           fc.numeric_column('a')])
 
   def test_one_column(self):
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       net = fc_old.input_layer(features, [price])
@@ -3832,7 +3822,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc_old.input_layer(features, [price])
@@ -3840,7 +3830,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
-    price = fc.numeric_column_v2('price', shape=2)
+    price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       with self.assertRaisesRegexp(
@@ -3849,7 +3839,7 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [price])
 
   def test_reshaping(self):
-    price = fc.numeric_column_v2('price', shape=[1, 2])
+    price = fc.numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc_old.input_layer(features, [price])
@@ -3857,8 +3847,8 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
-    price1 = fc.numeric_column_v2('price1', shape=2)
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       net = fc_old.input_layer(features, [price1, price2])
@@ -3869,13 +3859,13 @@ class FunctionalInputLayerTest(test.TestCase):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
     # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
     # creates a Variable.
-    price1 = fc.numeric_column_v2('price1')
-    dense_feature = fc.numeric_column_v2('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column_v2(
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -3899,19 +3889,19 @@ class FunctionalInputLayerTest(test.TestCase):
     # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The
     # EmbeddingColumn creates a Variable and the two SharedEmbeddingColumns
     # shared one variable.
-    price1 = fc.numeric_column_v2('price1')
-    dense_feature = fc.numeric_column_v2('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column_v2(
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)
-    categorical_column_a = fc_old.categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc_old.categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
-    shared_embedding_a, shared_embedding_b = fc_old.shared_embedding_columns(
+    shared_embedding_a, shared_embedding_b = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b], dimension=2)
     with ops.Graph().as_default():
       features = {
@@ -3951,13 +3941,13 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[shared_embedding_a][0].shape, [3, 2])
 
   def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column_v2('price1')
-    dense_feature = fc.numeric_column_v2('dense_feature')
-    dense_feature_bucketized = fc.bucketized_column_v2(
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
         dense_feature, boundaries=[0.])
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)
     with ops.Graph().as_default():
       features = {
@@ -3984,8 +3974,8 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10])
 
   def test_column_order(self):
-    price_a = fc.numeric_column_v2('price_a')
-    price_b = fc.numeric_column_v2('price_b')
+    price_a = fc.numeric_column('price_a')
+    price_b = fc.numeric_column('price_b')
     with ops.Graph().as_default():
       features = {
           'price_a': [[1.]],
@@ -3998,7 +3988,7 @@ class FunctionalInputLayerTest(test.TestCase):
         self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
-    animal = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
+    animal = fc.categorical_column_with_identity('animal', num_buckets=4)
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -4009,8 +3999,8 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [animal])
 
   def test_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': [[1.], [5.], [7.]],  # batchsize = 3
@@ -4022,9 +4012,9 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
-    price3 = fc.numeric_column_v2('price3')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    price3 = fc.numeric_column('price3')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -4037,8 +4027,8 @@ class FunctionalInputLayerTest(test.TestCase):
         fc_old.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
@@ -4051,8 +4041,8 @@ class FunctionalInputLayerTest(test.TestCase):
           sess.run(net, feed_dict={features['price1']: [[1.], [5.], [7.]]})
 
   def test_runtime_batch_size_matches(self):
-    price1 = fc.numeric_column_v2('price1')
-    price2 = fc.numeric_column_v2('price2')
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
       features = {
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
@@ -4068,9 +4058,9 @@ class FunctionalInputLayerTest(test.TestCase):
             })
 
   def test_multiple_layers_with_same_embedding_column(self):
-    some_sparse_column = fc.categorical_column_with_hash_bucket_v2(
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
         'sparse_feature', hash_bucket_size=5)
-    some_embedding_column = fc.embedding_column_v2(
+    some_embedding_column = fc.embedding_column(
         some_sparse_column, dimension=10)
 
     with ops.Graph().as_default():
@@ -4103,17 +4093,17 @@ class FunctionalInputLayerTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in input_layer
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
 
     # one_hot_body_style has 3 dims in input_layer.
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column_v2(body_style)
+    one_hot_body_style = fc.indicator_column(body_style)
 
     # embedded_body_style has 5 dims in input_layer.
-    country = fc.categorical_column_with_vocabulary_list_v2(
+    country = fc.categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column_v2(
+    embedded_country = fc.embedding_column(
         country, dimension=5, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -4159,17 +4149,17 @@ class FunctionalInputLayerTest(test.TestCase):
       return embedding_values
 
     # price has 1 dimension in input_layer
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
 
     # one_hot_body_style has 3 dims in input_layer.
-    body_style = fc.categorical_column_with_vocabulary_list_v2(
+    body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
-    one_hot_body_style = fc.indicator_column_v2(body_style)
+    one_hot_body_style = fc.indicator_column(body_style)
 
     # embedded_body_style has 5 dims in input_layer.
-    country = fc.categorical_column_with_vocabulary_list_v2(
+    country = fc.categorical_column_with_vocabulary_list(
         'country', vocabulary_list=['US', 'JP', 'CA'])
-    embedded_country = fc.embedding_column_v2(
+    embedded_country = fc.embedding_column(
         country, dimension=2, initializer=_initializer)
 
     # Provides 1-dim tensor and dense tensor.
@@ -4207,7 +4197,7 @@ class FunctionalInputLayerTest(test.TestCase):
 
   def test_with_rank_0_feature(self):
     # price has 1 dimension in input_layer
-    price = fc.numeric_column_v2('price')
+    price = fc.numeric_column('price')
     features = {
         'price': constant_op.constant(0),
     }
@@ -4245,10 +4235,17 @@ class MakeParseExampleSpecTest(test.TestCase):
     def transform_feature(self, transformation_cache, state_manager):
       pass
 
+    def _transform_feature(self, inputs):
+      pass
+
     @property
     def parse_example_spec(self):
       return self.parse_spec
 
+    @property
+    def _parse_example_spec(self):
+      return self.parse_spec
+
   def test_no_feature_columns(self):
     actual = fc.make_parse_example_spec_v2([])
     self.assertDictEqual({}, actual)
@@ -4358,7 +4355,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     self._wire_vocabulary_size = 3
 
   def test_defaults(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -4370,11 +4367,11 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3)
 
   def test_all_constructor_args(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file='path_to_file',
         vocabulary_size=3,
@@ -4386,7 +4383,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     }, column.parse_example_spec)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_vocabulary_file_v2(
+    original = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file='path_to_file',
         vocabulary_size=3,
@@ -4401,16 +4398,16 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_vocabulary_file_none(self):
     with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa', vocabulary_file=None, vocabulary_size=3)
 
   def test_vocabulary_file_empty_string(self):
     with self.assertRaisesRegexp(ValueError, 'Missing vocabulary_file'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa', vocabulary_file='', vocabulary_size=3)
 
   def test_invalid_vocabulary_file(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4426,18 +4423,18 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_vocabulary_size(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=-1)
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=0)
 
   def test_too_large_vocabulary_size(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size + 1)
@@ -4455,7 +4452,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file='path',
           vocabulary_size=3,
@@ -4463,7 +4460,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype(self):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file='path',
           vocabulary_size=3,
@@ -4472,7 +4469,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
   def test_invalid_buckets_and_default_value(self):
     with self.assertRaisesRegexp(
         ValueError, 'both num_oov_buckets and default_value'):
-      fc.categorical_column_with_vocabulary_file_v2(
+      fc.categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=self._wire_vocabulary_size,
@@ -4480,7 +4477,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           default_value=2)
 
   def test_invalid_input_dtype_int32(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4496,7 +4493,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_invalid_input_dtype_string(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4512,7 +4509,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_file_v2(
+    a = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -4534,7 +4531,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -4557,7 +4554,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_none_vocabulary_size(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file=self._wire_vocabulary_file_name)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4578,7 +4575,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
                                   id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -4586,7 +4583,9 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         indices=((0, 0), (1, 0), (1, 1)),
         values=('marlo', 'skywalker', 'omar'),
         dense_shape=(2, 2))
-    id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column]
+    id_tensor = fc._transform_features_v2({
+        'aaa': inputs
+    }, [column], None)[column]
     with _initialized_session():
       _assert_sparse_tensor_value(
           self,
@@ -4596,7 +4595,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size)
@@ -4615,7 +4614,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4639,7 +4638,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4666,7 +4665,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     # 'marlo' is the last entry in our vocabulary file, so be setting
     # `vocabulary_size` to 1 less than number of entries in file, we take
     # 'marlo' out of the vocabulary.
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size - 1)
@@ -4689,7 +4688,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4714,7 +4713,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4735,7 +4734,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_file_v2(
+    column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
         vocabulary_file=self._warriors_vocabulary_file_name,
         vocabulary_size=self._warriors_vocabulary_size,
@@ -4760,7 +4759,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_file_v2(
+    wire_column = fc.categorical_column_with_vocabulary_file(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4787,7 +4786,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_file_v2(
+    wire_column = fc.categorical_column_with_vocabulary_file(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4814,7 +4813,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
-    wire_column = fc.categorical_column_with_vocabulary_file_v2(
+    wire_column = fc.categorical_column_with_vocabulary_file(
         key='wire',
         vocabulary_file=self._wire_vocabulary_file_name,
         vocabulary_size=self._wire_vocabulary_size,
@@ -4839,7 +4838,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
 class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_defaults_string(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -4851,11 +4850,11 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key=('aaa',), vocabulary_list=('omar', 'stringer', 'marlo'))
 
   def test_defaults_int(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36))
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
@@ -4865,7 +4864,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     }, column.parse_example_spec)
 
   def test_all_constructor_args(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=(12, 24, 36),
         dtype=dtypes.int32,
@@ -4876,7 +4875,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     }, column.parse_example_spec)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_vocabulary_list_v2(
+    original = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
@@ -4887,7 +4886,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype(self):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa',
           vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.float32)
@@ -4895,13 +4894,13 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
   def test_invalid_mapping_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary dtype must be string or integer'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12., 24., 36.))
 
   def test_mismatched_int_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'dtype.*and vocabulary dtype.*do not match'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa',
           vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.int32)
@@ -4909,42 +4908,42 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
   def test_mismatched_string_dtype(self):
     with self.assertRaisesRegexp(
         ValueError, r'dtype.*and vocabulary dtype.*do not match'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.string)
 
   def test_none_mapping(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary_list.*must be non-empty'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=None)
 
   def test_empty_mapping(self):
     with self.assertRaisesRegexp(
         ValueError, r'vocabulary_list.*must be non-empty'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=tuple([]))
 
   def test_duplicate_mapping(self):
     with self.assertRaisesRegexp(ValueError, 'Duplicate keys'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12, 24, 12))
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=-1)
 
   def test_invalid_buckets_and_default_value(self):
     with self.assertRaisesRegexp(
         ValueError, 'both num_oov_buckets and default_value'):
-      fc.categorical_column_with_vocabulary_list_v2(
+      fc.categorical_column_with_vocabulary_list(
           key='aaa',
           vocabulary_list=(12, 24, 36),
           num_oov_buckets=100,
           default_value=2)
 
   def test_invalid_input_dtype_int32(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4957,7 +4956,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_invalid_input_dtype_string(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -4970,7 +4969,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_parse_example_string(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -4992,7 +4991,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_parse_example_int(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(11, 21, 31))
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
@@ -5014,7 +5013,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -5035,13 +5034,15 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('marlo', 'skywalker', 'omar'),
         dense_shape=(2, 2))
-    id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column]
+    id_tensor = fc._transform_features_v2({
+        'aaa': inputs
+    }, [column], None)[column]
     with _initialized_session():
       _assert_sparse_tensor_value(
           self,
@@ -5051,7 +5052,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
@@ -5068,7 +5069,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         default_value=2)
@@ -5091,7 +5092,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=100)
@@ -5114,7 +5115,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32)
@@ -5138,7 +5139,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32,
@@ -5160,7 +5161,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
-    column = fc.categorical_column_with_vocabulary_list_v2(
+    column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=np.array((30, 35, 11, 23, 22), dtype=np.int32),
         dtype=dtypes.int32,
@@ -5184,7 +5185,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_list_v2(
+    wire_column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -5210,7 +5211,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    wire_column = fc.categorical_column_with_vocabulary_list_v2(
+    wire_column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -5236,7 +5237,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
-    wire_column = fc.categorical_column_with_vocabulary_list_v2(
+    wire_column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
         vocabulary_list=('omar', 'stringer', 'marlo'),
         num_oov_buckets=1)
@@ -5260,7 +5261,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
 class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_constructor(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual('aaa', column.name)
     self.assertEqual('aaa', column.key)
     self.assertEqual(3, column.num_buckets)
@@ -5271,10 +5272,10 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_key_should_be_string(self):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
-      fc.categorical_column_with_identity_v2(key=('aaa',), num_buckets=3)
+      fc.categorical_column_with_identity(key=('aaa',), num_buckets=3)
 
   def test_deep_copy(self):
-    original = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    original = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     for column in (original, copy.deepcopy(original)):
       self.assertEqual('aaa', column.name)
       self.assertEqual(3, column.num_buckets)
@@ -5284,24 +5285,24 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_invalid_num_buckets_zero(self):
     with self.assertRaisesRegexp(ValueError, 'num_buckets 0 < 1'):
-      fc.categorical_column_with_identity_v2(key='aaa', num_buckets=0)
+      fc.categorical_column_with_identity(key='aaa', num_buckets=0)
 
   def test_invalid_num_buckets_negative(self):
     with self.assertRaisesRegexp(ValueError, 'num_buckets -1 < 1'):
-      fc.categorical_column_with_identity_v2(key='aaa', num_buckets=-1)
+      fc.categorical_column_with_identity(key='aaa', num_buckets=-1)
 
   def test_invalid_default_value_too_small(self):
     with self.assertRaisesRegexp(ValueError, 'default_value -1 not in range'):
-      fc.categorical_column_with_identity_v2(
+      fc.categorical_column_with_identity(
           key='aaa', num_buckets=3, default_value=-1)
 
   def test_invalid_default_value_too_big(self):
     with self.assertRaisesRegexp(ValueError, 'default_value 3 not in range'):
-      fc.categorical_column_with_identity_v2(
+      fc.categorical_column_with_identity(
           key='aaa', num_buckets=3, default_value=3)
 
   def test_invalid_input_dtype(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=('omar', 'stringer', 'marlo'),
@@ -5313,7 +5314,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           }), None)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=30)
+    a = fc.categorical_column_with_identity(key='aaa', num_buckets=30)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -5334,7 +5335,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_get_sparse_tensors(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
@@ -5354,12 +5355,14 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_transform_feature(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0, 1, 0),
         dense_shape=(2, 2))
-    id_tensor = fc._transform_features({'aaa': inputs}, [column], None)[column]
+    id_tensor = fc._transform_features_v2({
+        'aaa': inputs
+    }, [column], None)[column]
     with _initialized_session():
       _assert_sparse_tensor_value(
           self,
@@ -5369,7 +5372,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': ((0, -1), (1, 0))
@@ -5385,7 +5388,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_inputs_too_small(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(1, -1, 0),
@@ -5401,7 +5404,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         id_weight_pair.id_tensor.eval()
 
   def test_get_sparse_tensors_with_inputs_too_big(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(1, 99, 0),
@@ -5417,7 +5420,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         id_weight_pair.id_tensor.eval()
 
   def test_get_sparse_tensors_with_default_value(self):
-    column = fc.categorical_column_with_identity_v2(
+    column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
@@ -5438,7 +5441,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           id_weight_pair.id_tensor.eval())
 
   def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self):
-    column = fc.categorical_column_with_identity_v2(
+    column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
     input_indices = array_ops.placeholder(dtype=dtypes.int64)
     input_values = array_ops.placeholder(dtype=dtypes.int32)
@@ -5466,7 +5469,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           }))
 
   def test_linear_model(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column.num_buckets)
     with ops.Graph().as_default():
       model = fc.LinearModel((column,))
@@ -5488,7 +5491,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column.num_buckets)
     with ops.Graph().as_default():
       predictions = fc_old.linear_model({
@@ -5510,7 +5513,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
-    column = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
 
     self.assertEqual(['aaa'], column.parents)
 
@@ -5529,9 +5532,9 @@ class TransformFeaturesTest(test.TestCase):
   # All transform tests are distributed in column test.
   # Here we only test multi column case and naming
   def transform_multi_column(self):
-    bucketized_price = fc.bucketized_column_v2(
-        fc.numeric_column_v2('price'), boundaries=[0, 2, 4, 6])
-    hashed_sparse = fc.categorical_column_with_hash_bucket_v2('wire', 10)
+    bucketized_price = fc.bucketized_column(
+        fc.numeric_column('price'), boundaries=[0, 2, 4, 6])
+    hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     with ops.Graph().as_default():
       features = {
           'price': [[-1.], [5.]],
@@ -5541,7 +5544,7 @@ class TransformFeaturesTest(test.TestCase):
                   indices=[[0, 0], [1, 0], [1, 1]],
                   dense_shape=[2, 2])
       }
-      transformed = fc._transform_features(
+      transformed = fc._transform_features_v2(
           features, [bucketized_price, hashed_sparse], None)
       with _initialized_session():
         self.assertIn(bucketized_price.name, transformed[bucketized_price].name)
@@ -5578,12 +5581,12 @@ class TransformFeaturesTest(test.TestCase):
       column1 = _LoggerColumn('1')
       column2 = _LoggerColumn('2')
       call_logger = {'count': 0}
-      fc._transform_features({}, [column1, column2], None)
+      fc._transform_features_v2({}, [column1, column2], None)
       self.assertEqual(0, column1.call_order)
       self.assertEqual(1, column2.call_order)
 
       call_logger = {'count': 0}
-      fc._transform_features({}, [column2, column1], None)
+      fc._transform_features_v2({}, [column2, column1], None)
       self.assertEqual(0, column1.call_order)
       self.assertEqual(1, column2.call_order)
 
@@ -5591,23 +5594,23 @@ class TransformFeaturesTest(test.TestCase):
 class IndicatorColumnTest(test.TestCase):
 
   def test_indicator_column(self):
-    a = fc.categorical_column_with_hash_bucket_v2('a', 4)
-    indicator_a = fc.indicator_column_v2(a)
+    a = fc.categorical_column_with_hash_bucket('a', 4)
+    indicator_a = fc.indicator_column(a)
     self.assertEqual(indicator_a.categorical_column.name, 'a')
     self.assertEqual(indicator_a.name, 'a_indicator')
     self.assertEqual(indicator_a.variable_shape, [1, 4])
     self.assertTrue(indicator_a._is_v2_column)
 
-    b = fc_old.categorical_column_with_hash_bucket('b', hash_bucket_size=100)
-    indicator_b = fc.indicator_column_v2(b)
+    b = fc_old._categorical_column_with_hash_bucket('b', hash_bucket_size=100)
+    indicator_b = fc.indicator_column(b)
     self.assertEqual(indicator_b.categorical_column.name, 'b')
     self.assertEqual(indicator_b.name, 'b_indicator')
     self.assertEqual(indicator_b.variable_shape, [1, 100])
     self.assertFalse(indicator_b._is_v2_column)
 
   def test_1D_shape_succeeds(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_hash_bucket_v2('animal', 4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_hash_bucket('animal', 4))
     transformation_cache = fc.FeatureTransformationCache({
         'animal': ['fox', 'fox']
     })
@@ -5618,8 +5621,8 @@ class IndicatorColumnTest(test.TestCase):
 
   def test_2D_shape_succeeds(self):
     # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready.
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_hash_bucket_v2('animal', 4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_hash_bucket('animal', 4))
     transformation_cache = fc.FeatureTransformationCache({
         'animal':
             sparse_tensor.SparseTensor(
@@ -5633,8 +5636,8 @@ class IndicatorColumnTest(test.TestCase):
                           self.evaluate(output))
 
   def test_multi_hot(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
 
     transformation_cache = fc.FeatureTransformationCache({
         'animal':
@@ -5646,8 +5649,8 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output))
 
   def test_multi_hot2(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
     transformation_cache = fc.FeatureTransformationCache({
         'animal':
             sparse_tensor.SparseTensor(
@@ -5658,17 +5661,17 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
   def test_deep_copy(self):
-    a = fc.categorical_column_with_hash_bucket_v2('a', 4)
-    column = fc.indicator_column_v2(a)
+    a = fc.categorical_column_with_hash_bucket('a', 4)
+    column = fc.indicator_column(a)
     column_copy = copy.deepcopy(column)
     self.assertEqual(column_copy.categorical_column.name, 'a')
     self.assertEqual(column.name, 'a_indicator')
     self.assertEqual(column.variable_shape, [1, 4])
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_indicator = fc.indicator_column_v2(a)
+    a_indicator = fc.indicator_column(a)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -5689,67 +5692,67 @@ class IndicatorColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_transform(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_indicator = fc.indicator_column_v2(a)
+    a_indicator = fc.indicator_column(a)
     features = {
         'aaa': sparse_tensor.SparseTensorValue(
             indices=((0, 0), (1, 0), (1, 1)),
             values=('marlo', 'skywalker', 'omar'),
             dense_shape=(2, 2))
     }
-    indicator_tensor = fc._transform_features(features, [a_indicator],
-                                              None)[a_indicator]
+    indicator_tensor = fc._transform_features_v2(features, [a_indicator],
+                                                 None)[a_indicator]
     with _initialized_session():
       self.assertAllEqual([[0, 0, 1], [1, 0, 0]],
                           self.evaluate(indicator_tensor))
 
   def test_transform_with_weighted_column(self):
     # Github issue 12557
-    ids = fc.categorical_column_with_vocabulary_list_v2(
+    ids = fc.categorical_column_with_vocabulary_list(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    weights = fc.weighted_categorical_column_v2(ids, 'weights')
-    indicator = fc.indicator_column_v2(weights)
+    weights = fc.weighted_categorical_column(ids, 'weights')
+    indicator = fc.indicator_column(weights)
     features = {
-        'ids': constant_op.constant([['c', 'b', 'a']]),
-        'weights': constant_op.constant([[2., 4., 6.]])
+        'ids': constant_op.constant([['c', 'b', 'a', 'c']]),
+        'weights': constant_op.constant([[2., 4., 6., 1.]])
     }
-    indicator_tensor = fc._transform_features(features, [indicator],
-                                              None)[indicator]
+    indicator_tensor = fc._transform_features_v2(features, [indicator],
+                                                 None)[indicator]
     with _initialized_session():
-      self.assertAllEqual([[6., 4., 2.]], self.evaluate(indicator_tensor))
+      self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
-    ids = fc.categorical_column_with_vocabulary_list_v2(
+    ids = fc.categorical_column_with_vocabulary_list(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    weights = fc.weighted_categorical_column_v2(ids, 'weights')
-    indicator = fc.indicator_column_v2(weights)
+    weights = fc.weighted_categorical_column(ids, 'weights')
+    indicator = fc.indicator_column(weights)
     features = {
         'ids': constant_op.constant([['c', 'b', 'unknown']]),
         'weights': constant_op.constant([[2., 4., 6.]])
     }
-    indicator_tensor = fc._transform_features(features, [indicator],
-                                              None)[indicator]
+    indicator_tensor = fc._transform_features_v2(features, [indicator],
+                                                 None)[indicator]
     with _initialized_session():
       self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
-    ids = fc.categorical_column_with_vocabulary_list_v2(
+    ids = fc.categorical_column_with_vocabulary_list(
         key='ids', vocabulary_list=('a', 'b', 'c'))
-    indicator = fc.indicator_column_v2(ids)
+    indicator = fc.indicator_column(ids)
     features = {
         'ids': constant_op.constant([['c', 'b', 'unknown']]),
     }
-    indicator_tensor = fc._transform_features(features, [indicator],
-                                              None)[indicator]
+    indicator_tensor = fc._transform_features_v2(features, [indicator],
+                                                 None)[indicator]
     with _initialized_session():
       self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
   def test_linear_model(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5768,8 +5771,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5787,8 +5790,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
-    animal = fc.indicator_column_v2(
-        fc_old.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc_old._categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5806,8 +5809,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_dense_features(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5819,8 +5822,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_input_layer(self):
-    animal = fc.indicator_column_v2(
-        fc.categorical_column_with_identity_v2('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5832,8 +5835,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_input_layer_old_categorical(self):
-    animal = fc.indicator_column_v2(
-        fc_old.categorical_column_with_identity('animal', num_buckets=4))
+    animal = fc.indicator_column(
+        fc_old._categorical_column_with_identity('animal', num_buckets=4))
     with ops.Graph().as_default():
       features = {
           'animal':
@@ -5845,8 +5848,8 @@ class IndicatorColumnTest(test.TestCase):
         self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_serialization(self):
-    parent = fc.categorical_column_with_identity_v2('animal', num_buckets=4)
-    animal = fc.indicator_column_v2(parent)
+    parent = fc.categorical_column_with_identity('animal', num_buckets=4)
+    animal = fc.indicator_column(parent)
 
     self.assertEqual([parent], animal.parents)
 
@@ -5915,10 +5918,10 @@ class _TestStateManager(fc.StateManager):
 class EmbeddingColumnTest(test.TestCase):
 
   def test_defaults(self):
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column, dimension=embedding_dimension)
     self.assertIs(categorical_column, embedding_column.categorical_column)
     self.assertEqual(embedding_dimension, embedding_column.dimension)
@@ -5935,18 +5938,18 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertTrue(embedding_column._is_v2_column)
 
   def test_is_v2_column(self):
-    categorical_column = fc_old.categorical_column_with_identity(
+    categorical_column = fc_old._categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column, dimension=embedding_dimension)
     self.assertFalse(embedding_column._is_v2_column)
 
   def test_all_constructor_args(self):
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         combiner='my_combiner',
@@ -5969,10 +5972,10 @@ class EmbeddingColumnTest(test.TestCase):
     }, embedding_column.parse_example_spec)
 
   def test_deep_copy(self):
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
     embedding_dimension = 2
-    original = fc.embedding_column_v2(
+    original = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         combiner='my_combiner',
@@ -6001,16 +6004,15 @@ class EmbeddingColumnTest(test.TestCase):
       }, embedding_column.parse_example_spec)
 
   def test_invalid_initializer(self):
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
-      fc.embedding_column_v2(
-          categorical_column, dimension=2, initializer='not_fn')
+      fc.embedding_column(categorical_column, dimension=2, initializer='not_fn')
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_embedded = fc.embedding_column_v2(a, dimension=2)
+    a_embedded = fc.embedding_column(a, dimension=2)
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -6031,15 +6033,15 @@ class EmbeddingColumnTest(test.TestCase):
           features['aaa'].eval())
 
   def test_transform_feature(self):
-    a = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
-    a_embedded = fc.embedding_column_v2(a, dimension=2)
+    a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    a_embedded = fc.embedding_column(a, dimension=2)
     features = {
         'aaa': sparse_tensor.SparseTensor(
             indices=((0, 0), (1, 0), (1, 1)),
             values=(0, 1, 0),
             dense_shape=(2, 2))
     }
-    outputs = fc._transform_features(features, [a, a_embedded], None)
+    outputs = fc._transform_features_v2(features, [a, a_embedded], None)
     output_a = outputs[a]
     output_embedded = outputs[a_embedded]
     with _initialized_session():
@@ -6084,9 +6086,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6146,9 +6148,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc_old.categorical_column_with_identity(
+    categorical_column = fc_old._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6207,9 +6209,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6268,9 +6270,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6339,9 +6341,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         ckpt_to_load_from=ckpt_path,
@@ -6387,9 +6389,9 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6478,9 +6480,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6540,9 +6542,9 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer,
@@ -6602,15 +6604,15 @@ class EmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
 
     # Provide sparse input and get dense result.
-    dense_features = fc_old.input_layer({
+    feature_layer = fc_old.input_layer({
         'aaa': sparse_input
     }, (embedding_column,))
 
@@ -6623,7 +6625,7 @@ class EmbeddingColumnTest(test.TestCase):
                           tuple([v.name for v in trainable_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
+      self.assertAllEqual(expected_lookups, self.evaluate(feature_layer))
 
   def test_old_linear_model(self):
     # Inputs.
@@ -6650,9 +6652,9 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6729,9 +6731,9 @@ class EmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column = fc_old.categorical_column_with_identity(
+    categorical_column = fc_old._categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column,
         dimension=embedding_dimension,
         initializer=_initializer)
@@ -6790,9 +6792,9 @@ class EmbeddingColumnTest(test.TestCase):
       return ValueError('Not expected to be called')
 
     # Build columns.
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    embedding_column = fc.embedding_column_v2(
+    embedding_column = fc.embedding_column(
         categorical_column, dimension=2, initializer=_initializer)
 
     self.assertEqual([categorical_column], embedding_column.parents)
@@ -6837,9 +6839,9 @@ class EmbeddingColumnTest(test.TestCase):
 class SharedEmbeddingColumnTest(test.TestCase):
 
   def test_defaults(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_b, embedding_column_a = fc.shared_embedding_columns_v2(
@@ -6861,9 +6863,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }, embedding_column_b.parse_example_spec)
 
   def test_all_constructor_args(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
@@ -6892,9 +6894,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
     }, embedding_column_b.parse_example_spec)
 
   def test_deep_copy(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_dimension = 2
     original_a, _ = fc.shared_embedding_columns_v2(
@@ -6923,9 +6925,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       }, embedding_column_a.parse_example_spec)
 
   def test_invalid_initializer(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
       fc.shared_embedding_columns_v2(
@@ -6934,11 +6936,11 @@ class SharedEmbeddingColumnTest(test.TestCase):
           initializer='not_fn')
 
   def test_incompatible_column_type(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
-    categorical_column_c = fc.categorical_column_with_hash_bucket_v2(
+    categorical_column_c = fc.categorical_column_with_hash_bucket(
         key='ccc', hash_bucket_size=3)
     with self.assertRaisesRegexp(
         ValueError, 'all categorical_columns must have the same type.*'
@@ -6948,13 +6950,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
           dimension=2)
 
   def test_weighted_categorical_column_ok(self):
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    weighted_categorical_column_a = fc.weighted_categorical_column_v2(
+    weighted_categorical_column_a = fc.weighted_categorical_column(
         categorical_column_a, weight_feature_key='aaa_weights')
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
-    weighted_categorical_column_b = fc.weighted_categorical_column_v2(
+    weighted_categorical_column_b = fc.weighted_categorical_column(
         categorical_column_b, weight_feature_key='bbb_weights')
     fc.shared_embedding_columns_v2(
         [weighted_categorical_column_a, categorical_column_b], dimension=2)
@@ -6965,9 +6967,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
         dimension=2)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    b = fc.categorical_column_with_vocabulary_list_v2(
+    b = fc.categorical_column_with_vocabulary_list(
         key='bbb', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2)
     data = example_pb2.Example(features=feature_pb2.Features(
@@ -7001,8 +7003,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
           features['bbb'].eval())
 
   def test_transform_feature(self):
-    a = fc.categorical_column_with_identity_v2(key='aaa', num_buckets=3)
-    b = fc.categorical_column_with_identity_v2(key='bbb', num_buckets=3)
+    a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    b = fc.categorical_column_with_identity(key='bbb', num_buckets=3)
     a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2)
     features = {
         'aaa': sparse_tensor.SparseTensor(
@@ -7014,8 +7016,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
             values=(1, 2, 1),
             dense_shape=(2, 2)),
     }
-    outputs = fc._transform_features(features, [a, a_embedded, b, b_embedded],
-                                     None)
+    outputs = fc._transform_features_v2(features,
+                                        [a, a_embedded, b, b_embedded], None)
     output_a = outputs[a]
     output_a_embedded = outputs[a_embedded]
     output_b = outputs[b]
@@ -7069,9 +7071,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7132,9 +7134,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7173,9 +7175,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       return zeros_embedding_values
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7299,13 +7301,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
     )
 
     # Build columns.
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    categorical_column_c = fc.categorical_column_with_identity_v2(
+    categorical_column_c = fc.categorical_column_with_identity(
         key='ccc', num_buckets=vocabulary_size)
-    categorical_column_d = fc.categorical_column_with_identity_v2(
+    categorical_column_d = fc.categorical_column_with_identity(
         key='ddd', num_buckets=vocabulary_size)
 
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
@@ -7363,9 +7365,9 @@ class SharedEmbeddingColumnTest(test.TestCase):
       del shape, dtype, partition_info
       return ValueError('Not expected to be called')
 
-    categorical_column_a = fc.categorical_column_with_identity_v2(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    categorical_column_b = fc.categorical_column_with_identity_v2(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=3)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b],
@@ -7381,8 +7383,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
 class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     self.assertEqual('ids_weighted_by_values', column.name)
@@ -7394,16 +7396,16 @@ class WeightedCategoricalColumnTest(test.TestCase):
     self.assertTrue(column._is_v2_column)
 
   def test_is_v2_column(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc_old.categorical_column_with_identity(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc_old._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     self.assertFalse(column._is_v2_column)
 
   def test_deep_copy(self):
     """Tests deepcopy of categorical_column_with_hash_bucket."""
-    original = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    original = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     for column in (original, copy.deepcopy(original)):
@@ -7416,23 +7418,23 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_invalid_dtype_none(self):
     with self.assertRaisesRegexp(ValueError, 'is not convertible to float'):
-      fc.weighted_categorical_column_v2(
-          categorical_column=fc.categorical_column_with_identity_v2(
+      fc.weighted_categorical_column(
+          categorical_column=fc.categorical_column_with_identity(
               key='ids', num_buckets=3),
           weight_feature_key='values',
           dtype=None)
 
   def test_invalid_dtype_string(self):
     with self.assertRaisesRegexp(ValueError, 'is not convertible to float'):
-      fc.weighted_categorical_column_v2(
-          categorical_column=fc.categorical_column_with_identity_v2(
+      fc.weighted_categorical_column(
+          categorical_column=fc.categorical_column_with_identity(
               key='ids', num_buckets=3),
           weight_feature_key='values',
           dtype=dtypes.string)
 
   def test_invalid_input_dtype(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     strings = sparse_tensor.SparseTensorValue(
@@ -7440,21 +7442,21 @@ class WeightedCategoricalColumnTest(test.TestCase):
         values=('omar', 'stringer', 'marlo'),
         dense_shape=(2, 2))
     with self.assertRaisesRegexp(ValueError, 'Bad dtype'):
-      fc._transform_features({
+      fc._transform_features_v2({
           'ids': strings,
           'values': strings
       }, (column,), None)
 
   def test_column_name_collision(self):
     with self.assertRaisesRegexp(ValueError, r'Parse config.*already exists'):
-      fc.weighted_categorical_column_v2(
-          categorical_column=fc.categorical_column_with_identity_v2(
+      fc.weighted_categorical_column(
+          categorical_column=fc.categorical_column_with_identity(
               key='aaa', num_buckets=3),
           weight_feature_key='aaa').parse_example_spec()
 
   def test_missing_weights(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -7463,13 +7465,12 @@ class WeightedCategoricalColumnTest(test.TestCase):
         dense_shape=(2, 2))
     with self.assertRaisesRegexp(
         ValueError, 'values is not in features dictionary'):
-      fc._transform_features({'ids': inputs}, (column,), None)
+      fc._transform_features_v2({'ids': inputs}, (column,), None)
 
   def test_parse_example(self):
-    a = fc.categorical_column_with_vocabulary_list_v2(
+    a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    a_weighted = fc.weighted_categorical_column_v2(
-        a, weight_feature_key='weights')
+    a_weighted = fc.weighted_categorical_column(a, weight_feature_key='weights')
     data = example_pb2.Example(features=feature_pb2.Features(
         feature={
             'aaa':
@@ -7501,8 +7502,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           features['weights'].eval())
 
   def test_transform_features(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
@@ -7513,7 +7514,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0.5, 1.0, 0.1),
         dense_shape=(2, 2))
-    id_tensor, weight_tensor = fc._transform_features({
+    id_tensor, weight_tensor = fc._transform_features_v2({
         'ids': inputs,
         'values': weights,
     }, (column,), None)[column]
@@ -7532,15 +7533,15 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_input(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     weights = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0.5, 1.0, 0.1),
         dense_shape=(2, 2))
-    id_tensor, weight_tensor = fc._transform_features({
+    id_tensor, weight_tensor = fc._transform_features_v2({
         'ids': ((0, -1), (1, 0)),
         'values': weights,
     }, (column,), None)[column]
@@ -7559,15 +7560,15 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_weights(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(2, 1, 0),
         dense_shape=(2, 2))
-    id_tensor, weight_tensor = fc._transform_features({
+    id_tensor, weight_tensor = fc._transform_features_v2({
         'ids': inputs,
         'values': ((.5, 0.), (1., .1)),
     }, (column,), None)[column]
@@ -7586,8 +7587,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
   def test_linear_model(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7616,8 +7617,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model_mismatched_shape(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7638,8 +7639,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         })
 
   def test_linear_model_mismatched_dense_values(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7662,8 +7663,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           self.evaluate(predictions)
 
   def test_linear_model_mismatched_dense_shape(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7688,8 +7689,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7718,8 +7719,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model_mismatched_shape(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7739,8 +7740,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         }, (column,))
 
   def test_old_linear_model_mismatched_dense_values(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7763,8 +7764,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
           self.evaluate(predictions)
 
   def test_old_linear_model_mismatched_dense_shape(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc.categorical_column_with_identity_v2(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7789,8 +7790,8 @@ class WeightedCategoricalColumnTest(test.TestCase):
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
-    column = fc.weighted_categorical_column_v2(
-        categorical_column=fc_old.categorical_column_with_identity(
+    column = fc.weighted_categorical_column(
+        categorical_column=fc_old._categorical_column_with_identity(
             key='ids', num_buckets=3),
         weight_feature_key='values')
     with ops.Graph().as_default():
@@ -7821,9 +7822,9 @@ class WeightedCategoricalColumnTest(test.TestCase):
   # TODO(ptucker): Add test with embedding of weighted categorical.
 
   def test_serialization(self):
-    categorical_column = fc.categorical_column_with_identity_v2(
+    categorical_column = fc.categorical_column_with_identity(
         key='ids', num_buckets=3)
-    column = fc.weighted_categorical_column_v2(
+    column = fc.weighted_categorical_column(
         categorical_column=categorical_column, weight_feature_key='weight')
 
     self.assertEqual([categorical_column, 'weight'], column.parents)
@@ -7907,8 +7908,8 @@ class SerializationTest(test.TestCase):
       })
 
   def test_deserialization_deduping(self):
-    price = fc.numeric_column_v2('price')
-    bucketized_price = fc.bucketized_column_v2(price, boundaries=[0, 1])
+    price = fc.numeric_column('price')
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 1])
 
     configs = fc.serialize_feature_columns([price, bucketized_price])
 
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index c187b1f8e5..2ceef6ba50 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -49,7 +49,7 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_sequential_model(self):
-    columns = [fc.numeric_column_v2('a')]
+    columns = [fc.numeric_column('a')]
     model = keras.models.Sequential([
         fc.DenseFeatures(columns),
         keras.layers.Dense(64, activation='relu'),
@@ -70,7 +70,7 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_sequential_model_with_ds_input(self):
-    columns = [fc.numeric_column_v2('a')]
+    columns = [fc.numeric_column('a')]
     model = keras.models.Sequential([
         fc.DenseFeatures(columns),
         keras.layers.Dense(64, activation='relu'),
@@ -94,8 +94,8 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_subclassed_model_with_feature_columns(self):
-    col_a = fc.numeric_column_v2('a')
-    col_b = fc.numeric_column_v2('b')
+    col_a = fc.numeric_column('a')
+    col_b = fc.numeric_column('b')
 
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
@@ -114,8 +114,8 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_subclassed_model_with_feature_columns_with_ds_input(self):
-    col_a = fc.numeric_column_v2('a')
-    col_b = fc.numeric_column_v2('b')
+    col_a = fc.numeric_column('a')
+    col_b = fc.numeric_column('b')
 
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
@@ -137,8 +137,8 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def DISABLED_test_function_model_feature_layer_input(self):
-    col_a = fc.numeric_column_v2('a')
-    col_b = fc.numeric_column_v2('b')
+    col_a = fc.numeric_column('a')
+    col_b = fc.numeric_column('b')
 
     feature_layer = fc.DenseFeatures([col_a, col_b], name='fc')
     dense = keras.layers.Dense(4)
@@ -163,9 +163,9 @@ class FeatureColumnsIntegrationTest(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def DISABLED_test_function_model_multiple_feature_layer_inputs(self):
-    col_a = fc.numeric_column_v2('a')
-    col_b = fc.numeric_column_v2('b')
-    col_c = fc.numeric_column_v2('c')
+    col_a = fc.numeric_column('a')
+    col_b = fc.numeric_column('b')
+    col_c = fc.numeric_column('c')
 
     fc1 = fc.DenseFeatures([col_a, col_b], name='fc1')
     fc2 = fc.DenseFeatures([col_b, col_c], name='fc2')
-- 
GitLab


From 4a818c7f7bb44587ec55958b180452a6d01afe6e Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 19 Nov 2018 11:00:27 -0800
Subject: [PATCH 0496/1554] Disable FusedConv2D if Tensorflow compiled with
 xsmm

PiperOrigin-RevId: 222103775
---
 .../core/grappler/optimizers/remapper.cc      | 19 ++++++++++++++++
 .../core/grappler/optimizers/remapper_test.cc | 22 ++++++++++++++++++-
 tensorflow/core/kernels/conv_ops_fused.cc     |  4 +++-
 3 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index 5282b08066..d8e62e0b24 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -36,6 +36,15 @@ constexpr char kFusedConv2D[] = "_FusedConv2D";
 constexpr char kDataFormat[] = "data_format";
 constexpr char kIsTraining[] = "is_training";
 
+// TODO(b/119765980): Upgrade upstream Eigen to set `m_can_use_xsmm=false` for
+// contractions with non-default contraction output kernels.
+bool EigenSupportsContractionOutputKernel() {
+#if defined(EIGEN_USE_LIBXSMM)
+  return false;
+#endif
+  return true;
+}
+
 struct RemapperContext {
   explicit RemapperContext(const GrapplerItem& item)
       : nodes_to_preserve(item.NodesToPreserve()),
@@ -116,6 +125,8 @@ bool IsInPreserveSet(const RemapperContext& ctx, const NodeDef* node) {
 
 bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* node,
                         Conv2DWithBiasAdd* matched) {
+  if (!EigenSupportsContractionOutputKernel()) return false;
+
   // Root of the pattern must be a BiasAdd.
   if (!node) return false;
   if (!IsBiasAdd(*node)) return false;
@@ -144,6 +155,8 @@ bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* node,
 
 bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* node,
                                Conv2DWithBiasAddAndRelu* matched) {
+  if (!EigenSupportsContractionOutputKernel()) return false;
+
   // Root of the pattern must be a Relu.
   if (!node) return false;
   if (!IsRelu(*node)) return false;
@@ -172,6 +185,8 @@ bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* node,
 bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
                                   const NodeDef* node,
                                   Conv2DWithSqueezeAndBiasAdd* matched) {
+  if (!EigenSupportsContractionOutputKernel()) return false;
+
   // Root of the pattern must be a BiasAdd.
   if (node == nullptr) return false;
   if (node->op() != "BiasAdd") return false;
@@ -219,6 +234,8 @@ bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
 
 bool FindConv2DWithBatchNorm(const RemapperContext& ctx, const NodeDef* node,
                              Conv2DWithBatchNorm* matched) {
+  if (!EigenSupportsContractionOutputKernel()) return false;
+
   // Root of the pattern must be a FusedBatchNorm or a FusedBatchNormV2.
   if (node == nullptr) return false;
   if (!IsFusedBatchNorm(*node)) return false;
@@ -263,6 +280,8 @@ bool FindConv2DWithBatchNorm(const RemapperContext& ctx, const NodeDef* node,
 bool FindConv2DWithBatchNormAndRelu(const RemapperContext& ctx,
                                     const NodeDef* node,
                                     Conv2DWithBatchNormAndRelu* matched) {
+  if (!EigenSupportsContractionOutputKernel()) return false;
+
   // Root of the pattern must be a Relu.
   if (node == nullptr) return false;
   if (!IsRelu(*node)) return false;
diff --git a/tensorflow/core/grappler/optimizers/remapper_test.cc b/tensorflow/core/grappler/optimizers/remapper_test.cc
index 1711c3dbcd..ffc242decc 100644
--- a/tensorflow/core/grappler/optimizers/remapper_test.cc
+++ b/tensorflow/core/grappler/optimizers/remapper_test.cc
@@ -24,7 +24,17 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-class RemapperTest : public GrapplerTest {};
+class RemapperTest : public GrapplerTest {
+ protected:
+  // TODO(b/119765980): Upgrade upstream Eigen to set `m_can_use_xsmm=false` for
+  // contractions with non-default contraction output kernels.
+  bool EigenSupportsContractionOutputKernel() {
+#if defined(EIGEN_USE_LIBXSMM)
+    return false;
+#endif
+    return true;
+  }
+};
 
 TEST_F(RemapperTest, FusedBatchNorm) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -92,6 +102,8 @@ TEST_F(RemapperTest, FusedBatchNormNCHW) {
 }
 
 TEST_F(RemapperTest, FuseConv2DWithBias) {
+  if (!EigenSupportsContractionOutputKernel()) return;
+
   using ::tensorflow::ops::Placeholder;
 
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -153,6 +165,8 @@ TEST_F(RemapperTest, FuseConv2DWithBias) {
 }
 
 TEST_F(RemapperTest, FuseConv2DWithBiasAndRelu) {
+  if (!EigenSupportsContractionOutputKernel()) return;
+
   using ::tensorflow::ops::Placeholder;
 
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -216,6 +230,8 @@ TEST_F(RemapperTest, FuseConv2DWithBiasAndRelu) {
 }
 
 TEST_F(RemapperTest, FuseConv2DWithBatchNorm) {
+  if (!EigenSupportsContractionOutputKernel()) return;
+
   using ops::Placeholder;
 
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -291,6 +307,8 @@ TEST_F(RemapperTest, FuseConv2DWithBatchNorm) {
 }
 
 TEST_F(RemapperTest, FuseConv2DWithBatchNormAndRelu) {
+  if (!EigenSupportsContractionOutputKernel()) return;
+
   using ops::Placeholder;
 
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -368,6 +386,8 @@ TEST_F(RemapperTest, FuseConv2DWithBatchNormAndRelu) {
 }
 
 TEST_F(RemapperTest, FuseConv2DWithSqueezeAndBias) {
+  if (!EigenSupportsContractionOutputKernel()) return;
+
   using ops::Placeholder;
 
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
diff --git a/tensorflow/core/kernels/conv_ops_fused.cc b/tensorflow/core/kernels/conv_ops_fused.cc
index 4f30d625e0..a0484e9235 100644
--- a/tensorflow/core/kernels/conv_ops_fused.cc
+++ b/tensorflow/core/kernels/conv_ops_fused.cc
@@ -1366,7 +1366,9 @@ class FusedConv2DOp : public OpKernel {
 
 // If we're using the alternative GEMM-based implementation of Conv2D for the
 // CPU implementation, don't register this EigenTensor-based version.
-#if !defined(USE_GEMM_FOR_CONV)
+// TODO(b/119765980): Upgrade upstream Eigen to set `m_can_use_xsmm=false` for
+// contractions with non-default contraction output kernels.
+#if !defined(USE_GEMM_FOR_CONV) && !defined(EIGEN_USE_LIBXSMM)
 TF_CALL_float(REGISTER_FUSED_CONV2D);
 TF_CALL_double(REGISTER_FUSED_CONV2D);
 #endif  // !USE_GEMM_FOR_CONV
-- 
GitLab


From edf3518e1a76be5b9186e13957ed5b186b4e1757 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 19 Nov 2018 11:01:00 -0800
Subject: [PATCH 0497/1554] Add a "serving" tag to the MetaGraph exported from
 tf.saved_model.save

Apparently saved_model_cli will choke on a SavedModel with an un-tagged MetaGraph. I don't think the tag will be particularly meaningful since we'll always export one MetaGraph, but hopefully this will help usability a bit.

PiperOrigin-RevId: 222103865
---
 tensorflow/python/saved_model/save.py      | 2 ++
 tensorflow/python/saved_model/save_test.py | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 63575f631e..02c8dc7c13 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
+from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.saved_model import utils_impl
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import util
@@ -526,6 +527,7 @@ def save(obj, export_dir, signatures=None):
   saved_model.saved_model_schema_version = (
       constants.SAVED_MODEL_SCHEMA_VERSION)
   meta_graph_def = saved_model.meta_graphs.add()
+  meta_graph_def.meta_info_def.tags.append(tag_constants.SERVING)
   meta_graph_def.saver_def.CopyFrom(saver_def)
   # TODO(allenl): Factor out some subset of SavedModelBuilder which is 2.x
   # compatible (no sessions) and share it with this export API rather than
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 42ff508b38..04cd9d0683 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -36,6 +36,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import loader
 from tensorflow.python.saved_model import save
 from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import adam
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
@@ -68,7 +69,7 @@ class SaveTest(test.TestCase):
     """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
     graph = ops.Graph()
     with graph.as_default(), self.session(graph) as session:
-      model = loader.load(session, [], save_dir)
+      model = loader.load(session, [tag_constants.SERVING], save_dir)
       signature = model.signature_def[signature_key]
       self.assertEqual(set(inputs.keys()), set(signature.inputs.keys()))
       feed_dict = {}
@@ -246,7 +247,7 @@ class SaveTest(test.TestCase):
     save.save(to_save, save_dir)
     graph = ops.Graph()
     with graph.as_default(), self.session(graph) as session:
-      loader.load(session, [], save_dir)
+      loader.load(session, [tag_constants.SERVING], save_dir)
       func, = graph._functions.values()
       complex_node, = [
           node for node in func.definition.node_def if node.op == "Complex"]
-- 
GitLab


From 0c5cc9db663fb3b77756dd601a6a6b86a54f02c4 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Mon, 19 Nov 2018 11:01:14 -0800
Subject: [PATCH 0498/1554] Change sparse.reduce_sum(sp_input, axis, keepdims,
 reduction_axes, keep_dims)   to sparse.reduce_sum(sp_input, axis, keepdims,
 output_is_sparse, name)

and unify with reduce_sum_sparse via dispatching (see new output_is_sparse_arg).

PiperOrigin-RevId: 222103917
---
 tensorflow/python/ops/sparse_ops.py           | 73 ++++++++++++++++++-
 .../api/golden/v2/tensorflow.sparse.pbtxt     |  6 +-
 2 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index a65778b9aa..dc091336b2 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -993,7 +993,6 @@ def sparse_reduce_max_v2(
       `Tensor` (the default).
     name: A name for the operation (optional).
 
-
   Returns:
     The reduced Tensor or the reduced SparseTensor if `output_is_sparse` is
     True.
@@ -1138,7 +1137,74 @@ def sparse_reduce_max_sparse(sp_input,
   return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
 
 
-@tf_export("sparse.reduce_sum", v1=["sparse.reduce_sum", "sparse_reduce_sum"])
+@tf_export("sparse.reduce_sum", v1=[])
+def sparse_reduce_sum_v2(
+    sp_input, axis=None, keepdims=None, output_is_sparse=False, name=None):
+  """Computes the sum of elements across dimensions of a SparseTensor.
+
+  This Op takes a SparseTensor and is the sparse counterpart to
+  `tf.reduce_sum()`.  In particular, this Op also returns a dense `Tensor`
+  if `output_is_sparse` is `False`, or a `SparseTensor` if `output_is_sparse`
+  is `True`.
+
+  Note: if `output_is_sparse` is True, a gradient is not defined for this
+  function, so it can't be used in training models that need gradient descent.
+
+  Reduces `sp_input` along the dimensions given in `axis`.  Unless `keepdims` is
+  true, the rank of the tensor is reduced by 1 for each entry in `axis`. If
+  `keepdims` is true, the reduced dimensions are retained with length 1.
+
+  If `axis` has no entries, all dimensions are reduced, and a tensor
+  with a single element is returned.  Additionally, the axes can be negative,
+  similar to the indexing rules in Python.
+
+  For example:
+
+  ```python
+  # 'x' represents [[1, ?, 1]
+  #                 [?, 1, ?]]
+  # where ? is implicitly-zero.
+  tf.sparse.reduce_sum(x) ==> 3
+  tf.sparse.reduce_sum(x, 0) ==> [1, 1, 1]
+  tf.sparse.reduce_sum(x, 1) ==> [2, 1]  # Can also use -1 as the axis.
+  tf.sparse.reduce_sum(x, 1, keepdims=True) ==> [[2], [1]]
+  tf.sparse.reduce_sum(x, [0, 1]) ==> 3
+  ```
+
+  Args:
+    sp_input: The SparseTensor to reduce. Should have numeric type.
+    axis: The dimensions to reduce; list or scalar. If `None` (the
+      default), reduces all dimensions.
+    keepdims: If true, retain reduced dimensions with length 1.
+    output_is_sparse: If true, returns a `SparseTensor` instead of a dense
+      `Tensor` (the default).
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced Tensor or the reduced SparseTensor if `output_is_sparse` is
+    True.
+  """
+  if keepdims is None:
+    keepdims = False
+
+  # reduction_axes is the deprecated name for axis.
+  reduction_axes = None
+
+  if output_is_sparse:
+    output_ind, output_val, output_shape = (
+        gen_sparse_ops.sparse_reduce_sum_sparse(
+            sp_input.indices, sp_input.values, sp_input.dense_shape,
+            math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims,
+            name=name))
+    return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
+
+  return gen_sparse_ops.sparse_reduce_sum(
+      sp_input.indices, sp_input.values, sp_input.dense_shape,
+      math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims,
+      name=name)
+
+
+@tf_export(v1=["sparse.reduce_sum", "sparse_reduce_sum"])
 @deprecation.deprecated_endpoints("sparse_reduce_sum")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
@@ -1193,8 +1259,7 @@ def sparse_reduce_sum(sp_input, axis=None, keepdims=None,
       math_ops._ReductionDims(sp_input, axis, reduction_axes), keepdims)
 
 
-@tf_export("sparse.reduce_sum_sparse",
-           v1=["sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse"])
+@tf_export(v1=["sparse.reduce_sum_sparse", "sparse_reduce_sum_sparse"])
 @deprecation.deprecated_endpoints("sparse_reduce_sum_sparse")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index 161744a914..960d1ba425 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -62,11 +62,7 @@ tf_module {
   }
   member_method {
     name: "reduce_sum"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "reduce_sum_sparse"
-    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'reduction_axes\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'sp_input\', \'axis\', \'keepdims\', \'output_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reorder"
-- 
GitLab


From 38c9b12464b23aac132fbc8005cb74de86eee241 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 11:09:09 -0800
Subject: [PATCH 0499/1554] Run gRPC worker parsing and async callbacks in a
 threadpool, instead of in the RPC handler thread.

gRPC assigns a single thread to service the CompletionQueue of associated
with each peer.  Typically the parsing overhead of the response is fairly
high, and there's no limit on the computational duration of the caller
supplied callback.  By moving this work to another thread the RPC handler
can get back to unblocking incoming bits more quickly.

PiperOrigin-RevId: 222105449
---
 .../rpc/eager/grpc_eager_client.cc            |  2 +-
 .../rpc/grpc_remote_worker.cc                 | 14 ++++++--
 .../rpc/grpc_remote_worker.h                  |  3 +-
 .../rpc/grpc_rpc_factory.cc                   | 10 +++---
 .../core/distributed_runtime/rpc/grpc_state.h | 36 ++++++++++++++-----
 .../rpc/grpc_worker_cache.cc                  | 16 +++++++--
 6 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc
index 181422118c..3626a48171 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc
@@ -40,7 +40,7 @@ class GrpcEagerClient : public EagerClient {
       override {                                                          \
     new RPCState<protobuf::Message>(                                      \
         &stub_, cq_, "/tensorflow.eager.EagerService/" #method, *request, \
-        response, std::move(done), nullptr);                              \
+        response, std::move(done), nullptr, nullptr);                     \
   }
 
   CLIENT_METHOD(CreateContext);
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
index 885c5e87c1..2daefcb399 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/worker_interface.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tracing.h"
@@ -42,10 +43,12 @@ class GrpcRemoteWorker : public WorkerInterface {
  public:
   explicit GrpcRemoteWorker(SharedGrpcChannelPtr channel,
                             ::grpc::CompletionQueue* completion_queue,
+                            thread::ThreadPool* callback_threadpool,
                             WorkerCacheLogger* logger)
       : channel_(std::move(channel)),
         stub_(channel_),
         cq_(completion_queue),
+        callback_threadpool_(callback_threadpool),
         getstatus_(Method(GrpcWorkerMethod::kGetStatus)),
         createworkersession_(Method(GrpcWorkerMethod::kCreateWorkerSession)),
         deleteworkersession_(Method(GrpcWorkerMethod::kDeleteWorkerSession)),
@@ -258,13 +261,15 @@ class GrpcRemoteWorker : public WorkerInterface {
                     protobuf::Message* response, const ::grpc::string& method,
                     StatusCallback done, CallOptions* call_opts = nullptr) {
     new RPCState<protobuf::Message>(&stub_, cq_, method, *request, response,
-                                    std::move(done), call_opts);
+                                    std::move(done), call_opts,
+                                    callback_threadpool_);
   }
   void IssueRequest(const protobuf::Message* request, TensorResponse* response,
                     const ::grpc::string& method, StatusCallback done,
                     CallOptions* call_opts = nullptr) {
     new RPCState<TensorResponse>(&stub_, cq_, method, *request, response,
-                                 std::move(done), call_opts);
+                                 std::move(done), call_opts,
+                                 callback_threadpool_);
   }
 
   // Helper function for initializing the RpcMethod objects below.
@@ -273,6 +278,7 @@ class GrpcRemoteWorker : public WorkerInterface {
   SharedGrpcChannelPtr channel_;
   ::grpc::GenericStub stub_;
   ::grpc::CompletionQueue* cq_;
+  thread::ThreadPool* callback_threadpool_;
 
   const ::grpc::string getstatus_;
   const ::grpc::string createworkersession_;
@@ -298,8 +304,10 @@ class GrpcRemoteWorker : public WorkerInterface {
 
 WorkerInterface* NewGrpcRemoteWorker(SharedGrpcChannelPtr channel,
                                      ::grpc::CompletionQueue* completion_queue,
+                                     thread::ThreadPool* callback_threadpool,
                                      WorkerCacheLogger* logger) {
-  return new GrpcRemoteWorker(std::move(channel), completion_queue, logger);
+  return new GrpcRemoteWorker(std::move(channel), completion_queue,
+                              callback_threadpool, logger);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.h b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.h
index b85c1dc5b4..d1f0e94ba5 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_remote_worker.h
@@ -19,18 +19,19 @@ limitations under the License.
 #include <memory>
 
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 
 namespace grpc {
 class CompletionQueue;
 }
 
 namespace tensorflow {
-
 class WorkerCacheLogger;
 class WorkerInterface;
 
 WorkerInterface* NewGrpcRemoteWorker(SharedGrpcChannelPtr channel,
                                      ::grpc::CompletionQueue* completion_queue,
+                                     thread::ThreadPool* callback_threadpool,
                                      WorkerCacheLogger* logger);
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
index cde6b785dc..4f5975bbc1 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_rpc_factory.cc
@@ -206,11 +206,11 @@ void GrpcRPCFactory::StartCall(const Tensor& address_t, const Tensor& method_t,
 
   int index = call->index();
   // This object will delete itself when done.
-  new RPCState<string>(get_stub(index), &completion_queue_,
-                       *get_method_ptr(index), call->request(),
-                       call->response(),
-                       /*done=*/[call](const Status& s) { call->Done(s); },
-                       call->call_opts(), fail_fast_, timeout_in_ms_);
+  new RPCState<string>(
+      get_stub(index), &completion_queue_, *get_method_ptr(index),
+      call->request(), call->response(),
+      /*done=*/[call](const Status& s) { call->Done(s); }, call->call_opts(),
+      nullptr /*threadpool*/, fail_fast_, timeout_in_ms_);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_state.h b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
index 61c5bc285f..b67f3c4563 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_state.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_state.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/rpc/grpc_client_cq_tag.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/distributed_runtime/tensor_coding.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/platform/notification.h"
 
 namespace tensorflow {
@@ -36,16 +37,18 @@ class RPCState : public GrpcClientCQTag {
   // Default behavior is to set fail_fast = False and handle timeouts manually.
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const protobuf::Message& request,
-           Response* response, StatusCallback done, CallOptions* call_opts)
+           Response* response, StatusCallback done, CallOptions* call_opts,
+           thread::ThreadPool* threadpool)
       : RPCState(stub, cq, method, request, response, std::move(done),
-                 call_opts, /*fail_fast=*/false, /*timeout_in_ms=*/0) {}
+                 call_opts, threadpool, /*fail_fast=*/false,
+                 /*timeout_in_ms=*/0) {}
 
   template <typename Request>
   RPCState(::grpc::GenericStub* stub, ::grpc::CompletionQueue* cq,
            const ::grpc::string& method, const Request& request,
            Response* response, StatusCallback done, CallOptions* call_opts,
-           bool fail_fast, int64 timeout_in_ms)
-      : call_opts_(call_opts), done_(std::move(done)) {
+           thread::ThreadPool* threadpool, bool fail_fast, int64 timeout_in_ms)
+      : call_opts_(call_opts), threadpool_(threadpool), done_(std::move(done)) {
     context_.set_fail_fast(fail_fast);
     if (timeout_in_ms > 0) {
       context_.set_deadline(gpr_time_from_millis(timeout_in_ms, GPR_TIMESPAN));
@@ -77,11 +80,27 @@ class RPCState : public GrpcClientCQTag {
       // to Finish for client-side unary calls, ok should never be false
       s.Update(errors::Internal("unexpected ok value at rpc completion"));
     }
-    if (s.ok() && !GrpcMaybeParseProto(&response_buf_, response_)) {
-      s.Update(errors::Internal("could not parse rpc response"));
-    }
-    if (!s.ok()) {
+
+    if (s.ok()) {
+      if (threadpool_) {
+        // Run parse and callback in another thread, returning this
+        // one to service more RPCs.
+        threadpool_->Schedule([this]() { ParseAndCallDone(); });
+      } else {
+        ParseAndCallDone();
+        return;
+      }
+    } else {
       VLOG(2) << "Call returned with non-ok status: " << s;
+      done_(s);
+      delete this;
+    }
+  }
+
+  void ParseAndCallDone() {
+    Status s;
+    if (!GrpcMaybeParseProto(&response_buf_, response_)) {
+      s.Update(errors::Internal("could not parse rpc response"));
     }
     done_(s);
     delete this;
@@ -90,6 +109,7 @@ class RPCState : public GrpcClientCQTag {
  private:
   CallOptions* call_opts_;
   ::grpc::ClientContext context_;
+  thread::ThreadPool* threadpool_;
   std::unique_ptr<::grpc::GenericClientAsyncResponseReader> call_;
   Response* response_;
   ::grpc::ByteBuffer request_buf_;
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_cache.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_cache.cc
index e1541db69b..60d5881d4c 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_cache.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_cache.cc
@@ -43,7 +43,17 @@ class GrpcWorkerCache : public WorkerCachePartial {
         local_worker_(local_worker),
         channel_cache_(channel_cache),
         threads_(kGrpcWorkerCacheThreadCount),
-        next_round_robin_assignment_(0) {}
+        next_round_robin_assignment_(0) {
+    // NOTE: We don't yet have any reason to assign NUMA affinity to this
+    // ThreadPool.  If there's only a single NIC it shouldn't make any
+    // difference since presumably it is handling memory from all nodes.
+    ThreadOptions options;
+    options.numa_node = port::kNUMANoAffinity;
+    const int kNumCallbackThreads = 10;
+    callback_threadpool_.reset(new thread::ThreadPool(
+        Env::Default(), options, "grpc_wcache_callback", kNumCallbackThreads,
+        false /*low_latency_hint*/, nullptr /*allocator*/));
+  }
 
   // Explicit destructor to control destruction order.
   ~GrpcWorkerCache() override {
@@ -67,7 +77,7 @@ class GrpcWorkerCache : public WorkerCachePartial {
       if (!channel) return nullptr;
       return NewGrpcRemoteWorker(
           channel, threads_[AssignWorkerToThread(target)].completion_queue(),
-          &logger_);
+          callback_threadpool_.get(), &logger_);
     }
   }
 
@@ -138,6 +148,8 @@ class GrpcWorkerCache : public WorkerCachePartial {
   WorkerCacheLogger logger_;
   std::vector<GrpcWorkerCacheThread> threads_;
 
+  std::unique_ptr<thread::ThreadPool> callback_threadpool_;
+
   mutex assignment_mu_;
   std::unordered_map<std::string, size_t> target_assignments_
       GUARDED_BY(assignment_mu_);
-- 
GitLab


From 680799688cd0ff984f92cde3c9f1eec8e322a5d9 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 19 Nov 2018 11:14:02 -0800
Subject: [PATCH 0500/1554] Remove GraphKeys class symbols from tf 2.0.

PiperOrigin-RevId: 222106202
---
 tensorflow/python/framework/ops.py            |   2 +-
 .../golden/v2/tensorflow.-graph-keys.pbtxt    | 140 ------------------
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 3 files changed, 1 insertion(+), 145 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index ee7323fc9d..c465d2bc10 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5703,7 +5703,7 @@ def _get_graph_from_inputs(op_input_list, graph=None):
   return graph or get_default_graph()
 
 
-@tf_export("GraphKeys")
+@tf_export(v1=["GraphKeys"])
 class GraphKeys(object):
   """Standard names to use for graph collections.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt
deleted file mode 100644
index ffe4790933..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-graph-keys.pbtxt
+++ /dev/null
@@ -1,140 +0,0 @@
-path: "tensorflow.GraphKeys"
-tf_class {
-  is_instance: "<class \'tensorflow.python.framework.ops.GraphKeys\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "ACTIVATIONS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "ASSET_FILEPATHS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "BIASES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "CONCATENATED_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "COND_CONTEXT"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "EVAL_STEP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "GLOBAL_STEP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "GLOBAL_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "INIT_OP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "LOCAL_INIT_OP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "LOCAL_RESOURCES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "LOCAL_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "LOSSES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "METRIC_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "MODEL_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "MOVING_AVERAGE_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "QUEUE_RUNNERS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "READY_FOR_LOCAL_INIT_OP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "READY_OP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "REGULARIZATION_LOSSES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "RESOURCES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "SAVEABLE_OBJECTS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "SAVERS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "SUMMARIES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "SUMMARY_OP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "TABLE_INITIALIZERS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "TRAINABLE_RESOURCE_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "TRAINABLE_VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "TRAIN_OP"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "UPDATE_OPS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "VARIABLES"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "WEIGHTS"
-    mtype: "<type \'str\'>"
-  }
-  member {
-    name: "WHILE_CONTEXT"
-    mtype: "<type \'str\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 10aa8b781f..aefd5df2b2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -40,10 +40,6 @@ tf_module {
     name: "GraphDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "GraphKeys"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "GraphOptions"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-- 
GitLab


From 5a422845f204b1ddf6d6b7e85522cd0d0f606bdc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 11:16:12 -0800
Subject: [PATCH 0501/1554] Automated rollback of commit
 8aeb9686e85ca7e4a8586841ce0de41acfe8a769

PiperOrigin-RevId: 222106589
---
 .../xla/service/buffer_assignment_test.cc     | 18 ++--
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  8 +-
 .../xla/service/cpu/cpu_executable.cc         |  3 +-
 .../compiler/xla/service/cpu/cpu_executable.h |  2 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    |  2 +-
 .../compiler/xla/service/cpu/ir_emitter.h     |  2 +-
 tensorflow/compiler/xla/service/executable.h  |  6 +-
 .../xla/service/gpu/gpu_executable.cc         |  2 +-
 .../compiler/xla/service/gpu/gpu_executable.h |  2 +-
 .../xla/service/gpu/gpu_hlo_schedule.cc       | 18 ++--
 .../xla/service/gpu/gpu_hlo_schedule.h        |  4 +-
 .../xla/service/gpu/gpu_hlo_schedule_test.cc  |  2 +-
 .../xla/service/gpu/thunk_schedule.cc         |  4 +-
 .../compiler/xla/service/gpu/thunk_schedule.h |  2 +-
 .../xla/service/heap_simulator_test.cc        |  6 +-
 .../compiler/xla/service/hlo_computation.cc   |  6 +-
 .../compiler/xla/service/hlo_computation.h    |  2 +-
 .../xla/service/hlo_memory_scheduler.cc       | 82 +++++++++----------
 .../xla/service/hlo_memory_scheduler.h        | 14 ++--
 .../xla/service/hlo_memory_scheduler_test.cc  | 33 ++++----
 tensorflow/compiler/xla/service/hlo_module.h  |  6 +-
 .../compiler/xla/service/hlo_ordering.cc      |  3 +-
 tensorflow/compiler/xla/service/hlo_parser.cc |  6 +-
 .../xla/service/hlo_rematerialization.cc      | 19 ++---
 .../xla/service/hlo_rematerialization.h       |  5 +-
 .../compiler/xla/service/hlo_schedule.cc      | 25 +++---
 .../compiler/xla/service/hlo_schedule.h       | 12 +--
 .../compiler/xla/service/hlo_schedule_test.cc | 14 ++--
 .../xla/service/interpreter/executable.cc     |  2 +-
 .../xla/service/interpreter/executable.h      |  2 +-
 30 files changed, 151 insertions(+), 161 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index b1fc50cb18..8f482e6ba8 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -137,8 +137,7 @@ class BufferAssignmentTest : public HloTestBase {
   }
 
   std::unique_ptr<BufferAssignment> RunBufferAssignmentWithInstructionSequence(
-      HloModule* module,
-      absl::Span<const HloInstruction* const> instruction_sequence,
+      HloModule* module, absl::Span<HloInstruction* const> instruction_sequence,
       int64 alignment = 1) {
     HloSchedule schedule(module);
     schedule.set_sequence(module->entry_computation(), instruction_sequence);
@@ -1853,7 +1852,7 @@ class WhileBufferAssignmentTest : public HloTestBase {
   std::unique_ptr<BufferAssignment> RunBufferAssignment(HloModule* module,
                                                         int64 alignment = 1) {
     HloSchedule schedule =
-        ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie();
+        ScheduleModule(module, ByteSizeOf).ConsumeValueOrDie();
     return BufferAssigner::Run(
                module, absl::make_unique<SequentialHloOrdering>(schedule),
                ByteSizeOf,
@@ -2162,7 +2161,7 @@ TEST_F(WhileBufferAssignmentTest, ColocatedBuffers) {
   // nodes are traversed during BufferAssignment.
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
@@ -2391,15 +2390,16 @@ TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) {
   RunCopyInsertion(module.get());
 
   HloSchedule schedule =
-      ScheduleModule(*module, ByteSizeOf).ConsumeValueOrDie();
+      ScheduleModule(module.get(), ByteSizeOf).ConsumeValueOrDie();
 
   // To trigger b/38494731, we want a specific Hlo schedule for the
   // root computation, so we overwrite that entry with a manually
   // crafted sequence.
-  schedule.set_sequence(module->entry_computation(),
-                        {input1, weights1, one, output1, while1->operand(0),
-                         while1, input0, weights0, zero, output0,
-                         while0->operand(0), while0, gte0, gte1, root_add});
+  schedule.set_sequence(
+      module->entry_computation(),
+      {input1, weights1, one, output1, while1->mutable_operand(0), while1,
+       input0, weights0, zero, output0, while0->mutable_operand(0), while0,
+       gte0, gte1, root_add});
 
   // If this ASSERT fails, we constructed a bogus sequence above and this test
   // itself is buggy.
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 6fe76e18ca..a6f357df20 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -588,9 +588,9 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::RunBackend(
   // Select an order for emitting the HLO instructions for each
   // computation. Using this sequence enables tighter buffer liveness analysis
   // and reduced memory usage (as compared to using DependencyHloOrdering).
-  TF_ASSIGN_OR_RETURN(
-      HloSchedule schedule,
-      ScheduleModule(*module, BufferSizeBytesFunction(), DFSMemoryScheduler));
+  TF_ASSIGN_OR_RETURN(HloSchedule schedule,
+                      ScheduleModule(module.get(), BufferSizeBytesFunction(),
+                                     DFSMemoryScheduler));
 
   // Run buffer allocation on the HLO graph.
   TF_ASSIGN_OR_RETURN(
@@ -780,7 +780,7 @@ CpuCompiler::CompileAheadOfTime(std::unique_ptr<HloModuleGroup> module_group,
     XLA_VLOG_LINES(2, module->ToString());
 
     TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                        ScheduleModule(*module, BufferSizeBytesFunction()));
+                        ScheduleModule(module, BufferSizeBytesFunction()));
 
     // Run buffer analysis on the HLO graph. This analysis figures out which
     // temporary buffers are required to run the computation.
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 29abf38e43..818b2b0d0d 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -51,8 +51,7 @@ namespace cpu {
 CpuExecutable::CpuExecutable(
     std::unique_ptr<SimpleOrcJIT> jit,
     std::unique_ptr<const BufferAssignment> assignment,
-    std::unique_ptr<const HloModule> hlo_module,
-    const string& entry_function_name,
+    std::unique_ptr<HloModule> hlo_module, const string& entry_function_name,
     std::unique_ptr<HloProfilePrinterData> hlo_profile_printer_data,
     std::unique_ptr<HloProfileIndexMap> hlo_profile_index_map)
     : Executable(std::move(hlo_module), std::move(hlo_profile_printer_data),
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.h b/tensorflow/compiler/xla/service/cpu/cpu_executable.h
index 3c3c047bfe..3b91b15ba9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.h
@@ -49,7 +49,7 @@ class CpuExecutable : public Executable {
  public:
   CpuExecutable(std::unique_ptr<SimpleOrcJIT> jit,
                 std::unique_ptr<const BufferAssignment> assignment,
-                std::unique_ptr<const HloModule> hlo_module,
+                std::unique_ptr<HloModule> hlo_module,
                 const string& entry_function_name,
                 std::unique_ptr<HloProfilePrinterData> hlo_profile_printer_data,
                 std::unique_ptr<HloProfileIndexMap> hlo_profile_index_map);
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 620c45fa39..9b731b0a6a 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -111,7 +111,7 @@ IrEmitter::IrEmitter(
 StatusOr<llvm::Function*> IrEmitter::EmitComputation(
     HloComputation* computation, const string& function_name_prefix,
     bool is_top_level_computation,
-    const std::vector<const HloInstruction*>* instruction_order) {
+    const std::vector<HloInstruction*>* instruction_order) {
   string function_name = name_uniquer_.GetUniqueName(function_name_prefix);
   VLOG(2) << "Emitting IR for CPU function [" << function_name_prefix
           << "]; ordered? " << (instruction_order != nullptr);
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 136b88ff75..e03ba42b0d 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -101,7 +101,7 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   StatusOr<llvm::Function*> EmitComputation(
       HloComputation* computation, const string& function_name_prefix,
       bool is_top_level_computation,
-      const std::vector<const HloInstruction*>* instruction_order);
+      const std::vector<HloInstruction*>* instruction_order);
 
   llvm::IRBuilder<>* b() { return &b_; }
 
diff --git a/tensorflow/compiler/xla/service/executable.h b/tensorflow/compiler/xla/service/executable.h
index 45f620f3f3..b34bca55a4 100644
--- a/tensorflow/compiler/xla/service/executable.h
+++ b/tensorflow/compiler/xla/service/executable.h
@@ -61,7 +61,7 @@ struct ExecutionOutput {
 class Executable {
  public:
   explicit Executable(
-      std::unique_ptr<const HloModule> hlo_module,
+      std::unique_ptr<HloModule> hlo_module,
       std::unique_ptr<HloProfilePrinterData> hlo_profile_printer_data,
       std::unique_ptr<HloProfileIndexMap> hlo_profile_index_map)
       : hlo_module_(std::move(hlo_module)),
@@ -162,7 +162,7 @@ class Executable {
     return hlo_profile_printer_data_ != nullptr;
   }
 
-  const HloModule& module() const { return *hlo_module_; }
+  HloModule& module() const { return *hlo_module_; }
 
   const bool has_module() const { return hlo_module_ != nullptr; }
 
@@ -199,7 +199,7 @@ class Executable {
   // HloModule this was compiled from. BufferAssignment keeps pointers to
   // HloInstructions owned by the HloModule so we need to keep the HloModule
   // around.
-  const std::unique_ptr<const HloModule> hlo_module_;
+  const std::unique_ptr<HloModule> hlo_module_;
 
   // HloSnapshot this was compiled from. Null if not dumping executions.
   std::unique_ptr<HloSnapshot> hlo_snapshot_;
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 5742632782..ae2e718db2 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -51,7 +51,7 @@ GpuExecutable::GpuExecutable(
     const string& ptx, const std::vector<uint8>& cubin,
     std::pair<int, int> compute_capability,
     std::unique_ptr<const ThunkSchedule> thunk_schedule,
-    std::unique_ptr<const HloModule> hlo_module,
+    std::unique_ptr<HloModule> hlo_module,
     std::unique_ptr<const BufferAssignment> assignment,
     std::unique_ptr<HloProfilePrinterData> hlo_profile_printer_data,
     std::unique_ptr<HloProfileIndexMap> hlo_profile_index_map)
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.h b/tensorflow/compiler/xla/service/gpu/gpu_executable.h
index 0e276282e4..2b3c77f5b8 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.h
@@ -54,7 +54,7 @@ class GpuExecutable : public Executable {
   GpuExecutable(const string& ptx, const std::vector<uint8>& cubin,
                 std::pair<int, int> compute_capability,
                 std::unique_ptr<const ThunkSchedule> thunk_schedule,
-                std::unique_ptr<const HloModule> hlo_module,
+                std::unique_ptr<HloModule> hlo_module,
                 std::unique_ptr<const BufferAssignment> assignment,
                 std::unique_ptr<HloProfilePrinterData> hlo_profile_printer_data,
                 std::unique_ptr<HloProfileIndexMap> hlo_profile_index_map);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
index 91609c730b..1126943624 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.cc
@@ -37,7 +37,7 @@ class GpuHloOrdering : public PredecessorHloOrdering {
  public:
   GpuHloOrdering(const HloModule* module,
                  const StreamAssignment& stream_assignment,
-                 const std::vector<const HloInstruction*>& thunk_launch_order);
+                 const std::vector<HloInstruction*>& thunk_launch_order);
   ~GpuHloOrdering() override = default;
 
   // Only the entry computation can possibly be sequentially ordered, and only
@@ -56,7 +56,7 @@ class GpuHloOrdering : public PredecessorHloOrdering {
 
 GpuHloOrdering::GpuHloOrdering(
     const HloModule* module, const StreamAssignment& stream_assignment,
-    const std::vector<const HloInstruction*>& thunk_launch_order)
+    const std::vector<HloInstruction*>& thunk_launch_order)
     : PredecessorHloOrdering(module) {
   // The entry computation has a total order when there's only one stream.
   if (stream_assignment.StreamCount() == 1) {
@@ -150,7 +150,7 @@ GpuHloOrdering::GpuHloOrdering(
 // However, if the total order is A,B,D,C,E, then C and E can run
 // concurrently.
 void BFSLaunchOrder(const HloComputation* computation,
-                    std::vector<const HloInstruction*>* launch_order) {
+                    std::vector<HloInstruction*>* launch_order) {
   // This topological sort uses two data structures:
   // 1. `incoming_edge_count` which keeps track of the number of incoming
   // edges to each HLO;
@@ -158,9 +158,9 @@ void BFSLaunchOrder(const HloComputation* computation,
   //
   // The sorting algorithm repeatedly pops the top from the queue and deletes
   // that HLO from the graph, making more HLOs incoming-edge free.
-  std::deque<const HloInstruction*> queue;
+  std::deque<HloInstruction*> queue;
   std::unordered_map<const HloInstruction*, int64> incoming_edge_count;
-  for (const auto& hlo : computation->instructions()) {
+  for (auto* hlo : computation->instructions()) {
     if (hlo->operand_count() == 0) {
       queue.push_back(hlo);
     } else {
@@ -172,10 +172,10 @@ void BFSLaunchOrder(const HloComputation* computation,
   }
 
   while (!queue.empty()) {
-    const HloInstruction* x = queue.front();
+    HloInstruction* x = queue.front();
     queue.pop_front();
     launch_order->push_back(x);
-    for (const HloInstruction* y : x->users()) {
+    for (HloInstruction* y : x->users()) {
       --incoming_edge_count[y];
       if (incoming_edge_count[y] == 0) {
         queue.push_back(y);
@@ -195,14 +195,14 @@ StatusOr<std::unique_ptr<GpuHloSchedule>> GpuHloSchedule::Build(
   std::unique_ptr<GpuHloSchedule> schedule(new GpuHloSchedule);
 
   // Initialize thunk_launch_order_, the total order of thunk launches.
-  const HloComputation* entry_computation = module.entry_computation();
+  HloComputation* entry_computation = module.entry_computation();
   if (stream_assignment.StreamCount() == 1) {
     // All kernels are launched on a single stream, so there's no loss of
     // concurrency by optimizing for minimal memory usage.
     TF_ASSIGN_OR_RETURN(
         HloInstructionSequence sequence,
         ScheduleComputation(
-            *entry_computation, [pointer_size](const BufferValue& buffer) {
+            entry_computation, [pointer_size](const BufferValue& buffer) {
               return ShapeUtil::ByteSizeOf(buffer.shape(), pointer_size);
             }));
     schedule->thunk_launch_order_ = sequence.instructions();
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
index 07a7fc67aa..7f224ffe4f 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule.h
@@ -46,7 +46,7 @@ class GpuHloSchedule {
 
   // Returns the total order of thunk launches, represented in terms of HLO
   // instructions.
-  const std::vector<const HloInstruction*>& ThunkLaunchOrder() const {
+  const std::vector<HloInstruction*>& ThunkLaunchOrder() const {
     return thunk_launch_order_;
   }
 
@@ -60,7 +60,7 @@ class GpuHloSchedule {
  private:
   GpuHloSchedule();
 
-  std::vector<const HloInstruction*> thunk_launch_order_;
+  std::vector<HloInstruction*> thunk_launch_order_;
   std::unique_ptr<HloOrdering> hlo_ordering_;
 };
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
index 5f857a1a54..91db7151f2 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_hlo_schedule_test.cc
@@ -33,7 +33,7 @@ namespace gpu {
 
 class GpuHloScheduleTest : public HloTestBase {
  protected:
-  using HloVec = std::vector<const HloInstruction*>;
+  using HloVec = std::vector<HloInstruction*>;
 
   // Pre-canned shapes.
   Shape f32_2x2_ = ShapeUtil::MakeShape(F32, {2, 2});
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
index 141f321938..6b2d76764a 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
@@ -45,7 +45,7 @@ void ThunkSchedule::AddDependenciesOnTransitiveOperands(
 ThunkSchedule::ThunkSchedule(
     std::unique_ptr<ThunkSequence> thunks,
     std::unique_ptr<StreamAssignment> stream_assignment,
-    const std::vector<const HloInstruction*>& hlo_total_order)
+    const std::vector<HloInstruction*>& hlo_total_order)
     : thunks_(std::move(thunks)),
       stream_assignment_(std::move(stream_assignment)) {
   std::unordered_map<const HloInstruction*, Thunk*> hlo_to_thunk;
@@ -53,7 +53,7 @@ ThunkSchedule::ThunkSchedule(
     InsertOrDie(&hlo_to_thunk, thunk->hlo_instruction(), thunk.get());
   }
 
-  for (const HloInstruction* hlo : hlo_total_order) {
+  for (HloInstruction* hlo : hlo_total_order) {
     if (hlo_to_thunk.count(hlo)) {
       thunk_total_order_.push_back(FindOrDie(hlo_to_thunk, hlo));
     }
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
index d3352994f8..43b628a1ba 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
@@ -46,7 +46,7 @@ class ThunkSchedule {
  public:
   ThunkSchedule(std::unique_ptr<ThunkSequence> thunks,
                 std::unique_ptr<StreamAssignment> stream_assignment,
-                const std::vector<const HloInstruction*>& hlo_total_order);
+                const std::vector<HloInstruction*>& hlo_total_order);
 
   // Returns the total order of executing all the thunks.
   const std::vector<Thunk*>& TotalOrder() const { return thunk_total_order_; }
diff --git a/tensorflow/compiler/xla/service/heap_simulator_test.cc b/tensorflow/compiler/xla/service/heap_simulator_test.cc
index fad3215fc8..dc40b9446a 100644
--- a/tensorflow/compiler/xla/service/heap_simulator_test.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator_test.cc
@@ -258,7 +258,7 @@ class HeapSimulatorTracker {
   // Constructor for testing a single entry computation.
   HeapSimulatorTracker(
       const string& name, std::unique_ptr<HloComputation> computation,
-      const std::vector<const HloInstruction*>& instruction_sequence) {
+      const std::vector<HloInstruction*>& instruction_sequence) {
     HloModuleConfig config;
     module_ = absl::make_unique<HloModule>(name, config);
     module_->AddEntryComputation(std::move(computation));
@@ -286,7 +286,7 @@ class HeapSimulatorTracker {
   // Similar to the single entry computation constructor above, but runs the
   // simulation over the entire module.
   void RunWholeModule(
-      const std::vector<const HloInstruction*>& full_module_sequence) {
+      const std::vector<HloInstruction*>& full_module_sequence) {
     points_to_analysis_ =
         TuplePointsToAnalysis::Run(module_.get()).ConsumeValueOrDie();
 
@@ -294,7 +294,7 @@ class HeapSimulatorTracker {
     HloSchedule schedule(module_.get());
     absl::flat_hash_map<const HloInstruction*, int> reverse_position;
     for (int i = 0; i < full_module_sequence.size(); ++i) {
-      const HloInstruction* instruction = full_module_sequence[i];
+      HloInstruction* instruction = full_module_sequence[i];
       schedule.GetOrCreateSequence(instruction->parent())
           .push_back(instruction);
       reverse_position[instruction] = full_module_sequence.size() - i;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 0c20d207dd..65bd251dd8 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -795,7 +795,7 @@ Status HloComputation::AcceptWithOperandOrder(
 template <typename HloInstructionPtr>
 Status HloComputation::AcceptOrdered(
     DfsHloVisitorBase<HloInstructionPtr>* visitor,
-    const std::vector<const HloInstruction*>& order) const {
+    const std::vector<HloInstruction*>& order) const {
   VLOG(3) << "Accepting visitor with order.";
   for (HloInstruction* root : CollectUnreachableRoots()) {
     TF_RET_CHECK(std::find(order.begin(), order.end(), root) != order.end())
@@ -825,9 +825,9 @@ Status HloComputation::AcceptOrdered(
 
 // Explicit instantiations.
 template Status HloComputation::AcceptOrdered(
-    DfsHloVisitor*, const std::vector<const HloInstruction*>&) const;
+    DfsHloVisitor*, const std::vector<HloInstruction*>&) const;
 template Status HloComputation::AcceptOrdered(
-    ConstDfsHloVisitor*, const std::vector<const HloInstruction*>&) const;
+    ConstDfsHloVisitor*, const std::vector<HloInstruction*>&) const;
 
 Status HloComputation::Accept(
     const std::function<Status(HloInstruction*)>& visitor_func) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index fc7d2035e5..be1ce33696 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -301,7 +301,7 @@ class HloComputation {
   // be a topological sort of all instructions in the computation.
   template <typename HloInstructionPtr>
   Status AcceptOrdered(DfsHloVisitorBase<HloInstructionPtr>* visitor,
-                       const std::vector<const HloInstruction*>& order) const;
+                       const std::vector<HloInstruction*>& order) const;
 
   // Same as Accept() above, but the visitor is given as a function.
   Status Accept(const std::function<Status(HloInstruction*)>& visitor_func);
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
index 234fcd266a..d2740bcce2 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc
@@ -73,7 +73,7 @@ class ListScheduler {
   // Construct and return a memory-minimizing sequence of HLO instructions
   // containing the given HLO computation.
   static StatusOr<HloInstructionSequence> Run(
-      const HloComputation& computation,
+      HloComputation* computation,
       const TuplePointsToAnalysis& points_to_analysis,
       const LogicalBuffer::SizeFunction& size_function,
       const absl::flat_hash_map<const HloComputation*, int64>&
@@ -98,7 +98,7 @@ class ListScheduler {
   // comparison operators.
   using Priority = std::pair<int64, int64>;
 
-  ListScheduler(const HloComputation& computation,
+  ListScheduler(HloComputation* computation,
                 const TuplePointsToAnalysis& points_to_analysis,
                 const LogicalBuffer::SizeFunction& size_function,
                 const absl::flat_hash_map<const HloComputation*, int64>&
@@ -111,7 +111,7 @@ class ListScheduler {
     // instruction. An HLO instruction "uses" a LogicalBuffer if the
     // LogicalBuffer is in an operand of the instruction as indicated by
     // points-to analysis.
-    for (auto* instruction : computation.instructions()) {
+    for (auto* instruction : computation->instructions()) {
       absl::flat_hash_set<const LogicalBuffer*> instr_uses;
       for (auto* operand : instruction->operands()) {
         points_to_analysis.GetPointsToSet(operand).ForEachElement(
@@ -126,13 +126,13 @@ class ListScheduler {
 
     // Create map containing the number of unscheduled uses (hlo instructions)
     // of each logical buffer.
-    for (auto* instruction : computation.instructions()) {
+    for (auto* instruction : computation->instructions()) {
       for (auto* buffer :
            points_to_analysis.GetBuffersDefinedByInstruction(instruction)) {
         unscheduled_use_count_[buffer] = 0;
       }
     }
-    for (auto* instruction : computation.instructions()) {
+    for (auto* instruction : computation->instructions()) {
       for (const LogicalBuffer* buffer : buffer_uses_.at(instruction)) {
         ++unscheduled_use_count_[buffer];
       }
@@ -141,7 +141,7 @@ class ListScheduler {
     // Buffers live out of the computation have an implicit use at the end of
     // the computation.
     for (const LogicalBuffer* live_out_buffer :
-         points_to_analysis.GetPointsToSet(computation.root_instruction())
+         points_to_analysis.GetPointsToSet(computation->root_instruction())
              .CreateFlattenedSet()) {
       ++unscheduled_use_count_[live_out_buffer];
     }
@@ -157,7 +157,7 @@ class ListScheduler {
   // HloInstruction, plus some cached metadata, saved for the purposes of making
   // BytesFreedIfScheduled fast.
   struct ReadyListEntry {
-    const HloInstruction* instruction;
+    HloInstruction* instruction;
 
     // The total size of all buffers defined by this instruction.
     int64 bytes_defined;
@@ -171,7 +171,7 @@ class ListScheduler {
   };
 
   // Creates a ReadyListEntry for the given instruction.
-  ReadyListEntry MakeReadyListEntry(const HloInstruction* instruction) {
+  ReadyListEntry MakeReadyListEntry(HloInstruction* instruction) {
     ReadyListEntry entry;
     entry.instruction = instruction;
 
@@ -250,13 +250,13 @@ class ListScheduler {
     // Populate the ready list with instructions which have no operands or
     // control predecessors.
     absl::flat_hash_map<const HloInstruction*, int64> unscheduled_pred_count;
-    for (auto* instruction : computation_.instructions()) {
+    for (auto* instruction : computation_->instructions()) {
       // TODO(b/34466113): Replace this and above with successors() or
       // predecessors() when these methods are added to HloInstruction.
-      for (const HloInstruction* user : instruction->users()) {
+      for (HloInstruction* user : instruction->users()) {
         unscheduled_pred_count[user]++;
       }
-      for (const HloInstruction* succ : instruction->control_successors()) {
+      for (HloInstruction* succ : instruction->control_successors()) {
         unscheduled_pred_count[succ]++;
       }
     }
@@ -275,7 +275,7 @@ class ListScheduler {
       ready_instructions[inst] = it;
     };
 
-    for (auto* instruction : computation_.instructions()) {
+    for (auto* instruction : computation_->instructions()) {
       if (instruction->operands().empty() &&
           instruction->control_predecessors().empty()) {
         add_to_ready_queue(instruction);
@@ -287,7 +287,7 @@ class ListScheduler {
       // schedule.
       auto best_it = ready_queue.end();
       --best_it;
-      const HloInstruction* best = best_it->second.instruction;
+      HloInstruction* best = best_it->second.instruction;
       VLOG(2) << "Schedule instruction: " << best->ToShortString()
               << " Bytes freed: " << best_it->first.first;
       ready_queue.erase(best_it);
@@ -348,13 +348,13 @@ class ListScheduler {
         }
       }
     }
-    CHECK_EQ(schedule.size(), computation_.instruction_count());
-    CHECK_EQ(scheduled_instructions_.size(), computation_.instruction_count());
+    CHECK_EQ(schedule.size(), computation_->instruction_count());
+    CHECK_EQ(scheduled_instructions_.size(), computation_->instruction_count());
 
     return schedule;
   }
 
-  const HloComputation& computation_;
+  HloComputation* computation_;
   const TuplePointsToAnalysis& points_to_analysis_;
   const LogicalBuffer::SizeFunction& size_function_;
   // Computations are analyzed in post-order. When scheduling an instruction
@@ -386,13 +386,13 @@ int64 SumLogicalBufferSizes(
 }
 
 StatusOr<HloInstructionSequence> ScheduleComputationHelper(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
-  VLOG(2) << "Computation: " << computation.name();
+  VLOG(2) << "Computation: " << computation->name();
   if (algorithm) {
     return algorithm(computation, points_to_analysis, size_function,
                      memory_by_computation);
@@ -404,17 +404,17 @@ StatusOr<HloInstructionSequence> ScheduleComputationHelper(
 }  // namespace
 
 StatusOr<HloInstructionSequence> DFSMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
   // These variables are a hack to prevent overflows.
   int64 cumulative_total_size = 0;
-  int64 total_hlos = computation.parent()->instruction_count();
+  int64 total_hlos = computation->parent()->instruction_count();
   absl::flat_hash_map<const HloInstruction*, int64> extra_users;
   absl::flat_hash_map<const HloInstruction*, int64> total_sizes;
-  for (const HloInstruction* hlo : computation.MakeInstructionPostOrder()) {
+  for (const HloInstruction* hlo : computation->MakeInstructionPostOrder()) {
     if (ListScheduler::IgnoreInstruction(*hlo)) {
       extra_users[hlo] = 0;
       total_sizes[hlo] = 0;
@@ -448,8 +448,8 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
     total_sizes[hlo] = std::min(total_sizes[hlo], cumulative_total_size);
     extra_users[hlo] = std::min(extra_users[hlo], total_hlos);
   }
-  CHECK_EQ(extra_users.size(), computation.instruction_count());
-  CHECK_EQ(total_sizes.size(), computation.instruction_count());
+  CHECK_EQ(extra_users.size(), computation->instruction_count());
+  CHECK_EQ(total_sizes.size(), computation->instruction_count());
 
   // Construct a total order based on DFS post-order, visiting operands in
   // decreasing cumulative extra user order, and next by cumulative size, with a
@@ -459,7 +459,7 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
     sequence.push_back(hlo);
     return Status::OK();
   });
-  TF_RETURN_IF_ERROR(computation.AcceptWithOperandOrder(
+  TF_RETURN_IF_ERROR(computation->AcceptWithOperandOrder(
       &visitor, [&extra_users, &total_sizes](const HloInstruction* a,
                                              const HloInstruction* b) {
         if (extra_users[a] != extra_users[b]) {
@@ -470,12 +470,12 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
         }
         return a->name() < b->name();
       }));
-  CHECK_EQ(sequence.size(), computation.instruction_count());
+  CHECK_EQ(sequence.size(), computation->instruction_count());
   return sequence;
 }  // namespace xla
 
 StatusOr<HloInstructionSequence> ListMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -485,16 +485,16 @@ StatusOr<HloInstructionSequence> ListMemoryScheduler(
 }
 
 StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
         memory_by_computation) {
-  return HloInstructionSequence(computation.MakeInstructionPostOrder());
+  return HloInstructionSequence(computation->MakeInstructionPostOrder());
 }
 
 StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -513,7 +513,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                           memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 list_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          computation, list_sequence, points_to_analysis,
+                          *computation, list_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory list sequence: " << HumanReadableNumBytes(list_memory);
 
@@ -522,7 +522,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                                          size_function, memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 dfs_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          computation, dfs_sequence, points_to_analysis,
+                          *computation, dfs_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory dfs sequence: " << HumanReadableNumBytes(dfs_memory);
 
@@ -532,7 +532,7 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
                                memory_by_computation));
   TF_ASSIGN_OR_RETURN(const int64 post_order_memory,
                       HeapSimulator::MinimumMemoryForComputation(
-                          computation, post_order_sequence, points_to_analysis,
+                          *computation, post_order_sequence, points_to_analysis,
                           size_function, &memory_by_computation));
   VLOG(2) << "Min-memory post order sequence: "
           << HumanReadableNumBytes(post_order_memory);
@@ -555,17 +555,17 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
 }
 
 StatusOr<HloSchedule> ScheduleModule(
-    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
+    HloModule* module, const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm) {
-  HloSchedule schedule(&module);
+  HloSchedule schedule(module);
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
-                      TuplePointsToAnalysis::Run(&module));
+                      TuplePointsToAnalysis::Run(module));
   absl::flat_hash_map<const HloComputation*, int64> memory_by_computation;
-  for (const auto* computation : module.MakeComputationPostOrder()) {
+  for (auto* computation : module->MakeComputationPostOrder()) {
     if (!computation->IsFusionComputation()) {
       TF_ASSIGN_OR_RETURN(HloInstructionSequence computation_sequence,
                           ScheduleComputationHelper(
-                              *computation, *points_to_analysis, size_function,
+                              computation, *points_to_analysis, size_function,
                               algorithm, memory_by_computation));
       memory_by_computation[computation] =
           HeapSimulator::MinimumMemoryForComputation(
@@ -583,11 +583,11 @@ StatusOr<HloSchedule> ScheduleModule(
 }
 
 StatusOr<HloInstructionSequence> ScheduleComputation(
-    const HloComputation& computation,
+    HloComputation* computation,
     const LogicalBuffer::SizeFunction& size_function) {
-  CHECK(!computation.IsFusionComputation());
+  CHECK(!computation->IsFusionComputation());
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
-                      TuplePointsToAnalysis::Run(computation.parent()));
+                      TuplePointsToAnalysis::Run(computation->parent()));
   absl::flat_hash_map<const HloComputation*, int64> empty_map;
   return ScheduleComputationHelper(computation, *points_to_analysis,
                                    size_function, nullptr, empty_map);
@@ -600,7 +600,7 @@ HloMemoryScheduler::HloMemoryScheduler(
 
 StatusOr<bool> HloMemoryScheduler::Run(HloModule* module) {
   TF_ASSIGN_OR_RETURN(HloSchedule schedule,
-                      ScheduleModule(*module, size_function_, algorithm_));
+                      ScheduleModule(module, size_function_, algorithm_));
   TF_RETURN_IF_ERROR(module->set_schedule(std::move(schedule)));
   return true;
 }
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
index cca5dc4939..7227bfb27c 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h
@@ -36,14 +36,14 @@ namespace xla {
 // that describes buffer aliasing, together with a target-specific size function
 // that maps a tensor's logical size to its padded size.
 typedef std::function<StatusOr<HloInstructionSequence>(
-    const HloComputation&, const TuplePointsToAnalysis&,
+    HloComputation*, const TuplePointsToAnalysis&,
     const LogicalBuffer::SizeFunction&,
     const absl::flat_hash_map<const HloComputation*, int64>&)>
     MemorySchedulerAlgorithm;
 
 // List scheduler
 StatusOr<HloInstructionSequence> ListMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -51,7 +51,7 @@ StatusOr<HloInstructionSequence> ListMemoryScheduler(
 
 // DFS-order scheduler
 StatusOr<HloInstructionSequence> DFSMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -59,7 +59,7 @@ StatusOr<HloInstructionSequence> DFSMemoryScheduler(
 
 // Naive Post Order scheduler
 StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -69,7 +69,7 @@ StatusOr<HloInstructionSequence> PostOrderMemoryScheduler(
 // and the DFS scheduler, and chooses whichever returns a lower min-memory,
 // not accounting for fragmentation.
 StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
-    const HloComputation& computation,
+    HloComputation* computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function,
     const absl::flat_hash_map<const HloComputation*, int64>&
@@ -79,13 +79,13 @@ StatusOr<HloInstructionSequence> DefaultMemoryScheduler(
 // the computation. size_function is the function returning the number of bytes
 // required for a LogicalBuffer.
 StatusOr<HloSchedule> ScheduleModule(
-    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
+    HloModule* module, const LogicalBuffer::SizeFunction& size_function,
     const MemorySchedulerAlgorithm& algorithm = {});
 
 // Computes the schedule for a single computation.
 // Currently only used by the GPU backend.
 StatusOr<HloInstructionSequence> ScheduleComputation(
-    const HloComputation& computation,
+    HloComputation* computation,
     const LogicalBuffer::SizeFunction& size_function);
 
 // A pass which schedules the HLO instructions in a module. The HloModule's
diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
index 3d8482065a..bc0d7e2bc0 100644
--- a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc
@@ -78,7 +78,7 @@ TEST_F(HloSchedulingTest, LastUseScheduledFirst) {
   TF_ASSERT_OK(module->schedule().Verify());
 
   // Verify that all instructions are in the sequence.
-  const std::vector<const HloInstruction*>& sequence =
+  const std::vector<HloInstruction*>& sequence =
       module->schedule().sequence(module->entry_computation()).instructions();
   EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size());
 
@@ -124,9 +124,9 @@ ENTRY root {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, size_fn, ListMemoryScheduler));
+      ScheduleModule(module.get(), size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
-  const std::vector<const HloInstruction*>& sequence =
+  const std::vector<HloInstruction*>& sequence =
       schedule.sequence(module->entry_computation()).instructions();
   EXPECT_EQ(module->entry_computation()->instruction_count(), sequence.size());
 
@@ -175,12 +175,13 @@ TEST_F(HloSchedulingTest, TuplesAreAccountedCorrectly) {
   auto module = CreateNewVerifiedModule();
   module->AddEntryComputation(builder.Build());
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
-                          ScheduleModule(*module,
-                                         [](const BufferValue& buffer) {
-                                           return ShapeUtil::ByteSizeOf(
-                                               buffer.shape(), TUPLE_SIZE);
-                                         },
-                                         ListMemoryScheduler));
+                          ScheduleModule(
+                              module.get(),
+                              [](const BufferValue& buffer) {
+                                return ShapeUtil::ByteSizeOf(buffer.shape(),
+                                                             TUPLE_SIZE);
+                              },
+                              ListMemoryScheduler));
 
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
@@ -225,12 +226,12 @@ TEST_F(HloSchedulingTest, MultiOutputFusionAccountedCorrectly) {
       {tuple, mul, add}, HloInstruction::FusionKind::kLoop);
 
   TF_ASSERT_OK_AND_ASSIGN(HloSchedule schedule,
-                          ScheduleModule(*module,
-                                         [](const BufferValue& buffer) {
-                                           return ShapeUtil::ByteSizeOf(
-                                               buffer.shape(), 2);
-                                         },
-                                         ListMemoryScheduler));
+                          ScheduleModule(
+                              module.get(),
+                              [](const BufferValue& buffer) {
+                                return ShapeUtil::ByteSizeOf(buffer.shape(), 2);
+                              },
+                              ListMemoryScheduler));
 
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
@@ -284,7 +285,7 @@ TEST_F(HloSchedulingTest, HeapSimulatorAccountsForSubcomputations) {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, size_fn, ListMemoryScheduler));
+      ScheduleModule(module.get(), size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
   auto entry_computation = module->entry_computation();
   EXPECT_EQ(module->entry_computation()->instruction_count(),
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 7974999e07..66622a1d26 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -104,11 +104,7 @@ class HloModule {
                                        HloCloneContext* context = nullptr);
 
   // Return a pointer to the entry computation of the module.
-  const HloComputation* entry_computation() const {
-    CHECK_NE(nullptr, entry_computation_);
-    return entry_computation_;
-  }
-  HloComputation* entry_computation() {
+  HloComputation* entry_computation() const {
     CHECK_NE(nullptr, entry_computation_);
     return entry_computation_;
   }
diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc
index f5f99bece1..ca6a154809 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering.cc
@@ -356,8 +356,7 @@ void SequentialHloOrdering::Initialize() {
   // Create a map from instruction to its order position.
   TF_DCHECK_OK(schedule_.Verify());
   for (const auto& computation_sequence : schedule_.sequences()) {
-    const std::vector<const HloInstruction*>& order =
-        computation_sequence.second.instructions();
+    const auto& order = computation_sequence.second.instructions();
     for (int i = 0; i < order.size(); ++i) {
       InsertOrDie(&order_position_, order[i], i);
     }
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 4390145c6b..4bf287a9ed 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -47,11 +47,11 @@ const double kF16max = 65504;
 
 // Creates and returns a schedule created using the order of the instructions in
 // the HloComputation::instructions() vectors in the module.
-HloSchedule ScheduleFromInstructionOrder(const HloModule* module) {
+HloSchedule ScheduleFromInstructionOrder(HloModule* module) {
   HloSchedule schedule(module);
-  for (const HloComputation* computation : module->computations()) {
+  for (HloComputation* computation : module->computations()) {
     if (!computation->IsFusionComputation()) {
-      for (const HloInstruction* instruction : computation->instructions()) {
+      for (HloInstruction* instruction : computation->instructions()) {
         schedule.GetOrCreateSequence(computation).push_back(instruction);
       }
     }
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 49e46ecd00..48add75523 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -130,10 +130,10 @@ using ItemList = absl::InlinedVector<Item*, 3>;
 // before arbitrary elements.
 class InstructionList {
  public:
-  explicit InstructionList(const std::vector<const HloInstruction*>& order) {
+  explicit InstructionList(const HloInstructionSequence& order) {
     int64 position = 0;
     Item* last = nullptr;
-    for (const HloInstruction* inst : order) {
+    for (HloInstruction* inst : order.instructions()) {
       // Add a new item to the linked list.
       Item* item = new Item;
       item->next = nullptr;
@@ -151,7 +151,7 @@ class InstructionList {
       // to be monotonically increasing through the list, and so is still useful
       // for quickly(-ish) determining the order of arbitrary instructions in
       // the list.
-      item->instruction = const_cast<HloInstruction*>(inst);
+      item->instruction = inst;
       item->position = position;
       position++;
 
@@ -927,7 +927,7 @@ Item* PickRematerializationCandidate(
 
 StatusOr<int64> HloRematerialization::ComputePeakMemory(
     const HloComputation* computation,
-    const std::vector<const HloInstruction*>& order) const {
+    const HloInstructionSequence& order) const {
   InstructionList instruction_list(order);
   MemoryUsageTracker tracker(computation, size_function_, *points_to_analysis_,
                              instruction_list);
@@ -971,8 +971,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
           << HumanReadableNumBytes(computation_peak_memory_.at(computation));
   CHECK(!ContainsKey(rematerialized_computations_, computation));
 
-  InstructionList instruction_list(
-      schedule->sequence(computation).instructions());
+  InstructionList instruction_list(schedule->sequence(computation));
   MemoryUsageTracker memory_tracker(computation, size_function_,
                                     *points_to_analysis_, instruction_list);
   bool changed = false;
@@ -1184,7 +1183,7 @@ StatusOr<bool> HloRematerialization::RematerializeComputation(
   sequence.clear();
   for (auto* item = instruction_list.first(); item != nullptr;
        item = instruction_list.next(item)) {
-    const HloInstruction* instruction = item->instruction;
+    HloInstruction* instruction = item->instruction;
     sequence.push_back(instruction);
   }
   rematerialized_computations_.insert(computation);
@@ -1235,10 +1234,8 @@ StatusOr<bool> HloRematerialization::Run(HloModule* module) {
         if (node.context() == CallContext::kSequential) {
           TF_ASSIGN_OR_RETURN(
               computation_peak_memory_[node.computation()],
-              ComputePeakMemory(node.computation(),
-                                module->schedule()
-                                    .sequence(node.computation())
-                                    .instructions()));
+              ComputePeakMemory(node.computation(), module->schedule().sequence(
+                                                        node.computation())));
         }
         return Status::OK();
       },
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h
index 70d83c04f0..a07d348041 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.h
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h
@@ -87,9 +87,8 @@ class HloRematerialization : public HloModulePass {
   // peak memory is the maximum total size of all live HLO instruction values at
   // any program point. 'order' is the order in which the HLO instructions will
   // be emitted which is used to determine lifespans of HLO values.
-  StatusOr<int64> ComputePeakMemory(
-      const HloComputation* computation,
-      const std::vector<const HloInstruction*>& order) const;
+  StatusOr<int64> ComputePeakMemory(const HloComputation* computation,
+                                    const HloInstructionSequence& order) const;
 
   // Returns the peak memory usage of the called computations for the given
   // instruction. Zero is returned if the instruction calls no computations.
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.cc b/tensorflow/compiler/xla/service/hlo_schedule.cc
index a5780b7551..8f6eb974c5 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule.cc
@@ -46,8 +46,8 @@ namespace xla {
         << "No computation exists in HLO module with id " << computation_id;
     const HloComputation* computation = comp_it->second;
 
-    absl::flat_hash_map<int64, const HloInstruction*> id_to_instruction;
-    for (const HloInstruction* instruction : computation->instructions()) {
+    absl::flat_hash_map<int64, HloInstruction*> id_to_instruction;
+    for (HloInstruction* instruction : computation->instructions()) {
       id_to_instruction[instruction->unique_id()] = instruction;
     }
 
@@ -81,9 +81,8 @@ StatusOr<HloScheduleProto> HloSchedule::ToProto() const {
   return std::move(proto);
 }
 
-void HloSchedule::set_sequence(
-    const HloComputation* computation,
-    absl::Span<const HloInstruction* const> sequence) {
+void HloSchedule::set_sequence(const HloComputation* computation,
+                               absl::Span<HloInstruction* const> sequence) {
   set_sequence(computation, HloInstructionSequence(sequence));
 }
 
@@ -114,8 +113,8 @@ Status HloSchedule::UpdateComputationSchedule(
     const HloComputation* computation) {
   // Map from unique ID to HloInstruction pointer for instructions in the
   // computation.
-  absl::flat_hash_map<int, const HloInstruction*> id_to_instruction;
-  for (const HloInstruction* instruction : computation->instructions()) {
+  absl::flat_hash_map<int, HloInstruction*> id_to_instruction;
+  for (HloInstruction* instruction : computation->instructions()) {
     InsertOrDie(&id_to_instruction, instruction->unique_id(), instruction);
   }
 
@@ -128,7 +127,7 @@ Status HloSchedule::UpdateComputationSchedule(
   // Map from HloInstruction X to newly added instructions (instruction is in
   // computation, but not in schedule) which use X. If an instruction is not in
   // the map, then it has no users which are newly added instructions.
-  absl::flat_hash_map<const HloInstruction*, std::vector<const HloInstruction*>>
+  absl::flat_hash_map<const HloInstruction*, std::vector<HloInstruction*>>
       new_instruction_uses;
 
   // For each newly added instruction, this is the count of the instruction's
@@ -138,9 +137,9 @@ Status HloSchedule::UpdateComputationSchedule(
 
   // Create a worklist of newly added instructions which are ready to be added
   // to the schedule. Initialize worklist with those that have zero operands.
-  std::queue<const HloInstruction*> worklist;
+  std::queue<HloInstruction*> worklist;
 
-  for (const HloInstruction* instruction : computation->instructions()) {
+  for (HloInstruction* instruction : computation->instructions()) {
     if (ids_in_schedule.count(instruction->unique_id()) == 0) {
       // This is a newly added instruction which is not in the schedule.
       if (instruction->operands().empty()) {
@@ -161,17 +160,17 @@ Status HloSchedule::UpdateComputationSchedule(
   // Lambda which schedules all instructions on the worklist.
   auto schedule_worklist = [&]() {
     while (!worklist.empty()) {
-      const HloInstruction* instruction = worklist.front();
+      HloInstruction* instruction = worklist.front();
       worklist.pop();
       new_sequence.push_back(instruction);
-      std::vector<const HloInstruction*>* new_users =
+      std::vector<HloInstruction*>* new_users =
           tensorflow::gtl::FindOrNull(new_instruction_uses, instruction);
       if (new_users != nullptr) {
         // This just-scheduled instruction has users which are newly added to
         // the module. Update the number of unscheduled operands and push the
         // newly added instruction to the worklist if it is ready to
         // schedule.
-        for (const HloInstruction* new_user : *new_users) {
+        for (HloInstruction* new_user : *new_users) {
           unscheduled_operand_count.at(new_user)--;
           CHECK_GE(unscheduled_operand_count.at(new_user), 0);
           if (unscheduled_operand_count.at(new_user) == 0) {
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.h b/tensorflow/compiler/xla/service/hlo_schedule.h
index 0a714101ee..486ddbf499 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/hlo_schedule.h
@@ -35,14 +35,14 @@ class HloInstructionSequence {
  public:
   HloInstructionSequence() = default;
   explicit HloInstructionSequence(
-      absl::Span<const HloInstruction* const> instructions) {
-    for (const HloInstruction* instruction : instructions) {
+      absl::Span<HloInstruction* const> instructions) {
+    for (HloInstruction* instruction : instructions) {
       push_back(instruction);
     }
   }
 
   // Adds the instruction to the end of the sequence.
-  void push_back(const HloInstruction* instruction) {
+  void push_back(HloInstruction* instruction) {
     instruction_sequence_.push_back(instruction);
     id_sequence_.push_back(instruction->unique_id());
   }
@@ -56,7 +56,7 @@ class HloInstructionSequence {
   int64 size() const { return instruction_sequence_.size(); }
 
   // Returns the sequence of HLO instructions.
-  const std::vector<const HloInstruction*>& instructions() const {
+  const std::vector<HloInstruction*>& instructions() const {
     return instruction_sequence_;
   }
 
@@ -65,7 +65,7 @@ class HloInstructionSequence {
 
  private:
   // The sequence as HloInstructions.
-  std::vector<const HloInstruction*> instruction_sequence_;
+  std::vector<HloInstruction*> instruction_sequence_;
 
   // The sequence of HLO instructions, represented by their unique IDs. The
   // sequence is stored as both HloInstructions and unique IDs because the
@@ -98,7 +98,7 @@ class HloSchedule {
 
   // Sets the sequence for the given computation to the given sequence.
   void set_sequence(const HloComputation* computation,
-                    absl::Span<const HloInstruction* const> sequence);
+                    absl::Span<HloInstruction* const> sequence);
   void set_sequence(const HloComputation* computation,
                     HloInstructionSequence sequence);
 
diff --git a/tensorflow/compiler/xla/service/hlo_schedule_test.cc b/tensorflow/compiler/xla/service/hlo_schedule_test.cc
index 1424569ac1..0e56e6f760 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule_test.cc
@@ -56,10 +56,10 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
-  const std::vector<const HloInstruction*>& entry_schedule =
+  const auto& entry_schedule =
       schedule.sequence(module->entry_computation()).instructions();
 
   EXPECT_EQ(entry_schedule.size(), 6);
@@ -90,7 +90,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -139,7 +139,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -183,7 +183,7 @@ ENTRY main {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape());
       }));
 
@@ -244,7 +244,7 @@ ENTRY %WhileLoop () -> s32[] {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
@@ -313,7 +313,7 @@ ENTRY %WhileLoop () -> s32[] {
                           ParseHloString(module_str));
   TF_ASSERT_OK_AND_ASSIGN(
       HloSchedule schedule,
-      ScheduleModule(*module, [](const BufferValue& buffer) {
+      ScheduleModule(module.get(), [](const BufferValue& buffer) {
         return ShapeUtil::ByteSizeOf(buffer.shape(),
                                      /*pointer_size=*/sizeof(void*));
       }));
diff --git a/tensorflow/compiler/xla/service/interpreter/executable.cc b/tensorflow/compiler/xla/service/interpreter/executable.cc
index a06d6113e8..7635fbfed6 100644
--- a/tensorflow/compiler/xla/service/interpreter/executable.cc
+++ b/tensorflow/compiler/xla/service/interpreter/executable.cc
@@ -37,7 +37,7 @@ namespace xla {
 namespace interpreter {
 
 InterpreterExecutable::InterpreterExecutable(
-    std::unique_ptr<const HloModule> hlo_module,
+    std::unique_ptr<HloModule> hlo_module,
     std::unique_ptr<HloEvaluator> evaluator)
     : Executable(std::move(hlo_module), /*hlo_profile_printer=*/nullptr,
                  /*hlo_profile_index_map=*/nullptr),
diff --git a/tensorflow/compiler/xla/service/interpreter/executable.h b/tensorflow/compiler/xla/service/interpreter/executable.h
index 3b1ebce0c7..bda13d3763 100644
--- a/tensorflow/compiler/xla/service/interpreter/executable.h
+++ b/tensorflow/compiler/xla/service/interpreter/executable.h
@@ -42,7 +42,7 @@ namespace interpreter {
 // buffer allocation. Refer to interpreter/README.md for more.
 class InterpreterExecutable : public Executable {
  public:
-  InterpreterExecutable(std::unique_ptr<const HloModule> hlo_module,
+  InterpreterExecutable(std::unique_ptr<HloModule> hlo_module,
                         std::unique_ptr<HloEvaluator> evaluator);
   ~InterpreterExecutable() override;
 
-- 
GitLab


From f5cb8c654f5c35178c6b0c2bb034298180836619 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Mon, 19 Nov 2018 11:18:58 -0800
Subject: [PATCH 0502/1554] Fix TRT conversion of bias_add and add
 corresponding c++ tests.

PiperOrigin-RevId: 222107103
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 132 ++++++++-----
 .../tensorrt/convert/convert_nodes_test.cc    | 186 ++++++++++++++----
 .../tensorrt/test/biasadd_matmul_test.py      |  32 +--
 .../test/tf_trt_integration_test_base.py      |  28 +++
 4 files changed, 271 insertions(+), 107 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 431e363612..af9bbbfdfd 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -54,10 +54,10 @@ limitations under the License.
 // would work!
 #define TFTRT_CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2)
 
-#define TFTRT_INTERNAL_ERROR_AT_NODE(node)                               \
-  do {                                                                   \
-    return tensorflow::errors::Internal(                                 \
-        "TFTRT::", __FUNCTION__, "failed to add TRT layer, at: ", node); \
+#define TFTRT_INTERNAL_ERROR_AT_NODE(node)                                \
+  do {                                                                    \
+    return tensorflow::errors::Internal(                                  \
+        "TFTRT::", __FUNCTION__, " failed to add TRT layer, at: ", node); \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
@@ -187,10 +187,34 @@ Status ValidateTensorProperties(const string& producer_node_type,
   return Status::OK();
 }
 
+string DebugString(const nvinfer1::DimensionType type) {
+  switch (type) {
+    case nvinfer1::DimensionType::kSPATIAL:
+      return "kSPATIAL";
+    case nvinfer1::DimensionType::kCHANNEL:
+      return "kCHANNEL";
+    case nvinfer1::DimensionType::kINDEX:
+      return "kINDEX";
+    case nvinfer1::DimensionType::kSEQUENCE:
+      return "kSEQUENCE";
+    default:
+      return StrCat(static_cast<int>(type), "=unknown");
+  }
+}
+
 string DebugString(const nvinfer1::Dims& dims) {
   string out = StrCat("nvinfer1::Dims(nbDims=", dims.nbDims, ", d=");
   for (int i = 0; i < dims.nbDims; ++i) {
-    StrAppend(&out, dims.d[i], ",");
+    StrAppend(&out, dims.d[i], "[", DebugString(dims.type[i]), "],");
+  }
+  StrAppend(&out, ")");
+  return out;
+}
+
+string DebugString(const nvinfer1::Permutation& permutation, int len) {
+  string out = "nvinfer1::Permutation(";
+  for (int i = 0; i < len; ++i) {
+    StrAppend(&out, permutation.order[i], ",");
   }
   StrAppend(&out, ")");
   return out;
@@ -381,7 +405,7 @@ size_t TRT_ShapedWeights::size_bytes() const {
 
 string TRT_ShapedWeights::DebugString() const {
   return StrCat("TRT_ShapedWeights(shape=", convert::DebugString(shape_),
-                ", type=", type_,
+                ", type=", DataTypeString(type_),
                 ", values=", reinterpret_cast<uintptr_t>(GetValues()), ")");
 }
 
@@ -935,6 +959,8 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor,
   for (int32_t i = 0; i < dims.nbDims; ++i) {
     permutation.order[i] = order_with_batch_dim[i + 1] - 1;
   }
+  VLOG(1) << "TransposeTensor permutation: "
+          << DebugString(permutation, dims.nbDims);
   layer->setFirstTranspose(permutation);
 
   nvinfer1::Dims reshape_dims;
@@ -1750,73 +1776,71 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertScale(OpConverterParams* params) {
+tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights()) {
-    return tensorflow::errors::Unimplemented(
-        "ConvertScale only supports tensor<op>weight: ", node_def.name());
+    return errors::InvalidArgument("Input expects tensor and weights, at ",
+                                   node_def.name());
   }
+  if (params->validation_only) return Status::OK();
 
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-  TRT_ShapedWeights weights = inputs.at(1).weights();
-  if (params->converter->is_fp16()) {
-    weights = ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
-  }
-
-  TRT_ShapedWeights empty_weights(weights.type_);
+  const nvinfer1::Dims original_dims = tensor->getDimensions();
   TFAttrs attrs(node_def);
-
-  const auto data_format = attrs.get<string>("data_format");
-  int channel_index;
-  const auto dims = tensor->getDimensions();
-  if (data_format == "NHWC") {
-    //  1). NHWC is really N+C
-    channel_index = dims.nbDims - 1;  // batch dimension is implicit here!
-  } else {
-    //  2). NCHW is really N+CHW
-    channel_index = 0;  // batch dimension is implicit here!
-  }
+  const string data_format = attrs.get<string>("data_format");
+  const int channel_index =
+      (data_format == "NHWC" ? original_dims.nbDims - 1 : 0);
 
   nvinfer1::Permutation permutation;
-  for (int32_t i = 0; i < dims.nbDims; ++i) {
-    permutation.order[i] = i;
-  }
-
-  if (channel_index >= 0) {
+  if (channel_index != 0) {
+    // Permute the dimensions so that the channel dimension is the first
+    // dimension.
+    for (int i = 0; i < original_dims.nbDims; ++i) {
+      permutation.order[i] = i;
+    }
     permutation.order[0] = channel_index;
     permutation.order[channel_index] = 0;
-  } else {
-    return tensorflow::errors::Unimplemented(
-        "TFTRT::BiasAdd cannot apply on batch dimension, at ", node_def.name());
   }
+  VLOG(1) << "ConvertBiasAdd permutation: "
+          << DebugString(permutation, original_dims.nbDims);
 
   // TensorRT addScale requires input to be of rank 3, we need to apply
-  // transpose as well as reshape
-  if (channel_index != 0 || dims.nbDims != 3) {
+  // transpose as well as reshape.
+  // TODO(laigd): this doesn't match what the TRT doc says, fix the doc?
+  if (channel_index != 0 || original_dims.nbDims != 3) {
     nvinfer1::IShuffleLayer* shuffle_layer =
         params->converter->network()->addShuffle(
             *const_cast<nvinfer1::ITensor*>(tensor));
     TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name());
+    // NOTE(laigd): for some reason we need to apply the reshape
+    // unconditionally. The default shape has nbDims==-1 and it seems the
+    // behavior is undefined in some cases.
     nvinfer1::Dims reshape_dims;
     reshape_dims.nbDims = 3;
-    reshape_dims.d[0] = 0;                          // 0 copy from the input
-    reshape_dims.d[1] = dims.nbDims >= 2 ? 0 : 1;   // 0 copy from the input
-    reshape_dims.d[2] = dims.nbDims >= 3 ? -1 : 1;  // -1 infer from the rest
+    // 0 means copying from input; -1 means inferring from the rest.
+    reshape_dims.d[0] = 0;
+    reshape_dims.d[1] = original_dims.nbDims >= 2 ? 0 : 1;
+    reshape_dims.d[2] = original_dims.nbDims >= 3 ? -1 : 1;
+    shuffle_layer->setReshapeDimensions(reshape_dims);
+
     if (channel_index != 0) {
-      // maybe we do not need this check. concerned about TRT optimization
       shuffle_layer->setFirstTranspose(permutation);
     }
-    shuffle_layer->setReshapeDimensions(reshape_dims);
     tensor = shuffle_layer->getOutput(0);
   }
 
+  TRT_ShapedWeights weights = inputs.at(1).weights();
+  if (params->converter->is_fp16()) {
+    weights = ConvertFP32ToFP16(params->weight_store, weights);
+  }
   nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL;
   if (weights.shape_.d[0] == 1) {
     mode = nvinfer1::ScaleMode::kUNIFORM;
   }
 
+  TRT_ShapedWeights empty_weights(weights.type_);
   nvinfer1::IScaleLayer* layer = params->converter->network()->addScale(
       *const_cast<nvinfer1::ITensor*>(tensor), mode, weights.GetTrtWeights(),
       empty_weights.GetTrtWeights(), empty_weights.GetTrtWeights());
@@ -1824,17 +1848,22 @@ tensorflow::Status ConvertScale(OpConverterParams* params) {
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
-  // restore transpose & reshape
-  if (channel_index != 0 || dims.nbDims != 3) {
+  // Restore transpose & reshape.
+  if (channel_index != 0 || original_dims.nbDims != 3) {
     nvinfer1::IShuffleLayer* shuffle_layer =
-        params->converter->network()->addShuffle(
-            *const_cast<nvinfer1::ITensor*>(output_tensor));
+        params->converter->network()->addShuffle(*output_tensor);
     TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name());
-    nvinfer1::Dims reshape_dims = dims;
-    int tmp = reshape_dims.d[channel_index];
-    reshape_dims.d[channel_index] = reshape_dims.d[0];
-    reshape_dims.d[0] = tmp;
+    // NOTE: for same reason as mentioned above we need to apply the reshape
+    // unconditionally.
+    nvinfer1::Dims reshape_dims = original_dims;
+    if (channel_index != 0) {
+      // NOTE: according to NVIDIA dimension types are deprecated, so we don't
+      // need to copy them back.
+      reshape_dims.d[channel_index] = original_dims.d[0];
+      reshape_dims.d[0] = original_dims.d[channel_index];
+    }
     shuffle_layer->setReshapeDimensions(reshape_dims);
+
     if (channel_index != 0) {
       shuffle_layer->setSecondTranspose(permutation);
     }
@@ -1842,7 +1871,7 @@ tensorflow::Status ConvertScale(OpConverterParams* params) {
   }
 
   params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
+  return Status::OK();
 }
 
 Status GetTensorDimsWithProtoShape(const Tensor& tensor,
@@ -2660,6 +2689,7 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
 
 void TrtNodeValidator::RegisterOpValidators() {
   // TODO(laigd): support all op types.
+  op_validators_["BiasAdd"] = ConvertBiasAdd;
   op_validators_["Const"] = ConvertConst;
   op_validators_["Transpose"] = ConvertTranspose;
   op_validators_["Reshape"] = ConvertReshape;
@@ -2673,7 +2703,7 @@ void Converter::RegisterOpConverters() {
   op_registry_["Relu"] = ConvertActivation;
   op_registry_["MaxPool"] = ConvertPool;
   op_registry_["AvgPool"] = ConvertPool;
-  op_registry_["BiasAdd"] = ConvertScale;
+  op_registry_["BiasAdd"] = ConvertBiasAdd;
   op_registry_["Const"] = ConvertConst;
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index c3a39395f3..862754f3d2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -36,6 +36,8 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/protobuf/config.pb.h"  // NOLINT
+#include "tensorflow/core/public/session.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
@@ -341,28 +343,46 @@ TEST_F(ValidatorTest, ConvertToTensorOrWeights) {
                                           graph_properties, &output));
     ValidateWeights<float>(output.weights(), {2}, {1.0, 2.0});
   }
-  // Convert non-Const. We test the case where the non-batch dimemsion is
-  // unknown as well, to make sure the validator allows that.
-  for (const int32 non_batch_dim : {-1, 2}) {
-    const int32 batch_size = 12;
 
+  // Helper method to run ConvertToTensorOrWeights() with predefined parameters.
+  auto convert_to_tensor_or_weights = [this](const std::vector<int64>& dims,
+                                             TRT_TensorOrWeights* output) {
     Scope s = Scope::NewRootScope();
-    ops::Placeholder::Attrs attrs;
-    TF_EXPECT_OK(TensorShapeUtils::MakeShape(
-        std::vector<int32>{batch_size, non_batch_dim}, &attrs.shape_));
+    const auto attrs = ops::Placeholder::Shape(PartialTensorShape{dims});
     auto feed = ops::Placeholder(s.WithOpName("feed"), DT_FLOAT, attrs);
     auto add = ops::Add(s.WithOpName("add"), feed, feed);
 
     grappler::GrapplerItem item;
     TF_EXPECT_OK(s.ToGraphDef(&item.graph));
-
     grappler::GraphProperties graph_properties(item);
     TF_EXPECT_OK(graph_properties.InferStatically(true));
-
-    auto& node_def = add.operation.node()->def();
+    const NodeDef& node_def = add.operation.node()->def();
+    return this->ConvertToTensorOrWeights(node_def, /*output_port=*/0,
+                                          graph_properties, output);
+  };
+  // Convert non-Const with #dims > nvinfer1::Dims::MAX_DIMS+1.
+  {
     TRT_TensorOrWeights output;
-    ExpectStatus(ConvertToTensorOrWeights(node_def, /*output_port=*/0,
-                                          graph_properties, &output));
+    ExpectStatus(
+        convert_to_tensor_or_weights(
+            std::vector<int64>(nvinfer1::Dims::MAX_DIMS + 2, 1), &output),
+        error::OUT_OF_RANGE, "Input tensor rank is greater than 9");
+  }
+  // Convert non-Const with #dims < 2.
+  {
+    TRT_TensorOrWeights output;
+    ExpectStatus(
+        convert_to_tensor_or_weights({1}, &output), error::INVALID_ARGUMENT,
+        "Input tensor with rank<2 is not supported since the first dimension "
+        "is treated as batch dimension by TRT");
+  }
+  // Convert non-Const. We test the case where the non-batch dimemsion is
+  // unknown as well, to make sure the validator allows that.
+  for (const int32 non_batch_dim : {-1, 2}) {
+    const int32 batch_size = 12;
+    TRT_TensorOrWeights output;
+    ExpectStatus(
+        convert_to_tensor_or_weights({batch_size, non_batch_dim}, &output));
     EXPECT_EQ(true, output.is_tensor());
     EXPECT_EQ(batch_size, output.batch_size());
     EXPECT_NE(nullptr, output.tensor());
@@ -691,6 +711,7 @@ class OpConverterTest : public ::testing::Test {
     validator_inputs_.clear();
   }
 
+  // TODO(laigd): test fp16 and int8 support.
   void BuildAndRun(const char* input_name, const std::vector<float>& input_data,
                    const char* output_name, std::vector<float>* output_data) {
     // Mark the output tensor as TRT engine output.
@@ -1070,15 +1091,14 @@ TEST_F(OpConverterTest, ConvertMatMul) {
         "Input expects tensor and weights, at my_matmul");
   }
 
-  // Get the NodeDef for Reshape.
+  // Get the NodeDef for MatMul.
   auto get_matmul_nodedef = [](DataType dtype, bool transpose_a,
                                bool transpose_b) -> NodeDef {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), dtype);
     auto weights = ops::Placeholder(s.WithOpName("weights"), dtype);
-    ops::MatMul::Attrs matmul_attrs;
-    matmul_attrs.transpose_a_ = transpose_a;
-    matmul_attrs.transpose_b_ = transpose_b;
+    const auto matmul_attrs =
+        ops::MatMul::TransposeA(transpose_a).TransposeB(transpose_b);
     auto matmul =
         ops::MatMul(s.WithOpName("my_matmul"), input, weights, matmul_attrs);
     return matmul.operation.node()->def();
@@ -1094,45 +1114,125 @@ TEST_F(OpConverterTest, ConvertMatMul) {
         node_def, error::UNIMPLEMENTED,
         "Data type is not supported, for node my_matmul got int32");
   }
-  {
-    // transpose_a is set.
-    for (bool transpose_b : {false, true}) {
-      Reset();
-      NodeDef node_def =
-          get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/true, transpose_b);
-      AddTestTensor("input", {2}, /*batch_size=*/1);
-      AddTestWeights<float>("weights", {2, 2}, {0, 1, 2, 3});
-      RunValidationAndConversion(
-          node_def, error::INVALID_ARGUMENT,
-          "transpose_a is not supported for TensorRT FullyConnected");
+  // transpose_a is set.
+  for (bool transpose_b : {false, true}) {
+    Reset();
+    NodeDef node_def =
+        get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/true, transpose_b);
+    AddTestTensor("input", {2}, /*batch_size=*/1);
+    AddTestWeights<float>("weights", {2, 2}, {0, 1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "transpose_a is not supported for TensorRT FullyConnected");
+  }
+  // OK.
+  for (bool transpose_b : {false, true}) {
+    Reset();
+    NodeDef node_def =
+        get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/false, transpose_b);
+    AddTestTensor("input", {2}, /*batch_size=*/1);
+    AddTestWeights<float>("weights", {2, 2}, {0, 1, 2, 3});
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(2);
+    BuildAndRun("input", {0, 1}, "my_matmul", &output_data);
+    if (transpose_b) {
+      EXPECT_THAT(output_data, ElementsAre(1, 3));
+    } else {
+      EXPECT_THAT(output_data, ElementsAre(2, 3));
     }
   }
-  {
-    // OK.
-    for (bool transpose_b : {false, true}) {
-      Reset();
-      NodeDef node_def =
-          get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/false, transpose_b);
-      AddTestTensor("input", {2}, /*batch_size=*/1);
-      AddTestWeights<float>("weights", {2, 2}, {0, 1, 2, 3});
-      RunConversion(node_def);
+}
+
+template <DataType dtype>
+void TestConvertBiasAdd(OpConverterTest* test) {
+  // Get the NodeDef for BiasAdd.
+  auto get_biasadd_nodedef = [](const string& data_format) -> NodeDef {
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), dtype);
+    auto weights = ops::Placeholder(s.WithOpName("weights"), dtype);
+    const auto biasadd_attrs = ops::BiasAdd::DataFormat(data_format);
+    auto biasadd =
+        ops::BiasAdd(s.WithOpName("my_biasadd"), input, weights, biasadd_attrs);
+    return biasadd.operation.node()->def();
+  };
+
+  typedef typename EnumToDataType<dtype>::Type CType;
+  for (const string& data_format : {"NHWC", "NCHW"}) {
+    for (const int trt_input_rank : {1, 2, 3, 4}) {
+      test->Reset();
+      NodeDef node_def = get_biasadd_nodedef(data_format);
+
+      // Add input, dims_array will be like {2, 1, ..., 1, 3}
+      std::vector<int32> dims_array(trt_input_rank, 1);
+      if (trt_input_rank == 1) {
+        dims_array[0] = (data_format == "NHWC" ? 3 : 2);
+      } else {
+        dims_array[0] = 2;
+        dims_array[trt_input_rank - 1] = 3;
+      }
+      test->AddTestTensor("input", dims_array, /*batch_size=*/1);
+
+      // Add bias weights.
+      const int channel_size = (data_format == "NHWC" ? 3 : 2);
+      std::vector<CType> bias(channel_size);
+      std::iota(bias.begin(), bias.end(), 1);  // bias will be {1, 2, 3, ...}
+      test->AddTestWeights<CType>("weights", {channel_size}, bias);
+
+      // Run the conversion.
+      test->RunValidationAndConversion(node_def);
       TRT_TensorOrWeights output;
-      TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output));
+      TF_EXPECT_OK(test->GetTensorOrWeights("my_biasadd", &output));
       EXPECT_TRUE(output.is_tensor());
-      EXPECT_TRUE(TrtDimsEqualsArray({2}, output.tensor()->getDimensions()))
+      EXPECT_TRUE(
+          TrtDimsEqualsArray(dims_array, output.tensor()->getDimensions()))
           << output.DebugString();
 
-      std::vector<float> output_data(2);
-      BuildAndRun("input", {0, 1}, "my_matmul", &output_data);
-      if (transpose_b) {
-        EXPECT_THAT(output_data, ElementsAre(1, 3));
+      // Build and run the engine.
+      const int num_input = TrtDimsNumElements(GetTestDims(dims_array));
+      ASSERT_EQ(trt_input_rank > 1 ? 6 : (data_format == "NHWC" ? 3 : 2),
+                num_input);
+      std::vector<CType> output_data(num_input);
+      test->BuildAndRun("input", std::vector<CType>(num_input, CType(0)),
+                        "my_biasadd", &output_data);
+      if (trt_input_rank == 1) {
+        if (data_format == "NHWC") {
+          EXPECT_THAT(output_data, ElementsAre(1, 2, 3));
+        } else {
+          EXPECT_THAT(output_data, ElementsAre(1, 2));
+        }
       } else {
-        EXPECT_THAT(output_data, ElementsAre(2, 3));
+        if (data_format == "NHWC") {
+          EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 1, 2, 3));
+        } else {
+          EXPECT_THAT(output_data, ElementsAre(1, 1, 1, 2, 2, 2));
+        }
       }
     }
   }
 }
 
+TEST_F(OpConverterTest, ConvertBiasAdd) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_biasadd", "BiasAdd", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Input expects tensor and weights, at my_biasadd");
+  }
+
+  // OK.
+  TestConvertBiasAdd<DT_FLOAT>(this);
+  // TODO(laigd): uncomment this after cl/220663893 is submitted.
+  // TestConvertBiasAdd<DT_INT32>(this);
+  // TestConvertBiasAdd<DT_HALF>(this);
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 7545bb9df2..6546ef6477 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -41,6 +41,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     input_name = "input"
     input_matrix_rows = 4
     input_matrix_columns = 144
+    # Note that tf.nn.bias_add supports up to 5 dimensions.
     input_dims = [input_matrix_rows, input_matrix_columns]
     output_name = "output"
     g = ops.Graph()
@@ -74,18 +75,18 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
       x5 = nn.bias_add(x5, b)
       x5 = gen_array_ops.reshape(x5, [4, -1])
 
-      x6 = gen_array_ops.reshape(x, [4, 12, 12])
-      b = self._ConstOp((12,))
+      x6 = gen_array_ops.reshape(x, [4, 24, 6])
+      b = self._ConstOp((6,))
       x6 = nn.bias_add(x6, b, data_format="NHWC")
       x6 = gen_array_ops.reshape(x6, [4, -1])
 
-      x7 = gen_array_ops.reshape(x, [4, 12, 3, 4])
-      b = self._ConstOp((4,))
+      x7 = gen_array_ops.reshape(x, [4, 12, 4, 3])
+      b = self._ConstOp((3,))
       x7 = nn.bias_add(x7, b, data_format="NHWC")
       x7 = gen_array_ops.reshape(x7, [4, -1])
 
-      x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2])
-      b = self._ConstOp((2,))
+      x8 = gen_array_ops.reshape(x, [4, 4, 3, 2, 6])
+      b = self._ConstOp((6,))
       x8 = nn.bias_add(x8, b, data_format="NHWC")
       x8 = gen_array_ops.reshape(x8, [4, -1])
 
@@ -94,13 +95,13 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
       x9 = nn.bias_add(x9, b, data_format="NCHW")
       x9 = gen_array_ops.reshape(x9, [4, -1])
 
-      x10 = gen_array_ops.reshape(x, [4, 12, 3, 4])
-      b = self._ConstOp((12,))
+      x10 = gen_array_ops.reshape(x, [4, 3, 4, 12])
+      b = self._ConstOp((3,))
       x10 = nn.bias_add(x10, b, data_format="NCHW")
       x10 = gen_array_ops.reshape(x10, [4, -1])
 
-      x11 = gen_array_ops.reshape(x, [4, 12, 12])
-      b = self._ConstOp((12,))
+      x11 = gen_array_ops.reshape(x, [4, 6, 24])
+      b = self._ConstOp((6,))
       x11 = nn.bias_add(x11, b, data_format="NCHW")
       x11 = gen_array_ops.reshape(x11, [4, -1])
 
@@ -116,9 +117,14 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-    return super(BiasaddMatMulTest,
-                 self).GetConversionParams(run_params)._replace(
-                     max_batch_size=4, maximum_cached_engines=1)
+    conversion_params = super(BiasaddMatMulTest,
+                              self).GetConversionParams(run_params)
+    return conversion_params._replace(
+        max_batch_size=4,
+        maximum_cached_engines=1,
+        # Disable layout optimizer, since it will convert BiasAdd with NHWC
+        # format to NCHW format under four dimentional input.
+        rewriter_config=trt_test.OptimizerDisabledRewriterConfig())
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 1763f3b22c..c3cff28574 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -30,6 +30,7 @@ from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
 # pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import graph_io
 from tensorflow.python.framework import importer
@@ -65,6 +66,28 @@ class GraphState(object):
   INFERENCE = 2
 
 
+def OptimizerDisabledRewriterConfig():
+  """Returns a RewriterConfig with all default Grappler optimizers disabled."""
+  rewriter_config = rewriter_config_pb2.RewriterConfig()
+  off = rewriter_config_pb2.RewriterConfig.OFF
+  rewriter_config.layout_optimizer = off
+  rewriter_config.constant_folding = off
+  rewriter_config.shape_optimization = off
+  rewriter_config.remapping = off
+  rewriter_config.arithmetic_optimization = off
+  rewriter_config.dependency_optimization = off
+  rewriter_config.loop_optimization = off
+  rewriter_config.function_optimization = off
+  rewriter_config.debug_stripper = off
+  rewriter_config.disable_model_pruning = True
+  rewriter_config.scoped_allocator_optimization = off
+  rewriter_config.memory_optimization = (
+      rewriter_config_pb2.RewriterConfig.NO_MEM_OPT)
+  rewriter_config.pin_to_host_optimization = off
+  rewriter_config.auto_parallel.enable = False
+  return rewriter_config
+
+
 class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   """Class to test Tensorflow-TensorRT integration."""
 
@@ -443,6 +466,11 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       # types.
       scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
       dims = params.input_dims[i]
+      # TODO(laigd): add debug options. E.g. we can set the input data to be
+      # continuous natural numbers:
+      # seq = np.arange(np.prod(dims))
+      # seq.resize(dims)
+      # input_data.append(scale * seq.astype(dtype))
       input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
     self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL)
 
-- 
GitLab


From 5f9852c3ea1046513369ef9c0b2a2c6c103b147d Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 19 Nov 2018 11:22:32 -0800
Subject: [PATCH 0503/1554] Fix how keras's model_to_estimator function is
 exported.

Fix issue if estimator is not present when API files are generated
but installed later.

PiperOrigin-RevId: 222107827
---
 tensorflow/python/keras/estimator/__init__.py | 61 ++++++++++++++-----
 1 file changed, 46 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/keras/estimator/__init__.py b/tensorflow/python/keras/estimator/__init__.py
index b244beb5b5..3c1a63d6df 100644
--- a/tensorflow/python/keras/estimator/__init__.py
+++ b/tensorflow/python/keras/estimator/__init__.py
@@ -24,23 +24,54 @@ from tensorflow.python.util.tf_export import tf_export
 # As long as you depend //third_party/py/tensorflow:tensorflow target
 # everything will work as normal.
 
-try:
-  from tensorflow.python.estimator import keras as keras_lib  # pylint: disable=g-import-not-at-top
-  model_to_estimator = tf_export('keras.estimator.model_to_estimator')(
-      keras_lib.model_to_estimator)
-except Exception:  # pylint: disable=broad-except
-
-  # pylint: disable=unused-argument
-  def stub_model_to_estimator(keras_model=None,
-                              keras_model_path=None,
-                              custom_objects=None,
-                              model_dir=None,
-                              config=None):
+
+# LINT.IfChange
+@tf_export('keras.estimator.model_to_estimator')
+def model_to_estimator(
+    keras_model=None,
+    keras_model_path=None,
+    custom_objects=None,
+    model_dir=None,
+    config=None):
+  """Constructs an `Estimator` instance from given keras model.
+
+  For usage example, please see:
+  [Creating estimators from Keras
+  Models](https://tensorflow.org/guide/estimators#model_to_estimator).
+
+  Args:
+    keras_model: A compiled Keras model object. This argument is mutually
+      exclusive with `keras_model_path`.
+    keras_model_path: Path to a compiled Keras model saved on disk, in HDF5
+      format, which can be generated with the `save()` method of a Keras model.
+      This argument is mutually exclusive with `keras_model`.
+    custom_objects: Dictionary for custom objects.
+    model_dir: Directory to save `Estimator` model parameters, graph, summary
+      files for TensorBoard, etc.
+    config: `RunConfig` to config `Estimator`.
+
+  Returns:
+    An Estimator from given keras model.
+
+  Raises:
+    ValueError: if neither keras_model nor keras_model_path was given.
+    ValueError: if both keras_model and keras_model_path was given.
+    ValueError: if the keras_model_path is a GCS URI.
+    ValueError: if keras_model has not been compiled.
+  """
+  try:
+    from tensorflow_estimator.python.estimator import keras as keras_lib  # pylint: disable=g-import-not-at-top
+  except ImportError:
     raise NotImplementedError(
         'tf.keras.estimator.model_to_estimator function not available in your '
         'installation.')
-  # pylint: enable=unused-argument
+  keras_lib.model_to_estimator(
+      keras_model=keras_model,
+      keras_model_path=keras_model_path,
+      custom_objects=custom_objects,
+      model_dir=model_dir,
+      config=config)
+
+# LINT.ThenChange(//third_party/tensorflow_estimator/python/estimator/keras.py)
 
-  model_to_estimator = tf_export('keras.estimator.model_to_estimator')(
-      stub_model_to_estimator)
 
-- 
GitLab


From bbea5170c8dcf80fa25166fbf5ca06c74ad2d156 Mon Sep 17 00:00:00 2001
From: Vinu Rajashekhar <vinuraja@google.com>
Date: Mon, 19 Nov 2018 11:25:38 -0800
Subject: [PATCH 0504/1554] Adds conditional tracing around
 ExecutorState::PropagateOutputs.

PiperOrigin-RevId: 222108412
---
 tensorflow/core/common_runtime/executor.cc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 1e68954827..b9115f86ff 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2046,6 +2046,23 @@ Status ExecutorState::ProcessOutputs(const NodeItem& item, OpKernelContext* ctx,
 void ExecutorState::PropagateOutputs(const TaggedNode& tagged_node,
                                      const NodeItem* item, EntryVector* outputs,
                                      TaggedNodeSeq* ready) {
+  auto activity_handle =
+      [&]() -> std::unique_ptr<tracing::TraceCollector::Handle> {
+    if (TF_PREDICT_FALSE(trace_collector_ != nullptr &&
+                         trace_collector_->IsEnabledForActivities(
+                             false /* is_expensive */))) {
+      const string& op_name = item->kernel->name();
+      // Intentionally using ExecutorPropagateOutputs as the first key so that
+      // users are aware that it's not the op invocation.
+      return trace_collector_->CreateActivityHandle(
+          "ExecutorPropagateOutputs",
+          strings::StrCat(op_name, "#id=", step_id_, "#"),
+          false /* is_expensive */);
+    } else {
+      return nullptr;
+    }
+  }();
+
   const Node* node = tagged_node.node;
   FrameState* input_frame = tagged_node.input_frame;
   const int64 input_iter = tagged_node.input_iter;
-- 
GitLab


From b4fc367cdd3ecc2c05d4b56470358fc1381265c5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 11:30:18 -0800
Subject: [PATCH 0505/1554] Speed up distribute_coordinator_test by changing
 sleep-when-worker-is-waiting-for-chief-to-initialize-variables from 30
 seconds to 0.5 seconds.

PiperOrigin-RevId: 222109155
---
 tensorflow/python/distribute/BUILD                    |  1 -
 .../python/distribute/distribute_coordinator_test.py  | 11 +++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 04592ee6ed..83c3901441 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -109,7 +109,6 @@ py_library(
 
 py_test(
     name = "distribute_coordinator_test",
-    size = "large",
     srcs = ["distribute_coordinator_test.py"],
     srcs_version = "PY2AND3",
     tags = [
diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py
index 5d336648ce..0c1ee8c87e 100644
--- a/tensorflow/python/distribute/distribute_coordinator_test.py
+++ b/tensorflow/python/distribute/distribute_coordinator_test.py
@@ -47,6 +47,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import monitored_session
+from tensorflow.python.training import session_manager
 
 
 CHIEF = distribute_coordinator._TaskType.CHIEF
@@ -930,4 +931,14 @@ class RunStandardTensorflowServerTest(test.TestCase):
 if __name__ == "__main__":
   # TODO(yuefengz): find a smart way to terminite std server threads.
   with test.mock.patch.object(sys, "exit", os._exit):
+    # Reduce `recovery_wait_secs` from 30 seconds so the test completes quickly.
+    orig_init = session_manager.SessionManager.__init__
+
+    def new_init(*args, **kwargs):
+      kwargs.pop("recovery_wait_secs", None)
+      kwargs["recovery_wait_secs"] = 0.5
+      orig_init(*args, **kwargs)
+
+    session_manager.SessionManager.__init__ = new_init
+
     test.main()
-- 
GitLab


From 4a79a2c11547717daa248eda45fae4fd2bf95ef5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 11:34:02 -0800
Subject: [PATCH 0506/1554] Improve tfgan test asserts.

PiperOrigin-RevId: 222109876
---
 tensorflow/contrib/gan/python/train_test.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index 64d6706199..31d9e82700 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -519,7 +519,7 @@ class GANLossTest(test.TestCase, parameterized.TestCase):
     """Test output type."""
     loss = train.gan_loss(get_gan_model_fn(), add_summaries=True)
     self.assertIsInstance(loss, namedtuples.GANLoss)
-    self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0)
+    self.assertNotEmpty(ops.get_collection(ops.GraphKeys.SUMMARIES))
 
   @parameterized.named_parameters(
       ('cyclegan', create_cyclegan_model),
@@ -528,7 +528,7 @@ class GANLossTest(test.TestCase, parameterized.TestCase):
   def test_cyclegan_output_type(self, get_gan_model_fn):
     loss = train.cyclegan_loss(get_gan_model_fn(), add_summaries=True)
     self.assertIsInstance(loss, namedtuples.CycleGANLoss)
-    self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0)
+    self.assertNotEmpty(ops.get_collection(ops.GraphKeys.SUMMARIES))
 
   @parameterized.named_parameters(
       ('gan', create_gan_model, False),
@@ -923,8 +923,7 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase):
         model, loss, generator_optimizer=g_opt, discriminator_optimizer=d_opt)
     self.assertIsInstance(train_ops, namedtuples.GANTrainOps)
     # No new trainable variables should have been added.
-    self.assertEqual(num_trainable_vars,
-                     len(variables_lib.get_trainable_variables()))
+    self.assertLen(variables_lib.get_trainable_variables(), num_trainable_vars)
 
     g_sync_init_op = g_opt.get_init_tokens_op(num_tokens=1)
     d_sync_init_op = d_opt.get_init_tokens_op(num_tokens=1)
-- 
GitLab


From 96f1e4eedc0f421c90fb92b58ffa3268f90a874d Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 19 Nov 2018 11:34:43 -0800
Subject: [PATCH 0507/1554] [XLA] Simplify while-loop induction variables into
 a single variable.

PiperOrigin-RevId: 222110003
---
 tensorflow/compiler/xla/service/BUILD         |   6 +
 .../xla/service/while_loop_simplifier.cc      | 262 ++++++++++++++++++
 .../xla/service/while_loop_simplifier_test.cc | 108 +++++++-
 3 files changed, 364 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index da185db2e3..8295a9530d 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1730,7 +1730,9 @@ cc_library(
         ":hlo",
         ":hlo_pass",
         ":hlo_query",
+        ":pattern_matcher",
         ":while_loop_analysis",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
@@ -1743,9 +1745,13 @@ tf_cc_test(
     name = "while_loop_simplifier_test",
     srcs = ["while_loop_simplifier_test.cc"],
     deps = [
+        ":algebraic_simplifier",
         ":hlo",
+        ":hlo_cse",
         ":hlo_dce",
         ":hlo_matchers",
+        ":hlo_pass",
+        ":hlo_pass_pipeline",
         ":tuple_simplifier",
         ":while_loop_simplifier",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index 646f75ca2a..c4790a7f19 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -19,13 +19,17 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/service/call_inliner.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_query.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
 #include "tensorflow/compiler/xla/service/while_loop_analysis.h"
 
 namespace xla {
 
+namespace m = match;
 using absl::optional;
 using hlo_query::ContainsInstrWithOpcode;
 
@@ -738,6 +742,248 @@ static StatusOr<bool> TryFlattenNestedTuples(HloInstruction* while_op) {
   return true;
 }
 
+// Tries to merge loop induction variables of a given type.
+//
+// In this pass we're only concerned with elements of the loop's tuple that
+// are effective-scalars of type `elem_ty`.  Some terminology:
+//
+//  - The trip counter is the first element of the loop's tuple that starts at
+//    0 and does x++ on each iteration.
+//
+//  - An induction variable is an element of the loop's tuple that is not the
+//    trip counter and does `x += <constant>` on each iteration of the loop.
+//    Negative constants are OK.
+//
+// This pass adds a trip counter if one isn't already present, then replaces
+// each induction variable with
+//
+//   <initial_value> + <trip_count> * <constant>.
+//
+// This reduces the number of scalar operations in the loop, which is important
+// e.g. on GPUs, where each scalar operation is nontrivially expensive because
+// it's a separate kernel launch.
+//
+// Returns the new loop if a change was made, or null if no change was made.
+// Note that the new loop is not a valid replacement for the old loop; it may
+// need to be wrapped in a tuple that changes its shape.  We return the loop
+// itself so that you can call TryMergeInductionVariables in a loop, once for
+// each integral type elem_ty.
+static StatusOr<HloInstruction*> TryMergeInductionVariables(
+    HloInstruction* while_op, PrimitiveType elem_ty) {
+  CHECK(primitive_util::IsIntegralType(elem_ty)) << PrimitiveType_Name(elem_ty);
+  HloModule* module = while_op->GetModule();
+  HloComputation* computation = while_op->parent();
+  auto* while_init = while_op->mutable_operand(0);
+  auto* while_body = while_op->while_body();
+  auto* while_cond = while_op->while_condition();
+  auto* while_body_root = while_body->root_instruction();
+  if (while_init->opcode() != HloOpcode::kTuple ||
+      while_body_root->opcode() != HloOpcode::kTuple) {
+    return nullptr;
+  }
+
+  TF_RET_CHECK(while_cond->num_parameters() == 1);
+  TF_RET_CHECK(while_body->num_parameters() == 1);
+  TF_RET_CHECK(
+      ShapeUtil::Compatible(while_init->shape(), while_body_root->shape()));
+  Shape while_shape = while_init->shape();
+
+  // The tuple index of the trip counter, if one is present.
+  absl::optional<int64> trip_counter;
+  // Maps the tuple index of each induction variable to its constant increment.
+  absl::flat_hash_map<int64, const HloConstantInstruction*> induction_vars;
+  for (int64 i = 0; i < while_body_root->operand_count(); ++i) {
+    const auto& elem_shape = while_body_root->operand(i)->shape();
+    if (!ShapeUtil::IsEffectiveScalar(elem_shape) ||
+        elem_shape.element_type() != elem_ty) {
+      continue;
+    }
+
+    HloInstruction* constant;
+    if (!Match(while_body_root->mutable_operand(i),
+               m::AddAnyOrder(m::GetTupleElement(m::Parameter(), i),
+                              m::Constant(&constant)))) {
+      continue;
+    }
+    if (!trip_counter && constant->literal().IsAll(1) &&
+        while_init->operand(i)->IsConstant() &&
+        while_init->operand(i)->literal().IsAll(0)) {
+      VLOG(10) << "Found existing trip counter at index " << i;
+      trip_counter = i;
+    } else {
+      VLOG(10) << "Found induction variable at index " << i;
+      induction_vars.emplace(i, Cast<HloConstantInstruction>(constant));
+    }
+  }
+
+  // There's only something to simplify if we can either:
+  //
+  //  - combine one or more induction vars with an existing trip counter, or
+  //  - replace two or more induction variables with a new trip counter.
+  //
+  // Put another way, there's only something to simplify if the number of
+  // induction vars plus the number of existing trip counters (0 or 1) is >= 2.
+  if (induction_vars.size() + (trip_counter.has_value() ? 1 : 0) < 2) {
+    return nullptr;
+  }
+
+  // OK, we're going to do the transformation!  Set up some helpers.
+
+  // `new_instrs` holds instructions created outside of a computation for
+  // cloning.  Elements added here just need to live until the end of the
+  // relevant CloneWithReplacement call.
+  std::vector<std::unique_ptr<HloInstruction>> new_instrs;
+  auto add_new_instr = [&](std::unique_ptr<HloInstruction> instr) {
+    new_instrs.push_back(std::move(instr));
+    return new_instrs.back().get();
+  };
+
+  auto add_binary_op = [&](const Shape& shape, HloOpcode opcode,
+                           HloInstruction* lhs, HloInstruction* rhs) {
+    // Reshape lhs/rhs to the output shape if necessary.  This deals with the
+    // fact that induction variables need only be effective scalars, not true
+    // scalars.
+    if (!ShapeUtil::Compatible(shape, lhs->shape())) {
+      lhs = add_new_instr(HloInstruction::CreateReshape(shape, lhs));
+    }
+    if (!ShapeUtil::Compatible(shape, rhs->shape())) {
+      rhs = add_new_instr(HloInstruction::CreateReshape(shape, rhs));
+    }
+    return add_new_instr(HloInstruction::CreateBinary(shape, opcode, lhs, rhs));
+  };
+
+  auto add_gte = [&](HloInstruction* src, int64 idx) {
+    return add_new_instr(HloInstruction::CreateGetTupleElement(
+        src->shape().tuple_shapes(idx), src, idx));
+  };
+
+  // Our new while loop will have the same shape as the old while loop, except
+  // we'll add a trip counter to the end if it wasn't originally present.
+  Shape new_while_shape = while_shape;
+  bool added_trip_counter = false;
+  if (!trip_counter) {
+    VLOG(10) << "Adding new trip counter to end of loop's tuple.";
+    trip_counter = new_while_shape.tuple_shapes_size();
+    *new_while_shape.add_tuple_shapes() =
+        ShapeUtil::MakeShape(elem_ty, /*dimensions=*/{});
+    added_trip_counter = true;
+  }
+
+  // Converts `instr` into a tuple of the "old" form -- that is, to a tuple with
+  // shape `while_body->shape()` and where the induction variables are "reified"
+  // (i.e. they have value <init> + <counter> * <constant>).
+  auto convert_to_old_form = [&](HloInstruction* instr) {
+    CHECK(ShapeUtil::Compatible(instr->shape(), new_while_shape));
+    std::vector<HloInstruction*> tuple_elems;
+    for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+      const auto& elem_shape = while_shape.tuple_shapes(i);
+      if (!induction_vars.count(i)) {
+        tuple_elems.push_back(add_gte(instr, i));
+        continue;
+      }
+      tuple_elems.push_back(add_binary_op(
+          elem_shape, HloOpcode::kAdd, add_gte(instr, i),
+          add_binary_op(elem_shape, HloOpcode::kMultiply,
+                        add_gte(instr, *trip_counter),
+                        add_new_instr(induction_vars.at(i)->Clone()))));
+    }
+    return HloInstruction::CreateTuple(tuple_elems);
+  };
+
+  // Converts `root` into a tuple of the "new" form -- that is, to a tuple with
+  // shape `new_while_shape` and where the induction variables (but not trip
+  // counters) are replaced with their unchanging <loop_body_param> values.
+  auto convert_to_new_form = [&](HloInstruction* old_root,
+                                 HloParameterInstruction* loop_body_param) {
+    CHECK(ShapeUtil::Compatible(old_root->shape(), while_shape));
+    std::vector<HloInstruction*> tuple_elems;
+
+    // In the new form, induction variables come from `init`, everything else
+    // (including the trip counter if it's not one we created ourselves) comes
+    // from the `root` tuple unmodified.
+    for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+      tuple_elems.push_back(
+          add_gte((induction_vars.count(i) ? loop_body_param : old_root), i));
+    }
+    // If we created a trip counter ourselves, add 1 to it in the next
+    // iteration.
+    if (added_trip_counter) {
+      tuple_elems.push_back(add_binary_op(
+          new_while_shape.tuple_shapes(*trip_counter), HloOpcode::kAdd,
+          add_gte(loop_body_param, *trip_counter),
+          add_new_instr(
+              HloInstruction::CreateConstant(LiteralUtil::One(elem_ty)))));
+    }
+
+    return HloInstruction::CreateTuple(tuple_elems);
+  };
+
+  // Creates a new init tuple, which is the same as the old init tuple except if
+  // we added a trip counter, it's set to 0.
+  auto get_new_while_init = [&](HloInstruction* init) {
+    CHECK(ShapeUtil::Compatible(init->shape(), while_shape));
+    if (!added_trip_counter) {
+      return init;
+    }
+    std::vector<HloInstruction*> tuple_elems;
+    for (int64 i = 0; i < while_shape.tuple_shapes_size(); ++i) {
+      tuple_elems.push_back(add_gte(init, i));
+    }
+    tuple_elems.push_back(add_new_instr(
+        HloInstruction::CreateConstant(LiteralUtil::Zero(elem_ty))));
+    return add_new_instr(HloInstruction::CreateTuple(tuple_elems));
+  };
+
+  std::unique_ptr<HloComputation> new_while_cond =
+      while_cond->CloneWithReplacementPairs({
+          while_cond->parameter_instruction(0),
+          convert_to_old_form(add_new_instr(HloInstruction::CreateParameter(
+              0, new_while_shape,
+              while_cond->parameter_instruction(0)->name()))),
+      });
+
+  // Creating the new while body proceeds in two steps.  First we convert the
+  // users of the parameter to the old form.  Then as a second
+  // CloneWithReplacement operation we convert the root to the new form.  We
+  // have to do this in two steps because the new root needs to use the new
+  // param0, and during the first clone operation, only the *old-form* param0 is
+  // accessible.
+  //
+  // We have to add temp_new_while_body to the module because cloning a
+  // computation touches the module (to get its NameUniquer).
+  HloComputation* temp_new_while_body =
+      module->AddEmbeddedComputation(while_body->CloneWithReplacementPairs({
+          while_body->parameter_instruction(0),
+          convert_to_old_form(add_new_instr(HloInstruction::CreateParameter(
+              0, new_while_shape,
+              while_body->parameter_instruction(0)->name()))),
+      }));
+  std::unique_ptr<HloComputation> new_while_body =
+      temp_new_while_body->CloneWithReplacementPairs({
+          temp_new_while_body->root_instruction(),
+          convert_to_new_form(
+              add_new_instr(temp_new_while_body->root_instruction()->Clone()),
+              Cast<HloParameterInstruction>(
+                  temp_new_while_body->parameter_instruction(0))),
+      });
+  TF_RETURN_IF_ERROR(module->RemoveEmbeddedComputation(temp_new_while_body));
+
+  // Create the final while loop, and add any new instructions created to
+  // `computation`.
+  new_instrs.clear();
+  auto* new_while = computation->AddInstruction(HloInstruction::CreateWhile(
+      new_while_shape,
+      module->AddEmbeddedComputation(std::move(new_while_cond)),
+      module->AddEmbeddedComputation(std::move(new_while_body)),
+      get_new_while_init(while_init)));
+  TF_RETURN_IF_ERROR(computation->ReplaceWithNewInstruction(
+      while_op, convert_to_old_form(new_while)));
+  for (auto& instr : new_instrs) {
+    computation->AddInstruction(std::move(instr));
+  }
+  return new_while;
+}
+
 StatusOr<bool> WhileLoopSimplifier::Run(HloModule* module) {
   XLA_VLOG_LINES(3,
                  "WhileLoopSimplifier::Run(), before:\n" + module->ToString());
@@ -813,6 +1059,22 @@ StatusOr<bool> WhileLoopSimplifier::Run(HloModule* module) {
     if (result) {
       continue;
     }
+
+    bool merged_induction_vars = false;
+    // Notably missing from this list are S16 and U16.  These don't currently
+    // work because S/U16 literals are not implemented.
+    for (auto elem_ty : {S8, U8, S32, U32, S64, U64}) {
+      TF_ASSIGN_OR_RETURN(auto* new_while_op,
+                          TryMergeInductionVariables(while_op, elem_ty));
+      if (new_while_op) {
+        while_op = new_while_op;
+        changed = true;
+        merged_induction_vars = true;
+      }
+    }
+    if (merged_induction_vars) {
+      continue;
+    }
   }
 
   XLA_VLOG_LINES(3,
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
index e036c2aabc..4950e8269e 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
@@ -17,6 +17,8 @@ limitations under the License.
 
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_replace.h"
+#include "tensorflow/compiler/xla/service/algebraic_simplifier.h"
+#include "tensorflow/compiler/xla/service/hlo_cse.h"
 #include "tensorflow/compiler/xla/service/hlo_dce.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_matchers.h"
@@ -31,6 +33,14 @@ namespace {
 using ::testing::_;
 namespace op = xla::testing::opcode_matchers;
 
+// Returns the first kWhile instruction within m's entry computation.
+HloInstruction* FindFirstWhile(HloModule* m) {
+  const auto& instrs = m->entry_computation()->instructions();
+  return *absl::c_find_if(instrs, [](const HloInstruction* instr) {
+    return instr->opcode() == HloOpcode::kWhile;
+  });
+}
+
 class WhileLoopSimplifierTest : public HloTestBase {
  protected:
   // Makes an HloModule that contains a loop with `num_iters` iteration.
@@ -542,11 +552,7 @@ TEST_F(WhileLoopSimplifierTest, FlattenNestedTuple) {
   // it easy to find.
   EXPECT_TRUE(HloDCE().Run(m.get()).ok());
 
-  const auto& instrs = m->entry_computation()->instructions();
-  HloInstruction* new_while =
-      *absl::c_find_if(instrs, [](const HloInstruction* instr) {
-        return instr->opcode() == HloOpcode::kWhile;
-      });
+  HloInstruction* new_while = FindFirstWhile(m.get());
   Shape flat_tuple =
       ShapeUtil::ParseShapeString("(s32[1], s32[2], s32[3], s32[4])")
           .ValueOrDie();
@@ -634,15 +640,9 @@ TEST_F(WhileLoopSimplifierTest, RemoveConstantFromLoopCarry) {
   // Run the tuple simplifier to make the resulting HLO a bit easier to check.
   EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok());
 
-  const auto& instrs = m->entry_computation()->instructions();
-  HloInstruction* new_while =
-      *absl::c_find_if(instrs, [](const HloInstruction* instr) {
-        return instr->opcode() == HloOpcode::kWhile;
-      });
-
+  HloInstruction* new_while = FindFirstWhile(m.get());
   Shape new_while_shape =
       ShapeUtil::ParseShapeString("(s32[1], s32[3])").ValueOrDie();
-  SCOPED_TRACE(m->ToString());
   EXPECT_TRUE(ShapeUtil::Equal(new_while->shape(), new_while_shape));
   EXPECT_TRUE(ShapeUtil::Equal(
       new_while->while_body()->root_instruction()->shape(), new_while_shape));
@@ -659,5 +659,89 @@ TEST_F(WhileLoopSimplifierTest, RemoveConstantFromLoopCarry) {
               op::Tuple(_, op::Constant(), _));
 }
 
+const char* const kSimpleMergeInductionVariablesModule = R"(
+  HloModule Test
+  Body {
+    param = (TYPE[], TYPE[], TYPE[]) parameter(0)
+
+    a = TYPE[] get-tuple-element(param), index=0
+    one = TYPE[] constant(1)
+    a1 = TYPE[] add(a, one)
+
+    b = TYPE[] get-tuple-element(param), index=1
+    negone = TYPE[] constant(-1)
+    b1 = TYPE[] add(b, negone)
+
+    c = TYPE[] add(a, b)
+
+    ROOT tuple = (TYPE[], TYPE[], TYPE[]) tuple(a1,b1,c)
+  }
+  Cond {
+    param = (TYPE[], TYPE[], TYPE[]) parameter(0)
+    a = TYPE[] get-tuple-element(param), index=0
+    b = TYPE[] get-tuple-element(param), index=1
+    sum = TYPE[] power(a, b)
+    ten = TYPE[] constant(10)
+    ROOT cond = pred[] less-than(sum, ten)
+  }
+  ENTRY Loop {
+    a = TYPE[] constant(10)
+    b = TYPE[] constant(100)
+    c = TYPE[] constant(0)
+    init = (TYPE[], TYPE[], TYPE[]) tuple(a,b,c)
+    while = (TYPE[], TYPE[], TYPE[]) while(init), condition=Cond, body=Body
+
+    a1 = TYPE[] get-tuple-element(while), index=0
+    b1 = TYPE[] get-tuple-element(while), index=1
+    ROOT sum = TYPE[] add(a1, b1)
+  })";
+
+TEST_F(WhileLoopSimplifierTest, MergeInductionVariables_Simple) {
+  string hlo_string = absl::StrReplaceAll(kSimpleMergeInductionVariablesModule,
+                                          {{"TYPE", "s32"}});
+
+  auto m = ParseAndReturnVerifiedModule(hlo_string).ValueOrDie();
+  EXPECT_TRUE(WhileLoopSimplifier().Run(m.get()).ValueOrDie());
+  // DCE away the old loop so there's just one while loop in the module, making
+  // it easy to find, and run the tuple simplifier to make the resulting HLO
+  // easier to check.
+  EXPECT_TRUE(HloDCE().Run(m.get()).ok());
+  EXPECT_TRUE(TupleSimplifier().Run(m.get()).ok());
+
+  HloInstruction* new_while = FindFirstWhile(m.get());
+  // We should have added a new loop counter for s32[] to the end of the tuple.
+  SCOPED_TRACE(m->ToString());
+  Shape new_while_shape =
+      ShapeUtil::ParseShapeString("(s32[], s32[], s32[], s32[])").ValueOrDie();
+  EXPECT_TRUE(ShapeUtil::Equal(new_while->shape(), new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      new_while->while_body()->root_instruction()->shape(), new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      new_while->while_body()->parameter_instruction(0)->shape(),
+      new_while_shape));
+  EXPECT_TRUE(ShapeUtil::Equal(
+      new_while->while_condition()->parameter_instruction(0)->shape(),
+      new_while_shape));
+
+  EXPECT_THAT(new_while->while_body()->root_instruction(),
+              op::Tuple(op::GetTupleElement(op::Parameter(), 0),
+                        op::GetTupleElement(op::Parameter(), 1), op::Add(),
+                        op::Add(op::GetTupleElement(op::Parameter(), 3),
+                                op::Constant())));
+  EXPECT_THAT(new_while->while_condition()->root_instruction(),
+              op::Lt(op::Power(op::Add(), op::Add()), op::Constant()));
+}
+
+// We shouldn't merge S16 induction variables; we can't create constants of this
+// type because S16 literals are not implemented.
+TEST_F(WhileLoopSimplifierTest, MergeInductionVariables_SkipS16) {
+  string hlo_string = absl::StrReplaceAll(kSimpleMergeInductionVariablesModule,
+                                          {{"TYPE", "s16"}});
+  EXPECT_FALSE(
+      WhileLoopSimplifier()
+          .Run(ParseAndReturnVerifiedModule(hlo_string).ValueOrDie().get())
+          .ValueOrDie());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From e2fb964f15c53a305849799a81f14116d9d4ea9c Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Mon, 19 Nov 2018 11:35:59 -0800
Subject: [PATCH 0508/1554] Add SquaredDifference to the op schema

PiperOrigin-RevId: 222110246
---
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   2 +-
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   5 +
 tensorflow/lite/schema/schema_generated.h     | 124 +++++++++++++++++-
 5 files changed, 126 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index 89f5aac922..a3843deb00 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -124,6 +124,7 @@ typedef enum {
   kTfLiteBuiltinRange = 96,
   kTfLiteBuiltinResizeNearestNeighbor = 97,
   kTfLiteBuiltinLeakyRelu = 98,
+  kTfLiteBuiltinSquaredDifference = 99,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 3e67badfe4..3b592a6e0b 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -620,7 +620,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       *builtin_data = reinterpret_cast<void*>(params);
       break;
     }
-
     case BuiltinOperator_LEAKY_RELU: {
       TfLiteLeakyReluParams* params =
           allocator->AllocatePOD<TfLiteLeakyReluParams>();
@@ -681,6 +680,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_FILL:
     case BuiltinOperator_FLOOR_MOD:
     case BuiltinOperator_RANGE:
+    case BuiltinOperator_SQUARED_DIFFERENCE:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index a1be2a5abc..e78ce554ed 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -683,6 +683,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_FLOOR_MOD:
       case tflite::BuiltinOperator_RANGE:
       case tflite::BuiltinOperator_LEAKY_RELU:
+      case tflite::BuiltinOperator_SQUARED_DIFFERENCE:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 3eeae708fd..e40a0409f9 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -201,6 +201,7 @@ enum BuiltinOperator : byte {
   RANGE = 96,
   RESIZE_NEAREST_NEIGHBOR = 97,
   LEAKY_RELU = 98,
+  SQUARED_DIFFERENCE = 99,
 }
 
 // Options for the builtin operators.
@@ -280,6 +281,7 @@ union BuiltinOptions {
   RangeOptions,
   ResizeNearestNeighborOptions,
   LeakyReluOptions,
+  SquaredDifferenceOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -664,6 +666,9 @@ table LeakyReluOptions {
   alpha:float;
 }
 
+table SquaredDifferenceOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index b054f6bb0c..e93cb3d81d 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -256,6 +256,9 @@ struct RangeOptionsT;
 struct LeakyReluOptions;
 struct LeakyReluOptionsT;
 
+struct SquaredDifferenceOptions;
+struct SquaredDifferenceOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -504,11 +507,12 @@ enum BuiltinOperator {
   BuiltinOperator_RANGE = 96,
   BuiltinOperator_RESIZE_NEAREST_NEIGHBOR = 97,
   BuiltinOperator_LEAKY_RELU = 98,
+  BuiltinOperator_SQUARED_DIFFERENCE = 99,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_LEAKY_RELU
+  BuiltinOperator_MAX = BuiltinOperator_SQUARED_DIFFERENCE
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[98] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[99] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -607,7 +611,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[98] {
     BuiltinOperator_FLOOR_MOD,
     BuiltinOperator_RANGE,
     BuiltinOperator_RESIZE_NEAREST_NEIGHBOR,
-    BuiltinOperator_LEAKY_RELU
+    BuiltinOperator_LEAKY_RELU,
+    BuiltinOperator_SQUARED_DIFFERENCE
   };
   return values;
 }
@@ -713,6 +718,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "RANGE",
     "RESIZE_NEAREST_NEIGHBOR",
     "LEAKY_RELU",
+    "SQUARED_DIFFERENCE",
     nullptr
   };
   return names;
@@ -800,11 +806,12 @@ enum BuiltinOptions {
   BuiltinOptions_RangeOptions = 73,
   BuiltinOptions_ResizeNearestNeighborOptions = 74,
   BuiltinOptions_LeakyReluOptions = 75,
+  BuiltinOptions_SquaredDifferenceOptions = 76,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_LeakyReluOptions
+  BuiltinOptions_MAX = BuiltinOptions_SquaredDifferenceOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[76] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[77] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -881,7 +888,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[76] {
     BuiltinOptions_FloorModOptions,
     BuiltinOptions_RangeOptions,
     BuiltinOptions_ResizeNearestNeighborOptions,
-    BuiltinOptions_LeakyReluOptions
+    BuiltinOptions_LeakyReluOptions,
+    BuiltinOptions_SquaredDifferenceOptions
   };
   return values;
 }
@@ -964,6 +972,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "RangeOptions",
     "ResizeNearestNeighborOptions",
     "LeakyReluOptions",
+    "SquaredDifferenceOptions",
     nullptr
   };
   return names;
@@ -1278,6 +1287,10 @@ template<> struct BuiltinOptionsTraits<LeakyReluOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_LeakyReluOptions;
 };
 
+template<> struct BuiltinOptionsTraits<SquaredDifferenceOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_SquaredDifferenceOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1909,6 +1922,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_LeakyReluOptions ?
       reinterpret_cast<const LeakyReluOptionsT *>(value) : nullptr;
   }
+  SquaredDifferenceOptionsT *AsSquaredDifferenceOptions() {
+    return type == BuiltinOptions_SquaredDifferenceOptions ?
+      reinterpret_cast<SquaredDifferenceOptionsT *>(value) : nullptr;
+  }
+  const SquaredDifferenceOptionsT *AsSquaredDifferenceOptions() const {
+    return type == BuiltinOptions_SquaredDifferenceOptions ?
+      reinterpret_cast<const SquaredDifferenceOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -6708,6 +6729,46 @@ inline flatbuffers::Offset<LeakyReluOptions> CreateLeakyReluOptions(
 
 flatbuffers::Offset<LeakyReluOptions> CreateLeakyReluOptions(flatbuffers::FlatBufferBuilder &_fbb, const LeakyReluOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct SquaredDifferenceOptionsT : public flatbuffers::NativeTable {
+  typedef SquaredDifferenceOptions TableType;
+  SquaredDifferenceOptionsT() {
+  }
+};
+
+struct SquaredDifferenceOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef SquaredDifferenceOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  SquaredDifferenceOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(SquaredDifferenceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<SquaredDifferenceOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct SquaredDifferenceOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit SquaredDifferenceOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  SquaredDifferenceOptionsBuilder &operator=(const SquaredDifferenceOptionsBuilder &);
+  flatbuffers::Offset<SquaredDifferenceOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<SquaredDifferenceOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<SquaredDifferenceOptions> CreateSquaredDifferenceOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  SquaredDifferenceOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<SquaredDifferenceOptions> CreateSquaredDifferenceOptions(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -7066,6 +7127,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const LeakyReluOptions *builtin_options_as_LeakyReluOptions() const {
     return builtin_options_type() == BuiltinOptions_LeakyReluOptions ? static_cast<const LeakyReluOptions *>(builtin_options()) : nullptr;
   }
+  const SquaredDifferenceOptions *builtin_options_as_SquaredDifferenceOptions() const {
+    return builtin_options_type() == BuiltinOptions_SquaredDifferenceOptions ? static_cast<const SquaredDifferenceOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7397,6 +7461,10 @@ template<> inline const LeakyReluOptions *Operator::builtin_options_as<LeakyRelu
   return builtin_options_as_LeakyReluOptions();
 }
 
+template<> inline const SquaredDifferenceOptions *Operator::builtin_options_as<SquaredDifferenceOptions>() const {
+  return builtin_options_as_SquaredDifferenceOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -9914,6 +9982,29 @@ inline flatbuffers::Offset<LeakyReluOptions> CreateLeakyReluOptions(flatbuffers:
       _alpha);
 }
 
+inline SquaredDifferenceOptionsT *SquaredDifferenceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new SquaredDifferenceOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void SquaredDifferenceOptions::UnPackTo(SquaredDifferenceOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<SquaredDifferenceOptions> SquaredDifferenceOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateSquaredDifferenceOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<SquaredDifferenceOptions> CreateSquaredDifferenceOptions(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const SquaredDifferenceOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateSquaredDifferenceOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -10472,6 +10563,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const LeakyReluOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_SquaredDifferenceOptions: {
+      auto ptr = reinterpret_cast<const SquaredDifferenceOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -10790,6 +10885,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const LeakyReluOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_SquaredDifferenceOptions: {
+      auto ptr = reinterpret_cast<const SquaredDifferenceOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11096,6 +11195,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const LeakyReluOptionsT *>(value);
       return CreateLeakyReluOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_SquaredDifferenceOptions: {
+      auto ptr = reinterpret_cast<const SquaredDifferenceOptionsT *>(value);
+      return CreateSquaredDifferenceOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -11402,6 +11505,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new LeakyReluOptionsT(*reinterpret_cast<LeakyReluOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_SquaredDifferenceOptions: {
+      value = new SquaredDifferenceOptionsT(*reinterpret_cast<SquaredDifferenceOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -11784,6 +11891,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_SquaredDifferenceOptions: {
+      auto ptr = reinterpret_cast<SquaredDifferenceOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
GitLab


From 932ac256f5761d148e9317508cfa796ae368488a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 11:37:21 -0800
Subject: [PATCH 0509/1554] Fix `tpu.rewrite` documentation.

PiperOrigin-RevId: 222110519
---
 tensorflow/contrib/tpu/python/tpu/tpu.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index e3e791faac..a02361241c 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -1001,8 +1001,8 @@ def rewrite(computation,
       `rewrite` is a list of tensors corresponding to the tensors from the
       output of `computation`.
 
-      All `Operation`s returned from `computation` will be executed when
-      evaluating any of the returned output tensors.
+      All `Operation`s constructed during `computation` will be executed when
+      evaluating any of the returned output tensors, not just the ones returned.
     inputs: A list of input tensors or `None` (equivalent to an empty list).
     infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple
       of arguments as inputs to `computation`.
-- 
GitLab


From 06eb734f21e510ac938cae6148baf5a9d765b669 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 11:39:59 -0800
Subject: [PATCH 0510/1554] TF 2.0 Symbol Renames

Update nn.weighted_moments(x, axes, frequency_weights, name, keep_dims) to
change: nn.weighted_moments(x, axes, frequency_weights, keepdims, name) in v2.

Update sparse.add(a, b, thresh) to change: sparse.add(a, b, threshold) in v2.

Rename sparse.matmul to sparse.sparse_dense_matmul in v2.

PiperOrigin-RevId: 222110970
---
 tensorflow/python/ops/nn_impl.py              | 26 +++++++-
 tensorflow/python/ops/sparse_ops.py           | 65 ++++++++++++++++++-
 .../api/golden/v1/tensorflow.sparse.pbtxt     |  4 ++
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../api/golden/v2/tensorflow.sparse.pbtxt     | 10 +--
 tensorflow/tools/compatibility/renames_v2.py  |  5 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 15 +++++
 7 files changed, 116 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index d697834206..30c8058a35 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -895,7 +895,7 @@ def moments(
       return (mean, variance)
 
 
-@tf_export("nn.weighted_moments")
+@tf_export(v1=["nn.weighted_moments"])
 def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False):
   """Returns the frequency-weighted mean and variance of `x`.
 
@@ -967,6 +967,30 @@ def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False):
     return weighted_mean, weighted_variance
 
 
+@tf_export("nn.weighted_moments", v1=[])
+def weighted_moments_v2(x, axes, frequency_weights, keepdims=False, name=None):
+  """Returns the frequency-weighted mean and variance of `x`.
+
+  Args:
+    x: A tensor.
+    axes: 1-d tensor of int32 values; these are the axes along which
+      to compute mean and variance.
+    frequency_weights: A tensor of positive weights which can be
+      broadcast with x.
+    keepdims: Produce moments with the same dimensionality as the input.
+    name: Name used to scope the operation.
+
+  Returns:
+    Two tensors: `weighted_mean` and `weighted_variance`.
+  """
+  return weighted_moments(
+      x=x,
+      axes=axes,
+      frequency_weights=frequency_weights,
+      name=name,
+      keep_dims=keepdims)
+
+
 @tf_export("nn.batch_normalization")
 def batch_normalization(x,
                         mean,
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index dc091336b2..9981f389c8 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -325,7 +325,7 @@ def sparse_concat(axis,
   return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
 
 
-@tf_export("sparse.add", v1=["sparse.add", "sparse_add"])
+@tf_export(v1=["sparse.add", "sparse_add"])
 @deprecation.deprecated_endpoints("sparse_add")
 def sparse_add(a, b, thresh=0):
   """Adds two tensors, at least one of each is a `SparseTensor`.
@@ -408,6 +408,65 @@ def sparse_add(a, b, thresh=0):
                                                   a.dense_shape, b)
 
 
+@tf_export("sparse.add", v1=[])
+def sparse_add_v2(a, b, threshold=0):
+  """Adds two tensors, at least one of each is a `SparseTensor`.
+
+  If one `SparseTensor` and one `Tensor` are passed in, returns a `Tensor`.  If
+  both arguments are `SparseTensor`s, this returns a `SparseTensor`.  The order
+  of arguments does not matter.  Use vanilla `tf.add()` for adding two dense
+  `Tensor`s.
+
+  The shapes of the two operands must match: broadcasting is not supported.
+
+  The indices of any input `SparseTensor` are assumed ordered in standard
+  lexicographic order.  If this is not the case, before this step run
+  `SparseReorder` to restore index ordering.
+
+  If both arguments are sparse, we perform "clipping" as follows.  By default,
+  if two values sum to zero at some index, the output `SparseTensor` would still
+  include that particular location in its index, storing a zero in the
+  corresponding value slot.  To override this, callers can specify `threshold`,
+  indicating that if the sum has a magnitude strictly smaller than `threshold`,
+  its corresponding value and index would then not be included.  In particular,
+  `threshold == 0.0` (default) means everything is kept and actual thresholding
+  happens only for a positive value.
+
+  For example, suppose the logical sum of two sparse operands is (densified):
+
+      [       2]
+      [.1     0]
+      [ 6   -.2]
+
+  Then,
+
+      * `threshold == 0` (the default): all 5 index/value pairs will be
+          returned.
+      * `threshold == 0.11`: only .1 and 0 will vanish, and the remaining three
+          index/value pairs will be returned.
+      * `threshold == 0.21`: .1, 0, and -.2 will vanish.
+
+  Args:
+    a: The first operand; `SparseTensor` or `Tensor`.
+    b: The second operand; `SparseTensor` or `Tensor`.  At least one operand
+      must be sparse.
+    threshold: A 0-D `Tensor`.  The magnitude threshold that determines if an
+    output value/index pair takes space.  Its dtype should match that of the
+    values if they are real; if the latter are complex64/complex128, then the
+    dtype should be float32/float64, correspondingly.
+
+  Returns:
+    A `SparseTensor` or a `Tensor`, representing the sum.
+
+  Raises:
+    TypeError: If both `a` and `b` are `Tensor`s.  Use `tf.add()` instead.
+  """
+  return sparse_add(
+      a=a,
+      b=b,
+      thresh=threshold)
+
+
 @tf_export("sparse.cross")
 def sparse_cross(inputs, name=None):
   """Generates sparse cross from a list of sparse and dense tensors.
@@ -1995,7 +2054,9 @@ def deserialize_many_sparse(serialized_sparse, dtype, rank=None, name=None):
   return sparse_tensor.SparseTensor(output_indices, output_values, output_shape)
 
 
-@tf_export("sparse.matmul", v1=["sparse.matmul", "sparse_tensor_dense_matmul"])
+@tf_export("sparse.sparse_dense_matmul",
+           v1=["sparse.sparse_dense_matmul", "sparse.matmul",
+               "sparse_tensor_dense_matmul"])
 @deprecation.deprecated_endpoints("sparse_tensor_dense_matmul")
 def sparse_tensor_dense_matmul(sp_a,
                                b,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
index 32bd8d5f8e..ee4f31774e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
@@ -112,6 +112,10 @@ tf_module {
     name: "softmax"
     argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "sparse_dense_matmul"
+    argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], "
+  }
   member_method {
     name: "split"
     argspec: "args=[\'keyword_required\', \'sp_input\', \'num_split\', \'axis\', \'name\', \'split_dim\'], varargs=None, keywords=None, defaults=[\'KeywordRequired()\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 0d2d6d682e..ecbda448f1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -310,7 +310,7 @@ tf_module {
   }
   member_method {
     name: "weighted_moments"
-    argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], "
+    argspec: "args=[\'x\', \'axes\', \'frequency_weights\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
   member_method {
     name: "with_space_to_batch"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index 960d1ba425..bd7592e019 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -10,7 +10,7 @@ tf_module {
   }
   member_method {
     name: "add"
-    argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], "
+    argspec: "args=[\'a\', \'b\', \'threshold\'], varargs=None, keywords=None, defaults=[\'0\'], "
   }
   member_method {
     name: "concat"
@@ -40,10 +40,6 @@ tf_module {
     name: "mask"
     argspec: "args=[\'a\', \'mask_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "matmul"
-    argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], "
-  }
   member_method {
     name: "maximum"
     argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -100,6 +96,10 @@ tf_module {
     name: "softmax"
     argspec: "args=[\'sp_input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "sparse_dense_matmul"
+    argspec: "args=[\'sp_a\', \'b\', \'adjoint_a\', \'adjoint_b\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'None\'], "
+  }
   member_method {
     name: "split"
     argspec: "args=[\'sp_input\', \'num_split\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 294cd66fdc..f96bbe342c 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -438,9 +438,10 @@ renames = {
     'tf.sets.set_union': 'tf.sets.union',
     'tf.space_to_batch': 'tf.nn.space_to_batch',
     'tf.space_to_depth': 'tf.nn.space_to_depth',
+    'tf.sparse.matmul': 'tf.sparse.sparse_dense_matmul',
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
     'tf.sparse.reduce_max_sparse': 'tf.compat.v1.sparse.reduce_max_sparse',
-    'tf.sparse_add': 'tf.sparse.add',
+    'tf.sparse_add': 'tf.compat.v1.sparse_add',
     'tf.sparse_fill_empty_rows': 'tf.sparse.fill_empty_rows',
     'tf.sparse_mask': 'tf.sparse.mask',
     'tf.sparse_matmul': 'tf.compat.v1.sparse_matmul',
@@ -462,7 +463,7 @@ renames = {
     'tf.sparse_slice': 'tf.sparse.slice',
     'tf.sparse_softmax': 'tf.sparse.softmax',
     'tf.sparse_split': 'tf.compat.v1.sparse_split',
-    'tf.sparse_tensor_dense_matmul': 'tf.sparse.matmul',
+    'tf.sparse_tensor_dense_matmul': 'tf.sparse.sparse_dense_matmul',
     'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense',
     'tf.sparse_to_indicator': 'tf.sparse.to_indicator',
     'tf.sparse_transpose': 'tf.sparse.transpose',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index bd556c575c..e7e6fc9d77 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -55,6 +55,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "t": "x",
             "msg": "message",
         },
+        "tf.sparse.add": [
+            "a", "b", "thresh"
+        ],
         "tf.sparse.split": {
             "split_dim": "axis",
         },
@@ -81,6 +84,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
+        "tf.sparse.concat": [
+            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
+        ]
     }
 
     # Mapping from function to the new name of the function
@@ -164,6 +170,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "filter", "padding", "strides", "dilation_rate", "name",
             "data_format"],
         "tf.nn.crelu": ["features", "name", "axis"],
+        "tf.nn.weighted_moments": [
+            "x", "axes", "frequency_weights", "name", "keep_dims"
+        ],
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
@@ -189,6 +198,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
         "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
+        "tf.sparse.add": [
+            "a", "b", "thresh"
+        ],
+        "tf.sparse.concat": [
+            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
+        ],
         "tf.sparse.segment_mean": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
-- 
GitLab


From 9c3751f171b53c3c19a90fbbd777bc5eb21ea240 Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Mon, 19 Nov 2018 12:12:11 -0800
Subject: [PATCH 0511/1554] Remove tf.image.resize_area,
 tf.image.resize_bicubic, tf.image.resize_bilinear,
 tf.image.resize_nearest_neighbor in TF 2.0 API.

PiperOrigin-RevId: 222116892
---
 .../python_api/api_def_ResizeArea.pbtxt       |  4 +--
 .../python_api/api_def_ResizeBicubic.pbtxt    |  4 +--
 .../python_api/api_def_ResizeBilinear.pbtxt   |  4 +--
 .../api_def_ResizeNearestNeighbor.pbtxt       |  4 +--
 tensorflow/python/ops/image_ops_impl.py       | 31 +++++++++++++++++++
 .../api/golden/v2/tensorflow.image.pbtxt      | 16 ----------
 tensorflow/tools/compatibility/renames_v2.py  |  4 +++
 7 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_ResizeArea.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResizeArea.pbtxt
index 2f1b4aee00..e1a1f883d8 100644
--- a/tensorflow/core/api_def/python_api/api_def_ResizeArea.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ResizeArea.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "ResizeArea"
-  endpoint {
-    name: "image.resize_area"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_ResizeBicubic.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResizeBicubic.pbtxt
index 3ec8e0ad63..e0bec8c116 100644
--- a/tensorflow/core/api_def/python_api/api_def_ResizeBicubic.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ResizeBicubic.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "ResizeBicubic"
-  endpoint {
-    name: "image.resize_bicubic"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_ResizeBilinear.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResizeBilinear.pbtxt
index eb3b8d6f45..6121c1128c 100644
--- a/tensorflow/core/api_def/python_api/api_def_ResizeBilinear.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ResizeBilinear.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "ResizeBilinear"
-  endpoint {
-    name: "image.resize_bilinear"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_ResizeNearestNeighbor.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResizeNearestNeighbor.pbtxt
index 25c5d5701f..0e86e4ce3e 100644
--- a/tensorflow/core/api_def/python_api/api_def_ResizeNearestNeighbor.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ResizeNearestNeighbor.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "ResizeNearestNeighbor"
-  endpoint {
-    name: "image.resize_nearest_neighbor"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 3ab3695a03..c3dea18807 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -37,6 +37,7 @@ from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import variables
+from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 ops.NotDifferentiable('RandomCrop')
@@ -2808,3 +2809,33 @@ def sobel_edges(image):
   output = array_ops.reshape(output, shape=shape)
   output.set_shape(static_image_shape.concatenate([num_kernels]))
   return output
+
+
+resize_area_deprecation = deprecation.deprecated(
+    date=None,
+    instructions=(
+        'Use `tf.image.resize(...method=ResizeMethod.AREA...)` instead.'))
+tf_export(v1=['image.resize_area'])(
+    resize_area_deprecation(gen_image_ops.resize_area))
+
+resize_bicubic_deprecation = deprecation.deprecated(
+    date=None,
+    instructions=(
+        'Use `tf.image.resize(...method=ResizeMethod.BICUBIC...)` instead.'))
+tf_export(v1=['image.resize_bicubic'])(
+    resize_bicubic_deprecation(gen_image_ops.resize_bicubic))
+
+resize_bilinear_deprecation = deprecation.deprecated(
+    date=None,
+    instructions=(
+        'Use `tf.image.resize(...method=ResizeMethod.BILINEAR...)` instead.'))
+tf_export(v1=['image.resize_bilinear'])(
+    resize_bilinear_deprecation(gen_image_ops.resize_bilinear))
+
+resize_nearest_neighbor_deprecation = deprecation.deprecated(
+    date=None,
+    instructions=(
+        'Use `tf.image.resize(...method=ResizeMethod.NEAREST_NEIGHBOR...)` '
+        'instead.'))
+tf_export(v1=['image.resize_nearest_neighbor'])(
+    resize_nearest_neighbor_deprecation(gen_image_ops.resize_nearest_neighbor))
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index 0a231f1b65..c4cd166ad5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -172,18 +172,6 @@ tf_module {
     name: "random_saturation"
     argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "resize_area"
-    argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
-  member_method {
-    name: "resize_bicubic"
-    argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
-  member_method {
-    name: "resize_bilinear"
-    argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
   member_method {
     name: "resize_image_with_crop_or_pad"
     argspec: "args=[\'image\', \'target_height\', \'target_width\'], varargs=None, keywords=None, defaults=None"
@@ -196,10 +184,6 @@ tf_module {
     name: "resize_images"
     argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\'], "
   }
-  member_method {
-    name: "resize_nearest_neighbor"
-    argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-  }
   member_method {
     name: "rgb_to_grayscale"
     argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index f96bbe342c..41ae1a2a68 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -193,6 +193,10 @@ renames = {
     'tf.igamma': 'tf.math.igamma',
     'tf.igammac': 'tf.math.igammac',
     'tf.imag': 'tf.math.imag',
+    'tf.image.resize_area': 'tf.compat.v1.image.resize_area',
+    'tf.image.resize_bicubic': 'tf.compat.v1.image.resize_bicubic',
+    'tf.image.resize_bilinear': 'tf.compat.v1.image.resize_bilinear',
+    'tf.image.resize_nearest_neighbor': 'tf.compat.v1.image.resize_nearest_neighbor',
     'tf.initialize_all_tables': 'tf.compat.v1.initialize_all_tables',
     'tf.initialize_all_variables': 'tf.compat.v1.initialize_all_variables',
     'tf.initialize_local_variables': 'tf.compat.v1.initialize_local_variables',
-- 
GitLab


From ed6172f6c72e667887e7b4631bee4044cc718020 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 12:22:41 -0800
Subject: [PATCH 0512/1554] Speed up and shard further estimator_training_test.

PiperOrigin-RevId: 222118412
---
 tensorflow/contrib/distribute/python/BUILD           |  2 +-
 .../distribute/python/estimator_training_test.py     | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index b5318dcfe6..e99b0b4039 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -475,7 +475,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:summary",
     ],
-    shard_count = 5,
+    shard_count = 48,
     tags = [
         "multi_and_single_gpu",
         "no_pip",
diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 9d8bc89b33..3e7d5df4c4 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -50,6 +50,8 @@ from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary import summary_iterator
 from tensorflow.python.summary.writer import writer_cache
+from tensorflow.python.training import session_manager
+
 
 BATCH_SIZE = 10
 LABEL_DIMENSION = 2
@@ -622,5 +624,15 @@ class RunConfigTest(test.TestCase):
 
 
 if __name__ == "__main__":
+  # Reduce `recovery_wait_secs` from 30 seconds so the test completes quickly.
+  orig_init = session_manager.SessionManager.__init__
+
+  def new_init(*args, **kwargs):
+    kwargs.pop("recovery_wait_secs", None)
+    kwargs["recovery_wait_secs"] = 0.5
+    orig_init(*args, **kwargs)
+
+  session_manager.SessionManager.__init__ = new_init
+
   with test.mock.patch.object(sys, "exit", os._exit):
     test.main()
-- 
GitLab


From 0dff427f4ebee8486db686cef28a8d17c4c663e5 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Mon, 19 Nov 2018 12:28:22 -0800
Subject: [PATCH 0513/1554] Removed unnecessary iterator class from
 PartitionedVariable

Note that PartitionedVariable is still iterable.

PiperOrigin-RevId: 222119232
---
 tensorflow/python/ops/variables.py | 30 +-----------------------------
 1 file changed, 1 insertion(+), 29 deletions(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 8da740d265..5bee522481 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -2457,34 +2457,6 @@ class PartitionedVariable(object):
   @end_compatibility
   """
 
-  class PartitionedVariableIterator(object):
-    """An iterator that allows accessing the underlying `Variable` objects.
-
-    This iterator is necessary to control order of access when Variables
-    are not partitioned in a standard way along a single axis.
-
-    Allows e.g. `list(partitioned_variable)` to return a proper list.
-    """
-
-    def __init__(self, partitioned_variable):
-      self._ix = 0
-      self._partitioned_variable = partitioned_variable
-
-    def __iter__(self):
-      return self
-
-    def __next__(self):  # For python3 compatibility.
-      return self.next()
-
-    def next(self):
-      # pylint: disable=protected-access
-      if self._ix >= len(self._partitioned_variable._variable_list):
-        raise StopIteration()
-      variable = self._partitioned_variable._variable_list[self._ix]
-      # pylint: enable=protected-access
-      self._ix += 1
-      return variable
-
   def __init__(self, name, shape, dtype, variable_list, partitions):
     """Creates a new partitioned variable wrapper.
 
@@ -2541,7 +2513,7 @@ class PartitionedVariable(object):
 
   def __iter__(self):
     """Return an iterable for accessing the underlying partition Variables."""
-    return self.PartitionedVariableIterator(self)
+    return iter(self._variable_list)
 
   def __len__(self):
     num_partition_axes = len(self._partition_axes())
-- 
GitLab


From d5a620d8b95dfb0ffc001dac34ecc1349d48bab8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 12:38:49 -0800
Subject: [PATCH 0514/1554] Passing feature engineering fn to custom export

PiperOrigin-RevId: 222120842
---
 .../estimator_batch/custom_export_strategy.py            | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
index a3df272e69..b314b4d74d 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
@@ -41,7 +41,8 @@ def make_custom_export_strategy(name,
                                 convert_fn,
                                 feature_columns,
                                 export_input_fn,
-                                use_core_columns=False):
+                                use_core_columns=False,
+                                feature_engineering_fn=None):
   """Makes custom exporter of GTFlow tree format.
 
   Args:
@@ -52,6 +53,7 @@ def make_custom_export_strategy(name,
     export_input_fn: A function that takes no arguments and returns an
       `InputFnOps`.
     use_core_columns: A boolean, whether core feature columns were used.
+    feature_engineering_fn: Feature eng function to be called on the input.
 
   Returns:
     An `ExportStrategy`.
@@ -59,9 +61,12 @@ def make_custom_export_strategy(name,
   base_strategy = saved_model_export_utils.make_export_strategy(
       serving_input_fn=export_input_fn, strip_default_attrs=True)
   input_fn = export_input_fn()
+  features = input_fn.features
+  if feature_engineering_fn is not None:
+    features, _ = feature_engineering_fn(features, labels=None)
   (sorted_feature_names, dense_floats, sparse_float_indices, _, _,
    sparse_int_indices, _, _) = gbdt_batch.extract_features(
-       input_fn.features, feature_columns, use_core_columns)
+       features, feature_columns, use_core_columns)
 
   def export_fn(estimator, export_dir, checkpoint_path=None, eval_result=None):
     """A wrapper to export to SavedModel, and convert it to other formats."""
-- 
GitLab


From 03bbe80587382daf94c75892c7c99773a7201ee3 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 19 Nov 2018 12:46:36 -0800
Subject: [PATCH 0515/1554] Add support for LeakyRelu

PiperOrigin-RevId: 222121991
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 14 +++++
 tensorflow/lite/kernels/activations.cc        | 62 ++++++++++++++-----
 tensorflow/lite/kernels/activations_test.cc   | 33 ++++++++++
 .../internal/optimized/optimized_ops.h        |  1 +
 .../internal/reference/reference_ops.h        | 13 ++++
 tensorflow/lite/kernels/internal/types.h      |  4 ++
 tensorflow/lite/kernels/register.cc           |  2 +
 tensorflow/lite/testing/generate_examples.py  | 28 +++++++++
 .../propagate_fixed_sizes.cc                  |  1 +
 tensorflow/lite/toco/import_tensorflow.cc     | 16 +++++
 tensorflow/lite/toco/model.h                  | 16 ++++-
 tensorflow/lite/toco/tflite/operator.cc       | 20 ++++++
 tensorflow/lite/toco/tflite/operator_test.cc  |  8 +++
 tensorflow/lite/toco/tooling_util.cc          |  1 +
 15 files changed, 202 insertions(+), 18 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index bc98dc57bc..8666855fff 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -248,6 +248,7 @@ def generated_test_models():
         "sum",
         "l2norm",
         "l2_pool",
+        "leaky_relu",
         "less",
         "less_equal",
         "local_response_norm",
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index b0dfb0fed1..2a7f8f1f23 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -378,6 +378,20 @@ Options {
 }
 ```
 
+**LEAKY_RELU**
+
+```
+Inputs {
+  0: a tensor
+}
+Outputs {
+  0: a tensor equivalent to max(input, input * alpha)
+}
+Options {
+  alpha: slope of the activation at x < 0 (provided alpha <= 1)
+}
+```
+
 **LESS**
 
 ```
diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index 9c525d9640..bcd2b160f7 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -288,8 +288,8 @@ TfLiteStatus ReluEval(TfLiteContext* context, TfLiteNode* node) {
       return kTfLiteOk;
     } break;
     default:
-      context->ReportError(context, "Only float32 supported currently, got %d.",
-                           input->type);
+      context->ReportError(context, "Only float32 supported currently, got %s.",
+                           TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
@@ -309,8 +309,8 @@ TfLiteStatus Relu1Eval(TfLiteContext* context, TfLiteNode* node) {
       return kTfLiteOk;
     } break;
     default:
-      context->ReportError(context, "Only float32 supported currently, got %d.",
-                           input->type);
+      context->ReportError(context, "Only float32 supported currently, got %s.",
+                           TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
@@ -328,8 +328,8 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) {
       return kTfLiteOk;
     } break;
     default:
-      context->ReportError(context, "Only float32 supported currently, got %d.",
-                           input->type);
+      context->ReportError(context, "Only float32 supported currently, got %s.",
+                           TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
@@ -367,8 +367,8 @@ TfLiteStatus TanhEval(TfLiteContext* context, TfLiteNode* node) {
       return kTfLiteOk;
     } break;
     default:
-      context->ReportError(context, "Only float32 supported currently, got %d.",
-                           input->type);
+      context->ReportError(context, "Only float32 supported currently, got %s.",
+                           TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
@@ -407,9 +407,8 @@ TfLiteStatus SigmoidEval(TfLiteContext* context, TfLiteNode* node) {
       break;
     }
     default:
-      context->ReportError(context, "Only float32 supported currently, got %d.",
-                           input->type);
-      return kTfLiteError;
+      context->ReportError(context, "Only float32 supported currently, got %s.",
+                           TfLiteTypeGetName(input->type));
   }
   return kTfLiteOk;
 }
@@ -604,8 +603,8 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) {
     }
     default:
       context->ReportError(
-          context, "Only float32 and uint8_t supported currently, got %d.",
-          input->type);
+          context, "Only float32 and uint8_t supported currently, got %s.",
+          TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
@@ -636,8 +635,8 @@ TfLiteStatus LogSoftmaxEval(TfLiteContext* context, TfLiteNode* node) {
       return kTfLiteOk;
     }
     default:
-      context->ReportError(context, "Only float32 supported currently., got %d",
-                           input->type);
+      context->ReportError(context, "Only float32 supported currently., got %s",
+                           TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
@@ -652,8 +651,8 @@ TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* alpha = GetInput(context, node, 1);
   TfLiteTensor* output = GetOutput(context, node, 0);
   if (input->type != kTfLiteFloat32) {
-    context->ReportError(context, "Only float32 supported currently, got %d.",
-                         input->type);
+    context->ReportError(context, "Only float32 supported currently, got %s.",
+                         TfLiteTypeGetName(input->type));
     return kTfLiteError;
   }
   reference_ops::BroadcastBinaryFunction4DSlow<float, float, float>(
@@ -663,6 +662,28 @@ TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) {
   return kTfLiteOk;
 }
 
+TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* output = GetOutput(context, node, 0);
+  const auto* params =
+      reinterpret_cast<TfLiteLeakyReluParams*>(node->builtin_data);
+
+  LeakyReluParams op_params;
+  op_params.alpha = params->alpha;
+  switch (input->type) {
+    case kTfLiteFloat32: {
+      optimized_ops::LeakyRelu(
+          op_params, GetTensorShape(input), GetTensorData<float>(input),
+          GetTensorShape(output), GetTensorData<float>(output));
+      return kTfLiteOk;
+    } break;
+    default:
+      context->ReportError(context, "Only float32 supported currently, got %s.",
+                           TfLiteTypeGetName(input->type));
+      return kTfLiteError;
+  }
+}
+
 }  // namespace activations
 
 TfLiteRegistration* Register_RELU() {
@@ -721,6 +742,13 @@ TfLiteRegistration* Register_PRELU() {
   return &r;
 }
 
+TfLiteRegistration* Register_LEAKY_RELU() {
+  static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr,
+                                 activations::GenericPrepare,
+                                 activations::LeakyReluEval};
+  return &r;
+}
+
 }  // namespace builtin
 }  // namespace ops
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/activations_test.cc b/tensorflow/lite/kernels/activations_test.cc
index fff4121dc0..8d705b8f1d 100644
--- a/tensorflow/lite/kernels/activations_test.cc
+++ b/tensorflow/lite/kernels/activations_test.cc
@@ -606,6 +606,39 @@ TEST(FloatActivationsOpTest, PRelu) {
                              }));
 }
 
+class LeakyReluOpModel : public SingleOpModel {
+ public:
+  LeakyReluOpModel(const TensorData& input, float alpha) {
+    input_ = AddInput(input);
+    output_ = AddOutput(input);
+    SetBuiltinOp(BuiltinOperator_LEAKY_RELU, BuiltinOptions_LeakyReluOptions,
+                 CreateLeakyReluOptions(builder_, alpha).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+  void SetInput(std::initializer_list<float> data) {
+    PopulateTensor(input_, data);
+  }
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+
+ protected:
+  int input_;
+  int output_;
+};
+
+TEST(FloatActivationsOpTest, LeakyRelu) {
+  LeakyReluOpModel m({TensorType_FLOAT32, {2, 3}}, 0.5f);
+
+  m.SetInput({
+      0.0f, 1.0f, 3.0f,    // Row 1
+      1.0f, -1.0f, -2.0f,  // Row 2
+  });
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({
+                                 0.0f, 1.0f, 3.0f,    // Row 1
+                                 1.0f, -0.5f, -1.0f,  // Row 2
+                             }));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 6f2cd4faab..4ff875091e 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -65,6 +65,7 @@ using reference_ops::Greater;
 using reference_ops::GreaterEqual;
 using reference_ops::GreaterEqualWithScaling;
 using reference_ops::GreaterWithScaling;
+using reference_ops::LeakyRelu;
 using reference_ops::Less;
 using reference_ops::LessEqual;
 using reference_ops::LessEqualWithScaling;
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 1bd9129488..7c37dab7e4 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -558,6 +558,19 @@ inline void ReluX(const tflite::ActivationParams& params,
   }
 }
 
+inline void LeakyRelu(const tflite::LeakyReluParams& params,
+                      const RuntimeShape& input_shape, const float* input_data,
+                      const RuntimeShape& output_shape, float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("LeakyRelu (not fused)");
+  const int flat_size = MatchingFlatSize(input_shape, output_shape);
+  for (int i = 0; i < flat_size; ++i) {
+    const float val = input_data[i];
+    // Note that this implementation matches that of TensorFlow, and corresponds
+    // to the traditional LeakyRelu equation only for alpha <= 1.
+    output_data[i] = std::max(val, val * params.alpha);
+  }
+}
+
 inline void L2Normalization(const tflite::L2NormalizationParams& op_params,
                             const RuntimeShape& input_shape,
                             const float* input_data,
diff --git a/tensorflow/lite/kernels/internal/types.h b/tensorflow/lite/kernels/internal/types.h
index a05bd5e003..dea571039e 100644
--- a/tensorflow/lite/kernels/internal/types.h
+++ b/tensorflow/lite/kernels/internal/types.h
@@ -1006,6 +1006,10 @@ struct UnpackParams {
   int16 axis;
 };
 
+struct LeakyReluParams {
+  float alpha;
+};
+
 template <typename P>
 inline void SetActivationParams(float min, float max, P* params) {
   params->float_activation_min = min;
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index c683453767..3645507cc7 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -123,6 +123,7 @@ TfLiteRegistration* Register_SQUARE();
 TfLiteRegistration* Register_ZEROS_LIKE();
 TfLiteRegistration* Register_FLOOR_MOD();
 TfLiteRegistration* Register_RANGE();
+TfLiteRegistration* Register_LEAKY_RELU();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -256,6 +257,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_ZEROS_LIKE, Register_ZEROS_LIKE());
   AddBuiltin(BuiltinOperator_FLOOR_MOD, Register_FLOOR_MOD());
   AddBuiltin(BuiltinOperator_RANGE, Register_RANGE());
+  AddBuiltin(BuiltinOperator_LEAKY_RELU, Register_LEAKY_RELU());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 71db1bcdb4..f54ab9dfbf 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -755,6 +755,34 @@ def make_prelu_tests(zip_path):
       use_frozen_graph=True)
 
 
+def make_leaky_relu_tests(zip_path):
+  """Make a set of tests to do LeakyRelu."""
+
+  test_parameters = [
+      {
+          "input_shape": [[], [1], [5], [1, 10, 10, 3], [3, 3, 3, 3]],
+          "alpha": [0.1, 1.0, 2.0, -0.1, -1.0, -2.0],
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.nn.leaky_relu(input_tensor, alpha=parameters["alpha"])
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-3, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 # This function tests various TensorFLow functions that generates Const op,
 # including `tf.ones`, `tf.zeros` and random functions.
 def make_constant_tests(zip_path):
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 78ea54e452..54f4b5601b 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1714,6 +1714,7 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) {
     case OperatorType::kRelu1:
     case OperatorType::kRelu6:
     case OperatorType::kPRelu:
+    case OperatorType::kLeakyRelu:
     case OperatorType::kSoftmax:
     case OperatorType::kLogSoftmax:
     case OperatorType::kLog:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index c13517566f..d4ca594ba6 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2217,6 +2217,21 @@ tensorflow::Status ConvertUnidirectionalSequenceLstm(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertLeakyReluOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "LeakyRelu");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
+  CHECK_EQ(GetDataTypeAttr(node, "T"), DT_FLOAT);
+  const auto& input_name = node.input(0);
+  auto* op = new LeakyReluOperator;
+  op->inputs.push_back(input_name);
+  op->outputs.push_back(node.name());
+  op->alpha = GetFloatAttr(node, "alpha");
+  model->operators.emplace_back(op);
+  return tensorflow::Status::OK();
+}
+
 }  // namespace
 
 namespace internal {
@@ -2280,6 +2295,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
        ConvertSimpleOperator<TensorFlowGreaterEqualOperator, 2>},
       {"Identity", ConvertIdentityOperator},
       {"LRN", ConvertLRNOperator},
+      {"LeakyRelu", ConvertLeakyReluOperator},
       {"LegacyFedInput", ConvertPlaceholderOperator},
       {"Less", ConvertSimpleOperator<TensorFlowLessOperator, 2>},
       {"LessEqual", ConvertSimpleOperator<TensorFlowLessEqualOperator, 2>},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index f85e1c2878..7b9c78eaad 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -152,7 +152,8 @@ enum class OperatorType : uint8 {
   kCTCBeamSearchDecoder,
   kUnpack,
   kZerosLike,
-  kResizeNearestNeighbor
+  kResizeNearestNeighbor,
+  kLeakyRelu
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -699,6 +700,19 @@ struct PReluOperator : Operator {
   PReluOperator() : Operator(OperatorType::kPRelu) {}
 };
 
+// LeakyRelu
+//   x -> max(x, alpha * x)
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: LeakyRelu
+struct LeakyReluOperator : Operator {
+  LeakyReluOperator() : Operator(OperatorType::kLeakyRelu) {}
+
+  float alpha = 0.2f;  // 0.2 matches the default value for the TF op attribute.
+};
+
 // Element-wise Logistic operator:
 //   x -> Logistic(x) = 1 / (1 + exp(-x))
 //
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 84ae448246..44da6a97d0 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1218,6 +1218,24 @@ class Unpack : public BuiltinOperator<UnpackOperator, ::tflite::UnpackOptions,
   int GetVersion(const Operator& op) const override { return 1; }
 };
 
+class LeakyRelu
+    : public BuiltinOperator<LeakyReluOperator, ::tflite::LeakyReluOptions,
+                             ::tflite::BuiltinOptions_LeakyReluOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateLeakyReluOptions(*builder, op.alpha);
+  }
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->alpha = options.alpha();
+  }
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
 std::unique_ptr<flexbuffers::Builder> WriteFlexOpOptions(
     const string& tensorflow_node_def) {
   auto fbb = absl::make_unique<flexbuffers::Builder>();
@@ -1516,6 +1534,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
                                    OperatorType::kOneHot));
   ops.push_back(MakeUnique<Unpack>(::tflite::BuiltinOperator_UNPACK,
                                    OperatorType::kUnpack));
+  ops.push_back(MakeUnique<LeakyRelu>(::tflite::BuiltinOperator_LEAKY_RELU,
+                                      OperatorType::kLeakyRelu));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 8a776cbf0b..96aac5f9d5 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -517,6 +517,14 @@ TEST_F(OperatorTest, BuiltinUnpack) {
   EXPECT_EQ(op.axis, output_toco_op->axis);
 }
 
+TEST_F(OperatorTest, BuiltinLeakyRelu) {
+  LeakyReluOperator op;
+  op.alpha = 3;
+  auto output_toco_op = SerializeAndDeserialize(
+      GetOperator("LEAKY_RELU", OperatorType::kLeakyRelu), op);
+  EXPECT_EQ(op.alpha, output_toco_op->alpha);
+}
+
 TEST_F(OperatorTest, CustomCTCBeamSearchDecoder) {
   CTCBeamSearchDecoderOperator op;
   op.beam_width = 3;
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 4a96450e07..2afd4685d7 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -411,6 +411,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(ZerosLike)
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceLstm)
     HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor)
+    HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From 9a83d2111f6d25ee7e2311b2e232be746acd0b31 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 19 Nov 2018 12:48:46 -0800
Subject: [PATCH 0516/1554] Export initial batch of contrib summary symbols
 under tf.compat.v2

PiperOrigin-RevId: 222122302
---
 tensorflow/contrib/summary/summary.py         |  1 +
 tensorflow/python/ops/summary_ops_v2.py       | 90 +++++++++++--------
 tensorflow/python/summary/summary.py          | 18 ++--
 .../tensorflow.summary.-summary-writer.pbtxt  | 29 ++++++
 .../api/golden/v2/tensorflow.summary.pbtxt    | 40 +++------
 5 files changed, 104 insertions(+), 74 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt

diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py
index 42898e797c..605625c305 100644
--- a/tensorflow/contrib/summary/summary.py
+++ b/tensorflow/contrib/summary/summary.py
@@ -79,6 +79,7 @@ from tensorflow.python.ops.summary_ops_v2 import image
 from tensorflow.python.ops.summary_ops_v2 import import_event
 from tensorflow.python.ops.summary_ops_v2 import initialize
 from tensorflow.python.ops.summary_ops_v2 import never_record_summaries
+from tensorflow.python.ops.summary_ops_v2 import record_summaries
 from tensorflow.python.ops.summary_ops_v2 import record_summaries_every_n_global_steps
 from tensorflow.python.ops.summary_ops_v2 import scalar
 from tensorflow.python.ops.summary_ops_v2 import should_record_summaries
diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py
index fac9055ef1..a0ad43b444 100644
--- a/tensorflow/python/ops/summary_ops_v2.py
+++ b/tensorflow/python/ops/summary_ops_v2.py
@@ -40,7 +40,9 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import training_util
+from tensorflow.python.util import deprecation
 from tensorflow.python.util import tf_contextlib
+from tensorflow.python.util.tf_export import tf_export
 
 
 # Dictionary mapping graph keys to a boolean Tensor (or callable returning
@@ -56,6 +58,7 @@ _RUN_NAME_PATTERNS = re.compile(r"^[^\x00-\x1F<>]{0,512}$")
 _USER_NAME_PATTERNS = re.compile(r"^[a-z]([-a-z0-9]{0,29}[a-z0-9])?$", re.I)
 
 
+@tf_export("summary.should_record_summaries", v1=[])
 def should_record_summaries():
   """Returns boolean Tensor which is true if summaries should be recorded."""
   global _SHOULD_RECORD_SUMMARIES
@@ -64,58 +67,64 @@ def should_record_summaries():
   return should() if callable(should) else should
 
 
-# TODO(apassos) consider how to handle local step here.
+@tf_export("summary.record_summaries", v1=[])
 @tf_contextlib.contextmanager
-def record_summaries_every_n_global_steps(n, global_step=None):
-  """Sets the should_record_summaries Tensor to true if global_step % n == 0."""
-  if global_step is None:
-    global_step = training_util.get_or_create_global_step()
+def record_summaries(boolean=True):
+  """Sets summary recording on or off per the provided boolean value.
+
+  The provided value can be a python boolean, a scalar boolean Tensor, or
+  or a callable providing such a value; if a callable is passed it will be
+  invoked each time should_record_summaries() is called to determine whether
+  summary writing should be enabled.
+
+  Args:
+    boolean: can be True, False, a bool Tensor, or a callable providing such.
+      Defaults to True.
+
+  Yields:
+    Returns a context manager that sets this value on enter and restores the
+    previous value on exit.
+  """
+  # TODO(nickfelt): make this threadlocal
   global _SHOULD_RECORD_SUMMARIES
   key = ops.get_default_graph()._graph_key  # pylint: disable=protected-access
   old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False)
   try:
-    with ops.device("cpu:0"):
-      should = lambda: math_ops.equal(global_step % n, 0)
-      if not context.executing_eagerly():
-        should = should()
-      _SHOULD_RECORD_SUMMARIES[key] = should
+    _SHOULD_RECORD_SUMMARIES[key] = boolean
     yield
   finally:
     _SHOULD_RECORD_SUMMARIES[key] = old
 
 
-@tf_contextlib.contextmanager
+# TODO(apassos) consider how to handle local step here.
+def record_summaries_every_n_global_steps(n, global_step=None):
+  """Sets the should_record_summaries Tensor to true if global_step % n == 0."""
+  if global_step is None:
+    global_step = training_util.get_or_create_global_step()
+  with ops.device("cpu:0"):
+    should = lambda: math_ops.equal(global_step % n, 0)
+    if not context.executing_eagerly():
+      should = should()
+  return record_summaries(should)
+
+
 def always_record_summaries():
   """Sets the should_record_summaries Tensor to always true."""
-  global _SHOULD_RECORD_SUMMARIES
-  key = ops.get_default_graph()._graph_key  # pylint: disable=protected-access
-  old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False)
-  try:
-    _SHOULD_RECORD_SUMMARIES[key] = True
-    yield
-  finally:
-    _SHOULD_RECORD_SUMMARIES[key] = old
+  return record_summaries(True)
 
 
-@tf_contextlib.contextmanager
 def never_record_summaries():
   """Sets the should_record_summaries Tensor to always false."""
-  global _SHOULD_RECORD_SUMMARIES
-  key = ops.get_default_graph()._graph_key  # pylint: disable=protected-access
-  old = _SHOULD_RECORD_SUMMARIES.setdefault(key, False)
-  try:
-    _SHOULD_RECORD_SUMMARIES[key] = False
-    yield
-  finally:
-    _SHOULD_RECORD_SUMMARIES[key] = old
+  return record_summaries(False)
 
 
+@tf_export("summary.SummaryWriter", v1=[])
 class SummaryWriter(object):
   """Encapsulates a stateful summary writer resource.
 
   See also:
-  - `tf.contrib.summary.create_file_writer`
-  - `tf.contrib.summary.create_db_writer`
+  - `tf.summary.create_file_writer`
+  - `tf.summary.create_db_writer`
   """
 
   def  __init__(self, resource, init_op_fn):
@@ -210,6 +219,7 @@ def initialize(
     session.run(_graph(x, 0), feed_dict={x: data})
 
 
+@tf_export("summary.create_file_writer", v1=[])
 def create_file_writer(logdir,
                        max_queue=None,
                        flush_millis=None,
@@ -285,7 +295,7 @@ def create_db_writer(db_uri,
       `tf.Graph`.
 
   Returns:
-    A `tf.contrib.summary.SummaryWriter` instance.
+    A `tf.summary.SummaryWriter` instance.
   """
   with ops.device("cpu:0"):
     if experiment_name is None:
@@ -334,7 +344,7 @@ def _nothing():
 def all_summary_ops():
   """Graph-mode only. Returns all summary ops.
 
-  Please note this excludes `tf.contrib.summary.graph` ops.
+  Please note this excludes `tf.summary.graph` ops.
 
   Returns:
     The summary ops.
@@ -502,7 +512,7 @@ def graph(param, step=None, name=None):
   """Writes a TensorFlow graph to the summary interface.
 
   The graph summary is, strictly speaking, not a summary. Conditions
-  like `tf.contrib.summary.never_record_summaries` do not apply. Only
+  like `tf.summary.should_record_summaries` do not apply. Only
   a single graph can be associated with a particular run. If multiple
   graphs are written, then only the last one will be considered by
   TensorBoard.
@@ -546,14 +556,13 @@ def graph(param, step=None, name=None):
 _graph = graph  # for functions with a graph parameter
 
 
+@tf_export("summary.import_event", v1=[])
 def import_event(tensor, name=None):
   """Writes a `tf.Event` binary proto.
 
-  When using create_db_writer(), this can be used alongside
-  `tf.TFRecordReader` to load event logs into the database. Please
-  note that this is lower level than the other summary functions and
-  will ignore any conditions set by methods like
-  `tf.contrib.summary.should_record_summaries`.
+  This can be used to import existing event logs into a new summary writer sink.
+  Please note that this is lower level than the other summary functions and
+  will ignore the `tf.summary.should_record_summaries` setting.
 
   Args:
     tensor: A `tf.Tensor` of type `string` containing a serialized
@@ -567,13 +576,14 @@ def import_event(tensor, name=None):
       context.context().summary_writer_resource, tensor, name=name)
 
 
+@tf_export("summary.flush", v1=[])
 def flush(writer=None, name=None):
   """Forces summary writer to send any buffered data to storage.
 
   This operation blocks until that finishes.
 
   Args:
-    writer: The `tf.contrib.summary.SummaryWriter` resource to flush.
+    writer: The `tf.summary.SummaryWriter` resource to flush.
       The thread default will be used if this parameter is None.
       Otherwise a `tf.no_op` is returned.
     name: A name for the operation (optional).
@@ -600,6 +610,8 @@ def eval_dir(model_dir, name=None):
   return os.path.join(model_dir, "eval" if not name else "eval_" + name)
 
 
+@deprecation.deprecated(date=None,
+                        instructions="Renamed to create_file_writer().")
 def create_summary_file_writer(*args, **kwargs):
   """Please use `tf.contrib.summary.create_file_writer`."""
   logging.warning("Deprecation Warning: create_summary_file_writer was renamed "
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index 9e9e6ed903..0c13016712 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -52,7 +52,7 @@ from tensorflow.python.util import compat as _compat
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('summary.scalar')
+@tf_export(v1=['summary.scalar'])
 def scalar(name, tensor, collections=None, family=None):
   """Outputs a `Summary` protocol buffer containing a single scalar value.
 
@@ -82,7 +82,7 @@ def scalar(name, tensor, collections=None, family=None):
   return val
 
 
-@tf_export('summary.image')
+@tf_export(v1=['summary.image'])
 def image(name, tensor, max_outputs=3, collections=None, family=None):
   """Outputs a `Summary` protocol buffer with images.
 
@@ -138,7 +138,7 @@ def image(name, tensor, max_outputs=3, collections=None, family=None):
   return val
 
 
-@tf_export('summary.histogram')
+@tf_export(v1=['summary.histogram'])
 def histogram(name, values, collections=None, family=None):
   # pylint: disable=line-too-long
   """Outputs a `Summary` protocol buffer with a histogram.
@@ -179,7 +179,7 @@ def histogram(name, values, collections=None, family=None):
   return val
 
 
-@tf_export('summary.audio')
+@tf_export(v1=['summary.audio'])
 def audio(name, tensor, sample_rate, max_outputs=3, collections=None,
           family=None):
   # pylint: disable=line-too-long
@@ -228,7 +228,7 @@ def audio(name, tensor, sample_rate, max_outputs=3, collections=None,
   return val
 
 
-@tf_export('summary.text')
+@tf_export(v1=['summary.text'])
 def text(name, tensor, collections=None):
   """Summarizes textual data.
 
@@ -269,7 +269,7 @@ def text(name, tensor, collections=None):
   return t_summary
 
 
-@tf_export('summary.tensor_summary')
+@tf_export(v1=['summary.tensor_summary'])
 def tensor_summary(name,
                    tensor,
                    summary_description=None,
@@ -325,7 +325,7 @@ def tensor_summary(name,
   return val
 
 
-@tf_export('summary.merge')
+@tf_export(v1=['summary.merge'])
 def merge(inputs, collections=None, name=None):
   # pylint: disable=line-too-long
   """Merges summaries.
@@ -371,7 +371,7 @@ def merge(inputs, collections=None, name=None):
   return val
 
 
-@tf_export('summary.merge_all')
+@tf_export(v1=['summary.merge_all'])
 def merge_all(key=_ops.GraphKeys.SUMMARIES, scope=None, name=None):
   """Merges all summaries collected in the default graph.
 
@@ -404,7 +404,7 @@ def merge_all(key=_ops.GraphKeys.SUMMARIES, scope=None, name=None):
     return merge(summary_ops, name=name)
 
 
-@tf_export('summary.get_summary_description')
+@tf_export(v1=['summary.get_summary_description'])
 def get_summary_description(node_def):
   """Given a TensorSummary node_def, retrieve its SummaryDescription.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt
new file mode 100644
index 0000000000..6715c14e16
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-writer.pbtxt
@@ -0,0 +1,29 @@
+path: "tensorflow.summary.SummaryWriter"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.summary_ops_v2.SummaryWriter\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'resource\', \'init_op_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "as_default"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "close"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "flush"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "init"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_as_default"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
index 7ed9cd77a0..26c979c0c6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
@@ -24,44 +24,32 @@ tf_module {
     name: "SummaryDescription"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
+  member {
+    name: "SummaryWriter"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TaggedRunMetadata"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
   member_method {
-    name: "audio"
-    argspec: "args=[\'name\', \'tensor\', \'sample_rate\', \'max_outputs\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'3\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "get_summary_description"
-    argspec: "args=[\'node_def\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "histogram"
-    argspec: "args=[\'name\', \'values\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "image"
-    argspec: "args=[\'name\', \'tensor\', \'max_outputs\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'3\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "merge"
-    argspec: "args=[\'inputs\', \'collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    name: "create_file_writer"
+    argspec: "args=[\'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
-    name: "merge_all"
-    argspec: "args=[\'key\', \'scope\', \'name\'], varargs=None, keywords=None, defaults=[\'summaries\', \'None\', \'None\'], "
+    name: "flush"
+    argspec: "args=[\'writer\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
-    name: "scalar"
-    argspec: "args=[\'name\', \'tensor\', \'collections\', \'family\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    name: "import_event"
+    argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
-    name: "tensor_summary"
-    argspec: "args=[\'name\', \'tensor\', \'summary_description\', \'collections\', \'summary_metadata\', \'family\', \'display_name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    name: "record_summaries"
+    argspec: "args=[\'boolean\'], varargs=None, keywords=None, defaults=[\'True\'], "
   }
   member_method {
-    name: "text"
-    argspec: "args=[\'name\', \'tensor\', \'collections\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    name: "should_record_summaries"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
 }
-- 
GitLab


From c5695df38cbdae987a8f3f1c45b80f953368de39 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 19 Nov 2018 13:01:19 -0800
Subject: [PATCH 0517/1554] Cleaning up safe_div implementations

PiperOrigin-RevId: 222124095
---
 .../contrib/losses/python/losses/loss_ops.py  | 43 +++-----------
 .../contrib/metrics/python/ops/metric_ops.py  | 52 ++++-------------
 .../python/keras/engine/training_utils.py     |  2 +-
 tensorflow/python/keras/metrics.py            | 29 +---------
 tensorflow/python/ops/losses/losses_impl.py   | 42 +++-----------
 tensorflow/python/ops/metrics_impl.py         | 56 ++++++-------------
 6 files changed, 42 insertions(+), 182 deletions(-)

diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py
index 619294b518..d8ac4163b2 100644
--- a/tensorflow/contrib/losses/python/losses/loss_ops.py
+++ b/tensorflow/contrib/losses/python/losses/loss_ops.py
@@ -22,7 +22,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.framework.python.ops import add_arg_scope
-from tensorflow.python.compat import compat
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -67,34 +66,6 @@ def _scale_losses(losses, weights):
   return math_ops.reduce_sum(reduced_losses)
 
 
-def _safe_div(numerator, denominator, name="value"):
-  """Computes a safe divide which returns 0 if the denominator is zero.
-
-  Note that the function contains an additional conditional check that is
-  necessary for avoiding situations where the loss is zero causing NaNs to
-  creep into the gradient computation.
-
-  Args:
-    numerator: An arbitrary `Tensor`.
-    denominator: A `Tensor` whose shape matches `numerator` and whose values are
-      assumed to be non-negative.
-    name: An optional name for the returned op.
-
-  Returns:
-    The element-wise value of the numerator divided by the denominator.
-  """
-  if compat.forward_compatible(2018, 11, 1):
-    return math_ops.div_no_nan(numerator, denominator, name=name)
-  return array_ops.where(
-      math_ops.greater(denominator, 0),
-      math_ops.div(numerator,
-                   array_ops.where(
-                       math_ops.equal(denominator, 0),
-                       array_ops.ones_like(denominator), denominator)),
-      array_ops.zeros_like(numerator),
-      name=name)
-
-
 def _safe_mean(losses, num_present):
   """Computes a safe mean of the losses.
 
@@ -107,7 +78,7 @@ def _safe_mean(losses, num_present):
       then zero is returned.
   """
   total_loss = math_ops.reduce_sum(losses)
-  return _safe_div(total_loss, num_present, name="value")
+  return math_ops.div_no_nan(total_loss, num_present, name="value")
 
 
 @deprecated("2016-12-30", "Use tf.losses.compute_weighted_loss instead.")
@@ -612,14 +583,14 @@ def mean_pairwise_squared_error(predictions,
         math_ops.square(diffs), reduction_indices=reduction_indices)
     num_present_per_batch = _num_present(diffs, weights, per_batch=True)
 
-    term1 = 2.0 * _safe_div(sum_squares_diff_per_batch,
-                            num_present_per_batch,
-                            name="value")
+    term1 = 2.0 * math_ops.div_no_nan(
+        sum_squares_diff_per_batch, num_present_per_batch, name="value")
 
     sum_diff = math_ops.reduce_sum(diffs, reduction_indices=reduction_indices)
-    term2 = 2.0 * _safe_div(math_ops.square(sum_diff),
-                            math_ops.square(num_present_per_batch),
-                            name="value")
+    term2 = 2.0 * math_ops.div_no_nan(
+        math_ops.square(sum_diff),
+        math_ops.square(num_present_per_batch),
+        name="value")
 
     loss = _scale_losses(term1 - term2, weights)
 
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index d6932f6e4b..09fe65b73f 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -24,7 +24,6 @@ from __future__ import print_function
 
 import collections as collections_lib
 
-from tensorflow.python.compat import compat
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -46,32 +45,6 @@ from tensorflow.python.util.deprecation import deprecated
 _EPSILON = 1e-7
 
 
-def _safe_div(numerator, denominator):
-  """Computes a safe divide which returns 0 if the denominator is zero.
-
-  Note that the function contains an additional conditional check that is
-  necessary for avoiding situations where the loss is zero causing NaNs to
-  creep into the gradient computation.
-
-  Args:
-    numerator: An arbitrary `Tensor`.
-    denominator: A `Tensor` whose shape matches `numerator` and whose values are
-      assumed to be non-negative.
-
-  Returns:
-    The element-wise value of the numerator divided by the denominator.
-  """
-  if compat.forward_compatible(2018, 11, 1):
-    return math_ops.div_no_nan(numerator, denominator)
-  return array_ops.where(
-      math_ops.greater(denominator, 0),
-      math_ops.div(numerator,
-                   array_ops.where(
-                       math_ops.equal(denominator, 0),
-                       array_ops.ones_like(denominator), denominator)),
-      array_ops.zeros_like(numerator))
-
-
 @deprecated(None, 'Please switch to tf.metrics.true_positives. Note that the '
             'order of the labels and predictions arguments has been switched.')
 def streaming_true_positives(predictions,
@@ -3247,24 +3220,20 @@ def streaming_covariance(predictions,
 
     # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount)
     # batch_mean_prediction is E[x_B] in the update equation
-    batch_mean_prediction = _safe_div(
-        math_ops.reduce_sum(weighted_predictions),
-        batch_count)
-    delta_mean_prediction = _safe_div(
-        (batch_mean_prediction - mean_prediction) * batch_count,
-        update_count)
+    batch_mean_prediction = math_ops.div_no_nan(
+        math_ops.reduce_sum(weighted_predictions), batch_count)
+    delta_mean_prediction = math_ops.div_no_nan(
+        (batch_mean_prediction - mean_prediction) * batch_count, update_count)
     update_mean_prediction = state_ops.assign_add(mean_prediction,
                                                   delta_mean_prediction)
     # prev_mean_prediction is E[x_A] in the update equation
     prev_mean_prediction = update_mean_prediction - delta_mean_prediction
 
     # batch_mean_label is E[y_B] in the update equation
-    batch_mean_label = _safe_div(
-        math_ops.reduce_sum(weighted_labels),
-        batch_count)
-    delta_mean_label = _safe_div(
-        (batch_mean_label - mean_label) * batch_count,
-        update_count)
+    batch_mean_label = math_ops.div_no_nan(
+        math_ops.reduce_sum(weighted_labels), batch_count)
+    delta_mean_label = math_ops.div_no_nan(
+        (batch_mean_label - mean_label) * batch_count, update_count)
     update_mean_label = state_ops.assign_add(mean_label, delta_mean_label)
     # prev_mean_label is E[y_A] in the update equation
     prev_mean_label = update_mean_label - delta_mean_label
@@ -3926,9 +3895,8 @@ def cohen_kappa(labels,
       po_sum = math_ops.reduce_sum(po)
       total = math_ops.reduce_sum(pe_row)
       pe_sum = math_ops.reduce_sum(
-          _safe_div(
-              math_ops.to_double(pe_row * pe_col),
-              math_ops.to_double(total)))
+          math_ops.div_no_nan(
+              math_ops.to_double(pe_row * pe_col), math_ops.to_double(total)))
       po_sum, pe_sum, total = (math_ops.to_double(po_sum),
                                math_ops.to_double(pe_sum),
                                math_ops.to_double(total))
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 61d40d1caa..a0d1e5488d 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -654,7 +654,7 @@ def weighted_masked_objective(fn):
       score_array = math_ops.multiply(score_array, weights)
       score_array = math_ops.reduce_sum(score_array)
       weights = math_ops.reduce_sum(weights)
-      score_array = metrics_module.safe_div(score_array, weights)
+      score_array = math_ops.div_no_nan(score_array, weights)
     return K.mean(score_array)
 
   return weighted
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 2c3822e38d..7848be33f1 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -27,7 +27,6 @@ import weakref
 from enum import Enum
 import six
 
-from tensorflow.python.compat import compat
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
@@ -173,32 +172,6 @@ def weakmethod(method):
   return inner
 
 
-def safe_div(numerator, denominator):
-  """Computes a safe divide which returns 0 if the denominator is zero.
-
-  Note that the function contains an additional conditional check that is
-  necessary for avoiding situations where the loss is zero causing NaNs to
-  creep into the gradient computation.
-
-  Args:
-    numerator: An arbitrary `Tensor`.
-    denominator: A `Tensor` whose shape matches `numerator` and whose values are
-      assumed to be non-negative.
-
-  Returns:
-    The element-wise value of the numerator divided by the denominator.
-  """
-  if compat.forward_compatible(2018, 11, 1):
-    return math_ops.div_no_nan(numerator, denominator)
-  return array_ops.where(
-      math_ops.greater(denominator, 0),
-      math_ops.div(numerator,
-                   array_ops.where(
-                       math_ops.equal(denominator, 0),
-                       array_ops.ones_like(denominator), denominator)),
-      array_ops.zeros_like(numerator))
-
-
 def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
   """Squeeze or expand last dimension if needed.
 
@@ -697,7 +670,7 @@ class Mean(Metric):
       return ops.convert_to_tensor(update_count_op)
 
   def result(self):
-    return safe_div(self.total, self.count)
+    return math_ops.div_no_nan(self.total, self.count)
 
 
 class MeanMetricWrapper(Mean):
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index ffcd12b38e..e8cadf931b 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.compat import compat
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -92,34 +91,6 @@ class Reduction(ReductionV2):
       raise ValueError("Invalid Reduction Key %s." % key)
 
 
-def _safe_div(numerator, denominator, name="value"):
-  """Computes a safe divide which returns 0 if the denominator is zero.
-
-  Note that the function contains an additional conditional check that is
-  necessary for avoiding situations where the loss is zero causing NaNs to
-  creep into the gradient computation.
-
-  Args:
-    numerator: An arbitrary `Tensor`.
-    denominator: A `Tensor` whose shape matches `numerator` and whose values are
-      assumed to be non-negative.
-    name: An optional name for the returned op.
-
-  Returns:
-    The element-wise value of the numerator divided by the denominator.
-  """
-  if compat.forward_compatible(2018, 11, 1):
-    return math_ops.div_no_nan(numerator, denominator, name=name)
-  return array_ops.where(
-      math_ops.greater(denominator, 0),
-      math_ops.div(numerator,
-                   array_ops.where(
-                       math_ops.equal(denominator, 0),
-                       array_ops.ones_like(denominator), denominator)),
-      array_ops.zeros_like(numerator),
-      name=name)
-
-
 def _safe_mean(losses, num_present):
   """Computes a safe mean of the losses.
 
@@ -132,7 +103,7 @@ def _safe_mean(losses, num_present):
       then zero is returned.
   """
   total_loss = math_ops.reduce_sum(losses)
-  return _safe_div(total_loss, num_present)
+  return math_ops.div_no_nan(total_loss, num_present, name="value")
 
 
 def _num_present(losses, weights, per_batch=False):
@@ -620,18 +591,19 @@ def mean_pairwise_squared_error(
           keepdims=True)
       num_present_per_batch = _num_present(diffs, weights, per_batch=True)
 
-      term1 = 2.0 * _safe_div(
+      term1 = 2.0 * math_ops.div_no_nan(
           sum_squares_diff_per_batch,
-          math_ops.maximum(num_present_per_batch - 1, 0))
+          math_ops.maximum(num_present_per_batch - 1, 0),
+          name="value")
 
       sum_diff = math_ops.reduce_sum(
           diffs, reduction_indices=reduction_indices, keepdims=True)
-      term2 = 2.0 * _safe_div(
+      term2 = 2.0 * math_ops.div_no_nan(
           math_ops.square(sum_diff),
           math_ops.maximum(
               math_ops.multiply(num_present_per_batch,
-                                num_present_per_batch - 1),
-              0))
+                                num_present_per_batch - 1), 0),
+          name="value")
 
       weighted_losses = math_ops.multiply(term1 - term2, weights)
       loss = math_ops.reduce_sum(weighted_losses)
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 601b75fe06..03de8d552a 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.compat import compat
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -213,26 +212,6 @@ def _maybe_expand_labels(labels, predictions):
         lambda: array_ops.expand_dims(labels, -1, name=scope), lambda: labels)
 
 
-def _safe_div(numerator, denominator, name):
-  """Divides two tensors element-wise, returning 0 if the denominator is <= 0.
-
-  Args:
-    numerator: A real `Tensor`.
-    denominator: A real `Tensor`, with dtype matching `numerator`.
-    name: Name for the returned op.
-
-  Returns:
-    0 if `denominator` <= 0, else `numerator` / `denominator`
-  """
-  if compat.forward_compatible(2018, 11, 1):
-    return math_ops.div_no_nan(numerator, denominator, name=name)
-  t = math_ops.truediv(numerator, denominator)
-  zero = array_ops.zeros_like(t, dtype=denominator.dtype)
-  condition = math_ops.greater(denominator, zero)
-  zero = math_ops.cast(zero, t.dtype)
-  return array_ops.where(condition, t, zero, name=name)
-
-
 def _safe_scalar_div(numerator, denominator, name):
   """Divides two values, returning 0 if the denominator is 0.
 
@@ -246,7 +225,7 @@ def _safe_scalar_div(numerator, denominator, name):
   """
   numerator.get_shape().with_rank_at_most(1)
   denominator.get_shape().with_rank_at_most(1)
-  return _safe_div(numerator, denominator, name=name)
+  return math_ops.div_no_nan(numerator, denominator, name=name)
 
 
 def _streaming_confusion_matrix(labels, predictions, num_classes, weights=None):
@@ -401,13 +380,12 @@ def mean(values,
       update_count_op = state_ops.assign_add(count, num_values)
 
     def compute_mean(_, t, c):
-      return _safe_div(t, math_ops.maximum(c, 0), name='value')
+      return math_ops.div_no_nan(t, math_ops.maximum(c, 0), name='value')
 
     mean_t = _aggregate_across_replicas(
         metrics_collections, compute_mean, total, count)
-    update_op = _safe_div(update_total_op,
-                          math_ops.maximum(update_count_op, 0),
-                          name='update_op')
+    update_op = math_ops.div_no_nan(
+        update_total_op, math_ops.maximum(update_count_op, 0), name='update_op')
 
     if updates_collections:
       ops.add_to_collections(updates_collections, update_op)
@@ -779,19 +757,19 @@ def auc(labels,
       """
       dtp = tp[:num_thresholds - 1] - tp[1:]
       p = tp + fp
-      prec_slope = _safe_div(
+      prec_slope = math_ops.div_no_nan(
           dtp,
           math_ops.maximum(p[:num_thresholds - 1] - p[1:], 0),
           name='prec_slope')
       intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:])
       safe_p_ratio = array_ops.where(
           math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0),
-          _safe_div(p[:num_thresholds - 1],
-                    math_ops.maximum(p[1:], 0),
-                    name='recall_relative_ratio'),
-          array_ops.ones_like(p[1:]))
+          math_ops.div_no_nan(
+              p[:num_thresholds - 1],
+              math_ops.maximum(p[1:], 0),
+              name='recall_relative_ratio'), array_ops.ones_like(p[1:]))
       return math_ops.reduce_sum(
-          _safe_div(
+          math_ops.div_no_nan(
               prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)),
               math_ops.maximum(tp[1:] + fn[1:], 0),
               name='pr_auc_increment'),
@@ -1074,7 +1052,7 @@ def mean_per_class_accuracy(labels,
     update_count_op = state_ops.scatter_add(count, labels, is_correct)
 
     def compute_mean_accuracy(_, count, total):
-      per_class_accuracy = _safe_div(
+      per_class_accuracy = math_ops.div_no_nan(
           count, math_ops.maximum(total, 0), name=None)
       mean_accuracy_v = math_ops.reduce_mean(
           per_class_accuracy, name='mean_accuracy')
@@ -1083,9 +1061,8 @@ def mean_per_class_accuracy(labels,
     mean_accuracy_v = _aggregate_across_replicas(
         metrics_collections, compute_mean_accuracy, count, total)
 
-    update_op = _safe_div(update_count_op,
-                          math_ops.maximum(update_total_op, 0),
-                          name='update_op')
+    update_op = math_ops.div_no_nan(
+        update_count_op, math_ops.maximum(update_total_op, 0), name='update_op')
     if updates_collections:
       ops.add_to_collections(updates_collections, update_op)
 
@@ -1394,15 +1371,14 @@ def mean_tensor(values,
     with ops.control_dependencies([values]):
       update_count_op = state_ops.assign_add(count, num_values)
 
-    compute_mean = lambda _, t, c: _safe_div(
+    compute_mean = lambda _, t, c: math_ops.div_no_nan(  # pylint: disable=g-long-lambda
         t, math_ops.maximum(c, 0), name='value')
 
     mean_t = _aggregate_across_replicas(
         metrics_collections, compute_mean, total, count)
 
-    update_op = _safe_div(update_total_op,
-                          math_ops.maximum(update_count_op, 0),
-                          name='update_op')
+    update_op = math_ops.div_no_nan(
+        update_total_op, math_ops.maximum(update_count_op, 0), name='update_op')
     if updates_collections:
       ops.add_to_collections(updates_collections, update_op)
 
-- 
GitLab


From ea3ee9804de5b403bc3fe8d618fc88242d8c3321 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 19 Nov 2018 13:05:58 -0800
Subject: [PATCH 0518/1554] Deprecate arg_min and arg_max in the TF 2.0 API.

PiperOrigin-RevId: 222125023
---
 tensorflow/python/ops/math_ops.py               | 4 ++--
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 8 --------
 tensorflow/tools/compatibility/renames_v2.py    | 4 +++-
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 2 ++
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 679ce07f3f..952a2a1e79 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -51,8 +51,8 @@ linspace = gen_math_ops.lin_space
 
 arg_max = deprecation.deprecated(None, "Use `tf.math.argmax` instead")(arg_max)  # pylint: disable=used-before-assignment
 arg_min = deprecation.deprecated(None, "Use `tf.math.argmin` instead")(arg_min)  # pylint: disable=used-before-assignment
-tf_export("arg_max")(arg_max)
-tf_export("arg_min")(arg_min)
+tf_export(v1=["arg_max"])(arg_max)
+tf_export(v1=["arg_min"])(arg_min)
 
 # This is set by resource_variable_ops.py. It is included in this way since
 # there is a circular dependency between math_ops and resource_variable_ops
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index aefd5df2b2..175e8820b7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -444,14 +444,6 @@ tf_module {
     name: "add_n"
     argspec: "args=[\'inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "arg_max"
-    argspec: "args=[\'input\', \'dimension\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
-  }
-  member_method {
-    name: "arg_min"
-    argspec: "args=[\'input\', \'dimension\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
-  }
   member_method {
     name: "argmax"
     argspec: "args=[\'input\', \'axis\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'int64\'>\", \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 41ae1a2a68..314888f14f 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -70,6 +70,8 @@ renames = {
     'tf.all_variables': 'tf.compat.v1.all_variables',
     'tf.angle': 'tf.math.angle',
     'tf.app.run': 'tf.compat.v1.app.run',
+    'tf.arg_max': 'tf.compat.v1.arg_max',
+    'tf.arg_min': 'tf.compat.v1.arg_min',
     'tf.assert_greater_equal': 'tf.compat.v1.assert_greater_equal',
     'tf.assert_integer': 'tf.compat.v1.assert_integer',
     'tf.assert_less_equal': 'tf.compat.v1.assert_less_equal',
@@ -562,7 +564,7 @@ renames = {
     'tf.variable_scope': 'tf.compat.v1.variable_scope',
     'tf.variables_initializer': 'tf.compat.v1.variables_initializer',
     'tf.variance_scaling_initializer': 'tf.keras.initializers.VarianceScaling',
-    'tf.verify_tensor_all_finite': 'tf.debugging.assert_all_finite',
+    'tf.verify_tensor_all_finite': 'tf.compat.v1.verify_tensor_all_finite',
     'tf.wrap_function': 'tf.compat.v1.wrap_function',
     'tf.write_file': 'tf.io.write_file',
     'tf.zeta': 'tf.math.zeta'
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index e7e6fc9d77..28d30a047f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -145,6 +145,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.multinomial": "tf.random.categorical",
         "tf.random.multinomial": "tf.random.categorical",
         "tf.load_file_system_library": "tf.load_library",
+        "tf.arg_max": "tf.argmax",
+        "tf.arg_min": "tf.argmin",
     })
     # pylint: enable=line-too-long
 
-- 
GitLab


From ee20d8c029c44e2ff9b33a119f9c4a1e657c9e9a Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 19 Nov 2018 13:15:12 -0800
Subject: [PATCH 0519/1554] [TF] [XLA] Fix a number of compiler warnings on Mac
 OS X.

* Mostly these are unused/dead code.
* Fix an ignored Status in shape_util.h

PiperOrigin-RevId: 222126279
---
 tensorflow/compiler/xla/array2d.h             |  2 +-
 .../compiler/xla/service/buffer_assignment.cc |  3 +-
 .../compiler/xla/service/buffer_assignment.h  | 10 +----
 .../compiler/xla/service/computation_placer.h |  2 -
 .../compiler/xla/service/copy_insertion.cc    |  2 -
 .../xla/service/cpu/compiler_functor.cc       | 13 +++----
 .../compiler/xla/service/cpu/cpu_options.cc   |  1 -
 .../compiler/xla/service/cpu/ir_emitter.cc    | 37 +------------------
 .../compiler/xla/service/cpu/ir_emitter.h     |  8 +---
 .../xla/service/hlo_evaluator_typed_visitor.h |  2 +-
 .../compiler/xla/service/hlo_instruction.cc   | 30 ---------------
 tensorflow/compiler/xla/service/hlo_value.h   |  3 --
 tensorflow/compiler/xla/service/service.cc    | 15 --------
 tensorflow/compiler/xla/shape_tree.h          | 11 +++---
 tensorflow/compiler/xla/shape_util.h          | 17 +++++++--
 .../collective_param_resolver_local.cc        |  2 +-
 .../common_runtime/collective_rma_local.cc    |  2 +-
 tensorflow/core/graph/graph.h                 |  2 +-
 18 files changed, 35 insertions(+), 127 deletions(-)

diff --git a/tensorflow/compiler/xla/array2d.h b/tensorflow/compiler/xla/array2d.h
index 782c966b4c..e4aca98f67 100644
--- a/tensorflow/compiler/xla/array2d.h
+++ b/tensorflow/compiler/xla/array2d.h
@@ -104,7 +104,7 @@ std::unique_ptr<Array2D<NativeT>> MakeLinspaceArray2D(double from, double to,
   int64 count = n1 * n2;
   NativeT step =
       static_cast<NativeT>((count > 1) ? (to - from) / (count - 1) : 0);
-  auto set = [&array, n1, n2](int64 index, NativeT value) {
+  auto set = [&array, n2](int64 index, NativeT value) {
     (*array)(index / n2, index % n2) = value;
   };
   for (int64 i = 0; i < count - 1; ++i) {
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index 4b3ab56dea..8d7c624478 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -746,8 +746,7 @@ StatusOr<std::unique_ptr<BufferAssignment>> BufferAssigner::Run(
     LogicalBuffer::AlignmentFunction color_alignment,
     bool allow_input_output_aliasing, bool allocate_buffers_for_constants,
     BufferLiveness::Colorer colorer, ReuseAllocationFunction reuse_checker) {
-  BufferAssigner assigner(allow_input_output_aliasing,
-                          allocate_buffers_for_constants, std::move(colorer),
+  BufferAssigner assigner(allocate_buffers_for_constants, std::move(colorer),
                           std::move(reuse_checker));
   return assigner.CreateAssignment(module, std::move(hlo_ordering),
                                    std::move(buffer_size),
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index d8e1612b89..0a9fdede80 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -545,12 +545,10 @@ class BufferAssigner {
       ReuseAllocationFunction reuse_checker = nullptr);
 
  private:
-  BufferAssigner(bool allow_input_output_aliasing,
-                 bool allocate_buffers_for_constants,
+  BufferAssigner(bool allocate_buffers_for_constants,
                  BufferLiveness::Colorer colorer,
                  ReuseAllocationFunction reuse_checker)
-      : allow_input_output_aliasing_(allow_input_output_aliasing),
-        allocate_buffers_for_constants_(allocate_buffers_for_constants),
+      : allocate_buffers_for_constants_(allocate_buffers_for_constants),
         colorer_(colorer),
         reuse_checker_(reuse_checker) {}
   virtual ~BufferAssigner() = default;
@@ -640,10 +638,6 @@ class BufferAssigner {
                       LogicalBuffer::Color::Hasher>
   SplitBuffersByColor(const absl::flat_hash_set<const LogicalBuffer*>& buffers);
 
-  // If true, buffer assignments assumes that input parameter buffers and output
-  // buffers can be shared if their sizes match.
-  bool allow_input_output_aliasing_;
-
   // If true, allocate buffers for constant instructions.
   bool allocate_buffers_for_constants_;
 
diff --git a/tensorflow/compiler/xla/service/computation_placer.h b/tensorflow/compiler/xla/service/computation_placer.h
index c899ffb9dc..844b42a38d 100644
--- a/tensorflow/compiler/xla/service/computation_placer.h
+++ b/tensorflow/compiler/xla/service/computation_placer.h
@@ -105,8 +105,6 @@ class ComputationPlacer {
   // Map from platform kind to computation placer singleton.
   static std::map<se::Platform::Id, State>* GetPlatformComputationPlacers();
 
-  se::Platform::Id platform_id_;
-
   TF_DISALLOW_COPY_AND_ASSIGN(ComputationPlacer);
 };
 
diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index 4e547d925f..df60596638 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -442,7 +442,6 @@ class CopyRemover {
               const HloOrdering& ordering, HloModule* module)
       : module_(module),
         alias_analysis_(alias_analysis),
-        ordering_(ordering),
         buffer_value_tracker_(*module, alias_analysis, ordering) {}
 
   // Try to elide the given copy. The copy is elided if the instruction is not
@@ -1003,7 +1002,6 @@ class CopyRemover {
 
   HloModule* module_;
   const HloAliasAnalysis& alias_analysis_;
-  const HloOrdering& ordering_;
 
   // Object tracking the HLO values contained in each HLO buffer.
   BufferValueTracker buffer_value_tracker_;
diff --git a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
index 73b03440cb..2852fc8666 100644
--- a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
+++ b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
@@ -61,17 +61,15 @@ Disabling these as a starting point.
 // TODO(b/64227304) Creating a custom pass pipeline will replace this.
 
 namespace {
+
+// TODO(sanjoy): remove this class.
 class FilteredFunctionPassManager : public llvm::legacy::FunctionPassManager {
  public:
-  FilteredFunctionPassManager(llvm::Module* m, bool disable_expensive_passes)
-      : llvm::legacy::FunctionPassManager(m),
-        disable_expensive_passes_(disable_expensive_passes) {}
+  explicit FilteredFunctionPassManager(llvm::Module* m)
+      : llvm::legacy::FunctionPassManager(m) {}
   void add(llvm::Pass* p) override {
     llvm::legacy::FunctionPassManager::add(p);
   }
-
- private:
-  bool disable_expensive_passes_;
 };
 
 class FilteredPassManager : public llvm::legacy::PassManager {
@@ -96,8 +94,7 @@ class FilteredPassManager : public llvm::legacy::PassManager {
 std::unique_ptr<llvm::MemoryBuffer> CompilerFunctor::operator()(
     llvm::Module& module) const {
   FilteredPassManager module_passes(disable_expensive_passes_);
-  FilteredFunctionPassManager function_passes(&module,
-                                              disable_expensive_passes_);
+  FilteredFunctionPassManager function_passes(&module);
 
   VLOG(2) << "IR before optimizations";
   XLA_VLOG_LINES(2, llvm_ir::DumpModuleToString(module));
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_options.cc b/tensorflow/compiler/xla/service/cpu/cpu_options.cc
index b8ace57026..92debb83e3 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_options.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_options.cc
@@ -22,7 +22,6 @@ limitations under the License.
 namespace {
 
 const char* const kXlaOptimizeForSizeCpuOption = "xla_cpu_optimize_for_size";
-const char* const kXlaDisableVectorizedReduce = "xla_disable_vectorized_reduce";
 const char* const kLlvmIrDotTilingFactor = "xla_llvm_dot_tiling_factor";
 const char* const kXlaEnableExperimentalLlvmIrGemm =
     "xla_enable_experimental_llvm_ir_gemm";
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 9b731b0a6a..cf97a8bde0 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -140,7 +140,7 @@ StatusOr<llvm::Function*> IrEmitter::EmitComputation(
   // readcyclecounter if it is unavailable.
   bool use_rdtscp = arch_type_ == llvm::Triple::ArchType::x86 ||
                     arch_type_ == llvm::Triple::ArchType::x86_64;
-  profiling_state_ = ProfilingState(use_rdtscp, GetProfileCountersArgument());
+  profiling_state_ = ProfilingState(use_rdtscp);
   if (instruction_order == nullptr) {
     TF_RETURN_IF_ERROR(computation->Accept(this));
   } else {
@@ -1379,33 +1379,6 @@ Status IrEmitter::HandleCrossReplicaSum(HloInstruction* crs) {
   return Status::OK();
 }
 
-// Fills up the free variables in 'index_with_free_var' with values from
-// 'filler_index'. The size of free variables must be the same as the
-// size of 'filler_index'.
-//
-// This is often used after dimension reduction, where
-// 'index_with_free_var' has one or more dimensions reduced, which serves as
-// free variables (represented as nullptr). For example, if we have a 4
-// dimensional input and index for the dimension being reduced is
-// 2 (third dimension), we will have an index like [i, j, NULL, k]
-// after reduced dimension.
-//
-// Here we fill up that free variable by 'filler_index', which contains
-// the value in the reduced dimension.
-static llvm_ir::IrArray::Index FillReducedDimensionIndex(
-    llvm_ir::IrArray::Index index_with_free_var,
-    llvm_ir::IrArray::Index filler_index) {
-  llvm_ir::IrArray::Index::const_iterator it = filler_index.begin();
-
-  for (size_t i = 0; i < index_with_free_var.size(); ++i) {
-    if (index_with_free_var[i] == nullptr) {
-      index_with_free_var[i] = *it++;
-    }
-  }
-  CHECK(filler_index.end() == it);
-  return index_with_free_var;
-}
-
 Status IrEmitter::HandleParameter(HloInstruction* parameter) {
   VLOG(2) << "HandleParameter: " << parameter->ToString();
   return EmitTargetAddressForOp(parameter);
@@ -2194,14 +2167,6 @@ Status IrEmitter::HandlePad(HloInstruction* pad) {
   return Status::OK();
 }
 
-// If `hlo` is a Transpose, returns its operand; otherwise returns `hlo` itself.
-static const HloInstruction* StripTranspose(const HloInstruction& hlo) {
-  if (hlo.IsRank2Transpose()) {
-    return hlo.operand(0);
-  }
-  return &hlo;
-}
-
 Status IrEmitter::HandleFusion(HloInstruction* fusion) {
   auto* root = fusion->fused_expression_root();
   if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(fusion, assignment_)) {
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index e03ba42b0d..f529c613a3 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -467,9 +467,8 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   // profiling a computation.
   class ProfilingState {
    public:
-    ProfilingState() : use_rdtscp_(false), prof_counters_(nullptr) {}
-    ProfilingState(bool use_rdtscp, llvm::Value* prof_counters)
-        : use_rdtscp_(use_rdtscp), prof_counters_(prof_counters) {}
+    ProfilingState() : use_rdtscp_(false) {}
+    explicit ProfilingState(bool use_rdtscp) : use_rdtscp_(use_rdtscp) {}
 
     // Record the cycle counter before an HLO executes.
     void RecordCycleStart(llvm::IRBuilder<>* b, HloInstruction* hlo);
@@ -494,9 +493,6 @@ class IrEmitter : public DfsHloVisitorWithDefault,
     // intrinsic?
     bool use_rdtscp_;
 
-    // The argument which corresponds to the profile counter buffer.
-    llvm::Value* prof_counters_;
-
     // The first read cycle counter in the program.
     llvm::Value* first_read_cycle_start_ = nullptr;
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index b79412e4d9..332fa874c3 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -593,7 +593,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   }
 
-  Status HandleDivide(HloInstruction* divide) {
+  Status HandleDivide(HloInstruction* divide) override {
     return HandleDivide<ElementwiseT>(divide);
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index ce255292d2..cd95052580 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -2628,36 +2628,6 @@ Status HloInstruction::AcceptWithOperandOrder(
   return Status::OK();
 }
 
-namespace {
-
-// Returns true if the given order is a topological sort of the instructions
-// it contains.
-bool OrderIsTopologicalSort(const std::vector<const HloInstruction*>& order) {
-  // Create a map from instruction to its position in 'order'.
-  std::unordered_map<const HloInstruction*, int> order_position;
-  for (int i = 0; i < order.size(); i++) {
-    if (!order_position.insert({order[i], i}).second) {
-      // Instruction order[i] is duplicated in the order.
-      return false;
-    }
-  }
-  // Verify that the operand of each instruction in the order is also in the
-  // order *and* the operand's position is earlier (defs are before uses for
-  // all ops).
-  for (auto* instruction : order) {
-    for (auto* operand : instruction->operands()) {
-      if (!ContainsKey(order_position, operand) ||
-          order_position.at(operand) >= order_position.at(instruction)) {
-        return false;
-      }
-    }
-  }
-
-  return true;
-}
-
-}  // namespace
-
 Status HloInstruction::Accept(
     const std::function<Status(HloInstruction*)>& visitor_func) {
   FunctionVisitor visitor(visitor_func);
diff --git a/tensorflow/compiler/xla/service/hlo_value.h b/tensorflow/compiler/xla/service/hlo_value.h
index b6670d409b..1f01b0bb36 100644
--- a/tensorflow/compiler/xla/service/hlo_value.h
+++ b/tensorflow/compiler/xla/service/hlo_value.h
@@ -166,9 +166,6 @@ class HloValue : public BufferValue {
 
   // Whether this value is live out of the HLO module.
   bool live_out_of_module_ = false;
-
-  // Whether this value is live out of its computation.
-  bool live_out_of_computation_ = false;
 };
 
 std::ostream& operator<<(std::ostream& out, const HloValue& hlo_value);
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 75f7413b3c..13fd6bc009 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -957,21 +957,6 @@ Status Service::TransferToClient(const TransferToClientRequest* arg,
   return Status::OK();
 }
 
-namespace {
-
-// Creates a clone of the given shaped buffer with the given device ordinal. The
-// shape and DeviceMemoryBase values of the clone are identical to the original.
-std::unique_ptr<ShapedBuffer> CloneShapedBufferOnDevice(
-    const ShapedBuffer& shaped_buffer, int device_ordinal) {
-  auto clone = absl::make_unique<ShapedBuffer>(
-      shaped_buffer.on_host_shape(), shaped_buffer.on_device_shape(),
-      shaped_buffer.platform(), device_ordinal);
-  clone->buffers() = shaped_buffer.buffers();
-  return clone;
-}
-
-}  // namespace
-
 Status Service::TransferToServer(const TransferToServerRequest* arg,
                                  TransferToServerResponse* result) {
   TF_ASSIGN_OR_RETURN(Literal literal,
diff --git a/tensorflow/compiler/xla/shape_tree.h b/tensorflow/compiler/xla/shape_tree.h
index df610102b4..7bf9772916 100644
--- a/tensorflow/compiler/xla/shape_tree.h
+++ b/tensorflow/compiler/xla/shape_tree.h
@@ -667,12 +667,11 @@ void ShapeTree<T>::CopySubtreeFrom(const ShapeTree<T>& other,
 template <typename T>
 bool ShapeTree<T>::operator==(const ShapeTree<T>& other) const {
   bool equal = true;
-  ForEachElement(
-      [this, &other, &equal](const ShapeIndex& index, const T& data) {
-        if (data != other.element(index)) {
-          equal = false;
-        }
-      });
+  ForEachElement([&other, &equal](const ShapeIndex& index, const T& data) {
+    if (data != other.element(index)) {
+      equal = false;
+    }
+  });
   return equal;
 }
 
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index f9f1347460..7f72e57d00 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace xla {
@@ -756,10 +757,18 @@ class ShapeUtil {
       pool.emplace(tensorflow::Env::Default(), "foreach", kNumThreads);
     }
 
+    tensorflow::mutex mu;
+    Status status;  // Guarded by mu
+
     while (n < rank) {
       if (pool != absl::nullopt) {
-        pool->Schedule(
-            [indexes, &visitor_function] { visitor_function(indexes); });
+        pool->Schedule([indexes, &visitor_function, &mu, &status] {
+          StatusOr<bool> result = visitor_function(indexes);
+          if (!result.ok()) {
+            tensorflow::mutex_lock lock(mu);
+            status = status.ok() ? result.status() : status;
+          }
+        });
       } else {
         TF_ASSIGN_OR_RETURN(bool should_continue, visitor_function(indexes));
         if (!should_continue) {
@@ -777,7 +786,9 @@ class ShapeUtil {
       }
     }
 
-    return Status::OK();
+    // Waits for the scheduled work to complete.
+    pool.reset();
+    return status;
   }
 
   TF_DISALLOW_COPY_AND_ASSIGN(ShapeUtil);
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index 3a03b6724c..624d3f2289 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -511,7 +511,7 @@ void CollectiveParamResolverLocal::FindInstanceRec(
         if (irec->is_init) {
           exit_outside_locks = true;
         } else {
-          irec->init_waiters.push_back([this, gr, cp, done](InstanceRec* irec) {
+          irec->init_waiters.push_back([this, done](InstanceRec* irec) {
             CallbackWithStatus(done, irec);
           });
           return;
diff --git a/tensorflow/core/common_runtime/collective_rma_local.cc b/tensorflow/core/common_runtime/collective_rma_local.cc
index 288ae9d794..d99565b49a 100644
--- a/tensorflow/core/common_runtime/collective_rma_local.cc
+++ b/tensorflow/core/common_runtime/collective_rma_local.cc
@@ -38,7 +38,7 @@ void CollectiveRemoteAccessLocal::RecvFromPeer(
     return;
   }
   buf_rendezvous_.ConsumeBuf(
-      key, [this, to_tensor, to_device_ctx, to_device, to_alloc_attr,
+      key, [to_tensor, to_device_ctx, to_device, to_alloc_attr,
             dev_to_dev_stream_index,
             done](const Status& s, BufRendezvous::Hook* hook) {
         if (!s.ok()) {
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 90a25d4e3f..6c6d98b5aa 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -59,7 +59,7 @@ class EdgeSetTest;
 class Graph;
 class GraphDef;
 class Node;
-class OutputTensor;
+struct OutputTensor;
 class VersionDef;
 class WhileContext;
 
-- 
GitLab


From 4a7b985fd2fd8b1c016bfb8b757f27983cda65f7 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 19 Nov 2018 13:44:52 -0800
Subject: [PATCH 0520/1554] Remove `tf.sparse.merge` from v2.

PiperOrigin-RevId: 222131011
---
 tensorflow/python/ops/sparse_ops.py                    | 4 ++--
 tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt | 4 ----
 tensorflow/tools/compatibility/renames_v2.py           | 3 ++-
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 9981f389c8..57f9f4e03e 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -1487,8 +1487,8 @@ def sparse_to_indicator(sp_input, vocab_size, name=None):
         sp_new, default_value=False, validate_indices=False, name=name)
 
 
-@tf_export("sparse.merge", v1=["sparse.merge", "sparse_merge"])
-@deprecation.deprecated_endpoints("sparse_merge")
+@tf_export(v1=["sparse.merge", "sparse_merge"])
+@deprecation.deprecated_endpoints(["sparse.merge", "sparse_merge"])
 def sparse_merge(sp_ids, sp_values, vocab_size, name=None,
                  already_sorted=False):
   """Combines a batch of feature ids and values into a single `SparseTensor`.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index bd7592e019..4ad94568b2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -44,10 +44,6 @@ tf_module {
     name: "maximum"
     argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "merge"
-    argspec: "args=[\'sp_ids\', \'sp_values\', \'vocab_size\', \'name\', \'already_sorted\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], "
-  }
   member_method {
     name: "minimum"
     argspec: "args=[\'sp_a\', \'sp_b\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 314888f14f..3b5559a9f3 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -445,6 +445,7 @@ renames = {
     'tf.space_to_batch': 'tf.nn.space_to_batch',
     'tf.space_to_depth': 'tf.nn.space_to_depth',
     'tf.sparse.matmul': 'tf.sparse.sparse_dense_matmul',
+    'tf.sparse.merge': 'tf.compat.v1.sparse.merge',
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
     'tf.sparse.reduce_max_sparse': 'tf.compat.v1.sparse.reduce_max_sparse',
     'tf.sparse_add': 'tf.compat.v1.sparse_add',
@@ -452,7 +453,7 @@ renames = {
     'tf.sparse_mask': 'tf.sparse.mask',
     'tf.sparse_matmul': 'tf.compat.v1.sparse_matmul',
     'tf.sparse_maximum': 'tf.sparse.maximum',
-    'tf.sparse_merge': 'tf.sparse.merge',
+    'tf.sparse_merge': 'tf.compat.v1.sparse_merge',
     'tf.sparse_minimum': 'tf.sparse.minimum',
     'tf.sparse_placeholder': 'tf.compat.v1.sparse_placeholder',
     'tf.sparse_reduce_max': 'tf.compat.v1.sparse_reduce_max',
-- 
GitLab


From 2c03c37711afee175d7664b4d859e79f018a6209 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 13:46:55 -0800
Subject: [PATCH 0521/1554] Rename batch_norm_with_global_normalization args
 for TF 2.0 API

PiperOrigin-RevId: 222131379
---
 tensorflow/python/ops/nn_impl.py              | 49 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  5 ++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 30c8058a35..9cf53f191a 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -1122,7 +1122,7 @@ def fused_batch_norm(
   return y, batch_mean, batch_var
 
 
-@tf_export("nn.batch_norm_with_global_normalization")
+@tf_export(v1=["nn.batch_norm_with_global_normalization"])
 def batch_norm_with_global_normalization(t,
                                          m,
                                          v,
@@ -1160,6 +1160,53 @@ def batch_norm_with_global_normalization(t,
                              else None, variance_epsilon, name)
 
 
+# pylint: disable=redefined-builtin,line-too-long
+@tf_export("nn.batch_norm_with_global_normalization", v1=[])
+def batch_norm_with_global_normalization_v2(input,
+                                            mean,
+                                            variance,
+                                            beta,
+                                            gamma,
+                                            variance_epsilon,
+                                            scale_after_normalization,
+                                            name=None):
+  """Batch normalization.
+
+  This op is deprecated. See `tf.nn.batch_normalization`.
+
+  Args:
+    input: A 4D input Tensor.
+    mean: A 1D mean Tensor with size matching the last dimension of t.
+      This is the first output from tf.nn.moments,
+      or a saved moving average thereof.
+    variance: A 1D variance Tensor with size matching the last dimension of t.
+      This is the second output from tf.nn.moments,
+      or a saved moving average thereof.
+    beta: A 1D beta Tensor with size matching the last dimension of t.
+      An offset to be added to the normalized tensor.
+    gamma: A 1D gamma Tensor with size matching the last dimension of t.
+      If "scale_after_normalization" is true, this tensor will be multiplied
+      with the normalized tensor.
+    variance_epsilon: A small float number to avoid dividing by 0.
+    scale_after_normalization: A bool indicating whether the resulted tensor
+      needs to be multiplied with gamma.
+    name: A name for this operation (optional).
+
+  Returns:
+     A batch-normalized `t`.
+  """
+  return batch_norm_with_global_normalization(t=input,
+                                              m=mean,
+                                              v=variance,
+                                              beta=beta,
+                                              gamma=gamma,
+                                              variance_epsilon=variance_epsilon,
+                                              scale_after_normalization=scale_after_normalization,
+                                              name=name)
+
+# pylint: enable=redefined-builtin,line-too-long
+
+
 def _sum_rows(x):
   """Returns a vector summing up each row of the matrix x."""
   # _sum_rows(x) is equivalent to math_ops.reduce_sum(x, 1) when x is
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index ecbda448f1..b4c845a3d7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -30,7 +30,7 @@ tf_module {
   }
   member_method {
     name: "batch_norm_with_global_normalization"
-    argspec: "args=[\'t\', \'m\', \'v\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'mean\', \'variance\', \'beta\', \'gamma\', \'variance_epsilon\', \'scale_after_normalization\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "batch_normalization"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 28d30a047f..fe74f9a00c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -67,6 +67,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.multinomial": {
             "output_dtype": "dtype",
         },
+        "tf.nn.batch_norm_with_global_normalization": {
+            "t": "input",
+            "m": "mean",
+            "v": "variance",
+        },
         "tf.nn.conv3d": {
             "filter": "filters"
         },
-- 
GitLab


From 717ac62eb16eb5069bc71e002286960c5649f56a Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 19 Nov 2018 13:50:24 -0800
Subject: [PATCH 0522/1554] Set TF_NEED_ROCM=0 when building with CUDA.

PiperOrigin-RevId: 222131980
---
 tensorflow/tools/ci_build/builds/libtensorflow.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/ci_build/builds/libtensorflow.sh b/tensorflow/tools/ci_build/builds/libtensorflow.sh
index 9b3ff0cba7..44abcc309b 100755
--- a/tensorflow/tools/ci_build/builds/libtensorflow.sh
+++ b/tensorflow/tools/ci_build/builds/libtensorflow.sh
@@ -55,6 +55,7 @@ function build_libtensorflow_tarball() {
   export CC_OPT_FLAGS='-mavx'
   if [ "${TF_NEED_CUDA}" == "1" ]; then
     BAZEL_OPTS="${BAZEL_OPTS} --config=cuda"
+    export TF_NEED_ROCM=0
   fi
   bazel clean --expunge
   yes "" | ./configure
-- 
GitLab


From 7b1e7138f1fc36dd83e0cb1e8ea9b49d2b576802 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 14:00:59 -0800
Subject: [PATCH 0523/1554] Improve thread-safety of ProcessState::singleton()
 and GPUProcessState::singleton().

One might wish to subclass GPUProcessState in order to override some of
its methods using special knowledge about the platform in use.
Suppose the new class is called SubGPUProcessState.  In such a case it
is necessary that all calls to SubGPUProcessState::singleton() and
GPUProcessState::singleton() return exactly the same pointer, to an
instance of SubGPUProcessState.  To accomplish this there must be a
static var initializer that sets the singleton correctly prior to main
routine invocation.  Here's an example program fragment:

SubGPUProcessState* global_initializer = ([]() {
  SubGPUProcessState* ps = SubGPUProcessState::singleton();
  return down_cast<SubGPUProcessState*>(GPUProcessState::singleton(ps));
})();

PiperOrigin-RevId: 222134183
---
 .../common_runtime/gpu/gpu_process_state.cc   | 23 ++++---------------
 .../common_runtime/gpu/gpu_process_state.h    | 22 ++++++++++++++----
 .../core/common_runtime/process_state.cc      | 20 ++--------------
 .../core/common_runtime/process_state.h       |  4 +---
 4 files changed, 25 insertions(+), 44 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
index 1a43d2c8fc..a9a19f0fe0 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
@@ -55,30 +55,17 @@ bool useCudaMemoryGuardAllocator() {
 
 }  // namespace
 
-GPUProcessState* GPUProcessState::instance_ = nullptr;
-
-/*static*/ GPUProcessState* GPUProcessState::singleton() {
-  if (instance_ == nullptr) {
-    instance_ = new GPUProcessState;
-  }
-  CHECK(instance_->process_state_);
-
-  return instance_;
+/*static*/ GPUProcessState* GPUProcessState::singleton(GPUProcessState* ps) {
+  static GPUProcessState* instance = ps ? ps : new GPUProcessState;
+  DCHECK((!ps) || (ps == instance))
+      << "Multiple calls to GPUProcessState with non-null ps";
+  return instance;
 }
 
 GPUProcessState::GPUProcessState() : gpu_device_enabled_(false) {
-  CHECK(instance_ == nullptr);
-  instance_ = this;
   process_state_ = ProcessState::singleton();
 }
 
-// Normally the GPUProcessState singleton is never explicitly deleted.
-// This function is defined for debugging problems with the allocators.
-GPUProcessState::~GPUProcessState() {
-  CHECK_EQ(this, instance_);
-  instance_ = nullptr;
-}
-
 int GPUProcessState::BusIdForGPU(TfGpuId tf_gpu_id) {
   // Return the NUMA node associated with the GPU's StreamExecutor.
   se::StreamExecutor* se =
diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.h b/tensorflow/core/common_runtime/gpu/gpu_process_state.h
index 43e9a31660..df51c10c80 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.h
@@ -37,7 +37,19 @@ class PoolAllocator;
 // Singleton that manages per-process state when GPUs are present.
 class GPUProcessState {
  public:
-  static GPUProcessState* singleton();
+  // If ps == nullptr, returns pointer to the single instance of this class to
+  // be used within this process.
+  //
+  // If ps != nullptrs, accepts a value to be returned by all subsequent calls.
+  // A non-null ps may ONLY be provided during program static storage
+  // initialization.  Must not be called more than once with a non-null ps.
+  //
+  // If a derived class of GPUProcessState is ever used in a process, it must
+  // always be used in place of this class.  In order to ensure that existing
+  // calls to GPUProcessState::singleton() all resolve to the derived instance
+  // instead, this function must be called once during startup, supplying the
+  // derived instance value, prior to any accessor call to this function.
+  static GPUProcessState* singleton(GPUProcessState* ps = nullptr);
 
   // Query whether any GPU device has been created so far.
   // Disable thread safety analysis since a race is benign here.
@@ -97,7 +109,11 @@ class GPUProcessState {
   virtual int BusIdForGPU(TfGpuId tf_gpu_id);
 
  protected:
+  // GPUProcessState is a singleton that should not normally be deleted except
+  // at process shutdown.
   GPUProcessState();
+  virtual ~GPUProcessState() {}
+  friend class GPUDeviceTest;
 
   // Helper method for unit tests to reset the ProcessState singleton by
   // cleaning up everything. Never use in production.
@@ -127,10 +143,6 @@ class GPUProcessState {
       GUARDED_BY(mu_);
   std::vector<std::vector<SubAllocator::Visitor>> cuda_host_free_visitors_
       GUARDED_BY(mu_);
-
-  virtual ~GPUProcessState();
-
-  friend class GPUDeviceTest;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/process_state.cc b/tensorflow/core/common_runtime/process_state.cc
index c8d5bf100b..3d8ac9b134 100644
--- a/tensorflow/core/common_runtime/process_state.cc
+++ b/tensorflow/core/common_runtime/process_state.cc
@@ -32,28 +32,12 @@ limitations under the License.
 
 namespace tensorflow {
 
-ProcessState* ProcessState::instance_ = nullptr;
-
 /*static*/ ProcessState* ProcessState::singleton() {
-  if (instance_ == nullptr) {
-    instance_ = new ProcessState;
-  }
-
-  return instance_;
+  static ProcessState* instance = new ProcessState;
+  return instance;
 }
 
 ProcessState::ProcessState() : numa_enabled_(false) {
-  CHECK(instance_ == nullptr);
-}
-
-// Normally the ProcessState singleton is never explicitly deleted.
-// This function is defined for debugging problems with the allocators.
-ProcessState::~ProcessState() {
-  CHECK_EQ(this, instance_);
-  instance_ = nullptr;
-  for (Allocator* a : cpu_allocators_) {
-    delete a;
-  }
 }
 
 string ProcessState::MemDesc::DebugString() {
diff --git a/tensorflow/core/common_runtime/process_state.h b/tensorflow/core/common_runtime/process_state.h
index 06a283c9a6..6849d305b3 100644
--- a/tensorflow/core/common_runtime/process_state.h
+++ b/tensorflow/core/common_runtime/process_state.h
@@ -87,7 +87,7 @@ class ProcessState {
 
   // Helper method for unit tests to reset the ProcessState singleton by
   // cleaning up everything. Never use in production.
-  virtual void TestOnlyReset();
+  void TestOnlyReset();
 
   static ProcessState* instance_;
   bool numa_enabled_;
@@ -100,8 +100,6 @@ class ProcessState {
   std::vector<SubAllocator::Visitor> cpu_alloc_visitors_ GUARDED_BY(mu_);
   std::vector<SubAllocator::Visitor> cpu_free_visitors_ GUARDED_BY(mu_);
 
-  virtual ~ProcessState();
-
   // Optional RecordingAllocators that wrap the corresponding
   // Allocators for runtime attribute use analysis.
   MDMap mem_desc_map_;
-- 
GitLab


From a41d366c5d346aba054dc0a9c8595213866c4936 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 19 Nov 2018 14:04:47 -0800
Subject: [PATCH 0524/1554] [XLA] Enable malloc zone statistics in Mac OS X
 LLVM build to silence warnings about sbrk() being deprecated.

PiperOrigin-RevId: 222135053
---
 third_party/llvm/llvm.bzl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/third_party/llvm/llvm.bzl b/third_party/llvm/llvm.bzl
index 54ca86f327..5a977f82c4 100644
--- a/third_party/llvm/llvm.bzl
+++ b/third_party/llvm/llvm.bzl
@@ -250,6 +250,7 @@ linux_cmake_vars = {
 # CMake variables specific to the Darwin (Mac OS X) platform.
 darwin_cmake_vars = {
     "HAVE_MALLOC_MALLOC_H": 1,
+    "HAVE_MALLOC_ZONE_STATISTICS": 1,
 }
 
 # CMake variables specific to the Windows platform.
-- 
GitLab


From ba3ee00a14559b5c8be1d5d5020989c1d7368b62 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Mon, 19 Nov 2018 14:09:24 -0800
Subject: [PATCH 0525/1554] Enable use of Keras graph functions with iterators
 and datasets during eager execution. This is done by routing `fit`/etc to the
 methods `fit_generator`/etc when passed an eager dataset or eager iterator.

PiperOrigin-RevId: 222136116
---
 tensorflow/python/keras/BUILD                 |  13 +
 tensorflow/python/keras/backend.py            |   8 +-
 .../feature_columns_integration_test.py       |  11 +-
 tensorflow/python/keras/engine/training.py    |  57 ++-
 .../keras/engine/training_dataset_test.py     | 324 ++++++++++++++++++
 .../python/keras/engine/training_eager.py     |  49 +--
 .../keras/engine/training_eager_test.py       |  29 +-
 .../python/keras/engine/training_generator.py |  99 +++---
 .../python/keras/engine/training_test.py      | 284 ---------------
 .../python/keras/engine/training_utils.py     |   4 +-
 10 files changed, 494 insertions(+), 384 deletions(-)
 create mode 100644 tensorflow/python/keras/engine/training_dataset_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 510fcc04b4..adfa2260c6 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -719,6 +719,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "training_dataset_test",
+    size = "medium",
+    srcs = ["engine/training_dataset_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":keras",
+        "//tensorflow/python:client_testlib",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "training_generator_test",
     size = "enormous",
@@ -743,6 +755,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 8ab8f2ae04..969c386fc6 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3161,8 +3161,12 @@ class EagerExecutionFunction(object):
         if value is None:
           raise ValueError(
               'You must feed a value for placeholder %s' % (tensor,))
-      converted_inputs.append(
-          ops.convert_to_tensor(value, dtype=tensor.dtype))
+      value = ops.convert_to_tensor(value, dtype=tensor.dtype)
+      if value.dtype != tensor.dtype:
+        # Temporary workaround due to `convert_to_tensor` not casting floats.
+        # See b/119637405
+        value = math_ops.cast(value, tensor.dtype)
+      converted_inputs.append(value)
     outputs = self._graph_fn(*converted_inputs)
     return [x.numpy() for x in outputs]
 
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index 2ceef6ba50..b7549e013c 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -18,10 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import metrics as metrics_module
@@ -42,7 +44,7 @@ class TestDNNModel(keras.models.Model):
     return net
 
 
-class FeatureColumnsIntegrationTest(test.TestCase):
+class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
   """Most Sequential model API tests are covered in `training_test.py`.
 
   """
@@ -112,8 +114,10 @@ class FeatureColumnsIntegrationTest(test.TestCase):
     dnn_model.evaluate(x=x, y=y, batch_size=5)
     dnn_model.predict(x=x, batch_size=5)
 
+  @parameterized.parameters(True, False)
   @tf_test_util.run_in_graph_and_eager_modes
-  def test_subclassed_model_with_feature_columns_with_ds_input(self):
+  def test_subclassed_model_with_feature_columns_with_ds_input(self,
+                                                               run_eagerly):
     col_a = fc.numeric_column('a')
     col_b = fc.numeric_column('b')
 
@@ -122,7 +126,8 @@ class FeatureColumnsIntegrationTest(test.TestCase):
     dnn_model.compile(
         optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
         loss='categorical_crossentropy',
-        metrics=['accuracy'])
+        metrics=['accuracy'],
+        run_eagerly=run_eagerly and context.executing_eagerly())
 
     y = np.random.randint(20, size=(100, 1))
     y = keras.utils.to_categorical(y, num_classes=20)
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index c3ec23c6b4..032ebeaeca 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1152,9 +1152,6 @@ class Model(Network):
     is_x_eager_iterator = isinstance(x, iterator_ops.EagerIterator)
     is_x_iterator = isinstance(x, iterator_ops.Iterator)
 
-    if is_x_eager_iterator:
-      self._run_eagerly = True  # TODO(fchollet): support using graph functions
-
     # Validate user inputs when data is given as a dataset or dataset iterator.
     if is_x_iterator or is_x_eager_iterator:
       training_utils.validate_iterator_input(x, y, sample_weight,
@@ -1207,6 +1204,8 @@ class Model(Network):
     all_inputs = []
     is_build_called = False
     is_compile_called = False
+    # Whether this is a subclassed model that expects dictionary inputs
+    # rather than list inputs (e.g. FeatureColumn-based models).
     dict_inputs = False
     if not self.inputs:
       # We need to use `x` to set the model inputs.
@@ -1236,6 +1235,10 @@ class Model(Network):
         self._set_inputs(x)
     else:
       dict_inputs = isinstance(self.inputs, dict)
+    if dict_inputs and context.executing_eagerly():
+      # No support for graph functions when the model expects dictionary inputs
+      # (i.e. FeatureColumn-based models).
+      self.run_eagerly = True
 
     if y is not None:
       if not self.optimizer:
@@ -1275,7 +1278,7 @@ class Model(Network):
           # Handle target tensors if any passed.
           if not isinstance(y, (list, tuple)):
             y = [y]
-          target_tensors = [v for v in y if tensor_util.is_tensor(v)]
+          target_tensors = [v for v in y if _is_symbolic_tensor(v)]
         is_compile_called = True
         self.compile(
             optimizer=self.optimizer,
@@ -1294,7 +1297,7 @@ class Model(Network):
     # mixed symbolic/value inputs.
     if (not self.run_eagerly and is_build_called and
         is_compile_called and
-        any(tensor_util.is_tensor(v) for v in all_inputs)):
+        any(_is_symbolic_tensor(v) for v in all_inputs)):
       return [], [], []
 
     # What follows is input validation and standardization to list format,
@@ -1766,6 +1769,18 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps)
+    elif isinstance(x, iterator_ops.EagerIterator):
+      return training_generator.fit_generator(
+          self,
+          x,
+          steps_per_epoch=steps_per_epoch,
+          epochs=epochs,
+          verbose=verbose,
+          callbacks=callbacks,
+          validation_data=validation_data,
+          validation_steps=validation_steps,
+          workers=0,
+          initial_epoch=initial_epoch)
     else:
       return training_arrays.fit_loop(
           self,
@@ -1873,7 +1888,6 @@ class Model(Network):
           max_queue_size=max_queue_size,
           workers=workers,
           use_multiprocessing=use_multiprocessing)
-
     # Validate and standardize user data.
     if self._distribution_strategy:
       distributed_training_utils.validate_inputs(
@@ -1908,6 +1922,13 @@ class Model(Network):
     elif training_distributed.should_run_experimental_loop(self):
       return training_distributed.experimental_test_loop(
           self, iterator=x, verbose=verbose, steps=steps)
+    elif isinstance(x, iterator_ops.EagerIterator):
+      return training_generator.evaluate_generator(
+          self,
+          x,
+          steps=steps,
+          verbose=verbose,
+          workers=0)
     else:
       return training_arrays.test_loop(
           self,
@@ -1981,7 +2002,6 @@ class Model(Network):
           max_queue_size=max_queue_size,
           workers=workers,
           use_multiprocessing=use_multiprocessing)
-
     if self._distribution_strategy:
       distributed_training_utils.validate_inputs(
           x, None, self._distribution_strategy)
@@ -2003,9 +2023,15 @@ class Model(Network):
       return training_eager.predict_loop(
           self, x, batch_size=batch_size, verbose=verbose, steps=steps)
     elif training_distributed.should_run_experimental_loop(self):
-      results = training_distributed.experimental_predict_loop(
+      return training_distributed.experimental_predict_loop(
           self, x, verbose=verbose, steps=steps)
-      return results
+    elif isinstance(x, iterator_ops.EagerIterator):
+      return training_generator.predict_generator(
+          self,
+          x,
+          steps=steps,
+          verbose=verbose,
+          workers=0)
     else:
       return training_arrays.predict_loop(
           self, x, batch_size=batch_size, verbose=verbose, steps=steps)
@@ -2151,13 +2177,12 @@ class Model(Network):
     # Validate and standardize user data.
     inputs, _, _ = self._standardize_user_data(x)
     if self.run_eagerly:
-      if (isinstance(x, iterator_ops.EagerIterator) or
-          (isinstance(x, dataset_ops.Dataset))):
+      if (isinstance(inputs, iterator_ops.EagerIterator) or
+          (isinstance(inputs, dataset_ops.Dataset))):
         inputs = training_utils.cast_if_floating_dtype(inputs)
-      else:
+      elif isinstance(inputs, collections.Sequence):
         inputs = [
-            ops.convert_to_tensor(val, dtype=K.floatx()) for val in inputs
-        ]
+            ops.convert_to_tensor(val, dtype=K.floatx()) for val in inputs]
       return self(inputs)  # pylint: disable=not-callable
 
     self._make_predict_function()
@@ -2463,3 +2488,7 @@ class DistributedCallbackModel(Model):
       logging.warning('You are accessing attribute ' + item + ' of the '
                       'DistributedCallbackModel that may not have been set '
                       'correctly.')
+
+
+def _is_symbolic_tensor(x):
+  return tensor_util.is_tensor(x) and not isinstance(x, ops.EagerTensor)
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
new file mode 100644
index 0000000000..e8b884e935
--- /dev/null
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -0,0 +1,324 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for training routines."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import metrics as metrics_module
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training.rmsprop import RMSPropOptimizer
+
+
+class TestTrainingWithDatasetIterators(test.TestCase):
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_training_and_eval_methods_on_iterators_single_io(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae', metrics_module.CategoricalAccuracy()]
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3))
+    targets = np.zeros((10, 4))
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+    iterator = dataset.make_one_shot_iterator()
+
+    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
+    model.evaluate(iterator, steps=2, verbose=1)
+    model.predict(iterator, steps=2)
+
+    # Test with validation data
+    model.fit(iterator,
+              epochs=1, steps_per_epoch=2, verbose=0,
+              validation_data=iterator, validation_steps=2)
+    # Test with validation split
+    with self.assertRaisesRegexp(
+        ValueError, '`validation_split` argument is not supported '
+        'when input `x` is a dataset or a dataset iterator'):
+      model.fit(iterator,
+                epochs=1, steps_per_epoch=2, verbose=0,
+                validation_split=0.5, validation_steps=2)
+
+    # Test with sample weight.
+    sample_weight = np.random.random((10,))
+    with self.assertRaisesRegexp(
+        ValueError, '`sample_weight` argument is not supported '
+        'when input `x` is a dataset or a dataset iterator'):
+      model.fit(
+          iterator,
+          epochs=1,
+          steps_per_epoch=2,
+          verbose=0,
+          sample_weight=sample_weight)
+
+    # Test invalid usage
+    with self.assertRaisesRegexp(ValueError,
+                                 'you should not specify a target'):
+      model.fit(iterator, iterator,
+                epochs=1, steps_per_epoch=2, verbose=0)
+
+    with self.assertRaisesRegexp(
+        ValueError, 'you should specify the `steps_per_epoch` argument'):
+      model.fit(iterator, epochs=1, verbose=0)
+    with self.assertRaisesRegexp(ValueError,
+                                 'you should specify the `steps` argument'):
+      model.evaluate(iterator, verbose=0)
+    with self.assertRaisesRegexp(ValueError,
+                                 'you should specify the `steps` argument'):
+      model.predict(iterator, verbose=0)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_get_next_op_created_once(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae']
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3))
+    targets = np.zeros((10, 4))
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+    iterator = dataset.make_one_shot_iterator()
+
+    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
+    # Finalize graph to make sure we are not appending another iterator
+    # get_next op in the graph.
+    ops.get_default_graph().finalize()
+    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_iterators_running_out_of_data(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae']
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3))
+    targets = np.zeros((10, 4))
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(2)
+    dataset = dataset.batch(10)
+    iterator = dataset.make_one_shot_iterator()
+
+    with test.mock.patch.object(logging, 'warning') as mock_log:
+      model.fit(iterator, epochs=1, steps_per_epoch=3, verbose=0)
+      self.assertRegexpMatches(
+          str(mock_log.call_args),
+          'dataset iterator ran out of data')
+
+
+class TestTrainingWithDataset(test.TestCase):
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_calling_model_on_same_dataset(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae']
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3))
+    targets = np.zeros((10, 4))
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+
+    # Call fit with validation data
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
+              validation_data=dataset, validation_steps=2)
+    # Finalize the graph to make sure new ops aren't added when calling on the
+    # same dataset
+    ops.get_default_graph().finalize()
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
+              validation_data=dataset, validation_steps=2)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_training_and_eval_methods_on_dataset(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae', metrics_module.CategoricalAccuracy()]
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3))
+    targets = np.zeros((10, 4))
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+    model.evaluate(dataset, steps=2, verbose=1)
+    model.predict(dataset, steps=2)
+
+    # Test with validation data
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
+              validation_data=dataset, validation_steps=2)
+
+    # Test with validation split
+    with self.assertRaisesRegexp(
+        ValueError, '`validation_split` argument is not supported '
+        'when input `x` is a dataset or a dataset iterator'):
+      model.fit(dataset,
+                epochs=1, steps_per_epoch=2, verbose=0,
+                validation_split=0.5, validation_steps=2)
+
+    # Test with sample weight.
+    sample_weight = np.random.random((10,))
+    with self.assertRaisesRegexp(
+        ValueError, '`sample_weight` argument is not supported '
+        'when input `x` is a dataset or a dataset iterator'):
+      model.fit(
+          dataset,
+          epochs=1,
+          steps_per_epoch=2,
+          verbose=0,
+          sample_weight=sample_weight)
+
+    # Test invalid usage
+    with self.assertRaisesRegexp(ValueError,
+                                 'you should not specify a target'):
+      model.fit(dataset, dataset,
+                epochs=1, steps_per_epoch=2, verbose=0)
+
+    with self.assertRaisesRegexp(
+        ValueError, 'you should specify the `steps_per_epoch` argument'):
+      model.fit(dataset, epochs=1, verbose=0)
+    with self.assertRaisesRegexp(ValueError,
+                                 'you should specify the `steps` argument'):
+      model.evaluate(dataset, verbose=0)
+    with self.assertRaisesRegexp(ValueError,
+                                 'you should specify the `steps` argument'):
+      model.predict(dataset, verbose=0)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_dataset_with_sample_weights(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae', metrics_module.CategoricalAccuracy()]
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
+    sample_weights = np.ones((10), np.float32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets,
+                                                      sample_weights))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+    model.evaluate(dataset, steps=2, verbose=1)
+    model.predict(dataset, steps=2)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_dataset_with_sparse_labels(self):
+    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'sparse_categorical_crossentropy'
+    model.compile(optimizer, loss)
+
+    inputs = np.zeros((10, 3))
+    targets = np.random.randint(0, 4, size=10, dtype=np.int32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+
+    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+
+  def test_dataset_input_shape_validation(self):
+    with self.cached_session():
+      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+      model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse')
+
+      # User forgets to batch the dataset
+      inputs = np.zeros((10, 3))
+      targets = np.zeros((10, 4))
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.repeat(100)
+
+      with self.assertRaisesRegexp(
+          ValueError,
+          r'expected (.*?) to have shape \(3,\) but got array with shape \(1,\)'
+      ):
+        model.train_on_batch(dataset)
+
+      # Wrong input shape
+      inputs = np.zeros((10, 5))
+      targets = np.zeros((10, 4))
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
+
+      with self.assertRaisesRegexp(ValueError,
+                                   r'expected (.*?) to have shape \(3,\)'):
+        model.train_on_batch(dataset)
+
+
+class TestMetricsWithDatasetIterators(test.TestCase):
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_metrics_correctness_with_iterator(self):
+    model = keras.Sequential()
+    model.add(
+        keras.layers.Dense(
+            8, activation='relu', input_dim=4, kernel_initializer='ones'))
+    model.add(
+        keras.layers.Dense(
+            1, activation='sigmoid', kernel_initializer='ones'))
+    model.compile(
+        loss='binary_crossentropy',
+        metrics=['accuracy', metrics_module.BinaryAccuracy()],
+        optimizer=RMSPropOptimizer(learning_rate=0.001))
+
+    np.random.seed(123)
+    x = np.random.randint(10, size=(100, 4)).astype(np.float32)
+    y = np.random.randint(2, size=(100, 1)).astype(np.float32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
+    dataset = dataset.batch(10)
+    iterator = dataset.make_one_shot_iterator()
+    outs = model.evaluate(iterator, steps=10)
+    self.assertEqual(np.around(outs[1], decimals=1), 0.5)
+    self.assertEqual(np.around(outs[2], decimals=1), 0.5)
+
+    y = np.zeros((100, 1), dtype=np.float32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
+    dataset = dataset.repeat(100)
+    dataset = dataset.batch(10)
+    iterator = dataset.make_one_shot_iterator()
+    outs = model.evaluate(iterator, steps=10)
+    self.assertEqual(outs[1], 0.)
+    self.assertEqual(outs[2], 0.)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index b0f40dcb4f..b2dace84aa 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import copy
 
 import numpy as np
@@ -215,7 +216,7 @@ def iterator_fit_loop(model,
   assert isinstance(inputs, iterator_ops.EagerIterator)
 
   # make sure either x,y or x,y,sample_weights is provided
-  if (not isinstance(inputs.output_shapes, (list, tuple)) or
+  if (not isinstance(inputs.output_shapes, collections.Sequence) or
       len(inputs.output_shapes) not in (2, 3)):
     raise ValueError('Please provide either inputs and targets '
                      'or inputs, targets, and sample_weights')
@@ -340,7 +341,7 @@ def iterator_test_loop(model, inputs, steps, verbose=0):
   """
   assert isinstance(inputs, iterator_ops.EagerIterator)
   # make sure either x,y or x,y,sample_weights is provided
-  if (not isinstance(inputs.output_shapes, (list, tuple)) or
+  if (not isinstance(inputs.output_shapes, collections.Sequence) or
       len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3):
     raise ValueError('Please provide either inputs and targets'
                      'or inputs, targets, and sample_weights')
@@ -461,7 +462,7 @@ def iterator_predict_loop(model, inputs, steps, verbose=0):
   """
   assert isinstance(inputs, iterator_ops.EagerIterator)
   if not isinstance(inputs.output_shapes,
-                    (list, tuple)) or len(inputs.output_shapes) > 3:
+                    collections.Sequence) or len(inputs.output_shapes) > 3:
     raise ValueError(
         'Please provide data as a list or tuple of 1, 2, or 3 elements '
         ' - `(input)`, or `(input, target)`, or `(input, target,'
@@ -583,16 +584,17 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
   Returns:
       total loss and the loss associated with each output.
   """
-  if len(inputs) and tensor_util.is_tensor(inputs[0]):
-    inputs = training_utils.cast_if_floating_dtype(inputs)
-    targets = training_utils.cast_if_floating_dtype(targets)
-  else:
-    inputs = [
-        ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs
-    ]
-    targets = [
-        ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets
-    ]
+  if isinstance(inputs, collections.Sequence):
+    if len(inputs) and tensor_util.is_tensor(inputs[0]):
+      inputs = training_utils.cast_if_floating_dtype(inputs)
+      targets = training_utils.cast_if_floating_dtype(targets)
+    else:
+      inputs = [
+          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs
+      ]
+      targets = [
+          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets
+      ]
   if sample_weights:
     sample_weights = [
         ops.convert_to_tensor(val, dtype=backend.floatx())
@@ -630,16 +632,17 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
   Returns:
       total loss, loss and metrics associated with each output.
   """
-  if len(inputs) and tensor_util.is_tensor(inputs[0]):
-    inputs = training_utils.cast_if_floating_dtype(inputs)
-    targets = training_utils.cast_if_floating_dtype(targets)
-  else:
-    inputs = [
-        ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs
-    ]
-    targets = [
-        ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets
-    ]
+  if isinstance(inputs, collections.Sequence):
+    if len(inputs) and tensor_util.is_tensor(inputs[0]):
+      inputs = training_utils.cast_if_floating_dtype(inputs)
+      targets = training_utils.cast_if_floating_dtype(targets)
+    else:
+      inputs = [
+          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs
+      ]
+      targets = [
+          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets
+      ]
   if sample_weights:
     sample_weights = [
         ops.convert_to_tensor(val, dtype=backend.floatx())
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index bd25af9ec6..d769143106 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.platform import test
@@ -144,16 +145,19 @@ class TrainingTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError, r'specify .* `steps_per_epoch`'):
       model.fit(iterator, epochs=1, verbose=0)
+    if not context.executing_eagerly():
+      # In eager execution, `keras.backend.zeros` returns value tensors
+      # which can be used for validation without a `validation_steps` argument.
+      with self.assertRaisesRegexp(
+          ValueError, r'provide either `batch_size` or `validation_steps`'):
+        model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
+                  validation_data=(x, y))
     with self.assertRaisesRegexp(
-        ValueError, r'provide either `batch_size` or `validation_steps`'):
-      model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
-                validation_data=(x, y))
-    with self.assertRaisesRegexp(
-        ValueError, r'must specify the number of steps'):
+        ValueError, r'specify the number of steps'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=validation_dataset)
     with self.assertRaisesRegexp(
-        ValueError, r'must specify the number of steps'):
+        ValueError, r'specify the number of steps'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=validation_iterator)
 
@@ -170,13 +174,18 @@ class TrainingTest(test.TestCase):
     x = np.random.random((10, 3))
     y = np.random.random((10, 4))
 
-    def iterator():
+    def numpy_iterator():
       while True:
         yield x, y
 
-    model.fit_generator(iterator(), steps_per_epoch=3, epochs=1)
-    model.evaluate_generator(iterator(), steps=3)
-    out = model.predict_generator(iterator(), steps=3)
+    model.fit_generator(numpy_iterator(), steps_per_epoch=3, epochs=1)
+    model.evaluate_generator(numpy_iterator(), steps=3)
+
+    def inference_numpy_iterator():
+      while True:
+        yield x
+
+    out = model.predict_generator(inference_numpy_iterator(), steps=3)
     self.assertEqual(out.shape, (30, 4))
 
 
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 21f2b9dfe3..45247a2751 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -21,12 +21,12 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
+from tensorflow.python.framework import errors
 from tensorflow.python.keras import callbacks as cbks
-from tensorflow.python.keras.utils.data_utils import GeneratorEnqueuer
-from tensorflow.python.keras.utils.data_utils import iter_sequence_infinite
-from tensorflow.python.keras.utils.data_utils import OrderedEnqueuer
-from tensorflow.python.keras.utils.data_utils import Sequence
+from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.platform import tf_logging as logging
 
@@ -54,7 +54,7 @@ def fit_generator(model,
     if do_validation:
       model._make_test_function()
 
-  is_sequence = isinstance(generator, Sequence)
+  is_sequence = isinstance(generator, data_utils.Sequence)
   if not is_sequence and use_multiprocessing and workers > 1:
     logging.warning(
         UserWarning('Using a generator with `use_multiprocessing=True`'
@@ -65,23 +65,16 @@ def fit_generator(model,
     if is_sequence:
       steps_per_epoch = len(generator)
     else:
-      raise ValueError('`steps_per_epoch=None` is only valid for a'
-                       ' generator based on the `keras.utils.Sequence`'
-                       ' class. Please specify `steps_per_epoch` or use'
-                       ' the `keras.utils.Sequence` class.')
-
-  # python 2 has 'next', 3 has '__next__'
-  # avoid any explicit version checks
-  val_gen = (
-      hasattr(validation_data, 'next') or
-      hasattr(validation_data, '__next__') or
-      isinstance(validation_data, Sequence))
-  if (val_gen and not isinstance(validation_data, Sequence) and
+      raise ValueError('Please specify the `steps_per_epoch` argument.')
+
+  if (isinstance(validation_data, dataset_ops.Dataset) and
+      context.executing_eagerly()):
+    validation_data = validation_data.make_one_shot_iterator()
+  val_gen = (data_utils.is_generator_or_sequence(validation_data) or
+             isinstance(validation_data, iterator_ops.EagerIterator))
+  if (val_gen and not isinstance(validation_data, data_utils.Sequence) and
       not validation_steps):
-    raise ValueError('`validation_steps=None` is only valid for a'
-                     ' generator based on the `keras.utils.Sequence`'
-                     ' class. Please specify `validation_steps` or use'
-                     ' the `keras.utils.Sequence` class.')
+    raise ValueError('Please specify the `validation_steps` argument.')
 
   enqueuer = None
   val_enqueuer = None
@@ -117,19 +110,19 @@ def fit_generator(model,
 
     if workers > 0:
       if is_sequence:
-        enqueuer = OrderedEnqueuer(
+        enqueuer = data_utils.OrderedEnqueuer(
             generator,
             use_multiprocessing=use_multiprocessing,
             shuffle=shuffle)
       else:
-        enqueuer = GeneratorEnqueuer(
+        enqueuer = data_utils.GeneratorEnqueuer(
             generator,
             use_multiprocessing=use_multiprocessing)
       enqueuer.start(workers=workers, max_queue_size=max_queue_size)
       output_generator = enqueuer.get()
     else:
       if is_sequence:
-        output_generator = iter_sequence_infinite(generator)
+        output_generator = data_utils.iter_sequence_infinite(generator)
       else:
         output_generator = generator
 
@@ -144,7 +137,6 @@ def fit_generator(model,
       batch_index = 0
       while steps_done < steps_per_epoch:
         generator_output = next(output_generator)
-
         if not hasattr(generator_output, '__len__'):
           raise ValueError('Output of generator should be '
                            'a tuple `(x, y, sample_weight)` '
@@ -167,8 +159,8 @@ def fit_generator(model,
           batch_size = list(x.values())[0].shape[0]
         else:
           batch_size = x.shape[0]
-        batch_logs['batch'] = batch_index
-        batch_logs['size'] = batch_size
+        batch_logs['batch'] = int(batch_index)
+        batch_logs['size'] = int(batch_size)
         callbacks.on_batch_begin(batch_index, batch_logs)
 
         outs = model.train_on_batch(
@@ -217,6 +209,13 @@ def fit_generator(model,
       if callbacks.model.stop_training:
         break
 
+  except (errors.OutOfRangeError, StopIteration):
+    logging.warning(
+        'Your dataset iterator ran out of data interrupting testing. '
+        'Make sure that your dataset can generate at least `steps_per_epoch` '
+        'batches (in this case, %d batches). You may need to use the '
+        'repeat() function when building your dataset.', steps_per_epoch)
+
   finally:
     try:
       if enqueuer is not None:
@@ -247,7 +246,7 @@ def evaluate_generator(model,
   steps_done = 0
   all_outs = []
   batch_sizes = []
-  is_sequence = isinstance(generator, Sequence)
+  is_sequence = isinstance(generator, data_utils.Sequence)
   if not is_sequence and use_multiprocessing and workers > 1:
     logging.warning(
         UserWarning('Using a generator with `use_multiprocessing=True`'
@@ -258,26 +257,23 @@ def evaluate_generator(model,
     if is_sequence:
       steps = len(generator)
     else:
-      raise ValueError('`steps=None` is only valid for a generator'
-                       ' based on the `keras.utils.Sequence` class.'
-                       ' Please specify `steps` or use the'
-                       ' `keras.utils.Sequence` class.')
+      raise ValueError('Please specify the `steps` argument.')
   enqueuer = None
 
   try:
     if workers > 0:
       if is_sequence:
-        enqueuer = OrderedEnqueuer(
+        enqueuer = data_utils.OrderedEnqueuer(
             generator, use_multiprocessing=use_multiprocessing)
       else:
-        enqueuer = GeneratorEnqueuer(
+        enqueuer = data_utils.GeneratorEnqueuer(
             generator,
             use_multiprocessing=use_multiprocessing)
       enqueuer.start(workers=workers, max_queue_size=max_queue_size)
       output_generator = enqueuer.get()
     else:
       if is_sequence:
-        output_generator = iter_sequence_infinite(generator)
+        output_generator = data_utils.iter_sequence_infinite(generator)
       else:
         output_generator = generator
 
@@ -302,11 +298,11 @@ def evaluate_generator(model,
       outs = model.test_on_batch(x, y, sample_weight=sample_weight)
 
       if isinstance(x, list):
-        batch_size = x[0].shape[0]
+        batch_size = int(x[0].shape[0])
       elif isinstance(x, dict):
-        batch_size = list(x.values())[0].shape[0]
+        batch_size = int(list(x.values())[0].shape[0])
       else:
-        batch_size = x.shape[0]
+        batch_size = int(x.shape[0])
       if batch_size == 0:
         raise ValueError('Received an empty batch. '
                          'Batches should at least contain one item.')
@@ -317,6 +313,13 @@ def evaluate_generator(model,
       if verbose == 1:
         progbar.update(steps_done)
 
+  except (errors.OutOfRangeError, StopIteration):
+    logging.warning(
+        'Your dataset iterator ran out of data interrupting testing. '
+        'Make sure that your dataset can generate at least `steps` '
+        'batches (in this case, %d batches). You may need to use the '
+        'repeat() function when building your dataset.', steps)
+
   finally:
     if enqueuer is not None:
       enqueuer.stop()
@@ -346,7 +349,7 @@ def predict_generator(model,
 
   steps_done = 0
   all_outs = []
-  is_sequence = isinstance(generator, Sequence)
+  is_sequence = isinstance(generator, data_utils.Sequence)
   if not is_sequence and use_multiprocessing and workers > 1:
     logging.warning(
         UserWarning('Using a generator with `use_multiprocessing=True`'
@@ -357,26 +360,23 @@ def predict_generator(model,
     if is_sequence:
       steps = len(generator)
     else:
-      raise ValueError('`steps=None` is only valid for a generator'
-                       ' based on the `keras.utils.Sequence` class.'
-                       ' Please specify `steps` or use the'
-                       ' `keras.utils.Sequence` class.')
+      raise ValueError('Please specify the `steps` argument.')
   enqueuer = None
 
   try:
     if workers > 0:
       if is_sequence:
-        enqueuer = OrderedEnqueuer(
+        enqueuer = data_utils.OrderedEnqueuer(
             generator, use_multiprocessing=use_multiprocessing)
       else:
-        enqueuer = GeneratorEnqueuer(
+        enqueuer = data_utils.GeneratorEnqueuer(
             generator,
             use_multiprocessing=use_multiprocessing)
       enqueuer.start(workers=workers, max_queue_size=max_queue_size)
       output_generator = enqueuer.get()
     else:
       if is_sequence:
-        output_generator = iter_sequence_infinite(generator)
+        output_generator = data_utils.iter_sequence_infinite(generator)
       else:
         output_generator = generator
 
@@ -415,6 +415,13 @@ def predict_generator(model,
       if verbose == 1:
         progbar.update(steps_done)
 
+  except (errors.OutOfRangeError, StopIteration):
+    logging.warning(
+        'Your dataset iterator ran out of data interrupting testing. '
+        'Make sure that your dataset can generate at least `steps` '
+        'batches (in this case, %d batches). You may need to use the '
+        'repeat() function when building your dataset.', steps)
+
   finally:
     if enqueuer is not None:
       enqueuer.stop()
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index cd25707f51..1009ef7138 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -1830,257 +1830,6 @@ class TestTrainingWithDataTensors(test.TestCase):
                            [output_a_np, output_b_np])
 
 
-class TestTrainingWithDatasetIterators(test.TestCase):
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_training_and_eval_methods_on_iterators_single_io(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
-
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
-
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-    model.evaluate(iterator, steps=2, verbose=1)
-    model.predict(iterator, steps=2)
-
-    # Test with validation data
-    model.fit(iterator,
-              epochs=1, steps_per_epoch=2, verbose=0,
-              validation_data=iterator, validation_steps=2)
-    # Test with validation split
-    with self.assertRaisesRegexp(
-        ValueError, '`validation_split` argument is not supported '
-        'when input `x` is a dataset or a dataset iterator'):
-      model.fit(iterator,
-                epochs=1, steps_per_epoch=2, verbose=0,
-                validation_split=0.5, validation_steps=2)
-
-    # Test with sample weight.
-    sample_weight = np.random.random((10,))
-    with self.assertRaisesRegexp(
-        ValueError, '`sample_weight` argument is not supported '
-        'when input `x` is a dataset or a dataset iterator'):
-      model.fit(
-          iterator,
-          epochs=1,
-          steps_per_epoch=2,
-          verbose=0,
-          sample_weight=sample_weight)
-
-    # Test invalid usage
-    with self.assertRaisesRegexp(ValueError,
-                                 'you should not specify a target'):
-      model.fit(iterator, iterator,
-                epochs=1, steps_per_epoch=2, verbose=0)
-
-    with self.assertRaisesRegexp(
-        ValueError, 'you should specify the `steps_per_epoch` argument'):
-      model.fit(iterator, epochs=1, verbose=0)
-    with self.assertRaisesRegexp(ValueError,
-                                 'you should specify the `steps` argument'):
-      model.evaluate(iterator, verbose=0)
-    with self.assertRaisesRegexp(ValueError,
-                                 'you should specify the `steps` argument'):
-      model.predict(iterator, verbose=0)
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_get_next_op_created_once(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics)
-
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
-
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-    # Finalize graph to make sure we are not appending another iterator
-    # get_next op in the graph.
-    ops.get_default_graph().finalize()
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_iterators_running_out_of_data(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics)
-
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(2)
-    dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
-
-    with test.mock.patch.object(logging, 'warning') as mock_log:
-      model.fit(iterator, epochs=1, steps_per_epoch=3, verbose=0)
-      self.assertRegexpMatches(
-          str(mock_log.call_args),
-          'dataset iterator ran out of data')
-
-
-class TestTrainingWithDataset(test.TestCase):
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_calling_model_on_same_dataset(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics)
-
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-
-    # Call fit with validation data
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
-              validation_data=dataset, validation_steps=2)
-    # Finalize the graph to make sure new ops aren't added when calling on the
-    # same dataset
-    ops.get_default_graph().finalize()
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
-              validation_data=dataset, validation_steps=2)
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_training_and_eval_methods_on_dataset(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
-
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
-    model.evaluate(dataset, steps=2, verbose=1)
-    model.predict(dataset, steps=2)
-
-    # Test with validation data
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
-              validation_data=dataset, validation_steps=2)
-
-    # Test with validation split
-    with self.assertRaisesRegexp(
-        ValueError, '`validation_split` argument is not supported '
-        'when input `x` is a dataset or a dataset iterator'):
-      model.fit(dataset,
-                epochs=1, steps_per_epoch=2, verbose=0,
-                validation_split=0.5, validation_steps=2)
-
-    # Test with sample weight.
-    sample_weight = np.random.random((10,))
-    with self.assertRaisesRegexp(
-        ValueError, '`sample_weight` argument is not supported '
-        'when input `x` is a dataset or a dataset iterator'):
-      model.fit(
-          dataset,
-          epochs=1,
-          steps_per_epoch=2,
-          verbose=0,
-          sample_weight=sample_weight)
-
-    # Test invalid usage
-    with self.assertRaisesRegexp(ValueError,
-                                 'you should not specify a target'):
-      model.fit(dataset, dataset,
-                epochs=1, steps_per_epoch=2, verbose=0)
-
-    with self.assertRaisesRegexp(
-        ValueError, 'you should specify the `steps_per_epoch` argument'):
-      model.fit(dataset, epochs=1, verbose=0)
-    with self.assertRaisesRegexp(ValueError,
-                                 'you should specify the `steps` argument'):
-      model.evaluate(dataset, verbose=0)
-    with self.assertRaisesRegexp(ValueError,
-                                 'you should specify the `steps` argument'):
-      model.predict(dataset, verbose=0)
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_dataset_with_sample_weights(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
-
-    inputs = np.zeros((10, 3), np.float32)
-    targets = np.zeros((10, 4), np.float32)
-    sample_weights = np.ones((10), np.float32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets,
-                                                      sample_weights))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
-    model.evaluate(dataset, steps=2, verbose=1)
-    model.predict(dataset, steps=2)
-
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_dataset_with_sparse_labels(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'sparse_categorical_crossentropy'
-    model.compile(optimizer, loss)
-
-    inputs = np.zeros((10, 3))
-    targets = np.random.randint(0, 4, size=10, dtype=np.int32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
-
-  def test_dataset_input_shape_validation(self):
-    with self.cached_session():
-      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-      model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse')
-
-      # User forgets to batch the dataset
-      inputs = np.zeros((10, 3))
-      targets = np.zeros((10, 4))
-      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-      dataset = dataset.repeat(100)
-
-      with self.assertRaisesRegexp(
-          ValueError,
-          r'expected (.*?) to have shape \(3,\) but got array with shape \(1,\)'
-      ):
-        model.train_on_batch(dataset)
-
-      # Wrong input shape
-      inputs = np.zeros((10, 5))
-      targets = np.zeros((10, 4))
-      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-      dataset = dataset.repeat(100)
-      dataset = dataset.batch(10)
-
-      with self.assertRaisesRegexp(ValueError,
-                                   r'expected (.*?) to have shape \(3,\)'):
-        model.train_on_batch(dataset)
-
-
 class TestTrainingWithMetrics(test.TestCase):
   """Training tests related to metrics."""
 
@@ -2144,39 +1893,6 @@ class TestTrainingWithMetrics(test.TestCase):
     self.assertEqual(outs[1], 0.)
     self.assertEqual(outs[2], 0.)
 
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_metrics_correctness_with_iterator(self):
-    model = keras.Sequential()
-    model.add(
-        keras.layers.Dense(
-            8, activation='relu', input_dim=4, kernel_initializer='ones'))
-    model.add(
-        keras.layers.Dense(
-            1, activation='sigmoid', kernel_initializer='ones'))
-    model.compile(
-        loss='binary_crossentropy',
-        metrics=['accuracy', metrics_module.BinaryAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=0.001))
-
-    np.random.seed(123)
-    x = np.random.randint(10, size=(100, 4)).astype(np.float32)
-    y = np.random.randint(2, size=(100, 1)).astype(np.float32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
-    dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
-    outs = model.evaluate(iterator, steps=10)
-    self.assertEqual(np.around(outs[1], decimals=1), 0.5)
-    self.assertEqual(np.around(outs[2], decimals=1), 0.5)
-
-    y = np.zeros((100, 1), dtype=np.float32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
-    outs = model.evaluate(iterator, steps=10)
-    self.assertEqual(outs[1], 0.)
-    self.assertEqual(outs[2], 0.)
-
   @tf_test_util.run_in_graph_and_eager_modes
   def test_metrics_correctness_with_weighted_metrics(self):
     np.random.seed(1337)
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index a0d1e5488d..8669daf99e 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -140,7 +140,7 @@ def convert_to_iterator(x=None,
   if isinstance(x, iterator_ops.EagerIterator):
     if steps_per_epoch is None:
       raise ValueError('You must specify the number of steps (number of batches'
-                       'to draw from the iterator).')
+                       ' to draw from the iterator).')
     return x, steps_per_epoch
 
   if not _nested_any(sample_weights, lambda x: x is None):
@@ -155,7 +155,7 @@ def convert_to_iterator(x=None,
   data = _convert_lists_to_tuples(data)
   if steps_per_epoch is None and batch_size is not None:
     num_samples = _get_batch_axis_size(data)
-    steps_per_epoch = int(math.ceil(num_samples / batch_size))
+    steps_per_epoch = int(math.ceil(num_samples / int(batch_size)))
 
   if steps_per_epoch is None:
     alternative_arg_name = (
-- 
GitLab


From 7d0afad9925cd7b00d12db3c4f6f16646e7ef27d Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Mon, 19 Nov 2018 14:18:21 -0800
Subject: [PATCH 0526/1554] Distribution Strategy: Make values test non flaky.

PiperOrigin-RevId: 222137537
---
 tensorflow/contrib/distribute/python/BUILD          | 1 -
 tensorflow/contrib/distribute/python/values_test.py | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index e99b0b4039..2a595e7c87 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -40,7 +40,6 @@ cuda_py_test(
         "//tensorflow/python/estimator:estimator_py",
     ],
     tags = [
-        "no_oss",
         "no_pip",
     ],
 )
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index fe9f688c8a..855b9c29ae 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
@@ -588,7 +589,7 @@ class InputIteratorTestBase(test.TestCase):
 
     evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
 
-    evaluate(iterator.initialize())
+    evaluate(control_flow_ops.group(iterator.initialize()))
 
     for expected_value in expected_values:
       next_element = iterator.get_next()
@@ -601,7 +602,7 @@ class InputIteratorTestBase(test.TestCase):
       evaluate([values.select_device(d, next_element) for d in devices])
 
     # After re-initializing the iterator, should be able to iterate again.
-    evaluate(iterator.initialize())
+    evaluate(control_flow_ops.group(iterator.initialize()))
 
     for expected_value in expected_values:
       next_element = iterator.get_next()
-- 
GitLab


From 7333382d1086a7cc136576347e2402a0e0a1083f Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Mon, 19 Nov 2018 14:20:55 -0800
Subject: [PATCH 0527/1554] Update tf.image.resize and tf.image.transpose_image
 for v2 API.

PiperOrigin-RevId: 222138068
---
 tensorflow/python/ops/image_ops_impl.py       | 38 +++++++++++++++----
 tensorflow/python/ops/image_ops_test.py       |  4 +-
 .../api/golden/v1/tensorflow.image.pbtxt      |  8 ++++
 .../api/golden/v2/tensorflow.image.pbtxt      | 12 +++---
 .../tools/compatibility/tf_upgrade_v2.py      | 17 ---------
 5 files changed, 46 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index c3dea18807..4d1357abbc 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -512,15 +512,20 @@ def _rot90_4D(images, k, name_scope):
   result.set_shape([shape[0], None, None, shape[3]])
   return result
 
-@tf_export('image.transpose_image')
+
+@tf_export(v1=['image.transpose', 'image.transpose_image'])
 def transpose_image(image):
-  """Transpose image(s) by swapping the height and width dimension.
+  return transpose(image=image, name=None)
+
 
-  See also `transpose()`.
+@tf_export('image.transpose', v1=[])
+def transpose(image, name=None):
+  """Transpose image(s) by swapping the height and width dimension.
 
   Args:
     image: 4-D Tensor of shape `[batch, height, width, channels]` or
            3-D Tensor of shape `[height, width, channels]`.
+    name: A name for this operation (optional).
 
   Returns:
     If `image` was 4-D, a 4-D float Tensor of shape
@@ -531,14 +536,14 @@ def transpose_image(image):
   Raises:
     ValueError: if the shape of `image` not supported.
   """
-  with ops.name_scope(None, 'transpose_image', [image]):
+  with ops.name_scope(name, 'transpose', [image]):
     image = ops.convert_to_tensor(image, name='image')
     image = _AssertAtLeast3DImage(image)
     shape = image.get_shape()
     if shape.ndims == 3 or shape.ndims is None:
-      return array_ops.transpose(image, [1, 0, 2], name='transpose_image')
+      return array_ops.transpose(image, [1, 0, 2], name=name)
     elif shape.ndims == 4:
-      return array_ops.transpose(image, [0, 2, 1, 3], name='transpose_image')
+      return array_ops.transpose(image, [0, 2, 1, 3], name=name)
     else:
       raise ValueError('\'image\' must have either 3 or 4 dimensions.')
 
@@ -939,12 +944,28 @@ class ResizeMethod(object):
   AREA = 3
 
 
-@tf_export('image.resize_images')
+@tf_export(v1=['image.resize_images', 'image.resize'])
 def resize_images(images,
                   size,
                   method=ResizeMethod.BILINEAR,
                   align_corners=False,
                   preserve_aspect_ratio=False):
+  return resize_images_v2(
+      images=images,
+      size=size,
+      method=method,
+      align_corners=align_corners,
+      preserve_aspect_ratio=preserve_aspect_ratio,
+      name=None)
+
+
+@tf_export('image.resize', v1=[])
+def resize_images_v2(images,
+                     size,
+                     method=ResizeMethod.BILINEAR,
+                     align_corners=False,
+                     preserve_aspect_ratio=False,
+                     name=None):
   """Resize `images` to `size` using the specified `method`.
 
   Resized images will be distorted if their original aspect ratio is not
@@ -980,6 +1001,7 @@ def resize_images(images,
       then `images` will be resized to a size that fits in `size` while
       preserving the aspect ratio of the original image. Scales up the image if
       `size` is bigger than the current size of the `image`. Defaults to False.
+    name: A name for this operation (optional).
 
   Raises:
     ValueError: if the shape of `images` is incompatible with the
@@ -993,7 +1015,7 @@ def resize_images(images,
     If `images` was 3-D, a 3-D float Tensor of shape
     `[new_height, new_width, channels]`.
   """
-  with ops.name_scope(None, 'resize_images', [images, size]):
+  with ops.name_scope(name, 'resize', [images, size]):
     images = ops.convert_to_tensor(images, name='images')
     if images.get_shape().ndims is None:
       raise ValueError('\'images\' contains no shape.')
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index 7eee70c0e3..de82f4fc27 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -1234,7 +1234,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.transpose_image(x_tf)
-      self.assertTrue(y.op.name.startswith("transpose_image"))
+      self.assertTrue(y.op.name.startswith("transpose"))
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
@@ -2613,7 +2613,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       single_image = array_ops.placeholder(dtypes.float32, shape=[50, 60, 3])
       y = image_ops.resize_images(single_image, [55, 66])
-      self.assertTrue(y.op.name.startswith("resize_images"))
+      self.assertTrue(y.op.name.startswith("resize"))
 
   def _ResizeImageCall(self, x, max_h, max_w, preserve_aspect_ratio,
                        use_tensor_inputs):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
index 0a231f1b65..15d0e099ba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.image.pbtxt
@@ -172,6 +172,10 @@ tf_module {
     name: "random_saturation"
     argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "resize"
+    argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\'], "
+  }
   member_method {
     name: "resize_area"
     argspec: "args=[\'images\', \'size\', \'align_corners\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
@@ -240,6 +244,10 @@ tf_module {
     name: "total_variation"
     argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "transpose"
+    argspec: "args=[\'image\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "transpose_image"
     argspec: "args=[\'image\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index c4cd166ad5..f25fb6541e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -172,6 +172,10 @@ tf_module {
     name: "random_saturation"
     argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "resize"
+    argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], "
+  }
   member_method {
     name: "resize_image_with_crop_or_pad"
     argspec: "args=[\'image\', \'target_height\', \'target_width\'], varargs=None, keywords=None, defaults=None"
@@ -180,10 +184,6 @@ tf_module {
     name: "resize_image_with_pad"
     argspec: "args=[\'image\', \'target_height\', \'target_width\', \'method\'], varargs=None, keywords=None, defaults=[\'0\'], "
   }
-  member_method {
-    name: "resize_images"
-    argspec: "args=[\'images\', \'size\', \'method\', \'align_corners\', \'preserve_aspect_ratio\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\'], "
-  }
   member_method {
     name: "rgb_to_grayscale"
     argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -225,8 +225,8 @@ tf_module {
     argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
-    name: "transpose_image"
-    argspec: "args=[\'image\'], varargs=None, keywords=None, defaults=None"
+    name: "transpose"
+    argspec: "args=[\'image\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "yiq_to_rgb"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index fe74f9a00c..92793f7241 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -55,9 +55,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "t": "x",
             "msg": "message",
         },
-        "tf.sparse.add": [
-            "a", "b", "thresh"
-        ],
         "tf.sparse.split": {
             "split_dim": "axis",
         },
@@ -89,9 +86,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
-        "tf.sparse.concat": [
-            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
-        ]
     }
 
     # Mapping from function to the new name of the function
@@ -150,8 +144,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.multinomial": "tf.random.categorical",
         "tf.random.multinomial": "tf.random.categorical",
         "tf.load_file_system_library": "tf.load_library",
-        "tf.arg_max": "tf.argmax",
-        "tf.arg_min": "tf.argmin",
     })
     # pylint: enable=line-too-long
 
@@ -177,9 +169,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "filter", "padding", "strides", "dilation_rate", "name",
             "data_format"],
         "tf.nn.crelu": ["features", "name", "axis"],
-        "tf.nn.weighted_moments": [
-            "x", "axes", "frequency_weights", "name", "keep_dims"
-        ],
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
@@ -205,12 +194,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
         "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
-        "tf.sparse.add": [
-            "a", "b", "thresh"
-        ],
-        "tf.sparse.concat": [
-            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
-        ],
         "tf.sparse.segment_mean": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
-- 
GitLab


From a603d9a9f705e8ec5dfba628b70de740465c27ae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 14:25:51 -0800
Subject: [PATCH 0528/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 222138921
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 102 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 102 ++++++++++++++++++
 2 files changed, 204 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 803e4b434b..dd1aaf966e 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -75514,6 +75514,108 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "TensorForestCreateTreeVariable"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "tree_config"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeDeserialize"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "tree_config"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeIsInitializedOp"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_initialized"
+    type: DT_BOOL
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreePredict"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "dense_features"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "logits"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "logits_dimension"
+    type: "int"
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeResourceHandleOp"
+  output_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeSerialize"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "tree_config"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeSize"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "tree_size"
+    type: DT_INT32
+  }
+  is_stateful: true
+}
 op {
   name: "TensorListConcatLists"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index f9474d4a16..bc35ce7513 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -35980,6 +35980,108 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "TensorForestCreateTreeVariable"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "tree_config"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeDeserialize"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "tree_config"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeIsInitializedOp"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_initialized"
+    type: DT_BOOL
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreePredict"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "dense_features"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "logits"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "logits_dimension"
+    type: "int"
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeResourceHandleOp"
+  output_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeSerialize"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "tree_config"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "TensorForestTreeSize"
+  input_arg {
+    name: "tree_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "tree_size"
+    type: DT_INT32
+  }
+  is_stateful: true
+}
 op {
   name: "TensorListConcatLists"
   input_arg {
-- 
GitLab


From 75ced89772f6ab5f90c5ee00f08570e472c2bf19 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 19 Nov 2018 14:28:14 -0800
Subject: [PATCH 0529/1554] [tf.data] Decrease the work done in
 model_dataset_test to avoid timeouts.

PiperOrigin-RevId: 222139319
---
 .../kernel_tests/optimization/model_dataset_test.py           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index 35dc9ecbce..f5a8399124 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -73,7 +73,7 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       for _ in range(5):
         sess.run(get_next.op)
-      for _ in range(1000):
+      for _ in range(100):
         start = time.time()
         sess.run(get_next.op)
         end = time.time()
@@ -137,7 +137,7 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       for _ in range(5):
         sess.run(get_next.op)
-      for _ in range(1000):
+      for _ in range(100):
         start = time.time()
         sess.run(get_next.op)
         end = time.time()
-- 
GitLab


From ecf0f5dc13cf0a01ee00958df347f0190ba512b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 14:32:57 -0800
Subject: [PATCH 0530/1554] Remove docs from `tf.distribute.Strategy` from
 methods that we plan to move from that class.

PiperOrigin-RevId: 222140061
---
 tensorflow/python/training/distribute.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 646c352a71..4ef784d121 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -546,6 +546,7 @@ class DistributionStrategy(object):
     return self._extended.update_non_slot(
         colocate_with, fn, args, kwargs, group)
 
+  @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
   def unwrap(self, value):
     """Returns the list of all per-replica values contained in `value`.
 
@@ -564,6 +565,7 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.value_container() instead."""
     return self._extended.value_container(value)
 
+  @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
   def group(self, value, name=None):
     """Shortcut for `tf.group(self.unwrap(value))`."""
     return self._extended._group(value, name)  # pylint: disable=protected-access
@@ -580,6 +582,7 @@ class DistributionStrategy(object):
     return self._extended._num_replicas_in_sync  # pylint: disable=protected-access
 
   @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
   def worker_devices(self):
     """DEPRECATED: use extended.worker_devices instead."""
     return self._extended.worker_devices
-- 
GitLab


From 50bdd7183c408dc04c3c4e13c839780d3d39b160 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Mon, 19 Nov 2018 14:37:06 -0800
Subject: [PATCH 0531/1554] Fix --config=mkl clang compilation error.

PiperOrigin-RevId: 222140708
---
 tensorflow/core/kernels/mkl_conv_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 14d134e2d0..dc6f783623 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -1468,7 +1468,7 @@ class MklQuantizedConv2DSumReluOp
             {"sum", {scale_summand / scale_output}});
       else
         params.post_op_params.push_back(
-            {"sum", {2.0 * scale_summand / scale_output}});
+            {"sum", {2.0f * scale_summand / scale_output}});
     } else {
       params.post_op_params.push_back({"sum", {1.0}});
     }
-- 
GitLab


From 73f15c66008da9922dbc7a3f5c1e3749f9d0ce3e Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 19 Nov 2018 14:48:04 -0800
Subject: [PATCH 0532/1554] [XLA:GPU] Set the requested block size for the
 tiling scheme to 1.

When we refactored the 0-2-1 tiling implementation with a kernel mapping scheme
implementation in cl/221393383, we intended to set the requested block size to
1 to preserve the behavior of the original implementation. We accidently set
the value to 2, which affects the programs with a block size in a normalized
dimension that is not less than 2.

No observable performance impact.

PiperOrigin-RevId: 222142668
---
 tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc | 2 +-
 tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h    | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 5751cfd952..efe335c1c1 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -3677,7 +3677,7 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
   constexpr int kNumRows = 4;
   KernelMappingScheme mapping_scheme(
       reduced_output_dims, /*tile_size_y=*/kWarpSize,
-      /*tile_size_x=*/kWarpSize, /*req_block_sizes=*/{2, 2, 2},
+      /*tile_size_x=*/kWarpSize, /*req_block_sizes=*/{1, 1, 1},
       /*num_threads_y=*/kNumRows,
       /*num_threads_x=*/kWarpSize, &b_);
   TileElementGenerator element_generator;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index c44ffb9e07..06002d57b0 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -90,6 +90,10 @@ class KernelMappingScheme {
   enum { DimZ = 0, DimY, DimX, DimTot };
 
  public:
+  // dims_in_elems: the normalized tensor dimensions.
+  // req_block_sizes: the requested block size in number of tiles for each
+  //   dimension. The actual block size is set to min(req_block_size,
+  //   dims_in_number_of_blocks).
   explicit KernelMappingScheme(absl::Span<const int64> dims_in_elems,
                                int64 tile_size_y, int64 tile_size_x,
                                absl::Span<const int64> req_block_sizes,
-- 
GitLab


From 0d2344c156c2a54e9407e214157a10170f38f7a7 Mon Sep 17 00:00:00 2001
From: Youlong Cheng <ylc@google.com>
Date: Mon, 19 Nov 2018 14:48:13 -0800
Subject: [PATCH 0533/1554] Repharese the docstring of TPUContext.

PiperOrigin-RevId: 222142693
---
 tensorflow/contrib/tpu/python/tpu/tpu_context.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index da6bdf67d6..6724624479 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -41,7 +41,7 @@ _NUM_CORES_TO_COMPUTATION_SHAPE = {
 
 
 class TPUContext(object):
-  """The context of current input_fn invocation."""
+  """A context that holds the current configuration of the TPU computation."""
 
   def __init__(self,
                internal_ctx,
-- 
GitLab


From 1bd82e09594e04835d06737de80737912c47444e Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 19 Nov 2018 14:57:03 -0800
Subject: [PATCH 0534/1554] Simplify some cuda_dnn logic: * De-duplicate the
 accumulator type logic. Currently we have two ways to specify them,
 "AccumulatorType" as template argument, or GetConvComputeType. Removed
 GetConvComputeType. * Simplified GetCudnnDataType.

PiperOrigin-RevId: 222144108
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 239 +++++++++-----------
 tensorflow/stream_executor/cuda/cuda_dnn.h  |  13 +-
 tensorflow/stream_executor/dnn.h            |   4 +
 tensorflow/stream_executor/dnn.proto        |   1 +
 4 files changed, 116 insertions(+), 141 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 3dd5b77ddb..9d691d0bc9 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -132,45 +132,6 @@ string ToString(cudnnStatus_t status) {
   }
 }
 
-template <typename T>
-cudnnDataType_t GetCudnnDataType(
-    dnn::DataLayout = dnn::DataLayout::kBatchDepthYX);
-
-template <>
-cudnnDataType_t GetCudnnDataType<double>(dnn::DataLayout) {
-  return CUDNN_DATA_DOUBLE;
-}
-
-template <>
-cudnnDataType_t GetCudnnDataType<float>(dnn::DataLayout) {
-  return CUDNN_DATA_FLOAT;
-}
-
-template <>
-cudnnDataType_t GetCudnnDataType<Eigen::half>(dnn::DataLayout) {
-  return CUDNN_DATA_HALF;
-}
-
-template <>
-cudnnDataType_t GetCudnnDataType<int8>(dnn::DataLayout layout) {
-  switch (layout) {
-    case dnn::DataLayout::kYXDepthBatch:
-    case dnn::DataLayout::kYXBatchDepth:
-    case dnn::DataLayout::kBatchYXDepth:
-    case dnn::DataLayout::kBatchDepthYX:
-      return CUDNN_DATA_INT8;
-    case dnn::DataLayout::kBatchDepthYX4:
-      return CUDNN_DATA_INT8x4;
-    default:
-      LOG(FATAL) << "Unsupported layout: " << static_cast<int32_t>(layout);
-  }
-}
-
-template <>
-cudnnDataType_t GetCudnnDataType<int32>(dnn::DataLayout) {
-  return CUDNN_DATA_INT32;
-}
-
 // RAII wrapper for all calls to cuDNN with a cuDNN handle argument.
 //
 // See CudnnAccess::GetHandle() for details.
@@ -868,6 +829,12 @@ cudnnDataType_t ToCudnnDataType(
   }
 }
 
+template <typename T>
+cudnnDataType_t GetCudnnDataType(
+    dnn::DataLayout data_layout = dnn::DataLayout::kBatchDepthYX) {
+  return ToCudnnDataType(dnn::ToDataType<T>::value, data_layout);
+}
+
 cudnnRNNInputMode_t ToCudnnRnnInputMode(dnn::RnnInputMode input_mode) {
   switch (input_mode) {
     case dnn::RnnInputMode::kRnnLinearSkip:
@@ -2347,27 +2314,6 @@ struct ConvDoFP32ComputationFP16Input {
   static constexpr bool kDefaultFlag = true;
 };
 
-// A group of helper functions to return the internal compute type for
-// convolutions in cudnn.
-template <typename T>
-cudnnDataType_t GetConvComputeType() {
-  return CUDNN_DATA_FLOAT;
-}
-
-template <>
-cudnnDataType_t GetConvComputeType<Eigen::half>() {
-  if (CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()) {
-    return CUDNN_DATA_FLOAT;
-  } else {
-    return CUDNN_DATA_HALF;
-  }
-}
-
-template <>
-cudnnDataType_t GetConvComputeType<double>() {
-  return CUDNN_DATA_DOUBLE;
-}
-
 // A helper struct to decide whether to use FP32 as the internal compute type
 // for rnn when the input data type is FP16. At present it is turned off,
 // users can explicitly control them through an env-var
@@ -2439,7 +2385,7 @@ port::Status CudnnSupport::DoConvolveImpl(
     const DeviceMemory<T>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
-    ScratchAllocator* scratch_allocator,
+    dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
@@ -2447,7 +2393,7 @@ port::Status CudnnSupport::DoConvolveImpl(
   CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
   CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
   CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  GetConvComputeType<T>());
+                                  ToCudnnDataType(accumulator_type));
 
   auto cudnn = cudnn_->GetHandle(parent_, stream);
   // Alpha is the scaling factor for input.
@@ -2538,8 +2484,7 @@ port::Status CudnnSupport::DoConvolveImpl(
   return port::Status::OK();
 }
 
-template <typename AccumulatorType, typename ElementType, typename BiasType,
-          typename ScaleType>
+template <typename ElementType, typename BiasType, typename ScaleType>
 port::Status CudnnSupport::DoFusedConvolveImpl(
     Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
     const DeviceMemory<ElementType>& conv_input_data,
@@ -2550,7 +2495,8 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
     ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor,
     const DeviceMemory<BiasType>& biases, dnn::ActivationMode activation_mode,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<ElementType>* output_data, ScratchAllocator* scratch_allocator,
+    DeviceMemory<ElementType>* output_data, dnn::DataType accumulator_type,
+    ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   if (activation_mode != dnn::ActivationMode::kRelu &&
@@ -2571,7 +2517,7 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
       GetCudnnDataType<ElementType>(conv_input_descriptor.layout()));
   CudnnTensorDescriptor bias_nd(bias_descriptor, GetCudnnDataType<BiasType>());
   CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  GetCudnnDataType<AccumulatorType>());
+                                  ToCudnnDataType(accumulator_type));
 
   auto cudnn = cudnn_->GetHandle(parent_, stream);
 
@@ -2940,10 +2886,10 @@ bool CudnnSupport::DoConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoConvolveImpl<float>(
-          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output_data,
-          scratch_allocator, algorithm_config, output_profile_result),
+      DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
+                     filter_data, convolution_descriptor, output_descriptor,
+                     output_data, dnn::DataType::kFloat, scratch_allocator,
+                     algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -2958,10 +2904,10 @@ bool CudnnSupport::DoConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoConvolveImpl<double>(
-          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output_data,
-          scratch_allocator, algorithm_config, output_profile_result),
+      DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
+                     filter_data, convolution_descriptor, output_descriptor,
+                     output_data, dnn::DataType::kDouble, scratch_allocator,
+                     algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -2975,11 +2921,15 @@ bool CudnnSupport::DoConvolve(
     DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveImpl<Eigen::half>(
-          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output_data,
-          scratch_allocator, algorithm_config, output_profile_result),
+      DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
+                     filter_data, convolution_descriptor, output_descriptor,
+                     output_data, acc_type, scratch_allocator, algorithm_config,
+                     output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -2997,12 +2947,13 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoFusedConvolveImpl<double>(
-          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
-          filter_descriptor, filter_data, convolution_descriptor,
-          side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output_data, scratch_allocator,
-          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
+                          conv_input_scale, filter_descriptor, filter_data,
+                          convolution_descriptor, side_input_data,
+                          side_input_scale, bias_descriptor, biases,
+                          activation_mode, output_descriptor, output_data,
+                          dnn::DataType::kDouble, scratch_allocator,
+                          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3020,12 +2971,13 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoFusedConvolveImpl<float>(
-          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
-          filter_descriptor, filter_data, convolution_descriptor,
-          side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output_data, scratch_allocator,
-          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
+                          conv_input_scale, filter_descriptor, filter_data,
+                          convolution_descriptor, side_input_data,
+                          side_input_scale, bias_descriptor, biases,
+                          activation_mode, output_descriptor, output_data,
+                          dnn::DataType::kFloat, scratch_allocator,
+                          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3043,13 +2995,17 @@ bool CudnnSupport::DoFusedConvolve(
     DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoFusedConvolveImpl<float>(
+      DoFusedConvolveImpl(
           stream, conv_input_descriptor, conv_input_data, conv_input_scale,
           filter_descriptor, filter_data, convolution_descriptor,
           side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output_data, scratch_allocator,
-          algorithm_config, output_profile_result),
+          activation_mode, output_descriptor, output_data, acc_type,
+          scratch_allocator, algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3075,12 +3031,13 @@ bool CudnnSupport::DoFusedConvolve(
     return false;
   }
   return IsStatusOk(
-      DoFusedConvolveImpl<int32>(
-          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
-          filter_descriptor, filter_data, convolution_descriptor,
-          side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output_data, scratch_allocator,
-          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
+                          conv_input_scale, filter_descriptor, filter_data,
+                          convolution_descriptor, side_input_data,
+                          side_input_scale, bias_descriptor, biases,
+                          activation_mode, output_descriptor, output_data,
+                          dnn::DataType::kInt32, scratch_allocator,
+                          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3114,7 +3071,8 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     DeviceMemory<T> backward_output_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<T>* backward_input_data, ScratchAllocator* scratch_allocator,
+    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+    ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
@@ -3135,7 +3093,7 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
   CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
   CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  GetConvComputeType<T>());
+                                  ToCudnnDataType(accumulator_type));
 
   const bool is_profiling = output_profile_result != nullptr;
 
@@ -3215,11 +3173,11 @@ bool CudnnSupport::DoConvolveBackwardData(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
-                                 output_descriptor, backward_output_data,
-                                 convolution_descriptor, input_descriptor,
-                                 backward_input_data, scratch_allocator,
-                                 algorithm_config, output_profile_result),
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kDouble, scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3235,11 +3193,11 @@ bool CudnnSupport::DoConvolveBackwardData(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
-                                 output_descriptor, backward_output_data,
-                                 convolution_descriptor, input_descriptor,
-                                 backward_input_data, scratch_allocator,
-                                 algorithm_config, output_profile_result),
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kFloat, scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3254,12 +3212,16 @@ bool CudnnSupport::DoConvolveBackwardData(
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
-                                 output_descriptor, backward_output_data,
-                                 convolution_descriptor, input_descriptor,
-                                 backward_input_data, scratch_allocator,
-                                 algorithm_config, output_profile_result),
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, acc_type, scratch_allocator, algorithm_config,
+          output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3271,7 +3233,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     DeviceMemory<T> backward_output_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<T>* backward_filter_data, ScratchAllocator* scratch_allocator,
+    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+    ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
@@ -3292,7 +3255,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
   CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
   CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  GetConvComputeType<T>());
+                                  ToCudnnDataType(accumulator_type));
 
   const bool is_profiling = output_profile_result != nullptr;
 
@@ -3408,11 +3371,12 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(stream, input_descriptor, input_data,
-                                   output_descriptor, backward_output_data,
-                                   convolution_descriptor, filter_descriptor,
-                                   backward_filter_data, scratch_allocator,
-                                   algorithm_config, output_profile_result),
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kDouble,
+
+          scratch_allocator, algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3427,13 +3391,14 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardFilterImpl(stream, input_descriptor, input_data,
-                                   output_descriptor, backward_output_data,
-                                   convolution_descriptor, filter_descriptor,
-                                   backward_filter_data, scratch_allocator,
-                                   algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+  return IsStatusOk(DoConvolveBackwardFilterImpl(
+                        stream, input_descriptor, input_data, output_descriptor,
+                        backward_output_data, convolution_descriptor,
+                        filter_descriptor, backward_filter_data,
+
+                        dnn::DataType::kFloat, scratch_allocator,
+                        algorithm_config, output_profile_result),
+                    /*report_error=*/!output_profile_result);
 }
 
 bool CudnnSupport::DoConvolveBackwardFilter(
@@ -3447,12 +3412,16 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(stream, input_descriptor, input_data,
-                                   output_descriptor, backward_output_data,
-                                   convolution_descriptor, filter_descriptor,
-                                   backward_filter_data, scratch_allocator,
-                                   algorithm_config, output_profile_result),
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, acc_type, scratch_allocator, algorithm_config,
+          output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 74f6f935b8..0641be140d 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -670,12 +670,12 @@ class CudnnSupport : public dnn::DnnSupport {
       const DeviceMemory<T>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T>* output_data, ScratchAllocator* scratch_allocator,
+      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
+      ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
-  template <typename AccumulatorType, typename ElementType, typename BiasType,
-            typename ScaleType>
+  template <typename ElementType, typename BiasType, typename ScaleType>
   port::Status DoFusedConvolveImpl(
       Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
       const DeviceMemory<ElementType>& conv_input_data,
@@ -687,7 +687,7 @@ class CudnnSupport : public dnn::DnnSupport {
       ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor,
       const DeviceMemory<BiasType>& biases, dnn::ActivationMode activation_mode,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<ElementType>* output_data,
+      DeviceMemory<ElementType>* output_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
@@ -700,7 +700,8 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T> backward_output_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<T>* backward_input_data, ScratchAllocator* scratch_allocator,
+      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+      ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
@@ -712,7 +713,7 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T> backward_output_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<T>* backward_filter_data,
+      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index ab88345873..c044a356ef 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -114,6 +114,10 @@ template <>
 struct ToDataType<int8> {
   static constexpr DataType value = DataType::kInt8;
 };
+template <>
+struct ToDataType<int32> {
+  static constexpr DataType value = DataType::kInt32;
+};
 
 // Specifies the types of a RNN model.
 enum class RnnMode {
diff --git a/tensorflow/stream_executor/dnn.proto b/tensorflow/stream_executor/dnn.proto
index 91943cc8e0..56b079c3f5 100644
--- a/tensorflow/stream_executor/dnn.proto
+++ b/tensorflow/stream_executor/dnn.proto
@@ -9,6 +9,7 @@ enum DataType {
   kDouble = 1;
   kHalf = 2;
   kInt8 = 3;
+  kInt32 = 4;
 }
 
 // Describes how a convolution input or output layer's data is formatted.
-- 
GitLab


From bbe996e251ce99daa00e5a8bc69b03e8fb9816bb Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Mon, 19 Nov 2018 14:58:13 -0800
Subject: [PATCH 0535/1554] [TF2XLA] Use xla::GetDimensionSize in reduce_mean.
 In the future when XLA has dynamic shapes, the MeanOp needs to divide the
 real dimension size.

[XLA] Add a HloGetDimensionSizeRewritter pass which converts a
GetDimensionSize op into a Constant.
The lowering of GetDimensionSize is not yet implemented on any platforms. So
add a pass that replaces it with Constant.

PiperOrigin-RevId: 222144256
---
 .../compiler/tf2xla/kernels/reduction_ops.cc  | 20 +++--
 .../compiler/tf2xla/kernels/reduction_ops.h   | 15 ++--
 .../tf2xla/kernels/reduction_ops_common.cc    | 11 ++-
 tensorflow/compiler/xla/service/BUILD         | 40 +++++++++
 tensorflow/compiler/xla/service/cpu/BUILD     |  1 +
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  2 +
 tensorflow/compiler/xla/service/gpu/BUILD     |  1 +
 .../xla/service/gpu/nvptx_compiler.cc         |  2 +
 .../hlo_get_dimension_size_rewriter.cc        | 66 +++++++++++++++
 .../service/hlo_get_dimension_size_rewriter.h | 36 ++++++++
 .../hlo_get_dimension_size_rewriter_test.cc   | 83 +++++++++++++++++++
 11 files changed, 259 insertions(+), 18 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
 create mode 100644 tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
 create mode 100644 tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc

diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
index 107fa62967..132160de70 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
@@ -113,11 +113,21 @@ class MeanOp : public XlaReductionOp {
     xla::Add(scalar_lhs, scalar_rhs);
   }
 
-  xla::XlaOp BuildFinalizer(xla::XlaBuilder* builder,
-                            const xla::XlaOp& reduce_output,
-                            int64 num_elements_reduced) override {
-    auto divisor = XlaHelpers::IntegerLiteral(builder, input_type(0),
-                                              num_elements_reduced);
+  xla::XlaOp BuildFinalizer(
+      xla::XlaBuilder* /*builder*/, const xla::XlaOp& input,
+      const xla::XlaOp& reduce_output,
+      const std::vector<int64>& dimensions_to_reduce) override {
+    if (dimensions_to_reduce.empty()) {
+      return reduce_output;
+    }
+    auto divisor = xla::GetDimensionSize(input, dimensions_to_reduce[0]);
+    for (int i = 1; i < dimensions_to_reduce.size(); i++) {
+      auto size = xla::GetDimensionSize(input, dimensions_to_reduce[i]);
+      divisor = xla::Mul(divisor, size);
+    }
+    xla::PrimitiveType type;
+    TF_CHECK_OK(DataTypeToPrimitiveType(input_type(0), &type));
+    divisor = xla::ConvertElementType(divisor, type);
     return reduce_output / divisor;
   }
 };
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
index 466e79828d..8f1667df5b 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
@@ -48,13 +48,14 @@ class XlaReductionOp : public XlaOpKernel {
                             const xla::XlaOp& scalar_rhs) = 0;
 
   // Applies a transformation to the output of the reduction. The desired
-  // computation should be added to 'builder'. Argument 'reduce_output' is the
-  // output of the reduction. 'num_elements_reduced' is the number of elements
-  // that contributed to the reduction. Returns the transformed reduction
-  // output, Defaults to returning 'reduce_output' unchanged.
-  virtual xla::XlaOp BuildFinalizer(xla::XlaBuilder* builder,
-                                    const xla::XlaOp& reduce_output,
-                                    int64 num_elements_reduced);
+  // computation should be added to 'builder'. Argument 'input' is the original
+  // input of the reduction; 'reduce_output' is the output of the reduction.
+  // Returns the transformed reduction output, Defaults to returning
+  // 'reduce_output' unchanged.
+  virtual xla::XlaOp BuildFinalizer(
+      xla::XlaBuilder* builder, const xla::XlaOp& input,
+      const xla::XlaOp& reduce_output,
+      const std::vector<int64>& dimensions_to_reduce);
 
   void Compile(XlaOpKernelContext* ctx) override;
 
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
index 118f2798d5..e96cabbb85 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
@@ -37,9 +37,10 @@ XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx,
 
 // Unless BuildFinalizer is overridden the reduction has no
 // finalizer.
-xla::XlaOp XlaReductionOp::BuildFinalizer(xla::XlaBuilder* builder,
-                                          const xla::XlaOp& reduce_output,
-                                          int64 num_elements_reduced) {
+xla::XlaOp XlaReductionOp::BuildFinalizer(
+    xla::XlaBuilder* /*builder*/, const xla::XlaOp& /*input*/,
+    const xla::XlaOp& reduce_output,
+    const std::vector<int64>& /*dimensions_to_reduce*/) {
   return reduce_output;
 }
 
@@ -71,7 +72,6 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
 
   absl::InlinedVector<bool, 4> bitmap(data_shape.dims(), false);
   std::vector<int64> xla_axes;
-  int64 num_elements_reduced = 1LL;
   for (int64 i = 0; i < axes_tensor_shape.num_elements(); ++i) {
     int64 index = axes[i];
     OP_REQUIRES(ctx,
@@ -82,7 +82,6 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
     index = (index + data_shape.dims()) % data_shape.dims();
     bitmap[index] = true;
     xla_axes.push_back(index);
-    num_elements_reduced *= data_shape.dim_size(index);
   }
 
   std::vector<int64> final_shape;
@@ -119,7 +118,7 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
 
   auto reduce = xla::Reduce(data, initial, reduction_computation, xla_axes);
   auto deconverted = XlaHelpers::ConvertElementType(b, reduce, input_type(0));
-  auto finalized = BuildFinalizer(b, deconverted, num_elements_reduced);
+  auto finalized = BuildFinalizer(b, data, deconverted, xla_axes);
   auto result = keep_dims_ ? xla::Reshape(finalized, final_shape) : finalized;
   ctx->SetOutput(0, result);
 }
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 8295a9530d..d5e73fe1a8 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2886,6 +2886,46 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "hlo_get_dimension_size_rewriter",
+    srcs = ["hlo_get_dimension_size_rewriter.cc"],
+    hdrs = ["hlo_get_dimension_size_rewriter.h"],
+    deps = [
+        ":hlo",
+        ":hlo_pass",
+        ":shape_inference",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+)
+
+tf_cc_test(
+    name = "hlo_get_dimension_size_rewriter_test",
+    srcs = ["hlo_get_dimension_size_rewriter_test.cc"],
+    deps = [
+        ":hlo",
+        ":hlo_get_dimension_size_rewriter",
+        ":hlo_matchers",
+        ":hlo_parser",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/compiler/xla/tests:test_utils",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "device_memory_allocator",
     srcs = [
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 2763d18121..ce4c2a9cc6 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -96,6 +96,7 @@ cc_library(
         "@com_google_absl//absl/types:span",
         "//tensorflow/compiler/tf2xla:cpu_function_runtime",
         "//tensorflow/compiler/xla/service:map_inliner",
+        "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:scatter_expander",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:protobuf_util",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index a6f357df20..2bf24c15c1 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -76,6 +76,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_cse.h"
 #include "tensorflow/compiler/xla/service/hlo_dce.h"
 #include "tensorflow/compiler/xla/service/hlo_element_type_converter.h"
+#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_memory_scheduler.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -249,6 +250,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
       &pipeline, module->config().debug_options(),
       ReducePrecisionInsertion::PassTiming::BEFORE_OPTIMIZATION);
 
+  pipeline.AddPass<HloGetDimensionSizeRewriter>();
   pipeline.AddPass<MapInliner>();
 
   // TODO(b/65775800): Fix wrong output bug in Call and remove the CallInliner
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index b1629616ac..bfd1b6cb14 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -701,6 +701,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_cse",
         "//tensorflow/compiler/xla/service:hlo_dce",
         "//tensorflow/compiler/xla/service:hlo_element_type_converter",
+        "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/compiler/xla/service:hlo_pass_pipeline",
         "//tensorflow/compiler/xla/service:hlo_proto",
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 902e2b0502..637b861f70 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -67,6 +67,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_cse.h"
 #include "tensorflow/compiler/xla/service/hlo_dce.h"
 #include "tensorflow/compiler/xla/service/hlo_element_type_converter.h"
+#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_pipeline.h"
@@ -142,6 +143,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
                          Compiler* compiler) {
   {
     HloPassPipeline pipeline("optimization");
+    pipeline.AddPass<HloGetDimensionSizeRewriter>();
     pipeline.AddInvariantChecker<HloVerifier>(/*layout_sensitive=*/false,
                                               /*allow_mixed_precision=*/false);
     pipeline.AddPass<GpuHloSupportChecker>();
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
new file mode 100644
index 0000000000..631b3ad735
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
@@ -0,0 +1,66 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h"
+
+#include "absl/algorithm/container.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/shape_inference.h"
+
+namespace xla {
+
+namespace {
+
+StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
+  if (instr->opcode() != HloOpcode::kGetDimensionSize) {
+    return false;
+  }
+  HloComputation* computation = instr->parent();
+
+  TF_ASSIGN_OR_RETURN(auto legal_shape,
+                      ShapeInference::InferGetDimensionSizeShape(
+                          instr->operand(0)->shape(), instr->dimension()));
+  TF_RET_CHECK(ShapeUtil::Equal(instr->shape(), legal_shape));
+  TF_RET_CHECK(ShapeUtil::HasPrimitiveType(instr->shape(), U32));
+  uint32 size = instr->operand(0)->shape().dimensions(instr->dimension());
+  HloInstruction* new_instr = computation->AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(size)));
+  TF_RETURN_IF_ERROR(computation->ReplaceInstruction(instr, new_instr));
+  return true;
+}
+
+}  // namespace
+
+StatusOr<bool> HloGetDimensionSizeRewriter::Run(HloModule* module) {
+  bool changed = false;
+  HloProto proto;
+  *proto.mutable_hlo_module() = module->ToProto();
+  for (auto* computation : module->computations()) {
+    // Replacing instructions will change the instruction list in the
+    // computation. So instead of iterating computation->instructions()
+    // directly, we make a copy of the list to avoid use-after-free.
+    std::vector<HloInstruction*> instrs(computation->instruction_count());
+    absl::c_copy(computation->instructions(), instrs.begin());
+    for (auto instruction : instrs) {
+      TF_ASSIGN_OR_RETURN(bool replaced, ReplaceGetSize(instruction));
+      changed = changed || replaced;
+    }
+  }
+  return changed;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
new file mode 100644
index 0000000000..30f44c23a8
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
@@ -0,0 +1,36 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_GET_DIMENSION_SIZE_REWRITER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_GET_DIMENSION_SIZE_REWRITER_H_
+
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+
+namespace xla {
+
+// Pass to replace a kGetDimensionSize instruction with a constant instruction.
+class HloGetDimensionSizeRewriter : public HloModulePass {
+ public:
+  absl::string_view name() const override {
+    return "hlo-get-dimension-size-rewriter";
+  }
+
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_GET_DIMENSION_SIZE_REWRITER_H_
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc
new file mode 100644
index 0000000000..a86aebdd5b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter_test.cc
@@ -0,0 +1,83 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h"
+
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/literal_test_util.h"
+#include "tensorflow/compiler/xla/tests/test_utils.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace {
+
+namespace op = xla::testing::opcode_matchers;
+
+class HloGetDimensionSizeRewriterTest : public HloTestBase {
+ protected:
+  HloGetDimensionSizeRewriterTest() {}
+};
+
+TEST_F(HloGetDimensionSizeRewriterTest, Ok) {
+  auto module = ParseHloString(R"(
+HloModule _
+ENTRY gds {
+  p = s32[3,4] parameter(0)
+  size0 = u32[] get-dimension-size(p), dimensions={0}
+  size1 = u32[] get-dimension-size(p), dimensions={1}
+  ROOT mul = u32[] multiply(size0, size1)
+})")
+                    .ValueOrDie();
+  HloGetDimensionSizeRewriter pass;
+  EXPECT_TRUE(pass.Run(module.get()).ValueOrDie());
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Multiply(op::Constant(), op::Constant()));
+}
+
+TEST_F(HloGetDimensionSizeRewriterTest, IllegalType) {
+  auto module = ParseHloString(R"(
+HloModule _
+ENTRY gds {
+  p = s32[3]{0} parameter(0)
+  ROOT gds = s64[] get-dimension-size(p), dimensions={0}
+})")
+                    .ValueOrDie();
+  HloGetDimensionSizeRewriter pass;
+  EXPECT_FALSE(pass.Run(module.get()).ok());
+}
+
+TEST_F(HloGetDimensionSizeRewriterTest, IllegalDimension) {
+  auto module = ParseHloString(R"(
+HloModule _
+ENTRY gds {
+  p = f32[2,5] parameter(0)
+  ROOT gds = u32[] get-dimension-size(p), dimensions={2}
+})")
+                    .ValueOrDie();
+  HloGetDimensionSizeRewriter pass;
+  EXPECT_FALSE(pass.Run(module.get()).ok());
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From dcbc3e1deb3355146ec94950ec665efa7356899a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 15:00:13 -0800
Subject: [PATCH 0536/1554] Write AssetFileDef to Metagraph's asset_file_def
 field.

PiperOrigin-RevId: 222144547
---
 tensorflow/cc/saved_model/loader.cc           |   9 +
 tensorflow/python/saved_model/builder.py      |   1 +
 tensorflow/python/saved_model/builder_impl.py | 250 ++++++++++---
 tensorflow/python/saved_model/loader_impl.py  |  31 +-
 .../python/saved_model/saved_model_test.py    | 337 ++++++++++--------
 .../v1/tensorflow.saved_model.-builder.pbtxt  |   1 +
 ...d_model.builder.-saved-model-builder.pbtxt |   1 +
 7 files changed, 438 insertions(+), 192 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index c6abe2f41b..ec116f68cf 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -193,6 +193,15 @@ Status RunRestore(const RunOptions& run_options, const string& export_dir,
 
 Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
                         std::vector<AssetFileDef>* asset_file_defs) {
+  // With SavedModel v2, we write asset file def into metagraph instead of
+  // collection, so read from metagraph first.
+  if (meta_graph_def.asset_file_def_size() > 0) {
+    for (const auto& asset : meta_graph_def.asset_file_def()) {
+      asset_file_defs->push_back(asset);
+    }
+    return Status::OK();
+  }
+  // Fall back to read from collection to be backward compatible with v1.
   const auto& collection_def_map = meta_graph_def.collection_def();
   const auto assets_it = collection_def_map.find(kSavedModelAssetsKey);
   if (assets_it == collection_def_map.end()) {
diff --git a/tensorflow/python/saved_model/builder.py b/tensorflow/python/saved_model/builder.py
index be49c70c60..b929934eeb 100644
--- a/tensorflow/python/saved_model/builder.py
+++ b/tensorflow/python/saved_model/builder.py
@@ -24,5 +24,6 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import
+from tensorflow.python.saved_model.builder_impl import _SavedModelBuilder
 from tensorflow.python.saved_model.builder_impl import SavedModelBuilder
 # pylint: enable=unused-import
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index 4f68f7c5ae..ce7641cd98 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 import os
 
 from google.protobuf.any_pb2 import Any
@@ -39,8 +40,7 @@ from tensorflow.python.util.deprecation import deprecated_args
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export(v1=["saved_model.Builder", "saved_model.builder.SavedModelBuilder"])
-class SavedModelBuilder(object):
+class _SavedModelBuilder(object):
   """Builds the `SavedModel` protocol buffer and saves variables and assets.
 
   The `SavedModelBuilder` class provides functionality to build a `SavedModel`
@@ -68,7 +68,7 @@ class SavedModelBuilder(object):
     builder.add_meta_graph_and_variables(sess,
                                     ["foo-tag"],
                                     signature_def_map=foo_signatures,
-                                    assets_collection=foo_assets)
+                                    assets_list=foo_assets)
   ...
 
   with tf.Session(graph=tf.Graph()) as sess:
@@ -105,19 +105,8 @@ class SavedModelBuilder(object):
     # weights.
     self._has_saved_variables = False
 
-  def _save_and_write_assets(self, assets_collection_to_add=None):
-    """Saves asset to the meta graph and writes asset files to disk.
-
-    Args:
-      assets_collection_to_add: The collection where the asset paths are setup.
-    """
-    asset_filename_map = _maybe_save_assets(assets_collection_to_add)
-
-    # Return if there are no assets to write.
-    if not asset_filename_map:
-      tf_logging.info("No assets to write.")
-      return
-
+  def _copy_assets_to_destination_dir(self, asset_filename_map):
+    """Copy all assets from source path to destination path."""
     assets_destination_dir = saved_model_utils.get_or_create_assets_dir(
         self._export_dir)
 
@@ -136,6 +125,25 @@ class SavedModelBuilder(object):
     tf_logging.info("Assets written to: %s",
                     compat.as_text(assets_destination_dir))
 
+  def _save_and_write_assets(self, meta_graph_def, assets_list=None):
+    """Saves asset to the meta graph and writes asset files to disk.
+
+    Args:
+      meta_graph_def: The meta graph def to which the assets will be added.
+      assets_list: The list where the asset paths are setup.
+    """
+    # Creates a function that adds assets into the meta graph def.
+    write_fn = functools.partial(_add_asset_to_metagraph, meta_graph_def)
+    asset_filename_map = _maybe_save_assets(write_fn, assets_list)
+
+    # Return if there are no assets to write.
+    if not asset_filename_map:
+      tf_logging.info("No assets to write.")
+      return
+
+    # Copy assets from source path to destination path.
+    self._copy_assets_to_destination_dir(asset_filename_map)
+
   def _maybe_add_main_op(self, main_op):
     """Adds main op to the SavedModel.
 
@@ -252,12 +260,8 @@ class SavedModelBuilder(object):
         for outputs_key in outputs:
           self._validate_tensor_info(outputs[outputs_key])
 
-  def _add_collections(
-      self, assets_collection, main_op, train_op):
+  def _add_collections(self, main_op, train_op):
     """Add asset and op collections to be saved."""
-    # Save asset files and write them to disk, if any.
-    self._save_and_write_assets(assets_collection)
-
     self._maybe_add_main_op(main_op)
 
     self._add_train_op(train_op)
@@ -280,7 +284,7 @@ class SavedModelBuilder(object):
   def add_meta_graph(self,
                      tags,
                      signature_def_map=None,
-                     assets_collection=None,
+                     assets_list=None,
                      legacy_init_op=None,
                      clear_devices=False,
                      main_op=None,
@@ -297,8 +301,8 @@ class SavedModelBuilder(object):
       tags: The set of tags to annotate the meta graph def with.
       signature_def_map: The map of signature defs to be added to the meta graph
           def.
-      assets_collection: Assets collection to be saved with SavedModel. Note
-          that this collection should be a subset of the assets saved as part of
+      assets_list: Assets to be saved with SavedModel. Note
+          that this list should be a subset of the assets saved as part of
           the first meta graph in the SavedModel.
       legacy_init_op: Legacy support for op or group of ops to execute after the
           restore op upon a load. Deprecated; please use main_op instead.
@@ -332,8 +336,8 @@ class SavedModelBuilder(object):
     # Re-mapping to main_op, as treatment is identical regardless.
     main_op = main_op or legacy_init_op
 
-    # Add assets and ops
-    self._add_collections(assets_collection, main_op, None)
+    # Add ops to collection.
+    self._add_collections(main_op=main_op, train_op=None)
 
     saver = self._maybe_create_saver(saver)
 
@@ -347,6 +351,9 @@ class SavedModelBuilder(object):
     meta_graph_def = saver.export_meta_graph(
         clear_devices=clear_devices, strip_default_attrs=strip_default_attrs)
 
+    # Save asset files and write them to disk, if any.
+    self._save_and_write_assets(meta_graph_def, assets_list)
+
     # Tag the meta graph def and add it to the SavedModel.
     self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
 
@@ -357,7 +364,7 @@ class SavedModelBuilder(object):
                                    sess,
                                    tags,
                                    signature_def_map=None,
-                                   assets_collection=None,
+                                   assets_list=None,
                                    legacy_init_op=None,
                                    clear_devices=False,
                                    main_op=None,
@@ -378,7 +385,7 @@ class SavedModelBuilder(object):
       tags: The set of tags with which to save the meta graph.
       signature_def_map: The map of signature def map to add to the meta graph
         def.
-      assets_collection: Assets collection to be saved with SavedModel.
+      assets_list: Assets to be saved with SavedModel.
       legacy_init_op: Legacy support for op or group of ops to execute after the
           restore op upon a load. Deprecated; please use main_op instead.
       clear_devices: Set to true if the device info on the default graph should
@@ -408,8 +415,8 @@ class SavedModelBuilder(object):
     # Re-mapping to main_op, as treatment is identical regardless.
     main_op = main_op or legacy_init_op
 
-    # Add assets and ops
-    self._add_collections(assets_collection, main_op, None)
+    # Add ops to collection.
+    self._add_collections(main_op=main_op, train_op=None)
 
     saved_model_utils.get_or_create_variables_dir(self._export_dir)
     variables_path = saved_model_utils.get_variables_path(self._export_dir)
@@ -434,6 +441,9 @@ class SavedModelBuilder(object):
     meta_graph_def = saver.export_meta_graph(
         clear_devices=clear_devices, strip_default_attrs=strip_default_attrs)
 
+    # Save asset files and write them to disk, if any.
+    self._save_and_write_assets(meta_graph_def, assets_list)
+
     # Tag the meta graph def and add it to the SavedModel.
     self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
 
@@ -471,11 +481,157 @@ class SavedModelBuilder(object):
     return path
 
 
-def _maybe_save_assets(assets_collection_to_add=None):
+@tf_export(v1=["saved_model.Builder", "saved_model.builder.SavedModelBuilder"])  # pylint: disable=missing-docstring
+class SavedModelBuilder(_SavedModelBuilder):
+  __doc__ = _SavedModelBuilder.__doc__.replace("assets_list",
+                                               "assets_collection")
+
+  def __init__(self, export_dir):
+    super(SavedModelBuilder, self).__init__(export_dir=export_dir)
+
+  def _add_collections(self, assets_collection, main_op, train_op):
+    """Add asset and op collections to be saved."""
+    # Save asset files and write them to disk, if any.
+    self._save_and_write_assets(assets_collection)
+
+    self._maybe_add_main_op(main_op)
+
+    self._add_train_op(train_op)
+
+  def _save_and_write_assets(self, assets_collection_to_add=None):
+    """Saves asset to the meta graph and writes asset files to disk.
+
+    Args:
+      assets_collection_to_add: The collection where the asset paths are setup.
+    """
+    # Add assets to the collection with key `constants.ASSETS_KEY`, in the
+    # graph.
+    asset_filename_map = _maybe_save_assets(_add_asset_to_collection,
+                                            assets_collection_to_add)
+
+    # Return if there are no assets to write.
+    if not asset_filename_map:
+      tf_logging.info("No assets to write.")
+      return
+
+    # Copy assets from source path to destination path.
+    self._copy_assets_to_destination_dir(asset_filename_map)
+
+  @deprecated_args(None,
+                   "Pass your op to the equivalent parameter main_op instead.",
+                   "legacy_init_op")
+  def add_meta_graph(self,
+                     tags,
+                     signature_def_map=None,
+                     assets_collection=None,
+                     legacy_init_op=None,
+                     clear_devices=False,
+                     main_op=None,
+                     strip_default_attrs=False,
+                     saver=None):
+    if not self._has_saved_variables:
+      raise AssertionError(
+          "Graph state including variables and assets has not been saved yet. "
+          "Please invoke `add_meta_graph_and_variables()` first.")
+
+    # Validate the signature def map to ensure all included TensorInfos are
+    # properly populated.
+    self._validate_signature_def_map(signature_def_map)
+
+    # legacy_init_op is deprecated, and going away in TF 2.0.
+    # Re-mapping to main_op, as treatment is identical regardless.
+    main_op = main_op or legacy_init_op
+
+    # Add assets and ops
+    self._add_collections(assets_collection, main_op, None)
+
+    saver = self._maybe_create_saver(saver)
+
+    # The graph almost certainly previously contained at least one Saver, and
+    # possibly several (e.g. one for loading a pretrained embedding, and another
+    # for the model weights).  Removing the preexisting ones was the
+    # motivation for the clear_extraneous_savers option, but it turns out that
+    # there are edge cases where that option breaks the graph.  Until that is
+    # resolved, we just leave the option set to False for now.
+    # TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
+    meta_graph_def = saver.export_meta_graph(
+        clear_devices=clear_devices, strip_default_attrs=strip_default_attrs)
+
+    # Tag the meta graph def and add it to the SavedModel.
+    self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
+
+  @deprecated_args(None,
+                   "Pass your op to the equivalent parameter main_op instead.",
+                   "legacy_init_op")
+  def add_meta_graph_and_variables(self,
+                                   sess,
+                                   tags,
+                                   signature_def_map=None,
+                                   assets_collection=None,
+                                   legacy_init_op=None,
+                                   clear_devices=False,
+                                   main_op=None,
+                                   strip_default_attrs=False,
+                                   saver=None):
+    if self._has_saved_variables:
+      raise AssertionError("Graph state including variables and assets has "
+                           "already been saved. Please invoke "
+                           "`add_meta_graph()` instead.")
+
+    # Validate the signature def map to ensure all included TensorInfos are
+    # properly populated.
+    self._validate_signature_def_map(signature_def_map)
+
+    # legacy_init_op is deprecated, and going away in TF 2.0.
+    # Re-mapping to main_op, as treatment is identical regardless.
+    main_op = main_op or legacy_init_op
+
+    # Add assets and ops
+    self._add_collections(assets_collection, main_op, None)
+
+    saved_model_utils.get_or_create_variables_dir(self._export_dir)
+    variables_path = saved_model_utils.get_variables_path(self._export_dir)
+
+    saver = self._maybe_create_saver(saver)
+
+    # Save the variables. Also, disable writing the checkpoint state proto. The
+    # file is not used during SavedModel loading. In addition, since a
+    # SavedModel can be copied or moved, this avoids the checkpoint state to
+    # become outdated.
+    saver.save(sess, variables_path, write_meta_graph=False, write_state=False)
+
+    # Export the meta graph def.
+
+    # The graph almost certainly previously contained at least one Saver, and
+    # possibly several (e.g. one for loading a pretrained embedding, and another
+    # for the model weights).  Removing the preexisting ones was the
+    # motivation for the clear_extraneous_savers option, but it turns out that
+    # there are edge cases where that option breaks the graph.  Until that is
+    # resolved, we just leave the option set to False for now.
+    # TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
+    meta_graph_def = saver.export_meta_graph(
+        clear_devices=clear_devices, strip_default_attrs=strip_default_attrs)
+
+    # Tag the meta graph def and add it to the SavedModel.
+    self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
+
+    # Mark this instance of SavedModel as having saved variables, such that
+    # subsequent attempts to save variables will fail.
+    self._has_saved_variables = True
+
+  add_meta_graph.__doc__ = _SavedModelBuilder.add_meta_graph.__doc__.replace(
+      "assets_list", "assets_collection")
+  add_meta_graph_and_variables.__doc__ = \
+      _SavedModelBuilder.add_meta_graph_and_variables.__doc__.replace(
+          "assets_list", "assets_collection")
+
+
+def _maybe_save_assets(write_fn, assets_to_add=None):
   """Saves assets to the meta graph.
 
   Args:
-    assets_collection_to_add: The collection where the asset paths are setup.
+    write_fn: A function callback that writes asset into meta graph.
+    assets_to_add: The list where the asset paths are setup.
 
   Returns:
     A dict of asset basenames for saving to the original full path to the asset.
@@ -486,14 +642,13 @@ def _maybe_save_assets(assets_collection_to_add=None):
   # Map of target file names to original filenames
   asset_filename_map = {}
 
-  if assets_collection_to_add is None:
+  if assets_to_add is None:
     tf_logging.info("No assets to save.")
     return asset_filename_map
 
-  # Iterate over the supplied asset collection, build the `AssetFile` proto
-  # and add them to the collection with key `constants.ASSETS_KEY`, in the
-  # graph.
-  for asset_tensor in assets_collection_to_add:
+  # Iterate over the supplied assets, build the `AssetFile` proto and add them
+  # to the meta graph.
+  for asset_tensor in assets_to_add:
     asset_source_filepath = _asset_path_from_tensor(asset_tensor)
     if not asset_source_filepath:
       raise ValueError("Invalid asset filepath tensor %s" % asset_tensor)
@@ -501,10 +656,11 @@ def _maybe_save_assets(assets_collection_to_add=None):
     asset_filename = _get_asset_filename_to_add(
         asset_source_filepath, asset_filename_map)
 
-    # Build `AssetFile` proto and add it to the asset collection in the graph.
+    # Call the passed-in function that builds AssetFileDef proto and adds it
+    # to either the collection or asset_file_def field of the meta graph.
     # Note that this should be done even when the file is a duplicate of an
     # already-added file, as the tensor reference should still exist.
-    _add_asset_to_collection(asset_filename, asset_tensor)
+    write_fn(asset_filename, asset_tensor)
 
     # In the cases where we are adding a duplicate, this will result in the
     # last of the filepaths being the one used for copying the file to the
@@ -542,7 +698,7 @@ def _get_asset_filename_to_add(asset_filepath, asset_filename_map):
 
   other_asset_filepath = asset_filename_map[asset_filename]
   if other_asset_filepath == asset_filepath:
-    # This is the same file, stored twice in the collection list. No need
+    # This is the same file, stored twice in the list. No need
     # to make unique.
     return asset_filename
 
@@ -589,6 +745,20 @@ def _asset_path_from_tensor(path_tensor):
   return str_values[0]
 
 
+def _add_asset_to_metagraph(meta_graph_def, asset_filename, asset_tensor):
+  """Builds an asset proto and adds it to the meta graph def.
+
+  Args:
+    meta_graph_def: The meta graph def to which the asset will be added.
+    asset_filename: The filename of the asset to be added.
+    asset_tensor: The asset tensor used to populate the tensor info of the asset
+      proto.
+  """
+  asset_proto = meta_graph_def.asset_file_def.add()
+  asset_proto.filename = asset_filename
+  asset_proto.tensor_info.name = asset_tensor.name
+
+
 def _add_asset_to_collection(asset_filename, asset_tensor):
   """Builds an asset proto and adds it to the asset collection of the graph.
 
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index 601133ce90..f50a07fee4 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -99,22 +99,29 @@ def _get_asset_tensors(export_dir, meta_graph_def_to_load, import_scope=None):
   collection_def = meta_graph_def_to_load.collection_def
 
   asset_tensor_dict = {}
-  if constants.ASSETS_KEY in collection_def:
-    # Location of the assets for SavedModel.
-    assets_directory = os.path.join(
-        compat.as_bytes(export_dir),
-        compat.as_bytes(constants.ASSETS_DIRECTORY))
+  asset_protos = []
+
+  if meta_graph_def_to_load.asset_file_def:
+    asset_protos = meta_graph_def_to_load.asset_file_def
+  elif constants.ASSETS_KEY in collection_def:
     assets_any_proto = collection_def[constants.ASSETS_KEY].any_list.value
-    # Process each asset and add it to the asset tensor dictionary.
     for asset_any_proto in assets_any_proto:
       asset_proto = meta_graph_pb2.AssetFileDef()
       asset_any_proto.Unpack(asset_proto)
-      tensor_name = asset_proto.tensor_info.name
-      if import_scope:
-        tensor_name = "%s/%s" % (import_scope, tensor_name)
-      asset_tensor_dict[tensor_name] = os.path.join(
-          compat.as_bytes(assets_directory),
-          compat.as_bytes(asset_proto.filename))
+      asset_protos.append(asset_proto)
+
+  # Location of the assets for SavedModel.
+  assets_directory = os.path.join(
+      compat.as_bytes(export_dir), compat.as_bytes(constants.ASSETS_DIRECTORY))
+  # Process each asset and add it to the asset tensor dictionary.
+  for asset_proto in asset_protos:
+    tensor_name = asset_proto.tensor_info.name
+    if import_scope:
+      tensor_name = "%s/%s" % (import_scope, tensor_name)
+    asset_tensor_dict[tensor_name] = os.path.join(
+        compat.as_bytes(assets_directory),
+        compat.as_bytes(asset_proto.filename))
+
   return asset_tensor_dict
 
 
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index a04f4dd099..e722b6ceae 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -54,7 +54,7 @@ def tearDownModule():
   file_io.delete_recursively(test.get_temp_dir())
 
 
-class SavedModelTest(test.TestCase):
+class SavedModelTestBase(test.TestCase):
 
   def _get_export_dir(self, label):
     return os.path.join(test.get_temp_dir(), label)
@@ -78,14 +78,16 @@ class SavedModelTest(test.TestCase):
     asset_collection = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)
     return asset_collection
 
-  def _validate_asset_collection(self, export_dir, graph_collection_def,
-                                 expected_asset_file_name,
-                                 expected_asset_file_contents,
-                                 expected_asset_tensor_name,
-                                 asset_id=0):
-    assets_any = graph_collection_def[constants.ASSETS_KEY].any_list.value
-    asset = meta_graph_pb2.AssetFileDef()
-    assets_any[asset_id].Unpack(asset)
+
+class SavedModelTest(SavedModelTestBase):
+
+  def _validate_assets(self,
+                       export_dir,
+                       asset_file_def,
+                       expected_asset_file_name,
+                       expected_asset_file_contents,
+                       expected_asset_tensor_name,
+                       asset_id=0):
     assets_path = os.path.join(
         compat.as_bytes(export_dir),
         compat.as_bytes(constants.ASSETS_DIRECTORY),
@@ -93,8 +95,10 @@ class SavedModelTest(test.TestCase):
     actual_asset_contents = file_io.read_file_to_string(assets_path)
     self.assertEqual(expected_asset_file_contents,
                      compat.as_text(actual_asset_contents))
-    self.assertEqual(expected_asset_file_name, asset.filename)
-    self.assertEqual(expected_asset_tensor_name, asset.tensor_info.name)
+    self.assertEqual(expected_asset_file_name,
+                     asset_file_def[asset_id].filename)
+    self.assertEqual(expected_asset_tensor_name,
+                     asset_file_def[asset_id].tensor_info.name)
 
   def _validate_inputs_tensor_info_fail(self, builder, tensor_info):
     with self.session(graph=ops.Graph()) as sess:
@@ -185,7 +189,7 @@ class SavedModelTest(test.TestCase):
 
   def testVerifySessionGraphUsage(self):
     export_dir = self._get_export_dir("test_verify_session_graph_usage")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
@@ -205,7 +209,7 @@ class SavedModelTest(test.TestCase):
 
   def testSequence(self):
     export_dir = self._get_export_dir("test_sequence")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Expect an assertion error since add_meta_graph_and_variables() should be
     # invoked before any add_meta_graph() calls.
@@ -222,7 +226,7 @@ class SavedModelTest(test.TestCase):
 
   def testTags(self):
     export_dir = self._get_export_dir("test_tags")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with a single variable. SavedModel invoked to:
     # - add with weights.
@@ -311,7 +315,7 @@ class SavedModelTest(test.TestCase):
 
   def testVariables(self):
     export_dir = self._get_export_dir("test_variables")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with two variables. SavedModel invoked to:
     # - add with weights.
@@ -363,7 +367,7 @@ class SavedModelTest(test.TestCase):
 
   def testGraphWithoutVariables(self):
     export_dir = self._get_export_dir("test_graph_has_variables")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with no variables.
     with self.session(graph=ops.Graph()) as sess:
@@ -398,7 +402,7 @@ class SavedModelTest(test.TestCase):
 
   def testNoOverwrite(self):
     export_dir = self._get_export_dir("test_no_overwrite")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with a single variable. SavedModel invoked to:
     # - add with weights.
@@ -417,12 +421,12 @@ class SavedModelTest(test.TestCase):
 
     # An attempt to create another builder with the same export directory should
     # result in an assertion error.
-    self.assertRaises(AssertionError, saved_model_builder.SavedModelBuilder,
+    self.assertRaises(AssertionError, saved_model_builder._SavedModelBuilder,
                       export_dir)
 
   def testSaveAsText(self):
     export_dir = self._get_export_dir("test_astext")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with a single variable. SavedModel invoked to:
     # - add with weights.
@@ -453,7 +457,7 @@ class SavedModelTest(test.TestCase):
 
   def testCollections(self):
     export_dir = self._get_export_dir("test_collections")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with a single variable added to a collection. SavedModel invoked to:
     # - add with weights.
@@ -503,7 +507,7 @@ class SavedModelTest(test.TestCase):
 
   def testSignatureDefs(self):
     export_dir = self._get_export_dir("test_signature_defs")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Graph with a single variable and a single entry in the signature def map.
     # SavedModel is invoked to add with weights.
@@ -563,7 +567,7 @@ class SavedModelTest(test.TestCase):
 
   def testSignatureDefValidationFails(self):
     export_dir = self._get_export_dir("test_signature_def_validation_fail")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     tensor_without_encoding = meta_graph_pb2.TensorInfo()
     tensor_without_encoding.dtype = types_pb2.DT_FLOAT
@@ -585,11 +589,11 @@ class SavedModelTest(test.TestCase):
     tensor_with_name.dtype = types_pb2.DT_FLOAT
 
     export_dir = self._get_export_dir("test_signature_def_validation_name_1")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
     self._validate_inputs_tensor_info_accept(builder, tensor_with_name)
 
     export_dir = self._get_export_dir("test_signature_def_validation_name_2")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
     self._validate_outputs_tensor_info_accept(builder, tensor_with_name)
 
   def testSignatureDefValidationSucceedsWithCoo(self):
@@ -599,16 +603,16 @@ class SavedModelTest(test.TestCase):
     tensor_with_coo.dtype = types_pb2.DT_FLOAT
 
     export_dir = self._get_export_dir("test_signature_def_validation_coo_1")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
     self._validate_inputs_tensor_info_accept(builder, tensor_with_coo)
 
     export_dir = self._get_export_dir("test_signature_def_validation_coo_2")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
     self._validate_outputs_tensor_info_accept(builder, tensor_with_coo)
 
   def testAssets(self):
     export_dir = self._get_export_dir("test_assets")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
@@ -618,21 +622,19 @@ class SavedModelTest(test.TestCase):
           compat.as_bytes(test.get_temp_dir()), compat.as_bytes("ignored.txt"))
       file_io.write_string_to_file(ignored_filepath, "will be ignored")
 
-      asset_collection = self._build_asset_collection("hello42.txt",
-                                                      "foo bar baz",
-                                                      "asset_file_tensor")
+      asset_list = self._build_asset_collection("hello42.txt", "foo bar baz",
+                                                "asset_file_tensor")
 
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
       foo_graph = loader.load(sess, ["foo"], export_dir)
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar baz",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
+                            "foo bar baz", "asset_file_tensor:0")
       ignored_asset_path = os.path.join(
           compat.as_bytes(export_dir),
           compat.as_bytes(constants.ASSETS_DIRECTORY),
@@ -641,64 +643,66 @@ class SavedModelTest(test.TestCase):
 
   def testAssetsNameCollisionDiffFile(self):
     export_dir = self._get_export_dir("test_assets_name_collision_diff_file")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
-      asset_collection = self._build_asset_collection(
-          "hello42.txt", "foo bar bak", "asset_file_tensor",
-          asset_subdir="1")
+      asset_list = self._build_asset_collection(
+          "hello42.txt", "foo bar bak", "asset_file_tensor", asset_subdir="1")
 
-      asset_collection = self._build_asset_collection(
-          "hello42.txt", "foo bar baz", "asset_file_tensor_1",
-          asset_subdir="2")
+      asset_list = self._build_asset_collection(
+          "hello42.txt", "foo bar baz", "asset_file_tensor_1", asset_subdir="2")
 
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
       foo_graph = loader.load(sess, ["foo"], export_dir)
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar bak",
-                                      "asset_file_tensor:0")
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt_1", "foo bar baz",
-                                      "asset_file_tensor_1:0",
-                                      asset_id=1)
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
+                            "foo bar bak", "asset_file_tensor:0")
+      self._validate_assets(
+          export_dir,
+          foo_graph.asset_file_def,
+          "hello42.txt_1",
+          "foo bar baz",
+          "asset_file_tensor_1:0",
+          asset_id=1)
 
   def testAssetsNameCollisionSameFilepath(self):
     export_dir = self._get_export_dir("test_assets_name_collision_same_path")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
-      asset_collection = self._build_asset_collection(
-          "hello42.txt", "foo bar baz", "asset_file_tensor")
+      asset_list = self._build_asset_collection("hello42.txt", "foo bar baz",
+                                                "asset_file_tensor")
 
-      asset_collection = self._build_asset_collection(
-          "hello42.txt", "foo bar baz", "asset_file_tensor_1")
+      asset_list = self._build_asset_collection("hello42.txt", "foo bar baz",
+                                                "asset_file_tensor_1")
 
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
       foo_graph = loader.load(sess, ["foo"], export_dir)
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar baz",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
+                            "foo bar baz", "asset_file_tensor:0")
       # The second tensor should be recorded, but the same.
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar baz",
-                                      "asset_file_tensor_1:0",
-                                      asset_id=1)
+      self._validate_assets(
+          export_dir,
+          foo_graph.asset_file_def,
+          "hello42.txt",
+          "foo bar baz",
+          "asset_file_tensor_1:0",
+          asset_id=1)
       ignored_asset_path = os.path.join(
           compat.as_bytes(export_dir),
           compat.as_bytes(constants.ASSETS_DIRECTORY),
@@ -707,35 +711,35 @@ class SavedModelTest(test.TestCase):
 
   def testAssetsNameCollisionSameFile(self):
     export_dir = self._get_export_dir("test_assets_name_collision_same_file")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
-      asset_collection = self._build_asset_collection(
-          "hello42.txt", "foo bar baz", "asset_file_tensor",
-          asset_subdir="1")
+      asset_list = self._build_asset_collection(
+          "hello42.txt", "foo bar baz", "asset_file_tensor", asset_subdir="1")
 
-      asset_collection = self._build_asset_collection(
-          "hello42.txt", "foo bar baz", "asset_file_tensor_1",
-          asset_subdir="2")
+      asset_list = self._build_asset_collection(
+          "hello42.txt", "foo bar baz", "asset_file_tensor_1", asset_subdir="2")
 
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
       foo_graph = loader.load(sess, ["foo"], export_dir)
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar baz",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
+                            "foo bar baz", "asset_file_tensor:0")
       # The second tensor should be recorded, but the same.
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar baz",
-                                      "asset_file_tensor_1:0",
-                                      asset_id=1)
+      self._validate_assets(
+          export_dir,
+          foo_graph.asset_file_def,
+          "hello42.txt",
+          "foo bar baz",
+          "asset_file_tensor_1:0",
+          asset_id=1)
       ignored_asset_path = os.path.join(
           compat.as_bytes(export_dir),
           compat.as_bytes(constants.ASSETS_DIRECTORY),
@@ -744,19 +748,21 @@ class SavedModelTest(test.TestCase):
 
   def testAssetsNameCollisionManyFiles(self):
     export_dir = self._get_export_dir("test_assets_name_collision_many_files")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
       for i in range(5):
         idx = str(i)
-        asset_collection = self._build_asset_collection(
-            "hello42.txt", "foo bar baz " + idx, "asset_file_tensor_" + idx,
+        asset_list = self._build_asset_collection(
+            "hello42.txt",
+            "foo bar baz " + idx,
+            "asset_file_tensor_" + idx,
             asset_subdir=idx)
 
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
@@ -765,18 +771,20 @@ class SavedModelTest(test.TestCase):
       foo_graph = loader.load(sess, ["foo"], export_dir)
       for i in range(1, 5):
         idx = str(i)
-        self._validate_asset_collection(
-            export_dir, foo_graph.collection_def, "hello42.txt_" + idx,
-            "foo bar baz " + idx, "asset_file_tensor_{}:0".format(idx),
+        self._validate_assets(
+            export_dir,
+            foo_graph.asset_file_def,
+            "hello42.txt_" + idx,
+            "foo bar baz " + idx,
+            "asset_file_tensor_{}:0".format(idx),
             asset_id=i)
 
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "hello42.txt", "foo bar baz 0",
-                                      "asset_file_tensor_0:0")
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
+                            "foo bar baz 0", "asset_file_tensor_0:0")
 
   def testCustomMainOp(self):
     export_dir = self._get_export_dir("test_main_op")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       # Add `v1` and `v2` variables to the graph.
@@ -811,7 +819,7 @@ class SavedModelTest(test.TestCase):
 
   def testLegacyInitOp(self):
     export_dir = self._get_export_dir("test_legacy_init_op")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       # Add `v1` and `v2` variables to the graph.
@@ -855,7 +863,7 @@ class SavedModelTest(test.TestCase):
     self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY)
 
   def _testInitOpsWithNonEmptyCollection(self, export_dir, key):
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     g = ops.Graph()
     with self.session(graph=g) as sess:
@@ -885,7 +893,7 @@ class SavedModelTest(test.TestCase):
 
   def testTrainOp(self):
     export_dir = self._get_export_dir("test_train_op")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       # Add `v1` and `v2` variables to the graph.
@@ -914,7 +922,7 @@ class SavedModelTest(test.TestCase):
 
   def testTrainOpGroup(self):
     export_dir = self._get_export_dir("test_train_op_group")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       # Add `v1` and `v2` variables to the graph.
@@ -943,7 +951,7 @@ class SavedModelTest(test.TestCase):
 
   def testTrainOpAfterVariables(self):
     export_dir = self._get_export_dir("test_train_op_after_variables")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       # Add `v1` and `v2` variables to the graph.
@@ -975,28 +983,28 @@ class SavedModelTest(test.TestCase):
 
   def testMultipleAssets(self):
     export_dir = self._get_export_dir("test_multiple_assets")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
       # Build an asset collection specific to `foo` graph.
-      asset_collection = self._build_asset_collection("foo.txt", "content_foo",
-                                                      "asset_file_tensor")
+      asset_list = self._build_asset_collection("foo.txt", "content_foo",
+                                                "asset_file_tensor")
 
       # Add the asset collection as part of the graph with tag "foo".
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
       # Build an asset collection specific to `bar` graph.
-      asset_collection = self._build_asset_collection("bar.txt", "content_bar",
-                                                      "asset_file_tensor")
+      asset_list = self._build_asset_collection("bar.txt", "content_bar",
+                                                "asset_file_tensor")
 
       # Add the asset collection as part of the graph with tag "bar".
-      builder.add_meta_graph(["bar"], assets_collection=asset_collection)
+      builder.add_meta_graph(["bar"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
@@ -1004,43 +1012,41 @@ class SavedModelTest(test.TestCase):
     # Check assets restored for graph with tag "foo".
     with self.session(graph=ops.Graph()) as sess:
       foo_graph = loader.load(sess, ["foo"], export_dir)
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "foo.txt", "content_foo",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "foo.txt",
+                            "content_foo", "asset_file_tensor:0")
 
     # Check assets restored for graph with tag "bar".
     with self.session(graph=ops.Graph()) as sess:
       bar_graph = loader.load(sess, ["bar"], export_dir)
-      self._validate_asset_collection(export_dir, bar_graph.collection_def,
-                                      "bar.txt", "content_bar",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, bar_graph.asset_file_def, "bar.txt",
+                            "content_bar", "asset_file_tensor:0")
 
   def testDuplicateAssets(self):
     export_dir = self._get_export_dir("test_duplicate_assets")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
       # Build an asset collection with `foo.txt` that has `foo` specific
       # content.
-      asset_collection = self._build_asset_collection("foo.txt", "content_foo",
-                                                      "asset_file_tensor")
+      asset_list = self._build_asset_collection("foo.txt", "content_foo",
+                                                "asset_file_tensor")
 
       # Add the asset collection as part of the graph with tag "foo".
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], assets_collection=asset_collection)
+          sess, ["foo"], assets_list=asset_list)
 
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
 
       # Build an asset collection with `foo.txt` that has `bar` specific
       # content.
-      asset_collection = self._build_asset_collection("foo.txt", "content_bar",
-                                                      "asset_file_tensor")
+      asset_list = self._build_asset_collection("foo.txt", "content_bar",
+                                                "asset_file_tensor")
 
       # Add the asset collection as part of the graph with tag "bar".
-      builder.add_meta_graph(["bar"], assets_collection=asset_collection)
+      builder.add_meta_graph(["bar"], assets_list=asset_list)
 
     # Save the SavedModel to disk.
     builder.save()
@@ -1048,9 +1054,8 @@ class SavedModelTest(test.TestCase):
     # Check assets restored for graph with tag "foo".
     with self.session(graph=ops.Graph()) as sess:
       foo_graph = loader.load(sess, ["foo"], export_dir)
-      self._validate_asset_collection(export_dir, foo_graph.collection_def,
-                                      "foo.txt", "content_foo",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, foo_graph.asset_file_def, "foo.txt",
+                            "content_foo", "asset_file_tensor:0")
 
     # Check assets restored for graph with tag "bar".
     with self.session(graph=ops.Graph()) as sess:
@@ -1059,13 +1064,12 @@ class SavedModelTest(test.TestCase):
       # Validate the assets for `bar` graph. `foo.txt` should contain the
       # original contents corresponding to `foo` graph since an asset with the
       # same name across multiple graphs is only stored the first time
-      self._validate_asset_collection(export_dir, bar_graph.collection_def,
-                                      "foo.txt", "content_foo",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, bar_graph.asset_file_def, "foo.txt",
+                            "content_foo", "asset_file_tensor:0")
 
   def testOp(self):
     export_dir = self._get_export_dir("test_op")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with session.Session(
         graph=ops.Graph(),
@@ -1108,7 +1112,7 @@ class SavedModelTest(test.TestCase):
 
   def testCustomSaveable(self):
     export_dir = self._get_export_dir("custom_saveable")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with session.Session(
         graph=ops.Graph(),
@@ -1137,7 +1141,7 @@ class SavedModelTest(test.TestCase):
 
   def testCustomSaver(self):
     export_dir = self._get_export_dir("test_custom_saver")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
@@ -1159,7 +1163,7 @@ class SavedModelTest(test.TestCase):
 
   def testNoCustomSaver(self):
     export_dir = self._get_export_dir("test_no_custom_saver")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
@@ -1181,7 +1185,7 @@ class SavedModelTest(test.TestCase):
 
   def testMultipleCustomSavers(self):
     export_dir = self._get_export_dir("test_multiple_custom_savers")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
@@ -1211,19 +1215,19 @@ class SavedModelTest(test.TestCase):
 
   def testImportScope(self):
     export_dir = self._get_export_dir("test_scoped_assets")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Build a SavedModel with a variable, an asset, and a constant tensor.
     with self.session(graph=ops.Graph()) as sess:
       self._init_and_validate_variable(sess, "v", 42)
-      asset_collection = self._build_asset_collection("foo.txt", "content_foo",
-                                                      "asset_file_tensor")
+      asset_list = self._build_asset_collection("foo.txt", "content_foo",
+                                                "asset_file_tensor")
       constant_op.constant("constant value", name="constant_tensor_name")
       builder.add_meta_graph_and_variables(
-          sess, ["tag_name"], assets_collection=asset_collection)
+          sess, ["tag_name"], assets_list=asset_list)
 
       # Save the asset file path for later comparison.
-      asset_file_path = asset_collection[0].eval()
+      asset_file_path = asset_list[0].eval()
 
     # Save the SavedModel to disk.
     builder.save()
@@ -1244,16 +1248,14 @@ class SavedModelTest(test.TestCase):
 
       # The loaded asset tensor should be scoped, but the asset file path and
       # contents should be unchanged.
-      asset_collection = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)
-      self.assertEqual(1, len(asset_collection))
-      self.assertEqual(asset_file_path, asset_collection[0].eval())
-      self.assertEqual("scope_name/asset_file_tensor:0",
-                       asset_collection[0].name)
+      asset_list = ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)
+      self.assertEqual(1, len(asset_list))
+      self.assertEqual(asset_file_path, asset_list[0].eval())
+      self.assertEqual("scope_name/asset_file_tensor:0", asset_list[0].name)
       # The static asset data inside graph_proto.collection_def should not be
       # scoped.
-      self._validate_asset_collection(export_dir, graph_proto.collection_def,
-                                      "foo.txt", "content_foo",
-                                      "asset_file_tensor:0")
+      self._validate_assets(export_dir, graph_proto.asset_file_def, "foo.txt",
+                            "content_foo", "asset_file_tensor:0")
 
       # The constant tensor should be scoped, but its contents should be
       # unchanged.
@@ -1264,7 +1266,7 @@ class SavedModelTest(test.TestCase):
 
   def testClearDevices(self):
     export_dir = self._get_export_dir("test_clear_devices")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Specify a device and save a variable.
     ops.reset_default_graph()
@@ -1288,7 +1290,7 @@ class SavedModelTest(test.TestCase):
 
   def testStripDefaultAttrs(self):
     export_dir = self._get_export_dir("test_strip_default_attrs")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Add a graph with two float32 variables and a Complex Op composing them
     # with strip_default_attrs enabled.
@@ -1361,7 +1363,7 @@ class SavedModelTest(test.TestCase):
   def testInconsistentConsumerDefaultAttrs(self):
     export_dir = self._get_export_dir(
         "test_strip_default_attrs_no_consumer_defaults")
-    builder = saved_model_builder.SavedModelBuilder(export_dir)
+    builder = saved_model_builder._SavedModelBuilder(export_dir)
 
     # Add a graph with a single variable and a test op with a defaultless
     # float32 attr, "test_attr".
@@ -1428,5 +1430,60 @@ class SavedModelTest(test.TestCase):
       loader.load(sess, ["foo"], export_dir)
 
 
+class SavedModelV1Test(SavedModelTestBase):
+
+  def _validate_asset_collection(self,
+                                 export_dir,
+                                 graph_collection_def,
+                                 expected_asset_file_name,
+                                 expected_asset_file_contents,
+                                 expected_asset_tensor_name,
+                                 asset_id=0):
+    assets_any = graph_collection_def[constants.ASSETS_KEY].any_list.value
+    asset = meta_graph_pb2.AssetFileDef()
+    assets_any[asset_id].Unpack(asset)
+    assets_path = os.path.join(
+        compat.as_bytes(export_dir),
+        compat.as_bytes(constants.ASSETS_DIRECTORY),
+        compat.as_bytes(expected_asset_file_name))
+    actual_asset_contents = file_io.read_file_to_string(assets_path)
+    self.assertEqual(expected_asset_file_contents,
+                     compat.as_text(actual_asset_contents))
+    self.assertEqual(expected_asset_file_name, asset.filename)
+    self.assertEqual(expected_asset_tensor_name, asset.tensor_info.name)
+
+  def testWritingAssetsToCollection(self):
+    export_dir = self._get_export_dir("test_writing_assets_to_collection")
+    builder = saved_model_builder.SavedModelBuilder(export_dir)
+
+    with self.session(graph=ops.Graph()) as sess:
+      self._init_and_validate_variable(sess, "v", 42)
+
+      # Build an asset list.
+      ignored_filepath = os.path.join(
+          compat.as_bytes(test.get_temp_dir()), compat.as_bytes("ignored.txt"))
+      file_io.write_string_to_file(ignored_filepath, "will be ignored")
+
+      asset_collection = self._build_asset_collection(
+          "hello42.txt", "foo bar baz", "asset_file_tensor")
+
+      builder.add_meta_graph_and_variables(
+          sess, ["foo"], assets_collection=asset_collection)
+
+    # Save the SavedModel to disk.
+    builder.save()
+
+    with self.session(graph=ops.Graph()) as sess:
+      foo_graph = loader.load(sess, ["foo"], export_dir)
+      self._validate_asset_collection(export_dir, foo_graph.collection_def,
+                                      "hello42.txt", "foo bar baz",
+                                      "asset_file_tensor:0")
+      ignored_asset_path = os.path.join(
+          compat.as_bytes(export_dir),
+          compat.as_bytes(constants.ASSETS_DIRECTORY),
+          compat.as_bytes("ignored.txt"))
+      self.assertFalse(file_io.file_exists(ignored_asset_path))
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt
index 67457de070..e4cc0061a9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.-builder.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.saved_model.Builder"
 tf_class {
   is_instance: "<class \'tensorflow.python.saved_model.builder_impl.SavedModelBuilder\'>"
+  is_instance: "<class \'tensorflow.python.saved_model.builder_impl._SavedModelBuilder\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt
index 83bd703540..44860b1172 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.builder.-saved-model-builder.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.saved_model.builder.SavedModelBuilder"
 tf_class {
   is_instance: "<class \'tensorflow.python.saved_model.builder_impl.SavedModelBuilder\'>"
+  is_instance: "<class \'tensorflow.python.saved_model.builder_impl._SavedModelBuilder\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-- 
GitLab


From 6d5b2211f5ffca29a736c9af7abadbf41e4efcd5 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Mon, 19 Nov 2018 15:03:21 -0800
Subject: [PATCH 0537/1554] Replaced 'is 0' and 'is 1' with equality

Small integer caching is an implementation detail of CPython and should
not be relied upon. In general reference equality between small integers
is not guaranteed.

PiperOrigin-RevId: 222145271
---
 .../contrib/learn/python/learn/estimators/linear_test.py      | 2 +-
 .../contrib/linear_optimizer/python/sdca_estimator_test.py    | 2 +-
 tensorflow/python/keras/backend.py                            | 4 ++--
 tensorflow/python/ops/distributions/util.py                   | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py
index 64ee2eef0a..dfc76bfde6 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear_test.py
@@ -1745,7 +1745,7 @@ class LinearRegressorTest(test.TestCase):
           'place_holder':
               constant_op.constant([[0.0]] * num_examples),
       }, constant_op.constant(
-          [[1 if i % 4 is 0 else 0] for i in range(num_examples)])
+          [[1 if i % 4 == 0 else 0] for i in range(num_examples)])
 
     place_holder = feature_column_lib.real_valued_column('place_holder')
     sdca_optimizer = sdca_optimizer_lib.SDCAOptimizer(
diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py
index 6476671882..7a5354222f 100644
--- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py
+++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator_test.py
@@ -524,7 +524,7 @@ class SDCALinearRegressorTest(test.TestCase):
           # LinearClassifier requires at least one column.
           'place_holder':
               constant_op.constant([[0.0]] * num_examples),
-      }, constant_op.constant([[1 if i % 4 is 0 else 0]
+      }, constant_op.constant([[1 if i % 4 == 0 else 0]
                                for i in range(num_examples)])
 
     with self._single_threaded_test_session():
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 969c386fc6..ac896218b6 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3684,13 +3684,13 @@ def in_train_phase(x, alt, training=None):
   if training is None:
     training = learning_phase()
 
-  if training is 1 or training is True:
+  if training == 1 or training is True:
     if callable(x):
       return x()
     else:
       return x
 
-  elif training is 0 or training is False:
+  elif training == 0 or training is False:
     if callable(alt):
       return alt()
     else:
diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py
index 760e7a8a84..24314e8fc9 100644
--- a/tensorflow/python/ops/distributions/util.py
+++ b/tensorflow/python/ops/distributions/util.py
@@ -343,7 +343,7 @@ def embed_check_categorical_event_shape(
     x_dtype = x.dtype.base_dtype
     max_event_size = (_largest_integer_by_dtype(x_dtype)
                       if x_dtype.is_floating else 0)
-    if max_event_size is 0:
+    if max_event_size == 0:
       raise TypeError("Unable to validate size of unrecognized dtype "
                       "({}).".format(x_dtype.name))
     try:
-- 
GitLab


From c01a6d66514886d2d75f9f3d0d46301827d2b7e9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 15:04:02 -0800
Subject: [PATCH 0538/1554] Add UINT8 PReLU op in TFLite

PiperOrigin-RevId: 222145407
---
 tensorflow/lite/kernels/activations.cc        | 65 +++++++++++----
 tensorflow/lite/kernels/activations_test.cc   | 81 ++++++++++++++++---
 .../internal/reference/reference_ops.h        | 47 +++++++++++
 tensorflow/lite/kernels/internal/types.h      |  8 ++
 .../toco/graph_transformations/quantize.cc    |  5 +-
 5 files changed, 181 insertions(+), 25 deletions(-)

diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index bcd2b160f7..82072bccb2 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -45,6 +45,11 @@ struct LogSoftmaxOpData : public OpData {
   int32_t reverse_scaling_right_shift = 0;
 };
 
+struct PreluOpData : public OpData {
+  int32_t output_multiplier = 0;
+  int output_shift = 0;
+};
+
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   // This is a builtin op, so we don't use the contents in 'buffer', if any.
   // Instead, we allocate a new object to carry information from Prepare() to
@@ -57,6 +62,10 @@ void* LogSoftmaxInit(TfLiteContext* context, const char* buffer,
   return new LogSoftmaxOpData;
 }
 
+void* PreluInit(TfLiteContext* context, const char* buffer, size_t length) {
+  return new PreluOpData;
+}
+
 void Free(TfLiteContext* context, void* buffer) {
   delete reinterpret_cast<OpData*>(buffer);
 }
@@ -65,6 +74,10 @@ void LogSoftmaxFree(TfLiteContext* context, void* buffer) {
   delete reinterpret_cast<LogSoftmaxOpData*>(buffer);
 }
 
+void PreluFree(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<PreluOpData*>(buffer);
+}
+
 TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
@@ -253,13 +266,18 @@ TfLiteStatus PreluPrepare(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* input = GetInput(context, node, 0);
   TfLiteTensor* output = GetOutput(context, node, 0);
   const TfLiteTensor* alpha = GetInput(context, node, 1);
+  PreluOpData* data = reinterpret_cast<PreluOpData*>(node->user_data);
 
-  // Currently only Float32 is supported
-  // TODO(ycling): Support other data types.
-  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
-  TF_LITE_ENSURE_EQ(context, alpha->type, kTfLiteFloat32);
+  TF_LITE_ENSURE_EQ(context, input->type, alpha->type);
   output->type = input->type;
 
+  if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt16) {
+    double real_multiplier =
+        input->params.scale * alpha->params.scale / output->params.scale;
+    QuantizeMultiplierSmallerThanOneExp(
+        real_multiplier, &data->output_multiplier, &data->output_shift);
+  }
+
   // PRelu (parameteric Relu) shares the same alpha value on "shared axis".
   // This means it's always required to "broadcast" alpha values in PRelu.
   TfLiteIntArray* output_size = nullptr;
@@ -650,16 +668,35 @@ TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* input = GetInput(context, node, 0);
   const TfLiteTensor* alpha = GetInput(context, node, 1);
   TfLiteTensor* output = GetOutput(context, node, 0);
-  if (input->type != kTfLiteFloat32) {
-    context->ReportError(context, "Only float32 supported currently, got %s.",
-                         TfLiteTypeGetName(input->type));
-    return kTfLiteError;
+  const PreluOpData* data = reinterpret_cast<PreluOpData*>(node->user_data);
+  switch (input->type) {
+    case kTfLiteFloat32: {
+      reference_ops::BroadcastBinaryFunction4DSlow<float, float, float>(
+          GetTensorShape(input), GetTensorData<float>(input),
+          GetTensorShape(alpha), GetTensorData<float>(alpha),
+          GetTensorShape(output), GetTensorData<float>(output),
+          ApplyPrelu<float>);
+      return kTfLiteOk;
+    } break;
+    case kTfLiteUInt8: {
+      PreluParams op_params;
+      op_params.input_offset = -input->params.zero_point;
+      op_params.alpha_offset = -alpha->params.zero_point;
+      op_params.output_offset = output->params.zero_point;
+      op_params.output_multiplier = data->output_multiplier;
+      op_params.output_shift = data->output_shift;
+      reference_ops::BroadcastPrelu4DSlow(
+          op_params, GetTensorShape(input), GetTensorData<uint8_t>(input),
+          GetTensorShape(alpha), GetTensorData<uint8_t>(alpha),
+          GetTensorShape(output), GetTensorData<uint8_t>(output));
+      return kTfLiteOk;
+    } break;
+    default:
+      context->ReportError(context,
+                           "Only float32, uint8 supported currently, got %d.",
+                           TfLiteTypeGetName(input->type));
+      return kTfLiteError;
   }
-  reference_ops::BroadcastBinaryFunction4DSlow<float, float, float>(
-      GetTensorShape(input), GetTensorData<float>(input), GetTensorShape(alpha),
-      GetTensorData<float>(alpha), GetTensorShape(output),
-      GetTensorData<float>(output), ApplyPrelu<float>);
-  return kTfLiteOk;
 }
 
 TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) {
@@ -736,7 +773,7 @@ TfLiteRegistration* Register_LOG_SOFTMAX() {
 }
 
 TfLiteRegistration* Register_PRELU() {
-  static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr,
+  static TfLiteRegistration r = {activations::PreluInit, activations::PreluFree,
                                  activations::PreluPrepare,
                                  activations::PreluEval};
   return &r;
diff --git a/tensorflow/lite/kernels/activations_test.cc b/tensorflow/lite/kernels/activations_test.cc
index 8d705b8f1d..1de3dbc44f 100644
--- a/tensorflow/lite/kernels/activations_test.cc
+++ b/tensorflow/lite/kernels/activations_test.cc
@@ -563,15 +563,29 @@ TEST(QuantizedActivationsOpTest, LogSoftmax) {
               ElementsAreArray({189, 93, 221, 253, 142, 63, 255, 111}));
 }
 
-class PReluOpModel : public SingleOpModel {
+// A base class of PRelu op model. It provides the constructor for
+// FloatPReluOpModel and QuantizedPReluOpModel.
+class BasePReluOpModel : public SingleOpModel {
  public:
-  PReluOpModel(const TensorData& input, const TensorData& alpha) {
+  BasePReluOpModel(const TensorData& input, const TensorData& alpha) {
     input_ = AddInput(input);
     alpha_ = AddInput(alpha);
-    output_ = AddOutput(input);
+    output_ = AddOutput({input.type, input.shape, input.min, input.max});
     SetBuiltinOp(BuiltinOperator_PRELU, BuiltinOptions_NONE, 0);
     BuildInterpreter({GetShape(input_), GetShape(alpha_)});
   }
+
+ protected:
+  int input_;
+  int alpha_;
+  int output_;
+};
+
+// The FloatPReluOpModel class handles float input and output.
+class FloatPReluOpModel : public BasePReluOpModel {
+ public:
+  using BasePReluOpModel::BasePReluOpModel;
+
   void SetInput(std::initializer_list<float> data) {
     PopulateTensor(input_, data);
   }
@@ -579,16 +593,35 @@ class PReluOpModel : public SingleOpModel {
     PopulateTensor(alpha_, data);
   }
   std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+};
 
- protected:
-  int input_;
-  int alpha_;
-  int output_;
+// The QuantizedPReluOpModel class handles quantized input and output.
+class QuantizedPReluOpModel : public BasePReluOpModel {
+ public:
+  using BasePReluOpModel::BasePReluOpModel;
+
+  template <typename T>
+  void SetInput(std::initializer_list<float> data) {
+    QuantizeAndPopulate<T>(input_, data);
+  }
+  template <typename T>
+  void SetAlpha(std::initializer_list<float> data) {
+    QuantizeAndPopulate<T>(alpha_, data);
+  }
+  template <typename T>
+  std::vector<T> GetOutput() {
+    return ExtractVector<T>(output_);
+  }
+  template <typename T>
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<T>(ExtractVector<T>(output_), GetScale(output_),
+                         GetZeroPoint(output_));
+  }
 };
 
 TEST(FloatActivationsOpTest, PRelu) {
-  PReluOpModel m({TensorType_FLOAT32, {1, 2, 2, 3}},
-                 {TensorType_FLOAT32, {1, 1, 3}});
+  FloatPReluOpModel m({TensorType_FLOAT32, {1, 2, 2, 3}},
+                      {TensorType_FLOAT32, {1, 1, 3}});
 
   m.SetInput({
       0.0f, 0.0f, 0.0f,     // Row 1, Column 1
@@ -606,6 +639,36 @@ TEST(FloatActivationsOpTest, PRelu) {
                              }));
 }
 
+TEST(QuantizedActivationsOpTest, PRelu) {
+  const float kMin = -1;
+  const float kMax = 127.f / 128.f;
+  QuantizedPReluOpModel m({TensorType_UINT8, {1, 2, 2, 3}, kMin, kMax},
+                          {TensorType_UINT8, {1, 1, 3}, kMin, kMax});
+  m.SetInput<uint8_t>({
+      0.0f, 0.0f, 0.0f,        // Row 1, Column 1
+      0.5f, 0.5f, 0.5f,        // Row 1, Column 2
+      -1.0f, -1.0f, -1.0f,     // Row 2, Column 1
+      -0.25f, -0.25f, -0.25f,  // Row 1, Column 2
+  });
+  m.SetAlpha<uint8_t>({0.0f, 0.5f, -0.5f});
+  m.Invoke();
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(ArrayFloatNear(
+                  {
+                      0.0f, 0.0f, 0.0f,       // Row 1, Column 1
+                      0.5f, 0.5f, 0.5f,       // Row 1, Column 2
+                      0.0f, -0.5f, 0.5f,      // Row 2, Column 1
+                      0.0f, -0.125f, 0.125f,  // Row 1, Column 2
+                  },
+                  kQuantizedTolerance)));
+  EXPECT_THAT(m.GetOutput<uint8_t>(), ElementsAreArray({
+                                          128, 128, 128,  // Row 1, Column 1
+                                          192, 192, 192,  // Row 1, Column 2
+                                          128, 64, 192,   // Row 2, Column 1
+                                          128, 112, 144,  // Row 1, Column 2
+                                      }));
+}
+
 class LeakyReluOpModel : public SingleOpModel {
  public:
   LeakyReluOpModel(const TensorData& input, float alpha) {
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 7c37dab7e4..fd37865c3e 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -4567,6 +4567,53 @@ inline void ResizeNearestNeighbor(
   }
 }
 
+inline void BroadcastPrelu4DSlow(const PreluParams& params,
+                                 const RuntimeShape& input_shape,
+                                 const uint8* input_data,
+                                 const RuntimeShape& alpha_shape,
+                                 const uint8* alpha_data,
+                                 const RuntimeShape& output_shape,
+                                 uint8* output_data) {
+  TFLITE_DCHECK_LE(input_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_LE(alpha_shape.DimensionsCount(), 4);
+  TFLITE_DCHECK_LE(output_shape.DimensionsCount(), 4);
+  const RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input_shape, alpha_shape, &desc1, &desc2);
+
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          int output_index = Offset(extended_output_shape, b, y, x, c);
+          int input_index = SubscriptToIndex(desc1, b, y, x, c);
+          const int32 input_value =
+              params.input_offset + input_data[input_index];
+          if (input_value >= 0) {
+            output_data[output_index] = input_data[input_index];
+          } else {
+            auto alpha_index = SubscriptToIndex(desc2, b, y, x, c);
+            const int32 alpha_value =
+                params.alpha_offset + alpha_data[alpha_index];
+            const int32 unclamped_output =
+                params.output_offset +
+                MultiplyByQuantizedMultiplierSmallerThanOneExp(
+                    input_value * alpha_value, params.output_multiplier,
+                    params.output_shift);
+            const int32 quantized_min = std::numeric_limits<uint8_t>::min();
+            const int32 quantized_max = std::numeric_limits<uint8_t>::max();
+            const int32 clamped_output = std::min(
+                quantized_max, std::max(quantized_min, unclamped_output));
+            output_data[output_index] = static_cast<uint8>(clamped_output);
+          }
+        }
+      }
+    }
+  }
+}
+
 }  // namespace reference_ops
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/internal/types.h b/tensorflow/lite/kernels/internal/types.h
index dea571039e..859ec8c682 100644
--- a/tensorflow/lite/kernels/internal/types.h
+++ b/tensorflow/lite/kernels/internal/types.h
@@ -904,6 +904,14 @@ struct PadParams {
   ResizingCategory resizing_category;
 };
 
+struct PreluParams {
+  int32 input_offset;
+  int32 alpha_offset;
+  int32 output_offset;
+  int32 output_multiplier;
+  int output_shift;
+};
+
 struct PoolParams {
   FusedActivationFunctionType activation;
   PaddingType padding_type;
diff --git a/tensorflow/lite/toco/graph_transformations/quantize.cc b/tensorflow/lite/toco/graph_transformations/quantize.cc
index e28b7288f0..1146078c30 100644
--- a/tensorflow/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/lite/toco/graph_transformations/quantize.cc
@@ -64,7 +64,8 @@ bool SupportsQuantization(const Operator& op) {
          type == OperatorType::kRelu1 || type == OperatorType::kRelu6 ||
          type == OperatorType::kShape || type == OperatorType::kExpandDims ||
          type == OperatorType::kPack || type == OperatorType::kTopK_V2 ||
-         type == OperatorType::kResizeNearestNeighbor;
+         type == OperatorType::kResizeNearestNeighbor ||
+         type == OperatorType::kPRelu;
 }
 
 // The quantized op allows output arrays of type float using
@@ -360,7 +361,7 @@ bool ChooseQuantizationForOperatorOutput(
       op.type == OperatorType::kSpaceToDepth ||
       op.type == OperatorType::kReshape || op.type == OperatorType::kSplit ||
       op.type == OperatorType::kRelu || op.type == OperatorType::kRelu1 ||
-      op.type == OperatorType::kRelu6) {
+      op.type == OperatorType::kRelu6 || op.type == OperatorType::kPRelu) {
     int data_input_index = 0;
     if (op.type == OperatorType::kSplit) {
       data_input_index = 1;
-- 
GitLab


From 43a573e666addad24c05ea0f67a1efe9486553d1 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 19 Nov 2018 15:04:34 -0800
Subject: [PATCH 0539/1554] Change API for tf.zeros_like/tf.ones_like in TF
 2.0.

PiperOrigin-RevId: 222145504
---
 tensorflow/python/ops/array_ops.py            | 76 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 14 ++++
 3 files changed, 90 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 3ce1a0cf97..2a7989e889 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -1740,7 +1740,7 @@ def zeros(shape, dtype=dtypes.float32, name=None):
   return output
 
 
-@tf_export("zeros_like")
+@tf_export(v1=["zeros_like"])
 def zeros_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to zero.
 
@@ -1767,6 +1767,42 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True):
   Returns:
     A `Tensor` with all elements set to zero.
   """
+  return zeros_like_impl(tensor, dtype, name, optimize)
+
+
+@tf_export("zeros_like", v1=[])
+def zeros_like_v2(
+    input,  # pylint: disable=redefined-builtin
+    dtype=None,
+    name=None):
+  """Creates a tensor with all elements set to zero.
+
+  Given a single tensor (`tensor`), this operation returns a tensor of the
+  same type and shape as `tensor` with all elements set to zero. Optionally,
+  you can use `dtype` to specify a new type for the returned tensor.
+
+  For example:
+
+  ```python
+  tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
+  tf.zeros_like(tensor)  # [[0, 0, 0], [0, 0, 0]]
+  ```
+
+  Args:
+    input: A `Tensor`.
+    dtype: A type for the returned `Tensor`. Must be `float16`, `float32`,
+      `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`,
+      `complex64`, `complex128`, `bool` or `string`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` with all elements set to zero.
+  """
+  return zeros_like_impl(input, dtype, name, optimize=True)
+
+
+def zeros_like_impl(tensor, dtype, name, optimize=True):
+  """Internal implementation for the v1/v2 zeros_like API calls."""
   with ops.name_scope(name, "zeros_like", [tensor]) as name:
     tensor = ops.convert_to_tensor(tensor, name="tensor")
 
@@ -1793,7 +1829,7 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True):
       return gen_array_ops.zeros_like(tensor, name=name)
 
 
-@tf_export("ones_like")
+@tf_export(v1=["ones_like"])
 def ones_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to 1.
 
@@ -1820,6 +1856,42 @@ def ones_like(tensor, dtype=None, name=None, optimize=True):
   Returns:
     A `Tensor` with all elements set to 1.
   """
+  return ones_like_impl(tensor, dtype, name, optimize)
+
+
+@tf_export("ones_like", v1=[])
+def ones_like_v2(
+    input,  # pylint: disable=redefined-builtin
+    dtype=None,
+    name=None):
+  """Creates a tensor with all elements set to zero.
+
+  Given a single tensor (`tensor`), this operation returns a tensor of the
+  same type and shape as `tensor` with all elements set to zero. Optionally,
+  you can use `dtype` to specify a new type for the returned tensor.
+
+  For example:
+
+  ```python
+  tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
+  tf.ones_like(tensor)  # [[1, 1, 1], [1, 1, 1]]
+  ```
+
+  Args:
+    input: A `Tensor`.
+    dtype: A type for the returned `Tensor`. Must be `float16`, `float32`,
+      `float64`, `int8`, `uint8`, `int16`, `uint16`, `int32`, `int64`,
+      `complex64`, `complex128`, `bool` or `string`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` with all elements set to zero.
+  """
+  return ones_like_impl(input, dtype, name, optimize=True)
+
+
+def ones_like_impl(tensor, dtype, name, optimize=True):
+  """Internal implementation for the v1/v2 ones_like API calls."""
   with ops.name_scope(name, "ones_like", [tensor]) as name:
     tensor = ops.convert_to_tensor(tensor, name="tensor")
     ones_shape = shape_internal(tensor, optimize=optimize)
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 175e8820b7..b03c8c212d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -866,7 +866,7 @@ tf_module {
   }
   member_method {
     name: "ones_like"
-    argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
+    argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "pad"
@@ -1170,6 +1170,6 @@ tf_module {
   }
   member_method {
     name: "zeros_like"
-    argspec: "args=[\'tensor\', \'dtype\', \'name\', \'optimize\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
+    argspec: "args=[\'input\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
 }
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 92793f7241..9b2abb9d2a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -72,6 +72,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.conv3d": {
             "filter": "filters"
         },
+        "tf.zeros_like": {
+            "tensor": "input",
+        },
+        "tf.ones_like": {
+            "tensor": "input",
+        },
         "tf.nn.conv3d_transpose": {
             "value": "input",
             "filter": "filters",
@@ -233,6 +239,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         " the data and summarize arguments have been removed."
     )
 
+    tf_01s_like_no_optimize_comment = (
+        "WARNING: tf.zeros_like and tf.ones_like no longer have the optimize "
+        "argument in TF 2.0 or after (also, `tensor' argument is renamed to "
+        "`input')."
+    )
+
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
     self.function_warnings = {
@@ -300,6 +312,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.conv2d_backprop_input":
         "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" "
         "was renamed to \"filters\"",
+        "tf.zeros_like": tf_01s_like_no_optimize_comment,
+        "tf.ones_like": tf_01s_like_no_optimize_comment,
     }
     # Right now we can't have both a rename and a warning.
     self.symbol_renames = {
-- 
GitLab


From 026a687079fabf38fb0bd43348f0130ee1d14c09 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 19 Nov 2018 15:30:41 -0800
Subject: [PATCH 0540/1554] Implement Keras V2 Adagrad optimizer.

Several notes regarding this change:

1) initial_accumulator_value and is kept from the public signature, since
internal search suggested many people use that.

2) included new argument epsilon from Keras, just the same as other optimizers.

PiperOrigin-RevId: 222150112
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  20 +
 .../python/keras/optimizer_v2/adadelta.py     |   2 +-
 .../python/keras/optimizer_v2/adagrad.py      | 144 ++++++++
 .../python/keras/optimizer_v2/adagrad_test.py | 349 ++++++++++++++++++
 tensorflow/python/keras/optimizer_v2/adam.py  |   2 +-
 .../python/keras/optimizer_v2/adam_test.py    |   2 +-
 .../python/keras/optimizer_v2/adamax.py       |   2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |   8 +-
 .../python/keras/optimizer_v2/rmsprop.py      |   2 +-
 9 files changed, 524 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/python/keras/optimizer_v2/adagrad.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/adagrad_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 2296a9562b..7defc7d308 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -14,6 +14,7 @@ py_library(
     name = "optimizer_v2",
     srcs = [
         "adadelta.py",
+        "adagrad.py",
         "adam.py",
         "adamax.py",
         "gradient_descent.py",
@@ -35,6 +36,25 @@ py_library(
     ],
 )
 
+cuda_py_test(
+    name = "adagrad_test",
+    size = "medium",
+    srcs = ["adagrad_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+    shard_count = 4,
+)
+
 cuda_py_test(
     name = "adam_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index 29407484a9..21a3f06f4f 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -54,7 +54,7 @@ class Adadelta(optimizer_v2.OptimizerV2):
   def __init__(self,
                learning_rate=0.001,
                rho=0.95,
-               epsilon=1e-8,
+               epsilon=1e-7,
                name='Adadelta'):
     """Construct a new Adadelta optimizer.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
new file mode 100644
index 0000000000..7d090e8b84
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -0,0 +1,144 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Adagrad for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+
+
+class Adagrad(optimizer_v2.OptimizerV2):
+  r"""Optimizer that implements the Adagrad algorithm.
+
+  Adagrad is an optimizer with parameter-specific learning rates,
+  which are adapted relative to how frequently a parameter gets
+  updated during training. The more updates a parameter receives,
+  the smaller the updates.
+
+  Initialization:
+
+  $$accum_g_0 := initial_accumulator_value$$
+
+  $$t := t + 1$$
+  $$accum_g_t := accum_g_{t-1} + g * g$$
+  $$theta_t := theta_{t-1} - lr * g / (\sqrt{accum_g_t} + \epsilon)$$
+
+  References
+    See [paper]
+      (http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
+    or this
+      [intro](https://ppasupat.github.io/a9online/uploads/proximal_notes.pdf).
+  """
+
+  def __init__(self,
+               learning_rate=0.001,
+               initial_accumulator_value=0.1,
+               epsilon=1e-7,
+               name='Adagrad'):
+    """Construct a new Adagrad optimizer.
+
+    Args:
+      learning_rate: A `Tensor` or a floating point value.  The learning rate.
+      initial_accumulator_value: A floating point value.
+        Starting value for the accumulators, must be positive.
+      epsilon: A floating point value.
+        Starting value for the accumulators, must be positive.
+      name: Optional name prefix for the operations created when applying
+        gradients.  Defaults to "Adagrad".
+
+    Raises:
+      ValueError: If the `initial_accumulator_value` or `epsilon` is invalid.
+
+    @compatibility(eager)
+    When eager execution is enabled, `learning_rate` can be a callable that
+    takes no arguments and returns the actual value to use. This can be useful
+    for changing these values across different invocations of optimizer
+    functions.
+    @end_compatibility
+    """
+    if initial_accumulator_value <= 0.0:
+      raise ValueError('initial_accumulator_value must be positive: %s' %
+                       initial_accumulator_value)
+    if epsilon < 1e-7:
+      raise ValueError('epsilon must be larger than 1e-7: %s' % epsilon)
+    super(Adagrad, self).__init__(name)
+    self._set_hyper('learning_rate', learning_rate)
+    self._initial_accumulator_value = initial_accumulator_value
+    self._set_hyper('epsilon', epsilon)
+
+  def _create_slots(self, var_list):
+    for var in var_list:
+      dtype = var.dtype.base_dtype
+      init = init_ops.constant_initializer(
+          self._initial_accumulator_value, dtype=dtype)
+      self.add_slot(var, 'accumulator', init)
+
+  def _init_constant_op(self, v, dtype):
+    def init():
+      # Use a Tensor instead of initializer if variable does not have
+      # static shape.
+      init_constant = gen_array_ops.fill(array_ops.shape(v),
+                                         self._initial_accumulator_value)
+      return math_ops.cast(init_constant, dtype)
+    return init
+
+  def _resource_apply_dense(self, grad, var):
+    var_dtype = var.dtype.base_dtype
+    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
+    epsilon = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    acc = self.get_slot(var, 'accumulator')
+
+    acc_t = state_ops.assign_add(
+        acc, math_ops.square(grad), use_locking=self._use_locking)
+    var_update = state_ops.assign_sub(
+        var, learning_rate * grad / (math_ops.sqrt(acc_t) + epsilon))
+    return var_update
+
+  def _resource_apply_sparse(self, grad, var, indices):
+
+    def _resource_scatter_add(x, i, v):
+      with ops.control_dependencies(
+          [resource_variable_ops.resource_scatter_add(x.handle, i, v)]):
+        return x.value()
+
+    var_dtype = var.dtype.base_dtype
+    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
+    epsilon = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    acc = self.get_slot(var, 'accumulator')
+
+    acc_t = _resource_scatter_add(acc, indices, math_ops.square(grad))
+    acc_t_slice = array_ops.gather(acc_t, indices)
+    var_update = _resource_scatter_add(
+        var, indices,
+        -learning_rate * grad / (math_ops.sqrt(acc_t_slice) + epsilon))
+    return var_update
+
+  def get_config(self):
+    config = super(Adagrad, self).get_config()
+    config.update({
+        'learning_rate': self._serialize_hyperparameter('learning_rate'),
+        'initial_accumulator_value': self._initial_accumulator_value,
+        'epsilon': self._serialize_hyperparameter('epsilon'),
+    })
+    return config
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
new file mode 100644
index 0000000000..7d0f55c7d7
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -0,0 +1,349 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functional tests for aggregate operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.optimizer_v2 import adagrad
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+def adagrad_update_numpy(param, accum, g_t, lr=0.001, epsilon=1e-7):
+  accum_t = accum + g_t * g_t
+  param_t = param - lr * g_t / (np.sqrt(accum_t) + epsilon)
+  return param_t, accum_t
+
+
+def sparse_adagrad_update_numpy(param,
+                                accum,
+                                gindexs,
+                                gvalues,
+                                lr=0.001,
+                                epsilon=1e-7):
+  accum_t = copy.deepcopy(accum)
+  param_t = copy.deepcopy(param)
+  # first loop accumulates repeated indices if necessary.
+  for i in range(len(gindexs)):
+    gindex = gindexs[i]
+    gvalue = gvalues[i]
+    accum_t[gindex] = accum_t[gindex] + gvalue * gvalue
+  for i in range(len(gindexs)):
+    gindex = gindexs[i]
+    gvalue = gvalues[i]
+    param_t[gindex] = param_t[gindex] - lr * gvalue / (
+        np.sqrt(accum_t[gindex]) + epsilon)
+  return param_t, accum_t
+
+
+class AdagradOptimizerTest(test.TestCase):
+
+  def doTestBasic(self, use_callable_params=False):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = lambda: 3.0
+        if not use_callable_params:
+          learning_rate = learning_rate()
+
+        ada_opt = adagrad.Adagrad(learning_rate)
+
+        accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+
+        if not context.executing_eagerly():
+          ada_update = ada_opt.apply_gradients(
+              zip([grads0, grads1], [var0, var1]))
+          self.evaluate(variables.global_variables_initializer())
+
+        # Fetch params to validate initial values
+        v0_val, v1_val = self.evaluate([var0, var1])
+        self.assertAllClose([1.0, 2.0], v0_val)
+        self.assertAllClose([3.0, 4.0], v1_val)
+
+        # Run 3 steps of adagrad
+        for _ in range(3):
+          if not context.executing_eagerly():
+            self.evaluate(ada_update)
+          else:
+            ada_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+          var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np,
+                                                    grads0_np, 3.0)
+          var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np,
+                                                    grads1_np, 3.0)
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testBasic(self):
+    self.doTestBasic()
+
+  def testBasicCallableParams(self):
+    with context.eager_mode():
+      self.doTestBasic(use_callable_params=True)
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = resource_variable_ops.ResourceVariable(
+            [[1.0, 2.0], [3.0, 4.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = adagrad.Adagrad(1.0).minimize(loss, var_list=[var0])
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType(
+            [[1.0, 2.0], [3.0, 4.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[0, 1], [3, 4]], var0.eval(), atol=0.01)
+
+  def testTensorLearningRate(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = constant_op.constant(3.0)
+        ada_opt = adagrad.Adagrad(learning_rate)
+        ada_update = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        # Run 3 steps of adagrad
+        for _ in range(3):
+          ada_update.run()
+          var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np,
+                                                    grads0_np, learning_rate)
+          var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np,
+                                                    grads1_np, learning_rate)
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
+  def testSparseBasic(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0_np_indices = np.array([0, 2], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np[grads0_np_indices]),
+            constant_op.constant(grads0_np_indices), constant_op.constant([3]))
+        grads1_np_indices = np.array([0, 2], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np[grads1_np_indices]),
+            constant_op.constant(grads1_np_indices), constant_op.constant([3]))
+        learning_rate = 3.0
+        ada_opt = adagrad.Adagrad(learning_rate)
+        ada_update = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 3.0, 4.0], var1.eval())
+
+        accum0_np = np.array([0.1, 0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.1, 0.1, 0.1], dtype=dtype.as_numpy_dtype)
+
+        # Run 3 step of sgd
+        for _ in range(3):
+          ada_update.run()
+
+          var0_np, accum0_np = sparse_adagrad_update_numpy(
+              var0_np, accum0_np, grads0_np_indices,
+              grads0_np[grads0_np_indices], learning_rate)
+          var1_np, accum1_np = sparse_adagrad_update_numpy(
+              var1_np, accum1_np, grads1_np_indices,
+              grads1_np[grads1_np_indices], learning_rate)
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
+  def testSparseRepeatedIndices(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var_np = np.array([[1.0], [2.0]], dtype=dtype.as_numpy_dtype)
+
+        repeated_index_update_var = resource_variable_ops.ResourceVariable(
+            var_np, dtype=dtype)
+        aggregated_update_var = resource_variable_ops.ResourceVariable(
+            var_np, dtype=dtype)
+        grad_repeated_index = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1, 0.1], shape=[2, 1], dtype=dtype),
+            constant_op.constant([1, 1]),
+            constant_op.constant([2, 1]))
+        grad_aggregated = ops.IndexedSlices(
+            constant_op.constant(
+                [0.2], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        repeated_update = adagrad.Adagrad(3.0).apply_gradients(
+            [(grad_repeated_index, repeated_index_update_var)])
+        aggregated_update = adagrad.Adagrad(3.0).apply_gradients(
+            [(grad_aggregated, aggregated_update_var)])
+        variables.global_variables_initializer().run()
+        self.assertAllClose(aggregated_update_var.eval(),
+                            repeated_index_update_var.eval())
+        for _ in range(3):
+          repeated_update.run()
+          aggregated_update.run()
+          self.assertAllClose(aggregated_update_var.eval(),
+                              repeated_index_update_var.eval())
+
+  def testSparseRepeatedIndicesByEmbeddingLookUp(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var_repeated = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype)
+        loss_repeated = math_ops.reduce_sum(
+            embedding_ops.embedding_lookup(var_repeated, [0, 0]))
+        var_aggregated = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype)
+        loss_aggregated = 2 * math_ops.reduce_sum(
+            embedding_ops.embedding_lookup(var_aggregated, [0]))
+        update_op_repeated = adagrad.Adagrad(2.0).minimize(
+            loss_repeated, var_list=[var_repeated])
+        update_op_aggregated = adagrad.Adagrad(2.0).minimize(
+            loss_aggregated, var_list=[var_aggregated])
+        variables.global_variables_initializer().run()
+        self.assertAllCloseAccordingToType(
+            var_repeated.eval(), var_aggregated.eval())
+        for _ in range(3):
+          update_op_repeated.run()
+          update_op_aggregated.run()
+          self.assertAllCloseAccordingToType(
+              var_repeated.eval(), var_aggregated.eval())
+
+  def testSparseStability(self):
+    for dtype in [dtypes.half]:
+      with self.cached_session():
+        shape = [1, 6]
+        var0_np = np.array([[
+            0.00872496, -0.106952, 0.110467, 0.226505, -0.0147257, -0.0105945
+        ]],
+                           dtype=dtype.as_numpy_dtype)
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        grads0_np = np.array([[
+            -5.91278e-05, 5.31673e-05, -2.5779e-06, 4.29153e-05, -8.4877e-05,
+            -9.48906e-05
+        ]],
+                             dtype=dtype.as_numpy_dtype)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np), constant_op.constant([0]),
+            constant_op.constant(shape))
+        ada_opt = adagrad.Adagrad(1.0)
+        ada_update = ada_opt.apply_gradients(zip([grads0], [var0]))
+        slot0 = ada_opt.get_slot(var0, "accumulator")
+        init = variables.global_variables_initializer()
+        for _ in range(100):
+          init.run()
+          ada_update.run()
+          self.assertAllCloseAccordingToType(
+              np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval())
+          self.assertAllCloseAccordingToType(
+              np.array([[
+                  0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573,
+                  -0.01029443
+              ]]), var0.eval())
+
+  def testSharing(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = 3.0
+        ada_opt = adagrad.Adagrad(learning_rate)
+        # Apply the optimizer twice.  Both applications will use
+        # the same accums.
+        ada_update1 = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        ada_update2 = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        slot0 = ada_opt.get_slot(var0, "accumulator")
+        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        slot1 = ada_opt.get_slot(var1, "accumulator")
+        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values.
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Mix the first and the second adagrad for 3 steps.
+        ada_update1.run()
+        ada_update2.run()
+        ada_update1.run()
+
+        accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        for _ in range(3):
+          var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np,
+                                                    grads0_np, learning_rate)
+          var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np,
+                                                    grads1_np, learning_rate)
+        self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+        self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 1cab03eec7..962680fad6 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -44,7 +44,7 @@ class Adam(optimizer_v2.OptimizerV2):
                learning_rate=0.001,
                beta_1=0.9,
                beta_2=0.999,
-               epsilon=1e-8,
+               epsilon=1e-7,
                name='Adam'):
     r"""Construct a new Adam optimizer.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 76d3307e5f..46a45af224 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -41,7 +41,7 @@ def adam_update_numpy(param,
                       alpha=0.001,
                       beta1=0.9,
                       beta2=0.999,
-                      epsilon=1e-8):
+                      epsilon=1e-7):
   alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
 
   m_t = beta1 * m + (1 - beta1) * g_t
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 539654580e..6712427f91 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -43,7 +43,7 @@ class AdaMax(adam.Adam):
                learning_rate=0.001,
                beta_1=0.9,
                beta_2=0.999,
-               epsilon=1e-8,
+               epsilon=1e-7,
                name='AdaMax'):
     """Construct a new AdaMax optimizer.
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 99902ff9a9..fa7cfa5b8a 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -374,12 +374,16 @@ class OptimizerV2(optimizer_v1.Optimizer):
     else:
       super(OptimizerV2, self).__setattr__(name, value)
 
-  def add_slot(self, var, slot_name):
+  def add_slot(self, var, slot_name, initializer="zeros"):
     var_key = _var_key(var)
     slot_dict = self._slots.setdefault(var_key, {})
     if slot_name not in slot_dict:
       slot_key = _get_slot_key_from_var(var, slot_name)
-      weight = self.add_weight(name=slot_key, shape=var.shape, dtype=var.dtype)
+      weight = self.add_weight(
+          name=slot_key,
+          shape=var.shape,
+          dtype=var.dtype,
+          initializer=initializer)
       slot_dict[slot_name] = weight
       self._weights.append(weight)
 
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index 4b036c005f..e34397c06d 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -56,7 +56,7 @@ class RMSProp(optimizer_v2.OptimizerV2):
                learning_rate=0.001,
                rho=0.9,
                momentum=0.0,
-               epsilon=1e-10,
+               epsilon=1e-7,
                centered=False,
                name="RMSProp"):
     """Construct a new RMSProp optimizer.
-- 
GitLab


From f62b85410ddac7be9bb8ec25e84cf1712cedff53 Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Mon, 19 Nov 2018 15:32:00 -0800
Subject: [PATCH 0541/1554] Add support for SquaredDifference

PiperOrigin-RevId: 222150470
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md |   1 +
 tensorflow/lite/kernels/BUILD                 |  14 ++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/kernels/squared_difference.cc | 129 ++++++++++++++
 .../lite/kernels/squared_difference_test.cc   | 157 ++++++++++++++++++
 tensorflow/lite/testing/generate_examples.py  |   4 +
 .../testing/generated_examples_zip_test.cc    |   1 +
 .../propagate_fixed_sizes.cc                  |   1 +
 tensorflow/lite/toco/import_tensorflow.cc     |   2 +
 tensorflow/lite/toco/model.h                  |  12 ++
 tensorflow/lite/toco/tflite/operator.cc       |  22 +++
 tensorflow/lite/toco/tflite/operator_test.cc  |   7 +
 tensorflow/lite/toco/tooling_util.cc          |   2 +
 14 files changed, 355 insertions(+)
 create mode 100644 tensorflow/lite/kernels/squared_difference.cc
 create mode 100644 tensorflow/lite/kernels/squared_difference_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 8666855fff..59b1d9d2a0 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -293,6 +293,7 @@ def generated_test_models():
         "split",
         "sqrt",
         "square",
+        "squared_difference",
         "squeeze",
         "strided_slice",
         "strided_slice_1d_exhaustive",
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index 2a7f8f1f23..5a7bc2deeb 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -75,6 +75,7 @@ counterparts:
     0D tensor*
 *   [tf.squeeze](https://www.tensorflow.org/api_docs/python/tf/squeeze) - *as
     long as axis is not provided*
+*   [tf.squared_difference](https://www.tensorflow.org/versions/master/api_docs/python/tf/squared_difference)
 *   [tf.strided_slice](https://www.tensorflow.org/api_docs/python/tf/strided_slice) -
     *as long as ellipsis_mask and new_axis_mask are not used*
 *   [tf.transpose](https://www.tensorflow.org/versions/master/api_docs/python/tf/transpose) -
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 010ba83466..0bf4f01ac3 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -219,6 +219,7 @@ cc_library(
         "sparse_output_fully_connected.cc",
         "sparse_to_dense.cc",
         "split.cc",
+        "squared_difference.cc",
         "squeeze.cc",
         "strided_slice.cc",
         "sub.cc",
@@ -1379,6 +1380,19 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "squared_difference_test",
+    size = "small",
+    srcs = ["squared_difference_test.cc"],
+    tags = ["tflite_not_portable_ios"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 3645507cc7..f4aa5cc438 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -124,6 +124,7 @@ TfLiteRegistration* Register_ZEROS_LIKE();
 TfLiteRegistration* Register_FLOOR_MOD();
 TfLiteRegistration* Register_RANGE();
 TfLiteRegistration* Register_LEAKY_RELU();
+TfLiteRegistration* Register_SQUARED_DIFFERENCE();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -258,6 +259,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_FLOOR_MOD, Register_FLOOR_MOD());
   AddBuiltin(BuiltinOperator_RANGE, Register_RANGE());
   AddBuiltin(BuiltinOperator_LEAKY_RELU, Register_LEAKY_RELU());
+  AddBuiltin(BuiltinOperator_SQUARED_DIFFERENCE, Register_SQUARED_DIFFERENCE());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/kernels/squared_difference.cc b/tensorflow/lite/kernels/squared_difference.cc
new file mode 100644
index 0000000000..59b53a6287
--- /dev/null
+++ b/tensorflow/lite/kernels/squared_difference.cc
@@ -0,0 +1,129 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/quantization_util.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace squared_difference {
+
+constexpr int kInputTensor1 = 0;
+constexpr int kInputTensor2 = 1;
+constexpr int kOutputTensor = 0;
+
+struct OpData {
+  bool requires_broadcast;
+};
+
+template <typename T>
+T SquaredDifference(T input1, T input2) {
+  const T difference = input1 - input2;
+  return difference * difference;
+}
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new OpData;
+  data->requires_broadcast = false;
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<OpData*>(buffer);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  OpData* data = reinterpret_cast<OpData*>(node->user_data);
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1);
+  const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  TF_LITE_ENSURE_EQ(context, input1->type, input2->type);
+  output->type = input2->type;
+
+  data->requires_broadcast = !HaveSameShapes(input1, input2);
+
+  TfLiteIntArray* output_size = nullptr;
+  if (data->requires_broadcast) {
+    TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast(
+                                   context, input1, input2, &output_size));
+  } else {
+    output_size = TfLiteIntArrayCopy(input1->dims);
+  }
+
+  return context->ResizeTensor(context, output, output_size);
+}
+
+template <typename T>
+void EvalSquaredDifference(TfLiteContext* context, TfLiteNode* node,
+                           const OpData* data, const TfLiteTensor* input1,
+                           const TfLiteTensor* input2, TfLiteTensor* output) {
+  if (data->requires_broadcast) {
+    reference_ops::BroadcastBinaryFunction4DSlow<T, T, T>(
+        GetTensorShape(input1), GetTensorData<T>(input1),
+        GetTensorShape(input2), GetTensorData<T>(input2),
+        GetTensorShape(output), GetTensorData<T>(output), SquaredDifference<T>);
+  } else {
+    reference_ops::BinaryFunction<T, T, T>(
+        GetTensorShape(input1), GetTensorData<T>(input1),
+        GetTensorShape(input2), GetTensorData<T>(input2),
+        GetTensorShape(output), GetTensorData<T>(output), SquaredDifference<T>);
+  }
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  OpData* data = reinterpret_cast<OpData*>(node->user_data);
+
+  const TfLiteTensor* input1 = GetInput(context, node, kInputTensor1);
+  const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  if (output->type == kTfLiteFloat32) {
+    EvalSquaredDifference<float>(context, node, data, input1, input2, output);
+  } else if (output->type == kTfLiteInt32) {
+    EvalSquaredDifference<int32_t>(context, node, data, input1, input2, output);
+  } else {
+    context->ReportError(context,
+                         "SquaredDifference only supports FLOAT32, INT32 and "
+                         "quantized UINT8 now, got %d.",
+                         output->type);
+    return kTfLiteError;
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace squared_difference
+
+TfLiteRegistration* Register_SQUARED_DIFFERENCE() {
+  static TfLiteRegistration r = {
+      squared_difference::Init, squared_difference::Free,
+      squared_difference::Prepare, squared_difference::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/squared_difference_test.cc b/tensorflow/lite/kernels/squared_difference_test.cc
new file mode 100644
index 0000000000..32bcab3b87
--- /dev/null
+++ b/tensorflow/lite/kernels/squared_difference_test.cc
@@ -0,0 +1,157 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class BaseSquaredDifferenceOpModel : public SingleOpModel {
+ public:
+  BaseSquaredDifferenceOpModel(const TensorData& input1,
+                               const TensorData& input2,
+                               const TensorData& output) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_SQUARED_DIFFERENCE,
+                 BuiltinOptions_SquaredDifferenceOptions,
+                 CreateSquaredDifferenceOptions(builder_).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  int input1() { return input1_; }
+  int input2() { return input2_; }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+class FloatSquaredDifferenceOpModel : public BaseSquaredDifferenceOpModel {
+ public:
+  using BaseSquaredDifferenceOpModel::BaseSquaredDifferenceOpModel;
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+};
+
+class IntegerSquaredDifferenceOpModel : public BaseSquaredDifferenceOpModel {
+ public:
+  using BaseSquaredDifferenceOpModel::BaseSquaredDifferenceOpModel;
+
+  std::vector<int32_t> GetOutput() { return ExtractVector<int32_t>(output_); }
+};
+
+TEST(FloatSquaredDifferenceOpTest, FloatType_SameShape) {
+  FloatSquaredDifferenceOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
+                                  {TensorType_FLOAT32, {1, 2, 2, 1}},
+                                  {TensorType_FLOAT32, {}});
+  m.PopulateTensor<float>(m.input1(), {-0.2, 0.2, -1.2, 0.8});
+  m.PopulateTensor<float>(m.input2(), {0.5, 0.2, -1.5, 0.5});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({0.49, 0.0, 0.09, 0.09})));
+}
+
+TEST(FloatSquaredDifferenceOpTest, FloatType_VariousInputShapes) {
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    FloatSquaredDifferenceOpModel m({TensorType_FLOAT32, test_shapes[i]},
+                                    {TensorType_FLOAT32, test_shapes[i]},
+                                    {TensorType_FLOAT32, {}});
+    m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.3, 0.8, 1.1, -2.0});
+    m.PopulateTensor<float>(m.input2(), {1.0, 0.2, 0.6, 0.4, -1.0, -0.0});
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetOutput(),
+        ElementsAreArray(ArrayFloatNear({9.0, 0.0, 0.09, 0.16, 4.41, 4.0})))
+        << "With shape number " << i;
+  }
+}
+
+TEST(FloatSquaredDifferenceOpTest, FloatType_WithBroadcast) {
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    FloatSquaredDifferenceOpModel m(
+        {TensorType_FLOAT32, test_shapes[i]},
+        {TensorType_FLOAT32, {}},  // always a scalar
+        {TensorType_FLOAT32, {}});
+    m.PopulateTensor<float>(m.input1(), {-0.2, 0.2, 0.5, 0.8, 0.11, 1.1});
+    m.PopulateTensor<float>(m.input2(), {0.1});
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetOutput(),
+        ElementsAreArray(ArrayFloatNear({0.09, 0.01, 0.16, 0.49, 0.0001, 1.0})))
+        << "With shape number " << i;
+  }
+}
+
+TEST(IntegerSquaredDifferenceOpTest, IntegerType_SameShape) {
+  IntegerSquaredDifferenceOpModel m({TensorType_INT32, {1, 2, 2, 1}},
+                                    {TensorType_INT32, {1, 2, 2, 1}},
+                                    {TensorType_INT32, {}});
+  m.PopulateTensor<int32_t>(m.input1(), {-2, 2, -15, 8});
+  m.PopulateTensor<int32_t>(m.input2(), {5, -2, -3, 5});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({49, 16, 144, 9}));
+}
+
+TEST(IntegerSquaredDifferenceOpTest, IntegerType_VariousInputShapes) {
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    IntegerSquaredDifferenceOpModel m({TensorType_INT32, test_shapes[i]},
+                                      {TensorType_INT32, test_shapes[i]},
+                                      {TensorType_INT32, {}});
+    m.PopulateTensor<int32_t>(m.input1(), {-20, 2, 3, 8, 11, -20});
+    m.PopulateTensor<int32_t>(m.input2(), {1, 2, 6, 5, -5, -20});
+    m.Invoke();
+    EXPECT_THAT(m.GetOutput(), ElementsAreArray({441, 0, 9, 9, 256, 0}))
+        << "With shape number " << i;
+  }
+}
+
+TEST(IntegerSquaredDifferenceOpTest, IntegerType_WithBroadcast) {
+  std::vector<std::vector<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    IntegerSquaredDifferenceOpModel m(
+        {TensorType_INT32, test_shapes[i]},
+        {TensorType_INT32, {}},  // always a scalar
+        {TensorType_INT32, {}});
+    m.PopulateTensor<int32_t>(m.input1(), {-20, 10, 7, 3, 1, 13});
+    m.PopulateTensor<int32_t>(m.input2(), {3});
+    m.Invoke();
+    EXPECT_THAT(m.GetOutput(), ElementsAreArray({529, 49, 16, 0, 4, 100}))
+        << "With shape number " << i;
+  }
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index f54ab9dfbf..eccef341a0 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -1177,6 +1177,10 @@ def make_floor_mod_tests(zip_path):
   make_binary_op_tests(zip_path, tf.floormod)
 
 
+def make_squared_difference_tests(zip_path):
+  make_binary_op_tests(zip_path, tf.squared_difference)
+
+
 def make_gather_tests(zip_path):
   """Make a set of tests to do gather."""
 
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index aedea52065..6f31daa5fb 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -101,6 +101,7 @@ std::map<string, string> kBrokenTests = {
     {R"(^\/mul.*dtype=tf\.int64)", "119126484"},
     {R"(^\/add.*dtype=tf\.int64)", "119126484"},
     {R"(^\/floor_div.*dtype=tf\.int64)", "119126484"},
+    {R"(^\/squared_difference.*dtype=tf\.int64)", "119126484"},
 };
 
 // Allows test data to be unarchived into a temporary directory and makes
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 54f4b5601b..664424860e 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1760,6 +1760,7 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) {
     case OperatorType::kEqual:
     case OperatorType::kNotEqual:
     case OperatorType::kPow:
+    case OperatorType::kSquaredDifference:
       ProcessSimpleBinaryOperator(model, op);
       break;
     case OperatorType::kAddN:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index d4ca594ba6..c51031b4e5 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2348,6 +2348,8 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Split", ConvertSplitOperator},
       {"Sqrt", ConvertSimpleOperator<TensorFlowSqrtOperator, 1>},
       {"Square", ConvertSimpleOperator<TensorFlowSquareOperator, 1>},
+      {"SquaredDifference",
+       ConvertSimpleOperator<SquaredDifferenceOperator, 2>},
       {"Squeeze", ConvertSqueezeOperator},
       {"StopGradient", ConvertIdentityOperator},
       {"StridedSlice", ConvertStridedSliceOperator},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 7b9c78eaad..92be42f47c 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -123,6 +123,7 @@ enum class OperatorType : uint8 {
   kSplit,
   kSqrt,
   kSquare,
+  kSquaredDifference,
   kSum,
   kSwitch,
   kTile,
@@ -1303,6 +1304,17 @@ struct TensorFlowSquareOperator : Operator {
   TensorFlowSquareOperator() : Operator(OperatorType::kSquare) {}
 };
 
+// Element-wise squared difference ((x-y)*(x-y)) operator.
+//
+// Inputs:
+//   inputs[0]: required: the left-hand side array
+//   inputs[1]: required: the right-hand side array
+//
+// TensorFlow equivalent: SquaredDifference
+struct SquaredDifferenceOperator : Operator {
+  SquaredDifferenceOperator() : Operator(OperatorType::kSquaredDifference) {}
+};
+
 // Transposes a tensor.
 //
 // By default, this operation performs a regular matrix transpose on 2-D input
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 44da6a97d0..83325f1f79 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1236,6 +1236,25 @@ class LeakyRelu
   int GetVersion(const Operator& op) const override { return 1; }
 };
 
+class SquaredDifference
+    : public BuiltinOperator<
+          SquaredDifferenceOperator, ::tflite::SquaredDifferenceOptions,
+          ::tflite::BuiltinOptions_SquaredDifferenceOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateSquaredDifferenceOptions(*builder);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {}
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
 std::unique_ptr<flexbuffers::Builder> WriteFlexOpOptions(
     const string& tensorflow_node_def) {
   auto fbb = absl::make_unique<flexbuffers::Builder>();
@@ -1536,6 +1555,9 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
                                    OperatorType::kUnpack));
   ops.push_back(MakeUnique<LeakyRelu>(::tflite::BuiltinOperator_LEAKY_RELU,
                                       OperatorType::kLeakyRelu));
+  ops.push_back(MakeUnique<SquaredDifference>(
+      ::tflite::BuiltinOperator_SQUARED_DIFFERENCE,
+      OperatorType::kSquaredDifference));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 96aac5f9d5..16514760de 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -525,6 +525,13 @@ TEST_F(OperatorTest, BuiltinLeakyRelu) {
   EXPECT_EQ(op.alpha, output_toco_op->alpha);
 }
 
+TEST_F(OperatorTest, BuiltinSquaredDifference) {
+  SquaredDifferenceOperator op;
+  auto output_toco_op = SerializeAndDeserialize(
+      GetOperator("SQUARED_DIFFERENCE", OperatorType::kSquaredDifference), op);
+  ASSERT_NE(nullptr, output_toco_op.get());
+}
+
 TEST_F(OperatorTest, CustomCTCBeamSearchDecoder) {
   CTCBeamSearchDecoderOperator op;
   op.beam_width = 3;
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 2afd4685d7..cff387782f 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -412,6 +412,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceLstm)
     HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor)
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
+    HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
@@ -440,6 +441,7 @@ bool OperatorSupportsFusedActivation(OperatorType type) {
     case OperatorType::kMaxPool:
     case OperatorType::kMul:
     case OperatorType::kSub:
+    case OperatorType::kSquaredDifference:
       return true;
     default:
       return false;
-- 
GitLab


From c0f8f54005df9e6ef91ae4eda44ee9c32c907355 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 19 Nov 2018 15:46:37 -0800
Subject: [PATCH 0542/1554] Adding a deprecation statement to
 `tf.sparse.merge`.

PiperOrigin-RevId: 222153035
---
 tensorflow/python/ops/sparse_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 57f9f4e03e..58cd8291e1 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -1488,7 +1488,7 @@ def sparse_to_indicator(sp_input, vocab_size, name=None):
 
 
 @tf_export(v1=["sparse.merge", "sparse_merge"])
-@deprecation.deprecated_endpoints(["sparse.merge", "sparse_merge"])
+@deprecation.deprecated(None, "No similar op available at this time.")
 def sparse_merge(sp_ids, sp_values, vocab_size, name=None,
                  already_sorted=False):
   """Combines a batch of feature ids and values into a single `SparseTensor`.
-- 
GitLab


From 810a383baa41dbefe3902ed7b2d9963ab81a662e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 15:57:50 -0800
Subject: [PATCH 0543/1554] Remove unnecessary include.

PiperOrigin-RevId: 222154730
---
 tensorflow/lite/kernels/register.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/lite/kernels/register.h b/tensorflow/lite/kernels/register.h
index eb5ce667d4..059c9d165e 100644
--- a/tensorflow/lite/kernels/register.h
+++ b/tensorflow/lite/kernels/register.h
@@ -15,7 +15,6 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_KERNELS_REGISTER_H_
 #define TENSORFLOW_LITE_KERNELS_REGISTER_H_
 
-#include <unordered_map>
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/mutable_op_resolver.h"
-- 
GitLab


From b7bc8650cd673e2b12ed2a9b5a81d8074cee1e2a Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 19 Nov 2018 16:08:25 -0800
Subject: [PATCH 0544/1554] fixed grammar in dataset_ops and readers

---
 tensorflow/python/data/ops/dataset_ops.py | 28 +++++++++++------------
 tensorflow/python/data/ops/readers.py     |  2 +-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index cf51fdffdd..37802b9105 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -278,9 +278,9 @@ class DatasetV2(object):
     Note that if `tensors` contains a NumPy array, and eager execution is not
     enabled, the values will be embedded in the graph as one or more
     `tf.constant` operations. For large datasets (> 1 GB), this can waste
-    memory and run into byte limits of graph serialization.  If tensors contains
-    one or more large NumPy arrays, consider the alternative described in
-    [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
+    memory and run into byte limits of graph serialization. If `tensors`
+    contains one or more large NumPy arrays, consider the alternative described
+    in [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
 
     Args:
       tensors: A nested structure of tensors.
@@ -297,9 +297,9 @@ class DatasetV2(object):
     Note that if `tensors` contains a NumPy array, and eager execution is not
     enabled, the values will be embedded in the graph as one or more
     `tf.constant` operations. For large datasets (> 1 GB), this can waste
-    memory and run into byte limits of graph serialization.  If tensors contains
-    one or more large NumPy arrays, consider the alternative described in
-    [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
+    memory and run into byte limits of graph serialization. If `tensors`
+    contains one or more large NumPy arrays, consider the alternative described
+    in [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
 
     Args:
       tensors: A nested structure of tensors, each having the same size in the
@@ -566,7 +566,7 @@ class DatasetV2(object):
     ```
 
     Args:
-      *args: follow same semantics as python's xrange.
+      *args: follows the same semantics as python's xrange.
         len(args) == 1 -> start = 0, stop = args[0], step = 1
         len(args) == 2 -> start = args[0], stop = args[1], step = 1
         len(args) == 3 -> start = args[0], stop = args[1, stop = args[2]
@@ -852,10 +852,10 @@ class DatasetV2(object):
 
     Raises:
       ValueError: if `num_shards` or `index` are illegal values. Note: error
-        checking is done on a best-effort basis, and aren't guaranteed to be
-        caught upon dataset creation. (e.g. providing in a placeholder tensor
-        bypasses the early checking, and will instead result in an error during
-        a session.run call.)
+        checking is done on a best-effort basis, and errors aren't guaranteed
+        to be caught upon dataset creation. (e.g. providing in a placeholder
+        tensor bypasses the early checking, and will instead result in an error
+        during a session.run call.)
     """
     num_shards = ops.convert_to_tensor(
         num_shards, name="num_shards", dtype=dtypes.int64)
@@ -892,7 +892,7 @@ class DatasetV2(object):
       batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of
         consecutive elements of this dataset to combine in a single batch.
       drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
-        whether the last batch should be dropped in the case its has fewer than
+        whether the last batch should be dropped in the case it has fewer than
         `batch_size` elements; the default behavior is not to drop the smaller
         batch.
 
@@ -949,7 +949,7 @@ class DatasetV2(object):
         respective components.  Defaults are `0` for numeric types and
         the empty string for string types.
       drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
-        whether the last batch should be dropped in the case its has fewer than
+        whether the last batch should be dropped in the case it has fewer than
         `batch_size` elements; the default behavior is not to drop the smaller
         batch.
 
@@ -1573,7 +1573,7 @@ class DatasetV1Adapter(DatasetV1):
 class Options(object):
   """Represents options for tf.data.Dataset.
 
-  An `Options` object can be for instance used to control which static
+  An `Options` object can be, for instance, used to control which static
   optimizations to apply or whether to use performance modeling to dynamically
   tune the parallelism of operations such as `tf.data.Dataset.map` or
   `tf.data.Dataset.interleave`.
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index 8a563122a0..85999444d9 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -180,7 +180,7 @@ class TFRecordDatasetV2(dataset_ops.DatasetV2):
 
   def __init__(self, filenames, compression_type=None, buffer_size=None,
                num_parallel_reads=None):
-    """Creates a `TFRecordDataset` to read for one or more TFRecord files.
+    """Creates a `TFRecordDataset` to read one or more TFRecord files.
 
     NOTE: The `num_parallel_reads` argument can be used to improve performance
     when reading from a remote filesystem.
-- 
GitLab


From 8414baa94d62b23024b3a5a6b3dea23f3dcb4568 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 19 Nov 2018 16:08:45 -0800
Subject: [PATCH 0545/1554] fixed grammar in experimental enumerate_ops

---
 tensorflow/python/data/experimental/ops/enumerate_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/enumerate_ops.py b/tensorflow/python/data/experimental/ops/enumerate_ops.py
index a1af98f552..f38cab12a7 100644
--- a/tensorflow/python/data/experimental/ops/enumerate_ops.py
+++ b/tensorflow/python/data/experimental/ops/enumerate_ops.py
@@ -26,9 +26,9 @@ from tensorflow.python.util.tf_export import tf_export
 
 @tf_export("data.experimental.enumerate_dataset")
 def enumerate_dataset(start=0):
-  """A transformation that enumerate the elements of a dataset.
+  """A transformation that enumerates the elements of a dataset.
 
-  It is Similar to python's `enumerate`.
+  It is similar to python's `enumerate`.
   For example:
 
   ```python
-- 
GitLab


From 06b671ee519da066ac32f081df487c2bdafdf38e Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 19 Nov 2018 16:16:49 -0800
Subject: [PATCH 0546/1554] Internal Change.

PiperOrigin-RevId: 222157726
---
 tensorflow/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 2dc70c359c..17577afecb 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -355,7 +355,7 @@ package_group(
         "-//third_party/tensorflow/python/estimator",
         "//learning/meta_rank/...",
         "//tensorflow/...",
-        "//tensorflow_estimator/...",
+        "//tensorflow_estimator/contrib/...",
         "//tensorflow_fold/llgtm/...",
         "//tensorflow_text/...",
         "//third_party/py/tensor2tensor/...",
-- 
GitLab


From fc8f4eee1c65bdcd25ce19bbb8063e61a859dc6b Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 19 Nov 2018 16:41:32 -0800
Subject: [PATCH 0547/1554] Reenable tests since the TODO for grappler has been
 addressed.

PiperOrigin-RevId: 222161267
---
 tensorflow/python/eager/function_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index fe5f0ef57f..a206b1f791 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1695,8 +1695,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
                                  'be Tensors;.*'):
       graph_function('Not a Tensor.')
 
-  # TODO(scottzhu): Revive the test once the grappler plugin is updated.
-  def disabled_testSwapImplementationWithGrapplerPlugin(self):
+  def testSwapImplementationWithGrapplerPlugin(self):
     rewrites = rewriter_config_pb2.RewriterConfig()
     # function_optimizer has to be turn off, otherwise it will delete the
     # registered function if it does not get called.
-- 
GitLab


From 63d37cc71946c513af6c9c3a0dceacdb458228cf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 16:56:10 -0800
Subject: [PATCH 0548/1554] Support spatial partition for input dimension not
 being able to be divided in TPUEstimator

PiperOrigin-RevId: 222163267
---
 .../experimental/xla_sharding/xla_sharding.py | 30 ++-------
 tensorflow/contrib/tpu/python/tpu/tpu_feed.py | 63 ++++++++++---------
 2 files changed, 38 insertions(+), 55 deletions(-)

diff --git a/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py b/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py
index fb135f5ced..1fea816a80 100644
--- a/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py
+++ b/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py
@@ -18,12 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import math
-
 import numpy as _np  # Avoids becoming a part of public Tensorflow API.
 
 from tensorflow.compiler.xla import xla_data_pb2
-from tensorflow.compiler.xla.python_api import xla_shape
 from tensorflow.core.framework import attr_value_pb2
 
 
@@ -64,22 +61,18 @@ class Sharding(object):
             tile_assignment_devices=[core]))
 
   @classmethod
-  def tile(cls, tile_shape, tile_assignment):
+  def tile(cls, tile_assignment):
     """Returns a Tiled sharding attribute.
 
     This causes an op to be partially computed on multiple cores in the
     XLA device.
 
     Args:
-      tile_shape: A xla_shape.Shape describing the tile shape that each core
-        will compute.
-        The tile shape does not need to be divisible by the tile assignment.
       tile_assignment: An np.ndarray describing the topology of the tiling and
         which device will compute which part of the topology.
 
     Raises:
-      TypeError: tile_assignment was not of np.array type or tile_shape was
-         not of xla_shape.Shape type.
+      TypeError: tile_assignment was not of np.array type.
 
     TODO(jmolloy): This concept is nefarious and is not
     something we really want to expose to users (especially as the
@@ -87,14 +80,11 @@ class Sharding(object):
     """
     if not isinstance(tile_assignment, _np.ndarray):
       raise TypeError('Tile assignment must be of type np.ndarray')
-    if not isinstance(tile_shape, xla_shape.Shape):
-      raise TypeError('Tile shape must be of type xla_shape.Shape')
     dims = list(tile_assignment.shape)
     flattened_devices = tile_assignment.reshape(-1, order='C')
     return Sharding(
         proto=xla_data_pb2.OpSharding(
             type=xla_data_pb2.OpSharding.OTHER,
-            tile_shape=tile_shape.message,
             tile_assignment_dimensions=dims,
             tile_assignment_devices=list(flattened_devices)))
 
@@ -118,14 +108,8 @@ class Sharding(object):
     shape = tensor.shape.as_list()
     if shape[split_dimension] < num_devices:
       raise ValueError('Split dimension was smaller than the required number '
-                       'of splits: shape=%r, dimension=%r, num_devices=%r',
-                       shape, split_dimension, num_devices)
-
-    tile_shape = shape
-    tile_shape[split_dimension] = int(
-        math.ceil(tile_shape[split_dimension] / num_devices))
-    tile_shape_proto = xla_data_pb2.Shape(
-        element_type=xla_data_pb2.F32, dimensions=tile_shape)
+                       'of splits: shape=%r, dimension=%r, num_devices=%r' %
+                       (shape, split_dimension, num_devices))
 
     tile_assignment_dims = [1] * len(shape)
     tile_assignment_dims[split_dimension] = num_devices
@@ -133,7 +117,6 @@ class Sharding(object):
     return Sharding(
         proto=xla_data_pb2.OpSharding(
             type=xla_data_pb2.OpSharding.OTHER,
-            tile_shape=tile_shape_proto,
             tile_assignment_dimensions=tile_assignment_dims,
             tile_assignment_devices=range(num_devices)))
 
@@ -149,7 +132,6 @@ class Sharding(object):
           type=xla_data_pb2.OpSharding.TUPLE, tuple_shardings=tuple_shardings)
     else:
       proto = self._proto
-
     attr_value = attr_value_pb2.AttrValue(s=proto.SerializeToString())
     # TODO(jmolloy): This need to be seriously revisited before declaring this
     # API available for public use.
@@ -194,8 +176,8 @@ def assign_device(tensor, device):
   return tensor
 
 
-def tile(tensor, tile_shape, tile_assignment):
-  Sharding.tile(tile_shape, tile_assignment).apply_to_tensor(tensor)
+def tile(tensor, tile_assignment):
+  Sharding.tile(tile_assignment).apply_to_tensor(tensor)
   return tensor
 
 
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
index e75a09492e..cf36103277 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
@@ -26,7 +26,6 @@ import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.compiler.xla.experimental.xla_sharding import xla_sharding
-from tensorflow.compiler.xla.python_api import xla_shape
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_sharding
@@ -92,8 +91,7 @@ class InfeedQueue(object):
       else:
         raise ValueError(
             "number of tuple elements cannot be inferred from InfeedQueue "
-            "constructor"
-        )
+            "constructor")
     if number_of_tuple_elements <= 0:
       raise ValueError("number_of_tuple_elements %d must be > 0" %
                        number_of_tuple_elements)
@@ -293,9 +291,8 @@ class InfeedQueue(object):
         self.number_of_tuple_elements
     """
     if len(input_tensors) != self.number_of_tuple_elements:
-      raise ValueError(
-          "input_tensors is %s, but should be a list of %d Tensors", (
-              str(input_tensors), self.number_of_tuple_elements))
+      raise ValueError("input_tensors is %s, but should be a list of %d Tensors"
+                       % (str(input_tensors), self.number_of_tuple_elements))
     self.set_tuple_shapes([t.shape for t in input_tensors])
     self.set_tuple_types([t.dtype for t in input_tensors])
 
@@ -451,8 +448,8 @@ class InfeedQueue(object):
       for i in xrange(1, self.number_of_tuple_elements):
         if devices[0] != devices[i]:
           raise ValueError(
-              "input devices for shard %d are %s, but should all be the same",
-              index, str(devices))
+              "input devices for shard %d are %s, but should all be the same" %
+              (index, str(devices)))
       with ops.colocate_with(inputs[0]):
         return tpu_ops.infeed_enqueue_tuple(
             inputs=inputs,
@@ -792,18 +789,14 @@ class _PartitionedInfeedQueue(InfeedQueue):
 
     Args:
       tensor: Input tensor for partitioning.
-      dims: A list of integer describes how to partition the input tensor.
+      dims: 1-D np.array of the list of integer describes how to partition the
+        input tensor.
 
     Raises:
       ValueError: If the tensor can't be partitioned by dims or the
         num_cores_per_replica doesn't match the number of
         partitions(dims.prod()).
     """
-    if dims is None:
-      return
-
-    dims = np.array(dims)
-
     if (dims < 1).any():
       raise ValueError("All input partition dims must be >= 1.")
 
@@ -823,11 +816,6 @@ class _PartitionedInfeedQueue(InfeedQueue):
           "partition dims = {}).".format(tensor.shape.as_list(), dims))
 
     tensor.shape.assert_is_fully_defined()
-    if (np.array(tensor.shape.as_list()) % dims != 0).any():
-      raise ValueError(
-          "All input partition dims must divide exactly into the `Tensor` "
-          "shape (tensor shape = {}, input partition dims = {}).".format(
-              tensor.shape.as_list(), dims))
 
   def _partition_or_replicate_on_host(self, tensor, dims):
     """Partitions or replicates the input tensor.
@@ -840,16 +828,33 @@ class _PartitionedInfeedQueue(InfeedQueue):
     Returns:
       An iterator of `Tensor`s or a list of partioned tensors.
     """
-    self._check_input_partition_dims(tensor, dims)
     if dims is None:
       return itertools.repeat(tensor)
-    else:
-      output = [tensor]
-      for axis, dim in enumerate(dims):
-        if dim > 1:
-          output = [array_ops.split(x, dim, axis=axis) for x in output]
-          output = nest.flatten(output)
-      return output
+    dims = np.array(dims)
+    self._check_input_partition_dims(tensor, dims)
+    output = [tensor]
+    divds, remainders = np.divmod(np.array(tensor.shape.as_list()), dims)
+    for axis, (divd, remainder, dim) in enumerate(
+        np.dstack((divds, remainders, dims))[0]):
+      if dim <= 1:
+        continue
+      if remainder > 0:
+        # For each dimension, when it cannot be evenly partitioned, XLA assumes
+        # the size of last parts are smaller by 1. E.g. 2D tensor with shape
+        # (5, 14) and dims are (2, 4). Since 5 % 2 = 1 and 14 % 4 = 2, [5, 14]
+        # => [[(3, 3), (3, 3), (2, 3), (2, 3)],
+        # [(2, 3), (2, 3), (2, 2), (2, 2)]]
+        output = [
+            array_ops.split(
+                x,
+                num_or_size_splits=[divd + 1] * remainder +
+                [divd] * (dim - remainder),
+                axis=axis) for x in output
+        ]
+      else:
+        output = [array_ops.split(x, dim, axis=axis) for x in output]
+      output = nest.flatten(output)
+    return output
 
   def _tag_sharding_attribute_for_dequeued_tensor(self, tensor, dims):
     """Tags appropriate XLA sharding attribute to the dequeued tensor.
@@ -866,13 +871,9 @@ class _PartitionedInfeedQueue(InfeedQueue):
     elif np.prod(dims) == 1:
       return xla_sharding.assign_device(tensor, 0)
     else:
-      tile_shape = np.array(tensor.shape.as_list()) // dims
       tile_assignment = np.arange(np.prod(dims)).reshape(dims)
       return xla_sharding.tile(
           tensor=tensor,
-          tile_shape=xla_shape.CreateShapeFromDtypeAndTuple(
-              dtype=np.dtype(tensor.dtype.as_numpy_dtype),
-              shape_tuple=tile_shape),
           tile_assignment=tile_assignment)
 
   def _tag_sharding_attribute_for_dequeued_tensors(self, dequeues, dims):
-- 
GitLab


From 8242bee9287540979a509afb4e6f3f21e18f0fe4 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 19 Nov 2018 17:00:51 -0800
Subject: [PATCH 0549/1554] Fix some crashes caused by adding kInt32.

PiperOrigin-RevId: 222163922
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 2 ++
 tensorflow/stream_executor/stream.cc        | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 9d691d0bc9..1f2e2f48bb 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -824,6 +824,8 @@ cudnnDataType_t ToCudnnDataType(
     case dnn::DataType::kInt8:
       return data_layout == dnn::DataLayout::kBatchDepthYX4 ? CUDNN_DATA_INT8x4
                                                             : CUDNN_DATA_INT8;
+    case dnn::DataType::kInt32:
+      return CUDNN_DATA_INT32;
     default:
       LOG(FATAL) << "Invalid DNN data type: " << static_cast<int>(data_type);
   }
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 4dc7c59921..3edc66cde8 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -191,6 +191,8 @@ string ToVlogString(dnn::DataType data_type) {
       return "dnn::DataType::kHalf";
     case dnn::DataType::kInt8:
       return "dnn::DataType::kInt8";
+    case dnn::DataType::kInt32:
+      return "dnn::DataType::kInt32";
     default:
       return "unknown DataType";
   }
-- 
GitLab


From 955a6a614adf9303dc2efcb1dca922c87f14a8aa Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Mon, 19 Nov 2018 17:01:34 -0800
Subject: [PATCH 0550/1554] Add upgrade script to pip package.

PiperOrigin-RevId: 222164025
---
 tensorflow/python/tools/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD
index 384c7a82d2..901d6bc335 100644
--- a/tensorflow/python/tools/BUILD
+++ b/tensorflow/python/tools/BUILD
@@ -29,6 +29,8 @@ py_library(
         ":optimize_for_inference_lib",
         ":selective_registration_header_lib",
         ":strip_unused_lib",
+        # Include the TF upgrade script to users can run it directly after install TF
+        "//tensorflow/tools/compatibility:tf_upgrade_v2",
     ],
 )
 
-- 
GitLab


From 1ff6d4ad09b3a2b9482ceedbeb244eb4ee83451e Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Mon, 19 Nov 2018 17:53:39 -0800
Subject: [PATCH 0551/1554] Rename ecosystem to resources

PiperOrigin-RevId: 222170251
---
 tensorflow/compiler/xla/g3doc/_book.yaml  | 6 +++---
 tensorflow/compiler/xla/g3doc/_index.yaml | 4 ++--
 tensorflow/lite/g3doc/_book.yaml          | 6 +++---
 tensorflow/lite/g3doc/_index.yaml         | 6 +++---
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/_book.yaml b/tensorflow/compiler/xla/g3doc/_book.yaml
index 756b932332..12b7094705 100644
--- a/tensorflow/compiler/xla/g3doc/_book.yaml
+++ b/tensorflow/compiler/xla/g3doc/_book.yaml
@@ -3,11 +3,11 @@ upper_tabs:
 - include: /_upper_tabs_left.yaml
 - include: /api_docs/_upper_tabs_api.yaml
 # Dropdown menu
-- name: Ecosystem
-  path: /ecosystem
+- name: Resources
+  path: /resources
   is_default: true
   menu:
-  - include: /ecosystem/_menu_toc.yaml
+  - include: /resources/_menu_toc.yaml
   lower_tabs:
     # Subsite tabs
     other:
diff --git a/tensorflow/compiler/xla/g3doc/_index.yaml b/tensorflow/compiler/xla/g3doc/_index.yaml
index 7934cd11ba..858de42711 100644
--- a/tensorflow/compiler/xla/g3doc/_index.yaml
+++ b/tensorflow/compiler/xla/g3doc/_index.yaml
@@ -17,7 +17,7 @@ landing_page:
   - classname: devsite-landing-row-cards
     items:
     - heading: XLA - TensorFlow, compiled
-      image_path: /ecosystem/images/tf-logo-card-16x9.png
+      image_path: /resources/images/tf-logo-card-16x9.png
       path: https://developers.googleblog.com/2017/03/xla-tensorflow-compiled.html
       buttons:
       - label: Read on Google Developers blog
@@ -28,7 +28,7 @@ landing_page:
       - label: Watch the video
         path: https://www.youtube.com/watch?v=kAOanJczHA0
     - heading: XLA on GitHub
-      image_path: /ecosystem/images/github-card-16x9.png
+      image_path: /resources/images/github-card-16x9.png
       path: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla
       buttons:
       - label: View on GitHub
diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml
index ab0d186848..a51c7a667f 100644
--- a/tensorflow/lite/g3doc/_book.yaml
+++ b/tensorflow/lite/g3doc/_book.yaml
@@ -3,11 +3,11 @@ upper_tabs:
 - include: /_upper_tabs_left.yaml
 - include: /api_docs/_upper_tabs_api.yaml
 # Dropdown menu
-- name: Ecosystem
-  path: /ecosystem
+- name: Resources
+  path: /resources
   is_default: true
   menu:
-  - include: /ecosystem/_menu_toc.yaml
+  - include: /resources/_menu_toc.yaml
   lower_tabs:
     # Subsite tabs
     other:
diff --git a/tensorflow/lite/g3doc/_index.yaml b/tensorflow/lite/g3doc/_index.yaml
index 093f86b542..1b3f1d616a 100644
--- a/tensorflow/lite/g3doc/_index.yaml
+++ b/tensorflow/lite/g3doc/_index.yaml
@@ -189,7 +189,7 @@ landing_page:
       - label: Read more
         path: https://cloud.google.com/blog/products/ai-machine-learning/ai-motion-designing-simple-system-see-understand-and-react-real-world-part-ii
     - heading: "Introducing the Model Optimization Toolkit"
-      image_path: /ecosystem/images/tf-logo-card-16x9.png
+      image_path: /resources/images/tf-logo-card-16x9.png
       path: https://medium.com/tensorflow/introducing-the-model-optimization-toolkit-for-tensorflow-254aca1ba0a3
       buttons:
       - label: Read on TensorFlow blog
@@ -205,7 +205,7 @@ landing_page:
     background: grey
     items:
     - heading: "Using TensorFlow Lite on Android"
-      image_path: /ecosystem/images/tf-logo-card-16x9.png
+      image_path: /resources/images/tf-logo-card-16x9.png
       path: https://medium.com/tensorflow/using-tensorflow-lite-on-android-9bbc9cb7d69d
       buttons:
       - label: Read on TensorFlow blog
@@ -216,7 +216,7 @@ landing_page:
       - label: Watch the video
         path: https://www.youtube.com/watch?v=FAMfy7izB6A
     - heading: "TensorFlow Lite on GitHub"
-      image_path: /ecosystem/images/github-card-16x9.png
+      image_path: /resources/images/github-card-16x9.png
       path: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite
       buttons:
       - label: View on GitHub
-- 
GitLab


From 7b327e27d67fdd0e7bfca1cbcf55ad63a6904bfe Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 19 Nov 2018 17:57:30 -0800
Subject: [PATCH 0552/1554] Clean up the step and batch size calculation in
 case of numpy inputs with DS in Keras.

PiperOrigin-RevId: 222170615
---
 .../contrib/distribute/python/keras_test.py   | 201 +++++++++++-------
 .../engine/distributed_training_utils.py      | 141 ++++++------
 tensorflow/python/keras/engine/training.py    |  48 +++--
 3 files changed, 217 insertions(+), 173 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 2df8d7a66e..29d85fe971 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -220,8 +220,8 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
   # TODO(b/118776054): Use global batch size for Keras/DS support.
   use_per_core_batch_size = (
       with_distribution and
-      not isinstance(with_distribution, (
-          tpu_strategy.TPUStrategy, mirrored_strategy.CoreMirroredStrategy)))
+      not distributed_training_utils.global_batch_size_supported(
+          with_distribution))
   if use_per_core_batch_size:
     batch_size //= with_distribution.num_replicas_in_sync
 
@@ -480,84 +480,133 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # Verify that the numpy value is copied to the variable.
       self.assertAllEqual(x, val)
 
-  # TODO(sourabhbajaj): Test this with all strategies.
-  def test_calculating_batch_params(self):
-    # This verifies that we calculate the number of steps when the batch size
-    # is specified.
+  @combinations.generate(strategy_combinations())
+  def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
+    # Calculate the per_replica_batch_size scaling factor for strategies
+    # that use per_core_batch_size
+    replica_scale_factor = 1.0
+    if not distributed_training_utils.global_batch_size_supported(distribution):
+      replica_scale_factor = distribution.num_replicas_in_sync
+
     with self.cached_session():
-      # 64 is the number of input samples.
-      inputs = np.zeros((64, 3), dtype=np.float32)
-      # The number of replicas is equal to 3.
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
-                                                     '/device:CPU:0',
-                                                     '/device:GPU:1'])
-
-      with self.assertRaisesRegexp(ValueError, 'The number of samples is not '
-                                   'divisible by batch size.'):
-        # The batch size(128) is larger than the number of input
-        # samples(64).
-        distributed_training_utils.get_input_batch_params(inputs,
-                                                          128,
-                                                          strategy)
-
-      with self.assertRaisesRegexp(ValueError, 'is smaller than the number '
-                                               'of replicas'):
-        # The batch size(32) * num_replicas_in_sync(3) is 96 which is greater
-        # than the number of input samples(64).
-        distributed_training_utils.get_input_batch_params(inputs,
-                                                          32,
-                                                          strategy)
-
-      # The number of replicas now is equal to 2.
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
-                                                     '/device:CPU:0'])
-      # 32 is the batch size per replica.
-      steps = distributed_training_utils.get_input_batch_params(inputs,
-                                                                32,
-                                                                strategy)
-      # The number of batches is the ratio of input samples(64) to
-      # batch size(32) which is 2. The number of steps(1) is the ratio of
-      # number of batches(2) to the number of replicas(2).
+      # Input samples of different sizes
+      input_20_samples = np.zeros((20, 3), dtype=np.float32)
+      input_63_samples = np.zeros((63, 3), dtype=np.float32)
+      input_64_samples = np.zeros((64, 3), dtype=np.float32)
+
+      # Default global batch size 32 for input with 64 samples run in 2 steps
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_64_samples, steps=None, batch_size=None)
+      self.assertEqual(batch_size, 32 // replica_scale_factor)
+      self.assertEqual(steps, 2)
+
+      # Computed global batch size 20 is lower than 32 if we pass less samples.
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_20_samples, steps=None, batch_size=None)
+      self.assertEqual(batch_size, 20 // replica_scale_factor)
+      self.assertEqual(steps, 1)
+
+      #  Default global batch size 32 cannot be used with 63 samples.
+      with self.assertRaisesRegexp(ValueError, 'not divisible by batch size'):
+        distributed_training_utils.get_input_params(
+            distribution, input_63_samples, steps=None, batch_size=None)
+
+  @combinations.generate(strategy_combinations())
+  def test_calculating_input_params_with_steps_no_batch_size(self,
+                                                             distribution):
+    # Calculate the per_replica_batch_size scaling factor for strategies
+    # that use per_core_batch_size
+    replica_scale_factor = 1.0
+    if not distributed_training_utils.global_batch_size_supported(distribution):
+      replica_scale_factor = distribution.num_replicas_in_sync
+
+    with self.cached_session():
+      # Input samples of different sizes
+      input_63_samples = np.zeros((63, 3), dtype=np.float32)
+      input_64_samples = np.zeros((64, 3), dtype=np.float32)
+
+      # Computed global batch size is correct for number of specified 1 step
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_64_samples, steps=1, batch_size=None)
+      self.assertEqual(batch_size, 64 // replica_scale_factor)
       self.assertEqual(steps, 1)
 
-      # 16 is the batch size per replica.
-      steps = distributed_training_utils.get_input_batch_params(inputs,
-                                                                16,
-                                                                strategy)
-      # The number of batches is the ratio of input samples(64) to
-      # batch size(16) which is 4. The number of steps(2) is the ratio of
-      # number of batches(4) to the number of replicas(2).
+      # Computed global batch size is correct for number of specified 2 steps
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_64_samples, steps=2, batch_size=None)
+      self.assertEqual(batch_size, 32 // replica_scale_factor)
       self.assertEqual(steps, 2)
 
-  # TODO(sourabhbajaj): Test this with all strategies.
-  def test_calculating_batch_size(self):
+      # All samples can not be consumed in specified number of steps
+      with self.assertRaisesRegexp(ValueError, 'not divisible by steps'):
+        distributed_training_utils.get_input_params(
+            distribution, input_63_samples, steps=2, batch_size=None)
+
+      # This cases is different for different strategies due to the
+      # difference in supported batch size being global or per-replica.
+      if replica_scale_factor == 1:
+        # Computed global batch size is correct even if not sharadable
+        steps, batch_size = distributed_training_utils.get_input_params(
+            distribution, input_63_samples, steps=3, batch_size=None)
+        self.assertEqual(batch_size, 21)
+        self.assertEqual(steps, 3)
+      else:
+        # Computed global batch size can not be sharded across replicas
+        with self.assertRaisesRegexp(ValueError, 'could not be sharded evenly '
+                                     'across the sync replicas'):
+          distributed_training_utils.get_input_params(
+              distribution, input_63_samples, steps=1, batch_size=None)
+
+  @combinations.generate(strategy_combinations())
+  def test_calculating_input_params_no_steps_with_batch_size(self,
+                                                             distribution):
+    # Calculate the per_replica_batch_size scaling factor for strategies
+    # that use per_core_batch_size
+    replica_scale_factor = 1.0
+    if not distributed_training_utils.global_batch_size_supported(distribution):
+      replica_scale_factor = distribution.num_replicas_in_sync
+
     with self.cached_session():
-      # 64 is the number of input samples.
-      inputs = np.zeros((64, 3), dtype=np.float32)
-      targets = np.zeros((64, 4), dtype=np.float32)
+      input_64_samples = np.zeros((64, 3), dtype=np.float32)
+
+      # Computed steps is correct for specified batch size
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_64_samples, steps=None, batch_size=16)
+      self.assertEqual(batch_size, 16)
+      self.assertEqual(steps, 4 // replica_scale_factor)
+
+      # Computed steps is correct for specified batch size
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_64_samples, steps=None, batch_size=32)
+      self.assertEqual(batch_size, 32)
+      self.assertEqual(steps, 2 // replica_scale_factor)
+
+      # Number of samples is not divisible by the global batch size
+      with self.assertRaisesRegexp(ValueError, 'not divisible by batch size'):
+        distributed_training_utils.get_input_params(
+            distribution, input_64_samples, steps=None, batch_size=20)
+
+      # Number of samples is not divisible by the global batch size
+      with self.assertRaisesRegexp(ValueError, 'not divisible by batch size'):
+        distributed_training_utils.get_input_params(
+            distribution, input_64_samples, steps=None, batch_size=3)
 
-      model = get_model()
-      optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-      loss = 'mse'
-      strategy = mirrored_strategy.MirroredStrategy(['/device:GPU:0',
-                                                     '/device:CPU:0'])
-      strategy.extended._require_static_shapes = True
-
-      model.compile(optimizer, loss, distribute=strategy)
-      iterator = model._distribution_standardize_user_data(inputs,
-                                                           targets,
-                                                           batch_size=None,
-                                                           check_steps=True,
-                                                           steps_name='steps',
-                                                           steps=3)
-
-      # The global batch size(21) across all replicas is the ratio of the input
-      # samples(64) to the steps(3).
-      # The batch size(10) per device is the ratio of the global batch size(21)
-      # to the number of replicas(2).
-      # The global batch size and batch size are rounded integer values.
-      self.assertEqual(10, distributed_training_utils.get_batch_dimension(
-          iterator))
+  @combinations.generate(strategy_combinations())
+  def test_calculating_input_params_with_steps_with_batch_size(self,
+                                                               distribution):
+    with self.cached_session():
+      input_64_samples = np.zeros((64, 3), dtype=np.float32)
+
+      # No change to steps and batch size if both specified and feasible
+      steps, batch_size = distributed_training_utils.get_input_params(
+          distribution, input_64_samples, steps=5, batch_size=3)
+      self.assertEqual(batch_size, 3)
+      self.assertEqual(steps, 5)
+
+      # Number of samples is less than global batch size * steps
+      with self.assertRaisesRegexp(ValueError, 'less than samples required'):
+        distributed_training_utils.get_input_params(
+            distribution, input_64_samples, steps=10, batch_size=13)
 
   @combinations.generate(strategy_combinations())
   def test_calling_model_with_numpy_arrays(self, distribution):
@@ -656,7 +705,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # `predict` a list that is equal in length to the number of model outputs.
       # In this test our model has two outputs and each element of `outs`
       # corresponds to all the samples of one of the model outputs.
-      self.assertEqual(2, len(outs))
+      self.assertLen(outs, 2)
       # Each of the output samples have a dimension of 7. We should process all
       # the available input samples(6).
       self.assertAllEqual([6, 7], outs[0].shape)
@@ -1147,7 +1196,9 @@ class TestDistributionStrategyCorrectness(test.TestCase,
           distribute=distribution)
 
       batch_size = 64
-      batch_size //= distribution.num_replicas_in_sync
+      if not distributed_training_utils.global_batch_size_supported(
+          distribution):
+        batch_size //= distribution.num_replicas_in_sync
       train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
       train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 8e0e2adca0..208602b90f 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -401,58 +401,84 @@ def global_batch_size_supported(distribution_strategy):
   return strategy_name in ('TPUStrategy', 'CoreMirroredStrategy')
 
 
-def get_input_batch_params(first_x_value, batch_size, distribution_strategy):
+def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
+                     is_training=False):
   """Calculate the number of batches and steps/steps_per_epoch.
 
   Args:
+    distribution_strategy: The DistributionStrategy used to compile the model.
     first_x_value: This is the first input numpy array that is passed in as the
       model input.
-    batch_size: The specified batch_size or the default batch_size of 32.
-    distribution_strategy: The current DistributionStrategy used to compile the
-      model.
+    steps:  The specified number of steps.
+    batch_size: The specified batch_size.
+    is_training: Boolean to relax the constraints on consuming all the training
+      samples to keep compatibility till we support partial batches.
 
   Returns:
-    The steps or steps_per_epoch argument depending on if a user is
-    calling `fit`, `evaluate` or `predict`.
+    steps: The steps or steps_per_epoch argument depending on if a user is
+        calling `fit`, `evaluate` or `predict`. If the is_training flag is set
+        we don't require the number of samples to be used completely.
+    batch_size: The batch size to be used in model iterations.
 
   Raises:
     ValueError: If the number of batches or steps evaluates to 0.
 
   """
-  if batch_size is None:
-    # Default the global batch size to the minimum of 32 and the size of
-    # the numpy array. 32 is chosen to guarantee backward compatibility.
-    batch_size = min(first_x_value.shape[0], 32)
-    if not global_batch_size_supported(distribution_strategy):
-      if batch_size % distribution_strategy.num_replicas_in_sync:
-        raise ValueError(
-            'The batch size (%s) could not be sharded evenly across the sync '
-            'replicas (%s) in the distribution strategy.' % (
-                batch_size, distribution_strategy.num_replicas_in_sync))
-      batch_size = batch_size // distribution_strategy.num_replicas_in_sync
-
-  # Calculate number of global batches
-  if first_x_value.shape[0] % batch_size:
-    raise ValueError('The number of samples is not divisible by batch size.')
-  num_batches = first_x_value.shape[0] // batch_size
-  if not num_batches:
-    raise ValueError('Please specify a batch_size that is smaller than '
-                     'the number of input samples %d.' % first_x_value.shape[0])
-
-  # Number of steps required to run needs to be divide by the number of replicas
-  # if global batch size is not supported.
-  if global_batch_size_supported(distribution_strategy):
-    steps = num_batches
+  num_samples = first_x_value.shape[0]
+  # TODO(b/118776054): Use global batch size for Keras/DS support.
+  # Currently this is only supported in TPUStrategy and CoreMirroredStrategy.
+  use_per_replica_batch = not global_batch_size_supported(
+      distribution_strategy)
+
+  if steps is None:
+    if batch_size is None:
+      # If neither the batch size or number of steps are set. We choose the
+      # global batch size as the minimum of number of samples and 32. 32 is
+      # chosen to provide backward compatibility.
+      global_batch_size = min(num_samples, 32)
+    else:
+      # If the user provided the batch size we need to handle the case
+      # between different strategies that use the global/per-replica batch size
+      global_batch_size = batch_size
+      if use_per_replica_batch:
+        global_batch_size *= distribution_strategy.num_replicas_in_sync
+    if not is_training and num_samples % global_batch_size:
+      raise ValueError('The number of samples %s is not divisible by '
+                       'batch size %s.' % (num_samples, global_batch_size))
+    steps = num_samples // global_batch_size
+  else:
+    if batch_size is None:
+      # We calculate the batch size based on the number of steps specified
+      if num_samples % steps:
+        raise ValueError('The number of samples %s is not divisible by '
+                         'steps %s. Please change the number of steps to a '
+                         'value that can consume all the samples' % (
+                             num_samples, steps))
+      global_batch_size = num_samples // steps
+    else:
+      # If the user provided the batch size we need to handle the case
+      # between different strategies that use the global/per-replica batch size
+      global_batch_size = batch_size
+      if use_per_replica_batch:
+        global_batch_size *= distribution_strategy.num_replicas_in_sync
+
+      if num_samples < (global_batch_size * steps):
+        raise ValueError('Number of samples %s is less than samples required '
+                         'for specified batch_size %s and steps %s' % (
+                             num_samples, global_batch_size, steps))
+
+  # We need to return the per replica or global batch size based on the strategy
+  if use_per_replica_batch:
+    if global_batch_size % distribution_strategy.num_replicas_in_sync:
+      raise ValueError(
+          'The batch size (%s) could not be sharded evenly across the sync '
+          'replicas (%s) in the distribution strategy.' % (
+              global_batch_size, distribution_strategy.num_replicas_in_sync))
+    batch_size = global_batch_size // distribution_strategy.num_replicas_in_sync
   else:
-    steps = num_batches // distribution_strategy.num_replicas_in_sync
-  if not steps:
-    # TODO(anjalisridhar): Number of replicas in the error message may not
-    # convey what we want to the user. Is there another terminology that we can
-    # use that is consistent across different strategies?
-    raise ValueError('The number of batches %d is smaller than the number '
-                     'of replicas %d used for DistributionStrategy. ' %
-                     (num_batches, distribution_strategy.num_replicas_in_sync))
-  return steps
+    batch_size = global_batch_size
+
+  return steps, batch_size
 
 
 def get_batch_dimension(iterator):
@@ -463,43 +489,6 @@ def get_batch_dimension(iterator):
   return dims[0] if dims else None
 
 
-def get_batch_size(distribution_strategy, num_samples, steps):
-  """Calculate and return batch size for numpy inputs.
-
-  Args:
-    distribution_strategy: The current DistributionStrategy used to compile the
-      model.
-    num_samples: Total number of input samples in the input numpy arrays.
-    steps: Number of steps that we run the model for.
-
-  Returns:
-    batch size used to create the Dataset object from the input numpy arrays.
-
-  """
-  if num_samples % steps != 0:
-    logging.warning('The number of input samples %d is not evenly '
-                    'divisible by the number of steps %d. '
-                    'Some samples will not be processed as expected.' %
-                    (num_samples, steps))
-  global_batch_size = num_samples // steps
-
-  # TODO(b/118776054): Use global batch size for Keras/DS support.
-  # The Keras API supports using the global batch size which is currently only
-  # supported in TPU Strategy. For other strategies we use a per_replica
-  # batch size so we need to divide it by the number of replicas.
-  if global_batch_size_supported(distribution_strategy):
-    return global_batch_size
-
-  num_replicas = distribution_strategy.num_replicas_in_sync
-  if global_batch_size % num_replicas != 0:
-    logging.warning('The total number of batches per step %d is not evenly '
-                    'divisible by the number of replicas %d used in '
-                    'DistributionStrategy. Some samples will not be processed '
-                    'as expected.' %
-                    (global_batch_size, num_replicas))
-  return global_batch_size // num_replicas
-
-
 def get_cpu_device(distribution_strategy):
   """Returns the CPU device of the TPU host or the default CPU device string.
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 032ebeaeca..edd6b69dcc 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1002,11 +1002,6 @@ class Model(Network):
 
     first_x_value = nest.flatten(x)[0]
     if isinstance(first_x_value, np.ndarray):
-      assert steps is not None
-      x_shape = first_x_value.shape
-      if batch_size is None:
-        batch_size = distributed_training_utils.get_batch_size(
-            self._distribution_strategy, x_shape[0], steps)
       # We need to use the drop_remainder argument to allow for a static
       # input shape which is required for TPUs.
       drop_remainder = self._distribution_strategy.require_static_shapes
@@ -1041,7 +1036,6 @@ class Model(Network):
         var_x = distributed_training_utils.get_var_for_numpy(
             self._distribution_strategy, x)
         x = dataset_ops.Dataset.from_tensor_slices(var_x)
-        x = x.repeat()
         x = x.batch(batch_size, drop_remainder=drop_remainder)
 
     assert isinstance(x, dataset_ops.Dataset)
@@ -1661,9 +1655,11 @@ class Model(Network):
           x, y, self._distribution_strategy)
 
       first_x_value = nest.flatten(x)[0]
-      if not steps_per_epoch and isinstance(first_x_value, np.ndarray):
-        steps_per_epoch = distributed_training_utils.get_input_batch_params(
-            first_x_value, batch_size, self._distribution_strategy)
+      if isinstance(first_x_value, np.ndarray):
+        steps_per_epoch, batch_size = (
+            distributed_training_utils.get_input_params(
+                self._distribution_strategy, first_x_value, steps_per_epoch,
+                batch_size, is_training=True))
 
     # Backwards compatibility
     if batch_size is None and steps_per_epoch is None:
@@ -1708,9 +1704,10 @@ class Model(Network):
         distributed_training_utils.validate_inputs(
             val_x, val_y, self._distribution_strategy)
         first_valx_value = nest.flatten(val_x)[0]
-        if not validation_steps and isinstance(first_valx_value, np.ndarray):
-          validation_steps = distributed_training_utils.get_input_batch_params(
-              first_valx_value, batch_size, self._distribution_strategy)
+        if isinstance(first_valx_value, np.ndarray):
+          validation_steps, _ = distributed_training_utils.get_input_params(
+              self._distribution_strategy, first_valx_value, validation_steps,
+              batch_size)
 
       val_x, val_y, val_sample_weights = self._standardize_user_data(
           val_x,
@@ -1893,9 +1890,9 @@ class Model(Network):
       distributed_training_utils.validate_inputs(
           x, y, self._distribution_strategy)
       first_x_value = nest.flatten(x)[0]
-      if isinstance(first_x_value, np.ndarray) and not steps:
-        steps = distributed_training_utils.get_input_batch_params(
-            first_x_value, batch_size, self._distribution_strategy)
+      if isinstance(first_x_value, np.ndarray):
+        steps, batch_size = distributed_training_utils.get_input_params(
+            self._distribution_strategy, first_x_value, steps, batch_size)
 
     # Backwards compatibility.
     if batch_size is None and steps is None:
@@ -2006,18 +2003,25 @@ class Model(Network):
       distributed_training_utils.validate_inputs(
           x, None, self._distribution_strategy)
       first_x_value = nest.flatten(x)[0]
-      if isinstance(first_x_value, np.ndarray) and not steps:
-        steps = distributed_training_utils.get_input_batch_params(
-            first_x_value, batch_size, self._distribution_strategy)
+      if isinstance(first_x_value, np.ndarray):
+        steps, batch_size = distributed_training_utils.get_input_params(
+            self._distribution_strategy, first_x_value, steps, batch_size)
+
     # Backwards compatibility.
     if batch_size is None and steps is None:
       batch_size = 32
 
     # Validate and standardize user data.
-    # TODO(anjalisridhar): We don't pass batch_size here for some reason. This
-    # means that we end up calculating it twice which we should avoid.
-    x, _, _ = self._standardize_user_data(
-        x, check_steps=True, steps_name='steps', steps=steps)
+    if self._distribution_strategy:
+      x, _, _ = self._standardize_user_data(
+          x, check_steps=True, steps_name='steps', steps=steps,
+          batch_size=batch_size)
+    else:
+      # TODO(anjalisridhar): We don't pass batch_size here for some reason. This
+      # means we need to special case distribution strategy which needs the
+      # batch size.
+      x, _, _ = self._standardize_user_data(
+          x, check_steps=True, steps_name='steps', steps=steps)
 
     if self.run_eagerly:
       return training_eager.predict_loop(
-- 
GitLab


From 3affd7655e2318d3b12c9513fbacd6aae796c192 Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Mon, 19 Nov 2018 18:02:45 -0800
Subject: [PATCH 0553/1554] [XLA] Remove duplicated XlaOp comments for private
 XlaOp methods.

PiperOrigin-RevId: 222171307
---
 tensorflow/compiler/xla/client/xla_builder.h | 296 +------------------
 1 file changed, 5 insertions(+), 291 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index e6ee41b300..78c90dbccc 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -280,31 +280,14 @@ class XlaBuilder {
   // Build helper which takes the id of the root operation..
   StatusOr<XlaComputation> Build(int64 root_id);
 
-  // Enqueues a "retrieve parameter value" instruction for a parameter that was
-  // passed to the computation.
+  // Description for the methods below can be found in the corresponding public
+  // functions section in this file.
+
   XlaOp Parameter(int64 parameter_number, const Shape& shape,
                   const string& name);
 
-  // Enqueues a constant with the value of the given literal onto the
-  // computation.
   XlaOp ConstantLiteral(const LiteralSlice& literal);
 
-  // Enqueues a constant onto the computation. Methods are templated on the
-  // native host type (NativeT) which corresponds to a specific XLA
-  // PrimitiveType as given in the following table:
-  //
-  //  Native Type   PrimitiveType
-  // -----------------------------
-  //   bool           PRED
-  //   int32          S32
-  //   int64          S64
-  //   uint32         U32
-  //   uint64         U64
-  //   float          F32
-  //   double         F64
-  //
-  // Note: not all primitive types defined in xla_data.proto have a
-  // corresponding native type yet.
   template <typename NativeT>
   XlaOp ConstantR0(NativeT value);
   template <typename NativeT>
@@ -334,181 +317,78 @@ class XlaBuilder {
   template <typename NativeT>
   XlaOp ConstantR4FromArray4D(const Array4D<NativeT>& values);
 
-  // Enqueues a rank one constant (vector) onto the computation. The vector has
-  // size 'length' and every element has the value 'value'.
   template <typename NativeT>
   XlaOp ConstantR1(int64 length, NativeT value);
 
-  // Adds dimensions to an array by duplicating the data in the array.
-  //
-  // The new dimensions are inserted on the left, i.e. if
-  // broadcast_sizes has values {a0, ..., aN} and the operand shape
-  // has dimensions {b0, ..., bM} then the shape of the output has
-  // dimensions {a0, ..., aN, b0, ..., bM}.
-  //
-  // The new dimensions index into copies of the operand, i.e.
-  //
-  //   output[i0, ..., iN, j0, ..., jM] = operand[j0, ..., jM]
   XlaOp Broadcast(const XlaOp& operand,
                   absl::Span<const int64> broadcast_sizes);
 
   XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape,
                        const absl::Span<const int64> broadcast_dimensions);
 
-  // Enqueues a pad operation onto the computation that pads the given value on
-  // the edges as well as between the elements of the input. padding_config
-  // specifies the padding amount for each dimension.
   XlaOp Pad(const XlaOp& operand, const XlaOp& padding_value,
             const PaddingConfig& padding_config);
 
-  // Enqueues an operation onto the computation that flattens the operand based
-  // on the dimension order (major/slowest-varying to minor/fastest-varying)
-  // given, followed by reshaping it into the shape with the given dimension
-  // sizes (also major to minor). Conceptually, this is a limited form of
-  // "shape casting".
   XlaOp Reshape(const XlaOp& operand, absl::Span<const int64> dimensions,
                 absl::Span<const int64> new_sizes);
 
-  // Enqueues an operation onto the computation that collapses the operand, from
-  // first to last dimension (C order), then reshapes it to the given dimension
-  // sizes. Conceptually, this is a limited form of "shape casting".
   XlaOp Reshape(const XlaOp& operand, absl::Span<const int64> new_sizes);
 
-  // Wrapper for Reshape.
-  // Enqueues an operation to collapse the provided dimensions; e.g. an
-  // operand with dimensions {x=256, y=2, z=2, p=32} can be collapsed to
-  // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
-  // be a consecutive, in-order subsequence of the operand dimensions.
-  //
-  // Note that collapsing a single dimension does nothing:
-  //
-  //    {256} collapsing {0} => {256}
-  //    {1} collapsing {0} => {1}
-  //
-  // Collapsing multiple dimensions produces a single result dimension:
-  //
-  //    {256, 2} collapsing {0,1} => {512}
-  //    {256, 2, 3} collapsing {0,1} => {512, 3}
-  //
-  // This could potentially cause data to be moved -- it provides a more
-  // structured form of reshaping than an arbitrary Reshape operation.
   XlaOp Collapse(const XlaOp& operand, absl::Span<const int64> dimensions);
 
-  // Enqueues a slice operation onto the computation that slices the operand
-  // from the start indices to the limit indices; e.g.
-  //
-  //        x
-  //   [ 0 1 2 3 ]
-  // y [ 4 5 6 7 ] => slice(start={1, 1}, limit={2, 3}) => [ 5 6 ]
-  //   [ 8 9 a b ]
-  //
-  // Note that "limit" means up-to-but-not-including; i.e. [start, limit) in 1D
-  // range notation.
-  // The strides parameter determines the stride over the slice
   XlaOp Slice(const XlaOp& operand, absl::Span<const int64> start_indices,
               absl::Span<const int64> limit_indices,
               absl::Span<const int64> strides);
 
-  // Enqueues a slice operation in a given dimension, taking all other
-  // dimensions as they are; e.g. if dimno is 1 from start_index 2 to
-  // limit_index 4 by 1, and the shape is f32[7,8,9], this call is short-hand
-  // for:
-  //
-  //  array[:, 2:4:1, :]
   XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index,
                    int64 stride, int64 dimno);
 
-  // Enqueues a slice operation onto the computation that slices the 'operand'
-  // from dynamic start indices which are passed in 'start_indices'.
-  // The size of the slice in each dimension is passed in 'slice_sizes',
-  // which specify the end point of exclusive slice intervals in each
-  // dimension [start, start + size).
-  // The shape of 'start_indices' must be rank == 1, with dimension size
-  // equal to the rank of the 'operand'.
-  // Slice index calculations are computed modulo input dimension sizes to
-  // prevent dynamic start indices from generating out-of-bound array accesses.
   XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                      absl::Span<const int64> slice_sizes);
 
-  // Enqueues a dynamic update slice operation onto the computation, which
-  // updates a slice of 'operand' with 'update' at dynamic 'start_indices'.
-  // The shape of 'update' determines the shape of the slice of 'operand'
-  // which is updated.
-  // The indices specified in 'start_indices' specify the offset of the slice
-  // of 'operand' which is updated.
-  //
-  //               update = {10, 11} // calculated at runtime.
-  //   [1 2 3]     start  = {1, 1}   // calculated at runtime.  [1 2  3 ]
-  //   [4 5 6]  => DynamicUpdateslice(data, update, start)   => [4 10 11]
-  //   [7 8 9]                                                  [7 8  9 ]
-  //
-  // The shape of 'start_indices' must be rank == 1, with dimension size
-  // equal to the rank of the 'operand'.
-  // Slice index calculations are computed modulo update dimension sizes to
-  // prevent dynamic start indices from generating out-of-bound array accesses.
   XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                            const XlaOp& start_indices);
 
-  // Enqueues a concatenate instruction onto the computation. 'operands' must
-  // have >= 1 entry.
   XlaOp ConcatInDim(absl::Span<const XlaOp> operands, int64 dimension);
 
-  // Enqueue a tracing operation onto the computation; the computation will emit
-  // a logging message with the operand.
   void Trace(const string& tag, const XlaOp& operand);
 
-  // Enqueues a conditional-move-like select operation onto the computation;
-  // predicated on pred, selects between on_true and on_false.
   XlaOp Select(const XlaOp& pred, const XlaOp& on_true, const XlaOp& on_false);
 
-  // Enqueues a tuple-creation instruction onto the computation.
   XlaOp Tuple(absl::Span<const XlaOp> elements);
 
-  // Enqueues a tuple-element-get instruction onto the computation.
   XlaOp GetTupleElement(const XlaOp& tuple_data, int64 index);
 
-  // Enqueues an equal-to comparison instruction onto the computation.
   XlaOp Eq(const XlaOp& lhs, const XlaOp& rhs,
            absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a not-equal comparison instruction onto the computation.
   XlaOp Ne(const XlaOp& lhs, const XlaOp& rhs,
            absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a greater-or-equal comparison instruction onto the computation.
   XlaOp Ge(const XlaOp& lhs, const XlaOp& rhs,
            absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a greater-than comparison instruction onto the computation.
   XlaOp Gt(const XlaOp& lhs, const XlaOp& rhs,
            absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a less-than comparison instruction onto the computation.
   XlaOp Lt(const XlaOp& lhs, const XlaOp& rhs,
            absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a less-or-equal comparison instruction onto the computation.
   XlaOp Le(const XlaOp& lhs, const XlaOp& rhs,
            absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a dot instruction onto the computation.
   XlaOp Dot(const XlaOp& lhs, const XlaOp& rhs,
             const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues a general dot instruction onto the computation.
   XlaOp DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
                    const DotDimensionNumbers& dimension_numbers,
                    const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues a convolution instruction onto the computation, which uses the
-  // default convolution dimension numbers.
   XlaOp Conv(const XlaOp& lhs, const XlaOp& rhs,
              absl::Span<const int64> window_strides, Padding padding,
              int64 feature_group_count = 1,
              const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues a convolution instruction onto the computation, with the caller
-  // provided padding configuration in the format returned by MakePadding().
   XlaOp ConvWithGeneralPadding(
       const XlaOp& lhs, const XlaOp& rhs,
       absl::Span<const int64> window_strides,
@@ -516,8 +396,6 @@ class XlaBuilder {
       int64 feature_group_count = 1,
       const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues a convolution instruction onto the computation, with the caller
-  // provided dimension numbers configuration.
   XlaOp ConvWithGeneralDimensions(
       const XlaOp& lhs, const XlaOp& rhs,
       absl::Span<const int64> window_strides, Padding padding,
@@ -525,8 +403,6 @@ class XlaBuilder {
       int64 feature_group_count = 1,
       const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues a convolution instruction onto the computation, with the caller
-  // provided padding configuration as well as the dimension numbers.
   XlaOp ConvGeneral(const XlaOp& lhs, const XlaOp& rhs,
                     absl::Span<const int64> window_strides,
                     absl::Span<const std::pair<int64, int64>> padding,
@@ -534,8 +410,6 @@ class XlaBuilder {
                     int64 feature_group_count = 1,
                     const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues a convolution instruction onto the computation, with the caller
-  // provided padding configuration, dilation factors and dimension numbers.
   XlaOp ConvGeneralDilated(const XlaOp& lhs, const XlaOp& rhs,
                            absl::Span<const int64> window_strides,
                            absl::Span<const std::pair<int64, int64>> padding,
@@ -545,80 +419,53 @@ class XlaBuilder {
                            int64 feature_group_count = 1,
                            const PrecisionConfig* precision_config = nullptr);
 
-  // Enqueues an FFT instruction onto the computation, of the given type and
-  // with the given FFT length.
   XlaOp Fft(const XlaOp& operand, FftType fft_type,
             absl::Span<const int64> fft_length);
 
-  // Enqueues an infeed instruction onto the computation, which writes data of
-  // the given shape to the infeed buffer of the device.
   XlaOp Infeed(const Shape& shape, const string& config = "");
   XlaOp InfeedWithToken(const XlaOp& token, const Shape& shape,
                         const string& config = "");
 
-  // Enqueues an outfeed instruction onto the computation. This instruction
-  // generates outgoing data transfers for the given data.
-  //
-  // shape_with_layout communicates the laid out shape that we want to outfeed
-  // -- if !ShapeUtil::Compatible(GetShape(operand), shape_with_layout) an error
-  // will occur.
   void Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
                const string& outfeed_config);
   XlaOp OutfeedWithToken(const XlaOp& operand, const XlaOp& token,
                          const Shape& shape_with_layout,
                          const string& outfeed_config);
 
-  // Enqueues a call instruction onto the computation.
   XlaOp Call(const XlaComputation& computation,
              absl::Span<const XlaOp> operands);
 
-  // Enqueues a custom call instruction onto the computation.
   XlaOp CustomCall(
       const string& call_target_name, absl::Span<const XlaOp> operands,
       const Shape& shape_with_layout, const string& opaque,
       absl::optional<absl::Span<const Shape>> operand_shapes_with_layout);
 
-  // The following methods enqueue element-wise binary arithmetic operations
-  // onto the computation. The shapes of the operands have to match unless one
-  // of the operands is a scalar, or an explicit broadcast dimension is given
-  // (see g3doc for more details).
-
-  // Enqueues a complex compose instruction onto the computation.
   XlaOp Complex(const XlaOp& real, const XlaOp& imag,
                 absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a complex conjugate instruction onto the computation.
   XlaOp Conj(const XlaOp& operand);
 
-  // Enqueues an add instruction onto the computation.
   XlaOp Add(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a subtract instruction onto the computation.
   XlaOp Sub(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a multiply instruction onto the computation.
   XlaOp Mul(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a divide instruction onto the computation.
   XlaOp Div(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a remainder instruction onto the computation.
   XlaOp Rem(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a max instruction onto the computation.
   XlaOp Max(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues a min instruction onto the computation.
   XlaOp Min(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Element-wise logical operators
   XlaOp And(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
@@ -637,32 +484,23 @@ class XlaBuilder {
   XlaOp ShiftRightLogical(const XlaOp& lhs, const XlaOp& rhs,
                           absl::Span<const int64> broadcast_dimensions = {});
 
-  // Reduces an array among the provided dimensions, given "computation" as a
-  // reduction operator.
   XlaOp Reduce(const XlaOp& operand, const XlaOp& init_value,
                const XlaComputation& computation,
                absl::Span<const int64> dimensions_to_reduce);
 
-  // Reduces several arrays simultaneously among the provided dimensions, given
-  // "computation" as a reduction operator.
   XlaOp Reduce(absl::Span<const XlaOp> operands,
                absl::Span<const XlaOp> init_values,
                const XlaComputation& computation,
                absl::Span<const int64> dimensions_to_reduce);
 
-  // Convenience wrapper around the above that reduces all the dimensions in the
-  // operand shape.
   XlaOp ReduceAll(const XlaOp& operand, const XlaOp& init_value,
                   const XlaComputation& computation);
 
-  // Enqueues a windowed reduce instruction onto the computation.
   XlaOp ReduceWindow(const XlaOp& operand, const XlaOp& init_value,
                      const XlaComputation& computation,
                      absl::Span<const int64> window_dimensions,
                      absl::Span<const int64> window_strides, Padding padding);
 
-  // As ReduceWindow(), but the padding is given in the format
-  // returned by MakePadding().
   XlaOp ReduceWindowWithGeneralPadding(
       const XlaOp& operand, const XlaOp& init_value,
       const XlaComputation& computation,
@@ -672,48 +510,22 @@ class XlaBuilder {
       absl::Span<const int64> window_dilations,
       absl::Span<const std::pair<int64, int64>> padding);
 
-  // Returns the sum of the operand value within each subgroup of replicas. All
-  // replicas supply one input to the sum and all replicas receive the resulting
-  // sum for each subgroup.
   XlaOp CrossReplicaSum(const XlaOp& operand,
                         absl::Span<const ReplicaGroup> replica_groups = {});
 
-  // Enqueues an operation that do an AllReduce of the operand cross cores. Here
-  // AllReduce means doing a reduction on the input operand cross cores and then
-  // broadcasting the reduction result to those cores. The reduction function is
-  // defined by `computation`, which should be a commutative computation on
-  // scalars, e.g., add, min, or max. The way that AllReduce is applied is
-  // configured by:
-  //
-  // - `replica_groups`: each ReplicaGroup contains a list of replica id. If
-  // empty, all replicas belong to one group. Allreduce will be applied within
-  // subgroups. For example, we have 4 replicas, then
-  // replica_groups={{0,2},{1,3}} means, replica 0 and 2 are in subgroup 0,
-  // replica 1 and 3 are in subgroup 1.
-  //
-  // - `channel_id`: for Allreduce nodes from different modules, if they have
-  // the same channel_id, they will be 'Allreduce'd. If empty, Allreduce will
-  // not be applied cross modules.
-  //
-  // TODO(b/117564385): Rename this to AllReduce when it's ready to use.
   XlaOp CrossReplicaSum(
       const XlaOp& operand, const XlaComputation& computation,
       absl::Span<const ReplicaGroup> replica_groups = {},
       const absl::optional<ChannelHandle>& channel_id = absl::nullopt);
 
-  // Enqueues an operation that do an Alltoall of the operand cross cores.
   XlaOp AllToAll(const XlaOp& operand, int64 split_dimension,
                  int64 concat_dimension, int64 split_count,
                  const std::vector<ReplicaGroup>& replica_groups);
 
-  // Enqueues an operation that do an CollectivePermute of the operand cross
-  // cores.
   XlaOp CollectivePermute(
       const XlaOp& operand,
       const std::vector<std::pair<int64, int64>>& source_target_pairs);
 
-  // Enqueues an operation that scatters the `source` array to the selected
-  // indices of each window.
   XlaOp SelectAndScatter(const XlaOp& operand, const XlaComputation& select,
                          absl::Span<const int64> window_dimensions,
                          absl::Span<const int64> window_strides,
@@ -721,8 +533,6 @@ class XlaBuilder {
                          const XlaOp& init_value,
                          const XlaComputation& scatter);
 
-  // As SelectAndScatter(), but the padding is given in the format
-  // returned by MakePadding().
   XlaOp SelectAndScatterWithGeneralPadding(
       const XlaOp& operand, const XlaComputation& select,
       absl::Span<const int64> window_dimensions,
@@ -730,217 +540,119 @@ class XlaBuilder {
       absl::Span<const std::pair<int64, int64>> padding, const XlaOp& source,
       const XlaOp& init_value, const XlaComputation& scatter);
 
-  // Enqueues an abs instruction onto the computation.
   XlaOp Abs(const XlaOp& operand);
 
-  // Enqueues a atan2 instruction onto the computation.
   XlaOp Atan2(const XlaOp& y, const XlaOp& x,
               absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues an exp instruction onto the computation.
   XlaOp Exp(const XlaOp& operand);
 
-  // Enqueues an expm1 instruction onto the computation.
   XlaOp Expm1(const XlaOp& operand);
 
-  // Enqueues a floor instruction onto the computation.
   XlaOp Floor(const XlaOp& operand);
 
-  // Enqueues a ceil instruction onto the computation.
   XlaOp Ceil(const XlaOp& operand);
 
-  // Enqueues a round instruction onto the computation, rounding to nearest even
-  // with half-way cases rounding away from zero.
   XlaOp Round(const XlaOp& operand);
 
-  // Enqueues an log instruction (natural logarithm) onto the computation.
   XlaOp Log(const XlaOp& operand);
 
-  // Enqueues an log1p instruction (log(x+1)) onto the computation.
   XlaOp Log1p(const XlaOp& operand);
 
-  // Enqueues a sign instruction onto the computation.
   XlaOp Sign(const XlaOp& operand);
 
-  // Enqueues a count leading zeros instruction onto the computation.
   XlaOp Clz(const XlaOp& operand);
 
-  // Enqueues a cosine instruction onto the computation.
   XlaOp Cos(const XlaOp& operand);
 
-  // Enqueues a sine instruction onto the computation.
   XlaOp Sin(const XlaOp& operand);
 
-  // Enqueues a tanh instruction onto the computation.
   XlaOp Tanh(const XlaOp& operand);
 
-  // Enqueues a real-part instruction onto the computation.
   XlaOp Real(const XlaOp& operand);
 
-  // Enqueues an imaginary-part instruction onto the computation.
   XlaOp Imag(const XlaOp& operand);
 
-  // Enqueues a lhs^rhs computation onto the computation.
   XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs,
             absl::Span<const int64> broadcast_dimensions = {});
 
-  // Enqueues an operator that tests if the operand's values are finite, i.e.,
-  // not Inf or NaN. Defined only for floating-point types. Returns an array of
-  // booleans with the same shape where entries are true iff the corresponding
-  // entry was NaN.
   XlaOp IsFinite(const XlaOp& operand);
 
-  // Enqueues an iota operation onto the computation.
   XlaOp Iota(const Shape& shape, int64 iota_dimension);
 
-  // Enqueues a rank-1 iota operation onto the computation.
   XlaOp Iota(PrimitiveType type, int64 size);
 
-  // Enqueues a convert instruction onto the computation that changes the
-  // element type of the operand array to primitive_type.
   XlaOp ConvertElementType(const XlaOp& operand,
                            PrimitiveType new_element_type);
 
-  // Enqueues a no-op instruction onto the computation that changes
-  // the element type of the operand array to primitive_type. The
-  // bit-widths of the source and destination element types must be
-  // identical.
   XlaOp BitcastConvertType(const XlaOp& operand,
                            PrimitiveType new_element_type);
 
-  // Enqueues a negate instruction onto the computation.
   XlaOp Neg(const XlaOp& operand);
 
-  // Enqueues a transpose instruction onto the computation.
   XlaOp Transpose(const XlaOp& operand, absl::Span<const int64> permutation);
 
-  // Enqueues a reverse instruction onto the computation. The order of the
-  // elements in the given dimensions is reversed (i.e., the element at index i
-  // is moved to index dimension_size - 1 - i).
   XlaOp Rev(const XlaOp& operand, absl::Span<const int64> dimensions);
 
-  // Enqueues a sort (as increasing order) instruction onto the computation.
-  // If only keys are provided:
-  // * If the keys are an rank-1 tensor (an array), the result is a sorted array
-  // of keys, in ascending order.
-  // * If the keys have higher rank, the keys are sorted along the provided
-  // dimension. For example, for a rank-2 tensor (a matrix) of keys, a dimension
-  // value of 0 will indepenently sort every column, and a dimension value of 1
-  // will independently sort each row. If no dimension number is provided, then
-  // the last dimension is chosen by default.
-  //
-  // If both keys and values are provided:
-  // * The keys and all values must be tensors with the same dimensions. The
-  // element types of the tensors may be different.
-  // * The result is a tuple that consists of a sorted tensor of keys (along the
-  // provided dimension, as above) as the first element, and tensors with their
-  // corresponding values as the other elements.
   XlaOp Sort(const XlaOp& keys, absl::Span<const XlaOp> values = {},
              int64 dimension = -1);
 
-  // Enqueues a clamp instruction onto the computation.
   XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max);
 
-  // Enqueues a map instruction onto the computation.
   XlaOp Map(absl::Span<const XlaOp> operands, const XlaComputation& computation,
             absl::Span<const int64> dimensions,
             absl::Span<const XlaOp> static_operands = {});
 
-  // Enqueues a N(mu, sigma) random number generation instruction onto the
-  // computation.
   XlaOp RngNormal(const XlaOp& mu, const XlaOp& sigma, const Shape& shape);
 
-  // Enqueues a U(a, b) random number generation instruction onto the
-  // computation. Returns values in the semi-open interval [a, b).
   XlaOp RngUniform(const XlaOp& a, const XlaOp& b, const Shape& shape);
 
-  // Enqueues a while node onto the computation.
   XlaOp While(const XlaComputation& condition, const XlaComputation& body,
               const XlaOp& init);
 
-  // Enqueues a conditional node onto the computation.
   XlaOp Conditional(const XlaOp& predicate, const XlaOp& true_operand,
                     const XlaComputation& true_computation,
                     const XlaOp& false_operand,
                     const XlaComputation& false_computation);
 
-  // Enqueues a ReducePrecision node onto the computation.
   XlaOp ReducePrecision(const XlaOp& operand, const int exponent_bits,
                         const int mantissa_bits);
 
-  // Enqueues a Gather node onto the computation.
   XlaOp Gather(const XlaOp& input, const XlaOp& start_indices,
                const GatherDimensionNumbers& dimension_numbers,
                absl::Span<const int64> slice_sizes);
 
-  // Enqueues a Scatter node onto the computation.
   XlaOp Scatter(const XlaOp& input, const XlaOp& scatter_indices,
                 const XlaOp& updates, const XlaComputation& update_computation,
                 const ScatterDimensionNumbers& dimension_numbers);
 
-  // Enqueues a Send node onto the computation for device-to-device
-  // communication, to send the given operand to a Recv instruction that shares
-  // the same channel handle.
   void Send(const XlaOp& operand, const ChannelHandle& handle);
   XlaOp SendWithToken(const XlaOp& operand, const XlaOp& token,
                       const ChannelHandle& handle);
 
-  // Enqueues a Send node which sends data to the host.
   XlaOp SendToHost(const XlaOp& operand, const XlaOp& token,
                    const Shape& shape_with_layout, const ChannelHandle& handle);
 
-  // Enqueues a Recv node which receives data from the host.
   XlaOp RecvFromHost(const XlaOp& token, const Shape& shape,
                      const ChannelHandle& handle);
 
-  // Enqueues an AfterAll operation with no operands producing a token-shaped
-  // value.
   XlaOp CreateToken();
 
-  // Enqueues an AfterAll operation with no operands producing a token-shaped
-  // value.
   XlaOp AfterAll(absl::Span<const XlaOp> tokens);
 
-  // Enqueues a Recv node onto the computation. The data comes from a Send
-  // instruction that shares the same channel handle and its shape must
-  // be the same as the given shape.
   XlaOp Recv(const Shape& shape, const ChannelHandle& handle);
   XlaOp RecvWithToken(const XlaOp& token, const Shape& shape,
                       const ChannelHandle& handle);
 
-  // Normalizes operand across spatial and batch dimensions for each feature.
-  //
-  // Returns a tuple (normalized, batch_mean, batch_var) where `normalized`
-  // is the normalized result and batch_mean and batch_var are the mean and
-  // variance, respectively, across batch for the operand.
   XlaOp BatchNormTraining(const XlaOp& operand, const XlaOp& scale,
                           const XlaOp& offset, float epsilon,
                           int64 feature_index);
 
-  // Normalizes operand across spatial and batch dimensions for each feature.
-  //
-  // `BatchNormInference` is equivalent to calling `BatchNormTraining` without
-  // computing `mean` and `variance` for each batch inside the operation. It
-  // uses the input `mean` and `variance` instead as estimated values. The
-  // purpose of this op is to reduce latency in inference, hence the name
-  // `BatchNormInference`.
-  //
-  // The output has the same shape as `operand`, and contains the normalized
-  // values for each batch.
   XlaOp BatchNormInference(const XlaOp& operand, const XlaOp& scale,
                            const XlaOp& offset, const XlaOp& mean,
                            const XlaOp& variance, float epsilon,
                            int64 feature_index);
 
-  // Calculates the gradients of a batch norm op.
-  //
-  // The inputs `batch_mean` and `batch_var` represent the mean and variance
-  // across the batch.
-  //
-  // Returns a tuple of three elements:
-  //   - grad_operand: Gradient with respect to input `operand`
-  //   - grad_offset: Gradient with respect to input `offset`
-  //   - grad_scale: Gradient with respect to input `scale`
   XlaOp BatchNormGrad(const XlaOp& operand, const XlaOp& scale,
                       const XlaOp& batch_mean, const XlaOp& batch_var,
                       const XlaOp& grad_output, float epsilon,
@@ -1409,6 +1121,7 @@ class XlaScopedShardingAssignment {
 // Free functions for building XlaOps. The intention is that these will
 // become the public API for building XlaOps rather than calling methods on
 // XlaBuilder directly.
+//
 
 // Enqueues a "retrieve parameter value" instruction for a parameter that was
 // passed to the computation.
@@ -2154,6 +1867,7 @@ XlaOp BatchNormGrad(const XlaOp& operand, const XlaOp& scale,
 XlaOp GetDimensionSize(const XlaOp& operand, int64 dimension);
 
 // Implementation details below this point.
+//
 
 template <typename NativeT>
 XlaOp XlaBuilder::ConstantR0(NativeT value) {
-- 
GitLab


From be27c92dde98b08f5e2e4cf501584fa442bcb905 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 18:56:17 -0800
Subject: [PATCH 0554/1554] fix unrollbatch matmul toco transformation.

PiperOrigin-RevId: 222176219
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/kernels/fully_connected.cc    |  2 +-
 tensorflow/lite/testing/generate_examples.py  | 26 +++++++++++++++++++
 .../unroll_batch_matmul.cc                    |  3 ++-
 4 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 59b1d9d2a0..8255211d27 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -303,6 +303,7 @@ def generated_test_models():
         "transpose",
         "transpose_conv",
         "unpack",
+        "unroll_batch_matmul",
         "where",
         "zeros_like",
     ]
diff --git a/tensorflow/lite/kernels/fully_connected.cc b/tensorflow/lite/kernels/fully_connected.cc
index 63cca1cf54..a1eecb284a 100644
--- a/tensorflow/lite/kernels/fully_connected.cc
+++ b/tensorflow/lite/kernels/fully_connected.cc
@@ -117,7 +117,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   // Note that quantized inference requires that all tensors have their
   // parameters set. This is usually done during quantized training.
   TfLiteType data_type = input->type;
-  if (data_type != kTfLiteFloat32) {
+  if (data_type != kTfLiteFloat32 && data_type != kTfLiteInt32) {
     double real_multiplier = 0.0;
     TF_LITE_ENSURE_STATUS(GetQuantizedConvolutionMultipler(
         context, input, filter, bias, output, &real_multiplier));
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index eccef341a0..9b0f59f9da 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3470,6 +3470,32 @@ def make_logical_xor_tests(zip_path):
   return _make_logical_tests(tf.logical_xor)(zip_path)
 
 
+def make_unroll_batch_matmul_tests(zip_path):
+  """Make a set of tests to test unroll_batch_matmul."""
+
+  test_parameters = [{"dtype": [tf.float32], "shape": [[(2, 2, 3), (2, 3, 2)]]}]
+
+  def build_graph(parameters):
+    """Build the batch_matmul op testing graph."""
+    input_tensor1 = tf.placeholder(
+        dtype=parameters["dtype"], shape=parameters["shape"][0])
+    input_tensor2 = tf.placeholder(
+        dtype=parameters["dtype"], shape=parameters["shape"][1])
+    # Should be unrolled and replaced with fully_connected ops in the end.
+    out = tf.matmul(input_tensor1, input_tensor2)
+    return [input_tensor1, input_tensor2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value1 = create_tensor_data(
+        parameters["dtype"], shape=parameters["shape"][0])
+    input_value2 = create_tensor_data(
+        parameters["dtype"], shape=parameters["shape"][1])
+    return [input_value1, input_value2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value1, input_value2])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 # Toco binary path provided by the generate rule.
 bin_path = None
 
diff --git a/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc b/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc
index d59954fc74..41a735394d 100644
--- a/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc
+++ b/tensorflow/lite/toco/graph_transformations/unroll_batch_matmul.cc
@@ -117,7 +117,8 @@ namespace toco {
     auto* slice_b_op = new SliceOperator;
     slice_b_op->inputs = {
         batch_op->inputs[1],
-        CreateInt32Array(model, batch_name + "/slice_b/slice/begin", {0, 0, 0}),
+        CreateInt32Array(model, batch_name + "/slice_b/slice/begin",
+                         {batch, 0, 0}),
         CreateInt32Array(
             model, batch_name + "/slice_b/slice/size",
             {1, input_array_b.shape().dims(1), input_array_b.shape().dims(2)}),
-- 
GitLab


From 435f4605b7d1e7d2c9d7937ab9c0cbf4069bdc74 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 19 Nov 2018 19:40:01 -0800
Subject: [PATCH 0555/1554] [tf.data] Shard the map_vectorization_test to avoid
 timeouts.

PiperOrigin-RevId: 222179740
---
 .../python/data/experimental/kernel_tests/optimization/BUILD     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 9946ef5a42..1d0e6af649 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -202,6 +202,7 @@ py_test(
     name = "map_vectorization_test",
     size = "medium",
     srcs = ["map_vectorization_test.py"],
+    shard_count = 8,
     srcs_version = "PY2AND3",
     tags = [
         "no_oss",
-- 
GitLab


From e13fa8d670b2dba7ba337471c918e5dbbeca3072 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Wed, 14 Nov 2018 08:33:51 -0800
Subject: [PATCH 0556/1554] TF-TRT: Improve log

Add more information to the log including unsupported ops.
---
 .../contrib/tensorrt/convert/convert_graph.cc | 13 +++++----
 .../contrib/tensorrt/convert/convert_nodes.cc |  2 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc |  4 +--
 .../contrib/tensorrt/segment/segment.cc       | 27 ++++++++++++++++---
 4 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index f95ffe4100..9428fad309 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -435,7 +435,8 @@ tensorflow::Status GetEngineInfo(
                  << "but this shouldn't have happened";
     info->device = *segment_devices.begin();
   } else {
-    LOG(ERROR) << "Can't find a device placement for the op!";
+    VLOG(1) << "No device is assigned to the segment. "
+            << "A device will be assigned during graph execution (inference).";
   }
   return Status::OK();
 }
@@ -875,10 +876,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       // need to check the input edges.
       [](const Edge* edge) { return true; }, OutputEdgeValidator(),
       segment_options, &initial_segments));
-  if (initial_segments.size() > 1) {
-    VLOG(0) << "MULTIPLE tensorrt candidate conversion: "
-            << initial_segments.size();
-  }
+  VLOG(0) << "Number of TensorRT candidate segments: "
+    << initial_segments.size();
 
   // Get the EngineInfo for each segment.
   std::unordered_map<string, tensorflow::Node*> node_map;
@@ -910,7 +909,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
              : EngineInfo::EngineType::TRTStatic);
     curr_engine.cached_engine_batches = params.cached_engine_batches;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
-    StrAppend(&curr_engine.engine_name, "my_trt_op_", t);
+    StrAppend(&curr_engine.engine_name, "TRTEngineOp_", t);
     status = RegisterSegmentFunctionToFunctionLibrary(
         &graph, curr_engine.segment_graph_def, curr_engine.engine_name);
     if (!status.ok()) {
@@ -971,7 +970,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                 &graph, alloc.get(), &engine_nodes);
     // If status is ok, we successfully added the node to the graph and can
     // remove segment ops. Otherwise graph is not modified.
-    string msg = StrCat("Engine ", engine.engine_name, " creation for segment ",
+    string msg = StrCat("Adding TensorRT node ", engine.engine_name, " for segment ",
                         i, ", composed of ",
                         converted_segments.at(i).first.size(), " nodes");
     if (VLOG_IS_ON(1)) {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index af9bbbfdfd..68015a9a78 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2982,7 +2982,7 @@ tensorflow::Status ConvertSegmentToGraphDef(
     }
   }
   *common_scope = local_scope;
-  VLOG(0) << "Segment @scope '" << local_scope << "', converted to graph";
+  VLOG(1) << "Converted TensorRT candidate segment @scope '" << local_scope << "' to a GraphDef";
   return tensorflow::Status::OK();
 }
 
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 019446813a..3cdad69f49 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -491,8 +491,8 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     }
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
-    VLOG(0) << name() << " Constructing a new engine with batch size "
-            << batch_size;
+    VLOG(0) << "Building a new TensorRT engine for " << name()
+            << " with batch size " << batch_size;
     // Up to this point, calibrator_ can never be empty, since otherwise it
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 4f64b7a952..9ee7b76e30 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -406,22 +406,43 @@ tensorflow::Status SegmentGraph(
   // Use a union-find to collect the nodes that belong to the same
   // segment. A node value of nullptr indicates that the node is not a candidate
   // for TRT.
+  std::unordered_set<string> unsupported_ops;
+  int num_unsupported_ops = 0;
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
     if (options.exclude_node_list.count(node->name()) != 0) {
-      VLOG(1) << "Not a TF-TRT candidate: " << node->name()
-              << " (excluded by segmenter option).";
+      VLOG(1) << "Not a TF-TRT candidate, "
+              << "(Op type: " << node->tf_node()->type_string() << "), "
+              << "(Op name: " << node->name() << "), "
+              << "(Reason: excluded by segmenter option)";
+      unsupported_ops.emplace(node->tf_node()->type_string());
+      num_unsupported_ops++;
       node = nullptr;
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
-        VLOG(1) << "Not a TF-TRT candidate: " << node->name() << ": " << status;
+        VLOG(1) << "Not a TF-TRT candidate, "
+                << "(Op type: " << node->tf_node()->type_string() << "), "
+                << "(Op name: " << node->name() << "), "
+                << "(Reason: " << status << ")";
+        unsupported_ops.emplace(node->tf_node()->type_string());
+        num_unsupported_ops++;
         node = nullptr;
       }
     }
     node_segments.emplace_back(node);
   }
+  string msg = "There are "
+               + std::to_string(num_unsupported_ops)
+               + " unsupported ops of "
+               + std::to_string(unsupported_ops.size())
+               + " different types in the graph: ";
+  for (const auto& elem: unsupported_ops) {
+    msg += elem + ", ";
+  }
+  LOG(INFO) << msg << "(For more information see "
+    << "https://docs.nvidia.com/deeplearning/dgx/integrate-tf-trt/index.html#support-ops).";
 
   // The segmentation algorithm below visits nodes in reverse topological order
   // and attempts to merge nodes along output edges. That means that subgraphs
-- 
GitLab


From 65ab9a8abf45c763348af019f77be6c40a78e54b Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 19 Nov 2018 21:31:59 -0800
Subject: [PATCH 0557/1554] Revert absl version to latest healthy one for
 windows.

PiperOrigin-RevId: 222187718
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index bb339f60f2..14a5d0a25c 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "28a6cb644dcebe7d3e0ee347706fec2e6975fae2bceb0add834c77140c7b6632",
-        strip_prefix = "abseil-cpp-f6ae816808cd913e0e2b3e2af14f328fa1071af0",
+        sha256 = "3cf6132129ba87f0781c383bfaf381b7174b5818e81fffcc5d04bb451154f0f2",
+        strip_prefix = "abseil-cpp-f95179062eb65ce40895cc76f1398cce25394369",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f6ae816808cd913e0e2b3e2af14f328fa1071af0.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/f6ae816808cd913e0e2b3e2af14f328fa1071af0.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
         ],
     )
 
-- 
GitLab


From 3bb55e919b8afe3d466edf2392ade769f3196ba2 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 19 Nov 2018 21:45:31 -0800
Subject: [PATCH 0558/1554] Clean up all matches to the TPUStrategy to easily
 trace things we need to fix.

PiperOrigin-RevId: 222188691
---
 .../keras/engine/distributed_training_utils.py       | 12 +++++++++---
 tensorflow/python/keras/engine/training.py           | 12 ++++++++----
 .../python/keras/engine/training_distributed.py      |  6 ------
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 208602b90f..41da3930e2 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -333,7 +333,7 @@ def configure_and_create_session(distribution_strategy):
   # TODO(priyag): Throw error if a session already exists.
   session_config = K.get_default_session_config()
 
-  if type(distribution_strategy).__name__ == 'TPUStrategy':
+  if is_tpu_strategy(distribution_strategy):
     # TODO(priyag, yuefengz): Remove this workaround when Distribute
     # Coordinator is integrated with keras and we can create a session from
     # there.
@@ -379,7 +379,7 @@ def validate_inputs(x, y, distribution_strategy):
                      'Iterator. You must pass a `tf.data.Dataset` object or a '
                      'numpy array as input.')
 
-  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
+  if is_tpu_strategy(distribution_strategy):
     for i in [x, y]:
       if isinstance(i, dataset_ops.Dataset):
         shapes = nest.flatten(i.output_shapes)
@@ -401,6 +401,12 @@ def global_batch_size_supported(distribution_strategy):
   return strategy_name in ('TPUStrategy', 'CoreMirroredStrategy')
 
 
+# TODO(sourabhbajaj): Remove this once we use the same API for all strategies.
+def is_tpu_strategy(strategy):
+  """We're executing TPU Strategy."""
+  return strategy is not None and strategy.__class__.__name__ == 'TPUStrategy'
+
+
 def get_input_params(distribution_strategy, first_x_value, steps, batch_size,
                      is_training=False):
   """Calculate the number of batches and steps/steps_per_epoch.
@@ -504,7 +510,7 @@ def get_cpu_device(distribution_strategy):
     NotImplementedError: We currently don't support copying numpy data to
     multiple hosts in the case of Cloud TPU pods.
   """
-  if distribution_strategy.__class__.__name__ == 'TPUStrategy':
+  if is_tpu_strategy(distribution_strategy):
     if distribution_strategy.extended.num_hosts > 1:
       raise NotImplementedError('TPUDistributionStrategy does not '
                                 'support numpy inputs when running on Cloud'
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index edd6b69dcc..d926b53189 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -986,7 +986,8 @@ class Model(Network):
                                 'when using DistributionStrategy.')
 
     if (sample_weight is not None and sample_weight.all() and
-        self._distribution_strategy.__class__.__name__ == 'TPUStrategy'):
+        distributed_training_utils.is_tpu_strategy(
+            self._distribution_strategy)):
       raise NotImplementedError('`sample_weight` is currently not supported '
                                 'when using TPUStrategy.')
 
@@ -1755,7 +1756,8 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps)
-    elif training_distributed.should_run_experimental_loop(self):
+    elif distributed_training_utils.is_tpu_strategy(
+        self._distribution_strategy):
       return training_distributed.experimental_fit_loop(
           self,
           x,
@@ -1916,7 +1918,8 @@ class Model(Network):
           batch_size=batch_size,
           verbose=verbose,
           steps=steps)
-    elif training_distributed.should_run_experimental_loop(self):
+    elif distributed_training_utils.is_tpu_strategy(
+        self._distribution_strategy):
       return training_distributed.experimental_test_loop(
           self, iterator=x, verbose=verbose, steps=steps)
     elif isinstance(x, iterator_ops.EagerIterator):
@@ -2026,7 +2029,8 @@ class Model(Network):
     if self.run_eagerly:
       return training_eager.predict_loop(
           self, x, batch_size=batch_size, verbose=verbose, steps=steps)
-    elif training_distributed.should_run_experimental_loop(self):
+    elif distributed_training_utils.is_tpu_strategy(
+        self._distribution_strategy):
       return training_distributed.experimental_predict_loop(
           self, x, verbose=verbose, steps=steps)
     elif isinstance(x, iterator_ops.EagerIterator):
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 49ef44e4fd..878451d4cf 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -674,9 +674,3 @@ def _per_device_aggregate_batch(batch_outs, model, mode):
       total_batch_outs.append(np.concatenate(nest.flatten(nested_outs)))
     return total_batch_outs
   return batch_outs
-
-
-def should_run_experimental_loop(model):
-  """Whether to run the experimental loops in this file."""
-  return (hasattr(model, '_distribution_strategy') and
-          model._distribution_strategy.__class__.__name__ == 'TPUStrategy')
-- 
GitLab


From 89289dada9ba16617d125e4790c82e634b8d842b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 21:51:49 -0800
Subject: [PATCH 0559/1554] Replace iteration over depsets with an explicit
 .to_list() call.

The old pattern did an implicit iteration over a depset which will be forbidden in the future since it is potentially expensive. The new to_list() call is still expensive but it will be more visible.

PiperOrigin-RevId: 222189113
---
 tensorflow/tensorflow.bzl           | 4 ++--
 third_party/gpus/rocm_configure.bzl | 2 +-
 third_party/repo.bzl                | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index f01dc64bcf..2d67d1f466 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1307,13 +1307,13 @@ def _py_wrap_cc_impl(ctx):
         ctx.outputs.py_out.dirname,
     ]
     args += ["-l" + f.path for f in ctx.files.swig_includes]
-    args += ["-I" + i for i in swig_include_dirs]
+    args += ["-I" + i for i in swig_include_dirs.to_list()]
     args += [src.path]
     outputs = [ctx.outputs.cc_out, ctx.outputs.py_out]
     ctx.action(
         executable = ctx.executable._swig,
         arguments = args,
-        inputs = list(inputs),
+        inputs = inputs.to_list(),
         outputs = outputs,
         mnemonic = "PythonSwig",
         progress_message = "SWIGing " + src.path,
diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
index 9108639b0b..6df6799bd7 100644
--- a/third_party/gpus/rocm_configure.bzl
+++ b/third_party/gpus/rocm_configure.bzl
@@ -105,7 +105,7 @@ def get_cxx_inc_directories(repository_ctx, cc):
     return includes_cpp + [
         inc
         for inc in includes_c
-        if inc not in includes_cpp_set
+        if inc not in includes_cpp_set.to_list()
     ]
 
 def auto_configure_fail(msg):
diff --git a/third_party/repo.bzl b/third_party/repo.bzl
index 07b853ff11..bad6d20a08 100644
--- a/third_party/repo.bzl
+++ b/third_party/repo.bzl
@@ -84,7 +84,7 @@ def _apply_delete(ctx, paths):
 def _tf_http_archive(ctx):
     if ("mirror.bazel.build" not in ctx.attr.urls[0] and
         (len(ctx.attr.urls) < 2 and
-         ctx.attr.name not in _SINGLE_URL_WHITELIST)):
+         ctx.attr.name not in _SINGLE_URL_WHITELIST.to_list())):
         fail("tf_http_archive(urls) must have redundant URLs. The " +
              "mirror.bazel.build URL must be present and it must come first. " +
              "Even if you don't have permission to mirror the file, please " +
@@ -150,7 +150,7 @@ ensure best practices are followed.
 def _third_party_http_archive(ctx):
     if ("mirror.bazel.build" not in ctx.attr.urls[0] and
         (len(ctx.attr.urls) < 2 and
-         ctx.attr.name not in _SINGLE_URL_WHITELIST)):
+         ctx.attr.name not in _SINGLE_URL_WHITELIST.to_list())):
         fail("tf_http_archive(urls) must have redundant URLs. The " +
              "mirror.bazel.build URL must be present and it must come first. " +
              "Even if you don't have permission to mirror the file, please " +
-- 
GitLab


From 7554e91876d9b37ea1b5708140848a665c15180d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Nov 2018 22:44:36 -0800
Subject: [PATCH 0560/1554] Remove "_native_" from
 depthwise_conv2d_native_backprop_filter in TF 2.0 API

New name (depthwise_conv2d_backprop_filter), same semantics.

PiperOrigin-RevId: 222192663
---
 .../api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt     | 5 +++++
 tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt        | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt        | 8 ++++----
 tensorflow/tools/compatibility/renames_v2.py              | 1 +
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt b/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt
index 6f9df4b1a1..01c4a50ca6 100644
--- a/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt
@@ -2,5 +2,10 @@ op {
   graph_op_name: "DepthwiseConv2dNativeBackpropFilter"
   endpoint {
     name: "nn.depthwise_conv2d_native_backprop_filter"
+    deprecated: true
+    deprecation_version: 2
+  }
+  endpoint {
+    name: "nn.depthwise_conv2d_backprop_filter"
   }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 233567d600..e781287d6c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -116,6 +116,10 @@ tf_module {
     name: "depthwise_conv2d"
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'rate\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "depthwise_conv2d_backprop_filter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+  }
   member_method {
     name: "depthwise_conv2d_backprop_input"
     argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index b4c845a3d7..34ca2078b9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -109,12 +109,12 @@ tf_module {
     argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
-    name: "depthwise_conv2d_backprop_input"
-    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    name: "depthwise_conv2d_backprop_filter"
+    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
   }
   member_method {
-    name: "depthwise_conv2d_native_backprop_filter"
-    argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
+    name: "depthwise_conv2d_backprop_input"
+    argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], "
   }
   member_method {
     name: "dilation2d"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 3b5559a9f3..55a2804c80 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -308,6 +308,7 @@ renames = {
     'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
     'tf.nn.depthwise_conv2d_native': 'tf.compat.v1.nn.depthwise_conv2d_native',
+    'tf.nn.depthwise_conv2d_native_backprop_filter': 'tf.nn.depthwise_conv2d_backprop_filter',
     'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
-- 
GitLab


From f827bad24efc949e657e141464ea1010c18d812c Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Mon, 19 Nov 2018 23:03:47 -0800
Subject: [PATCH 0561/1554] Automated rollback of commit
 41d29682e9c35802825b69996d512825990522a3

PiperOrigin-RevId: 222194093
---
 tensorflow/lite/BUILD                      |   2 +
 tensorflow/lite/core/subgraph.cc           | 976 +++++++++++++++++++++
 tensorflow/lite/core/subgraph.h            | 476 ++++++++++
 tensorflow/lite/interpreter.cc             | 916 +------------------
 tensorflow/lite/interpreter.h              | 284 +-----
 tensorflow/lite/interpreter_test.cc        |   2 +-
 tensorflow/lite/nnapi_delegate.cc          |  87 +-
 tensorflow/lite/nnapi_delegate.h           |   5 +-
 tensorflow/lite/nnapi_delegate_disabled.cc |   6 +-
 tensorflow/lite/util.h                     |   6 +
 10 files changed, 1588 insertions(+), 1172 deletions(-)
 create mode 100644 tensorflow/lite/core/subgraph.cc
 create mode 100644 tensorflow/lite/core/subgraph.h

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index be84fc5db1..bb2c53b8c9 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -131,6 +131,7 @@ cc_library(
     name = "framework",
     srcs = [
         "allocation.cc",
+        "core/subgraph.cc",
         "graph_info.cc",
         "interpreter.cc",
         "model.cc",
@@ -155,6 +156,7 @@ cc_library(
         "allocation.h",
         "context.h",
         "context_util.h",
+        "core/subgraph.h",
         "error_reporter.h",
         "graph_info.h",
         "interpreter.h",
diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
new file mode 100644
index 0000000000..05a60962b1
--- /dev/null
+++ b/tensorflow/lite/core/subgraph.cc
@@ -0,0 +1,976 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/core/subgraph.h"
+#include "tensorflow/lite/arena_planner.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/context_util.h"
+#include "tensorflow/lite/graph_info.h"
+#include "tensorflow/lite/nnapi_delegate.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+
+namespace tflite {
+
+namespace {
+TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
+                           const TfLiteRegistration& registration,
+                           int node_index, const char* message) {
+  context->ReportError(
+      context, "Node number %d (%s) %s.\n", node_index,
+      registration.custom_name
+          ? registration.custom_name
+          : EnumNameBuiltinOperator(
+                static_cast<BuiltinOperator>(registration.builtin_code)),
+      message);
+  return kTfLiteError;
+}
+
+// Stub method which returns kTfLiteError when the function is forbidden.
+// We're registrating this function to several different function to save
+// compiled binary size. Please note the restrictions:
+// * The type of first parameter have to be `TfLiteContext*`.
+// * All paramteters must be trivailly destructible. (E.g. No C++ class)
+TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) {
+  context->ReportError(context,
+                       "The function is forbidden if not calling in delegate.");
+  return kTfLiteError;
+}
+
+// Set the ForbiddenContextFunction to a compatible function pointer.
+template <typename FunctionType>
+void SetForbiddenContextFunction(FunctionType* func) {
+  *func = reinterpret_cast<FunctionType>(ForbiddenContextFunction);
+}
+
+// Returns true if at least one tensor in the given list is kTfLiteDynamic.
+template <typename TensorIntArray>
+bool HasDynamicTensorImpl(const TfLiteContext& context,
+                          const TensorIntArray& int_array) {
+  for (int i : int_array) {
+    const TfLiteTensor& tensor = context.tensors[i];
+    if (tensor.allocation_type == kTfLiteDynamic) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool HasDynamicTensor(const TfLiteContext& context,
+                      const TfLiteIntArray* int_array) {
+  return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
+}
+
+}  // namespace
+
+// A trivial implementation of GraphInfo around the Interpreter.
+// NOTE: this interpreter info represents the subset of the
+// graph that is executed according to execution plan. Thus,
+// the indices are execution plan indices rather than raw node
+// indices.
+class InterpreterInfo : public GraphInfo {
+ public:
+  explicit InterpreterInfo(Subgraph* subgraph) : subgraph_(subgraph) {}
+
+  size_t num_tensors() const override { return subgraph_->tensors().size(); }
+  TfLiteTensor* tensor(size_t index) override {
+    return &subgraph_->tensors()[index];
+  }
+  size_t num_nodes() const override {
+    return subgraph_->execution_plan().size();
+  }
+  const TfLiteNode& node(size_t index) const override {
+    int node_index = subgraph_->execution_plan()[index];
+    return subgraph_->nodes_and_registration()[node_index].first;
+  }
+  const std::vector<int>& inputs() const override {
+    return subgraph_->inputs();
+  }
+  const std::vector<int>& outputs() const override {
+    return subgraph_->outputs();
+  }
+  const std::vector<int>& variables() const override {
+    return subgraph_->variables();
+  }
+
+ public:
+  Subgraph* subgraph_;
+};
+
+Subgraph::Subgraph(ErrorReporter* error_reporter,
+                   TfLiteExternalContext** external_contexts)
+    : context_(&owned_context_),
+      error_reporter_(error_reporter),
+      next_execution_plan_index_to_prepare_(0),
+      external_contexts_(external_contexts) {
+  context_->impl_ = static_cast<void*>(this);
+  context_->ResizeTensor = ResizeTensor;
+  context_->ReportError = ReportErrorC;
+  context_->AddTensors = AddTensors;
+  context_->tensors = nullptr;
+  context_->tensors_size = 0;
+  context_->allow_fp32_relax_to_fp16 = false;
+  context_->recommended_num_threads = -1;
+  context_->GetExternalContext = GetExternalContext;
+  context_->SetExternalContext = SetExternalContext;
+
+  // Reserve some space for the tensors to avoid excessive resizing.
+  tensors_.reserve(kTensorsReservedCapacity);
+  nodes_and_registration().reserve(kTensorsReservedCapacity);
+  // Invalid to call these these except from TfLiteDelegate
+  SwitchToKernelContext();
+}
+
+Subgraph::~Subgraph() {
+  for (auto& node_and_reg : nodes_and_registration_) {
+    TfLiteNode& node = node_and_reg.first;
+    TfLiteIntArrayFree(node.inputs);
+    TfLiteIntArrayFree(node.outputs);
+    TfLiteIntArrayFree(node.temporaries);
+    if (node.builtin_data) free(node.builtin_data);
+    OpFree(node_and_reg.second, node.user_data);
+    node.builtin_data = nullptr;
+  }
+
+  for (size_t i = 0; i < context_->tensors_size; i++) {
+    TfLiteTensor* tensor = &context_->tensors[i];
+    if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
+        tensor->delegate->FreeBufferHandle != nullptr) {
+      tensor->delegate->FreeBufferHandle(context_, tensor->delegate,
+                                         &tensor->buffer_handle);
+    }
+    TfLiteTensorFree(tensor);
+  }
+}
+
+TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels(
+    TfLiteContext* context, TfLiteRegistration registration,
+    const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
+  return static_cast<Subgraph*>(context->impl_)
+      ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace,
+                                              delegate);
+}
+
+namespace {
+
+// Copy a std::vector<int> to an existing TfLiteIntArray.
+// This is a low-level data manipulation function, and it's caller's
+// responsibility to ensure TfLiteIntArray has enough size.
+void CopyVectorToTfLiteIntArray(const std::vector<int>& vec,
+                                TfLiteIntArray* arr) {
+  arr->size = vec.size();
+  memcpy(arr->data, vec.data(), sizeof(int) * arr->size);
+}
+
+// This function allocates a continuous memory space that contains a
+// TfLiteDelegateParams followed by a several TfLiteIntArray.
+// When calling `free` at TfLiteDelegateParams*, all the allocated space
+// will be freed together.
+//
+// +-----------------------------------+
+// | TfLiteDelegateParams              |
+// | TfLiteDelegate* delegate;         |
+// | TfLiteIntArray* nodes_to_replace; |--\
+// | TfLiteIntArray* input_tensors;    |--+--\
+// | TfLiteIntArray* output_tensors;   |--+--+--\
+// +-----------------------------------+  |  |  |
+// | TfLiteIntArray (variable size)    |<-/  |  |
+// +-----------------------------------+     |  |
+// | TfLiteIntArray (variable size)    |<----/  |
+// +-----------------------------------+        |
+// | TfLiteIntArray (variable size)    |<-------/
+// +-----------------------------------+
+TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate,
+                                           const NodeSubset& node_subset) {
+  // Step 1: Calculate the allocation size.
+  int allocation_size = sizeof(TfLiteDelegateParams);
+
+  int nodes_to_replace_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size());
+  allocation_size += nodes_to_replace_size;
+
+  int input_tensors_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size());
+  allocation_size += input_tensors_size;
+
+  int output_tensors_size =
+      TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size());
+  allocation_size += output_tensors_size;
+
+  // Step 2: Allocate the memory.
+  // Use `char*` for conveniently step through the allocated space by bytes.
+  char* allocation = reinterpret_cast<char*>(malloc(allocation_size));
+
+  // Step 3: Fill all data structures structures.
+  TfLiteDelegateParams* params =
+      reinterpret_cast<TfLiteDelegateParams*>(allocation);
+  params->delegate = delegate;
+  allocation += sizeof(TfLiteDelegateParams);
+
+  params->nodes_to_replace = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace);
+  allocation += nodes_to_replace_size;
+
+  params->input_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors);
+  allocation += input_tensors_size;
+
+  params->output_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
+  CopyVectorToTfLiteIntArray(node_subset.output_tensors,
+                             params->output_tensors);
+  allocation += output_tensors_size;
+
+  return params;
+}
+
+}  // namespace
+
+TfLiteStatus Subgraph::ReplaceNodeSubsetsWithDelegateKernels(
+    TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
+    TfLiteDelegate* delegate) {
+  // Annotate the registration as DELEGATE op.
+  registration.builtin_code = BuiltinOperator_DELEGATE;
+
+  // Analyze the graph to find all independent node_subsets that are either
+  // fully not-this-delegate or this-delegate computation.
+  InterpreterInfo info(this);
+  std::vector<NodeSubset> node_subsets;
+  PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace,
+                                           &node_subsets);
+
+  execution_plan_.clear();
+
+  for (auto& node_subset : node_subsets) {
+    // Subsets calimed by the delegate should have a "macro" op created, the
+    // other node_subsets (kTfNonPartition) just have their nodes added back to
+    // the execution plan.
+    switch (node_subset.type) {
+      case NodeSubset::kTfNonPartition:
+        for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end();
+             ++it) {
+          execution_plan_.push_back(*it);
+        }
+        break;
+      case NodeSubset::kTfPartition: {
+        int node_index;
+
+        TfLiteDelegateParams* params =
+            CreateDelegateParams(delegate, node_subset);
+        TF_LITE_ENSURE_STATUS(AddNodeWithParameters(
+            node_subset.input_tensors, node_subset.output_tensors, nullptr, 0,
+            params, &registration, &node_index));
+
+        // Initialize the output tensors's delegate-related fields.
+        for (int tensor_index : node_subset.output_tensors) {
+          TfLiteTensor* tensor = &tensors_[tensor_index];
+          TF_LITE_ENSURE(context_, tensor->delegate == nullptr ||
+                                       tensor->delegate == delegate);
+          tensor->delegate = delegate;
+        }
+
+        // Associate the node with the delegate.
+        TfLiteNode* node = &nodes_and_registration_[node_index].first;
+        node->delegate = delegate;
+      } break;
+      case NodeSubset::kTfUnexplored:
+        return kTfLiteError;
+        break;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteExternalContext* Subgraph::GetExternalContext(
+    TfLiteExternalContextType type) {
+  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
+    return external_contexts_[type];
+  }
+  return nullptr;
+}
+
+TfLiteExternalContext* Subgraph::GetExternalContext(
+    struct TfLiteContext* context, TfLiteExternalContextType type) {
+  return static_cast<Subgraph*>(context->impl_)->GetExternalContext(type);
+}
+
+void Subgraph::SetExternalContext(TfLiteExternalContextType type,
+                                  TfLiteExternalContext* ctx) {
+  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
+    external_contexts_[type] = ctx;
+  }
+}
+
+void Subgraph::SetExternalContext(struct TfLiteContext* context,
+                                  TfLiteExternalContextType type,
+                                  TfLiteExternalContext* ctx) {
+  return static_cast<Subgraph*>(context->impl_)->SetExternalContext(type, ctx);
+}
+
+// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns
+// this memory and it is only guaranteed to exist during the invocation of the
+// delegate prepare.
+TfLiteStatus Subgraph::GetExecutionPlan(TfLiteIntArray** execution_plan) {
+  // TODO(aselle): Do not make a copy here
+  plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size()));
+  *execution_plan = plan_cache_.get();
+  static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]),
+                "TfLiteIntArray and execution_plan do not contain same type.");
+  std::memcpy(plan_cache_->data, execution_plan_.data(),
+              sizeof(plan_cache_->data[0]) * execution_plan_.size());
+  return kTfLiteOk;
+}
+
+// WARNING: This is an experimental interface that is subject to change.
+// Entry point for C node plugin API to get the execution plan
+TfLiteStatus Subgraph::GetExecutionPlan(struct TfLiteContext* context,
+                                        TfLiteIntArray** execution_plan) {
+  return static_cast<Subgraph*>(context->impl_)
+      ->GetExecutionPlan(execution_plan);
+}
+
+TfLiteStatus Subgraph::SetInputs(std::vector<int> inputs) {
+  TF_LITE_ENSURE_OK(&context_,
+                    CheckTensorIndices("inputs", inputs.data(), inputs.size()));
+  inputs_ = std::move(inputs);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::SetOutputs(std::vector<int> outputs) {
+  TF_LITE_ENSURE_OK(
+      &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
+  outputs_ = std::move(outputs);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::SetVariables(std::vector<int> variables) {
+  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
+                                                  variables.size()));
+  variables_ = std::move(variables);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::CheckTensorIndices(const char* label, const int* indices,
+                                          int length) {
+  // Making sure kOptionalTensor is not re-defined to something other than -1.
+  static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1");
+
+  for (int i = 0; i < length; i++) {
+    int index = indices[i];
+    // Continue if index == kOptionalTensor before additional comparisons below,
+    // size_t(-1) is always >= context_tensors_size.
+    if (index == kOptionalTensor) {
+      continue;
+    }
+    if (index < 0 || static_cast<size_t>(index) >= context_->tensors_size) {
+      ReportError("Invalid tensor index %d in %s\n", index, label);
+      consistent_ = false;
+      return kTfLiteError;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::BytesRequired(TfLiteType type, const int* dims,
+                                     size_t dims_size, size_t* bytes) {
+  // TODO(aselle): Check for overflow here using overflow.h in TensorFlow
+  // MultiplyWithoutOverflow.
+  TF_LITE_ENSURE(context_, bytes != nullptr);
+  size_t count = 1;
+  for (int k = 0; k < dims_size; k++) count *= dims[k];
+  switch (type) {
+    case kTfLiteFloat32:
+      *bytes = sizeof(float) * count;
+      break;
+    case kTfLiteInt16:
+      *bytes = sizeof(int16_t) * count;
+      break;
+    case kTfLiteInt32:
+      *bytes = sizeof(int32_t) * count;
+      break;
+    case kTfLiteUInt8:
+      *bytes = sizeof(uint8_t) * count;
+      break;
+    case kTfLiteInt64:
+      *bytes = sizeof(int64_t) * count;
+      break;
+    case kTfLiteBool:
+      *bytes = sizeof(bool) * count;
+      break;
+    case kTfLiteComplex64:
+      *bytes = sizeof(std::complex<float>) * count;
+      break;
+    case kTfLiteInt8:
+      *bytes = sizeof(int8_t) * count;
+      break;
+    default:
+      ReportError(
+          "Only float32, int8, int16, int32, int64, uint8, bool, complex64 "
+          "supported currently.");
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::AllocateTensors() {
+  if (!consistent_) {
+    ReportError("AllocateTensors() called on inconsistent model.");
+    return kTfLiteError;
+  }
+
+  // Explicit (re)allocation is necessary if nodes have been changed or tensors
+  // have been resized. For inputs marked as dynamic, we can't short-circuit the
+  // allocation as the client may have done the resize manually.
+  if (state_ != kStateUninvokable &&
+      !HasDynamicTensorImpl(*context_, inputs())) {
+    return kTfLiteOk;
+  }
+
+  next_execution_plan_index_to_prepare_ = 0;
+  if (memory_planner_) {
+    TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations());
+  }
+
+  TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
+
+  state_ = kStateInvokable;
+
+  // Reset the variable tensors to zero after (re)allocating the tensors.
+  // Developers shouldn't rely on the side effect of this function to reset
+  // variable tesnsors. They should call `ResetVariableTensors` directly
+  // instead.
+  ResetVariableTensors();
+
+  return kTfLiteOk;
+}
+
+// TODO(ycling): Support non-zero default values.
+TfLiteStatus Subgraph::ResetVariableTensors() {
+  for (auto& tensor : tensors_) {
+    if (!tensor.is_variable) {
+      continue;
+    }
+
+    // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be
+    // allocated after the initial `PrepareOpsAndTensors()` is called.
+    TF_LITE_ENSURE_EQ(context_, tensor.allocation_type,
+                      kTfLiteArenaRwPersistent);
+    TF_LITE_ENSURE(context_, tensor.data.raw != nullptr);
+
+    memset(tensor.data.raw, 0, tensor.bytes);
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::AddNodeWithParameters(
+    const std::vector<int>& inputs, const std::vector<int>& outputs,
+    const char* init_data, size_t init_data_size, void* builtin_data,
+    const TfLiteRegistration* registration, int* node_index) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError("AddNodeWithParameters is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  state_ = kStateUninvokable;
+
+  std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
+                                                              free);
+
+  TF_LITE_ENSURE_OK(context_, CheckTensorIndices("node inputs", inputs.data(),
+                                                 inputs.size()));
+  TF_LITE_ENSURE_OK(
+      &context_,
+      CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
+
+  int new_node_index = nodes_and_registration_.size();
+  if (node_index) *node_index = new_node_index;
+  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
+  auto& node_and_reg = nodes_and_registration_.back();
+  TfLiteNode& node = node_and_reg.first;
+  if (node.inputs) TfLiteIntArrayFree(node.inputs);
+  if (node.outputs) TfLiteIntArrayFree(node.outputs);
+  if (node.temporaries) TfLiteIntArrayFree(node.temporaries);
+
+  // NOTE, here we are not using move semantics yet, since our internal
+  // representation isn't std::vector, but in the future we would like to avoid
+  // copies, so we want the interface to take r-value references now.
+  node.inputs = ConvertVectorToTfLiteIntArray(inputs);
+  node.outputs = ConvertVectorToTfLiteIntArray(outputs);
+  node.temporaries = TfLiteIntArrayCreate(0);
+  if (init_data) {
+    node.user_data = OpInit(*registration, init_data, init_data_size);
+  } else {
+    node.user_data =
+        OpInit(*registration,
+               reinterpret_cast<const char*>(builtin_data_deleter.get()), 0);
+  }
+
+  node.builtin_data = builtin_data_deleter.release();
+  // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size`
+  // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels.
+
+  if (registration->builtin_code == BuiltinOperator_CUSTOM) {
+    // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer
+    // `Operator` table is passed in.
+    node.custom_initial_data = init_data;
+    node.custom_initial_data_size = init_data_size;
+  } else {
+    node.custom_initial_data = nullptr;
+    node.custom_initial_data_size = 0;
+  }
+
+  node.delegate = nullptr;
+  node_and_reg.second = *registration;
+  execution_plan_.push_back(new_node_index);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::ResizeInputTensor(int tensor_index,
+                                         const std::vector<int>& dims) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError("ResizeInputTensor is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
+  // TODO(aselle): All bounds checks can be implemented as one-sided bounds
+  // checks by casting to unsigned for efficiency. Profile before doing this.
+  TF_LITE_ENSURE(context_,
+                 tensor_index < context_->tensors_size && tensor_index >= 0);
+  TfLiteTensor* tensor = &context_->tensors[tensor_index];
+
+  // Short-circuit the state change if the dimensions don't change, avoiding
+  // unnecessary (re)allocations.
+  if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) {
+    return kTfLiteOk;
+  }
+
+  state_ = kStateUninvokable;
+  return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims));
+}
+
+TfLiteStatus Subgraph::PrepareOpsStartingAt(
+    int first_execution_plan_index, int* last_execution_plan_index_prepared) {
+  for (int execution_plan_index = first_execution_plan_index;
+       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
+    int node_index = execution_plan_[execution_plan_index];
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    const TfLiteRegistration& registration =
+        nodes_and_registration_[node_index].second;
+    EnsureTensorsVectorCapacity();
+    if (OpPrepare(registration, &node) == kTfLiteError) {
+      return ReportOpError(context_, node, registration, node_index,
+                           "failed to prepare");
+    }
+
+    *last_execution_plan_index_prepared = execution_plan_index;
+
+    // Discontinue if the node has dynamic outputs. Note that we don't
+    // stop for dynamic temporary tensors since they won't affect the
+    // sizes of other tensors in the graph.
+    if (HasDynamicTensor(*context_, node.outputs)) {
+      break;
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::PrepareOpsAndTensors() {
+  if (!memory_planner_) {
+    memory_planner_.reset(new ArenaPlanner(
+        context_, std::unique_ptr<GraphInfo>(new InterpreterInfo(this)),
+        /*preserve_inputs=*/true, /*preserve_intermediates*/ false));
+    memory_planner_->PlanAllocations();
+  }
+
+  int last_exec_plan_index_prepared = 0;
+
+  TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt(
+      next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared));
+  TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations(
+      next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared));
+
+  next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::Invoke() {
+  if (!consistent_) {
+    ReportError("Invoke called on model that is not consistent.");
+    return kTfLiteError;
+  }
+
+  TfLiteStatus status = kTfLiteOk;
+  if (state_ == kStateUninvokable) {
+    ReportError("Invoke called on model that is not ready.");
+    return kTfLiteError;
+  }
+
+  if (nnapi_delegate_) {
+    if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) {
+      TF_LITE_ENSURE_OK(context_, nnapi_delegate_->Invoke(this));
+      return kTfLiteOk;
+    } else {
+      // TODO(aselle): In the future, we would like this to be an
+      // automatic tflite CPU fallback.
+      ReportError(
+          "NNAPI was requested, but dependent sized tensors "
+          "being used.\n");
+      return kTfLiteError;
+    }
+  }
+
+  // Invocations are always done in node order.
+  // Note that calling Invoke repeatedly will cause the original memory plan to
+  // be reused, unless either ResizeInputTensor() or AllocateTensors() has been
+  // called.
+  for (int execution_plan_index = 0;
+       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
+    if (execution_plan_index == next_execution_plan_index_to_prepare_) {
+      TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
+      TF_LITE_ENSURE(context_, next_execution_plan_index_to_prepare_ >=
+                                   execution_plan_index);
+    }
+    int node_index = execution_plan_[execution_plan_index];
+    TfLiteNode& node = nodes_and_registration_[node_index].first;
+    const TfLiteRegistration& registration =
+        nodes_and_registration_[node_index].second;
+    SCOPED_OPERATOR_PROFILE(profiler_, node_index);
+
+    // TODO(ycling): This is an extra loop through inputs to check if the data
+    // need to be copied from Delegate buffer to raw memory, which is often not
+    // needed. We may want to cache this in prepare to know if this needs to be
+    // done for a node or not.
+    for (int i = 0; i < node.inputs->size; ++i) {
+      int tensor_index = node.inputs->data[i];
+      if (tensor_index == kOptionalTensor) {
+        continue;
+      }
+      TfLiteTensor* tensor = &tensors_[tensor_index];
+      if (tensor->delegate && tensor->delegate != node.delegate &&
+          tensor->data_is_stale) {
+        EnsureTensorDataIsReadable(tensor_index);
+      }
+    }
+
+    EnsureTensorsVectorCapacity();
+    tensor_resized_since_op_invoke_ = false;
+    if (OpInvoke(registration, &node) == kTfLiteError) {
+      status = ReportOpError(context_, node, registration, node_index,
+                             "failed to invoke");
+    }
+
+    // Force execution prep for downstream ops if the latest op triggered the
+    // resize of a dynamic tensor.
+    if (tensor_resized_since_op_invoke_ &&
+        HasDynamicTensor(*context_, node.outputs)) {
+      next_execution_plan_index_to_prepare_ = execution_plan_index + 1;
+    }
+  }
+
+  return status;
+}
+
+TfLiteStatus Subgraph::ResizeTensor(TfLiteContext* context,
+                                    TfLiteTensor* tensor,
+                                    TfLiteIntArray* new_size) {
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function ResizeTensorImpl
+  // (this function is static).
+  return static_cast<Subgraph*>(context->impl_)
+      ->ResizeTensorImpl(tensor, new_size);
+}
+
+void Subgraph::ReportErrorImpl(const char* format, va_list args) {
+  error_reporter_->Report(format, args);
+}
+
+void Subgraph::ReportErrorC(TfLiteContext* context, const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  auto* f = static_cast<Subgraph*>(context->impl_);
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Subgraph to call into the member function ReportErrorImpl
+  // (this function is static).
+  f->ReportErrorImpl(format, args);
+  va_end(args);
+}
+
+// Entry point for C node plugin API to report an error.
+void Subgraph::ReportError(const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  auto* f = static_cast<Subgraph*>(context_->impl_);
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Subgraph to call into the member function ReportErrorImpl
+  // (this function is static).
+  f->ReportErrorImpl(format, args);
+  va_end(args);
+}
+
+TfLiteStatus Subgraph::AddTensors(int tensors_to_add,
+                                  int* first_new_tensor_index) {
+  const size_t base_index = tensors_.size();
+  if (first_new_tensor_index) *first_new_tensor_index = base_index;
+  tensors_.resize(tensors_.size() + tensors_to_add);
+  for (size_t i = base_index; i < tensors_.size(); i++) {
+    memset(&tensors_[i], 0, sizeof(tensors_[i]));
+    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
+  }
+  context_->tensors = tensors_.data();
+  context_->tensors_size = tensors_.size();
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::AddTensors(TfLiteContext* context, int tensors_to_add,
+                                  int* first_new_tensor_index) {
+  // Note here that context->impl_ is recovering the this pointer for an
+  // instance of Interpreter to call into the member function AddTensors
+  // (this function is static).
+  return static_cast<Subgraph*>(context->impl_)
+      ->AddTensors(tensors_to_add, first_new_tensor_index);
+}
+
+TfLiteStatus Subgraph::GetNodeAndRegistration(
+    int node_index, TfLiteNode** node, TfLiteRegistration** registration) {
+  TF_LITE_ENSURE(context_, node_index >= 0);
+  auto nodes_size = nodes_and_registration_.size();
+  TF_LITE_ENSURE(context_, static_cast<size_t>(node_index) < nodes_size);
+  TF_LITE_ENSURE(context_, node != nullptr && registration != nullptr);
+  auto& node_and_reg = nodes_and_registration_[node_index];
+  *node = &node_and_reg.first;
+  *registration = &node_and_reg.second;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::GetNodeAndRegistration(
+    struct TfLiteContext* context, int node_index, TfLiteNode** node,
+    TfLiteRegistration** registration) {
+  return static_cast<Subgraph*>(context->impl_)
+      ->GetNodeAndRegistration(node_index, node, registration);
+}
+
+TfLiteStatus Subgraph::SetTensorParametersReadOnly(
+    int tensor_index, TfLiteType type, const char* name, const size_t rank,
+    const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
+    size_t bytes, const Allocation* allocation) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
+  TF_LITE_ENSURE(context_,
+                 tensor_index < context_->tensors_size && tensor_index >= 0);
+  // For most tensors we know exactly how much memory is necessary so we can
+  // ensure the buffer is large enough. However, we need to skip string tensors
+  // because their sizes change with the contents of the individual strings.
+  if (type != kTfLiteString) {
+    size_t required_bytes;
+    TF_LITE_ENSURE_OK(context_,
+                      BytesRequired(type, dims, rank, &required_bytes));
+    TF_LITE_ENSURE_EQ(context_, required_bytes, bytes);
+  }
+
+  TfLiteTensor& tensor = context_->tensors[tensor_index];
+  if (type == tensor.type &&
+      EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
+    // Fast path which does not invalidate the invokable property.
+    TfLiteTensorDataFree(&tensor);
+    tensor.data.raw = const_cast<char*>(buffer);
+    if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
+    tensor.params = quantization;
+    tensor.allocation_type = kTfLiteMmapRo;
+    tensor.allocation = allocation;
+  } else {
+    state_ = kStateUninvokable;
+    TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
+                      quantization, const_cast<char*>(buffer), bytes,
+                      kTfLiteMmapRo, allocation, false, &tensor);
+  }
+  return kTfLiteOk;
+}
+
+// Set description of inputs/outputs/data/fptrs for node `node_index`.
+// This variant assumes an external buffer has been allocated of size
+// bytes. The lifetime of buffer must be ensured to be greater or equal
+// to Interpreter.
+TfLiteStatus Subgraph::SetTensorParametersReadWrite(
+    int tensor_index, TfLiteType type, const char* name, const size_t rank,
+    const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  TF_LITE_ENSURE(context_,
+                 tensor_index < context_->tensors_size && tensor_index >= 0);
+  size_t required_bytes = 0;
+  if (type != kTfLiteString) {
+    // These types will be allocated in our arena so we need to record how
+    // many bytes we will need based on the dimensions. String tensors are
+    // allocated dynamically and we can't know ahead of time how much space
+    // they will require.
+    TF_LITE_ENSURE_OK(context_,
+                      BytesRequired(type, dims, rank, &required_bytes));
+  }
+
+  TfLiteAllocationType allocation_type = kTfLiteArenaRw;
+  if (type == kTfLiteString) {
+    if (is_variable) {
+      // We don't have a real use case for string variable tensor.
+      ReportError("String variable tensor isn't supported.");
+      return kTfLiteError;
+    }
+    allocation_type = kTfLiteDynamic;
+  } else if (is_variable) {
+    allocation_type = kTfLiteArenaRwPersistent;
+  }
+
+  TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
+                    quantization,
+                    /*buffer=*/nullptr, required_bytes, allocation_type,
+                    nullptr, is_variable, &context_->tensors[tensor_index]);
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::SetExecutionPlan(const std::vector<int>& new_plan) {
+  for (int node_index : new_plan) {
+    TF_LITE_ENSURE(context_, node_index >= 0 &&
+                                 node_index < nodes_and_registration_.size());
+  }
+  execution_plan_ = new_plan;
+  return kTfLiteOk;
+}
+
+TfLiteStatus Subgraph::ResizeTensorImpl(TfLiteTensor* tensor,
+                                        TfLiteIntArray* new_size) {
+  // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too.
+  if (tensor->allocation_type == kTfLiteArenaRw ||
+      tensor->allocation_type == kTfLiteDynamic ||
+      tensor->allocation_type == kTfLiteArenaRwPersistent) {
+    tensor_resized_since_op_invoke_ |=
+        TfLiteIntArrayEqual(tensor->dims, new_size) == 0;
+    if (tensor->type != kTfLiteString) {
+      size_t bytesRequired;
+      TfLiteStatus status = BytesRequired(tensor->type, new_size->data,
+                                          new_size->size, &bytesRequired);
+      if (status != kTfLiteOk) {
+        TfLiteIntArrayFree(new_size);
+        return kTfLiteError;
+      }
+
+      // Realloc space for kTfLiteDynamic tensors.
+      TfLiteTensorRealloc(bytesRequired, tensor);
+      tensor->bytes = bytesRequired;
+    }
+    if (tensor->dims) TfLiteIntArrayFree(tensor->dims);
+    tensor->dims = new_size;
+
+    if (tensor->allocation_type != kTfLiteDynamic) {
+      tensor->data.raw = nullptr;
+    }
+  } else {
+    // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore
+    // of fixed size.
+    TfLiteIntArrayFree(new_size);
+    ReportError("Attempting to resize a fixed-size tensor.");
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+void Subgraph::UseNNAPI(bool enable) {
+  // TODO(aselle): This is a workaround for finding if NNAPI exists.
+  // We also need to make sure getLibraryHandle() is renamed to be NNAPI
+  // prefixed.
+  if (!NNAPIDelegate::IsSupported()) enable = false;
+  if (!enable) {
+    nnapi_delegate_.reset();
+  } else if (!nnapi_delegate_) {
+    nnapi_delegate_.reset(new NNAPIDelegate);
+  }
+}
+
+void Subgraph::SwitchToDelegateContext() {
+  context_->GetNodeAndRegistration = GetNodeAndRegistration;
+  context_->ReplaceNodeSubsetsWithDelegateKernels =
+      ReplaceNodeSubsetsWithDelegateKernels;
+  context_->GetExecutionPlan = GetExecutionPlan;
+}
+
+void Subgraph::SwitchToKernelContext() {
+  context_->GetNodeAndRegistration = [](struct TfLiteContext* context,
+                                        int node_index, TfLiteNode** node,
+                                        TfLiteRegistration** registration) {
+    return ForbiddenContextFunction(context);
+  };
+  context_->ReplaceNodeSubsetsWithDelegateKernels =
+      [](TfLiteContext* context, TfLiteRegistration registration,
+         const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
+        return ForbiddenContextFunction(context);
+      };
+  context_->GetExecutionPlan = [](struct TfLiteContext* context,
+                                  TfLiteIntArray**) {
+    return ForbiddenContextFunction(context);
+  };
+}
+
+TfLiteStatus Subgraph::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
+  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
+    int last_execution_plan_index_prepared;
+    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
+                                     0, &last_execution_plan_index_prepared));
+
+    bool has_dynamic_tensors = true;
+    // Dynamic tensors exist if not all nodes can be prepared.
+    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
+      // If all the nodes can be prepared, check if the last node has dynamic
+      // tensors.
+      int node_index = execution_plan_[last_execution_plan_index_prepared];
+      TfLiteNode& node = nodes_and_registration_[node_index].first;
+      if (!HasDynamicTensor(*context_, node.outputs)) {
+        has_dynamic_tensors = false;
+      }
+    }
+    if (has_dynamic_tensors) {
+      ReportError(
+          "Attempting to use a delegate that only supports static-sized "
+          "tensors with a graph that has dynamic-sized tensors.");
+      return kTfLiteError;
+    }
+  }
+
+  // TODO(aselle): Consider if it is worth storing pointers to delegates.
+  // Setup additional context interface.
+  SwitchToDelegateContext();
+
+  TfLiteStatus status = delegate->Prepare(context_, delegate);
+
+  // Remove additional context info.
+  SwitchToKernelContext();
+
+  TF_LITE_ENSURE_OK(context_, status);
+
+  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
+    // Reset the state to force tensor/op reallocation.
+    state_ = kStateUninvokable;
+    TF_LITE_ENSURE_OK(context_, AllocateTensors());
+    TF_LITE_ENSURE_EQ(context_, state_, kStateInvokable);
+    // After using a delegate which doesn't support dynamic tensors, make the
+    // entire graph immutable.
+    state_ = kStateInvokableAndImmutable;
+  }
+
+  return status;
+}
+
+}  // namespace tflite
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
new file mode 100644
index 0000000000..9783747ff6
--- /dev/null
+++ b/tensorflow/lite/core/subgraph.h
@@ -0,0 +1,476 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_CORE_SUBGRAPH_H_
+#define TENSORFLOW_LITE_CORE_SUBGRAPH_H_
+
+#include <cstdlib>
+#include <vector>
+
+#include "tensorflow/lite/allocation.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/memory_planner.h"
+#include "tensorflow/lite/util.h"
+
+namespace tflite {
+
+// Forward declare since NNAPIDelegate uses Interpreter.
+class NNAPIDelegate;
+
+class Subgraph {
+ public:
+  friend class Interpreter;
+
+  Subgraph(ErrorReporter* error_reporter,
+           TfLiteExternalContext** external_contexts);
+  Subgraph(const Subgraph&) = delete;
+
+  // Subgraphs should be movable but not copyable.
+  Subgraph(Subgraph&&) = default;
+  Subgraph& operator=(const Subgraph&) = delete;
+  virtual ~Subgraph();
+
+  // Provide a list of tensor indexes that are inputs to the model.
+  // Each index is bound check and this modifies the consistent_ flag of the
+  // interpreter.
+  TfLiteStatus SetInputs(std::vector<int> inputs);
+
+  // Provide a list of tensor indexes that are outputs to the model
+  // Each index is bound check and this modifies the consistent_ flag of the
+  // interpreter.
+  TfLiteStatus SetOutputs(std::vector<int> outputs);
+
+  // Provide a list of tensor indexes that are variable tensors.
+  // Each index is bound check and this modifies the consistent_ flag of the
+  // interpreter.
+  TfLiteStatus SetVariables(std::vector<int> variables);
+
+
+  // Adds a node with the given parameters and returns the index of the new
+  // node in `node_index` (optionally). Interpreter will take ownership of
+  // `builtin_data` and destroy it with `free`. Ownership of 'init_data'
+  // remains with the caller.
+  TfLiteStatus AddNodeWithParameters(const std::vector<int>& inputs,
+                                     const std::vector<int>& outputs,
+                                     const char* init_data,
+                                     size_t init_data_size, void* builtin_data,
+                                     const TfLiteRegistration* registration,
+                                     int* node_index);
+
+  // Adds `tensors_to_add` tensors, preserving pre-existing Tensor entries.
+  // The value pointed to by `first_new_tensor_index` will be set to the
+  // index of the first new tensor if `first_new_tensor_index` is non-null.
+  TfLiteStatus AddTensors(int tensors_to_add, int* first_new_tensor_index);
+
+  // Set description of inputs/outputs/data/fptrs for node `node_index`.
+  // This variant assumes an external buffer has been allocated of size
+  // bytes. The lifetime of buffer must be ensured to be greater or equal
+  // to Interpreter.
+  TfLiteStatus SetTensorParametersReadOnly(
+      int tensor_index, TfLiteType type, const char* name, const size_t rank,
+      const int* dims, TfLiteQuantizationParams quantization,
+      const char* buffer, size_t bytes, const Allocation* allocation);
+
+  // Set description of inputs/outputs/data/fptrs for node `node_index`.
+  // This variant assumes an external buffer has been allocated of size
+  // bytes. The lifetime of buffer must be ensured to be greater or equal
+  // to Interpreter.
+  TfLiteStatus SetTensorParametersReadWrite(
+      int tensor_index, TfLiteType type, const char* name, const size_t rank,
+      const int* dims, TfLiteQuantizationParams quantization, bool is_variable);
+
+  // WARNING: Experimental interface, subject to change
+  // Overrides execution plan. This bounds checks indices sent in.
+  TfLiteStatus SetExecutionPlan(const std::vector<int>& new_plan);
+
+  // Get a mutable tensor data structure.
+  // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this
+  // read/write access to structure
+  TfLiteTensor* tensor(int tensor_index) {
+    if (tensor_index < 0 ||
+        static_cast<size_t>(tensor_index) >= context_->tensors_size) {
+      return nullptr;
+    }
+    return &context_->tensors[tensor_index];
+  }
+
+  // Get an immutable tensor data structure.
+  const TfLiteTensor* tensor(int tensor_index) const {
+    if (tensor_index < 0 ||
+        static_cast<size_t>(tensor_index) >= context_->tensors_size) {
+      return nullptr;
+    }
+    return &context_->tensors[tensor_index];
+  }
+
+  // Read only access to list of inputs.
+  std::vector<int>& inputs() { return inputs_; }
+
+  // Read only access to list of inputs.
+  const std::vector<int>& inputs() const { return inputs_; }
+
+  // Read only access to list of outputs.
+  std::vector<int>& outputs() { return outputs_; }
+
+  // Read only access to list of outputs.
+  const std::vector<int>& outputs() const { return outputs_; }
+
+  // Read only access to list of variable tensors.
+  std::vector<int>& variables() { return variables_; }
+
+  // Read only access to list of variable tensors.
+  const std::vector<int>& variables() const { return variables_; }
+
+  size_t tensors_size() const { return tensors_.size(); }
+
+  // Return the number of ops in the model.
+  size_t nodes_size() const { return nodes_and_registration_.size(); }
+
+  // Read only access to list of variable tensors.
+  std::vector<int>& execution_plan() { return execution_plan_; }
+
+  // Read only access to list of variable tensors.
+  const std::vector<int>& execution_plan() const { return execution_plan_; }
+
+  // Mutable form of tensors (TEMPORARY for refactor).
+  // TODO(b/119495520): remove when refactoring complete.
+  std::vector<TfLiteTensor>& tensors() { return tensors_; }
+  // Mutable form of tensors (TEMPORARY for refactor).
+  // TODO(b/119495520): remove when refactoring complete.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
+  nodes_and_registration() {
+    return nodes_and_registration_;
+  }
+
+  const std::vector<std::pair<TfLiteNode, TfLiteRegistration>>&
+  nodes_and_registration() const {
+    return nodes_and_registration_;
+  }
+
+  // Get a pointer to an operation and registration data structure if in bounds.
+  const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
+      int node_index) const {
+    if (node_index < 0 || static_cast<size_t>(node_index) >= nodes_size())
+      return nullptr;
+    return &nodes_and_registration_[node_index];
+  }
+
+
+  // Change the dimensionality of a given tensor. Note, this is only acceptable
+  // for tensor indices that are inputs.
+  // Returns status of failure or success.
+  // TODO(aselle): Consider implementing ArraySlice equivalent to make this
+  //   more adept at accepting data without an extra copy. Use absl::ArraySlice
+  //   if our partners determine that dependency is acceptable.
+  TfLiteStatus ResizeInputTensor(int tensor_index,
+                                 const std::vector<int>& dims);
+
+  // Update allocations for all tensors. This will redim dependent tensors using
+  // the input tensor dimensionality as given. This is relatively expensive.
+  // If you know that your sizes are not changing, you need not call this.
+  // Returns status of success or failure.
+  TfLiteStatus AllocateTensors();
+
+  // Invoke the subgraph (run the whole graph in dependency order).
+  //
+  // NOTE: It is possible that the interpreter is not in a ready state
+  // to evaluate (i.e. if a ResizeTensor() has been performed without an
+  // AllocateTensors().
+  // Returns status of success or failure.
+  TfLiteStatus Invoke();
+
+  // Entry point for C node plugin API to report an error.
+  void ReportError(const char* format, ...);
+
+  void UseNNAPI(bool enable);
+
+  // Return the subgraph specific context.
+  TfLiteContext* context() { return context_; }
+
+  // Set the value of an external context.
+  void SetExternalContext(TfLiteExternalContextType type,
+                          TfLiteExternalContext* ctx);
+  // Get the half precision flag.
+  // WARNING: This is an experimental API and subject to change.
+  bool GetAllowFp16PrecisionForFp32() const {
+    return context_->allow_fp32_relax_to_fp16;
+  }
+
+  // Ensure the data in `tensor.data` is readable. In case delegate is used,
+  // it might require to copy the data from delegate buffer to raw memory.
+  // WARNING: This is an experimental API and subject to change.
+  // TODO(b/119495520): make this private when refactoring complete.
+  TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
+    TfLiteTensor* t = &tensors_[tensor_index];
+    TF_LITE_ENSURE(context_, t != nullptr);
+    if (t->data_is_stale) {
+      TF_LITE_ENSURE(context_, t->delegate != nullptr);
+      TF_LITE_ENSURE(context_, t->buffer_handle != kTfLiteNullBufferHandle);
+      // This can be null if the delegate doesn't use its own buffer.
+      TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle != nullptr);
+      t->delegate->CopyFromBufferHandle(context_, t->delegate, t->buffer_handle,
+                                        t->data.raw, t->bytes);
+      t->data_is_stale = false;
+    }
+    return kTfLiteOk;
+  }
+
+
+  // The default capacity of `tensors_` vector.
+  static constexpr int kTensorsReservedCapacity = 128;
+  // The capacity headroom of `tensors_` vector before calling ops'
+  // `prepare` and `invoke` function. In these functions, it's guaranteed
+  // allocating up to `kTensorsCapacityHeadroom` more tensors won't invalidate
+  // pointers to existing tensors.
+  static constexpr int kTensorsCapacityHeadroom = 16;
+
+  // Reset all variable tensors to the default value.
+  // If a variable tensor doesn't have a buffer, reset it to zero.
+  // TODO(b/115961645): Implement - If a variable tensor has a buffer, reset it
+  // to the value of the buffer.
+  // WARNING: This is an experimental API and subject to change.
+  TfLiteStatus ResetVariableTensors();
+
+ private:
+  // Prevent 'context_' from accessing functions that are only available to
+  // delegated kernels.
+  void SwitchToKernelContext();
+
+  // Add delegate-only functions to 'context_'.
+  void SwitchToDelegateContext();
+
+  // Give 'op_reg' a chance to initialize itself using the contents of
+  // 'buffer'.
+  void* OpInit(const TfLiteRegistration& op_reg, const char* buffer,
+               size_t length) {
+    if (op_reg.init == nullptr) return nullptr;
+    return op_reg.init(context_, buffer, length);
+  }
+
+  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
+  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
+    if (op_reg.free == nullptr) return;
+    if (buffer) {
+      op_reg.free(context_, buffer);
+    }
+  }
+
+  // Prepare the given 'node' for execution.
+  TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) {
+    if (op_reg.prepare == nullptr) return kTfLiteOk;
+    return op_reg.prepare(context_, node);
+  }
+
+  // Invoke the operator represented by 'node'.
+  TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) {
+    if (op_reg.invoke == nullptr) return kTfLiteError;
+    return op_reg.invoke(context_, node);
+  }
+
+  // Call OpPrepare() for as many ops as possible, allocating memory for their
+  // tensors. If an op containing dynamic tensors is found, preparation will be
+  // postponed until this function is called again. This allows the interpreter
+  // to wait until Invoke() to resolve the sizes of dynamic tensors.
+  TfLiteStatus PrepareOpsAndTensors();
+
+  // Call OpPrepare() for all ops starting at 'first_node'. Stop when a
+  // dynamic tensors is found or all ops have been prepared. Fill
+  // 'last_node_prepared' with the id of the op containing dynamic tensors, or
+  // the last in the graph.
+  TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index,
+                                    int* last_execution_plan_index_prepared);
+
+  // Tensors needed by the interpreter. Use `AddTensors` to add more blank
+  // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
+  // `context_` whenever this std::vector is reallocated. Currently this
+  // only happens in `AddTensors()`.
+  std::vector<TfLiteTensor> tensors_;
+
+  // Check if an array of tensor indices are valid with respect to the Tensor
+  // array.
+  // NOTE: this changes consistent_ to be false if indices are out of bounds.
+  TfLiteStatus CheckTensorIndices(const char* label, const int* indices,
+                                  int length);
+
+  // Compute the number of bytes required to represent a tensor with dimensions
+  // specified by the array dims (of length dims_size). Returns the status code
+  // and bytes.
+  TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size,
+                             size_t* bytes);
+
+  // Request an tensor be resized implementation. If the given tensor is of
+  // type kTfLiteDynamic it will also be allocated new memory.
+  TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size);
+
+  // Report a detailed error string (will be printed to stderr).
+  // TODO(aselle): allow user of class to provide alternative destinations.
+  void ReportErrorImpl(const char* format, va_list args);
+
+  // Entry point for C node plugin API to request an tensor be resized.
+  static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor,
+                                   TfLiteIntArray* new_size);
+  // Entry point for C node plugin API to report an error.
+  static void ReportErrorC(TfLiteContext* context, const char* format, ...);
+
+  // Entry point for C node plugin API to add new tensors.
+  static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add,
+                                 int* first_new_tensor_index);
+
+  // WARNING: This is an experimental API and subject to change.
+  // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels
+  static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
+      TfLiteContext* context, TfLiteRegistration registration,
+      const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate);
+
+  // Update the execution graph to replace some of the nodes with stub
+  // nodes. Specifically any node index that has `nodes[index]==1` will be
+  // slated for replacement with a delegate kernel specified by registration.
+  // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller.
+  // WARNING: This is an experimental interface that is subject to change.
+  TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
+      TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
+      TfLiteDelegate* delegate);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Gets the internal pointer to a TensorFlow lite node by node_index.
+  TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node,
+                                      TfLiteRegistration** registration);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Entry point for C node plugin API to get a node by index.
+  static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*,
+                                             int node_index, TfLiteNode** node,
+                                             TfLiteRegistration** registration);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Gets an TfLiteIntArray* representing the execution plan. The interpreter
+  // owns this memory and it is only guaranteed to exist during the invocation
+  // of the delegate prepare.
+  TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan);
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // Entry point for C node plugin API to get the execution plan.
+  static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context,
+                                       TfLiteIntArray** execution_plan);
+
+  // Retrieve an existing external context by type.
+  TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type);
+  static TfLiteExternalContext* GetExternalContext(
+      struct TfLiteContext* context, TfLiteExternalContextType type);
+
+  // Set the value of an external context.
+  static void SetExternalContext(struct TfLiteContext* context,
+                                 TfLiteExternalContextType type,
+                                 TfLiteExternalContext* ctx);
+
+  // Allow a delegate to look at the graph and modify the graph to handle
+  // parts of the graph themselves. After this is called, the graph may
+  // contain new nodes that replace 1 more nodes.
+  // WARNING: This is an experimental API and subject to change.
+  TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
+
+  // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra
+  // capacity. Calling this function may invalidate existing pointers to
+  // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
+  // more tensors won't invalidate the pointer to existing tensors.
+  void EnsureTensorsVectorCapacity() {
+    const size_t required_capacity = tensors_.size() + kTensorsCapacityHeadroom;
+    if (required_capacity > tensors_.capacity()) {
+      tensors_.reserve(required_capacity);
+      context_->tensors = tensors_.data();
+    }
+  }
+
+  // The state of the Interpreter.
+  enum State {
+    // The interpreter isn't ready to be invoked.
+    // `AllocateTensor` need to be called to enter an invokable state.
+    kStateUninvokable = 0,
+    // The interpreter is ready to be invoked.
+    kStateInvokable,
+    // The interpreter is ready to be invoked, and graph can't be further
+    // modified. The interpreter will enter this state when calling
+    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
+    kStateInvokableAndImmutable,
+  };
+  State state_ = kStateUninvokable;
+
+  // A pure C data structure used to communicate with the pure C plugin
+  // interface. To avoid copying tensor metadata, this is also the definitive
+  // structure to store tensors.
+  // TODO(b/119495520): Get rid of owned and just make context_ a instance.
+  TfLiteContext owned_context_;
+  TfLiteContext* context_;
+
+  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
+  // function pointers to actual implementation.
+  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
+      nodes_and_registration_;
+
+  // Whether the model is consistent. That is to say if the inputs and outputs
+  // of every node and the global inputs and outputs are valid indexes into
+  // the tensor array.
+  bool consistent_ = true;
+
+  // Array of indices representing the tensors that are inputs to the
+  // interpreter.
+  std::vector<int> inputs_;
+
+  // Array of indices representing the tensors that are outputs to the
+  // interpreter.
+  std::vector<int> outputs_;
+
+  // Array of indices representing the tensors that are variable tensors.
+  std::vector<int> variables_;
+
+  // The error reporter delegate that tflite will forward queries errors to.
+  ErrorReporter* error_reporter_;
+
+  // Index of the next node to prepare.
+  // During Invoke(), Interpreter will allocate input tensors first, which are
+  // known to be fixed size. Then it will allocate outputs from nodes as many
+  // as possible. When there is a node that produces dynamic sized tensor.
+  // Interpreter will stop allocating tensors, set the value of next allocate
+  // node id, and execute the node to generate the output tensor before continue
+  // to allocate successors. This process repeats until all nodes are executed.
+  // NOTE: this relies on the order of nodes that is in topological order.
+  int next_execution_plan_index_to_prepare_;
+
+  // WARNING: This is an experimental interface that is subject to change.
+  // This is a list of node indices (to index into nodes_and_registration).
+  // This represents a valid topological sort (dependency ordered) execution
+  // plan. In particular, it is valid for this ordering to contain only a
+  // subset of the node indices.
+  std::vector<int> execution_plan_;
+
+  // In the future, we'd like a TfLiteIntArray compatible representation.
+  // TODO(aselle): replace execution_plan_ with this.
+  std::unique_ptr<TfLiteIntArray, TfLiteIntArrayDeleter> plan_cache_;
+
+  // Whether to delegate to NN API
+  std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
+
+  std::unique_ptr<MemoryPlanner> memory_planner_;
+
+  // Tracking bit for whether a tensor was resized in the course of an op
+  // invocation. This is a useful hint to ensure that dynamic tensor outputs
+  // trigger downstream reallocation after op invocation.
+  bool tensor_resized_since_op_invoke_ = false;
+
+  // External contexts (kTfLiteMaxExternalContexts).
+  TfLiteExternalContext** external_contexts_;
+};
+
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_CORE_SUBGRAPH_H_
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 0bac5b07b2..4f4a999210 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include <cstdint>
 #include <cstring>
 
-#include "tensorflow/lite/arena_planner.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/context_util.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
@@ -32,110 +31,14 @@ limitations under the License.
 #include "tensorflow/lite/util.h"
 
 namespace tflite {
-namespace {
-
-TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
-                           const TfLiteRegistration& registration,
-                           int node_index, const char* message) {
-  context->ReportError(
-      context, "Node number %d (%s) %s.\n", node_index,
-      registration.custom_name
-          ? registration.custom_name
-          : EnumNameBuiltinOperator(
-                static_cast<BuiltinOperator>(registration.builtin_code)),
-      message);
-  return kTfLiteError;
-}
-
-// Stub method which returns kTfLiteError when the function is forbidden.
-// We're registrating this function to several different function to save
-// compiled binary size. Please note the restrictions:
-// * The type of first parameter have to be `TfLiteContext*`.
-// * All paramteters must be trivailly destructible. (E.g. No C++ class)
-TfLiteStatus ForbiddenContextFunction(TfLiteContext* context, ...) {
-  context->ReportError(context,
-                       "The function is forbidden if not calling in delegate.");
-  return kTfLiteError;
-}
-
-// Set the ForbiddenContextFunction to a compatible function pointer.
-template <typename FunctionType>
-void SetForbiddenContextFunction(FunctionType* func) {
-  *func = reinterpret_cast<FunctionType>(ForbiddenContextFunction);
-}
-
-// Returns true if at least one tensor in the given list is kTfLiteDynamic.
-template <typename TensorIntArray>
-bool HasDynamicTensorImpl(const TfLiteContext& context,
-                          const TensorIntArray& int_array) {
-  for (int i : int_array) {
-    const TfLiteTensor& tensor = context.tensors[i];
-    if (tensor.allocation_type == kTfLiteDynamic) {
-      return true;
-    }
-  }
-  return false;
-}
-
-}  // namespace
-
-// A trivial implementation of GraphInfo around the Interpreter.
-// NOTE: this interpreter info represents the subset of the
-// graph that is executed according to execution plan. Thus,
-// the indices are execution plan indices rather than raw node
-// indices.
-class InterpreterInfo : public GraphInfo {
- public:
-  explicit InterpreterInfo(Interpreter* interpreter)
-      : interpreter_(interpreter) {}
-
-  size_t num_tensors() const override { return interpreter_->tensors_size(); }
-  TfLiteTensor* tensor(size_t index) override {
-    return interpreter_->tensor(index);
-  }
-  size_t num_nodes() const override {
-    return interpreter_->execution_plan().size();
-  }
-  const TfLiteNode& node(size_t index) const override {
-    int node_index = interpreter_->execution_plan()[index];
-    return interpreter_->node_and_registration(node_index)->first;
-  }
-  const std::vector<int>& inputs() const override {
-    return interpreter_->inputs();
-  }
-  const std::vector<int>& outputs() const override {
-    return interpreter_->outputs();
-  }
-  const std::vector<int>& variables() const override {
-    return interpreter_->variables();
-  }
-
- public:
-  Interpreter* interpreter_;
-};
 
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  context_.impl_ = static_cast<void*>(this);
-  context_.ResizeTensor = ResizeTensor;
-  context_.ReportError = ReportError;
-  context_.AddTensors = AddTensors;
-  context_.tensors = nullptr;
-  context_.tensors_size = 0;
-  context_.allow_fp32_relax_to_fp16 = false;
-  context_.recommended_num_threads = -1;
-  context_.GetExternalContext = GetExternalContext;
-  context_.SetExternalContext = SetExternalContext;
-
-  // Invalid to call these these except from TfLiteDelegate
-  SwitchToKernelContext();
+  subgraphs_.emplace_back(error_reporter_, external_contexts_);
+  context_ = primary_subgraph().context();
 
   // Reserve some space for the tensors to avoid excessive resizing.
-  tensors_.reserve(kTensorsReservedCapacity);
-  nodes_and_registration_.reserve(kTensorsReservedCapacity);
-  next_execution_plan_index_to_prepare_ = 0;
-
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
     external_contexts_[i] = nullptr;
   }
@@ -143,673 +46,75 @@ Interpreter::Interpreter(ErrorReporter* error_reporter)
   UseNNAPI(false);
 }
 
-Interpreter::~Interpreter() {
-  for (auto& nodeAndReg : nodes_and_registration_) {
-    TfLiteNode& node = nodeAndReg.first;
-    TfLiteIntArrayFree(node.inputs);
-    TfLiteIntArrayFree(node.outputs);
-    TfLiteIntArrayFree(node.temporaries);
-    if (node.builtin_data) free(node.builtin_data);
-    OpFree(nodeAndReg.second, node.user_data);
-    node.builtin_data = nullptr;
-  }
-
-  for (size_t i = 0; i < context_.tensors_size; i++) {
-    TfLiteTensor* tensor = &context_.tensors[i];
-    if (tensor->buffer_handle != kTfLiteNullBufferHandle &&
-        tensor->delegate->FreeBufferHandle != nullptr) {
-      tensor->delegate->FreeBufferHandle(&context_, tensor->delegate,
-                                         &tensor->buffer_handle);
-    }
-    TfLiteTensorFree(tensor);
-  }
-}
-
-TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
-    TfLiteContext* context, TfLiteRegistration registration,
-    const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->ReplaceNodeSubsetsWithDelegateKernels(registration, nodes_to_replace,
-                                              delegate);
-}
-
-namespace {
-
-// Copy a std::vector<int> to an existing TfLiteIntArray.
-// This is a low-level data manipulation function, and it's caller's
-// responsibility to ensure TfLiteIntArray has enough size.
-void CopyVectorToTfLiteIntArray(const std::vector<int>& vec,
-                                TfLiteIntArray* arr) {
-  arr->size = vec.size();
-  memcpy(arr->data, vec.data(), sizeof(int) * arr->size);
-}
-
-// This function allocates a continuous memory space that contains a
-// TfLiteDelegateParams followed by a several TfLiteIntArray.
-// When calling `free` at TfLiteDelegateParams*, all the allocated space
-// will be freed together.
-//
-// +-----------------------------------+
-// | TfLiteDelegateParams              |
-// | TfLiteDelegate* delegate;         |
-// | TfLiteIntArray* nodes_to_replace; |--\
-// | TfLiteIntArray* input_tensors;    |--+--\
-// | TfLiteIntArray* output_tensors;   |--+--+--\
-// +-----------------------------------+  |  |  |
-// | TfLiteIntArray (variable size)    |<-/  |  |
-// +-----------------------------------+     |  |
-// | TfLiteIntArray (variable size)    |<----/  |
-// +-----------------------------------+        |
-// | TfLiteIntArray (variable size)    |<-------/
-// +-----------------------------------+
-TfLiteDelegateParams* CreateDelegateParams(TfLiteDelegate* delegate,
-                                           const NodeSubset& node_subset) {
-  // Step 1: Calculate the allocation size.
-  int allocation_size = sizeof(TfLiteDelegateParams);
-
-  int nodes_to_replace_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.nodes.size());
-  allocation_size += nodes_to_replace_size;
-
-  int input_tensors_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.input_tensors.size());
-  allocation_size += input_tensors_size;
-
-  int output_tensors_size =
-      TfLiteIntArrayGetSizeInBytes(node_subset.output_tensors.size());
-  allocation_size += output_tensors_size;
-
-  // Step 2: Allocate the memory.
-  // Use `char*` for conveniently step through the allocated space by bytes.
-  char* allocation = reinterpret_cast<char*>(malloc(allocation_size));
-
-  // Step 3: Fill all data structures structures.
-  TfLiteDelegateParams* params =
-      reinterpret_cast<TfLiteDelegateParams*>(allocation);
-  params->delegate = delegate;
-  allocation += sizeof(TfLiteDelegateParams);
-
-  params->nodes_to_replace = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.nodes, params->nodes_to_replace);
-  allocation += nodes_to_replace_size;
-
-  params->input_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.input_tensors, params->input_tensors);
-  allocation += input_tensors_size;
-
-  params->output_tensors = reinterpret_cast<TfLiteIntArray*>(allocation);
-  CopyVectorToTfLiteIntArray(node_subset.output_tensors,
-                             params->output_tensors);
-  allocation += output_tensors_size;
-
-  return params;
-}
-
-}  // namespace
-
-TfLiteStatus Interpreter::ReplaceNodeSubsetsWithDelegateKernels(
-    TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
-    TfLiteDelegate* delegate) {
-  // Annotate the registration as DELEGATE op.
-  registration.builtin_code = BuiltinOperator_DELEGATE;
-
-  // Analyze the graph to find all independent node_subsets that are either
-  // fully not-this-delegate or this-delegate computation.
-  InterpreterInfo info(this);
-  std::vector<NodeSubset> node_subsets;
-  PartitionGraphIntoIndependentNodeSubsets(&info, nodes_to_replace,
-                                           &node_subsets);
-
-  execution_plan_.clear();
-  for (auto& node_subset : node_subsets) {
-    // Subsets calimed by the delegate should have a "macro" op created, the
-    // other node_subsets (kTfNonPartition) just have their nodes added back to
-    // the execution plan.
-    switch (node_subset.type) {
-      case NodeSubset::kTfNonPartition:
-        for (auto it = node_subset.nodes.begin(); it != node_subset.nodes.end();
-             ++it) {
-          execution_plan_.push_back(*it);
-        }
-        break;
-      case NodeSubset::kTfPartition: {
-        int node_index;
-
-        TfLiteDelegateParams* params =
-            CreateDelegateParams(delegate, node_subset);
-        TF_LITE_ENSURE_STATUS(AddNodeWithParameters(
-            node_subset.input_tensors, node_subset.output_tensors, nullptr, 0,
-            params, &registration, &node_index));
-
-        // Initialize the output tensors's delegate-related fields.
-        for (int tensor_index : node_subset.output_tensors) {
-          TfLiteTensor* tensor = &tensors_[tensor_index];
-          TF_LITE_ENSURE(&context_, tensor->delegate == nullptr ||
-                                        tensor->delegate == delegate);
-          tensor->delegate = delegate;
-        }
-
-        // Associate the node with the delegate.
-        TfLiteNode* node = &nodes_and_registration_[node_index].first;
-        node->delegate = delegate;
-      } break;
-      case NodeSubset::kTfUnexplored:
-        return kTfLiteError;
-        break;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteExternalContext* Interpreter::GetExternalContext(
-    TfLiteExternalContextType type) {
-  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
-    return external_contexts_[type];
-  }
-  return nullptr;
-}
-
-TfLiteExternalContext* Interpreter::GetExternalContext(
-    struct TfLiteContext* context, TfLiteExternalContextType type) {
-  return static_cast<Interpreter*>(context->impl_)->GetExternalContext(type);
-}
+Interpreter::~Interpreter() {}
 
 void Interpreter::SetExternalContext(TfLiteExternalContextType type,
                                      TfLiteExternalContext* ctx) {
-  if (type >= 0 && type < kTfLiteMaxExternalContexts) {
-    external_contexts_[type] = ctx;
-  }
-}
-
-void Interpreter::SetExternalContext(struct TfLiteContext* context,
-                                     TfLiteExternalContextType type,
-                                     TfLiteExternalContext* ctx) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->SetExternalContext(type, ctx);
-}
-
-// Gets an TfLiteIntArray* representing the execution plan. The interpreter owns
-// this memory and it is only guaranteed to exist during the invocation of the
-// delegate prepare.
-TfLiteStatus Interpreter::GetExecutionPlan(TfLiteIntArray** execution_plan) {
-  // TODO(aselle): Do not make a copy here
-  plan_cache_.reset(TfLiteIntArrayCreate(execution_plan_.size()));
-  *execution_plan = plan_cache_.get();
-  static_assert(sizeof(plan_cache_->data[0]) == sizeof(execution_plan_[0]),
-                "TfLiteIntArray and execution_plan do not contain same type.");
-  std::memcpy(plan_cache_->data, execution_plan_.data(),
-              sizeof(plan_cache_->data[0]) * execution_plan_.size());
-  return kTfLiteOk;
-}
-
-// WARNING: This is an experimental interface that is subject to change.
-// Entry point for C node plugin API to get the execution plan
-TfLiteStatus Interpreter::GetExecutionPlan(struct TfLiteContext* context,
-                                           TfLiteIntArray** execution_plan) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->GetExecutionPlan(execution_plan);
+  primary_subgraph().SetExternalContext(type, ctx);
 }
 
 TfLiteStatus Interpreter::SetInputs(std::vector<int> inputs) {
-  TF_LITE_ENSURE_OK(&context_,
-                    CheckTensorIndices("inputs", inputs.data(), inputs.size()));
-  inputs_ = std::move(inputs);
-  return kTfLiteOk;
+  return primary_subgraph().SetInputs(inputs);
 }
 
 TfLiteStatus Interpreter::SetOutputs(std::vector<int> outputs) {
-  TF_LITE_ENSURE_OK(
-      &context_, CheckTensorIndices("outputs", outputs.data(), outputs.size()));
-  outputs_ = std::move(outputs);
-  return kTfLiteOk;
+  return primary_subgraph().SetOutputs(outputs);
 }
 
 TfLiteStatus Interpreter::SetVariables(std::vector<int> variables) {
-  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("variables", variables.data(),
-                                                  variables.size()));
-  variables_ = std::move(variables);
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::CheckTensorIndices(const char* label,
-                                             const int* indices, int length) {
-  // Making sure kOptionalTensor is not re-defined to something other than -1.
-  static_assert(kOptionalTensor == -1, "kOptionalTensor should be defined -1");
-
-  for (int i = 0; i < length; i++) {
-    int index = indices[i];
-    // Continue if index == kOptionalTensor before additional comparisons below,
-    // size_t(-1) is always >= context_tensors_size.
-    if (index == kOptionalTensor) {
-      continue;
-    }
-    if (index < 0 || static_cast<size_t>(index) >= context_.tensors_size) {
-      ReportError(&context_, "Invalid tensor index %d in %s\n", index, label);
-      consistent_ = false;
-      return kTfLiteError;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::BytesRequired(TfLiteType type, const int* dims,
-                                        size_t dims_size, size_t* bytes) {
-  // TODO(aselle): Check for overflow here using overflow.h in TensorFlow
-  // MultiplyWithoutOverflow.
-  TF_LITE_ENSURE(&context_, bytes != nullptr);
-  size_t count = 1;
-  for (int k = 0; k < dims_size; k++) count *= dims[k];
-  switch (type) {
-    case kTfLiteFloat32:
-      *bytes = sizeof(float) * count;
-      break;
-    case kTfLiteInt16:
-      *bytes = sizeof(int16_t) * count;
-      break;
-    case kTfLiteInt32:
-      *bytes = sizeof(int32_t) * count;
-      break;
-    case kTfLiteUInt8:
-      *bytes = sizeof(uint8_t) * count;
-      break;
-    case kTfLiteInt8:
-      *bytes = sizeof(int8_t) * count;
-      break;
-    case kTfLiteInt64:
-      *bytes = sizeof(int64_t) * count;
-      break;
-    case kTfLiteBool:
-      *bytes = sizeof(bool) * count;
-      break;
-    case kTfLiteComplex64:
-      *bytes = sizeof(std::complex<float>) * count;
-      break;
-    default:
-      ReportError(&context_,
-                  "Only float32, int16, int32, int64, uint8, int8, bool, "
-                  "complex64 supported currently.");
-      return kTfLiteError;
-  }
-  return kTfLiteOk;
+  return primary_subgraph().SetVariables(variables);
 }
 
 TfLiteStatus Interpreter::AllocateTensors() {
-  if (!consistent_) {
-    ReportError(&context_, "AllocateTensors() called on inconsistent model.");
-    return kTfLiteError;
-  }
-
-  // Explicit (re)allocation is necessary if nodes have been changed or tensors
-  // have been resized. For inputs marked as dynamic, we can't short-circuit the
-  // allocation as the client may have done the resize manually.
-  if (state_ != kStateUninvokable && !HasDynamicTensorImpl(context_, inputs_)) {
-    return kTfLiteOk;
-  }
-
-  next_execution_plan_index_to_prepare_ = 0;
-  if (memory_planner_) {
-    TF_LITE_ENSURE_STATUS(memory_planner_->ResetAllocations());
-  }
-
-  TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-
-  state_ = kStateInvokable;
-
-  // Reset the variable tensors to zero after (re)allocating the tensors.
-  // Developers shouldn't rely on the side effect of this function to reset
-  // variable tesnsors. They should call `ResetVariableTensors` directly
-  // instead.
-  ResetVariableTensors();
-
-  return kTfLiteOk;
-}
-
-// TODO(ycling): Support non-zero default values.
-TfLiteStatus Interpreter::ResetVariableTensors() {
-  for (auto& tensor : tensors_) {
-    if (!tensor.is_variable) {
-      continue;
-    }
-
-    // Variable tensors have to be `kTfLiteArenaRwPersistent`, and must be
-    // allocated after the initial `PrepareOpsAndTensors()` is called.
-    TF_LITE_ENSURE_EQ(&context_, tensor.allocation_type,
-                      kTfLiteArenaRwPersistent);
-    TF_LITE_ENSURE(&context_, tensor.data.raw != nullptr);
-
-    memset(tensor.data.raw, 0, tensor.bytes);
-  }
-  return kTfLiteOk;
+  return primary_subgraph().AllocateTensors();
 }
 
 void Interpreter::ReserveNodes(int count) {
-  nodes_and_registration_.reserve(count);
+  primary_subgraph().nodes_and_registration().reserve(count);
 }
 
 TfLiteStatus Interpreter::AddNodeWithParameters(
     const std::vector<int>& inputs, const std::vector<int>& outputs,
     const char* init_data, size_t init_data_size, void* builtin_data,
     const TfLiteRegistration* registration, int* node_index) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(&context_,
-                "AddNodeWithParameters is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-  state_ = kStateUninvokable;
-
-  std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
-                                                              free);
-
-  TF_LITE_ENSURE_OK(&context_, CheckTensorIndices("node inputs", inputs.data(),
-                                                  inputs.size()));
-  TF_LITE_ENSURE_OK(
-      &context_,
-      CheckTensorIndices("node outputs", outputs.data(), outputs.size()));
-
-  int new_node_index = nodes_and_registration_.size();
-  if (node_index) *node_index = new_node_index;
-  nodes_and_registration_.resize(nodes_and_registration_.size() + 1);
-  auto& node_and_reg = nodes_and_registration_.back();
-  TfLiteNode& node = node_and_reg.first;
-  if (node.inputs) TfLiteIntArrayFree(node.inputs);
-  if (node.outputs) TfLiteIntArrayFree(node.outputs);
-  if (node.temporaries) TfLiteIntArrayFree(node.temporaries);
-
-  // NOTE, here we are not using move semantics yet, since our internal
-  // representation isn't std::vector, but in the future we would like to avoid
-  // copies, so we want the interface to take r-value references now.
-  node.inputs = ConvertVectorToTfLiteIntArray(inputs);
-  node.outputs = ConvertVectorToTfLiteIntArray(outputs);
-  node.temporaries = TfLiteIntArrayCreate(0);
-  if (init_data) {
-    node.user_data = OpInit(*registration, init_data, init_data_size);
-  } else {
-    node.user_data =
-        OpInit(*registration,
-               reinterpret_cast<const char*>(builtin_data_deleter.get()), 0);
-  }
-
-  node.builtin_data = builtin_data_deleter.release();
-  // TODO(ycling): Filling `custom_initial_data` and `custom_initial_data_size`
-  // properly for nodes generated by ReplaceNodeSubsetsWithDelegateKernels.
-
-  if (registration->builtin_code == BuiltinOperator_CUSTOM) {
-    // When it's a CUSTOM op, the `custom_options` field in the Flatbuffer
-    // `Operator` table is passed in.
-    node.custom_initial_data = init_data;
-    node.custom_initial_data_size = init_data_size;
-  } else {
-    node.custom_initial_data = nullptr;
-    node.custom_initial_data_size = 0;
-  }
-
-  node.delegate = nullptr;
-  node_and_reg.second = *registration;
-  execution_plan_.push_back(new_node_index);
-  return kTfLiteOk;
+  return primary_subgraph().AddNodeWithParameters(inputs, outputs, init_data,
+                                                  init_data_size, builtin_data,
+                                                  registration, node_index);
 }
 
 TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index,
                                             const std::vector<int>& dims) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(&context_,
-                "ResizeInputTensor is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-
-  // TODO(aselle): All bounds checks can be implemented as one-sided bounds
-  // checks by casting to unsigned for efficiency. Profile before doing this.
-  TF_LITE_ENSURE(&context_,
-                 tensor_index < context_.tensors_size && tensor_index >= 0);
-  TfLiteTensor* tensor = &context_.tensors[tensor_index];
-
-  // Short-circuit the state change if the dimensions don't change, avoiding
-  // unnecessary (re)allocations.
-  if (EqualArrayAndTfLiteIntArray(tensor->dims, dims.size(), dims.data())) {
-    return kTfLiteOk;
-  }
-
-  state_ = kStateUninvokable;
-  return ResizeTensorImpl(tensor, ConvertVectorToTfLiteIntArray(dims));
-}
-
-bool HasDynamicTensor(const TfLiteContext& context,
-                      const TfLiteIntArray* int_array) {
-  return HasDynamicTensorImpl(context, TfLiteIntArrayView{int_array});
-}
-
-TfLiteStatus Interpreter::PrepareOpsStartingAt(
-    int first_execution_plan_index, int* last_execution_plan_index_prepared) {
-  for (int execution_plan_index = first_execution_plan_index;
-       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
-    int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
-    const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
-    EnsureTensorsVectorCapacity();
-    if (OpPrepare(registration, &node) == kTfLiteError) {
-      return ReportOpError(&context_, node, registration, node_index,
-                           "failed to prepare");
-    }
-
-    *last_execution_plan_index_prepared = execution_plan_index;
-
-    // Discontinue if the node has dynamic outputs. Note that we don't
-    // stop for dynamic temporary tensors since they won't affect the
-    // sizes of other tensors in the graph.
-    if (HasDynamicTensor(context_, node.outputs)) {
-      break;
-    }
-  }
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::PrepareOpsAndTensors() {
-  if (!memory_planner_) {
-    memory_planner_.reset(new ArenaPlanner(
-        &context_, std::unique_ptr<GraphInfo>(new InterpreterInfo(this)),
-        /*preserve_inputs=*/true, /*preserve_intermediates*/ false));
-    memory_planner_->PlanAllocations();
-  }
-
-  int last_exec_plan_index_prepared = 0;
-
-  TF_LITE_ENSURE_STATUS(PrepareOpsStartingAt(
-      next_execution_plan_index_to_prepare_, &last_exec_plan_index_prepared));
-  TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations(
-      next_execution_plan_index_to_prepare_, last_exec_plan_index_prepared));
-
-  next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1;
-  return kTfLiteOk;
+  return primary_subgraph().ResizeInputTensor(tensor_index, dims);
 }
 
 TfLiteStatus Interpreter::Invoke() {
-  if (!consistent_) {
-    ReportError(&context_, "Invoke called on model that is not consistent.");
-    return kTfLiteError;
-  }
-  if (state_ == kStateUninvokable) {
-    ReportError(&context_, "Invoke called on model that is not ready.");
-    return kTfLiteError;
-  }
-
-  TfLiteStatus status = kTfLiteOk;
-  if (nnapi_delegate_) {
-    if (next_execution_plan_index_to_prepare_ == execution_plan_.size()) {
-      TF_LITE_ENSURE_OK(&context_, nnapi_delegate_->Invoke(this));
-      return kTfLiteOk;
-    } else {
-      // TODO(aselle): In the future, we would like this to be an
-      // automatic tflite CPU fallback.
-      ReportError(&context_,
-                  "NNAPI was requested, but dependent sized tensors "
-                  "being used.\n");
-      return kTfLiteError;
-    }
-  }
-
-  // Invocations are always done in node order.
-  // Note that calling Invoke repeatedly will cause the original memory plan to
-  // be reused, unless either ResizeInputTensor() or AllocateTensors() has been
-  // called.
-  for (int execution_plan_index = 0;
-       execution_plan_index < execution_plan_.size(); execution_plan_index++) {
-    if (execution_plan_index == next_execution_plan_index_to_prepare_) {
-      TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-      TF_LITE_ENSURE(&context_, next_execution_plan_index_to_prepare_ >=
-                                    execution_plan_index);
-    }
-    int node_index = execution_plan_[execution_plan_index];
-    TfLiteNode& node = nodes_and_registration_[node_index].first;
-    const TfLiteRegistration& registration =
-        nodes_and_registration_[node_index].second;
-    SCOPED_OPERATOR_PROFILE(profiler_, node_index);
-
-    // TODO(ycling): This is an extra loop through inputs to check if the data
-    // need to be copied from Delegate buffer to raw memory, which is often not
-    // needed. We may want to cache this in prepare to know if this needs to be
-    // done for a node or not.
-    for (int i = 0; i < node.inputs->size; ++i) {
-      int tensor_index = node.inputs->data[i];
-      if (tensor_index == kOptionalTensor) {
-        continue;
-      }
-      TfLiteTensor* tensor = &tensors_[tensor_index];
-      if (tensor->delegate && tensor->delegate != node.delegate &&
-          tensor->data_is_stale) {
-        EnsureTensorDataIsReadable(tensor_index);
-      }
-    }
-
-    EnsureTensorsVectorCapacity();
-    tensor_resized_since_op_invoke_ = false;
-    if (OpInvoke(registration, &node) == kTfLiteError) {
-      status = ReportOpError(&context_, node, registration, node_index,
-                             "failed to invoke");
-    }
-
-    // Force execution prep for downstream ops if the latest op triggered the
-    // resize of a dynamic tensor.
-    if (tensor_resized_since_op_invoke_ &&
-        HasDynamicTensor(context_, node.outputs)) {
-      next_execution_plan_index_to_prepare_ = execution_plan_index + 1;
-    }
-  }
+  TfLiteStatus status = primary_subgraph().Invoke();
 
   if (!allow_buffer_handle_output_) {
-    for (int tensor_index : outputs_) {
-      EnsureTensorDataIsReadable(tensor_index);
+    for (int tensor_index : outputs()) {
+      primary_subgraph().EnsureTensorDataIsReadable(tensor_index);
     }
   }
 
   return status;
 }
 
-TfLiteStatus Interpreter::ResizeTensor(TfLiteContext* context,
-                                       TfLiteTensor* tensor,
-                                       TfLiteIntArray* new_size) {
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function ResizeTensorImpl
-  // (this function is static).
-  return static_cast<Interpreter*>(context->impl_)
-      ->ResizeTensorImpl(tensor, new_size);
-}
-
-void Interpreter::ReportErrorImpl(const char* format, va_list args) {
-  error_reporter_->Report(format, args);
-}
-
-void Interpreter::ReportError(TfLiteContext* context, const char* format, ...) {
-  va_list args;
-  va_start(args, format);
-  auto* f = static_cast<Interpreter*>(context->impl_);
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function ReportErrorImpl
-  // (this function is static).
-  f->ReportErrorImpl(format, args);
-  va_end(args);
-}
-
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
                                      int* first_new_tensor_index) {
-  const size_t base_index = tensors_.size();
-  if (first_new_tensor_index) *first_new_tensor_index = base_index;
-  tensors_.resize(tensors_.size() + tensors_to_add);
-  for (size_t i = base_index; i < tensors_.size(); i++) {
-    memset(&tensors_[i], 0, sizeof(tensors_[i]));
-    tensors_[i].buffer_handle = kTfLiteNullBufferHandle;
-  }
-  context_.tensors = tensors_.data();
-  context_.tensors_size = tensors_.size();
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::AddTensors(TfLiteContext* context, int tensors_to_add,
-                                     int* first_new_tensor_index) {
-  // Note here that context->impl_ is recovering the this pointer for an
-  // instance of Interpreter to call into the member function AddTensors
-  // (this function is static).
-  return static_cast<Interpreter*>(context->impl_)
-      ->AddTensors(tensors_to_add, first_new_tensor_index);
+  return primary_subgraph().AddTensors(tensors_to_add, first_new_tensor_index);
 }
 
-TfLiteStatus Interpreter::GetNodeAndRegistration(
-    int node_index, TfLiteNode** node, TfLiteRegistration** registration) {
-  TF_LITE_ENSURE(&context_, node_index >= 0);
-  TF_LITE_ENSURE(&context_, static_cast<size_t>(node_index) < nodes_size());
-  TF_LITE_ENSURE(&context_, node != nullptr && registration != nullptr);
-  *node = &nodes_and_registration_[node_index].first;
-  *registration = &nodes_and_registration_[node_index].second;
-  return kTfLiteOk;
-}
-
-TfLiteStatus Interpreter::GetNodeAndRegistration(
-    struct TfLiteContext* context, int node_index, TfLiteNode** node,
-    TfLiteRegistration** registration) {
-  return static_cast<Interpreter*>(context->impl_)
-      ->GetNodeAndRegistration(node_index, node, registration);
+TfLiteStatus Interpreter::ResetVariableTensors() {
+  return primary_subgraph().ResetVariableTensors();
 }
 
 TfLiteStatus Interpreter::SetTensorParametersReadOnly(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
     size_t bytes, const Allocation* allocation) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(
-        &context_,
-        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-
-  TF_LITE_ENSURE(&context_,
-                 tensor_index < context_.tensors_size && tensor_index >= 0);
-  // For most tensors we know exactly how much memory is necessary so we can
-  // ensure the buffer is large enough. However, we need to skip string tensors
-  // because their sizes change with the contents of the individual strings.
-  if (type != kTfLiteString) {
-    size_t required_bytes;
-    TF_LITE_ENSURE_OK(&context_,
-                      BytesRequired(type, dims, rank, &required_bytes));
-    TF_LITE_ENSURE_EQ(&context_, required_bytes, bytes);
-  }
-
-  TfLiteTensor& tensor = context_.tensors[tensor_index];
-  if (type == tensor.type &&
-      EqualArrayAndTfLiteIntArray(tensor.dims, rank, dims)) {
-    // Fast path which does not invalidate the invokable property.
-    TfLiteTensorDataFree(&tensor);
-    tensor.data.raw = const_cast<char*>(buffer);
-    if (!tensor.dims) tensor.dims = ConvertArrayToTfLiteIntArray(rank, dims);
-    tensor.params = quantization;
-    tensor.allocation_type = kTfLiteMmapRo;
-    tensor.allocation = allocation;
-  } else {
-    state_ = kStateUninvokable;
-    TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                      quantization, const_cast<char*>(buffer), bytes,
-                      kTfLiteMmapRo, allocation, false, &tensor);
-  }
-  return kTfLiteOk;
+  return primary_subgraph().SetTensorParametersReadOnly(
+      tensor_index, type, name, rank, dims, quantization, buffer, bytes,
+      allocation);
 }
 
 // Set description of inputs/outputs/data/fptrs for node `node_index`.
@@ -819,186 +124,52 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly(
 TfLiteStatus Interpreter::SetTensorParametersReadWrite(
     int tensor_index, TfLiteType type, const char* name, const size_t rank,
     const int* dims, TfLiteQuantizationParams quantization, bool is_variable) {
-  if (state_ == kStateInvokableAndImmutable) {
-    ReportError(
-        &context_,
-        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
-    return kTfLiteError;
-  }
-  TF_LITE_ENSURE(&context_,
-                 tensor_index < context_.tensors_size && tensor_index >= 0);
-  size_t required_bytes = 0;
-  if (type != kTfLiteString) {
-    // These types will be allocated in our arena so we need to record how
-    // many bytes we will need based on the dimensions. String tensors are
-    // allocated dynamically and we can't know ahead of time how much space
-    // they will require.
-    TF_LITE_ENSURE_OK(&context_,
-                      BytesRequired(type, dims, rank, &required_bytes));
-  }
-
-  TfLiteAllocationType allocation_type = kTfLiteArenaRw;
-  if (type == kTfLiteString) {
-    if (is_variable) {
-      // We don't have a real use case for string variable tensor.
-      ReportError(&context_, "String variable tensor isn't supported.");
-      return kTfLiteError;
-    }
-    allocation_type = kTfLiteDynamic;
-  } else if (is_variable) {
-    allocation_type = kTfLiteArenaRwPersistent;
-  }
-
-  TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
-                    quantization,
-                    /*buffer=*/nullptr, required_bytes, allocation_type,
-                    nullptr, is_variable, &context_.tensors[tensor_index]);
-  return kTfLiteOk;
+  return primary_subgraph().SetTensorParametersReadWrite(
+      tensor_index, type, name, rank, dims, quantization, is_variable);
 }
 
 TfLiteStatus Interpreter::SetExecutionPlan(const std::vector<int>& new_plan) {
-  for (int node_index : new_plan) {
-    TF_LITE_ENSURE(&context_, node_index >= 0 && node_index < nodes_size());
-  }
-  execution_plan_ = new_plan;
-  return kTfLiteOk;
+  return primary_subgraph().SetExecutionPlan(new_plan);
 }
 
-TfLiteStatus Interpreter::ResizeTensorImpl(TfLiteTensor* tensor,
-                                           TfLiteIntArray* new_size) {
-  // Note that in theory we could resize kTfLiteArenaRwPersistent tensors too.
-  if (tensor->allocation_type == kTfLiteArenaRw ||
-      tensor->allocation_type == kTfLiteDynamic ||
-      tensor->allocation_type == kTfLiteArenaRwPersistent) {
-    tensor_resized_since_op_invoke_ |=
-        TfLiteIntArrayEqual(tensor->dims, new_size) == 0;
-    if (tensor->type != kTfLiteString) {
-      size_t bytesRequired;
-      TfLiteStatus status = BytesRequired(tensor->type, new_size->data,
-                                          new_size->size, &bytesRequired);
-      if (status != kTfLiteOk) {
-        TfLiteIntArrayFree(new_size);
-        return kTfLiteError;
-      }
-
-      // Realloc space for kTfLiteDynamic tensors.
-      TfLiteTensorRealloc(bytesRequired, tensor);
-      tensor->bytes = bytesRequired;
-    }
-    if (tensor->dims) TfLiteIntArrayFree(tensor->dims);
-    tensor->dims = new_size;
-
-    if (tensor->allocation_type != kTfLiteDynamic) {
-      tensor->data.raw = nullptr;
-    }
-  } else {
-    // kTfLiteMmapRo tensors are stored in the flatbuffer and are therefore
-    // of fixed size.
-    TfLiteIntArrayFree(new_size);
-    ReportError(&context_, "Attempting to resize a fixed-size tensor.");
-    return kTfLiteError;
-  }
-  return kTfLiteOk;
-}
-
-void Interpreter::UseNNAPI(bool enable) {
-  // TODO(aselle): This is a workaround for finding if NNAPI exists.
-  // We also need to make sure getLibraryHandle() is renamed to be NNAPI
-  // prefixed.
-  if (!NNAPIDelegate::IsSupported()) enable = false;
-  if (!enable) {
-    nnapi_delegate_.reset();
-  } else if (!nnapi_delegate_) {
-    nnapi_delegate_.reset(new NNAPIDelegate);
-  }
-}
+void Interpreter::UseNNAPI(bool enable) { primary_subgraph().UseNNAPI(enable); }
 
 void Interpreter::SetNumThreads(int num_threads) {
-  context_.recommended_num_threads = num_threads;
+  for (auto& subgraph : subgraphs_) {
+    subgraph.context()->recommended_num_threads = num_threads;
+  }
 
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
     auto* c = external_contexts_[i];
     if (c && c->Refresh) {
-      c->Refresh(&context_);
+      c->Refresh(context_);
     }
   }
 }
 
-void Interpreter::SwitchToDelegateContext() {
-  context_.GetNodeAndRegistration = GetNodeAndRegistration;
-  context_.ReplaceNodeSubsetsWithDelegateKernels =
-      ReplaceNodeSubsetsWithDelegateKernels;
-  context_.GetExecutionPlan = GetExecutionPlan;
-}
-
-void Interpreter::SwitchToKernelContext() {
-  SetForbiddenContextFunction(&context_.GetNodeAndRegistration);
-  SetForbiddenContextFunction(&context_.ReplaceNodeSubsetsWithDelegateKernels);
-  SetForbiddenContextFunction(&context_.GetExecutionPlan);
+void Interpreter::SetAllowFp16PrecisionForFp32(bool allow) {
+  for (auto& subgraph : subgraphs_) {
+    subgraph.context()->allow_fp32_relax_to_fp16 = allow;
+  }
 }
 
 TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
-  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
-    int last_execution_plan_index_prepared;
-    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
-                                     0, &last_execution_plan_index_prepared));
-
-    bool has_dynamic_tensors = true;
-    // Dynamic tensors exist if not all nodes can be prepared.
-    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
-      // If all the nodes can be prepared, check if the last node has dynamic
-      // tensors.
-      int node_index = execution_plan_[last_execution_plan_index_prepared];
-      TfLiteNode& node = nodes_and_registration_[node_index].first;
-      if (!HasDynamicTensor(context_, node.outputs)) {
-        has_dynamic_tensors = false;
-      }
-    }
-    if (has_dynamic_tensors) {
-      ReportError(
-          &context_,
-          "Attempting to use a delegate that only supports static-sized "
-          "tensors with a graph that has dynamic-sized tensors.");
-      return kTfLiteError;
-    }
-  }
-
-  // TODO(aselle): Consider if it is worth storing pointers to delegates.
-  // Setup additional context interface.
-  SwitchToDelegateContext();
-
-  TfLiteStatus status = delegate->Prepare(&context_, delegate);
-
-  // Remove additional context info.
-  SwitchToKernelContext();
-
-  TF_LITE_ENSURE_OK(&context_, status);
-
-  if (!(delegate->flags & kTfLiteDelegateFlagsAllowDynamicTensors)) {
-    // Reset the state to force tensor/op reallocation.
-    state_ = kStateUninvokable;
-    TF_LITE_ENSURE_OK(&context_, AllocateTensors());
-    TF_LITE_ENSURE_EQ(&context_, state_, kStateInvokable);
-    // After using a delegate which doesn't support dynamic tensors, make the
-    // entire graph immutable.
-    state_ = kStateInvokableAndImmutable;
-  }
-
-  return status;
+  return primary_subgraph().ModifyGraphWithDelegate(delegate);
 }
 
 TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle buffer_handle,
                                           TfLiteDelegate* delegate) {
-  TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  TfLiteTensor* tensor = &tensors_[tensor_index];
+  TF_LITE_ENSURE(context_, tensor_index < tensors_size());
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  TfLiteTensor* tensor = &tensors[tensor_index];
 
-  TF_LITE_ENSURE(&context_,
+  TF_LITE_ENSURE(context_,
                  tensor->delegate == nullptr || tensor->delegate == delegate);
   tensor->delegate = delegate;
   if (tensor->buffer_handle != kTfLiteNullBufferHandle) {
-    TF_LITE_ENSURE(&context_, tensor->delegate->FreeBufferHandle != nullptr);
-    tensor->delegate->FreeBufferHandle(&context_, tensor->delegate,
+    TF_LITE_ENSURE(context_, tensor->delegate->FreeBufferHandle != nullptr);
+    tensor->delegate->FreeBufferHandle(context_, tensor->delegate,
                                        &tensor->buffer_handle);
   }
   tensor->buffer_handle = buffer_handle;
@@ -1009,8 +180,9 @@ TfLiteStatus Interpreter::SetBufferHandle(int tensor_index,
 TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
                                           TfLiteBufferHandle* buffer_handle,
                                           TfLiteDelegate** delegate) {
-  TF_LITE_ENSURE(&context_, tensor_index < tensors_size());
-  TfLiteTensor* tensor = &tensors_[tensor_index];
+  TF_LITE_ENSURE(context_, tensor_index < tensors_size());
+  std::vector<TfLiteTensor>& tensors = primary_subgraph().tensors();
+  TfLiteTensor* tensor = &tensors[tensor_index];
 
   *delegate = tensor->delegate;
   *buffer_handle = tensor->buffer_handle;
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index de6c42bfcc..d89affff45 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/memory_planner.h"
 #include "tensorflow/lite/profiling/profiler.h"
 #include "tensorflow/lite/stderr_reporter.h"
@@ -73,9 +74,6 @@ constexpr TfLiteType typeToTfLiteType<string>() {
   return kTfLiteString;
 }
 
-// Forward declare since NNAPIDelegate uses Interpreter.
-class NNAPIDelegate;
-
 // An interpreter for a graph of nodes that input and output from tensors.
 // Each node of the graph processes a set of input tensors and produces a
 // set of output Tensors. All inputs/output tensors are referenced by index.
@@ -104,12 +102,6 @@ class NNAPIDelegate;
 // foo.Invoke();
 //
 
-struct TfLiteIntArrayDeleter {
-  void operator()(TfLiteIntArray* a) {
-    if (a) TfLiteIntArrayFree(a);
-  }
-};
-
 class Interpreter {
  public:
   // Instantiate an interpreter. All errors associated with reading and
@@ -121,6 +113,7 @@ class Interpreter {
 
   ~Interpreter();
 
+  // Interpreters are not copyable as they have non-trivial memory semantics.
   Interpreter(const Interpreter&) = delete;
   Interpreter& operator=(const Interpreter&) = delete;
 
@@ -201,34 +194,40 @@ class Interpreter {
   // Functions to access tensor data
 
   // Read only access to list of inputs.
-  const std::vector<int>& inputs() const { return inputs_; }
+  const std::vector<int>& inputs() const { return primary_subgraph().inputs(); }
 
   // Return the name of a given input. The given index must be between 0 and
   // inputs().size().
   const char* GetInputName(int index) const {
-    return context_.tensors[inputs_[index]].name;
+    return context_->tensors[inputs()[index]].name;
   }
 
   // Read only access to list of outputs.
-  const std::vector<int>& outputs() const { return outputs_; }
+  const std::vector<int>& outputs() const {
+    return primary_subgraph().outputs();
+  }
 
   // Read only access to list of variable tensors.
-  const std::vector<int>& variables() const { return variables_; }
+  const std::vector<int>& variables() const {
+    return primary_subgraph().variables();
+  }
 
   // Return the name of a given output. The given index must be between 0 and
   // outputs().size().
   const char* GetOutputName(int index) const {
-    return context_.tensors[outputs_[index]].name;
+    return context_->tensors[outputs()[index]].name;
   }
 
   // Return the number of tensors in the model.
-  size_t tensors_size() const { return context_.tensors_size; }
+  size_t tensors_size() const { return context_->tensors_size; }
 
   // Return the number of ops in the model.
-  size_t nodes_size() const { return nodes_and_registration_.size(); }
+  size_t nodes_size() const { return primary_subgraph().nodes_size(); }
 
   // WARNING: Experimental interface, subject to change
-  const std::vector<int>& execution_plan() const { return execution_plan_; }
+  const std::vector<int>& execution_plan() const {
+    return primary_subgraph().execution_plan();
+  }
 
   // WARNING: Experimental interface, subject to change
   // Overrides execution plan. This bounds checks indices sent in.
@@ -238,27 +237,18 @@ class Interpreter {
   // TODO(aselle): Create a safe ArrayHandle interface to avoid exposing this
   // read/write access to structure
   TfLiteTensor* tensor(int tensor_index) {
-    if (tensor_index < 0 ||
-        static_cast<size_t>(tensor_index) >= context_.tensors_size)
-      return nullptr;
-    return &context_.tensors[tensor_index];
+    return primary_subgraph().tensor(tensor_index);
   }
 
   // Get an immutable tensor data structure.
   const TfLiteTensor* tensor(int tensor_index) const {
-    if (tensor_index < 0 ||
-        static_cast<size_t>(tensor_index) >= context_.tensors_size)
-      return nullptr;
-    return &context_.tensors[tensor_index];
+    return primary_subgraph().tensor(tensor_index);
   }
 
   // Get a pointer to an operation and registration data structure if in bounds.
   const std::pair<TfLiteNode, TfLiteRegistration>* node_and_registration(
       int node_index) const {
-    if (node_index < 0 ||
-        static_cast<size_t>(node_index) >= nodes_and_registration_.size())
-      return nullptr;
-    return &nodes_and_registration_[node_index];
+    return primary_subgraph().node_and_registration(node_index);
   }
 
   // Perform a checked cast to the appropriate tensor type (mutable pointer
@@ -289,28 +279,28 @@ class Interpreter {
   // index must be between 0 and inputs().size().
   template <class T>
   T* typed_input_tensor(int index) {
-    return typed_tensor<T>(inputs_[index]);
+    return typed_tensor<T>(inputs()[index]);
   }
 
   // Return an immutable pointer into the data of a given input tensor. The
   // given index must be between 0 and inputs().size().
   template <class T>
   const T* typed_input_tensor(int index) const {
-    return typed_tensor<T>(inputs_[index]);
+    return typed_tensor<T>(inputs()[index]);
   }
 
   // Return a mutable pointer into the data of a given output tensor. The given
   // index must be between 0 and outputs().size().
   template <class T>
   T* typed_output_tensor(int index) {
-    return typed_tensor<T>(outputs_[index]);
+    return typed_tensor<T>(outputs()[index]);
   }
 
   // Return an immutable pointer into the data of a given output tensor. The
   // given index must be between 0 and outputs().size().
   template <class T>
   const T* typed_output_tensor(int index) const {
-    return typed_tensor<T>(outputs_[index]);
+    return typed_tensor<T>(outputs()[index]);
   }
 
   // Change the dimensionality of a given tensor. Note, this is only acceptable
@@ -325,7 +315,6 @@ class Interpreter {
   // Update allocations for all tensors. This will redim dependent tensors using
   // the input tensor dimensionality as given. This is relatively expensive.
   // If you know that your sizes are not changing, you need not call this.
-
   // Returns status of success or failure.
   TfLiteStatus AllocateTensors();
 
@@ -346,14 +335,12 @@ class Interpreter {
   // Allow float16 precision for FP32 calculation when possible.
   // default: not allow.
   // WARNING: This is an experimental API and subject to change.
-  void SetAllowFp16PrecisionForFp32(bool allow) {
-    context_.allow_fp32_relax_to_fp16 = allow;
-  }
+  void SetAllowFp16PrecisionForFp32(bool allow);
 
   // Get the half precision flag.
   // WARNING: This is an experimental API and subject to change.
   bool GetAllowFp16PrecisionForFp32() const {
-    return context_.allow_fp32_relax_to_fp16;
+    return context_->allow_fp32_relax_to_fp16;
   }
 
   // Owning handle to a TfLiteDelegate instance.
@@ -370,18 +357,7 @@ class Interpreter {
   // it might require to copy the data from delegate buffer to raw memory.
   // WARNING: This is an experimental API and subject to change.
   TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
-    TfLiteTensor* t = tensor(tensor_index);
-    TF_LITE_ENSURE(&context_, t != nullptr);
-    if (t->data_is_stale) {
-      TF_LITE_ENSURE(&context_, t->delegate != nullptr);
-      TF_LITE_ENSURE(&context_, t->buffer_handle != kTfLiteNullBufferHandle);
-      // This can be null if the delegate doesn't use its own buffer.
-      TF_LITE_ENSURE(&context_, t->delegate->CopyFromBufferHandle != nullptr);
-      t->delegate->CopyFromBufferHandle(
-          &context_, t->delegate, t->buffer_handle, t->data.raw, t->bytes);
-      t->data_is_stale = false;
-    }
-    return kTfLiteOk;
+    return primary_subgraph().EnsureTensorDataIsReadable(tensor_index);
   }
 
   // Set the delegate buffer handle to a tensor. It can be called in the
@@ -439,7 +415,7 @@ class Interpreter {
   const char* OpProfilingString(const TfLiteRegistration& op_reg,
                                 const TfLiteNode* node) const {
     if (op_reg.profiling_string == nullptr) return nullptr;
-    return op_reg.profiling_string(&context_, node);
+    return op_reg.profiling_string(context_, node);
   }
 
   // Set the value of an external context.
@@ -450,131 +426,19 @@ class Interpreter {
   friend class InterpreterBuilder;
   friend class InterpreterTest;
 
-  // Prevent 'context_' from accessing functions that are only available to
-  // delegated kernels.
-  void SwitchToKernelContext();
-
-  // Add delegate-only functions to 'context_'.
-  void SwitchToDelegateContext();
-
-  // Give 'op_reg' a chance to initialize itself using the contents of
-  // 'buffer'.
-  void* OpInit(const TfLiteRegistration& op_reg, const char* buffer,
-               size_t length) {
-    if (op_reg.init == nullptr) return nullptr;
-    return op_reg.init(&context_, buffer, length);
+  Subgraph& primary_subgraph() {
+    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
-  // Let 'op_reg' release any memory it might have allocated via 'OpInit'.
-  void OpFree(const TfLiteRegistration& op_reg, void* buffer) {
-    if (op_reg.free == nullptr) return;
-    if (buffer) {
-      op_reg.free(&context_, buffer);
-    }
+  const Subgraph& primary_subgraph() const {
+    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
-  // Prepare the given 'node' for execution.
-  TfLiteStatus OpPrepare(const TfLiteRegistration& op_reg, TfLiteNode* node) {
-    if (op_reg.prepare == nullptr) return kTfLiteOk;
-    return op_reg.prepare(&context_, node);
-  }
-
-  // Invoke the operator represented by 'node'.
-  TfLiteStatus OpInvoke(const TfLiteRegistration& op_reg, TfLiteNode* node) {
-    if (op_reg.invoke == nullptr) return kTfLiteError;
-    return op_reg.invoke(&context_, node);
-  }
-
-  // Call OpPrepare() for as many ops as possible, allocating memory for their
-  // tensors. If an op containing dynamic tensors is found, preparation will be
-  // postponed until this function is called again. This allows the interpreter
-  // to wait until Invoke() to resolve the sizes of dynamic tensors.
-  TfLiteStatus PrepareOpsAndTensors();
-
-  // Call OpPrepare() for all ops starting at 'first_node'. Stop when a
-  // dynamic tensors is found or all ops have been prepared. Fill
-  // 'last_node_prepared' with the id of the op containing dynamic tensors, or
-  // the last in the graph.
-  TfLiteStatus PrepareOpsStartingAt(int first_execution_plan_index,
-                                    int* last_execution_plan_index_prepared);
-
   // Tensors needed by the interpreter. Use `AddTensors` to add more blank
   // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
   // `context_` whenever this std::vector is reallocated. Currently this
   // only happens in `AddTensors()`.
-  std::vector<TfLiteTensor> tensors_;
-
-  // Check if an array of tensor indices are valid with respect to the Tensor
-  // array.
-  // NOTE: this changes consistent_ to be false if indices are out of bounds.
-  TfLiteStatus CheckTensorIndices(const char* label, const int* indices,
-                                  int length);
-
-  // Compute the number of bytes required to represent a tensor with dimensions
-  // specified by the array dims (of length dims_size). Returns the status code
-  // and bytes.
-  TfLiteStatus BytesRequired(TfLiteType type, const int* dims, size_t dims_size,
-                             size_t* bytes);
-
-  // Request an tensor be resized implementation. If the given tensor is of
-  // type kTfLiteDynamic it will also be allocated new memory.
-  TfLiteStatus ResizeTensorImpl(TfLiteTensor* tensor, TfLiteIntArray* new_size);
-
-  // Report a detailed error string (will be printed to stderr).
-  // TODO(aselle): allow user of class to provide alternative destinations.
-  void ReportErrorImpl(const char* format, va_list args);
-
-  // Entry point for C node plugin API to request an tensor be resized.
-  static TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor,
-                                   TfLiteIntArray* new_size);
-  // Entry point for C node plugin API to report an error.
-  static void ReportError(TfLiteContext* context, const char* format, ...);
-
-  // Entry point for C node plugin API to add new tensors.
-  static TfLiteStatus AddTensors(TfLiteContext* context, int tensors_to_add,
-                                 int* first_new_tensor_index);
-
-  // WARNING: This is an experimental API and subject to change.
-  // Entry point for C API ReplaceNodeSubsetsWithDelegateKernels
-  static TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
-      TfLiteContext* context, TfLiteRegistration registration,
-      const TfLiteIntArray* nodes_to_replace, TfLiteDelegate* delegate);
-
-  // Update the execution graph to replace some of the nodes with stub
-  // nodes. Specifically any node index that has `nodes[index]==1` will be
-  // slated for replacement with a delegate kernel specified by registration.
-  // Ownership of 'nodes_to_replace' and 'delegate' remains with the caller.
-  // WARNING: This is an experimental interface that is subject to change.
-  TfLiteStatus ReplaceNodeSubsetsWithDelegateKernels(
-      TfLiteRegistration registration, const TfLiteIntArray* nodes_to_replace,
-      TfLiteDelegate* delegate);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Gets the internal pointer to a TensorFlow lite node by node_index.
-  TfLiteStatus GetNodeAndRegistration(int node_index, TfLiteNode** node,
-                                      TfLiteRegistration** registration);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Entry point for C node plugin API to get a node by index.
-  static TfLiteStatus GetNodeAndRegistration(struct TfLiteContext*,
-                                             int node_index, TfLiteNode** node,
-                                             TfLiteRegistration** registration);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Gets an TfLiteIntArray* representing the execution plan. The interpreter
-  // owns this memory and it is only guaranteed to exist during the invocation
-  // of the delegate prepare.
-  TfLiteStatus GetExecutionPlan(TfLiteIntArray** execution_plan);
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // Entry point for C node plugin API to get the execution plan.
-  static TfLiteStatus GetExecutionPlan(struct TfLiteContext* context,
-                                       TfLiteIntArray** execution_plan);
-
-  // Retrieve an existing external context by type.
-  TfLiteExternalContext* GetExternalContext(TfLiteExternalContextType type);
-  static TfLiteExternalContext* GetExternalContext(
-      struct TfLiteContext* context, TfLiteExternalContextType type);
+  // std::vector<TfLiteTensor> tensors_;
 
   // Set the value of an external context.
   static void SetExternalContext(struct TfLiteContext* context,
@@ -591,105 +455,31 @@ class Interpreter {
     return ModifyGraphWithDelegate(owned_delegates_.back().get());
   }
 
-  // Ensures that `tensors_` has at least `kTensorsCapacityHeadroom` extra
-  // capacity. Calling this function may invalidate existing pointers to
-  // tensors. After calling this function, adding `kTensorsCapacityHeadroom`
-  // more tensors won't invalidate the pointer to existing tensors.
-  void EnsureTensorsVectorCapacity() {
-    const size_t required_capacity = tensors_size() + kTensorsCapacityHeadroom;
-    if (required_capacity > tensors_.capacity()) {
-      tensors_.reserve(required_capacity);
-      context_.tensors = tensors_.data();
-    }
-  }
-
-  // The state of the Interpreter.
-  enum State {
-    // The interpreter isn't ready to be invoked.
-    // `AllocateTensor` need to be called to enter an invokable state.
-    kStateUninvokable = 0,
-    // The interpreter is ready to be invoked.
-    kStateInvokable,
-    // The interpreter is ready to be invoked, and graph can't be further
-    // modified. The interpreter will enter this state when calling
-    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
-    kStateInvokableAndImmutable,
-  };
-  State state_ = kStateUninvokable;
-
   // A pure C data structure used to communicate with the pure C plugin
   // interface. To avoid copying tensor metadata, this is also the definitive
   // structure to store tensors.
-  TfLiteContext context_;
-
-  // Node inputs/outputs are stored in TfLiteNode and TfLiteRegistration stores
-  // function pointers to actual implementation.
-  std::vector<std::pair<TfLiteNode, TfLiteRegistration>>
-      nodes_and_registration_;
-
-  // Whether the model is consistent. That is to say if the inputs and outputs
-  // of every node and the global inputs and outputs are valid indexes into
-  // the tensor array.
-  bool consistent_ = true;
-
-  // Array of indices representing the tensors that are inputs to the
-  // interpreter.
-  std::vector<int> inputs_;
-
-  // Array of indices representing the tensors that are outputs to the
-  // interpreter.
-  std::vector<int> outputs_;
-
-  // Array of indices representing the tensors that are variable tensors.
-  std::vector<int> variables_;
+  // This is the primary subgraph context.
+  TfLiteContext* context_;
 
   // The error reporter delegate that tflite will forward queries errors to.
   ErrorReporter* error_reporter_;
 
-  // Index of the next node to prepare.
-  // During Invoke(), Interpreter will allocate input tensors first, which are
-  // known to be fixed size. Then it will allocate outputs from nodes as many
-  // as possible. When there is a node that produces dynamic sized tensor.
-  // Interpreter will stop allocating tensors, set the value of next allocate
-  // node id, and execute the node to generate the output tensor before continue
-  // to allocate successors. This process repeats until all nodes are executed.
-  // NOTE: this relies on the order of nodes that is in topological order.
-  int next_execution_plan_index_to_prepare_;
-
-  // WARNING: This is an experimental interface that is subject to change.
-  // This is a list of node indices (to index into nodes_and_registration).
-  // This represents a valid topological sort (dependency ordered) execution
-  // plan. In particular, it is valid for this ordering to contain only a
-  // subset of the node indices.
-  std::vector<int> execution_plan_;
-
-  // In the future, we'd like a TfLiteIntArray compatible representation.
-  // TODO(aselle): replace execution_plan_ with this.
-  std::unique_ptr<TfLiteIntArray, TfLiteIntArrayDeleter> plan_cache_;
-
-  // Whether to delegate to NN API
-  std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
-
   // List of delegates that have been installed and are owned by this
   // interpreter instance. Useful if client delegate ownership is burdensome.
   // WARNING: This is an experimental API and subject to change.
   // TODO(b/116667551): Use TfLiteExternalContext for storing state.
   std::vector<TfLiteDelegatePtr> owned_delegates_;
 
-  std::unique_ptr<MemoryPlanner> memory_planner_;
-
   bool allow_buffer_handle_output_ = false;
 
-  // Tracking bit for whether a tensor was resized in the course of an op
-  // invocation. This is a useful hint to ensure that dynamic tensor outputs
-  // trigger downstream reallocation after op invocation.
-  bool tensor_resized_since_op_invoke_ = false;
-
   // Profiler for this interpreter instance.
   profiling::Profiler* profiler_ = nullptr;
 
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
+
+  // Subgraphs
+  std::vector<Subgraph> subgraphs_;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index 7f03c3ceba..2e0dc77dcd 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -38,7 +38,7 @@ class InterpreterTest : public ::testing::Test {
   }
 
  protected:
-  TfLiteContext* GetInterpreterContext() { return &interpreter_.context_; }
+  TfLiteContext* GetInterpreterContext() { return interpreter_.context_; }
 
   Interpreter interpreter_;
 };
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index e78ce554ed..292dedf1c9 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -140,13 +140,13 @@ NNAPIDelegate::~NNAPIDelegate() {
   // ANeuralNetworksShutdown();
 }
 
-// Adds the tensors of the interpreter to the NN API model.
-TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter,
+// Adds the tensors of the subgraph to the NN API model.
+TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
   uint32_t next_id = 0;
-  for (size_t i = 0; i < interpreter->tensors_size(); i++) {
+  for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
     if ((*nnapi_ids)[i] == kOperandNotNeeded) continue;
 
@@ -156,7 +156,7 @@ TfLiteStatus addTensorOperands(tflite::Interpreter* interpreter,
     // NNAPI requires 32-bit float scale to be zero, tflite doesn't care
     float scale = 0.0f;
     int32_t zeroPoint = 0;
-    TfLiteTensor* tensor = interpreter->tensor(i);
+    TfLiteTensor* tensor = subgraph->tensor(i);
     switch (tensor->type) {
       case kTfLiteNoType:
         // Tensors added during initialization of Ops don't have a type yet and
@@ -240,12 +240,12 @@ void MapAndAddTensorIds(const int* from_ids_buf, size_t from_ids_count,
 // Adds the operations and their parameters to the NN API model.
 // 'next-id' is the operand ID of the next operand of the model.
 TfLiteStatus AddOpsAndParams(
-    tflite::Interpreter* interpreter, ANeuralNetworksModel* nn_model,
+    tflite::Subgraph* subgraph, ANeuralNetworksModel* nn_model,
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
-  for (size_t i = 0; i < interpreter->nodes_size(); i++) {
-    const auto* node_and_registration = interpreter->node_and_registration(i);
+  for (size_t i = 0; i < subgraph->nodes_size(); i++) {
+    const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
     const TfLiteRegistration& registration = node_and_registration->second;
     tflite::BuiltinOperator builtin =
@@ -291,9 +291,9 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [interpreter, &nn_model, &next_id, &augmented_inputs,
-         &model_state_inputs, &model_state_outputs](int tensor_id) {
-          const TfLiteTensor* tensor = interpreter->tensor(tensor_id);
+        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
+         &model_state_outputs](int tensor_id) {
+          const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
@@ -388,11 +388,11 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [interpreter, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
-      const TfLiteTensor* tensor = interpreter->tensor(scratch_buffer_index);
+      const TfLiteTensor* tensor = subgraph->tensor(scratch_buffer_index);
       ANeuralNetworksOperandType operand_type{
           ANEURALNETWORKS_TENSOR_FLOAT32,
           static_cast<uint32_t>(tensor->dims->size),
@@ -584,7 +584,7 @@ TfLiteStatus AddOpsAndParams(
         // The permutation input tensor value dictates the output dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((node.inputs->size > 1) &&
-            (interpreter->tensor(node.inputs->data[1])->allocation_type !=
+            (subgraph->tensor(node.inputs->data[1])->allocation_type !=
              kTfLiteMmapRo)) {
           logError("NNAPI does not yet support dynamic tensors.");
           return kTfLiteError;
@@ -601,14 +601,13 @@ TfLiteStatus AddOpsAndParams(
           return kTfLiteError;
         }
         if ((node.inputs->size > 0) &&
-            (interpreter->tensor(node.inputs->data[0])->dims->size != 4)) {
+            (subgraph->tensor(node.inputs->data[0])->dims->size != 4)) {
           logError("NNAPI only supports input rank 4 for L2Normalization");
           return kTfLiteError;
         }
         break;
       case tflite::BuiltinOperator_HASHTABLE_LOOKUP:
-        if (interpreter->tensor(node.outputs->data[0])->type !=
-            kTfLiteFloat32) {
+        if (subgraph->tensor(node.outputs->data[0])->type != kTfLiteFloat32) {
           logError("NNAPI only support HASHTABLE_LOOKUP with float32 output",
                    builtin);
           return kTfLiteError;
@@ -708,7 +707,7 @@ TfLiteStatus AddOpsAndParams(
   return kTfLiteOk;
 }
 
-TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
+TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
   // TODO(aselle): This is not correct. need to handle resize invalidation.
@@ -720,7 +719,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
     // inputs and outputs and mark the mapping in tensor_id_to_nnapi_id with
     // kOperandIdNotSet. addTensorOperands will replace those with the
     // corresponding NNAPI operand ids and skip kOperandNotNeeded entries.
-    std::vector<int64_t> tensor_id_to_nnapi_id(interpreter->tensors_size(),
+    std::vector<int64_t> tensor_id_to_nnapi_id(subgraph->tensors_size(),
                                                kOperandNotNeeded);
     auto set_ids_to_not_set = [&tensor_id_to_nnapi_id](const int* buf,
                                                        size_t count) {
@@ -731,35 +730,31 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
         }
       }
     };
-    for (size_t i = 0; i < interpreter->nodes_size(); i++) {
-      const auto* node_and_registration = interpreter->node_and_registration(i);
+    for (size_t i = 0; i < subgraph->nodes_size(); i++) {
+      const auto* node_and_registration = subgraph->node_and_registration(i);
       const TfLiteNode& node = node_and_registration->first;
       set_ids_to_not_set(node.inputs->data, node.inputs->size);
       set_ids_to_not_set(node.outputs->data, node.outputs->size);
     }
-    set_ids_to_not_set(interpreter->inputs().data(),
-                       interpreter->inputs().size());
-    set_ids_to_not_set(interpreter->outputs().data(),
-                       interpreter->outputs().size());
+    set_ids_to_not_set(subgraph->inputs().data(), subgraph->inputs().size());
+    set_ids_to_not_set(subgraph->outputs().data(), subgraph->outputs().size());
 
     uint32_t next_id = 0;
     RETURN_ERROR_IF_TFLITE_FAILED(addTensorOperands(
-        interpreter, nn_model_, &next_id, &tensor_id_to_nnapi_id));
+        subgraph, nn_model_, &next_id, &tensor_id_to_nnapi_id));
     RETURN_ERROR_IF_TFLITE_FAILED(
-        AddOpsAndParams(interpreter, nn_model_, next_id, &model_states_inputs_,
+        AddOpsAndParams(subgraph, nn_model_, next_id, &model_states_inputs_,
                         &model_states_outputs_, tensor_id_to_nnapi_id));
 
     std::vector<uint32_t> augmented_inputs;
-    MapAndAddTensorIds(interpreter->inputs().data(),
-                       interpreter->inputs().size(), &augmented_inputs,
-                       tensor_id_to_nnapi_id);
+    MapAndAddTensorIds(subgraph->inputs().data(), subgraph->inputs().size(),
+                       &augmented_inputs, tensor_id_to_nnapi_id);
     augmented_inputs.insert(augmented_inputs.end(),
                             model_states_inputs_.begin(),
                             model_states_inputs_.end());
     std::vector<uint32_t> augmented_outputs;
-    MapAndAddTensorIds(interpreter->outputs().data(),
-                       interpreter->outputs().size(), &augmented_outputs,
-                       tensor_id_to_nnapi_id);
+    MapAndAddTensorIds(subgraph->outputs().data(), subgraph->outputs().size(),
+                       &augmented_outputs, tensor_id_to_nnapi_id);
     MapAndAddTensorIds(model_states_outputs_.data(),
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
@@ -772,7 +767,7 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
 
     if (GetAndroidSdkVersionCached() >= 28) {
       CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-          nn_model_, interpreter->GetAllowFp16PrecisionForFp32()));
+          nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
     CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
   }
@@ -783,9 +778,9 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
   return kTfLiteOk;
 }
 
-TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
+TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   if (!nn_model_) {
-    model_status_ = BuildGraph(interpreter);
+    model_status_ = BuildGraph(subgraph);
     if (model_status_ != kTfLiteOk) {
       logError("Failed to build graph for NNAPI");
     }
@@ -798,19 +793,19 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
   CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
-  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
-    int input = interpreter->inputs()[i];
+  for (size_t i = 0; i < subgraph->inputs().size(); i++) {
+    int input = subgraph->inputs()[i];
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
-    TfLiteTensor* tensor = interpreter->tensor(input);
+    TfLiteTensor* tensor = subgraph->tensor(input);
     CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
   // Tell nn api where to place final data.
-  for (size_t i = 0; i < interpreter->outputs().size(); i++) {
-    int output = interpreter->outputs()[i];
-    TfLiteTensor* tensor = interpreter->tensor(output);
+  for (size_t i = 0; i < subgraph->outputs().size(); i++) {
+    int output = subgraph->outputs()[i];
+    TfLiteTensor* tensor = subgraph->tensor(output);
     CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
@@ -819,16 +814,16 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
   // current invocation.
   for (size_t i = 0; i < model_states_outputs_.size(); i++) {
     int state_tensor_idx = model_states_outputs_[i];
-    TfLiteTensor* tensor = interpreter->tensor(state_tensor_idx);
+    TfLiteTensor* tensor = subgraph->tensor(state_tensor_idx);
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
     CHECK_NN(ANeuralNetworksExecution_setInput(
-        execution, i + interpreter->inputs().size(), nullptr, tensor->data.raw,
+        execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
     CHECK_NN(ANeuralNetworksExecution_setOutput(
-        execution, i + interpreter->outputs().size(), nullptr, tensor->data.raw,
+        execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
@@ -841,9 +836,9 @@ TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
 
 #if 0
   printf("From the NN API:\n");
-  TfLiteTensor* tensor = interpreter->tensor(interpreter->outputs()[0]);
+  TfLiteTensor* tensor = subgraph->tensor(subgraph->outputs()[0]);
   if (float* data =
-          interpreter->typed_tensor<float>(interpreter->outputs()[0])) {
+          subgraph->typed_tensor<float>(subgraph->outputs()[0])) {
     size_t num = tensor->bytes / sizeof(float);
     for (float* p = data; p < data + num; p++) {
       printf(" %f", *p);
diff --git a/tensorflow/lite/nnapi_delegate.h b/tensorflow/lite/nnapi_delegate.h
index 63b408c141..b4f8e4ecf3 100644
--- a/tensorflow/lite/nnapi_delegate.h
+++ b/tensorflow/lite/nnapi_delegate.h
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/subgraph.h"
 #include "tensorflow/lite/interpreter.h"
 
 class ANeuralNetworksModel;
@@ -50,10 +51,10 @@ class NNAPIDelegate {
   ~NNAPIDelegate();
 
   // Convert a tflite graph to NNAPI
-  TfLiteStatus BuildGraph(Interpreter* interpreter);
+  TfLiteStatus BuildGraph(Subgraph* subgraph);
 
   // Run
-  TfLiteStatus Invoke(Interpreter* interpreter);
+  TfLiteStatus Invoke(Subgraph* subgraph);
 
   // Whether the current platform supports NNAPI delegation.
   static bool IsSupported();
diff --git a/tensorflow/lite/nnapi_delegate_disabled.cc b/tensorflow/lite/nnapi_delegate_disabled.cc
index 44dc21f1b6..a8f2c0bfe3 100644
--- a/tensorflow/lite/nnapi_delegate_disabled.cc
+++ b/tensorflow/lite/nnapi_delegate_disabled.cc
@@ -35,13 +35,11 @@ NNAPIDelegate::~NNAPIDelegate() {
 #undef UNUSED_MEMBER
 }
 
-TfLiteStatus NNAPIDelegate::BuildGraph(Interpreter* interpreter) {
+TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   return kTfLiteError;
 }
 
-TfLiteStatus NNAPIDelegate::Invoke(Interpreter* interpreter) {
-  return kTfLiteError;
-}
+TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) { return kTfLiteError; }
 
 bool NNAPIDelegate::IsSupported() { return false; }
 
diff --git a/tensorflow/lite/util.h b/tensorflow/lite/util.h
index 64a5b52e2f..dbb87528d0 100644
--- a/tensorflow/lite/util.h
+++ b/tensorflow/lite/util.h
@@ -52,6 +52,12 @@ bool EqualArrayAndTfLiteIntArray(const TfLiteIntArray* a, const int b_size,
 
 size_t CombineHashes(std::initializer_list<size_t> hashes);
 
+struct TfLiteIntArrayDeleter {
+  void operator()(TfLiteIntArray* a) {
+    if (a) TfLiteIntArrayFree(a);
+  }
+};
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_LITE_UTIL_H_
-- 
GitLab


From 0864b3eb3e3fa56c42501656703dec52e46ab8e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 01:02:12 -0800
Subject: [PATCH 0562/1554] compat: Update forward compatibility horizon to
 2018-11-20

PiperOrigin-RevId: 222204359
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 43a5eb4e7f..1b8114dd26 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 19)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 20)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 30dcdb99aaf7139ea09bb049f15192ec72ebf47b Mon Sep 17 00:00:00 2001
From: Ilya Biryukov <ibiryukov@google.com>
Date: Tue, 20 Nov 2018 01:43:24 -0800
Subject: [PATCH 0563/1554] Update downloadable clang to r346388, subrevision 3

PiperOrigin-RevId: 222210141
---
 third_party/clang_toolchain/download_clang.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/third_party/clang_toolchain/download_clang.bzl b/third_party/clang_toolchain/download_clang.bzl
index 9023e250b2..a941ee1c99 100644
--- a/third_party/clang_toolchain/download_clang.bzl
+++ b/third_party/clang_toolchain/download_clang.bzl
@@ -40,14 +40,14 @@ def download_clang(repo_ctx, out_folder):
     # Latest CLANG_REVISION and CLANG_SUB_REVISION of the Chromiums's release
     # can be found in https://chromium.googlesource.com/chromium/src/tools/clang/+/master/scripts/update.py
     CLANG_REVISION = "346388"
-    CLANG_SUB_REVISION = 1
+    CLANG_SUB_REVISION = 3
 
     package_version = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
 
     checksums = {
-        "Linux_x64": "5e5564e4e743414c7eaec9fd9e739732ddd2a343e49bde4c88fc2530b1c598b9",
-        "Mac": "19271a7cc5c2bcaf9643d3dd622b5458569dc662bbc58f63b129cf6e3a4e3243",
-        "Win": "60b0bd1f11e53892109f4159e2aba0f803604823e07875ca98b82bd5628d7f4d",
+        "Linux_x64": "d47b7ac4756c3f8e3bbfa0e81bf199ec8e9faa3a6b11573f0705e9c04af7ad51",
+        "Mac": "de2b0c701e19cda633ea02804866dd24d8506afb8cae51fbcce3415b76f4ded3",
+        "Win": "c7d27f13b41aa9eaaf9760903962e9b2b0f8261058df0d35170711dc60545a7d",
     }
 
     platform_folder = _get_platform_folder(repo_ctx.os.name)
-- 
GitLab


From 07f6ed9896bcf6f193f794a6c22398c3a6bcada6 Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Tue, 20 Nov 2018 18:48:51 +0800
Subject: [PATCH 0564/1554] Fix comment for AddGradients in graph.go

---
 tensorflow/go/graph.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index fb191fb51e..84e81c7755 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -147,7 +147,7 @@ func (g *Graph) Operations() []Operation {
 	return ops
 }
 
-// AddGradients add operations to compute the partial derivatives of the sum of tensors in y
+// AddGradients adds operations to compute the partial derivatives of the sum of tensors in y
 // with respect to tensors in x, i.e., d(y[0] + y[1] + ...) / d x[0], d(y[0] + y[1] + ... ) / d x[1] etc.
 //
 // prefix, if non-empty, is the name prefix used for all operations added to the graph to compute
-- 
GitLab


From 8b118ec4bee5f76b6672ddb79fcb095d75705e22 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Tue, 20 Nov 2018 02:57:03 -0800
Subject: [PATCH 0565/1554] Improve the sort implementation for the CPU
 backend.

We can use the same trick as on the other backends to cast the floating types
to integer types and then comparing these. This is faster.
Also use stable_sort instead of sort to simplify the custom comparators.

PiperOrigin-RevId: 222217591
---
 .../xla/service/cpu/runtime_key_value_sort.cc | 80 +++++++++----------
 1 file changed, 39 insertions(+), 41 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
index 669eeb95f3..c7fc101ec7 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include <algorithm>
 #include <cmath>
 #include <cstring>
+#include <limits>
 #include <memory>
 #include <string>
 #include <utility>
@@ -41,61 +42,58 @@ void KeyValueSort(std::pair<KeyType, int64>* row_to_sort, int64 num_elements) {
   std::sort(row_to_sort, row_to_sort + num_elements);
 }
 
-// For floating point numbers, we want a total order comparator. -NaN and NaN
-// should appear at the beginning and end of the ordering, and -0.0 should
-// appear before 0.0. Also we want to have a stable sort, so if the keys are the
-// same, we compare the index values.
-template <typename KeyType>
-bool LessThan(KeyType lhs, int64 lhs_index, KeyType rhs, int64 rhs_index) {
-  bool lhs_is_negative = std::signbit(lhs);
-  bool rhs_is_negative = std::signbit(rhs);
-  // If the signs are different, we can just compare the signs.
-  if (lhs_is_negative != rhs_is_negative) {
-    return lhs_is_negative && !rhs_is_negative;
-  }
-  bool lhs_nan = std::isnan(lhs);
-  bool rhs_nan = std::isnan(rhs);
-  // Exactly one number is nan?
-  if (lhs_nan != rhs_nan) {
-    if (lhs_nan) {
-      return lhs_is_negative;
-    }
-    return !rhs_is_negative;
+// We would like a total order of floating point numbers so that the
+// sort has a predictable behavior in the presence of NaNs. Rather
+// than using floating point comparison, we use the following trick:
+// If f is a float, and
+// x = bit_cast<int32>(f);
+// y = x < 0 ? 0x7FFFFFFF - x : x;
+// then y is ordered as an int32 such that finite values have the
+// obvious order, -0 is ordered before 0, and -NaN and NaN appear at
+// the beginning and end of the ordering.
+template <typename CastType, typename KeyType>
+CastType Convert(KeyType value) {
+  CastType casted_value;
+  memcpy(&casted_value, &value, sizeof(CastType));
+  if (casted_value < 0) {
+    return std::numeric_limits<CastType>::max() - casted_value;
   }
-  if (lhs != rhs) {
-    return lhs < rhs;
-  }
-  return lhs_index < rhs_index;
+  return casted_value;
+}
+
+template <typename CastType, typename KeyType>
+bool LessThan(KeyType lhs, KeyType rhs) {
+  return Convert<CastType>(lhs) < Convert<CastType>(rhs);
 }
 
 template <>
 void KeyValueSort(std::pair<double, int64>* row_to_sort, int64 num_elements) {
-  std::sort(row_to_sort, row_to_sort + num_elements,
-            [](const std::pair<double, int64>& lhs,
-               const std::pair<double, int64>& rhs) -> bool {
-              return LessThan(lhs.first, lhs.second, rhs.first, rhs.second);
-            });
+  std::stable_sort(row_to_sort, row_to_sort + num_elements,
+                   [](const std::pair<double, int64>& lhs,
+                      const std::pair<double, int64>& rhs) -> bool {
+                     return LessThan<int64>(lhs.first, rhs.first);
+                   });
 }
 
 template <>
 void KeyValueSort(std::pair<float, int64>* row_to_sort, int64 num_elements) {
-  std::sort(row_to_sort, row_to_sort + num_elements,
-            [](const std::pair<float, int64>& lhs,
-               const std::pair<float, int64>& rhs) -> bool {
-              return LessThan(lhs.first, lhs.second, rhs.first, rhs.second);
-            });
+  std::stable_sort(row_to_sort, row_to_sort + num_elements,
+                   [](const std::pair<float, int64>& lhs,
+                      const std::pair<float, int64>& rhs) -> bool {
+                     return LessThan<int32>(lhs.first, rhs.first);
+                   });
 }
 
 template <>
 void KeyValueSort(std::pair<Eigen::half, int64>* row_to_sort,
                   int64 num_elements) {
-  std::sort(row_to_sort, row_to_sort + num_elements,
-            [](const std::pair<Eigen::half, int64>& lhs,
-               const std::pair<Eigen::half, int64>& rhs) -> bool {
-              return LessThan(
-                  Eigen::half_impl::half_to_float(lhs.first), lhs.second,
-                  Eigen::half_impl::half_to_float(rhs.first), rhs.second);
-            });
+  std::stable_sort(row_to_sort, row_to_sort + num_elements,
+                   [](const std::pair<Eigen::half, int64>& lhs,
+                      const std::pair<Eigen::half, int64>& rhs) -> bool {
+                     return LessThan<int32>(
+                         Eigen::half_impl::half_to_float(lhs.first),
+                         Eigen::half_impl::half_to_float(rhs.first));
+                   });
 }
 
 template <typename KeyType>
-- 
GitLab


From 4a51de309e95173c78558873542fd2dcdd1889dd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 06:11:31 -0800
Subject: [PATCH 0566/1554] Optimize XlaDevice::Sync by allowing it to happen
 asynchronously.

PiperOrigin-RevId: 222234647
---
 tensorflow/compiler/jit/xla_device.cc      | 25 ++++++++++++++++++++++
 tensorflow/compiler/jit/xla_device.h       |  1 +
 tensorflow/core/common_runtime/device.cc   |  2 ++
 tensorflow/core/common_runtime/device.h    | 10 +++++++++
 tensorflow/core/common_runtime/executor.cc | 17 +++++++++------
 5 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 738bac54cc..4201ff91a8 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -410,6 +410,31 @@ Status XlaDevice::Sync() {
   return Status::OK();
 }
 
+void XlaDevice::Sync(const DoneCallback& done) {
+  VLOG(1) << "XlaDevice::Sync (asynchronous)";
+  std::shared_ptr<se::Stream> stream;
+  {
+    mutex_lock lock(mu_);
+    stream = stream_;
+  }
+  if (!stream) {
+    done(Status::OK());
+    return;
+  }
+
+  stream->ThenEnqueueOnBackgroundThread(
+      [this, stream, done](se::StreamExecutor*) {
+        tracing::ScopedActivity activity("XlaDevice::Sync::Callback",
+                                         /*is_expensive=*/true);
+        mutex_lock lock(mu_);
+        while (outstanding_asynchronous_operations_ > 0) {
+          outstanding_asynchronous_operations_cv_.wait(lock);
+        }
+        done(stream->ok() ? Status::OK()
+                          : errors::Internal("XlaDevice::Sync() failed."));
+      });
+}
+
 Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto,
                                       const AllocatorAttributes alloc_attrs,
                                       Tensor* tensor) {
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index dc8f49a9c9..c8bb276cdb 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -135,6 +135,7 @@ class XlaDevice : public LocalDevice {
   void ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context,
                     AsyncOpKernel::DoneCallback done) override;
   Status Sync() override;
+  void Sync(const DoneCallback& done) override;
 
   Status FillContextMap(const Graph* graph,
                         DeviceContextMap* device_context_map) override
diff --git a/tensorflow/core/common_runtime/device.cc b/tensorflow/core/common_runtime/device.cc
index 8fc64fff69..9925814a48 100644
--- a/tensorflow/core/common_runtime/device.cc
+++ b/tensorflow/core/common_runtime/device.cc
@@ -36,6 +36,8 @@ Device::~Device() {
   }
 }
 
+void Device::Sync(const DoneCallback& done) { done(Sync()); }
+
 // static
 DeviceAttributes Device::BuildDeviceAttributes(
     const string& name, DeviceType device, Bytes memory_limit,
diff --git a/tensorflow/core/common_runtime/device.h b/tensorflow/core/common_runtime/device.h
index 2ef1547cd9..8dfbb21eda 100644
--- a/tensorflow/core/common_runtime/device.h
+++ b/tensorflow/core/common_runtime/device.h
@@ -55,6 +55,9 @@ class DeviceMgr;
 
 class Device : public DeviceBase {
  public:
+  // Callback type that takes a Status and returns void.
+  typedef std::function<void(const Status&)> DoneCallback;
+
   Device(Env* env, const DeviceAttributes& device_attributes);
   ~Device() override;
 
@@ -112,6 +115,13 @@ class Device : public DeviceBase {
   // at completion.
   virtual Status Sync() = 0;
 
+  // Calls the given callback when all operations queued on the device at the
+  // time of the call have completed. The callback is passed any error pending
+  // on the device at completion.
+  // TODO(b/112409994): Consolidate these two APIs, removing the synchronous
+  // version.
+  virtual void Sync(const DoneCallback& done);
+
   // Override this to return true for devices that require a Sync() call before
   // session completion.
   virtual bool RequiresSyncOnCompletion() const { return false; }
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index b9115f86ff..77b249c2b4 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1713,7 +1713,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
         auto done = [this, state]() {
           Device* device = impl_->params_.device;
           NodeExecStatsInterface* stats = state->stats;  // Shorthand
-          Entry* first_input = state->first_input;     // Shorthand
+          Entry* first_input = state->first_input;       // Shorthand
 
           nodestats::SetOpEnd(stats);
           EntryVector outputs;
@@ -2394,18 +2394,23 @@ void ExecutorState::Finish() {
   auto done_cb = std::move(done_cb_);
   auto runner = std::move(runner_);
   mu_.unlock();
+  CHECK(done_cb != nullptr);
   Device* device = impl_->params_.device;
+
   if ((sync_on_finish_ && status.ok()) || device->RequiresSyncOnCompletion()) {
     // Block until the device has finished all queued operations. For
     // devices like GPUs that continue to execute Ops after their Compute
     // methods have completed, this ensures that control is not returned to
     // the user until the step (and its side-effects) has actually completed.
-    status.Update(device->Sync());
+    device->Sync([=](Status new_status) mutable {
+      status.Update(new_status);
+      delete this;
+      runner([=]() { done_cb(status); });
+    });
+  } else {
+    delete this;
+    runner([=]() { done_cb(status); });
   }
-
-  delete this;
-  CHECK(done_cb != nullptr);
-  runner([=]() { done_cb(status); });
 }
 
 void ExecutorState::FindOrCreateChildFrame(FrameState* frame, int64 iter,
-- 
GitLab


From cdd3a982536242b123559c6ae785c2894888184a Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Tue, 20 Nov 2018 07:10:28 -0800
Subject: [PATCH 0567/1554] Handle deeply nested concatenates with same
 operands.

Currently we would generate an exponential number of basic blocks.
With this CL, we generate each unique operand only once, and use a PHI node for
the source index input.

PiperOrigin-RevId: 222240704
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../xla/service/elemental_ir_emitter.cc       | 65 +++++++++++++++----
 tensorflow/compiler/xla/tests/concat_test.cc  | 26 ++++++++
 3 files changed, 80 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index d5e73fe1a8..1bd04d2785 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2984,6 +2984,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/strings",
         "@llvm//:core",
         "@llvm//:transform_utils",
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index f98c943669..00bb430206 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -22,6 +22,7 @@ limitations under the License.
 
 // IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc"
 #include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/str_cat.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Instructions.h"
@@ -1671,26 +1672,66 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitElementalConcatenate(
 
   b_->SetInsertPoint(init_block);
 
+  // Assign a unique id for each *different* operand, and count how often each
+  // operand is used. If all operands are different, the usage count will be 1
+  // for each operand.
+  absl::flat_hash_map<const HloInstruction*, int64> to_unique_operand_id;
+  std::vector<int64> operand_usage_count;
+  for (const auto* operand : hlo->operands()) {
+    if (to_unique_operand_id.contains(operand)) {
+      ++operand_usage_count[to_unique_operand_id[operand]];
+    } else {
+      int64 unique_operand_id = to_unique_operand_id.size();
+      to_unique_operand_id[operand] = unique_operand_id;
+      operand_usage_count.push_back(1);
+    }
+  }
+
+  // To avoid that we emit the same operand more than once, we create one basic
+  // block for each *different* operand with a PHI node for the different source
+  // index inputs.
+  std::vector<llvm::BasicBlock*> emit_operand_blocks(
+      to_unique_operand_id.size(), nullptr);
+  std::vector<llvm::PHINode*> source_index_phis(to_unique_operand_id.size(),
+                                                nullptr);
+  for (const auto* operand : hlo->operands()) {
+    int64 operand_id = to_unique_operand_id[operand];
+    if (emit_operand_blocks[operand_id] != nullptr) {
+      continue;
+    }
+
+    emit_operand_blocks[operand_id] = llvm_ir::CreateBasicBlock(
+        exit_block, StrCat("concat_index_from_operand_id", operand_id), b_);
+    auto saved_insert_point = b_->GetInsertPoint();
+    llvm_ir::SetToFirstInsertPoint(emit_operand_blocks[operand_id], b_);
+    source_index_phis[operand_id] =
+        PHI(source_index.GetType(), operand_usage_count[operand_id]);
+    auto operand_index = source_index;
+    operand_index[concat_dim] = source_index_phis[operand_id];
+
+    // Create the terminator of the block before calling operand generators,
+    // because they require non-degenerate basic blocks.
+    b_->SetInsertPoint(llvm::BranchInst::Create(
+        exit_block, /*InsertAtEnd=*/emit_operand_blocks[operand_id]));
+    TF_ASSIGN_OR_RETURN(llvm::Value * value,
+                        operand_to_generator.at(operand)(operand_index));
+    output->addIncoming(value, b_->GetInsertBlock());
+    b_->SetInsertPoint(init_block, saved_insert_point);
+  }
+
   for (int64 operand_idx = 0; operand_idx < hlo->operand_count();
        ++operand_idx) {
     const HloInstruction* operand = hlo->operand(operand_idx);
-    auto true_block = llvm_ir::CreateBasicBlock(
-        exit_block, StrCat("concat_index_from_operand", operand_idx), b_);
     auto false_block = llvm_ir::CreateBasicBlock(
         exit_block, StrCat("concat_index_not_from_operand", operand_idx), b_);
     auto concat_dim_size =
         llvm::ConstantInt::get(source_index[concat_dim]->getType(),
                                operand->shape().dimensions(concat_dim));
-    CondBr(ICmpULT(source_index[concat_dim], concat_dim_size), true_block,
-           false_block);
-
-    // Create the terminator of the true block before calling operand
-    // generators, because they require non-degenerate basic blocks.
-    b_->SetInsertPoint(
-        llvm::BranchInst::Create(exit_block, /*InsertAtEnd=*/true_block));
-    TF_ASSIGN_OR_RETURN(llvm::Value * value,
-                        operand_to_generator.at(operand)(source_index));
-    output->addIncoming(value, b_->GetInsertBlock());
+    int64 operand_id = to_unique_operand_id[operand];
+    source_index_phis[operand_id]->addIncoming(source_index[concat_dim],
+                                               b_->GetInsertBlock());
+    CondBr(ICmpULT(source_index[concat_dim], concat_dim_size),
+           emit_operand_blocks[operand_id], false_block);
 
     // Subtract the size of the concat dimension of the current operand
     // from the source index.
diff --git a/tensorflow/compiler/xla/tests/concat_test.cc b/tensorflow/compiler/xla/tests/concat_test.cc
index 9811a015e9..4f5b525a34 100644
--- a/tensorflow/compiler/xla/tests/concat_test.cc
+++ b/tensorflow/compiler/xla/tests/concat_test.cc
@@ -492,6 +492,32 @@ XLA_TEST_F(ConcatTest, ConcatR3WeirdDims) {
   ComputeAndCompareR3<float>(&builder, expected, {p0.get(), p1.get()});
 }
 
+XLA_TEST_F(ConcatTest, ConcatDeeplyNested) {
+  XlaBuilder builder(TestName());
+  auto a_literal = LiteralUtil::CreateR1<float>({256.0});
+  auto a = Parameter(&builder, 0, a_literal.shape(), "x");
+  auto b = ConcatInDim(&builder, {a, a}, 0);
+  auto c = ConcatInDim(&builder, {b, b}, 0);
+  auto d = ConcatInDim(&builder, {c, c}, 0);
+  auto e = ConcatInDim(&builder, {d, d}, 0);
+  auto f = ConcatInDim(&builder, {e, e}, 0);
+  auto g = ConcatInDim(&builder, {f, f}, 0);
+  auto h = ConcatInDim(&builder, {g, g}, 0);
+  auto i = ConcatInDim(&builder, {h, h}, 0);
+  auto j = ConcatInDim(&builder, {i, i}, 0);
+  auto k = ConcatInDim(&builder, {j, j}, 0);
+  auto l = ConcatInDim(&builder, {k, k}, 0);
+  auto m = ConcatInDim(&builder, {l, l}, 0);
+  auto n = ConcatInDim(&builder, {m, m}, 0);
+  auto o = ConcatInDim(&builder, {n, n}, 0);
+  auto p = ConcatInDim(&builder, {o, o}, 0);
+  auto q = ConcatInDim(&builder, {p, p}, 0);
+  ConcatInDim(&builder, {q, q}, 0);
+  std::vector<float> expected(131072, 256.0);
+  auto a_data = client_->TransferToServer(a_literal).ConsumeValueOrDie();
+  ComputeAndCompareR1<float>(&builder, expected, {a_data.get()});
+}
+
 // Describes a binary rank-2 concatenation test.
 struct R2BinarySpec {
   int64 lhs_dim0;
-- 
GitLab


From f25493f9c0ae0dec6db31dfe7fd19ed01b57c23d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 07:44:36 -0800
Subject: [PATCH 0568/1554] Re-create the gcc-nvcc remote configuration.

Given that the nvcc path is hard-coded in the config, move it from a
cuda-agnostic location to one that includes the cuda version.

PiperOrigin-RevId: 222244416
---
 .../ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD     |   10 +-
 .../cuda9.0-cudnn7/cuda/build_defs.bzl        |    6 +-
 .../ubuntu14.04/gcc-nvcc-cuda9.0/BUILD        |   87 +
 .../ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL    | 1431 +++++++++++++++++
 .../bin/crosstool_wrapper_driver_is_not_gcc   |  264 +++
 .../windows/msvc_wrapper_for_nvcc.bat         |   20 +
 .../windows/msvc_wrapper_for_nvcc.py          |  192 +++
 .../preconfig/ubuntu14.04/py3/BUILD           |    4 +-
 8 files changed, 2003 insertions(+), 11 deletions(-)
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.bat
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py

diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
index 247e0ace24..c6930904b5 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
@@ -1188,7 +1188,7 @@ genrule(
         "cuda/include/vector_types.h",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp "/usr/local/cuda-9.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp "/usr/local/cuda-9.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp "/usr/local/cuda-9.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp "/usr/local/cuda-9.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp "/usr/local/cuda-9.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp "/usr/local/cuda-9.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp "/usr/local/cuda-9.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp "/usr/local/cuda-9.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp "/usr/local/cuda-9.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp "/usr/local/cuda-9.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp "/usr/local/cuda-9.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp "/usr/local/cuda-9.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp "/usr/local/cuda-9.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp "/usr/local/cuda-9.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp "/usr/local/cuda-9.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp "/usr/local/cuda-9.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp "/usr/local/cuda-9.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp "/usr/local/cuda-9.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp "/usr/local/cuda-9.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp "/usr/local/cuda-9.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp "/usr/local/cuda-9.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp "/usr/local/cuda-9.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp "/usr/local/cuda-9.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp "/usr/local/cuda-9.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp "/usr/local/cuda-9.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp "/usr/local/cuda-9.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp "/usr/local/cuda-9.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp "/usr/local/cuda-9.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp "/usr/local/cuda-9.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp "/usr/local/cuda-9.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp "/usr/local/cuda-9.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp "/usr/local/cuda-9.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp "/usr/local/cuda-9.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp "/usr/local/cuda-9.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp "/usr/local/cuda-9.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp "/usr/local/cuda-9.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp "/usr/local/cuda-9.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp "/usr/local/cuda-9.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp "/usr/local/cuda-9.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp "/usr/local/cuda-9.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp "/usr/local/cuda-9.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp "/usr/local/cuda-9.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp "/usr/local/cuda-9.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp "/usr/local/cuda-9.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp "/usr/local/cuda-9.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp "/usr/local/cuda-9.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp "/usr/local/cuda-9.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp "/usr/local/cuda-9.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp "/usr/local/cuda-9.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp "/usr/local/cuda-9.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp "/usr/local/cuda-9.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp "/usr/local/cuda-9.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp "/usr/local/cuda-9.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp "/usr/local/cuda-9.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp "/usr/local/cuda-9.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp "/usr/local/cuda-9.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp "/usr/local/cuda-9.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp "/usr/local/cuda-9.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp "/usr/local/cuda-9.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp "/usr/local/cuda-9.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp "/usr/local/cuda-9.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp "/usr/local/cuda-9.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp "/usr/local/cuda-9.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp "/usr/local/cuda-9.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp "/usr/local/cuda-9.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp "/usr/local/cuda-9.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp "/usr/local/cuda-9.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp "/usr/local/cuda-9.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp "/usr/local/cuda-9.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp "/usr/local/cuda-9.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp "/usr/local/cuda-9.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp "/usr/local/cuda-9.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp "/usr/local/cuda-9.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp "/usr/local/cuda-9.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp "/usr/local/cuda-9.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp "/usr/local/cuda-9.0/include/device_double_functions.hpp" "$(@D)/cuda/include/device_double_functions.hpp" && cp "/usr/local/cuda-9.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp "/usr/local/cuda-9.0/include/device_functions.hpp" "$(@D)/cuda/include/device_functions.hpp" && cp "/usr/local/cuda-9.0/include/device_functions_decls.h" "$(@D)/cuda/include/device_functions_decls.h" && cp "/usr/local/cuda-9.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp "/usr/local/cuda-9.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp "/usr/local/cuda-9.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp "/usr/local/cuda-9.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp "/usr/local/cuda-9.0/include/dynlink_cuda.h" "$(@D)/cuda/include/dynlink_cuda.h" && cp "/usr/local/cuda-9.0/include/dynlink_cuda_cuda.h" "$(@D)/cuda/include/dynlink_cuda_cuda.h" && cp "/usr/local/cuda-9.0/include/dynlink_cuviddec.h" "$(@D)/cuda/include/dynlink_cuviddec.h" && cp "/usr/local/cuda-9.0/include/dynlink_nvcuvid.h" "$(@D)/cuda/include/dynlink_nvcuvid.h" && cp "/usr/local/cuda-9.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp "/usr/local/cuda-9.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp "/usr/local/cuda-9.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp "/usr/local/cuda-9.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp "/usr/local/cuda-9.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp "/usr/local/cuda-9.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp "/usr/local/cuda-9.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp "/usr/local/cuda-9.0/include/math_functions.hpp" "$(@D)/cuda/include/math_functions.hpp" && cp "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.h" "$(@D)/cuda/include/math_functions_dbl_ptx3.h" && cp "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.hpp" "$(@D)/cuda/include/math_functions_dbl_ptx3.hpp" && cp "/usr/local/cuda-9.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp "/usr/local/cuda-9.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp "/usr/local/cuda-9.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp "/usr/local/cuda-9.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp "/usr/local/cuda-9.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp "/usr/local/cuda-9.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp "/usr/local/cuda-9.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp "/usr/local/cuda-9.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp "/usr/local/cuda-9.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp "/usr/local/cuda-9.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp "/usr/local/cuda-9.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp "/usr/local/cuda-9.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp "/usr/local/cuda-9.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp "/usr/local/cuda-9.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp "/usr/local/cuda-9.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp "/usr/local/cuda-9.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp "/usr/local/cuda-9.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp "/usr/local/cuda-9.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp "/usr/local/cuda-9.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp "/usr/local/cuda-9.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp "/usr/local/cuda-9.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp "/usr/local/cuda-9.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp "/usr/local/cuda-9.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp "/usr/local/cuda-9.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp "/usr/local/cuda-9.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp "/usr/local/cuda-9.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp "/usr/local/cuda-9.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp "/usr/local/cuda-9.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp "/usr/local/cuda-9.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp "/usr/local/cuda-9.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp "/usr/local/cuda-9.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp "/usr/local/cuda-9.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp "/usr/local/cuda-9.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp "/usr/local/cuda-9.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp "/usr/local/cuda-9.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp "/usr/local/cuda-9.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp "/usr/local/cuda-9.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp "/usr/local/cuda-9.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp "/usr/local/cuda-9.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp "/usr/local/cuda-9.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp "/usr/local/cuda-9.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cpowf.h" "$(@D)/cuda/include/thrust/detail/complex/cpowf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp "/usr/local/cuda-9.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp "/usr/local/cuda-9.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp "/usr/local/cuda-9.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp "/usr/local/cuda-9.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp "/usr/local/cuda-9.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp "/usr/local/cuda-9.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp "/usr/local/cuda-9.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp "/usr/local/cuda-9.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp "/usr/local/cuda-9.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp "/usr/local/cuda-9.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp "/usr/local/cuda-9.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp "/usr/local/cuda-9.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp "/usr/local/cuda-9.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp "/usr/local/cuda-9.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp "/usr/local/cuda-9.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp "/usr/local/cuda-9.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp "/usr/local/cuda-9.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/memory_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp "/usr/local/cuda-9.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp "/usr/local/cuda-9.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp "/usr/local/cuda-9.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp "/usr/local/cuda-9.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp "/usr/local/cuda-9.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp -f "/usr/local/cuda-9.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp -f "/usr/local/cuda-9.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp -f "/usr/local/cuda-9.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp -f "/usr/local/cuda-9.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp -f "/usr/local/cuda-9.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp -f "/usr/local/cuda-9.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp -f "/usr/local/cuda-9.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp -f "/usr/local/cuda-9.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp -f "/usr/local/cuda-9.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp -f "/usr/local/cuda-9.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp -f "/usr/local/cuda-9.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp -f "/usr/local/cuda-9.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp -f "/usr/local/cuda-9.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp -f "/usr/local/cuda-9.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp -f "/usr/local/cuda-9.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp -f "/usr/local/cuda-9.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp -f "/usr/local/cuda-9.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp -f "/usr/local/cuda-9.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp -f "/usr/local/cuda-9.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp -f "/usr/local/cuda-9.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp -f "/usr/local/cuda-9.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp -f "/usr/local/cuda-9.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp -f "/usr/local/cuda-9.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp -f "/usr/local/cuda-9.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp -f "/usr/local/cuda-9.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp -f "/usr/local/cuda-9.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp -f "/usr/local/cuda-9.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp -f "/usr/local/cuda-9.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp -f "/usr/local/cuda-9.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp -f "/usr/local/cuda-9.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp -f "/usr/local/cuda-9.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp -f "/usr/local/cuda-9.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp -f "/usr/local/cuda-9.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp -f "/usr/local/cuda-9.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp -f "/usr/local/cuda-9.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp -f "/usr/local/cuda-9.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp -f "/usr/local/cuda-9.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp -f "/usr/local/cuda-9.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp -f "/usr/local/cuda-9.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp -f "/usr/local/cuda-9.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp -f "/usr/local/cuda-9.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp -f "/usr/local/cuda-9.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp -f "/usr/local/cuda-9.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp -f "/usr/local/cuda-9.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp -f "/usr/local/cuda-9.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp -f "/usr/local/cuda-9.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp -f "/usr/local/cuda-9.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp -f "/usr/local/cuda-9.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp -f "/usr/local/cuda-9.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp -f "/usr/local/cuda-9.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp -f "/usr/local/cuda-9.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp -f "/usr/local/cuda-9.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp -f "/usr/local/cuda-9.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp -f "/usr/local/cuda-9.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp -f "/usr/local/cuda-9.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_double_functions.hpp" "$(@D)/cuda/include/device_double_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_functions.hpp" "$(@D)/cuda/include/device_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_functions_decls.h" "$(@D)/cuda/include/device_functions_decls.h" && cp -f "/usr/local/cuda-9.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp -f "/usr/local/cuda-9.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp -f "/usr/local/cuda-9.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp -f "/usr/local/cuda-9.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuda.h" "$(@D)/cuda/include/dynlink_cuda.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuda_cuda.h" "$(@D)/cuda/include/dynlink_cuda_cuda.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuviddec.h" "$(@D)/cuda/include/dynlink_cuviddec.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_nvcuvid.h" "$(@D)/cuda/include/dynlink_nvcuvid.h" && cp -f "/usr/local/cuda-9.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp -f "/usr/local/cuda-9.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp -f "/usr/local/cuda-9.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp -f "/usr/local/cuda-9.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp -f "/usr/local/cuda-9.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp -f "/usr/local/cuda-9.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp -f "/usr/local/cuda-9.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp -f "/usr/local/cuda-9.0/include/math_functions.hpp" "$(@D)/cuda/include/math_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.h" "$(@D)/cuda/include/math_functions_dbl_ptx3.h" && cp -f "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.hpp" "$(@D)/cuda/include/math_functions_dbl_ptx3.hpp" && cp -f "/usr/local/cuda-9.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp -f "/usr/local/cuda-9.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp -f "/usr/local/cuda-9.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp -f "/usr/local/cuda-9.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp -f "/usr/local/cuda-9.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp -f "/usr/local/cuda-9.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-9.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp -f "/usr/local/cuda-9.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp -f "/usr/local/cuda-9.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp -f "/usr/local/cuda-9.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp -f "/usr/local/cuda-9.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp -f "/usr/local/cuda-9.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp -f "/usr/local/cuda-9.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp -f "/usr/local/cuda-9.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp -f "/usr/local/cuda-9.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-9.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp -f "/usr/local/cuda-9.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp -f "/usr/local/cuda-9.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp -f "/usr/local/cuda-9.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp -f "/usr/local/cuda-9.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp -f "/usr/local/cuda-9.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp -f "/usr/local/cuda-9.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp -f "/usr/local/cuda-9.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp -f "/usr/local/cuda-9.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp -f "/usr/local/cuda-9.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp -f "/usr/local/cuda-9.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp -f "/usr/local/cuda-9.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp -f "/usr/local/cuda-9.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp -f "/usr/local/cuda-9.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp -f "/usr/local/cuda-9.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cpowf.h" "$(@D)/cuda/include/thrust/detail/complex/cpowf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp -f "/usr/local/cuda-9.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp -f "/usr/local/cuda-9.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/memory_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp -f "/usr/local/cuda-9.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp -f "/usr/local/cuda-9.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp -f "/usr/local/cuda-9.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
    """,
 )
 
@@ -1198,7 +1198,7 @@ genrule(
         "cuda/nvvm/libdevice/libdevice.10.bc",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
    """,
 )
 
@@ -1235,7 +1235,7 @@ genrule(
         "cuda/extras/CUPTI/include/openacc/cupti_openacc.h",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h"
    """,
 )
 
@@ -1253,7 +1253,7 @@ genrule(
         "cuda/lib/libcupti.so.9.0",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0.176" "$(@D)/cuda/lib/libcudart.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0.480" "$(@D)/cuda/lib/libcublas.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0.176" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0.176" "$(@D)/cuda/lib/libcurand.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0.176" "$(@D)/cuda/lib/libcufft.so.9.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.1.4" "$(@D)/cuda/lib/libcudnn.so.7" && cp "/usr/local/cuda/extras/CUPTI/lib64/libcupti.so.9.0.176" "$(@D)/cuda/lib/libcupti.so.9.0"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0.176" "$(@D)/cuda/lib/libcudart.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0.480" "$(@D)/cuda/lib/libcublas.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0.176" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0.176" "$(@D)/cuda/lib/libcurand.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0.176" "$(@D)/cuda/lib/libcufft.so.9.0" && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.1.4" "$(@D)/cuda/lib/libcudnn.so.7" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0.176" "$(@D)/cuda/lib/libcupti.so.9.0"
    """,
 )
 
@@ -1263,6 +1263,6 @@ genrule(
         "cuda/include/cudnn.h",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/include/cudnn.h" "$(@D)/cudnn.h"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/cudnn.h" "$(@D)/cudnn.h"
    """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
index 5c6703aab4..a53c891d8b 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
@@ -9,15 +9,13 @@ def if_cuda(if_true, if_false = []):
     return select({
         "@local_config_cuda//cuda:using_nvcc": if_true,
         "@local_config_cuda//cuda:using_clang": if_true,
-        "//conditions:default": if_false
+        "//conditions:default": if_false,
     })
 
-
 def cuda_default_copts():
     """Default options for all CUDA compilations."""
     return if_cuda(["-x", "cuda", "-DGOOGLE_CUDA=1"] + [])
 
-
 def cuda_is_configured():
     """Returns true if CUDA was enabled during the configure process."""
     return True
@@ -29,5 +27,5 @@ def if_cuda_is_configured(x):
     --config=cuda. Used to allow non-CUDA code to depend on CUDA libraries.
     """
     if cuda_is_configured():
-      return x
+        return x
     return []
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
new file mode 100755
index 0000000000..6442e7628a
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
@@ -0,0 +1,87 @@
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain(
+    name = "toolchain-linux-x86_64",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    toolchain = ":cc-compiler-local",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "local|compiler": ":cc-compiler-local",
+        "darwin|compiler": ":cc-compiler-darwin",
+        "x64_windows|msvc-cl": ":cc-compiler-windows",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-local",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    cpu = "local",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    # To support linker flags that need to go to the start of command line
+    # we need the toolchain to support parameter files. Parameter files are
+    # last on the command line and contain all shared libraries to link, so all
+    # regular options will be left of them.
+    supports_param_files = 1,
+)
+
+cc_toolchain(
+    name = "cc-compiler-darwin",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    cpu = "darwin",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 0,
+)
+
+cc_toolchain(
+    name = "cc-compiler-windows",
+    all_files = ":windows_msvc_wrapper_files",
+    compiler_files = ":empty",
+    cpu = "x64_windows",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":windows_msvc_wrapper_files",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "crosstool_wrapper_driver_is_not_gcc",
+    srcs = ["clang/bin/crosstool_wrapper_driver_is_not_gcc"],
+)
+
+filegroup(
+    name = "windows_msvc_wrapper_files",
+    srcs = glob(["windows/msvc_*"]),
+)
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL
new file mode 100755
index 0000000000..0d89a539b8
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/CROSSTOOL
@@ -0,0 +1,1431 @@
+major_version: "local"
+minor_version: ""
+default_target_cpu: "same_as_host"
+
+default_toolchain {
+  cpu: "k8"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "piii"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "arm"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "darwin"
+  toolchain_identifier: "local_darwin"
+}
+default_toolchain {
+  cpu: "ppc"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "x64_windows"
+  toolchain_identifier: "local_windows"
+}
+
+toolchain {
+  abi_version: "local"
+  abi_libc_version: "local"
+  compiler: "compiler"
+  host_system_name: "local"
+  needsPic: true
+  target_libc: "local"
+  target_cpu: "local"
+  target_system_name: "local"
+  toolchain_identifier: "local_linux"
+
+  feature {
+    name: "c++11"
+    flag_set {
+      action: "c++-compile"
+      flag_group {
+        flag: "-std=c++11"
+      }
+    }
+  }
+
+  feature {
+    name: "stdlib"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-lstdc++"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        flag: "-Wno-builtin-macro-redefined"
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  feature {
+    name: "alwayslink"
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-Wl,-no-as-needed"
+      }
+    }
+  }
+
+  # This feature will be enabled for builds that support pic by bazel.
+  feature {
+    name: "pic"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        expand_if_all_available: "pic"
+        flag: "-fPIC"
+      }
+      flag_group {
+        expand_if_none_available: "pic"
+        flag: "-fPIE"
+      }
+    }
+  }
+
+  # Security hardening on by default.
+  feature {
+    name: "hardening"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases.
+        # We need to undef it before redefining it as some distributions now
+        # have it enabled by default.
+        flag: "-U_FORTIFY_SOURCE"
+        flag: "-D_FORTIFY_SOURCE=1"
+        flag: "-fstack-protector"
+      }
+    }
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-Wl,-z,relro,-z,now"
+      }
+    }
+    flag_set {
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-pie"
+        flag: "-Wl,-z,relro,-z,now"
+      }
+    }
+  }
+
+  feature {
+    name: "warnings"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # All warnings are enabled. Maybe enable -Werror as well?
+        flag: "-Wall"
+        
+      }
+    }
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  feature {
+    name: "frame-pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-fno-omit-frame-pointer"
+      }
+    }
+  }
+
+  feature {
+    name: "build-id"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        # Stamp the binary with a unique identifier.
+        flag: "-Wl,--build-id=md5"
+        flag: "-Wl,--hash-style=gnu"
+      }
+    }
+  }
+
+  feature {
+    name: "no-canonical-prefixes"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-no-canonical-prefixes"
+        flag: "-fno-canonical-system-headers"
+      }
+    }
+  }
+
+  feature {
+    name: "disable-assertions"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "linker-bin-path"
+
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-B/usr/bin"
+      }
+    }
+  }
+
+  feature {
+    name: "common"
+    implies: "stdlib"
+    implies: "c++11"
+    implies: "determinism"
+    implies: "alwayslink"
+    implies: "hardening"
+    implies: "warnings"
+    implies: "frame-pointer"
+    implies: "build-id"
+    implies: "no-canonical-prefixes"
+    implies: "linker-bin-path"
+  }
+
+  feature {
+    name: "opt"
+    implies: "common"
+    implies: "disable-assertions"
+
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # No debug symbols.
+        # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt
+        # or even generally? However, that can't happen here, as it requires
+        # special handling in Bazel.
+        flag: "-g0"
+
+        # Conservative choice for -O
+        # -O3 can increase binary size and even slow down the resulting binaries.
+        # Profile first and / or use FDO if you need better performance than this.
+        flag: "-O2"
+
+        # Removal of unused code and data at link time (can this increase binary size in some cases?).
+        flag: "-ffunction-sections"
+        flag: "-fdata-sections"
+      }
+    }
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-Wl,--gc-sections"
+      }
+    }
+  }
+
+  feature {
+    name: "fastbuild"
+    implies: "common"
+  }
+
+  feature {
+    name: "dbg"
+    implies: "common"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-g"
+      }
+    }
+  }
+
+  # Set clang as a C/C++ compiler.
+  tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" }
+
+  # Use the default system toolchain for everything else.
+  tool_path { name: "ar" path: "/usr/bin/ar" }
+  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "cpp" path: "/usr/bin/cpp" }
+  tool_path { name: "dwp" path: "/usr/bin/dwp" }
+  tool_path { name: "gcov" path: "/usr/bin/gcov" }
+  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "nm" path: "/usr/bin/nm" }
+  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "objdump" path: "/usr/bin/objdump" }
+  tool_path { name: "strip" path: "/usr/bin/strip" }
+
+  # Enabled dynamic linking.
+  linking_mode_flags { mode: DYNAMIC }
+
+  cxx_builtin_include_directory: "/usr/include/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/c++/4.8/backward"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include-fixed"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-9.0/targets/x86_64-linux/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-9.0/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-9.0/extras/CUPTI/include"
+  cxx_builtin_include_directory: "/usr/include"
+}
+
+toolchain {
+  abi_version: "local"
+  abi_libc_version: "local"
+  compiler: "compiler"
+  host_system_name: "local"
+  needsPic: true
+  target_libc: "macosx"
+  target_cpu: "darwin"
+  target_system_name: "local"
+  toolchain_identifier: "local_darwin"
+  feature {
+    name: "c++11"
+    flag_set {
+      action: "c++-compile"
+      flag_group {
+        flag: "-std=c++11"
+      }
+    }
+  }
+
+  feature {
+    name: "stdlib"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-lc++"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        flag: "-Wno-builtin-macro-redefined"
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  # This feature will be enabled for builds that support pic by bazel.
+  feature {
+    name: "pic"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        expand_if_all_available: "pic"
+        flag: "-fPIC"
+      }
+      flag_group {
+        expand_if_none_available: "pic"
+        flag: "-fPIE"
+      }
+    }
+  }
+
+  # Security hardening on by default.
+  feature {
+    name: "hardening"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases.
+        # We need to undef it before redefining it as some distributions now
+        # have it enabled by default.
+        flag: "-U_FORTIFY_SOURCE"
+        flag: "-D_FORTIFY_SOURCE=1"
+        flag: "-fstack-protector"
+      }
+    }
+    flag_set {
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-pie"
+      }
+    }
+  }
+
+  feature {
+    name: "warnings"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # All warnings are enabled. Maybe enable -Werror as well?
+        flag: "-Wall"
+        
+      }
+    }
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  feature {
+    name: "frame-pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-fno-omit-frame-pointer"
+      }
+    }
+  }
+
+  feature {
+    name: "no-canonical-prefixes"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag:"-no-canonical-prefixes"
+      }
+    }
+  }
+
+  feature {
+    name: "disable-assertions"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "linker-bin-path"
+
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-B/usr/bin"
+      }
+    }
+  }
+
+  feature {
+    name: "undefined-dynamic"
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-undefined"
+        flag: "dynamic_lookup"
+      }
+    }
+  }
+
+  feature {
+    name: "common"
+    implies: "stdlib"
+    implies: "c++11"
+    implies: "determinism"
+    implies: "hardening"
+    implies: "warnings"
+    implies: "frame-pointer"
+    implies: "no-canonical-prefixes"
+    implies: "linker-bin-path"
+    implies: "undefined-dynamic"
+  }
+
+  feature {
+    name: "opt"
+    implies: "common"
+    implies: "disable-assertions"
+
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # No debug symbols.
+        # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt
+        # or even generally? However, that can't happen here, as it requires
+        # special handling in Bazel.
+        flag: "-g0"
+
+        # Conservative choice for -O
+        # -O3 can increase binary size and even slow down the resulting binaries.
+        # Profile first and / or use FDO if you need better performance than this.
+        flag: "-O2"
+
+        # Removal of unused code and data at link time (can this increase binary size in some cases?).
+        flag: "-ffunction-sections"
+        flag: "-fdata-sections"
+      }
+    }
+  }
+
+  feature {
+    name: "fastbuild"
+    implies: "common"
+  }
+
+  feature {
+    name: "dbg"
+    implies: "common"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-g"
+      }
+    }
+  }
+
+  # Set clang as a C/C++ compiler.
+  tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" }
+
+  # Use the default system toolchain for everything else.
+  tool_path { name: "ar" path: "/usr/bin/libtool" }
+  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "cpp" path: "/usr/bin/cpp" }
+  tool_path { name: "dwp" path: "/usr/bin/dwp" }
+  tool_path { name: "gcov" path: "/usr/bin/gcov" }
+  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "nm" path: "/usr/bin/nm" }
+  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "objdump" path: "/usr/bin/objdump" }
+  tool_path { name: "strip" path: "/usr/bin/strip" }
+
+  # Enabled dynamic linking.
+  linking_mode_flags { mode: DYNAMIC }
+
+  cxx_builtin_include_directory: "/usr/include/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/c++/4.8/backward"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include-fixed"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-9.0/targets/x86_64-linux/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-9.0/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-9.0/extras/CUPTI/include"
+  cxx_builtin_include_directory: "/usr/include"
+}
+
+toolchain {
+  toolchain_identifier: "local_windows"
+  host_system_name: "local"
+  target_system_name: "local"
+
+  abi_version: "local"
+  abi_libc_version: "local"
+  target_cpu: "x64_windows"
+  compiler: "msvc-cl"
+  target_libc: "msvcrt"
+
+
+
+  tool_path {
+    name: "ar"
+    path: ""
+  }
+  tool_path {
+    name: "ml"
+    path: ""
+  }
+  tool_path {
+    name: "cpp"
+    path: ""
+  }
+  tool_path {
+    name: "gcc"
+    path: ""
+  }
+  tool_path {
+    name: "gcov"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "ld"
+    path: ""
+  }
+  tool_path {
+    name: "nm"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objcopy"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objdump"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "strip"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  supports_interface_shared_objects: true
+
+  # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
+  compiler_flag: "/DCOMPILER_MSVC"
+
+  # Don't define min/max macros in windows.h.
+  compiler_flag: "/DNOMINMAX"
+
+  # Platform defines.
+  compiler_flag: "/D_WIN32_WINNT=0x0600"
+  # Turn off warning messages.
+  compiler_flag: "/D_CRT_SECURE_NO_DEPRECATE"
+  compiler_flag: "/D_CRT_SECURE_NO_WARNINGS"
+  compiler_flag: "/D_SILENCE_STDEXT_HASH_DEPRECATION_WARNINGS"
+
+  # Useful options to have on for compilation.
+  # Increase the capacity of object files to 2^32 sections.
+  compiler_flag: "/bigobj"
+  # Allocate 500MB for precomputed headers.
+  compiler_flag: "/Zm500"
+  # Use unsigned char by default.
+  compiler_flag: "/J"
+  # Use function level linking.
+  compiler_flag: "/Gy"
+  # Use string pooling.
+  compiler_flag: "/GF"
+  # Catch C++ exceptions only and tell the compiler to assume that functions declared
+  # as extern "C" never throw a C++ exception.
+  compiler_flag: "/EHsc"
+
+  # Globally disabled warnings.
+  # Don't warn about elements of array being be default initialized.
+  compiler_flag: "/wd4351"
+  # Don't warn about no matching delete found.
+  compiler_flag: "/wd4291"
+  # Don't warn about diamond inheritance patterns.
+  compiler_flag: "/wd4250"
+  # Don't warn about insecure functions (e.g. non _s functions).
+  compiler_flag: "/wd4996"
+
+  linker_flag: "/MACHINE:X64"
+
+  feature {
+    name: "no_legacy_features"
+  }
+
+  # Suppress startup banner.
+  feature {
+    name: "nologo"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      flag_group {
+        flag: "/nologo"
+      }
+    }
+  }
+
+  feature {
+    name: 'has_configured_linker_path'
+  }
+
+  # This feature indicates strip is not supported, building stripped binary will just result a copy of orignial binary
+  feature {
+    name: 'no_stripping'
+  }
+
+  # This feature indicates this is a toolchain targeting Windows.
+  feature {
+    name: 'targets_windows'
+    implies: 'copy_dynamic_libraries_to_binary'
+    enabled: true
+  }
+
+  feature {
+    name: 'copy_dynamic_libraries_to_binary'
+  }
+
+  action_config {
+    config_name: 'assemble'
+    action_name: 'assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'preprocess-assemble'
+    action_name: 'preprocess-assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'c-compile'
+    action_name: 'c-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-compile'
+    action_name: 'c++-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-link-executable'
+    action_name: 'c++-link-executable'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+  }
+
+  action_config {
+    config_name: 'c++-link-dynamic-library'
+    action_name: 'c++-link-dynamic-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'shared_flag'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+    implies: 'has_configured_linker_path'
+    implies: 'def_file'
+  }
+
+  action_config {
+      config_name: 'c++-link-nodeps-dynamic-library'
+      action_name: 'c++-link-nodeps-dynamic-library'
+      tool {
+        tool_path: ''
+      }
+      implies: 'nologo'
+      implies: 'shared_flag'
+      implies: 'linkstamps'
+      implies: 'output_execpath_flags'
+      implies: 'input_param_flags'
+      implies: 'user_link_flags'
+      implies: 'legacy_link_flags'
+      implies: 'linker_subsystem_flag'
+      implies: 'linker_param_file'
+      implies: 'msvc_env'
+      implies: 'no_stripping'
+      implies: 'has_configured_linker_path'
+      implies: 'def_file'
+    }
+
+  action_config {
+    config_name: 'c++-link-static-library'
+    action_name: 'c++-link-static-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'archiver_flags'
+    implies: 'input_param_flags'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+  }
+
+  # TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
+  # not used in this crosstool
+  feature {
+    name: 'legacy_compile_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'legacy_compile_flags'
+        flag: '%{legacy_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "PATH"
+        value: ""
+      }
+      env_entry {
+        key: "INCLUDE"
+        value: ""
+      }
+      env_entry {
+        key: "LIB"
+        value: ""
+      }
+      env_entry {
+        key: "TMP"
+        value: ""
+      }
+      env_entry {
+        key: "TEMP"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: 'include_paths'
+    flag_set {
+      action: "assemble"
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      flag_group {
+        iterate_over: 'quote_include_paths'
+        flag: '/I%{quote_include_paths}'
+      }
+      flag_group {
+        iterate_over: 'include_paths'
+        flag: '/I%{include_paths}'
+      }
+      flag_group {
+        iterate_over: 'system_include_paths'
+        flag: '/I%{system_include_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: "preprocessor_defines"
+    flag_set {
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-header-parsing"
+      action: "c++-module-compile"
+      flag_group {
+        flag: "/D%{preprocessor_defines}"
+        iterate_over: "preprocessor_defines"
+      }
+    }
+  }
+
+  # Tell Bazel to parse the output of /showIncludes
+  feature {
+    name: 'parse_showincludes'
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-module-compile'
+      action: 'c++-header-parsing'
+      flag_group {
+        flag: "/showIncludes"
+      }
+    }
+  }
+
+
+  feature {
+    name: 'generate_pdb_file'
+    requires: {
+      feature: 'dbg'
+    }
+    requires: {
+      feature: 'fastbuild'
+    }
+  }
+
+  feature {
+    name: 'shared_flag'
+    flag_set {
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/DLL'
+      }
+    }
+  }
+
+  feature {
+    name: 'linkstamps'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      expand_if_all_available: 'linkstamp_paths'
+      flag_group {
+        iterate_over: 'linkstamp_paths'
+        flag: '%{linkstamp_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: 'output_execpath_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'archiver_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'input_param_flags'
+    flag_set {
+      expand_if_all_available: 'interface_library_output_path'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/IMPLIB:%{interface_library_output_path}"
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libopts'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'libopts'
+        flag: '%{libopts}'
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libraries_to_link'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        iterate_over: 'libraries_to_link'
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file_group'
+          }
+          iterate_over: 'libraries_to_link.object_files'
+          flag_group {
+            flag: '%{libraries_to_link.object_files}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'interface_library'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'static_library'
+          }
+          flag_group {
+            expand_if_false: 'libraries_to_link.is_whole_archive'
+            flag: '%{libraries_to_link.name}'
+          }
+          flag_group {
+            expand_if_true: 'libraries_to_link.is_whole_archive'
+            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
+          }
+        }
+      }
+    }
+  }
+
+  # Since this feature is declared earlier in the CROSSTOOL than
+  # "user_link_flags", this feature will be applied prior to it anwyhere they
+  # are both implied. And since "user_link_flags" contains the linkopts from
+  # the build rule, this allows the user to override the /SUBSYSTEM in the BUILD
+  # file.
+  feature {
+    name: 'linker_subsystem_flag'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/SUBSYSTEM:CONSOLE'
+      }
+    }
+  }
+
+  # The "user_link_flags" contains user-defined linkopts (from build rules)
+  # so it should be defined after features that declare user-overridable flags.
+  # For example the "linker_subsystem_flag" defines a default "/SUBSYSTEM" flag
+  # but we want to let the user override it, therefore "link_flag_subsystem" is
+  # defined earlier in the CROSSTOOL file than "user_link_flags".
+  feature {
+    name: 'user_link_flags'
+    flag_set {
+      expand_if_all_available: 'user_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'user_link_flags'
+        flag: '%{user_link_flags}'
+      }
+    }
+  }
+  feature {
+    name: 'legacy_link_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'legacy_link_flags'
+        flag: '%{legacy_link_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'linker_param_file'
+    flag_set {
+      expand_if_all_available: 'linker_param_file'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '@%{linker_param_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'static_link_msvcrt'
+  }
+
+  feature {
+    name: 'static_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MT"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MD"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'static_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MTd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MDd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dbg'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+        flag: "/DDEBUG"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEBUG:FULL"
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'fastbuild'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+        flag: "/DDEBUG"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEBUG:FASTLINK"
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'opt'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/O2"
+        flag: "/DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: 'user_compile_flags'
+    flag_set {
+      expand_if_all_available: 'user_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'user_compile_flags'
+        flag: '%{user_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'sysroot'
+    flag_set {
+      expand_if_all_available: 'sysroot'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'sysroot'
+        flag: '--sysroot=%{sysroot}'
+      }
+    }
+  }
+
+  feature {
+    name: 'unfiltered_compile_flags'
+    flag_set {
+      expand_if_all_available: 'unfiltered_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'unfiltered_compile_flags'
+        flag: '%{unfiltered_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_output_flags'
+    flag_set {
+      action: 'assemble'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+        flag: '/Zi'
+      }
+    }
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_assembly_file'
+        flag: '/Fa%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_preprocess_file'
+        flag: '/P'
+        flag: '/Fi%{output_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_input_flags'
+    flag_set {
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'source_file'
+        flag: '/c'
+        flag: '%{source_file}'
+      }
+    }
+  }
+
+  feature {
+    name : 'def_file',
+    flag_set {
+      expand_if_all_available: 'def_file_path'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEF:%{def_file_path}"
+        # We can specify a different DLL name in DEF file, /ignore:4070 suppresses
+        # the warning message about DLL name doesn't match the default one.
+        # See https://msdn.microsoft.com/en-us/library/sfkk2fz7.aspx
+        flag: "/ignore:4070"
+      }
+    }
+  }
+
+  feature {
+    name: 'windows_export_all_symbols'
+  }
+
+  feature {
+    name: 'no_windows_export_all_symbols'
+  }
+
+  linking_mode_flags { mode: DYNAMIC }
+}
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
new file mode 100755
index 0000000000..63893d3722
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs.
+
+SYNOPSIS:
+  crosstool_wrapper_is_not_gcc [options passed in by cc_library()
+                                or cc_binary() rule]
+
+DESCRIPTION:
+  This script is expected to be called by the cc_library() or cc_binary() bazel
+  rules. When the option "-x cuda" is present in the list of arguments passed
+  to this script, it invokes the nvcc CUDA compiler. Most arguments are passed
+  as is as a string to --compiler-options of nvcc. When "-x cuda" is not
+  present, this wrapper invokes hybrid_driver_is_not_gcc with the input
+  arguments as is.
+
+NOTES:
+  Changes to the contents of this file must be propagated from
+  //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc to
+  //third_party/gpus/crosstool/v*/*/clang/bin/crosstool_wrapper_is_not_gcc
+"""
+
+from __future__ import print_function
+
+__author__ = 'keveman@google.com (Manjunath Kudlur)'
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('/usr/bin/gcc')
+GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc')
+
+NVCC_PATH = '/usr/local/cuda-9.0/bin/nvcc'
+PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
+NVCC_VERSION = '9.0'
+
+def Log(s):
+  print('gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from the argv list.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    option: The option whose value to extract, without the leading '-'.
+
+  Returns:
+    A list of values, either directly following the option,
+    (eg., -opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., -opt val1 -opt val2).
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-' + option, nargs='*', action='append')
+  args, _ = parser.parse_known_args(argv)
+  if not args or not vars(args)[option]:
+    return []
+  else:
+    return sum(vars(args)[option], [])
+
+
+def GetHostCompilerOptions(argv):
+  """Collect the -isystem, -iquote, and --sysroot option values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be used as the --compiler-options to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-isystem', nargs='*', action='append')
+  parser.add_argument('-iquote', nargs='*', action='append')
+  parser.add_argument('--sysroot', nargs=1)
+  parser.add_argument('-g', nargs='*', action='append')
+  parser.add_argument('-fno-canonical-system-headers', action='store_true')
+
+  args, _ = parser.parse_known_args(argv)
+
+  opts = ''
+
+  if args.isystem:
+    opts += ' -isystem ' + ' -isystem '.join(sum(args.isystem, []))
+  if args.iquote:
+    opts += ' -iquote ' + ' -iquote '.join(sum(args.iquote, []))
+  if args.g:
+    opts += ' -g' + ' -g'.join(sum(args.g, []))
+  if args.fno_canonical_system_headers:
+    opts += ' -fno-canonical-system-headers'
+  if args.sysroot:
+    opts += ' --sysroot ' + args.sysroot[0]
+
+  return opts
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be passed directly to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, _ = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return ' '.join(['--'+a for a in options])
+  return ''
+
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling os.system('nvcc ' + args)
+  """
+
+  host_compiler_options = GetHostCompilerOptions(argv)
+  nvcc_compiler_options = GetNvccOptions(argv)
+  opt_option = GetOptionValue(argv, 'O')
+  m_options = GetOptionValue(argv, 'm')
+  m_options = ''.join([' -m' + m for m in m_options if m in ['32', '64']])
+  include_options = GetOptionValue(argv, 'I')
+  out_file = GetOptionValue(argv, 'o')
+  depfiles = GetOptionValue(argv, 'MF')
+  defines = GetOptionValue(argv, 'D')
+  defines = ''.join([' -D' + define for define in defines])
+  undefines = GetOptionValue(argv, 'U')
+  undefines = ''.join([' -U' + define for define in undefines])
+  std_options = GetOptionValue(argv, 'std')
+  # currently only c++11 is supported by Cuda 7.0 std argument
+  nvcc_allowed_std_options = ["c++11"]
+  std_options = ''.join([' -std=' + define
+      for define in std_options if define in nvcc_allowed_std_options])
+
+  # The list of source files get passed after the -c option. I don't know of
+  # any other reliable way to just get the list of source files to be compiled.
+  src_files = GetOptionValue(argv, 'c')
+
+  # Pass -w through from host to nvcc, but don't do anything fancier with
+  # warnings-related flags, since they're not necessarily the same across
+  # compilers.
+  warning_options = ' -w' if '-w' in argv else ''
+
+  if len(src_files) == 0:
+    return 1
+  if len(out_file) != 1:
+    return 1
+
+  opt = (' -O2' if (len(opt_option) > 0 and int(opt_option[0]) > 0)
+         else ' -g -G')
+
+  includes = (' -I ' + ' -I '.join(include_options)
+              if len(include_options) > 0
+              else '')
+
+  # Unfortunately, there are other options that have -c prefix too.
+  # So allowing only those look like C/C++ files.
+  src_files = [f for f in src_files if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+  srcs = ' '.join(src_files)
+  out = ' -o ' + out_file[0]
+
+  supported_cuda_compute_capabilities = [ "3.0" ]
+  nvccopts = '-D_FORCE_INLINES '
+  for capability in supported_cuda_compute_capabilities:
+    capability = capability.replace('.', '')
+    nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s,compute_%s\" ' % (
+        capability, capability, capability)
+  nvccopts += ' ' + nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += std_options
+  nvccopts += m_options
+  nvccopts += warning_options
+
+  if depfiles:
+    # Generate the dependency file
+    depfile = depfiles[0]
+    cmd = (NVCC_PATH + ' ' + nvccopts +
+           ' --compiler-options "' + host_compiler_options + '"' +
+           ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+           ' -I .' +
+           ' -x cu ' + opt + includes + ' ' + srcs + ' -M -o ' + depfile)
+    if log: Log(cmd)
+    exit_status = os.system(cmd)
+    if exit_status != 0:
+      return exit_status
+
+  cmd = (NVCC_PATH + ' ' + nvccopts +
+         ' --compiler-options "' + host_compiler_options + ' -fPIC"' +
+         ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+         ' -I .' +
+         ' -x cu ' + opt + includes + ' -c ' + srcs + out)
+
+  # TODO(zhengxq): for some reason, 'gcc' needs this help to find 'as'.
+  # Need to investigate and fix.
+  cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd
+  if log: Log(cmd)
+  return os.system(cmd)
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.bat b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.bat
new file mode 100755
index 0000000000..e896e654fd
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.bat
@@ -0,0 +1,20 @@
+:: Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+::
+:: Licensed under the Apache License, Version 2.0 (the "License");
+:: you may not use this file except in compliance with the License.
+:: You may obtain a copy of the License at
+::
+::     http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+:: =============================================================================
+
+:: Invoke msvc_wrapper_for_nvcc.py, which is located in the same directory.
+@echo OFF
+set arg0=%~0
+for %%F in ("%arg0%") do set DRIVER_BIN=%%~dpF
+"/usr/bin/python3" -B "%DRIVER_BIN%\msvc_wrapper_for_nvcc.py" %*
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
new file mode 100755
index 0000000000..859b3196d5
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs with nvcc on Windows.
+
+DESCRIPTION:
+  This script is the Windows version of //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc
+"""
+
+from __future__ import print_function
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('/usr/bin/gcc')
+GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc')
+
+NVCC_PATH = '/usr/local/cuda-9.0/bin/nvcc'
+NVCC_VERSION = '9.0'
+NVCC_TEMP_DIR = "C:\\Windows\\Temp\\nvcc_inter_files_tmp_dir"
+supported_cuda_compute_capabilities = [ "3.0" ]
+
+def Log(s):
+  print('gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from options.
+
+  Args:
+    option: The option whose value to extract, without the leading '/'.
+
+  Returns:
+    1. A list of values, either directly following the option,
+    (eg., /opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., /opt val1 /opt val2).
+    2. The leftover options.
+  """
+
+  parser = ArgumentParser(prefix_chars='/')
+  parser.add_argument('/' + option, nargs='*', action='append')
+  args, leftover = parser.parse_known_args(argv)
+  if args and vars(args)[option]:
+    return (sum(vars(args)[option], []), leftover)
+  return ([], leftover)
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    1. The string that can be passed directly to nvcc.
+    2. The leftover options.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, leftover = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return (['--' + a for a in options], leftover)
+  return ([], leftover)
+
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling os.system('nvcc ' + args)
+  """
+
+  src_files = [f for f in argv if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+  if len(src_files) == 0:
+    raise Error('No source files found for cuda compilation.')
+
+  out_file = [ f for f in argv if f.startswith('/Fo') ]
+  if len(out_file) != 1:
+    raise Error('Please sepecify exactly one output file for cuda compilation.')
+  out = ['-o', out_file[0][len('/Fo'):]]
+
+  nvcc_compiler_options, argv = GetNvccOptions(argv)
+
+  opt_option, argv = GetOptionValue(argv, 'O')
+  opt = ['-g', '-G']
+  if (len(opt_option) > 0 and opt_option[0] != 'd'):
+    opt = ['-O2']
+
+  include_options, argv = GetOptionValue(argv, 'I')
+  includes = ["-I " + include for include in include_options]
+
+  defines, argv = GetOptionValue(argv, 'D')
+  defines = ['-D' + define for define in defines]
+
+  undefines, argv = GetOptionValue(argv, 'U')
+  undefines = ['-U' + define for define in undefines]
+
+  # The rest of the unrecongized options should be passed to host compiler
+  host_compiler_options = [option for option in argv if option not in (src_files + out_file)]
+
+  m_options = ["-m64"]
+
+  nvccopts = ['-D_FORCE_INLINES']
+  for capability in supported_cuda_compute_capabilities:
+    capability = capability.replace('.', '')
+    nvccopts += [r'-gencode=arch=compute_%s,"code=sm_%s,compute_%s"' % (
+        capability, capability, capability)]
+  nvccopts += nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += m_options
+  nvccopts += ['--compiler-options="' + " ".join(host_compiler_options) + '"']
+  nvccopts += ['-x', 'cu'] + opt + includes + out + ['-c'] + src_files
+  # If we don't specify --keep-dir, nvcc will generate intermediate files under TEMP
+  # Put them under NVCC_TEMP_DIR instead, then Bazel can ignore files under NVCC_TEMP_DIR during dependency check
+  # http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-guiding-compiler-driver
+  # Different actions are sharing NVCC_TEMP_DIR, so we cannot remove it if the directory already exists.
+  if os.path.isfile(NVCC_TEMP_DIR):
+    os.remove(NVCC_TEMP_DIR)
+  if not os.path.exists(NVCC_TEMP_DIR):
+    os.makedirs(NVCC_TEMP_DIR)
+  nvccopts += ['--keep', '--keep-dir', NVCC_TEMP_DIR]
+  cmd = [NVCC_PATH] + nvccopts
+  if log:
+    Log(cmd)
+  proc = subprocess.Popen(cmd,
+                          stdout=sys.stdout,
+                          stderr=sys.stderr,
+                          env=os.environ.copy(),
+                          shell=True)
+  proc.wait()
+  return proc.returncode
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))
+                             and not flag.startswith(('-nvcc_options'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
index e021df9e1e..460c879d32 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
@@ -136,7 +136,7 @@ genrule(
         "python_include/weakrefobject.h",
     ],
     cmd = """
-cp "/usr/include/python3.4m/Python-ast.h" "$(@D)/python_include/Python-ast.h" && cp "/usr/include/python3.4m/Python.h" "$(@D)/python_include/Python.h" && cp "/usr/include/python3.4m/abstract.h" "$(@D)/python_include/abstract.h" && cp "/usr/include/python3.4m/accu.h" "$(@D)/python_include/accu.h" && cp "/usr/include/python3.4m/asdl.h" "$(@D)/python_include/asdl.h" && cp "/usr/include/python3.4m/ast.h" "$(@D)/python_include/ast.h" && cp "/usr/include/python3.4m/bitset.h" "$(@D)/python_include/bitset.h" && cp "/usr/include/python3.4m/bltinmodule.h" "$(@D)/python_include/bltinmodule.h" && cp "/usr/include/python3.4m/boolobject.h" "$(@D)/python_include/boolobject.h" && cp "/usr/include/python3.4m/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp "/usr/include/python3.4m/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp "/usr/include/python3.4m/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp "/usr/include/python3.4m/cellobject.h" "$(@D)/python_include/cellobject.h" && cp "/usr/include/python3.4m/ceval.h" "$(@D)/python_include/ceval.h" && cp "/usr/include/python3.4m/classobject.h" "$(@D)/python_include/classobject.h" && cp "/usr/include/python3.4m/code.h" "$(@D)/python_include/code.h" && cp "/usr/include/python3.4m/codecs.h" "$(@D)/python_include/codecs.h" && cp "/usr/include/python3.4m/compile.h" "$(@D)/python_include/compile.h" && cp "/usr/include/python3.4m/complexobject.h" "$(@D)/python_include/complexobject.h" && cp "/usr/include/python3.4m/datetime.h" "$(@D)/python_include/datetime.h" && cp "/usr/include/python3.4m/descrobject.h" "$(@D)/python_include/descrobject.h" && cp "/usr/include/python3.4m/dictobject.h" "$(@D)/python_include/dictobject.h" && cp "/usr/include/python3.4m/dtoa.h" "$(@D)/python_include/dtoa.h" && cp "/usr/include/python3.4m/dynamic_annotations.h" "$(@D)/python_include/dynamic_annotations.h" && cp "/usr/include/python3.4m/enumobject.h" "$(@D)/python_include/enumobject.h" && cp "/usr/include/python3.4m/errcode.h" "$(@D)/python_include/errcode.h" && cp "/usr/include/python3.4m/eval.h" "$(@D)/python_include/eval.h" && cp "/usr/include/python3.4m/fileobject.h" "$(@D)/python_include/fileobject.h" && cp "/usr/include/python3.4m/fileutils.h" "$(@D)/python_include/fileutils.h" && cp "/usr/include/python3.4m/floatobject.h" "$(@D)/python_include/floatobject.h" && cp "/usr/include/python3.4m/frameobject.h" "$(@D)/python_include/frameobject.h" && cp "/usr/include/python3.4m/funcobject.h" "$(@D)/python_include/funcobject.h" && cp "/usr/include/python3.4m/genobject.h" "$(@D)/python_include/genobject.h" && cp "/usr/include/python3.4m/graminit.h" "$(@D)/python_include/graminit.h" && cp "/usr/include/python3.4m/grammar.h" "$(@D)/python_include/grammar.h" && cp "/usr/include/python3.4m/import.h" "$(@D)/python_include/import.h" && cp "/usr/include/python3.4m/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp "/usr/include/python3.4m/iterobject.h" "$(@D)/python_include/iterobject.h" && cp "/usr/include/python3.4m/listobject.h" "$(@D)/python_include/listobject.h" && cp "/usr/include/python3.4m/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp "/usr/include/python3.4m/longobject.h" "$(@D)/python_include/longobject.h" && cp "/usr/include/python3.4m/marshal.h" "$(@D)/python_include/marshal.h" && cp "/usr/include/python3.4m/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp "/usr/include/python3.4m/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp "/usr/include/python3.4m/methodobject.h" "$(@D)/python_include/methodobject.h" && cp "/usr/include/python3.4m/modsupport.h" "$(@D)/python_include/modsupport.h" && cp "/usr/include/python3.4m/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp "/usr/include/python3.4m/namespaceobject.h" "$(@D)/python_include/namespaceobject.h" && cp "/usr/include/python3.4m/node.h" "$(@D)/python_include/node.h" && cp "/usr/include/python3.4m/object.h" "$(@D)/python_include/object.h" && cp "/usr/include/python3.4m/objimpl.h" "$(@D)/python_include/objimpl.h" && cp "/usr/include/python3.4m/opcode.h" "$(@D)/python_include/opcode.h" && cp "/usr/include/python3.4m/osdefs.h" "$(@D)/python_include/osdefs.h" && cp "/usr/include/python3.4m/parsetok.h" "$(@D)/python_include/parsetok.h" && cp "/usr/include/python3.4m/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp "/usr/include/python3.4m/pgen.h" "$(@D)/python_include/pgen.h" && cp "/usr/include/python3.4m/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp "/usr/include/python3.4m/py_curses.h" "$(@D)/python_include/py_curses.h" && cp "/usr/include/python3.4m/pyarena.h" "$(@D)/python_include/pyarena.h" && cp "/usr/include/python3.4m/pyatomic.h" "$(@D)/python_include/pyatomic.h" && cp "/usr/include/python3.4m/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp "/usr/include/python3.4m/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp "/usr/include/python3.4m/pyctype.h" "$(@D)/python_include/pyctype.h" && cp "/usr/include/python3.4m/pydebug.h" "$(@D)/python_include/pydebug.h" && cp "/usr/include/python3.4m/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp "/usr/include/python3.4m/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp "/usr/include/python3.4m/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp "/usr/include/python3.4m/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp "/usr/include/python3.4m/pyhash.h" "$(@D)/python_include/pyhash.h" && cp "/usr/include/python3.4m/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp "/usr/include/python3.4m/pymacro.h" "$(@D)/python_include/pymacro.h" && cp "/usr/include/python3.4m/pymath.h" "$(@D)/python_include/pymath.h" && cp "/usr/include/python3.4m/pymem.h" "$(@D)/python_include/pymem.h" && cp "/usr/include/python3.4m/pyport.h" "$(@D)/python_include/pyport.h" && cp "/usr/include/python3.4m/pystate.h" "$(@D)/python_include/pystate.h" && cp "/usr/include/python3.4m/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp "/usr/include/python3.4m/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp "/usr/include/python3.4m/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp "/usr/include/python3.4m/pythread.h" "$(@D)/python_include/pythread.h" && cp "/usr/include/python3.4m/pytime.h" "$(@D)/python_include/pytime.h" && cp "/usr/include/python3.4m/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp "/usr/include/python3.4m/setobject.h" "$(@D)/python_include/setobject.h" && cp "/usr/include/python3.4m/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp "/usr/include/python3.4m/structmember.h" "$(@D)/python_include/structmember.h" && cp "/usr/include/python3.4m/structseq.h" "$(@D)/python_include/structseq.h" && cp "/usr/include/python3.4m/symtable.h" "$(@D)/python_include/symtable.h" && cp "/usr/include/python3.4m/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp "/usr/include/python3.4m/token.h" "$(@D)/python_include/token.h" && cp "/usr/include/python3.4m/traceback.h" "$(@D)/python_include/traceback.h" && cp "/usr/include/python3.4m/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp "/usr/include/python3.4m/typeslots.h" "$(@D)/python_include/typeslots.h" && cp "/usr/include/python3.4m/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp "/usr/include/python3.4m/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp "/usr/include/python3.4m/warnings.h" "$(@D)/python_include/warnings.h" && cp "/usr/include/python3.4m/weakrefobject.h" "$(@D)/python_include/weakrefobject.h"
+cp -f "/usr/include/python3.4m/Python-ast.h" "$(@D)/python_include/Python-ast.h" && cp -f "/usr/include/python3.4m/Python.h" "$(@D)/python_include/Python.h" && cp -f "/usr/include/python3.4m/abstract.h" "$(@D)/python_include/abstract.h" && cp -f "/usr/include/python3.4m/accu.h" "$(@D)/python_include/accu.h" && cp -f "/usr/include/python3.4m/asdl.h" "$(@D)/python_include/asdl.h" && cp -f "/usr/include/python3.4m/ast.h" "$(@D)/python_include/ast.h" && cp -f "/usr/include/python3.4m/bitset.h" "$(@D)/python_include/bitset.h" && cp -f "/usr/include/python3.4m/bltinmodule.h" "$(@D)/python_include/bltinmodule.h" && cp -f "/usr/include/python3.4m/boolobject.h" "$(@D)/python_include/boolobject.h" && cp -f "/usr/include/python3.4m/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp -f "/usr/include/python3.4m/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp -f "/usr/include/python3.4m/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp -f "/usr/include/python3.4m/cellobject.h" "$(@D)/python_include/cellobject.h" && cp -f "/usr/include/python3.4m/ceval.h" "$(@D)/python_include/ceval.h" && cp -f "/usr/include/python3.4m/classobject.h" "$(@D)/python_include/classobject.h" && cp -f "/usr/include/python3.4m/code.h" "$(@D)/python_include/code.h" && cp -f "/usr/include/python3.4m/codecs.h" "$(@D)/python_include/codecs.h" && cp -f "/usr/include/python3.4m/compile.h" "$(@D)/python_include/compile.h" && cp -f "/usr/include/python3.4m/complexobject.h" "$(@D)/python_include/complexobject.h" && cp -f "/usr/include/python3.4m/datetime.h" "$(@D)/python_include/datetime.h" && cp -f "/usr/include/python3.4m/descrobject.h" "$(@D)/python_include/descrobject.h" && cp -f "/usr/include/python3.4m/dictobject.h" "$(@D)/python_include/dictobject.h" && cp -f "/usr/include/python3.4m/dtoa.h" "$(@D)/python_include/dtoa.h" && cp -f "/usr/include/python3.4m/dynamic_annotations.h" "$(@D)/python_include/dynamic_annotations.h" && cp -f "/usr/include/python3.4m/enumobject.h" "$(@D)/python_include/enumobject.h" && cp -f "/usr/include/python3.4m/errcode.h" "$(@D)/python_include/errcode.h" && cp -f "/usr/include/python3.4m/eval.h" "$(@D)/python_include/eval.h" && cp -f "/usr/include/python3.4m/fileobject.h" "$(@D)/python_include/fileobject.h" && cp -f "/usr/include/python3.4m/fileutils.h" "$(@D)/python_include/fileutils.h" && cp -f "/usr/include/python3.4m/floatobject.h" "$(@D)/python_include/floatobject.h" && cp -f "/usr/include/python3.4m/frameobject.h" "$(@D)/python_include/frameobject.h" && cp -f "/usr/include/python3.4m/funcobject.h" "$(@D)/python_include/funcobject.h" && cp -f "/usr/include/python3.4m/genobject.h" "$(@D)/python_include/genobject.h" && cp -f "/usr/include/python3.4m/graminit.h" "$(@D)/python_include/graminit.h" && cp -f "/usr/include/python3.4m/grammar.h" "$(@D)/python_include/grammar.h" && cp -f "/usr/include/python3.4m/import.h" "$(@D)/python_include/import.h" && cp -f "/usr/include/python3.4m/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp -f "/usr/include/python3.4m/iterobject.h" "$(@D)/python_include/iterobject.h" && cp -f "/usr/include/python3.4m/listobject.h" "$(@D)/python_include/listobject.h" && cp -f "/usr/include/python3.4m/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp -f "/usr/include/python3.4m/longobject.h" "$(@D)/python_include/longobject.h" && cp -f "/usr/include/python3.4m/marshal.h" "$(@D)/python_include/marshal.h" && cp -f "/usr/include/python3.4m/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp -f "/usr/include/python3.4m/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp -f "/usr/include/python3.4m/methodobject.h" "$(@D)/python_include/methodobject.h" && cp -f "/usr/include/python3.4m/modsupport.h" "$(@D)/python_include/modsupport.h" && cp -f "/usr/include/python3.4m/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp -f "/usr/include/python3.4m/namespaceobject.h" "$(@D)/python_include/namespaceobject.h" && cp -f "/usr/include/python3.4m/node.h" "$(@D)/python_include/node.h" && cp -f "/usr/include/python3.4m/object.h" "$(@D)/python_include/object.h" && cp -f "/usr/include/python3.4m/objimpl.h" "$(@D)/python_include/objimpl.h" && cp -f "/usr/include/python3.4m/opcode.h" "$(@D)/python_include/opcode.h" && cp -f "/usr/include/python3.4m/osdefs.h" "$(@D)/python_include/osdefs.h" && cp -f "/usr/include/python3.4m/parsetok.h" "$(@D)/python_include/parsetok.h" && cp -f "/usr/include/python3.4m/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp -f "/usr/include/python3.4m/pgen.h" "$(@D)/python_include/pgen.h" && cp -f "/usr/include/python3.4m/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp -f "/usr/include/python3.4m/py_curses.h" "$(@D)/python_include/py_curses.h" && cp -f "/usr/include/python3.4m/pyarena.h" "$(@D)/python_include/pyarena.h" && cp -f "/usr/include/python3.4m/pyatomic.h" "$(@D)/python_include/pyatomic.h" && cp -f "/usr/include/python3.4m/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp -f "/usr/include/python3.4m/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp -f "/usr/include/python3.4m/pyctype.h" "$(@D)/python_include/pyctype.h" && cp -f "/usr/include/python3.4m/pydebug.h" "$(@D)/python_include/pydebug.h" && cp -f "/usr/include/python3.4m/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp -f "/usr/include/python3.4m/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp -f "/usr/include/python3.4m/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp -f "/usr/include/python3.4m/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp -f "/usr/include/python3.4m/pyhash.h" "$(@D)/python_include/pyhash.h" && cp -f "/usr/include/python3.4m/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp -f "/usr/include/python3.4m/pymacro.h" "$(@D)/python_include/pymacro.h" && cp -f "/usr/include/python3.4m/pymath.h" "$(@D)/python_include/pymath.h" && cp -f "/usr/include/python3.4m/pymem.h" "$(@D)/python_include/pymem.h" && cp -f "/usr/include/python3.4m/pyport.h" "$(@D)/python_include/pyport.h" && cp -f "/usr/include/python3.4m/pystate.h" "$(@D)/python_include/pystate.h" && cp -f "/usr/include/python3.4m/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp -f "/usr/include/python3.4m/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp -f "/usr/include/python3.4m/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp -f "/usr/include/python3.4m/pythread.h" "$(@D)/python_include/pythread.h" && cp -f "/usr/include/python3.4m/pytime.h" "$(@D)/python_include/pytime.h" && cp -f "/usr/include/python3.4m/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp -f "/usr/include/python3.4m/setobject.h" "$(@D)/python_include/setobject.h" && cp -f "/usr/include/python3.4m/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp -f "/usr/include/python3.4m/structmember.h" "$(@D)/python_include/structmember.h" && cp -f "/usr/include/python3.4m/structseq.h" "$(@D)/python_include/structseq.h" && cp -f "/usr/include/python3.4m/symtable.h" "$(@D)/python_include/symtable.h" && cp -f "/usr/include/python3.4m/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp -f "/usr/include/python3.4m/token.h" "$(@D)/python_include/token.h" && cp -f "/usr/include/python3.4m/traceback.h" "$(@D)/python_include/traceback.h" && cp -f "/usr/include/python3.4m/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp -f "/usr/include/python3.4m/typeslots.h" "$(@D)/python_include/typeslots.h" && cp -f "/usr/include/python3.4m/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp -f "/usr/include/python3.4m/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp -f "/usr/include/python3.4m/warnings.h" "$(@D)/python_include/warnings.h" && cp -f "/usr/include/python3.4m/weakrefobject.h" "$(@D)/python_include/weakrefobject.h"
    """,
 )
 
@@ -171,6 +171,6 @@ genrule(
         "numpy_include/numpy/utils.h",
     ],
     cmd = """
-cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h"
+cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp -f "/usr/local/lib/python3.4/dist-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h"
    """,
 )
-- 
GitLab


From d692984d21ff115067164aea9629fa9e9dc0dce1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 20 Nov 2018 08:06:34 -0800
Subject: [PATCH 0569/1554] Fixes issue loading cached_value from
 resourcevariable proto.

PiperOrigin-RevId: 222247164
---
 .../kernel_tests/resource_variable_ops_test.py     | 14 ++++++++++++++
 tensorflow/python/ops/resource_variable_ops.py     |  5 ++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index eedc2d263d..c351a18c8f 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -568,6 +568,20 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     v.load(2.0)
     self.assertEqual(2.0, self.evaluate(v.value()))
 
+  def testToFromProtoCachedValue(self):
+    with ops.Graph().as_default():
+      v_def = resource_variable_ops.ResourceVariable(
+          initial_value=constant_op.constant(3.0)).to_proto()
+      v_prime = resource_variable_ops.ResourceVariable(variable_def=v_def)
+      self.assertTrue(getattr(v_prime, "_cached_value", None) is None)
+
+      other_v_def = resource_variable_ops.ResourceVariable(
+          caching_device="cpu:0",
+          initial_value=constant_op.constant(3.0)).to_proto()
+      other_v_prime = resource_variable_ops.ResourceVariable(
+          variable_def=other_v_def)
+      self.assertTrue(other_v_prime._cached_value is not None)
+
   def testVariableDefInitializedInstances(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v_def = resource_variable_ops.ResourceVariable(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 488b6fcbcd..c20f8fb938 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -519,7 +519,10 @@ class ResourceVariable(variables.RefVariable):
       snapshot = g.as_graph_element(
           ops.prepend_name_scope(
               variable_def.snapshot_name, import_scope=import_scope))
-      self._cached_value = snapshot
+      if snapshot.op.type != "ReadVariableOp":
+        self._cached_value = snapshot
+      else:
+        self._cached_value = None
       while snapshot.op.type != "ReadVariableOp":
         snapshot = snapshot.op.inputs[0]
       self._graph_element = snapshot
-- 
GitLab


From dcc76a5c4945406455213b05fb8c084b5a8b9b6d Mon Sep 17 00:00:00 2001
From: "William D. Irons" <wdirons@us.ibm.com>
Date: Tue, 20 Nov 2018 10:35:18 -0600
Subject: [PATCH 0570/1554] Add PYTHONPATH to bazelrc when necessary

When the user choosen python_lib_path was retreived from the
PYTHONPATH environment variable, need to set PYTHONPATH in the
bazelrc file so bazel includes it in all the build operations.

Fixes tensorflow#23695
---
 configure.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/configure.py b/configure.py
index 234561d94a..17ab7a0d63 100644
--- a/configure.py
+++ b/configure.py
@@ -238,6 +238,13 @@ def setup_python(environ_cp):
   write_to_bazelrc('build --python_path=\"%s"' % python_bin_path)
   environ_cp['PYTHON_BIN_PATH'] = python_bin_path
 
+  # If choosen python_lib_path is from a path specified in the PYTHONPATH
+  # variable, need to tell bazel to include PYTHONPATH
+  if environ_cp.get('PYTHONPATH'):
+    python_paths = environ_cp.get('PYTHONPATH').split(':')
+    if python_lib_path in python_paths:
+         write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
+
   # Write tools/python_bin_path.sh
   with open(
       os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
-- 
GitLab


From 1e4bb841565552ea2e27474abae8c8c1833a753e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 09:02:05 -0800
Subject: [PATCH 0571/1554] Add Dockerfile for cuda 10.

PiperOrigin-RevId: 222254468
---
 ...Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
new file mode 100644
index 0000000000..85b9d94313
--- /dev/null
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -0,0 +1,75 @@
+# To push a new version, run:
+# $ docker build -f Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 \
+#       --tag "gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04" .
+# $ docker push gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04
+
+FROM ubuntu:14.04
+LABEL maintainer="Manuel Klimek <klimek@google.com>"
+
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \
+    rm -rf /var/lib/apt/lists/* && \
+    NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \
+    NVIDIA_GPGKEY_FPR=ae09fe4bbd223a84b2ccfce3f60f4b3d7fa2af80 && \
+    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \
+    apt-key adv --export --no-emit-version -a $NVIDIA_GPGKEY_FPR | tail -n +2 > cudasign.pub && \
+    echo "$NVIDIA_GPGKEY_SUM  cudasign.pub" | sha256sum -c --strict - && rm cudasign.pub && \
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \
+    echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list
+
+ENV CUDA_VERSION 10.0.130
+ENV CUDA_PKG_VERSION 10-0=$CUDA_VERSION-1
+ENV CUDNN_VERSION 7.3.1.20
+ENV NCCL_VERSION 2.3.5
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV NVIDIA_REQUIRE_CUDA "cuda>=10.0,driver>=410"
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV PATH /usr/local/cuda/bin:${PATH}
+
+# TODO(b/110903506): /usr/loca/cuda/lib64/stubs should not be needed in
+# LD_LIBRARY_PATH. The stubs/libcuda.so is not meant to used at runtime. The
+# correct way to pass the path to bfd-ld is to pass
+# -Wl,-rpath-link=/usr/local/cuda/lib64/stubs to all binaries transitively
+# depending on libcuda. Optimally, builds targeting cuda would do that
+# internally.
+ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
+
+LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        cuda-command-line-tools-$CUDA_PKG_VERSION \
+        cuda-compat-10-0=410.48-1 \
+        cuda-cudart-$CUDA_PKG_VERSION \
+        cuda-libraries-$CUDA_PKG_VERSION \
+        cuda-libraries-dev-$CUDA_PKG_VERSION \
+        cuda-minimal-build-$CUDA_PKG_VERSION \
+        cuda-nvml-dev-$CUDA_PKG_VERSION \
+        cuda-nvtx-$CUDA_PKG_VERSION \
+        libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
+        libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
+        libcudnn7-dev=$CUDNN_VERSION-1+cuda10.0 \
+        libnccl2=$NCCL_VERSION-2+cuda10.0 \
+        libnccl-dev=$NCCL_VERSION-2+cuda10.0 && \
+    ln -s cuda-10.0 /usr/local/cuda && \
+    apt-mark hold libcudnn7 && \
+    apt-mark hold libnccl2 && \
+    rm -rf /var/lib/apt/lists/*
+
+# TODO(b/110903506): Provide a link to the SONAME of libcuda.so.
+# https://github.com/NVIDIA/nvidia-docker/issues/775
+RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
+
+# TODO(klimek): Once the TODO in tensorflow's configure.py to correctly find
+# libnccl is resolved, delete this block.
+RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \
+ && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2
+
+# Copy and run the install scripts.
+COPY install/*.sh /install/
+ARG DEBIAN_FRONTEND=noninteractive
+RUN /install/install_bootstrap_deb_packages.sh
+RUN add-apt-repository -y ppa:openjdk-r/ppa && \
+    add-apt-repository -y ppa:george-edison55/cmake-3.x
+RUN /install/install_deb_packages.sh
+RUN /install/install_pip_packages.sh
+RUN /install/install_golang.sh
+
-- 
GitLab


From 6afb388d49ab32055caf71974e863f5f1db65e0d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 09:13:31 -0800
Subject: [PATCH 0572/1554] Add tensorflow implementation of ctc_loss that runs
 on gpu/tpu.

PiperOrigin-RevId: 222256226
---
 tensorflow/python/kernel_tests/BUILD          |   2 +-
 .../python/kernel_tests/ctc_loss_op_test.py   | 546 +++++++++++-
 tensorflow/python/ops/ctc_ops.py              | 793 +++++++++++++++++-
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  12 +
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  10 +-
 5 files changed, 1354 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 41099ba2e8..19facca5a6 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -270,7 +270,7 @@ tf_py_test(
     ],
 )
 
-tf_py_test(
+cuda_py_test(
     name = "ctc_loss_op_test",
     size = "small",
     srcs = ["ctc_loss_op_test.py"],
diff --git a/tensorflow/python/kernel_tests/ctc_loss_op_test.py b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
index cfc7cb98aa..b38776ec5b 100644
--- a/tensorflow/python/kernel_tests/ctc_loss_op_test.py
+++ b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
@@ -23,9 +23,15 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ctc_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
 
 
@@ -52,6 +58,24 @@ def SimpleSparseTensorFrom(x):
   return sparse_tensor.SparseTensor(x_ix, x_val, x_shape)
 
 
+def _ctc_loss_v2(labels, inputs, sequence_length,
+                 preprocess_collapse_repeated=False,
+                 ctc_merge_repeated=True,
+                 ignore_longer_outputs_than_inputs=False,
+                 time_major=True):
+  """Call ctc_loss_v2 with v1 args."""
+  assert not preprocess_collapse_repeated
+  assert ctc_merge_repeated
+  assert not ignore_longer_outputs_than_inputs
+  return ctc_ops.ctc_loss_v2(
+      labels=labels,
+      logits=inputs,
+      logit_length=sequence_length,
+      label_length=None,
+      blank_index=-1,
+      logits_time_major=time_major)
+
+
 class CTCLossTest(test.TestCase):
 
   def _testCTCLoss(self,
@@ -66,7 +90,7 @@ class CTCLossTest(test.TestCase):
     inputs_t = constant_op.constant(inputs)
 
     with self.cached_session(use_gpu=False) as sess:
-      loss = ctc_ops.ctc_loss(
+      loss = _ctc_loss_v2(
           inputs=inputs_t, labels=labels, sequence_length=seq_lens)
       grad = gradients_impl.gradients(loss, [inputs_t])[0]
 
@@ -234,9 +258,9 @@ class CTCLossTest(test.TestCase):
     inputs_t_transposed = constant_op.constant(inputs.transpose(1, 0, 2))
 
     with self.session(use_gpu=False) as sess:
-      loss = ctc_ops.ctc_loss(
+      loss = _ctc_loss_v2(
           inputs=inputs_t, labels=labels, sequence_length=seq_lens)
-      loss_transposed = ctc_ops.ctc_loss(
+      loss_transposed = _ctc_loss_v2(
           inputs=inputs_t_transposed,
           labels=labels,
           sequence_length=seq_lens,
@@ -253,7 +277,7 @@ class CTCLossTest(test.TestCase):
     v = [1.0]
 
     with self.session(use_gpu=False):
-      loss = ctc_ops.ctc_loss(
+      loss = _ctc_loss_v2(
           inputs=inputs_t, labels=labels, sequence_length=seq_lens)
       # Taking ths second gradient should fail, since it is not
       # yet supported.
@@ -272,7 +296,519 @@ class CTCLossTest(test.TestCase):
     with self.session(use_gpu=False) as sess:
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "batch_size must not be 0"):
-        sess.run(ctc_ops.ctc_loss(labels, inputs, sequence_lengths))
+        sess.run(_ctc_loss_v2(labels, inputs, sequence_lengths))
+
+
+class CTCLossTestV2(test.TestCase):
+
+  def testCtcLossV2(self):
+    random_seed.set_random_seed(5)
+
+    batch_size = 8
+    num_labels = 6
+    max_label_length = 5
+    num_frames = 12
+
+    labels = random_ops.random_uniform(
+        [batch_size, max_label_length], minval=1, maxval=num_labels,
+        dtype=dtypes.int64)
+    logits = random_ops.random_uniform([num_frames, batch_size, num_labels])
+
+    label_length = random_ops.random_uniform(
+        [batch_size], minval=2, maxval=max_label_length, dtype=dtypes.int64)
+    label_mask = array_ops.sequence_mask(
+        label_length, maxlen=max_label_length, dtype=label_length.dtype)
+    labels *= label_mask
+    logit_length = [num_frames] * batch_size
+
+    ref_loss = ctc_ops.ctc_loss_v2(
+        labels=labels,
+        logits=logits,
+        label_length=label_length,
+        logit_length=logit_length)
+    ref_grad = gradients_impl.gradients(ref_loss, [logits])
+
+    sparse_labels = ctc_ops.dense_labels_to_sparse(labels, label_length)
+
+    def assert_same_loss_and_grads(loss):
+      with self.cached_session() as sess:
+        self.assertAllClose(*sess.run([loss, ref_loss]))
+        grad = gradients_impl.gradients(loss, [logits])
+        self.assertAllClose(*sess.run([grad, ref_grad]), rtol=2e-06, atol=2e-06)
+
+    assert_same_loss_and_grads(
+        ctc_ops.ctc_loss_v2(
+            labels=sparse_labels,
+            logits=logits,
+            label_length=label_length,
+            logit_length=logit_length,
+            blank_index=0))
+
+  def testCtcLossDenseIsSameAsCtcLoss(self):
+    with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
+      random_seed.set_random_seed(5)
+
+      batch_size = 8
+      num_labels = 6
+      label_length = 5
+      num_frames = 12
+      logits = random_ops.random_uniform([num_frames, batch_size, num_labels])
+      labels = random_ops.random_uniform(
+          [batch_size, label_length], minval=1, maxval=num_labels,
+          dtype=dtypes.int64)
+
+      label_lengths = random_ops.random_uniform(
+          [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64)
+      label_mask = array_ops.sequence_mask(
+          label_lengths, maxlen=label_length, dtype=label_lengths.dtype)
+      labels *= label_mask
+
+      logit_lengths = [num_frames] * batch_size
+
+      ctc_loss = ctc_ops.ctc_loss_dense(
+          labels=labels,
+          logits=logits,
+          label_length=label_lengths,
+          logit_length=logit_lengths)
+      ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0]
+
+      # Shift labels down by one (move blank from 0 to num_labels -1)
+      tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32) - 1
+      tf_nn_ctc_logits = array_ops.concat([
+          logits[:, :, 1:],
+          logits[:, :, 0:1],
+      ], axis=2)
+
+      tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse(
+          tf_ctc_loss_labels, label_lengths)
+
+      tf_nn_ctc_loss = ctc_ops.ctc_loss(
+          labels=tf_ctc_loss_labels,
+          inputs=tf_nn_ctc_logits,
+          sequence_length=logit_lengths,
+          time_major=True)
+      tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0]
+
+      with self.cached_session() as sess:
+        for _ in range(32):
+          self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
+          self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
+                              rtol=2e-06, atol=2e-06)
+
+  def testCtcLossDenseUniqueFastPathIsSameAsCtcLoss(self):
+    random_seed.set_random_seed(5)
+
+    batch_size = 8
+    num_labels = 6
+    label_length = 5
+    num_frames = 12
+    logits = random_ops.random_uniform([num_frames, batch_size, num_labels])
+    labels = random_ops.random_uniform(
+        [batch_size, label_length], minval=1, maxval=num_labels,
+        dtype=dtypes.int64)
+
+    label_lengths = random_ops.random_uniform(
+        [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64)
+    label_mask = array_ops.sequence_mask(
+        label_lengths, maxlen=label_length, dtype=label_lengths.dtype)
+    labels *= label_mask
+
+    logit_lengths = [num_frames] * batch_size
+
+    ctc_loss = ctc_ops.ctc_loss_dense(
+        labels=labels,
+        logits=logits,
+        label_length=label_lengths,
+        logit_length=logit_lengths,
+        unique=ctc_ops.ctc_unique_labels(labels))
+    ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0]
+
+    # Shift labels down by one (move blank from 0 to num_labels -1)
+    tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32) - 1
+    tf_nn_ctc_logits = array_ops.concat([
+        logits[:, :, 1:],
+        logits[:, :, 0:1],
+    ], axis=2)
+
+    tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse(
+        tf_ctc_loss_labels, label_lengths)
+
+    tf_nn_ctc_loss = ctc_ops.ctc_loss(
+        labels=tf_ctc_loss_labels,
+        inputs=tf_nn_ctc_logits,
+        sequence_length=logit_lengths,
+        time_major=True)
+    tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0]
+
+    with self.cached_session() as sess:
+      for _ in range(32):
+        self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
+        self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
+                            rtol=2e-06, atol=2e-06)
+
+  def testCtcLossDenseWithBlankIndexIsSameAsCtcLoss(self):
+    random_seed.set_random_seed(5)
+
+    batch_size = 8
+    num_labels = 6
+    label_length = 5
+    num_frames = 12
+    logits = random_ops.random_uniform([num_frames, batch_size, num_labels])
+    labels = random_ops.random_uniform(
+        [batch_size, label_length], minval=0, maxval=num_labels-1,
+        dtype=dtypes.int64)
+
+    label_lengths = random_ops.random_uniform(
+        [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64)
+    label_mask = array_ops.sequence_mask(
+        label_lengths, maxlen=label_length, dtype=label_lengths.dtype)
+    labels *= label_mask
+
+    logit_lengths = [num_frames] * batch_size
+
+    tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32)
+    tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse(
+        tf_ctc_loss_labels, label_lengths)
+
+    tf_nn_ctc_loss = ctc_ops.ctc_loss(
+        labels=tf_ctc_loss_labels,
+        inputs=logits,
+        sequence_length=logit_lengths,
+        time_major=True)
+    tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0]
+
+    # Shift the blank logits/labels to be somewhere in the middle.
+    blank_index = 2
+    shifted_logits = array_ops.concat([
+        logits[:, :, :blank_index],
+        logits[:, :, -1:],
+        logits[:, :, blank_index:-1],
+    ], axis=2)
+    shifted_labels = array_ops.where(labels < blank_index, labels, labels + 1)
+
+    ctc_loss = ctc_ops.ctc_loss_dense(
+        labels=shifted_labels,
+        logits=shifted_logits,
+        label_length=label_lengths,
+        logit_length=logit_lengths,
+        blank_index=blank_index)
+    ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0]
+
+    with self.cached_session() as sess:
+      for _ in range(32):
+        self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
+        self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
+                            rtol=2e-06, atol=2e-06)
+
+  def testCtcLossDenseWithNegativeBlankIndexIsSameAsCtcLoss(self):
+    with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
+      random_seed.set_random_seed(5)
+
+      batch_size = 8
+      num_labels = 6
+      label_length = 5
+      num_frames = 12
+      logits = random_ops.random_uniform([num_frames, batch_size, num_labels])
+      labels = random_ops.random_uniform(
+          [batch_size, label_length], minval=0, maxval=num_labels-1,
+          dtype=dtypes.int64)
+
+      label_lengths = random_ops.random_uniform(
+          [batch_size], minval=2, maxval=label_length, dtype=dtypes.int64)
+      label_mask = array_ops.sequence_mask(
+          label_lengths, maxlen=label_length, dtype=label_lengths.dtype)
+      labels *= label_mask
+
+      logit_lengths = [num_frames] * batch_size
+
+      ctc_loss = ctc_ops.ctc_loss_dense(
+          labels=labels,
+          logits=logits,
+          label_length=label_lengths,
+          logit_length=logit_lengths,
+          blank_index=-1)
+      ctc_loss_grads = gradients_impl.gradients(ctc_loss, [logits])[0]
+
+      tf_ctc_loss_labels = math_ops.cast(labels, dtypes.int32)
+      tf_ctc_loss_labels = ctc_ops.dense_labels_to_sparse(
+          tf_ctc_loss_labels, label_lengths)
+
+      tf_nn_ctc_loss = ctc_ops.ctc_loss(
+          labels=tf_ctc_loss_labels,
+          inputs=logits,
+          sequence_length=logit_lengths,
+          time_major=True)
+      tf_nn_ctc_grads = gradients_impl.gradients(tf_nn_ctc_loss, [logits])[0]
+
+      with self.cached_session() as sess:
+        for _ in range(32):
+          self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
+          self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
+                              rtol=2e-06, atol=2e-06)
+
+  def testCollapseRepeated(self):
+    collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
+        labels=[[1, 3, 3, 3, 0],
+                [1, 4, 4, 4, 0],
+                [4, 2, 2, 9, 4]],
+        seq_length=[4, 5, 5])
+    self.assertAllEqual(new_seq_lengths, [2, 3, 4])
+    self.assertAllEqual(
+        collapsed,
+        [[1, 3, 0, 0],
+         [1, 4, 0, 0],
+         [4, 2, 9, 4]])
+
+  def testCollapseRepeatedPreservesDtypes(self):
+    collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
+        labels=constant_op.constant(
+            [[1, 3, 3, 3, 0],
+             [1, 4, 4, 4, 0],
+             [4, 2, 2, 9, 4]],
+            dtype=dtypes.int64),
+        seq_length=constant_op.constant([4, 5, 5], dtype=dtypes.int64))
+    self.assertEqual(new_seq_lengths.dtype, dtypes.int64)
+    self.assertEqual(collapsed.dtype, dtypes.int64)
+    self.assertAllEqual(new_seq_lengths, [2, 3, 4])
+    self.assertAllEqual(
+        collapsed,
+        [[1, 3, 0, 0],
+         [1, 4, 0, 0],
+         [4, 2, 9, 4]])
+
+  def testCollapseRepeatedExtraPadding(self):
+    collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
+        labels=[[1, 3, 3, 3, 0, 0, 0],
+                [1, 4, 4, 4, 0, 1, 2],
+                [4, 2, 2, 9, 4, 0, 0]],
+        seq_length=[4, 5, 5])
+    self.assertAllEqual(new_seq_lengths, [2, 3, 4])
+    self.assertAllEqual(
+        collapsed,
+        [[1, 3, 0, 0],
+         [1, 4, 0, 0],
+         [4, 2, 9, 4]])
+
+  def testCollapseRepeatedFrontRepeats(self):
+    collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
+        labels=[[1, 1, 1, 2, 2],
+                [1, 1, 1, 2, 2],
+                [1, 1, 1, 2, 2]],
+        seq_length=[5, 4, 3])
+    self.assertAllEqual(new_seq_lengths, [2, 2, 1])
+    self.assertAllEqual(
+        collapsed,
+        [[1, 2],
+         [1, 2],
+         [1, 0]])
+
+  def testCollapseRepeatedAllLabelsTheSame(self):
+    collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
+        labels=[[1, 1, 1, 1, 1],
+                [1, 1, 1, 1, 1],
+                [1, 1, 1, 1, 1]],
+        seq_length=[4, 5, 1])
+    self.assertAllEqual(new_seq_lengths, [1, 1, 1])
+    self.assertAllEqual(
+        collapsed,
+        [[1],
+         [1],
+         [1]])
+
+  def testDenseSequencesToSparse(self):
+    labels = [[1, 3, 3, 3, 0],
+              [1, 4, 4, 4, 0],
+              [4, 2, 2, 9, 4]]
+    length = [4, 5, 5]
+    sparse = ctc_ops.dense_labels_to_sparse(labels, length)
+    new_dense = sparse_ops.sparse_tensor_to_dense(sparse)
+
+    self.assertAllEqual(labels, new_dense)
+
+    padded_labels = [[1, 3, 3, 3, 0, 0, 0, 0],
+                     [1, 4, 4, 4, 0, 0, 0, 0],
+                     [4, 2, 2, 9, 4, 0, 0, 0]]
+    length = [4, 5, 5]
+    sparse = ctc_ops.dense_labels_to_sparse(padded_labels, length)
+    padded_dense = sparse_ops.sparse_tensor_to_dense(sparse)
+
+    self.assertAllEqual(padded_dense, new_dense)
+
+  def testUnique(self):
+    labels = [
+        [3, 4, 4, 3],
+        [1, 1, 1, 0],
+    ]
+    unique, idx = ctc_ops.ctc_unique_labels(labels)
+    self.assertAllEqual([
+        [3, 4, 0, 0],
+        [1, 0, 0, 0],
+    ], unique)
+    self.assertAllEqual([
+        [0, 1, 1, 0],
+        [0, 0, 0, 1],
+    ], idx)
+
+  def testSumStates(self):
+    idx = [
+        [0, 1, 0, 1],
+        [0, 0, 0, 1],
+    ]
+    states = math_ops.log([
+        [[1.0, 2.0, 3.0, 4.0],
+         [5.0, 6.0, 7.0, 8.0]],
+        [[0.1, 0.2, 0.3, 0.4],
+         [0.5, 0.6, 0.7, 0.8]],
+    ])
+    sum_of_states = math_ops.exp(ctc_ops._sum_states(idx, states))
+    self.assertAllClose([
+        [[4.0, 6.0, 0.0, 0.0],
+         [18.0, 8.0, 0.0, 0.0]],
+        [[0.4, 0.6, 0.0, 0.0],
+         [1.8, 0.8, 0.0, 0.0]]
+    ], sum_of_states)
+
+  def testStateToOlabel(self):
+    labels = [
+        [3, 4, 3, 4],
+        [1, 1, 1, 0],
+    ]
+    num_labels = 8
+
+    # 3 frames, 2 batch, 10 states (5 label, 5 blank).
+    states = [
+        [[0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.20],
+         [0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.30]],
+        [[1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0],
+         [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0]],
+        [[11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0],
+         [21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0]],
+    ]
+    labels = ops.convert_to_tensor(labels)
+    states = math_ops.log(states)
+    olabel = ctc_ops._state_to_olabel(labels, num_labels, states)
+    olabel = math_ops.exp(olabel)
+    blank = olabel[:, :, 0]
+    self.assertAllClose(blank, [
+        [0.16 + 0.17 + 0.18 + 0.19 + 0.20,
+         0.26 + 0.27 + 0.28 + 0.29 + 0.30],
+        [1.6 + 1.7 + 1.8 + 1.9 + 2.0,
+         2.6 + 2.7 + 2.8 + 2.9 + 3.0],
+        [16.0 + 17.0 + 18.0 + 19.0 + 20.0,
+         26.0 + 27.0 + 28.0 + 29.0 + 30.0]
+    ])
+    self.assertAllClose(olabel[:, :, 1:], [
+        [[0.0, 0.0, 0.12 + 0.14, 0.13 + 0.15, 0.0, 0.0, 0.0],
+         [0.22 + 0.23 + 0.24, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+        [[0.0, 0.0, 1.2 + 1.4, 1.3 + 1.5, 0.0, 0.0, 0.0],
+         [2.2 + 2.3 + 2.4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+        [[0.0, 0.0, 12.0 + 14.0, 13.0 + 15.0, 0.0, 0.0, 0.0],
+         [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+    ])
+
+  def testStateToOlabelUnique(self):
+    labels = [
+        [3, 4, 3, 4],
+        [1, 1, 1, 0],
+    ]
+    num_labels = 8
+
+    # 3 frames, 2 batch, 10 states (5 label, 5 blank).
+    states = [
+        [[0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.20],
+         [0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.30]],
+        [[1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0],
+         [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0]],
+        [[11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0],
+         [21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0]],
+    ]
+    labels = ops.convert_to_tensor(labels)
+    states = math_ops.log(states)
+    olabel = ctc_ops._state_to_olabel_unique(
+        labels, num_labels, states, ctc_ops.ctc_unique_labels(labels))
+    olabel = math_ops.exp(olabel)
+    blank = olabel[:, :, 0]
+    self.assertAllClose(blank, [
+        [0.16 + 0.17 + 0.18 + 0.19 + 0.20,
+         0.26 + 0.27 + 0.28 + 0.29 + 0.30],
+        [1.6 + 1.7 + 1.8 + 1.9 + 2.0,
+         2.6 + 2.7 + 2.8 + 2.9 + 3.0],
+        [16.0 + 17.0 + 18.0 + 19.0 + 20.0,
+         26.0 + 27.0 + 28.0 + 29.0 + 30.0]])
+    self.assertAllClose(olabel[:, :, 1:], [
+        [[0.0, 0.0, 0.12 + 0.14, 0.13 + 0.15, 0.0, 0.0, 0.0],
+         [0.22 + 0.23 + 0.24, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+        [[0.0, 0.0, 1.2 + 1.4, 1.3 + 1.5, 0.0, 0.0, 0.0],
+         [2.2 + 2.3 + 2.4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+        [[0.0, 0.0, 12.0 + 14.0, 13.0 + 15.0, 0.0, 0.0, 0.0],
+         [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
+    ])
+
+  def testScan(self):
+    with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
+      out = ctc_ops._scan(
+          lambda accum, elem: accum + elem,
+          constant_op.constant([1.0, 2.0, 3.0]), 23.0)
+      self.assertAllEqual([24.0, 26.0, 29.0], out)
+
+      out = ctc_ops._scan(
+          lambda a, e: a + e,
+          constant_op.constant([1.0, 2.0, 3.0]), 23.0,
+          inclusive=True)
+      self.assertAllEqual([23.0, 24.0, 26.0, 29.0], out)
+
+      out = ctc_ops._scan(
+          lambda a, e: a + e,
+          constant_op.constant([1.0, 2.0, 3.0]), 23.0,
+          reverse=True)
+      self.assertAllEqual([29.0, 28.0, 26.0], out)
+
+      out = ctc_ops._scan(
+          lambda a, e: a + e,
+          constant_op.constant([1.0, 2.0, 3.0]), 23.0,
+          reverse=True,
+          inclusive=True)
+      self.assertAllEqual([29.0, 28.0, 26.0, 23.0], out)
+
+      out = ctc_ops._scan(
+          lambda a, e: a + e,
+          constant_op.constant([[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]]),
+          constant_op.constant([23.0, 24.0]))
+      self.assertAllEqual([[23.0, 25.0], [25.0, 28.0], [29.0, 33.0]], out)
+
+  def testScanCapturesVariables(self):
+    with self.cached_session() as sess:
+      x = random_ops.random_uniform([])
+      fn = lambda accum, elem: accum + x * elem
+      out = ctc_ops._scan(fn, constant_op.constant([0.0, 1.0, 2.0]), 23.0)
+      self.assertAllEqual(*sess.run([
+          [23.0 + x * 0.0, 23.0 + x * 1.0, 23.0 + x * 3.0], out
+      ]))
+
+  def testScanMultipleAccumulators(self):
+    with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
+      def fn(accum, elem):
+        accum_a, accum_b = accum
+        return accum_a + elem, accum_b * elem
+      out = ctc_ops._scan(
+          fn, constant_op.constant([1.0, 2.0, 3.0]),
+          (23.0, constant_op.constant([1.0, 2.0])))
+      a, b = out
+      self.assertAllEqual([24.0, 26.0, 29.0], a)
+      self.assertAllEqual([[1.0, 2.0], [2.0, 4.0], [6.0, 12.0]], b)
+
+  def testScanMultipleElements(self):
+    with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
+      def fn(accum, elem):
+        elem_a, elem_b = elem
+        return accum + (elem_a * elem_b)
+      elems_a = constant_op.constant([1.0, 2.0, 3.0])
+      elems_b = constant_op.constant([[1.0, 2.0], [2.0, 3.0], [3.0, 4.0]])
+      out = ctc_ops._scan(
+          fn, (elems_a, elems_b),
+          initial=constant_op.constant([0.0, 0.0]))
+      self.assertAllEqual(
+          [[1.0, 2.0], [5.0, 8.0], [14.0, 20.0]], out)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py
index e1071afd8e..3a7eb9355a 100644
--- a/tensorflow/python/ops/ctc_ops.py
+++ b/tensorflow/python/ops/ctc_ops.py
@@ -19,17 +19,27 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_ctc_ops
+from tensorflow.python.ops import inplace_ops
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops.nn_grad import _BroadcastMul
+from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
 
 # pylint: disable=protected-access, invalid-name
-@tf_export("nn.ctc_loss")
+@tf_export(v1=["nn.ctc_loss"])
 def ctc_loss(labels, inputs, sequence_length,
              preprocess_collapse_repeated=False,
              ctc_merge_repeated=True,
@@ -336,6 +346,785 @@ def ctc_beam_search_decoder_v2(inputs, sequence_length, beam_width=100,
 
 
 ops.NotDifferentiable("CTCGreedyDecoder")
+ops.NotDifferentiable("CTCBeamSearchDecoder")
 
 
-ops.NotDifferentiable("CTCBeamSearchDecoder")
+def _ctc_state_trans(label_seq):
+  """Compute CTC alignment model transition matrix.
+
+  Args:
+    label_seq: tensor of shape [batch_size, max_seq_length]
+
+  Returns:
+    tensor of shape [batch_size, states, states] with a state transition matrix
+    computed for each sequence of the batch.
+  """
+
+  with ops.name_scope("ctc_state_trans"):
+    label_seq = ops.convert_to_tensor(label_seq, name="label_seq")
+    batch_size = _get_dim(label_seq, 0)
+    num_labels = _get_dim(label_seq, 1)
+
+    num_label_states = num_labels + 1
+    num_states = 2 * num_label_states
+
+    label_states = math_ops.range(num_label_states)
+    blank_states = label_states + num_label_states
+
+    # Start state to first label.
+    start_to_label = [[1, 0]]
+
+    # Blank to label transitions.
+    blank_to_label = array_ops.stack([label_states[1:], blank_states[:-1]], 1)
+
+    # Label to blank transitions.
+    label_to_blank = array_ops.stack([blank_states, label_states], 1)
+
+    # Scatter transitions that don't depend on sequence.
+    indices = array_ops.concat(
+        [start_to_label, blank_to_label, label_to_blank], 0)
+    values = array_ops.ones([_get_dim(indices, 0)])
+    trans = array_ops.scatter_nd(
+        indices, values, shape=[num_states, num_states])
+    trans += linalg_ops.eye(num_states)  # Self-loops.
+
+    # Label to label transitions. Disallow transitions between repeated labels
+    # with no blank state in between.
+    batch_idx = array_ops.zeros_like(label_states[2:])
+    indices = array_ops.stack(
+        [batch_idx, label_states[2:], label_states[1:-1]], 1)
+    indices = array_ops.tile(
+        array_ops.expand_dims(indices, 0), [batch_size, 1, 1])
+    batch_idx = array_ops.expand_dims(math_ops.range(batch_size), 1) * [1, 0, 0]
+    indices += array_ops.expand_dims(batch_idx, 1)
+    repeats = math_ops.equal(label_seq[:, :-1], label_seq[:, 1:])
+    values = 1.0 - math_ops.cast(repeats, dtypes.float32)
+    batched_shape = [batch_size, num_states, num_states]
+    label_to_label = array_ops.scatter_nd(indices, values, batched_shape)
+
+    return array_ops.expand_dims(trans, 0) + label_to_label
+
+
+def ctc_state_log_probs(seq_lengths, max_seq_length):
+  """Computes CTC alignment initial and final state log probabilities.
+
+  Create the initial/final state values directly as log values to avoid
+  having to take a float64 log on tpu (which does not exist).
+
+  Args:
+    seq_lengths: int tensor of shape [batch_size], seq lengths in the batch.
+    max_seq_length: int, max sequence length possible.
+
+  Returns:
+    initial_state_log_probs, final_state_log_probs
+  """
+
+  batch_size = _get_dim(seq_lengths, 0)
+  num_label_states = max_seq_length + 1
+  num_duration_states = 2
+  num_states = num_duration_states * num_label_states
+  log_0 = math_ops.cast(
+      math_ops.log(math_ops.cast(0, dtypes.float64) + 1e-307),
+      dtypes.float32)
+
+  initial_state_log_probs = array_ops.one_hot(
+      indices=array_ops.zeros([batch_size], dtype=dtypes.int32),
+      depth=num_states,
+      on_value=0.0,
+      off_value=log_0, axis=1)
+
+  label_final_state_mask = array_ops.one_hot(
+      seq_lengths, depth=num_label_states, axis=0)
+  duration_final_state_mask = array_ops.ones(
+      [num_duration_states, 1, batch_size])
+  final_state_mask = duration_final_state_mask * label_final_state_mask
+  final_state_log_probs = (1.0 - final_state_mask) * log_0
+  final_state_log_probs = array_ops.reshape(
+      final_state_log_probs, [num_states, batch_size])
+
+  return initial_state_log_probs, array_ops.transpose(final_state_log_probs)
+
+
+def _ilabel_to_state(labels, num_labels, ilabel_log_probs):
+  """Project ilabel log probs to state log probs."""
+
+  num_label_states = _get_dim(labels, 1)
+  blank = ilabel_log_probs[:, :, :1]
+  blank = array_ops.tile(blank, [1, 1, num_label_states + 1])
+  one_hot = array_ops.one_hot(labels, depth=num_labels)
+  one_hot = array_ops.expand_dims(one_hot, axis=0)
+  ilabel_log_probs = array_ops.expand_dims(ilabel_log_probs, axis=2)
+  state_log_probs = math_ops.reduce_sum(ilabel_log_probs * one_hot, axis=3)
+  state_log_probs = array_ops.concat([state_log_probs, blank], axis=2)
+  return array_ops.pad(
+      state_log_probs, [[0, 0], [0, 0], [1, 0]],
+      constant_values=math_ops.log(0.0))
+
+
+def _state_to_olabel(labels, num_labels, states):
+  """Sum state log probs to ilabel log probs."""
+
+  num_label_states = _get_dim(labels, 1) + 1
+  label_states = states[:, :, 1:num_label_states]
+  blank_states = states[:, :, num_label_states:]
+  one_hot = array_ops.one_hot(
+      labels - 1, depth=(num_labels - 1),
+      on_value=0.0, off_value=math_ops.log(0.0))
+  one_hot = array_ops.expand_dims(one_hot, axis=0)
+  label_states = array_ops.expand_dims(label_states, axis=3)
+  label_olabels = math_ops.reduce_logsumexp(label_states + one_hot, axis=2)
+  blank_olabels = math_ops.reduce_logsumexp(
+      blank_states, axis=2, keepdims=True)
+  return array_ops.concat([blank_olabels, label_olabels], axis=-1)
+
+
+# pylint: disable=redefined-outer-name
+def _state_to_olabel_unique(labels, num_labels, states, unique):
+  """Sum state log probs to ilabel log probs using unique label indices."""
+
+  num_label_states = _get_dim(labels, 1) + 1
+  label_states = states[:, :, 1:num_label_states]
+  blank_states = states[:, :, num_label_states:]
+
+  unique_y, unique_idx = unique
+  mul_reduce = _sum_states(unique_idx, label_states)
+
+  num_frames = states.shape[0]
+  batch_size = states.shape[1]
+  num_states = num_label_states - 1
+  batch_state_major = array_ops.transpose(mul_reduce, perm=[1, 2, 0])
+  batch_state_major = array_ops.reshape(
+      batch_state_major, [batch_size * num_states, num_frames])
+  batch_offset = math_ops.range(batch_size, dtype=unique_y.dtype) * num_labels
+  indices = unique_y + array_ops.expand_dims(batch_offset, axis=-1)
+  indices = array_ops.reshape(indices, [-1, 1])
+  scatter = array_ops.scatter_nd(
+      indices=indices,
+      updates=batch_state_major,
+      shape=[batch_size * num_labels, num_frames])
+  scatter = array_ops.reshape(scatter, [batch_size, num_labels, num_frames])
+  scatter = array_ops.where(
+      math_ops.equal(scatter, 0.0),
+      array_ops.fill(array_ops.shape(scatter), math_ops.log(0.0)),
+      scatter)
+  label_olabels = array_ops.transpose(scatter, [2, 0, 1])
+  label_olabels = label_olabels[:, :, 1:]
+
+  blank_olabels = math_ops.reduce_logsumexp(
+      blank_states, axis=2, keepdims=True)
+
+  return array_ops.concat([blank_olabels, label_olabels], axis=-1)
+
+
+def ctc_loss_and_grad(logits, labels, label_length, logit_length, unique=None):
+  """Computes the CTC loss and gradients.
+
+  Most users will want fwd_bwd.ctc_loss
+
+  This function returns the computed gradient, it does not have a gradient
+  of its own defined.
+
+  Args:
+    logits: tensor of shape [frames, batch_size, num_labels]
+    labels: tensor of shape [batch_size, max_label_seq_length]
+    label_length: tensor of shape [batch_size]
+      Length of reference label sequence in labels.
+    logit_length: tensor of shape [batch_size]
+      Length of input sequence in logits.
+    unique: (optional) unique label indices as computed by unique(labels)
+      If supplied, enables an implementation that is faster and more memory
+      efficient on TPU.
+
+  Returns:
+    loss: tensor of shape [batch_size]
+    gradient: tensor of shape [frames, batch_size, num_labels]
+  """
+
+  num_labels = _get_dim(logits, 2)
+  max_label_seq_length = _get_dim(labels, 1)
+
+  ilabel_log_probs = nn_ops.log_softmax(logits)
+  state_log_probs = _ilabel_to_state(labels, num_labels, ilabel_log_probs)
+  state_trans_probs = _ctc_state_trans(labels)
+  initial_state_log_probs, final_state_log_probs = ctc_state_log_probs(
+      label_length, max_label_seq_length)
+  fwd_bwd_log_probs, log_likelihood = _forward_backward_log(
+      state_trans_log_probs=math_ops.log(state_trans_probs),
+      initial_state_log_probs=initial_state_log_probs,
+      final_state_log_probs=final_state_log_probs,
+      observed_log_probs=state_log_probs,
+      sequence_length=logit_length)
+
+  if unique:
+    olabel_log_probs = _state_to_olabel_unique(
+        labels, num_labels, fwd_bwd_log_probs, unique)
+  else:
+    olabel_log_probs = _state_to_olabel(labels, num_labels, fwd_bwd_log_probs)
+
+  grad = math_ops.exp(ilabel_log_probs) - math_ops.exp(olabel_log_probs)
+  loss = -log_likelihood
+  return loss, grad
+
+
+def _ctc_loss_grad(op, grad_loss, _):
+  grad = op.outputs[1]
+  grad = [array_ops.reshape(grad_loss, [1, -1, 1]) * grad]
+  grad += [None] * (len(op.inputs) - len(grad))
+  return grad
+
+
+def _ctc_loss_shape(op):
+  return [op.inputs[2].get_shape(), op.inputs[0].get_shape()]
+
+
+@tf_export("nn.ctc_loss", v1=["nn.ctc_loss_v2"])
+def ctc_loss_v2(labels, logits, label_length, logit_length,
+                logits_time_major=True, unique=None,
+                blank_index=None, name=None):
+  """Computes CTC (Connectionist Temporal Classification) loss.
+
+  This op implements the CTC loss as presented in the article:
+
+  [A. Graves, S. Fernandez, F. Gomez, J. Schmidhuber.
+  Connectionist Temporal Classification: Labeling Unsegmented Sequence Data
+  with Recurrent Neural Networks. ICML 2006, Pittsburgh, USA,
+  pp. 369-376.](http://www.cs.toronto.edu/~graves/icml_2006.pdf)
+
+  Notes:
+      - Same as the "Classic CTC" in TensorFlow 1.x's tf.nn.ctc_loss setting of
+        preprocess_collapse_repeated=False, ctc_merge_repeated=True
+      - Labels may be supplied as either a dense, zero-padded tensor with a
+        vector of label sequence lengths OR as a SparseTensor.
+      - On TPU and GPU:
+          - Only dense padded labels are supported.
+      - On CPU:
+          - Caller may use SparseTensor or dense padded labels but calling with
+            a SparseTensor will be significantly faster.
+      - Default blank label is 0 rather num_classes - 1, unless overridden by
+        blank_index.
+
+  Args:
+    labels: tensor of shape [batch_size, max_label_seq_length] or SparseTensor
+    logits: tensor of shape [frames, batch_size, num_labels],
+      if logits_time_major == False, shape is [batch_size, frames, num_labels].
+    label_length: tensor of shape [batch_size], None if labels is SparseTensor
+      Length of reference label sequence in labels.
+    logit_length: tensor of shape [batch_size]
+      Length of input sequence in logits.
+    logits_time_major: (optional) If True (default), logits is shaped
+      [time, batch, logits]. If False, shape is [batch, time, logits]
+    unique: (optional) Unique label indices as computed by
+      ctc_unique_labels(labels).  If supplied, enable a faster, memory
+      efficient implementation on TPU.
+    blank_index: (optional) Set the class index to use for the blank label.
+      Negative values will start from num_classes, ie, -1 will reproduce the
+      ctc_loss behavior of using num_classes - 1 for the blank symbol.
+      There is some memory/performance overhead to switching from the default
+      of 0 as an additional shifted copy of the logits may be created.
+    name: A name for this `Op`. Defaults to "ctc_loss_dense".
+
+  Returns:
+    loss: tensor of shape [batch_size], negative log probabilities.
+  """
+  if isinstance(labels, sparse_tensor.SparseTensor):
+    if blank_index is None:
+      raise ValueError(
+          "blank_index must be given when using SparseTensor labels.")
+
+    if blank_index < 0:
+      blank_index += _get_dim(logits, 2)
+
+    if blank_index != _get_dim(logits, 2) - 1:
+      logits = array_ops.concat([
+          logits[:, :, :blank_index],
+          logits[:, :, blank_index+1:],
+          logits[:, :, blank_index:blank_index+1],
+      ], axis=2)
+      labels = sparse_tensor.SparseTensor(
+          labels.indices,
+          array_ops.where(labels.values < blank_index,
+                          labels.values,
+                          labels.values - 1),
+          labels.dense_shape)
+
+    return ctc_loss(labels=labels,
+                    inputs=logits,
+                    sequence_length=logit_length,
+                    time_major=logits_time_major)
+
+  if blank_index is None:
+    blank_index = 0
+
+  return ctc_loss_dense(labels=labels,
+                        logits=logits,
+                        label_length=label_length,
+                        logit_length=logit_length,
+                        logits_time_major=logits_time_major,
+                        unique=unique,
+                        blank_index=blank_index,
+                        name=name)
+
+
+def ctc_loss_dense(labels, logits, label_length, logit_length,
+                   logits_time_major=True, unique=None,
+                   blank_index=0, name=None):
+  """Computes CTC (Connectionist Temporal Classification) loss.
+
+  This op implements the CTC loss as presented in the article:
+
+  [A. Graves, S. Fernandez, F. Gomez, J. Schmidhuber.
+  Connectionist Temporal Classification: Labeling Unsegmented Sequence Data
+  with Recurrent Neural Networks. ICML 2006, Pittsburgh, USA,
+  pp. 369-376.](http://www.cs.toronto.edu/~graves/icml_2006.pdf)
+
+  Using the batched forward backward algorithm described in:
+
+  [Sim, K. C., Narayanan, A., Bagby, T., Sainath, T. N., & Bacchiani, M.
+  Improving the efficiency of forward-backward algorithm using batched
+    computation in TensorFlow.
+  Automatic Speech Recognition and Understanding Workshop (ASRU),
+    2017 IEEE (pp. 258-264).
+  ](https://ieeexplore.ieee.org/iel7/8260578/8268903/08268944.pdf)
+
+  Notes:
+    Significant differences from tf.nn.ctc_loss:
+      Supports GPU and TPU (tf.nn.ctc_loss supports CPU only):
+        For batched operations, GPU and TPU are significantly faster than using
+        ctc_loss on CPU.
+        This implementation runs on CPU, but significantly slower than ctc_loss.
+      Blank label is 0 rather num_classes - 1, unless overridden by blank_index.
+      Logits and labels are dense arrays with padding rather than SparseTensor.
+      The only mode supported is the same as:
+        preprocess_collapse_repeated=False, ctc_merge_repeated=True
+        To collapse labels, the caller can preprocess label sequence first.
+
+    The dense implementation supports both CPU, GPU and TPU. A fast path is
+    provided that significantly improves memory use for large vocabulary if the
+    caller preprocesses label sequences to get unique label indices on the CPU
+    (eg. in the data input pipeline) using ctc_ops.unique and simplies this in
+    the optional "unique" kwarg. This is especially useful for TPU and GPU but
+    also works with if used on CPU.
+
+  Args:
+    labels: tensor of shape [batch_size, max_label_seq_length]
+    logits: tensor of shape [frames, batch_size, num_labels],
+      if logits_time_major == False, shape is [batch_size, frames, num_labels].
+    label_length: tensor of shape [batch_size]
+      Length of reference label sequence in labels.
+    logit_length: tensor of shape [batch_size]
+      Length of input sequence in logits.
+    logits_time_major: (optional) If True (default), logits is shaped
+      [time, batch, logits]. If False, shape is [batch, time, logits]
+    unique: (optional) Unique label indices as computed by unique(labels).
+      If supplied, enable a faster, memory efficient implementation on TPU.
+    blank_index: (optional) Set the class index to use for the blank label.
+      Negative values will start from num_classes, ie, -1 will reproduce the
+      ctc_loss behavior of using num_classes - 1 for the blank symbol.
+      There is some memory/performance overhead to switching from the default
+      of 0 as an additional shifted copy of the logits may be created.
+    name: A name for this `Op`. Defaults to "ctc_loss_dense".
+
+  Returns:
+    loss: tensor of shape [batch_size], negative log probabilities.
+  """
+
+  with ops.name_scope(name, "ctc_loss_dense",
+                      [logits, labels, label_length, logit_length]):
+    logits = ops.convert_to_tensor(logits, name="logits")
+    labels = ops.convert_to_tensor(labels, name="labels")
+    label_length = ops.convert_to_tensor(label_length, name="label_length")
+    logit_length = ops.convert_to_tensor(logit_length, name="logit_length")
+
+    if not logits_time_major:
+      logits = array_ops.transpose(logits, perm=[1, 0, 2])
+
+    if blank_index != 0:
+      if blank_index < 0:
+        blank_index += _get_dim(logits, 2)
+      logits = array_ops.concat([
+          logits[:, :, blank_index:blank_index+1],
+          logits[:, :, :blank_index],
+          logits[:, :, blank_index+1:],
+      ], axis=2)
+      labels = array_ops.where(labels < blank_index, labels + 1, labels)
+
+    args = [logits, labels, label_length, logit_length]
+
+    if unique:
+      unique_y, unique_idx = unique
+      args.extend([unique_y, unique_idx])
+
+    # TODO(tombagby): Update to tfe.defun
+    @function.Defun(*[x.dtype for x in args],
+                    python_grad_func=_ctc_loss_grad,
+                    shape_func=_ctc_loss_shape)
+    def compute_ctc_loss(logits_t, labels_t, label_length_t, logit_length_t,
+                         *unique_t):
+      """Compute CTC loss."""
+      logits_t.set_shape(logits.shape)
+      labels_t.set_shape(labels.shape)
+      label_length_t.set_shape(label_length.shape)
+      logit_length_t.set_shape(logit_length.shape)
+      kwargs = dict(
+          logits=logits_t,
+          labels=labels_t,
+          label_length=label_length_t,
+          logit_length=logit_length_t)
+      if unique_t:
+        kwargs["unique"] = unique_t
+      return ctc_loss_and_grad(**kwargs)
+
+    return compute_ctc_loss(*args)[0]
+
+
+@tf_export("nn.collapse_repeated")
+def collapse_repeated(labels, seq_length, name=None):
+  """Merge repeated labels into single labels.
+
+  Args:
+    labels: Tensor of shape (batch, max value in seq_length)
+    seq_length: Tensor of shape (batch), sequence length of each batch element.
+    name: A name for this `Op`. Defaults to "collapse_repeated_labels".
+
+  Returns:
+    tuple of Tensor of shape (batch, max_seq_length) with repeated labels
+    collapsed and padded to max_seq_length, eg:
+        [[A, A, B, B, A],
+         [A, B, C, D, E]] => [[A, B, A, 0, 0],
+                              [A, B, C, D, E]]
+    and int tensor of shape [batch] with new sequence lengths.
+  """
+
+  with ops.name_scope(name, "collapse_repeated_labels",
+                      [labels, seq_length]):
+    labels = ops.convert_to_tensor(labels, name="labels")
+    seq_length = ops.convert_to_tensor(seq_length, name="seq_length")
+
+    # Mask labels that don't equal previous label.
+    label_mask = array_ops.concat(
+        [array_ops.ones_like(labels[:, :1], dtypes.bool),
+         math_ops.not_equal(labels[:, 1:], labels[:, :-1])],
+        axis=1)
+
+    # Filter labels that aren't in the original sequence.
+    maxlen = _get_dim(labels, 1)
+    seq_mask = array_ops.sequence_mask(seq_length, maxlen=maxlen)
+    label_mask = math_ops.logical_and(label_mask, seq_mask)
+
+    # Count masks for new sequence lengths.
+    new_seq_len = math_ops.reduce_sum(
+        math_ops.cast(label_mask, dtypes.int32), axis=1)
+
+    # Mask indexes based on sequence length mask.
+    new_maxlen = math_ops.reduce_max(new_seq_len)
+    idx_mask = array_ops.sequence_mask(new_seq_len, maxlen=new_maxlen)
+
+    # Flatten everything and mask out labels to keep and sparse indices.
+    flat_labels = array_ops.reshape(labels, [-1])
+    flat_label_mask = array_ops.reshape(label_mask, [-1])
+    flat_idx_mask = array_ops.reshape(idx_mask, [-1])
+    idx = math_ops.range(_get_dim(flat_idx_mask, 0))
+
+    # Scatter to flat shape.
+    flat = array_ops.scatter_nd(
+        indices=array_ops.expand_dims(
+            array_ops.boolean_mask(idx, flat_idx_mask), axis=1),
+        updates=array_ops.boolean_mask(flat_labels, flat_label_mask),
+        shape=array_ops.shape(flat_idx_mask))
+
+    # Reshape back to square batch.
+    batch_size = _get_dim(labels, 0)
+    new_shape = [batch_size, new_maxlen]
+    return (array_ops.reshape(flat, new_shape),
+            math_ops.cast(new_seq_len, seq_length.dtype))
+
+
+def dense_labels_to_sparse(dense, length):
+  """Convert dense labels with sequence lengths to sparse tensor.
+
+  Args:
+    dense: tensor of shape [batch, max_length]
+    length: int tensor of shape [batch]
+      The length of each sequence in dense.
+
+  Returns:
+    tf.SparseTensor with values only for the valid elements of sequences.
+  """
+
+  flat_values = array_ops.reshape(dense, [-1])
+  flat_indices = math_ops.range(
+      array_ops.shape(flat_values, out_type=dtypes.int64)[0])
+  mask = array_ops.sequence_mask(length, maxlen=array_ops.shape(dense)[1])
+  flat_mask = array_ops.reshape(mask, [-1])
+  indices = array_ops.expand_dims(
+      array_ops.boolean_mask(flat_indices, flat_mask), 1)
+  values = array_ops.boolean_mask(flat_values, flat_mask)
+  sparse = sparse_tensor.SparseTensor(
+      indices=indices, values=math_ops.cast(values, dtypes.int32),
+      dense_shape=array_ops.shape(flat_values, out_type=dtypes.int64))
+  reshaped = sparse_ops.sparse_reshape(sparse, array_ops.shape(dense))
+  max_length = math_ops.reduce_max(length)
+  return sparse_tensor.SparseTensor(
+      indices=reshaped.indices,
+      values=reshaped.values,
+      dense_shape=[
+          math_ops.cast(reshaped.dense_shape[0], dtypes.int64),
+          math_ops.cast(max_length, dtypes.int64)])
+
+
+@tf_export("nn.ctc_unique_labels")
+def ctc_unique_labels(labels, name=None):
+  """Get unique labels and indices for batched labels for tf.nn.ctc_loss.
+
+  For use with tf.nn.ctc_loss_v2 optional argument `unique`: This op can be
+  used to preprocess labels in input pipeline to for better speed/memory use
+  computing the ctc loss on TPU.
+
+  Example:
+    ctc_unique_labels([[3, 4, 4, 3]]) ->
+      unique labels padded with 0: [[3, 4, 0, 0]]
+      indices of original labels in unique: [0, 1, 1, 0]
+
+  Args:
+    labels: tensor of shape [batch_size, max_label_length] padded with 0.
+    name: A name for this `Op`. Defaults to "ctc_unique_labels".
+
+  Returns:
+    tuple of
+      - unique labels, tensor of shape `[batch_size, max_label_length]`
+      - indices into unique labels, shape `[batch_size, max_label_length]`
+  """
+
+  with ops.name_scope(name, "ctc_unique_labels", [labels]):
+    labels = ops.convert_to_tensor(labels, name="labels")
+    def _unique(x):
+      u = array_ops.unique(x)
+      y = array_ops.pad(
+          u.y, [[0, _get_dim(u.idx, 0) - _get_dim(u.y, 0)]])
+      y = math_ops.cast(y, dtypes.int64)
+      return [y, u.idx]
+    return functional_ops.map_fn(
+        _unique, labels, dtype=[dtypes.int64, dtypes.int32])
+
+
+def _sum_states(idx, states):
+  """Take logsumexp for each unique state out of all label states.
+
+  Args:
+    idx: tensor of shape [batch, label_length]
+      For each sequence, indices into a set of unique labels as computed by
+      calling unique.
+    states: tensor of shape [frames, batch, label_length]
+      Log probabilities for each label state.
+
+  Returns:
+    tensor of shape [frames, batch_size, label_length], log probabilites summed
+      for each unique label of the sequence.
+  """
+
+  with ops.name_scope("sum_states"):
+    idx = ops.convert_to_tensor(idx, name="idx")
+    num_states = _get_dim(states, 2)
+    states = array_ops.expand_dims(states, axis=2)
+    one_hot = array_ops.one_hot(
+        idx, depth=num_states, on_value=0.0, off_value=math_ops.log(0.0),
+        axis=1)
+    return math_ops.reduce_logsumexp(states + one_hot, axis=-1)
+
+
+def _forward_backward_log(state_trans_log_probs, initial_state_log_probs,
+                          final_state_log_probs, observed_log_probs,
+                          sequence_length):
+  """Forward-backward algorithm computed in log domain.
+
+  Args:
+    state_trans_log_probs: tensor of shape [states, states] or
+      if different transition matrix per batch [batch_size, states, states]
+    initial_state_log_probs: tensor of shape [batch_size, states]
+    final_state_log_probs: tensor of shape [batch_size, states]
+    observed_log_probs: tensor of shape [frames, batch_size, states]
+    sequence_length: tensor of shape [batch_size]
+
+  Returns:
+    forward backward log probabilites: tensor of shape [frames, batch, states]
+    log_likelihood: tensor of shape [batch_size]
+
+  Raises:
+    ValueError: If state_trans_log_probs has unknown or incorrect rank.
+  """
+
+  if state_trans_log_probs.shape.ndims == 2:
+    perm = [1, 0]
+  elif state_trans_log_probs.shape.ndims == 3:
+    perm = [0, 2, 1]
+  else:
+    raise ValueError(
+        "state_trans_log_probs rank must be known and == 2 or 3, is: %s" %
+        state_trans_log_probs.shape.ndims)
+
+  bwd_state_trans_log_probs = array_ops.transpose(state_trans_log_probs, perm)
+  batch_size = _get_dim(observed_log_probs, 1)
+
+  def _forward(state_log_prob, obs_log_prob):
+    state_log_prob = array_ops.expand_dims(state_log_prob, axis=1)  # Broadcast.
+    state_log_prob += state_trans_log_probs
+    state_log_prob = math_ops.reduce_logsumexp(state_log_prob, axis=-1)
+    state_log_prob += obs_log_prob
+    log_prob_sum = math_ops.reduce_logsumexp(
+        state_log_prob, axis=-1, keepdims=True)
+    state_log_prob -= log_prob_sum
+    return state_log_prob
+
+  fwd = _scan(_forward, observed_log_probs, initial_state_log_probs,
+              inclusive=True)
+
+  def _backward(accs, elems):
+    """Calculate log probs and cumulative sum masked for sequence length."""
+    state_log_prob, cum_log_sum = accs
+    obs_log_prob, mask = elems
+    state_log_prob += obs_log_prob
+    state_log_prob = array_ops.expand_dims(state_log_prob, axis=1)  # Broadcast.
+    state_log_prob += bwd_state_trans_log_probs
+    state_log_prob = math_ops.reduce_logsumexp(state_log_prob, axis=-1)
+
+    log_prob_sum = math_ops.reduce_logsumexp(
+        state_log_prob, axis=-1, keepdims=True)
+    state_log_prob -= log_prob_sum
+
+    cum_log_sum += array_ops.squeeze(log_prob_sum) * mask
+    batched_mask = array_ops.expand_dims(mask, axis=1)
+    out = state_log_prob * batched_mask
+    out += final_state_log_probs * (1.0 - batched_mask)
+    return out, cum_log_sum
+
+  zero_log_sum = array_ops.zeros([batch_size])
+  maxlen = _get_dim(observed_log_probs, 0)
+  mask = array_ops.sequence_mask(sequence_length, maxlen, dtypes.float32)
+  mask = array_ops.transpose(mask, perm=[1, 0])
+
+  bwd, cum_log_sum = _scan(_backward, (observed_log_probs, mask),
+                           (final_state_log_probs, zero_log_sum),
+                           reverse=True, inclusive=True)
+
+  fwd_bwd_log_probs = fwd[1:] + bwd[1:]
+  fwd_bwd_log_probs_sum = math_ops.reduce_logsumexp(
+      fwd_bwd_log_probs, axis=2, keepdims=True)
+  fwd_bwd_log_probs -= fwd_bwd_log_probs_sum
+  fwd_bwd_log_probs += math_ops.log(array_ops.expand_dims(mask, axis=2))
+
+  log_likelihood = bwd[0, :, 0] + cum_log_sum[0]
+
+  return fwd_bwd_log_probs, log_likelihood
+
+
+# TODO(tombagby): This is currently faster for the ctc implementation than using
+# functional_ops.scan, but could be replaced by that or something similar if
+# things change.
+def _scan(fn, elems, initial, reverse=False, inclusive=False, final_only=False):
+  """Repeatedly applies callable `fn` to a sequence of elements.
+
+  Implemented by functional_ops.While, tpu friendly, no gradient.
+
+  This is similar to functional_ops.scan but significantly faster on tpu/gpu
+  for the forward backward use case.
+
+  Examples:
+    scan(lambda a, e: a + e, [1.0, 2.0, 3.0], 1.0) => [2.0, 3.0, 4.0]
+
+    Multiple accumulators:
+      scan(lambda a, e: (a[0] + e, a[1] * e), [1.0, 2.0, 3.0], (0.0, 1.0))
+
+    Multiple inputs:
+      scan(lambda a, e: a + (e[0] * e[1]), (elems1, elems2), 0.0)
+
+  Args:
+    fn: callable, fn(accumulators, element) return new accumulator values.
+      The (possibly nested) sequence of accumulators is the same as `initial`
+      and the return value must have the same structure.
+    elems: A (possibly nested) tensor which will be unpacked along the first
+      dimension. The resulting slices will be the second argument to fn. The
+      first dimension of all nested input tensors must be the same.
+    initial: A tensor or (possibly nested) sequence of tensors with initial
+      values for the accumulators.
+    reverse: (optional) True enables scan and output elems in reverse order.
+    inclusive: (optional) True includes the initial accumulator values in the
+      output. Length of output will be len(elem sequence) + 1. Not meaningful
+      if final_only is True.
+    final_only: (optional) When True, return only the final accumulated values,
+      not the concatenation of accumulated values for each input.
+
+  Returns:
+    A (possibly nested) sequence of tensors with the results of applying fn
+    to tensors unpacked from elems and previous accumulator values.
+  """
+
+  flat_elems = [ops.convert_to_tensor(x) for x in nest.flatten(elems)]
+  num_elems = array_ops.shape(flat_elems[0])[0]
+  pack_elems = lambda x: nest.pack_sequence_as(structure=elems, flat_sequence=x)
+  flat_initial = [ops.convert_to_tensor(x) for x in nest.flatten(initial)]
+  pack = lambda x: nest.pack_sequence_as(structure=initial, flat_sequence=x)
+  accum_dtypes = [x.dtype for x in flat_initial]
+  num_accums = len(flat_initial)
+
+  # Types for counter, [outputs], [accumulators] loop arguments.
+  if final_only:
+    loop_dtypes = [dtypes.int32, dtypes.int32] + accum_dtypes
+  else:
+    loop_dtypes = [dtypes.int32, dtypes.int32] + accum_dtypes + accum_dtypes
+
+  # TODO(tombagby): Update to tfe.defun
+  @function.Defun(*loop_dtypes)
+  def cond(i, num_elems, *args):
+    del args
+    return i >= 0 if reverse else i < num_elems
+
+  # The loop *args are [output tensors] + [accumulator tensors] which must
+  # be paired. Each output corresponds to one accumulator.
+  @function.Defun(*loop_dtypes)
+  def body(i, num_elems, *args):
+    """Loop body."""
+    i.set_shape([])
+    if final_only:
+      accum = args
+    else:
+      out, accum = args[:num_accums], args[num_accums:]
+    slices = [array_ops.gather(e, i) for e in flat_elems]
+    accum = fn(pack(accum), pack_elems(slices))
+    flat_accum = nest.flatten(accum)
+    if final_only:
+      new_out = []
+    else:
+      update_i = i + 1 if inclusive and not reverse else i
+      new_out = [inplace_ops.alias_inplace_update(x, update_i, y)
+                 for x, y in zip(out, flat_accum)]
+    i = i - 1 if reverse else i + 1
+    return [i, num_elems] + new_out + flat_accum
+
+  init_i = (array_ops.shape(flat_elems[0])[0] - 1 if reverse
+            else constant_op.constant(0, dtype=dtypes.int32))
+  outputs = []
+  if not final_only:
+    num_outputs = array_ops.shape(flat_elems[0])[0] + (1 if inclusive else 0)
+    for initial_accum in flat_initial:
+      out_shape = array_ops.concat(
+          [[num_outputs], array_ops.shape(initial_accum)], 0)
+      out = inplace_ops.empty(out_shape, dtype=initial_accum.dtype, init=True)
+      if inclusive:
+        out = inplace_ops.alias_inplace_add(
+            out, init_i + (1 if reverse else 0), initial_accum)
+      outputs.append(out)
+  loop_in = [init_i, num_elems] + outputs + flat_initial
+  hostmem = [
+      i for i, x in enumerate(loop_in)
+      if x.dtype.base_dtype in (dtypes.int32, dtypes.int64)
+  ]
+
+  # TODO(tombagby): Update to while_v2.
+  loop_results = functional_ops.While(loop_in, cond, body, hostmem=hostmem)
+  out = loop_results[2:num_accums + 2]
+  return pack(out)
+
+
+def _get_dim(tensor, i):
+  """Get value of tensor shape[i] preferring static value if available."""
+  return tensor.shape[i].value or array_ops.shape(tensor)[i]
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index e781287d6c..48501e1b58 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -44,6 +44,10 @@ tf_module {
     name: "bidirectional_dynamic_rnn"
     argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'sequence_length\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'parallel_iterations\', \'swap_memory\', \'time_major\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "collapse_repeated"
+    argspec: "args=[\'labels\', \'seq_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "compute_accidental_hits"
     argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
@@ -108,6 +112,14 @@ tf_module {
     name: "ctc_loss"
     argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], "
   }
+  member_method {
+    name: "ctc_loss_v2"
+    argspec: "args=[\'labels\', \'logits\', \'label_length\', \'logit_length\', \'logits_time_major\', \'unique\', \'blank_index\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "ctc_unique_labels"
+    argspec: "args=[\'labels\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "depth_to_space"
     argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 34ca2078b9..eb1e793b0f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -44,6 +44,10 @@ tf_module {
     name: "bidirectional_dynamic_rnn"
     argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'sequence_length\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'parallel_iterations\', \'swap_memory\', \'time_major\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "collapse_repeated"
+    argspec: "args=[\'labels\', \'seq_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "compute_accidental_hits"
     argspec: "args=[\'true_classes\', \'sampled_candidates\', \'num_true\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
@@ -98,7 +102,11 @@ tf_module {
   }
   member_method {
     name: "ctc_loss"
-    argspec: "args=[\'labels\', \'inputs\', \'sequence_length\', \'preprocess_collapse_repeated\', \'ctc_merge_repeated\', \'ignore_longer_outputs_than_inputs\', \'time_major\'], varargs=None, keywords=None, defaults=[\'False\', \'True\', \'False\', \'True\'], "
+    argspec: "args=[\'labels\', \'logits\', \'label_length\', \'logit_length\', \'logits_time_major\', \'unique\', \'blank_index\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "ctc_unique_labels"
+    argspec: "args=[\'labels\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "depth_to_space"
-- 
GitLab


From 177b34c3b1520c846dc83cb187bba7eca75e3503 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Tue, 20 Nov 2018 09:21:31 -0800
Subject: [PATCH 0573/1554] Disable header parsing for broken targets

PiperOrigin-RevId: 222257218
---
 tensorflow/lite/experimental/writer/BUILD | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/experimental/writer/BUILD b/tensorflow/lite/experimental/writer/BUILD
index 506c668cf2..57ce636367 100644
--- a/tensorflow/lite/experimental/writer/BUILD
+++ b/tensorflow/lite/experimental/writer/BUILD
@@ -1,6 +1,9 @@
-package(default_visibility = [
-    "//visibility:public",
-])
+package(
+    default_visibility = [
+        "//visibility:public",
+    ],
+    features = ["-parse_headers"],
+)
 
 licenses(["notice"])  # Apache 2.0
 
-- 
GitLab


From 1bcdfbac8161b9c925ef0dc10288cfe207f21d36 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Tue, 20 Nov 2018 09:39:18 -0800
Subject: [PATCH 0574/1554] [Perf] Use a shared lock to avoid slow paths

Whenever a tf function is invoked, it calls ResourceMgr::Cleanup.
Because most functions do not use the step container, this is often a no-op.
Unfortunately, it grabs an exclusive lock on the process-wide (or
session-wide) resource manager. This sometimes causes the slow-path to be
taken when there are lots of concurrent function invocations (each using
their own step counter).

As a performance optimization, we first check (using a non-exclusive lock)
whether there is a step container that needs to be cleaned up in the first
place.

PiperOrigin-RevId: 222259910
---
 tensorflow/core/framework/resource_mgr.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/resource_mgr.cc b/tensorflow/core/framework/resource_mgr.cc
index 508a8d3149..9f3204ab96 100644
--- a/tensorflow/core/framework/resource_mgr.cc
+++ b/tensorflow/core/framework/resource_mgr.cc
@@ -204,12 +204,19 @@ Status ResourceMgr::Delete(const ResourceHandle& handle) {
 }
 
 Status ResourceMgr::Cleanup(const string& container) {
+  {
+    tf_shared_lock l(mu_);
+    if (!gtl::FindOrNull(containers_, container)) {
+      // Nothing to cleanup.
+      return Status::OK();
+    }
+  }
   Container* b = nullptr;
   {
     mutex_lock l(mu_);
     auto iter = containers_.find(container);
     if (iter == containers_.end()) {
-      // Nothing to cleanup, it's OK.
+      // Nothing to cleanup, it's OK (concurrent cleanup).
       return Status::OK();
     }
     b = iter->second;
-- 
GitLab


From 7f4a9f8467afee61d037c25ba65fd0601dcf9dcd Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Tue, 20 Nov 2018 09:40:02 -0800
Subject: [PATCH 0575/1554] Handle edges within same XLA cluster in
 ExtractOutsideCompilationForFunction().

This change will make sure that outside compilation to outside compilation data edges don't go through XLA cluster.

PiperOrigin-RevId: 222260022
---
 tensorflow/compiler/jit/encapsulate_util.cc   | 312 ++++++++++++++++--
 tensorflow/compiler/jit/encapsulate_util.h    |  97 ++++--
 .../compiler/jit/encapsulate_util_test.cc     |  21 +-
 .../jit/extract_outside_compilation_pass.cc   |  16 +-
 .../extract_outside_compilation_pass_test.cc  |  15 +-
 5 files changed, 374 insertions(+), 87 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_util.cc b/tensorflow/compiler/jit/encapsulate_util.cc
index 28ec37b1b9..bcc3213285 100644
--- a/tensorflow/compiler/jit/encapsulate_util.cc
+++ b/tensorflow/compiler/jit/encapsulate_util.cc
@@ -86,7 +86,7 @@ Status ProcessControlEdges(Graph* g, const string& xla_computation_attr_name,
       continue;
     } else if (src_xla_computation && !dst_xla_computation) {
       if (src_outside_compilation) {
-        // Case 1d: outside compilation to host computation control edge.
+        // Case 1c: outside compilation to host computation control edge.
         edges_to_remove.push_back(e);
 
         TF_RETURN_IF_ERROR(AppendToListAttr<string>(
@@ -94,7 +94,7 @@ Status ProcessControlEdges(Graph* g, const string& xla_computation_attr_name,
       }
     } else if (!src_xla_computation && dst_xla_computation) {
       if (dst_outside_compilation) {
-        // Case 1d: host computation control to outside compilation edge.
+        // Case 1c: host computation control to outside compilation edge.
         edges_to_remove.push_back(e);
 
         TF_RETURN_IF_ERROR(AppendToListAttr<string>(
@@ -103,40 +103,24 @@ Status ProcessControlEdges(Graph* g, const string& xla_computation_attr_name,
     } else {  // src_xla_computation && dst_xla_computation
       if (*src_xla_computation != *dst_xla_computation) {
         if (src_outside_compilation && dst_outside_compilation) {
-          // Case 1c: outside compilation to outside compilation control edge.
+          // Case 1b: outside compilation to outside compilation control edge.
           edges_to_remove.push_back(e);
 
           TF_RETURN_IF_ERROR(AppendToListAttr<string>(
               e->dst(), kXlaControlDependenciesAttrName, e->src()->name()));
         } else if (src_outside_compilation && !dst_outside_compilation) {
-          // Case 1b: outside compilation to another XLA computaition control
+          // Case 1a: outside compilation to another XLA computaition control
           // edge.
           TF_RETURN_IF_ERROR(AppendToListAttr<string>(
               e->src(), kXlaConnectedToOtherXlaComputationAttrName,
               *dst_xla_computation));
         } else if (!src_outside_compilation && dst_outside_compilation) {
-          // Case 1b: another XLA computaition to outside compilation control
+          // Case 1a: another XLA computaition to outside compilation control
           // edge.
           TF_RETURN_IF_ERROR(AppendToListAttr<string>(
               e->dst(), kXlaConnectedFromOtherXlaComputationAttrName,
               *src_xla_computation));
         }
-      } else {  // *src_xla_computation == *dst_xla_computation
-        if (src_outside_compilation && dst_outside_compilation) {
-          if (*src_outside_compilation != *dst_outside_compilation) {
-            // Case 1c: outside compilation to outside compilation control edge.
-            edges_to_remove.push_back(e);
-
-            TF_RETURN_IF_ERROR(AppendToListAttr<string>(
-                e->dst(), kXlaControlDependenciesAttrName, e->src()->name()));
-          }
-        } else if (src_outside_compilation && !dst_outside_compilation) {
-          // Case 1a: outside compilation to its XLA computation control edge.
-          ReplaceAttr(e->src(), kXlaConnectedToXlaComputationAttrName, true);
-        } else if (!src_outside_compilation && dst_outside_compilation) {
-          // Case 1a: XLA computation to outside compilation in it control edge.
-          ReplaceAttr(e->dst(), kXlaConnectedFromXlaComputationAttrName, true);
-        }
       }
     }
   }
@@ -181,12 +165,6 @@ Status ProcessXlaToXlaDataEdges(Graph* g,
         edges.push_back(EdgeInfo{e->dst_input(), e->dst()->id()});
         VLOG(4) << "XLA -> XLA edge: " << e->DebugString();
       }
-    } else {  // *src_xla_computation == *dst_xla_computation
-      if (src_outside_compilation && dst_outside_compilation &&
-          *src_outside_compilation != *dst_outside_compilation) {
-        edges.push_back(EdgeInfo{e->dst_input(), e->dst()->id()});
-        VLOG(4) << "XLA -> XLA edge: " << e->DebugString();
-      }
     }
   }
 
@@ -594,14 +572,242 @@ Status AddControlDependencies(
   return Status::OK();
 }
 
+// Step 1 for `PreprocessEdgesBetweenOutsideCompilations`. See comments of
+// `PreprocessEdgesBetweenOutsideCompilations` for details.
+Status PreprocessControlEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name) {
+  // Gather edges to remove. We should not remove the edge while iterating.
+  std::vector<const Edge*> edges_to_remove;
+  for (const Edge* e : g->edges()) {
+    if (!e->IsControlEdge()) {
+      continue;
+    }
+
+    auto src_outside_compilation =
+        GetStringAttr(*e->src(), outside_compilation_attr_name);
+    auto dst_outside_compilation =
+        GetStringAttr(*e->dst(), outside_compilation_attr_name);
+
+    if (src_outside_compilation && dst_outside_compilation) {
+      if (*src_outside_compilation != *dst_outside_compilation) {
+        // Case 1a: outside compilation to outside compilation control edge.
+        edges_to_remove.push_back(e);
+
+        TF_RETURN_IF_ERROR(AppendToListAttr<string>(
+            e->dst(), kXlaControlDependenciesWithinXlaClusterAttrName,
+            e->src()->name()));
+      }
+    } else if (src_outside_compilation && !dst_outside_compilation) {
+      // Case 1b: outside compilation to its XLA computation control edge.
+      ReplaceAttr(e->src(), kXlaConnectedToXlaComputationAttrName, true);
+    } else if (!src_outside_compilation && dst_outside_compilation) {
+      // Case 1b: XLA computation to outside compilation in it control edge.
+      ReplaceAttr(e->dst(), kXlaConnectedFromXlaComputationAttrName, true);
+    }
+  }
+
+  for (auto e : edges_to_remove) {
+    g->RemoveEdge(e);
+  }
+  return Status::OK();
+}
+
+// Step 2 for `PreprocessEdgesBetweenOutsideCompilations`. See comments of
+// `PreprocessEdgesBetweenOutsideCompilations` for details.
+Status PreprocessDataEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name) {
+  // Gather edges between outside compilation and host computation. Notice that
+  // we do not store `Edge*` directly because we remove some nodes while adding
+  // Identity nodes, and those Edge pointers might be invalidated.
+  struct EdgeInfo {
+    int dst_input, dst_node_id;
+  };
+  std::vector<EdgeInfo> edges;
+  for (const Edge* e : g->edges()) {
+    if (e->IsControlEdge()) {
+      continue;
+    }
+
+    auto src_outside_compilation =
+        GetStringAttr(*e->src(), outside_compilation_attr_name);
+    auto dst_outside_compilation =
+        GetStringAttr(*e->dst(), outside_compilation_attr_name);
+
+    if (src_outside_compilation && dst_outside_compilation &&
+        *src_outside_compilation != *dst_outside_compilation) {
+      edges.push_back(EdgeInfo{e->dst_input(), e->dst()->id()});
+      VLOG(4) << "Oc -> oc edge: " << e->DebugString();
+    }
+  }
+
+  // Remove the edge from host to outside compilation. Add a placeholder as
+  // outside compilation node input.
+  std::map<string, Node*> placeholders;
+  for (int i = 0; i < edges.size(); i++) {
+    Node* dst = g->FindNodeId(edges[i].dst_node_id);
+    const Edge* e;
+    TF_RETURN_IF_ERROR(dst->input_edge(edges[i].dst_input, &e));
+    Node* src = e->src();
+    int src_output = e->src_output(), dst_input = e->dst_input();
+    g->RemoveEdge(e);
+
+    // Find or create placeholder node.
+    string new_name = absl::StrCat(src->name(), "_oc_to_oc_placeholder");
+    auto iter = placeholders.find(new_name);
+    Node* placeholder_node;
+    if (iter == placeholders.end()) {
+      NodeDefBuilder placeholder_builder(new_name, "Placeholder");
+      placeholder_builder.Attr("dtype", src->output_type(src_output));
+      string outside_compilation_attr;
+      TF_RETURN_IF_ERROR(GetNodeAttr(dst->attrs(),
+                                     outside_compilation_attr_name,
+                                     &outside_compilation_attr));
+      placeholder_builder.Attr(outside_compilation_attr_name,
+                               outside_compilation_attr);
+      placeholder_builder.Attr(kOutsideCompilationOriginalNodeAttrName,
+                               src->name());
+      placeholder_builder.Attr(kOutsideCompilationSrcOutputAttrName,
+                               src_output);
+      NodeDef placeholder_def;
+      TF_RETURN_IF_ERROR(placeholder_builder.Finalize(&placeholder_def));
+      Status s;
+      placeholder_node = g->AddNode(placeholder_def, &s);
+      TF_RETURN_IF_ERROR(s);
+      placeholders[new_name] = placeholder_node;
+    } else {
+      placeholder_node = iter->second;
+    }
+    g->AddEdge(placeholder_node, 0, dst, dst_input);
+
+    // Replace `e->dst()` because its input node changed.
+    NodeDef new_def = dst->def();
+    *new_def.mutable_input(dst_input) = placeholder_node->name();
+    TF_ASSIGN_OR_RETURN(Node * dst_replace_node, ReplaceNode(g, dst, new_def));
+
+    // Other edge in `edges` might have `e->dst()` as src or dst
+    // node. Before removing `e->dst()`, replace those edges with
+    // corresponding edges for `dst_replace_node`.
+    for (int j = i + 1; j < edges.size(); j++) {
+      if (edges[j].dst_node_id == edges[i].dst_node_id) {
+        edges[j].dst_node_id = dst_replace_node->id();
+      }
+    }
+  }
+  return Status::OK();
+}
+
+// Step 1 for `PostprocessEdgesBetweenOutsideCompilations`. See comments of
+// `PostprocessEdgesBetweenOutsideCompilations` for details.
+Status PostprocessDataEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name) {
+  // Gather all outside compilation to outside compilation nodes.
+  std::vector<Node*> placeholder_nodes;
+  for (Node* n : g->nodes()) {
+    if (n->type_string() == "Placeholder" &&
+        HasNodeAttr(n->def(), kOutsideCompilationOriginalNodeAttrName)) {
+      placeholder_nodes.push_back(n);
+    }
+  }
+
+  // Remove the placeholder nodes, and reconnect original edge.
+  auto node_name_index = g->BuildNodeNameIndex();
+  for (auto n : placeholder_nodes) {
+    string node_name;
+    int node_src_output;
+    TF_RETURN_IF_ERROR(GetNodeAttr(
+        n->attrs(), kOutsideCompilationOriginalNodeAttrName, &node_name));
+    TF_RETURN_IF_ERROR(GetNodeAttr(
+        n->attrs(), kOutsideCompilationSrcOutputAttrName, &node_src_output));
+    auto iter = node_name_index.find(node_name);
+    if (iter == node_name_index.end()) {
+      return errors::Internal(
+          "Cannot find original node for oc -> host placeholder node ",
+          node_name);
+    }
+
+    // Change all usage node to use the original node instead.
+    Node* original_node = iter->second;
+    std::vector<const Edge*> control_edges;
+    std::vector<OutEdgeInfo> data_edges;
+    for (auto e : n->out_edges()) {
+      if (e->IsControlEdge()) {
+        control_edges.push_back(e);
+      } else {
+        data_edges.push_back({e->dst(), e->src_output(), e->dst_input()});
+      }
+    }
+    for (const Edge* e : control_edges) {
+      g->AddControlEdge(original_node, e->dst());
+      g->RemoveEdge(e);
+    }
+    for (int i = 0; i < data_edges.size(); i++) {
+      Node* dst = data_edges[i].dst;
+      NodeDef new_def = dst->def();
+      int dst_input = data_edges[i].dst_input;
+      *new_def.mutable_input(dst_input) =
+          absl::StrCat(original_node->name(), ":", node_src_output);
+      TF_ASSIGN_OR_RETURN(Node * replace_node, ReplaceNode(g, dst, new_def));
+
+      const Edge* edge_to_replace = nullptr;
+      TF_RETURN_IF_ERROR(replace_node->input_edge(dst_input, &edge_to_replace));
+      g->RemoveEdge(edge_to_replace);
+      g->AddEdge(original_node, node_src_output, replace_node, dst_input);
+
+      // Other edges might have `dst` as dst node. Update those edges with
+      // `replace_node`.
+      for (int j = i + 1; j < data_edges.size(); j++) {
+        if (data_edges[j].dst == dst) {
+          data_edges[j].dst = replace_node;
+        }
+      }
+
+      // Other placeholder node might have `dst` as original node. Update
+      // `node_name_index` with `replace_node`.
+      node_name_index[replace_node->name()] = replace_node;
+    }
+
+    // Remove placeholder node.
+    g->RemoveNode(n);
+  }
+  return Status::OK();
+}
+
+// Step 2 for `PostprocessEdgesBetweenOutsideCompilations`. See comments of
+// `PostprocessEdgesBetweenOutsideCompilations` for details.
+Status PostprocessControlEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name) {
+  auto node_name_index = g->BuildNodeNameIndex();
+
+  // Reconnect outside compilation to outside compilation control edge.
+  for (Node* n : g->nodes()) {
+    std::vector<string> control_deps;
+    Status s =
+        GetNodeAttr(n->attrs(), kXlaControlDependenciesWithinXlaClusterAttrName,
+                    &control_deps);
+    if (!s.ok()) {
+      if (s.code() != error::NOT_FOUND) {
+        return s;
+      } else {
+        continue;
+      }
+    } else {
+      n->ClearAttr(kXlaControlDependenciesWithinXlaClusterAttrName);
+      for (const string& control_input : control_deps) {
+        auto iter = node_name_index.find(control_input);
+        if (iter == node_name_index.end()) {
+          return errors::Internal("Cannot find original node for ",
+                                  control_input);
+        }
+        g->AddControlEdge(iter->second, n);
+      }
+    }
+  }
+  return Status::OK();
+}
 }  // namespace
 
 const char kXlaInferredShapesAttrName[] = "_xla_inferred_shapes";
 
-const char kXlaConnectedToXlaComputationAttrName[] =
-    "_xla_connected_to_xla_computation";
-const char kXlaConnectedFromXlaComputationAttrName[] =
-    "_xla_connected_from_xla_computation";
 const char kXlaConnectedToOtherXlaComputationAttrName[] =
     "_xla_connected_to_other_xla_computation";
 const char kXlaConnectedFromOtherXlaComputationAttrName[] =
@@ -616,6 +822,15 @@ const char kHostToOutsideCompilationOriginalNodeAttrName[] =
     "_xla_host_to_oc_node_name";
 const char kHostToOutsideCompilationSrcOutputAttrName[] =
     "_xla_host_to_oc_src_output";
+const char kXlaConnectedToXlaComputationAttrName[] =
+    "_xla_connected_to_xla_computation";
+const char kXlaConnectedFromXlaComputationAttrName[] =
+    "_xla_connected_from_xla_computation";
+const char kOutsideCompilationOriginalNodeAttrName[] =
+    "_xla_oc_to_oc_node_name";
+const char kOutsideCompilationSrcOutputAttrName[] = "_xla_oc_to_oc_src_output";
+const char kXlaControlDependenciesWithinXlaClusterAttrName[] =
+    "_xla_control_dependencies_within_xla_cluster";
 
 Status PerformStaticShapeInferenceBeforeEncapsulation(
     Graph* g, const string& xla_computation_attr_name,
@@ -699,4 +914,39 @@ Status PostprocessForEncapsulation(
   return Status::OK();
 }
 
+Status PreprocessEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name) {
+  // Remove edges from source node to outside compilation nodes, and edges
+  // from outside compilation nodes to sink node.
+  std::vector<const Edge*> edges_to_remove;
+  for (const Edge* e : g->source_node()->out_edges()) {
+    if (HasNodeAttr(e->dst()->def(), outside_compilation_attr_name)) {
+      edges_to_remove.push_back(e);
+    }
+  }
+  for (const Edge* e : g->sink_node()->in_edges()) {
+    if (HasNodeAttr(e->src()->def(), outside_compilation_attr_name)) {
+      edges_to_remove.push_back(e);
+    }
+  }
+  for (auto e : edges_to_remove) {
+    g->RemoveEdge(e);
+  }
+
+  TF_RETURN_IF_ERROR(PreprocessControlEdgesBetweenOutsideCompilations(
+      g, outside_compilation_attr_name));
+  TF_RETURN_IF_ERROR(PreprocessDataEdgesBetweenOutsideCompilations(
+      g, outside_compilation_attr_name));
+  return Status::OK();
+}
+
+Status PostprocessEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name) {
+  TF_RETURN_IF_ERROR(PostprocessDataEdgesBetweenOutsideCompilations(
+      g, outside_compilation_attr_name));
+  TF_RETURN_IF_ERROR(PostprocessControlEdgesBetweenOutsideCompilations(
+      g, outside_compilation_attr_name));
+  return Status::OK();
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/encapsulate_util.h b/tensorflow/compiler/jit/encapsulate_util.h
index 5e0c4bf6a0..e363bc5754 100644
--- a/tensorflow/compiler/jit/encapsulate_util.h
+++ b/tensorflow/compiler/jit/encapsulate_util.h
@@ -44,14 +44,6 @@ Status PerformStaticShapeInferenceBeforeEncapsulation(
     Graph* g, const string& xla_computation_attr_name,
     const string& outside_compilation_attr_name);
 
-// Attribute indicating that some ops in this node's XLA computation has control
-// dependency on this node. Attribute value will always be "true".
-extern const char kXlaConnectedToXlaComputationAttrName[];
-
-// Attribute indicating that this node has control dependency on some ops in
-// this node's XLA computation. Attribute value will always be "true".
-extern const char kXlaConnectedFromXlaComputationAttrName[];
-
 // Attribute indicating that some ops in other XLA computation has control
 // dependency on this node. Attribute value will be a list of string (XLA
 // computation names).
@@ -81,6 +73,14 @@ extern const char kOutsideCompilationToHostOriginalNodeAttrName[];
 // int (src_output for original edge).
 extern const char kOutsideCompilationToHostSrcOutputAttrName[];
 
+// Attribute indicating that some ops in this node's XLA computation has control
+// dependency on this node. Attribute value will always be "true".
+extern const char kXlaConnectedToXlaComputationAttrName[];
+
+// Attribute indicating that this node has control dependency on some ops in
+// this node's XLA computation. Attribute value will always be "true".
+extern const char kXlaConnectedFromXlaComputationAttrName[];
+
 // Attribute indicating that this is an Placeholder node added to act as a
 // temporary input node for an host node. Attribute value will be string
 // (original input node name).
@@ -91,19 +91,31 @@ extern const char kHostToOutsideCompilationOriginalNodeAttrName[];
 // for original edge).
 extern const char kHostToOutsideCompilationSrcOutputAttrName[];
 
-// Preprocesses the graph for encapsulation. It will perform the following
-// operations in order:
+// Attribute indicating that this is an Placeholder node added to act as a
+// temporary input node for an outside compilation node. Attribute value will be
+// string (original input node name).
+extern const char kOutsideCompilationOriginalNodeAttrName[];
+
+// Attribute indicating that this is an Placeholder node added to act as a
+// temporary input node for an outside compilation node. Attribute value will be
+// int (src_output for original edge).
+extern const char kOutsideCompilationSrcOutputAttrName[];
+
+// Attribute indicating that this node has control dependencies on some other
+// nodes within the same XLA cluster. Attribute value will be a list of string
+// (node names).
+extern const char kXlaControlDependenciesWithinXlaClusterAttrName[];
+
+// Preprocesses edges between different XLA clusters for encapsulation. It will
+// perform the following operations in order:
 //
-// 1a. For control edges between outside compilation and its XLA computation,
-//     add attr "kXlaConnected{From, To}XlaComputationAttrName = true" to the
-//     outside compilation node.
-// 1b. For control edges between outside compilation and another XLA
+// 1a. For control edges between outside compilation and another XLA
 //     computation, add attr "kXlaConnected{From, To}OtherXlaComputationAttrName
 //     = XLA computation node name" to the outside compilation node.
-// 1c. For control edges between different outside compilations, remove the edge
-//     and add attr "kXlaControlDependenciesAttrName = src node name" to dst
-//     node.
-// 1d. For control edges between outside compilation and host computation,
+// 1b. For control edges between different outside compilations (in different
+//     XLA computations), remove the edge and add attr
+//     "kXlaControlDependenciesAttrName = src node name" to dst node.
+// 1c. For control edges between outside compilation and host computation,
 //     remove the edge and add attr "kXlaControlDependenciesAttrName = src node
 //     name" to dst node.
 // 2. For data edges between different XLA computations, if either src or dst
@@ -146,26 +158,53 @@ struct XlaClusterInfo {
   const std::map<string, int> host_compute_core;
 };
 
-// Postprocesses the graph for encapsulation. This function reverts what
-// `PreprocessForEncapsulation` did. It will perform the following operations in
-// order:
+// Postprocesses edges between different XLA clusters for encapsulation. This
+// function reverts what `PreprocessForEncapsulation` did. It will perform the
+// following operations in order:
 //
 // 1. Remove Placeholder nodes between outside compilation and host computation
 //     (created in `PreprocessForEncapsulation` step 3).
 // 2. Remove Identity nodes created in `PreprocessForEncapsulation` step 2.
-// 3a. Reconnect control edges between different outside compilations (marked by
-//     `PreprocessForEncapsulation` step 1c) and control edges between outside
-//     compilation and host computation (marked by `PreprocessForEncapsulation`
-//     step 1d).
-// 3b. Reconnect control edges between outside compilation and another XLA
-//     computation (marked by `PreprocessForEncapsulation` step 1b).
-// Notice that control edges marked by `PreprocessForEncapsulation` step 1a are
-// not handled here. They are handled in `RewriteOutsideCompilationSubgraphFn`.
+// 3a. Reconnect control edges between outside compilation and another XLA
+//     computation (marked by `PreprocessForEncapsulation` step 1a).
+// 3b. Reconnect control edges between different outside compilations (marked by
+//     `PreprocessForEncapsulation` step 1b).
+// 3c. Reconnect control edges between outside compilation and host computation
+//     (marked by `PreprocessForEncapsulation` step 1c).
 Status PostprocessForEncapsulation(
     Graph* g, const string& xla_computation_attr_name,
     const string& outside_compilation_attr_name,
     const std::unordered_map<string, XlaClusterInfo>& clusters);
 
+// Preprocesses edges within the same XLA cluster. It will perform the following
+// operations in order:
+//
+// 0.  Remove edges from source node to outside compilation nodes, and edges
+//     from outside compilation nodes to sink node.
+// 1a. For edges between different outside compilation clusters, remove the edge
+//     and add attr "kXlaControlDependenciesWithinXlaClusterAttrName = src node
+//     name" to dst node.
+// 1b. For control edges between outside compilation and its XLA computation,
+//     add attr "kXlaConnected{From, To}XlaComputationAttrName = true" to the
+//     outside compilation node.
+// 2.  For data edges between different outside compilations, remove the edge
+//     and create a Placeholder node as dst node's input.
+Status PreprocessEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name);
+
+// Postprocesses edges within the same XLA cluster. This function reverts what
+// `PreprocessEdgesBetweenOutsideCompilations` did. It will perform the
+// following operations in order:
+//
+// 1. Remove Placeholder nodes between different outside compilations (created
+//    in `PreprocessEdgesBetweenOutsideCompilations` step 2).
+// 2a. Reconnect control edges between different outside compilations (marked by
+//     `PreprocessEdgesBetweenOutsideCompilations` step 1a).
+// Notice that control edges marked by
+// `PreprocessEdgesBetweenOutsideCompilations` step 1b are not handled here.
+// They are handled in `RewriteOutsideCompilationSubgraphFn`.
+Status PostprocessEdgesBetweenOutsideCompilations(
+    Graph* g, const string& outside_compilation_attr_name);
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_COMPILER_JIT_ENCAPSULATE_UTIL_H_
diff --git a/tensorflow/compiler/jit/encapsulate_util_test.cc b/tensorflow/compiler/jit/encapsulate_util_test.cc
index 7255df3112..25c32cef01 100644
--- a/tensorflow/compiler/jit/encapsulate_util_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_util_test.cc
@@ -107,28 +107,19 @@ TEST(PreprocessForEncapsulationTest, ControlEdges) {
   identity4_node->AddAttr("_xla", "1");
   identity4_node->AddAttr("_oc", "0");
   identity5_node->AddAttr("_xla", "1");
-  // Case 1a: control edges between outside compilation and its XLA computation.
-  g.AddControlEdge(add_node, identity0_node);
-  g.AddControlEdge(identity0_node, identity1_node);
-  // Case 1b: control edges between outside compilation and another XLA
+  // Case 1a: control edges between outside compilation and another XLA
   // computation.
   g.AddControlEdge(identity0_node, identity3_node);
   g.AddControlEdge(identity1_node, identity4_node);
-  // Case 1c: control edges between different outside compilations.
+  // Case 1b: control edges between different outside compilations.
   g.AddControlEdge(identity0_node, identity4_node);
-  // Case 1d: control edges between outside compilation and host computation.
+  // Case 1c: control edges between outside compilation and host computation.
   g.AddControlEdge(const0_node, identity0_node);
   g.AddControlEdge(identity0_node, identity2_node);
 
   TF_CHECK_OK(PreprocessForEncapsulation(&g, "_xla", "_oc"));
 
-  // Case 1a: add attr "_xla_connected_{from/to}_xla_computation = true" to the
-  // outside compilation node.
-  EXPECT_TRUE(HasNodeAttr(identity0_node->def(),
-                          kXlaConnectedFromXlaComputationAttrName));
-  EXPECT_TRUE(HasNodeAttr(identity0_node->def(),
-                          kXlaConnectedToXlaComputationAttrName));
-  // Case 1b: add attr "_xla_control_deps_{from/to} = XLA computation node name"
+  // Case 1a: add attr "_xla_control_deps_{from/to} = XLA computation node name"
   // to the outside compilation node.
   std::vector<string> attr;
   TF_CHECK_OK(GetNodeAttr(identity0_node->def(),
@@ -140,13 +131,13 @@ TEST(PreprocessForEncapsulationTest, ControlEdges) {
                           kXlaConnectedFromOtherXlaComputationAttrName, &attr));
   EXPECT_EQ(attr.size(), 1);
   EXPECT_EQ(attr[0], "0");
-  // Case 1c: add attr "_xla_control_deps = src node name" to dst node.
+  // Case 1b: add attr "_xla_control_deps = src node name" to dst node.
   attr.clear();
   TF_CHECK_OK(GetNodeAttr(identity4_node->def(),
                           kXlaControlDependenciesAttrName, &attr));
   EXPECT_EQ(attr.size(), 1);
   EXPECT_EQ(attr[0], "identity0");
-  // Case 1d: add attr "_xla_control_deps = src node name" to dst node.
+  // Case 1c: add attr "_xla_control_deps = src node name" to dst node.
   attr.clear();
   TF_CHECK_OK(GetNodeAttr(identity0_node->def(),
                           kXlaControlDependenciesAttrName, &attr));
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
index 8b3587c508..e3c7e2f89b 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
@@ -366,7 +366,7 @@ Status ReplaceOrRemoveOutsideCompilationCallNode(
 //    replace this node with compilation result node.
 // 3) all outside compilation graphs.
 Status ConstructHostGraph(
-    const string& xla_cluster_name,
+    const string& xla_cluster_name, const string& outside_compilation_attr_name,
     const std::vector<string>& outside_compilation_host_graphs,
     FunctionLibraryDefinition* fld, std::unique_ptr<Graph>* host_graph) {
   host_graph->reset(new Graph(fld));
@@ -476,6 +476,10 @@ Status ConstructHostGraph(
       host_graph->get(),
       std::unordered_set<const Node*>{(*host_graph)->sink_node()});
 
+  // Postprocess edges between different outside compilations.
+  TF_RETURN_IF_ERROR(PostprocessEdgesBetweenOutsideCompilations(
+      host_graph->get(), outside_compilation_attr_name));
+
   if (VLOG_IS_ON(4)) {
     dump_graph::DumpGraphToFile(
         absl::StrCat("extract_outside_compilation_host_graph_for_",
@@ -801,6 +805,11 @@ Status ExtractOutsideCompilationForFunction(
       },
       &fbody));
   std::unique_ptr<FunctionBody> fbody_deleter(fbody);
+
+  // Preprocess edges between different outside compilations. They will be
+  // restored in `ConstructHostGraph()`.
+  TF_RETURN_IF_ERROR(PreprocessEdgesBetweenOutsideCompilations(
+      fbody->graph, outside_compilation_attr_name));
   if (VLOG_IS_ON(4)) {
     dump_graph::DumpGraphToFile(
         absl::StrCat("extract_outside_compilation_for_func_before_", func_name),
@@ -860,8 +869,9 @@ Status ExtractOutsideCompilationForFunction(
 
   // Construct host graph.
   if (!outside_compilation_host_graphs.empty()) {
-    TF_RETURN_IF_ERROR(ConstructHostGraph(
-        xla_cluster_name, outside_compilation_host_graphs, fld, host_graph));
+    TF_RETURN_IF_ERROR(
+        ConstructHostGraph(xla_cluster_name, outside_compilation_attr_name,
+                           outside_compilation_host_graphs, fld, host_graph));
   }
 
   // Remove the outside compilation graphs from function library.
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
index c5bd64f004..bff956100d 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
@@ -290,21 +290,18 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
   TF_CHECK_OK(GetNodeAttr(host_compute_1->attrs(), "shapes", &shapes));
   EXPECT_EQ(shapes.size(), 1);
   EXPECT_EQ(shapes[0].dim_size(), 1);
-  // Check XlaHostCompute nodes' "shape_inference_graph" attr. "0" should have a
-  // non-empty value, and "1" should have an empty value.
+  // Check XlaHostCompute nodes' "shape_inference_graph" attr. Both should have
+  // empty values.
   string shape_inference_graph;
   TF_CHECK_OK(GetNodeAttr(host_compute_0->attrs(), "shape_inference_graph",
                           &shape_inference_graph));
-  EXPECT_EQ(shape_inference_graph,
-            "_outside_compilation_shape_inference_cluster_0");
+  EXPECT_EQ(shape_inference_graph, "");
   TF_CHECK_OK(GetNodeAttr(host_compute_1->attrs(), "shape_inference_graph",
                           &shape_inference_graph));
   EXPECT_EQ(shape_inference_graph, "");
 
   // Check `shape_inference_graphs`.
-  EXPECT_EQ(shape_inference_graphs.size(), 1);
-  EXPECT_EQ(shape_inference_graphs[0],
-            "_outside_compilation_shape_inference_cluster_0");
+  EXPECT_EQ(shape_inference_graphs.size(), 0);
 
   // Check `host_graph`: verify we have key placeholder and sequencer.
   Node *key_placeholder = nullptr, *sequencer = nullptr;
@@ -333,8 +330,8 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
       send_recv_nodes.push_back(n);
     }
   }
-  EXPECT_EQ(num_send_from_host, 2);
-  EXPECT_EQ(num_recv_at_host, 2);
+  EXPECT_EQ(num_send_from_host, 1);
+  EXPECT_EQ(num_recv_at_host, 1);
   for (Node *n : send_recv_nodes) {
     Node *input_node;
     TF_CHECK_OK(n->input_node(n->num_inputs() - 1, &input_node));
-- 
GitLab


From f1324400026ce07b859cc0f4c2db989ad0559f42 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 20 Nov 2018 09:48:45 -0800
Subject: [PATCH 0576/1554] Update tf.export and deprecation notice for the
 rest of RNN cell and layers.

PiperOrigin-RevId: 222261282
---
 tensorflow/python/ops/rnn.py                  |  10 +-
 tensorflow/python/ops/rnn_cell_impl.py        |   5 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   8 -
 ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt | 205 ------------------
 .../golden/v2/tensorflow.nn.rnn_cell.pbtxt    |   4 -
 5 files changed, 12 insertions(+), 220 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt

diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index 57ecb50557..c23b85847c 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -348,7 +348,10 @@ def _reverse_seq(input_seq, lengths):
   return results
 
 
-@tf_export("nn.bidirectional_dynamic_rnn")
+@deprecation.deprecated(None, "Please use `keras.layers.Bidirectional("
+                        "keras.layers.RNN(cell))`, which is equivalent to "
+                        "this API")
+@tf_export(v1=["nn.bidirectional_dynamic_rnn"])
 def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None,
                               initial_state_fw=None, initial_state_bw=None,
                               dtype=None, parallel_iterations=None,
@@ -1490,7 +1493,10 @@ def static_state_saving_rnn(cell,
   return (outputs, state)
 
 
-@tf_export("nn.static_bidirectional_rnn")
+@deprecation.deprecated(None, "Please use `keras.layers.Bidirectional("
+                        "keras.layers.RNN(cell, unroll=True))`, which is "
+                        "equivalent to this API")
+@tf_export(v1=["nn.static_bidirectional_rnn"])
 def static_bidirectional_rnn(cell_fw,
                              cell_bw,
                              inputs,
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 050b486893..2bc1ad4e04 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -1394,7 +1394,7 @@ class DeviceWrapper(RNNCell):
       return self._cell(inputs, state, scope=scope)
 
 
-@tf_export("nn.rnn_cell.MultiRNNCell")
+@tf_export(v1=["nn.rnn_cell.MultiRNNCell"])
 class MultiRNNCell(RNNCell):
   """RNN cell composed sequentially of multiple simple cells.
 
@@ -1407,6 +1407,9 @@ class MultiRNNCell(RNNCell):
   ```
   """
 
+  @deprecated(None, "This class is equivalent as "
+                    "tf.keras.layers.StackedRNNCells, and will be replaced by "
+                    "that in Tensorflow 2.0.")
   def __init__(self, cells, state_is_tuple=True):
     """Create a RNN cell composed sequentially of a number of RNNCells.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index eb1e793b0f..19380337d8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -40,10 +40,6 @@ tf_module {
     name: "bias_add"
     argspec: "args=[\'value\', \'bias\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "bidirectional_dynamic_rnn"
-    argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'sequence_length\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'parallel_iterations\', \'swap_memory\', \'time_major\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'False\', \'None\'], "
-  }
   member_method {
     name: "collapse_repeated"
     argspec: "args=[\'labels\', \'seq_length\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -292,10 +288,6 @@ tf_module {
     name: "sparse_softmax_cross_entropy_with_logits"
     argspec: "args=[\'_sentinel\', \'labels\', \'logits\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "static_bidirectional_rnn"
-    argspec: "args=[\'cell_fw\', \'cell_bw\', \'inputs\', \'initial_state_fw\', \'initial_state_bw\', \'dtype\', \'sequence_length\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "static_state_saving_rnn"
     argspec: "args=[\'cell\', \'inputs\', \'state_saver\', \'state_name\', \'sequence_length\', \'scope\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
deleted file mode 100644
index f53567af52..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
+++ /dev/null
@@ -1,205 +0,0 @@
-path: "tensorflow.nn.rnn_cell.MultiRNNCell"
-tf_class {
-  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.MultiRNNCell\'>"
-  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
-  is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "graph"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_size"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "scope_name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "state_size"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'cells\', \'state_is_tuple\'], varargs=None, keywords=None, defaults=[\'True\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'synchronization\', \'aggregation\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\', \'state\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_initial_state"
-    argspec: "args=[\'self\', \'inputs\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "zero_state"
-    argspec: "args=[\'self\', \'batch_size\', \'dtype\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt
index 3c78b07b39..b1f687f529 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.pbtxt
@@ -12,10 +12,6 @@ tf_module {
     name: "LSTMStateTuple"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "MultiRNNCell"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "RNNCell"
     mtype: "<type \'type\'>"
-- 
GitLab


From 4c75fb1cb917320acb386cf26adeb8e5151ca4f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 09:58:14 -0800
Subject: [PATCH 0577/1554] Improve error message reporting for check_numerics
 gradient.

At present the op message is only printed if the numeric check fails during
the op's 'forward' computation. If the check fails during the gradient, there is no
identifier on *which* op's gradient failed.

This is the Python equivalent of
https://github.com/tensorflow/tensorflow/commit/7e48bada

PiperOrigin-RevId: 222262823
---
 tensorflow/python/ops/array_grad.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 68c392bf28..6edc193361 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -489,10 +489,12 @@ def _GatherNdGrad(op, grad):
 
 
 @ops.RegisterGradient("CheckNumerics")
-def _CheckNumericsGrad(_, grad):
+def _CheckNumericsGrad(op, grad):
   """Gradient for check_numerics op."""
   return array_ops.check_numerics(
-      grad, "Not a number (NaN) or infinity (Inf) values detected in gradient.")
+      grad,
+      "Not a number (NaN) or infinity (Inf) values detected in gradient. %s" %
+      op.get_attr("message"))
 
 
 @ops.RegisterGradient("PlaceholderWithDefault")
-- 
GitLab


From 3275747a481a5bbc8d099a063c356d0938fdeea9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 10:07:02 -0800
Subject: [PATCH 0578/1554] [TF:XLA] Fix the output type of ArgMaxCustomCallOp.

PiperOrigin-RevId: 222264582
---
 tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index e310db2162..42bf4b06e5 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -119,6 +119,10 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
                         ", but got shape: ",
                         input_shape.DebugString()));
     }
+    const DataType dtype = output_type(0);
+    xla::PrimitiveType output_type;
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dtype, &output_type));
+    output = xla::ConvertElementType(output, output_type);
     ctx->SetOutput(0, output);
   }
 
-- 
GitLab


From 0add015954ded4642855918e8346b4f107f8c2b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 10:17:06 -0800
Subject: [PATCH 0579/1554] [TF:XLA] Implement StatelessMultinomial.

PiperOrigin-RevId: 222266213
---
 .../compiler/tests/categorical_op_test.py     | 31 +++++++++
 .../compiler/tf2xla/kernels/categorical_op.cc | 66 +++++++++++++++++--
 2 files changed, 91 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py
index 532e2b5748..15108487cf 100644
--- a/tensorflow/compiler/tests/categorical_op_test.py
+++ b/tensorflow/compiler/tests/categorical_op_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import stateless_random_ops
 from tensorflow.python.platform import googletest
 
 
@@ -138,6 +139,36 @@ class CategoricalTest(xla_test.XLATestCase):
       chi2 = self._chi2(probs, freqs)
       self.assertLess(chi2, 1e-3)
 
+  def testStatelessMultinomialIsInRange(self):
+    for dtype in self.float_types:
+      for output_dtype in self.output_dtypes():
+        with self.cached_session() as sess:
+          with self.test_scope():
+            seed_t = array_ops.placeholder(dtypes.int32, shape=[2])
+            x = stateless_random_ops.stateless_multinomial(
+                array_ops.ones(shape=[1, 20], dtype=dtype),
+                1000,
+                seed_t,
+                output_dtype=output_dtype)
+          y = sess.run(x, {seed_t: [0x12345678, 0xabcdef12]})
+          self.assertTrue((y >= 0).sum() == 1000)
+          self.assertTrue((y < 20).sum() == 1000)
+
+  def testDeterminismMultinomial(self):
+    # Stateless values should be equal iff the seeds are equal (roughly)
+    num_samples = 10
+    with self.cached_session(), self.test_scope():
+      seed_t = array_ops.placeholder(dtypes.int32, shape=[2])
+      seeds = [(x, y) for x in range(5) for y in range(5)] * 3
+      for logits in ([[0.1, 0.25, 0.5, 0.15]], [[0.5, 0.5], [0.8, 0.2],
+                                                [0.25, 0.75]]):
+        pure = stateless_random_ops.stateless_multinomial(
+            logits, num_samples, seed=seed_t)
+        values = [(seed, pure.eval(feed_dict={seed_t: seed})) for seed in seeds]
+        for s0, v0 in values:
+          for s1, v1 in values:
+            self.assertEqual(s0 == s1, np.all(v0 == v1))
+
 
 if __name__ == '__main__':
   googletest.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
index ad85940920..3e398fff95 100644
--- a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
@@ -21,10 +21,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/client/lib/prng.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.pb.h"
 
 namespace tensorflow {
 namespace {
@@ -57,8 +60,6 @@ class CategoricalOp : public XlaOpKernel {
     const int64 batch_size = logits_shape.dim_size(0);
     const int64 num_classes = logits_shape.dim_size(1);
 
-    xla::XlaBuilder* builder = ctx->builder();
-
     xla::Shape uniform_shape;
     int class_dimension;
     if (num_samples > 1) {
@@ -83,16 +84,16 @@ class CategoricalOp : public XlaOpKernel {
           xla::ShapeUtil::MakeShape(uniform_xla_type, uniform_shape_array);
       class_dimension = 1;
     }
-    xla::XlaOp uniforms =
-        xla::RngUniform(XlaHelpers::Zero(builder, input_type(0)),
-                        XlaHelpers::One(builder, input_type(0)), uniform_shape);
+    xla::PrimitiveType type;
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(input_type(0), &type));
+    xla::XlaOp log_uniforms = GetLogUniforms(uniform_shape, type, ctx);
 
     // Use Gumbel softmax trick to generate categorical samples.
     // See:
     // https://hips.seas.harvard.edu/blog/2013/04/06/the-gumbel-max-trick-for-discrete-distributions/
     // TODO(b/68769470): Switch to using a cumulative sum approach.
     auto softmax_entries =
-        xla::Sub(logits, xla::Log(-xla::Log(uniforms)),
+        xla::Sub(logits, log_uniforms,
                  /*broadcast_dimensions=*/{0, class_dimension});
 
     xla::PrimitiveType xla_output_type;
@@ -107,6 +108,16 @@ class CategoricalOp : public XlaOpKernel {
     ctx->SetOutput(0, argmax);
   }
 
+  virtual xla::XlaOp GetLogUniforms(xla::Shape uniform_shape,
+                                    xla::PrimitiveType type,
+                                    XlaOpKernelContext* ctx) {
+    xla::XlaBuilder* builder = ctx->builder();
+    auto uniforms =
+        xla::RngUniform(XlaHelpers::Zero(builder, input_type(0)),
+                        XlaHelpers::One(builder, input_type(0)), uniform_shape);
+    return xla::Log(-xla::Log(uniforms));
+  }
+
  private:
   TF_DISALLOW_COPY_AND_ASSIGN(CategoricalOp);
 };
@@ -115,5 +126,48 @@ class CategoricalOp : public XlaOpKernel {
 REGISTER_XLA_OP(Name("Multinomial").CompileTimeConstantInput("num_samples"),
                 CategoricalOp);
 
+class StatelessCategoricalOp : public CategoricalOp {
+ public:
+  explicit StatelessCategoricalOp(OpKernelConstruction* ctx)
+      : CategoricalOp(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &dtype_));
+  }
+
+  xla::XlaOp GetLogUniforms(xla::Shape uniform_shape, xla::PrimitiveType type,
+                            XlaOpKernelContext* ctx) override {
+    xla::XlaOp seed = ctx->Input(2);
+    auto seed0 = xla::Reshape(xla::Slice(seed, {0}, {1}, {1}), {});
+    auto seed1 = xla::Reshape(xla::Slice(seed, {1}, {2}, {1}), {});
+
+    xla::XlaBuilder* builder = ctx->builder();
+    if (uniform_shape.element_type() == xla::BF16) {
+      uniform_shape.set_element_type(xla::F32);
+    }
+    auto uniforms = xla::StatelessRngUniform(
+        {seed0, seed1}, uniform_shape, XlaHelpers::Zero(builder, DT_FLOAT),
+        XlaHelpers::One(builder, DT_FLOAT));
+    return xla::ConvertElementType(xla::Log(-xla::Log(uniforms)), type);
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    TensorShape seed_shape = ctx->InputShape(2);
+    OP_REQUIRES(ctx, seed_shape.dims() == 1 && seed_shape.dim_size(0) == 2,
+                errors::InvalidArgument("seed must have shape [2], not ",
+                                        seed_shape.DebugString()));
+    CategoricalOp::Compile(ctx);
+  }
+
+ private:
+  DataType dtype_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(StatelessCategoricalOp);
+};
+
+REGISTER_XLA_OP(Name("StatelessMultinomial")
+                    .CompileTimeConstantInput("num_samples")
+                    .TypeConstraint("T", {DT_FLOAT, DT_BFLOAT16})
+                    .TypeConstraint("Tseed", DT_INT32),
+                StatelessCategoricalOp);
+
 }  // anonymous namespace
 }  // namespace tensorflow
-- 
GitLab


From f0335444ce854f5e47e26631b2be708dff2d2224 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 10:24:36 -0800
Subject: [PATCH 0580/1554] TFTRT: change all instances of VLOG(0) to LOG(INFO)

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc  | 4 ++--
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc  | 2 +-
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc  | 4 ++--
 tensorflow/contrib/tensorrt/resources/trt_resources.h | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 9428fad309..69c8be8b57 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -170,7 +170,7 @@ tensorflow::Status BuildNodeMap(
 tensorflow::Status ConvertCalibGraphToInferGraph(
     const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* infer_graph,
     bool is_dyn_op) {
-  VLOG(0) << "Starting Calib Conversion";
+  LOG(INFO) << "Starting Calib Conversion";
   infer_graph->CopyFrom(graph_def);
   auto trt_rm = TRTResourceManager::instance();
   auto calib_rm = trt_rm->getManager("TRTCalibration");
@@ -876,7 +876,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       // need to check the input edges.
       [](const Edge* edge) { return true; }, OutputEdgeValidator(),
       segment_options, &initial_segments));
-  VLOG(0) << "Number of TensorRT candidate segments: "
+  LOG(INFO) << "Number of TensorRT candidate segments: "
     << initial_segments.size();
 
   // Get the EngineInfo for each segment.
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 68015a9a78..0e4dca5960 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -130,7 +130,7 @@ void GetOutputProperties(const grappler::GraphProperties& graph_properties,
     *dtype = out_shape.dtype();
     *shape = out_shape.shape();
   } else {
-    VLOG(0) << "Unknown output shape" << node->name();
+    LOG(INFO) << "Unknown output shape" << node->name();
     *dtype = node->output_type(out_port);
   }
 }
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 3cdad69f49..cff6be9759 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -491,7 +491,7 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     }
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
-    VLOG(0) << "Building a new TensorRT engine for " << name()
+    LOG(INFO) << "Building a new TensorRT engine for " << name()
             << " with batch size " << batch_size;
     // Up to this point, calibrator_ can never be empty, since otherwise it
     // means calibration_mode_ is true and this path won't get executed.
@@ -567,7 +567,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
   const int64 workspace_size_bytes = workspace_size_;
   cres->thr_.reset(new std::thread([cres, label, segment_graph, shapes,
                                     platform_gpu_id, workspace_size_bytes]() {
-    VLOG(0) << "Starting calibration thread on device " << platform_gpu_id
+    LOG(INFO) << "Starting calibration thread on device " << platform_gpu_id
             << ", Calibration Resource @ " << cres;
     auto err = cudaSetDevice(platform_gpu_id);
     if (err != cudaSuccess) {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 840da6e78d..6b68f92d16 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -39,7 +39,7 @@ namespace tensorrt {
 class TRTCalibrationResource : public tensorflow::ResourceBase {
  public:
   ~TRTCalibrationResource() {
-    VLOG(0) << "Destroying Calibration Resource " << std::endl << DebugString();
+    LOG(INFO) << "Destroying Calibration Resource " << std::endl << DebugString();
     builder_.reset();
     engine_.reset();
     // We need to manually destroy the builder and engine before the allocator
-- 
GitLab


From 1c850ad297ff2b3236a440893b1a3b1ebc8a8ca7 Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Tue, 20 Nov 2018 10:40:47 -0800
Subject: [PATCH 0581/1554] Change signature of tf.image.crop_and_resize and
 tf.image.extract_image_patches for TF 2.0.

PiperOrigin-RevId: 222270163
---
 .../python_api/api_def_CropAndResize.pbtxt    |  4 +-
 .../api_def_ExtractImagePatches.pbtxt         |  8 +--
 tensorflow/python/ops/array_ops.py            | 45 ++++++++++++
 tensorflow/python/ops/image_ops_impl.py       | 69 +++++++++++++++++++
 .../api/golden/v2/tensorflow.image.pbtxt      |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  9 +++
 6 files changed, 127 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_CropAndResize.pbtxt b/tensorflow/core/api_def/python_api/api_def_CropAndResize.pbtxt
index ce65f8172d..2559a6c80b 100644
--- a/tensorflow/core/api_def/python_api/api_def_CropAndResize.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_CropAndResize.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "CropAndResize"
-  endpoint {
-    name: "image.crop_and_resize"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_ExtractImagePatches.pbtxt b/tensorflow/core/api_def/python_api/api_def_ExtractImagePatches.pbtxt
index 0bd8b1c11a..17921dea4d 100644
--- a/tensorflow/core/api_def/python_api/api_def_ExtractImagePatches.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ExtractImagePatches.pbtxt
@@ -1,10 +1,4 @@
 op {
   graph_op_name: "ExtractImagePatches"
-  endpoint {
-    name: "image.extract_image_patches"
-  }
-  endpoint {
-    name: "extract_image_patches"
-    deprecation_version: 2
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 2a7989e889..85a2c9192d 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3201,3 +3201,48 @@ def searchsorted(sorted_sequence,
 
 
 quantize.__doc__ = gen_array_ops.quantize_v2.__doc__
+
+
+@tf_export("image.extract_image_patches", v1=[])
+def extract_image_patches_v2(
+    images,
+    sizes,
+    strides,
+    rates,
+    padding,
+    name=None):
+  # pylint: disable=line-too-long
+  r"""Extract `patches` from `images` and put them in the \"depth\" output dimension.
+
+  Args:
+    images: A 4-D Tensor with shape `[batch, in_rows, in_cols, depth]
+    sizes: The size of the sliding window for each dimension of `images`.
+    strides: A 1-D Tensor of length 4. How far the centers of two consecutive
+      patches are in the images. Must be: `[1, stride_rows, stride_cols, 1]`.
+    rates: A 1-D Tensor of length 4. Must be: `[1, rate_rows, rate_cols, 1]`.
+      This is the input stride, specifying how far two consecutive patch samples
+      are in the input. Equivalent to extracting patches with `patch_sizes_eff =
+      patch_sizes + (patch_sizes - 1) * (rates - 1)`, followed by subsampling
+      them spatially by a factor of `rates`. This is equivalent to `rate` in
+      dilated (a.k.a. Atrous) convolutions.
+    padding: The type of padding algorithm to use.
+      We specify the size-related attributes as: ```python ksizes = [1,
+        ksize_rows, ksize_cols, 1] strides = [1, strides_rows, strides_cols, 1]
+        rates = [1, rates_rows, rates_cols, 1]
+    name: A name for the operation (optional).
+
+  Returns:
+    A 4-D Tensor. Has the same type as `images`, and with shape `[batch,
+    out_rows, out_cols, ksize_rows * ksize_cols * depth]` containing image
+    patches with size `ksize_rows x ksize_cols x depth` vectorized in the
+    \"depth\" dimension. Note `out_rows` and `out_cols` are the dimensions of
+    the output patches.
+  """
+  # pylint: enable=line-too-long
+  return gen_array_ops.extract_image_patches(
+      images, sizes, strides, rates, padding, name)
+
+extract_image_patches_deprecation = deprecation.deprecated_args(
+    None, "ksizes is deprecated, use sizes instead", "ksizes")
+tf_export(v1=["image.extract_image_patches", "extract_image_patches"])(
+    extract_image_patches_deprecation(gen_array_ops.extract_image_patches))
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 4d1357abbc..c4b5db9418 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -2861,3 +2861,72 @@ resize_nearest_neighbor_deprecation = deprecation.deprecated(
         'instead.'))
 tf_export(v1=['image.resize_nearest_neighbor'])(
     resize_nearest_neighbor_deprecation(gen_image_ops.resize_nearest_neighbor))
+
+
+@tf_export('image.crop_and_resize', v1=[])
+def crop_and_resize_v2(
+    image,
+    boxes,
+    box_indices,
+    crop_size,
+    method='bilinear',
+    extrapolation_value=0,
+    name=None):
+  """Extracts crops from the input image tensor and resizes them.
+
+  Extracts crops from the input image tensor and resizes them using bilinear
+  sampling or nearest neighbor sampling (possibly with aspect ratio change) to a
+  common output size specified by `crop_size`. This is more general than the
+  `crop_to_bounding_box` op which extracts a fixed size slice from the input
+  image and does not allow resizing or aspect ratio change.
+
+  Returns a tensor with `crops` from the input `image` at positions defined at
+  the bounding box locations in `boxes`. The cropped boxes are all resized (with
+  bilinear or nearest neighbor interpolation) to a fixed
+  `size = [crop_height, crop_width]`. The result is a 4-D tensor
+  `[num_boxes, crop_height, crop_width, depth]`. The resizing is corner aligned.
+  In particular, if `boxes = [[0, 0, 1, 1]]`, the method will give identical
+  results to using `tf.image.resize_bilinear()` or
+  `tf.image.resize_nearest_neighbor()`(depends on the `method` argument) with
+  `align_corners=True`.
+
+  Args:
+    image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
+      Both `image_height` and `image_width` need to be positive.
+    boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
+      specifies the coordinates of a box in the `box_ind[i]` image and is
+      specified in normalized coordinates `[y1, x1, y2, x2]`. A normalized
+      coordinate value of `y` is mapped to the image coordinate at `y *
+      (image_height - 1)`, so as the `[0, 1]` interval of normalized image
+      height is mapped to `[0, image_height - 1]` in image height coordinates.
+      We do allow `y1` > `y2`, in which case the sampled crop is an up-down
+      flipped version of the original image. The width dimension is treated
+      similarly. Normalized coordinates outside the `[0, 1]` range are allowed,
+      in which case we use `extrapolation_value` to extrapolate the input image
+      values.
+    box_indices: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0,
+      batch)`. The value of `box_ind[i]` specifies the image that the `i`-th box
+      refers to.
+    crop_size: A 1-D tensor of 2 elements, `size = [crop_height, crop_width]`.
+      All cropped image patches are resized to this size. The aspect ratio of
+      the image content is not preserved. Both `crop_height` and `crop_width`
+      need to be positive.
+    method: An optional string specifying the sampling method for resizing. It
+      can be either `"bilinear"` or `"nearest"` and default to `"bilinear"`.
+      Currently two sampling methods are supported: Bilinear and Nearest
+      Neighbor.
+    extrapolation_value: An optional `float`. Defaults to `0`. Value used for
+      extrapolation, when applicable.
+    name: A name for the operation (optional).
+
+  Returns:
+    A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
+  """
+  return gen_image_ops.crop_and_resize(
+      image, boxes, box_indices, crop_size, method, extrapolation_value, name)
+
+
+crop_and_resize_deprecation = deprecation.deprecated_args(
+    None, 'box_ind is deprecated, use box_indices instead', 'box_ind')
+tf_export(v1=['image.crop_and_resize'])(
+    crop_and_resize_deprecation(gen_image_ops.crop_and_resize))
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index f25fb6541e..dcf80e3342 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -38,7 +38,7 @@ tf_module {
   }
   member_method {
     name: "crop_and_resize"
-    argspec: "args=[\'image\', \'boxes\', \'box_ind\', \'crop_size\', \'method\', \'extrapolation_value\', \'name\'], varargs=None, keywords=None, defaults=[\'bilinear\', \'0\', \'None\'], "
+    argspec: "args=[\'image\', \'boxes\', \'box_indices\', \'crop_size\', \'method\', \'extrapolation_value\', \'name\'], varargs=None, keywords=None, defaults=[\'bilinear\', \'0\', \'None\'], "
   }
   member_method {
     name: "crop_to_bounding_box"
@@ -86,7 +86,7 @@ tf_module {
   }
   member_method {
     name: "extract_image_patches"
-    argspec: "args=[\'images\', \'ksizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'images\', \'sizes\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "extract_jpeg_shape"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9b2abb9d2a..1074ba5702 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -31,6 +31,15 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Maps from a function name to a dictionary that describes how to
     # map from an old argument keyword to the new argument keyword.
     self.function_keyword_renames = {
+        "tf.image.crop_and_resize": {
+            "box_ind": "box_indices",
+        },
+        "tf.image.extract_image_patches": {
+            "ksizes": "sizes",
+        },
+        "tf.extract_image_patches": {
+            "ksizes": "sizes",
+        },
         "tf.expand_dims": {
             "dim": "axis",
         },
-- 
GitLab


From a3650434746cba6b78d9846fbde3b7f31f6a33c8 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 20 Nov 2018 11:00:02 -0800
Subject: [PATCH 0582/1554] [tf.data] Enable `tf.data.experimental.Optional` as
 a dataset component.

This change introduces `DatasetStructure` as a `Structure` subclass for representing the structure of a `Dataset` value, and changes the representation of "legacy structures" for non-Tensor components to use the new class.

Previously, a nested `Dataset` was represented by a `_NestedDatasetComponent` that appeared at the same position in `Dataset.output_types`, `Dataset.output_shapes`, and `Dataset.output_classes`.

After this change, we allow a `Structure` object to substitute for `_NestedDatasetComponent`. Thus `DatasetStructure` becomes the direct replacement for `_NestedDatasetComponent`, and `OptionalStructure` can be used in the same position to represent an `Optional` as a component of a dataset element.

PiperOrigin-RevId: 222273133
---
 tensorflow/python/data/experimental/ops/BUILD |   2 +-
 .../python/data/experimental/ops/grouping.py  |  16 +-
 tensorflow/python/data/kernel_tests/BUILD     |   4 +
 .../data/kernel_tests/dataset_ops_test.py     |  61 ++++
 tensorflow/python/data/ops/dataset_ops.py     | 260 +++++++-----------
 tensorflow/python/data/ops/optional_ops.py    |  12 +-
 tensorflow/python/data/util/structure.py      |  26 +-
 tensorflow/python/data/util/structure_test.py |   9 +-
 8 files changed, 202 insertions(+), 188 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 170fda90b6..b6c1376b6a 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -165,7 +165,7 @@ py_library(
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index 80ca7104d8..db10ea3b7f 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -21,6 +21,7 @@ import numpy as np
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -448,7 +449,10 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
 
   def _make_reduce_func(self, reduce_func, input_dataset):
     """Make wrapping defun for reduce_func."""
-    nested_dataset = dataset_ops._NestedDatasetComponent(input_dataset)  # pylint: disable=protected-access
+    nested_dataset = dataset_ops.DatasetStructure(
+        structure.Structure._from_legacy_structure(  # pylint: disable=protected-access
+            input_dataset.output_types, input_dataset.output_shapes,
+            input_dataset.output_classes))
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
         reduce_func,
         self._transformation_name(),
@@ -456,11 +460,13 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
         input_shapes=(tensor_shape.scalar(), nested_dataset),
         input_types=(dtypes.int64, nested_dataset))
     if not isinstance(
-        wrapped_func.output_classes, dataset_ops._NestedDatasetComponent):  # pylint: disable=protected-access
+        wrapped_func.output_structure, dataset_ops.DatasetStructure):
       raise TypeError("`reduce_func` must return a `Dataset` object.")
-    self._output_classes = wrapped_func.output_classes.output_classes
-    self._output_types = wrapped_func.output_types.output_types
-    self._output_shapes = wrapped_func.output_shapes.output_shapes
+    # pylint: disable=protected-access
+    element_structure = wrapped_func.output_structure._element_structure
+    self._output_classes = element_structure._to_legacy_output_classes()
+    self._output_types = element_structure._to_legacy_output_types()
+    self._output_shapes = element_structure._to_legacy_output_shapes()
     self._reduce_func = wrapped_func.function
 
   @property
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 21eed2b070..fa1f6d701a 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -117,8 +117,12 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:optional_ops",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
index a5324af4d0..1f22a37c2e 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
@@ -24,10 +24,14 @@ import numpy as np
 from tensorflow.core.framework import graph_pb2
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import optional_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.platform import test
 
 
@@ -249,6 +253,63 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertTrue(ds.options().experimental_autotune)
     self.assertTrue(ds.options().experimental_filter_fusion)
 
+  # pylint: disable=g-long-lambda
+  @parameterized.named_parameters(
+      ("Tensor", lambda: constant_op.constant(37.0),
+       structure.TensorStructure(dtypes.float32, [])),
+      ("SparseTensor", lambda: sparse_tensor.SparseTensor(
+          indices=[[0]], values=constant_op.constant([0], dtype=dtypes.int32),
+          dense_shape=[1]),
+       structure.SparseTensorStructure(dtypes.int32, [1])),
+      ("Nest", lambda: {
+          "a": constant_op.constant(37.0),
+          "b": (constant_op.constant(["Foo"]), constant_op.constant("Bar"))},
+       structure.NestedStructure({
+           "a": structure.TensorStructure(dtypes.float32, []),
+           "b": (structure.TensorStructure(dtypes.string, [1]),
+                 structure.TensorStructure(dtypes.string, []))})),
+      ("Dataset", lambda: dataset_ops.Dataset.from_tensor_slices(
+          constant_op.constant([1, 2, 3])),
+       dataset_ops.DatasetStructure(
+           structure.TensorStructure(dtypes.int32, []))),
+      ("Optional", lambda: optional_ops.Optional.from_value(37.0),
+       optional_ops.OptionalStructure(
+           structure.TensorStructure(dtypes.float32, []))),
+  )
+  def testDatasetStructure(self, tf_value_fn, expected_element_structure):
+    dataset = dataset_ops.Dataset.from_tensors(0).map(lambda _: tf_value_fn())
+    dataset_structure = structure.Structure.from_value(dataset)
+    self.assertIsInstance(dataset_structure, dataset_ops.DatasetStructure)
+
+    # TODO(b/110122868): Add a public API to `tf.data.Dataset` for accessing
+    # the element structure.
+    self.assertTrue(expected_element_structure.is_compatible_with(
+        dataset_structure._element_structure))
+    self.assertTrue(dataset_structure._element_structure.is_compatible_with(
+        expected_element_structure))
+
+    self.assertEqual([dtypes.variant], dataset_structure._flat_types)
+    self.assertEqual([tensor_shape.scalar()], dataset_structure._flat_shapes)
+
+    # Assert that the `Dataset` survives a round-trip via _from_tensor_list()
+    # and _to_tensor_list().
+    round_trip_dataset = dataset_structure._from_tensor_list(
+        dataset_structure._to_tensor_list(dataset))
+
+    value = tf_value_fn()
+
+    if isinstance(value, dataset_ops.Dataset):
+      self.assertDatasetsEqual(value, dataset.flat_map(lambda x: x))
+    elif isinstance(value, optional_ops.Optional):
+      self.assertDatasetProduces(
+          round_trip_dataset.map(lambda opt: opt.get_value()),
+          [self.evaluate(value.get_value())],
+          requires_initialization=True)
+    else:
+      self.assertDatasetProduces(
+          round_trip_dataset, [self.evaluate(tf_value_fn())],
+          requires_initialization=True)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 4a11619112..5c0cfe994d 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -31,6 +31,7 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import random_seed
 from tensorflow.python.data.util import sparse
+from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -1868,57 +1869,6 @@ class SparseTensorSliceDataset(DatasetSource):
     return (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64)
 
 
-class _NestedDatasetComponent(object):
-  """The structure of a `Dataset` nested in a component of another `Dataset`.
-
-  A `StructuredFunctionWrapper` around a function that returns a `Dataset` as
-  one of its components will have a `NestedDatasetComponent` in the
-  corresponding position in the `output_classes`, `output_shapes`, and
-  `output_types` properties.
-
-  TODO(b/110122868): Add this class, or something equivalent, to the public API.
-  We are considering revising the public API for accessing Dataset structure
-  (`output_classes` etc.) based on experience with nested datasets and other
-  custom component types.
-  """
-
-  def __init__(self,
-               dataset=None,
-               output_shapes=None,
-               output_types=None,
-               output_classes=None):
-    if dataset is None:
-      if (output_classes is None or output_shapes is None or
-          output_types is None):
-        raise ValueError(
-            "Either `dataset`, or all of `output_classes`, "
-            "`output_shapes`, and `output_types` must be specified.")
-      self._output_classes = output_classes
-      self._output_shapes = output_shapes
-      self._output_types = output_types
-    else:
-      if not (output_classes is None and output_shapes is None and
-              output_types is None):
-        raise ValueError(
-            "Either `dataset`, or all of `output_classes`, "
-            "`output_shapes`, and `output_types` must be specified.")
-      self._output_classes = dataset.output_classes
-      self._output_shapes = dataset.output_shapes
-      self._output_types = dataset.output_types
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
-
-
 class _VariantDataset(DatasetV2):
   """A Dataset wrapper around a `tf.variant`-typed function argument."""
 
@@ -1935,15 +1885,73 @@ class _VariantDataset(DatasetV2):
 
   @property
   def output_classes(self):
-    return self._structure.output_classes
+    return self._structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    return self._structure.output_shapes
+    return self._structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._structure.output_types
+    return self._structure._to_legacy_output_types()  # pylint: disable=protected-access
+
+
+class DatasetStructure(structure_lib.Structure):
+  """Represents a `Dataset` of structured values."""
+
+  def __init__(self, element_structure):
+    self._element_structure = element_structure
+
+  @property
+  def _flat_shapes(self):
+    return [tensor_shape.scalar()]
+
+  @property
+  def _flat_types(self):
+    return [dtypes.variant]
+
+  def is_compatible_with(self, other):
+    # pylint: disable=protected-access
+    return (isinstance(other, DatasetStructure) and
+            self._element_structure.is_compatible_with(
+                other._element_structure))
+
+  def _to_tensor_list(self, value):
+    return [value._as_variant_tensor()]  # pylint: disable=protected-access
+
+  def _from_tensor_list(self, flat_value):
+    if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
+        not flat_value[0].shape.is_compatible_with(tensor_shape.scalar())):
+      raise ValueError(
+          "DatasetStructure corresponds to a single tf.variant scalar.")
+    return self._from_compatible_tensor_list(flat_value)
+
+  def _from_compatible_tensor_list(self, flat_value):
+    # pylint: disable=protected-access
+    return _VariantDataset(flat_value[0], self._element_structure)
+
+  @staticmethod
+  def from_value(value):
+    # TODO(b/110122868): We can simplify this when a `Dataset` object has a
+    # `Structure`-valued property.
+    element_structure = structure_lib.Structure._from_legacy_structure(
+        value.output_types, value.output_shapes, value.output_classes)
+    return DatasetStructure(element_structure)
+
+  def _to_legacy_output_types(self):
+    return self
+
+  def _to_legacy_output_shapes(self):
+    return self
+
+  def _to_legacy_output_classes(self):
+    return self
+
+
+# pylint: disable=protected-access
+structure_lib.Structure._register_custom_converter(DatasetV2,
+                                                   DatasetStructure.from_value)
+# pylint: enable=protected-access
 
 
 class StructuredFunctionWrapper(object):
@@ -2001,6 +2009,9 @@ class StructuredFunctionWrapper(object):
       self._input_types = dataset.output_types
       self._input_classes = dataset.output_classes
 
+    self._input_structure = structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
+        self._input_types, self._input_shapes, self._input_classes)
+
     self._transformation_name = transformation_name
     readable_transformation_name = transformation_name.replace(
         ".", "_")[:-2] if len(transformation_name) > 2 else ""
@@ -2008,35 +2019,18 @@ class StructuredFunctionWrapper(object):
         readable_transformation_name,
         function_utils.get_func_name(func),
         str(ops.uid())
-
     ])
 
     if defun_kwargs is None:
       defun_kwargs = {}
 
     @function.Defun(
-        *self._defun_args(), func_name=self._func_name, **defun_kwargs)
+        *self._input_structure._flat_types, func_name=self._func_name,  # pylint: disable=protected-access
+        **defun_kwargs)
     def tf_data_structured_function_wrapper(*args):
       """Wrapper for passing nested structures to and from tf.data functions."""
-      flat_args = []
-      for arg, arg_class, arg_shape, arg_type in zip(
-          args,
-          nest.flatten(self._input_classes),
-          nest.flatten(self._input_shapes),
-          nest.flatten(self._input_types)):
-        # TODO(b/110122868): Add a registration mechanism for new component
-        # types.
-        if arg_class is sparse_tensor_lib.SparseTensor:
-          arg = sparse.deserialize_sparse_tensors(
-              arg, arg_type, arg_shape, arg_class)
-          arg.indices.set_shape([None, arg_shape.ndims])
-          arg.dense_shape.set_shape([arg_shape.ndims])
-        elif isinstance(arg_class, _NestedDatasetComponent):
-          arg = _VariantDataset(arg, arg_class)
-        else:
-          arg.set_shape(arg_shape)
-        flat_args.append(arg)
-      nested_args = nest.pack_sequence_as(self._input_classes, flat_args)
+      # pylint: disable=protected-access
+      nested_args = self._input_structure._from_compatible_tensor_list(args)
       if not _should_unpack_args(nested_args):
         nested_args = (nested_args,)
 
@@ -2054,50 +2048,14 @@ class StructuredFunctionWrapper(object):
       if isinstance(ret, list):
         ret = tuple(ret)
 
-      # Convert any `SparseTensorValue`s to `SparseTensor`s and all other
-      # values to tensors.
-      flat_ret = []
-      flat_classes = []
-      flat_shapes = []
-      flat_types = []
-      for t in nest.flatten(ret):
-        # TODO(b/110122868): Add a registration mechanism for new component
-        # types.
-        if sparse_tensor_lib.is_sparse(t):
-          t = sparse_tensor_lib.SparseTensor.from_value(t)
-          flat_ret.append(sparse.serialize_sparse_tensors(t))
-          flat_classes.append(sparse_tensor_lib.SparseTensor)
-          flat_shapes.append(t.get_shape())
-          flat_types.append(t.dtype)
-        elif isinstance(t, DatasetV2):
-          flat_ret.append(t._as_variant_tensor())  # pylint: disable=protected-access
-          component = _NestedDatasetComponent(t)
-          flat_classes.append(component)
-          flat_shapes.append(component)
-          flat_types.append(component)
-          if t.options() != Options():
-            warnings.warn("Encountered a nested dataset with non-default "
-                          "options. These options will not be propagated to "
-                          "the outer dataset.")
-        else:
-          try:
-            t = ops.convert_to_tensor(t)
-          except (ValueError, TypeError):
-            raise TypeError("Unsupported return value from function passed to "
-                            "%s: %s." % (transformation_name, t))
-          flat_ret.append(t)
-          flat_classes.append(ops.Tensor)
-          flat_shapes.append(t.get_shape())
-          flat_types.append(t.dtype)
-
-      ret = nest.pack_sequence_as(ret, flat_ret)
-      self._output_classes = nest.pack_sequence_as(ret, flat_classes)
-      self._output_shapes = nest.pack_sequence_as(ret, flat_shapes)
-      self._output_types = nest.pack_sequence_as(ret, flat_types)
+      try:
+        self._output_structure = structure_lib.Structure.from_value(ret)
+      except (ValueError, TypeError):
+        raise TypeError("Unsupported return value from function passed to "
+                        "%s: %s." % (transformation_name, ret))
 
       _warn_if_collections(transformation_name)
-
-      return flat_ret
+      return self._output_structure._to_tensor_list(ret)
 
     self._function = tf_data_structured_function_wrapper
     if add_to_graph:
@@ -2108,32 +2066,21 @@ class StructuredFunctionWrapper(object):
       # in case (e.g.) we need to rerun the function.
       self._function._create_definition_if_needed()  # pylint: disable=protected-access
 
-  def _defun_args(self):
-    """Returns a flat list of `tf.DType` for the input element structure."""
-    ret = []
-    for input_type, input_class in zip(nest.flatten(self._input_types),
-                                       nest.flatten(self._input_classes)):
-      # TODO(b/110122868): Add a registration mechanism for new component types.
-      if input_class is sparse_tensor_lib.SparseTensor:
-        ret.append(dtypes.variant)
-      elif isinstance(input_class, _NestedDatasetComponent):
-        ret.append(dtypes.variant)
-      else:
-        assert isinstance(input_type, dtypes.DType)
-        ret.append(input_type)
-    return ret
+  @property
+  def output_structure(self):
+    return self._output_structure
 
   @property
   def output_classes(self):
-    return self._output_classes
+    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    return self._output_shapes
+    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._output_types
+    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
   @property
   def function(self):
@@ -2156,30 +2103,12 @@ def flat_structure(dataset):
     A dictionary of keyword arguments that can be passed to many Dataset op
     constructors.
   """
-  output_classes = []
-  output_shapes = []
-  output_types = []
-  for output_class, output_shape, output_type in zip(
-      nest.flatten(dataset.output_classes), nest.flatten(dataset.output_shapes),
-      nest.flatten(dataset.output_types)):
-    if isinstance(output_class, _NestedDatasetComponent):
-      output_classes.append(output_class.output_classes)
-      output_shapes.append(output_shape.output_shapes)
-      output_types.append(output_type.output_types)
-    else:
-      output_classes.append(output_class)
-      output_shapes.append(output_shape)
-      output_types.append(output_type)
-
-  output_classes = nest.pack_sequence_as(dataset.output_classes, output_classes)
-  output_shapes = nest.pack_sequence_as(dataset.output_shapes, output_shapes)
-  output_types = nest.pack_sequence_as(dataset.output_types, output_types)
-
+  # pylint: disable=protected-access
+  structure = structure_lib.Structure._from_legacy_structure(
+      dataset.output_types, dataset.output_shapes, dataset.output_classes)
   return {
-      "output_shapes":
-          nest.flatten(sparse.as_dense_shapes(output_shapes, output_classes)),
-      "output_types":
-          nest.flatten(sparse.as_dense_types(output_types, output_classes)),
+      "output_shapes": structure._flat_shapes,
+      "output_types": structure._flat_types,
   }
 
 
@@ -2902,11 +2831,13 @@ class FlatMapDataset(UnaryDataset):
 
     wrapped_func = StructuredFunctionWrapper(
         map_func, self._transformation_name(), dataset=input_dataset)
-    if not isinstance(wrapped_func.output_classes, _NestedDatasetComponent):
+    if not isinstance(wrapped_func.output_structure, DatasetStructure):
       raise TypeError("`map_func` must return a `Dataset` object.")
-    self._output_classes = wrapped_func.output_classes.output_classes
-    self._output_types = wrapped_func.output_types.output_types
-    self._output_shapes = wrapped_func.output_shapes.output_shapes
+    # pylint: disable=protected-access
+    element_structure = wrapped_func.output_structure._element_structure
+    self._output_classes = element_structure._to_legacy_output_classes()
+    self._output_types = element_structure._to_legacy_output_types()
+    self._output_shapes = element_structure._to_legacy_output_shapes()
     self._map_func = wrapped_func.function
 
   def _as_variant_tensor(self):
@@ -3048,10 +2979,9 @@ class WindowDataset(UnaryDataset):
     self._output_classes = nest.pack_sequence_as(
         input_dataset.output_classes,
         [
-            _NestedDatasetComponent(  # pylint: disable=protected-access
-                output_classes=output_class,
-                output_shapes=output_shape,
-                output_types=output_type)
+            DatasetStructure(
+                structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
+                    output_type, output_shape, output_class))
             for output_class, output_shape, output_type in zip(
                 nest.flatten(input_dataset.output_classes),
                 nest.flatten(input_dataset.output_shapes),
diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py
index 91cf883ce9..4113b7ed31 100644
--- a/tensorflow/python/data/ops/optional_ops.py
+++ b/tensorflow/python/data/ops/optional_ops.py
@@ -183,19 +183,13 @@ class OptionalStructure(structure.Structure):
     return OptionalStructure(value.value_structure)
 
   def _to_legacy_output_types(self):
-    raise NotImplementedError("The `output_types` property is not supported on "
-                              "structured objects containing an `Optional`. "
-                              "Use the corresponding `structure` property.")
+    return self
 
   def _to_legacy_output_shapes(self):
-    raise NotImplementedError("The `output_shapes` property is not supported on"
-                              " structured objects containing an `Optional`. "
-                              "Use the corresponding `structure` property.")
+    return self
 
   def _to_legacy_output_classes(self):
-    raise NotImplementedError("The `output_classes` property is not supported "
-                              "on structured objects containing an `Optional`. "
-                              "Use the corresponding `structure` property.")
+    return self
 
 
 # pylint: disable=protected-access
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 9a3118297d..3cf67b0745 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -208,14 +208,16 @@ class Structure(object):
     flat_ret = []
     for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes,
                                                  flat_classes):
-      if issubclass(flat_class, sparse_tensor_lib.SparseTensor):
+      if isinstance(flat_class, Structure):
+        flat_ret.append(flat_class)
+      elif issubclass(flat_class, sparse_tensor_lib.SparseTensor):
         flat_ret.append(SparseTensorStructure(flat_type, flat_shape))
       elif issubclass(flat_class, ops.Tensor):
         flat_ret.append(TensorStructure(flat_type, flat_shape))
       else:
         # NOTE(mrry): Since legacy structures produced by iterators only
-        # comprise Tensors, SparseTensors, and nests, we do not need to support
-        # all structure types here.
+        # comprise Tensors, SparseTensors, and nests, we do not need to
+        # support all structure types here.
         raise TypeError(
             "Could not build a structure for output class %r" % flat_type)
 
@@ -381,6 +383,13 @@ class TensorStructure(Structure):
     return self._from_compatible_tensor_list(flat_value)
 
   def _from_compatible_tensor_list(self, flat_value):
+    # TODO(b/112266545): It would be cleaner to create a new `ensure_shape()`
+    # op here and return that, instead of mutating the input's shape using
+    # `Tensor.set_shape()`. However, that would add extra ops on the arguments
+    # of each `tf.data` function, which could impact performance. When this
+    # bug is resolved, we should be able to add the `ensure_shape()` ops and
+    # optimize them away using contextual shape information.
+    flat_value[0].set_shape(self._shape)
     return flat_value[0]
 
   @staticmethod
@@ -406,7 +415,11 @@ class SparseTensorStructure(Structure):
 
   @property
   def _flat_shapes(self):
-    return [tensor_shape.vector(3)]
+    # NOTE(mrry): The default flat shape of a boxed `SparseTensor` is `(3,)`,
+    # but a `SparseTensorStructure` can also represent a batch of boxed
+    # `SparseTensor` objects with shape `(?, 3)` (and batches of batches, etc.),
+    # so the flat shape must be unknown.
+    return [tensor_shape.unknown_shape(None)]
 
   @property
   def _flat_types(self):
@@ -428,8 +441,11 @@ class SparseTensorStructure(Structure):
     return self._from_compatible_tensor_list(flat_value)
 
   def _from_compatible_tensor_list(self, flat_value):
-    return sparse_ops.deserialize_sparse(
+    ret = sparse_ops.deserialize_sparse(
         flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
+    ret.indices.set_shape([None, self._dense_shape.ndims])
+    ret.dense_shape.set_shape([self._dense_shape.ndims])
+    return ret
 
   @staticmethod
   def from_value(value):
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index 630a0c912b..65a41a50f1 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -44,7 +44,7 @@ class StructureTest(test.TestCase, parameterized.TestCase):
        [dtypes.float32], [[]]),
       (lambda: sparse_tensor.SparseTensor(
           indices=[[3, 4]], values=[-1], dense_shape=[4, 5]),
-       structure.SparseTensorStructure, [dtypes.variant], [[3]]),
+       structure.SparseTensorStructure, [dtypes.variant], [None]),
       (lambda: (constant_op.constant(37.0), constant_op.constant([1, 2, 3])),
        structure.NestedStructure, [dtypes.float32, dtypes.int32], [[], [3]]),
       (lambda: {
@@ -58,14 +58,17 @@ class StructureTest(test.TestCase, parameterized.TestCase):
                 sparse_tensor.SparseTensor(
                     indices=[[3, 4]], values=[-1], dense_shape=[4, 5]))
       }, structure.NestedStructure,
-       [dtypes.float32, dtypes.variant, dtypes.variant], [[], [3], [3]]))
+       [dtypes.float32, dtypes.variant, dtypes.variant], [[], None, None]))
   def testFlatStructure(self, value_fn, expected_structure, expected_types,
                         expected_shapes):
     value = value_fn()
     s = structure.Structure.from_value(value)
     self.assertIsInstance(s, expected_structure)
     self.assertEqual(expected_types, s._flat_types)
-    self.assertEqual(expected_shapes, s._flat_shapes)
+    for expected, actual in zip(expected_shapes, s._flat_shapes):
+      self.assertTrue(actual.is_compatible_with(expected))
+      self.assertTrue(
+          tensor_shape.as_shape(expected).is_compatible_with(actual))
 
   @parameterized.parameters(
       (lambda: constant_op.constant(37.0), lambda: [
-- 
GitLab


From 231d94406b9bb4c85e43ae15b2df391a439d6ccc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 11:02:05 -0800
Subject: [PATCH 0583/1554] Update docstring for CategoricalAccuracy.

PiperOrigin-RevId: 222273525
---
 tensorflow/python/keras/metrics.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 7848be33f1..030bf4768c 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -760,6 +760,9 @@ class CategoricalAccuracy(MeanMetricWrapper):
   ultimately returned as `categorical accuracy`: an idempotent operation that
   simply divides `total` by `count`.
 
+  `y_pred` and `y_true` should be passed in as vectors of probabilities, rather
+  than as labels. If necessary, use `tf.one_hot` to expand `y_true` as a vector.
+
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
   """
-- 
GitLab


From 02c1056b0d35e53dc8173a8bd1b70196cf9ea0c4 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 20 Nov 2018 11:02:23 -0800
Subject: [PATCH 0584/1554] Rename random.set_random_seed to random.set_seed
 for TF 2.0 API.

PiperOrigin-RevId: 222273633
---
 tensorflow/python/framework/random_seed.py    | 103 +++++++++++++++++-
 .../api/golden/v2/tensorflow.random.pbtxt     |   2 +-
 tensorflow/tools/compatibility/renames_v2.py  |  20 +++-
 3 files changed, 119 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py
index 0d20693429..6b7f56a92c 100644
--- a/tensorflow/python/framework/random_seed.py
+++ b/tensorflow/python/framework/random_seed.py
@@ -82,8 +82,7 @@ def get_seed(op_seed):
   return seeds
 
 
-@tf_export('random.set_random_seed',
-           v1=['random.set_random_seed', 'set_random_seed'])
+@tf_export(v1=['random.set_random_seed', 'set_random_seed'])
 def set_random_seed(seed):
   """Sets the graph-level random seed.
 
@@ -183,3 +182,103 @@ def set_random_seed(seed):
     context.set_global_seed(seed)
   else:
     ops.get_default_graph().seed = seed
+
+
+@tf_export('random.set_seed', v1=[])
+def set_seed(seed):
+  """Sets the graph-level random seed.
+
+  Operations that rely on a random seed actually derive it from two seeds:
+  the graph-level and operation-level seeds. This sets the graph-level seed.
+
+  Its interactions with operation-level seeds is as follows:
+
+    1. If neither the graph-level nor the operation seed is set:
+      A random seed is used for this op.
+    2. If the graph-level seed is set, but the operation seed is not:
+      The system deterministically picks an operation seed in conjunction
+      with the graph-level seed so that it gets a unique random sequence.
+    3. If the graph-level seed is not set, but the operation seed is set:
+      A default graph-level seed and the specified operation seed are used to
+      determine the random sequence.
+    4. If both the graph-level and the operation seed are set:
+      Both seeds are used in conjunction to determine the random sequence.
+
+  To illustrate the user-visible effects, consider these examples:
+
+  To generate different sequences across sessions, set neither
+  graph-level nor op-level seeds:
+
+  ```python
+  a = tf.random_uniform([1])
+  b = tf.random_normal([1])
+
+  print("Session 1")
+  with tf.Session() as sess1:
+    print(sess1.run(a))  # generates 'A1'
+    print(sess1.run(a))  # generates 'A2'
+    print(sess1.run(b))  # generates 'B1'
+    print(sess1.run(b))  # generates 'B2'
+
+  print("Session 2")
+  with tf.Session() as sess2:
+    print(sess2.run(a))  # generates 'A3'
+    print(sess2.run(a))  # generates 'A4'
+    print(sess2.run(b))  # generates 'B3'
+    print(sess2.run(b))  # generates 'B4'
+  ```
+
+  To generate the same repeatable sequence for an op across sessions, set the
+  seed for the op:
+
+  ```python
+  a = tf.random_uniform([1], seed=1)
+  b = tf.random_normal([1])
+
+  # Repeatedly running this block with the same graph will generate the same
+  # sequence of values for 'a', but different sequences of values for 'b'.
+  print("Session 1")
+  with tf.Session() as sess1:
+    print(sess1.run(a))  # generates 'A1'
+    print(sess1.run(a))  # generates 'A2'
+    print(sess1.run(b))  # generates 'B1'
+    print(sess1.run(b))  # generates 'B2'
+
+  print("Session 2")
+  with tf.Session() as sess2:
+    print(sess2.run(a))  # generates 'A1'
+    print(sess2.run(a))  # generates 'A2'
+    print(sess2.run(b))  # generates 'B3'
+    print(sess2.run(b))  # generates 'B4'
+  ```
+
+  To make the random sequences generated by all ops be repeatable across
+  sessions, set a graph-level seed:
+
+  ```python
+  tf.random.set_seed(1234)
+  a = tf.random_uniform([1])
+  b = tf.random_normal([1])
+
+  # Repeatedly running this block with the same graph will generate the same
+  # sequences of 'a' and 'b'.
+  print("Session 1")
+  with tf.Session() as sess1:
+    print(sess1.run(a))  # generates 'A1'
+    print(sess1.run(a))  # generates 'A2'
+    print(sess1.run(b))  # generates 'B1'
+    print(sess1.run(b))  # generates 'B2'
+
+  print("Session 2")
+  with tf.Session() as sess2:
+    print(sess2.run(a))  # generates 'A1'
+    print(sess2.run(a))  # generates 'A2'
+    print(sess2.run(b))  # generates 'B1'
+    print(sess2.run(b))  # generates 'B2'
+  ```
+
+  Args:
+    seed: integer.
+  """
+  # TODO(go/tf2-random): change doc, update to match design doc
+  set_random_seed(seed)
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index ce8d277ec8..de5cb6b717 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -29,7 +29,7 @@ tf_module {
     argspec: "args=[\'shape\', \'lam\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\', \'None\'], "
   }
   member_method {
-    name: "set_random_seed"
+    name: "set_seed"
     argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 55a2804c80..f002418020 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -39,6 +39,7 @@ renames = {
     'tf.GRAPH_DEF_VERSION': 'tf.version.GRAPH_DEF_VERSION',
     'tf.GRAPH_DEF_VERSION_MIN_CONSUMER': 'tf.version.GRAPH_DEF_VERSION_MIN_CONSUMER',
     'tf.GRAPH_DEF_VERSION_MIN_PRODUCER': 'tf.version.GRAPH_DEF_VERSION_MIN_PRODUCER',
+    'tf.GraphKeys': 'tf.compat.v1.GraphKeys',
     'tf.IdentityReader': 'tf.compat.v1.IdentityReader',
     'tf.InteractiveSession': 'tf.compat.v1.InteractiveSession',
     'tf.LMDBReader': 'tf.compat.v1.LMDBReader',
@@ -198,7 +199,9 @@ renames = {
     'tf.image.resize_area': 'tf.compat.v1.image.resize_area',
     'tf.image.resize_bicubic': 'tf.compat.v1.image.resize_bicubic',
     'tf.image.resize_bilinear': 'tf.compat.v1.image.resize_bilinear',
+    'tf.image.resize_images': 'tf.compat.v1.image.resize_images',
     'tf.image.resize_nearest_neighbor': 'tf.compat.v1.image.resize_nearest_neighbor',
+    'tf.image.transpose_image': 'tf.compat.v1.image.transpose_image',
     'tf.initialize_all_tables': 'tf.compat.v1.initialize_all_tables',
     'tf.initialize_all_variables': 'tf.compat.v1.initialize_all_variables',
     'tf.initialize_local_variables': 'tf.compat.v1.initialize_local_variables',
@@ -354,6 +357,7 @@ renames = {
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
     'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
     'tf.random.multinomial': 'tf.compat.v1.random.multinomial',
+    'tf.random.set_random_seed': 'tf.compat.v1.random.set_random_seed',
     'tf.random.stateless_multinomial': 'tf.compat.v1.random.stateless_multinomial',
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
@@ -437,7 +441,7 @@ renames = {
     'tf.serialize_many_sparse': 'tf.io.serialize_many_sparse',
     'tf.serialize_sparse': 'tf.io.serialize_sparse',
     'tf.serialize_tensor': 'tf.io.serialize_tensor',
-    'tf.set_random_seed': 'tf.random.set_random_seed',
+    'tf.set_random_seed': 'tf.compat.v1.set_random_seed',
     'tf.setdiff1d': 'tf.compat.v1.setdiff1d',
     'tf.sets.set_difference': 'tf.sets.difference',
     'tf.sets.set_intersection': 'tf.sets.intersection',
@@ -449,6 +453,7 @@ renames = {
     'tf.sparse.merge': 'tf.compat.v1.sparse.merge',
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
     'tf.sparse.reduce_max_sparse': 'tf.compat.v1.sparse.reduce_max_sparse',
+    'tf.sparse.reduce_sum_sparse': 'tf.compat.v1.sparse.reduce_sum_sparse',
     'tf.sparse_add': 'tf.compat.v1.sparse_add',
     'tf.sparse_fill_empty_rows': 'tf.sparse.fill_empty_rows',
     'tf.sparse_mask': 'tf.sparse.mask',
@@ -459,8 +464,8 @@ renames = {
     'tf.sparse_placeholder': 'tf.compat.v1.sparse_placeholder',
     'tf.sparse_reduce_max': 'tf.compat.v1.sparse_reduce_max',
     'tf.sparse_reduce_max_sparse': 'tf.compat.v1.sparse_reduce_max_sparse',
-    'tf.sparse_reduce_sum': 'tf.sparse.reduce_sum',
-    'tf.sparse_reduce_sum_sparse': 'tf.sparse.reduce_sum_sparse',
+    'tf.sparse_reduce_sum': 'tf.compat.v1.sparse_reduce_sum',
+    'tf.sparse_reduce_sum_sparse': 'tf.compat.v1.sparse_reduce_sum_sparse',
     'tf.sparse_reorder': 'tf.sparse.reorder',
     'tf.sparse_reset_shape': 'tf.sparse.reset_shape',
     'tf.sparse_reshape': 'tf.sparse.reshape',
@@ -496,6 +501,15 @@ renames = {
     'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
     'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
     'tf.string_to_number': 'tf.strings.to_number',
+    'tf.summary.audio': 'tf.compat.v1.summary.audio',
+    'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
+    'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
+    'tf.summary.image': 'tf.compat.v1.summary.image',
+    'tf.summary.merge': 'tf.compat.v1.summary.merge',
+    'tf.summary.merge_all': 'tf.compat.v1.summary.merge_all',
+    'tf.summary.scalar': 'tf.compat.v1.summary.scalar',
+    'tf.summary.tensor_summary': 'tf.compat.v1.summary.tensor_summary',
+    'tf.summary.text': 'tf.compat.v1.summary.text',
     'tf.svd': 'tf.linalg.svd',
     'tf.tables_initializer': 'tf.compat.v1.tables_initializer',
     'tf.test.compute_gradient': 'tf.compat.v1.test.compute_gradient',
-- 
GitLab


From a96a27e92faa3e908c2a5952d5f8f0dfcb6fa4d2 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 11:23:36 -0800
Subject: [PATCH 0585/1554] TFTRT: split log for adding TRT node to two lines

---
 .../contrib/tensorrt/convert/convert_graph.cc  | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 69c8be8b57..5038e74fdd 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -970,16 +970,9 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                 &graph, alloc.get(), &engine_nodes);
     // If status is ok, we successfully added the node to the graph and can
     // remove segment ops. Otherwise graph is not modified.
-    string msg = StrCat("Adding TensorRT node ", engine.engine_name, " for segment ",
-                        i, ", composed of ",
+    string msg = StrCat("Added TensorRT node ", engine.engine_name, " for segment ",
+                        i, " consisting ",
                         converted_segments.at(i).first.size(), " nodes");
-    if (VLOG_IS_ON(1)) {
-      StrAppend(&msg, " (");
-      for (const string& node_name : converted_segments.at(i).first) {
-        StrAppend(&msg, node_name, ", ");
-      }
-      StrAppend(&msg, ")");
-    }
     if (status.ok()) {
       LOG(INFO) << msg << " succeeded.";
       for (auto node_name : converted_segments.at(i).first) {
@@ -987,8 +980,13 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       }
     } else {
       // Graph is not modified.
-      LOG(WARNING) << msg << " failed: " << status << ". Skipping...";
+      LOG(WARNING) << msg << " failed: " << status << ". Fallback to TF...";
+    }
+    msg = "Segment consists of nodes: ";
+    for (const string& node_name : converted_segments.at(i).first) {
+      StrAppend(&msg, node_name, ", ");
     }
+    VLOG(1) << msg;
   }
   cudaSetDevice(old_cuda_device);
   graph.ToGraphDef(params.output_graph_def);
-- 
GitLab


From 5708ff3b7535cc22c054afe66d98beb71211603b Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 11:25:54 -0800
Subject: [PATCH 0586/1554] TFTRT: fix alignment

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 2 +-
 tensorflow/contrib/tensorrt/segment/segment.cc       | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 5038e74fdd..d688ebd7b4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -877,7 +877,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       [](const Edge* edge) { return true; }, OutputEdgeValidator(),
       segment_options, &initial_segments));
   LOG(INFO) << "Number of TensorRT candidate segments: "
-    << initial_segments.size();
+            << initial_segments.size();
 
   // Get the EngineInfo for each segment.
   std::unordered_map<string, tensorflow::Node*> node_map;
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 9ee7b76e30..ab44718256 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -442,7 +442,8 @@ tensorflow::Status SegmentGraph(
     msg += elem + ", ";
   }
   LOG(INFO) << msg << "(For more information see "
-    << "https://docs.nvidia.com/deeplearning/dgx/integrate-tf-trt/index.html#support-ops).";
+            << "https://docs.nvidia.com/deeplearning"
+            << "/dgx/integrate-tf-trt/index.html#support-ops).";
 
   // The segmentation algorithm below visits nodes in reverse topological order
   // and attempts to merge nodes along output edges. That means that subgraphs
-- 
GitLab


From d1fec4590b803a51765706d9d7a35c0a1e917ac1 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 20 Nov 2018 11:03:44 -0800
Subject: [PATCH 0587/1554] Support calling AddN on nested TensorLists. Disable
 SimplifyAggregation optimizer when aggregation variants.

PiperOrigin-RevId: 222273920
---
 .../optimizers/arithmetic_optimizer.cc        |  4 ++-
 tensorflow/core/kernels/list_kernels.h        | 15 +++++++--
 .../kernel_tests/control_flow_ops_py_test.py  | 14 ++++++++
 .../python/kernel_tests/list_ops_test.py      | 33 ++++++++++++++++++-
 4 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index cf294cd20b..566701ec2a 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -2309,7 +2309,9 @@ class SimplifyAggregation : public ArithmeticOptimizerStage {
   ~SimplifyAggregation() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
-    return IsAggregate(*node) && NumNonControlInputs(*node) > 0;
+    return IsAggregate(*node) && NumNonControlInputs(*node) > 0 &&
+           GetDataTypeFromAttr(*node, "T") !=
+               DT_VARIANT;  // TODO(b/119787146): Enable for variants.
   }
 
   Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index c2591f5314..75d91aff49 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -374,8 +374,12 @@ Status TensorListBinaryAdd(OpKernelContext* c, const TensorList& a,
           b_tensor.shape().DebugString(), " in position ", i);
     }
     Tensor out_tensor;
-    TF_RETURN_IF_ERROR(
-        c->allocate_temp(a_tensor.dtype(), a_tensor.shape(), &out_tensor));
+    AllocatorAttributes attr;
+    if (a_tensor.dtype() == DT_VARIANT) {
+      attr.set_on_host(true);
+    }
+    TF_RETURN_IF_ERROR(c->allocate_temp(a_tensor.dtype(), a_tensor.shape(),
+                                        &out_tensor, attr));
     out->tensors.push_back(out_tensor);
     switch (out_tensor.dtype()) {
 #define DTYPE_CASE(dtype)                                        \
@@ -387,6 +391,13 @@ Status TensorListBinaryAdd(OpKernelContext* c, const TensorList& a,
       TF_CALL_NUMBER_TYPES(DTYPE_CASE)
 
 #undef DTYPE_CASE
+      case DataTypeToEnum<Variant>::value: {
+        Variant* v_out = &(out_tensor.scalar<Variant>()());
+        TF_RETURN_IF_ERROR(BinaryOpVariants<Device>(
+            c, ADD_VARIANT_BINARY_OP, a_tensor.scalar<Variant>()(),
+            b_tensor.scalar<Variant>()(), v_out));
+        break;
+      }
       default:
         return errors::InvalidArgument("Trying to add unsupported dtype ",
                                        out_tensor.dtype());
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 0d6d2cc6da..3394df20d8 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -435,6 +435,20 @@ class ControlFlowTest(test.TestCase):
 
       self.assertEqual(1.0, control_flow_ops.cond(rv, case, lambda: t).eval())
 
+  def testCondWithTensorArrayGrad(self):
+    with self.cached_session() as sess:
+      with ops.device(test.gpu_device_name()):
+        pred = array_ops.placeholder(dtypes.bool, [])
+        x = constant_op.constant([1.0, 2.0, 3.0])
+        y = control_flow_ops.cond(
+            pred, lambda: functional_ops.map_fn(lambda z: z * 2.0, x),
+            lambda: constant_op.constant([1.0, 1.0, 1.0]))
+        g = gradients_impl.gradients(y, x)[0]
+
+      self.assertAllEqual(sess.run(g, {pred: True}), [2.0, 2.0, 2.0])
+      # TODO(b/119791601): Enable this.
+      # self.assertAllEqual(sess.run(g, {pred: False}), [0.0, 0.0, 0.0])
+
   @test_util.disable_control_flow_v2("b/113293074")
   def testCondIndexedSlicesDifferentTypes(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 2bc8ba463b..09cb5cf0ba 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -29,8 +29,8 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradients_impl
@@ -834,6 +834,37 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(result[:2], [-1, 5])
     self.assertIs(result[2], t)
 
+  def testAddN(self):
+    l1 = list_ops.tensor_list_from_tensor([1.0, 2.0], element_shape=[])
+    l2 = list_ops.tensor_list_from_tensor([3.0, 4.0], element_shape=[])
+    l3 = list_ops.tensor_list_from_tensor([5.0, 6.0], element_shape=[])
+    result = math_ops.add_n((l1, l2, l3))
+    result_t = list_ops.tensor_list_stack(result, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(result_t), [9., 12.])
+
+  def testAddNNestedList(self):
+    l1 = list_ops.tensor_list_from_tensor([1.0, 2.0], element_shape=[])
+    l2 = list_ops.tensor_list_from_tensor([3.0, 4.0], element_shape=[])
+    l3 = list_ops.tensor_list_from_tensor([5.0, 6.0], element_shape=[])
+    l4 = list_ops.tensor_list_from_tensor([7.0, 8.0], element_shape=[])
+    a = list_ops.empty_tensor_list(
+        element_dtype=dtypes.variant, element_shape=[])
+    a = list_ops.tensor_list_push_back(a, l1)
+    a = list_ops.tensor_list_push_back(a, l2)
+    b = list_ops.empty_tensor_list(
+        element_dtype=dtypes.variant, element_shape=[])
+    b = list_ops.tensor_list_push_back(b, l3)
+    b = list_ops.tensor_list_push_back(b, l4)
+    result = math_ops.add_n((a, b))
+    result_0 = list_ops.tensor_list_stack(
+        list_ops.tensor_list_get_item(result, 0, element_dtype=dtypes.variant),
+        element_dtype=dtypes.float32)
+    result_1 = list_ops.tensor_list_stack(
+        list_ops.tensor_list_get_item(result, 1, element_dtype=dtypes.variant),
+        element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(result_0), [6., 8.])
+    self.assertAllEqual(self.evaluate(result_1), [10., 12.])
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From b07a00ab431cced4f51984e824e474df07ba6d57 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Tue, 20 Nov 2018 11:13:22 -0800
Subject: [PATCH 0588/1554] Fix one typo.

PiperOrigin-RevId: 222275519
---
 tensorflow/core/protobuf/rewriter_config.proto | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index d68f273536..515d673828 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -38,7 +38,7 @@ message RewriterConfig {
   }
 
   // Enum controlling the number of times to run optimizers. The default is to
-  // run them once.
+  // run them twice.
   enum NumIterationsType {
     DEFAULT_NUM_ITERS = 0;
     ONE = 1;
-- 
GitLab


From 3478a4e928431b45cec68c781cf2b0cbd94015fd Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 20 Nov 2018 11:13:30 -0800
Subject: [PATCH 0589/1554] Internal Cleanup.

PiperOrigin-RevId: 222275534
---
 tensorflow/python/ops/array_ops.py            |  70 +++++-
 tensorflow/python/ops/control_flow_ops.py     | 226 +++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |   6 +-
 .../tools/compatibility/tf_upgrade_v2.py      |   6 +
 4 files changed, 302 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 85a2c9192d..ed050d740e 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -1507,7 +1507,75 @@ def split(value, num_or_size_splits, axis=0, num=None, name="split"):
       value=value, size_splits=size_splits, axis=axis, num_split=num, name=name)
 
 
-@tf_export("transpose")
+@tf_export("transpose", v1=[])
+def transpose_v2(a, perm=None, conjugate=False, name="transpose"):
+  """Transposes `a`. Permutes the dimensions according to `perm`.
+
+  The returned tensor's dimension i will correspond to the input dimension
+  `perm[i]`. If `perm` is not given, it is set to (n-1...0), where n is
+  the rank of the input tensor. Hence by default, this operation performs a
+  regular matrix transpose on 2-D input Tensors. If conjugate is True and
+  `a.dtype` is either `complex64` or `complex128` then the values of `a`
+  are conjugated and transposed.
+
+  @compatibility(numpy)
+  In `numpy` transposes are memory-efficient constant time operations as they
+  simply return a new view of the same data with adjusted `strides`.
+
+  TensorFlow does not support strides, so `transpose` returns a new tensor with
+  the items permuted.
+  @end_compatibility
+
+  For example:
+
+  ```python
+  x = tf.constant([[1, 2, 3], [4, 5, 6]])
+  tf.transpose(x)  # [[1, 4]
+                   #  [2, 5]
+                   #  [3, 6]]
+
+  # Equivalently
+  tf.transpose(x, perm=[1, 0])  # [[1, 4]
+                                #  [2, 5]
+                                #  [3, 6]]
+
+  # If x is complex, setting conjugate=True gives the conjugate transpose
+  x = tf.constant([[1 + 1j, 2 + 2j, 3 + 3j],
+                   [4 + 4j, 5 + 5j, 6 + 6j]])
+  tf.transpose(x, conjugate=True)  # [[1 - 1j, 4 - 4j],
+                                   #  [2 - 2j, 5 - 5j],
+                                   #  [3 - 3j, 6 - 6j]]
+
+  # 'perm' is more useful for n-dimensional tensors, for n > 2
+  x = tf.constant([[[ 1,  2,  3],
+                    [ 4,  5,  6]],
+                   [[ 7,  8,  9],
+                    [10, 11, 12]]])
+
+  # Take the transpose of the matrices in dimension-0
+  # (this common operation has a shorthand `linalg.transpose`)
+  tf.transpose(x, perm=[0, 2, 1])  # [[[1,  4],
+                                   #   [2,  5],
+                                   #   [3,  6]],
+                                   #  [[7, 10],
+                                   #   [8, 11],
+                                   #   [9, 12]]]
+  ```
+
+  Args:
+    a: A `Tensor`.
+    perm: A permutation of the dimensions of `a`.
+    conjugate: Optional bool. Setting it to `True` is mathematically equivalent
+      to tf.conj(tf.transpose(input)).
+    name: A name for the operation (optional).
+
+  Returns:
+    A transposed `Tensor`.
+  """
+  return transpose(a=a, perm=perm, name=name, conjugate=conjugate)
+
+
+@tf_export(v1=["transpose"])
 def transpose(a, perm=None, name="transpose", conjugate=False):
   """Transposes `a`. Permutes the dimensions according to `perm`.
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index eab9b3f993..4417632e69 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -3136,7 +3136,193 @@ class WhileContext(ControlFlowContext):
 
 
 # pylint: disable=redefined-outer-name
-@tf_export("while_loop")
+@tf_export("while_loop", v1=[])
+def while_loop_v2(cond,
+                  body,
+                  loop_vars,
+                  shape_invariants=None,
+                  parallel_iterations=10,
+                  back_prop=True,
+                  swap_memory=False,
+                  maximum_iterations=None,
+                  return_same_structure=False,
+                  name=None):
+  """Repeat `body` while the condition `cond` is true.
+
+  `cond` is a callable returning a boolean scalar tensor. `body` is a callable
+  returning a (possibly nested) tuple, namedtuple or list of tensors of the same
+  arity (length and structure) and types as `loop_vars`. `loop_vars` is a
+  (possibly nested) tuple, namedtuple or list of tensors that is passed to both
+  `cond` and `body`. `cond` and `body` both take as many arguments as there are
+  `loop_vars`.
+
+  In addition to regular Tensors or IndexedSlices, the body may accept and
+  return TensorArray objects.  The flows of the TensorArray objects will
+  be appropriately forwarded between loops and during gradient calculations.
+
+  Note that `while_loop` calls `cond` and `body` *exactly once* (inside the
+  call to `while_loop`, and not at all during `Session.run()`). `while_loop`
+  stitches together the graph fragments created during the `cond` and `body`
+  calls with some additional graph nodes to create the graph flow that
+  repeats `body` until `cond` returns false.
+
+  For correctness, `tf.while_loop()` strictly enforces shape invariants for
+  the loop variables. A shape invariant is a (possibly partial) shape that
+  is unchanged across the iterations of the loop. An error will be raised
+  if the shape of a loop variable after an iteration is determined to be more
+  general than or incompatible with its shape invariant. For example, a shape
+  of [11, None] is more general than a shape of [11, 17], and [11, 21] is not
+  compatible with [11, 17]. By default (if the argument `shape_invariants` is
+  not specified), it is assumed that the initial shape of each tensor in
+  `loop_vars` is the same in every iteration. The `shape_invariants` argument
+  allows the caller to specify a less specific shape invariant for each loop
+  variable, which is needed if the shape varies between iterations. The
+  `tf.Tensor.set_shape`
+  function may also be used in the `body` function to indicate that
+  the output loop variable has a particular shape. The shape invariant for
+  SparseTensor and IndexedSlices are treated specially as follows:
+
+  a) If a loop variable is a SparseTensor, the shape invariant must be
+  TensorShape([r]) where r is the rank of the dense tensor represented
+  by the sparse tensor. It means the shapes of the three tensors of the
+  SparseTensor are ([None], [None, r], [r]). NOTE: The shape invariant here
+  is the shape of the SparseTensor.dense_shape property. It must be the shape of
+  a vector.
+
+  b) If a loop variable is an IndexedSlices, the shape invariant must be
+  a shape invariant of the values tensor of the IndexedSlices. It means
+  the shapes of the three tensors of the IndexedSlices are (shape, [shape[0]],
+  [shape.ndims]).
+
+  `while_loop` implements non-strict semantics, enabling multiple iterations
+  to run in parallel. The maximum number of parallel iterations can be
+  controlled by `parallel_iterations`, which gives users some control over
+  memory consumption and execution order. For correct programs, `while_loop`
+  should return the same result for any parallel_iterations > 0.
+
+  For training, TensorFlow stores the tensors that are produced in the
+  forward inference and are needed in back propagation. These tensors are a
+  main source of memory consumption and often cause OOM errors when training
+  on GPUs. When the flag swap_memory is true, we swap out these tensors from
+  GPU to CPU. This for example allows us to train RNN models with very long
+  sequences and large batches.
+
+  Args:
+    cond: A callable that represents the termination condition of the loop.
+    body: A callable that represents the loop body.
+    loop_vars: A (possibly nested) tuple, namedtuple or list of numpy array,
+      `Tensor`, and `TensorArray` objects.
+    shape_invariants: The shape invariants for the loop variables.
+    parallel_iterations: The number of iterations allowed to run in parallel. It
+      must be a positive integer.
+    back_prop: Whether backprop is enabled for this while loop.
+    swap_memory: Whether GPU-CPU memory swap is enabled for this loop.
+    maximum_iterations: Optional maximum number of iterations of the while loop
+      to run.  If provided, the `cond` output is AND-ed with an additional
+      condition ensuring the number of iterations executed is no greater than
+      `maximum_iterations`.
+    return_same_structure: If True, output has same structure as `loop_vars`. If
+      eager execution is enabled, this is ignored (and always treated as True).
+    name: Optional name prefix for the returned tensors.
+
+  Returns:
+    The output tensors for the loop variables after the loop.
+     If `return_same_structure` is True, the return value has the same
+     structure as `loop_vars`.
+     If `return_same_structure` is False, the return value is a Tensor,
+     TensorArray or IndexedSlice if the length of `loop_vars` is 1, or a list
+     otherwise.
+
+  Raises:
+    TypeError: if `cond` or `body` is not callable.
+    ValueError: if `loop_vars` is empty.
+
+  Example:
+
+  ```python
+  i = tf.constant(0)
+  c = lambda i: tf.less(i, 10)
+  b = lambda i: tf.add(i, 1)
+  r = tf.while_loop(c, b, [i])
+  ```
+
+  Example with nesting and a namedtuple:
+
+  ```python
+  import collections
+  Pair = collections.namedtuple('Pair', 'j, k')
+  ijk_0 = (tf.constant(0), Pair(tf.constant(1), tf.constant(2)))
+  c = lambda i, p: i < 10
+  b = lambda i, p: (i + 1, Pair((p.j + p.k), (p.j - p.k)))
+  ijk_final = tf.while_loop(c, b, ijk_0)
+  ```
+
+  Example using shape_invariants:
+
+  ```python
+  i0 = tf.constant(0)
+  m0 = tf.ones([2, 2])
+  c = lambda i, m: i < 10
+  b = lambda i, m: [i+1, tf.concat([m, m], axis=0)]
+  tf.while_loop(
+      c, b, loop_vars=[i0, m0],
+      shape_invariants=[i0.get_shape(), tf.TensorShape([None, 2])])
+  ```
+
+  Example which demonstrates non-strict semantics: In the following
+  example, the final value of the counter `i` does not depend on `x`. So
+  the `while_loop` can increment the counter parallel to updates of `x`.
+  However, because the loop counter at one loop iteration depends
+  on the value at the previous iteration, the loop counter itself cannot
+  be incremented in parallel. Hence if we just want the final value of the
+  counter (which we print on the line `print(sess.run(i))`), then
+  `x` will never be incremented, but the counter will be updated on a
+  single thread. Conversely, if we want the value of the output (which we
+  print on the line `print(sess.run(out).shape)`), then the counter may be
+  incremented on its own thread, while `x` can be incremented in
+  parallel on a separate thread. In the extreme case, it is conceivable
+  that the thread incrementing the counter runs until completion before
+  `x` is incremented even a single time. The only thing that can never
+  happen is that the thread updating `x` can never get ahead of the
+  counter thread because the thread incrementing `x` depends on the value
+  of the counter.
+
+  ```python
+  import tensorflow as tf
+
+  n = 10000
+  x = tf.constant(list(range(n)))
+  c = lambda i, x: i < n
+  b = lambda i, x: (tf.Print(i + 1, [i]), tf.Print(x + 1, [i], "x:"))
+  i, out = tf.while_loop(c, b, (0, x))
+  with tf.Session() as sess:
+      print(sess.run(i))  # prints [0] ... [9999]
+
+      # The following line may increment the counter and x in parallel.
+      # The counter thread may get ahead of the other thread, but not the
+      # other way around. So you may see things like
+      # [9996] x:[9987]
+      # meaning that the counter thread is on iteration 9996,
+      # while the other thread is on iteration 9987
+      print(sess.run(out).shape)
+  ```
+
+  """
+  return while_loop(
+      cond=cond,
+      body=body,
+      loop_vars=loop_vars,
+      shape_invariants=shape_invariants,
+      parallel_iterations=parallel_iterations,
+      back_prop=back_prop,
+      swap_memory=swap_memory,
+      name=name,
+      maximum_iterations=maximum_iterations,
+      return_same_structure=return_same_structure)
+
+
+# pylint: disable=redefined-outer-name
+@tf_export(v1=["while_loop"])
 def while_loop(cond,
                body,
                loop_vars,
@@ -3536,7 +3722,43 @@ def group(*inputs, **kwargs):
       return no_op(name=name)
 
 
-@tf_export("tuple")
+@tf_export("tuple", v1=[])
+def tuple_v2(tensors, control_inputs=None, name=None):
+  """Group tensors together.
+
+  This creates a tuple of tensors with the same values as the `tensors`
+  argument, except that the value of each tensor is only returned after the
+  values of all tensors have been computed.
+
+  `control_inputs` contains additional ops that have to finish before this op
+  finishes, but whose outputs are not returned.
+
+  This can be used as a "join" mechanism for parallel computations: all the
+  argument tensors can be computed in parallel, but the values of any tensor
+  returned by `tuple` are only available after all the parallel computations
+  are done.
+
+  See also `tf.group` and
+  `tf.control_dependencies`.
+
+  Args:
+    tensors: A list of `Tensor`s or `IndexedSlices`, some entries can be `None`.
+    control_inputs: List of additional ops to finish before returning.
+    name: (optional) A name to use as a `name_scope` for the operation.
+
+  Returns:
+    Same as `tensors`.
+
+  Raises:
+    ValueError: If `tensors` does not contain any `Tensor` or `IndexedSlices`.
+    TypeError: If `control_inputs` is not a list of `Operation` or `Tensor`
+      objects.
+
+  """
+  return tuple(tensors=tensors, name=name, control_inputs=control_inputs)  # pylint: disable=redefined-builtin
+
+
+@tf_export(v1=["tuple"])
 def tuple(tensors, name=None, control_inputs=None):  # pylint: disable=redefined-builtin
   """Group tensors together.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index b03c8c212d..6adce4b3fd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1110,7 +1110,7 @@ tf_module {
   }
   member_method {
     name: "transpose"
-    argspec: "args=[\'a\', \'perm\', \'name\', \'conjugate\'], varargs=None, keywords=None, defaults=[\'None\', \'transpose\', \'False\'], "
+    argspec: "args=[\'a\', \'perm\', \'conjugate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'transpose\'], "
   }
   member_method {
     name: "truediv"
@@ -1130,7 +1130,7 @@ tf_module {
   }
   member_method {
     name: "tuple"
-    argspec: "args=[\'tensors\', \'name\', \'control_inputs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'tensors\', \'control_inputs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "unique"
@@ -1162,7 +1162,7 @@ tf_module {
   }
   member_method {
     name: "while_loop"
-    argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'name\', \'maximum_iterations\', \'return_same_structure\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'None\', \'False\'], "
+    argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'maximum_iterations\', \'return_same_structure\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "zeros"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 1074ba5702..3cef1f1602 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -219,6 +219,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
         "tf.strings.length": ["input", "name", "unit"],
+        "tf.transpose": ["a", "perm", "name", "conjugate"],
+        "tf.tuple": ["tensors", "name", "control_inputs"],
+        "tf.while_loop": ["cond", "body", "loop_vars", "shape_invariants",
+                          "parallel_iterations", "back_prop", "swap_memory",
+                          "name", "maximum_iterations",
+                          "return_same_structure"],
     }
 
     # Specially handled functions.
-- 
GitLab


From 97a16c7b0237b4da9276981515d13a38b0de8019 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 20 Nov 2018 11:14:45 -0800
Subject: [PATCH 0590/1554] [tf.data] Propagate private threadpool size
 parameter during iterator creation so that it is (actually) used as the
 maximum degree of parallelism for autotuning.

PiperOrigin-RevId: 222275729
---
 .../experimental/threadpool_dataset_op.cc     | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index ab21dfc6bc..7bd393f0f4 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -187,20 +187,15 @@ class ThreadPoolDatasetOp : public UnaryDatasetOpKernel {
           : DatasetIterator<Dataset>(params) {}
 
       Status Initialize(IteratorContext* ctx) override {
-        return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_);
+        return dataset()->input_->MakeIterator(
+            IteratorContext(CreateParams(ctx)), prefix(), &input_impl_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
                              std::vector<Tensor>* out_tensors,
                              bool* end_of_sequence) override {
-        ThreadPoolResource* pool = dataset()->threadpool_;
-        IteratorContext::Params params(ctx);
-        params.runner = [pool](std::function<void()> c) {
-          pool->Schedule(std::move(c));
-        };
-        params.runner_threadpool_size = pool->NumThreads();
-        IteratorContext iter_ctx(params);
-        return input_impl_->GetNext(&iter_ctx, out_tensors, end_of_sequence);
+        return input_impl_->GetNext(IteratorContext(CreateParams(ctx)),
+                                    out_tensors, end_of_sequence);
       }
 
      protected:
@@ -211,6 +206,16 @@ class ThreadPoolDatasetOp : public UnaryDatasetOpKernel {
       }
 
      private:
+      IteratorContext::Params CreateParams(IteratorContext* ctx) {
+        ThreadPoolResource* pool = dataset()->threadpool_;
+        IteratorContext::Params params(ctx);
+        params.runner = [pool](std::function<void()> c) {
+          pool->Schedule(std::move(c));
+        };
+        params.runner_threadpool_size = pool->NumThreads();
+        return params;
+      }
+
       std::unique_ptr<IteratorBase> input_impl_;
     };
 
-- 
GitLab


From 3c28434fe21b5feb29973632f393a27b0b5afd84 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 11:49:27 -0800
Subject: [PATCH 0591/1554] TFTRT: use StrCat and StrAppend in the log

---
 tensorflow/contrib/tensorrt/segment/segment.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index ab44718256..917ff41fbf 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -33,6 +33,7 @@ namespace tensorflow {
 namespace tensorrt {
 namespace segment {
 using ::tensorflow::strings::StrAppend;
+using ::tensorflow::strings::StrCat;
 
 // A simple graph representation to mirror tensorflow::Graph. This structure
 // helps saving memory since segmenter modifies the graph in place, preventing
@@ -433,13 +434,12 @@ tensorflow::Status SegmentGraph(
     }
     node_segments.emplace_back(node);
   }
-  string msg = "There are "
-               + std::to_string(num_unsupported_ops)
-               + " unsupported ops of "
-               + std::to_string(unsupported_ops.size())
-               + " different types in the graph: ";
+  string msg = StrCat("There are ", num_unsupported_ops,
+                      " ops of ", unsupported_ops.size(),
+                      " different types in the graph that",
+                      " are not converted to TensorRT: ");
   for (const auto& elem: unsupported_ops) {
-    msg += elem + ", ";
+    StrAppend(&msg, elem, ", ");
   }
   LOG(INFO) << msg << "(For more information see "
             << "https://docs.nvidia.com/deeplearning"
-- 
GitLab


From a51e09caf77453c4713686888082dae7b6cfa5f8 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Tue, 20 Nov 2018 11:21:15 -0800
Subject: [PATCH 0592/1554] Log when a new FuncGraph is created.

PiperOrigin-RevId: 222276868
---
 tensorflow/python/eager/function.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index e863cf57bc..609a340915 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -48,6 +48,7 @@ from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
@@ -66,6 +67,11 @@ WHITELIST_FUNCTION_ATTRIBUTE_REGEX = [
     BACKWARD_FUNCTION_ATTRIBUTE_NAME
 ]
 
+CacheKey = collections.namedtuple("CacheKey", [
+    "input_signature", "parent_graph", "device_functions", "colocation_stack",
+    "uses_xla"
+])
+
 
 def _parse_func_attrs(attributes):
   """Convert the keyword arguments into function_def attributes.
@@ -927,17 +933,17 @@ class PolymorphicFunction(object):
     """Computes the cache key given inputs and execution context."""
     if self._input_signature is None:
       inputs = (args, kwargs) if kwargs else args
-      cache_key = pywrap_tensorflow.TFE_Py_EncodeArg(inputs)
+      input_signature = pywrap_tensorflow.TFE_Py_EncodeArg(inputs)
     else:
       del args, kwargs
-      cache_key = self._flat_input_signature
+      input_signature = self._flat_input_signature
 
     ctx = context.context()
     with ops.init_scope():
       # The graph, or whether we're executing eagerly, should be a part of the
       # cache key so we don't improperly capture tensors such as variables.
       executing_eagerly = ctx.executing_eagerly()
-      execution_context = executing_eagerly or ops.get_default_graph()
+      parent_graph = None if executing_eagerly else ops.get_default_graph()
 
     # pylint: disable=protected-access
     default_graph = ops.get_default_graph()
@@ -966,8 +972,8 @@ class PolymorphicFunction(object):
       else:
         device_functions = ()
     # pylint: enable=protected-access
-    return (cache_key, execution_context, device_functions, colocation_stack,
-            uses_xla)
+    return CacheKey(input_signature, parent_graph, device_functions,
+                    colocation_stack, uses_xla)
 
   def _canonicalize_function_inputs(self, *args, **kwargs):
     """Canonicalizes `args` and `kwargs`.
@@ -1083,6 +1089,9 @@ class PolymorphicFunction(object):
                         "must be hashable.")
 
       if graph_function is None:
+        logging.vlog(1,
+                     "Creating new FuncGraph for Python function %r (key: %r)",
+                     self._python_function.__name__, cache_key)
         if self._input_signature is None:
           arglen = len(args)
         else:
-- 
GitLab


From dadd38ede6fa5a039a14e01714da9eff2dd00459 Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Tue, 20 Nov 2018 11:22:33 -0800
Subject: [PATCH 0593/1554] Add tensor_name field in VirtualScheduler
 CreateSendRecv

So that we can compare ChannelCostEstimator estimation with measured RecvTensor
time in StepStats.

PiperOrigin-RevId: 222277076
---
 .../core/grappler/costs/virtual_scheduler.cc  | 21 ++++++++++++-------
 .../core/grappler/costs/virtual_scheduler.h   |  3 ++-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index b9b240e72c..ae5200b359 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -469,8 +469,8 @@ Status VirtualScheduler::Init() {
         } else {
           // Different device, no cached copy; transfer input_node to the
           // curr_node's device.
-          auto send_and_recv =
-              CreateSendRecv(input_node, curr_node, input_node_name);
+          auto send_and_recv = CreateSendRecv(input_node, curr_node, input_node,
+                                              input_node_name);
           // Note that CreateSendRecv() already connected input/output between
           // _Send and _Recv ops.
           const auto* send = send_and_recv.first;
@@ -608,7 +608,8 @@ string VirtualScheduler::ChannelDeviceName(const NodeDef* from,
 }
 
 std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
-    const NodeDef* from, const NodeDef* to, const string& input_name) {
+    const NodeDef* from, const NodeDef* to, const NodeDef* input_node,
+    const string& input_name) {
   CHECK(!initialized_) << "CreateSendRecv is called after Init().";
 
   // Connect "from" node to "to" node with _Send and _Recv such that
@@ -639,10 +640,14 @@ std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
   send->set_device(ChannelDeviceName(from, to));
   auto& send_attr = *(send->mutable_attr());
   send_attr[kAttrInputSrc].set_s(input_name);
-  // Use input_name as tensor_name.
-  send_attr[kAttrTensorName].set_s(input_name);
   send_attr[kAttrSrcDevice].set_s(DeviceName(from));
   send_attr[kAttrDstDevice].set_s(DeviceName(to));
+  // GraphDef generated by AutoGrappler has tensor_name field when removing
+  // _Send/_Recv nodes.
+  if (input_node->attr().count(kAttrTensorName)) {
+    send_attr[kAttrTensorName].set_s(
+        input_node->attr().at(kAttrTensorName).s());
+  }
 
   // _Recv op.
   auto* recv = new NodeDef();
@@ -652,8 +657,10 @@ std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
   recv->set_device(DeviceName(to));
   auto& recv_attr = *(recv->mutable_attr());
   recv_attr[kAttrInputSrc].set_s(input_name);
-  // Use input_name as tensor_name.
-  recv_attr[kAttrTensorName].set_s(input_name);
+  if (input_node->attr().count(kAttrTensorName)) {
+    recv_attr[kAttrTensorName].set_s(
+        input_node->attr().at(kAttrTensorName).s());
+  }
 
   // NodeState for _Send op.
   auto& send_node_state = GetNodeStateOrCreateIt(send);
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index 92e0a88782..6a835f32d1 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -317,7 +317,8 @@ class VirtualScheduler {
   void MaybeUpdateInputOutput(const NodeDef* node);
   NodeState& GetNodeStateOrCreateIt(const NodeDef* node);
   std::pair<const NodeDef*, const NodeDef*> CreateSendRecv(
-      const NodeDef* from, const NodeDef* to, const string& input_name);
+      const NodeDef* from, const NodeDef* to, const NodeDef* input_node,
+      const string& input_name);
   string DeviceName(const NodeDef* node) const;
   string SanitizedDeviceName(const NodeDef* node) const;
   string ChannelDeviceName(const NodeDef* from, const NodeDef* to) const;
-- 
GitLab


From a0b8cee815100b805a24fedfa12b28139d24e7fe Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Tue, 20 Nov 2018 11:24:30 -0800
Subject: [PATCH 0594/1554] Remove `seed2` arg in sample_distorted_bounding_box
 op for TF 2.0.

PiperOrigin-RevId: 222277431
---
 tensorflow/python/ops/image_ops_impl.py       | 110 +++++++++++++++++-
 .../api/golden/v2/tensorflow.image.pbtxt      |   2 +-
 2 files changed, 110 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index c4b5db9418..1618b79504 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -24,6 +24,7 @@ from tensorflow.python.compat import compat
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -1965,7 +1966,114 @@ def total_variation(images, name=None):
   return tot_var
 
 
-@tf_export('image.sample_distorted_bounding_box')
+@tf_export('image.sample_distorted_bounding_box', v1=[])
+def sample_distorted_bounding_box_v2(image_size,
+                                     bounding_boxes,
+                                     seed=0,
+                                     min_object_covered=0.1,
+                                     aspect_ratio_range=None,
+                                     area_range=None,
+                                     max_attempts=None,
+                                     use_image_if_no_bounding_boxes=None,
+                                     name=None):
+  """Generate a single randomly distorted bounding box for an image.
+
+  Bounding box annotations are often supplied in addition to ground-truth labels
+  in image recognition or object localization tasks. A common technique for
+  training such a system is to randomly distort an image while preserving
+  its content, i.e. *data augmentation*. This Op outputs a randomly distorted
+  localization of an object, i.e. bounding box, given an `image_size`,
+  `bounding_boxes` and a series of constraints.
+
+  The output of this Op is a single bounding box that may be used to crop the
+  original image. The output is returned as 3 tensors: `begin`, `size` and
+  `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
+  image. The latter may be supplied to `tf.image.draw_bounding_boxes` to
+  visualize what the bounding box looks like.
+
+  Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`.
+  The bounding box coordinates are floats in `[0.0, 1.0]` relative to the width
+  and height of the underlying image.
+
+  For example,
+
+  ```python
+      # Generate a single distorted bounding box.
+      begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
+          tf.shape(image),
+          bounding_boxes=bounding_boxes,
+          min_object_covered=0.1)
+
+      # Draw the bounding box in an image summary.
+      image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
+                                                    bbox_for_draw)
+      tf.summary.image('images_with_box', image_with_box)
+
+      # Employ the bounding box to distort the image.
+      distorted_image = tf.slice(image, begin, size)
+  ```
+
+  Note that if no bounding box information is available, setting
+  `use_image_if_no_bounding_boxes = true` will assume there is a single implicit
+  bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
+  false and no bounding boxes are supplied, an error is raised.
+
+  Args:
+    image_size: A `Tensor`. Must be one of the following types: `uint8`, `int8`,
+      `int16`, `int32`, `int64`.
+      1-D, containing `[height, width, channels]`.
+    bounding_boxes: A `Tensor` of type `float32`.
+      3-D with shape `[batch, N, 4]` describing the N bounding boxes
+      associated with the image.
+    seed: An optional `int`. Defaults to `0`.
+      If either `seed` or `seed2` are set to non-zero, the random number
+      generator is seeded by the given `seed`.  Otherwise, it is seeded by a
+      random seed.
+    min_object_covered: A Tensor of type `float32`. Defaults to `0.1`.
+      The cropped area of the image must contain at least this
+      fraction of any bounding box supplied. The value of this parameter should
+      be non-negative. In the case of 0, the cropped area does not need to
+      overlap any of the bounding boxes supplied.
+    aspect_ratio_range: An optional list of `floats`. Defaults to `[0.75,
+      1.33]`.
+      The cropped area of the image must have an aspect `ratio =
+      width / height` within this range.
+    area_range: An optional list of `floats`. Defaults to `[0.05, 1]`.
+      The cropped area of the image must contain a fraction of the
+      supplied image within this range.
+    max_attempts: An optional `int`. Defaults to `100`.
+      Number of attempts at generating a cropped region of the image
+      of the specified constraints. After `max_attempts` failures, return the
+      entire image.
+    use_image_if_no_bounding_boxes: An optional `bool`. Defaults to `False`.
+      Controls behavior if no bounding boxes supplied.
+      If true, assume an implicit bounding box covering the whole input. If
+      false, raise an error.
+    name: A name for the operation (optional).
+
+  Returns:
+    A tuple of `Tensor` objects (begin, size, bboxes).
+
+    begin: A `Tensor`. Has the same type as `image_size`. 1-D, containing
+    `[offset_height, offset_width, 0]`. Provide as input to
+      `tf.slice`.
+    size: A `Tensor`. Has the same type as `image_size`. 1-D, containing
+    `[target_height, target_width, -1]`. Provide as input to
+      `tf.slice`.
+    bboxes: A `Tensor` of type `float32`. 3-D with shape `[1, 1, 4]` containing
+    the distorted bounding box.
+    Provide as input to `tf.image.draw_bounding_boxes`.
+  """
+  seed1, seed2 = random_seed.get_seed(seed) if seed else (0, 0)
+  return sample_distorted_bounding_box(
+      image_size, bounding_boxes, seed1, seed2, min_object_covered,
+      aspect_ratio_range, area_range, max_attempts,
+      use_image_if_no_bounding_boxes, name)
+
+
+@tf_export(v1=['image.sample_distorted_bounding_box'])
+@deprecation.deprecated(date=None, instructions='`seed2` arg is deprecated.'
+                        'Use sample_distorted_bounding_box_v2 instead.')
 def sample_distorted_bounding_box(image_size,
                                   bounding_boxes,
                                   seed=None,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
index dcf80e3342..3c6ed1cfb8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt
@@ -206,7 +206,7 @@ tf_module {
   }
   member_method {
     name: "sample_distorted_bounding_box"
-    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'seed2\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0.1\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'0.1\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "sobel_edges"
-- 
GitLab


From 541ca6d17420e6f421e38e9d86204699a519eac9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 11:24:53 -0800
Subject: [PATCH 0595/1554] Multi-label quantile regression

PiperOrigin-RevId: 222277514
---
 .../estimator_batch/estimator.py              |  54 ++++--
 .../estimator_batch/estimator_test.py         | 159 +++++++++++++++++-
 .../boosted_trees/python/utils/losses.py      |   4 +-
 3 files changed, 198 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
index 99ecded653..a178820841 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
@@ -428,6 +428,7 @@ class GradientBoostedDecisionTreeQuantileRegressor(estimator.Estimator):
                learner_config,
                examples_per_layer,
                quantiles,
+               label_dimension=1,
                num_trees=None,
                feature_columns=None,
                weight_column_name=None,
@@ -448,6 +449,10 @@ class GradientBoostedDecisionTreeQuantileRegressor(estimator.Estimator):
         layer. It can also be a function that computes the number of examples
         based on the depth of the layer that's being built.
       quantiles: a list of quantiles for the loss, each between 0 and 1.
+      label_dimension: Dimension of regression label. This is the size
+        of the last dimension of the labels `Tensor` (typically, this has shape
+        `[batch_size, label_dimension]`). When label_dimension>1, it is
+        recommended to use multiclass strategy diagonal hessian or full hessian.
       num_trees: An int, number of trees to build.
       feature_columns: A list of feature columns.
       weight_column_name: Name of the column for weights, or None if not
@@ -489,9 +494,11 @@ class GradientBoostedDecisionTreeQuantileRegressor(estimator.Estimator):
           loss_fn=functools.partial(
               losses.per_example_quantile_regression_loss, quantile=quantile),
           link_fn=array_ops.identity,
-          logit_dimension=1)
+          logit_dimension=label_dimension)
       return head
 
+    learner_config.num_classes = max(2, label_dimension)
+
     super(GradientBoostedDecisionTreeQuantileRegressor, self).__init__(
         model_fn=model.model_builder,
         params={
@@ -548,6 +555,7 @@ def core_multiclass_head(
 # Core..QuantileRegressor directly,
 def core_quantile_regression_head(
     quantiles,
+    label_dimension=1,
     weight_column=None,
     loss_reduction=core_losses.Reduction.SUM_OVER_NONZERO_WEIGHTS):
   """Core head for quantile regression problems."""
@@ -562,7 +570,7 @@ def core_quantile_regression_head(
 
   # pylint:disable=protected-access
   head_fn = core_head_lib._regression_head(
-      label_dimension=1,
+      label_dimension=label_dimension,
       loss_fn=loss_fn,
       loss_reduction=loss_reduction,
       weight_column=weight_column)
@@ -747,6 +755,7 @@ class CoreGradientBoostedDecisionTreeQuantileRegressor(
                learner_config,
                examples_per_layer,
                quantiles,
+               label_dimension=1,
                num_trees=None,
                feature_columns=None,
                weight_column_name=None,
@@ -766,6 +775,10 @@ class CoreGradientBoostedDecisionTreeQuantileRegressor(
         layer. It can also be a function that computes the number of examples
         based on the depth of the layer that's being built.
       quantiles: a list of quantiles for the loss, each between 0 and 1.
+      label_dimension: Dimension of regression label. This is the size
+        of the last dimension of the labels `Tensor` (typically, this has shape
+        `[batch_size, label_dimension]`). When label_dimension>1, it is
+        recommended to use multiclass strategy diagonal hessian or full hessian.
       num_trees: An int, number of trees to build.
       feature_columns: A list of feature columns.
       weight_column_name: Name of the column for weights, or None if not
@@ -799,18 +812,31 @@ class CoreGradientBoostedDecisionTreeQuantileRegressor(
           mode=mode,
           config=config,
           params={
-              'head': core_quantile_regression_head(quantiles[0]),
-              'feature_columns': feature_columns,
-              'learner_config': learner_config,
-              'num_trees': num_trees,
-              'weight_column_name': weight_column_name,
-              'examples_per_layer': examples_per_layer,
-              'center_bias': center_bias,
-              'logits_modifier_function': logits_modifier_function,
-              'use_core_libs': True,
-              'output_leaf_index': output_leaf_index,
-              'override_global_step_value': None,
-              'num_quantiles': num_quantiles,
+              'head':
+                  core_quantile_regression_head(
+                      quantiles[0], label_dimension=label_dimension),
+              'feature_columns':
+                  feature_columns,
+              'learner_config':
+                  learner_config,
+              'num_trees':
+                  num_trees,
+              'weight_column_name':
+                  weight_column_name,
+              'examples_per_layer':
+                  examples_per_layer,
+              'center_bias':
+                  center_bias,
+              'logits_modifier_function':
+                  logits_modifier_function,
+              'use_core_libs':
+                  True,
+              'output_leaf_index':
+                  output_leaf_index,
+              'override_global_step_value':
+                  None,
+              'num_quantiles':
+                  num_quantiles,
           },
           output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC)
 
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index 7863b5a4f8..ee052ac603 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -81,7 +81,7 @@ def _infer_ranking_train_input_fn():
 _QUANTILE_REGRESSION_SIZE = 1000
 
 
-def _quantile_regression_input_fns():
+def _quantile_regression_input_fns(two_dimension=False):
   # The data generation is taken from
   # http://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_quantile.html
   np.random.seed(1)
@@ -90,20 +90,28 @@ def _quantile_regression_input_fns():
     """The function to predict."""
     return x * np.sin(x)
 
+  def g(x):
+    """The function to predict."""
+    return x * np.cos(x)
+
   #  Training data.
   x = np.atleast_2d(np.random.uniform(0, 10.0,
                                       size=_QUANTILE_REGRESSION_SIZE)).T
   x = x.astype(np.float32)
 
   # Labels.
-  y = f(x).ravel()
+  if not two_dimension:
+    y = f(x).ravel()
+  else:
+    y = np.column_stack((f(x).ravel(), g(x).ravel()))
 
   # Add random noise.
   dy = 1.5 + 1.0 * np.random.random(y.shape)
   noise = np.random.normal(0, dy)
   y += noise
   y_original = y.astype(np.float32)
-  y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1)
+  if not two_dimension:
+    y = y.reshape(_QUANTILE_REGRESSION_SIZE, 1)
 
   train_input_fn = numpy_io.numpy_input_fn(
       x=x,
@@ -439,6 +447,78 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     self.assertTrue(frac_above_lower >= 0.92)
     self.assertTrue(frac_above_lower <= 0.98)
 
+  # Multi-dimensional quantile regression.
+  def testQuantileRegressionMultiDimLabel(self):
+    learner_config = learner_pb2.LearnerConfig()
+    learner_config.num_classes = 2
+    learner_config.constraints.max_tree_depth = 3
+    learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
+    learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.tree_complexity = (
+        1.0 / _QUANTILE_REGRESSION_SIZE)
+
+    train_input_fn, test_input_fn, y = _quantile_regression_input_fns(
+        two_dimension=True)
+
+    # 95% percentile.
+    model_upper = estimator.GradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.95],
+        learner_config=learner_config,
+        label_dimension=2,
+        num_trees=100,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_upper.fit(input_fn=train_input_fn, steps=1000)
+    result_iter = model_upper.predict(input_fn=test_input_fn)
+    upper = []
+    for prediction_dict in result_iter:
+      upper.append(prediction_dict["scores"])
+
+    count_below_upper = np.count_nonzero(upper > y, axis=0)
+    count_both_below_upper = np.count_nonzero(np.prod(upper > y, axis=1))
+    frac_below_upper_0 = round(1. * count_below_upper[0] / len(y), 3)
+    frac_below_upper_1 = round(1. * count_below_upper[1] / len(y), 3)
+    frac_both_below_upper = round(1. * count_both_below_upper / len(y), 3)
+    # +/- 3%
+    self.assertTrue(frac_below_upper_0 >= 0.92)
+    self.assertTrue(frac_below_upper_0 <= 0.98)
+    self.assertTrue(frac_below_upper_1 >= 0.92)
+    self.assertTrue(frac_below_upper_1 <= 0.98)
+    self.assertTrue(frac_both_below_upper >= 0.92)
+    self.assertTrue(frac_both_below_upper <= 0.98)
+
+    train_input_fn, test_input_fn, _ = _quantile_regression_input_fns(
+        two_dimension=True)
+    model_lower = estimator.GradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.05],
+        learner_config=learner_config,
+        label_dimension=2,
+        num_trees=100,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_lower.fit(input_fn=train_input_fn, steps=1000)
+    result_iter = model_lower.predict(input_fn=test_input_fn)
+    lower = []
+    for prediction_dict in result_iter:
+      lower.append(prediction_dict["scores"])
+
+    count_above_lower = np.count_nonzero(lower < y, axis=0)
+    count_both_aboce_lower = np.count_nonzero(np.prod(lower < y, axis=1))
+    frac_above_lower_0 = round(1. * count_above_lower[0] / len(y), 3)
+    frac_above_lower_1 = round(1. * count_above_lower[1] / len(y), 3)
+    frac_both_above_lower = round(1. * count_both_aboce_lower / len(y), 3)
+    # +/- 3%
+    self.assertTrue(frac_above_lower_0 >= 0.92)
+    self.assertTrue(frac_above_lower_0 <= 0.98)
+    self.assertTrue(frac_above_lower_1 >= 0.92)
+    self.assertTrue(frac_above_lower_1 <= 0.98)
+    self.assertTrue(frac_both_above_lower >= 0.92)
+    self.assertTrue(frac_both_above_lower <= 0.98)
+
 
 class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
 
@@ -685,6 +765,79 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
     self.assertTrue(frac_above_lower >= 0.92)
     self.assertTrue(frac_above_lower <= 0.98)
 
+  # Multi-dimensional quantile regression.
+  def testQuantileRegressionMultiDimLabel(self):
+    learner_config = learner_pb2.LearnerConfig()
+    learner_config.num_classes = 2
+    learner_config.constraints.max_tree_depth = 3
+    learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
+    learner_config.constraints.min_node_weight = 1 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l2 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.l1 = 1.0 / _QUANTILE_REGRESSION_SIZE
+    learner_config.regularization.tree_complexity = (
+        1.0 / _QUANTILE_REGRESSION_SIZE)
+
+    train_input_fn, test_input_fn, y = _quantile_regression_input_fns(
+        two_dimension=True)
+    y = y.reshape(_QUANTILE_REGRESSION_SIZE, 2)
+
+    # 95% percentile.
+    model_upper = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.95],
+        learner_config=learner_config,
+        num_trees=100,
+        label_dimension=2,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_upper.train(input_fn=train_input_fn, steps=1000)
+    result_iter = model_upper.predict(input_fn=test_input_fn)
+    upper = []
+    for prediction_dict in result_iter:
+      upper.append(prediction_dict["predictions"])
+
+    count_below_upper = np.count_nonzero(upper > y, axis=0)
+    count_both_below_upper = np.count_nonzero(np.prod(upper > y, axis=1))
+    frac_below_upper_0 = round(1. * count_below_upper[0] / len(y), 3)
+    frac_below_upper_1 = round(1. * count_below_upper[1] / len(y), 3)
+    frac_both_below_upper = round(1. * count_both_below_upper / len(y), 3)
+    # +/- 3%
+    self.assertTrue(frac_below_upper_0 >= 0.92)
+    self.assertTrue(frac_below_upper_0 <= 0.98)
+    self.assertTrue(frac_below_upper_1 >= 0.92)
+    self.assertTrue(frac_below_upper_1 <= 0.98)
+    self.assertTrue(frac_both_below_upper >= 0.92)
+    self.assertTrue(frac_both_below_upper <= 0.98)
+
+    train_input_fn, test_input_fn, _ = _quantile_regression_input_fns(
+        two_dimension=True)
+    model_lower = estimator.CoreGradientBoostedDecisionTreeQuantileRegressor(
+        quantiles=[0.05],
+        learner_config=learner_config,
+        num_trees=100,
+        label_dimension=2,
+        examples_per_layer=_QUANTILE_REGRESSION_SIZE,
+        center_bias=False)
+
+    model_lower.train(input_fn=train_input_fn, steps=1000)
+    result_iter = model_lower.predict(input_fn=test_input_fn)
+    lower = []
+    for prediction_dict in result_iter:
+      lower.append(prediction_dict["predictions"])
+
+    count_above_lower = np.count_nonzero(lower < y, axis=0)
+    count_both_aboce_lower = np.count_nonzero(np.prod(lower < y, axis=1))
+    frac_above_lower_0 = round(1. * count_above_lower[0] / len(y), 3)
+    frac_above_lower_1 = round(1. * count_above_lower[1] / len(y), 3)
+    frac_both_above_lower = round(1. * count_both_aboce_lower / len(y), 3)
+    # +/- 3%
+    self.assertTrue(frac_above_lower_0 >= 0.92)
+    self.assertTrue(frac_above_lower_0 <= 0.98)
+    self.assertTrue(frac_above_lower_1 >= 0.92)
+    self.assertTrue(frac_above_lower_1 <= 0.98)
+    self.assertTrue(frac_both_above_lower >= 0.92)
+    self.assertTrue(frac_both_above_lower <= 0.98)
+
 
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py
index f8da20a54c..7a99dccdd1 100644
--- a/tensorflow/contrib/boosted_trees/python/utils/losses.py
+++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py
@@ -65,9 +65,9 @@ def per_example_quantile_regression_loss(labels, weights, predictions,
   below is this loss but squared in the region where the loss value < 1.
 
   Args:
-    labels: Rank 2 (N, 1) tensor of per-example labels.
+    labels: Rank 2 (N, D) tensor of per-example labels.
     weights: Rank 2 (N, 1) tensor of per-example weights.
-    predictions: Rank 2 (N, 1) tensor of per-example predictions.
+    predictions: Rank 2 (N, D) tensor of per-example predictions.
     quantile: The quantile to use.
 
   Returns:
-- 
GitLab


From 51f54a7a95cd4bef139967f12fe6ce67b8e63c83 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Nov 2018 11:25:44 -0800
Subject: [PATCH 0596/1554] Remove FilteredFunctionPassManager; NFC

PiperOrigin-RevId: 222277666
---
 .../compiler/xla/service/cpu/compiler_functor.cc    | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
index 2852fc8666..796a7cf94d 100644
--- a/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
+++ b/tensorflow/compiler/xla/service/cpu/compiler_functor.cc
@@ -61,17 +61,6 @@ Disabling these as a starting point.
 // TODO(b/64227304) Creating a custom pass pipeline will replace this.
 
 namespace {
-
-// TODO(sanjoy): remove this class.
-class FilteredFunctionPassManager : public llvm::legacy::FunctionPassManager {
- public:
-  explicit FilteredFunctionPassManager(llvm::Module* m)
-      : llvm::legacy::FunctionPassManager(m) {}
-  void add(llvm::Pass* p) override {
-    llvm::legacy::FunctionPassManager::add(p);
-  }
-};
-
 class FilteredPassManager : public llvm::legacy::PassManager {
  public:
   explicit FilteredPassManager(bool disable_expensive_passes)
@@ -94,7 +83,7 @@ class FilteredPassManager : public llvm::legacy::PassManager {
 std::unique_ptr<llvm::MemoryBuffer> CompilerFunctor::operator()(
     llvm::Module& module) const {
   FilteredPassManager module_passes(disable_expensive_passes_);
-  FilteredFunctionPassManager function_passes(&module);
+  llvm::legacy::FunctionPassManager function_passes(&module);
 
   VLOG(2) << "IR before optimizations";
   XLA_VLOG_LINES(2, llvm_ir::DumpModuleToString(module));
-- 
GitLab


From 8aeab3d5e13d2b4e4db3e4720d017d70a38c8394 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 11:33:19 -0800
Subject: [PATCH 0597/1554] This CL allows subclass models to work with float64
 datasets.

PiperOrigin-RevId: 222279047
---
 tensorflow/python/keras/BUILD                     |  1 +
 tensorflow/python/keras/engine/training.py        |  7 ++++++-
 .../python/keras/engine/training_dataset_test.py  | 15 ++++++++++++---
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index adfa2260c6..37dcc9eb04 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -728,6 +728,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index d926b53189..ce01777b2d 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1227,7 +1227,10 @@ class Model(Network):
       # to match the value shapes.
       if not self.inputs:
         is_build_called = True
-        self._set_inputs(x)
+        cast_inputs = x
+        if training_utils.has_tensors(x):
+          cast_inputs = training_utils.cast_if_floating_dtype(x)
+        self._set_inputs(cast_inputs)
     else:
       dict_inputs = isinstance(self.inputs, dict)
     if dict_inputs and context.executing_eagerly():
@@ -1243,6 +1246,8 @@ class Model(Network):
       if not self._is_compiled:
         # On-the-fly compilation of the model.
         # We need to use `y` to set the model targets.
+        if training_utils.has_tensors(y):
+          y = training_utils.cast_if_floating_dtype(y)
         if isinstance(y, (list, tuple)):
           if not all(isinstance(v, np.ndarray) or
                      tensor_util.is_tensor(v) for v in y):
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index e8b884e935..2e6bec6c62 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import logging
 
+from absl.testing import parameterized
+
 import numpy as np
 
 from tensorflow.python import keras
@@ -33,11 +35,18 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
-class TestTrainingWithDatasetIterators(test.TestCase):
+class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
 
+  @parameterized.parameters(
+      {'model': 'functional'},
+      {'model': 'subclass'},
+  )
   @tf_test_util.run_in_graph_and_eager_modes
-  def test_training_and_eval_methods_on_iterators_single_io(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+  def test_training_and_eval_methods_on_iterators_single_io(self, model):
+    if model == 'functional':
+      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    elif model == 'subclass':
+      model = testing_utils.get_small_sequential_mlp(1, 4)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-- 
GitLab


From 06d5348a3da499ecfb3745ec48cc1a8411fcf674 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 11:40:12 -0800
Subject: [PATCH 0598/1554] Add Jacobian API to GradientTape.

PiperOrigin-RevId: 222280321
---
 tensorflow/python/eager/BUILD                 |   1 +
 tensorflow/python/eager/backprop.py           | 109 ++++++++++++++++++
 tensorflow/python/eager/backprop_test.py      |  76 ++++++++++++
 .../golden/v1/tensorflow.-gradient-tape.pbtxt |   4 +
 .../golden/v2/tensorflow.-gradient-tape.pbtxt |   4 +
 5 files changed, 194 insertions(+)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 55728b19ab..d3457ed241 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -341,6 +341,7 @@ py_library(
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:execute",
         "//tensorflow/python/eager:tape",
+        "//tensorflow/python/ops/parallel_for:control_flow_ops",
         "@six_archive//:six",
     ],
 )
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 5b6b42155f..84b61f47c1 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -42,9 +42,20 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
+from tensorflow.python.util.lazy_loader import LazyLoader
 from tensorflow.python.util.tf_export import tf_export
 
 
+# Note that we need to lazy load the following two modules to avoid creating
+# circular dependencies.
+# TODO(b/119775953): fix the circular dependencies.
+pfor_ops = LazyLoader(
+    "pfor_ops", globals(),
+    "tensorflow.python.ops.parallel_for.control_flow_ops")
+
+function = LazyLoader("function", globals(),
+                      "tensorflow.python.eager.function")
+
 _op_attr_type_cache = {}
 
 
@@ -937,3 +948,101 @@ class GradientTape(object):
 
     grad = nest.pack_sequence_as(sources, flat_grad)
     return grad
+
+  def jacobian(self,
+               target,
+               sources,
+               unconnected_gradients=UnconnectedGradients.NONE,
+               experimental_use_pfor=True):
+    """Computes the jacobian using operations recorded in context of this tape.
+
+    See http://en.wikipedia.org/wiki/jacobian_matrix_and_determinant for the
+    definition of a Jacobian.
+
+    Example usage:
+
+    with tf.GradientTape() as g:
+      x  = tf.constant([1.0, 2.0])
+      g.watch(x)
+      y = x * x
+    jacobian = g.jacobian(y, x)
+    # jacobian value is [[2., 0.], [0., 4.]]
+
+    Args:
+      target: Tensor to be differentiated.
+      sources: a list or nested structure of Tensors or Variables. `target`
+        will be differentiated against elements in `sources`.
+      unconnected_gradients: a value which can either hold 'none' or 'zero' and
+        alters the value which will be returned if the target and sources are
+        unconnected. The possible values and effects are detailed in
+        'UnconnectedGradients' and it defaults to 'none'.
+      experimental_use_pfor: If true, vectorizes the jacobian computation. Else
+        falls back to a sequential while_loop. Vectorization can sometimes fail
+        or lead to excessive memory usage. This option can be used to disable
+        vectorization in such cases.
+
+    Returns:
+      a list or nested structure of Tensors (or IndexedSlices, or None),
+      one for each element in `sources`. Returned structure is the same as
+      the structure of `sources`.
+
+    Raises:
+      RuntimeError: If called on a non-persistent tape with eager execution
+        enabled and without enabling experimental_use_pfor.
+      ValueError: If vectorization of jacobian computation fails.
+    """
+    flat_sources = nest.flatten(sources)
+    target_static_shape = target.shape
+    target_shape = array_ops.shape(target)
+    # Note that we push and pop the tape here and below. This is needed since we
+    # need gradients through the enclosed operations.
+    self._push_tape()
+    target = array_ops.reshape(target, [-1])
+    self._pop_tape()
+
+    def loop_fn(i):
+      self._push_tape()
+      y = array_ops.gather(target, i)
+      self._pop_tape()
+      grad = self.gradient(y, flat_sources,
+                           unconnected_gradients=unconnected_gradients)
+      return grad
+
+    try:
+      target_size = int(target.shape[0])
+    except TypeError:
+      target_size = array_ops.shape(target)[0]
+
+    if experimental_use_pfor:
+      def f():
+        return pfor_ops.pfor(loop_fn, target_size)
+      if context.executing_eagerly():
+        f = function.defun(f)
+      try:
+        output = f()
+      except ValueError as err:
+        # TODO(agarwal): Fold this error message into err.
+        logging.error("Encountered an exception while vectorizing the jacobian "
+                      "computation. Vectorization can be disabled by setting "
+                      "experimental_use_pfor to False.")
+        raise err
+    else:
+      if context.executing_eagerly():
+        if not self._persistent:
+          raise RuntimeError(
+              "GradientTape must be created with persistent=True"
+              " to compute the jacobian with eager execution enabled and with "
+              " experimental_use_pfor set to False.")
+      output = pfor_ops.for_loop(
+          loop_fn, [target.dtype] * len(flat_sources), target_size)
+
+    for i, out in enumerate(output):
+      if out is not None:
+        new_shape = array_ops.concat(
+            [target_shape, array_ops.shape(out)[1:]], axis=0)
+        out = array_ops.reshape(out, new_shape)
+        if context.executing_eagerly():
+          out.set_shape(target_static_shape.concatenate(flat_sources[i].shape))
+      output[i] = out
+
+    return nest.pack_sequence_as(sources, output)
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index d9f2a95828..237b7f304e 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -1227,5 +1227,81 @@ class BackpropTest(test.TestCase):
       self.assertAllEqual(da[0], tf_da[0].eval())
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class JacobianTest(test.TestCase):
+
+  def _jacobian(self, experimental_use_pfor):
+    persistent = context.executing_eagerly and not experimental_use_pfor
+    with backprop.GradientTape(persistent=persistent) as g:
+      x = constant_op.constant([1., 2.])
+      y = constant_op.constant([3., 4.])
+      g.watch(x)
+      g.watch(y)
+      z = x * x * y
+    jacobian = g.jacobian(z, [x, y],
+                          experimental_use_pfor=experimental_use_pfor)
+    answer = [array_ops.diag(2 * x * y), array_ops.diag(x * x)]
+    return jacobian, answer
+
+  def testPfor(self):
+    jacobian, answer = self._jacobian(experimental_use_pfor=True)
+    for j, a in zip(jacobian, answer):
+      self.assertAllEqual(a, j)
+
+  def testWhileLoop(self):
+    jacobian, answer = self._jacobian(experimental_use_pfor=False)
+    for j, a in zip(jacobian, answer):
+      self.assertAllEqual(a, j)
+
+  def testPforDefun(self):
+
+    @function.defun
+    def _f():
+      return self._jacobian(experimental_use_pfor=True)
+
+    jacobian, answer = _f()
+    for j, a in zip(jacobian, answer):
+      self.assertAllEqual(a, j)
+
+  def testWhileLoopDefun(self):
+
+    @function.defun
+    def _f():
+      return self._jacobian(experimental_use_pfor=False)
+
+    jacobian, answer = _f()
+    for j, a in zip(jacobian, answer):
+      self.assertAllEqual(a, j)
+
+  def testPersistentTape(self):
+    if not context.executing_eagerly():
+      return
+    with backprop.GradientTape() as g:
+      x = constant_op.constant([1.0, 2.0])
+      g.watch(x)
+      y = x * x
+    with self.assertRaisesRegexp(RuntimeError, 'persistent'):
+      g.jacobian(y, x, experimental_use_pfor=False)
+
+  def testPforException(self):
+    var = variables.Variable([1.])
+
+    @custom_gradient.custom_gradient
+    def op(x):
+      def grad(_):
+        # Note that we perform a stateful operation here that will not be
+        # compatible with parallel for construct.
+        with ops.control_dependencies(
+            [var.assign(random_ops.random_uniform([1]))]):
+          return constant_op.constant(1.)
+      return x, grad
+
+    with backprop.GradientTape() as g:
+      x = constant_op.constant([1., 2.])
+      g.watch(x)
+      y = op(x)
+    with self.assertRaisesRegexp(ValueError, 'No converter'):
+      g.jacobian(y, x, experimental_use_pfor=True)
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
index 0a16d6ab92..50af42f4fc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
@@ -10,6 +10,10 @@ tf_class {
     name: "gradient"
     argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], "
   }
+  member_method {
+    name: "jacobian"
+    argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+  }
   member_method {
     name: "reset"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
index 0a16d6ab92..50af42f4fc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
@@ -10,6 +10,10 @@ tf_class {
     name: "gradient"
     argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], "
   }
+  member_method {
+    name: "jacobian"
+    argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+  }
   member_method {
     name: "reset"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 619b3f452923780de3a6164d5d692ee9343adeec Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 20 Nov 2018 11:41:25 -0800
Subject: [PATCH 0599/1554] Minor update to the unified LSTM.

The activation function (callable) can be passed into defun function, since its
is not tensor, it does not change the signature for the generated underlying
functions.

PiperOrigin-RevId: 222280524
---
 .../python/keras/layers/unified_rnn_test.py   | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index 015a079a0d..a16d0f383a 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -174,6 +174,8 @@ class UnifiedLSTM(RNN):
 
   def __init__(self,
                units,
+               activation='tanh',
+               recurrent_activation='hard_sigmoid',
                kernel_initializer='glorot_uniform',
                recurrent_initializer='orthogonal',
                bias_initializer='zeros',
@@ -196,7 +198,8 @@ class UnifiedLSTM(RNN):
     cell_spec = collections.namedtuple('cell', ['state_size', 'output_size'])
     self.cell = cell_spec(
         state_size=(self.units, self.units), output_size=self.units)
-
+    self.activation = activations.get(activation)
+    self.recurrent_activation = activations.get(recurrent_activation)
     self.kernel_initializer = initializers.get(kernel_initializer)
     self.recurrent_initializer = initializers.get(recurrent_initializer)
     self.bias_initializer = initializers.get(bias_initializer)
@@ -292,7 +295,8 @@ class UnifiedLSTM(RNN):
 
     outputs, [new_h, new_c], runtime = normal_lstm(
         inputs, initial_state[0], initial_state[1], self.kernel,
-        self.recurrent_kernel, self.bias, self.units)
+        self.recurrent_kernel, self.bias, self.units, self.activation,
+        self.recurrent_activation)
 
     function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1],
                       self.kernel, self.recurrent_kernel, self.bias, self.units)
@@ -385,7 +389,8 @@ def _is_multiple_state(state_size):
         'experimental_api_implements': 'lstm',
         'experimental_api_preferred_device': 'CPU'
     })
-def normal_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias, units):
+def normal_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias, units,
+                activation, recurrent_activation):
   input_shape = K.int_shape(inputs)
   timesteps = input_shape[1]
 
@@ -405,12 +410,12 @@ def normal_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias, units):
     z2 = z[:, 2 * units:3 * units]
     z3 = z[:, 3 * units:]
 
-    i = activations.get('hard_sigmoid')(z0)
-    f = activations.get('hard_sigmoid')(z1)
-    c = f * c_tm1 + i * activations.get('tanh')(z2)
-    o = activations.get('hard_sigmoid')(z3)
+    i = recurrent_activation(z0)
+    f = recurrent_activation(z1)
+    c = f * c_tm1 + i * activation(z2)
+    o = recurrent_activation(z3)
 
-    h = o * activations.get('tanh')(c)
+    h = o * activation(c)
     return h, [h, c]
 
   _, outputs, new_states = K.rnn(
-- 
GitLab


From e808d6d895502c0fb97f8fc1eb4e60b10ff2b8c5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 11:47:20 -0800
Subject: [PATCH 0600/1554] Copied sort_ops.py and sort_ops_test.py from
 contrib/framework/python/ops to python/ops and adjusted them.

PiperOrigin-RevId: 222281467
---
 tensorflow/contrib/framework/BUILD            |  15 +-
 .../contrib/framework/python/ops/sort_ops.py  | 172 +--------------
 tensorflow/python/BUILD                       |  28 +++
 tensorflow/python/ops/sort_ops.py             | 197 ++++++++++++++++++
 .../python/ops/sort_ops_test.py               |   6 +-
 tensorflow/python/ops/standard_ops.py         |   1 +
 .../tools/api/golden/v1/tensorflow.pbtxt      |   8 +
 .../tools/api/golden/v2/tensorflow.pbtxt      |   8 +
 .../tools/compatibility/tf_upgrade_v2.py      |   2 +
 9 files changed, 250 insertions(+), 187 deletions(-)
 create mode 100644 tensorflow/python/ops/sort_ops.py
 rename tensorflow/{contrib/framework => }/python/ops/sort_ops_test.py (96%)

diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index cd747df4d6..53efae1e10 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -66,6 +66,7 @@ tf_custom_op_py_library(
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:script_ops",
         "//tensorflow/python:smart_cond",
+        "//tensorflow/python:sort_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:state_ops_gen",
@@ -311,17 +312,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-py_test(
-    name = "sort_ops_test",
-    size = "medium",
-    srcs = ["python/ops/sort_ops_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":framework_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:random_ops",
-        "//third_party/py/numpy",
-    ],
-)
diff --git a/tensorflow/contrib/framework/python/ops/sort_ops.py b/tensorflow/contrib/framework/python/ops/sort_ops.py
index 1921a77c1e..42184a4e55 100644
--- a/tensorflow/contrib/framework/python/ops/sort_ops.py
+++ b/tensorflow/contrib/framework/python/ops/sort_ops.py
@@ -22,173 +22,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
+from tensorflow.python.ops import sort_ops
 
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import ops as framework_ops
-from tensorflow.python.framework import tensor_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nn_ops
-
-
-def sort(values, axis=-1, direction='ASCENDING', name=None):
-  """Sorts a tensor.
-
-  Args:
-    values: 1-D or higher numeric `Tensor`.
-    axis: The axis along which to sort. The default is -1, which sorts the last
-        axis.
-    direction: The direction in which to sort the values (`'ASCENDING'` or
-        `'DESCENDING'`).
-    name: Optional name for the operation.
-
-  Returns:
-    A `Tensor` with the same dtype and shape as `values`, with the elements
-        sorted along the given `axis`.
-
-  Raises:
-    ValueError: If axis is not a constant scalar, or the direction is invalid.
-  """
-  with framework_ops.name_scope(name, 'sort'):
-    return _sort_or_argsort(values, axis, direction, return_argsort=False)
-
-
-def argsort(values, axis=-1, direction='ASCENDING', stable=False, name=None):
-  """Returns the indices of a tensor that give its sorted order along an axis.
-
-  For a 1D tensor, `tf.gather(values, tf.argsort(values))` is equivalent to
-  `tf.sort(values)`. For higher dimensions, the output has the same shape as
-  `values`, but along the given axis, values represent the index of the sorted
-  element in that slice of the tensor at the given position.
-
-  Args:
-    values: 1-D or higher numeric `Tensor`.
-    axis: The axis along which to sort. The default is -1, which sorts the last
-        axis.
-    direction: The direction in which to sort the values (`'ASCENDING'` or
-        `'DESCENDING'`).
-    stable: If True, equal elements in the original tensor will not be
-        re-ordered in the returned order. Unstable sort is not yet implemented,
-        but will eventually be the default for performance reasons. If you
-        require a stable order, pass `stable=True` for forwards compatibility.
-    name: Optional name for the operation.
-
-  Returns:
-    An int32 `Tensor` with the same shape as `values`. The indices that would
-        sort each slice of the given `values` along the given `axis`.
-
-  Raises:
-    ValueError: If axis is not a constant scalar, or the direction is invalid.
-  """
-  del stable  # Unused.
-  with framework_ops.name_scope(name, 'argsort'):
-    return _sort_or_argsort(values, axis, direction, return_argsort=True)
-
-
-def _sort_or_argsort(values, axis, direction, return_argsort):
-  """Internal sort/argsort implementation.
-
-  Args:
-    values: The input values.
-    axis: The axis along which to sort.
-    direction: 'ASCENDING' or 'DESCENDING'.
-    return_argsort: Whether to return the argsort result.
-
-  Returns:
-    Either the sorted values, or the indices of the sorted values in the
-        original tensor. See the `sort` and `argsort` docstrings.
-
-  Raises:
-    ValueError: If axis is not a constant scalar, or the direction is invalid.
-  """
-  if direction not in _SORT_IMPL:
-    raise ValueError('%s should be one of %s' %
-                     (direction, ', '.join(sorted(_SORT_IMPL.keys()))))
-  # Axis must be an integer, not a Tensor.
-  axis = framework_ops.convert_to_tensor(axis, name='axis')
-  axis_static = tensor_util.constant_value(axis)
-  if axis.shape.ndims != 0 or axis_static is None:
-    raise ValueError('axis must be a constant scalar')
-  axis_static = int(axis_static)  # Avoids NumPy casting error
-
-  values = framework_ops.convert_to_tensor(values, name='values')
-
-  return _SORT_IMPL[direction](values, axis_static, return_argsort)
-
-
-def _descending_sort(values, axis, return_argsort=False):
-  """Sorts values in reverse using `top_k`.
-
-  Args:
-    values: Tensor of numeric values.
-    axis: Index of the axis which values should be sorted along.
-    return_argsort: If False, return the sorted values. If True, return the
-        indices that would sort the values.
-
-  Returns:
-    The sorted values.
-  """
-  k = array_ops.shape(values)[axis]
-  rank = array_ops.rank(values)
-  static_rank = values.shape.ndims
-  # Fast path: sorting the last axis.
-  if axis == -1 or axis + 1 == values.get_shape().ndims:
-    top_k_input = values
-    transposition = None
-  else:
-    # Otherwise, transpose the array. Swap axes `axis` and `rank - 1`.
-    if axis < 0:
-      # Calculate the actual axis index if counting from the end. Use the static
-      # rank if available, or else make the axis back into a tensor.
-      axis += static_rank or rank
-    if static_rank is not None:
-      # Prefer to calculate the transposition array in NumPy and make it a
-      # constant.
-      transposition = constant_op.constant(
-          np.r_[
-              # Axes up to axis are unchanged.
-              np.arange(axis),
-              # Swap axis and rank - 1.
-              [static_rank - 1],
-              # Axes in [axis + 1, rank - 1) are unchanged.
-              np.arange(axis + 1, static_rank - 1),
-              # Swap axis and rank - 1.
-              [axis]],
-          name='transposition')
-    else:
-      # Generate the transposition array from the tensors.
-      transposition = array_ops.concat(
-          [
-              # Axes up to axis are unchanged.
-              math_ops.range(axis),
-              # Swap axis and rank - 1.
-              [rank - 1],
-              # Axes in [axis + 1, rank - 1) are unchanged.
-              math_ops.range(axis + 1, rank - 1),
-              # Swap axis and rank - 1.
-              [axis]
-          ],
-          axis=0)
-    top_k_input = array_ops.transpose(values, transposition)
-
-  values, indices = nn_ops.top_k(top_k_input, k)
-  return_value = indices if return_argsort else values
-  if transposition is not None:
-    # transposition contains a single cycle of length 2 (swapping 2 elements),
-    # so it is an involution (it is its own inverse).
-    return_value = array_ops.transpose(return_value, transposition)
-  return return_value
-
-
-def _ascending_sort(values, axis, return_argsort=False):
-  # Negate the values to get the ascending order from descending sort.
-  values_or_indices = _descending_sort(-values, axis, return_argsort)
-  # If not argsort, negate the values again.
-  return values_or_indices if return_argsort else -values_or_indices
-
-
-_SORT_IMPL = {
-    'ASCENDING': _ascending_sort,
-    'DESCENDING': _descending_sort,
-}
+sort = sort_ops.sort
+argsort = sort_ops.argsort
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 1010678533..3def23bc44 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2843,6 +2843,33 @@ py_test(
     ],
 )
 
+py_library(
+    name = "sort_ops",
+    srcs = ["ops/sort_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":array_ops",
+        ":framework",
+        ":math_ops",
+        ":nn_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "sort_ops_test",
+    srcs = ["ops/sort_ops_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":array_ops",
+        ":client_testlib",
+        ":framework",
+        ":random_ops",
+        ":sort_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "spectral_ops_test_util",
     srcs = ["ops/spectral_ops_test_util.py"],
@@ -2957,6 +2984,7 @@ py_library(
         ":random_ops",
         ":script_ops",
         ":session_ops",
+        ":sort_ops",
         ":sparse_grad",
         ":sparse_ops",
         ":special_math_ops",
diff --git a/tensorflow/python/ops/sort_ops.py b/tensorflow/python/ops/sort_ops.py
new file mode 100644
index 0000000000..c3e23d701e
--- /dev/null
+++ b/tensorflow/python/ops/sort_ops.py
@@ -0,0 +1,197 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Support for sorting tensors.
+
+@@argsort
+@@sort
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops as framework_ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export('sort')
+def sort(values, axis=-1, direction='ASCENDING', name=None):
+  """Sorts a tensor.
+
+  Args:
+    values: 1-D or higher numeric `Tensor`.
+    axis: The axis along which to sort. The default is -1, which sorts the last
+      axis.
+    direction: The direction in which to sort the values (`'ASCENDING'` or
+      `'DESCENDING'`).
+    name: Optional name for the operation.
+
+  Returns:
+    A `Tensor` with the same dtype and shape as `values`, with the elements
+        sorted along the given `axis`.
+
+  Raises:
+    ValueError: If axis is not a constant scalar, or the direction is invalid.
+  """
+  with framework_ops.name_scope(name, 'sort'):
+    return _sort_or_argsort(values, axis, direction, return_argsort=False)
+
+
+@tf_export('argsort')
+def argsort(values, axis=-1, direction='ASCENDING', stable=False, name=None):
+  """Returns the indices of a tensor that give its sorted order along an axis.
+
+  For a 1D tensor, `tf.gather(values, tf.argsort(values))` is equivalent to
+  `tf.sort(values)`. For higher dimensions, the output has the same shape as
+  `values`, but along the given axis, values represent the index of the sorted
+  element in that slice of the tensor at the given position.
+
+  Args:
+    values: 1-D or higher numeric `Tensor`.
+    axis: The axis along which to sort. The default is -1, which sorts the last
+      axis.
+    direction: The direction in which to sort the values (`'ASCENDING'` or
+      `'DESCENDING'`).
+    stable: If True, equal elements in the original tensor will not be
+      re-ordered in the returned order. Unstable sort is not yet implemented,
+      but will eventually be the default for performance reasons. If you require
+      a stable order, pass `stable=True` for forwards compatibility.
+    name: Optional name for the operation.
+
+  Returns:
+    An int32 `Tensor` with the same shape as `values`. The indices that would
+        sort each slice of the given `values` along the given `axis`.
+
+  Raises:
+    ValueError: If axis is not a constant scalar, or the direction is invalid.
+  """
+  del stable  # Unused.
+  with framework_ops.name_scope(name, 'argsort'):
+    return _sort_or_argsort(values, axis, direction, return_argsort=True)
+
+
+def _sort_or_argsort(values, axis, direction, return_argsort):
+  """Internal sort/argsort implementation.
+
+  Args:
+    values: The input values.
+    axis: The axis along which to sort.
+    direction: 'ASCENDING' or 'DESCENDING'.
+    return_argsort: Whether to return the argsort result.
+
+  Returns:
+    Either the sorted values, or the indices of the sorted values in the
+        original tensor. See the `sort` and `argsort` docstrings.
+
+  Raises:
+    ValueError: If axis is not a constant scalar, or the direction is invalid.
+  """
+  if direction not in _SORT_IMPL:
+    raise ValueError('%s should be one of %s' % (direction, ', '.join(
+        sorted(_SORT_IMPL.keys()))))
+  # Axis must be an integer, not a Tensor.
+  axis = framework_ops.convert_to_tensor(axis, name='axis')
+  axis_static = tensor_util.constant_value(axis)
+  if axis.shape.ndims != 0 or axis_static is None:
+    raise ValueError('axis must be a constant scalar')
+  axis_static = int(axis_static)  # Avoids NumPy casting error
+
+  values = framework_ops.convert_to_tensor(values, name='values')
+
+  return _SORT_IMPL[direction](values, axis_static, return_argsort)
+
+
+def _descending_sort(values, axis, return_argsort=False):
+  """Sorts values in reverse using `top_k`.
+
+  Args:
+    values: Tensor of numeric values.
+    axis: Index of the axis which values should be sorted along.
+    return_argsort: If False, return the sorted values. If True, return the
+      indices that would sort the values.
+
+  Returns:
+    The sorted values.
+  """
+  k = array_ops.shape(values)[axis]
+  rank = array_ops.rank(values)
+  static_rank = values.shape.ndims
+  # Fast path: sorting the last axis.
+  if axis == -1 or axis + 1 == values.get_shape().ndims:
+    top_k_input = values
+    transposition = None
+  else:
+    # Otherwise, transpose the array. Swap axes `axis` and `rank - 1`.
+    if axis < 0:
+      # Calculate the actual axis index if counting from the end. Use the static
+      # rank if available, or else make the axis back into a tensor.
+      axis += static_rank or rank
+    if static_rank is not None:
+      # Prefer to calculate the transposition array in NumPy and make it a
+      # constant.
+      transposition = constant_op.constant(
+          np.r_[
+              # Axes up to axis are unchanged.
+              np.arange(axis),
+              # Swap axis and rank - 1.
+              [static_rank - 1],
+              # Axes in [axis + 1, rank - 1) are unchanged.
+              np.arange(axis + 1, static_rank - 1),
+              # Swap axis and rank - 1.
+              [axis]],
+          name='transposition')
+    else:
+      # Generate the transposition array from the tensors.
+      transposition = array_ops.concat(
+          [
+              # Axes up to axis are unchanged.
+              math_ops.range(axis),
+              # Swap axis and rank - 1.
+              [rank - 1],
+              # Axes in [axis + 1, rank - 1) are unchanged.
+              math_ops.range(axis + 1, rank - 1),
+              # Swap axis and rank - 1.
+              [axis]
+          ],
+          axis=0)
+    top_k_input = array_ops.transpose(values, transposition)
+
+  values, indices = nn_ops.top_k(top_k_input, k)
+  return_value = indices if return_argsort else values
+  if transposition is not None:
+    # transposition contains a single cycle of length 2 (swapping 2 elements),
+    # so it is an involution (it is its own inverse).
+    return_value = array_ops.transpose(return_value, transposition)
+  return return_value
+
+
+def _ascending_sort(values, axis, return_argsort=False):
+  # Negate the values to get the ascending order from descending sort.
+  values_or_indices = _descending_sort(-values, axis, return_argsort)
+  # If not argsort, negate the values again.
+  return values_or_indices if return_argsort else -values_or_indices
+
+
+_SORT_IMPL = {
+    'ASCENDING': _ascending_sort,
+    'DESCENDING': _descending_sort,
+}
diff --git a/tensorflow/contrib/framework/python/ops/sort_ops_test.py b/tensorflow/python/ops/sort_ops_test.py
similarity index 96%
rename from tensorflow/contrib/framework/python/ops/sort_ops_test.py
rename to tensorflow/python/ops/sort_ops_test.py
index 791b32cd1e..8a92f49266 100644
--- a/tensorflow/contrib/framework/python/ops/sort_ops_test.py
+++ b/tensorflow/python/ops/sort_ops_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.framework.python.ops import sort_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -28,6 +27,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import sort_ops
 from tensorflow.python.platform import test
 
 
@@ -88,9 +88,7 @@ class SortTest(test.TestCase):
       self.assertAllEqual(
           np.sort(arr, axis=0)[::-1],
           sort_ops.sort(
-              constant_op.constant(arr),
-              axis=0,
-              direction='DESCENDING').eval())
+              constant_op.constant(arr), axis=0, direction='DESCENDING').eval())
 
   def testSort_staticallyKnownRank_constantTransposition(self):
     # The transposition array should be a constant if the rank of "values" is
diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py
index 03e491a315..c614d072ba 100644
--- a/tensorflow/python/ops/standard_ops.py
+++ b/tensorflow/python/ops/standard_ops.py
@@ -72,6 +72,7 @@ from tensorflow.python.ops.partitioned_variables import *
 from tensorflow.python.ops.random_ops import *
 from tensorflow.python.ops.script_ops import py_func
 from tensorflow.python.ops.session_ops import *
+from tensorflow.python.ops.sort_ops import *
 from tensorflow.python.ops.sparse_ops import *
 from tensorflow.python.ops.state_ops import assign
 from tensorflow.python.ops.state_ops import assign_add
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 6a45bc7b7f..367f506b21 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -700,6 +700,10 @@ tf_module {
     name: "argmin"
     argspec: "args=[\'input\', \'axis\', \'name\', \'dimension\', \'output_type\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"<dtype: \'int64\'>\"], "
   }
+  member_method {
+    name: "argsort"
+    argspec: "args=[\'values\', \'axis\', \'direction\', \'stable\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'False\', \'None\'], "
+  }
   member_method {
     name: "as_dtype"
     argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None"
@@ -1956,6 +1960,10 @@ tf_module {
     name: "slice"
     argspec: "args=[\'input_\', \'begin\', \'size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "sort"
+    argspec: "args=[\'values\', \'axis\', \'direction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'None\'], "
+  }
   member_method {
     name: "space_to_batch"
     argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 6adce4b3fd..1711348ec3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -452,6 +452,10 @@ tf_module {
     name: "argmin"
     argspec: "args=[\'input\', \'axis\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'int64\'>\", \'None\'], "
   }
+  member_method {
+    name: "argsort"
+    argspec: "args=[\'values\', \'axis\', \'direction\', \'stable\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'False\', \'None\'], "
+  }
   member_method {
     name: "as_dtype"
     argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None"
@@ -1036,6 +1040,10 @@ tf_module {
     name: "slice"
     argspec: "args=[\'input_\', \'begin\', \'size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "sort"
+    argspec: "args=[\'values\', \'axis\', \'direction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'None\'], "
+  }
   member_method {
     name: "space_to_batch_nd"
     argspec: "args=[\'input\', \'block_shape\', \'paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3cef1f1602..c60c6a4b32 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -153,6 +153,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.contrib.data.sloppy_interleave": "tf.compat.v1.contrib.data.sloppy_interleave",
         "tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
         "tf.contrib.data.unique": "tf.data.experimental.unique",
+        "tf.contrib.framework.sort": "tf.sort",
+        "tf.contrib.framework.argsort": "tf.argsort",
         "tf.quantize_v2": "tf.quantization.quantize",
         "tf.sparse_concat": "tf.sparse.concat",
         "tf.sparse_split": "tf.sparse.split",
-- 
GitLab


From bf8544fccf8d4f909600b1840758680f411deaa7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 11:48:08 -0800
Subject: [PATCH 0601/1554] Switch some code to the new tf.distribute.Strategy
 API.

PiperOrigin-RevId: 222281620
---
 .../collective_all_reduce_strategy_test.py    |   2 +-
 .../python/cross_device_ops_test.py           |   2 +-
 .../python/estimator_integration_test.py      |   4 +-
 .../python/estimator_training_test.py         |   4 +-
 .../python/keras_optimizer_v2_test.py         |   4 +-
 .../distribute/python/mirrored_strategy.py    |   4 +-
 .../python/mirrored_strategy_multigpu_test.py | 133 +++++++++---------
 .../distribute/python/one_device_strategy.py  |   6 +-
 .../python/one_device_strategy_test.py        |   2 +-
 .../python/parameter_server_strategy_test.py  |   6 +-
 .../distribute/python/strategy_test_lib.py    |   7 +-
 .../contrib/distribute/python/tpu_strategy.py |   6 +-
 .../contrib/optimizer_v2/optimizer_v2.py      |  17 +--
 .../distribute/distribute_coordinator.py      |  22 +--
 .../distribute/distribute_coordinator_test.py |  67 +++++----
 .../python/distribute/estimator_training.py   |   4 +-
 tensorflow/python/distribute/values.py        |  10 +-
 tensorflow/python/eager/tape.py               |   4 +-
 .../keras/engine/training_distributed.py      |  16 +--
 tensorflow/python/training/distribute_test.py |   4 +-
 .../python/training/monitored_session_test.py |  32 ++---
 tensorflow/python/training/optimizer.py       |  16 ++-
 22 files changed, 183 insertions(+), 189 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index a47eef94e9..eb2b859aa5 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -249,7 +249,7 @@ class CollectiveAllReduceStrategyTestBase(
                               expected_values):
     distribution, master_target, config = self._get_test_object(
         task_type, task_id, num_gpus)
-    devices = distribution.worker_devices
+    devices = distribution.extended.worker_devices
 
     with ops.Graph().as_default(), \
          self.cached_session(config=config,
diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 00672a4401..a0f7cf182c 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -119,7 +119,7 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
               sess.run(list(left._index.values())), list(right._index.values()))
 
   def _testReductionAndBroadcast(self, cross_device_ops, distribution):
-    devices = distribution.worker_devices
+    devices = distribution.extended.worker_devices
 
     values = [constant_op.constant(float(d)) for d in range(len(devices))]
     per_replica = _make_per_replica(values, devices)
diff --git a/tensorflow/contrib/distribute/python/estimator_integration_test.py b/tensorflow/contrib/distribute/python/estimator_integration_test.py
index 264dca6f38..e17085628b 100644
--- a/tensorflow/contrib/distribute/python/estimator_integration_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_integration_test.py
@@ -77,12 +77,12 @@ class DNNLinearCombinedClassifierIntegrationTest(test.TestCase,
     train_input_fn = self.dataset_input_fn(
         x={'x': data},
         y=data,
-        batch_size=batch_size // len(distribution.worker_devices),
+        batch_size=batch_size // distribution.num_replicas_in_sync,
         shuffle=True)
     eval_input_fn = self.dataset_input_fn(
         x={'x': data},
         y=data,
-        batch_size=batch_size // len(distribution.worker_devices),
+        batch_size=batch_size // distribution.num_replicas_in_sync,
         shuffle=False)
     predict_input_fn = numpy_io.numpy_input_fn(
         x={'x': data}, batch_size=batch_size, shuffle=False)
diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 3e7d5df4c4..202e92d1e7 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -204,10 +204,10 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
     train_input_fn = self.dataset_input_fn(
         x={"x": DATA},
         y=DATA,
-        batch_size=BATCH_SIZE // len(train_distribute.worker_devices),
+        batch_size=BATCH_SIZE // train_distribute.num_replicas_in_sync,
         shuffle=True)
     if eval_distribute:
-      eval_batch_size = BATCH_SIZE // len(eval_distribute.worker_devices)
+      eval_batch_size = BATCH_SIZE // eval_distribute.num_replicas_in_sync
     else:
       eval_batch_size = BATCH_SIZE
     eval_input_fn = self.dataset_input_fn(
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index 0d7e11c3b6..fba06283ce 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -83,11 +83,11 @@ class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase):
     train_input_fn = self.dataset_input_fn(
         x={'x': data},
         y=data,
-        batch_size=batch_size // len(distribution.worker_devices))
+        batch_size=batch_size // distribution.num_replicas_in_sync)
     eval_input_fn = self.dataset_input_fn(
         x={'x': data},
         y=data,
-        batch_size=batch_size // len(distribution.worker_devices))
+        batch_size=batch_size // distribution.num_replicas_in_sync)
     predict_input_fn = numpy_io.numpy_input_fn(
         x={'x': data}, batch_size=batch_size, shuffle=False)
 
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index f7432162cb..a2ab95487a 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -742,7 +742,7 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
       self.coord = coord
       self.distribution = dist
       self.device = device
-      self.replica_id = dist.worker_devices.index(device)
+      self.replica_id = dist.extended.worker_devices.index(device)
       self.variable_creator_fn = variable_creator_fn
       # State needed to run and return the results of `fn`.
       self.main_fn = fn
@@ -944,4 +944,4 @@ class MirroredReplicaContext(distribute_lib.ReplicaContext):
   def devices(self):
     distribute_lib.require_replica_context(self)
     replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
-    return [self._distribution_strategy.worker_devices[replica_id]]
+    return [self._distribution_strategy.extended.worker_devices[replica_id]]
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 9fd4cca319..d20ac144bf 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -90,11 +90,11 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
       return list(range(replica_id))
 
     with distribution.scope(), self.assertRaises(AssertionError):
-      distribution.call_for_each_replica(run_fn)
+      distribution.extended.call_for_each_replica(run_fn)
 
   def testReduceToCpu(self, distribution):
     with distribution.scope():
-      result = distribution.call_for_each_replica(_replica_id)
+      result = distribution.extended.call_for_each_replica(_replica_id)
       reduced = distribution.reduce(
           reduce_util.ReduceOp.SUM,
           result,
@@ -114,7 +114,7 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)
     iterator = distribution.make_input_fn_iterator(input_fn)
-    self._test_input_fn_iterator(iterator, distribution.worker_devices,
+    self._test_input_fn_iterator(iterator, distribution.extended.worker_devices,
                                  expected_values)
 
   def testGlobalStepUpdate(self, distribution):
@@ -150,7 +150,7 @@ class MirroredOneDeviceDistributionTest(
       mode=["graph", "eager"]))
   def testReduceToMultipleDestinations(self, distribution):
     with distribution.scope():
-      reduced = distribution.reduce(
+      reduced = distribution.extended.reduce_to(
           reduce_util.ReduceOp.SUM,
           1.0,
           destinations=["/device:CPU:0", "/device:GPU:0"])
@@ -204,7 +204,7 @@ class MirroredStrategyVariableCreatorStackTest(
     with context.graph_mode(), \
         distribution.scope(), \
         variable_scope.variable_creator_scope(main_thread_creator):
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       result = distribution.unwrap(result)
       expected = ["main_thread:thread_0", "main_thread:thread_1"]
       self.assertEqual(expected, result)
@@ -221,13 +221,13 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     def model_fn():
       # This variable should be created only once across the threads because of
       # special variable_creator functions used by
-      # `distribution.call_for_each_replica`.
+      # `distribution.extended.call_for_each_replica`.
       v = variable_scope.variable(1.0, name="foo")
       ds_context.get_replica_context().merge_call(lambda _: _)
       return v
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       self.assertEqual("foo:0", result.name)
 
@@ -238,7 +238,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return v
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       # Default name of "Variable" will be used.
       self.assertEqual("Variable:0", result.name)
@@ -252,7 +252,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return vs
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       for i, v in enumerate(result):
         self.assertIsInstance(v, values.MirroredVariable)
         self.assertEqual("foo" + str(i) + ":0", v.name)
@@ -268,7 +268,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return vs
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       for v in result:
         self.assertIsInstance(v, values.MirroredVariable)
       self.assertEqual(4, len(result))
@@ -285,7 +285,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return v
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       # The resulting mirrored variable will use the name from the first device.
       self.assertEqual("foo_0:0", result.name)
@@ -316,7 +316,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     features = iterator.get_next()
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn, args=(features,))
+      result = distribution.extended.call_for_each_replica(
+          model_fn, args=(features,))
       suffixes = ["", "_1", "_2"]
       for (kernel, bias), suffix in zip(result, suffixes):
         self.assertIsInstance(kernel, values.MirroredVariable)
@@ -348,7 +349,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       v = variable_scope.variable(1.0, name="var-main0")
       self.assertEqual("var-main0:0", v.name)
 
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       self.assertEqual(4, len(result))
       v0, v1, v2, v3 = result
       self.assertIsInstance(v0, values.MirroredVariable)
@@ -385,7 +386,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         v = variable_scope.get_variable("var-main0", [1])
         self.assertEqual("main/var-main0:0", v.name)
 
-        result = distribution.call_for_each_replica(model_fn)
+        result = distribution.extended.call_for_each_replica(model_fn)
         self.assertEqual(4, len(result))
         v0, v1, v2, v3 = result
         self.assertIsInstance(v0, values.MirroredVariable)
@@ -418,15 +419,15 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     devices = ["/device:GPU:0", "/device:CPU:0"]
     with distribution.scope():
-      v0, v1 = distribution.call_for_each_replica(create_fn)
+      v0, v1 = distribution.extended.call_for_each_replica(create_fn)
       self.evaluate(v0.initializer)
       self.assertEqual(2.0, self.evaluate(v0.get(devices[0])))
       self.assertEqual(2.0, self.evaluate(v0.get(devices[1])))
-      self.assertEqual(2.0, self.evaluate(distribution.read_var(v0)))
+      self.assertEqual(2.0, self.evaluate(distribution.extended.read_var(v0)))
       self.evaluate(v1.initializer)
       self.assertEqual(3.0, self.evaluate(v1.get(devices[0])))
       self.assertEqual(3.0, self.evaluate(v1.get(devices[1])))
-      self.assertEqual(3.0, self.evaluate(distribution.read_var(v1)))
+      self.assertEqual(3.0, self.evaluate(distribution.extended.read_var(v1)))
 
       def replica_id_plus_one():
         return math_ops.cast(_replica_id() + 1, dtype=dtypes.float32)
@@ -437,7 +438,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         update1 = v1.assign_add(7.0 * replica_id_plus_one())
         return update0, update1
 
-      update0a, update1a = distribution.call_for_each_replica(update_member_fn)
+      update0a, update1a = distribution.extended.call_for_each_replica(
+          update_member_fn)
 
       # Update "sync on read" variable.
       self.evaluate(distribution.group(update0a))
@@ -446,7 +448,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       # so device[1] can end up with a different value.
       self.assertEqual(2.0 + 2*5.0, self.evaluate(v0.get(devices[1])))
       # Always reads from device 0.
-      self.assertEqual(2.0 + 5.0, self.evaluate(distribution.read_var(v0)))
+      self.assertEqual(2.0 + 5.0, self.evaluate(
+          distribution.extended.read_var(v0)))
 
       # Update "sync on write" variable.
       self.evaluate(distribution.group(update1a))
@@ -454,7 +457,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       # Writes are synchronized for v1, only the argument to assign_add on
       # device[0] is used.
       self.assertEqual(3.0 + 7.0, self.evaluate(v1.get(devices[1])))
-      self.assertEqual(3.0 + 7.0, self.evaluate(distribution.read_var(v1)))
+      self.assertEqual(3.0 + 7.0, self.evaluate(
+          distribution.extended.read_var(v1)))
 
       # Update using state_ops.assign_add global function.
       def update_state_ops_fn():
@@ -462,7 +466,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
         update1 = state_ops.assign_add(v1, 13.0 * replica_id_plus_one())
         return update0, update1
 
-      update0b, update1b = distribution.call_for_each_replica(
+      update0b, update1b = distribution.extended.call_for_each_replica(
           update_state_ops_fn)
       self.evaluate(distribution.group(update0b))
 
@@ -470,14 +474,14 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(v0.get(devices[0])))
       self.assertEqual(2.0 + 2*5.0 + 2*11.0, self.evaluate(v0.get(devices[1])))
       self.assertEqual(2.0 + 5.0 + 11.0, self.evaluate(
-          distribution.read_var(v0)))
+          distribution.extended.read_var(v0)))
 
       # Update "sync on write" variable.
       self.evaluate(distribution.group(update1b))
       self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(v1.get(devices[0])))
       self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(v1.get(devices[1])))
       self.assertEqual(3.0 + 7.0 + 13.0, self.evaluate(
-          distribution.read_var(v1)))
+          distribution.extended.read_var(v1)))
 
   def testNoneSynchronizationWithGetVariable(self, distribution):
     with distribution.scope():
@@ -540,7 +544,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
           "/device:GPU:0": "bar"
       })
       with self.assertRaises(RuntimeError):
-        _ = distribution.call_for_each_replica(model_fn, args=(names,))
+        _ = distribution.extended.call_for_each_replica(model_fn, args=(names,))
 
   def testReplicaLocalVariable(self, distribution):
     all_v_sum = {}
@@ -575,7 +579,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     with distribution.scope():
       # Create "sum" and "mean" versions of ReplicaLocalVariables.
       ret_ops, ret_v_sum, ret_v_mean, regrouped_sum, regrouped_mean = (
-          distribution.call_for_each_replica(model_fn))
+          distribution.extended.call_for_each_replica(model_fn))
       # Should see the same wrapping instance in all replicas.
       self.assertIs(all_v_sum[0], ret_v_sum)
       self.assertIs(all_v_mean[0], ret_v_mean)
@@ -609,9 +613,9 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       # applying the reduction across all replicas (whether you use
       # read_var(), get(), or nothing).
       self.assertEqual(expected_sum, self.evaluate(
-          distribution.read_var(ret_v_sum)))
+          distribution.extended.read_var(ret_v_sum)))
       self.assertEqual(expected_mean, self.evaluate(
-          distribution.read_var(ret_v_mean)))
+          distribution.extended.read_var(ret_v_mean)))
       self.assertEqual(expected_sum, self.evaluate(ret_v_sum.get()))
       self.assertEqual(expected_mean, self.evaluate(ret_v_mean.get()))
       self.assertEqual(expected_sum, self.evaluate(ret_v_sum))
@@ -631,7 +635,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return outputs
 
     with context.graph_mode(), distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       # Two variables are created by the RNN layer.
       self.assertEqual(2, len(result))
       for v in result:
@@ -652,7 +656,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       return var.assign(value)
 
     with distribution.scope():
-      ret_v_sum = distribution.call_for_each_replica(model_fn)
+      ret_v_sum = distribution.extended.call_for_each_replica(model_fn)
 
       # Initialize variables.
       self.evaluate(variables.global_variables_initializer())
@@ -663,7 +667,8 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
       self.assertEqual(2.0, self.evaluate(ret_v_sum))
 
       # Apply updates.
-      update_ops = distribution.update(ret_v_sum, update, 5.0, grouped=False)
+      update_ops = distribution.extended.update(
+          ret_v_sum, update, args=(5.0,), group=False)
       self.evaluate(update_ops)
       # Assert that the aggregated value of the replica local vars is the sum
       # of the individual values after running the update ops.
@@ -691,7 +696,7 @@ class MirroredStrategyNameScopeTest(test.TestCase):
 
     with context.graph_mode(), distribution.scope():
       with ops.name_scope("main"):
-        result = distribution.call_for_each_replica(model_fn)
+        result = distribution.extended.call_for_each_replica(model_fn)
         self.assertEqual(2, len(result))
         for v, name in zip(result, ["a", "b"]):
           self.assertIsInstance(v, values.DistributedValues)
@@ -708,7 +713,7 @@ class MirroredStrategyNameScopeTest(test.TestCase):
       return a, b
 
     with context.graph_mode(), distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       self.assertEqual(2, len(result))
       for v, name in zip(result, ["a", "b"]):
         self.assertIsInstance(v, values.DistributedValues)
@@ -734,7 +739,7 @@ class MirroredStrategyNameScopeTest(test.TestCase):
     with context.graph_mode(), distribution.scope():
       with ops.name_scope("main"):
         a = variable_scope.variable(1.0, name="a")
-        result = distribution.call_for_each_replica(model_fn)
+        result = distribution.extended.call_for_each_replica(model_fn)
       result_b = result[0]
       result_c = result[1]
       self.assertIsInstance(result_b, values.DistributedValues)
@@ -763,7 +768,7 @@ class MirroredStrategyNameScopeTest(test.TestCase):
     with context.graph_mode(), distribution.scope():
       with ops.name_scope("main"):
         a = variable_scope.get_variable("a", [1])
-        result = distribution.call_for_each_replica(model_fn)
+        result = distribution.extended.call_for_each_replica(model_fn)
       result_b = result[0]
       result_c = result[1]
       self.assertIsInstance(result_b, values.DistributedValues)
@@ -805,7 +810,7 @@ class MirroredThreeDeviceDistributionTest(
       return v
 
     with distribution.scope():
-      result = distribution.call_for_each_replica(model_fn)
+      result = distribution.extended.call_for_each_replica(model_fn)
       self.assertIsInstance(result, values.MirroredVariable)
       self.assertEqual("foo:0", result.name)
 
@@ -828,7 +833,7 @@ class MirroredVariableUpdateTest(test.TestCase):
       return v
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
 
@@ -839,7 +844,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           ValueError, "You must specify an aggregation method to update a "
                       "MirroredVariable in Replica Context."):
         self.evaluate(distribution.unwrap(
-            distribution.call_for_each_replica(model_fn)))
+            distribution.extended.call_for_each_replica(model_fn)))
 
   def testAssignMirroredVarReplicaContextWithSum(self, distribution):
     # Test that we don't reduce a non-per-replica value with the "sum"
@@ -850,7 +855,7 @@ class MirroredVariableUpdateTest(test.TestCase):
       return v
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
 
@@ -861,14 +866,14 @@ class MirroredVariableUpdateTest(test.TestCase):
           ValueError, "A non-DistributedValues value 5.0 cannot be reduced "
           "with the given reduce op ReduceOp.SUM."):
         self.evaluate(distribution.unwrap(
-            distribution.call_for_each_replica(model_fn)))
+            distribution.extended.call_for_each_replica(model_fn)))
 
   def testAssignMirroredVarCrossDeviceContext(self, distribution):
     def var_fn():
       return variable_scope.variable(1.0, name="foo")
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1.0, self.evaluate(mirrored_var))
@@ -881,7 +886,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1.0, self.evaluate(mirrored_var))
@@ -893,7 +898,7 @@ class MirroredVariableUpdateTest(test.TestCase):
         return mirrored_var.assign(value)
 
       self.evaluate(distribution.unwrap(
-          distribution.call_for_each_replica(model_fn)))
+          distribution.extended.call_for_each_replica(model_fn)))
       self.assertEqual(0.5, self.evaluate(mirrored_var))
 
   def testAssignMirroredVarReplicaContextWithSingleValue(self, distribution):
@@ -902,7 +907,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1.0, self.evaluate(mirrored_var))
@@ -911,7 +916,7 @@ class MirroredVariableUpdateTest(test.TestCase):
         return mirrored_var.assign(5.0)
 
       self.evaluate(distribution.unwrap(
-          distribution.call_for_each_replica(model_fn)))
+          distribution.extended.call_for_each_replica(model_fn)))
       self.assertEqual(5.0, self.evaluate(mirrored_var))
 
   def testAssignAddMirroredVarCrossDeviceContext(self, distribution):
@@ -919,7 +924,7 @@ class MirroredVariableUpdateTest(test.TestCase):
       return variable_scope.variable(1.0, name="foo")
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1.0, self.evaluate(mirrored_var))
@@ -942,7 +947,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1.0, self.evaluate(mirrored_var))
@@ -954,7 +959,7 @@ class MirroredVariableUpdateTest(test.TestCase):
         return mirrored_var.assign_add(value)
 
       self.evaluate(distribution.unwrap(
-          distribution.call_for_each_replica(model_fn)))
+          distribution.extended.call_for_each_replica(model_fn)))
       self.assertEqual(1.5, self.evaluate(mirrored_var))
 
   def testAssignAddMirroredVarReplicaContextWithSingleValue(self, distribution):
@@ -963,7 +968,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1.0, self.evaluate(mirrored_var))
@@ -972,7 +977,7 @@ class MirroredVariableUpdateTest(test.TestCase):
         return mirrored_var.assign_add(5.0)
 
       self.evaluate(distribution.unwrap(
-          distribution.call_for_each_replica(model_fn)))
+          distribution.extended.call_for_each_replica(model_fn)))
       self.assertEqual(6.0, self.evaluate(mirrored_var))
 
   def testAssignSubMirroredVarCrossDeviceContext(self, distribution):
@@ -980,7 +985,7 @@ class MirroredVariableUpdateTest(test.TestCase):
       return variable_scope.variable(5.0, name="foo")
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(5.0, self.evaluate(mirrored_var))
@@ -995,7 +1000,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(5.0, self.evaluate(mirrored_var))
@@ -1007,7 +1012,7 @@ class MirroredVariableUpdateTest(test.TestCase):
         return mirrored_var.assign_sub(value)
 
       self.evaluate(distribution.unwrap(
-          distribution.call_for_each_replica(model_fn)))
+          distribution.extended.call_for_each_replica(model_fn)))
       self.assertEqual(4.5, self.evaluate(mirrored_var))
 
   def testAssignSubMirroredVarReplicaContextWithSingleValue(self, distribution):
@@ -1016,7 +1021,7 @@ class MirroredVariableUpdateTest(test.TestCase):
           5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)
 
     with distribution.scope():
-      mirrored_var = distribution.call_for_each_replica(var_fn)
+      mirrored_var = distribution.extended.call_for_each_replica(var_fn)
       self.assertIsInstance(mirrored_var, values.MirroredVariable)
       self.evaluate(variables.global_variables_initializer())
       self.assertEqual(5.0, self.evaluate(mirrored_var))
@@ -1025,7 +1030,7 @@ class MirroredVariableUpdateTest(test.TestCase):
         return mirrored_var.assign_sub(1.0)
 
       self.evaluate(distribution.unwrap(
-          distribution.call_for_each_replica(model_fn)))
+          distribution.extended.call_for_each_replica(model_fn)))
       self.assertEqual(4.0, self.evaluate(mirrored_var))
 
 
@@ -1045,7 +1050,7 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase):
         return v
 
       with distribution.scope():
-        mirrored_var = distribution.call_for_each_replica(var_fn)
+        mirrored_var = distribution.extended.call_for_each_replica(var_fn)
         self.assertIsInstance(mirrored_var, values.MirroredVariable)
         self.assertFalse(self.evaluate(mirrored_var.is_initialized()))
         self.evaluate(mirrored_var.initializer)
@@ -1064,7 +1069,8 @@ class MirroredAndReplicaLocalVariableInitializerTest(test.TestCase):
         return v_sum
 
       with distribution.scope():
-        replica_local_var = distribution.call_for_each_replica(model_fn)
+        replica_local_var = distribution.extended.call_for_each_replica(
+            model_fn)
         self.assertTrue(isinstance(replica_local_var,
                                    values.ReplicaLocalVariable))
         self.assertFalse(self.evaluate(replica_local_var.is_initialized()))
@@ -1088,7 +1094,7 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
       return v_sum
 
     with distribution.scope():
-      replica_local_var = distribution.call_for_each_replica(model_fn)
+      replica_local_var = distribution.extended.call_for_each_replica(model_fn)
       self.assertTrue(isinstance(replica_local_var,
                                  values.ReplicaLocalVariable))
       self.evaluate(variables.global_variables_initializer())
@@ -1116,7 +1122,7 @@ class ReplicaLocalVariableAssignTest(test.TestCase):
       return v_sum
 
     with distribution.scope():
-      replica_local_var = distribution.call_for_each_replica(model_fn)
+      replica_local_var = distribution.extended.call_for_each_replica(model_fn)
       self.assertTrue(isinstance(replica_local_var,
                                  values.ReplicaLocalVariable))
       self.evaluate(variables.global_variables_initializer())
@@ -1181,7 +1187,7 @@ class MirroredStrategyDefunTest(test.TestCase):
       mock_model = MockModel(two_variables)
       self.evaluate(variables.global_variables_initializer())
 
-      result = distribution.call_for_each_replica(
+      result = distribution.extended.call_for_each_replica(
           model_fn, args=[mock_model] + inputs)
       for device in devices:
         device_result = values.select_device(device, result)
@@ -1194,8 +1200,9 @@ class MirroredStrategyDefunTest(test.TestCase):
         # call_for_each has one trace per device. To check that the expected set
         # of variables was accessed on each trace, we first retrieve each
         # device-specific graph function.
-        per_replica_graph_functions = distribution.call_for_each_replica(
-            defun.get_concrete_function, args=[mock_model] + inputs)
+        per_replica_graph_functions = (
+            distribution.extended.call_for_each_replica(
+                defun.get_concrete_function, args=[mock_model] + inputs))
         for device in devices:
           graph_function = per_replica_graph_functions.get(device=device)
           self.assertEqual(set(mock_model.variables),
@@ -1281,7 +1288,7 @@ class MirroredStrategyDefunTest(test.TestCase):
 
       gradients_fn = backprop.implicit_grad(loss_fn)
       gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn)
-      grads_and_vars = distribution.call_for_each_replica(
+      grads_and_vars = distribution.extended.call_for_each_replica(
           gradients_fn, args=(None,))
 
       optimizer = gradient_descent.GradientDescentOptimizer(0.25)
@@ -1361,7 +1368,7 @@ class MultiWorkerMirroredStrategyTest(
           expected_input_pipeline_id=None)
       iterator = distribution.make_input_fn_iterator(input_fn)
       self._test_input_fn_iterator(
-          iterator, distribution.worker_devices, expected_values, sess)
+          iterator, distribution.extended.worker_devices, expected_values, sess)
 
 
 class MultiWorkerMirroredStrategyTestWithChief(
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 2f6d38547c..f881c234ac 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -204,10 +204,6 @@ class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
         distribution_strategy,
         replica_id_in_sync_group=constant_op.constant(0, dtypes.int32))
 
-  @property
-  def device(self):
-    raise RuntimeError("Use .devices instead")
-
   @property
   def devices(self):
-    return [self._distribution_strategy.worker_devices[0]]
+    return [self._distribution_strategy.extended.worker_devices[0]]
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index b0a2ba3415..d46cd6f529 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -55,7 +55,7 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
         expected_input_pipeline_id=0)
     iterator = d.make_input_fn_iterator(input_fn)
     self._test_input_fn_iterator(
-        iterator, d.worker_devices, expected_values)
+        iterator, d.extended.worker_devices, expected_values)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index b4c098aa57..1ada6a6ba4 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -522,7 +522,7 @@ class ParameterServerStrategyTestBase(
                               expected_values):
     distribution, master_target, config = self._get_test_objects(
         task_type, task_id, num_gpus)
-    devices = distribution.worker_devices
+    devices = distribution.extended.worker_devices
 
     with ops.Graph().as_default(), \
          self.cached_session(config=config,
@@ -698,9 +698,9 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
           v = variable_scope.get_variable('v', initializer=10.0)
           _ = v * v
         v, = tape.watched_variables()
-        w = distribution.value_container(v)
+        w = distribution.extended.value_container(v)
         self.assertIs(values.AggregatingVariable, type(w))
-      distribution.call_for_each_replica(f)
+      distribution.extended.call_for_each_replica(f)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index de0abc6f04..5a8e8ed0dd 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -191,17 +191,18 @@ class DistributionTestBase(test.TestCase):
 
   def _test_replica_id(self, d):
     with d.scope():
-      expected_devices = [False] * len(d.worker_devices)
+      expected_devices = [False] * len(d.extended.worker_devices)
 
       def mark_devices_fn():
         replica_id = self.evaluate(
             ds_context.get_replica_context().replica_id_in_sync_group)
-        self.assertLess(replica_id, len(d.worker_devices))
+        self.assertLess(replica_id, len(d.extended.worker_devices))
         self.assertFalse(expected_devices[replica_id])
         expected_devices[replica_id] = True
 
       d.call_for_each_replica(mark_devices_fn)
-      self.assertAllEqual(expected_devices, [True] * len(d.worker_devices))
+      self.assertAllEqual(expected_devices,
+                          [True] * len(d.extended.worker_devices))
 
   def _test_call_and_merge_exceptions(self, dist):
     with dist.scope():
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 314dcc5e01..478e3accd7 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -556,13 +556,9 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
         # TODO(b/118385803): properly initialize replica_id, instead of always 0
         replica_id_in_sync_group=constant_op.constant(0, dtypes.int32))
 
-  @property
-  def device(self):
-    raise RuntimeError("Use .devices instead")
-
   @property
   def devices(self):
     distribute_lib.require_replica_context(self)
     ds = self._distribution_strategy
     replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
-    return [ds.worker_devices[replica_id]]
+    return [ds.extended.worker_devices[replica_id]]
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index d6dedc2774..a72db5e12f 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -447,7 +447,7 @@ class _OptimizerV2State(object):
     if v is None:
       if colocate_with is None:
         colocate_with = self._non_slot_devices
-      with self._distribution.colocate_vars_with(colocate_with):
+      with self._distribution.extended.colocate_vars_with(colocate_with):
         # TODO(josh11b): Use get_variable() except for the legacy Adam use case.
         v = variable_scope.variable(initial_value, name=name, trainable=False)
       self._non_slot_dict[name] = v
@@ -928,7 +928,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
 
   def _distributed_apply(self, distribution, grads_and_vars, global_step, name):
     """`apply_gradients` for use with a `DistributionStrategy`."""
-    reduced_grads = distribution.batch_reduce(
+    reduced_grads = distribution.extended.batch_reduce_to(
         ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     var_list = [v for _, v in grads_and_vars]
     grads_and_vars = zip(reduced_grads, var_list)
@@ -945,7 +945,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
     with ops.name_scope(name, self._name) as name:
       per_graph_state = self._get_or_create_state(var_list=unwrapped_var_list)
       # Include the current value of any dynamic hyper parameters in `state`.
-      non_slot_devices = distribution.non_slot_devices(var_list)
+      non_slot_devices = distribution.extended.non_slot_devices(var_list)
       state = per_graph_state._copy_with_dynamic_hyper(  # pylint: disable=protected-access
           self._hyper, distribution, non_slot_devices)
 
@@ -990,7 +990,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
       # Use the processors to update the variables.
       update_ops = []
       for grad, var in grads_and_vars:
-        update_ops.extend(distribution.update(var, update, grad, grouped=False))
+        update_ops.extend(distribution.extended.update(
+            var, update, args=(grad,), group=False))
 
       # Give the child class a chance to do something after applying
       # gradients
@@ -1002,8 +1003,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
 
       update_ops = control_flow_ops.group(update_ops)
       with ops.control_dependencies([update_ops]):
-        finish_updates = distribution.update_non_slot(
-            non_slot_devices, finish, grouped=False)
+        finish_updates = distribution.extended.update_non_slot(
+            non_slot_devices, finish, group=False)
       # We said grouped=False, which means finish_updates is always a list.
       # It will be [None] when finish() returns None.
       if finish_updates == [None]:
@@ -1018,8 +1019,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
           def update_global_step(global_step, name):
             return global_step.assign_add(1, read_value=False, name=name)
 
-          apply_updates = distribution.update(global_step, update_global_step,
-                                              name)
+          apply_updates = distribution.extended.update(
+              global_step, update_global_step, args=(name,))
 
       # Add the training op to the TRAIN_OP graph collection in graph mode.
       if not eager_execution:
diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 07d291e037..c0f9b8a1fd 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -245,7 +245,7 @@ class _WorkerContext(object):
     else:
       session_config = self._session_config
 
-    if not self._strategy or self._strategy.should_init:
+    if not self._strategy or self._strategy.extended.experimental_should_init:
       logging.info("Creating chief session creator with config: %r", config)
       return monitored_session.ChiefSessionCreator(
           scaffold,
@@ -306,19 +306,19 @@ class _WorkerContext(object):
     return self._num_workers
 
   @property
-  def should_init(self):
+  def experimental_should_init(self):
     """Whether to run init ops."""
-    return self._strategy.should_init
+    return self._strategy.extended.experimental_should_init
 
   @property
   def should_checkpoint(self):
     """Whether to save checkpoint."""
-    return self._strategy.should_checkpoint
+    return self._strategy.extended.should_checkpoint
 
   @property
   def should_save_summary(self):
     """Whether to save summaries."""
-    return self._strategy.should_save_summary
+    return self._strategy.extended.should_save_summary
 
 
 def _run_single_worker(worker_fn,
@@ -632,10 +632,10 @@ def run_distribute_coordinator(worker_fn,
   The `strategy` object is expected to be a DistributionStrategy object which
   has implemented methods needed by distributed coordinator such as
   `configure(session_config, cluster_spec, task_type, task_id)` which configures
-  the strategy object for a specific task and `should_init` property which
-  instructs the distribute coordinator whether to run init ops for a task. The
-  distribute coordinator will make a copy of the `strategy` object, call its
-  `configure` method and pass it to `worker_fn` as an argument.
+  the strategy object for a specific task and `experimental_should_init`
+  property which instructs the distribute coordinator whether to run init ops
+  for a task. The distribute coordinator will make a copy of the `strategy`
+  object, call its `configure` method and pass it to `worker_fn` as an argument.
 
   The `worker_fn` defines the training logic and is called under a its own
   worker context which can be accessed to via `get_current_worker_context`. A
@@ -758,7 +758,7 @@ def run_distribute_coordinator(worker_fn,
     # The client must know the cluster but servers in the cluster don't have to
     # know the client.
     if task_type in [_TaskType.CLIENT, None]:
-      if strategy.between_graph:
+      if strategy.extended.experimental_between_graph:
         return _run_between_graph_client(worker_fn, strategy, eval_fn,
                                          eval_strategy, cluster_spec,
                                          session_config, rpc_layer)
@@ -804,7 +804,7 @@ def run_distribute_coordinator(worker_fn,
         environment=environment)
 
     if task_type in [_TaskType.CHIEF, _TaskType.WORKER]:
-      if strategy.between_graph:
+      if strategy.extended.experimental_between_graph:
         # All jobs run `worker_fn` if between-graph.
         _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
                            task_id, session_config, rpc_layer)
diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py
index 0c1ee8c87e..f2cb950aad 100644
--- a/tensorflow/python/distribute/distribute_coordinator_test.py
+++ b/tensorflow/python/distribute/distribute_coordinator_test.py
@@ -79,46 +79,53 @@ def _strip_protocol(target):
     return target
 
 
-class MockStrategy(object):
+class MockExtended(object):
 
   def __init__(self,
                between_graph=False,
                should_init=None,
                should_checkpoint=None,
                should_save_summary=None):
-    self._between_graph = between_graph
-    self._should_init = should_init
-    self._should_checkpoint = should_checkpoint
-    self._should_save_summary = should_save_summary
+    self.experimental_between_graph = between_graph
+    self.experimental_should_init = should_init
+    self.should_checkpoint = should_checkpoint
+    self.should_save_summary = should_save_summary
 
-  @property
-  def between_graph(self):
-    return self._between_graph
+
+class MockStrategy(object):
+
+  def __init__(self,
+               between_graph=False,
+               should_init=None,
+               should_checkpoint=None,
+               should_save_summary=None):
+    self.extended = MockExtended(between_graph, should_init, should_checkpoint,
+                                 should_save_summary)
 
   def configure(self,
                 session_config=None,
                 cluster_spec=None,
                 task_type=None,
                 task_id=None):
-    if self._should_init is None:
+    if self.extended.experimental_should_init is None:
       if task_id == 0:
-        self._should_init = True
+        self.extended.experimental_should_init = True
       else:
-        self._should_init = False
-    if self._should_checkpoint is None:
+        self.extended.experimental_should_init = False
+    if self.extended.should_checkpoint is None:
       if task_id == 0:
-        self._should_checkpoint = True
+        self.extended.should_checkpoint = True
       else:
-        self._should_checkpoint = False
-    if self._should_save_summary is None:
+        self.extended.should_checkpoint = False
+    if self.extended.should_save_summary is None:
       if task_id == 0:
-        self._should_save_summary = True
+        self.extended.should_save_summary = True
       else:
-        self._should_save_summary = False
+        self.extended.should_save_summary = False
 
     if session_config:
       if (cluster_spec and task_type and task_id is not None and
-          self._between_graph):
+          self.extended.experimental_between_graph):
         session_config.intra_op_parallelism_threads += 1
         if task_type in ["chief", "worker"]:
           session_config.device_filters.extend(
@@ -127,18 +134,6 @@ class MockStrategy(object):
         session_config.inter_op_parallelism_threads += 1
         session_config.device_filters.append("/job:somejob")
 
-  @property
-  def should_init(self):
-    return self._should_init
-
-  @property
-  def should_checkpoint(self):
-    return self._should_checkpoint
-
-  @property
-  def should_save_summary(self):
-    return self._should_save_summary
-
 
 class MockServer(object):
 
@@ -373,9 +368,12 @@ class DistributeCoordinatorTestBase(test.TestCase):
     context = distribute_coordinator_context.get_current_worker_context()
     self.assertTrue(context is not None)
 
-    self.assertEqual(context._strategy.should_init, strategy.should_init)
-    self.assertEqual(context.should_checkpoint, strategy.should_checkpoint)
-    self.assertEqual(context.should_save_summary, strategy.should_save_summary)
+    self.assertEqual(context._strategy.extended.experimental_should_init,
+                     strategy.extended.experimental_should_init)
+    self.assertEqual(context.should_checkpoint,
+                     strategy.extended.should_checkpoint)
+    self.assertEqual(context.should_save_summary,
+                     strategy.extended.should_save_summary)
 
     task_type = str(context.task_type)
     task_id = context.task_id or 0
@@ -385,7 +383,8 @@ class DistributeCoordinatorTestBase(test.TestCase):
       while len(self._strategy_property[task_type]) <= task_id:
         self._strategy_property[task_type].append(None)
       self._strategy_property[task_type][task_id] = (
-          context._strategy.should_init, context.should_checkpoint,
+          context._strategy.extended.experimental_should_init,
+          context.should_checkpoint,
           context.should_save_summary)
 
   def _run_mock_std_server(self,
diff --git a/tensorflow/python/distribute/estimator_training.py b/tensorflow/python/distribute/estimator_training.py
index 227b00fb3e..549fa8fb8a 100644
--- a/tensorflow/python/distribute/estimator_training.py
+++ b/tensorflow/python/distribute/estimator_training.py
@@ -308,7 +308,7 @@ def estimator_train(estimator, train_distributed_fn, hooks):
     raise ValueError('Only `STANDALONE_CLIENT` mode is supported when you call '
                      '`estimator.train`')
 
-  if estimator._config._train_distribute.between_graph:
+  if estimator._config._train_distribute.extended.experimental_between_graph:
     # TODO(yuefengz): remove this limitation once we figure out how to merge
     # return values from `_worker_fn`s.
     raise ValueError('`Estimator.train` API is not supported for %s with '
@@ -356,7 +356,7 @@ def estimator_evaluate(estimator, evaluate_distributed_fn, hooks):
     raise ValueError('Only `STANDALONE_CLIENT` mode is supported when you call '
                      '`Estimator.train`')
 
-  if estimator._config._eval_distribute.between_graph:
+  if estimator._config._eval_distribute.extended.experimental_between_graph:
     # TODO(yuefengz): remove this limitation once we figure out how to merge
     # return values from `_worker_fn`s.
     raise ValueError('`Estimator.evaluate` API is not supported for %s with '
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 33ca27c63b..5f69323bff 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -1608,11 +1608,11 @@ class MultiStepContext(object):
   """A context object that can be used to capture things when running steps.
 
   This context object is useful when running multiple steps at a time using the
-  `run_steps_on_dataset` API. For e.g. it allows the user's step function to
-  specify which outputs to emit at what frequency. Currently it supports
-  capturing output from the last step, as well as capturing non tensor outputs.
-  In the future it will be augmented to support other use cases such as output
-  each N steps.
+  `experimental_run_steps_on_iterator` API. For e.g. it allows the user's step
+  function to specify which outputs to emit at what frequency. Currently it
+  supports capturing output from the last step, as well as capturing non tensor
+  outputs.  In the future it will be augmented to support other use cases such
+  as output each N steps.
   """
 
   def __init__(self):
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 1326f09713..e501b403a3 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -63,7 +63,7 @@ def watch_variable(tape, variable):
   """Marks this variable to be watched by the given tape."""
   strategy = distribution_strategy_context.get_distribution_strategy()
   if distribution_strategy_context.get_replica_context():
-    variables = [strategy.value_container(variable)]
+    variables = [strategy.extended.value_container(variable)]
   else:
     variables = strategy.unwrap(variable)
   for var in variables:
@@ -78,7 +78,7 @@ def variable_accessed(variable):
   """
   strategy = distribution_strategy_context.get_distribution_strategy()
   if distribution_strategy_context.get_replica_context():
-    variables = [strategy.value_container(variable)]
+    variables = [strategy.extended.value_container(variable)]
   else:
     variables = strategy.unwrap(variable)
   for var in variables:
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 878451d4cf..53261fdd26 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -109,7 +109,7 @@ def experimental_fit_loop(model,
         mode=_Mode.TRAIN)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = current_strategy.call_for_each_replica(
+     grouped_session_args) = current_strategy.extended.call_for_each_replica(
          _per_device_fit_function, args=(model._grouped_model_train,))
     (all_inputs, all_outputs, all_updates,
      all_session_args) = distributed_training_utils.unwrap_values(
@@ -152,7 +152,7 @@ def experimental_fit_loop(model,
       name='steps_per_run')
 
   with current_strategy.scope():
-    ctx = current_strategy.run_steps_on_dataset(
+    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
         step_fn, iterator, iterations=steps_per_run,
         initial_loop_values=initial_loop_values)
 
@@ -300,7 +300,7 @@ def experimental_test_loop(model,
         mode=_Mode.TEST)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = current_strategy.call_for_each_replica(
+     grouped_session_args) = current_strategy.extended.call_for_each_replica(
          _per_device_eval_function, args=(model._grouped_model_test,))
 
     (all_inputs, all_outputs, all_updates,
@@ -335,7 +335,7 @@ def experimental_test_loop(model,
   with current_strategy.scope():
     # TODO(priyag): Use steps_per_run when we use new metrics as they will
     # allow handling metric computation at each step using variables.
-    ctx = current_strategy.run_steps_on_dataset(
+    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
         step_fn, iterator, iterations=1,
         initial_loop_values=initial_loop_values)
 
@@ -414,7 +414,7 @@ def experimental_predict_loop(model, iterator, verbose=0, steps=None):
         mode=_Mode.PREDICT)
 
     (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = current_strategy.call_for_each_replica(
+     grouped_session_args) = current_strategy.extended.call_for_each_replica(
          _per_device_predict_function, args=(model._grouped_model_predict,))
 
     (all_inputs, all_outputs, all_updates,
@@ -445,7 +445,7 @@ def experimental_predict_loop(model, iterator, verbose=0, steps=None):
 
   with current_strategy.scope():
     # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed.
-    ctx = current_strategy.run_steps_on_dataset(
+    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
         step_fn, iterator, iterations=1,
         initial_loop_values=initial_loop_values)
 
@@ -528,7 +528,7 @@ def clone_model_on_replicas(model, strategy, make_callback_model=False,
                             inputs=None, targets=None, mode=None):
   """Create a cloned model on each replica."""
   with strategy.scope():
-    grouped_model = strategy.call_for_each_replica(
+    grouped_model = strategy.extended.call_for_each_replica(
         _clone_and_build_model, args=(model, inputs, targets))
     if mode is _Mode.TRAIN:
       model._grouped_model_train = grouped_model
@@ -583,7 +583,7 @@ def _get_execution_function(model, mode):
     # Create train ops on each of the devices when we call
     # `_per_device_fit_function`.
     (grouped_inputs, grouped_outputs, grouped_updates,
-     grouped_session_args) = strategy.call_for_each_replica(
+     grouped_session_args) = strategy.extended.call_for_each_replica(
          _per_device_function, args=(model._grouped_model,))
 
     if mode == 'train':
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
index ad4d50c548..4758e3d3d4 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/training/distribute_test.py
@@ -92,9 +92,9 @@ class TestStrategyTest(test.TestCase):
                            variable_scope.variable(1.0, name="bar"))
 
     with self.assertRaises(RuntimeError):
-      dist.call_for_each_replica(run_fn)
+      dist.extended.call_for_each_replica(run_fn)
     with dist.scope():
-      dist.call_for_each_replica(run_fn)
+      dist.extended.call_for_each_replica(run_fn)
     _assert_in_default_state(self)
 
   def testScope(self):
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index ebe2f15a55..2ceb387ec3 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -382,6 +382,16 @@ class MonitoredTrainingSessionTest(test.TestCase):
         self.assertEqual(0, session.run(gstep))
 
 
+class MockExtended(object):
+
+  def __init__(self, between_graph, should_init, should_checkpoint,
+               should_save_summary):
+    self.experimental_between_graph = between_graph
+    self.experimental_should_init = should_init
+    self.should_checkpoint = should_checkpoint
+    self.should_save_summary = should_save_summary
+
+
 class MockStrategy(object):
 
   def __init__(self,
@@ -389,26 +399,8 @@ class MockStrategy(object):
                should_init=True,
                should_checkpoint=None,
                should_save_summary=None):
-    self._between_graph = between_graph
-    self._should_init = should_init
-    self._should_checkpoint = should_checkpoint
-    self._should_save_summary = should_save_summary
-
-  @property
-  def between_graph(self):
-    return self._between_graph
-
-  @property
-  def should_init(self):
-    return self._should_init
-
-  @property
-  def should_checkpoint(self):
-    return self._should_checkpoint
-
-  @property
-  def should_save_summary(self):
-    return self._should_save_summary
+    self.extended = MockExtended(between_graph, should_init, should_checkpoint,
+                                 should_save_summary)
 
 
 class MonitoredTrainingSessionWithDistributeCoordinatorTest(test.TestCase):
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 6fca4ca7d4..8cd5311b31 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -660,7 +660,7 @@ class Optimizer(
       replicas. If `global_step` was not None, that operation also
       increments `global_step`
     """
-    reduced_grads = distribution.batch_reduce(
+    reduced_grads = distribution.extended.batch_reduce_to(
         ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     var_list = [v for _, v in grads_and_vars]
     grads_and_vars = zip(reduced_grads, var_list)
@@ -695,21 +695,23 @@ class Optimizer(
       update_ops = [
           op
           for grad, var in grads_and_vars
-          for op in distribution.update(var, update, grad, grouped=False)
+          for op in distribution.extended.update(
+              var, update, args=(grad,), group=False)
       ]
 
       def finish(self, update_ops):
         return self._finish(update_ops, "update")
 
-      non_slot_devices = distribution.non_slot_devices(var_list)
-      finish_updates = distribution.update_non_slot(
-          non_slot_devices, finish, self, update_ops, grouped=False)
+      non_slot_devices = distribution.extended.non_slot_devices(var_list)
+      finish_updates = distribution.extended.update_non_slot(
+          non_slot_devices, finish, args=(self, update_ops), group=False)
       if global_step is None:
         apply_updates = distribution.group(finish_updates, name=name)
       else:
         with ops.control_dependencies(finish_updates):
-          apply_updates = distribution.update(
-              global_step, state_ops.assign_add, 1, name=name)
+          apply_updates = distribution.extended.update(
+              global_step, state_ops.assign_add, args=(1,),
+              kwargs={"name": name})
 
       if not context.executing_eagerly():
         if isinstance(apply_updates, ops.Tensor):
-- 
GitLab


From a8ae0f47e56d778b69278f7eef971c3b6b605bf9 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 20 Nov 2018 11:52:19 -0800
Subject: [PATCH 0602/1554] Minor error message improvement for incompatible
 type conversion.

PiperOrigin-RevId: 222282332
---
 tensorflow/python/eager/pywrap_tensor.cc  | 4 ++--
 tensorflow/python/eager/pywrap_tfe_src.cc | 6 ++++--
 tensorflow/python/eager/tensor_test.py    | 8 ++++++++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 55f0896e3b..ed19047f09 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -439,8 +439,8 @@ int EagerTensor_init(EagerTensor* self, PyObject* args, PyObject* kwds) {
       PyErr_SetString(
           PyExc_TypeError,
           tensorflow::strings::StrCat(
-              "Cannot convert value ", TFE_GetPythonString(value_str.get()),
-              " to EagerTensor with requested dtype: ",
+              "Cannot convert provided value to EagerTensor. Provided value: ",
+              TFE_GetPythonString(value_str.get()), " Requested dtype: ",
               tensorflow::DataTypeString(
                   static_cast<tensorflow::DataType>(desired_dtype)))
               .c_str());
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index f074b73a9f..9ce500bc08 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -2303,8 +2303,10 @@ bool ConvertToTensor(
       PyErr_SetString(
           PyExc_TypeError,
           tensorflow::strings::StrCat(
-              "Cannot convert value ", TFE_GetPythonString(input_str.get()),
-              " to EagerTensor with requested dtype: ", desired_dtype)
+              "Cannot convert provided value to EagerTensor. Provided value: ",
+              TFE_GetPythonString(input_str.get()), " Requested dtype: ",
+              tensorflow::DataTypeString(
+                  static_cast<tensorflow::DataType>(desired_dtype)))
               .c_str());
       return false;
     }
diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index d0500a413d..8c9d5dabe7 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -323,6 +323,14 @@ class TFETensorTest(test_util.TensorFlowTestCase):
   def testConvertToTensorAllowsOverflow(self):
     _ = ops.convert_to_tensor(123456789, dtype=dtypes.uint8)
 
+  def testEagerTensorError(self):
+    with self.assertRaisesRegexp(
+        TypeError,
+        "Cannot convert provided value to EagerTensor. "
+        "Provided value.*Requested dtype.*"):
+      _ = ops.convert_to_tensor(1., dtype=dtypes.int32)
+
+
 
 class TFETensorUtilTest(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 2cbeb0cb762d31e69d4dd1adba9008c27441a1e5 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Nov 2018 11:59:09 -0800
Subject: [PATCH 0603/1554] Enable iota in HLO evaluator for all integral and
 floating point types.

PiperOrigin-RevId: 222283442
---
 .../xla/service/hlo_evaluator_typed_visitor.h     | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 332fa874c3..f975ba63bb 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -2543,12 +2543,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
   template <typename NativeT,
             typename std::enable_if<
-                std::is_same<NativeT, float>::value ||
-                std::is_same<NativeT, int32>::value ||
-                std::is_same<NativeT, uint32>::value>::type* = nullptr>
+                std::is_integral<NativeT>::value ||
+                std::is_floating_point<NativeT>::value>::type* = nullptr>
   Status HandleIota(HloInstruction* instruction) {
     auto* iota = Cast<HloIotaInstruction>(instruction);
-    std::vector<NativeT> data(iota->shape().dimensions(iota->iota_dimension()));
+    // Avoid using std::vector since std::vector<bool> does not convert to
+    // absl::Span<bool>.
+    absl::InlinedVector<NativeT, 1> data(
+        iota->shape().dimensions(iota->iota_dimension()));
     std::iota(data.begin(), data.end(), 0);
     auto result = LiteralUtil::CreateR1<NativeT>(data);
 
@@ -2565,9 +2567,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
   template <typename NativeT,
             typename std::enable_if<
-                !(std::is_same<NativeT, float>::value ||
-                  std::is_same<NativeT, int32>::value ||
-                  std::is_same<NativeT, uint32>::value)>::type* = nullptr>
+                !(std::is_integral<NativeT>::value ||
+                  std::is_floating_point<NativeT>::value)>::type* = nullptr>
   Status HandleIota(HloInstruction* iota) {
     return InvalidArgument("Unsupported type for iota");
   }
-- 
GitLab


From 7aeef997590ef434a71db1546eec173ff29640ea Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 20 Nov 2018 11:59:30 -0800
Subject: [PATCH 0604/1554] Fix profiler after subgraph refactor and remove
 commented tensors_.

PiperOrigin-RevId: 222283502
---
 tensorflow/lite/core/subgraph.h |  8 ++++++++
 tensorflow/lite/interpreter.cc  |  8 ++++++++
 tensorflow/lite/interpreter.h   | 13 ++-----------
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index 9783747ff6..120c307ef2 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/lite/allocation.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/memory_planner.h"
+#include "tensorflow/lite/profiling/profiler.h"
 #include "tensorflow/lite/util.h"
 
 namespace tflite {
@@ -242,6 +243,10 @@ class Subgraph {
   // WARNING: This is an experimental API and subject to change.
   TfLiteStatus ResetVariableTensors();
 
+  void SetProfiler(profiling::Profiler* profiler) { profiler_ = profiler; }
+
+  profiling::Profiler* GetProfiler() { return profiler_; }
+
  private:
   // Prevent 'context_' from accessing functions that are only available to
   // delegated kernels.
@@ -470,6 +475,9 @@ class Subgraph {
 
   // External contexts (kTfLiteMaxExternalContexts).
   TfLiteExternalContext** external_contexts_;
+
+  // Profiler for this interpreter instance.
+  profiling::Profiler* profiler_ = nullptr;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 4f4a999210..078c2306c4 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -190,4 +190,12 @@ TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
   return kTfLiteOk;
 }
 
+void Interpreter::SetProfiler(profiling::Profiler* profiler) {
+  for (auto& subgraph : subgraphs_) subgraph.SetProfiler(profiler);
+}
+
+profiling::Profiler* Interpreter::GetProfiler() {
+  return primary_subgraph().GetProfiler();
+}
+
 }  // namespace tflite
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index d89affff45..7da4a2cbfa 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -380,9 +380,9 @@ class Interpreter {
                                TfLiteBufferHandle* buffer_handle,
                                TfLiteDelegate** delegate);
 
-  void SetProfiler(profiling::Profiler* profiler) { profiler_ = profiler; }
+  void SetProfiler(profiling::Profiler* profiler);
 
-  profiling::Profiler* GetProfiler() { return profiler_; }
+  profiling::Profiler* GetProfiler();
 
   // The default capacity of `tensors_` vector.
   static constexpr int kTensorsReservedCapacity = 128;
@@ -434,12 +434,6 @@ class Interpreter {
     return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
-  // Tensors needed by the interpreter. Use `AddTensors` to add more blank
-  // tensor entries. Note, `tensors_.data()` needs to be synchronized to the
-  // `context_` whenever this std::vector is reallocated. Currently this
-  // only happens in `AddTensors()`.
-  // std::vector<TfLiteTensor> tensors_;
-
   // Set the value of an external context.
   static void SetExternalContext(struct TfLiteContext* context,
                                  TfLiteExternalContextType type,
@@ -472,9 +466,6 @@ class Interpreter {
 
   bool allow_buffer_handle_output_ = false;
 
-  // Profiler for this interpreter instance.
-  profiling::Profiler* profiler_ = nullptr;
-
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
 
-- 
GitLab


From 9947625c3f1c3d43f12125b7af1cdbdd8bea7ff2 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 20 Nov 2018 12:00:43 -0800
Subject: [PATCH 0605/1554] Add new method to expose all the function names in
 the registry.

Also refactor the ReachableFunctions so that it does not rely on the ToProto()
method for all the functionDefs.

PiperOrigin-RevId: 222283700
---
 tensorflow/core/framework/function.cc      | 20 +++++++++++++++-----
 tensorflow/core/framework/function.h       |  3 +++
 tensorflow/core/framework/function_test.cc | 11 +++++++++++
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index 838f899911..6809c27197 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -1241,6 +1241,16 @@ const FunctionDef* FunctionLibraryDefinition::GetAttrImpl(
   }
 }
 
+std::vector<string> FunctionLibraryDefinition::ListFunctionNames() const {
+  std::vector<string> function_names;
+  tf_shared_lock l(mu_);
+  function_names.reserve(function_defs_.size());
+  for (const auto& it : function_defs_) {
+    function_names.emplace_back(it.first);
+  }
+  return function_names;
+}
+
 FunctionDefLibrary FunctionLibraryDefinition::ToProto() const {
   FunctionDefLibrary lib;
   tf_shared_lock l(mu_);
@@ -1357,12 +1367,12 @@ absl::flat_hash_set<string> ReachableFunctions(
     if (!grad_func_name.empty()) add_to_func_queue(grad_func_name);
   }
 
-  const FunctionDefLibrary library_proto = flib.ToProto();
-  for (const auto& it : library_proto.function()) {
-    const auto attr_it = it.attr().find(kExperimentalApiImplements);
-    if (attr_it != it.attr().end()) {
+  for (const auto& func_name : flib.ListFunctionNames()) {
+    const auto& func_def = flib.Find(func_name);
+    const auto attr_it = func_def->attr().find(kExperimentalApiImplements);
+    if (attr_it != func_def->attr().end()) {
       if (reachable_api_interface.contains(attr_it->second.s())) {
-        reachable_funcs.insert(it.signature().name());
+        reachable_funcs.insert(func_name);
       }
     }
   }
diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h
index 6792cf1653..9cf4b0f4cd 100644
--- a/tensorflow/core/framework/function.h
+++ b/tensorflow/core/framework/function.h
@@ -407,6 +407,9 @@ class FunctionLibraryDefinition : public OpRegistryInterface {
     return function_defs_.size();
   }
 
+  // Returns all the function names in the FunctionLibraryDefinition.
+  std::vector<string> ListFunctionNames() const LOCKS_EXCLUDED(mu_);
+
   const OpRegistryInterface* default_registry() const {
     return default_registry_;
   }
diff --git a/tensorflow/core/framework/function_test.cc b/tensorflow/core/framework/function_test.cc
index f57a79b167..75d45fa2c8 100644
--- a/tensorflow/core/framework/function_test.cc
+++ b/tensorflow/core/framework/function_test.cc
@@ -1213,6 +1213,17 @@ TEST(FunctionLibraryDefinitionTest, ToProto) {
   EXPECT_EQ(f3->DebugString(), f4->DebugString());
 }
 
+TEST(FunctionLibraryDefinitionTest, FunctionNames) {
+  FunctionDefLibrary proto;
+  *proto.add_function() = test::function::XTimesTwo();
+  *proto.add_function() = test::function::WXPlusB();
+  const FunctionLibraryDefinition lib_def(OpRegistry::Global(), proto);
+
+  const std::vector<string> function_names = lib_def.ListFunctionNames();
+  const std::vector<string> expected = {"XTimesTwo", "WXPlusB"};
+  EXPECT_EQ(function_names, expected);
+}
+
 TEST(FunctionLibraryDefinitionTest, GetAttr_FuncNoAttr) {
   FunctionDefLibrary proto;
   *proto.add_function() = test::function::XTimesTwo();
-- 
GitLab


From 08d0de579e94c78bf5561bf312939e38d78c7d32 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 20 Nov 2018 12:31:09 -0800
Subject: [PATCH 0606/1554] [TF:XLA] Remove no-op return in literal.cc.

PiperOrigin-RevId: 222288471
---
 tensorflow/compiler/xla/literal.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index fcc59f6d21..36ad7c6486 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -1123,7 +1123,6 @@ void DenseArrayToStringHelper(const LiteralBase& literal,
             }
           }
           pieces->push_back(brace_to_string("}"));
-          return;
         }
       };
 
-- 
GitLab


From 0b0767e77f0e5bdc3a625de56cfed8a57d951e98 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 13:07:36 -0800
Subject: [PATCH 0607/1554] Automated rollback of commit
 e510289890c19b4ebab990e101d918b4c37029a7

PiperOrigin-RevId: 222294175
---
 tensorflow/lite/toco/import_tensorflow.cc      | 5 ++++-
 tensorflow/lite/toco/import_tensorflow_test.cc | 4 +++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index c51031b4e5..4c3a0717e7 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -219,7 +219,10 @@ tensorflow::Status ImportShape(
   // allocation code gets a bit confused. It seems that the code expects an
   // empty shape for zero-sized shapes, so we will do just that, except for the
   // [0] case.
-  if (zero_sized_shape && input_dims_only_sizes.size() > 1) {
+  // TODO(b/119325030): In order to correctly import the "scalar" shapes the
+  // following test must include "&& input_dims_only_sizes.size() > 1", but
+  // that seems to slow everything down a lot.
+  if (zero_sized_shape) {
     shape->mutable_dims()->clear();
     if (input_flat_size != nullptr) *input_flat_size = 0;
     return tensorflow::Status::OK();
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index 07b52d3970..0be358b1f7 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -190,7 +190,9 @@ TEST_P(ShapeImportTest, ShapeIsOneDimZero) {
   EXPECT_TRUE(ImportNode(node, &model).ok());
 
   const auto& array = model.GetArray("Node1");
-  EXPECT_THAT(array.shape().dims(), ::testing::ElementsAre(0));
+  // We would like to have [0] shapes actually import correctly, but
+  // for some reason that slows everything down.
+  EXPECT_THAT(array.shape().dims(), ::testing::ElementsAre());
 }
 
 TEST_P(ShapeImportTest, ShapeElementTooLarge) {
-- 
GitLab


From 5c4efd94603671df138d8df6bf1c9b51b8259270 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 13:19:32 -0800
Subject: [PATCH 0608/1554] Fix a convolution test's name

PiperOrigin-RevId: 222295785
---
 tensorflow/compiler/xla/tests/convolution_test.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index a5e9cfd0cc..b52d30fd66 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -1282,7 +1282,7 @@ TYPED_TEST(Convolve2D_1x4x4x1024_3x3x1x1024_Depthwise_Valid, Types) {
 }
 
 template <typename T>
-class Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid : public ConvolutionTest {
+class Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid : public ConvolutionTest {
  public:
   void RunTest() {
     XlaBuilder builder(TestName());
@@ -1341,8 +1341,8 @@ class Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid : public ConvolutionTest {
   }
 };
 
-TYPED_TEST_CASE(Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid, TestTypes);
-TYPED_TEST(Convolve2D_1x2x2x6_2x2x1x12_Grouped_Valid, Types) {
+TYPED_TEST_CASE(Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid, TestTypes);
+TYPED_TEST(Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid, Types) {
   this->RunTest();
 }
 
-- 
GitLab


From 760ef2cc7ec0d28b34e4544efb6433315f8fc99d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 13:34:35 -0800
Subject: [PATCH 0609/1554] Allow intermediate string tensors

PiperOrigin-RevId: 222298206
---
 tensorflow/lite/toco/BUILD                |  46 ++++++
 tensorflow/lite/toco/export_tensorflow.cc |  21 ++-
 tensorflow/lite/toco/toco.cc              |  82 +---------
 tensorflow/lite/toco/toco_convert.cc      | 108 ++++++++++++++
 tensorflow/lite/toco/toco_convert.h       |  34 +++++
 tensorflow/lite/toco/toco_convert_test.cc | 173 ++++++++++++++++++++++
 tensorflow/lite/toco/toco_tooling.cc      |   6 +-
 tensorflow/lite/toco/tooling_util.cc      |   8 +
 8 files changed, 390 insertions(+), 88 deletions(-)
 create mode 100644 tensorflow/lite/toco/toco_convert.cc
 create mode 100644 tensorflow/lite/toco/toco_convert.h
 create mode 100644 tensorflow/lite/toco/toco_convert_test.cc

diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 1430287444..82aa1f557e 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -395,6 +395,28 @@ tf_cc_test(
 
 # :toco is the main public command-line tool exposing the functionality
 # of the :toco_tooling library.
+cc_library(
+    name = "toco_convert",
+    srcs = ["toco_convert.cc"],
+    hdrs = ["toco_convert.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":model",
+        ":model_cmdline_flags",
+        ":model_flags_proto_cc",
+        ":toco_cmdline_flags",
+        ":toco_flags_proto_cc",
+        ":toco_port",
+        ":toco_tooling",
+        ":types_proto_cc",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/core:lib",
+        # We cannot embed the core:ops dependency directly into :toco_tooling as
+        # it can conflict with downstream deps when toco is used as a library.
+        "//tensorflow/core:ops",
+    ],
+)
+
 tf_cc_binary(
     name = "toco",
     srcs = ["toco.cc"],
@@ -404,6 +426,7 @@ tf_cc_binary(
         ":model_cmdline_flags",
         ":model_flags_proto_cc",
         ":toco_cmdline_flags",
+        ":toco_convert",
         ":toco_flags_proto_cc",
         ":toco_port",
         ":toco_tooling",
@@ -416,6 +439,29 @@ tf_cc_binary(
     ],
 )
 
+tf_cc_test(
+    name = "toco_convert_test",
+    srcs = ["toco_convert_test.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":model",
+        ":model_cmdline_flags",
+        ":model_flags_proto_cc",
+        ":toco_cmdline_flags",
+        ":toco_convert",
+        ":toco_flags_proto_cc",
+        ":toco_port",
+        ":toco_tooling",
+        ":types_proto_cc",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/core:lib",
+        # We cannot embed the core:ops dependency directly into :toco_tooling as
+        # it can conflict with downstream deps when toco is used as a library.
+        "//tensorflow/core:ops",
+    ],
+)
+
 tf_cc_test(
     name = "toco_port_test",
     srcs = ["toco_port_test.cc"],
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index 1752745aae..bdc3a5b0fb 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -48,7 +48,8 @@ using tensorflow::TensorProto;
 namespace toco {
 namespace {
 
-tensorflow::DataType GetTensorFlowDataType(ArrayDataType data_type) {
+tensorflow::DataType GetTensorFlowDataType(ArrayDataType data_type,
+                                           const string& error_location) {
   switch (data_type) {
     case ArrayDataType::kBool:
       return tensorflow::DT_BOOL;
@@ -66,14 +67,21 @@ tensorflow::DataType GetTensorFlowDataType(ArrayDataType data_type) {
       return tensorflow::DT_COMPLEX64;
     default:
     case ArrayDataType::kNone:
-      LOG(FATAL) << "Unsupported data type: " << static_cast<int>(data_type);
+      LOG(FATAL) << "Unsupported data type '" << ArrayDataTypeName(data_type)
+                 << "' in " << error_location;
       return tensorflow::DT_INVALID;
   }
 }
 
+tensorflow::DataType GetTensorFlowDataTypeForOp(ArrayDataType data_type,
+                                                const string& op_name) {
+  return GetTensorFlowDataType(data_type, "op '" + op_name + "'");
+}
+
 tensorflow::DataType GetTensorFlowDataType(const Model& model,
                                            const string& array_name) {
-  return GetTensorFlowDataType(model.GetArray(array_name).data_type);
+  return GetTensorFlowDataType(model.GetArray(array_name).data_type,
+                               "array '" + array_name + "'");
 }
 
 // TensorFlow sometimes forbids what it calls "legacy scalars",
@@ -1285,7 +1293,7 @@ void ConvertRangeOperator(const Model& model, const RangeOperator& src_op,
   *range_op->add_input() = src_op.inputs[1];
   *range_op->add_input() = src_op.inputs[2];
   (*range_op->mutable_attr())["Tidx"].set_type(
-      GetTensorFlowDataType(src_op.dtype));
+      GetTensorFlowDataTypeForOp(src_op.dtype, /*op_name=*/src_op.outputs[0]));
 }
 
 void ConvertPackOperator(const Model& model, const PackOperator& src_op,
@@ -1298,7 +1306,8 @@ void ConvertPackOperator(const Model& model, const PackOperator& src_op,
   }
   (*pack_op->mutable_attr())["axis"].set_i(src_op.axis);
   (*pack_op->mutable_attr())["N"].set_i(src_op.inputs.size());
-  (*pack_op->mutable_attr())["T"].set_type(GetTensorFlowDataType(src_op.dtype));
+  (*pack_op->mutable_attr())["T"].set_type(
+      GetTensorFlowDataTypeForOp(src_op.dtype, src_op.outputs[0]));
 }
 
 void ConvertFillOperator(const Model& model, const FillOperator& src_op,
@@ -1887,7 +1896,7 @@ void ConvertRandomUniformOperator(const Model& model,
       GetTensorFlowDataType(model, src_op.inputs[0]);
   (*new_op->mutable_attr())["T"].set_type(shape_type);
   (*new_op->mutable_attr())["dtype"].set_type(
-      GetTensorFlowDataType(src_op.dtype));
+      GetTensorFlowDataTypeForOp(src_op.dtype, src_op.outputs[0]));
   (*new_op->mutable_attr())["seed"].set_i(src_op.seed);
   (*new_op->mutable_attr())["seed2"].set_i(src_op.seed2);
 }
diff --git a/tensorflow/lite/toco/toco.cc b/tensorflow/lite/toco/toco.cc
index 9740015850..4a3d6a5848 100644
--- a/tensorflow/lite/toco/toco.cc
+++ b/tensorflow/lite/toco/toco.cc
@@ -16,87 +16,9 @@ limitations under the License.
 #include <memory>
 #include <string>
 
-#include "absl/strings/string_view.h"
-#include "tensorflow/lite/toco/model.h"
 #include "tensorflow/lite/toco/model_cmdline_flags.h"
-#include "tensorflow/lite/toco/model_flags.pb.h"
 #include "tensorflow/lite/toco/toco_cmdline_flags.h"
-#include "tensorflow/lite/toco/toco_flags.pb.h"
-#include "tensorflow/lite/toco/toco_port.h"
-#include "tensorflow/lite/toco/toco_tooling.h"
-#include "tensorflow/lite/toco/toco_types.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace toco {
-namespace {
-
-// Checks the permissions of the output file to ensure it is writeable.
-void CheckOutputFilePermissions(const Arg<string>& output_file) {
-  QCHECK(output_file.specified()) << "Missing required flag --output_file.\n";
-  QCHECK(port::file::Writable(output_file.value()).ok())
-      << "Specified output_file is not writable: " << output_file.value()
-      << ".\n";
-}
-
-// Checks the permissions of the frozen model file.
-void CheckFrozenModelPermissions(const Arg<string>& input_file) {
-  QCHECK(input_file.specified()) << "Missing required flag --input_file.\n";
-  QCHECK(port::file::Exists(input_file.value(), port::file::Defaults()).ok())
-      << "Specified input_file does not exist: " << input_file.value() << ".\n";
-  QCHECK(port::file::Readable(input_file.value(), port::file::Defaults()).ok())
-      << "Specified input_file exists, but is not readable: "
-      << input_file.value() << ".\n";
-}
-
-// Reads the contents of the GraphDef from either the frozen graph file or the
-// SavedModel directory. If it reads the SavedModel directory, it updates the
-// ModelFlags and TocoFlags accordingly.
-void ReadInputData(const ParsedTocoFlags& parsed_toco_flags,
-                   const ParsedModelFlags& parsed_model_flags,
-                   TocoFlags* toco_flags, ModelFlags* model_flags,
-                   string* graph_def_contents) {
-  port::CheckInitGoogleIsDone("InitGoogle is not done yet.\n");
-
-  // Ensure savedmodel_directory is not set.
-  QCHECK(!parsed_toco_flags.savedmodel_directory.specified())
-      << "Use `tensorflow/lite/python/tflite_convert` script with "
-      << "SavedModel directories.\n";
-
-  // Checks the input file permissions and reads the contents.
-  CheckFrozenModelPermissions(parsed_toco_flags.input_file);
-  CHECK(port::file::GetContents(parsed_toco_flags.input_file.value(),
-                                graph_def_contents, port::file::Defaults())
-            .ok());
-}
-
-tensorflow::Status ToolMain(const ParsedTocoFlags& parsed_toco_flags,
-                            const ParsedModelFlags& parsed_model_flags) {
-  ModelFlags model_flags;
-  ReadModelFlagsFromCommandLineFlags(parsed_model_flags, &model_flags);
-
-  TocoFlags toco_flags;
-  ReadTocoFlagsFromCommandLineFlags(parsed_toco_flags, &toco_flags);
-
-  string graph_def_contents;
-  ReadInputData(parsed_toco_flags, parsed_model_flags, &toco_flags,
-                &model_flags, &graph_def_contents);
-  CheckOutputFilePermissions(parsed_toco_flags.output_file);
-
-  std::unique_ptr<Model> model =
-      Import(toco_flags, model_flags, graph_def_contents);
-  Transform(toco_flags, model.get());
-  string output_file_contents;
-  TF_RETURN_IF_ERROR(Export(toco_flags, *model, toco_flags.allow_custom_ops(),
-                            &output_file_contents));
-  TF_RETURN_IF_ERROR(
-      port::file::SetContents(parsed_toco_flags.output_file.value(),
-                              output_file_contents, port::file::Defaults()));
-  return tensorflow::Status();
-}
-
-}  // namespace
-}  // namespace toco
+#include "tensorflow/lite/toco/toco_convert.h"
 
 int main(int argc, char** argv) {
   toco::string msg;
@@ -126,6 +48,6 @@ int main(int argc, char** argv) {
     return 1;
   }
   toco::port::InitGoogle(argv[0], effective_argc, &effective_argv, true);
-  auto status = toco::ToolMain(parsed_toco_flags, parsed_model_flags);
+  auto status = toco::Convert(parsed_toco_flags, parsed_model_flags);
   return status.ok() ? 0 : -1;
 }
diff --git a/tensorflow/lite/toco/toco_convert.cc b/tensorflow/lite/toco/toco_convert.cc
new file mode 100644
index 0000000000..28e7b10ecd
--- /dev/null
+++ b/tensorflow/lite/toco/toco_convert.cc
@@ -0,0 +1,108 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstdio>
+#include <memory>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/model_cmdline_flags.h"
+#include "tensorflow/lite/toco/model_flags.pb.h"
+#include "tensorflow/lite/toco/toco_cmdline_flags.h"
+#include "tensorflow/lite/toco/toco_flags.pb.h"
+#include "tensorflow/lite/toco/toco_port.h"
+#include "tensorflow/lite/toco/toco_tooling.h"
+#include "tensorflow/lite/toco/toco_types.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+namespace {
+
+// Checks the permissions of the output file to ensure it is writeable.
+void CheckOutputFilePermissions(const Arg<string>& output_file) {
+  QCHECK(output_file.specified()) << "Missing required flag --output_file.\n";
+  QCHECK(port::file::Writable(output_file.value()).ok())
+      << "Specified output_file is not writable: " << output_file.value()
+      << ".\n";
+}
+
+// Checks the permissions of the frozen model file.
+void CheckFrozenModelPermissions(const Arg<string>& input_file) {
+  QCHECK(input_file.specified()) << "Missing required flag --input_file.\n";
+  QCHECK(port::file::Exists(input_file.value(), port::file::Defaults()).ok())
+      << "Specified input_file does not exist: " << input_file.value() << ".\n";
+  QCHECK(port::file::Readable(input_file.value(), port::file::Defaults()).ok())
+      << "Specified input_file exists, but is not readable: "
+      << input_file.value() << ".\n";
+}
+
+// Reads the contents of the GraphDef from either the frozen graph file or the
+// SavedModel directory. If it reads the SavedModel directory, it updates the
+// ModelFlags and TocoFlags accordingly.
+void ReadInputData(const ParsedTocoFlags& parsed_toco_flags,
+                   const ParsedModelFlags& parsed_model_flags,
+                   TocoFlags* toco_flags, ModelFlags* model_flags,
+                   string* graph_def_contents) {
+  port::CheckInitGoogleIsDone("InitGoogle is not done yet.\n");
+
+  // Ensure savedmodel_directory is not set.
+  QCHECK(!parsed_toco_flags.savedmodel_directory.specified())
+      << "Use `tensorflow/lite/python/tflite_convert` script with "
+      << "SavedModel directories.\n";
+
+  // Checks the input file permissions and reads the contents.
+  CheckFrozenModelPermissions(parsed_toco_flags.input_file);
+  CHECK(port::file::GetContents(parsed_toco_flags.input_file.value(),
+                                graph_def_contents, port::file::Defaults())
+            .ok());
+}
+}  // namespace
+
+tensorflow::Status Convert(const string& graph_def_contents,
+                           const TocoFlags& toco_flags,
+                           const ModelFlags& model_flags,
+                           string* output_file_contents) {
+  std::unique_ptr<Model> model =
+      Import(toco_flags, model_flags, graph_def_contents);
+  Transform(toco_flags, model.get());
+  return Export(toco_flags, *model, toco_flags.allow_custom_ops(),
+                output_file_contents);
+}
+
+tensorflow::Status Convert(const ParsedTocoFlags& parsed_toco_flags,
+                           const ParsedModelFlags& parsed_model_flags) {
+  ModelFlags model_flags;
+  ReadModelFlagsFromCommandLineFlags(parsed_model_flags, &model_flags);
+
+  TocoFlags toco_flags;
+  ReadTocoFlagsFromCommandLineFlags(parsed_toco_flags, &toco_flags);
+
+  string graph_def_contents;
+  ReadInputData(parsed_toco_flags, parsed_model_flags, &toco_flags,
+                &model_flags, &graph_def_contents);
+  CheckOutputFilePermissions(parsed_toco_flags.output_file);
+
+  string output_file_contents;
+  TF_RETURN_IF_ERROR(Convert(graph_def_contents, toco_flags, model_flags,
+                             &output_file_contents));
+
+  TF_RETURN_IF_ERROR(
+      port::file::SetContents(parsed_toco_flags.output_file.value(),
+                              output_file_contents, port::file::Defaults()));
+  return tensorflow::Status();
+}
+
+}  // namespace toco
diff --git a/tensorflow/lite/toco/toco_convert.h b/tensorflow/lite/toco/toco_convert.h
new file mode 100644
index 0000000000..ebbd336d3f
--- /dev/null
+++ b/tensorflow/lite/toco/toco_convert.h
@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOCO_TOCO_CONVERT_H_
+#define TENSORFLOW_LITE_TOCO_TOCO_CONVERT_H_
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/toco/args.h"
+#include "tensorflow/lite/toco/model_flags.pb.h"
+#include "tensorflow/lite/toco/toco_flags.pb.h"
+
+namespace toco {
+
+tensorflow::Status Convert(const string& graph_def_contents,
+                           const TocoFlags& toco_flags,
+                           const ModelFlags& model_flags,
+                           string* output_file_contents);
+
+tensorflow::Status Convert(const ParsedTocoFlags& parsed_toco_flags,
+                           const ParsedModelFlags& parsed_model_flags);
+}  // namespace toco
+
+#endif  // TENSORFLOW_LITE_TOCO_TOCO_CONVERT_H_
diff --git a/tensorflow/lite/toco/toco_convert_test.cc b/tensorflow/lite/toco/toco_convert_test.cc
new file mode 100644
index 0000000000..c3c440db94
--- /dev/null
+++ b/tensorflow/lite/toco/toco_convert_test.cc
@@ -0,0 +1,173 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/toco/toco_convert.h"
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace toco {
+namespace {
+
+TEST(TocoTest, MissingInputFile) {
+  ParsedTocoFlags toco_flags;
+  ParsedModelFlags model_flags;
+  EXPECT_DEATH(Convert(toco_flags, model_flags).ok(),
+               "Missing required flag --input_file");
+}
+
+TEST(TocoTest, BadInputFormat) {
+  TocoFlags toco_flags;
+  ModelFlags model_flags;
+
+  string input;
+  string output;
+
+  EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(),
+               "Unhandled input_format='FILE_FORMAT_UNKNOWN'");
+}
+
+TEST(TocoTest, MissingOuputArrays) {
+  TocoFlags toco_flags;
+  ModelFlags model_flags;
+
+  toco_flags.set_input_format(TENSORFLOW_GRAPHDEF);
+  string input;
+  string output;
+
+  EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(),
+               "This model does not define output arrays, so a --output_arrays "
+               "flag must be given on the command-line");
+}
+
+TEST(TocoTest, BadOutputArray) {
+  TocoFlags toco_flags;
+  ModelFlags model_flags;
+
+  toco_flags.set_input_format(TENSORFLOW_GRAPHDEF);
+  model_flags.add_output_arrays("output1");
+  string input;
+  string output;
+
+  EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(),
+               "Specified output array .output1. is not produced by any op "
+               "in this graph. Is it a typo. To silence this message, pass "
+               "this flag:  allow_nonexistent_arrays");
+}
+
+TEST(TocoTest, BadOutputFormat) {
+  TocoFlags toco_flags;
+  ModelFlags model_flags;
+
+  toco_flags.set_input_format(TENSORFLOW_GRAPHDEF);
+  model_flags.add_output_arrays("output1");
+  string input = R"GraphDef(
+    node {
+      name: "output1"
+      input: "input1"
+      input: "input2"
+      op: "Sub"
+      attr { key: "T" value { type: DT_FLOAT } }
+    }
+  )GraphDef";
+
+  string output;
+
+  EXPECT_DEATH(Convert(input, toco_flags, model_flags, &output).ok(),
+               "Unhandled output_format='FILE_FORMAT_UNKNOWN'");
+}
+
+TEST(TocoTest, SimpleFloatModel) {
+  TocoFlags toco_flags;
+  ModelFlags model_flags;
+
+  toco_flags.set_input_format(TENSORFLOW_GRAPHDEF);
+  toco_flags.set_output_format(TENSORFLOW_GRAPHDEF);
+
+  // Inputs are automatically selected (but that might not be a good idea).
+  model_flags.add_output_arrays("output1");
+  string input = R"GraphDef(
+    node {
+      name: "input1"
+      op: "Placeholder"
+      attr { key: "dtype" value { type: DT_INT64 } }
+    }
+    node {
+      name: "input2"
+      op: "Placeholder"
+      attr { key: "dtype" value { type: DT_INT64 } }
+    }
+    node {
+      name: "output1"
+      input: "input1"
+      input: "input2"
+      op: "Sub"
+      attr { key: "T" value { type: DT_FLOAT } }
+    }
+  )GraphDef";
+
+  string output;
+  EXPECT_TRUE(Convert(input, toco_flags, model_flags, &output).ok());
+  EXPECT_TRUE(!output.empty());
+}
+
+TEST(TocoTest, TransientStringTensors) {
+  TocoFlags toco_flags;
+  ModelFlags model_flags;
+
+  toco_flags.set_input_format(TENSORFLOW_GRAPHDEF);
+
+  // We need to do a couple of things to trigger the transient array
+  // initialization code: output format must support memory planning, and the
+  // input array must have a shape.
+  toco_flags.set_output_format(TFLITE);
+
+  model_flags.add_output_arrays("output1");
+  string input = R"GraphDef(
+    node {
+      name: "input1"
+      op: "Placeholder"
+      attr { key: "dtype" value { type: DT_STRING } }
+      attr { key: "shape" value { shape { dim { size:1 }}}}
+    }
+    node {
+      name: "indices1"
+      op: "Placeholder"
+      attr { key: "dtype" value { type: DT_INT64 } }
+    }
+    node {
+      name: "intermediate1"
+      op: "Gather"
+      input: "input1"
+      input: "indices1"
+      attr { key: "Tparams" value { type: DT_STRING } }
+      attr { key: "Tindices" value { type: DT_INT64 } }
+    }
+    node {
+      name: "output1"
+      op: "Gather"
+      input: "intermediate1"
+      input: "indices2"
+      attr { key: "Tparams" value { type: DT_STRING } }
+      attr { key: "Tindices" value { type: DT_INT64 } }
+    }
+  )GraphDef";
+
+  string output;
+
+  EXPECT_TRUE(Convert(input, toco_flags, model_flags, &output).ok());
+  EXPECT_TRUE(!output.empty());
+}
+
+}  // namespace
+}  // namespace toco
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 5f96e833fb..d8b111d037 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -210,7 +210,8 @@ std::unique_ptr<Model> Import(const TocoFlags& toco_flags,
       CheckInvariants(*model);
       break;
     default:
-      LOG(FATAL) << "Unhandled input_format";
+      LOG(FATAL) << "Unhandled input_format='"
+                 << FileFormat_Name(toco_flags.input_format()) << "'";
   }
 
   LogDump(kLogLevelModelChanged, "AT IMPORT", *model);
@@ -424,7 +425,8 @@ tensorflow::Status Export(const TocoFlags& toco_flags, const Model& model,
       DumpGraphviz(model, output_file_contents);
       break;
     default:
-      LOG(FATAL) << "Unhandled output_format";
+      LOG(FATAL) << "Unhandled output_format='"
+                 << FileFormat_Name(toco_flags.output_format()) << "'";
   }
   return tensorflow::Status();
 }
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index cff387782f..084169548e 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -1770,6 +1770,14 @@ bool IsAllocatableTransientArray(const Model& model, const string& array_name) {
   if (!array->has_shape()) {
     return false;
   }
+
+  // The size of string tensors is rarely known ahead of time, so all transient
+  // tensors of this type will need to be dynamically allocated.
+  if (array->final_data_type == ArrayDataType::kString ||
+      array->data_type == ArrayDataType::kString) {
+    return false;
+  }
+
   return true;
 }
 
-- 
GitLab


From ca4a34eb9e8333f216eccb0adff0e6e9186b9510 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Nov 2018 13:39:02 -0800
Subject: [PATCH 0610/1554] [TF:XLA] Bump open source llvm revision to r347298

PiperOrigin-RevId: 222299016
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 14a5d0a25c..101d0e4c7a 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "286465fc41ade5c1c44e4a6dce9681106664fcdd12264dc9be63fc22bbee3c9c",
-        strip_prefix = "llvm-0478924a3727c74fd482d07eed45a8347540576e",
+        sha256 = "7b4f705c532ee2aafb6e8b9013ad22ec8bb1823a153cd2d6ddb6b7faef818874",
+        strip_prefix = "llvm-9ad322c7dfd4385be9a515d734f70700f192ebae",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0478924a3727c74fd482d07eed45a8347540576e.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/0478924a3727c74fd482d07eed45a8347540576e.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9ad322c7dfd4385be9a515d734f70700f192ebae.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/9ad322c7dfd4385be9a515d734f70700f192ebae.tar.gz",
         ],
     )
 
-- 
GitLab


From 1e904bd524771c851421860de39091976ef8616e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 13:41:02 -0800
Subject: [PATCH 0611/1554] This CL allows sparse losses to work in Keras even
 if they take a different dtype for the labels than the model outputs produce.

PiperOrigin-RevId: 222299347
---
 tensorflow/python/keras/engine/training.py    |  6 +++-
 .../keras/engine/training_dataset_test.py     | 34 ++++++++++++-------
 tensorflow/python/keras/losses.py             |  7 ++++
 3 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index ce01777b2d..56f069c057 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -693,11 +693,15 @@ class Model(Network):
             target = None
           if target is None or K.is_placeholder(target):
             if target is None:
+              target_dtype = losses.LABEL_DTYPES_FOR_LOSSES.get(
+                  self.loss_functions[i],
+                  K.dtype(self.outputs[i]))
+
               target = K.placeholder(
                   ndim=len(shape),
                   name=name + '_target',
                   sparse=K.is_sparse(self.outputs[i]),
-                  dtype=K.dtype(self.outputs[i]))
+                  dtype=target_dtype)
             self._feed_targets.append(target)
             self._feed_outputs.append(self.outputs[i])
             self._feed_output_names.append(name)
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 2e6bec6c62..e79e5842a1 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
+from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
@@ -146,7 +147,7 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
           'dataset iterator ran out of data')
 
 
-class TestTrainingWithDataset(test.TestCase):
+class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_calling_model_on_same_dataset(self):
@@ -249,20 +250,29 @@ class TestTrainingWithDataset(test.TestCase):
     model.evaluate(dataset, steps=2, verbose=1)
     model.predict(dataset, steps=2)
 
+  @parameterized.parameters(
+      {'model': 'functional'},
+      {'model': 'subclass'},
+  )
   @tf_test_util.run_in_graph_and_eager_modes
-  def test_dataset_with_sparse_labels(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'sparse_categorical_crossentropy'
-    model.compile(optimizer, loss)
+  def test_dataset_with_sparse_labels(self, model):
+    if model == 'functional':
+      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    elif model == 'subclass':
+      model = testing_utils.get_small_sequential_mlp(1, 4)
 
-    inputs = np.zeros((10, 3))
-    targets = np.random.randint(0, 4, size=10, dtype=np.int32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
+    for loss in ['sparse_categorical_crossentropy',
+                 losses_impl.sparse_softmax_cross_entropy]:
+      optimizer = RMSPropOptimizer(learning_rate=0.001)
+      model.compile(optimizer, loss)
 
-    model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+      inputs = np.zeros((10, 3), dtype=np.float32)
+      targets = np.random.randint(0, 4, size=10, dtype=np.int32)
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
+
+      model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
 
   def test_dataset_input_shape_validation(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 9f548bfe04..f871ee409e 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -26,6 +26,7 @@ from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
+from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -197,3 +198,9 @@ def get(identifier):
   else:
     raise ValueError('Could not interpret '
                      'loss function identifier:', identifier)
+
+
+LABEL_DTYPES_FOR_LOSSES = {
+    losses_impl.sparse_softmax_cross_entropy: 'int32',
+    sparse_categorical_crossentropy: 'int32'
+}
-- 
GitLab


From 813449805cf8cba7f3fbff7c7b77fa2f1f0ee2bb Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Nov 2018 13:49:58 -0800
Subject: [PATCH 0612/1554] Add uint64 support to Merge and Switch.

XLA's auto-clustering sometimes creates uint64 Merge ops.

For symmetry I also added support for uint64 Switches even though I don't
strictly need them.

PiperOrigin-RevId: 222300834
---
 tensorflow/core/kernels/control_flow_ops.cc               | 3 +++
 .../python/kernel_tests/control_flow_ops_py_test.py       | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/core/kernels/control_flow_ops.cc b/tensorflow/core/kernels/control_flow_ops.cc
index 382c9d5e50..1587eb5114 100644
--- a/tensorflow/core/kernels/control_flow_ops.cc
+++ b/tensorflow/core/kernels/control_flow_ops.cc
@@ -71,11 +71,13 @@ TF_CALL_ALL_TYPES(REGISTER_CPU_SWITCH);
 TF_CALL_ALL_TYPES(REGISTER_CPU_REF_SWITCH);
 TF_CALL_QUANTIZED_TYPES(REGISTER_CPU_SWITCH);
 TF_CALL_QUANTIZED_TYPES(REGISTER_CPU_REF_SWITCH);
+REGISTER_CPU_SWITCH(uint64);
 
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_SWITCH);
 TF_CALL_QUANTIZED_TYPES(REGISTER_GPU_SWITCH);
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_REF_SWITCH);
 TF_CALL_QUANTIZED_TYPES(REGISTER_GPU_REF_SWITCH);
+REGISTER_GPU_SWITCH(uint64);
 
 #undef REGISTER_CPU_SWITCH
 #undef REGISTER_CPU_REF_SWITCH
@@ -263,6 +265,7 @@ TF_CALL_QUANTIZED_TYPES(REGISTER_GPU_KERNEL);
 TF_CALL_QUANTIZED_TYPES(REGISTER_GPU_REF_KERNEL);
 REGISTER_GPU_KERNEL(bool);
 REGISTER_GPU_REF_KERNEL(bool);
+REGISTER_GPU_KERNEL(uint64);
 
 #undef REGISTER_GPU_KERNEL
 #undef REGISTER_GPU_REF_KERNEL
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 3394df20d8..3b8f917282 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -3360,6 +3360,14 @@ class ControlFlowTest(test.TestCase):
       result = control_flow_ops.ref_merge([v_f, v_t])
       sess.run(result)
 
+  def testUInt64SwitchMerge(self):
+    with self.cached_session(force_gpu=test.is_gpu_available()) as sess:
+      constant_uint64 = constant_op.constant(np.array([42]), dtypes.uint64)
+      cond = constant_op.constant(True, dtypes.bool)
+      v_f, v_t = control_flow_ops.switch(constant_uint64, cond)
+      result = control_flow_ops.merge([v_f, v_t])
+      sess.run(result)
+
   def testQIntArgAndRet(self):
 
     @function.Defun(dtypes.qint8)
-- 
GitLab


From 7ee43ac030645fc3aa41fc61047e3164c53c99b1 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 11:52:25 -0800
Subject: [PATCH 0613/1554] TFTRT: fix wording in logs

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index d688ebd7b4..1e0ea5f7e2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -970,8 +970,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                 &graph, alloc.get(), &engine_nodes);
     // If status is ok, we successfully added the node to the graph and can
     // remove segment ops. Otherwise graph is not modified.
-    string msg = StrCat("Added TensorRT node ", engine.engine_name, " for segment ",
-                        i, " consisting ",
+    string msg = StrCat("TensorRT node ", engine.engine_name, " added for segment ",
+                        i, " consisting of ",
                         converted_segments.at(i).first.size(), " nodes");
     if (status.ok()) {
       LOG(INFO) << msg << " succeeded.";
-- 
GitLab


From f32501accfbbc5e76f182a20df93e4154737a11e Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 13:10:12 -0800
Subject: [PATCH 0614/1554] TFTRT: rename my_trt_op_ to TRTEngineOp_ in tests

---
 .../tensorrt/python/trt_convert_test.py       |  6 ++--
 tensorflow/contrib/tensorrt/test/base_test.py | 32 +++++++++----------
 .../tensorrt/test/batch_matmul_test.py        |  6 ++--
 .../tensorrt/test/biasadd_matmul_test.py      |  2 +-
 .../binary_tensor_weight_broadcast_test.py    |  2 +-
 .../tensorrt/test/concatenation_test.py       |  2 +-
 .../tensorrt/test/const_broadcast_test.py     |  2 +-
 .../tensorrt/test/memory_alignment_test.py    |  2 +-
 .../multi_connection_neighbor_engine_test.py  |  2 +-
 .../tensorrt/test/neighboring_engine_test.py  |  4 +--
 .../contrib/tensorrt/test/rank_two_test.py    |  4 +--
 .../tensorrt/test/reshape_transpose_test.py   |  6 ++--
 .../test/tf_trt_integration_test_base.py      |  6 ++--
 .../contrib/tensorrt/test/unary_test.py       |  4 +--
 .../tensorrt/test/vgg_block_nchw_test.py      |  2 +-
 .../contrib/tensorrt/test/vgg_block_test.py   |  2 +-
 16 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index aa82f4207f..7f1045c199 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -162,7 +162,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
       node_name_to_op = {node.name: node.op for node in graph_def.node}
       self.assertEqual({
           "input": "Placeholder",
-          "my_trt_op_0": "TRTEngineOp",
+          "TRTEngineOp_0": "TRTEngineOp",
           "output": "Identity"
       }, node_name_to_op)
 
@@ -189,10 +189,10 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     execute_engine_test_value = ("done" if expect_engine_is_run else "")
     execute_native_segment_test_value = ("" if expect_engine_is_run else "done")
     self.assertEqual(execute_engine_test_value,
-                     trt_convert.get_test_value("my_trt_op_0:ExecuteTrtEngine"))
+                     trt_convert.get_test_value("TRTEngineOp_0:ExecuteTrtEngine"))
     self.assertEqual(
         execute_native_segment_test_value,
-        trt_convert.get_test_value("my_trt_op_0:ExecuteNativeSegment"))
+        trt_convert.get_test_value("TRTEngineOp_0:ExecuteNativeSegment"))
 
   def testCreateInferenceGraph_MinimumSegmentSize(self):
     if not trt_convert.is_tensorrt_enabled():
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 18096e0ff1..5caa61c6fe 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -75,9 +75,9 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     # TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
     # breaks the connection check, fix it.
-    # - my_trt_op_0 should have ["weights", "conv", "bias", "bias_add",
+    # - TRTEngineOp_0 should have ["weights", "conv", "bias", "bias_add",
     #   "relu", "identity", "max_pool"]
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
@@ -131,10 +131,10 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     # TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
     # breaks the connection check, fix it.
-    # - my_trt_op_0 should have ["mul", "sub", "div1", "mul1", "add1",
+    # - TRTEngineOp_0 should have ["mul", "sub", "div1", "mul1", "add1",
     #   "add", "sub1"];
-    # - my_trt_op_1 should have ["weights","conv", "div"]
-    return ["my_trt_op_0", "my_trt_op_1"]
+    # - TRTEngineOp_1 should have ["weights","conv", "div"]
+    return ["TRTEngineOp_0", "TRTEngineOp_1"]
 
   def ShouldRunTest(self, run_params):
     # TODO(aaroey): LayoutOptimizer adds Transpose(Const, Const) to the graph
@@ -153,7 +153,7 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     """Setup method."""
     super(PartiallyConvertedTestA, self).setUp()
     # Let it fail to build the second engine.
-    trt_convert.add_test_value("my_trt_op_1:CreateTRTNode", "fail")
+    trt_convert.add_test_value("TRTEngineOp_1:CreateTRTNode", "fail")
 
   def GetParams(self):
     """Create a graph containing two segment."""
@@ -190,7 +190,7 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return {
         # Only the first engine is built.
-        "my_trt_op_0": ["c0", "c1", "add0", "add1", "mul0", "mul1"]
+        "TRTEngineOp_0": ["c0", "c1", "add0", "add1", "mul0", "mul1"]
     }
 
   def ShouldRunTest(self, run_params):
@@ -207,13 +207,13 @@ class PartiallyConvertedTestB(PartiallyConvertedTestA):
     super(PartiallyConvertedTestB, self).setUp()
     # Let it fail to build the first engine.
     trt_convert.clear_test_values("")
-    trt_convert.add_test_value("my_trt_op_0:CreateTRTNode", "fail")
+    trt_convert.add_test_value("TRTEngineOp_0:CreateTRTNode", "fail")
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
         # Only the second engine is built.
-        "my_trt_op_1": ["c2", "c3", "add2", "add3", "mul2", "mul3"]
+        "TRTEngineOp_1": ["c2", "c3", "add2", "add3", "mul2", "mul3"]
     }
 
 
@@ -257,8 +257,8 @@ class ConstInputTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": ["add", "add1", "mul"],
-        "my_trt_op_1": ["add2", "add3", "mul1"]
+        "TRTEngineOp_0": ["add", "add1", "mul"],
+        "TRTEngineOp_1": ["add2", "add3", "mul1"]
     }
 
 
@@ -289,7 +289,7 @@ class ConstDataInputSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return {"my_trt_op_0": ["c", "add", "add1", "mul"]}
+    return {"TRTEngineOp_0": ["c", "add", "add1", "mul"]}
 
 
 class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase):
@@ -324,12 +324,12 @@ class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": ["add2", "add3", "mul1"],
+        "TRTEngineOp_0": ["add2", "add3", "mul1"],
         # Why segment ["add", "add1", "mul"] was assigned segment id 1
         # instead of 0: the parent node of this segment is actually const
         # node 'c', but it's removed later since it's const output of the
         # segment which is not allowed.
-        "my_trt_op_1": ["add", "add1", "mul"]
+        "TRTEngineOp_1": ["add", "add1", "mul"]
     }
 
 
@@ -373,8 +373,8 @@ class ControlDependencyTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": ["c1", "add", "add1", "mul"],
-        "my_trt_op_1": ["c2", "add2", "add3", "mul1"]
+        "TRTEngineOp_0": ["c1", "add", "add1", "mul"],
+        "TRTEngineOp_1": ["c2", "add2", "add3", "mul1"]
     }
 
 
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 4b88808178..f42308ecb7 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -79,12 +79,12 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     if (run_params.dynamic_engine and
         not trt_test.IsQuantizationMode(run_params.precision_mode)):
-      return ["my_trt_op_0", "my_trt_op_1"]
-    return ["my_trt_op_1"]
+      return ["TRTEngineOp_0", "TRTEngineOp_1"]
+    return ["TRTEngineOp_1"]
 
   def ExpectedEnginesToRun(self, run_params):
     """Return the expected engines to run."""
-    return ["my_trt_op_1"]
+    return ["TRTEngineOp_1"]
 
   def ShouldRunTest(self, run_params):
     """Whether to run the test."""
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 6546ef6477..053b38ff1c 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -128,7 +128,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
   def ShouldRunTest(self, run_params):
     """Whether to run the test."""
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index b53cb3c091..e8b5a8ff62 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -70,7 +70,7 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_%d" % i for i in range(16)]
+    return ["TRTEngineOp_%d" % i for i in range(16)]
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index 465cb02296..c3576f81d9 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -79,7 +79,7 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index e32f047866..c1c883312d 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -64,7 +64,7 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ['my_trt_op_0']
+    return ['TRTEngineOp_0']
 
   def ExpectedAbsoluteTolerance(self, run_params):
     """The absolute tolerance to compare floating point results."""
diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
index bc7c90081f..104bac43a0 100644
--- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
+++ b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
@@ -68,7 +68,7 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
   def ExpectedAbsoluteTolerance(self, run_params):
     """The absolute tolerance to compare floating point results."""
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 11be4feaf7..f09c7b70bc 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -83,7 +83,7 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0", "my_trt_op_1"]
+    return ["TRTEngineOp_0", "TRTEngineOp_1"]
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index eddeafa38b..3e1e4b088b 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -66,8 +66,8 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": ["bias", "mul", "sub"],
-        "my_trt_op_1": ["weights", "conv"]
+        "TRTEngineOp_0": ["bias", "mul", "sub"],
+        "TRTEngineOp_1": ["weights", "conv"]
     }
 
 
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index 74a4a05925..0cd733dca1 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -68,11 +68,11 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": [
+        "TRTEngineOp_0": [
             "add0_1", "add0_2", "add0_3", "c0_1", "c0_2", "c0_3", "abs0_1",
             "abs0_2"
         ],
-        "my_trt_op_1": [
+        "TRTEngineOp_1": [
             "add", "add1_1", "add1_2", "add1_3", "c1_1", "c1_2", "c1_3",
             "abs1_1", "abs1_2", "reciprocal0", "reciprocal1"
         ],
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
index bbc724ab18..419e52b768 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
@@ -79,7 +79,7 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": ["reshape-%d" % i for i in range(7)] +
+        "TRTEngineOp_0": ["reshape-%d" % i for i in range(7)] +
                        ["reshape-%d/shape" % i for i in range(7)]
     }
 
@@ -117,7 +117,7 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase):
         # Note: by default Grappler will run the TRT optimizer twice. At the
         # first time it will group the two transpose ops below to same segment
         # then fail the conversion due to the expected batch dimension problem.
-        # At the second time, since the input of bridge op is my_trt_op_0, it
+        # At the second time, since the input of bridge op is TRTEngineOp_0, it
         # will fail to do shape inference which then cause conversion to fail.
         # TODO(laigd): support shape inference, make TRT optimizer run only
         # once, and fix this.
@@ -136,7 +136,7 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return {
-        "my_trt_op_0": [
+        "TRTEngineOp_0": [
             "transpose-1", "transpose-1/perm", "transposeback",
             "transposeback/perm"
         ]
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index c3cff28574..da22c8371b 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -217,9 +217,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _PrepareRun(self, graph_state):
     """Set up necessary testing environment before calling sess.run()."""
     # Clear test values added by TRTEngineOp.
-    trt_convert.clear_test_values("my_trt_op_.*:ExecuteTrtEngine")
-    trt_convert.clear_test_values("my_trt_op_.*:ExecuteCalibration")
-    trt_convert.clear_test_values("my_trt_op_.*:ExecuteNativeSegment")
+    trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteTrtEngine")
+    trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteCalibration")
+    trt_convert.clear_test_values("TRTEngineOp_.*:ExecuteNativeSegment")
 
   def _GetGPUOptions(self):
     gpu_options = config_pb2.GPUOptions()
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index 8736bfb644..9fc50e0595 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -107,8 +107,8 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return [
-        "my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3",
-        "my_trt_op_4"
+        "TRTEngineOp_0", "TRTEngineOp_1", "TRTEngineOp_2", "TRTEngineOp_3",
+        "TRTEngineOp_4"
     ]
 
 
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index b0271a04b3..b29626d2c2 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -76,7 +76,7 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index d7c165784b..9b0b189626 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -67,7 +67,7 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 if __name__ == "__main__":
-- 
GitLab


From d4127586167c47679cee38c61bc44b7055d90be8 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Tue, 20 Nov 2018 14:07:16 -0800
Subject: [PATCH 0615/1554] Fix vlog of non-functions (e.g. callable objects).

PiperOrigin-RevId: 222303881
---
 tensorflow/python/eager/function.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 609a340915..bc92a0c974 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1091,7 +1091,7 @@ class PolymorphicFunction(object):
       if graph_function is None:
         logging.vlog(1,
                      "Creating new FuncGraph for Python function %r (key: %r)",
-                     self._python_function.__name__, cache_key)
+                     self._python_function, cache_key)
         if self._input_signature is None:
           arglen = len(args)
         else:
-- 
GitLab


From 95fbc0e4db095ec6e947d1bb20319441cc31e505 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 14:12:53 -0800
Subject: [PATCH 0616/1554] TFTRT: fix alignment

---
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index cff6be9759..8037d47040 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -492,7 +492,7 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
     LOG(INFO) << "Building a new TensorRT engine for " << name()
-            << " with batch size " << batch_size;
+              << " with batch size " << batch_size;
     // Up to this point, calibrator_ can never be empty, since otherwise it
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
@@ -568,7 +568,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
   cres->thr_.reset(new std::thread([cres, label, segment_graph, shapes,
                                     platform_gpu_id, workspace_size_bytes]() {
     LOG(INFO) << "Starting calibration thread on device " << platform_gpu_id
-            << ", Calibration Resource @ " << cres;
+              << ", Calibration Resource @ " << cres;
     auto err = cudaSetDevice(platform_gpu_id);
     if (err != cudaSuccess) {
       // TODO(aaroey): should return error here.
-- 
GitLab


From fd46ffb7bd04d0959294a09ae8a9d2cbd74a48ca Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 20 Nov 2018 14:13:20 -0800
Subject: [PATCH 0617/1554] Automated rollback of commit
 a3650434746cba6b78d9846fbde3b7f31f6a33c8

PiperOrigin-RevId: 222304840
---
 tensorflow/python/data/experimental/ops/BUILD |   2 +-
 .../python/data/experimental/ops/grouping.py  |  16 +-
 tensorflow/python/data/kernel_tests/BUILD     |   4 -
 .../data/kernel_tests/dataset_ops_test.py     |  61 ----
 tensorflow/python/data/ops/dataset_ops.py     | 260 +++++++++++-------
 tensorflow/python/data/ops/optional_ops.py    |  12 +-
 tensorflow/python/data/util/structure.py      |  26 +-
 tensorflow/python/data/util/structure_test.py |   9 +-
 8 files changed, 188 insertions(+), 202 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index b6c1376b6a..170fda90b6 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -165,7 +165,7 @@ py_library(
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:structure",
+        "//tensorflow/python/data/util:sparse",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index db10ea3b7f..80ca7104d8 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -21,7 +21,6 @@ import numpy as np
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -449,10 +448,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
 
   def _make_reduce_func(self, reduce_func, input_dataset):
     """Make wrapping defun for reduce_func."""
-    nested_dataset = dataset_ops.DatasetStructure(
-        structure.Structure._from_legacy_structure(  # pylint: disable=protected-access
-            input_dataset.output_types, input_dataset.output_shapes,
-            input_dataset.output_classes))
+    nested_dataset = dataset_ops._NestedDatasetComponent(input_dataset)  # pylint: disable=protected-access
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
         reduce_func,
         self._transformation_name(),
@@ -460,13 +456,11 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
         input_shapes=(tensor_shape.scalar(), nested_dataset),
         input_types=(dtypes.int64, nested_dataset))
     if not isinstance(
-        wrapped_func.output_structure, dataset_ops.DatasetStructure):
+        wrapped_func.output_classes, dataset_ops._NestedDatasetComponent):  # pylint: disable=protected-access
       raise TypeError("`reduce_func` must return a `Dataset` object.")
-    # pylint: disable=protected-access
-    element_structure = wrapped_func.output_structure._element_structure
-    self._output_classes = element_structure._to_legacy_output_classes()
-    self._output_types = element_structure._to_legacy_output_types()
-    self._output_shapes = element_structure._to_legacy_output_shapes()
+    self._output_classes = wrapped_func.output_classes.output_classes
+    self._output_types = wrapped_func.output_types.output_types
+    self._output_shapes = wrapped_func.output_shapes.output_shapes
     self._reduce_func = wrapped_func.function
 
   @property
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index fa1f6d701a..21eed2b070 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -117,12 +117,8 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
         "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:optional_ops",
-        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
index 1f22a37c2e..a5324af4d0 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
@@ -24,14 +24,10 @@ import numpy as np
 from tensorflow.core.framework import graph_pb2
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import optional_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import structure
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.platform import test
 
 
@@ -253,63 +249,6 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertTrue(ds.options().experimental_autotune)
     self.assertTrue(ds.options().experimental_filter_fusion)
 
-  # pylint: disable=g-long-lambda
-  @parameterized.named_parameters(
-      ("Tensor", lambda: constant_op.constant(37.0),
-       structure.TensorStructure(dtypes.float32, [])),
-      ("SparseTensor", lambda: sparse_tensor.SparseTensor(
-          indices=[[0]], values=constant_op.constant([0], dtype=dtypes.int32),
-          dense_shape=[1]),
-       structure.SparseTensorStructure(dtypes.int32, [1])),
-      ("Nest", lambda: {
-          "a": constant_op.constant(37.0),
-          "b": (constant_op.constant(["Foo"]), constant_op.constant("Bar"))},
-       structure.NestedStructure({
-           "a": structure.TensorStructure(dtypes.float32, []),
-           "b": (structure.TensorStructure(dtypes.string, [1]),
-                 structure.TensorStructure(dtypes.string, []))})),
-      ("Dataset", lambda: dataset_ops.Dataset.from_tensor_slices(
-          constant_op.constant([1, 2, 3])),
-       dataset_ops.DatasetStructure(
-           structure.TensorStructure(dtypes.int32, []))),
-      ("Optional", lambda: optional_ops.Optional.from_value(37.0),
-       optional_ops.OptionalStructure(
-           structure.TensorStructure(dtypes.float32, []))),
-  )
-  def testDatasetStructure(self, tf_value_fn, expected_element_structure):
-    dataset = dataset_ops.Dataset.from_tensors(0).map(lambda _: tf_value_fn())
-    dataset_structure = structure.Structure.from_value(dataset)
-    self.assertIsInstance(dataset_structure, dataset_ops.DatasetStructure)
-
-    # TODO(b/110122868): Add a public API to `tf.data.Dataset` for accessing
-    # the element structure.
-    self.assertTrue(expected_element_structure.is_compatible_with(
-        dataset_structure._element_structure))
-    self.assertTrue(dataset_structure._element_structure.is_compatible_with(
-        expected_element_structure))
-
-    self.assertEqual([dtypes.variant], dataset_structure._flat_types)
-    self.assertEqual([tensor_shape.scalar()], dataset_structure._flat_shapes)
-
-    # Assert that the `Dataset` survives a round-trip via _from_tensor_list()
-    # and _to_tensor_list().
-    round_trip_dataset = dataset_structure._from_tensor_list(
-        dataset_structure._to_tensor_list(dataset))
-
-    value = tf_value_fn()
-
-    if isinstance(value, dataset_ops.Dataset):
-      self.assertDatasetsEqual(value, dataset.flat_map(lambda x: x))
-    elif isinstance(value, optional_ops.Optional):
-      self.assertDatasetProduces(
-          round_trip_dataset.map(lambda opt: opt.get_value()),
-          [self.evaluate(value.get_value())],
-          requires_initialization=True)
-    else:
-      self.assertDatasetProduces(
-          round_trip_dataset, [self.evaluate(tf_value_fn())],
-          requires_initialization=True)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 5c0cfe994d..4a11619112 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -31,7 +31,6 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import random_seed
 from tensorflow.python.data.util import sparse
-from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -1869,6 +1868,57 @@ class SparseTensorSliceDataset(DatasetSource):
     return (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64)
 
 
+class _NestedDatasetComponent(object):
+  """The structure of a `Dataset` nested in a component of another `Dataset`.
+
+  A `StructuredFunctionWrapper` around a function that returns a `Dataset` as
+  one of its components will have a `NestedDatasetComponent` in the
+  corresponding position in the `output_classes`, `output_shapes`, and
+  `output_types` properties.
+
+  TODO(b/110122868): Add this class, or something equivalent, to the public API.
+  We are considering revising the public API for accessing Dataset structure
+  (`output_classes` etc.) based on experience with nested datasets and other
+  custom component types.
+  """
+
+  def __init__(self,
+               dataset=None,
+               output_shapes=None,
+               output_types=None,
+               output_classes=None):
+    if dataset is None:
+      if (output_classes is None or output_shapes is None or
+          output_types is None):
+        raise ValueError(
+            "Either `dataset`, or all of `output_classes`, "
+            "`output_shapes`, and `output_types` must be specified.")
+      self._output_classes = output_classes
+      self._output_shapes = output_shapes
+      self._output_types = output_types
+    else:
+      if not (output_classes is None and output_shapes is None and
+              output_types is None):
+        raise ValueError(
+            "Either `dataset`, or all of `output_classes`, "
+            "`output_shapes`, and `output_types` must be specified.")
+      self._output_classes = dataset.output_classes
+      self._output_shapes = dataset.output_shapes
+      self._output_types = dataset.output_types
+
+  @property
+  def output_classes(self):
+    return self._output_classes
+
+  @property
+  def output_shapes(self):
+    return self._output_shapes
+
+  @property
+  def output_types(self):
+    return self._output_types
+
+
 class _VariantDataset(DatasetV2):
   """A Dataset wrapper around a `tf.variant`-typed function argument."""
 
@@ -1885,73 +1935,15 @@ class _VariantDataset(DatasetV2):
 
   @property
   def output_classes(self):
-    return self._structure._to_legacy_output_classes()  # pylint: disable=protected-access
+    return self._structure.output_classes
 
   @property
   def output_shapes(self):
-    return self._structure._to_legacy_output_shapes()  # pylint: disable=protected-access
+    return self._structure.output_shapes
 
   @property
   def output_types(self):
-    return self._structure._to_legacy_output_types()  # pylint: disable=protected-access
-
-
-class DatasetStructure(structure_lib.Structure):
-  """Represents a `Dataset` of structured values."""
-
-  def __init__(self, element_structure):
-    self._element_structure = element_structure
-
-  @property
-  def _flat_shapes(self):
-    return [tensor_shape.scalar()]
-
-  @property
-  def _flat_types(self):
-    return [dtypes.variant]
-
-  def is_compatible_with(self, other):
-    # pylint: disable=protected-access
-    return (isinstance(other, DatasetStructure) and
-            self._element_structure.is_compatible_with(
-                other._element_structure))
-
-  def _to_tensor_list(self, value):
-    return [value._as_variant_tensor()]  # pylint: disable=protected-access
-
-  def _from_tensor_list(self, flat_value):
-    if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
-        not flat_value[0].shape.is_compatible_with(tensor_shape.scalar())):
-      raise ValueError(
-          "DatasetStructure corresponds to a single tf.variant scalar.")
-    return self._from_compatible_tensor_list(flat_value)
-
-  def _from_compatible_tensor_list(self, flat_value):
-    # pylint: disable=protected-access
-    return _VariantDataset(flat_value[0], self._element_structure)
-
-  @staticmethod
-  def from_value(value):
-    # TODO(b/110122868): We can simplify this when a `Dataset` object has a
-    # `Structure`-valued property.
-    element_structure = structure_lib.Structure._from_legacy_structure(
-        value.output_types, value.output_shapes, value.output_classes)
-    return DatasetStructure(element_structure)
-
-  def _to_legacy_output_types(self):
-    return self
-
-  def _to_legacy_output_shapes(self):
-    return self
-
-  def _to_legacy_output_classes(self):
-    return self
-
-
-# pylint: disable=protected-access
-structure_lib.Structure._register_custom_converter(DatasetV2,
-                                                   DatasetStructure.from_value)
-# pylint: enable=protected-access
+    return self._structure.output_types
 
 
 class StructuredFunctionWrapper(object):
@@ -2009,9 +2001,6 @@ class StructuredFunctionWrapper(object):
       self._input_types = dataset.output_types
       self._input_classes = dataset.output_classes
 
-    self._input_structure = structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
-        self._input_types, self._input_shapes, self._input_classes)
-
     self._transformation_name = transformation_name
     readable_transformation_name = transformation_name.replace(
         ".", "_")[:-2] if len(transformation_name) > 2 else ""
@@ -2019,18 +2008,35 @@ class StructuredFunctionWrapper(object):
         readable_transformation_name,
         function_utils.get_func_name(func),
         str(ops.uid())
+
     ])
 
     if defun_kwargs is None:
       defun_kwargs = {}
 
     @function.Defun(
-        *self._input_structure._flat_types, func_name=self._func_name,  # pylint: disable=protected-access
-        **defun_kwargs)
+        *self._defun_args(), func_name=self._func_name, **defun_kwargs)
     def tf_data_structured_function_wrapper(*args):
       """Wrapper for passing nested structures to and from tf.data functions."""
-      # pylint: disable=protected-access
-      nested_args = self._input_structure._from_compatible_tensor_list(args)
+      flat_args = []
+      for arg, arg_class, arg_shape, arg_type in zip(
+          args,
+          nest.flatten(self._input_classes),
+          nest.flatten(self._input_shapes),
+          nest.flatten(self._input_types)):
+        # TODO(b/110122868): Add a registration mechanism for new component
+        # types.
+        if arg_class is sparse_tensor_lib.SparseTensor:
+          arg = sparse.deserialize_sparse_tensors(
+              arg, arg_type, arg_shape, arg_class)
+          arg.indices.set_shape([None, arg_shape.ndims])
+          arg.dense_shape.set_shape([arg_shape.ndims])
+        elif isinstance(arg_class, _NestedDatasetComponent):
+          arg = _VariantDataset(arg, arg_class)
+        else:
+          arg.set_shape(arg_shape)
+        flat_args.append(arg)
+      nested_args = nest.pack_sequence_as(self._input_classes, flat_args)
       if not _should_unpack_args(nested_args):
         nested_args = (nested_args,)
 
@@ -2048,14 +2054,50 @@ class StructuredFunctionWrapper(object):
       if isinstance(ret, list):
         ret = tuple(ret)
 
-      try:
-        self._output_structure = structure_lib.Structure.from_value(ret)
-      except (ValueError, TypeError):
-        raise TypeError("Unsupported return value from function passed to "
-                        "%s: %s." % (transformation_name, ret))
+      # Convert any `SparseTensorValue`s to `SparseTensor`s and all other
+      # values to tensors.
+      flat_ret = []
+      flat_classes = []
+      flat_shapes = []
+      flat_types = []
+      for t in nest.flatten(ret):
+        # TODO(b/110122868): Add a registration mechanism for new component
+        # types.
+        if sparse_tensor_lib.is_sparse(t):
+          t = sparse_tensor_lib.SparseTensor.from_value(t)
+          flat_ret.append(sparse.serialize_sparse_tensors(t))
+          flat_classes.append(sparse_tensor_lib.SparseTensor)
+          flat_shapes.append(t.get_shape())
+          flat_types.append(t.dtype)
+        elif isinstance(t, DatasetV2):
+          flat_ret.append(t._as_variant_tensor())  # pylint: disable=protected-access
+          component = _NestedDatasetComponent(t)
+          flat_classes.append(component)
+          flat_shapes.append(component)
+          flat_types.append(component)
+          if t.options() != Options():
+            warnings.warn("Encountered a nested dataset with non-default "
+                          "options. These options will not be propagated to "
+                          "the outer dataset.")
+        else:
+          try:
+            t = ops.convert_to_tensor(t)
+          except (ValueError, TypeError):
+            raise TypeError("Unsupported return value from function passed to "
+                            "%s: %s." % (transformation_name, t))
+          flat_ret.append(t)
+          flat_classes.append(ops.Tensor)
+          flat_shapes.append(t.get_shape())
+          flat_types.append(t.dtype)
+
+      ret = nest.pack_sequence_as(ret, flat_ret)
+      self._output_classes = nest.pack_sequence_as(ret, flat_classes)
+      self._output_shapes = nest.pack_sequence_as(ret, flat_shapes)
+      self._output_types = nest.pack_sequence_as(ret, flat_types)
 
       _warn_if_collections(transformation_name)
-      return self._output_structure._to_tensor_list(ret)
+
+      return flat_ret
 
     self._function = tf_data_structured_function_wrapper
     if add_to_graph:
@@ -2066,21 +2108,32 @@ class StructuredFunctionWrapper(object):
       # in case (e.g.) we need to rerun the function.
       self._function._create_definition_if_needed()  # pylint: disable=protected-access
 
-  @property
-  def output_structure(self):
-    return self._output_structure
+  def _defun_args(self):
+    """Returns a flat list of `tf.DType` for the input element structure."""
+    ret = []
+    for input_type, input_class in zip(nest.flatten(self._input_types),
+                                       nest.flatten(self._input_classes)):
+      # TODO(b/110122868): Add a registration mechanism for new component types.
+      if input_class is sparse_tensor_lib.SparseTensor:
+        ret.append(dtypes.variant)
+      elif isinstance(input_class, _NestedDatasetComponent):
+        ret.append(dtypes.variant)
+      else:
+        assert isinstance(input_type, dtypes.DType)
+        ret.append(input_type)
+    return ret
 
   @property
   def output_classes(self):
-    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
+    return self._output_classes
 
   @property
   def output_shapes(self):
-    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
+    return self._output_shapes
 
   @property
   def output_types(self):
-    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
+    return self._output_types
 
   @property
   def function(self):
@@ -2103,12 +2156,30 @@ def flat_structure(dataset):
     A dictionary of keyword arguments that can be passed to many Dataset op
     constructors.
   """
-  # pylint: disable=protected-access
-  structure = structure_lib.Structure._from_legacy_structure(
-      dataset.output_types, dataset.output_shapes, dataset.output_classes)
+  output_classes = []
+  output_shapes = []
+  output_types = []
+  for output_class, output_shape, output_type in zip(
+      nest.flatten(dataset.output_classes), nest.flatten(dataset.output_shapes),
+      nest.flatten(dataset.output_types)):
+    if isinstance(output_class, _NestedDatasetComponent):
+      output_classes.append(output_class.output_classes)
+      output_shapes.append(output_shape.output_shapes)
+      output_types.append(output_type.output_types)
+    else:
+      output_classes.append(output_class)
+      output_shapes.append(output_shape)
+      output_types.append(output_type)
+
+  output_classes = nest.pack_sequence_as(dataset.output_classes, output_classes)
+  output_shapes = nest.pack_sequence_as(dataset.output_shapes, output_shapes)
+  output_types = nest.pack_sequence_as(dataset.output_types, output_types)
+
   return {
-      "output_shapes": structure._flat_shapes,
-      "output_types": structure._flat_types,
+      "output_shapes":
+          nest.flatten(sparse.as_dense_shapes(output_shapes, output_classes)),
+      "output_types":
+          nest.flatten(sparse.as_dense_types(output_types, output_classes)),
   }
 
 
@@ -2831,13 +2902,11 @@ class FlatMapDataset(UnaryDataset):
 
     wrapped_func = StructuredFunctionWrapper(
         map_func, self._transformation_name(), dataset=input_dataset)
-    if not isinstance(wrapped_func.output_structure, DatasetStructure):
+    if not isinstance(wrapped_func.output_classes, _NestedDatasetComponent):
       raise TypeError("`map_func` must return a `Dataset` object.")
-    # pylint: disable=protected-access
-    element_structure = wrapped_func.output_structure._element_structure
-    self._output_classes = element_structure._to_legacy_output_classes()
-    self._output_types = element_structure._to_legacy_output_types()
-    self._output_shapes = element_structure._to_legacy_output_shapes()
+    self._output_classes = wrapped_func.output_classes.output_classes
+    self._output_types = wrapped_func.output_types.output_types
+    self._output_shapes = wrapped_func.output_shapes.output_shapes
     self._map_func = wrapped_func.function
 
   def _as_variant_tensor(self):
@@ -2979,9 +3048,10 @@ class WindowDataset(UnaryDataset):
     self._output_classes = nest.pack_sequence_as(
         input_dataset.output_classes,
         [
-            DatasetStructure(
-                structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
-                    output_type, output_shape, output_class))
+            _NestedDatasetComponent(  # pylint: disable=protected-access
+                output_classes=output_class,
+                output_shapes=output_shape,
+                output_types=output_type)
             for output_class, output_shape, output_type in zip(
                 nest.flatten(input_dataset.output_classes),
                 nest.flatten(input_dataset.output_shapes),
diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py
index 4113b7ed31..91cf883ce9 100644
--- a/tensorflow/python/data/ops/optional_ops.py
+++ b/tensorflow/python/data/ops/optional_ops.py
@@ -183,13 +183,19 @@ class OptionalStructure(structure.Structure):
     return OptionalStructure(value.value_structure)
 
   def _to_legacy_output_types(self):
-    return self
+    raise NotImplementedError("The `output_types` property is not supported on "
+                              "structured objects containing an `Optional`. "
+                              "Use the corresponding `structure` property.")
 
   def _to_legacy_output_shapes(self):
-    return self
+    raise NotImplementedError("The `output_shapes` property is not supported on"
+                              " structured objects containing an `Optional`. "
+                              "Use the corresponding `structure` property.")
 
   def _to_legacy_output_classes(self):
-    return self
+    raise NotImplementedError("The `output_classes` property is not supported "
+                              "on structured objects containing an `Optional`. "
+                              "Use the corresponding `structure` property.")
 
 
 # pylint: disable=protected-access
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 3cf67b0745..9a3118297d 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -208,16 +208,14 @@ class Structure(object):
     flat_ret = []
     for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes,
                                                  flat_classes):
-      if isinstance(flat_class, Structure):
-        flat_ret.append(flat_class)
-      elif issubclass(flat_class, sparse_tensor_lib.SparseTensor):
+      if issubclass(flat_class, sparse_tensor_lib.SparseTensor):
         flat_ret.append(SparseTensorStructure(flat_type, flat_shape))
       elif issubclass(flat_class, ops.Tensor):
         flat_ret.append(TensorStructure(flat_type, flat_shape))
       else:
         # NOTE(mrry): Since legacy structures produced by iterators only
-        # comprise Tensors, SparseTensors, and nests, we do not need to
-        # support all structure types here.
+        # comprise Tensors, SparseTensors, and nests, we do not need to support
+        # all structure types here.
         raise TypeError(
             "Could not build a structure for output class %r" % flat_type)
 
@@ -383,13 +381,6 @@ class TensorStructure(Structure):
     return self._from_compatible_tensor_list(flat_value)
 
   def _from_compatible_tensor_list(self, flat_value):
-    # TODO(b/112266545): It would be cleaner to create a new `ensure_shape()`
-    # op here and return that, instead of mutating the input's shape using
-    # `Tensor.set_shape()`. However, that would add extra ops on the arguments
-    # of each `tf.data` function, which could impact performance. When this
-    # bug is resolved, we should be able to add the `ensure_shape()` ops and
-    # optimize them away using contextual shape information.
-    flat_value[0].set_shape(self._shape)
     return flat_value[0]
 
   @staticmethod
@@ -415,11 +406,7 @@ class SparseTensorStructure(Structure):
 
   @property
   def _flat_shapes(self):
-    # NOTE(mrry): The default flat shape of a boxed `SparseTensor` is `(3,)`,
-    # but a `SparseTensorStructure` can also represent a batch of boxed
-    # `SparseTensor` objects with shape `(?, 3)` (and batches of batches, etc.),
-    # so the flat shape must be unknown.
-    return [tensor_shape.unknown_shape(None)]
+    return [tensor_shape.vector(3)]
 
   @property
   def _flat_types(self):
@@ -441,11 +428,8 @@ class SparseTensorStructure(Structure):
     return self._from_compatible_tensor_list(flat_value)
 
   def _from_compatible_tensor_list(self, flat_value):
-    ret = sparse_ops.deserialize_sparse(
+    return sparse_ops.deserialize_sparse(
         flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
-    ret.indices.set_shape([None, self._dense_shape.ndims])
-    ret.dense_shape.set_shape([self._dense_shape.ndims])
-    return ret
 
   @staticmethod
   def from_value(value):
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index 65a41a50f1..630a0c912b 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -44,7 +44,7 @@ class StructureTest(test.TestCase, parameterized.TestCase):
        [dtypes.float32], [[]]),
       (lambda: sparse_tensor.SparseTensor(
           indices=[[3, 4]], values=[-1], dense_shape=[4, 5]),
-       structure.SparseTensorStructure, [dtypes.variant], [None]),
+       structure.SparseTensorStructure, [dtypes.variant], [[3]]),
       (lambda: (constant_op.constant(37.0), constant_op.constant([1, 2, 3])),
        structure.NestedStructure, [dtypes.float32, dtypes.int32], [[], [3]]),
       (lambda: {
@@ -58,17 +58,14 @@ class StructureTest(test.TestCase, parameterized.TestCase):
                 sparse_tensor.SparseTensor(
                     indices=[[3, 4]], values=[-1], dense_shape=[4, 5]))
       }, structure.NestedStructure,
-       [dtypes.float32, dtypes.variant, dtypes.variant], [[], None, None]))
+       [dtypes.float32, dtypes.variant, dtypes.variant], [[], [3], [3]]))
   def testFlatStructure(self, value_fn, expected_structure, expected_types,
                         expected_shapes):
     value = value_fn()
     s = structure.Structure.from_value(value)
     self.assertIsInstance(s, expected_structure)
     self.assertEqual(expected_types, s._flat_types)
-    for expected, actual in zip(expected_shapes, s._flat_shapes):
-      self.assertTrue(actual.is_compatible_with(expected))
-      self.assertTrue(
-          tensor_shape.as_shape(expected).is_compatible_with(actual))
+    self.assertEqual(expected_shapes, s._flat_shapes)
 
   @parameterized.parameters(
       (lambda: constant_op.constant(37.0), lambda: [
-- 
GitLab


From b64c750d37d6d59d853e0f75e89fdae498dffb03 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Tue, 20 Nov 2018 14:24:48 -0800
Subject: [PATCH 0618/1554] Revert buildifier changes and exclude nccl kernels
 for windows builds

---
 tensorflow/core/BUILD | 114 +++++++++++++++++++++---------------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3237c34a99..41d3f6eab8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1349,63 +1349,63 @@ cc_library(
     name = "all_kernels_statically_linked",
     visibility = ["//visibility:private"],
     deps = [
-               "//tensorflow/core/kernels:array",
-               "//tensorflow/core/kernels:audio",
-               "//tensorflow/core/kernels:batch_kernels",
-               "//tensorflow/core/kernels:bincount_op",
-               "//tensorflow/core/kernels:boosted_trees_ops",
-               "//tensorflow/core/kernels:candidate_sampler_ops",
-               "//tensorflow/core/kernels:checkpoint_ops",
-               "//tensorflow/core/kernels:collective_ops",
-               "//tensorflow/core/kernels:control_flow_ops",
-               "//tensorflow/core/kernels:ctc_ops",
-               "//tensorflow/core/kernels:cudnn_rnn_kernels",
-               "//tensorflow/core/kernels:data_flow",
-               "//tensorflow/core/kernels:dataset_ops",
-               "//tensorflow/core/kernels:decode_proto_op",
-               "//tensorflow/core/kernels:encode_proto_op",
-               "//tensorflow/core/kernels:fake_quant_ops",
-               "//tensorflow/core/kernels:function_ops",
-               "//tensorflow/core/kernels:functional_ops",
-               "//tensorflow/core/kernels:grappler",
-               "//tensorflow/core/kernels:histogram_op",
-               "//tensorflow/core/kernels:image",
-               "//tensorflow/core/kernels:io",
-               "//tensorflow/core/kernels:linalg",
-               "//tensorflow/core/kernels:list_kernels",
-               "//tensorflow/core/kernels:lookup",
-               "//tensorflow/core/kernels:logging",
-               "//tensorflow/core/kernels:manip",
-               "//tensorflow/core/kernels:math",
-               "//tensorflow/core/kernels:multinomial_op",
-               "//tensorflow/core/kernels:nn",
-               "//tensorflow/core/kernels:parameterized_truncated_normal_op",
-               "//tensorflow/core/kernels:parsing",
-               "//tensorflow/core/kernels:partitioned_function_ops",
-               "//tensorflow/core/kernels:ragged_ops",
-               "//tensorflow/core/kernels:random_ops",
-               "//tensorflow/core/kernels:random_poisson_op",
-               "//tensorflow/core/kernels:remote_fused_graph_ops",
-               "//tensorflow/core/kernels:required",
-               "//tensorflow/core/kernels:resource_variable_ops",
-               "//tensorflow/core/kernels:rpc_op",
-               "//tensorflow/core/kernels:scoped_allocator_ops",
-               "//tensorflow/core/kernels:sdca_ops",
-               "//tensorflow/core/kernels:searchsorted_op",
-               "//tensorflow/core/kernels:set_kernels",
-               "//tensorflow/core/kernels:sparse",
-               "//tensorflow/core/kernels:state",
-               "//tensorflow/core/kernels:stateless_random_ops",
-               "//tensorflow/core/kernels:string",
-               "//tensorflow/core/kernels:summary_kernels",
-               "//tensorflow/core/kernels:training_ops",
-               "//tensorflow/core/kernels:word2vec_kernels",
-           ] + tf_additional_cloud_kernel_deps() +
-           select({
-               "//tensorflow:no_nccl_support": [],
-               "//tensorflow:with_cuda_support_windows_override": [],
-               "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"],
-           }) + if_not_windows([
+        "//tensorflow/core/kernels:array",
+        "//tensorflow/core/kernels:audio",
+        "//tensorflow/core/kernels:batch_kernels",
+        "//tensorflow/core/kernels:bincount_op",
+        "//tensorflow/core/kernels:boosted_trees_ops",
+        "//tensorflow/core/kernels:candidate_sampler_ops",
+        "//tensorflow/core/kernels:checkpoint_ops",
+        "//tensorflow/core/kernels:collective_ops",
+        "//tensorflow/core/kernels:control_flow_ops",
+        "//tensorflow/core/kernels:ctc_ops",
+        "//tensorflow/core/kernels:cudnn_rnn_kernels",
+        "//tensorflow/core/kernels:data_flow",
+        "//tensorflow/core/kernels:dataset_ops",
+        "//tensorflow/core/kernels:decode_proto_op",
+        "//tensorflow/core/kernels:encode_proto_op",
+        "//tensorflow/core/kernels:fake_quant_ops",
+        "//tensorflow/core/kernels:function_ops",
+        "//tensorflow/core/kernels:functional_ops",
+        "//tensorflow/core/kernels:grappler",
+        "//tensorflow/core/kernels:histogram_op",
+        "//tensorflow/core/kernels:image",
+        "//tensorflow/core/kernels:io",
+        "//tensorflow/core/kernels:linalg",
+        "//tensorflow/core/kernels:list_kernels",
+        "//tensorflow/core/kernels:lookup",
+        "//tensorflow/core/kernels:logging",
+        "//tensorflow/core/kernels:manip",
+        "//tensorflow/core/kernels:math",
+        "//tensorflow/core/kernels:multinomial_op",
+        "//tensorflow/core/kernels:nn",
+        "//tensorflow/core/kernels:parameterized_truncated_normal_op",
+        "//tensorflow/core/kernels:parsing",
+        "//tensorflow/core/kernels:partitioned_function_ops",
+        "//tensorflow/core/kernels:ragged_ops",
+        "//tensorflow/core/kernels:random_ops",
+        "//tensorflow/core/kernels:random_poisson_op",
+        "//tensorflow/core/kernels:remote_fused_graph_ops",
+        "//tensorflow/core/kernels:required",
+        "//tensorflow/core/kernels:resource_variable_ops",
+        "//tensorflow/core/kernels:rpc_op",
+        "//tensorflow/core/kernels:scoped_allocator_ops",
+        "//tensorflow/core/kernels:sdca_ops",
+        "//tensorflow/core/kernels:searchsorted_op",
+        "//tensorflow/core/kernels:set_kernels",
+        "//tensorflow/core/kernels:sparse",
+        "//tensorflow/core/kernels:state",
+        "//tensorflow/core/kernels:stateless_random_ops",
+        "//tensorflow/core/kernels:string",
+        "//tensorflow/core/kernels:summary_kernels",
+        "//tensorflow/core/kernels:training_ops",
+        "//tensorflow/core/kernels:word2vec_kernels",
+    ] + tf_additional_cloud_kernel_deps() +
+    select({
+        "//tensorflow:no_nccl_support": [],
+        "//tensorflow:windows": [],
+        "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"],
+    }) + if_not_windows([
         "//tensorflow/core/kernels:fact_op",
         "//tensorflow/core/kernels:array_not_windows",
         "//tensorflow/core/kernels:math_not_windows",
-- 
GitLab


From baad4e65fc8c4e5313e70bfd9fe0240cd0972842 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Tue, 20 Nov 2018 14:21:31 -0800
Subject: [PATCH 0619/1554] Remove some `num_gpus` and `auto_shard_dataset`
 from CoreMirroredStrategy.

PiperOrigin-RevId: 222306139
---
 .../python/cross_device_ops_test.py           | 16 ++--
 .../python/estimator_training_test.py         |  8 +-
 .../distribute/python/mirrored_strategy.py    | 94 ++++++-------------
 .../python/mirrored_strategy_multigpu_test.py | 32 ++++---
 4 files changed, 62 insertions(+), 88 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index a0f7cf182c..40410b90be 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -381,27 +381,31 @@ class MultiWorkerCrossDeviceOpsTest(multi_worker_test_base.MultiWorkerTestBase,
       distribution=[
           combinations.NamedDistribution(
               "MirroredCPU",
-              lambda: mirrored_strategy.MirroredStrategy(num_gpus=0),
+              lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=0),
               required_gpus=0),
           combinations.NamedDistribution(
               "Mirrored1GPU",
-              lambda: mirrored_strategy.MirroredStrategy(num_gpus=1),
+              lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=1),
               required_gpus=1),
           combinations.NamedDistribution(
               "Mirrored2GPUs",
-              lambda: mirrored_strategy.MirroredStrategy(num_gpus=2),
+              lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=2),
               required_gpus=2),
+          # pylint: disable=g-long-lambda
           combinations.NamedDistribution(
               "CoreMirroredCPU",
-              lambda: mirrored_strategy.CoreMirroredStrategy(num_gpus=0),
+              lambda: mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus_per_worker=0),
               required_gpus=0),
           combinations.NamedDistribution(
               "CoreMirrored1GPU",
-              lambda: mirrored_strategy.CoreMirroredStrategy(num_gpus=1),
+              lambda: mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus_per_worker=1),
               required_gpus=1),
           combinations.NamedDistribution(
               "CoreMirrored2GPUs",
-              lambda: mirrored_strategy.CoreMirroredStrategy(num_gpus=2),
+              lambda: mirrored_strategy.CoreMirroredStrategy(
+                  num_gpus_per_worker=2),
               required_gpus=2),
       ],
       mode=["graph"])
diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 202e92d1e7..0f35657a80 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -522,7 +522,7 @@ class RunConfigTest(test.TestCase):
       run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               train_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus=2)))
+                  num_gpus_per_worker=2)))
 
   def test_should_run_distribute_coordinator(self):
     """Tests that should_run_distribute_coordinator return a correct value."""
@@ -546,11 +546,11 @@ class RunConfigTest(test.TestCase):
       config_with_train_distribute = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               train_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus=2)))
+                  num_gpus_per_worker=2)))
       config_with_eval_distribute = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               eval_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus=2)))
+                  num_gpus_per_worker=2)))
     self.assertTrue(
         dc_training.should_run_distribute_coordinator(
             config_with_train_distribute))
@@ -564,7 +564,7 @@ class RunConfigTest(test.TestCase):
       config = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               train_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus=2)))
+                  num_gpus_per_worker=2)))
     self.assertFalse(dc_training.should_run_distribute_coordinator(config))
 
   def test_init_run_config_duplicate_distribute(self):
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index a2ab95487a..d671d1d562 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -303,58 +303,21 @@ class CoreMirroredStrategy(distribute_lib.DistributionStrategy):
   This strategy uses one replica per device and sync replication for its
   multi-GPU version.
 
-  When `cluster_spec` is given by the `configure` method., it turns into the
-  mulit-worker version that works on multiple workers with in-graph replication.
-  Note: `configure` will be called by higher-level APIs if running in
-  distributed environment.
-
-  There are several important concepts for distributed TensorFlow, e.g.
-  `client`, `job`, 'task', `cluster`, `in-graph replication` and
-  'synchronous training' and they have already been defined in the
-  [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed).
-  The distribution strategy inherits these concepts as well and in addition to
-  that we also clarify several more concepts:
-
-  * **In-graph replication**: the `client` creates a single `tf.Graph` that
-    specifies tasks for devices on all workers. The `client` then creates a
-    client session which will talk to the `master` service of a `worker`. Then
-    the `master` will partition the graph and distribute the work to all
-    participating workers.
-  * **Worker**: A `worker` is a TensorFlow `task` that usually maps to one
-    physical machine. We will have multiple `worker`s with different `task`
-    index. They all do similar things except for one worker checkpointing model
-    variables, writing summaries, etc. in addition to its ordinary work.
-
-  The multi-worker version of this class maps one replica to one device on a
-  worker. It mirrors all model variables on all replicas. For example, if you
-  have two `worker`s and each `worker` has 4 GPUs, it will create 8 copies of
-  the model variables on these 8 GPUs. Then like in MirroredStrategy, each
-  replica performs their computation with their own copy of variables unless in
-  cross-replica model where variable or tensor reduction happens.
+  The multi-worker version will be added in the fture.
 
   Args:
     devices: a list of device strings.
-    num_gpus: number of GPUs. For local training, either specify `devices` or
-      `num_gpus`. In distributed training, this must be specified as number of
-      GPUs on each worker.
-    num_gpus_per_worker: number of GPUs per worker. This is the same as
-      `num_gpus` and only one of `num_gpus` and `num_gpus_per_worker` can be
-      specified.
+    num_gpus_per_worker: number of GPUs per worker.
     cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not
-      set, the `configure` method will try to find the best one.
-    auto_shard_dataset: whether to auto-shard the dataset when there are
-      multiple workers.
+      set, nccl will be use by default.
   """
 
   def __init__(self,
                devices=None,
-               num_gpus=None,
                num_gpus_per_worker=None,
-               cross_device_ops=None,
-               auto_shard_dataset=False):
-    extended = CoreMirroredExtended(
-        self, devices, num_gpus, num_gpus_per_worker,
-        cross_device_ops, auto_shard_dataset)
+               cross_device_ops=None):
+    extended = CoreMirroredExtended(self, devices, num_gpus_per_worker,
+                                    cross_device_ops)
     super(CoreMirroredStrategy, self).__init__(extended)
 
 
@@ -364,21 +327,12 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
   def __init__(self,
                container_strategy,
                devices=None,
-               num_gpus=None,
                num_gpus_per_worker=None,
-               cross_device_ops=None,
-               auto_shard_dataset=False):
+               cross_device_ops=None):
     super(CoreMirroredExtended, self).__init__(container_strategy)
     self._cross_device_ops = cross_device_ops
-    self._auto_shard_dataset = auto_shard_dataset
     # Remember num GPUs which might be needed by `configure` method.
-    if num_gpus is not None and num_gpus_per_worker is not None:
-      raise ValueError(
-          "You cannot specify both `num_gpus` and `num_gpus_per_worker`.")
-    if num_gpus is not None:
-      self._num_gpus = num_gpus
-    else:
-      self._num_gpus = num_gpus_per_worker
+    self._num_gpus = num_gpus_per_worker
 
     self._initialize_local(self._num_gpus, devices)
 
@@ -493,8 +447,9 @@ class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
   def _distribute_dataset(self, dataset_fn):
     if self._cluster_spec:
       return values.MultiWorkerDataset(
-          partial(self._call_dataset_fn, dataset_fn), self._worker_devices,
-          auto_shard=self._auto_shard_dataset)
+          partial(self._call_dataset_fn, dataset_fn),
+          self._worker_devices,
+          auto_shard=False)
     else:
       return values.PerReplicaDataset(
           self._call_dataset_fn(dataset_fn), self._devices)
@@ -873,26 +828,29 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                auto_shard_dataset=False,
                cross_tower_ops=None):
     assert not (cross_device_ops and cross_tower_ops)
-    extended = MirroredExtended(
-        self, devices, num_gpus, num_gpus_per_worker,
-        cross_device_ops or cross_tower_ops, auto_shard_dataset)
+    if num_gpus is not None and num_gpus_per_worker is not None:
+      raise ValueError(
+          "You cannot specify both `num_gpus` and `num_gpus_per_worker`.")
+    if num_gpus is None:
+      num_gpus = num_gpus_per_worker
+    extended = MirroredExtended(self, devices, num_gpus,
+                                cross_device_ops or cross_tower_ops,
+                                auto_shard_dataset)
     super(MirroredStrategy, self).__init__(extended)
 
 
 class MirroredExtended(CoreMirroredExtended):
   """Implementation of (contrib) MirroredStrategy."""
 
-  # pylint: disable=useless-super-delegation
   def __init__(self,
                container_strategy,
                devices=None,
-               num_gpus=None,
                num_gpus_per_worker=None,
                cross_device_ops=None,
                auto_shard_dataset=False):
     super(MirroredExtended, self).__init__(
-        container_strategy, devices, num_gpus, num_gpus_per_worker,
-        cross_device_ops, auto_shard_dataset)
+        container_strategy, devices, num_gpus_per_worker, cross_device_ops)
+    self._auto_shard_dataset = auto_shard_dataset
 
   def _make_dataset_iterator(self, dataset):
     """Make iterator from dataset without splitting the batch.
@@ -912,6 +870,16 @@ class MirroredExtended(CoreMirroredExtended):
       worker_device_pairs = [("/job:localhost", self._devices)]
     return values.DatasetIterator(dataset, worker_device_pairs)
 
+  def _distribute_dataset(self, dataset_fn):
+    if self._cluster_spec:
+      return values.MultiWorkerDataset(
+          partial(self._call_dataset_fn, dataset_fn),
+          self._worker_devices,
+          auto_shard=self._auto_shard_dataset)
+    else:
+      return values.PerReplicaDataset(
+          self._call_dataset_fn(dataset_fn), self._devices)
+
 
 class MirroredReplicaContext(distribute_lib.ReplicaContext):
   """ReplicaContext used in MirroredStrategy.call_for_each_replica().
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index d20ac144bf..1027da857d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -1304,21 +1304,23 @@ class MirroredStrategyDefunTest(test.TestCase):
       self.assertAllEqual([0.5], updated_var_values[1])
 
 
-@combinations.generate(combinations.combine(
-    distribution=[
-        combinations.NamedDistribution(
-            "Mirrored",
-            # pylint: disable=g-long-lambda
-            lambda: mirrored_strategy.CoreMirroredStrategy(
-                num_gpus=context.num_gpus()),
-            required_gpus=1),
-        combinations.NamedDistribution(
-            "CoreMirrored",
-            # pylint: disable=g-long-lambda
-            lambda: mirrored_strategy.CoreMirroredStrategy(
-                num_gpus=context.num_gpus()),
-            required_gpus=1)],
-    mode=["graph"]))
+@combinations.generate(
+    combinations.combine(
+        distribution=[
+            combinations.NamedDistribution(
+                "Mirrored",
+                # pylint: disable=g-long-lambda
+                lambda: mirrored_strategy.CoreMirroredStrategy(
+                    num_gpus_per_worker=context.num_gpus()),
+                required_gpus=1),
+            combinations.NamedDistribution(
+                "CoreMirrored",
+                # pylint: disable=g-long-lambda
+                lambda: mirrored_strategy.CoreMirroredStrategy(
+                    num_gpus_per_worker=context.num_gpus()),
+                required_gpus=1)
+        ],
+        mode=["graph"]))
 class MultiWorkerMirroredStrategyTest(
     multi_worker_test_base.MultiWorkerTestBase,
     strategy_test_lib.DistributionTestBase):
-- 
GitLab


From 81493f17e8538691f7affae4458f5e4b2c1d74ce Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 20 Nov 2018 14:21:42 -0800
Subject: [PATCH 0620/1554] Allow TFE_Execute to be invoked in a new thread.

Returns an TFE_ExecuteOpOnThreadResult which can be waited upon.

PiperOrigin-RevId: 222306168
---
 tensorflow/c/BUILD                            |   4 +
 tensorflow/c/c_api_experimental.cc            |  54 +++++++-
 tensorflow/c/c_api_experimental.h             |  19 +++
 tensorflow/c/c_api_experimental_test.cc       | 123 ++++++++++++++++++
 .../core/common_runtime/eager/context.h       |   2 +
 5 files changed, 200 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 84238ffc1f..71089a87c7 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -121,6 +121,7 @@ tf_cuda_library(
         ":c_api",
         ":c_api_internal",
         "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:c_api_internal",
         "//tensorflow/compiler/jit:flags",
         "//tensorflow/contrib/tpu:all_ops",
         "//tensorflow/core:core_cpu",
@@ -274,8 +275,11 @@ tf_cc_test(
     # the shared library must be able to use core:framework.
     # linkstatic = tf_kernel_tests_linkstatic(),
     deps = [
+        ":c_api",
         ":c_api_experimental",
         ":c_test_util",
+        "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:c_api_test_util",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index f160f204de..069de45e33 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -15,7 +15,10 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_experimental.h"
 
+#include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/c/eager/c_api.h"
+#include "tensorflow/c/eager/c_api_internal.h"
 #include "tensorflow/compiler/jit/flags.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
 #include "tensorflow/core/framework/tensor.pb.h"
@@ -8740,8 +8743,55 @@ void TFE_TensorHandlePrintDebugString(TFE_TensorHandle* handle) {
   TF_DeleteStatus(status);
 }
 
-TF_CAPI_EXPORT extern void TF_MakeInternalErrorStatus(TF_Status* status,
-                                                      const char* errMsg) {
+struct TFE_ExecuteOpNotification {
+  TFE_ExecuteOpNotification() : status(TF_NewStatus(), TF_DeleteStatus) {}
+  tensorflow::Notification n;
+  std::unique_ptr<tensorflow::Thread> thread;
+  std::unique_ptr<TF_Status, decltype(&TF_DeleteStatus)> status;
+};
+
+TFE_ExecuteOpNotification* TFE_ExecuteOpInNewThread(TFE_Op* op,
+                                                    TFE_TensorHandle** retvals,
+                                                    int* num_retvals,
+                                                    TF_Status* status) {
+  TFE_ExecuteOpNotification* n = new TFE_ExecuteOpNotification;
+
+  n->thread.reset(op->operation.EagerContext()->TFEnv()->StartThread(
+      tensorflow::ThreadOptions(), "ExecuteOpThread",
+      [op, retvals, num_retvals, n]() {
+        TFE_Execute(op, retvals, num_retvals, n->status.get());
+        n->n.Notify();
+      }));
+
+  return n;
+}
+
+void TFE_ExecuteOpNotificationWaitAndDelete(
+    TFE_ExecuteOpNotification* notification, TF_Status* status) {
+  if (notification == nullptr) {
+    status->status = tensorflow::errors::InvalidArgument(
+        "Passed in notification is a nullptr.");
+
+    return;
+  }
+  if (notification->thread == nullptr) {
+    status->status = tensorflow::errors::InvalidArgument(
+        "Passed in notification didn't start a thread correctly. Cleaning up "
+        "this notification. Please re-execute the operation to get a new "
+        "notification.");
+
+    delete notification;
+    return;
+  }
+
+  notification->n.WaitForNotification();
+
+  status->status = notification->status->status;
+
+  delete notification;
+}
+
+void TF_MakeInternalErrorStatus(TF_Status* status, const char* errMsg) {
   status->status = tensorflow::errors::Internal(errMsg);
 }
 
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 25c03df518..728445e8fd 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -180,6 +180,25 @@ TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_DequeueVariantTensor(
 TF_CAPI_EXPORT extern void TFE_TensorHandlePrintDebugString(
     TFE_TensorHandle* handle);
 
+typedef struct TFE_ExecuteOpNotification TFE_ExecuteOpNotification;
+
+// Allows invoking a kernel asynchronously, and explicitly returns a
+// notification that can be waited upon. This always executes the kernel in a
+// new thread.
+// 1. `retvals` and `num_retvals` can only be consumed after
+// `TFE_ExecuteOp` returns successfully. They shouldn't be used
+// if the return is unsuccessful
+// 2. These new APIs cannot be used together with the TFE context level async
+// support.
+TF_CAPI_EXPORT extern TFE_ExecuteOpNotification* TFE_ExecuteOpInNewThread(
+    TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
+    TF_Status* status);
+
+// Waits to complete the op execution, and cleans up the notification.
+// Errors reported by op execution are set in `status`.
+TF_CAPI_EXPORT extern void TFE_ExecuteOpNotificationWaitAndDelete(
+    TFE_ExecuteOpNotification* notification, TF_Status* status);
+
 TF_CAPI_EXPORT extern void TF_MakeInternalErrorStatus(TF_Status* status,
                                                       const char* errMsg);
 
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index 881dbaf35a..daa7701b7f 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_experimental.h"
 #include "tensorflow/c/c_test_util.h"
+#include "tensorflow/c/eager/c_api.h"
+#include "tensorflow/c/eager/c_api_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
@@ -173,5 +175,126 @@ TEST(CAPI_EXPERIMENTAL, IsStateful) {
   EXPECT_EQ(id, 0);
 }
 
+TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Simple) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+
+  TFE_Op* matmul_op = MatMulOp(ctx, m, m);
+
+  TFE_TensorHandle* retvals[1] = {nullptr};
+  int num_retvals = 1;
+
+  auto* r =
+      TFE_ExecuteOpInNewThread(matmul_op, &retvals[0], &num_retvals, status);
+
+  TFE_ExecuteOpNotificationWaitAndDelete(r, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  float product[4] = {0};
+  EXPECT_EQ(sizeof(product), TF_TensorByteSize(t));
+  memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t));
+  TF_DeleteTensor(t);
+  EXPECT_EQ(7, product[0]);
+  EXPECT_EQ(10, product[1]);
+  EXPECT_EQ(15, product[2]);
+  EXPECT_EQ(22, product[3]);
+
+  TFE_DeleteOp(matmul_op);
+  TFE_DeleteTensorHandle(m);
+
+  TFE_DeleteTensorHandle(retvals[0]);
+  TFE_DeleteContext(ctx);
+  TF_DeleteStatus(status);
+}
+
+// Perform a send/recv test. Recv blocks, so they need to be executed
+// asynchronously.
+TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Blocking) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  // Returns a 2x2 float32 Tensor on the CPU, with data 1., 2., 3., 4.
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+
+  // Build a send op.
+  TFE_Op* send_op = TFE_NewOp(ctx, "_Send", status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_OpAddInput(send_op, m, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  string tensor_name = "Tensor";
+  TFE_OpSetAttrType(send_op, "T", TF_FLOAT);
+  TFE_OpSetAttrString(send_op, "tensor_name", tensor_name.c_str(),
+                      tensor_name.size());
+  string send_device = "/job:localhost/replica:0/task:0/device:CPU:0";
+  TFE_OpSetAttrString(send_op, "send_device", send_device.c_str(),
+                      send_device.size());
+  TFE_OpSetAttrInt(send_op, "send_device_incarnation", 1234);
+  string recv_device = "/job:localhost/replica:0/task:0/device:CPU:0";
+  TFE_OpSetAttrString(send_op, "recv_device", recv_device.c_str(),
+                      recv_device.size());
+  TFE_OpSetAttrBool(send_op, "client_terminated", true);
+
+  // Build a recv op.
+  TFE_Op* recv_op = TFE_NewOp(ctx, "_Recv", status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  TFE_OpSetAttrType(recv_op, "tensor_type", TF_FLOAT);
+  TFE_OpSetAttrString(recv_op, "tensor_name", tensor_name.c_str(),
+                      tensor_name.size());
+  TFE_OpSetAttrString(recv_op, "send_device", send_device.c_str(),
+                      send_device.size());
+  TFE_OpSetAttrInt(recv_op, "send_device_incarnation", 1234);
+  TFE_OpSetAttrString(recv_op, "recv_device", recv_device.c_str(),
+                      recv_device.size());
+  TFE_OpSetAttrBool(recv_op, "client_terminated", true);
+
+  TFE_TensorHandle* send_retvals;
+  int send_num_retvals = 0;
+  auto* send_result = TFE_ExecuteOpInNewThread(send_op, &send_retvals,
+                                               &send_num_retvals, status);
+
+  TFE_TensorHandle* recv_retvals[1] = {nullptr};
+  int recv_num_retvals = 1;
+  auto* recv_result = TFE_ExecuteOpInNewThread(recv_op, &recv_retvals[0],
+                                               &recv_num_retvals, status);
+
+  TFE_ExecuteOpNotificationWaitAndDelete(send_result, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_ExecuteOpNotificationWaitAndDelete(recv_result, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  TF_Tensor* t = TFE_TensorHandleResolve(recv_retvals[0], status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  float product[4] = {0};
+  EXPECT_EQ(sizeof(product), TF_TensorByteSize(t));
+  memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t));
+  TF_DeleteTensor(t);
+  EXPECT_EQ(1, product[0]);
+  EXPECT_EQ(2, product[1]);
+  EXPECT_EQ(3, product[2]);
+  EXPECT_EQ(4, product[3]);
+
+  TFE_DeleteOp(send_op);
+  TFE_DeleteOp(recv_op);
+  TFE_DeleteTensorHandle(m);
+
+  TFE_DeleteTensorHandle(recv_retvals[0]);
+  TFE_DeleteContext(ctx);
+  TF_DeleteStatus(status);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index 4de807bde3..51109f8f1a 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -206,6 +206,8 @@ class EagerContext {
   bool UseSendTensorRPC() { return use_send_tensor_rpc_; }
   bool PinSmallOpsToCPU() { return pin_small_ops_to_cpu_; }
 
+  tensorflow::Env* TFEnv() const { return env_; }
+
  private:
   void InitDeviceMapAndAsync();
   Status MaybeRegisterFunctionRemotely(const FunctionDef& fdef);
-- 
GitLab


From 178b4349ae1710869c625ec843213584253faeb1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 14:43:22 -0800
Subject: [PATCH 0621/1554] Adds string deserialization support for V2
 Optimizers

PiperOrigin-RevId: 222309566
---
 tensorflow/python/keras/BUILD                 |  2 +
 .../python/keras/optimizer_v2/adamax.py       | 14 +++----
 .../python/keras/optimizer_v2/adamax_test.py  | 28 ++++++-------
 .../python/keras/optimizer_v2/rmsprop.py      | 32 +++++++--------
 .../python/keras/optimizer_v2/rmsprop_test.py | 16 ++++----
 tensorflow/python/keras/optimizers.py         | 39 ++++++++++++++-----
 tensorflow/python/keras/optimizers_test.py    | 38 ++++++++++++++++++
 7 files changed, 114 insertions(+), 55 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 37dcc9eb04..81ef941ba6 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -266,6 +266,7 @@ py_test(
     name = "optimizers_test",
     size = "medium",
     srcs = ["optimizers_test.py"],
+    shard_count = 2,
     srcs_version = "PY2AND3",
     tags = ["notsan"],
     deps = [
@@ -273,6 +274,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 6712427f91..7530e629d1 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""AdaMax for TensorFlow."""
+"""Adamax for TensorFlow."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -27,8 +27,8 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.training import training_ops
 
 
-class AdaMax(adam.Adam):
-  """Optimizer that implements the AdaMax algorithm.
+class Adamax(adam.Adam):
+  """Optimizer that implements the Adamax algorithm.
 
   It is a variant of Adam based on the infinity norm.
   Default parameters follow those provided in the paper.
@@ -44,8 +44,8 @@ class AdaMax(adam.Adam):
                beta_1=0.9,
                beta_2=0.999,
                epsilon=1e-7,
-               name='AdaMax'):
-    """Construct a new AdaMax optimizer.
+               name='Adamax'):
+    """Construct a new Adamax optimizer.
 
     Initialization:
 
@@ -86,10 +86,10 @@ class AdaMax(adam.Adam):
         rate for the exponentially weighted infinity norm.
       epsilon: A small constant for numerical stability.
       name: Optional name for the operations created when applying gradients.
-        Defaults to "AdaMax".
+        Defaults to "Adamax".
     """
     # pylint: disable=useless-super-delegation
-    super(AdaMax, self).__init__(learning_rate, beta_1, beta_2, epsilon, name)
+    super(Adamax, self).__init__(learning_rate, beta_1, beta_2, epsilon, name)
     # pylint: enable=useless-super-delegation
 
   def _resource_apply_dense(self, grad, var):
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index 23eb718429..c6b45ccbe9 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for AdaMax."""
+"""Tests for Adamax."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -76,7 +76,7 @@ def get_beta_accumulators(opt, dtype):
   return beta_1_power
 
 
-class AdaMaxOptimizerTest(test.TestCase):
+class AdamaxOptimizerTest(test.TestCase):
 
   def doTestSparse(self, use_resource=False):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -100,7 +100,7 @@ class AdaMaxOptimizerTest(test.TestCase):
         grads1 = ops.IndexedSlices(
             constant_op.constant(grads1_np),
             constant_op.constant(grads1_np_indices), constant_op.constant([3]))
-        opt = adamax.AdaMax()
+        opt = adamax.Adamax()
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
@@ -110,7 +110,7 @@ class AdaMaxOptimizerTest(test.TestCase):
 
         beta1_power = get_beta_accumulators(opt, dtype)
 
-        # Run 3 steps of AdaMax
+        # Run 3 steps of Adamax
         for t in range(1, 4):
           self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
           update.run()
@@ -135,7 +135,7 @@ class AdaMaxOptimizerTest(test.TestCase):
         var = variables.Variable([[1.0], [2.0]])
         indices = constant_op.constant([0, 1], dtype=index_dtype)
         gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices))
-        optimizer = adamax.AdaMax(3.0)
+        optimizer = adamax.Adamax(3.0)
         minimize_op = optimizer.minimize(gathered_sum, var_list=[var])
         variables.global_variables_initializer().run()
         minimize_op.run()
@@ -157,9 +157,9 @@ class AdaMaxOptimizerTest(test.TestCase):
                 [0.2], shape=[1, 1], dtype=dtype),
             constant_op.constant([1]),
             constant_op.constant([2, 1]))
-        repeated_update = adamax.AdaMax().apply_gradients(
+        repeated_update = adamax.Adamax().apply_gradients(
             [(grad_repeated_index, repeated_index_update_var)])
-        aggregated_update = adamax.AdaMax().apply_gradients(
+        aggregated_update = adamax.Adamax().apply_gradients(
             [(grad_aggregated, aggregated_update_var)])
         variables.global_variables_initializer().run()
         self.assertAllClose(aggregated_update_var.eval(),
@@ -189,7 +189,7 @@ class AdaMaxOptimizerTest(test.TestCase):
         grads0 = constant_op.constant(grads0_np)
         grads1 = constant_op.constant(grads1_np)
 
-        opt = adamax.AdaMax()
+        opt = adamax.Adamax()
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
         if not context.executing_eagerly():
@@ -198,7 +198,7 @@ class AdaMaxOptimizerTest(test.TestCase):
           self.assertAllClose([1.0, 2.0], self.evaluate(var0))
           self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 3 steps of AdaMax
+        # Run 3 steps of Adamax
         for t in range(1, 4):
           if not context.executing_eagerly():
             self.evaluate(update)
@@ -232,7 +232,7 @@ class AdaMaxOptimizerTest(test.TestCase):
         var1 = variables.Variable(var1_np)
         grads0 = constant_op.constant(grads0_np)
         grads1 = constant_op.constant(grads1_np)
-        opt = adamax.AdaMax(constant_op.constant(0.001))
+        opt = adamax.Adamax(constant_op.constant(0.001))
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
@@ -242,7 +242,7 @@ class AdaMaxOptimizerTest(test.TestCase):
 
         beta1_power = get_beta_accumulators(opt, dtype)
 
-        # Run 3 steps of AdaMax
+        # Run 3 steps of Adamax
         for t in range(1, 4):
           self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
           update.run()
@@ -268,7 +268,7 @@ class AdaMaxOptimizerTest(test.TestCase):
         var1 = variables.Variable(var1_np)
         grads0 = constant_op.constant(grads0_np)
         grads1 = constant_op.constant(grads1_np)
-        opt = adamax.AdaMax()
+        opt = adamax.Adamax()
         update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
@@ -279,7 +279,7 @@ class AdaMaxOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], var0.eval())
         self.assertAllClose([3.0, 4.0], var1.eval())
 
-        # Run 3 steps of intertwined AdaMax1 and AdaMax2.
+        # Run 3 steps of intertwined Adamax1 and Adamax2.
         for t in range(1, 4):
           self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
           if t % 2 == 0:
@@ -298,7 +298,7 @@ class AdaMaxOptimizerTest(test.TestCase):
     with context.eager_mode():
       v1 = resource_variable_ops.ResourceVariable(1.)
       v2 = resource_variable_ops.ResourceVariable(1.)
-      opt = adamax.AdaMax(1.)
+      opt = adamax.Adamax(1.)
       opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
       # There should be iteration, hyper variables, and two unique slot
       # variables for v1 and v2 respectively.
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index e34397c06d..eae5620349 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""RMSProp for TensorFlow."""
+"""RMSprop for TensorFlow."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -23,8 +23,8 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.training import training_ops
 
 
-class RMSProp(optimizer_v2.OptimizerV2):
-  r"""Optimizer that implements the RMSProp algorithm.
+class RMSprop(optimizer_v2.OptimizerV2):
+  r"""Optimizer that implements the RMSprop algorithm.
 
   A detailed description of rmsprop.
 
@@ -36,7 +36,7 @@ class RMSProp(optimizer_v2.OptimizerV2):
       mean_square_t + \epsilon}$$
   $$variable_t := variable_{t-1} - mom_t
 
-  This implementation of RMSProp uses plain momentum, not Nesterov momentum.
+  This implementation of RMSprop uses plain momentum, not Nesterov momentum.
 
   The centered version additionally maintains a moving average of the
   gradients, and uses that average to estimate the variance:
@@ -58,8 +58,8 @@ class RMSProp(optimizer_v2.OptimizerV2):
                momentum=0.0,
                epsilon=1e-7,
                centered=False,
-               name="RMSProp"):
-    """Construct a new RMSProp optimizer.
+               name="RMSprop"):
+    """Construct a new RMSprop optimizer.
 
     Note that in the dense implementation of this algorithm, variables and their
     corresponding accumulators (momentum, gradient moving average, square
@@ -83,16 +83,13 @@ class RMSProp(optimizer_v2.OptimizerV2):
         True may help with training, but is slightly more expensive in terms of
         computation and memory. Defaults to False.
       name: Optional name prefix for the operations created when applying
-        gradients. Defaults to "RMSProp".
-
-    @compatibility(eager)
-    When eager execution is enabled, `learning_rate`, `decay`, `momentum`, and
-    `epsilon` can each be a callable that takes no arguments and returns the
-    actual value to use. This can be useful for changing these values across
-    different invocations of optimizer functions.
-    @end_compatibility
+        gradients. Defaults to "RMSprop".  @compatibility(eager) When eager
+        execution is enabled, `learning_rate`, `decay`, `momentum`, and
+        `epsilon` can each be a callable that takes no arguments and returns the
+        actual value to use. This can be useful for changing these values across
+        different invocations of optimizer functions. @end_compatibility
     """
-    super(RMSProp, self).__init__(name)
+    super(RMSprop, self).__init__(name)
     self._set_hyper("learning_rate", learning_rate)
     self._set_hyper("rho", rho)
 
@@ -182,7 +179,7 @@ class RMSProp(optimizer_v2.OptimizerV2):
           use_locking=self._use_locking)
 
   def get_config(self):
-    config = super(RMSProp, self).get_config()
+    config = super(RMSprop, self).get_config()
     config.update({
         "learning_rate": self._serialize_hyperparameter("learning_rate"),
         "rho": self._serialize_hyperparameter("rho"),
@@ -191,3 +188,6 @@ class RMSProp(optimizer_v2.OptimizerV2):
         "centered": self._centered,
     })
     return config
+
+
+RMSProp = RMSprop
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 8d7afa54cc..2ac090724e 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -52,7 +52,7 @@ _TESTPARAMS = [
 ]
 
 
-class RMSPropOptimizerTest(test.TestCase):
+class RMSpropOptimizerTest(test.TestCase):
 
   def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, rho, momentum,
                             epsilon, centered):
@@ -98,7 +98,7 @@ class RMSPropOptimizerTest(test.TestCase):
         var1 = resource_variable_ops.ResourceVariable(var1_np, dtype=dtype)
         grads0 = constant_op.constant(grads0_np, dtype=dtype)
         grads1 = constant_op.constant(grads1_np, dtype=dtype)
-        opt = rmsprop.RMSProp(
+        opt = rmsprop.RMSprop(
             learning_rate=learning_rate,
             rho=rho,
             momentum=momentum,
@@ -135,7 +135,7 @@ class RMSPropOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 4 steps of RMSProp
+        # Run 4 steps of RMSprop
         for _ in range(1, 5):
           update.run()
 
@@ -164,7 +164,7 @@ class RMSPropOptimizerTest(test.TestCase):
         x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
         pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
         loss = pred * pred
-        sgd_op = rmsprop.RMSProp(
+        sgd_op = rmsprop.RMSprop(
             learning_rate=1.0,
             rho=0.0,
             momentum=0.0,
@@ -188,7 +188,7 @@ class RMSPropOptimizerTest(test.TestCase):
         x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
         pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
         loss = pred * pred
-        sgd_op = rmsprop.RMSProp(
+        sgd_op = rmsprop.RMSprop(
             learning_rate=1.0,
             rho=0.0,
             momentum=0.0,
@@ -224,7 +224,7 @@ class RMSPropOptimizerTest(test.TestCase):
         grads1 = ops.IndexedSlices(
             constant_op.constant(grads1_np),
             constant_op.constant(grads1_np_indices), constant_op.constant([1]))
-        opt = rmsprop.RMSProp(
+        opt = rmsprop.RMSprop(
             learning_rate=learning_rate,
             rho=rho,
             momentum=momentum,
@@ -261,7 +261,7 @@ class RMSPropOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 4 steps of RMSProp
+        # Run 4 steps of RMSprop
         for _ in range(1, 5):
           update.run()
 
@@ -295,7 +295,7 @@ class RMSPropOptimizerTest(test.TestCase):
         rho = lambda: 0.9
         momentum = lambda: 0.0
         epsilon = lambda: 1.0
-        opt = rmsprop.RMSProp(learning_rate, rho, momentum, epsilon)
+        opt = rmsprop.RMSprop(learning_rate, rho, momentum, epsilon)
 
         # Fetch params to validate initial values
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 09dd708b93..9c8020dc05 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -22,8 +22,16 @@ from __future__ import print_function
 import six
 from six.moves import zip  # pylint: disable=redefined-builtin
 
+from tensorflow.python import tf2
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2
+from tensorflow.python.keras.optimizer_v2 import adagrad as adagrad_v2
+from tensorflow.python.keras.optimizer_v2 import adam as adam_v2
+from tensorflow.python.keras.optimizer_v2 import adamax as adamax_v2
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.python.keras.optimizer_v2 import nadam as nadam_v2
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_v2
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import clip_ops
@@ -796,16 +804,27 @@ def deserialize(config, custom_objects=None):
   Returns:
       A Keras Optimizer instance.
   """
-  all_classes = {
-      'sgd': SGD,
-      'rmsprop': RMSprop,
-      'adagrad': Adagrad,
-      'adadelta': Adadelta,
-      'adam': Adam,
-      'adamax': Adamax,
-      'nadam': Nadam,
-      'tfoptimizer': TFOptimizer,
-  }
+  if tf2.enabled():
+    all_classes = {
+        'adadelta': adadelta_v2.Adadelta,
+        'adagrad': adagrad_v2.Adagrad,
+        'adam': adam_v2.Adam,
+        'adamax': adamax_v2.Adamax,
+        'nadam': nadam_v2.Nadam,
+        'rmsprop': rmsprop_v2.RMSprop,
+        'sgd': gradient_descent_v2.SGD
+    }
+  else:
+    all_classes = {
+        'adadelta': Adadelta,
+        'adagrad': Adagrad,
+        'adam': Adam,
+        'adamax': Adamax,
+        'nadam': Nadam,
+        'rmsprop': RMSprop,
+        'sgd': SGD,
+        'tfoptimizer': TFOptimizer
+    }
   # Make deserialization case-insensitive for built-in optimizers.
   if config['class_name'].lower() in all_classes:
     config['class_name'] = config['class_name'].lower()
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index 9664f09fff..46bb0274c6 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -19,11 +19,14 @@ from __future__ import division
 from __future__ import print_function
 
 import gc
+import os
 import weakref
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python import tf2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -208,5 +211,40 @@ class KerasOptimizersTest(test.TestCase):
       _ = keras.optimizers.Adam(clipnorm=-2.0)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class KerasV2OptimizersTest(test.TestCase, parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('adadelta_tf2', 'adadelta', True), ('adadelta_tf1', 'adadelta', False),
+      ('adagrad_tf2', 'adagrad', True), ('adagrad_tf1', 'adagrad', False),
+      ('adam_tf2', 'adam', True), ('adam_tf1', 'adam', False),
+      ('adamax_tf2', 'adamax', True), ('adamax_tf1', 'adamax', False),
+      ('sgd_tf2', 'sgd', True), ('sgd_tf1', 'sgd', False),
+      ('nadam_tf2', 'nadam', True), ('nadam_tf1', 'nadam', False),
+      ('rmsprop_tf2', 'rmsprop', True), ('rmsprop_tf1', 'rmsprop', False))
+  def test_load_from_string(self, optimizer_string, tf2mode):
+    old_mode = os.environ.get('TF2_BEHAVIOR', None)
+    if tf2mode:
+      os.environ['TF2_BEHAVIOR'] = 'enabled'
+    else:
+      if 'TF2_BEHAVIOR' in os.environ:
+        del os.environ['TF2_BEHAVIOR']
+
+    # Sanity check.
+    self.assertEqual(tf2.enabled(), tf2mode)
+
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(1, input_shape=(10,)))
+    model.compile(optimizer_string, 'binary_crossentropy')
+
+    self.assertEqual(optimizer_string,
+                     model.optimizer.__class__.__name__.lower())
+
+    model.fit(np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'))
+
+    if old_mode is not None:
+      os.environ['TF2_BEHAVIOR'] = old_mode
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From cfb6e1634ecb89ca32ee60ce394c119fabe5d61c Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Tue, 20 Nov 2018 14:53:45 -0800
Subject: [PATCH 0622/1554] [TF2XLA] In Resampler, correctly handle out of
 boundary samples by returning 0 in the forward pass.

PiperOrigin-RevId: 222311168
---
 .../compiler/tests/resampler_ops_test.py      | 53 +++++++++++++++++-
 .../compiler/tf2xla/kernels/resampler_ops.cc  | 56 +++++++++++++++++--
 2 files changed, 102 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/tests/resampler_ops_test.py b/tensorflow/compiler/tests/resampler_ops_test.py
index f87ac3360c..d8ca0eab27 100644
--- a/tensorflow/compiler/tests/resampler_ops_test.py
+++ b/tensorflow/compiler/tests/resampler_ops_test.py
@@ -63,8 +63,8 @@ class ResamplerOpsTest(xla_test.XLATestCase):
   def testSimple(self):
     for dtype in self.float_types:
       input_shape = [1, 2, 2, 1]
-      input_rgb_data = [0, 5, 13, 54]
-      input_np = np.array(input_rgb_data, dtype=dtype).reshape(input_shape)
+      input_data = [0, 5, 13, 54]
+      input_np = np.array(input_data, dtype=dtype).reshape(input_shape)
 
       warp_shape = [1, 2]
       warp_data = [0.7, 0.6]
@@ -151,6 +151,55 @@ class ResamplerOpsTest(xla_test.XLATestCase):
                                             expected_grad_data,
                                             expected_grad_warp)
 
+  def testOutOfBoundWarps(self):
+    # (x, y) are both less than 0.
+    for dtype in self.float_types:
+      input_shape = [1, 2, 2, 1]
+      input_data = [10, 5, 13, 54]
+      input_np = np.array(input_data, dtype=dtype).reshape(input_shape)
+
+      warp_shape = [1, 2, 2]
+      warp_data = [-1, -1, 0.7, 0.6]
+      warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape)
+      expected = [[[0.0], [27.62]]]
+      self._assertForwardOpMatchesExpected(input_np, warp_np, expected)
+
+    # One of (x, y) is less than 0.
+    for dtype in self.float_types:
+      input_shape = [1, 2, 2, 1]
+      input_data = [10, 5, 13, 54]
+      input_np = np.array(input_data, dtype=dtype).reshape(input_shape)
+
+      warp_shape = [1, 2, 2]
+      warp_data = [-1, 0.1, 0.7, 0.6]
+      warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape)
+      expected = [[[0.0], [27.62]]]
+      self._assertForwardOpMatchesExpected(input_np, warp_np, expected)
+
+    # Both of (x, y) are greater than image size.
+    for dtype in self.float_types:
+      input_shape = [1, 2, 2, 1]
+      input_data = [10, 5, 13, 54]
+      input_np = np.array(input_data, dtype=dtype).reshape(input_shape)
+
+      warp_shape = [1, 2, 2]
+      warp_data = [-0.1, 0.1, 1.2, 2.1]
+      warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape)
+      expected = [[[0.0], [0.0]]]
+      self._assertForwardOpMatchesExpected(input_np, warp_np, expected)
+
+    # One of (x, y) is greater than image size.
+    for dtype in self.float_types:
+      input_shape = [1, 2, 2, 1]
+      input_data = [10, 5, 13, 54]
+      input_np = np.array(input_data, dtype=dtype).reshape(input_shape)
+
+      warp_shape = [1, 2, 2]
+      warp_data = [0.1, -0.1, 1.2, 0.1]
+      warp_np = np.array(warp_data, dtype=dtype).reshape(warp_shape)
+      expected = [[[0.0], [0.0]]]
+      self._assertForwardOpMatchesExpected(input_np, warp_np, expected)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
index 847704608f..8a8f33c8f3 100644
--- a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
@@ -44,9 +44,6 @@ namespace {
 
 using xla::XlaOp;
 
-// TODO(b/112295522): note that sampling from image boundary is not currently
-// being handled properly.
-
 // Calculates the bilinear weight tensor, given basis ratio (px, py) of the
 // sampling position:
 //    W = [(1-px)*(1-py), px*(1-py), (1-px)*py, px*py]
@@ -421,12 +418,13 @@ class ResamplerOp : public XlaOpKernel {
     OP_REQUIRES(ctx, warp_shape.dim_size(last_warp_dim) == 2,
                 errors::InvalidArgument(
                     "the last dimension of warp must be exactly size 2."));
+    xla::PrimitiveType warp_type = ctx->input_xla_type(1);
 
     XlaOp data = ctx->Input("data");
     XlaOp warp = ctx->Input("warp");
 
     // Find the coordinates of the top left corner for the 2x2 region to be
-    // sampled from. The dimensions are (batch, dim_0, ... dim_n, 2) where the
+    // sampled from. The dimensions are [batch, dim_0, ... dim_n, 2] where the
     // last dimension of size 2 in turn is [x, y].
     XlaOp top_left = xla::ConvertElementType(warp, xla::U32);
 
@@ -457,10 +455,56 @@ class ResamplerOp : public XlaOpKernel {
     dot_dims.add_lhs_contracting_dimensions(warp_shape.dims() - 1);
     dot_dims.add_rhs_contracting_dimensions(warp_shape.dims() - 1);
 
+    // The dimension is [batch, dim_0, ...dim_n, data_channels].
     auto blended_pixels = xla::DotGeneral(weights, neighbors_data, dot_dims,
                                           /*precision_config=*/nullptr);
 
-    ctx->SetOutput(0, blended_pixels);
+    // Handle out of boundary cases by constructing a predicate mask array based
+    // on the in-bound condition, and output 0 for the blended pixel value if
+    // out-bound. The dimension is the same as top_left: [batch, dim_0,
+    // ...dim_n, 2] where the last dimension of size 2 is the [x, y] coordinate.
+
+    auto is_ge_zero = xla::Ge(warp, xla::ZerosLike(warp));
+
+    auto is_lt_image_size = xla::Lt(
+        warp,
+        xla::ConvertElementType(
+            xla::ConstantR1<float>(
+                ctx->builder(),
+                {/*width=*/static_cast<float>(data_shape.dim_size(2) - 1),
+                 /*height=*/static_cast<float>(data_shape.dim_size(1) - 1)}),
+            warp_type),
+        /*broadcast_dimensions=*/{warp_shape.dims() - 1});
+
+    auto is_in_bound_x_y = xla::And(is_ge_zero, is_lt_image_size);
+    // Reduce along last dimension. The resulting dimension is:
+    // [batch, dim_0, ...dim_n].
+    auto is_in_bound = xla::Reduce(
+        is_in_bound_x_y, xla::ConstantR0<bool>(ctx->builder(), true),
+        xla::CreateScalarAndComputation(xla::PrimitiveType::PRED,
+                                        ctx->builder()),
+        {last_warp_dim});
+
+    // Broadcast 'is_in_bound' to the same dimension as 'blended_pixels', which
+    // is the dimension of the result:
+    //  [batch, dim_0, ...dim_n, data_channels].
+    auto warp_dims = warp_shape.dim_sizes();
+    std::vector<int64> result_dims(warp_dims.begin(), warp_dims.end() - 1);
+    result_dims.push_back(data_channels);
+    xla::Shape broadcasted_shape =
+        xla::ShapeUtil::MakeShape(xla::PrimitiveType::PRED, result_dims);
+
+    std::vector<int64> broadcasted_dims(warp_dims.size() - 1);
+    std::iota(broadcasted_dims.begin(), broadcasted_dims.end(), 0);
+    auto broadcasted_is_in_bound =
+        xla::BroadcastInDim(is_in_bound, broadcasted_shape, broadcasted_dims);
+
+    // Set out of bound samples to zero.
+    auto zeros =
+        xla::Broadcast(xla::Zero(ctx->builder(), data_type), result_dims);
+    auto result = xla::Select(broadcasted_is_in_bound, blended_pixels, zeros);
+
+    ctx->SetOutput(0, result);
   }
 };
 
@@ -473,6 +517,8 @@ class ResamplerGradOp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &output_dtype));
   }
 
+  // TODO(b/112295522): note that sampling from image boundary is not currently
+  // being handled properly.
   void Compile(XlaOpKernelContext* ctx) override {
     TensorShape data_shape_tf = ctx->InputShape("data");
     OP_REQUIRES(ctx, data_shape_tf.dims() == 4,
-- 
GitLab


From bf4acd8775e93dfc54250e2414cc3971a29a1de1 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 15:01:36 -0800
Subject: [PATCH 0623/1554] TFTRT: include log in VLOG_IS_ON(1)

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 1e0ea5f7e2..bc82b3c0f0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -982,11 +982,13 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       // Graph is not modified.
       LOG(WARNING) << msg << " failed: " << status << ". Fallback to TF...";
     }
-    msg = "Segment consists of nodes: ";
-    for (const string& node_name : converted_segments.at(i).first) {
-      StrAppend(&msg, node_name, ", ");
+    if (VLOG_IS_ON(1)) {
+      msg = "Segment consists of nodes: ";
+      for (const string& node_name : converted_segments.at(i).first) {
+        StrAppend(&msg, node_name, ", ");
+      }
+      VLOG(1) << msg;
     }
-    VLOG(1) << msg;
   }
   cudaSetDevice(old_cuda_device);
   graph.ToGraphDef(params.output_graph_def);
-- 
GitLab


From 0e4fced6195fb73f1c3091d8bfd2c341102f347e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 14:57:06 -0800
Subject: [PATCH 0624/1554] Fixed/improved ragged.from_sparse() precondition
 logic for enforcing that the input sparse tensor is statically known to be of
 rank 2.

PiperOrigin-RevId: 222311695
---
 .../python/ops/ragged/ragged_conversion_ops.py     | 11 ++++++++---
 .../ops/ragged/ragged_from_sparse_op_test.py       | 14 ++++++++++++++
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/ragged/ragged_conversion_ops.py b/tensorflow/python/ops/ragged/ragged_conversion_ops.py
index 0385be02d4..83212e49cf 100644
--- a/tensorflow/python/ops/ragged/ragged_conversion_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_conversion_ops.py
@@ -361,9 +361,14 @@ def from_sparse(st_input, name=None):
     st_input = sparse_tensor.convert_to_tensor_or_sparse_tensor(
         st_input, name='rt_input')
 
-    if (st_input.dense_shape.shape.ndims != 2 and
-        st_input.indices.shape.ndims is None or
-        st_input.indices.shape.dims[1].value != 2):
+    static_rank_from_dense_shape = (
+        None if st_input.dense_shape.shape.ndims is None
+        else st_input.dense_shape.shape.dims[0].value)
+    static_rank_from_indices = (
+        None if st_input.indices.shape.ndims is None
+        else st_input.indices.shape.dims[1].value)
+
+    if static_rank_from_dense_shape != 2 and static_rank_from_indices != 2:
       raise ValueError('rank(st_input) must be 2')
 
     with ops.control_dependencies(
diff --git a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
index ff19ddedeb..77418ff20d 100644
--- a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
@@ -64,6 +64,20 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
                             ragged.from_sparse, st3)
 
+  def testGoodPartialSparseTensorRank(self):
+    st1 = sparse_tensor.SparseTensor(
+        indices=[[0, 0]],
+        values=[0],
+        dense_shape=array_ops.placeholder(dtypes.int64))
+    st2 = sparse_tensor.SparseTensor(
+        indices=array_ops.placeholder(dtypes.int64),
+        values=[0],
+        dense_shape=[4, 3])
+
+    # Shouldn't throw ValueError
+    ragged.from_sparse(st1)
+    ragged.from_sparse(st2)
+
   def testNonRaggedSparseTensor(self):
     # "index_suffix" means the value of the innermost dimension of the index
     # (i.e., indices[i][-1]).
-- 
GitLab


From fcc79c953348a95c611ed8a05dcf27349597bf34 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Tue, 20 Nov 2018 15:23:38 -0800
Subject: [PATCH 0625/1554] Changing subgraphs to vector of unique_ptrs.

PiperOrigin-RevId: 222316187
---
 tensorflow/lite/core/subgraph.h | 3 ---
 tensorflow/lite/interpreter.cc  | 8 ++++----
 tensorflow/lite/interpreter.h   | 6 +++---
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index 120c307ef2..e85d6df974 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -57,7 +57,6 @@ class Subgraph {
   // interpreter.
   TfLiteStatus SetVariables(std::vector<int> variables);
 
-
   // Adds a node with the given parameters and returns the index of the new
   // node in `node_index` (optionally). Interpreter will take ownership of
   // `builtin_data` and destroy it with `free`. Ownership of 'init_data'
@@ -167,7 +166,6 @@ class Subgraph {
     return &nodes_and_registration_[node_index];
   }
 
-
   // Change the dimensionality of a given tensor. Note, this is only acceptable
   // for tensor indices that are inputs.
   // Returns status of failure or success.
@@ -227,7 +225,6 @@ class Subgraph {
     return kTfLiteOk;
   }
 
-
   // The default capacity of `tensors_` vector.
   static constexpr int kTensorsReservedCapacity = 128;
   // The capacity headroom of `tensors_` vector before calling ops'
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 078c2306c4..326aff5ce4 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -35,7 +35,7 @@ namespace tflite {
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  subgraphs_.emplace_back(error_reporter_, external_contexts_);
+  subgraphs_.emplace_back(new Subgraph(error_reporter_, external_contexts_));
   context_ = primary_subgraph().context();
 
   // Reserve some space for the tensors to avoid excessive resizing.
@@ -136,7 +136,7 @@ void Interpreter::UseNNAPI(bool enable) { primary_subgraph().UseNNAPI(enable); }
 
 void Interpreter::SetNumThreads(int num_threads) {
   for (auto& subgraph : subgraphs_) {
-    subgraph.context()->recommended_num_threads = num_threads;
+    subgraph->context()->recommended_num_threads = num_threads;
   }
 
   for (int i = 0; i < kTfLiteMaxExternalContexts; ++i) {
@@ -149,7 +149,7 @@ void Interpreter::SetNumThreads(int num_threads) {
 
 void Interpreter::SetAllowFp16PrecisionForFp32(bool allow) {
   for (auto& subgraph : subgraphs_) {
-    subgraph.context()->allow_fp32_relax_to_fp16 = allow;
+    subgraph->context()->allow_fp32_relax_to_fp16 = allow;
   }
 }
 
@@ -191,7 +191,7 @@ TfLiteStatus Interpreter::GetBufferHandle(int tensor_index,
 }
 
 void Interpreter::SetProfiler(profiling::Profiler* profiler) {
-  for (auto& subgraph : subgraphs_) subgraph.SetProfiler(profiler);
+  for (auto& subgraph : subgraphs_) subgraph->SetProfiler(profiler);
 }
 
 profiling::Profiler* Interpreter::GetProfiler() {
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index 7da4a2cbfa..405cf640b9 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -427,11 +427,11 @@ class Interpreter {
   friend class InterpreterTest;
 
   Subgraph& primary_subgraph() {
-    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
+    return *subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
   const Subgraph& primary_subgraph() const {
-    return subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
+    return *subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
   // Set the value of an external context.
@@ -470,7 +470,7 @@ class Interpreter {
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
 
   // Subgraphs
-  std::vector<Subgraph> subgraphs_;
+  std::vector<std::unique_ptr<Subgraph>> subgraphs_;
 };
 
 }  // namespace tflite
-- 
GitLab


From 095b54be3fa6dc206d7151d65c901e4273fd5b08 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 20 Nov 2018 15:38:53 -0800
Subject: [PATCH 0626/1554] PR #22788: TFTRT User provided INT8 quantization
 scales

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

Imported from GitHub PR #22788

TF-TRT now supports the following quantization nodes:
- QuantizeAndDequantizeV2
- QuantizeAndDequantizeV3
- FakeQuantWithMinMaxVars
- FakeQuantWithMinMaxArgs

When these nodes are converted:
1. Their quantization ranges are extracted and stored
2. The nodes are removed
3. The ranges are applied to the relevant tensors

This enables a path for TF-TRT to deploy models trained with quantization in the loop, for example those trained with tf.contrib.quantize.

trt.create_inference_graph() has a new boolean argument, `use_calibration`.
If we are in INT8 mode and use_calibration=True, create_inference_graph will return a calibration graph just like it did previously.
The calibrator will not override ranges provided via quantization nodes.
If we are in INT8 mode and use_calibration=False, a warning will be issued for every tensor which does not have a calibration range. Since TRT may fuse some operations, users may not always need to provide a value for these tensors. If a tensor that TRT needs is missing, the conversion will fail.

This PR also adds support for Relu6 nodes, and fixes a bug with not renaming tensors.

Current Issues/Considerations:
- If a model was trained with quantization nodes in places where TRT will not quantize (i.e. due to op fusion), then accuracy may drop dramatically. The solution is to figure out which ops will be fused by TRT and avoid placing quantization nodes between those ops. Another option is to place clip ops with each quantization node, so that if the tensor is not quantized it will still be clipped to be in the expected range - this will impact performace negatively.
- TRT does not have documentation for op fusion yet.
- TRT only supports symmetric quantization. Ranges are converted to symmetric ranges using `max(abs(min_range), abs(max_range))`.
- Since the tensors which are inputs and outputs to the TRTEngine are renamed to "TensorRTInputPH_{X}", "TensorRTOutputPH_{X}", if these tensors are missing ranges the warnings will not make sense to users.
- MatMul and BiasAdd are not fused, requiring users to provide a range between these ops.

Copybara import of the project:

  - 58f69a5899c8a902956b876c3fa8baab6e60e7c7 Allows user to specify int8 scales via TF nodes. by Trevor Morris <tmorris@nvidia.com>
  - abda4224e8d6b8e227a579a481e9f0ebe5fdb346 Add FakeQuantWithMinMaxArgs by Trevor Morris <tmorris@nvidia.com>
  - 6e07831307ebad7f154052cc387da22b6169a4d1 Add more checks on quantize node inputs. by Trevor Morris <tmorris@nvidia.com>
  - 9dda56272f87cc1ebd66eeaa6e8f3cf0421fe4fe Fix merge conflicts by Trevor Morris <tmorris@nvidia.com>
  - 08159b275a5e66e5713ab25befcd7b2034d631bb Improve use_calibration arg documentation by Trevor Morris <tmorris@nvidia.com>
  - f60e430827c759755922cdaea80ae6af954cb934 Quantization-aware training full test on MNIST by Trevor Morris <tmorris@nvidia.com>
  - de0ed5a2bf01342d4c23357d3abf39eb68bb0b2a Add quantization mnist test to BUILD by Trevor Morris <tmorris@nvidia.com>
  - da089db8e6c868f7fab11e45cbea182daccdb393 Fixes for @aaroey's review. by Trevor Morris <tmorris@nvidia.com>
  - 481c4904c6cdc129827c6b9c625cbd899fd3a555 Merge branch 'master' of https://github.com/tensorflow/te... by Guangda Lai <31743510+aaroey@users.noreply.github.com>
  - cdccbc859f1fc7e5041d61f378934ab1125e7ccc Fix some formatting/compilation issues by Guangda Lai <31743510+aaroey@users.noreply.github.com>
  - 218a89386fb8c00c0bf1de7f1ce6ec69a3991130 Merge cdccbc859f1fc7e5041d61f378934ab1125e7ccc into 38b3d... by Trevor Morris <trevoraidanmorris@gmail.com>

COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/tensorflow/pull/22788 from trevor-m:tmorris_tftrt_int8prequantize cdccbc859f1fc7e5041d61f378934ab1125e7ccc
PiperOrigin-RevId: 222318529
---
 tensorflow/contrib/tensorrt/BUILD             |  29 ++
 .../contrib/tensorrt/convert/convert_graph.cc | 144 +++---
 .../contrib/tensorrt/convert/convert_graph.h  |  10 +-
 .../tensorrt/convert/convert_graph_test.cc    |  43 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 420 +++++++++++++++++-
 .../contrib/tensorrt/convert/convert_nodes.h  |  58 ++-
 .../tensorrt/convert/convert_nodes_test.cc    | 293 +++++++++++-
 .../tensorrt/convert/trt_optimization_pass.cc |  10 +
 .../tensorrt/convert/trt_optimization_pass.h  |   4 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc |  14 +-
 .../contrib/tensorrt/kernels/trt_engine_op.h  |   4 +
 .../contrib/tensorrt/ops/trt_engine_op.cc     |   3 +-
 .../contrib/tensorrt/python/trt_convert.py    |  25 +-
 tensorflow/contrib/tensorrt/test/base_test.py |   4 +-
 .../tensorrt/test/quantization_mnist_test.py  | 290 ++++++++++++
 .../tensorrt/test/quantization_test.py        | 144 ++++++
 .../contrib/tensorrt/test/testdata/checkpoint |   3 +
 .../model.ckpt-46900.data-00000-of-00001      | Bin 0 -> 686728 bytes
 .../test/testdata/model.ckpt-46900.index      | Bin 0 -> 652 bytes
 .../test/tf_trt_integration_test_base.py      |  51 ++-
 .../grappler/optimizers/layout_optimizer.cc   |   4 +
 21 files changed, 1421 insertions(+), 132 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
 create mode 100644 tensorflow/contrib/tensorrt/test/quantization_test.py
 create mode 100644 tensorflow/contrib/tensorrt/test/testdata/checkpoint
 create mode 100644 tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001
 create mode 100644 tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 20bcd2447e..784acce444 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -29,6 +29,10 @@ load(
     "if_tensorrt",
 )
 
+exports_files(glob([
+    "test/testdata/*",
+]))
+
 tf_cuda_cc_test(
     name = "tensorrt_test_cc",
     size = "small",
@@ -491,6 +495,7 @@ cuda_py_tests(
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
+        "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
         "test/vgg_block_nchw_test.py",
@@ -527,6 +532,30 @@ cuda_py_tests(
     ],
 )
 
+cuda_py_test(
+    name = "quantization_mnist_test",
+    srcs = ["test/quantization_mnist_test.py"],
+    additional_deps = [
+        ":tf_trt_integration_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/keras:keras",
+        "//tensorflow/python/estimator:estimator",
+    ],
+    data = [
+        "test/testdata/checkpoint",
+        "test/testdata/model.ckpt-46900.data-00000-of-00001",
+        "test/testdata/model.ckpt-46900.index",
+    ],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_pip",
+        "no_tap",  # It is not able to download the mnist data.
+        "no_windows",
+        "nomac",
+    ],
+)
+
 cc_library(
     name = "utils",
     srcs = ["convert/utils.cc"],
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index f95ffe4100..21f505b7fe 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -82,60 +82,73 @@ std::vector<int> GetLoadedTensorRTVersion() {
 }
 
 TrtCandidateSelector::TrtCandidateSelector(
-    const grappler::GraphProperties& graph_properties)
-    : graph_properties_(graph_properties) {}
+    const grappler::GraphProperties& graph_properties, int precision_mode)
+    : graph_properties_(graph_properties), precision_mode_(precision_mode) {}
 
 Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
   // TODO(laigd): move this set to TrtNodeValidator where it should belong.
   // LINT.IfChange
   static const std::set<string> candidate_ops = {
-    "Identity",
-    "Snapshot",
-    "Const",
-    "Conv2D",
-    "MaxPool",
-    "BiasAdd",
-    "Relu",
-    "Add",
-    "Mul",
-    "Sub",
-    "Rsqrt",
-    "Pad",
-    "Mean",
-    "AvgPool",
-    "ConcatV2",
-    "DepthwiseConv2dNative",
-    "FusedBatchNorm",
-    "FusedBatchNormV2",
-    "Div",
-    "RealDiv",
-    "Rsqrt",
-    "Reciprocal",
-    "Exp",
-    "Log",
-    "Sqrt",
-    "Abs",
-    "Neg",
-    "Transpose",
-    "Reshape",
-    "MatMul",
-    "BatchMatMul",
-    "Softmax",
-    "Minimum",
-    "Maximum",
-    "TopKV2",
-    "Sum",
-    "Prod",
-    "Max",
-    "Min",
+      "Identity",
+      "Snapshot",
+      "Const",
+      "Conv2D",
+      "MaxPool",
+      "BiasAdd",
+      "Relu",
+      "Add",
+      "Mul",
+      "Sub",
+      "Rsqrt",
+      "Pad",
+      "Mean",
+      "AvgPool",
+      "ConcatV2",
+      "DepthwiseConv2dNative",
+      "FusedBatchNorm",
+      "FusedBatchNormV2",
+      "Div",
+      "RealDiv",
+      "Rsqrt",
+      "Reciprocal",
+      "Exp",
+      "Log",
+      "Sqrt",
+      "Abs",
+      "Neg",
+      "Transpose",
+      "Reshape",
+      "MatMul",
+      "BatchMatMul",
+      "Softmax",
+      "Minimum",
+      "Maximum",
+      "TopKV2",
+      "Sum",
+      "Prod",
+      "Max",
+      "Min",
+      "Relu6",
   };
-  // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc)
-  const bool is_supported_op_type =
+  bool is_supported_op_type =
       (candidate_ops.count(node->type_string()) ||
        PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string()));
+  static const std::set<string> quantize_ops = {
+      "QuantizeAndDequantizeV2",
+      "QuantizeAndDequantizeV3",
+      "FakeQuantWithMinMaxVars",
+      "FakeQuantWithMinMaxArgs",
+  };
+  // In INT8 mode, we will always apply the quantization ranges provided by
+  // these ops to the relevant tensors. This happens regardless of the value of
+  // use_calibration.
+  if (precision_mode_ == INT8MODE && quantize_ops.count(node->type_string())) {
+    is_supported_op_type = true;
+  }
+  // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc)
   if (!is_supported_op_type) {
     return errors::Unimplemented("Op type ", node->type_string(),
-                                 " is not supported.");
+                                 " is not supported");
   }
 
   std::vector<const Edge*> input_edges;
@@ -220,7 +233,8 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode, int minimum_segment_size, bool is_dyn_op,
-    int max_cached_engines, std::vector<int> cached_engine_batches) {
+    int max_cached_engines, std::vector<int> cached_engine_batches,
+    bool use_calibration) {
   // Create GrapplerItem.
   tensorflow::grappler::GrapplerItem item;
   item.fetch = output_names;
@@ -287,6 +301,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       list->add_i(batch);
     }
   }
+  parameters["use_calibration"].set_b(use_calibration);
 
   // Run optimizer.
   tensorflow::grappler::MetaOptimizer meta_opt(nullptr, config_proto);
@@ -566,27 +581,30 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
       }
     }
   }
+
+  const bool calibrate_int8 =
+      (info.precision_mode == INT8MODE && info.use_calibration);
+  // Build the engine and get its serialized representation.
   string segment_string;
-  if (info.engine_type == EngineInfo::EngineType::TRTStatic ||
-      info.precision_mode == INT8MODE) {
+  if (info.engine_type == EngineInfo::EngineType::TRTStatic || calibrate_int8) {
     // Create static engine for fp32/fp16 mode, and test validity of the engine
-    // for int8 mode. We don't want engine to fail at the calibration time.
-    // So we are constructing a FP32 engine here to check its validity, and if
-    // it is a valid engine then we put the serialized graphdef to the op.
-    // Otherwise we skip node creation for this engine.
+    // for int8 calibration mode. We don't want engine to fail at the
+    // calibration time. So we are constructing a FP32 engine here to check its
+    // validity, and if it is a valid engine then we put the serialized graphdef
+    // to the op. Otherwise we skip node creation for this engine.
     Logger trt_logger;
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     // TODO(sami): What happens if 1st dim is not batch?
     TF_RETURN_IF_ERROR(ConvertGraphDefToEngine(
-        info.segment_graph_def,
-        info.precision_mode == INT8MODE ? FP32MODE : info.precision_mode,
+        info.segment_graph_def, calibrate_int8 ? FP32MODE : info.precision_mode,
         max_batch_size, info.max_workspace_size_bytes, input_shapes,
         &trt_logger, alloc, /*calibrator=*/nullptr, &engine,
+        info.use_calibration,
         /*convert_successfully=*/nullptr));
     TrtUniquePtrType<nvinfer1::IHostMemory> engine_data(engine->serialize());
     segment_string =
         string((const char*)engine_data->data(), engine_data->size());
-    if (info.precision_mode == INT8MODE) {
+    if (calibrate_int8) {
       // See above comment about why not putting this inside the 'else' branch.
       segment_string = info.segment_graph_def.SerializeAsString();
     }
@@ -598,7 +616,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   // conversion.
   string prec_string;
   TF_RETURN_IF_ERROR(GetPrecisionModeName(info.precision_mode, &prec_string));
-  if (info.precision_mode == INT8MODE &&
+  if (info.precision_mode == INT8MODE && calibrate_int8 &&
       !TRTResourceManager::instance()->getManager("TRTCalibration")) {
     LOG(ERROR) << "Failed to construct calibration storage";
   }
@@ -634,6 +652,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
           .Attr("cached_engine_batches", {max_batch_size})
           .Attr("workspace_size_bytes", info.max_workspace_size_bytes)
           .Attr("precision_mode", prec_string)
+          .Attr("use_calibration", info.use_calibration)
           .Attr("OutT", out_types)
           .Finalize(&trt_node);
   if (!status.ok()) {
@@ -866,7 +885,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   }
   segment_options.minimum_segment_size = params.minimum_segment_size;
   tensorflow::tensorrt::segment::SegmentNodesVector initial_segments;
-  TrtCandidateSelector candidate_selector(*params.graph_properties);
+  TrtCandidateSelector candidate_selector(*params.graph_properties,
+                                          params.precision_mode);
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
       &graph,
       std::bind(&TrtCandidateSelector::IsTensorRTCandidate, &candidate_selector,
@@ -904,10 +924,14 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
       continue;
     }
     curr_engine.precision_mode = params.precision_mode;
-    curr_engine.engine_type =
-        (params.is_dyn_op || params.precision_mode == INT8MODE
-             ? EngineInfo::EngineType::TRTDynamic
-             : EngineInfo::EngineType::TRTStatic);
+    if (params.use_calibration && params.precision_mode != INT8MODE) {
+      return errors::InvalidArgument(
+          "Calibration with FP32 or FP16 is not supported.");
+    }
+    curr_engine.engine_type = ((params.is_dyn_op || params.use_calibration)
+                                   ? EngineInfo::EngineType::TRTDynamic
+                                   : EngineInfo::EngineType::TRTStatic);
+    curr_engine.use_calibration = params.use_calibration;
     curr_engine.cached_engine_batches = params.cached_engine_batches;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
     StrAppend(&curr_engine.engine_name, "my_trt_op_", t);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 1c9d82105a..1f39f56f63 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -35,7 +35,8 @@ namespace convert {
 // supported by TRT.
 class TrtCandidateSelector {
  public:
-  TrtCandidateSelector(const grappler::GraphProperties& graph_properties);
+  TrtCandidateSelector(const grappler::GraphProperties& graph_properties,
+                       int precision_mode);
 
   // Returns OK iff 'node' is a TF-TRT conversion candidate, which will be added
   // to TRT subgraph and later converted into TRT engine.
@@ -49,6 +50,9 @@ class TrtCandidateSelector {
   // GraphProperties of the graph whose nodes are to be validated by
   // IsTensorRTCandidate().
   const grappler::GraphProperties& graph_properties_;
+
+  // Quantization ops are only converted when using quantized precisions.
+  const int precision_mode_;
 };
 
 struct ConversionParams {
@@ -63,6 +67,7 @@ struct ConversionParams {
         cluster(nullptr),
         is_dyn_op(false),
         fixed_input_size(true),
+        use_calibration(true),
         max_cached_engines(1) {}
   const tensorflow::GraphDef* input_graph_def;
   const std::vector<string>* output_names;
@@ -76,6 +81,7 @@ struct ConversionParams {
   bool is_dyn_op;  //  Whether to create engine on conversion or execution time
   bool fixed_input_size;   // Assume non-batch ranks of input tensors are fixed
   int max_cached_engines;  // maximum number of cached engines
+  bool use_calibration;
   std::vector<int> cached_engine_batches;  // list of cached engines
 };
 
@@ -95,7 +101,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode = 1, int minimum_segment_size = 3,
     bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batches = {});
+    std::vector<int> cached_engine_batches = {}, bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
index f10729987f..2d2bfeb192 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
@@ -85,27 +85,42 @@ TEST(TrtCandidateSelector, Basics) {
       ops::MatMul(s.WithOpName("matmul_with_incompatible_input"),
                   incompatible_feed, const_2);
 
+  // Quantize ops.
+  auto quantize_attrs = ops::FakeQuantWithMinMaxArgs::Min(-6.0f).Max(6.0f);
+  auto quantize = ops::FakeQuantWithMinMaxArgs(s.WithOpName("quantize"), feed,
+                                               quantize_attrs);
+
+  // Get GrapplerItem and GraphProperties.
   grappler::GrapplerItem item;
   TF_EXPECT_OK(s.ToGraphDef(&item.graph));
   Tensor feed_tensor(DT_FLOAT, input_shape);
   item.feed.push_back(std::make_pair("feed", feed_tensor));
-
   grappler::GraphProperties graph_properties(item);
   TF_EXPECT_OK(graph_properties.InferStatically(true));
 
-  TrtCandidateSelector selector(graph_properties);
-  TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node()));
-  ExpectStatus(
-      selector.IsTensorRTCandidate(incompatible_matmul.operation.node()),
-      error::INVALID_ARGUMENT,
-      "transpose_a is not supported for TensorRT FullyConnected "
-      "(op: MatMul), at: incompatible_matmul");
-  ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node()),
-               error::UNIMPLEMENTED, "Op type Sin is not supported");
-  ExpectStatus(selector.IsTensorRTCandidate(
-                   matmul_with_incompatible_input.operation.node()),
-               error::INTERNAL,
-               "Failed to convert input with index 0 to a TRT_TensorOrWeights");
+  for (const int precision_mode : {FP32MODE, INT8MODE}) {
+    TrtCandidateSelector selector(graph_properties, precision_mode);
+    TF_EXPECT_OK(selector.IsTensorRTCandidate(matmul.operation.node()));
+    ExpectStatus(
+        selector.IsTensorRTCandidate(incompatible_matmul.operation.node()),
+        error::INVALID_ARGUMENT,
+        "transpose_a is not supported for TensorRT FullyConnected "
+        "(op: MatMul), at: incompatible_matmul");
+    ExpectStatus(selector.IsTensorRTCandidate(unsupported_op.operation.node()),
+                 error::UNIMPLEMENTED, "Op type Sin is not supported");
+    ExpectStatus(
+        selector.IsTensorRTCandidate(
+            matmul_with_incompatible_input.operation.node()),
+        error::INTERNAL,
+        "Failed to convert input with index 0 to a TRT_TensorOrWeights");
+    if (precision_mode == INT8MODE) {
+      TF_EXPECT_OK(selector.IsTensorRTCandidate(quantize.operation.node()));
+    } else {
+      ExpectStatus(selector.IsTensorRTCandidate(quantize.operation.node()),
+                   error::UNIMPLEMENTED,
+                   "Op type FakeQuantWithMinMaxArgs is not supported");
+    }
+  }
 }
 
 class FakeCluster : public grappler::Cluster {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index af9bbbfdfd..27c20204bb 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -449,7 +449,9 @@ class TRT_TensorOrWeights::SimpleITensor : public nvinfer1::ITensor {
   void setLocation(nvinfer1::TensorLocation location) override {}
 
 #if NV_TENSORRT_MAJOR >= 5
-  bool setDynamicRange(float min, float max) override {}
+  bool setDynamicRange(float min, float max) override { return true; }
+
+  float getDynamicRange() const override { return 0; }
 #endif
 
  private:
@@ -810,8 +812,11 @@ Status TrtNodeValidator::ConvertConstToWeights(
   return status;
 }
 
-Converter::Converter(nvinfer1::INetworkDefinition* trt_network, bool is_fp16)
-    : trt_network_(trt_network), is_fp16_(is_fp16) {
+Converter::Converter(nvinfer1::INetworkDefinition* trt_network,
+                     int precision_mode, bool use_calibration)
+    : trt_network_(trt_network),
+      precision_mode_(precision_mode),
+      use_calibration_(use_calibration) {
   this->RegisterOpConverters();
 }
 
@@ -836,13 +841,18 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
     TRT_TensorOrWeights& output = outputs[i];
     string output_name = node_def.name();
     if (i != 0) output_name = StrCat(output_name, ":", i);
-    // We need to check the name before setting it. For Identity op where the
-    // output is the input, if its input is one of the engine input, setting
-    // the name here will overwrite engine input bindings which will cause
-    // runtime error.
+    // We need to check the name before setting it. If the input is one of the
+    // engine input, setting the name here will overwrite engine input
+    // bindings which will cause runtime error.
     if (output.is_tensor()) {
       const char* tensor_name = output.tensor()->getName();
-      if (tensor_name == nullptr || std::strlen(tensor_name) == 0) {
+      if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) {
+        // TRT initializes tensor names as "(Unnamed ITensor* N)". We rename
+        // them to match their corresponding TensorFlow name.
+        // Note: ITensors that we create internally within TF-TRT which are
+        // not inputs or outputs of a node will not be renamed. This is a
+        // potential cause of confusion if an error message or warning
+        // mentions the unnamed tensor.
         output.tensor()->setName(output_name.c_str());
       }
     }
@@ -954,6 +964,7 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor,
 
   nvinfer1::IShuffleLayer* layer = this->network()->addShuffle(*input_tensor);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Transpose");
+  MarkQuantizationRangesAsInferrable(input_tensor, layer->getOutput(0));
 
   nvinfer1::Permutation permutation;
   for (int32_t i = 0; i < dims.nbDims; ++i) {
@@ -976,6 +987,38 @@ Status Converter::TransposeTensor(nvinfer1::ITensor* input_tensor,
   return tensorflow::Status::OK();
 }
 
+Status Converter::GetWeightRange(const TRT_ShapedWeights& weights,
+                                 float* out_min, float* out_max) const {
+  switch (weights.type_) {
+    case DataType::DT_FLOAT: {
+      auto inp = static_cast<float const*>(weights.GetValues());
+      auto result = std::minmax_element(inp, inp + weights.count());
+      *out_min = *result.first;
+      *out_max = *result.second;
+      break;
+    }
+    case DataType::DT_HALF: {
+      auto inp = static_cast<Eigen::half const*>(weights.GetValues());
+      auto result = std::minmax_element(inp, inp + weights.count());
+      *out_min = Eigen::half_impl::half_to_float(*result.first);
+      *out_max = Eigen::half_impl::half_to_float(*result.second);
+      break;
+    }
+    case DataType::DT_INT32: {
+      auto inp = static_cast<int const*>(weights.GetValues());
+      auto result = std::minmax_element(inp, inp + weights.count());
+      *out_min = static_cast<float>(*result.first);
+      *out_max = static_cast<float>(*result.second);
+      break;
+    }
+    default:
+      return errors::Unimplemented(
+          "Data type not supported for GetWeightRange: ",
+          DataTypeString(weights.type_));
+  }
+  return Status::OK();
+}
+
 Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
                                         const nvinfer1::Dims& dims,
                                         const nvinfer1::ITensor** tensor) {
@@ -1002,6 +1045,8 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
           *const_cast<nvinfer1::ITensor*>(input.tensor()));
       TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
       layer->setReshapeDimensions(dims);
+      MarkQuantizationRangesAsInferrable(
+          const_cast<nvinfer1::ITensor*>(input.tensor()), layer->getOutput(0));
       *tensor = layer->getOutput(0);
     }
   } else {
@@ -1009,10 +1054,123 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
         this->network()->addConstant(dims, input.weights().GetTrtWeights());
     TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
     *tensor = layer->getOutput(0);
+    if (precision_mode() == INT8MODE && !use_calibration()) {
+      // If we are in int8 mode and not calibrating, we need to explicitly set a
+      // quantization range for the output tensor of the IConstantLayer. Here we
+      // set the range to [min(weights), max(weights)].
+      float min_range = 0.0f;
+      float max_range = 0.0f;
+      TF_RETURN_IF_ERROR(
+          GetWeightRange(input.weights(), &min_range, &max_range));
+      // Avoid setting range to 0 because TRT will throw an error. If the
+      // weights are zero then the range doesn't matter: using 127.0f should
+      // ensure the quantized weight will be exactly zero.
+      if (min_range == 0.0f && max_range == 0.0f) {
+        min_range = -127.0f;
+        max_range = 127.0f;
+      }
+      ProvideQuantizationRange(const_cast<nvinfer1::ITensor*>(*tensor),
+                               min_range, max_range);
+    }
   }
   return tensorflow::Status::OK();
 }
 
+void Converter::MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input,
+                                                   nvinfer1::ITensor* output) {
+  quantization_infer_.push_back({input, output});
+  quantization_infer_.push_back({output, input});
+}
+
+void Converter::ProvideQuantizationRange(nvinfer1::ITensor* tensor,
+                                         float min_range, float max_range) {
+  float symmetric_range = std::max(std::abs(min_range), std::abs(max_range));
+  quantization_ranges_[tensor] = symmetric_range;
+}
+
+void Converter::MaybeApplyQuantizationRanges() {
+  if (precision_mode() != INT8MODE) return;
+
+  // Infer ranges across marked ops.
+  PropagateQuantizationRanges();
+  // Apply ranges.
+#if NV_TENSORRT_MAJOR >= 5
+  for (auto pair : quantization_ranges_) {
+    nvinfer1::ITensor* tensor = pair.first;
+    const float range = pair.second;
+    VLOG(1) << "Setting range for: " << tensor->getName() << ": " << range;
+    // TODO(laigd): if 'tensor' already has a range set which doesn't match
+    // 'range', it should report error.
+    tensor->setDynamicRange(-range, range);
+  }
+#endif
+
+  // Warn user about tensors that are missing ranges. If TRT fuses some layers
+  // then these tensors may not actually be required, which is why this is
+  // just a warning. If we are still missing ranges even after fusion,
+  // Builder::buildCudaEngine() will return nullptr and we will catch the
+  // error at that point.
+  if (!use_calibration()) {
+    // Get all tensors from network
+    std::set<nvinfer1::ITensor*> all_tensors;
+    for (int i = 0; i < this->network()->getNbLayers(); i++) {
+      nvinfer1::ILayer* layer = this->network()->getLayer(i);
+      for (int j = 0; j < layer->getNbInputs(); j++) {
+        all_tensors.insert(layer->getInput(j));
+      }
+      for (int j = 0; j < layer->getNbOutputs(); j++) {
+        all_tensors.insert(layer->getOutput(j));
+      }
+    }
+    // Find tensors with no ranges
+    for (auto tensor : all_tensors) {
+      if (!quantization_ranges_.count(tensor)) {
+        // Note: there may be some warnings for "(Unnamed ITensor* N)". These
+        // are tensors which are created internally by TF-TRT. The ranges for
+        // these unnamed ITensors are always inferred from user provided ranges,
+        // thus there will also be a warning for the range(s) the user missed.
+        LOG(WARNING) << "Quantization range was not found for "
+                     << tensor->getName() << ". "
+                     << "This is okay if TensorRT does not need the range "
+                     << "(e.g. due to node fusion).";
+      }
+    }
+  }
+}
+
+void Converter::PropagateQuantizationRanges() {
+  // Propagate ranges across edges in quantization_infer_ until no new
+  // information is added.
+  // Note: this function modifies quantization_infer_, it might be better to
+  // modify a copy instead if we for some reason need quantization_infer_
+  // later.
+  bool information_added = true;
+  while (information_added) {
+    information_added = false;
+    for (auto it = quantization_infer_.begin();
+         it != quantization_infer_.end();) {
+      auto input_tensor_range = quantization_ranges_.find(it->first);
+      auto output_tensor_range = quantization_ranges_.find(it->second);
+      if (input_tensor_range != quantization_ranges_.end() &&
+          output_tensor_range == quantization_ranges_.end()) {
+        // Input has range but output doesn't: copy range
+        // TODO(laigd): consider reporting error if it a different range is
+        // already set.
+        quantization_ranges_[it->second] = input_tensor_range->second;
+        information_added = true;
+        VLOG(1) << "Copy quantization range: " << it->first->getName() << " -> "
+                << it->second->getName();
+      }
+      // We can remove edges when the output range is known
+      if (quantization_ranges_.find(it->second) != quantization_ranges_.end()) {
+        it = quantization_infer_.erase(it);
+      } else {
+        ++it;
+      }
+    }
+  }
+}
+
 Status Converter::GetInputs(const tensorflow::NodeDef& node_def,
                             std::vector<TRT_TensorOrWeights>* inputs) const {
   for (auto const& input_name : node_def.input()) {
@@ -1247,7 +1405,7 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
     }
   }
 
-  if (params->converter->is_fp16()) {
+  if (params->converter->precision_mode() == FP16MODE) {
     weights = ConvertFP32ToFP16(params->weight_store, weights);
   }
 
@@ -1264,6 +1422,10 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
           *const_cast<nvinfer1::ITensor*>(tensor),
           nvinfer1::UnaryOperation::kNEG);
       TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+      // Since quantization ranges are symmetric, the same range as the input
+      // will work for the negation of the input.
+      params->converter->MarkQuantizationRangesAsInferrable(
+          const_cast<nvinfer1::ITensor*>(tensor), layer->getOutput(0));
       tensor = layer->getOutput(0);
     } else {
       TRT_ShapedWeights neg_weights =
@@ -1275,6 +1437,25 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
     }
   } else if (node_def.op() == "Div" || node_def.op() == "RealDiv") {
     if (swapped_inputs) {
+      // We need to infer the quantization range for this intermediate tensor.
+      //
+      //   x -> [Recip] -> 1/x -> [Scale] -> s/x
+      //                    ^
+      //            need range for this
+      //
+      // We have the quantization scales for x and s/x - can we divide the scale
+      // for s/x by s? Only if it is a scalar.
+      //
+      // Because of this issue, fall back to BinaryTensorOpTensor if we are
+      // doing INT8 with no calibration. There is most likely no performance
+      // penalty by falling back here.
+      if (params->converter->precision_mode() == INT8MODE &&
+          !params->converter->use_calibration()) {
+        return errors::Unimplemented(
+            "Intermediate quantization range cannot be determined without"
+            " calibration. Falling back to BinaryTensorOpTensor for ",
+            node_def.op(), ", at ", node_def.name());
+      }
       scale_weights = weights;
       nvinfer1::IUnaryLayer* layer = params->converter->network()->addUnary(
           *const_cast<nvinfer1::ITensor*>(tensor),
@@ -1350,7 +1531,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
     return tensorflow::errors::Internal(
         "Conv2D expects kernel of dimension 4, at: " + node_def.name());
   }
-  if (params->converter->is_fp16()) {
+  if (params->converter->precision_mode() == FP16MODE) {
     weights_rsck =
         ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
   }
@@ -1397,6 +1578,8 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
         nvinfer1::DimsHW(padding[0].second, padding[1].second));
     TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name());
+    params->converter->MarkQuantizationRangesAsInferrable(
+        const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
     VLOG(2) << "TENSOR after: " << DebugString(tensor->getDimensions());
@@ -1741,6 +1924,8 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
         nvinfer1::DimsHW(padding[0].second, padding[1].second));
     TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name());
+    params->converter->MarkQuantizationRangesAsInferrable(
+        const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
   }
@@ -1748,6 +1933,11 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
   nvinfer1::IPoolingLayer* layer = params->converter->network()->addPooling(
       *const_cast<nvinfer1::ITensor*>(tensor), type, ksize);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+  // TODO(tmorris): Average pooling may not be entirely safe to infer
+  // quantization range through (at least forwards - backwards should be fine).
+  // Max pooling is okay.
+  params->converter->MarkQuantizationRangesAsInferrable(
+      const_cast<nvinfer1::ITensor*>(tensor), layer->getOutput(0));
 
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
@@ -1776,6 +1966,148 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
+Status ConvertQuantize(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if ((inputs.size() == 0) ||
+      (node_def.op() == "FakeQuantWithMinMaxArgs" && inputs.size() != 1) ||
+      (node_def.op() == "FakeQuantWithMinMaxVars" && inputs.size() != 3) ||
+      (node_def.op() == "QuantizeAndDequantizeV2" && inputs.size() != 3) ||
+      (node_def.op() == "QuantizeAndDequantizeV3" && inputs.size() != 4)) {
+    return errors::InvalidArgument("Invalid number of inputs for ",
+                                   node_def.op(), ", at ", node_def.name());
+  }
+  if (inputs.at(0).is_weights()) {
+    // TensorRT will automatically quantize weights, so we will ignore ranges
+    // for weights.
+    params->outputs->push_back(inputs.at(0));
+    return Status::OK();
+  }
+  float min_range = 0.0f;
+  float max_range = 0.0f;
+  if (node_def.op() == "FakeQuantWithMinMaxArgs") {
+    // Get ranges via node attributes.
+    TFAttrs attrs(node_def);
+    if (attrs.count("min") == 0 || attrs.count("max") == 0) {
+      return errors::InvalidArgument("Min or max attribute not found for ",
+                                     node_def.op(), " at ", node_def.name());
+    }
+    min_range = attrs.get<float>("min");
+    max_range = attrs.get<float>("max");
+  } else if (node_def.op() == "FakeQuantWithMinMaxVars" ||
+             node_def.op() == "QuantizeAndDequantizeV2" ||
+             node_def.op() == "QuantizeAndDequantizeV3") {
+    // Get ranges via inputs.
+    if (!inputs.at(1).is_weights() || !inputs.at(2).is_weights()) {
+      return errors::InvalidArgument("Min and max inputs for ", node_def.op(),
+                                     " must be weights not tensors, at ",
+                                     node_def.name());
+    }
+    auto get_weights_value = [&inputs](int index) {
+      auto raw_weights = static_cast<float*>(
+          const_cast<void*>(inputs.at(index).weights().GetValues()));
+      return raw_weights[0];
+    };
+    min_range = get_weights_value(1);
+    max_range = get_weights_value(2);
+  } else {
+    return errors::InvalidArgument("Unknown quantization op ", node_def.op(),
+                                   ", at ", node_def.name());
+  }
+  if (params->validation_only) return Status::OK();
+
+  // Store ranges for tensor
+  params->converter->ProvideQuantizationRange(
+      const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor()), min_range,
+      max_range);
+  // Sometimes, TRT may not quantize a tensor, either because it chooses to
+  // execute a higher precision kernel or because of op fusion. In these cases,
+  // accuracy will suffer if the model was trained to expect quantization at
+  // that tensor. We should consider adding a clip(tensor, min_range, max_range)
+  // operation here to ensure that any arbitrarily placed quantize node will
+  // execute as expected. However, this will negatively affect performance. If
+  // users train their models in a way which models inference as close as
+  // possible (i.e. not quantizing in place where fusion will occur), then there
+  // is no problem with the current implementation.
+  params->outputs->push_back(inputs.at(0));
+  return Status::OK();
+}
+
+// TODO(pdavoodi): we should update relu6 implementation once TensorRT supports
+// Relu6 natively.
+tensorflow::Status ConvertRelu6(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        "Invalid number of inputs for Relu6, at ", node_def.name());
+  }
+  if (inputs.at(0).is_weights()) {
+    return tensorflow::errors::Unimplemented(
+        "Relu6 is only implemented for tensors, not weights, at ",
+        node_def.name());
+  }
+  if (params->validation_only) return Status::OK();
+  // ***************************************************************************
+  // TensorRT does not implement Relu6 natively. This function converts Relu6 op
+  // to available TensorRT ops: Relu6(x) = min(Relu(x), 6)
+  // ***************************************************************************
+
+  // Input Tensor
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+
+  // Relu operation i.e. Relu(x) = max(0, x)
+  nvinfer1::IActivationLayer* relu_layer =
+      params->converter->network()->addActivation(
+          *const_cast<nvinfer1::ITensor*>(tensor),
+          nvinfer1::ActivationType::kRELU);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(relu_layer, node_def.name());
+
+  // Large range of relu is problematic during quantization in INT8 precision
+  // mode. Setting dynamic range of relu = [0.f, 6.0f] helps with quantization.
+  // TRT only uses dynamic ranges in INT8 precision mode,
+  // and this does not affect the FP32 path.
+  params->converter->ProvideQuantizationRange(relu_layer->getOutput(0), 0.0f,
+                                              6.0f);
+
+  // Create a constant layer to store the floating point weight i.e. 6.0f This
+  // tensor will be broadcasted uniformly during elementwise `min` operation.
+  // The constant has to have the same rank as the input in order for TRT to
+  // broadcast
+  nvinfer1::Dims dims;
+  dims.nbDims = relu_layer->getOutput(0)->getDimensions().nbDims;
+  for (int i = 0; i < dims.nbDims; i++) {
+    dims.d[i] = 1;
+  }
+  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
+      tensorflow::DataType::DT_FLOAT, dims);
+  auto weights_ptr =
+      static_cast<float*>(const_cast<void*>(weights.GetValues()));
+  weights_ptr[0] = 6.0f;
+  nvinfer1::IConstantLayer* const6_layer =
+      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_layer, node_def.name());
+  params->converter->ProvideQuantizationRange(const6_layer->getOutput(0), 0.0f,
+                                              6.0f);
+
+  // ElementWise Min Operation
+  // Min op is a nop for INT8 execution path, as the input tensor
+  // to this layer will only have values in range [0.f, 6.0f].
+  const nvinfer1::ITensor* tensor_l = relu_layer->getOutput(0);
+  const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0);
+  nvinfer1::IElementWiseLayer* relu6_layer =
+      params->converter->network()->addElementWise(
+          *const_cast<nvinfer1::ITensor*>(tensor_l),
+          *const_cast<nvinfer1::ITensor*>(tensor_r),
+          nvinfer1::ElementWiseOperation::kMIN);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(relu6_layer, node_def.name());
+  nvinfer1::ITensor* output_tensor = relu6_layer->getOutput(0);
+  params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 6.0f);
+
+  params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return Status::OK();
+}
+
 tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
@@ -1786,7 +2118,8 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   }
   if (params->validation_only) return Status::OK();
 
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  nvinfer1::ITensor* tensor =
+      const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor());
   const nvinfer1::Dims original_dims = tensor->getDimensions();
   TFAttrs attrs(node_def);
   const string data_format = attrs.get<string>("data_format");
@@ -1811,9 +2144,11 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   // TODO(laigd): this doesn't match what the TRT doc says, fix the doc?
   if (channel_index != 0 || original_dims.nbDims != 3) {
     nvinfer1::IShuffleLayer* shuffle_layer =
-        params->converter->network()->addShuffle(
-            *const_cast<nvinfer1::ITensor*>(tensor));
+        params->converter->network()->addShuffle(*tensor);
     TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name());
+    params->converter->MarkQuantizationRangesAsInferrable(
+        tensor, shuffle_layer->getOutput(0));
+
     // NOTE(laigd): for some reason we need to apply the reshape
     // unconditionally. The default shape has nbDims==-1 and it seems the
     // behavior is undefined in some cases.
@@ -1832,7 +2167,7 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   }
 
   TRT_ShapedWeights weights = inputs.at(1).weights();
-  if (params->converter->is_fp16()) {
+  if (params->converter->precision_mode() == FP16MODE) {
     weights = ConvertFP32ToFP16(params->weight_store, weights);
   }
   nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL;
@@ -1842,8 +2177,8 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
 
   TRT_ShapedWeights empty_weights(weights.type_);
   nvinfer1::IScaleLayer* layer = params->converter->network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), mode, weights.GetTrtWeights(),
-      empty_weights.GetTrtWeights(), empty_weights.GetTrtWeights());
+      *tensor, mode, weights.GetTrtWeights(), empty_weights.GetTrtWeights(),
+      empty_weights.GetTrtWeights());
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
@@ -1867,6 +2202,8 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
     if (channel_index != 0) {
       shuffle_layer->setSecondTranspose(permutation);
     }
+    params->converter->MarkQuantizationRangesAsInferrable(
+        output_tensor, shuffle_layer->getOutput(0));
     output_tensor = shuffle_layer->getOutput(0);
   }
 
@@ -2025,6 +2362,9 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
 }
 
 tensorflow::Status ConvertIdentity(OpConverterParams* params) {
+  // TODO(tmorris): TRT's Identity layer does not get optimized away as of TRT
+  // 5.0, however once we know that it does it would be nice to use that
+  // instead.
   params->outputs->push_back(params->inputs.at(0));
   return tensorflow::Status::OK();
 }
@@ -2046,6 +2386,13 @@ tensorflow::Status ConvertBinary(OpConverterParams* params) {
         node_def.name());
   }
 
+  // TODO(tmorris): TRT plans to deprecate IScaleLayer and will replace it with
+  // IElementwiseLayer. At that point, we can remove BinaryTensorOpWeight. For
+  // now, the performance will be slightly better with IScaleLayer because it
+  // can be fused in more situations. However, most of the benefits of
+  // IScaleLayer are when the layer performs both a shift and a scale, which we
+  // don't do except for convolutions.
+  //
   // Try to convert into Scale layer first (for better performance)
   // Since scale layer supports restricted broadcast policy and op types, we
   // allow failure and try to handle it through Elementwise op
@@ -2088,6 +2435,20 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
 
   nvinfer1::IUnaryLayer* layer;
   if (node_def.op() == "Rsqrt") {
+    // We will need a quantization range for intermediate tensor if not using
+    // calibration.
+    //
+    //   x -> [Sqrt] -> sqrt(x) -> [Recip] -> 1/sqrt(x)
+    //                     ^
+    //               need range here
+    if (params->converter->precision_mode() == INT8MODE &&
+        !params->converter->use_calibration()) {
+      return errors::Unimplemented(
+          "Intermediate quantization range cannot be determined without"
+          " calibration for Rsqrt, consider replacing with "
+          "Sqrt -> FakeQuant -> Reciprocal ops, at ",
+          node_def.name());
+    }
     layer = params->converter->network()->addUnary(
         *const_cast<nvinfer1::ITensor*>(tensor),
         nvinfer1::UnaryOperation::kSQRT);
@@ -2647,6 +3008,8 @@ tensorflow::Status ConvertSoftmax(OpConverterParams* params) {
   layer->setAxes(1 << (nbDims - 1));
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  // Quantization range for SoftMax is always (0, 1)
+  params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f);
   params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
@@ -2694,6 +3057,13 @@ void TrtNodeValidator::RegisterOpValidators() {
   op_validators_["Transpose"] = ConvertTranspose;
   op_validators_["Reshape"] = ConvertReshape;
   op_validators_["MatMul"] = ConvertMatMul;
+
+  op_validators_["Relu6"] = ConvertRelu6;
+
+  op_validators_["QuantizeAndDequantizeV2"] = ConvertQuantize;
+  op_validators_["QuantizeAndDequantizeV3"] = ConvertQuantize;
+  op_validators_["FakeQuantWithMinMaxVars"] = ConvertQuantize;
+  op_validators_["FakeQuantWithMinMaxArgs"] = ConvertQuantize;
 }
 
 void Converter::RegisterOpConverters() {
@@ -2744,6 +3114,11 @@ void Converter::RegisterOpConverters() {
   op_registry_["MatMul"] = ConvertMatMul;
   op_registry_["BatchMatMul"] = ConvertBatchMatMul;
   op_registry_["TopKV2"] = ConvertTopK;
+  op_registry_["Relu6"] = ConvertRelu6;
+  op_registry_["QuantizeAndDequantizeV2"] = ConvertQuantize;
+  op_registry_["QuantizeAndDequantizeV3"] = ConvertQuantize;
+  op_registry_["FakeQuantWithMinMaxVars"] = ConvertQuantize;
+  op_registry_["FakeQuantWithMinMaxArgs"] = ConvertQuantize;
 
   plugin_converter_ = ConvertPlugin;
 }
@@ -2754,7 +3129,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     const std::vector<tensorflow::PartialTensorShape>& input_shapes,
     Logger* logger, nvinfer1::IGpuAllocator* allocator,
     TRTInt8Calibrator* calibrator,
-    TrtUniquePtrType<nvinfer1::ICudaEngine>* engine,
+    TrtUniquePtrType<nvinfer1::ICudaEngine>* engine, bool use_calibration,
     bool* convert_successfully) {
   engine->reset();
   if (convert_successfully) *convert_successfully = false;
@@ -2769,7 +3144,11 @@ tensorflow::Status ConvertGraphDefToEngine(
     builder->setHalf2Mode(true);
   } else if (precision_mode == INT8MODE) {
     builder->setInt8Mode(true);
-    builder->setInt8Calibrator(calibrator);
+    if (use_calibration) {
+      builder->setInt8Calibrator(calibrator);
+    } else {
+      builder->setInt8Calibrator(nullptr);
+    }
   }
 
   // Create the network.
@@ -2782,7 +3161,7 @@ tensorflow::Status ConvertGraphDefToEngine(
 
   // Build the network
   VLOG(1) << "Starting engine conversion ";
-  Converter converter(trt_network.get(), precision_mode == FP16MODE);
+  Converter converter(trt_network.get(), precision_mode, use_calibration);
   std::vector<std::pair<string, string>> output_tensors;
   // Graph nodes are already topologically sorted during construction
   for (const auto& node_def : gdef.node()) {
@@ -2838,6 +3217,9 @@ tensorflow::Status ConvertGraphDefToEngine(
   TF_RETURN_IF_ERROR(converter.RenameAndMarkOutputTensors(output_tensors));
   if (convert_successfully) *convert_successfully = true;
 
+  // Apply user provided quantization ranges to tensors
+  converter.MaybeApplyQuantizationRanges();
+
   // Build the engine.
   VLOG(1) << "Starting engine creation";
   engine->reset(builder->buildCudaEngine(*converter.network()));
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 5cc28b33e7..50100a142f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -92,7 +92,8 @@ struct EngineInfo {
   EngineInfo()
       : engine_type(EngineType::TRTStatic),
         max_workspace_size_bytes(0),
-        precision_mode(FP32MODE) {}
+        precision_mode(FP32MODE),
+        use_calibration(true) {}
 
   string engine_name;
   string device;
@@ -109,6 +110,7 @@ struct EngineInfo {
   int maximum_cached_engines;
   std::vector<int> cached_engine_batches;
   int precision_mode;
+  bool use_calibration;
 };
 
 // Constructs a graphdef from the segment in the given graph. Adds placeholder
@@ -145,7 +147,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     const std::vector<tensorflow::PartialTensorShape>& input_shapes,
     Logger* logger, nvinfer1::IGpuAllocator* allocator,
     TRTInt8Calibrator* calibrator,
-    TrtUniquePtrType<nvinfer1::ICudaEngine>* engine,
+    TrtUniquePtrType<nvinfer1::ICudaEngine>* engine, bool use_calibration,
     bool* convert_successfully);
 
 // Helper class for the segmenter to determine whether an output edge from the
@@ -392,7 +394,8 @@ class TrtNodeValidator {
 // Class to convert TF nodes to TRT network.
 class Converter {
  public:
-  Converter(nvinfer1::INetworkDefinition* trt_network, bool is_fp16);
+  Converter(nvinfer1::INetworkDefinition* trt_network, int precision_mode,
+            bool use_calibration);
 
   //////////////////////////////////////////////////////////////////////////////
   // Methods used by the TRT engine builder to build a TRT network from a TF
@@ -422,8 +425,27 @@ class Converter {
   // to add TRT layers.
   nvinfer1::INetworkDefinition* network() { return trt_network_; }
 
-  // Is the converter operating in fp16 mode?
-  bool is_fp16() const { return is_fp16_; }
+  // What precision are we targeting?
+  int precision_mode() const { return precision_mode_; }
+
+  // Calibration will be or was previously performed on this network?
+  bool use_calibration() const { return use_calibration_; }
+
+  // This should be called on the inputs and outputs of any layer we create
+  // where we know that the quantization range does not change during that
+  // operation. (e.g. Reshape, Transpose, Identity, MaxPool).
+  void MarkQuantizationRangesAsInferrable(nvinfer1::ITensor* input,
+                                          nvinfer1::ITensor* output);
+
+  // This function should be called when we know the quantization range of a
+  // tensor, either from a quantize/dequantize node or when the output is a
+  // fixed range (e.g. SoftMax, Relu6, Sigmoid).
+  void ProvideQuantizationRange(nvinfer1::ITensor* tensor, float min_range,
+                                float max_range);
+
+  // Should be called when full TRT network has been constructed and before
+  // building the engine.
+  void MaybeApplyQuantizationRanges();
 
   // Below are helper methods for op converters to add different layers to the
   // TRT network.
@@ -457,6 +479,12 @@ class Converter {
 
   void RegisterOpConverters();
 
+  void PropagateQuantizationRanges();
+
+  // Gets the min and max value in a TRT_ShapedWeights
+  Status GetWeightRange(const TRT_ShapedWeights& weights, float* out_min,
+                        float* out_max) const;
+
   // Registered op converters by op type.
   std::unordered_map<string, OpConverter> op_registry_;
 
@@ -472,7 +500,25 @@ class Converter {
   // Store the weights added during construction of trt_network_.
   TrtWeightStore weight_store_;
 
-  const bool is_fp16_;
+  // During conversion, this table is populated with quantization ranges per
+  // tensor. MaybeApplyQuantizationRanges() will use this table to set the TRT
+  // quantization ranges. Since TRT only supports symmetric ranges, we will
+  // store the range as a single float = max(abs(min_range), abs(max_range)).
+  // Range refers to the floating point values, e.g. min_range = 0.0f, max_range
+  // = 6.0f for Relu6.
+  std::unordered_map<nvinfer1::ITensor*, float> quantization_ranges_;
+
+  // Edges where quantization ranges can be inferred (copied) across ops - from
+  // first tensor to second tensor. PropagateQuantizationRanges() will propagate
+  // known ranges from quantization_ranges_ across these edges, adding the new
+  // ranges to quantization_ranges_ so that they can be applied in
+  // MaybeApplyQuantizationRanges().
+  std::vector<std::pair<nvinfer1::ITensor*, nvinfer1::ITensor*>>
+      quantization_infer_;
+
+  const int precision_mode_;
+
+  const bool use_calibration_;
 
   // Batch size of inputs to trt_network_ added by AddInputTensor(). During
   // network construction it will update this, use it to verify the batch
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 862754f3d2..b46a7c6c8a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/protobuf/config.pb.h"  // NOLINT
 #include "tensorflow/core/public/session.h"
@@ -49,6 +50,7 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
+using ::tensorflow::strings::StrCat;
 using ::testing::ElementsAre;
 
 // TODO(laigd): put this into some test utils file.
@@ -135,11 +137,12 @@ void ValidateWeights(const TRT_ShapedWeights& weights,
 // Fake ITensor implementation for testing purposes.
 class FakeITensor : public nvinfer1::ITensor {
  public:
-  FakeITensor() {}
+  FakeITensor() : dynamic_range_(0.0f) {}
 
-  FakeITensor(const nvinfer1::Dims& dims) : dims_(dims) {}
+  FakeITensor(const nvinfer1::Dims& dims) : dims_(dims), dynamic_range_(0.0f) {}
 
-  FakeITensor(const std::vector<int>& dims) : dims_(GetTestDims(dims)) {}
+  FakeITensor(const std::vector<int>& dims)
+      : dims_(GetTestDims(dims)), dynamic_range_(0.0f) {}
 
   void setName(const char* name) override { name_ = name; }
 
@@ -168,7 +171,12 @@ class FakeITensor : public nvinfer1::ITensor {
   }
 
 #if NV_TENSORRT_MAJOR >= 5
-  bool setDynamicRange(float min, float max) override {}
+  bool setDynamicRange(float min, float max) override {
+    dynamic_range_ = std::max(std::abs(min), std::abs(max));
+    return true;
+  }
+
+  float getDynamicRange() const override { return dynamic_range_; }
 #endif
 
  private:
@@ -176,6 +184,7 @@ class FakeITensor : public nvinfer1::ITensor {
   nvinfer1::Dims dims_;
   nvinfer1::DataType type_;
   nvinfer1::TensorLocation location_;
+  float dynamic_range_;
 };
 
 TEST(TRT_ShapedWeights_Test, Basic) {
@@ -425,7 +434,9 @@ class ConverterTest : public ::testing::Test {
   ConverterTest() {
     builder_.reset(nvinfer1::createInferBuilder(logger_));
     network_.reset(builder_->createNetwork());
-    converter_.reset(new Converter(network_.get(), /*fp16=*/false));
+    converter_.reset(new Converter(network_.get(),
+                                   /*precision_mode=*/FP32MODE,
+                                   /*use_calibration=*/false));
     weight_store_ = &converter_->weight_store_;
   }
 
@@ -452,8 +463,21 @@ class ConverterTest : public ::testing::Test {
     return converter_->GetInputs(node_def, inputs);
   }
 
+  Status GetWeightRange(const TRT_ShapedWeights& weights, float* out_min,
+                        float* out_max) const {
+    return converter_->GetWeightRange(weights, out_min, out_max);
+  }
+
+  void PropagateQuantizationRanges() {
+    converter_->PropagateQuantizationRanges();
+  }
+
   int batch_size() const { return converter_->batch_size_; }
 
+  std::unordered_map<nvinfer1::ITensor*, float>& quantization_ranges() {
+    return converter_->quantization_ranges_;
+  }
+
  private:
   Logger logger_;
   // These members are ordered in a way such that the destruction order is:
@@ -676,6 +700,88 @@ TEST_F(ConverterTest, AddAndGetTensorOrWeights) {
                "tensor/weights my_tensor already exist");
 }
 
+template <typename T>
+void TestGetWeightRange(ConverterTest* test, TrtWeightStore* weight_store) {
+  TRT_ShapedWeights weights =
+      weight_store->GetTempWeights(DataTypeToEnum<T>::v(), GetTestDims({2, 3}));
+  const std::vector<T> values = {T(3), T(1), T(2), T(6), T(5), T(4)};
+  memcpy(const_cast<void*>(weights.GetValues()), values.data(),
+         weights.size_bytes());
+
+  float out_min = 0.0f;
+  float out_max = 0.0f;
+  TF_EXPECT_OK(test->GetWeightRange(weights, &out_min, &out_max));
+  EXPECT_EQ(1.0f, out_min);
+  EXPECT_EQ(6.0f, out_max);
+}
+
+TEST_F(ConverterTest, GetWeightRange) {
+  TestGetWeightRange<float>(this, weight_store_);
+  TestGetWeightRange<Eigen::half>(this, weight_store_);
+  TestGetWeightRange<int32>(this, weight_store_);
+}
+
+TEST_F(ConverterTest, ProvideQuantizationRange) {
+  FakeITensor fake_tensor;
+  // Assymetric range
+  converter_->ProvideQuantizationRange(&fake_tensor, 0.0f, 6.0f);
+  EXPECT_EQ(6.0f, quantization_ranges()[&fake_tensor]);
+  converter_->ProvideQuantizationRange(&fake_tensor, 1.0f, 6.0f);
+  EXPECT_EQ(6.0f, quantization_ranges()[&fake_tensor]);
+  converter_->ProvideQuantizationRange(&fake_tensor, -8.0f, 6.0f);
+  EXPECT_EQ(8.0f, quantization_ranges()[&fake_tensor]);
+  converter_->ProvideQuantizationRange(&fake_tensor, -8.123f, -6.123f);
+  EXPECT_EQ(8.123f, quantization_ranges()[&fake_tensor]);
+  // Symmetric range
+  converter_->ProvideQuantizationRange(&fake_tensor, -6.123f, 6.123f);
+  EXPECT_EQ(6.123f, quantization_ranges()[&fake_tensor]);
+}
+
+TEST_F(ConverterTest, MaybeApplyQuantizationRanges) {
+  // input -> infer1 -> infer2 -> infer3
+  FakeITensor input, infer_1, infer_2, infer_3;
+  FakeITensor not_infer;
+  Converter int8_converter(/*trt_network=*/nullptr, INT8MODE,
+                           /*use_calibration=*/true);
+  int8_converter.ProvideQuantizationRange(&input, -5.0f, 5.0f);
+  int8_converter.ProvideQuantizationRange(&not_infer, -100.0f, 100.0f);
+  int8_converter.MarkQuantizationRangesAsInferrable(&input, &infer_1);
+  int8_converter.MarkQuantizationRangesAsInferrable(&infer_1, &infer_2);
+  int8_converter.MarkQuantizationRangesAsInferrable(&infer_2, &infer_3);
+
+  // Input range should be inferred along the chain and applied to tensors.
+  int8_converter.MaybeApplyQuantizationRanges();
+#if NV_TENSORRT_MAJOR >= 5
+  EXPECT_EQ(input.getDynamicRange(), 5.0f);
+  EXPECT_EQ(infer_1.getDynamicRange(), 5.0f);
+  EXPECT_EQ(infer_2.getDynamicRange(), 5.0f);
+  EXPECT_EQ(infer_3.getDynamicRange(), 5.0f);
+  EXPECT_EQ(not_infer.getDynamicRange(), 100.0f);
+#endif
+}
+
+TEST_F(ConverterTest, PropagateQuantizationRanges) {
+  // infer0 <-> infer1 <-> infer2 <-> infer3
+  //              |
+  //            infer4 <-> infer5
+  FakeITensor infer[6];
+  FakeITensor not_infer;
+  converter_->ProvideQuantizationRange(&infer[4], -5.0f, 5.0f);
+  converter_->MarkQuantizationRangesAsInferrable(&infer[0], &infer[1]);
+  converter_->MarkQuantizationRangesAsInferrable(&infer[1], &infer[2]);
+  converter_->MarkQuantizationRangesAsInferrable(&infer[3], &infer[2]);
+  converter_->MarkQuantizationRangesAsInferrable(&infer[4], &infer[1]);
+  converter_->MarkQuantizationRangesAsInferrable(&infer[4], &infer[5]);
+
+  // Input range should be inferred along the chain.
+  PropagateQuantizationRanges();
+  auto ranges = quantization_ranges();
+  for (int i = 0; i < 6; ++i) {
+    EXPECT_EQ(5.0f, ranges[&infer[i]]);
+  }
+  EXPECT_EQ(ranges.count(&not_infer), 0);
+}
+
 // Class to test various op converters, using both a TrtNodeValidator and
 // Converter.
 class OpConverterTest : public ::testing::Test {
@@ -704,7 +810,9 @@ class OpConverterTest : public ::testing::Test {
 
     // Reset the validator and converter.
     validator_.reset(new TrtNodeValidator);
-    converter_.reset(new Converter(network_.get(), /*fp16=*/false));
+    converter_.reset(new Converter(network_.get(),
+                                   /*precision_mode=*/FP32MODE,
+                                   /*use_calibration=*/false));
 
     // Reset other related artifacts.
     scope_ = Scope::NewRootScope();
@@ -847,6 +955,11 @@ class OpConverterTest : public ::testing::Test {
     }
   }
 
+  // Expose quantization_ranges_ for tests
+  std::unordered_map<nvinfer1::ITensor*, float>& quantization_ranges() {
+    return converter_->quantization_ranges_;
+  }
+
   std::unique_ptr<Converter> converter_;
   std::unique_ptr<TrtNodeValidator> validator_;
 
@@ -856,6 +969,11 @@ class OpConverterTest : public ::testing::Test {
   TrtUniquePtrType<nvinfer1::INetworkDefinition> network_;
   TrtUniquePtrType<nvinfer1::ICudaEngine> engine_;
   cudaStream_t stream_;
+  // Used to create placeholders with shape and data type information. The
+  // created placeholders will be used as inputs to the node to be verified,
+  // thus we need the shape and data type information to get a non-empty
+  // GraphProperties.
+  // TODO(laigd): consider use this Scope to create the NodeDef to verify.
   Scope scope_;
   std::unordered_map<string, NodeDef> validator_inputs_;
 };
@@ -1233,6 +1351,169 @@ TEST_F(OpConverterTest, ConvertBiasAdd) {
   // TestConvertBiasAdd<DT_HALF>(this);
 }
 
+TEST_F(OpConverterTest, ConvertQuantize) {
+  for (const string& op :
+       {"FakeQuantWithMinMaxArgs", "FakeQuantWithMinMaxVars",
+        "QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3"}) {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_quantize", op, {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        StrCat("Invalid number of inputs for ", op, ", at my_quantize")
+            .c_str());
+  }
+  {
+    // FakeQuantWithMinMaxArgs attributes are empty, should fail.
+    NodeDef node_def =
+        MakeNodeDef("my_quantize", "FakeQuantWithMinMaxArgs", {"input"});
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Min or max attribute not found for FakeQuantWithMinMaxArgs "
+        "at my_quantize");
+  }
+  {
+    // FakeQuantWithMinMaxArgs ranges set via attributes, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto quantize_attrs = ops::FakeQuantWithMinMaxArgs::Min(-6.0f).Max(6.0f);
+    auto quantize = ops::FakeQuantWithMinMaxArgs(s.WithOpName("my_quantize"),
+                                                 input, quantize_attrs);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(1, ranges.count(output.tensor()));
+    EXPECT_EQ(6.0f, ranges[output.tensor()]);
+  }
+  {
+    // FakeQuantWithMinMaxVars ranges set via inputs, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto quantize = ops::FakeQuantWithMinMaxVars(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights_min", {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", {1}, {6.0f});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(1, ranges.count(output.tensor()));
+    EXPECT_EQ(6.0f, ranges[output.tensor()]);
+  }
+  {
+    // QuantizeAndDequantizeV2 ranges set via inputs, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto quantize = ops::QuantizeAndDequantizeV2(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights_min", {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", {1}, {6.0f});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(1, ranges.count(output.tensor()));
+    EXPECT_EQ(6.0f, ranges[output.tensor()]);
+  }
+  {
+    // QuantizeAndDequantizeV2 Range inputs are tensors, should fail.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto quantize = ops::QuantizeAndDequantizeV2(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestTensor("weights_min", {1});
+    AddTestTensor("weights_max", {1});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Min and max inputs for QuantizeAndDequantizeV2 must be weights not "
+        "tensors, at my_quantize");
+  }
+  {
+    // QuantizeAndDequantizeV3 ranges set via inputs, ok.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto weights_min = ops::Placeholder(s.WithOpName("weights_min"), DT_FLOAT);
+    auto weights_max = ops::Placeholder(s.WithOpName("weights_max"), DT_FLOAT);
+    auto num_bits = ops::Placeholder(s.WithOpName("num_bits"), DT_INT32);
+    auto quantize = ops::QuantizeAndDequantizeV3(
+        s.WithOpName("my_quantize"), input, weights_min, weights_max, num_bits);
+    const NodeDef& node_def = quantize.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights_min", {1}, {-6.0f});
+    AddTestWeights<float>("weights_max", {1}, {6.0f});
+    AddTestWeights<int>("num_bits", {1}, {8});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_quantize", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(1, ranges.count(output.tensor()));
+    EXPECT_EQ(6.0f, ranges[output.tensor()]);
+  }
+}
+
+TEST_F(OpConverterTest, ConvertRelu6) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_relu6", "Relu6", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Invalid number of inputs for Relu6, at my_relu6");
+  }
+
+  // Get the NodeDef for Relu6.
+  Scope s = Scope::NewRootScope();
+  auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+  auto relu6 = ops::Relu6(s.WithOpName("my_relu6"), input);
+  const NodeDef node_def = relu6.operation.node()->def();
+  {
+    // Input is weights, should fail.
+    Reset();
+    AddTestWeights<float>("input", {1}, {1.0f});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Relu6 is only implemented for tensors, not weights, at my_relu6");
+  }
+  {
+    // Clip tensor values and set quantization ranges, ok.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_relu6", &output));
+    EXPECT_TRUE(output.is_tensor());
+    auto ranges = quantization_ranges();
+    EXPECT_EQ(ranges[output.tensor()], 6.0f);
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {-100, -1, 0, 3, 5, 9}, "my_relu6", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(0, 0, 0, 3, 5, 6));
+  }
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
index b30d94b028..4ac7e21d34 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
@@ -67,6 +67,9 @@ tensorflow::Status TRTOptimizationPass::Init(
     TF_RETURN_IF_ERROR(GetPrecisionMode(
         Uppercase(params.at("precision_mode").s()), &precision_mode_));
   }
+  if (params.count("use_calibration")) {
+    use_calibration_ = params.at("use_calibration").b();
+  }
   return tensorflow::Status::OK();
 }
 
@@ -222,6 +225,12 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(true));
   tensorflow::tensorrt::convert::ConversionParams cp;
 
+  if (use_calibration_ && precision_mode_ != INT8MODE) {
+    LOG(ERROR) << "Calibration with FP32 or FP16 is not implemented. "
+               << "Falling back to use_calibration = False.";
+    use_calibration_ = false;
+  }
+
   std::vector<string> nodes_to_preserve;
   for (const auto& n : item.NodesToPreserve()) {
     auto tokens = str_util::Split(n, ":");
@@ -250,6 +259,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   cp.is_dyn_op = is_dynamic_op_;
   cp.cached_engine_batches = batches_;
   cp.max_cached_engines = max_cached_batches_;
+  cp.use_calibration = use_calibration_;
   auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp);
   VLOG(1) << "Returning from " << name_;
   return status;
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
index 71b51d1368..3e8dc0978e 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
@@ -38,7 +38,8 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
         maximum_batch_size_(-1),
         is_dynamic_op_(false),
         max_cached_batches_(1),
-        max_workspace_size_bytes_(256LL << 20) {
+        max_workspace_size_bytes_(256LL << 20),
+        use_calibration_(true) {
     VLOG(1) << "Constructing " << name_;
   }
 
@@ -67,6 +68,7 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
   std::vector<int> batches_;
   int max_cached_batches_;
   int64_t max_workspace_size_bytes_;
+  bool use_calibration_;
 };
 
 }  // namespace convert
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 019446813a..1e907e0d2a 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -124,8 +124,10 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
   OP_REQUIRES_OK(context,
                  context->GetAttr("segment_funcdef_name", &funcdef_name_));
   OP_REQUIRES_OK(context, GetPrecisionMode(precision_string, &precision_mode_));
-  calibration_mode_ =
-      (precision_mode_ == INT8MODE && calibration_data.size() == 0);
+  OP_REQUIRES_OK(context,
+                 context->GetAttr("use_calibration", &use_calibration_));
+  calibration_mode_ = (use_calibration_ && precision_mode_ == INT8MODE &&
+                       calibration_data.size() == 0);
   if (calibration_data.size()) {
     calibrator_.reset(new TRTInt8Calibrator(calibration_data));
     calibration_data.resize(0);
@@ -308,7 +310,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
   std::vector<void*> buffers(num_binding);
   for (int i = 0; i < ctx->num_inputs(); i++) {
     const string input_name = StrCat(kInputPHName, i);
-    const size_t binding_index =
+    const int binding_index =
         trt_engine_ptr->getBindingIndex(input_name.c_str());
     if (binding_index == -1) {
       LOG(ERROR) << "Input node not found, at " << input_name;
@@ -345,7 +347,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
   for (int i = 0; i < ctx->num_outputs(); i++) {
     // Create an output tensor
     const string output_name = StrCat(kOutputPHName, i);
-    const size_t binding_index =
+    const int binding_index =
         trt_engine_ptr->getBindingIndex(output_name.c_str());
     Tensor* output_tensor = nullptr;
 
@@ -497,7 +499,8 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
         segment_graph_, precision_mode_, batch_size, workspace_size_, shapes,
-        &logger, allocator, calibrator_.get(), &engine, &convert_successfully);
+        &logger, allocator, calibrator_.get(), &engine, use_calibration_,
+        &convert_successfully);
     if (!status.ok()) {
       if (convert_successfully) {
         // This means it fail to build the engine even when the network is built
@@ -586,6 +589,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
         *segment_graph, INT8MODE, cres->calibrator_->getBatchSize(),
         workspace_size_bytes, shapes, &cres->logger_, cres->allocator_.get(),
         cres->calibrator_.get(), &cres->engine_,
+        /*use_calibration=*/true,
         /*convert_successfully=*/nullptr);
     if (!s.ok()) {
       LOG(ERROR) << "Calibration failed: " << s;
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 8fe0675891..b545f497f3 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -130,6 +130,10 @@ class TRTEngineOp : public AsyncOpKernel {
 
   // The finalized calibrator for inference.
   std::unique_ptr<TRTInt8Calibrator> calibrator_;
+
+  // If true, create calibration graph for INT8 mode. Otherwise, we are using
+  // user-provided quantization ranges.
+  bool use_calibration_;
 };
 
 }  // namespace tensorrt
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index e0c7b62723..ce04e5806e 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -39,8 +39,9 @@ REGISTER_OP("TRTEngineOp")
     .Attr("cached_engine_batches: list(int) = []")
     .Attr("max_cached_engines_count: int = 1")
     .Attr("workspace_size_bytes: int")
-    .Attr("precision_mode: {'FP32', 'FP16', 'INT8', 'INT8CALIB'}")
+    .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}")
     .Attr("calibration_data: string = ''")
+    .Attr("use_calibration: bool = true")
     .Input("in_tensor: InT")
     .Output("out_tensor: OutT");
 // TODO(jie): TF requires concrete output shape for concrete input shapes.
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 0e59fdd1fe..74a2c2392a 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -70,7 +70,8 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
                                  minimum_segment_size=3,
                                  is_dynamic_op=False,
                                  maximum_cached_engines=1,
-                                 cached_engine_batch_sizes=None):
+                                 cached_engine_batch_sizes=None,
+                                 use_calibration=True):
   """Returns a RewriterConfig proto for TRT transformation.
 
   Args:
@@ -95,6 +96,15 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
+    use_calibration: this argument is ignored if precision_mode is not INT8. if
+      set to True, a calibration graph will be created to calibrate the missing
+      ranges. The calibration graph must be converted to an inference graph
+      using calib_graph_to_infer_graph() after running calibration. if set to
+      False, quantization nodes will be expected for every tensor in the graph
+      (exlcuding those which will be fused). If a range is missing, an error
+      will occur. Please note that accuracy may be negatively affected if there
+      is a mismatch between which tensors TRT quantizes and which tensors were
+      trained with fake quantization.
 
   Returns:
     A RewriterConfig proto which sets a TensorRTOptimizer to run Grappler.
@@ -141,6 +151,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
                        "maximum_cached_engines items.")
     optimizer.parameter_map["cached_engine_batches"].list.i.extend(
         cached_engine_batch_sizes)
+  optimizer.parameter_map["use_calibration"].b = use_calibration
   return rewriter_config_with_trt
 
 
@@ -153,6 +164,7 @@ def create_inference_graph(input_graph_def,
                            is_dynamic_op=False,
                            maximum_cached_engines=1,
                            cached_engine_batch_sizes=None,
+                           use_calibration=True,
                            input_saved_model_dir=None,
                            input_saved_model_tags=None,
                            output_saved_model_dir=None,
@@ -184,6 +196,15 @@ def create_inference_graph(input_graph_def,
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
+    use_calibration: this argument is ignored if precision_mode is not INT8. if
+      set to True, a calibration graph will be created to calibrate the missing
+      ranges. The calibration graph must be converted to an inference graph
+      using calib_graph_to_infer_graph() after running calibration. if set to
+      False, quantization nodes will be expected for every tensor in the graph
+      (exlcuding those which will be fused). If a range is missing, an error
+      will occur. Please note that accuracy may be negatively affected if there
+      is a mismatch between which tensors TRT quantizes and which tensors were
+      trained with fake quantization.
     input_saved_model_dir: the directory to load the SavedModel which contains
       the input graph to transforms. Used only when input_graph_def is None.
     input_saved_model_tags: list of tags to load the SavedModel.
@@ -333,7 +354,7 @@ def create_inference_graph(input_graph_def,
   rewriter_config_with_trt = get_tensorrt_rewriter_config(
       rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
       minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batch_sizes)
+      cached_engine_batch_sizes, use_calibration)
   session_config_with_trt.graph_options.rewrite_options.CopyFrom(
       rewriter_config_with_trt)
 
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 18096e0ff1..7e826fb7e1 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -197,7 +197,9 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     """Whether to run the test."""
     # Disable the test in fp16 mode since multiple matmul and add ops together
     # can cause overflow.
-    return run_params.precision_mode != "FP16"
+    return ((run_params.precision_mode != "FP16") and
+            not (trt_test.IsQuantizationMode(run_params.precision_mode) and
+                 not run_params.use_calibration))
 
 
 class PartiallyConvertedTestB(PartiallyConvertedTestA):
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
new file mode 100644
index 0000000000..e7d6ec4ad3
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -0,0 +1,290 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Script to test TF-TRT INT8 conversion without calibration on Mnist model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.tensorrt.python import trt_convert
+# pylint: disable=unused-import
+from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+# pylint: enable=unused-import
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python import data
+from tensorflow.python import keras
+from tensorflow.python.estimator.estimator import Estimator
+from tensorflow.python.estimator.model_fn import EstimatorSpec
+from tensorflow.python.estimator.model_fn import ModeKeys
+from tensorflow.python.estimator.run_config import RunConfig
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import graph_util
+from tensorflow.python.framework import importer
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.datasets import mnist
+from tensorflow.python.layers import layers
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import metrics
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.losses import losses
+from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.summary import summary
+from tensorflow.python.training import saver
+from tensorflow.python.training.adam import AdamOptimizer
+from tensorflow.python.training.checkpoint_management import latest_checkpoint
+from tensorflow.python.training.training_util import get_global_step
+
+INPUT_NODE_NAME = 'input'
+OUTPUT_NODE_NAME = 'output'
+
+
+class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
+
+  def _BuildGraph(self, x):
+
+    def _Quantize(x, r):
+      x = gen_array_ops.quantize_and_dequantize_v2(x, -r, r)
+      return x
+
+    def _DenseLayer(x, num_inputs, num_outputs, quantization_range, name):
+      """Dense layer with quantized outputs.
+
+      Args:
+        x: input to the dense layer
+        num_inputs: number of input columns of x
+        num_outputs: number of output columns
+        quantization_range: the min/max range for quantization
+        name: name of the variable scope
+
+      Returns:
+        The output of the layer.
+      """
+      with variable_scope.variable_scope(name):
+        kernel = variable_scope.get_variable(
+            'kernel',
+            shape=[num_inputs, num_outputs],
+            dtype=dtypes.float32,
+            initializer=keras.initializers.glorot_uniform())
+        bias = variable_scope.get_variable(
+            'bias',
+            shape=[num_outputs],
+            dtype=dtypes.float32,
+            initializer=keras.initializers.zeros())
+        x = math_ops.matmul(x, kernel)
+        x = _Quantize(x, quantization_range)
+        x = nn.bias_add(x, bias)
+        x = _Quantize(x, quantization_range)
+      return x
+
+    x = _Quantize(x, 1)
+    # Conv + Bias + Relu6
+    x = layers.conv2d(x, filters=32, kernel_size=3, use_bias=True)
+    x = nn.relu6(x)
+    # Conv + Bias + Relu6
+    x = layers.conv2d(x, filters=64, kernel_size=3, use_bias=True)
+    x = nn.relu6(x)
+    # Reduce
+    x = math_ops.reduce_mean(x, [1, 2])
+    x = _Quantize(x, 6)
+    # FC1
+    x = _DenseLayer(x, 64, 512, 6, name='dense')
+    x = nn.relu6(x)
+    # FC2
+    x = _DenseLayer(x, 512, 10, 25, name='dense_1')
+    x = array_ops.identity(x, name=OUTPUT_NODE_NAME)
+    return x
+
+  def _GetGraphDef(self, use_trt, max_batch_size, model_dir):
+    """Get the frozen mnist GraphDef.
+
+    Args:
+      use_trt: whether use TF-TRT to convert the graph.
+      max_batch_size: the max batch size to apply during TF-TRT conversion.
+      model_dir: the model directory to load the checkpoints.
+
+    Returns:
+      The frozen mnist GraphDef.
+    """
+    graph = ops.Graph()
+    with self.session(graph=graph) as sess:
+      with graph.device('/GPU:0'):
+        x = array_ops.placeholder(
+            shape=(None, 28, 28, 1), dtype=dtypes.float32, name=INPUT_NODE_NAME)
+        self._BuildGraph(x)
+      # Load weights
+      mnist_saver = saver.Saver()
+      checkpoint_file = latest_checkpoint(model_dir)
+      mnist_saver.restore(sess, checkpoint_file)
+      # Freeze
+      graph_def = graph_util.convert_variables_to_constants(
+          sess, sess.graph_def, output_node_names=[OUTPUT_NODE_NAME])
+    # Convert with TF-TRT
+    if use_trt:
+      logging.info('Number of nodes before TF-TRT conversion: %d',
+                   len(graph_def.node))
+      graph_def = trt_convert.create_inference_graph(
+          graph_def,
+          outputs=[OUTPUT_NODE_NAME],
+          max_batch_size=max_batch_size,
+          precision_mode='INT8',
+          max_workspace_size_bytes=4096 << 19,
+          minimum_segment_size=2,
+          use_calibration=False,
+      )
+      logging.info('Number of nodes after TF-TRT conversion: %d',
+                   len(graph_def.node))
+      num_engines = len(
+          [1 for n in graph_def.node if str(n.op) == 'TRTEngineOp'])
+      self.assertEqual(1, num_engines)
+    return graph_def
+
+  def _Run(self, is_training, use_trt, batch_size, num_epochs, model_dir):
+    """Train or evaluate the model.
+
+    Args:
+      is_training: whether to train or evaluate the model. In training mode,
+        quantization will be simulated where the quantize_and_dequantize_v2 are
+        placed.
+      use_trt: if true, use TRT INT8 mode for evaluation, which will perform
+        real quantization. Otherwise use native TensorFlow which will perform
+        simulated quantization. Ignored if is_training is True.
+      batch_size: batch size.
+      num_epochs: how many epochs to train. Ignored if is_training is False.
+      model_dir: where to save or load checkpoint.
+
+    Returns:
+      The Estimator evaluation result.
+    """
+    # Get dataset
+    train_data, test_data = mnist.load_data()
+
+    def _PreprocessFn(x, y):
+      x = math_ops.cast(x, dtypes.float32)
+      x = array_ops.expand_dims(x, axis=2)
+      x = 2.0 * (x / 255.0) - 1.0
+      y = math_ops.cast(y, dtypes.int32)
+      return x, y
+
+    def _EvalInputFn():
+      mnist_x, mnist_y = test_data
+      dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y))
+      dataset = dataset.apply(
+          data.experimental.map_and_batch(
+              map_func=_PreprocessFn,
+              batch_size=batch_size,
+              num_parallel_calls=8))
+      dataset = dataset.repeat(count=1)
+      iterator = dataset.make_one_shot_iterator()
+      features, labels = iterator.get_next()
+      return features, labels
+
+    def _TrainInputFn():
+      mnist_x, mnist_y = train_data
+      dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y))
+      dataset = dataset.shuffle(2 * len(mnist_x))
+      dataset = dataset.apply(
+          data.experimental.map_and_batch(
+              map_func=_PreprocessFn,
+              batch_size=batch_size,
+              num_parallel_calls=8))
+      dataset = dataset.repeat(count=num_epochs)
+      iterator = dataset.make_one_shot_iterator()
+      features, labels = iterator.get_next()
+      return features, labels
+
+    def _ModelFn(features, labels, mode):
+      if is_training:
+        logits_out = self._BuildGraph(features)
+      else:
+        graph_def = self._GetGraphDef(use_trt, batch_size, model_dir)
+        logits_out = importer.import_graph_def(
+            graph_def,
+            input_map={INPUT_NODE_NAME: features},
+            return_elements=[OUTPUT_NODE_NAME + ':0'],
+            name='')[0]
+
+      loss = losses.sparse_softmax_cross_entropy(
+          labels=labels, logits=logits_out)
+      summary.scalar('loss', loss)
+
+      classes_out = math_ops.argmax(logits_out, axis=1, name='classes_out')
+      accuracy = metrics.accuracy(
+          labels=labels, predictions=classes_out, name='acc_op')
+      summary.scalar('accuracy', accuracy[1])
+
+      if mode == ModeKeys.EVAL:
+        return EstimatorSpec(
+            mode, loss=loss, eval_metric_ops={'accuracy': accuracy})
+      elif mode == ModeKeys.TRAIN:
+        optimizer = AdamOptimizer(learning_rate=1e-2)
+        train_op = optimizer.minimize(loss, global_step=get_global_step())
+        return EstimatorSpec(mode, loss=loss, train_op=train_op)
+
+    config_proto = config_pb2.ConfigProto()
+    config_proto.gpu_options.allow_growth = True
+    estimator = Estimator(
+        model_fn=_ModelFn,
+        model_dir=model_dir if is_training else None,
+        config=RunConfig(session_config=config_proto))
+
+    if is_training:
+      estimator.train(_TrainInputFn)
+    results = estimator.evaluate(_EvalInputFn)
+    logging.info('accuracy: %s', str(results['accuracy']))
+    return results
+
+  # To generate the checkpoint, set a different model_dir and call self._Run()
+  # by setting is_training=True and num_epochs=1000, e.g.:
+  # model_dir = '/tmp/quantization_mnist'
+  # self._Run(
+  #     is_training=True,
+  #     use_trt=False,
+  #     batch_size=128,
+  #     num_epochs=100,
+  #     model_dir=model_dir)
+  def testEval(self):
+    if not trt_convert.is_tensorrt_enabled():
+      return
+    model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata')
+
+    accuracy_tf_native = self._Run(
+        is_training=False,
+        use_trt=False,
+        batch_size=128,
+        num_epochs=None,
+        model_dir=model_dir)['accuracy']
+    logging.info('accuracy_tf_native: %f', accuracy_tf_native)
+    self.assertAllClose(accuracy_tf_native, 0.9662)
+
+    if trt_convert.get_linked_tensorrt_version()[0] < 5:
+      return
+
+    accuracy_tf_trt = self._Run(
+        is_training=False,
+        use_trt=True,
+        batch_size=128,
+        num_epochs=None,
+        model_dir=model_dir)['accuracy']
+    logging.info('accuracy_tf_trt: %f', accuracy_tf_trt)
+    self.assertAllClose(accuracy_tf_trt, 0.9677)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
new file mode 100644
index 0000000000..28353273ed
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -0,0 +1,144 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.python import trt_convert
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+def _GetParams(add_quantization_nodes, dtype=dtypes.float32):
+  input_name = "input"
+  input_dims = [8, 8]
+  output_name = "output"
+
+  def _Quantize(x, r):
+    if add_quantization_nodes:
+      x = gen_array_ops.fake_quant_with_min_max_vars(x, -r, r)
+    return x
+
+  g = ops.Graph()
+  with g.as_default():
+    x = array_ops.placeholder(
+        dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+    x = _Quantize(x, 10.0)
+    x = x + 5
+    x = _Quantize(x, 15.0)
+    x = x - 5
+    x = _Quantize(x, 10.0)
+    x = x * 0.1
+    x = _Quantize(x, 1.0)
+    w = constant_op.constant(np.ones((8, 1)), dtype=dtypes.float32)
+    x = math_ops.matmul(x, w)
+    x = _Quantize(x, 10.0)
+    x = array_ops.identity(x, name=output_name)
+
+  return trt_test.TfTrtIntegrationTestParams(
+      gdef=g.as_graph_def(),
+      input_names=[input_name],
+      input_dims=[input_dims],
+      output_names=[output_name],
+      expected_output_dims=[(8, 1)])
+
+
+class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Create a graph containing single segment with no quantization ranges."""
+    return _GetParams(add_quantization_nodes=False)
+
+  def ShouldRunTest(self, run_params):
+    if trt_convert.get_linked_tensorrt_version()[0] < 5:
+      return False
+    # Only test static engine mode, with or without calibration.
+    return (trt_test.IsQuantizationMode(run_params.precision_mode) and
+            not run_params.use_optimizer and not run_params.dynamic_engine)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    if run_params.use_calibration:
+      # In static engine mode with calibration, it should build a calibration
+      # engine.
+      return ["my_trt_op_0"]
+    # In static engine mode without calibration, the engine building will fail
+    # since no quantization ranges are set, which results in no TRT nodes.
+    return []
+
+
+class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Create a graph containing single segment with no quantization ranges."""
+    return _GetParams(add_quantization_nodes=True)
+
+  def ShouldRunTest(self, run_params):
+    if trt_convert.get_linked_tensorrt_version()[0] < 5:
+      return False
+    # Test static/dynamic engine with/without calibration.
+    return (trt_test.IsQuantizationMode(run_params.precision_mode) and
+            not run_params.use_optimizer)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+  def ExpectedAbsoluteTolerance(self, run_params):
+    """The absolute tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
+  def ExpectedRelativeTolerance(self, run_params):
+    """The relative tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
+
+class NonQuantizedPrecisionsWithRangesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Create a graph containing single segment with no quantization ranges."""
+    return _GetParams(add_quantization_nodes=True)
+
+  def ShouldRunTest(self, run_params):
+    # Only test FP32/FP16 mode.
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    # The fake quant ops are not supported in FP32/FP16 mode, and will split the
+    # graph into three TRT segments.
+    return ["my_trt_op_0", "my_trt_op_1", "my_trt_op_2", "my_trt_op_3"]
+
+  def ExpectedAbsoluteTolerance(self, run_params):
+    """The absolute tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
+  def ExpectedRelativeTolerance(self, run_params):
+    """The relative tolerance to compare floating point results."""
+    return 1.e-05 if run_params.precision_mode == "FP32" else 1.e-01
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/testdata/checkpoint b/tensorflow/contrib/tensorrt/test/testdata/checkpoint
new file mode 100644
index 0000000000..a603e1aec9
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/testdata/checkpoint
@@ -0,0 +1,3 @@
+model_checkpoint_path: "model.ckpt-46900"
+all_model_checkpoint_paths: "model.ckpt-0"
+all_model_checkpoint_paths: "model.ckpt-46900"
diff --git a/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001 b/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.data-00000-of-00001
new file mode 100644
index 0000000000000000000000000000000000000000..88a998f184b275121e1e76eb51d2310da149f10a
GIT binary patch
literal 686728
zcmZQzU|{flKUY@aQ}cmBy$k!*ua_KP-g01n%h#p*|LuCZ-!ZBEK&D62fvP{R_CL_p
zI8YRkdBA)t=Yc2#(F2tY5BIlE*E;ZRYSsbq1*`{N-T1w~ob$r|rYy<*^?&sbFdU6J
z5Oc)jz{5oG1F@FO2kyUDK5#D5=fGDdkptJ0*banTbUfgA>Cb*Ee!&Cw8{&35*|=Mm
z>2KaGTXVs3p8TR+?(3HAdMWqEihFymwa=1oR>}{TSu6Rd?*6}-cek!=msM<{r?o&6
zzjfXmw%vC>@3PqZzu)S%eueeA;MZ0yi{I?pTE@3q^vvPihd7s6l^@z{y-mZ;DyrFT
z_ljV1Yx{E&yJRn2-Sx~=&MM7a#5$6p*W$LCkflc5eoN~FU(2mM8kS!^uv#)~YPFb}
z9%s32orR_6(_%~0&J`ASv~?^UFJ7>C|5eE{>Qk7d!C_C!>U$lQ7wW<-uhoiL{SAC)
z`M>Oy<$@)#mUI4uSc)dRvADZT!*cPb3d@Am`Ig_NOtvg?*R~81=(Fg2C1sf(9cn4R
zSj}Oh;C1_B&mZnhX=t?5<VdvtopRfrKl<T5ADv6~->2TP>yun!zh%SD{YP3D?DH~O
z?e_~8+ebV(zW05Y?*3Mm4fah-`43zyk#ShMX_DQF$oYHM^B>*c@c7yON<*Ck3nKdL
zQ+%rSFSpFG-;?3IpHXzW!<3DG>;-52-S?s4pnYjzr@cZ`#=buNq<u>mboW(l(XyXy
z8n!=sk?eliPl|RS!q4|}G@rGXSNXb6^|+M%G1VIT$~&j*?Dugx{C$34zupsO`-z=l
zb_c~n?LT}j*e~vQazBULm3=P-zS^bpoVHKRzGA2O=ghvF4ioHUX0O~I6Srvp<ZiEh
z*ROuvyCopl{!fS4fv~Ix`~L>U?`Q45zh7fk#r|zG7!P#jvK~;g=05Q9Z}WcZ(~A4;
z_Z_s46zJR6wOzvD>y#_|ez5U7aAdFEKWm1%L&cIWcF~dC2l$?3IfxaXwJo_6d!Q&j
z&R+Sz{JozTocEiqZ?{)s>9ntRka1vNcgV)}-mm>z%!=*K^j)%l_@%;ruK9|+D=sDP
zm*={?-+fiZex95;_Ccpw_no@N=D<`d>d?M>ll?r_<NH>yZ`_~ie0#4)>K*%IJD2SD
zv^#9~$4YZQ;|33VHS<&Zr?R=*#wJzo?_bqszjbZX{-CKo_S+Y=+pmlg*r#HpVBh=B
zd;gvdvIj0`o9|z=|KdKo=e`c>*6Qp(^D1Wl%vZg3X}2Ej)BV46f7Q}Ac5a#?`%mR8
z+|Rg5+99ED!~UQjZ|%*i0`^Y{xUjF-Pk--?$-Vn?U-{UtTCr!JY2iitziLPA4USkH
z$UnYhAG6ApeP8B&wwu6nWZ(bbX8U!O8|){2Nwqs_GR^+)niu<XQ(o9@n|yeGo@UDa
zWApg;XDnX2FO5Zc|4he~_FHw%?hg@ry6>&GqJzzq^ZP&DV{>@_;*PyP^Xh#ow#gh&
z|L3`1J$CK>Ge61?tnFvtzbyU6{=id)`!~&Cw|>F6ZJ%gUsLh5Y><*VgyX>8ht+qGu
zyteOn{vUhZAFu2(msi+_U%X~}Up;ESb;HB`On*A}TbOy+CvN>>&)ikA&v}BZ!^YQ*
z_ElaH_PQ(C4oC@f*lCGw+<(|v-(l<0`}=d}EZ)C2c>n%yJ09%+_{rNLWdV=F8YZLt
zjlZ-VIt^a!H|fdSXP2^T|CZ$p4#$_r?{{E%W3L_TxNps$%lkdOEDvax>Ny<E&E1#u
z^|hUPqOrpp<M{nwZ<+1)6|&mr+Qhow$8(0w-2XT28%z23{+IW)e`9{e-e|eV0rw5l
z_KT=bv0JP++y1mR_x|hKm+mhYueA4wUuo|*D|Y|J=BoWF8GrW8c4f0)X830B-_PG{
z|GsS7zwcP${t1ij?CaT^xj&QDd4J2-2bOIY!}f{D8an(CakX3bjqQMLXyg8m<!|?A
zPh>c7$v$uY!nzjwj;EjZ^<R<RAAUI0VY^wXz2B0Q{l6A>+5a!+ap>H?*<OM-%;9C~
zSzB3#`u(}f7!Mp*vOe&K`GTEEoY>yE&I%4}Y2o(HcUtX?|4G@GPEkIPxklsw?{@72
zUOAroYcm-4OL^?tzco$DF20RppD>I1?m2U}STgQOwVrjfX=l0Tw_U93ICjO%W8K}i
zW|CEH$1RKF+l#E{X9`&t?3`!CzjBg=&u{HrE7t2;i%;6UYf0ZpD?W#b)|xw-tk*n?
z-<_NC#j5lm_m1lSf!2TGeXZ^q23hmZO4<F-gVDM(vt+l4?gA^ty*qc!S>?7%uQAa2
zS$Vcq%Li5Kj}d#Udrjqc?OQCk`)bJ%tBi&3c1buaw>-!<eOL4sA?xQ>{JT=0U9*bT
z7O+l{Ik3y=W9_cv8oZX4vdX*Gm8`UGD{!(hWv<%gEV^cw{-S)VYYGv&n%KFmbmb$g
z&$lw~{(h8Yx5A=?UFW}8>{=mkeAl+85<9Pbv$I-N*<jtjcedq;gA1&cw>#`&Wte6)
z$LXr2+Ki@MlJj=%+UK5Uv3BcHtJKzOyDAustZzy-THWN)-L3d=_pY-xPpxd;Ki#Ew
z(7-yr=F+Zxw<cO~rmop_`(l~ZZ%eJ+w+w}L<{!$iJZooV6@J`s_v?mCtKIcac70m9
z-=fxfm*uPHQ+EBka(`Ez$sdbZ6Zcrh2i&qWUwdfh!p%OrLIl`%C;3QP*Jw=Gc`bLg
zby3bitNo9pt$Q?Dt*QbPcRjM5X34~`e%F$=WmboucU$KlQ?rf=`@8GyD^Bb08W(n{
zgf6lUd1`0PBYxCMqU-CfC$>S>e9k|1-dJ$fDsDR0?jrwYt7xYztCF%1E4zb>tPbT|
zvEFle#;*NYa+a4qy|#LB@ZGKr%M`8O=Gg9v>9)3BP%vp1ORA^!o2Q)CV%qa|efYw$
z`-|YiUEK9dI~?AYSZ_0!YW37$=`L=q?j3tHzwct)(!1+%P@5G`qv)<0`yV@=c^<Qj
zo6Bf@diE)c_t*CA3T)J{4x2D<=PCmS>nN`?yVA2(SZ}m;-mQCH$(kkgndQbCE;}T%
z<#w7FF0x9Lmb2a%!nM1_Il!9dV63(Gw`u07-1fVR`5AUsYwg~tFBQ7$6nDqY9kIdI
zdlxY64&8okmx1$&-7U^jtW4e{?=B74WGOFtcvpqLy|s8-zvcD%-kty2Cs|3&Ik9Wf
ze7>E=eFm1FmOitNP}Z<Ix+P;*&O_myo0#73YOg(JHK}|5uA}bZ)}QVRSZi;WvU+oQ
zqSfk6?7LpZT3FqE#=Dzu%4VyD%-YsgNwasn-_BzdxGu*!D&KhbMe&W+3nd(^w`gqL
z^{-;G74O-8>tDu)t@u>~tj#WS?#?@`XRUl|?RKS`{Z^Zz-K^$)IcPoQY4c7!_7baY
zt2M2dSGMi^$rx!}$Hu$M{e!qQ(`>(8wazJ4C5z8noe5gH>j1;CU2m1;cduRY-Rh%u
zvi0hYDVATw0(P9r53-VeY_OYGsN3quliODHyApRddB$0L6il?f;&Z}klhJKUg+<eL
z)!pl~*6A<ZwQ!k)^{ca6cM1MjY2BZ2Y*+A`pH>MEu3K?doU~Mb-)NO>?P{r4sbhVs
zL}9m*!E-B}3r5y_^A=cL3VpZpz!b6FQ&gIF@p?yDmTge5I`J>q`p3R~yDn`N-MunD
zb+@CYq?PNlxx3jkoUF3iq^&jWmRPlj$?clA+sTSaTG3i9UwXHnnwP~EnZp)2dHt4i
zK8RXA)BbM3{co{F^rs4o(yPBMJRMXl@5^ght_{w#$cbRK+&6QZ#grLGES@TEu~;Ct
z)#C9{QA@*1QkF%Vn=S9L^;;_WAFvd?z-d{)^u&VoezS#x)f9_dW=YHL-M=k1su^0I
zntk8m_!?h}Pp{Woa3(TX%6RKr=oxRfxchdlCGUH2ORcA8Etbyiw3xiO$RaTFx<#{>
zkR?lijAc{hU-RZYuPw|2`z;O$&b6>?x@6(>^SZ^3auG|l7aErKy;+uTp1NCp_UyAv
zT+Cs4IW)_n&)m&ozvxPf_<$7_dc9XHw%q$`F-45s^3Oa?i=PJ$Td;P&w}`wGXpwyC
zvV|}I97_c)QOmf=ES9RAJ1jm<*0G35<+T*cJ!f%jCBJ3OsTzx{CD$y{<u+M-@#(X8
zQGd&#;0v4OMK>YKxC=^_nTqc%riZgys(nwhWDel6tap23;gHg2v95Z%#e07S%g{r&
zEv`>jvHUE+YZ+5%YN4j`+`?A-lf~n36^p4B>n%QinQ8f9I-}+K>+db3{`OehiYc{-
z{KaZnZ_97V6c}XLnq+FRVWY66-{e^q%{-eeHnQxq*eBFw!JsW+IpLX*<)r4F7Hq7G
zEYHhdv5cB?z@q1wwB_>04i+^gwH7B&OIS9nI%^Ra{=-7&$4?9G@Nf%@1v4!i74}#J
z)~i|UvE5{`;N^Nt=HBNP9qpGa9%^-4q-bbbJfD2t;=DAk<;f5o%g-_57S8n_EEK1-
zTiD*2V<FP8%VMQVki~s&Udw~(I+kl%!Y%bPwJg`91zF}*PPI@sRkyhCIo?8YQLRPi
zgl!f@E?X^{ef=yK<*{1&sS8`&dNtFce&$+>t_3L;mMYgR+%9dh44TDg$s~W;VzbH$
zi}06^%zrX1wU`(8&7%E*m}P(4REv+#W?0m8-nN*i(Qk3mjn%TnZJEXT?c$d1yFOU>
z9b&Uwf7sq~$1NSpZ6X^iwzR*rNSnIFVphm9i{A?lTi8gQxA>muVfj~*-!lB9gvIuy
z$1LtSezcH@)Uc4*v&cfMWVWSQ>{SbeTl*|LbtYQ)TLf4L8S_|9|9Z(n_o%w%_e%~I
zUJMMD#*SSU;Z{>E7<LO;x^VurXt$EGES<$^De+du@~=vVWx<0<mQ%cMSVZiRu&gMp
zv9P|r$l}C(6U*hVr7YL`sas|%2v`RG2(x%Tz1m{=qh%IbIlL`2YgbvU+rHS6oAJ8^
z?~e@@YhNv~*yF=zvG<a+<^JV6EdKFuS@tm5SVRY|vtTcnYVkHa)8e-GUW-+?*IO89
z8Cu%D`)aW@o5ymYDzBxW{C^84PDaaB-m5JdUNl+kvs`VF`BuSl`g2uF*~zAsPRjh2
zyA?Do_Mhyxa5On#;rKGc;#25xiy)rGmamn0Emv*1Wg*(N-J*ic#iD4+Qws(uX3K*G
zzb!t`Y_mAE@1%vC*(;0t?8hzAoR3@BN-nS{We&AG`;yJF=)HiY^S3C=<4qZsQPUYL
zKYJ{(SjD@`!t=#9i-WKDETa=8Ep-(5E&X;0TfVp<W>MR;(PF<km!<Bp(!Ie`SM5D=
zYS!L!X6yFtzn!=@r;-1_nwdQNi~dRN{rg$)0E?E-UVZiD`wGiv?yd35-us>_Y_GXh
z?cTh!kiFWe1^bj5SMQS%k=`FJ(X}_kNO<qnz`*^V3(EH1Z<)IP#Ni$L{$*S5{c1RG
z-wMXEy~(#r_Xh9d+h64nWc!zMy8Y(gnRegmT=#~|J+^;dU)o+}ubRE9_@>$3m07Si
z{Lq%YDt^=UO?Z^MSMYE7UdGp@d;h6d?Ok!XV(-z0w!QbRtlyidvu7W_w7%VhOQm}^
zahL7A@Rsqw`ggT^UGls3I}5b#t^88DcQcdS-tR0M_p$G&+WRi4b8pLqefx523iqlM
z_3Z_X1EAx%cP3cg*i&u=68B-X+6AJqVf)qzU^VD+wI(kuL1I7owu5O98yTCfezhyi
zv(?hNI@5~zuOwKF#KZ_N4N@1|orcp+kUTn0)USi9U$)^cnEhpI2HZT5`uVqKfW<*<
zWPI?wC2yRaq(tZC2HpqJ?P7CcnZ<vGE;7HqZK3&71{UE<iq00{JC6xpwOFWdR_>vo
zGQ)1+C6YD5dzh5@-<I2$YpBlPfBSrfwCufn%`DYQ0mn%(!gJ(G&0A}a3C_@eW5(5d
zUhKd6WFgr%J0+j(W#P-(^@mq4Cd{JTqsaV~nxeUw<Ti`nO*4hptzj~Eb$1duci4n4
zSbDX&>!QQFGCuKwpHz0Ltru_;UR=V+H%;v+-;yZ~LQ4x~30;2_E^zSFME;6f%S0_^
zuv#z+&*a;Gs#v)7^mFsN7<0jxXZq%HbGHhfTc9jx)IUvVwL*mX<}<NE4GfzF1rJUX
zxn-y(oOQ!V_<UB|-mG}>y<PXk_A0$?-j`;&Z||%JKlfXFTCqQ#qharJ!*d4|m$KR>
zoIkkFP`r9ibI9-A)!G4jZ!c`yd-yf8L$n;<KDY8cdlqXo?Nh8?xHtLmw7mfgUk^x5
z&)A!HcH6$Tlhu2Fe>-9Gyrh4(?~QeP{uqYs&C1@mzjM`ld!G632PEIg?H8Bm*?X9w
z@4%JCJ8T@;SMBY)P+`xTvd?;9w&uQLiW~ODwTIYxUf5?-Tw1qx=9G22w+h^{3HnjE
z_i(|hy``R|d!(3l?|<g}b9cgJ!QIoGC+v^xYuF<uYJTwIBc^?~w!GQPbxv`2^-Sme
zPD&?jG+kSFPu<6`$Dnu7UN3Qpy>6wS_U@hh(1u$tb`O8j!F_uLxot(N<L$P+*4rQZ
zxqi=imCge?=@V>LT2I^O`A}wG?~N&Y^)7breP712FHWgwpUsiRz5i~7+0O82+_xr;
zX<xyI=)ICAdk!RN<n4WBbI^WI;T0RlZJf5pWDIN<+MDnB&&6rAkJtJD`}6{PUWW|_
ztn%*cV_mjnuieC7``M$W>}Biyw6{9#-~KF#vwNjPMfU|C-n;L4kJsMdj>CJtNuS@_
z*&Ms4iKT6i!>oe6e|$9e*@cGe(LQ@?|Bm^=dyRGG+pNrT*#A1CZ}0kr+Yh)LXWQo;
ze`K%iiC25Lua(+g;k;+<wQB!f-S9q}J7>E0oKBJ2bH|%;-zFy3J%5$k_ij%;zHggd
z>)yy29s5r$-@e~^%06qge|`Ix%sRR^?Bjxc5mFC!YpTB6Tl#$A-gmF|*w{}|JMi(D
z-#&}nnfscg8TViA{kr$*>#KYHJiP5McD3(qz0<#6)br`yh1)Og)%`ko-@cFidozp|
z?R5*#w=>x5X;+aaXs2UQx&LMBD@&>GjQh5xM($CtJ+rrJTbb>zuKqoD9!u`q8ZWW$
zT9c^_%j*++Uo~~x%)Rw?ujJM3d)toxwyr%Cymy`UhJ863Cmu+=%e?O!<K{gl!)ER)
zoE@`g-V_&G_wPn~cdR?Mw|DE0-3GiBd;J_LZFaxUusJ)wXy2Ay-Fv5<Y}zNVK63Bk
z);)Vmy4UYrwR*=s<J@(7U+X0A_xkW_U(kECy?f%9?f>$A-d@Xul6{{0wD!8p`MLL?
zZP%X8-R%1!6y^3aWee@now|N6cU{W9v<bC)C%AstdzXV_KV$5xJ?9cW?-N-$b>FJE
zk9#Nh+uNAN@7|m7C}Qs>&+Yr0by5y|QIk8c#M#n*jy&f+ul9eoDYde;pWSBL2$z`b
zUw3N5UP)j1z1Qbl-4~s4$foJ-+r8>n8TZZXmEE`MFWX*e^Ba4ZXN2wDxTnj;u*}|m
zg^=&wtizdmXD+MWe|qPJy&K;x-Y=iaxsP$_-aU7mnf3`z?6DWL{II7uW8U7Hy0`oO
znmpgzp(41?X#K6dp?5m=y4|$hn-@~LZ~N&dd+##$?|YUje_%>@w#|pD@^<F^d3%KF
zYxf<g5#Gzx`F8K!LfO5okr(&=ydAjz-?2$H^9~5@aq$=3Yux&OPt;|OegE5(_lf;@
zxc6ydo_*}xj(yd;&h0&2$+XW&vVCvM{ndL@G?WkYGOgKf_G9_}gOfMeFIIiCYwN|Q
z`!ugC-~C4B)!x}!EcTK0oqOkZZrQ0=Vz{qyb>f~^aqRn&8g}pcDA2O^Ym)U|C$5*f
zV!JNvJ>7E5_P+Lm{hger_TH9b-e>WxV}Jf5wte~YmfEfHaNhaIW!D~qp0~Rij(xG`
z-(9{-^1;Eq*8c?ePi54zvF>=aM<=w?X6pVw)>VJk?ClcRz3+Bf?%u@WPWx-Ov<_$~
zZrVGE<JEzdLnSu7sww-Xo~zwc6xOuYMQ^^1cWKPt)jA3GYBu-x-p***W3`iI-}$AF
z_rBQKv3p|g;seY-*4r39XLR^<mSrFJJpH|wcg@-RzlCq_4E4^ve2g6j{Pv#PYj}(2
zVAJix`>!~=@6q16azE#mzk8(J3-_K$6gr@2*uHn8!>YZjgI4Z48fdu3#sAA*WyX(t
zKUQV!-Cxwdw|UaMz451*_bs{mWjCX$`+)}w+4ikZ+Pt@Oh4DV;&)fE9UvxfjOh0hX
zy|O90zpuV!qrdH+{ppq`dw*Iq?`8e@c8_JZlQoC*JR1?=vwI5JAMCway<ztgi3|I<
z>zVgXyB}~MWRI0Y>gR^NUHADMCe2CT^V`vAA4BNUJ+t*>_x=#yVe`tqXK!$$?E#z5
z>-Kj1WZzqM=k*>*|0R2iFHYFK-u$?|blsiZ)|G7s;<H%xX=O3kmM?VNYqg(apYPYR
zdlU9aIqW!p!M-YQll`3Yng`bH(A|@yRkC;a#_cxV?fds063ja=J9Wd}XRNyWPIXP*
zSL_(H%jV2lo7tZx+06KVcaKs19Ghe3^=!ibE!;aNC2sGo1KkH^PO9Ad^5Hw1k`e*?
zgYi%Hgui8TFs}WzSGD)t-iA*%_a5?`u|JlZb5G3UgL~!5cI^!oRkTsx_<HZkUAt_a
z9cbJ8<(kkwZqxPqc5K_e_j15S+cG1)gR&d{S?kTSI1tbHbMFhT+<gn%w(Rk~{CMwE
z5rI7mzg*h;;O<-py(yCW^rzSFJ#C+8*L6f@pVvpHy<)Ha9?<*vY;R$lh{Ks_bM|$I
zH`*w^tk{!OzQIQNjmusJp1K2<Kk*+BiobmzI%?(q?-`%=PUF75e|0U_-ntd5_AX^%
z+P{Wn(%vbu+x9RxY~Jg4R?g}RN41UW@tVD&Z(R1KRi)Z={5Y|<EsS%o)e@F{pWmI|
zzf4hZ@7D{a*8iBb53olI>@zyP-aaLoYftVh_1%Y6823qYcR1|fUTM8z&c?m#!qxZp
z2yWhe@ZTbv!?qcFy`!0IU!PdC_xZ1t`%bl=+jHu{c6$eAUc1bMq`em=H`!;OjkjKB
z;k-{?fAbz4k(D+anT*!o9X{<*wP&?6;cMT^&nmNbd-?RehVPj6h3wnANAWS+0h#LT
zy-Wt3_U7#i_ZhEa+^076`raU&{=Mu^KJDFnu5!PuqW1xogDdv)F;>`@Un$yKdalud
z&E?bHe9awut3v16-wIp3=gEc3d)v})S}%+4*kf<Xuuo65%Eq+7%zB>pr@e_>g8TTa
zuk8&_Tex@KyQB67lezbNpT@f{ZKub9&#s^M`d)I|zc_ihb#sO2-rssBY^Hy?wQpi=
z+}{4K@ZB5tFz<itv2Cvu=V6;x+dF$UtPtEcSDta-`i=MY$z*M`IrS^hfulxrZ@SFc
zJwAu#9pH$bwKv3Q+P=AOKkS+Oo5Q9*{p#LzmpS*%Q2%CsW!=fW6T{5*vb~yZ%OZSq
zuhx^;z2d3z_NVvM+w7_qclcnnXJ3v7$KHeWd-e)ehVDHu_0`^nNptppPVe7$qdCf6
zVoKlsyB2)=xczwdmlqwf7WL)dm&yFnZjG(r-iGy2dlmd@_P#w^V#8^<V6VM8^S<*r
z>U(*aFYkTT(zo}PrNF-W1@U{Hy!^0lkIt&yzgK*-DK_IeX!h;;-c7d*4hX(GW^J0N
zzsKM~>RylB`~x>z`)xG8&fLo%Xt~dFN0H5~dxo|d^Ah&%JEv>={rla$<)Q2L34}k|
z!~9TZ|Bu()2X<X4+-s;BZ2wfCVQ*XGWUFJY(tGzTEVsUF@^f!}W%KS`%)tlt2VB}a
z>&6_L-+>$VZ9e~dZ(K#|-e7+<hs2%p_XZuEeZW{`);^APKlYf<(BFIghveSRa+SMx
z&Cx$lqP5+=eP*3~lUujF{sb8t9la_0^^<1W?5<<lH(gldpz1@%JuezL_Av?Q+xjmp
z*!%i|!`?sNT5V(({oNbR%eK#!<>el+mkai;spi|4eRbIZ)n6XF{dLXv{^3<{;8|5`
zBklQj|BptOy@%eOu%22hviGiQqy3|p=DpKiAKClhZ{9wiH}m&e@o(5G!FFnonc9ZE
z94u;kdu-0^+s?PwX7=1$`<6+sv`>)Lx3LoI+As5(eecu-z54`P4%<kqkKXI`=ER<C
z{%6*gg}&N}gi7yS*M7z3>R0Q1k#0L|-g4UQsfjV)Cth`Kk7!cX{yQHs_GV|FvA*3}
zy!SCL!yebJLpG9;|Mn`LoVowWhb8uJ0>0bNKP0%<ZN+2zH`|2v_It0g2`Q1VZ{@Jw
zqyAuvHSdrA`}l;9?B#Od-?y(R(B{kKDSNbyCheV=CBCP2XYbzZ;Au8;p$qM=)-KxP
zaB9sS`R_&h<0QHE1wYo>?|gdp-btFad*#;n?>&-u!``=L-rj&!iF;~pAKNE+w0iFn
z<x_h<cnI!O<(A#cIQjJ+|H9^d-J;L-DxDRw<BpTF4-Ij!)lL3)K*3C5w?}F3KEVsC
z_HJ3fX3w!_EVe#;*Y^~<-M4F8B(QJeSvT9$f&cebw7uJ_AiQsH!<#2|<}+pYNWQyi
zzcw^?U;jMgy_shi_OX>6vi{%xb9YCLxBcS7&-XShZ`r?k|8YBpWo-MRdwUMNHci~4
zy7kQ70vDbAf4c<s&AzeIhK1wvUPklZd;Lu8_r!(o-<$rhdGE5my?b|`^|N`m_w?RR
zI;OjVy>IR-(%rsSoQ-4eBDwYhb5E?XdY<y`!2N}Vd-**itzS7_->a4Eejxtjp}m>q
zyn8ELCHGIf>$~^1Pvl+$pUZof8NRWawlc|f&*b_0BGQ-bohlQ!U-aEdd$+_ZHWO7E
z_fI(2zPGHUW8bvJ|MqTnezrH<-rDNef@^zQj`Q#Bnels1^!=8-SGT|3cf#z(o{95!
z>}fi^f4|6wfW3PDzIN9{ZtQ!r<Aja)uOoXO6fy6eb$|6<_Le#O>lIiJXhbU?D78DV
zUx$CEwQ+|3{(V~a_8!gk-+P+f>VQz+={<+o9`7{>yt?-cSK!`oxw5^VwlnPcu%%(|
zDV^(k_n&^YcT%L(-tT5N_MT^Ha!BXfzSk|{{oa?Wj_uc9d4KO*(YO2mFHYS1%~Eb}
zhUuZb3y<(S82!!LTe9QM9)aYG`_{btvp0ru;$G89y?aaQr|o_JeEXjF<`ed1z3SU*
z{`!Xf<J|`K`WegiD9%=O`0!GDuad!{eRWbC`}XaR-D|RD+TQ6`efC7L-m<@*y~XBZ
z<JP^d6V2>AFR1PFGyJrdXTI6~x6HfuHiezEKPt3j-^Ym;_Rd%!zWeIl{d;v}+V|E>
zV7AZY>~lDyH)B7G|H%V#2Rip=hv)8>vbEa#BIMCtf&3u*(|>O7z4+y#%{(RLy*ke(
z?_CqRe(&KE3-)q7+P2r3dEZ`HCaHbjZYb@wY<jiZCPURe#OLguLhF^*9pzRB^V%5q
z{WLmlA9X@*-!fmj-4=ZoHom#N`}3Js>^X15wfBC-$-N&f@7P?Fzq5z2&3|vx@2h(q
z?L};(dT;DY*m={&cS(#r=gH^;#>;;1HOT5Xa6O83pXScyeb*l+?PYJU+cPUzX0Ne0
z)4rBfS^HfsPTtGe<+;~?-Ijfl%kJ;CWS_rxd6b%i#l@DrDQ;W$eq}nj@44IUy}fhT
z_E~Rk*u7dicJG%D76+0__Z-NMmOh|(DbL}=hSz)9)+F1X30<*g%^k(P+;`X8UB3Q&
zujzpqdwJK+-m9M0uy?N4jJ;AjH}BoOY5(5VcGJCwWyJQCG#|5haVp;CrQ*rGuWl^c
zd+WldJ+@D`?l1fMYOh)}^S*rjBYX7~zwFJ7duJ`K`qMsvg=1d@+n&93JMZkfW3_6p
zPTQ8fTVJs6dog+QUV~*b_on1j?hDiH-aFAyz%GZ^!0wru=iWcNdJnMX%I=*%`OChu
zQ_t<qc(-`(??8rqNmtnR-RIh1TXKKP-k_h_dyBbN>~mw`-PcfDySLz=*nUlp#d~Lz
zeXz@WdUbD6+RMFZKR@lRJY%>wZ&CZ6J82v3cQ^mqJ1_s{{_J-j?R3Ov>^;lHbkJzQ
zxxHuCE!oRd9kTzj=7YU^w?4Ds=h?Gw{fzp(Ve%LEy!sltrzl8!uV~4wy~X?6_e$L3
zv9;0=-y1M}zJ2_%cY8&*vg|Y5DRscoV#(eo8S4+||6$n68Em>&=huwAvWHh5@YmPb
zyK~dDy)n1<?AsIJV6)=r$=&lUe(h=Ly}q}V-+He{|IvK~O&9mNlr7mG;*(?lYYwAj
zK@a1;<by6YE}7Z;7(<WkeW<eC#^rC0jYcHnzTPmi{j;SX?9JEPZnNcd!@eCAllF!N
zthZhiy}<stjOE_A7kl<E^O>~ovm(R3N_mex`_%gPe*E}vZ&3-?fm@bS_qW?r9(eP-
z=YZ6PZ+p+KIb`n>_InR+>-D|OJkRzYNRqKxee=Yg?rO<>y=~j~{w=w;C(G8+#$G9L
zZ{<#<z0r@__O9c3w0HNu%Doyp_t+oaKilTtmc4ro?*us96x^`)%}h0iZOlyjUVK_+
zbEq!ThP~0xA$@K3o(-D>_szS^w6Aq~;NCskHtlu#@M~|3!HvE8-_rN|yV|mk=hz&Z
za{(^)97kgIDWvb)+pIY2z=NO*7FL%w?^_YTuunpqYu}ue%WP(>`MS4C^!Wbgm7aSi
zNq(|%DtNhf%~htoymfVZJ{rm$Y%AYl(=kuT{$l0ceGw&W`{eeWv$320dGET?r+cqE
zh#YvyDzKkhG~vL@7kzeu8{GG9-O_!)^x_elXUVVke)zCszmUnHJ!{QR?Rk9OaKDS&
z+Pz1P1@7G~C%LDZNqC?9^K5I8O7=Y~)-mjL$>Z3kH*uLmjMJ^X^+`?E6CZIOXnlKl
zubR>J17Gg0+tc#jYj5^KCfk6I>+KD`PO>^CP_@_jo5=nrCfxh@W?b1bBk0wh6(`L1
zKI!V*vpW0YzGoIUY!(zQu>by?<G_xeSN0e#PCC$jv|{fmpWc0!5AyAGxgx!H*-_?w
z<}5q+M({BnxW4-EUgh4NJ>LYk>{HkHyZ6@D`+IA4NbRqdeX+-&HUGegpriXVYxh{+
zaa*zX$-2ol%c}zSdhV3mUw{7VK0mRo_Ay5?Y{FJX?6rE=eL!;Og1zY;tM<Ad5Zu2k
zYr<ZacsrXjGk5P>v6z3Kt?}Evo~5VumPs=1o8>vx>h9d%doO$l-^*I{VDF~&lJ@=w
z|L?6=f3U~&)4cs9p*!|6UadKBYZ=?V?YT_*#H>a43boGOpS8wtFIWD>Jx?3>_t$8w
z-)pdT;vPL!1KXoU$M)VmynPQ>)xLc;^N;N{?v^-EB$sPfc*JV=ROR`0uZ92ZnO?ee
zpRM5Ay&X?x@6q_{y!UK4&%TMSUi%YGnf8XJe%Sk6#L{-}>79EgOi$VS+?{)$kl>8H
zwi#&$6uU0(WBPn+ueR#qy*%2VcQftTu!l8uzy15n*8NdOuGr7ESbLy<d)ux;rWt#`
za?G*rzbn7jyOnugWbcl>9_EwxHU@O>Grp0(H~q}!y%GhpZJ4Gg?X8sExHn?%gT1@`
zJog&*ckQX0DSlw*sb-t}>7Q)+|39%6QGd8s+x7dtF2VDAzluNJ`y%Jwo>ayp+tiB+
zd--*|Y<9kE-B;_oYOlT5-o56G5BE+~%h-GWy5%0@>~s6NcW&AfI@x4@p{B$ExjUcs
zUYRF$p!7|x%`TS{`+ofUw<kdP^j^k(@x3KwtouF}`Z~<-pR>38+VZ{Ge!O=6uAzJW
zH5To)Eq6O0pjW)tpy<1u;M^no>P>d<UAueY-b3NhdmcO~-uu%d;Q%j#(}6#~_aE4~
zM8=`<&%M1Wa(#P*)=Ta&TOGZZV?vkxPCM7VE8m#duF>4SFKlPfUJvENdtYvC+xw)Q
zec$6~jeT286ZS|lFWbv&+GJxg|D9dft7SIp-m~pn>TZ3&=2_w1TUoOEHoCO#&AP+4
z?_5CI-i%Et`_+C--J|W$xmUvS{JvV2e|y)v1nv=+p0#(m@VC7g-je&)U--4pr(omW
zL*?!EV)2*vvq(JL%QV?#|GKrG_r&F&vh|ajzSrQg!rmJ$v-f6i`Lo;nn8X3@q@{b=
z9$4BcXP)1u^Xcr~T%J2Nzr(}r%N?8d_Wubx=&|*|KK8)Fd*0WV+G=c&+sn_(x-aIb
z{Q&`iEB1UJ)g7)+y<k7P!DG+K*0TrVmzwSlOuw?%Y|epw+?kj6e4XL6$9+QdUf&7F
z_vBdavx)fDytiB3U~k-(=Qc$=Li>&gJ=?or57)kgbBzat7EH3v=KX84j%~HQ)aU1W
zQ@=VKIM6+B*TEctz1(hJ_F7o9*zbLQdhf>x8}}wE>h7za^2_EolfBK0dy8$p9Aw)Y
z%(P<fYTK3j9;Q#;`{hUAfse(d`}e15+h*;ZvEO^X{N71@kM`a8<FhyG{F%K*tbTiL
zc751e(A2nZ=3dTyeOiq+LH;-Q{l441r%u{sPo<iV!^&5wd*{w8-XFht>c0PXefG|b
znYedSapB$x$?y06er<ekO*6m4PkXrom5+P(vrm~~UC#1x_pP0`_AX7lx3}<%iGzu+
z&fXuIyZ2u5Xxsa*Fn3Sw`=s5MbYAYAC;Dse`n~&hyV!o-`%|%dZ)WWFJ$*;^?r(nf
zd#{G+hP}_`#~k>wb;jNsey#hH<rwz$yx6&yv2f!axhIVKEtSglW|eHPE<UEU@69Kk
zeeRP!>=n_tw&(Nw|29i1c=xg`pT19he$k%1sK@(Q*Rj~~z2~&;cPu|3lQnUV)HMHn
z&K58BZrI1YHzH%@UI|v#eKtR2?Rh#*?qy9oW5ca6Yv26(=)KJMO7>>Yzj=UBRA67$
z&&K_Z*E@FQ8=LI$Q|sCL^yBP3w>neyiX3s>pK5Ptzgcm~zMoMK_EbiE+B0RA-htO%
z3VYPv>g^3W;b7PGcf+1_t2XR?d~E%`PUWwA7rijE$x{^CH`^=5y6AN4Uj4rp_ZFRG
z*;lR0uuuL`{{GbRyL$@1o!BeBk#&FXvyFRpcKkljU{<*M{c6E|OO=B5>Twz!*fm>x
z@2*7deH90q_I6%X+WU8pvaS83#=X)@=GpZB{I_@MnmPNtG>-1_S6H&YW>1*ilJ`ve
zO!p-0_x-zK@6rWt_Nm{rw>i+Ud+)nPN_!U;Nmy@MD!s3P>BZjbRVI5Dt+d<UB(Zhx
zmc`fi9{&-rf5I7#-G5d159FPiv2Q}uv%Qul&fDy8-C-Sbk73`ktu6L^MceFiPa4=Q
zE^OUjc7NU8d0F=l<Sg>ub4Q|QZ?K@oezWV7_qI09+uQy>aUaKt;=OTe8u#AR*<kbd
z1OGmYFRc6K%vim*VP@Cfu(Gdv-NY*Gb9jvQKE8Zp@BX#i2UajA?-4w`cAxp;uDzEM
zHt&7bX0pebmEGQ2WA9!83)j7OqCf3(zEp2hxY2u062tnvoGTJ{Cnz7>Qz28b@8!(v
zd#_nZ*<51Tv%g3yc`t8k(ScbtGxjdg+p>>m_47Ro^fUKHIw<dr^AX*<QOe3*ExBT^
zkng3v&KqX#d&jbPZ<dadEt6)?0mkp|_bw3B+uzX@ZF|6A{oarB8uln<p4b~}%ev3#
z^eX!W-4XjYhjSh<SQ}v<8+~tY@I7OPPR+S{J5Kx9?CYOqf9m$ly+3`TY(<?8?vsg{
zwpTHI{@zHDpOz)jEBAJNG~C;*@N4foqtv~|`%U&1b-l9Z+$*}ryUlLz3nt409_36n
zPd?q;m#p}DZ&233y-|L5Y(5?=IuKrZZ_o9&6ZfvMWZVDMhkf4~FSfmxErRxL&=21`
zx9X5hW%Idxa-QNgLG$Mv*dw&T?mpYHy*)B+cKg)w_oi>G+}HGZ>)tK3oqJs_uiops
zKYMRon3a90?V7#2-U!&fj#<3#WL@8$PntXS7W8D;x?Wwlcf~qp`v~UxeVd9-+H^U7
z+WYNr)1FBsclK`U%{Wl~JlCOHLudc3BB=usiSKtmPU_x2&z*fA+w`k@c*XYE$Mas;
zWA|g?o<RLIdph=u?weATyZ48$+TLbv#(feOg!ZlF-@LbOoyhK2b({9y>*BGy^Q&*S
z_3Z6?OWD8fU%~rhZ-`vjfifqdeP#Rk_bGF!*{ZGBWY0W(%U)*3pL>(f*X+}=o42=8
zYrV~D8#e30Mag>|T$J}NS#fONvBNWMs?`teuW^`s;3I37&HPiXcK4&r_kQ}?wC`QN
z``%lZnD=}Ql;5WnkhS-Lb<F-bUp=k6GEVNZKC^7!H37Fh`(~%_NtmR3&`^YV5A&8~
z_Di4U?aTWUv{xhV!QOdtO#2S4n`?6_iuFK#N7nw$b?FCf|2FScy7_Kz;D(%oGi0Cd
z+0}DwuN<@B{?A;G_MSA4*z582&c5@JA$yM|H|#C&`?oh*<j`KVv-x||-n`t~!<D+b
ze8cixER#4K@;1xuy*sJd`uh_514a+d?Op!r$ANjby7s0w^X@B7UAb2yYx#kR4DNfw
zb~4)bXh`n=RB5$$RZ#NYouzyCKDhgHuZ~TcjZ49VeO|>E_f|6G?91J9Z$I-6jlEM-
zcn_4QxZAq?U%M}^`^jFO=?nJmw3xMbp_#Ld^}}`hH|6l}n{0k_kIb>Dw&#5R?Fr;$
z+?RUo|NcV$V|&gjPPP{coVPcwmu>I+=}EQ+AHLmfb*RrK_uzbc^=A(I8C)hF$eK8P
z|1XBydkcc24xB#ld2i(;?tMMWyZ0w)zTdO_edgY2{uO)92U+c{+J0#-Padzeb-2L3
zs`+R4uA3RV_qZ+l-UH9K?Mc|vd?03W#@^XSWA`phdB2yL_4Hn;&$0*n1$OP3tH{3Z
z?nKr-A6|vpTj>4StF>vt-r~&JdkPQUw&|+4U{h!_*Jk~rqP;gF_w8Mjxp1GoVv%)v
zEQdq!l;-_6lR5S*lr1?R%6Py=Y~}sE1$=jRo0x>`Ibk8VPx3X#-s-&H`{xEI?VB@s
z``*H^oBP%}DeN_MWY`y6RJrHZgf^RXKLhM9&T83L@{en8;rfHSukyX#`+M`Xy(!zX
z56tumKM-jkvH#bqRtM+)qkEF~8QCtKePQq6_bmH31RM67igNC=?pUzbq-f*5zmv80
zp4&Xzrpc6d-@g}=_9|SuwMR<f#h%v_dTnRMNbNZu>1ki}>+&9p^#}K^4d1<QdhVy4
zD`sxD<+JqJo9n%BFKg9|y)|df*)PlB+qcE8%0~R)ntlHq_wU`mdyCDR@1OR1Sx?xL
z&-Hun>W~Bb`0sb^+3794zwXn;{a5Ce?|pMi_h7{m4qMig<NKDr__%jPUfu5eqO*HN
znO^KQwX}E8n09S%#hWdAeHO3S#}qbq7k7oCt<5?61M#5;_9#fy+D@4lu+K(d`<}~F
z?(cPE+O~JjDu+F)Wv+Hn(Y^be4(lASYt*+bJMnez?bJj2_Au_XvcJ)5v)k&KebKJV
zdrMv(+rzi#`d)q6Eqgk<1orN*<J`NTRcN0;xa;mhv3GmBH1_VfE53iP+>Gx1Vq0JB
zt$XvwhO@YQ-;|$4RyVhH9axdfwy*L}*Pag<oA+|4>^zXD!*6@?$ECexMVt4sAK7hF
zlH$GB$nN6awjC9F6?_-&ElZ!iPvf1$UNQb|yNmbR><@}P+9|i{vc1^L3p@D!P21Pg
zpu8`eF>Y_6?T)=mHTUlIyZ&;Y*wp^LMi+AS>e+Dbjh-)IGo|F@Uf0y-{dFY`dy_ma
z+iBD|?TtS2!^ZS_jO`It&V4Fu%l2H%VL7-W?5SPl)^qz_Jd-?7Jk4cK#+6A2JPZ%-
zE%;et6WOS7z@C+DpWs{Vy$U{B`z|iauwC+SlTF8@Q+wC8$M1c3%W<#f?g=)IJH7XA
zv}fLz@b9AC>^UFyrh6USYc4VSz^a}tHvAf^59F8g?Rnz)d~e_t5!)aIb9=X~K6~aX
zGVH51|GaP7<}h2=kbQe(w{G2gHgegXG<KG~PF`pBJ-Q{ZC;y#>{f6~3>>pj2ZF5SC
z<-qRD{Jqgem-c-X652PBPi0TMn9lCW{y}?BIn>+b^-bD+rJ-)`h3PW;t*nanGF;-?
z`$F-aeTR41ZmUV9_7{Dw?N#?}-a9F{Y1dl`&V4U>0{5Oi{&@d}>l${?V&>Uj7R<6=
zu$*a+&=1A~^Vz=aT|e>l-b<Z5`^&l}?M*wscTcqWntiQP?(Mm=gJ<9DoVQjEAsseM
zt6BHD)Ku8`+Hvh;+{wDnU3lI80}@4h6*OP(-l4SMz!JMWn;Vr&_uuq(vw3gQZBwxJ
z*Phzt8TN_OPVP}Z!Lx6w-{XC!e|@lF<p{B!JL}J0_X^p)yDN6>Jz_p}-%pJvdslpG
zv#*~hb|6dc%-*Q&Zu_n<T-odFKXu<WwS9XvbN21k&ik-eZ4$@6h$rg%e?O_&_hi9@
zeb?Hz?sL^+-M3L%-AXPb%RaBre(!3YEB0x6+xO1zX4~s9bM?OUW}<uVo_M&IX~Pve
zj>8l8sR^yx@A<WF|D=+?dlh+Z?#Z-#xAz3ou|4KL<@Yzau<aB5#JMj<EpOk$4gGud
zemvZ>T=U$XL!YJhPTuRXHy~y4-as+qy$f9#_o+{FJ`m5~wD;XRq1|3RclYgI`pqgb
zCc^%!z>K|NJB_R@T(|69)6TFz<4N`2y!$bGA5P`p%UQnOM)+LtzGvBoc3YeI+boV!
zvAt)$bKfTwx4o`c3+zoKJM7Mg_3y3NC$#VWjLmyw&-Lw_5+!Q;tNPR4x}@5@pL<U4
zT_C_>zwv{qtsd*Oy+u0@?>*JgYm=DUy!Xk6(tVR#H|<^Jx^u5@ZtuR{EgSY;wc55f
zum0?wy_==?&a(Kh@AQI}{o%P!_Mh18w*UC@>wCA|y|d5N|KQ#~-AQ|OKl1Otle%HA
zwam^vCrX9)wR1n;8xqa9&v<*`UfKT<yRNGq-y0XbV$aWOC-#1gV%pafxMBaUmLq#Z
zQ+`<cd#2et77Oj;-ppgCv0&PsxvBg1zW3s^na13|ug&ZCUbmq2dzZWH-uL&<{k`8>
zzwiCs{&i3NR))RzBMSD;=BnQJJdb(bmzz`fsT{a!fA^1_&7@^^wxW~P?5$5|+c$06
z_PqjsoAy}D&)U26(#E~(ltb)!z8&6cpf_c2zQ((KXZD)!ZT>geMoye<zn0{+JzI93
zwqFw-uun0MVXy!3=X*=5LiWnKm+oCvSG8ZKkaPbw(_{NjE@j)#T9moh?&#-zO}vNp
z^4(+E*BMp6??AupUg7S7J@?iu-N$yKV(;xE-Ft(BU+#Uf!qrwT(0Px`r*C`rRBzdP
zQ%hlw@<yZm%TJu&8~Q-sX6JtH12qpO?0r$R&2H{zc3X>YY<mxI@a=ueyTzVM@cCXP
zn;&}>7|!imyk6aA?V){p1Nl$vJ+y|))>-x1-n=hs_NC8d+2<T*Yd?Fgjr~RE6MG|V
zl@82Et*|b8yLaEDMXh`Jc6;vjlXtZ#Q~$nu&YQY@uYT>_`!Q?!o+(#u?X&+Ty63RX
ztX&rtu^spk!m!VdTf{-_x!b-Z{pouT+H}}3uAjO0;>_=RV%7xhe{Ej6U-s51yBGJB
z4xHn?yH}oz=|HEy!yb1@o89wHHSJ#`F1+XRQOSM$%i8v`K33oR_H54{CH)wiUjCnZ
z%A7g&sV{!8=fUBnd&Ae>-uvqQ<NeN+&-Ypz9N0U{!{7kZX@-3a)0gkBx?65DGgD`8
z|Lm_em$F{k>zq8Wci)y{yOVYa?EmQ0vG<bA+P&!}Z}*;)m9kZwthDc;&)$75O4IjV
z=G?q*UFC-TO2YeWmasPN*F7`G=Ev69J(0=``?6=xv-!Whbx%0&hrLx3&L0SU@on!H
zZ5b=a9lADL0<JqPukh@>Cz7;p|IF`uE#KK5_}pV`Q@Ua9-W*G#-3fa9`{X^=@4ac8
zZdYi>YCpkU+RoB^jr}^_^LuT!9oki<xMDARiqO7_H;wi>k}bOv+$8pWx>9Mw^WSgp
zE|=eX`_}#2d&+;k&F|;ldrfjHY#hwx_UZjE+q2|n>4CSKrtkHdmArS&;x4<Fj}7+h
z>#DV9UT|dZGLv>2mQ7-NZ|~Z?|Nq8iyCo!Y_bwCA-lukdk4?ml(|fxVTXq}N9NGQt
zd-vYlU&r@tTDxJdAiIVA9j^`h(_S|2-OyrkAY^Cb9u2c~`&Ngav+?fA*?Z;oW}AnJ
zi)@(VpWElJoxFFBnD^cTn>+U%S=+W(h4H|iYvpU~4#*nrjS-t>@9}-XzO^oQ_HH|r
zZ*%<Q<h`!zoAy4~5`N%_)fW3jW(NBobu8aMsa$sV#vK>!PwJi9`-Sn8b#l;kTRu(y
zy&NoFdt(|_?Ui06vA4l*nRVUU?|V7aR_uLye#YL@$=`MhGAy^*nccW2sp;E3iAg*5
zR$k%UH)+O<{mn)Ptc5RX+24)au*ZD&nLYEL@!D>lbi{sh!q&aND~k6j6kgpo!<J*;
zr-j$|BwqWwhiTL6y$)F`ZOa#&*vHzDU^7+s*8U%tjU8TlsM$(xVK@*cB)WH<<&1qT
z$!lyr%$T$5-`{Rap@8jsHTvB4TPkd^KE0#Cy6v~*ezj=+-OLwP?lsPTykC3DlD&aP
zr`a#nI=7D{^xWQ(gqXc&ueR;gy2H2k-O|r?Mzg-!*zc5dsK2`2{+8zp>xF#24yB3?
zdxY;E-Wz=D-~N*<hxgtpFx+#+ZSC&OFCF(fGpO!u-oUbNo~Vb-lWl@~Z+yJFTlfFD
zz5fp^u}O1w+%IsEd2jr*hkG8}Fx+==OZi@f^n|_gr>^XYo4<H(QOfPTrx=RuD;99?
zdoQH9SGeuzKA-01z1E*L*qG)Vx0$1OY|n&ky*7s%UhPw3=i0NSZR38?6EXYm`s(bR
zUS74|)l1Npop0$rmB?*-?PKQe-D{?9tMT*v-n2u1?CUJw?tS{V&^Dw|VsHFz-hI`&
z!F%J^l-c`<vhA}t{CRKhx0-#ojM?|r)J@!T+HTYCt(tfDZsA&Uz-IP0`>n3!_OrZ|
z_H)Lb*qe5v)4utU&EB@PANJ%qPd?zaX~N#9O~3b~Px0Jy{Bh4-)iWFRXdcwwtNe;(
zpYX5G*3K^j_f{Sd+Sih|V^3R!s(oA1!M*nypYQqgW5a>#9*q0`noK$1w_j^-o&B!8
znkRnk-B;JMzn8CQ?}eC6d;3x*?n~>}-n)0!j=d)N@Am$*wb~=C61b;EbN0TMcB}X1
z9E;uG7$CGy)BNq;meX?k*Izc;`zUL}-r~n{`wTYr?cH>6(caKor}uhYZ$GeSebe3+
z_n5tpA9vcdR7C8(_~7zhRpW_v3sz0pyX$7efe8zy?fck&b+3R{&0Zl%RvX#hZ2L57
zmLF&cj@$p?&whK?cN-1}aZRvc*}dL=g<tMof6+?oKaZ6h&Q#CX`>#4_?>kPHy^JT{
z?)_vr)rQ;dz+TB?%6l)m^V!Iq6Wbg2j$@x>@?M*ki<}2ap6=ee{?N3&X4+l*xjw7x
z{bc+6fZ!RieG6ZC?%iUvac{=+<@Pc?y!&RzzuNo!bnm{y((HRDyl&sC*D10$aK*$u
zWr1JzsvbVQFY{7`^>2Ni1M-uf+PBAD*i-hf&c1Qo-o5(QckheuylHd){tau}-HiLX
z^WW@Uwcpp?<f`Ky=k(5f*;BgrDO7*5*?j+m<pzU=c5e+h_Id8_wmVt#VectP#l3cs
z+I!7i{_H)!%VMv{%sKYG4O8qG>udLClux!lw>4ysblQgfI(kQJPA;9lck`K*dlt6E
z?LE$UV6Vi5ihXACeS7zo9k9vJxU#pm;*rfBi}QO|u0FkIb$!6z&&u64*A*G}eLM7j
z?*$f{J%#tQ?Av<Q?|Cs<+D_-A^d4!B+P$-jqW7wLDcZNn{@#;tY^BY{lZyM<X0F})
z^hv_r2OHV;ZJ45L!#yW-@6x1Y`;MMBwK@OnoV~%{H?}Kkqpe#mL>*|oant6N;Jkg;
zmL%-;z8h~-y~M=YLw)6*2@K2acivyM=lVIry$c@C*jGFI^WHlyH}@uVFSGyWF1OcV
z;S_s;zKQ#ud|JO}7T40f9$j<xRLq{fSGCdGe#wDPcKh_U+MCV#zVD*Z3Y&;)oCo&i
z{@q)-e&1e`WS0Z7Tu-fK!#?co{5xq+`o!ct_49&u`?Mw6tbOOZ=cCuZy^jjc>|J^7
z-X3@Mj6J&Zz4y=M-mtf@wRf*WlE?wam239KB_2PJYsj}RU47bKj%O0KMTuz#PON9!
zx4}wcx2NF0eLeGc@A><0_TH(lm2FBkH0=I1ZM#iz;?jLAo)c`YZM?XjZC|tfciXGG
zgReB%?_?F+cVy-KeeWK3+N_X%vzL`gYVS(nDSLHxEZ84e>ScW~f8yTr{x|lWQGH}%
zd;Z5>&aUrvHu)_!)AoJZAG>(TzMn;N_bzLmy7!nv*WM)&PxkWr%-H*|X~|yWzWH|k
zN+;}ASbehj&U<}-=gX6O9v=9;H{0BCzYq7My=$gE+q>ZX{JqD{3EJ*F;;>ianEhV6
zolSeqi+}9?>BY29OGbCEU^n;Py!V3pZ-0KgW9@ST>$^+#+27!3+B@?Glf$~$2{!)6
zEB9K(zPCAag6+VATkrNB`EYEn!3>H0i_TowqjG^|pQH5Az0<ni@4k3)#$LOuj(xL!
z3)rsu_iuksvFZL`|Gd4)XLKC4emC7?tvGWZTP^S2D;X>Hnq0lS=ePRJy?fS8b$Iad
z)1EDf&-Pwr>a_j$`@x=DE28#(YF=m0kh5%W^mDfZ-<HqXH<gKZZ^mWI-4Tvw_P(5A
zx|glla{mF@U-phR(g*4bR_!;?yR~;Y`xAQ^?zX*4^6K_3a5}%w{9X56+l6cPKD@uv
zMqy(6-owQ_`(EFa-s^C3-QML5i}rquW8BwqO~iKX0lU4sqq^<0<hJiU#k0~zKeB88
zvyGN}%?~x~KPi4@ujo><z2&L>Hb!>@>^&x4-kTI!y|?Ae-+ixyzwe20$=`eInZe%I
z(ckv+6jbfKZE|^^f;j8G;^Tkp^Upll@A|f9Pfg3F{Y$sX?qd*6+vg;AZqGu7$$R;2
z74}7?|KIz-?4Et_hfjMC$ll${d0cW|V6Nrf^5S$G2hAA!747YN3pF>`O2>)q74d(&
z*GF1s@55l_y`Lo__S|{E=n&Lwc;MA3!2`Ehl<aSPIb<`-OwC?(`IX%hyXM)1|E{pV
z)w6o<gAEUCR(@~Yx9sAZy%JrF`#7F^?`iIk-oszBe$Uz9346mA1@3jbAF-F+)^Gno
zlgoRJ<)!xeE3UPRcAvPHIU-?i->f@()4Uk=oj9?0@AL^y_F^Xl_x^Ha-FJ1ym3=ld
z6>X}7-|S)M72EqM<c(G3bd|lE4g2?<+LE}p$Kj^k3og-ppLgxr+bw)>|GW1r`}UvO
zwa;Mf8k^(4r|wyyrMKs3jKp5X-+BAa2tKgR*7$BCmU-JUO3q*}_oLvw%#Y0uRCfK|
z`}0zn{jXCId-r&3+p}xt$-UeAiuYQ#pWQoo)(yKWU#j<)$ci2Kn>^QEF>8bM>ztqa
z=P-BgJ^80&uVUuz1Hz%x_moAl*v9T#y3hLg3LDlpmv=jq1??4CxXxyEtM8sAhraBQ
zeCWN`Q{vB_lO~Jywy;_3{i3h7H^{ty|NBM1_VzJ}9o$mVWi#oU$li7H1NPoaJ9D6O
z!P`BIyo~!+Kl-t++wtHY(FK$D8eHYsx4-{@&6KID_pF(4Vc+?6fA$J7HSObi)?>#I
z`F5{bVUR;l@r}K&k`wkN@_pUQQ?Y7q{p?qJ^;hxj()68Rzs&L5p4*!|Z09?v*oPh7
zX#Iof?cROK@eZ4Iv+ldk#CAZ#WB<Oag2Q_+*x$67@^Z%B#l`3Ly8LB5@am41-Lc<`
z?8Vxy+ZQB+?Devcw4avNw0G^_w|lo4ciV5Sy=t9Ow{)-DiF>x2j@;dA|9HyYEjtAF
zeLpvQ*OWWA_sARku=;T4*IsF(_j^7yeA|CW^u=DmAYt3b4=f$5#5DIFOksD3$P~8e
zH=Vrq(c>F7zbe1jpI@+eulb5sdn=3g>|?EA+so(EWRoP{y!ZX;D|?&x7wlbe;pD!j
z@22do>Xdg_FXiEIFj9T*^&5u{%v)!*_xQqX`%+|+Y(D=GvT5r*v|Hkl{oeXRyY2VY
zdGGa}mbWK<x7GgiOSkuSxy$X&`!LZi$ccBan%YeJqRP|ze(9gx8^^=4=S=ssz3FF6
z_fF7d+|R$(Xa7f0Py1K>7WRub{NFo2yvMGh;jMMYi2|E_C(;jm%+A}>_2u85@<*%o
z*^39+aITixE7SI7Z>`V8y%Ssb_AO}sx_7?sRvWd1$9tzUgxhU!^xb=V&8EE`?=tM)
ziFNH+(4KJM?+WfclbwI=HD$iNCq3A1|D}to_tqvK+N;&0v_CBC!(PP~@Avx7F5BCY
zwQ=v7KcDwB`hVE>#6)GU<3SMzNlE<!JSqbFq%uSg+<t4mH`VCmzSR>9_WsvdVdE>j
zcMlK4<vq?ZEA~Ixz`Adr>)X9Pd{z7Ira4$MJQub3F?WLf(d``j<g#7%8A)#6XPte2
zZypo--u1lvdncsk**I@)+JD@;!QP3r#bN79XZskA#y$RDGaXp9ChR@uG{+{yV)Fha
zk=}dnZ(O%mi-TeRYaOS(Q&Qhp7e6SmaqH#Uw@;&Y?|b%;y{op`+ujd$*;lz**uk2c
zecxfx|9kHq{I_51TlC&rV%HDMVrSgPadVT+)QDYs1tm|}tG-~}Ti^F#??R4E`?@MV
z?)|f-a*yD&bvCQt-q`!%@8!ML5BBe4@;<fqp<mbj?X1E3n7+Q*vs01VE_>fn8_~Cm
z_bCVn?EAK>Z0{5O!+ULvGWIMq*}p$uXZ_x+PaF0vj^46QNW5h)|I0^v|2z=c-_owL
z*LA_`edZZnd*5yTv^OAd?%wJX*?W$)p4j_Dzjl9-A=81Xi*NUwy5YS45$Bh^ZO(=V
zX50VXGgI>7UbfYf?b4hy_P#G?+_%qW-QN6*eS15EnD$ja582ynJ;|nI!mYinpNnjk
zH%0A@tlPft>1@$`pV!9jJ#2k#FXNmIcHH4J_b&Rrd0#I#>%Km_TpJ&S*L(k&zp<-z
z?AY6xF>$YL@P&O!ccS-xb`;!uJe_49=bKqpf6Tl0Y+ygKPx;=Pz3q3N>|fN|Zg(sx
z-FlAj+`XljckTIn{oKCdbDQ?Q6uD}>fbFmK?*pgy3h41W+<U8O^TcMNO@h(RJz}=4
zdl|Kk?EM%#<-oz-+k4+%mfm;%+PQt7GOYK`Jmha3uw$#utkb;vI5Q8~JzFVwKw);T
zUC+$o154^2?e^B6Zok!b(%#9(p6+#ev}eD?4$-}fWeWB<x=-BKWBYi|kCJ^hij$c3
zvF9Gzo6-?u{qJ+*-kB=j_arZ7+$VT*_P&`11ot_wn!2auxrV)h#qT|@HMRFQ&aSqZ
z)>XQ<sjPU<-Sa#5{py>rcUAn#y{rB(?pF!hwl`?<?7gbfR@rR3?6~*KL8X1^5*PP5
z7DrjVJ9XCna6<mRoAayo=0xq<cg{p_@Ar(E`vjZ#_iVfJa<8F$$6n_<|Mr@dJKBb}
z)$OgD)V7x`Veamu$8`35S#WjFu~R4Q|E|y3BOqgNV3WjweP^$P?QKn&YNIiUYhP;Z
zY@64W9S2NIJ?-E4?X+vIK5VZQC9-cq=j;Pae;N0k(fql0VdRg!7rQz3c|U8|^XTMi
zTglCHZI;hoy=P1K(mh2X%XWGuJMX<Sw_uOQoqv1t)939AcbmRHPGQg9kGH+{dY)|E
zKRr`)Z~OCv1If&4R%ajg+VJ+S+uIoKXrG*3w71Rj+g{<BCi`q;6l@C3<7|t*JlG>Q
z+jx(AYn083*_ZbHlJMLcp!0LT(DEzxzy287UhaNn-*PW*?=-#r`-0=I+T0NPvFAoX
z*Ph=@SFNkv`1Vh26WeQbd-Go1m^u4aC^uODlIq{PSxkN(kKv)c8%w+GKWR_fCus9v
zZ(?8a-msL()@k8Ct-pO|Jz%z7@4&b6#{Kf&*X(!y|77ondF=-loLRAZ-zqtq?pHGV
z7rM^edwombUi)U|eah!d_h^24xo2zQ<h@Oa8+S{Y@7lZ0e(T;zcKdfrwq@>>KX7$F
zcOn12pI4o1Y<|bvN5qxv-S9ulzIa*t9?>n6Y$SA|_w0B4XU{p~)ZUHrm)kVtneB7-
z&D!&kDQ)l3K%RY%9<AP6RU2r-eP-jn+23~TSt?#>KWFiAyDJW7_ntb&WS{>)Z?BBu
z<b6dKIreSey>74Jf?a!N%P-oizRJ%2p=8ru^QjIt*BiR_h3*&GcV6<RP5XQ20~?HQ
z?hV;Cale>Y?cSyvA$#THyY^o2?AiO_|Lfh8Skm^hp4+=$ZJ&ib>(&`IFRvWl^SbBi
zfm<u9Y*@8*_FfKC+sF1Vcdvy0vOOz}cJFg|^>1(1v2A;eRWIz#3z)Ol+mdZxHJjev
zb9X20vHt&i?;pSQ`@J^1?(SvUZS%^@>Og-n<G#0#W*=~xGkveXTaLZrFE8wUf4zEt
zCCiaLa((T4r+Piw$Ni3V-xUo5Tjd!S_A-PX*c)Lmb?>*p&-;$qb?r5Kp|XEP;Ya&p
zZs+$JtP|e<Y^~0o4^<ELO}O`G@1cxYdpUeo?iIetx97pLIrf$Um-g;#ySZ2W{gi!f
zTiW-YQl7tyXU#WzUy~}Er*EFyrl+;<ZRl><lYjpG-o4+o_m(Yrx##uguKjPGEwgue
zIoZDZ$U8gN=)^sFJu(N6P0HFOQu}nTu6xq{E%6-tzMoXzdn0w>zLSX$_j2!N*d2B3
z`QB2~$$MqowfEjgdb&5$_J+-K(_8!e9xd6QYZ0*bmG9?0Wd+Okb5E7n%Q(~Y!0+Jm
zdvllj?sc`iZDUb#!(Qh!_rCDT<9iRyf4Hyoz_GoKo0IppURJf?|LDE<nij*}i<6h{
z>yVhX_tPVH+n*Ya2Wn;<-RmR2+wP!Y+g`=WiTiY?l-LwWGwv(iCcG!G`p({YH?!>H
z<dyeco@cnraCzarvc_|JpYD3T_hR<?19L){_icLf*xo0H%l7sAS$oTN-`IPHHDQmI
z-HpAn7fkIpz4)>3fZp`|ULD%@Y_}ZtC>V7d*tq`5-m4xn_U_`c*#B(G;l1w;{<8_Q
z-nGwJaN*v2hyC|D+?lfXNI3gGd2gXTCQokdm1Jq!vt%>RzTOGCwu_J3?Rofo!CrU&
z5Bn#k*VtU~pR~XH?&Q5oB$)R0$m?6P<o-V(v%`LG(Ve$@i(f6-mpN(8UYAMS`&RJY
z-22Gt)!t`+7u$Lt{I+lYsY90A^EnS{nQ7Y^JZRr5dR@)o+64uh;|eqP<=k9o6J1!l
z=ZgOM-F#uc_h?%r*~cDTuy@aGZrdd@xc9kznzEPQm(_OGnXLydeu~<=anZ8<DK_!@
z^tXMta<yH!_rECTUZ%4CJ%V}j4!m>?bx4`Vy06#W*}+vX)MnS&qX%Y~tl3j^wr=lM
zM{)bbVNQDmAFtdyv4eTv%d&%ee3=jIEzOMFbNS^etM$hg@72DrX7B1RfA`*-xNpzz
zV~eaKxgP92dhPUH&F}~NcMH4j<+`nMkg29*uh8F>dj$>)+nQWt+V5`@wfAFP>K;#>
z|9j7DDcP%d<o({Je&fB1COqCND4M%h;=sv$H+#13Q3~Wd5THKIVNQJh-e12N4~W}-
z-D|XS$-V@6o4o?z&-XsrD7Tj@KVZ*|fF}F*4hO6Zqh{<=d8KA|CP{D~U&EulFL)Xq
zv=zDc8P`be*St5?_S($gJ>2Xc_Z}7E-Frs)#@_Dg2m4%4MA+Z-)^YF_nPe|e&bwEx
z{LX&erx&gNDt_MUb}RY7LuYN9FZN!0wJ+`8XKwO-@4KRNd)<!A-W|$zYR}@U*Z0<3
zV%S&s?7&|0>s<S;-7UAzGMl}-eeQSbgmR1hqAauaG%sO45PNLt-m5X3`_vr{?@f9w
zcc9E&data^-MuTg`1XYzIkbn%tA6hk?c2LIlv(YS&-uJ}!M9cWrZgzqiq&4UpJIN_
zF5ffI)>@VIfVjrIz05mj?ORrOceliw?mbI(CG6Es7T)(F`qe%LyNP?=%3Ii8-m-jO
ziJsiv8!|g>#4c9t->bT7?}qKo4mQu%?aMl~Y;Vo;%{E$J7Vmu!%)77S#6qjuf>isf
z_R)6F9?shTa>u{D&MSNmBqv<8IrTMk@4+e0_w%lMxz}eQ@7{B%)AlA$J-@dnaFNX?
z(Gz>j9YuEc+OOYZJhyl6)RMD%ZPqN?E5%>3Up19+-wi3YJz+L4ZSRVA+N7Ig*mETA
z+{5d>%;w1I*?TYCIBL(E(YANxFVnq}kL310eqFeCvV!2=Et{+NxH|9HeO%<gUhS^=
z`?SmL_i_XuuuruX-#4d~ckeF$aJ%VuGxu;==<S==ws7yd+_=5(T9WrFOyaP<mT0nn
z;-{{?ruKJiriS+JU7y{z+d=$^O~iabhbZGEHusz6+JB$GXZw)l_}<Rlm3xhi()Zr!
z+q!3~`I&t(4>#|XYnZWbiFD3^h&jwQOtlm3{&c^$5twpn_nx%weJc!??D@fcY446b
zNjB!||LoP5TDRBnX6W9iB@+9-K9Sse)_cCqZhND>?Yq;gZL`D=EZK2#uhw;;eFjqR
z_Fpr|*~^~0+&*<y@7}&lp}ig)i}zlrir?dE*|m3p^@Y9L6FBylCn(vpKj_$7%CXMo
z;k6}upV@Kkou_bR-_uuddsQRt4)pE(X;<>~!rmhHefE9l1Z{VOpWF9#uIJvTKi}=u
zK6z#DrHSc#J4HS1?+Rbq%Q$(>UZ>No`z~0o-n*mg-QERZUI*mPnC*VhRBP{Sy<)GG
z$)vq&C*QV~+a$QJY3AO&L9#RV&*<D^AG-eK{x1^Mb^+xZtgq>>Jm8XTY%5>hyw|+m
z!9iY9c<%&loqZel*X`NHk-mG*WafP?DUCL9e;fAB`ekQhD)MH}7K1r^w@le$onQ5F
z|BP4td%bMe?fGoZci{2n8++~F?LYAA*t^}&GB@vCK7X?H>Lc+79^|R-jaa;3@3PXk
zz2`-l_DP7l+px~9-+S?$tnF3a<9ppAkL-Kyv&cqDTH5}}$>a70GoRY*?7eK?xHxF<
z(Y{stn6s4k9sM)cM#wkFHbH@RpVFKM`{lm&+GI`Ix%cCwrhWX5UVG*nS?{?&L3LkT
z!<)UE)^E4(xwm_tmh{rSA7(S#YF;<Axx8S5)!K^h2g;B8?+;(i@38d0@BxRJCVQWB
zA3DHPB5Lbm@nY})3#kq(gRkuQxlC+d@Xmv~TkfUUoczJP@An&yeaowU?LDcoW-s?E
z!M*R^Pu$z#+_ERQy=A|Wz`4DbwU+H!lO1v3(DLTJBDyd3iR`oAdu8d?z0;ri*{nG9
z%APYz-1gPn$9s9*KJI%kMbq}#8%Nt}6HlAPC(QN+{c7K9sB~yw@w;7nO|GZdC$jvq
zzZmghuQLzNfxGSLHtD8)`)cm-?0qb|aj%=wN}K)(%WaCk)b8K(?$qAh=^yvxJQLgh
zmoH(@V@vD3uaZyit9;qEmsQ}#ejT-=`<Ak-+-u<7ZrlIr`rgMge(cqd7dl`ZSh;`h
zQv-(wdwll2vHP}X@1e#$52Z48iO#p(D<$W5fGd0P-ZejX_PKJN+P6J@^4@O6WqX5_
zx9tsCCcSs+Rk6J%PV?-Y^5*T{q#B96YRQfJs@MP7d#J+1c6m?a{@b(u?%g57cwnc#
z;J)=*se7EW*YEY`{cF$7zh$r7l%IQ!rF`G_^Xitp`+KwZMsNn(oH9PJcgAGfJ?(W4
z`&^<d_LgnR+HV!sXS+NrbZ^9k!}}xnm93XD&D_WA8n<^z^1eNZ-<xf&rA^)Ikg0JX
zz+uroTb<?mwr<+KZ+R=jJ~igMdyD?G*}rs_-rcW%X@58W^1V+wkJ|)Xt=Sjw^WmO3
zbGF*#Ek0qVs2Q?fyo>!nXM*+q`&zsAu4fapKae4`_s~hEy?!Dr_NDT__5=%w?Av(N
zWUrun%O1C9hxUFv)U&6>Xp+q}?HhZtuLkXnST%cZwQJAbnMW2MSZU$2H>Os?*7fG3
z{h7Kx*2iq$*<Eb1w7DcNX=8A*XD>fn&HiNzuUfe(M(@2UD!8wry>a*3CjNbgmVDV;
z8$4&vLEExDw>M4N$IsDZ^DyS$zKGj`_BQ6z_p+Hq*dGiJ-TS}u%D(#voO`5=H|;(b
z>1Zu|>+0V7KY1PUmaN&k`gYFVppt3(?zP|CyCa2rUq(-m-8SC+dmZw;><+h2*q0jQ
zX>;c6^1ah6`fW~Fx9t__DBpkn+{yiy4f^-rUAo0yWns;pRAaOKi+1eY!(e`GZ^LoH
z1G71&?y<9O*s~?MYM+Qz%$~15YW60(a_&)K?b@rk@BH3BUZ?iT-dVoq3=`Ady$!kc
zh9<}Ne$eo<7I9~Gn4kJ-PsH^VcGGnC?Rms+w>L(mZ142Ip8a<YFzs8ZJ$0{^zVQC*
zFMjMTP)ObT&u{DA`In3LMoc=gE9T1NePSnN_Wr$aYj5`<oBeP0m+fXeqGoS>?(*I{
zX5IS~-uCXDQMYn$laus5HnHi}^7~Ef8(S^*rgC!Z`+eojzG>Dydxf8*?%~T~*f(R3
z(7vF+hxVPPv-Uj@nY8=q&#!w<JL>M`{mH%0xN^z9vW~s>68bg!^K6UlS6zR*cXdPm
zK4!U?z1=6}ZStSA+P!DLy4PS%w#|gXo%<FvX<C1p^Kq|4FzY@opWimZ0cCryM)mI%
zYV5F{lYDLO%5%#0^Iz=RJMA>rzKAGGhj-ekdzUYe-WRf|cem$Rr#=6U30pO7d9(kF
z6vIB{z(+R8xxe;JF5}oYyXE`dfaA~h>RGt$4V}!ekD+hbzT7A6d%2#jwm-Lo*+J~&
z%Du{}ng_T7iuSM+Zr(RzHT&LtHMV`1J*Mw9*8RKp%K6j-{a&|sr)*ulcS+LieRm4k
z_AxJKwSBNabbr&43D&~Q()+K(F54F@^L=mEuJ*nDeRKBCy{@qLL%@msHx#$+U&@^8
zkbf|6f2t$f?s-Ss?e7Ol@7cSGe_!wQ2?sKt%-%brP+;%tg@^Z+?f<<;`UuOue#c*X
z&6sBIeV@|3*SGH6-sIh~d!zPr?49p@(!N`2`mW>FoA<u;^*wOi_Q~#x7v}93dN$ws
z`Qwv&IXoxXm}qF&2O93M=}xS%UbI+zKUe<Ay)W%jY;B}1cAuIgXgh5U&%T}2tM`?N
z2is)&ov@Rc$Lnx)hSc6Tt6uwF{?5H0_66@dSE97<!2Jn((~orT<($3K=47by0sW`<
z_TIVqXs^1?@_hv#6!+$N2kqU=VC_)(jB8)sm)QMb`xoyitYF#8CYo%!LzZQq!fX3I
z&et^$e0f@V;I0<OK9z-w4!n8xYmZ9-mz{;G{9e~2|2>fnF$d(IEAIOy^JlN$yTHAB
zJnrtzJrQ7&z4ymnmN!Xz<LBh=eb=&XkA6nc-qq^-dre>D*_ZWe@4dvtXnXYRH@kyU
z27CAMOE`${m)tYgL}71z>!Q7;eDCaEf4{M-)|zvl!JO55)$3Q=n3<;T4V|fB)9$Og
zH&p1_-oVPPeV_ZM?One)(|($@%Yj7o^u2Q;zwg&fop0lR<;cE&6JPGVePhL5zim2u
z<@dkas~CNIZ>3%P-l{6&y*WMm_Gx$?+5PiO-d=`HYz}5V?7Q_2-?E#X-o3Aa{o!7x
zzPo#Wo?zK0+|0K(?SZF#cT<(!dG$GVr=Ms$Z2YxiuYuDnd;bqV_Wn2hvDbBxz`ma!
zRrYix9o`%M?CQSGfDL>4k}mH(d4zx8;s+b{ihqB)XVu}ydqYB+_sTcx?wwQm)Ry^l
z&)#psU3+)9aXIW;5xGZv2dDic{f&EF;sf^@n{wJL_+xWm;=vVr?_OfqcSi2RzBvYl
zdmDVt>@9azv|0J+;ofIERrgs{E!p>--*@khWmD{RbFUtFl|FTEY~<?w-xhi9<#g@d
zC#}o0@2gyg%_nQdec$f6>=wV#zyH>`347P3{<FSy`un~|0Zn^a=d<jKEf=ugB-poi
zv-E|1t21Wq>&{`=dpgW-Z%E@=>w9Uedky;o>^poq91bz8us`J#vbXMD&)&TcQV%Gu
zYu$U};@dq}H76hFSAV%%k>%&!&-bV9Q|J=e%X94H-hCYV_kR7~xHm`f((Vmn%dNVz
z*6sBboxPVib-DfLKUa5GX0G1LP@d{gv**NKF2jfRPoJ&bJG=eqURw*Lz024Y9r9$^
z_E~t$*}HK8!~Xl*{_oAKi{Dc|seMmI;nv-Uj-TA?_VLL+zHgm-jwst7m>$M_V9OUP
z+i+hK`-e;X_IkU_-=}?vf8U0~%WW2ko!uKa@9^Gv@pJ9+T-RDZRf@6gseG|lRKDHD
zmE-1K-qTC=t$Z<gZ}^K12V8<O_Qf+X@0%VmW$(wc8}=^N^xtz$b;<$d0}b}drjPA!
zJo&Ic&tcErTj_rfScr(({+!Ram;ILfJ~zW1d%ugR?)!Z3uJv@+H8zqAQF|CqtL#nc
zaobZfvwQC~A?|%XJra9o{^Qv9oad^&+rAxpOc(94F4_Bk|G6uVtff;<9bow_xKC=v
zzTMX+&)BP&e#3TE_xHVQ@?ZD*yieYD=5^fO3lZ1%7SDFs+p=cv-V(WUdvDA3@B2UV
z+}@kzfp$rr$MzRY`D_!y>tO%TV}^~Bo9{l^-P(ID<cse4^zqJKfis_X-JD`}ATn_4
z-smX9y_)<FcHcDpzUPe5<-PwqMfS^FciOwZcHX{Ywxav^y7unf<1}M0_xrPZBSgdY
z-aowfK(Zjq0rP|u`;MqKhhV>>d*(2=9AKX8WOH2P-rjW%9`-i>l=qtMI=ko0yqLY`
ze#`IK8q2uPXhV*5n81`hTcaEI)~IvuDX&!A_d>*bZ+v&0-6V@lo7yRod)}K*+@G;!
zg3X`k(7g<6xb{Um9NjzV`S!iK_B;-owgm5ex_iQ2!#_*+Dzpk%aj4$e`~Mi<z8t=X
zdsCJy-0M5@<i2fJIQN<yP&lyb!HfgJp<8#R#zojSDKqTzZd|Z$;<DX)8GSF<6a?Ph
z6TIWv-u7s|eQWHZZNtuZ*z{KI*cW|MX|M0?lY1pUD(o+n+_2a1Ui*HJ+SU7t&DHi)
zDdp{r62G+PcT9tg@{{fRqn>ZG|1sJ1Kt@%;{!^Eo_a3U;c_3Q%<lX{X#(g_3i`su!
z*KfVk+SPV~#+H3`?@n1)NOAA8*V(r>fLC|7Ma$Z~dv|ZyJI!F{-V4_r?3Ie=b=V=X
zVeh$T7xr%Jj&-<yEPQX639~)>g}gnUVF&k$*T?StUG{wcD<k*4cN83K9L^rvx5vnI
z&w(i)_E;S~zvq&e`d&8)fqnZfE#8-Riq)p6`jP#HD#HWuV)yr)?g`z$Nxsh}M)|<L
zTHZ;!B^Si(y(Gu7&-(AwJt3a!?VpP=+Js#2*!%D5@qOyc_U~O}a?(a|!n6GWXX<U9
zt*G7q$p7HJH+@lizdh#L>t_FBPXcrDUTu#A`%Uw<*gHSuaFA?#vj1A)i@n=TTO8JE
zZQd&~hhbk}%(ngda<BGO2IlW|p8C<&@QBD>hsoKyzqt$U(^qcatGRaFPS)-3_BPaN
z>?=yyw%h$vxP7P3R-238ZtY!ZR=AJz8H-Jab>%_ZDzUxi<L>UgZrQs>ZbJNiRf)xW
z_u6Fd=@8=I|2^gK9_AZaHWR~+>?xMu-a9e*(%yhlt9>`rChTe7sbVK-&aq$1UcvUs
zlX?5PZ#~^x6}e^KyX*V+eo5T2_jkeOy+Qv2_vYl*+ixx2wCBOf*1Z>ub@xxseq-ZU
zwta7;bNhblzZ3WJ8Beg^9zSK@A!F6OdziEKa<Jdn%aAR#_kz--1J^_P?LByR?Y~oc
zfB*89%)RcuOZQz_w|eic*A{z!7|+?C@@C;~vsF*_GF~{lXWn79ea>z6Hgd{a_N;fm
zzT30o<et}B2lqDrxv<x=Q-7~nNua%<?6*DAf(Q3rHx_eP@bTW>6Z*;f!$Vv4bV_sV
z^$1|wyY=JV{TAz1?LBk2-X_ZI-97_Lj(x4oEc+T(ZQE;mLUnIn&g8wE`Ahc&ZM?HL
z=^>AUOz-J^&KF<rX+OYpV1?06>lHWl?fYX8w^!|4%bv6AJgg`0?%CTh-+q5pm)PD5
zXF~SICOzNhX!2^WME?Cfy?K)zVsmfpZ8+$3pde(?zG%}sd!I2)+#~&`dat^~W}6+0
z<ZO$#@;WH&nZCc>?AQTO-K%>Ka=Ywje9^r($G*g7Q{0YyQQ;E%jGt%heP9&6_uBVB
zn?Krf_ecq|?R{7|-AY*X(w?e)l6&J;pWnMtYO~F)>yihgZ=c$GHim6qYiHbkBl(uS
z!Ou(&Y`G)8uR`GQ-iO*=HbOB^?bDw*?LB**ec!8%SNBC}UEaG}U(nX*Qm@Tt=KFiS
z+n4W6PTjcgoFK35w>};FZ;{LFkBIi}ZIqd0f2U%p)#i+;`~FGY+j~{u+g{0~414c9
zpKGHexY)juY0qvQTl2kqv77d3aJ%m{Z^_^LBu4MR*ZMPim(S#J5Nl}P*C!ffQ?0UL
zZ_p=)y($tBdxf4wIxH;vu>YN@^Zu3vSNEQ<`?)vA@UQ)=*cp2+wl3SdGBN1Dle1g*
z2D#6)*|%-szPpd7+C2ETZg0dQ$31KAGunFV^zCImcz3VcR>wUvg*x_@pPjI`;vu(9
za+cse-+8(Y5ALPd90*9Z-|Zg0cgES(d-J|5*|S3I+5QWG%=_lM?c7rvAisaP>Ce4K
z3?A-XQ0BPzu>Hn8D=RM9B)2}?ca-(cPDz&O2PRet9`NVgvpe}ir#(|y$DW5AEc+iE
zIkWdfZTsGgbC>p1=X~2+eR0R$xLuO_4oMr>`rT{aXI3?5?~VBPdu#3Y*dG<**%y1w
z{=nZ$F?-uLhV4=O{&epvX6Aid*V*<h4innnF4D5!UtRn_FAJ9g$JFk<&n%Abk9{_2
zXPI`_-m6PR4sPL2v)x;M*~WGHw0(U?GIwv)c)9n+s>gdy-*?!%FNb4a7_Z@;1DvKd
zXSbf-o5eV9fAZUFd+*vX?fdXmeE*YThrMfOo<9(=m1Xa{TSxZpSP*Kn+@I~>s*7{>
zbgh`ZSN!SeeI@QMY_itQ+<Rl&;k}9;Og5Da5qodHUBA!6s%CGTeyKf!JFETqu#0>B
z0}JfDdluMabf4Y#nUP_it*q7_v46Aop8Iytx^RcE!(_Imy;noOSQ&`%?|+vjYcrqi
zt&Kox>Yk{;SvH=tj@kd7ereyWkn4MAXCB&P`fB>#z?WzCq}*jc@ZU4np10I*|Ew#{
z4rSu1d&MWT+aEAEx>xXZ%ijFuI{V*k=h~b6<msL=MpfJG)5LAG#jfu*xU_n2gyG5E
zVK3L&%#>5vo3v73p99;8y^J1lwwghE_D<p2ws-QZ7yI^X`e>bd$ZKzNkkCH%><7F5
zZu`4;|GLusEVBHzk3H=6-aXR1Z_cmAy?eS8ZPj*j?d!hWxp())UwdZHJiRaQ)aTtQ
z5)Ka6nO5%mZ|!Sy)M4>{JHEW#Gei&Ui#l_B&$SCCd)M6X-ODL_W^dEk>-Gx-WcTen
z^~Z*Z+k5}H|LptB4=>w0C9B8&jew)gjRtf3Ukmd0sc$v3TJ^ba@2NIUoA{%$dxbBA
z?_1Zo*{<;1)V-E$Hv0>YDOoK#6?<T%(Dc364~y+mWSO(yTx#atf3hum0;LY`GtBw0
zcP8WOy+2D2?hU*8d2fC9i@mGar|!L=b;d@7Veh_2dz$Qwihk_<>~_ogaEXWgfhz@j
zE6lF%^O^p2@3s8td(F1K*t?B?)qaf`Pxj{Qe6)L=3(NjYdDeY$6^Hk}<XW=#lli<o
zlOJmDmA06(Z}Y!1dlO>kJ2-~KA28wX-#f85`ar|wmc5k?v-gQHY1ua14zelu&$ZX}
z@Y20|@BFmC=NGv*uVTM#^W~NMSf>?PKK{pMBfF`~zC2fR@6?IL`-4wiwBb?xx0l)A
z=-zut{(CHh?(Xf{|Ihwy-Cp~77kV6SEpl?G@|d!Rv7o_W$*h3Ae6tz$G4*>LxR%Dc
z?~jA~-tDZj_ubv+u-7TQc<=6|`)!=d`1Tdt=(6Svp1!y0`y(5P=C^w<t4y-nS?{sO
zeCqYRm!4W2VBq_)_uNV5gNIT@_HE;Tx_8r|^VZK-Z`l9Fw{fqh)ug>IUBB<+dMUm4
z%QW_Vf<15cp8YMmH=*kD-u<<Q_Ze^Q-MdhO^T5&BHujPCr|m8G?6l`}dcOB}Y5hJG
z?E`yymfy0Od0*Ienswry#7z_SN9|$X*ScQGX2OrAeFx0K_gWgi-kTZ{Y#-^*yH8`6
znce=9?RygncJ7V6)3h&6`PrVOVm5pHnLiy+`9AG{$Ys0zEj>vN4u8(=<>F8|_-E?6
zy{m3++xx5Ab^kG`XSQEFH21me5wlIWacPga(9ykdZ{P0OU%7MdG=|fA1GDvQEbq1L
zeO0h+?~9}L2lOVd*?Xc?WpDkx_4c~qEPDlBGVj-P=Gk{Co?)-x`c->x2T!qg&<o#V
ztvPA0@YUdb8yA1yBT=wnuf3$w9>x9)o98L__xwm+wr|6(|9e+vSsrLJd3&J$Lc(7A
zT@DV1=kM6_Pv*cr4xx&@XPoEny`;0lX8DTMdkc4)*q7Yz+WRz5&!#pZWdEu?x_ebj
zJNIfW`)V(_qThO7=)b)VX{-0WdD**X_U3JSTPN`Dz3nw??}@Mb?E7t`9nz2V?QeS<
zaNy$o=)GGWt=nI7mC>dzVEf+h=cEr5@^I{ndwj~qf5Z8G`M-DVwd57r=h?2g_iQ%T
zzD+ML?e#O8uve;|bzhijgUwY|nY|P2kL}&UC9-c%5068}bD@20OLXkL-mC7*2;01u
z>C_GzC4bia%Y~%&iKs8z+a|8Jf2l*w-n|7;Hdju}+dJ>phP_^&Q};Of@7#AeRKYg$
z^;G*K3H%4x?C0)b4Q$=(E<4etHu%Ioi^#6M?kx(ouT7HooGW49Bg&a$Uth9$&w=K=
zy>aI|_SIb%*jIA1VDF@hV*44hHTH`5q#vl6$gub2oT5ERs*QU$1_tf*<KDM-LaMj@
z>HWe7u4KsCuk4I*SY#5ir?UQw?cMK__w3s;V^78=M~CIiuXo#W%-H+gG;!a9CD-<x
z>Yl!Lhm_miXV+iv@kp4y=fyVfy&2`I`;5-5+MBR$>HgZd6MH{|FWh^&X3ze4i~j7D
zS!RFW_&e5pW>XgJRV-xLb6okb-R`N+tu7l~+2e5I{N634*Y<GiF|vIS^=a?BUDx+c
z)4aWR^NkJr#Ak5ZdQRB9-|OQd`$-m?_P)+4-nU1KY46_;+xFdJP2Fqx@y%ZGTONCx
zuW#9VIexo+<7(l3hs}6wW}gq;x4Tqm--La<d;PVq?^8VPxtBjUVSntl)BBci-`V?a
zAM3t92VU$wyLq3r&k@`Gf`L^B6xM0%3#nVQ-^!DH-_k2P4_rL9z-FTa|GrnL5B6PG
zkJ}r5-N*L$|7rUg0*~xH;~!$9U%q$Gw0kmp84jiH4NMW-chhj;UVm5ieMSjE`@^T*
z*c+b}vG>1ku>JkLGxloyay;;Wecc}CNP&H?-u>U(%q6*RrNh^~DSr3%h8>%{Z&{+;
z-meqq?iIUIvRC_1>K=6~>3x3^_V24@|F)N>jKltUnZJDoPv@Sq50Bc*HZ<)0p}ce7
zvFzu2uSh@MdsO$>-btPU`!)qj?|+*<b1(C}xZOT)7548CKe#vMc<|oCZ&vPedmL^3
z$ZY+AuA({nZtXa|Cu?cV-pS@i_I$i~z<T$AHT$3cxwOBWN7O;MJ<0xcqrhGxq1gwP
zM>OyGxire={@3;Pi`Kr|`^_+TugI%ywsEW4_jU;H-1G0$ianR}3-`*sl-axf)cU=u
z@)P$;y>H)>`SZno<tpxdMLO~}>ukQ-|CuDbcX8XP{RVSh?>!i8vA41A<DOTHA`UvH
zQ}))ht=oI1>A=1l%)xu}eq7&cxODkmiDQ%Z#<MZ(Jyv~o-_doe_kJu*ao{(Ua45Us
zWcAeCbN?OPWqa5C+pzCYZt32}sn&a(cAeV$ua13h#|L!>iNk;QN*z76_qOr9ee*L~
z_Da@B?8&Ip+P^N+U@uovt^?mvj=i6}m+pRdVDsLLMNjuERNHSor~J|Wllw&voRN<{
zz&3A+{g<Qft^Wvq+jpVs&E7X>Z1>FBxM%+<Cat}HO*ibFF1=u1p!mr>Gp4QI+xzzI
z-ddwmd$aoQ?s=6Kxp#hP?OuJYX?xW_ueQ5#L}Ks8%80#l65|fE$Nk%T;)u+Fm62_G
zGIqA_<&`~e9hE+7|E*Bhy}h2Zt!s2H?%Va7b>HK?_IpGQaPQ0gc);edgQ(T{sY~{?
zRW|Q^X0>X+`t*zYD>E+dy&WLx@HqGHUiD*Z_N||MX79t-@_Vnz#@M*LD6-MJzG&~0
zulx3v#QojdnB2K9h-u2+%SB)JsJ1LQu=M}ry{2o|?03%2+&d@n_U`%_WqT*D7u@&e
zY|h@6ZOZ%G+qUhWte17*iZFvis`IA3Vs10{f7;lxH=ECYuh7mT2RL$9?@fH!x!36P
zoqdxu-|ty?pLfr9mHxe6_N+GNV$=3A8_QTnJ#^iBPWt`cgN@(qtaJ|SZTOaHqr6GL
zUd?vF-rsJs50uE+?2SC|WbYxN(|el}3=XEheYp4c%ep<cw{6@vyXD{Bdls#G-+em1
zXOhtJz3Y3@_uP)&w=Z;JpiTNG#RD@`h3u}`&EHe<a*dta#VIz@t%vt{zB{sa^9-&%
z%Z^XpyCZk|o=%w+_A1kP_TBlvXRn1;%)S${HG8EycJ7^I$#_t+^zq)dJy-X>F<r8c
z&t%u$1Ezv|w%&NXw{dQz^~rZ{4=mF5KX7sG?EQQOM*H7#J+Z1?*mmG(t;gP}*;RXe
z8;{#>4m`6b=F-KzJ1@VrsatYouglGeduPl_+N)d9xtHs>*gp4;S9=?~-|Z>VmD;PC
z^JjnOLymoixOMlf+3|7zR;%Vcg4?F;7kt#R*Xq@_y$?9__XeI{vERz0ZO;<6_j}f@
zt=zZy*V4V;T{rD*+qZh}Wo=KJFA41XLSq-~i;8C47r>`^!1_trenB3&eRp|Z*>i07
z-aG%;+I`IB-}jbJ+OzjWU+Ufm$qV;Zuia}en<=$Vz_4eZ3D?BE45cFbdM9nMdA9N2
z{=&ycthMHZ95`~t&05l;b+3C@>)r#;#P;f1@7a4oy=TwV@Nf1)sjB;%XFJ-TUjB8j
zQF`hAem#dh7wkFqwyVuQFh%13o`wG!_RNpDvd@zF{GLZ^diExkG4In4HQMXvd~t7t
zp6i}&S-U+Un)miPPq#b}eDK=df3JA<l^kc@&mMGP&ostm2eQf%Y?Ei3?2UQKu<vTY
z%KaAoQu|KCuGu^79{>J0mJNF)6?At;Kfb)z_r<q8LaxI5v@RXr7bZA+Z<Omo`{g}S
z4u%`|?yc55azJUz<GqW+x9?l9OJd(5b^E>MZw##SH<|2Z`xtP5G2F;{sgs+nan7ZE
zw|>vDahxP#v+ePm{ZsUQ?Ja1Yw_j)N;(dKDrtkGvo3-aAcl+ML3gx{SlUWZeyd<_?
zE>quOMudq&Q}3QVt7hxjn=Q!N)9Jr(kEBc5{-@~=_Of)T?YsMebzgnDu&t{<|31FI
z2dz^4wDz73yRi3m@$bFYFY)bLEl{=BTr|}_+P~N4>ldbd;s2!%SnsXfbN)%@{#TC*
z_Ri+rW8*S0ZttacJ@(J9Hts!XE4;TaZ`Hoz2j}i>W}Clv53|?awFwNim#-eOnR#~e
zKIJ9*_u7gqus5~bw%_ju_g+@^_4^jDUuNTSyMLd?th&A5ltlObmwUTcbvgS!-s^wt
z>wllyz0I<FuhTriy{`?p_dPGNvz1GmxNlR!p}nn{zwFum?cO)%-j6+5y{q>c-D|hL
zcX7ks!Vd~|FI4OHe=+LcU&Cs;-|EWCy|2C(*d9`0+V`ig+2;Rl&HcY$t=ePuNMUdB
zyrugZ_D<bnchJISll8$pE#^UckHxp|;ad1|Z(*?Z-aDaTdv5)&v!8c_ZQp0hwR>Z}
zYC8OxaLIaY7r*^1z4*PWoPO@jX>#1l`kP_DZgin_+J%>UXLHN#@7n0PcYSiwp2QsH
zy*)9FHcObF?)iIn>b`}xhI@;<P9DfDU$cMjrDmIjxBl5T#j@Cb&b+*@=a9snIKR9-
zvu*|2__+Pqd-6`A!@HNZHeOb9ZM4&N?bU4&*cba_v(53>-46E>@9mLwI<WtKd(l4b
zn=N~e-M86m^O|el=fh|AJa%F^z*cRr|4Qthee0Ra?f){I*&8SM<-p-<xAz(+$nL#)
zZ=Zc{5c|GI@BQ{l-L>Dlan@!l|F(#|_7<P^$cil7#l1^;?*>M$eE}a%@4fpa-==k<
z>VAQMcY6bGOxwHI*lPctV=8<8T3a7jIgxE&T|LWQyZ%W#F3a5Bzh%z9J#JIy>~(s!
zeBX+zReL{s^X%o4Y~Q1CvS!cw6EF9+Y~8(YF6-+(g-i4gEY^~;&nW-D^U;w{_O?$|
z_r5s(dvDX#j=fPeM{U%PMeKDj4c|Lc{KS3^4vu{mixl=U{T8-);V!o)<eseUC6R}A
zvl$NWe*fvje&I#i_cFHJ-@83}=ibK6U3;JYW851bB6i@u_f^|ly7LaGsPP^UPvqUp
zZ&|s&I{wh!l;0LM?k<c6_Ly(o^VA@5ukN{%d;Kyr_pr`j-n;fE*FL>PeR~zPrtj@r
z%Cj%AP;j4igw@{r>%DESosr!;udQ}(=LJ=VT0ynF94p)%=G^hKX-#~(yWzuyy>Wcg
z?dH7awS6CX()!u&oi=~pZP>fdqsu0L1>4?)lRbOqe0JSEjceb&jE`lzGYVSmMZdM}
zYe{n3d-cRthkr%Fdw<E5?v-Hl*z?cUd9T^S#=R^bV)kBq!)@RH=I35n?qfEMPZ#XF
zb<%q8wAKn6qpxZXtTjCQw*1(#FZ0EsJz<_ad++P4-79!5+@|Z^;=Ob3>e+u$2())T
z`t88w^E>uun&#X1xt_Jp{<d_l+V*anru3Hm=OS(QPM&^f@46E=_sLBvv;82*wQr5l
z(!J$>rtWr?X|z7Hq<*iSwEo`Ufc8Dd(#-dTwOjAq=&rmcGqb~f!uxqvGBr^SA1$Ky
zNKfBpqc{Ki-rdFP?Zg!(?zR6cu&-z1?p+doK6`$B+-dXp_wqef^Bnd#yyD*HaBcs-
z7Q3G|l{c>0&ophZE$WNe<9;uG-|>%&_TD`7c;D_f+xGS>{=fIOS+C8uzIl7qcK_X{
zn%ut6-m!O|Mc9pfY^y)*eWGt+d!*!)-C51@y)*459AG=Az1Qt&v(1jihJA~^U)+0D
za>L$*n_t+fT+FpOAos}b*Jr_fZ&J1P2D|(2Pwl;Ky+)FKpTf-P`zQ1n+Mdwbym#5M
zIM9Lv>!0^e?bTS+z4uB6>%MsAe|x_eitJr^?&RJbJTvwkoy=js;BoUF7B98EIo7VW
z|F|yi<z*7xFT5pt?{vF8d%v<2>`{x_Yxn3w#omvLtgY8s=~#bCII{Qe8^e9yj)>Vv
zG`!jUUy0W?=ivQ)((Nbr{*Rx2VBdP{1EnVId+$|UK9Ij-;qJnG<9&BZ9_*>>x3pRG
z!))(lH;#SBc75KTzT9$WwQsZS#VdUK*^@u)p0e@IUa{c&`<QO{@7*Hw-u{egz`l!i
z`}WH8h3~CO_Sju}M00Nf`{VtReHHu9%njXNX4GfDd3C2vTZ^WB?h3BG`f*$K3O~DO
z?<}}=?;W>YdpGz@*!MQ(!rqcMI(z#h?pVzhPTA{m&wDQi@2S1Hlh*F7XfCx;yKcJw
z>+OYmH@}uQf2sLtfBnDPmamqp+p+Jq-hEtP%^uyN=X;L?rrUd5eYCgXBexCzq%-@j
z?>n-`&wkn7=U)8#q_5rFdt&`k%jZFJ_HEg(zUOTv&w=#AtoCyL&3iQ>6%S}F2-tgT
zQ~y4m;I%d_Wt;cxx!<r?bL!naXH0|acNjjhX|XV~5#>$Y7n?fWX0y%Cy%wS$_Q&`9
zvOd_GXkT{h@}AmR&-c#S+P=4`;nm*lrj2_WH@WP8c%9pR=gz?W+iPFi&tMMN%gexI
zUv&56-nU7AcQ4&~$-X|iWzUIQC-;66Ke#XXW08%qi|*dALjAqcA3yF&T(fk~N3XO!
z;@_QY?$z$Lov5F0uOn5mH`q#O?^W&jwqCwp_eO9h?EM*g(8lIn+TPSap*`i=YWACb
zH|;qu!)e<;|Ng#@y4&|;GrR9K*xb6?Hh^{SyQ`6VWtP6!*Zle6-ul>w`%^er_8ond
zZ}Vb_#{r=jfxT6qKkZ#+;js6!|Es+$=|}b$MGEdMx0Kkwx3+NaYLgRtpRjJ)_xx7N
zUe4dE_Qt!Nv@_64+8e#}t?eJb(|gx;xb6-87rcj6e$yTu8Cz?ki=X$qXZP4KJDxw_
z^x*UU|A9(-Z>~<U57T*QBcS1Bt#WMA{yvSnds%0!+ave-=DzN|uWfAEtM@p_r0$hC
zd|_|ZkzSj|2MhMD+V*m<m`L{C`ss`I&!5|4V^@51ujt|u`&Sp0_X-&BIGp&zxi96y
zy*)1vKHK|0;G+GTC8B#4dY|6Q-r8d$n{aLSvx26*XDn9lEmnWBSF7RR9)Ze^eP8Wt
zY+@q6*%v>Wv#-Wx*50U|&9)Is)b{%3oZS22*1tXLcAMH<wobH}m$=rZxHkVlgv`mk
z|0n#g@oD(5H|dbjz6JA+?`5<)X21E2$zIu`od@FDJocJTp0ZbiIb`pH&!KxmljrVT
z^TOYL=P|<rA13wKs~*eQcjw3DJufwR51cSOv1jd{|9f8Df3n{)LvHUs9mc)P=I8h7
zMLpa*yWexqR`2V3@9{D2n~_<(SFz4|?@t!_-B!61_i}#TU~l=-bx(kP&n}JP#db!Y
z5AQt~w0D24>3(b36X*9@<UO|*+wW`t=#=2zxJs+Noa&$UFesGny)pgyUXNt<J?Cx<
z+OFO@d+!6y^ZTk7&e^NyA8kL&aJGY)s`6fYj;DLQ3b*Zje0}{sx82M3gndx5me{v-
zuhXV?)-ERv99#`o?_H_=aBo=jIom@WDtj%bx7vK0d~Sb1_=3Gc_nPd3%4h8Rng4$8
zp8cVFqW!q`&2&7p$0f?y{#A~M{b%jQ{beVoIuy&K>|J<$>AuX{llRWh?%lJk_Q&3i
z9qN1eEY9w&)Nk7NlI4m`PTZm0+yCd=n6JFQSIoO<FFWV?y&@Tl_Ix}VvUio>GJDSE
zLTi~-xqA{%F4@2IOW)oT@?86mMz!n>-Fs=T_r&RY6-~Mw>icf*4dGjElM%?jf5VZt
zdsj29-0Qq2cJIy^g1f8ikL`WA@ZCO%51M-}El#lCG3BJa(WDNWTLrlXCYl`Fd#QQ;
zzJN6%d&9hr?cMe2;+`j^H}{zSadFUSlH6zB%dmH6^45L#I1~0tcRjG#x}@j8QD^>r
zVvE1oZD+Z+PfbL0*G#utHt*+M-208IV(;W6?*l3`r4F3lpmbpGp2-d`zs=Y?_v1tR
z)U$niqPJeReqk%^!0IEsH*;mT&C04X`!-#b-CG#Ees8mDolTkOk-e%}XZIv8db0P#
z{f~P@9z58qUa#!1_q^ENZ4-9yHTB(kAf=6EZ@ayLeaE_~yO*Bj+tX@Lw>M|fqkU_p
zH19Py5V6PFm}$R^_OHEfT&C>J)?BlfsW*I2*1H3Hb+4`5C&W8_Z^$aI{ioh7+<&qB
z+TP45dk^&4sM?Aq`0qP4!*uVPx3Bi@wVrJG{}=B*rH@Pw&u$d$<@3t2{a0$UPhO&X
zuhg0kyJW)e9PpOjvAZf>)4uS`k$uSve(zl$)3jI4=k{LLUu^rfZ>rq?{37=Om*$WA
z*WLYL-~Vj;UY_ss_RSV<*n2nS^j_J}i#CTt5A9WbI&1Hvnv%Wwv&(GWMz`(t>u23t
ztF+1b)r%E2Q>F;->+Y4_myyJ_&mePyU4!uIy_-%k>?@bty8o)_hdm)5?%RLpn7zB>
zPQcz)h5EfIKi};C9LceJgZlhEspU8Jy;wJU?}@BGHW~8kZKD3K-c$Es&fbvZsr#nA
zUA5=OhRORZ4o2JWUM;`p_^u55mfZ5az1t)AUb14_8*H*~Z~DCrd;a^Z-aGB6f&KT$
z^LzR3XYF1uy=R~5hoyTNycG88a|-TPR9>`e<D_#37G0UTPkm9up27u=d%w++*t?MN
z(cXE%XY6lRowQ5sn(1&PZioGg=BT}050v+%l&0+cFipW``lmAoCffbj`(s)1-bEh{
z?-RZ{YtIVBrF#U9zTCU}>6<;Pa=z{jp8S9BiQBjK$}Qir*D`16{zD(W>@Dec-czh=
zWcPjIgFOjrPT3!Lxqq*u(yP6e_t)*sU^%(JMfdw2tv_yi^sW}{^W0#w*XK#N?faxn
zd((V=?&0&4vOUvsb{`{4(w_JgpZ0m{?Xa`h{d@19*QfUHHs#%0cdTXKw6I%y?=SY*
zdz7_w@84OGmJ9Cw-JjmNd~f2E+C2|f>+gHS(YcptXRy^><EQ(&*6rKNVz6hwnZo|P
zY<C*>c0aALsy6z$H=&eipX$|BRx<YOb_U5G?8Rrh+SiCP*hDS6X%}AlW3R))<$Kd+
zl<%pRU$FPS_1wLJ)B5(Y{$<@)<IK8G`D2o`j*h%-=I6(I?OwC*`}QhjuXR`9p6C7U
z_Kz7B?d{9T*{i<hk*&Skbeqg|4SV7r%kEv5F=4O0ndm;LAE)*UJBaKJ(Ok8s;@p>g
zPgu_G{o>xUSKYp7ukaM6y{2ks_XKbq+V{8ehV`|>r}o|XY_tEZ=Yu^LpLW_$jytl~
z)N$HAd8=J}*9!32o-<|GXa1Ud-;t*`_pg4=weQYO4cnIK;`?62GVC*Ln7#LQ-n{)&
zuKDd<EzxBkkQTh}B>&evJ61L9jb8U+@1En~dnPPuIFPDZYX4I9#-1(49d@mk+4ub~
z*<_#1;$y>e)oc&DN0xnX#lPKO_Vn$YvWs=^<fiDoO<YNPSW<V|tWagycl)gWUd5%a
z_C(EJz4x7)$lh&P!uB=B6?<*+Z|sc~2tL3bbz<+f5}gB$ub0`JZ3(n_zMR9>V8?m;
z)$7Ig`ELEQ$D!lfz8!1!?LE2PbMNMd-FyG<UcL8i=Hk69Q*HOP9_O)ENS<i#V=duu
zrRmAu?<JG%_ryKiYdn4Sz8|-3Y!VY{_A(#K-F>?1``#^*LJr?e#cWSmMC^55n!ZnF
zr>M<eXHlDLOsV@_9hO_ihXmSnu<qGcxsY$~$7>h%C{N#Rvn|bF@6!)^?8;{P+1s6c
zwSQT3o1L9?;a=~S)B8;Sx7o~;W7+4vL4AMk?aO;l-m>0nc<JcAgtwRWa+y8Y`$g>T
zo|m!z_e$okwC1cd*z0*^>Rv0B(mk`wME330T(EcJY0<s=jZ^F|F!kA(y5{Vkc&2G@
zkZHqS+uibeA{f{2<2yNbPiw*dy&TIK_cPr8vez>5-rh?dWi}Gm{@9$F7`XT2(rNqJ
zWMAzuk=}0all8*>h3o&lg3p%hKbQB~QZ8x!zP}o$Y&OT*?qx2Sz5BdQ$6mFCzxQSD
zyl*paRkO{8Vkf(b@~?Z&FI>GhMW26fn)FHQ!i+-uo>}Mj8R-1o>&HD|Z}zrS>qOT5
zHmB0;9P~of9d>C?-*0^Us{I|uJ$rNd-|zdw%3`~$$=XV>Pubyc$M?NAI%e)lThXzP
zdGGH%nzn!UF5mrP?}KewdwZJC?mZRLy4Pa&wOx~sRNJK9e7etfe*K=!vtI8_+hld1
zcLL|W_;<hdzwoTv`!}X<uWM81o|AjD4`lc6-+N~U`<^q0+V*|d<+ceBSZQ-M$8YZw
z$rkI20u{T@DxBQ+RJ~*Gj_}zBj<#|h2-xs>?<E%71Knp1?X`cyv2V5+zt!(|_pG9C
z9o>6P<ij3e|Goo1HcsB#Au!kGOYymVt&)%SS{&otb75M(edFKjd*?Epu`|+oyH_;n
zq|E`5C3|PGz1yo+y2g6_`Puuo7cbs_rp0mpOvbDB4l(Te7Hm}Azl%p@j}7CWy=yr>
z+xyj8?77q4zjyM~<NGW=e%t#tsn@1yUZPFn&1ZYBiHYt_VV<;io=Vf+tofQYu|)z0
z&U+u*d$Div9+5er`=9I0-s=<LvESnSVVk$#9`0>a>D@bXwXD70X|uhzs?7H8+52VR
zifJ8t&DL$$>(BmdZ-DTYy`hN;yYHEu-&gOqW6w8tPP>f1*Y+n~6}34VaAN<YW3785
zPcGTVy;OPc8t(PhjGxl?O!&aD@5?+s`&oX)dpGWg-|J%iZ?D3l(|cP_{oK1}ufc)T
zkC*nGUvu0}==#sSQPX$reazgtcVoBi-sp9Sdp;B#-9P_G$bN}GSNHRC$nDSgFk^2+
zVAsA)vnuyKP2}9O#Z=kh!k=?iwG+PWP0Rbck1h1t-rl(Wy-^~9`*eR~?rj$TvFE+J
z{a$HJXPeA^7VAy&p7w@??t2b)uim5f{OtZ+>Fzdx63P1tp8nrkab()wf0K9Z7V6rt
z=kw{id!C6|TmOtZw@+--g1uesGxwNfecgM*WY*q?S7+@t?0>uO-|YE&>z?dAVCKGm
zpL@=?y<w+=4&<J=vb(LNeV^$i{(ZH(yY|j8{A?r3v~^F>*E9Bw_22fU)+X3J(e&N-
zCdXxOe1^i_Q*Tb&|9&cG{d~2J{a5!{`-Bv~?p5f>vlS?1*=K5Tde4NQH}<z4{@VYy
zB-wsC_nLiyS-18C3g6n-?RMLy*H~t6^dkX>{T)5K+&?Sq<BDFtFa7V<Jywqw?&jdX
zy;p5h(q85l8}}X*kg+ao+rIY%@2b5LHv{+YulTul{+d-bn~R+7KdW}_-FkbUee&BA
z)~fer+1#EhxbNUD<9+rF4tp0Y*|N7#EOy@km8E+dTQBU53OKN5@3gZvH;=5@YivJh
z-@E=U8$Vt?hm9f6?C*SK*k}GmcK<7Vv3=(+yY4e=Yu}rDWBT3=uUPkWFFv!^oK4hW
zzmniy+e6=M17BR-XB#(Zuk8~9o2<C)_5nv-_C8#yZ~xSE<GyQBLN+~|NA{HPFz=I_
z-fxxth0FfT;nj9tB8v`i-?+Aa#+8CS9S^hZj-LtKlk)r3-p@DV_H%K?@2YNT*&7<V
zXrJI$rhTfyclP{IVA;D(;=<l_)6eb|ogu&P_oi1ivl~|K{a(cB5F$9yCUno9z3P*<
z99VIqZEs}bll}KAR_@KzvE7?te|t}##e2Ihh9!I6bH3giJ?Y6lAuXnT77dMi__VzC
z_GnMuyFljC9=)W?`;@kH*eqPjy)U;l!G2MG=iZ0+XV~AkKF^wW*Xw;Z?Zx)4E-cw=
z9RGChXZ0AXm;bi!7r528_hk6wy<761?OQM2wD*q5&%M9mrrIrQI<cp6iLt}Igx&j|
zOx<ns>h+Gj(qU}-j<9dvX)bnge~9@dyGL(g_cOfjwEw+W+(zQCtKGX=Hk-XO6m1->
zKG`pwx^=JGEhgKv;9t87sta~o_fFl*t9xqiC&w>)ZwPJPdnn@T?n95>?AhnXws&)-
z_x@EvynFX%`PuA}oV4G3?ltQ#_9Av`HuUe6`mkc}4Cy4B4SlEgA5Y%2hgWCno<moq
z_y3=_Zf|hS^}XIUYpfl3L~S>$d$Y$v_QF2>ply3SX65dGH`ifbdH3%<SA9b5B)V4a
zb+q2SkN@M^y*3-C@0RZTy2pr-Z{H2W^!<C!E!(?e$N#;uhFP{VLKCf*@(Ay4Q|Px3
zVr|;%_&D-_n2plD`_dEkZrAm)?KzgZcix;Pn{L@cJD;i#c8vU`_OUG8_QzZ7_jW&0
z+{be)-Ny6N*S+~}clMsXkZa>~`_EnztFQasnt9nAp0;Z5l_{(DGQ762x$bnpW>dD~
zo-;dg_vYrE-}~sl^Zxjm<$EQWF7Lg+q~C4}AM4(OMOJqCEv0*(-kNCDdU@6!Q?0*t
zffpw3&3=4g_qNU@`y49f?Kzn|b<cHs!@cu2F1Lwz^mxw`weR}^9(dWP&5t_}z_o8*
z&$-6E-S_A2zg_CO_rvsQ`xGbe+E#AawdeLz&b>MHTlOf<cRuhVXyslPUc0?<vt9R1
z`ku1ejpg5tXbwSp39H_{>wKr#@)u9v_dZI{cIlb#d)IO=-(&X5YcJn?u>+s<rX2{M
z|J&}{PA`W@u~ZxOryuri=l{4jKYgQhknJ}6<GluZdms1jIVj~~XHvg;PgHE9b=?>5
zJ-tSL)|=8L_r@M**!$OW>RzU=-FwrQRqbb<nYydtEXST@MlbfKWIo$_JmCI5IftEl
zR1zla&5z}^J#?{upJ8C<Uh|M$yW`ey?tij5W$y_){(awhr|m5jsNOq$*2KM<LD%+i
zO}(|ZS~JA{t<ER==OXuPYL>Lwd!=yOdZ+l=vZ!9#?Q`GDX6w8ydtJjX@2zM4z5l|>
zAG@9MX6$+Xbj`l{d*b`rpKP-+G>^0QNttA$ZE|zJz0cx(?*rLv88-6l{dk{eZ}73L
zdn42$?eEtc*|UhKI-C`Ev_G|H@1F7>D-XD<Z`sRX`*$ysZsq}vr3!l@tho0%ge=(i
z?8c$J`QKmczPtL*UJah-d&{Ga*c|tLwpV_K=)T`_ihBcU*bijo?cbYnZn2HJXuJJ{
zrUN?;WyL#WuG_e`?qA$q<>jS&{Tuh%Z(gu`uOs8Sy%x$n`>y2Q-P^Bu)23QHXYUe|
z>3eS;d$)I|<%4}o&a~~hS2O=W=C)b;zMYHNt(N!1e#hm=-4z~-_9^b^*;_Pq)1HHI
z1$%caKesnYGT^}cBE@~{%%|_&X>(|w`OXV_%erOu)(EaRQ1j>2-aOSQb{k%|?{h!W
zzWc4(<Gq2`?(e<m<zsbC@aX=<g_{q2U!&qMXM@Xu8G843?@KA&8*lY~&!zVq`yyPV
z_D_(^-<!Cb$=Y!JgFQmi-|to0eQNKk#a=cl35<LHAKkZC_U+2uk{5sO?Ps*#yW>Ex
zJ*VdMJv?@N`-)^+_jM`q?fbCpn_c*!g?rQ3Z|!Zk+pu>_Y~+6XNwfC&S-jq}%{zKu
zdUm~asyy?)gO?WCc<nl8ZM=?g@5x<z_E~&6w0G5q5W9=3^$#r3S-0oH{Cx)=9y8k;
zU6Z?S`$?XC=h%Gr{J(yC&()sidwL(Q+IxAc(4N&#r|<m}l)2B&`NrNUw`cBs|GU#+
zT~5c|?}Z-rVFzUPDz`c8eX_~S<`%=#y^+cpHf{d%4)`cc*zfRHXaD!IL;D$nKkfN5
zW4ryPdGfp0a?RN*lR9?~qaEws$cOEF%VZ|+v;I7BZ`0E=d!?UT+4G{Ieb479ANQ7@
zi`+YH(v7`Q+^hF?&oQ+7xZuj(uZH$}|8BlzpXkZBZ&CF;`)e0p?Oi&LW8b{HHhW?<
z*4n+zm|+vO>D=B0zl8S29ACdjuCr^eS<a2UQA-^69!lr4nSPjKU;5^B%c=Ec_EL4*
z?KeoP>@7bqeShf^hJEXo?Av!*WWnCsdjI!y1jp}HFcGkcbGfpAW+C(5r`c2XeBPF9
z_w~W*y;2@QHe7oT+Rt}fwCDfdYy113_3ty~;os+xI$=+s)CH@6jK_Pw9An?l-8j{5
z$-6B3JO5Mm&uXpQ%X!(vp3&ZIuVbRj-kQUP2ik(x>`h4y*gN-q<Gv4Dj_h5$a+=NV
z`c#`M`Um#LM$g(C;dp3oy6@FJRu0qmuKw`B&du@aUMIcVyU+74-&cO`<DQ9!XYUI#
z(cByKkaKU=#J;_Sv9<R7$rtuY=1;ZZa6hu|+HGB%9}Z0WzU5xCwA(JfclPXcdrwDR
z+vgS&wfFR_`8M{qg$^iBoVjO-*5&<a0gLwp_Q&sQ5j}2`%g(g#<(?mV&s^Z&SATxV
z{`oCsd#lXm?)9&9+rQhIW1msomA$V6Cfh5X6W#Oq*e=_S%{}|RwsG#8(NwYboFvme
zpO4ymgB_pkU-PYRzs;nlyLSq@@4p>Wzjv`!pZ)($B6|`%y7rc3aX6H;MDLyVdWntK
zmE(IK9yi%Lbz<b6*u?jH4yB&nT~V`hZ{iu@J%Q)BtPJfL_eCF_XrHdtx+iw-(!HwN
zO%Ax4Zm=<OoxDGncm3X-ZnJHeI3sMr<v1OdxUJp$VCJEn+q*983%XZtm2=I-MlkQ@
z-u<gr+h|(7-{W`g{XV4<UR&Wf*8|VDG8{-NjNF^D;KY81gE@P2Chgs~viRWM)`ZPA
z;qK0REhh2qy>e*D{s2Xty{4B}?iFO|+*iM=W$z2;=sjssHV5N><n0ycKe4~`+=+cs
z(hF=(m<jIrV3o61@k{ewQ5zA5!o?y7#rm`D8~?K$DAQZ8_f2=efl9smdy~%f?A^Ck
zaQ~j_d3$gDpJj7=3j4n5j(2;5%YW~&n!>toeGS*%O?jX8PKlqoxAl+6KDo);_qKS6
z+OvI%wh3I7x3}e)?1A$OGWOnHJA41}9rb$?nLGF9Oy{+|-6&~4TWrRj*(U{UR?ODi
zr*P}MO=COLK6A~!y|$sw)&a2xti-ww?kk#Bw&(1FX1fjcs}4v6F5Am~fBS(%H~zgV
z*#-CUZ(p{zvAkvPl35@2Zv7&>uS`(J;g0R%y^)r&`=%aZ+t=&Qx6kcA(>~1;Qyt7U
zCG9<K>ab7NdD*@<Y3%z1^>*&bQr*92lS}E|UCi<O)>uwD5OzTRK=F<E{lV8S?G@c}
z$j<tT{9X}(Eqha{QtTHzdba1j_?$hzdLQo7U=rGUI;DD#(KX{eHCxj5?l!-%SCZ}L
zUOpo>+r6F__n!1sclfzt;a;n3*S%J67w$Kl!@Y0zoP2wZWxw{``SWt`y#3NUZ>^EH
zKe_nq-mh;`_O6z=zt82s_PuG1YxkbB__RxPpZwmZ&$##TnJnD5Oz7>N#TA149i8Uw
z-)6yX%h){ifcd2Tdz0R1?aNU5v-fxIa~q}?;(Pxa80`{{YTl>+<H6pv{CRs>r`_KB
zX~mhn4f`(aW%Ac@aDKUJ@6uUM_a81T+gngOY0u&dtb6zD|G#(HVK?jehUfbi7w|ju
zt8U-F@C3&JubMx5%Pt5wY(Ckz>xlQ)y<4+v_E#wI?75M<Z;!yMi+lT;-S_@x@!K2m
zXRghH=AgX`(zoor{r%LQYdbvmM78tp+rQ3y{|4cvy>D+_-+QQe!+vddu6@_MI`;RA
zT-j^1|KeU3?@pU55@8N1ZSVGW95LOK`Bu&@arWlD+ax4x-8wn<ZLe<MTP9GmcZ2Bs
zeZlFs_9~gm9x#cXw{OXw3wu9o*|2}w=B+k4sd4*Ex^?%)dvosNn$f+-bxQl5H+3)e
z3n>chv;S_nx7vwm|87UMz310N?o|%U-hbI<<(~JFukEL5bJ@;p{<3>n75m;<Zzk@E
zSDm#--DSVs7k9mV4CjCB|IWE+@3&yny<evY*-B5iZew+seP2jF+y1G6TlV}hTxnDN
z`p`ZXw}m!|&mZg!%iXm1LH}YKLyzFSk;QMUh3i%KF4}Qr?-K3J`(q+E?ls-fXY=}<
zrtQmDr}wf1MDPDNWA|Rk6FmEB&R^UU7wxv+E+Au1qWkVWu{S>MvpnLs_p{=gy{p%1
zS|4BX!$v&KcF)I<WBcqqA6s*FM(q<=vTlEduY*m38vDU%U;FnK-Cw=WTp)UHPRfM6
ztG><M%e3*T^{E|O_t*J$?%C*_w^wnw+J4vQKYME>MQl&*H$AZS9?RaP^WND??VG=^
zq2jsCA)VyC<-%unJ&I@B_w#4|fs5Cb56rK5Z08cAZ1?z2-rknwya(*R#M!*8X4{t~
z!{m^;@AB>flM8!&U1#p)tJc|@pU$yQGM#y!WrpBBrWC$?k;YDY>#a2RPSFq8lghQ*
z-j=&}&&%f%tuM-@+TYu#wD)#;$^ow8i+kT`PTyOZrMI`-<(l2IS1C5TqId4Kxz4rE
zZNIwBjc@IH{;GZ38{xZi?@9K|J*=h&_ObWfvySsQv-c0vT-zY!IeX`=VskLrS8bb?
zvUwl>YvFxyr?=V^oxi^Ky%^&@;gnnZ=TC0gy|iPoMb5c^y-zmr?OX6SZtpU4XFFx5
zC3`EREe|-CY~EWxyWPe@r^%WzRdC-9-IIH#PfW6JiTS_pPQmnjfs>^i0x!(nbK?W^
zfeHUE+H@4pv5D#wJJ9v$)ZVXsTKj5lwe0Q7-*0oNfMaiUDEGdN5`6p4<?P!l@oU!}
z^Nb^V;~akNb!&363o3fH*EaLo-h>X>12gY7+OYBVI=l$Exwo>r#inF_(%!IzN9`t@
z{JeMTneM$j{~Gtbj7r?Ata@h8v#G6n+n4Xz`yhM$-mlpg_pyI_vGaVe`u=I&7x!K(
z{<7CxX1o3C`lh{}E=%`4bk^T<d)bveEHC<ZPQTc)H}>T5{hsSZ_sKkDx6OHaX7APe
zqI-O<UEiy;cAtG*xX0e(g5mo+zPIk(tdX;K<xLjb3|*(aOOlxOx*nNpAM4h(?_rLP
zy|SC3{k0EF`{cGM@4x)}%w92X{(bWIS?xC^EZh6|&hg!qQCsX5PAj+W6X4!AZJx^B
zLe~--qm4^#c%--QO)mex*FU#@Pr@4B{VQ+W-)pkr-JYnch6lE-f4cYB$|8rI_t)=X
zWbxda={$X}*@+`|o=4`}91?HYEAMk)-?E(odz)tH?-h9)y<7QT>)t!hYWCjJy|PcV
zymzl?RM~;0Cz<wtHl4m#`Sr7Xt~(y>;rO+AU(LNsdwP9s_Wsy(WABnZPxdBQuHM(v
z?6<F*f6>0b_BZ$K7Js_u$Ba!jn<VGkac{NTyYIe$eX!ESy*BE%_eLJ~+xMO4@t%o)
z{_g$Se#7pMlgR$Yhg=7;9pd)6r=QsSxAy+NP20_FrPeg;6`1X^UpP^0-}`OX_U@bF
zwQpVQ%H6zv+<T`k=-wOoYPU@*>-@bL#~b#fh92K*CBAY`i`5?c+1>n>M~=SQ6LeU@
zK3{=*Pld#Tz276;cIPEl?8(@`weQf5{{4)f{q~;cU2K`dzjm)%+V#EJ$zS$1zIwWs
z=kMLU9GdI){;Zq4Pn5NF@1xxB`?p`bZZ96RZg2ax>H9a?F5CO^!JK_AxvTcNUO#TL
z%fH#0A?L~7XA@TLXV3j_)91*$Z+AQEz8QiJd%LFa?)90cyZ_!&51SA7*4Qt6G<PrS
z)1JNNGZ*g-YtGxdK=8tzqZ@tpPqb~`zg=|t{?!eV2l~&N?Jd^-xG#6#oxMNj23v<X
zYuG<`TCn$lQ0iU@)0cKXrwQ+Eyz_PMQAzH-!Ihu)9Qw+)&xrZd-sSF3_sZWoxAzy@
z9sARN?(DVT^x7Nrn9FvLm(aeJ6N>hK)%EwTo7ugmB$|2Ot(5Qf)^j`eEN$Gp_vEwt
z`|jy1+WSYNdhdRQB%9Q45B4tj6Jh&^Gh?6Rh9!G>{yg4Wu3T*Y{9WwcviA7>DY|d=
z=0$AUcfDebP1ZlRy_4T_?9tJiuy@vH#(i4?OKmN8ckQ<1Vc+-RkAltLt$X&~^f|kC
z?Lou6C(6$4Tl#h8-g~e1?%lG!bnhX@BYS70?A-g}=$C!vA7|Pt@-MgdIoY!R_n*MM
zpUkiCYt$9mdm^TH@4MZ{_U&>L+$Su?ytiX+&A#)t;`<)`f3dsd*5199?^f<TmnOXT
zp+$-{_kqg2T(zxx_Zx@V#|Ny~>**@CZ*P9T-B!&BHcE@0+V<Tu-cuIWx!3Nh%HHLi
zO#5dtvg`}Kv~h1{$n|}faz5=nZhB(xhD%+0V|R7h*oZamdCPWwpH*IwjiBtLy_a7o
z>@}-hY_sZiyuGAjwvFl2u6>``MfUa=+}$Ji;=j#|6G!%Rf8MtLj0(fPwydwaee@RZ
z-Ra|MBXBKluPwtW`?b#>?vX6JV{c?xX8rBH$KD-B?Dn30AHR3W(cgQYziGE`dCYB>
zr4h2bF3Z<$(T$sX>nFF{Rq{yKW>|6VW#&oPw<6@s-t{}z?>)b0o^6Kd%)Qr+G3;x%
zy1lpj`<*>Wg}r;3O#Amfxw?LD*7UD?P0#D@m;Lj0kEqam8-=_22fi;qx_3(RRQtuj
zJNIrqA-FGc^^QH&_KNniHtgDywb^IyC*`yI*w+8qyDEJ9-pw;V?_%`mw#g9Rz4z$;
z=6&<|`)qDhY_+d=d2m11B$>T#K5_exE7<Ir!)~!}j)?J|dq;cs6y0XC-67Gi_x$e?
zyKtZQJ^Lko?|oIjXYb?NQhO$HoVF4DplT=ekbU3gwI=(S7n$yzw~f^{LwVQU1M_tD
z9u8^V>)){2w&DAgeW`&Q_Kulp_Q}eh_FlJLuwU$c&)yC97VK$VbH|Q}rDN}zO`?0F
zen;$$S~blka@PAjLd-dPg%(-a%r%;_x85gmuX|b6-jW($o05I(c6qMmdxRR+@3oL|
zx98czw~x>1*nTysH+z1~?y@fW@PF_2uS@oObj9q|(<|B=b|Y$!kd)l6RWGCVYG2jd
z%aJa*SDLAPkLaRH`y6<B_N?BUV9&F<d*7msrn_oFyzQ1w=h-`pW92>@myLU;i0{~&
z<I`>PoAuRRf%_BePpR<kD-?F#`<!#tz8UUv`{G2(_C}@hI}~_W?&WP+V!tmnU~iq?
zguUiYyY^1Cn!dM7>F=I<&lLBY1x>e4$o#v1m&oG%*WVx6BRi*e|4jvkJ=ZH#_fG!O
zZ+~G~!X9_+G~2gLxAz@h_;7E!ptQ~JLjHZp$I|x}t=hh~ebU;!_oLVEUH4aR&!#I;
z_De#0ZQNw0+gvUyvFFh^xOZOj5xZ%gNA_B&9<gz$U9p#Ou8O^S9ml?kEloBJ*`E8B
zE^y!be$Udqm)tk+ott}M58IBCy$@_R?=zjuytgh}+g|zo)qP)X^X@fth_yePlD$`E
z!?JxA?br8mw9MJ-ek^%!N%QBuceKvg{|!8}SM#m)-k=j#_A&qG*e9RzWbe*hckE=}
zZrZK&w%y+F_4IwJv!!elq_*$r>5|#Im*d%<D!xB<yXT*<|NCIy{tWS)1L5y>*j!^>
zZ?C5Db#L!o=6&6n>GmuP275)AnD&1EuyLQRa?#!`3-|8zeqq16$xUou{B4K5vqQc2
z=9x|1{eQ)&z1lzd?dQZx+U(Gk*!LlvfB)MnANOi^N;-Ub7qK^bS%b}NrZszG?{Mrr
z!hU&Aq1U3li-HRFl^y<N{q)4Q-3z`*><tZBzGqoz$lkLum-pFme%@;sTC_hvq-lS@
zo1yInwIF+`op<)8%-Xo`hd=i|&i$UYWy=@ZOmkYh_kf|Y{gUdpdt)0O?T%Y$v+sOO
z#$MUq9Jbl{S`K?Rp4#g;y~%z@{jPnRTQ=?$pUG*PCRw`snkdV@%r9T;4ECM24?fCl
zzkKpa`-S2h`$}sf?7wVWvUi~Y|GtwSxesid&cBya>c^g``#JZqc{lI99kFO{*n!)-
zFNio<Z|I!9_vvcgy_o?`dp|Qe?e)mLw<oQOVc#z)e(TRmuJ2cv=CybJl1=uvt7P_W
z{=RW<*vag@8KSTE*Q@kfAAgv-*UV+XKBJGb_g-Wbwq5n~)}9voX?uB}E#6zP`qIA0
zuh;foS!?L<#pa&vhPU%~e7tke&i?tjJtfCg_FgvLzW2w~340rT7VTwZVB43L#&+QN
zbLG7`ce?l5`5f7MvO#{Ym(3p=?*>MPqyMh${VM#?u5jwqeUE;=-kWny+O}xvsy*Ky
zh1zJWer^9+qu2gGJD)=i=W=_WD9^nm`JN6lzH8ckao61Y#qpP|Wb50#c~cJV-Fc~Y
zuau3@URLXwd*65&?@@1M-lr2fX>V(O|K8`P<@Qy~p0rnBQQrQRDcLq2z0rFwEiJa2
zAg8?d`|Bn9uRr*{w|&;tJwKdT_AM^7J8)-PyN!bKl06Oq*Y`!spWHk1SjFC|H*<H3
zOk1{R<>b)4hiBC86MgKn=bgkY+YtM0_Ub%8_8JQv*muuSZ|^p*)%%VbhwWu?IJ9@q
z{h+<VeN*=Q5}Cd?M$y@3#gD?h3zqEJSFka6uZ$F@&CAb{cGmNF_NAJb>`$EWW$(=|
zE!Mj83ilZ*vF+Ov#J5-9c#++d?o0L}3WxST?p<n|Ak4K-x`1VWfcM6|oBUSpRZ3d0
zf4TnAy*KCG-m{HcW8anbYkQOhmhP!`S+F-aWwK4x+5I**mowOGJZZA`X5`MjjUFNg
z!hQU98zu4WYf${U-(pAjUK6{?_6mEZ@0nilVDEJProCosdJl--JGw_~`e7U9{ipVo
zPnx;6^zxs*2aY-Jy>Yb1##&5Z-^{O{_vzbx-otbC{eHiFd+bF|Z`iBM$K;^D>HF^8
zPbcqtmN#dwrodI3xR4b#m!C22oBsaizIpXO_xdbdw0G}^?0x5+f7^4CQP)Oh#n%0N
zd3}3}W`4BS46oSdc(2~Z{mJINDzEtW8M?gQJNNVded>-%`&Z8|*|)Xkhg}Fm{oYKo
znfq5nd+gaZ>E+(t!CUwFFYn%KyKDE(MCFh@QE5l^hP*kvdn$ALUj0R9_ZG76u{p9@
zd{2Ro%)S$M_w5aiWwKSNX4xD3>G<AlrE~UQ4Jq8q)XI9G=ZEB81@psu?3btS&3ro9
zuI7UF-sh%Ldl&4G+%Ic?+2;Sg?R$N`u<ccT)UkJwzpd@TnQi+PE*9N;<ME|^!9@}d
zuh%TvBkFm=URFeJkKeJA`)n@O?P=~#-1Ajss?CeWCwns%?6&7;JZ{s<Te45t<ioyy
z>PPm}ZcMeQOJm-@a0<)5O{z!td#u^8FHiT{Uc(g+_olyS-<udSVejokcZaRV|L>Ro
zJ$rv$UCaI_7uN4_WnHrW-DSSL3hm7{dDAZ26|WH6d!i$5?_+<7{f^!}d$%2&zqfMj
z!oA|&uWSTa+4pKToZUMuR&cNIzpOp`AI{#_KWYEovllq_%B?tJcjfM$Jpl`*?C(uC
z+UwzTU~g8%#l4Z8-1eD21ooxqJlyl}|E_&U&Nx{gtJ`bc61?0xd@Y}CGFQ-E39Xg;
zPTpF+SFU8~ew8~4`<k<-?yWw}?a-N0Z|%ytW#6)24SVx^{_eGU`EsvHY}y{JM^pDt
z*P6O_SsCL#-PDh^mTH^!Dt<`V!(QBIzbR?n-c22m2d-V;v2V%5zP(Spw(s2#{=)VZ
zlkDEwe+mv2=Wf}FA1ZWE`|@R<-B!E3kDQbC&wbf$^ZNYLJs~R`?H9f}w>Naf%Dv)(
zns%G}e(tII_hE0x&bqx1e(l{mKgVXT>8=NRT{C_5PW^MnM*QX8eR_=?``Bka-^)L7
zk3GxL$$S6ZS+oCJr`z6x3peep&|A9KubbCF@eb?01F78mj=rC{kNpAXzVgV!d!3XH
z@13)H%dQJ5xwbXJ9s5{#AMZW>-C@7wzl-+EHB<NAUo&(6?Wqk`#jH*H=3HXfXZUyj
z-c<i(d$0WcwYNCBYyW!Pn|lo%w(Xr_pT5slc$rO@`j_24jRN)>U(Z^1TZY;Hvf5}9
z<@a&V`_LtOW;{B#xAXkgJ#{}D?7wa4-QRn(*S^>O$NuE+oqMB<0{6tQNAEqpEZ@fD
z?mYYR4IB4P<9)n0MtzgrK7$jxuduf5RSlfKcb(MUJ*>v7cQ<{?*gNxP(cVW@$$J;R
z|GmFvR`1@M6YKUwm{i;6z4^AcU%}UYM+4X1j4cU!ml$^KI^PmtKY5$O-W_K5ZBBoG
zvG3!gZ<alaSoV5YP1tK0+P8OF^`yPOuN~ee?ec4HY!Rz{j!BK(ijOPzE)D%;m!><(
zCRlj!KK@VYdqb+~tiF7|y;uLkjlIg(&)EAvn!LAk?wP&nsps~cFnz!0Rl}FvTUJfj
zUo1FtZ^GSPYcAUxdxL&o-n*a4%6i)MO?yAfaO|BhuW=tJ2mbCjYOnFId!NLcO?!2G
z?%8>(?%o}d-o5vjq^Sdg?7F=RQr-8K1#8&3TixCJ@zbZhT904s&5}B~H{EaM9^Z?$
zdsary+#MpuX)~u?ZT~UH1J+{I5BA=9z`mCuCw*^1+Zp>OW*_%<p6cAo_2k^%IYGPZ
z-UiOzvw1DIwfTaz`@UtI-lMk7cn{M<j(zJk&e>aRpSm~C=gq!sgBg2!cg)$(vs+}(
z4(0VWOoiv{Z#?eY`|G9izONU*@7e17+a}aGXph6Z3Tt2Mt@|&>zS>ifV!n5sxR6~?
z@=fcK{sVjG2Q9GYZS3B=WYNEU(a-w#g}+kUyH{rN-kWD%@13mnXs@MSht2V>Yc}r{
z8trF1_u6lzEVIWrj>Y~-ih=E_$(egajyvvWRZH8Qy!`2&DHnF{eeQ8|PoNaH%`=}5
zd;TX)-TTP-hqYYamA%ogf9y&Aa&oVGUAW!Wkjs10-f`}oeUNKE+b6v}*Jez&yPtk>
zciswtefgrPHk<pH_KD6sy*IW?WZylp#rx)p&E6B#Q*PUB_IvL)+j!fR-)8N(u%l|<
zhOb}t<}&QIH`3PI|LoSuy}K+_?PoBv?)AF4exK##qkAQb&hOO{e`md7>ltfC<AwGz
zhx_&_+IsFi^ZwkvSMnVDCKylHyUKWneWk+Oy|=AQ_oqkB+~-{KcyG|IhkJbjzU-Y9
zFnzD}()xV`)3(?}P5iQNRg=j6KTH+7WsbYp`>^-#I#9BDuf<de2d33h`!1)?*?U@~
z*H+kj$KF%wu6tBtHtqd={=lA~k2-sVxm5OY)G6)hJm0-nG5Mlx)5aK^ebse)v%l`O
zUr=GQcU5}ZeudjIdk>bH?3rXUbFb2!eS4(h_4d}hZ`(WN8~^@0PP_N6Y5l!-&WZ<n
zIOVSG_1bq~&!$&1_dR&7zh{fk#Qn^DN9`Y-d9s(?;FJCD;~T9-OqcGf*J7|;Y{6x7
zG5`7A>!#tm6?A*-Llt)Kox5kp-hJH$`yZAs*=xc-(I)n|>Va!9##SL))EsUoEZ7^t
zw`CV^-kv?&Aw2tZ#AEh;e>8vpqO4T=%4@y$YOG)PE&TAsI(6xT16)D;d*?f_?X&wh
z!S2$#(|f+I3E3lTvVY&rV5xmqIoIyJnK^ZDZtFsueGzVZKKF&~Ww|bAQ?PQ`zIQ7h
z+OZ04-}`PU%f4*2iTgz-&);*eP0E4O)no6voVLAx^4RvZ%wxCDKeuczZ}XnL9HvbB
zA5Gn|H{j3I-F7V7_xh?c>^&c-wvXF!&%QtZR_=A2b7kMFFZ=dK+;QKN>p0QwxuwXS
zUeO!-4Bj2tv-Jq`-VNW@?tT2{?A~ylPj>s3^6U$a3E1~{e#1Vi{_}f}UuWJc_CVFH
z-e~(Cfdz~9@1MA3?^fex>lue9>{}xyy4U>gp}iNa8SS@ywz8j?d))SDoZ$hFn_u_p
zHg(t+)dlXoF>~8qvE@(q*Pl4N*Z<6}y<r)r_npWt+*@DKu(!<W)t*fia(n&E*4xaU
zw{P#c%hLPW1X=bikUqPA7Gtwb<>ZTd84fV+S6a(%BY*XxP5Pmhy@wl|_gsFtb?=_#
z4hMF=(6C*fBD?oS`H_8-1ljlYB){96AKt!qyTYTrRcpTPU0QH{-wPckn}sbQb}l#k
z9BSp3*+iLj+pAW++M6(W%Dz6`^u33#a@vLz&Dphy{q$~^)Ae?xJJ#({-s`bfQB-yB
zl~?EXE|!nm>z?kq|L*PQd)94bwUK?hZ12MU_q(TxuCj?bY-W8Ym}Q^VdhP?9lQ!(T
zx4CVfWW@e`0+YAf1U^%+|NHan9#^Z2d%m-j+1FTZ+x@xv<=*@LT6<TAyY5ZgcY4pm
zI&B-Puv>dsZ!g<jbE<T&T;ztm+HMAWC%xOdzhg_xo|<nRdygJFu>Z&VO*WZ11@_@*
z=I_<apK85f=b^pcZVT+JOwR00zBk`y)y=2-HdZk0JG1u6-o!&p`*L<%-4#Fkz@A*m
z75h}AzuJU1p0$sCb#dPl$AY~pAII%u&2G0*oql*<&h3u9Eq)JeYF4`MslDrM)38!|
zAM;z2y>DOk@3l|lu>I7(YcErzitRRo0K5HdJ$n~?+_L}L)YW@eZ#!k}+S;+l`#Hlt
zR=>WzGM2~oEj}T$Uq8mwRzy(J-eW4$KGnS}2QuOn_wvN>>=UwQ+CSfE<KAeivOU|1
zCht3M%V(S4etFN42CjYEn$GXx-M3`#xofg}`Y$frGi%A#eMjQ8?GH+w-8-Z7*4~b!
z3VW43bN4z3hT32FH^WBMaK^5e(>eB~GA-V(c9U_h^sZBTFQ3}JFKzD9J@SSAHlI9w
zcRn+--NUr`>)sDcOZOdgW7!uw_uD>6qs99}+K=ze$lS9})@$e9w?>=xO-((xTbKLk
z-Yx&R_Pw8a$2v&-p`FRT?|YwLez|ws%BTAtJk{FQD!+H{l#a*)7PE}@YAR?Pkeqsb
z-+%W-dmXM;+DzH}YVRh^zk7ad6mw{6u-xCO+rD3s#l>z?UCl1hBlq?ja;WY-%DKqW
z^Gc4rb^^=3uC(iWb-r|4OFlE(YY=F-_mINoy_W-a?=4m9-xKV%chASfg?pF2;@{`R
zc3|(3i2Zx>S2yqNbrsvkXu!OW>6VNA{oqS`7ybCZ*W%#9y~Q(p?4#Cg+v`@_xHq^z
zbsyhthrNDDJbOb6n)dA8H`Au=K!bH?#@c;BmumLxf04G|bo+{ZhRvV%ihKOrCpKZn
z-V>WQ?sNOcz0b>CbC0U|^xY}8J$rrEPTKd@rfbif?|=4e|FpyQgyDs~8*C@<ZENb?
zpZnmz-pg<H?%m=NVxw%9v)7U-Y)|!}!+R&>@a}8&IJD1eW08H*d<F+yhOf3dDVz7+
zV|r}g?VDutYU<{_pQatPonpbZ&*akIy^4Fb+nSa8?7b8}XV3j#WqUUKJh{ha8}Ghd
z$G+?hT2#E({)W<?iOc5hdmK={_gmA1y_P&{_eb$|?A=qs?U25-Y&T29w7mw&-8N^7
zoA-+_W$abXe7skvZt6ZomK%F}#Fh8jyw=(K<<GvoMiU?Jl`JaVmppUT-Xk*?+Rw3_
zZO;_Mx6h(QX#f6okM}xeZrXR{_xruZr44%&T>tJ(f8w)Ok#(kBKX>R}p=Eu0wVhV(
zbJthdcYYrCo`;jC?%As6WfQYnYroU3)V*uB%-LI}^<?jN#r=B@25|3Nbvfz)BkPp?
z51)G4v&`DC|Jl4#d;Wm#amW<iyFKgjUhZw`2iC0Q+;?$Hsm*rX!}}_WSoZz3<=v+k
zer4~&^q##E>rD3MuXwWe_Qd4fB0uKryZElfp6SJjy{|sq-7B&y-mXUF>|U0F)%%@a
z3GFlB7TO#CY2x1MZ*uk}9!+}}*G<{GbHm?#OIKdq`{uyHy_?Q%-@7Q}<KBgN_xC=n
z+p*8-zvv#N?@W6gpLg1?Gf3XMeodLZ>oV27mZzKc6)SSuPB;u&gL8LJg#XpO1({du
zb5|bQ8}j7PUe~;)J=q;Q_p(Sj?fuI3$$sLoW7d-Q^6hIE?b{t-eSYt+ASW9g)+c)v
za&GKdpRDb0AUnbS$huwo!ZR9dd8b<J<qh3s|Go6+p2BUv_v(L&u~%xnvOD_Ts=b%C
zvF&X-v3$?H;JbU(71!>49r1IwMu76ZW%ld#a@jH1n(d3SxgF)P&;PE~-j<H-dwoLo
z+O_Xow6`IV(Qd|J#l1PEGxq*+IAXnjy^nqD5}CbIJ~Hi_&An%ziQBWi-xmn&J7~Xc
z@0(pu_Oec$X`BE1?7n!xHXGe5>+KhLJM5oxu)=18<Rbf3zuEVGl3%~C|3Ld*-r%Qu
z)w5RYJ>lWKNAFj>z0-HbeQKG9_tt#z-FNi#%)P1hO1A99clO5|zrA;YK*N5oBL967
zCl~D%I`Y;g;KJWM4;23F-CWYRH-M4BZa(XQ{k#nl4kdqk_U?TrbHHM*+g?s3)xG=g
zr`t>D$yps){Bp1G%A<Q1W!|)zF2%Vo@9^`zf$h7k_Z9o@y;;z^ms9w{-W-cv*7s~C
z?UOoJxTmh;*IvUj!uG3fz1h2tWv0Dt!TCL+e@<I3d9-=2#r4VdTc-=}TP0JtcMY%Q
zzN>w_`&OU(y=SZPwA~S>?(LcU&T8*lgNge}bogx#EMwg_v;CdjlxUtk`L~zae-KOB
z^OEb}zOJ>*`+`|j_uhQ9Y452EEY`=1kJ>J|&b#m5`bGQl<2$W)O9}3qJA;4km7N#%
zTQ7UFw@qU9{+dg(_kLkqXS2&Ach84^@Aqn4__cRU1&70J&KNt5WvlnCo3ec0^Ea(~
zS>`U<zam+5pU$zay_+UKw*SwNzPGf-)ONMl0z3EewR>Z7+4m_-SKpI<_1#`~xwgHJ
z9<%L}N#45m;`AkZnRgZJyRWx+@4?g=drnu~-T&q9t-UweuG%j-B)Bi-ebZh!LCJlQ
zW-7L)&duMO$nt#e{*1NzUdJ5U8<Cy8w@#01-+8l~y_Y09_gK4Z+2?TL=w8v;es(P}
z`}YNQu-F<&+uCnGXR_D8`@%jc`O3X--97epTJ8s*yP3m3egCv@u6<=+cx{(A`r7(8
ze%$k$fB#;tk_3m$oS41aca%FkJ>X><H2;f@`?6(w9gn=;d(5C^&+&Qf_63Sd?NSTY
z+ov<z?dw*S*|XumYdgm*|GnRO8}{_O@;Ur%`nq@9v5vjxc9h!5l<Vz%VZMIv49A|m
z^Uh7Q{CqEeuSwgIJux4-_i;UB-{;i&)ShWS$G$WFWA;iPzqh~p@QJ;$8w&T&OuWB4
z)9cONxSFNbm$UZok6u-@ch4Wgz14+R_gxpgY7?<&vh|W*fqUD3E!f*QM{#fL?+yF<
zT6XTu|L|k~tI%Wit0gz>^>R}_U|Gey&uMe}zN77XZR|75ZDz*n*jx&3-TUM9k^TPd
z^Y*GQd}(v<ZS=mZrE+_w21eLkS#G=k$D4ASXRo#G_x&{8J3n;Y-n3;m_jY}dvD_&v
zy!Xl#ce{<oQ|;vX%<SD96dcYxY2ACBY0*A|rP6!(btmjS7^-huc=PvO9=C;iZ&+>G
zH<ek@<`46Qy^|Jj?EB2bxi_n1(jKk(fA)q<-eTilthIMSR?LA19%6gGl)kZ!5lpo!
zI-b0@qsL%xxA&sG)<<^lE!KKt<Lx14>vUn$-eVI^?fu%%xPPl_+g`U_kN4g@x@~XY
zA@04o3%>1rW3^@9JN5N@^#Y6!oL_t2uK!Bk-WsKy`$c9i+#7Uc?LLp<(!H`7H}^^(
zY1(`9_m({wN}LW!zc%gdT(D~IQpfB2_~lRR30PaQr`^K&Kpk6!^&|Zdy8|5&`(n!&
z_jz&6+S^{aXs`1x*FDNFq7OLuEViF-R%Gu#<&d55Oz%B@StSRU=2q;@Z$G(LjY;sp
z=S83PO5c~>=d|Sd-nNBTZL%vI_R8`v-1FXG!(N6%t$XcRIQQ(BCbchV3*SDwxz_s^
zG%4;?{C0Vd)y>xZ*B%#HvnyBZw~YU|_i3-nUar^)d!NqxWT*E~Z0{36|2+#!Yxb2t
znYQQJ%&@)lY7gzLp0jLk)I@=OXH3rT^VLw@^IO`_o@2kCeZyt5Jr%xu`@2-F_pWZN
z*ta%e@?I0uw7v5y%WPfR_wN00L2-YL!kxVrdDid!zHZvS{8g<xy$T-g$@w_P{><4a
zdk-Fbdf@WC&HMJHY_NH_e%fAkRnt92wgG#(+xhniuI|}i<L~6~FnqrKmz`ldO+_l~
zzD!uPcX3<q-V5iy?eBVWaBo>S`@W|<7xqd@S=q|JXW8@R*y+9JN~i6WFWkTP%d;JO
zCkrs{<7<uD+fr{}zmkJt-wX4{ds>RK53qbNurU<6=ivI}<=!2qkM5l^Wzycsydn-O
z4S(&uB9gZ!U!{E?w^h7tPx7L@`tR=VeYW-P-uA|OdsoI^+Ltl+#@?4)lkLP=n(Vxi
zT5X<59<`HBzQ0#?$N7Exc7*Mf&3m@T!HLWETczN>Q&9)^Pxkn~_i@9_z0>#a-`75k
zW3QRUXY1doO#9|!&fJ^$k@?_5cjbK-U7Gf?AKtttHo<)Fht^elleHr4>ReU!ANVZl
zka6j({R^GzdwsnX?Drp(wt0A&f1iL{w%tQB=6zCyReR^Su<Wz0SiU#PeTxnA?dyAY
z&H23d*{N-NXPtbwXQO-D-Wnm!eGxT+_D>x7_thRS*n6h))_(INPxf*h{cCrpDshj~
ziQjv#>Ac?aR(-d9?$nig^V@gs{Std>U-JHWds{f{_m)O+?+aoT+0C@JYj2w7vVCfs
zAMWMJXWGNu5W9E##g%(s8!X*l6#swEBZoQrye<jvV-;f9=l#phmTT1ln+M%W2VQU1
z*mpmrai2QV^?lZxZdrFdx8L*O4ZppV_TRlPqOTwL+;n#DzdA<SNvWIn@K_$&d$=xe
zZ@tVc`?*o8_p25P?A~ZM&u*JO+rIeI_xFYBU)lS0-=jSbfA{VC`1-(Jj~f1cuePM_
z+yC*DO=m;uUWF~p`?#AJ_pVrfZST(h`+HTCtM<-qpR+f6Y1M(Yus?g3A6&M#hi94H
zlQzM9Paj(E&nuR)>0)}nXWo(jHX<oI_b0DjW4%K2^xpRQ6ZYkQ)UmB^d9_z9f79OW
ze>`@VZMwX7)4Tcm*qMv>+;OzGD;B!9zxTtFJz~Yj>>YSl?oAC{vCr{n?B2A{qkDJF
zmD#s7m1AFR_|3hKrzGyxH;CKY9`j-MLG$Q6evvQs9yr{!xBKz6J%<9q_ZxF`?<*62
zyZ2tz6YB{(Ht*@++qt*1m%(n&{<HS47aG~KO>;jG)*NhodZYaQ9`)&aLa#aPHQ6xH
z{(a<|Jy{3b_L}_Ju~%`2&|WWDMVsivqkE4{-?3-cInKS3^AGN|d9-ft9uMAqjZYu$
zUtZO*S7sg8UYCq7_J`w{td)6w+nwFxx;L>wckkr~AND#w3Oo?+cwl$$O_|-Ns$}*r
z;7Qo~b$Qa>^DZoVeXd(uUyGS!<9E4mUn1L@y-kr>_RL>9?Tx?f+dIYUgFVkg`MvX}
z@7u?+_w(M9#`$~COX=7M<UQIeFjK<8eWI-G%8aJHH@<A#H^JI&Z({W8y}l2-_J!BY
zv98R_-+#1b)4smImwUS#Z|vRF{$#IO@-&+l%O>r+yZej1QQDFHXA2$nr|f3i8z}s8
zU+AA#){icqw-%ndY=8C2;Jte{blOZ}oVD-Q!;rlzu5|BlTX<pjt++PZE?%X*kHg>X
z?R>^--KIEWuSTJ}-Gjxi_uhX!ZLiKg76;W*6`Si?C+&M}uJ8TD#=eh5_T1iU6G{&R
z3h&zM`q6DK-;dk-oCKBjmLB@Dck(*seUDpb><w%CwO1<r!M?i>F51{cO|Uy#&0;UR
z?e5;3`*HhEeLcLVq-f<nhkM`miq0vpzQ}xO@3G2Ko68kV``K<P?alL&w_UrfV_#Z_
z;67c29X8tYeC!?Cw(kw)pK!pbe!;%(GjDe9Kf1s=^|skw|Ac>gkGz_`?-<kM{mqdK
z4!ldX?elze_GWm@vDfmK+q?SO+r5i-7##5XbabzKqWNAW?c%*ZPHos5mfyRVVcnrU
z?wzamyf^UL(^IXpck%g{J;wriZ7L_V*&e!;xA(2b^u2eEdmcD>Hh*tEhxNV!Rpq_h
zGjHw5RoQ4&ygYYrRDSl}2i{?O9rEt(`+ZV>uhiXQ>%)d0_WHT8@AdPTy@$o9ci$x&
zLEFI07YEioS#9U+x@Rw+uHOFgsUP+_Pu#Tc>e=SKp^^Lcti0K}H)H=s8@3MB{e=d<
z_Dpu)vq$s%ntdw@*X%v_@$%lZFIjey3=I2NUR<%CSCq9^Eo8;s`?8w*E-z`?v+C7c
zo06dOwrum$4><It9|#MZX#cBp^4=3`oAz6HhwqKbw%g<OKk&eQmzR5+=9cW8r=+x3
zt6XBws?LbLXV&lEGx^Jly*sR)?lrBb*lSxDwI{-Bw>75;qr(G^{=NQ8?R(8y8tlt$
z&+V<?S-<a856`~5hE03t={?_j*kJAcRX-i~I;~>cSF@XSZ<$}i9{0@Ry?e8E>}e2S
zvH6&MVb7PBNA@wcJ+|5N=7YU<Ym|MSM(*C=+9dlcpEm9Fo;79Po$4ohjJap;Em7gw
z>+v<u#!NWNVe>buz1Qv9_8J*K+9%X_VsC+U)82VEx%Mwwzin@(o9lk7fJ6H}zMZtU
zJ?8Kp*P=d~-h1r(*sY@+68?DXpM9j+{&B}DyO5Jt_r7~$Z66b>WP317Vebs4?)?v5
zuHLK9aAB{1_Mv^%&;IUJ_cY!8WL~z-3a{q9wvQwB7BGeHT_S#L@8?xYHhwoV_veVo
z?3F+NW$yu}I*0YQ4({cS*>3;BOK{)R7a2Bl&6ez~Kf3xrB=f|*DsuDp?wc>SU)1H?
z-qpVu_ASuevRA}!$=-J?@AjnqIKI#6!^FKyzjW@?nC*XH%7Tu)k4%s4Uq6R!uQK=I
zefL6t?LF0Ze(#j`OZQItAF%fpyMukR@v%K2n^<j@#whMj$(^w~sCM$6Wa$`tr<p=~
z{j7K#jJ_<}r(*VM&)T07dw1&J*sJLfxmWeHgu};!4i0+%1RV+z5AXL%{bK#srg48q
zx}xpf%Xxd&JMlTFC%Ej5w7hI%GIiO$r?=kjwQexnyXCu=%}QOleJ>fB_nr!7+PCND
zv%L-vx9kn8x@o(uB5KdXH@r3n4o=wLr_r`|wszyblk*z)UU)jmM&#lzn_r<`_Su^?
z_FeAjwkZ;4-oM)F%U;%mg?oK=KHKZE?e(6^9q;$PNPM(!v$4(IU@6o64<}9Ax6W>k
zm4r`+{m-<&*0;Ql>{Daq-*-iFp^fczSzCtZe7kjb<=AcwoV`cGV&-1!e5ZY%43FB}
zS~zF#GDf2VHH!cDWYqE>IB1%;Z#v5go7JV;tz&$b?OlA~_g<yP5A6aU>+kQpIM2@c
zmDT=Np1b$Xbrg5dNcp&TvFg^nvM+=8-RqIs>yx6g@3+dcy{vIRtg05R*drRLySJ%l
z<(_GkF?;Q^_wG7u6u6hcQq@NDAhZ3Yv+MV&cP-iLdtA~^$>qY{@Q-B&rktK^z1HBs
zp3rb!+gmSp>^IqcW^b~r;+_v}k^5Slvusbw#o8(^?A)7s|M=d3490!s-{$U{#rS*g
z&OCvAzs0;9Vizs7xiC|A|JBFL`?!x=+Fh7`ZO@YP@_Wwq3hn(?G1I0sujW8juIS$T
z2HdutGQN9RQs3{*?Z35G)8M4t-X$mZ{?&hKzi7^sy=mO?d$>7w+vHu{WYfV{vzP1f
z%>CkgcJ}NmLibP2|7oY&{CRKT^Zom02u|NyS}MJ_ed=7h^0vCYQ|{l~Q=k;FS6AWq
zUb`5%y~XJ}Y*zhY+~>NlcyGnKJ$t<kwD<iz;<Ibn4nzCc@{@aIB3$;&|IM;b@yz7C
zznCQKXO!~qy==07?`CyY+Y8xB`zCIBy7$L=?tQ<6>-S!_+_U#%Lc`uC?c4SS%wE3N
zLG;AlScfP37@8yYvYE0ts4V|`z{9Y4ukxA-`|cG=`_4>wx6hn;%C5)#E_>Sql6JF6
zM(^FGxyF|9!Ir&>pUU=b+Vf=Jr+JI_Myi(YUGI>+zw5!>y=BrG_N!;g?pbaAZm;t-
zzJ1%R*4pf}UcBel%Om@>4p}+`&gVK1;lXvlN4a{hd54-qd%;6%ukIIn#bm$kEwwtZ
z_ug0Ey*8rj_C=JG?ooXCYj44Fy}j|z3T%(9|G78+;qu)a;byk3uNwA6{LQx)_$amK
z$35x2;cCbBhsy7?n(6w?_KDx6y)qG@dv{OSz4x8^xBcH`6!zU|{<@cQ(WZT;$|mkz
znsQ?AtUWb*PcNHq^KN3&UQdm4`?f7OxA$f3j{QFtJl>yB{b*0>uSC1UUzqm&FnqYr
z%2#5azmdmY@2k7^Fh1J7H)9Ed!~V$ydsVHz?dm)7cJFhy>wAS5r|kXqTzEh0L+`zp
zKBezpJZag!WlvA+{q^a%jsFzuy}uaS_MCs^zfXS$@4ov#8}}c|KEIz)y4hy(LEZyv
z6BpY2=yBh}q$1~Fxc%+k-!Inf4Sm{c{eQ|K>zJ9xtvfYt>^byh`yTCM$M>e@3+~hX
zcWZC?BF26FcNXkF{cZmq_cH!{Z#Y>TBv@JYHBUHazkci1Z7bNh_I_U~X2pIhZr?5=
ziQOGJclTbgk=|cCX{Yr;u8n(NuU%#38g$cU`QAr+S&jtlyZX>(Z||8J``T^C_KCmR
zXLDzU*M0*zvAu~QoA;GzWbeJ4ynheJ&HTMJ4;SrG-_&9svuxMi%laI9cRrEd@8Mv)
zNB`34y$>R$?N^z?v+v`{U-thKy7#G_<=wmT1NXigC%4&z=xpD+O~!n`@SAgXW)U;@
zpJlGy-}LeS-sueuc4BY7?%jVNdiSBg+56ANmhSy;Ghz2-_NIO8$JqBBuu$Bqr=+-t
zWx>I{%*<MQul7u{xx_bb@7k``-B<q}+rQ#;iS^SR7x(Ut2tA;bdSh>k{}%i7$hN&}
zlK$>}?fbwcarblk+dF>k{Zz1W?=k(G`-DC-?A?A|&!#DveNQ>N_1=oFmV1p?PTN;n
zy3yuimi7K_(Sm(n@@4n7)OPR7pSr;2MgF;cN8j$+`=Tpwk3gyI-c=f1HX6xF`>Py<
z_U(9bf3I!iwS5nF2<+RxQQX#_quIXj&V#)Qi7V_iHZIz?Ml5LW)S?f254x_m{lPkC
z@2T%T4${o-`>U-P?9_I3ICwD2?{!G)vtO`w#~#P*?|WWO=h^QleQuA!J9V2Anv?g<
z?A^CF_t^rQ<@THRMkTiI-OH4}*TFe&uT$H_z1O)-_sVL7+NbBP-Q!WgZQCVv$=*SX
zd0&gyO8au%ReLYImf1VMrfZM%+HQvpZrApjcRk#DX?oYb(9W(s+%w{Ayc;L)txfo}
zmo?_W?t(q@_h||i?A5Dj-2Y`V^Zw3aroDpC7#wVVa_`%(J#F8I6ZiJ+l)h)PK|OX)
zQ^|zA904-+`geEl<~Sj2aaKWQ-`Y)ud#4#E+qkQ5+0Rk=eQ(2@*LLouZF|eBFYjfa
z_<iqdj}<n~Qs4IK-mtdIc%yH9;S<||bDBR6yt(^xuj$@n`}W2;?uppeu!nUvpRHQw
z^SyTY=WXQO7Vqocf72>3<Jn%e@H5uCr~a|otjx6UgUPqOl}_h&$FqOjGm+t^-Is%t
z_c}=}+M6fu>F|1Ivz1E6AKPppKHG;i6Zagg7u;7dv*tiU=%T$JFY@h^pD4B8b=CR3
zO7rvfDsOzZm*WG&z9QBf>qSeS?bEr?vDaha!~<+Gn)ZF)3~VpVIAk};foY%V-d+1%
z-ru*kU2Wgq-&y<i3SK|HccqWBz1k7ay;ls?ZLGd^?Yqq&u+Kr^_}+&F$_Gzw<JtEj
zDd2#PP5nOE`&ahf|NPCyW5w;g_jkPAdzK^2K1``_f7I=N`=bh#_gCIZvCi@FcSzvh
zyXVTuN!BeqUI#ogX74^w)44}7X!pLNpjaE0Pb~YME}Fd8^vjjK_cH(P{qK5WZ-;Tl
z-ec8Q_r6QKv!7}6`MtNKrtB^3|75>oA+HVht55bqr>5^sn0{?<QIx0kg(;B-f&>@t
zIpvwWcT>QWeFkECY@UU0-WyQ+YL87E<K9O(!u!^lo!H0w_?nIPtIquoRTkLipIW-N
ze*e6^0beiMT+LX!&vBFN-eZp)cduj<wdL~^+AGj)Y4<;B{a*IZ*0x>#Yxn)iTDs@!
zDhAtVTg?NK&Y64b{y(%$D&D{E*1>gqW}0U2Eq=?fw|Do?z5QK``;#_w*mox^+L!i>
z-+uSLo;_@?|LjjLxxV+~4CZ~AcWe&KZb;l?CG>wU*Y4%}F4XexU1GwpPcHZ4UdQv2
zdqtPc-ut2I#$M~}BYO`_f3$bk%?10NW+v?APkp)f&B}TEzrSeP{gO}Wz}wXCdy0bh
z@45Q1ZjZll;C>H{`n|`+Pwf4%apJ!13uf<;dw*r``Qw-NJ~>-r%k+YKUtapjeM^Jt
zY_`V!v+ue5-hLg!?Y(9@x$M+ZcJF19-m<U1|A>uvzWrWzjfH!Cl>Ke|)-l_9yePGK
z?Hj&F;O~)rG8=B}ohIM2H{^5Y{*Olgb|r?z+0T8ne&4MdSNE0{G44CLvv04@^%r|f
zk59D^`@7mcZPOe3eRJR0&iTQz=kQZThcFxWy+<dm-#b(P=6;u(C-<J>n7dcb`r2NX
zZFzg+ICfcYF`uz_i*&`_Q${&fxyRP-?Vi8RI??pNzI@Br{Rd{)>|Gx-Vei5$#RFy&
zSoez9aUaOz{<fD>?z?s72BW=!D;L{Ws&VgI6(_#WX#M1UQgeFu+?yo3r||Jc>%;2b
z_Hf5_?G3vzWnV5gtL^N&r}jO}7dlYf^nS0&pB`HVg>!o|&y?@8*P6NK+B(L4@`kE=
z)p+mh;*fB5ko0-KmvL*p&9TQ*_c^~|-8DNdV(;5eP6wtJG3|Zidc%H>y!JkiS*vVj
z=3m^inf=S&o><Pk`8rJdcmB|}Z#PJ=PhY=%|E^6Hd!vPS9k|%J!Ny?8gS}E!Tzi*V
zi0yT6HQjSA{`0<>y`1}a%p3OZ`1xjU*W#ypZO<>-{gF>)pSF$f-cxHeY&iDk+b^EA
zeRtp-fqiK=r`xv+EZB3*?B)J6fwg;?R<Z8=7Sy*_$iCTrvGK&c-^)7oEO^Vgf7jid
zz2d@~_WnF}aqs)wwR^wpXx@AF?}UAPI&=1Nl(QY!6QFr;*3ZVhTnCQW?u|dS+d}Z%
zzFPJJd!^0Y_U_SpxqExYguS89kM5Nf<lXbQNO7<B(VP2H_ub#S^7M?oN@qm(^G&_A
z*GhV#-PMAGy}mxVdp#a5*?0El@ja>HJp0;@njFZuEq&mE=iI&4wigZ@3^U)$>#)QA
zzC_jD)t__r)bHUu(7tiy-ixRA?F|&VyDyO8#NHV8FMB;t_wBWBVBf2<bnf1ZY-{&E
zz8$;wd+C~e3;zi0fB0zEo-2OFd({Qk?q&MpV^fmryMN;@hJ96+C+?kd>B#N}5$g7H
z@1C()E@-zm?$uA5%~`u_sxFGzCj3<0`|OCt-U-`h>{;G8b)PtEsZECdiv1DWEDruY
z(Yv={tA@k(seOC-m?!T$zVG^8hN6VMYWo-OwR+mOcb}1x!<`(teX|1Q?aq26zCXX|
z_1^Qw*Y~Eb<KOS5yJ@e*t&8?kP5Src+FI?pQ2WUG;4QwrK8&h+w&z^jzjaFI-lN70
z2lTI|9%znY+xPaU&;i@s|83rHb=oT#{@gxaS8(6jcPV>WH_zI4dF8piUMsxze(MXd
zxy-+0@1Fas_8d2!ymxo|#JzF>pZCh>PB~C5*}1pbKXz|_N}R)tCxZK~G{3RSQsLOQ
zQ**)IKUraW*>3*X|29x}Z>@jeUeSJ${V6=Ad(WEQux@<WX|=CKcJI1XEB9t7&DqDi
zqjZm3zWu(`?6Z3+!o+P4uGnK&|Ks7_!;fd|OLh6Px9YIP-k1B;_code?R&Odd;e_n
zMSJrm3hd21uy>zfo1;xA-<!SNC(8Fv?0mns+B4w523F&}GtaW^iCMm8?-o`gn*-}^
z?0J9r+J4)KzxUT!KC@%cFx{V3B(iU_g5W{pn@W3^e=)K7wNv)M>WQ!RiuxMt%ev;X
zw=v@RUX|9Ld%iyi+iR`qzqfGH<30cMPVO}~6yCSjxp3c74*R`add_=uHnG}n?(Mb@
zInlLuQ!mSY^~Dx@XXd}yyKF1NzRCQP?A4`~?Nt#L-(zsVeqUkCJew_F8ExyPO|tUn
zJFrJ%>6X3EnwReTt#!%9>EQK!R~F^&m)bLR?=lNUJ9UGoy+7)g@8cKxvBx9u!QLO{
zI(z3m)8C`w(6`?&)_rf?hp0VvXB}++-j&<8$C-D}l8*)U94EK!U2AW@-}K3*eQJxf
z_PW*I+_UEQOdElS?R%SM96E6B!N2{^;stgbx+eP`v##HJSH$1|+oG3y-2IR4owIMP
zZGX<~y$5T%_ntm?cyF24gS{7yZ`u1vk9VIj-}=4Zc0Sn~IF)x_dehmx4Bn6S6o<ap
zXQcLZ*YoGA_u72$voHBkuq(&WX1`w4s=ccx*6#`YF1XL<^SS-;5i<LJsm|N0Vtj4i
zf|twoa!T&9E?xe5PvYhWyI05c?UmuZv~QQ-rM+LeSMEDvA8en&eaI%I+k8*X#m+r(
zA+Pt%Pch%qxa6tL?dE9fW82vF9qQmY5WP`gALo&&HW}$x_KBG`?7e=$Y)9aQ;|CPH
zYxbUOl|Qh$jCr5R3`N@q2QTh@bgO?a1KYX1i~mpCZ*^7pz!vxO_Bu&(?5AA6yXW$Z
ze*5M^8Qc86{=I?E?jK0exUu)P+VR~lTo&y0Gr4S2y>k6t?ZxNzvO3A^(R)#0qqOPU
z-UTmL?)CaAYO^tG<v#AV<$J|${oN~FJacbmN~5)&`VITE6ZdzYIm^E<)4tjIMy|yE
zwf?{Ma?M+0qsFpvpSEJaUiFeUd+U<7@6`(Uv-@CL&))s@7xpb=ZruB&qTOC2IBuW(
zlNWo6br<h5|HioY)A5D-?wyI>%k#;0Z|RN~d-s|>w4T)PeqUmd^1kB>lJ>H%oW1Xd
zXQ9;*)mt{4Gji>&s@Uw^(71FzQ&W(Q`J0~IyLMIYUE3SHSGM!#-i^O*?%$%hWdHJ?
zHTzRfNA4G2w|=kDEn&x}BJcKc_qp19IyC>lvlXFxPhVTG=LXlMeJkxh?mc|VZ7-+#
zmc6>t)q6FrAKc?&thihClFQyr^X~1H^^LKuIHtPCUe9r_(z6Epc+nku<4PIrPaFK&
z6Q$|6caeqWUgec5_wPs%*mty#b)O9Xv3+c_#rDoyHhpj8$C|yRevvlYPW11&-1>Z9
z+Ri)Hu?ZIr#4m2MzpwjqZ^g4K_I9_Y+C1HRX5YUhFKx1U8}_<eZr%G}rRUy@Of~kC
zb@}&t6y4jaeqrXm3+0-7dkUxSnPh+bz;%h3Jx0AV4lMDMwOJ*YwKrqZ-aYsJ#q5nZ
z_GNF#sx$Uq@*mh;{pozbef#BobCs7{JD>M=Sl|0|&!ly7dw*`<wcp`s;oh$|C)vp9
zxa<v^y?n3Y!<Ic(NfY)&mkI4N`KGXM=Ih?QJ$85ZD!K06lX>Coe(58Id$(<zyjMc)
zr#&C<^SxW`XW5@RX=ig|_oKbKCj#~!SF${?XnMom^!0svKOf$^@APVmy>5=P_Wtm_
zx_6KIZ5zjbS+*j3F74}8d$E^qZnqsb-`l-Es~_z>o^{B+$K}V~nbH0GCNKWG$GF5`
zuiCMFdp#d5-Me|Cwtb%`=RTfR`@PkZRrcxrliyd|xo_{>He-7`nU=l%A#4ZY!uRfb
zw{!a5dGS?y8fC8Uebqg6?-%8;152|64$Rrzu>ZJ}<bI3Ny*A=j3J33t-`eZ06R=lC
za>ssQ#d&*~Y!2>S6Kl11;lC=YboC#5Kg#~uYcNA)?{3z4d;f{1?QK>1xA)f4c5Cyg
z9s4uuC+_;vyJ|1b=L$Oq(d@k%t#kKEDEaNR{++OwzdB@ZVxEb8=$XX5^EQ6p`-IzQ
zUl#AG-BqnAdzH-@_MMcQu{S?z|6WI{HT%3h5AJy)anbJe6h`}v+6;T`TGreBuI1df
zE9mmRZSC*(il(vcRbtEEJ8{F&y_qxT+0V~eZ7sB>WuNdLwY}Ca()J!sQL%me{>=Wr
zY&?71gwyu(`}FT!lgD8jeB{br)>m`(K5CTTd(LU%zQxy=9XuP=_8zO!vCnof+`H&;
z-u^orZF`#21@@ipn`y6F@nUb_uMc~kHFED2Op)3<=jZgjswa=_eH>u6clq+adoSGi
zwztdg%wA4KX<NChCi}3k{JndH7w%O`5w=&_vdHES7qf$&)QP?CHQ(*ExhS$P+2Xpr
z%9mez{~NvCTU~QzADhZdo3Prsd(|7y>@`a?vo+Luwm164#(nV)kM?Scx7&VeV7Ir)
z(b*f&n`Q6*&wp=M@XdY9W}^E}td!rwf5?9C_K$D&-jbI*5bn*e?<)(F?UBh%`&ONN
zv^UoB<eo{cO!i#yta}~omL7O^<-ooK{))Z-ncHpK+2-ucTF<!e*WH)<B?2YwWe%7+
z{7zT1KYh@|W~b0;dkc=;d&=%~?FmkJwLdppY+tbd+P%%m5BDAWI%BWP!O*>Y-`s6@
zXENCIAKSJ^W6jRJX;N$VYOi3}TiBp%zs6NyU%i&$-aCt34piUM-@7nW)4|Mo=HA%N
zNA~t@Y_?G@KD)m}aNgdtHb-n$O`Ex|`R2>rGS}zr)T@o!b12DR?^!LUy*Z})_o*iS
z+<h(in{8ggHai36guNw;b?rNUPTBh;_~1T;yH0yEAI#d5`-NlQIq{8FYID!+Q&=Lp
z?~nMay(`VR_Bnjv*~=r-w}(l1?f#rfj(t~FPuufMrPi9`&fh&@8V~mh${6l#4btB`
z?SIKW!R1~1<Vthw|Kw~qAZIMGS7l=NJ^@WFn<WpYTkl^LzyE4y)83OWl5F4htlB4`
z_;7D%R`g!>^3A&o^p@;B@2j>a!I5cSFq8CN)4QDebfwJpNp4=dXJVb?UN-YJ_NTNa
z*tq#D-=}r`-yV(wd-v+!Ro%N`(f0iv4-f23_37QaBA#jgsx`m%R%-I^t6bT%XT1#5
z-bEZud+Ov1_wCxeY4?lhIr}%|dF)MlIb-jZHU;~JhY$7?e!RBtW5SfZQzo$PRSQ_P
zSK}ntzNJc!?H8~7WW6V$Ztv9_*Y`&Kowaw5^@hDy9lZ~@a24!j-?#I?hD{#(N`GG1
zd-lPS-TP!7?d9|5-M8lY`u$Q{ybt_kd}L>LpmX2;i(PxFVlwUhy)NuczmmRJ?e~`b
z=YN>(J$p}kuY7#|UQWf1z3x|w_TIUfvX>!<X<zXUQ>&+UC+>-y{B=)p+mF4+zjoTo
z{GV;Z-sHJA!j<1)-;PUr9xH#gKQ|-ChCz*a@3s8N*5OQE2N*7@+uADs+*>?x+CH_G
zFMAU|vD<JJuHCE5$h=Q>$?d(mCbRdwX1=<|maWBZ+3%@#txl|aSy#sHcfI*-PpMhc
zzRSnA?p@<rzW0Z;g>B2a%Dp}MG6(kkpJd(A)xXEev|-=tb1!x)ys6&X{q^1cxTg_&
zu7>#^usyMHk1n_K-kB^nc2`7l@0-$pd@oCe%6_B#@B^m}+Yc~xcsN8j*VyczG~F)!
z_1C?|Wt;XW-=A*3#qN+*Wxd$Gw+5T{6$P5?wVo8acaPJWy{3%Q_FhrCu=n9}o_#L0
zxAw+(weN|TuzLS_i7k7Z^{?z*`fj2__=S6W_c&_V-%h!+OX%8_y&~8C?B#vgvY)Si
z!d~HhllHRvvF%&5CCOHK(bc^g4JY>2>Acu0IGtthq?;@D*@v-O@9Vl`zc-TIA;z&_
zkJjYo12JFs+WfQ%+qYh#eXoM};k`T;8TN=Ckg_RYR6oFNyJxT6++}+s547*wQ8m%p
zVa<iThZ7s^re|C234c(qZ{51*dnct`*dv_v*}AWnao_2b2YY?yZMAQDpm1Q?>V5kb
zN&Gy}yy@y*DaNPv6?@zEM%N$P>-2Mh{Q*a%eRGQh_Z7wO+IQb>-d^FAfqR!O^|tw*
zd3bNXGQ+-^Mp}D|LSpx-Pi@^>KFw*LtlizcpB}E<%Os=bFyYmbJ&Wi6-?zH>%3cPC
z<$EuuEZDns!Myz!KXL9A*VwuzeV*H152bB;?A~$i{dalo-lKgt_PDV>+AF<e;l8WW
zXYBnyW2(LQM`nAYdjfkt1QglrZ#=vA&YyMrS{?uFjh>XW_W^h0-lezJ?hWZtwGU{^
z-Pas-df&TcxArx?o3nT4k7s)g)btL_R>|9a_4LXEQ)bm#WeV5X@W__$+pRIn=H{iH
zdmIy{>=(NAW8YKrsC|d$&amz;jokBj(xL;ZtAFg-T6tpkv$Yln3>LnyiHf_m_si#1
z`<6eMZc`s5uy4En>Ahb3?ECJ|dAn!XOw+yG|Gf7KMQ`7CXjAikvxb${mqPaMQ7x0R
zk1<|v9W7z8-_nw2-=3Rn`}QYp*~9rP)IQejpUtMY{=Gf28nzr;&h0Wf61A@-_t9P-
zPmjGuzMuA(axCB1p|x*Mt7y5sXcFswIkx?K3aWJunlN+h%i3bLujioT-p{7@_8L#t
z*;DteeQ)2Dt9#u(PTzZH;-<YPw;$gZvm@2!`A&<y>?srN_f#L>D{jJMuf%kA?|Ie7
zd)Bks?aeI>vN07}uy^aMr?%_LyX<#FvK*MI{or6t$>}||3R(6k0^xf+mh$ZV8^-A1
z6x6wQlfItqn==#kg}-^f_wpXLeZ{Q{_huK$?fdY3@7^z0JNBNrb6~HYX3pM8g)IA1
zIlT8yTc>LCs9^O0U*-Nivua-0-`~=<_wD4_d#5QH?X{^@IM`LIy|3i-F`JYMrv05V
zx_dQqd-s0UnYH)BjK6!0)P!v3bT#aoZ!Wdxj$4hrcWu`GJ8_aW=`-C9da26od;0y~
z-p$;VHde}=d)EA6-zT*^b}#=zH@g=OtM**-W!`r$@WZ~A6NhXj+&{jz<%9kHlzPX#
zpEWof+PFpc9{!-R_fJ699!FuueMciU?h$^{W*^?b;_!dv>ix^vW*#(3Ikor5j)V4l
z+u5wHUzOT>|ALQ$N@>mB=ihSnp0zlzulc6LzN-y;_Ha#}viHKaJA1qOKJBdveYp3W
z$BsRxn7-_Jcg4&8+!@)uI=*-JZpjfnc<9Yqo6<wQ_Mbg{_dZ-a-`ZJ$ZSQf@*#{<O
ziR{zuiP`(^`HOw!>~VW;-*(x1ZdK@BpQ<ivgL%CB47D!pd!Amux1nyo{q2jQ4vaTm
z@9kX2dcaD4pEci9^?kRBi}#*Nt=s$W#6_E_`vmq%u>INJ-~48;71!3i-k0C(6KP@C
zC&qi#X2qA?`{%RQ?3H@Yb3h<I%I4sM(mhKTWZQPFU9-1nli<GeRqPI8mt_ykcJABX
zYb9`?C98LDD#si<yZWrX7w)Op@cD|{KlhQ@=k5@&cjxWZ``Q&B?q!?Txce~!_dZd+
zw!L4hR_}Gu;@H=1?z6Y?<*B_~X=Vq`tS;JHwXb3CeK!^d?$a~&Iy$%R4`bT5SF?j@
zpLO((J@>m8?O*$0(cVRR`g@(;O6`ld`^2X6ypnBD{+_)Zwr)1Qd@R;7Q3v+<N{a2_
z-|%U_n6;_>zo-p+10VbDzp(bfUVg#+eLq~=cP){6Z2ft8=U$Tv=6xz_mhPW8ZP{MW
z#AkaG16%e;ew4KS^*dqjQsyc9gsOJ!ElyvxU-Qg|y;p)??!Du?d$0ZcdwYam$J-oy
zUuef)Eb4IMNBaIZWxMVAS9tH0<G;KAoZ0=o%KI1Wbr5)KXA-BhPn=O`&#a3@`#yR4
z?p@^@u=kmD#V&hUrhU`4PTm{euD3hxgZbXM>4G-xYwzt{Qn6`I(tbAEgu`#_PCp9W
z)6+27{&Nn;zFMD-y&Wt&_j){RcTjrVyf-Fz`|fG(7xwNoezn)`!@a%JzE9ty7j<G!
z`Gl!^BtD$o_t~yyuXv-CeR74_fvoyjdyi@|9^8~$W&PQ#W8abMyKQ6^gzWj1-MIH~
zr2O6$YN8GcBb)Xf7MNjEt+s3L<KV@6&+KNkoqt&QKyW(KzG<^s9Ud*6Vx2#we7Ewe
zPuBCSr`lZlJ8AE;tLN>RU!S(y5g4(*zr6gw2BBwrQ@CE*|6w<=DN^~q`{fA+`?)o9
z_O@+Uwl_Fs^S*ck-#x6$rtV!Ic7Ct%3+8>tm)Pv-N>$o>UA)S6w#I3j@bjPT&#gGV
z_x!ILdrzHUKj<Favv<vHxqVEAm-jx^J-xT`;Oo67Ee`M7_?vSN&-4|0&j<b7=O7lf
zH{5Tv%|-Vmd)=l@-&6YG#9l+q+xvV3mfEy8s_btpxPKsd$LhT?r;PX8?hmpNi|X39
z{YkfVT<wOvFU>dY<vP@~hiB_qyQQ8S`}~w0_wGA%YTp5iroDBi4(w&xyy^f)CeOa^
zJ<|61&nN6V_0M>(S=s8n+h$(bJJ<O5Ubiz|_QscX@87q@;lM+gzP-+^SNCX4?6Hqy
zy}kFR*wnoRK6Q3#y0`bPv{<~?<4&NRp*_RC-W9!j^?hFL75*2p*Td?xmC<r%o5icI
z*gP|OySMghiT(d5-hEHpdG`IO`g0)X-oL$v&j~psdrY^kk7wQ|SrBPczLeWO*lgoo
z!<1cnFZy2JSDMhhC%I_O-Y;>Ido5q_@4YlnWN&cVhJ7J_yKML_^&hbD=XSWgMbox&
zg2RE(Zz8t)d&2e!7*E=(Qz5rkEmh3+vOe41tMj-ITyAZ%Ipe~y_jY!lt<!b4y^;)U
zdzIIg9rTWxwO4-cT06a;H}<Xk8@hMWOvk;m-ly-Kc6#-miXTVzh32y!6g{|e|Nh7+
z_FF`R_jTmFvuB@wbngy!B^!p27xv}ZpLbXJoZQPVvTk2(Uz_cfH!XX$y?^g9DN)*6
z<|bfUI)!<!tV!h_#h<$@{m*gQ1+G88=WY5@o8w!`_uuONxL3Go@qu-{3vFI(I<q(V
z*NVM=SKixS>GstoRrJN)-c$YiZ1^Vb{p2aIm;bf--azSvdsTJ??NxSNwr_^=OPegM
z)%%ypF5GwHX#CzeXZ7qSTOG5B5DDAIJNM}xhW)#(ZJx^PebF7ix9R_%{Z}qX@8zg^
zzqeI6Wp8E0%{?|Br|jMNp6!6Fa@pP$A!7D+ezW(n<(Tcw-8f~hdtt#Io<&!zl@cTD
ztR~E|lVesmu=-ew{e!5My-$BnvlmQg+tcD9u_sXeh<$Md(>`%V{=ISC)wUnC)%Nb3
zFS0i`TyC$&WA=Ro?O*olvG3d4(ks5NC3?x8l#T5BUA-FjT<>Pvll0!+-X+9sug~Wu
zhnttY_ROp(wz<`lu(x_f?18;UFYKMfv1D)Flau?HHx%z}TXbdb*<iuFn_O1!%{$Sy
zw?4pg-(uOm-B)JFI;foyvA?8nb1y@^$$|N6|L?tG)wAze#W|bJB8T=`#_ivGG-9jG
z^v%)xWqz--w*JYmZ}DTD{U2|)@10uOu=nEYEPJ+J5_|rx?A?FzWar)zel43r$_BeP
z%O0><9u>NK%dL0&E6qaekLH}We`~LMVAt`=J@SS1wg+c&?iKvMb7!ixw}W2A-@SR&
zH}+U9y}55@n5tFo^s~DZSM%=M`%!#fii5^p@iPzidYxUkSEFv@?k~IN*w25oZ0|&o
zi+k5QT|6Kw$g%hM5v2q9v7LJ|?UeVFnn>(hsvhp}=ab0Z*UvKc#xSqk_vHT~Yn}z)
zZMr{9-~07W(cX=h758<AT-;abYq(3nDAVr#AF~66KUeLxV^iM0Opa;a%Q=VkJ)GUV
z=Z{6+-eYT=_A0gB-uvMtmxHR#t34b32JB7Rd3Ud|qUgSDz4vUKmp1Rey4z>hCWU|d
zKc8yZCl|zNdvDs&z3ytYdpYb+@0q@Cy4`&P+5Ib@2_BI7*1kXLna5t0cY+Rk7JlAq
zVjj3Rb*Z)e`7P{wkMG@T<;$tG_iOXQy;GQt_X<wo+9w;tuy0xfx6Li4<<^(W!fgBx
zZrsDRqRC!1DQEAQj8l8J&iTA=)}IZ#?xgMC-(%BY^K?V?-pheKyFca?+wYM*vseAg
z;XQVC-1}p_Cs{YF5ZM!E@n-MR=O=e1zG$#Lw<>5~?an28ZH3qDTe0WO0pouud*!#i
z-~XkId2h7poqhh%6ZbR~sqb0CDYWl8zlZg=_qq->bF25rTmRo{P~y3-B6qFz@`&Ag
z9e*1-Tsrk_@7qm=```4f*muhFjtx_^x6LBcJ9|&)Y46oJZ*{=+$3FXoQX2d3iZSia
zmvOY-R@}7z_uMR->q`6g^2}=A_srzb-oUnldoJ$SyYJuANE`nzzxPg<b#-sEn~Aj|
z+x)%O^-6nZmYVGK*FLz%Z`QW`I~BV3?$yuTn_9PM|JCBtdpR5`>^Fs+-)r8Su~+V^
ztgZE2Z-<FYZ2M|D8TVP7;@H1EXu)n7vHHDTs+Vl`PK@89f1_#djTfE!7Pqb0yD)Q>
zy`B6i`><ZNy(#l%A8?82-uvNz!@lEBnQWSm-rsvsYv0~eA*Fl&zf?W&e8yxOa~Yn!
zKb7|Fi(#3vcfqCwyS2(D+2?og?0Y;V&EcHo!+nnzJh55g{&8<fdz{UN*DU+qYx+Cv
zG1|EQeT$C6(y4q0H{X}sd;QyS`{d%id)xQT+M_J(aNyMw@qO7p%JvlN?Av#*w%Nuz
z>ZA46ZIAczs7$ghoZ4mmWdG*9Wm+@$Zo0O7?<;{n_B;N5-n&ASbDx=~p#9EYjQdJ=
z3hyuHdb9V%%WHdUy{7NI8}j?Wz0&P_+pFdF@Sa|@&uPE;Uh&4iHu<TGZG_iNwiY<O
zaStnp+dg$s|Gi;X9_@cq6mlT>+TXq0a?1M`s+a7Q;C9-ln;@{a*?Gd=Nypdi&9#-?
z>-uYt{R``_HplL#@0D#@x39k0dT-0tb2e+TzuDRMudopn-nx%_rPSVQUzYDJRb|;H
z7QnHu;byXpi-4uWqP?mPvr}3QEaq@MDB1dVFT;yYd)H8gz4g0%_spKM&i>E~`Mr^)
z;<idMSN7SurSFZov~-V7@Y=nf-6D4HO%>hOw9L)=DBIJ$tu0sf_7^VQulnQFo<HSF
z_wMr9w*TJz`+L=2-nSRt&$N%{#@4;NTn^Y6&RKBaeGS9jt!ESVUbW=kZ<qRG?|O+F
zdkP$;?cHd_y03q=uT9X075jdxbnP|DpSNGjc-{WVH@EG|)DGK!^1IMJ;ltDRbvsVp
zTfd-v_c^BZdoP}yv)3n}*Z$|EqCIUFFYo2p(qnh{59_{muA_ThmSos(*?(tm=$yzs
zZ+zD7tIgQI=V;NFy~eg1_D=sgbMJw^Py1f>tlS?YpJ|smfBQc5z}`L1Q|BMJa#z&m
z*oU=y*P3m%6MQDNH)Z<gy`LWi?~e&$vrXK_vA62YhrKro=Is6UfMwrO2Ijr1lg{j2
z`Q+5z<j+bDOL+43wtIZr>w59>{`|Dld;Kn`?O(9|{oXo;qkB)z53zo*>agAW+57fx
zI{1I@+qb*-O_Q6oS8@4in~YqReLXAE_AXBr+PDA6*?nn7U-#B~-f}o_VvoI6>usB2
z9iD^Y&HT34(;n_sXrHt<bj{hl)%k1ozOY`nxA5$_{Wb0@_XhS~we`OywU15Fb*E^e
zzU@rGh4zLHoqOJFh_PQ*T(PgK@cQ0Msv0&yTQ=_9^@evJL%|k%iKU14`}i<8)J|M{
zAm!4wy`LEk?XPMi?A=y%b?>C-iVh5m+V;-*ExYfip22>m&oz6?PN~}5-8^OQ(O)ck
zrHj_@nZ~YX^SrQeucjlzzDEwz9Q3~Y-5XW<X7{{q6?=2v`+I*%t~(GezQ%eh>-)V&
zbeHbAyUYH-bV1X-*IVoMe1EoNUyjh6z4K#KY?Xg}-<x}O|6Y$XDtkZu{<W{Iws-Fg
z2X*@=Np|*^ymnbTKDFF`c1rD@PhCy>Lf?q*os({1GiOotp82O&?De|%&;Fd-k-eW8
zt9EDd1lsK`SihI?{#<LD8513(Zt(4m{Wag#_sINxG4|YhnlC%-k}zZ2Cvv>YMl)}_
zUHEP(`_m`14v4R^v={guxmPtq#{Tm~1sffiqCMrCFYY&H?zQ5VSY-3-+roX3=U(kS
zu+VU?!u4%?{*`y_&9i0S7rjepAH&RLd$knz?X~7sKhR^Yu~$T*aPQoUiw@+^JGeJw
z$wd1j(k%PF-Q(M*Grw)`v8T7}MY<&SO|&rCbE)^pJ`PUxJ^z0@*!*E;-OH!myf;y4
zhfV3vuloe%aP4KWt=eyYziWR|_}{%g=hp0NSH88UH)`9yL>XTz&o2k}UUNLPx2rJN
z=0lz0KEdgC_eyep+q-(>mwo=fjQ4zg_HFN_ZSVH`@A|hlLyzBndB)~_%30I*xJ|gV
zch<)Zdwv_N-LoQK^S=MTtM@s`iSO5U|82dFZT()`%4&P>OQL(ZV;}EYs5Ez9`R65j
zP0b(gF+2Zi-%h=jJ&PV}*em#%!-mIU_U<$1KJT%)z`JkZF1fv&0l`*RzF6*YDOqDP
z=~&O6&0e$YYCL=Q^6<&*-+$%g-kIz9tV_Q!?(2xv*{^E(b1z33_rB*ZrT4FLyRp}J
zE7QI;4fpmoelXjc^JewlX<wG?TbA~E?}Ax;_8&FX_qU|4-8+MS*8X}Pm%TnrYxdPv
z{@c6x)LiSj+J-%f&FMBv_sbrvP1D;W-TrsaeK+I%+u2_1$)0m=FRS}r`#)cb_fGz`
z^1ve1N&9%Y_4n!q9N0bM?B%^d(>VA2sOZ_(H}inq<F5zp!#AiO_~Nu;Z*!}}0o|4D
zdp}tmu;H^2ckumn#YUp^<=$=PllEPT{IK`Mjn{jNM6CB*<9NNdGhqMT>PPK+3ugH3
zwOV7h`+ilA{eil@dmeUw-}Cdb*Z$4g+<W6@H64hWw0)0!5yL*M@I8C0vx^V7oZez%
zxci^=k8*+i>AP<35t3rtJAMDzz1iOD_Ds(4-D|aB>%J*BSob~A`e?st2mgV(1!rs~
zhWqZ1c(!tHc*@y*i4*tiSsT4^&olY`d%dEs@3nA~JMihD)83_tO#6gA=IzrpoU)gX
z<AKers$2UiUKH((({0?>u>Jeqf)zS@JBsJ;-muqUuZC~p-j}_s`@Hf!>^FIv?9blI
z;gJ0D{+<TD9=lknxqHPY_E>JTc5zUQW#89tGk15!qs9BK&Jx}GC|tp&TKn_fO*<L)
zmK9Fhn>_2l-m<3Md#*1x+$+T&>5y}GzfH5=!M)#E&h77Czk6@==N<NcGLG#%mbPin
z%~JWjL5~~l%On@>xpVN|-gS1*_N{Mtx%c6+yuFJLJlosd+H2#tsB!P+RcrU@s4v}H
zzUjw)F~KCeXOnDfqW<^UGu@rFSLWG~eY{1y`wmr}-rJP^#^%oik3BZG%ny7mT(Nhd
z)#N=NL%;2r{A1o;;k1i;5AXQ5@7PX$8#l8J`=z7r>^qlku(vLI!d|b(U3)FPU+m3!
zd&d4#Adh|L%9s0pO|9JP8MSP$*Q>w#g%@}3omsGX?~mgv4ybk5?frg`b>EZ>&ixS`
zOt!xrrT5MaVX)ry&|t6ogr$2vsq^pIesI;^-s4hxl_y&5pS-tuud$)oUZK)@`=!?u
zZ0?+xXRq&7x7W0=e)mK_{(ZZ0vhAh#kMCW1OL6ZG`xX11KghK?KjHRXwJ3pog*JgU
zCF`sAFfzT_SHXI9@1o=@_IgHh?NcQC_sThMIN)BCwI^eK+dl0b$$Kq}mhL&YHOVT6
zUwTjeghczhbFc4pz5UAOmAv`>`KgEYJ_?W9&656g|D$*7_G)DRx7*RQW}nU)A)Cf%
z=e-VTJ$pB+8|>lpWj^5T#&BTUN&$z8#|s^5q8RqoG+y0XeRSGh=e<Jv0=^uvGZovr
z*W!lu-dQQ``%2f|+4D+Xc#qR+=e@#GQ}%jfGw*xp)v;IogUFuAkuUZ%IqEvhzFfBV
zGhfGE$46`T-#c!*hwEUs{c>N%eGA<)_6Gg_ws(E5*@0_3yZ4&>nYm}#s&ji&4(!{@
zt;M@{KI4Kt9S<A!UJhQdXHoZ+edeyNyYE|T+Pk$>><^pGW~)&iyKh_X**&4ZX7Bqx
zfqmbz?ghIQuWr~~7JX;W?e8l4CssY)V|VT5o;8Q2?d3b%w^!{>_ufM1{QaC$c=!Dk
zm}AFX*1gB*!{)v3&zaaLUAt&~b*1s{165z`cRdN+ukF#ZTQFq)ex<yZdrOq~4~R0)
z-ot9W!1m1y1AF_WclKILwA$<Ye#ySgGiL71KE7?YneD8-9xp}r+PP=#sc+u9w|c_2
zy*xJ-?47Ev<8Xgs^Ik2HJ2ua(qwUWu=-Qj5H|M|xef7OR1m*Sw$U5w4FneUHxq#os
zGNWa0XsZ9ds;Dn}r@WeNeV&zn-vaN0dr!WYvUh7g!~U{PXKVh&QU}hz<2f*^b=lrR
zn;G_*_6zpbcFfq9UHW-%<BeXMQ^!v3-T3t5o@e};_S0gv?<v@I$mXn2>pm{Q(|bR$
zOxe5Lq-&2|;k&(jpBC&-D%h~^?cK7y%3PAW;_GMcwYkT&uV^LPfrXhD?e9$CIk2`-
zWq<VlvwJ_aX4>D<VBeP@+rHQRm(&4<|N48kNUyVT4nMX>y5`*8P`0wY+55ii)yw4E
zw>?j6ubVpWzL+%by*sSd*>0Y?X20>%-n{{VaeE@R`Rso`hk4(M|NIVL8z=4+-odf&
zecCST^&h?M?-?B4vn%X_&5e-P`?z8z+iVp3wU=q3(_WL(n|nlAllGRizugzJKzwhz
z1mA&zd-o2^+Q+%ilgDWfPwdV;>vt{P*Q@?x&*49Ldvj~P?-fvYw<#7~?XdW~*uH(a
zhpfYyyY_7;ZrSS;*tJ)_o7X`);l*Bu%ENn?MqS$X_)o*0lUx7q4Un9+S15SVUeWzO
z_J^)x+Y@A-W5*uLyuZD4(q2*JyZeJenD*_Q`D^cjq9^uZ3b*#WC=a#eTa>(SrVjhQ
z%K_%rHgkmb6qg>@%kkydUMZoQdn@@L@73-u+Ix;W#6Bc%>RvA%nSIOO6&*0kI=A;{
z?3sOUAH3aj?#|cUYgc{Tdq_;${>z0ed$ylwvWaZHy>DmftG!!yN$#2b<L#cK|MT`5
zESB7pudr@kXWWTBWh}}L`@PrO-~5)mr-7II;NJauHr2n*?#uFBy+^}v{hrq2thTEU
z>g_pcEpMM?BeUmy7sEa-!K{7v)mHBjT(D;kv%`D)s{OzAJo2)(d$MHDzMcEn_U-)p
z*4pIVm%X1)HSRsn<F$W!0r!4ZhL^V2wfpxQ-<!BsKJWfMQRTUNH#&dcs~(zVA8|@-
zAKT<O8{Q5DJ6nslds)^!-TUvizRmKEf7UX(oA;<h`R~1V_2%9Wb06&WF5+`Aa}nPA
z)B5=y>t+7?ZDw=rt72NQzco&7pUK<KJ(F0P_O`CMbKo7j<zB5{ZhJ0k3GFjHqq|p>
zdHUXSmt6MlpLEVf-009=W{+?CK7Eng+s_xWpWC|g07qZ$UVG8D1Mep<usX~fyRRxa
z)+(ZweV@7q;~tO0nR^Y(zwSNGT(bAu5&eCKf*<c&emZBb>dakx%ec4guV<^-Q}uYN
zeY5nueJAGh@AXM~xOdh6+j}$iJldOLaby2sx$Or6wi-E1b@Di1B4NL$_^G3PY_{v3
zj24|eZ4<cdgW9<Earx}u^PlJWKJjJzd%c`vZB)#@?!8o}zV8qBk-dk+By1K>@wHWS
z&)C~;qvkMA`qy5k)6)C&6}b1YCcfNrWb1j`ce{`5jg*kFS?IKQ??Ho&_RB-F_WsId
z+gBg{d0&6a**z0F-tPTx7QEN_k@4>8xAl9fC(YY;E$`6Y*&AE-v&@>mf2rE-y^{)O
z9f&@AV)rhGO?z4AZr|&+`r_X5mMwcy6l-_Yd+)Vxx+k&s)yt)OgY}g5M!3zhIbYtl
zSE*swewnQTR&zda?O#>-c5lw<Gkdj|cJ5vOe&HU)?DczlzPsDavSr>c9Ok=!zm?4X
ziycR73ZEXecl-Qm_s%r-ecfSP4!?eK?Mq*7y!ZKBu6?!czP25go2--ONACR}SGITC
z{{Fo_0$=w0UO0JgA9McRxk}6IRW<JHEfNpg)3rVAK%Rf)-e>9l`%8^oZTQM(?CtAz
zw&l>QJn)lS*k<$f_j{LRweE4S@Y}nur)_UY{T%DXuJ(JoOWODPJvp+kq~XJ!)i+NZ
zn4sEoVClMrdp4SVJy3Ndckja+7x$jGKe%_H2JgN}J=uFt6!GlqI{0pX(RH=miR!oa
zcC0+V@5Y_Cd)<6n_8K;~*tbn>x4C^-&cU_p;$C&dFME%sEZ94Je!<><rc11t>tgJ$
z`mM2ZmoPq%{O|g~H|>-5%EYKS+!5*8`9h^&Pp7wq!{O-ZyQ9skZ111Hy|2H4Yv0|r
z$9pX%vF?d7VBQyQ#lKHt;)1<rs_SeSuQAv;@Af`0^T*x2PwbRzUI|4UV9k!*vqOf#
z;l+lZyVIlv_WiGC*|VRs@<4G3%RcSXNA?=+y1q|Q@zUN*CgD9_%&Ycx{64&U<Nd{Z
z{&64P_pOI<-#2G9hlj452iD7O+q1fj{eU7v#$HwDmHQ$)PVbF~J-WBWd-7gxE5Ut*
zCBgP5KP=q)TaCx|r$fYkx6f*O>*P&s&V<$+Fy}MgyJxoX{yh&@?@J0=xi@6bhTYl6
z`u0}FM((XJG2CCQ@%(^o-Mjr;PdGSiE4JDDXinw<rRyEL{cP{<T|UXl?nyJ>-j;^R
zdxigR*e7;%+TO11JbTNY3+$U>E@AUBal+o)-RXOsrt|Mp`q8jAe15jW?Y$58zSwKG
zcV?}WJ*(x4y{kC39%#71urD;Ae{W{kslBa>PVW8Qn7=pEZSEeAYaRRM<)7T^w`$Yg
z14q8?tysQs&sHIE>laC<_dR`{zsF?xTDz`aI{P;z&$hWIYkD9-P{vmA*z$cRSnBuo
znOxa>pliuqT{X6S1#xNzgw40_xf3X|*QT3wuX1LZ&Hkrv_g?AVb|AXr+1|dc=?Cs)
zz1@>EGi>j)3q`iCgD34xp1F6=;nZ37-Hw_Erk{RdpTaiFer7}N-f-45d!5~Cd!@@B
z?2W33-(RLKyjN?U;~s+_d-t~XaoalI_`f$&;{V=vOWmw^d>Hnzy#KN1RM?I^_cW&O
z_O#PGa9+N0@2|}Ky-`Uk?W((H?`EAOd!UTr+1|M^Pxc0^yt_Ac@|OMfkKgW{6mxm+
zX5R_>IHZ;K{z-gmBWHPWZ>8O=y<3D7ZM;}^?)zC}u=nqFBm4KWT=uI-FW5U@!ScYi
zl^uJX@(uQV`(d@$z%_Mm?(7wN_cl51ZF|jopn58Ub<@YoHnk2X_N6d*+Gt&UzV}C>
z(t#cKg7-{V(0G7b{_x(0=L`0{R9~>Cz%<fQ`Bdy)29^r@2|QwZ@66$FC~bAIuQR!2
zeLiN=zG>UF_uM<Pd#}6adHehSZTGCJy17SY)|!12!Y<l;eCBO)UBzgxvQXt7R?DJ2
zJChdfwG*h@+jhIu`eahify{ubds~C8Y|p&U+IKj@dheTC3=XD&u6qw1*}3Oz_QAdH
zEcp+}x8B{u#<pVb>4xro{0+}+3cd^31PJEtl{Q+kx3_nt&7Y>J`*;?-+<VEt@L)v*
zk6r&9?Y%D52kgI#Httz@uV!Dj-L}2Yzs=pd)y!-!!(_pI8w@4vFHYaF_kjPkeH*Pl
z?VGZeb)QV%qdiJTn+~|n*<zy>Z*icEWzSxvC6jiax|?mw{Mux%>OXUv1q+4tvn<Hi
zpAuJaU<LoCy=!uA?ah;Uyx-}No9zuo&prH_3j57d58L>PsqRg?V85?^qs?CP?q_?{
z<4#&@%k%DQTYBGG<HeCZ88;jE)Ectvd&sqTpKzS}9`-}u_bN5qwR4T#x98i&qkFm4
zoNSi;oU>PHUe8{kMuGi;1?%=Y=c?~L_U7JRiH@^2RaZs#ZoM$udTHzN-3bg2_MQ}4
zy6;WO@x9-e4%?oSmT@S|{;~JXob?B$`F+{DojGvdx-(XL!k(<Q7X8M$FC}Nk-c33o
z_P6F7-*ZRBaBr$<@4nTC5A6NaCT}xSM#y2C`};i#4;%MC$iJ~yfVpBXi$ImtZU4!8
zecYRNN57k8_x@hIgY|=Md#C@q4n#1NT5mOFwm1EmV|~+(f8RUdQv2cyK6@=c&e@}0
zv0>l);~9GceXsAHFvn=GZUX<lH8Udj22V4#`Ny(;@4<~XcSo;XWIOk2z0Kc6rF;MD
zUpc_X`gJdVcJKagXZ!Xf8;R}ZKEknYB?tS#nL-TvGWN0VUSa-g-_t;^y>VgHdtEoN
z>|>lheNR%^R9k_*j(t;7FWES~USrF&O6WkS?wh@Hjuam7t?%8ttA6LcQ;m%K7M$qW
z^FHpt-cM6S_I{UpxbN83f_;y-ckNA@cYE(0^MbuQwJ+_hOx3o#dE?dI@2ZpRAN<{9
zDRfd~uMSVw-W~tA_uZVkdavD=753Gk4E9x_>ue`$2^~~8zi&_QDO0=h(@xfxPhQ+>
zYVyH8UPNQ>9-~)#Qr#Z!Tm0|9UbhT^efnJ&_ZB8d@8xM~w7KA?yf^%;<leOc4SP=}
z$=Yv`*V-%h#%r(Ig+&JrR{HIocFfk!;K0(oeakcVa0LkOz3t_A@W<)rd&688+RTx;
zvv2a<r+c*(8~2>~GkY(q>h-;jY)bns_rKkjyXN#>{*;aT-YcBh7dB7ImU{=szF5t7
zHkud0_6dBb+OtvXpiP$Ip<Op#+}yiuGyj2{f2;Re{rG2-)v(uQNrBE@eG47i<iAb(
zH(y!1S7l}5fphaG>^&v-ey_E_xxKG7s`ox~{I-|vkdM8>uDScGdjuRr4Mpu4US8S#
zBiCjBtlKO0X#EP^lk_Cbe(Tb2dv>d2@4fwT@!n;Y8~1MB(Y4o;&2#S$vnHFrnv?ft
z&FA0Cr0{o-{ItJ&Kl~6n@FKo?*S=}9_B<_IY&ZAUi9Kgr59~i7uxjswKlOX17JS%y
zXlCnv4vm9*uQ@aCo!?%tZ`G*{R*6$q?Co2cw|9E&!@VgcEB1D7JGpOj@h_Xa{mlpX
zX4yNKRN3#9dXQ=_x7EhR<HDSM9xVBL9vxk}ckR~`dp`<&+N+S!?69UsY~N{{i#8uq
zBy6`n|G3xPW%=Gq`=kye|2(#LQCyS5FY(5`6_+RNowjBD-hVUr_w{^OvG>~&afj7>
zO8a>mT=#Ff_1B)4apvBjg46p$`2F^>u4CADE|2BFH@@b*2B#<O6?rvfUs9y=-u9-1
zy>$g9dzRG-?vq;4xYuLJ1nVDF5BJ{RqQ9qa#dZ4w3WEC>ie}sVSTe`XPW<DZ_jAwh
zXVq)o+wZVoZ<`F`zS>}){kN-~Y}#-6@15)^w0ENa*S-5WO!iJbwrg+sRUccgYZkkA
zn$6tDGS_E!QPIi+vTjxOsRrxzTJ#3m&rMg^J9p0QeU>8IY`EN+_ObnXv1clG=WfHD
zTn?U_ZT4DC7PP&|5oD*cm3LpwYFpbYCaec^F6`a=+r7sw>I&;V(K!$I7Oz;ox8}(_
zo1=>B_NGaPJAD7l>9A+l-2Ly~bRL)`G|8sb?3?{;5%oRu8*lC1q|jr}f8*5Nz{U%E
z*!-gH%HDGC>pSvmua(`uT^n4}_uii@zW3UMCwmSrU9wkx&)U6}*&+64Uwzv>n}69}
zhWJefrqAuzt9bpq-8!*#dq2$ly|>C!#CG$_p#4v$h*&8Vw(kv)d%N$*?q(ZiuK9bb
zH~imow&SJMYC+vSU;KXVJF9hi&+DKf```Z4_O}^tweDNB-ag(%*@jytX`fJ_{oWbH
z+pWVK+4kK$b82tKKMVWDKNI#!*xcT0=s0!XhOaF9t}Qj+lOtEQpKTk*-n+A7?3(+|
z?~9IW-BZiGVXvRmn!SImHtvl+Zf;*DwA#LL$;ACC?rZj6e0bev@)R-q8ynZ}UEnll
z@0KU4_l4d2yQi0R!rruCvwe}98}=%he%bTSuhqIRO>$rL;bVK5uD9);eX7RhlHtz1
zifT*uACQT(vH#Drr*6?_yZt>KHa03;_O+Rk`<B0v-}~we`<`#N@9xXIp0jsTU-w>y
znBDtkp1insT@$mVQP#q}GQHe3Ou<|C1{luSSDhlZCoYxgKtZ(J{wZC%_HO^JYR~5<
zZ#yTqW1r9rmA#_N+-yEyp1N0pVZvVbw_W?JqvZEWRT|so24CB=hEHHmo0R(Ah%<8!
zOu6{RB2u_>zrv5KeVd%`>=9{=*t_J^^Sy0u-}lZjd1g1qc-n#ck1y;`xpVhG?!s+*
z+^^rTUBBY)UVcT!eG}67_BnEV+PiT&^FH>UD{T62_3UwHoV;g`+s!?O#=UzDSWoWR
zrYpX0>B*COUu~SSck8B&``M)S?43S^&Dv-2N85!aFZbkbYTxhqX69bCt!cYcuWz&2
z_3HROpQe|4)%cIwtgtcMXKIwWw@t@vZ@uD=y;HBS@0H}+w0GUL+xvol+wZykXXSoL
z*ZupKINjLG;F4`$@%q4?#&1RY{AC#SMVFNCojh}e%_Elny{Zly4*b$wx%ZMn@7`5~
zJNBJt{<e3+<l{EC%Kz-IWVpP?=JgzVW0$r)fjr-ACYmhS^CMhuuW)$9UPYI^_TJvV
z_fI(Hb0F$elfBKF>w9;#Ot6n;y}S3m+J(JlyW$RL+r8Q=RJUyJ6#XUpWN&uvoqL6G
z@7#Zk`|`Cm?oHF%z1L{>_PyPwb*!TEXIQhXKe;bwTlQXq6Ib?DtKPN$JhgStTfr%|
zGtS=J8{RNuuW)bH-UJJq0}sAmwAmK^ZLjv)%lqo3&+e7U*RwrfaAB{j(y_g_y%cPh
z-kQDdosPiXRV@?kbE9<*oVg;u=f3*2{W1P4_B{7lyKnKE`PLRM-|Q{;^=7a5iJrZV
zvY7|enJo5Yvo6_p@wV80>8M`Y=i9d17;^RO+slz}vu4|T`__MXdoR76ws+q@p*>2^
zYW5yJCb=i&&FcN`ced|;RXxu>=<3M>B{TQ#xo}Ox-fh#2z0S8b@9my{)4uoXY8%n4
zzP+u!`}f_QRBkPBonzk_;o!Y_frt0%nVhwGB$~gs<Jpfrd!B9CS88i+|J8PmP2&=$
zy%Wsx4roTa*_-tF-o8)YJNEAIT(dXt=Zw8!tA5+NS$^5eTc*4B;bf`(Di4_UZP;$P
zuZDHs-h<sN`*hc|?%iLxa39|uuDug2=h_PwuD1U!xqPo`d#`<|%bdNxdbaEf4q)7C
zn!I%H1(tPtQcj22)LjkQ|NQQ!y^VkO?f%-he;+#s>)vl|>U)2;i5y7ycVut!o%j0!
zrZw)HsB~>FPsp6TFE4ZM+hN4NM?3G&{yuZb{j90>_AcCu_too4?2S^HWq(P_(<W8l
z)5_X|VSo3N_j_9$TW!KJ4(;>OklMp(Exz|tc+uYP+>v{vZ?o--7V6*o%#M4nS}x~4
zshW2C!Y|8qbqO5bd%jJ?rXWLb-*b!1{la_;_8!o>v&U47bzj+$=lk1SPVeoWXur4e
zn&AGW4IO)*`2_Dtd0eoUyQszHOvcVVTR*Jbw^i<wO;sC*gU&^{0~b^l*_`;^u-EEo
z_+Ha(oA>Q{&Acye`p!Lm%lGfi2(;aOZMug&+mvbh?tk^)C$#SSzK<-Mtoh&G*}G3S
z#lA8_dr$vQVf#<_C-39gIL$`Mly{%M%%8n)b*Amfd3tYer^T&(5<=7V{w_7N-~D=z
z&C_-I4qMuo_r5z4VjUvdWH<9w!Cv*)%X?0R*x9{rd$jk}Cysrui@WyD=at;2a(UO@
zKcOr3whR8ZSs<jccm1bn_SO^D?@d^}Z?A-V^Z|F%V|y!lrrLen%CXOS<+{C+rCfWh
z6xQxHvwgp(-0#3%H<wrYG6k6SET1X7$1r!cP5k$ky&n|1_a@Eyv9Ek%_ug_pHV1uY
z9(%z()_Z4|Y96?sGIj6i=Uew3J~zqQa+2|$%!N<(_PZS3>#;AuAz+XE&MUW$?3J5+
zYaiRedwavWckZ3|kICU&iqO7Kf`R*$A~)_+bI95o{d~*bw6H##TT^)VO>7EuuoZ|n
z;1~YHu3uaJz|GSWZSJNt?7t|QzgIMH?w;Gz5)Vu~Dz@iM|E;~s?gF+&>ksU$-n@8E
z#LGu}BfnhV`y#|_ua`^Co}J3FdmK#fTi2*|?_YF^-DclX)7_^Rp0KHw?%Vr$VW(Y7
zt@ECVtLN@n@yB=X6VWI8%d>j+Y<7OSCn0v#zAV{id(C<$>}7hqbZ_d)4}15P#O+PJ
z*tf6n!(khz-xKzoQPi`SPCc?$#c2Kh&X1z|7CIO2dl}7T+gbE%uflerecQLM-8(m<
z=)gk1^)^TEdf3#=_1~v&A7wk?kImk|^G<e0@4wv3@$~h6-SVw_<IWx1yC9Bx?=I1)
zd$)g2+_QsWp>6-S&-PWRt@b)L<p;iPZ?rM`62HIqNSlrP8Hc^`7x&oz_-12$R6D^&
ze{aXWr8SH9{$u>UXXTy9J**Qe_VAfs*z57>gvFneMK;OS|MxO_-`+hjDRS@pTW|M1
zJs`FJ#%7(p<^O;0`!XSJuanIBJ>F}@_uV_kesIycb$jo(Y_SP$=iG0js=G@-?9QH3
zj1z4F3TN(}x3s`!>&4Ccrt@sxV`C$AAijT&{k*P2d-7H}+N6G+u-9^9{=RR{9Q%xl
z`u6@VJhAs{;@7>V3Nm)Oj5>SOlr;A~4V|>l=)>c^-t&ZPOZQH;-@wOgrD4f!Z}Y2X
zZ}FL3d)032-FyE2-Mz;yuG(wd+Pk0av&sGk7nbg?<7GRL+qryitLDQ4ytDoHwzJIH
zYdy=+?or6|z25sD+XU`ywXH7f+3Tg-wWqd@dtd31KYO=3Rqp+0)4cb4ce1s|ABKI`
zrg9#re3ZR+Q7QAjS!)FMUriU-8+=W9|91w4y@rp)_HJ2pX78DYwf3{Vv+kQ#xnu7%
zlSTUk%x>)Mnx$m3Tv2qd{E16@Uw&P_mpO0cKKVE?+s0HChw`oe?JmySzqdG}=)jNg
z{d>|`7w$W!Rk`<`e&OC1HH>?uqZs$O`)eGy&pmtZg!+`dxi^LOaUO2on>1fz&(eiy
z_HXYk-s`wSz(MaH%f9k;xA$glII#OT>l~X~)0gahGj*fAljJ7*`4!*nOL9-_n=a_Q
zch)bv1C9o6wiC=<_BK58a&T^4yEpi_@}47er|ml{bH(OmILF?V;yruo%bfSxf81+h
z5WatJ&(D9m3tlJeG4I^4|9R&uo4wno?meuyX#bDzZ2Kz8WA^V;-MIJX#>;yzxvblB
zh|TlB&w>*+29kn%r+U`xi#x!#>!k6ey%jYx_Rh5Bwe9DW*cV#Hu)pETf<1~;-rM^(
zMcE%@Jh!)S>3Tbz%dC699dg^pm^IVJ*>dk*HK8MWrz|hpyS-7#;n-FlTkdJCd#B{z
z+sD26+g=v=oqLw*WjkCyz_d@m*JZyA+thulWuEQ5Y_(wTNgnmRqANGs1in0I*AzO(
z*6aBq+vlg&?|XIY?cV*9a_nETx9@dWc6oRFjKg+XK|l81w9nhS@U+POdj5HP7W3WS
zYm~^auRNP^-xZD>d*8=C+cj%v{vN-hyY_aNzu33ibgK33a6cR4bJ_M!YS{OkIrG&{
z{))fNZ)@Q_3uY|aJH2$#-YXU-_AY7@-uLyyhke%VaeIH}pV|A^Zime_Hr3s_1;%?F
z18(nQ{hPN(*nGzR_Y!LM#uec<=Xc+>t+iOPd%?F&`{ceF?tZ@8#YX4j_ucu;F*c4L
zo%egZ{;~Jf^Ok)l7tY$(k@tF!OzHET!TS65uRhPdZ<pUhJBC^B_wqEX+#O`pxYz0A
zQp+xpMSI`YS~&3ARPOgyFS0MSn0dg7;i^@Tp{ITDsi%8aZa%xW=eoguGf}m@*UBI5
z*)VtZzIPjTTV3GbvXRi8xaV8S)V&vdX6%(X%C~RMoYi~7m)Y!FGKp!wo7T6z=eSJw
zDg>3<RsQANS7mEt&+}Jtubk52z0d6T?3p~Z!S?x4k$v?&^Y@<JD7oK9;E#1~fZ)DJ
z=KXsw%(}eyl#RkZqkX;m&Xqjd6F<$zK1tm4z%GNid%LUW?^oWIw%6qKt9@^rbobh7
z$?v_NCS%J!qj~RPmjwrAPMfz+b<&J|&ns8%+w8v0YU%|g+qO5&`=+d%w8yf)$u9lT
z*}X@W?%i8{s%77{O;`4Oyx*{AL+TaV#<gGe-~4sXo=27OVDGw9dp>nY90>Z8wfBtM
z#l4HlC+>@Xzj5#3GgtS{>YuxhHNRsoUoG#xmoa?%>b9NVdtQ6O-qXD=_h!Cr-kT;g
zV;?8qA=}$hH~0QeT(vhqEM=d|(a(E%g(VJn@A<o@kI#4a-X8b8i>^;P@Wu4`-gTiI
z`&f)G?A>6!)P_OUf8TRHjlF`)PVeb?)4cc3jpluoCpPU3`|EO`r0V7V<0rEA{$~{1
zfAn<M?roNf_X!^OzNfv5VPCWVo4v<O681cZo4W7t{n>ll+&<gnNl)9`vb=q7xcNhy
z5>|Kn`_u32t-dCCKsSidR=H2mR&%xBKBhp%eQm$vZJKAv+S@iiu;=h`*~^#Vx}RHd
z#~ugX*>=t%HG3_eI`4JyjdoaJlD0=<sj{t;;njWJ7dGs^8S{Pb8NYpd_srwpm&>cY
zXY1xwd*dc3fZ8Q{e`{s!duKMurtr|Lz2A3-99R^>x^Kk~*?lUxU3<$UBW>nBIcg~v
zSLTpZc60B^1LAwnuV1_Gn8mfdffLkivW`97n_j77bNY3}9wU*}`?5mr><L<abw8J6
z*S@cdj_#ePW^aEp;j)dup0s_xEl%&f<-KlifkyXU@fADwCYSUdSQ@ivPa22)zQmiK
zty8sk?b5mVc<-56+xPb*&9OOd*|I-j4*x!`kM4UoHhr@BGg)9C+rR$3E9Lnd{MO&u
z&mrcqfAc&EdsUu|d#_sw95|RPwohUP&%PC`#s@C;vg~~`p=+=C!N2=-CwlK?aC@_N
zc?{RSb?dG7?sM(mdnRa_jZ)u@y@$Re?Yp~<;lP@3p*{ajz1tiAa*jQx+sVCW96s3l
zCHPx=8(!XP`t90Y#!FcTw)}a%_o?6hy&D4F?rSSPv*(4S>fY;~0ece@FYUQB>+s&b
zGYj^8aelCO)A2I<Ma}8^SI5uZdoXgtf!Oexy=!i*-S;lxvGpsJm3yBCEZbX<ethrA
z=NIg^t=HRI*%o2*mu=y`>fqMBIrDb!dG9A>zh}){n;Qpa?oFRFZ{NmTUfa~26?=Jl
zU+*$_&1_RAwbAZ{UZ4HbyfFL!yQL0V%yHSPQc`Z8|8v`(HQ%P~oqUhiK2-G3Ubf?c
zdtSON-1q0J-rnT0n!W8tZ2RtIPu!dS`PtrVNuhmzK5yK6Nhx}-Q{m!$8Y>v~UAcbL
zI$F16|ALpd_DcDi?=RZ@VsEwey1i_!jQbQP&$HbzwR`W0-OT&02m0^(zU<}R{tBUe
zIW25^S@uuaGmHD@-nHKj?bA;AzBk+2#9mFBYhPl~Pn+llsr}Jyi}o7y9o=W^e0*>4
zG3I?gLX-BK|F&xH;#SoIsy}n~rf)9a+aSnmmuGl!FLTj_y+^Ki9(W}`V^8L41^a-z
z`)rn-zP)$F>A8EiPh;F`)6Z`cdXa74!;-E8a(Qd*eXh6fuY0<2H;>KJ{W~H~>{VPk
z({}#D>HD-!{N0-vGT-`Do6SDwqpz(Zw&(7ZzF%T<^?Bp&Rpq&RXKr}5cWv;3J=b~W
z?p3J&y#L}y?tK~R<$EVcE!)p}EoQHD<h%pF{L5`@BX92w`!#iM`Rrx;L>~+8E1V~=
zk8P{hzCS1b?cF1P+$J=hVNZ$euRWjHjQ46;G3}QsKEL;&#I*g(nhosv*17Gixj)yQ
zO}KT>w7ir19*Ah}UFg=hH_UJHUgqu%dpXPQ+3jdE-Mjvg@xF&K)Auc(F>UYSl(TzR
z_=p_XX}Dx>;GE+9a^AD{MlEC6S21hxUbc-t_L@jv+nXyCvwu^>JG=SSJO{M@GC8=P
z__-&cKEu9vSMT1>Qx5Fm`^RSQCGm7Gx6tGro@EubYvxbhvnGmTUqyY<-g};0d)JDu
z-s@+;w2vXz+t%;m-@PW04-WAEXR*22Dz>ka&wAf>N%p-RPFL;yem>dVE4p{LeyRIj
zx!4Q#v6sB}#^rPFQ?lmYUwENl?|X$Cdm{8q_m(;4+pIm!vF}R9)_tp)1#B;_i#agy
z@F)8i)<rh?oBQ_lD{a~1)!n(TyQJCXbjcT+k5fPG<=fqB)3t-we!HXLUagiBdz;SK
z*=>(|zNfW>bFbH$Y4&U9e&2iRZub7_S?Bgn-kZ6%<DsLC)hyP1Ul;z}!&TM3e_iTJ
zyQ^$J_9cC)+rN(I&EANW><1<+c)quNmFM1Orxtt7TQ~L|*?PsM@LlIV_badWy0|Ue
zv!U_!-h3gmy~=l4_pZ~n+{<;dbFcZyse8BU1RvOI(`55lTWFu1it7HHDa)+QOhWhP
z%{{w!s!{Xa?#`>bkIPTD|1_m%@8;Q?_sn!XwePs$&b@4hHTQlHUA$LY^}6)}F)JJA
z&FA*%9tqt$F*4eovHa4$&A-=JwMB;>5YYFsvE6ZS-@Y5m_C}~N@BPm6bgz(a)80U_
zdG;=w@9bf8I=0t{bNk*2M>zK>D6H9|B6!b!Q=`-#+vr*LSu8dCN_Lm+&AI5c`?+=C
z-T<!XJq@*52l~E>JFL6K<?y&j@?hqNuX`({zVBPoDQpw(W3+ck-^_ihr!ehvn{#b1
z%Z?@cI0NVGI%t2-y4sD`Mp4||`u~nYd+x3b*&FNpZttDF?t5nxHrmf;O4*y!Id$(p
z=KK3i-S_W(ueaas%Gy_Zqvd+`raJH3oBCz)0YAg4y~o&Y@AVX_+ZQ&&Vt3c4S9{~u
zGwc)O?cRHvbHd)e=a%nVy4QVgkI2FOArt1>r&-kQTC29ozW2e*y`{NP``%Bvuvf%?
z!CuMAAA9beV%hgKb+`Q~7pFaP?S1?FYq|HbZn3i|VBy~DcJH};nf$c9&m9!@Ka1PF
zFI7lt&&FT-_qMF--peK7xA)YGUb_>Qp6%QAOJJW$)WiL~ZL{}w-*4Q<9@cI>k40p!
z*?WckGd}FzTgT^PJD;)5=0oJ<y)(Ru_d193?9-Qdzt{Tm_q~@i#r75Pv)G;xJ!;!<
z^qu|o>$j}^V^#Nb*F^3=F@<;U^50YUsRkC<92Iund;b2Gy*Y=~_b+&mZNu^P=iWUB
zHthRyXzrdo4Uau%I-cyEDf(+q)9JR|dHfsq_1ewZ8(Jo^U;G`<0k*ERy{jZ*_i4PW
z-)pIVY+psH)Lx^E6MNrZ%d}oExnfVz+J}2(=5X&zb2i@V)a$>u?P34kISeoNHk978
zeYQ1VulHJmeec4T?$ubbXV0=_k~WUZzgb1y*k!%1bHRQ)+kkz`=Set7axArv;XAPR
zqGgbMw8r_p=>q$9*IwRaf8~+LzBVQHy}8_MHY?6<-}C6wFKfF6B73);X4$)a@{zs0
zXT$ei6&Ku>@$J~&lD4w_?5Y3u_AyPjp6pxXpzv|q-lIpg?0<Ru*^?Y5w092=gYCfy
z{RiCL=IlMXOmA=8%BFo&HP-KaVdZ1nZnk9a?u)6m3sMsIZpqobk2{okuM11kft_Y&
z_p3K??+bdr`#^Nl%e^Xwn)`ej{@cvp-Lp4ut@@rt;e~sjTPW_|F_mTC&WVb9r|#Ig
zuUXMyuhnV2y}J~+_RDM*+N-s`d(XZ%>-H-AZr$s}vuW=}!znf$3+nb5N<Xw;6(Qu1
zp?+=O?q|gZv=uV<W<;3UO^e&M_uy2qz1Q|KIFy~7XZ^Bsx{Y7OynS!@TK1fO{cDeV
z4DUWc2Jw9^auRzR)MxEowMf_IRae%Y9~asW9E_j1m*KI{zM#to_X}%q><eLJ-2cV=
z=-%r;c=q+BwC*i0EIJ^oTebU=Qvco>`B(eqx#aITy;*Iq_u|04x|yf<zTL^cPsV8D
zzTXR4_Y|B|cKDm3wqN$u(LK8VPuqEa>)Eq1uzw#f)6u<a0#)|h;_2R#$8~JaZF#Bv
z_kyqPdAj%CUgd4Z`<mzQ?Q@-5yhr`Z?ETJD|L=8b*0O)*wq@VUqnvvjSrTmI3fcF?
z{0^~M^WOP@$W8{k*}K=+wJAs(sNJx0@2a~s2Ug`fSXao|Tkq}IxPQa??Ry<nl=qcz
z&fNF#yyV`f?5}%s95(LVD>`v+#P=n8r<YdlO<uZk&yM74dqbavJ3PLWx;Nm)NgJd7
zo%>4;pW73YWN{$eUv}r#InjG89eMUKN&nva)Z^jaki~cRKKoF;Z`btudzq&w+TPpx
zV6Rd2nZ2&-rS{I2IkN9QWA|QN2@|^&%_jDTo=@9rpUHY)u7IWOVUC6SoID%$UbuB^
zug}(wy>6Phd#gG*55zF{?q#$}xBZbCviJSp(7lT$J=^nx(eyyd2abJK?cet+6l~dN
zu%~2i%Cl*EjW~PvzMRCd*SA5-!6Epj-R^@72ULn#4=5jI*k`t)d;g<i&%OKfkL~nd
zIOBkK#_c`l&A9e8KWyLYz3bNQ_oXlP{uJ81_cY7gy({j{-Yew%e9z_Lj=i7K-|rRf
zZQILR%Cm3n%w2n){7K)p{tWXzkNB;&A@Q1fSDIej^Z&%lz2ONi_G`=?_B!5Puvd8U
zlzkQ=W_!hsP279>>6*O{uCCnMQn7uno95$v>zNz(PBnkLPw<Mi{VPsxTjyWT_b+r@
zu=m2^3H$2Su<rYnK7B8<+YK8h*H?R%=$_u6x@Fnk``_m6eRu2GK9&gPeUS~DZT4Si
zI&dcQ^xmSQbN2bLgzdY!lxZK!X9wHI+cWn}w!OG_mG@iw5azx6w)PyhllrS{&#;Dn
zUqGM1{);u=_OKok+-J-5V?Wo;BYQs{She@v&pG=-UYXj=+@!U4(a-X|fy>QpX8v&A
zGw0gLz2AiV_ikM_bMN$T0rqERZP`8JCEMQ2jCuCU9xmIfdW-wu+SM2LuI8C%b27l#
zreH?kzL>Y#wx1s7SUcUCyO00kguU68C41c~IrsIvD&1ROP`773)3SZ3>rJd5`t{j&
zb-UTW`R2AqaKguZKkq8+4OzHoUwOc)J$(;2_X;n+u=`WR>)lFhr}iKDwQ}znkJ^1t
zS8d&w_iN4Gpu`({?>JT*od0OIwP9)P!8_OP?ad9y+WU>UX|KcfXL~Zwe%Qmdx8uP5
zoBQ_{+*Uted)U(c%h8nGp5F8JTg12Sj`CT*+hFMwTkHGx_ts42-Dlt4zOP5pY!64^
z`n?63`ZhtTjQh6rY}jkb$G*==QfeP7@0Y#zd9NOP>SVe1?6PTlceVuXlZ!dNH_?gZ
zAnV?DJ5yDT@BMJ$_uh+_H`<9C8|>ARk=c8+W8=Q-hj&=tHu}4_uIuPtlk=CXOKk=A
zsw|woFFuWJZ|kG@ePw&U?q~YGX-~L=ro)N(v-YyePuLf8`~03$OXF=e6wlb}sAsol
zqIj3x%B$u!cV>Lt6E`Vs??1Dzdwg%b+N)B}WdHG)z^-(6d%K%2*!G3bDB0_6@3W`<
zE9>4J4>#<+GwbkPkKlFo=l@){`=B9m(6DCy-jyrG?N>f)*|Rcq`raG7j1Cu+BlaeK
z*W4TYeAm8D46k=H-K)0Yb?>oR<a1%qskFwu9~y4#_4{44XLaC&y??izvOlEUx3`N~
zVxP>{wRYXlU+mo^y~a*S<K^Cer@!xA|H5gnfymbbYhQ2P>!f>j@2vYu`&Axp-1D=3
z`QC|7KkQv4X0rEEw$R=fuEu>AuAJZV<68a!`6mkx$j7|h`*}~1z0387d-wFO*{62*
z(Oy$-;XMjQxqGkGT;0Q;(z$P?UD&=Q7Ik}_U8VOcE@s(VK3QVV=?mxVP1~;R{i`(V
zz&G|u`+|zt_PX@`-2JM0-`*0gUYoC+2kaC4cG<tavv7a2!LkGYozLz)Xx(ccn99HR
zzOdY0=Qm62%hylY%l_SOZ{h3x`-DaB>{W04vs*H7sSUfC_&(d)%=_~681~hE^xl2o
zZu8#v8Bzz@QUvxbG&0)DQ$6Xx=I0mo{@U~0UR7_KjrLkjn>8MH_ikcea)9mW#l1}*
z*X)%FW!_)@ymXIdQr}+2C($+!_S)_Z{=a7L+YKl7Jw18PhOx_fKM$ju-O^P{_C8&3
zeecE9PxiV$c(ZTik{x^gt9R_3zgo$LC2X!uH^apJSLYquJ1>dN=Ht%u`&`*`_nIG7
zvfU)be8Aww#l6aM>Ib6KHt$OiKDsyQnbuy78!z_moiWRX(K*Rprfbdq9eXC*2fo;~
ze`$xX%`29M{cHzKZ2ojE*_$*gX8#|FCwtTyzwTyj+O;p!vSRQ1Rj>E@&2rqGWZ1R0
z-Mn$HOU9+WfeO#|HY@kpTuJM*pAmCvZ;kR|n>m}x?e+9i_DV`i9N6Q+yYHprlfAqL
z823&th_lyucy&*3iPGM!E9dTWw>@HQmDscQPIa$M_OXL|K2E>2ms@+mz9V9{_c$GX
zb-;L?@&T#+v-VgjsM&w0`n>zf<Zt_w{$AXh8~k$bR_1Nio1GZ<No`+Z?{Vptwd%#Y
zd#^_v-gjil#=Xi8eS3m@ciUu7VB8n>eBu7LA5QPx@IY>F@t6L+8SS&Jr=)D%yZor-
z0X7LIhrLQt`@-)>*<YxS-1|^@&i;@Bo4wBtaqUyGSaV?ajaAkPdq3^na-rB(etzX%
znU42+qsy81onOtrPi)H0J)F89_9|)d?mM}{Y44F6ItNa!Si9GA(gExJ${h#Tn(psO
zzt?i0=I+8h#hbqFWz^%@XWC`BKW#UYZL>(v-Wf)Ud#h^J?e%5)yr-z={N4@g753VS
zXYSoLb=JOi`-yws#>_b|Z<omdYgK{0Pld18mrJqin=yCKJ}Emc>x0S5_pt3`+#9<1
z>t59&Hv2!V2ls9dK4UX2_s+f!JK=rZoDcVgTR812nBl&c(>HN{ufU>xXL@vOmc*^x
zyF+J&^_|_E`=(TTI!HZ}afrBW;LvOtuy1YIg1uj#->@(2ma=uJKe*Sipu{1D({Ate
zx5m3KMV{W*v?SPCHvWjsoJsR-?j1gEedy85y|!^H_R2(Y*(#U4*{eH4>Hy;$bsMHV
z{d*TZ2{^zoUvcjvo@j^fv)}CAcQa?NwE9Aug_&vxr|s?Av(fz8Ue_?*{f>%OdoTR`
zz31a;>AeM;9BdbDFWUQT>Y;tPE@6A^1m5fu+Y@%cgK6vD9jnvrMK^KnRhV{apZkjm
zdk<7x-uq;2lg-)8FzXpN<Q%T?u<e_7_U0ZH+19<O-s|^jUR2zd+|+CT&Fa@)oy9u#
zH*ZYa$KF$I!?xUZuhqkcd!2cg?UlZmX0N)R+o4x8X1|tlul@S1zxMhcm}P%P^xoc6
zjU6`Y?(#e6D@*M?m?mmle&2Y1{etX0J1S*t_PrO}XC?V^@2pU+eZK4Y_Em)c-Fs$3
z%AO-?$p@^3Ht*fe_kZt%Pn+!zZBDY`E}CL5ubjQN_~4E`b9O%1Tk%}M-mLoG-h~o%
z)_R9e?`!V4x5rZU>mDQX1vYgX+V}j+>9>6@y?UQu+~mEJG8ft}zT>vfF-O`)VQKjO
zybrZ|W$SP4d;3CS-{zo;d#j}w_O_l5*!#RuY=3G)#cs!sR(s9WqxUV_w#p{J_>)cB
z|5^6ezlQI%@iE(9`D^;VSrv2k#<TG5-OB%A@B4tidySeo9ct}04xD@XXkX;|^ZQ+b
zH|*Wwl5=3?*ByJgdZKJ{53b)A8YjEYZ~cS4N$E589kY<Loj?8a-Yj93eP37i?PZCY
zxl44>&%KWy`Rrv;*|T@O1H*pjxTDr99j@<PV()GLHal~-+12UxQu+mZi#BxZc|Q63
z-no8f_P(m=-#ekKeeb;2`}h8eiLp&mytc<Tj(H!`iaUFk%)4qGSv`B7@39SgmG(^B
zfAH+P{qtJ)@12w<y<b&VX73;S?fc#cY}oVg9@pMEQRY@33ps6`zMpg;R%z#6^KCjd
zEmGBcm)%O(TeCuEPv^lTTi)}&d;jz;we#I%y;u6>yS-wre{4QJk=@(8d&%Az$#(Yg
zy4L%(#me`)OuV#z$~%X>M!w7k#Qt>bU3;ix@AYZ!4zp^O?%l@uVy~G?%f3}ku6tgu
z*t~a&$Mn4n-gkG4*j?Wv?fqo0clPDIQ=<a+it5a<&)t}^m+e%Lb!ch3z5cJ`d)G*H
z9ys#UVsA9(Y#YabxqFoYSPlqppSyRzmBb#sTxVNvWk;)&nI^Ue(`5GAOnkCeEyQMz
zLgLDOOIzCZ%F6z<*PhdLp#0g9U9tO>9KLpb+{>`y@;;fJsy1_THt$W|`h4$;jqmnK
z?Rjp`@hfqk-nU<SA5S{8@7RY#o7o9$HU>sV>{ma2xW{pdpF>ib^`62d@AmF`b#(9L
z;`Y6TANluA(D`6Lb??M|95IIuF!4tpeErwha;D+zy)T|U-MjMIq`hZkf7=B(6z#2?
z|7A~7!-IV+ObmN9w)^h+wmsD5o@U+NEv1uftTuA*J2rc}P0G}-dp$CX_Z{-wx;Ov)
zuf6lyitW#tAKp9bkG%c!M}m9vZ#~`XDI&Rd-EYYQjn8EEm26eE<uMZ9zp_qx-@};C
zHuY9odllZY?Tgg!v|VgId*AV#_j_Y5upOw|HsOG%)}FnV*=Ow@2Or*3E_!aCseseo
zq~=3=@83SO*Zh{t-aSSW_D1?W+Plf+p!Jd_ru{OXzwSAzuD_S<vi|<V2Vd?rDd69~
z^l|jQeF_KnR<)eld&`P-PkM0RUgyhl2LcMd+5Cz1+`qm-X8(!joA!LW<9=Xv$g{nC
zDxCXF&Eg$ieYmw(N@~d-{uf8~Ek7Q!_Xx+!y}NI}*?UiP{a)*Z6KxV@*!MQB;MkX$
zV!L~ZfbRbLnum9*e$v_9Uh8GI?Cj3HA2)Y6Fzmm*mnAQKZzWHh%@<_}`@=h)?p-F@
zyjRYjY5zF}3!69cXZB8*(rcY%e15Oe#>;#8S;F?cwVA)yr&!xzmTSh|(B#5B*5|kF
zyY%P6-iZDi`!tzF_qG`C*yCl|v6r)C{oY9{Yxe)HFW<NRt=3+_(^dQQQUmwQe7I;&
zvXZd<lX<Os6Z6U)ex)4Sm#LP#cZt@Cz3EH$?vZ#u+tz>IJp1IB*$y{OEU^Ew()vJw
ztj?a+8`cgjr#|d0YS7$UH2=e1nZ=y@?u(`D4OnqzpZ=d0J7Sc0_sJdP+{Y=ndhhkf
zroAtZo!TpAJkct*X!GtBJsta6B{uFYHt5}3aPa8<H3gsc+MV9F@7swO>(Zs1`&K+V
zwMU|<`hdtIiG5bTeptPJerw;}9+rK_62taN8D6(}u4J}H%;v?Oy<9W)1x2LWn8obe
zUr^Gp{}=nOz4sS09mqNByVuY8!@fhWFYWPV++y9wE50|n<LaKgrpkRjWz7511e^C}
z1b^RGkbla?+M!_2X3uZ?zuuc>qhOxnaPQlSeamZ~@2y+Lxwlq*%bv@>W%m9&=;@GC
zlz(83L6-ePj-mrgu5I7T`RI>*v1s_-wJ+lKRxozi%{|1vFWHaJHmSnb?&5L9y{j4=
z_FnV+yZ4;>)4kskZtmG5&$q8AzIU%t{ieMe)4BJHKbx|*eZ#W79XnUqx_z0q_p)r(
zfu)zZ_tn|$*sGHCZO@xTQ-=oE-+S2-uUS=v?B2I{@<p3tT4j4>)P-$2?=0SvDaN`d
z?1cTkr|*sS$~<thSM=sSpt^L?Ud`!k4!0|^_x=;yvQJ%9e{b8-^t}`O<M&wKShDxg
zCAI?_q%Z9?4wc&5CMUKpjcd;C#xoXst#>Wizw48j&G|j|5BT3cytixFoxSNtuJ3hY
zY1v!JBCyY}J;6RC&%+_g_xyf?t2_sNE4J^Q!0l#tN_4%Ap6$WCXU}mv$eSDQ`B^8r
zr_OouzKp8hdz%h3?7be7zW2g|mwQ8^p6uDxzj?2I3+q1B@R@sAPcOAwB*nb%!QsBW
zpMPrF^KX2y_hZKTeSF&+_Gb16SSil`vRfzF{NSqvNA_Aa2JG=Uti1n+>fybr0bKjm
zB(v=GTf?;P^^059r<AYlyRfs(dV?(E0W}Rr`$Z2P?Do7k$6oUr#~%Cim-{3i2<%&T
za?{?Ka-n-uULD!1`Izs(rv*p$u3ejEy<oT6{%Jwq_D)Pu*%R+N*S_KRt-Y>$3-;w-
z>fRT)q-U>r-v=9`WXHX~?W$~UM_sp1ESzKCl^}CqrFZdx+q)I_8a$UjC|hW-*OC9%
z-iM!@4rH!t*!$ZwWpC?toBcfV_t{kKI<qJE+>X7^?%vyLQE0!1sn~RHR_EHi59%cL
zn0B5&z_aT6-mQi!_nO4BIqW}jb+2yTeS7=7GkZN>a9c-NZrxid<!LW5|JUAIHzw_^
z;nCe!uK8y7XEndQinjdwtbMs{gU>SVyU2cV-{<WaHt#rv9JasuY2U57Z*P29n*BL*
z1Dj^u-TStgE!lHr)sa2f4ZCf054_o1ro?mL`3bgtc?LZDg4RmzSL)ik_sX)xd$+Qz
zvHQs4xOdB8T?dBcclRZA)$Td^(8`u|>ax8SMGX5EpO@Nib?Lgj&hvWvEjrT=Jdv{A
zE7Br#;EGh|UJ;`;HdoWn@4qvPao?9;U3-{vO!furJ+SA+mV0}O&aT+29?!V%j>XQs
z$_v@|g)%Ge%jmeixAO8KJ4Mk~ds(uS_WG;{x0}CjuGLQY4f{6jow#Spj%U^%=Ckc}
zQ9iMM^6tBPPigS(JG9DmUw_K=y^D7&-^)4a{+|Ce%l39ITfF<u%f<Wdvv=>kr@h}k
zY57@upPvu+O1JdcC)Cc{`|<4DeO|wR+I;<gYp>OV_C4RCPgp<spk=EmE4%l{OXa<t
z-*)YbOm4H$F<rd3&m+-ZyJy92(Lf{notY>0iA$>5EU#U&cjHUtz4p6K>~Yxi$^HiW
zD*O3nbNB10O|pMyon&Kk!NcL2<1(vLJh%2v7hAOdp$Fq`f15I!995frO5qH9MU)u!
zK1!HvBQgEXo`m)z*6E$+_WV5QvX{X+Wbe{g6MIM9Q+oy0Jl~@&vebUp>YIDB`u^|t
zvEI1XQ}g^DFZ<<tCn(OhFEgLMXWx{lJ)2f8+xIkUqRojKu6?tnPTTvtd#-i<oBqA4
z{TJ_Jd!4;k?*r$7zU!d}{;uoaW09<E-+p?>p3kk@_g&Jtu$L*Hb6@Mko;|K_AMJg!
zO3+@nG-FQ!zsBBxRo(m2mUr&myzHAzMJVrqz20Uvb8dI-<*hwnGrQ~E?n&O0_g;G6
zZWA%#*xt)8yZ38dcxYd}Tz0>X+r$HNjxDsgy6?n+stxD&JPXjX35kxdzt<?S@3{%1
zt)bzGeOGQX+rHUZyvMSPf1jiJ`n|V}R#@#X{klizrruuZtsMJmzBk!V`}1LMw>IlO
zaW)Ny#y$7<a-KA^PhUB0uXoh%JqcSi_wG}hVgEej?%vq<-}Y{r!?AywLi(QbNgwxa
zHh;HQ)L?~8Q3LnB<KkQQ_57K+XRDgTf$wv}4ps<e?0xn0*namwp1ol;Klbfx+rMWI
zNBLd{@0EL-F3j31E@!kq`hD%b$B$O*RbXGT&%wB2FH6-mn}{hk`@gPg*u8i9FT0(O
z#rM1^li$msp0;mqzokw4q5FG-E=;vgtyq1a`1&0CZFf%{oU7u$SNHye184sF?Ok5I
zeDCki%l2!Xe7W~Z>a{&t7eDS3eD-{=9rvca@k0H3H$89NYw5buCTM4!&B8<HcGm|V
z-*;W-uKkrs7xqeM&9W-p{9ym$JKy)lwC>)2IQYTd&1$xLm1g$sm1z(-(0T8b&HsJt
z_j1lVy=&{<i#DouoA)}TKie~D*RehA|J(N-3uxb`De_?Ny0xkYW=!fkaESH#?%7cZ
z`;?p(>^6OMVBhnfllH3CU)($2<;ULdXIu76k6p80#I$YiC7;f{{F0~l<r=Kn8?U!{
zFQ@e_`?)4Id*{Ec-Cwe<WnY1$q^;w@=X*_SmfI+saqd0f@?xKtC$EFgbms$c%N`u)
zuFu$Am0oV2XO+Cy<Z-9<8?Qt5wd@l6=I_X|5ji|-pMzKQp7(zq?>TkMYOm6^`n}0N
zS@-VO5!iSC>5;vs*Na&nSfsuG`?~JE;)}oRIrC)efjKF`dzr+v9FkV??dzEMV^9CS
zi+h9HRva+;@yB}JRfS#unpyT=)r#CJdb)qlH@jmt3-0&qz3;-k&&%u9zK#<rdk=OS
z9dP|^b>Oiim+hMA1^Wf^eQkYR5AK^5$F}#hCCi?5z1(~6MlId_)MWMk-z6Juwxt={
z9%0zFFKC;K&5ngSwhXrB`(x}o_wvmZI#8<6Xydl)z730dx%F;_LhEC{?e~7H*LO(a
zlRU6worVo)Ett5~`n@SKn+~kY+PEuWHOoHTil_D^llk|R{&3zi<xTg#RbTGz^;|Y%
zuj;3ld;Q}0_i^g7?pxN}VsoYY+8&29SNBa@!(_W{6Ysv={_M7^_FuGfynS--hKAGo
z6C&>Hl?yq(*Cdx`A7|*JJ-0Q*_v%~?-}7+S>b=KjJl>O8dDG@u_=&w&`xorp75ZUs
zYu3?ymXAN%ywSUAXEm4UfU9_<%^9C<`^vr@-@E^7^FAY+qP<s^e%osocy{ma1H9I@
zj-2~TtEcYGTKsSC8iC9EY^Iu9PiQaSdnw7%e$8%yy%Umc51c4vv^90pvh6FX+-r16
zU|*D?+1}kAJNDJHP2T5THpza~^3(P|*u3{LPFiShYJ6_5U2yl_h^CYK6HOTQ?V5Ph
zhEsakz5s>Zy^ndA_O0}~y7yP}^u6M@c=r8>k=!Tl*04w3WP$ahd&}+f8>;ulg#FyR
zs4v~_>Bok>f-4vg<dmG*o3(V|-cn8Fy>APD?w>e+;-1HQ1FUO8_U`L8`?ojn`KsMq
z0<ZT3oISr+_FJp1@~MOS=4d?M8<QZgpHY3eee=%;d;gx_u)q12jm<T#XZsF4;og_Q
z$-7r1M}IGa=>}`})vgCDukh}B`+SRyVFma8MaJLu3T$cHE6Ff-Kii^>d-riR?U$Xg
zW8WlE!@YaoJ=nAKr_^4n8!da2?gs6viu!Qin11JeO_l|A%hRXZ%(0qwVD-|edsmCH
z>}}rj{(wV2=RU{x8hbV7sqPDTqqryd4D&vNl*qj?FIe_9A6#J*zE^i|<}{zZg`aQl
z-L5m)e#!oTJ$&-3_9o1lZeR0j!JfPt?*sb(R`0zz`O#kU*n+*rUk}@tEO}zR^SRdE
zvjHpi`KlhU3ADSmH=EUJ@9U#?_FlaDYj4?xr~4k?P__MebkY9RXYcOkOl9Bu)nli9
z8Xv<R-@=}K9c8EXdT+PcD|T3DpI*#A%d<~c+AjMgu`e~0dtc;qt$h&!b$bg$4(#<3
z_-gORA+mR;8mE2U`z`zARhw+)Y3$$2x$eQ<Qtq95Wf<4nl?NWR&#LP<;E^wVVCOCM
zy`^1_4mTg~*z2>pW=~ph>;chT&-VQO`DpL;MYn9DKgI98C;fk~dY1TJgJsY6o(<#N
zXEkHO-tB4ydvENJ+1s(U|G?#6{QFkOUD<0h<+MGU^OL=P-=Z8OZ|&Y|>dn9J=L(U1
z<yV*;x;!NJI>|8Ydsd^k@1tXo&5DxZy>A}P*gN;n-@OIXSoY27J+#kHeCyr>p`{0I
zO`g3^ea0f2U*8+-l~?oZJJI@LpIeahUVhPOduDWt?3<Csxv%fk-~CHc&hOc^bLU<e
zdEUK``ZD*N@AKWu8-KxeMbzWHRxYjk5AjFtt+@Gr5BvIkduP7ivFGOoo_$%L58AJ2
z(y?FBb?U&>Ej$PE|9{<8pu6F~#>XssXX<_3<2P;T{w;;y_i%FU-)r`_ZlCdj&wDd1
z!uPIi;@+3>O?IFEjEQ^I1Eu%mmrCzlvijZLFL`nY_j4ZID=;Z)Z~YG;hk45_Z4!cB
z*qc{m?~(f`viHpWyuB}brr2*;wrQ`ApZVTLuln}=jXAf+elpL#slF_GOKR5d_1J!J
zui@T>`>roe-ZO{wkUjI|J@)+S(zbVHJrCRo{k=E-;k12Hd)M#%SburX)cwkP+rIYL
zR50$}yEVUcPu03}dsU5{_ATS*-1nt%-CkB@IR{S-v%Q=B8tpr#r0=_`F==nrWW&9A
zg(voQ2-fedxU^=!Gs9&2r$@N#-wK~TuuH6dFJIE@{TtOD?TxOr+S@+q`F^zt5BG}y
zQ`+;`X8pdHjJ~}eSZ!@yr*iJAIQ?+1v9|Kw$ZLQ0Sl7ht&7R}7XL;g98;-5UdxRbt
z?0s~3hfQ|{!@leC8}{Y(O6*hEz`W<w@#Qw(K5w=Ec53Nf5$7L!&Tw4ad!&wKU#Qfs
zy-XH9d;UZ=?)_T$VQ*FJv3=QsjJxI5GT0fdJ7V|s=kL9TuC?0<Z!_Qfe%h9OX?|b#
ze$Qmwx6StbUN*r=dp{k%zmGqhY2S}&?fX=9*V~>>|F`FM!^^!}1DOt-40vs=92d90
zmO;+8qj29|t<<}F+5}GQJ+PzQru5<p`#)jL_HL(Y?JA2M_xru_+nW`vY2RxdvRBz|
z^WOP}d+oov@a<a>@p!L5>G6F|A0O_?ys~-kvR$%!Cw+Rd`}015ebZdnZT?*B+B1Fe
zZkydz-20yW>as4J{9>=|W_|nV#}Dr1$iKQzE$_Vb?I+XrCeAowwb|p_-d2wPdn>uw
z_W3hU-ly=+c<=q%&wH)Km+Uo<5ZG(-{Dbv+@iqGzlZ9=Mudv>K;pff$uTu8!wYpqp
zFI~2Lw^)IlU9<t`zWnPJwk1<F_k_;k+si$3(>}F_?Rytp)3wTYDzIO&b;sVT5;yjK
zTo-@9Q<QJ-1-pIw!vwbO)2e*2cg{b-y}YLzY?63FY$Q)e+l9o<+5hjmz5UBrqXR)b
zWqT{tU+kaTGi7g!;Hte-CK>K67e2CASm46m4ICHtX}C1(ElO$K^U`C_9#atpn>j{5
z_6DgJ?fqfRvahKyX7BztTkVUh-S#qGp0)R)e#C*akcE4vWzIT~!78-p!9C->vs&Bs
zPCs^ke_{2RJ+~Q!_Hyt#?R#vubFb@}?!8wOr|-Gs<6vw1n0cS__i6hC*qLp6U#so^
z`qJ)z?X{hI?>}5?FMR5qO|a7PeX5ff_bg+Zx%cvq4|{c4c=z#c*|wh}c<J6}EDZZz
zN($SW$T958alf$V)S;!e1~rfNhJ1K`z_DxL-Xy#Jy&bu)_g+&_-CMIPaj)ZjX@^2?
zWBWH(!|i3V8x91=)!5uSX}a%F@2R~LB)qM|UQDrH5~Q(D&wsH^bnDuEl0j$oTx0Ls
zYr?a{Mn#8xk0AfFy$!r9yL)?9?U}Q9#oh`3?(P@69=hjB=B~ZZKASsq7Paj;GdIY7
zT`2oLoy7}n?)xp=tFFa;Ag)wx-;~pRdu-MI>{Ab0zI&JY$-V0sIQH&eX|k)tLTR6c
z;Q4(#pa1NsVOeIEvd6*xO&pu;<RuLD6QiB?>fJuDZ+7<T-9OwX*jSjD?nyfBy*o$6
z(mwY5o83&+xAsPqJ=(V?PGH}A5e^$~*G+cIZ)n^6Y}>atnB(rgkZBwC{@N;I)3xBy
z-jMITdv&hz99W;rd7x$EqWzuS8V7vO?cK|8SmnT*hZcJsn`&+Jr>)xe<&*PX%|hk9
ztEx8b`>3{hkHbseea|kR-81{~w!KGwP1qCuv~92AuEl%y{tn*Dd|Ti^_HjRJ;mTus
zc`jtxe|wX^=R#TBKBKM7`^?*T_x)?SYr}Owa=-c9b9>EgF73VLnz%0}gK3|~gG^fn
zEFf)Xs^yZ<E_0C7QvrT;5RDB_o>XFmO?=ArWhNl;K+EeEAQ~NOpXj$}s7bJNte$Kh
zlFPFLB;Q;XwjD%+aLdHIJ3%zD_;l+_uo)orhfYL-_3}Gk1G^t22EtEnRe;4oY(hB2
z$<1<oA-^R^-P-cKaCd?D$XFwC4_5zs7xjYeer>w}YzDF(5PN;jt{p))K3abH{M!8E
z$ys1`o&V(rw<jhjau-M)vH0YUaJc`QP9%Z#F6Y|;7Xz8Y)0Ybt2eAoZ>3p`GGIcUw
zx3evg0lN>Io_8y`vHJh`O<u5gzr`yQH^9t@5M$eYJD<VQC;pY?qGH2sAn~ikMPU0t
z?2LckaM}ryN5?OI-GryJnK!nA{VA2_4L1*@ezigh(Pk#7cv}>=%Yya$EqspE4<Iw=
zCSC!n0kM(stDUpK@t^-2)eSH;w}ocx__A-G1+%%r);XChU^6~#hu8;F6D8CH_Ya6q
z2)8?Xf&C0pcWajbnC&OD3&q_B&Mzd||0b71Et_w1fz`iLW(LzBv(Yh&m=HLg(Z$#N
zg5>)cwbNj~q03)cr?BIT&Tk9uiPDxQ|33ozb=sCzFbz_>;CLfWJ3;d3SpN54xcZ4{
zg<y7&{}(U~Q`_9k01g)zA0K^l!b!`^Pd9+w!*_xeOk-2eY}AF-|KU&V!C|l4xB#pF
z^D0-H-#gQ5Imvc`Me<!MaK6vcLX=_5x8;a-E680h#MXiB1F=;~uYvu&WU>m1-Jwp=
zM4MUte9`v*SJs04t>>)-HV<SrI@VQv0#4`X;+r!=!0z$)-VF8|y8P=Vshvmjo6Wm?
zvn}0sS%dAF(d`B|tK_^DSU<WxT>MRwd*Jr$*4+TsyL>gI{D$dSmD~arhw<^z%Gw;3
zes+pr^-Fv<!rg_gXLFV+N*KfBb@)ZLg4lN5OjaNo9Y5XcyW@0lx~1IpE{p!^TCo3j
zU7G=>L255Mci{|kkUTn`bUy&DUS<y`*xft_D#0{N?TavFqT@d%M&D|Iq8-@n7IqwP
zvteez=p!k<SpCns7gA@QP`C&86HE_?Zaj8lhs$9_%TIb2E$m$a!0t?~Q8fd}fv~lO
zBUXJNIc#`Sl@r`=JwJYe&C@YlkK#6;J%MmF*vvo{v)&<QS-5S&W{}vDx$|-Q-<w+>
z97Z5@$k^8571%GQ(w<`V$9DE(790g3maANqET<Xig45TTfBax~gUqoGlEY~yNFE(;
z&QXH<O@H23usL-LI8n@CyutxjgRUQ)zq(S~(*KMz*l%rqVJPMxtIY^ng4KVaHjws+
z;pctea6r}rVmrs)*zt2#ujRKiE=xUoNV)HEQ3h_-aR)!JA3$n}!6FS=aP_{6)WPnJ
zU^D~MF#Z2F<)ZLm;@Ic|U7}`jbt>TSYOui_|M7vG;4nfr<LsP?;I!wJD*~p`<)2+%
zxKo2IZ-?M*T`Px*Wnh2n@HD{v;28LvXt#pQPTi9QSO5Jc6WHDIRr}#$AT?*^zlWER
zAU-}Ex?avwX_YqEZ?)1=DCQvR%ip#F?q+0hMh{3@-Y3osHUn7>#Ez2H*#5m;!^+WH
z*W$q4t6={Zb54W1?~|@GR&zmmvEja&D!3aZs#?J2+3q((F@xWs4Xy^88OUP$<EB_H
z_|y-MyO<JN6myW(6=t)8)gX(3*cReb!EVUDwFRrZhP%Yh;|BvR>^NRoFj{tk^T-M%
zF|ZzxepfFIqTLEo@3(3XTz#g#2bk?1qK#s=#Rng-I7mGq{Q1)Qt&;;)!2TAgkU%j9
zS#R-{HCW|7gq;AFX?jA{Smmuv-<v7zPPV+%8DT!}yD?a;nP>*w|3*$VV7rm+!pCmx
zn+*5c!WY6|y@ikD;bI`Y&wJTW_#iP{*nG|=%Qtrfz;1VaunlY;HnrvJd%<eZ<(f|`
zfb}TulEW&$#`2>1<Pvtv<`2%>?;p(t`+L!Xo8WK&nPtu^Otf1;>Lpt~!PT29-Ui$K
zeBog*4Uz}p0~b!f)qwc;u=5e09p-{t!TP7Y+>TQpuR=an{~d^hl;1`Rzhad)^x9;x
z^4=NCkXxbVTMq94=lhL1{BZvpM%v=E6PvrAEms4Vc_6v?XA8mZo^7-koUcG)Ae<B+
z2Nnmh3E`FUb1V}swSm=#{n`zVA8dNo`KDnl-}Y?01lIfK+cB(qljd7lJha-nW8;cW
z%U34cV86Z$ngRF!+r7eY|6sEhU2L1ycd!{Caeq6A-rJj-;O2qEvwB5{HuK7zSr$){
z!@zO3_TNEp{D90r$Nf{W$FJ_T`Cz+Mr6z&RK-X7#NPP#(c4bSawHGbI{<DDHbx_6#
z9A+T3DHo=}{R83?!cLvMU^77Krs)WP^==D5?f*=@T>usbsUd`~o|U$AaBl|3ZO|h(
zxY_9Xv@b+q4I6pOwP3pr{XLG={~pIXcE0Wvw=!M-)?(>Dep8USuNjQsX~m=V3p^}9
za)j`bm+o-Ao=-W!?hQJ47%m3VU%b5!>?RPK5Pq|Fk>!kfcd-90*nZ;F_s8Z9R{tG{
z*#HigIqHkC`a^t?!>$KCmX`A8%B{ZMbpo3aDb5HU2LidRnHAE80kMg}WtNcq22y)Z
z8`91{@!SN(-Sgf~N8y9i;=)hQnp&C3-vNi;8>3@5^<^|I1-l*HoXPSF!2S<T$;K*g
z;d5$xqEWDgTj@s&v)PbwAFCUX{wByQRrU+u`~zYWgD3l52FE!_?X8XpVD}z#fV2ZZ
zVj%2as0~*G;^V_gdd8NE|IG%+|4!GpVDqr4KfQf5IE>KcE*+Ew>k*hJidEhv+h*q*
z<#(1>_G?%^&SnCKk=%pBa6hvZt^tQ1y1BUc6OTl~(-}WUG1zWCyLl*fH|*U37RO}<
zviNiBDhnxPNdHmqffG)Be7e$Lw<GHXv1hg10=s{D;Rkq{28qM)6tzP}nwiUX=zo>8
zJW=QXcKb<5NSXnu(GJ{+(@u~)IzBrGG5%M1=Ly)nwi&4P-I2&SM4P#&Wsl{|4c%Zr
zs2u7Amrv++g7_IPn!srU#74$E5|Usyc#C_W!~sl=jg*Y#iA_b85A0`H?7qtcc30kS
z#CSDN>;teEOg$lbM)NYb`cwDg!0fla&rrht<VWK3e{)TmW$O1pu-_Ecn&ZqjYh+Gf
z^&i{39I*e7xy(jsgTVY@!W3&Bs##?@$K$z$_@BLCGprH~;pyM*u`HYo(}S1(z8vgl
zkeUqH9x&VN*-4bN$tpqon9?iX8cT)MF<||3p5P9*)hr2MH6VMDahpy$IPJK6$6o*O
z%@nr$X7$r@#)KQ@C)>Qi{{Mf{8mtFo&T?@zupdBdVsL_j9XyVvHU0pbbNovQN*Hdu
zB?XQXV$8e!d7%a8v%O%yO+0%Zr(0rUo`BV$o0-B_1$M*!cyqAd(B&)ZgDhTb%-nI{
z-$l!9N|1Q*QCtsCD?7P#;r>C_i;Z9WvlXuXoyQumdw<NGi{k&cbH(6lu$h4@<~}K7
zXWNokusIWWZQ$-gRs&*3R{DVR2#Af0PcU2o=e>6;?qCfc71kXVInwb~kD0z$?0sMb
zcGtS5m0&#}bD}@}2ipx|6N5MW7lx~k-_Q$YUx@TY@qfdNXs|dj=BY{aT6zdS1nU<P
zc!ab3o9&Oi|7UQY8?0CP`G2hOKS8u}2XhFQWrxpU3pdFNTR?8&tI>n|U(L*sXt#p&
z#!vqQPum*r!og`zS1JP~>=PC&BHBz%^K}*t3%7yw-{#p5HydOZGS2+@6s!hW48%U8
z?+5nxtEZo^%D;43zWtAzq~+5c^4mESK7!Tmu2q7o&9c!U+N~h_9{KHttKT?Z5S*Sq
zT{;671F2b4WDFJuu?gXvd}*s6+znv&MCb~E!xNjHhSx?|(|;j{3^*?Rq#lCJK-YV>
zpv<zIUBaqnH@oFNR>&B^wB$T+ngOY`{);+x2NJ`F*NFMR{pRdz3HJBBNL{#jAidTT
zh#&v3@qM{t%3W5lABz7ufa3>b20A|8{SmAG1f8CP%Z$fn53%aqa{a1#7t6J6E~?io
zvQ|RoJ&L6fb2wqExWRU#+lPz4L^%QMXOR4y{nB8))6BTQG)N4Dy=RKU)qwc;a6|WV
z3!k5mcDZxKH>`dD>3^5kjWz!D9`u3r_~~E78a@$8uXZ>~|F_uN(QUcuy9Xqm_<O;6
zK;~Y_K~1|LF?=|`iVGh9x0~*P^Xtyw&2aNTdO4Jt;c7s9eE7}{yB*wp+rj!}nQ-TS
zH5&=6{tq^i1dD&#dlGBjNx8_rTk~a+rSYCj%kQGPU^NLPkoE@1U3U^m8~?s~VHqS0
z85kG}&v$|S`SRBSu=_#gfpBBRT%zOOepRI9^0UjqX8PYg0uE1*8R)n%a30q9HJH8v
zY<G+06s+<ETMBmwy7XK4GkREVvK0oaeXThOOoQxr`+qG?J3;d3_(UEz*v}yGYsJ}M
zfA(Bm38q0}AiThK3JM=2h6^kG&9JDLSpjx~;HxGujZJN@C?uZI<$e|Lg8d!r^&d>5
z%Zq0l?yP>i%)BP9)zZ}x(nr*7-3j;q-mNEzb}Pv28`rMD)xSIs@ppPRY9Hc8EFz3R
zdhlWIZx^?Hd9xhs_A@;uaCf2Wxhk&#PUq<29vcO~X{Vk20oV+5`OrYSopxHwcC;N<
zwQO@11gHN636S&#QhTZNA<i%d$)n?x^DglC-_8W-3+LIQ&ds&&$K8)dH}7)(DT~)h
zkg?rfi;pPfD6-nD=veSPH?kOry;>MD@5Q5}jMX38kLvHtnf-CwItDvSp=CYb@O`!5
z6TF?JdWIbypCI!H;T1)T!C?qeCtgtw)@%5r3nlFNrY*zi50E}&Tp-_K$?68l+X5kw
zaUf)Q7~4}P9>orrc<!pN;QH^rzCG6T!58#s$9|?p^V`yKJ3h{g0EbbGgc{tefO6vN
zYLJ~Pa&CkD3}TnFtpb~8Qja=sKcn$JTn$JaK76^w%qq|*3hef6Ce=9gnRYe7-HdGB
z_KRxZ^v|~UJUDKV<v{G1^Z&O0EEl!Bzs%Qi>B(%cdmG+P1JfY+IG;MK`ap8n@a+H2
z@bs^1vL5W-4cGa=?uV&=yu}x;1}27!4lT~IsJg5V_M2)qYB(b6nYI=C_@B3>IygN>
zZ1BSBk7aee7KiQUn(tU=Xvt;82KM)Dj<w)412Uguw=&odAT}|$v;?x|3#4|w&?c~X
zkG4a`pg>|EtZ)W0E(~H5!mn$1EtIra!Qq$j2(=zX)+^A4n6pO~cRSArcK^Z6zrpDM
zSq{YhCwq8D!IyH&RgA)x&%5lvVe~@U8D5q+gclQ?S3%~P1~S9#sr#G+wp+6KCtM7q
zW}74NWeNvRmgSQR+re=-?IZ4S^UeeJz-a_zJ~DoBay3{yBasbEBg=u<uTM1WWT}+g
zDW#%n<+KUXzE06f2HOXc?+=$DI<11#ADj^a_A`h*V+CZKH?TGp#qL=*I^b$R>hNLS
z__>yucV>X~XMODhrzdRcWj-;(-Ha^H{~0p>d387T`Jq0;q#f<;f>x!RN|r7vtHAMp
z#Y6}mU+Q;H;ItFj9vJ)obI6!4h##*r59}sgu10u%0*OC&sU+I}N46ifJi7K6*#D(m
zL16cR%s|KU6(IE-y7(4zCa|8A@=IVd(B)0cm3DBQceS+0YP2lP)CK$X!UB1CnqOL_
zLbO{!W~-*B!PO^pK<bWz)nzDl%daGU{KHBk)_ke~WNp`)dyw_1AbZg9*Uz;mcEIFs
zhC|l<iCp*#4hNVxjQ+UtxA|6U`R(V~zgr~9p9A}UeZ~ZEyn)m-ZzQe!+j9Q~*bI=~
zou?DQ?Dhka@Gt?1AK38%EDmB5!oR0FTb7+`1D8i0@&DoOLf1Ecaxqr_w?}J&)3TR?
z7gl+_6`DI+_dVKv>xhJ9RGk($jBI0A;qGJJdJ&!$(Cx&=uXvjPFE3b`A?p?9YpjQx
z2hy|R=sA?|0EywkB`yyw>is3a>Z9NAz|BTi&yyt%t|QRJMLu|f?LJu%2d2^G^+V2B
z>`Pm?^OKgp#p$AZ;IJycW(N23oVloTSLpiD`R6Yn*2`~PD+CTt=|WGG{6C4~778ET
z3}n9d+g}z2MVes0Wq6ii^#e$K&co+e<F`0J4XkIA{T8rWKx&ck!#Qhqgeo1f_`!6|
z(&_Y8a5zucX$sZ@QYVzQ0xSk%6N8tjq`>3IViROMdG+1|6uaBBiC_P`PI9NEi7BN0
zG`0FQ&TxC@hkbto^ILVWU(%KDVhx`xkNFl!uahk={g`R-H4PF*mIa7@=Ia-FB>DfI
z;Y+xBL1)O^<J9vvQ2ejrya((iV!~59ecetrw|V9uwaqQ6@UR8(k+Jr|8(94}^EYJ8
zShv_Vtn%v`*Y4n!5;dFLf5lR)svYdEH$i9N<;>HzyYRe#Y#%PRzltQ>9+5qe`hWi$
z)VawuO1RhYqnjJ%&t>VE2HC@LNCeUjKvx6eFOfL~w->~RVU_dOz~YR&sB6$*@~1Z)
z-nny^@XmUsjaE$IE?~9R_l3dX4N~(pk+k*i9uHT8!w{r*&Vwc3_;P6p0q0+k7zjUe
z{QyrRAU;0aZRuz=rC$#0o?M+goch*Hxd3)Mx>>ba<={Ls<KAqn@|PC`?_Bjh$-=}j
z+4AkIbZ|Pgzl9iQv~x4SX(zf}Abt?<5wM>@Z2ppUV7+`77JzAx7zk^J5I_FMvwHdV
z?`@Fs@ADzV+8U5K$k_TKWRC!{7>HdIdlwuZn|2=r`wb+9j29;!-rnT1$#Tz=O3VF0
zki82nPnf{<ElAx`qik^4gV@C2jAL8j;jEDfS^xA*E(<OOGGk`RXRw<<Y(iMvu*AYV
z2eP+G^HL5<I3nu}=<q?Y114v1#TRUs-Yrh7{(mNTa!2t&LvyZmGM1$rYrtw9YZ~Bw
zJ~+7@r=2i6km-|Ctl{c!=|bi!4R#5F&4sD!*SrT;0~5nVzfBI`dFByh|JPH$30VCA
z(tB;OI!Yb`iGgsgwg}iS9*1MWG)N2?n;C4dcp~h){lld`OO}@{U^O$Zcfs9T{==AP
zw}Q;IJb>uiau&0J%hC4VTT$F?H}ekMe2^Y|*xZq6$LB+kvAdT)j=}R5x}I<1khv=i
z@duFg=(p9TfZc*FU!r)-ywTU&^63>v3&RI|V85=sJQH5#-&v1+-2}QhFuq{@CAeQ1
zts!gA-Y#K4iJK~0+~YVfb6_-Q@f8cPtsG$Y1baZ%Q^Dkr=@<Q7aC?!()#m;J&uPlc
zgoFdK9EiOsyTRh};iKj;CuJ?ZN<qp!(Tx+pdO-50q#%2EKx|_0jcbVd;@oCeu$wfj
z1;I2-f3Wu@tZ@O8N2aypwwmiFNQ1*qJ`}OG09g%)y;HvfYyQs_hU|X`sJ@NW&W8)k
zcD{60+VRlvxMlf)zhE^7>z%+f$lUK!4}#MKh)oPWJ2x1fMp-7ZgW3M!LMZ9|`X^1W
zI5Fn&O#5d(*%{LQXW#J~oDQ(*Gq_<04kL6q`^ptyzp(W(VwI2Hy3gY9^L3WGO+PKR
zc6o!{`7`eYy!?;Zi&)Q$ZZ<Z4(A+Ax-)u@QfX%tra}&kgYC@mEZo;M?UF^m5OFNy`
z&j-hCpQRE?Izm?4(9H=}gDeJOAJgOq>v7OF!79(>a%IQD_bS_u&beVRw;f{s-sdS`
zJs|x7tQ_Dl0kMg}E<#V?>7{etA#i!xUVs=+1)1@@Clsy*#K(tyruJBdZ><2kXGwu8
z*gS0N7nokg>OYQFNS(oz_!+CbxXhXzx;p|as%`IBy!)mKwuf~mWE~L5p54a+iFPYU
zy>$p=%ooId%Tff^o4BC?OoPNg_^vAMek@2H87u9swR~Qj3D!SXKL^~FLzaiJ&uJ=S
zP2XA%{)5xtMpX^0@%VVIr)3tmgXJ{FBbJ8GyTIXlbkhxZ`n)aE3AP(%4<Y*Yb{ly5
zf3y7q*qozlKElgWkbd2th&3W0K0ciLcfyWyCuPBYQ-41Lr(3ktA!{Vj&0-6L>^*e*
zU<>vex_odiqnYj}G0XIA!j@Z?hJwS$?`s}BoD-~2_pYJqN9Q}=tB1Q$qGdJM9A_!i
zetTvT?r|J+^Qx_W@7yZA8|?r7PItK3=<3+cuz=N|i#OXr)*~%^c@%5<50_fG)3}M-
z@}0?U%QI6U`<|+IoQH>%=DkPYI7BxW7eB2WvHt5|&RVcJ_xmrQxVz<F3<@8YSs?N7
zufbbCb-V@VOR)o!!RCR)(XnL(cK>~R{s-*u?!%C|D0F?rbARm$SZ!p<UsG!7Jh>n2
z*Fp^gupW@wQ1x3xyA`B<OJ@MstswTzX2@9h;+?*5F_8G#SH$oCYv<<O$<{Upte?pZ
zvYr-X20C6b;T_z~$a3qFi@|!UgU@1>kE@8@X&p3Un;O$r%RQb?!Sxj5;{{+EWR42c
zMsS`0v5CR69cAJ9|K8-i;ILD9a~e#;^ow{R_K3m6aM9N%Fj>hYLiT^WTqTNP4ziwN
zyGL04R~_6B_J6&`cdYV?F$;H|PSf4l-nQLh+gnH<vX=J+SP#hjiI)}N{sHj`;h!cE
za5p|QhwL4gA=!fxhFzAZ;epRg>n%GhF3Bwf=k2|#dEjQF>-p%l4y*r6s~}@gMUL3Z
z$4gq#JL|F+TgZp5-yZFE7M#Cg8uq~b&ya|^cNN`Ubbj^K3Gnb*E~*N4ul)~1od{CD
z(|#{Xc!0!kVb0xKEaJY)f!!mYjk_<#XC{x;%xOCzby=%k1K137dn6iNw>Rq5?06uN
zZ5id$46dg-nOlrt>UISp_5#7g@X>*LI>2QJNKM;}X0ZPSg;3Z3zcXyb>Q;~*bbRS4
zuhpL^kToo;IeD@A0i;Lhh7i{H7e96sY}Y#$$QUX}Ei#tUE!tUDzs|yBiq6hCagcK+
z*4jzI&6@fS_m~s1IWYFU2Z;SrI!ci9WEu_ygTn!)=2jf;b=)vDFnVc*pyja_hr#aY
zoq(8IfXTz?3L(@zRxt60y~^M)+nJ0y1`U&MFz(p)^>vS>?b<NQATP+ex7TXB!Syys
z&D~1Wvr9l?_^{P#FSuW){e!HX-uxN0ZWcB(g69!@W`X2pCavARd-@S@{*S(od;R}A
zZbY30(g(xSMbyFRaf^jI+)pq$5S{OOV29DMZI<7IXISo#f~566J_c}`5hTB29%KzS
zh)oPGN9v<TeuT7zWGo?bFfjdRPFoP|f4?h*JG-){g5&?3$#HO8!puRYBOgHaIU|dK
z*k&`m!DX6w&{?c{P1^36r7oChvGsj}<#IVeu-eTr-QaQnq@S$|F&6^j6T<JpAZZAs
zE_tyD*q?EKQ2XX9g^8b2d~?v>GGRC5+>g}1v*BqAWIi%pz*PwjBV;iU`^S06+OaEP
z*vE%f%l_DLTza8-Rcf=P>cIrC{{!C5f~OVLf*`OTKxPqx1tk&Vg{!?F=|6-MweCF<
zP5k_Cx$sVl%zG=q?ke8D38(*;6{5~Dz}%|LvILy|HyxS|_Y+JGME_grVqT$`V7X50
zlSS4<f3Q8<W2L})K=RrxQ^DmYh)oPGo7oOeXO?du<KFp8q`~IG^v{uy#_11qFmvZS
zi};rF;PXGWiOR#>g|3dRUl42jznlg+Ge}|ge7L)i^<MpHZILeBZ~5fuQH!qJ*<kbA
zE=+~HSLysjqTLEImt_ZJy$y(M>;T#4{nc;UCJ-NnuT?<W1u#B7x;JN+dFyo*aQVlZ
zg4kP(u1Dfz8&?1Gb~1q7|He=mtGvdS@|}DCP2b6?9%4B``8>GYd%PGjW(KljmKtg6
zKe>(<!qbbq(seNVR=ojA*vkn#28)BtCxmr(-Ltf5f$UvcDhBDFVAFHz?@X-z%ko(Y
zPWSVp=3te-H0$`zxYJ4&zUq%HF709gyQ}llX}Ej8`#l1O2{t>iitXxwmlwqy`rxql
zTj>v`Vd`(DL*fL+$49ejcUhh~x(e*}`^#{z8>oNL3Qi;FX1%-sS^x4i-V*L7WHq0x
z+P0s)W3_8>{(8$BtQKH*uDgm@E45ScI5<p@_26UY=OWhs1zv9g=cmg(3&3#z)Avr%
z9=^U5CWedVxw&!YcjYT!zn!;w0(KuZwW*ER_xD~Fg{*1(?T5X7*!cJV&eZ~~JM3lj
ztP*0lz<yOue*mUI_V`9R;ItDYkB$%T*aVONp5HgYdMDgGiW2q(f{=MJbiLU4nLH*t
zG(`)*`c-<;!RBF8XP-X{tOi}qsSdJ!Jt`CX`p|73dMt{bR9W(h?6df^To|0r&Og+H
zmwy>LsAspJn~%=Fe+#nb86?i#s{nTI-n)=>bs#YiUY=qH4_^=;A69)7W%2(n1K4lP
z{n+CWq(A4^F>o3Iv5~R5I^^7|Oa9$pzah(k*vF*ucdneoyK8o5o29fJqz*B(Ld=zO
zt$R<jTS4Z1Tz(epXApZK(+_Z3n8%H}_iRd%I9ME{h7f-I)4<&9>k+X3Ga7i|W~1x#
z75j<Re{cUl%EhKaJ*@JFFE801H|O<^W1G^gLizTC<9*Vat?>3=R%8g#ZUwnXSQgR#
z>5o?14H65oLY;eXev2q0K=SzT_7Wzm?^~{d<3QX5at<Ok^`=YJu*UzHr=8%iJC(Kv
ztN;6Ce(c;OyvM9;f|Qjbr!-jYe+|T3`Pqqx@eFMCqKhSoPlSh8y_FxBef&A<d0$*{
zh%sVx{pkGF-zV?fel-&8cJ)KJ>*6!}mw?rv>)&#S860-?0@&M!JW=<}Cw=U-$j^4O
z{P(;A9RCs=AK~To*-gK}WjMOIxcJ|FTH)?^`F0K1|FN==bzm?(*(~#Lrel~I7@hZ{
z-(pwxMQ|P5`W3OZ9wrZ?Q-Auy?S+Yf=-l&$VE-TahW#AFqO&Wu*Cf2RXmvPZd24St
z*bHT(3E*%5=`WbL609G@CI;XBbPBFMSs#)YE>A}-?_?$?67B!U?uebInYV${wv_>5
zZWY}v-SykSZbuitd-ovNJ=gUhYjM%#U)*}TqapFrj!E^;Ewfp~!0r;e;|kUTQmf5F
z+WDVfQxM~Y8;$0J&0%S21m`P|c_6%+l?7}Mh)oEai>K{);sjaSKecKHI6Yz0Q~d{f
z`)<$b<>2(YIolX(_>>*HZZ7%J({jF4vjx|KEU-I8xI@7-$R1BMdpH}!Cxq2=FN6II
zQa2+Ivi>VJ4|V*5wQC#E{x{kA+hWoI$a<`c9BnxLACmR~tN)5u`-Ah&*F*`pyFhM(
z;XS5HEX><>?tHgG(DK_X$o$Isd1dhQpFGnTt3H^1bUKC^F-FtYlngfKxIigN7^Z%K
z?6X1Fi;W*~^pfS1bja9U?P5Nh{{K017FPf7`Og6sm(;ldb_=>4RWdtwe2nq1Jb%^9
zszdAySnbP3#5~Ka5GkB?qMHTctE)lIPy(^zmqYeK`zxJ6@&B>a>O`BV=Ok`1s{>Mo
z={I9v;|j8C#stV7DG(bOt8ae`j=z&{H=(!zrp9rFkfjpy!X1~VYgzQVd<LsYyKo#H
zRvTXVgZ0DI6QXZOc*6Y_C#nq2d$%rcfY&D={U`4D5N+l~#%q>Z$6UbW-@IcMVDms`
zpyT4Hh&_wQa#@Lxx%Ov@kiFf=av*lfBlewVFY#K;3;w)AZ|?!H|2NP03Z_Bw&#q&y
zH$ZaOu<CZi-VR1X$h>EmEMy)WrarQHD@u64#Iez{O{VV1Q9J=omml{`#aV9TJ3!V*
zp_`$m0qM(fKHvw3A-eqX?wvbxzCPbEX`!ZNhXmxTpTH+-aQ}C6hU2so-7FCQ)ha~W
z?pk0nSTFlq$UX>=JP4oDx(XHtu?gW5bE<bt@%09~U1J4eZVg@E->2C3cc@Gd1*e&o
zOvo8}82VmLu<%!Vxnsp6MXR1B$oYEd3lZo4%(WuEtpYOJT@SHF>9N{xa5_(~T!9kb
zV#gkV#X;%`VXKOC%Z8<OVD$%$yussO*z`zU@&bnuy4<y}DPX-}Pd32wG_o3jNxc@1
z_i`=X&75pDON#*<R{vQL=bvu+09g}{tOp-Ee=*{$!w*7`^FNl|M3@KCTUxxC==hK8
zDzdcvkO@wYtY?mc-3KxQ9lz#YiPisd?7?8YEvDGl$G`c|wDVrtIm<-pT62l|58!wb
zpMh9Qv@e<Xy^0_=Wp^RY_NaVQ2rdiHG@_n&;4&T3E&!<~gr7(%nfFcl2adZxKKVGq
z?W!SUjU2jp%+^1^Wyk3b$X*n5dCfew-8s{CnMu1&vvi7F4xaZ2aLEPh0jcfcCw{IL
zq~3$?GuW*lcJ`XJVE?c94mnp2BnHBZGNW+%10;`(pH?*Ncx>GQR{xH#6=(eGI?jc=
z8Cm~V0m%OUm*T8g<sB;}cBWpbvixt@Yst~H2<&!S707uLAafF#h##v5sjm^x0=pH&
z&fWeW%>KZFx`+I$BV=6!NDU#puCZ#Xzz!vF9==(HY7Vkq*G-T;vdCf}c8Ja=aQfFX
z^~36aHKs`x@8nY~7biZpyuS<5M!LNOah5R44@emfGK&~2lZ9B@5&q;3*#DN*sB2Rf
zKE}O{8{LjW!4oad&WG%KTX17C*e~d6K>YKEA!mev*vOdM{{c8ok6c`c)&ENt9=7=U
zcZTH?23hk{dY<6)^|MD09Nr*vtX?yM(=dol3|_DWv9`<38q#0me2+Tc5n2P87bC_z
zlT+N5CUMnZ_cYJJz5c=b4rGiF-OR0bEy4blo63ka>^2mn?DUa}*l}{EgQd+~$XP+l
z4a~sh0LZMvt08NmL2P1hPatHx1*G<OJ|v!v|Dm4$Wd0B`Cks+T2wPaN>^wUKvhR~S
zZ4p*KfXp~<4LQ3I#74$mGd#fkpYp}n`^W7|?Je8QPh0*xmSAxxe+}6DxMz_17GzHS
zLDaL0L1Osu-d8bTGeBydHbCx?=@WvSgAWn|;U&$#!TAEjCWN~VGg&Q?hm8A9&Bc8e
zkKu;N;CRL`?-AtezylYtkB`};o!x$E(?c`GtX~!nS3&mBW)~y&QgEIk?fld8Gc(}%
z#b)<eaJrr6rh(#Leup}^8VvWM2{`Bfv^Z-InNyBDxEX97ni4Q~<$e>a^}n|?q^*5s
zk2}`ziTK-UF>_zPh0BXG7E@nA&ObCMMYI)ngg-`cJJ>QjM7e7-+;7wUe}Ubb!iK05
zKw2%Y6F>f|_h;6QNKMGuR{!6(;&jWOzo=&nf$RZeFG#(A%jGY)Z37lVBCMIbcfRnS
zWwCf2pXH<l-@$43y%3_jUhoAnhl8XAFLU9Fm0&YKiriz2!0z3@1@*jNixAxR7=iR5
z<6Ap=E$n{m0>|BMZAd;qmWQ#uKA@g402AkW3|aU0tgagz7cg-ceYN=hj)?{;JGJj=
zSv7YVfaj&o86(bTw^>M9{NG%SSfjh_7v%nzoQe<dFag>7uBHg9TR?K?SnTs@%cOsh
z{?9)DSK#nOR|Dd!-j71D10)8*$}Ydab=utC%UI>b);MmLIR4e5HNL}AR+A4L&aV$$
zgsc6fV+F21L1qzyO%o7j`=2O-oH-M47jmxxOn>kV4R{`biQ%G?=1jIMO@^EwvtbH9
zPXFJ3fn$C@;1jqm+h}SJ_8YoA>*Z8-y81lZvFYGL%Qh3p`Qo?Dbm40COlJ`tS0J-%
zzah>)TJuXB9FB44Q1>1PBoSZ!AHH;N`>CmrH3c^p{=n%LR!7Jg2_QEh<Hk>u!Fo2y
zVPF4VrkJrk#BQO5h3G7cG%3g#(Xj^HV0VMeS+bV+y{aJf_m(2ozukKdG4D|OdANBX
zH5EIsuT=!eq2o8PtvjA`rGw)mt5_QB7j!ise%llauo@5>86We6?Ehd<!d^dyIEnA@
zJ=JB|$}!J^WgF!DBg=HedZoTSmq<$g`qhv#gh6hwG~ESGubR_pz%)n<gtbm0?#KY~
z@!_zDd6xURAbWeJX_?~8H|+WHSnI$3c71TzU-++pHGBjOmRnrzdTYt#`Ni^E5u|?A
z^?=M7gWRQk5pkX^h))PVS}_ssSG75iv-76&$)LE+QX?CuKhVuz?7YLW{YN@D-tEqw
z0Q&`94T$gcUJ=C(kQfN(orCOYG0(%k|NF<2cRS2iC|mA*>TmgT^+vGU8BHF;!|KYP
zM_@6KS;Szgt%&;%xWXap9K8;r=DmRF#IGrow!3Y4upY85dv-blI2^FK!|5sF&OT)M
zS(71qkLH^~?!iHp1F^j{thV3zxX1j4tB9qD0OYL1GbI6V{~u39l;<Gzgz)zrkTc&v
z>J~kQ%)O*#l)}veiEF*WJ&q5ON5(hn+^jqcSgb)}A*K2_)6K<8*w6n6ErZ-OX}kve
z{J<9J-8-FhqO3e$9k<LXfSmhvrW$f?8^})MYDoBj*u>y1Z5QC_<+GFs*u4s=sOkTr
zCnQ}EW8R`&vn(21rh?OUS_PtRLDyTfBmyOjVe$txAZG@|UWJUI!Ng(os=pq)-f8Pw
z{xY3swKcK?Y{nbreeiVVs7L%>Es&W@w<6BE(pw1G|F~w-Ih1_cEmaFQAEXB#UL+!D
zq42W~9H#0s48b%u^$MpTZDe%0=h~2dq1$Hcb0_ABK|5~r1X)Nd+hS?F46=qe<;Q<`
zng2)%QJ<lkjg2qx5OU@#NY3RqWdFnBSk(PLiq=PQ`U9j78NWNjX1R9P25|h(dWQQf
z+NYZ#Yj=?Kg4oYrL&gjwmLS%&gT!H2&t~3Ew`E*b_kT^Xj7Vev$A9}BZFqb&i*ke8
zk1+Lw=v}c5;Qls9o$@Y7-t!TOL5Xj_Jlyv<fb=2bPoKNDKYtCmyKLh$+-DJU2|>>J
zLe>jndpAPnG#)&L?8O9$A>&JVGk2_GNU*55yTQDkWePanJ7i_y?$w<LS;LR42OoP<
zGi1yQq=r2Pvd;goH0rt?4mIM>KbZdikLAA$lfdqY_`VM8K9Jey*tn?*oJP>aYqxQO
z_3qo(0gqE;HD8=+Ej~4Io1J#8w9NPr4^Ed6>z9G`fYhD%3>m`$v5CQ{OCfCtklLsv
zhr#A0H6Yg0fW!?F4Z-Cih)oC!YMr!@VTY`36i7zYE$I3lec}SAU3778<t^a!Yw@QA
ztNdpltDRCUw=Kn+Pg>p+&<CsaTnAYP1Tt$$E#wX-5StkMzuyn+XOLRHr%S<X&KaoZ
ze`!Xf5$%8Hi$RtQtUTbj<=4Tz7xTU#_VvGpE|4=_`(3b~f3;uc@Aey}8!T=Aa#`Jm
z&XK)KhxFS(e(1PITKbn(f}EiO())WpWbag45aj%RkQfN(9_uCA{|o)qESqPTfYW1f
z#Sw6Lg3Lh22Tx4J8owIz#ldl!ehG0d8M5BQMdx=Y=G`z)3$CzSy7E5Q9yTv8Fbz@{
zeFjp8gV@C2)h7`9KaceN0K0bz^d3x@ex6;Z^#M#A8@=z1ftB8_MPR?ZP;~?MWwELI
z86N>oBj|DtB9JvLVuw-NU@&#>wq3XQYT0hNTkMwkx-><wJxky8!OIyR6-XN%rk)@z
zf>`Tz;Sgj#)#cI;l(J`^67l8#0<&Y5TYMq?$NR#1DCQu$rLY?N__xAK$l9~qt&qMQ
zvK|mSdqcW~$EylUTbD$Ow)&;u@GVjE0;d^}e7ry6K1UFr5H9Qsf~OrTvk73fv=r*z
zlhP9xz~Y3=0?8jtRN5ha4stKA@-E!_ANDRqjM;+p!El`h<ouFd>mltxm^h4XQIp)c
zQ)<0s6K|?zZjKE&eBTxz&dPSlBL1u{keR+625`Ui?Sh<z{z9u2o}NHz3}z7DrfT1?
z-m+K?vUfRv?+)B-kU7YBi#?={Ko$eB6I&tc-!2v**0h7fVYp%cBnz|LZ99Z-idY31
zYJ&5t56?xoJzrcQet@YbM2jgQ?!P{44LLhNb?YaTvDcEVi-|7(mIygphTVmXDZJ`|
ztmVMw23y{@;Jk}2x8Nw`+?zGpkbA(<<^KjtnZK_RvS@Uywp3dO8DH6!dj_8V>!V4#
z|KfiKq|F1e=VrrkaDH-1hKwtM#6Y;r0rxl_NFEt?xdmC?4S}3(A$4s6RzHH&Zxcp6
zV+<q)!j6@YwVb8F*z3o=ngTn&>(8;AcV@NaZZ-yR+jZGF7jQlR>A$N*{8|N&`XlA4
z@VH&F0W$v0cl0sbJdm2Ehmd`NAT}YqZDp#({4HO>`A4Aj7Cc>|>s#Gui?#fpoCR4I
zw3js(tKNOnxptpy>)I}_?qK=+=P$55zuqC{>)DqPKUN8H(~~O#aC=T_Le?v?K1oFJ
ze}L6Oa6SX6Cxki7$}CE@I)n9pNkg4SMb@jg0DJtJ8$j0m%V}ZH|LY#wZoesNYuRu3
z#PX5<5wQPftVGO}ewc-O%@wl!Fm_Hr6Wni~?I3kw@sAXgbgS;40#^gmhl{TH@^R-G
zRmj@a`fZT;DQs$wp2Xh%wQ`*gE_+T`BH|QTzt^T?I~e?@o8LHHYpL)Ma_^gL17iMT
z-X`Mrs)5Wcxbg+=x5-+OV0JxsA-Eg>$%F8_ql>}fAT}X<Xa8~21@|EBlI`~q<Fe@b
zf}df(|2c8GCOADFor3-T-!~`QcS!7xvM9PxY8l}QSzpxe-3ZnLvg7M<L|B0Mgs|o+
zM1N_=d?&CuH~mq^Ua!u&NVNY8qb`~l-GiJ*Aj}a7FPG8X5zdPJ{!8bEN^suEVp0dw
z==#_aV|RR(_uaXKPt9`w?>FFmABEqXz~vxFZOVGm&i`V{gq(Q-(ra4Z2~K-&6;a0?
zpZ+6$O>wLIhMj(mkbeI*Z$#MwG9MXd=AoW32$L(G0vUTPd=8mIf{DZE>8i7KB&YAR
zh*%_Kaq`wuaQ?4e3E2k(QZrX?BREVzY+~@%+mJhCL29$2CxYXD`U2E@TRuxG!s`f-
zI(*oy@~`E^4?n=}l0Anwmjhjo^4&g^G6p6;V_P>k-dSg2pC9MGtGctiE7;<2=Oc^C
z$B?z|>uw>|S-9?i%wfXJBSdpR&pZRE^Hs6|r}GKbYf;keV=H%}{eR)jI?H8FkhN_G
zZX?##fy_t7OWB*T`v1yFZEzZ7ZiS5PA?pFLCFa|2b1%BIy|<#u#I6r=&vwn$Rq*s*
z5`-wvLFx(NRj$+Fe!CYE1or=}vOVB=WtiSI?1(dRVPd%Ga%L9G4;LV7`?s0|g3}W=
zwGV5tpa1_S*b%Jf;6X>M;iDlGvh%lcwq^HGBg=yQkaREh?It+PK=zyuM)ZY2d_wqC
zFJ!(Aq%L#z5pcY-+g5?oFGvi859C}%sSiM6xG?kWMLQ&)M1kGD>m!ae4h#$oe_xh^
z(>c01^&*h7lR8Bq<DlsBg1aSlPVr*2oGsB}$>-$(cCW(XXW%dcsdc{&Il~piCI)}_
zjF>NxW9b5?Bg;vlVE4oH?^<CGHy<X3i~g!9V6~tUa{lRFGu&s9EMYD{31gUfQx<*)
z`-#n}6U7ZM`Mpog%rBmNVLAWv0ZX3n#o#bJ;kgBF)}E!L)&B~NTfz2$%wHq_8tl)~
z-KcqUu>kRBDkRxvSv<c4sgD=TN4-Y}WCskZieW$h#m5P<{_%D__Wf`DaW>n+FMTnS
z3twWXtl|N7@1lKT;BWw$EyqrLTLq+EJq5Dn8N^mP%nUYX!y-|Xu-86M{F=i26IXXO
zpN|8#?H|oW+${n!9~n>7nGTOLWO0kUU~qiMu|wMS$Z{aI&$lx>)F(VL@1K6z;>lV`
zuzMSCBhKe9a9sor6OdYB@REnF;4%-SR;>lHZt*L#2a5j-G+4m)fYcDeciyw@%&~|D
z>zA8>d;jPEbnJaYtxu4;EY%hJ_+Zq^SvziQeq%BBptfa9iVoOa3nLKoOm}s2z+!~l
z36pnQ{s$gjz4IaKzTS8L1IGbO&5}~Y9T_k&Ty$>jr5zI^6T$Jn_xwqC-GZ*(><?s}
z6S{ch3CNo15((5ZSYhhe4m`2g6~5MT_n*&}2ZbQx|1H&s`+vV_la~LhJ|WKfx)&)A
zcJH4@PGI-L?A*ag{P?$&oW)LOnaN=N_vKf?(-zDu80~Zl`~8ON?F7K_Cz6*7reW$~
z^mIPwor{A7EP1%USuUTn1RQs_IS<0~Ql2U5+C`W?T=c1fh%<U$t3&qrSsS3PAyGT1
zh|?eF=6V$-Tbk!V)_({qse#>xt_H;4cOSAx2*gIlxeFj;Iy<-^XQ(2}f!M2Fe%PVE
zZnC9U&~!_wj`Lvi`K%G=Rj;<fUFU%GqvI7Si2DM6G9Lhk<DpxKHNhbD;#*!5?f<g1
zt9Hf*G=Sq@_7&tlL68~fm{&jmTt=XaPv;5+w`F#5Le3CDmshZ~wXixRV|D7JhSjPS
zGr{gvv#JKuAhmPz3*l@KpAh!?4Y{inr0&7%`QY?#qmd4FKS&IOBO4&|;vhC5{HWAs
z=cEkC-HtZ5|6=tc$c(9X<gupjb=x3qr?N#!SoMCtAhI(%UCpv0Gv3PPJ7lelNGIgZ
z1dzE~Yaw$OAT}|WQ3Nrz9gzU(=UnGENAZ7Vu`bc^uk2Z9vFAWAI9+mm#eEj@A46Rf
zJ78|TvKTTie4qOWR{!^uzT0u&8TSsKT`w(PeAxjG!!SNXe`VtCUqrhVWarjPh_$X_
z^^ko&4Lc$8pdfh=o^9s{P8T3HA^cczmc_TzreME4eC~|Z4<Iw9{%r-j9mGb)4Goa;
zIL+t)R=xFxK0A}8{#Xj`tFUap2U%w%SIYoT^X4Z>oBt2Am<xBu()*BkUbTHJVE4n^
z75)SFItG{;7|mM$&cbyrWDGiVF7ACp(O(v0_1~J3%iy|~+4T}Q4Z+NX(bL|2*s)+m
zkwtFuYD>|@oM1Dee~E(243L@`$4P7d2em-fYJ&7y?Sq{AxR2QdB@NjI7ozY%YH{Ik
z!_MsoOT)n7cjaI^JYLY%Ule8rt3el+S_wI8_wWtu^M5Zlx9zybF0`X<H=~tLB4n>8
zrwU}g1!UGN9$%u}3Q|8~h88?+xA;TiYa@#{I9x&UAbcb7J<(=fXI{4>YcJ$(&kG#5
z@4xu<G7-fNkUb#GehjjgXUAsj<0C~=rFLpZJhFV3yU3DjlPNfSU&Kqp<3Ec_2Wz?k
znT-w4+<P6ae$}p2a9DT<qR!1+ZVdp7W7Cf=_MusH$ImUs;J8hVz`bvLQ9JhY4^BT*
z2KPZ%U4*QmMK|9*d*iko+pm@v%$Tec7f6BQg(nBHZv&*3NtF1z`atUS-0p(&8;Jep
zJmj8}#9T43`$1wLY+O$K{QvEm%a+_Zko|Jg_FM<M4`c>9_J8&dYy2nmLH5Ooi9_zt
zMb|e!W8ThlYW?OqE*4h5b0KR4k1_ec{g85*__G>8W?RfdjBRb43u*s`-h#{nf#gBB
z<u37iD)(;vYWWXZwolrP`z)IC6S41qi&_u4({1+sZg{!`xdDb%`My~^U2bT(yU5xy
z>>*@~*g3o(uJ%qF@pJVcGn-`)XIoh$w1LgZ`GC4M^=T+%KPN~%AuK5^w5!7Z2{>*&
z5>>%8Ha#o8qRz3y)SUW17o3+fE@R(2Sk@80gH!gl*|tAYmX@KAb4r%w9{{@>WX1`_
zh4A_e#3zJ3Qy^^~kUFaw#$dg7br9=GLE?OY*vFzka_D$zqmV`S4aj`_ifu2!enD3Q
z;-3w|emCe&7RcI(v+q#H;6Zvo*rQ9(!bEa{<%!$;mU6-O!RDX%YXndKVox%O9;*YH
zm-rEJwq5;y$oS`S7S#LRw#5*Cr`!*Y<(6{nSHbZvb`<wnwB5f^+ZZ6Xfbf#b_24qy
z_ci1W9gr9@E+~_=*dBk_qTXE1s&)=!um3A&$esz1x+Twv&#NHy$I{H<Wt5@DaWMPB
zeAK(Srq|%^$AHX0#@o(r+o?EpEx7zl$i}^H^aRs$a9Tsw3u3=<g`8=5Dg<&4AV>@u
zU#^?8^M&FL^S%!gEcrqpb3GCvui)ulMip{?GO`}LY<<Muo+l3<gVU2i6Jk9vNbj*?
z$T_<pHX$q{>#&Qh{U_M})f*w_Gh)*d$br56v)K10IR1YdVqYIF)BJ4b((_dom-U#f
z&OEUM`+MhP#5{w3VhPr8#pVW(*ow)JJ?|j4?03l8t9|Pr?Ld$i2-~Yb>RAw*5YBmU
z$>MMIbg&yrODChuBOvQ#RKdQ#vtS?OJg!qcVc>8;)&pYyh&{i{wP&`)DhE%?ql<Qf
z+Y0}=5O>&x-XQ+2LXdf>nq1&G2eGd{gOu~@l`7%o2}pd_A;`QCh)oC!ub;j1yd7k0
zYs-B$aQtA?!zKF*97gDJlh_=<Wyh-mPcV%k-?4YcLQj^R6{%5{tK8DTZg=)Z+<B;S
znzZq+7#2j@NtA$`^Z%&gH;UW*OCjwUbhlvR=LbzPW7E$7yS-u&?zp%T0P!cf85Z1-
z{hQN|V;>(0zd32=%g3=6WtpeUXSRET<LPB1;w;G};l!U+2D01n8|2P25W6a7Iyk)=
zm@EaG3lan293j;GP#`f}_(T=s&LaIi;QYVhHSV>jxAl9$VT5kZsYj6eKlHw0?;o<w
ze!KIY)v9e*8R9HcCP2;?I+-sEkFQ6K+Bnl3x?Ldt*5s-1w*8mxY_Pu_(;(--f#gAW
z*)HPulsidiS{%Ht4X%SFQ<lTs1u_R23&yU&n!k_zg`AmxydH6<39>$u{mZr#GnH)n
z*R|Aq{W)E5e3fjA1=|Nwck&oy{Tzr*4F2~QQMSKygp7snDL|Yj0y5(R*A#F*1F;EV
zzGi+aw)>FtALMnN;9-ldPyb~g*7%Qbh0I5oDPdn9X(V!bXPjGxMfR>6J0|o&#;%@B
ztA?93`6FrLpNpUNfa`pa`|5ol>;BD`M5Cl310LMxxPbH_W5<QHmPgZ+!0w3;#=gfD
zq&`#bI#>;ejf@+PK=vMgeF<4Zjw}aa&&{5@(~FhM(z-|9Lc?_nxcqNA3RyD;l21@3
zeykFt{y>@pJdS)?A>+K_yAkCHNR9s`#2H~AK0ds)%4_GFt+T=T-_E8JZZ^7}H?c-o
z{jcZ>Y11qg+k-WKobXw+({!<|g@(Pgg(EX$o-Wk$AAHS+ohNDce~0pHfxB^fF=RiX
z|0&cpM0qNRGZNALfzH>QwrxiduMId}&Wkf+^&?16ZaMb#&Fim=fz#fSmymOCKx&ck
z6}dL^-`8@sUuLMc<enD+&X@C#e}>=xk?9K2kE{nDyT}N!c7UZ6(ic{^D+P`}n7-&*
z;`7whJLfEZ{mTZId-wUiV~s<Qxi^auZ59w8h6_qIgTt)$EMy-9OdLk{eqOor*1gph
z$%0K50rDc?u=><x1J(mlvvE4&ENBp)5N?(}1<(J6lOXB--&4rB8Zf;^PAy=2V0?V^
z^J7sKB1a%+`#p{4ho>!cJ+(_A^IYiSk*Sb5=%s0hJ^jdPq*7EYCUYuU%}WTfJgV#o
z4(G4Ah_fP?^AU5Q$og=xV^5~R!){(O<h(XFk>7AJkiO}>mN@+Zl1IjVhW-{T9?4++
zng+Pnf1Z=UUN-U<^n%k&`K(N^8OUaX*jGZ8c0LRKX7O%Dlf~i(kaI~dKY-LXAbEyh
z+;>iZ<<W@y^@rf`Z}l(?%w|=z2fG2Lw14YdusDp5kB;BAXJ^o|dEoq$`Ck%exCvWZ
zV)dU(3FQ7alj$~C)107E*G?si6&4#y>n*?khRoAn7DJp*F<~>}JSTkagvnJ4A;zdC
zY=hkU_k<g@ZT9Fs@$KJz`<CqNl82mKmEMwoGu%@6vCr>UeS^eh-W%-mU*g+UcBb;_
z?ht<xY2m;8Hn{xT6OjxLt93<?@e`Q;2+?Auh_fDxIw0femUngG=7IDF8$$L4fY^la
z>Q#%)Z(Z38j(=ZKe(-tP*!0Naxc`05Oiys#@j>$@IB%ott=9fv5xP~`@?YCG%cISZ
zd*455Bkn6zISeTi(e>fthj}CRKZ?wS<n!Xgo?!RG^sLk;{!H-=vK+g_ZbQy?|GEeJ
z8c&e9Uq4_U|D9O`nbTIPh1{VCQj3hM)-T!V*%iLC)i%tkS_D#F%gP|`YbZQ032VMW
zHUpWx%K$Rw4-zx{25Eb?*D8YB)gUnt7CMGFBNW8Phc(iZEY+1DbDKU^aVX^ovOcCm
z*w6nsC<eKY!?FPT{>Ok-yLWhV%(0k%uia903Z%WN{`49=y&CZnKUN8HPxgL$cpj}g
zwHnNp_=S4*{Es$7841#Z56i3H-yyrc9~=koRB_+`Y_R}x&po<Xz1|DKdD?yZ8?5R7
zqS;>4reD)76Qd0*A6$p5(@V96>=^}_^?5C6=l`wjodFNK>LOKeezje~j$*fuODxWE
z9AqalPB1uX!96=3oQJ>PhpYoemWQ#=Jr+dCV=!^<S&(!QdJVh(Q=^XUY;p~=tiC(X
z(mpp3Y)^l_7&yE^W=MZvBigMX^(6s_`ZCYy4Os82PZDtRKx&T1Bkl+X@$unT73G#P
z+K~C!igjXO^RTI3oQZw>cdH!4{~T}IQQQDCW3l_;ohN_B?+oU2wA6@g0>{%&k3(P@
zq=xTS0ZuzX^60oV407fdNPN;82e96S-ywTYL1G|m{{?qF4w6U4d`+D@-xfgD_Qh61
z#;1|xVeF8%h;=70F%TW0u@78koX;wR`w1it!xoRXoBQ{1Te2o(TJ~(N2j>-zu;=jd
zN`Ki4@E8V6Jt6v(7UCTKgp_+=|9?_NjsNK{iN8~NpL3gKSmzROIbyKv7(8#GyW<B3
z_VxYDH#UOfJ#>LBm`2wpsxGxNyK%Nf!$EPYJ&zz~MDuzFgTn!&_VyOi_W$y0K<-cn
z={-3Ea?Z=%O^|jFNDPF3uP1&@<@~ZEJ1$u>gU>(7JBc`#6J!oDHd=?hp4xQ<G9T-4
z4*UF1NK4O7KWm<yi%xM`?Qnw3*B#=3tlI^df8)(`oM8@<N5}2WrQkRRiO=wbv~zfy
zQTwfjYa#QZAT@-rFJJJ^g_nZCaTout3vM>LzDsMdw+%X;e*~xhtXAytuPe}Nv5M8o
z@|}3G<%A&U9huq7;qhN(30XgjZXYiGt*74Lv<i}E_GtvWSK!eKu=_z`AZ&9IGA|Bd
z6T*waG<OI(L)O1#uD${`51XE*O6>c)(sSIvetBFES=*1U*I-4jg}dfC%e$(tEY<eD
z0mt=yyC5(PQme(?2xo)%gzy9tXSn~}1tDi4bv;CFd#Ngw;PeN&`7SHITe1sTfy0mS
zE$+K`y6Pb1Ji7iWNl4!zi%$b=2D*Gk7w67LXT!Gd$=+?nG#zqAWVR||{ZmZ@Y2#m;
zZbQ!e0@*WjJ7m26%oo)4{!>__iT3}m_^_Q72f4s?`yy7zcoN8LbR6>lb&Ul~{%+7N
zaGgG<6q2`L;xKw+?Fx%6M!`F}eU4Z<Nt=S>t|b}LW(KJ_>J4ebg4o31=m5n2SGO7=
zuz8C^&!L3<g-wVy0wFV(ncT8m-Q^7Sf3G-XJt;Olrop$c)^BrWK*peVYGA(~^M%HO
z9p_dqGP@v?y?uv1Wd1+rA>#Z?pVXt^@WW;&HZkEo#QewIiQmBE=%<2F?@)>T^p)s5
z^;Oe)m#iWy*#8s1;XaG`@KMOvCAypVkH&$+CGmJ2N*ut{-CdQrGwYRrMWcn1<+7=1
z;Ih(LAqgCBAT{R<Yq9DB$zj79jfgvXnzGu!ZHH+K5NE=J^k^=(g_{rJ<HLq$gLckb
z`4Q}Yo`ztY`Ns4+_W3WBvkBn%TY1|Wt3Og=9e3vDnOZhpm$Ld43c3Gg`#i+`6mctv
z->VGr$43blxEp6JhpgA?j}ZpDA0!XL91%0&YCwE^SpCloODmx$aC)h{Y7CAaZ0g_K
z2t~02rY0!zE?Cdnxsb6vm^h5Cb#2`ty<B$t1ddr2&x>Ay?O|_&oC5|@V||18vkF1#
zEkve+;~c~`?}Us6?A=fXPJ<vZ5Wch#axOH8O$e{Pa^JFV5#$cHNaY!Dv(fckx;Y)I
z|BpIC?oLl>#=duWN^<Tlmz~;{M>$qo{GA6mL-vUT<jz=-9lcJ7`)ojbLikBIqz?^J
z=WqzJ&w*zv>ixfp-vx<|e_`W|+mqKp?){aTi@V+IBW#7$|NB@W>)wSQs$dPf?G-n6
z^{Q^)u~>1v#e_YxO+fBexr$iB`(JG;(Rmf5x9c5Z|DR0Bdayrt9E03J2a*S2q1TXe
z!9Z+6Sn>A8?Jot^g7fZD;e2pg2Adw$jt^MnPW*<ni=<{F?({|0&z@wo>xew_P6v@P
z%NbMmgZn-GwUG8ENS)kT$QTBQO$<J^2{E_3t{t*>Y)KC48B#B;MX<&NHuIH5<SYwj
zLH71HG;IXC51YEf7KlD8vi!EsOTc-LXNMLz43XtP?1o=lmgdilE&ZM`TYb_D1;>9x
zNISS30Liy`;XZQ^B#(~UP9fAU|8o)SPfL4Alr$v$h6h|GqwB@SU!~e$xse-kcAe7X
zojCOsopQ&T{`Z_-2KK+?1nm8Pu}1|Kod$)LMf=&zk4Dc0mz5iP3gBTiB@%a^8=HGs
zRT1k&>JLN84$qb#6nA@EJ%AD(*vx^6@qg~#G5<28?7A911MC-=95UVKkc~C{7e>th
z`$^}sCzwXo17fq(oZYcld(w_M(Rr5f#U5ZYp0F;1t39{<9Xu_7)Dyzh?>E8qZodz?
zdp_U_Wc&`M_jTwKl<<IwW20w#_UugJ-41qx$+X)z_02f|x#tVrj5rTS8+0PH9zd6G
zV_If0ZI*`RgG@`yA8Q|h-M(NO;(Vg!bBOgk=z6j7)Al3Ii@9l80uIkrY^dwMia*T-
z`v;qTbg`b_EZZF<A?N2s$9jXyQ*<>TJ}Wn5ju6B~#=f7q!D)HJ5)rKVqw&k{9h+@3
zEsZW~Sl(U<IcLPN#0i{lLFUv-F~Zp(J|Qe0i)jC`nL@(xAp2dEbQ@^rkJBIM=C7aZ
zXIYaB*_V88D&ky5baetpAbX_H#gAv60ms3q2R&H*@!!LGhs&eO=An~=EUl6u=L@IK
zL(KU&H}w&nS3!0+{y?13>~IUR{<Ho>35prJ`61^5g47ejDZRlvorE;Oai>3f1ByAw
zdJkX2-oBG~0XhHl-mHgM{eP@!<4*nm?=75;$ym<#dkq{<w>O>y=L3-W3;1!5xq;--
zam@$FdD9?q(?gK^3ij|o)|G?AKv+KnaxWH$O$dMfzTdKy&kdZ0D`w$7i@oU-<cuJ6
z^X9Xj0OvK|8>hg2Lzgcr58lD{c&(*jQodE8ObpoHKUyK@K!DT=l_K&ph))Rbhwk|S
zsnfbR6KtNWn*&PNzh6lF`p*UV-j+3HSHS-F*1<h)>=uH3{r8Ng3&D0LUx(ZS1#%ZM
ze%-fp2cKD|rQ~}yiyJPGGr|PSeZlsD)K#0T2kQs1iNVIR5dEJYQu^SqzY>*#Qf_Zv
z)lGE%*NAZ6-XE#}j=OyaA@ix&+)>#9Dd*7T9(<h-wtK3X9o$dIYKobXc1--e)>7#N
zvt|4~Z?Jo>`dtB+86b5xz9aT?f%t?l>uto?-r>wluz5}&5bH@n`dg<H-~OM*e$jHL
z0c8D%FZ(aBUqEJ{<5=dG;4nfLuL;`@PJ`JJjadEPt>0%c?HZ>g$Fg$EH$Nr8?aKgb
zh`T{%9m!I|S>}M`(ea0ch_Q{Ob&&NM+jk=NQ-aigQ6&EUkLUCIcgX*Qw0~7ERB40s
zW5fSa5N9?c%daqojEm3m#eV)vU&VZjy6;OYO`Uu#XYulb*L@`$AnuCLe}g#B2iaU)
zY>nCR;5Y}#|9J~N3-<YPxEM%WzkdSJc}n0eztzHZvEcGQ-w3jv3S<U4mi&VKe#c@~
zHn3eiDvn??(DmsZ_^?Bzpvj_-L(WPh@d-GtzvV9krx}pi{tLMKoFI90JiAL1p0_iM
zA?yF7>?Bdr-p=KB;A+tIqw`PwueM~&4h4tb?Tv@Arg4y-uhsb|c7Vh{xIPxr|Cgyp
zoM{UZhhfF@j=Pp_?6zdjWwk0l2dP6YW<Q0SCDKaT`KMis55QpvGM}v!GXIz27m4ET
z50_E*gM!rJ!YRBrEzit_+{<p(z5;GGy83@5U*K*=mh-HF>>1M73h4(U%YoSGx0h~H
zZa8UiO1RO|>=fjlYF&$MaQA*nWW{MGNIyDWYK~|xyNP`R$J;Jp)O}kM{#Otk|HtP>
zSS8;14^F%P-me3jhi)f`&v6RX4v-iKpZx}zkJ07#1DgR7L&opZ1$MEpMp@-}>RGW0
zK-$^sWFdV<kh%}0#P3xCsb7IKM!%qR3pnilyP}?d(>!?r(f;3+xX(f|U_03D>%ZbY
zH@J2EXK+3Txd9p5o-hZei%GZi!8EcQh<!YTeV5PbV5?a<ldR@Wf!zDR){JOp&&+xU
z_5(;QG5Go-#M~xJHe{W%O)g|#HBA4~iI977VSIda|2DB5%oibh_?auGfXioW>UShy
zAOB#oSqb)c!+q@MxXfGRVWCztbw^wtm(|vyso-+YBpoqcv2zM(`@h%aA;vbdg&}u`
zxagzSAr<+==c&!}6?eYAJ^>v6a>sDT|K%r;aUS%rdGn+loR@d}fvkf=mlqG)vh(TV
z1(u7x%&}a1p#mIMU0?Ts^?=lxIc~<9u0V3w@cg6}xZc9ekaJ!&93kgG!qkiO?Ik)*
zS=vWgUb+l9FXenICr-D_b8rQ_9cBlN=3faJGc)PDhqdh_YoWDspWVtGdBG<wJyc(U
z<6rU$;=I0w=uD#B3bM191yNqG-iMqWUH*v&-kt=hdGzN2R=0rU(6Q{tUzQ^OAZK{o
zy8)@6(ba(XU+W-qWFR&&X7phN$H$zXkU3OjIS^aCX}S57GwD0_&Q`N*j)R<C<?|Au
zc9F_`lKd|xy%w(i_bLN$IX9_;5hZSPzZrqWLG}~E7o?mm#isgz)BodM#NHZoeSwc5
zV`S*!Th~I?Ow>-<kKzWHy5dOVodR=~@4TMZVYyBw5FA$9?Gfq!#YAJce_(p>(Ps{%
z!PWECLGD8OnSgq3)vX>sqW%AC{$$HkJq57;V^;O!)Hh|xEv)`?y9}B4jeV&KHUr)5
zjT6<j3vtfbsj;NRa@ENbV7D(WMC>I!)Cal49$g<UzW)Kl-KP4lA^W`8>LBN!!}Q#%
zUy9QoFf}mx*e_12T>_AEU!LE~#F_sYd0u1ne;La}aGW+T!M=9jW6RW?hpd!$ZVrvI
zYBy*GtDP6H5#G*tnS<ER3Ud=ay7@#h+;8(&L*~MN&Okjwa=j&Le-NK}%imqFWPa8M
zc8`jQ2snPQslR)n0&DzkX@lJV{h<YQ3>Id_ejBIlVb*hYEOueFDz-@khi{-4WWEuk
zM#6x!^UrrWvB3Rr68jzO-mVYqU>al|2#4%m1y=*&<HPk$?UpyRA$Pas?wPa+m)f5j
ze86#sF8|vq3@n}(g}Md}rcN;Ug~c<LUlw2V&u$MeoD81-sAg6H_k%%dI@c0ER|QfZ
z_oWvcS0HxJ*E3+YY&GiMGm%+6;Ccq6h7g|h%6<EiP%UGSy8C7<;PnI`HacGO3o&Mi
zEN5#AITs{k8l)eDEC*uu$2;!2?%ZH`eE%FvtJcq8|4UB43C~Nt$4G1c#5p1EdXKJ!
zoO{u|4Kl9|vJZsQHf{%}GZ32)PS?C+*}H!xI2>a?=-^B@#y|3~rvKnC#bCRp{$;=_
zf2DAq`P0?CJ5JB)u{_?r51dYWw?NLD2DxpE6QVo=@d;t36BFQW^eup#*H{(B0X7$=
z_t|aY%Re)He@pknieR_zk}3wLWtch0^ujvq>znuzpM%>Tyy;W0>TTlm-C>vDXQ39+
zVR<zeve(Jpd=fa!K;}oP7=ptd#3ly6;Y8dQ8GaU8@7W;Mgo4Z{S^_y|8^k7r+v{do
zKGlKTZ4`Y0at|;zJ;zwEum3-P4|1m)@0NOannu?D;d$!L3(1o#@6TkiI{pr_?)@MK
zqAg)zPFnuooP?O`IeY-po{B4kxB+JG$ET1z$uK@XTJ^wh^Fxy$_wsor<32akPRA9T
zM$pY#!VbB6q3PXCc$^}uv7hu}*K`&CohHF+Elz4mfz@d2o&ZlPhh~tr|J!`+D|p$q
zKL)bSZ|mXhaPvTR+qTX}$qyhgT=?{p{TBRFRlsrAl@$p$8(ls3ZN!)rvYhB1$bCer
z;*hm{$Z{a|+>FgT^k)}ZESb5*(mOi|oL7F$-3xaw|1{F}Ki#X$fcx#t>}}w@^EMeW
zt^l$Rgcm&j2DS&pCWM_fgzb3p9nvq?j6|HRg0AnF%rm%~k>xHfo(GQ8w`(D1`ytDL
z*sre1nL7s^viuOCY^C|M4qWfq-|&L_A*K?NhCynH!JEV(YaT&rjpRka@g)+9y54W5
zj09L5q=pbKJ-c`Nq06b@wC$|mjn$7JGa_Ok<Gdg?GLCC$0o%1VsS2EikmW#Z!4KEA
zUx{k5ELW+pymtIPIIL#)u)+PG7X(?u08&c~UQ&n1{}r7J!SNsa19Gn(O#h)U)b(&M
zacuPKBL1B=R}O;vi*ad?^9ix3yYDWAwQVrV4Py7kpCw?opzD>n|JZ!S_fr;97ba}K
z(8CB$mwpE6;BWw`_55=U&Ia)b;eCvY!EObqn|1+mp0b=R>K+2y&nJmCv*~op&U0RN
z;CSDB&ki0hAoG#2#&YcE`y7~g3T)T5b=b#;%tO9xH!qjAWY{NZ$(G*<PXAijh&6A{
z=VoIKH)Ok!*(q6&IbV?2=Gf)n@hXE3)cIbnsiHXj0n&$zj~`aBY}r-_PP;q$J>YE%
zWHlgmOxtX*8W0;9mrsYx<rpeK?odaT1F>%j)|%%%_ukp;!eRMfD?iwb`Py~h@CM2M
zy_N)LgZPASsx#ugV8(wp!2Yy;x)YqPV0uHn5Pc+=7%p17aEWEWX~-V#_QSaAjp=3(
zHR$Gqa6;C-U9S9yH6AxOyxVbM<z9>8%2rFG4~xNN1tZe^AE8ym?^OfYz4hHBcpS}`
z_!jK{kH=8gZQE?V3>F8eCxpN8Zrgr(5@c<MV4DI~KY+|ws3s3~JBW>pYp$#U$KR>9
ztl+psmIJX@eDm4aYqimGTlZ$mEhUgO#2c0%)>$sx^aY%MKx&D>{gy&-cf3-C+#~+o
z9dZUEO#kIbNShqS$4BpBey~F$<2=~RqRrUXFoE>nSpFL9PY@d!?~ynI_BUIzE>?fk
za8B51-t)@x-nlE5+apea%dYrL$bBmyb2h)gz2^)hkB${C{)ER7YZ&DGqf4<j;bI{5
zm*PE%j{nBaf0iB7wnO}2`U<B$n?ql*`oE(aGWODHgnj=1QTps1a}C-puYMM?EItI8
zBQoYdoPX1FLj)WqAa@gk%U!$SZd5UX+#^~ia{|Q-VI5s~9wB7rt$zRQWp%>fxYG;S
zi4u;;dbebs!Rr4BSDL_Krw|nhrjhl4*f(G9+wso%@{W_4i*`sW^n=Z~t%bO+xa>FT
z-E|;6_%JIcV*Nkc&G}&WIy`~gUkTIqcLwBMU>F}C-F}CC=LgehaQo-*Q$*hiU5{)(
z;@kyfxuwypU^jdW4#(>Mok8_GJ1XW|ESYu3l0_*4T>r_3K+b^&nKLZ`_naF@9vz=a
zw};0q>zxl^yZ;5)gX0gT_J}d@^Z(zNEO!P<W`o_H^*9Gy4#3PorWbCI0LLA&7>FHa
zxCJcEv7#8OUisF;79XP1EakSBSXyjM0FQsLbwK7VK>F`b$OHQU#3ly+Dn;b=lS}x)
zdiUF+u3Jfv#l999oB7|i?YD@oDhJ2Ei#P7Qs54~cz-rLVh?89mu6uvZ#6JITsi&}G
z#-f83mz89!j_kJr=iNG8Yj8RMnf2xbY3CpAe~Fm?{>B11ujkrN$og%Nc_3UNiF+S6
zNFEuNB|Nbx`_2ybn^iXC9B5>D7(4d<A#fUj@nQ6e)nedyUlECYeW=*}_?^Fc%q&>M
z0<BivnGaU0=hp@|%f}9Ke<I90LiGP>i2Y)mvKipKFgbEP+&qwejn4OAdq8YLSm(01
zh5ai?d9+FtG7pAL&xVfaD0aZq#FRtI9>J%RvC1zfXSVQK+p%N2_%=(2--6)yKPCuS
z2MRLd<5A-CDoDM_)RSN{K<wSs%3yZtDPM3t0EvO{rQb>5Z~?Ii;q)Wz<|o8*!0~@F
z1p680ATyXtmSXkaJ<dRI*)wq>_VJOUt{--s=2f&R?US=IE`i*y?W-&gckkIc#C#6O
z9zr<H4$;^3d2$`>Pk!}Iu=`<pdv-4Zi^KT%XdV8kJ2&+}&hYw_@(HUSLHbwAK-z8~
zHZnfG@h4bMMzSH8MwSDyTi&bf<g4Sd3h838;*?DTmz59X5O;+O-oQQP3et~`3vOJ6
zhuscO$lQrZD`GzlSUne_d<FCI5Z_+q?^teQ4R%*VBJR1kUq;yXw;tAn)Zaogu&@7r
znfb*`=e@S2WbY=+iSK7^1UY4~2crM~-4HT|0TL$$r%Xk(f7Ty<3Jwb&gHK=@rhmra
zP82>&92>oz@r7l8I^=Fc{*REmJh7=O+dT~&M(A?AzajVE?YG2!{>P1v);m?#&$PUJ
z^tR=Bdke4`tPMxNc?M+G8kuEqHi%CMORRJPn*mZc&lA!Y>=HPSlJ;JFBYsTDL+puV
z{};%;Jf(gC;PM1yHab>4h$yp=<xZ!GgY6DudWyBoz4#-3r;~K3`5s|Lt57A#UC9ep
z6u{#rvxBtrFVY)Vz{9KYEabfMFU;rQVj#C|Hb$H;4C3R%)ANEY-tU?OE<gPL;qHrQ
zUWM%QLN{yuMg_3lZ$F`)!30zH%*1MYbiMY@-BO<|yS*UyD?heJtk;y_M?Jd$rVkg*
zyA-jvKc*IPUU7K|<Xl~to;!9^aK<A{4UFFUOvJKh3Z(x3ef}{{eP@`_${45t#%c4x
z{<n>Stf7Vq!zsxrvO6o|eRj^!du6%ov<^6pBnLsp3_*$x2sz-i6C{t0Pyba1n*kD^
zCu9vS=My)fuFaeg0+|;EsUd`QYhx_C%^~YQH<+}5&BLZA?>6ck128o;3pRoMAERai
zPD3zp7#$e5e8<`(gPqxdi!GVgX@mV5w_-B5%?MJn(T}wBf57n%*bI=~GhL9ouqSd}
z0lNVt2EvlAlX3b3B#(@nR)$*L<y8T@-MkKRmkF{wjJ@RC8gM>`@nQ6rcaU?@W*9)u
zfrN>}XyKf`osO)TJ35t{th`@C);$(PLC%2zsrepH+Wg<yAWyj8)WveaY(ph(c$xs2
zDJXXY9!4NOJ{-lxZ1s+L8n`@KZH9f07o<O71@`uhNauTS*ef<;zyEK>?S<Q3%rLar
z@ms?xJbOFXUD7WgYbHSE?%*W;u1b*l?FTo3%>c0vX?20w3%^13Z-B%=SVtuf94;U>
zA?$MC`py$SA!i$%H~9_@2W)z7PHuzOJIL~rL$-qZGO>>lZ5m|xT!H8v_qv=c|4TPo
zo(uc~PWPpvkbD4Aw{AD&{zMR)7<_QiO|YLqYL_WM?h(9p0kW<ZBnHBbtNOui0<j5U
zyDKV|Z|W={X<J4TWgG=rFTWo4`b{?$((bd{jJ^Nka_XqX%|lBqE4}_%%3V$d`@i7^
z;=V36f8y(EkbB~-`QZL{Zi3v~q;&xG+~(6ECE#&!ka|McWu>H*(RCMa{+C-+g)`kK
zT!q{bfNtL1iGRRhrXq*E{i7ao-{Q^Zj-9%X_F2|khl~;XUV)s04l*l!8>9>av5CR)
zoyBlBmfVA!fBB_37$yH_FcE+M(>Z3YT}9<b!2UlZyd9jEvAILy2juKVbh!&LJYfHG
z`C~sHq4MF9oj;Bp-PZ8otffHgYOp)mw<y8$*>3*FIPFBY3&dZag4q8iCTtB(3*m1b
zprm(?i!0&ggY@9TyE5W;MdS;D&p)bPiYS-R^|&`-zu)WsC&*os{dr3uc?eCTL%#j4
zpIc=t4{vL*y1y54E-P!tOt8B_N+&Os#;Ok_hYg?LSObs$)JKr@f29SxP{L3vhxj!$
zYdVxHKg@^3%k3+Oz6HA7@v#QrG=eUE*<dEPTx8>g%!8oIhrbTkW%8D9XXRm2%kF5%
z9WgRn!{C1I=R}<6iLMtLzkWG8JZ`;PAnSZ%=0f)Gz|`~B;XcO`rUph&tZ>`8n&lJN
z4_&_ybL%j97_IaVb&V-ZT;y2-*gdj*Mqo2w;xKy4%ch;2_oi5Gd?0Rh;y>iBK%c*m
zbO2J5bg~AlAH*gGbN_t?Pdj2;uY=3eqo+{okYnj3DCq&8xlLa+cV62KIsb(>1^2wM
zLJ#)w|ME7-{?GW=YFPcBZz{QCre?3DvFKe(`$Whca@iLl`?f&tDwaZw=YjZy@QYK3
z{U6SYJi-1H&PAMw1k%5N3AH{1iQ&RN95K7(E?)$v?U28??@bH)i@ksEo81VGzo>$4
zc$!AGYxCBZJHIfTu(%Z6W*ImevW7nC17Z*V(j!l?h8waO$ZY4U4e<Q`hF=_<7998y
z<LMx^Cs+~Z>wx(9aAyXKMfulN;Bo22)3Bf62+|*Pwg{^`EB`^lW&LWznI0hZFuX8m
z*A5G}{X1HD?Y4Vc%mS|=Fp@WioAt<O7r0J<sV796&)EzPZ;-lVA;?_-u4<m(ItU~N
z!i>-Nfzt(uO$g8O`(m!f4)KGXDq?O0UEh^T?CW3dKT-nwWfJQvFpaKnp4jXiGtvtz
zC*NqZ*gGQ}?AOBfTVNWb_TE<F&uRs!pZpimUySC2%)Kbdp97}@kUR)?$Dy7F1`@-C
zJM0}TnG+!Q5jc2Hfu}8W_4kvp@Be!G2Qu~~%+80^|0R3p?37Mly|Z%47K>^9ka3BC
z1jJl%!p+GjZb!ElonNvVasPqLZ^-<=S-Jp<8M1Z6@2M-k7`J1hu`M|N>l}iNC!yO3
z;s>9>e*UMQR07!F5A_ghIzj4T__qz;F1dyGEp<${S~7Y<?y1n!MbuZKsfhi&Fg^I_
zP>*%+I9fYL6wJ;JLF@?v>HVRLxFZ3?$A?QQy|>@Ky%OAhYO+PlOQ7p1od}udLKmOs
z*bB~UeJx+XW}wUW@7FV5oX>0-nd4=d$7&0%uO#$3!Rw7dYM(FYfahlrpAgO~Tn!Fy
zkh<G*AZy2+HWZ<hy`kcec?OUgLioIG+xC6;A^S(yEkNAMhOTekI>;Rv=;Hkfkh{}+
z&n(93j~5SHcX7NDwdm*iWieqL<gSC0XA$jRM;l*o9HN_xi=WtP22bn9LLgzMbayAb
zJOSyO69Ktr1jHtUdvr}LgX=GX%g^^q5NpfO^%+b|K?!4+{4H0={;geiEwK83TB*d2
zP17wc6V<0!{(SlfY|o3yw%{}aG9#z~GA9UP6N4G|OTzsxDhkQ3y`NFftqISB><c2s
zyyU}`meTuW!G3cM!ae>UcuWVZ2Hnh=L8f4Fain#*$ZE`|`t3Zy=CD)toR#GhDTv#f
zZc4)aU%Lcyh6S=7eC%g>kTc&vY6>nv_7iL8yP|~sotua=qCo2K;dIH<7W1Yb0GEHi
za$@0m3tdn8JM8;AdO9q@@u8rAeSWYz&3I?hR1r&NFDWbL*^qNDt-}!gpVs1^@UTF)
z6C3~HVmEkxd1VJVw`{Kf>b-q4DpAJ+u$coBbN>Hf=lujo-L7*L5}q(QWSZv&WR4VB
z48$&V3I)4=nkwXMACMR_wv0ZsJ*n@Pg{AX+%M}+Ef$u$e)_4~jW*~JTJ*aD!L1Or@
zj>Q7F-Uo9b?f(ToTEK3A>61G=6<p53`1okA-7_t-I!wUvKX1oHc-W%riIipot3el^
zVRRc@_I%XJ$14AKp6?Dv17i!>|FbMPT@QlYwT1_>4g_SD+z#j&1|SX*_{nX=c`3R1
zkh(DI`Z{nPglRX;g`5iq<Kv?*Z~wjXaM2sE+YfX1pqPWKuh$rHW*4%!+W))YGOeKZ
z1=hTy9&&8w_JyogKTPIW<#8r}{m-O`IP>e~c1Zsj*-U)wZ<A)g<7i&Y1#nuJz6Npz
zEll6ZMB>{&S4}0Y!m66V{<m3z`!3>5%l}~Y-^YKDy=>imkTx{T4KVuqf}h(ju3l&M
zn&Xirr|)#Id*`)rz|GqC3(=l~>A^>v=pp)x65k-_-0ZlFFb||x=po{cSP&l{PG_9C
z^KucS4|dym4m@np_1LVi1BVg1xZ~P?V83t|VxJ!_aXDnJc6qO*rc#$h<7&tm=}DRF
z;5-8|%jgI0F&B_LI-Z%f6<*f6|M3N%|NoL3_3lB=HN>C)nR#@Ad8PP2uzS|*L5xeF
z+v_VRiq-$u`TN1*Eq4)f+Q@n<SkCWyu|>l2PR>+|f{T#70(M!5u(F#%{9F~tT(4M2
zdIPc7&E5%)uiLLt-L3i(_dXVoK4hF1{%XfNJuYzh8Mhv?o&Z@M#<pU@e*S}U0AybH
zSqGv`57PspMW#0Fta#LD!E7XAX`(R?JT7fE4N~8L<ZtMa*8e&76ER*M%@5g=qJ9Um
z4*+By2-^o@j|-3-I$pb1e}}&TWbEqqZ0utgAU#j?P|w%|iGlDFCCK`1t}FM!`2Zw_
zjAzE>>?qpMZuyU~&+=TwYjFAZ`T-}H2C0ib3^{uR#3lw$=s=XwS9R`#<6lkV4NAG)
zwV(Jj)gI4bIlmmzcNJuB2bWLS+@Wmlj@5r9Paxy|=Qd=4^ANh;pS3r3yv_;QvGn^_
z%jwz$U^8NuK+f#~sqNAtZT|Dk48-{tZC4;?hu+<VdT-<RNwPTo0kRVrmpG;Fbnkx%
z&cmE*aL<27c443Y@O9V(E{~c{YJkl^HXFoVV7-0kqRmrxdK$d9v}9TY_P@;!$eH0F
z`EPZ7@U#Hp6T<N^&EPl(sXO2<4rb?YL+*hFiGgsnH7i&g#3qETEP8i*u7TX$6kH7%
zPr#-}Mrs908iT3vzV{65{*2=@u*#oIys$%3_qj!As=lS~zj@#^B4Y=cGY6T`wuQ9&
zUn;jE&Y{>U^BC;U{dONw>`wpA0Tu_DPYCz>E#J2H>{GD*W6!4H)c5D|cd#0C^Dg|_
z25$eHc@Tp&{%d2Sc1R!ly}c;K)3Uh;a(1U^8e|+CWR~z8$eb{UO$^?75OKCu_9I9=
zw|fcd+O#Q`ao^*BZpY~f*DOoDA@}jEFE|NK%jjxA{GWQ*=f;^!Ann2j)(^pH8>AK)
zui7rR)4TVV<)meeRx=X5fz?!QU<1=2b)M?D!yF`!j{i0w&i4QL0kRkVcRlKQugVL&
zIO7rByoXGo7C~zJz;SnN<yoBmS7$|>S%z%J)FbP`X<B~s39RYEJ;QEib}y6Vu?;F#
zl1Y%hu2?f4+@3#zsAm@-n~jZ~yVe=*H|Il;wXmE^QO9N$MI*+@v6+D^=9jm6=hctC
zVE^BU(!}Zqkos6@?D1c=12TWJs97GX-Yj;Zoo)}?EH-L$S!O#O0mu7Cb_sa-yzO8-
zSPWzbF}VH};*6fhQ@X)n|3U`!+=kL;v2gPVnYlZof9F(b$k`Uh*Qwxii=ysHta9vH
z5n%Ca3}#@rpxehP>$g)Q{e<Q0>&8|Q&W+%*@{=2+-vUx=zi=A3`~b0u!OO+I!2LGY
z;04&c7nVEWVjwddi<f}gb09V$+*4?7x!fFbw)y|$Zg|>4*Owm?1x_RA;$g2KcQ0Hk
zeU3H$e?Cs!ng8$qPU+R`RxELldmjXBSmEw-oC!G-0Nq?%{K>kAHCj*eg2C=(FGB36
z1?ej{MC_3O@nM)}P7uc-&1HOHr&YMN6=(`yow%0Y^z?m`79j&uU)x#yFZEVvu5{%P
zxMia*cSggA?|z4kK+v*B{FB^$xgTrDnrXi1;$fIHnSTnuwA3L-dA=tN#{65>b?du*
zW9I);_}{eDWIF#VK_UJ(&e!>7oYLmHXIIFb8`NRKdhoc(k-0w%yzaQ0eGK};Uwn3w
zv0U0F{#`GsxZb|GW@f?klw-1o3!g-cux#y$P5h1%rf}Vu*2Aq4@t9v*?j2vq_NTlD
zcs;mzcK7jwCCQmu>F?tT+qs9|IM>HC<^Kl0BYioBZ#8oHjrC6O9f%X)FJ#d-73|>U
zm#!D(``*{j?;JgupUZGQ|B40X?1JI#{B`|(Tt0KfMJ6xl61chcg=yU08>Xcl1p+dj
z0!9ZP?Gf->VJaUeb&hY%5^lkpo7V_9>uK<m-M(%1%Kj-&gNMC9bi+3}!8QGSytjn}
z^dt&R>I7=|?O)6_Yv0{0FiZ9UzmI4O|4g3S92=|taxU+3Fq^gaq$%6pbfbA;pUpy-
zF$%sGYcx}R(<s0r^qwn&|Froz&J&zLURU^z&$U)sbTd_8kwPPf{DI5d`+L0v?oZyv
z@1*I)C&xaWr$M}*N9E2g)48`-bL%oS32^!DGRtd`;AiKxG|gnZ$1na)o&Wun2mFrz
z3eCQXo#AI#a*r?E*+$^%JP(23V^IR_Z5*tZGE*fko!`ghSa?G8{=!EB(W^tuF2)v^
zhO9m+pu51%;;*l=pkRTp!a`4H{=;js1lv?O1Yc~C<LT|2YNoBWpKrG1CF#!O*{ZX4
zE#q79WQ7ptf~DqOCe!(4Ws^*Mf^P~O&NLNBD)}z?vrn2!wto)S;r4SD<sA3S4sAMY
z=G7i)Y4$!uP;HTsIiFUjNTWqRx5gF@^AFYLTv<Xt`K}$XSD&e|Q{YTsBiDxP5FQPi
zJb^%2F@f6}MSKeC{XE%jfuc%`7fqcUHgo@(c1=K;<)k^^87qF#SyJX)VxRb1o(1tg
zYt<82QqO6A-AqB?DWey^*MtiKd@>6J96SsJpZKV8M{)-6@$yJ=IUn#5E-9VN$C7r#
zWZk?DQ{$$&yjdS5OeZ%k<S$icku@s_<(p*un=k%aCcj%}6Hii(lj(D*m0S}RU*I!L
zH<7)u%ZhtV#W%i99?2%VT$b?~^~spEe&!H%{42w^X+b|9%Po2C{7qLl#g=KA-MDzk
zL|s_dWK(*usf3I+|Ew9VX0=ia`1Z~Z;$F0Jt(ohy?cB$?+W4FeedU;b`|vwtWOKD{
z-pqY2vYqQoY$M;ikG0%uG`xAUz2!JJX!V+IJ+PH~6H6cePH#`Ma>+jKnW|SzuXng{
z|CbTr&S&%BD|xG9_DEfkhe3HM&!XBM!3bL>e$DtVyd`lLIo4fl;Voxe%sG{pQ&^wv
z3vYdWj9G5O0aM2-vb?Fig{JM=pZRj-{bYZd@$r>!3*!@U+0M7St&)4;^2??P(O0>i
zW+?D^clq*ZP4VIub+qC0-6(E)bNLjWrxtU~nr$5g13q=|PBD1RdvoDg&b-ic4%d^Z
zroVGkOdqaDGnwt{ZhD)ymye~E-Ask$6|dq{X6{FSWz54DnQ<@XdBOYf)mw?6U(@(}
zuBLGIgz$4OmRQMkCSW`7yuamKO?3u5CaY$#zxN3-)vog9@;Z5z&u5j0Swuo1w{`k?
zvoBocJQZ{MxNpoU<W*i0ZuaonQ|^yfPjD}|zF$zcJch6Kb1v`C1Kb?e5wG~!98|ef
z!<GxzHM|n|n>xwteZmY=+lcc52WtC`7*9zE7TVsDi>Vjne><mGaIOQJ;LQBnTub$9
z%<i|D^2|5v5U}2vr!al%S>EMk#R5s!+l?cBZseQgoNXq=_)Oqjuf4zuk7xW%vL;;O
zHYVKfc5s?*(QY(#jIcIVc`0wodfrKJbJ%p#d3)aqi2c0HseZT9eBx~*&M!MB@`aTx
zkgsiACJ-%B!8IX9g*zr^GQWqWyuduk3A}-qEV!HVM0xUOmz(OY%;x&8{8&IlUC?az
zr6j(31xM4%))M?d;$?ixZaE9E%iEiE1$hYU^3CK|D4Z%#>@-c_oAd(#Ya2fHb&Q4r
zzwaL9l5H^&Iby0U`1ayf<EisDn10zVEa>aL(6nkzj$p&Dbpltvr1KxWvs$n|+EehZ
z=Ou1+?u*7x8)ozTPC6hU;S?eB`J5nM*x5}2uMfEyD}LbT&uR2GUV1`J@XX~l0d)@(
zfkPi|b5<Fw=j@3*XtLVF(IhK#uZf<IuJK2Om4d1(yp0XlstfKoCd7T*na!l&r!AK;
z<1Ib~ML*S(r#=g~XzOv-x&GpgExf@0eRG;XQQ>*suwQ$5ibbDryT6-ge0jzs?)5j7
z1XE^8n_ix?obOafr^)4C@%%qK7V|OYwF>kbzA$Fy%oq5cJC|S2TT<}#+<OAti3);y
zn=Lu7wyN^np1hLtlP#xUMPCUIzb~7)`}1~_)}4wxuCJs_Cd(M|Hm$dl*}X@RCy9-b
zcge^1Jn1!|TvO{Wn6%bNa%x%6;pxrUsTAiig)@3~6^}o2hq1?v5^mck_GWM8^#tn<
zvG6c-8}S@aa^etKEX*14q}`<AmW%23&)bYo_r;iO{#?uR(5lvyduj=f_#q9>-0ziU
z3l1#g^qgSHQ@tpb)!FYbPv&wpjwz?VajJXTbC+ui@-QZE<V*>i$`$g>nO$2-#_V9c
z6h~~2JntFV1{0^k6fV^xQD*s1e{db$v5E7_zOUT%FXhc9F39EXs(HpWW4)$;b<=L1
z0OMAkMXy*m)1s#FcJQ@uJm+Q<;Mg=-EOzfoGoK`7v$RSF-ZI6drd#vY@NLRvmeFgB
z=RJHOlh0#yDW9~-AFj*k5vGlaNnD#vIQYJPJED5>*&(ivyc~S*(xw|T-+RE*GJmC6
zk?l93<h#+l?96Sv>)v~Dxz6b3vOa2LdgMdDY1DLnQ~$qFW}n|T@=14Un#P5E=ly1X
zirwIusoCZbX)e)&4ZOK(-vqNxMe!YZb)92>PXPCc^NV<%UGL_N{hGpE)mF+QKbL{K
zQRJ|hS9l0FYsoY|qql~p!C4GERzj;xgXidQakAXt(mYYeyTIDl>_F%*uACN59{D4-
zLMe|W_?}s7^Dfp9<v9CRmoKSGj;o5LLdaQZx_FXtq1jc50JEb`%lL%lnN9gqEBP~z
zU6Bp?vz&Knc`CoeR3rYx>Ta%Mk1v|GZ|36Zd(X(P^@m0CVc~pUsrUxI9}4Nl7oyhi
zMctJ(3!nW_RD72apZfhYzEd0rx#X@oaI19hG+BR<$CPLHMALxu1k(x24)LwsY-ZB)
z=ON$m`D|R*%CgMleox`peqcGTLS_f+olFJ(^LN>}ujRht5|hv5kNl*?x6?D7ck5{f
z?$?H!xaY}gTjaI}a;*;&<5&8)+=SiCp6_8_hw<aNFZfn$cjH?v8^9-<KhspECYn#n
zNuKwbb*zxd-miSmPkiSqa^qxM7P*pt-ck=P-W6^94O^EBG$c<pJ;8L(ltHdfAdt(-
zEHU+lz*OgS!9IOa{_As81ygH}2)ukdo%`(`E|aA-x48>@Yy~cRpVbIhqshy?##BI5
z-_%s*o(o@si@uTJ2Wb(7aCU(|Wi$9ip80T^J-x*lD>BXac3HfsoK>iadj1=e+cT5}
zS1Se@1r#k7NDhwWGAju*GF+O%B^?~W2WqF@3CI$VT_eJoz&nY1N$hJr+tZKuZ?SRl
zmZcl>)ToznpL?{{B!0Cdw~*r^0d~b&Q@y8`c;#f)8lTV4<-7CX3h%xS1_3kqH6}*)
z9`J`n2=afvmMuI{Bv?RhLW4lC@DEO|VsqXLT{Ahmzt#$_IDLSZ=gc3|6>pZ9`KL4T
zKK><cHdAH~-|Xw}r7k6}<Grztmv8^%seJZ{*SOy~&NOxZI+;sx$v58J^ZjHv^q+D)
zFjC@cKeNKrbN+Fjt7V#Irt)08+bUyuMJtc<E;M<~`Qyzqb|c4T)45LfP29znm^#fX
zHIbPbz*kpx+$41J0^W|bbzCQ(i<?d6>*Kz=<}$C`qQlCs<|gyW9d_YR+04paz2A##
z`G-lornB~Q&DvAXV-#?kz3t`!Q|=FExR__J;In%_%`9TmO|ELQ31&ZXI=I<uWVm&=
zxbo^sxtN;%?c-jv>o_-S&oQC4i8_2X9#Xtkl4m%=rgHL4T{(%9_ug-Tt?s^jwT6$(
zwtTfTyK+a5uhx*ql*`14{{UyP<e$S_d<&JH^Q|dj;Wyu^#Z_A@Y<e!;oVzn^J74@H
zZ?%Rghj}Jddhj(Y`fI`;UCCFz=aOlS*+$WcS=V^o?icgzEZfBWGi4HY*p)9Pi|$yP
zNts_VF}y2cR(VE<_m^voaiL2yUs~-kt}`7@CS?bhxfm<g^D<0#VDn$V%D=(x1-Bu?
zR<2voTlv1OVdLXCeVWJc8z;|0Ek5px?DNeu%p$lJE*9WdYnWhCb?_!HgK4<Yu}C()
zyr;K$xkXs`jJ|9)iHQ{FdpYSW&)@Seg}!W9%h%+xi%)u47l*1&HXrYrMlQS1DMCT5
zH~79z-)&YtG0=>sshzLcz{QklLLPs_8AWM@-L`z$8BYAG-7Wa%GB|OIFJv}-&tk~E
zbK+{gmPvl9TyMW{eHGfn_x|N76FG~UJl*EsP4Dp>7Bb{@;B#ko;5+e|h3iCP7}tTX
zUraQk?M&6R)J%6@^)@}QrhxylS)Pf^+UtDFuXJ$UNi#CNl>3>h`G^Q#a?WJ|2gh%G
z!SUjpW#Qf26D4MHFNg`?J1X76-Em|KPvEy$u9H@tW>>GNa^J7;;{V~PZ<=B{m%H|=
zkg06{2JSr<A9Fpgk>*o%l{ZaW$j<xQNr=~E@f=}Ct&@D~nzi}#CYiFgYu@7vJ;lSh
zB4)eL^4#0}X}iiypH34uW3}JJpD#AuWX^I{fvX?FWjEY!;Cs{NEwC_?MPTE+E-nVy
z9#hWNW-hCjMf_EF{S{b^7`fMOtl*#XNzLSLh#`;I#F?h%Q@;qO=$Y})Hr&B?oqq~@
zQ@1pyYKDZVnZsn$o{sCr8#Mh)q;I+ksPlX`nX7r0|JL=toN8ND&78gExwgw3;q6GA
zBVr#qliz}IF30hzjhwSR#d%)x^6+<TJi+y~M1^~^Z3xE-wnWqXSFbq|{@&uxX})i?
zCv-FSzYs;!hhlL&hviOlCk1NqpWNeQ+PYMdpV?lDFTcu_f8nhz{_KuL{1xX!xOW;&
z=KBzC!nHm%M0knCGQPL(M9d^UwwMI5?Bumjs4>yV*vIeoyjNC;uaB=hN`gQ6Su_7M
zwk)3KOutNQ1ov}^$h_whdB7lD`nHyP{&7zJ>&JLZ8_ulcZJNVrCSST*Ft^NrPv`Uu
zzSV5u+>0mmbDcdNXu2k}!{nWKqw&phF4N{I&itAQ_GUZV5Anq?wR6uXcx$%Dx{Evd
zKtEr)F_XYq#~gnC?l)Xgf0%ikguJ=U4o~9qu?gVbR_D&Ur{pu|qcvToM;^@PR{k)T
z-+9pl(~j()+_R-6%)GBM^W;|qaKE}2$5;1*(`;(A9nV?yy*!^bu?rf@2=Xuf%gU#l
zv6Z!U_g}uMR8`JpLeatwa~|-k*X%bl|FqWZQ}cTM(luhH?|g*>_AhIe`>*5AclT4E
zK-V;W0qHtrt``<}%vSOkbDikv;J;eUCEsQj$u;$46~E4@lg9OnHF#(34>7ClJj#Fi
zt`Yx+SC{yn%$&$|e7Y59UC1=E26+c_kNx*e=JT6be2TIVSmMEJ*w%iD|B&2T&f7}^
zO#hn{bIPbp;XV2Ez2f0b8~DFEPv)xfw&iL{o5~X+EXaR#`!}u&r)+qvoE_LrLz2zC
z*Xwa@a(c^Oo7ikR_ni+<OUzqCll{Kj#yZouSIlzZujXep`K{#2zjLY>pYWWc{HuP?
z;GZyMFMrwVCXSxCr~EA+t++RR))MYnep|paRNwUe>{(`}YTE^rJ{~f8>7XQdL-3j0
zhPyue@5Q48Zx!<k3b^(2RGIpl?Mi*X!!voN06#aITGPYTd=>(g0!#gEO`pzK$)6y|
zYQ}qDwa~R+CIU)%pZU*izRX$mVlo$RrL9@!;VGt%P5+sEnzg`m(qccsh@%tDk^|2O
z9N^!<Z5W+lzNOTg+vSiBzr#sKnHdLn2(V3R<-Ew8#bd-;CE(^RB@nOQ$amk3i&ro?
znrEiDkeSt0MjqaucLmfRgqWGl_Tkr@u)_4}41NI-v2gx7i3S3@KQx-H6A%(OZ=%mX
z|D2BCrEQA^wk_H$aC0XU*ZJd)yqp0Ioa|3u3O#+jnYZKHQ8SkQeP${fgn8?p`J3D<
z>Ev6V$t1^8uEX2E&5>`ef)n4QFCE<BOKr?vJo(0HzlMn~-#l9BVSWy0N81BlNvjFQ
z@;og(*IQJ~VqQ)Ye*G<&SFUg(@A)6CoEP?$aHa(}nc8uSnQ{1BHEG{{(tKH#3*Wsx
zW+uPY&hhH8t>k217GoMHzMpG}N;dDs`CF8Jm3Z<={xRUVn%m38mRZ4F)X>2DP|1bs
zJckF*w(uNIZRs;+AKR92#h+iur|CGy#BuUPuD|_yCU?3{bNb0LaSE=D;pLfBYU;r>
zox9?jJWu15TY{UtW%v$Or10h&ZRbq(x8l9pbf05+-VK4}N4t4TH_bA?;U;EY;jh4Z
zE?U#PZc!uOW{ahg(;g@AHlG&aTeDlAZ}o!5T&uam&1_zHar*LJ=8Z8}A-C}AO^&-I
z>v*|&bj-~Auk)m;_M3ik_Y=weBEx$+bR(~(%_6ST?}fSEF`JtyY?x!d`R7a1)V!Hy
z&W0?!`-)|ZGB@nzJ-^74EB0KBDdQ0Xt|k2vyxB}l@>|!3@>LZq;yPjG!})w#7k5`m
z6z|>z+FXBlI=L11x^T?RFEURyoW$9DJ(KVCf%#@LR%dcQF)KHo?qkMPxLuTcszW;O
zObbD?;$B7Gi`GxMH=oH9zOjOl&oBEsuju<{Oh??l@*B6Ba#i$w5eRAcA}}Rh&U~T7
zN%MQluLw+;chFo|NkVXn`#;I;zoq#Xm{bT(W#AUP9udUt{`G?CkD72EyYJls&b)TA
zN!L&F%AKzgSQIhSY|Z^`eD8y}%}$902^ADM2;94JnqO_V4_6E4OHSpg0yB?Y8D>vi
zj+u$~beVjgkuE5qa?~U@^qoNCs}mfW?(@uiTh4GQx_0ojt&vk(8?a2^P|j)&Z#g6G
z`dQullYL|bs>8Z?YyYq0G4~VW*^!fG_Iq^?cf$Hd0_O9Z%%^bq^6?!yY8JBhGvB>R
z6TXhA_5%LL&YD^VS_pjbjO5?7c(%y8FVh8-`OXNac`o9VOi<!m$Kl2$KKqhjoM-~y
zf?!{>h`jHn(wlGcawMKHH4~BImuusaVN!DC<9Ya;&wR#BzL5StZnw%n)0yYuxwn0+
z=iBq%Tw$pNA9san1>dWg8;srW$n)x1JvTex%OmW`eUW#fp#b0O3-h>|j<R!SPBu5)
z8LeV`Fld5FMdfnSb5Gd$XV>+ZDsQUhbFJ#(^ggxNY=spQx1ogupZhdPm3N7I_!@fK
zIUDBR;?C@D<*|Jy!na_`EbfOk%Xwx@lje@p*=Y7|O#*i!uNXfYi;LO*awZ<($S5<B
z`7^mW#pStIRWIayl{&*Ly<3r6Lvl4wdh#2=jUf~G%05){DkmRceaSkJ|8w6DuEbN(
z{Gl%00{TrsW?Rjj&5obS5J+^HZ?-(@ltAeVPGNIl8Gel<9zmV{jRHbP<al1pooK>%
z<TsCvii1EM_hfbJZOwcSyCno(6kast-`v1|z3{WqYVTzts}J1g{~y!CUuhQ1_3o1c
z*NaJlCZDecnQc@(X2KX5YU&@xBdBGpX%xSsMPSyodE7@Hx)_@*uHm}4<p$r0Iog6T
zDft3D0Vg>Z2PX08#M=uPB<<qgQRK*X+4mgJ?~Tno8?9|k7aZEfo#WFY5Fqo<H0iSh
z|FeH;Ch11U_+PwG<<~rMg?|E<kLiPo#r(1-&hb4mN*0-)<0WwEr>DT;luWJ;XDPmJ
zg*%*VMymuSU3<=(pVeuye3h7)x}7Pn@5>g`<dQplr%f`YzI<8E`)k7tzOVn9`FwV7
z<d)j4Xj*LR!Zjtpg-_^px#sVSCS0bqVtk=og~lHRuJR}<xS7Vj6cSx%-pqSPV=wQ&
zJ6pJvcvQHe3uYR<xu9pVpy9Jg@7_?8kklH!tHvjdD>7yI1T;5rCi(3(J>kEMYtw{O
z-V5dh?28*$@cE=);9QsM!Nthq$s_f34==-^^<2}lN_c!{2Xa*~w3=zRpW%97b(b$M
zbdd>%SPXY^UY7ClhQpj&{uXgYT5RGqdYx!`+fa<>u#i8G>Y1y;<}yiqx7`o%KK^r;
z!_cpd|J!02u4C)A2p_p!BhXbd&$LWU-7KUuUcmQ#o#BJKvjq$n9+aE)WEY>(LMB1h
zB@+auyj;npB|Fi~UoMqfP*XzSrGSSV;~@dwqf6KYJocS5T5^`1@3x_q+1~nefyF1T
z^4Hvq=VvtB$~jTuCO5a3k!c%yk*T5m2cuaty3L<oJR!h(MBilk+C~9S#swTn-4f>S
zC%@-hm>kY`_-eR9D!0CXl2SXDwOAPUtS`6tl!e#vi|l&E^Ze*zF2BT7?(fYuX89hQ
zI16{y3)tS$GtC#;$9qKVg3;>SG`>lPmh$eFyv3i_*=}mfaEt%ix{rL9_!$L$9a0e3
zcQ938!n$kRzb`)GTlqMGdzoYdUqNOQpCC7*neGf<v#K3uc)d2<GUMr4z|Wm}Q_48w
z4)3o+3jE8D>GEIvUCVQfcY<m0jH%q>AD{7Ay7w!s(RjeM)M7thbIW6st@0mufADFV
za!xuSbpE<5pJ~7hzP;bsxxL#<xw+JuOkS)%Z}w-Fjj4x0o|%r$Exupo#|@p1U*j|T
zHji6p|3ecOIeqS$T3x<5ZvN7LUZ?VFSx(`e$8&`{@&h05XM+m9RWhdBg6dqnFDxq9
zm&_0`ThJ}d9jM&Fe^64}tj0ExXWpA_CStp0@g(kE!KHt$ozKW+j_Hz01>Spnk9qPs
z8U**;|HJp`(;hzFRm|L>YZUpO%zMJca%rK^>w{N#->Br8UCWF#YtA$0Z3(qD_1yM=
z@56m2Io)|myaA1kd=1^re7vcvxL0nmH`Bi;$jw-1#K)ItD0_BRIoD55KE4vRX-2C)
z`S8|zn3zoy4CZCg%;6O-+Q<8IWhNJw<$5k2Q)V;$Nzcq?pQ<sLe45GPZlD|A*$K&p
zsR43)*TY%4&wofWeK6UIt2(ET_s$zWg`X1_@ZC>%!<AFr!abj36;E2qHr^MfS8=hv
z^Wza|zsO#|_}%QLSQD2t_d!0^Rz=gCTYNlbI{%F{FMQ@M*}IjK$NCI!=>&Nb)%CVK
zKH6qH(k^`b>yOy;UDEl%Ymmpo{v!S;Pd?{jcJC%v{%)pq!WEpyO+u%tn19-7!LwJU
z&}<Gv1F!N^4Vjl|`aA~rJ$di#u;V@U@gC>ZA~v(S-V>ajIxM_zZl`L>1TN;d6mXR%
zXy*pwIPSOHx9(3fRdDVW^HY)KS-zy4XR*dq&a^jyoW`BHCJM?7rZy#NW)q)HH08=L
z;+0iCZoK#NU!MK@{WzX3?=ur|7UtNol9wmW@gSSzJTKnRk3TsX-iLB2_)2r<H1_k@
zPIBigYf|Di5?jWh$a~jZ%bb_1M{+K2w@Rm}Otc2Kjg`AG+mB70t^toZEVfkfZ0Hd(
zlRo%^OQx@qd-ZKOVMA6<Ua#r%cow_w;<nnpfbX})cdn)0`-C=aisXA(-ey+xy3fog
z#FY0qAG1kb)Kvb2_p9X8pWf$vsn5$l|En6mN>>+;=KC*ZeN)1@|E&ubixWsy5LY_G
zX?tcL-#yN&M$O?<c-Q<%GdpG;A|TTyz~`dg&i6a~I@dy@dajOpYG&a*1!iVRXN<oJ
zmsn`%Jm7mfh0Qp3)>=Nvm=^9`$&zNg`N7-;S0woMsIn+K2xRl?J@~-2!Cjhr+Ghuz
zw8N=<2d_@#Vl~s{ExB32;awqQE+uh?i~Cn6|8CA;)2hZv+;dk>H$MDiFSmZ@Y)+fs
zjeK+7C78t7PvH?so5Az)hM=JKliz$7u6OgP1ij&Sw<DL&+oPE4D)T+z=p&kZ{s$_|
z7AKuBO};PBcl60DqvhMZ_~&@dkU4c`HgEc@cYKZoH~1Wm+j9k(@R+_gILn=$vz;&J
z8;`UvPbRM@BOBlE&gsVNiY<KK;%ZC}NuA)Ge{Kb@u(=c8@za9b_3QU=M=CU$oGOzv
zm3+V5xWR(m*!!dv?_1fcCXAd-d_KijINLWKG+W0N%IR;pm{()#O4j=yZ}C;c2XXsk
zYH~kgHsD_o_lWl<qap9S$(G!!4=mwUeZgTOcK#mccC|ZvT^H9F?<o=IJI1rjWU0&&
zKJA+ld_0p*^IqK6Zo1*zMc(v++dPMKH}eUX&EzxRx0vsKk05tauQ>0H0w1oTX~x3u
zqFZ?1EonDh&6aBtqbAHdspW#nE&CY0JHef@;+Nui&#jT*bF^pUi-_OF9d><^Y5oT>
zuI7Y8yftN0rHYF4xDG#@#v2)}Zn`>nJI}prbF)=R*93p&Nb%13R><4FV<zW>1+3gj
zX})HEcLtkYwdgWVW)L;~5_gx^!|;&V7lR$VUYtzaM-^1fr*?efvI`33O;<FK)>o0@
zJ7~eg6_@*%EB0tTkMm4t-hf}9xf*Ux<+l9U!ui+vwTWK75C`AMaK1-Dv1WOi%ekE=
zx0&hKrty4tPU24OR^{E#5@@#1^&>a0^*e4!erthydvEeidfm<2HJO3!`*ulzy$W1h
z{8?`V9nYN*i#o8-Y**}lQ<2qw1gxDojcyeM3PyHLkhA*b&!5IVLvVYkfnYDU7`Ny;
zIWv{&I-bPn4FV1Gn3V)CeCItgW43@tSd{UB0ABv}m0f1nd=&+^e9RGenX4qgep80~
zJf{xVql7c24?J1S_OAGCY#}$(T*$Rla9LS^vBG;1!4T2;Tya~&&0>ZBaAXM1=Ti-T
zDN@}0QsDd>e(ux--??sEY~}a3<|#1$@OIvu`e2^h+qHQF`L3Bw&llxh*DfL$YqZwH
z|NK<GJ1HfGq7O9rS*I@H)45nKu<yYn6CSrlfkz*@`9HqDETFXGv_P<+px~bO3Y;dh
zSMYVU^>TBn3JU%9Udtbu++mhB@t~Q1(JcN1w;v`q@3IRlF<mPw@lAnm4WErbOXLgw
z74^%w1bru&-d;P8=ep>0e)%LT{xhcPeE(k?@rxW&Ffnap;@`x7(`44unfyoU`T31Z
zuJf^!OypwPxsJPXr>+T8`v(&{hQ}t4A5Aiyb%aYGLoC)POKl^6m%wAL2Y=6*<`!M$
z^w$jIt86!r;JmqvfAgoyT;E<C=Kg!<J%6?8Gd_U{7kCe<W^m6bUdz2VL(bG>cRE*>
z)^7ekEOv&iAshKD9QlpEyEX93WG?6PdMU}zDrsRVuf)RN@|2g4$2&scb8SDr&a1in
zckM2+&j@ShzqL7*`$(IX@P*bcfjMPe<~deL<{Q`x1gd9FHNCWPkAO4hKDnK@Zt<yD
za|?3(Tr034U4}>GUX@wau4g=6RSp8?UQLSkRc7)j-4qfyppa===2^slwT#P*WplGY
z_q==jMl5ao@xfZ0CpNC&D!K5~^wqvsreAVqm<BxRHS?7JCtx7E%JkdCMuEBoU%6Tx
zjLii<P2pl%^_cIXO1Qk*yd(j!lnWf`r{;5~D@F<kuy5nH6LjUv*K6cS{guSy==Q{P
zeSjYKj_r*C>rx&VuQ@8ozkQ#MY1EVV{EysJ`6K1t@}Dl!H|y&=&2N@*gRjE<zkr^a
zyFfZ?l)%AR5*%VJ^Th9j{opzhe@^rU?<0YqVmhXqzd4ykbRH96RZ1|A>`@Y&W3Qob
z<(NAE=_i?jAB8vss~3LbeyzR2?05KXKK_QY(&s$psJ=5_!Dn@2g-~s5zWHC1nfwK6
zk*2Lyw*>m8S%ByNPi{-&ijtG&ejs+s;&8$gv-ZVeW>@OtEz>V}2yVNbWA6Orfxukz
zE!<_^`sM+p+c_WUvhceXJE{Aw+aVxzUYL8Sl{k-4QISBkn3%w%tUSK%Egn27Gj{Xb
z+Qng79hb`egYCLN#f~_0?N1K;Ggh23bGpeQ&?OPefBL_f!0PVnW;_1s2zUti@&Dbp
zMxax2fk5(aQ^B+U6}eqf-S{?NKFt*oaz!Yle=6U9E;h4}nzyF4x90HXPi!z1owSIb
zV`ZaEjQ%7(Y45*$FZZVNPhUQrha<Ydw8VZXclVLgd`7ES_+#s;xwV)6<YOuPY?AA?
zm^ZE2)~vTZjPI+nCSRTLJibZO1GxRw!nnHiwwa3cE;rfee#OMq>z2tG8$JG~&##y=
zurJ{AOAh9q6F$+5zvDOe@nt=HJoTEYS#Ny#MZWIkywQ4vyLI&&E{&5_d>6wmao4iT
z^1jM$WWRIY+H?hP0e4>QME=c>B+b^8v~#C>o0*A-a`ALeI>z0<Q=L!m9kZEz|6A_L
z&+~aK*k1_p88Pv53vc8-Iej&UlXo+(xVZ#pwBbgfo4>yCE)^Fr&tx<(OU;z!Wq&1S
z)~584kGHy3Mn6l8&*pX*-^#<A`68}5aT|tTHkCY|#jP_zjxX(+r|OTUX6{r$J3e*B
z*QP)3tl>#GxyJ0#IXS)+MpJk#FMr_mUUh);y4g$)&a4#EnASa}b=DH5ZL1nhlJ`&K
zd!Dk*R6qU&@3wwj?wYOJ%vSAj;jS!u#p}G=MkTOv2H)$nGn~BT;@sUOzqyVt-^lA3
zY{9kviU!ZrY5p7s?njw*Z0X^8rgWa~#NRqIe%(rLcV=_*IZ>KCw%ayv7pA)Mdi~Zm
zyScrSJNN!E?&xrLp-kT>KEGH8Uggwqj<TO`_|LbwaG#1@CFCLWN+3vmj=7|RvU%R?
za{?ktC8oPIB?XnkHRSvwnfcGe6bo`Fu?xz5NaZ^5g56AyF^}i$ybggUp+{AZq}<}|
z3n&pN?z>^)|7b1WKi{vWAqtFwhAa*OsydJPm#s48dXpE$?fWy!v~cMa)2Y*En9fu-
zH=SkVD%d!k!_>z7t-uZ$2Cn0i`^~=2xXS6Svw+V)RZeV5_acGV-z{7la=N%Ths@@m
zEiEg+9975r{n`}n#8_4C5Z4|v4IW#r?{gjs$oTM@>?ld%<JNUCUBa)zA1LO`Cp;%u
zAaTB|*^(P>0(P6@`JWx1D!A>*WC5>#cLgq-zQm?I(NN$F`!=qYXU-xH^4fx7mKROT
zqSH((`9uVFXsVjIXXOc&DKqn{)@AamUtBG?^R$N`!|li1V)+clN3YJ{`SNL>fZdNs
zfw#5_eEqGP1^yZPo9Nu<;<xCYV9aw<RZzjCLm;)*N<d|EB!`#2J(q86gh^}jE#uNu
zZqq-3uEw9@<_S*G%rd(9P+jouxhq`Po^hLeX6@(73w;ir|Ks2FRbXbmB4-EhPHxt>
z5Bcq@(*<rXxy-vnbTW@gQWDR}zbB2G*T-|;-=!@0W!qHK^W`h~L}p8wZsE`3pIyJ1
z&t!YIz`<4>6R!SB0j<2b{7wCz1T4)T2s~S+BDiErEaxP9a~|`*`#GX^I11R$E8+1A
z3NY9E@WJ#&EiVtNoQ>&B2|eBqZ+xVy@}+r1S2OT>e|f^QIar13WxkH-f0<pJy_aV5
zxH=lDua}nMT$b0#GqFhBlrJogJHI5`>@8EX!0Yti-0yG8^5|dh<=7$9z)`6<(WLzU
zH&dONQl=dzcbPo8yoskteW~fwwhSI&Azsd=*mb61lkakhe{<pK>~Cb#$U4e%DBPN3
zWA-Y}>k9w5nmUDerp)@p(edsC7vp0?&i1#$<~^}ZoM!$Cyy5GTOq&{FxF#H|HJkmQ
zf$PAUY7X_pZ`@bxWz4c9*|=XtpWxcOhFhR2YZs4KXAsX`mqw13lpNk#8!e6&D^I?Y
zZ$63_zVk7w7B)6}6&b)Q{#D8Ba?BFGuCtBOt0H@OFG*(c6}#v2O+LrO{cQGR)50%P
zxRxrg^M$iHDtFyv=aRqik2hd(kg?}pUf!iS+svBM4-4#(PvgzJvXHm?S1A{JcNgdH
zJzA#XYKCS@_j#M@COVjfDYx)(?4N3)?eUZMjkN-2Nb4ulN6R$0c3z6%?U#5g@Mc#$
zU%@;bPTc@q?&K@8xeNdG@E(l3$G!HjEstRwFE@|!ZZjb_EAIFW)A^b&@S5FvqQ!GK
z`kLvA**%<vn|OJ)?UUi{PA)T3H&fy6(ADF)o^xL?&|HS^7E3y>K%O+mg3Z=^d%p{E
zuU+&*V50+%c$z?#S@LX7^G3-<d;#|SW?Nax`F*#u%KDnF;8ne!%5Qhej{m2g1ecBY
zW7Euv2%g91S@|DsW6=_xxQsVyRRdqM=4um;iA(r8Bi5U8#?BOLZBpesGc}PfMe`Y#
z<I-U6;QQSsTZJZ=hPh~)efm4i^vESP{#`+`CPwpa@a<b6&Gqu`2~+RsYdFq+S<Smy
zd;>ez4q<+ANiOa;yTZ5^Zmi^2DAD0t^wfj*(v*YTd)}|)GIUO`u)M6#b*Dm<e~C}D
z$)jKgzW+}1j3gue@ZFm%&9})QmQN~nx2ZFmB_HcwAzmJzmqNnQ@A-lPU-3;ZGG#yV
zYB_&xtPa<=05!qzDa!>;&Ehwc&-iAlRM01I<LMkT{mnN7c63_{OnxfJzkimh;D>F8
z1@^}zahLpiXSB-i9`_GD8v!k$ISMy;lzD?=Oa<cR`<dqS*z+0I2^(wto-1^tltaMl
z_ALHK|HC*ZYAxdYcZAtQv}A+n`z1?F%uFPWf9gmH&Tr~Ay18btz(b$IT(9Hin>3pS
zacz5>!l!fYil9(^mVoxfK2GHc`P{!G-thI?KH~S!5#VLZy3EbiFq!-L-DxI|HlO3V
z=(R|oW4fbhpw)BU+3!V+uS-<(t(d{WS0%$N@FwlBiSf$U`~{PR_^(~NC6p~5EFkaP
zA@J##0+)lTDX-tmtDOH0W(r=HKg_FZENaI3-^OfDJtMC=!wj=W6L;}ldr%{#H|r?x
zHAQ|t@8HRN2d#E-cgFjea;Q3Thpdef-m<n=OtM^tdxoPbpQdWLS@OYSJUR2t&E)sI
z<*^bk;$=K=n)kvk3$E*YIULK@JDCPDNt)&BoHd=T^27L&RWx6su#Cxppar~cS9fwP
zp3h~r%wre#Y=tYl)3;g5%xy{M+oN5?VLItGSD1Di*Rq8@yw|68aRoQ~@i<&R#6G*?
zgy|xck6d4WEa#i0ci*g9<|WtU@1M*njOK7x%+=%OT(8ZW6(MD6A1}-Ok?9ck{D{8-
zFCw-1UfyEm70X-4k(eOPx8LGA*Sy`yf-MZ9VlzALm}yOzZF=UuBcIg1EYpr-j{K7w
zWF_a%e8KyA*(*MdHUWO|{P!GkTHL0b0q3~2w(RF~o;*{@zw`vpwhf+qA|~%lc3dmw
z>)vwGbnc&}qH%qPc~$tb`2>6Vxj)xW;ojPkWV+gFmf2?SD<=Ns1*VJ7zU5uywaU;T
zteme)Yd2Tot6e7h7Kd||HLl>Tw@>28yzq@L-uN~5)0u0zkIU@loA-;2Ph!$Mp3^hp
zc|QMK%D!>pTr=BRFD?mR9{&GX%BFgYZ}Y0|dSxgnF2eURqM0|Qo{z71$|Dn>&X>HI
zNh^6WZMy_BbyxCD{kWX(L=Q8k!7*pPZkZ;o{*4_%=4t)nizXg1<3B2CHve!3UwQ8}
zQ!VLqe);sLQX6*!^95u(^Piq&!XFl*#$CSM)O1ndW$qhKR`7+Lc%ia&p(wY-!`*xv
zOJYp_N510G_$FbdB{H8sb&fCJxpl#Odal-7TI!QHgREFhb7h3h)^(Sf9!r%o-J6uo
z|51#|r1SQ5zR<-UTo-o7nHHHyaA(x<@M&d9GhKM{i?4agVNUmXwcOuz7jlJe@Z#H}
zx|I99W&qE^6@6U0S6G={7f|KiUgyovBKF$!#99WP{FYSH{9lYbT$}lMewBaZy}QG~
zH0TREZ%KwIukzYDL6#k-_!w>{^UaUyV9(~f$M+-PA_u?lBEdM-+x(ic24)L-9nHRL
zZsPx-BV~5Gj8&lQHjB)(LuGuH$=(8`>MR1`lg+t)-?1}YzB8GNe{~Umd!(GQn4AQ6
zT4x2n|FnlDcN4UD4CI7N#WhPr_FS;wFP*)e?~luB_P%;gj@Q=(P1T(5n!fg9GTl9G
ztI4dRP6B7x_nIvIcZUDftYEI%ikYT6!UeeM#jo=&O?uAM;XISye(p~Wx2eWl>laG#
z1byP-ce^^Dt2M@yTjd!uJJ*!erZ;%saHy@i#qT~b#ALqYHtwcl-ll)QMe$q=ox?3*
zV91|5$IA4CtvJ7bvN#{-vIfC|1zr3-vdj5zZhye_`B)cU+S9dMx(s)P8pKxeeX4n8
zX0)f<^v1t!yvvS-n6j_g&);|Bpp5RfgM7CqOY;A9Z{m+OYU63S`QPOD?={>(_n-5n
zScveSdAN=Hss%THUVMYexqoYTCmqu>ixXSI`+b`&U)c2Zd`Am|xfP#W;?h07-}LI1
z9+MBfOHHO)d^4G{-Hrc^<!#e@lMeBD<#uxyEyy&}DB8g7cWox$vVJqw1$T4!zcpEM
zUF1{c`M)lYTkUKgUvKqQ?&}fqyrqUKIqw)Mngv_BaDT{N#Bcd#vT60}yWBh0nwaef
zZs*C{p}^yjtjot={>qd+)|4mAej88F(lvtiGX(i_ZXf22=7?cyEM?#i{>aP0lG`NI
zR{enAa{d9+jWzLRF*+Ohe@@deOP(euaE4z*Zhu@NpBPh+fG>l9fY|a*u96K`&F-jw
z=RE(souBPas7#sQOfGHxYJQo_t;PYFYP@Dgn@uI-5A(A;FyKGaa*6M8Wiwa5avW!5
z*etX8kJQZFrY4!@y*p~Yx86{Ivxdd^{<KT{ulNqL-+ymw`XhP+hp>GcZ>nFe!pjRA
z_^)S8<+9mW%_YRNlIOar2>+!nDekz5iaair-#NMEtjtgGT;<vn|CYaYg^H=+R6CwN
zp_c}8A3f(f{d*$!%1$4CGh;=Q^Jby^H$I8;P2D9S_=JBZ|E0h^{L>~CvRjwD;QyE2
z!kuQ|Dx~xCw!o{5du9vvdzvpw*d}mh$qX}tRZ4<IOOMFzsf^~oJvCA=MpQs>+pB(_
zvMt<Z-nufp&#Pw%99_9YX~pp!e9dZA0!-;errgep_$?1ho4)mWDm3+wvB1TgH~bU7
zD{<z1*vobMtfrY<%RJME{VHZEt?gz9?ED2yb;ZpdUp_4$wsIqP^2@JgC)RG_E`M&x
zzp5~hZ_WK}0)lleoN_{PJZab31ZJ~J3vA@g=Ij2Z%_~<vheu5Ns44TTt=z5&cLn}`
zT5ig<*OOm<W{jzfr<8!fAus+TuPp?)4|<w4&1M&P!miH$`=E|UHuqwI-zU}x>?v2`
zx+x*TJ6mo6=Sw>dAseIZyh__Qo8^i>HLL%s!n-80&ZP2uJzt9P30ba9Gk6=8JM#st
zw&&A)wwl|kNzp9EQiCh*5(D4wq*|42yPY{N2R!0Uy>`~*j^7lXi7X;!**~rbh{Z<p
zo?JG8cgw*-F0F@ZoN+tcP2+fT&BU2COvPF(%%A4D@wwVOHugxqz<W7KhO4gbm&u(S
zC%HVkU3qyw2g+9+4B$I$_=#iImB*a+XH&ShndkG)TYG`4^szS2@2B%Q8{h9VYtA^w
zRrY@&-;&swravwTb2nC&n}`^5aymqna?S1*<Mq}}H#I$6%6(~)FONZ0pWp*CdA|1e
zI^N@aA2_Q|8u9*$KgStYxkxa0UKj5xpVj6U8qCaA-wfhC(V}57#i)rdRI*d*&dzAw
zEA_&B4*x9p?6_sPuN~WNW~ZXg`Q7X~Z&-StbY4X|$86WNypJ4To1VIOgQsuCFVi2N
z#07JA^YF&`FXjEj)6Vsxdn4y*9e=Y^nttYeqSj`!Bd40FP3qv~UjNj1J424B=@eeB
zgKbMpi_6_PR||;p2Dc^1yDEn9t=PVpE4-(O>(u53Zi|>OUYUFmuAY)@+~zxZ#1~s#
zH}BYC#ksyUmG6e+T{H90Om2?5ye6x^+~GXok-#;NCzY4MJkV^1=~tfe2S>Q~A2JbI
z*}=%S$LSewfNcZIy}uv${T;4w7JDfQS#W+4xGS;5JYiXixtQoxff(Hm3&%_1f}1~o
zmz<x^%&+yYT#)B6m*94>IPT*P4yM+AZaj*qr$i!K?~9%CJI1?gMx}s(mzsIE`fff)
zffh3ppVtDjKiCO~2wvbnJM$3dk5qTgrAIcJwg@gYJD;FqE<XFB@z+_Af=++78=sln
zD{@Zb97mexTr)@0ha5HK*?dR*EyWlATPBcZk-{P3#lUU4sf~Z-XL*6JZ&P`B>a%z*
z&HgQN@#ir!PRn5Kg09B`Z`R&37jq8f3w_{i{-uhEzgjMsFTKx6K+i|Y%)!f7fVVr5
z|Nr$cVar7`1eV>uCLq7AgHuFhAFsQ{P0suFy@LLy)A(i=1(*pq|2Ew&{)o5ejj)-T
z5I_I^r0LTCk4Er`H^1U*);Py^X<ZYyaZbCb=d2stI$YI!MF(!nNG$)tbz^oupa0jN
z#yu;ncx|hM&HwBC5#UXI#+$WLo3F&~CKsQjK9?HHWK%KbOcT`wqNb})v70vhV&Olw
z%E9!ka~)q^cQco*SdQ7JDn{=5PENj$y=-crbzuta3|ucxY~zml>(0$!Bgkj7hmB{(
z2RoiT9Vu=VhrMP?0vx!tEJgSQZyqyqF!bTk;ag|+VDc_*z6I$#GMiO-JJ}bRU0fc_
zePP`h9+s2@K_lzQd|RhI=DqPyk-hVDKmS3#$y~1N&VtS+-2#tZE;CEu-fw!nB}-t{
z!8r3>^G*tM{g^Mh@uM(*&=+n&`NRzZSC$C#*x9Z(VQ7BJb0XSa;Po{I0okjae7o%=
z1f+9(OogvD@rx{FFgY(cSLkNWGyYi%`}xbZe&D?Lcqy0e<<rKdRtaWOeafZ@XG%;g
z7?=e&*v>GF(ETs;ZNe(<^S>HRUK*_93J842*S+Y70Ppo80gZrJT+0iEd2YJ72{3E!
z;y=6Di7!KG3eOX%Mp3;yd$Uz1Te-Kyv<R^8lQ-)qlH>1qa>RII<u(42cc%QkdbjwG
z-<oVHbbd8|PwQnqlU<*MPptM5unr9s(4MEsb<Xf0Z<9Y8*SSOW{Hwy>@H$m8nS@Sh
zG}Fxp;nh0NYsUZK0^gCC84@NBzw-WnF@x`RR}<gDG-jUvxqPOJrVDVzb2;;^Y%SL~
zvp$3~#Y32n@3)Sz;VlMUkDW25Q~FnnOxs<|>r{1`H<U}1yUwwU^X7to#>MRECj2=P
zrqf#UO{Ox|@@1y2Hu`y3f$v<QB3IDr-KLAOCUaJ=@aA3K-OuJ2vXU?PZzt!CUq?9w
zUG{Kq{;-vIk8LEk%*_a%Z7sT7y}vokq~v+Iw71;jE3{i`D!s3oyD7WP*efQ4vrfvI
zJ6O|?S21v==^IZgo{-Egp0W!SA~qW{`L<+S;x*J+#vy9a$*(m>hkL5%CZUxxYXqz`
z)|gpclQrKb8Y^&3WQOU|*x3SF`}E`%DDL6gJef)GCDSB<7ZFc6x%gDg8u_GnZu&|J
ztP;N=E>X|JySI-+fRRznWX_`>yo*~|On)rN5U729f&U>>Fn?I)Q_keO(cIhS>X|<L
zyu$RLUxVpgX@2vHwp#+rPtGzr>(U_bNpKq{=Q&fe>povN6-3JT`ZC#N_4aEEoM`Cg
z+G%;7JLu$fK4pnD{Bu{Y;Bkrn#{Fkw6Ib6f0khA$Y`Kga>jZw~NEruJ?BR_QP&HNy
z&*t0O;>K&A^NK&dwb#^U$3gygS$Fs*Y`8Db@m*fP=4q6G(Z(a(n*W~h-B-`y+Adql
zXQ|o1*IvKHbo(b}vnTZ(ylnH9noGIO=0A3yLn?XS1KxA4^8EQbb@^Axtmbihnq#W}
zemi$Z^$R}r8|9LB)Y`bRUYz2qDq}MFp7@Yg-Eq4~$efwHKR0pl6&@|)yISte9UaBa
z9q7?&Vsdu7+51v|Q<g_fX8PCp`L}M~ZJ4-)Tlj@V1^0nvQ%(M#y33UvqRf{+M^NUc
zbuxc(U=nvv$Wd<XUl({XPd4$biMqh`<%ccrRvtysjXN{VY897pzxdh7KXZMr*~KDJ
zo)3D9O;``z=C0T!&*9tN%vZ8xrs?_OsXSThZ}4o5I3^T+;TPY&!g{{Fl8?9~Q#JW=
z{#@l)UhqLs=JzGu6Bis!ucV(g&3>ZGyM1SpnbOo7d?wG2%YIXI;Jy2{fsd!WiO=K_
z1JAk5A5DEu3v)l|G~~NFd$;7mGqbr~T1xRf*|NjXY^obC!-H<qWe0xoG~A8hy;QiG
zck;EFT+>vhb1``SF%5HKGUpMPYvTH}-aJ{oknc|WWP_OxCHQ9kwc*;zC1F~yR+ekC
zZ!_<Sj0v*;rqAcAWnIUmQ53|jE!W4RW_*y>oo5Bt3(W?eHGB-*y0g!leO}tY9rWS|
zpPYECsrKgW+_yADjLv8u<Z9m##x5(fn>TZ_iLt-@L+%xcHawCYi+S(y*zpx!U&Q<U
zDlhxIRmXXDKYGrQHP?~vx$#1t#gAv1XcjLppPtFVv)T5B*{9kX-jCrMq&#!_cs@n=
z@<#r);JuwC!zE|QZT4w%BPXvI1Mi&#0nIlZlAI5iukZxK))}8(a-G{fxXHA4&UCQ^
zEjFGwohF`I&aWJX@0~aU9!HuAFB33jTIOul^5KMO)_e=z{8E17+dseXEXhw}3)?)^
z?B(UT>@U76^2i6f^A*N<^R{tTaw<Jtz$I?0&i#GKG@e;G0$dypmfSp(xwx8Mtu{B2
zZ01^!vxxTwuf5r>z2CUh7YG{{ukPW{d$pHSCB=p(_JpL_V-W`KC)@J5TYqqgu)pHq
zohcE(GyCBN?t=ld`J789aD89PDfERkg3o@tq}dG%DRb7-vb={@Y8fwn-_M^`H%)Hy
zdttshJ|6zWP<8$#Q;zd&OFM4%CgB5@&4e(q$7k+IFW&!vv!v)K-_vEUjf?h9<K>sO
zFx$?q#5*CKi;wY3HlMFi8MlbyOs*4tDrO=ay=GjIpN!8w*07kj=O>@nmlh-Dhb#GN
z_N8!Z?b0%{xW>tS>^e7JN9G!Z<#w6;PP!YozH!~>GPrBX<NQ38&qm=Pms_k0FH^)^
zPB!ghW;0y)xEY0d_-meXn_T#*$i42*VI%eOot*O{xj5^lCiC5Dnq%@}o+Zz%)J;5r
z`4)VJdw=pNZ*$}O@a{TC$k}GTP5X?vz3#sdQZLZvi>eAT&zJQy>z*sfS0>qRYJI|m
zf3D_PN#1V_yc^3t@<my^<J;9dhdp}g4U;J$0X!=r_Vb-f6IRtt&fslP;^Gqyn{Co^
zqMGmA5d%~I4d3~s%ct`CF4yKOS|`EX{%k#WP>iH$NsPTIuLzH+$lt@p9#?qz%-t57
ztmcp5`?_3@%g9y2WKI5b&i%8N@`l-OV~@GCk8hh^I5*GX1>6&5O!;qU+~YlK%Er4X
z>K8XhmK&F@Wun<-Lt`#R%d32{cUBp<eHQ22?)%7u_2xQ0n}(CTTeM&BF6&Y-Wq#4l
z>*&3j$G&r(V4cDwKKZrHe2kSG+^b%x^Uj~Qj?3`hX`ye`O}t^M>&<xQRhueYe9m+1
zNSmo>bu?d1r>tzJxfk!#tKxiTj<fOIVV=gVd#Be_#P%9ja?BClCC%-;9F2Nha*@k;
zOFibBrq5f$b39VVEHL#M&p!b{-alWhc-KE?=i)H5<LdEWZMwa~(==2y!DRO9O(v-y
zB=}abN0=qNTgMxl?ZN$GzO1?6@*`YNZwB!4={}JG_1S$x?sCeSm~oqZH{|)tXUBVT
zy$;u^lXtj1EEjN^s%w~<3h{D0m>9;#^<a_dlj=V17M`7E73=wUgv^q;)KwICJq!5E
zqW>6kU%R`RJHD+);Nq4Wyk?o!ywCckvr9~q6!<xBAD74DBLX4Mo5go8*lSwSwa`ql
z{*OTM{R-2AyMhFDrM2ZOn@spipG+6@eQPLKyOV?a_M~5?yUwQaXdK%hu%l|XqTNa+
zzJD>Z1<pR1Y`l6Z4?kbvN7J7Xyn@LAc>+w{8Ul<5Sh!bg6XFh<v&ghCJH)KIQrPtI
zTrG1s`Fz33PB}wO9Z^B|Diy9KFE6vpPo8r4tenR;MQb~|^SoyQ>nyps#i#jm&uH4m
zU%kLffWdDQZ(ySo4}V(|_XLL}W|spFa24zo5oAkTWxQT@8sG9yj|>+J+3<hmnasCS
zyG6jYi_>&cdY8cTA6@)W*B%JuS)UZR873n5XX;H3!>!x+K73~9_Ei%Xn8LM=|L~D#
zW@?p(&H5W=@NYiHVrF=UU0~}uZ<#-=LVPxAHUeu}-totLDdswJMa9&04+HPEH#7L>
z$fhef`>F7;>@w%~UU|r5()k~JQ^L%R87}n;Ki$s5|MTo2zURIbT<^5!a5IE<88fLU
znliNTn(qI<)O6E4X@Ot&*-cL0SkJHHwTNqC-ei*(zvgrCa>np=|J=eRtG0-Loyaq;
z$BU+NZ+!TLKePV@-_ARWdHE))@bLX*<Kd1qF>|k%;Z~B_!yj5-Wi0Y@9iQ^CNycn?
zHT?fiwD8s5QR26YayRXn`;xCAot5wG`xfEM!hZgIrb+yZ?^JQHKkDGW7Gul3$e>d&
zXjYfNb9n&^mGAS-J*)}@9E6ve?KIyj;Ho`cR$<9gK1l~|LAHSP0<$On<9=VMW~Nd1
zfhRfML7-oKrYgVbY(C{R!U9K+&oy0Dn8`1Hc8zJC%sgS)>IeM7@0<B=b{BAMRu1J}
zyz{*2HCsNjcO73$oxNktnl>>CwppJsJ#)TMfOnxb_ky{rP2~?ua8F<HhmX_qrtCtA
zc!BJ$cuwm%5!})B$pU9SZsFhk#gMP!Y$;EGIVaCvpLp}w8O+>&S2haFF~4B)XPN}R
zWM8XkWAqn((+d*(hJU{Dd+rc5n{Kv`U+L8)zN?LUg*mNU1?Gi@3Ai1(%u)YWPT0$K
zD)&CAL!yU{KNJw0U2mHCjMZ!=$8iCsy4|L-uF8S}>x>kh-4*4}tH~4;Y2Xx$E<DC<
z8!2Pf+_sm`^YR7hH)gX{Q+s#t*@|WeJltq!UbJ@_|NDEYrctf81PoKn1<plUOW3IF
z<k~7bgZsACWs3{Wy=JRY(@niBqAjgroCO=MHkdc4JQSE(wTD~Sr^DRk*&43<yuAF}
z<#y`K%eM+RG23$6{kX}^KC4B*W|pWxa(xWn!mE)yLdV>At`|QtdHdgxJ8sT3fiF+e
z&C+5W_{%rtnOUtC5Ks;D<Tvqk6sXHTZaU$MkpS~UFaA5s4+SoKnJ2KX*+Ou?oB}uN
zDrvr~hE}eB_C`W(^QZIuV4iOF&Pvtn8pC4VRdVuXSET0h9~3K)KDlTu-)TPv{x2p;
z{O%zWd8WB_n7Wtg^W2+!lJDNl!}47%Z@E`*|HF6XbdTw$lJ&eNxAd54e|Rap?|?Pm
zlm7L5!4ea=f7X2C+!4RW^t8brlUm*|)2zo@CNFcf`LCJPn<}#};CtaWf%|^HgPFA8
zAMS^*+xTQ<SXJxm1NbYZ%;7q8Yd1GX#e1$J*@b*F)NXNq?Apbn=)Q%$HQUUrX|X-G
zvE@X5IoY{phKt{Dmq=bU^WN#g^Xz;dPq^k+Ug7M|rk4)y;8qFT#B+VeJfYRQnfd!U
z+4xFcKjAoizm)e`<yH<htxTb~*WY=k-_bH(7%ppOe_oaM;l}x9oo}A<afd0$a6S;=
zt85SDo4RKsU$k{3_oLGnOvTfdaBu6C<+GC#Rm+TP;GS7v$2Vv93^QAo$2>=c7nwP|
zu@d!rI+s`d;d@@j;A@-=jr%x`OmH+^^|Z)r(ZNR3D=Rxq<CJ>&++NQ$6-;`;o6DBY
z{fuj_*;=!B?iHQ)c{?^uR4(J4!*`?Akjo|MCfAJiZCqCvSMwez_T}<;^PT(r%?8%h
zy_e0R5-)IhygSRsV|d8yG*ct@s^V~Sj;uG_n=~(R2X7JK75*n?=3LXj9sc(Pce2d}
zAx*<bzKcpxyuS-XI5IOo@fWr`a0U9E6e#w4C16_q+<dEKlzG(ra{|vw&CGsIloXWG
zTOq?Y^#z|ncd_868g{|7LPxH^nYT@)C1>&2ZS4?vU+<&N?|Pm$E3aH2^u-<1<c~}F
zzRdV&D$n&=bk}oRfqio>^Y^Wh;c_ka<F+t+Vv=`P(X6)TvuV^6FEgi}K*5Bq-KLxS
z-Ux6ePT*vDbkC$-AdkylU@zZqv26Cp3kwA{mQ`}q-gM^X_L$9I?I$a+z{G?1Q+*WA
z=H%tvZvHvuKXW*_Ly8^<+)MQ|<@1f;>)s@4>c2pO|BRj#-yGF=fzk&uX6rXv34C}T
z#LrtRCsHdlS%B&H1%VxVb2!xUE{UoYdvj^Ma1&X0M@w+Q&LR`<lLt&bhl>bq+&952
z`$w*zVyT?)#9LAP9Jf~sF7x*g{4@6mxBHw{<0OMPo*bWj0{T~OiZ5c6;5%EsSwLLk
zfyvUPT>MAx-7=0_rYgw0r%Pb|IXi(Z_Io&eXC2_0d_&Oq#svqH+yB)}Yoa=gp0mvr
z6j`vvFpXD3P`<*4`@OiQajw%6F7G33{Gaxwh~-TAB5-CS50{d_H15}8_xRn|QU$p7
zT;WyMb>dlO`jY#6>I~y=Hj>;M<|_-b27fSh+rFOf?-~cwP?v0esr)^BC%7gEsA)|z
z>2t0USUGtvf5f7f0*jB|6>uuj6x>$l%IVfE!y_b-!m(3o0ssCq5#eT|F!ODR3}&KB
zC3!CXy=fYdtH_(V=fBi0gMZx8KNxuXS3Kl-d3iF|*Vx6TYwXQA*~6yu_#BbZC_HnH
zBQI()kMF!_vsK^ca`RfQG<!QUOf*EDokvaa9k;Ec0B5e_Zw|TgYbGn)GtK5MIBROl
zYGG!7{W{OP^-DnWOgy3YY&p57Y%=}yeivuZBz+z^?~^K;yRP!gpLv^|ak2{MKCxF^
zm4)m)XKp8P?oYeR<({U)x!|mi`N4O`Ib9|w@t(OEXtv9Ug=<%Ws@bch+#KdE?i~ES
zPq+<})XXFv@8DW-^A%U}MRviqw!J(vouzmJL$`BOR2uMdw!GmC)o|wPd2&QlD*l+M
z_2XrxKHa{&3D35eh3GHlYqVvPzJ9Kiw`6BJ-@MD2d@DMpaBWUlU~2lPhfDPm7hixF
zpK7>JH`n<~zj#%DNttZRk>OR!nqrn-vRL@W`XJum$vb)P1u1cTWm(3a%Ass}|8$Gl
zLgNC{LyuL=UI@AIZ4;Peyx8Lx@2=FH9Qy(~Os_UC<&1C>;dK<6sJvok0$)?%ZO*-k
zrQGRPVz?)EPT_5R`kecJ%6;y8YeKkZxok80buNT^j@&f9B=!|%L1#jEwyVE2-CVkg
z`;SdCkKUo{Jaf;MnE7*m;dXxD!XxlRUP$AFJm02|D|iilKW1M#(U<RX?R3t=j4XU1
zfoJ(H{>wA7dKzwiO?@U`!qLxW+3QOAXRw}>R7+aK8x)bwpYX(qf42e;*LLBXrfPA`
zJdC1j{Ez!Av@Yn)<9)HXl`r(8qG{f>8GKpU2BsNChs3nBMfsl9x$t$^KjxAZap8XY
zeuv58$qZ%{DRySA4(6tpbXEC<Lo<x;F<s>o@oM0dWcg@%^W7tk15-Bg^0=;Jf4WV8
zKWQB&_o80`+*u}7{L}mm`7)&yc+as1^KAK;!)>rK*WzsTTP~-~V*Ihjt4v?5H02AI
zEH>Jj{FKk*)DPauvx$76d+wSBdyDhAWpMGHo!2R%cIF-5kD7~ov3BYl6}Oi03xu<C
zxoNKvN()~uAoo?@?Dj-v(+@lP1n&7InE&j)F2Hy-N63(ag@0y+s^H1M!vfLoS-EFu
zTAEx?UC(VMU?ae`;sSruA|2i{A58>WeIA;GJ#ps?uk<!4W>_k)KZIML;>{fXAI^oG
zOfy)x`eYJ}j(FZQIXXqn^qas=BVB$m!F~Hr={@dPETDcxoV!~-#biTzJXft+GhZLW
z1I9MXECF_o0<Os5W^Tv-ulb_(+~*hJWanj>zJlB6UIMr5wIilhUAwrZ+bj}bUBGBo
zbNLN#NcJO>X-|6iZWXZeX?bu8*r@uMie|j$Z|oE1Us1#^_`fJfV0}!V0M}+6F69(0
z-n{3UoEv6*5L_XBgqMH0ooV$S856xDoV=XR<;-<cxA4_Ryq1cZbb|NSG66pOuM_yh
z&F*n?vVJz1`>2z<(EBa#hx2?gnfuMTlRm5Qd0PjXrImc)v3mc^^xQ>v!Ntd0d8Z~H
z;8m5K#Kn14gwsti*+lZLt(j3}i0QJE+{R00gzz<cO*DG-dOokkBSCI85l*w2Q+9H%
zKXZ_G^*ep(0NZ@NC!1DsPG8Z-HKCTDtAA}9Z>+K@*D@w?o{K-X2)Qbkn_ewY=RWjk
zDPQC44Q8t@KIc;5IAkWdq?`M$Kry#lKO^tod#<K#k;k~+ioWJP;Kj&ap{L8Yz*U#`
z_ES!dG#?#4t$Xe42d|e2DQ^}MYm+cBi#ere5~yv&7bT%*<}P8+f1{I6@<iKy-X}j_
z^X2M`^LJWjaq2(GH`U=*;z?FG$hYk19HpS^TX>#Cg!4uAbC`S%N#^VK&Np3o!%EcY
z;c8xo=^lK0=E`yJTxZT*5SD7n8`)u&_wI$sms5JCx@HM{7Paw)E_<^0lIGTPUjBRB
z<YBW2Cum+I@3oS|$<WVyTNz()FV20!?KFKo-`hxGzLK3mJjyaxd9)IkxR)G1Znj>f
zkh^L+FaL-1yQUfIPw~1uoMzDQ;yZ7Csv__1Tyeg-*eFw8u{FFQVY7IGZ=4l)slA%-
zTY4v7=TZgEx+-QqzVGWf6_qy&X}|m{)~v;1ma%$=iSEHJzWBKoX8+Qo_!pK$O0}=m
z<~yF}#DCO8m;YbU1+Lk;Dy9x|=W{<zUCFmEt53CGu9<tg-hRFbv$ae$Q~7vh8d%KO
z65a?`^o8<8y$R(@$l~U@bTo;b;XjYbw&vfa3J;A<E3OBcB!)Zi7rgjn)HdNd-xK#q
zT#3pJrgx0jb9IP3<K0@A&E&ZCFW=Po^<0~uv~q8WddB(R&697vYzp_BW8b(}H*$05
z|Ftz^ITXQtl*f&q>!*TQrl=y%%kU>AGnw~sGqp7E?4N&*cY0c{X-QEyuSt|HZ~Kz#
z{B`W-_?8+i;tQ#7W51wspKsRD#cUH~Kkz>Yy3OykWSQBj_H|}c{WtO76z?!w+`=kQ
zJ=so@K_`@twbEO_){#Y^;<PZ=9}8R41%C>;Hk>Nr4_mxOb&Awmu1SWK{1q>{OugQj
z^XONom@?{o5IJz#f`8lR&3uP{G_dPr@^Y*aK5lC4@ynDs-@)|0?KTtlrOpE9pYWRA
zQa;Py`ZS&EO<S1hmwHXEg-joKLFa#bJ}{I2Z8k6Gx(5tgyElvTMEvLCw+jj6YJW48
zn=QwJbBR%^*~vBQIfY-`;y0FZG(8%%l6&v5$)<Z}tMRng)^P9EvEb(ri81Bv66C+u
zDa^ObxJuX{tBZe0!eaiIKR38eat86`IOlK)pEeSjQn8xvuGcHGG%qGIQR_XtTh6UB
z^|su}FB-5=I#KuzpPZXCKTmZHzm8cF&$X>)rot6LJQey+_{450t8Blj$fF#}!yh8C
z%;e05UA(J33d~fzgarR@_vbU1af&Zdww3!9hdq~u^EFe=CIwTEcv;i>Tqe_f$DR03
z<ZU%o-+qWse)2(X&+4P5n+?x!Pgb14H;H47YIJ@9|GAhGTrTenc%I1WaQo$T@g*%}
z=ZOf~#KWKcigWpv?`GbIesg^cSi~<@r*4+h=*hF%ti$Y+-a;N(!zVn#I)8YtzWHVv
zGE0VM7WW~ZU&q%8EiVw_FL@@)S0b^3wYB6c-^aiW9E$E+1TPpo;BUI9Wcp2gotfgB
z&HPU#H<+~u2nz7O2$A*gjo`Ck3=*hW%`b58p&M6Hmznv6G;YpEDsqBR`7-hbReL!_
zva9&}o-~-uF4g2cE_c`1=j##vLVg2&i>>GQPXBP>(#qV#u2}ZM?CtgcX6tMZn{Lsb
zZ=M=xDxmWIpvn2Om-zRs-^_76a)U|onq2laty#SL*))_6TW#durR)eA-{-o`X2>(2
zNra!h#ftl30V|KGv<TOO!s})qFV5w%&v?s!VosPzX_zxlWW*f9bF;F!Tyw5)UvCcO
z*W4~;GBYZYzvr41pXv5Y5yR6n_{Br^@h|;!n`3wOYyRlXU%5m${|c~~+!5F|r^npl
z{W^2cHQNO0x|GeM<dp??%U+RQ-XFvNZf&Gs)>(eRuh;u|=JVV#&FK2iV}Esq0Mk_s
zm5Fz^^4+~wE$~fji-~0MJbuNCYfV4DZWT_<G!t;`d&u9`QpD96yPR83>#&({$4t}T
z_sYydcPN_8&<+ti$@|<?JM*+a?hjU;?T7A~-pu6XiMk!ce^kLmAb81k0kOv^oTV<$
zxWDCh3%Ku*5ZFB_j;}*oh4(CT7SGT1JmxQa_Hh^dxFaC)xZP|PmkYl~Wxwf>Kz@M<
z-gf*V>#YUq^6ktVlYa0gx2W(ZF0vI}vTvb)^n^tMX6d&%S>J5uVPd<(>3u|0Q2Y4~
z-c>P;X2L5K%%Znj@&*d5FulB|obS$ua@osY=J8f<b>?HRwc=aBaE3d+h{24NuaNV0
zBQxJ_Q$;nkQbP`t-iN#|*}F|9-P^%4t>&$1<dh#Ga+&tL4@~CszA9YK_263-TTp?5
z=?jz1W)J;iO($GgU{>(ifNybExKV8KAwe!RO)l0hF4NUdH*;2A;pYvJ<d#+74CPxG
z@6TDbT!70|VmarM-8H;3d4;$)^|0_{Uw08vQ2uK6e`P;c>)H8y!t5Mohua*vpUj_S
z+#D{#`QkwWw@32?9*61qru)_8c^ds&c+9fA1k=>i_#^}N@;=Qw###H*k=G^RB)j>g
z&qDV3UA&VLWX%O%IGG+j+rsNtCu-hwzL9TdSG~-t`4POXZ-n_6gDm-E7D{rvX?2<#
zeci=*y+Bjw_tH7?GnKPAW+?{>u6oa9)|+~Y=l<Czrl$6Xc-S;P^4#K`z#AbD!}V$H
z8IF{Xm(6mI&NmCQTWaQ#m|||ou!C31A;`3aaXoK<^eoPY&q7R}o-^fm=qJE?BW0@6
z*FQmgFH{@3Sn`)}T}brdj+9U3wVk++^K~5;&+Z$Q9GX2`<`22$xKbO__{vsCn}6JB
z%zcKV$3*^G2Iuv2%Q!a$7V<7#Ze+em-~x}y{_Wf~-`5BvRx|KjoV1HqbM7p*?+NyT
z=LDy56|YSbb`AY3uyETZv&CN+&19Rd2rPX%!D5w!gdl_KBI!~E2L5l|<$`9h+=BA*
z>fAl0#%A(jPCVWVx&^ih?3Imra++5rszRXXk(BvOtsQ(zLQ~EBZ@(7U5o{+AuPG+E
zG`@xNmd<m|Ti!M%D}&q37!!@mmmFPb{OC=fphnU!qxwG|1d6K`a45C~n0<RE$#H!_
z3E%9QFC|vgFB1q|U&FCp*q!_Nv`+rMT?zup+>?2WjtTNynaIrJcHy1rgA`uwxmTYE
zY`S4(!Sp+V&!;WHT(ei4KV?QR-%{^Dfn>iF^XV~B0;fME@^i7R6KK3LL*PNqZGjuJ
zCvx7Z%;Z(M`G-@kq(jhgQVQSJIv2BDkN2A@|9sBN5n5}uX%Q#?$9f^@!!nwD^;e$r
zwLdt>7yDkA+pBxK>4vf{Zt=K!zNFu!svM^yxuw$b_?%SMnD)&{<}LlGVP05ZCvxKd
z8{XgTrhIF@F6ZKNXyCYT>4xdc#g(SZ)Yh98h)y#J6xhzkSiR3=;p$pG{Wa6MqBiK6
zF>OrYQkFc(`{>kGHK{o#_+C9d!gXH$Ik(EqI!?C^F}`Zs58R%|4{<NCna_Rr{Y|sz
z#v<;sdHnn)a{A_*3fg$OWfaVdU3YSyynUTVh2<xYZpVDHf9G6y!YtW&)kNb2`y8k9
zP405!b2&VnL#LycU(AG&JHL}xc*lz_0qz|pW@=%TCJE9x0+n+nn@4A#5ZJhImE^Ls
zfBDX}aSJjY+aTbwhmYrNYO(1_&b>UW3626)d>pb@6#My(EEN}!GF@jPF44k&(Q>v)
zifW@k^@QjArfC!TuVr;}&7JJbeP#6q<NqrYOn=tTG}UH^HocYdM?fV^-KbT>M)>@`
zi`>?ZnkLqwEZlZIFZi0D-;=K2S0K=}kCVGm;uJUI4S#|7<X!x?%kB6ogra#$XBqJ9
zeHms}w=#xXBD_UFka@q^*O&7A^$YS$_?sT{Z!tFFH&naBFSIt=tR;O1|GWt|`3#n;
z3&k(+6mUG9BoH=TfNRGsE8YZ;V6M=83k8;~d&m3l6rZU_^Ln$F8n(Rm<owNYZeQff
zTj(fZvY(ePWX}vf;mr+vW~s-xS&QUMTR(SjX?eKu-CO6WI`?%E=UFEqzPSNMjV&cO
zcy*@FFl7t9BDkq4f>*i!CGXv}*SM5F{$%gc5;YN;cHFpRZ@cLO(K3?{N}ha){%l76
zmdbp$CUJ1B$^2lNbX%I;dy)okx@fCPMBGZgK&y1lq;_WRxE*@jSKlAvbrPS({k!Wq
z_a3`vTpWF~O{aR@;mp5$o3HuhYtxlSR&!s;>@|_T-^#W2$0hEM6Zv?jx!0Md-#pLr
z-fT9{HYG-ppT#+R3)V35O<&){A^E$PU)PzFGid4;f$yQU0v&Pu=AS$B%-N?!3*1^b
z%Zz2gY=Oq<A7v8ftmT`!nMqJ5VWNQ0s*Ri{>W`XkGPmI2la~~DW@@goc<whIq3c`%
zsd|%4q|UwOy-;_>sIKa=;HI#1{JVUt`Ca#{=A3^)ftz8TzUhWtTxOS6TsMtAbi-`w
z=eGiFj5;QhrRoKmS8{M($rLtS8sg67sXv)-^1`3uiF0%XO5MG=L?&(F=HNKVw|(;(
z{=3f;cnm|<@c2bB@f=&U!tAGJ4LAGxIsr$?$tDiM3waq{?Jzdf(&dYo%+1U7^&|hc
z#p_HDzF*ECC3v2XD~mzIutq^(I=`d9fw?=l>ymErv6Q{zRJ_D2P;<9|@8jc%rmw%w
zGVMIy#A`HVw)rmkY5b`_-$}LU*z)cFC&$0!oHoDy2PWQa@79?LUuNeP&7CQr*gHdR
z;-52ImU}PqwJpgop7rGx?~LayCQG+^@FvM|@*Oj%;hVB!8Ml110r%>cT&5+d=S`n%
zvp3ZWd}j7eQG(yxlgH$>BZI(-Kx6Knp#R3-gpPCluvg@>c0R1IggJ?S=Hnw=D|&Bp
zcb(hMqvO-Sx83g$_q}>I-hww%xgxiIGA&GA#(l1(m;cF%ZnGB~cW|>z{bJ%~;K^xN
z7s|DwFo18}?2V?)!HqnNj(+30vH!Wyq`yD;z8^Q{Ta$m5OXw;qU&TIF&OGl{!CPY2
zc<1hSFmYP{!PK-Wowtegq1n9I=lPVif@Hte%;7y8-^llfy`IlT&5h@MoSoUz9T8kX
z`?dIT_HoKBIpe@}rCW-R_v<wy|5?7gdoK%{yh}OEv-Ecoucz-(-YD;{Tnp=?x!R|w
zngvX|Yc?fV&QycL&HS=ZJzrpFlCjN6Nxnz3ytuAUn`4rc!_6i4zJ_;Whm?YK#sa>Q
zS64axYdyGIljV8jv-j}w3D4$k3CrNQ#nZ;cx3<en?~5(>Pu3HB)(4qPrN7PNz7zM$
zuq|&Lr}nSCT*_CPcvmgyGoCK`kNZJFJWtQ5t%6Tl?fKe7Ch~TQPh>ZfUe5zMyT{tb
zfZy)XLY`LhRx@q$1Lj=Yn0W##?aeO>*781IzD<%R{2`B$zCSPHH%(sej=h|9+y0n-
zou0*cBj6*?)%sSA$Q6ItO2y9eRAtXL(Uv&O{VrA8>~i8+(NkL_cwX?$;@O>=#?fOF
z#{M?rtI5?l7ffF6S!<SiQ_D2o&zRR!ti*WvvM)R@*(^AWiw>HZr%mAa?4!f;vQA!j
zeZ4nt)O1$P$)8tp<)2~Xa-QAG<G-|&%Q}LY`^;l2uH|oA%?<7va9Lkl##?ylq8X>E
zA@^BUN7M2gL9X<SpIi^$e&S~9&^CMK$<D3PmBX#a^-EZ#lau$3RTNK<_f&4N%Qbw7
zU)OU@j29HN^@-v;@LIx*FZYYt&3W3q6Z~sT%{F!OZ)eewTk4$7$GVuCU-hX5|Iy!!
zyvYwHm=zQ(<~n;PTukE53n~9D0j`xD$M`b8<(tfPnajIo`EQf$_r-X7?)~H4-`c_F
zUVMT3hh`JkGh1sjiB}uV1T~mVMK8vg?>qR1PyAPt(ay`O`1C&NazD85XPUGxnaeVh
zl`nj*vHa((bpCHsB)Ryk7`bPh{=wZHoyw<hX&d+QE_L2gx5XTX66c!bOLcHnKk4QF
zlkRJL(oThY=j0_u>)X3HFXu>c&#sl{`}*Xu$^D!8JehhIc=GP~3o=Cg=G(wx!S}i|
zh(lO#5})VGG|n%R%>?Hz*5V7~%Qyegy2ecI-b>z;Q(C5JGB*5185QCpl0m%Lt3L2;
zIrx#!I#-@kaN-Y>t7#oP=?;7NLUm`W{XQ4Uo7*nKw`1=-(?2J3`ILHFOmDN9ivD-(
z;uY;-=WCd{hl{PQpL^%AP?Nmw9j2$t?M*kCaGF&etK^fg%r><;>dkkZ+kjKU;e)Yg
z(_>D9_KCbZz9n<4n7)V4FDjn<iRo@`hJ||kpRc{<H7q~O!x}P!hwsNMZrf?H=3)$y
z++2A#`OfkEHYw=j=G&xt(d65SKE90ooxJ+B?|20b(@ahIHF-BLna7hK@l<%>`YC+o
zKOFdC%f4{!SJdTwcKa0PCN^Us5!ohQM-3D6p8L#ZI^WOod<!==vpp8dXKeLM+A%VO
zw<TGOuf~m$Z~Em^+&oK{o4!t4!1d+*9^NOrwuwDeSLfn6u$?!0<ss9S>LWaCkA2Oa
zUoR8*k}Jx)DNdSKe|-zrx`m6m9RBc_O;+4)`h$bbG%qyB<laYFzU-OoW^=jL@WvTx
za97k%F_UVV$Mx0Goj2U?lEU<2X}(Vx^Ek^sGjZ!?nDYEwYR#ME-^5ja?*(@b{|T;R
zKPH<+?fb^*em;ut%n>28=UR%~vS-=M8^7J=`jPdO^N$M;uk-S^rXnGSxlU%D;^y6w
zFVwsCDsR%gV?0~FB(ZC%$Oxn`&f=P7W6SSx=(UjgY;V&AUuDcAh5iUQp1EcA=zOps
z`@K~%Pn$LPU%sC%IB~tPAUF4EF1eV)rfXdn@I-#wAaH!&Y0>#1tb8>^a|HPPHBFQP
z+4%K0>^D(o^%XqlnIo{}x0*m=$4#yoUqrZ5?kqP=a-M3oU@424mA`~}Ie(ep+jkR;
zRtty<{{8og)2VHtX>0sj4kooNeE##fIHcL12>jJ%<+d{j;=WmSg8!M4w?O^ZnY<tS
zOL<fz6}cDXWSjB)+rnj%C?Y5)WoB9)Ifd_C^m-%C8SeaT6{UQs4t)Y9k};-2d)oz`
z=Ctt7(aII}Jat0gN`R1{%;zeOZms2fo>CoL${#KZX^5}k7w;-J`6zzGG~mfx{tYo<
z`qgdh0xEWvax-U0^IdAU5(v5Vi2q1$7gz7LFQ!?#(|M9FPvI~3H|N*bslcahWx(Ie
ztY<6`%Ef=8<+LgD`sw`4JNWtCU9a#x-f@j9X5UWkLkn0;zKT_t>^Zi~fLlw<glo?m
z{zD<}Oz!b+;@3Bn<a!~%X)gDC0;|?iXFlmIH9~po*YclmVCL>&3gmV@@|s_A#tXi!
zyYBJ&A9UmPz30Lsulm7c#nx`F{?Hx#pN`HpEpy$>*EdVp<j$#F{<7}9d@^Q2{JKs3
zrYTPH{K~h)_>N7P!|(ro0)L$BV*V?F0&ERWTKIPg3398xW)ac4)Goly7HH~MXJ^*8
zHe2BDm3{-q`kexQuDZ$>7ry5E*~}>@ePX4+!qtDd&-KkPTjl?Ur}LAoKzgc(-21AD
zeAmA62rT$6Zu*d|fnR1$zgYuklK@ZX4gQ;AP5i$mX>cu@qQ_NXxzH@g;HGImf1$DC
z)XS#f=idp4%o8#{uiGH-)$#$C@|jcSx4Z;7GvrV4nH{NDyzZVXu=txQ*W9W~?#J`|
z1*QdU<j<b%%D2+jmnU<D9naP!9i}Q%dAR#r>jVrvzM9#cmgK*akz=~!-VOflUwZt8
zRrmR?#)g^Mhd$ts+H#xkwW5u{{t`EVg*Ore_-;*MTWVM+G4u8=u6q&NL|dBf2?QU%
zYuZ}6(zNi#ApwT~S&P40<OR!wr4<zGtofhwX9)hSV-+;dozK18jM03V{Q*ANomZp}
z&6%zG)oMAP*_;(ZKJP8fk3XHupMGJZN%OL60>9b}!2AEt9}DKLom<5<{l-d*&}FC1
zY9unuawW4YL*tzU59LlWV@Wd=NlvZh)>B<$#+WjPbNT+4e4l6At4B=SDIk9%k^Au%
z79N>D#R9sY1O)<%^Z7*Nym@To!$j3o49(=`ZsFF}xh!zFz1Mu_6bF9WdDG19xV`7U
zClt$HeL`R0*;H=xlt3eaIY+(tMZH%EY>1sN5X598c&>C8SMWkTzBmSbt{OERkt64)
z@O>3qW0HF6uBk@VEMD!R^Tyc%EBVu`S!A6r2=T8?V&vaFD}%pd?L?ll{9dLFdDFO~
zn?48&UfL_AK2eKjfzB_!h>ne>`#$IJ?wb{0rt2ohZ=R^bSHZfCZ^0x^Zt+vTT&_Z~
zX5!WhOsliLnoM7)ZnAxvCjUi|gJ!QH=kpbrN%MR?o^5va$7Al@K3#k-R)#1fDhKmV
zpEH9??I;V+>Ksq*6MnUP0mr(ywx2iQ$-c9aohcySG%hWN>z?0a{wVfJvpd=ExVx9e
zn(aLi$jeyj&a;~RAFmRVj%jD#9v%yY13W3gn*^0u1^Ksp{lI&0Z68P8i5y;|geLaJ
z1`!d)4WD^ev7R-}n|;G{$5d(FeebTDd|CL8Z|0H*GLu6)`J6UI^4S(_=F6S4i+iqO
zrRlSp+gu_CW%(Wliis|%Yvzthw&6RMYG@`HbA{*K<;`Zk<<h*?UNd;lwhQoWY<|O$
zYTL)=5Ix^)MTd)7%lQJ+?|MIse>QaS`T5T?+c@btFV{j3?pv+%%v?@uai<1e;hj`6
zQ|?vk0zRH8tXw{&AGr7r@8kR@v5~i}RE=xVmg!uNe>-wG7T+;F%WcjzPxKVuWcfd)
zg&p&`7s*XDv#QkQdAt1)ce<c6uim4xCi&5PJf$bNd3^ak2^yZs<+HS3#Cz=GHTGXS
zU-Iv9naZ`}(NW<h<`)9SPv4rZRuM3B@H#8de{hD;oN#f$KdLX~-rc;zxBqIPU~)LC
zpk>@%uE(>T%qot$@%U<V2$&V-OK(@a%o|ygC&1%0$E1JVHojk<?ahp{KMJ@SSqp^C
zc*1Wc636wappkpV`x&PC9A%~-{e_HY_dGDxXR{G>T_0ekH{*>!asn4e|FOB|Kh8F@
zf81Be=ks}qEdP}y0@F4u;kp~8$t~G4jX$AYR^V)J5AU(bPq{ZuI>UXj`HTre-bOAq
zwub@|jNWE<Et2_cG-^zfyQKNotdHR{o#!QRdz-G=^KW4SYbrAM*+VA_EDM_=5Rmjp
z;G^>dhF7<B1rjFka=-nlA!2k`UGPGTlxcBbg{gy}fM9meV$(jwOhIe$R^I*v_52&I
zt`y8Na20%$eVF@HxWDl--PJtY1^Wa_`2qx5`Gomav8@q^Z#-h6C?LT9^SiIfTs;**
zC(dSpi!CMsX0iNSk5o5vetWjfWGmlUV~e9FO>8{sjIw6V7EE5^Z>+ReRj?sVgZooJ
zg^97;eNOBCy?ieWzla{#^HqRb(Ufb6?OyIX4A=RWaU=-HDV^axe7l*)CGIr$%k(Cb
z6`V@krhy8A#~braKYUolcdm%f)J;B|f1}bCzW-gV0_90NjYV%Z37oE<!++u8cY*3f
zcLntSD+>C`>T@dED)Vr*YIAb1P7pfF-oSJEx}n)l0X9>^qnbP`^p=@)ZnWSP+-@WD
ze0BiOI$svvn_^775s7oSB%g(v{(6|g`C|DTp2}++iod*FIsbp@<l(rm-z3*EmHXRi
zN3#gwJb`(7ygY%goOn2$=CHr@ddeYo{;R2O=W|noy+@3n*&3OoeOS#Cx%#!KSV|d>
zjhYQ7@8;!Zx0*9JcRqFIu}txmcxQW*hi~6Q_KRB-xgLZmb1V5s@#w@<avt7fz$F%V
zlwEM$Y}2flOE{KaROB_;?qia|-orKFhq9T$fk#|Cg*}|Rk1+BuZs;~uIX9JiXEFnK
z{Ew~t*4Ga3bai&|%+kBezV~GpZ$0yQ4&%6b!LoK!!N9D^W>brPo3x*C;;p;8-ZZ~s
z3!h^Jvuyi57QT#0Ieh1W%K0Xrn!#mTaN1-^9xJzEG7I0e-v;V&)hygWDx7>xR&u6=
zC%5wS%v@wvCOuL3wnh=}=GpUjpB2e*)x4j`CEXxvQaI<gN&MYtlcuN)Q^Du0d~BSv
zO!Jg}@Y*ic;9{S%+06F52UkeLTwcA$TU9Rw#`CF$OydmVYvKO!v4LlOWG`<6M-lg*
zGZH*6pL=ubb7YuVKi|Q1gJ%XG$1@JoUQ<~fN%I^t^{p$o%fe1@*UwPkb!1s*rrqSk
z#WM9g_rt#n1v56u@^$7j@SbRU%(3yi6`x7TVy@K%$wDU%UFK_y?=rhqdBbd4&_+J{
z0uR&Y^40v_TK%#!ObvNA8m04RyV&!;yrsdVoXlf(R+o$C%vNT8!JllJw-znsO%m<q
zd%wWS$fbH6Utpf8nWq$oSjIY2KFe4BeES|v<O=>-&VBw&zsXDaJ`<+DD@;G$)-+AY
z`N|h(e$nJj@gu&AcE310>^7Rky3JufvFsr4f#dJl^SxyFa~QsHeLcg(-7k>OKUdk5
z&mlCEclz~v+_y}dxQ#@-EiSO#<a#Y3$=@?O%XH~YOFqSjPLqQ=pZRn*sq!s}isLKE
z<TpFLMwKtoM2>d?+Y7-($3OD9{Cmges560WhUPMUUd}t5z8cno66==;{9>MM8oQ0x
zY$aQ_z(PAuv+VFI0_P8&=GFeJ#=qx}vS3HU0fCdD#oXZ<OeQxQ-*NBcv=)#zmC;ZM
zP~*){)E9Vp`Ju_0ZT@`KdD%u%XJkZrrn3lC3(Vl(D>au>wPYFROpawHGlU{cW3nfi
z{J#9z<W7~CV0VL#F@NJif%fw!xP+5F85Oc|a@kI_<1>5MrlOsmBk(&&iOXCznOkVn
zcfNabZt=hR$j+N`e>3;8t?AsH^BGL-UcKbf6Pzy)CfH<pR{1fn#VrAoA9aa*Qfz;C
z<AoUn9>_MBZ0-NXKTBSae`RT?@XPW*fw-BS0=JIu<~*`Nlb1DdEr%=XH=&=~4)PxO
z)n>-6q;AIcih=jnY%Mcq-<^DX7fwnXzVFVb<|52@_}?TxNpVh|g{JW)8OvXB&8qvv
zE3$Bx%&fL@?qk&|d`qj-%x;z}=9xN~+02&VAI~0_YTm1w?|E<S6Xu+ATAIUFj@8Up
z`J-viuSceT`8Z6zUi0H)2x&4^%b(9%k?zawD)HFV?ol)M60S468T{M|PMjrtcAhny
zy}@$aJpLYB4)=R_Z{88&D!ORFed_LJj>e1UOoQgF<^0LHjPJyDLo-3i2<~N3EoP5J
z8hAE{p5+cHX68*g#cymc#m_U1=_|Jc3%{W4J{!I(QTe>B>Q0>If%1GQuZ+2D_PX(L
zeMyj5D!j|A<(q=p#Vb~PA5R@IJ!TldA8~iKRIL4fp3^0N`O-VZ`T4(Ra$Q{a&17nU
z4EOA;1AGq-?$IdS{f_6ZMFStlWdYM$S~YyKIf7=pc58?|Tkwtd_2gK-lTW{Jb#UL~
z_Vin399{Fnv?M^))cXmanZ$Y(zS@S(CRexg@v-pi<dnEnVY>Ph3s;ofCEkXbc^r2w
ze(`O4Kc73<-<SIg$1c9P>2iGgcyI8$`Mr{7^CkiAicN>j7MeG6Pji;$?|KzsA}4i$
zSKz#hNx>o&KJK}Vynh?T`Bb?Vn(8fI&6{!VI!|c9X`$Oe>-Y{duIJ;~(8Vr(#G8*b
zY6s`O9bbglGgeB}gxoUonSa6b$=_DK+IU8@ytSqLS8PM2EBd$dJ!y94pSHl5U%p(7
z`<B5>lM=<<-1iSJ<Ks)=(WqzY=ibS=n@_sG(Db<GIv%~5XH5gSJw&>yeE3pV#qd44
z^_eq{$DOO^T(?Qx{zazRJj$k=5z9<ig3|aWmnoUfSaOALMGFh}qvsz@!>-NX+GZop
zr|Z2^?yL51zNrVbxMI#N;Qsbw9ryD(XTC~KChpopr?^iRAL5$5+0tx7_ExS$K2Lt>
zwL+$9GCVx0N83y*gI4maj%4IXoPUG&+QVSe@77|x_rCJ*3T^ig`f=$p-)ze)zWTkf
z?0*Vw@%iodWozHJT}WB;7QcRwmTAp_M6=3C8~CLfHkeFtVi8bt*(!5Bt$^>hrH??x
z6lQ^wt2T3;dz@|Rmwbq`I<}Pm^^`ecixe8UruvugvsphjmX&SiaaIg9-L~qku>S{R
z{wJL~`H~qmS+-o5!7=;P9McK?ji!=o_)S+o6*rxbZY5Bd&}y3Z`82=Xp--H8JX~hi
z<9=~^ch2Q~{Oz^mo!VLa*Xt&71i5E&Zg*1RS$Bq$-<M+_S3%27E*|R;ww3HnrYt@z
zoVPdK;um_wX2P+1EBEp6$tL+mFS#ZE?c}yFG35WZy3OQ<t|7m$j4WUE0Y-s3z8-%0
z&FlHUadvT)mXz=<ns|*fyg^08Gio_s^Q|Ms;wLtlY%$)%JNL;$<N8_0_^<z*AZxcu
zpZ|`N3_s(nMt*CKKAy?^45qy%len$dzT^wMZzY|4)sW}?0WN+Yh1VwEk5A)0pu%9r
zymJ-r$A$KMoUAAL`rn#xw}l<#O5WUHcJ|^N(^4h}lVdNJnaDkN;MZlpXI6IV0H1xS
zD9`u%u4c0AC%6y)@8?_7peuV+q?kYa&O<Jqxk@~LCf9NA`qIxAZ?m6^>$E7(X;E8_
zPl9?T3Sm>YZWS%zPqf`-YH(hiM`@p&*~E5t-a~bXJhi+Md_L1#OyiI5<@uoUlt-yr
zRPe${1^yNOe|hT)E^wUUWadv}WaAX<6Bdezb`ayetZkO|Zl~GeSDW}3uUKqm@>o=0
zvUZQ0W84{Dp1LrBx&Fce%~c}YcG=-(f%e-tuZi{WueWxPT`+Sam-mKd{`f3L6G0^n
zUYWkdW(#Cr@z1?t$-lDm4Bx9MZk*>+GC1w0TATVVeQb7F?Um`Twu$BvyFCP2)6Gmi
zyIkX6>YUC|z}{>2Sndtyg_~=6x5PNg8(vt)uOsz^Q|5s`SNoX>JbHGL{Co5~xfgxk
z#bfJxfpewf2eY>>rrhRhzVoxG*_p++s`8vLSZ|uPa3A-h`paC3UZMP_U!F8e6Vc)K
z_L1R}{`Wzk&viEcvAO&BTbJ;$bA7nSAKBl}9a&`|eDM1<f!WP^rstlRnoCdKB#<c*
zW%_W1yx{Vl3^~DPMf`5^;exfryn;=4m-5Km5HRB_<>Hk)T`Iz_;U@P_Yy+Q0WWK;`
zgBa7$vm5z8yn0~j6v`;fXQn40=l`AmiK!}=Zh$V=M5oVY1*Qv4tMA-0b-tixmUrDz
z&?Sb?{9?^XfhB9Fa$Bs_HNQ~A!~LgOgn!p}MuqOT+XX(TRdD$%72*lbY7ofr7ZVVw
ztl=}?G@U2Hm4RoD#|qOrXK!xr)LR1YgAbZzNBHt5?cq1waFj=YH$9erU6X-;!JC<8
zDL$G4##P$<CVOjz(^D1;v~JodFlA>Pm*^r#UjHn8PNO;pAx%dS@#uSx&CVR$WZLB^
z$@|O1)%4zlZa%5`_Hx|2X7FyB;mB9>(}}P1S{t|K%M#P*_;#)!>%}647jLNlWq!(8
zq5YP3#YR)p^EV@Sv|H5779E)_vNkW9w|`YP@6=~~oN=2YIXCjCn(kY-+LW20*yM2G
zEHiD52tFwm8`Bwnmw1bpSaN-=(>Hr8KZDCGX#(%ZbAqyMH@*0RR1UKnIqm1V@x7kA
zrLv6ovW_*^j%q)iZDm?qT$j4c?A)WdPG&9OyCCIlyfpA0*Oi;jrd1~&aNhYEz;!>%
zhxdi?d((GS0bHy>f4IGt84E6Mmf@41W6Zm3k1v<jCokSQl~3&IFGBcA4I{<&d#yHq
zv#i0q#*l-zfBG)-_|3h1GWN<+x4W8nT-Gb_wSIEqJE7skT{nr@OeI2qL(Jzc?^W5^
z%Ef`AoTo~z^7fw2G@BA}okyx7+-%<Z*#g?VX1q_$X7XO)c*;3%h9;N$!Xu{BUHi<r
z*__R+@9s8Rzhf1zih_+vk?ASkU43sjP3N$f1*cVWb#=P&PJJCCeyur>@3yHL*N@L%
zI9t0Nxs%RR@rG>6<9f!<&g0}ZfxTrfulWIuD_n9$4SeyhznFQmYH<J5oMM`E{WMpr
z$uCZx&K6#yOSNX(|4rw))%lm3(cD8=<ry>Il1I0AQy%bhI&b~LzaYGYt045b(0Q{@
z0{e8gn`Z}2F+Vo<q5x~-Q}f3{VuCZwi$r<PJm-sVE)^{I<`n!;IfGlipV723GL}c=
zQkOu-qsy`eMQ3@p_Lm7LU$!&f*sz%|^`WrY40dh7mRNfM-ET+v6$55*8mwjFay&WH
z?0nuGv$co0%(<Foo8-qQ3C8|xHf_H3Mqt5u5%$xkR+@44tl@g|wS#YsQlgT|_9X%@
zSnW7>%@^Wk`qa)Z@kmC%<yarDmc)CW+xLxm<{z1EHc@5=ccS700S$|0^8>5h`K}$8
zF^_lr%BTNXmrv2tL*Q!6K9i}l^aQRMMDS~bJrdqAX{vyt>sbLw*0mfR@qBz0pOrWd
zD~1a@)+F;~tc^Bxo6lo5f973Ykrk1qhk7OW8I<Z}u0$Q-i=6s~uea<1pSq3%_ngbT
zX0IOJ=4QTD!Z&H!P2qR<`ni+-l=J=1?=p2sW#e6V@|l@pwv^y=sn@*o+70*)ZP~?@
zWBH9syz;JT+fg6W^k5ZJ&mK+FXiYYL{e8k_k9St`)!h{2-jM&$?9}T$TvOL5@YST|
zO8P1u<TJ_j;97G18@Inz4Uge6E<WyE#oRouuedkV6mmQNn__C-#KT<@E5t9axXtX!
zgFqgM>xSkbQMEiP_vrBKiZSK&>N7Um>lw`LXFr4I&T271AE9Y{f)7{lo|_cUa^hwW
zf2HaxF8=FY0y&}`0)G`V%$zdo%*-p21x#*TH#1#vR6uP~novT!CBJ$JmmsI~T7l(T
zUUR4S%{R#u5acbnVk=PjC{1;xd@EmGt)Rf;JJ(GQn6&bXahRJpD`$z!F2Bvc>t+vs
zi#Hed;|=>c3xiLa@R%8zwQ7r-8df!%R=i*k+^^VX+`gwtATwhnw@d_+F`vB!r(4)j
zzHdz2YO5dT36xa}aX;Rxz+?8?Ss<osGykCq2fmm8l6bDB_VO69)tZ*+t>v!eY!u)*
z%W3xdzZ}1j<PMX4)ra`68kz97>^jH)xF+7z`uPt2dC_P2RxF4Yd3DZHAUq{TASz%X
z*XIM0d^{zqIJJLC3fX@6%DYXx-(+uhsM*Jj#=I`?3r**De&lml@=7A)+CtuS#k2XI
zhj#Ft_t?pOW1@lSvWw|lclDk44pzu%>~}2Y`hG`^FD#MQ_>1)s9^V}<rY4t?L`pPf
z@}70t&MPnD#nsR$z$IoLYU1bUZ~RA3!}R`Pb`!spS$wU{pNuDdSK>>!`JMB1=2cUR
zkUd=dGneol`TCXZ>$>H9y<7A-p08QM)f{iaqy29m@9CdUxcFZ5@RWOXbG2-dHsdg3
z;kFWZ%6GP>-&EaRiTiFdpXsdSZk)v{!#SVnZsonvqhji{f|a|J-I&MnQM1V7#R+^H
zcBS*~%CqLs<8I?m;7sG{m}D=^daz0$geAfB|J!1->&3AGa-r9ZdV^*ORJLE1;|uQQ
z+xwAGaL?um0;i<AxI6@G%`A@hanC&_E^z1JE}8KCg1qI6SOv0^!cE#{N%94)VKmD$
zOBd+YI?w;OGM@jP(-h8&MQ6BeRSiwDE^IULXk#-q(z7yie{xu0f2*Zw_3;LQtiuJY
z=bk&5U;1apskhRY?{2+}Z2lTufi<UExRw<pa7!}Y;d^p>4S%WNBOcRF)3{f}uI7?g
zyk|O<lab5&aIFCE`U|GPJbQUpv#c{|y^zP3y1t#)=K3T4g?z1Mzf|7y*L1$&`%}j#
zuuxiFK<HeWfXTz(+@Xed_>?zo;M!^MlUtCxk?%1pgPB?Ge6ugtmh*0$tZZH=ypVsY
z%v9;mv&H!y?UCgVUSiH~+`Ed0b6>paYNamjYcC%2U9a#}$Ww3Ny2W>jZzj)D(@Ae{
z@Y-(PW*YKJT6pOl4!%h1HoiB0G2AlQoZRO{WKH!l8q5xvwwg9;rkX0=7U7pjTxTqH
z^BkX=<0kI&|NoloSDwSIHaUz>@oJ-*a(^QKsjI%+y2;EupR(uh%uR3Qi;UCf7Fi;}
zyE5k_$MvrLrZHj(+*}iS_|Lc3m>v2m%hT(TZ+gdn4v%=<JFecV4SX_&Ys?I}Qh36*
zUFLc2!Xvy}_%~mqaW>y4V^Qu&GnD!2(ml9luqz5)FFz=%V7AA!bD@}-m<2y?$V@)7
zX6gHUnH3>&><=gM9(`QHXTG6>@6|FEo(VG-n!aY0<mP&(&$nRuaz(jc_FV3JB={Iz
z0!<#++w)GG*kcyf?!~`WCWtro!**VuXOp?SH<fbz_T6Z5EMCiOM#2PBzCB##`mIfT
z#cshSwZEkJf+B8n<sHp7yJxYS>o4CT-ebxa#Lnu@<6Fixf$K|oFZYs_-aP%yM|gjG
zujTp@7t50*+Rf=JJI!p7{2#8h0Y~|!>|bHppK*q}^YI*0yG~}FiPzY;%)R&Uo_Bp>
zw)0p9_lAF}JbSOs6-e7;!*_3AKkxe)mFyKu5AtxO_i?ze+3>4vS<YjvvBT8-XP>$5
zW(l5RjX$Pa+S_<Nq>oG5m>To!n&HE{OvHtEYHk|WqY?!({;r9f-5!j*{!x$BRn*uy
z7jZn{5o6Ue^4a#5dr8w}(_m#ivG>8EJk`Z5Jk?CsIc`ml<m?NYWxBp?hH>T_BeTud
zzL{_?^ymHjNz5ei@-H6NrPJ6yEbldw;t%CKc-ogo-R?i@vJN-i>292yHvM5-)eMr{
zaV@iXOvDaw>ZzM@Ut9f=V+P+@^Pm+DT-zL1^X7)8n*CqE$Q`lzkx8V=Jx+$DnH>I2
zbv&UaXG{%(E4cKfYq=FA-U-KfbMkh6H0F_4yun?xXFgwRur}A@^{)gkrAPA3JXdIT
z!ou5h)*CtA69oardL7gG1FcTWJ$~uRmoSx^e{F;|f2qlAp30*$&EBqY;O6Vs7hQSx
zjNG1`5nReUkMlJa_L^uaweV*4wVGA@@8OrK<ly@+Tg@ka--LT>jt19t_64TXKGm3Z
z&*wGivD#&RaW@D5xu>kAEU9bwt~V^^o-H=pjKi#j+la}T&;QOGg@&dK{+lmKxCB}Y
zxkbL5<Nmv)fUh#UnM-4mDR01j4bIQ)r_A02UE&J=)yto=YntiFRBi5e3z$v6y_?Bh
zSvQmGY<eM|@AF5deoHQL8;AAstX7i~;9l^TZ`%w>zI(hzoSU+n_)=7Fb4@V4DC~CE
zoG-71!EE``gQjoyIPq;;*lTo{(T88V_lQg#BNy+Y&To8kWq<IUw!6f6;l)dn$Fa}3
z%Nuv`F$8-F>eZI<?pBfG;}df?IqcHTr}EFpG<xcP-pbt@c>kLk^LfnOz~!)jfoHXa
zpsC6vD^p*cOU7LX_8Ps~^nv%9>JC%q<VL>hfz2GdmaQ-=`Ig18I&m-Ws;58LH%~an
z$2Q4@`(r^CcS(XNKi~InyvICsc{39}a_>-#<6d&0-Ppu)BiGx`JA966TqcV(MENdd
zY%)18V*}p@-dDVrZ{FbD=>FYg|Kwg?t_xRrTBkGed-qM|)6AI87qjIjSLb;FUQW(G
zoOZhyMGl;9=2cvrV)FQUxd}5fAFo4Bjd2)X4Bu@%cUe=-rM&w5;(XfwIru)lzs$Yp
z?=F+%1rA*6AK3|J<TJ^n8b0Neb)3bUvhT8~66Xn?=DuJv;a*l=fhZ~7uP)8J?;n2P
zWc$9H%l@9Q`O6yyW~Y8`HE~#~VbVC|9dCrCxw)*#Hs0k1cHEy#+RSuqk8`P;d-I-O
z)F?ltREn=C{T-*HtTMOhkzk&hch0=RC%15A`I~TG=IUZwdnn5E_^jz{t3AT_<{vRM
zd!cxk+gOsztmv8zPZVn)cf6<}Zv!*8savcI4`&J=&z0~4{EtNM@^1g!!kc;MB*(8)
zQUXt(6mSKZ{1Di;J4r(F=xI|8MnSVkx4!~=jE)$)e-04bcW}8}L<cj!x6ceg%~b}1
zH=~2N&v&w!JyOi$31{3SuuR)rZMiuM-?Y861!lWVGtN%t<)1CvZDxDSO88lEuD}u-
zc>z(CJ6vbwZ*mo@OPX9g&1f3%e3EHL>k2c!j55L4mWw7oLPQ0piK=quer7V?{9z6!
zv*Jv?Hwh9P`wl)8Fb!blmOJ{Ei+AI8exKPs0v{Z9@$%Wc<C(`Q&%^R%mf3ukW!z3N
zqJr|5rkk?Qn#k8xb=^cz%b4Fow}<ckye5J8^1Ws+QtJdPi~IQhoxU#MuYO7(=(CWZ
z|I$*9pPj4u%4e+O-d=KAkm1N`{%wEPnjN@x-puXPEdJTcSxwinunTle^_9KgB)}KW
zVI^Rk{hFW0fQ##7QGw|z)w?`*J!kNLQ8VWkyrjkFxW|B>^W+}mVj&j(+y#G)gJ#U)
zZ%N?c-<NWkufKl+*OYxrxz{ECG%43QW>WVo-PDWU*!22TE&+ALXU1U<*Yj(;=y4Ux
zKQgU;`I~*u2T#77AFJ8lELy;yUd74%<M4fMoi#uB!^5BQg?u@|YxU+B_l)RDZh`re
zOv3EVxE5U6&OfEB-RRuH&3smVtBpU*Y~XKVox;b>C(o~AkZHQynwMX82RGm0p2<Q<
zV!ixIJoEW8^m5tf3%BxbxtPekYOATR>HKzqf=y}Wq9vi`Js-0KF1gJyJ@2topfhKp
zoXq{7d;v+Eg02Ru1#CYa=l=X&(JYLYoA)oDoq+9^D3Q<GXY+;X@Cy9hZe&{4R>t3a
zrrxyjQ=>rkp4<F!p)LIC^EY!HnZ?6>U#-N9?+b@n$oC7ThK=h@AH07dARW?Vrm5N>
zF!KU4_t#)0^To$`x!Mk2<MW@pMdl|@g24LnP_Fw<QQV;=i2^dB8~7RB?fAGq$nfNH
zaq+Z^OPl7gZ0C{+sTJT}>1U!jSBig;qrPdj<x766jcWXtBtGzKurZj)UU<SUQFfD0
zyrEj?%6S)o_9aOIovkh$^<K&1t9W#|z5Ta~UjKYoz(3j7B$%hql=Z}6f#q*_%*}J<
z1vfbvDlAD9;}`Hu7fdu~6YQOv%KiG@U$cFGd--PR9hde@o1<#iwU&=}>I$KX#Jgq>
z|199Y60p}q_3?Fql|04*jM3d<Y+X#;nqqd`*OJ#*@JuT=3-~eB^mlcRrPL)e!Jecy
zW(T+36R5fu#r-Xc#r)E?DQpHTpZK&^JF1s3?GRW}-psxD<T>u-b2S38EkXieIvIRV
z)>-hJ`Pt6%YkHY!$w6sueU>W%{}1SzGYL8Jd)K&_F>|vDm`R25yVP3?^w(WAldy9U
z=vVRO-<7ydz;yLIfv)gx0uIwpaP4?4!<XGWjYIW|ph$GybiUe`n@vBuahr8KoX^X&
zM9_3e&LaMtSw1p19;xt`xH0oDnwh}QwJ(mRMbO;TmED;~TIK|wTFz?W?8AjT)93%=
zoBl=4Y_)JF?^de@Gc&y|UgcGGe1Bi;=PO%p&V5KZkn7(BEwi7y#LSk>Pczk@W^EiE
ztHq!2veN9l&U`+dDt{g=867j*yiMG)JNx;jFE}8lzbk_Ob}TFR4R;})yWL6LudikC
zow+Q^-M2}YM=5?8duK(1>CzvGTn%?7@H=Y;n>n>A@-VI0Z}vysjd$s+Ej%TWYj__%
zxnQ!1t%9fg;zk}1(?0?$mI(2unsM^EnBL-;5mLzegZnv0nmfBlNzymol*4n(UavcA
zs??^yE0mXPwqEKb-<Mit>1K(Ee1EP-@u~Z7;FJ6?hua~k*|egRpWCZjp3j%to$p^#
z2e+=U9pAT-9J4#-w|GAHH=6aVQs#|_o5Oo!B|o26e-?+4yfFLWQ}4{AU#gf%8_zIR
z+A-VM=v^=0n^j?E{Q=K-tr|SI&3v=W7)-di4`*EG<>V?-khERIH^;-3%h3HJ*I88_
zt~aXdczY%YaA}7A<f<%s!lB&#+_d)e0j{m?C;1jg&NAisw}sogdXCuzE@7UVSH5$b
zm<aR6x^6d_78A-n_w57j4NnAxK8R=Y6`z~Qd-Ibt#~GJ5{1X(dxV_aRg&n(}3#`9B
z(fq8Mn7JYQ8G$5T&>ALjL778uWp{*K;gi}}D0plNo8afLD6TzqmrW-*&E$EW)*%p(
z5iKG1=q~Tl7x@BQ{{>8amagGb$)02S?8<$CPxH(KY~SAFXDRC7TK&q6yN2hYshHhr
z)AxI(m<n@lHt|q*5VU`_*i_`>YXRx+Q#fQ>PMO7Dt>QSZ(96d(p-Eyw(L#YuKNoUU
zern`q4qm`Ncdm@U5vx+(uC0bVXE(8P|KmDsYOTJPt0D2e!1q<hjoY2$`R-}9nXXGw
z;XlKp$7ir9THsvXXVbYA!2(YIqWPni9~L^<G)dq~?LC2*KR4J8S?CFTQ9r^NWNj+q
zxk6pgr$or~&gJVSP00d+oRj3tu9s#Co|dr^Fo~_?Pkg*mkmrbtpmgA5?s-ZsMoh=&
z@>smzBVao}KwyrIEZ@|9YXmq;EKCz?dHLmBml?YhC<|)bZxuM=Zz;gYk<6u`TFrH4
zorwuY>wS|ecYc}7y6S6G)ignHK@^|S>nEy$9(&BWvyO_Iw0GU$3{E)3r>4#%8XoyW
zAi8%UR}p^)_os8u`FCxN7ntdLp7)BbIFHLAbDoOU%f^kq{@e==C<wk=yv}rS^aj2y
z&Vr`-W%2xcrw;R}EbbLB*ty4euS=JJQ29Lm;)*u{-p%&}Qg0~=Mhcp8HaeK_B*g7u
z&z|QlwBEXbr$_yV*-WJt(}f-qJlpP@n=N55=Vks7DBV{b%%hmk!pnX23y*}UH`i*$
zFw<?b8ab~lo6B=OL0?s_?E{B!+!UU<PiLDZD|&Nl37eSB^!&qpSN9k9>J~X32I~?I
zF*6H}xaD(9Cmj85de!5UN!)%LW0RK;cupohG5xhEk4Il!oO7pXm|5_fB+lE5gLvY1
z?vgzraFj>n)CBg;RX;gjeqiFRS|G<0G<h>eqR2U}p4rnm-Fkb?_U@j|=^LcTdu85k
zliMa9T+i&&%t{-rx%NI6;GDnv8@G#Ix9ROqC%JAH9pq{XapgDb-_OJUA&p1<KPSfv
znE+n@g@T-A<@*G5_ut~Z`#Qwz!8tqA+W8T@)yGuK-r26?dk~x<W3T^>cgKnhz6*Qu
z`QCZ1<vKfy-;_1hhWoKID_?FotLnB|Hg2nMCcevO{Y)1;VCD_rJYeQh!pMK^L>;f~
z_O-l<Y95>%m*#LfEEYCh#@cF{&(&;_5K(O+$kD=g$x+JmugFi{n{s_z*2i1SR<}oU
zMW30)`|5qV+OrSoe7p7=a820f${l!2pZkS*H!ok{Htt6^PjcIC_2gc2KG>|D*O2S|
z{pozHn|_$Gv3T&vuy>oyUc8^XKs1AA+vDXtrjrkwb^Myg^{=0wCw6MP;BF2Dz67sq
z-v5G+IGX1>^2N{iz{M=jF6dBlnXh&EQL}3cyv_BU7xO(|6Jl2JtCHX2=`v}NeR8}t
zIT`$Gwz>1S%uV15s~0f)@w}c#^9(Ehrz2Wgm4{aIzP{bf=dR;qQqi!Cue+biEVyo$
zSeB?Vp97y4-@!+#xPFNFaQ}&CHl0)MZhAM{&+O!n118shYw$l`ztm*j-<y1k&T??x
zDC9GH$Iro87ITVMV~HAv>@I$O&a`J-XD!{h|A$ub+ew=9)l6~YZC~ucV<NGhD_fz-
zqEnlPJ3v~B|3~IrQ?5KSzSY4kCT|NF_yacn;eGKWm+$=TH>QH#%zURzgm|T<u#1?^
ze9ag5@;;xnRx5jA!4m#e0$m)7q@;!3HZ2iQQ*}1eS$n|rOk%e{js01(+T2S5iafkr
z=cR=Bo9`+Ma^BuA@NK&$ckSC)qowI<xnJ(J68Ii?jKAxVH1E|NdIH(2Doigt3E`_@
zX*AZl!7M2ElTASE%uId-O%^V(U(K94kLyhIzvr9&R+lyHYL+$e{>39Wz51uo$4v_b
ziXTgJ&o7KI>8shxIbYC?FW)9YsB34g0E6{XF8}Rn+&}By@p-4-;@@S(#an0Kz+Jhb
zjhlb^Q<FEYzqn>zo+t1<EZp>8$0uHvtA!@kOa*-B&hhhg@v{nC-)m#SEXX1d`9p;N
zZ}C6DGn)bhO5`UAY~^^#$=;*QD<Z|t_R8&x(2F;Rc|V_EFsoNzW7775g;(PAWwT`^
zTllKX8zq*eIP!g~5aIiNvY+o@&1UX2eQT2^H<P&&lsH5qtQL!L{ORR>dtHt1#EcrV
zfA*VrwlpP~sx!ak`MbA?_qhO*#P$!XIKTQ=v8(QzZ&tE)uW8=FM$?ks2gX|#h46_V
zJ7YRMzgqCe>t^nLwXde-_NCkozt8i2|F~D7{CowU?yeu4n|YPECnRQZ-C*kEoiMSD
zvsiE@x9$WV){m)EOa)ugxca{@<a53F(^Tht0{2r-L$hyFwRu`Bu5uf-9^zSkf1A<S
z$$i`(Q%`fdlr-`MPPXD}-!+YIrk@^XRgXH~)4xl(Dk?Jgzv<PB&)?f-wxoH3sYIh7
z-_}}3v!j2#`6Hf0NWC)N!}GuF58sAFRsOcXlU$$bUYIPu^NL$8_!!@I1rb$7f!925
z1snN-#5GN?gqHICa5gY&Pv8>MZ#u?nQ|`^juDFFO_d)^p(Rx`E5nfBP&kSCszZ@@{
z?hxF?=XUF=F;7erUv;N5XWWXJrh?lxaEf0)$a~h>P%!M;3%)4s+1&3MpK#kIZ|2+I
zsLB^pw2nt<As26(DL2;~rV=yL#=G3xBW3u1%=9!h{B(-<=&b+~$7v>f|7UaY))~t2
z{WrO5s{F{9w_(awo{PsE1TQ^W&1ZjVCZCsy9*6&A3BENi1UO1{&k0S;J}<_SW^MM*
z(bx3g+D^X7i*n5-m&Wt6EL4ydcI@X<S99h!xvIlIW%nK~tI(AucRyX^_FCd7%%pNr
z<;aJ{+{g6x@V(x=#k3^zE01a`x7iM_d_gVO2)_9%Bl)&{-pu*i>@SB#o4l#>!WE{K
z+y<tn84jBioDb&@_xoUy$FfvtgMA`*fAoFRoT;f?id}+yWeXTo>t-?XUoL&Xr6fIx
z+u~a~mkGN)-+?KH+!v-U;0{~*k?W4Mp4sN@Pq|_yI`LofXfjn=WWi%~@Rdomfh|wP
ziw2&>9<z9@pH4P)uTtTi5hThRckd#fiQiQ|CDuiJxv|sOO*3!tCFtj{YkvAC=zrrT
zf3VIs)8}gaX6aEI_$NA<n>oB<7I+xTCpoXanD3IbkHF<jW&s_Cvs@cuKA7Cuvx>vK
zs+3<@!ASYz&I&FKjS_wnF<+C-c{x0y+b@{BJE$%4>$ow$#LOLhd%qrLUBI@PjprV>
z+4s{EOdnUPn#StCG}(5`Qb2F&1(U*yr}@u5;o{PKBV)=~{*3e6k9E91+$<#pqGs_Q
zWLU`&lJ$o3p^Z9Ek{=hp@cv0$?o&^3sl_pK{BvAuI>E$>b9&4z{>w}6n@qRc$KAlS
z)Z|?32X4=;UEFQy*8JY)#wLRIO!!@zCHY>3eGm#_=;6;)+sHrb%p|T{_9(vJvgbHD
zpYn<@KV8MQu#n%3X~JYv^({MiT|R1<P8QhDufVoe=Fh)ie(PL${?&gf`LFWk^4w~P
zHr>o`&$EB=BfhLTw}i~T*72NKz{5ZBNsj6MODlNa28f!?wVcJf`&=-eQ`23(BMuSV
z4wb99Zbe#~ZC*CR^byB56Fwt$lkH(n{Ds^;W)E&0;FCMy!($v>V76DJm;3F;X?!eU
zZZf_hRs6f7b+~i)So0{|%;VmX)5<3<-o(A==?U)ao$;J6pBkHpOU&Y0Q?iKv_rf)%
z>r4uGOx$wJQdFaP*FXBq<1^t9Z`Wcg)6X|sc+UR0%JW}>OR$<%g+Ho7l25yojZ@?z
z6Tf&%Cg<yE&H{^WJ>yrkd1q36#?ZXpdNcn%=0)ZT^F#$+Zrv>_clHFY{E{$%kEX%`
zUR_??KCJO(&zRLX@3?pK7hPbL5XwKzCHB3Uzc3-%NQYmV_sSQ3GhL12{HAkF_#gd0
z!IvI7k5joXmD8g|#gwgx$2?nH$}HEb!8}{sNuZ`B!gylVHU3OZ2M$(cDYN-CtX#i8
zEaLrN!y*;=eJ%g?L(4cFH#=~h3F_x*4V2=)|G1aiyLT>+?aaws8eg`VNd+$ER(|rG
zUtV9pJXB1UCuH?G(>XrI+}hWbxQ}d)<X0&<XU4G8gx@w_o^NvYA7SghS^ON&cJuGO
z_LRN5?-73;w<dS7>~W!GhSvrB96y=OpVDV;xMZWi^vFoFpGV{c6%I4X?wvA;Um`MG
z@S+x<pkBvxp5vZdOs8%)=iOI4L*Sm-A+fUad-#GE6$tE#mNxw$u#ErQ(pjeajdg@1
z|LF@nU-phaC29)StBxOB*Ol&><*u4yddXhFOx4`YG<&JNV3gh(vz0j~1$5%3aUa@k
zYnFOFnLD~jmH)%aV7_MM?E+^HC2)2By}^C<Td%;U3t|Ea>r(g>u6FYzPH5zbf0b&w
zZ9+Qt)j2l={Qs>on=sj%-|M%o>75_K0$Hnr_!n%p7GT*bX2uXEFW{@D&Y!&Dmq_&Y
zMFO{Tw+Xy`70D&GoRjy-(mu}54l@O;&+p<@x;NWQ>t%yk>v<*KN32gwcS_ar*&KW%
zyIQ=D*VWgNPj`|XpD#xr_gTi%rXK6JbJ<1y=B-#|r6Rp%A*auSx4itTlufr)PvNnT
zS2O$a;;}&Lv})e6&rQ5W*Pe4`AA7}75@lps_wc_dhpngSd!+`m2Q0CC*FD)yH@v>Y
z`^F}eOFfI#%=zXgF5aWvynE{&h^iR{@Xa&)$00PemrL=)b}nY7JYKftSGfA?EqL68
zIk@hdZ!}X>>f_?SG>`ApJ8RPmVjsDztprUkt^LlvSbH{C(GC&b)Blyt7RT@4bSP2d
z5xdMLD9$X!=Wx-D*Zz7q*Q?j&yt4$9Ii1f<5Kw3@6!*|vXTIstax<HWA9)<J7Fc|J
z(aU$+%T;QRdMA%YoFZR#su$m?gxTDGPivcnuS@0FX!Vd+hkcgvy0iN@7V=!?-Q;%1
z)OObe9&fuwvjh(&-V?_~dDkxO<&|=i=5o5}!Fk|(qv_sL#pW52U1oa@%{KE~D#WK*
z)nHU?!!A0t?<41iiC0ZePE+I3J7~^(uh~)RkBl$h#-LlAQM1yxbh<gY;}4eco)E3z
zYKqg~ajdczdvs%wxun4@uHd)zeCD;S=31-RxYx|$F`HSO&Lw!@0>_yz6}+a0H=Dit
z=)qGLai6<kcZ=ZasjPg??z?%VIwUxk34h?<BHziSuF@pD<={twee+kD|CwZBcA)-}
z0Po9UixZ8ag1=rwh%cCZfUm-}RIvLmhhS39QttaFCYi+T(Bp~axG(lo>7m%&mVLad
z<w^zI!kf(}w(sV<>-o&Iz4(;?&mUWXydSG2cZY1}yu0rh=e623X19MnG?QH<U>;lX
z$*5v?px~pc6HNU%8bnL%W^nA7R$<opmXYg5R0H3J13C&%OqU9*$u{H!%>l&*Hu4wA
z$_rG^oz8otcp*=FnK)1D%xz|}?~1rNemoGkk+;=+$#!49BOhOwb=ETR%TMy<o9*N)
zkozFn^pAm)03%llztO&E!IkHx3KS}+3W{F2!qJ&Hop<(@8Egr+eS|**W$<y-2%7OT
z?=ZE<`^eke$ZKX%$Ih=>9WP_SH<R!2h1Yy1FCOJ<36|ovY3DNgIQKTU#pN<STV@|V
zuOm~qodOH^f>%eFT~l-AU32<_*~H&-d6`^)@kYg)@*VtD!o|+=j8pMskLfWkDYK4a
zx~3nLBTZ`W|Kt0qd)f3jQw`r^i3)D5f`4Y*$IH1ASw;AkGA1#0H=O3X_lkkrX2}ii
ze&O9*pZvM_#QBbKC*9u4ed$Fq_wz;TO<x6H;`-^r%WpmZhnYZbKhN>D8RnLz#yqdi
zo#Z(?FQ4blWnQz3db7B#%a8J?R+{l&v75=aeu5fbn#2qCv&XvmStDb)mh3t#81lbe
zAjJ8AS@Fv6rr+kJ3LLL$HkU~}BG4eeL$v;@3g{j(!Mv<B0z1~O=l<|U#zg%7N1nJ<
z_5z6$zwyr$?%|X46ckv?Gu>2RLkIsOk7*{s->L<CxgYW`YwzU`^?bx-*}9J_D{X?w
zwsIRYrfFTKQ#b`pinHDd)V^*ustG(Ov?OsF_pgttCbyQ?a2~HZ!?&H8N0Iqtp}?VK
ziQLPl9_GF&?k6xiYzsg0Jv+YV{<=J#ub&G)K7QX+MLwT9P_R+Jwn*3PNU<XS6LAUC
zwnvxwg&C~)cOJOLA8NMFR7Cg~|DUu=d`iK_!mlm71T3V|1^VZAbKRS`j`zz>L9Pc^
zErfzU|K`oOrDK|IvDmC>wI{DcQlaUBeV6%ol@>}ydOqTPtTUVMCU-kuZ5c0*)a)Qr
ztMUe}6M>F=Q?7@rrk|R`x&N*b-%n|6laxMYUfH*irU@&Rgm<ye=Pi15j@O5CFPHMs
zY|b$M(<YK%jZGpH8cf~KNtyUm&f!z`$TYqdsLFRV%aY4~kBC`i>jbXN`}%nKWkT63
zm{#+BWN_x-mXhOIrfR~SwQUD)ZbbvP$)Ru__AGs_#NZF6OU;_Oj(mE`w?3G~Y!_oB
z_s3o9OkKBiaI9O*#vSUK&ui!|Y`Tq4n!7K$iKnT)OoZP*g>PEeOy2U{J?wIQ9sG-y
zo#X12HV_KPs1jKF=7$+?RjhelPNcve!)K<kLbC*R@|=>rm%Em){X3&z#QKQ>Q+QW%
z-PQkT8X)4qbK{4EKv~odp|^{<c*7R53CwN!VqBBUz!$vygUN#ZX#%bbkMaM1<;U+(
z_MB7tk}@~v+EUZsW&)<X;(tt^9}YE}qIyHX`%j$7Ie~hCLMCC3Ems1}1kYaPbQF%~
z%iHo^d~<-d!28eETodLT;6C~5Cf|oQYxvzI=kcVjR^a)c^o;AT`*c&m`kh>bLNx;a
z;xbITIXCkz6K^xANJ!<=_@u*YeDNE<#l>2)ly%qmy}fVnRUP&czAYv%Aows!K+NYZ
z_hX)ie2Nd4xx(h&=kBO%<eNX?f$6!|6U|!dDtJXwPnoy6EaIQ{*;(qPb1a{fnk@g$
z5Iz31u<Ja2{u@o-FgI~uc=dp9<zXYSQ_Rb`rYyR~caLq4iI@2y-YeYQrZr1ad70&T
z`2Kg-^G!HBh5H1T0QU+FQPcL5jb>6G8cbE1d`+y|75E=H6&bzKJ;A5nx0?I)yb==w
z{%PD+@1ywa81j^4GvoOSJg0MoU-{4N>s-szqtM28_Y*Jou?^b1^ImC+UJ80_dM7-H
zd)4U){H}J~=K4~LxtX3kHEq(L&b?;-9WJg3X?#`{*Ud5-zjFJA9OEe{GZ3D7=ojDC
zbFX+0EM(<Q3)SZ9nApZCwcCQfEaw_;(yd05-5O`iI8HL~-i#73k6-?PPjcU0*@v%=
z^B$0{<D2N#%(qHHhe!0^FH=svqg>20H2Gd%;^T9fRK~TiNrkU?MU8RhCM{kI2N|>V
zH;(bh%4G4TKH1K@#%wFso;#blw$~mpDZM|{EG=Q9>CA$!W`*Bc_&Ap)7}u*w^36Eg
z&XsM;Y*zaBJy-aG)x1u}F0nTMoX2OXaDlV&;Z)H6Ii8O(hj{OBFmvA%jO8ii;N-s9
z?PtbP70O-EdWug|Y>nCHvRv+G2QHaTm%Yh#wIzuwe&$@>UK=j+&bN%*w=86NuJk?^
zl&!bsYkkedo3p!^y(n)FPoU#hj{R=G_>bIO!4q<1r&+|7ljes_gn4S;#G84|@8oT*
zKPP#J>m1L`1HrsYKN$07*0*s*E>bf~eKwO*{MLIO=1-F}y3gryYUn-Zky_$q`f~k#
z?rpa#%x=x`6??tah$ni@G@h6wK~BRXoSaM{GfnSvtTnl3zQ$~l_9GL&N<UtasBB|S
zuOB>DtxY+K5+9pw4y)tb6>Z1EqPmx<(AtZ)Af=IG%AO{!6|!>NG0P|L7`}JnI^V_5
zy<b+3%WU2m^NS~DaUCsR%R6BPoB4SS9`1F;{ATJtk2qPvD!Jw_Rp5~jxon!jox+tO
zP|y9K#!}?YWiDRPOZT}W?;PP?`gkVaTuooD#j~afH0+Dwd-rL%Y0~EzX3m><c)w}-
znSSw{%HMHuzTDJzC4Aq#x%mHG*WjPNXAjS29d|R9dNFR6mHuMuE<O>m^xem0opYXV
z!O^+Kdaa$jQdbqta*LUHPiKhmh1{s%lTw|^9qYK0YxjRg(;yEnv&*dQCOWLn=3bw;
z_{H{!n;ieYitqRRe(nv2-<#bz*~+cL?84`;DL^_(D1*OObT8+g70KLRJ<oGLn2^fX
zBP+z+6RXO5_xx&(r)526euq1_G+8I|w@tcYlKX5a*Nn}-O*Sw0;QD@QEtk$MZ@#G=
zA534oY2=QXJe}wEA9DfI?LYV)ge~G-c<mlXR9+9?#}CrnD#!VSA1pWHo0z3*9?`39
z_URE9pRfM~(;x<Se(Sk&C4yGH=4s*h%4e{Ph5s4H7S3yO4^7nPHSqk;IKbyTb0MEs
zUm@=$78O2z-l-<{>}&YAH8@NY{eSUJj+ww)=Ox8g#5;%UN5wSmYULfKaZ40U_i=nO
zDXQ@|RyZ(;&n}V8^igdzUw~B)$A(wVrr&cfaGcBB&0A~3Ev=Til}}#XlDnJ#I(PIN
zWBx_@419fTti0RJCh}w$?cxf1a@*APpaXa8<GXw}6-AB1lLYz1n3tOL2XE(#%bLV%
zvilwHX~Q`tuk+sVI7BYy@%wsFNOgW6AA?gKUvl#$E>C?m-nVPTx$1gu3vZj<#B1cz
zY&ti(%9OF>2G4tCWz*7|k$m4S?~*Z?cbE5jw>Y2p91gw-@*BAMZ?KuF{d~@Kr0WQ;
z+n2k-OZqvuPS04$`~Kk%(?G_JJiD^x%zU>V<7sr_;?=ny%e!QXG#8HpGgr~^+h*dd
zvSxDc?M(L=9X4jYAkF9d_m3Gz<2v4-D^s~QU3NBO{p`n8waSZENT^A6ilhwR&bDq&
zfrtQZz9;fLQ~W)6C5~`#arzc=H+4MWyl8sSq$<mbW4CS?pN8H+la0&favRU(H(Qo?
zn)~3ly<8j%HF;C@mzqQt&g5>Kew+Jn$Qpq&f%kc*#H#aNESKlZj+7FZAH~6Ke58ec
zg^`Hht4#@}iAh%G?+*VF*x)W<=2sph_)mR@Z0GvZd>&de1T)Kw1ZPS+a@V9iF@1QY
zj)#MPlR)<RZsixRIQb5onJv(^d#my9)ja&ZuldbXAD0V;_@@i_7%L0>O}@<4?WWG1
zx$3A%P3I)j=!JG>_OtezUA<Q%_^o`cabvHjV4v6%j&Cy?%qE?$;K-XhgYQU<E61th
z4+W$IUvlZpQ|HcIet_Tqt&f0W>{i})pD*$Ru%vQViWr!^PoBV?T`VftDZ_2Xl{<lN
z2IDW2uCp%u9)~OWcCYRfShx3q+5E+A0<r=<{GP9W3F#Z35Ln4BDrh?86#LDaJNYi}
zp1@Uo_bUHipEdjiYlY2^y<{<;kT8v3MxWoTMwMN_Ry$oP?<zarJ~t}?Cx!R?2Uy;6
zN<G_WvR9s$*YWfW{-jMV%D?ZZ@F|p-@~g~TZ~UJ9JD*V6Q$w%Ie*{jrvGJdIxs%T@
zp_%LJipkuStNcyRaV$28j?goe_ee0kaamg6Nt~_8gOls{|Gd=VESH*Q{OE)&=L?H4
zzE2f=9Ix2t@k>^H<H{6V$jv(M7ytV5mwY}!i+EptPULA?rOtEwU5%OCQAzH?%{%yC
z^WQR_<*<tHVppGW_vJ!<kJ2K(S1UC5YtKfS{*iynCmh1WS9>K+#L2mr-)Ch%zukvJ
z>{-g~{2f8U+_~@Ag=Xz(7tk@{wD_OD$~?s&Tj0;ymu5T7cM4oHUL$kNmx+I06Q^M8
z`_%&b-}iC{yu53A<(>$y;#xa_OUhL$r+Mb^g}&z%nBc5sdTv%Wzg*ILlUW6~1tz-P
z<B#-d=8uZz<z6gS&3$p{46~-&?@TS%=$mccea%$5;H>~#5|7zt;|76)`WoDdx6hbX
z-8bj@mh*=1_4nJda~h)s>=cW*eg+0}M^$GCJl(&ZU#e1v@AgT4o+gn4+&lGy&1WV(
z<yx#&D{yG{XOpYEQv5+70j3Xl-|%<566UWy`=9^t*+-@xJ1+5W{ePJ+sI^Jt(<3JV
zZi_?#u9zAQ-r{@wR<@J4<NDT!UR!xrpk(hAlURNev)t0d0_P@lnnXQP5R@*oQrNnK
zk>5}@T`)eJLvV*^I`@HV>&;3Nmh<JEI48X}eYWb?#65hEa}xyC`PG^gpP9}7l=G``
z;qU7L7w;MgeBAtxf6|g~Twkr$a32jSv2d)+GgGzKH%*Jrv}8SPD)?!toVoM;djfVj
z65L-(yv@7*ad2Id`OUXb&0c-`?=1r9#tqzWZcXE!8q_VYd7F@c$%`;P;hJ2Y6>hvd
z*RQslJoT*P%CNs8@bO}dnM#TSe|_Q^)7lVT0o4^&{7Q?x1d_ronckc1DNq^U&i_DE
zPVmjPc>*&zv;_AmujgWa&A=xlY{BVs@09REuNizQU+m2U`!<@+j9<s=FBxnWq%oa;
zU8%P;w<I^e@)suli0p9w5D5>S=@QDOawoia9y*=mGl;PexfN2+lfI0B|L=MyvmlF&
zyyu^*n5q8C5HgkY<eO@?pKoV%0Jr$by__pp4b2WtP&PBUIL~ww(^{kD-xc}0=6*8m
zlbFwU@Nfao<JZAvI~133OUd{0y<fgjUV48LzixUHx6XbJo-fT|+=WNe`6~Cua{vGS
zk$d0eI1c~bLeraFi@B_)_3-D|%a|=Z=FD?^?S3<X*LQg8*Zkx0o9E5D=Ghz*?ky=i
z+vWD~bi8~iXueO7pJ}HT-|M|joIW#>d2e0#%wcowobX=%Z@ly0UpCYCmp4l|pvL>n
zp3mIi&m%tNJyBB6*s}OOXGHQzMX%@UuUN!g)YM|Cs%FPsRc0&XdS;T!Nw#9{=TGeT
z{Ex3UtL0?oJ;Ug2){@8~^6ULv-q4lId{(>C*-vKLvrkmgGyjxnYi2w#!qiun!_<0O
zC*O1JMP`d`KIQdwjpDX$3o=u5(c-?*f1J1c@f!srjU{|)O7&b`Mu)g;r#o;ZUR=q0
zV}>HvqeZUV@yQkJhnFoi+jV&|*O!wg_*8!snf~b6$XyY%#Ee7Y0QXj*U)-&GHu3c6
zY%rO(tBw2Z|1;c+!nX>^Tc+_%4W7^Y!+SIPyR9Gjd!&AFM(qDBsCD|eK<$K|=9;rJ
z%oDXv3*7deXT}pIA^72Mg><3cO}=j*3I+Q)*#tfK;<%FkO)#}KS;52M(;<+)VTWq%
zob$XJ_7@49@=P<ekDtrebbN;CkJ?Eh%YT{)1id@YzkhuTm$b0~_w-LJW)_dmoAyYY
zH1%g#Y+9)2CwM9Ay6I=X*8;i|XR_OT<uVn_FXTLHxQTCV=zrl|nF|CavpREaJKfA(
z@^?0WExW9M!DBbxrsvyvVg>hb_i5CcKm8uaeZ~KtfS9|8>B4w_zO=acrt7Z=@*A!c
z;M?V!DR3e6f~kdqgMgSuAirAUDv_AJi2_ekE(nN9>T`VBqAT#QlbLI#m8nQny1L+m
z`3DWlEVdhb{U9L7v9sK4+k|vMrd7!z=ai!PH(p#RXtdEqFhnGb`^lBXM%6oldHQPi
z3UCCT6nA_q%=dog8i9Aa*-W$L1^MUOYnWWwr!07DX1l;P$#dd;>GoWIM6YozD-bYV
zYW>l8v2VPw@?;((p;`Tce13Bc-Q?5+y?q<GUnm?kUa)!{=iSXO`A%QsW}SBKn?T!w
z*Ias*b=)#G_xU-E;{=3l&hlP2zR#@@vWWZY<6h$%4>Y;C6BPtmx@Mc&pV`d!VEucO
zW2|ZX>oxcBac52yX!$hP<jT5k0Zxy({P#m&3#|EcOQ8OPrQif_JI>ddTs+Z_ia0h|
zJrUe5*vxa#MBV(2X1LkP`BFS=?NiN8#cJ_dzHN|FE)wO5lV{<zOnuEG{iBMD{cVHk
zBbR=T)F0D$3XMzDOw(6#Opu$)W5?}cChp(E-C|^AcH(}%2<sP49`^@~JPxnCITH3P
z;dprand!=JE@q)3-KIs(?@jrq@$!CIZfwSSH=Bn+N{92`cLB3>&Y7H6_U1eZl3H>L
zM9%UoIJTcHeB&F=MN1}fO*G`^Q8MV|)Ox|i?Y>cnlb46h{CDy~PPflWyhpZOG<97j
z$`v=i)og}aHpkjok2s$A-r(N6wbs<NtA^{9@)53x%ZCKCgZA^NAC%zvW!T2yn<>e=
zBq)lDb*jFA&+MDL_hjVE4)kQ2*4~ch&Aqk7?9PIve31o{q$jt}<t@@p;ajpfi|@s&
zd0dJAC!1DPaC7gx$HB+`PFQtIH8=M}Hda1~A6rbl=kxO(e7xN3i0Nvf!Z+2tS?4$N
ze#>~n@jv1r`^K_%Q=TLLOiu+qGWmG!l(A=s8J}-=gUKS>U%U*@Cv&~4ax{A?x}7s9
zG=(>4D~sBuqIACYIW=5sUWsxSZJWp)(>IZK+k=hV9pQ!C{Yz(Y+q_mZ%i5{KwVYuZ
zpX!MjX5z`oJZ~niGIP*3<LQ{bh-bEN0?%FBM`l7fJGecBym=}&rU{-5R^scLbcok#
z;u;Qn<{-YxvOvx+F$H{+PqqlTIXo~ES-H}Dz20oToo+1V$<8(W{96`@Em`ivtNbyO
zf32cBzrNgCu7}&X%$(wzc%Epm@IP+~)N=7z%p09Ci7$*t&{Xf+WWM|wAv5zCU&P!e
zDDg#E*zhs#oyhgF$%Z@Tca~{+!EV#VHR)z?s@kR(g*^H5?s}LUm%721vvLYY!wgk3
zo;x9&LFT7<xhJOZrBw0q`}`{AdVDO8TjW|Xzx8=5K4vyG-pNO2@$f3o;(qDbX|X+W
z4!7l7Y5q1{53_WCb-qN6N5=d|?(+5hzr<U2A&;*xSkml6!fD=OHzD46J6?%oZg|bN
z;L>)!M{~?McHCOb@BH*X=be+R!U9%H1P(}Pn_2R|G}heQEznh^XTIanMFH!}tRh@l
z4E)CyDGTZe9}w7lg@L<2zR|cryotMIi?u-CqNDt8c4_n8o2VyX@z~CEGkXNz;b3c%
ztk78kBInr!>bPd}pJlY*e5!eg^Rm=V<L@gXOp+f@H2wU6(QpO_r(o8(2L@-hEEEv3
zV&|6py3d&R+#}9?Geh~(&m9r-*^w)d6StRZ=}i}IhyB0!cFn!b&#|A8cPVEkx1y68
zx5lI8rp4dib5-%r7w`|THcNc;h4-GLg6Ygjy?ozfnE5sz<`Y<O*Vgo;5r+WZDk1)o
z$5sNnp9KgQ_)ZmYp8kNdHCUTBL${xGa>*Sb)<1`MXY<9F^6c1Yy!Smf?~Y?<&7RKQ
z!1rk<r_>xd5x#wAg!oLS^zy~8KgL~mInCr)s}*<PulKxrX9UTD#weXC_4phoPB44_
z{5{X=o+#6-8P9pPt!n4p%yXLe?D1oqa_0|nEV;yJZraCcCZxK{<eR~I<N03Re1*=k
zX0QLx<DHf@f%}5S0aKHUecbY^_VC`wX;nD;q>(Q=#+HkDDi617uM5}Vo_5|jb^AD(
z*`{%C6KNCOHD|BsvLnyAzFuCyCtTHS61~=$`_9YjrkgpYbKhICk(*~j7mwPK^~Q_Y
znz_4sE^yDS=M{)IH0RrYq?lLxzA@*mqvm||Rasn?MzaN@_T=*If49{%w?)qMZrMxT
z=iSH6(l&VWzp8SVYGdHzJ!<)%FS0|0zfz}(d*d@r(+}*7Ja<nW;oHsSBRyaID$j{4
z?R<_We;BL#hw*LfWHd{8(=BRuWHIk)DI>mR%|%?H3cTDdS1e5Cf7LcC-0Nv-scK;Q
zg84Jws;^%R*DtQ-lio0meQ|T0snS9&E_uI$yl<_11&?<<=li8*$t{`uiM#9n5<X8?
zL%u}MHlC^b+<1>ITFfnz+Gi%wlE^coQIY>gysp`yv=zKR7OXK@HI1Dw_sVsi<xOII
zGh!vpj%VHG3GH0N6IFFsP~rSqzRio{`2@TgIP^cf<Tc$U&v|O)DWTIb*ZC4;j+sUn
zr<$Hs?cpmss9<I<5x_rF+EGe;ra51OmjnOLsp|Y5YInKp*q@oKj^N?W-?V~H&@@Ep
za;Go%fzA8*W(H0(jbszy<zoMBI_-#}@R2Fed<%Jk`PvTWa74fHWjm6w+r(hX4b%A&
zk)|om?IxGss`BTrK4HSwcZE-^!iIaF<|5PN<}$8TlOORWoSdPey^fv#oF)(VF{Nnk
z>h^t{^H#a@bttQG>zkKz`!#8CcU@CAJCXQ>Yu!m({#`#qP0Rk<^Aw+5W5Q#8iCast
zhsVugGVfHa6Q*HL7Vt)88S*|8P7>O~f0gg)x`TWbn(x>@&b`ey|H&=(eQaWa7g%ob
zKmXEcw&&47vz~<;_$U6pVs=r4Mc{*vf|wL%IiHuekHGBZ%mSfzmT|3PJZsY3?#EGb
zyp%u9>bCN29UCsgJth1bj<TBWKO4%EEw5uLZZciO{*W=hqTN=$F6sNMt86XVyS}KH
z-8k24S{ZiP^rn=C>EE070`{S%rnX;B^Y1qN$*CR|ZW@355NFrx6TFK#%;h&`&*JCW
zFq<RC^DL)po+i%|S#EwmM}MwgkHWY`cJAS<id$s%{L~Rn&Y8FPgL~$f?v>rZy;J{|
z$=|t$xl6z0a=$Nf=AW?lyU8?tP5!DhVZP1GTq45nyZP^Wt>M3{JCSR5qXJ*?>A4(-
zZ=4hk3tq!#FFwa?*Xy08dv2WMjnGs!tGu;=zxqUz^ve~-{81J1{FZm}_~qvq^Tg<P
zo385!;aT4EgfG2{m;VjNbe@w8{QQzfuA9zxIM2H+*WPR^-)dgR{#d@&Ef4t)-?rgi
zeQpIOn~IDXW7vMv6|vf;Cm2>537T2(KR+OAcJk{1KBb^)o-fzE%+4>1<yLE&#usq?
zsocl*CjL()>$$J5vFB;!wC6SsXy$8<-@@(CxRJa1u>zO7TC^GS+F4w8rRVZ5pZ>$t
z&~gz^Ysw+B+GZ}^)_6DG-HSYVc`s;~ZrESMGqvXmkINZjp~X9u`B_(2@o@>Ma|Q@}
z<C73l;|w-$<$b^S8GqmNNhT94`poZFZsy;)p50>Veh~q6%ZW16qu=mOoEj#ObY4hc
zLr?;DsHVNyznSbDbD6sN(>uast?x*2IaD<BPwby&pu?@f>lCrm<k)ULf%pz%e&N`2
zeCtDBa~!y9&2jn422<_}SIr)|NSSF?HJa~T?J2<fZJn`u+ckb^QzlN~9lECdQh}T<
z1*>_jk`~Cd8m#5NvMG@BfA&<a-K$P>XSGW4_Z*(XonY0%W8v1q_4%Zi*`9fg+y{hy
z@JF@gnxDU-$&)qZs_D-P74BbJUEFI76ZpT_oivlZXwJXFU4@Ug!a;PG{A~V`gnj%$
zO&pxhW<TX`6tm?zmJ%=2({)|IbiqFJ&F$<Ki|1_;P>j|$_cf9i{Psme>TB^-{twH;
z1#RE*2<}`L%=2A;n(5}NoV?7_rwe?{bX00=+{$MpQY7#$cb92a-+X@8x0}s&RyYY6
zx0?!_di|82Aw`^9S!@Y+yTD_!>5(f;nYJD_6Af)LIn?heC^$>f?8fsG0>5Yc;LdNA
zGTZ&yjoXjWj=!qwn#>{2Z35e*CU7-5KIM+|?iKhYBO>4t5ydw}y^^OTL4im3?|U=(
zyDPc10<Q~vx#DZS>w+i$BaeTk;+HuDT&f)S6))QhgdcJ+dz8;3Fz2rl|0Rh4(Ja4(
z0#Z}g3s~l>a#epk$s=-rl`BNSl8<Z3F5dYuxn?)gj++^5)#YtioM9H;UdkuceoSV&
zTn=y4HG94-KWzApwO!?&x!}BM!&+7@30DR_%frW2#PzmtM8?18o&J-<wC~+so_CAH
z%_O$UiIg2F<8|oo<$Zc6l`G@>40f(s8PnS*+s&Ra{x!W96l1ob%#QD1oQuf`?Q6VS
z?0C7Bt1UBKo87@xoZ-i78MKsb;jv)8`4fL}h^}JdlIHW}I2YN@yJEU9chYoro~30o
zMCM0KFyryA=JGO|&o@u!uj#C_!rYwe{+TSewutk1m^Sx~lFdBFFIAiEjF#s<6j#F2
zHT9KHP_``JqtA1B>vb%+nC^M<7L*%tZ2eO$BqubFcWTaLv*5W7rUt#2c^qbCTJ(zc
z@x@#Zl5q@X<NYM2#8(jJ#dqk|ZtlDzcg+}-4LRjHAM;9`5tiFCL7(%bw!2``#%5EM
zyv;n*6UEJLZIR%OOy}lRnmw6!$y8&muryW9hTV%yLrpfDt&+KAwn9kXyyc`8ANP$P
z#&SD%^Tu#)=iGR+9CU^imx7!M?=_w`avnn7e6R8iIGgyFasAVM%*7(w%*$XjjjQpD
zFVDBwV;r{>jm)PVR^~qM-^kbF$7Hcs^cI)#ZGN*8_uq479hc%1*^<wD%Xxu0JBuif
zQ_dCcM1fqv+7)bkMm8<HaZL7{C$7BVpSN9^J9agn2$#f1fwhNZ%o>9w3^_zD3AA!(
zT0EOCCa9y7Bz<)2TE6JNC4$<SoPt%yW4J{`)XbJNIPoY)bqNGNnl8uZaE@25Zmz(Q
zR9o|JZ+GzB<FYjC`S?m8_oB7Hp4QcpGdMW8;{K*`-F9O#7gP^2bAEo`OgiwTp>vSC
zV1CyNQ@=Uy1$JLC;`~sfW)`~n9f#HFM!uQ{>*NnvEf%OTmgD?+_bV4;We2|#zr283
zz(n4aIi5UXXIn)oZq}Gp<d$*Au6Zb+|LcIc{AqtayRA#j*jUB+kL3ICIb4YnxTUIN
zzW;TQz-Rds{@eQf0!?L81&*J%Ex^28hI8xoSl(mlu3RQ!I>O~iNqif-jLi%sZyJYu
z{=&QK!3wkMci8!N-!hSTq-xApv*anC?ui3@D;p%ZHD13o{Z=c&%~n#+_s+mw`M8oJ
z_cZ@fJ`=CSW*TRcczv?wnaL!F3I_`Q;Y|rP;`_z5fb;jWr|dNc=b0w{$Tr*5^V($E
z%^AiUUeDk&kX>TR&s4*gl;X#|q~euXPm37WpVs5N6<rHu{ye_UXSr09+e={$H)Gf#
zu9$WSzK!{(xfSB;xZjGN<o<d&(e&!QPOfu7-2BHr|1e8rp2K5&bej3J4>~*>e1G%I
zylcl}6Y$*h#ME;<QI7n)3D0W<o?e~Dr`lZ3XFOvOht}RM{)^`9+-YiR!r`mh1um}R
zHmmGuF)?;e7g*PD-aJ3)u)uoT1(K68|MM+d&MCM}VV%H+&RyJh^%j`sxL)DWu67bw
zb5~Jj-KBoMl^+EJet1na;Z^M5@1C>Sq?xH!U}NDUew~$l{CWKcxY+p;xqFW#8b3KO
z(e#nXBhxvmvy8cgp9(y9bWpGGf3#3h-E8jqdHakP%nak&z;}sn$zL9^+Lw6(KI|#n
zhIi|^WkezcUgd4&|HNa-XJDkwlX%QdP}J7X?9IAWT-DKy0yiJAnd|E+@_U}^H<j1C
z&wnCFn}5aCyZozjW}DeWUFSa=bB%9aVw8aMMGpZ*&OCu{;>lb;a!q){X07GAdg`^%
z!Qa1m`-)7>)^ATRGhE=p>$F(Utik;<Ux+rp)WP~Qy!@uK`O4E<_*z9dc$R7lnX+?C
z<$9;)%9s68Lr#4&GuOqV%6zuXt4%kx^YT8~d(%{P_c{LcwmrOFuCI7g7cg<Rs4#Gt
zPi`=Fe{sdcZRK9m^7sgo!1@Bdsiz(qip<mCJ1!*1Rd`q1?CiM?&TVzBybkmKvZW`l
z;<Gb4#&IimAy=>w1Gn++lf11vS99|Ta`0IEKFFoFfYI!=@N}+d*2jE#EYW6Hji+&+
z^Yu4N&tm6x$iBp_mB_|hk<4Uz{RBTx=h6i{vwi*w<x6Gp>CSk{dqT>TL$SS=pD*Pw
z$6dQXfz|HS0@Edf&2vTenFqQ@2>cB>YsUF>mOxb57HOu^YkU_BnFO`vCJ3}1p2;OA
zz1cJ+O`E6OUqax+3R|VM<-d4-PvH>Aj^{PL&wYXS<-2N=b#e&;Z4IaRMVf5*PfWhW
z`P1t?*W};tOrxa(Ol8h%m_545V&)+8S|HtRhKZJOoj{-dRJM-~T}-&rrgHv&+{<@h
zem@&yl8%6=iaRIUx6|DJb&m7tnXTm)h|1x4{9zN%Tro=?Vej*1IqCDb(;R99_+Go3
zE&V^6m+y0z$(IxvzS9l@yq25{0ybF_%ozV~;1A6>&v$wDMUi6{<ps8fI|?YRyu$6g
z;TWIkRyVHl@D=<s4>a>})diYPboVw}DpAUN=$xd*r;z#lYj$stys>UG->WII{Ov_r
z{9*pDd7d2NG&^*1I=6=Rb-sKLF4^tB)wqjh-{!O8TWxZ)`!KI^|18s2^_sk&w=45y
zsP*&tzP-ae`JD^*0ihix-#6QtsZUKXz3`~kR7+Wr--0c~=)6KNzf$f_?!zg|jGz9T
z$Gz}c9A9<!aY^ghN&Gw~4Y;aVSa~k<XYq7zFX0P)c9r`>8XIrG6h^Mn!*!;6A2M)P
zf0)F-aao-CuZUgTa`h=@K5<96L)ZV|-m^)E@7r-<^GC1ObKh*e&GXNzMd*0vPrkIR
z%X!u3@pAjzf5mHbeky0_Bptq8qIY<G)<qd7y|Xa?{-2$fk#Dzo>E5e+f<gT<DRcPv
z9Cz3At(;NAHzUH4CqzTeY#OIDx8)f{z8x1Dr2Xg4;#y;%#CJ=()Ub`$f%l-cld19I
z-8`X{&AbdAw|F<$KjkXF7Qv;aqi%YC#cngxxOt|xQ*N5|#<%jNslPUI-y+G^$@PV+
zEt${M;KFCFvh>ZoK1+p|FWp|mr&q9w^V#W2ZawuT?&DQkcukg0<31WM%ro6Rk1NDz
zyJ_0T4P1Ne&hS}%vNey;U(0=@oyW|eA(eaD@{`<WBYb)1>5G^zlw;xc;C1EceB&TI
zr{9JzX`vG@`_m)rhZlA5*t7JpZ+xaK_#`G;jOBg3`M<Pl<|ls3@D$o;n@3tV^DdIp
zlbSP2n0KE=FmG*|60i28cbuYUkC?8J{LMMd<1J5*@>Y$9*%LWWpTEPC^m(nB{{C0o
zKD$1deSg9zcC5vk=S$Bnp1)P|I6j`cz$X8?*2I5$xY^}^*Jg*GUNl)??Z&%8Bi~RX
z<QI?jYI)8z5f98n)*t7n{AkYeA?OAZOGz+qyrB*!SLSA}Ny~0<i21kj1Wzd9>SXKY
zY8UL|GGf?eE^?-gOQLi&Z;DovdG)LaZvU8TW^3emxjt{T=e{Plgj@K<3)B8n`CL0!
z_Hf6`xeIUoz{gwk-j9bfYc=<WoIF0>vk$m3tRwg=W#ae@BDu{hk`&Bs5A*OInY_kK
zJFJi2wYg9BSl(*BpMN;{#p5*j3(nu?@%zVPR`If%`{^owG0lrlgceT|<t{5f$7i#7
zf$>4nnY`?`xlC<DS$XwlN%5VUIgxMDmA~9vk;}MbW*D2^*%xJ2@SfGI+TF|idNdb*
zm`aba#e&s*3yhn%H?0shD}0~Ltvl6@Z>p%Jn2bXf|MPFxIZs9CaC7$8bGxg?@>RJX
z;(k$cjVE>fFSZ4B+GeR!q&Od(n#8YitkZPwcNy+X!>cCWms)e1yME@j_u=L{ZpdsF
zeSJRn&diNGN<Bux*Pj01YtcEy`~B1ljxF8O`HbD?b1r=OS};h{oNwiFU-SJBcbcu^
zeZb4`Uf!%E#f9JP=3~)^+k|;<Z2!V{nfo`Nh}8tH__S*#WqB1mOSpFO)tGy$>PE%$
zcJ8p?dl{}{x;Zb2FWW=TwCXL3NM2(LFOT*&UW?~eTx}+C+}+EiOzod{nwlQgH(h66
zXd2{jl+X74eA6s}K)(0eud-}b4l#+7>E)EzyPx;g*9kHk#dh#1C*9@}_;#Ip+fNPt
zsef7cY}a1osp0;~)3V|hw`aPOIm=dNp8vC+@jbrdYx3s%58g*dR7_*Fy7}IPr|?R7
zyx={q^v>k2p$$*q!#O;6LvuyCH74>UPZ8%!$==PCJ<Evqhr$ugQs0Tf@0U08b|@}0
zJ67}D)THPv&n)?JvpB{GzH@hYrPBEy@gCxq;5)sL0koEZdrNwlDeL<>u3!Il@;b)U
z@y>OT<ywDl2XD+_Ewhtz_VS4L$e8_jlg48(ONw_9qbu+2WqDk;*7b1BTg+)*V0qqD
zj;q~NmD}6+@oQPWr{|`c^);{IeaPU=-PGk~)_O^sYwcVw-e@%$g=>;>e3GAyIqTdK
zxied}dHNq&@b1c7z~ym%E%#2RC0vWX9x;8q;~J;*)Cj(Ho4%UZ2c>hj^M{*x-95)O
z!QdI!(a`rif5jG?%-+n$t<bfX+d_Q4(6{(oy!uP#@LYK!!g+I(jDXC6C0zV&<~-Sw
zB7$;f*v)=KuQX4c@>}5gepU0lYr%rrObeuEm*3`7a+x8Rb;wxIwm5@(#)6Ng(}Fhg
zqzi8nXl2qAvlrp!Q++c>KtY7tm_3!5|B<<kslv@9!K)f60=LwZ1g158=UQd1$$dd`
zyQ$&ZBc?_tOwGE^YnfFps1_`Wo@LyVEGlRb{Ep*?O@`@*1D%{{99#HQ6ht|u^WGPj
z6}p5=?m#y8Dz)SM&sX~j6a>%aWe?}zt!Yr=w(t`+o4;}gcTk<Epqa#9v&IeGeD_b-
zn%Y)-^6%FO<-5Ffia_y%=Vn$fngkYpZ{?qNIa4Hh!7+h<iGqT!!~@um|6j&8*@}(3
z*LJngD)u$}OaEUt;m)Wq3)(oJf3IVOQGPYMK({Eb+|8(Gyz7@*3M`%Qf`5yiJlFYk
z2TW__SMkWWOyhs7x>`bqN0aaA7kz%GUo(ssPT=LY;!87q9X6AHk`gb!@U=^PN2?BS
zy?Atso6l^XiI~=0<Mnw(CYe%~O>9nm=Py4s!}QSJjr{os1vu~Y3YqUbuE~+DV#zm4
z|BdA2*ya2Sp0jXUeCy-(aeT}F+T{hGT=+d+h2v|u6#_YVl7yz2tX*-H%bRxx|C<{(
zP0O!r<hx$<)%astG5>1wg?t&+;{4jO$4!@JDDVqj65;!KMo>`nL_dFk=wkj2fx4`E
zj?Mhu8N%F_qW6VE8`=djb^A=u=kGQvGsqQK^OeO&(0qqL*g_roV`}I4tlx182G3b3
zux@7xclj0%v#(qnynAQa2`JA}lfPp<i|@4$w*Xf!r>WAyX8sfXNoLEIGztivy3W5@
zsgZy3>mOXLo)KKEGjhyJ<_Vka`?<|zD{GmFB;y@{5U)V<cP<SAmv<lGI=cLox#2%=
z4i>2+e5x@4O5yvH1ny1M;(C75ntRcoP=SL2>-p8fUHB5y+<ETjx$#W1(Ko#(5X}8n
zyFnmj7LR$*3km+cbJR>(AKu{i&r{)-n*M}8*4@nPedH^CS*3@39IxgG=wEgh*l;aP
zAoJA=R?zwX%P+j<lKMDb)L7=8!246#roR?)nw2*m7I<E!Yq6PKUQqaptitMjEd1SB
z8G`E3tb!JHLEPVOE0|B&d5EvtvR(Ri>?~FB%j@|(SF;IatjRX(WLm(_>aoS-#HVWl
zK79J%^Z&hm&*sjyIK}mVCEUW`Ot4wHE0ftN?ovy^>DGepzB8C^?(7wEVcWxXs+`@d
z+qIiBbMk$@+3T#;PjK!O&^g!0z3kFD?rG^20vlL_1U|Cm^BJ6I2cQ3cdb_sSjFoG+
zub#UiAh5E_yd=<pKfZ#;O!3Vpey>+P{0f&X1uXjX&Eq#Y3Eax^;}0lXBf#}xfdI!U
zPQiWqS8xUQit(MkKbu|0q+Mhc!!$mwx8F?{TTC{6wqP!=llMfU($<;$ZZRumCUSc4
zpWeX8&y^9z&!67J^CCLMgjchZ+xXl?zE<<!vLBYP@T5%t#kc>1lbKriCf@4XLT2wu
zdwIoOZ1@&UxX0)J@)1|t@gB}S+kDK8G-Ax&&(k;kFtx|P_Ma@j{qA*UpSSK4US-C@
zb1J;YEbZx9?!1ONK3-1+rAPLe{1?qsxi>#L%FX@$B$uCcCf}l6x432>zRKlaRmQga
z{c_V6Eh5}KPrLaSIJTLtE0g6ZUr}vVB$>}68@QY2*pB_YXHSb7y%D+3bIRg8k9hJ#
z0fqPC{L8w7`K)f+a6H)_!@K_Vch*?9NRdN<yyDWc=9yY`vzp$!rp9Y|!raI?<u)Hv
zv%jo=bQd3=Xei&h8LRlDg648B(&jQ<>{P?G@|G;0{5&=3clY|aB>vd&dCZ?^CR`!G
z>tg!E)T&gH*W_+L@5E#czMpHpurDvPW&I*_%*-t**Q{NCzUh_TbNW>u^Z9Z_Z=3zz
z@Q{~F&XsG%CqJ{lPp5IoXl~-&yD&gu*{;QWcmGOp1?k0eW&e1_+2*%~*Doi4D>6-(
ztI}^J+ul${GyTsET&D#N@_m`pYV4Z5g<Id}gITrZ9IiedMjrbNE8au*4;UQ?e8R(d
z`5*T}K3l=XZ?pM!emTn<`)fD5X2dIg#h#U1iJQ+0Yn^^BAh|Bdlx0q<+3U(P0;@mw
z8m&AeE|?OtOV0g$8()J%k>L3stb!J6)^lws<}jO=mcXN2*C|kR`m2nO=w03~yK)8o
zy^An8_Gu?yn6ZkP7s~^Ig?ttQmsULB7rN5TbyKvLn|Jy`v-rJhO;_$NH(710WO7=|
zMvz_SquB(rw*qUmpRnCH+HSsd))o%d_;S8Er#8s1vt1$}8QsUl?j*_0n>LMKO<PVN
z`$8A5R__#^8iN?_6Ak-K4$9r+I>+=_AX2!+ti&vqZ+W_!>GVKZ{!{nt_?XLr1xy^D
znjWYR64-etgFpAiR{`h0lLcZwJQ7H%4re_*OGjXr^#QI$%VkAQ<f;oMNSm70%F3F)
zeabJG`TL!zXndOBcV#Exja$6=3udnpG=J|bC><QZ&GMts$i`?pPmTKlfw|%S0y!IG
z`7&Bo2|TNkH}&2o#=j}g&p1X?S+HBKS>X78V}a$TFLJ%?jN_C}X*NmEykla2{fbF8
zYmt#x$|S-3`9dZhN@{`}TR(E`m6>g_xyP8(oM9`Udl!S8zuqr_;-qvg*X?cG&VL^A
zZ^(-kn04+fuMhu9o-ZvbJQJ?#nLG=h$=!KOQ80dInd$GywR{B=znZLV3FX&s+swyX
z)F}{eFyE+sN0)%Q*L?n~oNomFG2Rz=d_Z2XZ-W>o&k13k#+3^=)HgE=znD_T^VoEQ
ziS)ltlfJ(yJlB_OGFGxO;XQh#L8dEPfM-V}1Fx~_f1ZC%i@BuVdYh)1Oyum_FpcMn
zbRJJM(>)I1DSbSuR%=ZchO}_sxyxv_US5!A+NM9;yZv2xf?B6?99j_1ad>~T*|N7x
zW}c^;OinI7Zn!bwD9^txJF|`>g`l%UI6S-5%vN((aV9dm@tpIqmF}(G&$H&rJ@&Ix
zPjjAE<K~|CT82k`aSG=gUM;SzYbUYqx4vK+w4t5D_KhrWSj105*1JhuW{a84TC?tR
zb#HsinWxXobMW9H<7K_4xu?ZH;`;LC7vI|{yLjXlH}W{YT*iL8zJS*<<~aL5T}k1y
zjyHJUE1H|_tl4dHYnKb}iKnm4Zf@Sj#}$?@^O!4{&t!QaAD@2#AOEb!Tv0;vO#go7
z;+8jI;Y$lWpjPu-g_|pfm9M^8-1Ji7d7cRipP3nz9uaY^ui<_C@+hz1(^H&rM<cnq
zv@Fe*J(V;|Sn<rn?{lwdk#jTOaTh)_NyRU`;&%^negA#X?97oOPP6jKyaqM*q>^u>
z@I`iMa`irm<o@m)$g^Cmjd#kH3T{zh25x<^B<{|MWo9>Kg>(I9o5t5Jdec;9Y6wrJ
z?;NwsJ3n!&+bi)%oM+&TzL{xua$ObICgZo<cVy=Y`nPNINl2vd-t!aYWK1#RWBo9R
zi|5>Wq3v#0_)@}}%)Djo&AV=H;QP!bZF*sT1HT35KbhY3Uc8$eQ~7K6dGPP=Im&hW
zxu_Xauo@4`JqCVF=c^jI<;!`0ZSLn|zxCbF?C?@PR_$=J^={l^+n4L})k#_NZ84k4
zbs%m6_uP*wOe(MNn3SJ7XFBKN9FrzVaemR09;WX1@A0(-tl^BA|He#2s-5%ZnRC2`
zqO)Wo`NjBG1svxpFE!=P3d-c4;$hC$@+O?O$Xb@?)GIsg&39caN<SNL>z<Y558T6H
zHg}~7pFydfX?WE)zSh@Vd?A6cd~;0Cnr`3sfj7lpfVVl}rck8%dp@(~BYYV)r`guo
zF5wqCug!USySb2N{t^K#rBkNrma=C1^tuJ!vPGHgX}BOTDPe`6O6xDa+l?xMv(@(t
z)L(hd#Toa=_{x`q+_OH|2w00wR8LgZ<GngjS76Px2POhh@qB0dW*aB_>59B-Wf7R$
zKb?QGd?6R7mJMgFB!{U~y1A)?H?!%3m}(QtCSJieDbtK6)+`i=ZEE1kW}RemZC4d%
zc(V@QWRHd7)*%G~uXWnECcfn6PU>Ldmk_+ce^-`=cQ2bdPvoan+)q}Rn#Lzaao;mp
zAdvE=)^x&$x4iF`oibVZJAscal!0$Y8MD9&7j6?*BUXWsdxHFyX1v17WP=2HLiz=C
z{8n+!=#%07_*R?khG?Ylr~ETwdJk>P4vCqV*j;AhwO_N-B=+_iz7^jyq)*&T<r91%
z$j2z&&3E}WH_y{8dyK<&2XNi9mJ!yi_@I1Tbt_k8)e3>qd(~!qJS@DMd9IkOy(Pfw
zy{>_Gk{}zO&Ei!Yt1nJv-E$<?EIY@*Ojw7_%*jyKC}N>CpYqENv(EK%ctvzBbG`lg
z-ZXuMD|gzA?YycdQ{|m(n)nKYnYnoW9pLIZ>(BWquZ>sglL%MKByFx$uiH4<|6VfP
z!5YtH`FIZBjCM)WCjpt<a{|=Ng!H#@vF%~wnKyxf*InwV@l(N0o}z%?+`s>><x_iZ
z%V(6aiFcoy71!}qs(jAjeO!tAcks!z=krCUOf|FMX*9jFLZ0tMp{d!@^gw<F9Z6}w
zKi7CPW_{%AELP)JowktsPT(YyXX=vNvF*F~*tcnFPJeih=kuLzzScq()5Vj^`O4p`
zm^oE05;OnufOpGfJ-+Q5|8u6M+~Ypg&t~##`wCNO9|qGIEsIPK+pOpFiIOyp`qRc&
z!r96h;(6NiY4|m+lL;JrUD+2nc1ykID`IKmUjMeAyN`7-Urw4P-`zKtdCV=J^O(CF
z=W6~dWIpXY3(vM38UCs%RVF9q?c$xX#@^IM#ggyWeq-JXKPC9&JVMPnZp`K3Sh$j>
z=lxkB>G{j}<_cHyZ7^!!s5qp{cSdzRM?+Pp@cxI_`Q#nl%_P~M81IU1=UdF~X}b1H
zH2;+BTxs`^V|?o`IrFo0Yw}-vWXK)tv(IEpr7rh<pJjZ4uDjI>Hk{?ktlYt8v#Zav
z@!3zF?6?)CK1D{N*Ov$JDN5w=X+HYFx%_G`XTsrp)6_S|O*y+nO$+CkndT<P@F#_?
zFwIuGz*q9~4wrR_yqWGdCa$BF%zV4;X2>>6aqxdSu#+o3+L=33aWS`4jWu6Sn;iF@
zObPCd#=P8i&%(@H4RyG8G}!W&#_u;dY;3?2Xl!pf_4+@q6<V%5JS(p7>gBeZCUTYV
zzNqEp-6-@~Kxf+>K3?I)e8MlH*yVbz^I24XV7vEplTfSGEq<%$E2eYh=bAap+sOZG
z+5+Q+|Cj{UsTRw)%gOQO#`_4U&SesKQRKo^EvIQ}`|d7B?ZPtty+7*Y%1b%9nmUU4
z`L=yBQJOWE$N#CJsp|9D!YA5{_}!AW^I0z}VSD{dnnUeUv#H2eYt#3ML8jAZG8<jk
zt|!3p{gFuu$7z1`HA^^lwrn&t*OcTe6>a1_lKW3~+0PmL+uB??e)w~7PJQFY!=1y$
zU&{KJD=cgtcc57v``e42rZ*2%ah~G7&42uUuxaA0-Q0;U)J-3Y{^AyEUd~;>X~Vy)
zV!p}dR0n>2ei=Tc`2T|M4|MbIb>GO(SrNr`+Blfcl}(eg^-_fhv-V29|G#&coU&hH
z;_AJVw|YaY?&-iy{L%(lvhAJm{K0%O{MY$%_$$1<dCa$cH`ZJ~gImk*J|ByovD}9n
z-?-Vex%uzC(lKibxXY^`^~Ur|+<e}kelNb44}S6Wx(IUD>3eeRofl-5y~*Ef(UM&z
znUiw#J<jX$EB<+DmKU~%Z*kx$?z<aQO>J9>xJ{Uw_}20-RJa@4!oTrjAa`OKBae{E
z1Mc!4&3wBQb-AtgN^%z%_;Q-;=P~`dID$Lq&uso%a>q<$`{H;Cd7hg(f6U@3dvlAY
z$@32Hqof6fx*oi|wf7(Mti7QjaDJ^Cf5G{1zVBD(aK2P#<(Iz8#x?y)mEh^W&-o{5
zW|%qN7Bo)@-o(%HA=iBWbO`|;UVS-{*z>&Vb3z3e{6z#-KV86mwav|J!chsXr&GK5
z=gF;=7q)2Q`dQn`FQ9hXv?Z2{S8KV9`NIxX;j}ax{_eD0e7uR9*b_2mbN;PUGL^sK
zZgxjj$*kL|-prRTRNxWMVH45d8~i@5Ok5YX9W!ftvW9EhOh&$QodIebYu55x%#h+N
zyxG82B9_HtS|-ImeYGX`i<nzHIwzNKIq%tE=K0c!`~0z={JJ|&m=%Pv@o1N;nt88Z
z$IYevj4S6tB>!ULjpk8oto+6)Qha}Q778EcnZ?h!eI5V)k|k_x9Jl$`JXppp#gHv5
z)OJlk&VIGo8s1)W0jn(nA$!eCG<M1eE<W~GHispI|7%~k;HPRHK{;m&o;RjDO?Tgv
z<1K4@EcE2cZTVNFyZKaJ<_M_VS#SEsX%By)R<xPn3w6PLGTH)WmEZYYUp(iUHTgNG
z`jZ*v@$bFON{w%s+IgKc7XPg$c;arN`8Ao70@_>dad}+lGS_r3<4Vh5<F8k9Q+jrF
zhrq6n5?s?Pws99Sb_ndP5D`$-tmbQES;f=esmY^dGtG4Kq*>f+`|b#=u;n&CFYm>_
zU{jT;)goR2kECG!X{XHuzHXXgx-ZjCK&4WLf8qTQk<9{21PV{>6EOO($<<hE!#nHs
zH+IK&212K$B*iCwpJLY6Az?blPJws&UIDWQ+?{+YIRfPr&Z+Q;E^_8e)OF^288DeU
zxM!NF#^hEmn=OBMuYdZevFmsmm!8NcUX3ODO`ESs@jR3jGc((KUNlOrjCX<7Lf$V)
zy_|KcdpUi@pPG1gA2!W*y<yU@bgrp=R2*Ne@;1{+`j>cTDL&$Q^k;|JrwhtlIaem|
z?tfq{t!w1Zx6;Ol<Fm&bF5agO++uyzycZ6bbH(}m=icpK%T;&V+w7C$V$MU&bNMc9
zC^E5gQ{rB@=CG-I0U!6>LMHC2iR!#}7?aG}7oXrV(Bj~Ex~M|%ZIcS0v#&AldWOkd
zJdEDFLKb^DG#xMSEt6TmyL#Dr^DE!;&2#*jd5^L%S@68=<GYe^UAkrcah|=wa(ums
zo_r~RQ@Ph?A2!_)X2ltJ=^C%LQIqQIr+S=R{7-mK%{^^)V)qW7n1bVG=k|sPbFb9p
zt=d(|`)=n)&Z({ZTqoqU%$V1yo1af;FcZzTFl$j~<Xff`Z|YrklvmLsmCNm(s@V@u
zM((F?%6U1Cn5pH-`0^dI-plEv>BZ$KFoDbcN+YjeXfs#VY#E*nZS&b3yBN%!C(q{k
zeZPUv%8S$d+|O%V7VO$)_d{QEExF>xb=<9jH{hzOdCAI&+;=9P;;!1^CY)f!$d~f1
zg}3(;H;1L;H~u8uD6WI~D}?7re-a2@bj^Iy!&hdxnHL1=7><}nHAx6sv454U^u5Wa
z#9k(Nvzb$n>*-|fX?jnMmG8#z9Ai`zov9w8<e_<jm&2q?;FH2+bI-5^d~Y}`Oe@-U
z38vh35a`%+h`-;MkBiaVgUj0akJ-0I74y4?Ma=u6CYapnix<?q*J^fA;hjK5=3K6T
zOPkDo%zDgaJZ}-7$stz-`Ke0;I7>fr%wOWey>eAE|E71+0?$^r@S3rR@kWXn^B8*1
zH=7gV&b?mek$}$)EAyS<-h4^N+RWZRe8#7DLYME-bYFpY|2vF>K4=Iur$qC=G}<8K
zvV5Ar0+rJO4gVN8#hYL8vL&uzE8g@``1-n3zDPC}GsUTeraU{I@Sb(IHQ89s%Ri-G
zNSagP248R78@_+%j_{?kMsaVgyJ^BA+RUBhSH#!ttt&P84JUWFMiE~@(kU~&Y9n6X
z{v~Eb=el@>`M&UeN^<2Z=U&J4`FJ0fw%r;th4ad0MZKp?Isfc5@(^L-zxXNE?97Qm
zz860}aBb=SXqsdChI57p51(j^jDpgdQ+!VzzvP+~x`aE?sglPvl!b4<_B`&;E%Mx|
z9qrsz)^=uV*bKOr$Z+x>d%w}tL%xG2kN=k0#*AX_8}So(1oz7GT8hRP*M7dlJ+HEl
zr+oHdJ}%x_eAx{md>2c+SsgUH`2Fjja(V1uFIcdxT|n?xnb{}Bc(WPDQv@Dv6*U*_
zIwUY5_PVH4hXnu9xm<#ZHfsc!dzW$VxzJ+LaZQ2Oe43*`;GAX^$(CNeW($6S4-XHS
z?w!-cuk19-guVZ`NQUN3{%=2f`1cDobNBdc<E-d6F<thV%`8bP!Sr~^4AXf+p9R_v
z#hR!VH3>Mp=I4I0cek-8i!O)yvz>er<}RuZ8w&&$eYfIP{JNRDx5`_<#%vS++;vWT
zacY4)mrt$c`8M;WsXpsA?)~>01rB`pYI-6-fuFfe+Vr^o5&jBmJ$@ViTl{iD5~fF{
z-{8N0|2*G+CO(mqGkpYdDl-KF`X+F7o#EjN|FMPh<t+_i(Mvyhr*BwknzwP5*|uhL
z-pZW6#@QU7_^wY;m*{EL;0sls&9^hZlTRXk0e4_YpsD=nXIw(1_IyGm3sj?8Z*eK^
z*5*^U5I2nvoy#-vS)Xagc01wCe2aK_H(uejVRhv4GBf3hU#(+WQQK^ieW=Q`?S8#+
zlF9<UimBly7j0DdET+!jTA;Jc^!0}+Ts-xAd2cw+5$9UIl20pMfTK}ZlzYyjAKc#O
z_whERF>)XLlE(9a;RM$mNd>c$Uxc{Q@}BUuXx=epePhmTD6+zoX{9{(wPiQBGCue6
z+RFYiGVu%Ln!ACA=Tt_w$eDt4zE7_DykBd)I6@w_@IU`t#U)$BDQq>NO2Ca-!0bXN
zxA}&SSb_DeT4pWJXA1nPjF;1AHsIT&#U%JqWuic*Pas$QW_`2sbLMlKCP@hFo_|gz
zEJBcXp*5?3ta7%=)oKMkB}a8LUWasnmf0uxZC-}+&&d(x`lvUNTVP+M>8BNTCV!dw
zO%0u2o0dP{Cy;hY*vwO|LEyHx1^Y=k2lGO~shk0O6!;d)`^r9gt0S=MvnW??VKq0a
z=S{v<KiBe`gxuvZO9<k*dt92!dTFrf9qUf6&q@sfxlB!FtcF{84P(xk)Kr!5B^^)W
zwU2$te=+~Q+0PZv_*Y$e#^-5iA`~DeFR<J%S>VyyJ>0jo4)b*+?dJ@gxslK2ZX@5G
zfHbpd;ss_;WBPdYdFGlg>s`gaXy#$5s5{&F@^j_*k4`b?S1jGZ^S3F+)Z)z~?p?``
z`NFg|D4)ve<ktOlny==XgxSu^t9V<Le9R2K?H3kr<L1*jJ%jI}=??C_i#KpFi&vU{
zf0u5SQc`YOVbN+j>8b>OSKv7liM-Q%wvAdmC5dxQze=+4@QWt%t$y}F_5Q?U{y);p
z+)R^9cx2QfdG_q-;M+ZW3s=y+9v-EsH`pUuKbmrXQ{-Zin9R?yeyW)(qa2UyEpxNb
z>(6<nYW(09HSy%jub*JXo$;Qh+U6^d?9^i6h3Rbk(!Roc6MsDAdLt&mXUwsL^WI7m
zffTcAynVC6%<eS_nI$u`@cw#JX&!U=EnoIYPB~9bWxg{}^?Z7Q?R>j36?nKVZ#6ws
zD8b!WrOOw2fLYb|)n%?Z&Qg3g{w*<0U=ihQ+Rtu2SK+IW*yT{(l`Xe<brSAy9d7aA
znpG!mQa1Ct={n!3rZ>es&8AOi;(H)*-Q-z?B%f_a40rU|?PeTx65KYK8+e^{#nn_Q
z=kdME<L7!Rypj97nI?~Q*)iTZ>VaIV%2#q9d8g0G$lhYsutJpU?)x)*Jz)}N*IV{;
z`*s?eRm=JCsAm{*b5HK$Rh!&o-mfmqb2re9XTz;9p_fLkeCswk^4`kR<oK_<f@gw!
zBfHD;mjZ?fD|!B%motx8e$0G>g#gd9AABZ33_ZLIrzS}Lx_XUA{G30phrJc=y7@6&
z(XA$Cx)-l=8s@#^@%ZVfZt-X#=lAU&csyL#&F;tC;*PCdYUVa^n&_q=eV(`Z(|I~w
zr*p)YD{-pi?=h{dHa4B#-)k1N&)Zn)l_#%O|8$eq)}K6mQ@3#xl^!#jm$;Sl_`yP+
zE5BSN^Y8ocnxFUOP<c9)OL&h5x3|kAo>%s>IRBmc!u4zGW6qu7N6kf_#d4Xjt>(>a
z<}?pI@ruj#WrtbD<`yo$g$Fomx9IaEukts^6#dT0z#hwOaYbHa&trDp*^axoO_=9%
zhtKZgYoD{9Q}$w!;Qddt#lAl|XF9)*-SpBn5#A@}awbc6%;VQs^hwU<;&Z<DAGrCA
z&+78u_MO7xKg-fAkSm<qT<wm~v#C#2d0#H&&bK<r=ko2e$(vX&-W6$mX8Nz@3QO1U
z@m(;P%a<}chg&z#lxzFtVB?2>!cC^MsF<347c;BW;N)LxHN&*2X${{iWiFofjSA-0
zo2PK=Px0bg*mP6A^jj|fZ);O7qx{X>Ih$s2ug)psi?4R#nqIe_=e%7l`zDncX0z+p
zbL@LMi9hsRiJA05C+^<?F{Z_C^Lf;m_i`I)8}LP)5-@kDXy&=Dw3?^p>{)@m-#Gar
zny&H|tZe1H`M82l<moN0IOTo9P7&sO3(vZnE>78Q`u?6VpV#R=!<sXJ{0)q!WqQy2
z;K^O}mM^WDn?LaCQ7#MqEhd&Txp=OJY~y2AHWZY+T*CW!nI_-Hb0Q`#F0FjQkNHeb
z%oE_twq3|;8863or>23+Tk0G4n>W8q73ZdzMrJ6PEJ<ND)bdW_JCmek*8U`yk8836
z$F)=5W{DM3I7R-P<Yiv5PiWViLwr+jX>mthTFh;wsl)#uf{Bm8LWMVSwK`AwbXM-)
zzWv6YJL0%Q*4^XN4LEIV-_Fk$JY}0n3)4<MZ@w+OY?2RoTO~Ie&leElog}q|r|$``
zprU_2->JRnd|7`da(%7l<z1$g!!^rMQDj+IGq1t7<t8)u@=c!YW8-C$s5jWo@52{+
zx=VIN;4I#vjiP*@wWIr&RdetB+HYcZIf2VEkDq_3U7OtXOD>#}wX=9lUc{QMZv4W-
z^yQvuOT`bKWhWVV|J6<4-5+hjbzIDv>;C+S=KkGN&2ndLGtv4NZE!m3CU4n04fD7g
zcZHL#yymL2-DM^d;lP#o&XRYc`#i<>NfLbaZ4z8Ezg4-f2RieJ{Pf~IUUi1+kkUae
zFRmZ#UbO|L9V`3T<Ez5?m``Sy-VEHy{Y7W0*^`&rJVw|0xV5a6dHEt18Sw;d<%#tA
z%DrW`G=D_qZQeU|4ZN;-4V?Rr$_N;F^mFC191-YWA|kj~+S+WBJCFIb>%RrK{4SVH
z&<qw_uJ&K{`O__YHWo7kyN(zNx{B}QezVTm?6^e|&%BA71ZpP+sU{h*^4+^VTfl@{
z%*6608~>}`8)iE<-4$uk&J~DPkQA_5F^@}bfirh}R+vfh1TB-!6}L>)B`27@`co@d
zRTXB+_DfW-b{z+ox3-$O*Y7!8ViR`oRU4Yiyj6T8Ah-1nm+U=uZc&>}{Or4Y1-zDR
z<~_-3&3onbH}0n`?PhLY_HlP^5fxl<AlFRLtD7(ERKDrb+s6F7JBs<P&Tbc2@+HN5
z;-^%Bwi*5WI?E!2+OM7v_+~C3$Q@A09^$*2Fa6kL?gH*lg7-34^KY2-#B4F+f3r2!
zGx-}%@S85a!zQp{cE5~X&o5qs9!r5uTp#$gq7=C}jsKd6cJT4?1<l~!CvC<r;A_Np
zV1@yIK}xEz<9il<-#yMID;Lh<zs}FZUvcp?-%^_gTrZ@0x&2n@nR1nTniNJ9nHGvG
zntEmk3fL_?Ya&#>o?qW&Eysm@YfMW!m$HU!cHy(oHsDyWX%2sf{tqt8%jdX7P5$tk
zxxV1bu|C9m(^iH@bY&g)68m$;AGfn}H?ZvBfA6^2<j~7?d^gq0jrYxJ;-99}$aijm
zGJpL8JJXzIK7OISoP4DwXN98<_3&3Ln#;dk@F1Jlgl7I<OQX5tgX4u+quT`%qJqs0
zlB>*De8~~e@Xs(kTDD!FsxDb>Gut&j&0U;=&!?{vh<fD3J?+j4)2F*-dEG4?1oC<O
zB;Q6X;p4C67N{~YHGTc2k^f__p6S!6jRGmZuk(w&Xy%Xd;O9<hQs*}5oMJX5L&xm<
zI({?7=ulIhTXzIb&bn%*Y~3Jm@n{Q|Y}R|TGymH-+w#xyUE01|?$F~nfjKwzxlTL}
z;r8Oq5I88Zo`3spE56mQIeGl@?{ixV)|tp1_2PC<s29*Z;A~o<B*j0)Rn*k??Mr_3
zU<v-Xc^~+Vm#sCOFy$-1*OohcJ-5#a^J};Xu$5*AeD3+c20s5ktA+d7r+K0)+wKa?
z?@%;4ew*8D_O-(T*%RB%8I2VL85xZfHt)Q`7keOGP}q=NkmcoDu0tha<{qqeytU7Z
zrI}aGR$U^#hcDSWR$yJi5)+mk3;3T0OPaiWa9!ZjJzW8A#-IG#ce`=VY%k+h5(%}?
z{bgj9W}swx`$D<p`d!9?a#_Y^e0dK9($=YRIZB-{+j#CH$Be(X`8xgW)qTCU3G5fh
z;y%p2ggg0Tv%nWtVS%~()A@R4Pvu#4I-93)Z?x&8dIj!VY1ahO*9MsTr`YpPpV(tM
zcL%$`!?m{jxsP20_UvFVKl3k8;A4p&KZDO@fs<Ja1d`w06gaMNgzM5TcD_W;2#%_W
z?IPF2X7K6pzBZkuByH-Mx{~*HWvR*CQ<L}|KU>OFh=uS!dBMbgS|EtOdPNS;)#<J#
z4RY<=KXr5Y=N9f0?+xbU$uRxHmzAJz7QBBKuTNXIsYF2+uht7UK1<F=d}Yf-xYNIP
zaBkRl)NFQ{n3;W8pXres42IhRmH2m@+i2z<Gn-HPq!7={?KeypypHB}?W*AmeV?O{
zT2;W`syd0gweAY{m4}R6Pa;zJPWSn6|Im5FRa$eOy>3~OX>G4K_md^<{OeYzo2=gu
z%;T9o$!x2>B9C4J6YsA23A~qicNlK<Na69{xS6N_g}T5T7D;|hfd)R?ouVAV`U$+N
z&v0@+z8fxLd+96hOucWW1(8msA$1D8oQLX7)$iWrYh7e1(|2ehU!it5-?Ez3d`izY
zaaUbXF};xD%s07j1)sS52l2d`Os-RU_Iyzj*P1n7WaaJaOEJyL6Xd<Tbqen_H5ER$
z|IKX5mlv|>?SE^g+Z=6XznH;{%a_?G_(T(*RM!hLpUL-mqhmz4RzH+8lYCgfrRTnq
zS7(Wmd^zJ{zQmngTxLnCT$)c$a^4SG$D2?b!d2g>$;BP;lT$6>z3HJiF)k&?1AK}v
zW*fQJ9pIJ~cy8A8wt`D!CIe5&F<V}Cvk69`4fA*wbzkLPSgR%!V4Kh9%XyyHZ{}5Y
zyNz%7`TntRKeNggR*ZTsVD-|`JaoOh`HH`%1Xw1XG`*W6At-*XPd0MXOuh`8BEcWG
z*aTlEe&gC-Ji&DRwoN>fqB;di6Y|A2r#<3rs>>Hp=HoKma$-5(=6XZZm8<Uy@Mjwf
zD74?<fAj7J*Y`^n+=8#O%&s&ZF+KNSsp*|%>rLjAItiAX_B3<f^hUtrOCX26`6jbZ
z2@g3xe(2!i=r0z#`g4K6o+3T2Rr4lt&x)VVzhsT9z>DYEycw43c&`0r<aW+qZ}MuN
zI`<;)hXUp*H70Tzp?n5)Jf>#H<oQ?0$ncq8OcaQ63^hF)=r6FWC5C_DhabY6uO|tt
z$h{>{#HYaSv|UHQ{Y^Kg$pvMRjWOzity3~gj>_5?JqQ;NoZaqb)_W*bkn5g?kYk%U
z|BU`sf?Jn43r1HfaW6~=H|E(ggU7n}qrg91e}TmzN_+|LR|$wT%`x3|M~pwAdB5>O
zc4fggpPL1GDrCi?yk2l|Ep6ozpUr9d<@iC91z#;pl(!TZ&9><h<e$3Pc-esyLeq|(
z;mQ@QF<JdghtsWYGatwLIdU%p{|H3=I?45@U7y>o>n%Uqwm5-v_7`{~>=StAd8zZT
z?fY;1prfB#?XjYud0(q(Nx~++^(R71{;rPax8vB&ceQb%K=ad6Mv4=s3OF<`;P<U~
zClIN3Pe8!kP%t!S6^FLH0?%8iSL`3QJQG@Ws)1*7o0q9V(mxY%Au*n)w;E=??~Qo<
zQ*KDPNk#Kad&bB+LGBxm*qRAk%-1KFE)PuNT=Zi)&n!_<rR|oL9QsFR^CWCLYMPxY
z&#fy|Yx=32ooDwCb{;o=HJ<3&HjbIs7jg9LZZ*qLV>0`k&2H*H^{!#=cW&OTs!V2o
z_;PrHRtR!R?A0~9a@wEs&r1iMhqbe0E_Cna>B<&h^LOFrl54ug#r;5m=S00N=gbRL
zToV^>;jB)7Z2C4wgH!yuG;dHjn^D&(11^PITxO~74{^5MT+OK;C&D8pHp6&It|Ygf
z_A#z9*_Zq&zxVK@o$=>c`8$FAF{du?nr+89zNcOkTJ3d{H|BnunP{ns>7rdxyb8b6
z&6Cfq=98apFH^G8hVPML9$)6;OumZ3%Uo()TTEYjaC6(Vv+>;!S+3e-BF3E+!Niwx
z>YM2vZ&BXaFWJm_|7i;^-rvA0a$qZO)A4qW{9G~46kR#9p9^%%j`STfIou&>+Izg3
zZ~3fWrU_@i@^ZD_;tH|YZub7%OwM$(3A}tV3i6MwbNQA^)o>ktY0lmCK$d$`K@+dW
zthwBM25Q{WJ9D^0tW3=&<Q(Btxip3EZI_|hcdKlkMFQK+9Ix%<p0&G%XK8Xd&-(wK
zW@i&0bG_l?<S{SbC^+e(7N5e|<-C(87H}r7ap03@i{vW*{ZH_E)fGNJw>M_Ch7-(J
zt1aYv^Jb5!)b9rV-s(_k!JKs7!u~Y=ne3kYQ*sr#P16+2T7qhM)DAK7Ps>rznq$9$
zch~m`eC#)*O-}Be%l9cG#7xJsMQpya0-wTuGrotf_i^zROLGfcaWa)Y|H@=nt+!dE
z+5+Rg7zh4eXBbWWZ`|aoK6-&u;&`iBK92&I$-29|i}R#8o~+~JPq(h-`oz)3Epn}x
zKdr}tZ|QGCUU7{m9@!;dxI)eAEY?n~;a(&q&HrJwt=Zm0Exxo%OH4&vfACr5Jmfvq
zm%_I^w#Rhtm7ToJ&OE$9SEh-?JH6za!n}rW&e1gXPdgX!Km5(bnNUAhsM%_Xz_O?q
zvnl&yOrK2b7EqqgYyQ~xyuf+aXTmoQp61))q$1eMvtPhak(2vQPLxs1?F#O&U$z3$
zJsL{VFBEy7{?-v_pES?Z{9Zbr;88=9ieIw?e%)db_}Dp%f5M-aoUX~OoUQ)TOo9wO
zO-1x#P1iZhG+B6pT~K?5wXy2Lg#v+3oVYAc?lG2fe83^RNr^A5g-=@KQGq~a<SMSO
z4?l9PSN_3QX?vUBj)RN$LNh;4#KHz{4RJx!7gPRm?eSV55W8-ksnGP#yy533nJo3J
z<@@!Xh3`c!m%t&#*+zx-!U7!zqWp$@)`H=FK?1ho(*({o&gLxMsLpG1u8rO2fTu8*
z_95QGQ`$_cZRQ%=CA0HtI$SoHEWM7eKWmDVO>z!jFpn@_drl7@ulhf3LAK|{eD!ex
zj(=x~T(gW(cy(?g*V!Elg}-=wH>;cagr_J**pzY3JD#{tjl5-we|Z@Lm^e8OoMbhN
zoMU#DPu}c9-CEP0;6+9}$36JU_b8g_pP9{@mT{Zwrk9qP*e`MJnd{c`E|8fjpU&CH
zckqWYmjllmF4yxB9Lh31ysodFaXw;R#dYt7o$$xz3#O;vc5=zo%;GZ&?KC;!Rn2WC
z!)mr!se^0RgqPeGl>hKBu&SG^zAeTRoO_)+^7;XO4Ou(BDc2SGGD8x%W?j(WOL}>n
zi*be;f7$nZzL~}`W{<e*Os6J(<dylo(d?Cb2>;cVAgPO9O1zzlU-=#eYw&;X;OANV
z?SaV*rajzQQ}*(4{SZ?-rFEX?#k?Lq(fzWf@pZ9$2~CM+du$`cO67O)`d?Mx+h*Cs
zbt6red%u#TiRRa1rdB~Jrm>N`O??=d_{%(&n(!9a@^N0|<oca`*mUz=d+wT<;(SYg
zEl`Pn_mJ=6e+_QQNq4vpPMgOkDyh$RpL-_Hr!!)_{@k%#k2uzwS?tu{sR~i%-}xfN
zRAbFD-l_6xriU!__}(x2#`8i|lJCsg1T&cr_qkU|%;4E3-79F8w3Lr`f(2jEtZf{O
z0Ze>84bAMbhO)wo_gv>|46!on>3w9vn$W{{uAI@VU}XUR)Quuij*gS~9^7%_k9wfa
zZ@S5dTl(Bp6CEi<fzwMJg&V(YQ*D>r#<jI<H~*8D7fk25$?^IHyf=+XXcM`=IgF2Q
zO&nkLGYJk6=6h^`1-Yin9;TYk*&}9Zm}zg?;N-<Waf_;{$A&2ab2X1}xfUv!rRJ>T
z3}Jc1dtbvt+R%WT-(wXccUz?!cl-`Xj^&C@d<n0NxL2-h;I8o(<vz74&TQecOI+WY
zEcsb9R+(r=x$y|w)R}6&Y2~t=*36S1zm@k>OuFfz`-^!)9R+w-Du@UPYux4&dvu5|
zFNmAn%=tRsgugG@lSK4|*zVutpQ6}c_D5{3Sw+VN{{5*%rWc+t3H(>yEET<6iqFN{
zM<9M0lfdo`bGUpq?J==mGlBi@`ZE4TRaT|h>qWWxYm51_EbL6@p6%vIoqNqBM`)Gs
z%>#P;&xE$~O;dQoCd<Bt&HFa5S+aJI34hiG(<8T?jZ$723vhpSF-?4aioYe)pQEgB
zmdWjN5}bKg`+2kUCy2$Yn!z8KtIiR++L)86G>pgR85jTj`t@Abqr14n)-!P4)mvv;
zT_?%;ZT>C(GS3I5#qqniOP3^>PF(el`|-2}ZsymX{AES=Ox`(p^Y4_9<cs;dNVt@#
zhrcj(BYzKD7gy6ZBfdxV!5kM>H;BBBSi`sPd!6a!1~t<gbw_x;ssv4vC#~d{__9Xk
z?Y#p2tY5PH%T&|&?XUasI4=+~xzn|p`<lf=zCUsML>ZfZanIG|=J)*|XqL(Lj5p_r
zjj2-YblwMTQGDl)GV$-#_2a&{^&_WOZL68wlrGb9)^<~=R~rnDE;Zw?UcJri2G1V8
znyN3{45eYF5-j}OXYV)ieQ|Y_H$BtI|Lp%k?gj6ja63%U<380}&-bZmEq6if2`;Ob
zpE!TUTr>G6YsCH7e>#7N;RfSF*0XsYXUm!`Oy=Wpl{ezu>$I3xpyrifWcw-};s0lN
zPEVRD@XK9`|Ex<t-?Fp=oV;bs{JVYra3=iy%&*Jvg1@Q!u4(w=mF9i_H}PNLeP>?a
zC?Rm;ouurp_RYN4Cx;5$J1i`4&1E-tV&4R_Jw>5hO<kS*e{S$fWIYPvnslm_U-j5j
zQ$IIB-bF6l=CUWw@jLiC@M}x&<y&t%fvw#jg7Y!=Jkz*ybIpD+D4Pj{Ofmg+&{yDF
zc8c-stQ-6<Ci-$c4t6!0T(g(!!pcv)DsAi4j7!$?XIx<Abb40EWy~+lbLOBFf8En&
z?qz-Jd3MiU$>smb&Ft>iRPKzYKlvq(cbl8f;pS-zOff6AV&!%@=*AtlKbHUP?Y-tE
zHEjIzmn!g;KfEEbu4*Pf@AFms8S?|!b+_E(Z=QFLYyG1T;e)%b2^@-_Wv=r3w0YKo
zO#<B0QcSmT$OvY7aL9`7tL1079WI#pjz^GnmMu?rIJ4=hLKogWFV_gD3++^h3O&Zx
zEt)4Vd99h*5{-@g|2CgDYk&1iV2z2cfM)*}{=MCX+?S52ar?4eHG5Sv(d^*5>1O8V
z+KjVnjRo&yUNk%L>x4kF`hG4+XDPFdKVEX_6o~LI4)l`wac_r!Q5FZ6^4j^_hbK%C
zc*P_t@c&aPUz|WHPxp;hp6e_4%#!z==gv&MDX`Jk#{6!)2mk33e>1xof&y{|uKb60
zI|&@TCv6(`-BzG|xduOno0{m>Gm8ZbQg;hnXkq92u!@^EE~%We^zs&g`VTvKEp3mP
z3CyZCn<=TrJMsQvvv=yvd@0FOWs{|t`KClU@mcS7;tRc=&g~}NVVb14g6n|hKi(tL
zy;U<UCUZ^h|Hx}2vCA~RDW9iETiuMAxlLrhVh!*4OYOYQw~9C)@fdMVmw8}PadEk6
zRpVsS?YFhfEQ@3L=14Z1&b)S!*H~&L*VC*}vpI)Xa{X(Z%$xk9MpZ`BpRZf@5yw4u
z9`2A#U9OUnO5Q~;S93MpQ{i!_)aIJrHOuV5>xrBv9Ov>$6sntEFjC~ce13)LG{X;E
z^D||*jTb%PxqZCdjD6NbuE=fwx%W3N71GU;=W{(^!h6DI7MJsEYu*Ble;k`Q&kH=Z
zvJt(+Im=vN{$VqP#V>j4BxhJCwD$3>diq0(P4hR;<lnM<+#h`S?B<;3zID*XOlW%-
z=QLq;Q3Hu>k~JFpIYl2m;GKIk#cZ<nMxL{`?wZ}a6~Y_dpu!vF-oz`uPmPOF{yWFk
zNq<e>X(^hgv>Y;fnj~w|=4s8hBcI1)>aGtW9G@Dv=C-n%@t8m0dSO<_d-up{mC`aV
zzI&e~I71rda!E`U;F{dlz<YOhIhTJJ56^{54@6T$yv?`WHsUJ#+{AagaGm)<i-lZ;
z4;GnmGtB1tx{{M?_1hTUADe5<4I+HG4gF7YThCoDz?i|z=Q-b<w?a>sW0T@X{v`*$
zab`>X6J}iXQQ%NiviU15W|QgrE()w_dv4CeC@z>>`amKjY6G9m;Znh=3Y>!W=H=Yp
z71K;ggk}p&dYmi1VO5Ci^wm3gJ1TjEzbt%YF7s#uU-w>ivsa&A2}Ip+5NKSyRZ3iu
zg=<l^H`n}rC37juon})ny)|pAU1lU;?I*bR2b0;M_?@ClUVY|TA~w(LsiqTG&c-Qx
zQ9pjki}x)Rc>VedNATR&T(QS1_;;?76?oU(&dYxD0?(PV38D(#=gjWj7vetB{Yc=$
z;tcZ}MgDwC-tw3k*Zkt!_}`vSEXiMB(t)EU8SIt<oWU9VD_G_T9FLhMa5{UIU`y0j
zj+J`#yiI|f9Gm_B3h#T6!spHsV_Nl2(1g|S53kp&)u!ev+4!4wt(WSpn9ZjY_KMG~
z=n&uB13BDhn>kJO?%n1(u%?=iYnmNji|RA3kb?z$QZ0MUZcHfP<>l}*6Hs2s%f<ea
zx8uDDUt#wu&R<2!oaSpB%;iOQn{N70WO{jRh|%xnH+-8Cmzg~gE8z<h`Oozx<(;X3
z${NlxuGhSo4UG!HiC6f{gmk!5Y^HPb-*Vy-{K~~ABXXAeE^`NW+J))d3%6f4opK?T
zOC^Avzw+W`6LG0oJdpu+%~VymxchW(^Dz0Z;`x-2WBhP_GtaW}ojmV5Hu2v%J%_L3
zY9gOUm<c;`cqhMP<s7c=9f?9MliCG3R%|lc96raiE+I|eveGPbqt-(LrR(j)8Pu5g
zuO)B^%89NOu&Ybqe*NL5@vKK&yeR=<BCQuc@z?#I%9mutFAy#F&{Ubfo1b6yyGed$
zjR1qkE&isK9)ACQj@(O(Sh&AEW;X5h;WX1ebj(!7lEHL&+Cu^9s13%x>WhV+%<bkf
zKl;je*Z*zooH<+gj+bR9G)yTFX!h~v{$-HG{b6~yz>?Kl`0uW;=d<>6<I%8t#^1VW
zx@qt;1Mb|PjRG4yjLqU?75HoJ6qs&Zca7h_SBpQ&@;QI)-9XcfFYox1c`x(DHCzxZ
zn&2a_n<rmjv3moT_N~dh+dg=6?a!?i_CEKQSMPYa*({eyX8RYr@P;j2ZNk=fh41ry
zNr~eR75EZFX7QQ6Z|B>w{vbC?6p!hQgBIKY*>-$OE7mKo>ORd?Jw=`GO{t(+(56>B
zvQalp<<GVAJ-1rU%N22n_nK-h*WrkjoaZLUo3h3gnHJ4nYsx=uveCZi`Fw3=b|!*3
zs(kuCPjDI68=D=N_<}3z{Xt$vr<F4KJJ<5{Jl?>uX3tM9!LS8fmNU2U{<xpbeQCZv
zPfA1u*M_48W;cQsag<p+;j8oEGP~IC&)x3qV8*(Yk=sDOmAgPdlQ+j&&bXst8rQWO
z?mW>4)(WTPrSPTO*Ylb&9cDlEp`G7qXCl|M-*!UFjH?7T+_W{<xHixH$jb<UH`>e0
zdalhBXfypHQ)Ae{C!^0KX#07BfLO6G_Zrn*rov%vJd9r?1Y(;-L{-~(cu)DW31mEM
zGl^kh=hK<~-;^;jRp8*QgZzeWKKuvo{pS>ae3olV)*jP0)<;cRk29Fr{^&QYX1Xq*
zZ7gm2Y<iu*EB2FYYv-7l9pDt>lHD1~7j=87@~bFqfxfq|IJ_tA<>vf(oiDa*9sj=F
zQ+Wiu8h8TMIdkuA{Am(7Nr^k4xmG|@I?T*DY%y=8=SdSrp=dtc<Ep$dc|Z7hCrmZ#
zy?%>-(#acqmewI6lC$LnUYUjnlmu_!wwQm6@7&!5ocuX@{Qq*A`6h6Bn;rdGYPNJ+
z6>oYnqs9I=EBJ4UJ4>-_VdG!(NRD4XR-4~WW))8bhqGz8*m-W#013ge-^OD5r=8=D
z+J1viP{-P=SaUJ&xv)gD)sZ&5nd!28ca@j(ooCv~EmADPz3geH>C2VV%~V@2npQqK
zXtL(N9RI!=Ws`RyC-_baIPeJV>M;Ghu9w@rFM&_2{GV#!igf-P`u$vzo@P8N8mf2>
zo@?McQTdoFET@m>Xp|<GW>UTBy}9ySjh`p;Z;s(Hk1b^2Y1um6EZy%c&+NVSJgGv`
zd~fFLHH&}N!?TI`5l_Xbqe9z_*!i88Zs2`jca-ajsRZ9#U2cwqh6Q|lZZ~<){<&&e
zr7UXByM~ds#BHm2tm13F+e%Ebu8IkKEZZCTCLOEeQ(wiy!(}OJW^mJ=o7-85Z`*Y~
z`QI4|+)SBjeDSh3O@$5Fc`vDQm<w>M=joW-%p27Go>%1(Gk2rFPOdvXBBn~Oe9bhj
zPB1-fRBFnl*uqzBc+9wFy98g6d=t0gBTKU?g>zh$dYgEwf}g4}Z&}FazT`URY~u~w
z%Et_N4u9Xu`~ATKuHxmZxLq2qb9KG@W?CWG!PTI7jxQmU%bZJRA9ty&yV=}h;yjYl
zr?@NnLV1OwC!2p>)X)7`&wwY%;Jr}EM_0ZjY3#fTY7HFgrp@8mp5f0CVLVlE!(1DQ
zvMuY)`@df>|6DA~v-;~H(})92yn&C_OGdoq=B?fyz`OIcGOxO`2iJ)YcTKlE4(GZM
z^?+yTwHl3_jk`JTB)#Q{dU(L>M#O*aR|nsk-M@ELv{^if=jEquJZmL4aNM7iz~Q>d
z*EBa^wb?(D3ud#{wi%o1`0~~$@|m<d{Nm}n;lw%HM94fuF@{rMWfqUaN_E)-SE6`D
zpR#gp<*?*xUT})5jir?*%gl_+-gyF-@ycwj#?pJ{1`oDyom{e<x8RSBxvrKjcdaRp
zxu~rP*DC4HTpr&yazFWd#-#5{2IuaH<=j^~77FjZ%E>!pI|EOEQV+K;XB!`5#9^+7
zjIsjjB;)uB-pn?2JI!N$q=%1po7yYWlb2`n*Jh{7{fuGbS2O44w{g?t_e-qhIr5X&
zEav$h?xTVFBJ%&%D=(YH%Cr8(IX*WdP16G%3A`8XXq$z8{m1{|qYU4ki;MWS2Q23H
zGW^KJdUB4*jm6@o-#r=3Ca-Qav%SyBzi;;u6Q8B4`Oapu@yz|mZSLk6%3Y=4$M;jC
zUUk={eEz-@I$Xz#E_44qb&q?_mpHy0S8?u(Qd4-=Pd(1Me2Ti+tWX)YimEC6J$Ig(
zMrXQmn_p5jeWkR9=SrI}kJw};zKd>B<~9>mc#iy?%=1!cwov<VcK))INxZEs^_=g6
z+W8!&F65fn=_Z^gVa}JQa>(pTr?lA%yC1w6t{Y6JYz*T6#3&*aQ|-$uwCV$2<Wn~O
zdI2-;+Hc2A{9iTmEIhK4uY)U6?9IO{-q*?ce0!><m>i1E<SX@0Gnx89o$r%uBQMi-
zHoh~88@RsIWO5tXF_^V-n3>v(el^Lv*I?wMe~7R7>k`wyM?(36{P{R{EZJ@<yTy@f
zZ`DKIif!r=@uC~~Zp^yQ_4xGz?nx3l{Jo#p`J@lM;R*UShev)52lploNz;O|4ctk`
zpYa{k-C;Z<jhWAXqP5A!DI56ar8o07^*-a(*gM1ck6{E)o$yqit{pdp&n@lYyBwp!
z_u+R4m-lof-aA(!xhA}m5?Phn#2dYY&9psF)^xq{X&%;DuZ%7<`0;&na*|cG|IE8&
zh8Ulk5-Z<cg)Z*tRtrqtnhSD=3d!)Py1Wz*nrzG&zI-LGtwEvLtm>US6$|oAcXq7e
ziOXZ;{nFFTd-t9lm%uS5F0aLt&5X61%vf9|nR-tzGSn~?<5OQGX>Q%Vns;UU6RzGD
zOHCs?mT+!);LN*a)@9i|9Vx!Xh83Kpv98=l9+>kOe)8hg_iW}ec*MY6vsaUA?HXCr
z1fPW*UW_4p5%RLeCT>04I*seiZl9aMt@y%&`@|aw-s^jrjCh`U@~HS<<etbqO;A4j
z0q>MUCcH&@OE@<g$_TW6e8V-@Y&-v=2oXWk6>H6uuFNqveEC~IYO%KY{g4pB;v11N
zOHJqU%~qcwxX{*E@Y(ZA+%*hJX2qvlcxrt&2|QgIt-^7ao$sc`9Dy0Rr6wT>%=}*i
zo|?5AbBSy#%M{opDJ#&X@R{pNj1#x4vA5~nCSKDgwp?ax`)o{Y<*EdyUg|aZrY|PA
zz%iYRwJ6n$|I|FLy|=dTd1bFxVokUwFyUPfS8-Vi_pG-&`Mb^i1cVN+;5E6f!+XD4
zlDl=Hn3>dtZQS3j#RQ*idSEs`vWu_o{0vi<=XU&+6|sCCpLzs@_o|s2Ddq@tzUkpF
zGI=U&`{kHGPYA!@#I4-ycOUKKv$ws#c_`z7fY#mB{5_X^%}-tBH%|(h%CGh9q3MGu
z>;m`hu9bLwO_s0S!b-r@={>)eU_aNxchgO@EJS$kZ=c5hrQV!h$Uuv)m)ng0`BgC^
zQM;FX4_J&$4s|38MDAeXKUTPn?~dh5u9~;a+{`B*ncnC3FrJpSz;w%&6UIr&@&XL{
zlTDo7tmA)q_b<DnzJ*ciqC}2;wlRErS*Efd(45Wx?rsX#boS%idt83=Z()1QXZ&yh
z?{evCo;4cYJQI}<nsIr|=N2>D!T;#)KGPWv3;7PcGBR=~OXoiw7td#%qs1@xKiZUi
z%S*l~dMtd-8QVluGkW;H^LO*tD(11pUTo!W^8Ut^Q?gXZF{54J$vk$8vq#RFSFFqy
zn6I_VENS6(fv<1OWF|2^=CioQDfnQ`Y5@_ET5jLIg{IPfHF$&b9R=)m1*&WmU&NQ`
z%p;IhXJR_Tt(d>#AB(A*)qa7^_pb9_^=RUMXrs=pf4!eusfyA3yufnPQqI|CX`W9_
zUb{UPSTEjZCT-s!pwMK<H9dK+>1Ez*&dQcMe69ilayu=f1j7GWa}^(n<G!boC$NuY
z1OJB)T72hI#CV!NALf1-6=i;8qanAkTfM+^eJwNbatZ!b`|g`8TJ)N~&zY6Knx8>n
z`KHsR8&ux%Ul6>;ce?nGNQ9`1fQMVUz#7XM_WKj>@`q;Uax<KpCc4`7uE3Q19VRQc
z%`sbX<B&i}hlN=fkD{Qno`HhRywiLiU!@B^_{1TYwBj&Vt?y+s#<+CeB&#~<!rjwV
z1;mf?1!}|#$o6ZRD6N>yFLwH+5$D$H0&;;`0^War@i*^|=1%yzojc(}lKG0|re>Q`
zznk2cTy7bwW+8Y-^tjm<jr#(|tG95zKOttW<F$zW|CLvKGQt+>Ip&)M?50L=znC$D
zTim-}AWB47fGZ$`ukp}9p3^FCxD%U1jY0c{Sy-<NT$NgGCYNE&ziEM?3DavHfkHhs
ze&*f50yh~=%}(V72s}LR#(%lyg5VF``2u}u(t^%@%emw?f99R8H<9zi!UU14#?$!b
zL<^afD6Kd7dw(;pt?dfaeA!9-u`h+Bldfv>XU=2dXI>h}Uv8blBbl6Gay&|pdy<ct
zP~-+<{^^U~aes7X;7{zCX2!#QmDewe+0^Xs242|?S3WPH%Y3(FwsC2H`or<cpWXas
zyOP=A=5wZQr!x#$%EkFVhyE~23!crVx|oS)K{JDy>PvU-ITz#jVoOvM6okt8fADSO
z4%@qr`*Pk%&Ie`%d^7r+xO;5ExYboPIdA^oYU*-~kL%@!Hh#~Uy~eh0<9NPxl$cdN
zIm&I5%FC-z5W&07Qr5^S=`hdS)hl?~watZIE|=i1jhe!jHlc+>OgMyhf8h*{Uu^~=
zISk);?KCva?h4;FJ$g%-w>RC-Z04Q&d}+(Lr6uhv_>MS4@XdX)hA-)UJ9o9Hg=vgI
zo4_Y-6`?C9mMPY*o6A+VCSAzFm(jd<J2&sGHPWV>!8}6qY^L+B{36QNvhpia@>@mr
z9Sz*(t4}qUrIcSWRk}0T_~Ef;zRewT%s%wq<9%^WgDd!AfSJE|8khOTX}pV_B@~+G
z&gWb9I)KYVI)*FfRWHZ?V{3R%PS50;U9g91|5a(u2^Le$d=8!Glw5m&&$qhM`1+ik
z+<R8MFq@s3%eBep9k+LdA@6n7NaIX_4xaN<HgY?EW*45YBbV=noCn{_Iz5iP)t~r<
z9%XVqO*kW@vFf?Ns(07T7d|L7pPhJ0U~1P%GrOk}g3srhNwFNd0XoB2P)m|caQVOI
zTpev)ri+r7@ch5gA)vIaU1fjbHQu%R3I&4wu9~FJn#{+vqQ*4dfJM0Wype#t^?80z
zi!)qpOXa!KRVJIwubO4LG?(2hLvx0SYr4On5TCeNljj?O(;7h>PwEz$n*1~2ToJU9
z@9^yw+32+k1ir7R;Jm>zgF9yROnxbLIRTT4uDllPFL>l9eB~DWmTLY!dm^`O?tK9{
z&iSVNuRQtw%r7y$vX_g0trRz3>9<?~ckTvL=Ll<ox&xv7`hUelm!6#{@cG7Rfg=6G
z>?z?o0?+K?xm2PQMfS3(3sxVvZ^V7E%OHKefZ+Dn-KK}OqzNVnY!GEKP~gwaTO}BG
z*hNt0i7mG^;{;=i$`YQfzYYjoQAm|Yx-HEo^Lv#*2Lp#`gr+!uW)`za?<{4()w`Po
z9zS9fUFmDZ<>~o>OPH_E<lf^>le4G0jGwHDHd4*#5!{yb(Rfden&AI{Uap0L3dRRt
zHgX)AdxY->H;eQUwO<0IvA4Jm{&nW=HGR&X<Q*@tD(5usJBAFN6G}_CBU3a@uH1aX
z)h4GT_<rV2<9i1-@rh==Fb@8c&3{32HQzqpnF57NrW<dQn<d~LF^~U~-g|*k)w=@s
zWK9K?cRu7er^(23_2zF5xAHkc;nQ1rY#e>f!s^(}xF^c+EMq@mcIlWd@4m%);%l5L
zd8V&s;uSvsif50dCKvaTN>fXjd+Y}mOyW5!`BquxW;@3X=lMKu+#i~raS!8ue}2E|
z<Dl6>v$rtu6l~_?iFIDjk(T$F<LQ=QvxQ}*W+8&5rY?*3ncV#<$@_{&+-#|3CXc1v
zBaSd_F|!AY>^MEY+VSMS-y|0uvzO<QeI{GObU&^Ms##oWasoVg)+aeHO})u=g;|!<
zPhy$b(Gp)y=_N9}6;BvUek-|h@x{iO={q@bR!FjO{oKUG!~e|O`05EZu4%W|az)PH
zCzx?_FOSDWex4fDN9;d#u<^>X-Q=t|-XQd#?K<x%sV=i8**&J`|Aq6us<k)&@^K+w
zM&f<xNV&JX3C0<G6Xs^}-Kf9MwJ!CV>7iv^T;Iz$`BG1~DgV!Y%JsaCm5)g&!EAew
z8ZT4N3p4+#>O%gF^}H8*Ht@1eWaD@sY{<rtkz%&%kFVLvlHVq0Sk9R^T+`!=uK8p7
zU*bFO=?CYz%si%;sjHsibUhu)yZ`xC)z5Y%d=@OZTy6WVab^7B;<0q@;?>}o!JYhA
zj(gYY819p@k4$|hEau$2aT4ESk!fZV7%F&V^iP_pU(@1o4xGsIf}e{gC@#>9vyqSI
zK!^iRX_lXmW12Q!^0HgJ51E&9nwk6XF^R@-`j_?#-uOIMjMXmNyjkn4d7{&7KJKaQ
zW{X}o@J9%>NGc^Q;^lQn=kKU=<zJ{(%H5XlZgw&>naBR=cfOqq!nBN-7w~E<n87#E
zX}4+TqCP&hvIevMhuq?xo7MQ7wwmxg74G6%U@yV_hjp>(G1C*KN4d6|tzIo|y5V^+
z|M!ITCObaf;B!j)#i=m$h}pwA+c@o%zVMn~{HDOULx8^`_$uf2vV3mNJ(>I$IL-Oq
zxybUyKS<)~j@!ZQbZoYTy59-z=gu<x!P>{olF!QV%})+6`;mN}kInG{FIRpHABW0g
z(?$I~yfKMFyr-0piUhBG&UeFi5?|%hDeO~p7V)o%ljPF6#xH#I?qUH=tvjaO$7&4s
znso~tS|DdWgY%NW(cQB|!k@3>>y%It^w2vXu&j=gyL{Pt<A#kJxD|Zu1Z-9u<X>T7
z!+XYESKvp(HIr$R6Z!Of)|i+{%o3RQjY;5U_H6$BcVsyY`&qe;+P^UFU!ZET`{8WU
zpqt!=Q$yGU%~qKiT->ryz^q{|=Nic!#{1@*a~8y?@hL{$l(G#h5V#hxgiHSXZZ1t#
zZvI50yZjbDOuTPTN%JhJv*SL*Q)?<`ZO)~&bAiATDH}7Ud7pR}N31hBE;Ny^efewN
zyc|&h#wXmS%6!rSrmuwfm;baAVE7gyP_$yY!0y>AIafF8@veUJknMsCzc63OL0+~l
zKGP{(UyUcc;pEk8@ibk^zLD>H<{fG6Xe&O=JYl{=FM9b-_nqMu4^S|XmzyGRw)qe5
zpL0?Qk^<|wB267c7}Jc+e?>C!+8eT)8hn1lBd62Id&J>BuVK3@N8;rwjz8-so2UM7
zGyT)2VKR$rp5dH1E_{8m3Z@CSX7a9^yq!x@fXQrErULgziRrwi6IUzxKd<LIUTn^F
z;wuC9Nxici*7N#!`LZOq<Rav_#h==<?Up%UDrb?yrI|5@PwLkTW1CG$+@G%rnlZ*K
z<eI$i1oyES_j&g9ikkfVugYUqvyJ<xp1t5I2U|Xd?wP!XQ+v6dyf){H@8{&^7g``F
zb)=9_SpKkSOoF@Vi^Lnezf#!D%brK_@3{3&(n0DV?^*F*d^)<y{PlZ%d91sPOrvb(
zaPzTk;FAy1RZ5?AfJeP>8ehcxHzuo^toeSmWt%O~-YzN-ae?>kKNZj#7%qbauei=0
zO*Sb`yl5K#vcUA4sep-9As2sOlb!Kwy-L28v$b3*XEvMOk#gp4*e}WFG_6}D^!jtY
zStkRzt|&g@-uGhyUv-u)-=5$3Jd1vF^L{Tr$vy2wmRW6Y3D46LYW#1G%b0~tpTOHY
zBi?l8?EAd$N*Q?H%JcFCE?jNqI#rST{E=lmqL=Oq&c3je&v5lU-pu7&IKH#q<=rmA
z!MQtli;&XVD}253TxN~G3QU==bn!JERX1BOKZu{7ZL^fwC1butJdXV53^n-Y9#G`Y
zeRI|1WRM%*vfHcpPEB-|=eT3T^~+d7C}D?^nU<m+ubc*pnO5%=!Rbvwe48iu@y$rR
z!EP8J&m5&@X=eE_&s1zNt7+7xr6v|)3jDH<PMO&MI?s1tNgmh4)<dQ~y!u=>CeG$f
z%~&G8M_Y*h81q-I4~On??G9z+KDy0^kMocUxA2@^?hEskxL-atH1qI!&KdF4ivJ3u
zkm+(E7oJoBd(+I2MDAYh8Xo>r+j#Z68%$^1<l;MKti}8AaFwt}&jY>)vD18MtM%E0
z4Q}vd9rEE=YkW=cSgX9)E{O+b54(cQ?XRupuXs1xOg@-dAo;DJWL%Ff-=9_=f$ld<
z0=iNiT#ZlIOpAFk*{{AV;}?2hr~YCiBiB-?68=k^cTF1?B=UrPUT7NLI#EQ_NRNMY
z?-o9`=NT-8XN5RscYQTgeA;NbT-ed<Y=?reBA>m$=J+gAPVUqEyCe%aWQ4w$JiL38
zLpgi}Z{~(<={wys_yzTTuorJC<V>66z%$pDn?GM-9+$@CMcj}7@pAbHEHTpxp2W#<
z{TBZptxz+s#7*4EO*N+S`75|9{NuQ<&WYgv|Gvr8+smH6`o9QY<`sSs*}xwD^C@fj
z_w6a-l62<hTVUtGxg$1F<fZp2J_BbKGt(pi)9t^m@ZRN=G!weKj9)-jP+CLVoj+Ge
zmS1UG3jbZrP@Z+`PnsMw)8{T>c*VE(SPp-%!UOJ5Gk$(|(O@%=dH;AF=WaELShSXR
zn`Ja#v*~ZX(v-7Y$1a9(aQ(Yr_J^&(bZgW$)6G8X4W=H{<mZZ7Z8kMv4_|2DYwqi@
z$4pPX-otfOKAlh5zFpx-bsvA@>W|zJPi}C(ex${@G`o%O?}1C)o%yQV?rD3uiWXiq
zT{-zb=P|yi{5mnWjVso#;gM*LFymf3kK5><AFtr7Cf*lP3yhXUvGO*S9OwBI>L>KT
zL7V^Y?cID&uYcxD^Z&-@)UcfM|BOWb&I`}^kLrIj4YX3UxV>aE|N8?87Jhrh1a^s)
zNmtxE%zL*yRKU$bSYYD9-Q4!i*O|TOiRB6?>EzGf&7%0^oGe$le=Glsuh&d9c4_fy
z1j?Jw6z~=5^S9zJOFYcyZ?S`=a_vSA#{ToBM!c`gIKySkR?n0+O`PB@;ApzuFt+vv
zzeuSE7nj3S)9F%)TrQX1@IE-+ruO&MT7E}sYtBV=&0GuCg>o;tEX^-FtA+cqVFu3<
z>jhjt1=39KC}(lMz44QOV~)7R(_9&zFMaRLmWGvb?`GV=eYPixKXS@B^Y3iD{N2Ab
z_<k8Qh=S(-L)Wb1w+riI-@51_Ki}~xuBpBC!i`mt64TacS(xkUSzNrkNg&!!-F!Z;
zj9{+fdTHarPX5W_5rW06Jc5d;Mm!OV(@nXIlzF$jRu{X}H&Jau%Wl4{Ciw#4H8;$D
zaWCWV(Eey9;K?a6eW`(fkK<eZLl5-0HyCf^4hgwp7ViAq?76t9d78-t<In5t1t%_8
zY__xLvCz!NN4PGmUul*dCCug9uf}i3?=ANzWvjrGi78wR2hMV@dDABlzFJt|&DuCV
zCAT;p*KA{+mv+wPrEV9vKiJ+7I2GG$epJGXUo!BT*<^oCfmN~w{I^+s1^9Q)FugfX
zTfj<2g@4icyCNL(76}wupBH$saWAK_)P0^VPf4yv%JqDuB0G2&dh9gQaQJ5yB&y7t
zd6LQeg;OJ6>T?#EG~N}wLL3f!x5}LO%-TA+D<b<$<J4wyo$BHd_}s;&B-vrcwXE$E
z@6=<*P2DFf;>o=kVfJ09T`1^$74Ng@b-c3c|8weYZQ&4|n`auTEMiuje%`budWPx8
zSyp@&$JI;|wqNG`Y_yW==d!(~voBuddR<b-D-g6?)$MW!Uz*$vj(@EyxERcjaGpKh
z%4@RrDHrR*zua7(T)7f&cAL2$+{XS-XCa@9ysFuUchcO;(>hEKw;kk~U>n9A)my{k
zE;-li=B;LKRZl;jjIx`;{uxSqu3ZhhaUK0!fnJ`xW*2ohC;V#?Y7}0`n<DYU^!a*z
zv%BZ6@FcaJw>WD%f$#ck8JWhOSl(x4@_a>aJozTFT<4zPqiVkRpbpp1v)6d#i~q~k
z2yt`St}+k`zsF@dU3V!@Ysn6?tE$I&&i+y1J!~_X_vT@Ft~DuzoFB`qP3!8OnyDn6
zGE1#iFztVB$tQk3!YDa)H}AS9lei>iY%z^qy_2hOeI9S7U$*QfCvU#?YaE=iJNvoT
z2_$fFuWI5AT-42#`}R5aZ0Qdi-y5XOzucI>HMy;YuXU53#g~>PT+f|^&D(!6a*OZU
z$yNTqg}1cS#G>NYPcFyj`?%*d_Xs|+VdnerUW#{J^J)&mXK(p8HGklG+@UNogX^Qf
z!+%RmCuvMF<avBipw?H^Jng5L;C1z0nRmI<_?Cv13dUx03g$4`atEwpF^jz_CgA$1
zOJKIiG=;zU=XfXUCy3}zoMZkxYAattjfvR{ht~o!Ms@-h_OwbeFe-7q{@=}2sZnAP
zx_X-FUe6@6WLGnzi*sBA*VH~Wu}gk0@PO+s*V1M|v%-qooO8sx`RX=2R?I)NL?A7i
znN#ns1h>eBPJY3SaspN_ns^Uf>*WdR?h=(=aL!D7ax}Ny??(c&9_X8wmjv^1M$Ira
zS|ZGETVln>Efp^CN1Vfaak`s8%h?S6JxY@WmU~VUNO!$1FlFZ<4zHuZysJ5Exg;Vk
z3X7Y?@ns)zF?E(cXdGDofj50|y_s4CFaO0!izV3(81p6CKI3cQJj$of;lUk~&0{Kg
zpMmREaRc9c{ypNJLY-W<DvS6g&VOn4=SLH-#1jEC;h7T!wC8=~J$Xr!?^^FVju>rL
z_5|rJ^UW)H&CVX=Gj&a`H(LL265n3Qhi1atO88z0AK-es%FOI!&>7ArcPI1uyB$}U
zcJ&cow~8Wn-5E!2uV;I>SC)zMZEZin9Vk1EJNm#n?y>{tOpj)2a88=c#h<v{(qvcs
zTps1k7tI1HxOk>tdd#D!zL@9tsubfh$sxQqudwoFJBA9*j$OmIdTJZr9Hy!4bx%6@
zJAMjrPm{J4F0gMG(C0m6I!|J@vHJZqf%jjInIGP9NMK#Xb;+)~SNU9ZxCD73)(bGH
z*l>%n=bMJiW#P?Za25D^ja7Db{S3Yw9ex4N;#`xwGoAe2)k>zXYik4+AG*b_w569n
zijR$(bxl2Yxv{#*QQk1qtxUXTEc`hJa;}dB*00qvygjW)V0T<8m-1q9lhd{q914rJ
z^J&zFNH}Ze2=He+a>tyH<eqvvLE!u1t^6@h&G|Mp=ka{WloaVUS2LUaT$Rhcs!70m
z-%GOxKjiptd+jpyE`PvpXd%J>vG@~zIP*rcyZS%*Rc>D9TQS{6Aj8I6U_*70z?Bp3
zTo>PK^Dex4lglmcsc>vjqG;HYHnTS@wx%c4-FR>Hi<rIaKhO70x>2H^<pu9_;hB6F
znp^n-Ki}mx5RftnF%jSnHFDs)qQ6!-cXAKc9%d~*yVZ7P^XD=1-gr6J^zkHp-uySq
zc#p)~=iT_}7MJD31`gNzwx(vE+0EL{tunpsf7$41VIiNes*OqBJPp3qxYJxOLZ6um
z$WP~5qB4zloB3B+mv?LVQZLGIs=u1WC6}>|Yu&yRyq3AsxdVT^;Li5B#MQK(%j^aF
z4vuce$9%cICT5k5?cASNi<?bnOXKF-_M6)v`#R5U=TpXcsb$>yx2N+I-HjGl5thj(
z_4^)g)yI5})Fs{gO_e;H!aZJs|2?V&R?b*!Ua2c*5u6(?Fwa%ge5Kngf&V-5r7H5S
z^3|0w3D&3f3*0Fd;O=_3%+&Fu1W$vMguvmcJ}OtMnRso#unWwHH8s6&evbE3Rjt{A
zs|AAWrN{aAXFKwXAK>OPP`JgVy+zUN;7%>m#Hb82k^9?BE-ig2uwvp<lSAKX1+4Zt
zvn{{GY#KI|i8GC}moMsBv#RPg9f9iaE7${S)^fl6c!V!+)f#^NyKOw5t(WrHz4qaG
zZ6R(h%X)x2=v$3|nu(bCzSVPhgFked&M6k=bEwqj^_<Eea42QI*(Keb{0w~8`I4C=
zMK|Oq2)sV)An>(j4)@|YbNRHcU*OtkC@Sc&p^Z<Zy~UJ+vCve4w}E$wZ-K?W@HzaP
zD#xUH=ZN#axhT&sIz^dZLA{sf)-@Kh*$=gO4hml3TdTKOK1oiFhjZyoKJ7R*v!&(-
zd22I$&5F*)@@E{=;nV)Uf$vQNH;=QbA$Mt(rD<_pr&({;YSZ46F2?r{%JJ9Q@|uXw
zbrq326v)F+*I;_T!-o67mkhqzzt(D36m$9i`<rloeXq}RPQ!^utS^=Cc*{lZw)!ZZ
zYTHQ8uP5?N-<)FSx}`mdU-4dx`LR|jp7p8h=GSkp;<*=ihi7%`4&I6e26O4zHayXi
zcX*r(l?BE1+4zM{pXZ%+dK=f4k6U<W8$ah<H#3c2webP3bicl-*uysSt-A$zt6MlN
z7A9Qg+t}76z3gH-UqN3J-_C~xe4p36<(|LM-Hc7Rm0RnrJfC5;rt+Trd~U{D@_f=0
zq|7RVY<Z6tNSIfNALCKw=;9Uk`Nk{4Cc%C3cmS9Ag>KVS?iFT+BC4i~1kM@HZfxdD
z`~B9?g;A1^_x?I=(OKtA7u}x4_2uYp-t7<g)P)|b;7g60$>nl+A2-JjKJNDYD|k7(
zo4LPqG;@pTrgJf}HkgEc6XlxKdy=nFZ<YB;7fzl#ew)qSraa)5J5a&nHT4$H{q9cl
z31P+DIjWvK-+87A-_>#FYhRtg8>imK!MVVOCu3U_yTDB!q1zlE#7%xpG8dTn(M)Hn
z1<!7d)n>gHig>@@jFnnh&dAI7)0_9gTS;EIDo3um=H;dn&)nxc!26cx)yZ!f{7P>*
zC1W1(Y+*1lXRDFrIeB=aS*wDK*gE+Fo+{guJU^#hVek1V%XVUEnW@7xM)Rk>kIh<H
z&KpN<Hs%!&%`n;Z;unvL_I-{uxl-m5x2JO$+k5b2UGbLnNl4+<Ub~u8q_C1}b6*R`
z8sQe669>|{Jj&F#I&FKo{{FsdUis-D*MkL%c(<I7HJ=%g%+2@Yn%S#A+FVU44cz@7
znYqOmiJFE;@^f`+_i#_VrzsSCo`<*7W)hE%WCFK{rybu)B_r;p=jMDzs}lG+7Hu_a
z-!#=cYrhDu_QdOE_J{lU=jGPOEQw?14>-of@9|%iKdifnN3Mz0tZkMo&o+)FBLCQ*
z3YDIa;W?*qj_)6*fyw%qKHgjWw@fdYf8deWD90!AektEJeF2`ehxc-6?z?H~9GztP
zf1Re;)RX#V9M?JcuN-4BIlgH%-_u=UJPF@7nYNw@<bHW7luuBoK)JT6n7>BhBG;Bl
zySP^zE#l7q6vStEW+%6KwmpvyGaHAJ*JD!;-5$<8JX843@7rj$Fkk|=HgllqqHl#f
z3UbqV;x|m^ovtKqzAIjdXVTQUJljRtM63?5@^3YG&ig{~Fz3VPJ$zD(=3Ew277L}F
zu;4q<!exH${Q@(qzc+cS*~-oQo`vvtFW)2l%lr`U+%KQ_IAy-`$!+iEKK`85wDp=d
z&+9uY_~u1ES7F@{#rt)gJD<kmStebj&U~xfUYaC6`XZ$FcQUWnfiJwCQyaLN?F_gb
zMUI&&aOs(PZ4EW`nALB>IpaECrrK#!hEM){JZ)W^@qB+w&Sp7s*(iVDO%^yOQ+aL&
zU)SHCoT|H@a{t?*z<<+(f$vtvah~?W+jzcn$?&k+=$P-iA;R;k;T7MCJ<=wkEiZWg
z?YnDo>uMQaM@bd$=ijG!`A;r2xp&!{JMz{no{qUYM6#QD`QE2&=QX))##MAnkGD#-
zkgI6-H{o3|t-LBhyk^tSHJj=QoaAx*IoEXN@c_Pi7P2yx_QHJcFN^an7yixL;HJeb
zlK;?T!v6hSj0`(?+g@K6G*X+(Imc-m@9TSwW(D@Ec?{pznXqh6;bA>4&3lWdomXkj
zEUvXV6S(RgY%^2l@i5a}t6(;>Hpo!^n-t$9j<;sD8&>jO*zkbsgv%|{YsTiBEt-zJ
zsej|;-aAY3UHE*Fv*@Qkx0s_kPf~yxukdVH?lga6ZuY&$xo#@9n1(&==8St0%$Mt?
zWb$X)3~s|UOU)uD&*MsnpU?gE7$YyE%W|Vu<3#S+N7r+gh5Q%VmHL49-4_Pl>2^;z
zZ&k?&xR;u9wMZS}lkyf3ydTYHZgoY_;-<@QflG61&Ci_-6%1WBRch+)<9rM3X9#M%
zG8UY=@+tQ+C1$huDT{bi?`{-0d^}IdDu#n^fy`WiKP>xAvzmYL?afs(|8+<~c>cs>
z0j-@f0;aS7aUJH?;#SEzY1%nyyXnt|VP@eGSB$%I>I9QB#EsRZ!~{Wi#I7&DZ~DJ-
z9T(T_V|?$G)~UX#yelv#Qi4lYsGs}hgaiC%`uzpeUiI@XT&2w``7D;(*@n%G!Jm;Q
zq+e8UOU(pxv;8f6=e2{(%yjMfCtY;sD-N705H7D^9@~*2z?Rd=?-o5vq-x0#fzmge
zf<hBr_U1>%+RvOjZEuFs=e>!K#P@yrR=U@m|Mi}|iemeNK6dPN6y`f1t@v_JVDm}a
zn|s~&XWkCnyXwHk{Ux^(_OE7}xmWo6rQQF}TkW}Sz1;3x;>o>{=U4A#U|?YIyw$z8
zqVs_5*30huj70hDZ~ZFWyOtr|?)(y!{ilw--?!~ruH8|C7kjOLmhSDSbhAHxcIjSm
z5#zmEf0phwjJv*f;?{<}8$9OjHF$b|56C=*rj>hnk^;eIvv%~^zW!Re_s+k`dk?KV
zxci>Q<GpdO`s^QmD&2ejXX#$IJ6ih-JYU*P*&M$2Tg+Cw(78SK=YN#$l_}NU%fICC
zo^thm+uf25V6!;5!uFmiso86={LtRD^ELO@2gUB)G$VP>*RbHdr=N-0{+|)K_W?VL
zgUpxGz1Mgx_MWf^vv2WSx>u2N{@xQyX71barF8E-PGJY{+`zqCznAW9`Yp8Q;jC+W
z8WorA-BRPU@BYE?z1-hR_kQ}ict2<OioMZWa`x4TH|)Ltvvlu~Z;STw|18}*`*G>s
zQ(NTse!Nk-H&3;2ulUByy}Ni^_vSW9g58+CCVB7jD|%q|nm?g?-}27eyC5KWZ(;-2
zJ}bAKdkm!R?fKvvvbW_;nB9HH)%%WzAK1IH<<*`wOH20NH7&Mhn4Pn?TZCbM;KUDm
z?GA3)lNGdkKSvGc-g#e3_iFo0*qigFWN)Zl+rEiftM;1f&)O>+vvMzQh2!3gkEMGr
zU;4Cn&a=|Jl6-}Gqdqh2pCBx_S3P*P-Mr|)z1Oy}9BA#kYtL$!yifh#&pmP9O7}*5
zFWoEA9=NwwWVikMbL;n*{4Cw;+pv1ioFAooBgLNX{qXP7o`Va#_eMRg-}|2Z&%Oyy
zO7~6)+hqre<BuA5_I^LSVV}g;(!CDHZtc1I^2*-auccsrpL<-o_udx>8>F`E>ejt>
zR;ByZrmfz)=v(Pt6VDbqo%GZ_@#_rsrW@|DTNrM>_ZM&HzEgMZ*(eHW?*Edy%<khl
ze%sW=M*C;+b{=>)<>emZ-1xoaF^~3Dh8pi@<}BJP`6GGXvh$^Tx6ORE_pkz&J)6?E
zJx=~zdwCDc-51N=VE-f6!%l5&^4{>!8+#0XAly=Pt8}k>&#rwxeU9(B^d*0vwAY%w
zORp{8$0*;k_wtR>y^6Q2>_Kh@;l&=4_E*%k+3YjfyZ6%6!+ZC7PT9M2b@E=leG+@!
ze=s-*`kmgtC(OcO+uJ33H}P%U2h!smCTBNKy4-GJv%-D`S?}Gg-%IyusUFxf;Z@sS
zkAm5I_p^uXbviO-@42|1Jz~E~_g?n#w9j8NXYb^nrF)~rQ}!0#GTW<L@@(&~p2mHh
zJm2=(FVfljNhxeEgYUe(CSPjyR&~|vtN&iQS2)zrVFf6@7PQ$gYzf?}F;Uq556eaS
zML$aSs@`bWe^6@no=sDZ@9q3qx>w|v;r>Ny*Y7?1CuyI+-1+;T**fmm{Z+bm%Pi0R
zPu<$=J!35P#_Q|Yum2=x`~FAQ-pcFW?4COs?A`Sf5^g`l>i34uUb6ReQN+G$Zvywu
zC<@xU<$H+z1`heXYx8RM_9PkGf4w<%@9bZtdz&Axv2XkmxOZ*lefylVDf<FFlfY@h
z`(x?e$~T7gyw95VF&&H8$MI;|UIV$PeGZFG@0l6xwEyM>uf0sXEc*l=UbKtv<K8zh
z?6~#6Lx=3H-JQNK)BX8gL+NRISO2Ns`|oGz-onqNd!6Rq+9&q2bnl$k9ea8<oZjQZ
z1hMDtza4vT1$ORLs$IO#=V$5OJ&zWG;{_BqtTTG{ia%IlUv{l@?~45;dv!mT?iJg@
zwf9%vD|@D2rF%JTSod$$I=<KTSLxnoX%YLJwlnXY{yuc?oS&t8tD3_1?p|_p-->TG
zd&)kQ?yZ{gYESf!(!J?7B=_$Dl`FHq?0fLFbg$@Uy}fT{3EEBiIA{NN%fEYOrMK^C
zFJB7IGns3W_eQfQ?#=6(y_dgAcdzGKUEBL=+xJ}Bcy(|6$<n<?KGg5sv_5$++pp5S
z3rZI6wb^BB=iR+-SI_xf_8LC=dr$KI-}CsK(%w|3;JyB<jP`d<|GPI(b;h3m^A-2n
zXj<&amk8VY;cMyM_PZf_y|(<>qx>puFHght-PS)!_rB?wwbv%S&F(Fq?!LFzXM)qX
z+0W9wX0J;3HvKqe`-LlPZ{Uy8y~1;s?iDuuX>0ehbnk-nBYVQWmhOFerF8FXC%*lg
zDmU!g66d@3e2nNmEjj0X&0k9Qx|w$F2ZimfsLH)dHX7`8SsA|f#H-T1%@W1?9DZu;
zwfk7QSMSjloAcjG_jdg(-Rr5eWbg3_Z}v|8TDtf2@2I_}j+q=NJ7Q^*_`G!Qjw=QG
z(ztu~Ei?AFTgCEkU-@UPy&(Y`_ipMqZhv#K{{FUqW%gHQ&)+L^<<s8MpQU?0w#YbS
zYliKu*)?G=$Ch~er7kn~eScHBm*sou-Yw!H`|_&=?4wRk+56>b)ZWg09(!cU`RxBZ
z(cJ&*VErEbm+}XM9yRWj@iyJtoY!O%wMcMZ$<NZg;rxg9>^G|0ck#kfyTD(id(UK_
z-S^{L>E8PlynD+%`1Vhq8NTn&eujNgb&<AyU3YCkeg|REX}sWc);l+2pZzj}y?Xy{
z?sfXLXW#X2rF$o*w(nJXX}I^vt)Dh2tD^SuD_88@{-boS-YZD{vTbbKGktB_-jW}s
zdus)4_SR0@Xlv@-Z1eTgiapzJodkz3$c{~R_pCrP2%D9@+z6tPaY3!ZZW#a4n+j_X
zyQA0`OoP}kEWAf-7mSTgM`T-Ls1Mk)YYt4E_`@e~H^9X3(L4s4a5ur!@%l7kH6teW
z>)Jc9g;?cadI`}nn>pcrds@Y7gRWj6N*u#I=xTgiZo|z$m!lS6{`O)Fcf61e#PIu<
zdwdw`NO8l#jQQ#y^{J}MtxZ&$u$oo!RRya!NFO@3%~T^=ExKM9pXb~{qQm##gl8CG
z0y1aE&6#lV2kMb1Y1VOb5z*#@%tFROwybMl{0$Q~<22`!OxUgs&lq;j{450ykNpA1
zQPkdhxESs}n0eUfwZU9!*u)8mP48cfVjoETnXGJZIGix+-vQ!VJ>pymqLH!g#YtAD
zKc;}){QuH(g{6i|cY@?ESnZGp(I9;0{s}V>jSm~W+XnaFeCsT*y&&~$WqM$?-zqK)
z_icRZAOTW?jybq@gWZ5GF7lEet2nwkQuvN11i)c~tVeSh3r0DCERT<!A@CpU&s(xa
zaJO=Xt%8RQNN@kUZ}2n);=}M9fz!)jY<#qRw*%M?WHtBMR59WfSsojE&XP_`qZS^?
zd3LickmbXoPH7_ZiDCO|Ib*nq7=7q!<}h>+H0P_HEjT<w;yJ<T7G{Qh;sSV>I~u-%
zvtjan9$l6&Hi(XM3InH25PS6&5jcBoX9JuKk^|u>7rQXTK;keQm$PIEj7@FYUufoB
zbTd<L<Ox}qvYYEGEadtA`jp6v)}tm)3ug+jFZ&~OL!?8IaYD0@Ly(}QQ$&h<!10R`
zzkf?x<>)=(Wv%hD^!O;HdDz!fz`K!2Xic4*@G=88i!U$VihL<%7Y5mL(U4o<uj&*r
zx24xab$zWYi<74b&0nxjaoy$1@@bP9l+R1*h)0Y5VZU0MFPI_TWAW3!Payfp9)aTC
zwSqS5ehaJiJ`_Bjc}>V|ZVA{t(=ST|TteCTL1uT~JZ3SOBSOHOnNf&s*E0Qe8fS!a
zgo-RH{SNb=Ik-!(_2F&>7p4u`iJokH`|mdE#yR>~bVykWGCDB{SsO%)JoTO_ro2o?
z5M<WM13P%v9#|&WGl4<y|MGbL^WO6WZ!a+x>Q3tCdwL^W3bcN?c^A9Ix0{RtYuF3~
zCv9=J<kMvpj9Rln@YsQKTrXtB1!knaw#f3BB<OnQlE6pidck*hxdi*T=Ll^3I9<5F
zgqweB!2$uTY9>qGq$2{1_tcbXe@TfoPB<cX!uWzf^7075?4t>MnuQU<7gyfm{p8y$
zdo1^-Nb$Z_K}}U|up7^?Efa_nvJ?WjSJmJFUvt`bp@fI-JV8n70_nF`3n|a&64F>`
z!z*!Rv1MGIk=W~NCq%z1Ckh$Po5a=rw%_8S)eM0TaeQ*QZ?uKNHr@~l^Vw^0_OYMP
zvN&gfBU1x~lf1SIC7!pJ-XgPEAZm?)pzte2;cwwx!gE?C@v9ZH^WTx4#24gKFZ3*X
zkNmb*u|m&0owY9<t>BY2GPhiEB-A3F%b8cl-d9M?xk4cQ{U(9qTi^1ZdK74R(%Ma^
zNGMyd%uHM8ij|6By#EBDq-R`;r<C=@p3OZf^xxk^&UsZa-_-A)z-7ldiTi>x_aBy9
zc1c5kEjmR=@p`3Ttcj%n$lssWr0_*Pmjkmw{N?>81aC4rsPAU76R6tLAt2m$$?|Qx
zrO<9Ob)ltlY|6>%eL|<SqWKw^twlHFiKs;!xv1>5^^xivDSNGdt@A8i3M>~qzJrgu
z{P{j<iFN_`Np;*p#^2^!_`U7m{dU1p#PiT(%OkvI!b%?Pg8OBk=m^&^TSPyaBxRTQ
zLO{E;O~}iO1spdZx3pe($NS$*OiEzF1)=nvh61b~ghVc=nMqEP(-S<Tdy{v^>I<eI
zbs+q=W{QP-cY#RC@)g1s{|uzmmlp`SH&hEKRT>JFw7FVn+1|34ePyvl#%ndfH))&9
zL3%XShKPlyEmZ&hY^lZluag8j)D8&*T{jWVa`P8>>b6gCE$1R$gN2r0_jvbw5&S4{
zNA*MPZ@~+Ca|ET9FBh7`GLP@Kv$E)}R}!LWJDNn8Mf8OyKHJ29#EVsAwu!R9PGLvU
z^@axo0w<VScuia;s2cc5Tqtxo&x-Z8rGIT?wrsUj6HM(_wJ5guFYx@sLY>(r(**B$
zPn9#AUMCP@TqW@3bi2j#RrZ!|tXYIk9-gA5RGe$ssrgut!+5sxf!+-w9My_K4~13C
z`*TbLC-2?}b|-JwM4@S|zolxv>j?ELYVl7_-!Ise!EU)P;-;YR^Xmfn4yu+WO=W@-
z8PP(kgq1AUvCI{y-?Q1`@zew{?=od@Uf5^a&%ahDNHgKt1g;s^YuUC$3kVAGsBo96
zw+N};b+qhs>k!P^c8$NYib=Wr&sJdv-xEs0XM4=c!X;$Q4(?Qtl03<;e&oB@{4I9`
z%cbw}-*B*zdJq&X*j(T!Xtnsb5WC-JaC&QfG)qY4%51^rvKDFi+tq?s-wDcq;su25
zU#$_G{jb>a@Xmd_5)Y0F+ErfQ{~^XBY^Qz3a@+TNf*1ckm33r@=bxm@FXXv<D#wK$
z7olXgXFSDE7YK5)^zm4gomUoSvJ~2)xS7AbD?-R}yMW+c24_oqWeI_`J*DO`l_3H<
zc>{%Z9M`f)+uf#>UUoppchy(HdXA3<Ap6;KmI};pm>_iTaiNIWQh%XDUnN=2Ib4E#
ztP5rOO@n!7?`;v97ndRQZ0=*hS%qFAq7O7BC2q+JN(5_KeqEs>Qn6ZE=!JZiP?dEP
z@1}QITF26bMVCHo5?aIXK<LQ32>BJC`}j)FMheX7&EnFpoguXNc(1^^l?<ZqQrQLn
z?X3_o4nA)#7@r__&RrOshflo95oBz;#AkhEnat~Hhj}?<c?IRpujV@>d`e7Lq)>2g
zskpFEp@{(3!}YvZ`B-EbW{PRwn6sO|d6mA9iFJ|1gY*0XCi&6ecH@U7EdtzKTtYWi
z_VZ43c)&OL?Ns52DK7+H9`EJfyirJ2N$8Hi+YQqNukDy8WFR@8&n?<f;MI+b{MM`e
zEm;oT5M&md%XjXcpK#;4dai!W6-s^g7}Qz%4+=@CFXTV^%f(`qS&PE%#A?f#f1QOz
ze6PrI1@;Lh>Rz=-JbYBpRjW-fsbPY}(_2$ij28;=XZg$%FsbJg@%cZ)a+!^q$S;*o
zyk{f1g@Wy+Ec*(sYp-e&6JD3K)-tDU2fw5GJi!Ie4_OxUUf@0TtyfU!5|?m$rYqNJ
z%?XxGtkOcx91=Q9V`mFIvYIC(ym6x9h7UJ{{(0&PmQ;Mz<bLHU28ve@{^|Cf4@Bn$
zyJ^Kf<`BADvP0<gl9TeOJW~WbeHsOLr?(3l)yx&);FaJk<_i!Kzo08PXW~V$o{3i5
zgvx$@5D@p05G>S;5!(5QO@`@Dm&lRn5yJ0R+>-;DyLxY@?fh>IHh1QF+A48~?=fxN
zV8g-OyZ6;T>Aj*-r)?#a{_Qy_Ai2kZU&rQ_a?{=q!53_6n!ENy9oTQ%q2{ya&gRK`
zM55JfG~DZVA31+!Z$g@}?SxiKn*+D|_A)qFS-smSVRKpX@}4>0CfQaUUB0KNMQ!he
zWL;a?)Yp5v7`E<J*!XC-mD>EhR-GUAO8n;B%b-xb{m2PZ+YU|-o1mt@dsa=KWc%Xl
z#JwVySM2?NyTo=0!(y8Uw?eIt1la7&Xp-A^$BTcTt3%h`s<xGTYaS`sx*D?Xo0De0
z_Xzt;+pk5swmtI7w$p@@Z5tk3+pBG-Vz;>EtL?5AlWjexaoK&ie`9w_eu%BisV<up
z%cA!FXG*m$UBGHHaq47S)!)+lo&-;@efeju?S(m3dtPu`+WuZwYikg@Yp<pKl)ZM`
zQ+K~B?zervWZPZ=o%MTU86Vpmi1XO(@vqHF=3tZUt}A<OP4(4mN|KmuU6<7FooMsh
zdR@~t+e?8r_D<kWw!Km?!RC?iZ<`J0zU~zf;oSRSC*!_n7VGwgeiGfscUi%1!M@A8
zxdPVh{k12=T8(jm?R&3-d(Zr1*(bBE#nwVT!6yFr>Aee#FWAUz-(>qs^_*=^n$+F{
z|EqV`%v@!2uHu!gcP^7{nN_EC_ncpQKevbNJ*IYIuZf7LEyujhz5Rt&dtX=!*m!CR
z+m*Vr@9ntHX!Aw3-L_xk@18ZwzV4m2GGgzsS4?&j&I;OYnImGy6>xCx{B!wxqoTI$
z%{nvPcHTvXy~!3w_kPpj-uuMA!ZzxrjBP^ylHHFkTH4-OGSil^T5?~*T%|pww+i>p
z&)sROu}937(Rbb6n->E1PHQ-4n{z1Dc7q+`o+X#J*e;xX)h6#i`kvM;yY{e&T(Gse
zw8G|B&T5;kD|hzB#2mD3xpvmN{jjv{9*acV-cOQt(JU4=eKDJDXL+2oHTBrD_j%cQ
zTknPJb`5KG?hUHpu{%EH$KI@a9Qz86RP5al&1om5>9%)6vyaWJQ#1CYPHMH~32U~=
z)?Bo=Fjn6-BKXW+hE8qU8udlC6J+<<?%!Lm*Tgh^FV|BA+d1;T_DJdX?)^TS$8OnS
z$Gxi@GxqApow2=>d3R5M_CA}a)@`=?SS0s#?3T3Qd7)t&=YD0c*7OcrgJ_Sv>kpgk
z6}-5?c74%z+Xr`f_Zlq9-y2}Na&H3*r%l{}Www7?%xx^zv)Qz*>fL)jk=d3ZQQk)2
zpPlW6o~O2#cFXK(3K!kG`7Fb}1xIJtRNFG!J_zU8Eq(RmUb`!7wr%M<ZCiSO+jeJc
z-V^=6Xm6Z1*WPo^S$ln$58F=2zPHC{ZjWujjB|Tl)xX;-o@u@}K%r;v?j1_DP1c9@
zYMVdYB_kDJyZoiBZN&4{)>f0|?Dfvn*t?+P*xnc6cD4lu;(I41yxq&FJ8AFjI6>PG
zKRMf=ASt^p%Z;`_UiI3RW-hUj|21jPoK#8MKZQMe@87ew4SD0RH-L#}?-Y9<TRkC<
zy&unf+4CSl#y0M3_TJNHSnXb#v+jL!PS*C#j+3??Wrz37{`G$E=H-2RUu^qjvm?cC
z@BIMfy%x#0Yy!$WY&*MV*e+G&vpVUtVz=4}p?y6HLbgU<_uG2;uh~1NP<gM&t0Y^^
z?^b)?82Q-hd=#>qb<ch8MxM2MHNL6XPIBws%j278yJXI@J=V=r_x{$GwY|D1Xpi%o
zKejFj+xI+3=HByY^+%f)|0-Mim&f*6v+&r7Z#uPi->tc}8;=<5JuW(Luk}W4+gCFh
zY<ZL{_x=&%w>8mFw&mCov1h@rmc6?k#_pNC%XhDUzu{g!r=C3=W`?#dH)QrYT%WRM
zM)4`zmWf~X#O>W^`$?g3?@psNdpFuPSodC2u<lP?x5v8DVfXg;k+$x+(|3DbU9q>9
zk#Wzi*`|9%9{sjqo43!_YyIrqH#hX!em*9#kJE~4_de6j)(Y02Y@X~>wVv`TbT3!v
zZ)=w>M!OqR*4p0x!@u_hhw0vg#$$VV^rLM%z31%R^=I>5#Y$D%7|UPQpPB#LvMyM?
zce?ge+XWWu_Smee*lY8J-%ezbp6yYU*?V^^6SV!M=4ESPQ)+v#Y5QKL4V-(!-%j2e
zuw|AlgNUcK^rEeM@9>=3Gyl-dJ$Kd~-Baf0vG>T1{yod*z2Dn%eC6I1vu@hF)Zbuh
zP-<*rJxgluBd>+F>`znnJ_?>@n=mcGmQ#5DUNHu_eJ)kKdk?ri+--6Bg^eX&_1?%v
z;eE5Zw(Xtc#&73me&6Q)cYj;yYO%e|5%Ig0zFfI?j|H!7j*Xcu^ZuN@JES%C3RKOv
zKHz<Ecd50t&1-gvJq|t@yVs_1?Mprux;It1#^#`$l&z#j?;fL968lc1U*Fqz^sMcM
zneVLaj5pc7VeGc83YlxWiQ&NRSEo{K9QID!YZR5bca9U^UXDr+>w=XhY)U6*+ElDM
zWqV-Hc3XzWw!0n+?yzlD_S`#l-SfQ-zfapdNWZ!J>3>_>h4YTtUNKp|XIZS=K5;=2
z+dqMeY~>3s+N?dBz4u0Fwe6$pf_t|LT;J2}vwp7?JOAEB8+V&E-yZEbrYCJzQq5+2
zq-}z20k?+D&sjTmL@-UV+5a`gde+~WdzsEN*i|{X?)5C?v3s4gbML;Lv-aM4!?M@%
z>+!u@Cu;ZT-CAb5ZqWqW>)}85SS+xyDbsY=tHEDwb8fEBUeQbZb|;tc*jnv7XTu|3
zW}AH3fA5_2TlS<WF4_CJ(sxg>?z+AEiWk|gzb9fh!MM%#ZgiN9@v%9!Vb>P#RWdzf
zJHPRft&iPN+Z7pNd#7$s+FRmWu(vB~&fZw30^5(xlWlE9p6`9H+i4?ixz+a8|A@UW
zYyEarU$L}3vv=0sD7{0r5mwy$6y;X#O+N6#=D#H8KEBd9dlfT^_B!<$+kU;Rv$wBf
zhwTOLC%d0@@$H+fwb~~93+KK!RZI3d?LW8Ye#+FnHQKvu&o5ZG_fd4!-XnqAY?Zcq
z?0v<RZoBxFw$0y|LK|U*Mw=B?kM>r2sO}a1UbT1Ghx4{SHeR<mCva%@uj;>hCw-W+
z_uoBUy9J)Y`*h3~?|HV8V_$F0Yuku&hdt_Bp6{tYban5VjQ=(ZLbmR`!D?XhJ?#4)
z8^%<dWr-5I9v=C<+xp-Y+oON<Y-fFT*_-#c(^kFp=-y0DKU?$o`n~h!^X{$eYuvN%
z2HWngUvF)WIv&|ulC;S-RR8jxnmr=6R*&!QKD=t<-h=nn*kpeZ+glJY$u=rMX5Y5n
zIo2QFIqp5_xMi<%Hp|}qp+EQNzFWNa^>U7V2j(2M7M7IQckH8}-G_^F_uhW}#5VM_
zo$dD6!+YC)KD2R&KD%e?_OG@*x0P(mUu?IH-&Abd5OHAd`wj-Xs8{><en@NE{aCJS
z?_I5rHW3pHY^IfM+$+;lym#sP`!)gFChmRJ^I=b>z0K}~#X)=ie$BJKbyaOIv&0;m
z<eQ83?wI>#ulF<Ey;BYu>=WN1XxA`rj_vi%>b)1O@9aG@$8zsyR+W9L-Sqc*GH%$r
zM*5D8=0^eBj~Pn){`{@Bsh;SzSDZn0-|LvuyK59r?csSiYp>NpdAm(tfA0NuVzcet
z8qK{r_MNuEE5Gesz<$?yuLsM%l3wL~!d;8^*2o;$)7Zam@4{y*_x@fs!`gy3$7b0+
zfqexgGpvskKeyfGw%Jxlf!X@w8^%2q#>so8n`qgd>zT24uC0{qC;8J>LS^#%c6a6O
z(GGIAF|20T=d_e(?>P;HeY)w}_CDcWZ^Iy1W&PM+!TO`?TpRJ|_T5b$jJ6wXovanN
z?X|i2Mr_|fi(fY36IpF@Ci~dVeeG}io}<}zv469z!J`GXyvMfMT$wLzx81gIZ(4@+
z-Vc}A_dZ{LZC9T&o9)3(w{7hrW%lLm+HM^Ywb&-*Qo&w>%=>%)Gs*3-Qod-bT)fIw
z_r2_%rxPmn>P??$bAp9!?}xkl_MT!ozjuDZ*1bAfJhs}UoqNrc#B6``w(XU9EwZn_
z*Lv^d)||Z)VtQ@b{&d-Dia)dU(1^CJ^*+A)nXK`?R#l~azn%H_y*n{;@08j#d!@P_
z+oY`eYop-JzE4zmsqKSzOKs~LC+$@U39=QKIp3Dujb-1e1uXkK{5kgh=G?KjiGyz+
zM_tpNhbv6>We1(vJJIyQ-q3{kyT5rB>}@#WyLTR!#J)P=<$Ejq?peS3!MpFVYP;=6
zWft3p_TpW^8awxNbxH5r!<lHyQh#f&#`A=|30%_qObeN9`|jrMHTs>n_fP92ThsEY
zz4tEO-|L*Z+^Y43(q8+CBKwxEoNl|r`O}^SM_1VNXt3|~U!!I_oqOlr_U(6UALgs=
zlkm^n%d=L)HsRiP8_8uGcL^)S+8SE#v`NifzE|eZ9^0^+8n(TkPVIfu#If(U3#YAp
zT+7~smmzjW@%#6d^7LA>EM03IFuQSYu-$&^HD*4x?Qgkk^{-~zT7*y7D=+tBufRVR
z8@VYhdl`2}+Bz7`+9SsAzIWlx7d9TzNB3kd*|4|Z;YXXe%tw1OPR8x=aJjVG<<LCa
zqw0KnA00Qc)v#r@^D&ugGa+4Q-)6yIwhePu?_xZ)b=Oz<HrwpbPkW9<SnfU2tFiZm
zQi@Gd+6CLcv#0N!e}i?eSuvmO=NTe<1rMp%E?|(bopL7Kc0zjRo~Y!Xdz4n|?sdFy
zaQ8B9Iop+cp4fzK;<e3t(z(~Gb@6WNVAGwloud10?Csm5Y@lk(!OygBhsf)_N`KpJ
zl_aj)UU+?G?`p29y$9ntY`zwX?eoy&vI}VXYAyBT`yQ!j@AqCev9dkye0uML-?weE
z)Yt5lc;&mdf5DF327j0BJswiKcglkQyXT(WU~Al>Y1=HdcyB~Pm(9n;T>IF*PTK4L
zyJPRYijy|0!WH&e|76+s^(gDUt#|eJ2=5fP-SL;%u4P%oUS&}u+oVf-_Rc=eZfD6A
zVymCbw{P1v(S5xYJhuNXe%tf+h?H$tgP)DhzMZxwB7FBYcOBTfyi{;cjo5jchL>OV
z?4BaJPv)h;-W{>Qwk+N=_uiCH-z(a_!uH(u1$$mDJi2#H?wdVtTlj1@x-H);c<s{O
zut!FFgFEH-P2I;~_i;tTUd?qrdt*-Y*a(FF+silYm+i#rFE$T${<U$OBf59D>&Cqs
z9{jL=Z*{}Q)BfR}^qVhjyptF0Wem-;QFZg*`@v6S-^=Fqy(?a3+FrW!cW<e(`<`h2
z)3#wxuI_ExBE0w7BeT6T8V}n%30t#AwC~~`-<}D3U9|o87Ay|3Wwhqm$9-4QcCq9+
z+wfbzZN<|c?ESq)z_xGQ!@aJ?o;JTeY}`9-@50@ZH52x5xT);huyeD`8avs&ty`n_
zy1Pm3bIB9fmvZgM-d48{wv%dh*@UTIu`SBJwm0Sm|K7scYi&KcuI`C9W!cww;+Rdw
z>m=J_S={?3m*3dAxb3rTR+7ZN<fd%f?DBhi;uLE4zUvgXeSGq_?W3lKJr)y`_O5(;
zes6i!Gs_)`%(hk6r&=$uYO#H@Nyye`X8vBrjS99lTRZkvzmc&u{r6}u!}=_1v8)F+
zKITn(X3GZc4dIlqy&IWl`)0|jy~ZnLZC-AEw$~_UkL{(}<+h*yO4w?Y9kZEwKVvVe
zaj|V&9it7)0h_(+(`5JR&-uEymPvJA`66yRp=%p#IbUwu`!VQ+ZNR~rJ!kA0_pSEs
zv*l<k+;fdNYwwM=LwiI%?6g(v-DK;mc;417_rhM|XU2QyWsC0p@gvHnqwJ>5*4Bfz
zOP;6h%~F-I&1+w5v(b9;-ikFYw&|}W_A0DByXV}D4qGK2>Af~7+iaU$KiZx)y=7a?
zz`f7bH)=08lZH+242FFttfOsN4o<VZla^=Gap%XLJN^x}5w*8$Gh%k{ea7}_uS8RX
zt@Tb;8};-Xd)fCKv)RLPa_=$qS+*~v5^erTp4t0PZRQ>ZOOd_WJJ{`nK26-ath?8?
zL}2sYd-a=b?=<Mx_BI*rJ({0qbNtg@TXBuqd(=`*_t{$6?CoGtv<Vkzv1Qm8ZnKkp
zzwMcmbN6OWD&0HhUZ-u|jITCj6SHmS+)KCR{=mGinZ18+#e-_wiVZ<FyXQ`}UGqhE
z@9MQh)=%OX>=O9z?*1eyy6;Yxs%_g;ZM$H<<9i=)UfV5GBV&_M!L|2eTc{1Q>EXRS
zmly8cuJO=DvXN(>Ra(*B2ftKz|IJd^w<ocA&ng2I+eMXvb~0COY+wA_V9O_YWG_pv
zk?pnG<+h2_H}83V+h_0or#*W;S8m+voGoZq6MDybUlxbmR;4dCB|lR3-nk%dyY1Zq
zTbB3+o2;@AJDyma->v?<dG|HRoV{X8X4tyi-fg>kN9EoPGAy=-_;~j1t#RFZ;JB)-
z>{F$Et8N<GF5#YT^QN|A?}8nSwg(=(->byVVt2+u*EZAP)Ly@uoqN9b1?}BC_p$9>
z2Nhe6ve`Czb7$J-waV?5NbuQva#_>fqq7g}l`P`5)!5Z(v#L^O@4fJcdqO|3+8Xau
zw~e)6w_RZQZ}%UGejCBk2-}CII(yH(IJ|eo)ec+s{TFPbf`V;sFxS~|+x6L=IN!2&
z@wZufZ~S)L^Wrs&?Yil2teXVB?fJZge^2<WDSIyaU9n|MH?wB2Y2AC`BBR}vL-Mv8
zHeI!re)i1zt2n=%0AJH?mPd?x=XI;vUaaP|>o_f7)6V|O_C=1TZH|A#Ui}niJArNA
z_w<`B+j}WHeJ@|vL0bs{;k^yTYitkMaoOoH9<==tcG`xsaI&qm0r%dXC+BTrw&(2q
zD74)6{K|K>3OVd{ERRk1u84}aZFO9?r}>8FUh5pbeGS_8_Z-jOws*^si8jk`f7m-C
z!Q7Uq#Lo7%#6+9TKla!f=D6ET|8>^3;JV&Ep$Xl#8LXf7YShl&n;a-)>u`6{uKNAD
zdmW;C_OefI+S|!5Vi$LA{oYRREZcSWn)m+M{d!Nul>;^p3Y>ReshDod@2Y7d^Cx5P
z&Z0loja#<uO};N?muP6acW0{6-V3{hY;AX5wT+5;X3N3vx~D+<$DYa$ZF>w1vi77!
zGwoZd|6}hCt=W4cE?lwcXky-L(OJF6W%|QCHhJZHZTiY=xi((eYmy{m`*#!P-iPk$
zHWj@rcHt}+_eMBOu<hRe!6qm`+@{Q=d2b`H(!Q&GhjzKGIc)QIwaf1H3<qr!V%%*P
z{|T}+n%20Nqn~H*g3sUg{H*HSTXIKuZ^*}aw$u3M*oLdTwk?-Pv;EMlXY0J_&z_SW
zO#7w_J+Nj_+`T*P=$^g*SCs5^Tx7Oa;xC`=g^r&#^VUq;d%BHx-;IN-_LRu++a3@x
z-m~S;^F79^SoelM)82b`{)|1^JEQjA(EVbwzeUaFM6#tV`~UT}p8i(0PRAYh-pOCI
zXa6G$o1m_ndwH+6?6o$uw9WNew5K9!_wK(}-`QG)THCJqIDfCH%{|*6CCB%c-(}kO
zJzv^xg8rtxf9@>Y%MtK!k7oEITO+egdyTa?_dT!|-1p8@d{6zqH@1=tQ*4t>gl##a
zCfl|ge{6f+Lu}uJ0LMMN<#BsBDzo=2E@HQ2IO1jd^lp{yubxU<;Xr|X|BfuSRei<2
zH`hhdR_E&OJqE4+ZT;#e*dAE=%eF^Y*ES^}!6xHihRvT%r8dd`J@%e+UTeGB#9*(i
zu#yeizSXwx6TjLhm2a{+*80G9y<Ugy2h*>6i-LM>=di`?J#Kn=@4<6zwrUGS?feR+
z?=@y>v}L^2xz|@|qmAO6{WekOkJwi6wAvcDOt-!hbKbTyGI(#BuD7klVzs?Fz5n(e
zDnGkdPrqSr;(cvflQtV$c3qLZ84K6%v8rg?>ujZN7Z==QvzaAeuelAo?cP8w+qdGU
zZD;M?z4zns?Ryj0r7d4M?6qCx^53RUcZ2Pgq}O|Y9e=Y|V>^qjl)C(0iBy%nJL(SE
z2<+sx6$#?EjnV3`4XU2HS90kRTeY3L_AEHeV13~w=U(k&CVPa{Liaw?m}r$;{nU0w
z{O-L^HY)91a=T?O3tzVF;rTmk73yc~<?HFSwM<#Dd+~t_yA2FgZKpM#*lTxCYVV#m
z#<m6GVs;g)5^c91IJwvPZ0z0_Q%>5RU6-}{`{{Linbth9Eb(BpZ93UvYkg_Dtz(|-
zUVBruy-PD!@14uyX?vo7qAmL*Pg}oBNwyRA|JuFoQ`_FRle+dk-^#XI@Xn9Da!YM(
z9E;laI&9Xp`|$ICt<s0Nwq{FI_x}I;+oo^ZG~4yf>U-uzzu!B#ZkDZtx$?gHT@!31
zm*2JJH|5<mbDi{FecQghVq8ycHt=4szOrY_p1Fr3_Jk`Q-}``zfA5o>#<ox1&)mD0
z?Y(Ws^e=n44$s&fS2o9X;@uG231<D)uf8|gu9RNA=k30mdu$@2_Qrk{wA+zCXYc(p
znzmc_n(lp6J<;|-uH9ab96cN9{f>KQcz(3W?ptWf7gA*>*09pHv*fqUQ<h0~-I0uY
zCX_F>^%pALr)=N4_iu~RK5@Z2b_)um>{10E>}z7$Z0k8^v7OvA>wR0^PqLl%b%E`L
zIWKJ2|Gd1fQD=&+Uh}@a0la258UYLTa!AYCnay5ds~*v3BiJau@2%Xleet{9?X+I}
z*%vgk%eHB<@?Q6=_jm15erD%-ceRa$jGAre@>kZ<e--W4F+H%>2t8=?M7r1ZwBEtJ
zOU(J~=Cy9Hot<dA_saDd`%*45*ywE+-rF&&-R@SH*WN8_*Y4Y?FTF2v-{HObm6!L)
zgs!pMd2RAuhpe-<0nVT8&KfD${xH(t8xUw<dnQBKE-ad9Z^gPyo1FSq+r+rPwm+_P
z+2)7#?7c1|yjQkZc5iLuzP&t;dTrkr?%u;G`pWLd3CVpctm|#(>I>WZOEFqADsk?e
z(!^tXwa#g;h34aZQ|xxz#Opq?HlCPedo?3#k3_Bb-nN~$Y)|Cuu-&S<&c@*H>V4})
z^=<!7ce3m83b$RvG;QyuPeQge6PfnSsS(^4cWSEbHc_R$Iu|<ZE`H9~8*o3+W|yqm
z-U&T!drbDNvn@JxWl#Up#=XZLpW3U@8M$YR`6Szd=@a*~+s?FIwq=j)nn+{YD~C1h
z6wd#)y`Hzs_U~yU8;#=;d%`tu>=RhTX|sR&^?mvC`uFn7T-a-qy2>_8Uw$uRGS9vh
zPe1P2_0GihPR13xUD7PJ=Np~&Zky}9oAJ?=JqZCzcDE#M+cRe;i*<-7w{3fNg`Gy@
zF<T|!V!Jto$8GmU-P&hf{nc*I9y8kp%X79;H%{-(OFdzApsalN9<#;!9L&zx#78aK
z+tCndd!zOG-jdu^Hqo=s?S0diXe$@YygM`V;l3Fgg!dM5-PxO9U$OV`$pd@;-JfsA
zoX)=|zW%`8-~vfInW7_mZG0nb(-VaD-JYdp!@l_F-Y7%<y@$Ri>^1j#voH3Az`jN?
zo4sncPuitaO4`oa*S<$y`k|f03JJSu=Qi3-VQID1yDe@XXmxF0wEhaaQ@w8cSVg_|
zUEI0X=0yGvn+N};_a*I{yXV=icpLW>llFya{M(bj;=1?o?!9(9c68YWJZjnf%j2fq
zg~T~_1-d)!icB}!#xGIWQ+9gS-kLy7TLoKZ+gB4_@6~wyeeZ1sYuk=<k~X}zg!b(C
zWV!Fc0RbBg6aT$W9?9D?Ox$hBz_QdveGQ9!MuDI$gTVVeb`2Z$?Z5WGwqbLMRi5=B
zTl@bnZEjh7-Z!E2h25P8jW+YQytZ?(kF-??KVz4F{K7ub*W341{%5zX%2(JYBEoCu
zcw^7r8IKHXetfX7<*0VF6)HPoYp;{Hcj|*FcB{86-TQb>_ul;umbT4;HhT+GKiYOl
z$n3dqHQDxyhk~s_AHVgduGxF9Nxt42BG_%K7U{I-<c{zA9{$?3ca_l-+ZHinTRZWo
zc4s)}+kM@&Yfs0HZTtA%z1yQ3ILYpWAG<9(GnakM#Ev}-TFra;-nZHv7I|iO@6*w}
zzo)j^8H)SbaTng$qkLV{u3@qHUawiaR@-K8w0r;h%wFw+Gd5q$EN#=@o!h(8f16!=
z`R2V1LBDO{H015dmvrvSbv$ch=rCz7ziG?93re4Dy8IsO)s~-NCwVJ<uWIB?yDddE
zdvjV=?bVCjV(TjqySGSc`QDsIcD568FI(^MXxek>>vG$-FCOhp)o-!;tG;V5OU1su
z`~7d)RmRQS9kyzT-OY>odrd$4?YsG?)-G(q-o39fC-40{p?ObL+M&HW9$en@WogNt
zNi+Z3o~`4zJ@7Mh&jG{vdzj?@+cr+*vwgxoZ(m$-kj<X7m39|=+-;u(mfF@<M%Xet
zwAk&=y}LI_=;PkMAFcOfUlOvjNL^v)FTHN>1G$iWDdlHu=HGPKyX1?i?YS)udpv$x
z?fGxYYtx{6!G_biaqmgxM%(*09QPL1=-JFqQs3LPUv1x>jq0}fd>8jU+z`Ka_M<*q
zzh8ZJQnteO-II^*u~(h6m;D5X{Y+M^y$4n#?vA#4XKUYj+SaE_a_^56UAtKsyn7Ra
z9@|-TaP8yf6x-9`6S`-|2Nj!FJFe|IZ^dEf&BAOY5;=R{i+emaDsvpH#QRk3p2!~A
z`=avizB!ZD*<REawOix1*yhew*8LCIi+0D~cia1@y4}`}<&5o7Rt3AGRlR#>r$67@
zFz>U~=GKRHXTr<vt~EZ~7jWkE?viUldv}EI*;66bYj;NI_}+ru%KKheT;Kabq|8pN
zLuYTV<d@xtf4$#Zb^GDo3w`hQP1}0IcCL-au6frd+n#BkX;-$W*LK&pl6`YZO7}|E
zH|>oUUb)BT6T@D?g}?Wn`N(V+vFpj+b>~?3DQbV-m#pZt_e#9Z9)V<uy}!fv?={h1
zY{T7Fv-d`8`99IxEw)?svG4n^@RrSswtn0H$JzHc&uiaj$T`WjJ4a@3??svY)v6o!
zvX@QRH?gP6cD|9FofqSby_=_AvitO3X0P~fKHJ8)4R-gt-S&M~ci;Q}y6m1R-9xs&
zSeo|s&)8!d@@JYY-^6xXoAbx_)nx|mTh+8@k6-f?yE{q?ZPy(x-&OKQ(7q|NZ0{C7
z)4dzZ_-)_)?y&v*<Fwta+#|auJe^|a*siqC#XQv3GiC0+YiBw4x}9ETtG3~iU3HC;
zZGmUY?gZ({HvDG=?QWf$ZEG=|(O$$($Tq-F%En4QabHw|{@w@oSM61qwsy~-n7_6G
z>Bf6rJ(u3As&Ic#mE^j;3Ujvang3AEE^f=Fy$stp_Hl2XW&7^xtbHPWJNC4l_-MOd
z$b8?6iN|f@?mgWrdWv!16e(}pw9hB@{y!$LS5)caUd>K^>s`NMZ8yv~Xp{bu!OlSY
zpzS2ije94^udt0y-n4hs#-Do*-mcnfbE9zY)%yqc8csg5$92jFo1&fkwp-P+_ZnR0
zv1NGaU@a-NZO?_iS9{qfOYQycD7IH2cJp3c(T#f|^2=;GG@14Vq+hqTu;Sah#V}!S
z(C)>17S)v4w3fN<t&94%_dwG5y{jyEce(uUvr#><aqsRG-1`<>&bDn>JjJFw$H;oA
zzQA7A1l>Igct7s_Cvwr&=%VbtJLYoxj$NFz_xId|ws&0GZ1t|L-@9v_!5$Nv7klz%
z@7g>27N6}?Q9E0`C2@Q2s~_I`poM#{zTD3}|58?4JF-sN?PA4joBVOwUa=tiz3e~Q
zY@F7-*>$#d;oeN9M4Rv1U+mrWkJa`S7q@Mp#D%>L2j*Bm@G-U3$z8km%>4GfZ=5S^
zO)^gGesgyF-s0qT8~;F_ee5Tv?u}uPw&mK@XLESh$-UE*8TWFWo@TpIdEs6=^VGdx
z4#?~^Hs8EgVL|tvoiaQ3%4+oP`C_==T7jX$Mu&0E-ux<oeGZQTY+XfP*zj+#-@_Ez
zym#K6^?PMjH}Boizr@Da+hp%wSH`{4M~~YyYpCr_ov_n3a37z|zR<aQ*-U-+E|t&T
zTPrTKXVa0|y>l%(ZPYF<-7E07V^7fI!o8i--1o-%uv(X{m$qGS<(ZAwOGeuSTRrRT
z41IfNe4S!5=X}B5wK6Sx%-IEP>rPMF+p+ni?Vqn{wu}vjb|-Ri*@aK&+xtH&!1m$4
zm9_%s1#Maj8}_bg{kXU8(Z{`hXI9#BFtOVP>^^6+;O>dNE+4CGtIyoryV9&_ua@n7
z+bPc!_MH96wlDUdo9!mam3wbU*Y3UWVvDV8)V00#5{LJ0XlC7W?X|4!|4+>Ow#Z-K
ztJiVd_M>pZ-ap(=_ipl8Wc&R2guNl3pX~|cw6MyU`^+Y%w`Xtco6WYV65KW=uKRZ1
z7Fe?P@wVH06HK1jJi0T<_V=T6wl-}qc5j@z+4j$ulY7Ji9PDxg)pvWj&bRfx6>EFL
zO2g(QN9NuHHqpI5f3CB+b(>{xj>F--aZ{6b8#KS%ZMXdC-U-1wY?t{Rx7EGlv-i0C
z>%F&r%k1M|jNX&=;(~4Pf%!I7-#hn)eXO_5xixq1y+#hZ^Dd&+{H+2u-qD}-uF$_@
zGlNHJPf^@C+w^Xoz09#Y_FBtrvT5GVy*F<CC!1$Ac6+Y33GJKo?#=F}clvEB>bC67
zu&&(Obdu37q^Z|t=?Y)lgNOF+WuL0CmpPez?_u+aHq$ty_crXEVtwL?&EA_4!ut+B
zoxbPhY}LI50mU{R<=ghQ6<pYBTe^8~Ze5D4$FXg;Mx6hxIjs2hRmNSm;S4-!^Ka=i
zn=gCqY;AoQ_GJ`pv|04daIgB6uX`ENX4;%lYuIzCJ9h6iCg!~}Z0_y}TzzKm_gx$I
z_J3Npcfx+{y$m;(?lqL*v@Nw+x97vJG+RC9&3g+M-L^e<nSF1y!cm*64!ie;DogB}
zQ0cq(K=4G{l_CQBK3(qKv**ng>*8nWdwYHa*mnG1zDHtOqm9Sb!?rQ}(tCfL*t2)(
zHDkN~kv)5Ft%}(z%zAV0rjHBvivP;m+wHJmkM^R1y{p6@+jg7}w=r*+vWG#cY_Hn%
z1ADy>pR;WcY~IZ!K6~$qMQk>QS5Mr#cioY_Tu&YL&FEp=H{oC7o|3biwyey4)@*6)
z`*Qzo-h0q^xAhsd%X_VxxouY*m9dRY?y#;+I%X3X&uAz6^|MWG|HZu>OZoS)uD@op
zWwOlP^;2Z_JyUqLr~cOpn@Z>7dw+3F+rzYbmu+nC`n?OJ|Ly5z-?Mj1g5zFsp0vGs
z4T82ex82`sd`R9p__vJhY(0~`tC{!M3SIuS*Jj}?n<|e$TeCNDwp%8P?|WC!V7t)a
zoUJkk<35cX&VA3Kx%V`?E#CXIv0=~e)}?!^Ix_a!?QPm)%64gQ%)H5aQ(Fvdg)^dU
zi@r?RYrH3O&k9~c8=vw&d!!T|?fGBIWcPITavKlHJ2rE+e%zxRCb4hEbXi+5BLSO3
z<<s_FSS7fxymj4P1+8g&&Td_?*Tq3@*B3FCeRePQ?&;$fv<<VA*mGDv-gd=HfxWkP
zZn2SCD732~Pi`;!#m2qX!ixJYhg#XrcVw|!I)!E5f;xk}Srzts9iE%o*gTrCM{<wE
zzDd8N?G$R4?Y5d;V&nOXbsu}8rrqg&seMaUbMCuX?_<*}&uZ6ju+HYsjfFNJH5>N6
z=hN7`W?9D`j-vv$JJ>n*UMmjX>;7Z=-X}tb_ew~;vt6*^ll2m(#=RXIrtirv*|)d;
z$xiDlmi>GEwNv)8=$+o%R(;Fnb(qk;tf}F9T@HEJ9?d_xSFA>T?-z-Ly>EE`?S12U
z*Cukwuf6*gZLs}+Vf)^R7v9<0U2WfMlOtuz;>fmlab@uy2A0QrZ%sU~=f}Y^+oEEn
zz2`j3_xe}s>`huRXHVk3*}E6dZMLzA{bwt@Z{c3&ra3kOLC0(p_loZQe2I5o{WXid
z|G)F^J;5hoW3I(yCt<Q+_j;Dgwr<6udv#{c+x!3RdfP6!^?T>9-?FznB5LpM$?Ny-
zjQ+IO+}v%~uhT-dE3E$Su@8N;x2np{R^t0@Td!|Z_ax-3+0z>>wwK}cL7N3(X0~(K
zS#5jbkJ}nsAG7skoV54J+l@9i&Rn<g{4ZzAdE=~2!oO_Wk4oWt3wCSRe7js~Q!(v>
z&8zkCHc{p6w)YEq_w*i_w|D8YroCG)%(vbxl)v}ef*pH>9!=VFXEo#A1&V^U{e|Is
z_wJv3KxFyDy_4^>?G4<&Z2!s5xV>I~Q*1A;V76b*|735g$YT5Mvb<fp7cAYs@uQ<d
zy1J=NhU)!&!FHSX2Z=A=dolgf-X-yVdv8QqAIMjz+Pf%=WuJ&#ob3(4Uwa=u58dD7
zu*&w4xc`A-*3EmFYWD1(Wx00$%sD&vSEqj4%m0FN-@#7{the3NvhUiu(`MIU$$dM{
zr0+F-p}BAQOy+&QH=gh9o-1h^QM$!e@+p^X{rc~FZ>epx{qN@Du=>xty?25wZD!1r
z+;6jFsm-SIANQqP{I@s9WB=aU3oQ08{oK6Yt>>0a^@IBTtlF#X&2oA7nJ;D8H>1pT
zUz^mVeT*p;wr$BL_TIjBZ|~fx_x75`i|p%d+GhRsDC55V54n2}DJSh+%F?^<#cosE
z>HgXd4_Du{3F+OvccF%r-J7)w_H4D+-J4QZzi-Bgb9;pf7!JfeU9xxQ#MZr$jXir0
ztx~ipZ4t0#p3Je2A)><8s^iJtT?$MNRXjUwR1TKeyYbZRwVx@lZ(hmTJuMFu_Bm?3
z+$&VUv~QJ@%s$Q7wfn>DP4<2B<JcSbR(fB@q_;ME;+E}Y6M3_@L^sS<G;i(Roy(5y
zS^P}F_EgT}y*8$IZ6<%$*_+_C$A)DJ^8te`HTzT}-0UW|DDGQ#<cW><y-D_80=f3o
zf8*G{X~ERJ-}5f+Ej_#TKuZJvKGrmzy?PxYdpRz;*e0|;-yh=<xcBT6roAWc`|SO^
z(P8g9QKfxFvsdno-aTcn_tS>Gr!IBbGj?`aEqUvAV0n1x9zWhL2hHj12Y&m^+PA;<
zwN3qQseKh@4Eth!N$h*SGuD3MF_yhE7#Hp>*v7dpF^Fg1rG<+7@_#TNm}eBZry-$l
z&%#%McBQ2qd*0lexlim(x9y+q8G8k9$J_r;m9*u*#J_L<v;MtZlZE!qxIcd{=P?P}
z97BP<bK2!>3wTBLdej*0ODnrzzd+*7UJ=idy=V1i*ne9)fA9W)m3su)&h2OGV%zKA
z6mQr6n05c0FZH(b=l|Qw7BOqz60M!~uT~a2WO4G?-v7?DPuFCj{en+t?Ot3~*<)4X
z?%?Zie=jrN?Y(aCtox;B*zMK0v(RQ%<I(+%F<bU8uAQ^Ljn98?4$JMm&E<3V7VNO!
z)8pE`ckX$XeaUmS?%Ne@ve(mEWZ&20;<lX|<?J}NtM7|-`eq&TX4?Lq*3Ekjy4&{5
zV%&Hj`Sav`vi_?3@*5lX9T#ri#dw>^R*a=}cUZ~C{b};W4uaCR57=*(-^a6B+~LFw
z&OJ*eNA5SwliK&;u+CoAb1b$QbJT5Or`p?ooV#r=@A-#&?PM5j{ocK>xqB#TZ{Y&1
zy;~fY?LTaJ!)8~2@V<2hU-zuuk-E1%oXzgpgPeW6A<b6RN<DjcPYT(Zf8el=e`v7#
z;D;r)JG-Xtb>4SvU!5|~KHg)m_Bk)EuvstIu+PtHqMc|F(>}AkG6yELY3~zkJF>sF
z;@Ivf>YMD1-p|=9B;K;m$K>1oJrn0SyjaDtcg4j{dj(^K19v<w?453ye4s=oYoE@E
z4}0ThueN_=(Ylu_>iXXMoCo&`@95lncyjr^Bk$DrDQz*{tyHpek4ILp!;v}n_RdH&
zv3lsU%RVheY+tG1$Gyk98V($|I(bj0%?`W2+%EgJPh+swF<ZPh=PT#Fg+ZV8XX`E4
zr}<*r-VNEm_q@`bz3-UAr9F9(Q|){dZ|+&Y<^0|SbK~p+qWkvrL{8mraa8Ysmr<C#
zlqmQ9V=VIf<kFY!-7>v($9_$dz4l&>_GSDBt$%+j+k4B_c<*~p-@SfzA$zC3Q`+~3
zC4H~ODSrDyjk0zt->$SXJH)#8fLQV#vwf2LW;-3-Z)`Jtuj<|0J?eQ^_8!_d!6u(E
z{y>+6<UU8H)%JUvcG~RvWV$!nOkrQQhVR~aV(ELoM(?w^ax2r8tEb6cdF$c5e|4Vk
zwN=uzJ-qXfP0&5D19o4f_D%TX?{Jg%*52E9SM9xX=*EFB(hLr%Mjm^!ZoJ&9zPH9+
z;n#bcEAiEPckW}dPD@O(d?8zIt^SmEKl|fTd;c$7v~SLXd3!_IzwEty#me?qt(VQQ
z8$b5;XK&o!R;Rl6QboZ2N57?QFG^qD>m@#6ug``uo5#+s`@A)$*xr?&v3I*T*S;^x
zUHjY4aPN!GOWgDE`k}pg|9kgF?a|#E!)(6q`{b*(dmHs_j~Z0$Z(4Y0-%?4=gEy{k
z-g{bj@m}_f8+)(*UB2H;*4%cZuE6djrhR+YOmE-+V|9jYbkLQ(7H5|24NzONSB-Cp
zy`9Dsd!?}0y<eu7?z{Y4VZV28|K9bQZ|(CxH}5SD*0afq__lXOO3)rXO>LX>e|Gzl
zZ6EGYi$1caquj^7f2Hg`xv7tK>EB(rS3I<5@A_7@y)XG5?|rn=&|#bX+r6iZ7wuj6
zbmhM5yVve}z5dKTiKsR9oN>Z?JM&Li|IX>R%b2*>enCg^zPydrd;fP&-5;6Wuvgps
z;{KCWQ}<*`3G92!eEq;h%Vc|z<IOfJC!eyt^Ez|iLx*Df71Ouvz4c|+-V5_$_VPx5
z*+233QoFWaT>El%Pqf!m?XvAX{dG^*2`#%>-uw67Eqt<fiTKvN+k2w+HMiI~$W$NL
zn=0OV;3>QGzBNvQ`$~ll+8<iBWnbAU&VA=s_3!6mI<jwurMPX}dYQdK+7tFJC`{QM
zJ+pQn@1nXry))+RD^Z+i&yeL~!>MDlr)IgQtwfpR{>5MJ*;qKr?YkYyxKI2qx9z0$
zL3`y}jd!T;erR(;aktHsBSQOPt#<B9+MTj5Z11nVQ)8s}Upa1R8=LWQ&p$R*hqBmL
zdl%(d?{zj?VgK&l9vg}EZ}+CWk=*ZY&$@5#?)C%MwsYD?eXQB{>YLwzz%7#d{;n*u
z&(Sm5KUrC6-&_OEy}O>e+CE+(bfA2b;J*Klzwh1A9%<uoz-+ILP3OMo&(`}&51+B$
zQE_jtXw}#K(^J>)tth*{-(mV5oAyq{eczfESbH?7+iy~9-Q(|<yI0{r!`?mny!Krb
zo3wY{<DR|89>2Bup_I8dv%P4qN$!ljD}v|mo$JPZV7lwkz1NR4@6jsawYQovXK(m9
z#r<{rPwnk9d$ZT~`Rx6l1bA$>oRhMdaw*(SL1g-YYi>gOq+^fmT{?HEZE%aao$xQF
zy&9YR_w1g*vA2eO`kugy3ww_4{j+z;B?nu6l|y^yY?Rn@Ywv06k{_ab7kN+Ee{<ib
zy-7I__s*7bcc_f4+iknPdiR5+-uwRZ+3bC!WOkr^%agsT1<!4s)I8q1^X8eo+_O3N
zt(RE1*W-fz&KY&qd#6ao9Qga^%O1|32KzQXxWCurY1rQNtg`#oXf3xs+_HCX!0}^y
zj(vQv=eNg9yU$ZM?(5WewwK%I(H^^P4STcS9N25Qo74J(%6A(D4ZeME6Lt2Ooe|zQ
zsU&h=#k7BWZilYjdtK$v&R5D3`wy9N?stA#zHiE1$$jTM1NIb6S396H{m-tB1IzY5
z)$!Qpv0~=lL(6&hzMcHnM)LCJy{eJLdsEJ<@AER|-#=l)+r4Zr81|i=e_(IOVQZVE
zid_39Z#Lh@eo1~``nTe}v0HrXgI5UHdR<(($EIz@-dsKA{Sv8P?WN_X@9XYU-1qj@
z^}RdR^Y7zS+`o5jo7I6$iW6;Wt@-!qZrZ(f#nfecYc$RG&2&Du&&9QTPn(jdjn>oC
zdrxdrwfUP7U?<wRd+%{|@qMEAIQGUTFz!1&(bML@7u|i1O&|BPDO}haD!*XwtPG}o
z5xZaQiC$f>S3ljsx;D;y|D=rp`<!mD+a4_Kx1aZoeP8C>U3>jjuG{}nfn#5QM$^8f
zR~hWyWINcaE3nz@WZPmp`B~w<>+CwV2{tqKeA~Hk?}y9F?QASK_idkg&BobeiQP17
zp?wYy;`i+nbaPP46W%vTmvi5&8G3uNHZ8YP+_J@P(&F2DtCeo=-M>M6p9h=dp1iGw
zd-W&Z+iSyfY~R0Tp}pGY6ZW0mBfU>j;I8ctv70t)((c*56ytHY#Ibnq7r`leI(E!*
zh|ivE?RjmY&C{BR`%>QW?mNpbxNquQ_C1CrRr_<im)rX+y0JgawPlZ$+g<y<mgv2b
z8lARUNy__GR_HpMt9-Oas=aUToZUC~YRQK04PvR;Yje_PZ+Vpbo;e*$_HKPHxX&S{
zdjAcVmc8r_DSK)!OtCREJHOZWblJWmr3(9!ZF}~9n!~eC{{8pehMY3{QdS+?yF6j_
zzFKL)eLENM+F$*~xX*pjPTL79@9mv$ma>1k)hYYsyv=*oEHpdNJoU$3sR?5C!Vc>!
zL&P-fz3yJ!%V)fJpJq*+J?kch{a;Qh?0X-TZ{Iw5zWwU6bN2545NqGEyw7%qpvXQ+
zTV{u^B1*eo-Im+8ufAsQUZE3vID9krMp;_!yZB-I-b23`_if^nIIueL|6Zq6C-<Cp
zS3XcO%VzKDjcaUNz8u`oox!-T{;ZM1mrE!1EpKDrC)d`r_Z@4)9*YOZ_HkYA-RC_o
zf8Q%c!F{P)-1q%mJ!S9ONk{BoybRmRF*|)<{+h)79T%7GJv%$W{-aN!ePxv1{^hT}
z+wpWR-aGMN`(E3$MZ1&B8TTn#Y}~0)dT{TB+?l)MThwh88NK%0TE?+gMonN(tEAb!
zzx)UHg~=b^TQWh)KEkhf?}F2eHapL8?^_pPV1L)W&!$C=ZC`c3gS{0mf9+|l>e#Ow
zBe`#~#)5r!z0~&o;hedb;d=Kz^FOlt?yqOsyDhV3@9(^GHZ!ki?AbI~Xs?T)>fU9$
zx%bI9{Mp-;F@2wW<<Y%QFZJ&a6y@7@Vp8?q!y<YPJM(VaHPvzK3(ncM<G_k*_9ej^
zY<Bs}?A<JPY%fz<%HC3Wb=z77AN%9imhFj{DztxnrO>`fEpPW`NT0S&QM$h8_PN)4
zCyUM5_mu1MUSE#+2Nq9WwrBe1WSjf_%=^|okJ+nI++iQI-h9uFV;A-Y{gK@Fz-XqO
zo>%o=7Tq6v4jt36eIT}SZ$js-z3VScv#B=Nx_3QS%)Uo07xqhdMcSWNIch&IY5LxH
z;ir4T*PPuOIw{eP<?!`AUiWzRmAC)hdu92{y<N7l`x^U1_X=*~-TO)A+McsqI{P!@
zFYoodlekaGP<o$!(C&TRFaFs4Xq;zv#V^1%_F3^>rE`z=wpOv)9?>|sw=(_E-iXa!
zd#!B0?TOaX-@nRAXy5$h()%8Lo?+v*Lwuj;$*+6gXV}_qs93(A{rUdAlRri6HJn(w
z@1tedJ{Pg|`&D0E-!Jo=cVFKpCEE}o5qm?YXZyrkJNK!d{<P<ax9)zIBiHuCte9tK
zvesm;&e``i{zs<Tb6bYly<L9K=D3XDUUpaSeUf*T?Io(d@8zrJ*mrHR@LrLssJ+hj
z=GyMQpki&==;u)KM1SukVdj0hLb?v-2Q&7)$(P@k_vp>uBd59ciGSC%Z%f&>H{PUa
z-_ii9y|ra$_m<eO9I$?+vM>Cb!#?9BN%ndx8f>FjChh&<`fyKnbLU>>c};t7{W@cJ
z`*Hf-heaLx4u6l{fBV3_y-v%y_kDQBwePE9uB~Xd_`X}~?QDgcSobYoD`(5oU%R*K
z6~{iywedDqnV<I_7n!qny2K;9{kK%?HZGiGZS%c;|G|qsd#^uQx@YfgJ6jhwHQU<G
z<h|Rk$n3o+F22|4$5fk!4Y{^&ZnxMnURbwBCF}7X@w#2s>T4g{{&{)cwlj43-f0)Q
zZRPm;Z4Hkt-uuOm(N^5#imj`_e%l?XYi(cka_^NcG2Y8RQ*7_yP3|_Q|NpSzKlfuV
z<IY94K~gtt<##OEduE!*-Ugm*t2Y<CZA|C<vb9@zW3Q_%`(CH1U#vTLd90T@P21c4
znZb6_0l~d%kIvj%*ZX;ImiE!TXI?L~ZMi*pPcygY-YY!Uta%qo?VIuUn5|`1k8RLC
zhrQQ+_UwHdH_vviz>2+7R&m;f79O%)IAwwD`WD80s&3D1ikbv%=N)`xBYeSYZ^-(E
zwpu-FZGuIe_geobwRzjJbnp82mu=1`Fz)GZ>a*GZW7XbgN?Z4~FhtoVww&I3wtC6l
zSMld|Yk6_+o3E<6_jK9Ly~?2s`@X-vzxU6^6}F7?e(n99m~MMRBVcbIzmQGaBFTLW
zYJIy+uQTmiH{+14)4Vf#SIRuK`M?>x$C6vomUYI8y>F+s?QL`ZwD&?IpY4G!YCBx|
zvu!UJM(h!J)?~}sSHAbd6Jy(kj~#nfe15rG$NldfiPev6X1=(#x9GdAwSr)ojrq+K
zo02eoJBHS1Yqp1scExTC`=r~K?~Ojb$)+MeVDF0Je|sk!XxM8OanvS%<+;82m#5fT
zG@9?tKF(nKm%m}}*DsfC>-K2x6<FG{r+>!Ny~pNk*}W-~W8c38etVr9pYP54pRw2Z
zxcTmEla;pON_n=y{EPPrhHkU*=-R$FZt-55Z!Gz?tNt$8yQq(AUnIMmO^*E9U0nOl
z?(NX)wVm3eut&t|{oamC?6$sB8TN_nFtnZI?P&X~e3tFH249<1`<3i=v(4E1q4B=0
z#E~_&O#7GYRmtGB^_Tdw`%C()y^G6l*z!jx*s?tl+51SPe~+TQ;9dreXV#Y#_uJmd
zKD;-$DQdUHB;|dRz8=_f%00qXTzm7L58I#YaVb@?d8qJrZ{PRRwvVRx+WvC9WBY&h
ze_O+49CmB%=I*(%CuGl@`G5D8?Je0Wu_e&<#r++7uj-w$j+eh@6D0a%kHjPk+e>^)
zZ9jkHw$^#M&^AGnW#6yEUv1~cblEcQooQo`HgWH*<x2a^`bGBM*>!V|(8k%j-CoJr
z{tf=IXU8iCoAV;GZ6&_%wEY%v$ENFS{VqR!SDTcHHnwlhb?+@`USezcBw_E<G=96E
zy%~F>@09QT<UVb8;YzlBYnWYatra%evgw-IUesT@x9JFvo#Dr(y#+45*4C%x_jad6
z+1~2fvzO({%)Ok<-)!WjZnb5%d9-(S$d$eR3=y^^C4zece*D?1n%S`T0!Q**`LK<9
zyP|&YW$5hMTi&^8&#N3Z+i5@K>`osRwEZLHX=^=m+YX61)x83_Pi-VR8}|MaXWX~!
z*|fb)Uq0Gi+h(<wRl#DHOPHmtM1#b>+V-n^gSqYZ1)AzwU;KM@?>e^bz4uOi*dw9s
zWV5Vxf~}|qpWQ`k_q|<h5_UXhANM|aqO|X#2gAPS*WzstNSxSP<sWWqvr1+kXZ`uT
zEDQGSd1jz(Ta~6~J0)_fEeHPr+h2yS_QnRjvPn4lbGP@4<u*~TFWQJqkl$w)uyJqN
z9R}MQo{4+%-nZ>q>KeB9tnRkG1(PNBJbQd{@A6<FJ0;WgwtUL@dn@ke+V1!=Yfsqx
zgL@3ueBQfTM!<HK<kGzi^)GiFRGeZX=i9mWf$=8W8D^hsmz&<S*_`&x=EennYkq4*
zTOY}jdkwP^Y&Yfd?tAy3d2hsZv3+yZChQgHdSm_A$=kNJS<5z+Ief42zB9Jhx%TaT
zoZh>aLF&WK8XK9tA10sMyJ4p89@BR``x+h|-P6*hWILzh{$6vzTYHmUAKiO!3Y*=P
z6La^TIhD6p?nu&J-3OCxe5(%F<a?R!{U9)DPj>U8-O`)2Y<KCh*k09Q+NZr|kL~_n
z;`^rTt+lax&9?8mC4+58+7a6;qNi;yJ~*~_?q&IXlf->&WwOd`7i+$;-sr7k>#E#k
zlfucq&;I+Iy|0fN?%h?`x7W`4=H6{P_w3#Jq}Rsj)3v?kzWRIJm=5oG^>u<xNa06Y
z%iMLg8qV&v!Lu#)))`FNdw<LBJ+61Z?C$wAchA!03v9R_2kwy)*4(?DOT#vOlD_T5
z++^F%$6NM(@Qky0t|DpoI&%Nsf72N3_GHi8>mkr*yF$Ff=J`?Ey-A$%`@~oRY*~5O
z_P&?fw|83jt=&&7_S@dC`)Awh$-6I;Uv}U7g(`a&bXo6>;O5w~UF`Ut2t`?21L3#U
z7PZH0USFDQJI%7+ma9tC&U=}u&6@pDwqny>?)fYyuy1W3ukA_U{=JvfPumDraP8yK
zc)oXA%?n!xuY-Hmr2MfBDtov0x6kpt>e&bP8f2c{bAj!wEz{KtdlQb??vBxAw^f=^
zwD(9f-@d2uJ8V~MIJZ|={*!Ga&mY^y16yo+rG)qOao)Fe@wDBWoteLPnak<jj9-u0
zdbV%etET>c?}^l)y_(OwZDlPM?2Te-*xNVb>mE}<-@OvS-)-N|Wwo8wKif8W`M$jg
zzt#3`l4Y>tS<A4OC6{5}ig>lXPj-FWBiYEYZ%%B(-s>!Dtyk~Zx>x>R&)$2CXRKRw
zui0K(9&E$DQE+dfhUMPNoGbT!6FF;}y!4@U58o17wSGz46^1Q)XME1Jt-Swv&&JN5
zdw(2RvUiU2L0jXFnR^r1bZp-lt+GA!T6o{Djf(pwEt#?R#<k?V9Yxl*+Qxi)Z`9x3
zJLy5c?T5L0trIt~+8FAtwqg)pwP*fm@qJs4HXg{BJl!UD<rZ6}KUeHrx!Ly~Q}(c3
z<-EzR{?+4MbunA^zsqwyaA%9@ez9-m4!3q)-78ip=I~1Lg#FF9DBFhm`Fn1KPu%-i
zV($KIQ;~fuuUGH8F)haSiF4lGm#3BX&(CYvyMNl>1LqVU?p4_?u;2calzrtXUHcq|
z#(ghDBy5|OEw(OGoqr%X<ep9OKZ$*=-?;bLuFTqJen8Vky->z>Z6EVK8N~)$8J3y0
zvL2`QwzS^eGo!uJ?(Eigd)lAI?fn!iw*U0%w!K-qckiECnq}KQ<Dt!w)2t3=J|_++
zEot5p`m}VvSa`zzY?E5sTko~^&Ga$Yx2fKApGNm2Yl*FCHX;|d?J-)PV&gfJ#a24;
z_g?WV&waNK+}su7t!3-)QNORC@`FuIwYrT(?7=<1Jp^qVr`+8C?C8Qh7j$>qJa=s0
zw})@zo~j3H?77nS?v)gfv~90?zPtH!;2!-ihxUHnV6-ppfaKme-?;ZhgcUhhiip|%
zx^dHf<Bij{8w<YdU9&HA?~HdR_BOx$zt{B4HyfAUwYEm_CvESEpW7??Qg5&9l%73u
zN6+omj5@k^&Y@j4E*Vqye%SeHZ_ciHd-xyg*&do!xUZghvdv4TD|`2g&ak=0I`hCI
zAMbrm8k~0a0(<xNK2zRnC&TH`rOUbZ*R+-P3cr*09@rzkFW`olgWm31+u1i??$r!E
zWqD$`#a>knt9@a!*Y6Ey?6t{xa%Jyw51D<dXJze~kaTPBn@Wzo?pvkz@zg6G_&Is$
z-qqdL_M7tDv63}Ey??XLvICA>Cj0#F?c00RIA;&fJ{G%T-LrcS%$acD^F#A}GDVDg
z_UTmYofRj#Z{b}1eeb<y+xv1ov~|ld-dnP*X`c;?=-z;P-}c>!60!Zd-+%Adx|#d^
zrC0CO6BgJjz|6k)=1JjwH$r`F10EXf%~86%_t^^8y>5v+ZGBF^+`EuN-JxEfeV?;2
z)4uIHCO8PhKi%_SpUghrhHv{L!jJFS>%Z6T-o88gZvC9S@5}pB_E|ZX53~j++M6-<
zIDG3Bv3;%eYfsl9f&Kr~&)XlWlG$sMa&qt6#vGf{lihoNZg;SY+Vpl0@5V$c<%Y%f
zhxhY1I9l24S7J!oedtPx4X?$<y*UTg*jN<v*~-Z#Sy%k#-Dk9O<6fb(o_(9mXYOs6
zmDzpO_u^j9&k1|-tsd{snIvlaV5*k2?)nw`<!&18>&;WzccG|Y|H{S}d++6n*qLnR
z-uG$Eo&5#cXZM>O(X}_PjoUNt%Rjr2CB=K!Wa#V{Fq*qReYK(e+lj3EO6=tJIp}lP
zt}ANVJE7{ro}ZgU_CC!N-dDb6=H3a*`Sxwy$i9Djh^@_~u-kjDb{@8wF<EKf-(zgH
z8@%7`-LXE==DA(>-eCVNHce)`_R4ff?EBcIXS>Pn*Iw><TkMz5mf7dG>dU^4^Njlr
z7mMzH-z#!ZQa@qu4S!V!ww`%=-5dYyZ}_^x=F@`Hw(rzxZ37Oq>^r?6$l+Fjlf%x~
ziF*#4NgwEzNZluHt+h|=aqIrncM1Czt!A~mxJC0o?ndvu#*I?@J}|Uc7jmrLn<iqo
zf5P9sy=)JE?J<<ywRgpq`3FAFowc`^b;q7KP3isN4g$6pqZiriII-0}_XqDj)}lEF
zjtHFGXD;n++puecbwv#SzD*CD4!r-Ax$kI)<i5JwhxU{PPTsf9!O(Ww<|_{S>&5rI
zbm!V9FFD6P;P<q>XP>OwFJb(6|Lx@W_D1|H2duS~_OU;|vv>NLlD)c%S@w1JCfa{A
z@Y(Bdm1kd`_`$s&W90T_Xjbf9zVzeXBAXSv@3NcOztGuZYozFI@1t{OuT5+F-U$_+
zdxg0r9X4n**%<ll*?XhrqD^Px1e=q`MfbZa3+#!Sx5oCw-pak4B_=ix-}~<=y&b-{
zN62As^cjJ@O?$g-<7D>lQxX^5_wFYDzIv5pTSrDl+gQ11dqbO5Hn&|$9ae-2?B27~
z%y#C~%Do+8+wHC%{cIzBXvUs=&PDsDM)>UYcp$Rx;=2X5aaR@g?R#vvchAQL+YX1x
zd+ti89k}>ceBa9rkM>@Bd}L2^Wu?v2%)2(W{|ooWzMQi6&b+Pz*Go3o9Bw(Z_jXX_
zp1QpbdmpW;+}~O9!)BT8*}X5-dH1c;R^Pw7so%Ecw$Q#`If8p%IX<#pe93U{g8Qzv
zE1rt(Z8rPA=kJ;4yONi;?KP}@w|~{9Wwz~a<ZSnT{BLc2Z~K1NE!*}!klDX?Z|JVQ
zk||pIcdd!B<xOnZQ)t4s*MOa8uLGZveefdY{qmC(_Qma%-?w6uzJvE62D_`X683J6
z+rC%r{{!3Yo~3(x!aDctd8A`wz}2x&o5z2zWdQeHt0oP5PLn!YgW$%!DT<kUA0K_P
zcb=Y|E!)>_o6_nH`&w`Rw)yTYw2xEc<X(>rbM}2)Vz+<Qg>80o#kuzVe|6L*bn>bL
z!Abk=RxjyysIfh_SLOC{TmNE4+bOkw_qTHF->aJvz4z>ux%(BuHTNqAO6+sWld!w_
zs&U_^*YXaUu@`r$%Kx?b=g79N&v>5QvSf3IA8&Z}#!miioA|q6w{H3`n}$hd2li)l
z?A6?Lb?@9SrhB`KUG^2+c)P!hGt<WCFSi53xz4?3Yuapu?=P~S>V0}&LUa4xM``W*
zckY|N@4>Wao0V0Gd$$Rs?3;9F{$7JkZ}$hjU%NM*|G-{`wY_`Ij`i)aQBAjvownO1
z)#K2<RUg0ay&k?{@0XlId+$x*-e>S?rOo<toqKnN&aypsYPn5tRi#biyMq0WCK3DU
zQ!eif{`J-_VHvaSu5Tyz`u$h2;d!`n@2llCwijC)ZLiN<VYAI!+x9@Ar0vqjbN9w<
z`)13+_TP44QMUD0X};a5?JM^B9zSk-@P*QzwktFDa9Ydn*;%u9Z>P4L-TDI;ZJqY-
zvAt1gXcH82dGGec-Fw+e{r1LKO4-ILEZX}<eu8zahJx)}oiv+SAx3*Mx*zQ6J}kZO
z($<wb9~b}Ld&c3{UY*JvdkQ={_a0%pV<Y-MY;Ve8?!6UDzw9vxaoM}m|LI<Ve*rc(
zG%wlaUR<_!#$4mQGM~2XoiF#+);DdpZDPWSJ&F^A_U_3(VkMc>ZX3z8VDCPo>o%QM
z>~<36N;W?}oUv&V(XmZe`C;qPRJk|cq57U5T5tD;E#GM^YR<KH=DekQ(wX`8742`>
zvrqHw-ub6@?cNa<Y@1Yhdk-rI*S<xId-ks958CVCG|^^{c+cJm{~Pzl$En#&ow34(
zQ!#w+2bK+cjqP0aY_u-5y`5mTm*uh5zRHzL_pZ8sVvoU!1G{%8?%(@Xcha6l|D(3;
zkHz*eGTqx-nc=nfNF>WXk9U`Bj_ta;XS!CqEyG*gU2HG7?JOoq>`5rB+56q&=iYsr
zx%Wj0%-LI^ENd%%U1jgz?Hl%b>|ePjbNVTp=lPtr3w*t-)r?=+q`ob(4VxKk>uYLa
zUAirGPi*_#Jvu78taG)l*sdrIw0+b5%jSu2>~5B39@~P`zxGVLZD1>OLfiI4Qs~~d
zM~wIJhwre~iCt*>`$);&=6JQeCo35DRm>6I7k+->Ufqikwk^G0ws-%x+kS{xX!F1)
z*Y@ch)4ivvmhF`(vA3P@lzrdRg%9_bteau0C%@SCytTw$Y3(4J83*O}h{TBO3%R;=
zuj(&58@`?udu-cT_pRP?!1h4U_PsW`nzq~0H|%{U=Ck+5{);wm&MNMc=$*Xx*?;MM
zeh=2~X?VnK`~OFu)y3Mwdn=Zu+5GPE+-v$%ci#-26MNrZn{OR!`pqV3lgysE73=p-
zik9E|<bR*-$?XsKUe4LJ*InR?jk)WpJ(j9-Y>mJDvXx*LwELkPY+JljaPQTY&3lEn
zAKWXnC}FRkk<`AOmCSoz6n5GyX4$bvUwxi+?okKZoWct`+g2a9&DwRw#;;<LZC&EB
zy`Mg*+9n^|wdXeXyuD24W%hbbnYdTdSZJSS-1fcioloo)i`;2zmZ@a>w?lLv+e{VP
zy>VK5JxX-;h+LPjjd$L>*P!|Ep5ND#ZQsPS?)KW4vq#6}zs;eRDSM2cKC`t<Y}#vY
ze8R?JuG3z7GeNtCLj|@CrTVtg^JHufJmufpY8Y<&RfBh*)xC)A@eNb<?&LnOcVWbj
zy*Diw_WfJYVjJAdzK_vI)wVH0abNP1ExW@d3-(^Ndu==aoutj9nHhU^6*_I#=Ih(|
z2J!B@zF2VYtIKSAcYK+%*O6!Q-udGH_bh#}VXtZZD%;A+mo{(L-n2c@b$kz}kJP@;
z9^AH(ea*HxjGESdw>|bIX0Nb$@il4h@7e=<-Pd*5UeW)w*Yl^Ib&dLIt91Q8))VYM
z?CrjB-p0d+(@yo$kG+boFWXvQYTJ9_$h<vE7BAe>$Z&b@+jru7Uw)I==Y3Y&+V@bp
z?eq;3ZM*NP?S1*%daum+|9kSxUHAU^#<ou>Xr1jXW}$s*EE;wamk!%3HCS&myUJtl
z=Ze=hNyYv)H&U%^r<|#>b*NplN9MiYUR!DLy&u;;-V@o;w730E>fX17TlQ?16WzP{
z`u{!Z3kvrt&TF=w<afu0Zx6%152h`)|0c8A)_F<on=aU4`|0K~n@cLE_b!b;u=ksx
z?cR?!C++QKUTR}je8|>SjdNc^5`#_j<Eyp-l85)s^<TcXSNr+iFG7yCI~TF+v#a3P
ztA3u-X2Lpu+mMt^Heaimta<aIZ6}@nxaZBEcY6%ZI`6fQ^xi8fJ8y5lS(lBB#oImd
zm-_e2nqy_F@$>E8XvqhAZ8aEnFL(N2`>1c??t)p3dz&jo_6D7^vn{!_Z|~BrlWY^c
z9Btk*ciYy_T)JmYey{ENy8U}z?~vake{QlZyEEV3Xqz>*S{FI?9rjsm?V-Z6Q^NYC
zjrhJddv}xz?z?aE*5>Bw9Gif!^)}Zs+HGSB5^S%kY3=>R&S={pb8T;&FZ-TXIbpUh
z+N1X_4d>W*rJ>cf(<f`MS=FDtxkk-<x9D!OnH$NwHzjt$-mTIy`_?lx?3wxT_ihWT
z4|^hR9JP({Ub2^Gn&`g2y@LB1d!E~}Oa0n=dBVHB^UihceRQVYMs%syUR%F8dzJsK
z-_xSCdJl_WzU_;n=l1p$FWTEDxN@&)o9^Dqx?eW`uFTzAv8>dVx1Md^<T)(+tYka)
zdSCmwx4Y!Ro=6AbeY+>S?e%$Out&~IWuMHCXM5E)&)X|7VeOue88h~tI>fqnQ%3Wi
z2X9wc%O2+0D{^Xyjm^zM+d2NLZ8O&%-y>Gfx$ixz@jeyjhP{8MaP2#^QPoyqN`dXn
zH;Zlad|CEo*FV~8u+(v{otufZzV4B|`|oks>`^S(%k^s2ZifHWdlDK<_pXbOwbfLU
z+<S40-JWleDz?|BGug>52(^`y*>3y(!WA2Ky&HQcooBamQ0cZkVXI|Zx_|%Pj8kE@
ze!i?$QcF7bB!50@v!Q&}-lyw6SvSAa+PkmZe771`ukF<2jk`bOO}06+_~ssi2Q&6&
zZ-2aZfxDoM{jrs{w@gp&m0|t0mpyaJo(GC0dmr9o+1K*-^Io?_Pxn4Py=JfOTa`T;
zl5h5C2QAoppe}6hiRuG3<>F%d;$y#AyZ4CfwOh|-d(p9TZ{<x!TMjX%eJkD=?(s|z
zv735o^PU&Wi)|TnLF-vr_D`%;vQ3D1wx{U|tKA_HjlBw|uiMqtx$eEebHF<Lnw4FP
z@aDZ=H|pDYtmd}OVQbnG%CyUR-Z>^a+mwfPPxE_i=ZQ(|m6)bzt8m+AcaPuNy~p0>
z?Ct*CxKB;nY>!iA?OukHPj>z|s9>ALU$J+O!ZzEjx109H|D3RQn#}S&4?DwcnEvtX
zlYM#K#%bNJy(jNJwZ38Fv)4@Of^Ea39^0*4C-(lh-(s_3+6_B>wnm#^^9_4rrZ2J4
zs5)yadHU$SO(AYJ3j3J$?XPUJS@q?gZ3kb*UYV1;dj#fgvNLr3wP(lo<9i#rQ+J(M
zv)1;^;nlV>N;CHEIo)Ji6Lezl41sQ2RSC|0yM3#6Z~c0G@5Rlvdl$WZymx`Wu<iDs
z3wF(Q9Cl3WH`~q+@!YqA<ImooN?-R*$=YeNTS{W@`mU~hH&^f9%RS?p&3`VVy{ERn
zwC>u@ZWmyyvuEAER@+JZL3<oFMcZ9Crm=Uy(iM9*gfQ&g(P6ZU)tGbN;~!u5MvDpW
zjVoSZtG0KIjZDEmTaLtb+jA9Sd&M@W?hCSLvz@eI!`_eQ-`glmiL|{Gz1Wr^hTU?0
zuA}XgJ4}0@iaG98xy)hHaEE1&N6}8(JbB5zI)_>JzM8$t)~kVG@6NXpHfzdG*m_kz
zw`nl5-=*-iaktvthx^_o3-0aXTea6FCeQZfbI-juVrTA=JSx8L@28D6OHJMO>M^g`
zJJ0au-YfD;ZQd@Ouvfr9X^%tbDx3aejeB|2s%;BwPwd(K=b-KJlRxZkMenrDdL(H5
z;EJf7@tl);->^m6?y~c+`!bKwj_>cry;nNK_m<6J+r#mG?w<a?>~@uBIP4}}dTwhj
z`PFu3zx>`Kktud^``z~D|H`)A@KM?J>6_1c{~va@EzRY#$z}a!v&dw_-iOn9?A9nW
z?nx<MXj4CR{$8(}du-=i$+hJbH?oyJ)o3fTa<bjo15Ep}%%9mDI`rQ*qs7Yl$%9FI
z<eJvl>|+eGbr#UFvv5DKC(-e!UAroyJ;&4BJ*yn1?6t{{*!#t1>E4Lb9J{u*i0%J*
zD0FY0(W<>gyOsAw3(dA=*xtUk^ZaF7N2dt8X~p7uOYeT(TUKYfSMHL~o&_C!)(!9X
z+xUI`wf7T)g3Y?5p1prAf8JyN=!$Jo^eo#F|FgEr7v}A~|0K*N;6;=5CY?2VS<b%L
z=P+Ag?@B&dyKTQBZLi!pyVv-|Y@2mEP4+N3e6VZZHs2<)u-ooH+<Y6iy5DvN<??%$
zWZKwn5_r1j&5?_CqTlb^B=M}W*%q<F_I-BG-YKDztwl~h+hbv6WILm0x9z*qr+eoV
z>+S87lH6xl{&_EZ(HR@Hkj=Ki(X(xLKGWFKt9rvWS$z6lo2xOlR~M|Y%~P6e&0u$M
z=i<4qt<{g8v}?MhYWp_$vvu>=jeE1^tgzK=+_Z1O=~=rCwHfUV47hBLo$s{WcJTS$
zti$td12=HnJxJbb(=dIr-PRedcKZUQ_i^cn+WozqWV5$I$yVzW(>~{79=ls>{_TF|
zzt%1y>Z*;DTI61a>MMI{gir1HcB|6%P8+*zSl+R{(@#9G{iFEHHc=vH&+Rbdy~ezn
z`!9T)vN!epwLNckY~JU<>$H3JK|4Evi5K?l+aqWvk?m)j^~H0qT=c4aEzR9~BeETA
zH<)YMZYYtmeOTVR=hKD;yQ2k{Y&n=)Z5Ny<+M`ukZhIp#*QTyoey`hxMBDcarF$Bl
zP21z<Dzi8E;oiOf-Ie#P@%UpqUF+iBqG@h>SbsO}{ob3q_eoRdo?9n#ZJcL3+xtRv
z=U%NVMYbI?1@?M4{<6_nvEQaeztP5e={h?fTlsxliA?)=c1!Hl|DtF2K!I^z(emcK
z<%*fM4o7os&+orxyX5UY+fNpoY!$Sv_OPuJ-?u|9#<pYrLR;6Iq&@j!ANFKzw%PMz
z>xMlk!VY^K?jGB_>L%O%ychdybWFT$w=7;}XZ0=7&iHHGUgkwzdry7|w$<{uV5=z}
zvu9yStL+?RCA+i=HoL{|diILUePJ^#^YNZDxwEa4s}uKj*k|wkoAzVx2cM3;IkS}a
zuDZF(mM>_@z74FW_xxjT**o#Q=iUt$n{9XI&$C-MYx-VB`<=UWLO0lQ^<1%i*jBgq
z(fnU~D~_hxDsPUm)q0=1H&$ZDUfJT;)(0#f?0L}9ZTE_iWACxR)O}|{SK6%pyk_r}
zWZ8WlllJY2F*&n$-I|FuyY2P%ZtAGEZoDLJEvI(NwtIrY-qX7`+a!MF+uIUyYA?HM
zgYCb2llOK7G1_mvCbie^<*dDr_I22LZCkbH^XWOZYYscxo;H7Hvys=(_SnN8d*>|f
z+Uv>l-}dB|)q7*_DeX(B)U}m&o3~f%kEq=Q`;B`OrIze7^?Pf3=P8G618?RYjet`&
zZS%!!@+uD4eVqT)cDts~KEdY)?K-Cl?fcHjU>~qs*{-s&Y|pLt=l4!NzR~W%-hP`8
zmKXNs$n3DYA=YNISK^xucNOREhR0d9x91D*6Pnp*J1MJq@4~~scmKV~xlfc&z&7X6
zpFNFUntPv%9p1~wdc>B|bf)c|m1}JGK0CKJX5QC5&hh8=vcK-zXLw81R^NWHUHQUG
zdz+(<*i2l<yEiGa-1d3;O50^WoNc4~?QD1^8QS)w$k<+ayVANvI?J{~)zr4bNZa<p
zEG^smpEqoE*CyFiRjcg%H@$1`6>VADicObn{-0dC_YV8iy?>(q?x|8)Y^#&Ww5K)g
zrESmx9$P_)w7rjhO}EXFHL?B3eAmXWvt;jr(Aqr<4?VW+zCPd9v$<w(%=3-bp&ftr
zo{!kDcX#1N+q{^j-4cr`ZNC^Z+exhVw%Mt0WUp(A!Cpt{t$Q=t-rKZnbGPl8Fvn&#
zr@77S(#GBOj`DV2CO@!#Ice)&2Z5K?n`Umc?fU6qduFfhK9ygowz1(Bdp)aq_pW-j
zeDCyC_ibj>McU3%I=pw2biYmN@r^e8`xfkV-}GVcW&Znno~;S6{gtt8Z+UIZ-rotq
zduuM7-)qfyYi~pGKbxbO=l42WmF>CGskSGVzsY8f>^j?|2P}J=8J_P=e3`me;z`|J
zuC;Nt=fY>&US&J9*YV9O+l`LSdk@^u+S{z!Z`*P9z@84_iMDDpRQA4c(6NoHSYmr_
zRmfg5$!)d^{*~>S%Hd?IHo0bRO{%=@H-RSGhGh%)Z0en0)BLJ)uS1{Y?hKcYHm{x?
z+I@MS<h}==C)s-4;;}7BpSHIoq1jfk=JDR%nXGo@2?uSD__NwtoGjbTaiP~Xaq;!N
zS=adYG6%QWP6@8G{TFt0@2u&3wnuKU@0)f+a^H`R{JjnfzuE-Pn6i6&jgT$J65&1j
znzrrzT07r%)p3r!TYA{`RetBOiz`>!>vKh7Z@gf|-ruS{dsi%JvAt)?XA`wnd|x!v
zjJ@-cHtk(^ey6SO)2Vx9!Wr$@FNN7O8co?7Hoem}#wcu$z^sEdN^YWd>|7^o<2Svr
zEeICdx8RGGt$Agq?d7Q}ZD*hTx3@=c+1}gQ{d-H8B<xb!nf7sZ_U!GdR<!Zx{bb`F
z8NAnPX_qZeUGyG?0EeA2LVC7ax9i&O@jGQZYf6>PEN1CFs%t0jRk4z@ZLfH0Z4}J8
zr={`5-h!;Y-K#UA_HN&>Y_Dmpg>Ab^{@!J)SL_LW|9y|3(cV3cUU7RHj&|-f@qS@b
z*mZyRraP*(+Ck#Br9W@(NLs05!}6BF_Rhh(dlIfq+;ipSn!N!c9=4M$ChYw?mEG=*
z_T4=X(vDkcZ+T}s;mA(gZ?B&0EtP(6+vXOy_W+BjZRAfmyX!wE*ly$P-^*e%druqB
z0b3*f4%>UGEOrNWz1hX}aL?`^PptOT*)HGf%W%Tx;l`f5GpzsFSg3dH+1eywcY(cW
zZ$+bqO}+%1?T>Zr`%d+7?Te6O-<!UA`(CA)>ul#;*<yR{NYCE84|ea-@DaCN6=ky5
zq-4MCq=O=RwK^BuGQR%4*MR4%jcMrgy&mr`+We}O-B;StviI8g54QCz4{a|#k+Rz>
z$g!{d)f(G?7TrB6;dQqDKJ#q(UM{!ox!AmSTV2ZDg5=42+BS9D&gYr8Hz7oD&x}o(
zdu{K^*)rVNXBn~Q?%tG)DSHzPC+v<1h_LN(DY4N%{@-Sg?5VvIA06J)^h0aUy1Q%l
zX7`xdZdx6?=YArSUFyZTwpQ=@_ST8bvsL|ddhh433w!svP22mG^^k3>YU*Ck<F{;m
zwjbN8S$D|Bs7Yb3Tsg<S(>~{I9&4YpRrq~=chnuZeR;l1ZSVR%+Uu}xyUi`#lY7tg
zm+YDO=C#fKLIInwu7$fbH%zroVfbxp;J<LM^ooVHHS>?|^;TBd+xxc1rr_X&y$|Q<
z@13;0-S$A9n{^3uysgGIR=anx-22Wp+_Ft+R<>Q4`_*>dYbm=8SEua#vrTUAjt#PV
zZz%1u*;cZ6uWQEsy^op??bY1+Vek8L{yi@&GWOhBb8PRt|2ca-E`HwQwf3GZYuInw
z*;A+5==4mmWte@=D%~<|Z_$yZw(l2)?Y&m}*EU`Aw{75ei@mxVC+yMIS+*yMaq`}(
zZXcTqrFZtuS@PM|L;HbkztpC^0;xi_*SaR`H9eTWx9s$e-A1?A_I!9SZO@iiBip=&
zn!VBG-}Wq+9JqV#Ue$fKXBgS``0(5IzPn<xd%gFb`3DR3zCQAJubTAYy$@Htu*q_1
zw%sXz!1mhIzP(LLmhV+)Jzz6K-e&J1uP$rR+WUL|T=lfQ%CyANJ?PY)MeBR_R$dp_
zd+~X%&6Ifx`_fLf+eSnd*`#)^*<)~TimiiRpS48qojog-{M)0qtJwCAoUN_HPZO&{
zHCy)h-TP-F7B06}D&*bXv~_KJA01cO^Z%~A?LLKVHnW!N*nWT4wwGu1X{#5-*Y+$o
z)v?<=hi&hQ`6p~+?6=sg;d)~$@%pf>xZG9S=?fcoKdDNzonyDfW>MX?y~V0Nwo`XK
zwPp96ymzb8G3zCchxghX6R{KE@32+<#%Z_UxwEz7BZj>;y!ULgw@BJCMB3WEnk}$b
z%ZO*6MY7-CHi>SVIgi3^Pacf6Iq*Yd-<50LwjrGl_po1DYU6fmzio!3h^_9QZF@69
z&28Q8^zGd^-)wKn%!xLYuHJjSs^;wd&U@cxmvfZu_A6$4?k|43`>Pl4UV(|<Y#QbU
zSUp&L)B4eNmVFE?qPzL+BlgONpSAsUKYp)C>hwKx&Nl2hA=0vYU5d1=rPF(xXNx-a
zK3#6OH>pl|-}!{?wr~G2*i}|cwyB*|vbU~szU_;r2W>w3u-KlvxXVVpV&dLi&dU2r
zn``!J{0ZLE`Dvl;lf4RiX7Cu=rYv4$bISYe-ni9Y_sq69V9V+;!)8US(4LT&FZPs6
zneM$JqP_R}!6SQ)Ur4r<=-OvHt0{KR0@3t6g_;ZY=qAMPU6rwW&rXdudmb!2v-e-#
zOY3KI|J!t4owDbF%1s-FA1$^yAFkN2e3)+YOt*Qj3EOg8qw|t`HQU$OTK&`6RdxEn
z9yeF>y_Ie|Y+1q|?vcDzu(y4_%-+8(5B4-g?y|khdSs8mxyd%|_U?N>2lDS*ZIQZn
z8Rt&hx+gbnKfIe@!+F?p?-a)ad-u+Jwr9g79h+<O8TNg;!D>^pM0QX1jGeo!^E_-q
z?ECiKd%S7S=7zStA5To*8@qtP?m>5{t^CbNHb;e~?UfhT*?T^+XYa?`&bDvMmF=D%
zkh5#fUb}aPQt6(kHS28W`|R55C~?_Z?`68Jg2YT4hszp!ChgI-Q|Oyut9M>>@0=qI
zw(Hc{ZO^~twVhVAfA7VY`}SPhZ?so>%JjV#f5q-T7RRvH{=)3N1$~lxP8~4bE3<N!
zt%**U?Uvu2HtqX%?pa{I%68u+PP>oa`0V2SuG%W|x9;_uc44o&3ybYr-PzXek8Sp{
z8PwZWxZBwodi~k^@W+PT1`GQ3w0F(2<=Voz_w3?bd(SVg-0PHFyyxpS$$er6ukE$o
zve{-w%!J+juioyhFF(9@kJp^N_Lr~k&0Y|*_k-;Wn_Ej;ZKDz|?@ga3WAk9!x;;BB
z=j=_~FlF~Pjs3Ra7QD8dA4K+MykXoog>9#eMO?$)N3)OaZR2Iz_oAB3cAMaPo6b^x
zyV~&ly=fZ`+OTRLvyo<O+Oy}0|K6jXhxV!lZQWx%yTMi{(rfRV{+%|(E}Z*x7|z&U
zz9DOU?T)m~pT0ADDm0^PtAEV5zV(TH-wlQUo3*?9_fBg#Y1_jxbB`3$6PuIOi}pO0
z%-J(>Q^4LikCyMU)mgiDR=lUJq+PhJ8fT|%S@`}v35U;H<z2I~iP`A8SFtP1w$0}2
zUX9gddl@-&Yz{8s+WSA{nay>nz&%>JeEZIHv)Tr}nXp&Gu-(?S&%iDtwbiEBSi??Y
zrQjazl=ZgyS~qPUygRYG|61c-k%l~*_vVUwd6#b7yYkB;TUF+sJre93`~KF<-+OKP
z<~_1OQF{-5yJss<xX||Pnf*3qqI|ZGA};Oibp5+~0&A*GwT{Q$Ztht&%C=MX#wTmq
z6dilF_j=fby|Wgy+umap+h^nV!^WknVz0#HO}07zj_*yqXlNTQyJzoDrsI1SEk0#y
zdhGq)<Z9M^t^PK48~1J8v!QOyUe*9dYerT*+qw(&){^X8wl7<^SgAET?CmM^*~_tf
zzs;gUdbSpa_wD()`r+Q&uf=WGRaEVHa8A_Pw*S9v(xoLf3eGaNqB@iIzHpA-lVG>Y
z_P>nio>d%s_wN6=Y3~f9vo;<x1#I)4uHCyaMbP%FWSFg*$F{vmncHnYOh~kq?he?y
z_{H|UT8*=9FDz@aS=M@L?|Q%ed)fCU?tN{>Ynx!Kwb#2&!S;R2l)cwa9k3RkDPb3?
zeq-;lcYd}}>L>Q*^%(38nzDVbaqHGSMV{fd%F+D$*rt@)wg&L;wX<@y?QpZRJ?X)@
z_w%(adnWNu-npRhjP17fV*3&=a@cg+aPIql_U+yR5y?F*`k(jYzGdE9c46t>=UTCQ
zY<>jq=6JGWuc!JiTOr<>J$Jsv?0soEZLg?Uug$3@&VAWmeeEKQJZ#N-X4_tiHrd;7
zZvO7wQ(O1E_~&5jw8nhTo%D%&52SpuDN4Gvm(OzT-i`BD+tgkZ-FIxY(7pxTM{H`v
zI``ChEw))*r(nbXR>E$|lLLDfZai%L-?3_M?2egxSNJZq4PAU}ufXfXy%Ots_kOWd
zvOT$8*5=Qy&b?*N&R8>C4A`UMD`+#D(RnXd{jNO`)#7^vcHXxMitgN9)3|(Z=uAJG
zme;I%XNx`E%h++icER;8HlIAs?@ju&ZtwIzetS=Hyxb$gJ=@l&sn=$We5viDIOn}r
zlWy&~dGzt#wT-iEyjX7U?T}q<<x|eSuQFzp?TQ7f_f|>x>@huh%GUbv_q`u}EZ<WU
z`N`(A?)5z(CyQ(^z23Ih(#O;GonQYRgPaw6wHseqZ)l0M6}!LOHl*U5^`5eodkvLm
z+lVAh-P;!{z0X)*!FKnAm_6&`rELzbT)Rj2+P%H|c7^R;xk6{}jm<0fh%fQl+dlc%
zo(Vpxwm+BT?MYcFus6#7*51N^y1n5xdV3Tz{rAl1HQBpYA$#xT-NJjeZau#HdFjEu
z5fA_GnYC7J@1rL*w)>*9Y_*OE?T!58zUSAUyVgH<U9r8N_jj+a>}p%-vY&gc4=C8S
zwTSP{ZL-|!?%`tdbL-?itL8u4yY7+aUb6_1y=8flw#CzK?1{Yk)pl{Fl8t-q_dRAY
zTzk@WmfC)Cjo6$2HDm9-Fk9P(eC@rmGg$YYvwUGA>11N-blZOKuau*E)t)Qvd!fl;
zJKuAv?UMQTZJH;1wv|ymZR=V3c&|(EWSeJqL-%Y<OWm_xzh}?f?=1URd%o?Kn5eXm
z`TD~>^M8w4D@?w)NBPQf+X<)S_DbDOwKbLS-fLrBwD)1ahP^B9dfIU2D(%g*E!sPw
zxN^@DX|}yv8Kn39dS$ogQRHEp`jruTXE<=~HT}P1kBR*;>+|^^Y+a0}>=nJSz^1kQ
zwT<!3GkaAZhT57riSAW7V`#g8NzbNMKGaq)Sk;#C-kH5iUO(MCZTl`;=B*6-e!f^|
z`=q(hw)g!n+nSP~y&^Xi_p;?(w^C&~wpTn}#nv~g-IgJNZQr2_+iX&%<?Nl$`f87o
zj^<u}*ND9=q6vHNT7TZVF@L{p^I=)r2=PyQ`yMmz2{|`=56g?`wtbhl_6GkwviH=~
z5bL&@ep{vV6?+5up6t<gcwsvyrhm_KAub!6tQK3*c}#ZMGadJuJe9FqGS6qvo7m%f
zs)SAU8qesl5vY{kD?j&(t=p4MTSbN=dlmmR**LF$w^v`ZYVV8kIkuNxJK25`-DPWM
z!M<0%Y3JT8Uzgbk%$Bs3TPU-)!a;dooZg~cHtRa~99Y`Er$;s4HisqM_Da_|+k{09
zyTucXY{MeW_saAY?u}|;+PAWh!8VQW#omh(1@~o&XYZBU7QT1upDNoF7NxywV$*Ez
z+>zSrlK5^9L$H7ygUdG?=FQ%Fx%ixInK#VetCBBatIpVGyW!qun}#@1+w3>#d#nCb
zTDX0FwYNElZ{Nuy9Q%CUuCR@@7qz<k?XvBauNQ3|XwKUkx<cL7Z4R65pV@zR|0-Bx
zGx36gb*1gky~VjV_9`T1?k$mzwiVdlWBccX#oj}kdu&x7oY|}J_?zv=7XQ6Lrk1w(
z>;blqjXiC5aUZgsxx0OD0{@S_0!P#K{JSW(caFv3y}$Xz_ZnY)WIN^X)xFQ3GVhyT
zx4`z`Yz;dj1!G%*_DR<94PAR&f;(*7-ap#4PrhTXJ(K1h{@z17{MESk{$MNLyFup0
zo<GSi_8zHa-0Q?^Y3u7Iws%40+&zXuJNDK*lC(8_;IjM8^$E5w+E3ajx)kn>(_FTv
z+Bekpu9NlNKX<q9<vZTJce5MY-Z_!^wg!8D*xbFmz-G%me%qFnm-ni>Xz#N%Y_Qq2
z#ddFj#eQq{X{+~o=Pb23Q?qW5!(Jg<)jf@SFUUyk^>X>Xmtzy-zJv*<Y%7u%?J2BT
zw3p+K#6E>Gojv_2J8YID3fQ?FKVqwP?z&C%J005(2^o8CD>Cg3UAWDbdqMqP@2M)b
z3C)viHz?h+S+aGuZ9r<P?W&nhwg<hU_Wo$vVH+TFX1CDcN!E9?m~HHS>g}zyE!ew(
zQOl00q<gQU^2@zT7C72`Y}{${%|g{~yNIT(i{oQkEzaJ(+cJLGOpRS>t)qKz?>Q4|
zTkUF|ea4lCY^EfgwY_<x*mmQN`+Ftk9NDATb8hdRT`%{_Op3SVWnEz#Jl$_^&_p5I
z;_u3~hnH>KQ@8c{o;DA~y)~>0_cpJdWFy#<w|Db>RojLMntM0TJhJD(rOw@R;;-&;
zIIXg;r_##y%f(fDn;C!Y;r%SS=Szg$-baBd`($?W*j>EYwRiujA2zEd+S(qxv~yR1
z>q(oVCsgecPHFBvl5^R1_vyF0(<)EyHq&adZMJ6LXUj8nZ)Dufy=zLA+y0ziw0p%)
z@4b6=$n296=-4}9iOpVjDNE~eQ?0#SEarRTo7rtbxY_K=-bwE3<J@Jl`0RnbGNEVo
z-e`-n{nD^*ugRiWdzLsf+1|3U-dmw(XuC|BWA9lt#(l!A<$Gsmh1&d>>btk+)GpgB
zS0>vX3|V{Te3@n2e`fn$$61+s6^dutc$Ty5%bfAZ_V<~id-s@Y+nWAT+Sj(c*7ix#
z4%-5k%X|85Rc#704E8d7-nys2ZpL1-T_3IA{N1^C_6*fMs`~`@1!z`UC%k6elj69>
z#^+Gd9vPpay}LF^?%Ol7X^%xq-`@Ft)_Y|h=Iq^fRo(Vl#%dd9ezU#CyXEcV4HxhI
zq2p&8v|oPTN_JMeY)hLxPKO`b9$B(`?*T0h+dZaF_lUkx-@AXK?p_Xt$i1s3vF|Mq
zklH6_vVLz$+N8aDky3jDO3vE`3Z1v{QsS_Ucw4h)!{1wbITU%Vr)}D9E4h1~?Y-?j
zHc#64_U;ce*qfHpzjJE(nY{<ATK3*~WxeNe_OiVO5w&~QP1wEnztJXJao<j>d#MZd
zHZFO#_tG!_J>je7>@}X$Yn!2NVk2s1W#`fL!N%$3nZ4IsH|>oLxU)xPri`7-_ANFi
zS}be~)-~@HStD(mm{+uSPGErTJXy_s3@UT2EN1=L6F9rf#xBZY?;jpjyDh<6_dI$V
zYr8;c#_patMZ5KyeEY5zckEqpYx`c!j(0YWyit1(T{GP)Br)B5!O1hbSBI(YovCnt
z_j0`ndpFb^-79$ir){U^p1tS3^zT)WP2QWv+_HBj=Z!s#{g-WA+!btjW^c5qv9a1S
zd2*WdUCXU|8_RF+{V+*rpMZ+6O`Evz-nnXKwv{g@?tL&vdXHsW+ulVAGi;<TH`!Rc
z_t<OmaOPfL)!(*>?Cg8P3|nmNOBU|UWxlxgQit|lZR57R%iR@he*O5oH&!5KmtWPn
zy^IHc?!C9>gZ0tQf<4^fH}*QmKeD|NAi3|!NvXXXmiO+><}}*dYsj?ww1&ps>1Vk2
zJy$xsccqJkZL*%3?JDm@w%^@u@8P#uWz$*6wy#1h-PYNW!Hzxd#a^o|340G+Zrk0l
zQE2ZsZN9x3%f9asyDYNTc18W3q;;EYRhDz@ld!C|{pnL-8}``Grd2M?RzccfkAT~n
zy_dHh+}nM3r|s|0TlR`BN!;6DxovNnmxWzt*>juR=#)K<AK3Q&tlYGBlk;_3SB{f=
z*Y2sZ-B!-M*GepBui0<AJ<g^odo9|(*;q!n?wNT`#?~)$rEUGf3404ZFz@3_ud!aI
z^l~rT6B%2%tPgvm_cQFvxPIB@-ooI$Z@qr+P5OId?~;i7Hfw}7T6^87v$1g5v*$!i
z!yfLyYqsvUukD#H)ozo^?7H{F`Al2SBl@<pAI`C!HZ$DXq+7&B-}2Ai3~$zbl69sw
zofRgwVR<&T9IkbH|2`415o6l8H(~3F-Op6p_x_xC%0}gQ#oqa+D{P}5uiLxt=@MJF
zCJ`IM>u!6uuX5k(*5tjnRnmO7$GU4aU3E|Q^tUnXnPV|+?|jK?d#zSh?cs4)ZL6*r
zvNyf4d2h99r0w;0lWqTqKHHlmKGn9{vc;A$cmCc9@@ID|Rq*c<i3_xiiHWmmIPhn;
zMDDIVX4eh&Obg-Nw`s|5>jykXY;V50vBzlRL|a7(uH8HJW%fk|bM1M;ddT)Re~WE@
zv-{pm`Q>|Jm;T-R$nc(R?Wg#?Zykg8oG(n>v!YRF&%#4*ZKJ>Z+QZ=R(BhH7tUavE
z?R(Q^TI`v1ncMFFx)XaRsV%qV46w8fo<DQ1?D>l}Qr3O83@Wm=anZ7S4a(|l6(n=4
z&5SnNF4Qu%UC{8sM&X<N-rT>Z_g>q)(e^;xJ)7RA*|ri>{@8{{y4otAl;3O6a>?ez
z)!;oncTU+juAOc3ZCkJHiwoSg3L-wXr?xBGZISNZ8|mk6-E@oJ_R*BDdy6Nn*y|Ct
zV6Tn$eA^dCdG-}G2<?5+`eo08!&j{r1Ssya(s*K7xp2<jjKV8>*05FC&acSWoBhde
z&nNjww!cn%woOT0v1d`l8e8XGdA5(O`R%kMm)UfvCfl+ERNHEJt+tu*y>`#<4FA1t
z@1^&K{MEGm6vANp+fUf;>QRw>Hc>10MovFx`$u7xZFKV@n`5#LdoJ8swa4IspzS}#
zd3$Fr*uH1RoV~WP6ZrS7&Ro97r0lS5&8#zfr!flK{g>aqXQ_zr-V+T9c1|m2*)EUl
z-_!6r*ebdC<lZ?AM{Jk;?Xl(gl(pBb=!{MDL6LpkT0d>{Jm=WXHJG}m`ACWN6p`gN
z6}iuBrt!?R33;Dk>viY2t?im<+XY!^)<>=h+lHM#X=}4|n$3wQvprwb@7S_!pJ*Fk
z+O;=lyOynA{SDjQ2Bo&UvrYC$KfAqmS;z0)FS2Lt{ZYSaZ$qVrE$6wDdlxlu?v*^v
zU?<S|Y!6q#Ion5^Mz$?$^!EH06y2wNY5m^Z$*XLQZZg}YCNu0iVkNm(Y5~_ihNh%l
z4Mt&B2JeOJ0^VJ<{;@)I50k<<+rZEk+k`t=)@$p<_8wPxw8ypK!Cr=pG|Maj*}bnq
znCw_$A6Q%1U$+Sq+-QAaaiDF&nRr`~8QObql?U72yw|xmM{~)Z*HIyR52T*8E*EjK
z4o;W1(@ejzJ1=3b?Ui-^_uM@sXFH3JeQ#61|Gob-SoXfy!De&el%}l><H5Z-ZwhuN
z91*dLe;~fE{K^5FGw){EUfD6hmgA%K-q@T}+XYr)wkOmZZ4S*kZhQa5r@bYo7VXK8
zvDzC^6KVVV;uRZ-sII*nqGxQ)&YsxYfA#m?Q?;vX<{$gKS8>Vwy;qc5Y$e&a_hign
zXJf<FwdaNWZJV0e6Kpr!*0&X$bj#MNTyWpR?&iHNJPx)EtBv=b`V(ryQ22OHhV!RA
zFW;WrnHeZ&+Y)VIs}g!;kInyGd%d>1?hV;wzIXA-2AizHIks!APP26g61H=>9<{e(
zef%EJeZOpFPK(%0IIL?EQ=(*h?&5CSpM7&|3s$!5W&78$ckf^8y&3Z_+gz!BZF7I$
z)4f*~NZA!wXzjUj+t^mYDbl(mcZJQiqBVP!-u3R?z&~Y=#mwNnn!96cBo7POH82?N
z6@RtQ=6uv*TYnFpy&0!o+OBz9Ve3%-Y*)19?!9l9AKW`@+gj^q-uw3c?wYo@D?-*T
z=xVp^f)D(*U#_3AJ)Y9Ox8xzWP0#lId+u%H-g_^i-A3Wn!#(}2pKTs|S-3kPJIuQK
zwYXgg^Rc}ZxqEFC1aI&0o2zIyXW4U`H%wD)W2U<AVJq*k&6oJGN0UupU&ZB9d#5>V
zuywEaVk@&W#nxfpzP<8yI&3yQWwk33i`|nqRm0Y5j+mXtyarqSiy<}>)0u1qu3xa7
znX-Sch1q&r_5-JE^A4}F{Xc#GUYoz?ZP}!w_O4+0YICPz@*bJ<mbTkDpX{Bb#kkK>
zUDEEt>h*g%G_CfkH7M<y6LxU#g^u&P9bHpxC*BsYO-S&vZJ495*THJ_-r3*#cN=^^
zWy`C+aqlDLTHCov=k^+Wn!6{!w7~XO_GR0N+duBD*cxbCefas_4=!h{*TuT-^}V=i
zuh{aHdsf`tx9515^WFn7XZL0=+p;$&U&B^oOSA2H=d*inD4nzwnImi0&~b8a`683O
z#ro>Df22?E{p6u(d)6*{??O)#+g0sMwqluk_wM?2U^j=`ggx8VH0@osWd7a*NjbI!
zJ-=*ruCunSnBZb#^hD44oYe-KySqef-|YCXC(Uc~UJa{9d(K<$-DB(DYb$8E(e{_g
z{=HeE>uf!&F4!(TYr6LX=V{wFC$8+7;<jwBej2l#Ri3P!zU19K9|C*#gdA956MIW?
z-^_2)wtYSG_A-~*?{(1Wvc6v2W$TkFzgJf(Z*N90gKgjaBYSNgPO^2C^WR&&W%}-u
z(kXjQrc~{jF}v7?;d{cag=&&^7e0Tsj^I6O^X$;Yy|+yN?X3}=WXr=JVq1|ob?=T%
zO15UjS$nUTbJ{s9ytiky&M6!At@CXP1>WopdCF{S^kRzbmFPKpeq>wkEm*N@@12rh
z+XH4XwjX{q@6}JXv^DeXw7qavX|Kh0F}rPf=6hdPUERB9%alD1C&Tty&3tatmeOR)
zqPKZZRkpysN3|1eS4>;CcftEJ)~gP^-y3PNW-r6KnY%W9IAzNb*|Yb_$y0kw4$j`|
zp<ZOOK)P>_*$IJt*Y2|JopyWc-gEa@_jLr$w$<5hYCEgxmTlmr4qKBPW?O;M!oBkw
zyKNUpY3_Z#Y?jR<PgT2r>GSqJeVw{T&zWJ5OyZk8<qa=wH_Lw7+ca;L?aGNeZPRXS
zwOt^zeow-Zz`gH-{#)NMpSWkzRj$3ozijp{ue`W>+KGTYf$pEJxcEx;9;of#eg2|`
z&10>Yz0VZ(+WJkKYO4^{wD<l;Zo55Mi}ya-7qwSmKclsIWcXfgRt8(!&h0iI#82%F
zXw<i@xck+1!JKP*9Imih^KNI~_vNR6?VQ!JdmqS5+au9()W)U%{oaLVV{AE2q}kR?
zSh+{ySB7oqbDO;j<ZSmYc~G|ZT)|tr<r(w$+6J@P%}HTBVCkZ|PyDa7?Vo3Y`x?Ve
z>|N};*gm<We}9?2+y0DgE9`=G^lTWu&9`n`Yrb#Z>So*b{+W9(s-NBY^J3F}<~g_a
zv0v`kn;@}sw|(^%8?9M)_RiG%Z~Y>7hW*;@Gxz!!a_m<-=e<8u!)ia%^GEyUaQxq!
z8TfPW+PZq{_8-o+iX8v<mP}3F`{YrBt;v6`eLoLc?Bi*_wYOluh3zTPAG;3RZ?f)x
zv1jkszMB1s_Fr~ix~;xf>D!e3liJw!M65Vz|L@+0y_Nfy?7cNvaR0QI>-X<ov)p>m
zFYA3OnYH`7CjH#wwR`<uwVVn2?iDZHC#T4@cW2n%y<V4FZCjt(@7>~@wO2#^(_Y<O
zYxXXmS-UrCS)#RB<Ai+*VVn2(zCW`6--|7K4^&3%<w|O{zsh`L&ppTAdygx&?YmsI
zY40JCM*A1C6ZWK8z1aKIe)isTcjxV?bNFf#ofTv=WfR{%$9J83xh3z~ZH-;Lr)7G>
z{+eh7n_cOgd#^}d+0*97V&i|TfA8(Z8+YXv2=DfuZM|Pp#&93|*5$Te-uCRZe71hC
z{5oBmS$z$A-&s%H%lBd0UhREn_bUAUV|}*b(cXs1aeKS}eAt_P=;_{sH%bS5Oy=#C
z>U7$>F>2f1O&{C#ev&Y;bI5tSS1-wY--U;3_DW7XxhKG2v;FDkw{3P5|JdUia&~Xs
zCUKi7Qd{=sIP>lk>1wj+>G9rM^h;~s1E<q_oPX@ytK)0FS8vXqz2~HV+W7kG?7Ph5
zv469}&AqLSEA9T-c<;A~a^1I*`Q`5R>F4*}x+$^eW$JaC%b_B+uglwQF5hvoG5>yN
zZ_2e_dv9m-@2&QpyicQCXE&p`?cRr)zxKVb*uVGS{LcNZ-I05bMCjVo#7){OkbZ4X
zw28-Vp5&Z8BF;+t?rBWin|p`T`nHSM9x0B(z30SJ_SU3x@0};IWB=@_)qCgVwc8kP
z7Tn*(w0-Z24yL{L0_*o0eP!Rvb9eGSkNCWO6K-1BZrnF_|61-z`xQb>>|NN6>;kVX
z+#OSUV6T#Bm2L9kKX%DQ+O`7bEB7_bUAULoX766v{y5vrMY?;=*f8uh<d)c9#oN38
zX_5c_ttR{TR{8DOD=qqeZ-CdA-6q|Iw$ms4*}Fi~W1mHn;NJCS>ie!fHMTL(zF_kz
zZnjMr$Kk#IbRO<IV0F`4$7|{ywI`45W-VQ@_sTMkeQYy%_L`N>*;^JRXUiC{b?=6W
zoA<KozukX@@t{3Fi|RflFNOU#-el}K_K<OZl}nVJv&^o2C;7kamHznOM&rb`z3w|!
z?QM7xzvrYL?>_g|ExXjTX6)S{v2xF$j{5y8uS@OAtd!qp<ji5)QM_-j@3s)zJ$EnK
zE?HA)E70w`_t+}leM}vT_lDN}*>fu*dT)Em>%Fs!UH8|k8tzT)YT7r^<-^{20;&7L
zKCZO)T&1+H@<OlupIaPz4|BEcS3EOqZ_}BGealZzwpo?bzwc_v9{Yol_xByxSF?B9
z7v25qv|jA5DR{a!$G?2PP|);!W!sP2PW{4kK=9S{y|oKZ?)|ak`kp!M&U=4cSG4=}
zt#DuHv6j7$C+*qObI#T-fxmlCqq2o<+OY}t6J2|)H0{)F4wZH8UzZTR_xIIr`yM4a
z?|U=h^xp2u1$(C}x9pjorDQK)nzoNQQ)ZuZ^X|Q}!u9*=V^a3MpWVCvdb``+3+|$O
zdDum4SWV_xeY(A7Ka=de{gn%R_KBo^w{z9nuy@TLH`{ePnfER7+`adz=WAR0$%1<}
zuexQUY|pnZ>*M}C*LKV8ec$+Zuda2(zW0CJ?6MwBu~B5y+Rss~z2{B;-90lTifk{x
zTDkwkBZYk%@9f{3a`ENf_LA#+bR7@bZ)la>+j!c}&f{6r-i+fNHox{a>`Ud}y65}b
z(|ecQTD|wr@7a6&E<CYW@>6v0y9=EAUU#MKt*||^rz_>~{?;ARHfhe^_Gxcz+Vd;4
zY476;#``z@y=0rDerj)H>Epdz)(Lw(jkoWeGgEY5%>lE$b6GX^eRJ2@^Vp$pzw)Eg
zdkfg?_I=U3xc8=|i0x%-tG!QGe%QP5z^h$3Gnn@Oo;Yi-_f{7BC+>B7?^!eMYjA(C
z=i#r?y<g{X*>i4-w!X9Uz}^)NqPCo?W%kc4eZKeZ>(e${GFI+AHNjx-A88Yt|H~w7
zOB%M=@Lg)LJ}1(*@5+nL{SV&E-oNFR|K7~A^KB-ZOxRPdB)z}nUd-N4y6pQNGjZ?b
z+#R?tVV&9T<q^3yyTexOjnAI4$FQ+sUsPM~ULI@3eL9A=d-oJv*k91azSp`kcE7?d
zTWh~CqdjW2n|D8XY;3!x&E6(ZnPuPZl$m?8lKJ;F<}b2a*0FT&vWd5?{ZBC2>fPAC
zcjmSn+mFv=_BFZe+sA6!y;shmb?>EM>wQPnT=p%x*|w+XMBct@JyUGMHq707q3FiG
zAl1co1v|{_xWiTVN&Vp1FBS7@uTPoYeup~Vy@%#2?Xy$0-p8}Aci%i;;eAC)oHnjE
zBKO(cR<nO`q|4^aMP=LYU#feTdROn~Wop|qaqaOv@ALTWo9hnl*}kx5&#WKa`z~Kt
zxL3oGc^~Vp<GXnm&D^)Sw9YO!NNitG>8kw?19|pYZ7bV5>rvaj6(2Y6OPk2KZ~k@f
z{j(G%?_+p!-p27<*xrYellD!p$=Typ|6nh>dc&TNJEraZB5~F(BUNr6^WM9Ao7`IW
zez>-EpK9oqJ&CUk_C@6D?D=VVYwwg(oqH>9IqsbyuVVW?S7M)g-@3hvB(wI_EpOUa
z;w!f=swQ(^X?D%ty{i*!h2OukF@Jw(ufp{Wwv*D?_g3og+Pur-uzSX9vNuM(bMJ#$
z$u?CVX4#rv-?is_X^<^L3ggbK>ScQc-X7h1$6%FhP3~tKr8s`O^UHtQF5S{-!;mOq
z%cIa~qwjETujwUOJDphZeY5ARwvG71y>E@M>fXLDowkmgEVc))r|o_5;E3(gtoM8E
zlP~RAXfSJ!fb@&Ke_wF#4J}+|>z{LRFPFZ!ZOAi~y?@FM+dfpjw71K?Wbca2<+hvx
zR@VGJt9Ku<zGpj6UTAOl$+tFUZhQB#xh>escW}X8{$(e3J&NJ5<=FgZ&ves+w$kUX
z?4F=_X)nuH?!AQyllS)X@a*Q9US=zhwb1&u^NzhKTcqt;)$ZBy?&aTm)JtgJ`;(=6
zBWC2=GDdLNMHF7K`LWi2uc^kQy$<Xfcjr7mZELhkYu~FR&V3DODz<7<wD+FWdv7DP
zTxRe8&8qtvA5GcgwVT6k^^AtS`<F@XE12bN%hxQpcV3A0-opwCd;eWtX)AWva_^+?
z8+UJ)y|*{!#xI*aKkD~>xzfA0Y-a1;)*}qID!T=36@;5?u5Nd-jp$Lf74^TpcPZz`
zJ+;Rc@BQT3XS?ariM<gI=k2}OthIM_w$>i+5BK-%wzu4Sp)JmKzWT~NO)Wb6B5z9U
zZP<Oomfh-*?WF(wdxcc@*ed+8*qeXBaWC(AQQM2F{B7@lp0Vde&W$}>3y;`Zb8^}J
zdj59rHf^W9H^kT2Bz(5CW^Z`4S69H=wr-AsT}9imJp~-KHtXWM_n0cM+U}6wwb#c_
zbnmAf(zY2H{d<i9`)vwOGw*ANQ@7QVo4j|&<`Nq%4Jo_sTj6^Z<UZPL4PLg_;Kpxj
zp|Z`py?(v3J-@!+M)K{Sy~+!=TRs2$dar}-)jivljQ4WycxD@OIehPWr_8-S9v<8)
zXg+&yhDqOE&1#9g@78hbvq<OPqq4YZkLjBkdykYY*{#%ebI+>iO?zUhC)&Qevd7k{
z_pxn^&2<|?@4I_F>KpfZb^h41V}pfVYwoN)GXhTAx+JdO<E&q9Yq?e0Zax2+y>BnS
z+{0rs+4e-<3L6{yvOOtxjrL4>u++BY+`T<sJC53>TwSwQ=>BorC2wQ*I2~BAH*1NQ
z4QGPkUQLy<-7EjM?M=#=Ya3w5wfEQTb2duhJNF!jsI}!>_{`Rmb-&GrApX4!J0<po
zo!nx(Va|SQgHv~QGkW*$iBMs*xl}UMR$+IswYw|7o#1x&z4|8oyVtFfwcV>dVQ;$b
z4qJtV>9(4i?Drm1wA#zJdeYutr|o;**tgq0{J-AzXujY+zxtoMP2cD4wcyLO`J=tT
zwnKH*UMaf;d-LzU+8dmH%GSBfX0OSQ=X=-g;<9Zx=4U&%anWvG_5C)c6YY2Jo9SZ9
z<J@6;aCVRFS^1571NQv0kz`KVEBnb}@7zd6+uX>Dw(dQ#doLE}?z!uIXzxyqLwnC_
znb_nrgxLzMaM-*2t(0wjiu9fzTZQ)Vd~dehv#)=z+{HCE2f{n{7CEoldv=4>-j_R1
z*{Y;Gw#i%|zfb9GpKb4og?numo7v9$%VxVJ<oli-50CBHx%7w4xh*U9uCPzBouQSz
zC)wovUjDFOHVHZW`zCRU+jSnhY2#MWXnp7z(>{h(xw}<X>)O^NHSDoi+-O@6kz~82
zXz8AU0@b}|dZ*aRh_l+p6b0?A2>E6+i}k*(yMT<{r)VX+j@NH(zAMb$>%5C)-!F$r
zd;d1SvvD_VusO@{Y*$Oxg1s8g4%l40z`nQfmB>D$%1bu-+yeW~ZZxue^?1H*mQU~A
z5~KMx{{%nmxm&Y)uf%Nq-P>1{?OC8uwP%CV2kXt@hxbmJwr#Io=ia@{clq`-1##NV
ze>TsCd*10i3di@@-bp^WXG#d`zO%RN_DUA9?Q7Y&%a+gUvF)1v?!B8n?%x|_vTX0(
z6O;B#@nG5O{WENLPN9YEmKynecjmJ1{nPPdZ-LskJ(J^i@0Q4(v*&PH>)r>~#dq)B
z@We*y%#}Tw>x1_e$ZXo<tlVjPod1>$>or%~sm^Nqn1yxs-YC3dE1Aq=cd+^MUWQOp
zn;Wcr``Y)m@6`ztwDWk*u-CTy*B;LcpKT5-*Rq`vduESu=4RX6j=g(-Wvt&LA^yN-
zT4l5C-o^d4{Oc_D21JS29{ML}_2gv3UddG)d!6~-*fQFv?=_H9+v_lQ!QQ0T{C12_
zxNTn;9<=poUu5Hzd&#Q0L&}b^VTSFG2tC{Tf4TNPc*wVJIxGKP-4yA4Iz2mfJD4!+
z+wt!CUcR*Gy-bgrY#C=Y+wQrs)@I_g=)FnbG;MZFb+%CsHrZ?K;kftW`N_5syt8b#
zB=he3DCcJr^XiZ7oW4nW|4zGY+a9-nZ#`H4-eXUD_KH}>+Rlk^vu0Q}b$82`IeSzM
zF4=ZWSJ}7Wd6MmfLs7Qxujub(dwG0M-@#KhFDEV8Te39AR%mm&ZQK57d;RX6-<`7I
z>7K{?1@?Iu-rEykU21d2O>=MNe=*y?d!zU2ewnv-^1(a1-<ZqpEj?yv_pEfC?Vpdg
z_Q(tO+lKzWYFoaYfA8EIr)=4lzuDW7xxjWsl+E4;Yo=RU$+y}zRYdQ(xQEj&V9oE{
zJawwJ-qW`4J#x#(cG*=8+e6_D`>x(Mx0UR$x6zq-Vb6ii8r#PYzwcq#_|U3g;z?We
zb8Bq&w(Z+Hr8&=bUl;4%5-yg#aY40vm(BaN_x=Bqd;grO*?UN3u1$rOwyndIU3-lu
ziR_c)Ro-W@`LK=K#Z|Vas#ok4U*~4Cx9s@dKb`CMgk|>c<zFdgvrk)M@1IrMY_+Z!
z+1$6V-RrPq(cWe4xqG8mmD)bZtK3uQc*k1g_2Ru-6sm0NY%8q({&C!UXKUZylP`Ae
zx%7JZ-p-c2HZ2bnZOi13+HSF*yVq1$)%F<o1l#rZ8f@p-9NfEA)M@W&1Gl|;r)~GB
zMf2}l;C^WDhmW%RX8&8b*U5G2o;B+i?!7sC%iaTRR=Y#WEA|u#ZQlFgXseCe50QP2
zYea0zf}ZTzvF^s+mup__J+C)^uSM-jTNe@GJq%M;*;?ejvUzlRscpT+o;`b9HTOF9
zoZl<HVU^9ndzE`yEYtTMTfKiTqp_83^fc?eYAqY~{%^ctTREM>HqmLD^|g7E_P$+n
z&X(`sA=@gUMq7n5Q*89UU)gK#T)20$m%*M>bqn`S>ucU~)b_>RYoZLc8oPe)c6gSz
zXG`UTy`|hM_k;!(+vrw(u_^d}(1yz{boYZh^KD(^`1ekDb#u>~W9#=YHcs0+tE$P?
zX!+;8OB}kae(l({w<qbwp6Nxb`}PQW>=j_#yk|o6zTK;0*=)P^Ua?laCbjqSg4KJY
zo22$l*db+8CH`tp=feHA``;Yc+i;L)Z}?2ceRer(_P$yvy;twCsjWo+dK-lUQ*EyY
zNbFlRt<R=pZueeOe-Ycws~1^I%$seyLxtH+EmG8$>&8W!+12fPE$^$_7|+^lon<ew
zkJXIdCg8`ZJ=1Or+9}U?Xj{{<e(yn*AC{lCe%#aaX_}477DwA>+c@{lUCy#^R_V;W
zmRvG>m3eOMaf?vc*PYw9_W>{8UI{KqyB$AWY=cFY?a6Qx*gHq@if#E01>2lQd3!k&
znD%)b6W*8EI%BU{o!H)qJAT@Jc|C7$tXs`qhx2-SDoxaOO;~hzZ`{`jHvAC>chC3X
z-sjD`)OObf(S6?W3#`L_CGY(im$diQ@^d!g-`?%MX6dr`iNVgjEb3f)f0P`y4fLD7
z_somid(}Vm?hWw!wr8I}_}+h|#d~Ch!}rt|@a#L0pl;hcGimqE``WhUtUPvWCoJ6~
zW7)QMdgie`QFE5q&hG8s>$dNc?Y0}uw&q^bY+sb}?CW8!-~0EYxGhgHuicAI?tQoS
zt+82{8nD;I<mn!sklD6PF-d#f*F4#q!s=}EcZQUm$zJWf=dIT6nQ;5{-h+w~`x+*>
zSSh^<+`IF)?cTM&822TvUA{N7B5Lm|jfd7XPp{bYzvI~ZHFk?_@<F-1SB|pUM(9}W
zG4^<{$H4o?o+}fM+s6NUx3_!E<2{@FdG-Z=?b(wevcWcH3h$n4Z*6RJd*<61H$JpI
z@Oj-{uV-s*muX$xdq<vaudHpr-j5chdlW4%TOZ_=vfZ>|`JQ_-Y;2n@OW7<>N!xpW
z1OGnD!dZL!E8T5c_x#;^k=bzXx@Q44;vv&)TP7~qJ8N-}t>5y$d*3PF+ijS#$EN?z
z+Pyp$fA;vZX724U|7+V3xp(gwmtNcS15>Q+Klj-(NT1(*dwIp)=Nig;&zeQrYKSJ=
zzJJHPZ<d3Xt^d{rn=Wyly(hV5*&g+ruxIv8_I=GaZrQ%&a<}CEEWXdiYwq6MoCw=R
zGcWA9lCsM7m!ZAww42d;<~07^Gs#h5pTk}Yo41`@`;vY>*?ar6-5%4pJ+>k$+O`d!
zrtQt-TDZsGV$$A{Z|t@!o@QImN%^pM+ItzhAm7!tDRXb{UBDS+d-{U?9%17@yANq|
z?cMOkV9%~-)_vt)YxmkJp4{8g)L<LN#l7zod%#}1f0}z=A75x2_Pcp+!joUNQDp*q
zs{~v2zR+2)w^E*E=iU^aeLB)AHo^NCY|_FM_HMQF+q1hTdhhGqclWX}`R`@3n7#ME
z(IML#R~YsM?YX;0{AH1?#)F$Srx}#@?iEkpyQPHB?%mDwy^r6mv3U?DxKFcyVc)Hb
zW!52+IrixpY_XAA#JSJ-#k0K*d#kOvy!P+C^PX+rri(p$gOVF;KPhb6Q=njAoBBI=
z?;EQ`+w$-aHqOVE+teyA-IHrxu~#@ja^L&UC40ABTCwL`7OUN+&J8v}%a`n}$#}45
z0rw&63B^Tw=Z9|JJ4IB;&W}%ZALmRdyUD5y`+Uqr_g(Tlw5RjY6`QB*S$m}t|L*Pi
zBx0-d;Le^&oD=q{w(PZec&cLW^QYW<lM7k)ZcdQj`=3q5E^AHm9@9s5w%5X@?%l)i
z)^=5iz`pH~l{=if?e;91bi($|<g&dtPW0}b{NmK^l>2M;o-|r)QyF(^Z{mbkw*MN=
z+j8$!**kO7%e~KQkJ?`Tv}<pN(Z0R&xEI=}>725ap7MLI_Vc5=P5Bn@?O1KPH#9)o
zw%VL~uW(Y!9+s}jdt7!$?%jJ-(e_TtQrm=yYpmb@%iil!^?C23eJgEuJ~G%_?$EWD
zd)i!^n>V-azE(bC@05s%drnCI*t2@pcbkB1tM+)-p0siNl(6@~t5(~qvv%2tWgOg_
zQn$-Se`Ud5Tlu~{w=LQB%`D>B*C>*=SESI<wtktwzUX;dZ4aftw$|LS*mjL2|K6^M
zl-;&(TK7)kW!|^xP^?Xyf}maV^W}R584~usxz%O6bdsd)+`T?Fb5^tL%YQP#<{HyO
zn=MN0*3p7{_A>cd+y1^gbMNj&!u!6SaoO|a>;>C`McsQ}s<PTnQ1G=;e`2|Jh3nV7
zK71Q&H^;Z{*?RoPo)4Ed*@oqM*xnU0w#}Kh(pL7djjhWsNn5sU{d?Uc^!D;FAF=g3
zV7KR;`YPK<&%=AInrH4YIbvtoak9gvTR3#Du5XXciwXB^-keI_dvD5fE9nUFeJSq3
zdtE$?_FjA^x%V#n+P&O9-}VT;5w)>cv3_sMt}>f2sU>@t#7f%=zL(f1>(aINR@W5U
zgsA<yzgQ*h{bF!!cf5ztu1{9Ed)GZ#xHmSQ(f+A#>|U204K|LO((E2w6WQl9)5kX4
zXVqRG6A@d!9p7y7pKjPQQ*M`CmTZ%)fY`M?)vCdJ-S!*r{d%}<ulJRcHc5M3?7KF8
z+dI#gVK4t~CcABAMz#mI-|fk_U%OXf@4dZgzw_)GQ^WUm+%m9zmZiAY;wsx-!!5V>
z&ig22TNKW|H*)`uz1)AV?Bz&v+`HtB!d`~sMRpk?`}S5EitRmg{hJN{wv;_3;ScN<
zeg3#tApPc^H2;gX&!bM-dCb|g_qN$&+xA#?d->C+_k5i5YcKP;3ART4Lbivx#P=rj
z?zTC2<^Eo=+TDBam2&Ovz4do*^S=gji$g4X6N)+aO|hM1v+!Tz-ephi_sIzt?#-O^
zV6WAV7P~)6s{8!i*4P9bxMs8F<z5?|?Sb}MB5ZqewoBV2ObN7E{@iwNiQvQC4*u<X
z7cSej*LTWI8@*)l{hHT|_MUW|Xlo%Yu_v&QW8e3xS$kjlitbCuSK24=afbDG<=uPh
zKxg7Soot)kq`mimu+lCI=Pfq57Z&YZwYziAkzb9r4h(_2Kjw?>Yv2>HnP$?mS7jf^
zKF4e|n;(BO_MX`=(axZ?W^c5@EgSyx>ANT8S?(?TU21o8*DIUmwbFafbe`NRxs}_t
zbE2T#Tq`}>m-(K1_mt1F*;T%H@9CvicU!(Yv6qec`rdO>lJ=fgFWEa?S8QMQRGEFp
z?G$X!-uh!>s{3R2H15W|pSg={l}sY{Cf%86Bld!OpCC`)UM_D|+hcB0`=oxq+?)EK
zXz#rxTkYm7G_eh?)!CbHHN{3ZSHxDra*ge`{SWr6={~r(TUpW8X#uyLXvfOE*Uj4Z
zIQ(m|Iw>2p=S-5e?Q%Xj+q~&Md&H*c?KO0nZ<TPmYmY?H(!E;~rT6-Gp5ME{Ol$9<
zWe4|8Qe3_FgI&TN7W;M^v%i=2X^CFjQ_9D0+jy+ec9Hq~y=EJy>`e;I+TEo3)b=Br
zm+hRCOSV1vntS+5diLeM_uLy(YHX{n`gO0e*XKQ^j8pe;i)Gq(UEtWKT>jSD?8NPT
z9?SXn?k?uv*S7rd-X$Nz_CK1>Y8O+)w@)f7#rElnjW#`<r}tiSX|r{(xV!h^-Lkz+
zOH=l)FkrO3#uZ|-IBE9YB#t$^*PJiid-|x{UjBj|dyh+;*?V!}z1=KpR_x{EOxrs*
z!es9;{gS<HmOpJeTd(h(X=ZNk6mi0Ke&{+oZElu*ckZ+9pBFvLR`wO^?%)!4+dmF%
z_L?#a_U@`YzPI4Rn%(NRR_^`#OJR@AC#}5;s^090i`cn$bEKdx|1|l%5ALtnYojrH
zZ?moVUZw!!y|bL=*`7Q9+U6LqyzSJi{d@VFx9)9hQ?tFd^zz;lC3ZHd3ETIk{V%gS
z@N)LvMHwG$Cpb6keHff=t8rkiUAbra-kyg`>;iPQ?o|u8XM20+`aPkAyY?u!ZL_)b
zuFKY0E8XU|#=(8gHTHX!Wu3BJTs(2Ful@DCx#Hrs3O9^xo&3aXefU)EHX4fVJMXk=
zufP#++u&-Jy}^fiZI{YU*qd}-cppFKq+LFFC3Zdji}xOmV6o-0YqkIG!?N#LMdn_m
zIqm!QIyl*$Ijd`XZN=KX34z%*yM8t7y;8Sw&#Eiu_KJwI*&6(6v3Yh))7Hgf{~pmZ
z^K3NNoVAlIsox_TxP5Pl?y)^qYiw=hOBwc+9aw7j!lBQ$_4#hw51TahJt~=R)8uk#
zue~6n?GJ(HyXJ1<-Dgs2y(eBp-&QD#bzj4qJvMtk*6nqhGGULVq1e71E(-e&?qk`L
zAU4ICW#clNtR+o*18h$3TbBOOM%9aZU-Yx9dpqj3?+IjnWLs}>aW7lC*Iu@Toi=ti
z<@ep!pSm|>ch}yYceiblJO%bXIbvuVa7x;GQR~CKOL%wh<~vfpJ8IX;y;pDb>^;-V
zwzqGCl-(hx*?Sk%mhFBfre}BWL-Ss%qggg*U#9QvNtv*3=E}KyKf4$0$<bP}x9U2-
zUG{2@eTz7z?mafs)b?+Z<i1PpH}@8q$lC}`y|Y(hnb%(TcSrU*9{jQQlxn+e{H+c<
zqpBUd0ydtttrdH+m&f_&-oM)y?=vhDwdIyk-K+NI;oe^!PPS>wH`y3GX0?@jD!$LZ
z{)O$|fb)AdN^IGi+QYfGV&;Xt+V6DiUn>6JYtnk$?#zx|d-Wen*sR<gY|Zu~Z_fo4
zW}6<{n|u9)H1-sKykMKT`Iz<3Fn8O9_gnX_WdCdP;?8fI73^X*Kj(GrRf$<_lU~DM
zAA4V9-`o1uy;D~&-YfU@uQiu5pY5ismc1G4WcP8Nf3Ww%uH}1w)ynVF==!-=tIuQa
z`MNWE`)6+3eNwk|?}9X@eNKG~_pYzVw_PVJVq3Om^WL|6{PzlcXy3E`;!c}fl@E3b
zzlHaH`qr@LU@f=(>xfBvIan^(c139HtzY`Z_M?l;J~t0u`*R_Ndy8DF_U`1~Y1eUA
zz)s?&z&_<Ik$dJ}3*S5Idg%U=K2=)=7D?-cQxa|0Enc^;P*T(`I=I31uDpcZw+!)p
z{QrgbZJV-dZ<EK)z3P+y?Ui!p*_V*!zc+dNG`qJkOrW*!_8X3e?$vRexObI>k8Qhx
zh3%n4Ufb`tw%g9zwsG%+pmnxkACB)0pSi-OC-(H-MDHtm+(c*DDwrhOJ~$Y&m(h@O
z-^_R7`_$S5_TIm2zenTa<h^!U-23#ouIzm)>9I?Lzj}`u<EOn0z8yB-{yf;*;?28v
z#dmGHIaemxew*`j&#N-EeR}CLY{TDF*i<c)vD@Rl#WqEU)o%4OjlK6|>-R25{bJj>
zL~pN(zQW#YMt-{k|1Md7=1|^e*>~9X(Tz!a8IpW##g2E_dVOELH|6~s+e`6tZR>v&
z+9vH6-1FkC^B#-&=l3}D@>`o}i`pr)>g^L*xoPjOVClWPm}l93v0T6RvcXzgy=l#R
zUt2HR`|0fBy|ry3Hh(Mk>^<r7b8nBwvb|xml5Cgs9oft6d3EoB!%?<TSB&<G6&CJU
zq5IB8u6cuP`f7H&I4LHZMxo`lYwk?no8<a>k6^>Jy?gV|?CpM(yw~v6ggvJ+Eo=qK
z=G!W6n`Ip;eRTJ&!UcPiIX_z$ESK8L60yNnpd!Tf#W`P_KX(IcO=8$>gi;yzUi4Ag
zcXyKO?*G5C_VOi&?PbmIurY~PXq%KTz2`2A{@$9;EA}qoUb3hAy3W4i#VY$A*cR@I
z3T58cvHhm)euad+E4Sa<`(g*TUGD4dJw~lZZKNCz+E_X-+51jLY2QKft$PDq)>^MP
zW4z}~9*eE5|56)y>qmP6Qg-f5zFcC{qL8&W!O?2(C4FX_O9dzP%3SWUjh<AoH)osi
zUNzN@y^k0!?BV&^YFiL(YV*K|eYXzF7F!X%4YtNRS!~a9a_?KVS8T89xzl^+TCB3s
zl``CGq^z_zEaZi)LEa%-PV-HB@9g+(Tj9UW*5K_z+pBdZwiRbPY?61{TBZ5&@3nGS
zZygZtZ~OeDwT-*^!9DFO*V*3sAaBQ@D6(f`UgI98R&5)OITLLJLay1IZ*;ZQ^qOwV
zcV_+G7Y>v6ZgKu>y?Oqsy&KYw*c#oF-1n)nVej+(Yxj0tIk;Emg3w;EKAXLFs{L$*
z-*N2wmw3|la;&UfuB*mgX3=f77X?lC7G6BKx6F8njr2Z^y>-HkHZ%Tf+BGqLwb|g4
zvuEOaX4|vZ=i4;QzP4*p$>Y7ruFCru48HHUslR%!UR|5*q}Ok4m0Q?sayKj6Ix63=
z-QeqOYkzC1jbQf#n{6Sl_iQy$-|L-gV;kvdW#{m$WzVa}t8C|`?Ad!b^R2B(*N#1J
zOz!OYA8Bv9^7E0s2lo8f^R4yCo~GKRHVmwtd*_Jr+Q$6rvi-0pWN*RGhP_kn%(i`g
zM{wU=9;tl-;#c>`ec59DL*|jq7w*ZnaU}<K^{@KACqc!*_Q1QNHXY|}_FQnxv<;D5
zym!8NgY_-lvwJyy^6q)G?d)E!wUYZ7^3<$bdSvZ->bBeL>`<|5d-G_o%SC3p{H+^p
zxwzKa#$J`MHn`YmdtM}MZ_b<*wlYhe?zy;D($?W}vF&-m54QZrKJ5J}W54%@^a|Tu
zg)O#gHtgT))zGl_0oOB|tKt{;{=an8wtdc}y{{hJ*&}m8z|N;eV()i#6WfBy-8O*-
zb?mH||J#IothHSb-?~S&<mcW;n}zqDid(RE)vM!sUZ1_Pch&T3Rx3*O?0vLLdSBP0
z@V$lkReRNZTKC9x^4etxueIIDres^%Dr^^*e0HynL!(X9G?u*^TNL-LDYD%AL}2Eg
z?P6hjwc|h9*7qK?oquco9>qtOY$MiW*o3Ds*{#p#-gk+4!`_DV0ag_+PuWPPYV1|}
z-n3Wc_I2A``LFkC{WIS?F@bIGRmOj|1s~-0$k-X!)Xx2Avm|87UIn=-TL+o`UH7ZF
z_QkBs*ju^Zb??8fje8$YSKph{D{q^~ox8VZwV&;5+kLyQ7aLltubFK7rg4GohI{sV
z?sB&5{qSJRUXAx=dt>AZY!|z#+d5p@waevY;ht-*_x7w#mfCmhh1A|%8M6C~7BJb}
zcl&00*+F#ggy>hh9UW6_*M=$B_Melqds%mR?*s`;8@I+?wl7?M**HbA?c3o{x<_ll
z(Y+;av-U{Z?X!9-wB0uN$R(SdMW)vG4KntgKFnZK$f|9-=ZUfHw<<onb#o8d&TL<>
z_r&Y3HbF~SZ6jt~+_UXdvDM$rXKkwvcI<Vl<FhLeW8L?CH-p_P9hQBkOAPkaoKUu1
z^-aW<uSwK)PSJH+4}Xb0HVM;gKL4=TJM&4t?L1$3yOnNIdv$p?+b%RcYAbzzqs_s7
zg?&jM?DjHlmAC0%IBBnp<uRMLZ_e*w{lQ^p@+H^Cz?9Wi^5#$5h@UHLyUug(Ev%8;
zo9DmFRwG_`@58-M_SkL|w{vj*u!p-L%BJDh&pjS?BD+iDFYju(*T47QzGr(5yRF~*
z!$WQF6e~j;F#$1auhi4F%2tQ>?tPK9cLvKRTlQt!_rAD)XK!lW&AqEmTkqkvPu@Ez
zOV3vA{o%dyb{^mRjcejwhLrXlTsdNP6C8i<5sE*uSDZ!N_D)U9-iD{It?xeY+H<5c
z%C<pL&n8i^XRqSKcv}TWXWPo>IeUNJK59EF^!r{7W8r;~b}oCp9Gh%6`T5vHy!fzZ
z&NKtti!O_I25fz~%VmnAtxL}ZTiJ>UdnEU5+mm<ajBWAy#d|leFWc*L{D<v>hL*iQ
zZVB6o`#IQN*uQ&E=uhvxH}+iLW4(Nbt>_DmJt^{wY*sGLwE33Nx|d^m=$>~=*zJ0z
z-Q4ZRao9GId*!ay*PfO)Vs=|wNZ+vKdnvQ;oAwUdrf(bf&aqu$JH=Pbw)zR1O{3G?
zJwA_j?u`xLwd+^0+Z)m7YwN$AVPESvX4^g<;l0t>Jho3IcJJ{xeZ+R|-Fa3%=eOG~
zNNBO`=J{oPz}t7v1e0GjA`>Us+%P*}lTa>Zw>@R$-s29EwtLc<ZNDhI+B0M48yl79
z>uo=Ly=tQ$U}y7Sx`4H@vdx|=A2#f{JBMTM#!R8T0-uWaJkZjy-TCO6%?X_odn4ZT
z>^&6sVy~SH!@iQ4X0~bdQG0Ht1ng=qY}}))c-iLD;#a%2Iri<{pr2_yqa@VEZ-VpQ
zDT;}<^E8F_K6Guey*tTk@0)wPwzfrk_DVldven=Jd(YW(l6HZ;E_>yqnr%H6<?NlH
z_131r>6yj8{~!0>|MAeqO0$2jix8WwZ^y^IQ%^_jo%ct4uS}-MzD1$WY|Gl%ZA2%l
z*#0uSxp(s6?>5U^qxLq=zhmRoQ?i%ou<TydZsvW*1eV$+*-o>q{r`E-%e^=E6uHi`
ziJQG_Z%8JOU1;=V+i5?__sl&xeUEYMCfirP823I|DrP(9OZZ-f=ZtpdXZUS>v-xZT
zR~p;>NPTI0>g}q%9^U>oC9_p+ze{%P{h?{HH{hArzBfLb_Xrgy?X6kSXuFrU*yhdM
zmOV1RTdgg6kL`6Rf4|qe{OI1<TRHcnrt0ov*t&XeL2>%tdpcjO|EwyqO$nZEQyd*`
z`&)X`UW17j_D1^K?@4=jes8&k!ajvxRn{y0P4*dAaM?XRUcT4krp!K>*-d+Iy^G$v
zf~{fqck%YU$?rGr`L+MC?U%)@wjr%^_ikqv+&g2@W?SB=9yZ*+RQ9e=(ziKtk#8@f
zo2#u|-h@3333K=CI<accvnOY5G^|x@%zi1_{&~-Bdv1-0ZJ4UUz67<|yRYa9>|<NE
z&&K36qs@sH1KY!!s%%RSWY}i@5#Cq*o@?*XS_Zo-VZ!^KWSZJe)z;Z}e$vUknire*
z=9JF24cvKlZ{>2feT{qAY#;AgWV_FDn(ZeOp?#LzXRP_`_V0aew%T@g$4%QUB|^3m
z_vY?dTgPtm?ct<7DzE0+rfG26=^B{seR+8GUQ3Ikd;7Nw?$a>Pu|1*j)Jjv5+3tq-
zoW1Tn=WX+Xf9@^ax_A%U?Paz*r@XOcR(id6P1c&dev9jD7qZ;iW%vBy-e3JQ_m=-;
zwX2%RxbH<gmrcpHa@(YTj(am7sMy|1+PEjaR$}iKj|$slp_?{O*)QzAwzp>wm&EhE
z8!j~2+E3%Mv#{D`dy?svty_Art%reto$D4=I~nfoy>-_o*{(OZvsdTJ`n@IVY;8F`
z`Sx<uxbA(>>bYC^`Ba<f&l+ti9%}5}_h7p1_5am25<gzquDi;&w@O0VR<C2#o+W3#
z?lH-0u&oS|uwBRgYiIcJQrk@pZMKf)8*B|P%h@@&zS*_G&v@^h?#R6fQTwc~xIEtz
zd^F6)VfNa+g0FAv7J3$D>$_CO_H1nH-pCC+`}$7>*fQLmZ2SA>guN4Pz25tD_sKmr
z=NN6*XZhGp3)8UK`^IPQ<6C-rTG);D&Y5v?uS00Y?jNrWZBG|%-dncrq3waM`n&lT
z&EIQmP`sD(Z17(3S&w)3%=X;d@}zyw?wd3Bo|KZYovEsBcimB9-|1x+_v$rmvN`tr
zfz5%_#(S(CIroLDKG`d9LU!Nof)}<m8h7_}FLbv3cc^1;<H`8F7bh|8i?$Qq=j(sX
zcKLpuedqp6vDqBSzwbrBLmS?0Q*7DSKG>6gk9nVC;xb#0?ceuuO<c1_T&!~Mp?*c%
z{Y$p(xfOJ85648MeZ>Y0`#hu=Y;|j%?`2sSvBz&gx9zL8&b>VKO#AlaoUuJ9qi(a<
znr|<wIh#$(lCIsS@?>o-N?Gk1uK3zS^U2#zQj*@wDBrdBgU2zOI%D3wxsSiwR;1PJ
zQD_OYZ79jJ63FV_yR>%h-jcxkdzHm}Y$HNhZR+xO+t%LYx6__@bnhANMYhV1|JrB=
zm+rN_K5y^67bQErje_?Cv@_ZE{!`z5$Z5ge6&{avcchEj9w?r=S7`aNy(~)}?X`(|
zZ|j~iV{gZ2-hF&`_U=_#mSZd3GH36j#W8z1D!<$CHHq&_QMTAOyPn-9e|PZStf}mK
ze|pO7bF{m-_eYMx-dg92dspp}+uQQhcdtWY-X8bMg?qowRNGs2b(1ZtDE~h0EZ%)V
zN__i*L%jE%3%_IYM(@R5yKB7r;;!Ac+4hTXZ_gt!Tb=C!dpjyB_r5n1*jHz;+4jYk
zg?qm|Ub1((<|UgM$9e63b4=YEcXg?)qTkKEhhDzhs~QrvH{-=7YlaCrd*v^S>}!p&
z+*=*K%tq_}9NY9we0EEJFWGf!Re|mA|FU~E-gEC=zI&!^g|C8b!L<W>liBC(eLN}0
zX0O?bJu?Jt_eN)YvRVE}*zS?ur@jBSZP~l!fa9Jo_W$;%U0G!N(4%}WpHrJnN!FV^
zx6AqWp2}6&H)l@0E&uc8J+CxfcZXCw+*8~pW0%v#W!IeSZ=3(?-QG=o=l5DiTI_wZ
zI>lCt>-S!c?R#tsEZc0#_G#=n$9s40k*;<dkuv3darfo+sH&XVJL^T(-fSxk8-EeY
zy>EZZ?tL5MySJTn_TD)rkM<tezj)7^pb%T>q}O}&=6dfsBP+ab=UaChkMqrYdFHe2
z?Oif|@4tFs8=E83ZC%!J?M?i-c5hY+r=3epj!m|oq}|8LYqp<7Iqk}F4%(!%uiEpv
zglk`6*S0;^))?3rsG8c<WbWK+|Kx}5hvuo)OSG%3J&m^6Bs@B@mnl|c--LkTJx~6=
zw#ii!+Bb#sz0HzDiG4ShKiq42|J@#)#f`R&2POAzS$)-J&D+y^q|%)BZ1nEjn;dx9
zHd1KYUdz~ZHl@FuY!hOb_ne$~-1f_^Ydf=EG3}e-UTPa~UUiQzSC)+!`<A^9Hhgvk
zA~kzDCQjY^^8K;BH{9IzS}Ca6s^kdRYMq&G<KTPI=H)ZrJ>Pmo_wjLv**YZNv0*A)
zu-ANr>b_a5@AiE9?QF|&aJ_9qt<av335RSX1xxo%uPxiV@R;J>qxl*RWxE_~I9`j`
z?P}II&{(o{@0re1Hq!$-ZA(|M*e-jqZ6DuF&i&_wy6qmszP5Loy={;0yrT!y&Nb}c
zrgm&^d(F4KzZdV@d*XuI0fAp!`wWv_>~&Bnwb3$c-kYT!Znts53ET87Ui-H#o3huM
zZ|S}-cbfLAx^CR>?R$LRjeS3Dq%$q|u9Vofe^;xl-4uoWd!0{d+rAapxHsp2%WkO!
zjQeiSyKc?iQMY$(|2doewjcI#N%h&j-R^83wy@Xc*OP;Ll}%>t-_+~6cZPt}{+eBH
z_ZTr;+Iulr+4lF%dA4q*=l7~cT-$g4Xz_vhmwxYFH!XZ`O%U5Y1>e{Egzw4iUAL&;
zHsXDwZEovjn~n%wn=JQRdzVdD*eA5#Z?EE68C&5U@%ygpWbO6zcDD<wX51&$Icbmi
zqpSm4cggHMVfn-+!?1SWt=%g7!rWi)fAP=MruS>)Ug3t^y^YS+d$omg_s+A|+dD-e
ze(&++s=M_~^6d^wrSBD9puQt}x~J`q1!6WKk4*M?o>bna%)V^zVQrSZ*+CokY-5ny
zKl^Nejhf?5+qtvY_iYIFuwA-y_TJU};`=g_ckg8|ciwBheWvX;NvAz$Bii?hFSu$`
zyXL~)%$?i!9&FaO`%}DfpZ-%ZyW9+qy@D%`?zP)^*S`9p`0n}95q5nA;rnjhG`5+c
z|6;#LaNFJ-v8j8vFDtZ-F}uF^yDZ-UWzDyHMclG%)!mQpEvTKmH^<I?@6~9ReF@qh
zZ1(r(?)jdy$gb3U&EC&Pi|st0{@c4dH`w-h@)B$2MxTBBsTF$-uPE%3&Su(MyVh>+
zKZCdS&-r6)t8XaoeK1$amaBlzW`}U}K8uNW_qn{Qwsrixckk_N0Xvhwt8FqgtsGk3
zbMCc|zPZ=;cESExSK4h9XK&y8yFz`>c1ON_3UBuBDPTynJ{$9D?*&$ATN@^yy|44M
z_x-=rv%f?nVV|ds_`c;cl=oMh<+rK%$+dTxM$Ud|-ep!d{%)~bo7}Na(UH}Do|%w+
zS9-JEJ*hKxla_heGl>iCyB5f|uP?iI|NbYecJGgW*(>uw(!PyZb8m3tJli)y+wHw|
zdiQ$ei0;Yz+qVCoO!)r5eIffF#Eb7ORr|KvDqMQs8H3$60ekFi<+FqLOm$1%r^2v!
zuVYZ#zP_WHdx}1H*l8|1u=m*whrN<fE%qj1pZ6T-KV_rq-EOzn_siY|h1>U*PUYIK
z$?mdO$@}Ks!mc@cC-d4J__~nSp~9Wp_Hx{vz15%3*}J@)u(vgF;l39<4)z?z<@*99
zb*!8ErS>he*tNI7gLPk9IG1(W!sfl~rcd^8l&rVD`}FgkLcbgPr>3**l@WTq*Sh}l
z-X$p(wv#4s+ii1K+{YU8V$X-=C-!bzF?o;LyM(>Z_t#mspM7n0W@5u0)kmK8%S9t?
zUKYCU*MD?<PuY|c`xf@>vM*+n+n0WM+x~Ay_U(NkuXtep$rUznbEn#|x^J~<depS9
zq$A0GgIvM>GtCjZZTGPs*e~91e>Asg&o{+ryWC%F`;=xj>}9+Wwb$N!)6N|>&-PS2
zcd-3@nsuMq4L-ZBqTannehTe7tHQMJxnHb9{C^SKqm#<_=6*V1pKQ8eZ@u2*y<TP4
z_lHiD+-r8x(&1ui{yzP$%=;$1V%@V&dgbnrn=Sie=O*pDx3gfM8q?-IaqO@5a(UR;
zbRYLS(EHkQZ{qcwy~iS7?pHCLxA);aJ^PE_!}hP7`P%M-*!BIl7f;!1W&7Or^Edu|
z3#-3bpM9rq=fSD6Px<ofy~pAh_K29-+wQ(>w^#Sb|6NlR>}{nj8}@~0itc;#_KqFT
z@iN=M!y$VUgy-&!`OLQ8W#Npy`a9J3eXi%QohW_FDt=$rKK+$)`##LsW~acNyZ7JA
zn!T6bsqWQ~KDeh~{~6mY5h~VA;%$4?W~AC~->}}gZ0q*DUJqsVTAz^J6I`&umfP|4
z-Wt|__Fi&HdkkM}-gQ?|*kQ|+=)FGdiu=@Bxb{^y&)c_4VVkYqj-9r>Zy)c?JMOY~
zqxkN<{PijOcl~GI8yldx_xsx4d!1BW_Y{0yzPBS?aUaim;XPG)U-$aHzq2Q#<*oe!
ziGMaMmdd*?ElS_J?5O_UGb>~d@H_0V{d9Bh-lun#?X^|nvJ2`n+pE~H)8^4F<Gq^A
zmV3=Zy=>VT9c|Vt-muNfdT492M8sY&b&s9GB3JtrZOpc<vd8zZ?XI`^v*rFim(9ob
ziYM^yy-`1ZZ|LEDdkaku+rHX)3v^z&t-2roK4IBj`^eu4Hac7^`>a=b@B5gr`oK4C
zNt+!NTkZd>c)#Z$N8H{3&iOXl(^>X~OenTtOr5sRJG{x}=JWk~+UtAvGxt8;`>Zc|
zZ*lIny$OX3`yR!I?LDwL*yeA0g1yVS1A9*<zuU_Ym9|&NZ0f#0JG$%*S1q)^uD0FA
z)9~5e3G-VIG{kD`?}~c2FVw7U&l8TR`~Re<+G_vfw~;rou{rdZd7nVigT3uiiu?C5
ztJxm0?cF<j#>IU)y@3b3QfJwI{d~=4=^Hg0=bGvEtTBfBMTO5<-JI~wKH=OQ8v)gO
zdyCy}+h>;=>|^xu-hHU*!rt08wSBT72krk$SnZ8q`n<o_SYqEoy9iqw-9G!cB?tGh
z*DCETyJWmyBy_p8gTp@ScZXN+{j+(+-W}d1w&&Z9*>+78uukB}*jL(o+Ggv0-hHZF
zzxT8xvh7<NGHu_I?Bjbsi|g(SIm~bS>M7&C4!?tYCW?gF&goaRoqExBFYg3d+mFwL
z_PjP%-sf~xV_$WO>Ar^M(%oJ){(Bzl*6uEGO|yNlc!jN1cii52Th8xYb&_rG;i?6D
zg~dzuTCq>I?GoN=8!(|}&#aKGdqeIV-y^16zt`fZo9&DR9(#-vKU+K8KW>|$#%mYR
zkZN1gyml{xnyU@Znd5sld;YP_zPxg;rTKzAOB+A!W#}l|%j2?i@2iIkZEJZ}+t`Vo
z-Fs_=%0B;2hJD}f3hdh>ynC;y$??5~t1GRn-#6}Eac{@oBV9{ur!-~kO{kE!F<~^b
z&6Sw3*Wl6g-9B%h>{WmJ!p18*ZEv-~*}V;+>-IAFUf%nzYvtZE`Mox0gHPB#UwCNm
z_Fe<qg31rJ+y5Wk6}?2<Zo#ouo7djlb}ZUWdl+V(w=J>$zK8kaB`c4$llRV0SZ&k&
zTY7KYe;&ITh0?YQSe5KPt?S%tzrcI1)u%~&u9O_w``o)}@1*Hl_ukM{*%x1(ytm=n
zj6L&8PTPLvvfjHrMR1R8Z=db2zJs=c3M=>S{IhFsLt&Kd@q}4>beyKx`bdA;TX0Nc
zkK&(odo|}<?`?R}u($1(*WO|&0oyIwOKq-)pWExZP<r14$Lls`9y)vNXU?(};E%MG
zPtV-zud8M2Fo|bxpwHU9XPEBnIdL+=rlo3@?PSkodlneJ+ili8*S4TMb59h%@SYRg
zLDuW{`|eWxcxi9KF-E%$1+6v-8<VUHYM1Xl@iE=z&niK?=Ml4Pwh2z$vms^P-jk~w
zY;TGy*=@VbX1k(GYTsg^Pd5K&p0f1|l;2k-deZjRivGO~zoKmtCe`dY<IAx3_>U8N
zOD^5n^FiR1%||KcJ@aI{_x=@7-<uvGxc620p4~m)C+}snEZzI=44<7yd-<N9u`Bof
zHW#oJDEnls?C-L7#uB-`i$BWRZSl<5Yj;e{deuUH+g+2J_Pz;H+IwE`!=7*RPVJqu
zQ{L8jzwF*aiv;)W+nc!e3x}4Cru53a3R52K5xr5lS0P8-R`FlU?lRWkz0npQ_wM_t
zY{ztlW3Qs)QQHeG+jhUUdt<v`?;-2hjc4|TC9JSDlbg9W^+?Lz70KsquTOtvd$aD&
zo_9i%Z57vFv7NJSf-OtU8Jj#F&V8%*yxg1Wws3FHn|*t8GvsYq9=+M)v&!AJ;p7+V
z{=H}Ra^xu5h-Iv_eSBfno>>#u*(&wV+xtUQ+jd$Z$DUkM6I%{@1>3;gY<qKCPVZT<
z@rljRqY``jrFHk#oLRfqdVc)gYndnaHpZUW>%zNq?=G=zHsUkP_Do4!z4uf^oo(IE
zWLue8JNC>uv&uGQ<Bq-Cn9KLxd9J&6&8K`@(VMEaEum*^^)H#&cuvXN^R@n%jY(y{
z?HA!RTQASry)vg~?%BM~aj)ho|GkWdms)#duibk^ug5xV;xt=1|G9g!!dUD)u0P%z
z?sv$xFt^`!hX3rnM;9mTUDP~d?}Ok~doJmj+h_|(*{xsqb?@RwDt0TMb?&XHI%pHS
zv~_R4+QvOaKhD~&y1dxNp?%iwDJEunF2?cN*07)6eS??ZmZ9^mjrHo%y-Eq6_MSXh
zv-eIZv)zLm6*g}-ud;oyGRS7PwY9B+xUQ{+{43jo4~6#GpI^AQVdFBZZ%euM*0}Dp
z&Hp%M?><R6+k?MO?fG|kx$Tn!Vz&Q%=G(rRd35iu^*wu)(_ie>*&1hCu2s8Nj&;}G
z`SZTo-thIYjlLFRv*3T>Zic(=yRO{Zx@XD~oxS{p>-Ks@i0^6J&%Vbf;k9kug*CQ5
zibrj8g0I>fkmR#I&?{l{$a8|tp^4IVyA{vwElJYYE6>8a@25@g-uG#T_nL_D*k%?s
z*~T)iw2iv7VsFExOdIZ*(`+5;zwb@3u(r-!yL)fMvduO&OPOs8UhUeuDTr-fLia5j
zo*5VRE;pQHld{}pZ=d=FTh|+wwhIrc?rqnq+AHaJ+(vc!Ya7*|#=WLx?Y1#@X4@R|
zRM=<Xwa0c=tN1>K2On*AN<Y}U)bNw-J)T8-c=s;aTcIs#E8A|fSMQk6zJ)tjZ7=+o
zVG|W|Zcl3yzunGMS=*oSbN05pG_$q)bKLf+^C#Pw=*fE+Yrj}8cim~*-YssMZ^2<#
zG5yxw>e`ume`(IMP2JmK^XJ)hYuztQc9)u5Y;J2!+#8X6W>0iji|v9V&-NJHK4#;;
zaMRuyH?HlO>D+9a{z2JpQPcUoEQ+1B@<I1)eplP?Jz-_Cr?Fbn?$x3ldo#-z_Fag&
zZu4R8_dO?%T(vzO7rr<8<(a)#LXE6{NKD%6@p|K4)9)wu9=rT%ueiL+-lkutZ2cSk
z_wr<RS^sA5-Lw7h<-KMty!&d~+V*lkl(W0B`Ib$9_f*@hHPiMstg73SV6@Vvd)3L^
zBHoX8ADEJAqr?_wbNHUZ-gADowu?i(_TG8IxX16~9h=JeCw8ydBWcSKJk!Q*_VK+Y
zE}XWf>X+F{CQ94weRE~c-zj!`1NI2)J+N=v9@Xu$_HJ1!v3FBV-`*LvkM}H56t!LY
z`u!fp+8x&an7{31e$i(=mvh-(nfl9nZe6=?>mnIyyRE6;wzOAr-})uz_i#Sixi`;^
zX>XGbtF7V2qkD}@XWG0KF5YvGxoR)FKChj0E~8D{?MZueS1Q<*XU^EmqB(!B?>TN;
zUJq|;4JBrqUZyQJzaDVd3W{*pDGA@Q`TSnlZkKG&UYF0`_vjzKy7z#2^B!w<Rl9~r
z9_u9s+V`wu5#1xVD1C2&)-jvQp<DL~bhPZ<eOTVkc-9^p?)lI5{$HQIH}Z#<jfpSQ
z-U9Q*dktlF+N>9swpDE~*?U8Ew(XklclX>(X0VBgk+6C3tlaiZSlnKr7qWXl1TEOJ
z(q-k|4Tsm-&iSo-;L<<VJ-!lZd(#Wq4*ZKhu=jbw+r6jK?e@*c_O#)hmuQ>eG-n@!
z?Ct#*rZOC8e$2VAa9R8Tr=y4V$<N$rqf{cbx8&8dy$UIx_kDBj*thXr<KFf;v-a|y
zj@#>-;=6xMw}@?wPt!hEZe?4A(D40PlMdTy=NQ>td&#)}-P@~s*?&ISJ7wwtdu!i}
z-Hx*9HdD{G?oH3{u$B2Dzwc+V#=dXqbM^+D72P{KoYl7b&5FGX!k+EP;-9+TXJ63X
zN;lno*Ee<TpXRD;b8*cy`^oM<_XKXt-D5a6bibUB&Hg1*By3AJX6;W7OFkfxcg=?T
zsmwl&de41EHDdey-ml#I*)4RhU9#rhOZFS~DloFx`d{bWqxM(U#`T=QzJ(@R_nN)9
zzVDvT+Pz(yavZL$H{5HyP-@@H4he^UvxN6m?$z8kH*CVbT*pm&a}7To;J(GT?|kAJ
zn^x^LdoKkG>{%Z2)_Rp$ylrQs>0Ys1*?p5`mh4~mV8vdIq~iT?B|dxpNcZl&b3b~o
z>DPmMix@Tb#?C0)yKTjX-6rDS_IW+a-1{MG^{$1M`g<9^{j!PAxx07G;~9GoSZ=iW
zcmK4_hhIf|dH<E|eav)hZ^C_{eRE!Jw~<*avF};E!G5!+ar^fC6R>6eyJ4@k<HtQQ
z3EsBmSJ?Lz&EVaCX`27ObDq5Wyna;MXRg>_8^*F@ukSCBeUl!P*skG<+4r48cVF|V
z+P(kOTK4ucPTjjV{`6jd$Dey{`*`kkIJtZ8MhiJR|KC-6oqnqwa9~W>%j+`1;qbMe
z`=0mC+;?@|R;v?2Q*1Uby0rJqPgNVAe@T18LSt=;gs<(<(_z~;S)ph5AJz?f1Ex&d
zCzZ%#_q1AVpUdZfecc-r_HAA}-@f2{@SY8x*LKT3nqWVz_U9f3{V97@XSMJB`OsnS
znJ(GA*_(y-Eqb+Z&+q8Tdt3g_+*^45^4`fDyZ5Is-{1SVTy~$lueF0T%euXLs@B--
z^hntMR9JEEhO0ex^ColdKO6UYUpmu+{ai|Q`zQQ4u%9=4<$jA}*Y_UpK4W8PaMVug
z|C)U(HhkN=eCF4EzxGS*J;Zx|&;H_<dv;B9v;Akh*;?bi>%Ox!o%@{?BoADAK6#Jz
z2CKa%73b`|XRv(l-NXKSxr5(W87}YG_v0hGZQDkfeO{T9cXLUx*nSDou@+jzyHET_
z&H+`&3w!q*KfAZt?wH+{;`#d|7JS`n=Wk}m$e*@1>T;;<+Op|;C)GC`c$XaIu;%5u
z1I#yW>}g#)(ILPxc#r#o{{1uFKDEm@rM{o*F6+LtUUt@>*9Gr2H*c_ho}|BzcM`|G
zb1Q%E&3L$V@63HacSqlta=_$W&E9=a#rD;BiSIpD=ecLEbm|@!D}CE}j|^=$)<xKI
zy%yOgls9d!-zlX%zdpO~o@~>-XaCGu2UH_H_xiA{-p99wV_(FtHMVP?ud%xt8@yK}
z?6`ec*NMGbWdsi7#9Z35dQ!f<m|4i4qrOc0Y@@RrzB2E&Z~akVt)KK||LKVg4ifX1
z>@{|8u?t$8y6=LOj?I!Sh5PwF`|aK9_IvMw<|vzgGj8tf>XzBJGDvQpaGa#=pX=wX
z?*IF|Z{_Z5HsuF0_vTsiILt_G+jHnF*Ium^|Mu(L$liPYf{VSC%Bg*acgyX)uwu4N
zlH~Edo%eM2y)0b5Z(&UJzD=+EY>c==_HA0!yLa)9$NNve?6*0ZIDPNRgvR|Z9v|Ji
zY6JKFFI)B;2t87@e})XdJ?}56eK+%F*iQIpy;p4e*1gR4CF~|9v+ukAC(~y2rvAMX
zy`=X&N$s_<RMy_T%u--q!n3V*8jpY3q^f`3XJfu$@5X6edt=L5ZRhCj*eiUFZC`|h
z@ZP&ykMF&`pLgGi&vWh5E!6j(u#mPt_f==_fn#nqD@0oMY!!I3caB`Rt?up9drw(x
z*=s1lz5m0XI$ISXp?%Z$DcP2tKf2dt&(Xc?E!+31DxKNCEBb?tl->t3-?Xjv%c{HW
zL+8!0>EdqQlf9aKf4zI)-gg>@_ddGXw_8%wZTHKCr)|Q@84j?U$L|sB?c1x|^Kx&P
zw!_|~{}T3Q{rj<3^5vUdyQSakPp@d(`$Td1zAZ;)>@{k+xi`|dZ?BxyjlK6iUf8?v
zSIOR!3SV{`d=uUK+l*&#OW)N!t~D$6N<8VcT~hUE@1`uaz1!z@Tj$y^?_=zkxtB46
z>)@RyMf;_f1Rq$Wx?}IxgR^a>*tFX8{tiE2ZTWrAu5|Ogkt-SZq`&Xnf33-X&-s4|
zw&%8c?_E1hV&BwlMz(sL^7|)z5ZX84lit2Lwm0_|6{*;|HOTHeuRUq6`7y`6+tRi6
zZjBV#$60)Hui&xBeHG1z_MGy}+3UG1$SzoI)83j%FZUd3IB&C|Sb3j|_x8O^sr&b`
z*fbspJYBwb%>pi4zB8})@?4y=Z>7G3{S4Wk`--d6_N44sZsT8k;y|imi(Q`7hW)$t
zOx}C<+{eA2&IMYAPC0AuTJN)~@X?IDul(3;yG~^8_dk4ePpVSDUc)V$_qiP4IB=Ll
zYj4TsCwnAnnf7?{80~X6liu(1S=s8P-~sy!Sy%U#MJ3vPHVQv5ft_pLwYQo!$K$$p
z3w&$Y_uJXTL2c@_-3orU_8r=Df3L~uWqXBxHt(DBYUjRL$*p_awcQWIr26jrkab{>
zOz5sX3)SoP8l<-EUAX?>KD*_;wufZ`_N|yyV$VHSbl-zr%KHQrCfFL>`LnMmZ0Fv`
zhx%+@U0iLmZByT#h~Q3JpG6DzWZ#&uw^rKSCWk|Lugp3ZyIUsH_gyX&wf%nAYOg_U
z@SanRCbm%`JNG_XxZ5^V|I}Vz{mpxxdvWfQW#+TJ6?4Is?e~ej=U3jeo4YE<wu-4|
z@1|Y*>@HrsxhHtyA=?-74tr`Mn{7|U&fLp*g2VdVooTi&?^M{bJlnGOLf@j@Q+~|e
zbMyquz7KD5ZTiyfY(IZKZ*$*j=N`GVC|lzti}oyFf4z5$rMT^^)-!u69}4VKUp0Sk
z(vuf^WX|*L>+BQVS2X9~-X)BkHVOO_Y#XHNtjrSD?!EEmjSb7=Gqy%n$8C$0H1}3j
zcGw>03f}u_k+|(st?YdZ-t*Y4xOj8#*9TU%VH$~hFYNnZqiQ~HuQa#bzE{N@wzn2u
z-g9BOoSmRww6#6Us=d#IdiH*-*s)h-@5Q}ug7|E6J|5lEo!Y#2uR-+Q8>b8REPQ&)
zrpA2RUZpAz+XlZ4d&NR$?VVW>xYyXP+N$yXr`-%wSJ<ewtlrDG`-RP-AB$|SRnNEf
zem8gTmY`dE+b;0!YhH9??~RQrd!#2T?)zE3-|pkrX?riTOx&Zlo53!SU1i_&lM;LD
zYL)i79_QSrF@<?g1-pRVl}wMlH4?jQa(rgkY+_^Ecg|+Mt?Nqjy&pQ}?q!JLw5}`N
zYa9R4Y;Wd^)iw%`6>KLt@!8t1@z}d5Yl5weOss9&KjwWOo=4bj3N^5uv_5EWdKrgp
z>!bI3vsccwb&yiG4LrBewxeaOO-<3}z3&(J?)8|?u=ni~72B4rpLQEIaqP`1zF~c8
z-mJa(ANXxcvzcwazt*)Cv*X;iReFl;n@{b#=Cx$)dGfj0retE8t^XZIJJ+rA_9_JZ
zve_%~cK7TK&%M4o-|R83KD&2P?OeOU4#&OBADQ<l*lF!qDj>XXM#$v7y|q5JQ>JV0
zm0v2fZ~D|-wt0ned$;lK+xI81%SLKS$zJZ4#e2JX^!8k;ZQt8}x?rz%>1~?|w~D>9
zx|#L`EWNS!!6P>71rIjbIt421OTO8>TOw%B-j9>n_Wg6Lv=u0mvK6U$zBhehm+kgA
ze%msoiM9<PN_!9ZuHWap-)Jw>wGP{3Av^7^*&nmLw>V_4z;tF?Kk0ea0-H5$5*JL_
zJDq{ou1k8>-hcHRb`BXSc0V5cvU$}rbI&B_|9jup%I^K6%4*j&U(D9kUBGtDP5C_@
zyEpE6QL)=*O<mmHMF%JDt>MzOWpOyYM@M*_t-*XAJ5QCBdrt{x+r;wSww1HkZ2KYZ
zug!iX2D<{zM(bU7IqXta@7}w5yYRk;1t<45vw7|{5$Lg1(C)U|mn^%x*L~Gqm5Ik}
zO*Z@5zGRf#vqzP2kJ3z2TW1xEy^V$J``ScW_Y@}G-YeC{yDxbEyS<s4S@!+-vBGxU
z0}0!WV*mF}eHFTQT7ayreCE8p2H^{Bu2$IZS-k$po+sT#c1`OT_H2ImY`32QgWdbK
z{d;R%`t}~VTeT<ZLBig~Ut;?VdTMN+++1bNBIIj5<=-y5i4r<{C3O_+1id8o_HMMY
z4Uz4)xxRzXhTF=@HsW^ko*D7&dx949+qJNY*fK|5w0(Nw-abxompzm2xY_hg4z@K)
z)v-PE_Ofju2b(Ql;Ub$5t(|**#cZ?5TKU@g{~Z2(DcRd>)Oq*sy}SLeo$yzty>qn=
z?diRCdS73YmaTovzP%?`OxZJKe#@R~4(fI)MrZe)xol<=^JJq9$N!$auWK*aB?M>h
znbX%|%M$c^ubRjO+qJ)H_kNZ+wr_&;W1F0KM%&|)57`FAOy6@Z*kw;m+<n^@u6?%8
zS}yNhA@svqR$#Ra(=zwHkAE$&y0Un-?T6T7dkueW-_y3_!QO?U`g_>kM(owkJhpd}
zihy0oL+QQUER6d~RMd7y@ZPpra&3d{$B27-O852HoxLHqZ_lr(do3OH_C7hyvqww9
z&Mr>z|DG+@+x9+P9&W2M?e$(E;a*z_@4h_`-T&{^HvF=8#g~<P?mPS2YDIGF38`7K
zw^glkuebdCz0J8NcV)TF+3TvbZl9n(>pr)_B-`E28GE1P2JJhY`eIM1fRt^{vW<II
zbY$De_U_x`%e25cZNo3yZxLJe9y-5v*Pc%o_q?t>vRBD{?cRM4=I^m)@3YZ0VA|*Z
zYRca2U!Ls!IcxdeJLfLjy%kp2yQ}SxZHuMQzTZZHdmkvb*{oaXxOc_fjeFIk#B8Ke
z8EsAZWA<DN-?DdqXvkjQlQMP=JCm&U9ox0{t+?ynbMIUC25PqKd2-cYZ>cw@UANr)
zy(<~_?VU4E+xET1Yui<S8Eu;i4)1-q|McFy-x94SJ!0RdF~@iBro~ZvdFLJ2vwGTp
z+oO8Z_Ih8KxJTpbq&>|BPxcr^`R#evq`6n)F4G?4>u>kQ_W#>!bZWugPg<!~_jXtA
z)eyR36S6FE&xy%P_g+7uxGypL!QSdbCfj>0ZMNmjk~Vv6o%YB!SJ+9rm+YNaerqqo
z91bf7wp~{7qEBqkDsQs;mv_=;f}hadf9v9GTVj^j%=Ug`m&MOwn_jWbwz2%cUg^?_
zc6R&M?KO_LwC8x7z&?XX(t9JHzS?WMkZ0c<`7m3>{{DSDtAFjab(XQwT{CM>%!Ubj
zg@1(Zv%8*ZQ(7Fn&)S`JZ<LC_zS&~GZHxYJ?X9Sk+4n&9#$KIT%e`m!?BAC;`}|(b
zLx*g%8z$^+`fP6dsa(vCw=Zsw@eNixv8l<n8x5E4J@G?wZ^6x`J-d&6vON+QV*8-@
zt*w%?{az!vqqbia{_TCyD6uc)&ZE8W7e3gV-p8~rrGMYvMbj1c>R)EsXUo~S_ek-Y
zz4L|t?A=~@eD5UPySqz+HtpWU(`~cgzj5#GM8CbBweR-My3D!PN9e&`4K_tvwae>l
zi>_zv5qR0S$0(3z--+k{Y<}rpwDmb|v6tn=bz6o5Ikp#DH}4hHNZrHzB64rYl;FLb
z<*WA^nRM7H^t0?snd)XMBgwtzD#uJ)37>y97E*?`J?S%TRkEu0a;wbS>tQUgHz{cM
z-q{gzZD&}$wb4qSWV2z`9h(*QclVrN)!aM(yR2QQ=JLI-cpvO-*r%}9;$`LDi{Ibw
zy_r9AuS=?-ZJl|a?S!K%_AI(H+tz5}4qKHIy0*`9rS?qo=CZ9lymxO^&0*V1n+5k8
zI49Zc&OfpDy+-$*1*bS{cE~Tb{kJ>aYFFm|J?`Nhd*^2y+3U3H{N86iZ2O!JZ?aXH
zAhq`hyOgc@?^%0ePg~gv?c1=|Z3By~q)Yy8mxc{{_HTc;H{lb*UUL%{+xc=A_g>Qw
z+!t(Jx!aykX0LDo*WL$>p0>+FC)t>PzqFUfrgzWw$_v(RrR@9SuJ5oeoq54VeV^n$
z`_<KZOaHF0Nn@I1+hVtN@3EO@c7G5)V7u6?Z!gb=eKzWrZ2LN;;%yiNckMRhWVLC#
zu-Nv?ylu7)UgvCoT-so>;%}F2VB564lg;>TzibrW%W&?+&dLz(eSdF%+<RkogpH)j
z_r0gKuDAKOamL=RgO_b@HaJ@^7MIxDz@%mCen4nnb&|@S0{N+X?fN&`@SHN-E0Xic
zc81o}J=%|gY<+r~_wITfwKwYZ=DkJ^PxkV!H{WAl{$@{5o5tRR5RX0Y?oZhBHgU(E
zkmR#_ZF8U58ojx^Cw_(h-r$ri)-9jg_I6&;u&aG6WTPQiYkP6++P&Nh)NCuo6>ZJc
zD)+W@$?oGykllC8uwm~RX5qb$S`_x(=}fV;`ln#WSEz5x#mKskFCfr1PV(<w#(C^^
z)$3O8{gC+1dfH-l+wyK!8y<d!-3O-d?koH8b??%a_`N4jY3}{sCvJP?rp{iMZ)^50
zYJRaN%<aV9xk8KfwlH$<U9|R!?RUk+dsjAY-fQkS$M$p0BpaU3>uqn&>9FP(Ft-hT
zIn6e*b;+J1FU9Q?^EB<uETXLQQtS3ATu9!-V5(;oe0u8MLq1b%Z=8wSyJpvXTeo@B
zY{Tky?cu9!-#dBw*S!-Y&)EF^`PTO85*=GP6>Z!3+urPbu*zcZ(W6WEzSrNhchBc1
zds$aKu-TB*w)bFHzHOb+THC}Y|7^sZF7AD6^})7Ad+%PEs4!cuY-YP{*LT}aKXunO
z>Q3X{Gs-jfGQBvu_qfZ2y_L;d_Ly5Nv3X_Qx;H8&&c@Mvm5l?7zO6F{gYAUYSlcCM
z=GjhIP`bC*vUji3V$;1}I%@VBxbW|5bicFrUBt1y70Ubf?mc(kCg(TzKAF=gHV)yL
zd$V@_+RJ@n<KCEcy?d7%IoY<T%(St%#AN&4$7*lnwWvMm&hoYntM6M&EtqE2(zj{P
z#B`B;kD5De&q&(a+OD}}yCC_<-n+93cc0{nv^CC=+h^tZ(bm2*+q!?%OPig%D{Tr^
zh1x{h@!92yFS2RPD7Hzp`fj__{r?{0NHN<}Tv>ZFq($v=bW`>eznX5-uadXh$h&o~
z-TG$R;6m|zvnQJDZfRX?%W<jC_5^eL-nu)TwiiBg?$cORYiq@pZ>#)z*<Rx?j(ra`
zIri?j=d|~n=3!f%5AXLZcp<ZIjR2pmoRH+c<<@z&-O-hMchp+iy!qU)XWKN5y}>ol
z_r90r+-KwOxHr4haj%0eyKQpTZ`+?6*!L-2x3PV2O5XPWgRb3wuI$*maeDHOE3dEE
z=+6|`$60P>XDxZyW~#~GJyL&e?_FQwu=n&Q&Al7^*6y7;Ws)s-DU*$(r_esLsok~*
z<ic$~lnL1O?Ge~rKHGfnwb+$==QPdRQyQnWSGbUEU(M3`y&+GR?FuQIviG0)*WC-<
zP4*rMmf0)6{Ht}(=~`R%AK&)yDEr#R{#Un)x#F-lI%3yeNv^efo0PcsGAjMEedq7H
z$2-t^uVnbTy{A;(?rrE~+x^aahRv=oi)}dHUf8?isKH*1MQnSY`LXYfD^j;*p2}^v
zGV}P}C!)Legg=wEJ-oTZ_Ti*OdyA`n?fgBJeV?k`%e^PB#@Z?}N$usHBxbY6`o8V-
z|8w?EIrPTHc;))NbFvoLI0wzQwfK~4`$p&P-U2p;-PYE8*2@<&?dvw(yT?;%*Iub5
z?baT*|L=X7AGEiF_4wX<FIL!2ir8euv`%%eJ^NmpTaz01>IR?QBjC7mZ-ZZ<?Y%^S
zed~gK_Aa#lvsYe9&Q5Bn#J+o~)q9rN3GdskJA3aR-nV;t?lIc>`rX;n!8h6VM(0eM
z)~h0W<2V)fl|+izKKA0@$Kl4huVl&@+t^C+ec2ZnZ5O@WwKw_S?A=aJx9_dGYPL7`
z<J-Ob-zM)(c;#Xnr#E-^-^U#LyjE-2Zn507ck8c}RzGBGZ4EdU?)96PY8zN3Zp-16
zxR-@}^<MGnPc}JKyKOE81=xfiUv0b0`MB-(B_FI)7f!G}x7Ky9g8d#FhIt(OBJT;=
z?Vl>Q?|b|0y-eps_I57bw72S<+Fl)LJzF1-$u>d_|7|W?9JTdn|Fvs}fT6X@ZsC36
zrT_P`$@be?OgV1jyK&#%sU10cc{Z`zDwX!`O|o6QcZNE%-N6opeXova+VURuvK0w^
zZ`+ybv!|fw{;oSNpZDB<8@jjR^XxqbUYgl_2~4rIRLQW-INfbybNa&G)Lij>TwA;L
z3ct75Q}uYRt>fvLdvo?Jw@GHG+4GBa-mb*GkM~-<m$YTA3%0$tO?YqlVWE91BDU=n
zmA+})yZPwewM%*S>6IL{H5ZKAd#1j3Z{?R|HZOL*vuU<^v**G5Gq%$eGxj>a{kXS&
z&imapEV8zVk-KezCFj}NMLAnFo=md!h;O&qGw0FHnZk$n8nGDd)%E_lS87+)-ZOUJ
zZNuKpv)yrg^WKJf-#rc8>uo!JKHq)klJ4Hz50SQN%G>rXaB8!%-5_F9nl;%r?#gBx
z_G0_JUu&N1ak(&aS9F8=zU!AI_x)54*gJWhtxaFgw7tpmmRfDgl-X;uk9%*k!EtMW
z`}6kx&^fSo6)%%*$^zwm9d+;b)Linh{m3Y}`(_`<o^)H|z2?jp_plqEwe1l2ZQF3R
z-R6VAHCuhR4YsGwXznrC7-#!;n)L2#zmD3p^?$VSuxYW?d@8m#@<Qj{B$m~Cr!eMO
zKffceFW>FtUY@HH_qJGF-MeGvR9pKju6@%L7VIrOX217~l)ygzX_xo<d6e%pdbD7#
z+fV<!uGjhYJnC-S>tQarcT>IRUbnl;_C~3{-ODe$XOE89roAss-tBeiT(ft|8L@qF
z$?vUyR_)$<mwVctn{Ai(YCe$OC&_er?~+@4_I_h3-g|?M&(6WvdGCw;)q7@LVcvIK
zuYd1e-Zgt(U+3RvF|XO?Nj0yn+zuC8CX0?ecl0OhjZW#<^Sf-1?Vj*fn-e!AZ2ky0
z+aw$--@9nB)V^EOB=%PQE40mzVz;%~c-VG@WwGrkEf%XCN%nRNtY7WDG55>fRX?un
z>Fi#$w=P<1Z+^Do-ZlkZ>x}uqww=~%Y`33g+}C?r!&+|3G284pDtm=aF0nnay38hP
zw!Yn*S)2Dt`nB19x0T(q=Y+DYSH_&Zv%*<yPxy1$Za6AnW3|i6F8^omUPTuUTLtZ1
zyM_1N*(<@lb8q<n&(<OK6Zd{CJh3;v{e*4t|7m+SOg>>VWAFPtzE5oS`eYs7EBs$z
zU*_JGyVqS;-gjn#z`i+er|*%Fy0iDFs^q>YhLdb_!V+!Qyb9UtCM#)M6)?rRG~x1I
zd-h4Y>*@ve^&Ixwvp_M)){*D>-UDY2TVHGRvOQ<ay?29|&E6?D-1Z(gI)6_n)4aX?
z=GD7Pbo*^yKIPk|IZwcLj*^(|fkhAZrhI1E7jd9%@9%F#wmd45`?v(sZ9m^)wi6KF
zYP)X2XWJ>k3-*RaU$W8kX5G8y`<cBA&Ik91e0sh2Z@{y?^}L*Wtq<1kUAwp2c2Qr2
zt?b>&d*#)b_XdAoym#{XnR_J~@7O%a5!v^k-EMa#3%A{!lFxf)t(s-?iIZ>N)@K{{
zR_V#xMFwox`~F+A?M%(?Jw1&=wi*}1Z6Z<{_FOnGXZvVI$DWwiAvQO|llM;Obl$z-
zuIyg-4?_DE@ObUH+1<F;;a{!oaz)jB_4_pKo^|}(b8DOAzWJHb`_8U9W8D|^VQ<`)
zb9=Pxb9VP{yly=+^x<B&38J=%fzmb{w@hreG)%P-5fHIkRn232XyP5)7d!i{dn{G0
zg#ypnN`BnB_j3@x?coGb+rT47Y`;lK?=@9kyw~KoqHV;L!?v12Y<4=PJhlsOtlwSm
zX3Fl$@1=VqYQ$`t_ZQhNX{p>JBz$$Rhs#FWZM!P=-uof2&t{FWE&G~vd#1c(+!N%J
zVRK61#%|4wt$R0mytCQ*fNAfI+sF1Y%UrV2Nt|STo%P_}65d0*r`eg?Cft;_+tHq|
z=W@_Yn;BPPY^P4xu-oaF)*c(94BLC#JolQuh~K;E>;l^zF&4H@7xUTuy(O~GX2mJn
z8NO_G0RhKs_PMRF<=U)jSO4PL-nPqccW<fRw70mn$>xaq*}ZY!MfOSha_(_>s<_wa
z<kr2*xZL)He7Uq&<@7<@Pb&oX&Gx%&8_79$?|~mKdp@)3+HG08Yfqf!dRvFzMSJW_
zrR=g@%JyFE;@ubd=EL3>Yb0zgD7o8a_NCg0PHC`t7jkHC$6<rLJJN;stXo!MQ$6MM
z-pFqX`y`T{*~I8^?EBG?zPI_>o4rk{{Cn?(N9=w4XRA$?j{Lqisoi@S1pn@t_(F0Y
z<Ay~$`_@g}yS+NaHp3))PesqUJ?C;x+g8q+v&Z~6)85%ut$T`DmRiUBQnHhtB)T`%
z)?x3)9j<#D($3jT+0wE1-rI|N@_v}^X%?7l)0$Ll8@}nL?SA9EyKlVi-}6hhey@LT
z*<O(#ep}&^XSRPOG;H;E8tmm*AZqt>o5tR6)3f&;{c~*Z?gNZ_67NdylR3>`yGQNl
z-Veg__sF{awSB_Fy6?>LueKcv7wuhsOmLsJZHcYH@x`_t$q)Db*uQ=6Q~o!5qHTrj
zTsk;*zYBb3BRNgNW~rszzJqI~+Mae{*_&lLXKzMgrLD_($-OQmp?m%6mG(+3j<J2T
zTE+JCuP-(suUGBOy)3oQQ$uv$^Ls4&e*J2;;j=!rr$2Va-c1*F?>#(8b?*eu?%mRr
z7xpsB-P(KMoR+Q59__uYM_=zvu}Qc6=hkh@l=swDwPDs?mulgC=EWAa-wqtHZ7Dgn
zH+3U}UH58kJDJbBZQXTs_byz;yziw+^xlMQX4`~{UR!4iE!#PN#rCxnu-Y`Nme{*4
zn{QwKz4^O4oD25$R-d(bte0RbJXzH4xmc#{r=1h_db|+W_jfMezQ7ZfwgsPF?$Nq`
zXRqO<2{sSbEAE#6Kg;?{rsTfkA7@#s9Jp-bSP{9`Agj@qPjjEGP2O7DXASJO^)7XL
z%oKOqR>T|bec9f(cT3FmJ;keJ_pbinws(ia!oBT}#P$WZSM9ytsJpk>r*-eg^KRCa
zDF%D@TW{WbV8^b#)=SvzRP^d?C7+z%EA#CCUX}o+eT(L0?>&{Au;;?oje9fCo!k?&
zKG}w2rOBQIqj|P(&fK(N$jG++wLIB&zb~7u(7uMf&U4xJu0DHg_dWT!wn94`Y#l!F
z+32-SvG{QChV_XeNgEC|&AlIwmG6CUTz+rbLngcEj8@yzJ$Lq8e&MzE<0a+20t@|Z
z_cTtmefc%biihu@ZIle_uC`x|HU{6O+G^O|-_yQ*%ih9D3A+Z*Csv1N_3V9gJlJ;q
zg5Es|@vE&boT%Ho=!%qGK{l`LwC9O??>yXNt8rPvR^`u?y#=p-Ss!^NZR;`hg6;I$
zse4|pFxlJCQDvLBNOiBs;f=NrH~iYurP#3d%tJxj;-qc1Q)aHT4gJSy_aI@n)y<py
z`;ziD+1zVwu$|H{*H*@Kv(1tHe6}jSO?w+ADB7-w>Dt2~{nm;}Tgm!CA;VsWtVPzz
zH|=Z_R#@)g<Ib~gOLDb6a6f5JYpccHt+7}42CnPbYjtae?d7`uJ&H$H?`^obahJU_
z@7|fBwYDb<1@^Lkd}b3SvBp+`=lJdiPxtP1+3Re3bG6>yE7Q)~atLg(Jumut@49uo
z`{r)p+$-@`Zr>-F?Y2{=a__rWcx10y{8`(qJIid%j1+7)TsGOm^?lA>tv$N7cNqF@
z1M=_gnUb(#?*^7ryMN5;v0Wf@*><~*_1-O$LT%r0tl!g=^w)O!hh<jQA6slUr^wj$
zhH~%Q^1XGhl60r7iPUaep@&_zeCOWSUcR7cw`=#_y$VWf*59kd?Vi=J+L_*8wfFg-
zkJd-Nf3exF$*?!3PGMh@+m^k~M_Be2sO`6TwQj=R2>}yrCH^tkbhKsdxwU8M-fagP
zY`dJ7*zPtL-W@JtzIV~@2YW5NC)n7Qvf4hK%wl)>r0~AV0+qe|w;$|1a&E0{gS(FP
z?zxtGk32KlGkIIME$cM_8%<lreKVdV?Fn_UwvCG3vU}5$vwJ&~qW3U|irIyk-?zD-
zUT3RtOxf<<PUd|Is(O1re_OS8&;I*+7JgV_^KsfIo43x4`;xbD**Q&Pwp+ceZ`X_M
z=WHC6t85>=oN2qPb(O8E#7f&mM!tJx?#bD1Q|`32eZpf~FUPz0$gAc(Ufy$UlOopJ
z{!s?i9kzQjF0S5NP_*0j7OTWw)&;_L8@QYIHuTQhD|Gtw-d~c+dt<l{?Y(lsWl#4P
z1zWK_d-l%QHhquB=Xrbke*WI;@i5hP#qU2hn`+qYj<U?LQShE=v+eB3y|tfD?_C#o
zdhfHN3-%`KNbGxle&gP`;ZA!iw0GHPiM#HtFnwcl{LaQbE`hDKQ;a2Tv&<$~ulu!R
zZ}jd!+aEkkHV1w!+*_pmWUoNQC!129Pd3kY>Fq6k`PjD0*?jLQ`PH^pv}A2tk{I{i
z3!Q8m(f7gT;hK|s#Le&SvDqlR@4r2pEgSPB+Y?D*cDtn4+H&OS+I(QOwdI(<*VbU}
zVe5NBYxag@INQbse%)(%H++vxg}^>Zf1SN=<?8pk_Y~~qC>F9kZ#dCfb;->=Rn@ch
zh=?%l-KyiTw_7>V_DeqdK80z{d)Lg0v90$$Yb`Id-{!%hQtPAj5B4roT5NM6aHj3F
zS5158R<qeXnbm4@Oh;p{DRasm)3potvOJfu<@g_N8}a3!ZGokitx)G8TZIe1Y%k8f
zZu_*yYVT>EcH3VU4()wZ{n_TM%I>`?%q#Ztte9X^aOaP;<-R3*X52YtlV|9>_sU=4
zy*_U??G0G>)#gJ+)}Drvqc-cMrT0B#SZkBE)YJCEg6Df`>bCFsCM~qDp=QUfKa9M#
zlh-e@t+M@Tb7hC6Z9u`!-TpzV_MF@KX0OI3S6lb%Y_<{wjJ7?meQcYKGVe{XVzXVL
zEo65?t!OvH<%c_$-j>+=r*_5OslCB_Id-qLt=C_^$LYnky|eYE><y}3yf;~R(Vlrb
zqiideytkEp%CS$Gk>74@9+$16xct5YIvzG#mDu-sIB&OA;hDMj$<+yaU$EKPe%X26
z=5gqaz2X)t_ImB**|(r8$@cM=;=NjGi}zj&-DZ6-W$~V6tM>0LWE8jaEPZVIVlA)T
zo!MQsi!u(|-r03-Pl+k}-oix!cESIi+E(n?vPUH$Ymd0|7TZ%t8TL6lthdoxJj2$}
zd(GYrHydmVoHpCWrvJAMxe;Ys(&cX3=evJzy73>|4<2EA(;bfNJ#n_fCcxyEP4DwN
zHXbb9dt4qe**cyP*tdzDXK%c4_TH{pTldt|@3F1OJ-Ig{>hGRgN9XPR?LWcBqGa=4
zgUQ!yxR3tW`)ktSJr%!A_pE7{WUEvky*DF?)iy*U%eFuItF4UrDXUFeh4$Lckhfvj
zqiyT={*<li5_a2Zj%RIWWLnzfwolo6XD#R6tcVM?Hlm#S3OU!>h!ifg<@vOF&%NT$
zwoK{^_I7wn?-Mt#x7pl3WB1dS*?XrtHSFDz%Ve9Xdv&kZspYl_W+k>Zv2$$S?+ds2
zWZ|^;g3uQ08#PR}^Lx1LY7M$=|G!(eS47TnZ$x<89{(jm``FnzY__~xy7$fcGkX`=
zE#Awuf0k|YfyuU$-(9m|Ffz5~xF&BKru%&lgW46#<x`~Xj!e(o<9A`1t+cp@?WJv}
z_FlPrV$YU6jCO|1cWgr<PuSi#yV}+uvvse`fvI~>2PN)Z+Qe$7y?f5yv*!bCUp1Yy
z-Wu3ulVZcM?{cc3?Hl*WdlnttxHqZe+8&n`)A!`s3h(u~kY)Qo+Q8;b9KT&iz;W9G
zrujCnJiGS3VVbbVeMQfnb$^8R&HTo)ucIJwZ<hTW+j|{7wh{BC_83p**_U>*b8moR
zwe9=qhPDUPH`$&~onpgV-o01q`m((a(|v7kYxLVX*ooL`xL4U)dL{1p=QF{!<bIXy
zwMU7zF4dd%zUg<`yS^yI`d5CYje@?3wL@f+?FE~IHYZId@A-17*j8b;kL^0wzP-=P
zx7zYuSJ>-bx_|ebuMcd`@(J3!OWSB$v+mZOdAW9bHylW{)qB=%b0B{5-aUIPY;|UT
z-<$t*^InPEX?vy~WZ(C9$1&TM#+9~{>{x7@Vqe;Z6!6+<96z{szDwHPfR(<sSMzS{
z^|-&?W<h-Y?i-I4Z8aiIZI|!N-TQ-AV(*>2xwhBVZrnS==8LsKKK~w81--ooeE4?@
znMm(tcr|5DfVS!$m(|95w^%9IE|!t7W4)(qyM9Z+-fdAFb_*T{?CJA5us7&fkL@b<
z?R)oTMeJP_xXbqHw*`B4<xSbE@Puby>24<5?hL!V;*VtPl=C(A@>yxxJg^SiyXNIG
zn{_M=wz@Yy?#cYO(B?*Qr>&Xhs=bTXHSB$zdTQ^wu$sM{d_r~-M>=<{;Z55s7ZhMy
zQ_Z{g$VEfz1?+eCxF2w}{WDL>_J`V4n|bs5_nM`huwHcb`QE!>A9lY8&)hTFS-{q9
znuFD;3o(1$yKMJPX=B(s@9cD2hrf4h;+_lHy?E8WCn<cfZNrK?);6}Xdl%gAvHhGh
z)i!B^++G8bn|uBjhV0!OeA~AAkb|wD&!IirmJ00?eUiHO(es$SQQ;G9*P2J|jkw=s
z%PP0rMpshaZc^&!Jq#`PY)n{x+L~%U+bcYiXJ2WE``$Axb8J5uN847Y%-*|YQLt^?
z2`-zOPpkIsW^CL0icP|H+TwQGncg9L-{)5Ey>Vsw-WA0QZP}g+*!o;|-P?Eh``%Td
zn{B7^H|*)|l-)ZoX5U^r%VoCbbS3sNWv<;T6XCx1(r3oK8CMzY-rfGV_w_p^Tb=Ls
z_bxd)#a8jw`aL(FvDk4k7w-wOy|8yuyT_iF0do6hv}@bUKErQ&)bq?9l|2f!4vh?Z
z%2cHH`oEFd8-GmMw&nV*y>BNb?)6=H(snTyryc)X&AsRArEDdpuHT!yRe$fY#EpA|
zjArd&{d;I{!rwQ$o9t%U1Z|eEyT#tUSMr~eZ9`DWo>LvZyPr6Gvpv13!$#obh21=E
zezvz_?pv44@8A1ziT_@OPi?karuo?xuyfih@Mhc-xLwV5YSr?+W@c7<mxZsfRk+Qw
zcSqG*TUoQyd$)8Rwry0|w?|2Tr|ry0NxKg-6>T+wZrCLJQ?P9`T5daO;uPBlQ^jnw
zrrg}izs}W`ZS6GM_X}6ro)=--myw)j`?z(RjX~Nm>qR%(ZFSGD+3Pod$=+s#o3;i!
zcJ0p0nrv%xZp&Vqw1u`ayuJ6t3YYJlkTKQTV9PmM5lIKz^;w7ZI`3Grw`<!JTZ2O{
z_r5qAWA!OXc(3x!Wb1eH9@{e7U$lK2!eb|}m1FORPfKiXv%K1~DsPhQwl#wL{+v3w
zS0txvuVcl!y&lgaY)i`(_deKlanITQM|*!dc-dy0uHPNd|Ja6wQDILLm!O@|hNU)k
z1uJb2q@A+*qQGjm_+9f}hR@e_FR>5b^C~dXwmJULUIt#bUBR=T+3I|@wKXqQv@7>a
z+WYqN(Y;rkHrxEO=G(Vm^6%YEnihLSCx+X){o=D*_+4=C%TM?B7%lv>E9d5X+ZDG2
z_EjF3zPE!jdXEHahHcW)^u1j9S8RDq*V(T8r?dCi^Yp!2ekku1uzIpvi{Zdt?d3gt
zPpp=-)hT+l_l4o7Js~Onws*=hZ4VpG+`GX(*j8uZ3fqlZD{X$xvEJM9`TQP(Ck{5n
z0lhYrrL}uC?uFW(^WC%8`TLYT$5S+|)l8S}ZOBjDb6aYjZCvBJJv-bd*zUKM-uL+N
z!o6W@ChW~SthhHfYoe`#@Hy)T;TpDERHgSxT$pF$vw&xB#WCK!bN?vYURv_hX4_Rq
zTONmY>wjqnZTa)HY!|Pcy0_k(XYYTLZF{XIH`>~Iwc4t`726wI%VcNzP0zL=(`MJX
z?(chTO=|WY7FxUabrP%HtG|AG7y7=jt*GPLbM(S?TLt~sHYy6twl502ZBsAySbxx0
z*w>k<Y}XXqv)A8v%3jq9@x8C7Zr{6DhRtpVe~#^<DeU`-I2rcxf7agnW^IFwkIOn6
z1EK!CcB&uuDnFWH+p+hI4Fl`XJqJFm-)k?@Xd`YeWoPoQ!q%fKeb4r~Lwif!-`F!F
zWtHvqOJDX}xG>fBxFWaRq`B*C`&hT^jlN`PYr?R@Hsakro3`_Z_WEz&+-t3+vo|6{
z(MITuf~`xZ^u8k(ciKL=Fv;fE?{zk3-FNT(CMdcu*7wIAmv!fC-HfK$o=@Mjmtnh(
zm4&tH9-$2@Z39w8_bzz+evj4(K|ABDz`eir9^1P!@yPD1b?<DahRm@lm@8p>=#A0d
z+tHnSUmR1mF^@gJSAXt_J%1h?x4kj<qfLoS#h%k4^7|N=%B&}tXxm<RDX~W~P2TQG
z@C(~H{491;SbA*)wB`4v>Mq>d*Un`p`-po_YkcBf29|QGp9@d!Rj6p&&FA;ScHd>b
zy@&Kq@8RJ+WfQUZ&K{XwLtDdhe74IrGw!`~{o3ADQkQJHIcM+H*~7RusKUh7L#*0H
zF-p!hV2-!VjTbq48UovGEdG?)JbR_Qm!)#A?eZR0Tj!Za_q<k~wfFJWZMH`|Rcx)?
zO!nrT`fOV>)7f^}oHe%Jlo!}8@t?a_>E-4<B{C+qaXCVJw{+&)9@}%$mW4ykR^j%{
zJ?39DY#7SqcMHwDYMXFHVz1a{p1rl+D*H4djBI-rUf=V)W~c2LpVPL<0oQk{A9CM&
z=%@VND{4iy(`#b(teU5|SD|@{b*1fU+x&Gq_MMfTwkIR~-QGPb8|)RH3GbUxsJZu-
z{hED_Usd-B+?98b{{C%Wn4rM^zecV00qWg*^KZ*Ia9gjpy?k}uUW;#sZ4U^a-(ytP
zvCrW^)V|wyUhRGNr){s;?ESVJf41)1H7jP%iDNtVC#?Ih=Zf)UJ2OsWd+SZV?WX8V
z-p5{bZtwK-oqN)|n+|MkyJxdHOktnP(w8=eL!)*(|2e()-6zAnYZ)8%u3NNx?>zJ0
zdp8^uv0X8h(I(4U!a>@7yG?sqj7?FD^S;@?j_tWWG5vt&(Wf@MroY;IMQ5J9iiG_B
zbAE63%9(%Ix6|3({-V~Cz1j17_N4!txcB36_Wj-E`}Z2$d}FhZ%WQAVTN7Jtbu-)F
z(fT&I>IJs953bw$#=L1yvxUvR6VI;M=y*2lZ&*;Wx1l7$DyDGyzB|r8_P(%<+H0iO
zw9oDOro9|OOb5I+%kEt|y~w6uub*wCz*XCZvorTDQsvltp*CfY%<30=ZgAhQH+JT-
zYdt>6er;Lf-k?z1-9H>Y?!D{Ww|AAm*1c=CJlK1K^}t^Jl-_-f0SEWIW1MD_5N%=0
z^YqwWrphCG-<zGXHC#M*Z+gwiy(i+f?AfEpy3Z(R*51=EL~J+2z1w@EYUdusWxwo<
zwsY^>%3-|kY4cjEYdtIXgrDWM^KdS+-q^XrZt|bTy=$hm*>2vy(f+>MD_hBxi}qZY
zByBfi>5g53vlrSlmwes(dDf4;W>-A--3a}<$9`M)-iGM@eTGji*+^_sv$^~D#(veC
z4{Qr6w(OhuH-B#w56^*&PcHix_37_pH-BR*IrrOMj=$pjX0S8Z^jiG1&*js#owfb)
z-kPX8dlSya*c{!Wy-)dxjQ!bjp0@vIaqSc5YT3u>vT(1CZG&BnSiNn`FX_Dvf0pj|
z)|zV*8!x)=Y<bIGVFj6e0-IJ_Gd9NVU9f)nUV+7@_a+3avp$o&(1zu)sy*j|-+Se3
zIrcpeoVI^9i-GMoh55GSTUGXd<MOaodpO^&`i1tsd+%rLpJStGFBXuypKtlfeOz2u
z>_rzZ-4l_rbkD9uJ^RxpKejvnjcM=2w*m(W?&#RWh5p#9py#$<e)WA@{}ar1LUD)p
z{doJ{K7MQ2eh~?!z2a+BZ2ufQyLZ8tU)BQEoz_!By!JM%Y1yZJux8I*{abr^*SYLI
za`=I5O0I)#s-eT~FsbhSI<p?`Nidtd$M)2g{n!31-q&!@cCX~Gy8V?Al6GHmIP7A}
zciWs*Q9YoOn74n8n%h2wf<rdXUTxVgqPBFe#ob+donOq~|Ma|=!}^Fu+sY;T_a2q+
zwf$htVe?bk%;wqcdA1f!Ha2yZBGz3`?(7x!n7yxen%us^Lk)XYbbhyO&~&x^9xJ`K
zR4#7cf*kd|r&+CSFD#YWQ&c@^@9i|6eI;3mdoJ!~u+n*WeV@jLFME3qs_p+G!)MzU
z-m@=O`SkvkA1~RcnIExV9ni4%3G3{A*Iu#j%bzlP|IOzT`$Pj+_D#-Wb2!6zW50AX
z?_SBZ6ZU=G9c=%xhTYD?q<x=%bI`uo({|e!?)BY$Qh&!@!(IW~8PAPuIZofRIj+~S
z|IM_febWLO_j*kzxAn8*+V{Oj&~8ie@4X8x(huy}%&_;`tNuN)K2P?|tvR;$`Ib=o
zdvDA4O__GUw#Kev?=SbZy=86F_vcQ{-sj%uxzFLw46Al^{(Wl-)b@%mHnl%^;P;+{
z-p0LOF5KNO_tki>1ADvO8okx_+cO32l78_YFjHdQTR;EHzT3M?_ge1#yLam%%l$#i
z@9y>Gy<>a*;#XUlTNC##f4FF`M3bPk(3^dG&wjDoZ*bq;rqA=G?FPSuJv}?y_td;w
zuqXEFOndXM()$YKvTY|{*<&s8;N~8Mh$Z`8>8aa3xxHt9DBJoy>YD`iKHNKDZ&|@6
zn<bx*?|r{}_a2V0nR{73-q=6i@}8|q;N?9&qN{ALo_n+Rtju%!hu`+@WpW9!&sUe(
zt9_Px-*TyYb_ZT?9=MVJ!lv~XgN?t>$^BcNx!CFjY_@%4ecV=CYqCwvgS@@^XJ_sI
z+pyRsBEZ8ws6=z`makj(T4_n{og>P=Z*Fde?fsWG_O+OC+ZEU<+UpjC?NK|QZu_9^
zlkJ=&N86B?Zo79mu6v)RnAtvfIMM2Vrm+2$3?<vA=Hhz|<R{q7<$Js*=Bk<PyQW)v
zec!y=Gim*At6YBW0|(@G+wI(VYX5?TH?8j<l(aU@R@}EES7(3pqLjUcvsUcM=VRMv
zb4O(V^ZvBGGZ%N*6uk?vE>V)+n{<BVzPd&8_lKWT*n8_*+`h%fuh~yDJh<DdVB5Z&
z>I-{fx2@gF$o}27_L|OK*NV-%vtP>Y`&9mQ&l~f?z2R@B?ccEH=-xRp)AufOZ?UPE
z-nmz-P|YTj<<wrgXF?7|{2qIkP7>R9^@{sGd$+m!#Ej?eKjF(^&%Tqx_RVwUy`ASw
z_w%{z*k83G!+tAMf^7<4^<JM|cAIM)hxXTOTC+!UiSXWSe2eyb#t0oK4!E)R_s)ZR
zPd#_qS5i9bfXMxiduJpY+B``=XJeK>)y~&=@%}4auWUbmP(EPxdfV=xUz+=pf&%T&
z@^bClH^<+4`PQ7h-ws*rd+XD8;K`>IdlmPZ+JB0)-y7y}c+b@7QhU$zO4w@fKeXE2
z#lL?!pZ-1rwrSRU&zA2^o7%r8q4t8ci`A|Dk-4gS{~15q_wVb${hD9C@12s&x<_qi
zyv>`-OKtCpKeTLDIlR~OG1orveuaJhCtUaH^nSJ7W^v5Mwd(KQ^`CF<Rg-$PZ_WBk
z`>f|au<iM-XXkcjnQi+jue}xZJO{q;uHMVGPtZnHj_m;N#i-py+U+)f9^bL)+2wBA
zywlgdl~H?N@~7nex=}mq@8<ICd*ZUl{#3u`{v}~Kd-JzH-ec9Fz4u7Qs{NNz7wr2o
zQ*~eB_UU^cxo_P2c#-@5t#7#Ogk>xo)}5%?Yj$$-{s_^RdvB<)?%yBFykAvKZeLE$
zo4uzzTKDm$aqQ!LF0-%7OJHBH(%ijLQm^*b#q-%-aCoqnp{&$quK2#a(TlqF$V!~p
zTYK$}-Qu`|HpxoI_nfZx-G5kJXm6CarhWgiH#UdA?ce)ErFU<#jqLuDOZM#DEg`vI
zoAd47q`eV)w>aG2`>1u<-c4yz`)5vDx_8B-xV_x7|L(o~`{mw+im8@%bKY4S%SG;8
z`AlK2!1~2|nZ7^TccUfS)??TFedW4*d*9nK?fW}(^1eT=_xFBN3fOzcsDGawpMdQv
z{w4MX^M&{AxDmD2bKbwb<%jZZPV@EI{8#YXSL9!1!(;w+&)J4f`^$5e?X7ujzTf}u
zWZUc=Qu|Ub9N2SzviDxW+pl(KbBOP2{<y<N>U{V9%aY7{tF|xQv*?!YzFA^id*9@V
z><awayEn-7*xt9NrtY2GFlDcss@>injfMM`?|-y+%6ry*QTIG;=GocVZ&fSU_d(;A
zZEl<E-j}VCdl&F4I7spw-^<O#VaxXV(cWG9O?#R53mmB2d~&a);I2LHUiy1E1E=p{
zwR7G-YpdM8Hye3u*1liAx1Zz1-m^FO_Wiw-v@cKK!(LB?WqZ~Juh^@(`ODtB3oh=z
z&Qr3d`Q2nY%`*)9^=`H9J6Zp5Z?xR8y{!J+`(#`t_J!`|+kd6Qa`!{7ioLUUAKe@M
zGk0%^N8mo$6K4CRDx-Hxx)$z!W{_wn<kW1Vy+P){ySuCRa+f9T&5N?y?do)W&*4bU
zebVV(`&1<rZOh&q+<QXJVej`ZUu+f#9I)Ond&6!%la+fUY<Tx~t>)Qx;^cH2Clkv9
z1{Ga<r_`zL-RBpyf6p?uJ$_1S_MN=Tw*S`b8~ZQLlHK?5>%x7-`rZ3gvODbm-w@lY
zK9^^osffSr?~7vlSyUqSL<EZMzj&U>Hhxdk9-qiH`{xG;*qpy4xbI5e+5P{#T=v&7
zJ=%XT<HugZX}9*?H~hbMhvS94Mh`^xmhRWudne%hzFkVs_Eu*x?|ap8--i1XpWVq{
zLDoDQEcXih{k8vx$+A7Wmv7kn^1O+?<mB*u_FJd#)hXxPH^rmRCSL2{-k^}Fdvh#m
z?WatbZ9k9a?EdKOlKWUJ%n#__SiIM?G;;r@E(!b9;;-zjeA@Q*w0z$yI(^oj%_p7r
z{y3h#=W;jmzRiz+?tK`x%Jz`@DeGfO4YuO(-)wHN_}RSKo4M!pJf6MkGy80O1B>^C
zK9aOupj~0}gFj&p`^{;47A!8^JAr+M?f<1sdzZ{>v;TGW#9n?mnf)Cb59}?we_~&_
zF{}OUJmEc)<PPo^-s!gQ%GUq;<+_FUaxeV8&ph9F@1=`v`!4mf?W@1XXlKK@Xs^+F
zp8dQ|2lr26U%2<yua*0@?#SCG#I$91%X5PR$5)=->$gwUw!=Zu_DB=gJ}dWZJI7lS
z_pM{xxOd|`W?Kih+4eslY~8-VW9HsvOE?{zrbO?3|M1yfCL^8$-4A;Ao|x3Uf5)+H
z`~KA%>@C?@YFlt~-(Gd*1^coKx9$rpt=Sh4qrGqKYPNk*v#0E4>H51*T)$<HAz$g<
z!lLT^yH?%a%aGN*-|rCfzEgpl_kRD?VjJniu}}1l^4{b!!F>v{(R(jF-neg(_&w_u
z!}q&eo-eYAInKLp*Yr+X71m{YjhpXT_oPVe`!SDw-&AhL{XN0WHZHMeY}b7Hx_8ER
zl>>_BU+!JKJ96*qd6V|qm9*^r^ytw3xQW;IM*rlnKkBx2&;HvfHj?F{`#w3$*;~J9
z%U(b2guOB6&+n}a(b}Ia)V}AI`SHEx8rpkvCUxu)&raKO%bj&!yU!;3^J(3Cf4gnn
z`-rFIz;fH^`!~DK+|%N(Wt+;dY5$rO0ow`Q=l90ObL?Bo*|8_-xYyo&`Oo+7ZdSB?
z7m#l|g^6?DN)B(^sddlyuG)5NZ#pCE-pQ4V_6JnVv-wjNVJooMY44_o0@iHP#P><$
zu<lOKV6u-rrDXePnvgBy-j{o7=C9eSY&LC=%n4rG-E(BD_a6AWSNO}Py~&b$_Qo9L
z-WPkA-FD3_KfB8|e*34y=pHD!JJa@E<qxZHrH6Zui|yJkctPFz^PSs!g({xy4dpc5
zzrz3JUY$*A_Z;_k*?XOlW8Y$4dprIFce`!(%=S61S+`d>jcNZH)xUecOgyvC<A3y?
zBYFn=M9jHuznoIDk+f&nxA@8WeHE`a?0pcdXgf>Oai7^&m3{i`eS1A5d#t~2V%R5`
ze|4|q^;Vn2i#zSvy>#|X+xdC#Yx!CGiXztTYxhgCe|2$*JxiYWzS?GveU3Bq9a>ZP
z_D}nIZvVqIuC^X;IQC1ud$CuVd(-~7(sz5evYyy$$5y^?-W{v`hwjYYE1SW!Fa5^G
zedQJ3?DW{UtUaVz_MV&0wXcOC{eba=>HAYlChk#M@N<9G!X0}!JofKppLA`1-DmcF
z+WWrlWwf%eUAJ@kzPS45b~&@Z?Ul3X+yCdA^gh0o?R%%VtJw=`-Pp%`{<GylOUVNf
z%jWJ|V!n0n7Pmm#`wzwUEuP}G*Y>#Peq~P9ecSdH?_0z^WuL;<>wCSO_w82wUt}xt
zYvVrSM_GHs&rIKI*Iu)C%{2La&)8k|ep@NKH=?m@@7Ddt_bxK*-5V*byq9h1t$nXQ
zN$g|X^>J^^k@b5vGpp<iWUaQ}y^(*PxYpIZ&aMsi^CQytOcxN`cYDUJebG(!dz-Sf
z9oDToVCC{bU~h5d9Qz09y!-g7l^qtGZ`i+Jlfb^`=fwA2%In&DZ^I<}wRWugzC|<d
zV@_DN_fh(vz0G&t+v+Wj-e=LJyx(?<!oHQ~=hzlr>a%-ZEM%8iywyga_V?b?Oq{mC
zk_YV9Exow+YP7$N+|BiS4exvHTWBY+cWtqN&B?om_Uc@`xQ9(};a<6W?E9|n(%yUF
z>TA2!_SJij|C8Dq;_$|DvE22&R&#9kmi}G5*R6iv-W%pe_iH$@?Ay6RVqaQxx1E+b
z@BY-T1$$3-sO;l16}I&(S!<JVQGBn0fz;k7({1)H?&IBOcEQM|S$*N&n^#}$`DPqu
zTc0**?{dpxTct(!_By*QJP?(;VsE-&i_PMN%{KDdQ+B_c%4!ofW7R&vf<l`Zejoe(
zBRu<r9^3AnzTv}O7Ut)BlMEU6t<ku#w>t3hUd|jrTfenk_H4}$_Wt@h*ETHC+4gTu
zlg--OJ=Q9Y%lG!?E#9kjv3hTY(eu5=7PI%yb<MEx|2KCp*Js{+iJK?vos!bFS8?Xs
zy*4^e_gH`1X>;|J%s%hZbGx%vu<Se8zjW`pZ{oJQooDZrZhT^QLzZoS)YV!0CUXky
zV~sgzvrETr|D(=1dmTz^_pjmA+b3hRdv9pMV*76b%6spd9@{I`U1$5#x_0j;nX7hd
zUN`T(z;({%h~V?RO^;n|CZ6Qmr>~m2Z->Wno9D@LR=eguw^NK-x<~Yak^Sn|TzeN(
zUa^ZjcXz)^sL?)=0}T5v%_!fyao>x*KQ~?9EBHCmZkzc1J(6!U_THajxhLzR*50+Z
zr1zBvPu;Jp<G454M`+(yPrm($S334C=w;kf6EtU!dcw!O*Os&Fk8~5VUQr;scZr$m
zUNuIwy~QRzdp#8P*=nxz+FRh1WpgJyVeh;>t~OWi%i1fvnYPc9;rw0`vE}<k_wnov
zk2SFE_Wxq5w1sise4W01HMuUf?|0<vmzOxYU#9%FonS)oeu*^A1Do7f_E`zI?@d4C
zwJ$+v?mqRh9X5YtPS`*DJ#X)#RVVlAY-!lP_xb0&c5zDkUKp&|KSA5zK-bC+JL8Ol
zdm|nD_Z|%E-P?JiZ*SJUMSCCqc(XS>d;Pu{MKAZ3Xl3mS*5TRrdrA4eKiX>he!W<=
zS5V&EZqjLqy&D>r?kPOowco#lZLbAmo6RYv(!H%`YxYX+xw<!f3G2RDs#opR{U+}>
z-OXq3e!IcubR+M9Gku}9zf{Hc9&UYTbC~gwZQfsB+x1W0+ho@i?&Vm=ux~qa;_l65
zclSET9Ni_d;?~|NFCzB-V%ccdJ$J+2qilEgdb#rMd$x1t-j6W{_Zl*vwh^#5+c&9K
z)Ao;H^WHhtR(lNIu<W}TI%{9la>jj=v)=3vF4x=3#_wQvwkCS-V%66Dv0<Th6VvBe
z_J=RGXE<=x=1*6-z0)W8eNzf!_s`v1y61n_;(eP<?%6e8F+R}qTFiD^^)!2()BgL9
z7~kJ><*dkl8y-d*jSICl9!ie(t@fRJ3xZ_#&05-RJLindUbm&P_J(I>?^7t^+V?Pd
z|DMv|Y<uH3o_o6@c=nag+;1OnS!-{=!^?YZf1a?{eDHOz=rq*>9lMt96IcGUSLRaH
zUYo~md-Q~V>~$#Y-1l50cwb1T<i5Gv=I&emb?zR6Y~};5k<9xtRde_0eYml2fe`Dy
z<0p6TTb8uaUTI;vZAMwqzV{V@Hd1XC`?znu-OD!n!`|4f^Y>*e*0x>ZBC{{Vs$lPZ
z!8N;oL`UzLz*V?+Cy(O3*!yn#8nw>u-F>@tpUTgyy~c8dd$0fR-+M`n>44H6@qL?a
zi|-TY`n&hL_tHJRJN4|<b<gch^DDI#>O5&9ANPFky;TAGe)j#}y`^pDUIQ+^ebZ*L
z?GwHwV>hd)f3H*Muf5g1A$vo2iS6Urw%b1QOUbUNv-$SIx1a6p6c^mfx#{V?M;(Uy
zrl{rb`QkQr@5-aq`z=&+_N?<ezIWcFKYL4rd#!!{FWIY^A#Z;wN@!n$0;j!eM9yC8
z{*`;R<{#NB>GpJY;+jW$PTXX)%YSfkk80?@eY+=C?CqXmz4uR3yUpPRjW#nE%h{*Q
zKel(vpKp8DX9nyIv}WF4yZHDXw#||@X{OWmt}S?IrPdg<=i%%pd*h3E_I)-F-Ipq1
z=Wv&Ouib%LyX^n0Te~NR<^0~#uQq#?8|3UGwa)JKp7(#RCEx75^{pCq75+;5f^9f#
zTTGts`TcB}&Fa{cea%zS?KPi-?<<q>+o!*}(q4;Sc%Q4tMVko^boZ(YD(}lo{;@aQ
zOL(7a`rEz7V^j7`J|4JhlU3W^jOw*}|8HX0$0Kxf@9j11d%X{C*?ax}-90?Z9_$TW
zW#I5#==9#&&6D>y3peg-65P6PL-cc75$m-5OA|%+y|m=r_c6M`zW(H_z3mq)53G~m
z-<MYExL@6}ckkinXZA0eEVnoD-ju!a4y*QWd$igvCW_T|Te-4ru@T!orq#T5JHj>h
zEe-l;bGH2c-u#ru_V4Dk@Bceh##Um5<o=6$MECso!?|y1$8P)QX8QZSGQ00RE`NP*
z<3qoF?3WkXP5Sg_@1xfE{jYB{?iG_%-77AgyN}0d<-S~N!F|=i0`@M+%WYT!8uwmY
ze|WF^^m}_d%>UWMb}hGEZ=<zu+OkRewj1#7-PM0;Z;7_yzVK+4eMahSw#Du2b^*3M
zd+IHN_V#X?WAotbx!qz+_x3*Gp1rqi&QF`1!nJ$7nN{|l$SJn{ZvS!*o3iKrpN(_&
zDw^}}TXAZ({mzDOdp~YA+biADv!9)Nkxl#^(S24GcKg+4v+lcJ+p>S*@;m!GPc`p<
z>mzxf<@Oz$7bRi)Zy!?G|L6Zro1`@#_G}BXwN=${*<Uv~W8cLmO#7@eFYLAa6}Z<w
z)N_B%sXextz5)(=;t$%ydw1>kTX}R}%fIdW*%Tu89jaYzv#_yx?-qe}yQhb{_NM=v
zy4Sku$zG;)Y5O$4UD!MG&<~pfPSUmv34NAlk4@e?#hZ2CqOS}0cD`S37tk%b?}9Af
zzH7C)`zz-4?a}wEbg(}%&)VMjmUZ)%HTzFVhTHevp1jxK4)^{U59T^dyF6|0_Irx^
z?3ldwE!{D3@9_fToxMj)_nw`~Yy0ucroGt>wR>|fs_ir1;AnF^bC1om#*=%Czb)LC
zmAHBDjRy_(OS-jfe!qORm(gH>y{}xPP0ivbdrOvV-xss|;a>jyt@grS_U$cd{;}8O
zeB<7Qx1Q`R3t`-MO+IVix^KsAErk;HzMQmaKd-{&y-X#y_q(?<?^}0j%HH7YFq@82
z?tP17KkS`cICJl}!;O3NSw8KnuCv%DtovlQ;0}qsQ*L$HdLFLXdvW__n>9`I_q>kS
zuy;%S;l1~R&+JL@<=Dsh!`0@gR)_5_A&0#VCE^YxUA_BOx@p;cd(LE2+F7`Fp<T}b
zwfQA`s}@Y$A1L{M@BGVowin&E?9=1?Y%@__({|0hg?m@box4Z&Xz9NBoe?&{QK$D-
zYbEcl$vv}I=f-@Si0?l8-nM4#HT%Y~Pb*`Z-N^~x_vW^%+J|Ir+}rKoWG^PS&R(&b
zWnW3SoUQt9_I)a6uI!y7GIeiRjp_ln;FP@^7vJAw;@)YyFjQfm&V2WMdpdXzOj9+q
zDZcr1?}cam`yJKS+Fa3%K9DeV{a(XIa{G>l?Xu~3YiTpFUc&bITDv_{xf}N0ka}a&
zaad^YDvi#)1vzrI8DYQn<V!T}Iq7k3|4QZjeRtl>*?TRb&B12*uDyGHAK%OI!fPK(
z_TIfZ+q?J0TwvJWw{5+BtLW|hw<4|VV^>?-cm5Bwm*h|0W1afKMyo^CMq<Z$yH&fr
z_5^P@c3{(;7kj_FoVs`MLuH4(n)=r3yJzfm@R+?{T)E_clE}Y(Yd=5Tv+3E*y)!3#
z+rtra!6xGU%)NW06!z6H?cc|lxMr`f<<fnJ_iwYFe(3PNm%I7*&Ub&fXVYuf{m)i8
z?p+{xWw+6%d^>H;75j>Ky7tcV>DcSv*u48b^Xt8H&;8kJ`(*Wjd0OZ8n|(Dnuy?8X
z-r$+~4h@Ts?p52zu>WW@@BUZE&hK|OTD$jC?VBCtYmV<_Z{*p_CDgHZ!sT<;k9802
z{oA$FCVk<#y}!I#_HWc!vd4S>#J&GgT&@30G25(@SY^u)yMNz8yEQgkTOaL-agpEa
zY8$(E?XEbRk7_-8r{!(hD|~dxfpw4X@0D9}es5oc-k#-N8unB3H`{TWz1&+qE8xJf
z9Z#(%e>rKFVYFxOk`FQVVlUNge}^sDr?Jrd!2Y*C>;<&<?XBS6zhC5H`+<{Zw(t46
z?#6zl=!N^DO%!Znch1`{wBf`ax4%OBGRrFWva4L*8-G)Icl(B^`|fXO*i#&}XK%?w
z=>spzw(r%xAZz{HvDfZiyx_iV6aVjR{yS^`WW|ELv2n}oYVzXu<xhODcV&Xf-Yt%7
z`>YnU+AmY}*ysMBX<zJ`#=UiZ5BHvYuDnnCvVvW(p7CBs-IRR^yBQr?3bt6K-&(!T
zL2j%4*5Wy~w?4Vs->AylbL4EmzOsef`waIp+6es3vC9)w*jIh=?_Q<jfqTniPVSj9
zcm1B+AH4gNp0n+pDc5GltNCZICU@ul-T%(*eUa+F$KC#yP1mfJ{r5s#_AVAFuzI@j
zqHWeAdz<KO7wxz8-rrNPuzv5<&b@mNyQ}P5mpO6YIyQ-YKV83B?Kv;K*WmWvz3;Qy
z_v|~qey^mm%)ZIY0k)GOHtzj+X0g4Y5ZAt6r}_`vnl#l$E2-HgRA2CbA@?DBo62u{
z<xd~jJ9qAX`x_R!Y`$+^ws&>DkuA^HmwUH_#O|Hd*}HF=*3G?}x8K~qOliaJIm$cs
zUbrv3k8efH-o*b0_V{ek+kci@$~Gj`&|V{F`ksI_Jp05HgZ3`zlG-Oe@9w@Jvjcmr
zKC|x4dii;;;{~Pz`{l0fy$~w3@7GoJy=}`F_P+dcWv|lI1ADbrP2cle__<Alm!^Hg
zqksD+S<2WOSiIiNETU*Df3nMFm20Nm;<>MPvzSfU8)m`3H}tvq{!5`Ad$(!o+n(7X
zyzhKU`reCnQ}*AJ*4eMd>$uM*Bzf<J|GOPrYRqll->BQ4)49vWfMM6}HM76$t%>Qe
zIqM{6JIm8*pJ4B?z5C7{+52ltlkNPn#d}xom$f~T!?;g^Y35$OwIchx;`sLE?R{-;
z_JnocpZ?u@jj9&!<NDmN@4M@&eWw-p4)jUI+0^V^VN+r4vwvdDD!Xe(4ILtM8f*i^
z?e}N1Z`hj`xyb(4)}wn5*&p1?m=knBJ9F2*7uV116}Ac7Q(``O-}$1Y4z~-!_MYPT
zw%1qw>YkET{ryiC%R5*oo!b-fNXH@H;L=_dp$&VEWm)Y1;FYm&pQ+^DNY`IBl41S(
zW?LM!FR^Ia!zjSLzkkb(J&Nyd?KPQwZa=g5gnb?9)_d1KDz@?7b!)F<a_indWpDQg
z=`-0{^Q_<d@^|sRnC$a56?#qky5lwM6+ge*YpyxRx_Ig5z0VDg?Muj7xVP2+>z?q#
zO19}CKlfggUc9%Z^n;C2()_)47ccIWGBmRNyyWA)!%O=2na(lY+gJE(->OjCy{-&h
zw#%#B_kNgtcy~qG9NQnB=d4+N=<U7uKGb%Z-gO&=%Zj#1Y&UK1bx7_#WFWieY~u{u
zf@apemJ%|yv!b~6K0ao-x99BSz4hG9d!u88?XLL9+g<TzvK3e_Wt%qHac_CL!S1a;
zZT762<FR+m_2#`>mTs`k`f$j0HS5$prtY8i>=4;+vrByU-qV+aZMWLJ+0`2_wD0N>
zwcYk7g>0pKSM4?X>TbjEH^nArL%i*(xQ{joH&5^0;j+(G@4&k~JE~Xj-LE{!Rxz5_
z?rpn--BH2qd)GYa-uqeRw#`9i<GluNZrZxtSZ!-yd&V~RY5E?SFL(F8ILN*4-BS*`
z&MAlXW?kK2+vwA{_rouPy>gdK_EelZv14jl_g)=_4SO%dO51i^6R~Mdp1sGU#l==8
zOVw^dB!g{;7t_AOPT4l!yZ`OA+O%o!fe?kgTt1I%;_rp*jk~#h@5ed+_Wlt)W?feF
z+D6Is%icKOrM3;%C)nPLTxcU9!M@ixi(?;`?osOo)00+vrmnV8Vc^~4;iX~A!FY4G
z|APL#amw5F{#=}~C&S?6o)2bgZJ*wA+#9g8aqq1&$M&{7=C$2)V4khP^vBi*KJMIY
z!a99#LgcpH%YQ}LoZiG?+ak5oHs?C8-L()Q+sfVdZNKeVY|E0pWN+ZfqP_e<*Y|$<
zvA{OK|DTP@ZNWXtmeRY|EmE_MSt_t6HF&-4YUAR)iU0oFG^|*+r>LiA?_#f>-PsrZ
z+f2D7W$R@6)8-f_!@l2A7xz@(K5iS{dS$PE?jf5u6GCkLJ_y;lPTgShNp{xW?Y1**
z+Z6=&On5SRZ_bIOds535?C`$DxHtH;(!LIDzC9ir)%MK!)@d6O;JRn(r*F11ycSqj
z1n}8(uUxlB@V?Go8{dhx_mWg?^Jg>LS?pl2UD^KGW<uLmo8kc3eGSLitkgx0?6LlS
zXODx}#a%ZV?Q9O(%-ZXbaD0#Tf{lC6sBE&4U1zoT&&^#na!#LYmRGs$6_A#(HPPs{
zv6b)L%lTBm&VJQeTi<O~d*9udz3069L0hLMQZ@{4l6Dzisj;nzkF>2Tm#|&jtYo|B
zRQ=u?x1wxydUx%~vYlZ&<K>q<w`MlkK5m$3D>?U{?KZ~Fy+xZ-_uMr)u~*~Bu03<S
z18pz+i0oCV{b_U0_`u$sX%%~wulU%yMzq)(NJ;M5X7G3Kg_oVXXY5_E_rT=7Jzf(l
ztQ`D4?r|_Hx90w^Wv^sZ+uo;*{Wh{&_wBY6D7Jm_^T%$6Z+G{UiM+BFlz(r#zTSQB
z;b!H%(fMV27$W!E&TIDETO`Y4lU4X+kMPCly&gO&wv*-F?cS9qyYIp52R2v!PPEE3
z6WAv#^UIpSM#J{(u9`g(o$9vlMa6AnDq3w%FR`?ntFB<1xv|AYNN0zwc+fGM85v2o
zRg6=tS9WsklibF$kHK}@UV+ZTHmfC9?rqTYw>`UT_1-%^f_A%Nx9<Jj$i0`RK!5L#
zh0ARee#z{cY<O;O(R)U_mYoOo?t0O`d+*8rwli7H_wr1fZL7sHXYZ#n37eWzZ!DQ_
zOth6Z<=pp8xp-HD%e+0WX6@Mf%;@UgISUr=U45@<?{n=*wuKQ*wz2iDd$*WW+9p)x
z+jPEF+}CrGd!Ix@rR@@NW?N2eNxLKZ=lA@|&D(o?y3Jnk3mf;G2$bHp{>FyA4i#JW
z{xTKc`=Do+?XGJcdvpG??hSdKzUN%lB3r&7$Gunef7{f*FyEW1HN|G%3TeBa9=v<|
zE@|z3+9qYUkb}uaPf&H=8>zxQ(-M#Fc_6E}H(#~Ic85mEu0wx1_r4NYwl|%<Xz$jS
zl6#M}?b*v9bHw68>JjUzX79a<cH*`l1l??<_deab`h5DHoqLY&m08xgSFm=H^~!`#
zd(S+YyI1R;gq=*O!QLC6HEo-65AKPwD&M=SOKvY)Oz&PDLB>5#FaEH(v--^5BjvC6
zL{&uZ<~jOp*OXPgdw;NU?LG1D_MRifUABwY#O$5VAZvSAyWZAihQ^-yzt*;8^)-8o
z=I`H|FyX?UW1hw~ftP35@E-ZO=g!~Fw%ZC=Y*%$$+*=X5*ydHphrN!CcWivGt=rq-
zW^KJKn`bZoOSZk2*RR`KSaixpI4;em<nTP(e|LV`?k<_UcT$$m?k>*#djxf>_B>GC
zYf~TdU~j^mlY9QIu(y@BU%mIi{5<RK{LOnObX>4r7Vy?qIDPM4{;j)g7p?8wdwX-?
zULCWUHcPu6**HA^wmac}^`7=U%KI3&H`uPvzOpwddcN&v{vUh$Rx9i&y=`U7aCVxt
z`B^r*rD4nVPI7Lt?S9>}cTa<ptx@MR+qM}S_g*NRvG+n3;~pO8%)K!$GHe;=>F<5X
z)Ufxc*k0QR>5108%N1>(ya=_mIA6U-f2!2JwK8Y-W@-r7vdHeSwW;N?eIk8)55wF5
z8!syn+uLtCZMS^1wCVRZ+53wp+QwCfdH1>bi*1;%YTGKEW!@*vw$bKj$%Q?W46STC
zy1jOOp3S;%Y2#ztRCzVqpbo`-5|d`_^$pr)yTPEu=JkuYd(AQ~*<Ab@y!T<kI-9FU
zkM3P^YuDau@kjSQIh<g-`?&UAhf4yx@8>n_-ErArPl2_jZT;!pHpfpH?KRhZX=|b+
zzt2<ix{c!F#kLiOY_>b!bldi>y>7F2%bUF~k1yQ&pm~{X?{87NkZbO?Xa0P*y%A<;
z!}G#zZ&s+^Ub8UKeTPIR+vI7S*ej<iY2&~<!&ZOyk6i|jWA<9Fwb@gkaC}d`kH5_(
zt;@Co%RcUX$=SST#rbcxAx|4@{>i=DEAxH%o)1}7dyaAc+RMA3=Kz1qyFH=XyY`;d
z%d(d>n7a3m$kx4kT3_zF7Q(+zPJXhz!T*VNL6ev7TQz<8{!_dGd(GpY?A<#{z}{{C
zoIUprW$ekm#ke<M>4E(<FW2rdH;~!aeXwM2{)c1Mtk<^eEBSTT_OMILzR4|j_I!KW
zXdkjv>wx4=jsxNc1owqq*lx@7<I>(~d=(CBSyXI-0+#RP4?M8f`Mt%ylgkSCYTgpq
zx52J`ulT~rds7+p_sp}>+w<Q0<ldEh>UI;Yn)a;xb->!0O?v;7D_3kzwKMOJyUo1s
z!TK9}e}4aDBjW02ch9I|Pv<O!{fQGq?9JB6?BmYJ-n%J<ai964yZe}vn`{q63-9*1
zd2#Qq-fcEtYVxezRFCX6<k+_N!Kww;e~JS39#Pt`@7?h;wuk5Y?{8Y_u=kjW&0f<R
zP4)#<*Y+}<-MaVh_i6i%Z{XUisZnb8aogd&f&2gNU0tfS&rNu}&8u#P-KVzo>^<_s
zV$Y4ed-wW2W<8MmZ0p|DvHbRJRk!wL9eA|YZGzRFy;hU<Zp;_nxBvKoy^R}8Y#Fbg
zw_9pAe~+j3vb_No=6gRp;k0Q}mD%_Dx4QKehxL1z-n-k1c^t7_ncBN|gZksWA!76P
za?KR6S+Q8sX0pZ7{ZlI%_wBzqeXqi<C41+-Vc)knlWV``g>`#(c1i7XI_b1eEZ~Bz
zk*Ljq9FxU+lQX;b)=6HsO?SAu_tcH8``^ec+v|32<6gar6T7+hZrL07{LJ3`mBxGj
zDXQ$t(o^1ZK(ue4WrgOxEALJA3p+9IJ1{$S|BuTd4wpD3_wD$myXU>?x4lwJ<o0bX
z{IMsyW4-;?sxNz|=pNdwG+WKK%dL6Oi)p5NP2%p@Ehw6}cdF*Qy{o&w?7RAR@?Pnr
z`F2~LaoRfA-rgG#a$@h&dBS@GEaj~aB_-P|aFW^AE?B-dOYHofn8r41kJ|@rf)>rP
z{m&_|k5`ZHz{;&__X@VZ+q-qO@qxJWv-gVqJ+NoGQ_r6Ft77*W#;o1<|FQQznOJT6
zDgICGx_9r{CsJEy@6uVZe}9&%Er-~vJw6?(`+v<mU^i>kW}Ds&A^UaT&+p|wT)9{A
zf`?rztH8c0>l1q;N@DDnob0pTW&dnnsf@Ml*13&)`Q63#xm=O9z0)ysZ~M&E)>?8w
zdn<M4?R96<-{(-bXOHb$Vf&kHFZOPDvDM}@x9kCpBZq7o?04@zHmQ8yD$%rkB9p$_
z#4*+F*Sx5^m-+m?Jzw54*z*0Wu)i|7#lFp8zujidroBr{Irls8?y$XkHeq*fHTwa7
z`LY9bF{`Xo4s6()<z;It^Pb0M){=L7wSG_8z2ojI+o*nry-O$Y?$Z&UXfMK<v$t_6
zn=MoMxjhb??zWcd3-`XWH{5s0tKa6%57~V)c0IERo3-2K^2g}CA9Wtvwmq`kJ9)}3
z+h0a2Z0bJf?H4}ku-Auqf!)#1T>HPNwe2<RzPx{n{3;vGd%J8;oMqZOX=&B|DL3=?
z&WnrQch-sd;M!Xqwi^3c_Hp)Zvp;{b(0<CgpL?%Oa5!+0muKIZSAX{Et50=MjMUki
zQOUM%L36&%6TL@!e<sb}FTAR6pXm=f+ZxuTHcR#_+ZQe^y!VjxIvbV^$_LCoZQY|R
zEoobLPTao3Yx~}1k<kv#O6~iOES<A=Ufn_4zi$$4jJ(_S&zm)KpY~1ReLL0}?28Sw
z-&eK!^xmZA0GpR<zuQ#4H`{wX_`7Y`y3D=h?sM%n&X#kCopNyBZO_*Ii+@z_Jz^WR
z@Am=ueS25F+3Qz0d7pfN!`>fN^Y=DiV%^96if^CJRbyLbq0YU=;dgC?t19+iE&986
zy6gcvja!|2t|*o5HI+8oqdig3LDAyfZtbmod*zd!?B$(w&PGBpVZXQL@x4)jSN6Xs
z{A061!NsPFHGiLyo9Mm`r+)AKygq#Ihn-vZE<Eykuix6gHajk?+I#QZew&XS%=>t)
z_a89vp160<Y~};krY^N{KH#(0m`!{CUE6c}j_+#T%kHjeduiJh`)t0zy-g+i_rBr1
zzE^%qqOJa;)q5PDtlMAt|I?ne`X}vv+}LRoyq9O6qsn|6;aej6Zk{Z&HSb-#PgwEO
z-rm+Y+oBB_d-)oe>}F0h+;{ML!rtHWWA>l%cx>DLZP^}U=DWLBw6Cx$-1Kg*Szh?w
z46Da`d-;uQXLktO?E2cWN8G#JditTxJ+c+74quI|?X(P^@0SqL+4Fl&w#^^YS$oSC
z-nTpYS!iD?m&9J>l#abD>kjU7^@_9=pQ&tX$1Jo@Q1|WLwGV3S-hGa?`!^?WUsplS
zzSZ&T_uE!}+4F48McV`SU+p?%yk+l~r=RzBYMI;AJ(Sp&6yULMuklWsFG+{?h8gbI
z$0{Sf??2nkz2}ax*!FFive#+x%{?DJ&fa_2J9+<y6yAN?>|6I{9B|o}D^j-a>t?<K
zJd2L(d%irtMsW6|y*l5fIIPTb+CN*}#D2vM>Af*5rhCh>Z1&z+xzc_Q?><|p3d4O_
z3_1Is9JJW`{mQJpZ>#6n{Il!Z7rCq3W_dy3-t#RLd(ZiV?cMup&wk&s1^YRFZnXIs
zG|&FD#+kjmm)`EZ6jJ4oUBj}^e*2!imtP;;+pr^R-`#^B?fPUl?`8WCyMNnHxqYTS
zBKw}Iwe2h3W3%rQ`|Q0-HZ|>!b^EhVBEQ2nqEK>=!L_+oFD^Rl+5RVW|EbM~_SX8i
z@4LQo{=NsPANNKYUfi2o{d#X+g!;ZiKXq-n&)wPkNB+#-A2tkowd|JL&NrF3*Sh=I
zUf(cz+cUS?_nx24wO_%!XP<j^_ul7~DtlPU!)<+CzwMn@@p`Xz-;BN7=OgxhsWP-(
z_T`+7K+2=N8AZ8!FNjXs{dMXY+g-mF+g|6{zo%W=dasqplD((DZne$()4lge;o&{I
zwPS5^l0NS}zJ2nZEeB(48yp07pXwFb%OHAoZ^_IhRz?LU_xfnN*ygi|?XzBAvR9}p
z%vR~)>^;8{w(jL~+h@yfzi02VM0cCpXZP&QYEjyg$9vmm-&?MI`;@r$mEQQh=fn3S
zHveRl_D<gX#in7Y(4LvTc6*Oy3fpld=IoAHw_vY^-wfO3*L`=dV!mV>p(e5C^RYR5
zE*)pEJ<6xMr}5k5z499CZ70t%vXe~Rz30@~N4xurC)*Z?7TBKNmAcpW;<~;2yLa!c
z?t8vh?AA=1w+2i1{<Ar>_uEl+YyOSP_J$TMvw6<Fa&O`G1RI5lcYDPa9k+gFylwB+
z6La?NG%~khd%wsw=cCHL>h}lsI%#y<-mG%p`{FPEzN&|_cRNkx-22ZWeUC+;x6Q}y
z4|}cqckf*ucYN=X8SD0(ahzcLHfHi(*OqJ6wKH4xZi?7w^JKvV+ngCKd+ufw+OCmb
zw6~%;Vo$)$se4ZCpSbteoE|H;M@#px&7QTl#A3sq{qc=^^U}8O73q|+P0?SmSHypU
z?ThQnY!oCG+N#`IyLVA5%f1$^9eca}$?OX_$iGkT=aXGLpY&`W?1|g`??1Pl`c(nD
ztqw_hnU;OpyZol<o;4GA_x(7v(B`nB^xo_)`Mr*enzjM8N_!XeowZSatYIs>zIQL9
zli;2LE=IcvEAH=g+I7~pfvd=-*`dp}$TfQJhwRJN-LIJU2{3Kn`}iEsz8`GLd!_sq
z?$S=%ws-ciReNsmtM18Bv)B`0Q)DC8ZfPqkv(&cJ#cGdPda>=J(9L`6HlDKm^0ds>
zPRVoc_hM7qZ{Mfx{W9Uj-nTW^_LkgH-sk+Tb?^2nmVJ|S1okORv)Q8%@zusaRbiib
z*Xg~<H@NqRO=H@3;b(_!O#N|dzlMc-7q`B)arrE0Tchb@n`yG%s#>YdmLu<`jZ5}{
zJ(Dlbx4p#nYmfb$mOT&rn)cq*yKa*h^Ks9sjaT;eW|iBDYVF<YafrpvH1^+~yaS(X
zp3Dic-7X`%PkHloTlb4RwwLU#+VmgWwfE2Z7rQ-n?y}W+JJ*(-NqWz3>nnTuj!dyB
zxHI1-^q;cbYPW~B2~r(<pZpTu_wRVKt$=l(wS0c=-Z`A-_D(z9wl}dMb?**?PMa^g
zx%R}hi0qp<`{3T$EE;y)cj|0A-iX*f>Q&ii8Wm}KpXc1(hT`Qm%NN$zUMX$0?SJLG
zSN_Nan`dj+?%_9y-Rr5ywNH$-WG|ymneB|CM|(VWMDNY`qi-v-`=d?JU#Goh?_chn
z^of7pr912QewZ(`FLhqG?F+uWdksu4?zMWxWycYwxi@R^eA}JwHG7vC{<8^RK51|1
zXARqT@7V0bGP`XXP6XS!q;TzV5wF}Uc)V}#6ZzA&r-at+`MM%}ufP<uy~pM+v|Xpp
zx@XZ_B|D+i#=Y}gwd_8Am)^@iSKroQ)&ISs9AP_cjxz3(T=K`JLzjP#d*F|~4zJs6
zmv%hb`-!b;?}bTCdxSD-_sp+2zn5jEknIkSxIGUaEV0S`ma|9x=8ioP*}8jYZdqb`
zZwl|e>T`Xz3a9_r^xR>wouL$HTQXzoo`rk#_87!Jv?(y&Z=-P1+E$}Cd#_%0m-X`O
z9X5w&zt|hScIw`HUsU#fS@Y92*LmvRp1c*dKbQTt-Kn$RMnbu3uiB&N-4{8O_D#EG
zX1&)#W6z-{j(e^0oA<6Wp1IdaP{?-C_o;i86KC%|rK4n<cKg6yrs&|k+jeN|eOc7I
z_nXE++YMJm_xesOvz2)Ea<4$5;6C>6>3bJHF5LTX{^q?;?*HE#yJGrY$r&uR{Kj?G
zvxMf^-dMnGdwbf)z2|FJ?$!9`V>{t@`tAo4@7qY6(%ie};q$%sMON%desju}c~|b<
zt4<!a6N^~(@%`Mpw^wtv?Gu3+HeU~Jwn@ux-_2ITwbx*c>YlFBnKoa$y7vA|=dok_
zV!gMc=ENT5sXO;%%;4K+Xyt9QF8$7)tLts|JTX4E%gUbD_C)#~8-qs<Hf%Q%ZSUmS
z?{&U6-8PE(>z<PvENvrpZ?NqUTV>0q=4|_V(<_^6FX!3@wsGxyxqItgDT_aQm)y8(
z`}DQW-mfZtd#CM{vt{^GWOYs`W{+N{(%yz^Z+2%sJik{_H+Qe#kIcPl=2vXDEMeQX
zcf<TW1{WvUtpBcHXE*KA-s!TQdw({u?)!g4!gl-K%Du}1PS~DZer*rKfi$aS8vVOx
zNG;zh6F0%S<K97=1_2h^$L|<z-(5RvyJy9Py{i(ZTGu^bw3)4Gx7V@g@Sg3jO!vx#
zl<l2abbW7NhV5Rv<rX$}wY>YnzsT%;ePfr+rj|Q4?^KQU>TKlRdwI!DE882__9_Xo
z?%h#Yyr)X8-DbhR9eahY&apK-?QOf@_p9C8r*GN&l6&jkN*9TJADQRvJ*;wJ@537s
zd&P3it-~tX_xgnS**4D!-rbv}Y+E_=(cTlutos5t@7o)2TGUomNN+Dw?p)if^*(zf
z{}fq2TDQ$MMS0Gi8|jLBd(60OD|kI^Q?{_!8Y>F#E6EYDy&lA4^S*WJ-fx!{+HQ4`
zu@mu_v*+-kZri3)oAwrz1=}w0_+xXCVbva!Pup#ciZgBA+ga~zxxdl2HDr~I&KrZh
zJ2w^Xby)4b*O^siuY>Ot>mt|Sy=v!p_s-+&-h24Ol)dZE9<XuQu+qlvT+g0gHM};x
z<*~L84)yOjp&DlU%kS7;pR1X+N2e;;T`E6rbLz@k+h<WGwwE`pwTUUq+8g!mzpbO*
z6<Z(KW?O~#(`?grO|?zrXWe%om1l3#x^;US-tV&ZIT&Zl`n_~-n_G}A$L_s0OTIqY
zyDeIL?}L<S*23O1_Fjn3uxVdzYAf;Lg6-B*xx05x`n%UhR?C)yXQ2%j<KMk+<%Mk-
zx{P)eJ8s*{`hJh?iQ?6^JlhxS-L+@OUVe$~wt9Al_q16o+k0f&THA;{<u**Lb8TP!
zoVE8%;rhKwZ727>*?V#C?KABA%qv>=KDe>NhDS1P?^<O!JI!B@_hz&e?mcXxVXOT^
zXYY&Yw=93pw6rbVx!RUzMuK&$%+|f<&b+Yw_m6k4?d;yY9}1rCzP0`Bp3Yx)Z67^m
zvh_H`Yq!{I+uoh$?e|{GTx~l;Z2n#i=Wg3|`W$;yzAUoYu=TCY9N~3V)&<Oa>kb~Z
zJ^gpTb!*6{y~fHedwpvR_WEyV-P57)*|vWw|30B)?Y$a%^lWE&Z?g&5eP(wulk(oY
zm=(5eH<#L~X05ZyeP3<OUcg|p_kzSeDV7U+t-gos`MpYL-=<$1Y<q5R-fP##yzivw
zl)VmI^Y`pobjWr^k(7<xvhcliJE!dBNh`7y$qU@Oee!MF8_QnU+_`UK`|+Li-ue#<
zZOp`3?2fE?xyO0u*S(V0dG~Fwwc8UDb9j$}#Jt@XHoddi{<3ec?uYGr8ap@c?w(n>
zw@0hT)~w;m?g`Vq_s)EEa!*an37h-Z7T9teuiv}ioVKm-9?^XYt@m~>_+)IO=qh06
zXmeoin;ZA`$WJ}BH{Xic)@Jj9y&8@`ZDLp$_g%LW-uH{OY44vy|Mn<8{AjyHfB7Dn
z=$(6C2L<kxxcJM4M_6(1i(7|mv-9@s@hxAvx7F(HF3k_}d+*rH+iNcR(}p+gyzLSm
z<Gt)FHTTYpxNlpvZ?a9|chkN5Ygg`-o^@{TngHItSt{MO5$hZF=5dzVZp=Mq^TuSW
zO@7EP+j%>b_XcRG*oAej+neo>W6N-Ez0C&!PTOfALVGS;`?8lqhkuW8<&wQ=S<SY_
z(G2@SX1DDTxMyS&<X^D&%?8<hUp&R_{yD$gBhvqRPyC-9dpE6Px64S_w&w>I=U$y_
z+ilzapR~34G-2<1+fbV*uSt6u^lfb>%{XiuKIzh~0~XKsPTJ3Hcc_2X-sLNV?Pi`?
zvL__)(VnH!S$ihvJ-1c~*<$NE_2^#Cr59~atkkvTIJnBT%KWKq<55Ao+0IAydL5Rq
z>up%Fcaq5#+yCV|Z5i4R*<5xLw#{JJyf-X<ifu}0?_R$-r)*UV=Go5dk=y5FzRkAh
z<%+#+M$`7b`nTJ*q+H*&VE07Zw$vRq+2(KdgvF@tSru_^?~VD6wh0=7wyk<mwjFQI
z?Vikg$TlwNr;UK`l)V9;rtVq&{oAf9L7jU7C-mApbV%D<r?t&0Jy*)sp~J%FyxpU{
z;tRIyJ)^C<??s)X?HPTxeH)w=_hfqO*?7Cl?cH%g+0N>9m#xm~=Dlsg)_Z#QN$(5z
z{J`dmTJ>J;5)E7X`{H&9i#G20IZwszlVhtb%d8iBdqoe}oKx<yt>O5%=l{F!wj1iL
zZNsK=@6GG;ww*U&gDt-XzvaxT^?PqkV&1E9Ud*;^tBmaz2Kjv=D|l^RiOt`epenm}
zBG2MIPrfX(4GtC9r}3w1&yMDjy$%x&TV43}d$+;WFMHVk&E0!K>+jzG-D~!0Jaw}T
zV65JoSg5#LCqU75#f^nF`xvG6Xn0@SGb7#0=HK4^dynlsV^bovVef@Mi+7*8@q2Gl
z$c;TWZmh7qu6bzh0)tt5TV&jK-RBS8Ta?>r8}xYI-iEdsTL$qu+XlDQyH_V0+A6Rv
z+5J5ybuZtF_j|)VoA-!Cy|!JZbjP-9!ATpx04clOW-DzCZ<p_}&|tHTR$;OWs1e+|
zE%w{qJ(mP+&sVzJ{@VP{R`<!Zz5lK#+P$jyythUA*d8_G{kBdkTlVJcRNCiv$j@r7
zPtV?Y!WZ|X1~u7!sp7CTI25*LV?q4hh>S_L4O_YF{;a-idu_eD?X&A1d*<HyVLLs^
z$Ckb3we<xdF<X{@6Zh=--?3LD&C0gy#p6AJkt_FH@tA9y`Dgpy>AQCA)xE~KH{p%a
zo(Cs=Y+W=wZ0~GbX#F&CyX}F1)?I&`lx(+c{=TQjxohu*!f2Z*(T{9?{LHm|@nZF^
zId!}Ca;*Gi^YJ~)-h#4SyRYOPv%O+>*E&Ax*4~^2+O`M!6ZdfXYwwxCdTH;EYm@dq
z?_jj+Em7P1f4}IyUounomM<vUBe6+%FX#wI2KF7h7UW9XxZZ2FHn?uM_sR)xTaPQX
zdn$Tn+dfFUxLc-b*PgggNxLgq?R)o{eBJw~`I@afYyIA3rYmhiuBF%psBf@Q&pf-g
z;oU0h6(5)G5kAXfw{!;U-Ve$fZ8bF2ZRc5^wEdC6vX9kol5I{DlkJ9iUA9yD9c+!{
zx9{od=(cq@QnuG<$$~w5ejeWQ-=SphUAJCai#Jkx&Uj?o=FL2_SKFF@-z}44d(R(k
zvpFuvV)tT~jP3bqS-Xqvi}!v@J8hdAaL@LZ*NHtH_b2Q<C)~G3$zsyp1?*uqTdh~@
z$zONa_S=Kwdw-{Hw=KzW+v{hOX0vq~pY7dwM{NyG?b~~0uKeD4p=b9LJeq24TPCyj
zMCOIPHXqOJeYZ!==7s^EZGzmS-MfD8+RJu(hi&O;al1|Zk8QN?P2YRK%F@PAE5PP}
z>h!(GG`H@38vfjd`^3k+z8VR8jcS!_9Xx*SQR95Rd%1qG?Te%;+ZQ~Fd%B#~*#=y2
zw@tmgZm*%;MVspCm3z188QRKr8t#48o3gJmQGVZ&$$$2~$z8tRS9pT0{t>DD5AT@U
zDd{oV_eaYd_{uiTW@321J?EOfeUqY!_x_OkvDaCXW3Sn-yY`zDj_#YkG;&{;%P!l^
z;e7iF5^mY)>{i=nJng~0#f)?IILhShoe*zezmv<?PH@kJeG7OO*_@M|y{A`X_5QQ-
zV{Lzjo!q<W$BVrOHGkP`kvO)e+kf-k%+)LQ`Zk=}d-S`sO^V3my=(5Q-79P|Z~vbS
z47N+otlsPLEN<VQ!a26vuI$~v^77HWmn`<|o!d5dpTUNC_SdyK_P)F?vOnj@b^9~X
z4Yu58H|`CPW7(V6wRvCR^;dhJgs-&Gjo-EB<5s7=tO2k0c7<!&uABX4@0zm`d#Apa
zxBc6Bbf1spsy$g(C)#z3ecAQR>gnFScW&+9*nNGkhVjY0+dnScckg!NZsp`_`wJI@
z?Gf%5-d8q#uFVC>^Lu4ap5I$o?YoboYWJSd?k6_)=Tz*U)}gX5>BSK{2`2e{dDi!A
z&XzXsF}8cUCvt<tzO^L@mdpXi_n7TxwtpirWv_4h*1fwPKi>1<XUd+SwVit>Hc9Mz
zSJr16w)xcFotsYVeYxt+-XrmP`+C25+5XY2w*A04&6c}X-=RypZ6908$GwkNU*7Bc
z_4(e3tDE*ODec>Ps>@{mtr?2@K<5*j?b&Uw*(AB|RO81zD^@e^b@<u1x2^r1t(fuh
zJ$!}g`&>l@_c5#z-nZt&6PpZ^uzml79rsuVH|`CbJ9l47hyR|8r?2epWvsWER<nA4
zU&`Hm+IdU&t)8Z^Z^2KFeUnyk?=us(vw8Dy-u_DEh`o(|V*6%KR@v9Nck14%#rpf=
zjoIzAbH(<}@SCw`#`=`~Vbg!?y_6{HP~$Fa`$&V`MxI%3|GSB^_Hx@T-c$cca9`k;
zr+XC|_-rkW*X`MV`OV%_9UJ$aZI9e5Q*_g2kJ^I$?S6Ci2AzFq`~KoHd;eE1dl&a_
z-mCiEX77Q_+k5{Sr|)ZvylbO$liU9K0jK>neEjyGyc73-%syea`au5PHIrBDnY(!A
zekPaidpY{H?`?FDvv+%UeJ@wq>b*NQvOAPto?)}#v+~{z!k_K>Ek$kF#IEd<U;BA)
zm#5@jhN%<w?nu(NJ+ki69`TEN_6BX`*tdtL$5vFEb6-{PK3l^MxBXY=GVVLz`ebil
zJKuhf%J4lrm)7h(aMsq|J-TpTR<Qows`H8aIl4FP{T#7$clKXa+oIJA_lfYQ?U&zK
zzW;=t`M%=q;<g5tWA<i97w@$Uox0z9%B%g)VgmNqZ`{4N?C#9n+so~2!;bvfJI(Fz
zZua+$dk-tL*xus%wpXQg*1q!DjC)0by!T$%vvto5_iZ-)(o6T&+pXF=tz_Ds*%1r&
z+VAh&{b$XWJ@10rck6B0wpX8>XCI6Aru`K+-tE1U&#^y%Q(~Wr_U*mzKF-{4c2~^i
ze!z_V+X^r3eJ=UQ_V?TBy&r}B?7ZA;ZL`Dm_pxet?)SG>w=Yu8+U1+vz0ZPakG+?q
z;l7`o>ih1$GvC)P#kp?_!$LdXTk89G3;*op@87WZud?XAr<)}9Pnw>&@8p^ZdmC?T
z*?Yt4=f2~-^KJGw>Dr$0RzL7{QH729rImX+KkTtT6R>h`Nm<kWD2>hgKIR(O`aIgO
z$E0iS-uQ@#`@1vU_nk`kyf^c|-(GD7*L~9Rocoq+=&^gPduMOj?%utp3bpp9ytmt1
zT;^|Y89mK@_U5wvU5u;;u1;&-`!HE$e_!_eJ@en)u`xa#zHb^6_r52xYWx1_PTR}$
z=l`A`cdpn3Ox3o%+E`#4ae`yNFxR5Ju|`|$roHL0wOCqbdyJEDAID2>hwTq-_MVMl
z+Z)uxvv1on$Gu_Jn)Z7%QugYzH11C;7qX2zE4J^7LFvA(^(K2n7QVKr$cx;Y(tX0#
zO{Z)B%$TIT>pyAksR~%JC+&ygzRpve4s&gm*v>wBc>li1jQiTxv+R>DcDAq9T4$sD
zopImmmS<Le_uK5VK3v%A5hk{;YeSgro$K@W9?`Mc(;vBcUs|o|t`v{5eb-L<*=Wji
z*nB-CxG#`>$KEsbFLrO)dC{)j`^w&1T$=m+*SxizrYy5h>r?CAPW6vA`o6vU_i6R)
zR*?9%_nCvl-i-Led-u-Sy7$oUjW(9|U96{e>+SV>qhR}qQE|tU$CvhMzrM4laFV0_
zsjiKCjQ*~(FELHs`$JiM@9wIJd++%h*@p(T*j&2Jw9nIB-TJs`xZV0)=C(zzIri~<
z`m^^t&pIoi?LGTDPtDywY1!qy{hnU?o`0Wi?<VTEHz0Z8UWL6U_Pl*@d#}#6nR`EM
zn7?=Bo{cu|W7qFX?s&d;kK7WQDR&z8GdjBLU2^*U-cY&EdnbfU*gJO)!@kZZLVN$j
zOWA$)_-n&<<>THrFZlP(l-{{-dee&ipM&=8Pv5&^kNX<~Th4+X`_9VT+Sj<t&VK$z
zm%Y)Mi|v*t2JPiv`PE)KFKX}S(^|GCO_J<u_pIB$yNJ<t3e&VbXUpg8>pb7OfA+kl
zec~J5*ckl|vbi)z)PC;r*ZUsJChUzDx3YJ<tG~}wKhAdEl_Wc#aOHhH`!DZxoWf#z
zMZ##`7ugxMEha5{H4=^OHCPn(_H@PAO!~WEe{^uhzAIU^dkb@y?(5rWZD*i#+q&V5
zgq7UWt2P%DkMDh`TDtd&>sDJ4>7;!w)2;UBEbH65FaFqGj{b#vu7oz)URcMx?|$^P
zy&qO`?akG@xYy<Qa$6yXbv9;8ns)n7=iMi1$GbO1GT%;WkHy}s*O7a(O>XbYIMcbe
zVxEMZ+YV{F4IAd}ee+k&PW16&o6YRPwz9Lg*w&xovRmO}VY8%6!<L0vdGFJmO125{
zTzhBEoxfK~amL<9$vbW5q#N$_Z~bUvQd(m>Sx0H#mJ=)YHt-wmEtq_K_kwIko9dE3
zd-^sC@3R+MZOgERX<tvldRw=7XZNmFUudJVMbIvJj?Ug+p*4HkS0wD=TB>AM_iWDI
z)UW<~J&j-66c>u_)so(~S9nk7-n5&iZBM8O+R7Cz*gMH=s_mXKUfbm@Q*G}1vD)tV
z)VQZ>!qmOj1ibcc73SRcr*fT*)Z>Y^R*f=t0WZ(kuyeiH^JrR_?Ngpbw(El?+Wa>&
z-kVr?Z;wF4ZyRAh$GrhRf9;my7TLRK?^WB`8uGSb583SI`MlV>;ln%|PfJO=iqIW4
z7dP?R)i2+(H^(=7?}N2gwm(%@?|HfU-rfg)m+d|2!eg_fblctqz1FsyPVU+p&BkZT
zc6h$+ia>F@2u3B_^pDYdHytwFd!pTZPp53%9<M#SZBLwWwr%^uWP9B9)?Viu6Zg)Z
zEWU5a!}EJPE*!R=cW%1PW$m!NOw0%O7JN9hS4_Oo_GMew-oT9tcG4lo_VVo7z4yS<
z%X{X;81AiD=DUYsLY4LLKQs28(>h^$!f^XulU_!<*Z-F8-Weiq_s;6T-sZX6_D-C3
zcCWzCH8zs}cJF;*$G?|L=ec!5?>bxNS(j`qJ~rCk^0{YotVn$Cm3<4X4<0*Z^HN2~
zR@jT#Zgb({-4*`>ZC^Lb?PJl@vc0u%r)>hOkL?799Gflg6!-2pblB$6%k8!dn%;Zv
zr`_9=cK4sn$)8vDupB*SJK6of-kpEW*|hlb?bCWa-FE(C|Gg)#?%Vrde(Y`r|DAh3
z$eiD^?~AnEf^dty%d$CbS^7<FtC!5OxvRu$w<qk<UIWgCJ!~;s_b5(1v}YafzP;BO
zTWqCXz1kbZw0keZfu(y{_T1gG&3m&g!*^yoW4?cT8@7J6{lACTPSQkpufZ91+Z*3F
z?1a3`_lE7Cv3C_aw;k&RCA%-N%Wc(9?A`0PY44tE^2~chl6LGBl`6MB5wXwa<YkV1
zdtUhMS;ObQ*Ws|Ft(4(`y$r88ZBIR&yLW<5jIGd-xwdZ3=WN3=+HIY_^x4{1$lI<;
z>)HER+-mQ(7w`A}k!-R}++%55WZGhzl)S*UH@3<8!spF<o|LZLyCFNv=76+?%>~v#
z+dE6AS<AiKxwql|cbgU6^Y(no`(XQP!`Z#Z*7n(M&t$aSv9D^c_OJduDz=ksPWVXJ
zvfLN2oz-Wtcd5qQy?zf)?ae4+v*TRav-kA1yS6<FakgeJ7womDsIWEo{c6t*oBcKu
zC9C(GoU~_8&cf@~oBjCqZ9H#kn>4w7uY!z#?Ul=XHeQ(*ZPaaC_gFt&u=m@6vwPM3
zR_)c)GTvMI*LBar*gbn$YtQVx<yXG9!Qia5`r5C1=BQ2H(|Y93UL`9byDjkw`%JQo
zZ1+fp?S1}ck*)E3)jbZ=yR39huH7^55To5$35k75zxM5I6t3I5V$OP7<B9QmGj>+*
zEw_=k6FD<w@8_G#trvvd+Pm-Bmc1Pop?g}*gzN&&-`+FhPvD-V&kxyDzq8pb^q0qW
zd7;#v8!mTtTbw>(vx#A|&AM$<ZD#DAuqT51*&c=?v-aj3Q?zAx)Mx!UPR902O1P~+
zREPBr?y0r{F9r58bmm&CvA*8>dE45(y%T@zS$eEwuf$0yTaR__dmSV{*i`hj?5(*X
zXtyA)a*xKdg|-6qp0-W`B73GvF12C!pk)2WS#ICmv~HVcpO)Cp*w(W*#H`lVDQTVU
z^A$?_YV=m`J#p;#9*^~>Y(C81y*uh>i!JN_Hrw2dJ$r**3hz6x_|F~*`6YYSsY>px
z%v@^AoOpTf6^3RTt{2N}f6ZOI=f~kwdw&$$+Zwd5-BUF0;@&=9iM=yyw%Uqnrr2I`
z=C$TJx5uVGjmehJV4dw|fAPIB><jj!tQWQOUUhNzGM5#$qGGFU@6I@BV=Y&<ciEh_
zy%(C*Y?p~Q*@|n<w3!@u$fiK-oXw9;-90;v<?IYx-|u;p%CqM?FVDVL9R0TP0UP%A
z8!*`|N#@+UDrmj!gS`2BDmLA<S-S1kjv0&Y*qm51b#K;-Zd(f$lfC?`zxTXeDQ3sl
zJa_Mht$jAa*_`_#JWBRXvb|uNy|c&m%#JWy*@;_hD;SsB_AXtt_xgn^yYE<e?N#a!
z*t<~NZx6fgADa#5TK3jV=e89Rj<6QYoU!-GX<=I#7iqhily$asDkA$P$n3O<@nhMi
zvTu&<jr;(c9pWW>66BxScpQFXdq-t~?G4%eHmnV^_Ldc-+9cdH+g<J5x;IJF)V8r+
zV&9n)d-mE)I%)fRQpuh#U#IL*2{^V_kL~#0Ea6!;8{W*aRVb<3J4yMet#|96JsR`R
z>^*oXb+1F5;-2yuC-;b~J#9T(F??@B0*j5q+)CR67tij^etOk*S@rI{!W($^F_^O0
zgy@&ss)kGL6BJ{%O^{cy;XTi?ui?Z>+q{!!ZGFUL?6Pf??4}4V*gMftz%KjK%)MLI
z9<*hedCS&kj^y5m95dTxzu5Lp$how;Pqf|k=w~_GS)2dc?krB+yT+8y&Njn-?*!o-
z+bf(~_9R^pv{T|cu=l@O&E5sP4SN<8FS0&5W34sECr8@{ekr@Tnp^hdyfC+|+MBx9
z>i;_11)rsCcRu`L+itpa*PAUT_A30oV{_Jy({9o16Sf?G+HK2!`Ro-qCAZfszSDNc
z+EcbWZtmTCX8M=CV#eETET<dneX;t>9s|XGTg#1y_A;!tvTZlqu*bD@n(YP&UfT&-
z_ierWe(zl`bzqmvr8e7k4-VTeQf;=JZ@=#8*&w-BER|*7o&Ke^8GS9b&Ka9*7p&2<
zt~i*#=W(6%p2ePv_e|XIbJsD0Q`S8tr}l)*_1ybm@<kgL_afWdmRkFawlnQ>+VFPI
zylv^W^S-&*KD)Paui*<WTMJnaTc^i+_kNf^-KO2_ukDYMU3(9$NZT9Uv2T}VrKoM>
z3dw!XFB$A*+sU>!f628yCtUgV)G4*@eZJ+<?nP4K`>OhDY!lZ1+sl7*tF7_2(|hb}
z-tIkclz(4tzOqf|VzE6Z<bK%L7k2LPlU-|*81im!?GDC$P9+m;%HLkMR=E^wyI|JZ
zy)zQC_WWSC-;;Bs*w#zI+eT1xvCYO4X|@gd2ds6JKH01cp0c;Kf6JcCHH`b*G??~H
zsy=C>Xe6<3&cR1}w>_-iyYIg9p0qNKeLlt4_Eb;k-xDWQv-kG#44dCpmv>K%`es`Z
zS!BD|o@0+#>1NyAmt1WFdp&F?-<!7gNbK6ZN?YCb25&mP$Kv+;z5iEj+<SiC?7gXG
zC->Y8+`D&0YlQ9k>IZwhezw?J24CAdDdV!$<TcaxDp>H?&L~gc!#nZIo+pg%wywL3
z_x@P_+x9BMti9LdnQUjo_wP|&_SE)j-J?C33?jC&ZfEwU$BFNI^G?xLD|+4DgsvFd
z<+BoOcTZeoJJYOg@9RLuJtaSv+HzY3?(z;iwzsjqVXxM-Q+xG~7VZ69(!94hBxLWF
zd2;*wxb64ol!WhjDsk0jFUPXo2O6BMJqns^J>?(Uo|$!f&%*hNdww0ix>xqt^}YZ7
zmh64|t9$Qelk>K9leXF%V7qRe^doNXf{$l*8@e*usv61L-kB$2`+Rzi?Jp+rJv(J4
z*&dk3VEdYZecy&yS-YS$H}_Po%&~Qn>fHO1XVKmd?KXRQ3cl?AEyuX8W4gCZpZvtV
zpU>Ufn;Ms~*SJP(uZ8g6y_G-sc7NUJuxC^3-aYDv+ilzJj@jxRVcA>KxYAaxXo+p^
zV#_^_#*1ytUz+XxwSKX+O~!c}p-|4f9n(Z?KX2c=cSD4nZN!g@w#V5o@4b-|W#f>_
zX)`fH)HWfee)rM)L3<aiyld;UQe>aik_~&J-><O!;AykhL9%<#W`WszBV|<guC5W>
zr?z_HUW-o-wz>WnZ6}zVx7{#zvbDRX{2r5FuDu0$Ypl=wo4r@rV#cmh3x)S`ZtL57
zV8gaOvALRi<v&#IZR`27_mANno0X3&_uhFLzL!C3{qB_GYIZTA6KyRL7w=_Uz1=#l
zaH8!lm3w<uy^`N&ey(-zy7R5JcVk}dO}*J`BgnUQ?^B&kd-H#v+WV1<e=jGe``)Ra
zBKC&FTkZ{*yI}9zr>b^hVQ#iRKRehdNR;heyq3*YHB!QEv7zd|jy5&hCRtY7Z>vph
z&#nG$Ey8rrW<uNV-Qfy3HXj0i?q+hHzW3IGv$lKPTCINHYqzOyJ#Ql`C1zKCLv)`(
z_OZR+6<_bYC(Ug8U$1+wQ2$-q_)Vv6lwLbpos-nudvUtxK95Pfc4uRAY!8$>*``%K
z-@|^gd2jWBt$Sv0a@gv;XxVe7C2VhAt-zk8*T33snQ_8aZtDHLVw#ioUUcNxC#-p5
zZ^;6-ecQY^_WnEKYHKk6*Pa#9&9-@c+xD*dEVxJKuA7ZQjm&QMpNseYaJgn9p|IAb
z&S3T47raxfzijfdJ#tFjwnsL|_FL!AJ=s+uR)sT|_ubgRYS%H{!sb?o@7|b|$M;BZ
ziS9jon`2+K$V6L1*|fa|Zh^M7`W<^e?R&M?WUJHOH#Tec*nVxcP8Qp2+ptY^uSofZ
zJrzDRHs`L*+uOY}%4P<$zHP1)n~g!f%ii}U)qAZyPueOAE!tCOF1s)M+x)#7H2U|Z
z{#Li`xSDG-_s{vgE4NSIJ7Md6n+*s5*hFzpwAH`AelOq3&wH<i>hEQ&-?;ZLbE|C^
z@3~zw3TN6fco*%}K6}}AMfm}n<4bt#X55P1yS%^ARy^mt&AWY*_qOsa*gL_m-?sh6
zY1=C~Kld&;@O$sJsO~+HF%7mAT#<V#`@;8r+_K8%!S-8wgMWD1L<L&!`SNkIt>ska
zJvExPw*Dv9*uGm<V%=dPYJ26=Pg{oEse2ynIka~{M7eFk=1Y67eV5!TI)iuLgIDgh
zcMnzD7PCp(C~TOw$8@d9UW@8-TmIX5dw-X#-n;c+r|pZR$+nD=H}*QjoZcI^LDgoj
zyXd|^fjKtmFEni*_g}TXeKmE@iLFVt`j#*DK5A0iqo;k$R^i$|oAUR%JJ@x$?OpJN
z!P<90*xuQD=I&Md)Vw$6%gx;_eN*=`oXpsz^I^+grFrfaKWBQ`PLu1hadhCd6K8&J
z!)UgDcZ&N7TaCnhd#|73wVk+<d0!h}&feu4zU>T5k>1Bs=C}8{^u)bu&tI@rn)lUK
zYDcK;>?2e5zS4MR`!K9~cT{MrE!R{HJNY)|y?L*i_I~GIWi#>HgFU{UT>BRE-rU`M
zR(x-QcGlh%JTLcHe2d+C{^(tsTfFmaG95qeb=f=7wqu9i-tcETZC|$~?dkr0X>aLA
zSzE*Ou)T9NPwYwEe`YV^F1x)`R3`0lQ{88~>-CmBq1HV6c#nkdU8uj;#;s4$Hqv74
z9<?)U`})^-?v-QxVO#t`WzV0m2wR7o1G@z@X6$9D>#@mITVPwV*v!@@-pV%S;9d)_
ze`>b9&ogZn2<z?EQ&hF}mvr3wWzrn0d?S^8-+oB%*>OYPw!p04R_=DwUIw;lHZ5}s
z_ReV9V)MZMf%SwQ6Wf^ep?epU%(8v*p>gl2%vHO$9O&Fz@FaK7?Bc053|GQ-HBLCV
z*W$P1o`xrhw$TE)b|$y<_X=c8w$&)_uv-(peXsZB?fa%b3Ems0x6qd5*aX{vN0Y2Y
z?2g*;+z_xeys>@nCy5h#8!}z?{F$F++n{p9n(0`t9n%-~z1F>J_iEHnww)uvY`cMb
z_ueBZ{d=5p7VRx<sI#51mD5&Yj<Ib;?$<pX{@ZNb#1`7lc&KjMA+*<4rQqhCmhY^#
z6^t8g)~RgU`)Whez8g`GZ8rR6v-OTnvvsJyZ^Pv_#V-1K^4^%7<u(?p(rqO6p4!`x
zeROw-(6>EX_WIbZaJ;@JPjiB;!UJ3Dh!SO+=S7iw&+whx`+z58&!2}|Y&+L)-1|bA
z%jU(RS9@%<#BB{$<=O_AX4_8C-);SL&Q4ne*+(|!+q(9yc*AP<ruu^Ih4?lbj}=<B
z5r>&<boO=H8F-$v)mya0_DAb68*zo5djn2?-K{WJ!{)%7lsy@#qI(o#755p2&fWW_
zX6>FS1+KQ&cg)-CcP@M{vy1iKIhK?6=5-#it&kJlbK;Tp9u|#TdtU5vwVmd9*4Cn(
zZO^jMjkXag5w<dag={~_hU~GK+-{S#Vu|f5<|VdA?gZ@JAUWUq&xhT6I;Kys+wib#
zZ$aTP8-|vOJr%rn_M{k2+{Y5V+m`9+&b>1XdG<a?53>EW`M8}5gOzRM!O*>bG|$>9
zwcGE_km=adv2Ba37RP$qYfSBXUw!J?`=Mg$?f|2;d+NV0+xz5=qOC*Z_dRz`hTDq$
z4%j<;%EY~v5tsKymoD1(b%V*?E8_CD9!uD5bSzHU9zVN&uWkpseI57Gy$?7`b{l+p
zWW%t?$NE8pqV<i*=WS-#FSDK3w{UN6{jR+){8rn9E}3n&X>I1-2fYG&W-zeZMl9O3
z_g48qTi42id*5W9-+M)eVXw=PCAJLui>*tXf9`p-=*(V`D?EFRyf*L6dFf}HHFe?M
zaQP#4ADWVFbS4$--Br7G@3c7%d$(mR-a9R8{oV)v584LY;<WW>RJXk$DZKaY-mrbY
zejKn>n4n?%tF3i!%A+(}zHf7E?-;z>_hHfl>zq|Nw!6Ed_O-F>+`H!FF<S@G>9&>Z
zxA*<p*S5E3O5fgfRnu&9R5shxE!bsybj8xW4KKFYoH(9uyP~mV51%aKUZcZ3dvg@+
zZPzUl-CL`;Y41icW@`n%#627t{Ch7xoxAtLHub%WH?QAYlXcLRgIC&ine+u)iT@w=
z1S~qSFWUXzzO!GK*{!H@-&2*a!0y>~#l2xZ8*I%p&+dJwa^9|>q;gLP&tcmT3x&NW
zUe(%mRc^9P_;ki*NAs;cdWGtHi{zKss&^i;eXx1At<0ZIHYt-f?s+hG^4?vEx_f6_
z_O<N@m9kxv>}NaSSjt|rl})w^qV0QHW_H**q#fB+Rvo-g?SSRpFZWt)1rB7}9^2Dq
zyR&WfJ{yL)wno$9?dB<J?3E6kXB%L-$mY(wJ$o4R8uuP}esa&2i4*oc_xIT+kUnG2
z;Thd_KW47k=Oh1YPfBa+-i~K0Z7(lL+dJc^fIZ*;X|^3XQ*95FFWi0MPrt2*>yteX
zGJov$FxKBAGVio)qUx4?43~^-+J(06{WZhVF0H1~cGtazz4pgH@B4P+@ZMDwCVT(%
z&9-UmvfcYabI0C;eXI96cx&38SY}}BB-CTOW6oKd*K5x0{Z|{jH?8aIz7rBxY&h&&
zZNIIP+RLnIyZ2K2THE@!o9t5F9p9r-!@uXqnxnQBOI2(wvzzvs3hmx|<4p733Ud|P
z2-o?x(w8##wuFh<9?8wO{UN+@@78pqy_fo(_deKOv-?47kB!@n<9p5SG}>+mN!eq1
zrEK4m_IG=J?6<J}mB_ib$!>$~k(!Hk2W;Q$QRpn){lo6w9+T@E_J;Uvu{}`m%bG*c
z#dhH*`@Lrs4fggkP1&2$x54H>+@{?u3uf84Z7JC+yMot#N%=gR*)P}c)408F?;rm)
zw)ZS=+bO*9wkeyyyI1PoNm~iYEBo9`)c0Qac6-kk_se^}W;E=5Ip1!t=&4nEi&pKk
zo$+&xZOqI5y&sC^S|@B>ZnJICIa~Yd<$FbL9NCk(%xmxMi+uL#)f#($y_{wHV{Vy^
zMbR4@YxmN<#rI10K1-6`d+tQ9EsMi;TNCe9dmbz_wUxbha_>jkb$c0n+boUdZ?wJA
zFyD4YMXJr~nU(uC@>=cPX;o;~;PS{ufm?G=#9T8Q&S(646{i2+(-Y8a)AV$vZP{YJ
zy%!4R?w#{=xs5=~Me7qbJ8dOyGVO^;_Oe}Iy~tK@!QQ<^Z{OI7XfCsPA^6-T=H|*h
z2{-C&8Kh6{KEUE{`+4>3y_L$x_LeK3-0Nr2Yb*9^#@<El#e4S^_1mWXH`?3ao@ndl
z)@?iQ$LhVej4$p}++?!1uX6I<h`!3bcYo~JtFU~VwZqPhyF+GgwsDwQX6t{n%+BS$
z``-G$^Y;mS@wGj6Zl<k9*nHbdj3@Rzkg>5Xk}}>qv!>53;!W3{Wd~N<3YZ<*+Y=zV
zS5BDM?vdpQ+gn*ads&K(*#6Dsw|!Q8e(#xwjkdbt+wIOol<xiTRNmI${-S*mJDY46
z)QxNxz1?YhZsWGSd-<Q+UYxmpZ_kCBd$M+~*&D{S!L~t;V>d(Xsa+d_Lu`*b?c94{
zevqw<f3|Idg@$#7*!?{XMclT9%M<rr5oWjbv8l6l+pyGj$DPl6E@*PvUU-?kN5c5f
zp0eVpwk4&Kd(XJK*&a~7yKCyy>3iEwpWG{#c-;0z@$5ahJxBKb*|BU7$9@CbjT#ku
zZ|w5ko6#g|yZP(xJ#yi5tV9-w?!CP2l67w5XWJ*{i}srT{=BF9p_lC`b+*0vjL&Tw
zzLxEk_|$3hGH8R1L9wLuO}>P^A8tz6o-$p#ch0fRdpn}aZRPas?W{FZZ7;mfw$XT5
zU^C?b@7{x({A`c+8t<*s`>|K-=z+ako~^KL>}a-?UiRMByDY+XN5ZB($D^3`W*C|6
zeUz%akHP8r9;sh<_AJzyw)d#s&b`<3B=&8pZrUs0vdgyG%y;hs3;8|bNptr`E3?@x
z+0V8&NlRvL>4na{A2puZGM~F?E0XqSkH?t<dn$L{x1B5W#MWY-@E*0(pLX|h@3PUm
zb!Bf&HH+=i+Ml+;>uqhnPY~YAb929K<B6I*7qlw(9x0x)H)op4zO0KyyP}V7+jBy`
zX!qt@eYO|6PS_lu)wcKeG26W?Ei?E2vHQAraWCh-lg=CV90>QbQEB7eSKes5XPqb0
z-j2$fd+xZb-pe7vyYHd!y1lR0@a=oKz<969-TQmlpOx<I`L|^6!96^7oAjA%?q1^9
zXArw%ulmk~Hcx){?Ol;S+os)Y`(CX!Z(D|^KD*y1hwV)|c4?1>$-KSB=@0j=KO|<8
zaB#WptGiolZ$*~d2>bHvn{#0M9t}H5+gj5nwlS4c_AZ(8&AQ?P(?0z|@qIf|cx{>V
zpY6G+yTbaL!gSleA7|U>YpvdUZjpeU@0BRqeVw9vOZ1y;r1z-pE;&2JMu+9mUMHhu
z8(oHgz0ap9@Ac#SYNLJKe(#<y*Y`-7-n1^~YuI~Xs*0V?M~}UqfBD#iq^;ar8+XU{
z?Da>s7n)yKPu*d?SEBFB?)d)?_dc2GZF}_91)FC-+4j{MZ{F)6x@50o%#FP|3T=A}
zrF?8txt7?T(zvtNZ8Dpk*mJpk$_meHMSjZct+eRe8#0OAcALgC>j`~wdzYPly7ymi
z`W}xtS8NJ;()Kd`PT4afB+}-qr|;hXKW^9@{Qq>XncUO8rxknlWL2x$8BCGe`~JnD
zy)tUHwl=0r`vMB4?iHMN-1fooox9a1huHFDeYK5WJIQ8gRKebDLB6(0r!4n&Ywq9c
zwT@@+DqB6<Y1%(*v=uaLXRh?NS#f;zUV*bbw)rJNw#MIlZB9QrW&8FjgWZZ--}Yv_
zzPPtk?%H0BFmBt1qXoN`v5VT~Zu!6Wh%DQ_3l=-AH(p$~=c&~7J!w@Z_WEt<*qgp<
z`QAA`ezpbodaPydUAL)Px@s@i<Fk7NUaj6M#xAzcxO4N~m4Qzzebd(1ni(<eOMjkW
ztAFFzp4d)~y@7L<?Om{9w~hZ)GuzaMZ>)KBl5EwDyZ3&q=e1?JvwUx8(U-l-&u-Xk
zxw_LP_Hp;#mZxiNbe;Fu9$ox+4~O1N8!71sTNVw+z4F<Q_v&e|?(^K<ZEG#NYVYn#
zQv06Ikl!=&GS5D<hc$a!1efkL=JB($_F86hE`5pZ2DfK>%ZuA~FSg&f_h>`&-Wpla
zy>6T5*#0X$Y0Kl5v6r1$bnk%`68j!}kh5OUddEh71;@Va)${kx{~==6an8ba&6l+{
z1rpr0XMb4R8n=Gmv+wzqJ*^6p_m*|5>?<wYxOeN(Y@0aC2YdG}+PXI<=kDGK*0XF*
zoxW~!H%NA$k$&D@y_&6ik0uH4TVL~VZ{H#*TN|Iiy}z$a-96=2vrTcHyp0q0Lfg^`
z)qP8DYS@WP4zy(uyJ6FOD%YyZ`<$)&N7j7|E-!3D`||c|5PP=UnbpvCddU=9C6haQ
zbN08`8fPuv%M;0FE1GwHugIC5wt|{Udo`Ev>^o7+Y_s?0!M*>wyY_m?CGV<Qpt0LA
zGJ5aUR2f@S`$AifW}$t)VUo6C;tsZ71!wM^ayWW#o8)(!VkvREZED4P!s>Tft(fbx
zhv`{^4TrbJo}DMl_Nx3g+Q&LUcW+YX`Mndu_SziC6y9gK*K5xyjp{wjKkD~R+aSMB
zd~)XA{!ZOJ84notbuH%Hd;OfS&6N75dlRR7+H#)YvAur#(BAVhn{2aRvF=+ln`JMH
z)ifLP%u3s{8ohgO)vNB6j}x^GaILW2Qp{v~a*4Gq&)Q{sbQXWKQLp~8_w=TddwvV?
z?BjD)-OGFV|L&#Yk+zmk3T@X}GuZj`ZQbi&K6j6fY_M&yqSU@vVO~3r@F1IGy6<gL
z*In7uGiQ};lC#*pYyG)<<x8gTIkR?wZFJPPJr6z_+V17}Y_m&st?i0A>-V%Qi{E=p
zWUY-u>gm0$uJ89oHOTJWI)~Fv!R^&v`<1-=R{of|x5?RPr|KoneFgJP_GZmB+iRvg
z*?N!VCEJJ)r@e9<&9;Xty!KvRG1u0eTg3K4_tU)-`qcKMg-qUievihU4cnjW*$`j9
z*Gh%oc3yAj-YGTg`(}6X>{Iq?+4DlLcW?0l`Mo>N7TbPX7`a>DS8|`_w{<pGGmh<D
zxw&#Lhd8h8gz(L_QX86V4|&Az_4(v%dy~D-CQWXGt;Mvbdnf9BvgO*j()Nkf<~_EY
z+xCQV3hmn(D6qG@CuMKZ4Gue{^=tNqDX`mShP~a}!O*>T*;J=J=F8dksri@L?7mdK
z_fq@rz0dab*zjNcYLk+BY40MhhkNT8Wo+ZR<o9e)`@F~C9ml@)ZQE>4OfKDHy`f;w
zk#ep*r<pI>uKmonui&M(t&M}*-XP<P))#9U_SPFNw28ZDw?}7t{@#D5w%I1n+P}Bs
zVzh1CDTci|s?%-%NUYl9_Fr(Ht3A)YAYJ~wF}=HN|4%U4`z1TdHd$lG-kWL9_skV%
z+qamnaIZ?<EE|VHfqh9U*6b~5xU!d9Npi20)e>8$C|%n{<}&-jIy3i5D+}3u)%CNr
z(`DbQkTcijbCvwwPktQx<~_HtJ>Xus*NMB^*8J}cTdx!Q_CB2d(V9c=>z)VmXY4ib
zUS_M%QE2^OvZ+l_!B^X;v~#x8LKoX^TXoL%GDH7fhUXJ@S_pO8@SHzsyCG-k-u~|E
zw$3sAdktT0-@D})>)w?e8}^t*T-%fJ*T>dqU&CGoy~lgA|FrLY7tw0FL3@sk(uq&D
zYpkB_tys;vr(%V_&83N|d#~56-Fw0Bh7HG+9Gi2~O7?ni@$Pvga$;}8)ofcU*0jA0
zRX;5ctkmCQ^gU^>M1Z@^AEWEGELW2DE?A#pTjI{Nw}zv0uY~#=+bN+>_pDy`)>`oI
z#JvocwD)T4NVZMyWU<SbwZ*3Mag^1PVlCSln#XMg5>DC}U%YMmyheHNPj5lHqTbVc
z<95%nof<xSuT}bH8<uGPeINcl*loPDX0OF+d)osHEW3F&a_=j-I(@hJ%{W^-jput^
z*#GZ|Q%T%waG`ncpC}<)33Fk)Wxcca_J#G^uFK}%*Y>AxZ@rrIUNcXNy%7(x_O#ud
zy4OBr>E3UPoc2bpP1~Ej(rk}I;^$qPDtm3iHuKo2yb!XDyI#0At>vg~r{bZ#6B_2(
zOmIGBEA!vMRwr`C-UZV0Y!<{Xx0%nXxi>X^flY#__`bR6OnV=!HL>M;KYOphpRhf5
z*mv(e`c}~Hnt1;nVTGh!8Png{JU<({XTs4Nwp#txd&9f~_pYhmZR_-Sjcv?IjXeT_
zPB!8D9^34ZY_R=wS!^$h+8o=6PsTPXE39o<_BPp`SLC-#;OVxmI^4XQW$U`VDGYUc
z+k!UQ>WJ*Je)n72?(W)8dpF2lwU#nqw*B^Ry6vuOlWh5?tl9g?{Qhp~H5Y99u7BQR
zkS4p&L7~Q`-hb;}p=nR{2wF1QD)Xf8&7YuXn=IJ4mv#T1y|0oN@AcWFZnK6X*|vmH
z(00l44YpzLrT4y@EWOXmBzN!bLJ_;XxM#M`Pv_ajE|1t-ttVx-sbTM4gXYV-Kk9|q
z9$gl+_eE*z-UP|dHg6Jo_vpXy-klJ4)q3)$_cn8W-Pv<5+;MODo_~8Ug}&R<#?iBP
z*_YL}yb7Xwo7-3J-Nn|tSEEL3&l52dTb@1rd#`W0Z~N)Wj6FN-ZtdAI#bqz+GmX7_
z4}9HgZ^>x;f8n0Ji)5JXo~{z!_uzxj?hP_r`$ATC*&J}0v0FrLgYA>UnzkIf*leCR
zH|*VcK+JBY8;`AD@5;Rq7oXW4``ct&wPLmHL(A{B`-F=2F1UKf_QCA;HVhtHY%7je
z>}kmevNhs+WXpF^aqrU1N46FZ4EGwShwhnmcdhLLt<%;zmuK!-vQ%QPN&l?f9{rp4
zM!X8#YqiDGw);T+-d&TdY<V_M+H13NqwVc$l6E==n`{qEO0YRs{crE&_SU`kXDrz(
z$FXZqagKy-$<H&kg6VF1pTE0kBht#dSIS(-=1QKNT}R((8^6%Kw$VPv_8xCpWt;8J
zXscYHx>sez7Tc%1Q8ugN|LxT?IB&aCjBW4!*Dbb}?Kj$*Debh4beFd=zY)CmVW_NK
z&D_g-74EOM{brH1w^ena?V2Q0+lty&+r-NAwp?~EZ0|~K+v75I?e6ui7i<jf{NB@+
z!Lx6!-^RUbm@n@6@H2hS<eK$+*XZ=x)*P|6HGlkXkH;jly$*#}_Bcczv;C33)wV&n
z*7ndOemm);*|s`7KWq<7>)3l>|D|1(ym5OQ)Z46jcqZ9w{Jhb&V3*Mz&3L1|HPWoM
zi5quX_qr?Xy|6vqMkIaj-pS`H_P9B)?0s=-%igMm^K7|I<ZS;P*=n7#xO<QAgyKB{
z;+=aM;uCHD*-zSAI9bLnW?$xB@#UBH>Pc4Y>0B+huW2Kr-L9$fc2-$td$mtp+RGNs
zYx~6Ull2x=!@VjpqIMchzI$etv)g(YS=z44>e-t+=Ys9Z-KKjib5!iwCtu#{Y~Z^$
zp=I0NA2N-$S=C2&-<f$~Pl4D>+sFuI+d1#f*=8mQ+No^u*!xD*%=SjU>7Iwiw{6y#
z9kWSnwzrKaziZ?Ep3PP!aOUo8lg)cBas=*mGdQ<*<LY<2vfeQ7S+S|Zc3G3E-Gb9?
zyVY;2?tOky%(lUUWsiZJ>7E8Y{=GLecJEn{@y@pX!2{b2X9nA@iR<>VC;i#mGPBpV
zX9?Hd>(;YvZ&XdU<>F?xd9cH5kBB#u?W)iBY<Do4+Iq39wS8-=v@blB#kPb0m`#UC
zrLE{=pS^~++xJF&@7Vi=DSx-pWf8mo5#Ox3MVIeww)WZk<-ECV-t~WbWImqV`)2;~
zz1Hea_G*;8+LM%!wbzSp%^vO|^}TZ1$87C&cJ1ZLI%2ytSkjJfF2g?lz#!W{Vw3jH
zO4zbD<r=?ja=Gr_Lx<1p-4frw*Fs?a9-%3}Z37uT@AYWevUkHTew&6Vx2>10ePa{5
zmdDm&S=Jt(u)aMDv{h{8^DyuGmGs0`XR?HC*uN)x7nCsXS^KnlZ)jJo_0gFQwwabK
zd*!dr+%wbr*6t^#;%(i{_t-N0oMzLPaAGfC(@NWwZJ(_*{7&v+?z(Dwbeq@i53VM*
zW*;W**|4#|)-rq1-rv4Ic2Dx_vi((lc5j>7e$aUpdsj(3-lN^(v-f<{3)`9yJKHP%
zqI;84*6uA(wYF_?Pu^3`KXEUY&L-Q9F7GW@GaTA0zA0|6cGvE`-*UFw{xIBVt755V
z)6zS2?}gX}d+voyw*A8`Z_Dxcy3LnYVK#3jzP53&>fbB9cdu<naFb1j_KiLKEFHED
z@7t_4oSJAW#+zyT;LuTPfmySx1um?#ogn?qRx4L=Z*k@%+x=3)`=)AY+e(TU+HNyE
zz2`-(hHdN8DYhT%R_+n~{c5j+ZoloFBr)4-3GKEvvg>Uc{?=J3u}j(=PE**sV3y_H
zoJ)GPKe9Y*Pc;A7vtj=?8{IzTJ$<H|Y<CJU*-Z<U+1qh|$F_h+&}Q$G6ML+<xa|(a
zHSS7bUb*LflH4AHLnV8*8LzgzW4_&H<+C$;1iwGn%fRqp=NF$-wiY!4dwDr!Y|S*b
z?|mkDz;@Y{Nw(8|KH2;86Nl}dEOuLqrq_FFsse5AEu6B;Q7?CwQF8lU(f(()AEt@e
z$~zd_`ng@TJuAL@uYZ2X?t78DZR>@X+Fq=mv3H8%yFDumZEU%_Rjt?_OWLI$+O}ui
zncFt3&*k<t{>$9^Cs%!M3%90i;~6EpBkMNp1=ZbKs+4V`csTcMv6rzuU}9-KgY%{J
z#;Hg4d}!6*qgME4FZ)g{>kcLHeOLM=>@2?@u;p3oZ|gjD<KDLSt+od?aqN>gy<xA(
zW*ghTGBW$li5}RiVXVB5IdT48qsSSySAKKaOjBpI(+gwS+n||m6Y_oD9*tg;Js~Rc
zwib4>dqp%@Y-NH>ZEX@|ZEa80?|s{KXzyxIMcV+=UYiS>l5K2!J#CwBs@Sf*EWG<Z
z|883i&ZfNzev117Qf&4He?PldCCAiObG66bdu0myIK%nup4?&ByCO=}_WOa^HuK&j
z+8%hrzwf(%wC&YIaoaS8(|i6rliFLtJHhtjZLhs5Zi2R7_AcM^{mz6vE-%b&XZd#T
zWhk4!(<<Yx4d3Hww)ZpS?Y7Kh-N!P!XYYlo3AR1=^7k^cGTHi9U*22YX|}iibEAz!
zx5b_&qa}ME<Vx@6Fp%A=@6xyDq|fBNnvGR9TkcfZI=_8md;S9N9zQFey*d9D+H}_K
z+2gR8ci)CcUEAerN^Ot4_S(B=uZZmmKO@^;DpPGWQXkvKPF`;N`~UtuTwRl_eR3A=
z<=38I`=g?BuV%?B+pun#y(h9(?v>ZMW7|D#rES#e-n}mS7wqlb`O9WWvy9yVE=lV|
zA8tE~sLM8dug>mm-h9(G$@1i$gn33bzg`^LJGZOLHe_S(9-dZ(z0cog?-iK#*Jgoq
zicMDd{XKi9bnQwxJkM5(&vdWE!6|zu2ko@sNVl@>IG<~Cd`ID)Ir;K?7td7MyZj;J
zzA0}kY)?G;V>5SO=3b8A+`U>y&f3nIxn<8f!6&v8UNYM)yL8;PT#L!>-m<-W`eKgQ
zx*m|;Ym~os@0<K7wszGQY-23v@3m^o-K%rxs4aUf-(J^MhinddTibq_+pzac)8joY
zA}97#yic%wrXjL-PK=oC6qjkXhr-47uo$WC+gkU>w&e8;TaDKj_SCCfu<89OXLsnE
zskMMmo{hmREnB}iJk|+kzS>yb6}5T8_;Rn~^H(-v<~M9M@E_dECfa2qFqdzyhQ8X~
zCz<=KimHY8iJhHc%em~{&I@rod-tt<Z5y!T$nFiVxc9C%(%QS>a{pf0hb#6@{c?P7
z#*#g@OT(mXuYDKXE1E8_SHO7nUgmm%JzJH<_VsR>yO%lGX7Ag-#<mY;b=oXbliS-K
z@@>y?-HkTqn%~;&ofo%v=fP{X9`iTZnr@l3_wl*^dwwwZ+sp|zvz=hJWlzG?P}>jH
z(`}vp-`rDlNpx=r$JadzX0Y1otlqoFh;xT6Z*usan^Qj6tUoSbw?kx?t;rEy+vLlp
zd)sGh*()LyWNT3N!Iop^2b<#;t@d6JYPS~Yc)fSG#_GKb%&hk$&g9s4SnJ$gh9IlG
zMK&^fdlcJkFH7g|zNo{xPh`#4Jzb5n_ZU18-Lv6V?`|=>qqgg|J+oatd&k~AG8KC@
zt=H|jA@XGJ5q1{aTb>LyumAM!$t`+ot05R=`_;2&@2_14ZEvJ4vsKyHzUNlU*1Z)n
z6ZZBlIAeS9=mXoC3)uE49hhi)Y9`m-oDZFQUD{l%m;IT!*Z7&d-9!#A+otjfdxfnx
z?YXvl-X5>xu{JA$X4rnINVjd6T521zh}mYHJHy_{!wI&M6Ta*fyY^)F^RzE}uD$-Y
z_tj<2eY2*|v$bp%-*@%Vo;`J%EOv9>3E8dqu+LWSU#qRnC6T?q;??&iWR%%(TAkf{
z$6>9_-7SiCyOh`3Hr%<l`#{_ET@`;j_sriCy!V5!w(W$NUA7EA>a1_nR@fep`($Hc
z@qSOU%y!!g*VpgiXnSSjbJuY1g1_tbrWj<|Zg|aZ8#znGmj5H;9u}TF+hfHtc23Ks
zY=4SQ*<14S%iaf{me|aQo3@ATp`xA0zQcRU414z;G|}F>BcIv!md*LSE;Dx9gj~A1
zr@_9*CjWnz&5~uFw)N5n_5>WiVI7}(clYK!%=^Bany@$EjLlxXd$Vo79W1um(Zamf
zMqtBUtGA4H>oczG`T0|I?}CpjY<@_K+wL~`W4o!4+s;^3-qyf?bx+E@>9$7C`1kGZ
z=Cox$F?H{An+-NP*XP?By*g^U&FJ`^I};Y|Et%Z2=N{;eQMrfKDUWPy&u;!~yFB*0
z?bc1J_CDtJwee48+UKxo?w*M{7xo?~O|k80Z`ymsyJt_s=Hq)Z%vRgVE56ygrBmMS
z!-O@vnY#AdN-GNPyll{7!{gVn_s2s)yDv@$Y!B@4wvEtwYI|kn{yjO$%=;SlmhTl4
zYPW51@Y_43q1;wQWUuXxv*&Cp+NAfoJT|m_asB<?omV7m6PA|kKEHpRZA0~Q>xNbL
ztvW52?s?_HweQH5EZc=o|Ly*8{mY)Btb2P?*f-nWi#cH%5y!Dd(?iz!UrO)Zh5L5x
zEmiBbNzYTYT~?yLx9`<;+k(i(-41Pv`{W%xY()-m+s3%fx7k{zwRc(f)V-TB)b^?_
z%-DOkz-_OX$_886_@2E99m{RxS@W#bFK)BSlFGH^ntWmJtJ<TsKGv<aQIqHIoo}tZ
zH|bh~O~=(Xo0moZ_ZD1Tv3F0#aa*B@r)`v0+_8NhuzByDO^o~ae;nQ`<}9}Fv32iW
zmn$6m{yby0z2V1ZTftmun>TCe-cP!`d)I8(y7xkE-kv|ZSMFWGxMOdUgtkqCN~^7C
zPWj%1^Ci}0bLQJB$cOF~iz(h4HLqvy&jX@%6Xv<>$>E!7yR`6>ZSjo8y;3)M?RXev
zZ8bARY&S4T*e;n-Z7Zp^Vy~(B2HSO<oA-9Uk=$o2FmX=+=SSO`OC`3A5fk^UQ0cSj
zIxV+X>sZ3xu)uw`3p8d}=SlCkKC@VT@1ldPw&95)dxeaa+s^1y*elyG!-l;?(QeNn
zK3g4ExxFg)-|U?b-fz3*;g!8Lf%>*hC4#oe2TS+<dU3%v`0G=fSME#q{xQ$oJ2m0l
zu3h`j+MY8LwdI^YX|F=5-`)?K)?1q#du}_U`sm)6whG(H8_(}H*lW5cz-X;a#)l5u
zQ|+s5d$<(#{tv#t=ftK7doz|g+KS9OWb0GMVmF6Zdf%<4SGIZ0qIQiRnQb|4CfjB>
zckf+t<ky}H4gJ=+*J^DQHLu%Ryp^&2@pHYcgtWY!PuH?N#*rWQhHrUj6T`B{R&cA7
zT|0l%-Vb;5Y%cmguq`+nZ);%Ky?4q*X}hiuN_+S4Xxl#Zp0ihe>lPa+p}n>_a|G@F
z+KAa6{(sn}pqz7WXrzqYg1RfVF6%$;{lT$kkND*BJvJBS*zn}v*dt^sw%7QGkll;b
z0(%3WoUkb{RoNHNztsBN(SSW41C#d%SiRa?b4=S-e_`96%U`DKHGI2tZ{#zUy$6>3
z*`wgvWs~>q<ldU;2lm#x?Xaz4|FZY`^!`1&9vraUYBbm8e(_V=16$1ZvKBtFS@87l
zZuULL_m;-avz?{-ckhjFvus`1G;Obb{9&VcSkf+F`;6V%vb{D8TfbOZZ+)?s>)_43
zN4Ie9+j6+o_JYO?n}+q*_MBs!YWv0T^xh3Zk+u?1Q}^6hxOVUQpj&(9UA=3g9ACJ1
zTJ=g>Nu33IC+t49H|Z9GolNz-J*OU4S(_%_-g7~^(OQI2*LK3(AlrYtckY#TEZqCM
z_=Jsu#-rUG*CyNkv7TqMf0L@6fbR0$JnuQ}_TBBaQJL{)54)F!txfg@+gXnTcXKV}
z*lY8&*S6>1@jZFGQ}(7Bf3)2>>)-Bd*`IqBx%Jzo%xm7u5WUd)!rQ<-CLL34i)82Q
zX=Qn6YcJTe$1&>J-nE{hb`B9UZ5dB}-lKC-%I3Auwmmy1O53fNb#ssCqu#v=#f$e&
zkk7Qyh*#LVfh*Q_>63Xj6{06?511Xd3FPM6XL7q??+SS>+kNdXZFVZpvduUjZxdU7
zcF#$PKepL+^0qrn-tGA@QGajvqEmZ~*L}6&lV4%Gw_Ry(U97!rsqEj~+Z5LBz3|+{
zcI$7wy}D`h_tbl2*zCx0u)T2egw4(~^Q|7Vt+U?Hu+}zZHlwXW@$@|&X9{h_cvbd_
zt=6=ya#(14bf1$=@yvO9V`A3Z7Ca2Jy-<>~_sib<dkpv&?GC7YW%DkccW<Wn?!8l%
z)YxvxYO?XoS+e)~r<XR?;^wwFVV`YhY_GQM$?e<Qdw0D}*gKKEEnDPlU7DZn&b05a
z{d0NA-t{*o>|Hx0dH0nAt$QxC?b@3)>B*k$mt$?OHY~JV>K?pTB<jYVj=8-y0W-Mw
z@_yd9muo@R-UE7HcJqcz+^fL;+~$av(%x4tb8I(GyS``hSGK*6CVkvH)9sn<wh}(O
zp0u{TQ}QJC&OEnb7sq>#-IHgX-OH%*($*pOfz9h#Gxshq<Jx<WMc7v9!71DFmA!jE
zuWH&OBO$ieLp;r9;(L+Z$I374ee_an-^WM!dmV~p_Wr$kbC1;{f1AQXC3|LihVQ+@
zFw6GO`*V9tl?3eGJ*~85&DduXYawW-{MyF$rPn4K*ByKI@_6jq<7K?bR$|}PJ%@D#
z?Ks>5_ssNdvHj9uZ=1&0zjs6UbXz0oCfmr{8GA1%?cJ->yl`*)qGP)avn6e}{0_F&
zshex7zv<wfNmYw&RW{w(V-ek8tCQclH@jNVHbHHfO$GlNTLZBgTZ6E<doOIZv3bnj
zwwKdR*-p>ad#{I!o^8e55St&8@Aew(&bBo_^LNjkXzsnA@`U$wZWY-hzN2B+Zqeg=
zAFYtKjkq+!wrBmyz0rq0>^;HSY|~ccy0<5L!`@GyW$e~$K5ct|v3)P|%(UI|$2ZyX
ze5~7DAh*Nzn5h5Wh|cGGHoQ2n_sD^XHk;nKTIYA1*;}CU)yA5c)z13fI~xJr*)|>>
z%6k_y2=2aib>r@7ix%42o!!6tk#@iBf%2xkPybo&%~^ZjmdjaUPu3gRJvtAqY;T1A
z*|Yn<qD|7(xwb#{o!`UCaAL3Wp+~zn*)Z)jTKma1V!pO5#}_tR;TYRJdeQf7c4f8M
zPO;x%{aKd5w#4j=?J7$FTbI`-_g;x&uw7|eyO*U#bk7Rb-+TNkc<hpo2<*FZncwbh
z%Bj7j`!?J3?_IN3Zo)#_N)4fX2{C?qpQ$kI=2>rJ`<A_6@1y;dw&(s^@0}1mVb8(;
zHhW8>&g|u1lDgO8?n#?npH27735>Iyu;aOHjq5SnMV-@amsT<E`*B2OujG{ldxJDh
z_q?e~u-*7!qV;vDHe0_?{=GL|tgtz&S!TPze$w6wzD>4{f&shFC}i)A`o3##+qXx1
z|Jv=cnUXYZZ~W=Ad+k>2w>2tTYWwHlEL*8KhP`qV=kGo9>bFhAABH^%5;}XQ)lRm}
zSi@*@xMBXDY`vMgs&mS1ul60;)3T#*?+Hsro7pDbduPs6*t2ro<vqnkEc=|^aPOND
z6=6F+kio9u%xv4vjHbO>nY;I{e9*CXovhTpcU(C;PpZi7UARGP@7s^1w#rPW_P&j;
z+q=$GX78(~tM@XdsO??m=&<*IpwC_z)dpLR@a=n_tlDLJprdc^T=!o#*K&pJj`Pjk
zJ4s31mg|F&ZP1;&wmDs0wx|Dnx4pIgrERho<GwpC5__aJUbK36SKRgh_j;=r+*9{1
zdgQ$K6_?N6@_WsDa}0LxIm5Si_le7=_VylGyjSa_++NosNqci$x%Y)Kp4l5K_rdng
z-*#KU+NCyU8PxU_9h<b*WV_hDgW+#&Zk*q^=d}Bez3m(QZRh2F-P>tCZBLhb@!rf^
zllR_BPuP2cxo)p5<6r9vnV3BiMmzSNewDp9ZnBb%M`rxqgY#|nN`}nYYuR7CNBP4~
zTY;IAZ4(x*-791oW4mKQs_mf!*}Xn8pKTJB#@T#Yw_xw0J)8HgV%oCz#E*<UykWh2
z_P&s}eQ~{VuT<poJ<X!O_gWfi?>$`LZBrtlWEU}!-S*ErcDo<PH`uK6Zm<<nRj^Co
zXtfoLW!%>?*Lp8!mgL^$o;J4b?;hF|R%q@sICO8Xm7vf*t{_c2RqM%n-#t&Z+3q7`
zoBCm!ZPOQLo4z{+d#e;|Y%5N^-t+ow*6uT1QMMW_;(Jw}_v}3v_-Hrx_f<AJzkK%A
zmFn1r=dH8}xhJ}>F0OCy=A%V>9NvaopAC|-Q~t8iX2B&c+bbIHZSy7acF$dqy*K1=
z+n(5_t9u(xp57B1$+GWLaEq;l*B0w9W;6B*@v`mxeD2I%k(X^YN@9w8=QPZ<{j`#M
zUya0py{FP8Y_|$+w3%9CWS99QZSQXPJA3N$U)dCxf7@$fyU*54Mb<Xs?rPiCpvK*H
zxtsUSm}X<!73^xWBi`6nA$qrMvkUh=jil^7e>Q%zxwdAv?XPu<_NGK=+V1l)*sH4Q
zyVq=n-`<kTr}u6ukhLwARN8BC>ATI2`iQ-^E8F)jUZ}cPg|X2l#FgLn-s8i2H*Ctc
zwONyGd*eI1-Oj^1tv6`5*_u|Iv7T(WXs>US$KFXHoA&bftk^qe-U1t?mD_E6*G=Ee
zEY-a?V&wuGiyHqu1`V#Zr&KI#P4=zWo3O%f@9g!TcN>V^vi=r(&UUt?k!`|dKWo;s
z`n{RYx9nZ0EnwTNc4V)x;QGA@v6pQAPwcWS`0>xi{^;tx%NK63F<fr4cm9!GHm08z
z?LFpOy7wAilTBfU(7whG8g@D#YWH$?MB4^Nu-aBEw%Hqy9ATR~vD_wcMTV{J)rP(5
zr>yKARM+ml&UD*WLT`&rVW`|bDf5qe=DC>J23fH0)nV=1>wJ9n?n7Q|`)1rfYvaGr
zd(ZkGr}uXLW8SCn)^hhusUF+NiQM~S7-sEpNt$k(rsHTkZR)GN-&=V1<wi>E`!dPh
z)@SV_>pPF9@3p_aZf{2Z`91wYSN9ZFIPBSZZpH4O++BN$-4E^UDP-H{{;+!QtU@un
zHTjOV`z%{*xu%@lTg*9mZ-lDIo|A{Z?hP_#w0mQ;b?*i3$Gi8NFWT!->SVLGtACFQ
zubHjzvE;q?ik9x>nHOYxOY4j6zjrfiB_8du-LP$nZE|<c-rWmN?~Tl6va98qzGux+
zGaLE--Fw$1_Uwr*XWExKC2CKMb%*7HY0b96ZI-r0w;1+zDDmuB5R<ld%9^UZetVj2
zSJW)9l}Wy4bH1{5uZQ*2y{5-?_p+KT+N-T<ZyWva=WZUp?7iQa*YE9G!D?HvMcwwu
z`JTOo@^|-&-xuHeSfF+94EY6nHii}L&G);%_u&o}8>V*leWe0#_eP5I*esE`w70bT
z&ED{)-PV5|G3{eW*s%S6{>i<q<=uPF*u_~JCr-9?i`ZrR-i%{kd$fSfzM9*%KTa|3
zwMyW(leKiTJr%FDhbOv!ud>S;>(=KYHm{AI?p26+zPC(j&EB}FyY^bK{<Q7d&A*qe
zKGW82Tb=EVmrZ-64)NOA#c#5`aht_vu@|fDad}nST#YSz7q41o6RH%v_tNYQ)^;I^
z`z&k^*fa!4?<*0RX?r8%ne76-#Jy*~OtJmZ*u7^@KG(kXo2U1jNStMpbEACk@efOP
zOC9>K=Nxyhtv=hNy%I;K?Y-7<aPQsA-1`hPV)ydB-nVyyxP*-XbC30dNy&TWKfbb8
zaqS<Q6Y^=c4l{o5efIs3&AS&1_x}0Cy>B_g%DoR8G<GX2V%xjTXzAW}F{w7%H{|zy
zYF=*3kR@aj=QL^W_ctr{cHKL=S9a>@J?jfE?n$|`#x{!eylpFwp<U|2rF&)tO7493
zal+nbam#I4-#@aMvF)yH|F!eBYUz2lhZm~tnPVkkdr#4BkB}e7K1q%@wv+!R?^*n3
zqs`eRT(-&&vi1i1f3xXMVcln!vUE>n{o}p&gah|9X{_C|d##P_j=&dt_fOlsSHC^n
zwm?>F@3L+4Y^@p1Y^0?=@7<Pu!S+?c+Px>Z_F4z<aoYWCu-u#H{AjO{{#9E;|9)Fr
z_gi}wwtuqyxBS80mC>j6PBY)X_m|#F+xOFb_r7KK+iS5~*oNg~yA7lKw!H#H-8Q?n
zN$fq*J!9|k7fbfq&0M&5``ib%r*h`*o%>i|-+ji2z4G4=?-h0UxmQZVWACb-ZF}Av
z?$}!(J$G-(Kk+>i9?R{u*HPKqy<oYmPM(b2mARMq<jmW?XY-s{wr0H__v-GFvwOF9
z-QIO>yY{}iIbqL+d-M0KmHlBGIG@9=Vd3vxcjhhFo0i70w~A+nZGx7YP3Qv^yM-xi
zcDn41wklegwsr5$+s;T?X&b;KZ#RDmm$min6E<h%PuaXUUu&x{>5xs>`>D18t(<lN
zUS4~4U0=F)`!<oidoTX8UL3?>+tH)6Z&Qwfo$;2pd&Oq{+?%jG)mFpl^xkPJv-f5_
z-(_3yQ_r?1Eq+hdt`mE8_=W5$>Q2}&7*4kRu|s>Wc8SCuhbs!Uz3xA(_x)C}(_eaY
zZx%=Y9+`+!dziM%*}lJg-G;+s&YpKA+xN^c3D~nkifxbSjP-ln-!bgH^;p(+i;{}1
zZ<Uel^Zw_yVKozNXQ%VqZD*?4d(4$<-@N(xdnBGF?orsXa`)j=%l1Z=OYWVaq-3+7
z`;*O<%z3t78G84wm1f%)KJS$+SFWb*y9C*NKkbWlr3ePw?i0Ma=S|+1Jy&KY+a`#6
z?TOiXb<d~sf;QULHG8>^aocVv-f1h~#cTWcn3U}YIkUY+du8@IdGYOivGtGbjtiwW
zDy~axzt7xnTaz?%Z{&=<dtW43*gRko*wa?Gb+6(DZ`&7fMSB%ywA(OD7v7upN!8Bq
z-q*eV_hi~yTv}o)bS}=OLS(sZb!Ca|4c?`;>nDip{j{3jPBw3bb-3{9y<1i|?G2y3
z+4e~fgWZmjUAyI<bXvXSo@Mir!N_*$P9K|z8KV1?JR5B_cw+W29G_}ysP@%%VdPDl
z1&MCf`!<^G{hJkNdo!+N?;&<MJIy8A_n3$_S#ca*ym$8wCA%zb4!f26x%b-dKCxHk
zZ|$C$PonmwFUs0`?GT^cYWYjHXII{}`CX{B=j#3CdkV5#ZNv6V-D?&TW}80au?<7?
z8B4Dda{JPA4%sY`Tx%=5&TsEF9|!9+&wYDY%xBsj6wcYJxRKrN>*Bk6tppkOMD_RD
zil__O{9C7Dcg;U$Z&cNny>Gat*}8novYpG=xo7j-wKj{2y7#VLynQdDoBZDWBDZa2
zVz_L@y4`GZ*jL-m|HHL!RpveGTW4?X$vSml?}Fb+Hm}?{><a!avt^j^akpV3*S>4>
zp6umK{$ZQx_`%lf>OmWZ_+_?PiA(pgFwfcRV&b?r;M*jdL$&j4+1IP?SzUK(Z;+Vm
z-lVT<Y|Gf1_s)70z4r(EblZIq4{UGrN!#w7@_+A?$$WOa8M-!FbN}zHufJfsZA-lE
zrZdNE+b)IJ7ACasEt;#k_eb1nn}Ucpd%jK3+55n1)t*R0Zo6vVYkQp6pR|>ju*&v^
zt)Y!~?5VvT<$QaR+UD+cGP+^2G+)MUK}!AJSFOf->z9RF8{Ct!{d-u>_G`89-t<Gt
zc0UYe*&2Lnw&7x1YuoVe^v)*EO}0+Yx7n~Co?-i6;FisK&-1o!BHL`=mT21AcI)nS
z>z`-y$XUfsQ+=0hch$_jr;h*Fn>WRDkBW=vz6se?wzstUY_p$9?wgRrz0XI3)%N=4
z>9%UFCN}5qp4k&!CcbZ(gOuHdKWuv^PEodtKmO7>%i!GJm{a%n7|j*1(<|R;d&{T8
zR<&)RZSuA~))!VxvF&-?w>N9Tb=y>~OM6%XT5V>`{<OEodAjZ0w=?$65P4+NI;(!K
zENkN4)DKGg_E+23-f^C^cgAre8;yO__Ih90y0>Z8Mca*X^KIkS)$QFVaAt49jSYKu
zo^jsmAbZz3*D~IA0Xx^;sqXf+0+Rdo227XV`(2`AuisYT-A23BZQ=`B_FmUau{9BY
zyXV6skG&-^HnyKd#O!X*d9tVR)mqyRkDB*t$hh0Q>YTOr-|4-3C(ggTH#BeCUb{m)
zdt^8+*ve!y?EO*mXwM>-CAJ54v+cR@e8t`yhrZbe`A*w=pefZ_e9vT?3zw^H7tIj0
z&C%xG8~^s{?tMD@Z8PoX@BN~?+ScG+-d+=PL)(-GT>IAR3fSFAh}ip3fA5}EN!xAf
z8YA}J*|WrQ-@GtehTr`*8h_^Pxt1wx_bpP|HZs}Ix^XVE^^#pvZ7UwEw|U6A#&)KF
z!oI~bc=x@UIoIZGGOKOy<d8kL9@f~F9kH=}!zr-$gWdz1JV`FQnS5+}KWw;fBfIp=
zUUsQzd%rz9X)9BI&~{<ZqP_V_OZIx++P8Ou<h4DAZC~s*i<Gu|lGL>4o;91TpTtyK
zS?`;B*SHtiKA6F_*W&#On>m)V_6Sb)w{3Cr+Z)w9*H-1(;k_Cz6?--vzQ1Qf?UTLX
z4DW6CuQ;^Zr(b!WiEq$ema0v*pEFhLg5J%wZO~P*J#r;^?*_IK+jog)_kQB9-0R28
zZ|AY#fz9WTki8dQ-L~Chx6ro4=f&Ri)6#Y_o&WYSFTA_=#^mxnOldpzy5~ymVR&%F
zmh0LY+q6KPT@yHu@7;57vQ5w3gL^r?1njNU$+el>%W3Cvpl5HmrSQHAn}2&2U6j}t
z=&@$cqfa09NO1Dm9$=qi6>xIy-rF-}?LNO*Yg_TKd2a%P$ljw3`}gqX2Jd~Tp<uVP
z@UrdXb>@3LVmqyWYO&ZYaAmfews()M`R}5=_6w``KGYYoaX9qdIw!5dc4g+WJuBAn
z*a{aN+S~O@#_nB2^{%CJp4z@#7O;0Fd%A5_efwVa<cqe7_2qjd%(m^>WyWRq@7@gC
zCsL<u!*9y&TeL@iuSf5bJr&Ev_fEaFb#LH9uDu(g_Sk$s$+A!B$pKrNxi@SR>hi3o
z%qrQtOhnw)Ez)K0n$-vQ=qG%&Rd}|4_o;&2dyUle_6np5*i7k*vz6=?u`6)cXB|+d
zy+>_MgzZ<(w7s8oL-y)*?6=M2?Y5b?q-d{X-H*M>dmq>)2_CZ5dc$hhbIX12zsMGw
zlV)pet%`Q;RWVW7yCrghZNpMU+eF3xd*psG?JbPlx9h?bL7Nw!*X)&0d1Dhb<E(8#
z>!m#<CHriW+aKB7{<C=R>HY;avX!-axJzH{Js3A*@4p)6y>dsC_E-wA?Q!q;XDjBj
z&-#Gf*S)EFwtEdeY~AH(%Dzv>e}V0<WlL;>FG<*)KecqP${itF(U5Dq_cngt-On)7
zCSr4_ZTPOKwjth9`=qu!+AGvEWp78vMBBgv$$K><7T7HN^2s`t%hNVXX_1X`Bi~;4
zwzYfCwC~v4!8do$fwjSVB<{%C{yw;MFT<ul+eqsS>+-B*Tb~m-dmT3I+O1QOYMXtg
zcW?PmQ(KN>NB3^H#<17I&C7<P?$zE(vGaQ?WV`kzZojm*^SbiB9m|i|=5k5ecHLOI
z_rujKd;Jg2we`9G*)}<S&)y5+$-DW!H0)il@sABx*0Q}1e)sQr-`#7IVavKt$<Wls
z;mg&%CQch{UmlyZ_v@x9dp!~^+X(F6Z7r64e~(3mnC%0`d3$I5*}r#=6^E^d_^Q1Q
zH%?eHPg}Rwpkwat{KWTrt;2-v8vec7b8J$R?FsXpwsJfIc2^!Ru<a_kwfDiN-+Qgr
z+}~s8{%TKy>}8wR$|bgw6jtnMmpNc##k+HFz)u^SdiTz~({s6Or`w;hu@*kKXPT+x
zUa4ZCz0bO5?e%NYu}xRW+IxUY!&>x!-CizkS(}^A8*N%1JKIhav9%Tb+ittee3z}>
z+-G|?^X}c-Q}K3h#!^??FMQ|s-n+_aE7fpiug|U$+h?BDdsXx@Y#TYkZNtK!*v?2a
z+|w7BY@_?XanGb~FWdc}_U&a2Yua-l{i4mAyR9}S_inLWF;~aNHd1)sij{%3o@|nK
z3#T>hZMe;DlhHKU=Jh1|z47mw_HGPz-#cMR=k7^=XYEaW<Z8R|s^H$n2g<gNYM1vg
z2npHn#UHa#n)t`|P}_Ff9giw(-v{dKEjik69p?FOuZl+dUgypUw&ri=>=k|dd+)+y
z%WSJAzT0d6(0lJsmb|?SR-W4XCo9zUYr^Kef6ew<$624YY2v%R=Zf~my#|6Z`&2w4
zZD*u@u#v53vzk0{l8xj<4;!IF)AzD+2ils>Ja239@s^E^eUr_bDuunOT~T}g+}dut
za4zRQt}k<Kw^shLDf-U3_r<;4wl4}QZO{Mxyf>2L!rmi?B=%apwcGun?vw2T1v9%_
zGv?VI`_p9Gk$q>c-~FAoKbUxI8-MQGyG-|uwGV5C?Zuw{y(xcJ**e^m-TOp-vTeiG
z*gcw)EOslu<K3e^?d6^#wN-nM2)FKOi+pd(C1q)M^7KO62;ud67rzPEJ0q3D?vnn}
zJsuA^_O@SUv=!fVWY^JfE}QzqE}IUsse5a_r0-dAXUCqxv!`~8){5B~PZr!~CnmIS
z3g@o9K{8)$?xwxi6Q;O*@B1|4y*x_aY(;gZ?iDilV7pTL#a=~?NqZkm<lQG<n7Dh=
zR!+Mg>jiAS^lr8(^_JRK{ZM!Bp|pP6)gOXv*Oe>m{dI1k?OyIfdkq$e+I=gZvuC|9
z|6Y!l2lmbo{%!5@<KW(cbq#xGd}Fnlf4ak_M{}i(<T>{}E}u5rhAx`0x5}M+m&=KT
zw#zgm_hxzQvo-#;Xzxw!_Pr~^boLcq`?7cS*R6YBl}xn#(^a;2XRF%Yf)@h&mKY1!
z-qCrm_rN`&J=~j*+b+H(x9>!Uz`nwzA8ln{i|)<mSg>dDkM(=6-e}w#A<J&_YD34~
z4SYNIeo>!nyUsP$=KX@~z3(&j?|snHZL^*8qAj0v_uifA+xFydb=<?Iadz(wO~<`&
zYbtC5pQzdT#fsX=o%p$D)1_~FkN?}b_p3nb?g_kAw#7>n_O5kZwD<O*iMC%4PTc$Q
z|2~^Tb@jWQSG4clx?go~ZiR!b{OU!vowKuce?Rlf_TF2zy&64gc3V#w*|JCN-|JC!
z!FIuAY1=bAyf&vS<@Pi_*S1ULwb^6vNNV3*-|4mz_j7Eo3H{$Qmv`yj8S7fCcjt5L
zn^`!0uTWNk?Z4>twn|l#_V9>rwh=eC-z)m0Vo%VyGq$XYe%nk7@7e2oaE9$mmV0||
zbY0#3guC2!>M{0xOSUev4W2&Dwn{+Kc7f}@y(fK^?6qo}z4!P!1>3Dft$WSiWLUlH
z?%g|^d!Y?iee@ouLz8zYY&WqM&JfzS?3%#7FOogBALsJfZIyGdv73BvZ?aI^o|{*W
z+ot<O>{VQ7V7sApm(3HE&3nJMCE8|aDeN)%dvuSH4evf<SAN?yj}PqKZ>qHS_vMzo
zXG6sH&Gxm~y>0Dr+p3%Tmd!O*yRQVz+G{bV*mnK=i?-UU_-r@yD%)-jv9oo(d&fqn
zcgkLetEsk(i}~za61#1G8H(?1%#PgKa*W;1tI*um%iDbSCr=SO!7svlS3Zs3;}_j)
ztN&r|UcD3dZI4ZSxA!yaW?SJWu6sJRJlbo~CTz1W`l#(9ak;%~);8OU8qC<^`M+`R
zt(gM0slSx13peuE9cekVSLGbzz7Mjut+Q4n><#$Cw(r!18rx}38}|C|I%ji#q13*=
zLXWIP=O5nvC!@<YenP)(@X@_{SAM%<)0@7**6fqr?kOAnY#(UL?X{RCZWmU!ey><O
zhn@V%V|#0aci8Bc@3B=`DYsAIqU64WAN_mfwg~T;>$hO9MDdP2G4l`GY*q8z%k$!x
zt>%h|y~<rawj#F-c2Alkv5(`S&7QQ`5ql?nytpS}&rKTvhvGe#qGsF5o}6#9W1*6*
zTO6C+Qh&aER&&MnSuHrer=jSX?PQtBdzIF4**d;Ev$s$#X>Z1o*1as8HhUdTecba*
zVawi%<9)VQt=`#2@rCWVDcZCr<$L$u<x5)jevsI>$77$4P3A$(y&dy-Y*&3gV!L7A
zp1lGq=UC6tUuf&i$+ItWncUth$D8-syz<_g6UMM_=}HdU2jArP$e-icmw7*SkNL9d
zz0cIQ?)~uNn9Yu}^0wBEv3r98nrt^+UT>?SzRlL8H`Mx28_V7^llyFLm`mAMSm@eD
zS=sM3bGU6i#Xa2CLuj*Y#<E>|E`QeElhnsyS9l<K@Ai)qcO7}5Z|l&-X?K9H)8^Z)
z>b)u93-+476}5X?$YVEAZM_ZCQl@=x)=aY%XsFtgoi@Ryg(+k2@kLU058rX_69|vk
z(~?zVTjO}q+VpwfUX8$sd!uu2>|Og$dfx}jFE*<#=k1;RfyJ&uM0TIT!c*2iAL#5o
zu*Au>U?Pw03|;fRw)@+6-`E#o+vYvlHdw{OnqkQYn`ajr_BMVF*xS$^Zo6t#<KCh}
zllNA*eAsiZ>74D<3<2AWWVSsOlg`>oSueD<DwwfXqo>+N<o)G6o4QZh1_<TYxGplZ
zjf&FUD|4m8Mqz6H?ggur?W_VA?3Nh(+_N%xh3)m?NqdzA684_ivtmy}O4*(}yVvY>
zHfXV#wd9X2)9&MYUua#l;pH;0bu3cexAAVr9;5iVdjlU;Sr_Zg*^~SF<nAlG!)*Ib
z?A|N#muGLo(jeQ`<tcj@HgWG<pjBgiL^s4%=dGZfK~MAE=kK1|ZeB8VkDYYg-Wf7#
zwjomY_VhEP?OC`m$7X@_V%w)*F6|X(_u5-fWW1+=_s6d3{}pWyOk~=_$CqxC?Pa_-
zjA57U{5L;r)GS17b}u_+Biea#?@pP@JwNrg?3HGoy7zKCgI!9}hrJAWina!;&e*)0
ze$v+Zp}VcX71cd^dIk2H-wv?lTNq+<@$ynzokk_w;{pF|_dMHTTjsOh);NpD?&q_I
zduErI?fuYZvS-UhA-nJUj@!)XKe*SzSKh`%a+i(Wdg*<AlEQnpf19=U#+LoISN3h%
z>$i+??~T)Jwx=Tx?>%rJXU_^2|2;X&PT58uIBRptQ(>P8^IV%Y^A6j2Z{_!;gs|)m
z5}ab|F5a|vaU7T3y&!QrhL~SFHt{drE77~h=0tM1t;r@v+uBccRuk@>-<>6YdhdjF
z#<uIX?X`X2Tfg`HdbYhTCmr@GL`m=Y8^&s9ry;QSp`wJ{4YOz)kC2_VRR^nVdc1>e
zXMYm1;$OPcMnyr%F2mi=*5QGHjl>K)n-v>1Y%lB!u;rNFxwl44*w%pQs_l{e+iZUE
zHtf!kU2Yq@(ZyCPrNed+=N{WLZ<y?UnB1~C^XclI$e%p>+^VPS-62_FQ*@8hcFGAI
zn+LgHY!4jnveuYvW<JMhr_G;74Ey4l&+qvbV6vy;$DX}~QfqDI_Xyd3Tzh-ZkLwd`
zW-Xa$TX@pWmf2av_QkFLdt~Z1?EU@dx2?Q8lig<BwR_jRJ8xTIAiuY(mB&_E`}rOT
zuGzL5*pFGa9=&TTkoCw$A+~pqTWXJu#JVMWTqVxgI5WMoZTddf)^2*yUeSQ*d$wKU
zvMrloy7yM-#=TNj+iVY3we2n6uyU`d4b#3gD`xGL7O&p>LHU%efX8m@0GCTP6SlnC
zyMOA&JwLWd@7+_qYLCa~UpC@y<u(G<@2zKKvDn1sU9e4<%xTNgdU&rbm-b$5n<snU
zZPc{cf4Y6IZJ5ZuMJe<5MC=o{zQcKKPmV_Cp8m4!wtFTYu-P6^ymz*=ruCZjEw(Er
z@b7t`+_HD#e>OX|12b&D-Z-?Ebw}BrOAdK^pWL`@%Wx)s_j+5-eLe{sdsoi9z31fH
zmc3W(Lu{KbGuQ>R@Y^yjF4_H~BHZTnvqYPT^IL57Q}y<`)b#CUygg$NPygP%60>CY
z86VNwtMbxp@2ZW0b}GsjY-<%fZ9BS{?6Mh-?=G3$vxnis?HvmomhJsgo@1-F>5;9#
z{9U_O7S!&I5>~KfjDBo8Px{{8b<MZ;?%@=%)mrmsFX!v`wg=2DY;S$yvN`FnYwtC#
zt+sX=l6D;Dc<m~tG}^TH&);hkF>UXgQ(|^k$`$OMJbtw2&5!oIISRjQ8m_lm)iKYu
z^_nBJ*I@G{Yej?Odk-AHZT(NU)%L+EM!Q!xj_=*YF>SAj_F0>0GV^Q%KEJThJms->
z&ZGN#7w!|^vq^ZN?W{@x>+|fbw!40L?p@7tb?*+J(!H<RrtSS9eZ=-nd6%u;$2qnO
zb=3BX_$;%jDPF(FbHgqhdkrbOMRMYMHDxF6ohu}3*Lvc-?MFS8eOp3a+9rCX?Agiw
zdG8OdF54=#*?aFxHSLM&*kW67d%Csz?N7Gxk(>73h;G^=VB@%_fIHLn$q|LUCyE#C
z2{|EP+orR7@1B5`y~l+f>=p_2-qZM|X|KQq@jYB0_t}0sxyUx~)+O5&2XE{Zh<$0p
z*{ZgORdCzh59ObBuf06qR$5!ejzhxS_DeF0T|&0XUWrd{_FVe$+?L05$=-%H=l2Rd
znP;PLbN-%Lo??4T%&zY}C+WV&+1_NY;#w&?_n)G+VKxi*Op;!=*Vt>bt;V{1+chUE
zY?CwdZ56UIcDHj+-Xs2T!`=fe3br5K?y|X2^4+E(O5Rq^z+&&DFN<wCSv+hacPj2%
z@b1Q*(*Cx+S5$TO6f`vN^$s|{SHfkHjpF4Ed;bbY>|Jn&ZLiy(X|{W;-q|EhH?vLq
z8NQd{!wwts9|g7xa&GQf(ABv6tx%-3fE@3E!?#}U{bHrH*Ck`Y0rSHv_DW2jzxTcR
z!u?m6IQKnYyZnI5<8=Eh#TomLeSK!vf6di)7vJvvE9AKB^B*tVt0?NYmuqE>jc#<R
zL*jJ7eVr=q`%cUY+EXTebnoPCz55S(3){(-r5xBgU2@;W->nBs7i`=2_3;Y(TTd44
zd-YUpueG@NzNp_i`;Vkw-E-vhlD)rIN$s=tZr``TMquA#uChH}KF04|r#F4C&)Z3R
zEH6sgez{_{S0$DE;PWZcd;2?=?Bxn-+Hc18XwSZFd<Uvm_U%n&irQ0gw!u!~!E*c4
z&%f_gYwX(JUHjU;f7%k8<(C)jJz*`mPxjUNeY%$>+6t#_+IukR$=>%|xAvq;KG}O?
ziuT^OW&-=ZInT2ZxH@ldw(XOBH!hU#P3wtv$Pt^eS8BbZZQ0C=2i`P%+^aqH@}6DG
z7Vm4)KW?MO#dAPbl*@M8ZNt6kNh156Gnnt&Y_rH(F*tkQ3rER)>|ZKvrd(sPH@rA!
zulwQ&_Og}Xdy|;#cbdL-+`Hr$tF72=Rokimnf9sngzt?qy|p(Z=k(smT>-n61(xhR
zJ7L@25IyOA;Tr|^y{TQZ*H-23-j;7lwz~fh@2%>5wRhKoi+fKri0+$e{B5s=pTC2(
z7015nqYQS}yC3YWzRIxY+F?Wcjuo8yg5(nHdQCg`Dn%LDDt%Mg?;5jV&*o*<_IxsI
zww||*dtW&7<o!!b)Ap`RxwKdIx1Mb`U(a4{r&)Uqm2LNZ_;hx+{(Q4NobpclLY~X)
zyY+6-ffntBdyCpq_wn3YVec#&yKiT}_PxTM+xHqjI=xrSYu4VQ_96$2zl836|F?UO
zkP-jhyKOT2rcGYGSM7PKy^C~>ZOm8BeOyjiyB#(k-`jC7#bJgmhi$|*BbyDpH*5os
zJ+xi5rORe-R+sH=tIc~A=I*i0C_1%wq9(__jlxs*7E4awyKwD+y;95c_g$P(x3_rT
zX<PH>Z|&B<W!ZOIP<3A!pYVZug)P>HboT8N^7*%KYswP4O(F*SI<?IA=go3IAoQ}>
zzPRno-p(Zg`_8Xu-RnDh%l;Q$k8RQd>>cdOwD+c(weMZS%W@$5hxeZ1ESbF;OYhrj
z+BMp*_S(6h&1=1l@lG}yzB8h>FI-b?Ego|3eYD)tw&k_cz8%l+?0p|Mb6<Ph-@WAw
z3j3EWYO(srIBV~#2@>`S-zM!=%QxHmIF<dtw7?bnLIr~NY1bO=-_)wQ&)cWd#$@Bd
zy%y$r4&A<$4oN$_4?GMwv^S_n#Gz7Q)?N+9==~FqpV=q$d*}YfWvlicJk7pO&QjTG
zPqex9*ETbou7-^^u2vGZ!W-7@UA9JL--mnW_TLRNww>X~X1ihC!M(rkKHPgp`;@J+
z=fr(=+m!cieHpTMfdR|DV-K76a&BweQ>gZ3kNW%Oy`G65_c6^mVY@$pXa7162OFbr
zC-?8x7uk1AxpA+X`$v1X-feq#Z#;RRiXqbGdRUA7O?jq$PmEah-7cSaAoEPK!z=eW
zyUX@49=I+j>hSBp#l0~+Z2MK-XYHG%Y_RV_v97)9hjV*>pA+8ab$8}oKC{_-W0uU=
z`+2(dK2<;7y*Jlw+56jls=eup4|{dvuk0;-Ahq9ef%x7ew}L&p_9-1;XxFx#ad+YV
z+on_YO@BLK@1glK_wEQ(*vB|0@<94yi+wJZm-il%X0uU=o3ziaCflYp(!gQqxh;EN
ze+=BW<Zao0p06)0j0H^XxBl%o$mPCxf50)<{i23Jd!s(+?{nB4v}f~v(S7P?@7fjj
zU$xcBdB0~3`x@&5lb7#(GX3CQnJ3A6^;f&vcr4P~Kk4&i+f!>-+yBU_+2heDYQrS;
ze(ynzZu>|9seO;KR_|5a=CijWV3Xy_na2)Dc<r&_Tcx&NIrizEo#(>#wkUtvYb&g4
z`^^5^-u$XNw)1E6>{F}?-FMCRy^Z@U?Y(6MkM?Q^x$iZ)ZM@%4kI{DJq1*PqB$nI$
znajT~OwY{j*?ObB@z*<TIH%9FiB?;_@676Fdo&yL_7)yh-#evz$KI&Q1$+A!3hZlN
z{BiG8wj{gzx!?9Ey4&uZpZ{Tx8}r7!!TRpDFK=JpA5kN0Q)095K!xuXTh5a`dzXc$
z>^-W)x$n8=`~&6H0eg>56W?c>)UoH1_(!|kTPyc^v7NOM%@^C-l)vA`X3B%TjP{v(
z&n#MNYxUA@@61h$4}>{N*sI&M9!P()aqn-5cY9X|$y>|j+Z}jy>eb!|k7%1yYczK0
zWEbq`ad>L8;<LcMaF=s?>jQ=M&8RT6?+n{$w`!IDzLa&I`$V(T_qX@{-NVoI$v(U&
zVDHys0k%B;DSI!kP~R)OZ~NY&iAnn`uRY#7z2ViKZ!FCF{V$!}6Obai_k@tq-sG8@
z`%cHNw9WhOy*Fxij-7Vri#_ssvuylbC+&;p2;LWXxPAX0YstMW-nlkrYxMRQ@GLm6
zHHytXLf7QLjYRo<o7Ohj?}+(k{iKY=K~-bV-UUyb_l3@guxELvWG{4AX<ytCF`M+b
z75nbj^z3`NV#A)djzW8xZy(y*oT_u+yw-WU%r5?YFVaQ!=XG=LyS?j-O*N0{e&&9k
zeFp81tQP&4wD<cm!+mA%B<;o9uk7X7%-~?xadnT?ho9D_{wMbF#!ucCw?=B;(KhP?
ztzt*_omREmYv3zwE7!hd_tw6Ky%nNOcIQt#+xz}gkL{P2eEUmOl=ppIU~1EwCTLsn
z>EXUCJLY})Ii-7&<yP-qd$Zj}hJE**Mlptc=hLkB&aYdt_kQB<y$f}%_VKGN+o$?g
zYVVJ|pZ4w37TDVv;bU8}dW&s?`Cse%)|Iy6yBhXJgnr#yoa?ve=!Au~ceXa{-Med|
z?Y8A>ZI5obwb!B4*7n5VHk+=KJ+>Bd7JC-#iQijm7HRuU{;+kS+^fBF<fhrIb2@5!
zVB+UJuYc^?d)bxSt}C8z@7C9!ZDnSh+q=Vi)gG=up}m#g*z6QNL~R*bd8|0MTkLIQ
zeziCL-u68<K^OMiN_}B-%lM6L`Mza)1GIc?gE+JIW<;~vhX46yd*NUHo*O5e_elIR
zwYC4SV6R-clZ}psk*!p~cH0MY1@|nyvVJemgUfptg|*tAuG(drr88?UR~D0<!z@YL
zuUFaju{SpF&X~V_uiS~5dz$XC>{SuoVjIAuvPZ0W`yR2>BR2J?KJK~HEMq76{_36^
zoEkRP882)ntX*OqBoJzA%Kvlkrac#IGq~5<nixsje(lz>eXwGK&8>@j_rwYQ+ADCr
z(MB}m*`EB<9Cmxp=<Ip)WBK0Ws}9*}sUEfEKfP_w4IXFP%!0YLl~-r%y|mbG?_}XK
zdpRdh*yDNk+um;T&^<qV&)7Du_-f;^pu(!b=CI8IImNw|j*9ypH1yeWF21n$@Xa;0
zX4$fP_bBYOWss8E#o+wis<eH}-p#!VcJks;dr$nRwzbjIxAjO(u{BA#V^gzT((cGN
zS=(<LIrk>)wz5sru-*IR#r!?x?g#glsCL+h)nB&_O1{6>q3x$l8I$s!jmLi5?C<B?
zTbuA@cY5pHy?s*xcbWTI?ag`HyLa^jHrrM2&+YxNb?WZP8B=WyiYDx}bi1%upv7Qs
zYXg^E(7qFU15SzCIVA1gvw!*QJrz9{Y+n3P+;?&2;k_r5-S=K`Qr_z@_0L`<n<m>I
zE#5Xy_}lmXjh<_}+h~Jr+P=CylMUGSi7Z^ar-!wB@5FOIY%gU_*n8zpxb2d^QoB1o
z-`Klhulrs(eeQiNJkxBBELgr*C3K!m!W8G-eRodSs%|#g%Xjw2Uf)Pv+tW9jZPRXi
zvfV%5V6UG1QCpGq^K7HTHtoIIy1~|Yi;{Io<1U*umtt%)7fRY)jg_%8ir>8V|E)S(
z(`<viGq3L7yNxB=_Qc({Hcmgf_oiH(vBzVwnAL?>Q*3qI18gN;AKxpoc9u<K!}7h-
zdOUX9y?brm1TV0SFnVEgCt}^6-E}#8w_eq?ebLmn=f@OryY5*_cF(Dm-urvz1e=on
zH+wXcH|}|{c$dw?Q(SvpZdTY@Jp8dI<;tTyOITX=%uXuY+rxQp&;RQS_a2xQxF_uP
zY1@)H+iYdl*4ysoys&p4cgfxzGaYSC7**RYEr{GRq3*-ph3hQ$^kmGiQBo4w+rV~d
z_fHo6y=k3&wvD|1_nzjC+5M!NanCx<yL)A7#r7u5a^G#hQ()^X_+xuEH-p`W`-*#t
zYs2<hI7r*>napY1;s0Zg<)P3$TLk~@DZg{thNrg4cFHqn+p@c#_A<;gus$6tYx_*8
zcu$4$8e5mE8}{mbU%1zJ-ml%eZQS=d*cI)m3z%oS_Q^LJoq%iBdu96fCjKtn6Z}Zd
zE_N-iU5eDgy%XBr*d!#a+q-$+r@ewZ*V{<&J>1hU_lNboQ1;y!`!#GDqTX4pcvN9q
zlFYw%YQuk9<@l1lLdBAHLe^Vtt98HcnQ!}b&z1HXTNZ!ey{iPI_SW6eww^KfgpJEj
z&OK#J2ljr@x3|r>K7CI{&T`xAJ)8E{b3e0Pxm0wY?p3LM4quMj{>kjHEfW5|Cq-M$
zmLXbV?-L6p+lp*%yNI?-n;Yl%+bru|ym!;@g?klVckQ;8zp=L<YN_qk%t@9dkELzp
zT95BNwD6(TKaNi}b0(?S7Db)hdm$m(RzoAl#-Lkp&)Vb<dmh|8v3o}RhP}sJ1#La^
z80|jpKECH%^WWWVKhN*J8@SD8X6@_UTki7jl|Q<1?{<eO+bfQHZ9lH<+`A@l^InGE
zinfJ|cx~hUz26)DCd+n$pQw#X;2ImZOMmvd9GPzW@6n|_lANk`4HL|EH8sw+RXoUH
zx9mZp?Y&)7_fB}MZ@W-z{oWJ895z38tljhW3$xvU7ZQ7?HHhs!&c(L(%KIxebvJ(Q
zHt<ol-LZDQ&EB~eci&n(!FKnnBHNlH5BDtb*|qn2)wR7ltNN@2uJP|_Y5lS1L{8qG
zG8yGP8cSc=F1WzAw{G)H+at4CZLjYL*!w`+W%sUTNxP+A&g>2R)w-AO_$k{3$EWQ1
z>-}<%!s7+DR}6Y>{$IMjm!09o-WuUpTQ%io+im%gdu(=y?u&cFyI1S5sEuFL58KK<
zk$pV#&+M^V{BOsW74P<}$iHsuv;EYb2NwIRvkq(RPFQ)v=0LTYt-wvDeK#{C?PjPh
zvSkcixc73-72Aiuqij`6yY@;-ckXHHc)WMPP1d~}Hyrjvu-vtIciwTY%EoE7wQsBT
zepsuyH#lL*-WNIdZPJq~_p-g%ZhKwiifwna+}`x8xi%U)wzfWeeS4PLu-Tlxu+o-q
zo6^3%+OvCW`VDRO{90~PuDRa!wu{N$KO($)i{2i!`OV9=_p9Z}z0R+v+xnWT?md)a
zw72faJKLf}+r3$bmh64p-)P%5U2@-qSUwvy0V!LB?H8;s$bPg5KEt(d)(Ww`$CXR=
z^2-<RKD<Y3uSuztoqyATy~dIY_PlUpus!0DY&$XJj%~!u4>oMCwDvyW=CeKVedXSu
zj!Cv#t_SZeep9|z^VY<@C+b!9x=%9N6Im#?w}5~BUcUAPd*yBZ?-6<7Y<u95_ns6L
zrrjOaKJIzp!)$Z$nXFyG`KP;YPqp5g%-p?q!kjmI9GJgaGbmoPt=qn5Z_>7vdsVem
zY&l;|+<kZ6;=P9i*Y1k#DYL!PCbp05-xZsG%l-G<xb?#3X3GRy=Hh*}Dh3;EJ&hRc
z%tHkCo^4jJRoUIT=UVH8y|Q=S?Oo8ccdy-|rM5ekAKA5_DAW4Z*$uW6Dp+i4I(_z9
z%dFjNxABKf8Ov*%NHNpB$&;qqeD&P0XUE}ATZaWlcJ1TYxi?~Zlg-f$n{75IZnSCg
zytcRFjL2T^!$0;;tLfb95V&yf9+#Q6tM(n)8?a%~9;S=^dsn_-u)B9I+a|B^yRH8=
zbsP7WCVOod<84JdpX^>IvvBXp{<k($wsY8B*z|07<mr314km_nAH<XQ-1#84_xt9m
z-Ah<??dE)#YqMvWfo%m-?cT_ZEj!mQHs5#sny0OK)Aqf5Dv$SE3%<8E&T;czxp_<X
zx}4jv_kPT|JsiiI_sE{`wpsA*!rrJ4@Ap3NzG@?Jm}lSPg-7;sz3<<9<n;2ry9?a*
z>U@aVv$@e|U&_4Id#jeQ?%j6v(O!#hJ8X?QSMIfpylngJ_JzIE-*WDov6N|F_o_>~
zzwEuSSLPFo?Tsfl_FVq?)pm!Kkgd;(ioItPe%bKfh}}D1O~m$&h@)+USKQunmPhv9
zD1EiJ|3cK>Q-Q8~7u{6c`?8wHPRMb~UXe+XcKh{P_9i%Q*n4eu|K7{Gt+utyGi+z(
zZQ6TSr^Pn?d-C3Ir{CBtw->N8HxaYVUd6h1XY)tflpnc!BmPaa<u<U{d&^+rUN^~=
zdm}2D?6%zF*n4_n!(Nv+uWVPQtk`ROb%Bk)U$)Jfd(Ujk-Wlv|_c>>K{^xSrDd#TO
zp1j~^JJaci&8G+RtzVsW-@D{GuU+FmF&iOqS=$a>KHK}375AkC#_T=$q0`nTV%^>;
zrI+`#nX=o>x+=3TOMJU+M!b@(ZSJhSlP2@+dvb64-pSWAtP`?5_a5kAvQ?ShwCC*f
z$$Jl2neTDU+_(4E67Ic=9&g<HaF(v^p7gnU8CI;{+xyzoE_R8CEw6jIEysnwyHmK|
z?U8)pwl}$2+phA=MqA50RW@ZQs(XzMGWW`cZ`=FysP&#HS9jXXdp3D*{0bqv*W7M<
z>ZjQ4*>_5C&sv+7JzF%-?h)V2vG0M&&Ao@N@3OoW%Vjqs<IC<l9uq9*H3;psd#ALo
zA$#er4&jBiDW}ftWmj^viT_==$9><az55O<ww(~ty4Tb2`re0s5AC^;_Qys#TVwBu
zm?L|XJl^ijVEnSjYuf(Z8V?`Xo?&6ydpnMIUtaRoJuH_?ZQh;Sy{BJOY>&KG^`4uL
zW?Gw^Dcd`D!E;+7_w}}s{<*exrmy$>NbTPHqwD1EAMOwKir1Iy{q&4+&k;4Qy;FS>
z_jY&O+}pK>$F6MOy*)n|4(&~M<FPgHeX{4>vi&wks~Ywcd3x;O(qOlp+Ss?Z^Eijy
z5hb<V!WnscYiAhjPVf3>J8@CM-YIoEYy~@A_g-MyZ+jtq*6vliv#l=%)a;#UGTHW(
zx|rRxm-;r6e&_eT->tLPHEj0YJs)@N6pw1$dx3w)-u7)XZEF}BZ0Cx`*gSDuyT|)s
z;hs(Nw(o5W7PH&yzi!WEjwYLnQssNUs@m<%%(32MA>6Y!gOhV#lHgt&@vWVES$^p5
z*~{!?vw=;?=7OF4Ud5;ldl_9N?Ub{2>|MNWxeY_X1FIt%v-avA>E7GYwr%fPlU23?
zD<<zLaOJjn<0`z*rK8t2=fMJ-u%ETIkM@MyPKZ6UXZ5|~dtB23ZJ8Vn?w)dY@}4w>
zZ0jd6J8aKdKHi<MNN_J>Wrx)Vx3<0WtY+;w>im4Kb9<2Oi4{L~Z%$pir$L!zSDG^S
zzCiO?ww`L8d(XW6X|w8slAVkUk8Mfaro9$3zt||Q<lAf6G-q#SZKsXRZ{>YPOJ(-l
z)Z(&rn!vU%>XeF27wek6jec!=GF@`^_62I%Y@B~>FQ3=Wy=@i}d(T}_vYJ;Uw)g#%
z6MOB&m)lqc-rd`o*l%0n8@yNS)XTkc$9~z|ukW$eIKX53?;zj415pR}?kV4Avw;7W
zO<8iEt;q40d-XbB*<8$jZyU93&Yoo&dv?9c3A5SrMR8v}^ZLC-Q`xL%^!M+TEalqw
zRk?Q$>rSz~+deI@E#CBOPkdIw-j-dDZSS@k?zwT7dG8th?%mItb8MEWys}w7y>-v@
zyH{;LpR(NZ@u=C}P@7|WLjRZA7%nZ^tHHs%cd6~|JsHv6w&^cU*uIT_zvr#$8rui2
zvuu2yY`3<4Hg8Yv*7JKdv&igC<66GgJ^$q1t-buV{{s{EzKA%nCtZ<o->zzgeI0CV
zwx6yaw0ZUR<6e#2AX`tLvwNri7ut7I)@IKY>2TX6Qv~;|>2$M^a5migEM&WlWVPg;
z`4z1D{^YjVurKAb*`V3CS4g{Q&o{pgTZYZwY%I>n?R7ZMvrl-T=Du>}%{GGVB6}FD
zZ(CZYcJ0*;x3GPq_IK|+NyEKoMS5(~p0-;3FjciRUwgy$zrY*Y=F2~9x4ie+o6~z@
zZ~DF$yVjl-+<CNL%Er>l-gZIGL7Qtch4(f4mF@krRCC|UU(I{>OqjLTGosyQR@Z^O
zpQ?9SpL9L6*G$IH);;Xi-f5p$Y%kh2*`~zT+6uj@ve_+sWlw^M)NY^8-}ioZowYYQ
z`?jsQgTP+ydC&KTy)&@cxZs;@%IButR`#4We7m}B4_X}DTPLx_R(G$=z6%=jt$SzA
zw0*SZu<Z@A>w6YS&e`kCEw@)%hSj=d*3rF|N|m-JmettgZs@Vi*w3+Vm#^dAch?m5
zO5E7K=kB&+HV56-*kpKU*e;o%Wp{z^@$QO$X?wW^jrJ}{4z}HK|L~r<Z;tQ(vLeUE
zh0SPRwAne^^!I^#|2^BWcVXTiTLpc_y_>m{_8puWw7=HBd2ic<gZ2xA)c0JzxM=_5
zuT$&<66I`;Pw2PVyGeBKnx&`gB!#}&cI|(<xB5o*-W7q<_sogBZTsQ!`@OHeF0_rA
z&tm&+VUN88hx>jbBV)TsRrY(+o)zrPEBv-cA}VtK$2DuL6HHF+{l0Mho~Vbq`_^n=
z-g};dd0+3u>wDb}#_bJ^KfC9D-txT)isrVii&7od$5!usTff3qZk6bMQI$=5-!9l{
zrw}T=_j&rly%Q&L+Bc~S9LPM+v2Q`#LfePRZ|xpmZm`jgli54>#sb@y6Tk1VYmVI8
zkSMY{Ys-sWm5e;LCP&QnDIRCt_vrP-y%B5JY`01t*{#f1ysPi;O`A;S^!+&-YxbDT
zX543TS#<xC;(a#vUjE*D)oky+&MR@&Q*v+ZJF{h@&9rYWdp-Vh+f3UOWOMm2|31Mv
zfA>!R^KY-vl|6eZ{ly%f6ujBP`&E9wu*sdh$NM<;HnEA?#2!?%EnFABXTH$tz2e=w
z_V)a>v^&W#ZEyAcpL@F(Xxr>f%HJED&9rAyKJ&h1sf>GH35x9txOaA!eHZUO-tx$O
zLjCo&RTsVY%v%;}Tg9?)pYr;iea_lFd-)cy?ql+m-fL{UeV<G7ew+0Zxb3&7p5MFe
z|4iFQo+WmFeaiQ8)vnm9YUsZA7XRJ7+tRG}<#TB4Gc&N-H|Or=y{X&R@6}xOdXGi8
z)V_IJx%c@cUD+G$>AL^Gx8l9>Tj%cESysMxo3y+AjFW;62WqD7%Vlu3Rj)g>hv&P*
zUfG}Rd#_aJJLu>|?A`UBeP1WbEL+vX*Y-S*T)21X-YokY;>`P2O=Yq@^x4nO{=sjX
znpI}@>gFzcSxcq%)@aw+m!)sCKCZ31FIcjDSEHc#o;6|D_Utmhy4NUa{a%jxO?%!<
zNVT>9vvu#g{dW87L+$o;_jT<0EE0D>Xm`Htm&Y9Y#59WbO^m*~N8{!UyO{RLwr>1y
z_ec7lwP$k?JaFKCy`7QBT|04?*?Uet?z1l59cx$m`<?Bz=(;_7B-9T`XD!@YGqZYM
zV_xrpi(F^-%)0Y#Zw~u>yKL1chf5AM`_vDz?YnU8i_Nuf3^rzZXZ9X@v1qTIO`lD%
z>BN22rOWmTni=i8(*Ao-{^!qjzxL1Bt9qwxFY9-%19R5yusPEEX3vXzCi~YgRoMRR
z4A>(j!oM$TuZit8BZIw}Hy!sn9B<fv=)(p3&fHKtIYyR!Ea8jo_0=crJ$)_BHo;kP
z-wZ1e`}w`4d)IA}v<dM#xW`akVxM6D|GoO@$M>E-xpI%(#zxx!+wQ&7jCSq&o3ebb
z;l{7K*?o8HoimqvAD^n)zEzEq`~GHU@3VdCXVYDIZqLlSd-lG4&9FB%?ApGtEob(o
zPtdYka4KSN#8<|BXP39wGSpqMv(PI$&~X3S-c4L)_PZGb_ohe-@BgmkVDoFmCc9Gk
zqP?xkCi~R-|Ly;%!S7Ht_4D4n+A8}QSxomkY)-dT+jL-`&z2uHdr!^VYo<PJf4S^x
z>(7C8dw&-%*((#tvoAN-dB05Ax_zq@`1c+78NIjX+UNa!cl!5;-x1ySy`q1Aoz={}
zs@yO4HdMdZ-x0{aZ<Utlfzwv=_uUsfxOYje@LuVQNA~(~1lt8zPu|DXGHu`Kmd|_D
zr33cej`grz_A1j(c($b#U-^-}+X_W&_asc%lg8S-|EJ?GTb8Sv?D#co_ka1yweRAx
z4|XRvu<rY^tz)lm>3X{v#=6#DXD_qW@YuKKEUWasJ+=4t%xn_eCu7^Nr}@JwJISvH
zZ6cqyTXQ5w?tQ>MYp=}$!@bg)d+d+wNZXVDbMD^ujmrCU-5d6-*gxN{dYb0m0O9p}
zb1%s3bLD2*_c29qpW!k;TgTPR``&CkvS;FDj=cwx+V`!E<+XLSklVAX)66O<v23rQ
zP~iS2HX=4L3p(rqe!tlJJmmSFV;mdo@A5O-&wA;#H`pzDukp>01D|V_>|OnH#_sZo
z@AqzKQQ2$vZMMzlF7bV0X54#^#d+^%sS)3M>e#y7U*kmf@!3DKP7d+ib7iOKK3m!S
zd#jB-9VRdz+G}uGVUKBdpzXTc(7l)LNAI0H?a1yOA#<#M-Cn#$MI+R{Cy-~azWlGf
zn~iqw4OHUX^NB}h??$0xHpwj#`{E)k_qm>2XA`L=wf_P`+5UhNl6y|-ece0p`DdG$
z865k?=A7G`ClzU9AuF>_L~ViXr*4tGTYI<LxL1elJ@l1v@Bc-`d($K(_M3Z#?Yo;0
zv`;>&+wMSs(q6^3oP87OzV7i#6|>zPlDYSnWBp!sQOSKjem>lLs^8nz*^p!J87q1F
zg<?zhyq@Q>yEW|e9^Q>N_s&a8*t_wXh3(S?OZV+rxzFbHrH^}0zbf0Oeq_Twy)Gqt
z)lEG12kjp2Y2SKz&%@{12Y9Ed+a;WOZC_Z`V|!}#7dxiJeS5XnG#vP0JICgf1LvME
zPd3=UXu4zL|4MAHqL#|u{DdugCvVQSOWxAE*X^3A?R!bZJz}z}?09X@+w+=q?w$3;
z+d=lvOq;)kYi**wJ>TzOWxJ1^w|noe+8;KrIkWa1)>b+&sZ4p_T=tB8Hu)_3&YWhp
z^?qVx|I3JD-*mg$z3l>x`;LA!-Fx<J>Ryc=p}o!@;`ip9`oCvITigD#u0s31)$ZMQ
zE2Gu^sS>lT*ScGKlcYrU{<_&|lPj3Kw@iGi)p0w!y>3k#_SPlt*z56Q-QN7Uo_kq(
zh4<}{IkI<J<LZ6C7uxR?(AL?z_Tb9>s|-%><w+8;E&7maV{rNIo~)(`d$czR*e$(v
zaqrs6Uv0HkDewC{Y5CqSM!hysvp3te{CQ`+|6BE5j>kuBg-eq5F6`d2SF8Q}-b~gf
zw(NbYZTqbI_C!g??2Vh^y_Z2fem6%}!XA;5(|eTJ1onQZzp!&xS>4{+H%Ip#JAY!Y
zXT&dC{e<m%8h<a`8^`s-rsmwoy><qwwo779*<SF!u=nLF0XycZc{W$%lx@G5``B*S
z9lKYv^v>Q4{cszGI8n<f{AX+zu?yMOHL~w@V_$At5>{u+=E%R-W;V0U$qO&{YB8_e
z``Eb6cI_^my*u^h+ID9a*v^PiwRKjJwA<4dZ@YZ8g55O3KHF~s>-V-?-eD8<iNi)v
zu6VDK-vV1}1|~bT#2{M@w@rJ6j8yD$(~bA$u94WQ{NHnrUfp`z$fwSG^U4m|`aSr&
zhb7h9Hl^jy-qJ4xwkLL<v|X`3X0OS;ht`)5cH6{k3flAI%9_31uGV|`H(BoySQKXK
zYNKKMVcXl?e`jUyeZtyroq3MUuCbG2U-;1pHZz{Y?vejBXRn~^mpuhb_E}f>v)S%c
zm$!2`-L`kj=bL-3KHX;9@UUc$fo_-WKEBg?FHA46nKIYW)>Hq8?S(Tkwk^ucwkvYq
z*yJc`?%m#BwU@V3((ZNSCEME8w>Eyub@z6ZvfI?Za@+f536Jd}*C1O{&fB(^qBqzC
z+|}OGYpG!OqvVv8ZfDWnr;Ek+id<>5)wh^xV^V*7PtKwpHr29!Y#9<8cF(yRWb2<C
zXM5nuBHL>(6!$KOJho@?nnIf|0&8t^PqNr{Htx4cUGZ^Ggx@h+n`O#&`p<>!9Fx{s
z^KgjS{?W<X>!QK2cTVKWz3&)B_R2~6*gVL(x!3%|$-QivYxd5SPTE`aMQ6|A_v$wF
z_a^Vn`@DAVw-#Yrv$&K!pAPoe9?iOAb3RYRmO;yIcTLRQJ@5V;+dHM?iS4Z@C0k2l
zA=|?zs%($D%-X}a`M&K6`K5cl@|W*fGnr?v*{crQ{+%3lS^DpGJ1jeAGo@76cJtw2
zoAb@iwhGR#Y*y_^vHejod#`njmaVz#4{NVASNDefVBEXDX4c-QD!x5BZmajy2zc2B
zFs|BDP+MxV#dhA_$*OF&4-&WTwR$jTZ`tOWy{l9|*i>||?fdU9XWNuJdGE>X);6<~
z_U|=N5Vw{6qHK3YK**M1M#zqi>Li=~t$Mbcm)iHfJAKraH!{Fh?}xnI3s;Ffr?_X?
zCWpH3i8y@Dwr|s1>j>wry&|bSdul=s?Ahb8%cl72%{>mUuJ3xdP}1(nbHRPRHO+fE
zcQow{tt#5PLE^}69$iIS4_{u}>v{_NJeGFZ8dW~BDShj_SNI^mZNuD;dzSfM++EG=
zv-fpZi)}*4Y+FW$>3eS^B<%HD^=<Es-YuYUw7nk+_uHPo#A$nfvFzTWpXY7&7Z>e~
z<UF=lsZ7H*$ywbt)cusL-0}Xsx*jt7j6;R&-n4wOIixye@4MXoy*!7O?s1S>ZKH6l
z&31FAq}_d)t+uf&3U*S{&+Uyb=Cuppez3>zXQM59w2*BS!@|8ShtAoy2(7c_KXP>M
zhUbT^?@U;1+nj!D@8nN?wrkg4+2d>AY;*eK-o3FKB=?<Y{kHqAn8)4=K9)8GQB!PB
zAD7=7v%O>QU(*eHg+!V6Mu=6~EQz?hM`7ZYy))u!Y`1(+w0oGc*>(x<2^-Twr@hOH
zn)bFdownH$*=xJ^_R77tij?f${t~rgkj}FeN$9o_Q#i1<MDd1AoXNqx5)*lBX5IX<
zhhHYd_Lr{X-bmeJds%tRZI`}qv~tm(YkO7u(cT7AE9-Nq=6f41bnTkDX^+*jI-9*A
z<x6c|e3)e`JI%qm?Z~1%#~I6PnVFaFU4BE;_FMnt-KE(IHftPK+ElYG+ryW!)%IZH
zyS)i=6ZRB;$=|#5)>|8g)X+Uk<}um*lwZ1sVQ-+V9izZrA?uF4AF>wgoN({W-WkT#
zHtcz|yZ4t#*kvsIZWA$)#nwfk$HwRDo;`t<=WJVcP2W9x`mH^?H?rGJza?vz_4|=+
z%7d%B*WYH}*R*=x-l@!|_fAlmV=KnNY_nK%w{5_+r#25teRpTA;j;4yUvFEs<%n%S
z(&OC?i3hA)Ynp5Yp0(}q$k}gWG&5yyj#bOv4QF3k>+p8(UE4Rw_MVB#-em_8_ilc$
zZtwQDEB7Al`(~54W!>HdR`J&J%T#T(7M$C&ptH}~&uWRS#+)g8wWikZ-Tu&hZ`$h^
z>x}8EY<9WLvWeKVX0KZH_Prc@LAG`;F6{krQ*3X;u7B1a+`9H&2#K=oY1(8{wv%t)
zS`NiMCetU`+!NcfXVG)+eUS}5wo`SQ_g>vPY0p0iIa{MkA$xx@&$K;oHq>@S<IcS-
za{Kr2Fs`%Np5|kF>x0C;hAWP{CkVFfUG--3-no(M_IiHGwGFZswLR*byLZE~qt>DO
z%56R^xMVY>nANWF|CK!+j$t-gwKMk?PW)%v@0?*{u<V)jY+GJi>&HU-7_z_YUhV3=
z$F_HY^{od2d#$(pw*7PRm#u|p;ojga`}X{Q|7-8%;F7&XV#2$hXwBO5Ct;y&YLMc-
zf75vEQj)E0m9!ddPYE;IZe07_c5dzdy(bd{?G9OQw-r9ScklZP3^x2MNA~JAi`q2^
zUbdFWW!oE6#k==<+Je0$0`j&pAHVE*?(Myo$%xB#h5E|9=NKnhJ8V(iEisYDuH{zC
z-Wv<DZ8kjGw|8}t)?S;Le6~}r&9aq|eznJ4WAfe&O0j!WtORUT)_&M?=z4+80i#*g
z)yxO>THM>Y_w2dF2d>_h+*iG6iFJJUA$tZ}^}X}c=j=`5dcSXu{HMM7D_QKfp32`Z
z6tQIQdUat3n?vIJrY6s`*S~FPub>fSyYuUpy^8flwoI+O_FMHE_HFu|vTwocSvH5-
z-t7rGzIxw_po6w%@!1D@Q#|&v{#<Au{k(O5>OKE`ub0&9TXJpQ-ul|L*73sH4l%V{
z`;z@9+AL}4wUQ2>w5RO<p4~Iwe%U+4uWQfsU#IrQyj9(2mGyY9q5qk^LCs%nLs>fa
z*qQL!IBYK3XY+K6&DQVk2Yys5*_zE!u(=Q`Xn)nldH?A+;eE@Vdh9=|;CSG;(yKkc
z3_SLn>l593`tj*~Ut`bMeD3Jq%VP0oulbhiHbRjpd)0JT?UhshwdYya%sr3iIoOH>
zm+YItreXVH<9+*}x$b)kndSE0&DdzKm?*Yy{f_s0Irnw#dzkaZR+B^8p|$s$O(VmG
zy}XCd?J?z8yXWCz8{7B)jrQrZEZg(z{3Kg;o(Oy0iA(lO{UWyii{&*diy1mL+csa@
zvsP&LUX}O1t*l*n_JwH3+dMzsy7%#Vmwmq0k-NV|=Gr{k-)<XHF>lYG+QPj?#?^aO
z{X=b5CZ6A`r7E^B?~VGtb59KSu`cP{EA5`WH%{xu{$CrT_euPg*zLE}b8r2<RW|>&
zSsuuBowoPY$H#UDy0!Oxx;$Yov+UsmhD(p_efFkd=cid$Z0@UH*xN97|Neavv-j%H
zm)KXLq_sEg^y<Cq-|*X3+E?uR9DHN%hIJ?QIJR=_*KCd68!r2Lzra3u+w~Ll56m)`
zIw1FF>b|M~?R^qie0JOJ9k#u$sb#CiS7Z0b=J4Lfh7o(`nndiKBFboUg>~BAi#Lw#
zTl!hp#>z@^AM-VZ{gPJ%?W)Rd?Ri(;y;tDUBHOji$M@Sb-`ndbt+c0V=Ip(BTl4lL
z82;Gv_MrUUl;Dfj3)YwIsff(C;qfrs7x?V7ZS^&eeVZ1v+i1L6y?^_XS9?QVOtNCW
z(z*X|{^LDAUj^E&Jac59U2)PLiA~w|9Z5y|=S(uP?J456V?8r>Pk-yXJu90t_H&CY
z*uO%Cc~9o8B8MA}uWVjV@Y#FR%hUcq#nrt{HfQ%pCrRx4e^Y($8?jycf|6PG9oqS1
zZ+QP}+klLw-CRrg_eu%M?2VW_Y2Ve<*1d8j^Y?|guiv8^almfcf~>te8T@P&e5daZ
zIc#Jb&_92-1m`~68-K6wJ=~nPPoP_SztWOpy9yiG_8!~)-R36)!+{kH7WM%KANIf7
z^l@+K3a$fx^cih$USrw!<>#6G?T6bBNKb6qTUIi`mPaUTZ_#z<y_4dn?+rU0xYvL~
zY+q+e(q5LuqWc8Csq7cJA-H#;$IiXHf?xN3`>nRuXw!b%XL~aCrI%0NyPxme-XBvL
zZEsZ6?Dd<-y{D1qldZmz{660gr|d)u|L)zhUtoW*T*2NNmr^?=s~-En_(gjK?35f_
zGY;&PYIQo$U7uvTS#{U`dC?nfs@3}Uadg@4ce-(4|H+?B`xqPF+Vxn*+fUfHcW=-7
z!2OFG8}=z}S-vNU?X-Q@E$+Q8yI<@*_L_h1{|WlGdiJdQ)wE*wc`WSM>!)&JZ+*v6
z8}@9WeJhuJ-@9<qf&E8bU)u90&Ty~W)Hx0rkK^~we6-m9=dUUIzFkP&yGde$?SseP
z_6Bv!9hl46vhN13?mnHP3-^lbn!m3q>FwSxD^(q=)^hBXYI?dS;Y`r}H1&hl&XdyY
zU;bEoAcN(z-DT?~`=^yJvXQxEV{5ZN$7V%5yWJhthJDRfFYmp(ee>S7N6faSZ=c&7
zSvYy`R@2<Q91-*OZjoQQ|K&1f+ec?++6ZiaW;12tlD#uM`S;D8mbvdh)q=gN(i--z
zea5i&YhT&k2ce(#%Xci=yL#%X{k$AMY;@PM?7J;&zE6Q`sr9C9#a1U<!}hNGcXV&B
zd&6E|gAIFEZnxMg$9H4z&IS%U@gkP}Q~F)^mK`!b@Uo|K&qPVCeHr214$B`%AK-Rd
zy4UAImTea6J-dE(_q_)H9QJm0Ew*{j*S6>X57m8vciZ+cvQFOH5Y)BLW?6{srNfqc
z^Dms-`_FxrZJqpF+v7&!`-8u~*xT^TYxi%yQrk5$-nO5V)AxGEu<y&iBf4+f>@Bu(
zcDMH$neN>yF@4Q`%^0bD|8=JB^)LUnSN+byy@%s<_r{x<*%~M-?Oo`w($>uH(Ej5)
zZS6DrwmB5szOeh?G3k9S+HJN+0`KgLe_^yY-FS!XE35N1kAs%&`|8H9Z(9t<KAqxw
zyR<$pv6Wgi$v$yUk-bgBf_)cmmF(kdRX*@RQF@<O=<0pt`I&n!FSxa5$HPZ^*_0Xf
z@oe3(hudt^KEb@@dpCD(wRXu^zW>tUm3xjYk=vUv|JL4AiP(J%wXHUvj6(P7xOLjI
z>(92<IJ3}JWA~rEaov0N?K-P>;Aeb+Ex)MrzTyuIc4j8i_GiC1zCSW_k^POCuKSYL
z%kS^JTxSzfXSBaxNp<(+BNz7sYn|GkSG;cDrb7?+O8?zu+s9Y8@Axb!hb3BDtPcyn
z-`i0qxNouMV+YfHg8TnBUD)e8{o($F)r)QYQ|kA=6jE`B4b9xwy1QYoO&8<7&<3}C
zlGP9PUu538Tm9-ndx1+$yO*D^+-sntVz+Qr%05BCXM3l+9oYXnu4msQ4Vk@9t~lGy
zuG?;-;=a*VR#<a?^)XM|=~CtU&Ru)9UsjKQ->*g&TMH+(y=$bL_wAbb&gRQc)_v>G
zZ?L(ik!YI}B5Lb#P1mN0k!kPY{~mkGkM-~A`o3)6{NjdvZu-u9w_VrR$6!}(YyC}V
zFB4yx?ZX%5d*}S@x8>OKWtU}#^<I9?O}6GmhPEaL3~l!qKD5!eS7`gS<f`pqo42+N
zUGHqVLdEvVH6F7ys6DoK`r~HXo04~|)I@mgO#X4%YB+@3e({`Ovp`43c9CR+tzxj^
zKH1()wqE6IwsDi=_XZa)+k3HpvhD6|?Y4OZHG6-kY_<6=aCnck$ho~Q(iHaoWR|vF
zb4k|jt_0V<TeIc%WlURd+tE0G?-#Fodv1Qv-WyZCeeZwWb9=7&?%yNJ#cEej@?eir
zVZ`1|4r}*bHRrUQdrHLipvYuflg?YVCqyi5g_BO$>R!HM+n>D1rq|13uZ6JW-q{ly
zZROK7_qx1Pw2d!L-@8v-!S1qN-rf%-A8aeVt87bo*!FIFu*25W`oE1$it*ml`}Azr
zCm!098=t!OoX`BdS2Xl&c1YXp-Qag`k45m5z3iEn_bk#qu=j<a;NF`>?`+)cg!X#J
zGwi9?+^{FEM`^F4uj;;892;%FTmH1o3ADGJ5R<nj&imQkNVVtIBKPxc1b@oey-07`
zE0nc&Z>^`5%`=ztd-<-k*?yGSxhG8g$lm4y+<W)F`D^QO!NJDk$<Do|JAL-1e(&1z
zz{AY;*W@!cE0p#3?yjG`mu(*VUa$GrY|7N%*c_a8)3!@M+_quwU)!Iab$bn#IPH1y
z<iQ^Gqx^e<uL|w$&uHDV&;HL|n+*27lXOp6izhy?Jrbt0@9Tx1d*2?pZnfdsFWZ_x
zMqAPETlTt}vDk_9PPLutn7#LA?Dahh>L2Y^c+$PQ<(j+g$?zMtX*yDS8($__P5Y9y
zx8-^0-X5cydzurbSgvK>Y?Cd~YQw>-Y{Pl|&E8K|Yiw_4h1x{eaqhdkU&_Yf#G}1u
z(kAY8|NdvU^&9QIO;?%sC9VFqXZxC~dzQpau&uhbz~%)n&mM=Rv#i@^^6Wd+uf5lm
zS9dSRoEqB(hADeqpIl+<wBfwX^Y6UY5<VHW#;^9;s+xSa?zh>qx5BY#?-HA7w%pIp
z?iF|;Z2K<t)}9Akdba7_XZCugTiOb|UvJB!vwIJ}!b{tngqwRLt3U6VVzPFx$>!KS
zi>iBV-DFSiomf9}cgy6-dl!Tpvp)ZO<DR}mE!(nx!M48(Ds4g<G;N=(pKN<WZ}Q%|
z?U(n4ZeZPa*>I+<km(-VEsXzdB^$%{UYb(9H|O?T+Y`A{_jH?E?9F+ydXF!&w@nFu
z^WN@wX1iA!=WMgT^4a}-$FNU%y1Z@W>6Sg;wP)|GUVUZHm(P1_4<A{%SHP~x)@#j{
zz3+tN?KW)LZ?oXEkX?VJrmb0-k!{zU2HR~C+xE7ER@#Pq?%DI<x0p>(%kn)r8?M;y
zNd95P7;n1wpEifxi<+x8eqWB;%2jOJb4J(9_GMd(?E;A_yO&*`Yb(H_w6`Ou%2xQ=
z%)KUjXLtGPh}q8X;kK(&Uu4_azGHWTREG7!KU??y*ScbLJ5X%To((>>vnH_a6IPe8
z{(bz1wMF&my(^yk?K$vUdf$rZLwjG`f4N8e!h7r5hJ`kzQapQ?h4$~2^y0Vsu~=a5
zQz<z+GbX-$O{#lr+kUI>{d$MfuH;+So;dHxwi$c8_nc^XZu8Zh!_Gjx!scw2r|tGF
zGi~GJr|rpj&$-XXsN7a!+uc30u20@OGofWK*M|n%g5Asa{PU69_rY=N-dj5k*~+=C
z*sE}Tx%Kqxdu?(V-r0Ko7TUXJ-aH$hkFmC<T&MPC2;AMX>ho2b{|k8Syi-=%CN(Xw
zHD#Y^`{B%go1<G)ZKIYg*|Ykd^j^&<emmVoH*NRMkhVK`?eN|yw;Am=t@^O{Rd1hd
z(f<Q`AN+{5E(zGU*WgQ^_4b)fdus2i?%g}}{@!;Jxb3DnKis`r^oh+HhI-p%?xfu;
zch=ba-F#wi^r0GS7mb-V#<Nb?8mP+dN%pbYd-uZey`M~Y_QlDcwGG~PU~j3zTHF5J
zGj{hsJ+(J(uBeS>$AsOBj%>G?s<wCUhMkXgUuROZRcg6s`y={|?Tpt&w(ZH4du{&C
z+Oz1xj~zDO6zncMf3a7~Vw%l+4|dzSz#q2!um9|wHM?^!Yp&?NEv++ctVI{w?)hhI
zE5}j1dvd4UUICW`+by>b+V*vBw!Lu6a_>VKo_#!u4z{M>pV(enaKzT3N^<X$YCl`$
z<I4NQJa_Ef&NR)|Fz=o1Dw_^lua>^O0)|F5qDOOVQ=YH25s|gq+x36a-rz>9y;HYe
z-m`KRuT9K55nGn~T6?$tTE6#`vxDt{Lyp#7-V=9UR%F;`In!!yywB9V*5#-6&iSNZ
zn^LgPR_*D$z5liy-jg7lVf)~m@ot9cy?Yllez(045U}?;<EK5h)o$8a_q?*bA%4@U
zH+Pn8)nbl)XP2?=+j04~jkC&`z1+pDwkcD3_7=XMxYsQDu<fE_?E7B5*|qn;Nk?1B
zxUYNb7d_s~f9tL7Z24Jxx1LS1O>1MdO}&3<@5`ck+XkEcyN-3P+WSV)-S$HD<h`CJ
z()Mi7VYMqu?66HOdAT=I&tvbW%PaT()NI~yLu`TVz4N?#A6q)uFdy@?U8cCu`ndby
zJv=KH?Ok@G%XZWCRW@hZBW(jpH23Q4pSO4JzbCd6RM+j6K5)Qx=Bx|5Q_l+99oH_l
zxv$1&XR_U9uVVci8@Hv0_i9)Q?VWpsckh&WUACX5Z`;eHrLi|ie4*_-Ly^62HecT}
z;kl8mUyZBn1Fw+1$Itz-IpA-#w>kTWwTW+|Er(R|-hz51o1><|drdDp@4X#*cTdE@
zg?lR3>)9-{WwKqOkYcOzN!d0_T;9&XDP?yB>o1$?evdu;Q;ym+6i?Wzymx~2@*LK^
zH!aWadGEf|_Cdytz2|-`vHg5)mraa!<KBOJ{#zHk|Fipmx`)k?-Clc-TyMAep`2%1
zY`(_UUqZ$1e36u0!TBXNdP&`Tbxil{HE`s#owJj5FVC-(J;AFv?3n%D?#b9GU|WA<
z>0VQlRklA`*z9(3&bKX$k=j>Z!)?Pk*U|DoDdS#;V?z6u{*biu?~=AvS$cS{O<Bd>
z`9`XHUrT@3BOvW$BiX!X?;D~0dsQCH+8cAGWshG)uWjd{3wv*Gn7emk<-NTgv!rdN
z9bddxZpMxs1;-k83m@0p^VXPi@8d3|eHy%swn4&M_D(;s%ht1O#@>R8o7QV5Y}~8E
zcxm^q7u@?C7jxLnS;u6z$U)K0V3we5@m+yE8{BqUKk^XSx3be@&ySGOy&cY(whVjN
z_sOZ0?_KdvVz1Nlusww<IqWQC8f~RyefN6){IVy@AjSI8jBR^EdWvi$O1*5QE!OU3
z*yXWj!q%m>JmK25|K7~EwP{$sCv|`K-V^E~d*_S3wryEBX;1atPkU0%Wbd`8n!T6p
z+6vq4ZB4cc&v))EIv}~X#ZJn$Zx`!cH@0QA8#GT?A9(g?&kLr5dm|2RwTW5#XU_vp
z&)pBIyll>|x@PNqWai#^6{~Cy<TKeVo43`rW5fA9zRt(@)=ZkVSK{RqTd~T$HZ^+`
z>}FZC?v-d_w*9<Les7&zg6$`HgT30N5BG+zKeMO*IlrBuwyJH6?!LVV&sps{X3ntD
zsP(nU{q=f}gWrFfsBc|+8&1XB=$<-m8^$_eZ%6e7+w?;MwkMy+>}yyQU?aKz(H<KC
zCOi9M)_Ze)&9P0+65Y$>)3BFsXUpFFqWrzTe@wCM_q({qa6!P{cimt2rk>fhw}EHR
z-u8`?Y%R`y+S~j2^j`njtoszV{@){TtKGV7%agqUj~Vt!O`dAI%uQtP9DmQfZ`Zro
z-dwV6?@95VJ>^B~_q<V7vGcV!Y`bCcKAZ5~)iybbYi&3-t+C<j{%+gBVYK&Z=7hZ~
zrmEPkxNL8`HmTnBkKXS+6BhCAEtzArw<2ln-t5h#d(I^*?u+r5ytm=F)*egkCR>X}
zg}uFjhxbOCEZw`oYm;rq2Q%BtvQuo7_MO{vM>)av#FUP`1y_&m;gs94cb4@;+nLi7
z_s-aQch~DLNA~6wy6lbUVA>Zkz0KxA*$P{UgEF?Wb^h#Sy3%E9H*3;fvBjrt%_151
z#ze^My>_F^_SRV`yOIYPwx(;_ZNHc=-sAbGaaT`LpzRi$|9gB|dF}W=73}5eUupZ{
z!h0)4uQRp}mO9v8ThL+4!M)1X*3NG4qWO#VrZC^$<7@udHnq-WZ&v;in`QHN+CFjY
zv+caWU?Zmzv1jA%Hd`Zin>`a4n{4|lV)nKQAK&AYwZvAA<<=f;yXCgmucq&P;ICrW
zG}&}-bY1n{*h6#n-Za$Os~P{$cG>pHwvTL^_C{<gwXQhzVGqyIUwfG*?%ONia?ILe
z7w^8)kKXRFb`RaV{NgTKZv{oWsVAB3ZuCyDjhAb)z4*v~??H(STkg9`wm+6n-#Z~+
z+17=D+b;6ek-a))e0$rb$=e>XSh@Sujmvv78+dGmcb&3z2-e<{9l>h5P2=tE+Pw$&
zZs=TPeP^$*tw!aey(aDV_gHT`wYMj2>z*0B$M&AN^JouGlbo%C{AR27NhfSSR~gs}
zeV=C=aB+&Y#q;fZr!ckbRcJb}yJ2$v-s_QewrNui?b+w$zc<>(b+2<Oo2}=n%3V25
z#<p)=dH0@7Rkd?_ecG1$xYu40p)T9?Z~yG6TJ>V@!UtRT{$y|1TXjld?*rA9)&{bi
zwi|Y>u;w^aYIC{ileO?8Q(OH|kG;!Up4x6Ktg&sHx@hlrsj@u>4E6T@pZjX>hsM6W
zo0X#WykXyB^WiM3O#pko?TP)ldo_ZN*ruPkX!F-RbWeLN^S)^t`fYDE$nMi~wBGyt
zfZ85mUp;FFr8ic0`bGB|FDtM;)K_C`F#E)w2bP;{MX%ZJop^)Cmh1Z!+tVFwdyg}%
zv^D)PXZJc4MZ1lUF58&i;oZmoe)(SGWn1@7R93M&q}Xe_FG0y}%eCcut*Tk~8voO^
znY`|}ZH6iP-o~$s_DWW_?wxU6&E}<LgUzgyQF|Nm4(!=cF>kL$hPjQ1c7*NoBR6b0
z9u@9=I!WKAXZ_wi4tI5T-)a@H`;ilF+xLI9ZNXXTy-GnbHa>dS_B;{XvuEFF3EQ>d
zTlccnGVS#`D7Wuc=u8{Mw7%V6zxeOD{!C!shBKvm{n>BYzV6T3D`L!HJNdx5y&E!K
zSev&T+I!2JW#88(*}Xj%C+&SX$=Q1G64|{)Ym#k0yU6YH=1{h?`MYa(s5Hmk{ZI7v
zE?87xyYs}kJskZLY(2MF?loDNY!l>{yH~Gl;@$wAxV=e6zI#8t)ZDu{`=;$#wS{{c
zMCz?gdRFc2e5$z5<5t8Ti9!ZDm!BH8LX91^I`_}+u`{2x$Ie~KjzNiW&oi@|*4H(=
z_Rdre+#Aj#Z}*k!oOPPi8ryT{8TL9V?z3emHnNHTe93&T=>5GXDzo-1%u=(ho#(Rm
zg5b2hvu~{3!~B8Au4Qkp?LRxly{fA`t(|uX>`6Gly^rt3iair%sM;#6ZLtkIFwN%X
zxkY;$J}TJ#-I}>~POgXT28~3Uia$JicU)_<ogvJ$M{V8`+i9}pdtN9`+Qawph|Q53
z7xwhXCEBtXY}i|NV})%cKhIvBA5ZrBW}n)dSk!3y@Y%$@tDg((d!?Rg-SA}Et_Ky0
zHWBMoY%2n$?0GTIac>I$a@(m}rtOgkm9`BpNwSp`{cW3`dS|bGRnJ~;pVfPA{HWS%
zedVt0k11`o8Nny^p0|E$d*i*EjgDFA-UDHIyLZjsZX0f~$VUIC(B7uWRkj&#xUCsB
z%(oQym~Shx;=-PQp4qmUxBYBRobBFwpl8nB6Dy0Xc|{l4eC9Z@cS>UC-Zk$g+2s74
zzt<~Lb+4}Yti4BnUfT12UCrKEsdx8gE_$;!UvJJ{hsHBD_1eq#e*ALWMv8s%-ZkPu
zwk~_(Y@hln@0;v>+2;T4{ypm3Gi*I}n(y6ui*Zkm{r<h()2nS?3th6E<~C>VzI7e8
zB~i<53;dM#U9ooFyKD6p+s^en_qL_4v0WRmXYUllraj-9llJ`5=Ge>ncivv<tnR&q
zetT^g*;Q<INT}@LQI*-dtlGzR{TKGVbzaQ-1aw>We!Wn*=SSc_>m9N#wsTHB+jA~a
zbZ?E~>^&{LW_!ch3vJC<x^0C^n)cR&%I)3fcf|JTa+AF^nP2yOzr4&=W0jOm)|W|p
z%f2UBIjzy!tHltx_xWGBeFw|j_vW3Dw_Tom$2LW#&{po(CfkUr-n|=cx7seLdTX<^
zTWc@V%nP<N&bZlfpMANPf%E*HzhM{l>ZHW({Zo*=hp%ak?UnikdrGv|?VWM+g-y!5
z`+HX};<WvD?bY5Fip_iXvo70Z<lAOjo<G-SWpAYo(}%Zv<hIuB6}{qT8~ZN7R_Pwc
z-pUQT_C#%(ViR#gc+awkd|R`!>wDHDbMBk3=Ct>H)0w>nXY_0zp6jv6X>GA}V3D&e
zJayDYan@B^k3JLIqD7N!8#hVrog|&K_r=-gdm5Lw?Oh_UY;Seo<h=_D(`@!xFWTFZ
zX}kA-oZ=ovw{LrzqL13h*WS0au#n%k{-B$!$BMQ++WH6fe%rLrmYMbOUe3-=+wY4{
z*eud7+MA(}v-dhrzs;1>(zY3ea(l`i9o%a?wP|n4az5+bT$MH*??v}r2=2B$xo+Q{
z0|ra>v=mC)mQ7%`dC@k}CVBg(J#o)FZEh{vX>~zQ!?uS*+SWu;(Qe=E`+M>p>FiBw
z{<-(d3=v!LmQQ<Uo#eLbu+Q54AXsN_a`W~*ch}3>UW`xLyXdu&&D~QUY%hJdV#QG*
zXCuS0#MWp**WQM8skRBOUw1DI`n&f*^o_mZ@&0>0DEI8Sr6ptc^y>b-fu4+e-?P}-
zNQdd{jZqV|-LdVj%>pO2y$TlcHVfv&?af~7w%51(+1|8;k8R37RPT8?`<u<&w8?vd
z>ZjN)`^&kHZ_y3w3Edht{HeOOk2Uw$8Yf-f9h?|qn=K)_PdR`0-ddUXy{-0>_kMl4
zY)@QukM)(@F59S>X?wf>9o(zevvco*ZR|GI%&+#m$lGH3{AGsiqLmkIo~m`(M2Xed
zG+SP=StFKZtGJB8#!Pg%t+c!6-v3;gd+aa$-uo_A*X~}z!M*b5zV3No?QX*`_vG%@
zUo&hIeBRrB<mR%w6nS9p=E#G#^%+fj<@9#%jlN>J=d#&_J*Und-qrr`pRK&-X&aR#
z2lg(p+qn1PrN_1(uidiF3Gdn?%*3(pE#q^WB~c6aa?L%vSA6>Wy=@w+ZKXn__WrWC
zv*)9-sa?D;lih|?=6$z0qHWit3EJ-1e8I-r&&ry?dhedfJx{Hr6uNB>*9+`B;i|ny
zroV6RB9C|06D^MJEjg@c`+?up)<wg@HkHfXR{n$V-jqX?Hs?8Q_P*#9wLPA`)3)j3
zslC5irtgUocxTfy>B(+Z;VXNqS>^0D&Um`_yxhgTdD=7gsx7^~ch$y?wpOd>+t@uU
z+H1FamF=T8uD$O~*Y8c=chSaAK5p;BJA8ZdrT6X4-l@MgZ{IVU{jTb^+v2zF-MMVt
z-rvV2?=j7OxYuyM<6g}_2lwclk+OCC_RThJ#TT2Tqcis^fB3OC!Xs&Kq5PD+4lyw{
zHY&^a7JV__dpOU>dgbIvdw;Wa?w#pqWE)Y*X0zUQ%3k9+Wj1UI5BDwz{BC{T$lOjy
z{^Q>L6Pj!{pAxW>dA-b5^sMe)Z=GE>6`McUOzn=do&NK{?z7hg_G;YSv&Y4C{~pGb
zZ2Q!&U)<Hwb$V~)^Zq?Q_gU=~H`B6xz%I4NKwNvzr{GU}=Il<}>u_n3?N#$LHfn7b
z_ncoLYP00QG@Hn#4BMbXEqj;MP1tL)_U&%Ty;e4AYAL%frx@+EN>j0INVD79@X*e-
z_}cuv-JP|z4qg_vHg(K<h1{9;vB~_mJr?j{kJgNTduQit-Lt-(-R^K}{+>1_J6nsT
zOna|r%Gx%ao^5+zxy&A+`8#dq-EZ03aqgYXbB_aiFFKvw-LKWTS5K9BZ?R~e?Xg3h
zdxe;FZQ^d9+#8j7V{g*JMSH6v#P)27i?gxMOxU|c>74E91+jZ4SU1_YPJgo3Gq=~e
zWA{m0$A1_0{!o0mhjU5l-ty<AwiOem+bZ2&vG@4$vwOW)w%9y%p1tR2%p+S5Gd;U|
z;Xn7NZWFZKC1tj^%YMb~V^<w5_Y~jQ`)pI{UQV_Rdw+&}-<zS9V9WIH)80JK2HU@P
zTlYH0&a*w3DY@_8clo`KdnfGeI6rOgsn7iTDvW<yzn;!xcSW(vc5NrazR70~>=n^&
z-TQEJm#x~3^?OUZ?(CUX_-F6beox!~@A&K{-4C#KUNUV@ufzh|wBrRfTV|Ns9!Orl
zS81AnoyF<>d+qGl?8G^a?7d`kY;Qy``#wPifqfr3XYHO=y>{<H7v|l3D!sPPpYGZ-
zKe~I*hile*4-|x0i{D?kclAmQ+b#zMo1JeJ_VQ@i*mkMd@11bs<t}mi?7cD$%{FgK
zXYKhi<>3CE>DqhW+sEuX{qDEjg`J+emmb@^w@+Vezh~YW8#ft;eIe1y_J#W$-1q<S
z!+p!X$?r8Rc((uC$_IAK+?n^)uamY}?67w)2YZu4)DeSyx*K}-W@jJRtNQud-W@s*
z_deLBw=bb#x%~k>bDQt_U-$iRT4v8%$LnxIMsx4Wb-VXQ_uB5|DcrEXKzY{QEtc!{
z-Z&w@Z+3RmzWqAhHolHAd*xE)Z5Jfj?OFAgV{c=l(O$E|B6}roc-b5b-n}>M&Z0eX
zrmXvy{^H%2Wgfe?%DQ#0g1hzJ)}9Y`DX%T|NrcSWD`9)g<_)LG{%-E`dkn8$+H>pS
zeA~wwbN8%##BP<rbYsu=_h<LoS>CV_mz=%lN_f95_l^B~ZU6J`Tljm0?em|G`~D|9
z*>lKl=l+H^ro93$_-$F2Chb2Uad5Bg1-^YRk0kDk_^Z6{&yDx{7Hqz{r?%tnUL~{E
zy>tF9-}_Q+%iiX5toxS#K5DZjqsgXg#ul5snI~<UrqA7<_Vm(TyWJssB2KgJTlit~
z-dmH|Z8y~)u{k8UXRpw!&9?KHChW0ZzkcspnfAQ}PL;Ms3leRn*8bV6X1QhWk2S3O
z!nsrSs?8GHXP0HVudZ8YugmNQwgD<z_wIOO?cnS+dmo?p%)KxFi|+M%etM5a`(t~{
zJB<6D*Ynx2O-bF$<9&Be#|Z_8tew4k?T@|ReZeDbZ)~5)zK1ss*i|omyEooWW8XGD
zzI_=lKkt1noxOKUzQ(={o|U%AR~>ABW~|+Rav6*5<@RU$9E#5EN=Z(3FzffUZ-2qG
zPj2;rz0RyG`(|ypy;nO$X|Hy+x}A0Elf5@4l<(D9|9#KSl{;+w-skVFp0~p8arc3}
z+&RsAwo3}ze+mD&H_2q}ehJMjdwLz)_9hgn+dn=rac`l}qrJzH>-Lxn^6y(Kw_$Ix
z&)dB%j^g`H*xt4Izw^l6t%?iwOnSn(ufjKLpY$J}y#)uR+bix~v-i8Fi*4{U#eJ&t
z+IK~$XWGsEF=6+*4N3>(el6d3^a=C+Wy@s_OwT@HzjXT1y)&d%?cvA@+qc<%;r_~k
zrah~*?(P3rFSqZ7q36EbPrvue&b(%w%YSe0m8}Z<b!V-&U()_|A8(M?-XqTs@4fC@
zz9;33jjhb*B{oUB*X_NUw{~Ai!HzvuGXwUWt8m;a=JRFWsW-iQKSWDdEy?pZu%WnN
zZ%1i^jV+U!LtCJOt<+*hJH>NS``jk+?K5%PzPIwj2J7SBobC4?leb@G_i*35Xo-CX
z?6UWtD0HwfxVvqi^gV(7H3g{-(<WZtYamsyXSwn!>kIMsZA6d#+Z%Md-zKSR+McTi
z*6n3cRo%Vyq}9HdrPucCzPM<w!;Yf8dcjQlLT5JYQ`KzVH^=_m-Yb8n?p?P<$M(iQ
zv3)1EKiIp6%X8oCP^o?A4>;L%UFESBDH6673Txh*;P_+TT=kd(94AigT{`cZ{W=FO
zJG+mKcDC(O`#3vxSx=Lz-DC5<ao;J2?fcpuuh=h9$iH{e?4|pyrieN8L<rk{eX_`Y
z+4og@nI?DdHJ=-Q;BS5JUc*n!wl1Cf_O9byu{VU*e}C4Qv-_T>8`{d7tlr&qw{HKr
zIL3YHmR$P|1P0iZ=`q;ePv5rp;w%-1@|Nj)O^?+cxP5oczSVq&`{pcJy7w9LbQ{~w
zo%{B?_3iV?Fxuy-cVX|<_PP5Wb=TP1*Wa=~({*r9u(icLy;~yt6WmqyR&Q+Cm$Pk$
zT|;~Feoxs?`_wl5+tWL*!?yOa&CWHJpZE4P@b3@$SGo6Zzt&#$f3mjxwOTu5E^FAX
zTT-@Xn<$s9o>SZ21#4gI-8i{z|AeJ8_oht0zDGx2Vh>Ac`2Gr(yL&mV+S^*zoZtI&
zX0dJi@=5z5YlQY`NbcQti+A;IXPftXn72>gr}oTqPiU#G?G-PDea~jT-0L!F`+lo6
zj(hhgaND|V=G=E-%KJU`>Xi<yqEBsZ?iH|?*>iO7a{H-!tC{xiFP*kw|I|%4_bzW)
zv9~Bo>Oh4_>)uB<d2G!^JoawsZMQCXaeQyNDdT<#xh>X<YSZ_hIis-e%bt6C>-Ng+
zyYcMCURR&ey<W!}_t`I=y=NJ(p<Souw!N=YENzufSKE9QF0~akU$?(Nf_L9?A&tG)
zg~V;o-TZF1A(efv$_2AMPC+SqBPYvPpA@~nmrcCg+UKv+o;|J;_RjTB+GqJ>)&3(q
zS@uf(EPKDVT(s%vkFs5@vwNSpcDZ%`g^4zE{b%gGxop>7w#XTKtM&iw&13J`weRhg
zz3D6Y_lwV4xL>4w%|2f#r+r!BE_R2K^mg}7n7Dt_-nDyW*9q=DCD3o<W4Uy1Ur5^?
zmZ|*v{4*}^S+YNG?~%fTb{;b`_A(~l+EetMf3M60{e4@muD8<9nrfq&ae9BeGTYvr
zA5`{!U)#O!?alsuYKQLIDaoqY{bHJCv+T2m&BB;V`*%MK*l!TKWPjB5vwQhJY_#_=
zS+e(G<JA2RZ&&WgQ9ZJEg?)!Z=%hXSV-E7}vnVUr9slO)-g)Nt_rKv4+_x`$;@*2x
zL-u_960<*RX7zrdUkv*`*l--sW>ehvf>nEOL(u#Kxs(3vty!A4r=r=?=0<DRzMLni
z`=;OOviW-Zu>C=U>%04TCho1-wrg+CpRRq*;+OaGr3&qj<Eq<hz#C!HG=Gk54Xe8C
z55wJiIM(ml&ljMvugoNMpYRpV{X+LR_67(m?W^1H!}dZ%itRE_-hIw%S@tRYov~Lz
z<lvqpn_0HZ52|d^Tkh?>c8+<U)AVn9;~(+Z)k*x{yT-@Xwn060@7pP-_p%joTW4Pj
z-n(YYNm~&S=6zlIU#$4qM6Kg9rrYeB&$?H{{IG4xt695UMEh)4J$hs#{o{tM(*KUV
zHd}YwM%>oi<KRBqs<MV>pW2m!d)xm!v335%V9UVt%x1yaleT*=8|}T{r@!+<P_oT0
zTeZEyo0;~hEqi5qTl$;zD(N)a-&2L{{@tBydrS1(-k=;+8(vL*TjtZcwkHIR?zZoV
zwhfb1-#xj6$+lwAVVikZ4%k{NY}><An`(PU<M*ECP(HgUPtNUCV_0jG#6H0`z2Lm<
z%R+6tpcxXji63J2uA8*YcCO&u-An>qdwsqyv7J5P|IW2B$M#w*f4_I{G>yIG$B)~3
zC`H;dMNZqh;ftlMv5vr=S+1gc-%Pz@yWjVL?fo^uyZy{h?!6Lhuy@KW7u$yn6Zif&
z@p!j#hn{Vj#cEq7xpg*o&%E4g;5ucG&soWRh0YxNQUg}+Rg1p5_n8IfKKY3k_8Lk2
zv6)dRwr^RS(cXN^^1W~T&)Z5=blI$w4%)jz&dYX5+LFD6MooLZt531Xn!9w*0Zm2Q
z8w>XA4O(ivJ3zb8R)fLZmX){B<^bC<s|RZTY}DAl?`=ywvG)SsLhH2UGi?(bkL``%
zUtzP$pLw5zA&1?M4J@|8kFxgc72?@9Pr%sL#>`~TlF#4vDzd2BG9~)$u@>27TbDE2
zc6I1W+r2)LcCD7*_Fmi;wb#h$wC(Y<mOW9czVBZ4WTMR<UA?_Gk2mh|_B6ENTljWQ
z;~)OL`k$2dv1IkzvS~`~)tjSeJAu2^_CP?7^{njYwj2I(+8IfA*|J?-u`4i@X<r>X
zuT4yum~B92tgXy6=iLV?W%lJPPO;U{<Js$Gz0J1n(-m7b%j>%*Nxj+an|IPS;F#s!
z8*761NSr!k^IVg4U)c@ueZ2d=?G@R$XHS{dO`8d)6!z-WgxGeg?b>S>p>Dfqis@dy
zvh{m+i_F-o_*`^fun^Ne<0D7*-uSTC_N4OlJ!Wogdl_2f>_Vyq_Pq)9+pF<L-p)<A
zZO_{S341(N73@{JBfQt=W0TFb8TxzwrOvcf+s3_5;=)GTYm(Y~8&b~gEi{zar?b9)
zuZn@3t#|Uyy$qgDccoY#-oxGIX4CB;x%Z3G7n`Js`FroW+}XRY@Q|&5ytQprsNmiM
zZk{%q|F!R3;q7E|*rMBdkGjylJLaox^(q+m?bA7G`|f<{-Zj&2?3uFV_+Fj12{xCW
z%h^RFO}3pav3So$x5d_1_z&9r>SC}9Sif|S=8T5DeXkq#a<{12O+PqwZ{58uwz&t+
z?{3Ijvdd|?{$9;=`@Q)mS@-?pooU;+Rb)@)H0`|seav<hR#*2V|C_q^Opk<JU_;~X
zhoZcDo^CsDvt+Y|ZT+j3J>Om}*vo%mwe_qHC0md6LN;5agzfe&k=n<uZL)XK=kq&i
zbQJ6^%xc)%ZzN$?;C$Hj+G<X_3+Gt({!U4^tkp}mnauRfmht*!+s5VndzriwZJM6w
z?711WVDIji{I=>1(KZaB&3hM5%(C^*GP2#!Cbw_y&8EFee@(Hu{G-b{<M7Ho$(I!N
zJox%+uSmP<Ub!cSY>#*>*;~)qyqD+l1l#L7zU+<eXW5tiLDJSqLEGl#`Ng(fX2N@u
zPqOa|G@ibv;2!@T1I>-L3=CO2h3c>FJrL8d_e0dr-ETOb+q`&ide55VrF$8?CHBY{
z7VULvli#DlG{-h)=V6;`eH?q!)-~Ji?clUy3Yxt)STe*m<J6743T2VDH+CMiHDO`4
zo9NZC*KVKgUe$1=y*rO8?0d_-Xm8#94SO!^v9VRrKfCwWoXdN2ib`!m+=ceU*`3*2
z8{)tBM$JhZwVRrIPsnoY>oK;oeOKYX*C(6NT4HhO-bmk|Jp~qFw#{5Ady@};-+Swd
zjNL8E$GfxGMfYx7>br;S=^WcXzfbM?Qa5pLUJvu$?+$HyZ6`?ChBa2~mHN?Svo(O%
zPTSYUM(=;iUc0pYHcQ(*ZGXMlz4uOrn{CR<b2jIkIQE6@I%+GBCcS5gf{5+$qoVsh
z3w79}Tzj}TcVU!`p6BPio8`{!O$j?_vmy4t-oq0g?@`IRw0F<HXS-`8f9;Kq=-rby
znbj^J?8+Y5y6!#U&FuS@o>{kd>3b2|?Gkrw9KNl$QE~OydqmyQ);}k0?;ES8JqJ1;
zTNN)^W&5<9ai53f*}c4PME3=>`tGTJs&4!DRqXCd;zD~G&h%TQWv$#DQowEdc0H?Y
zO1qZrtc^W3UICo8jK>7_Sg&x|doGE|RxtLJ&Cc7$_nr!zY`NfYgRQy!`@L;!v-Xx{
zNblWtKztvs?wP$MFRN|0Ft_jasb9P2($C4ZP17Z9)gLUgwhfBiqpqR9_p~R!t-a@K
zoBB7e_rxq&W2;uXVeivQ@qKeED)!7WTWI@~>%?BMw+(xrTwh{4!{5S&yK<%N^UwNw
z8?3+VUE`l-n~}T7)<1Q@9<ie3R*wyj?)kWPlI^qwocmI?wc8j3Nba#eaM;$pGQqZP
z*B0B4kvw}PPt@!+xE;RtmgTWM=j8bJT|D|_&rPv>8x?!CJ>Ryt?`>0iwbx!@!roSg
zdwY%ca_wW!D&D*2=dL{~ge~?i`0ZosW4dE6-;e3LcjT#AtK~njwT^As+x|Ui_qmCu
z_J%Ii-_vkW$XX~?aqq5M8MYs+G;F@eb=$5z{djNBu7!KP&C<1<AzE+S^K0c^qji?H
zXUZCEHeQz9XRtcaHe+U}twuwX%??)az1x2%?%lzlxaaAnoqIW4xp#jm;N0i)e$QT?
zjh@z9Bi8QqvHQ8FL1OuyONFhrO*f9&`e(b?PJ6(!*G&1`-U*X8?`?agZ~M|BZm-Qr
z!9D#y=GvULRN1>haf{8)Go5?6{)z38pB`lU?qk8;jg?Ba4KpqFFce+geZ76VjbwfQ
z-i-Isb`q7nHVF^P_b7a?-gDu1iEXam!acTbO#8amp4_8hzj|-NCKsD@gDtkwU+?Y}
z_KUJzvTD9<Pbl}kHx+7j&HNAd^iBJ}mxrl)?}rF+Tj>drHfzlP?49AianB1QH{0V@
z_x4ujN9{ea+rYM&VW#b0mhim`qL15tc+YINV!ou^#TT!4n@tPb?JNCl_XdgEwp&AX
z?QODKXnXf?y{+=AQ@elh@a}rvKV{Fl?bB>!61eRgf=qTRX6Eb_)>gNj^8VIdxt6Va
z{i}laN~>D$^*4XGS74%wtwhiHy+3lzZSz%RZMHNovU&0B<KBbL9D6OcuD4b^9=i9#
zG_^fkxBU0=ADyx{;6u61qjh3?^+bQ{&0D$2Ccby4RcysW>)6xB_bN%`?_IHJy6p{?
z`aPP<?(Dgd$+^e>9nZd<lRoV|G241?{3*G8Kdg0Z|K|(tyL0KP^~r^6_O@6~-=p|e
zV&9~P^Y^y7RPIsY4c*(Lvfg$B^K<Kj=3_Q1r%l>(-oj?DSlRJC>I}U$N0@u|e$Eov
z`>Eyl-gOJo_g+<yv=Qr=Wve<fe(&dHI(r<Z{;>_{KD%4((<xh{!amzYx9e?Ex~}dy
z%v54~NrrEqG>ePP?ByHwri%RBvoD8ZpKObyor0%`4U;(AzUDeP+v5WNZ4UYU-1A!b
z=$;Q@ZMJt7JKE}>JGW=jS6RC`n@-xYR2A)E=MCG_?I^LYMX1^K{PM%L2fN$$nEUbV
zd$H(*?RU-zd(UL`+U{B+Z^xL-ZCB*J(&oXs&9>8)y|O+l-(kD%`i{NwcFlWJF74m@
z{Ib%%?dJdYgg0KZ<>7j<cS)_g?f;AOY&s*>?Y$vqY%`-*!)B&%olX7q_cnpNi}w20
z$nV|dueV#jm1!T31;d`hb46_Xc}(`$39Pl1e%819)-?&c7+v4J59Ju`o_6=`+4V-t
zcE3vc9<i4TY*kz`_Fj73X1k+Zd{2qe!aaMFx%Woj%HKP=)O_!YvfX?3Deu_pbTDvl
zyeZorp;>Hpv-iy0yOX(Pud4sHy(t%z_8w@MwAa{h|K4r34126+u<i@c^4=2_tYf?4
z>Iz$iMtxhx@?U!{?=iLI5IkwiVB>3LEO>X1>ZHYc=iGd4Yb5Nt*UIa$%@vUedvCsM
zvf<w@VaLyXVlT%x$-TmTZ}!SBRkZ7BXtFI`E@7LHtYJIvYW?2wWxH$&9IowMckuh}
z45ma|;r{!3GY-V>T_hQ0wI<`(-i=Qd?7inP-(~}^@V><hdG?wg60_YG$GNx5?BJeB
zCYE~)&Y!VOe8af!=jw>PTrASI;at~k4xOK5yJ+sjy-xcX_WpSLdoR<Z$F?8lys!>f
z-ff#vG0#S8;qSd#E)_PbN)z@9+s?O1sqNUqdQQSNR$5?Rg3vPCO3{h7yVKX~DfM96
zE1%S0tvYG<-j>C8_iC44*c&K0+qOAIYM*<lrLD@@<$H~<t+biMW4O0MPIBL+`LVnE
zIQH7!<}BH3(6+=n@CJ)r_0G1vt_#2Inevm>Hg7GzZTZ9ko6ml~ZCxFA@0DA3d3(d1
z%ezi{&)&=BylSuQg|xjh7T?<4qu8})fkL9~HRsU1CG&f2=2o!oOS6~UtLipm?+Zpz
zo6WQK>~&E+XRErXdv8|T3)`M=vu&4M_t~r8ec0OW@dcaW2QoIZc0IK@^nd2wnsws)
znkLQN`&%+>Z}`h;HZ_I2Y!bdm?VBDw-)0qa)82mnXM2szHrg7TSiif%XS;3dJ6{{M
zBhzdh(=_%P9*woJv1+hwh-}&2mGgCP@pSop-Fs^HOljuayVNm!ZwupE+XeqC_wEX;
z-@BmLf3N<w{d=coe%yP@Yr|fK6-R9H)DG@-sK2{s?vzE=HKistvI{M3qf+ke71U(k
z`|x$k-pY;kws)l2_WijoZvC(&)V3n=`<`OCygkb}!uH-&)U?T-d~9!>{~TMrpTc`<
z-x%$l68d*<&>6OUEhm@nz2JRy@4VhAdq4Ac*_`QpzT4in(bi;>jqM!vKHIy0lWkqH
zx@?w*+}iWbJl^)?LZ-cI(gkd{I6byqc|*r0TvK+h><_=a6D4%_tewiT_sYe2wvUR%
zY#)8@-^);&wnwN{anGqw^KCWeG}-L_DQ7F_H*IfLw6g6{>AyBiN%O5|bE)iow?V}2
z=ez8^bqoyqb~SYF`M2o$-g>zcwoxg)whl=~w$B0-_Ig<x?tOhqY~TFccY7NWp6pt?
zZja47n+bbaKE2;lxlVGg`%QaWp$j~=*FNa%`L^)Y-fcx9`vj9V>~&!IXI1mW()M)9
zPTRox_Pu(GIQK1{*0z_)fA3z8UHP^}>kjS>eS6Zj@ZiL~ihJ5@L@!3zycO!Q)r^0>
z*CDFb=4jk=TO}zbJIRR;Y)=2_*vqobz&2Gk$7ZF@wY?waPPI8wd}Z(Eez|>Vt!HeL
zzm)H_<TTy8X13>^1!{78m%hJXb13eJ?bi;WeaRe`Y~?MV?3p3QvhV8ZINK-wNwy`1
z1$!54x?^)})nwb8XZQ9h79{TNS2|()@KycZFT&UME_{|}Yp}N0wqd>e-psa!J<HCo
zw^{M!ij4+urEPNxi=D(G0h{|N)wV2_i}ncoOR@d7MtPs?<MVq@m8<QIP?~PLIbz}N
zJGXc3H9t0SPu%ppy)79h_I$V(V0)HHe6Q`H4||VC?cS4aI)AU!L%F^AVJG+Ue*I#7
zrKM}{q3GtlD?fMbxv}l#o^?_i_r_Z=@7t~Z&F<HvnR_c_Z|%v-c)vH`(FvQ0qQd*E
zg-+XfFKM-%-*U@NOoiDtpl|Iy)g5p4cI;Q!Yf}5)de%E0+XZUx?B0HJvpo{9Ztofy
zb=$!03-&&=;NJIni{L&v0q%W91^f0+D4cE=wqf(Wc(sOoA16Gs-6Av9X3j3weaVM=
zY+uL(*hKVL*(~Sj-WvfLCv;wKlh!U^o4{?qd&aRgTdw;G`yAdk+b%ged(WHxr~B6V
zuh}EMaF<O(ZiVe!8$TP_mB;sHcvkFnf2(L)ye`siYs3?~AM^NaclaH%o$bGGU-<gw
zJ-cHy_g16|*<GKr!B)7U-u8g@KHDd1YporZtlYDAz1luT!)Mls3g7n_uYSIFBgaRZ
zHD4O{)VyxDz4l+qK0`_0CMJY)U#Lrs-7(uedvYd-@7uL?;@%1Gg>0{`O54ZqNOG@W
z)Xcr%9Km}H*IMq~GJV!w_fXZnCiPaf^;XaJ+Ft#+?{n|mJtC5ZHc`*4Y?8N0?9*=4
z*_X$7XS>a#?>3FrNA~84-`}gDyVq{Qmvwt}Y{PB4qv!5vm=v|!j45&NxhpgF{%CC5
z`|Fh1-l}W7`|7+J_9WCSwY_-SZ!d!h>mHYNi}tqWuCtpxOJ?ue6U}>Prv&buP`GCA
z1^JV9U;P^P>S{>XvpF`|F4mE@%ky>H$54D>FSmr$K9iJf*2k(fY|EQ3?EQO|-};Q7
zsCDgS6I;{!tb3CCrT3iAQ?OY&<M-YPFPrv!btte~w6S{ck*)G}M<37JD|l<Go$1fZ
zc0HfuZ2eQe>=l+*wrLByZuR9)h^@A;n*IAIyS=-=PT70GDc|O5klmieRmb*53w^RH
zj=!+S<hYHEpy@uFN6dfsK6%Ar@6oGoyLRrLy-~AQ?3z{dckfk^TYJ~)EAHFnooanD
z$an8bC%HYm1{W=t?Otfh@Z#xCvwf`loS8Q5%dHXK7j4M3PorefK3UgB+w|MjdmZ~d
zZSM!q-YX|Db#HZDjcsmo^WH7%1?`zXNZ4OWXtcBLUTWtYQ)GMKmZz=R{&+hD_n&q~
z5yJcGzi!<(Nn!C`ahKmV%ep=H?O3s9ZxrjDy@%gL*g2>_vuoqdwoY7ma<32n-n}<6
zBlp<U_Smk@S-5xI`~!Q6@;df%K9#k-cWm3<^Pk1{u3<6WyX!~u-sZ3J`!a93+2%Yu
zy=Sv_^WOi}FZUHVPq*zCW3s#5zS;Kk-#0ewn}ck$TGef5#yajh)0l0$$u)N0>CKA!
z80!=4rs_oQ-E8b<`$Oh}?edcecK^Tr-y1!D=iUhleb$kaR_(Rak=i$H+vmLvVte;K
z2sppn$M4hLrOO=bSdJasYc%cA?i2SEZ9}?+Z1Wl4?^R`0*=tw#+III;rhWDk1@?VB
zp<ws*w8-9!yc4!F)cN+Z)hgNRoO`%eeC{;c4)b4oC7)JVPksDiw~{00zLOdk?f6=b
z?dC2?vN>ISY_Hz;88+z~jrTAFvhGzE3*Wau(thuR<*NG}_HfzW-kZD6lUdxx+-S|-
zGpY<WPmf>SyTs<&UbUS(`yL;EyGP(}s%@;e+n%HQ&+c2s_;&Bl1YKLJ;!k^x8`<pZ
z@7>v}s%N<G{JQ;n11sO}IUlmihT&PqzU9jWY*b2C*%gQ_w|=m#)#gRUy1msq@9dpX
zvS#1=YiITvZ(F_BhWVw<<dDL>7dI8y_Fn(6uhE%x-wh)R+XtuD*c@-Vxrd))qxHlh
zE?ZvC)BCh?|Lm1D(An4iS<>2f&D*^@4j1h;`zvnKc=Y&Q+bdanw<lcNEBp5L9-)Ty
zd-HDB?_H77y8G)VcDqD%mOYi-|LwL+US%twGu3AQu`PRFIQG~rn)qZ-_}<@pXB)Td
z6>AsU*Rf^OzRrcS_vV>ivVI=LxbG{=_q}O%j@dHH_3h0#tz+wWW%k~H?^kxuUbt>w
zXh-4R=*DF>7RwcEW47JiRUwhE@9JH~eadDR?bx>p?6Y8F*ekrX*Dj-7cwequx6PRx
zu6^w;Ec+_t-S+By3)y?-0LQ*%oAhjF%;n!#61Lhd%S7FV`Jl$Wx3|o8oeH(zb86D`
zy;^@??&Y7geebG!vu$d{H}093{=~*o%6jj$eUogR6`S`2{+?u`v47ca(}Fj<gPzIS
zPME}MYy4}YopyqtHMjroy~XR;_ib@|YWw?2@?NQ=?!CWTPT0(urfS>iTePqAsp_7s
zUuyQ%Y?HRVGskLg=H8!sDpPrF&tK8r+a1QWPd)Yg-aV3%c9-PN?F|#XY`rF~b#Juf
z!@X_cYwY~o&h4viXW4hb=84VJ>5}`tNW9#e{_p-ih3b=gRW$DIEtu45V<O*ZBRlQN
zu2-Lz?YXyFXy48~yZ5$OEwl4X__p`{OtF3I1a{aKWX!g9=l-+z&MWoZG7qZk7*`yy
z5qKoNyDYbB?-b>kdy+OS+ADqZy<K$O#l2HL@7>!UxO3mdOHXVs@pjuqx$E0@{SvZ&
z`M=v{)~3~VJ0kDzP0oqiJNxucJ1dp5cIVSIZH<23*lSSy%cl9zs(lw{E!->jxM#0Z
zXYStjTR!i5ef-6~BfUrVy88XJtvoqtuctE4Zu^cF+mvW$+Y6mC`%ao4+`D3jo$cOf
ztM*+}-nv)o+*#YBe|B3pN@?wVdwYuA$*NhlE0&4aR`GJ$sj^Podt$1+?Xj~}d;jUk
z?_1&U(5B>4lHH-?roBnhy|yz9n(daPE!(?IL~SppY{cG_y?px&9w*xt3ft}3D=~L3
zf0y7sfzEw4U02lhEJ@z7SLTJd?FT)_J>tpgb_-T~w0UZ7y|?%MwLK*V7TfOG(PtB9
z!C))4Y5HFAAJKdLH>BCBIkN3rTKs))q|>y$ueQvzwfUlK8$C14hM{)S-nGkx?KUX0
z+dN`tuzS~}u(xk!yX~6=+<Q;#er<C!UeK=F%+5CB{T!Paiid1!_^NFUjM(=%EV{IJ
zb+Uxbe|dp@GVb<!51nP%<E!-DcGYry+cjM>dj+o5S|`8F-|cYe*WRM#i*3UG=GeSr
zGqcSMEwGM@P}}?BV$PnXTH(Fw_c``8Tv%i^=kHuw`Ob%X0s^+}+0whvwrH8E?c>C$
zdtM~Zwe3*Tuw87$xo_elew)4q{ylyC%Wbu7#O!X=ncJ;y=H6!=xpuGltZlXzIcC^u
ztzENM;AXb1gQ>~hn(4Fl);Y}Gb83Cwp3wcmwn;OZZCzqD_I^ltzITU5-`+WA7VeFZ
zdAQf3-Q0$2<;A_pSKIcO#3fn3R!QBfx+&4-GL!e-`!~J!zI!UWC-vdS-HZ$^d$-^D
zYHhym%btcUtF4#m-PqedBWkab5A&X`@aR1@cMt9PGKbCf+q0UzA9pbBV`M*TbK%`~
z+X|I58`0uTwyJA(?OkVOu~#q1+V;bZ54*iT&Doo_cgfy8f*!W2pXctaTM)L_%!$p`
zq4SH)^W@jo+1su5{uMa2H^@ME@2ba>_k5nlw(rh@8#diRy?fPuys%-pR=?NcQRv=D
z&ocMA=6>H}{XxO*&&$hu*Uf*lclwhPwmi(McK)^exYu+E_uif7<o7!A$J#3Wn6hWl
z|L(oJz6$Q$diwI7+F%B|Njg_-ub%y|yJm`ybxXQ~?fl>)dtB02*lwD4!8Z8VSsUib
zNB8D(%I{<Ax7oY#g_d1G1MBXlLWaGs&35c%^m<^k;f(*D&J1C@Q=i}1%#4@W7r*=a
zo_jKq`{e8n+cG_r-^;yO*tVf_^WK;jv-ZR_sO@EKWZk<c|ID83^`3kGc|6?{XgOhT
z!~actWAc>udRa^Di@*1CZ(E+&uA6dl`z~fm+kMFXx5r!0U~jj-nqB$QuX}y?jrRJ5
z>)9PJJYxH!yw29AN7Yt;RjaM-OhenYD7`&D*0t<0E#}#G?8Q9WHqq@iv8w;}u1kBg
zSAsiiPiOIs-4(3gY*w5;xTkz>n)NGR2iqq}y|(rihxZ6~U)}Te&CR_Q7VdjE*0$|g
zRv>9Rl|S8@bEEKHP2SkO-AN34FGOCjEo_)<lb)KnTY<G`uSwQ~JxP0a?qvzzyH~-!
z*k<|NmR;T^5jM*&c3a!0%(eay_sL4SK-TW1A;-R4?asZaVh{FiD&?>fnLdB7gQ$h=
znr~lixPOZ54Re~fS149`-xHmGdshVu?cH_5&h}C8^S#?eGWTBOI=I)}dCp!QDR0||
zrQ7xv8?o(klVI7Wns9cn?8G=*k8-g+IeYin)@?kpSEK*k-ar3BZ7nC=-<u%v-TJ#$
z)ZSgcB<!NtzVGF0F}1zr<-0eiHV<^S$KD-lCHIxjQrT-*SZez$c%SXeGtzstAMf9L
zbn;W{9ec9&cxPU)iSJmlm$h-$o~waZZU5H3+5J&KbnjBx*?YB@^V=v3UbEqpnr(gc
z?Uy~XTTbo?FkWNZ6fC)?lV!F|$HC7w6}B_>OuAmNH`;<>@5Ku{_pW4~w)ZN>G}{-K
zmhbs`^14mm3VFMd7iag_tXpfl<5JWf3GvlCGq_auYTe<pdH14Y&jb61y8|ZAv}TCu
z*ef@E(%y*{akjr2t@oPiF5L6RKg*Uu=(f%5=~rw6&(7G}6l`P1zvraw>B6wRC+6_(
zQ*}|ZyP&~sD{H=Buk1=5+jm!`_r)n#?hWGhuvPoJ%O=Qbu`Q$M#62D?8*TY_&)L<y
z=Kfxzhr4YH&d%Q>Fl+7J4}bjj%=+lDN1Z=@@47WB_N)kI*mwO;ob3a?&wJ;+?%cD6
zam(J>Nqu{d#4+wYa_Qfm=zx`bYPWCLE4|?1-buDAZ6{i+*?YY5;U16rZ+pF7yxwDb
zY1-a{uE%T?`q-_x(wO$%J#lmI?eN8W4SwCVDQx&;6PA6<X1~OOy*a0t?N-!ZuqhYj
z-?!yG-@Z>=oOVk?zw9yI#ACO@XU1NJo0s<fS@L(6f)fAU_g5A6njU$*w?#H>Pun7$
zy&e1^drM++Y}ZWMx_9r+SgS2kSN0yd_IGd8(|@)fTGs7x-p##l&+J!wt#fN_zFrX9
z=fAIeucmkJ-nd0tc1ud*Y&@nG+vFXN-dnGHc<+Y(Y@16ZSvH|xOl+ssH19pf+qGxu
z6fH}Z{#e_Gx4CW49z1OG=AHhYe@&BY-+#Peb5QX8Ui%Q9efRcC?tA<%*4E>8ovq~g
z8@u;Mp5NQkJJI&M?cTi?%4PR%kS^Tw*Fk)r^^DznSw2VE_QbUAy`8LU^Gxo=UhS0%
zdmc_cu=fJj{JoRIIrs4v-q^FDVZN=Zmf*fFmwPs5S10ThO>ec$W9-~}a%<+^YyS@0
zu5K3H%NgFimxbZY?oBtEZ0_;J?d88LW;=z4#ja?Pw$-G1t-U(&nR~Y`*}u2s5#PSQ
z%!l`yi$~Zp2Q%yoSt7a5iGStZ10CCKo35?0>05Sxuk6!jd+({evrfrTu<h{X+RM&<
zW6x!4fqhli_-+4RX4%WPc=evFqUODkS<CnO%}L(7AVtGAbB(9%0Z*oVlMfi$B`I#*
z>*+MpChEpp+p^6Jd&T*sY(fr;+o>jq?VTFBX77|!cDC_+0(<WyOx>GRz+g8=)Xw&R
zOq=a7fyvgQH{18dzhmAf@J!3rp>y8eyj{I}8;ft+#PLqv<Cbu0SDf37z1s6H+U)Q?
zwpY|4aIdrMoxL+n{A?%W6xwPxT;CHCc6#rU<O6&1dgb<BpLJkw*2|W?r!K_Xwirm;
zz3AV(i$Tv}FXz>Ndo2WF_S~tNuq%UE%hqz*(Y?=?iSC`?^lz`mch9{a_HVIuxvX#7
z%w1qx>3GTJ?xRdw%`92FSyuhFhI#Y%I=pkS{km_OjnM3ydvs@>->Wv|^IoOZ?tA-t
zh3p(u-q=>z-LRdKePfSV@w+_&>l<v9_fOfYa;tCmW~VoMcUWlH79X0jS80aCK7MtJ
zy;s!ZZ2d02-KEgD&-y^Lv+d=lpKX^kW$p=N=e0A?n6TF?E#B7a?|0i%Axrn#zPP*h
zs5hT&onhJTM-qyABUl=2y*|d;Y@7CYui(K|dppBAZB-td*y+srxqBI3`(Ck7rM)_z
zUhJ*$II`#9N!i^xq1Lu#0=au6gUxLDLeE%x*7oc@S2WS4?$gb^+jq^|TeNGEE!&4D
zdnQFO+cunW-=jHEWZ&bt$M;Cr&foh#>dju(6TkQVyb!jx`%UuRWRoj<OP+7v!}URG
zA4B6wYuU+j_RM>{V%MX~du%zJ*!K3ZitdYT-neH{v;3Y&x17DNRKM<J$XjXKHOqPL
zguNSfce;A-jk>XS@9~+jd*6Fr+pAE&+4j}k{kF?r$k=pwrtFovxNEP3L(|?_it&54
zNQv$hel22Gvwg*0sp%8;=HC^wZMnMN=BA|7-fgWNd(ZMQ?lm|Qzjx}FpSBt1rgjzU
z^Y(H@U){Tw&vvic6o<Wv%D=3?ncvytyS37Wx%sNiG6_aI{l$KJQgvJRBt$muIg;{x
zuZZScTiJWYdp|DLw{xCsy*KFW?LCtV8f=eGs@|Jd9=dnuWX^pj9JTf?m>F$z{rtY&
z7HS*zw*BMZr+4}6-p9Ld?WwiQ*&Ffg<nDZ<EjIjqk=C;WFW6@Pim*KJqhZgAj0#%?
zqo%#;+ji`|m@T%~EoJ^5EyHbl4QlK5>}k&1tDwqjd!<=nUnSe2y*~>4Y>pcW?EA1$
z#I}9A(7w+8{d<iy7F$OwNw@WwXS4Su!|~lh?F;v2=GN~OJ9fs#%D>0<*r}T~?awQ0
z8x$n%?w%{M-4yW1#(SpzUco)7dlKGUv^^B|Y47eui}ya}$={>(%6;$E<LhkZ%T(`O
zmHT0@?cP6oe_j@}tDK^?ce=)(y@6L1ZIxGP+cLQC-}S@ukd3NUj?K%uQ+w<Wl<l1m
zGHH*J&_>$@+!Fh)tr6HeU#n-&2M+1o4T=nV-!Sg8eWko~uj`xqy@%YJtX1b2*lL=c
z-1D&f&EC6`+IxR&mA5(j^|Ou3>>OLq`+WP<<eP14T2Aa~WVm75*vw~pq3W@X?5($Z
z{aa%8NF_Mgeyq8)*GuWfUW1G&d#8P_+10Yy*4CuJVDHCsLUvPbIqc=P*|Ar?;?7=1
z-U{1%+rxWzIW_Ekz-(nVWvA_4`6I&n-n*#p<ua+*vp43*-gf?}HcPX<+eYqRVIvp6
z)%sb>wY|C*HEs9p+-579{C97`d6s>0Nyls_FZSKDjVs&c;JIx!|NqU}tJD8r&qc4-
zwqO40*nXeWw)f`Q@3yB|XYS2uP_T&>71?Xn^2w$~^!nbTI_-PX4)X5Rm07U&o%fu*
z?^|c=4RiLhm2;c0_x{mmdsuq8Z3TH=+K7Blv#n`=yO-ZW$@XF0!#%wsr}rlQthSvh
zA-(tUZ85uR4`=T+44%8^z|tvJj8^~m_OhGrc9@m9XPY+X-hV-Mdu_T^ZR>V0?Kzn9
zdCz}0R@)D6I`^zv6Sikn=?PoTKK^}jhbGu+bZ@r)EmLoktRlK^)+%wkUCGk>T8gjk
zo-fsAtCD+X&t8V~y(X9c?Rvr|x9`QbG@EV7BKv+!o^1PV`}Dm%%q@F=UO8kNC9%@>
z!<%V)(<2o3b$^z&byE@F*EPR-PeRF`Jy$O>?n`zQ+q-|!sXfUX#P^jn?zX;A+P!z1
zbMxMnd%xJYwQ}uaH&fc<?%J`}_P5Ml3G*78ZL$1&p6q>M!y|35_sr$Xwp$l3-Fs{^
zldUlCFPqMsReKY+EVb=VVX(d3{cNv1&!oNcvcKCXoMW_>72UM=Vd2_6htAyE!z13Y
z_w8;TyFHENwyl#xt(jM}?&US^u{l3um5oc2(cX@SX|^6acG{G`Te0`1Hm7ZD*#+D1
zIm~vOl`ij1+Oux&)op<`^(}Aq$os6`Ykxh<HtZ0C-ToO9_Xs`?*n8~bMVpH|r0qVe
zkl(XocDe1D=u>+OcrVyk=zX>|Y!<WKAf#h!<6B|-<^R&XEI(TII<-!>b+QxR8}s?j
zULD0(duK&Q*|gtXyeCx4Xzvpd2HO*gGW$;EN!vK3`0V*&WMRuu!?Q<ob(5{M(Mp>%
z;a7WJT|TzATD@gYr@oBs1>xzoj^~tYy^gQkbN&SPzN0_x?Db9Gws(q_zV+D~#e02&
z_-%e%ys&5elm5MVPmb;7jBVbt;%ejGteht{A#HB92bQ<&X3l@IXP?@KJ$@ob_mnwt
zSRYSZZY4AG+TQ8zC-z8Nyt7&H@vrRy-o-ZgH{16<4mR2wHG7V2b7;k0!QQERL(LiX
z-Hc}5w_(>b+h*rkdsqHjzW1<|*gk%-oqG-1T5Jr08}@wpbA6B2JIlR?r*5~|wSCTB
zE3O~5H5X^v=r;K5o#nV;_hhM+dyTx}_f8gDzgKfxk`33MuD$-kLVLd}eXxDo@yB-3
zpA~xxrB7JPS<cw=t!m9)j|;PHZ(n(|msdDruYAr*n>GIz?0J&!Zo9H;^WI-EtE{U7
z#P`PfKi~7xceahqtxJ0@K2WsT!633Xe1gMX58e|tKh1aAtT=RN@5}Xn_D)IZvDug|
zXtN+TX?M{2w|n<$cJ9r1m2JZ?3PwX<Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0q
zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(
zGz3ONU^E0qLtr!nhGYoL_{3)kbNIF+=hnm6FuLTo{Vo{W|5c9_h<(9a4NQaBFznjc
zVh&@Y)04OFh1&s>vkr@x4P!s}dkI4xCWnm<kLQKE2_}C&%^Ry3eV>`vYzmFWDhJa`
zh?cduzXPV`)ki^V7#l_>{yVu7#>Pb-yLE-28eDpb6*spyf)Oq!wokzDd-RHSirgaZ
zaz+*8Ci#D#tVD`)G3*4%eRvj!RU9OTj<>CBAX+WDUKl_A+FYXjBykZlOhD$GE#icy
zYfqhUI2$Av*zf?J20(mL@VX%1RWS83>@#q>({oyj`8=J%oo%kxU^NGKy#~`Ty`r<4
zFyay>hfMEj?N>qO6U+V^6oFzNNFV#vI&e7L-hCPDf4A=9<sflneA{Zf#leIYa2(1{
ze=J|f*#kDmf3l?vOr7^;!EG=$F1q~f9=QJ&UTOi`3sQTCp9{=3yV?eJ14s;pd6Ioa
zVQgf&Wt!4<WIl|o`1%)CahN<gbWx)K*uThnzp?XUloQDE_}C>^`oaE8esmq~*5|*>
z;9&#O8)f-oHHb#Xx>9D#(8aLv)%CS8+<e;S5Juc$(~B(jtl`4;-4(9l?A-TI;@4Bz
zSOZxdF>LqpGhlxcqZXeU>#d(K{D!W6=gD?(cph3T2Tr#z`8(pfEnw{awj>Pq=n0m9
z^}*B>>|z9`O%VU`#SXaK<IVuMI7kkJw;ShTh=Igmm^YStF^o-Z+T~$1yuL*?+u=YA
z+)c-XTQ?%hf!G4fLGbYT;&lP6_G0*3Fb$H2VS657oMGPQw-rPEyfVu<Fuh@!2^jJ)
zIc)TU83}MV!Q|~`VK-xWb>CWj#TIzHz|@mM8>u7w=JqxoBVQMnv0%6d-F*HT90b*%
z>milTlcT;HUC*ismofaF{)Mvo)=NiJ9c0h1rGnP`V@_iA*S~$Uv5JHAq2nTrPNLPK
z>xJ>1&#4e?#?EwStm$nh^G|qKZ7Di~qGmZCW}6dcHz7KI&J=iAeP(hRPIvmBQ`lvd
zuy&_*)g-VQjUTFD8fN}GZ*3Ge!Nk$&qWh}g_8Gc3A%69XFckYh>UY&G1KatEYYMn5
z+!EWf0wj)%cN}%GlJ<%Q>t&0{QFypY8q9vKEhP_A7xmj3rG9{kBhz~h34`qdiFy2Z
z1!t@D?*g+u?`^>-V^(Iq5eKP9$J=(!gqP#Ua(zo}!2UoMqm-SpW($Uy6Tf6&loQ0-
zv%Cb7caB>;g_}F=_)~b=1exQu@yHqwjgB7)1Q1n+^ViJA2+PNl&tb$Zy7}n*W6oKY
z{wwB5Y*lN4yA`CyNJ>!?M3aKAGOx#Q6Dj&(YJYR<fYTq0kB#0U$pB7Mk7fK&)K2_5
z1)gpq0(~&tqu+J{B_2d}-v^rmvg6y?lW_J4hTCv9NDhQ|MqR-W1Bt_M{}=ToFgCU6
zNu?5Vscq({d7~jP8UmvsFd71*Aut*OLo)<6bT5IoXWSlt+JNp0%bRE6ZJ@bv{^0hD
z&dc-gHWEy|oJcT+7)<WM5;m|t7#~DGwf;T}M1ydYlmXlgAU+}7V9$WkW&^3W)+xt2
z|Bxm&VNGXwHcq#J^b*2dt9apIka=+~hF^r@+u-uJ%qlxuNKg$fy~K)7NPLQ6hxobK
z7-9R^<u8UhV(kE_>lS5D1JPnLnyk{b6foifB*%a98qszfs@g@gT9Db;@IqB%tnR?3
z*6Iu&MjV3FWjp)9!^+NG9?k~I<!tdM+l|sW%i-hy5z|y~nxj?EwsYO*EX!ssNZ1{a
zzJQ`uU&{)`O(1t+#Tf!*jsKVbu0pX7WcG=$8nB($XZnK2qE9|5!02E8TxMjs_<9jI
z{I~G($iHn90`~=LXYZ8-nE}G0f{(U<Xnfdk<yWv>AT^J9?BH?qCeIJdzEB{J;l7Xd
zK4KvC=s2%R0z)lGjrW{jus=X-YT*wlg&5^a;huY7H<M$B)bSHwx7Y5G1&1TZj2(;T
z!@~x|ubkkFwg2Dp*_5d9e-pM2jPi2(pKgq}1=)oU|9LRO!nr+GeDmcKC}EIVZ$|R?
z{{qDXu)p!yM~qz1!y^RExhytuE6A*r8ZmIX1+hW+`~^FBnEPC2!Z2s<+H4fDURi#4
zJYR@=0uR&NLNB;D$XpQitu@6E1Bt_M>&tVCU~FpBPn9-e9sj4cJ)`E0hQMeDjE2By
z2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kW!
z5TJetbh*!m?@6#YUWIY~&!=f2@H1t!zbpZtX(9gS8(a^}MVUTQDEla2;vib~upd|-
zhz-Lx#k%495q|w*z>tTj!$#Yvn8Muzlb^PK9)_Dha#MdtuFdxE!YT)nCk9WNhd2W<
zC_ob9-sJTQ12No#ZmzzvBtbRkdPwE7gkQz5<C{tZhTnBrDRWC?{dDmDKM&q()=TT!
zG3*4nduD+%R&kIVI^G-|PPAHdy)b@$_)4P9@bc2dn%-0kzrw@HaJmtSngU})8i3hJ
zEY0Ir2S5Mk*@f3Q^NEJS*_}1r-d3xQ&Ihac(#enFCk^4vaQES}`{gtZvd;feo6-Ze
z7ujt0S<pNm8wr-nTl5NJk62HBrqx2l8DO^x&wVWaMQI;+kJL}^&2lias&;=Rc>Yh@
zGHH02b7l0v+2)^@gZ-VmN)crb9LQ|(_8zR~|0GSse$LWjIm|QNK;}?_Z8mv;{flfy
znSd}xIe{#XkDZ&X4-U&l#fflp{VgGTAYuAW)qa7eDVP|Dp2@nBsQrIJ8!*p$RM_1}
z^7%go#<MJb=x|D$Ps=j{*$=~B3+|G<|4-PX3!KJaW{^uCnvS}s51Tpe8}-3y>VGt9
zUGjg;H+Y!yNJ(J0=Y`87lr(r>;}O^#kR8uCe#7Nb)~|-AUyvLK&rjctAqEnMVaJ3i
zG&}#VPca65-X6@pQF=54MnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0q
zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(
zGz3ONU}%Ow&8hA1HH#aji(=gW^E}`td_5%paVGHkMgNOV@HLMxUuVisfZv+{6NAx>
z{(>0lHCAP}z~qmVzJ$90CWenL6iUTdmzlC-2UauoWv8vV{X7<{96mdc<-Dsq;9;<|
zL<JlM$npnv3ShVgSsujBVR!{M1H`8VuKD!`!yWuxM=|_f8j4usN{&17SsTFX%b$1d
zvAUmd6vIxCnR35~zmLcM6Xfm~kUnzp?t(>F-GR+K?LXM>ihh(K1W%_6rXcRc2AL7$
zF_-N4E7Y9<zyIg(8Y`UcjI9*i=_Ao$DJ}!Kb89b$A&T0W&JQr+667{?Tzl1C8C{GR
z{?e?U7=CQCYy*eMJk#gk{4J|D72}T5i*CJ^{=A#OZoS3MBR7Ak6WE-;_q^cu{|Kg=
z6I}n#{}!=V<>0ghV0%IC{(j;M*x&gMwitK!=*~VP3Q~iPgA|T}-GDAGa_lNWccAMb
z#22r)1r8fzbu(IgFv<yJd3@}7S53fS8T(;5+`s&5g27<`)29<Y6YKi_$@kM1z|`QQ
zUFI~v@2Y{R<9cF=v9|*zhl{>)^nkfwSiaby^D|Mx_5F8dbzFLIiBC>I+_Q#D4Kd>1
zs{Ufc2fDtwr)0t5>EIg&4r`cvz?EFM-f4oEcSzlM<cgB6_Wf0Yr_&j;4dLvx9qZw2
znAw{Zo@0o?<Ulmby!8t~G%c{AahDV=%pEmzGz3ONU^E0qLtr!nMnhoeg}~GgpWyd4
zKD=n8hwg*hZL#q7(IP9zKHNBVPH@`^S?!)5z3_GuvN(tx)EJDn-D4TYf>$171~R^3
zpagdlOnl|jxfpH&$+i0}UUq1z0o;!uF;elanQHJhUrrJqM!Vi~GvWU9*NuD_ZYI?o
zgw*$!T*9zp;_5jV;bKto0`4C|W@3}?{wEFYU%q;I-74QT0K-m@p337HSj9nd==gGs
z8_{af^}_f*^({o3p|jN&YkE`O`W7Bmfs><9)O3E^gfo4^+<=P?m>LG}3mx^^f-~$(
z7HRHCUE62*{arWME`uGcD1JI-J{j&lTy{|+&Rb!J;kM;#v%%qzTwM>ATeYqOV~_H;
z=P{P0E{R~b#!k2-7LjQLR%@=S3GZ_}-#XcZklX4rdBJvp)J<RM24^pIgXHb?v$kR6
z<Bq6h@Nq<tJ{Vs3!w2jJ7#~KzER06!r^3W3p}jj8G3??|`UiJEsb<G6vj&HyiNjQI
zIsoandCr1jM{9K#d`t->55vb#=FNn$@zFm5v{1|gsr|FKdp?LJ1)FSV0>_EUA`ukx
zq-P&dBE=k#+K>x9C}x1f$ie^a^b<5EOWzormoENA9lQPFl?@Mb1#=CIF_MD@{9t`B
zceNJw!1@0DX5g>@$<12ciV^=g`xat|fz-nAuKYVh#T`sNDfFJqEm+6NNHJqn?Pv&$
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin;0S>d%RTV*i$2Yq
z82A5ulvxa3iwbgWW!6RT`o#lt{lPRy42Cmp!*H&-R56%^q5egD1pNL#j{lk%@-TC;
z(MwM+fV&AM?{V!A+#fJ85WVl(j5V9~_G6U;$rFQFzP7;qR>M<@vHt&G<OB@&pqp#`
zznP#KbUmc<pT#MG?+ruN*XzFu!|xa7IaB18H%^h@`~Oxqa9CF=+{9{D{)6dQ#gW|x
zV;@eaB3dm>4>~<L74z;obTu&k($yVU(;KHE;+zWhx|1ksG<LMZ(*VpIV(DM8Oz^$h
zN2i$K^efw@r#tp6I&L|Sg%hl1F`piohS~EbOasMDFmZHxN&j^fbTMN1vrjz6@S~H{
zHn5#u84&+3W7&+cN8IWEBTJk9e_*$MP%4#QT5%P;Z@r<u3V#3J#FRvW>+ZwD-NAN&
z+`6x74V+!MTN=!E_+bcNn+;L}!jBtyu-^Y?cVaDuUqI>)>G5Ki0TQPK*Z$d#VP=G6
z9!5DqtR3t7qQGIv<WvPWcUB|n872=cuENt4vbhJjn~1vqPk#ajIF69@o-5-Z`Tjqd
z1AG>t0hh(Y59p(WcWX`y$@l-&yS_&;1KDk)u<gYc5;SM0;s$WJsLIp`4r`eCZzsHl
zhq+o?9Gnf4ztx1Au42DfgVQERp5Is=&K9k_1ZRWfK=|o9LA+^V(~UHYy)__x)WDaH
zietV1Pjum0a5#YMFwt+vIR8iT;xTymJUVX-cE74bE{5OMzv{&q=HEIGW2o;f*f|Si
zCkT6=Ys8QTiQ~c(&DO%*1XD9@EB3QdmWtS~sbEqj$$XfZF#3-38o1xMPRs>|0Za}=
zPu%<t!#yBzWV}Y9ouC?I_2jU*rlerh|DJ~s?K_y69A!Qfxg}-K6Y%*z*FHK}x8MAR
z)vU|<GqH-p+<;79_Sr|YT4a65>;P9pIz<-8#+GWGgf%^!jt_^6YgwFwvq5G=_an}L
z0`W<~LEmHGd$rs|OK|$t<i?V%Y5S*JGN)Ytr#CG|RJB*1BhJUcXFpT&1hVS?Z3j3o
z+%|m@BRIX^-(3TCQ}^0?81;*d{z}WhwJPAWXD5D1t~vy=CoFQ)5%~E(I*yK8@VOZz
zXTe$t5A*zWPvGp*>iOU@>T_BZm<FkZ;ffL;F&G<}{x|C>*uBVNAhz0f4vaD#Bu)wD
zw!}QQrmJ8QINZoF_fIFJKf(TW4&1Fy+!Im!{%Q6<O>AZ-$<0}cO&ne9OP?~_Y?$2g
z<?$G8JD40Uy6lks4*wf(#R9MCpoDkBt=I59uekK0i~o9k55)|0d1CnqY>>7pKK0oe
z_rT$qbz&PznvdGF10F7xf&1WWm|6c!r=hs<Q<)k(ou++9^dBmlL@?sNuTBj^3}!Bf
zp5H!U5s0P*mJPZJ?{6cUKRNjj-2F=??ZN2(*K831=WCFj_`d64wE_?IF#K`<p$tkm
zg4DzC-&=pc`e1wzy<fZ=-X8g6jmYaDIYQW{@(6}q;m2NJ*a0$UJ@dx38lIL|<v{Yp
z-~+Pe@HkI4xrE{OPswvI+=FiJG(Ft+zN4!pmw&9a4<1)AGrrbr0jF~qA4Icjv16zM
zi4%*jzwZI(?MLD5)^k;4u$q;oDuq=Xn;nJ|B#2gvO+QF%YWzupaR5>W!W$B>r-uz6
zi{R<xnPw15Jb(8@J&O-y4zbw!$Paj5t;Mwur#rv@(AyQ{ptVcO2@*d?!tzkmDs!d4
z-G|R^sj&Cp_8C4k*yLV+#5~L4T6Y6D97I~|!DV5>nui#DM8CjJtLECzV7DfPEK``^
z{ScgAwq{S1huL#kTOOmo36q1-7r%zW{r9uL18gryZr#^SVDT$Lr@-cd#9&yPM@<68
zMy8prx?rdU$!~pi0_+bEn_Bq4N<M~Ni@u+NyPs5hmN@?cho#}Qxo~sM54wWG0A|*c
zIHxr*HZq-Yd?`_NxZ<A%upP*HZ67eKMCKF24#@ptA?fl&V$J7nl=x+QwHj;t1fQ9#
z7RnfI!lxdWoJmL%L37x**@DCKoyt5EwclKA%wXnzyRsM_{xGo@7vF*P!TA2ZEbw@~
z)6WfOpK{TFvteqBH)msr!Q?=6$Ax-~vo%5D)WChS%I0D-$3Sxn{LBKe+k6=H|AiHZ
zGU%h{46xq?jvv9OQ&OL-#F?)u=419-4my;>`~N)8^}yz0b00P_*L#@pn>+t6hMPco
zxm;t`q=@@rl>^BWgM;$};OXG+=@5)KY?ZY~NkiDoWD<=as0N!pQp9HIuwc}qsiFrl
z{64{7oFccJVUz-o|3}%bu;RSD0;^eHr}|?R$L61Zbt*)w#ik!5)^>Oo(Ps2Mz<&Pk
zA>S@|Seef#MoE(kK8D~--yr+(VV`Fm@c#eRszva$gs#VL{S5Padl;+|OjE&Xq%2U!
z|Ls0ww&l>x0P(w?Igr)=-+lKLN*N9^OY$;g{QrfA71&LRvLP7tOWAB+%bI{M;PCGg
z=#e|IP7WM)Z0B6y{eN%Wxm!Wz;lnqY5q+se7Z7KuhVHNe#|v+(5|~CdBm4R$to{Fu
z?t8%QMOF)9S9l}N)CKXWfhVuF!*JKHi<o^RV(mIm<pK`NZ{<tjY3}j0dGK-?WKQti
z4tSdq#E0S9ztR}vGcY-1T1v(qBd&9jUJ|7j*_>y!sXLn7FNt|cN}+_ewzoOS<Ns67
z?nW^K*&b5Z9`1|b;f70}_PsW6ocY>Rg2Ni7F5mqjJY0@Fc#LuW55q>p`~XPLsYp4n
zIUu%!I^>)x5c?qWW^f(=v0->q&l?Oem>h_HfAc-f`u}2v%dz(VLH3T~(GVC7fzc2c
z4S~@R7!84u6#~BDGvVzQG4osN(S83&2{As>+A|m2emSv13QQxb<$4l~v+dTs$pl0F
zL5J_~vks@te+_p7vYEKp-V+K@+H4?o&7PrH%{W&gu==X$X{>S}eZ=7PYuw>}3m3M+
z=>Na5pNiogbaT~t3<#=0*F!3Q(Hjno{(nWwb(DSs$jo)q4#NEd;**NGCM*Y!FMo}=
zY1Q{@C02j!Z!5(rj?J!TH&TdJi%maB?8kQE$28LNZ7|X!$c!j2$UQS4cDC476gBVm
zUm)100O`YqXCD-T_y5)Trr-?wMFM7)%Xd$)6m5l^gPQC0A4M&T_Z6bjab)-kviko5
z-?C8b1NkNR!*p;sd}lKO%kA{Jg3(7@rv24Iq-qN|4rlW5$Q$`X=ELh(_QCuA>dnCf
z*Z)o1%mlUzWUqU49-MvC<|H@`3-#YO2gxC09%e7B=bBcQ&jY&`SuKeD-j|2$cm(Ok
zhMx-6Vb~?w%Y{)+VAG3DjQi;`uz7x=jqtu`Z!+rH&zE0CVD0}u`#hVd`TrurLa-gk
z_KTZu#)w;Fd2DPob&j2%RcDF4;XMbABbb^WyH0`U2w{9;=vK)T3^x&@4_(b#zRd*V
z_<(;eI4`Bl(gdelm>J*ST!V-Cv<ZlF-(m6`jFKpBoOXUW*c_0&nbsnB7`$Se0#Cmn
zIS}rAd<8=cBo4!EP0lpy|1XzO9qs?a0(z7l4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R
z7!85Z5Eu=C5f%dG7Kk-BZBs8}?Ejk-!UP{vR&a9wk7=<#gRBWc4k}IkK9q40WI2C>
zofzXD<sDM+@d#&`ehhhJ{pjq_g1abVHX!*e+~!!#V4UQ!=47KYRymM9Vz9*2=WxHx
z{DQfz_u3)IdR=^GDKKrua5Fyj<j75%HxV9JF#R)+Aoirf#9lupes3O34>HXbo(5k3
z*ZHN<x>ZCS!%mReT)qoL`*-QSzeKAAnT-u^`iHsp5t}-YSW@|8tm(ldyB995V#Nba
z>mc<YEFZBAo(4dCQgFEcGWhsE-?SMx{d&W()|_>piPit->0mWRi}!$Om|a&`A?xa3
zd|Y(bM<4L~3@&+G;)Xd)DE5KWy<~O*hr`5~FTwtgiN@Uj?>>Rk>YbK7I309|te1b-
z+zB2NW!<tFzW@J@)g6NS|K5h}gNOfxZ;0^UX37N?zxq`fY%a)MFl^y56YKiFGET@E
zI+!{bZJ}RJaBLW+mRS1uq<#!DJyUmp!;M%okk#40I}DD8IdX_Kb*YL%;4pxxI~KML
z>-zc-CgS%0KQdDU$0y7@5PfWAC&t=NkT^b^l+CnLwjo`dn{7Wz7^EfKCwcv!#$02t
zzwy~ejNIOxzcBoUuCG{>9UPt?-mF2flOyFM+#jkFi!sb8K|5DMBX||q9FX}-*b#MU
z=@!g+;<)TAylFz3&5~yO|L=Z@h3~fmxpfqehQMeDjE2By2#kinXb6mkz-S1JhQMeD
zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mk
zz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By
z2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kW!5Eu=C(GVC7
zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5TIQMd@MN$KU2GCx0pV<O^00-
z;pZcN`EVF~)-n4E5ipIccGmU-@N<)q#X;-~%Q!LAgT!HYuH1IG8(?Di=s!M?`%#hA
zxO(q_-@S+|p1DbVg?7+dlFUce4`SaIv4@`n{U-AlhF>1KsKdhuq#hd%loE!UflZzo
zVyiDqz;K6y?raRdpXT~Ukz1x5W>5mTcVFUDtCIO<7<Pi>ipq}@ZO7M3%~;I<nL{jo
z<l0KK8SWLj7;y-)XFdZPTzr21RFw3zV`Dqf{sEbVjPK@6SqbBF%gf_*XI-S~&TCpd
z+a;#`1>1GL-W^4)MvF8?T*AynrqcrarIGo>vVRoq2HT6Q-bZc@I80cwq`>)`WqAw6
z9i`_i>@8od)PuWKuzWh$oVaz1;b-$(Yz;F*HWwRP=d2#sE|6T!#?^54x4+lG{`RSS
zgK__FN0eF%NIg1ExZMVJ1G@P43Ev3bLyWG65MO4RCWc)N{S0`|ddFv`Q`$qYdCbhK
zz+nzj`@anm1|T*FUp;gdey0S855uO{Z%v1>@zH*J*Q2CwklH<dRSQ8hDfp+T-ws<j
zJ>ds7%-}eJ>HQlPK-4|NF!dnX&OQk2ZxEXjte*FopgA*NaD&~u&K#20VD<^du7ron
zM-$Aux^=o+!1`e7H3Ev^>2#r|6+FIbyBEO4VQNjgMKQ!+av*v^Yt;e}O$*%h@PHsK
z%pEmzGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhomhk)`P#901zcEgS6
zF5a@o2|li8pbokB_k?6N#yEzLC<D$hm&(Vk80ssey5aZ#DmqDG$fLUhnZHOp3+^VE
z_?k~mDC06P@w#{7YuO|lvC6^J5TfUPaf1786$|FQj@!=*W4H(1%x7CT393QYLn?n+
z++_?qBBtHL@OweEABH+o?CM@Q7kvM3R*$Ilvb<$j&HB?(fmIxvJ(^ZgM61Q7A0*}^
zWJt6bvpgNJrnj<b%y98zL0xb*$b3n)&G0k;;*)}FKQXU{sdwVQeK+ZKFRh&ymtESn
zj~kNbHrI)w_{rt$X^gmpnTt$Uhcc6O|8I)ZGYmiaH9^9mprg|iWXD4mo)sV(89$q|
z%knAHYOq^h2=>T7PM8KB2hF)O1AhN+-k%nN<L*yai^IcQ`_@c2`}3nHFx#oI0K<Lv
zC)$f6y9LHBvdYI$3zEBIe{MU7rY2@Aaomb--jdJ$WRL$h&6NX(rG4^oa5#d@F5Axy
z4g(MygxAhUTmz!f@fp?&%h1KJ@iTUqVuVlmMnR0Y#ikcoEP(m4#oE++;=XTp!N<)(
z^2yhZlYIX#@4VR<ZUUJ_Dqi=-9~^%8^a-m&?x_CI6oH~P*~A1M=2Z<k816~-oe7r*
znO|%>2_7%4dl2``uhm!q7YC^W;Z<*rV2FXlVR-eS88o~9_vrJ9qxb*90(z7l4S~@R
z7!85Z5Eu=C5gr1mrxEMsq<0)$j~*Ov<ILghqvZE#;P%T#-K`k?5LHQYly(uaIn`5H
z!TONJklFldGce3UmPcoYsGGvw1d~hptBA2S2qfoO%CUM!2|r2ZgY+TeyS#<)cA#6E
zB1XHuv(**DJ;?e&Y-?2;f@(nOD8cO8Ut-u1dczQ<T@Et;ds_nBKOjD-IFEIS3P{iX
zvguaQaqSp(g5->E5kKC+uYZ~%_kirvmtn)|4v-u=wl2)TnjS0uPKT$Hy$$s!YS?bv
z$C<v-%}3|YN~?#j|Lb%+g43_}n67RwTdigFcy}OJP2ToKFby;Bw7?#?`(R?&Xp0ti
zveprOSkRARAINMzTR(6(%>N+<mRtPhFUGnq#w}+p_&B$N!~ZcSk6e3@BG{c%Ct1PQ
zA)41b!ssJ|%tyva;=EwHKw`%N5M{<+uBTx0+zbP;=5dgEbZnuq3hZ8V@#Y!_s`vkE
zuSQ|yvG#OvjB*0qEx7pQf{-<7FK@hsyY;mGOmG;$^l(h$(}uB;>4rP!mmu?TvHKle
zP|O3VbG+e8)Yt(?EexmVEZb4D=cVYz*J>#F!!)pi<S{7Auhd!pcUrO;BR*j6h0($f
z&B6X(T&)I9w=nTHzpUWtW@1Y_il0Dg{8%GV)c=ZF2{s2L*Y{=>oXu#o1)i2cav+?y
zY&V7&NF0WjD%>P1-@^3cqgUpI&A=x=DmNMeqaiRF0;3@?8UmvsFd71*Aut*OqaiRF
z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFnmHFfgLe_@p$Xm4d_10^6!SvcZkG7
z&I0iZ#hgDibdQD4o1mNV;3Mi>Do8%at{lGppS4#E@4N^|Ju+^H&w;xMCa$*WH+=2|
zCI+GlU(8t((!CC=97vuRY_eSe?zhG#E*SHA4-Q0QxCh-_xr!u$YS8tN$}ck)*^REJ
zxPBsr-#75MQRJ4`jZWb8|BEl4u{zc<2di0nraoB3(cK5)_dX<k?LJ67HvE#OlW6;P
znHOPAZ>%ehz{6_&l?f<nUVLtXrvYsC;}dICi-(`p(xk(I)1CcBSGP+(<FWEd{RFlv
z(EAUVhM9fm;{~|;U}D(lZ6*99&%J`oS-E>JihUrl_=79K;V`QJvNquR?Fq|4;>bA7
ze!nHJ{#vlV4oL^ef3$?0t)jhj6MX%@;=eBh=l_GhGlJ~`nf)}T6V9&txg6~8BT<Vn
z=5^JkpMjq-1kwk?J4M%H_yr{AG(!$!P8TFj3HIaj$1wBq8e@!df>=95J0NAQokbJe
z+-sY1!C?S1<K!Ma_?{A&7>EuQ3m|F^f06TlxY;0mAS|+_2_tSn;`p#N(+YDYXG?MQ
z`%)<39XWRq$?N~i?Z`i43)zjHTQpGIf-DDPvlvVQhv%C>UvRpGiQC?_fQQS=wNo(G
zo8&ibLrH^TYuMrGG~qVl{3=Dxsc><a*=)^kFvMVTAX+0rlV<DxgEmx+uKx!G@+ck+
zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z
z5Eu=C(GVC7fzc2c4S`WG8UmvsFd71*Aut*OqaiRF0;3@?8UjN+1YWW0v2A&Loeks-
z?`Io07)u_r83e55&{JE(Zqt;>3KIY1@eeHj^xRes5FZ_{UbUO;PJ1WED&L3f=yEFi
zR<j+;iC}BhpUSp*mN{JB|3WvM{Vu1Cqgd|<>$Uhc7Lfji)>L+}djYH)m9~KG1&M)h
z|F(lD{9yHq9MdnlqqymNB0qaqwgA}dWVUV0Ablr~nQ%lfva@k@7I4fr_66Hfy@VI+
zH;_CC&$kuf;5v4ME&gg13+K^9j$8jEz;>ynvx3cHZtLWj&@=}u22#KM-*ncM?N7jF
zW%X$?v*u}Vg2b&a8F2)gbhFQ&%)oXhU=n+ZJ~!BX&7!x#wAjO!VDms~kDA(YByTlk
zH`boO@wI(5SbTz;4TsW<R=7QFy?ep>K<XKct-)%*Y$RgSf<U->knqgE``FZ7<T;m$
zer7k3yuoUnu>fHPNC5)_LkC9yql8H#o8KpHu$iCYR<rh3NwCe-so-cm-pny8>o+)j
zL{e(NW~Zef%;4C694rPkioxjn6tEeJYgyUf{@laf@^&(q&lK;$aXg5heb$ouVD&34
zCxOLb`ih=pbKEvy<+%6t3j5D*N5O7<T*=Jo5&emy%GVpLcHY%McKLaq!EAw#d)XXW
zZNO&1%xL-fnMF-_G28xz4{V?2v2ZMBW#f49RtK!+@|m?@8l?8J`$CTFDSP1jAD_3d
z_x67W^P}V2!RD_It7A{^KE?i~!UHT0()WM=eXtyejf|ITALWR-G#9SUWKRs6Xyjt{
zb8~FLX-@TqGRI6lUa+5ZdfHITShZIiEDkbHU+X@5{_`TR{F~J$SZ_XQ1DpA-oDZzV
z>C^+T97yfU3kF~@SAjCtgIcA`t9M@k+Xa$C#+voh*$*o}We-n%&&FQc&RVui1gtMq
zaSF$#&YA2@ddJx}hD?B)dpWBaEPwf%9LK-+oE#bE!fbMP-hty^cv&CVJ=<7Mu!cRK
z&+5}+4;EK5y~&aCnU&M;*;jBL0NMZM;(Qjz7jwb#U-G*-K9+1^I~2bjY<BB~JTT37
zNQ_<f;$<d~dJvxdCJ*io6PD@hXA+gzU02U%7h9~&p_=Lmw%erkCHs+Y)4}F~^nkF4
z6&F~a`PZ3Xb3tOr_@GMx$I+^KP7z@@xcdT@JOG;qQuiS?mEA`hl1@PUU~eXl{Wna(
z@+|KI*;%yMS+8vtg`3SGSjawsOB-y~lv+)&-5`A+tnRyl5k!Nqc0?oFe(_GWO*|qT
zKkn-@SNvPd{`Uemr%+oXXBZbmU(+6Gusx@YZh+0NbLrtwZ_Q^<_uyyiZBOBtUA6~o
z{;p1aFzptj!Zx!~3|ziwunBT1WvpQp-~9>f7Lc7FY`1F`XWp9S9B*7-bEFsZvtL>L
zn9XeVN%lz}*0Q~y#ss$SGoL%yJnxse95qvOz;=W5UY~H0UDNv_SPsMo;SWkD*{@~>
zz{NO!xx(3pHlByD^IRY_0|UdqzX#y)<E7Hb-q5-MF8A}rNmj<9J7E6mja3|03q#qH
znVVV4cF40GK6MN%KDX;8*o`2&jrr2xY$F#9w)s9iV6*@8adSM7oyJzE^@&~9u9scQ
zM~<T?M3h7Ck3BeSKziNR#)8#?*c-Nob3DJ_0X84R2Vs@7?^xeXc+3%(RtL8OB<{Q5
z8Qb#GB^=Xq55d)+;+@6t#4`$<$HH~YI4T`}faRDug4uWN%K(S}VeY?dn;x@+!*Ye*
zW7c2qTfk|1^~UXBwIH+Oa?{veG1h_AXw7hBf34WW8Fm|z)?V%14_7Chk^xo|!|Vum
zo19J-*nW^XF#NC3fIU^BlD+rGY_MLpi*a!E+j5#X>_caQ#UHF`28V@<zzL4r`^xY%
zdVO~zSR7>DgV<1hr?nyiAh!MP-~8+Eg!38pMRBcuc0s@=cmXd+d`s0OzKwY=xI}d3
z@PEx+EMU4yol|A49p4{ES<Y4?HIDt+j{FxYR|)*tw}5YL{1twX9uQtCBg+}KlTl#J
zeqKIl-&X!9%{n|c3TN=HoUxvF@#C3%sXyxYK<fSohI1!vH|MR~EzI}Ga6Mm>t{L|=
zTO$FFY8#&7lnr3Ffz(aDoWaXvb`I>fB3(v-;7#1zdKEYLG)p(~9GSnGKYQt8-kc3{
z*`1#;@cf)1At1C^5A6OuTUhy|<5T$9@-A~;`N$;jNbEJwzux71vLeyobhC2x9`+MH
zY5dKn*YWY?d-94JbMQ?$#U>D~#lU&^#Z<6=rlj!jUc1-9HGi5d&sA+zo|SxW`T90Y
z1Iv|_PT|`6>l0u9b4ONlZ9lHRHfH<@iu3vEINSNm(&fQ^RCq7W^?i;e=eeGRyjNU<
zc>*ppaV+9}%zgdWKE55c=eSi)F@xhoZ*LQ8W9}KSxo3_zvPsu+fZ0JgtNGYgF9owf
zVe!(=hxfm=D%kyz3pxcRW!(b%)x)`i*Lq=zz`M(5c!TU_^2g-9U|;+A3|~hR1Fx@G
zEm*(%(OCWy#fp5v8y51LXvcARo)zKK{8T64v_g=7-|<7d+mDy?=I>PGk8I`QwdtM9
zwPbA<5C0Y(4!&h9d@31-c(2a6!FQx>H$R7KKHGw12VTkBVPLyY73}1Hx=4^~k=h|H
zkQ*ac<nv~IpTwtfZ#FlGul?*XSX_RaoIs>wC-0<;egTuO7W|IC{5kfVSpW{FD77*!
zi^(0l1%}_aZf{<}>pQcTqpSJ>SfBROdiI6#ufTZ(WZs-ZQM?W&FZu4NPUqTqpo_ap
zcN5?GOaTGk3rqrgCpz<HOv(ZKxBT7*o=>jg{5Cz({7V}ZdB4Rl@OIDr$-DByJh1%P
zZ<7Ri<o0tLIX>pT+}zLirow|=WSJF5%A~CvXEwhQD5zE7agUhCr4@LYU$poMAOF_t
zd>5Bn3f%m=0PHT1TQ>i2;#ib4ozw2?7J-|O^w`3g{rLGdrSmEN7U!9~YAOd~N)tGY
zKx+RqF9pZjtXv2F``h~X+Ryg#-TNBCw^Ce1Ks$0e?_2Fgo)0ga_@(}L^Q7*V1G{m1
z`fA=A87cfTb!E7(WXAIEo#DcDMftISVPqKh>gHv<5?4|=S6MCO0h!N|a)A5&JwCn*
zN3Qehob2N6-pnkJYT3=1Y^BI&x}ujaH@BW|O%bEumK#gC%T*Kw_UGGx{lOZl&9%ba
zkB_nbDEE$Q-+0S^C9^l5(E#Va4ZCf)?%S#Jtznbpt=f2;H)ul_x9QDR{tcIYg3F%o
z0xNiY)&058xozcUX8XYBx4E1D)=fXYYg;4*<{#X^FKMg5->M+PyFg!@zfo*2I6ZKO
z7Vu`Nyyx$`s>>~TX(^Z6L@|DyEe3pt_TA^rZok1<E&H0!j4P1;m$(*Bfm%Pmpt2Cp
z<b9^#aD4No2Ap2p81Hd;9sCUz1Nq(Qup~GPUd~q$2s|Uj7hc)~4!=H4P9D>VUi=q}
zmh;>!ROCI$$-#FZ+K%5+pN-3MRTp32<vHA|8`p9E+WC^VxjvseM{zfwL}07HvVDKR
z_Oxru2(0U|;A1>p#(({C8kdE{J+L1@c3zn|jh|!XT6W8X9yVc#SpL;F4|9mGxypZW
z?F4=<`#c`kZAtuQk%}BuejH#ujC%zI)HGkS?Ot#m%r6zV!J}s`AyD&`g*$z!g}_{Y
zA1?QMxdPAS<v9GBR`EYy{F$?{(NrKN<vrJ`*Nb@n>J|#5a6aR=H&6rnwPV{nJ~#Gc
z0Z(340Z(OXfoq?=`S!VJ^RcYR<ky@l#eQ*miNLG6Fo9a0R&LQp-2(B)*ahr<+~l?I
zvlF<WxRf`Jr-`pMWD37iM><E(T6uwa`LB6}zF7;X9^NRRSaF+wpSU>ZQWgd7clL`}
zcz^cs$-1fX$&_s8X!$XVLt>H~=egd4yzPgy_)i_14-V`2gWtHm-+0G4N5Mj1`|cZ@
z>%0$hExLG-qxMD^pS!Fh7w6(N0<1aCEOA*v?22I(0&=X$oX4(8f&H>w-+`mKZ!(AZ
zT^4YhPSe@Ve*4A;u(`KcSU3x7773Uiy~FSS&7Sl9)Svv%J}Yy*DOke&Pu&8nzK2hj
z>rmJ-?#Ppz+>;~@^DTJR2`?`;?B2tt6`RJV@NuO;@#G1d?~cskGfbJmAGA4CV8ykg
z{Fw{B^ZNhU!DW~##k2eNLZ0)gLA(m5dHIeUH{dCJeTBnebq_e7h~~HPJG5;TU=5za
zdHKj1e(7K9`E8`*1Uw7p2&|g_o3|v(lsj_obpEN{4PZB!FI&Xz*?a+<kNCA83M8I+
z!d+hy$^ZVzf6krPItB7nZ}It_T+VkdcP($aMn7-l%7bA4g3?wNR|mMPm>+M$D>}~$
zT=&%(*YoW?-NtUe^%hsl0VeK=Y3X3IKXozi?Feq?zQ2AG*w2y^CJC55jp8ogwBgRM
zju6O;VgkE|`Rs3jH&>s6{bILXhWo3ID3~q2RG!;^S2Wk+osL}65i7Ws+_?j`oAb#&
zu)Qnh<a0B&PX?O}avKaY|6t+zS}e*f%O=UyqS63X3zLifA<X6TfStqnuoCCyx)5-B
z2g%7sf8v_9lND^%R<SOwm2Pi1`}R)bGFF|yDlV49xmBT^Yr)6)T>nj{!p-|0x&^Er
zq~A^GDtpg~+gxrr@4;^Ts#VE$`Em>U>r5t2f#+J>s}tKfD$h&;tKanYJ*R75BKwoq
zm)QQ)go4#qZTSn91DOTFvJ<9oB>(usx&8QO&gLDR9IBja!0LXTR0h+!jJvt2_I!ow
z$-6AU$$1OXj<QyX;WD|I0S-gMGmRV#`)+Z3|D?uooIRi8nRN+Ryj!gbOrPEf83zE_
z>2Ghzm23Qvt9(Th_nDkVu>6JSF80IhDsX#lZes?E*DP#i4JZ!;vq5^1vB(8=&f-m_
z;Pxm;+}fs(%gN_H7iV!lSFZdyj>WpG!G2&1iUWt`Q-_0WF?t<bZ?C0tFH_zImb)mh
zmWyxpERMtT%{kxh(d7zwbDh0y#YNVpj3ZpjGB>knZ=3|T`@wG$jw`z2V6#B>aXU1C
z&0tm1;9B_UE&K0R?O^`$587OFlZrX_@^XRoN`I3B$A9f-7dEHkUF@oBL%0QQb+Vfo
z2y@#0Z3C-mh+n~C!MBR5(2WmlM)RSEYzrSf2ebJ?mvYD-+XW9BkllT`Zk$_Z=W^ss
z`pOyWvWsK+!donDk0D{ak#i~7Opv<SWzC$@g`ePhBT6;6PpGa2^WR25#ut9CHR5vj
ze8_QWM=jiMo0Bht-3&5g26H0&wD7rXHkYTdu&93IQaINEH$UpZa`vhAee8Qe7J%Kl
zDMyUc_(COE{+`Pjuo%b;5WZeE4_s%yjI(0D#MjC>*LfG`mF|=5VQMot-!-v-)jD4l
z1G~-6{2<s2kiL0lU%_%^lU8sBE1uw*x!Dx%Hjq3FFF$mG>*K|_oX^#`IoLHaxgv7^
zfc1T<V&v`!oyB!0Cy8rks0mnaQ*$}H8Pj2~yx2-NF5M4nxg=8UxmLsnfW^)`w1DlG
zxogSgSKh`o@30owU2|6ObN4PN;4Zo<1y%>LTidXS^GnVbFh9mXgFAH1c20)MX`Bf>
zciG|>hk@;zoU@g!;^ZqZ8>A<hK^|-#i2c!YGG~L%POj-m$GMDdZ05MQ%b!D>X&q~M
zpaZwsh38;*fYgC-ll5D$KA$Z!xHLmv!Q%@g2f`os#dBP<n#6TeI2<mgp1L0FHt|3=
zcD*wV9M45l!TeocyEvLv#JShAFmW1|9AG=)W5y*vw}Cxx&rVL`>jB`n`pxo{eflM5
zE@Ng{_J&#9V0|EaKv*hr2H0)lp8o7>*?w`p*`maLzgLy3Y0VOjlE6}KF>PJ0lGiC*
zoQaSzmmkN^f%R)GeFfI1`>==0Qt%L`wPOp{k5$__<FyxYT%T&k_H6c3jy0-~@t?^F
zHsCNz3|-C@bU2l3vv)Vx43Pbo6Fb<hK0L_1^HvLoJAVsTy_6$2=kx=d#g2~Lp!(l7
z;SYP@{f*2=89s5<$gJj?kf^~eKUImNK5GHkZ@+f@<`9sG0lOu%<RQ4fxs*|nJK|<2
zXNt-au$ncB)nNJBi;k=}v%0`x|GX;zELPJb#v%K!jdOe6GO$`#wS`<vhAdpIn;>oV
zYXO%ye|~o6s5`*RG3#^_*UP2HxSU?~a#@_HWe@n%0X8e~94FW;kbi#thNLkN`{2EL
z&U@OeU^(Zd>p1pLJH;8-#lsP3IGb~4<`V9Hq5YgWP7A?d15!WL1yXK<*bx$wxHuO;
z>Hsi5AeVimbRee@O96M$v56et?r1X~-`B(;vc8`^=5#)%_<>ZeyM0aYa8-_o;dmN#
z58N;77f#{g@|g*C>z)!P?k!i{z;^CouHw{dn#pN!>?!-7qtn=P>mX_D{PtF`T9Ca0
z!Qouf)&_#jeg0RPV^=p5k4mKumuY!2hu85RTo%Wlva=rg$NWO-0ay+563AF6Nbi5y
zYPdNdJ_yS;wR7o-o#DE9d=WTpuh`TM4j+)*0<F7T`4{ei&9Quc7%aXnB8JPwCkSqi
zd$2Uv?I88(rzUa#6t)HPg9`h&qq#hVnSU^H&oy1f&E~Wm-1habWfx}pCoZ%%*Mqxc
z!#iP+zE9zfBD^0&!R#-hGTfT=_qii(?-c^cA>*~SySPptJHX0p-!9ZqY0C3w+BKnQ
zeRJVWeG$UDq#@&9FQ3PACry|tm@N7P?8Z;`bh(#%dvQ<6S`Bs=%)GZzj$nBZzw6L+
zk(?L>?uX~sag}V{3AVR<AFoKs>lF62`>t{Otz9I%eRqTiNNobkH?9{3Y{CV#uAIBC
zih#rHk>qK(Ss*njm-yN3Zo6?FJj2bwkg|~T)L$WRTE6i{1?+};`4-_68Fp}Z%5v5S
z<tN*4MjeLC4NQ9Ro6B=e8Q2YfFLZOda(?DoDzCyRp}K=LF;xt#XUAe0ZnOJVB6nJj
zvK#$(1gkyr>InCoZT(y+;fy?YXF&R3k8d?`OcHAbyBp*#!z+uyaS3AY-G_*05FduC
zqF#gT0P!`|T7~5}mAOSuFms1H*Kn6{WwYx)UII2x`Gyvl2B~B2(-+B^I+<Iv7LvC<
zPRio4lU~S|(<aEpTC2rv@n}8!(%xvc3zN=*&DP!T1E$$owg~CxMsfVsE(f<IgEwdi
z&D8k;HrJ4WkNfD}51jtnWWiw;bnUX}?-(`?b%jJQf5+T%Fzp}&@&6}@`P_`RX9>$1
zrf{xq7XhcGOWDUbdNvvezh=A$wlC&dC&xS=A+Wx0%k)`$0)64(0rH3F%<V!uZ+_-H
zKSP3B?MJ()+=MDlE=>cjT3b^Nn+qGkW`NY4Y!wqu;D{BfG*}@VFtJaB$)ZtcGEY7@
z4Y<{97Bpv#6xtZRjr)FIk<gn853u;9?;IlWF%jT?$!yjbk-knh;iC%$!E%p{u5n*q
zs|9u!$o#(rD>y#AbY!zsn+Ud3CG!?%wj{()p?gnp1}dxq`wyh{i6S?gU940qn8x%T
zY^Fk`K9|>XUSYltSz*2T;ov^N8b3>L*cxxw;t<U37p~yi%`dmKg6mY(N3Oo|{c!t1
z=E86>|5I+mQV*`<GS=+9YW>_F{guJ$Cg%5ZSIX&fak1{^dYUvJEC$juab-8y&9fuV
za0UHs7TFNd$}Jjk3!G0r{Fp1qem+O|*xAEeO`lan!)?y9IQgyNyjS;|dsdgBi0m&&
zTZV0htI#V`17Y4Lt=zTgmHaF76h(aByybK@Rt5XjpnVQ^Y>ueVC%uIn@ymF@VG6Rl
z)6rfKM1yd<Rt%S!tAyyQZKB+4`&GEK?>2({9^7flb88*5C`e7rzaFr8z1JagPJJI)
zxF4{YfYpKIVc5?ikA21a4<h?3AmfGUKi3N-NIm4T`Oqx#VU{_UlV-Qb9*NoDFb3%<
zu)oLssYIWvSYJ+v`EnCC%j(;l=j5k>!v!RNEOZaY-mQDMW~o~Xvx%s4X}xR!=V6fC
z=}JwmZ<&Uo5p(_7Pox%e7*AQpc74rs?r+NbMV9ee@XVOiF1)w=FxS<7NPX|Bb^~l~
zP1PH49BA<I3m+=K$Tjcg7p|U?PaK{OTwKe(MGAj->CMq`Z@b84Gb_#oVI|zMq5p-_
zYs-Zu-^${y-Mp23LR}i%-zPt*^CVjoa>nEcb0?ax@UIsR<ofL}RbbC!b`H}Ska^kt
zt;@I{PZQwQn9Rzv?1~hZ$SiqqnC@K0&siNL26l^a_G_@ewtZR4U6WeE)pc7P%%8s=
z;&zaKMYb*nr`I-7TX0^K@E7Bn^-@^q%p?VHy#7)%;Qk@k%Jnk$Em;16(o?QtOC8RN
zL%bZhQNrwpGq-ZDJH^Q5KiQds{dbSBQjVz5FII1Gm~474FKlW#kF8+iX>htwPjKRR
z`xnv%bt`G-^k`D&(p$Ni*>PqH=b4&YLUIA~xqj8kf$askZR;UnaJe0p;UnDX)X&W}
zDHiMokeps(zDVbe4<d?gXSu7Zyf_(l9TWOc_gyf1^F86?oj*B`rF;{9Y3$0)Yjyz~
zuKP;YaI#Cw3(IpS3%l_cae2L{1e?{qn@NP(`2{$<UgW;#I+f!i;#<crT(fE>7uPg9
zaCj+ZzT}YIqX$l_3iT=6)Bgv9)vr4)DU$uIktbxDm5BD!|D2JP6Sx!kT)^>g%Q25L
zt}~y*P-+%9d_iv3$qfVB8NHPY+{Xur$=;L{iOTKe`n*8~9A;Yus=;Q1<i7t}&V9rS
z(pKBW(gqd}sawF^#n!;|!BY-gpQ{Bs^IVKF1iOvt#BqK`{Wh@KSw?;W20R*k^Pk+}
zdGW$raCx~XSpA$*PyYYC`Rs3Y2@A*`5d_DJ#ii@u@*ia9l9I#xtr^0C+&tU)zN}CM
zn*|aF;Zl=ifk>xhJ~<->0RfphK7o>hJn`3;@;u%3gzui|eX#jK7M{GbFR}9!D24HT
zI%~y$*}01M#OApI`%L%qw#}0O+XpghK`aAUJ&5hIAy(kQ&pkX38<hFa`$+KgPiGUj
zzAlx=G-nd`m&zr)wwfIR&ePt3-RPE~%y(+SIbQCYJ9xgQ<_Tz8igBId;seWp%u3s;
z&N}hdA%5nRr+hlk`T33}rSdU6$QMYde#7S9B>?u%A_)%OJ-5zrzp7H_I=GjG^XJSQ
z{-)JzV7VyEbe@7Wkn%e0YABC`{5t*#>vj06<yY|;w-kWgJo5k}*uK)4ynIicAaif2
z+%tKbPMGk|F`3J|UI<bqo_Y9|vq8fKY%a)c2OhP7%cy6)YW&ZRi^9cuc|ADlg)e~P
zUoo6naPP(gVDq>Be$Knh@`Av>|I&P#7V-i&q8d1@B-Hth-SfGE{l9_Ds{Ox`pFfzJ
zkJX4r;PSB)9?gprcrtUY2&Bm7@tZG-;d^@V39nmN8vnEEA|4ZaRUXYBdc2a`6nJLG
znR1)X6XDtS%7=gBYhkck?i^*~4VK~t+g;mZA)tKb3OD1uec=44xP1;!B;Pi^b^qkS
z@^7Lu!E~Z*mq4U=3(u42w*n;+EBK`>j&Z%-_#YfjvmUE*D?i`HYdK4dN5WzzkD&z@
z_razRuzBwdtk_#UA>jlHn@H`eyo+2d`S}>8aC2pv@|>S_im&2Yvw#Its(_@33U9FK
zKCpjdBMf*iUaaPq)Strt@UJ2_Z_*L&x~bQBeRnZ{{qSh(XD}V|_7v~sK4ZR@;Vj^A
z4UAA{e{AXrPJgB%y=;CrtOZOjNbpA)Tk=1-zDD599A2=0KyEqw>J;Z6aaQiUiE?1|
z0dtn~KfCptFD%uTJt%uFcN?=I*i4XGrpA|G_5*_{{F}6o^6`Y8=1aM>k<V*>r@;Dr
zcHTU*%iK?r_w#SI_2oJJv<~ctd>3nOea3tI@3iMIuX?hZUwZdDo?mAj!FIL8hVh-}
z?&jDYKNp<eGmCEXE@%+si_5hVxOT3bcXekL*ncrgI{3DoJj?g#+Ec#Auj2(9?gVqq
zoX{p9cRUgtzNz2;@oe9{i&x-ZBkz6TzhE)3o72I1yQB<x!<jGeE&lYL_nN^CUN?hv
zy#8Ic`ERn`1N+73hb@oB`$o<^3R`*i$r<vqCH>&%bvVt}eYQ`)QSL84$KpBs9-^%r
z&fALk*;o7rhZ~2c8MoWP2(bB}aFL!;&d;d*iR*#(R6dC(S9$6m#q&$3PUElr>Bx8c
z@*Z$Fh%I>p_M^;XJ}_TARfgy5fyrR=KxV*jK+_a~eUT1)_1mw4)q%t<4=&*kpKHwR
zU&zFhZ)(feuegf;gvJz3Pu7=wlHZ^3E_`~EJMUl&-}wx0-h%gm{ChXP5xB<~2-d&)
zal3%X)5(0^w>I*db3foQIj0P+XF%pU{(lWtGxOd#&Zrw#`6nFy!tL_hP9UX-TVPw;
z9$uyD!~E|B3b}*+>;}8j)}lpV7kePf&X)_pe6@`MJd1go1Sb0O^UQcPPoQ=8avtf_
zZ33GQ-Q+&-`#JyRQ>(elcgz5X)32VdJU)A`2|VKQ5jcG*3LN%n<#+k^`W+H@rqM1C
zA-P!K!O|1FAq$uBJrg{`zaZuaXWG7N0v8T!7ij)@jdy#`8v&WVN`dWaj(les=L;;~
zpUHi$<tSg)|3Ca}O9a`O7M2Lu{angB^YUVWoS!m+y?1;CUc6n+7P`Bg=liQGj4L~~
z@NIUT3eKzhHa3FOh|m>ozQr<=_$RFT3l4{})EM4`>8rUvoSiS=y0wdA@y{nbYwrBu
zN_gJF_aGsj=bMeB;JsI;cru<#a&4`<2KJZN^H*TAK;gN6(njtMn^oXCP~U@-=Zm;I
zSk1vsGwzLt*#*OGodwdW*K^OgZzCYh_?xHk<9ptzx_`j!Rev@Ou9=GZJnz<NayU-(
z<yTG-0jC#`x#la`_|-Tj@~xiABRHq(2luXXkNK`%`ON>vYQKQxZWV#+3;}#+QyX{<
zkDlc630=XyXtoZYyW@GDyQejHmKEsnT!=aVHp~CP9{xUO8Nu#l>0DeN6a<oXNC_M`
zdP2bFET^Dj{C6J7fc@N&O^gEfmHvb4?io2wyhlSJdDYayOYlu|8ei<|YW^84(L6?i
zUj_PNJoq069^*6gx8i$u+=n;tm_FFQAUDsw2N@&3T%ZdNpI?#N_@+2<@p6A&3wDoI
z=2@`%skgLv52X9>{%aHltBrH|CtzXop7V941#e2kF@eU_Zs0I7b<GqEFX;o17jsDk
z^Jo0tZG3Uh6^>(KpM>^=el%SBL7QvR?LL8vCvxQT+-~vy5nwmhSu#a-F^hp%_jNUk
zmGi!HdAwsbXZxe7?tS|!@5XP+{1cfo1m<ksVVZeAMQG=j|KM`zn4k|&GM|fZU*CQa
zhUxO={7>?@-<I{s-&XaP<zKo*foaJ)k;*0(=9>{)Ig|Hfm|b1^n3ri)GhY^eAZLx=
zWTDm5ySQ0*6bSH6umGF4`eP&S=X@uypHn`cFk?~9;+lBHhd(OXTc<^8m!Q!z1M|r)
zQ`rU5S-IIbrpvXwT%vLNDJQpp=@G4!aV}<;9~H45%v-^4=yFleY0e_y<NlCvO53W=
z5$-F@Z}>Zv&*_a7uf$bZ{vzr9{Hv$sa$l(}6$iybnCc_b%2gbkpW|70v*i5DzX_h;
zYd#yo@3%mPbE|hNXSBAP8RKUS-nrotoO9d1^BKO&=JQ-C#J9;$8{8fgR4n0Iv5eU~
z?2#L<6lb}@XJ%s&D+@l(=S%AOqQmmJIaGIZ$;9sxtSR}x>a3?B&9A#gs4Z28=kuhU
z;JlW9JdIC!QwunM3B?I>Up(3(Ag`s*rMdei-yscmf#v!O!Rq`&7n#S*xGelYa-N9r
zxhDRyw<;XJ{g#_qE?mgF&R0~%?Y6(5x|fhZ|6(Dt8vi_g^Y!mI<$NspkGQA{gw@$f
zyh&KVyZ$*p?-{l!{7V^?1g|7r=c;vGz;ki=Wv;B7to;0I4$Fr6Df2IN;nYx!&u1^)
zy1@LVl(bpts&KC4h_(E;_j9w`^Izj!B%a7UuS4GajnQ$ww*6(C)B4N#1Uv1yP4Au;
z`2N60Zku+eh`?V5fhqobq=Uazb6UMVU<L|jHRT)pi{n{kIfC6;%a+~albHR6=g2z-
za5*<+eJRJ8-J;+;1ybuZzmxZZudV7?c2(XK{U(lgYz5{_>bv-_*B;@2%pj;R-}f^=
zyJr#4JbMnId5z)9AI)|sKD554By_4<<6_?vGm#(8{5SdhS(Z-~mRiiuBzyX0DVMhB
zTr<%4e?mm35dW@f^97UV2vp8l$9wXVsP>}U-KM!!-^B_#yZP+=nFUlfK;{EMZu$Nt
zltal*LHuQJxxlHpQv8Cy9tqA3d?l7CIFmQ>=w;3eH}Z`@>Oh!Pn%~Ug+Xq1lQ9HpC
zF$NNIwlwoS*dW6<wdV?den6h->zsdPWigqiCm-J6$zS!%6r`vAtBUZxW);<=9t~!D
zyd?NnyUgKon>bA%;8h?`w&WDPs{JRqJ|F7>*C%IuE^~<p<|yY&_47Li3UYbyJmg<^
zp_ps)aZZu5OO^^thb|O~TC+jGK=vumRo!;M2OUQ_{r>BT7(3>3d7s>8s&{lRXYIi|
zB3C3^IcHv4FUj2a&ur_FEnFL)&M<v6XAjR;&)Mqk%Qd+D*`~-WVdUlsxmd=>vSE(t
zUWOR+qZfF&&Q@&K*kP(<z9W1NpY#P;g@=>h3g%l2^7pZOn6}xR;Cr$05;zZEIi$cR
zzbjKhy^~X5fwwu&#IARIXP(QMJL>52g)A53QaQkA&br_upG2b__uJeqvsoWk@cwO0
zGfNDX5n<;(1y1k3oWFA>eV(c|*)oXzxr_<Z$<WWdwNeY%``BObzx47kuUVqP&t0&O
zuWMhdLe69%fva<FD9nFbYhpX~i`3FXU*-OJRB}I9CMr6aS(^J*Y#`?<6=R8=to&SV
zva|W-Z#EV1X6ysoqq+Ase}RHO-_5`dNe{<fHja7EB|-55!e?cD_;V&Sna3&da>z}z
z<#*lsl;iLlE`f|+tIh9jKFKwy<g4_~l@>gy^Q^fljFzzay*kCei#?G2LlvZ*=@KQS
zv?=xy|CGlIIroNt=bzfg%sJub5;GBw&zyxZ)ut+rENm+;Fbm{1{5H9Gc!GxXdtCwF
z$usy%G+*g~?B5??!)GHWCvY!gfly%%gMiEo6)FC&iF{QR-BO_O|Gxoh0{i*+1#~0-
za+xn_73$JjDY~SJk8i&#r#bU(J|TwrhxlUt)$wy5a^cFE@J@a6Iuqe1f4=Z*^kfU{
zbL^JA;@!`ATj4IJX5$AI>k27;6XR37scsj9r0s$^wX`gRmA*NfUa_4dJ->8?3CKN%
z;xBT!hJNOp(!NCUH<u3gO8F+vGx4vuPMg{a|8f1uIqh7T0NaYU;PL<Xizd=%uc>P8
zYUAQHj+7GIl_+2qy{d{MDqx{3$gB_NoH&ZB7J$pW^?X}7p4+Yvl<k_xU(_|5^EJDJ
zv`oe;&YGZ2+((ZT^9lc*${S|(nN#TpH+RCDPV@OkAMk<3{~atZ2x{vuVq3UaP~l?J
zVO2Kg3<2MJd$``8<1n+(UMwemywTiaRi?m!vs0wqGNQS%-*=kLJeJAxysC!JdCwlx
zpWJ_ya#*f#wrNh{=-SsV^i-zC+@-`_XtOU9=cYy7{L`MLnQ0z&(tNQnRUpxFy1DqS
zP2BYkPW+Ycc9?VSkm3#Ij^$b>FiGI8sS$@XSGT#ivLa7Ftb^vt*Y`MHu5=REYjr|C
z`ca|)-x(``-b_(-u9}_TzB$PM${!(P4hm<()R&jW@UIjP;y?D5RgUYG33qMq5w2XJ
z-MqmmErM58Cv)vsVZq;Fewb^;+v(sk^414u0b?mSp7@!vTxpN^1@@FAONudV6+F6h
zw}9B53E=R(!D24(Y`e4Bi`;znuX)iTF&$rxt8{a?eps&+SfT$${^gg8{1#P8=GS+e
zkaOC&R=jWLbc?-RN4fO4^ekMCUsRvH_#dxex-fs&n!N(Q@>iIx$jK71T)iJs=WRT|
z-Qmh7#(6PBv`~c2;@__=JW^{56xQtCAeVkfM3M35Z80x(LyirFVcfrB^UbZT4)WCm
zJMyjGdWpO4E32^B%4VL<zC8jJ-CE%A2v1VrYn(g<To=eY8=0>wn8<nW`$hgQhqmbT
zof8+1>EO4}IVaDx>Wny#n|rpLM%F(~%b$H*I-gQ>^wu|+9VtxU=6X7r|E!UT$Yd*R
zv8pa>aCsRW&(7oTG@t**9S{DX#uMC0^UC<&TxA!K`*DlAk|R#?)f@|+OED8oLmxA6
zRc%Y>pCH3;G3ihzpBQf|sQzbD(^=1X-f5EgzkYt+_q(0A-<WUXd%7r|@96&peEDvi
z!r~wKc(c5nxW0M?nKwNY<zJI1t@wDJir91Io7{Fy?tGQydEDjxZ`s{1pBJ*3-@$pl
zuSI&|{AI!h3Q@d_M^eD$kL{Yjyywy&;|(qr;oL&2429kcpW^(Vv5fzTovOfwt3?84
z#>!k~tDG&W*5rt6cxf$~$NiJPGLn_`_f8)3|8Gw5F|atvFrMcSx@O}dXdAc8d_T9Y
z0FVAF&OH+r3LHz?C$RWdi<HfmoxJDXzvaz!D-w9Yb6V&?pBh&Zk0!5PQx_Y@autER
z>U(l&?6v|-5=>fcvi)2tY9Gu!y|l~~1@CYk{(h8y)`dLo;8GLrSQ|@jd9$PD<&w4n
z!ZDY)udUP*a$9P|ZGLC3(5}Au@+%IRiD^Y{6Sy#Ufz1AZXPoE#c7V^>W;gB=*yz?T
zJ4sNK%WhM?z_o3wcveo-2G<wH5+Ypu=YE3IHYjgA&tep~vO-U7seuQN%$<*13jY>a
zOy0eb|II;0fje^+D|*F*2spfN;=g;SSu|v=f@+PLj?%om-zqg>sTx17T`|)REaK1H
zwVaDpXNFXdUZt#7S|E=B?;`W|8cFs!h9`ypHp^KwlrRahYtG@lwbfPI$?1t1o8TEq
z+p{zIK7WuBu)Nv_b{EJkALbt6j4l5zQMGiQfW|QcUPIm&LRqh6B_^&5<(re)$@ya5
zFL2oiQk%6Z!u+CQqlmHFY+;W=Rmt>wx%}c8<$SV<tODVaCCr~IO)}?Fo?xDFA)DX$
zP&c?;#HBJ%^l<ZRwPQCInRnhW;jd<5=Z-(YCa8L>oLBGU68^K6ecUUYcYxE%nsc%|
zZ`!<7B2ORSe-nL?dw0+jfgY|3&h74(MH&{Z74dUCFMQ(%v*3!On|aqt%ob|cFUi%d
z&>^zpw+C0vBysatgEic%g|x(v`Zsc}vXPfM;`YXTpOh2#xy}V<vQd}$m_%f?^I8RX
zTn{G5atGbyZuYL@yR%`n`K#Og<|ZCyJc18zX}(i;u;^hw$}eMjR4KT0vd9v4Edh-L
zOSAp5>ij)98Q^f+wRyjQ{c~2SCok6V&$LkG`E;nBU(|k<`G?J$`A^(5<25+(!~978
z1b&4}?L3n%-ZWQEpTgVMo@*}U|63$iKoi{OnZ&Y)vvq}rhV5TLj`)0M*5!#Myldb2
zu({vy6bNZrXD(;x$FJIckUv+=QL&!QR7kRLyW+BnUNcFSInuZ9aVS*WzRzo1+ACK5
z={olv#}`~o{tqSE|F&^ga$4~@ukRAr>)Z+sQ~Qvm0$T!<_)Q(%rOp&^@K|-;kpjgf
z2)l@F<rnvsuxQV><=|dj#Q$)E3%kRoI|8z)ZWel;);upZ@0T%Xn#z5!*oEi!<X-mX
zLl61?@6_OazoC=cc-M5U<GUv*2_E{+UzfIw^I)Nhz}a7I+;@-hSsXpHk;~S}+sxOC
zh5OZVcY!^p0?Z3~R%#YXx(loeo5=4zyIdb+KUeTpzTB#E0qX)@5$$EF0uLesWtvpN
z`Bwfglm^xR%N3mjs=G`C+%lMW6#l&s)~pbb2o;dwf8pk79u)Ch*v+7rf2rg-{wdLs
z+<n$dG{16O5t(q3SKt}jM1kuZ%5o(&XLw3C-{-s$x0d6^tn2*Eyxn}R`?y7SU$*A{
zdxSxR;Y+94?2E0kGOWVjJj|$Zk=x92Cud>OT4|@<6S<o>Ot}||SaW)pON&+C<l>o@
zbW$KY%YY+Ba4}on@l(?G9oK30#K-e43vCjV*<@z^WT6Y!%()$MAiw2G>vCJn{KWsP
zc`~<!%rlPmoPMFnK6Ck*WPLf)yB0}D&KBaDd7zP-r=*$x-||&_g<65!_19JT%G3oc
z79I8Dd)^VoadiTp@CQu+4(rpwifx%ms;XC33LFqS!+Y!2E3*jRM0tI-r{*r}-V1t{
z2FV=w{+v50o8SDBp{RhwggAav11EFw2foT;x36-_U$o*5<O&xS`D<eFZ_-5JnIhUe
zaV_5b_b)M7*e|-Pb>{pP0gY*?7F;(Z`BaQE`2UKmve>p>naBU66i?0mW`W<LA?&*Y
z_$@B1kmcFQGhM6H<2=`k)=Gh*&=vA~mGuR+e)$NfEHl>JVXzE59{}>d)2Re-`U_cc
zK~uY96Mw(3wSf1X3^_f&cCOFI4Y+^*=Hk7Ydr?Gf-Xqp~Au0lj3ny}4Eb;-DkxV_6
z0`-Dw{I@o`aT^5x;IG`RCjDerx5%xVrv;8r(~tp~|8cE8{}Rmy;Pw^ckth7;nxy#V
zJi5jC-DaJ@T#ZcdI6{3&1lSGzi!JzXM&^R^-R@Kka2^Jk5wwAe|5<5^z_XQpd~agb
zfXxDlgK&!64gP2BTY2|<@Z{eXQOv#U%|))C^>W+}S2yr%TOSOre;F>TaZR7%&n5k=
zfbR!)I$y<a4X&VDECLgp&T+dZLdG9KX4P=+f~aR;VE9;Zp1<05J(ovYKi>-xZ%(Dz
z-TV;;E^xaw^s_QuV&yiRdXs-`XEWH1T?HLHGmdca^+YFe{C&QI-#WXIeRrk|+$_-e
ze^t6Ce>3|5zVa&@c{-je@+PvL;Sbq!n6*^b9_*j_U)S^g{dbAel7pASl5GLoG3gb2
z4=ftMa_x@dT>LwhfWzp`2X@Zv%y)chf8zO?m$dMu%4&hd_vifp$BFT|RlN7VW^#c1
z6Bl`eOW|r2-_A=h+?Rgaf#sQE!#V2KF9MqjQq%W&8CZW<pEuv_#%8#9Z?*vEvqg|K
zTB1JA0vuY9wWiM}9N<2=G)!Q_-<7<TEdKmog#WYGaar@lhYN8EdC7wH2Tr!)-<%}P
z%{`@$U*1rRE3@H07w_2&0hWw8d`p>^^6E~X#am|mgs*zm9<Ftq&Rjj0%Xv;;{>HiS
zuLzexkuO)vj6l9cF*CqsN!(q@b$6!(IGi~z^zhFRe#>g|?HJfEn#(8gFl}e$>k}6T
zx7lX&9|6+`rhnq!QWVL(E2v)JuLLuH^q*vQ7Do@TKj*Gq!CrV`4sTljcTSJ9joh#P
zu4UajqX=x?&y@QdS6@_t!wwWS+hZAd8l2Yfeg3P%@olR-SJSjUKF8D7_!Z1<@J~1$
zz!ThB40hv8aY^nSZx-{luRF~*&v`GW#iC1`#+pey{#G?$`3KzX;50p7@&wm2#X4S4
z{SPwV`SAs|uA-A*IkwF^*qeUN;@|CbpYJ{23O?UZae-~K5o?9~692MUyfS1p)1C}g
zZ`v!tul22iS39qj{Qx^Z*UN6mI3CE{=MCK~AX-b1pZ}Ux4R7MVXg={0559P9c7Z*c
z1$etFr*W!8T;rQpI+ttt&O_jENw{jyHPMxse_F;7rZyW*{*?QhIBx{)2HSPR*n%g!
zYAKsE!z6Gxb(g&2I?~p{b8hZl{-@WoxN;n>f&JMntitDAvVm_&3o{?@?%M*f+b6TP
zUU<jP77SUxsqs{UOJ8Iw_wweqTpG)^gY!q^#-Cul>PIZOI0`uV9GANAB&EOQHk-4E
zt23v7KgVk>I6mfNa&n3I<Zyi3KAUU%lihqXF0t{O7VqaZJ+X~{uHYhm{!?=N=PP%y
zZ)g+dzp`!-IK6PxigINJ&H<Yb3Xc`*?(waT>*1=6n8ceDz{}IQB#%$g-;{q&R}^>s
zqFAuHuXk&~Y3P>d9I*R%-^p@i*CWaWkU1c%Rq~zRi0viM(V$?kK9Ja+5^;V%w}qUY
z2|YZ0d6hhe1dj2|?_A1$gZB+@a;7YIZ~9iYy`3+3FMpfIbzCcludX3dV2Q>?aQGiu
z$|E4h&&&7DuaEy(w;lU#V>9r&Fp#<bpHBwcp`dt<P1f!;-|=@j?1yz`^Y<V0<3Ffx
z%e_j(jsILvJm(HkNwC}P&prY5|5@7q{s;4oY+8A4Z~e$0o)pKm*i%8kby@(2L9(yF
z*Fbl6N4;76g-_?QJpZB%j_aVD``p%QmI8_om+<fRRRhQO@`|Z^r}}&atQ6P;9vCPJ
z)ZAs@3F<$^J2N1YKVfPiTQ0k&z#nE4fuIwIxvqZA6j0;d!e6Ld%DelAsK6(K$(({h
zFL~d%CV>6GWy&RB-guDvd$q8DcxjKoO70o_3#(Mv?2R^afyVz+eNFie`a$Ml*U1!u
z<GxkHf_HDiS-#uljbJmp<vF<GKId^hKg%u<+hfSarN4u7huBQ^%Qd{b*Uh;&FD`5q
zFcEX+k_qf*zyH(#?61H>=fPnC3eQb$<?LpYAbS8=tfJY3jzi{f!oHkgU#HP4@NVmB
ze!dcJ&Oc#)`ERcD<Feb;$o*&UIq<s0gaS^^swy3>SK4K4f4-#hxxRz!qXn6}^I-vB
zn!695<ErTbD^=S$Ta|<OnyfqcBku4Dth+OZf43nspWVWJoK-T-90CWTI8No9;dywg
zk~_W0o$HE#F4wVBZ@_atmFGM7P1)xNNKT)`!Lc=m|3ZHoKW~$yz_P#b0`pe6bI-kY
zk#kCAIsdCSGT``0ZS&zuoG=aS&%bJm1zxr8<Ywsh<bO3ukn^-=u7La;55C*S>-hxq
zO?V{}4Y{{^<bcBw<mN|yOyII&;;DsTvp{0fUVeOQ<~`@+3=9E>|Kv@OwDHA0jk_^@
z2iH25-(a=T+C>7}moH}b+F8kETPY!+ANdUI{(DV(1z_X<z3~p*>Rt&Z@0vGq&A2Zo
zTy#FbsHUofBcXZ+@86PI`TR*A`Rpw?%{Qf7mY%vTOKk2D4U1>4m7F`5{xkceHBX)8
zt~KB1?$bPT8_W1O)AG%Jrc4q#dvY1rubo_eJYl<3h2J}Kh(up#F{_q$<osXlBmZ(r
zsVrX@m*O68e{ehJ`YQ>}dj}tw9(CyA3F%qPlNw;oQQ^5ousOz(Q|pjBul&SH@c6%q
zx(E-u-B)lrNhs(rQ;JvP{C-N8x5%7VS7Gi&LFR+H=1*T3b1s=5!Rh;(MP60nqIyoB
zCC>~ER&AM00%rc*580~9cJXS-aSM6w<PeE8&IY${Kw)a7Yr|9eZ6c4^32Pqb=?Xk<
zDk;3*7N>LA?r@d>$xpZ}Xjb*;4?CxsH`|$YN6mKfoAJ~IGw}HBDq-LBE{pToo&{zf
z-R|&w(o<wp5Rl-F6<*9^ow<g~)Ib{S_Lg;p90jLY&Aqps=6u%lM_wR!v#{T;UbZ7i
z`+3}wwsSt`vF7~7`bsb@%ZB4iYlZZj$FqfU9q#ZRxeb|PF3ynRJ)`3dcCXr>cpmnt
zl>&?_q&d#+H{i{UGT>8qBFYoKv6c1kZ4>i47aoymo1O}9y0645Wd4Kg{qZ2PIbJik
z42;xdzIol{j};IT(30&k`}og|yY=`Awre|8_yxUMdD1wnCAPG`=lFHy0@t-m>-ilt
zA_Wg$UC3F_$j%j>?#{7Mu#NY<=?a;nEB^5mJaN_d^K}<{%(E9}OAl=^1@-^;-Ji$v
z?`Ii%&%UE<B_R`ezB(9~`!U?+7Fr$8t~_@k-?nfwPDjDz{2On6mTUZIB*H7Qg~!D8
zo|N$WZjOK*1>k<}(S2e9Wokvz@hl9im(DTrZQo?g*A;gVT-GkSX2@0|#0gIGpg3{L
zKgD_Fkb-L9Bvl?B240TH-udQpL-z4XwQc2jcF0lTVyLV@khnGPwzOG-X=k=8oyuoc
za{TY7RFqY%ao6Ra*}In?ctGR-xtymZq|dU;KHts4{%dKN*_#Rv*4W@xLM<85=2`Yl
zyehY5@})K#YCnv;Z)y}-BJSv9&tuWy&%31vv4(WjE+)=z4sznH(c5^Vx|Z<%b9yJ(
zz_?zl_--KY!TdcO7gk3bgWL?lPwoeqY06I*6#X+<u))z^{77IJ*RtDZIB$6x@y<VX
z&GgIp17^>vOwHO#Kd`_2yBj?IuXe;t_;ItE>hq*{v$7=zc~&>C=Dh#yF>hkK0=MYw
zP;OBDFFxl4xL)1MEX(nz{*3aq$+LMXBWyX?|G(x@(4Ng@Fn@}$GrOCR+3b@-TYgO7
zHT%ZEnd#6ec(MCDN6C{y;e+Zy9Q|pJP0OxM<#;P*D4Ly+!LfTwgJe&zuNh;QIY(}x
zw%G%ZGdznm-fLc7GKpQkJ4Pma-W*=r!_0iIIW5e-aVMI48OpF7KAxvOOIg*NqhE_h
z{h6jBLo<_5?Qw4IPj}~;SiGLZ;aj{5oQF52|7P!>VkDt;NLt|d|7Bd((VKW`qjs20
z(=Fwi`l*3u-nQ*#+3hksuX+VI|FNGl`?6e#hjDg<ndWjK5qDupuwR#pwQ;oM1*%=Y
zyNm6L<O1fq3~iwCe~uq_yLkB@w3v6SoX=x-OP=SmX`4b*TBCs5r+$UR`m-jS{%q22
z=hw(A*>#g$XuqzA*@2@Rb0#vdpE<%Vk$zc~eY(_Bo(qP3yqbG{f%6>07ek&lo~69o
z_A^O3#yfH*_}+n!|4Ys|##MU4-n>+28gq7MKCjZLF!lrg9`PnP+%uozafq|tSW~({
z--7ot?+y;BvTIDolRUv~mlIrXz-f7%%SMGGUvqdr+lX<79Notgbb1!MZQOseYgR(+
z-V@DDZ_lscn0(KayEpT<>Du>?)Gy{P=H+4B!h7&?2YCGd)6OQYg^LAw)HJRMP2s!2
z)65qu)$uZu`*XVvX#Ag5iMN3_V~#0r@`D`CLoADg+Hc$y<(RRXd&>(&^JgNyLQ$dY
zd~4R<;WMe6!FKlPTJ^gJbA%^m*70`mec<ipStYZig_$|MmJ#e$rld)H@w?b~locKd
zaffZ-Y`tSBeD}VOsdwCV>Cagm;Ix|7AkT4il`zMHYZoLz_5ZaKeH?u*JGuNS^M$+D
zT;jZN<|?nbrY2|bh9cIvI`^a`+b(N-7vIaZy3vCFSgneg&a3Yn0?Ajw{nIb(^_(W}
ze}m)tJkw7$V{JEq>ruMA@t>};b5HG+nk$jY@_T6*XLe5@e{hf@PjP7=Cy#(S>+2H&
z<`0&v<@wUJfNkD}J^^k0MeMmN-^iCukWpPZ=>yOCueRL9pYlxY<GJNpza^NTd|D!q
zK6j$@_16d36Tftsofj2hH>;51<~<c<_B+}`vG-9YXT%h4_G|0e1%(bZnI{}?5sKg3
z%W9+O#k*R0uUYVwKbliKocO-FtutTh<H_^q;td|}x%TGWoX5ED9iPLl9#X;QGUFY4
zacH9XRHKue_P=Gcg4zUG&vV@4k(;ty{^FH={2Qvy^ND`ZP?uqz2_EkTl@)cy_TX?|
zuqi|Rzr{X&l^3`9Z@tTv4bza|{PAl#*L=n8ybmUC7clyo%XMqAC(oo^0USk(g~9dz
zW03~l)dC+mte5w){%aKGGZMcoe!Z|+(05)O-)gsJ@c2Ks+8lvQZ+UPX^>z)5p!|{!
z{_v@ee2rqa1vk51<pa4vUG*jZ^7k*fi$AOrn6h@UU{{emd*7Kb{%x`W92cg!GP@ZR
z3rOm35e$#m!2h7_I=Jo!>6<OU#Cdokm*AI7Nq!-PNdiXKWCeN-&*7_XlHq?bZzuoT
zvylF?P~{)KkemR%BgdWipI_L*pLNNc|7fMP;4IBLex*e_z-|MXcdG0$-}XbNz-<`A
zZOnpKW%>BtTYTa_=NittTYHtjilh4cy?L&@b29??<Mc!XIhL7&+fmDB_3%6FYUh8u
zeF-1?uD=2iq8fY~X7da9H$e7(W!L3!b^4VH%t=1PAG$M$FHu~NzrUPc@W`w+oHBLG
z!Ty<Y$cb;u*D_x9Eo^+#XCCFbyZQ@1)7|-CxkT#=yvu|?@Za^)VP82lgzuE7mB6*$
zt^6TuP5foy-C#G^KMCNoog&0BLyCpJZ$A(3>bcqM-{iyiRhtg*FD_-{|9`RpT+YM^
z?BZVLcnfT<xU)Ljwreb4wyEY){=Wf`bsnIwkX^Kccl8B(aK6}4(ku9S=`(O#J+Eu!
z*EDVrJpA=KUzy}Nf#YY5xQwf>@E<xnfj2*+3T)QG)MNqqGv)lpB~J@Hy1#+<mVg+4
zw_J~4PP?aoZWtf`xjbe5Ge6V?e73yd^*CV5`$2sZ@0BeZcwPzy@^z_v<&%2IE3k?A
zy8vka|H3>yzSH%4z;-|4eJ;Qh!_KF3K8#OjD=+KUuxdWn{I~oTuRnw3H+dcd)8GA+
z1TUBK^0%y<DL9MKL!kJeKhK*D>%rk<$hVC5_J(bIocj;+{(oi6zi+h+@3w!C{oT1&
z-*YogUkolgL1Az(+>f6j_%;8n4|92Id06>^R_){O*Ao`}e2hcz`_)6dAC|O${mYS`
z!}sBYguvpx1_I}!j__S~R^*#HeHH)D53|7XyJyT1^t}|!by{W;f8LQ^{>!qK+$MYl
z?B4hPaXC(45DM;*=6b})!s*C*PC)x3gTQK&Tm0vGyab(-AY(otw^&xJ<vzes#mhEj
z7dX8#o_80>zR<<r|Dl-2<mChIWZu(Yzk}4iUE=_bw@s()1h(WX;NQ`;hoALDGJn6C
zs^Gk5+xd*=3G@0k2Mb)_^5)~ZBnfuoo~-M9^>W1mXCKbzmad2w=zE*R*K7P$@To-@
zC;#?T{uL&NxTb#+0f$Y(kp})LAH(^1%pV9;G|BPLzAhvfVmX1`UAT`wNOUnjv-@=Z
z)62PpY-c~=Q7DiTjE+1H)^}!)6W{%i5dLE;&G;Y8jN*T7HG@s8&<5<E7~3s;nP*P$
zFUUO3cV|l{zjBEs{}YeR0)Hxm!Fv0z3i8Q_9^ndl*Um4#RX||I%^rak-UIx-JJkhk
zLU#*j8OI6SYG>hHm!c%#SMLLk>*VMl{s-Ux3q-6q!08?t$MMcZR^Z;F8GKXC*6?RZ
z74u!$&LN=V;45(7;SE1u)e3=HNj|QkpO3)yb#oen!^NX5n0MK2$QcQsu-(2%7~Hm4
zo?{?rwO}!S@Tmr{x<w@wT#=Ds0ux><@l7v(#JgHtlHYqmn!pAz7M{N9bpG4Ex%|>g
z=J2-UP33P8|HHTT@@;;Rcin<J_PznTmEn-Spmv`%zeHfWfaTNIe15lOz;Ot&^YPVr
z0tb>3INNu+a39bs5y<Hd<bB!BBzWldA^{u4Dt`ZY%>v)c(%4-ay1-_o=t~OvCr#%3
z_SF*1U%KrmZ*7OPp!b|s-mgy`1sCT`=X-y%L@<0;FRxb00fFwN$9dTgIS3xzd6#p0
ziV~k}ev_a>=|cgg<8#4bQKB-3|CfKUU?;zZprDtN;Pch3{C<-%`O8yh3bdL?aO(wC
z30l925=?qn&ad1!MNoIPqTo+S75>$Uwt|@(@A0m<*UNu5bEd$bo##0}1}F(mRQbur
zEblC6`gXhE;i<O;CN;Zomu_O<dmtyp9<{fTzdOy0|10BHj`qWKeEuG_EVJ@w@%@mA
z7O;_<2M+6%Q`q<go-pz*@AnYwYW>Wm_h>TTgVzo`>q<}ZZ~1wVuk6`&!TE9Lc~#5;
zxaZ_F3R*tn;5hN)HrOv)t{!FE%3Q=-`sp5+fA<j&-%XcaU~@kwE#PtWSSIMzeOJJW
zRgO3P0gIr`%LKjxI}85ogD1h|C(oqAyz(Co@ov=I%oQKRBVe-_vbGfzPa2ml@NY5g
z<gfU=UU0Xc9j~hRCjKoe<_i4b3KLv>?zn&=4=?|2Wm*39AGY((JJi7UW#tyW_O|(a
z>wcT_{C;tT=fXKLa30>(u~@)g|A63jzeC)sPHq>dG~X(4W^1Hih4MVXY!6}nx34;R
z*A&kXcvlJ;Cvg&%;a3&82F^$EhA#xI=I`U@(`^+{-mAb@@^gY<m(D5vcd0Y^PxLVJ
z|NB0j?_S<1uzx{rmdIEJuFEsk6!`8}$FSHsSg;1PALKvpWy;sh$<1eII+OF&-YT%!
z*JRE4E$trhB~P0O4vQj_8G;4H+jyUDKft%iD@4$5wjo$Nf`L;=LcR-ZZbAen_n-HW
zJ;pt91w0#Gg^8uJ^6*S-mg2r*Tm;Tb(s!1~gWSUVBaCPNuEmNVwsn>@m<F-Ajb?Ke
zI4|T`!g60OS|c57E=U}NXRPn!e71+3vy6YCg1_1|c%G4JcjKwP203S~!DBxc$7D%`
zeKFqLTE}!)HBFzf3q+N2$8755S}7O^b_2+)jeLS|_5!_Nh3%Iwa5Fk_gZ<=lD1(zP
zg+Wn7_!y^qPcFA<!E{dBu!G<^q$nE)Za<|g`M?FYxftx9g57sE39?24WG)CF61>f|
z>Wdinq0AVrTcMWR{k4f;x5l-EgY8sb{y~PFbvanCX}!3@n{8fP{u497`~~iD+`cI)
z;P7xfo5FcGL5b^rSPo}m%pC4eamX4?5eZiwCL;!UmmgQa=7H?wvtG$%{J4W>m0Tmw
znKo6hKe$a<xUV`V!qrT-kOqtUY^>&ReLf8?4$^-mo(0?%O3ZHMDmkbHcFWzRmlVWP
z=Wy%j3-EY1xpG%^Fmb(q{#f2lM}XDiPZGE-IZaB0)90|3{A}?_+@IG#_UJg(UE>aI
z-p93AVkP%Y(=XiDtYf+In)}!eEVkly%{Jjs%F6}217z>aZ#i85BqoE+y3GAsZvIVn
z1(5iIbLTmBZK`EkJuMd;b|05MlRx6Z$+M4d33%P9|Kw7z85(BZoDSVL*{`tg;jwBL
z;fiGS<2uSK2@bzI;_tXGG=Jpod$*cXpJfy4lJ0dJtKYYP&1=6P&-Fq41=yb;`#_lY
zd<&nEhXD81dy6@ruE>-x^}Nfq)~J+I=-eN+)3z31eSaQzvWDl(;`S4|4Axtv-z66*
z^hcgc9&%>GW{c_Ia5d(A$I1LFRpG87<cx^v4LtJ4cX)!`ymCe;?-PTy^4g~2>_s-t
z91QV7JTLe^fXxTFV}YwKS7!_#_tQ|wIR;NQ_;UwX%YgN^_1bb@QCkKM1CZL}39rE6
z6)W|Q!_6;?t6OFx2lMVMw#|t;+zp;dGJiXQ<U?xqf%hQ&wmbuFr-Ss}{q=`?*>4rO
zj1LmrQ_pPY4k(0}-+JyIc#IIF288D?ZsR_z+0VH=OPovPnHtw=opoUUHBPtWNijaj
zefVKN_xy{HwF^_!B)KFx8^Hd!5ber+{oZvsKJipeiIsD}dfpzb;!=NnLt%+a3%7c0
zsGOh23@)v*CjR5MH}I_evz6yk$479O3h{oD?+G|4m;98U<K80%zUrkD6lxVVaKF%s
z;tA~QVxPVJF1N{XUU~gTY@Ah2UEpv5xmC<V609G@{_Z)8i|5-91?3zoZuc@dt|tk1
z<#xYx<~sWGF}KiLMsWIQP-O?xg8oflH&48?j$2$g1YACW%z)uTnK9fA?BVi1(<X!c
z0TNGnJORuWaAp?#vb&h;;9^KSY`5S>4yA{JJiAYP=ZbnL%%NGC%l-E6X0C%fPI8AP
zo&dY=M(ZN3fa9uMd%R|_&ozLU12PkY|7me^Kg?%QyfHPNV{y3**M;8SoXiJZIUnZm
zDg6sI=dQf4gge}0Cr{{8HgKL<t-25Fj^;~Oz<zke`k8y?`7_-6_B`QQ(l5a+Hpz)Y
zFTzCO?!*u5Z}}Nze=YdPVQ{XG=Y^Y!q+NQb%rv1MuIZb9aAh4~1g9f)6%)?QXJmO-
zsqk<It~Taj($V7y{2s@>r|~X#(SsSBQ%@Y@<hcKflT}~|_s)Y;xDPh=@Fb^{b1%Jp
z8ys%GJuh+mIRRNG?`pD7Zue9@=5Dq)9%joIT=Lhr!T$GOavbdD2$xNqf1O3a?q1|x
z2$uWkZ6@i)IGgLkX%Vn{xL=*;-XVUA+iCqnu>84o4>>O%ILPG?d5KM7mL^XU*GX>k
zn;*GTr+(tl-qHy+*H26sZ07a|8`)|0lfn8xVSCigf`g&j46Noj(;qIk)?DsKK`PvL
zdMdaYwr%CPztn=;JcR??Rsg9N-2^!&Av}1B{H+Z>+%Lr-`*A^XA1*3$&F`Bi*YoEL
zPj}Wejvsej#pksfa6bF2EdT7mbFMP}h1{AAo498lfQ0L5m#thsKJEhhN$qbJ_nDW`
zU^mt>u_&1QgY2PKzxJDJ!uHehk(ZRYmgdgndc}Scte(qk0at~^E^u09*7oK)T;m1S
zt9t#9{O!U9-t-mCqRG<)IZNeOxEEREa7LtYa31@3iu>ikx6IcI0>J(Sxw%E98=Phy
zzF7;;;~y5Y@|=sv;4zJu2UhoE1~1rb|FAs9gMrRG|8$eVd^g9tT<>|hS>GHm=80VH
z2UZI*GbsqNPYA@ed{xA^*K#koT{FGdgZF6bV($NnUA%cS3i-l>uY>cc_k<=eee60n
zuWG|Ya9F?LgPa2jGDG6SLB5y^BK&KD)OpXwK+gF9iGy%Jiy`l!`cR&~;m>#`N?ZoF
zcR}KRZI|$EoEiu=YjWIdzB9Yd@pwG+;k~rci>I5Fng7vjA^uKo0sf=!PJrzO>2Yun
z0=pB$mRVHE^QDf7x6jcAEN{iz#jT{&&dsA8%)7blEZ;*vA>I?=hr#OOn@jmP|MByw
zq#WRzdRYQ&CttNKSPo<s2=`xR;H`P6$}M^QK6eSn2X3P!hrs4Ox4j2e7gW*46X<9Q
z)(cXfIp33~PR$jpCaP=}&zYN$G^lV@hcD~c3*IMr6}&yCukg61HG|b`mYW3D`|dg<
zAA!uc+xdYnTT+C_!b*kj$=ytF{jkzu0jJuAsbD=IH4^*P!TOfTO7I+vmjb&5Bo4yT
zB39tEvDWnr-|f|H@H~?eu#x9kn*dLR!$Q8MPi$Q549UDV#pbg7bNUW8*W+<LS8KNq
zuiR=wKKCCzVDsPol;P=N=-~OV^)PSx9SuIW@0{H29mlz5GVkLq(L2YL<f;I+3uLd-
zV}3q9amW}hh`(;lH?SG<E_3*zHZ9@pp1%srfAe)BSFW}=@BQ#;V0W`e$%5%^-G_K|
z4kfX9XXNt6ssH0ybvm1C{q<h3o{uq?dAI)h#HIb2jh7*ghsAG#Ht+g4NL*aG|D5~B
z-wLogki8&mxYdWZ@$__FSLb$Kr%jJ|`9mh~?W|nHBYIAobCPa9IL)W6)@T36{gy}k
zoIYHygPIkecjh&4cyyfo3eGP@^WFH3br$k;2PA>@fy_6(paoB}%Ze8AtlFW+HMQ7=
z``~|HKKrl#;N~rMcjDS^)Xw{SCZrFZc}0<j@$E0LJi8*|+$xZnpIBFe&G%>e%jM0=
z$@jn2g?EnB1g>Jf3p`KnPXoJU|G&-Puxcwv1G^EV?`V+@SnRUw3f}SuM|fw%OM%^H
zw!a-bKLSz%!ZnGC{Ocv1dAS$Q=B+fl%;%9F2X@DuC|<q`Q4YLuN%g#<HFLmbO>kMm
z<JSn;o7$Gc%_n<Tjj#3LHNNNXxWMx3tvbMVte>01cb4S@Pw|v%V6&F)Z{^!uag0w>
zFayj7*?V__Ew^rm8^4j`W6taqvV5obr}JvEG4an4y3D3E{}k9?FM~LEgg0*in+?+A
zy;UC0c74Fe$M5LOQ=oB;SNXaw-`(?td}*`g*=-#s@Gs4Qi~)i4fH2c<NWC0$@CDbU
z>d9cULE<oMP_M<eM^2g7;v1x$0}}t;m<qPb<c}uLs*jyKw|O3e-M&YIi)R5l4}W0e
zN}h}U>^z>U3;6zLa`16^W%6xlgS3rStzhBV($>JU&D5QHr`8*=IUqAZ_~a}>a2(C7
z{lUd_xRlrTO)l@NI7L3*yK%fLxi0g?wO-~q@8QnN)dpF=v*;cqAJpDn3N}ZNSDjDn
zb2snZzQx>2>}T+*JkI1>d9|C{YQq%X1FWrF@i#Z~CJ2gy-MahvT<%Zz<oOFt+`xR0
z{Vo6Oxl#|z=W)CCou?pRDvzw00x$b6U)~+_OL&swpK!*ToaRwKQqFU3sU*L^t~5U3
zceea~+{?Ig;<~}^e!?xld$JwUS6%i<l6%z(epX4}34FSSQ~B7oa)Isn5TgmEZ*ZD$
zCL}@H)%lLSVE3yzGja(EAK_)!^#RAjHLFQ{Z{NglZk0a>HlJ;a9bfQv0q$#zS{$eK
z*!dgy+4+C5HS&Hrah@x68zh}a>p{*p2Ki_5iS=N+K<tS9zj=AnW5MRk-x$li?{xvs
zxBGi}CVZLC6O{LtXY&~qKIacjV0VDjx9(mJ*L(Wqb-uFOkUASA$H(Ep<sYWYV}2x%
zN7?5QcPRHaj{ECbxg%}+c+KuS<eBwx1=qQYN8x78U#7|P?5q?gTW13ECEIP>&*$)h
z<rZeL@Z_F=><hVnc@E#t?Vdcba|?OfdY<x~GVTVeIdN?<I4nT+PTszf&+C#OI3M**
z_{_I*iXgv7I4?i9<U3yZz!yBLrDyPP+&s&X&0fZ9p0SDf=%Grm86dr9P6UF@nVrW0
zK35nd_Mw%VD|EqH-i2QwW1)}lLiU~~$o8@{nKSXZ`T2qMblnR9o5OSdEAREpVtBYX
z%5Da`1!UH*UN?bhZYRLw4I+Op3;cK~$iJSIg?D=KY{8N=Nd7p_6a`k#y5COVom>cb
ztl`Uh$e1t43{4eo0e!VB!K-OL{GFAM@p6zj2-hc`5ZLi<FW;OF4}nl;Lw@#sHoTX&
zi10C(aPzk<x(;^Vgg2ag*SCG*VPh)f2c4f8HZ7EI)4Nv!M+`goRF8Lo?E{&$an*OQ
zdJuc3;y!_-pt(HemOcDVYU_Bcx0eVUzWs&wW%C4{h2d^|w^jBDoLm|PR{v{3FyAvD
zE&kOlJbdet=L!7SIhE(k>k6=+KxWP9oW~iSZ7R_GY(4)Z^%cAiRowVoQ+Ej5h!^0f
z7c&Eg&D+A|e6#&m^4xp<g-2%jWp+!;dHh%JXM^Pgc3JY&{e$eyv!C3<yEx)1zwY%2
z{*^v{{C}b|!S)2KVF!o*=}WixD!iwF>(V<t(R_P2eEE63H}kHx$pVWB8LZ>9TRjtO
zF34@OuTKK&Uo0fYzg?ykF5bG7le1s?F1Q}4QPmUN9;*lTt7xP$e_y7v;2Mv6e3^dP
z0>2in;E>TY;(yoC$s_QM8?1klsgA&u?|b-SOD79x2fXLm^5Oz-$Lk2eACI>4$Ijiw
zm*BCA&oSyWzdCa+?;h8s+=&efc&|JS=1Bcf$KzVPlK0#69DbXKtzfg*-B$2?IuAJ~
znt6J^fXY)*-dTK*vSPa~KRDgr`NIu%+jVzH+i}yQ+X8oDpYsZQP8a0c_M1Oop#xX(
z6+^H;Yd>r9ZvMH3kE3NK&yD{qd>t0SJZ~a9!TG<o#+5rcct4m83Y(7I7x>t%Ci7p_
zWaZsIQ<^vP+eH4$(mMrIHysf$>RZkG@@+EMjWdIo_{y6m@i&^E=D)drC9mPNHl7s)
zi})_*`Ge){8p^?GY|`=7d=m}t@ugVb0Jo(sZdu4F9|RfC=lr6;_F1?~fa}*W{<F=U
z{Ev)z1UEJof#Ve9mYegmx#sEI;7R9e0h_hi_6<M(x2=37$AlRleyQeguVR9$z3l^8
zw{=03QDEZhhkUzN<?zSMa^-J3^-O@HZ35qii%mQqr@!Fu&sxT-S+EKmE;E#u@fsig
z#UE8_%s#P6KwwdwA#eEdm0-KJZQRaR%d5ueIEfV;Ho5*ieA;h{_^gzc3!HAT;QLX0
z6ztE=S=RhN3s>-0GJoc?Sam{>KTeyQukW%zSW_EVU&G=!-b%jdd=AI$`OZA!1>1W_
z{4rSXbmKj|JBwQRvp2Z#g-IInyBV?Z#c5Xw%n(=w*88i<iZ_^jGM8-1GCs`{NBQGk
z|Kpdtp~c@Hx<#O6!4!cX|4jrY6j`#b_F)s~yQBh653P&=e3hb*IT%nFmTo!3@AoW(
zyQxB-|G=ALya|$({LK@^1+13*;xm344R*)xs%2pN*jd(s-Io*=%dI-?7uY<I88EzV
z-93TTPdD&gSOQsJ1`_8JU==v4wS?!&mJPg$7h?E|6+iQTSlr5W>fU}nt)xOeqdyir
zUt@#$H4`529?;F^pP=C)xPTjSesJ4s4#CASKln`a+XU(qEcnXS{f3vvuFjD4qY-)`
zTwhz?@xNa>iO2fJYyoctFM*lz34CWaI|;Ni1@b6eIstZPzW6zTRWpxrWN(3-$I|62
z#ka%hoPeb47Vhjve1glO_V5P(G!^tIzsqyMyH8-dy#%kLp%OT*&0Jl1Rj;TEGN0-c
z*w0b|4vX{21^j(S90a@j-v}Hm5fuC#f0j@1!+kz!)k=ZC+O3@3FAW4+Cd&%y`84r;
z{}UkiyJ5A!yS<C}nlt|i@IGI~B|KS)e?_O4K=ZcOY}3-e3T&|B<~uZpN$^;1rJyf=
ztH8Xjbgmr5-8`mlo@_<Tdi-Zsz61Lsf#WPVP0!xU&R5-hh5yE+Byb#A6*2J56VK#S
z&SMbt;E~~sC|b(9<@gFN_ooK@|60BItmBIXdv34cvEHJ^_3{v8?paIuF4!-icnaA1
zis%0ZO)&pYku~q_M+d;>T554|`#4kxKH;1y&@L^+Q*-5sfZCA;-souze7SXB!DFEZ
z{)_P{v6t~4YdFVo^LrKlmd=OZG7@C2cwr3xN;Y$T_YduY8cWo8U(E~UPv4&_U?T8S
zKqjVBz|Hb6-=)pUe2aFA@tR4^;eA&t&S%oo&%<8Q#(giGm%Fbt2khVL`#S`r_4)-X
z7q)SSZVDARUy>=1`hi(cyU9r~>;WU6#ugvmWwJp6Qg0z^Ri~Yl=hN(I1IHgj{3Jp1
z_w9V8aft$3K05Qhs*Dxv(l_Am?<wTJGO3ntcCI0BVmYK-0=YSaw+~zv-TZP593CLC
zkL+#y*Uz!?KJb?XyNBT$<cyroXhuHygZ;b@q#);>R=<iAtbNJB%YCJu_qqwQVE-&`
za9CVEvIgQ71_p+*3C053KJNmDFRR&FfeBx4^X-3N!7I($DR}l9q`kB6ZYsZF<N{9l
ztJVVV^_;<aS^h%KYX`aKM-GQT)0ZH@^{LkU?<9M`W`V>(SSWde!1Jq{`TE1n1q^zN
z_)SZac~xpm`A*$q=buw^2b_+C&7Js~x|DhQ7V7bD(hB6C_Og#}qslFTDTlZ7NtEV-
z?E{&$qv#-5J&2vUV7b78<aa!4o>uWMVPD0=wk=d(UmF`={;~-?*QNUT7;kSBV0h6E
zR<ESk%vTx1#xE3V!`B@?K_J8Z36HvEEdSLX`QSSdRCb=>)KZfXh&AfrH)p)Z+jFmu
zk7Lbpf!T+eSo2mIf&FvkObp*ozk579ib}lBdPg~rP443_-|PdH`|Y@r$4L=VPDngF
z&g;799KTMDFaL%`8vMIbA#;3@M=il|)iY@mpZ-S3Syz*0Oy!gR@6NyMFcaUVBarhU
zm7i;H{k(^$=O3JgtSy%Ot<7I}st{}+$p0#N#@r?gAmfNq8&w2@?&^a5`bL10ztT`s
z@QLyazR!xW0@6N<*)J5@@;}+IkcX`mQvauL2@0rtY~fR8tr1|=m*aJc5#{SnGZQr0
zw4eWqz*;`14b%A~IM(u?-tWa*Ca=b$kk-$eW2?vcHR%XXT-!_Dj<03>+|iIWO0?=x
z9+g{H!G5!JX%{G+6v;cMAQH?sp7x2iG|P$KbG8H6Z5R1&fW^y~9u>I$MS<`CBsW0^
zlb8IyobR|^^QnW&^C?D6ytSMyeEVCk^K^bU<(qwaD$j4-z2LOJEhCxx=}$;I3KRxP
zV*B{!JZ<IwFYdydSCYt^x+;eM#+#J__axT~tY2Qv+t_6f4lka~pLr8k_wk3mIKuxf
z{yFdYc_(=Kc#rXIxnKd7@9_)-r!k8cPxyl4&+uheL(U?NUw@SI)Q$yUxn`?rtaX2?
z1$1jy@b8mu;(vMghrnc&&0zn4+#+Cjm(%F05btEMG_d;n)pz(4IPUUoKNQBfNz8#`
ze?H`Va**1<W)ARJ51Yvy{@R!C_@aM@@UNPz%+L1iw1Cjm5I*y;Rvyn+&-hPXTFGmY
zJs%t{PFzoTA1(dOf9kk5XZIvlf$BBAyl(5KfbG)EUCS4kT+10P4LSS%Z~H#JBCQ_2
zIZpio{Gp9}T78?q{(Nvuncw?HGymH+Z2a>a*9-n~UC8}2ai74B{ae7{J8{cI-o3YH
z@cmlR$oKE_LvWl4)<e!``6tZ4cj{y^|6GF*zC||$_&qGF`A%>I2~7VEX^(LvtMJ~s
zmBLlTc%QHT+)92~4i15bahm+kBDM%jE+`RTFA@=0CaJ;Y%Xg3ebaXw~>@>CpJ{8-E
zV7Gz7W#^9d{NH{l@tApO@aL~t%NrZn!ryQ9pMOcBHveI1Td=y8y31g<zRg<<4ws%+
zm$?^DUJW)6WCjegA3rN#uK0-W^Z^;LK9IQIq;LG|`<QuIUxxC&_6y=ulz7K4`sygx
z72!L4ty;_Z@=U6E^!6I@ualMH`@1WPUt3pKFpdkdN93*hQ-Mj$clZ*m0|bf<a`=wQ
zK<2bS=B~REBe3IR6dUKwWn5nledM3`WD8H-;>iLlV$20rv8>?J;nWZ)%AL&9ocRyz
zPQFh&1vIt(bL@zM?(I@M&0BSMr@(S<V;;x1Uj@1^GV-xVN(t_|Db6cbktXoN)QeZ^
zBr`a!zghV6vIj{C{+M4O5Wow$@4<DlEB}NT1;KSJ=LHm5ehTz3F6GPVzQ_0CpRd50
zBg;4o1*8S9$#4k<%-Y0v#nntu<MuLv3+;>eEH$19q-yqZhrbo!7dUPvut+n2{SNm9
zfe&Rud>xT51e`v{3SM4YE6~$BjcYC^A8+QDMz&5PA%0VZ3t)fLE{60E6Rkh<W-qzG
zztYYb9M+2NCVcLCN?c4EUkdPSP2fxpd(JyI`ZCwleLDP-%j@{AuZ$9OaQw!jDC);`
zx=cv$==LeBrx!d1$15m2&jj)E3MQ(9`J8itc<&c12Ad0-&)dWoE;vc8O<=cg7|+9t
z+XReUR`BNCGv&MJd<$Gp)=hEYJyPz&JBwYC)A~&d|7w27yaLGFXZzgvuUQN8ht{VF
z#&k;ao_yoX9~BuWP$2Y5U{h9s0B^-%KHG2peBDcoc#X7A^A_Hf;VaqH!ZRVPf%}qp
zBKOuwGr;*>Un^eVuxSP8{2%W1Vg3SXR$&4g7W@?GnQtukV1o!Bn~gGW_D(Z_O{uZq
ze)ybE$$YBp)!_JBY}hLJk!c^_BnN!~*QPe!DMGe_^Z8u)PxD#x^ByhXQ<bmdJv-MI
z>|c<ZKZQWfQukl*4;&sKvF3MC{CoyZyy4-H{d1;&`M_p(x9IR42sq1orT8Sco<I1=
zM$q>0cb>3WYk9qRzX?=y7lOt4A{Pk$a8dxbm#k&Hxy4I)!0w#-N`U9<%=cXN{nNO)
zMZ&qG^C0IPZ(5!Q4%>O*h_N(~J7Bm&w2$j@od*xwggs#MVd5ZK*lsD8(CmY3ecL~B
zWjY1I{SdP!nXADUawnL?12yibDre4Vldpp9Df5!y{@s+#y+VE`*j*rVKv?`g;;eBH
zzdG8N`$6+o6g91_lR4`SJq3&D`o93X$0Rh3TgYe{XZ*j3tcDz#U^|%Fo4|4)b3l06
zzAn!1Kb+W?noVLey!)T)-aI?7yv+XdVD;HMoVZPHNrTI6kop5tl{wby&I0poe+O}&
zoD&DO^UQ<AoY9P&oF9K3<&@D0=B!HA2b;shwh1hMEy5Da2bp0nd5C+Z>k4ijNj07p
zXI-%T-=`b77D+?qkU?ttYoEZ?FG!pZXM^Nmn0Y(od|VjcnJ0`pVOcHrm#B5zTc&Vv
zavAD?&4$T?=;}pZIAYx*x!s~6Vfr$Zm%C%zWv(Xc^IYaf?{Hh+tL3noCc>V+05W%c
z^<6F843@?`6nAvYgUs_KMs#s$BrM`;aq9=G4V-Aloiz0bSGop2n4iYO29JM`8}|1I
z^XwE~%JEAkk!y`vFIdgHH~L)llWuVB{ICpc$CQcl*_JiT0LvwdK4IO~wg(&^ApIcB
zA-{utPHPxP<(f85u`Ca+WBCs`{T4ca^==J-+&chL_j4K}H?Iw3Ee(jj-sTARjvJ}q
z_?x)wDA-IZ(?#4lZH65G8~1|8VnFJ${z2xKKx|~ZVYMnxPSa<wyFlVCy24yLXPL5X
z-1-@8_m#IZxUR2^0-O7ieLt86sRLo5Z|Yz*AhtyAORknY$eBi=k`Y|(S&zVWFWJcf
z*0Ur85_TZ9SLI%U<+vBm=4$Kd<=&Xdh!UP4H6YBr<1Tlc*cy)ff)tLMCq20tdm(A4
z-0(5Cxm6q2Q6p~d#V?k@-Ol~M0L<2}Zs&Sqxsq$*8cXh{846(gSBpT_2KD(^a(~z*
z#vN5P1MG&Xd((L4>bLL|KhFi1BOrSn{_wGF*##MU{K==lT^?n`-L&r$*qko^JzzUt
z#2@9<SLTPid$R?kuL9DaKkF%HVx$PyRDC&ahC)v6x0}kr;dE%43(un6I$$<P9SDb%
zYyk6lw<E@PLE<o+*0Gs$VcvG`*Iy&R?gfcE%Cdp=fIw096;4O%IdD0J=Zrkoewtu6
zwjJ8TZSW$Doug|P*zF+owtv@iK4gQmw=G&_;pTwkG-7vv)%~+n<yhOihIOJ=BgX+T
zHtr&mRBmks8J_5w!rXI(-*C%+fvn$QagPAoGwH4nI2=4@C~?)#_u*b}nvYxj;BwB@
zL7`yt)eK|7?(^`9<6N;0vd+ov-YM<_)493hj|GF}K=y<1l$QBC3}S~kFZXZeUj5mH
zd#$Gk_t)}Z?nB@Ha`<`+fZaE_VjuS$dkya3oeeyH%h<qf0O_rLxsvOB>^88SAU+6}
z>uT`0t~7<K(LV*by9*>I=V%O8yKW_9-zZ2djt3DY4!Kvk*UasN>s=hXoXg|A0$9(s
zeqByRc~+kL*S2ykT9C{!Rl*yrZdo^C&h1z*;?ACm!^T`D)vaLnKUdG?DqDJuOId6Q
z=gqpaoF~|?bKg!}#w8^P8NUL#qs;UKJX};aXK-h4h1{P4k^|xSyKb@Ph3N9UD_#Uw
z_qe8kW4C5H*ZKfG?)Ev?!R`U6)!8D?k?{BwxQ}RkO^xeGxCJ<^C#PC-H+e$l(63xG
z;QnR=S=03>*nw+9ZzH%&X}@Czb~DHv(~Wbu@3XxJ`=9-tB)5GO2k-NZvD|YOaBv;2
zX@%RzrE(Ok=3+nO9srQu`{#J!=79JhoWJ@Z*F2r6+={wq!E!mz)!=3<={4o~_!Ls-
zXtrd5#rI56<&k&EX9np7VS~e=;B*IKuY4@Ut5!1EY;NuW?z!@}g>NiBY+NC8m3znf
z)jUfYZz#GoZs2cBT48zpx}`jy)eG@x|9-2hZIax(eXT8TR4>vDka^GN_5MDO2csN6
ztNR9vmlr1pn`JhG_b>1nx$(BCJ`}xi%2Mph^tqPZoj-XhbQqMZPX3YKBh{{a@4!c~
zV?t9|r|>T2{wcA*LTuk(ZgHh;+y@R_<6gklEFvHs!!!FZGmlK#esKHdUcDK&-b!Pz
z+n2pyvShx`$>TI{65n%kUcDFl9fdNs?Y2DC6TrFpZ4OU>Oupj8o6ogYWX17*5&WqW
zqBzmwiy#B%?}=M@TD4{ih0JXcy{=^lZqI?j;eP)z?%j3Md8Z#z<L-2`<h}26jYnUy
zm7A$Z7kth~)unn1rIRjP438POt$lJWyG$Q*Pma>()?2R4{XOVBcmCo17Kc@t_|~&z
zbIs|L<4N~j$9*ENj&rG#goyjIEOx(qUY@6On=Dr{F626Pwn6dWx>nK73-&zH+cS9l
zZXf3{eNxN)`lhas=+=E4N9JFV)mqalI@j$spU?w)uzy>vzj5#HJP&4by^rAiD1BYv
z%=%!?vRZAybq^YN3vNjAS*MG!w|>8Asde<LSfr}5Sn(M(-an16I9J!OSl(02;||)t
zM^2((z0f6>r2-3ozqR=GXeZADru|$E{>}n3KCj~5y@F9%<c|z@MXnByi<q&X@4SnG
zrD5&duWxhnhZdaX);P9-PhNsULFLVHUiXO2T7e7LIPTw=V|i&|tVK{nI<K3NKCk=p
zXPle<?d5*4xR1BdbeZK9W?A0wbNSp)8u#!WykEi7RS+Un89rNa_OWTA=|6Aq=}g%n
zn^aQHwME_E5)=<?vz80&O<pHwTI$GM5Y)yW&|=2#ZwHwZzcyKov#~(~oJT-v&;NeR
z?ZDBcdg~xR|IFe!Txb7&wG`g9nWuX1Y@Q^BIwk9iuLQzwbMWtf*&{3?%cuU!`KogM
z)vv1FCf-^F9&VNnm8-cy`~O;B>dP49^~lRk5aCh!f5)P|Mv}A1Z-Vf*j{;WG7iD>a
zW-aE|&pN8xz2cGi(f=`$Uq0U9TFrlvr)|S`aQ*_hCE>?z&c6qIrDU|P@o*fG5nN{F
zF0}FFWXTg{DST0r+_=4rg~5BHKx$tu;Is@oXe|6E@xP#YxV@Ci$_YFh_nzc5sQ$oR
zm2%eHN?6x2D9OU&=C&e^e=kKWKzbsc?h>t%&QSk2VWLI(>bX3|%7?gLiK+6g%Kyi$
z71F}nY5kBx`3oC3Ju90f@|Z2>RlUlP!F{OnAot?qth}K*zTAgCDT@lTtq`ql@D`pU
z&chqVC&?X~!z$GNGl^S9<Duwf<4kUGjg#g%?M6I_r#^|fO`OKL>bJkluNSSBw)^gJ
z^Dkd+;ge;_bIi_2=djIso`e-V^3Tps<CEG|BN)%M(BjqR<CaO6*?2Z^t83X!?6X{&
zpvIfP-KzZQ{~RI1b*(&N56+rBtWW2XjQ9gi#~-WyaC1G5k^cWMQphc*kf+qifp`DD
zHcLy}I-be(r@5D{&9LmP<K?=v^A=D34M9tr1y^`248B-|thg!G!#N*phS1_!+^VZ&
zH4~TXaMvw3#b&y41s|`*d2W@~<vbZK%Pm(5xNx72s1^8l`H<3qc5lI!x?M^RcQ>0A
zcs9y%T9hdGDop3t6!}?9=k+9>jw2tr{vCZJ^>vFS&(W`@JSUGS^X5MK29D#4EbKgv
zYftc*m|vF;Ik|}A)46o;{xneB9ALfAqpP>Wa#2DHhr~N3-c1?2T(b^e;wxNYWOYqw
z8jrERm|T~ZHJ@cl8+S;LI;&gb5wP18{UP)3%9%#W&zP6-Otjs{ozwb-+xXWUuA>`m
zESF9@#ZeIDX}(UzncXF1J@1hnLKfyS%e6{&<?;Id;N*R^e3Bu^e($D7>}D^&@f`hQ
zBD^z<i>GLhuuQhlSsnv^H(61YI~*&vcJml~-oc%mo57>}T~zpbxTwUVsVqElEQ>7H
z#%vODynL7M%G1mIhyHhSta_xVb&vOou=vtDJgY6Q@H}dnCGRWA%(9IC1~?qJ^j7n$
z2}*PO?QIq=__&UJ&MP62O9`dsHmav(@2oipKL2OtELI*Rsg>NpmUpBR^F6rU%(CbH
zeqM-Y-5E}?+yjwZ_SNfn3l8k!E;%U8IrZ=i*{09_+RA_a@`dkTCUmTl&%!HPgMH<b
zdEm8)b60C{&7H!+?LXlYt7f1Z*W`+Q!o4O!JVxttxxSe<$hxRZ;<mTr;5EAKBhb{O
z!ymr4ku&!HT=wr)M=k#=oAF*er^D4=cSN}5H80PJ<oQZpc@orx7TEHC7AWN2+<MR4
zB<qdB^<OV6t=~!sD$jJ0ll5ceu{TS$d>l4~<!|XOo`qujEG8ZMqU<Ez$ZdUFjeVz4
zyU3En^Ooi3KY-7;yIyF_`#+t-@~^;Z?V?+He3sqXR^baL@E$t6oV$#P)oS+Qe|!>#
zAGqJmROOp=+=*M;;frPL&vIU^OZ7TkTQ_nAos-~MbzD*@RC$lk&GraEv6~k)BDNWd
zf#MZ}XJ!2cw^PL3EVM3pHVZ^ewHMlYSWJP{>@9a_+js6qd<XdMt9}*Aj4b5-rv93D
zy<0a=^8!elSXQWm=lHEVT>m2ydFmoR^IUs$R?;}TQ26XRJHE0qRq#5+ua^yY9kw=u
z%Z+I(o^qGIJ;SMX+L1@{odWNtBv){p&)%~H+|F1~)WBsOZ3HiWLGA<LMgKZ^_o$ra
z?K{fF^X?cY%37pp|L1UTDfi}n`MQP6bLVXExDiOt3bqv9O^J~EeMA*Ycz>>$!Obox
z$kWTJ%2lFvoOe#)72cKMmw9(Cl!e;|+EWSA3&Q9AyK)BA?&Yql3I@AztM`324j)6#
z95x}I?jMtRC+ltHOxK32xx4p8op;{Pb39vjP2xS&d<|^gi(g;Bc@Sh42)C!7<zBb{
zBgb8f29C3(OWDg)CBWvUnJ)u}%`E{1?&723a6QcjcX7t9It30FI~y}@*qx?kOdol~
z-p}KiS}4s^-MNvw<dPxS9B*HDu-?tSqG0<$b~bHY%bOuMpHpx4ZJzxrqQULP>%TSF
zv}_?~1%cF@_nQUQubU#y$)0%!Yz9aiglBXFfY*eIs^##OG;xE|2}oS(P%W4BK{4(P
zu@StTTr8Zsv$eqbSS=*McIzxGWX}kk$GzTxg*VtX3@qnzyn<`R(>hM3AJcd?-oM8)
z$Iy!N#PM#<9g1;WhXn7lK4N4A`xRuqv%@K##_0`k`(#@odv8qVCGehDoylXmE(UC-
zUFk;7+DCV|%NL}8!;p2$8L+t6*(#1T_ZZn(GkJJ4bep(?S|d2(@0x+-UY%Rd^ZoH8
z&K++c>t|Md<K!{D2I(&?n}3I^CJ53V0=WZ(MICK;uJpI_?E4hVV{5R0XS$gk@3~F-
zTnn8o*!cJ$Yl@bK>9SO*MRM=&hqT#2`i@01^Dec3jAwB+C4>Fe(<jb%&O3{nZ>b8H
z4^kJk<ty0jAa>>wW6tEqjGS_9T&&-8zwj*Hxe@GUkldz_`)oSp?mTaH%7eq$cFR1j
z@c(nc^1+E$z;O#w2f{q&31IUT56<S8ws#HBB{O558<A}6npd5;x3=s9%l)yf1KVp6
z0=WkXr0<U1Ik0)ZxFdKbPMXeRWQe$f10)Z_bGhE}7B^qxer{F5Q<pZ6H{s?la2m4z
za*T)fvN+GLRDGUR3Z7u|H#IYI-dev6Ebsa18IMTNZypoo`MeKQAoKcS8iC+EyO>j-
z*Kk%0x2*hnup4ewX!EkE_wa_hnS%KsyJt`O&bjO#WR0%#n`1nFCmnb^S8n7jOOE9*
z`oRMZm*8SEZdbuJu-za%kB)DK>%Xw@EKlnaZEmxQSv*SOUwAar6nFz<-gB(b3FUj=
zbr`M=q_4+(H#q(2RIlM!Zp;P_Gmtn8U$XqkQ{T(UbM}oHTu#!^8f>pu(h|=7@w>RB
zw0ps3=2^X9{~mFQm;ZAnx3{hg*NH#!ybg0t^AuT`^FChu6`aSG&;G%AqUj@7UKA5o
z{Dlo*b6|F*E`*$2vu4pI&aB@TxxZK2@iduV=8?;q!TrQ}I`90*8Ql4IO?kX4T)|;^
z!f-9v-X(2?U~`ggKIb{LTac&uu>jX2zFMAMOJiQ`Q`~Hm&t>yWoc*2SUtbeX)4AW^
zI4V^;$u+s`6|a)|EifPChX2<WaDL2~!etPiz&%sq0@uIW$GG<!eB(ZT)r>nzV-g3)
zKQ^uduQqdJ=)L7l)llKB<YDIX-r2{g``QMaF8}p0@$`y9=Geq*>cHVOInJ2(!bfvn
zd&3|wKeO~0*uJjn1h(TdAoo{CS3<_0*B+k3A?nc1qr4h2C&ITni1%Cm3eHOb_rP&<
zKS7Z9kj+lcRhw=x<;*(6`zw4iuW*+qkHfdMoc{wLbA>{=kHKyM`6nr;792hxwvNIw
z9+AzgV7sSX{L6V_^-Fe%g>Sene=X#`uyQxI-21mYYTvZM;RsTnd@2R5_t3L>ys?~!
z^7d8kB=$QC-f{`sMR4AqF`sLDRSFAFTO_BWdJuQ~{bcSXl}|YJ{rcc$ed-G5`l)vc
z?5>QwM_eE0-UgfPP<n(*Gr|t6ZjoyUZ%wEsS6}ZBE*Iu`yf2Fa!Qxs6A^Yn<Zu;n%
z#{0031swM(wi9_xOpfqgQa{hzkhO-#oz0Ewg02kb%AHp1Di;&M=3iMS4>kj&SM?<1
zjt3AMhPArHId{&l<uQ8`2Uee#3t2}3lHXY}m*<}uWSl=v4bmU@v}-!gn=g?2XF+Nv
zUnvB;1;nmh{+)lu&kpc>!=r*+fz>{3e6AYbc#<Bv3ZA@i66}u8GN$~8w<K`RaupO<
zx)L(i(3lQ66A)y^?ZVUiI_o3_mHOZEJ*<Vyxq`$&xRSd<AkeFh@5Xc%f#5Jz{<;-v
zyl0ap@GiP|ov*bCvi`MW6Dwat=wY6E%fozc^HumC-1gwxVZ1<qp{|W@%WpMs-ydYw
zx9K@>ck|sW5Lm~vny0wNiofvb60SdK;sPc1s=T2Ip**VF0{CpDS_Sk1?ZE2AlP&lr
z?(OG0bN46j!(Y(?H5P(AnrRF8OwRRz$8)Z*Hgn`$xXIt~$(%p4^ce5UTONGuCu#)x
zvU=HldLZ{M9Z$c(yKT}Oo=^4$JXLoaxE6AU^8e2i1czI!n-7nzwj+2x()RB}UapIa
z_^bV7`T5)4@O@>6td&Wv`T&kA-l|x>t*;xnnCw$nL$3w%+1--o&)T+w*Kt}LIGsK7
z5#VxZg3P0V{G7i(18nX@BVm5a6dkaAAaT?1PVQa}$XUo1ia!N9g#Uo!itGAHK8ez|
z0#E+i@bOMo5y+i0nKkn(JO7tH5uTav?tsmzdvt=os91wf{i3$O`_kP!EW5pUch&w9
zi1W_iUo|s_Pk)_1Um#lvzu*Q=-gy_#b5C&5;r-+{o#W_vH=b@E7v6*^F8q0gUSPLq
zp5)<8-JS%tdz!tQKwG31@16g0V1Ce7Yu*Vfx%lIgAnX5bsX_AAtJS>%%BxCvBl)-l
ztu8O;U*W!l>$uwwa9o{K)8)0E$;&tG?i8ND6<NHU+d_E4rRIUdZr1b|?(X_^U^XZW
z!prjcj@!BOv%O*DO=wc!<(<mSpS7z-;81OqfXrSV-rg_<up5)#ui||W8_Lf;dn$j{
zvQIoQk8*fYR+;l<ma&87Yb68)*Ij?Zws_WJzHIg~zT_TAd-vep77oAniGtEDbJ)fD
zBH825TL~PhOXRl*cI9`?-Yg*34mooX<d%1{M7fql-r><$14*w>JXZ0qSh|Yu3-@YH
zjd(7$6@R_p?p|)H2_B16kzB~XVfrGz-k)OpVwbP-t&*K0@X_rKZ~r|Lu0Ii*_$QUf
z@_PKp1IO9A!>@Q`H4pMT-4No;d~}#U{Hhi2xsYJNvUg=1(?7L?*Z*BM+XXHswVN9G
zmRMir-LltP;48Nd--Wz-f%+4StQQi`@V)w_!oT9~RlWd~azQ!%Fz(XWP5~Vb$Q~6d
zEo0tx<>|aO4R(D07~}cof8b&Lz7KN7#f`)@yxf_h{F{?y`Fvzo@%f)+<J;-WBhdOB
zvX}4v=?gqT*SBz4znsU{a3Gcc8pk<)qx^q-`B|+3u@6)PrUdTe@3)!Ebv0u;|FboU
z;J99Sx{Oy+CtBd^n=*EfRhcY@Gn@Di-qz<~ob`lnf=3Z=<h3Av^=F&-jUC_fIo$oq
z?^1G{ZGUP4I2`LgLe6%)GH)Wc_;$oz9{yQP;CximKV3k<zmAVB`UyDvzP)DWsM@}b
zf1Obp&yflFJiZ(Xd>yxU@P7;E;>ya{%csm9%jf<piszQ#W4?;ETf9wmmi*2!zXaT`
zxPbLfN}4Tj=K3tYLsM7;VtCd0_N|Nt*KMG<m&q3q*j6siHp3)?D{R3Ley-o~Je*U4
z1j>#v3Y3Tx^Kqs<<NuNB&hs;P6WHIvd2IsSev8@Vzd`o%Ojw`9Yr@(ou=@Zj&pi3H
z0wFc~c)f%!3v80U!m}_`Tp<6c81Jf*%>vajFInuaFXY*p{z$+p#a>`j@OE%~b0~B3
zU$ed|@NnBCfyTCt0zAwb;5a(p!6A?^wUE=I;i16U6UPMhCYJJ97cvTZ*_8=|Z7t?|
z^L)8Lfax~wrzNlX9Cot`teBa=&Mh@Tz}MwI?~+-|1s2R#5Hwt4C$K0aiZev(Fpq4w
z8ta8=NBO>6L-zB`@rT@BG;hL9-jh7D`Kuy%!1?LdE+)R__(Cq_v#SM6-?MYR37*Hh
zLu@8jUg&E+?@6|Naf{^yjbqz+qCY?8oP77Xz=bPQ*&6R$0>>*TJi{Y)@vO{)oDo@f
zQjPb+Zpb=bIc72L)%(Q-jUTuQELbbTbCsh&V7WmcFPGALUY=(;;P$2T2_s$^Z$@68
z!|OO^?D6CObaFRX9AxesCLaE2xhMH1x+w~B=}+U)EEeMbeNs>$;`e%iU0d`7Vv`&B
z4loGw#qM9kef7&Eo~m!Jc~8xm%(LR%G%nYo`CP02Wr54JB?){2RsZz_?RGJ8zi^Qi
za9FP(z_w_=z@AEWK{tnMyc0ej;Bnr?Ca^g$4;(-0Q?>Y<GOWS%n9_4+!KvO2eBFyL
z@%vr0;>}BA6}%QH$$xy?cfS3LUHI&07V(;xNrA%=<mLsJAoCQT93ksoL1Krd3-ed~
z`oJ^krxx$|;JvIDmtO{}H{NiN_i*i8-j|6E;5J*D6}#Z%k~`dI-j(o9T(eVPM}-wQ
zEQ-8x1lO$+0;k`rmmCGsLW99>TNE%=;QIdG{F5d><eeANAh=-$WL-@_^=vh>;w{`d
zEJ*^2{v{ehsi|z-O0(5LZUN!fI<f-$Pe%!|^06pIm_g2%0f~dK^v)#$oOvhtDnGia
zIxt@4U(KG**AmFYZ}FyuKmLRTIBz9tZ{!Oq(Nhzhv7XQHbq@d2qks64>t6^w{G!f3
zqc<GvHjr7D?k0ls4~V_Dut4c)$2#8Sv!?J@d}HL@``%UH-=uJj6(64SG#ha8*MHb4
z(APRm8l?6cmm~l8uQiJ6j+gW4o|-QZr2UGwVErln9@B&1`v2Yn4=!sCJ^@yJYkr;W
zT6~o;T>J~Jtr2*#{ugI|a3nZvJ?=y)eNPPGP3d^f>v(X#T13NK{=_?$V7VD#`MjUk
zx+{U~68+E3cV*>!ez^nL{Lk00@!KsG2mA5FyLY_q^2gP$2=wti?^_H`>(Vp0_!%DL
z^Ox>k!T02~09fqjsuV7cd(B{TLFRGY-U!xzQYM`Lx^N9#JS)+Z_u=2Q;P`C)Xe?-I
z3R&y8t5raAQ>L)sR{wOq=WkmDZkjCQ_}KDQnfF@|@0TiNu>N%`83ew>GAgk@$`uG$
zF@aZa^KZToOF6;cx{?CTChPdVELy@h{mo_m`HT~J`D-|NW$qO6ZtHx{{YRFWPwCoj
zzD@my`CrFO0Gl=G(@N#%7uTqO+!_=<QQ*wGMZ6~3kbQAd7Mf~q2Nv+3wO$2|x5FKI
zV47jWF##Um_k5mt`huIDFbH_DpXZi7w_FWm2K${^yv)w7d>P)kygW5h{I#zS^UO+W
z2j@Gk%&pw&_aJ-OKx#vUg!mnKdiXiF^zr_^c!{@5M^!`7W`}^O#v*}RhunG3#zD#>
zhMBy4QNK>|XY)SgFJfNDSGD1b;Jn!L%7>Rg_IBJ9(HFct;U<UX2Y3D@RefqH);GA^
zHXLKwck7#4?ft2O=5b3owtm>l5q|W9>J{Erex|Rr{Hd=$3w*b8g!qSnfgx@|16R>8
zL*AUTvEcM7`2Q_`=ifN~SK<sjt-|xUWfl6t;RSMcfZRE7S`l9Oj6Yd4h2K&xmY?}6
z6Mx{E0|GaGGN}pAny7m6o3N6lmnNTe#ci<Ku=Z7a0h*cudzBiM4|;G2{GBPrr?h;c
zAjh-a9Lr`a@!$P9ookghWFKGYQ+@thGwt}9g&PIJf86Jrdu^jYdD0YCGh=0b<8|`<
zN2<K|H%ToP?Db#3BjkQiAov=jA1YcP%ICE0fzo!9uY3zHeC8{&&|s~a_ex+!5{Jr?
zw?%w@DNY(XY8`yFDfaw5nwR;eN%;!g<#_>4w;!|2l#NdrtLnVA<Zp5~#Q&VJT|GLk
zk^jb)X996N?gF3VSp?j|TotspujA+JhTPqk*{j1Zkhe(Sm2V=);}^Htwm;d;-_dqK
z_1e8;{@^3Rd=oQPsPNu@#{c^8JpS{FZfZ-OZ087bkp+jxFX!*ze6f`A5s&$#ci^#1
zP}t5$wgH!G{FiSEwAn?fE%_h@4)<H{-g0c+!6>kD7YE<2<mHOnm)z$&&nY3Gw)Z*r
z<rOOY3T<-yX(k%Hp`9lD5r1d$^+ZhOKXpq*kmY$l*dFEUX9SuZpYb*BWf!=8V=-U-
z7s$CFAUj=>v<0SK7t|1I{J>SaUqj%^Uol>uo67{Q%$%q3{fa985-wQ*=DmL0^CTf{
ziH?7J1YYu2aXKu14NgD(>XUgN?c61x_gs|MTJFAp!g*i5?)8nDHWE?1)2qZ(imID=
z>&34NG|KZR-u`x-H`!la&~bIQfPogjGRS`63>E$_N4W$GZyXXhfAxXDho1)gmrCXM
z{WvuRw$@MODwY!#yv6cgAj40SpYepQ;7;2s0zc2+=4%Vus`R}ppSSj-8Gq(23jvGw
zHk{A-^%ZnqEZ|#l_^p6hc)Z{di(Ubt$NJnYh6{KVkFl{g9+cs~ZhRN)Zk3<fV7hqg
zH-(H>0s?y9SAxgvA9yV1>#kGf(R}hlpw@c<*N3h^zKIokxutJ)@n1HX&gYVoD5%gq
zk$2@=UhXd+6$OJLqB+z$AbW$#zOGmB-KxO5O|TT4=X<9t<W+dw3ywpjE*YK$$Ls|q
z{8|Nc15WUGx%3IJ7K!ua&w0SNZ&o}w9lAXa<F%h$#XC``Rw>S54}bUP58yf&6i!V`
zTltS{za@9+g1ca|`!gPq*z@8FentYvmTgq`ip>`2?VZFI{O2g2Y?Xof(>(=z9jp8K
zmY=axR*~4q?f7dsk4D#hu$!J3>8ra<ju%{>*3TW=nJw`6lD@{$b?kyQ;Vy!Y6tuM7
zJl5fjzhEHn&Sn=le(r=G;k#(w4lWn!eooOyHB9Aa%Zrem%yW&mR9I7RCc`xTiy{X6
zZ#_2iIUMWf+uYOz_AkiIIu~`pb@`7&cfodp#J*0?;=d4bpI5+jJ#XAPS>~?&kTYgu
z#3lI;H&ya2`%nuG3x01C!3%%tc)Ntd`2JmC6MXP`A~-Bc4z3b>ckn1UpDYV$<#jKG
zob`P$VG>VI+g>h)hf{evWSV$?YpevTU$Or%xSr8Sl;!3shOA=)xdVoOSs&z8=#%FA
zx<r9T;a>*Wy)d~2b;>;4%)Z<laW&lY_S^uc7m%Fk>^feRbE05(`kO4{-8*kRcY2x_
zkCDPRuALnWe4pm<@i~4F<8#vF2D<^I=S4DN57nj1!CVhG<#^m@EdskuKRk`&LE%m=
z#~+qF$vrQ5ZOl!%FX<lvhgqXx2CthlBk$LqZM;Qo%faR)tVfKofXwzW66ayO<HluT
z=*-2nTZZcnUl3TG4)0~K8SzUjxi3CJtebdf^pI;w<xH^MUTfEKf3w^NR_`)RgIC(%
z70>_vES|eNXK=6bkp+uCZG+VLhYAsA`>g7D!^`tQn_D2ypZCtw*Whq#;AG;ISA>jr
zgWNT@5fa{+#x`7=Un1@-0Lc};uLjSxn0Tz?m1%^WO#~A6yPV1$vwAXDP{U&0_})z%
z0j!X7Vyad{?oJfD&d#yK{}1=9nPI$Nsvz!aV~^(+OI^lwhW#{88HX0{fiu@RjiUE+
zw)%H)v*rbJEO2T9rx%d<YIz@dEU$e8_p?C!oP&?S@wg~!GB4lOZ9G%w902p@l~3Y$
zD$B#;m9`Wdh7z~Uz;u@DEUxrDYuIOenaQ)}?n3T~T1s58lV^bS<X2winKvPl%ioI$
z%r8Cokvn-0q;2%}A`5r?9K<<9AiKN1`SIj`S;}K0x|XMP)=8eP8a=#PQYzf~%6aTt
zQ`UjQba|LAt6J14?uKSgu$drzw_XMEZi{;gj+?*BA?yF93H$J!HuL7rWNZWL1F5qJ
zhK$*Q*oAjraWypuasA9uW-t1g#A|a(4D26}oU?>4=ed?Vo_#+ifX!?@#>vev{VdqN
z1!a&r38W5$?S6NF&2QzJ!1-d$eV%Eoo;<G~wsMp)o#8&>0@-`8zkd<fY`6Nk;4}u(
zXJjG@7GvdE&2#J0JDy4JG~xLPBoD(}ow|I?yI*lT%Ej?)V0^-xr41P;+<jAy*RzR-
zho`BXr}!{rUh6{E9<GPic);%LnJUXWN&PdA+U(1`zj=Ow#X22(!FGJ-uHw~S#>`W&
z2r>q$!Ze4sefv$`lTXXQ>Ogis)Y{K^<)a(lPo)QJfu_fKW|U9o`ObQm_w&EsoIZ5{
z;Bbj9k>S=dxC!<LNY7dpb~t;|Ax_?R!gIOKOZ?=yy;q!f>$@o4t?LeOXg-|ISJnj?
zQv&G$;Zw^+z+uQ9xr~do1hFOrBnQIohxvH#G2i5#nGV@M1rpo$8M1#%^3f5l466|C
zKW&h6CCYkrxy+eF`6}<;;hy|Wn!Eg55%0=X%)HEO1-#o2LhiZ`sqNx=w|6dghtLMD
z+`1QVH-pTLuulNTQD~hq*RoJ19_v|-Ja6_(@J24T<oQ152XCfN0r!n{+&mX08o^=d
zDFQizq%40W*c{VDPTt+mX7h-+{Ny^qI-lpTP!8`;IZh5K%{4rEOSrgBKi$gXnFd);
z7hAlF)BcPMpO8Ib-sM{FR?el2|GC9Wd$?Z-&*bWUyPccouQku<-9N$W|D0-XalJEC
z<qCeS!e{osnRiEnE#DujMO?4b1i<dD^I_$Yx(g{=Pntu{L^a}@!+Tz?hgUonvbKz~
z8FF@9PC_KdrUJ<NWZ4kNSsb3r<v11op5loWF#^X)*pmgkuPbC&{tImfo4@6wKQCzg
z-^wF**yeSA=e;$Tm5;}<o5xD@J*Ry<<P5`U>#o7emw+NjnFI>kEY?Rn4;OZU(~e_O
z1((TVIj)t*qqrMAw{gqZ#Pi(TB*VLo4RRL*NWDxGWKIXfX1;xuH{yFe*bN~5rCMc9
z#S^_;ANN&qe|dP1%aqfH?Of9=j!mlKJip?Pao;ZWVxQJ>1{^0K_18}^adQjK2bXhG
zjZSi{Z-<<P@Gf{B_iT?@V0E*<bn|{LtL0jjvXA@0%&WWyua|+v&%ZH;+v_rMJFg@w
zWGs3^?IYeG3k|-BmpJ&0-#y{Ua%kgTocob0m0O1G;{HN#Uh3T%3pNAfhBJFL;Nb$|
zgYev@F3!w`%REXUv%%`O+>8RJE0Em7Il{bg0W-jMi)a^s&Djyh#Jf8QQa^#z%oT;)
zX#ry2?5XEpbyx$^-Z;)D;2U~|*E~j#J67eZzzvZOu>Jt8hkTdsin4vbwT6FP!Va+7
z!Z+4nIglAU-{$c5TkaLOx3`jSgFIxs4I~c2ZS@iYjjAGiduK1_mk&4N`O!6<JMJ7C
z&wV*pKKakFV0Vjf{p4cljpkZ%#D%Xk?kV5&_~l&19qt0Gy&^obmCl3h1DO?(jhOE|
z>m(wu=B*~T*Q^VCY1~C@EPUJe&$ry?dcn!Sc`|N0_xEZY0he`<`Pb{OfAM@gxPkYv
ziVMeoabbZ+@3^=`@*#U?K<2Joy@%}%V>AEf>g{~dA@g{TL_g)ds3$2PvHS~5f&k=P
z+C`q4ysHkp<Pi6o%K6vhKbzPce*VqQkaHB2BQ|rDt22W83HDQ_a?U*$%zwx25?>BS
z1)ty#Gq4{ycV>dqa6<Jd-hGcb!Q&3?HIm$yZ(ZY4d%?#OGYPUrKU!!O=Z)`Qz-EEm
zHff#$m~DFGFrSDD<UV_lxYq&+)@_d?!14Uocd@|rXh^za-BG|h(YIf~UHk#>uc}@A
z4<3Bw5T3e|&vu?M7w63>V6)C_&f<UZF^#v!YY+dy9A~Zrr<A$<11AYY9p~Y%4q)e-
zv!RyP!%>J|etROf#t%X6yEk9+uxo~MN5oFy%w}rguFbs0SAXa<*sKYLhq>1rnFIFQ
zjN?!E`@E)c-aW?*=HJrn<C(nCfv;Y1GuWMyJomuj2WJ}!#QLn^4ia4~AaK{0|JBp?
zoZmR6gTtxt^90W6H~M%_y<fw*k|Tk;>y!bz*e=Lg<&sVfj?#u-;4%gj1{+1Kc#VSJ
z@wL1&WEWVujx(XIhp+R3ia^pDNde}M^LW(j`oL~n|Gb|^S6-O^&P-!|j<*ZA{Ng@y
zc1q6VskyNXEYBRd3Y^CJc<Q*<pFYQXX-6$M&rM`K$>zz;4sN%D>^p1uiC@d}DPL^U
zTfR*(Ndo7ktHAyN$(ii8=G>5_#2&|R0c=*TR3QIji3+fvbZ6;sSuW*+yZdO-BQU$J
z$bo;kk1OA`+BQDPvL?Q5`ep(z;@<Im*>#gGVoyAOf7nUx58aUQBdZ5%xN2@g?pM+A
zE8^d}!H@e(=|8Yt4}~Z5g3tetn+gu+4`(-WTYGZx8ejRzzcpQ&d)6$-9ypd0W_+oO
zR`Eq77xHCviwgd{^^s%y7CnL1Es$}fQ~v+Bg#R4iiI9KA6(uJP&foJG?7((NF7)8`
z=*#B|`ya_0uT;S6P$S2qP&k#}GyE2KE~97dB<>}<|8N9YigB;Cf6HfR(9Lh|?8dh-
zTU%f{=XU-y#%O+<M@DQ_bM*K%KSI`UY*@z1Exi~r2LlR2r-QQm=8GnCIao^amHJQN
z>M>^EmwxQazb{ROr@0fdzjk^59k3tID?sKj6;HBryVqud-3l@ThTr|P5)j@nf!8an
z2OP&B@!Dkp{GO(!TpsmxTrLy0^Xlig^BYtfbDZX?<>S(L#J%B33D*mu_q=JH=G@}X
zKJk6go+F^j{u>;JDRtHYy@zf1Oy>0RpI8Ff8v!yCgbnLfgYB@8_Tylml*`{D6UBM`
z{4f5`>sIiekvYb#klDe1*)WUSdmUsS#u_dy0X5CntS?MD!0G4asl{9sf7AuO2+rs7
zI2|f5$?zxV#&uN!mWQ8kGHzMI&p2ZX$F%$ia6C-d>C59=&?#_j_Amak!qwn(^vz}_
zU**C!0pCnxfjF%g0S_lp-Zi(4`DA4$@$Xva%yBfJS72^#sX(LQKJI%f=LqEX3JF9h
zKI6Tq6ClugIhSMMqyoN~FPHHDN{D3H`%6P$;?(0jwf}<z-g=x8SUmF&|2Ix^_9Jgr
zaouB7V`lx4&A0D>H8@PBXVrkqv1?B!@-lO&@h`WBoK;cVF@@W9-#xCQue`wb|7?C5
z$@PYzg0rvaJ#VnhVa_|8M+M?E^ti*8_;Tb4LC#K);AH^UWuWkkxyH<?yXPjj-VKr0
z=X~bC3HJLo<^7zGjduuy3jN{lX;t8I=I0V%lHS4f&JZ$h4$>=U>dR^K@&I@Le=oMq
zzxVi7EP>pK1(N%dx`>an%AM~h-ys2}hxXisTYCAP@U7zS+mt0Bap^k0$b3n@lLxZ6
zA0$2DsO^yET6<|7FVFriZoZoz*j2CBa8(G3fXju0CKLFx8m<TwwH)PClsn3wZgh-4
zSFlDPJ$tRdZO%OIiDwJA4+XE}KXY{p*bnX8X55pnd<6TUW9>fyqs13_?glvU-%@<f
z$yvQj;3W4QzT3?`d?Af{c>RCH@T3Nx1E(jDn`2YLz;V1?EgYOyKw>+2^Z2H1Si&uA
z(g5!NsnkQxxbDjQ#T{0?otr5Ua%S$#l`91H{Q1b~A=AqJ!LUN$;~@>OdxAa-!OJx7
zHaXsciL$11SWUT4Clm;m?mTGdJ|~R(;zDD-M|af~4reaqf0#MLJd;gG_L=r-(f1Z>
zEmRpQxejItnr}VEpz$DM3EzhavAipHALsk>%EYY9{+p1{rcYqM-uchStIxes_}3FH
z5slF8=2JFV@%&i$M_y+Jo1FJXK1HTxfAIZ3E6Ss}Ga6IPIJ3*SUtN96vwQ*{_wqvv
z1uI&PbKiBm#@n&=HrTxa?;JR<8Jq-%gW*>X^Pk0@+*gc0@qS9_(z$U}OXx`YDf23~
zJskYAn0RKjH_PkBGHLdlUe5DW!dd(AVF@#?=}TDo(+~2-N=6F#evuUEZBT`m8<Adh
zJVn!g@oaEv<<XnJg4b>DB3@}3DegOg0^oZ-=Ey8IOYU=HZ@jjE^XRz-^I`#2Uf<0k
zyvL0<u)QoS<TlXWW_GfkgO4?umrMC>4Db4Yi##(I<#4UKDJpo`B#S*XG>ZG@(N^=_
zb_cl5zK~Z~!ShMDc*QF2@co;3q7%4yF6M6M`nVuT=;SpyF6Q;qrB5C+5ni^pn2&Ew
z9N3NQwd}mX@jhVo2bteIQgP;j|JjAv+r+o>eqfr!w@$8|H+*9&+l@IK7LkijiD(PU
zh#Z-}nl~wjn^S5XzxjoBCGLrdTV+&czTm%bu}k2e-3zm@gS&YeCcR@9oYlm?^tlV~
zo>Pk@7FD-$hj~ut_UzLWINWkZ(E0xdu1_lY+&umdxO^nv@t&;bl)HI+5pU#kInBTY
zZ0w2;mz%dcE;j?+|MU9kdEPSB$y`#wzqq(|KI8c(dcxdPaWzi|uP*o1eaifb?pwI$
z?K;T6qN`iJWmbj=qwyr(h|M-K&MMhlzt(1h*AAzw><|!9)sPXeox!zp*F?Vh#+7`6
zN|3(ep_h5=X<7fl`2my<_Oq38uU;IlvTf~Vp5RjrTpXST7SFyP=KXHk$7?&+O8)(Z
z=>n@eR`B>f-6?1(Xswz(?Uv%1ohMb+2dq=So*8U@S?C&%-o@>#m$<%5Zsu*1t&i;D
z)(LrG2CDyq5_yDeR~cK#J>lWAVqxRIBT%Wme!r|4Lq>!6>(nkDJN2u)55x|F(;~<%
zrMnihpLF{papj2!U(iBdKJMm8LSbvd#4mL0;Psm)!g(O$GW`CZMcaMM?aX<Fa&N5^
z%zZvng5Pxycfzbp?gvasydJmDnQm0fG_OfoY-W4Um&@@(82DZ!&HfVM@?FzZ*9AT{
z`?Tg8PmtD3?p6;)zR3%+xXbSI@UAd9#PKr%()U@;IfFZ)X^L`<<7pllQ5Wt5C%5y0
z_MJ>GIw>soq(?}-RZV!hy9=L@<UFp79(keHnq1s>`j-knn|GO0mLby2>~;kAmgTIX
z_ohteT>QgA>h^-^=1-efaB=omn#tCf@~m9yr;(&%&hx%ZMb_xBIv;;rE8p~c?B-Y3
z9ykB%V!&PU{IB}QOGnM`FJ<Pvvs^+c?Qf<~<>`Gqi`bb>i>LhJmffoaF6-aTPUO<B
zRg=_x+$nHzLOr+X|4%%}G!~n042a|rFzDdf@}b*2Ty8eE$U#}2q+&z!$4gjv<u5dt
zM?G2~;&u8S*o>WiUfjGnI_d=nnK>=C^{}+I{O4_rOy`tY{D)U+>k{+cLtA(%8De>M
ze%`LI;Uu?U(}7<K1+5oMv>#2C{$$TCcX*8<*A*92kyGufxlfdS=32F2qlB(p8+VJZ
zF7GUPK|W?N$oj+|Cpmaym2LTKm={UP=yGyS>Inw#3j)Rc6Yi(nQyia|d#O!gmuWl9
zyM~LA_1UL+d`63%Ep{Zv@Ywz~lgZ$|!ke`G3Ap~4@o^J49|zrJ;aUCu9gA^rlfs0s
z244PYw%p6-wenhT>*D&cyv4k%RD|RFl^Lct(l|ITb1Cx*eV=dE;QLPFi=aH8hD;Cd
zM77EKAp1A|)aPI?FXWwiKukFNRV{BtrM;AQ)Ks3tY9{IL`{r}lUAoNMc5?>rk>n-Z
zA@Wy+I(gZ|Ce2agNqKY3{Iz7S5bGgTzSPn@{?a6M&OTl@jd!=^2%j+9!s~FrlJC4t
zm+aff$*dmTXTa?)85srshx41cHJ*wJGf#QVu}!;3m{q6OG$F}K=BKS8IITv<&*4_p
zcjH!@`&H6)YZu3D=6hV;%WS#TQf~+wg*I|#uZrS3a(*-C{j=ZMJ1n<JD=x6njLn<N
zEw=I)zx}5drseOLIroL!0H1p;e^Qzwx9}d%qOBiUT||_)JXf9&(9J036}DjG65A;x
z{mE}Br--c#kCFKz{?ZjqJZ_9>oG&t?+5XAiHvja+oVSc)BWGAcs(`lsB96j(OT~5F
zy{Z#MF7U332;_QFmty)x`?FkWE{{djmahUYLzHC>Uy|eA$D3fzslJZ0_`eHpb`ppA
z?4R2eCZ$SneP!3@VE)l4Xf}79`F)$?Lh+I%>>6gLd5!-RnD3c%N%Qlrc0Rx4LW|wk
z&hz|vb%{r+LdD{nh!>CHhb7#pF{k+WHZJAl`pjrCp=b_IXy9b6?a%&mdi7o7b(tTa
z@b9jTz`VE@e7X^{)s}D06#>O72rry*4P4#{Gc{|>xMCx~qn#w6_d!6eF_Ry3E<1PZ
zd=oyer56NRD-Liqgq-HJJI=<_JPES)@W`W^ym#mMb5^h2&wVAtp6{Kjj`*}}F(JDt
zxB0H8Lhk=rGtrxGbpz!5M6rD{c)3DKx%3Wi=Jm7H<&$)d0Ozr1;gGww<R;t%rvZ?=
zknu%^X1;GZ@AxEAA?;aYxl$WR-uwD>+(o@#dAgkE!P_-^&M5F5UD6IVpWCT`Z-<r<
zPlhaHZL$2lvwVH$Pw}nfd=GAC!pw8XnE}o>Fuv4zcitjh)b){vXYlh@>8t_k+i|W7
zOlyP(@HwXK<=wj1lh42nvX{*3?RONjf*u^_{(HfWYfsHeu4B91c#eikfc^SY19Hye
zo9i8HF7Gdb-3zimew`@mao2RP8rfV+-YbtG_X)U(UgbTk-pjj}L4#NO(Nvx}29R;P
zF0I{Q`+fyL_O*k|SXc0XkAY8=XHTp%UuY+!|DSb8np+?ju|N9LycT#JQ$F_=*bI<f
zWW0EP1K6K1v6+%rc)Dc_IpZF-@U3@<VO=YRxOV}j#^v`7o{!<YeA)t#b44XCUFV*l
z(Z$0*y^B|Bzct_b7B{Xr8)~^&?nCy&=DdT9Ux4&$XP!oJhnn$0FhBWpD&L~(O}sWg
zA@Vi>bGi5b{lc?!c{1324@?!nG{_7PKG40ASAN-Co?RZ<+&NDna|Bk4t$81HUgW;E
z$qj6#@6W$H>kpWM+2M<pbLYt*&hQ19(Q$Gk&ypM^-mKS-ytmD^@)k@M;#>dO4IEFK
z-n|0n9gsTxtV=w)N|3o~5I<m&C7+NuWNt8SGvq${E62t74?Sw+*|>TQI9@^O>KCxU
z(=3P&!<PEte9_7RaQ`&xaB%NP6y=rohor9ycanK7mUx5RefJdV`3$~1%J4Wf=3(O7
zW4;vZUW@xNJZBB}gXNxd8iLJkQl1U2GeGwK>goiWw|q%Cuk*~qylj0}z-mBp$oS!{
zpM3ikU*`!mjNw)GoXw}{4Ox?XC0m)#@^A{zu6NbEd(|Lk;ezzt<#Yg>oAzisul|Qj
z-ammm_<9o|<FW-iRKfliaW>?O?S9D<^;H3^cHO!Ke6pPv`8FQh1$H~g9M{xaTst>H
z&JDb!EySzu)XDq71JX~QE%pek&UQx!r}ZMp*~1_`2QNMVn+IY)^qa-o$UU7~d)9Vd
zjScdA(kG0-{wvZ=;g{aI1g;LGwr4iv40<vB0I<0rF=YJeRyFU1CBNCj0ul3K1sNh>
z^FZqSp3dc7*9=*|4&panf5>-P1Tq&fS*wz7*_BDWomaLZ>@@P{Zu|<_=W?_jQYL`(
zfiPEo3^>k~ovr2an|YWg`H(1Y_6b?O)jSTo#XcwaQsO<i?tj(fevv5y4$FpBf5GN5
z9o!Cf&)2IRybY>}yweZO<5_oXDbHy;NIpr;dJFd3+<88{H;WH~`)(_@9_H2+xy&c}
z0rl*M1;#UZ_y6zUk$5S`U9p*o+xuod@2Oqqc|i03LCucfd=PW_EuT>bFJEb^BEPjz
z64-2z`<<JNxK}<q4t5iW55k%k8u;9%T7umN;;YPp^gTfA^*14R9~opJ=Ejr+qTu>(
zv=#6bPT_@{|Kp_~c>W)x_xR)<zS2ck_{x=Rd8ZuQ$F+EW8`zGSCa1w+2vSpg0+Pl+
z>_pG)yi?yn$}c}gVIJohPTWVcJGfju!+8!(G3E_i`jwZ{e-hXXka~s(`EYkkT)2ua
zG~+znT#$TrU<yy#wI4i}x)I|ICl21@mhD-=b83kfcM9_gxLF@AuH%*y`v%T`amr3S
zCg&jM61V<r<S`Y4tYz|?TEX{u#YS*>(R#X;Z{;j`u>O+*`@mx(AUj`N>g4;<`4cRD
zEvuVPphtnfzUUI4(VA7f=h`59Ktbx(cR=ncIx>?1tQI83BwLQ+evnvlk~nvF_ypeW
zM93V(=^xDCbOn-I{n(06V4^%Y?2BHAg3Z~v%ZhK>4Ic1V0Z7f#g>7*4i}{s!ukXKM
z9Ob=$TjTUb;f$nILwB)Wo}XqHcssUTmTzUc!EY3N*qkBHMfObEZIP=SlPvlq=5r|>
z5;fl;yHVZl_anYAUN_!-7nS(+XHPQA*vlYn<iQW#|NkiDJx{C2GLh`A`=UDY=9r&-
z(a(GRn!dtzNg+9pwpPX9Kgpt%O)N|f?X^7Odls9S%Rc6jNxsUXF}aQBS*WJSr)}K4
zv$iSl9>07Y-2dm99Ln>4`eAT*c-ZrpC&x7K_)k^iV~FwA{(jm`sN8?6`B%ktoJS@}
z^RB3wFVF17qWOH18!zvhm0As%QD(DZW^($Ozu}E}Un(SABp@<%=Y4QEfx^UaNfyuc
z;y&J!Qb9b6s#$r}KlJk^SM23luPX??|F^7TwV6U=2-hi%6`b#!dd%+yE#jHO&(Gtx
zK!z>FC!Z(VRNG7^cn<H!yWHGdt|q*<@}Bb?<mBVD71$}fqBo19=Jf)eySJOo1NYi<
z&*!d?KYE*4)Tj6zkKb!Y-g6p$ykD#pco^fngtq_M&1I9QDYKb}Ni<t(Ki|_zM{s)H
zWF^o0Ktce_=D!@ttK#%o;8V;6PT9T<0v_*^_`;Kdc|rUCfA0EbzDuk{w2bMLXiw2L
z-eWv+T&kZ1&E4fYx%G~Hk~t`$DU`KunLw2B5;M_^KRoNF{^rUxmlg28vXN)1_Y8^t
zWy`tO%irJ$<>U}_`S@6H`72Q#J>gY+OV(NNta)t7=eUtcZi??T-U?YKjmZ`HY?+T=
zm@hjYXvX2ahi7X|BJU)tI&LS;CLZ+_ZoJo{H<+*PXX5?a8_UzkCBV-o&cge+W3gbc
zMuU98Ol{E<F<pFfw+TxBRF2~|>CyxD|K;+R3rxzDl>MuBiu?WsOaA*}Ec{HnAnVf$
z?QXKG+=a{qfa1))rj7fd#yq9#uMhBL#&vM7GXHP>qTvniimm5)>aK|@d?<M;z?dh-
z&3|Evkf39$3b)!E#Vu2`l>Y{`s!vcWHh=T(2G5>)e%1xXGo`o|E|YC`d(30PamH-p
z>NT7?v!jKVL<(6X{Qb;(N3w{2dt{&Xygjc>&rNtEc66FLPi5UcUW+p~;OS?2jTM*R
z*PY@I7$5SkzsDr_@qUU>LbZp)163iu(jYtT6$TpMz1kpmiz{%O+ZFW))xJC~cu?+^
z#8n|D9-YiBTwScHy!Dz&rZ&u3=I@L>O?84gS+6xagZorM**+qhU)iYU>DHMU-r?Z&
zlby*^QZB*wcy$YR@i|G}PV0y44<6=&-81VxA8*y{S&EkLC-U^MOyYUDL!Z~YKA!t<
z+9?t7&(@;lAFYK=pB?43o2t$AGC)|!wp^7*S7(Cgjs8d6PlQWNdjr<-NCZw7eWv8k
zImK_Cl*YAq^YY3Vo;ym@%_ekM@vd#;)%+V&%RBR^qwL*_b$n|q<OCnDsWg*3eBC@;
zdMS@$zJx|X@eK2GR`R@=ib{&voDD)z(^+|y|6DYwE$-!F-VIqt{Vk1~dyie3<PWP|
z0+*hA;C`@Mo>%AFG;`~USv;H<-|`%H%`)e*n#FD2+sx}-A#PsH7RGyI&QCM;xBjBG
z_vV28Dz({<=Z?rOwXesyxURSDXAKDS;k%<=%010)Id7K2HuDRcdUy=A&+#8k=v4R~
z;U&oTdZ~h9?Kfk#Ys;hs9yZA>UZBe!Kg&-{fmxQ<(}stKV_t*g#%*VK{yo>_Jzlk%
z_ux`NaGsM<(%_k}F`dt7uCr8%H9xnoa3j3`e|5n;o*y%=m_K!T!*O}f7T(sU&pFq;
zoy4bPWNPtHB8FGY(ppArmldB}EoA*k`83GBWFI|EUQqx4todt&<|Qq>7e8C`$Xh<+
zseHbO+m1)fJna=H*SQ8m(?D%jt{}Uayl1#>n11-YQlr4&4ev37KfFs)8^Qhm-G^kj
z%Dm?A-kYu=%ynFycc=X<$#_2|-gv*c(%#NnIbU{8=DlGd&b#~rFYlw{+`{siJ)+Wq
z<~;51H<}xCED(C}s*_K?a4Y{UJ5@HHoR{kM9Vdi~g)j2{@Z;y>+Bh5B4;ONR?0uef
zHJ3lM(2YmoPM)yab6&1xZ@&mvA6ROVQk*RPuFVyE#>M--lROVO!+757GfCbr*v`2&
zouB7!_&x4c4l$9G=Sw*2xpesSEZ=c=>6UWVS$>uNniruNvwAw;M#n%Q_g#!;a)&Ee
zBhRe@+k1ZQKCa(-@42OvKe0J5S#gK1b`x&Vv*6X2{>#N9Jz4rT!+vgm+Ya9J$8rL)
zf;D{AuG}0KpQp01WE?c-Y;ESPZ+^_h-kd00xBDp9>$+Zroozj;!slP{B`T=%tejbC
z+L-i6ZtJoC=32^%f<diQWlnB7z%%Dmig}#8ADixSBcA1gi_L8AZBu+Zbt(5k#bcaB
zf$}1uo2Q#E_;C!p|9|hnt-N<MzM0kJ{nwb_(a0zFRmtM`yByx$+2464dp$Bg<FJJ9
z`O_^t_Lb}Tev5{18K3`dUM<1Ow^%1t%lnNxcekz?Z|wF61siWOp~lev0w<J@t9`7H
z2ak(`$_lM6WpLYK!j38G`afC)lB8}4%+)fHQ~uJ#<J$3no3E{kuiNp3plXOJcfb!T
z-l@u3yh>{zdncAAEaP33%gXiCoR2rMO^Hvw-#}vT#sfm17jEVIV6qH+H=eLnCcoJ{
z$Qs$^IA#H-2cLM?876}EG6`%lQ39uZ8-rJ3Z}WZG1q~<jzmjtkbvtsFxg~xhIRAm%
zxc8|SfBk{w0>yTXd`(-=!`l`hxoMI-0^*_qe0iEP__v(8&D|>|z*F-%l*dDZl~3e0
zWc||4g^#%QHw%beo*l@Se`*(>cJC1`gR_PL&zabH{=b8qO$;)NZOuKfdqC_*UpNKS
zM3!@zEjh$@LSBID_OCVkkCv?vIj^;b<#2uzcWJAXK(L?%*o}=En|L_Sd5iwMFom<F
z<QM<pe^a^2B|d}YKxU<FRcCvmk-^Wf@i1T3s&w9a-!Jm+2xb>p6MK-ATO2ZHUh5sl
z8<+Ey^S9VWF8v>oJZB64@WozO3ij*91R1WCE3bpwS_h7v;Jh)_jDNGrV!lhZd3?2@
zkoo@~-4R@QGv|sdowkm*<((ntGWoNt>-Pw7PkpwTZ_UIeZVBI=;5NF-2PZc3W7oju
zg50)YCnuO46|j)6wP_ug4HDmSX*E}O!c4FpJ(XSoj+Pp5*ldo_<~<nFF7UT~AMfAn
z+5ArSE7|?i1o-Wv4Y_3}SAz9>O^M<^z|PMbab_+5g$6+`w^hPChwrrs9BpRe-+cHV
z?~0NxUN;72{uh4RxDUil;9A@FooAZWdd>r;n%w`{*?D4bZR8X1*$p=9x?2<9j(`Q=
zHrda0=lH+HFmPt|DS^|hk;OIf%2n2Ur<Xng^B0T#2Gb%UiUJQ*w0T4i&k$I1(}sVw
z>3NPb7CGQCukULzI1iM~=Kb<!59jWJaGp4+Z;Y{e`e1#EuD&emmMjMQ859P>oC>@g
z^KSE9U$v66*?ThAi__hF*N$@ubQ$vroW0u3lll^}cH4x<lxO`$Uhy4nQvACvPv#a?
ztrV5~d5}l>|1_{4R=F<}5WZW=>XDPly=mbt(ELBU+lFJz(s|N+rkB15oZ`OCerZh~
zhf>95{)7Er`R4Ds#kXL+t3dWf)O+C03;pK!xnd`H?rY*=bACl{Wj;opTO6m)$8hHC
zf!z56a(A-wQgGiO)>DuF_D_AjX&T*p0`D66ERs|O5<h<B;d#`~r`>AFZ?Wno_th7W
zGh@~NWpK}(<jenmaX0%LqiBAXxM^IrrydD#U({zWFLUOt5*OxJ;Re|!E%POkTT5G%
z*Zs$R{)^XGxJ{UO1wJpdXOnC5;H%@`!e_SFk?(E<r{MX}SDcS^B?V+=vV+6-w0j9x
z^7SH~wN<j*%}+acmsBobIX**GAVkJSl*4!?SLRDT@kf7;@w{-=<NX+=$^D3@o1e8r
z4D1)_M>*UV%Gmf0<*9Jz+<eHltYDqUv?~gH6=FgHYWueFr`-4A-#Mp%bIMCi{wp6L
zd&#+0yyZ@jeaA06r-OBq_!HKeUUq&)BVEzVu$R2`v${EFpOzEb_Sv4_O!+joQ*b$d
z_jM`uwJAy9w6^Fc<m_AvMnA5zyf@(IrK~*C1nws<ouVqB^WIEU=1e^}j#r4?WZ$*e
znV(-vhWoEtIQQknoxBHqHTjS3KF{&WJDksJS0s0dMkOZ`GYj7;lY?Bx=kDeU*6J2m
z-*6M0uWM8_1e$yF`3`=Z!oTTbFK6=h2(Ue%_<SY1f}cHevFN!V5B8W>KK#6Ak8=E6
z`IO&x!wm6&!fo90U#j>oewX4Ddvh8b&T=_o0=w7)S#uh-!G76u#(-PpoTxxSkpowj
zhLeEwRbH+du^fSjkJ+61IqqT)Gw*O1n0gDa#zwI|d7&rvmAhPELgEGfr0up~^Pcrj
z;(HrjDX`^<vcMMwcLDRBjXc_~l=+Hgw()Dum11AwUn3B&7B6tzF_k-Rez!o^#sB>2
zS5EQH+~^~Ied<20=nejS?ANF9=b0a1)8SSUcow;uM?b<^AVXrazzL(<{L>GwX8)<@
z#WjDe8uNc+2R@%b$QcjUb=kpjpRq5DcU>V5f7piE;IQV1xX$IbtC!O<&_dwB-s9|>
zUYK(I{OruJXTxq@ztyfBb3`@>eE7k|we4^N`|d+I0+*En8DB=N0*A$!oFxo4yERxJ
zpS%ao=hK}e*?(Sy?Bh+B*}`E}xmw`t%lrJhj-_yBo3INcS@Cc!Y3}5{)nEu#zigob
z>(SX=+%dhjVjS;x^PLlH=K#eK2){~c<a@lDMP&D?tpX>H?&CT-v6JtQ<ud*n&-G&N
zhY$0stmEYq?CIwU6fx&bIKsfq?9#@2B8``at3i+}^7uK<>zl*Cc5LaHE9RelRDdIN
zF=zb8_53SeY~b%vXcJNtnkJAEc2oS6vlTc0*~$FVSPa1Nv$K?$+b{hxxLn|L;1HL{
zeaSuVyF35MV~aVTzn?BpTExp={IHcTXmJnk)|X{GSGTSJha<?%A+fW-?*ApJ2@W5S
z7{k#rK8e0aF6Bq5oaSBon7-O2fXzM@J)dXWG)C^Ei9KL<3hbUHU@+$er?1;gt_YDZ
z0Z*n<u(+fcqhR;n4shIk_+Y{3^UMrvU%;Xpd|&drc`ASRaRsE^<^O06>3eSGgY2)}
zxAiaIOP>>9wN?ooV84OfW4Eh{@0I^4{`C$qyk!b&!TALw4#Kx;e(_~pWarhJ-N$Fz
zEx=<uc{b<ao}FB5kvx2IPu_#;EVVCU+;IyRao*`&&3%~nAMa9)R_?C5N&H@&j68|y
zo51#g%ql%z4!6_l=YPIbElKXVM|bn~@9pPUc78b@i>MZl|0Wgof9H;Hzl?I^-}%%Z
z9A?b3=JUw>E#qEnSi!k*ss;Z{y-JQpHCkZ1LFR(+_k$&TO+`k$8^iAMbS}8f<273v
zY;W?qt6=}+9`NBh=bgp1>*OvDtNs|4*=r>EEQ%oeV0%j?xOo&IXWXyqdBb^*y^Zf>
z;A`HEHxqeNW(0%V8TNl2!EWDst&xYF_d57Y5jm3|+|&2)^PQe`k-P7^KRBLuPfg=U
zu7})<0CE?Tb`0458?X8Jy6cvL*&y*9-NG!YnTcRM-FZ9t=SV~5ew#Q<xvlmk@~bFV
z@-QhK<y-ttghTh27+>QP3+{%~JHh%7s@w5R-POj+?Apt>%J(;??He&peT7VZc42A0
z3p1DVd`x-E9d0JYmzy-3^KxV&*W4Xp+<SVraBRP9!J)Udlyh6J1YfpDF4!!wUCmsF
z?o9%R^Y7+kd{?J#<aDoi4fd;z%zv&6ft9>bfsk{sSDg6?7Qb$&!~aUuhC7w1g}-dQ
zJKr`IPR=xyqu}^xmC@$faAhjDwT3wN(gSa}YFRqi*Id~P*7s^t0sHo}ePBO>!hrR3
zEcezs?0g3oIdk<r*W~^j`j_XhmmB{gc2@q({=3{x_aNi*4O^FU?%~bjbM21gOA0IF
z47+=TgUd*e_gEC<-pjd#kiF#ZuI=OgTI0q0zi}_TY|qkq2Tli#ChE-2#-I3Jt|{lu
z&-lap=CL3D&!vdH#fBLjYz9|MIPWAK1FPTox|MJCJWjqvY^|(sBHnUNamazYJ8XkA
zI3ND4Fyk}6w2JrYOLyKQW-7dY?;G=5va)gg?bYV^bYu;m^uj}&o1!6kPi7V?=h;h#
zz+rAu+RDc*-N02>_zP^;@~)}eRpP>|hfMUq=Gt_x=ANRX&MP<fCg1<EB<_UckU5Nx
zA4+(9-(BLl$a9_d%6?`6t7Y?86Y?$jk9L-V!&h^W8kd&NB3|8pkGW60a|Oq#!p;b=
z9lpy_xSq~^!t-*n6z{<c7r2gX?cm;<t-<G}(g^mK&klReTTv^RKTMd<ow{uw@742v
z`KFae@q+d$71oCHdB)!2W&f+qIw|WN??vecV6!hq-Q-k_gp7%U;<H`boKI#UE7!*9
z8+p=~3-dft(&r0u4&zJ8l;`=r#SiRvH(kiyiEHhUy%qUV+qjlhpw6#=%;HS(<oA6V
z$}94HGFTr-j7v9-&p&B0hld9@&o0?|-dkJR_?CywX4}QHkM~LLY;LOw&p0`z@o~>w
z-_PwaUyAR|uF3q@7D3i+CrlIMpYte+H?!ptZ`ldNxxgTE_nn8dMeg|6vqs)p#wX+T
zi{*>US-y*__VQ($+VaG&Gv(vj6v?@<5t5F(LUh1k$|wWb<B~t|HIL_VJ%07CI^1<S
zHvB%Pthl~jjNyMQwt{{4D<?jN$7vj&n*_mnMA&b0n-=!+H^eFMzpnEDr_(;mF5XIE
zFaEXr6#1WSvEtt#7sC7FnmKP$jw;`-b<XT@EN%Qtza{d&df&o5C2%@FOS=ib{Mze0
zvW21iVo&FAge>slW$<1G4yPt(7Ji569-PO66Zt#tUE+UYDZ=mkXcEhf3sT@XW4zqX
zdu4SpIInKY(gdfs@|q_+54I)qF~%$cn{n1{1D9511lLgk5B}I5L)L)5+d=F9IF@i<
z;!bDR;%rMi!GGZ7JC4tVd)U9QLFR<d9*_Z>4+>9yjkTQYnRCE?dp2E=!#xXf7scKM
z+MFJi3;1Ktu<&;;Vdhx*Z42K&b$_n5|LeF@4`_qOx0e`NaK3Ug<P<90!cw|ckgrMX
z4mf>+%$2+Ok5?{Vl-FNn4*z^65pE0R5?=Nf?R-;TOy*yfdzH`N*K!_Fzk6K%X^k9e
z99Eo0i4%EZ#2#=j)avDMSZTvGg(DW6Pc%dY_?o}2=ilth#LgS_girqWCBBuLr}1xD
zeVU&~f`MD3E|7D|#&vxAdxXJ$=wnFWK3xboM`mJ^fWXI}JUq|eXY+18vY0c`rj0+Q
zESN8DN(k@7ss}tbAJ661=SSp;+p8yn{WZ-C(mw?GW3>u9??ewf4#NeIe%1O<kUpxr
zP$`$?w#nSldH!H`MzhT4mrf4kEaqzAUY;?He?A)nxZVl+BMojZfXvcrS;Eh#CjsUM
zaa#%;3Y@@~bk&LbfaX7eWfvfKmMm$X#CNAZkn2rbJO2xP$XyI)eyD@v8f3=7eh+@e
z*sTJtjJx<2rU-$}0*Qn0)QOq`ZIN<(!pST6w`^R&tHm~zCu3h3uSk3=-&qSUuo<cU
zOL_A8xOl|Av+}PzH=A!t_GzAfg$4qa9K5{8-o1dEmzD~dD*~DOLr_k@X}&Ahm#o`-
z%ID_sy!Sc7?`UwJcOSno_XCspJf94N1w4OCfzxR$?`hsQUB~(SCN1WP)xOW)&ehAq
z@<oCF{{~%f*rsh&=PWxjg<n?9kbjx?ah_d?>v#p_cm-@$YjW7^tOn;hrg!Old8Ti<
zF0j||aD4f}8DRF5Z&lqvu-t#1ot!t1K<=@0+b79$I#`hZ(dHR^XAW)U`)c_B?8h~2
ze&F=oBJ9L>Tn=(~uAtOx9>y<Se6eafxDVY{0;d(*vbh|S^FD*k1^M&lgacsx29v${
zR5l^*Pi~E!!mVQ<1&-%is*M6qy>@~9+MCnMx37Aoz>EpOd`=pR`5SKuaad}k@%81a
z@p!*+2b+63X)1sE;#@wfq@Vnu3lzDh7;EvKTDMamxl5A2RY8U?*NBtPk5`u8z4jl^
z&pY{?g;FBCEsBC1g~w!ga>TTGLxj8dyt?JV@v<=NCbz2fB5)h0<nlfKNhO=O>t`f_
z{rYO@7T&)L`uKA1<bvfJ_Cd}p<6EgK@bIA{uU6v;f$!CJ{4W_lax7D;1^e^G(k<MN
zU%K+~3oCJbSQEv&;kpbrj~8U0rPX0Cu96>+xmHlvteh*rcU$2uU*7RDZs`Z|+_#iC
z`I(ln3TQ{D3dG-V<h5_V3-+(f1}EO6$fJBtwZi;A>vef$?T&FT;Z)*V?7jdjfBw%+
zft4Gr*pBil@dlps;A7}`2#yQRFAF$lJ(Uz>dNh&s{u>TXkDOclHB;R9|A<}Vv$RVR
zxSR40>`#zeHi>1hOK&aWW){2z4wtRlb@*fUJmlTE+=1&s{(TOm4#a-bBU-z_Wm4%=
zZ~g^!X?zFX9OIMM+|K71WF^qxBEk18A)BWvOqAbLhJ|;TE~NjzqgILcnu!Ac>$lUn
zOzV>Qk3ZYa{j85&(9wDhdx_$1UN=oEj;Yb|VE5>2z2iCmUx4q<Zcc&J%(*=KjZ_5!
zmNl~;n7otk(0v1bBN0x%;v1@h?;fw<iY-(ZU@P7ZPDh+~k8(F}&gAP`IDzNroNas&
z3lB3L+XOi?VyC+cPx;)-d@fmyygwOc^1XBO;`uxI0e{}sda#|+)mM00jxFF6_$S4C
z=!ypaVmBTBSF8zq3Uz<^HM+S4CW$WNHxhZosl}edFaJFYoF0C!y2|Ub<2irbT~;>t
zGE3I(1A_c#)`#;1N;dI*FHqs#$G?itkgtLNUwRbp?b*lqS>NwxDd#={w(oD=IdFRU
z<MW(#0}o_h9Vi}Pc*8$cftBz6@~+znS^s~$<1(9WlNW#74=0}WajSU0Rvh3x=AgoV
zR79Ms@-`#in*#@V@*ZB`%H_Ypw^m$?$33E-Z(j3kfo+PA{!HG99D%z54t#10*#%@o
zZt@uHgq+I)vd>=s8-LWXI`)hk8l0CJ`1oH2n{z+D!Xyy#ViLb9?_{2J25b4F+d{a*
z>tn!qbl-19Q2(D}-Z5uzIOzMn<(1f@Bp|ACoQ=sgOJM)&c%FKrxdMj*S96~YVG%IT
zTE+c78FJo+MBYZ8xw9GtvRL->FDiwc378{yiSM!W3IXYbP6C$miUgwmbnw03sl=yt
z@hShL4{O<zck~Ku4Qdpa$@80MG4DzNH|J0M2k*Y&b=%`DprS9!-4{^8*Vwp#e<JTf
z)>Exv0?n3+ycc`C1eSb0AW*XKF#r1r?>QM6t+~Fxy2ARnU;^LG&Boxc*9nB=>#67D
zd6_P;^2e{=1<p^;R=4n6PW0vAvG5Q8t^e!MlILM8y2G*eL>pfr`+uJIS$hQ<RSmdR
z8N4{2X_g37r@v)gzv2)$UP0l>7POw*G1~zgHaFfp=Qi?C0q3bFi%d9=?7S#&T<{q`
zx6}l#Tf#vC&twm9Z(<hVQ8@+KbEI=?Gxx&Dhj;{6nQ`bd&*U>n{s7KvAaj)k=JN$H
z&*59Y@|8eU_$nTkuBCh_GcNPn-Zv2NUwwoB)RBBXCDuhed9Dw69tG*~W<O2f&AtAd
zXF1mvZiDtVcD`+p@J_zPCNPKTlfY5si(Egh&*N`Dw~2rL97loW6Z!-$&CcUBdb5GY
zQF}4J-HSS~pInoAd8CdX1N-4={|y1psRw!Ps6OUD`Q|XUVeKw~mrJwwP8i+h<DER4
zcY+5aPy8#y{QQ4T1F-wIorByp5AuiImKS`gB}=%uHLJN}`5&+x{RHX%vpd`JZf{w}
zeN}2YxNjfFxL&|bWHHZ1mS*mQ|Lg^ByKDjH^HM8j!DsgM;4;oRSefgS;WltPPHMtZ
zF58k_oWHFFxpsd1$|+Em1~z9w*hH{e3ll@IevliGam}{|&fou@b9s551iKYk&SAYI
z=M^Jic7=Rh&cY5kaCm^^B@NU$?RIj4%@x?i$hGqEERL726TtTUUgpSo^^!7I^i@F=
z`%ai$1gi(>2VwF1A32NkMN!l=l`UYu<~R>5R`(5Z&Z>o&8P~Mn5RM%$x;Q*CAm<(}
z*dGe^GsqkePFi=Cjq7;^`|H0h9I6$w**l+32m8g^1ada4^8*9U{Z6^y_5w(~U~@cs
z)hfu@kd7gjIM+uV0h^b2eK*JbjdGmo)o)m{Te~?XPK4a!a*+XY7fr#Oxo|&x`Pt3o
z8f(J!;P)Xe*FSb({dW(SaOTW91GXDv&yP#OU~v$eUz!Wf2JvAyx*4+G6ULu*^a!Ww
z2|uo+NhMspN*`IO8ZggCZP?n#YHwc7_1nJ<Y-ao>6Hexyb`FMXwVdG&4qPi0``L2%
zCE08(Ap57UC4B^|1KHtsQxNVx5MR50Cz$`jnUQnPX$H>CK7YV!t%Hg<+dpe_PL=x&
z=8JM}1;-0W4+wuc+R1fn>rFQ2Ge#ULu5n;FYh!8l(AZy`p><QhdM?k}%dT<wHJH7!
zL7v0A^b*)EkbV$8VmO!m|Lg?z+tt;aUc0q8mu|}7xPBUPe@Jsj09>vAxg5?Me`a{t
zFNqH4y0KmyEWgJ~0!%;n>BIF!WD)zD3P`^SWd516>hLrR;=}OW;2B&&mmue9fyCx%
z-Q);+wu<#c+B~pZ7v#U>U_P4xHuv$98E|_+YI`IhX%NIda^Vpt_us$ZboDXqGl$f2
z8?f4C(TBi#*|tK`6-aGhU@KVNb*)rRV-04mDZY?9a6od%SSI2S*AC~$Op_KbU^CoR
z%hg{2NjF#ir*Rd1eZmnEyOEQl)CU~iAboPnA#3E^*yeNY{IAJ*MRP7^qSIQi{gv~D
z!RjveZRYgsJ;9kfFAAJTk2lJ4uj3Wqu2L-o^Fij^uWMsjaS3wI%cjX^ID-m9I454X
z3wHAhJ;;4qt?O&qLsAjzh?RFw1&0?%e-D!>r|uMY&WE*cIWNs$&(Zz#GuRDT=h?VV
zWkb>_NF4}Qyn^(tGAD_E-3Jmw#)|jXaa^v{;1d2h7u@Fo$-P^Z3Aby4gFMGMjty|P
zOBjCOdMO6k>%U=cB3I7-CU(XP)8O*|X0mg*T$l^if7%doCN9WK5T2Z;1QxG+)WV+i
z?-gs%1};|T51E`&`Fl9IC;jD`_BoqV%R!oJ?%qXUbuO-ubHk^HZUehLk2#X#d*?+?
z52vY|7p{b}%LUkj{Vi2`9Gn)~&a-fCJrCK_{r{K|*ZrIqoJrS&;BEuCp>(4)_sm2+
z&YLcLT=Tpha|YaK;+&9sgA>&Mzrn@;zQcxVsR>tk`bExp{5;&Ae-ZmpZP}AJ+zMWR
z-3BrXgy#x9;bNOT5iV9&{u$2ZSX={k-vvn>u=_yjxxOOC<t7CtaveJg8K(isu`?%f
zzG0{U>&Z-;$C|om71z-}9-NkfPVD^c$HC^RB`txQ^?uo5uv(DbB~xqJCwSR{)vQ|B
z&+)HOnA4z6f!)yMB1g@q<y>Y$j9lTY4)8px|BV42E~jrqbA|j)hnod5Pop`7BhUOA
z*JE}jxOl<sc=mY?*Ev#*PjcFMK;~3HYCl|zVLxgh0Zu<WjiDSt=f%KoOm<`B+ROt<
z&njnDbEYiR0lRNTratFZ4?pm|(7E3s`ximx1kDfT;?e#FR`b4zk?YEg>D)UevT*G+
zKfqzQ)(e~lK<b=N7J}6rF<%LH+q>Hf;B1f@5SFZ(#ra~NB$vl)NT0Fd93;<y<Zj=e
z&$VVdr2OOmm;p9Be&=$o+UOnNGyqal5bzHy4q|urb8{IeK<+79a4DAybPuimuPV+y
zLn*G^SyRAasp#(j_P3L29b6wsFAU#$E6K@u=Ow3i`&V#Yhlzt|-Cy3EH~NpW^Y}kt
zTfhK0Hv%NSc#$#JlrxaBqkoDy*Q+13oJaW}ckP|Ot--$dl04_*xf{XZ0a6db?~)<o
z-5@pyr%!6+++LM}qUO*wM$Y2O>0ozm+;#@+e*ehZ?C!xvtp6siWHl;10v0pmvqlNm
zWi2tB0*~t1EfgF$4I)xGXU7(R{rrLd9hhF;Tg=fe!w6=B?3SG%!ZKy86WENkl@mB}
zDtCb8#B1C*96H`|Y*2a2Ca{^GvF-%q{)Y3MkoLwW+fKN9m(1zq@LIscRd&vjD|X#%
zu=u0eyBr7o4}sYrd$we%gY5vZpKf=7i-Y(uEIS!BT*UarIW4?2xcrPya9))RV4t%v
z6dw05J!iJ<=hP{f#kInt5^VmupH5uoCC_q{ZClCCni<QH8|A~M!F-sdQW~-k!f7r$
z-0ssBzrcQlnf3B*IGC@=caLMvbWP66laThJ===v9mwp^$4cHH<{|gd+!p#8bO{ik$
zT6>U@-D1~tj`JyybECA(7INHXVB!2K3`rZCl*Bn^>(ztxDQx0lpPf1Z>|T(55Ki^s
z;*jx)=WL#Ll=G*S1n05so}79=t-$VS+6X!03Z$<3R~JXdA|9|fi0_^4#TA%m2R7>v
zGo;Stc00&<^r1HAn+gwb8Ud;MbXW;qwt)CBe0zr?SF0W5tbdT$*Pogkxsf6)N7(hi
zY41z4EXNNvCve%JqL&G`7o@h<#S6~2{rZ)C!m*QJevH;u4v!>A-7;<OY_OQvk|eO-
zLHd~7RKV^!X{W*-cP5ZahjkO!E|45DmY?{O^G|vk=LvgG&TV^taSB*X2K%$m<{+m+
zvLvT;MLZ|#WXPBbNMDK*q<uJ*(}^?X(nij9jXms(^#s87&k5!Q+p%GGKgX6Es$Bc_
zK;~21O>DWsrk&;bTeKMLCXhKN?;T~IdmVC5MEVhBt_$gB*e_LEf#qYn`oQL#-aV7U
z{0?Ls9Hd9KR2wW0VlVtMjbp<VQLbz5pE*@hPIKh7>VVyy_A7=fXxU$QIDpj7TOkj2
zAES9Uia1CPgoW>Rvwsdb$$s#AH&`7=Y@@v&oc-J@m7P6u8Q6S~99#NbPUG)FV6!fy
zA7Q^8c!GoR2jZ@iD;etSGZJFK@@qakfSUu-``hviSp1M~Bm2jtX`Gwiuy80XWM}`}
zXvgW_`I+l|5F=+!?Ng3DX4}E(VSNB(p61*kNV-0_PlxkQP!5O6zRMgRS>?E5{_%j#
zvwd6urZcLxar`+32|t&Q$(*&z^*P)BLDDYBeh^mV+|G4u#w+#))_t5`HcD|VG~-}A
z>uSK2Ftw1)r;8u#&-NlMt~nmAT(RsLT(z2zz3d>pFV5w%g>HtlH$i+5=5u%DdX%yi
zY(I$qxpzK1o|ZPKfz5NCJPR&=PFE1l*1u`ZB@vzrRs+&2Xt9KIhuBPTm^~}j;BdM7
zo%6uv1MJJ(Cb6C9oemavXSfOu8<2YQj~Z~c=@t`Cor-5*v$+p5afGk<&9Q0DG4@DH
zE6#%I<y?8DZ&^?0n}E|6Nbj`0Yv6hn4yAB-@g#xG2g!l3|8_ObuxekfX>1B`@%^PL
zY=`|0a_qX{&S_VP=rgph-oUX<eFFG=i&ft`;p#QQmU24gMS|^23SZ9d(^(Fdb1*o-
z?(u0O*x&5kd%-lw&X2_`TsNV6^8ACuxO6AX=X$5i$7%ca9((ksUa&bJb=Hl?!EXPy
zXBIrH!=fN-yg>3WEVK0v$2GlgoHZR!!RltRAnM6k$EI+$*ZF|OgO#U%%}#JI<lvw6
z7^a<Jp|~tq93-$yegV(-Rz1^(rWq_du9XSBRggEfV>rVd`K6NQht?v6u(}1@R~Bbj
z7;oAn`&sd>Si&Az@I7t2<ykD6V|_J@|IOm=n6Qs0aJnpSo{xd~_r^{km4C|MwUHq6
zt?Y|M;=ON(JUl69@m}Q=ulLfe3ae_D$_3>ME4}>l1-#C2b9XOK{TDs+yZ<k6mF(^0
zn#k~!N8vNS@Jb&E-fPNhxYT$=1VCn)oPo?UgV+|!ADW*jUdnw(^$@3?z!}{_y=4N*
zzkMu<I=*oI(*DbH?XZqQ+}xd-@$VAZx4c@fU0ADOE@)QB<2$R8d&lf^0tSZ#M2dPK
z=fi=_>pN7&wY`{+tLU6Km#nux*TuV#vi0?orQr4d*>xV~JTD>pARW|1E!^FjxprQQ
z=Q3xy&&lzpmwQrHIC%Z~-5GK`cb4AcUOgd^d$#gwu30Pk!C@i3{1*4;3=xZBJsx)V
zB})}Vc-9HesORH-a_=#hUf^7w2OLj0I37+Acs|vMvyN@LjGnTp2&hisW@87}Ki&(o
zxGeukfycx)u6o9%u|JBhwB4Qkk}W^K`<!b$w|h9i>CIJYiiODLR#CIK)gt{j4)b*G
zzs??dy3o8uViRk?{!Ce!m|X(dO9FV+r$(Ajn_tGG9hb))6#9fKG$od6z40{3{Jci)
zc|yfJ^X@6}sTTg?%jDR}b<*oL_msJiy6)&*YdP)oE8Ipbnwp?+%S<q~xUMN+9u$$z
zbKi6}PiV?gZgu~kU^l0?W?4L5`Ic*%@B|)%8J4`sZ!3AO{N5yBz3GPh{`#xJo(<Bx
z8;lKQvaTQIY}K9%4nHyFwY=PZpJlWUPUEhh^Oc9s{{z?0i^X8`GF~s^$hhGMW`pbo
zVLqcfD*t58@y1PB$nCeb*Fs+~o5y1#2ltzoI*JbkP56{7TX|3a78LBg#H%`~>bp`K
z^E8!Z(PEklKk{2l(Vqy;2T!Yeq&^wS%H1o{=Y6fKY7V;pSB_6yX!_E%7UzP!c%S~?
z#4_XUD(#0NTg;+MDkRPa<#XQYTFm2Ks0)q{kUM6l?crqoJz3(4XES%^&13v?EFA<o
z7f6BA&NSHw&Ni_#@csXu`_`Jzc=uFb_P=QYtn8)|a=y>GEFAn;-rjq~_3~$$ndSFS
z=HHsGg4g}eW$FdfH9TG-@@)H5r5T;fTl98tAKCSdd*wOE89fK1B*FO|<PH$t`$&$L
zW6@J(v*YpHJH$`&Y%zf3o9A<8348a+i=?=}5VUUE4R(8E(rW%<8z1iF7qUhEEj-5^
zp4nkGk^ddfr#a8Tc~h0|l@yoaCUcRL2p%&HW%J+e-rO!HOSD9<T;W+cvs3oj*;!l$
z0m%aPr$x;}*Opr-`Wy28Ec>Ie;iR#}!F4;iZ=V04=$STIpltCx?p?pvm_8D=<9I#y
zC)jR~TVLwLNVZr*)=XNP?c(<JWVdLYAi*>BhZK*_{ln%GB|+?LRTFvtCOiVC4?*dz
z<~>_nL=Cd4!0Ej;{3X|BhuP|Tca(A5e+s#KOqg{i*R$LW+({2oEMB-A;&h0f#@jNd
zO|dB3hnML`l;UBbL#B`CO2{xZmxI?pCm%Ex)nk(6-97O;m+g0d$=WTecn&0A=T`bD
z&eJp%lJ1K<E^$7%v6y>~s+rWLoM}8^*(c!Z|KkPRxqqHXwD_q1hQrk+llwXUSFZCN
z!91}l&n?8?O7e#7EtTO?Tge%*(U|wEOCjrQuV}FUkEy%{)9vnC6fbYt&8@59&b?Jv
zl$$Ntgy*{8Tl2a@BHYZz4^0=}Y2{{Z3gQlS+-2rm&ZwCY&(2%**OEIvMa%#chB;S0
zvd=dY=8<PSC3tyT2bZF&v~=pz)!fWk8>MsEqc}pk*tv5PdbvW}`FW>EE)(=f*e`ZL
zWD=LDTDFDu23Nu2nU-8NHz9iy3`96J!?!02HDsRS3A*0TbM(I$IE;>5xeLyB&tAOZ
zXf~b8S-1L;;PHP#JbX4iLZVLEW>e2K$gH_k22QISXJ+xZ$Te_nm~{mlM$UEGJOP?r
zoKAveA`^d2;)zcY<=v56!#yea702;LF&Vo)ZmmWe6W-h-YJx?Lg62~!l(^5l(gNFS
zvSJ7KHrEy4d^&~cHRqk2B%uz+BV29~N4VCVekvX4uF5lY&I2CN=^b2`{@C*@JO#Pa
z>(Ze_i}DM1c~1A8;dCneD5&y#J!f9Klw!w51~vWy7QQ!;AzZ(LrkTb3u#xvm>$Xtc
zwVHRqsWUSF?rh=7-SGjO*5Y<-=2}q|V6G#Pq4Y1^lI!B0-P}%W?841F-WG-_6Ty4Z
z4$Ys=qw@2Bd29P0P2)?ec)02gSd<)`#BRnb$R$31s>Sd2Ox~T3PV=b$^WZ%spT-qE
zwbR1SD~`AFkAybQ*IV2X5|g<9{yVFnAk!)M{M8)(i}M3C6en&K0mUl_R~$G7PUrF_
zvow6}dGV@V`^T>>J6-N^><;clllO6-+c|?-A*4n?_eLAbO^fBsRh^wYT^^7#1B42@
zc+RYU!K5#j&$GICCHJmX`I6sN4he1v-^**^qYFN(nOkr^zxy6F(-Q4E?k8Vch5er`
zH(H#|#nYw2D%jq7NkPGSCcpXAB^L4eZgMStQ^X3(N-eJ)jpTkGWMq-NuUUgpwvW%~
zb}xU(&L;v-yz|UgZT%)Rw{iuz&QlCa<@M?564`yPLR2DWlEqOwA>Nx4z9<+wT$am<
zv{RBXjS{VFVqw|&i<L*cX@dD%9#cNuC13eO)~oP1?p!1+!MT;EW6fQG<~GRL$&-BN
z^EyWOf%A;iR#}UxS68{U&z18V+@Gf_aaU8Qqhh|r(haXUT^UMvT6%L8UT5lR{Y<&f
zBU`#!d&kmt^Gh=oxRO{k1-#$53)!-Ii|n7|1)h%st^e0tFo9o=|196cg`0TyEWE>K
z7-uLT+jf_`^y+$W89wXiRCDfgdpK9C+VN<=F1FZQrp5mz;}Ac4e*+iGt5e*LQ>UA6
z{%*}zy>v2HL~RCtSLA!X?`u2xPTe&YzHE}kE_X$W$BQx9LTTYe-njQ}3Qr35i^zCY
z@vL_L&G(~aF3;l|aooo_S%p5yed2VgYLh8<jTYJaZ5E&2C3Encnvu^-zGdfMfY~2@
zi}039B?&}bo6luma)w{!*e(HPxs!b18(UeQmi@PQBzant>p+5Nfyrxr9Tg`|C&mxv
zS5$xSSjB9UHJfr&(0Sqvf#6rO%{NwE<ZrI=;mYV`6zKMwC(tc5UoxeIjZe9@hxhZW
zDFQ_&Ckk?W5a6C!6T{oZ-^$I(pvj+edzsv%e~J8i{FpVrv+iau7T9Rvb2r)ixiTxy
z8+9ImTF<pyW=hw&H$TbdmDsk#;)KjczG-QDxPSkh!{^l+%(MKNlc3sbZ3W+aK2g@<
zO#%_=`(!?AHF3RK&Tj!q?*^+Q1!5)^%c?%L;WnHk!7o=Y#P1&qIRp9P;R)>RIgq`!
zAhp{5_j!$~V^wk&@bYQQpUov>d)i{sZCe38Ee(M`%R3c!F5M}>trW`bxvEd-iOLey
zf)}5a>Rt(`YKT42cyrdzg6Z6Te$f5DwH$iV-gCm`Txx1~bX@w)LHGY|i<J}peVD=0
z&s|7RcTq8aok)t#2eSuej;(LRa~5;(t6nJ;sNbanPKzM7eE#6Use12;#H);F0+ZVs
z1T^+>2zC6cldxJ*$;X^^j7zi8310u7k@{;c*c~P`<HAe9({o%Unb~&pMcXy-wN`rx
zd}D1e%W1Z@Fwr?;_Ht$&&jH0x;4-x~CrV^pzK)vqq7HMJh;IIhpfB7lnf!vPy;FF~
z`?l~|=lFAd%vcOg&)@P}cpkpwP_fy=Ca~fIGY?~vkwBA@EO&cppGeiDSt6V@lERnR
zR0JnYPUKlqc}}n(XBD^D9ez=6wG{598N19F@7eKO-gZi~tzDFhwM1PyNle#bZp$@p
zH>nKs^O8$>H%DL4n!e~HkBrDo+02RvzT<PP1#UIQnx84%ZSjWPoM(kJzvi>pI*X3p
zXMEiE43%^w?1f&4CGo$zxXzSE!iwMK$R2RG$+`yfUQv828B*^p$T)L4&$au0{8vPZ
zEhc_z;M-Zrz|;8K#o}mDCjVjXzdT_pc`f=H`S?m)znE{&TP7-Y6LKyh_u_W$O_fj8
z_bxTyG>8yq&57T|C$PSei!UukK=Rle3-+h;`Hi-_@yF&>Dl*QK5=<1mptxY_TvL_(
zAu`63nB=ccQ0CUneIyz?WfITRg%R9yKJSp^5_IBeohHPu@M5)q@*X*Go_jCnE+Do~
zg1_k8N~u#v9&pv%5(3{X0*d=fT^sr9K5Vjh{fwF8%l5_mPML9BymICO>*dufy`2{E
zOe|oR^;7%DyRz^y_g9xf7TW-O(ELBERK*e=(Eh*q!p{|dv^Waf_+G*-6`d_G?X($p
z)m#;e#(RdGu_Xp(u9xdL850u)%(P~i$yG&a%EitT@T?OPII}7RT>s1VZ{U+T+#~SO
zR!(^GyBq<_6&IzZHP7NZc937@`DQ=PRWs%Y{N`;F@cPxmbNeK}@Yl{aVmfJ>eCAxs
zEx32h5bD;j;#b|GC-8i_7MI`?R?WX#4hc8y$`&yH_Cz4Xtx4`+L^j7~`2=pz`u{UB
z$M}N`;`pwu_7$GM|BCC$wY|a&?8Ro=GqYs+<Ql<g)$^SekMo9E+#i)DNNwMJgKN>0
zKJMB$Rvv-yc9DhQd>kL<JQPrADB!l1J;1RmwpAu1QB5oD^IG1iRlfyGR*RT_`FMmw
zQ?LMB|IgXDl8gJIpaAInpM>p~xn4QF5Q^jA5!g6Qlk3e^FPYy9in;SP-{BFC=NE9;
ztHJj$VIhZJ%unv5j~guv)#Uk;mPB){IdE8Lrt?|O)s<5f`DQ1pRjOPTxO7aH`%>aB
zGu<6C<hi1sTO2fC5DbY?l$929<tevxuz1R_lPmvlBHvM+Ip&f}W-4v|V9#x%a)6VE
zdAIO0>A4mqYR84jlX<xkC%@%)k$7lcKCepa_Nj{ky&a5})3x|`w`lC=SJ-^pqUiE%
zUR_B(p0*G!LC?;qToHWtE%;|7@MdcnYR`UkkLy;+I)S`C8O2Py>4IG=R|u@!7ovV<
ziz2w~4N6<DbOgX-A0FM@ns**(2}C`4FEGpQo17faP42`u@3>D(T;-b=^-YkwVmH^4
zwvT)pT0414taQO`wdaer3n(_Zb9X4#@|<nx70|SvCXqj>P^dXXTQFrpCAj`?`V}D1
zbL*1XxAPa-ee|6~6#vaI>6&H6Eo*j8Kw5gKf^={Mzn7@DMMK*R`5KjJ62YHxE$@V1
z;__@zuuPcFsG(}Tp7%hh4FB)#_XOU**k}GXd$!2UA1dJe|EGVQ<*pLhA@*tOD$!5H
zS1q#MKjpc1T}N@-%7yaBwni%XEH4tTY+~W~EiseZj=#hrYbpo-?PuY9PZu5L{-$w2
zc(VKqo}YZr1%zcqz~;TI3gN5Qft)3KLxRJi<?Uo{d#5Y>!f_({n{xgL1;2c0(fDr{
z2hRt8o?9h8@?TfSXg{Ci#>IEmNmsV-jk)HGwOnGW*75r@Xo(!&+alJG4LM_7Us9E)
zuHgv(zQ+apE)(*&*ZeQxpT;RIU_A3W_kwd%C9gK_<<6GYG4s7s!zE$W#lJG=mBrJ)
zOZg%%EaZQ7!GQC?Tutsat;rV8FNyL_vts2|E!o2-^t+i)jpsODCDRt+WwH}^To?Iq
zHyOBCERM6_Z)AL-$bQ#Kto)?{k4T*Z-;6mSJa5a@xdJTK3!Mwx&+g#mC&TLfN+hZ|
zfmezRvd-Rm#S`AgQzXITAoVY~xV`Pzg%X}5aju!yE3lx<Mu1zoTi{CD9?o^0CYD-R
zRierXGexshc?6_q8gkn2;IR0;>MCEfmxt`}j}=19n!<vfQzu&3U-1!G{c$PRl_NU@
zOfBCE{L@XAuGW6QtM@~c?@Ld+z}H8;LIwU@+zX;#@oxTfnd|I2JAv#g7vxWChY5Un
zaY-xW-7YRWod*`_b8neDM9<>=Wp<rEYP~zxht&q$l~-4DZ!ceHv9>f?pjKcOx6dOn
zAts%_ptCfEGS9XsoVgGp_ObMVz}XjTWlPueakAMg0@wcyU(E#O?C+8@&VJ1`rM_RF
zaV-mPa5yJ8Jy-wz&aQ9;F<-|2M@As|&^<LvO#|LrSJ}8za=a{4-(TjxxmjEw{Su$D
zcDcR4>=-M-`Ttr(@7px1Ui@@iDVvE!t;4KMQ+8dm`6bpqem~i0F7^|Xq!-Pomb>n5
z#Ivic%Hn{b0Y~++?ZOL=zqLrHl@-*zet~z%PgxzaHEHIof8wOn=dI!kyl*A&Y;ObD
zU7&pWyZ<4_wWJx633>Yk7}rk)-EAq<U&k-$)ltW{sl|b7!`m+K-e{0or`!+b+gbWV
zZY}E)wtjm_vUGkU|G|gT`Qnr11b#LDGnapFYN0;U#KNk*m;VeC7q~AX9i1e$V@8R(
z>{K5M(|^JI-BMxPA=eZILu+R9>TKM~zpP*)ci4g#;B{Fot;RgzMnS57w|wCLsdto{
z+hUDCvG78!brL+HKWYL+HHFs+pL#1Ncv*TUZ=__3P?g3(F6{(v(eufC+@7Ut%!_Br
z^L)?UEIx%Zo->qPS^Ddoy%vW!CUT#=zry^-?H7D?$GdboZ|L&4osyEf{H~8X<?&R0
z!N(>RYLn9}ZYwGA@bs+FYIw?Ed1mz!etUr*N~NbdL>Afl34AZ*GQajWh<|<cWN`g&
z>iSZ^Vu^&b#;nWyJTIMi_U-NEuVbjUcp`hAzm%zmw|>WU3jyCv{Pk6vc>YY@Zjroh
zHE;5@dFFvDzlh#@3%Min@M8{crmupUcZ_dx*<`7*Z~ZZ!x3A|J$5P*X0nLkX7W!*S
z_#aBf3iQ5JR+=dMNbvi~SxP%L`kOcZ?U1=3`c;9ik%doZtBN?!IWL|SvCdq(g`P;(
z?Rw3vnPkt$?Y3Az><DE4fJf9>fk$p?{EcV5rHw0Raoub>39tX#vmWsuxOT_FtlyqP
zrMQuw_tbh$&gvfmcRq<*2HCXnbYEE{dnhA;`<d!ZZs(wM4r?=Zfx6?q+*5N6c<hbE
zx!Rn2m1WZe1Z)~SxhDpB3N-&@;$EQe&O%w{8mHzBN%Nno-#I5FHVEV_+-3g%VTD%x
z+C+g3F)R5kbF;zqf3w?7zGEe`1uD8f3hz+x5!lrDQpVFQnXh41x6JedZ`j|w$P#d7
zh!kKurpdFY;Fxej4v(ZQr#`=pvyO!_-x=YbCwuwVOnSl}_gaHnDI`d%=fGAGIZ17S
zDURy}x_MmX8LPhYWWAWieSFqMj%!ID`91vB@ii=B6kY25hyyf^dg$IVbB{Dbxg)3U
zgVXA*kEglQRE~1ZP@gF?n|lfOhxbz4S8na%<TN@gc3$BxXLtQuf#x;R+-(i292LUT
zWezYe)GEEQgV)`UO{nmlip3OzY!24H;o$no%;Yy`-*RDr%3UJdD>pZBGO>mT-#)s7
zUud=?*G=DRGWxpq+~<87c^a7K@(cf+%AX`(!P#o(&S!U%&2rasTRwv~p`4fgP7^-y
zX$n{CO>L!ZiY022F_#6hlm793W)!o?lW<lz@??`m|C|Ma_nM+)SzgZN{>RH@5m9R+
z@I<VXKZf_U`Tybo6@f`pxvWhkxTfx77n!m|%kt~YI^m5fQ9LH{ar~>7ZMImuAw~Ph
z+YbV_^SCYjk{tPzS$g=7{^+pe+#$u=U;m$*vweoZndcQ8n=LpjCz?Lx;jC)azUqFS
zD{<{ifi{te3N0U-gjPyu3Ud72rkVXf6~3P}rHUV%r}GWFwN6Av@y|2+Do~YFEN`Z3
z&s}D*g6sUoblz`!Zi_rFp1>-6(oo>DfDlj6V#r<FZOk(SeBK4{?=?Qmt=PvSz?>c~
zU1K;yL}K<Efx6ck;PVxph0o-#_|j`4)hNgn9(7#Ey-MC_oyG(%TiH(n6SO=OaxLfb
z9_D7ZaCG;SZC|-gw0ZAl3xhqn+(C>5=5I|ttN*(b&$}f%kN?%bqXOb_Gt9QHO%eKM
zKNCDQA}g_wr?^EzBxY`ph~L}~=Ios+Jh9fo3YS-D%N0FNR9t#I64d`^-hRZ5+x^82
zv)(IVeA6!;=L@Jkz{RaIQE1;310H_i^8y`5AZI5}X4%Ob-Y^|p=Dz7}GGDlujeCbf
zKEKexE!uTSErR~JY!(%t_1HX(A9DMhIw+rURY!AO(*~|{yDhXWS2LQQ_O<6^`^_(~
zP<yiAmOTrF+gCvDv{<=%5Bn11HvXNxSNI-TCi2Sm-Q)YeQ%)esik~|~<Ci$7%wFop
zWqw=OhKuX%7am*3h31Q*x%o{^Hu5*}PvO`%%Z7{Hk=wj8T$@)*Tb}FSB4_?Xsyq33
z1ZVOc5Z45cg?QTcbM4u-#oT-1CEf`y(&W!XGKyrUR&ZadzreT3@dmfak_gU;LVbcW
zbfnl%O|z9*|M#D8h^`>7)@31Z|Ks7eEquW*Am;+;zLDi_4V*2|{z8ZAzxyn{md@D%
zOWGi39>&+&TP&B85<Ov4DDv3y82{>~V2<tw4Cc<$dwFxUnPdgH%mo=*7zJi(yf%C6
zc$QyyQaV@m&nkX>tLXyyd=8S=)Zg(=STc>5>%~_7?>3DB8XYgVyzgG(Y0v(`d48fP
zfA_~zvRuJd{4=h|gU<iu@IAK0{LGpqX6L?G@?<!F=3oA{om1)Obgoj<OrCQ;7%Z$0
zvhhEi>CavGQH{?bU7LI7@6!UEZNc&j=U9r!rLPna*>gjBG3Q+ldkr^m*>NdWmVZv?
zCmChSG%ij3?R*S;|9LrnUjdK*-Hi)n2`ZBVw}(M-GIP^^-Vi<p6_3m#JUx-R+-bqi
z7XQC03cNCr5m?9LtQeEWC}1ip$rF9_rQmI?U{#^^Sfz?DvsB^@g*6?1&N1J!dIPxs
zzaZL2YDU{h*{i1(aI?*wWEMPc4!f0jk<g-?e2e{&oPxQpI`~}G-L)Uhx^EiLEg-&H
z;5{EhR;0kX_B~*Cf!y*(bT`LN|77u%3oi?V_cQY6ytyu@QgB(^qnm|q`W}AHOifvM
z|35`G(!5V;m*6yKX~BTPK#4tF7x->1Y~YJ3u@YEUw9J$tKGb|?9;caQH3RRl)!yJf
z)#V!uA{sYWsH$6UFtcZ><3D__jmx=&Meyv4`8<>F&f`1J-OW`{ybav{-)=RZd;i(_
z$~Fpr`O^$txmny)1!leS<+iflF8qE&r?3Z8qtJU_3BkGbJ9v)f*9aaq%H>jd#w@bb
zHJyvcz1;NR*KF>o-u9x0qHc2X%$X{s^s3OjAgGONPj92y#CwLk@AF=0Dwj^@esow}
zHb0Jo*FR(rpW=aDGu9Sfi{|qM+%3Ji8j7~F%&pX~^4&Qbtax+YYQec@Bl$(r-<w3N
zHs=4l<}o-A%V%@&ak=S9?kdm_h>_^#2|8xOf0ui;x&PHXzUAW2IHxgqn`dv#<gbh3
z=h>v>WZq);n|J-hK=ZZx3Pg0?i-7Y*vE~D=hW;eb_&+E6-YqN&FLU@#J(J<G-DEEy
zQ~SaEZR{leD3e)yp`3+^#_pf_S8VB2c(z@}RNz;uv`hYcIo{`5+##u9qRk#txR*`Y
z%XxemljQx=YTQ+IxA^81%@TN43fVj9x7|)ah5G<sfW$XR)<;)4^`yeVW80v(-yy!0
ze;(IW^IeCV*|nNG_+`Ug*b~B41und7wAgZ4kEir2r_6IF7aq_#7~6MyvDG(fg3kYE
zYv61GhlAxO8^sSR3<UUZZ{YI(5H8SZ#mhDE`XY08;dstDGvrMzOg3`1<-`gkmTWX#
zGV6$j$EO~FFH4vN)DFG_pa0W-d@J8Vm39H!$K^uy&!YsKVz)^;gm?1&ObwNe(wM~l
zKDbxFRjXWJb7%v1=LLJAbqRMwV<+nHO?KtCxbA;dkh98^pL@$0{@JzLIhzv$G%mgS
zB2;oFL_oIlw1C@+Rk9BjR&%zUe#-T#X*Ww`d_KR!R~x>`6WfIzB;RK|S@}Weu-zro
z<yKRr`P{j{X*KPyB=>B4aW0=7MpB0!i*eT~Y~Ttin!vq%hOCHs@JkM*bteT>o{Dn5
z%jjV@YThh;YeJjmML8wj>lYaW=b215V`kXL@-d+gTt7*#*}%cEga_R3`KY49x%c`5
z!Ge;X{9SLCa5_}=NQ>Q-<4RV%!hO6-oBvZ_0I!d-80XGvWu7!QQH$!|O#EWs4sb|0
zJ`hX{ui)fU-=NTIQmZQNcS0a2yPR|FVOO*0{*Q8d%VjL4oYEKg&Fn98=)@K-&xK3O
z?=|k?*?m!*@8bOFW{Wm3C>>=L;`G(p%UNi)O~{H}(8A3@M`+s{39eV{C;8v?nwY0-
z&(m6QXq$j=RI^3;-6PzmIM?tiY}c{q<~+s|wxF3iJN>o5ndcRpH-nTcrnJcO{At^$
z)%p4!=f{7G1QwUZDx7YK6+C)XUcmgOlzQQ*LiqT95$hLlc^5fHUiF#E0luw;KlxL|
zJme;Zh;!A>YU7@}#hy>&T8ChpZXstGlNi5ed=>Yj=r!OugUcsZ3QR8l&E5KhmwQq~
zzQFBE2I8xD&Io#c6&BpEJp??yVlah~?|Ilma2vz->@D88hRi&7ibZ(mB%ATuv9*KC
zlVo0Ru)K8iBR+c>FL1uw`+PdQ?Bn_n#}|+k#$R^QoyT`~F)t`jz_8pYV?L$6CZ1O{
zLVQn!_waov+Rv%Cayi$Y>x=j}lp*IwW_Qlvt?*ai+QqYkN8!|J-prU2yc<IV_%%5s
z`JQzg1KSNUPkiGNaK8b>?rF2-`?=}~&$?m>-l%>_j!8OKdG&gy@RZdY;qE@o!xwVx
z7N6#tKCoM^H=f~L(Za&JOj?kad+%Ys+kL$pd#>xi%>|kF@Zt>KBkJ3@5<}H_Q;alu
z`BUeC<(X&f0@Lo6lDsnS{&7ExdCWfb)kBv219Nzji;jWKKglqUN9Z6ESlyk6FL@5E
z3FPzpzmE6emb2WI;gIuSe|ji@)pn(-aLriK1G17Ke&bKx>FQs3=gD2<6}7trE(=a)
ze&Px{tpYX+r0C0g6EM3{OpGt-qAHjT5(nW!Jtx5NES6Eof5t`+95y>6O1KU5Tlu_v
zG<jm>rt-0FU&?vE{~zz^Cq>+<+_S*?`}WE3{#d}wmr(kFSBuY+N13mj<JB#HJ`=N(
zymKzz;*l3<<BgHv=6$q|kL!P)Cy&+SXfEFw+&tUYMzgxk(BLY{5$2PA`4?<o+lF$k
zRpnd3?1S$c`8u8~=lL^T1nk#6T<tva&U(DTb0FuPao?E+77yl_z{juA&ei;MHlLiz
zNnS>UWbUnrkn>%)etN+jvQm!cNPP}Zm^mx=Q{4vchv|?z=Waa8;_B>x?3n`jzv^=@
zw|>M#-cTK0o{g&)@+_*@$rJnTE}xK$4`1^2B5v2N9<aL-lNvdnA9vt;@=%JeeXAWO
z2WKPu>NE0uj#D6Kb+S9zgTp2ItU6!Q0v6u8ypT2Jo>~elcE+~g^bWEw_<A;9eccJ}
zD@=2E-?3fbi~Ldv_76zTyKyB~^|p)L4pls0^;2UN`Qn-kz~+9vc!z6N);n-qfz-Z;
zNCfAd9ua%qTYP_cj~cJwIkKvkJM&!+U&x{x+<jV790x_`^QA6O=bE?p0@x1LKZiKH
z{>=rOcXzHh-&_tO-ljuAV7vBzpA6prFE3*NE-NO;$@1N@oy}Vx^o>{X>`vah4@<!M
zc-Hpu23}U-zPaWfujmpxex_SD*+l*?<BQIR+%4Yv?mbUwk0W1S$0gorN7KOe?#@sG
z+hNI&!h81kF`n2R8N83aaB~0p#lvSY+lDWACS?ELA?_KRX4;oH4mgSPT})ojla<WG
zudUO-dqTs3KR9hRPrQgQ?^fnXjKX&&^Qg^%oFDUciX-R5nrmRUfx@MJ`&C}Khda2|
z6ua@3R!rx0+8o5Eb=i|w%WNXw?TZt@?opot*{hIXaReN<KQkkFCT!^dvq5ISF!SE)
zd{_LY^LEaC1=a@=FS{znH~;Ws4vifyyd|8Ec=tak;k&t@jwLYn3~!2qDBl5#W}fZ@
zN#37)3wU4mv-1@f{^YaV%?fs3+VTgyD;<9Gv>Tgo2W?M>n+Y=a{0T_-RfmXiT~Fxe
zEBbec+vPwDU+G<AzD>1Sd`qOvc=NR6xy=spg3X$^VL6z-aOy4CoMn-Bc<1HK=PUAx
z<Ei|d%l9~2oA>(^55AYadOU6MLOjL1Q9QisXM)vk7wO=+eNCCaG;T2;@1nKfb!Q9f
z`FW(GLij4C$M6-nNAs1ul;T^=Y{h$hl_u{kNkJ}#+!K5X%jWU@e<8tF{OTp&&u23H
zfBJ9p+|t^|=ekImeO=yd9xoQip1u#tS9m3EPUHB){h06m0Zaa)mAm<xW=>*F4P65c
z<HaW$c$x*ygV{$MAm^vfoZP|_U9H5o_RA@-8+T{!;$35-#&g{tQg-(981oiBHRnDv
zYdN>Ol{)Y9A0GUd`1*My?{IKUN;3ebum5Lufy+Noc-~=o$6a0rS#LMTb_$OY*F&(m
zDZR6~r!el}+Z|WR_wmGIZk7FsyoI{Id7Kg<;R@2bV!92_hMjjf?+2}8>Mru(i+^ni
zH#b2yjW^ptn%nj29KKcgb=*PUxABO2E#Wm%)#sNA)ZlxS@r=8*;V|!^2TRyGeoo@N
zr9FwKE0&2z$GM69((y&y{573mf6t9Sz|)oci!WZXgnd8DM!xukiG1QOocVvh&*o2_
z`i$r8rgF|CBUZj|iU+{!`KmHc@dgR60Q+;!k;(iA8z=Cs;o;<C*|w4AKF3_X8`;Tx
z6FsZBZpP2z`L4f_yF&0OIDLTp<6rlc14Q?m{{rW25W8o86c3N8D6gm?WE_q0x&zqk
zs_Cj+PdnD}y2=NG{XE^_1K+nzUYvn_UA$Jj#{7rXd%^B`|MUo0J;<yZf*t&Ow{#n)
zuDZjyFV9f;j>#TFwl9iY8ZS=>^b{~CL>~Ca_m)xDf}`=5jFj41F-6(K7Q9t`Tzen(
zn-@O6q`tE_k#F`!8~#T#rwCkWVlY3gI8(^G>nync&-2Tgr+=}9u>Pt^A_d}y%%zv+
za(~<ztPo>!N%mIR5ydHf7s2hE)NmHAs|r8NMB5hg&3QG6PdKKX(`>?iA@*InxXVs%
z7TC|%4>qsV-Hb2vDdb+>w(5HGIc-8*J9v2bqoTca3b;QB9u{M@(C^sK(sz%UJ9o<~
z`HDIn&27I<bL(4t*K!QvH80<{o1=2u0sg<Aq=Y;oZV8t>hty%9`~S=*`SZ{3ddU~n
zCB)mzwUtlp!gKz0bH#Y7|CWRM|Ke|X%pJ>ja};;Y=gvB~&%8A8E#K~mbNOdpUdXAo
z`v7N4<t?)by7PE-9?#-*b!O-Pf3k`1TYe_r$Io^`?$5H=K59g8`J^5*SE$>;TbPip
zpnmtR@HF>U&S<SFzWjJo?yg2ZF7Ic8LY!IJ?CPBdWSTzn3oBm`;Wg8TocS_Y&Vx@$
zW&=3hCe0G!dHH*@K%PSlr)I}?KHu6*f$)v-;50M!h@-{v+Z#mm&-I8b=bXV`+;y3~
zexanf+l#r}!FKGj$6c}nR==naXzuAWTkW}nKX2uF&Qpxv`Lep31xyboOPKPo@M*7~
z$IJF?EB{WmS%PLB+qh1eYw>Q^tl)}H`onwj`VraxMGyGo_ixacT#?Uu<OjEf^uLW}
zoQ|8gS@o~;E9zHsDD0lkDHtTr-JHQ|VRWgL?~nFM&SiFc__CE0xf3SF3W(R_$v@$=
z6e%^CE+BL?N?MX}Ca29-8*@;4d{pTn@QVAe%p&%^>^16Td~Am}d37WofYbi|yILGq
z*T#X_AhpvbxALwmKcpNyp_=!p{cnyhyPue+v)<y5l;9FL74=m<c5j?Or$rd|(GAlD
zqt0}yG+xM4D*I5Z{NVF!jfHp5na@61%ddBFJ1g@^c`5a0n`B)ow{o3P`eg>%|M&OX
z389V~aTY>lF9hE2wdK9zHA!1#vYgq{)33!}+pOdJ%wR4Mx2zppkAvK@%`J%|NRv&B
z?cj2O*9#i?UmuneI-Hd!el2S|&nZu7PEngSBak`}W=+sDKXv7r;Ef5bg7Qi%;^uzS
z`QkQB=KIAYB+%sc!_;1Fo_XDx-==dK_;^m9mk0N$Zhn#$mXYgGIp`Z_R&mRQUt;?t
zuIqEo3QWJ~&Sihfjn8bE3)k1Sso?U-K>9e>M3rXc%BTzc8B8u*HcUJMzrV3@Ni5zb
zY+$-n*d^zoQ04+ALC#h0xU0_{6}%F-p7VZ&naI{jj9d&Et4;5_?c;o?w_D`fzg?W4
zYHOtSsHdAR(`@E?KIMd&S<_>loN6&G#$_VheL|eFTqh6nnCPtFFG(si-5+IOQT1Ps
z>x5C2#<6Wz%xm|C@rjmYDG0Hu3oTgQ%D*%)-!#2jmp><a4Y+(-7s|_P$FW;tewK;g
z-J|u~MR(rv)#dLnf3eG!&+sEZ&%5S0bI)sf{O4C1aL+weZm#CV%gdXTXr8<MwaB}m
z<KXnJeu<weW7BE1$(BLvbqh{0pLwsq3+n&(>GKK{eN?d6`89^W@`pQbxCf`AcGOaV
z`}vNFC973U+I442-@I`~_D(?uw^`Cskz?C9x%C1SIptR!k$7IG%oW`|n{V>#3W2|G
zyTSInUZpJ%ekzzxyMMK0PTeB*ON+wb{r@)^Tlvo&kg`y^d567yr3SzL{RWPJ1a5(K
z@1|KW^D%N4M%<OYu!og*`389|(ER_s-Rz+LKkLNbeBkus{qv2&-bJheeO*g9xp|ER
zw%R#!^7`<Yzj?c#Geg<Z^xcQ8?8a-O1k#?LFm=k9tf6HQBM@jlkALrxpWyy~3P&Gb
z<AVkPw*~ivYWi#i9F+v5?0TK~CK*;rPe1U6t?^lcz{^~3fylkDxt6?GBE-;SC3@QA
z8}C0w5sO|KSD`-%fqbFnbNL0lGr6W0>1Zr`nJp}HUPWM1^c;cipeM3s>E;|hclUC>
zvtP{2{NW;B{o04T=eJ%Gy7Ebk(=l3Kc%I8L(}}+Q(sj-L;5jaZ_N`n+$M$nh`!-op
z{GuS&h555M_c4Cw?06_IoWjq~X;8O7AhW8FtF@+u{hrx;=^d#{H7|c;=4BQ9EReOb
z#cZi*EJuphRdE0RH`iwN^vvJ<{u4g2yuFpgF<Zz`=;X#d{K{*aIj_d|NFTFV#<^=&
z6SuFEG2fpcR-QCYHO>~JDcpwD>K3*U+xh<T*>Q-knJ38Mn8^A2F1y0*xkps<XKfSE
zel?r>u|THTh4;7QUPP%|7`a{*2s#igQyr<w<$q?oxw_|aE|%9_d>VI`n$1n0rs&VY
z%Nf=b%`VBmTgXp>-9n>zve0H<Ca%Pl)A_%jmN##iEu^_lyHX&iD%E1)k_4U~lPdXb
z<pL}~<Nv1NGq_%zTQ9)O`G<4eE*p#3!WZ0&_fFATJVlUm-`XhxSvAQDOt01nJlp5S
zFYd&rW-t2%Jl_o}D>|P-_TOYC2B`C9J>+M8U?-qh>m+y1?<3bLsXd%;PFnEhALbE^
zYbfFp*f)jmq0kyG@ikY$ZHDSCodRojBy;s83UfXAt0ADu*(|R1=dB>?hZh1hKRChj
zD}S7L1#Zk<0v=!K445Ur?Y5IIJC>by!t+u=X7j7ycGp}@U;YzQCUa~yR2O&~qAv(K
z|M#eMBsgDy-1y7?F28Y*o#3%_VScA?#bC2Q;vg)<JxjpYY$D&GDmeie3rD_npBQ;p
zo$cTa*|&`E`|tH&Gl~|d@cz$E<&k6N;1~Ymz<+;L6tB$6g90~>AL6Y%5e%;TL1ul~
z4ml$R#J1C!A~4_m5D(k7IQ|=za@<pG?F3$#=<_a_;?6xkat7~>!i56g4)=rI|J~*U
zuO#P2z7};xp3d7H0*T9*c$nv`<~wjpgB4`f%GG-~tTnj>bZ_VK3-O-e4P$WUD`J=-
z5I>!VRqYdG&V9lyExrr?A93wy4&f<qQRiaeXymt0GzI(hW9mMhXYM87y4j#zhUa<X
z4t_2r3w{GtDgG`dNPqB0h$z@CM{7Izc0GB``LgCY1Bd-Q-q0#ne#vdKcz;ci2bbNi
zqiwn07Z!re1-Z?xvjc3dpNbkk*9|YY_^MbtuA;BEz~zTvl&GNSc}Z|sZeF#D@8u#9
z!Nczx_^hPO1*9Eixt3SU^GDt}!oA6Y5v>1$%0K>RyL9<Zm<0+HUCie0G`Hqe&r=W#
z_U+==e9_J)_alz4oO>>R@`uAb{M){8cN_}h{SoEIli<$F(<*1otG2?M|4la}p1a>}
z=b1Hw6YRH-)$sz|wfa1Q&pv_uq8U7icU6-ZKeGV`SbnQ_JeZD|w_ZRZJePN8y}lsh
z`Q!ZN-<!C3pG$!K*)_R=bCIkI-|St!JhrigykTDzxDVYv3^p%PZY!sM)<tmL0ScR<
zC9!-D*o*l8pF7Kyw)7VFJ}G1V3wm<|IErQoh{w6{?wG9r_V27mUA!v2&HUSztl>Xa
z!O6S)ohi@02_k%KaoS+{Ku&YP`}_jz|K_je6@1Xk2kQU3Z8*lfs&5B}t7EU=`$#K}
ztI4l8)Vxv!X3O{Rmn-M-2XNjKxLoG}_7BJ{;{RuHeP*1?b)P*DoL=M8FY&Vltmj+z
zN|iHtxgYoUie#`KKx#9pzk%cJuG$Iy`|(To?rUrDZ`0x6KjpSd;BOEs-|EGT+($VN
z@{6pO;^oeSobB}>l99Kq`T>7RpeRRu(Fgvc3s`v-^IHUu%C@ja-bm!@w0_7D<vjr$
zHh~wH^KNU~!|RmYC~#KFls8a#j=(*SRcsvjpZFfk2<4x5`5m88_bfr<=lt9=8dnRH
zs?Gwt?dHr}o(P8tyxt*My!ZQ)`R0G%VcnXr9UMlN|NrJ$@2bH6F8(<$N9qc`Ss_n&
z&lu<k<S&Nozx}VUm#63ZQC5~O?YuET-TdFfKk#QqGw{zYm?;pV7AR0*{*FI?+cVA!
zvIqH@=I;Q9n-Aj*UTd}nfrX6!*%U8%u?3%)$N%yAE$#zbZ}N5gc*(=QHIo0U<7<AK
zC31W*t0e_E)z`5*vO)G-#9s{tr<a$PDtPwE)`QbNC~S53{)79(tKzl`JU*Jo_bgKY
z?EX#PBH8CWxynDk&x&WQ=m(x%lB#?YHXh}F?01TDrO!FO4S{QU1um8INMzjRlbW5(
ztMT8HUud6{;HDF0V0->NIVLcF`XRnw+Ij*y@2Bw;tG9sd0oj?e+g>2M>l4e(6lc!a
z7q0P7*q6;6m{KAj@>@Y*$HHU03;6g1`acx&bhz4q{jHq13{?NKYaEBn*L1tI@N`^T
zDIm(7#S?0HM}Q%HI*;N>c0vDnU%Bj0I1B9jp2^*?=DEN$!F`MZU7oyaO4tQ|>*Wgs
znH>Pfx5rBhesK#v!6%hF1<tL0Ai#Inn$KeLPQDME+5*nojk%->cm<Cf{wa`nV>$1|
zNy>sAy?p|@|LXa!q+b*$+!D)uX6s+RM^;J#PebRkT|Bp4AfR$K?_JMJ0!!|C2*&M6
z7f8^Y!O5^Ko5#H)pS9%~GrylTq)&2UDr7D8{}U{Hha1=Ps~$E4hxNKEF}#nbb@8mz
zJO?`emm~20PM*pm$=p+-Z}KfQKgjJn(@Aidyf4q~8D}|HU1Jg~nxeyz)DF4p2z35$
z^7H*%9<!%_{dRwb5Rd+K$k|6{p6=kTh;a~H^}bZVdqEa=Eyn@@MWY^`xIedezbwoH
z=PCJ)B<?9347^;Czc?xry!lW5@B*6yiYMmfru=u`GVsT?`U-wzZ{smgQsifTVkW?{
z|GL03p=g17DI54EWL5FXW$5uZXguQK{_~2LV<scdEpvCy-8%2Mm-F_6%@WTt7YMl&
zDad{2Cl_yvnSeOEi@>LI&jrk`sR&yC>*rY*bDU?*cP)Y3d_J)Hg%A9AGa?hg<-+eh
zg@UQiQurn>_|I>pWz3VgU0u+P$A~|oNR@v>vj?Bl3MXF2dP{IPg510&1hUuk-;-6~
zJPHz<YOBY;_|Pq$s{!&nD~u<z3fTSutKYJFHgCnsDZH01p90&v>c6Jor#NjMu8cO`
z^yyCo9{!&L?mHFR^a}p-Vga`iG}tY<&ly9`|4J8*<K8Oj$?2rY#Xa|$KR3sNDd76C
ze$Fni+Tx8OU~@rkK*oLr)41eRO1R&7WrE`XSx(^eS}yi8%h+nQ?sJ7WJc8@jUFFCb
z5*Q0M_t#}f?kQIOoGdMnb2a16eC6t}KF`HsPytsDGSg;pB-s5RHVD7+e8qM5^#c?;
zk4#&~{@SM-tS?|P<oxC9b6dHLqknNc=qzQGTGI&j(~S$y;Q1G1)-q)d_P0}FIVL<`
z$mXN?kMr}9EU??o=R^8<5l^kS;`|}|q(SNhyl1hkXH^8dCww*^_r8V4!0t%@w}T_%
zNd(u6)oh$Al`A;hr$Wy9Sa$|;Z%qbM3)ozc8O>!4+zelvxGI*d;=cXQ53HZ<=|zsV
zo?b8;tY&XJ*bWeTa)2{j9K?s=$A1uWF(5J3Hxk@#K7Y8lE~;^#tW@DhT>c$y21p)+
zxioY+Efh_;=elu&<!;H8a_zd>#`!){gmd!pe_Z8-ZtSyG^s`QrhMd_jbF~EA?%6)F
za6f>|QdhnSHsfC9b*`mTcXFy!oCd3HHax}ket84iKS5S-eB8Y&3#LJOK)6D(jQf+0
z5+`5&NlqoFi(oz9gmgJnq}#aO2SdU@^{@l$qel#2eM_&YaNN2KIjaJsAA}{B?PI?Y
zJ&`j}ZYtN|wu2lzo0~ayT!Gw~p(zI0GYwMLCXmdv!$<<`P7wd+=5FpS-N(RbVUHI?
z+%`Uhn|HA+J81nMNDT;EocIg28^lJ&pMw{2XD&B^%LRW4V_PBH&no-{vgh_012dN_
z2M0Ku+Gmu*?FE^!zm)?#4hLcfou9_#l*|E6i<)BlIJ24r!ES1kYz6BTSg{BkE+Dmr
zd<<Z*4xc@oHMtzzQ?K~I?FY%jFyoCB?nhyZIJ3X^vl?F%=6>(@9Bjs1^+Imbe+FDX
ze{A78c(VkqcXR(*u>41DZZ7dRom}e+wYU%GL(U)Y)`Og_yTQSNOYdL<*UQH3U^o12
zIlz6giH+xrtP_|IvU?q?GsipO?_jgeygb3BbAJx!iabad*howT+qZV}Vvg>!Tfl6P
zp0!elG9$^yjZ;%jfJ<@BX0GNSCazaH0${TW^v`o|TaCCk6Qu9y9mu|@zgDREA0!9D
zSveK#s!wgWpU6PUIgr?r1V~>B#P-rL<xD!H3Jz-!e`Q7_cf_)0uzj;u>TsLR^Wyy4
z$_VCz<p0fN=PWXS#6{dWL^%MG>lcLd<9BakW=rko;Fw<F!kREOn0v}9Auer3R-T1R
z9&pX6zQZ*q%NU#v_M|Tdo7?nq8#o*)818YNdu70NFYhuJ<J03D=hHuc)k)YzfYZrs
zj$Q1Y4imv@A06=EzPIQxmsBAn--7H1;rO3IJhD%P+3qsRa~IG4#U;2+o@>S>Y3^O?
zoH=?DA?N8h@Z@k$X6xswT(Oqha{p$q9U#4*vzfR~JVcxa0TNsID4qMfLJ-)UAbtxI
z<W6}IJ8uamIL@0_L(a_tiP=2b1Qr9apK1Nz5@Lj$Z3N<L?Oee1hM@v%--5bp9IJE6
zxc9!~;>@UJ;wZMt0^704HUUh7)R%KxfYpN7KF@@iW!~$6<z^kd!0G<ffNNPhFT3DO
z70!K1cHAlxx;VF%Z3c%8Nd2u#dEmSRVmG+5avz$GIP(c4w({cyc75H2+{&4pa61|e
zCb9qc`HC||{xX-r=O<vhKx&Q7z2eOIzZSgzzq?DGYsxJ~u%0mYXIx^b%fNO%DCy*~
z2%ZEMe|usJ=bKNE@}2!X<UDtfIb6KL+>X1xg3X=YyoKwekS34+v|C)Qb2~Zr3Paiy
zAa#q_Hh|5ED?0#Bm)&k%@NfmG0pSPX=Qu5sO}J8mCBbH<|3}<cTDh#7`_UFluy}d6
z2H1^;zv{SaqjxZY^n&n~*{{IjAa+M-5}&l{1k<H23^?y;dI`;O5;vAJVdJ?T&MIJ>
zrlUAz?<H<$=3f>i_ub@#PBn_pYTIvFD;~^gTCv>Xmfm-b*Vj9FjJBTT`&#~=zo+q=
zIp@UlLIM-z!E4?WROWG&1!jxNUU(u>{%ya-_MHwq>z^1azACkr|6#O7>2825c%9>n
z8!X)SKHN9w(Usu&yds6KNMSCw#u7*2<~kl8HK9%XhhIa+;CBD?;+CG(2o9quszDab
z$${JwyFK`3-J7TTg8w*wLn)(W!-;KNYZ}&a?+IP1aII*aR{Xm}E|)c~I&s^6n4i<S
z#rgeF3xA=atbp|7O(OHTAZJ>qZB=IzQ|siD$jsuolCzB0VAnI=Wm-A>;tj#<->gF=
zLE$9Pd(OQ3*)q=e9h<nf#V)j%urr>o>;EdgXBP~(ZgK4AmSJYGXb@86%8P2^xbW&J
zpJ-hm-)8lvJYW8w28V^^RX&~>CYvm@@|$?9wS^U*+x7|j&UEKl`>v0Vty-Puf;&I=
z;&vDQN;7Tl?S1O9-~7&tScabG-d}MJ+_!lbzL>A;669Wh76x^$CoVU6eQ!+WDxAZ>
ze>6l;z<pr>xNqUh;c6LD%PY1o?3l<gZ)yH(SA;o*(<CjzGS2Y4tLKtC5jjbqSZe{_
zYY}dXN{0$Q)n%rfrEc7O^DKP%J3sJBwVZy%^E@n(=VCY~|B|W3{Ef9W9IIa(<T2aS
z&E1&a%PVF2OD?cyJ)hi<1DYJJ`E09KeX<A$T4wGx=NQieQ91ss=U=%N@0H?t<hYJY
zXU=zv*}80e3xBe3*Ruz5*Z%y#Z7JF+uv2@wLgRx5VQbM!ev62iGRMW{aJ?&x1+V{K
z_Rol~Nnwuc<(hKt+Lj((&|Z#|PlDjKuir0YPQ8uG!0``?lhdBlc*<wbR86^blK1Pi
zoh&6LZkFz0z5ILrNbpbDvQlxAL@b~G%cDF60xJbpPZU>cnW~`d5SgdCqUNyX^@DRQ
z8a_V;hpk*;h%~S1Avva5cerhT{s!;=TkZTsFh17Z@|5>G{*8Bpcwap1)tO&mWNvZr
zuf)gYUVOE$YWekEYJ=Sca?4ix-&{-oMoXT#^pgK_{Azv+iEaEwHCrV1eMsR^ivQ24
zp5SEyQU}7ut1K+$Ty7P}VEV$}XT>MU<iDAB!uK^ioBf0N{Z{dtpFP%L!FDUmyyZ_M
z_oZGR@Vv;Xw*MkJyoG9q-t07QIK7ciDJ_V*zk^F)FAFEP!IVbc^W5DW5uIXS_o%JC
z%>A^dS!L3?hkW{XShxfH4f($%MR9o*qzUis@er|E)h77$2e*I;lNWdYjVt``jW=_L
zU+xs?EM(;DPJL-+p1g|tQKY}vYpWS-kMH(L|BtP*urKH5J{Wt#TyMok-V4`XYqd7L
z=h2pSkn_5;jQil8Ndix1OfhG4=COQen8!V@B3U!#-wlg7vpM*5{<SKpyWHV7dYQ^Q
zbFHJ9*n%y*e~lHu<qzA&BRuP7*+^M_`!2w`lbhS#xPk8x&jyQ4e(Jowe<tusv3FZk
zeb(W7aL$t_es{Y?!0$jFdmRl6t0V@o#E>80^xnR^iTnRragCJ~Gr4{_tztW}=R6O0
zq&WAMZBzNH>KHAxtk?2s>D}So#dBCOMwN-5*_U1EjLIuh<KtUo66Y$&Guh2%la0L~
zx=ZRFx7)Hk+&>QdmyC6^;5oQuCZAyOK7Mf{$eH%L`hN25eiy^bxc`e(U`q!Vdr2F7
z{r|N{X1>MNtd=Gb>TDSa8vJT=?r`w#Q{`uHYPIZc=i+&FTUT~|<7}QRe-o|>qb00j
z??U)@u?MnlX>0<=S1+%K67R(r{<XGN+^W}u`1zQ^xSr{DT3A%^aq&LPH>+;k!%=?y
z0$-}UrTK^NXEnvY9pYb^Sj>NhcRzUjf7U%W-W9D1{7a&J1=E6^__dr$q|Y3=%3HQ8
zMrO^CWt<aNEaBhP(7?}me;N0N5)Q$geWv2ul#+N?ntigk5HwTZuaya(-KU3qD-Z5s
zUR-XYxk7%Y;G<m){QY|v1k%o&m#b4|;$T!i2fqKu<;+js*BaA#Q*R0hZq04y7Jelq
zB=dWpS%tZTtjy#8W}x(aF7+(;Uv^2Jt?Of@nHQ*WH5AO{n)OtY`+A9ysP@vI>_X>%
z@qcG;<lJ!c68q7j%`)fKw`kRcUE<CBnjlzrYqk0Lxk~J3)E_|Brxm1d8~!^D&Zkql
zHMx^3dW8ffjrjSNePOrgogni-b|P00Q#Vi7ly$tI@&BHK%ek1Xa=A|5<+GHPpUUTS
zyMxPUONbDMV<Oii<`s&U4|S<6nlgib{V@}s^Yc~B!^Pgq3$K;5>~}Te50z||WuENA
zWBGZSMMsk;kLF@UUW=$V=BrzFDP>%E&AA}QnB$<oxbWF97E4Z7X2FZQ{Mi(e?D>;D
z11;KrF4g+7U0mR^U5@3W|4X^LQf~1TT{N(K-crJwV!np^ex#Rx&zfyq=StNqUA2{X
zJEk4hz80avbz@sQ|IbT7isqtxLhEEE^Iw=~q*2h<1mFKRo41i26sBCMGMa5a?(m*H
zx=G;JOE>vx;j-K&7hSpi4SRTaHgpNFOV8&Ni?ids{&N9O`xMBzCjEcc@=v@F%U#L2
zi|3%!V}6-mk&;)G(uAh=L<#Ji3)%l?7o5dYv+XK4Kl*hw@aXp)<F<Iq#k2LjCJ(DP
zq@DUv><YNd<?NaZPOl*MAmix8joc6KKH%AOA9C*?vYa}93AbQ*CuiyI6<m`H5$Aq*
zp4`Db^~Mx%+lKS;Z0=s;ubj^gUIDwKuHgvx{_h$*=N2}g*r)QS1FRlq{s$ieo|Ej|
zU_CIgW3M)G=uGMX>$`X&2Q25jw~+h2{!Gs999Ire56Ic0#{MnvHWo;4*Wc;vYqY(%
zcOU=E>GfZbduJ77?50TUKiKb4oozgs-w^%mppEj}v#$Ar<-F&f=AP&L3~bjna|5mb
zy+StmJU0%(88f-UxHf>r*~9I?Zu%0p8SaNy?@n^J+)3b>$;-)m@3azFKF$0co3m#i
zIL<-t;x5<+)(>JofA}2E2JvBd+A7HXIxv2(iy#j>`%>=cn2+4zyNtLhO(FM;z~n)+
z9lHeg<km8tbyBOqZABYad7fpH&v2R*gmUlIQsP;%b2{_;i^lBdxP`#(y3Az)ra^ke
zOJu-$Kx`29xevK#X({^*?%J}Y92_^-f!!%;a)x{7<>}n^&5(BJJx0hnDUcoz)>it-
zGr^{j-9Da^`%m9qu%6CMnOs`ZN4UM8LHhrif15cb@7MvB^HRFVA%A)<irrp+&T#JA
z)XABW!^$<;UXXjyWgX6_e8?VA*JKg6o7+s{xaapRhlgtu|5Ba_M}@#<^<0PCf461Y
zBkrHJ%R%S=f#VhA#-|e@YZ*aoWUOX4g(r297To*~oI;#^s>YnMId)((tF+j;CnP-v
zyL<ckjbIw4f9dH)uo@75^Q%U#Q@bJK6ioI}+<V_X0lSg6N&>8>aVJ0AjtzG4U@?}g
ziQIx&@;tdNd*N~*vtU?;$)6{e`vH5<s;QjDPincRG#v-)lU+EMr$UB-XZop!+~tx}
z;Ce0A%mB;xDK6*!c4-dxpPi~aSvAREH}Y(T_+e_MI?uwKcJBS3;=y6#A##w%`=JzX
z(``r}1Y~#1RbH;efsnCBt0$&B7p{Kb?ibDnn|*0b1lYc}R?VEoS6#t%H%QM9qX}U1
zK<t$!%-oBcd%5o)5#h02YQlAPeHYjbGb0}J@Z3}Z>j$X=;gzbQU~?@#LGl+!3>kaB
zkL7$_rNy&)0}I@IhwDyYJs@>kZdq|<e}ecC#Lu`hfya>}3M{YkL6m2~`55+qBu;Su
z8zlek`A5!*-GN~FRjr%gW`g8at0aQOe{Y_`5u|dCYrb?3m+hTP+=_xRT-m3)d79=<
z;^q}G;W61523Ggt*L*O&<PYR-u;UXJaA$Ea@E9-N&HZnIG-uPL<6wV(EV2gE`JBp}
zzn%qx{eSs@GEc%02A*qHkZ~E1{UBV|B*A-9{43|o1B^Tw7g~7)bQg0kygHpHty_<C
zLTnSb%$f3SCQsFIQ=U?-H#~1wK-N)#^sW+R;*ww21$GmN55hYCX7OlSSi{ARpO_10
z_kM!pL5^6&U7Xj=_~G&oeT{fB7kq;Ak9nDMmzY8B&e^%FpIgk;kLSoceV&e18BX>8
zkn>QlUcCobZxMD7>|T&NUTt2@$q@rNH~!Ys9b7_n)46^2Cvc^kZQ|;ZyvNh*;KLnz
zN)w)*A~!L@&C%W@!1Lr?88{3<W`VH8y_p=bJ1_Gr`|<`Jhw&UsIX*2j;NCrFDR;sl
zc5pa@)b8KZ$#HjgD>y$+mAKBWw%Z5nt^}i%JPLCf!Rr1ko64=n21$#5yP7yoe1VKn
zf3}32^$arS{<#O-FPB2@uB*2_!uIsmQeNE*cAhzT&0LBe>*4PGRHy`I_s&@gwg;p)
zWyedf{UA0BH{U(Y{p4^y4_ovTusX9^9k5$K@|uowc^FI<gUy*8co0nM@3rE2Q?LX)
zjsa5hJLno*eaX8@zBj^O!0juYT{n2u=WpbxcI@Q+^oF1Rcl;`FxL%tBx$DPz?>fG#
zt0DVt^aLPwfXs8O-^tg@AIv}dh#=3pyN}^!gXETN=HmM&(#mt^=U?7KS339~)F0z~
zVR(=8+MjEDZA=MZdz;du`R-+|=9c4A<4s(?hIjh4xqR166#3UnU*`K05(QQRGHYv8
z2G~6ywzAAM-pGys-ZzRHdH7Z~aCRTH;+@PP#%sG_Do>H?CO!dy`Fyr#GQfKI#8dgU
zq@Ut7@KfPCxO@iR<hM2)yQejP<v?bEu*RJ<-pMUrxb=1IdH1+m@lIVI43=-I{|2U)
zA1vfepToxMwnK{b@lS2${#D()x#p>0_r$cY^PX@z4rVVuy`OjG`2@agf3NVKZCB-a
z9u1kV{8~8$EPwv{Ij-eDw7}=J{kXJ@@6z52yzi!j@U2vWl$%Ub^|_`OFoNv_xvMBi
z2F$*q_ntSh6ml0CNF0Rwq<p|>xR*DT|D-Eq&F3EmbuPg*?0i8ilerm<F7WC6%H(kF
z`NXSsZ8ulS)@HC-OOHqLEZVKe=NjnEXYkpASH`N3qqBA^@BfSgyz7OEc{e%n^6gOB
z!1LE#jC;k(BRqjZHC$hsmT?QVb+K*huH<+mtj8DI16i-|lOv0J?rF&S{>ghb@iAE?
z^3JN74fZR4W)aWScU-(~Y!P64XXHWr^7WxTpH00K*GX*~zJ;o*c(*9d<eunz39M(&
zOHba|?X!8pJQ#Ub*zD(A_pFR(;zBL3oRLjFcid+IFdO9mU1!y}c|T_J{)#Z<^=oY8
zmApBb_o2i?zPr^wd5=Fo$}PtUsdv|D1#%U(8u8if=HOGTTFSO&(@qX9oesYIulvDf
z&GqU4(>Bk4^Ie;Nl4oZ|6THlL`7azi{|mBDZjmk@)9z|+RvvBMZ?7)%-7NkC)(4U+
zh;ZRrc;y|B@a8RGzxGz2;@ve5lKvL>3WM_@NF4}oS_`=!VUo)mo*5tP_%eI{^GNu$
za+$tU=2Q9Q!gHZUk2CA_IlkxuC60tWkbM!l*Qc=^m;>44d3Z7hU+$6PypLY_gWVQl
zQOpCn|8L!D$QkSTW%v1%%8v4~M{nWX!qmsdt7rpu)A`3zd^-iE@f>)>!?)v(1pnk$
zyV$$mMDZ<a<puj=`I5c79S&i9MSG|5o!!I^)>D120BlE<fjl4oIaA)byytjzo;~51
zSJlSn@%kX&jYp6@IkSZ{Ir|@!b2{C8%hz!C4)>DvkNL7PxAJD(^y0U=rp!|+W6yg-
zw2*a&eiHYWCp*AqZ!~(&#^H4k%m#%^jQ@Y$HS=RQBNspBRY~FH%f8je7hW5|lbGhk
zZ`cXhU$b$=3UHbC#|d(V5Tk7~4^Pl9lrjfomXqpgzJS&)-aGyIV84OHN>7>d@oTWN
zZ}ywSTmJhWZ~C_^u-)lN`*~UXPV%jKbc<)=)1|!GQ*8KtR#@@Pe0P~Id&@Gg{>ujG
zygbb^yfbYWd9(e(!TLaEf-uX>R&aV*#lDH_oo6}UJng+aoArA5*lrv1Et0#)7qDp)
zkD8S=_j5i*a9HN$LiQGY+6Ecdd}X7;7vP)C_rGf$PpqmkU%?g*zSZqhczwU9@J=*3
z#bdb7i1)(&4PZa|HZbsP+GfIk_hu6xqb4J`5A!sVkxTy+H{bX5zj-H4dCALuTaVB8
zM<?$*p$Og=IVxOfA*FnO=IQYHy58cOy!Z%T+rbe2Jrz56=A4h@`yX(VBlW;9o;{+F
z_5a`Fe(=~@n{uyFe!(}JOM*YS8FK!?v@#oTIE0?m;c@O;4tC3*cW1$3@7osgY?oK(
z>+!M$^XHeu^PPBbfcp<e0pEkY$JwTZG4lnb1aMzH;K;LNu?Js-ng_qIzay{kxvgB8
zQVQTOF*+v)HWL({YyppWmbpF!*LwzQ!g&8&fsAdIUS7*>dqS5l@0BWF*Mbh7OY<l5
z#<uU`H9yx2kB=i#e7x?Zq8yib7qGnW+rwu*2hl&gu|%1-{?9t@l+U7k6&(gVj~AHn
zJiWb#C+ZLn|5lzmyt8*L;ZCxx;rsB2kz;4*7Pba0cHYZDVLWG-?&M_nti|*GCS*TZ
zMUOgfUBWZI?q%t$$1Wb_d$Eq6ZxgF9zrB0_fBotEyn?50v4Yli8Sp5B{ot{xk?+Zd
zRbYQgEo<lBBO=7lUV5A_=Vv#sd4mgI6pugOQGa*NO<x3gKi`++`5OKV?w>!GH#334
zMEAvFa2f-#KZn-x{N6r?*K~R)IK2NFrh(1Y_15GPsg~y3Dg6>$-hADCg)j46GUw#-
z-@FrQIlyW_`ejs}!Py`_Px>18dcz~}=iy@yAUP1evhk!Uhz8-?=Rc@{=-)jrRX{W{
ze#XiRSBESPV%PJ$2kQf|v0;PyRT$=BQwI}^C{l%+r*X{#Tqb-`m<ShxsRPju9sjT|
zc~_|nVv~z^+5Lm-582X#VNa0o0-Sz_*_pMrfuI_gT5@R%9Ynam%s9;Z55r!@COr&4
zz|<2%>)ne6w>>l!B2}#=dNIrgnGvz#0MT*k;J;K6q=yo`<D@##erhjR2aiu=I~#iy
z)M0$~IAIijoz<;D;ltDsLmv%Y0bZj6Qk#093TL=1+i0tlX+Kr9K7|+TK3&6X6hBEA
zKP1WxhBcGGc>-h~DOl5Wi4wZ{HS&kRcAn~t1H0)-LmN2mk=5P(eP6}UE*b2v#L&|c
z+pPM*dXUvjE>FZT8(AL2UY&9fu1{9<FW5Yg_-FpfVDVr6cj00naeO$8TN|s|wW{GV
z`1Fw@Coyq7N<4wg;Je$6AqEmB20#3(2=?zH`y~o6^<VSm!`UFQ6XHQw&A)o%HhkOy
zqz)H8di@dD4rKXdflt8uosh){v2QPfl!s3n?7-;*re3K}7USL#m>e$J%YPY)8MxFD
z5>L9Bgf;Gw_081a2Csq7c)kfG9H(D)h3j4NIT1w;q(66#6^dB=*{N{#e{%ZZVLHYA
zABsAVUe0nR3^9;63~#%ZhLN{ma<rgBZiomR+ua8)oBq9+C*UXgkgsJT5AUf7iGo`<
zgoE1`%=_8+B@J}BRIh*MpZ!`~P$)H(^~vF4cpK&B<J0`B-m40p`y<Z(<zfZcESOto
z=1dh>Id=lzzesU`AO?5-DSkq{z5(}n<GBv=oxB0*gQq+4@!fNN!Xs7c!S8P?%P;sM
zi?6k9g~0EI4!(>#-r#kUAoE^Ld<D0&y`onjYvxQIuJfM!Ya4`k-bEV=1XbVVwK3Vr
z6Lism?_or{K+!*6u=>+M0epK;vhc5;q0SfdF;ZZ<{5+mQQxE=_r`6zo_&&!F&JF+n
z^WPI~<rldc#ank@l`qMnLEx?38P4g8nZf>1x17j#nPCr);FE<s7jyNvmfL0Wr<du2
z<(}wR^N28LgWFU`&QIa>7nsh!XR9dxvn)RT2p&zaxZz?ka9DCP%;w|%_mL~b`7m=@
zdK}*$E;jzT3EOzZ^|pfJ-F=@i$E~tBu(=>Vr<y^|Texa_jIaH%9b9}zr4qN9rzyBi
zxcu}tfu^P}VDlqZ9prnJD<rtzb}!#^8+Cy;YYxs6weR^FnCJ5RGu#Ju%Oj?L{B7r|
z`9ck21dM&fd28AC@?M)PE4b!V3;#>KxqLsC_wa4wnZ*BC(t}rM&U|jG0~2{Wjudh_
z-<!nq)G3G8$xxF2v3V=lEd@V9c>bEK1lwI97%I^E`yWq^1Y|tU;1L7g&(q)eCPcD<
z<H744q%V~qIA0)DY#Hz31TDc8C${k~JS@ny+wwP9jmKX--gghX`7Z41<vAOEl~=o{
zok!-OJ-BZmFVD!mzU&d04GJ627gc<7GXnTe{AA!YtzhCUlFsGlDy$P&@pFQJ?&omc
zOS2&Bk(-?N^Los-;FtT@!LM4E&HIBpou}^G6uyMH{9r%a)36jgx8NLm_uq|tnksYn
zrp)@s30nWZW5QC-u9ObJMs``YNRRCtQJii9@g+0(&Bg8bojcA57=6nE`y1qzl|Mdn
z7OUOj37l^WPOnXtv-$6pyYcOwY{b<lug&#BUm32pOE?T1Z@hbU@_(`2%(s23CjZ`O
zIsQj~77HAI9mV&5^(k&C`?>s1VgbC<Ba*>!wos^!S9A6P{)6ASxz_wW!yguJ%e&9D
zP|(+8JKNske7-7iVNM+u$og8&|1<c`-`3#E@+%TJ?BvS#<oyJJoh=@04na!%&3oJV
zr>yVit2{GF(7XEpm%)!I0xSF=^UyD9dwEqibnxwLbK_IE^Nz2~LW5=6$3@^Ude``b
zS6<zLUtX1oufXpKpBpnX-{QY60`d2Dg5yIXm4kO)+#k+m`kVN+2ln#+<6OXhwCW5W
z%f(Uw-rN*{m6k#R`BA}KSMHwX_m*}6$2H>_cE0eAaDk=<C-%b&%UOTsxASlLm&o(c
z=m($VpL4v|Sd95~&R*cp5Mk%f;Z_p(EuzR?E8GsYZ^=q;aJZDE{^n{}x(93?C?2}>
z1Hj{)UbAKkbP0s;wXyyHt8<^x&vt0bA^yHOvw3ESPvK1~{lU9CXEy)A`ZHY9eaiWg
zt2+7qeg>U;uF21G^9Apm<I4Oj|9J)LUqa452<JO3z>>C!uO=`|KtSA;Pv2)gynWSF
z<|j~WDaGEjF@tN>iaGo(_wVtDvE~STn#U&~-0Q*jL|sH++0$ko*=sMs?mVS9MZnX&
zg=5)41u);c(U9+g(*%KcUmCdIpFA(1F!Knnd@_rmt~n#Gw_%h(`UxT4H{X8<bgWil
zUl*j$yHoX(Kzg#dz@uq4;IMeHQ-lBeNkKubxYYtHUtbki`)np(-Nyxdaj}*Hx$IG#
z=gj^I{49GcaAoHNzLP8>f=<jO0!?Bud=YwE1X}E>x%>A%=KFg<M8K)AkiG52Hi3$S
zr@UsFy9M?K84Binw-ESMF2rT}{uEEo;(x3MRbTMsmCXdZ8+0xxD9m-I%;%M2Y3IM!
zt^`giRW&Sp+l!{KW;kyJum5j-vz9kNvx}=o`YGR%_n~|zL=6NZVnui#e5l|`E`1>I
zl23&#Pdy&&7ts3uB~l{1I}Ss};d|SIc(?2JgUz*Ab%M*i*i`V~haiCqU*vdX^wtT;
zZ&}8BwLyq)r}<BCp6X`1$Mac2lvgtA6^GANM}DPkZ^7aqcP_PN=T}r<<}W+oDOl~v
z$eWNW#BXh5AmIFFwLodCtAL~CQNFlOCVcG~Q+c-5EAZL}@$=n%ae(JrwFB2J`+3~0
z|5LzuIIGNGAnHb_;FnK!+*!w^1a!A33T!>FRUoyHUGTzcA-<y=uXy5?3JKWlh0KF8
z?h)s^?qv@y7tYP}6Ld+R#g{wFTEMt6ly{|~n&8UV8+;2@%=u*wsPGA}&frma!~ym%
z$jv#~khOi`dm-};AhE_WTYmm;ro8h$u=4J>)ySINc@3=IQ0p#l=H@0|54%Ee|6ga9
ztf1Nc?L5^lt9hqf+A1Iy54mq@QAvehj0@y`ugmLJ@jkk=1?(@ggRH!l7tZC%*m;FV
zN_`>k#^;bZ@Y?5)dD3nx$T{pFHy~p%`~AEwr&ajwrwD@WMV4EzqnD@ASB!g7|0iw+
zH^f;Vf-_lp-ES@dtN%886>nIcB=<b4mEgGA<;K7lUc|z;t3w-{&S2)5xAcP32#h~Z
zbP-R~4F^sbA4EI-d&w;m#{<!qw-M~GJr6^9b)zouG+CGMt}2GiH)*CFKrw5_{!iSE
zIlDNL9lcmt?+0)tr$W{WFF(BrY(@v?B2HG}9JroZ!vwCM4<YMxd7t_49O{PL`^qB8
z&+DUmoTs-sl4skVD()X1kh^iEd0v9`Zc}Cf`w`^UnOuK(b^aaU?y_3Vd*W;yI4qKD
z<2deTBi2zQI3VU%bJ>i*ZUN~<#{Z3HpoqWIvEaF)u%A=$<QiUIv09E9p^*EKk@b8F
zT*_tgvy1ooZpfJyTT~=@CLK2B+Vp8P&*PK;-a5V>wgoy%Id9E}%wISseSy2%vuP99
zFEI126+!A`WB2*I9*OIDYTY2`DwwR2;=W~lm;3B<$UIUHTMmjlE5FO}ZoNE%`+KMe
z_rVEA!Dc!io5SOFJdj&vwFQ{}XyPtzbMXmawr>?LxBdAbFdJsKtez;3*0pY)zDcKf
z_Pk%ovt&jz@3VQc!C@INMF&iS)J8hr;eJ+!SofN?cp7iqbBMY_%r;=Z@ff@Fz0J(#
zDp7#kLkZF+cN#KA3}PeWFD*-WZ%pup`^RazKIg_OraUEqGr@L#PfX;_|85C(>+e|$
zz~;mB7k>1Gr`aDer+7a7g3N7~*|6|P>f8p`EAocEV6(H2&VZ|p^H>ZvPxSL99x*8e
z-u?2Bv!P(-!swHWbot5@1-XNi#dy|v-ryDE+XvRyW01#tZ`E|JjXisK=03d$HXo#C
zT~!oV{;!7wZ_IHHp2g=r@~*hn2No+h1=(9C=U>aqZ=A{_7}fz6*LZrKx0gedZ|X5t
zFdt-&PIo8A$xKUd-ut|{k^AR|4LpoXJHhq@*0q4uIai8u{az{zRs+)05cm)*4r1#n
zz2P}FyM-h8u?nyJId<Nqx$a=SGw#ph3tt7fs{^DCgm<lK1Ba8$vz_4Z28ki#u9LfX
zQtiI7TO>t;$6!HnJI^BGOO$C2mqkVv-0t0l5`30y-eCT_lz!eP4=(fc?(qlP57K)j
zL!C=L8<H2MOGD;lK;j_$qUAQ&4nY+Gt~#D4T<a@^c}goHdH3t;^E_@;;+wa#k)?WV
z3fIF9FR;JG9_xY4U1)y=OxrEY;bHJSz;oaMCr>H=M4oA%V!`59p0$D9BYwY{=c4~U
za5@N5m*<wfD#Ul@LI{e#<Aj)bXKHJ6U-W#+#nF9$v*JZ5&-REtJPA_^Inxanf%`d(
zbw+##nfrMk8F})#PetsD`|0MyZ91PFY$wP+AiPLy2k+tBdbpTNE2P~8lB>M11e~5{
zE|-VP%bIKhyA>oKt-X%dc>-k34~VaLEtq$Q7-WBe)uS1__Cj2IGnnS`Y`ZSawT`I?
zYzD_7gnNvl1>k0_lUU1>KkXsdul&D%bE)~8a{HT4;(Gh%D);ZihdfNF%)F_)u7k}0
z*~M|d6R!6I^JQK~=Nn+Rg5*GW;jT)q@68i<{;Gd~i@Sed;GX^SCwI>3UtDH+C*fwD
z`p?KM(;WnkTd#vNxLKEd0Gqv0)t{$g17z+a;y@EGo2xWfO}E@!-Xr;{VEs<ZAbZ(B
zc7{|O=WVM~0>^#&^ewy@?kRj<dYSm{D_!PcTxSb56Qs`0_Ac0;rwq8jW`M*5_8tbu
z9f%FXpBodolK1`KiT%3{thYK1(w+v%w_Ysbt#Z!>oAXE+lJ{CB+wt<xdJJwCgVbzL
z*#>qCh~4+}EY}w^eQ;Q0HUHxB(C^?{k`l!AWu`gTW}TDZG^eN82lij`^-o}RAbrSq
z9eX?HtZAy;!Mh-9wvgqrO}}yezR$oGmz2vHDuGz17W8K}*W7E6b#9?C-#P2Qui;$D
z!38!yDOiF_=+zu9bqU0r2gpp1FQF*oTI?DXT)zY$_pO5DLHJzO0(P#xqhNDxEpi0A
zUs9ZrYu~O*>~H$du{AUO1(zF>WR8I6&OmxV*s3aw-EH$XPS>g)&is~M4)6DnI$)hz
zFj)QMfBU#*J_rG`LF$*RxW*ySaSJ>*vn2XD*ZIXQU_1Y~EaE7b|B-zYGY@BzzzvSI
z8_dAs5p1i$G|O4Y`Yw<e7s3j-7;XM?{Y#JHexOqfmbc`Z&nmBnxGy$6<QUiwAoaP-
zkp4c1jf{;08sKU`;`^Hpaalc0;WBvW!{xZgfJ17@9Ta(x8s2NWIlm|Ka@Pk#_Q$lZ
z^X6(x7UMLu(&H4X3+76z)@QwTW-4o==wz_FCR#z}I6->PiZ4Y;lRLE!gUz3x&%oue
zp^e?<C?p;`+7-CGV}-c5E1bac=FUdobO|y8g!_8ZxJB86*=h?;az2Uo1>3XQ!k6_3
zYX_IQ&H=EV^^-2Lojn8DU#NO?FNf!qd+;~_nc?2b&N=D8RMy+9i#R6lvg4FK@6U05
zGh~fgmI-7p9!TBZ9XmMV+xx-k7{s6P{xg?X?`5!Ab60)<i{Hq9z!@3L%+Ap95X=Xu
z>vwENaWhB^giB{G;&Pb)DbqlF_gS3mYmX{$<b^&5n>kZblC$yoEU=$=GC06ANF4|t
z|8*B`ezKQ1SL>UZV87<^t>g@`D+8ygeXRCibF5~)f$QBX0?DT*&j032cG|-A=?<bz
z1~M0h=bk*qH78w=O>{vAhhRe#m$C68u=#5jadPiWYv!6%8pw4eP8{y9uRS}!^3w57
zI71KUaxuTN<+`N-x$E}Pt<PXNhSQN;zrI}MG^n}-<{$CV;;t=?;GR_#2<C(AKAPCX
zsi_OuYg%iuhwDM>b<P=H9$<NauS{V3<{PoIo=JwR^9AY2y?YDnPY`?V17^;!ZYHi>
z`z*NvX5V8ej@tzer!_tr+)?}u@NfXBRb~|jn_Fdu*rNs#hvBFD*K?@HZsbx+HV2yl
z5<hww5f4!nJ{(_EAa@mm<Q6n4aj$#=xx+SW`wFg&HSrvBeNON)v&-Wed)6<=y{XO}
z#c(%+^y<A#0=w;@$#J$Yac_?6kKVICvJ&9>pnRJne3}sVmx<Rn+1}Q0#r}-~>wDDn
z6zq22xsZF<tuE<uT`ptb;<($+RkOvCo%3EbSlyq6o5A@^?Vvx$&gGExbvhOcxn^-C
zaIH9v*qaD)!;Fn<xj#HU$Whdv%%!tmh1)Mfg=>5IeJ;(pQtXYNg~8@&eLBTen{L3x
zbH<jtP53%k45as>+(r(?3GrY*fcPN1I$oZ;`GY&yogn^orYN|)fs`XSOp^j#;qvP9
zc;Rdwdq1vtE@!xUPM!UnKR-Kz?VI*Fohx<KRW7^r{alI;o*WxCEC-jvzak9b>Wgw9
zZ99;?x|6cmqa?J!a_e}UIZC^Yx%QhjbEvHN!NKg?&Yd8|&Gn0G0XS`e)SoFp#PP>T
zJzV@J_JHjL$${{l@15+A!>hO@pHG0hH`r(ro7065PLE|OTmssNea>_F{<2zUvV-G5
ziQk9o-*(77+A4RQxHA<YYxOP(2y^+}hO7sj=H<qo{r&{l+{Zuo!1_RTHowv3s=wd~
z78hDt!CDx_!;{;k&(&aZfFn3y64-Abb%(w}(pPg&Io!=Ee|N*vAV>`e|Fc}mWqEl!
zS7_8hupI>|9&j`I-+$)vy#`5-TOusL;;j}pxW2EP2%n2d?t$!k1DW-`a|JKM^&8;!
z#zy`lJiG25=RE&yIS*&{V%|B$E5YHqbM^(W+igXrg5w>e4;lMU{>u9_!<di#eiyhs
zf-LvX$cx9h-GSRr&4l~sN@nm_97sN<zK^$<2eM9KgTQ@Wi;z9se~!3=^}pOA#P_CM
zl}|O-7~W0+=~;Y)5A0458-z7a8u3hQgsk-j@j-aSd`F(e9#UX^-%q^&tLHh_!F#+!
zn0GVpdtS$MH?W@`?$HO!f%Jf|8&5t@*2)0Rsa9;9Qyw*OJ-nR<R;MBj8M~e{XAZa5
zQApnhq`okulymR=Y2fgk@^KmWiNkzgx974(@w%;3=XEY=;n`ljiM#rx1K1o{xs_mf
z^_z%3Y28l|J~fq7T(6#$@$N{T2##~^fS;U4IT8JTCx!E1{UAF}uQx<-Ge``CW3KK-
z5$ApE%e`sNG0vhdr+9x|ozD3y9@4)9sfFPV?ycM_geLRe*aJDMM>#8>`@u5}winKi
zc-Dql^1dtz<b1W`3+HUfGhq9@4VS^=elGtDuwP(iajt-vasI^`-iD~JJUr9xf#nOn
zJ>pQ*J<DzQ`zhFcXNw}hG)&(Lrzt$~T2r~D)8BFWzkr<Q`aVvcchB!mu48tPy>$yt
z@8jNW4CzNN*AnA?e`hM#Y?vA0YZ`c5I+%IQSj2ewSugXPh}goL@d2`So=xu$I7~rm
zPib^;7n)1J^?tFa<lV~*;j7+$2u|PWwT*oFKWw-!?WhIwLHd%FA!{Z<Y-GH#y^XiM
z)d23F7K=rkF3j_I-1kHJ|8a(#+<G&a!16-d_2K4$%vda=01wL>J3jJEUIQ6dx-suD
zw{4&nSnb^)NSMudgxL4)>Imrrw|+Xr^Jo_j?~M~-;B*c$6B&1hC-61hTf+VSl_L+w
zB^f@^eu(*PQ3kw1->rE-XR3ZOgq$l3(kGa#3=S8bZdcx}XKcKi{l4-l2&jSGo^*LT
z*pBU?eY{)tyYoDlDg+k4K6@Xp>N_UBmgX&By&!WgWbtq{=v9H$-!VVL(^@crM|T<I
z{t`BS$eDk0yW+S%E@A+y0qIeVlmVLuV*fm;$9wIk6qh5rBJZpgao+1~HDI}U*Y@ze
z>VvFx0;vPxoK=UwdCy~!A~=15#E|jIOl98GVo9EBJD0%2I<5n99vDd7%Xto5l1Kl;
z-QMA2!{_IG9c)(gfqve*@jAR`Zf^kF4^nR`d6H9}=POwL+&;)UR**OdD>5^I-6G%^
z!)3^@m3z+yYaUJ+9bTpwC!WYv27D(@c5!QZr}F&NnGg2&^_Qk#bH6Nw^da>k^?Cbv
zHu3cLe&w2*{e)+aDrBA6vI<B(dsM>B`}s~I*zVS7A+Er`Hhd}j8{lpSxnaAA0MCJg
zGr3hNFL3I-Xy!hvY{+v^l$+<4q#$SOs~&JZ;Hh%q`(VF}H*-n^-=r0gvlT#kWhcMl
z_S>Tfb`yvX!h4_Y=Urw4nXd!!MI#Vph~dI?uz!o%AnSfX^5-}pV}2mELeL4`RLyj7
zItKBVxHj_^J~apHQFKt_`@T<s&(gD+r%v+$=hHoqv!L1&AZONr)bCsS67Dy5!M8j*
zE8l_r9=m!H*Wo2gI9++CaxXjog8Q^|1J5KLd)~iGcER0lyzvZN?=&VxzL^)Q!1jX7
z1z`(IOHQfdz1-DlhvDKKYb?1-_T1rCIh)M7qY`o^Ge~W^^Lws$>e=A*BYmBNt2G2t
z{%Wpp<z|dK2evakX)5oA+D@=pVmi!xPlTp|#TyHr!~MTa@hflZtwM0z|1lTgGf1`J
z<MdbNV=WTkwf2D2u^{{Q=0n!fc1E^<%>aq5Z<_))2gC>ARd;LHPd$6U6QB&)|5NJK
z4mKMkw{Nu?Z)X2du%5=MwP17NiiCI@H+R6>5#PUDg{wC;J<K2R+8sQ9@P1mXz|8Z-
ze7beEJe!uv3rZ%|g2S?0YaLkqwy4$ok1Owh)pF`X&ISb8$uqf&ziZl4f&4Qk`2?+X
z!SWz+5H?s6DsbRX6rbmrhx}5m4SX$rT0Bv)`aFlG81w0bLdFX13_W>|GO2L0Y-i+8
zUH_JEM{qmur}k`tC);=Ne!j&Io;LxRwK;SS*c~9YNx84UpY;qp^0i$2-&L8}zmzcu
zJPPgLd62n^`}50*yxaTT1!lWL#+#4da_3#_GJ)^>L1$h&1{r}nJ-l2-+o!_KO53W=
zdVlL0ex6lk{2`9kyvh@A@hZ<Y2K!^@f*s&C%e)?QzSd+pp5Kch=Q?z&Y4a!D`vsQ!
zQR>I-q69g!`2hb8o@X)%{C9QF@~I2$<XiIcBG{hI?vVXE(@vX#`5-@R*i_5A{MjMC
zM{A{d-|zPUr=hi?``MDJ<iK`=)HHBNf$ey3Y7!s+W(K%8NbdR#NV{y()J+1j4npo~
zc=g?kFP7=Dz~|{Td_gI9_?zZsa{Mlu$EWUX%(Eh`2W%E+<a+*=eG7RdT0{ial-KfT
z@K5Ksk#Jq$jIatnS5!Qo_(wUuIx9Q=EY$@(>1FG=_Z8*xnwuWxv~T*%7R!5%Tjld+
zK3!97uv@rpMsdGfG9R2y9D+myf>!bK%rAwEqfS0Ik+<9=lkdTN$hu&~ec!<9zQ`pB
za0IU6Uf%dlz-LJVzty4xoIH$>Gto<PvUn5<t@t8BCUGm;{o@Q;q{H1be<?WsdoSka
zTBZrv;|}tF(w8H=pVS2S8;{xYyksxuaTgWmpINOh5F{5Qu(Uvmw`R>>uz$N&%JP1?
z{fqCmkuCpPju|W)CO5DO$L`_HXF36vU+|p?oW_<;c*xr+BE=V~Yy-|eAUm>5AoKYk
z_V;&u0yAxb_#>aP@hi=qAfT`t;%|_g%%mM0&u@O>-p|DbHmk}ih~GiuCOH2oRP5q>
z$^q%qfz<Z<LH0K-JJrCiGNqW$H1igpoaSr38JE)pE;PU7En->9ez-Y--+fXC&jNEu
z{I94v%$Bk>8th*Eh>863R`&DA{!{_m^}!U<{wsTH3l67>wdZ)x`kL^(@X!#@yVuHV
z9_I~?t0hVI_|BVH^M6d%<ong^Cn(dshxM;(puo-OJz#xPE3Wf6R8HbmODy7*=Yg!v
zX}B8$*86m}B=3boclnmTEavUG7R(pF+kn?lhgqPh@H<#<%?)=R$9+N^_HK813mVM%
zZ_f4S|MW71Ph`Eez_JE;f$q)|{72YkvWmXy<==lEa!+x9;Cl9jXQaXAgTm$XEEE1p
z^<uX91`T{(57>EA>c8^E9Pi<`_b}!QPP_p2`ywkfuzlqk`@#80MDh|BmoH{JNKr6c
z;O!D#-V+{c!TLbvUubUSZ>dpb7P%nJs~9ki_v{28{*_*~>|4Sd`JSik;4SvE;r_S$
z6yHi`4&K(tyL>5Ej|*5?+y>j@SKKXNwY7;)Znc=e-`+%Uyn@UGVT~VxU_X>4?C1O<
z7|mbS_lEo6B~^hXEr<BI<>v6pah%~VyIjkCL=Dmw@^T3R-T%wdUJS_>3vR6C4gBmO
zP|dxYOX2VYf&9fEc>LCH6FANv$@86EP#`YWl4q*&2CyBBwb?w){VN5|@B7WKdx#6%
zcQ!kJhwl}`d4d1C^90zurwhE9$->9<Fr4qwq#yj#_w3;~8L(NPSZ1MsK;m)UuQjIx
zeCH_%Nbh6dyK$^e!1?4-_O7;Be6EVuz~QvnxIy65Oa<Ocx0(gIKYkbR^JEp+yZAp7
zZ!9AZ>+wqr?*%6F?QDmPH_JCd_Ihbl+4HV7k>`Kt_84qNYpoeLeDd$tfZcrd|1+LB
z#_u`5p7G;jS4`&ZWBef?`pSzZ?$0bvxo@+<YC;teefTH*COqM-`QSDSo246%hY%$F
z{&{A>^=>AE;P<a$0_?F(+_gUo1kB$b<~h1J0c;;AE+V<Na^DZ`<>?9ZWoZxm$~VUe
zGX4sZo5VJYZ%gVtz64!<!Rs80c`|Ms<WrmYi~p%wiNKG$Tmlb1b@07B5(YNw_wlLh
zch2wUT_UB$qkCWi`#<jIT-O*N{l70wyaG?dBn5S)S94ic+~a3ad&0jvsYT!~-#&q>
z{{(ptUMuF|_<Ir@79e|#>;iae)^URKs!*<=VBG$@yz4f7<UgO%z>^*GUf}SADSUIP
ze(*_2ZsEPEn!^3wW-DB8%FSQk_Q%7I&ERwoGVgxO8@}4*6L`#jbAZd!HNum?VOYSt
zgXhomwLFzfTflV*<IH;kle((8MXb7bs!ucv=zaJJ9$Rd^s0Vfn0|NuYvsL+gjZ+}^
zitPPl#v327hLh1IjaT_{IA8K{$edkja3R>dl%&O6^16oLF$a)d7*;6X!MD=w4WE^v
zGOt^L4>(Q2<evEo^P0Eb<M#fN&l7SUvi=Mt=bPWnx8y72Ohe&$)A@WXcJQ$7x8qHj
z^n%%H)*HUesCRsp`fU717bn2&Ytn=4u><L!owI~F()lmX=O?`2FkqE^#^jV$#isC3
zn%A)M6kkJHE0b=(T5#SyAMVfhv*;~va@<<JG8V`^6C0bkz;Yn7K-hc>18;HrN-nLi
z7S7$ZksLxdOu^>b_FMw1Tc6Oylfa7TgRE<+;+z%)Ss!skZVFFn;UTd4qwUgse`7xJ
zW?b{(ePeu&=SyNFSiH?M2Ar;fu0zHjKz6DtzvHvp*~yXDB+vJ7mpV9%KR>YLxLBkC
z)&o*w{P+}@EpHpd_G8jduo)n6WGs}{&v%Ep3m)#}KX`fib}#1EztzhZRNug5Xad<k
z_rymHJbx0ClF2fy_b5-}9s|DPyCCgV?Luj8A@c;TxUL>vD^?!9fI214W51ViHM9QZ
zPW_?Hx_}*Vp4Xn!dw8w9OyKci(F++r+8o@&7ddw-Z^pLyV0VfuUFUqGp3n120+PP|
zSVQ`KwHwwkC#0@tHPn~k^}5`}^Ck5oXOTE$u93Iq9q+eIr@8XeApK9SNw;`JS}uaa
zz?$<Z_td8$V6$QV-{hywYw%+;Z${-IUP+zZyrCy^`22lX!0IKX=ECdoHCJqTR_;Rd
z_onn~^KCJh1XkBu2bs6}HPwc{cX=Go8(Bn|p)R2e&x?0MxLCaBn6sVfHD_aKH0IlM
z1TlxcTEl{?TeyXn;S%ILufpV=+(kZ+{vG4zr|_@>xm%rk1~{%(xgKNoXnf4;^jn+v
zd!Y$isAm$-wZo9Jagr+_X*X1y4eUmczPPI*;Iw$gteuy6!aUw3ERehDKyt`<j=Lm(
zb<HB4iC27hwL&iOrPo3BZXH_rnb$d4jo0X=Kkx2iO<=P?`uJBv#&62b{op;5{G3<j
z=Q+OKn(yFn)?{k}`#;guk#9xp37+Fxd%)rhBH4T^W!Cd8tBQi~85kJ4d&0OiEUm!m
z&uH)9ZSi;JHIRME=OP)ywrD$Kf7j}BE4XE17Q*eeRs09npY{GJZ@tBJ?y9VFyua^o
z@@<!n;9KbbgyBwAD}Vk}4!9nWo6|ZWXM?`2s^vCNHwBk#AaNKL{qu=;=a;iQ$1O#{
zX&xm0>yRJVjBYz=R&!=XZn<zsAI6f4lV!sD|9mQrEj(S0@od3wlK3WO{^NZfoyB*C
zH3h6cHmin9X4NX5N1Iewzns4XHV0%T2>(pm0xp~1D4pQCcjYus^J_I;c~wQe2^;S7
zWCgwC<Cqc8!^;xEyGSwv95x$|K=zHMDL}#@r-h$yO`9LDUj9?=O5PS;Wj-IU`RlHf
z@p@>UV6|Dfp7-B&$oSa2lgivJr?~hx9D$52f$aBj%VUeq_Tp*fDdSmJ*2eu-bP3O2
zE-_xK<_;cj=dGMSRie21yxmxjW^?doAF$`^PLSkZt*XcU>!v+89WG>_&119w88{xg
zdLet0KecD`{pk$j+qi54xZikag*jMW+~5X_h@~gEtm<}xjP=wlp3L&qYZ`B-x&&D5
zI+=1l1sip)vRyIYbic^ol<&x{r(Bk?oNQCl|M6MN-RBFLBgy+^%~Gy;pCS7T8<{tN
z>wA!YTztC0c7fQ;EJt~JFF@A(Ow)AW=2sBmV83g?)34XY<I%Q_hvgn0pE!FeIJ`jW
z*PdDq*E@U55x%W|4Z(2(lH0(;$XXz<lKZ4*6Zd?stK2{JPO)~%>M)x#GxA7(?BKa$
z>%e*94Wu6eQXgOOoBLba0kFFceRSrYcS!(j56i9j+?%dL_8i)qm-F!+cjB%%C&Fs?
ze;1#}%zUt#4S9$;zRBE6`F_7J1jqfJjH7%v-QMy&di#w}TKf{O8!HbF-}i4!J6Ks+
zZ#6^qw?7Uw2b%$MgU4LR8MYuc46`OUav4a4@*eDhj8BEOTZ7F8$+sPT!+V!Y8tmu%
z`;c_R_V^L+qx;?9xB#hH)H(;QKHWK==k#*OxU5?I6du-R`P>R>3_RO!&f@)&1leQD
zlwl1H)Ai-sc-xP!1G6tan+i??AoDC`mh-#{dC&XOdlmOYHq>(|Y-We>aJ4aW1xo1f
z=m&E1{`QM!pJBkjZmZeATeuI>empVh5I0X31J|BE5<Ds9syuTpAL3pyr;B&8dkRmd
z+#PuNclo|JoLzrHl_$DEfoDz#qz$CVev<ps%$eK?YqoP#8BE}@o$kPUGYPW3uqCCL
z$D+NK`;vGcN0Mka@9{`;)>F!eGe|&wh(7Mk)59sxo%Z_y_cwtX+#bszZP?OVko6oq
z(vx}CuSblTS$`DeDLDQO9L`g>3h?NymI8;1^|Vf|o=KZ{E=e}?ynIu^Rs90e-VlhA
z0=wN;8?wd`WT*J1Hl9C0Z+O&%Uh?D}hpd0$e#^u)cUeDJElAA|hnryb<0J2QF6-ri
z*&uNc4!QgYT)(AnWaN|f6#=`&l97Y+(y1OE_3chPFE0x4uDSh=^Vg*lJgjX-TyL9P
zz;1MrdCxWNFcWX`jTD|Q&mikyia)I5=@Zr9IrH=eSJE19o|&E3xGj0MbN#tFllwwo
z1Lw5pJ7Dvx-CNoCS}*V%n|l$QFDA_m<l^sy)Oo$GlX)&acjU@^3OW0ulv9sKK>9cL
zp&gL3BF^rI)VJnVEWE2iI=BjV^6|cUsn2t>sh7*YvK#C-c^6a8X}g_yy1%XFc3OUl
z<Kl!X%$+`vGym7OZsc4NC<jk>&QTGZ&(b(~gq$a{PIZvyeipx*o9*Fz-WcXI9-pF3
zTxX}A2D|xItS{#rEqUH=iikCzRjYM*?YABS+ZlHa(*M=xQsD_VpT~W`5OR+mD2ykp
zLaeQvXZoIpf9*9+GpSIX+S|drd*cMa`a$Zt-6OdcxOT9$+=q<aiPy68)(JquJI@?a
zr-RgiupT#L|J*WzN!&-1{dr#9(c^ma)`P2~N`!Y#807qt1u4S3|L=TdO6!4)nSk_7
z`UtsO^U0q%Jd56*;Z~5i1&+^!nUJ+7LOeC#^1OCM8_z4xxjY73c|0ky>^xu3Le^GZ
zc)OS9UWYQz{DL1mx($%=qPyYRyml*E!Qm91?9AQLlFwV<Y0I-YrWx$F%B<;NJ6eQy
zaKDXS#=WooArFW0QI3E^PkEvavGSf=^9q~>&s^QY`sB!K&c~X<JWPK6TsLAr@LmrK
z;IWrJ&nwqGf$L!>WdHiQT}Qdzn<DD2uBnz_^FjXWT-?mlH_4lOVF@Eo|LJI+uGUjL
z?=(JhzuonLXXdegU^hN+<pI~nH9sKdc<q??n_G1O<~hhLlR9_<FRSnhZA0WG_4k=z
zKh>vp^Biky;gQjUth1WL)x>kMWIfN^18=x~F&^R3xmU`Q#IudZY*jq3*FRyf`^tir
zaObyQ<GwMglt<fA80=?|n?X3aMFs4(^4T673%9882L5K?{KmA9XNB2Ko_N;VJfRuq
zx$E+Hd0x*p1*?;J47m%`XC7ql#O!VSyvqx1c@xdqcs7RJ<q_71=F(hW%o9IRjFoGl
zEcd?W_c&ZqA!`928U^u)IJWVIc|zg^WdBP65AOCpecp9eKY8L#T;;KST+Tc5GAmC@
zzZK7($LyT4KO%SyIk|YdRAYG}q*{27d|Ag^D7Tk;cPV6j<L?YH?v3rO;IMgUc9@&*
z%2xKOsb#$Ko~L+g3M;_=KN%(rme-v)lY5moCz!n=A{HDMMu~zv#n*&+y$lS%;#c|v
zcv5#==T^#40_)ZK>Catksl!#4c8g1)tDS4^;q|<0%Nw~1%C$Hhydh)v;W@LwVFU6{
z$wtVT_aOER?G>E=SRnH<!qqKYHdk191?*UOH>geIN?@7FV=T0WduO{c*dCDjWm94}
z1b^*@<OBcxJR6$5;pN5u#=Sgw>i0R<Tod5Eylxe@y@nsx&ke2IH*{|CvIVW@iD!Mx
zeWVWJH;`G2Uv+cc`goeF@ar`803I)HgR?8Z?ab+pvw5!DbAs*cd?LeJ`j4M?>EGMD
z7u@;zu3BB@Jjt8PeyjK-I4nTs?C((JS;DOduG3RA)%l)I?c$w2i;?&5qa9qUJ|5;_
z%(dX%{qq!O{V7eJP2aV+_c}x7L_m74?o$Srv$@QWIen1W+sW^_*KwWTmNbU+TeO=X
z{gAsMU7X2JH}T|HIfBD-jj{*t36|IFY|e9ezK229nu7Gpa3R_k+e_KFxBh~hJ+E|%
zhdXn*2ABNLhn$SJWw<3(AmgB2?x(<QhxjhY`PU#bknxu1bGgonS8?CwJ_1&UELXgK
z5+`${E9=21Nsc4MXTk0Q$$PC?!V%1~5<IqXC3G!U$HNR}$IBbQ`mOD@a&mc|=Ca-k
zSw96*55iB^UjeHJu|fF6GDohj%XCrl;`3xfj<+wp!S<|{4FJde9W7U`P{sldtt)C=
zyDD74cG&!?1cxig91y<hp2WHR^d8Qa_g8Q-xQB6?WLbjc_or+Io3W{T5r@-c#M%;u
zJ|Qknm1c<D4~#f%SuMfrLWNq++nZN#eNob4yR5~;HT&3auy{q-H!$t9;U(Byke%w5
zceyg=TXDtR$>R<>1z98aHe?Zd#U6jSnvWu^V0|F<I-D!uY!DxYIbD{bh%fvg$fc#~
z#kGFI0j}TS1?<oBPs7cF={ff#hqL#9AGg&!F>qMD$l1ZUqIntTle^MfzG{Ct*=?-Z
zz9mJo-L%;Y_P^7yOK>|Dy;MRm&x`#dSdMYa63!QUBss3HhV&1bbvQUT?+N1ApZpMP
zM*lC!Iax3_oZkJDYv%e5oJA|{v87n`gVm%Yz2f-ezlCe(7s$Dbr;iD6sb=ti<!-9+
zb5>7)tgnNaq5JwG=bz%moFC3Ja{upV=9+k`jMMGZ2eA9P48DTh1yb9iC&H=tWE(jD
zg7_Dr4sh-K%?vj85VH+9UH#-`<u1}?WqniO0p^3$mH&c_FM-&|Sl$08*R<J3;N~Bm
z5YMUC*2^Zo9kTYZU8k8txd^hZ&*=^1{tTG@bv)YOaudYY)ZED#qB<XJ&gwJrTsQ2z
z!Rhl~2PD70_H+fi5u|oW%Mq}<7Vn7Wx)Z*K^Wp;3GuuFFK)C0_bZ+rUi#bh>3UgfS
z2;%;n0l5#sLUA^i?7ue7>zdMBH`jB3;~k{_z@1dE{T*L#a`o2ra{emx<4Rfc4IGBY
zE<(<#>n^V0I%&6(vt1l={{Q8b`rPNfsd9hiMeJ8*uPWx)TLm%q<|0w9b){!GCVyW8
zHoNTvWFM2`ol9&6S60C7{x}n{eplUV64z@rYp#>~7I7{4tH^Gob{ed=kaInEVxs|E
z56I1v<sfUG^geXJ!v`b|!<=_?I6m21aqWmr1J?y0@egeG;c73Lu4aF^64G`7$yF_#
z$NgTc931wby)hmy3^*0EXT#NB$xvrEcK!i&PoAVLTpXl#S)?YI?;S7CQDUgi8E|wb
z>+v%MT<tp~***l!=6<t!J!d+<AE&<yq+fQA<1N@6sUPRS`Vv;QvD-}E!L?F+3g_z*
z7ET9S0dUyA5rCZ0|8(PiPQ|@0V6`t^vvc`h{>>E`B?&eIWIqT`DT?Ply!i&F@79Z)
z-SbVknD4T4+7}(*N|;*6w#G04oNmr6{l}$|_mk_}Y<X_0AjBO8MJKH}WhGC7%?FtU
z!i?n>+_l#sYd}H#FY6#{{z2@cGE2bjyDqK<ra@xU(|*F;b4ID3>(z@x;IIVA@!l5V
z+95U*>;_Jq{TwSiIl0g6<>i_X($D_rIb@x$4a*a_`Zes3J5xY<>;DC^g(NnE-BysY
zgPo--iF0#NGRF!_15U}~8r-x09pKVB2RS<pq`v7558NFeU&?Xwe9?ou5hPz+Q_r<V
z*@Jt6+H$yfe}xLWT^JuH_bFbkCEp<B5J+v&(qk-!bpc>^6~D^l_&#?6*bf|)=egFd
zPXVXLkG=Of4_#mfi?1!r<GTC&0a(423*@c`kU6Uxu5+!M2)Sct!JZ(lJ=&kRZ-=*Y
zy|%O9WVS)9@0oWRa@L?p<wLmJ0)AM5{SGn*h7Et)aNbl2<;uL40Z!X<a}ew53w?XJ
z#I0JvVGxmR26m&&;XW?@S&w1b86N$F+({1-*s(vK|4B)(Nwgj(&)oZ6!e{4O8d_`Z
z<S^^;5!kq0O@8f-Km03`u9zjw-Xjx#=ZL7`q+|;&Idh)BS>@(E{3YspdhYP$7JTFD
zeG(&ZJD1D6HIPgA5c3=G{DQe{4ByJNJ;E2}9uR4n{l)Bhz8&ZP9yj?-mNl|<69N=j
zT@=A*4ORS`!d_iaVa7X`pKqzSJYU?|Nvw-DbqQ4)adLn7U?C7%cnoabjnsv_sb?W`
zW=al>W<3YhISyPp%YQf0TPGsPRfsXW+kEN_PF9weUpbaPm?r1MyhPLgh63-ueID9B
z_D?k9F^pxqknhhg=U*-qC~hzE=!FirO}5W5gyZR+n|${edidCR7x46QhVhFWp2S}p
zypzYW?*;fg4E{MsOuaam*$>WQ=Y3uO-7G_A9$%f67XPCaoSdZ@y=-kU=gl504dmC3
zI?HtV!*srlcZB#fW?kkLIjA8db)T8TX}$^P^vzo4Pwi&#F24Oro}oEaB&%;7N9}i3
zK39X)oXZV4xE_cZ3(Y;eo+V~Vxb*Dc_d=CjQv8R{Le5C!n{tO&`d$lo+@>okl-KId
zb%Av<!W`d?7Vwpo|KmS>bQ`~Qx)}SScc;x)6-kMF5?v$m)uEODy8A>n&Di&5+djPI
zU4PzBW@ej?;GO3`1QyR_GoQS?j(_e?1&*i%TLqRR@(6rojFfnGpo3THlL)WA*K+~m
z!`B35Og3>I<=@T|w6L2~^|ro%_nGIi{G9XoFYTYAJ~ekb*NSO#%-7zxHY;5f&b_*M
zE`RwAPsYmqv)O}`mH56K7dQVCVZ?9r(1-Qo23G!nx$8M+SL_kk_5P5&mcR)SwImq<
z?Xy><{>Lxn*!9)K9F!jC6wMIOtuvRA7n{f75IddU@yAYn_EnJf5VzEIrrzauz<CX%
z)_Qe5pHo7gO27d<-up@i*%CXrELwi2^7k*B$ImgtO^)^LXMz9x2|W8EY=myT@KgEr
zN<^_!b+2-R?ML<Vt3I2Z&u8b~Q_s)(W#@Itd%HPgtKu$kzRy-Oe^cSX)^~(oSWPX-
z+%vpXpym^+z-oOJow&)&X5ZPCi<cfS=hOJRhkws)Z*V>Xxuq|(mGyGQ7x4#6O$9E^
zT)-zk$yX?<nL&Jlmo0z4LlwvAS*FGybs+5WDah=Sx2#Z?J)4kf$`bL**B9`e`y<Zh
zxn~ML%LQ$-IWxt~c{>!$COM_@Znm&61L=`xa28(M=%Om;S8euX>sh`g)+wBM7qbO^
z%vi_k_u7%aN?e#bx@ajlJvXwR;c$;ypmapsm!EleAG=rCKK}de@3}pb3`ELO3x$6d
z=?c%A+$+#^m4UZMR$izf%9^Wmj;@F~w>_t;{!>#mQFHeF2Zco$9!qlTZOfN@9nfHY
zATWh9;P?i!u5}Z5Z#)mx_~4Mo`Gn<)%&wnX`1-hI`0TYU%wpX;%>{QoWMbI1NPXkc
zZu8cIeEeDyRTOvjl?VmiG39^r<F`qy-g>_9$GPA#ZTe*|K7}9;$*qfj3m63D@D?V@
z@=tu>Wxn>zBEDr6LfrlgJ>~@$*YfSWuEE*2|EC$V;0wO>kCn^>R%VIp=ga};1%H*#
z?EkVv)h0H)VSg@T!s05|%fI%W5Bs&doB8K%>oM<jd%|aQ-<NN{F1Ny3!8Sps{u2uB
z*K&-GwTerB{rpeXf8BH*&8s}3E7Bx5XQ#GuNPE{xY}&%gx+f``Z{<e;f$hJg!D0IK
z*nWQdLkIY4&Q?pRS0}I^*q{Ty|2KZKG5@xX73NcBi?SO?Yx8g4v66kIV4#55_WR~{
zKCEKD^8J(a-fIec^+|#pQf1dzpSVuu_jGSzpPhY#ZT0(iEX$mk75}77=bt{AnWedQ
z7r#+xBfI^?TV@laR5_!!PBNXdN|oKg=NA9JE+Mm&^^-OBE@Kv0cF~%D!`tV2ptSKX
zorQ0SHKRb!OH1JiLHqbEr8A`d9r5OSq<30s(ucLIEtW6%*J+;PmyX`U`H4eFn9ZPD
zbWguKU;KxYW^TbdgsjTG@b9<F6fo3Tz<u{cl6rQ}Md9U+Yxozg*A}>Nv_N*Q=5bCB
z;iGI)E6iE*%6bL(3tD(5ZSxb(yD*tUB4e5Gj+Un;(riDa=NRakg4~nf5zN_hR-GgF
zZKUL($70;4j;nJlyVK9Z@w!F$6LS=&qr0a--=AZgYY(NfS<Uy7uDfrhX{yP@TbX`C
zKsG?etY7UMTVLX2@cqA&f;%{^=hpKp%XhGC+4zyodw#s&KQmYU^)D(o_Ds;1&f*GZ
z6E9)o(%HUBfL)`9_sqT?&JD9?@w`o|G*5ZHgs*J1BfH^`1A+@O|8hC6*HI{2zd_Y)
zjhR3u+avDVb-zr0oWCo_ak|I+9lw!a=Kasoh3EEj%sIhp{?p+ixBDAzevyKcW_}KJ
ziqmbEa7_Fy%fYymQ^;85k@>EuXF{7K?sBESHQ`^^dBm)$OkL~B+IWGv?+=&@JkRFc
zcHuu?hj53vI-?LT_vujf>NB1K&)kLCTOwDP?=GCqWxsy87W*tgRx^Du0h9f`@{Wi0
z33OkR5%{-AU0v?Cq6jD)K{)x)RB#z{@K%7v&FX3akBi!Z93^LEwI3RC)eGkG{7YHP
z&-AWEFumnJ*8%3^{I;wS97-QHfZGoWt$YGM-{0cqDivbW6W_)kv+k1E?zJL9w)_(X
z4(e-w!#B{amRmj?GPhgq&B+~m>mzFu4-;3wv2^a=vTwk91|FQA3@&pQm)`^DTadeu
zagK^B_s#`j+#8O1f!jXFatja6;_5x7$aZ;o6IY5VBYaHcVGkGQlWQly?)*JfhWp2j
z8EoMjS;74MkDhZS&-}{uaB~~D{Q@(uFC22W5{$o2=p@%7TR(94z{JWKs#xXbZUyU8
z`32dZwQt=*uIpm@oJ(JQ;GC8N+1vGX{S2_5VEW=Gc5r^&Xw1IIr;&AnzZ=_%Ly)~<
z+c~trW=S_MW4~Hx3-^CXf*t3_iV0x%N|^F+WibYU-J#hn&gGE$iYxlde9jjK>pApK
zLGJ1D_lMjOcJ9FoupW@TcW$<Eiwpnf$Sa%4eWwdD&a?W%PL@Pw#9lk&GhA?Ya&8TW
z+YizY!^dM$_et_s#&TVJ7su7H!;1UNzl-ck=exk|gy}hZ@;%22woLAmECpaQ*^VsY
ztiNE+u{wj3)B5&GuJtyV97}tn*(SL|+TIs#LiR?1^vBMg3vPFU*dXk|a0l%7SVwm5
zrHqd_AA3OVOw>61k#lC`LXPQ<kn!18e#G5cAbrc;9R%CG^!j%8Z+9W{BK8K)IsZ>l
z;kvse2P`+s$AUv@8N|HDYnxas7@AT1@|Sf7N9au*uGpDs+yY$dIZt?cb6M&`_TWr=
z%?UOGWS5JV2&dM+?{K}74v2DhS^fuyZFGD)n7+e4mD{RoGkbfDB$yA<cgb1-o@POO
z7>)>T<4(B=*|P}}o1CY_p`Ln<qkJ=D9p_=Y)f|5k0>Sn%uY%-9kU9|lGIuIi9K;rR
zWXiSFgaaIYyjt0u@!Xrid8j07E7<M(*u=ncAhmm{u7cU}TRFMvrfGA9zl7Y83X(&{
z3%1PSw)^v*Q~cHpw#Q2T-19t6fX(7edCGOFjE8gad<CvexiR2$4bnI9`y#Np;bQq*
zimm@Roo457Pk+7*EN1u<GWQ~38o?z#eFB&6{wRp}%@FRl$4j`EAJK*TC5=6d-LAC>
zY_91)buNp{V9q}=kh@64+g-upz5i}<Y<##J%m&&0;6W+aZ6LO$xGopNbZ4&eYaHBq
z4+=QVuR_KfgA|x}=3KUb>j9~qxGogzj=j8)I|4vr$awmxHg@YWH?C7bDqwXWIYn>8
zSTL8;d-gtk#9173r+0Fv{;&tDEqNx&y>?{+SI2kA`W}#84#7h9`jwErntXvf+|3}l
z=xJSGefuV^U~^*S<upEaj7@$~2>0{lW}N(~LOc=oCAeltS97?ELH4D5P+t$$-?g_E
z?5CaxVOFcd{#;c7eO%8}Z*oQ&2!rJhwyJ{7mHu^(GfWVY2cD`_a1}V1bEm%A4^N{Y
zHynDN$G!CEW{#klEL{6sw7Fi}EZ_`VB+NbO!&){m-8^u9UEJ!=&7=5&i^VUL+cFp8
zSCHP0erXPM3B*_jNbDkm9rw!1ko{vI{=o|uz-|JulNMbD+g-v1IS(Hsmd<|>>{bwa
zvC4C<*8h-up+Wq|ZIiediy-ax{9lY*2L(g8?fJMk7sPt7+Xq6%g_^Swdu|@5w}JBj
zNN<;hAjfAth&#?sPvdar_{;JBr4NUyIzMMi`VKB5HCL{UPiBC_5v2aUCc@t5PA=Sc
zZRFtT6D0re;7N{43QXL+>&)QdTRC>Ji*QwPrf>D-;%bH5g$Ys{&#uPx$YBO}tZuO@
zC+FvC$oTKqquE?MJ&^q@jPJs^`n{UK?mb=5#+7^uV*fApW^fpR%u(nH<=$9c33g+_
zHbL&_Nq4zZt3GgLxt-@MyloB^2dU$InFThd<C+3IEcYFRjLm`MVff3ZNcNSBJ2|)N
zD}v)hYeqiY41+Qq?nGV4eUC3b6oJJBHa+9Yy{`<e%Ry$8r%J-hw3H$<fsbap!0QLC
z)prWC{dmt;t~Q-_dt0ZV8n->T?OXXghM#lkO)lYjLxD<5NAUT7PG2B*K!eQtk;NmR
zdOA!{hr@^8c-~~NSs-x`mVduqV9l8gd?%7k1QHjX<F8co<K<@d<qM9t<v-E%0c`%l
zq6K`BDsjB&+a38I9Sr9Gvxc2NDCDj{>{T)T;N}u=-ydXFM)zm1dJtP<%~AnRvFAKa
z$4dFbMM8NWxrGR9kMZWSedoe^JNyh^_4YLa;m2Qt)z_aq#`pe)E`L4qHoh;llLUCa
z8uA8s&EXe&WC@O=sx)D)dmm*5*dlxQ8Gr2M{b%LH_epn&Krp8|``2rbyO_`Nm-4AP
z=<-%6`0@UaoX2(JK_7qDKQFLfuc}?<d2iha?vEU8QsjG4a*p3sF^vDz3uFFFNk_1_
z!H!wr`J#(Uw({L`Q{n!@#>={g`4?Y7gbP3G1O>hYwIbj+|2V^xE1|j*Y%a*391FX_
z`lAnt^Xueg!^JaSiSWb-L+;hxDWEK<c?>d^kyRkZZ#-W^u-oV~pL%tYfPripSI<rl
z{vX?C@I1c)Im=M9S5!dy$1grk;U)on$6{W!W1@VgWi18S4bJe-$vw<>gKZn%W9FUw
zEK3r3wQua^K6<;Eced(z&eYeRc~rh}@QFE=^JfUo0lOtX=`c?&uP!*8b@w(4OfcES
z%cXV@>{kVLalRkBZTU0PA?yDf+(g0Z3U3?|;H%K)6SndYH0giLFDWU*U1ntn_NU&A
z+q`F1Eas~|C(WCF%bSmRf&y=oAS>9s2}+hc!cmYpa8TGx-@2Pm?{G7}3F}hc*YXp1
z*QwX??{iuwV3V~*pqyhJ?~`v<U^g!L`hr&_eHwph?m_-oIZ-~_FUNQm-}%Mo)L;&l
z-*zoXFs5CCLqWuw|4Qj6KBt{!T%hy+gpIYi-mh9JSlYUf-81|a#|Fu20gj_f`Sr7N
z`5#RGCs4FU2<#t_TW(L*<gy8G;$40>6P#X`SX|-X|B{D)^OxCNPag}j_4@7v`vIi3
zQYa3bZ<m%o<?pf-<`2p7;5U&}<KKShl)%>)1$?UKlzHtIedYI=w1aoG$r5n5@G$W5
zO=M;f*tyP}TicLDAhqK<Z!PZ(LDQH;9QkG|_-c98I9pcmfb(_ADMfyl**E$4&rTM&
zbMOY=jqe)-HXAcAN-`Pp#|d}yzxn=yulmG#!Q~V3x!of63MkczgWV>3nSpOdtRVk}
z30L?s?g;RgS!l5Q4u1d+Bm1L?e1+b*{JULa_-@3@@=x)b#J6*Qs(?%e<P5}1i=udQ
zoddXIG@beV_15txr}7H8WhnE%dc9L%(wz!{(CKml+uF3axaZvB-^RBB9BwZaR`78r
zP7u&)<m1TaSk1Qa_ICb`6ir^6#ajGKnht!1vpV@ZH!}-}o$BM)-{B)*dP|mLqa0+N
z?aB?1e9>cK#l7?CVX%3iuw~hO3Y@23ai0+AXL-T*_pKDz{UZO>I4YO^;y-y+gxAt`
z3-9W{7{0;@&-ht(oZ>pXb06OU8+Lw=kMDW5q{{Kna<Aa~dar<g(+gd}Pj}M6;Xl3e
zqrfKH2Yf9nA_Z<P{=_H#<14)WxVI}wpn2+7_S2#_xEcZ<@-IE=#~aBvLBO=sQNU~6
z1HRc$^#oq$DDv6`mxImfd$?P`b%!};N$>`+yARm;^9eKU6j)=tn_F1zyFkX(JibI;
zc|lRZ7T%|YMFM|suHn5Gz%4jy-bD79J9>H7ER+)TtF9MFUkF*xQo7BL-)^V6;EvL>
z0`LF-6Y$>joG&r^4&Q~`Ac6fwnp~?tNDKCV<P=o7Bgx-(-9qs3m30Ev_EY#~-M=p|
z{nSeC{SK1+OAIXp)KC9oogHyez-iG}-l(Ma0-Es&f`wIO0$={0=Gv~o%bW6Q3tRq9
zMSd;|$ofZ7hv{JVuMKDBtC@J5|LA!)a9HQ;o4{8$Uya?&_>n+jz6YoAPY=H52g113
z^;P+we`V!Y*bpaJ^xlJ)+hh;dW)5+|n9w~OZ5OVB(=O=zzu!HxcvEyBXN*l$`N^xa
zb1B%|Kitc>yO_cSZQr&CH2T-@%x2v$aEn)fZ_bNSKH(n7`l-a|O}wWp{CSg&`MJCl
zrt%xqAo`H*@4fjuR?73If6Ef|I>pW#^TCXN-xd#nrlYR}!WR|_sBFH;r{i#z?^xJ;
zUiNPid?~?_d>Y~Myj}CddDdD_;;wQM1N--)Z>j)CUbW!7tPF0$<$eM)Rs98Y7Jd})
zI$<hU?z4g~Dp8#`>bs?YPBx_Nyye?!zG&wfaJj%?(kVD`DKEcme2Bo-Z_jz-pE(Nd
zT<*Xx#UIMgbZ-)0)m$Im+ZN$q|ANwUTo(_x4*&i_5S&LrVs-WL{Nhrxc<Yz%<BjID
zWhsjn1e>ikO^~mpkehFI#RRY(-CQTZnshhbhzY!Wf@U8D;_gA(-A`&43EEpt1m}~s
zDsKT>K`+zmE&S|X`9g(_RdkJaF8RrOw7E&}r;?V!zRlZtKU`a4zSogiZdp*jIFtDy
z3!&6h_C~hz=6%z?s$V;`nukT{J%9ET8^Ms6mu74Jo)JEC=@a<QTaa55d=`iac(#kS
z%k40aVK)}oVxOuIy4p={!qG{JZ{PBWG2gM`XxPKRZ?b~H{B_4!{s}q%`0`IB@{74$
z71rHU&%e~YS}^geF4(-kruE?b12T85Zj|}dRe8KyWE%L7AA6#^F_Te9#qGQKM&rG_
zyPFL7IVOk88y`Qc8UHSk!>H=1c1__iGbSA=egQ6b0gat=grA@67F}6x0xpNM>vFi3
z^C}7i89(5!W<Jf$tMHvyXmPW^j*T6h#lGzlA7?gmz1-Anwt-^_Z~gssyrHuXna?hX
z;#WBDD{$O+1LyhQT)ajfT)^Yk>g#&>E^ce$Uph^IKOb~258EtZ(|yd`ozpq_y4&WP
z8*=~U(@EW^AY3;^bj>Pv{uTQ#@MV0M!M8?uDOc{}BZ9|I_HkZjV3pyNx+T(*Hiz>g
zI~&-~rm}W?Jmv}DbTP|u3x|v8H-Vx@2e{@>`p#dp>8U{Mi$1>ajjine)r>6?_8W<|
zTQ`WBI+zG>ZvDY&oi1x0_ui5BhUa3L9Wk#2t>^U#q}e)~FN&EVV6@^fuffVfzS(tw
zf-i%1OO}8C$!BM($=?)nieL5UUcr+Mp*%|j5A(W+M{%ZkOyM<NDkG=frolhawMY{b
z4}Gg%nHx3SGShe^&U@`_hCr|y4_|(c95|nZ9C&H&yxM?&`^R5=`_>x^ZaCV*x4>+%
z(1ry~@-IKMiKeZM6>MB}S*EPqj@x5V894k7n6>ir_q)h$+7!v_th<`e@M0hDh7J#K
z+V@$K%2Rmb6qpUN`(^)mzJ*OoRi7ww^YP|i=4BLhx0vz1QNSX7nZS-Y(TZMox&)@Y
z@8Z^7DIlsiqfKQiW1P}|r2rL`^-P-Cr{c{WLSzM6kM?oSe-$cq_UUBVm98uJRWsDg
zuiv`E^;he@kbarC#ahF4g8B|BTnnGf(q6q_rdi1;QOU5|ru<=BUkRk1Mw|`E_4y8$
zVf{>rnDdc>(QZ=(6i==ZeBZJ};{B2R+;_jGa;<#u3_kv^`9aS7z{yIHn16jjmXB>E
zmp;qj_gNXrzqV$VK;GqPX1CRv%_}PZn^`RS#9J6w1|I*<oFpg~p(CIsq-<oq;`<%`
zU~WUcLZ-=r7JEy%_*WP4xoKK)PJUehE<<lU>EQpeT})-2%1i;RI9+~&^UDPeH)^mx
zU`P>>O#LMCwquKs|CtSfYx&o5AH3fuC={K=yZ+EKk&pKZ_*_rUHQThpf`9+M|6&23
zFEDeKL`gl}lWT6{beXTBVV7B^;vzntW9PM;J3jLrQnHq{zkHC_ELKgR;{k*Di@?w3
zMcbC~%bfeIvHkF8^S`G#`R{lASIiE1CT!atE>PgN$Ta^9hrmY{$ek}`-$nSHrmvS=
z?QxCIR_-sy=BbPL+uEm_-)eW}55N42PygR?bL-{20;Y$)@y}g9-8{R{mFw|0F7xQm
z7esa>#(?uh#l=YOd%-i*dAjd&)-5>2I`h~+&deDJ+&xq03TWN<Xg+Bbr$GO4SN`zd
zLW<uTg$1|&FjQ<laLknZ!$#=<u3d61Ka07`0=|iLGv)FtTATCs9Z!{v&pFSxeZK(z
zp)^Io*Nc9D({c$Pr(pS%JN*57y`=s(IP!bQOoNaAANx90pjwjCV$;=d?w|@U{sO0C
zy!obW0+;WVSeS1&;D0^eUq(Z^f%m1_J^nE1PaKg)7Yg_%P3GL~$ix3Kt&QWOxU^D(
z-aLW%U*veUZTTe-ly-vekLNCP*A4%9XRl!}%Q4~QyS2qYaQ4S#W?d_IHN#f32}V@w
z3AC&Y)Cc8*+fEJqCls6ot){#Y>fFsFxTe-xI{nE8zI$cH(q79`xq46X3+i#R2(CZV
z#xKmeQ>asvMPiLnAm91(hs~#O-WR%N!_P0quP(s!eHW|yyo(yCCX0m^thg%hveQX$
zlE4qyEyp#u-(2qI-Ks0Zp0`tt|H%2xe1)&B3g2dV&zHpaNNBE^iP`emD`c{!Re;l~
z{nP(^x&N6stnzP4op>{YQ+M`4z8&ooxhpuD#PV_?`A=}W3C3Ugz<a^OjoU1BnRF%7
z3QcF-l{`BySqV9(J~f-|&&~Io^D}sS8gxJXotEta=fAdde!PB*`=Gy*@b@!O0w1@r
z^KvotN-zG}#y5-cEnj*5Y#yH9MZDA8nE9HPH}F0bcxit8LI?lNxEtJ_hx~=Z_A7D9
zuNPJ1k)NqLC9PSIZ*?DU-Ih+Xz$HKBxYI=}Ud1vC9(9hDd9pyCKTyKNJn%saU%BQj
zUccE2=H`=+C^3I^;#t2rm+zz}lgJJi35z-GvxO=bwsFtMlM&E)FwNY*VuhA^FNfgn
zykd)Yjg`ELD^&!%GfgeN)pzs0yT{KTb;M5aw`d55q_KpBRZtuMPllOV3h!L`Hcj#o
z3_WY8Aedh$RNs{=5M`*MAtB!do)-tjZCfIw>|48Mt;Q==Grk`d@A%nF6XYD1sPL(t
z=jKaby~=iBMu*78vjVJ@_pJH+{EPUexiN#w0nYE%g155H@s>Eo@vWAbE5Q6=kEFP)
zyU29ccESEn^T6TzFlHU!fj^LYsS0K!^WJT*=Dshrnb&gD6uz(<R*<r`e>b?EZ0*+r
zn+tLmGXAF^#?Lt2jDO+bi4gTjlEH71cr6?hdEDRq;8`Rk4X(dH(){N*_?J3KfYnca
z_ma=zx-swF6_ESbWbgX&Kjw4fuYQpL&f_5UAbda68k`qFY!H@Oro_v6CLP61d*)2x
zZMByM>+6^%2&SjK+rsC1K7#MqB|-kCe~@!CVwezTCV<S!Ow!~%o2JRLePJK>3eys<
zU(H2eb4&JWfXz6$M3#5ew@$EoLF!GOv2h72IfL6-3Hv_q7^gwvDeQI!Upc!TpZ%}%
zydgquyiF@q!FqgTA!p$7Dcl8zCCChmR8RhurcXG(Z}I1=+t>g$>$qMFm+Jw<nJRqF
zi2ZJbtKCuj4>ALU#qa$^5r1aQ$@3+vf#<T}E53ZG4z6~ag<v;;)WWb%;#6LBpCrCk
z5s16;O>MYdrt<SVO1;XPo>9a1^SBb%&9egBU#--^?y{^#tj{fKaYC{0<_^f&mX=&^
z__lb9@%ep++|_oNVK3J_xvM;Gs~~qM+VMfodw{v&hO0d9;syiWE1LqjqFE1t-8h}e
zo^RRgMI0XUAnWM!e{}MS&w#A&eIeV%!xXU;tPW<znY<ug4ih%Mf2p5%H6kVXuF75E
zv%l^Kj=LoF+hDhY)NZZm<<WlwF%!ghxm(VsG8eLcU-dR*ed6ACbNJ0yz2~{KqZS@V
zk%tq(@ek62j4$l$=JPye1`h`T-|bxQIY086uAT?BbH557&$T?rzND|pkhOd;{rA7t
z!~2ds&F*|&wZULN?QV(X=~(m;tUpK#l6G%rBJQ@kVAc<|Z|$=~ys!5D;$?Jn$LhBA
zy4n0(>#})&vYy}-HB#U=yIBP`f693izF(Gyd3U^<#9Q<TvVR`rrX?Pby<Im;l=*7q
zviYjM>hjn2mVoWLXn7oLhgHE$z8Uvbd4=>LXV%vQ?dQA2%Eo`0YZKUPkU6=j+&o-r
zv%%^)4|wr0?A*eeECg}mjO@=~f0d}O;!ZC-4>lX5r`Z>B&nAc+(4fSZF!=<}`g&o$
z6LWg`6dU(~)$I?z!q0L`9?S-*17ZGf$ox?6(m(LH0*S-$wTei-p7tW%`%lHdW8Wb0
z&;=9VW*FDIaoYyz!TGC29QosQc))()<J!k}MJ<ePefMIxys6|#F3*hb;B?g|;{f&}
z$V?F4-FFZyUQ#T~eYUoeSN(q{@80+lKDCn*c!PH8@<-S5^6EF9<bCqz09aqp;W)6l
zXD>naZ$^9c@m=HU;<am);o-l!nb)pi2Uz|`{dTZ>3cic+b%v&c*-1q|xCJF+_$Tzl
zf%zc&LD<9YA@|YbEj+i1EqS(fW%K^g>fmkcGzXpk!__~n6&yy5+avhhTif|woUP;E
zBZgRmx#mzGkKQ!YGmRS0Y~j;UL5%Uvo81hz?`3Z@*gU6_=U{h&>|e|SIcp8XcAa^S
z&r}w21`UWmU49~8;Zw*sci75o{s%YA__+eSc&}^R;o|Dy28W69weRqF`XG)lOV!Su
zZ?7%nE&v(MS=^c5L^#(-R`W1^+r`rpAJ0?G5yAJ}4>Aq`vWwBp5$>jodXoJ6^bz~!
zLGsJG_VRRoT*NVpMGzckAhC`gnLH{5CwMf)1-J$8@`K$25|4DQ<*}5X1s<3E`h=02
z$E^`;_TDQCI90e0fZ5?0*ZJB%LFT3sm&@><D0>1Hmp_I$|94$6C%@6!8DMi=9rgJ;
zHt6!VZ1?Ba-E7Kd@C9<d8pysGnvnhN=jN;in*kD2y$iX+6vT$%6YcKYth1)_W{N=0
z$G<#d64-2z{Puo#z5^!}!SUmn2H9Kb>+i_dxVZy7j{#D%+<_5Z53`H7@zo~y7|WgF
z;FdND5_UU(*Z`!a_R|4A;ZN)3`!6N)Kl6EP9(qPWT3n@3#CvXs1;Z02?yAM>%=Sn~
zYdk6I;1jR&;AP>E;Q#e)nVHMEH9}uMHG#(?W*s-=b!>hkWT9UyVpOJM{w$G+{pj+q
z@&^vjm7ViFN72%2C3u}fk=H#o&MWF>SDd(c_eI$7L<=^svp*~m(#dG$oKmoZZ?jD|
z*u2&Ivv_{5R|3adsfncdv$M0<SGAqz%Xy%s8~cS(=v-2-d8fw<)<`cC_CT3a@<s7R
z>OH5I^E^||*LukzVYcYXCDxmd=J9f`&=<PGc1-vPALJh7m8<t~IGhgVwMc2_JsV@i
zb-8Ri-#Ul8yxTh2c{n9jg4h2A9sh3j?S2Q_lI^#6&WLE4>qyx1{@~lld)#;f`;^&J
z*>73Inz=cK^105v!JH<uho^d)4bSIQ%{)%Z9|YZ>Ww9v+TxHt#rQCdC?Qw2p{%-k}
zudjq3s%o$$ec<QWxUq_T+g3TY)~{SbXWHMfwmmD5_ELH+<kMo$=N<^z7q{)iR&MdW
zd*D2%V|j*0*`-22n&~~8;*M0lW~L~<-t=}*|DSc$=H=!bySIuweI76Jv0)c)y+$%i
zT3frB<)5=W%CTQ%R$rDEFnpFMP`m1;sphknyoL_y9DN`6^J{lb<Xc)VC+>bkjHl4=
zBDZX<r2tFT6u~X$@>msS?Bi}az{Bn|;UVAek{dF&W_j{m&(Bbg|FnyfQ_9=?P#A;R
z*6Tf7DLeW3jGoG{zt@*!^_ODc)zUa>zO>Pj@2J5HHofZ;`6}P8VB7snT42)zL-{L{
zqD2m}uI9Vl)hKbg@GQGA>tb-<;gnXi!1TS#rOndKS=y8M`IE0M<vaFaF}Q4KzT3jS
zE<_E?2Bp#47fpCsD)^KmHtO+yygi@2XwFn~k)ykL=b4A_{VaVV$GE>lpi-QXZ$-_1
z!DGu?RFuC-D?M54p;Yy3oBDjaMdr0z8h9VK?qFGZAV+ff-gw#Bk90URI-J1nc*b~F
z=x4EnMRw<VzJys{`GRJ$XlqZrU@9H`NUUVX44%q)x_mzBHQ+HckXwq@>$2TTd?&_!
zY#HA@3l)B$KdeIit4zdQBo+DGQ#x56u3<3(sRLmNRb}%R2M!9Ff7>T07B)xBMq8Qp
z-sC+zH}(1WjN(mA4L3%ZD<7O@>Y9F@hskp<cumzc*7ZU+uAEW+ZFJN0=sOqQGqcp$
z`HYzO3#wFk`pUlWp6Bl7I#U3tBi1g9VP9;gtlZvuf%nG6Eo{u2kMl{m#c(s5?Gug@
z_ZD96z$ko^k(b{?z@NuSe666&bOp`}5*#90clNU<`$m~22FS6>pKBMntj@`0^u<l;
z58Ev>-4pVx-zLs8v;6skJABF!^({>|m~I{2E>qpaz_+_6mH&j<WYd68kInyc%wuB-
z&{EIOZZr4Y-pV^udzQj<205X=W*t7GbJZpnw|VgjC^169O<j{O^qIcI;<g$AhbfXg
zk5?Y&UGUS!Tur5!_t;Tu?zUSy%#w_JdAk+;*%>Rf%zNg};^kauZ|<J8Mda`1R&ZWO
z(U)Xb@fK5?Y#GFMMREbtO`!z7#XApj?%1NjciQNc`MQJ{-bXtx^T;j~P!xLfQs8h}
zw*u!3ZsT6_G->V`Q)TM;pK@z>$BUTSonaS$x19a@>pqF(twGFp;}-KAxZA_0CzS*)
zZ)RI&@x@%6&BvAQD0wI|mhp)M3%vioF7_^O;H>%PY1}5P`PXuJ+YhZ`Vv%U!Fa76f
zF^7SH-8NcZ`seOtymML?u}hU*V=DLJ=2O#r&F<Y<3{K0}#3W?Aqy+e;DK)YPOtRri
zn){#aQYMSJoK6;-ZBUy@W~BqWepe&kqZS^sjPpCx^_CRzEnLyZn=rox-2b0*dN$9>
zvweJRQryD#4?FO^vr>_?;NQX%w31sIl-}weMDi_V^5UCYahYX#$3-D_R$~$Mv`Aj3
zOHa)O+gpV$?djqx_KfA<tJ%g?uyc{R)`r=_^Vci$u@=nZJESWmTW0ct^Yr5<?C)3g
zur4eL;eS=Ojl2Fco3MdCE8E<eVZtk-W|-_dU@U!fM>06AuDi&|Hs=8=d$3-K#P!?F
zT&m44SZh-nxQoh830;nP!`yH4nD2A-cJN#`!-^j1eT;^hk&hF28hR!Qd>4Lcnl8VZ
zJzP^3+&}elujbHc=HT<6@CjUhS+vd-eB8R4SK-bl*4E_tQj@IsIP6b2bErDZ=KmAK
z%F{1!l;xMne9nW9mzaN?8wgG#FO6LV@9)3Ek+^<|Jd=vI>KxuXd`H(Ra+Q{GnodYH
zmHTdS-n_zMmjJhlxJ==NBP=3yA?6BGCh%-*ir{<wvB`|F$xE6~+K!byaR%G}$3a3R
z$<xg<UMv?97o5fs@@o<Ata5qtjWa4VG;4VIFBb?}_*FgPd0>&qE2VMOyqohF&-ELI
ztk>7>;XCr0g{yhnIrAr*cd&2ESfmAtn^{k1^1bf&P&gE8Ah7O(2>;d%tg0yzN8$JJ
z?&R78?kmje%2%1RUQFQ5dk2BvQ)*@BT$W<{?-<OzOneR>%Y_F5+Yg=QdTYzUKeMNe
z=~?kxaJrhlzJt%8R-C7Pf*z}r5+9#=$y>4O_s<GuWQg(0PFpJnvfq!jj!&X+CAhtD
zZejqhvFTzi6P^ZMmz+etEH%jfXrBv^b#&g!8C;kDK=x&U^ulmz(NVr0y`OxS_VDqB
z_<RDp7bdrIr!BAQyZzjr*Dmq2NF9Z@Q&iHY@paWg&KH>9GlTDR_7R?{Dji<;)>)j(
zxmox<1HSRqzL4Pmzxy4y{RJ}5)a3*?K0)j^i#~Hqna0QavU4)HU%_HOf%VWSE6&xW
z>b#c@-sZb~riAm=97sEQ!p=Cp7iRx>m84ek{XY-c$IiW05bS4=Ss?tm?*$Lb_D1f{
z50$w$9a_dEeuNvG?<N01&hL9uoXb=C)(39CC#yT>S2szp-C>8%@R<0l2dke^tI4OT
z^O)E5ehY7VxG-;SaTi$pb8#hD?}_Pqz-=>-o!pxl`7_?eateMm;B(x29PH1cVo}cX
z3+IFNfYjW)a01LW+O(KmIhG5>&BZon!13JYxqz?O5E1T5Z0<acch_-O%FO1|xIT|F
z;U1)4^W-&T-?`C$O_qJ1*?Du{X!A|q06BA_f+vjo(89^w&sMka-a4(xr}R^tldpIq
z*Me=&xLN9Fv&Zd8hr3%u;5KhtZ!|ofU-q8|`|YXf9KI<x_wd?I+y~|tUgl-*3@hR}
zb^y|kn);6u?EYVuPH=3yxt@(-mLYH7r}I45IUaF^s6+OnFARCkd#S9O)6f92FFYq+
zkk>H^a^~#YQ`$V8bG*T3gTe-c*Vp{#ab2*Cm(yeuul<hQyh*~%e93)o;4s@;gV>Wa
zQ!<37wM_%AHz3@cZ^p-Pu)6J0khOLpUp)8^@&@pPDnsrA0-67@O$DB2m#wSiyrRg>
z>AtI*{p*WxzDZ)tVE=&BExbC3Yd=db@6T0Fz-glL?j7#uHz9X2YM!uzlyxA53=9m1
z8JC0k3!{BG&ZmFim0!uhtJliO@pPs?&kIAyJy5b&mV?dKUTzF_BS`DIE6QN8Y~4=Y
zN2*(Sw<U7H-3F3};nWk-{QMU$@vPDa;=On2JfHA%$k|ixI@$RGTD5rJJ__cQe%%K)
z>&aJE&UP_2aJZ!3xx=faagR60=Oy2?myq*z^pzUHc1(HY%csJ@%KLQ-WF5nj!Y;nP
za|ijB{)U`M46=LPqlw&wsjgu4;x5~HSM4w7tv&aYuZQ73`)-vqus@_9v2j1GfTRbI
zo|j3iV0VDnC7c&|TVFHq^nbq1>r$-2C&gUA_xnW%N6fz-{`MTm*-;>MAgpuW8k`=V
zJ#FVc(b*19iy%1=PMz|b*FSa>&*U9*AZ-l>28Jvf$oSJ~gDV`#@3pwi*Fnxe*8T0w
zaeXEO|M$=LdDMTqaC+Tn;0yT8%y&~if-lg~7Hqak^g*r{i8pwTE&b2lT>Jp;W{_PM
zZg7G9cHZ>_*OBO_Jp6|8y!(vJ`0Ae<@vb=Vo9}MC2hWQ-3*Pr^kUa}?6Cmf4hrizh
zHfK_Z7~h(|@w^Y_+w#m4D&{@<FP-oAr&Koa(rn%droHS(=B?x1V7Cx#NAzKTuJ`(U
z{M_qbgZUu)pM7WKRK3&7bFt2zN9&L@w}rxNo(;m*ycwJ3@+8MU;hgEo%@sE@g(K#i
z0DmN}3tz0AI)9ti2JWf=K6a4)Lo7->Iq{JF;hT%Az+$>_4Sb)3%lS?+L)K4b8MuPe
zh>-aMHl7a1p35wI$Qe?1x@WMRJU^4y%|#ZhH|t_2pF)!%XOi<(aM+p8x91DqF2F5#
z`!;*UaTfld#UJ_lzlrh+g-+orY=+c5Dhm*6n<Y1wfb9ag&8_t<Zvm4i*so_c)^fAF
zzR2m!C&klIRK^pWHk(JagN1MMscdjKg4F96K+bdmu`T8w<6HN@1Dqd0{O<aBtS1sI
zc@BS^#BFiyG<UKbJInEZ``LRpit}hU&g7}#lHpjoW+&V%E(trH1|`V71`py4xdT%n
zd-x-(OSyN%P6Vr~6>jHS@O3YjN!t@n_T_u|X7oeG2*Pf+g8dG%*C}l$-`CUj;CvDG
z^akIzw@m!{ADH-KUOwP`R_?`9U9^CsF8dJMo(s`ne;2NC2Ad7i8?xpwIGuyoFr33x
z#`S($EbpD|_rdCqh1-J72FWL{l;mSAxCPcD7aj^WCoY1Eug&cfxPAetk(#m^u0CU-
zIA8D9Sg<+?rNcab?XPiIOq1blis9yCIOGftckA8aVDn7YLB?=EW+3Cf!ZN-SlUMOw
zw*3lLk1Th%crMQe?ibvXU+v=Fc%~U{uOd?r--HJ#VD+8u&V2j!ui!bX0I8?AX0GBp
zn7f8=tr;RNKxQhfg`8&$VuSFX57T&@w;;}g1&PUC<>X}+hQz^+Lde}^C7XHqTn$$6
zdQ2_nE0RkErw7C7kU9&b2ZSYVF6X)SgpbST-FD7(x94-L*!>f%Zu#aOaM<WI^6<pC
z2!O*Iq`szXGN<(%$o(B3CB%80mqO}n-#=G*C40JfO{GP6#jdyWoZ^I>WB(*q8mxCp
z%Y1NHg3M?-HH%O6P9?|ed&haR*jd2wBlc65{bLYhPd-SE$Gkdl7=hSD&6~hxfY``*
zzMvb}jW98l##-+FU@o3_ugv)-X&G=`i-erV50eMc=Bz7toD44US}wB(n<-v+hb!Bn
zn(Ot73|_<bqr6(`Eu7{O`CNasLc!sX6$)7^0n!^{207mk#0KGx{zYIj+VgGs%9c3t
zawJ2}ld24G=UNre#iQot0d^m2%xbVYkRA}W(zwABZtKir^01OKwj44Sly_tjZ;;A4
z&gGLKb08Tv4)PS-f}DHw!g>d{^ApHD${_t9yixHFkBLkeuM(3tZ(PJg-XtG&zN;r8
zXQfY)Mx4Pd_?V3+diGj)SeiQA;=P#`2j**ZLiX6-t7GK%inihb-Tw<R8-y!&odvrc
z#74#r>JNFBuj+;SXG-u*PUb)hUguLH;Q0L%+Q_|oI%K|RQCA+kEC892`eQ#hu0ia_
zNfUXKgnonFk#$jpXa8OSaCk3&7zMUh;prPNAEb7&++nbN+nze}KKm2QD_FS|tOg{9
zi~~i_@#R~s<(Y9)pLdOPJKx6ck6`n;E-vCtp3KO*XE`UYm4rH6ukphLV0p2|&Ae}w
zPV%1Y?c>|K5pwo>yrK~}&q^B#@#P2E@hDb7{JcR)lF!gMgU_iIa>fhDer6Xv?y~}r
zxx{?V#k_3s%Di^*Gr;ypU4-m^UQ<1j+wf)xSPY~m_AeyufY^NVmhx^;6XZ#(ZsIlf
zzRsKRS{$tI(>ybN-eag~=W#t`J^Xe7NIL+e9~t}c@8Ru8+0S!X333JlNKU~NvVIH1
z&J)?p_5CWuFChNz^fi1J8X)b)W2;5^l$R{yy*dMOUN1=A_>n%>xr>l`{-4e)u)9Fw
zAl%sY031h_q5WK4N0;#gn2GR~l%3?g&(p?ZyQGcp-lsSoM*&G*wG<slSXM#q58TNO
zxu4v1;Z@$*_Ts!v%F^7H$257XcSG(+Jg*CB1ABi-;Z@r33>-&WOk25+I34EOeG#%g
z4rD(F*In}EUMp?RV`ev(TQ6q;&&z+ecp`22ctS4Ja-Qup2B*n~EC=}xc(U_}8r<Uh
z^Z{~?GDvR==Ug6k^=aU=1>%En=voQBrcaRY2l0h}oCDhrVlUcM0ygirFJc~WbpqtB
zc#!-dIUT-|=a4fIKzuK!54;`M7J&5}QQ6K{EisMH*oBAp*2Xlh#~&eQ)EPg3+{Fb}
zAAn@mmc_hUzL2%{dbj^`b%uRnmJB(}J=;}-r|qT}&!PHLygA<?>xw~o=S4!=x*+zN
zGi`iHb5Q4p{<Cf6eyrrrWtf1N7s#vF#(m(d9*>wr2iM<!<?y)u+{?(lM5Ga%cTagb
zaC6&3)*ih`Gv#(!r2$r__Ro}0#tU*cQ|Q%JKKGN5HfE2$0azc%9ct@?`DV#L?tcE_
zQ_0spVJV;9$F+P#JrjBB4Ip>xg4DfrfW)_y89Uf)kQig|HE_6r*f2cp5);>wZF;=t
z7N&yZ?D|?(aJmA?gU+?)c7l{41+QO&<@dBr;FXhq4?dd_q=tt*6t4b>y((YJ<b7cK
zWM=8`neFxA*{5=YXWLB^ezC2Px%CfwHuLTb_h45kZscpLQv&PN-8vWUu1Stod<U5v
z`NcTz@ho>m+{Fm8tAincFI2FR=l{KzyrFXr@g=Zr=UUXE$o<iMHLqw}BiMbqTNd(G
zPCd)@Z7~Pe{WKxo<Q+SByUIWF9atyEmvQ<%*li%QJVX(9jdJz{^39xmpQq@wD6df5
z5{?54h4_w3xpE)Kj^e)AF3ETO;zGV3n<4YPa`O4S{9NmJ<TZ_W^V@6q)Uvp_@+u(n
z@*s0T=l_2Z-o_hIc85pcrz(%NVmi;mw?%wU9z0_2Rf62x;QVv~*V?`qo(dCYE`H`&
z?Ce=#yq6muf!%+crIF{{zBypF^a2)M6MlBS?Gg&SPY&+qF?a%*&-Gz`2hKm!#AkBv
zi^~G1jk<g9cv+{k^VaXb!RyTrxwHBAtS6jDixB<%%y*DA5r3K6c_;5Pg@?t{Me$s>
zcAWv6^TA1uU+_ERyteZT=W}nKyNB<y?+xzjpJVwPFWGaJZ*1b-ytSNL<O(D|ZxQd}
z?VJ9BNA9T&-{V`Wd7}Dfa;>_%g0Dn=67RHAJ9vaIMewfquE*QX7tQsIXAY0y;Zs~<
z{wuhx#lpCzrrUF`Qk}uO(-Tr|ufAc#C3X#R_vie8Og^2pQ+eE1eF6Km&;1g2b*CvW
z*KA06_u)l9IE*f?YURsXqsx7E`ZGRL-sQYH_f@&CHbMIT-5<+&T7HZ1tc=X(iIK?Q
zo-*MgS1d2&?6?cw>p0I`H36FqinnRLz1$p2`g#BV&Et_P;o#-jxRYn;r`dcOInjKV
zSRZoTOn{tioDlz+^ZcSiys4+R@jC8$#dZ6u6IY<dMcxZSkp6V+$}Qk<t!Pl=OX|45
zlVW`XTt64uO=WN1oC!|vVTZd}7a!5)lXBDH{if8wD;2kyuX42`*gqh*tX-1CRda-u
z=h`fGuzHiEpS))8t$C#yBsr$|Zs4jv6%5ZWl42d;aT~+8(Y&>X+IeNf^m(tuwDA;$
zPvU!LB*_(Z@hivK>*joAlQ(ju+Cc8W`L=x}mz$<2-$TZ^>~Rj<d?t@&csCt^tdFtZ
zSjwGUwUn)XE#!=;6I0aq7Bqa}S@VmV&)VZE?>DY^u>WH6dU-o+RCvBL%;ufZlER-g
zF@uwTOAntZe<nCgkMVHwnmkYB&1$>MJC$t?SnQt`57<A(j8VMp3NLt0MfURuGPQ9Z
z6MW7q(B;Xe`ExqB?yCsB&bhs43R}-kIlhReio7}hv-x&j)Z^u2xX*VpY$I=QUoo$s
z&H;`Cy}rDmhcv<NaoK6lt&s$2UxUJMf#M$CM7a&zd3JHUCGXbqT>A8!H;Zi(uk@A4
zd;)tQdE!{L12`O~n=J>s@5uCcp2@n9x&>qg3{Om6!I#X;%<FyoD?E<OJf-*q-^g((
zMmch~3o-H<Ok?Jo=g7nMn%9r_#c~<G_cuj(Bxir&YJ50}cMVeq?>ncreCKr_bMV6X
zx_ondRd~e~Y4CnDP3LVY6atUofb4tgx}JCE6Axyt7Y1A%tulNcue5QC3MB9yGycuX
z`p=2aTPTY+XGbvmUoJ@d!^>ha-!Aq*may1sV7IkTUdIz#I+^cfS`m*ocRyd*Z#mxY
zb5`<|EKB6^-%`xm6;;kNP1*n)W?6a;JYA23`P(P^^A-0(&OWpLaf+vS_B=k<$}+w@
z?n=IMJnwk!-Id^dK7AE0sQ#bOvYRhp`AR-6R!P2=6TkT`pZUc1&borf`_mS_>I(}w
zx3^5>5%SjKbDj5*dBvmvKFPiouI7c;_zLf<@asrq@tsRIXTM{7lP5B)fGIVqlPC09
zJvi+5O@!RXKB4O<I6cHze*^nxo5BiS+3s1~QNPym#r7Do$8eg1>;Hp?7IX72G~rFX
zY{nnQ*~Vl4T!d@GG{~KBFLsoJ(-9~<<6^IK^K!<3^Wdjdwmf%^K-SrKIP!9DeEgd)
zv&D-~CW4zMuTPI}&oma^D;qia9>ocQ%fhMsd^~E07`a|)m$B`#QRF-M5VE!hWNzJG
zS>7vrDm*;rzw)I9ad3-!z2`Z(h>Nc>?H*sOqbA?x7+Idh-{<n)S+#((ZO&URv#<ZS
z?K9SJr+qPG_wFd?et3Q+*xzf^_w!DTRpj67%f!AXubxk6u^u0j_Yb~Lp6dL!b{*lF
z_H8z2!>ZT38@E=2{h+_#3hyPI3UEG}85zNEr&`DNWY2fr7w%hmW?kOR=l<asul}t_
zp1<y@JoC<_a=*G&4UTJ&o2Rz+gVXy{eZ*d7e~z6z77^chWDY~_WbU1&2sV2Qrw4b|
z*?wMgOJ#6avcLMr7o=vz*%BJS%d_hh-x;SquzPwq7xIUf^nvG7o+az?uie08+J3)|
zRr9l-@cR=_jlTYx&K)B<K_I0oMB%M)F7J5}4hyr#Q)J&W3WzI)9<<=#WMtQWcGcX(
zdcAtylT|zgMF;qvw|NWPZsIYYW??35mtP28uV2@_hHYupWYG(Zx}y6wy)ox~!pbMa
zGfTmrb)Fo1w1CoePe!qV{ae{Rp1<HZ<RM_*&T*RW+lP<5Th6HR{@*)G*p4HDcjC)d
zfiG`tz~+5bh3s(yg^RoTM01IoV%*O}XYs9Hk*Aw;h*jvJ9-qaz`Qh9gXQX)57Cw+K
z)>F^|t^YT3S*2ZYk-?nj*&Ckuhm-lg>xT)Sh%^!H@`c>D_$ZBmbIw;C{;;J?{73nZ
zb4etv;n}A*mH$zsKS#y1$r5u6L%Ax>eKfn>R>i&OtQ`;gpZn%13lsPrwa4)<JbjL1
z^UZ}^y=x7@>(|wv#`73@wDMiF_`-XzcLLwd2@b-0e{SP?={J#kdB$e*`9&vqwN9>-
zFMaAD+I7l+*Gim)ue8;Jr*Ja|C)bC2f@jV*aw?nh%D7H@B>aC83)hO365zCONqicw
z@3DArJYU#&m!phHUa-n6h;!nEUj8#zzwrl6InN6^|L4n$A`9=wa-#RNltp{Fc=-Qy
z_j3e2(Kr9jIE8!8CreqqrkR5CniT}X*?rAF@0!5BaoS35nN$s4;oJm)M-7)G8^12$
zUG($^?+ZZz{?gey1O+nQa$d<%=3TpI9jBG;eVz$&vT|lJ&w05zvcT*AlT(B(mbE`K
z<2^Bzhb17I|MdnX9+!*%xcRx;*ykP=vpBbcm#?Cok=L)?NpR^WX&wbbRiWi;Hpzz;
znTSe>lnHQcE0x)>G@awdjU;gRmHzkVmolCtyJliCchurpyiCSi+}}cT!0EZmYBiU~
z>QXQplm=#sOY)lB5m4P+7Q#FGf)dXY!48WLUS0fK?``KVd84K1tFfEks9+m63-5A~
zU#?44JS-L|*?(+MQ4O1}acKS$bFCwe{6cQ+oNfiJQcF%elFi#UiTCT_=VsjjNgOA=
zZVRpH^Rf`Tx<z2tLVm7XmwIj4_GxBkt->VOb`|h>)qUXiv9JJ_2Ozhk1Z?E!k&uvx
zo17=`^subJF870i^AbfRl$r#%E4XKHK0a;%U;p3gn_+(F_&4F*DcwRhCvZw$)Gy{^
za*N~p_~ADHhhz`414YZs_dHu*rp8mkQ_da&UXQ@-wMXR5(^ab9uBe%>KmMDqxZorA
zmv^%T*6ecRV)rQKk+_n|e)0rl-%rukDBix*Y06SQOZh8~N%Mx+ZsLC;)54aZE+ulk
zYoW;Bs$D`ZOZE#GSpDW?nbIgYlR<%d`+->^5uYS@<~|EH<8tKXHPUDkn-M>Yao*2O
zQfqg*o10I{;|UJDXU5>Si6^usK+C-GAWzB57+KIcq)&>H1wMBEF*Cg>XmPByjdvo~
zVU2b2Ocvp1r}DnfxTUyk-(ukx^DOye76+NS?~&kNWw{0%ZY#NG@X8-dlT_$p7x-T|
zjU!;ra=zIKYt6MHHt~LY)y3PXyWBkBl{WvEOM7__r|&g?f9wut>q1>~Cu42Vn!PK*
zbq#mURnC=+J?aGqnK^lqf?3rr%;a3W^B~t0u66ux&hT0^9Ms@%IdGWIJE2L@dBJl5
z!_{$$_ai2l1{}XGeeCorx!v+cTq1ik#7@_0@TPxM;V$*nmb9*(!?U*eF7L{@`T~Mm
z9KiW+K`gt#*X5S{RSil~0@sasm+LKp&;OUkE#?<h7Pjc)mf@`5>&y4YQ-J&K+XjAj
z(Fqo9skXdqO^q_U>L+juSuEh~s@=n;D!P`xflGno>-1MV$5vT!%wJrusH?b>UqnQL
zOW+chz?0t#cs|X(Z0=aUle_Budea*z@;r8?rUKh51I+qG88jyy;1Nh{@#eo2yc9hD
zALcQi@3^nGz|QB-g|x%?1cWP$rHgc?@&2i(lHT)}o#W_QMFGXVVggbfcD&V^n}r&Z
zpNL61%JQ;i3Rt`;ydyMY>Sf-@T4Vkx(KlI+ah}v@6I>zu)A0#^MnI@Q!8IniH6J!|
z?%W)}ozJ|OHE^jMpJ@L^-qg~E!d5BUc@lQs5RzyWG3)!_Ci5)I8=O|1@{aPzXl`Ox
zyS7)#^gTOA?%ocbVoM#)p#P1c&$|M6_cR0uT(CXBWfXjXBgFHn^k&yXnxV2&c^H45
z7G%w4HRsPf#NF%t7(D;)Hb<Dpf$tRm^{Y0V6VzHb#olrWTRB$q@8G}8Wf*x(T0Kmb
zC*(jokBL4T&(vifc=Bp(c~Xw;<o>^z(c+zx2w!znGUqYD4?^1di#XMs`4yQC98g_W
zH$`B}`%~Or;#Fq<XQ|8Qt~9lfzqLi6C1ROO;oUzx9ju+^XQNbj6+Ug@aXG4M?!Pob
zslhm#b**SFPdVQ-;j8n7EiNdw2uV(##5p%in*a8u7V|6T7_}Jc#01ndBQ2so-sk@9
z;K2XTQN`kn!xG-zNi4i~?|KM?)STe->@l|3YAMQl)=XSmUQ3K;PEV-72H`0R>HAg-
zY8rj!?=)PZ&J^+lye<w@uB0ldfz$c+J+T^d>^1nVi=E&vt8bJOpIykkx=M`a!rB|G
zx&A^TQP-!j^4*)n<8r~Ar>M{$+`lM#<0()osK#?rK$|DaW+(ruutLctD^ClTo$3&{
zY_k<S|Gy>n8DGpkU2wfJvtT`c9&0e4|Gy_(dD&(H3u7X{`PA^rVld5Y$;Q_ly8*2B
z->uDHF_0T2nr!%%y#2&~_5%-J;Hzq|JV+dbALf_yo6b<;HO$(@*Zg8Sj{w6(uH*-A
zxSqdN=bLZ`a<+81L?d@P_YaPndyn#&oQdZ>w{<ahX{;;%BOMc-08=ygTE!<1A@gP+
zbMtnG@Lze>&edxw3(lW*yQKIf<WzWbbfmd<|2@KOZ!XO5b_uekUUK1l-c*SezA7yV
z(3wVjA2=s*#%V+LAcM>T;Z&}7d`Yhl^S#aR=R7v!5;se?CfFa9Paxww<^CyrM-zKE
zcNfKj$5FmzUEw{z$q82T*zFhR2Gfb)crwyH&9#5^KHj5>t-KGzbNKQ-(!uUaTmBQ=
zj>{B>+<gYJ^P$uMZh@@{yoQJJxgT$F1GmkdUA5r6vQ8gtE=Y}_jXK!epXQT!zMVmx
z?^V703#@11#X|mt$01|Ln+~b*o=jm8(9nC%Yq(<~AO8y%_Jn1%JpUAYxDtQ92AfsJ
z_M0!e{}`{4ttWrmN_Mcj=CLRUT<P7)%kJ;St5dz1*FvL{?>wgtH%D>=XS3{k?%Z38
z!FKW1yyH5y=rGTt?~wMaMaU*jo$qJCW=vcChOaYiBG-|tkTEOyX&$`m+oteJzDNSs
z+0G@9xww-_G5kxWHgYMas|y@h?asICe;bEv4CF4+#S<rRa$dU2dtuFY&V!R<7>ya%
zbN0AG&Un#gn8(o=0qOsN!lwH}Ezi_AM!syD)109F_fMxT;VT!C<>$Sh#{c3;Bu|!;
zA2=L!FEnvGI|cG?mVE;DKS;0YiF3U2#V^2dv*V;aIF0>km*hFAbdcwN<6iJQCP<y~
z`9iQ9h^=*lnSXlCF1{9yPrT2vX7lr_Bj&@d-<ZbH;_;Ppavx;B`Lq0RzVp10agl#}
zA?I9!%m885qYU7%d~LR#@8>i|-WDcdzO5A^d_4AD{0p{1#w=t%H1mFHQsh1_#{+gF
zNT0YeWFN=n-Ti!ymT$Rwe?aa<Gu(dw9KN#b9$<6p<vw#4H)rxDre^aWHR|SG))E5_
zhiNU2e4Ae0;IsJe!n2^f0_?9BG6DSM$11?_C-e6R7wa?)zUY!U-1e!EdrY1?8-w-s
z=a_ICL<sSPCZ+K_F3{u+k2uGjnj6LMasE2k&W{<&+{#m$IaVBH;F-{0!^dT!%Deiu
z9B<#nRzAiR?fgDnkag8*>(27kPe$BtZ5Y1??mwsZc6{M;8(Ggf*YSEO&*D`+z{(rM
z@P&`DG?l0DqZ-&9lT0D&vCW#Y!Ew3Rdq3y(M>cRXK;Z<!iyxZtb3C5T6SV9cT#mI5
z;-`x*AMySTwB-HO3Yo`ZVg1A%BlduM%YIKTpG`G<H;*K82j~m)UhY57zx6p}zsZlg
zNBMK_PvGs6Uc|q=9WuWJvIm40rR0O%rZ!WA!@|v<_jeg5*Xg&M{BH`D@Ns{-$ZZ$Q
z#b5N5i(`WeWX-wRk`S=^v}nkF>}Tydd{;i_@OQWNbNV0L%%3M9%WbfOMPUCj6E3;G
zQ~0N-*K#rUu!G}E>cVcW(|NP_)qXyQ`->~AlFwK~NMLc<WPZlQ2l$=$+3^)_mghZ?
z;?F<*z#I0{i_Y+0kvYVFH{~67!%{|pxe`zLIF<MCWVCgF-EnJw9q*Zb$Xfp@o_+kG
z(+jwQ5_|alQy2yA?_UG9N6$bToJOv^{KM;J-T)4J?MQKOx-0W8;PFnl$SZkB7-F9K
zJnjy|8Jt(5AmJi!%*K6_O^kC!ksGf-RtmR5#3%k6#?f5IpKWJ1&V!t3>N|rQY(6MF
zKPDi?%1Tdd=ejws9c=Drof6LLqM8Dha|HNrYh2`XzH^`dx#VrGRYg0%?f~iCvZaNq
zLM#N#f3?(vmv=ehoCYtmtGvIoeE7~C^c0wp+rZ7E{f+n75_NuE6FL66G%5Z?|2X*?
zc@XRE&wSd+9TYy1+fMK*+f`Ko&aUiwup6}nyZEcsECqrJ#5mt|bn@-tzshI-NQeJb
z<Sc$o#|zy16>GR3Z2AHYH;|izb*FQeN1p=w!LH#uzjK-w&kBxC{w}t;T+Pw^0x~_)
zJS`O!d|ca7cqG+saPE6;2-llr5C^WyXBR-ui~^ZAYiAr^x6mi9?I|H(_bmM105<!5
zQzLiI=X7pARuk}8h`kkyz?(&xTuXaxxplUhfz^QY&+`)lrzH?OQB;v%a>`z?9J~BU
zeus?Lyg{W#oVE<p1cKkHg6sI&Ly+@b@}p(>tB(eN^$Lg}=B2k*eBvvZA0r@r(1Ndd
zHDsRy$ZQbaP;!8OmeK}ZhH?k~jD_d9!r#2$GMKNyouzx7$9OGd4@=SsJI<8vtek9`
z5q$Ci;e21)c{sN{dcd#8*};80r3Gvs$gI19kTohG_V+ux`8Pkk$<aHxmQVF*D7%D8
z5&v9=Ror>O&8$fs0bHG*_VV-ALhjmIyW5!O$&DMlMg0#s#8c<+Pb=8XvDPL5oQ6SW
zfyV#e8LIME{a(Vyv2ZRA-(p!_yB%x5VREGe(hhjWJB2spm>fs{<aQ4G7ps}=x6bGD
z_ysu|B=>+B=bXckyRbM99py;RxWu<U+m^5OgahApE6Bd3a2XkJIeFpMWZo@ja=~LS
zb?y5(^~Bxy{%(xmdZGIa9A|3+**Mm=Oa$8va@!w2$UVhv0y2C{4y3}xL2|9fe}MJ)
z=xGXgeT3ZKy`=gQ&mA9ofj7@~^LiYx<<Dj~$J)iJ%J=^BM^4=(Y+(Ib#!CE30z5py
zryKZ<4mNN${EFnPmT?z|)0@PX<+_yDiL;sahTA^A3)<1#3uj#8%uQawb4~Co$MLR>
z>?<DhbLpn&@a_KC12(Jucsti}oi=bf6iRF4*FANK!*PEn*soc->v(qlKF!-0aRAKE
zV7COT+tGZ5-#FtQm(;&#fw*ID`JQU5=Q!u90#5TO`GOqN1E%u^)ji;FlMm+-U)IO*
ze?vQ1|Ej`iZ0|qug8dBgf0x4)o|oPe_;%dz;`lXX5~pZbBVW^^b^Q7Y`}ys+%JZ-&
zLh3!kGjF+jRyOeM+_9c7QP7gpK%|!A>ed1t-AXU8A0!m>z-cV(bsblEcs%c$3J>_$
zF0;!zupG#~b-P;mHJA_a@kMs<d1$f-2tGatb{9x)#f1!x%wjI~J;8Ng^`Z|R^UeL2
z2~J}X{|>U+ol1kdTPtl5IR3AMe&uuUpUTVT=E!F^*M)EH><9b|vjlm0FNv^o2_NCB
zo@~e6YCjtsE{&7KINQ(P1hX@hxcMzyW^+zUf}Fwh=h{J@Dsf@f<-(A=CT?A?<uaVL
ziaU4neEy;!JFdFsU0{DQZc*Xem^_b9++i#4G1dJ7ds3CyL)IVXf9JLo?6#UsPMnWU
zsByD8vT&{m-v)O3MGwfmyZf$`a&A@=<ZCeG<+<{Ufj8=S50~AQDE=Sw-htan=UBgR
z?fo0X?!$DQ)2(L--<F)Ke7^%`@;V+_!2hgj0)MHT0zdC%aTaqc7XGCYkhA1-+b?n&
zRYKAcC=3%7Ht|(0kmFRlJ%u;8{T;XTd4Ikm?|At0kI&+A(}dhBSYZY^NAuCNJz)2V
zy=dm#)q!YJgUkWp8|oMNg$`x#JeUlba|ekjOaJ3bmY&Jsu`z|GaDErh;f<^Lw$x5y
zy~(tf_v@ZUu17C)IJg5D_@sLdaGFo@=Mz@670BX%+=JX*^n+jA>@#onw?_WgZocq%
z2bp`64^oFkc?+-`n_uFaI#H87S+SqL?t&@*gGnp63Z+c=_ldmYkP3#}N9me!gnt)%
zAd7+bHLy8BCsy$2JU+}HoXy6WBgrKo`!a;%{VQDohTn5IYK2?*1M@pLI!$H4=_+mG
zd~SAWc>%AP&HSq7=fPoLe>RoRx871fdD&zB5+Pmz%a#Q^g(tW2M!5y>Px`QyRqc<q
zfP;sm0LLF*uCTd|0(aL>=jSu==H0RM2mfX6-)wz#pLloIxq;J&@Q-)=Chi<OOLqR^
zFG|l7c+B3!-#Kv->!po0oS^%Ezf9ra`zi}L-^)0+AAFz2>D8Zk(i9f*DTqXY&9FAR
z#<`HWljG3O*Zi?PhO9!S{G5zMci5_Vop|^B7GOU*CqsaD>uC<7%9U)foT6ZV)rLaO
zpaq4eTss5DywcxbzwN(vg6){m9<aF{+pn;lSYIe`MQaj&;tzX{$sh0VKX8%b+_eF5
zW}m$FSvI`}BW|I>EiB7^dV$aX1(^xLeScN>+HT46b-T0(_(xpl+MA%u7s`{#&*Sik
zUwm~Ne{JkX-p?{yIhngfINaumb2c%t^Dey|&G9o>o;_x@Fw3)A$o(`^jui8=8FUMD
zFH7fKWb4QOBq*LgX9c6c^b~6WrG+QC)~4`s^Rs*N-(8&!E=NAzli-S;+64CJxsDzI
zZ6_w~j_r2*rsc^TOTL8*IJryl8SupMm0pqHReJD@>zp)VJ)g>CS8y5;zYjSR66BAr
zZU;UmkExt0g0sP4wCV;A*zCyubnZ1%UvZkna)a};SyZ4vjq+<wf$|%iJ>M7vj+*p>
z%biE}7J=1+%vw@;iGTNT3ot)xSE;~-jiSP<*`IT(zm^m1zA_(N-hAKpPN?bfG@-Bm
zH}X%l=My<N!-2i_(N7Wn0}wyy|4ZdR!TeYtNMxe$2R2Aw3S>42n;r=ki2WMJr<SKJ
zwr?XNU*1_};oPl%MCP~j3CHfc1UCP@<^>*?_7@`C?iupGH`v9uG*+5-i)x4fV|f~{
z;wxeB-an98DYcM!S`hoE*Lx9W%QkLhFJb=qpO^6--oqmBNqCCLWS=7L<$|KTCv>a@
z>WbRI`zS6>+s^lcKU+jO?f}moYc2t+;}duzG*9xcocRYFPMecuxx7sF@qfxL<ll2}
z0nc&SJG|`+4FqnV-OXP3U>Z2>DH>P_*&7CNf9I~^UBe?R`dyBPe}BRku$;u~H(b*)
zxWz#3id*ByQ+`pMe|y<RzTFT0@HwAQ0IT`s#=+xpK3vp@wTthZuaHQqj6d6TX(8V8
z%NFwGD)jMKZdwSA=li>)I8)3_!0rZ_SHg1*97YxmiG1HyF@fU=B>s4*1n<==iQxLQ
z<jhQg<9i@$susjg68^_>UBLauR6hBx<NO`b+u4{D<V3fg-^;_h?;O}%fxU<M4K6<r
zeGsZB5Z*eGyUjq9&m!TIz!WQ4;oufqJ`pD`zLtyx{-E>Icvt6iacy%{;SDyQ!fCy$
zl=t4ORl@3SNBLeU#DU%NJt0u2@#$yqc@-xsm<9UGUvnQi^#$DizHN72)M@5Zz7tdS
zgWHkiZe?J4`MGdm#|5f<@0}P0=bkI$SGb?SdBe&NYzFV6t=uP+R7Aet6ynY{@Bp3X
z$eksg05(r8N0Y0m199eUM0O{ifU&G-i2O9}&gBcZRh;`orQ#I?z8vxsF!MOhyJICU
z*o`MnF5-Rn^po(58`Fg!{0I~}$;~e2nHVixH4Sn{xB7*Rf*pBnA~|_=yb%(UL`oFS
zbMUvvvnDoN7L5>f5lr0qnDdEJlE~B9GeuqAP351ng_&PjrbocE3$lLz6dz9ddpHh7
zH*s6;W(B9$o9vPNS*f%5I^WIWj=$*6F@uu{>_3p&`>!^G(~6o|2fuCxJO7>yjQs5H
z9`eO_WeddcGm72|5EI?<zf#oU{Sw{{*H44Z+n=VuCtVTGzsqsAaGg*$e`S{v&p!?w
z!6`O!T%hw|Cb%<lUOFrT4qsCj58l+f|M{kh8VR&ie&aFI4iflqn~!PhWG4Qu*lhlJ
z{8oIv%>jZ(4is|#xfmeOR38Bj-?ltGo)`bN3NKOS;JuUN#_wu8f%QjjyFlH#WYMCl
zvv_X4;1;p}ah7*+6A!=s^xHhor#=ukzNQ4M_iVDBNbTMg!a*x@d3R<oiGDqGPxy1$
zJU)@qDI$jdT?OPa?(mx!Itxd8&El`!$PBK#Pa9t1o1zvh^g(_yr__??9IqrS_z(Um
z7OlI#jj!&}1m0c7|AlufTg_kGwu@K!Zi?u!N6K8HRe|8JTcs)i&R4~^Ik*;NmBPbz
z(Yn>(e6&=+M<C}4yJ*wojbL@{tg0dn>Gk|y*R%;wTw}@i<N7?I#|C;Lx>t3$Prc&h
zOZPY6ZG7s?{fbqc|De$vp0yTp`N~!w6fm%btpDWVoh5K!Ngm(68PkNW#xU@n6_^W-
z7f^g|4_6e3k-j7%Rd2?*G(n9&XHqHmlLU1E*&9biOHIUiF9$>jr}V7gVoHRJ?L19%
z6L_WV!s+L67aU*q$C`xoLcImJmW6TIPn{&ddSogOhy6U!M}LoTv#{(CZCrJo+l7Ch
zK=a#jp>=bg3vo8h74VRJ$6uVb8az(Cp-6y#f7dO6)mpUzGcL^*Sp7^%FfiMO&+H?U
zP?W|b4!e@g0-<$_1y-{9^WN*ZF7RrttbpAadA^7#-69)wSMW5h-pbd+cY^<ZPd=-}
z%}2tI7hdPxTv#LUdBS@E@j`on{r8@8{r?!oJ^kWwwpAyO^WBfh0hdX$7DDb;Ts!@g
za96F3kYSu5*gZvG@9<O?`*W^6T_M1;HG%U%dMVE#o1L6NO1JpZS90>&oqQu8y?q1s
z;R;#K|8n~UIZU>Te6xY9t@K)cTG;IUZtgYZn&7g^L`H%qcS;aAA9uas;}TuMD(H~T
zC*bvfoqNHV<HG&tO?g-+UEuL@%>sv|qw{5Mj(4)W`)s^LUgfXld-4j>#sbCDv)u)v
z^Y(R!TJIJYWK`niHR{b5Y}n`}%oB7}<eDP4K-3pbp{kpVyw)G5i|pEG&)3yFh1c~|
zq==WvK7lEX+FXwtq``LduJ#u(G?f&L@Ri`cw`v!EKC8ayg|4;2-#j-8_^tF5)yR<G
zy<TvEKkV^&aJu{$@sg*};yu_8r=M;W?OvG8`!Z>k$l@u2JfcV63Itx_6DgP`!T(~;
zeqMFXMLb3uc7xL_$j$Yi)`9D&Q#)+I;Q<nRaMxNSVX_2IPyPY!ny`tisr?;bv#S(8
z^PY{&;t67eoQ-`i>8`*&nO5GSixxZ#{z(FV_pS!J-}IE6V9}N?u(>reSOor_Y6I(k
z;y6X%<Bc7BX<^|!?FpfRGw(viwBG~>^RK;R#OaXopI>f<5IEh>^@8mE0J-N~)=vIM
zJo18^Q@Qw4uVsSG0*Qn0-jFVVa~+L*|B`tG?mw>QGvWEnW0ky#_m$8jzV=>-89y)C
z@_Nl^;VIBn;NS8{i9gU}KJOuec>))TFYr#@?*O(BWR`ye;#>sHjwXStJ)At|U3UDP
zQ_OizIj9Qc#5eF>TX%;$#<`Vu=d~t*epblZo$!q(d1Kwq@U5M0!F%+3yuhhQe;#E^
zA$S<4ZB^$iz4V1&<X0U3kN5^&?j9b#^b=JAm$wJ7J<9zGR`*^rfKT`|6VI-zZ9G$}
z_HfLvN#<X7N(t=O=*<(jecwaYNgpy@&*Nw?i9h*@H2+H%R{k}cAY)=HEvvy{KYc?V
zU$*RFa2d7aLo;tMhXDVoQ^LH35|I11a?Fj{Gfkqv=7RiuV3s%7em>_bd>*<6aB;Jf
z{ak&!uY&7;j;$XBoM%GTh6`04=Hs8jD!5H^Jzs*4qCm!ZbvC}!pZR2;#__y)e-Lce
zYwy?mtt$R}OD)|6c;2e=*cUYMet07!cqq7nf2Mm2-_dMGzP7r4{!fQ^c~u@dajPxP
z<h4C&#I-KPj3<>ZkoWdpasDs;MPRpN`)BcZTIqqy^C<<60$;<Fcs8^rgZ=u;{w%ND
z(YJit4IpzE_qIdU1*+9f7g+PFo40{gR&dtJHT-vEgt>OveFc{(TO>_+((O|Dj?Hf2
zp2fMGcgE6F+%NY+#*0B?4aXxzz-EKOra8ZeuPw}xpLrG^&q-e{9+TPb{Ql?b1$J#|
z6ZrTlo%gz^IM|KeF>86(dAsrJGIjA!({$jSbXS#U$qW-dn-FfWywDR3a2k8Q<~i?!
z^Zk6?0ZHKaaS>GHToh0TmNPrp$|`={QQ&`B4gbX!6aGfgBLWxw_rcu~>9L(nNB<u8
zsUzlK^;b=2^M9OK%%{9tjI*ktoJ&G}1zfFBivl>_Hd$=tUofeYPv#3hfAlYQ{-+1$
z2<)jd=G%0gm*-*abpA;;uDl!OK*msQS-p9=E!OatvS)D4nRAezDQXc<rGFaOu9lX3
zz7-~iIJnM2?waPhc#wDHMJ_(I*XaVE7&Cb}+uFhYRGIOWZ~47A{%L-b`NXbw2&T!;
z=enLVL16m+bKp8$?#C&fjRnqpSI)QbZk2_s+b_492G$#URh74~+k$^?=2hN#^%wc1
zJ1lvRf6*53mO28~tMue1&*B$}99CQJ@?Pg};-4V9nqU6yc|JXnB7rxj-37h~F$o0i
zXW$e+w1Hn(Z!S2!s5ZagjlBUGPXxuM%b!Mm$?Qh%s=JT*R#_?XDrze7Kb1Mb&$8Z_
zuVbzd*d2co0>R-ZcN%i85bL!}uI{-Dz-b3$1`J#CP7_FGwBfs%{0^MvLE;?=Yx(0N
zU3nG++w;Dg`GQy9Viy1XIytUY$Ljbrtk?7IwOqjcEtiphS%M(1!7Ua3RSj%{SvMf#
ze@*}P2^93K<XatVDsbmj7%$6rdvM<iWbW@YbAg$@o@|xZIyfu3=JOXGP~=(gCs<&b
zlz@N^PZRIoiR=O$^-A2rQjq)Og_rdS>|zh(IN1W(pDyrp4sWe(kHA%hdt49A4hy_v
z<m5GZ_*H<F*P7?@X(xfS{ro%&`yPY+|MG_>Z^w#<0vB#N2r$+_&U-G6Q|7;Rm`$*@
zeWAeGQ>O%U!YcWycTeVvt<w;ge&7v9guz#VD-1UT=FhvttF?w#Q0!!ifTE%|-}Ymx
z1?Kr3<+`c+lus{$Q^0u7WR|+O%LMjK+reA-bF)DI25rH0KP&|97X0S)dVZ1n`>QLg
z<>x-}1>9T$4(~)O$iC6$>8E%VU$pc0bwK967VNO&eY!uNb9eL_0iA2hIrcsOz;lyL
zjLSmo5#MuxSY919bwT;L4LtSHCpleAZ-M>g!!;Kiub}YUUn<J8=o(}k{sDUv=o}TW
z8jcz1T;Du31*M#P1tvNqaC2XoC1BIAkB4`DC~tEqWN&kA=TYwJCA_@8_q#dd&&cuH
z2}gqS8pzy$Zb5$4OJDd*`>h1oo%ixQKg`a*^O=mms-J5GbmrR#OgXfL&#SY9H)iev
z9>FE5yh*)(c%MDV=kb1XoJ+Jfk6UpQ<Sdrk%tit{RW5>JcjdSfcJK?R@0S%|-?T;G
z)&y3;4X=ZE_uPKU1G+oP`^f~b-Jdqw@}>q_g7d06tFNH+_4mBGH^l@xzqjxlD3lg_
z9{7fD+Ab~rP1lV0P9>h@nRAg3>|c<Z53BQn&4?6lfrpQ!mnOeV>@uDPXGnYhlJps{
z`f068d2^kQ@x)(~2j|J|yJCXZqL%O+R9(u${BEnj$`z0~mfY>xg5f27;CY33gXjDc
zVj<@Z`(J7km_PXmAJ^Y_Uemn>g0_6n{doG*_*4X^ax8i}i~n2FZ2_UwRMrY^#J=6=
z<+J!7x-kiU_xj9N`_mol7LeOPc;k~)fiM#fzAZ&J`HwiX@}GL$&NHdxDNmfjUcN>j
z$oWMEGJbr;8#8&Xy|(A~nRbFt^2{2(Q_hV7+U7U-tRob`;RZ6ReaA6yx&yKGcSQ)i
zc+bac$}7p={Wyh7L`*<nPFWCd&*$AdC7b5)El6|{P_E7ctFK^-;(KYw!C&#jl22li
zoPfa(7w$C@qWo%l)!@1OE3D1z*YeKtFP>S$A9J;fCp=^-@AkX)0ya;h*t1>sgUgSJ
zam)D5{$b$dQFP#byOn|KbCLqT*iSaFUv*}F;#nsN+20{ha*Fr;>Ja`1?Wg(V0|of+
ztb)}4ylWw6vs7-)<1;8-%lY$=IqL_8Eqp9{ck!9NQs7f4(*VaakM|AE=kZ!#b3y*B
zzxNaDho=rXe4L2_aPc`3^SEkG?FQFVhaRjGc%K5<C*XLfmajPbrNHBZGx_W%zv7S5
z|H^i1N*iB5NH2Ht!y>R>)^0t|fAEGf->a!c0!ROg@m8p6@XQit6x?`Mgx}*(Hs6uX
zWqdOi<?x3u*vG@(qs_DTmOgK9!BWn;rBU2^Gp6w{h3@4$B<uxt%kGQ2xUXbF`ZWfO
zQUamtYj{t4&jy$8li%*)WenQMS7sau_JhRVH(>E;rP%_X(r56b&Eyt*;Zo0kCnkh@
z&zsX=H3o;vd6ycr@V&hy&inU#B~OmkBA(?&!eD*R91e3g?)neU2i}>id~Bu4{EHJE
zc@_4R@J1Sk^I!L}7f3o5C*V=2%KJwFGPdaOnx9vA%@4kRg;xBPw>Wrw%B;96=C$zM
z3VsF-k5#G?g6*$Pu-ATL;y2o(%lD?jg9~*2zslKt?B$zc1?L`6<j9|v$l+ftARuqs
z#-DhDmw!FeT!E5zx4`ZK#giBBWUg22V!VD@BH;9zI;)63ke7)sV66#feu4m3(dG_t
zo&>4=Qm_}?_I~3yo!{eWDPMUDH-EXyKR%`MCV^eqx_s?6aa=F|CGlUJbA+crOBw7&
zp;&I7^(R93b-KGbZdA_WSF4)Idn(9LFnd}RyH?w2-t4NS?6P+ud%BwMT;+SVznr&9
z%tb&wubb~gexyKwYbujhk_5j~Y9)W|O=rFZ+tLM3KD@zso;gln@0wTOFulEZI<LnB
z1HRR2^Z4fO{K*G8|6kal5ggA`?fLjr{xR`0b1ma-e%isu&X~=|FQ+4L(hYJ($m-qA
zJTqbzaINO~#V55oo`2g#J^mRXU3^EpR0Qsxa2AM8y3g+v@}1LHc@=+m4dm?2kcTZi
zcMeJjtekd{-NtA;YpYKb|C;Vc+zC%t@}2vk$M<*<1Al-0F8)WAj{Nc;Sp`mt?`5xE
zz7iacH76c{(~DaOKTqOw#2SL^sgQHct>+~PRLm^lz3Fxg9DY?!TpSE-v-z8q%(*A{
ze&Ut$JHq?A(ubep+#U|u4eWgXE^OoTe{h<|>LCli(ryR7kko5@Zv$@$<o|=*Q8M}5
z0)ZowX7C-|>LTFZSkHGOW*5Bv-(O-Vz&ka7Rn4-KOWoOq|3}kW9@T5=0vk3Q=bzlV
zk8jeaC;W46>hLsfgRB>djY|~h_FK&M=2R!x53SAxe70#x0#&SSJlE$e6{wOG;A>;Q
zBj9hs!h0skUcgnwiFd{F^8$BI_Of|-*zzc?-YGD>mP=r}z(R1DQFv62|C{=I0S(4B
z0f+Qe0#`)Z`0n4#=i^8f7dZ6oFQ@3uqXKOWn+0|mpW@5Wdn+I{+d@Dfh?_5HLzh75
zl_OlT77O{hKRx4TmwwE8G<Ldx)1R9>E4-%&Y!BcNJn8X=|9^A?N4JJLZ)K?`%l(<V
z`50H`fZZ(>1vx`y)6pW{?3I@MX%kq$X@&KKGau){5bhMuHUZbIUF<$<nfN^3D{$wF
zX7S}LXy@Bh!!GE>{*mX_)gG=1{3`@>yc<}|4nyXe&*Us&Sbq2wk7hq)kD-Z7H}6ct
zEU>xFvpTqEsS63-aF!Ra<Fn+c?U*gVvUNT0IoDvm@Tmgeuy~sv&--@4E}p|jX0j{i
z%k$6cjRdC`P`(R(e4cO3zHfXl9n}N_C(PrC>$=Z(?=GLfG_!nx$={>}Jg=?e`^VbA
zH&=Npm-^x-pfZBjpOcU0<XvG-lb{9Mo@*ifNzE1ufdm_E!9~jZxNd%a$e(cIBmV=l
z1_8C(Ck2A5#rcH&m-D0~z2Xn)hpdZYUth;}U?C5<T$u4!LeMkmG2hH6LxFD}`+29-
zF$w<K9L}dPS%u%p<vwqZz5$P;`*v`e1-W^4857uFowu*U{n5#+%Kxaqo>y|kMBbGV
zqRgBwOTpn&bw-lcP?3}G>LJLz;C2sw3j{UK<^J~l8*h|;gTUDs$XJNFvyI@6v(v!o
zcTN19{T4qS@Aq+CWxxK$y!|ffZTpX3U2nJ5vdQk-%&h&~VQzL)%0JrsuVl1#Xn46(
zJn50e%e_Xs8naICNaZTo<rKBjs`=?&tBS{|R%@nb>{fJsXdY80Xa3c$)uM6^n?=I^
ztLEz_C0XcaKQZ^Yc;9^UQF{x8^EMWW!Cf}%{nTv5PWA2Od~t8jxwUupJbZS__TV!n
zJG)|yz5N^QZ6rAo_J(sku``+HX7B!V@;>%MQxE){%zZ%f*}4N~KB(>WG1##G+v8Ps
zX%pHHR2=TybHV2B-t5CyY+hJN?fW$~X0N=`9qTjEA=dTf{QDN}+GIU>{e``s<}9$C
zy3TQ*tEz@gdrqBgDQ|~OuY!Qx`t<g_^~c!ueb1U{t2S-^UVgjq{RS$1_6!+``;$1f
z?yr8Xx1Yc7@BXh@di(ZWoMW#RykS3sQ~H4{uhljYe71Yj&q?mQXDPLB&VeU3S1+*G
zdEIf?t2B{$?}-Pe_NK10wv~FXvOA&PX^;6_-n~X&7u)WBJlA%Cw9noK#W}W~Z;smT
z|E6Qhp1a(}Nr1&(>bdFu&kI-V(+V`R3F<Z8b7Yg0eTD7+eTU30?wKy}YH#K5zI`7i
zy|u0SIdP9f-Lu{Qm;?7n-P>);caM3mPx%WQj*DmaT=&Y~tKqz3&#RcFcAsbV9@sfQ
zd;gp4TYFBazu(vP@qzv8ONRT?w`Co0j(4%!Az`s!t-F06tB#{BkNna7?2Zlg^Yx7E
zcSb$kAK|oR?{E1o+k0D{?d7wRv%j=fa{qx6CHskr?shxs{@b>1?cVq5Q~JJ*U%v0F
z3A5X;s#3h~Xpyn~>Q|ZjL#E!_f0<q8z@Emv_80HC+ouYww&!ee*w1<L*8bhspV)Ud
z`Ro^rm|^3zY{wq0W4(J1)<xUOM$X%Nx`JcRgYyx4UpcVsYc2QMyD}<q?|Xk&`v*#O
z`+v7x+s`v)`hMkUst(iE?%dbVeah}lo7*1U?}_`(lMN0;9M;*tc9PUSZuLL=q<XdO
z3d^7GPiRuNSGHNQzrjk+zUkOyyY*_jY~Ay}*v5QJ-OHz!VtZ9kbg$FLYkLoGYPEGx
zYucN&c#E}%<Q`j_=@0i7IDfH`{`1k+mPvHq+jaYGu6+%#_1?#8x3omqZc(C$jb}EC
z?a#i9-H+;0_g*^pV-Hu-WZP?3nf9JeakA09qhWg^uygPFj|_W{zIEPf80fWoZ_*z-
zaj8c8XC5o|pPegqAhW5-{=@N|`x(XJ?bTy%+MnMawSP<Y6<g+KZ}+;Ft+RD!xU)B-
zPR2Gkj=}C#?7zK|y$g0r+Rd<OTKvamX1u;#njG7{{FM)FryZBEd;h+AU*@&|yNxa!
zdnN?lwf&&#y-y%teXr!Ni#rqI{A@1i>exDbJg|q={h002J!fp?-=4AkntN?;L&kpV
zUoFym4SZbpmjC0hzUIxk*V38C*5`%kUbg$o_ii{XWqUj~WN$*b#J&zcj(r|VEA}pW
z?Yr;SYhinS_exvFD{XsEC+*+&^4S3!hI1Tt6EA7*zf`PX8_S(;BbBFMcgv2!Zpo*#
zz1$%hcD{Fv_RVllw)<uJe{YFF-@dZF;dTrBlJ>sY<+lI%FOR)pqSgm)vX&hTj^?*F
z3plYqzwxBKZd?6<iiNHFT35}n_geOO?}RHhd+!GxunE5-ym#F+>AfGXYwR;Q?rD=h
zb-C^H(9e55rCaWG2xzvOTIXV`@~FaYWs}*y@~%7kr41$Q9hyV;si-;GeOVZ8&(t(=
ze;ZrsK2^a;`+&27`wjZ{?LWBbr(M6_g#AeeQum9Le6YKq+-8^XzSGXES!Mr?@5lE|
zIQ?;7rhv5VcX@9+?&fd%8qb98&p!CpUWi+G{}txTw#|h}``CTu?ayl~?vH-2WT!B1
z^PabS&uv#~rP_WJ7Tuq?r+)9gXRqv@ub95~Q9y!Cmr}GXL;CH#PZ^|b=I^_-=MC#=
zn;!|w_SR|swDGc^u-7D9$997EVOxPC+%{5g-`Q|Hb+R=t54L&eud<Ihz|`izO__aP
z_~L9Aozt*inY&_N!Oj!=PNk*qJLWBaz;Ch9{!<>4>`h#>?AH|b?|b=TnVs<S23zF|
zGJ7Nv6!ulwwC?@6ZS!86Q!ch@o;R!%4;{62Qkr4oZ98Z0p}XCCW79fpAAAwqw=b;9
zHslerZTSb*y&`Y2Y$Yx*?z<f8Wcwg_%U<!mzCHO9#rHMowCuf<DzNvD<ssXKtVPxt
zxk<L?%&hm8F&wkG>&CdpTJW!3<#T(x)`}{do}U%AcdCu-q>9hlXid9qYbEu5Z@BWj
zeJQrP_VTcO-OGD5-FEXvW?Q?2WE*S6&9>Xl+3hiQ6W(iiXvyyHTPE6M^Gn()KR&U0
z`p1iV=4yQ3%VsONH~vzH?GcZ)HhYBXZOwUj_C;Teuy!gD-m7*{dtdJhQCkrvzrFqJ
zfwtjyAKSz&=GiM`CBN_HW$wM2X>0c~#zgO0@WFo1sp;OfJmo)Z?m6(=wtlPGdpzKT
z?VIGPy^X?WZI`_NwCDcTlY3`W+_p)JFt9d0ZoSXlZ<o#OI7!>c@S?q<#te1`?w8r9
z)o|~2Sh8pD{mJIGX?8_6ir?Ao${0S|s7M9uHMZ2*8~AFvtwNmezMFYBY&ong_BOvw
z-<u|Qac|y)?!EVQ=kFD$NZ#9~K4ou%#cS(5m*?*hNolsZ|81M?{k!@0MuF1%O+R1W
zcd$y-e(Brf{R-E2+3%FSw9mlTcK@emD*Hp<e%bG&J7@Qk?g_RXw>tNJtIDz7Zt!{c
z_KrV$`=?9qD{AN5yHw-V-nDZi?Ot9uwU;Mk@}6ns-}j2Fo3__#Tfp8@?=ag9CS^96
z5%#vL80PGK%`a!`<1)j})8&%g$p^vvySPv7T^P8_j^lOc{{1%Q``_rCwtHoDU~gXG
zcAIj;jW(KBm+x(wV6vw=VUBH<{wiCurRuhyCcWQ%dXc#8uAp1CD)Fs*XEL+cb@CVQ
zPrjmSuVg-Lzlmb+{%uuP>@54&*)z_Zbl|wVzWu!KtM?1)OtG%JxXRY(XVczH73;lk
z+8b?}Ii~M*_Y|_N6N|K+bmFj$1?NdycF~?a3w7l8KAxg&^M&V$&HRGCJqx^d?=3uA
zvA0x9!&XQ7z}^LS<!lqb%-e4><H7!a90zS*HO#Pk$+L97d;`CoL*3MUi%+cH>unNa
zKfS@z?va6peSfU|zO^fl>`yuTXMf&|$@Yc!Sr0H>zPs=8jhnWMl5gzuowwXRWER8T
zCmFBzO13#$7hJrs=g2Q-n>~E0wkv*5++&-*Z%<rg-`?~F&V47CHSF7Vo7iXeJM90p
zbmCscm!10<J-_bVv$@~y+W)`%n7@D77gM+2cJ7TA`zN>xIrwQ=?F~CTYyZ=F0SCtU
z+xH)ydv(8U>74zhQjhj;*Z5*LO-tF%%1FxguUgq&hpB3|pPnAHo!Pr=uWh*9-Ze*_
z*}nbgWmA@>xc3(G%)QGZf7xDgxnUFjK-ktsm&Z1&IATx1$ND{jcjfnn*I&22$NbA?
z=dCR^3sha~_V7sT^L2c<_sde=z0nP~_a1Mw+Pjs>!sfg1#69}^9@zHsiSF5Ky1*`j
z$7Qc%=d^uN$GU9phH>wW%|Ekm|LLl||BvhK6Hf}Ub7;J<_ghiSo;AGjHZ@On@7cY1
zrmb1?#JyJ>B5h<^5AD5hi+3Mmg|zKW*=e>JuS@n!ubOOIR-?YR<@!RK^B0TuvWG9+
zJMl%?-hBry@71W+wYgHTdGCX>C-$uFG_ZX$E8XUV9?QNry&Lx){n2Q9Wy0A#Vhe8W
zm1tVAN9~vK-crR)HVs=j_w_Ebwe`8YWY5OLOSZh`s<wZ3t=x0K;IwVWn~gRKqAT|V
zS#$5rfAY(A<K7pxS@(bMS+{ey?Z0GQyH!giY%i_ku+@4XYO8wG)mCSQ`ra!xH}`Ne
zezVDN^|!hA?1}Z7pVfO>wyxMSt76LDu4#<><}0Y!+CGuBQ!6{VTky;tTSLXEHZ`*g
ztriux*(_biV|(;<++K#Y-2481dAs*g*Qs3%R=s<UHL=>cPI0icyq&an){;WorOU5c
z=UsnfvpdJx#;pG4o(si`_cnfH+80uCc#r8$=6yGKs`u0j@$LQV!@XB^Hq$<NHV50;
zs&n_Hy;rukT_0#`Wqo+xGS2z?Hcj8QuQmFf?e(Z{`+A;l+qe7USv#|6|J|QX3hs$h
zP}v*C>AIK8R(kJSuQz)y#0%{W{v>3(CgFxnTWtN_4{Q$hpO{?_y68{0|EPMyPG{{o
z`+euv@2`l+I<Q*NZ~t+zQv2-SRdyM>TlY@B_te&ZkCAOR!;!s_|AlS;w#?tV^2y1)
zeg2^~CvJt<Jm_+?owljRW_4S=&D=jGwmmJo_hwo$?EQD9*w%;j>z-Om1=|DR*?T|A
z?6Tc6S=)Zb)b0D{GO*dtUMIBQH^STgrOv{=InGP%xt7edfBr`Iz(&SJ`=#gj?!CWV
z%<jC=?%kaBu6wUdtlwLHyJ)Y6!75w-X`a^Q&u`l%q)P8i;IQ1&RPn&JPGs^P!-Ung
zLOa&&b$KMa*W}Y{+iR@edw;ky+oiT%waJz=-xq#mvfW|+u6_SvckKIR9J;T0XWhPo
zUF!SZ9$2zZm23V!XNP_JBC<~IDVg+duh+T<dj)zq_Wtx#vWt4X*yfw%xxF(OHS8kN
zY;AQ)C)uR@wK*`e<Foz#qf_m#sr<74*ByP}tIAHhzw;VxvV@rTht%%0KeveSz?{`H
zZ0AX>vH8Vdu-hU>X|GweyDg)B+n$%r4z}~}+}nG4#@4<2L)G_&NebF7bGp2D@hmR8
z?&_a5Ya@5qs^kmo6RkUD+hEXa{lH#i59^*&HX>(R_uh~$-y1xUfA7>R?ma0>N^O0o
zKiX3`k<HGVaob+uaz<Nz#=Dks?^|sC>)PAC<Zrk6F*(|HN~ynXU<ki$mrdaA<rik!
z7;b)I%O&!Buc3(0Ugzrq`_gwN*k&rU*?y^9ZhM{Ymu=FlsW#8`kMEf?am!xoJa5~%
z5_{~<N?){Vxn5;EDJRPAspLjmm9w&Ym1GU=8e{kDP5I-okH=-g-o`D>do}so_x`(R
zwD+ai+Pw>wHtb#CdB`SD`PbfrOZK)Cex2Ww_egZ#=YR9}<m*V;iX5=m``4O(pDfc_
z+tU^7`xLgf@4eZ1Wv|gjXPbLZ<@Z@ynC`!1sA6Yym~sEn9hvqiY>fNY3ESKMlAXD~
zg~w>$o!W_ZLccfcJJ;H>SKZ{V4G(YNzQFw8-HScn?djt<VY}e<Cfj*unD(6hyw^6j
z^Z#xYg<Jb3h)lDKikq>|i&N2l&Es2kRh4u0#qTQGpPuz|ANPDI`#K4qeMk4*w_SAc
z&_1EOv$h-!8*H!sO5WSKA=b93Z0{aMqs%>f<fhox@3z?8u(Zoo`IhIN%R3hDoylx!
zn_!}}CvVXT8zvsceeNQ=Y>%BtwsB{@Xlwm^w#`bN!+W-IuH2jKaemKvll`_)b1v`o
z6;ZPLz;NAm!;PrDYUbQ~Lzev9qY%kww<_kKwNla3Jvqw`SaW`UyjOSmq`k`aJ+{5P
zzil3uuebI1@o4WocQL!PkIegQ1!wJDX>--~>W$_-D>=$-1eclYDY&s_FQ38|n`;5l
zdvbM^t&?L{?yc|R-fQ%>#md+6%--vF{@L7kG~MRpg)@8o^LN;4gub!mv9I5odyspt
zt}e4}561$VdiS>7tDQLP;-1{Ku`iU{=f1Ul4-@B-{n7e!_x+ps)NaRmY5Pt2sr#Id
znb_`F)wS=!Q(Jp`+fDm-);_n(>@?fQw{el}{A;uKF0I_Vck<!IwvIAW_Ew(xVH2RG
zv`<Vt)@ms;iydobyzOLnp?!7d_;<A_*4pI#Y2AC=lE?bGR>vOB`38Geaz)!3#Xa5I
z{p|1F_uZFm7vIs|$MN^}-k_QBdt)M|>|L##xhF`9V{hfBw|n{7Rc+*dt+G97uySwA
z{#iDv`WtL#zg)04tmEunmh4A+EHah%PTYIbcE<KQdlXN7-0QsVt&RWklY6Z<@!Oev
zxodU(?cqJ)8&&rNPI_*2x5#R5!q>LF+_EpMw+I#4e&01~uf!Fmy<e_=*lWf!$#%N2
zg{{Rt=RGqdi+11o@nFw^1r0VqLD{w|zE$lB^ge34qhQvafOjf;ITl>l<5@e|*6q`d
zy;|!vY}Z8??%ivAbg$!~dwVoP5^TN%*z8p~+h`;At!!`YtcNxMfg5da_Vw+3(xbcg
z!MrcloU5evx|-<P-Ptc``=(XF?w*~r-TamXd)Dpmwv}EOyeHxFMO*Cy4z^Lk!h84q
z-DI1UKVfh0>WMZ!n+5kpiniO#`WLzPQK0aio4dK~LUxDkv%12t@12RyKGk}^eKnI`
z+s@r>weQ`j9J}K4RkkITclKVg^Rm&~q-!Umzs2Uk5>s2rSJJk5cA9%jtGIS|SDNkS
zUoT`G@=a^+T)t<!wG5@~wx_t;ifuh;{UBJ&MohAPcR0h=J@FoI_8JDd*nTZtwm10e
zH5(~c>pdsuyxJSte8J}JNydF_-R(B3kG<W!*nj)pihC^kf{l;v-IViS|EfP;_71`t
z2UwDN4z%|&IqX`je<1w0!~wzA*X?8ePuidMO3EQ>#VNbk$~(4dmoD##bd#{VYt^~W
z=#k96Uq<fxTs;@qs@wM1@|W@NYfpV@Q}%{opTeh@JxP<8?XFMjuw^*;#CF~!Rh#L5
z3ik<oNwK@;-LcR4i@go&&%d^tUozQdKT_Yz;@)d}T5pl9+JeWnHMZea%6s<MTAn()
zCm~?w9%sF_z58v&_Acai-s{pPV|%~9+a~^Cx^1$D;65L|n|lwN{<U41-@X69Zd03{
z2~PWGmRZ>wPG`42xy97h(XM~L*7P-dD;V4CjUQjLo8We6Z|PJY+lg-6cAu(JY)(J@
zv*(BIciY$#=WKj#bL{PwZ?lR@wX;!s`C{K|*_rz{v`g6ita7&FjtSrY(S3@Yjp^k5
z_Ir2ltDGrgukFol@4Ia4-b>#&_WA2t+O787YxC!=(>}G=Zg!f6$83Ln|8DEI>%|_G
z7FIik9m{Ou-j(dRB7A#KytTiLUsd|vs~00}-3`3=aUJirxjQF#kAQmpp4bD~`@^JG
z?Y|V?vR_MC^nk;r$M)Pml@16l*tYLt`{R8vrGNK}oi?@K{_4ygtHTzyu3kxd@AZW5
zHBwL76U1@ZHu3|X9p{X$y>WeUw&mYX*j)XlWLInRz=rF0<6cSMw!M3^ukMlae!bU2
zqiFYS%bT{YdVhDRl+3j#J!fz0u%Kn{-)}c<b!B^OEhn(=dy@3U_T0OtdlK)yvYBsM
zVEa$-nyslqq-}9E)9!N$!q%PKRko}AHTLcox3UenYG<>ky2ke0Iz~Iy|4O!d;tlOJ
z_oUeUG4Hco^Im%Y<Hz26FKAt`yT9hw-qU&a_u1Tz+5g^Y$G+0iI9sjCPqs$y%=fjb
zt+p2W5^N(=f5N&-o!9QB;_uzc$1C?Hdf(k0QMPxlpHRyltr||-6H<q5AO4)XcfP%n
zjn1Nnwtw|R_D+-0vVCXp&L(zF(_XfM{=IvXkJ#q#k+TdI=-K<W+s!)4*n6+dY@>at
zpH|y`JyN~bswc@daQ_xtf#b5f@4rvj?QJD!CvUcWPv4)ldl$_-X<I&J%HA8xJ#6*!
z{H-<?tJwYXXtXITIkNX_)H2%(DjW7L|DnED$dJMAPR**_YSCZzEO*YcwVf$#=Tx|A
z@00TmHgC%0_esby*?LsY-Lolr_Fj*l>-RjF>acfJKC^A7DXZPBOPlv9D0*9YnI_pr
zhY9WT`SZ!vPG*s<;!TmgM^-S~sdc*U(QixK%XWB^?Jd4<wj6I0ZEIMj+D1OuV0(7*
ziM=v+n`~A_?Xi{U6SCJ|vta*u6@z`^-!ATaRB&nU&sU50h2L?qUs+hS|90Q@ePK@b
z?Rre-?lqGCWIG}Mx6Oph`g^^)o%de<d(`Gy)XhCsf3Dg5J1cDa?f8<t>0H(OIS$m@
zA7!nvmskB^ch&Uv{))>Q`yM=H-Jh|x)lO`~|NR9g%=fpv6Sj7IdC7*AbN`+Tt0(T|
z+{b0DXs&I0=CF>f^f&Fjxm8JfXC8TAy<Ajmf3u;^LHR=$?Qb7--@ozd^Zf=_`u9H6
zI&SBySK=^Z;Ys_&s-g#EribjGmsqz?T-JVH@EOMa3cmdNcb2j_tp2EJ-`(G`&&N}I
zzg=L8{oeUv`>wnF+`Hft=f0$H{(TR2huf_z&)X-z`^R40n{)Sh|5~;8UiR<3jP8f6
zgikKC-LQgTpL|gL-rY+N?p^aid5@mcP1~9B$8EOwUD%_1pl7e`McIAnA;vbB8hQ84
zEitq4+q%o9>#m~RRWlRY2|?Ai^G$#6O_*`kdXdbhy}wIMZD-lA?>qMA{ocGM+I#0Y
zFW)^it#i+eX|1+wGv8V}Hg2^kT$^w6sHt<0wEKTs-J9vQd;U$e@%++i%W2)Y=Y)#r
zJ^{a`y?cwc*nIoQx$mIu!Ts(V=Ix8QxOLy*xvTf@zP#Gr?(-45&xg<K(_SNPKhZMP
z-s1C?y>kCI+5A>LWcy@x^WL1-H}~4;oV86Vm)tv{_rsny)&_fxZPoW`ADC%7F=&x(
zdkoLsfHPC~>||2hyD9Xbjp4gs+nw*&Y>f;a?{(q7Xk-3(iA~Xd0b8H&c-vXo=C<vR
zb@yh79kg{R=-pcu(70EnJ8!Rl#MwRkuKl~@Ph0POr}Am9aNjFyTgG79mROa2hG*3G
zI$N^arM-B%(|pN--J$bO>}}Dq-DfQ4W&5K{)b7oLS$nhR1lS4o8`#NL+^`l|J#+8x
z9A>*)ZEN?3_!#fc3q4~UVS2$Xt4DA@ugf*tR<Viu^=%yOpXlcA4QPF8*Z%pSZO*=1
z`xNV@+XkqM*=5Ib?Az*7Wt%YFdY8IX@Lq}bY+D~5xxJs8G;GzQ@@%}Un{6&Ar`iUV
zGVbl!`P9aLleNw22N`<|nsfI)X=t+9+4Fth`&%jdrS@Ik%dq8@Z3_47ecP|Ew>!gh
zaNmWS?0c1s0_^w>t+d+~S-j`d+>E{QzXkU$`MPSa#(jCaTahz&y%iF-``;;Kw^MzS
zjpdroJ-<bY_HoR9vag77tzC4?UAw+@^Xy(vGTyK2UwGhtc)Xn&m;Ao6nJ)Il*0*f)
z&#c%hykPR)2QmAsB?{&DS}g=^-8y4yy=J>@m%E{D(OG`mxl`BLimRXA`>3?t_5j<-
z-6uYMvwf^})#kC`!M$@W!fei3w%fj$F@5h{J-+>$qD}V4`sdhRpR#G6+dirNGYW)k
zqxsL+rSF?!^H1!fU1RX%eK$U?wNXys-*-G_?_RNINA?=6w%9Wx%5$&%52L-$p0V4#
z_;SML`)Wno%i#vL-<EjW%rLgyE4C)V=JOIR+l4_5wqpGjwlPa?@14A3=H8yU7M3$_
zwb_apr|!!RzPV?Hu9xk~pS*Uzo>%TszcFpEfl0$&-|hW-FO_%OF7vbA^M`MlZT>Xh
zy{1~{_Od=$zGt<Om(9|Ooi-1{w%GPAkJ<ZaNr>$g*1tCYx|ZzqS-NwN@S7gndlQOm
zE2hZU2<-0OExKxt?HR=zdoMhwwn<x5U^7)s*|zs$z3pntTYHnHW!ie*H?ZCC_s6bH
zYyRx{%=Bq*^q1>4N6dt6)=odR+hqUkz0;co_AOpM!FJ&iseRuszubLMqs!*!H>Q0;
zuWa|~UE$tqcZ9|EXUkbzuNOMDT$@vE`JGnp{hztOwwHVU-W%F3d;dz!-)o+|z;@QX
zVB1>mJ$s+2*4qZ(J7;?*V4;nNnupB-ee=C{-yGXJqf&kEN50#84=jISbHL#1-g|P!
zd%yR%?EQJiZC_;5`Mo;*T(&){-EB<6ChZaB3fwFCag&|I%m_Q5N1Aq(^EmCQR(S6J
ze)RahBd_P!Wx5}=SFV1#FWh#wZD!H*y}|rhdp*oK_s-k-$VTAJ&fWbH%zLwgg>8<M
zYwlaK;KClJ*=%;=qLOyX8%1oDCR^@3mHJ`NCSw*`lf57IzBd16d&ReC?}O*Lwu{zI
z+AqJ%(P5=`y8YGs^!;gjlO4WBh40V3^m+f?2OjoIqL=NPC*65q(#g`@v(4w)o_ON4
zm+61}p4+VdZERlNvE@A=zxU(Ur8X~=uG`++)oD}oU&A({_`2=WIl6nz7_ZqDnilPx
z{`s~IYetuC=j-izm$*Ca^=-?r`EYFJez`O)d!Czj>|A9%_TSslynoq)+<hOeRPS3I
zIb;8NBZmE&zWet+_vx?+I5}(YjTLM5<UDv|yKBR_y@|>_whz7<+n!p}wRd6olie#W
zitJUE@z{69u7BT#ndkP^MDMg?f8S^KndOMxf4@w-pKUAb9@RzI=}aiFdlFY|eZAp}
z-HLS1eJztt?1}!Cw=Za3)V^0!rtQo4)wgeZkitHtd2{#0&G)s}?a16e`)Itqhu}Ip
z&h0JxQ-4M8-&p*A-!ThOJLMmI`~D}c-nZcKqW#G;6zm;R829f>=dlm-iQb<TwR>NJ
zW~cqDr5tvCl9%@yNgc3TSK)4dHS5VflVqI(PF3&sANc8M7hCss-{DjFc6M!D_LIH`
z?4LF9g?;Rc|9iJRU$EaQW4>K>e72q1p^*JE-#XgubZy=z*TZSMCv?F+mA(u6RH6?Z
zc<wK7@O^mhzNwp-9i--%9OS*WYu}u*n*D~N)(1kBX4&8P^><(3YsP(Qq3d_$*+=i4
z9A{`7sG70&yjQF3#tjFop9@y+eY&81Pfdl7t@^KI+piw-HW7zj@4j!Xu($Dn(moc>
zT{itI{r8sJuGw2~S7D#`45fXo!qe@XoO||vn7L;6%$oLn8$(a*vDz}vHnG0MzRk#D
z{|CeO`!=Zc*k)FL*c-!9xBpIloZX+Nj&|w)lJ@ik*xSWi*|l$5ThqSTy8HHK=sE3c
z`Z3M+(&QG~IgAZ^D<+%Ts;BJRqj*->c3!mq-ifOEd$pK1?Y+M;z_v)@qWvGv!}i7^
zEeGy8EVMhC(!1YoMZf*2AE)=9$&%f_#D1cEy<)G8pO>f2-|uC6Zy%Uv)9v87`*+wP
zn+MW@yG7-d_a2l9+RLKZwU=*Ni0zz(n`~>lO!oP=aqO*Kw#z15@6cXh7jL^IE(`Xq
zYjfHww5iKhXUj&L$94PdXB4ipPwH2-4_qd>U&<}XzI6W6{d@j2>~}QVX7^e6*#3hd
zQv2V&n7yaYuz7E<50{O+ocZ27M|<1nE|s>I4!yFq4v5%$V{MV`1m+N1E?X&^Y*~jr
z!ghRiZ+k!4szw>?W1X{TPgqdY-sCr7d$az8?|pr_XK(id**)UVukJ76U11mWXSv-o
zrSkosEq>d5>t{HyD#^~CjWu+iLrT#8YM%N71(#Or>!?Vw`+U5_{>Ubl1CvYd+3t2-
zzyH|nZFVI;Z`$hVOtb$TzSFke*W7;B|26yP7QEh@<$7@6t+j9W8tjPQ%eKjRzu^)4
z{SR*k?EBNcc(1-~qHS_%zO96nlwG~xvAu?D^?PQQ@35J2VVjN5lKVEs-}<b1ch1>8
zb(8R}Z9KgDoGT{mVec}wU3pV&Z;RQrJ#p%j_S$T{zdPlW@V;Ja^}Rb@G1_*D3hp~m
zJYmld+u61=j?A^;cp+wc^NGLh<Ht|-t~?XAXM1yt?epvF_lD#zviZW|v3D6qtnKB)
zxqB5FYjztxT(Wm1=VaRi$5`v%XAar=3tq5I+5U4+ZSV@)sQ!+<pFVinIvz9Md)z8@
zuiYkh+qVVbdwGAW+HUO<x1FJtzE`4#X<u5a+1|gZGW#~v&EB&-Ps474=o6b;GdXQn
z9@=FWXZ>T}>SqV--YBizlk(Ko?rE&K-OM$s_l3o**tea3$v%gyCOiEPRyLWTO|}^q
zLu?|xUbHQnWoNr2aKWCQKp7j^nIikPA7rpQD3WW-RuN*ya#mvB-Cv>mZWVX#H9Nqw
zZ<(B{-ETXmePW+x>|3UG(q;=s$Ue&}5w>le^Y^TD^|QU*cgN;P<hQ*F`)1oJF$>r{
zU-@+J$wPd0YT=UmrhMCLd+T}3{=MC*cH4Ia*sTlG+keDI-mZ8;u-%I98+Tn-XRxzK
zf3kP!c~jev-~)RN?N8adXX)EcKk;jK$IK(UpQo$r`=7mGuPdv^-W^g3`~L1N-osj!
zx7Wab`Ch>lZ}xiX$J^d$=HB<=*zG+Ja~<|hx<AQ6ENhA_>vD&Eu38!nYke>6pD(0z
zAm*Cn{#T~z2NE~-?&l6~u$@@?Z-3g#!u?k^sO()~p|`i8?8Tm(ReSfIQm(Mg_nNlX
zg*9mJyY9}tYiBRBZ84g+$LnRs-g~MHwym<eZ9Ugs-uptg&t^d>=U$dGcWh>Bp0W*K
z-M6=HR;KNY)p7ehJlGv}v@Eb==v}qnU}lPa{;!AoTc4=gxBoh1duy-E{#|Rl>|5qd
zwUwOEYj>MX-j;KI?%s2{dA9F29NGJFQIl<Io#<W;EfdQN$0GLqZP(m8pJS2j{55rZ
z-$^ap+wtrF-f~Vo+x_h=wzWGp+6b_|u(A)Hwl_;sVqd^6MO(h08+&)l8}CnF-e&iq
zZu7n=6<h6=zwp_=>@$!3YtL1CPN|*SdwENnjkwI0T|N0z_nZv>ySHe5uWkCH_qHkz
z=k4(}Y~Is&^Mm!Hx|6mWdFR`(9^<ez*U;Zvwuoil)Q}?E4EJBQCYP4(y=QM`n-#xi
z@5HC4_s+GryvIZL(B939vTeowu<xDctFUJW%hbI-UJABG{N8(+wu{^B%erCr?8R35
z4#{8p*(d+me`H3BT}fL1f&5(|`=6}4vHvC4-8~;-Iqij;wfB9lJbYl<eG~irOdb1#
zT9@uO-P^j~bJb^?3u{#ls9M+Bd#>%bf0P}1ApPzo`)|Gxc30T0?Bx$wu)imA&Vj-M
z@9i_n`46xJ9No{{rRcCgb<y6gFGp>^h&9^WfAnpSo<!!}l|uG5ADTDXxQm?Gd#`No
zZd;X>y<rA9`|ho~yuW4j3Hy+bllOV&2s#`}o4x<lGN%1of;;wqaVp#=|0im{xYW$O
zH#9cwnIs-$yI66t?Xt-W_V%@G+Iy)%(RM?;>fU!r8}>e%>0tXMIDcP}+jhIzld|?Z
z{aLX$@Il+|n}!K?Z&YvX6P&HGZ}q`D`&R#o+Glb;z@G0o@BV`ir|mbceY}q^DQ2JP
zhvoZQH?6U|!#H)nOn#XCx;KvYdHmx0CnO~9SDYqwz?9?ae(~Z2dv@!n{f-;k><qpY
z?VtSjp1l)Wi(S&3y4^)ymu>evFtycQeaJTdsi&=Mpr{?IM2l@hX3d`e;!pOhJ(_6y
z;-u8R)T^wvh9_0`u97U<U6A?7HX`bjt;y2Ad;XiX+Z=f*x36uw#oiE}XM2qNHTQn0
zl-+x5xy4?NiYeO<1s=9N@vd@jWk=ZF^D->9tjkqw3#XdfhJM()NAybA-u{OMcK>@X
z@4XW5xTmL|Yai3)hkIrR=<Rbn<-6DAW0TF*eI|R|{&L$bE%LNorRcV|bg#mmxCI~g
z@ccL4Xa9AZO|`F@?bfeNd)GE9+ur7GvGshr&F1!1L)*F_w!O=b@Z0hhFWNIHb%9OM
z7nXg?PZ-&>*a+<Hn{<A!xy9o>zu6l1#x7vCQ{CLLci&g8eKp~$t)%BJ+&#bGoQ=qY
zn7!|~toHJ=G}-zZ^V`0g#A#c>TV-qI*0=Y?EkoP;?>5=aIe5YLhRu~dRxg*^e3SLw
zeQD+2J>Kn1d#7k++lG8!Z2MFqbk~#y#=SS=s`j(lY_v}@+-N5}o%g__KJNXe=3m+$
ze^6z=>$e^IOn1-Q|HoBy|MoxOHebqjTX)2~-^=N;(l)y%!S-B)gUzS@qkFvXui3jN
z$Jgfm%RM$fWBKi_hi<ohC3|A;yo-%CQ$<(W+*|V8M&{(4y-BY{?RFkMuxC-f+1{5O
zI=er3Ew;I{L3nR@{CnHCPA0ZjE(-2#ZFaTQ7vQpeG0D}6k$v&r$udW5W(6_qliOdl
z_szBsw#V37_gd?m*pu?-%bo-4c=v@%h3$QB!DXkelx{0<%4pB6Ym@h0ufJz|Hj!iB
zqAM$GeZ$Q6esM3_r>f3x_s&{x?^@5Vwl7aS+@~y-Y5R!Z##ZwEo4sD;?Rx{~l-N49
zEU<B0T4Kw+QhU#=twFXKnf&|QUhTD=RC(HVeM+OvYqN*8oqq&vi~d>e4e2~)`|$ID
zJu9m=+AtQMwfXDIv9I`r&R&J@ulF|oa@-@T=3_H);)%Vt4t?H}6=iE1{!YU7fpO*D
zh+ILt{eN2b%KM+TPUdd3Sr@j$_9)ZNJySou+I?XWm)(vslf7OtVS8P&4%p7j4zrb=
zTCg|%#g09<LblufT35Db$Ch4OP3c>E@<qScME%)kwYzGS?WZ?MdzbPE?ENQNWowi;
zeXrH2_PtZDncKV#Zr}SnigTZutn0qsO2vJ9?Mv)34qe~#<H$GL;JVGbKQ3(Cw@`7p
z?f-I{eg0d;>{_d=Y;${_@7=V$WKWG;qD|TipS@S6%h`Iez1m}EGixtjnyl@bomB_A
z|DCgM$v?J_`^!3e>sJ@{9k$Q4@0qb^ztJ{D+wAIy12OAo+b{X|W>1ymD=W_qJzJl*
z!PYA4gl(6sp0T&(|KGi~2Tb<PkLI%dZ^~e2u%63a&~>F<Quv>Jp>g~76)FkXt$yFI
zPp>L+ul13Adjk$c?qldYu(#;qsojP1f9>g%+qjowY1rP&zpmNL`Xaq=x<ZUizOCur
z@J2qnKgQ?xW~%+Rt9&lE{}lTTJ69*ueX%DC_ox55Vz(v1Z*SPlH#RfW@9q2V#k|kJ
z>%G0_S6K&->X`@r_^;c4*1K`PVndMqIqyV!nIkLqbL_pizb)Q(pXq}ed%py&x8472
z!(QJ>A~tIUSK3MizSujtZ-Lc$%}kpM9M5fXl~>wmIuz}H9L{+_#_p%poz5k8b6S5}
zM;%*i|E_7$el?@z_A|qK_Puq@+h6XivM(fG$!0~D>AwEVlD(#N%C;KzX0{PxjeBLT
zX4_UKU$9*f`g-p*i%oVMXA1T$h-EmikZbAwRi{4g=l7M`Kbuu%AFt-aeTteY`}M5X
z*nKx@+7lVxvG?<R>AeZUEB8bj>hI+@I<$vRUeV6y4X54mpQ?KUwJL13b#2(Us^P>w
zp6Q$HMD%Oz#N|Eh!{lxDJ>X!s|I8n{KUnmso$A~d`*uBcwNv{aY&$9F(B5^E`}XDw
z7w=8hWVTsn8N1hlrNLS}opqmBZo=MH9yz;J61Vq0Tz_h>Qr4_}-;RB<6*=Q-d(A;_
zUqf<)-I{m(c8x49`{ZVxvMYBo+NasscwqD8qyyQ!J`Nk7ecpGdsCB=sq^3Q?lJ5Qa
zUDx&-TspNsMc>SRe_MzB`wuJaUpBEGkhhq(e~ZVq{iflb`=5G6+0D2qxc_6(Z<`wn
z8TP%kQ`=kbq-R~<{bJAg^|$t#%d*;*?AW#2_T-*D!R~8zJI`*k*VwStCTH{I{WFCc
z_E|fv+V|kX?EPt+Ci|ymO|fb3FSfIHX4-e1<H!CCl|B1yc3j<OD95osv@6Ko*jsOZ
z<nf#Xi?467UprlF->NkS?ayx7ynDu%t+rc#U*D^3`FYR%J4Sm8-*DQsZ=bYxAN!0w
zD>cvW@msQGuhi^1`_37b_DB9c+b>aaYX7{DY4-C1WcEj`XxVT3c(>gg%jx@{czv{L
zia)k@$(mJmy(`Y{y}8hNUuHeC?Gi8Uy&7|5_il~Q-ea`%&mQ4beEYt!1luZk&apkv
zd2}z|vw*!V3lnX4mM7Y%$xGQi{J3(j@3KVOl!qJknzOX+Kb(|qA9l^h{&BppLw3%V
z{Rh`JA2{3`df?bjU;Fs=d+mOCNZLPK_h@gIU$>2`p|j1MlS}rxAJf~@uyNAf<x>57
zy>dD1CS*;t>0o+byV*oxpIo7kUFJ!ly>BddZ7*6V*a`2D-6w0zy6@xqEZe><>bB99
zMtgZRm+rOUmf73L(qXIc({N9cfZJXp!-acK@s!xMSlzRkvUY+E%bU)<KhKKV>iG-U
z++RF-@A^#&wmQed_SzRa?X@{}+ve))*u6m$O>O=NKea7AE^D9a#bs|9%(mausmwlc
z`<}i2h9B&d8J5|x9{;(obd#dpMEC#umWj&RZWeN}`@5NK-@SFtduJYR+Slkb#a8(S
z(_SBA;eF+;t$WJbnfIO3Twzno%e3#^4E4Q}Wxm^TnRM8!-^IE&<EyEyc2Dmf_JjZT
zM0xz$D<5-mZ&O}~%~Ahy+e2riY&|}7+CGoiYg@c!rmg8S5!>sQ3v551(Xn}aB*>oQ
zcFq2CkGJm6oT+7}ojPgXW2Qd4P4@Hm?cTH5Rv|XSKIT26{pQ<=d(!4e*>2f*Xz#~|
z+ItV=_t@;){K+<1aFXr2<*WBdU+%J<7qoY;#<He8$}Ved{?5C$cWpbHP3lLcJqJT>
z+5QthZQIbpw)fjR?>*5Pmu%<mKWcZ#uzLU2+R6KRbN1SGA6>FHYSjwclS&)y<CaJt
zWZ0T-FDb}*z;=7bUasuTyT9|Dwqd?E)3#3G`0kLSl6Jo$uI>E4YvJD6N9XQ2bzIUm
z(D01St<to;+Gh5)Gi@i@`1lI%72Kp_TkWP~8}fm3AMX@_z5cl}yWMVl*?%Fq)&5GQ
zg1y-{DZ5XrV)w`VYp`42DtVwLW50d0tL47^>va!YSRJ>wGjOHN&7`Y)ST?<~^<24n
z?{ZPIy<VINyFVQ@wE1_{!qzL9dvD1<TYK|Qx9!F5-QQ=KzI6Z5o;UU<9-P}RsnT*_
z<((#n>36#J7q7I~zi+X$-ODT?yI<!{@0%EV-Dal2<bBpH<#xpncG=yH{cC&R|G9m&
zeSLcmeTudHZt&Um`oFcdr>_<4-TU>;-b-7vZ8a`U*pn~ZYb&_xg)Q%j*LxRCS+#rX
zwWPf<spoA9;vQNHA33~NLutj{4IA}s7C*aUyTI`1-X~iZ?`dx8un|*I-W&e!v5oE3
z>3eJ4W%dS0weLMNb*jzz4}7+LPj>FTeZP6Hapi2=*@t%TQ~5A!-^skY``0<<+2&q3
zZ(AS5X7Bol;egW#Ci_zi&9;6e=KJ=%kl(v><+ObqGnec+x^UV)Rux7Y_g8v$DjDW`
zw|Z{ad#PoKZRIhZee6{}`#M9C_KMlo+dlm^*-orKXz$ln7TdU_#JwD6@9eRuI=6Rs
zJkQ?MuP*MrojSqR;Nx7I#t#c@7747hiQKtx@BWs(d#;r}v3c-!-QJ@qd3$Gz_3za?
zEoP@<-?Hze$p5`jJG%Gnu|K)zYuo&NvUgJUEK*)(tFYwEUWTSV+jI3?`#$Mg?=^7v
zzc*@q$=>-|y?ff$ud=n8#cBJZ=I!puM^D*b`0;rkV_nJqS+}0q2OQ+xzu^V<{zcE4
z9j@x{wC}s|$i8Ur^8JkWr|nf)Ds6M@{v(_6s%D$_>+SYhY;CY9+qQp?fiSzRpQ-rX
zZJr#vCE57)F>e;IRQ6tK>$zXc?q%yvTbZ{R`|cewx7oqCaqp+ahxU4uuHAc$C3W8l
zgFHKCwOYFb{W^Pwdqw-}l~e7WU(m5XpyIRdxKZ8y0Fzz&4=J?Rip9y>ma2B`U1YJ&
zW*x`#J>Oqnw~hbYwO3tW%3g+RY<6pNe(pUv`^moEC?C6Nw!7`PZhG2j#p&<kJ$7%m
zanAkyFYYd}|L|+({+TY*_rBG?vA-_9#NqXJBL~(~;Rkfj?KmLyu4=!}RfYqH{&^gb
zH+g$t&->H^Z+`3TXTG@8?!WIr+uJvI?U^@x+beBbV)te3OuI#b@%z>(d)jfX>Ds?l
zg5Un~mS=m5(x2NoKIGZ&q4B`Z^!j4^uR6i@|If(mFMo4v|9itf`y>8w*j`PvvGh*b
zvZr0n-zIUP%-)9ZBR2QnKe35>HDh<<qNRIp2yU^u$T@ExXUn@iuDOObyt}vTeOh&C
z-+jLWwkw}`+r7N|W$)Rg@2u;*4D9ku<M!4GMA}Z8E5C1c#B=MuDU5qxZ|2<>EckEl
zglEfarRF@{?ZB_Qch8Z`y-&2XY>(cXv-j!Ub$jpoMcFPo`NQVzJ5yV~Np*YfD&F5~
zZEj}!-lTr-D*qEUjVinBn1oK+y)<Us_aKaU-|anX_dX9hwvX$RwH@cRJ^R)taO`~%
zqq6q^&-uN3&)4n!)ZDzMLh#03i~TS5+PNOLy=A6odo-!t=GM&(dryis*uF`<v^REZ
z@7|tlMcdNZK701OPq)4AvdDJ#mnpWd3+L~>eE7v)cNb;b51%FW1+D6^R?O<!^LSe8
z-d9_0*`Df{ZS%MCr|q8$skTS8S@vzX9BC_gm2of6lm)imX205NpeSj3;1SQ>CA_@0
z$HGEvf3D)O{go4H`(1XqZS|CSdqozw?)^S}neDzQ>-Oe8u-ofU8M=3cxvtHHx?b!0
z<CZo*%=g;(t(MrgUdUiC=ZzV*XI?$9?RaEst0?nl@3H_>o5tzq_AE~PyqEQm#oon#
z-q>UoE!;EXteb7^{dId^%c<^tuEJ*5``mwTtjv6CWnbp~j~A@nx8(B!JO9H|?0So@
z**GSK*)^%ov<tFP-d`N#Vs|giV1L~~4ZD}tytd8%F7K`A>fT*;$a&AkIh$>4bL4Fw
zXdSc7PAS}b<d@A}kF3PKPxH>(<v$7Evu`1f{oAq^duz^a*qdIwVE@8>cWi6+?XZoS
z=C$wjW5>Pr9(!$LW-Yfd2q?69k@#TG_4$Ww|J@R{Jz9CoHsdM#o->vYZ5o!e?G0Ic
zWsk#6FYEF}3vIVt-nMt*&+E1f!p=5}JXrPyYpk@jn<>55*<E69GQ*BNA1X8UPI#%f
zC!}lb-bIT<Y|i)IwB5@!e{V^ikhQn_TpKQDi35HM9`9QcwA-%YG>3isD=j<6*igGb
zd9?$r5?k%rH@@4Sz^rWFB+j^(sdusM_4v9yPQhCHcusw_*-_54@Am2~dtVy%+d7mT
z*gMBYWN(f8vc1hJs&*f)O}0JZxzNT$tZeUO(fxZ?^orW95>VT#_t#~Qmi`9I;0YGC
z3z#EqKX>x%Tj6`&Qjt-_?rNohtxHX(ZT}|8y?5qR>``H!Yui3Q(njaJmhH_&DSI8K
zUb5{C5VG}at=hZPhRN=O`<312+cWk~D?PjS!m<b(o<nx~duki(yC1|Hs5-mHZs8kN
z`^6Et_S<f-?Z4U^xVNeA{Qf_D3^rRsH`!i0eauF%QqT5eQS+YtT0FK>42x}C7=`V4
zRF~{M?5Aw2&G^$U=kvM!Dvzhz_INz8i;G)pXEL$hwt7vM-Q5pT`+uI)uoumg-|rZ8
zz_wQ>eeZ<>!8Wcx9_=Zz{c3xt^WR=6MYnxcMbftCe>Uzlyw0>YV~>d4u|+0(A5_lS
z^UbBrhJ*dko_$jC`=SKeZ0c$^?`e^>ux<Wiw&z^trM(Lz)a`V4Ot+n2&b5!tglV5;
zR==&s;-<ZmPfgkP`$ov#A1{9FJ%3iqHirF_?dnOZ_I4@Q*v!&T-LvWR9NVjVrtJ~j
z^=hxNO4FXi{9Ajjn>z2kGTCs?KH1f~S03fqxARu4?OXGWd)W?O-NUCXxUV+v?q0P`
z%zI~Rwb%$0^XyZbEwk_IaWUJ6R~4<9r*E_|duFhAg`2$X<lcb2%%ulxXNiaH)mY<Y
zTiW(~uN2>^y)CW^)^Cp9wt9Jcu1(RENZW~(%C_v1H}@QvQ)PE@@$P*Ww<_&h_TR#;
z_*B>aH&>tSJ5k1BZ+w`|jwQX%PNK?d|J0Btd!F2zWBo+^_ui+%k8HPCRN2nHf57&4
zrR&~TQ7wDx{w&zrur|x8>EQdl8TLzT%d47pA7?&mqnrBBwsUKRP3fwFy~>xx?Oy%)
zZ&SZq)sC^Wac|{!vpq$RChzS_uH2ha!?90ozs}xwM?-D3o=&!PC}XudBi3PaFH*tw
z`(sbrzvox%-ddq=tGa~EZpVym)>jgwZS773+w^PK?wuO!XIt3)d+)D3nhwGVr}k?d
zn!SJX)`|As4txiubuk~f(-CA}b1&S^Fz~fi{2O1}!{2W2NzHBDd-$^3-t#-R+H7wU
z+~=os!S<p2v^~zBB=^mK$-j5y45|HcHfQ!(Ncioat-9Xc{D_b}UvBsQpTcVUpD60>
z58(5*|EA|^D{=j;ZA@6-ULD6-dzEi3vOce=y|?4%`n_V8me{VGvB<XbsnEW@-*&bS
z6=&N9A7!*<%kQ`8-P>x@e{q)0qQ(AuWz>$_7C$?>cha<bwvP{M@6G?~z0dISWc!Pi
zNA{=X1?-<bjq5;bysrIfwT1)To-ggiIoBRov6bt9>TI9gozVyO`7Z0Qoj<F`-XSf<
zuHB@1-wuNyd(-F*`%ivN-W#)&Z@<Q+xqE8@-E1cnZnQacbpM`vWioqr@-^Bpd2F&Z
zU3T4?@$i{FzF`+^&am?AGoLwe_g1yHHZO0UvXS|}+BPJ^z^2?UV6R+__1?`^=WK<Z
zao8;Rplfd+;Jkn7;oY{8uT^ZP{0^{N!5X(;n(4tlvnBlY99sQ0$%Rkswuh~=HNUjq
z_JBpC^@fc7)&UxPd;Y1q+DaU$u{lsb%lh5aT-(OgO|}Q`XYQ?eb$ZX)?Q*v59xQhM
zUo-8g5t(aats%89f%%xt%b?49Z%seC_tXzzTSn*PJ<p1-*xJO+*mX_dgw5U*#=WiQ
zI&C{_1@;B_tJ}^Aov?R8^ToYyzs1_tZEmxzGRU=Fy)VV~EOVo6>}fu`SGVkK%3?Ka
zv$Z|;K5IX2Gw*Sat+#!Mt$=Hit!(@q+w~5bd*4d6?W%m)Y5VYeh;7=Z3Y$D9OZ(41
zMeLtzT-;ajuVcT%Ar(9Br(OFiQ*YbxRjKVyj`ZAjeV^L?KT`cR>rY&>t(hHT+jZE{
zMnk}Uuj!6EwsVEjY^8a6_SUMt*<)>Azju`wll`v^JM5kxFWvtqc*(vQCLMOq&N14n
zHrDTT&g!$X7rAfuNjhlXQq~t%p4`#4zF(|tuQkoxd%=Kd-)W<-d-T7t?_0m+!QM0X
z7Vq7f?O=2A%dOpgyDDv&_BGmu*Zj5p@=ngS*fnvlq|jgM<7fD6AMAZ(8)<sVM!z<5
zZ@%uPJsWJztr8C|wC(aq+WYyd`rgfp_U$?Hd)M9odG|f1JWKYtHtgNIWbOr9`A{?4
zYLWImpHkM?rX2UTV%*7X>t86mmq(>^?}CZawkpiOY^+RP?Ntx*w@SIU%+|ljac@NM
zdE2GQ3-=seH^t_TdyVa$9mcj=3S#@^*Eretz1Ftp<>TJB_y*s8o#~rxZWIRZ%e^VL
z|5N2D8{35Qd!v&7T24vJu$f+a(dPKn3%2*?<=F~pFSAWL<6^Vqv%%iE&tC3{O@6$$
z;(L_+mq5+~r*)L}_bS%x7mR+p|6Y8*{e(r!?R&-E?`M9!)V}W?&z?I4^K55Z=<SQ=
zTePS4?xek&Z%Xb>oAznXPC?DROQIxg+|P3CowPT{mR(k2pWXXsdz=N_EfTjg?wJ|&
zVQ<<_QM<bvl5K-LrEP_#XYcJV^{}<v^LFnKa~G?l+KX-3f2!;WpQ69lbHT<vStl>q
zM)IHAdqsKTp2ZK&>{;}RWiQXAr?waOa_!^UvvAMclVW>%^k?r4ieR%{qGWCJ=&pzL
z=^4vy&bUO`vAaLtH;?n`-X(>T_X~T7+P}85uy;~owbyf5zMr3Ww%vPeuKnk3i|rL@
zso1CQb=r2ugLK;oz43daE8A`V8EfoazAAffkZkeZ#nZiQHiri8d0soicE-$mdph)u
z_g>!e!`eFHxb6NB3mdQfhxSf4VzoJDvcu+k5UcI%qt|UyLi25N_nz95!E$4d@e?K6
z3U%SVIw?nOoHUQvcBV($Rs~Mj&n4Vp_okoUj<X@u{)*lSJGt{W_Ldc1-Pii((7u}4
z=k{^B7~2&yb?nXg&tq$vmb!P&#ksr9r|;UkGFD>m+|^TTemaZn+qz=ro(7L~d$}0b
z?G;_0VY^<Gcduu=g6*q?u{*EN7uk1qvHzZfyF~21+|;n06s2Ux|9hs5^!hh8=iXf2
zD><QMZ<+ssy_HLZ_8ntow3QX)-8bQ3({8z~8}`onxMfejz(bqe|Bl-3n10fB8Rv<;
zJTHUyn*BayYu=r|C$_E4cF`=Q-S+EN>^aigZF{0oc+WXj-Q8XK2W_w3-D2B#^3Sfn
zp{H#RrRdoTWW3l@XT-m^^-TW0687V^MJHa`6!V<6apYWU`z&>uZRo9b+wiBqZ5F6+
zviTe+Wh<>Bvgh`<<9olw|JvKIQGAd2$}77U?_9B`t!0|c{udm!N12@VRHdEY`;*mg
z-<K1ccVB%ZW2f`@^xn5si)}wHh~Mk$xODIDe+@P_o^bBFu=I*uW%{PQE9-CDORxwX
zv`u0>AnTUoaAtA#{w0NJ`{O3R+P|!f^MKBNnf-yMd+d#sqV^py(YCL6(P#JNs=@vj
zKX?u}1?kz{^kKKp`><l~XJy+1pHD~H`+K$Rm%Fs_!1A{m2g15`I;47r+h3ICwOd)w
zc5u}#(*u*t3vBB*->|(>F==npg=pJA-RZVRrcbje`?zQ?)BMK0i+CCLPSN_aCt%|#
zYx`?8duN7U-aCQK(UzC>iFMnaqkGM^^V^-+pJx5jNp9a$D;C>h1vmCAKVh?Xx<tjE
zzfU&o$vpnTcALfg-TOb9*v_<Mvs3CkZ}U!C%x<pe3%kS<hW55@=j}I&2km<hy=T8{
zmV&*+8@GKwqM7%vW!t<j!C>9~#9X<(ryL?}vTTj__PylUckZEwt>@$Gds}wM@0&Pz
z@7~3Av3pniW!-xw^YY&Bipy;M)YjQNysT%d-lMU%-|4rlkn{7sXA9+Q(_$9xjjpNP
z!*gcWp1B{m_MNm6+C9IJWABRULwlPUI&Ch0mDoF7O?dCAEr$DM8sE0bH~+G?G;p)s
zyfw`BUM?*A)}8R$cQw#$-`+)xwg)eK*{3<<!fuw=mv^)L)Uwl2`o34eGT55q-7cH+
zcS3jX-nM(s)=AQKz28!8kE}M`tCl{;#*e3B*Q+o_yXIvtY%gB6++$dL*Cs0Pn@!A}
ziMB0CE_>bmoNV(N18vn0&fR<K;cr_v3zdDF_qN!)ihF6ht3rI=N(axq?0Y!&Zb)Cg
z(|n_|?d&E-J3nb1>z6?@ZSGnI?Txt}zjxK0=G}M0ckeN&_1~+!X~NzQO>6h;e7x6=
zE#%z3wiB)UX3jiryIf6hznn~!-HOe>_Ds=X-@mC#+}_^ll${!1yZyIvQ~RdERklG5
zs=GI6$?e~_*k=FlJq!0v&s)5YEu4FAd?Vw2<1XWU+b=QM9!jmUHRmhd8!S3;kGsXi
zy$Z@d_EvqpW4ris{=WR6z&&O+EceBqblo?n(rn+N1GD$a?GfI$hU14Vzw;5B{8^6s
zcD|WlJ2|y`-(}7P`$Kvg>{6JtZS^w0?tkgA#lB7AjNP7s19lJR8XQnO^l0zaW1ai_
zcG}v`YwEY%a+-Tz(lNU|-kD{4mWyicJN=PsFZZT}w%pRw_sU#dygRw@ubuV0==~ZG
zw%ARGu(m(3GSrrTQ{DcV4|Dfl=+xZbd}zNN1Anls(S$17gv>|QGJ8GuX4WpXo$|YT
zuT%x|-pL-@ZH2q*_eN}}wt1P|YwO(Zvgf*r#=iXyhiy_+`S-E4neRPjX0UJSg2%RU
zq1t<vTJ7AuucUDI**uZGXBuQ|g?bzJI?gb%JtOYDcap8LZI!El-O}i`y(?a(*eV7U
z+n($y*jxXf$2RbXqU{Rq&OJVX)_an9s`g4uT)sE1Qf42g^En%bz0+(qw5javiQ~7G
z(dV>Vvp8~ZXRf{N?PD!_rf<*LTjca^Pyf{3z0vbIZBHL>wz1rIfA>Q9Cc7fD+I?C^
z@_V=Lnr9<D>#VK5d)U6zmZW_?(ls`Tc7=NzBwB6fFf7??`=Q!4$*^jVK|!?5r$?T)
zf=3SQ$-1a$E8k#k%kX}JjhOK(+m8NywzsMTY`4D_uzjKXX>ZY9l|47qSK1n_71?)=
zo!QPlW7|H-#S`uRl@!}~rdZn-SnKRht+3hat9jh+@H-BBF|{wY8sSX)`OEojm%Goh
z-M#eVUgla3+q}p1HU}(3_x&yVyjMZ#lI?{eAzMZNg|@2qlx+Jh3fkR?zOd_6f`Lt<
zis)WeMy-9HXY}m-Sj21>nPz4?t18|0quvUeJ_q)FXYU`h+2@sMs}*u(@A8NLZ0`DS
z?=$~lX3Ll6u(wXR!8S72crUNZjlJ*As_hHC=)br3)7rf=^^WWn`N3ct_wCN^qCVk$
zfq_f+ESlNB*Xx<c-WsO;*6~MVY;R|B?)jR|y_f3=v)$Z|4%>Ch1@=`<7qVmg&9-k<
z+83MsY=>>~9o=mMQcv%lq$#=g(mn@Uhqy0$pV<A|)4jCW#(u#y`{HRk_Wg{zx&OJ~
zB0DDaQ2U<a$82Y4pWVm4_{DxXF`fM%i>KRrsy*LZcp%=kLzCUs%BOX2QwXPRnw+QY
zy}$2mw?EyuN7kWy@0XrQd#xBdcJFfXwu&*c-W$@lZm*`G<i6Z}NA^s*$i44Vtn9w)
z&qZuw-%hjHt>3kGL1mcTg+n@fdooz|xpw&Nt?|#cV+?p<yEvuUrr^o<y@!<4_B9>b
zyQg9opN%Wa&V82_Fzq|sr)qo7F=Ox9&?$RL`mgMXp5kH`{ikTJXXvB7YZETo6#4wy
z_ulca-46i<d;9N~_X#oHvx`qxJMi!HMZ2&I$Mz+yo48kP2hRah_r?1jEwHx_Z(p_d
z&))5J(H9=<y&z||pKBe5{k<(q?XJGrWyAd7_#Tbhn|J^IxYRak8n=yXB9qOD>k+nU
z3*YX&HAiZ%P31?Mj}M&o8Z~;_o~>E2cj`{=J>p+OZ58JS+8#Rles|&P&9;~R?XYF@
zUcA><ZqMFhQS}ZID|rtbxa51lcGZIYQvz=vFw1i|@TqH&{r&DQ_L+AT4v0Hmx6=|7
z*nhr3&_3nu_C4X^`|Ol7EB1*T>9@Hbm9>BO?~{9%ES<6Mx{BKV61ms=oPM*~KdEiE
z-+qegK+YXQ`$fML4@BhE*hx9Q+poW+d%xg3?Y%Vu=C-f)^zWTq?YGB0`Gk#Tbf<0d
zWi#6vcTStazcss;3m@GpBs_n=p<~LvIt2wgEs;pOLX94~c#qaSQz9JpyX@?<KRCx@
zA4kp&yM2OpZ9>FP*vL-O+@~xubuXu<%HHh<)%GTJZm?1MXSnBj63gDKb#wQezOvk2
zX6ftwYihaoZ|RWQ|FCSEy`ZqffeF>E`$Kfo?Xy$w+MOt9x6iW{uqoTOXb+DP!|wl!
z1Z@?$HrjBh&b4h1?BDzLM5gV(ORM+XzBR>Gq5Ad?`#I`%#<Sn-U3BcXg|)A#&BC?Z
zHVpU9*v?<MaBof1I@?Qc_wW7CG~a%2Ys0?qKlk>{bh*6$jhcsjQMuNEDU+P+pIsBO
z^KcW|FLnB`J(qBR-Ilp??RpFz?sFEHYoGF`dB4X5ul?4aC)l#Ap0iiBM0%glvW@#X
zgjU*KSj}R0KuO-V-0G)|<3)A5xeFFq)x<sD`&M3auVcsjJ#4R3?HDz`Sl>zbyVvsd
zpIv?%F7Lg0z1T)*>cKtnQycd-2wmM{U)8X8?_@#iog0tZFt=#f)jX5kr)p+m_b5!;
z?%lEJd(QIB->0!Ce&3g_$F^cT7j26l8rn_dQFVB^uEYL~=wW+dQN9DbXU+CoPO7li
zdfv6azF+9TOKrmg$xnQCyM&tWo3d@UP4<P4wp02q?e#E|vOSovYR{YehJDW@?%K@w
z;lH;eAjfvY74N;g8L_r1drWL4cFXSb-=Dg-t#OL2b;#m9XVWD1=0>+$T&aAyr?c#r
zt(?wI+Z^kIHic7T_8eyxw3Xo9W1D~G=I)IZL3_7ywC+jtliJ6;>8^dh^zwbpwaoU`
zQ-ANf^|{#o?+3L5EvMV<6*dRieLf|1;D&?ce(AVdb|pgdZ1X<&*n7NLv^V}*^1jq9
z%j{=Jb?!fr{9>O`!<qdZo-+FaZ>!iAhAyxbW8$@KIo@V_bXwA0HMZ-!KOPU*bK!)p
zt?I*bHXDvx+Rr+C^g!KLBZr)*&Gv=284j%YlXbvrwUhmcMHBZcIj!AiutjnIw5P}R
zzCE+k=HnVA+vu!)dv3av>^}R-bdNUYqrDw}xb3dmYwexS@OW=Z-P64ZyI<{@zhH@N
zh51XHJtkeY=cG*bxcr&E=a=Yv+wen0wl`L6ws|0b*}mx5Wc%<RE&DCk*4cC3Jikvb
zrS!l(J)HwRe@q>E8yoDDE-$f@kxblkri6Fz@%p2-(I-Uq*|@RVZn-{VZ{MNidrv(K
zw|%mvW6#4mSN8nWTD~WJR-aA&YCRjhKRVXC*Jkb(xN&XoLD|4P$IUqGcIBz;ElF#&
zj#HG`>&wJzcS!l{o}xRIdnG<H?e+L_Zcp9rB{rJO9Qy=VSM0g;&D!?0eBa)#f~LJ%
zKep@*5jeiLId=QruTP)aCLC|wD{#MSuX>l%zGr*A_d0mg*&Dv}IxuH0uU*^Iwfk4b
zHtf^-bAJDHYfXD`W9b8Vd13agfphFawsP++J<Vp<@1e2pU5KHrU8ljGY+L1h)AdhU
z%ht5nn$C8x4PNJL+nKm{f4{Qj{<6j1w(B)}Z5K&;?Aw)TX6JY-bN}1hiT2ZWtl4ME
zZ?P|@VZ+`Bi{I^8kvGMry6D5+Fo%}Cbxp;4qs?;nco{I+PV=6>cgrem+h*As+nuu8
zY)`m(?hVLQxBdJ!#&$Y)%U;<^4m)P<M|(M?SK2CC+F1vQf7`c^XV(5_37Pv6t|r+h
z{-0(W{Vv;1Cja_AEsnhX;jT>fvoo^o%2sZ+Ejhf@HY2CiZvU(7eXXz8?2C^pvlVUi
zx64lLw7Xg#vTvG1>%N$jzt&S%XWFXr8`xd@xx+^HyN&H~$3uHLkJ;{fxonGVvX<pO
zpP)51W>4bn9acozg^7zEsF~ZbfA7?<`xq6N56paZV1K~6nfs@H?b%;6D|mmfhQmGy
zg;qPSuW|eCUl7=vuXD%t;p&QgHWw}IUffz?dyO&BcH1tieR7(8d-ts{v3*{*(e~l8
zZMM%#R@(NQxVg8WTzF5;qWOEJXYbxSYjwUYgG=llosBbW5>$M5*GDe4_1F?=vsYk)
z?VPW^d$Z?GvyF4Mvo&;>ZtG+H-{z)MueHz12YaJ4jBKV$NZIZCvUTt2bfJAupGoe$
z>&U!!YY>+mqkh`n<f}(*Z}FPhc07>S8#k5NR$X%b-tc*cZ1VylZP&lvxR+<&@x6{p
zuJ)3DIP6b7vD+th?eOktvli|9Z+U3{*Q@&bF4bM!S5P`<-{Z#n`<|%_+O1F8yLYo^
zsjbKsHQRUG#`~VL_1henti9LrZ|z<k_D43JoyYgG1xW0(<~zAJwWM!fgvDoDcgY=l
zZ>@P{H*xA5JHCFQy+5z<?Ja(Aao^s5Z*2C3C+>H7`ND4HA02z=o3{Jpb}8+3i|@5t
z^vrkv`-hwDrYpAZH+!UJd&QmIE;>wXUq$7An>9O6+lC8p>`Ue3wk=<uZj&@|*Ir|T
zG#kNIE}M4>SNB|gywm1F-uk_q_8GR!4wtQCj(YF?Hhrya`igTl9xpv?-#lJq>)WNU
zPuNyqugbazo5!yiY`C8!+fL#$-aBvlp*_D|Y_Q(@=*(WzYxnl%zuaS&Z6&c!y>roC
zvmXZgCKPwuYBmJzQ@W_R@1dRLUL&EIdvgy=+UvHQ$7b97e%pnMxc2PGPS`u`tiJ7P
zOW}RXmvijBD9dLj&8xU?S?5gK){J+yf(y6q<?ivb<(fQg&xuWWd+L_x?3t`=vuE{u
zseSL4Jl~_kxN(m>ughLH{?E4C9e>%pi?g;hb7$RqHB!vZ<sH9WYM{RDcTRb`<@dR5
zdy`G}o+wt^r!Yly-x`xOwr9><w8~v5wNF5I{T}Wm^KItM`MT@V^Hkfi&M;f^;(dER
ze7nAzLG;P4vVLJ(htI2brF;|IYcToM?jLvd?peI;qebf3ESs-BjW#p7T5Uv+x7u#}
z;cXlLaG~vl-4FK4`M<DT;%#C(Vd=@8|Bjy6wc$m|UYSOwy`0w1ZT{&5>}6jhWuyI5
zXP@$uG@I9zTy{pWH?31VUhZ}L_GgdqQ=h%dw7B-%alCH*S74&;Lv;bWvrhW9qCa|U
z7hm++Yv6d!_Tx8xTQ*6*z0DqFd!`<KXk+?%`tFd(YkS%^OYiw1XTN9m^t-kW&dY3>
zmK@sW<R`j+jeM_N=_Kj>)BkL=t?;b07PTzgyK&`yyYPBtI}cemyP9X4>`XV9+wxgS
z+1+eW+BfytIolVJhWmEjG1`ASMP}cLH|K5KZ@jhbNj$e#WTwyFDRo}9pLSZ>KDaJm
zBmGclZ^4|ty&o3`+MM6&xWD0*z`mn4C-?rJFJ*te{=@#W{eHGzf1ld>x_k3JOPwWl
zqBhs|ZAullwaGlcr})BF+XL<qHVij(cj<iIW@F$t%X-(2347U}UEj-}QL-0wwjnY2
zoUXmuUT>D2uMT8xwOIO9zisz)i}HB^#*G4(Empj%wK&y$+$`Y9#~s9&MT(k>LdP^r
ziW)cWc`IOG#KWO-^MJJm6Kj~({<y1}(~cw<M(3^5So(LXDk)|Tgxb@VJ1qaaC@_1%
zbH%dkq3cezxZB$^*B{vFVUcLLa&4N$eq+mB_R1ZW)OMTjKVz${k0W;8@wD0b>)hg9
z&kX)s%t+p9+2^ryhZawk)uG=PtY$g?-bJjv*I836#npRva_^XI8NcSH`K@4%okfrN
zcKvI9Z23KCwx!KIFUuQ_EJT|}ti9A$XDnxG@j|tChw=vT9pzi5Sr~*@><qnRZC(~;
zw<Dp(f9LX<wL3oUX1AiYy@TDnxb1pYjq|>3*XI)6CG9rJNb*VSE<=?imM>OxSe<KO
z+`&GRWmj~K57qNIwZr9lQ3&{cqMsq9(jc>M{4f_R3tz_Tap|sjpXetkr}+zc9(_}W
zpBoKQLk&E2;XO;$+MXTGhU_~UpZjiKH&w>GcbSFRhaeNH^3|Kn->&<zQ}2?<pwA!F
z2#eup#v3LDiwVu?=E2VnTDt!2-7%@z-D1!DojZdBpX^ZNWUx^5&e^WoZEZ2w{eEGJ
zv*p?+nik7?Rd)Wa%d}YYQF@onOWB>Bzt&o<P;oY&aZ+`cw8FMQpU;U6=UFZt$|YQi
zYwkxV>r_oq;5*y0RkpI?puE}aU0O9$uWNnWH%a1b?i|_b<<rd1pFL`Re{#~!M;A>k
zHKoe7A1zn1=q-rdac70)&YP;HJHHAXr+Ph2tUpH84et=RQrBrNafH=ykB;KbS>N{T
zsNAAr@})A%!u6`W@g4DJ=C4#9T7)}EQoVc;vUjzrd)2=4^^HcW&3UKHUg@9P=^o3x
z+h|&gRo(1ntDS7KEzUfiHR$UtmhxDOYlkd$zV};cS$)0HLRb2XnLg9`oqCC@&HWhk
zceI*0?O2uIGn#)X%tlS&QdSd-zU|m^i)W`*<(6H?e{pZuyL!U1_fC%`lckN7LEc0w
z)&AXsKD|&F45QjdLtt2kfTjO@tI2%pcTS!5d*>UkN!!l)FWlKHG121IhV_>BR(e@%
zsIA}0-7{zBU=M^x#T}MYMfi6-xzMw7CR2muOQDk%+g~VKUKY8&W1@76<-$!*Ej=4M
z27P(e@jletsBW+MF6Vg`E!CS$CFQ$z?Wk`v_X+c|ddzuy$F!Op^V;wAqxIWhPmn{@
zeVx|IG9ag4vz1W;(I=g_C4C$tB%~)EQSv+fNqJl8X&$|K{6Zi(V(`k2g_bVs`i;L{
zDK=x7)3oFLmpeP!?D{Q=mI_++cD*<K%e!=^+{AY~ErZus-V`#l2y|E9QTKbQdEn}8
zJB)Pucix)RYw7f*(qf-}kmdP*?1Mi45)&pMHPSgtELUgT-qA3%+A7EW=+1S&=IyAs
zA-;<tb+7r)_lL}PF<!JRYLgf?{ssAYIADIYI;-^;wpiS|E4wSwUeW47&wk63+AFtz
zyEWCK;<ka+Z|5IY>#w)$9u9sT^~2B&f%yr~%w?rhwtZVPbLYNkN*1r5=k1hVyvO`S
zT7=~zz55nF@B5jrpZ=Wc;|xPL#0H1UPX#xce|3#E-4O1#L!h;GN08Y!<Hq}jmUW*s
zOm!F3?u_x^wCwHR8#eKFM(%~B^{<l_^Uvy<OX}UUm{>Q%;+U($PKT3G=B@wMTYfQH
zXZbaJ0oCilA3n#dzO{YdS>>X<OVCZjQpIbhIWNcMof7&6RtaAoTAn%9x$~;{!1Wh)
zBzPG=h>5TeV%lJS{`mJDa*2n`O3jw-csC)?`1D;)OC6P47B@1#44d)$!IArhzbhP<
znP~sg;8?LB-AqfWE=4@*uD9yTq#C);4-7SUGj8dedoo3Cy2o@SYKO$6Y(>inHXS>+
z1}R&v^Y+@&An|smf$lYnsMjAYH|w$OI^^PKS;ihR;`+6z3*TByKl$1sR?K<(m(@Eh
z&c66=_UBof*=G06=JVzIEEm7**nV@H)UG2(r7fRc;4ltbqrH=-IDE&yu&d@JmR6Q$
z8rE8HE==Cez5cir`_f0d6nEV;dve9YqB$XR+t0Uqjg~sRF;@)AHdT9b-%KNUho!)?
z^%jpFa}Zsgu6%gJ{AvCP6HiWtU2auDmbpp=o68<*najRX-Y#%_hN+s!a*M1!`vHr$
zRrVjv{ha%DNLF`SD1CprZE3UUt~yym%N;BGEK6K`cV6UKY56aD+IDKE=TY;phky?2
z90`z2JyQaB%~q6Yv)Jv%CNZyF^CUY1rzo-bbqiFaSQ0&c05WS758n{jbcDe?b>VA^
zqt=WTQQMxH-o7KV;~RsZd9e9DGv=;%izD{F7NK8?2L1SRoZMO~->Y)w;yX`nSGQ)f
zP)VL^A-k}7Cx5^U%N&Op+Z$Kj-&rZ!LUg(Nm3_bE<Ht=qwsT*#(7at~nY}!0$H|2!
zc6!}THaARdu{@y7vg_39Kjy@y48x!a@*uT6vtFz1>X^nH(IO(%aEC?0oj*(dy>z{*
z>z3KVOp<J(AT@&-ue2;N<7IR-{jg|(8T0J9hUd>RS!&%~Zc?t+yiMhg{LYfsbIqSO
zq?=u9Vcz+}K*s!K+NWJ>YS&vPb?-FKJg;Z<`_3fGpBHXg2>iZk)qJ;y==k8)(FBKI
z(62pGOE*vAyLe)O;H_l=Tuh?&BD+?`C|cSk2qzvE6td)Hw%jkQv2!+mwAF^hDLelC
zm|~H?WWmmNF{L|<+<seH%Gg;+{<%i9y?whRg)dz#Yuvn9Nfs0@I}Fo=d)9sDxvyX;
zWnR%C|CN0nOTv6>s+Xr_%!|zCTFo)fvb|=S9kke3wZ+s-Z>p>@%W+4e*z_2S*8X^t
z*ODto`yZf4p#VSZ5V!tm|7VBxgxsC)uFT$EUh~}YQ3%TpZDo5a-Q|C`OZ|Inx$a;J
z)yET;CNo&wSRuaS+sCilOF|}_*<N0@GxsXL#h*eos|SZl%oZQ}v$N~nT&kD5L7!in
z&)+A#D`NR(%T-?rcg()fWu9};V%IF0V2c~4gRO4A$gorudP#J;+P`3j#aFxOW_q``
zn5RtKxQ%&(oO$XK<85c1Ma{QtOx*E#{}S_E8GDH~kAkSANb6de7JkKOW!8<=A;KVo
z-c~Vkl!n}3yx5{A9xOEloWFNAJ_kt+3+x^C&+LG_ltt;`=pAW$ymwAM=w{BoYSoSz
zOG3;&1(sSa6Z>yICwa@Dk2hi?<4Dw2!<(l!Zd>@<Z0n-q|8&k>Tx>EmZ>Ir!*Cpex
zdv}>-nz`%832Zea*1S=5qaiR7LqP2QF^eN!OLxeNyxzfm_WX8>zFRxgX87)$745Pk
zFDKP%>4}mZ2cK`Fdc8&Mu+HYVWVP|1#jeHC6RmdCneH+@zIoS;Cjz^zBEDJ8`*GIl
zzZ&DNO^Z#4ZYNUPeWT_L%@9y4oMz$9;kj!jPuPx&;>z0t7B%nkKYV`2YX?EAq_06%
z*UB6%&n{R>^myIX4f8FIZl7T>W5*Kn9p6>ATRgmGs&`V?oPX8?^N+W7m?us6Xqk3V
zpX%u}>@utQ!F1^z{fEw&oj<N>{<l2cGO=3L?2SW^QHo@tCI3zi3-A5k2P~h9uuj!e
z<nmZ4>&>9+P|U4)bomC26UVkG=+3iLFEm=SE;4+NT3+cwouQeEFkQVaKtK{C`lR5R
zn(_G{W~0dtBD1t4#IHZUAr<_vRGC@Wj&Hh@E<Z@kD5iY~^v!3sRM+z|XPk0+XSqkj
z&bh1!=I`bDEJVe7Ew6_^u?X1eyED7eo9KK_dlzH3Vdkrc;JK!m6E3MH{7B(<I%+7L
z-X9|Q#IIlRO=*t$r8aGmNG<t6KQ4vcccY@jhJX!o!j2O^0(N@0%WhNeZZ`aPJJTX<
zPm=k8vNslI{&#J?vv#&c(|3n0w<lfW1DUz-Yzw%b99|#77JBk5%Xfx1Jn3tWg6AH*
z{_rr`p5-$6$~$wXzwFm7Ij^=Gh~`YPD9tn5emt9fvx43h%jn`ehUH1m2dsbDaw-VC
z&T^CXFLfPhKISuvCrWwEIUqfyxK=L1(MCPkUqf)C!x9mYUq|uq4FU6M?55L<87<{F
z-FKKa@ECu;aBW9!F_*>uDw7@J?C;F-mfba57PE)wasJ^OQlmZ_3LzkHdzICmRHGf&
zR&wv?|E{p>fa+rlYaRLRzYnTdF=yO1|9f-#&L>vARPQegg|MNf8yk8#Ei*TK-#)41
z;1;&!OHAbXW}C0g+HBI^JZ*cF(-d>7DMz>JKKem){r2Iij#XYL&#sW2Cw6{WduUhB
zzHK`q53k*^-Fuzo&brlBuV*Z_C_X-L`zoh<HCl$YIhdDTIJ}eLD~Dy*@%t9V-}Eid
ztvF#it>e5|SN!Ik%cfC(U;Ycp+uKA<indpJbekVG$TQm($-L`|r|>qeZU4=FRhC=)
z_+?<R#Pa;0Z)bNb7PRc&l(FNk2kWl+C%Jasn&i53bK0A2X@7aFl-?}0T>9_Zc46k}
zM5h;Ore%;>&!*qCXjTo`b$rua%dT(c+y50FwVXJ?WtUoa+s-!jn^s-NmRSnCYovO)
zQTcE0c7{y09n01{+8I!D!Lnh(&+T>$^S2dmU2Wkldd1?Y_eV=!HR9Jxf&4-YcAwN_
zHG8SXPQ^X#mJ7bBT6E4@U>VVGW`6T^`;P0TGM4w9PFXHLLHxNr#Mns-HHr>Q)_$D6
z7Jr^k+Lijs)N<VcMT@faYj+4r{;;|+Db?aa;$+Ll_Asi?U(&)a17T)GHY<2uzl3#~
zQinMc^S!O2qE268h3<MBlD(&RT=CRHXFj9Ay3ui!fry+@J4b8??Avw)Jl-i9$Pdn+
zl3AHT>-&OvB_3^-{G?PS|IjyqeZorPL7y)R9BqY8N7oCUFuKkAc<v>^=ep^F_57(q
z`EOPVJZ4zRU-ogoK)>h#p*eY{OkyvbFnKAl)ktlfrtTa2y(W7D<qY$+<V?>xnVFp4
zx!35v*G{7?WemI%?ACKjA64fzw_@WF(l_Hyle@vaS)7;K*U*i7%6E3|puMZOT2dwm
zZ{gDwd?>U?xGPgh<o~@)p<9>LMXJ<Ag_nKaCgkk7LAbwWvXHD#rtX!!M~vS8DAwUB
zTB1?A($C20x{zK>>Jj~?CDukk+$GwRG<WFFFgwY8*3gK{YX4iVXQ3-O>vcRiJOA@?
z7@pb5x%(~~SMX1Fu82-ot{u0f1S6U!@z(_S^L5X25Nt2y6I`)<v7po|Pk~6|Z2n*K
zqXeRId4;CDn60pAE~kRoK@%nA_EN>zjbG(AOw3X0yy~y;wbwwQ=BJ=ysr_k%^BdE7
z^-oOX-MING-_je!-1Boz@o?(T=iX7aiueDK8GLDZCVV1?nt5dA8=KGdwKU!-`qk{`
zrekJp*Q8DJel9m}Xkalv8|P<IC@5lHc>cMm*Hi^LDT8xTMGCT#5@szDnz2o?`C=>*
zkJ&!Tw)IBKyy2W7oh{KO!`a3z`fU0$QDr_SVM(zif*WNuME}^&7t~qmBW&EPE_!BK
zu+aCfjUtWaA53)$O-)mSmzoJp+h?w{uiZ44Ws&)#fcd5Z9!_T8RxC7+JGI8FY{p{F
zGieXluFhs>w_C5k&RMgaZBa=KyMUE2dvSU<TVtaF`|nNL*v^X_Gz&D&Fk5(dgK4$x
z2Gf~*ugy|@1<aghFE*R<sMqY4;&IdCkFJ`|I=WbC{gejPWnF7E8qS^5Sme4*+4T5i
zjRXm6rQ_L=8V_?Ws!PkiQ59fYz_<7cBcFTtEZzXKr@U-i`gm49UBG*Lehu$Wy?H#3
z&$sff56$NFu9Xt()vgw37QM}V{?jypgAc_8dL-2ZpE!H-ugfst+x4P~|D=4k;La24
zgr+jQ7c{Gx#2@Y*DCFCCSJ2Vsw9u|!@`Bu5VFE#Qrh;8f+QQzGzne&2`DVJBkI^*#
zikfNE?OkRm*+wQ6%Z{29)jOF*Xw5ObqJPkApJA%-=bKF;5eEW<c^{byOR=em@CA1Z
ze-#T9dGW_e<g{9^Fz=o&;a8{r3n?8?=Fd;LAoTeum+(}M0KvNx_6liA{}teP*(Vqx
zw_Iq=<l}<G7MR~>74w*7dGX{=D&}F8+s*ww_X>BjTQW!UR#DEkX)4@IQpw!Y)`xQ*
z4-FL7`mtK1JH%gT5l5_$VxG20RQ@TU6E~&^r>k!jzWQXl(6WpBggg1OMAlj{3I417
zAu?-MlIRDv`+}e5wTPbX)fS%XVkF$`w@@UEPf9q`OWxe(J%ef7q{F5~Mu~bW0*;tH
zc;jYPd#}$t;Do>N?&+~+f(1*=>edu-mrdBnv!vdKdtTZ!?yOhcTu+>LuuE9%<=XI9
zlbh4cntMT{68F<WHrf08gXGxx=F4pk%aC8Lcu3aw?pb-YngF@`8v62T{-$z1o9@be
zJ?<zYwriH0wwt$1nT&$8r@<XLaRXtQ-A2r^{&q&PtO-3bQ;%+u)wwiBZufIlc~w>^
zg%(9Yg;IWVh2n%Zg+oc7<UUL?QMhz$t%Bl9LB-gOo+6J7E5yQfHi~{~))x)^!zkgV
zD<JweUQbkKQInYBUTraF{+;5W_;BCZ%L<}Fc;g)*77(3&UYMy+Se+TfzLWKYSyWz}
z=b^PUuU14q56@?N-k$IKc%%)xxn>qfa_^kQ$K%GY##5fk%e!jwbaUZ%ou&qdc+8l?
zo|;UE=`l@RyTo+r`$Y4s2})*B8E4EqUM@BJ^6?@+i@p)R^_{)E3$6d~--r(9KWTlH
zf3kZDZ^M%oz9}YO_-fPK1>T;TEx_ilBhbw2$fI)hG5_{Ik^C#ldIZ)=&EQM6>E#Jq
z-NxtC|5?B&{wD9_FUz_8v?RG_ryk;y*d)fMSNN4@Yt{tLO7+h?ZGk6wGp1kXWhr<r
ztjDrSAYgyBu&n(Yk)w*OLX{rMqMkwO!cNl*h01P9h-9Zv6wWyHhuiA>T;9bE8+bNI
z&f)Vg<m8i#oXd0WxED9iiV)r^#~HjXJ#u`7hZhQltYQ&-$CbsaDyl0eD0NamGH<tF
z&^i@?d3>z=ceGUn=C4%~y582xlXf;wpjdwgf1SWiKJEp7`4y`^@tgdc#Jlr*2;UO%
zJ$y0SIKgB0pfD$dSETU>)KBT-_b@c%5n+@Q(CzBvf3WES|CHHNdBbF!`1(Ul_-|@O
z2(*?hVC&<H;K-Y&!8YaR9M&H~ejMlT+~mBJBEdGtMuS6wdoSyW>@*HS_QB-2na#~w
ztZGbuX!n|JuZ=T%^S8+~xZKwK_B&m(fBULUmv50W<JiC0^xVC3yzh5^=L&L}#;te#
z0RM+MO8n;v&hmcw>cDL#md$5&go!`5l9&Iou%CdgNsPeRjpF>4f)WDBQ>ODfXny0J
zadSO?$w6uURKAORc`_IHt$ETUWwJT-ZhqM(n>XvS#Ojh-Wu59B(s9OKa;e#zdU;cL
zrB8Z#Df9oV6h5=_yiiS$k+AxQOySc@9|`>v$`Stee53HIyRyP!do+Y&Hcb)wcmJsQ
zOmAJY8ku9}#W`GN%vrkT&7b<r?}r&$oRpnoGE?lJdC}feW_BELTsdr$c)!09=54J~
z;TPLIfiEjko!53>9M39lNxrwq4tx)fY~YJvZ0Gv@Fqi9k!*8yJH@UozUf$;kvf9b@
z<7+F&e3ek1`lAzgBA2oAzV*|Pk^H?!dXoAIh1!ez6c|jIrT5e`Dn6QaSXwr*RX(CR
zMxH}^x2%wvu)5nDZPkgl7pXcXIx0Rd;nmjf*ssL#(nWRE1r?3y8pl=GPH}3_G7jdf
zEVgEe(hy>gyncY~-IQ}IC(T!}t@lf1Yse~RJ)z;m#xFIAwd$(9PT%7?ouJEkT5K6#
zw89dW>ExZ@(0QOTO@~L_S7%?ikq+0d799<N%O<Pjo|zgndYNrh*lg<a=DF!v!wyrO
z``o6Q1=~#iJy0^0D|%#_WqN=&Ginddj^-;o<<iIa@BXsqU+SR9w`tQ&uBJ^feCOwU
z;meBV<KO@DlR&F*fWW26kGR|O%mq|lE#yD^=ZQd)cRSxB;n}>0t>pQyd*}%Mb$ce%
zaLQUl>Y$)7`@Pdb>vLxdGaURRH2Wd9$cFk|!bztO3mIyC61oszCCvD~LbO`mQ8<PD
ziqO70i$(NqO%*<{Z<fgaP1Pcdd-8=#A3PSh<R`{6_0bpJ$1LZ$*Pn6ZQCEM?!!UgY
zPsRT@?%rZnE^oD&JZ-9lT(ERah%TwfWZ7@Dn|=M%YL=<yEiAK{HnL0;+{e-%eT=1*
zV<L<6k6sp#nsXK6Li@}M1grJRc~705Ca6ANPVh^gijdNYD1p<KZv5u|n+1e^b_qS*
z^im*Z>3sfXk14!T^-lzjpIa>OJ~LSGhNP50l|LJQcm=nB(Y`~1`_8cOT{K(FlNA=s
z(=WZBZ|2eMe66p4^OYVF;S;`~$IJPMlTYcRKYy1Ro6x;56+vUS*?j)}#|00`rwLw>
zFcz98xLqJ#{4D>q-x~zhwagP@Y{?N6d(tV;&?&&n)1xhTl<Bm9T)=$6lq5-k`IX%K
zcVkrqE*SC&Wqz5+U7|XNyX5*ejz>2=d9<TcdFln;aesRi%-Mc(6W2AT-<;DwD|2tj
z$rDXf{wBKWl)Xe;f28PijT7Sj+Gm89FisR(Qs^Y19?K@Wb8C-8rT{;ih3#C<OgnM5
zCe>iJ9<RM@vOAZsB`z*v)4boz#=6Cb&Gyh)wh*Ts#)i$a^&hJin9Yz>Hp};#Z8SII
zp6TDNSq3q0!%g>JZ8g2LY`w|H^8Y;Cd9!$>R>$*d)XwIsJ@=PaE8l~sh3PbRzUNV1
zZN6!|`^*dYs(v2iNzxGHJ+n%d^ZR^B?gcu0oGz)KxKq!3;MsPUiMz>6n1{pOmY2<T
zzIp%dP_wr`n#{lH95H?E$ZNi9MyUCT?z85F7XQo)Zpxdv#s-_+`4B6p9&tgybIoGj
zwI8ksEL`m)kXaomcr!DO|I4xfzPD2j@asD55=>n1l7D{OdA@J^A92r}x{+TrB8va5
zp_xGZg*|+sZdZ9X#XREs`*DH5`mkvnm)MecwKimPujsPmoRRLzzWVz<E@i$m%piWT
zTp(Ao(|m68RxQ3mA8vC?#xU}nwm-(d^^pv}cIY17riJ@Bww#IQyJYZzZ}t&E{;6KA
zCjZQO&5ByI&0^bDnf~#RF;y%uFyorxVYW6g!&FCZw&|}6jV7JT?0GgndB_u{7sA7Q
za3gnyzBhN-W^0bF7FDhuH%0E_I(b}cV+^?5x3Tj2=~nYR@ZHCCU-~1DLs>dcmfu3|
z*jKgO2R%7B9=~AZIm4UJy^?(@uN2cA-V|qMUg4k1c(-5C<GVj?8n57-$2@n#t$B65
zV|imfnDK3&a$7*0a~}VbZ6|oAY`-E9y=Iz#*CjteP4{p7vOACSovr%B?-{a9Ffr^j
zuXW8N?$no++=o}a;A{C3&nG%Lm~WziE^qq3P~P4;CO*3lH~DmG&I$)SG8N>M%MdPK
zkRmeCkWu*CO-9l5y%U75KHn?!M9Es@mB|@lpBiQM!>(Oixv$mO-x#>FGhXjyuj21u
z@3swN-_8=k-fyDAen({%`v#VLo)(V`?*C_wa=%@|%k^90D%Xuez3gu-F0p?VW93%d
zF37$6moxX{7<04jNs~?I&+ah4(7|Xi^SZuSQ}1u{stqz`+x|{8bJ}ypJa_pWGoSPE
zLh*Wc1qC%@`5Jom3%<FiFDMt~FI4CfFK~KYJ^$g{RRR+?uM?W$l_aZjCtOzYuYoMD
zRgHq=lU~_Z9DK67`i@95p4U>SIk!{p)&mB4yDcxoEMLTk6xcF|3+?t6kDR6?##v@7
zZm;1d*0HWj?8~v;Voq}DV(~$1d7iNG@Jth3%Cn+1lmD61bv}!BPu^L2m$}%#@bjHD
zZR1Oe+{eeq6vkJzk&Ac6SrhK*ht&92&D_XWq*}?VyM8|R-S9ZxG+!S+Q&l^@t$XZv
zYmXk{3Hw{gTTuIj|7=Vcf4(jwU**jy+`ic>`QAR|=byGOiJ$fEE}mx}<ar-ju=Di&
zUBde)=P%E_$)4QtJ6L%9AI9^Zy28$@a=)CH)$Ai*p<}PW^F&pC$+>#`&udl+99;aC
zk2!FMK-A$@{-9NA{B>8m_$yu?;&q=@&uhir%#*2foM*DrBp#Eh$=uoE!Q37b40xQy
z4sad)`GjLmmWssZnSUit|NA4=!7y2>&)PzA`_w5?%=bAYqKtP+z4@_R!ui!+$t^yd
zqU&tfg>AzU#e_w-iH7kPi~LQPA{rJpPc*O1M1=9uCDA`cyG1|m`_3=)$dvz$M*v@y
zZ#Vym%qzSX+)wgtb-Bdb!rsHHvsIsO@$wmb(`&i-+gD!UD>ppGQxccWf5YJzUq#Fc
zK9($B9<d5_zBdisd`F(d5IqjE_TV0YgG})Poe9-Eb0#wgcy*TWpEj8%@V#LnUvmW)
z&*HzU_)7AH1Q$hj3qE~aA>g}p4R7wFIKgc`B7)HzcLep9CkS{Lx$uYk<OrPNv<L5t
zAXK-3<R3Z~a(2F+%l>5c3QnFmqFg)z-`UsH$#KeF{>LHlOo1aZ)tGBtbsLB7>p+3|
zw<hxK{WX~{`no&+79lZy0q0PDwi8x-kDg59Kcjz<|L%!UKKA#EcnTARxNK%k<$SL<
ziS5a4DejvK{&DhoTXHs?ozC^adkUxeG7j!k^DBHBOZz!5)@$)cE>0Hc{9VZNE^HnD
z_4{Y}u7xb(PLFFANL^^jcX5fSki0>(;I;j;c<&s3AXwbgA$Wgxv5<}OA%WZXH}j{n
z-V(^<JR=llZzizGDOW)3Qa7KT*j0Xp#%%uo!SehWM~e9ao^RnhvS=l5hajhbez6H#
z`o0=YZATlnpQ$cvdfG{B><^;B?7&=owoGPowkw~r*;v-THZ2cgG;94o!!+J(s`00;
zCUdS8HKu~$d}ebm=9sPQQ#QN5qr+V48K3wX#cGj^RXfD*ojfQ$mrF;iu5+3=7mKIZ
zZOLF!kthN2Yb=c7if=ZH99R=5<X7(_dUPLy*n`R#VW~tZF}4LZB0d{m2<tF0h^|x-
z5NYOLB3Qw?O<<{SG;g(MieUR$Ucnn2Zw1$ICkUL~row-qDqeuO*;i<iJ1e)_lNcVc
z?^8Kf3LN2HzrUVas%!#>&5lOy`YY*NE5kk6<)(CUt39+c{nc-2l4PA?X2^EPjO!<t
z*+I@PW}6;-H2wag)KtE;-+Z;gce9?@Ccb@)%lK|NzT-8XD$Td<+aw;@?an-hO0V-d
zv~cpBIIqTgb=OJ0%*Hm!*;lzFwuffO?M?8N<0%!9a7+=Biz}Tc`Qr3u>E`nvWL9r5
zklq@)kDq;-9Jg%xI&L#vd;XJ-c6{sSF6MRE#m;|arvQKR3KxMbOyc~TOdbmyVVfy%
z^3P}f^cF*bAjKK{I$1CI(@qESovo4QXF6giu+?`Tzo7LY(+>M7rk{nDn)Zb*GhTAu
z#audYuSsR?M6(OKHk)29zG%vD__*13KXIX3%^89|Ez<nYTVD$DY^o6KTxuz_E9R9z
z&Qcu#evVxN3lA?B%9Rb|ZHOx6uAQ-&dnvmFzp`xxpN2F8uW`v*_E4^=ysRmbe5Uf1
zeA~133M9?85r{i+lV7XFQJ_*Up8szyx4^CB^?V%ma{Sjy`2|eG4)bg8IxVQW=Cr`U
zORRjWTjmQkfAkW(%O@qIcd%bTx@R7L%({gFj{};7_GL0kMEvKHT(vw;-0jtL;jc~R
zlE-Ji65FOCAQ`u~RlI(hqj(S71##UOj=a?pth{T=S8{iqGT`+JzrwS1?@6AHyEC~D
z$SmQ0I5&r9Uu6Zamf9SFOLJfID+XNU&f8%wplZ2`-!EvffMf*=zZK(Z-ft_k`Ag~r
z1-Hbz2t8OfRnR({QQ-Wq3xdXcL4v-T9zxPL*9!1BeB$3GvroW;b%oG=Qw5<vKYs|a
zGqMYn`s@~*FZ^6kcjpAbgKMmXejH&HO4H314886k_;9`&@0LY;JO<)Yy#6-D{A}+(
z@p-)3#@k+_$kjhliEsYz8GP%m+~M>7na;mUl%GF3Ka%^kiy`07552sSR&xBh{L^^!
z7l?B@8s6tAcoE5e<mEIT_8)tBj{0ureX-{Xf32-4e@71kU$CDJSA9z-U!BGszSCmS
z{N-P6@L$fo%Re()hHt<48$MnGK7OBAdtT0N9v+*IuDlCA#qjAGI`JJkT`M=?NsctD
z=|mZ`y*ARaOvhz@u4j-w{Ki&p*MBwXSAo-ITNUQWo_f_Q<o*7<VB>rye#L!eLa+Nz
z2-@ka72=On6=Y8i70|pTA$WbCsBoQ?uacR^J`K|^Gc<HwycIv#*01i;Z=)eQIZIP%
zvXzFP&;j{ZzmKYje+gzgYc`v+u(O!WO1YWsx>pIC*qwg1-S_I*xb}L2#b$OEv#syj
zZ1&=4pMk>8vu0}<f=y+%x|%+@xZ3nh-BYta9-PMB52DPkZssuSUes(_rdnkBt9X{_
z&lmxdqj#*$4wWRDUe&*6nqPm{R8HB$bi40LGbi5$c1yt-T>s3KxjUA0v%K9Bz~lAp
zD~HxCRyGjb^iY&FJK+V(dL<Si?|2cx?BYedf*TJA{@xZPxa+HdP^<PtfemYx@C#^e
z5MWoICe*$<pZDVNMBZr|OL%^6n8^FpXBN-jr5+s1z3h10{A+l*6*+ht_iW&e>|e<f
z^JF3KtEne=S(hB<d$OL7uXFzqo}HEUTvHXl@-i{6<rN5=$v5TXa!%=kDx766wYetA
zp5fpVFy?gdci_76X*Q>}##=U?rgoN(xs9BDQcmVYGftT{uq`pqo~>%0^ex})0+WJ8
zz^rg{%lB{0ICtzd57TWkKPXWp80?iGkpG^MuOYfrP<^V9;Qyoh1Z@xb3M^2b!Qbbc
zAi%m%U5LADF4wK5**sJHCh;h8`|@qszm|8s)D@mgizghnH?r||?|0^1xoICSqr406
zo{4G#t2l1*8kM^6wTMOVPxlt%zxt?*&w~9eUtkswUx`!!-_uWdyh|^wWarNm<Xl|D
z%>ULqoG;&qhxf=L4Q_GEPTuboa(us|)cAPMSP3MoJHu^w^dxT}do};ftB?36o9^ek
z)^wO(MdUNTU3$2{sVY0Z%2zLC3{P~)KCM_N&(9{T&@34tXE3ozUQdxx#^U=mdF_Ww
z<@(gh<WKn}h#Ai@73NfI5li@cOY9U&qv)iahr~FpoD$u2<-6!ySwpey#~zD{<_QSR
z_Gu8*j#lPVRhcU|NmftrtGbGi*{ym3cF%17UYj0)6Kf_2*?IgIKk$F282>|S$!NB>
z5-vv+B_3O5$+$n>E1@53CJ_~BDD&p;O^MfBOx&u93%NywE_1p|2k@AmsprmoFUxhw
zwx4Ui{w9u9CoXfA$;$Cuo6lsv`r$M)jWs3arHbX|KFd7J!%UmZgtU81|AwqE|2XxW
z`Ri&uqR;25y3ophW7c#&+ega<{>z96IBaj>=c)4Ji@MUnzvZJpe~~MTU}RGrzoxpm
zw8G(C5)IrfGU>BRr3}&=rK;Kcq!#aFmi?<0FX=DlC*%0kRr;V$8{e~RU7muJ9Xzo)
zssfDHI{0TP8SqK(5aMp<S<M&o&Wm4Cw3dHXs0{yE&wT!R*DSux!Uy=ahg<S#uU*a4
zk!8le*~W?Q*CZL<6YUE8{(s)_N<O&Aqkg2F_rWP4{-+bu_=J;J@Xo#Zgr_)uIqzC4
z5xyAvNqndMckq~;?cjZIiIdN6YNP<O`DXqFQ(gIvM_Td}%`)S^6J5ifwfGLv?Zh|L
zg1oz=&vLvqz0NUf#zH=|>1=%0PxSNnD(7>3c<_Md{^{+!$x<tMU&v44vRW9*RklN(
zYwOILTvwZya^0<F<TxL`l2dKH5tohGCa!}e6S+WbKx$%zwZeRl8ZPl~bI;)Y<$Ryl
zX?6_noEukoPEE7rd$FB`*Ldbd-Veh3d_iu@74y!jt8OUOR+Q1QQVgGXSH-QOMImHb
zo}%RaXypTc7?mdQuTXTEvX#%LM~gT7yCql4|1JE!7P9>6YyR+tyA^QV;@Hpo`O<Md
zxvK~G=5M*f!_E}U+mgAB*L9AVz-<3+{sbN^z8x|R+`RK2@#XM4@;5%6#6M5h$i!vY
zJJYQEZ)Q`)e9aoNeavJ+CYqWDTr#m)^27Ai3_sJG>&#5wKFAOh&2JOfIy;vq-dS9*
zZ_!bKU;7se3ML8)-1xYTuileaVBZ=}p$(fd1%<n61P-(<;hiwWR<LNrJpr-9O9jKy
zR0a4Z`SPd9m<seq^9yZ=7M0p~vO)UvqgyglkDnL5ucI#a+w-#0{Tut09Mqj94t+Zz
z@nXk8u?IbO1dJ+s_~pBf^YiUHE>KvJC9wBZw4m&PfBd#z-1rqL*9fd$*d{nzqg!Bw
zM<xGulL|g#-|4(=zk~$RZ*=ihr0ei4Oy=W#{C5)XK{HuCJ7HJB|0y$s*Cw$G$gJHf
z@JVgEkfQZ=en#(Wf@?lJ5M<N)BfzHBCKxUGOL?;oqg-^^dxd~(JB6=TjN~Td{F6W1
z^<8<2^ga3QENb$mGZGcFxDsX6nx-jqAKfK?@bOy3AHp-`Lws!%H65qO1z4~uPMA1D
zA*1S^g24|*rKP9%6gqXq<v*(E$Y^jdC~3>($zKoGP!fIeQT}K3MumS0JW8!W{c4Z1
zRMd;56*XR|X{pQ97^oarzf0Zm8Hd`_?rUn_wSK66JeH`s?y#s)Pkf)j)|!)M%ewf?
z7Y5lH?HAZ##;P{gDBzQ&i8-sO*%Yy6lU7$&p^Hxq1?QAD@y2k^6lB<~FDNUjD&*Ri
zD9|<4lK;G4w}3%@hfwJ+ezpjf4z53EYT0gZUSjLJz|Wz*qLw}RDL?zEiTBx8nIB~{
z(URl{XqqB*W+JPY>-28PtJmcub|pWSP+!R@snNPxs?y-57(;f4RG8Wp$;UApWV>%F
z$cSJ5A*T_MBBz<!AtOC;p<H-cl&qsRuk2HeGqPMSOl2<=2k_MRoa1rYuFYFvu#TVW
z6Dz+!T0d{d{#o4rR!ro36?uWL=ZX%0rqf(A7IzQR*?r<B@;7gqsy1_(p1=0eL?tlY
z#GOUej9-k=ynNL)vtMckEawlZvn2DxvxDe)R(dS`({))40*zSecj~j~RamoZUTVpb
z(IYSX&1R-Rygsw=`SgBaj*<m}Ys`9u-8ou?CaUNQJ+0Lh)@0luXxo--er>C%*%MVJ
zi}fZ)%qE|oX?A~6yLrx69`mOqM&?3XjAs9?UNDonl_T`CF-FjBb_?G_cRL}ryibB3
zM0N>9uQC^$^VC$}r-`-TZfynO%TrDHG{kjy&p6xjwl>e^*~hYsC-AujZ=>!-?i+!c
zJlW6Z@tm_%<&oacqV;D*i^dHFah(N{COYp{y;NhYUal#pudbS-;;QxQRlLsE*vlGj
zZVL?5^xv5rm3v^Sm^|C;M$Q70!dpgW*DvrH3hs+A%R8=ZDrUxOW~nq&<Zwli*!;H#
zgidMj2<<8_6ZJi4FJyeyK{Q~#t{98&EYZO4aiV7}9`VKQ;N{))c^7v8n<T%U+i$*e
zV(t7(l_U5HCRp-LsE^{?dgKp3@8ZdV4T0+gt~6)zs;spW%)I+hAhG$FU{|-2KrDj{
zzXHF5!0SI6LNcKlyd6RtxbGco;{A2#9Dnf#J$^2YN4zgCPvSDrn9R3e{R2K0Jy)X3
zdt%Gx5Rq%VE7HGkvwIct1Q>khSG(uIzhloQ-Y%xi+)Gc-;}h5=$8Tw3&%bYGo7g17
zpF$_^UXy%$L|*)<JEP>>b3US5>awJ)MA)R#9tKJLP7jyzk2xV&Bz|6icSk#~;+tN<
zgf|X?(_Ey4o|{Y*XlV83zj$z>08@9fkpG1OLCL0eKH19k{N_tn3N*E^=X=^B$;Y51
z%x`&0j>peDM_|EzA%3xMoI?8yQw2R<G79|)Js>#ivbCU`FQ?GgRpx^C*7*tM-G40D
z{#ium537*G&c-dG!e1Urc$Ke~Q1NFIcQJS?v9@cE_>HJ@;>C|*C1&mZBi_Vs%V&4W
znm4n-i>LEnERSZ1GS7x?Cmx@4PM&_d*WAk=d-BLHjN^9O)6BEGeL9Eu^a(r`zkcz$
z@i+5Y3z+bfly`7nzg5nCUwJam#dtqniMtc@?H2Daii_(p6fJnAr<oUHxFh784#Oo|
z{cp3T8BJo_sGlY~f$HZF<jGr@RpnnbU2nS7?1jb>liVU!^M*^ZX0;bqni)*&F{>0x
zGgTLTXr^&9L&&`|L3o?~O~I<VTY^2U7Q(`}rwXP?2MM1oT`KhBrJ&Hgw-*FgPMs{6
zw^~NTRV`M?@cJjgWq;#^ukYI@l=YoUC~^CIVYiS(p_KDJLVNzr5trkbDK<g%hS-9d
zCeercuZnj?-V(iWVS>c%nou$SiVTTELIL7=s{F#5AMJ%6%}C(?dw;XgxwSSzAD5~N
z_k^_z{=L6hKz-t5!Q*^=!m|p!cw!4K@s>yi@H8!N;I&P=#``6HHm|*TIPbI<S9ns-
z81dS&i}6XVI3(~XJVW3p*AiZ{+O7O8OK<VHsWb}2FAn1^TiDCHPP~!t|Lc7M=3iP&
zx2a4tyB-=~BEQ_yq<3YQ+0^+l#*<#HG@V$&V`db2*YvKNw^>N!A~W5eLWXxvHk-??
zsy45k|K23As=@4I=zmiWMnBU%M>s4VL{2telXuw2;emk}XF0Q}o<^zBDvqfp1!6oV
zRYl#VWldeCe%|sX(gnPx;pg6SM@KB?`^vI~`?#$a&&+#QxQ>e~<ko&u#Pj`RKi8RK
zi+RG{@^MRT<u%zBKijbL%`(%~bB#>vAN?>c+W67%hg7$5<Fk!M+n$)38sAGXVIUIt
z?p`I_AU{Lo!SsJZcZyww%*$j%H|$pv7T}N)ahY2na_?)1@a5f&A`4<YRMor1R1WyP
zRk~5pCwo!iu4)F?6S*(%=BU2SVo{!ddYbB`$y=3=UGg{EAQWe|jM>?=>FX?$pdX9P
z?ZbLZub*Qx|1fo)nZ@PxX79zE&Aop13SQnaQ$T#aBHyHxc)`s=9|Z!0_Y0b;sR`W9
zoXlVM&R?LRTTw_swT)}n>xEp06J)ra`}*@NpUuvr^|XakKqH&G`1(ig+Y5tvl5Kjq
zy?ee03FK)BO}Sdf=Rdzos9}ky(5^51!U?Q_f+D`j0vnI#32r)CBy7zxi&yLIe4h9!
z7VdQ8&)m=Uf9J|8{KjRzc{aD`vqxNK?o8)CrJBfX|9BS97a?2T9}D`p{TE!|>D(g1
zlb9vMvq8Oq`>%m4kJzb39)*jmxdJwovqb+r%+lrG$#U}EOqQo*i&<=I_pxltxWlsf
zW;+YRnPco>8>3i+IkN;Vub&{GE3M7XfAS>%r;Q)^FCAFK`(14{|A)FKeCL#I^R00E
zNp$&a;V_9e>a#9yx$jKw8;_XzrnmX?GF}Sex!WJY6SjQ^_v-1fyj7A{d88hu%eSw8
zti0xtv7CfjreXlMsPgvXmy}c1wJLV!+9}1b`pVBTVNz~A_nI&5&{Q4`?c+R$Gq>`u
zZHeJOXx+ts?P@Ha=vz&`KdvA7w2tx##OjyvM9J&(ut(PNNEhhvYhPZ?_x-5>@7rC=
zxo7Kq<ejo7m5*`#EWX{Lje`Ek(*@d|P2x3hFcSPwa#G;JrImsz1-t^MH|X-8WRntj
zdW~Nw>-9OFH%bq<W~Z|9NN9+2%O~yPzG^#}`+Gw>XVz+YZtKk=JSW|CxwT^XM7-Wc
z3GPx@DdLgGE*d<OONiS{Q1sDD8R2yamxV4|GZfigX(H@DuUV{r|6bwl1A9e9Uu+hg
z6@5<R^g#i!Y7S{JJC;h(SB(Ee?<CfXHY=HngfEg37U0qsnW>s9V*jQ=*y(4LNLrkk
zNH6OFVTXJt5%uee!iybvgoMqV1n1t?=KC7BOYm25nBa<510l(bg#!Hxm+`Z?FBVYl
zo+ET(J|B0~tak3F%x>I!cdg>wy=Ma#iv|z(VWDH3`9<d38inFKOEjjirZWT!oqu~%
z(36{6NSf)r;My&%f)1Qh1Vv(`g*Gdm60CA$7V=tYEU3r5ojWB;gzL_XC@vK%A<lzx
zH#wK~?q$yw?dOy{E5+4mrp8sLsKT}L=U(9xTR8<oPA7?M4Qdp9ZJjQdZPY3Hea|1E
ztegs=dUb9Qg~L;Y9tP|2?Ws)T$v+gu)oC(?f5SBvex~(Xd6|9(aH*d7!^`pVFkkM(
zlYAQrT#d^Qs+eu^es8R7_|s(JeIe6RA_^vRa&8$PvN1I)+MZ`(kayLTNo+gM@*i5f
zcfX$E{bsHzP`0C;U-YUf-^0b)+{|~s@_CzC^5^ib<hOA7C!#W6TFhpNoXGa`TZAk2
za*F9_UK4J7d02EWd$XvZz&(+zXWc}<>YZX|pFfFB;b0G&IqM4cuovlUJUwgJ7VVnK
z)-m6dt$}+PdzVHGTgggh$!v*OiMV|#(i`p1Nv2BFh<i6ZlG<LjQR3%nDarq<y(A}}
znknhwX=0wUZLUdjrK8ECnr`E(IccU{2QHXcU*a=Qcs<$V#cDOPQ&k&HWnSLmK62<g
zPpj-1Zljy$dGe2+;rYwK#xvLE8fUt8Fn2)R74DT@r93K1tY$)je~kOTJDaI?UN>8l
zHPuvQ>lU-aoS~*#FB;7z)Za2IdhKl1Z?a4%?2ok2d-F$pyhl`pf+x)sOb=Wj)M0W#
zK>LK6K=lJw!R<!u!nP|eD{enwul1u#MR7_jtGLfEO%-JUM)@muU9>h{Xj065?<)6&
z{lEPA6U%r5+|ziQ7A@q7km}@}wPimK&m1<Mhr9o>&N=46Es^_@r~h*nkIVhp(vfxM
zlBb`@%0B(SKt{a4NqV2jH7SFvm(tBgTcnIEuE{8RTFa;^>+@87vE%X4)8yf-UdB89
zDmQN!hYRoRBd@tNd0udzWn<wvD=o;oV*LX?>Gx-N7kvLO5P!jjPkfCW|5O(p{$+n7
z`PQ9M5|9*H!e6mnlE<on-IV*(FXJ^C%*H&`{|x5GA2RLQw#9hgGJey;e^;Aq+|*`r
z>HGwfxTlO}KCd)PE9Gn~UYT227%VS0ElXuEU!S$XRMy_ljGyP3dC($P^W*Q{@NCp-
z;d0-(fqPF+4Bzv;`+3hf&gGF7Xy-m{Yr&IfXTV#aJ%#tw-bjsw-aFOJB{kH(*xRXI
zo_18d>enXKMe?)M?=JhME+lhC&Av%j{Z{uJ)2Oe<%vk=4nlIbVYaZ|3WS)O&otcP=
zx5;*+r{)<Cb<IDf@mZ{#ogl<J<CtLTtz14vq1%Gn_rwY^u1gns*1Arh-*p@R#x;8d
z)E^%by029rcTs7+;=Rf2^2U`O(uonP6q}bj%ZdN7liQ}_r&y(OSSB~lPtkPiAHh8F
z?ZT#yKM79S$|9uY%_KZ2mr01hCQ|s_yAMKwlMV}RIv^$FT+<+Ua{DZS`~|Ii!BR1T
z-(P+aXp3DTc+@~b;Gu0NzvCo#0gJ;TLRK~pOwQ(NnYyfgZv1L9s|j=HHDigp8iqEi
zUyNKAZZn+~sBGf7BGfcha06d&$wodWe?Ok-KDYQb8(Q*7JALJC`YFZT+nUZBDe#?l
z`nhhtr}n=&g(RNv`ub_}wVZy%-@@X~?{}4%?@ireF1xQ=__jJ-=L?zJLUetUWWvv@
zv|%CNiTVZHFDEVLah@=bd%}Y!oG~XldGFYnbM#)4=FKcK;$`|2EV<{1h(z{=CF186
zaEmd{m@n~Bs7kC^q*2mjUb~p?>pc>`e(^|zERo|+`Mj6^ykQc5G-DW_LNOz+ym}R1
zU&u$^K*2-24&KxG`@it=^#yF=w)h{zS-+x+^Tg6~9K7DfoD<q6a{79=bIx|+<UBB8
z3MZ$w0cScNi;%6Qvf$V0ulSl>_6at1dkV&vD+#r+&JtK{_nN=9X^{ZKiXNeg6W`32
zrr4RZcHTEz+jY(?sX)O@@R_lB@x>1F$sBu3AMZ1_=(n0_t|#9NS_>q2lf9PjiGY?+
z@Q&SrCJIZ0W*M*vev;7?s9q{87<ZFfSlPyve+TP+zCS9dyhk-n1TwCy;otMxlfT%L
zhc{93CZCXXGQY_#R)ME0g3af;$(w#u?J|qr-fVjKce7cqO`q9qfnM_$k2sBH#4k3#
zIhDigxdWfzmg&<3$}0_d9!$#==-~Lv&k`prcq__<pZnZK-Y0VY{C9sf39_zx#eML?
z7M_(KmT*me8pgfMN|8Is=q#s%-F(h6&n=wGUw+`6;D3y(xNowFZo&_<ZR_uu$@eFj
z1uxDu`}6*hnR8Bx>DAUWbG}MN(~Yy4%{1b^ab0p|;hbnK%6a9^EzT(uu5lE<cH`_5
zRpjjZyM#mOg8^sxr%N1ssV1fdte(aOT%D%ZZc3RfU-`gz`Be_n<6$}`M!OnK*Bf%0
zgnGO+aeEyizCf5!^vfnu@vk*967k+QL>D?Xh+7pfi+l6Pi`_Y<BW_@5A@THFpit+-
zc){1XKY6Dgk`~%iy<Je~-vpro*53mB8~6nF7zqmQ`}l?EdFSS7yLcZZbMvkj3gEf_
zg^{Okt~s}()>f`}g;zPP`}cCM(`@JL-1(Z@jagk?@l2F@Uzd}b`R{H;R;5sN_rP|w
zYr#qyYg{L*{1INFwj<+%>iwJf>=UZ^8B<E;vxA!4_m(dOuWh_Jbv7r6Uuv+P2}IAH
zmBYTpY6bJ<>Ac`Iu2P=+!0LO#rZGxBa0jn}T_yOJ!L(DDm-jsv_ohXioEg&-xw1De
z;7aRF;Fjf5;gr7gi!)uOpG(`2i3?=r?&Mo6OC(u%qj~RhZ7KN9uuH0o2P8h5zYgpM
z5Sw{dhLGNa!-6-K2lHL|#UNy9*DUz%Lxs?P>6HTe*S_FC6L??X?wiX(i+}DnR+DKl
zy_C*s-Ymb${N`~t(-(P<%ofN>8K2xQWxo9DJ<~e}r<-bfWC}K<Oc3zY<KR6d<tiAe
zepR5QY@*<`qc{1LQ!V+6+Ux|n!Wo2A@15f7?5JdkeAp+Twyss+I{Rdvn~m=T$~dz)
zeWd)jce|MJC;uqtj+;JR?&E76g`WH=inH(RmaCj6B>yr_Q*r%+vkE8na>&2<kS>4h
zth3y#-jjm=|HKF$<h>vuzoJ3#`h<G|qD~!x2@_)lB=0i_Y`=9)V0GeG!B3xf6n>T^
z%j<P7QnbEnu5eCWR$<v~Erln(2J)KCkL1nY^(ySJNmiW8;ALoOYGUHGG|p(QR;9{Q
z6(wWq9fyr%lD``3s-+lYt8;5@-&JNHc6YgyE&of&_$pcHGl#ZHXC$#pZFj#WEh>0a
za>*NK>6q$0Qj`DBk-So}MJ|2&bg2~CcIh=I4JD`Puat@XI8`#ue~;W+S4r_dBB!Mp
zV)sgBGFfpR%}iyNcX-X|#cIvY7ypi9MV%T)*IPl3>J!`8bFGhY)K$*qC|`3$)YL{p
z)H|P5OlMn+n4<n~(fm`1Vs(!M#GagB64P8DE_S$%U2Ij!0j}i+7r7Yc<njF2c8_<a
zfd{W#-+ykef5KdG`I@{<fsDMBf~R;_CdRQw>FMysu94=wKY1IUTZ#i;-UlUKQ_ohe
zFR$kCE>*w5JK4;h@5B{$iFBJYqQ{>INEQd4k+9jNBi`?lDj}S3TiirkOT6{}6N#Nl
zED}mqKe$d9?dF-T@5UomJehCJwJ*H9#ano8&adU%!|ca9=k{7&<+m?*I}@i0um?}%
z&lHj1UEHu(z-E@dz@?t60u!DY@L#ll#3#IeDgU20R)XK}2Mevcwn0#7!Y{r9tgi(v
zm{JAjZ;27A)7vHx@`gb`wfBg?&CK0GX_XFw@zbUY1f@yvd@nyNU}GC7kk+0kX!LXr
z|5M8_zIxvs{58)G5xwr;<cL4Nsd6vB<K-}(+#8(yTp2U@EN7SSH)W>sS#%}x9=qPn
z7k2Ij{}#In-p=p*d`Bft@p5zV@f&`d!uRR)4Bm4lOL%X&{p3~t_n$YDV<#W4%Xx{#
z=^MnSY(Fe1_(D+9((;GIdZQr8G~N}Gm*?FP_fCEw8JhN7;y=e*PA~80+$HiqxhLfu
z<MV70<>Rc@;gJaA;@&)I6Yoh*4!)-WZhUoXgr(+pESA(`^OXwsy)5N0`>&+(^KQx4
z-P0tB()LR6&)q1s_8X@(D6dl!J7%#9dOyC%Zx;K5H-EE?V20fRfq9|Hg1*1+@@qBO
z^Ot?s6-YUIOmIW#TvN-}Tg?J)ds)15(zYm&zG=2x=&N~V;#`xflh`cYrY|sizhRkq
z2*WB)*-xq5(+@4<<Pu-Xx#Wu$$E<JS9C4S!ISqo(a!gr!i1TmO5>D1x^98rF9TiCJ
zIKkU7DM`?cPe9PJ`;lN{gTFv_qY=OMuXq6-b~ho`x^S*Pjagg{S;sl|is*COGu-0b
z>s8IU$}f&H_3}2(<n)J}>Z)HjH#G<dD}DA5(mc*7@Z{$zAtMb#p|wwxgeOev7EE|^
zRG?U7qTsITE@9r!0u~HAv&>J=nqtwlpu<d5ozLR)w#gP>k5pJF{_8hi`r)|wsfjbp
z|0sm=6wRN<$!5vTBQLy}uXQ66pG4{gp7P)SxW23@<~?`mA@5Xw3BHh`C(=LlPD!M%
zn;?DaQ<>DSYkJb5uPmfux-?~_Ed(W^Ic;T&_!XrOO049K2)V+Ukh_e>I(ifTmwIu2
z)$5mdolo>~9a!Mbw|nzhzC-mw{M=bye3sXGxGpqS^SllI&(Fyd!O!xam(TwL54YWp
zYQBfU!u+Ogp5XS{O_B5L9cqUdwt2`hI*E3$fzrj7M<GnRzcaIKj7wr}eYS{o|0+%)
zb^{;5FC3=4xko1n?q*RDTx+5z^i8)=;E#?b|INlc0R@f@A<!D*{Y0!o_P*N8yQVab
z+hOi}9tBn@e(rx0_;Q7;c~@3!;PR_1<Mr-S;*)Uh<a@F~k?qlePEIayRknjd?rcH_
zd)az_wX^v?4hCZeDIGR0-dSv-w;GfWEo+jrDmfz4E$<{dOCwosnvk+oTb+S&e$P~y
zCx2L!IIi?5G=vKaCOqgDV7`2r=M{H^z(R@R{Cj?L3hvEg;IFjW&pYdY6aS8?B*CoE
zbe{If{=CcII`ZlrtK-uPf5f{Y)|uy--3;!Q9ZPwQZKv?EKXT=p&h|{8(C0q?Hl+(Z
z57G|_Xdh`4SmkIeSTgkt{~N`Hd?)T6=l`~LuHfaY9^MVtbUAJELU_7ZO!z}jP303z
zapL_mc^2o1#yPyt_Zje=p4ZBktG>Z>am5$235|MYzjzf*zka@D{>Mnw?A7d*W^vOQ
z&9}MPnW<dKGLNov;w#?qhfBbtg1g<ijsMe{4}1x?kMiEXyomFV%MM;O|M`471s3qV
z{CM1?s#x4q&EUGRSj<Ng(}hb+mw(kZF;>ntkvYA}<ani&8P_LPv$s}xd`s+9cpBnY
z@U(KP2=o?o@V_;b;^VQ~#KGscluu93o`0HpBmW-vT>|rNG6_s!eZ+G_piQ99PhVha
zB7@*kzZv|^#bJC)ESveWP8A7uaxxjOt=wo@)^*+Z^@4U|hwRNJ{jXV!bqj@!OpV3N
z#G)P<wx$}JCVI4KOe_vn?R&LMqc}lX<E;J?^;=KH)Pn1#Xeb<Cqh_hGL}Ot;qee}c
zuh7!p*@BN2uI5Xa$}iO1yioA!i54M`Q$GY4{>umm-uNi6g7>k|xn>?gTc;!Z*KAaH
zr!8g{oYOy7z-({0;OmL!`M*D6;h%6+NI>`1R>7Iu()Ih4GIeiSOwm{Pzf5mc&Sc%I
zUuWz6<Xfzl-MLuL@vn^DLa#}Bpu9i~E-AI=UspVXU#Xm*xBsyo&+cpId8|*d@il*N
z=lL{kF}JX^A@^sg6@2n_2L;2$)(B)C)8K2LuvGAsbF$#uDixviKC1=9jHd9jNNpA9
z;-4TiJxNSB@xyN6<9WKm)0c}2dt{sy=1IFRY;92^yzIe6;a~|l;hkr_h5LI~^IE0$
z^SJ+9!(GgIgs(Qmp06&^pYQGr4&LOy-MnAxpYl%dXW_pzF-p?^B(F5z<yYeUDrZIP
zKd4A+&RQy3HRG_vnM7YH&EQyx#g`bRR#t2fRNb;p;CaSW-cH6A!H6w-f<nC<LVxx)
z3-B&=<=0?r6WHO9Efi5IBrWe|FWLHhnq=8RUdbcJc%%yVJ(kQ8>yet;xj-`Y-D4?D
z&V5pGTVpxa@oBS09n)aF5vI=CnWx9vDW${8;Ah0TGftOP{;LJ+@1s_%ta7qK!rq#K
ztE}(wrk#B!C=)ta@Qi<g(3-;s1d<z1@e3S&EU<au9wDte$-H)JxOg^M_Hs>-(c(Rn
z^@8XB`bE44AJ68FzdM`zG{<$G><e6c>AU~&SV%qREoI{2{kkreSH5K?&k++z?)Pt=
zbF1Cq<1O{P%M)}iig(lPL&5>~ZwT6!z86*zixhDwj}<;GxJ4v&da3Y@%$dT*pE*Td
zznv_6n7dY3!%I?(``KLKkBM<Y2|`Ci9_HK^TDYZIgyYg2kynMwMJKu`iX@7C5s(zV
z!=D@O#amwWRKWY@5&<@jSV4Yq4uS1akNG}%{Nqn(Iw4q-$7#-3;9#P0|D^e(-2G;+
zS+q@+%C#*xq8Q9KKYwPru~)+U*NY;vhech&nyeFruHQT^e3bc=u)fAz;h!p%BC)@A
z3;QkGC2aZXyzrij{=zm-qXZ>mz6jXfU%($TM_RCHU6a6JJ8r>18!dsLh}--rCx7r8
zrnL$lyHhK~@M(eI1tBy3m46k4y3cMAymMfV&}}wO!JC`p1U^J@3I1nb71m8Q;1WBt
zfT#FjE7y^k6`c9EOS#sUeBn&XY2mmoX~W?-gMn+g10$EOd5q@6fOL&lbM@3We@at1
zD%P#(>^faN_Wx4NTsb%O3UM>_{zdQ99tE+8yspv`dT6I2Vwf^fghBP9@Zvj6qOviQ
zgyk->hy=`07m3<EUwHfd6M~D~bOhP<<ngfV(h<B<vR#1l&jmrj&0qLmY@Wh*bGw*;
zrnjz;k<T5$1$UJNqbhE5ok)rkyy5s(;E~}E!8u2S1x%R^^DX$`A+Y$QpOC@4WwP24
zOJ$V=nB`6s_sa%!704a(ULgD1jzhM#ezM$J#Wk|*MGbP#&Y$FXUZ}{^V|J6{`j)30
zKSCuqw=Y-bbXcX$`NjSxN7B<T9JN1>bNu+chij&tD34pmbnd99IXrew3_Q;Bow?=t
zq&QEmzs@yr%S3LM*`7S_x-(fX+U(}gw5Vq7u4`f4Ex3_YN^u{n@v&p9(IOLBZ!-6>
zN^j3#ZT)vwAn<aR09W<|{#uSI0ft#Z0t{~j1up&T=0Ep7m;dF{gZ%4OO%gbDO<CY~
zrYir6mCSq*lNJlS;L#AMoz@}Hc>5e5ON;`)Q^a<D+4f%oFSN=<Zaq3L<d)YjvLyMD
z$o>i|k;_SnqJ}YIBI$~SA`2I!ikS8(i}=jj&3!+^ndkgDF5V{!XL&UgWq747ujk$>
z>dBt%#LT-dd=Ag%%LjSCv)&alUocUyP^3ZFYObEhnnSY$_sdv_T<q!>k~$<P6!g+k
zI6ZTPV6^89lct~VP33%)O)HMZ=*{@G-DG<EG?Vl>Q%&EiZ8l*J{$c#<R+RCJTxn_j
z?e;R~r<O|IoZuj3bGcdCw6s^^$^K=MYs%VX+}yTG6<ob5eOpD&B%#>LY{uaW#>$>A
zbn~j@O<gyZn^abOGl|+^Yo@zgS^s%GAJOwx*LF(t)d&mnd^fJ(e6481AC$9-@0{-v
z-q!8i+>P5Kc;$4m_~gRl_>zr{c=q%4bB9Owa5uI+;GJ12!fTy%nmajGg1xqW6VH#s
z=Xj2`ck`~Ay;ox5OL3{VmEw{HW^YB7=rv1j*nVAn>#tDB+ULfS`wJz+8}(O-H>S&S
z`K=OWUvuz28?(`OwtIVju_;dxWS@9SpZ!zkDK<CG=WL>HSFka!`M|Z|f+qV_hh>~s
z(=)i__6Bk~J=)1N$Mp}F&$?1>vtuhckDO}fRt%WNIcdE=TP902o0UK_n|omhn|NG3
z8%ui=+n%EyY<`@bY;T_Avb__1EEMpJTh!QKn$VP{<AU<rlSM9P^$0fil#48{XcV6N
zX01@P@&w^!39Nh@o-^}2n{}SMU{x!>*`v#R>)6|Q!-ToG&i#1D%e;0L-+|gwe9VCs
zLbB@%1#6v_^GU`374-hxD|kpOTWE*xI{~&x9s$Fw8v+Y1oDuTl7UkU_{**U#(^a0G
z`J8<GPx5&eUw^{=O(u$`NH2_c>Jb^<i1<Ui{#QD<gSUxr%3Jkv7Rw29KC_eL{PeMp
zy?d7oXV@G`&UfdPIq!5Sb4vC0NGE!yNJ!27Db2m;kMu2fHYwxPozmRD4Wy4UpORuz
za+FDq&y{xlWutlOBEP10SDe<%BZ687C)#Qr6)Ms^{P38@)Tu3+?wsOU;--CClO+SV
zZ|<MPJ$2`O9!pVE-e><0@|@l(&fTk2$Z0U4g8SpY=RBX7HuG#vxx#NMUd7#3&A=}p
zzevDz(-A(2`P_os3#anANCfd^ZqE{M5I)7#f0@^`)Wh6t#)@uJa}EcS&HpRRCe^B#
z?oD4|%JxjpbZM!XN#;ap(_LFGbICRw=i$)Y#`C!B2=8q+Q{Md9ecVC81>8)>cko<M
zUdL0!*UCFbyjym~+IFc$&wORyK5CJjv0G7QMaLTH1!*=iGo>A6zI1BKm1Ky@5z{6C
zsZmuD<YE0S!NZaNk#p0QRv!ECiQL_eW}Mv5vN+cNZRLu8P|CgFz!~mYodQB<J<<gY
zWmogOwC)r%G8Yxx#iSs_WacLj%KwkAFSt}d<Up$svHO(f_1AK-ny=&dl~KUyWO9|g
z!|FH3mWvPBcgXMHcw?}YV@3Tz4t1Ra9Ii7S3o8rh3oZL9B7CpJUs%RoS=eUfG~t)8
z6NH~w9Tv2G`BO;!SgUY)7Q3LJu7$wqpAx*~#Zv@+Z;}(3V<Ri56KcwDKFNeH^Hm4G
z#?dCh_R3@q``F3cGSR0v=KbL1UJ-4<$+)bB(|x8H_vT~?uK({cI5qh=xwJ&LOG(Qv
zlg!U7lMeSUm+Jb*CpFD%iqz(*veHL%jU{DmyQF>7jir{9i*ZM#XYt-#&dBRkaGvi@
zmk}T9{M$V3omJcud#>?L4`k!pcr%f&g1eR5%4Q$OjXg^_r|59;Zuha~**^6tm+ke<
zoP|u2xf9aNcxJDE#eG6YR$x(>BmZaPw|s@^jr^adoAU7}f8d)i_a*P{gbBP3Au9Zv
zb_?=ngskQU#h>{(AC7rESJ>V>I>K6ECc#pFLXqcx@FYgoDKVUF*O=KdlzKQB9p~`u
ze5S{noqLPVoXwKYSnwrJ)W<)ZT$h*g3QoGs%bef9_upd?*U1(OUa!{^xo#?*;k=|f
zo$JUnJuZpUN}SVpm$1v`nsI$w^ORHOT&MZ88h6t<TPK?R(OO{^S+Ky&dE-5^-=Dsk
zZOTkC723#ZzRLWBnd4Iip)1+%1oUJ=__BXI6TJAOSJ1@LRj9H4w7`;lApxU?CjvJD
zR|+Na9Ob%Y%EaOH<vN?A1_Q^bc2<rf&8lpi9UL5%o~#^CwD~xuZ0F;+Qza&BQxqh~
z7i}bbQv1G8(-&^x39If3JMAwLKA$5Xtg>8H_@$~3eB3rA^b_ynRja|{prCMq;Vk9P
zydd_;3IF94cdV1QU1TG#WSJmqslHTUPTPO^$D&S(#hom&&%y*1@6X&VA0fn|(j;T2
zxHis3RrSL`)g7$+m6%%RshTtIRZg~htXvy%S@nM1Tb0Jy;)06b-tnI?KFYh=msQXr
zbG^Wo6^(+LEI0Y**fH|IImIau(0)m<c!ieWO`ZyYn5=cY`TJA_wIwzST=WPKod0z$
ze`7=hf8ho-fz{`B3!Y7=FzM-;Z?aJAjM2f*5(XyX-;Ce4#TrL2?={)_zts3;;v?fp
zVR^>dd9Ct}JC$S)-1;T|ef|+S0k(4a_s{j@q#pLl+ihDX8}!3iLDk$}-ekF@xo71m
zvy5gd%i05HEwV#u%=p_iEq!FxnH4BWSWMZIV!lk=&wTNuK%tZX4xt|zDnd-WqCziu
z*oC6zND6s{^$4w-CL^ThK3|aYmbuUib_+>IcQL7b9X>L5_P>$dn=UM!!#PnpjWJwu
zS7EjEgZJ~L{MJ@V%kB)}dAQh)r~bYKuia{U{_h)B^C^8u<(*t9#Z~q5GVc!)H$KUu
z=lLY>OPQU{i8PsAHqR{F@R%u^{sPl^WgAV)&D%_iu8Eu7U2xX)v~{}KrMNKOPZl@%
z?mn{RPM3YdBdh$0OYg=+?*72lyc!HkSo5MRd9x2m@lNpx7V<S+A-IQq4xiXhHX)1e
zGXyhUHwwwRu?beYatpX?z7WXS^i)W(-Oj}1F`wDmXKALpF14A|zLPdTQajh=myV{X
ziFcpb%;z1ZpD!7iFO*)T^xWM-Dl<$#N-|Vh%KKTb)U5_fi8$*`O0pBD%1UZ`$R<`z
zkUf0kB=<X(W{&gu>|E9{t-QJ0`FS_w>T&;{?Z_>;Jd?Y*wwtFid>YT=iN@TYWlXsl
zHz{-Nx-86_m>A3RJ28<<_NxuY#k~qV*TPhI=53kIBlV<D^vfMC;jC?8qE6HGMW%*a
z6Uj)77cI#U6`8qon(+R+H$~cFSBRW^7Gw6xc!HTzleBrG#so9gRZM0<mkP~Pns1ry
zowLSt<)nY68ifp|#|8Eay<hWD;F9e&p#y)_gu4VM3Ysq!5H?NZ6q?7iLeMYqsnAk;
zJ)x`@pIP|@blAE}?y$Q2>}Ne2qsC_5Eys3<<uCiTkN??M@!n%yEc%soPi30m(NI%?
zvlIXEKIRAz<SXS8On7rtaL>a~0cNute&&j30h_(LLhk}+n?8E^*6e^rtC^458`BrQ
zdS>4jwwSH@6K1-E!{1zl)7x}{)G^bR*3C-4Iixky(rh$snRaVbeUQ*(i_XyC%WhIH
zQwvv(Tot5Rw?<gCZLy`$VdF5tub-IsW7D~X;wCN>yk1l#)U}aKuv%0>;6B@X0fW^y
zg$^iOH)GY&G8XBQHuui2H)Al~X8LINN0Xe(U1rRJnWo2NTg~E*x0}VT+`>JpQJD9v
z|1z$Fsb@GHy_a%*iV5JzRJ_LdsaB55D@~P~`;HiwOawRY#z`f-lQ{%<ZUju{n|m&p
z@5uCczF)Dfyipv}dAt1|@HQ?!!FS6v)!fNzvYGT;ce57@Zkl$v>6!6(-8NzMoo&{r
z<YX?aEo;VQ&0)>}1vC98^PF0>A6%xpum*!^kW`Yj8mo%KCWhx%m3Tk>pTkkaqsd)c
z`Gs%eE)%{e9Z_B%q4Qk+Kd10=$Z+sU7rOEZUgZ%g-eo8_H#&#6uxGEJj%m5z^^NXA
z8aMX~Bzm^+i*j!ekZ4^ebW>~sc-{zPhiXhPD~K+B>Ij}=0<l9DDsno7wKFhqhFLH;
zvYYa+y>1>R|Jv-NnV7k7!dbHy5{4ELUMo%G-e+1Iw|{89+OC^DMRpd?vO+(eeQT!h
z`F)e&`<8f{XM05vr?8y^@9i5)d6haj_~curs2mM7)vQ`>rgh!<yjDfAx3+DPxE6zo
zsJdyKiS`QV6Pg{f_Grd=9ArD#^PAI}X&+ne#ocTt0{5}y+&;{9Y{p5p$&XI5E!=pQ
zt-tI%TiEmof^U8-7f|86z}u3NBe-&|fM71?CBaL_9R((RF66%?Q!OCoX(4o%S5!8n
zm|1$#b3yr_gdg&gijPYRHD8zK6PPRWbG@}}iFTa4#td7z<;EZQ6JuNW8v+vrpLl2q
zsVbf0e-Ydx(5H4=fMxAW!DqFqf(4dW1?uOx2-N8G@D`-Z6>z?<El@UjBOiAruYlA1
zqr6GarTHU0lLXfKeC1>Qx<P40(t6dyFDJ|8CeD_;TqCdk=++EXz2{R^pI+cnPoJ-$
z_(SHD+;PiKB4)WSMHc9Ai0Q=Hi)54?6uJM;T~uGlS5(+RO7zM`9nq|#VWOb8p(d`;
zT*>n<cq2~{>slVMpKbi7@|pP)r%mQ<)+prm6tw5N@^lNIjlnCT&vgl7U{_Zz*{R68
zH(f&WZ;SYHt{Lh-U%wMOev3<a#)e{vJsC=>=~I$b#I{)SH5squy}MX|-#UVscWY=b
z&(3WTykXbQ@)oFG;awvC7rbY?_%e&&?w~$?$A(h^TkJ~t|F2@^pY5I_&>mPS5Fy7Y
zkY3Lw`0$c0zn-YF+@kDe>4KhMxsA@YvL}>OWEm87WE*B$%E?9DmMUGdSg!diqpXg#
zCAV|C1MfNxH{Op;tNC9j3-Zt9*u<+98_E+fXF1=3U6=SACK>Wy{=0^2;_P%@lV^8%
z7j(4q&t7|t?|?}^@1fc%uKy8<d_23__)INS_)P?o`P7ZPc@Hti@E&^{!n1wW7oJ`o
zC*Hi&Z`^0E`0;Q&+s#w{XEM(WA9Eo?4PW8C!Ab)2^bQJMXl@mHy4^zP>A_aP=gYSV
z3I=lso_KdusORuCw*0yWEKVP^IN}ysbLt8?ux%2V%lYJzCx?K<S$5yaGdN$J`p@QP
z(8&K*M@}H_n=fC^Lw^2?+lBcHHB5Lb|Ht!Zv$677`J3>@oonMyaP#Mz8TO7xRMv)T
zd6+am_n9SpM}u2=x6eMs{brg5@6X~KKD`PSzIBI`_<EC%^W3~B%Pz$o&wsyQKi|HL
zAl@aBW?WHgckteN)Wo-L)l{O>LB*`Qf^WFL2%Jut$m>uuPw?oxFhMPU4WYQzD+Qij
zE8$=IZn3}}lSx7*wV!wd!a{f>W)<`7eszfFV7e<egYY!2pl_bMo^#gmtT9sO@m*-f
z`;R$-FY0&?Z(5@o=eLdF{CXeP@-=5i^6p#S%)MlhA#bFeE#E@Nc0QZcw*?lxeapYg
za}v+eme~RqjzkDNXVn!{{yLXm`B53)=jg@!ku{S9w+ZeRXK1&PYpH%AR&|$IjI;m0
z_|A3QVk)<;ifcOA%B>4)5?{J*hPcDMkAj(T48jXP%oc2U%qduE!Y!=P6fST}eYH^0
zlIKFpR?ZfjD|Ae7^OoB}mfW&J?y{o%<>@U#8NS>?Uu*vhJv|>OsB)@VK=@FMVCkMH
z;og7K#cMdNL??YKmt3&YRDRO*jUxBgo66^K{1B7znkl|%#taFT&kMlkS`m}aO`tbl
F0{~I{!JYsB

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index b/tensorflow/contrib/tensorrt/test/testdata/model.ckpt-46900.index
new file mode 100644
index 0000000000000000000000000000000000000000..537976571337508ab1798d33646c51d62a146ecc
GIT binary patch
literal 652
zcmaFOHiLnIje}8&iGz`Wn>#7BB+)RwAiq4dh=WmxL4(C~S+$@Q3p=k-yr}2_1r`k!
z(^q#&7Bg@QCFkdr8KvkaWhNGLFbc5<F>xp;aA-6znmW$Aki*R?tnZkTm@6v9(!t0m
ztxz&qF+_k#INnfFf`N%qBA}zWW<NWde0FM4UTO{pqmY0Q69+Rd3j-r-lt~A(Mgynm
zskSvUd0FN4d85758FY9+LWMekGD1x9tm$4V3=E=-Dhiwoy5(%_!tsWBhZsy86guQI
z8kkJgE7G`lS&a@e@Hi-RC@@M3#1=4F3dtS@ag`V)BxKud`#ISZf(|oqC<rkzY8a?!
zG#p?tt=|){k)Ku3?+6302Fnu`R!s?O;~gm?Oo~p&7<koKmT<DF>2OTm9LvBZnv$AV
zoGN(QhgFD~qk%=C;|Z@ugMjI}C*iCN{IW`d)q?5_Us%;8!oGXB^D~LE<_k)(wD2)H
zN`$;~X<}oO61~m9$#|YO*<=czMneOKsQ|ZvC=aWYV3d&(1A`c=k_77<U2Q=oDb{!+
zc?JeKR(XLZ9-lU_+VwLUa4B?5QPa>c{dB~nipQ*<k%LQN1FN*ax8JWE1#SB|xD;-%
zN(i(la0_x6ZDCLpb`fIY;8JLqs;1GffYEeXM_NCh-WCQ)C6=w~tV#mDkL=G0i*AvW
zXSu4uD!;*O`d3~CZvOO~{G`O3_~Me(0uD|g28D*Nni?FYna6J5WME)mc*)4Xz{J47
zz_`P0VgyKl5kwsLv~U9hBXb7BHWm;UEOsK&_QY-$juR~WAR+uX{C7jQN~!y80BP%#
Ang9R*

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index c3cff28574..0d23eeae1a 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -43,14 +43,15 @@ TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
     "gdef", "input_names", "input_dims", "output_names", "expected_output_dims"
 ])
 
-RunParams = namedtuple(
-    "RunParams",
-    ["use_optimizer", "precision_mode", "dynamic_engine", "test_name"])
+RunParams = namedtuple("RunParams", [
+    "use_optimizer", "precision_mode", "dynamic_engine", "test_name",
+    "use_calibration"
+])
 
 ConversionParams = namedtuple("ConversionParams", [
     "max_batch_size", "max_workspace_size_bytes", "precision_mode",
     "minimum_segment_size", "is_dynamic_op", "maximum_cached_engines",
-    "cached_engine_batch_sizes", "rewriter_config"
+    "cached_engine_batch_sizes", "rewriter_config", "use_calibration"
 ])
 
 PRECISION_MODES = ["FP32", "FP16", "INT8"]
@@ -162,11 +163,15 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_op=run_params.dynamic_engine,
         maximum_cached_engines=1,
         cached_engine_batch_sizes=None,
-        rewriter_config=None)
+        rewriter_config=None,
+        use_calibration=run_params.use_calibration)
 
   def ShouldRunTest(self, run_params):
     """Whether to run the test."""
-    return True
+    # This setting combination requires quantization nodes to be present in
+    # order to build the engine.
+    return not (IsQuantizationMode(run_params.precision_mode) and
+                not run_params.use_calibration)
 
   def VerifyRunForEngine(self, engine_name, graph_state, expect_run=True):
     """Verify the state of a particular engine after sess.run()."""
@@ -237,7 +242,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
           conversion_params.minimum_segment_size,
           conversion_params.is_dynamic_op,
           conversion_params.maximum_cached_engines,
-          conversion_params.cached_engine_batch_sizes)
+          conversion_params.cached_engine_batch_sizes,
+          conversion_params.use_calibration)
 
       graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg)
     else:
@@ -329,6 +335,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_op=conversion_params.is_dynamic_op,
         maximum_cached_engines=conversion_params.maximum_cached_engines,
         cached_engine_batch_sizes=conversion_params.cached_engine_batch_sizes,
+        use_calibration=conversion_params.use_calibration,
         session_config=config_for_trt)
 
   def _WriteGraph(self, run_params, gdef, graph_state):
@@ -428,10 +435,12 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         is_dynamic_engine = not node.attr["static_engine"].b
         self.assertEqual(run_params.dynamic_engine, is_dynamic_engine,
                          node.name)
+        self.assertEqual(node.attr["use_calibration"].b,
+                         run_params.use_calibration, node.name)
 
         has_calibration_data = len(node.attr["calibration_data"].s)
         if (IsQuantizationMode(run_params.precision_mode) and
-            graph_state == GraphState.INFERENCE):
+            run_params.use_calibration and graph_state == GraphState.INFERENCE):
           self.assertTrue(has_calibration_data, node.name)
         else:
           self.assertFalse(has_calibration_data, node.name)
@@ -482,7 +491,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                                 config_no_trt, GraphState.ORIGINAL)
 
     # Run calibration if necessary.
-    if IsQuantizationMode(run_params.precision_mode):
+    if (IsQuantizationMode(run_params.precision_mode) and
+        run_params.use_calibration):
 
       calib_config = self._GetConfigProto(run_params, GraphState.CALIBRATE)
       logging.info("Running calibration graph, config:\n%s", str(calib_config))
@@ -552,27 +562,38 @@ def _AddTests(test_class):
 
   use_optimizer_options = [False, True]
   dynamic_engine_options = [False, True]
-  for (use_optimizer, precision_mode, dynamic_engine) in itertools.product(
-      use_optimizer_options, PRECISION_MODES, dynamic_engine_options):
+  use_calibration_options = [False, True]
+  opts = itertools.product(use_optimizer_options, PRECISION_MODES,
+                           dynamic_engine_options, use_calibration_options)
+  for (use_optimizer, precision_mode, dynamic_engine, use_calibration) in opts:
     if IsQuantizationMode(precision_mode):
       if use_optimizer:
         # TODO(aaroey): if use_optimizer is True we need to get the inference
         # graphdef using custom python wrapper class, which is not currently
         # supported yet.
         continue
-      if not dynamic_engine:
+      if use_calibration and not dynamic_engine:
+        # Static engine with use_calibration=False will be static, so we want to
+        # test that. If use_calibration=True, only dynamic op is supported.
         # TODO(aaroey): construction of static calibration engine is not
         # supported yet.
         continue
+    else:
+      if use_calibration:
+        # Don't calibrate in FP32 or FP16 mode
+        continue
 
     conversion = "OptimizerConversion" if use_optimizer else "ToolConversion"
-    engine_type = ("DynamicEngine" if dynamic_engine else "StaticEngine")
-    test_name = "%s_%s_%s" % (conversion, precision_mode, engine_type)
+    engine_type = "DynamicEngine" if dynamic_engine else "StaticEngine"
+    calibration_type = "UseCalibration" if use_calibration else "NoCalibration"
+    test_name = "%s_%s_%s_%s" % (conversion, engine_type, precision_mode,
+                                 calibration_type)
     run_params = RunParams(
         use_optimizer=use_optimizer,
         precision_mode=precision_mode,
         dynamic_engine=dynamic_engine,
-        test_name=test_name)
+        test_name=test_name,
+        use_calibration=use_calibration)
     setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params))
 
 
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 7dc62e24df..f4653505f7 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -119,6 +119,8 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "Exit",
                                           "Exp",
                                           "Expm1",
+                                          "FakeQuantWithMinMaxVars",
+                                          "FakeQuantWithMinMaxArgs",
                                           "Fill",
                                           "Floor",
                                           "FloorDiv",
@@ -161,6 +163,8 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "PreventGradient",
                                           "Prod",
                                           "Polygamma",
+                                          "QuantizeAndDequantizeV2",
+                                          "QuantizeAndDequantizeV3",
                                           "Pow",
                                           "Real",
                                           "RealDiv",
-- 
GitLab


From e16716d94c0c6cde419129827c37a8f61bd8b023 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 20 Nov 2018 15:39:19 -0800
Subject: [PATCH 0627/1554] Remove unused lambda captures

This fixes the lambda capture <> is not used compiler warning.

PiperOrigin-RevId: 222318600
---
 .../image/kernels/adjust_hsv_in_yiq_op.cc     |  2 +-
 tensorflow/core/kernels/adjust_hue_op.cc      |  4 ++--
 tensorflow/core/kernels/barrier_ops.cc        |  2 +-
 tensorflow/core/kernels/deep_conv2d.cc        | 19 +++++++++----------
 tensorflow/core/kernels/tensor_array_ops.cc   |  6 +++---
 .../internal/optimized/optimized_ops.h        |  1 -
 .../internal/reference/reference_ops.h        |  1 -
 7 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc
index 478b716d88..108da04494 100644
--- a/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc
+++ b/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc
@@ -115,7 +115,7 @@ class AdjustHsvInYiqOp<CPUDevice> : public AdjustHsvInYiqOpBase {
         *context->device()->tensorflow_cpu_worker_threads();
     Shard(worker_threads.num_threads, worker_threads.workers, channel_count,
           kCostPerChannel,
-          [channel_count, &input_data, &output_data, &tranformation_matrix](
+          [&input_data, &output_data, &tranformation_matrix](
               int64 start_channel, int64 end_channel) {
             // Applying projection matrix to input RGB vectors.
             const float* p = input_data.data() + start_channel * kChannelSize;
diff --git a/tensorflow/core/kernels/adjust_hue_op.cc b/tensorflow/core/kernels/adjust_hue_op.cc
index 6079aa749d..52dec94305 100644
--- a/tensorflow/core/kernels/adjust_hue_op.cc
+++ b/tensorflow/core/kernels/adjust_hue_op.cc
@@ -216,8 +216,8 @@ class AdjustHueOp<CPUDevice> : public AdjustHueOpBase {
         *context->device()->tensorflow_cpu_worker_threads();
     Shard(worker_threads.num_threads, worker_threads.workers, channel_count,
           kCostPerChannel,
-          [channel_count, &input_data, &output_data, delta_h](
-              int64 start_channel, int64 end_channel) {
+          [&input_data, &output_data, delta_h](int64 start_channel,
+                                               int64 end_channel) {
             const float* p = input_data.data() + start_channel * kChannelSize;
             float* q = output_data.data() + start_channel * kChannelSize;
             for (int i = start_channel; i < end_channel; i++) {
diff --git a/tensorflow/core/kernels/barrier_ops.cc b/tensorflow/core/kernels/barrier_ops.cc
index 944564dfba..aa91235822 100644
--- a/tensorflow/core/kernels/barrier_ops.cc
+++ b/tensorflow/core/kernels/barrier_ops.cc
@@ -180,7 +180,7 @@ class Barrier : public ResourceBase {
         // SQSS is closed, nothing is left in the incomplete set,
         // the queue is not already marked as closed, and (most
         // importantly), the queue has entries in it.
-        [this, ctx, callback, component_index]() {
+        [this, ctx, callback]() {
           if (!ctx->status().ok()) {
             callback();
             return;
diff --git a/tensorflow/core/kernels/deep_conv2d.cc b/tensorflow/core/kernels/deep_conv2d.cc
index f9c8f16cb9..8736639881 100644
--- a/tensorflow/core/kernels/deep_conv2d.cc
+++ b/tensorflow/core/kernels/deep_conv2d.cc
@@ -434,10 +434,9 @@ struct TransformFilters {
         tile_spatial_size, base_filter_spatial_size, transform_matrix);
 
     auto shard = [&ctx, &args, &transform, &base_filter_rows, &base_filter_cols,
-                  &num_filters_transform, &in_depth, &out_depth,
-                  &filter_shards_row, &filter_shards_col, &tile_spatial_size,
-                  &filter_in, &transform_matrix,
-                  &filter_out](int64 start, int64 limit) {
+                  &num_filters_transform, &in_depth, &filter_shards_row,
+                  &filter_shards_col, &tile_spatial_size, &filter_in,
+                  &transform_matrix, &filter_out](int64 start, int64 limit) {
       // Allocate buffer for pre-processed filter:
       //   [base_filter_rows, base_filter_cols, num_filters_transform, in_depth]
       //
@@ -533,9 +532,9 @@ struct PackFilters {
     const int64 out_depth = args.out_depth;
     const int64 num_filters = filter_shards_row * filter_shards_col * out_depth;
 
-    auto shard = [&ctx, &packed_filters, &filter_transform_data,
-                  &tile_spatial_size, &in_depth, &out_depth, &filter_shards_row,
-                  &filter_shards_col, &num_filters](int64 start, int64 limit) {
+    auto shard = [&ctx, &packed_filters, &filter_transform_data, &in_depth,
+                  &out_depth, &filter_shards_row, &filter_shards_col,
+                  &num_filters](int64 start, int64 limit) {
       const int64 filter_coord_stride = num_filters * in_depth;
       for (int64 i = start; i < limit; ++i) {
         // Allocate filter buffer [out_depth, shard_rows, shard_cols, in_depth].
@@ -1004,9 +1003,9 @@ struct DeepConv2D<CPUDevice, T> {
         out_tile_spatial_size, tile_spatial_size, output_transform_matrix);
 
     auto shard = [&ctx, &args, &transform, &packed_filters, &in_depth,
-                  out_depth, tile_rows, tile_cols, out_tile_rows, out_tile_cols,
-                  filter_shards_row, filter_shards_col, tile_spatial_size,
-                  &input, &tile_transform_matrix, &output_transform_matrix,
+                  out_depth, out_tile_rows, out_tile_cols, filter_shards_row,
+                  filter_shards_col, tile_spatial_size, &input,
+                  &tile_transform_matrix, &output_transform_matrix,
                   &output](int64 batch_start, int64 batch_limit) {
       const int64 row_tiles =
           (args.out_rows + out_tile_rows - 1) / out_tile_rows +
diff --git a/tensorflow/core/kernels/tensor_array_ops.cc b/tensorflow/core/kernels/tensor_array_ops.cc
index a97a71b344..aa85f546a8 100644
--- a/tensorflow/core/kernels/tensor_array_ops.cc
+++ b/tensorflow/core/kernels/tensor_array_ops.cc
@@ -352,9 +352,9 @@ class TensorArrayGradOp : public TensorArrayCreationOp {
     }
 
     const auto key = strings::StrCat(output_handle(0), output_handle(1));
-    auto creator = [this, key, tensor_array, array_size, marked_size,
-                    element_shape, shape_to_prepend, tensor_array_output_handle,
-                    output_handle](TensorArray** ret) -> Status {
+    auto creator = [key, tensor_array, array_size, marked_size, element_shape,
+                    shape_to_prepend,
+                    tensor_array_output_handle](TensorArray** ret) -> Status {
       *ret = new TensorArray(
           key, tensor_array->ElemType(), *tensor_array_output_handle,
           array_size, element_shape, tensor_array->HasIdenticalElementShapes(),
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 4ff875091e..e2329c79c7 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -4293,7 +4293,6 @@ inline void LogSoftmax(const SoftmaxParams& params,
   using FixedPointScaledDiff =
       gemmlowp::FixedPoint<int32, kScaledDiffIntegerBits>;
   using FixedPointAccum = gemmlowp::FixedPoint<int32, kAccumulationIntegerBits>;
-  using FixedPoint0 = gemmlowp::FixedPoint<int32, 0>;
 
   const int trailing_dim = input_shape.DimensionsCount() - 1;
   const int outer_size =
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index fd37865c3e..920f154049 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -2736,7 +2736,6 @@ inline void LogSoftmax(const SoftmaxParams& params,
   using FixedPointScaledDiff =
       gemmlowp::FixedPoint<int32, kScaledDiffIntegerBits>;
   using FixedPointAccum = gemmlowp::FixedPoint<int32, kAccumulationIntegerBits>;
-  using FixedPoint0 = gemmlowp::FixedPoint<int32, 0>;
 
   const int trailing_dim = input_shape.DimensionsCount() - 1;
   const int outer_size =
-- 
GitLab


From f93cdcb3cd1dac7769ac393dbcbab801a304480b Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 20 Nov 2018 15:53:51 -0800
Subject: [PATCH 0628/1554] Replace use of tf.gradients with GradientTape in
 _FusedBatchNormGradGrad

Fixes #23794

PiperOrigin-RevId: 222320612
---
 tensorflow/python/ops/nn_grad.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 902653befc..34404edc9a 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -18,13 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
-from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 
@@ -948,10 +948,14 @@ def _FusedBatchNormGradGrad(op, *grad):
   grad_grad_x = grad[0]
   grad_grad_scale = grad[1]
   grad_grad_offset = grad[2]
-  grad_x, grad_scale, grad_offset = _BatchNormGrad(
-      grad_y, x, scale, pop_mean, pop_var, epsilon, data_format, is_training)
-  grad_initial = [grad_grad_x, grad_grad_scale, grad_grad_offset]
-  grad_grad_y, grad_x, grad_scale = gradients_impl.gradients(
+  with backprop.GradientTape() as tape:
+    tape.watch(grad_y)
+    tape.watch(x)
+    tape.watch(scale)
+    grad_x, grad_scale, grad_offset = _BatchNormGrad(
+        grad_y, x, scale, pop_mean, pop_var, epsilon, data_format, is_training)
+    grad_initial = [grad_grad_x, grad_grad_scale, grad_grad_offset]
+  grad_grad_y, grad_x, grad_scale = tape.gradient(
       [grad_x, grad_scale, grad_offset], [grad_y, x, scale], grad_initial)
   return grad_grad_y, grad_x, grad_scale, None, None
 
-- 
GitLab


From 03e9335993127eb9d6c18a86e83611583cdd5eba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 15:54:07 -0800
Subject: [PATCH 0629/1554] Test to show how strided_slice fails sometimes.

PiperOrigin-RevId: 222320648
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/testing/generate_examples.py  | 26 +++++++++++++++++++
 .../testing/generated_examples_zip_test.cc    |  3 +++
 3 files changed, 30 insertions(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 8255211d27..fcd7255910 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -297,6 +297,7 @@ def generated_test_models():
         "squeeze",
         "strided_slice",
         "strided_slice_1d_exhaustive",
+        "strided_slice_buggy",
         "sub",
         "tile",
         "topk",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 9b0f59f9da..b7e549cc5c 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -2520,6 +2520,32 @@ def make_strided_slice_1d_exhaustive_tests(zip_path):
   _make_strided_slice_tests(zip_path, test_parameters)
 
 
+def make_strided_slice_buggy_tests(zip_path):
+  """Make a set of tests to show strided_slice yields incorrect results."""
+
+  test_parameters = [{
+      "unused_iteration_counter": [1],
+  }]
+
+  def build_graph(parameters):
+    """Build the strided_slice op testing graph."""
+    del parameters
+    input_values = tf.placeholder(dtype=tf.float32, shape=[4, 2])
+    data = tf.constant([[0, 1, 2, 3],
+                        [4, 5, 6, 7],
+                        [8, 9, 10, 11],
+                        [12, 13, 14, 15]], tf.float32)
+    return [input_values], [input_values + data[:, :2]]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    del parameters
+    input_values = np.zeros([4, 2], dtype=np.float32)
+    return [input_values], sess.run(
+        outputs, feed_dict={inputs[0]: input_values})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_lstm_tests(zip_path):
   """Make a set of tests to do basic Lstm cell."""
 
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index 6f31daa5fb..91a4851fb0 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -102,6 +102,9 @@ std::map<string, string> kBrokenTests = {
     {R"(^\/add.*dtype=tf\.int64)", "119126484"},
     {R"(^\/floor_div.*dtype=tf\.int64)", "119126484"},
     {R"(^\/squared_difference.*dtype=tf\.int64)", "119126484"},
+
+    // Strided Slice chooses the wrong dimension.
+    {R"(^\/strided_slice_buggy)", "119786029"},
 };
 
 // Allows test data to be unarchived into a temporary directory and makes
-- 
GitLab


From 285bfa4f5231d18f0c4ef0169a79d1e71e1a601f Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 20 Nov 2018 16:15:53 -0800
Subject: [PATCH 0630/1554] Add V2 precision, recall metrics.

PiperOrigin-RevId: 222323900
---
 tensorflow/python/keras/metrics.py      | 120 ++++++++++++
 tensorflow/python/keras/metrics_test.py | 244 ++++++++++++++++++++++++
 2 files changed, 364 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 030bf4768c..0793585047 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -977,6 +977,126 @@ class TruePositives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
+class Precision(Metric):
+  """Computes the precision of the predictions with respect to the labels.
+
+  The metric creates two local variables, `true_positives` and `false_positives`
+  that are used to compute the precision. This value is ultimately returned as
+  `precision`, an idempotent operation that simply divides `true_positives`
+  by the sum of `true_positives` and `false_positives`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `Precision` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(Precision, self).__init__(name=name, dtype=dtype)
+    self.thresholds = [0.5] if thresholds is None else thresholds
+    self.tp = self.add_weight(
+        'true_positives',
+        shape=(len(self.thresholds),),
+        initializer=init_ops.zeros_initializer)
+    self.fp = self.add_weight(
+        'false_positives',
+        shape=(len(self.thresholds),),
+        initializer=init_ops.zeros_initializer)
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates true positive and false positive statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    return _update_confusion_matrix_variables({
+        _ConfusionMatrix.TRUE_POSITIVES: self.tp,
+        _ConfusionMatrix.FALSE_POSITIVES: self.fp
+    }, y_true, y_pred, self.thresholds, sample_weight)
+
+  def result(self):
+    return array_ops.where(
+        math_ops.greater(self.tp + self.fp, 0),
+        math_ops.div(self.tp, self.tp + self.fp),
+        array_ops.zeros_like(self.thresholds))
+
+
+class Recall(Metric):
+  """Computes the recall of the predictions with respect to the labels.
+
+  This metric creates two local variables, `true_positives` and
+  `false_negatives`, that are used to compute the recall. This value is
+  ultimately returned as `recall`, an idempotent operation that simply divides
+  `true_positives` by the sum of `true_positives` and `false_negatives`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, thresholds=None, name=None, dtype=None):
+    """Creates a `Recall` instance.
+
+    Args:
+      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(Recall, self).__init__(name=name, dtype=dtype)
+    self.thresholds = [0.5] if thresholds is None else thresholds
+    self.tp = self.add_weight(
+        'true_positives',
+        shape=(len(self.thresholds),),
+        initializer=init_ops.zeros_initializer)
+    self.fn = self.add_weight(
+        'false_negatives',
+        shape=(len(self.thresholds),),
+        initializer=init_ops.zeros_initializer)
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates true positive and false negative statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    return _update_confusion_matrix_variables({
+        _ConfusionMatrix.TRUE_POSITIVES: self.tp,
+        _ConfusionMatrix.FALSE_NEGATIVES: self.fn
+    }, y_true, y_pred, self.thresholds, sample_weight)
+
+  def result(self):
+    return array_ops.where(
+        math_ops.greater(self.tp + self.fn, 0),
+        math_ops.div(self.tp, self.tp + self.fn),
+        array_ops.zeros_like(self.thresholds))
+
+
 @tf_export('keras.metrics.binary_accuracy')
 def binary_accuracy(y_true, y_pred, threshold=0.5):
   threshold = math_ops.cast(threshold, y_pred.dtype)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index c6a49c3c3e..535802a64a 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.keras import metrics
 from tensorflow.python.keras.engine.training import Model
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -733,5 +734,248 @@ class TruePositivesTest(test.TestCase):
     self.assertAllClose([222., 111., 37.], self.evaluate(result))
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class PrecisionTest(test.TestCase):
+
+  def test_config(self):
+    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
+    self.assertEqual(p_obj.name, 'my_precision')
+    self.assertLen(p_obj.variables, 2)
+    self.assertEqual([v.name for v in p_obj.variables],
+                     ['true_positives:0', 'false_positives:0'])
+    self.assertEqual(p_obj.thresholds, [0.4, 0.9])
+
+  def test_value_is_idempotent(self):
+    p_obj = metrics.Precision(thresholds=[0.3, 0.72])
+    y_pred = random_ops.random_uniform(shape=(10, 3))
+    y_true = random_ops.random_uniform(shape=(10, 3))
+    update_op = p_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_precision = self.evaluate(p_obj.result())
+    for _ in range(10):
+      self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()),
+                           1e-3)
+
+  def test_unweighted(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_unweighted_all_incorrect(self):
+    p_obj = metrics.Precision(thresholds=[0.5])
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs)
+    y_true = constant_op.constant(1 - inputs)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0, self.evaluate(result))
+
+  def test_weighted(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
+    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
+    weighted_tp = 3.0 + 4.0
+    weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)
+    expected_precision = weighted_tp / weighted_positives
+    self.assertAlmostEqual(expected_precision, self.evaluate(result))
+
+  def test_div_by_zero(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([0, 0, 0, 0])
+    y_true = constant_op.constant([0, 0, 0, 0])
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertEqual(0, self.evaluate(result))
+
+  def test_unweighted_with_threshold(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 0.7])
+    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
+
+  def test_weighted_with_threshold(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 1.1])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[4, 0], [3, 1]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred, sample_weight=weights)
+    weighted_tp = 0 + 3.
+    weighted_positives = (0 + 3.) + (4. + 0.)
+    expected_precision = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)
+
+  def test_extreme_thresholds(self):
+    p_obj = metrics.Precision(thresholds=[-1.0, 2.0])  # beyond values range
+    y_pred = math_ops.cast(
+        constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32)
+    y_true = math_ops.cast(
+        constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertArrayNear([0.75, 0.], self.evaluate(result), 0)
+
+  def test_multiple_updates(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 1.1])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[4, 0], [3, 1]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights)
+    for _ in range(2):
+      self.evaluate(update_op)
+
+    weighted_tp = (0 + 3.) + (0 + 3.)
+    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
+    expected_precision = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
+                         1e-3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RecallTest(test.TestCase):
+
+  def test_config(self):
+    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
+    self.assertEqual(r_obj.name, 'my_recall')
+    self.assertLen(r_obj.variables, 2)
+    self.assertEqual([v.name for v in r_obj.variables],
+                     ['true_positives:0', 'false_negatives:0'])
+    self.assertEqual(r_obj.thresholds, [0.4, 0.9])
+
+  def test_value_is_idempotent(self):
+    r_obj = metrics.Recall(thresholds=[0.3, 0.72])
+    y_pred = random_ops.random_uniform(shape=(10, 3))
+    y_true = random_ops.random_uniform(shape=(10, 3))
+    update_op = r_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_recall = self.evaluate(r_obj.result())
+    for _ in range(10):
+      self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
+
+  def test_unweighted(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_unweighted_all_incorrect(self):
+    r_obj = metrics.Recall(thresholds=[0.5])
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs)
+    y_true = constant_op.constant(1 - inputs)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0, self.evaluate(result))
+
+  def test_weighted(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
+    weighted_tp = 3.0 + 1.0
+    weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
+    expected_recall = weighted_tp / weighted_t
+    self.assertAlmostEqual(expected_recall, self.evaluate(result))
+
+  def test_div_by_zero(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([0, 0, 0, 0])
+    y_true = constant_op.constant([0, 0, 0, 0])
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertEqual(0, self.evaluate(result))
+
+  def test_unweighted_with_threshold(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 0.7])
+    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
+
+  def test_weighted_with_threshold(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 1.1])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[1, 4], [3, 2]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred, sample_weight=weights)
+    weighted_tp = 0 + 3.
+    weighted_positives = (0 + 3.) + (4. + 0.)
+    expected_recall = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
+
+  def test_extreme_thresholds(self):
+    r_obj = metrics.Recall(thresholds=[-1.0, 2.0])  # beyond values range
+    y_pred = math_ops.cast(
+        constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32)
+    y_true = math_ops.cast(
+        constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertArrayNear([1.0, 0.], self.evaluate(result), 0)
+
+  def test_multiple_updates(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 1.1])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[1, 4], [3, 2]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
+    for _ in range(2):
+      self.evaluate(update_op)
+
+    weighted_tp = (0 + 3.) + (0 + 3.)
+    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
+    expected_recall = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
+                         1e-3)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 0fee95f2a98f047f074d6c16c1841f0f94834fcf Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 20 Nov 2018 16:28:50 -0800
Subject: [PATCH 0631/1554] Fix conversion for binary operations and add
 corresponding test. Main changes include:

1. reenable SimpleMultiEnginesTest, and fix the graph so that binary ops "mul"
   and "div" don't broadcast on batch dimension, also fix the rewriter config
   to disable LayoutOptimizer (by calling OptimizerDisabledRewriterConfig() in
   base class)
2. rewrite TensorRTGetBroadcastShape() to GetTrtBroadcastShape(), make it a
   member function of the converter and add corresponding tests.
3. implement RECIP computation for LambdaFactory::unary<Eigen::half>
4. consolidate ConverBinary() and related methods (BinaryTensorOpWeight,
   BinaryTensorOpTensor) and add corresponding tests.

PiperOrigin-RevId: 222325662
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 413 ++++++------
 .../contrib/tensorrt/convert/convert_nodes.h  |   7 +
 .../tensorrt/convert/convert_nodes_test.cc    | 610 +++++++++++++++---
 tensorflow/contrib/tensorrt/test/base_test.py |  51 +-
 .../test/tf_trt_integration_test_base.py      |   6 +
 5 files changed, 785 insertions(+), 302 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 27c20204bb..631c2575c0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -202,6 +202,21 @@ string DebugString(const nvinfer1::DimensionType type) {
   }
 }
 
+string DebugString(const nvinfer1::DataType trt_dtype) {
+  switch (trt_dtype) {
+    case nvinfer1::DataType::kFLOAT:
+      return "kFLOAT";
+    case nvinfer1::DataType::kHALF:
+      return "kHALF";
+    case nvinfer1::DataType::kINT8:
+      return "kINT8";
+    case nvinfer1::DataType::kINT32:
+      return "kINT32";
+    default:
+      return "Invalid TRT data type";
+  }
+}
+
 string DebugString(const nvinfer1::Dims& dims) {
   string out = StrCat("nvinfer1::Dims(nbDims=", dims.nbDims, ", d=");
   for (int i = 0; i < dims.nbDims; ++i) {
@@ -222,16 +237,15 @@ string DebugString(const nvinfer1::Permutation& permutation, int len) {
 
 string DebugString(const nvinfer1::ITensor& tensor) {
   return StrCat("nvinfer1::ITensor(@", reinterpret_cast<uintptr_t>(&tensor),
-                ", shape=", DebugString(tensor.getDimensions()), ")");
+                ", name=", tensor.getName(),
+                ", dtype=", DebugString(tensor.getType()),
+                ", dims=", DebugString(tensor.getDimensions()), ")");
 }
 
-// Return whether or not the broadcast is feasible;
-bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l,
-                               const bool operand_l_is_tensor,
-                               const nvinfer1::Dims& operand_r,
-                               const bool operand_r_is_tensor,
-                               nvinfer1::Dims* operand_l_new_shape,
-                               nvinfer1::Dims* operand_r_new_shape) {
+Status Converter::GetTrtBroadcastShape(
+    const TRT_TensorOrWeights& operand_l, const TRT_TensorOrWeights& operand_r,
+    nvinfer1::Dims* operand_l_new_dims,
+    nvinfer1::Dims* operand_r_new_dims) const {
   // ***************************************************************************
   // TensorRT Elementwise op supports broadcast but requires both tensor to be
   // of Identical rank
@@ -256,52 +270,59 @@ bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l,
   // -> T: 1 1 1 -1 3 5 1
   // -> W: 1 1 1  1 3 5 1
   // ***************************************************************************
-  const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1;
-  const size_t element_size = sizeof(operand_l.d[0]);
-
-  // fill in dimensions
-  int l_s[max_nb_dims];
-  std::fill(l_s, l_s + max_nb_dims, 1);
-  int l_d = operand_l_is_tensor ? operand_l.nbDims + 1 : operand_l.nbDims;
-  int r_s[max_nb_dims];
-  std::fill(r_s, r_s + max_nb_dims, 1);
-  int r_d = operand_r_is_tensor ? operand_r.nbDims + 1 : operand_r.nbDims;
-
-  int max_d = std::max(l_d, r_d);
-  std::memcpy(l_s + max_d - operand_l.nbDims, operand_l.d,
-              operand_l.nbDims * element_size);
-  std::memcpy(r_s + max_d - operand_r.nbDims, operand_r.d,
-              operand_r.nbDims * element_size);
-
-  // set -1 for batch dimension, since batch size is not supposed to be
-  // broadcasted
-  if (operand_l_is_tensor) {
-    if (max_d != l_d) {  // if broadcast beyond batch dimension, fail
-      return false;
-    }
-    l_s[0] = -1;
-  }
-  if (operand_r_is_tensor) {
-    if (max_d != r_d) {  // if broadcast beyond batch dimension, fail
-      return false;
-    }
-    r_s[0] = -1;
+  if (!operand_l.is_tensor() && !operand_r.is_tensor()) {
+    return errors::InvalidArgument(
+        "Broadcasting requires at least one of the operands be tensors");
   }
 
-  // compare broadcast feasibility
-  for (int i = max_d - 1; i >= 0; i--) {
-    if ((l_s[i] != r_s[i]) && (l_s[i] != 1) && (r_s[i] != 1)) {
-      return false;
+  const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1;
+  auto compute_output_dims =
+      [max_nb_dims](const TRT_TensorOrWeights& input, int broadcast_num_dims,
+                    int* output_dims_array, nvinfer1::Dims* output_dims) {
+        const nvinfer1::Dims input_dims = input.GetTrtDims();
+        std::fill(output_dims_array, output_dims_array + max_nb_dims, 1);
+        std::copy(input_dims.d, input_dims.d + input_dims.nbDims,
+                  output_dims_array + broadcast_num_dims - input_dims.nbDims);
+        if (input.is_tensor()) {
+          const int true_input_dims = input_dims.nbDims + 1;
+          if (true_input_dims < broadcast_num_dims) {
+            return errors::InvalidArgument(
+                "Broadcasting beyond batch dimension is not supported ",
+                "(tensor #dims ", true_input_dims, " vs broadcast #dims ",
+                broadcast_num_dims, ")");
+          }
+          // Set the batch dimension to -1, since batch size is not supposed to
+          // be broadcasted.
+          output_dims_array[0] = -1;
+        }
+        // Copy to output dimensions (stripping the batch dimension).
+        output_dims->nbDims = broadcast_num_dims - 1;
+        std::copy(output_dims_array + 1, output_dims_array + broadcast_num_dims,
+                  output_dims->d);
+        return Status::OK();
+      };
+
+  // Compute the output dimensions.
+  const int broadcast_num_dims =
+      std::max(operand_l.GetTrtDims().nbDims + (operand_l.is_tensor() ? 1 : 0),
+               operand_r.GetTrtDims().nbDims + (operand_r.is_tensor() ? 1 : 0));
+  int output_l[max_nb_dims], output_r[max_nb_dims];
+  TF_RETURN_IF_ERROR(compute_output_dims(operand_l, broadcast_num_dims,
+                                         output_l, operand_l_new_dims));
+  TF_RETURN_IF_ERROR(compute_output_dims(operand_r, broadcast_num_dims,
+                                         output_r, operand_r_new_dims));
+
+  // Compare broadcast feasibility
+  for (int i = 0; i < broadcast_num_dims; ++i) {
+    if ((output_l[i] != output_r[i]) && (output_l[i] != 1) &&
+        (output_r[i] != 1)) {
+      return errors::InvalidArgument(
+          "Infeasible broadcast scheme (", "batch_dim: ", output_l[0], ", ",
+          DebugString(*operand_l_new_dims), " vs ", "batch_dim: ", output_r[0],
+          ", ", DebugString(*operand_r_new_dims), ")");
     }
   }
-
-  // output new TensorRT Dimension (stripping the batch dimension)
-  operand_l_new_shape->nbDims = max_d - 1;
-  std::memcpy(operand_l_new_shape->d, l_s + 1, (max_d - 1) * element_size);
-  operand_r_new_shape->nbDims = max_d - 1;
-  std::memcpy(operand_r_new_shape->d, r_s + 1, (max_d - 1) * element_size);
-
-  return true;
+  return Status::OK();
 }
 
 inline bool DimsEqual(const nvinfer1::Dims& dim_l,
@@ -515,8 +536,7 @@ nvinfer1::Dims TRT_TensorOrWeights::GetTrtDims() const {
 string TRT_TensorOrWeights::DebugString() const {
   string output = "TRT_TensorOrWeights(type=";
   if (is_tensor()) {
-    StrAppend(&output, "tensor @", reinterpret_cast<uintptr_t>(tensor()),
-              ", shape=", convert::DebugString(tensor()->getDimensions()),
+    StrAppend(&output, "tensor=", convert::DebugString(*tensor()),
               ", batch_size=", batch_size_);
   } else {
     StrAppend(&output, "weights=", weights_.DebugString());
@@ -779,8 +799,9 @@ Status TrtNodeValidator::ValidateNode(
     Status status = ConvertToTensorOrWeights(
         *pair.first, pair.second, graph_properties, &tensor_or_weights);
     if (!status.ok()) {
-      return errors::Internal("Failed to convert input with index ", i,
-                              " to a TRT_TensorOrWeights");
+      return errors::Internal(
+          "Failed to convert input with index ", i,
+          " to a TRT_TensorOrWeights: ", status.error_message());
     }
     inputs.push_back(tensor_or_weights);
   }
@@ -1033,8 +1054,9 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
   }
   if (can_check_shapes &&
       TrtDimsNumElements(input.GetTrtDims()) != TrtDimsNumElements(dims)) {
-    return tensorflow::errors::InvalidArgument(
-        "Reshape shapes are not compatible.");
+    return errors::InvalidArgument("Reshape shapes are not compatible (",
+                                   DebugString(input.GetTrtDims()), " vs ",
+                                   DebugString(dims), ")");
   }
 
   if (input.is_tensor()) {
@@ -1227,12 +1249,11 @@ TRT_ShapedWeights ConvertFP32ToFP16(TrtWeightStore* store,
 }
 
 // ****************************************************************************
-// Constant folding functions
-// TODO(jie): once optimizer kicks in, we should have done constant folding
-// there.
+// Constant folding functions for weights.
+// TODO(laigd): we should probably use eigen directly.
 // *****************************************************************************
 struct LambdaFactory {
-  enum class OP_CATEGORY : int { RSQRT = 0, NEG, ADD, MUL, SUB, RECIP };
+  enum class OP_CATEGORY : int { RSQRT = 0, NEG, RECIP };
   OP_CATEGORY op;
 
   template <typename T>
@@ -1247,7 +1268,7 @@ struct LambdaFactory {
       case OP_CATEGORY::RECIP:
         return [](T t) -> T { return 1.0 / t; };
       default:
-        VLOG(2) << "Not supported op for unary: " << static_cast<int>(op);
+        LOG(ERROR) << "Not supported op for unary: " << static_cast<int>(op);
         return nullptr;
     }
   }
@@ -1258,15 +1279,18 @@ std::function<Eigen::half(Eigen::half)> LambdaFactory::unary<Eigen::half>() {
   switch (op) {
     case OP_CATEGORY::RSQRT: {
       VLOG(2) << "RSQRT GETS DONE";
-      return [](Eigen::half t) -> Eigen::half {
+      return [](Eigen::half t) {
         return Eigen::half(1.0 / sqrt(static_cast<float>(t)));
       };
     }
     case OP_CATEGORY::NEG:
-      return [](Eigen::half t) -> Eigen::half { return -t; };
-    // TODO(aaroey): can we support RECIP?
+      return [](Eigen::half t) { return -t; };
+    case OP_CATEGORY::RECIP:
+      return [](Eigen::half t) {
+        return Eigen::half(1.0 / static_cast<float>(t));
+      };
     default:
-      VLOG(2) << "Not supported op for unary: " << static_cast<int>(op);
+      LOG(ERROR) << "Not supported op for unary: " << static_cast<int>(op);
       return nullptr;
   }
 }
@@ -1298,50 +1322,48 @@ tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
   return tensorflow::Status::OK();
 }
 
+// If swapped_inputs is false, 'tensor' is the left operand and 'weights' is the
+// right operand. If swapped_inputs is true, those two are swapped.
+//
 // TODO(jie): broadcast is needed yet not implemented.
-// Only implemented channel wise for the time being
-tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
-                                        const nvinfer1::ITensor* tensor,
-                                        TRT_ShapedWeights weights,
-                                        bool swapped_inputs) {
+// Only implemented channel wise for the time being.
+Status BinaryTensorOpWeight(OpConverterParams* params,
+                            const nvinfer1::ITensor* tensor,
+                            TRT_ShapedWeights weights, bool swapped_inputs) {
+  static const std::unordered_set<string> supported_ops = {"Sub", "Add", "Mul",
+                                                           "Div", "RealDiv"};
   const auto& node_def = params->node_def;
-  // tensor is the left operand while weights is the right operand;
-  // when swapped_inputs set to true, those two are swapped.
-  // TODO(aaroey): use a set.
-  if (node_def.op() != "Sub" && node_def.op() != "Add" &&
-      node_def.op() != "Mul" && node_def.op() != "Div" &&
-      node_def.op() != "RealDiv") {
-    return tensorflow::errors::Unimplemented(
-        "op not supported: " + node_def.op() + ", at: " + node_def.name());
+  if (!supported_ops.count(node_def.op())) {
+    return errors::Unimplemented(node_def.op(), " is not supported, at ",
+                                 node_def.name());
   }
 
-  // Check type consistency
-  nvinfer1::DataType ttype;
-  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype));
+  // Check type consistency.
+  nvinfer1::DataType trt_dtype;
+  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &trt_dtype));
 
-  // Check scale mode
+  // Check scale mode.
   auto dims_w = weights.shape_;
-  auto dims_t = tensor->getDimensions();
+  const auto dims_t = tensor->getDimensions();
 
   // TODO(jie): addScale checks for input tensor dimension
   if (dims_t.nbDims != 3) {
-    return tensorflow::errors::InvalidArgument(
-        "addScale requires tensor with rank 3, " + node_def.name());
+    return errors::InvalidArgument("addScale requires tensor with rank 3, at ",
+                                   node_def.name());
   }
 
-  // default to element-wise
+  // Default to element-wise
   auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
 
   // TODO(jie): maybe use a permutation instead to support more cases;
-  bool permutation_flag = false;
+  bool need_to_permute = false;
 
   if (weights.count() == 1) {
-    VLOG(2) << "UNIFORM";
     scale_mode = nvinfer1::ScaleMode::kUNIFORM;
   } else {
-    // no broadcasting on Batch dimension;
-    VLOG(2) << "WEIGHTS DIM: " << dims_w.nbDims
-            << " tensor DIM: " << dims_t.nbDims;
+    VLOG(2) << "weights dims: " << DebugString(dims_w)
+            << "; tensor dims: " << DebugString(dims_t);
+    // Make sure no broadcasting on batch dimension.
     if (dims_w.nbDims == dims_t.nbDims + 1) {
       if (dims_w.d[0] == 1) {
         for (int i = 1; i < dims_w.nbDims; i++) {
@@ -1349,72 +1371,70 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
         }
         dims_w.nbDims--;
       } else {
-        return tensorflow::errors::InvalidArgument(
-            "Binary op cannot operate on batch, " + node_def.name());
+        return errors::InvalidArgument("Binary op cannot operate on batch, at ",
+                                       node_def.name());
       }
     }
 
     if (dims_w.nbDims == dims_t.nbDims && dims_w.d[0] == dims_t.d[0]) {
       scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
-      // default is element;
+      // Default is element-wise
       for (int i = 1; i < dims_w.nbDims; i++) {
         if (dims_w.d[i] != dims_t.d[i]) {
-          // if dimension does not match, switch back to channel;
-          VLOG(2) << "channel";
+          // If dimension does not match, switch back to per-channel
           scale_mode = nvinfer1::ScaleMode::kCHANNEL;
           break;
         }
       }
-      // if channel as candidate, validate it
+      // If the mode is per-channel, since channel dimension is assumed to be
+      // the third to last dimension, we need to make sure all other dimensions
+      // have size 1.
       if (scale_mode == nvinfer1::ScaleMode::kCHANNEL) {
         for (int i = 1; i < dims_w.nbDims; i++) {
           if (dims_w.d[i] != 1)
-            return tensorflow::errors::InvalidArgument(
-                "Weight shape not compatible at, " + node_def.name());
+            return errors::InvalidArgument(
+                "Weight dims not compatible for channel-wise broadcast at ",
+                node_def.name());
         }
-      } else {
-        VLOG(2) << "elementwise";
       }
     } else if (dims_w.nbDims == 1 &&
                dims_w.d[0] == dims_t.d[dims_t.nbDims - 1]) {
-      // channel wise and broadcast required;
-      permutation_flag = true;
+      // Channel wise and broadcast required. We compare the last dimension of
+      // the tensor shape because of tensorflow default broadcasting rules.
+      need_to_permute = true;
       scale_mode = nvinfer1::ScaleMode::kCHANNEL;
     } else {
-      return tensorflow::errors::InvalidArgument(
-          "Weight shape not compatible at, " + node_def.name());
+      return errors::InvalidArgument("Weight dims not compatible at ",
+                                     node_def.name());
     }
   }
+  // TODO(laigd): we should add validation_only support in TransposeTensor() and
+  // PrepareTensorForShape().
+  if (params->validation_only) return Status::OK();
 
-  // transpose last dimension
+  // Transpose last dimension.
   std::vector<int> permutation(dims_t.nbDims + 1);
-  if (permutation_flag) {
-    if (scale_mode == nvinfer1::ScaleMode::kCHANNEL && dims_t.nbDims > 1) {
-      // we swap the last dimension into channel for trt.
-      // because of tensorflow default broadcasting rules.
-      for (int i = 0; i < static_cast<int>(permutation.size()); i++) {
-        permutation[i] = i;
-      }
-      permutation[1] = dims_t.nbDims;
-      permutation[dims_t.nbDims] = 1;
-      TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
-          const_cast<nvinfer1::ITensor*>(tensor), permutation, &tensor));
-    } else {
-      return tensorflow::errors::InvalidArgument(
-          "Transpose cannot be applied, " + node_def.name());
-    }
+  if (need_to_permute) {
+    // We swap the last dimension into channel for trt, because of tensorflow
+    // default broadcasting rules.
+    for (int i = 0; i < static_cast<int>(permutation.size()); i++) {
+      permutation[i] = i;
+    }
+    permutation[1] = dims_t.nbDims;
+    permutation[dims_t.nbDims] = 1;
+    TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
+        const_cast<nvinfer1::ITensor*>(tensor), permutation, &tensor));
   }
 
   if (params->converter->precision_mode() == FP16MODE) {
     weights = ConvertFP32ToFP16(params->weight_store, weights);
   }
 
-  // prepare weights
+  // Prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
   TRT_ShapedWeights power_weights(weights.type_);
 
-  // Maybe I should do a switch
   if (node_def.op() == "Sub") {
     if (swapped_inputs) {
       shift_weights = weights;
@@ -1475,8 +1495,8 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
   } else if (node_def.op() == "Add") {
     shift_weights = weights;
   } else {
-    return tensorflow::errors::Unimplemented("Binary op not supported: " +
-                                             node_def.op());
+    // This should not happen.
+    return errors::Unimplemented("Binary op not supported at ", node_def.op());
   }
 
   nvinfer1::IScaleLayer* layer = params->converter->network()->addScale(
@@ -1486,8 +1506,8 @@ tensorflow::Status BinaryTensorOpWeight(OpConverterParams* params,
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
 
   const nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  // transpose back dimension
-  if (permutation_flag) {
+  // Transpose back dimension
+  if (need_to_permute) {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(output_tensor), permutation,
         &output_tensor));
@@ -1621,9 +1641,9 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params,
                                            params->node_def.name());
 }
 
-tensorflow::Status BinaryTensorOpTensor(OpConverterParams* params,
-                                        const TRT_TensorOrWeights& operand_l,
-                                        const TRT_TensorOrWeights& operand_r) {
+Status BinaryTensorOpTensor(OpConverterParams* params,
+                            const TRT_TensorOrWeights& operand_l,
+                            const TRT_TensorOrWeights& operand_r) {
   const auto& node_def = params->node_def;
   static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
       {"Add", nvinfer1::ElementWiseOperation::kSUM},
@@ -1634,50 +1654,52 @@ tensorflow::Status BinaryTensorOpTensor(OpConverterParams* params,
       {"Minimum", nvinfer1::ElementWiseOperation::kMIN},
       {"Maximum", nvinfer1::ElementWiseOperation::kMAX},
   };
+  auto op_pair = ops.find(node_def.op());
+  if (op_pair == ops.end()) {
+    return errors::Unimplemented("Binary op ", node_def.op(),
+                                 " not supported at: ", node_def.name());
+  }
 
-  const nvinfer1::ITensor* tensor_l;
-  const nvinfer1::ITensor* tensor_r;
-
-  nvinfer1::Dims dim_l;
-  nvinfer1::Dims dim_r;
-
-  if (!TensorRTGetBroadcastShape(operand_l.GetTrtDims(), operand_l.is_tensor(),
-                                 operand_r.GetTrtDims(), operand_r.is_tensor(),
-                                 &dim_l, &dim_r)) {
-    return tensorflow::errors::InvalidArgument(
-        "Binary op broadcast scheme not supported by TensorRT op: " +
-        node_def.op() + ", at: " + node_def.name());
+  nvinfer1::Dims broadcasted_dims_l, broadcasted_dims_r;
+  Status status = params->converter->GetTrtBroadcastShape(
+      operand_l, operand_r, &broadcasted_dims_l, &broadcasted_dims_r);
+  if (!status.ok()) {
+    return errors::InvalidArgument(
+        "Unsupported binary op broadcast scheme for op ", node_def.name(), ": ",
+        status.error_message());
   }
+  if (params->validation_only) return Status::OK();
 
-  TF_RETURN_IF_ERROR(
-      params->converter->PrepareTensorForShape(operand_l, dim_l, &tensor_l));
-  TF_RETURN_IF_ERROR(
-      params->converter->PrepareTensorForShape(operand_r, dim_r, &tensor_r));
+  const nvinfer1::ITensor* tensor_l = nullptr;
+  const nvinfer1::ITensor* tensor_r = nullptr;
+  status = params->converter->PrepareTensorForShape(
+      operand_l, broadcasted_dims_l, &tensor_l);
+  if (status.ok()) {
+    status = params->converter->PrepareTensorForShape(
+        operand_r, broadcasted_dims_r, &tensor_r);
+  }
+  if (!status.ok()) {
+    return errors::Internal("Failed to convert binary op ", node_def.name(),
+                            ": ", status.error_message());
+  }
 
-  // get trt type & shape
+  // Check type consistency.
   TFAttrs attrs(node_def);
-  // maybe this part has to be moved into the block of rsqrt later
   nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
+  TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype)
+      << DebugString(tensor_l->getType()) << " vs " << DebugString(dtype);
+  TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype)
+      << DebugString(tensor_r->getType()) << " vs " << DebugString(dtype);
 
-  // check type consistency
-  TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype);
-  TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype);
-  auto op_pair = ops.find(node_def.op());
-  if (op_pair == ops.end()) {
-    return tensorflow::errors::Unimplemented(
-        "binary op: ", node_def.op(), " not supported at: ", node_def.name());
-  }
-
+  // Add ElementWise layer.
   nvinfer1::IElementWiseLayer* layer =
       params->converter->network()->addElementWise(
-          // TODO(aaroey): will tensor_l/tensor_r get modified?
           *const_cast<nvinfer1::ITensor*>(tensor_l),
           *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
-
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
-  // pass the output
+  // Pass the output
   params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
@@ -2135,9 +2157,9 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
     }
     permutation.order[0] = channel_index;
     permutation.order[channel_index] = 0;
+    VLOG(1) << "ConvertBiasAdd permutation: "
+            << DebugString(permutation, original_dims.nbDims);
   }
-  VLOG(1) << "ConvertBiasAdd permutation: "
-          << DebugString(permutation, original_dims.nbDims);
 
   // TensorRT addScale requires input to be of rank 3, we need to apply
   // transpose as well as reshape.
@@ -2369,18 +2391,17 @@ tensorflow::Status ConvertIdentity(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertBinary(OpConverterParams* params) {
+Status ConvertBinary(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   if (inputs.size() != 2) {
-    return tensorflow::errors::FailedPrecondition(
-        "Binary ops require two tensor input, at ", node_def.name());
+    return errors::InvalidArgument("Binary ops require two inputs, at ",
+                                   node_def.name());
   }
 
   // Constant folding should have been done by TensorFlow
-
   if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) {
-    return tensorflow::errors::Unimplemented(
+    return errors::Unimplemented(
         "Constant folding is falled back to TensorFlow, binary op received "
         "both input as constant at: ",
         node_def.name());
@@ -2393,11 +2414,11 @@ tensorflow::Status ConvertBinary(OpConverterParams* params) {
   // IScaleLayer are when the layer performs both a shift and a scale, which we
   // don't do except for convolutions.
   //
-  // Try to convert into Scale layer first (for better performance)
+  // Try to convert into Scale layer first (for better performance).
   // Since scale layer supports restricted broadcast policy and op types, we
   // allow failure and try to handle it through Elementwise op
-  // (BinaryTensorOpTensor)
-  Status status = tensorflow::Status::OK();
+  // (BinaryTensorOpTensor).
+  Status status = Status::OK();
   if (inputs.at(0).is_tensor() && inputs.at(1).is_weights()) {
     status = BinaryTensorOpWeight(params, inputs.at(0).tensor(),
                                   inputs.at(1).weights(), false);
@@ -2405,7 +2426,10 @@ tensorflow::Status ConvertBinary(OpConverterParams* params) {
     status = BinaryTensorOpWeight(params, inputs.at(1).tensor(),
                                   inputs.at(0).weights(), true);
   }
+  // If both input are tensors, or one of them is weights but the conversion
+  // above failed, try the conversion using BinaryTensorOpTensor.
   if ((inputs.at(0).is_tensor() && inputs.at(1).is_tensor()) || !status.ok()) {
+    if (!status.ok()) VLOG(1) << status;
     status = BinaryTensorOpTensor(params, inputs.at(0), inputs.at(1));
   }
   return status;
@@ -3050,48 +3074,49 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
-void TrtNodeValidator::RegisterOpValidators() {
+static void RegisterValidatableOpConverters(
+    std::unordered_map<string, OpConverter>* registration) {
   // TODO(laigd): support all op types.
-  op_validators_["BiasAdd"] = ConvertBiasAdd;
-  op_validators_["Const"] = ConvertConst;
-  op_validators_["Transpose"] = ConvertTranspose;
-  op_validators_["Reshape"] = ConvertReshape;
-  op_validators_["MatMul"] = ConvertMatMul;
+  (*registration)["BiasAdd"] = ConvertBiasAdd;
+  (*registration)["Const"] = ConvertConst;
+  (*registration)["Transpose"] = ConvertTranspose;
+  (*registration)["Reshape"] = ConvertReshape;
+  (*registration)["MatMul"] = ConvertMatMul;
+  (*registration)["Relu6"] = ConvertRelu6;
 
-  op_validators_["Relu6"] = ConvertRelu6;
+  for (auto quantization_op_type :
+       {"QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3",
+        "FakeQuantWithMinMaxVars", "FakeQuantWithMinMaxArgs"}) {
+    (*registration)[quantization_op_type] = ConvertQuantize;
+  }
+  for (auto binary_op_type :
+       {"Add", "Mul", "Sub", "Div", "RealDiv", "Maximum", "Minimum"}) {
+    (*registration)[binary_op_type] = ConvertBinary;
+  }
+}
 
-  op_validators_["QuantizeAndDequantizeV2"] = ConvertQuantize;
-  op_validators_["QuantizeAndDequantizeV3"] = ConvertQuantize;
-  op_validators_["FakeQuantWithMinMaxVars"] = ConvertQuantize;
-  op_validators_["FakeQuantWithMinMaxArgs"] = ConvertQuantize;
+void TrtNodeValidator::RegisterOpValidators() {
+  RegisterValidatableOpConverters(&op_validators_);
 }
 
 void Converter::RegisterOpConverters() {
-  // vgg_16 slim implementation
+  RegisterValidatableOpConverters(&op_registry_);
+
   op_registry_["Conv2D"] = ConvertConv2D;
   op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
   op_registry_["Relu"] = ConvertActivation;
   op_registry_["MaxPool"] = ConvertPool;
   op_registry_["AvgPool"] = ConvertPool;
-  op_registry_["BiasAdd"] = ConvertBiasAdd;
-  op_registry_["Const"] = ConvertConst;
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
   op_registry_["Snapshot"] = ConvertIdentity;  // Snapshot should be removed
 
-  // resnet_50_v1 slim implementation
-  op_registry_["Add"] = ConvertBinary;
-  op_registry_["Mul"] = ConvertBinary;
-  op_registry_["Sub"] = ConvertBinary;
   op_registry_["Pad"] = ConvertPad;
 
   op_registry_["ConcatV2"] = ConvertConcat;
   op_registry_["FusedBatchNorm"] = ConvertFusedBatchNorm;
   op_registry_["FusedBatchNormV2"] = ConvertFusedBatchNorm;
 
-  op_registry_["Div"] = ConvertBinary;
-  op_registry_["RealDiv"] = ConvertBinary;
-
   op_registry_["Rsqrt"] = ConvertUnary;
   op_registry_["Reciprocal"] = ConvertUnary;
   op_registry_["Exp"] = ConvertUnary;
@@ -3100,18 +3125,12 @@ void Converter::RegisterOpConverters() {
   op_registry_["Abs"] = ConvertUnary;
   op_registry_["Neg"] = ConvertUnary;
 
-  op_registry_["Transpose"] = ConvertTranspose;
-  op_registry_["Reshape"] = ConvertReshape;
-
   op_registry_["Sum"] = ConvertReduce;
   op_registry_["Prod"] = ConvertReduce;
   op_registry_["Max"] = ConvertReduce;
   op_registry_["Min"] = ConvertReduce;
   op_registry_["Mean"] = ConvertReduce;
-  op_registry_["Maximum"] = ConvertBinary;
-  op_registry_["Minimum"] = ConvertBinary;
   op_registry_["Softmax"] = ConvertSoftmax;
-  op_registry_["MatMul"] = ConvertMatMul;
   op_registry_["BatchMatMul"] = ConvertBatchMatMul;
   op_registry_["TopKV2"] = ConvertTopK;
   op_registry_["Relu6"] = ConvertRelu6;
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 50100a142f..54e19b7395 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -462,6 +462,13 @@ class Converter {
                                const nvinfer1::Dims& dims,
                                const nvinfer1::ITensor** tensor);
 
+  // Return OK if the broadcast scheme is supported and compute the shapes after
+  // broadcasting.
+  Status GetTrtBroadcastShape(const TRT_TensorOrWeights& operand_l,
+                              const TRT_TensorOrWeights& operand_r,
+                              nvinfer1::Dims* operand_l_new_dims,
+                              nvinfer1::Dims* operand_r_new_dims) const;
+
  private:
   // Verify the provided batch_size is consistent with batch_size_ and update it
   // if necessary.
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index b46a7c6c8a..603c4f7b5e 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -52,6 +52,7 @@ namespace convert {
 
 using ::tensorflow::strings::StrCat;
 using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
 
 // TODO(laigd): put this into some test utils file.
 void ExpectStatus(Status status, error::Code code = error::OK,
@@ -73,6 +74,32 @@ nvinfer1::Dims GetTestDims(const std::vector<int>& d) {
   return dims;
 }
 
+nvinfer1::DataType TfDataTypeToTrt(DataType tf_dtype) {
+  switch (tf_dtype) {
+    case DT_FLOAT:
+      return nvinfer1::DataType::kFLOAT;
+    case DT_HALF:
+      return nvinfer1::DataType::kHALF;
+    case DT_INT32:
+      return nvinfer1::DataType::kINT32;
+    default:
+      QCHECK(false) << "Unexpected data type " << DataTypeString(tf_dtype);
+  }
+}
+
+DataType TrtDataTypeToTf(nvinfer1::DataType trt_dtype) {
+  switch (trt_dtype) {
+    case nvinfer1::DataType::kFLOAT:
+      return DT_FLOAT;
+    case nvinfer1::DataType::kHALF:
+      return DT_HALF;
+    case nvinfer1::DataType::kINT32:
+      return DT_INT32;
+    default:
+      QCHECK(false) << "Unexpected data type " << static_cast<int>(trt_dtype);
+  }
+}
+
 NodeDef MakeNodeDef(const string& name, const string& op,
                     const std::vector<string>& inputs) {
   NodeDef node_def;
@@ -115,6 +142,15 @@ bool TrtDimsEqualsArray(const std::vector<int>& lhs,
   return TrtDimsEquals(GetTestDims(lhs), rhs);
 }
 
+// TODO(laigd): define a parameterized matcher that can compare against the
+// vector.
+void ExpectTrtDimsEqualsArray(const std::vector<int>& lhs,
+                              const nvinfer1::Dims& rhs) {
+  EXPECT_TRUE(TrtDimsEqualsArray(lhs, rhs))
+      << "expected: " << DebugString(GetTestDims(lhs)) << "\n"
+      << "  actual: " << DebugString(rhs);
+}
+
 bool TrtShapedWeightsEquals(const TRT_ShapedWeights& lhs,
                             const TRT_ShapedWeights& rhs) {
   return TrtDimsEquals(lhs.shape_, rhs.shape_) && lhs.type_ == rhs.type_ &&
@@ -125,8 +161,7 @@ template <typename T>
 void ValidateWeights(const TRT_ShapedWeights& weights,
                      const std::vector<int>& expected_dims,
                      const std::vector<T>& expected_value) {
-  EXPECT_TRUE(TrtDimsEqualsArray(expected_dims, weights.shape_))
-      << weights.DebugString();
+  ExpectTrtDimsEqualsArray(expected_dims, weights.shape_);
   ASSERT_EQ(expected_value.size(), weights.count()) << weights.DebugString();
   const T* actual_values = static_cast<const T*>(weights.GetValues());
   for (int i = 0; i < expected_value.size(); ++i) {
@@ -276,9 +311,7 @@ TEST(TRT_TensorOrWeights_Test, Basic) {
           EXPECT_EQ(1, ptr->batch_size());
         }
         EXPECT_EQ(&itensor, ptr->tensor());
-        EXPECT_TRUE(TrtDimsEqualsArray({1}, ptr->GetTrtDims()))
-            << "- expected: " << DebugString(dims)
-            << "\n        vs\n-   actual: " << DebugString(ptr->GetTrtDims());
+        ExpectTrtDimsEqualsArray({1}, ptr->GetTrtDims());
       }
     }
   }
@@ -297,9 +330,7 @@ TEST(TRT_TensorOrWeights_Test, Basic) {
       EXPECT_EQ(false, ptr->is_weights());
       EXPECT_EQ(1, ptr->batch_size());
       EXPECT_NE(nullptr, ptr->tensor());
-      EXPECT_TRUE(TrtDimsEqualsArray({1}, ptr->GetTrtDims()))
-          << "- expected: " << DebugString(dims)
-          << "\n        vs\n-   actual: " << DebugString(ptr->GetTrtDims());
+      ExpectTrtDimsEqualsArray({1}, ptr->GetTrtDims());
     }
   }
   // Test constructor with TRT_ShapedWeights argument.
@@ -316,9 +347,7 @@ TEST(TRT_TensorOrWeights_Test, Basic) {
 
       nvinfer1::Dims dims;
       dims.nbDims = 0;
-      EXPECT_TRUE(TrtDimsEqualsArray({}, ptr->GetTrtDims()))
-          << "- expected: " << DebugString(dims)
-          << "\n        vs\n-   actual: " << DebugString(ptr->GetTrtDims());
+      ExpectTrtDimsEqualsArray({}, ptr->GetTrtDims());
     }
   }
 }
@@ -395,9 +424,7 @@ TEST_F(ValidatorTest, ConvertToTensorOrWeights) {
     EXPECT_EQ(true, output.is_tensor());
     EXPECT_EQ(batch_size, output.batch_size());
     EXPECT_NE(nullptr, output.tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({non_batch_dim}, output.GetTrtDims()))
-        << "- expected: {" << non_batch_dim << "} \n        vs\n"
-        << "-   actual: " << DebugString(output.GetTrtDims());
+    ExpectTrtDimsEqualsArray({non_batch_dim}, output.GetTrtDims());
   }
 }
 
@@ -548,9 +575,9 @@ TEST_F(ConverterTest, AddAndGetInputs) {
   EXPECT_EQ(nvinfer1::DataType::kFLOAT, inputs[0].tensor()->getType());
   EXPECT_EQ(nvinfer1::DataType::kINT32, inputs[2].tensor()->getType());
   EXPECT_EQ(nvinfer1::DataType::kHALF, inputs[3].tensor()->getType());
-  EXPECT_TRUE(TrtDimsEqualsArray({1}, inputs[0].tensor()->getDimensions()));
-  EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, inputs[2].tensor()->getDimensions()));
-  EXPECT_TRUE(TrtDimsEqualsArray({5, 3}, inputs[3].tensor()->getDimensions()));
+  ExpectTrtDimsEqualsArray({1}, inputs[0].tensor()->getDimensions());
+  ExpectTrtDimsEqualsArray({2, 3}, inputs[2].tensor()->getDimensions());
+  ExpectTrtDimsEqualsArray({5, 3}, inputs[3].tensor()->getDimensions());
 }
 
 TEST_F(ConverterTest, RenameAndMarkOutputTensors) {
@@ -596,7 +623,7 @@ TEST_F(ConverterTest, RenameAndMarkOutputTensors) {
       {{"my_op", "my_output"}, {"my_op:1", "my_output_1"}}));
   EXPECT_EQ(2, output_tensors.size());
   for (auto output_tensor : output_tensors) {
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 1}, output_tensor->getDimensions()));
+    ExpectTrtDimsEqualsArray({2, 1}, output_tensor->getDimensions());
   }
   EXPECT_EQ("my_output", string(output_tensors[0]->getName()));
   EXPECT_EQ("my_output_1", string(output_tensors[1]->getName()));
@@ -621,8 +648,7 @@ TEST_F(ConverterTest, TransposeTensor) {
   // OK.
   TF_EXPECT_OK(
       converter_->TransposeTensor(input_tensor, {0, 3, 1, 2}, &output_tensor));
-  EXPECT_TRUE(TrtDimsEqualsArray({5, 2, 3}, output_tensor->getDimensions()))
-      << DebugString(*output_tensor);
+  ExpectTrtDimsEqualsArray({5, 2, 3}, output_tensor->getDimensions());
 }
 
 TEST_F(ConverterTest, PrepareTensorForShape_Tensor) {
@@ -634,7 +660,7 @@ TEST_F(ConverterTest, PrepareTensorForShape_Tensor) {
   // Shape size doesn't match.
   ExpectStatus(converter_->PrepareTensorForShape(tw, GetTestDims({2, 3, 6}),
                                                  &output_tensor),
-               error::INVALID_ARGUMENT, "Reshape shapes are not compatible.");
+               error::INVALID_ARGUMENT, "Reshape shapes are not compatible");
 
   // TODO(aaroey): we should check the case where uninferred dimensions are not
   // an exact divisor of input dim ensions, e.g. for dims {-1, 7}.
@@ -642,14 +668,12 @@ TEST_F(ConverterTest, PrepareTensorForShape_Tensor) {
   // Infer shape, ok.
   TF_EXPECT_OK(converter_->PrepareTensorForShape(tw, GetTestDims({-1, 2}),
                                                  &output_tensor));
-  EXPECT_TRUE(TrtDimsEqualsArray({15, 2}, output_tensor->getDimensions()))
-      << DebugString(*output_tensor);
+  ExpectTrtDimsEqualsArray({15, 2}, output_tensor->getDimensions());
 
   // Regular shape.
   TF_EXPECT_OK(converter_->PrepareTensorForShape(tw, GetTestDims({10, 3}),
                                                  &output_tensor));
-  EXPECT_TRUE(TrtDimsEqualsArray({10, 3}, output_tensor->getDimensions()))
-      << DebugString(*output_tensor);
+  ExpectTrtDimsEqualsArray({10, 3}, output_tensor->getDimensions());
 }
 
 TEST_F(ConverterTest, PrepareTensorForShape_Weights) {
@@ -659,8 +683,7 @@ TEST_F(ConverterTest, PrepareTensorForShape_Weights) {
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_EXPECT_OK(converter_->PrepareTensorForShape(tw, GetTestDims({10, 3}),
                                                  &output_tensor));
-  EXPECT_TRUE(TrtDimsEqualsArray({10, 3}, output_tensor->getDimensions()))
-      << DebugString(*output_tensor);
+  ExpectTrtDimsEqualsArray({10, 3}, output_tensor->getDimensions());
 }
 
 TEST_F(ConverterTest, MaybeUpdateBatchSize) {
@@ -782,6 +805,96 @@ TEST_F(ConverterTest, PropagateQuantizationRanges) {
   EXPECT_EQ(ranges.count(&not_infer), 0);
 }
 
+TEST_F(ConverterTest, GetTrtBroadcastShape) {
+  const bool kIsTensor = true;
+  const bool kIsNotTensor = false;
+  auto symmetric_test = [this](const std::vector<int>& operand_1_shape,
+                               const std::vector<int>& operand_2_shape,
+                               const bool operand_1_is_tensor,
+                               const bool operand_2_is_tensor,
+                               const std::vector<int>& expected_operand_1_shape,
+                               const std::vector<int>& expected_operand_2_shape,
+                               error::Code expected_code = error::OK,
+                               const char* expected_error_msg_substr = nullptr,
+                               const int operand_1_batch_size = -1,
+                               const int operand_2_batch_size = -1) {
+    auto create_tensor_or_weights = [](const std::vector<int>& shape,
+                                       bool is_tensor, int batch_size = -1) {
+      if (is_tensor) {
+        return TRT_TensorOrWeights{nvinfer1::DataType::kFLOAT,
+                                   GetTestDims(shape), batch_size};
+      }
+      TRT_ShapedWeights weights;
+      weights.shape_ = GetTestDims(shape);
+      return TRT_TensorOrWeights(weights);
+    };
+
+    nvinfer1::Dims operand_1_new_dims, operand_2_new_dims;
+    TRT_TensorOrWeights operand_1 = create_tensor_or_weights(
+        operand_1_shape, operand_1_is_tensor, operand_1_batch_size);
+    TRT_TensorOrWeights operand_2 = create_tensor_or_weights(
+        operand_2_shape, operand_2_is_tensor, operand_2_batch_size);
+
+    // operand_1 broadcast operand_2
+    ExpectStatus(
+        this->converter_->GetTrtBroadcastShape(
+            operand_1, operand_2, &operand_1_new_dims, &operand_2_new_dims),
+        expected_code, expected_error_msg_substr);
+    if (expected_code == error::OK) {
+      ExpectTrtDimsEqualsArray(expected_operand_1_shape, operand_1_new_dims);
+      ExpectTrtDimsEqualsArray(expected_operand_2_shape, operand_2_new_dims);
+    }
+    // operand_2 broadcast operand_1
+    ExpectStatus(
+        this->converter_->GetTrtBroadcastShape(
+            operand_2, operand_1, &operand_2_new_dims, &operand_1_new_dims),
+        expected_code, expected_error_msg_substr);
+    if (expected_code == error::OK) {
+      ExpectTrtDimsEqualsArray(expected_operand_1_shape, operand_1_new_dims);
+      ExpectTrtDimsEqualsArray(expected_operand_2_shape, operand_2_new_dims);
+    }
+  };
+
+  // Both inputs are weights.
+  symmetric_test(
+      {1}, {1}, kIsNotTensor, kIsNotTensor, {}, {}, error::INVALID_ARGUMENT,
+      "Broadcasting requires at least one of the operands be tensors");
+
+  // One tensor and one weights.
+  symmetric_test({1, 1, 1}, {2}, kIsTensor, kIsNotTensor, {1, 1, 1}, {1, 1, 2});
+  symmetric_test({1, 1, 2}, {2}, kIsTensor, kIsNotTensor, {1, 1, 2}, {1, 1, 2});
+  symmetric_test({1, 3, 2}, {1}, kIsTensor, kIsNotTensor, {1, 3, 2}, {1, 1, 1});
+  symmetric_test({1, 1, 1}, {2, 3}, kIsTensor, kIsNotTensor, {1, 1, 1},
+                 {1, 2, 3});
+  symmetric_test({1, 1, 1}, {2, 3, 4}, kIsTensor, kIsNotTensor, {1, 1, 1},
+                 {2, 3, 4});
+  symmetric_test({1, 1, 1}, {1, 2, 3, 4}, kIsTensor, kIsNotTensor, {1, 1, 1},
+                 {2, 3, 4});
+  symmetric_test({1, 3, 4}, {1, 2, 1, 4}, kIsTensor, kIsNotTensor, {1, 3, 4},
+                 {2, 1, 4});
+  symmetric_test({1, 1, 1}, {2, 1, 1, 1}, kIsTensor, kIsNotTensor, {}, {},
+                 error::INVALID_ARGUMENT, "Infeasible broadcast scheme");
+  symmetric_test({1, 1, 1}, {2, 1, 1, 1}, kIsTensor, kIsNotTensor, {}, {},
+                 error::INVALID_ARGUMENT, "Infeasible broadcast scheme",
+                 /*operand_1_batch_size=*/2);
+  symmetric_test({1, 1, 1}, {1, 1, 1, 1, 1}, kIsTensor, kIsNotTensor, {}, {},
+                 error::INVALID_ARGUMENT,
+                 "Broadcasting beyond batch dimension is not supported "
+                 "(tensor #dims 4 vs broadcast #dims 5)");
+
+  // Both inputs are tensors.
+  symmetric_test({1, 1, 1}, {1, 1}, kIsTensor, kIsTensor, {}, {},
+                 error::INVALID_ARGUMENT,
+                 "Broadcasting beyond batch dimension is not supported "
+                 "(tensor #dims 3 vs broadcast #dims 4)");
+  symmetric_test({1, 3, 4}, {2, 1, 4}, kIsTensor, kIsTensor, {1, 3, 4},
+                 {2, 1, 4});
+  symmetric_test({1, 1, 1}, {1, 1, 1, 1}, kIsTensor, kIsTensor, {}, {},
+                 error::INVALID_ARGUMENT,
+                 "Broadcasting beyond batch dimension is not supported "
+                 "(tensor #dims 4 vs broadcast #dims 5)");
+}
+
 // Class to test various op converters, using both a TrtNodeValidator and
 // Converter.
 class OpConverterTest : public ::testing::Test {
@@ -820,8 +933,11 @@ class OpConverterTest : public ::testing::Test {
   }
 
   // TODO(laigd): test fp16 and int8 support.
-  void BuildAndRun(const char* input_name, const std::vector<float>& input_data,
-                   const char* output_name, std::vector<float>* output_data) {
+  template <typename T>
+  void BuildAndRun(
+      const std::vector<std::pair<const char*, const std::vector<T>>>&
+          input_data,
+      const char* output_name, std::vector<T>* output_data) {
     // Mark the output tensor as TRT engine output.
     TF_EXPECT_OK(converter_->RenameAndMarkOutputTensors(
         {{string(output_name), string(output_name)}}));
@@ -832,25 +948,33 @@ class OpConverterTest : public ::testing::Test {
     CHECK_NOTNULL(engine_.get());
 
     // Execute the TRT engine.
-    const int input_size = input_data.size() * sizeof(float);
-    const int output_size = output_data->size() * sizeof(float);
-    const int input_index = engine_->getBindingIndex(input_name);
-    const int output_index = engine_->getBindingIndex(output_name);
+    ASSERT_LE(input_data.size() + 1, 3);
+    void* buffers[3];
+    for (const auto name_and_data : input_data) {
+      const int input_size = name_and_data.second.size() * sizeof(T);
+      const int input_index = engine_->getBindingIndex(name_and_data.first);
+      ASSERT_EQ(0, cudaMalloc(&buffers[input_index], input_size));
+      ASSERT_EQ(
+          0, cudaMemcpyAsync(buffers[input_index], name_and_data.second.data(),
+                             input_size, cudaMemcpyHostToDevice, stream_));
+    }
 
-    ASSERT_EQ(engine_->getNbBindings(), 2);
-    void* buffers[2];
-    ASSERT_EQ(0, cudaMalloc(&buffers[input_index], input_size));
+    const int output_size = output_data->size() * sizeof(T);
+    const int output_index = engine_->getBindingIndex(output_name);
     ASSERT_EQ(0, cudaMalloc(&buffers[output_index], output_size));
-    ASSERT_EQ(0, cudaMemcpyAsync(buffers[input_index], input_data.data(),
-                                 input_size, cudaMemcpyHostToDevice, stream_));
+
+    ASSERT_EQ(engine_->getNbBindings(), input_data.size() + 1);
+
     TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context(
         engine_->createExecutionContext());
     execution_context->enqueue(/*batchSize=*/1, buffers, stream_, nullptr);
     ASSERT_EQ(0, cudaMemcpyAsync(output_data->data(), buffers[output_index],
                                  output_size, cudaMemcpyDeviceToHost, stream_));
     cudaStreamSynchronize(stream_);
-    ASSERT_EQ(0, cudaFree(buffers[input_index]));
-    ASSERT_EQ(0, cudaFree(buffers[output_index]));
+
+    for (int i = 0; i < input_data.size() + 1; ++i) {
+      ASSERT_EQ(0, cudaFree(buffers[i]));
+    }
   }
 
   bool HasStaticShape(const nvinfer1::Dims& dims) const {
@@ -865,18 +989,7 @@ class OpConverterTest : public ::testing::Test {
   void AddTestTensor(
       const char* name, const std::vector<int32>& dims, int batch_size = 1,
       nvinfer1::DataType trt_dtype = nvinfer1::DataType::kFLOAT) {
-    DataType tf_dtype = DT_FLOAT;
-    switch (trt_dtype) {
-      case nvinfer1::DataType::kFLOAT:
-        tf_dtype = DT_FLOAT;
-        break;
-      case nvinfer1::DataType::kINT32:
-        tf_dtype = DT_INT32;
-        break;
-      default:
-        ASSERT_TRUE(false) << "Unexpected data type "
-                           << static_cast<int>(trt_dtype);
-    }
+    DataType tf_dtype = TrtDataTypeToTf(trt_dtype);
     ops::Placeholder::Attrs attrs;
     TF_EXPECT_OK(TensorShapeUtils::MakeShape(dims, &attrs.shape_));
     attrs.shape_.InsertDim(0, batch_size);
@@ -1097,15 +1210,15 @@ TEST_F(OpConverterTest, ConvertTranspose) {
     Reset();
     AddTestTensor("input", {1, 2, 3});
     AddTestWeights<int32>("weights", {4}, {0, 3, 1, 2});
-    RunConversion(node_def);
+    RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_transpose", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({3, 1, 2}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_transpose", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_transpose",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 4, 2, 5, 3, 6));
   }
 }
@@ -1187,15 +1300,15 @@ TEST_F(OpConverterTest, ConvertReshape) {
     Reset();
     AddTestTensor("input", ok_params[i].tensor_dims, ok_params[i].batch_size);
     AddTestWeights<int32>("weights", {4}, ok_params[i].shape);
-    RunConversion(node_def);
+    RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_reshape", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({1, 3, 2}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({1, 3, 2}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_reshape", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_reshape",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
 }
@@ -1250,15 +1363,14 @@ TEST_F(OpConverterTest, ConvertMatMul) {
         get_matmul_nodedef(DT_FLOAT, /*transpose_a=*/false, transpose_b);
     AddTestTensor("input", {2}, /*batch_size=*/1);
     AddTestWeights<float>("weights", {2, 2}, {0, 1, 2, 3});
-    RunConversion(node_def);
+    RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_matmul", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(2);
-    BuildAndRun("input", {0, 1}, "my_matmul", &output_data);
+    BuildAndRun<float>({{"input", {0, 1}}}, "my_matmul", &output_data);
     if (transpose_b) {
       EXPECT_THAT(output_data, ElementsAre(1, 3));
     } else {
@@ -1294,12 +1406,15 @@ void TestConvertBiasAdd(OpConverterTest* test) {
         dims_array[0] = 2;
         dims_array[trt_input_rank - 1] = 3;
       }
-      test->AddTestTensor("input", dims_array, /*batch_size=*/1);
+      test->AddTestTensor("input", dims_array, /*batch_size=*/1,
+                          TfDataTypeToTrt(dtype));
 
       // Add bias weights.
       const int channel_size = (data_format == "NHWC" ? 3 : 2);
       std::vector<CType> bias(channel_size);
-      std::iota(bias.begin(), bias.end(), 1);  // bias will be {1, 2, 3, ...}
+      for (int i = 0; i < channel_size; ++i) {
+        bias[i] = CType(i + 1);  // bias will be {1, 2, 3, ...}
+      }
       test->AddTestWeights<CType>("weights", {channel_size}, bias);
 
       // Run the conversion.
@@ -1307,28 +1422,29 @@ void TestConvertBiasAdd(OpConverterTest* test) {
       TRT_TensorOrWeights output;
       TF_EXPECT_OK(test->GetTensorOrWeights("my_biasadd", &output));
       EXPECT_TRUE(output.is_tensor());
-      EXPECT_TRUE(
-          TrtDimsEqualsArray(dims_array, output.tensor()->getDimensions()))
-          << output.DebugString();
+      ExpectTrtDimsEqualsArray(dims_array, output.tensor()->getDimensions());
 
       // Build and run the engine.
       const int num_input = TrtDimsNumElements(GetTestDims(dims_array));
       ASSERT_EQ(trt_input_rank > 1 ? 6 : (data_format == "NHWC" ? 3 : 2),
                 num_input);
       std::vector<CType> output_data(num_input);
-      test->BuildAndRun("input", std::vector<CType>(num_input, CType(0)),
-                        "my_biasadd", &output_data);
+      test->BuildAndRun<CType>(
+          {{"input", std::vector<CType>(num_input, CType(0))}}, "my_biasadd",
+          &output_data);
       if (trt_input_rank == 1) {
         if (data_format == "NHWC") {
-          EXPECT_THAT(output_data, ElementsAre(1, 2, 3));
+          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2), CType(3)));
         } else {
-          EXPECT_THAT(output_data, ElementsAre(1, 2));
+          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2)));
         }
       } else {
         if (data_format == "NHWC") {
-          EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 1, 2, 3));
+          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(2), CType(3),
+                                               CType(1), CType(2), CType(3)));
         } else {
-          EXPECT_THAT(output_data, ElementsAre(1, 1, 1, 2, 2, 2));
+          EXPECT_THAT(output_data, ElementsAre(CType(1), CType(1), CType(1),
+                                               CType(2), CType(2), CType(2)));
         }
       }
     }
@@ -1344,11 +1460,344 @@ TEST_F(OpConverterTest, ConvertBiasAdd) {
         "Input expects tensor and weights, at my_biasadd");
   }
 
-  // OK.
+  // OK. Note that kINT32 is not supported by IScaleLayer, so we don't test
+  // DT_INT32 type here.
   TestConvertBiasAdd<DT_FLOAT>(this);
-  // TODO(laigd): uncomment this after cl/220663893 is submitted.
-  // TestConvertBiasAdd<DT_INT32>(this);
-  // TestConvertBiasAdd<DT_HALF>(this);
+  TestConvertBiasAdd<DT_HALF>(this);
+}
+
+template <typename OpType>
+NodeDef GetBinaryOpNodeDef(const string& input_name_l,
+                           const string& input_name_r, DataType dtype) {
+  Scope s = Scope::NewRootScope();
+  auto input_l = ops::Placeholder(s.WithOpName(input_name_l), dtype);
+  auto input_r = ops::Placeholder(s.WithOpName(input_name_r), dtype);
+  auto op = OpType(s.WithOpName("my_binary"), input_l, input_r);
+  return op.operation.node()->def();
+}
+
+void CheckAddedLayers(OpConverterTest* test, bool expect_scale_layer) {
+  bool element_wise_layer_found = false;
+  bool scale_layer_found = false;
+  for (int i = 0; i < test->converter_->network()->getNbLayers(); i++) {
+    nvinfer1::ILayer* layer = test->converter_->network()->getLayer(i);
+    if (dynamic_cast<nvinfer1::IScaleLayer*>(layer)) {
+      scale_layer_found = true;
+    } else if (dynamic_cast<nvinfer1::IElementWiseLayer*>(layer)) {
+      element_wise_layer_found = true;
+    }
+  }
+  EXPECT_EQ(expect_scale_layer, scale_layer_found);
+  EXPECT_NE(expect_scale_layer, element_wise_layer_found);
+}
+
+template <typename OpType, DataType dtype>
+void TestBinaryTensorOpWeightNoBroadcast(OpConverterTest* test) {
+  typedef typename EnumToDataType<dtype>::Type CType;
+  for (auto swap_inputs : {false, true}) {
+    test->Reset();
+    NodeDef node_def;
+    if (swap_inputs) {
+      node_def = GetBinaryOpNodeDef<OpType>("weights", "input", dtype);
+    } else {
+      node_def = GetBinaryOpNodeDef<OpType>("input", "weights", dtype);
+    }
+
+    const std::vector<CType> operand1{CType(3), CType(7.5)};
+    const std::vector<CType> operand2{CType(2), CType(3)};
+
+    // It requires the dims to be at least of rank 3 to apply an IScaleLayer.
+    test->AddTestTensor("input", /*dims=*/{1, 1, 2}, /*batch_size=*/1,
+                        TfDataTypeToTrt(dtype));
+    test->AddTestWeights<CType>("weights", /*dims=*/{1, 1, 2},
+                                /*values=*/swap_inputs ? operand1 : operand2);
+    test->RunValidationAndConversion(node_def);
+
+    // Make sure it does use BinaryTensorOpWeight, not BinaryTensorOpTensor.
+    CheckAddedLayers(test, /*expect_scale_layer=*/true);
+
+    // Check the dims of the output ITensor.
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output));
+    EXPECT_TRUE(output.is_tensor());
+    ExpectTrtDimsEqualsArray({1, 1, 2}, output.tensor()->getDimensions());
+
+    std::vector<CType> output_data(2);
+    test->BuildAndRun<CType>(
+        {{"input",
+          /*input_data=*/swap_inputs ? operand2 : operand1}},
+        "my_binary", &output_data);
+    if (node_def.op() == "Add") {
+      EXPECT_THAT(output_data, ElementsAre(CType(5), CType(10.5)));
+    } else if (node_def.op() == "Sub") {
+      EXPECT_THAT(output_data, ElementsAre(CType(1), CType(4.5)));
+    } else if (node_def.op() == "Mul") {
+      EXPECT_THAT(output_data, ElementsAre(CType(6), CType(22.5)));
+    } else if (node_def.op() == "Div") {
+      EXPECT_THAT(output_data, ElementsAre(CType(1.5), CType(2.5)));
+    } else if (node_def.op() == "RealDiv") {
+      EXPECT_THAT(output_data, ElementsAre(CType(1.5), CType(2.5)));
+    } else {
+      ASSERT_TRUE(false);
+    }
+  }
+}
+
+template <DataType dtype>
+void TestBinaryTensorOpWeightWithChannelWiseBroadcast(OpConverterTest* test) {
+  typedef typename EnumToDataType<dtype>::Type CType;
+  const NodeDef node_def =
+      GetBinaryOpNodeDef<ops::Add>("input", "weights", dtype);
+  const std::vector<CType> input{CType(1), CType(2), CType(3), CType(4)};
+  const std::vector<CType> weights{CType(10), CType(20)};
+  // There are two types of valid dim pairs which requires channel-wise
+  // broadcasting:
+  // - input dims (X Y Z) vs weights dims (X 1 1)
+  // - input dims (X Y Z) vs weights dims (Z)
+  // Here X=Z=2 and Y=1.
+  for (auto weights_dims : std::vector<std::vector<int>>{{2, 1, 1}, {2}}) {
+    test->Reset();
+    test->AddTestTensor("input", /*dims=*/{2, 1, 2}, /*batch_size=*/1,
+                        TfDataTypeToTrt(dtype));
+    test->AddTestWeights<CType>("weights", weights_dims, weights);
+    test->RunValidationAndConversion(node_def);
+
+    // Make sure it does use BinaryTensorOpWeight, not BinaryTensorOpTensor.
+    CheckAddedLayers(test, /*expect_scale_layer=*/true);
+
+    // Check the dims of the output ITensor.
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output));
+    EXPECT_TRUE(output.is_tensor());
+    ExpectTrtDimsEqualsArray({2, 1, 2}, output.tensor()->getDimensions());
+
+    std::vector<CType> output_data(4);
+    test->BuildAndRun<CType>({{"input", input}}, "my_binary", &output_data);
+    if (weights_dims.size() == 1) {
+      EXPECT_THAT(output_data,
+                  ElementsAre(CType(11), CType(22), CType(13), CType(24)));
+    } else {
+      EXPECT_THAT(output_data,
+                  ElementsAre(CType(11), CType(12), CType(23), CType(24)));
+    }
+  }
+}
+
+template <DataType dtype>
+void TestBinaryTensorOpWeightWithUniformlyBroadcast(OpConverterTest* test) {
+  typedef typename EnumToDataType<dtype>::Type CType;
+  const NodeDef node_def =
+      GetBinaryOpNodeDef<ops::Add>("input", "weights", dtype);
+  const std::vector<CType> input{CType(1), CType(2), CType(3), CType(4)};
+  const std::vector<CType> weights{CType(10)};
+  test->Reset();
+  test->AddTestTensor("input", /*dims=*/{2, 1, 2}, /*batch_size=*/1,
+                      TfDataTypeToTrt(dtype));
+  test->AddTestWeights<CType>("weights", {1, 1, 1, 1}, weights);
+  test->RunValidationAndConversion(node_def);
+
+  // Make sure it does use BinaryTensorOpWeight, not BinaryTensorOpTensor.
+  CheckAddedLayers(test, /*expect_scale_layer=*/true);
+
+  // Check the dims of the output ITensor.
+  TRT_TensorOrWeights output;
+  TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output));
+  EXPECT_TRUE(output.is_tensor());
+  ExpectTrtDimsEqualsArray({2, 1, 2}, output.tensor()->getDimensions());
+
+  std::vector<CType> output_data(4);
+  test->BuildAndRun<CType>({{"input", input}}, "my_binary", &output_data);
+  EXPECT_THAT(output_data,
+              ElementsAre(CType(11), CType(12), CType(13), CType(14)));
+}
+
+template <typename OpType>
+void TestBinaryTensorOpWeightFallback(OpConverterTest* test,
+                                      const std::vector<int32>& input_dims,
+                                      const std::vector<int>& weights_dims,
+                                      error::Code code = error::OK,
+                                      const char* error_msg_substr = nullptr,
+                                      const int input_batch_size = 1) {
+  const DataType dtype = DT_FLOAT;
+  typedef typename EnumToDataType<dtype>::Type CType;
+  const size_t num_inputs = TrtDimsNumElements(GetTestDims(input_dims));
+  const size_t num_weights = TrtDimsNumElements(GetTestDims(weights_dims));
+
+  test->Reset();
+  const NodeDef node_def =
+      GetBinaryOpNodeDef<OpType>("input", "weights", dtype);
+  test->AddTestTensor("input", /*dims=*/input_dims, input_batch_size,
+                      TfDataTypeToTrt(dtype));
+  test->AddTestWeights<CType>(
+      "weights", /*dims=*/weights_dims,
+      /*values=*/std::vector<CType>(num_weights, CType(1)));
+  test->RunValidationAndConversion(node_def, code, error_msg_substr);
+  if (code != error::OK) return;
+
+  // Make sure it does use BinaryTensorOpTensor, not BinaryTensorOpWeight.
+  CheckAddedLayers(test, /*expect_scale_layer=*/false);
+
+  TRT_TensorOrWeights output;
+  TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output));
+  EXPECT_TRUE(output.is_tensor());
+
+  // Check the dims of the output ITensor.
+  std::vector<int> expected_output_dims = input_dims;
+  for (int i = expected_output_dims.size() - 1, j = weights_dims.size() - 1;
+       i >= 0 && j >= 0; --i, --j) {
+    if (expected_output_dims[i] == 1) {
+      expected_output_dims[i] = weights_dims[j];
+    }
+  }
+  ExpectTrtDimsEqualsArray(expected_output_dims,
+                           output.tensor()->getDimensions());
+
+  // Check the result of running the engine.
+  const int expected_num_outputs =
+      TrtDimsNumElements(GetTestDims(expected_output_dims));
+  std::vector<CType> output_data(expected_num_outputs);
+  test->BuildAndRun<CType>(
+      {{"input",
+        /*input_data=*/std::vector<CType>(num_inputs, CType(2))}},
+      "my_binary", &output_data);
+  if (node_def.op() == "Add") {
+    EXPECT_THAT(output_data, ElementsAreArray(std::vector<CType>(
+                                 expected_num_outputs, CType(3))));
+  } else if (node_def.op() == "Minimum") {
+    EXPECT_THAT(output_data, ElementsAreArray(std::vector<CType>(
+                                 expected_num_outputs, CType(1))));
+  } else {
+    ASSERT_TRUE(false);
+  }
+}
+
+template <typename OpType, DataType dtype>
+void TestBinaryTensorOpTensor(OpConverterTest* test) {
+  typedef typename EnumToDataType<dtype>::Type CType;
+  test->Reset();
+  const NodeDef node_def =
+      GetBinaryOpNodeDef<OpType>("input1", "input2", dtype);
+  test->AddTestTensor("input1", /*dims=*/{1, 2}, /*batch_size=*/1,
+                      TfDataTypeToTrt(dtype));
+  test->AddTestTensor("input2", /*dims=*/{2, 1}, /*batch_size=*/1,
+                      TfDataTypeToTrt(dtype));
+  test->RunValidationAndConversion(node_def);
+
+  // Make sure it does use BinaryTensorOpTensor, not BinaryTensorOpWeight.
+  CheckAddedLayers(test, /*expect_scale_layer=*/false);
+
+  // Check output dims.
+  TRT_TensorOrWeights output;
+  TF_EXPECT_OK(test->GetTensorOrWeights("my_binary", &output));
+  EXPECT_TRUE(output.is_tensor());
+  ExpectTrtDimsEqualsArray({2, 2}, output.tensor()->getDimensions());
+
+  std::vector<CType> output_data(4);
+  // After broadcasting first input becomes {3, 6, 3, 6} and second input
+  // becomes {2, 3, 2, 3}.
+  test->BuildAndRun<CType>(
+      {{"input1", {CType(3), CType(6)}}, {"input2", {CType(2), CType(3)}}},
+      "my_binary", &output_data);
+  if (node_def.op() == "Add") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(5), CType(8), CType(6), CType(9)));
+  } else if (node_def.op() == "Sub") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(1), CType(4), CType(0), CType(3)));
+  } else if (node_def.op() == "Mul") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(6), CType(12), CType(9), CType(18)));
+  } else if (node_def.op() == "Div") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(1.5), CType(3), CType(1), CType(2)));
+  } else if (node_def.op() == "RealDiv") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(1.5), CType(3), CType(1), CType(2)));
+  } else if (node_def.op() == "Minimum") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(2), CType(2), CType(3), CType(3)));
+  } else if (node_def.op() == "Maximum") {
+    EXPECT_THAT(output_data,
+                ElementsAre(CType(3), CType(6), CType(3), CType(6)));
+  } else {
+    ASSERT_TRUE(false);
+  }
+}
+
+TEST_F(OpConverterTest, ConvertBinary) {
+  // Input size doesn't match, should fail.
+  for (size_t num_inputs = 0; num_inputs < 2; ++num_inputs) {
+    Reset();
+    NodeDef node_def = MakeNodeDef("my_add", "Add", {num_inputs, "input"});
+    AddTestTensor("input", {1}, /*batch_size=*/1, nvinfer1::DataType::kFLOAT);
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+                               "Binary ops require two inputs, at my_add");
+  }
+  {
+    // Both inputs are weights.
+    Reset();
+    NodeDef node_def = MakeNodeDef("my_add", "Add", {"weights1", "weights2"});
+    AddTestWeights<float>("weights1", {1}, {1});
+    AddTestWeights<float>("weights2", {1}, {1});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Constant folding is falled back to TensorFlow, binary op received "
+        "both input as constant at: my_add");
+  }
+
+  // Test BinaryTensorOpWeight() without broadcasting.
+  TestBinaryTensorOpWeightNoBroadcast<ops::Add, DT_FLOAT>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::Sub, DT_FLOAT>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::Mul, DT_FLOAT>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::Div, DT_FLOAT>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::RealDiv, DT_FLOAT>(this);
+#if 0
+  // TODO(b/119560144): it doesn't support FP16 constants and the following test
+  // will fail.
+  TestBinaryTensorOpWeightNoBroadcast<ops::Add, DT_HALF>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::Sub, DT_HALF>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::Mul, DT_HALF>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::Div, DT_HALF>(this);
+  TestBinaryTensorOpWeightNoBroadcast<ops::RealDiv, DT_HALF>(this);
+#endif
+
+  // Test BinaryTensorOpWeight() with channel-wise broadcasting.
+  TestBinaryTensorOpWeightWithChannelWiseBroadcast<DT_FLOAT>(this);
+
+  // Test BinaryTensorOpWeight() with uniformly broadcasting.
+  TestBinaryTensorOpWeightWithUniformlyBroadcast<DT_FLOAT>(this);
+
+  // Test BinaryTensorOpWeight() falling back to BinaryTensorOpTensor().
+  // Unsupported op.
+  TestBinaryTensorOpWeightFallback<ops::Minimum>(this, {1, 1, 1}, {1});
+  // Rank of input tensor dimension <3.
+  TestBinaryTensorOpWeightFallback<ops::Add>(this, {1, 1}, {1});
+  // Broadcast on batch dimension, should fail.
+  TestBinaryTensorOpWeightFallback<ops::Add>(
+      this, {1, 1, 1}, {2, 1, 1, 1}, error::INVALID_ARGUMENT,
+      "Unsupported binary op broadcast scheme for op my_binary",
+      /*input_batch_size=*/2);
+  // Incompatible dims with per-channel mode.
+  TestBinaryTensorOpWeightFallback<ops::Add>(this, {1, 1, 1}, {1, 2, 1});
+  // Incompatible dims.
+  TestBinaryTensorOpWeightFallback<ops::Add>(this, {1, 2, 1}, {2});
+
+  // Test BinaryTensorOpTensor() with broadcasting.
+  TestBinaryTensorOpTensor<ops::Add, DT_FLOAT>(this);
+  TestBinaryTensorOpTensor<ops::Sub, DT_FLOAT>(this);
+  TestBinaryTensorOpTensor<ops::Mul, DT_FLOAT>(this);
+  TestBinaryTensorOpTensor<ops::Div, DT_FLOAT>(this);
+  TestBinaryTensorOpTensor<ops::RealDiv, DT_FLOAT>(this);
+  TestBinaryTensorOpTensor<ops::Minimum, DT_FLOAT>(this);
+  TestBinaryTensorOpTensor<ops::Maximum, DT_FLOAT>(this);
+
+  TestBinaryTensorOpTensor<ops::Add, DT_HALF>(this);
+  TestBinaryTensorOpTensor<ops::Sub, DT_HALF>(this);
+  TestBinaryTensorOpTensor<ops::Mul, DT_HALF>(this);
+  TestBinaryTensorOpTensor<ops::Div, DT_HALF>(this);
+  TestBinaryTensorOpTensor<ops::RealDiv, DT_HALF>(this);
+  TestBinaryTensorOpTensor<ops::Minimum, DT_HALF>(this);
+  TestBinaryTensorOpTensor<ops::Maximum, DT_HALF>(this);
 }
 
 TEST_F(OpConverterTest, ConvertQuantize) {
@@ -1509,7 +1958,8 @@ TEST_F(OpConverterTest, ConvertRelu6) {
     EXPECT_EQ(ranges[output.tensor()], 6.0f);
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {-100, -1, 0, 3, 5, 9}, "my_relu6", &output_data);
+    BuildAndRun<float>({{"input", {-100, -1, 0, 3, 5, 9}}}, "my_relu6",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(0, 0, 0, 3, 5, 6));
   }
 }
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 7e826fb7e1..b325d76edf 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -56,8 +56,9 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
             strides=[1, 2, 2, 1],
             padding="SAME",
             name="conv")
-        bias = constant_op.constant(
-            [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype)
+        bias = constant_op.constant([4., 1.5, 2., 3., 5., 7.],
+                                    name="bias",
+                                    dtype=dtype)
         added = nn.bias_add(conv, bias, name="bias_add")
         relu = nn.relu(added, "relu")
         identity = array_ops.identity(relu, "identity")
@@ -73,11 +74,12 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    # TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
-    # breaks the connection check, fix it.
-    # - my_trt_op_0 should have ["weights", "conv", "bias", "bias_add",
-    #   "relu", "identity", "max_pool"]
-    return ["my_trt_op_0"]
+    return {
+        "my_trt_op_0": [
+            "weights", "conv", "bias", "bias_add", "relu", "identity",
+            "max_pool"
+        ]
+    }
 
 
 class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
@@ -92,7 +94,7 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+          dtype=dtype, shape=input_dims, name=input_name)
       with g.device("/GPU:0"):
         conv_filter = constant_op.constant(
             [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
@@ -105,10 +107,10 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
             padding="SAME",
             name="conv")
         c1 = constant_op.constant(
-            np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype, name="c1")
+            np.random.randn(12, 12, 6), dtype=dtype, name="c1")
         p = math_ops.mul(conv, c1, name="mul")
         c2 = constant_op.constant(
-            np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype, name="c2")
+            np.random.randn(12, 12, 6), dtype=dtype, name="c2")
         q = math_ops.div(conv, c2, name="div")
 
         edge = self.trt_incompatible_op(q, name="incompatible")
@@ -129,22 +131,21 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    # TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
-    # breaks the connection check, fix it.
-    # - my_trt_op_0 should have ["mul", "sub", "div1", "mul1", "add1",
-    #   "add", "sub1"];
-    # - my_trt_op_1 should have ["weights","conv", "div"]
-    return ["my_trt_op_0", "my_trt_op_1"]
+    return {
+        "my_trt_op_0": [
+            "add", "add1", "c1", "div1", "mul", "mul1", "sub", "sub1"
+        ],
+        "my_trt_op_1": ["c2", "conv", "div", "weights"]
+    }
 
-  def ShouldRunTest(self, run_params):
-    # TODO(aaroey): LayoutOptimizer adds Transpose(Const, Const) to the graph
-    # which breaks the conversion. We should fix it as:
-    # - Detect the invalid NodeDef earlier before adding them to segment
-    # - Let it able to change the RewriterConfig when calling
-    #   create_inference_graph().
-    # It will be good to add debugging feature for Grappler to print the graph
-    # after running each optimizer.
-    return False
+  def GetConversionParams(self, run_params):
+    """Return a ConversionParams for test."""
+    return super(
+        SimpleMultiEnginesTest, self
+    ).GetConversionParams(run_params)._replace(
+        # Disable layout optimizer, since it'll add Transpose(Const, Const) to
+        # the graph and breaks the conversion check.
+        rewriter_config=trt_test.OptimizerDisabledRewriterConfig())
 
 
 class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 0d23eeae1a..80eb8552fd 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -70,6 +70,8 @@ class GraphState(object):
 def OptimizerDisabledRewriterConfig():
   """Returns a RewriterConfig with all default Grappler optimizers disabled."""
   rewriter_config = rewriter_config_pb2.RewriterConfig()
+
+  # Turn off all default Grappler optimizers.
   off = rewriter_config_pb2.RewriterConfig.OFF
   rewriter_config.layout_optimizer = off
   rewriter_config.constant_folding = off
@@ -86,6 +88,10 @@ def OptimizerDisabledRewriterConfig():
       rewriter_config_pb2.RewriterConfig.NO_MEM_OPT)
   rewriter_config.pin_to_host_optimization = off
   rewriter_config.auto_parallel.enable = False
+
+  # Run only once for each enabled optimizer.
+  rewriter_config.meta_optimizer_iterations = (
+      rewriter_config_pb2.RewriterConfig.ONE)
   return rewriter_config
 
 
-- 
GitLab


From dc45615483758ffc5889d7cb61537634ce925c78 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 20 Nov 2018 16:33:58 -0800
Subject: [PATCH 0632/1554] Internal Cleanup.

PiperOrigin-RevId: 222326402
---
 tensorflow/python/ops/linalg_ops.py           | 73 ++++++++++++++++++-
 .../api/golden/v2/tensorflow.linalg.pbtxt     |  2 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 3 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index bbccc7e036..1a9e7112b4 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -423,7 +423,78 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None):
 
 
 # pylint: disable=redefined-builtin
-@tf_export('norm', 'linalg.norm')
+@tf_export('norm', 'linalg.norm', v1=[])
+def norm_v2(tensor,
+            ord='euclidean',
+            axis=None,
+            keepdims=None,
+            name=None):
+  r"""Computes the norm of vectors, matrices, and tensors.
+
+  This function can compute several different vector norms (the 1-norm, the
+  Euclidean or 2-norm, the inf-norm, and in general the p-norm for p > 0) and
+  matrix norms (Frobenius, 1-norm, 2-norm and inf-norm).
+
+  Args:
+    tensor: `Tensor` of types `float32`, `float64`, `complex64`, `complex128`
+    ord: Order of the norm. Supported values are 'fro', 'euclidean',
+      `1`, `2`, `np.inf` and any positive real number yielding the corresponding
+      p-norm. Default is 'euclidean' which is equivalent to Frobenius norm if
+      `tensor` is a matrix and equivalent to 2-norm for vectors.
+      Some restrictions apply:
+        a) The Frobenius norm `fro` is not defined for vectors,
+        b) If axis is a 2-tuple (matrix norm), only 'euclidean', 'fro', `1`,
+           `2`, `np.inf` are supported.
+      See the description of `axis` on how to compute norms for a batch of
+      vectors or matrices stored in a tensor.
+    axis: If `axis` is `None` (the default), the input is considered a vector
+      and a single vector norm is computed over the entire set of values in the
+      tensor, i.e. `norm(tensor, ord=ord)` is equivalent to
+      `norm(reshape(tensor, [-1]), ord=ord)`.
+      If `axis` is a Python integer, the input is considered a batch of vectors,
+      and `axis` determines the axis in `tensor` over which to compute vector
+      norms.
+      If `axis` is a 2-tuple of Python integers it is considered a batch of
+      matrices and `axis` determines the axes in `tensor` over which to compute
+      a matrix norm.
+      Negative indices are supported. Example: If you are passing a tensor that
+      can be either a matrix or a batch of matrices at runtime, pass
+      `axis=[-2,-1]` instead of `axis=None` to make sure that matrix norms are
+      computed.
+    keepdims: If True, the axis indicated in `axis` are kept with size 1.
+      Otherwise, the dimensions in `axis` are removed from the output shape.
+    name: The name of the op.
+
+  Returns:
+    output: A `Tensor` of the same type as tensor, containing the vector or
+      matrix norms. If `keepdims` is True then the rank of output is equal to
+      the rank of `tensor`. Otherwise, if `axis` is none the output is a scalar,
+      if `axis` is an integer, the rank of `output` is one less than the rank
+      of `tensor`, if `axis` is a 2-tuple the rank of `output` is two less
+      than the rank of `tensor`.
+
+  Raises:
+    ValueError: If `ord` or `axis` is invalid.
+
+  @compatibility(numpy)
+  Mostly equivalent to numpy.linalg.norm.
+  Not supported: ord <= 0, 2-norm for matrices, nuclear norm.
+  Other differences:
+    a) If axis is `None`, treats the flattened `tensor` as a vector
+     regardless of rank.
+    b) Explicitly supports 'euclidean' norm as the default, including for
+     higher order tensors.
+  @end_compatibility
+  """
+  return norm(tensor=tensor,
+              ord=ord,
+              axis=axis,
+              keepdims=keepdims,
+              name=name)
+
+
+# pylint: disable=redefined-builtin
+@tf_export(v1=['norm', 'linalg.norm'])
 @deprecation.deprecated_args(
     None, 'keep_dims is deprecated, use keepdims instead', 'keep_dims')
 def norm(tensor,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index 1a4098d121..d8259aa775 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -142,7 +142,7 @@ tf_module {
   }
   member_method {
     name: "norm"
-    argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "qr"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 1711348ec3..ec2d72290e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -854,7 +854,7 @@ tf_module {
   }
   member_method {
     name: "norm"
-    argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'tensor\', \'ord\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'euclidean\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "not_equal"
-- 
GitLab


From 4da08f347b444fa60b2327a15449a37740871224 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 20 Nov 2018 16:54:28 -0800
Subject: [PATCH 0633/1554] Implement Keras V2 FTRL optimization.

PiperOrigin-RevId: 222329163
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  20 +
 tensorflow/python/keras/optimizer_v2/ftrl.py  | 207 +++++++++
 .../python/keras/optimizer_v2/ftrl_test.py    | 426 ++++++++++++++++++
 3 files changed, 653 insertions(+)
 create mode 100644 tensorflow/python/keras/optimizer_v2/ftrl.py
 create mode 100644 tensorflow/python/keras/optimizer_v2/ftrl_test.py

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 7defc7d308..6b805781f0 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -17,6 +17,7 @@ py_library(
         "adagrad.py",
         "adam.py",
         "adamax.py",
+        "ftrl.py",
         "gradient_descent.py",
         "nadam.py",
         "optimizer_v2.py",
@@ -112,6 +113,25 @@ cuda_py_test(
     shard_count = 4,
 )
 
+cuda_py_test(
+    name = "ftrl_test",
+    size = "medium",
+    srcs = ["ftrl_test.py"],
+    additional_deps = [
+        ":optimizer_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+    shard_count = 4,
+)
+
 cuda_py_test(
     name = "gradient_descent_test",
     size = "medium",
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl.py b/tensorflow/python/keras/optimizer_v2/ftrl.py
new file mode 100644
index 0000000000..2faf65eab3
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/ftrl.py
@@ -0,0 +1,207 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ftrl-proximal for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.training import training_ops
+
+
+class Ftrl(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the FTRL algorithm.
+
+  See this [paper](
+  https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf).
+  This version has support for both online L2 (the L2 penalty given in the paper
+  above) and shrinkage-type L2 (which is the addition of an L2 penalty to the
+  loss function).
+  """
+
+  def __init__(self,
+               learning_rate,
+               learning_rate_power=-0.5,
+               initial_accumulator_value=0.1,
+               l1_regularization_strength=0.0,
+               l2_regularization_strength=0.0,
+               name='Ftrl',
+               l2_shrinkage_regularization_strength=0.0):
+    r"""Construct a new FTRL optimizer.
+
+    Args:
+      learning_rate: A float value or a constant float `Tensor`.
+      learning_rate_power: A float value, must be less or equal to zero.
+        Controls how the learning rate decreases during training. Use zero for
+        a fixed learning rate.
+      initial_accumulator_value: The starting value for accumulators.
+        Only zero or positive values are allowed.
+      l1_regularization_strength: A float value, must be greater than or
+        equal to zero.
+      l2_regularization_strength: A float value, must be greater than or
+        equal to zero.
+      name: Optional name prefix for the operations created when applying
+        gradients.  Defaults to "Ftrl".
+      l2_shrinkage_regularization_strength: A float value, must be greater than
+        or equal to zero. This differs from L2 above in that the L2 above is a
+        stabilization penalty, whereas this L2 shrinkage is a magnitude penalty.
+        The FTRL formulation can be written as:
+        w_{t+1} = argmin_w(\hat{g}_{1:t}w + L1*||w||_1 + L2*||w||_2^2), where
+        \hat{g} = g + (2*L2_shrinkage*w), and g is the gradient of the loss
+        function w.r.t. the weights w.
+        Specifically, in the absence of L1 regularization, it is equivalent to
+        the following update rule:
+        w_{t+1} = w_t - lr_t / (1 + 2*L2*lr_t) * g_t -
+                  2*L2_shrinkage*lr_t / (1 + 2*L2*lr_t) * w_t
+        where lr_t is the learning rate at t.
+        When input is sparse shrinkage will only happen on the active weights.
+
+    Raises:
+      ValueError: If one of the arguments is invalid.
+
+    References
+      See [paper]
+        (https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf)
+    """
+    super(Ftrl, self).__init__(name)
+
+    if initial_accumulator_value < 0.0:
+      raise ValueError(
+          'initial_accumulator_value %f needs to be positive or zero' %
+          initial_accumulator_value)
+    if learning_rate_power > 0.0:
+      raise ValueError('learning_rate_power %f needs to be negative or zero' %
+                       learning_rate_power)
+    if l1_regularization_strength < 0.0:
+      raise ValueError(
+          'l1_regularization_strength %f needs to be positive or zero' %
+          l1_regularization_strength)
+    if l2_regularization_strength < 0.0:
+      raise ValueError(
+          'l2_regularization_strength %f needs to be positive or zero' %
+          l2_regularization_strength)
+    if l2_shrinkage_regularization_strength < 0.0:
+      raise ValueError(
+          'l2_shrinkage_regularization_strength %f needs to be positive'
+          ' or zero' % l2_shrinkage_regularization_strength)
+
+    self._set_hyper('learning_rate', learning_rate)
+    self._set_hyper('learning_rate_power', learning_rate_power)
+    self._set_hyper('l1_regularization_strength', l1_regularization_strength)
+    self._set_hyper('l2_regularization_strength', l2_regularization_strength)
+    self._initial_accumulator_value = initial_accumulator_value
+    self._l2_shrinkage_regularization_strength = (
+        l2_shrinkage_regularization_strength)
+
+  def _create_slots(self, var_list):
+    # Create the "accum" and "linear" slots.
+    for var in var_list:
+      dtype = var.dtype.base_dtype
+      init = init_ops.constant_initializer(
+          self._initial_accumulator_value, dtype=dtype)
+      self.add_slot(var, 'accumulator', init)
+      self.add_slot(var, 'linear')
+
+  def _resource_apply_dense(self, grad, var):
+    var_dtype = var.dtype.base_dtype
+    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
+    learning_rate_power = math_ops.cast(
+        self._get_hyper('learning_rate_power'), var_dtype)
+    l1_regularization_strength = math_ops.cast(
+        self._get_hyper('l1_regularization_strength'), var_dtype)
+    l2_regularization_strength = math_ops.cast(
+        self._get_hyper('l2_regularization_strength'), var_dtype)
+    accum = self.get_slot(var, 'accumulator')
+    linear = self.get_slot(var, 'linear')
+    if self._l2_shrinkage_regularization_strength <= 0.0:
+      return training_ops.resource_apply_ftrl(
+          var.handle,
+          accum.handle,
+          linear.handle,
+          grad,
+          learning_rate,
+          l1_regularization_strength,
+          l2_regularization_strength,
+          learning_rate_power,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.resource_apply_ftrl_v2(
+          var.handle,
+          accum.handle,
+          linear.handle,
+          grad,
+          learning_rate,
+          l1_regularization_strength,
+          l2_regularization_strength,
+          math_ops.cast(self._l2_shrinkage_regularization_strength, var_dtype),
+          learning_rate_power,
+          use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices):
+    var_dtype = var.dtype.base_dtype
+    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
+    learning_rate_power = math_ops.cast(
+        self._get_hyper('learning_rate_power'), var_dtype)
+    l1_regularization_strength = math_ops.cast(
+        self._get_hyper('l1_regularization_strength'), var_dtype)
+    l2_regularization_strength = math_ops.cast(
+        self._get_hyper('l2_regularization_strength'), var_dtype)
+    accum = self.get_slot(var, 'accumulator')
+    linear = self.get_slot(var, 'linear')
+    if self._l2_shrinkage_regularization_strength <= 0.0:
+      return training_ops.resource_sparse_apply_ftrl(
+          var.handle,
+          accum.handle,
+          linear.handle,
+          grad,
+          indices,
+          learning_rate,
+          l1_regularization_strength,
+          l2_regularization_strength,
+          learning_rate_power,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.resource_sparse_apply_ftrl_v2(
+          var.handle,
+          accum.handle,
+          linear.handle,
+          grad,
+          indices,
+          learning_rate,
+          l1_regularization_strength,
+          l2_regularization_strength,
+          math_ops.cast(self._l2_shrinkage_regularization_strength, var_dtype),
+          learning_rate_power,
+          use_locking=self._use_locking)
+
+  def get_config(self):
+    config = super(Ftrl, self).get_config()
+    config.update({
+        'learning_rate':
+            self._serialize_hyperparameter('learning_rate'),
+        'initial_accumulator_value':
+            self._initial_accumulator_value,
+        'learning_rate_power':
+            self._serialize_hyperparameter('learning_rate_power'),
+        'l1_regularization_strength':
+            self._serializer_hyperparameter('l1_regularization_strength'),
+        'l2_regularization_strength':
+            self._serializer_hyperparameter('l2_regularization_strength'),
+        'l2_shrinkage_regularization_strength':
+            self._l2_shrinkage_regularization_strength,
+    })
+    return config
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl_test.py b/tensorflow/python/keras/optimizer_v2/ftrl_test.py
new file mode 100644
index 0000000000..c14cf75c26
--- /dev/null
+++ b/tensorflow/python/keras/optimizer_v2/ftrl_test.py
@@ -0,0 +1,426 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functional tests for Ftrl operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.optimizer_v2 import ftrl
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.training import adagrad
+from tensorflow.python.training import gradient_descent
+
+
+class FtrlOptimizerTest(test.TestCase):
+
+  def doTestFtrlwithoutRegularization(self, use_resource=False):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable([0.0, 0.0], dtype=dtype)
+          var1 = resource_variable_ops.ResourceVariable([0.0, 0.0], dtype=dtype)
+        else:
+          var0 = variables.Variable([0.0, 0.0], dtype=dtype)
+          var1 = variables.Variable([0.0, 0.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.02], dtype=dtype)
+        opt = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.0,
+            l2_regularization_strength=0.0)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllClose([0.0, 0.0], v0_val)
+        self.assertAllClose([0.0, 0.0], v1_val)
+
+        # Run 3 steps FTRL
+        for _ in range(3):
+          update.run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType(
+            np.array([-2.60260963, -4.29698515]), v0_val)
+        self.assertAllCloseAccordingToType(
+            np.array([-0.28432083, -0.56694895]), v1_val)
+
+  def testFtrlWithoutRegularization(self):
+    self.doTestFtrlwithoutRegularization(use_resource=False)
+
+  def testResourceFtrlWithoutRegularization(self):
+    self.doTestFtrlwithoutRegularization(use_resource=True)
+
+  def testFtrlwithoutRegularization2(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([4.0, 3.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.02], dtype=dtype)
+
+        opt = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.0,
+            l2_regularization_strength=0.0)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
+        self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
+
+        # Run 3 steps FTRL
+        for _ in range(3):
+          update.run()
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType(
+            np.array([-2.55607247, -3.98729396]), v0_val)
+        self.assertAllCloseAccordingToType(
+            np.array([-0.28232238, -0.56096673]), v1_val)
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = ftrl.Ftrl(1.0).minimize(loss, var_list=[var0])
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType([[0, 1]],
+                                           self.evaluate(var0),
+                                           atol=0.01)
+
+  def testFtrlWithL1(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([4.0, 3.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.02], dtype=dtype)
+
+        opt = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.001,
+            l2_regularization_strength=0.0)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
+        self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
+
+        # Run 10 steps FTRL
+        for _ in range(10):
+          update.run()
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType(
+            np.array([-7.66718769, -10.91273689]), v0_val)
+        self.assertAllCloseAccordingToType(
+            np.array([-0.93460727, -1.86147261]), v1_val)
+
+  def testFtrlWithL1_L2(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([4.0, 3.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.02], dtype=dtype)
+
+        opt = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.001,
+            l2_regularization_strength=2.0)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
+        self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
+
+        # Run 10 steps FTRL
+        for _ in range(10):
+          update.run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType(
+            np.array([-0.24059935, -0.46829352]), v0_val)
+        self.assertAllCloseAccordingToType(
+            np.array([-0.02406147, -0.04830509]), v1_val)
+
+  def testFtrlWithL1_L2_L2Shrinkage(self):
+    """Test the new FTRL op with support for l2 shrinkage.
+
+    The addition of this parameter which places a constant pressure on weights
+    towards the origin causes the gradient descent trajectory to differ. The
+    weights will tend to have smaller magnitudes with this parameter set.
+    """
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([4.0, 3.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.02], dtype=dtype)
+
+        opt = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.001,
+            l2_regularization_strength=2.0,
+            l2_shrinkage_regularization_strength=0.1)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
+        self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
+
+        # Run 10 steps FTRL
+        for _ in range(10):
+          update.run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType(
+            np.array([-0.22578995, -0.44345796]), v0_val)
+        self.assertAllCloseAccordingToType(
+            np.array([-0.14378493, -0.13229476]), v1_val)
+
+  def testFtrlWithL1_L2_L2ShrinkageSparse(self):
+    """Tests the new FTRL op with support for l2 shrinkage on sparse grads."""
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        var0 = variables.Variable([[1.0], [2.0]], dtype=dtype)
+        var1 = variables.Variable([[4.0], [3.0]], dtype=dtype)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant([0.1], shape=[1, 1], dtype=dtype),
+            constant_op.constant([0]), constant_op.constant([2, 1]))
+        grads1 = ops.IndexedSlices(
+            constant_op.constant([0.02], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]), constant_op.constant([2, 1]))
+
+        opt = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.001,
+            l2_regularization_strength=2.0,
+            l2_shrinkage_regularization_strength=0.1)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([[1.0], [2.0]], v0_val)
+        self.assertAllCloseAccordingToType([[4.0], [3.0]], v1_val)
+
+        # Run 10 steps FTRL
+        for _ in range(10):
+          update.run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val)
+        self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val)
+
+  def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self):
+    """Verifies that l2 shrinkage in FTRL does not change lr schedule."""
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session() as sess:
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([1.0, 2.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+        grads1 = constant_op.constant([0.1, 0.2], dtype=dtype)
+
+        opt0 = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.001,
+            l2_regularization_strength=2.0,
+            l2_shrinkage_regularization_strength=0.1)
+        opt1 = ftrl.Ftrl(
+            3.0,
+            initial_accumulator_value=0.1,
+            l1_regularization_strength=0.001,
+            l2_regularization_strength=2.0)
+        update0 = opt0.apply_gradients([(grads0, var0)])
+        update1 = opt1.apply_gradients([(grads1, var1)])
+        variables.global_variables_initializer().run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
+        self.assertAllCloseAccordingToType([1.0, 2.0], v1_val)
+
+        # Run 10 steps FTRL
+        for _ in range(10):
+          update0.run()
+          update1.run()
+
+        v0_val, v1_val = sess.run([var0, var1])
+        # var0 is experiencing L2 shrinkage so it should be smaller than var1
+        # in magnitude.
+        self.assertTrue((v0_val**2 < v1_val**2).all())
+        accum0 = sess.run(opt0.get_slot(var0, "accumulator"))
+        accum1 = sess.run(opt1.get_slot(var1, "accumulator"))
+        # L2 shrinkage should not change how we update grad accumulator.
+        self.assertAllCloseAccordingToType(accum0, accum1)
+
+  def applyOptimizer(self, opt, dtype, steps=5, is_sparse=False):
+    if is_sparse:
+      var0 = variables.Variable([[0.0], [0.0]], dtype=dtype)
+      var1 = variables.Variable([[0.0], [0.0]], dtype=dtype)
+      grads0 = ops.IndexedSlices(
+          constant_op.constant([0.1], shape=[1, 1], dtype=dtype),
+          constant_op.constant([0]), constant_op.constant([2, 1]))
+      grads1 = ops.IndexedSlices(
+          constant_op.constant([0.02], shape=[1, 1], dtype=dtype),
+          constant_op.constant([1]), constant_op.constant([2, 1]))
+    else:
+      var0 = variables.Variable([0.0, 0.0], dtype=dtype)
+      var1 = variables.Variable([0.0, 0.0], dtype=dtype)
+      grads0 = constant_op.constant([0.1, 0.2], dtype=dtype)
+      grads1 = constant_op.constant([0.01, 0.02], dtype=dtype)
+
+    update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+    variables.global_variables_initializer().run()
+
+    sess = ops.get_default_session()
+    v0_val, v1_val = sess.run([var0, var1])
+    if is_sparse:
+      self.assertAllCloseAccordingToType([[0.0], [0.0]], v0_val)
+      self.assertAllCloseAccordingToType([[0.0], [0.0]], v1_val)
+    else:
+      self.assertAllCloseAccordingToType([0.0, 0.0], v0_val)
+      self.assertAllCloseAccordingToType([0.0, 0.0], v1_val)
+
+    # Run Ftrl for a few steps
+    for _ in range(steps):
+      update.run()
+
+    v0_val, v1_val = sess.run([var0, var1])
+    return v0_val, v1_val
+
+  # When variables are initialized with Zero, FTRL-Proximal has two properties:
+  # 1. Without L1&L2 but with fixed learning rate, FTRL-Proximal is identical
+  # with GradientDescent.
+  # 2. Without L1&L2 but with adaptive learning rate, FTRL-Proximal is identical
+  # with Adagrad.
+  # So, basing on these two properties, we test if our implementation of
+  # FTRL-Proximal performs same updates as Adagrad or GradientDescent.
+  def testEquivAdagradwithoutRegularization(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session():
+        val0, val1 = self.applyOptimizer(
+            ftrl.Ftrl(
+                3.0,
+                # Adagrad learning rate
+                learning_rate_power=-0.5,
+                initial_accumulator_value=0.1,
+                l1_regularization_strength=0.0,
+                l2_regularization_strength=0.0),
+            dtype)
+
+      with self.cached_session():
+        val2, val3 = self.applyOptimizer(
+            adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1), dtype)
+
+      self.assertAllCloseAccordingToType(val0, val2)
+      self.assertAllCloseAccordingToType(val1, val3)
+
+  def testEquivSparseAdagradwithoutRegularization(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session():
+        val0, val1 = self.applyOptimizer(
+            ftrl.Ftrl(
+                3.0,
+                # Adagrad learning rate
+                learning_rate_power=-0.5,
+                initial_accumulator_value=0.1,
+                l1_regularization_strength=0.0,
+                l2_regularization_strength=0.0),
+            dtype,
+            is_sparse=True)
+
+      with self.cached_session():
+        val2, val3 = self.applyOptimizer(
+            adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1),
+            dtype,
+            is_sparse=True)
+
+      self.assertAllCloseAccordingToType(val0, val2)
+      self.assertAllCloseAccordingToType(val1, val3)
+
+  def testEquivSparseGradientDescentwithoutRegularization(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session():
+        val0, val1 = self.applyOptimizer(
+            ftrl.Ftrl(
+                3.0,
+                # Fixed learning rate
+                learning_rate_power=-0.0,
+                initial_accumulator_value=0.1,
+                l1_regularization_strength=0.0,
+                l2_regularization_strength=0.0),
+            dtype,
+            is_sparse=True)
+
+      with self.cached_session():
+        val2, val3 = self.applyOptimizer(
+            gradient_descent.GradientDescentOptimizer(3.0),
+            dtype,
+            is_sparse=True)
+
+      self.assertAllCloseAccordingToType(val0, val2)
+      self.assertAllCloseAccordingToType(val1, val3)
+
+  def testEquivGradientDescentwithoutRegularization(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.cached_session():
+        val0, val1 = self.applyOptimizer(
+            ftrl.Ftrl(
+                3.0,
+                # Fixed learning rate
+                learning_rate_power=-0.0,
+                initial_accumulator_value=0.1,
+                l1_regularization_strength=0.0,
+                l2_regularization_strength=0.0),
+            dtype)
+
+      with self.cached_session():
+        val2, val3 = self.applyOptimizer(
+            gradient_descent.GradientDescentOptimizer(3.0), dtype)
+
+      self.assertAllCloseAccordingToType(val0, val2)
+      self.assertAllCloseAccordingToType(val1, val3)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 0dbafc2eb5b3e5fdfcdb10d69e44c09d61373a08 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Nov 2018 17:05:54 -0800
Subject: [PATCH 0634/1554] Don't auto-cluster ops producing or consuming
 DT_VARIANT.

PiperOrigin-RevId: 222330733
---
 tensorflow/cc/BUILD                           |  1 +
 .../compiler/jit/mark_for_compilation_pass.cc | 37 +++++++--
 .../jit/mark_for_compilation_pass_test.cc     | 76 +++++++++++++++++++
 3 files changed, 106 insertions(+), 8 deletions(-)

diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD
index 83353b79f7..a09becc49b 100644
--- a/tensorflow/cc/BUILD
+++ b/tensorflow/cc/BUILD
@@ -489,6 +489,7 @@ tf_gen_op_wrappers_cc(
         "image_ops",
         "io_ops",
         "linalg_ops",
+        "list_ops",
         "logging_ops",
         "lookup_ops",
         "manip_ops",
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 60b962d2e8..25796435a5 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -72,6 +72,11 @@ struct OperationFilter {
   // to resort to a dummy implementation. Currently Assert and CheckNumerics ops
   // have dummy XLA implementations.
   bool allow_dummy_ops;
+
+  // Whether ops that produce or consume DT_VARIANT values are allowed.  We
+  // don't auto-cluster these ops because we don't yet support live-in or
+  // live-out DT_VARIANT values.
+  bool allow_ops_producing_or_consuming_variant;
 };
 
 bool IsDummyImplOp(absl::string_view op_name) {
@@ -84,6 +89,12 @@ bool IsStatefulRandomOp(absl::string_view op_name) {
          op_name == "TruncatedNormal";
 }
 
+bool OpProducesOrConsumesVariant(const Node& node) {
+  auto is_variant = [](DataType dtype) { return dtype == DT_VARIANT; };
+  return absl::c_any_of(node.input_types(), is_variant) ||
+         absl::c_any_of(node.output_types(), is_variant);
+}
+
 bool HasXLAKernel(const Node& node, const DeviceType& jit_device_type) {
   // There is a SymbolicGradient kernel on the XLA_JIT device, but the gradient
   // is really a kind of function call and will be handled by
@@ -246,6 +257,10 @@ bool IsCompilableCall(const NodeDef& call_def,
     if (!op_filter.allow_dummy_ops && IsDummyImplOp(node->type_string())) {
       return false;
     }
+    if (!op_filter.allow_ops_producing_or_consuming_variant &&
+        OpProducesOrConsumesVariant(*node)) {
+      return false;
+    }
     if (!HasXLAKernel(*node, jit_device_type) &&
         !IsCompilableCall(node->def(), jit_device_type, op_filter, depth + 1,
                           lib_runtime)) {
@@ -470,16 +485,15 @@ Status FindCompilationCandidates(
         XlaOpRegistry::GetCompilationDevice(device_type.type(), &registration));
     DeviceType jit_device_type(registration->compilation_device_name);
 
+    bool always_auto_cluster = registration->autoclustering_policy ==
+                               XlaOpRegistry::AutoclusteringPolicy::kAlways;
+
     OperationFilter op_filter;
     op_filter.allow_resource_ops = registration->compile_resource_ops;
-    op_filter.allow_stateful_rng_ops =
-        (registration->autoclustering_policy ==
-         XlaOpRegistry::AutoclusteringPolicy::kAlways);
-    op_filter.allow_control_trigger =
-        (registration->autoclustering_policy ==
-         XlaOpRegistry::AutoclusteringPolicy::kAlways);
-    op_filter.allow_dummy_ops = (registration->autoclustering_policy ==
-                                 XlaOpRegistry::AutoclusteringPolicy::kAlways);
+    op_filter.allow_stateful_rng_ops = always_auto_cluster;
+    op_filter.allow_control_trigger = always_auto_cluster;
+    op_filter.allow_dummy_ops = always_auto_cluster;
+    op_filter.allow_ops_producing_or_consuming_variant = always_auto_cluster;
 
     if (!HasXLAKernel(*node, jit_device_type) &&
         !IsCompilableCall(node->def(), jit_device_type, op_filter, 0,
@@ -503,6 +517,12 @@ Status FindCompilationCandidates(
               << node->type_string() << ")";
       continue;
     }
+    if (!op_filter.allow_ops_producing_or_consuming_variant &&
+        OpProducesOrConsumesVariant(*node)) {
+      VLOG(2) << "Rejecting " << node->name()
+              << ": produces or consumes DT_VARIANT";
+      continue;
+    }
 
     if (!op_filter.allow_resource_ops &&
         (HasResourceOutput(*node) || IsNonResourceVarResourceOp(*node))) {
@@ -639,6 +659,7 @@ bool IsCompilable(FunctionLibraryRuntime* flr, const NodeDef& ndef) {
   op_filter.allow_stateful_rng_ops = true;
   op_filter.allow_control_trigger = true;
   op_filter.allow_dummy_ops = true;
+  op_filter.allow_ops_producing_or_consuming_variant = true;
 
   return IsCompilableCall(ndef, jit_device_type, op_filter, 0, flr);
 }
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index 24d78c0772..bf2c5508ea 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/control_flow_ops_internal.h"
 #include "tensorflow/cc/ops/function_ops.h"
+#include "tensorflow/cc/ops/list_ops.h"
 #include "tensorflow/cc/ops/resource_variable_ops.h"
 #include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
@@ -1147,5 +1148,80 @@ TEST(XlaCompilationTest, DontAutoClusterDummyOps) {
   EXPECT_EQ(clusters["test/check"], "");
 }
 
+TEST(XlaCompilationTest, DontAutoClusterOpsProducingVariant) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  Output a = ops::Placeholder(root.WithOpName("test/a"), DT_INT64);
+  Output b = ops::Placeholder(root.WithOpName("test/b"), DT_INT64);
+
+  Output cast_a = ops::Cast(root.WithOpName("test/cast_a"), a, DT_INT32);
+  Output cast_b = ops::Cast(root.WithOpName("test/cast_b"), b, DT_INT32);
+
+  Output tensor_list_reserve = ops::TensorListReserve(
+      root.WithOpName("test/tensor_list_reserve"), cast_a, cast_b, DT_FLOAT);
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  TF_ASSERT_OK(root.ToGraph(graph.get()));
+
+  TF_ASSERT_OK(MarkForCompilationPassTestHelper::MarkForCompilation(&graph));
+
+  std::unordered_map<string, string> clusters = GetClusters(*graph);
+  EXPECT_EQ(clusters["test/tensor_list_reserve"], "");
+}
+
+TEST(XlaCompilationTest, DontAutoClusterOpsConsumingVariant) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  Output dummy_input =
+      ops::Placeholder(root.WithOpName("test/dummy_input"), DT_INT64);
+  Output variant_input =
+      ops::Placeholder(root.WithOpName("test/variant_input"), DT_VARIANT);
+
+  // Create one more node so that we don't avoid creating a cluster solely
+  // because it would be trivial.
+  Output dummy_cast =
+      ops::Cast(root.WithOpName("test/dummy_cast"), dummy_input, DT_INT32);
+
+  Output tensor_list_element_shape = ops::TensorListElementShape(
+      root.WithOpName("test/tensor_list_element_shape"), variant_input,
+      DT_INT32);
+
+  root.graph()->AddControlEdge(dummy_cast.node(),
+                               tensor_list_element_shape.node());
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  TF_ASSERT_OK(root.ToGraph(graph.get()));
+
+  TF_ASSERT_OK(MarkForCompilationPassTestHelper::MarkForCompilation(&graph));
+
+  std::unordered_map<string, string> clusters = GetClusters(*graph);
+  EXPECT_EQ(clusters["test/tensor_list_element_shape"], "");
+}
+
+TEST(XlaCompilationTest, ClusterOpsProducingVariantIfOnXlaDevice) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  Output a = ops::Placeholder(root.WithOpName("test/a"), DT_INT64);
+  Output b = ops::Placeholder(root.WithOpName("test/b"), DT_INT64);
+
+  Output cast_a = ops::Cast(root.WithOpName("test/cast_a"), a, DT_INT32);
+  Output cast_b = ops::Cast(root.WithOpName("test/cast_b"), b, DT_INT32);
+
+  Output tensor_list_reserve = ops::TensorListReserve(
+      root.WithOpName("test/tensor_list_reserve"), cast_a, cast_b, DT_FLOAT);
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  TF_ASSERT_OK(root.ToGraph(graph.get()));
+
+  string xla_cpu_device = "/job:worker/replica:0/task:0/device:XLA_CPU:0";
+  for (Node* n : graph->nodes()) {
+    if (absl::StartsWith(n->name(), /*prefix=*/"test/")) {
+      n->set_assigned_device_name(xla_cpu_device);
+    }
+  }
+
+  TF_ASSERT_OK(MarkForCompilationPassTestHelper::MarkForCompilation(&graph));
+
+  std::unordered_map<string, string> clusters = GetClusters(*graph);
+  EXPECT_NE(clusters["test/tensor_list_reserve"], "");
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From d26d2aa067455fa6ba2b013fb133f158ed488aab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 17:08:18 -0800
Subject: [PATCH 0635/1554] Allow installation of custom delegates during
 initialization

PiperOrigin-RevId: 222331040
---
 .../lite/tools/benchmark/benchmark_tflite_model.cc       | 9 +++++----
 tensorflow/lite/tools/benchmark/benchmark_tflite_model.h | 7 ++++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index 777d9dde7d..7768b75f76 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -181,7 +181,9 @@ bool PopulateInputLayerInfo(
   return true;
 }
 
-BenchmarkParams GetDefaultParams() {
+}  // namespace
+
+BenchmarkParams BenchmarkTfLiteModel::DefaultParams() {
   BenchmarkParams default_params = BenchmarkModel::DefaultParams();
   default_params.AddParam("graph", BenchmarkParam::Create<std::string>(""));
   default_params.AddParam("input_layer",
@@ -192,10 +194,8 @@ BenchmarkParams GetDefaultParams() {
   return default_params;
 }
 
-}  // namespace
-
 BenchmarkTfLiteModel::BenchmarkTfLiteModel()
-    : BenchmarkTfLiteModel(GetDefaultParams()) {}
+    : BenchmarkTfLiteModel(DefaultParams()) {}
 
 BenchmarkTfLiteModel::BenchmarkTfLiteModel(BenchmarkParams params)
     : BenchmarkModel(std::move(params)) {
@@ -319,6 +319,7 @@ void BenchmarkTfLiteModel::Init() {
   bool use_nnapi = params_.Get<bool>("use_nnapi");
 
   interpreter->UseNNAPI(use_nnapi);
+  ApplyDelegates();
 
   auto interpreter_inputs = interpreter->inputs();
 
diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
index 401ab5427d..83599e644d 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.h
@@ -77,11 +77,16 @@ class BenchmarkTfLiteModel : public BenchmarkModel {
   };
 
  protected:
+  static BenchmarkParams DefaultParams();
   void PrepareInputsAndOutputs() override;
 
- private:
+  // Allows installation of custom delegates during initialization
+  virtual void ApplyDelegates() {}
+
   std::unique_ptr<tflite::FlatBufferModel> model;
   std::unique_ptr<tflite::Interpreter> interpreter;
+
+ private:
   std::vector<InputLayerInfo> inputs;
   ProfilingListener profiling_listener_;
   GemmlowpProfilingListener gemmlowp_profiling_listener_;
-- 
GitLab


From b3e007394680801113f492fa1f5a9784e8502f19 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 17:12:09 -0800
Subject: [PATCH 0636/1554] Add parallel_iterations argument to pfor and
 jacobian functions to control the amount of parallelism.

PiperOrigin-RevId: 222331433
---
 .../ops/parallel_for/control_flow_ops.py      | 108 +++++++++++++++---
 .../ops/parallel_for/control_flow_ops_test.py |  34 +++++-
 .../python/ops/parallel_for/gradients.py      |  25 ++--
 .../python/ops/parallel_for/gradients_test.py |  13 +++
 4 files changed, 155 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops.py b/tensorflow/python/ops/parallel_for/control_flow_ops.py
index ead7ae5478..3c818f3d6c 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops.py
@@ -19,14 +19,16 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops.parallel_for.pfor import PFor
 from tensorflow.python.util import nest
 
 
-def for_loop(loop_fn, loop_fn_dtypes, iters):
+def for_loop(loop_fn, loop_fn_dtypes, iters, parallel_iterations=None):
   """Runs `loop_fn` `iters` times and stacks the outputs.
 
 
@@ -39,6 +41,8 @@ def for_loop(loop_fn, loop_fn_dtypes, iters):
       objects. The shape of these outputs should not depend on the input.
     loop_fn_dtypes: dtypes for the outputs of loop_fn.
     iters: Number of iterations for which to run loop_fn.
+    parallel_iterations: The number of iterations that can be dispatched in
+      parallel. This knob can be used to control the total memory usage.
 
   Returns:
     Returns a nested structure of stacked output tensor objects with the same
@@ -66,11 +70,16 @@ def for_loop(loop_fn, loop_fn_dtypes, iters):
       outputs.append(ta)
     return tuple([i + 1] + outputs)
 
+  if parallel_iterations is not None:
+    extra_args = {"parallel_iterations": parallel_iterations}
+  else:
+    extra_args = {}
   ta_list = control_flow_ops.while_loop(
-      lambda i, *ta: i < iters, while_body, [0] + [
-          tensor_array_ops.TensorArray(dtype, iters)
-          for dtype in flat_loop_fn_dtypes
-      ])[1:]
+      lambda i, *ta: i < iters,
+      while_body,
+      [0] + [tensor_array_ops.TensorArray(dtype, iters)
+             for dtype in flat_loop_fn_dtypes],
+      **extra_args)[1:]
 
   # TODO(rachelim): enable this for sparse tensors
 
@@ -79,7 +88,15 @@ def for_loop(loop_fn, loop_fn_dtypes, iters):
   return nest.pack_sequence_as(loop_fn_dtypes, output)
 
 
-def pfor(loop_fn, iters):
+def _flatten_first_two_dims(x):
+  """Flattens the first two dimensions of x into a single dimension."""
+  old_shape = array_ops.shape(x)
+  new_shape = array_ops.concat([[old_shape[0] * old_shape[1]], old_shape[2:]],
+                               axis=0)
+  return array_ops.reshape(x, new_shape)
+
+
+def pfor(loop_fn, iters, parallel_iterations=None):
   """Equivalent to running `loop_fn` `iters` times and stacking the outputs.
 
   `pfor` has functionality similar to `for_loop`, i.e. running `loop_fn` `iters`
@@ -99,8 +116,8 @@ def pfor(loop_fn, iters):
       reads, etc).
     - Conversion works only on a limited set of kernels for which a converter
       has been registered.
-    - loop_fn cannot currently contain control flow operations like
-      tf.while_loop or tf.cond.
+    - loop_fn has limited support for control flow operations. tf.cond in
+      particular is not supported.
     - `loop_fn` should return nested structure of Tensors or Operations. However
       if an Operation is returned, it should have zero outputs.
     - The shape and dtype of `loop_fn` outputs should not depend on the input
@@ -109,12 +126,21 @@ def pfor(loop_fn, iters):
   Args:
     loop_fn: A function that takes an int32 scalar tf.Tensor object representing
       the iteration number, and returns a possibly nested structure of Tensor or
-      Operation objects.
+      Operation objects. Note that if setting `parallel_iterations` argument to
+      something other than None, `loop_fn` may be called more than once during
+      graph construction. So it may need to avoid mutating global state.
     iters: Number of iterations for which to run loop_fn.
+    parallel_iterations: A knob to control how many iterations are vectorized
+      and dispatched in parallel. The default value of None corresponds to
+      vectorizing all the iterations.  If `parallel_iterations` is smaller than
+      `iters`, then chunks of at most that many iterations are dispatched in
+      sequence. This knob can be used to control the total memory usage.
 
   Returns:
     Returns a nested structure of stacked tensor objects with the same nested
     structure as the output of `loop_fn`.
+  Raises:
+    ValueError: If parallel_iterations is not None and not an integer > 1.
   """
   existing_ops = set(ops.get_default_graph().get_operations())
   with ops.name_scope("loop_body"):
@@ -122,9 +148,61 @@ def pfor(loop_fn, iters):
     loop_fn_outputs = loop_fn(loop_var)
   new_ops = set(ops.get_default_graph().get_operations()) - existing_ops
   iters = ops.convert_to_tensor(iters)
-  with ops.name_scope("pfor"):
-    converter = PFor(loop_var, iters, new_ops)
-    outputs = []
-    for loop_fn_output in nest.flatten(loop_fn_outputs):
-      outputs.append(converter.convert(loop_fn_output))
-    return nest.pack_sequence_as(loop_fn_outputs, outputs)
+  if parallel_iterations is not None:
+    if parallel_iterations < 1:
+      raise ValueError("parallel_iterations must be None or a positive integer")
+    if parallel_iterations == 1:
+      raise ValueError("Found parallel_iterations == 1. Use for_loop instead.")
+    iters_value = tensor_util.constant_value(iters)
+    if iters_value is not None and iters_value < parallel_iterations:
+      parallel_iterations = None
+  if parallel_iterations is None:
+    with ops.name_scope("pfor"):
+      converter = PFor(loop_var, iters, new_ops)
+      outputs = []
+      for loop_fn_output in nest.flatten(loop_fn_outputs):
+        outputs.append(converter.convert(loop_fn_output))
+      return nest.pack_sequence_as(loop_fn_outputs, outputs)
+  else:
+    num_tiled_iterations = iters // parallel_iterations
+    num_remaining_iterations = iters % parallel_iterations
+    # TODO(agarwal): Avoid calling loop_fn twice. Generate the loop body inside
+    # a tf.function and extract the graph from there to vectorize it.
+    with ops.name_scope("pfor_untiled"):
+      converter = PFor(loop_var, num_remaining_iterations, new_ops)
+      remaining_outputs = []
+      flattened_loop_fn_outputs = nest.flatten(loop_fn_outputs)
+      for loop_fn_output in flattened_loop_fn_outputs:
+        remaining_outputs.append(converter.convert(loop_fn_output))
+
+    with ops.name_scope("pfor_tiled"):
+      loop_fn_dtypes = [ops.convert_to_tensor(x).dtype
+                        for x in flattened_loop_fn_outputs]
+
+      def tiled_loop_body(j):
+        offset = j * parallel_iterations + num_remaining_iterations
+
+        def tiled_loop_fn(i):
+          return nest.flatten(loop_fn(i + offset))
+
+        return pfor(tiled_loop_fn, parallel_iterations)
+
+      tiled_outputs = for_loop(tiled_loop_body, loop_fn_dtypes,
+                               num_tiled_iterations, parallel_iterations=1)
+      tiled_outputs = [_flatten_first_two_dims(y) for y in tiled_outputs]
+
+    with ops.name_scope("pfor"):
+      iters_value = tensor_util.constant_value(iters)
+      if iters_value is None or iters_value % parallel_iterations:
+        outputs = control_flow_ops.cond(
+            math_ops.equal(num_remaining_iterations, 0),
+            lambda: tiled_outputs,
+            lambda: [array_ops.concat([x, y], axis=0)
+                     for x, y in zip(remaining_outputs, tiled_outputs)])
+      else:
+        outputs = tiled_outputs
+      return nest.pack_sequence_as(loop_fn_outputs, nest.flatten(outputs))
+
+
+
+
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 7d13014687..4470c0b958 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -73,9 +73,13 @@ class PForTest(test.TestCase):
       else:
         self.assertAllEqual(outputs[i + n], outputs[i])
 
-  def _test_loop_fn(self, loop_fn, iters, loop_fn_dtypes=dtypes.float32):
-    t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters)
-    t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters)
+  def _test_loop_fn(self, loop_fn, iters,
+                    loop_fn_dtypes=dtypes.float32,
+                    parallel_iterations=None):
+    t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters,
+                                    parallel_iterations=parallel_iterations)
+    t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters,
+                                        parallel_iterations=parallel_iterations)
     self.run_and_assert_equal(t1, t2)
 
   def test_op_conversion_fallback_to_while_loop(self):
@@ -96,6 +100,30 @@ class PForTest(test.TestCase):
         loop_fn, 3, loop_fn_dtypes=[dtypes.float32, dtypes.int32])
     flags.FLAGS.op_conversion_fallback_to_while_loop = False
 
+  def test_parallel_iterations(self):
+    for parallel_iterations in [2, 3, 8, 10]:
+      x = random_ops.random_uniform([8, 3])
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        return array_ops.gather(x, i)
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 8, parallel_iterations=parallel_iterations)
+      self._test_loop_fn(loop_fn, 4 * constant_op.constant(2),
+                         parallel_iterations=parallel_iterations)
+
+  def test_parallel_iterations_zero(self):
+    with self.assertRaisesRegexp(ValueError, "positive integer"):
+      pfor_control_flow_ops.pfor(lambda i: 1, 8, parallel_iterations=0)
+    with self.assertRaisesRegexp(TypeError, "positive integer"):
+      pfor_control_flow_ops.for_loop(lambda i: 1, dtypes.int32, 8,
+                                     parallel_iterations=0)
+
+  def test_parallel_iterations_one(self):
+    with self.assertRaisesRegexp(ValueError, "Use for_loop instead"):
+      pfor_control_flow_ops.pfor(lambda i: 1, 8, parallel_iterations=1)
+
 
 class ArrayTest(PForTest):
 
diff --git a/tensorflow/python/ops/parallel_for/gradients.py b/tensorflow/python/ops/parallel_for/gradients.py
index 1f026b3660..3ba1bde347 100644
--- a/tensorflow/python/ops/parallel_for/gradients.py
+++ b/tensorflow/python/ops/parallel_for/gradients.py
@@ -25,7 +25,7 @@ from tensorflow.python.ops.parallel_for import control_flow_ops
 from tensorflow.python.util import nest
 
 
-def jacobian(output, inputs, use_pfor=True):
+def jacobian(output, inputs, use_pfor=True, parallel_iterations=None):
   """Computes jacobian of `output` w.r.t. `inputs`.
 
   Args:
@@ -33,6 +33,8 @@ def jacobian(output, inputs, use_pfor=True):
     inputs: A tensor or a nested structure of tensor objects.
     use_pfor: If true, uses pfor for computing the jacobian. Else uses
       tf.while_loop.
+    parallel_iterations: A knob to control how many iterations and dispatched in
+      parallel. This knob can be used to control the total memory usage.
 
   Returns:
     A tensor or a nested strucutre of tensors with the same structure as
@@ -56,10 +58,14 @@ def jacobian(output, inputs, use_pfor=True):
     output_size = array_ops.shape(output)[0]
 
   if use_pfor:
-    pfor_outputs = control_flow_ops.pfor(loop_fn, output_size)
+    pfor_outputs = control_flow_ops.pfor(
+        loop_fn, output_size, parallel_iterations=parallel_iterations)
   else:
     pfor_outputs = control_flow_ops.for_loop(
-        loop_fn, [output.dtype] * len(flat_inputs), output_size)
+        loop_fn,
+        [output.dtype] * len(flat_inputs),
+        output_size,
+        parallel_iterations=parallel_iterations)
 
   for i, out in enumerate(pfor_outputs):
     if out is not None:
@@ -72,7 +78,7 @@ def jacobian(output, inputs, use_pfor=True):
   return nest.pack_sequence_as(inputs, pfor_outputs)
 
 
-def batch_jacobian(output, inp, use_pfor=True):
+def batch_jacobian(output, inp, use_pfor=True, parallel_iterations=None):
   """Computes and stacks jacobians of `output[i,...]` w.r.t. `input[i,...]`.
 
   e.g.
@@ -87,6 +93,8 @@ def batch_jacobian(output, inp, use_pfor=True):
     inp: A tensor with shape [b, x1, ..., x_m]
     use_pfor: If true, uses pfor for computing the Jacobian. Else uses a
       tf.while_loop.
+    parallel_iterations: A knob to control how many iterations and dispatched in
+      parallel. This knob can be used to control the total memory usage.
 
   Returns:
     A tensor `t` with shape [b, y_1, ..., y_n, x1, ..., x_m] where `t[i, ...]`
@@ -118,10 +126,13 @@ def batch_jacobian(output, inp, use_pfor=True):
     return gradient_ops.gradients(y, inp)[0]
 
   if use_pfor:
-    pfor_output = control_flow_ops.pfor(loop_fn, output_row_size)
+    pfor_output = control_flow_ops.pfor(loop_fn, output_row_size,
+                                        parallel_iterations=parallel_iterations)
   else:
-    pfor_output = control_flow_ops.for_loop(loop_fn, output.dtype,
-                                            output_row_size)
+    pfor_output = control_flow_ops.for_loop(
+        loop_fn, output.dtype,
+        output_row_size,
+        parallel_iterations=parallel_iterations)
   if pfor_output is None:
     return None
   pfor_output = array_ops.reshape(pfor_output,
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index b2be24e110..545c482df8 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -416,6 +416,12 @@ class GradientsTest(test.TestCase):
       self.assertAllClose(ans, pfor_value)
       self.assertAllClose(ans, while_value)
 
+  def test_jacobian_parallel_iterations(self):
+    x = constant_op.constant([[1., 2], [3, 4]])
+    y = math_ops.matmul(x, x)
+    self.assertAllClose(gradients.jacobian(y, x, parallel_iterations=2),
+                        gradients.jacobian(y, x, parallel_iterations=3))
+
   def test_batch_jacobian_bad_shapes(self):
     x = random_ops.random_uniform([2, 2])
     y = random_ops.random_uniform([3, 2])
@@ -459,6 +465,13 @@ class GradientsTest(test.TestCase):
       self.assertAllClose(ans, pfor_value)
       self.assertAllClose(ans, while_value)
 
+  def test_batch_jacobian_parallel_iterations(self):
+    x = constant_op.constant([[1., 2], [3, 4]])
+    w = constant_op.constant([[1., 2, 3, 4], [5, 6, 7, 8]])
+    y = math_ops.matmul(x, w)
+    self.assertAllClose(gradients.batch_jacobian(y, x, parallel_iterations=2),
+                        gradients.batch_jacobian(y, x, parallel_iterations=3))
+
   def test_fc_batch_jacobian(self):
     pfor_jacobian, while_jacobian = create_fc_batch_jacobian(8, 4, 2)
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
-- 
GitLab


From afca7030074c869313fcf75cd84fc7ec3dc99fec Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Tue, 20 Nov 2018 17:19:16 -0800
Subject: [PATCH 0637/1554] Add MirrorPad op to the schema.

PiperOrigin-RevId: 222332230
---
 tensorflow/lite/builtin_ops.h                 |   1 +
 tensorflow/lite/c/builtin_op_data.h           |  10 ++
 .../lite/core/api/flatbuffer_conversions.cc   |  13 ++
 .../writer/option_writer_generator.cc         |   2 +
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |  13 ++
 tensorflow/lite/schema/schema_generated.h     | 170 +++++++++++++++++-
 7 files changed, 204 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index a3843deb00..2300ff4ed2 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -125,6 +125,7 @@ typedef enum {
   kTfLiteBuiltinResizeNearestNeighbor = 97,
   kTfLiteBuiltinLeakyRelu = 98,
   kTfLiteBuiltinSquaredDifference = 99,
+  kTfLiteBuiltinMirrorPad = 100,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h
index 5a2f1fa4b1..33aaac3c80 100644
--- a/tensorflow/lite/c/builtin_op_data.h
+++ b/tensorflow/lite/c/builtin_op_data.h
@@ -35,11 +35,21 @@ typedef enum {
   kTfLitePaddingValid,
 } TfLitePadding;
 
+typedef enum {
+  kTfLiteMirrorPaddingUnknown = 0,
+  kTfLiteMirrorPaddingReflect,
+  kTfLiteMirrorPaddingSymmetric,
+} TfLiteMirrorPaddingMode;
+
 typedef struct {
   int width;
   int height;
 } TfLitePaddingValues;
 
+typedef struct {
+  TfLiteMirrorPaddingMode mode;
+} TfLiteMirrorPaddingParams;
+
 // Possible fused activation functions.
 // TODO(aselle): rename to TfLiteActivation
 typedef enum {
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 3b592a6e0b..aa9b372398 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -629,6 +629,19 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       *builtin_data = reinterpret_cast<void*>(params);
       break;
     }
+    case BuiltinOperator_MIRROR_PAD: {
+      TfLiteMirrorPaddingParams* params =
+          allocator->AllocatePOD<TfLiteMirrorPaddingParams>();
+      auto* mirror_pad_params = op->builtin_options_as_MirrorPadOptions();
+      if (mirror_pad_params != nullptr) {
+        params->mode =
+            mirror_pad_params->mode() == tflite::MirrorPadMode_REFLECT
+                ? TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingReflect
+                : TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingSymmetric;
+      }
+      *builtin_data = reinterpret_cast<void*>(params);
+      break;
+    }
 
     // Below are the ops with no builtin_data strcture.
     case BuiltinOperator_BATCH_TO_SPACE_ND:
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index 26d4a91c71..b44750e8b2 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -67,6 +67,7 @@ static const char* param_structs[] = {"TfLiteConvParams",
                                       "TfLitePackParams",
                                       "TfLiteOneHotParams",
                                       "TfLiteLeakyReluParams",
+                                      "TfLiteMirrorPaddingParams",
                                       nullptr};
 }  // namespace
 
@@ -153,6 +154,7 @@ class OpOptionData {
     op_to_option_["BIDIRECTIONAL_SEQUENCE_RNN"] = "SequenceRNNOptions";
     op_to_option_["UNIDIRECTIONAL_SEQUENCE_RNN"] = "SequenceRNNOptions";
     op_to_option_["UNIDIRECTIONAL_SEQUENCE_RNN"] = "SequenceRNNOptions";
+    op_to_option_["MIRROR_PAD"] = "";  // TODO(karimnosseir): MirrorPadOptions.
     // Manually specified mappings between ops and options (none)
     op_to_option_["EMBEDDING_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 292dedf1c9..58288a8dd4 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -683,6 +683,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_RANGE:
       case tflite::BuiltinOperator_LEAKY_RELU:
       case tflite::BuiltinOperator_SQUARED_DIFFERENCE:
+      case tflite::BuiltinOperator_MIRROR_PAD:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index e40a0409f9..652871d013 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -202,6 +202,7 @@ enum BuiltinOperator : byte {
   RESIZE_NEAREST_NEIGHBOR = 97,
   LEAKY_RELU = 98,
   SQUARED_DIFFERENCE = 99,
+  MIRROR_PAD = 100,
 }
 
 // Options for the builtin operators.
@@ -282,6 +283,7 @@ union BuiltinOptions {
   ResizeNearestNeighborOptions,
   LeakyReluOptions,
   SquaredDifferenceOptions,
+  MirrorPadOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -669,6 +671,17 @@ table LeakyReluOptions {
 table SquaredDifferenceOptions {
 }
 
+enum MirrorPadMode : byte {
+  // Doesn't include borders.
+  REFLECT = 0,
+  // Includes borders.
+  SYMMETRIC = 1,
+}
+
+table MirrorPadOptions {
+  mode:MirrorPadMode;
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index e93cb3d81d..1464c75613 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -259,6 +259,9 @@ struct LeakyReluOptionsT;
 struct SquaredDifferenceOptions;
 struct SquaredDifferenceOptionsT;
 
+struct MirrorPadOptions;
+struct MirrorPadOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -508,11 +511,12 @@ enum BuiltinOperator {
   BuiltinOperator_RESIZE_NEAREST_NEIGHBOR = 97,
   BuiltinOperator_LEAKY_RELU = 98,
   BuiltinOperator_SQUARED_DIFFERENCE = 99,
+  BuiltinOperator_MIRROR_PAD = 100,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_SQUARED_DIFFERENCE
+  BuiltinOperator_MAX = BuiltinOperator_MIRROR_PAD
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[99] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[100] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -612,7 +616,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[99] {
     BuiltinOperator_RANGE,
     BuiltinOperator_RESIZE_NEAREST_NEIGHBOR,
     BuiltinOperator_LEAKY_RELU,
-    BuiltinOperator_SQUARED_DIFFERENCE
+    BuiltinOperator_SQUARED_DIFFERENCE,
+    BuiltinOperator_MIRROR_PAD
   };
   return values;
 }
@@ -719,6 +724,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "RESIZE_NEAREST_NEIGHBOR",
     "LEAKY_RELU",
     "SQUARED_DIFFERENCE",
+    "MIRROR_PAD",
     nullptr
   };
   return names;
@@ -807,11 +813,12 @@ enum BuiltinOptions {
   BuiltinOptions_ResizeNearestNeighborOptions = 74,
   BuiltinOptions_LeakyReluOptions = 75,
   BuiltinOptions_SquaredDifferenceOptions = 76,
+  BuiltinOptions_MirrorPadOptions = 77,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_SquaredDifferenceOptions
+  BuiltinOptions_MAX = BuiltinOptions_MirrorPadOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[77] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[78] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -889,7 +896,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[77] {
     BuiltinOptions_RangeOptions,
     BuiltinOptions_ResizeNearestNeighborOptions,
     BuiltinOptions_LeakyReluOptions,
-    BuiltinOptions_SquaredDifferenceOptions
+    BuiltinOptions_SquaredDifferenceOptions,
+    BuiltinOptions_MirrorPadOptions
   };
   return values;
 }
@@ -973,6 +981,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "ResizeNearestNeighborOptions",
     "LeakyReluOptions",
     "SquaredDifferenceOptions",
+    "MirrorPadOptions",
     nullptr
   };
   return names;
@@ -1291,6 +1300,10 @@ template<> struct BuiltinOptionsTraits<SquaredDifferenceOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_SquaredDifferenceOptions;
 };
 
+template<> struct BuiltinOptionsTraits<MirrorPadOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_MirrorPadOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1930,6 +1943,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_SquaredDifferenceOptions ?
       reinterpret_cast<const SquaredDifferenceOptionsT *>(value) : nullptr;
   }
+  MirrorPadOptionsT *AsMirrorPadOptions() {
+    return type == BuiltinOptions_MirrorPadOptions ?
+      reinterpret_cast<MirrorPadOptionsT *>(value) : nullptr;
+  }
+  const MirrorPadOptionsT *AsMirrorPadOptions() const {
+    return type == BuiltinOptions_MirrorPadOptions ?
+      reinterpret_cast<const MirrorPadOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -2127,6 +2148,35 @@ inline const char *EnumNameCombinerType(CombinerType e) {
   return EnumNamesCombinerType()[index];
 }
 
+enum MirrorPadMode {
+  MirrorPadMode_REFLECT = 0,
+  MirrorPadMode_SYMMETRIC = 1,
+  MirrorPadMode_MIN = MirrorPadMode_REFLECT,
+  MirrorPadMode_MAX = MirrorPadMode_SYMMETRIC
+};
+
+inline const MirrorPadMode (&EnumValuesMirrorPadMode())[2] {
+  static const MirrorPadMode values[] = {
+    MirrorPadMode_REFLECT,
+    MirrorPadMode_SYMMETRIC
+  };
+  return values;
+}
+
+inline const char * const *EnumNamesMirrorPadMode() {
+  static const char * const names[] = {
+    "REFLECT",
+    "SYMMETRIC",
+    nullptr
+  };
+  return names;
+}
+
+inline const char *EnumNameMirrorPadMode(MirrorPadMode e) {
+  const size_t index = static_cast<int>(e);
+  return EnumNamesMirrorPadMode()[index];
+}
+
 enum CustomOptionsFormat {
   CustomOptionsFormat_FLEXBUFFERS = 0,
   CustomOptionsFormat_MIN = CustomOptionsFormat_FLEXBUFFERS,
@@ -6769,6 +6819,60 @@ inline flatbuffers::Offset<SquaredDifferenceOptions> CreateSquaredDifferenceOpti
 
 flatbuffers::Offset<SquaredDifferenceOptions> CreateSquaredDifferenceOptions(flatbuffers::FlatBufferBuilder &_fbb, const SquaredDifferenceOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct MirrorPadOptionsT : public flatbuffers::NativeTable {
+  typedef MirrorPadOptions TableType;
+  MirrorPadMode mode;
+  MirrorPadOptionsT()
+      : mode(MirrorPadMode_REFLECT) {
+  }
+};
+
+struct MirrorPadOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef MirrorPadOptionsT NativeTableType;
+  enum {
+    VT_MODE = 4
+  };
+  MirrorPadMode mode() const {
+    return static_cast<MirrorPadMode>(GetField<int8_t>(VT_MODE, 0));
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyField<int8_t>(verifier, VT_MODE) &&
+           verifier.EndTable();
+  }
+  MirrorPadOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(MirrorPadOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<MirrorPadOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct MirrorPadOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_mode(MirrorPadMode mode) {
+    fbb_.AddElement<int8_t>(MirrorPadOptions::VT_MODE, static_cast<int8_t>(mode), 0);
+  }
+  explicit MirrorPadOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  MirrorPadOptionsBuilder &operator=(const MirrorPadOptionsBuilder &);
+  flatbuffers::Offset<MirrorPadOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<MirrorPadOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<MirrorPadOptions> CreateMirrorPadOptions(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    MirrorPadMode mode = MirrorPadMode_REFLECT) {
+  MirrorPadOptionsBuilder builder_(_fbb);
+  builder_.add_mode(mode);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<MirrorPadOptions> CreateMirrorPadOptions(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -7130,6 +7234,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const SquaredDifferenceOptions *builtin_options_as_SquaredDifferenceOptions() const {
     return builtin_options_type() == BuiltinOptions_SquaredDifferenceOptions ? static_cast<const SquaredDifferenceOptions *>(builtin_options()) : nullptr;
   }
+  const MirrorPadOptions *builtin_options_as_MirrorPadOptions() const {
+    return builtin_options_type() == BuiltinOptions_MirrorPadOptions ? static_cast<const MirrorPadOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7465,6 +7572,10 @@ template<> inline const SquaredDifferenceOptions *Operator::builtin_options_as<S
   return builtin_options_as_SquaredDifferenceOptions();
 }
 
+template<> inline const MirrorPadOptions *Operator::builtin_options_as<MirrorPadOptions>() const {
+  return builtin_options_as_MirrorPadOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -10005,6 +10116,32 @@ inline flatbuffers::Offset<SquaredDifferenceOptions> CreateSquaredDifferenceOpti
       _fbb);
 }
 
+inline MirrorPadOptionsT *MirrorPadOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new MirrorPadOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void MirrorPadOptions::UnPackTo(MirrorPadOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+  { auto _e = mode(); _o->mode = _e; };
+}
+
+inline flatbuffers::Offset<MirrorPadOptions> MirrorPadOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateMirrorPadOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<MirrorPadOptions> CreateMirrorPadOptions(flatbuffers::FlatBufferBuilder &_fbb, const MirrorPadOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MirrorPadOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  auto _mode = _o->mode;
+  return tflite::CreateMirrorPadOptions(
+      _fbb,
+      _mode);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -10567,6 +10704,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const SquaredDifferenceOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_MirrorPadOptions: {
+      auto ptr = reinterpret_cast<const MirrorPadOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -10889,6 +11030,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const SquaredDifferenceOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_MirrorPadOptions: {
+      auto ptr = reinterpret_cast<const MirrorPadOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11199,6 +11344,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const SquaredDifferenceOptionsT *>(value);
       return CreateSquaredDifferenceOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_MirrorPadOptions: {
+      auto ptr = reinterpret_cast<const MirrorPadOptionsT *>(value);
+      return CreateMirrorPadOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -11509,6 +11658,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new SquaredDifferenceOptionsT(*reinterpret_cast<SquaredDifferenceOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_MirrorPadOptions: {
+      value = new MirrorPadOptionsT(*reinterpret_cast<MirrorPadOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -11896,6 +12049,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_MirrorPadOptions: {
+      auto ptr = reinterpret_cast<MirrorPadOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
GitLab


From 785cac6c8d43dadca123403f2212987a6230fd84 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 20 Nov 2018 17:22:34 -0800
Subject: [PATCH 0638/1554] Update tf_ops_compatibility

Alphabetize the list, and include the following ops:
 * Fill
 * FloorDiv
 * FloorMod
 * LeakyRelu
 * Range
 * ResizeNearestNeighbor

PiperOrigin-RevId: 222332617
---
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 216 ++++++++++++------
 1 file changed, 143 insertions(+), 73 deletions(-)

diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index 5a7bc2deeb..2864c6aaf4 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -1,4 +1,3 @@
-
 # TensorFlow Lite & TensorFlow Compatibility Guide
 
 TensorFlow Lite supports a number of TensorFlow operations used in common
@@ -155,6 +154,30 @@ Options {
 }
 ```
 
+**ARG_MAX**
+
+```
+Inputs {
+  0: a tensor
+  1: a tensor
+}
+Outputs {
+  0: A tensor of indices of maximum values.
+}
+```
+
+**ARG_MIN**
+
+```
+Inputs {
+  0: a tensor
+  1: a tensor
+}
+Outputs {
+  0: A tensor of indices of minium values.
+}
+```
+
 **AVERAGE_POOL_2D**
 
 ```
@@ -281,6 +304,18 @@ Outputs {
 }
 ```
 
+**FILL**
+
+```
+Inputs {
+  0: a 1D tensor
+  1: a 0D (scalar) tensor
+}
+Outputs {
+  0: A tensor of shape `tensor 0` filled with the value in `tensor 1`.
+}
+```
+
 **FLOOR**
 
 ```
@@ -292,6 +327,30 @@ outputs: {
 }
 ```
 
+**FLOOR_DIV**
+
+```
+Inputs {
+  0: a tensor
+  1: a tensor
+}
+Outputs {
+  0: result of computing element-wise floor of `tensor 0` divided by `tensor 1`.
+}
+```
+
+**FLOOR_MOD**
+
+```
+Inputs {
+  0: a tensor
+  1: a tensor
+}
+Outputs {
+  0: result of computing element-wise floor of `tensor 0` modulo `tensor 1`.
+}
+```
+
 **FULLY_CONNECTED**
 
 ```
@@ -393,6 +452,20 @@ Options {
 }
 ```
 
+**LEAKY_RELU**
+
+```
+Inputs {
+  0: a tensor
+}
+Outputs {
+  0: a tensor equivalent to max(input, input * alpha)
+}
+Options {
+  alpha
+}
+```
+
 **LESS**
 
 ```
@@ -436,6 +509,18 @@ Options {
 }
 ```
 
+**LOGICAL_OR**
+
+```
+Inputs {
+  0: a list of tensors.
+  1: a list of tensors.
+}
+Outputs {
+  0: A tensor of logical_or output tensors.
+}
+```
+
 **LOGISTIC**
 
 ```
@@ -513,6 +598,18 @@ Outputs {
 }
 ```
 
+**PACK**
+
+```
+Inputs {
+  0: a list of tensors.
+  1: an integer.
+}
+Outputs {
+  0: A tensor of stacked tensors.
+}
+```
+
 **PAD**
 
 ```
@@ -554,6 +651,35 @@ Outputs {
 }
 ```
 
+**POW**
+
+```
+Inputs {
+  0: a tensor
+  1: a tensor
+}
+Outputs {
+  0: elementwise pow of the input tensors
+}
+```
+
+**RANGE**
+
+```
+Inputs {
+  0: a 0D (scalar) tensor
+  1: a 0D (scalar) tensor
+  2: a 0D (scalar) tensor
+}
+Outputs {
+  0: A 1D tensor of type `dtype` defined by a sequence where `tensor 0` is the
+  start, `tensor 1` is the limit, and `tensor 2` is the delta.
+}
+Options {
+  dtype
+}
+```
+
 **RELU**
 
 ```
@@ -602,6 +728,22 @@ Options {
 }
 ```
 
+**RESIZE_NEAREST_NEIGHBOR**
+
+```
+Inputs {
+  0: a 4D tensor
+  1: a 1D tensor with 2 elements
+}
+Outputs {
+  0: A tensor of type `tensor 0` resized according to `tensor 1` heigh/width values
+  using nearest neighbors interpolation.
+}
+Options {
+  align_corners
+}
+```
+
 **RSQRT**
 
 ```
@@ -796,66 +938,6 @@ Outputs {
 }
 ```
 
-**POW**
-
-```
-Inputs {
-  0: a tensor
-  1: a tensor
-}
-Outputs {
-  0: elementwise pow of the input tensors
-}
-```
-
-**ARG_MAX**
-
-```
-Inputs {
-  0: a tensor
-  1: a tensor
-}
-Outputs {
-  0: A tensor of indices of maximum values.
-}
-```
-
-**ARG_MIN**
-
-```
-Inputs {
-  0: a tensor
-  1: a tensor
-}
-Outputs {
-  0: A tensor of indices of minium values.
-}
-```
-
-**PACK**
-
-```
-Inputs {
-  0: a list of tensors.
-  1: an integer.
-}
-Outputs {
-  0: A tensor of stacked tensors.
-}
-```
-
-**LOGICAL_OR**
-
-```
-Inputs {
-  0: a list of tensors.
-  1: a list of tensors.
-}
-Outputs {
-  0: A tensor of logical_or output tensors.
-}
-```
-
 **UNPACK**
 
 ```
@@ -869,18 +951,6 @@ Outputs {
 }
 ```
 
-**FLOOR_DIV**
-
-```
-Inputs {
-  0: a list of tensors.
-  1: a list of tensors.
-}
-Outputs {
-  0: A tensor of floor_div output tensors.
-}
-```
-
 **ZEROS_LIKE**
 
 ```
-- 
GitLab


From 3e3f71b3e4ac9872895f6ddc7482f97b7b22448c Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 20 Nov 2018 17:24:25 -0800
Subject: [PATCH 0639/1554] Make deprecated Split function use new one rather
 than the other way

This helps avoid a compiler warning since the deprecated function is no
longer being used in the header.

PiperOrigin-RevId: 222332803
---
 tensorflow/core/util/sparse/sparse_tensor.h | 50 ++++++---------------
 1 file changed, 14 insertions(+), 36 deletions(-)

diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h
index b9ca8ab395..89c163aa51 100644
--- a/tensorflow/core/util/sparse/sparse_tensor.h
+++ b/tensorflow/core/util/sparse/sparse_tensor.h
@@ -238,15 +238,6 @@ class SparseTensor {
   static Status Split(const SparseTensor& tensor, const int split_dim,
                       const int num_split, std::vector<SparseTensor>* result);
 
-  template <typename T>
-  ABSL_DEPRECATED(
-      "Use the form of Split() that takes an output pointer and returns a "
-      "status instead.")
-  static std::vector<SparseTensor> Split(const SparseTensor& tensor,
-                                         const int split_dim,
-                                         const int num_split,
-                                         Status* status = nullptr);
-
   // Slice() will slice the input SparseTensor into a SparseTensor based on
   // specified start and size. Both start and size are 1-D array with each
   // element of the array representing one dimension. The start is the start
@@ -578,10 +569,9 @@ SparseTensor SparseTensor::Concat(
 }
 
 template <typename T>
-std::vector<SparseTensor> SparseTensor::Split(const SparseTensor& input_tensor,
-                                              const int split_dim,
-                                              const int num_split,
-                                              Status* status /* = nullptr */) {
+Status SparseTensor::Split(const SparseTensor& input_tensor,
+                           const int split_dim, const int num_split,
+                           std::vector<SparseTensor>* result) {
   std::vector<Tensor> output_indices;
   std::vector<Tensor> output_values;
   std::vector<TensorShape> output_shapes;
@@ -601,17 +591,15 @@ std::vector<SparseTensor> SparseTensor::Split(const SparseTensor& input_tensor,
   const int split_dim_size = input_tensor.shape()[split_dim];
   const int split_size = split_dim_size / num_split;
 
-  if (!(num_split > 0 && num_split <= split_dim_size) && status != nullptr) {
-    *status = Status(error::INVALID_ARGUMENT,
-                     strings::StrCat("num_split must be in the interval (0, ",
-                                     split_dim_size, "]"));
-    return {};
+  if (!(num_split > 0 && num_split <= split_dim_size)) {
+    return Status(error::INVALID_ARGUMENT,
+                  strings::StrCat("num_split must be in the interval (0, ",
+                                  split_dim_size, "]"));
   }
   if (!(split_dim >= 0 && split_dim < num_dim)) {
-    *status = Status(
+    return Status(
         error::INVALID_ARGUMENT,
         strings::StrCat("num_dim must be in the interval [0, ", num_dim, ")"));
-    return {};
   }
 
   const int residual = split_dim_size % num_split;
@@ -649,28 +637,18 @@ std::vector<SparseTensor> SparseTensor::Split(const SparseTensor& input_tensor,
     }
   }
 
-  std::vector<SparseTensor> output_tensors;
-  output_tensors.reserve(num_split);
+  result->clear();
+  result->reserve(num_split);
   for (int i = 0; i < num_split; ++i) {
     SparseTensor tensor;
     Status create_status =
         Create(output_indices[i], output_values[i], output_shapes[i], &tensor);
-    if (!create_status.ok() && status != nullptr) {
-      *status = create_status;
-      return {};
+    if (!create_status.ok()) {
+      return create_status;
     }
-    output_tensors.push_back(std::move(tensor));
+    result->push_back(std::move(tensor));
   }
-  return output_tensors;
-}
-
-template <typename T>
-Status SparseTensor::Split(const SparseTensor& input_tensor,
-                           const int split_dim, const int num_split,
-                           std::vector<SparseTensor>* result) {
-  Status status;
-  *result = Split<T>(input_tensor, split_dim, num_split, &status);
-  return status;
+  return Status::OK();
 }
 
 template <typename T>
-- 
GitLab


From 96b0d298df643ea835a8e2aa7265cb365409205d Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 20 Nov 2018 17:28:20 -0800
Subject: [PATCH 0640/1554] Add performance test for LSTM that with grappler
 swap the implementation.

PiperOrigin-RevId: 222333112
---
 .../python/keras/layers/unified_rnn_test.py   | 190 ++++++++++++++++--
 1 file changed, 176 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index a16d0f383a..22d29f191b 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import time
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
@@ -37,6 +38,7 @@ from tensorflow.python.keras import regularizers
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine.base_layer import \
   InputSpec
+from tensorflow.python.keras.layers.cudnn_recurrent import CuDNNLSTM
 from tensorflow.python.keras.layers.recurrent import RNN
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import array_ops
@@ -47,20 +49,22 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import gradient_descent
 
 
 class RNNTest(test.TestCase):
 
-  def test_unifiedRNN(self):
+  def setUp(self):
     rewrites = rewriter_config_pb2.RewriterConfig()
     rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
     customer_optimizer = rewrites.custom_optimizers.add()
     customer_optimizer.name = 'ExperimentalImplementationSelector'
     rewrites.min_graph_nodes = -1
     graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
-    config = config_pb2.ConfigProto(graph_options=graph_options)
+    self.config = config_pb2.ConfigProto(graph_options=graph_options)
 
+  def test_unifiedRNN(self):
     input_shape = 10
     rnn_state_size = 8
     output_shape = 8
@@ -68,13 +72,13 @@ class RNNTest(test.TestCase):
     batch = 100
     epoch = 1
 
-    with ops.Graph().as_default(), session.Session(config=config) as sess:
+    with ops.Graph().as_default(), session.Session(config=self.config) as sess:
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=batch,
           test_samples=0,
           input_shape=(timestep, input_shape),
           num_classes=output_shape)
-      y_train = keras.utils.to_categorical(y_train)
+      y_train = keras.utils.to_categorical(y_train, output_shape)
 
       layer = UnifiedLSTM(rnn_state_size)
 
@@ -104,18 +108,36 @@ class RNNTest(test.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
+  def test_keras_model_with_lstm(self):
+    input_shape = 10
+    rnn_state_size = 8
+    output_shape = 8
+    timestep = 4
+    batch = 100
+    epoch = 10
+
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=batch,
+        test_samples=0,
+        input_shape=(timestep, input_shape),
+        num_classes=output_shape)
+    y_train = keras.utils.to_categorical(y_train, output_shape)
+
+    K.set_session(session.Session(config=self.config))
+    layer = UnifiedLSTM(rnn_state_size)
+
+    inputs = keras.layers.Input(
+        shape=[timestep, input_shape], dtype=dtypes.float32)
+
+    outputs, unused_runtime = layer(inputs)
+    model = keras.models.Model(inputs, outputs)
+    model.compile('rmsprop', loss='mse')
+    model.fit(x_train, y_train, epochs=epoch)
+
   def test_unifiedRNN_with_cond(self):
     # This test is to demonstrate the graph rewrite of grappler plugin under
     # the condition that the function returns different number of internal
     # states.
-    rewrites = rewriter_config_pb2.RewriterConfig()
-    rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
-    customer_optimizer = rewrites.custom_optimizers.add()
-    customer_optimizer.name = 'ExperimentalImplementationSelector'
-    rewrites.min_graph_nodes = -1
-    graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
-    config = config_pb2.ConfigProto(graph_options=graph_options)
-
     input_shape = 10
     rnn_state_size = 8
     output_shape = 8
@@ -123,13 +145,13 @@ class RNNTest(test.TestCase):
     batch = 100
     epoch = 1
 
-    with ops.Graph().as_default(), session.Session(config=config) as sess:
+    with ops.Graph().as_default(), session.Session(config=self.config) as sess:
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=batch,
           test_samples=0,
           input_shape=(timestep, input_shape),
           num_classes=output_shape)
-      y_train = keras.utils.to_categorical(y_train)
+      y_train = keras.utils.to_categorical(y_train, output_shape)
 
       layer = UnifiedLSTM(rnn_state_size)
 
@@ -169,6 +191,146 @@ class RNNTest(test.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
+  def _time_performance_run_cudnn_lstm(self, test_config, x_train, y_train):
+    # Get the performance number for standard Cudnn LSTM
+    input_shape = test_config['input_shape']
+    rnn_state_size = test_config['rnn_state_size']
+    timestep = test_config['timestep']
+    epoch = test_config['epoch']
+    warmup_epoch = test_config['warmup_epoch']
+
+    ops.reset_default_graph()
+    with self.test_session(use_gpu=True):
+      cudnn_lstm_layer = CuDNNLSTM(rnn_state_size)
+      inputs = keras.layers.Input(
+          shape=[timestep, input_shape], dtype=dtypes.float32)
+
+      outputs = cudnn_lstm_layer(inputs)
+      model = keras.models.Model(inputs, outputs)
+      model.compile('sgd', 'mse')
+
+      total_duration = 0
+      for i in range(epoch):
+        start_time = time.time()
+        model.fit(x_train, y_train)
+        end_time = time.time()
+        if i >= warmup_epoch:
+          duration_per_epoch = end_time - start_time
+          total_duration += duration_per_epoch
+          logging.vlog(2, '%s: Time consumed for epoch %d is: %s',
+                       'CuDNN LSTM', i, duration_per_epoch)
+      logging.info('Average performance for %s per epoch is: %s',
+                   'CuDNN LSTM', (total_duration / epoch))
+      return total_duration / epoch
+
+  def _time_performance_run_unifed_lstm_gpu(
+      self, test_config, x_train, y_train):
+    # Get performance number for Unified_LSTM with grappler swap the impl
+    input_shape = test_config['input_shape']
+    rnn_state_size = test_config['rnn_state_size']
+    timestep = test_config['timestep']
+    epoch = test_config['epoch']
+    warmup_epoch = test_config['warmup_epoch']
+
+    ops.reset_default_graph()
+    K.set_session(session.Session(config=self.config))
+    layer = UnifiedLSTM(rnn_state_size)
+    inputs = keras.layers.Input(
+        shape=[timestep, input_shape], dtype=dtypes.float32)
+
+    outputs, _ = layer(inputs)
+    model = keras.models.Model(inputs, outputs)
+    model.compile('sgd', 'mse')
+
+    total_duration = 0
+    for i in range(epoch):
+      start_time = time.time()
+      model.fit(x_train, y_train)
+      end_time = time.time()
+      if i >= warmup_epoch:
+        duration_per_epoch = end_time - start_time
+        total_duration += duration_per_epoch
+        logging.vlog(2, '%s: Time consumed for epoch %d is: %s',
+                     'Unified LSTM', i, duration_per_epoch)
+    logging.info('Average performance for %s per epoch is: %s',
+                 'Unified LSTM', (total_duration / epoch))
+    return total_duration / epoch
+
+  def _time_performance_run_normal_lstm(
+      self, test_config, x_train, y_train):
+    # Get performance number for standard LSTM on GPU.
+    input_shape = test_config['input_shape']
+    rnn_state_size = test_config['rnn_state_size']
+    timestep = test_config['timestep']
+    epoch = test_config['epoch']
+    warmup_epoch = test_config['warmup_epoch']
+
+    ops.reset_default_graph()
+    with self.test_session(use_gpu=True):
+      layer = keras.layers.LSTM(rnn_state_size)
+      inputs = keras.layers.Input(
+          shape=[timestep, input_shape], dtype=dtypes.float32)
+
+      outputs = layer(inputs)
+      model = keras.models.Model(inputs, outputs)
+      model.compile('sgd', 'mse')
+
+      total_duration = 0
+      for i in range(epoch):
+        start_time = time.time()
+        model.fit(x_train, y_train)
+        end_time = time.time()
+        if i >= warmup_epoch:
+          duration_per_epoch = end_time - start_time
+          total_duration += duration_per_epoch
+          logging.vlog(2, '%s: Time consumed for epoch %d is: %s',
+                       'Normal LSTM', i, duration_per_epoch)
+      logging.info('Average performance for %s per epoch is: %s',
+                   'Normal LSTM', (total_duration / epoch))
+      return total_duration / epoch
+
+  def test_performance_with_standard_cudnn_impl(self):
+    if not test.is_gpu_available():
+      self.skipTest('performance test will only run on GPU')
+
+    test_config = {
+        'input_shape': 128,
+        'rnn_state_size': 64,
+        'output_shape': 64,
+        'timestep': 50,
+        'epoch': 20,
+        # The performance for warmup epoch is ignored.
+        'warmup_epoch': 1,
+    }
+    batch = 64
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=batch,
+        test_samples=0,
+        input_shape=(test_config['timestep'], test_config['input_shape']),
+        num_classes=test_config['output_shape'])
+    y_train = keras.utils.to_categorical(y_train, test_config['output_shape'])
+
+    cudnn_duration = self._time_performance_run_cudnn_lstm(
+        test_config, x_train, y_train)
+    unified_lstm_gpu_duration = self._time_performance_run_unifed_lstm_gpu(
+        test_config, x_train, y_train)
+    normal_lstm_duration = self._time_performance_run_normal_lstm(
+        test_config, x_train, y_train)
+
+    cudnn_vs_unified = cudnn_duration / unified_lstm_gpu_duration
+    unified_vs_normal = normal_lstm_duration / unified_lstm_gpu_duration
+    # Assert the performance diff should be within 80% of the native cudnn impl.
+    self.assertGreaterEqual(
+        cudnn_vs_unified, 0.80,
+        'Expect the performance of Unified LSTM is within 80% of CuDNN LSTM, '
+        'but got {}'.format(cudnn_vs_unified * 100))
+    # Assert the performance diff between CPU impl and GPU impl should be more
+    # than 5 times.
+    self.assertGreaterEqual(
+        unified_vs_normal, 5,
+        'Expect the performance of Unified LSTM is more than 5 times of normal '
+        'LSTM, but got {}'.format(unified_vs_normal))
+
 
 class UnifiedLSTM(RNN):
 
-- 
GitLab


From a102a6a71844e194f3946f6318768c5367f1f16b Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 20 Nov 2018 17:33:45 -0800
Subject: [PATCH 0641/1554] Add use_gpu, force_gpu & force_gpu helpers in
 test_util

In order to make tests compatible for V2 behavior we should avoid using
sessions and instead use device contexts. To support the different
desired behaviors we add 3 helpers in test_util and covert a few tests
to make use of them.

PiperOrigin-RevId: 222333755
---
 tensorflow/compiler/tests/rmsprop_test.py     |   4 +-
 tensorflow/python/framework/test_util.py      |  21 +
 .../python/keras/engine/training_gpu_test.py  |   2 +-
 .../python/keras/optimizer_v2/rmsprop_test.py |  21 +-
 .../python/kernel_tests/array_ops_test.py     |  13 +-
 .../python/kernel_tests/bitcast_op_test.py    |   3 +-
 .../compare_and_bitpack_op_test.py            |   3 +-
 .../kernel_tests/determinant_op_test.py       |   3 +-
 .../kernel_tests/dynamic_stitch_op_test.py    |  15 +-
 .../extract_image_patches_op_test.py          |   3 +-
 .../extract_volume_patches_op_test.py         |   3 +-
 .../matrix_exponential_op_test.py             |   3 +-
 .../kernel_tests/matrix_logarithm_op_test.py  |   3 +-
 .../python/kernel_tests/numerics_test.py      |   7 +-
 .../python/kernel_tests/scatter_ops_test.py   |   9 +-
 .../kernel_tests/spacetodepth_op_test.py      |  18 +-
 .../python/kernel_tests/unstack_op_test.py    |  24 +-
 .../python/kernel_tests/zero_division_test.py |   3 +-
 tensorflow/python/ops/nn_test.py              | 673 +++++++++---------
 tensorflow/python/training/rmsprop_test.py    |  37 +-
 20 files changed, 449 insertions(+), 419 deletions(-)

diff --git a/tensorflow/compiler/tests/rmsprop_test.py b/tensorflow/compiler/tests/rmsprop_test.py
index 5138a4a2a9..dc3e90b4af 100644
--- a/tensorflow/compiler/tests/rmsprop_test.py
+++ b/tensorflow/compiler/tests/rmsprop_test.py
@@ -76,7 +76,7 @@ class RmspropTest(xla_test.XLATestCase):
           rms_opt = rmsprop.RMSPropOptimizer(learning_rate, centered=centered)
           rms_update = rms_opt.apply_gradients(
               zip([grads0, grads1], [var0, var1]))
-          variables.global_variables_initializer().run()
+          self.evaluate(variables.global_variables_initializer())
 
           mg0 = rms_opt.get_slot(var0, "mg")
           self.assertEqual(mg0 is not None, centered)
@@ -97,7 +97,7 @@ class RmspropTest(xla_test.XLATestCase):
 
           # Run 3 steps of RMSProp
           for _ in range(3):
-            rms_update.run()
+            self.evaluate(rms_update)
 
             var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
                 var0_np,
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 897122746b..bf0ebaea99 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1062,6 +1062,27 @@ def device(use_gpu):
     yield
 
 
+@contextlib.contextmanager
+def use_gpu():
+  """Uses gpu when requested and available."""
+  with device(use_gpu=True):
+    yield
+
+
+@contextlib.contextmanager
+def force_gpu():
+  """Force the gpu to be used."""
+  with ops.device("/device:GPU:0"):
+    yield
+
+
+@contextlib.contextmanager
+def force_cpu():
+  """Force the cpu to be used."""
+  with ops.device("/device:CPU:0"):
+    yield
+
+
 class CapturedWrites(object):
   """A utility class to load the captured writes made to a stream."""
 
diff --git a/tensorflow/python/keras/engine/training_gpu_test.py b/tensorflow/python/keras/engine/training_gpu_test.py
index 596d085f3f..45dcfe4399 100644
--- a/tensorflow/python/keras/engine/training_gpu_test.py
+++ b/tensorflow/python/keras/engine/training_gpu_test.py
@@ -69,7 +69,7 @@ class TrainingGPUTest(test.TestCase):
       return simple_model
 
     if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
+      with test_util.use_gpu():
         losses_to_test = ['sparse_categorical_crossentropy',
                           'categorical_crossentropy', 'binary_crossentropy']
 
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 2ac090724e..62b64d5cf9 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
@@ -87,7 +88,7 @@ class RMSpropOptimizerTest(test.TestCase):
 
   def testDense(self):
     for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         # Initialize variables for numpy implementation.
         var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
         grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
@@ -106,7 +107,7 @@ class RMSpropOptimizerTest(test.TestCase):
             centered=centered)
 
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
 
         if centered:
           mg0 = opt.get_slot(var0, "mg")
@@ -137,7 +138,7 @@ class RMSpropOptimizerTest(test.TestCase):
 
         # Run 4 steps of RMSprop
         for _ in range(1, 5):
-          update.run()
+          self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
               var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate, rho,
@@ -171,11 +172,11 @@ class RMSpropOptimizerTest(test.TestCase):
             epsilon=0.0,
             centered=False).minimize(
                 loss, var_list=[var0])
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
         # Fetch params to validate initial values
         self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
-        sgd_op.run()
+        self.evaluate(sgd_op)
         # Validate updated params
         self.assertAllCloseAccordingToType([[0., 1.]],
                                            self.evaluate(var0),
@@ -195,11 +196,11 @@ class RMSpropOptimizerTest(test.TestCase):
             epsilon=1.0,
             centered=True).minimize(
                 loss, var_list=[var0])
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
         # Fetch params to validate initial values
         self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
-        sgd_op.run()
+        self.evaluate(sgd_op)
         # Validate updated params
         self.assertAllCloseAccordingToType([[-111, -138]],
                                            self.evaluate(var0),
@@ -207,7 +208,7 @@ class RMSpropOptimizerTest(test.TestCase):
 
   def testSparse(self):
     for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         # Initialize variables for numpy implementation.
         var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
         grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype)
@@ -231,7 +232,7 @@ class RMSpropOptimizerTest(test.TestCase):
             epsilon=epsilon,
             centered=centered)
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
 
         if centered:
           mg0 = opt.get_slot(var0, "mg")
@@ -263,7 +264,7 @@ class RMSpropOptimizerTest(test.TestCase):
 
         # Run 4 steps of RMSprop
         for _ in range(1, 5):
-          update.run()
+          self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
               var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np,
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index b9d9d54c98..afc158f697 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -556,7 +556,8 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
   def testInt64GPU(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
-    with self.session(use_gpu=True, force_gpu=True):
+
+    with test_util.force_gpu():
       x = constant_op.constant([1., 2., 3.])
       begin = constant_op.constant([2], dtype=dtypes.int64)
       end = constant_op.constant([3], dtype=dtypes.int64)
@@ -1187,18 +1188,18 @@ class IdentityTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(x.numpy(), y.numpy())
         self.assertTrue(device in y.device.lower())
 
-      with ops.device("gpu:0"):
+      with test_util.force_gpu():
         a = constant_op.constant([[2], [3]], dtype=dtypes.float32)
-      with ops.device("gpu:0"):
+      with test_util.force_gpu():
         b = array_ops.identity(a)
         _test(a, b, "gpu")
-      with ops.device("cpu:0"):
+      with test_util.force_cpu():
         c = array_ops.identity(b)
         _test(b, c, "cpu")
-      with ops.device("cpu:0"):
+      with test_util.force_cpu():
         d = array_ops.identity(c)
         _test(c, d, "cpu")
-      with ops.device("gpu:0"):
+      with test_util.force_gpu():
         e = array_ops.identity(d)
         _test(d, e, "gpu")
 
diff --git a/tensorflow/python/kernel_tests/bitcast_op_test.py b/tensorflow/python/kernel_tests/bitcast_op_test.py
index 4dcf218d7c..5ceffcfeda 100644
--- a/tensorflow/python/kernel_tests/bitcast_op_test.py
+++ b/tensorflow/python/kernel_tests/bitcast_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -28,7 +29,7 @@ from tensorflow.python.platform import test
 class BitcastTest(test.TestCase):
 
   def _testBitcast(self, x, datatype, shape):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       tf_ans = array_ops.bitcast(x, datatype)
       out = self.evaluate(tf_ans)
       buff_after = memoryview(out).tobytes()
diff --git a/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py b/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py
index e1928c5a1c..215ea97f36 100644
--- a/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py
+++ b/tensorflow/python/kernel_tests/compare_and_bitpack_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -30,7 +31,7 @@ class CompareAndBitpackTest(test.TestCase):
                              x, threshold,
                              truth,
                              expected_err_re=None):
-    with self.cached_session(use_gpu=True):
+    with test_util.use_gpu():
       ans = math_ops.compare_and_bitpack(x, threshold)
       if expected_err_re is None:
         tf_ans = self.evaluate(ans)
diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py
index 78c1d74da0..602ceb6ebd 100644
--- a/tensorflow/python/kernel_tests/determinant_op_test.py
+++ b/tensorflow/python/kernel_tests/determinant_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_linalg_ops
 from tensorflow.python.ops import linalg_ops
@@ -62,7 +63,7 @@ class DeterminantOpTest(test.TestCase):
         atol=5e-5)
 
   def _compareDeterminant(self, matrix_x):
-    with self.cached_session(use_gpu=True):
+    with test_util.use_gpu():
       self._compareDeterminantBase(matrix_x,
                                    linalg_ops.matrix_determinant(matrix_x))
       self._compareLogDeterminantBase(
diff --git a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
index c0b0e3f193..3d063c4e0e 100644
--- a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -36,7 +37,7 @@ class DynamicStitchTestBase(object):
     self.stitch_op = stitch_op
 
   def testScalar(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       indices = [constant_op.constant(0), constant_op.constant(1)]
       data = [constant_op.constant(40), constant_op.constant(60)]
       for step in -1, 1:
@@ -47,7 +48,7 @@ class DynamicStitchTestBase(object):
         self.assertEqual([2], stitched_t.get_shape().as_list())
 
   def testShapeInferenceForScalarWithNonConstantIndices(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       indices = [
           array_ops.placeholder(dtype=dtypes.int32),
           constant_op.constant(1)
@@ -61,7 +62,7 @@ class DynamicStitchTestBase(object):
         self.assertEqual([None], stitched_t.get_shape().as_list())
 
   def testSimpleOneDimensional(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       # Test various datatypes in the simple case to ensure that the op was
       # registered under those types.
       dtypes_to_test = [
@@ -84,7 +85,7 @@ class DynamicStitchTestBase(object):
         self.assertEqual([8], stitched_t.get_shape().as_list())
 
   def testOneListOneDimensional(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       indices = [constant_op.constant([1, 6, 2, 3, 5, 0, 4, 7])]
       data = [constant_op.constant([10, 60, 20, 30, 50, 0, 40, 70])]
       stitched_t = self.stitch_op(indices, data)
@@ -94,7 +95,7 @@ class DynamicStitchTestBase(object):
       self.assertEqual([8], stitched_t.get_shape().as_list())
 
   def testSimpleTwoDimensional(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       indices = [
           constant_op.constant([0, 4, 7]),
           constant_op.constant([1, 6]),
@@ -113,7 +114,7 @@ class DynamicStitchTestBase(object):
       self.assertEqual([8, 2], stitched_t.get_shape().as_list())
 
   def testZeroSizeTensor(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       indices = [
           constant_op.constant([0, 4, 7]),
           constant_op.constant([1, 6]),
@@ -222,7 +223,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
     DynamicStitchTestBase.__init__(self, data_flow_ops.parallel_dynamic_stitch)
 
   def testScalar(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       indices = [constant_op.constant(0), constant_op.constant(1)]
       data = [constant_op.constant(40.0), constant_op.constant(60.0)]
       for step in -1, 1:
diff --git a/tensorflow/python/kernel_tests/extract_image_patches_op_test.py b/tensorflow/python/kernel_tests/extract_image_patches_op_test.py
index 4fe51e94e1..bb3c0ae806 100644
--- a/tensorflow/python/kernel_tests/extract_image_patches_op_test.py
+++ b/tensorflow/python/kernel_tests/extract_image_patches_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -43,7 +44,7 @@ class ExtractImagePatches(test.TestCase):
     strides = [1] + strides + [1]
     rates = [1] + rates + [1]
 
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       out_tensor = array_ops.extract_image_patches(
           constant_op.constant(image),
           ksizes=ksizes,
diff --git a/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py b/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py
index d99823d517..88f7df8fbb 100644
--- a/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py
+++ b/tensorflow/python/kernel_tests/extract_volume_patches_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -45,7 +46,7 @@ class ExtractVolumePatches(test.TestCase):
     ksizes = [1] + ksizes + [1]
     strides = [1] + strides + [1]
 
-    with self.cached_session(use_gpu=True):
+    with test_util.use_gpu():
       out_tensor = array_ops.extract_volume_patches(
           constant_op.constant(image),
           ksizes=ksizes,
diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
index 7fe6cd4141..d41b449a1f 100644
--- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
@@ -25,6 +25,7 @@ import numpy as np
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import random_ops
@@ -50,7 +51,7 @@ class ExponentialOpTest(test.TestCase):
 
   def _verifyExponential(self, x, np_type):
     inp = x.astype(np_type)
-    with self.cached_session(use_gpu=True):
+    with test_util.use_gpu():
       tf_ans = linalg_impl.matrix_exponential(inp)
       if x.size == 0:
         np_ans = np.empty(x.shape, dtype=np_type)
diff --git a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
index 102502ae0d..81c0b5a772 100644
--- a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_linalg_ops
 from tensorflow.python.ops import math_ops
@@ -39,7 +40,7 @@ class LogarithmOpTest(test.TestCase):
 
   def _verifyLogarithm(self, x, np_type):
     inp = x.astype(np_type)
-    with self.cached_session(use_gpu=True):
+    with test_util.use_gpu():
       # Verify that expm(logm(A)) == A.
       tf_ans = linalg_impl.matrix_exponential(
           gen_linalg_ops.matrix_logarithm(inp))
diff --git a/tensorflow/python/kernel_tests/numerics_test.py b/tensorflow/python/kernel_tests/numerics_test.py
index d25d97349d..e3210dcddc 100644
--- a/tensorflow/python/kernel_tests/numerics_test.py
+++ b/tensorflow/python/kernel_tests/numerics_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -35,7 +36,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
   def testVerifyTensorAllFiniteSucceeds(self):
     x_shape = [5, 4]
     x = np.random.random_sample(x_shape).astype(np.float32)
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32)
       t_verified = numerics.verify_tensor_all_finite(t,
                                                      "Input is not a number.")
@@ -48,7 +49,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
 
     # Test NaN.
     x[0] = np.nan
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       with self.assertRaisesOpError(my_msg):
         t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32)
         t_verified = numerics.verify_tensor_all_finite(t, my_msg)
@@ -56,7 +57,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
 
     # Test Inf.
     x[0] = np.inf
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       with self.assertRaisesOpError(my_msg):
         t = constant_op.constant(x, shape=x_shape, dtype=dtypes.float32)
         t_verified = numerics.verify_tensor_all_finite(t, my_msg)
diff --git a/tensorflow/python/kernel_tests/scatter_ops_test.py b/tensorflow/python/kernel_tests/scatter_ops_test.py
index 1c7006ac0b..a4daad7adc 100644
--- a/tensorflow/python/kernel_tests/scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -320,19 +321,19 @@ class ScatterTest(test.TestCase):
       updates = np.array([-3, -4, -5]).astype(np.float32)
       # With GPU, the code ignores indices that are out of range.
       # We don't test the implementation; just test there's no failures.
-      with self.cached_session(force_gpu=True):
+      with test_util.force_gpu():
         ref = variables.Variable(params)
         ref.initializer.run()
 
         # Indices all in range, no problem.
         indices = np.array([2, 0, 5])
-        op(ref, indices, updates).eval()
+        self.evaluate(op(ref, indices, updates))
 
         # Indicies out of range should not fail.
         indices = np.array([-1, 0, 5])
-        op(ref, indices, updates).eval()
+        self.evaluate(op(ref, indices, updates))
         indices = np.array([2, 0, 6])
-        op(ref, indices, updates).eval()
+        self.evaluate(op(ref, indices, updates))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index c32b4ff42d..8ac98a198c 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -36,21 +36,22 @@ class SpaceToDepthTest(test.TestCase):
 
   def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
     input_nhwc = math_ops.cast(inputs, dtype)
-    with self.session(use_gpu=False):
+    with test_util.force_cpu():
       # test NHWC (default) on CPU
       x_tf = array_ops.space_to_depth(input_nhwc, block_size)
-      self.assertAllEqual(x_tf.eval(), outputs)
-    if test.is_gpu_available():
-      with self.session(force_gpu=True):
+      self.assertAllEqual(self.evaluate(x_tf), outputs)
+
+    if test_util.is_gpu_available():
+      with test_util.force_gpu():
         # test NHWC (default) on GPU
         x_tf = array_ops.space_to_depth(input_nhwc, block_size)
-        self.assertAllEqual(x_tf.eval(), outputs)
+        self.assertAllEqual(self.evaluate(x_tf), outputs)
         # test NCHW on GPU
         input_nchw = test_util.NHWCToNCHW(input_nhwc)
         output_nchw = array_ops.space_to_depth(
             input_nchw, block_size, data_format="NCHW")
         output_nhwc = test_util.NCHWToNHWC(output_nchw)
-        self.assertAllEqual(output_nhwc.eval(), outputs)
+        self.assertAllEqual(self.evaluate(output_nhwc), outputs)
 
   def testBasic(self):
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -134,13 +135,14 @@ class SpaceToDepthTest(test.TestCase):
     input_nhwc = array_ops.ones([batch_size, 4, 6, 3])
     x_out = array_ops.ones([batch_size, 2, 3, 12])
 
-    with self.session(use_gpu=False):
+    with test_util.force_cpu():
       # test NHWC (default) on CPU
       x_tf = array_ops.space_to_depth(input_nhwc, block_size)
       self.assertAllEqual(x_tf.shape, x_out.shape)
       self.evaluate(x_tf)
+
     if test.is_gpu_available():
-      with self.session(use_gpu=True):
+      with test_util.use_gpu():
         # test NHWC (default) on GPU
         x_tf = array_ops.space_to_depth(input_nhwc, block_size)
         self.assertAllEqual(x_tf.shape, x_out.shape)
diff --git a/tensorflow/python/kernel_tests/unstack_op_test.py b/tensorflow/python/kernel_tests/unstack_op_test.py
index 6aea42990a..d314e1eaf9 100644
--- a/tensorflow/python/kernel_tests/unstack_op_test.py
+++ b/tensorflow/python/kernel_tests/unstack_op_test.py
@@ -41,7 +41,7 @@ class UnstackOpTest(test.TestCase):
 
   def testSimple(self):
     np.random.seed(7)
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
         for dtype in [
             np.bool, np.float16, np.float32, np.float64, np.int32, np.int64
@@ -53,14 +53,15 @@ class UnstackOpTest(test.TestCase):
           cs = array_ops.unstack(x, num=shape[0])
           self.assertEqual(type(cs), list)
           self.assertEqual(len(cs), shape[0])
-          cs = [c.eval() for c in cs]
+          cs = [self.evaluate(c) for c in cs]
           self.assertAllEqual(cs, data)
 
   def testSimpleGpu(self):
     if not test_util.is_gpu_available():
       self.skipTest('No GPU available')
+
     np.random.seed(7)
-    with self.session(use_gpu=True, force_gpu=True):
+    with test_util.force_gpu():
       for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
         for dtype in [np.float16, np.float32, np.float64, np.int32, np.int64]:
           data = np.random.randn(*shape).astype(dtype)
@@ -70,7 +71,7 @@ class UnstackOpTest(test.TestCase):
           cs = array_ops.unstack(x, num=shape[0])
           self.assertEqual(type(cs), list)
           self.assertEqual(len(cs), shape[0])
-          cs = [c.eval() for c in cs]
+          cs = [self.evaluate(c) for c in cs]
           self.assertAllEqual(cs, data)
 
   def testGradientsAxis0(self):
@@ -131,15 +132,13 @@ class UnstackOpTest(test.TestCase):
       for j in range(-i, i):
         expected = np_split_squeeze(a, j)
 
-        with self.cached_session() as sess:
-          actual_unstack = sess.run(array_ops.unstack(a, axis=j))
+        actual_unstack = self.evaluate(array_ops.unstack(a, axis=j))
 
         self.assertAllEqual(expected, actual_unstack)
 
   def testAxis0Default(self):
-    with self.cached_session() as sess:
-      a = constant_op.constant([[1, 2, 3], [4, 5, 6]], name='a')
-      unstacked = sess.run(array_ops.unstack(a))
+    a = constant_op.constant([[1, 2, 3], [4, 5, 6]], name='a')
+    unstacked = self.evaluate(array_ops.unstack(a))
 
     self.assertEqual(len(unstacked), 2)
     self.assertAllEqual(unstacked[0], [1, 2, 3])
@@ -156,10 +155,9 @@ class UnstackOpTest(test.TestCase):
       array_ops.unstack(a, axis=-3)
 
   def testZeroLengthDim(self):
-    with self.cached_session():
-      x = array_ops.zeros(shape=(0, 1, 2))
-      y = array_ops.unstack(x, axis=1)[0].eval()
-      self.assertEqual(y.shape, (0, 2))
+    x = array_ops.zeros(shape=(0, 1, 2))
+    y = self.evaluate(array_ops.unstack(x, axis=1)[0])
+    self.assertEqual(y.shape, (0, 2))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/kernel_tests/zero_division_test.py b/tensorflow/python/kernel_tests/zero_division_test.py
index 73ab382e53..7c82f9320a 100644
--- a/tensorflow/python/kernel_tests/zero_division_test.py
+++ b/tensorflow/python/kernel_tests/zero_division_test.py
@@ -21,13 +21,14 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class ZeroDivisionTest(test.TestCase):
 
   def testZeros(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       for dtype in dtypes.uint8, dtypes.int16, dtypes.int32, dtypes.int64:
         zero = constant_op.constant(0, dtype=dtype)
         one = constant_op.constant(1, dtype=dtype)
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 14cc1c6b5a..96b9d6fc0d 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -53,31 +53,29 @@ class ZeroFractionTest(test_lib.TestCase):
     x_shape = [5, 17]
     x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32)
     y_np = self._ZeroFraction(x_np)
-    with self.cached_session():
-      x_tf = constant_op.constant(x_np)
-      x_tf.set_shape(x_shape)
-      y_tf = nn_impl.zero_fraction(x_tf)
-      y_tf_np = self.evaluate(y_tf)
+
+    x_tf = constant_op.constant(x_np)
+    x_tf.set_shape(x_shape)
+    y_tf = nn_impl.zero_fraction(x_tf)
+    y_tf_np = self.evaluate(y_tf)
+
     eps = 1e-8
     self.assertAllClose(y_tf_np, y_np, eps)
 
   def testZeroFractionEmpty(self):
-    with self.cached_session():
-      x = np.zeros(0)
-      y = nn_impl.zero_fraction(x).eval()
-      self.assertTrue(np.isnan(y))
+    x = np.zeros(0)
+    y = self.evaluate(nn_impl.zero_fraction(x))
+    self.assertTrue(np.isnan(y))
 
   def testZeroFraction2_27Zeros(self):
     sparsity = nn_impl.zero_fraction(
         array_ops.zeros([int(2**27 * 1.01)], dtype=dtypes.int8))
-    with self.cached_session():
-      self.assertAllClose(1.0, self.evaluate(sparsity))
+    self.assertAllClose(1.0, self.evaluate(sparsity))
 
   def testZeroFraction2_27Ones(self):
     sparsity = nn_impl.zero_fraction(
         array_ops.ones([int(2**27 * 1.01)], dtype=dtypes.int8))
-    with self.cached_session():
-      self.assertAllClose(0.0, self.evaluate(sparsity))
+    self.assertAllClose(0.0, self.evaluate(sparsity))
 
   def testUnknownSize(self):
     value = array_ops.placeholder(dtype=dtypes.float32)
@@ -302,19 +300,18 @@ class DropoutTest(test_lib.TestCase):
     y_dim = 30
     num_iter = 10
     for keep_prob in [0.1, 0.5, 0.8]:
-      with self.cached_session():
-        t = constant_op.constant(
-            1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
-        dropout = nn_ops.dropout(t, keep_prob)
-        final_count = 0
-        self.assertEqual([x_dim, y_dim], dropout.get_shape())
-        for _ in xrange(0, num_iter):
-          value = self.evaluate(dropout)
-          final_count += np.count_nonzero(value)
-          # Verifies that there are only two values: 0 and 1/keep_prob.
-          sorted_value = np.unique(np.sort(value))
-          self.assertEqual(0, sorted_value[0])
-          self.assertAllClose(1 / keep_prob, sorted_value[1])
+      t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
+      dropout = nn_ops.dropout(t, keep_prob)
+      final_count = 0
+      self.assertEqual([x_dim, y_dim], dropout.get_shape())
+      for _ in xrange(0, num_iter):
+        value = self.evaluate(dropout)
+        final_count += np.count_nonzero(value)
+        # Verifies that there are only two values: 0 and 1/keep_prob.
+        sorted_value = np.unique(np.sort(value))
+        self.assertEqual(0, sorted_value[0])
+        self.assertAllClose(1 / keep_prob, sorted_value[1])
+
       # Check that we are in the 15% error range
       expected_count = x_dim * y_dim * keep_prob * num_iter
       rel_error = math.fabs(final_count - expected_count) / expected_count
@@ -330,19 +327,18 @@ class DropoutTest(test_lib.TestCase):
     y_dim = 3
     num_iter = 10
     for keep_prob in [0.1, 0.5, 0.8]:
-      with self.cached_session():
-        t = constant_op.constant(
-            1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
-        dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
-        self.assertEqual([x_dim, y_dim], dropout.get_shape())
-        final_count = 0
-        for _ in xrange(0, num_iter):
-          value = self.evaluate(dropout)
-          final_count += np.count_nonzero(value)
-          # Verifies that there are only two values: 0 and 1/keep_prob.
-          sorted_value = np.unique(np.sort(value))
-          self.assertEqual(0, sorted_value[0])
-          self.assertAllClose(1 / keep_prob, sorted_value[1])
+      t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
+      dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
+      self.assertEqual([x_dim, y_dim], dropout.get_shape())
+      final_count = 0
+      for _ in xrange(0, num_iter):
+        value = self.evaluate(dropout)
+        final_count += np.count_nonzero(value)
+        # Verifies that there are only two values: 0 and 1/keep_prob.
+        sorted_value = np.unique(np.sort(value))
+        self.assertEqual(0, sorted_value[0])
+        self.assertAllClose(1 / keep_prob, sorted_value[1])
+
       # Check that we are in the 15% error range
       expected_count = x_dim * y_dim * keep_prob * num_iter
       rel_error = math.fabs(final_count - expected_count) / expected_count
@@ -355,17 +351,15 @@ class DropoutTest(test_lib.TestCase):
     y_dim = 30
     num_iter = 10
     for keep_prob in [0.1, 0.5, 0.8]:
-      with self.cached_session():
-        t = constant_op.constant(
-            1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
-        dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
-        self.assertEqual([x_dim, y_dim], dropout.get_shape())
-        for _ in xrange(0, num_iter):
-          value = self.evaluate(dropout)
-          # Verifies that each y column as only one type of activation.
-          for i in xrange(x_dim):
-            sorted_value = np.unique(np.sort(value[i, :]))
-            self.assertEqual(sorted_value.size, 1)
+      t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
+      dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
+      self.assertEqual([x_dim, y_dim], dropout.get_shape())
+      for _ in xrange(0, num_iter):
+        value = self.evaluate(dropout)
+        # Verifies that each y column as only one type of activation.
+        for i in xrange(x_dim):
+          sorted_value = np.unique(np.sort(value[i, :]))
+          self.assertEqual(sorted_value.size, 1)
 
   def testDropoutPlaceholderKeepProb(self):
     # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate
@@ -409,20 +403,19 @@ class DropoutTest(test_lib.TestCase):
     y_dim = 3
     num_iter = 10
     for keep_prob in [0.1, 0.5, 0.8]:
-      with self.cached_session():
-        t = constant_op.constant(
-            1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
-        # Set noise_shape=[None, 1] which means [x_dim, 1].
-        dropout = nn_ops.dropout(t, keep_prob, noise_shape=[None, 1])
-        self.assertEqual([x_dim, y_dim], dropout.get_shape())
-        final_count = 0
-        for _ in xrange(0, num_iter):
-          value = self.evaluate(dropout)
-          final_count += np.count_nonzero(value)
-          # Verifies that there are only two values: 0 and 1/keep_prob.
-          sorted_value = np.unique(np.sort(value))
-          self.assertEqual(0, sorted_value[0])
-          self.assertAllClose(1 / keep_prob, sorted_value[1])
+      t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
+      # Set noise_shape=[None, 1] which means [x_dim, 1].
+      dropout = nn_ops.dropout(t, keep_prob, noise_shape=[None, 1])
+      self.assertEqual([x_dim, y_dim], dropout.get_shape())
+      final_count = 0
+      for _ in xrange(0, num_iter):
+        value = self.evaluate(dropout)
+        final_count += np.count_nonzero(value)
+        # Verifies that there are only two values: 0 and 1/keep_prob.
+        sorted_value = np.unique(np.sort(value))
+        self.assertEqual(0, sorted_value[0])
+        self.assertAllClose(1 / keep_prob, sorted_value[1])
+
       # Check that we are in the 15% error range
       expected_count = x_dim * y_dim * keep_prob * num_iter
       rel_error = math.fabs(final_count - expected_count) / expected_count
@@ -563,78 +556,78 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
           initializer=constant_op.constant(biases))
       with self.session(graph=g) as sess:
         variables.global_variables_initializer().run()
-        return sess.run([list(sharded_weights), list(sharded_biases)])
+        return self.evaluate([list(sharded_weights), list(sharded_biases)])
 
   def testShapes(self):
     np.random.seed(0)
     num_classes = 5
     batch_size = 3
-    with self.cached_session() as sess:
-      for num_true in range(1, 5):
-        labels = np.random.randint(
-            low=0, high=num_classes, size=batch_size * num_true)
-        (weights, biases, hidden_acts, sampled_vals, exp_logits,
-         exp_labels) = self._GenerateTestData(
-             num_classes=num_classes,
-             dim=10,
-             batch_size=batch_size,
-             num_true=num_true,
-             labels=labels,
-             sampled=[1, 0, 2, 3],
-             subtract_log_q=False)
-        logits_tensor, labels_tensor = _compute_sampled_logits(
-            weights=constant_op.constant(weights),
-            biases=constant_op.constant(biases),
-            labels=constant_op.constant(
-                labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
-            inputs=constant_op.constant(hidden_acts),
-            num_sampled=4,
-            num_classes=num_classes,
-            num_true=num_true,
-            sampled_values=sampled_vals,
-            subtract_log_q=False,
-            remove_accidental_hits=False,
-            partition_strategy="div",
-            name="sampled_logits_basic_num_true_%d" % num_true)
-        got_logits, got_labels = sess.run([logits_tensor, labels_tensor])
-        self.assertEqual(exp_logits.shape, got_logits.shape, self._eps)
-        self.assertEqual(exp_labels.shape, got_labels.shape, self._eps)
+
+    for num_true in range(1, 5):
+      labels = np.random.randint(
+          low=0, high=num_classes, size=batch_size * num_true)
+      (weights, biases, hidden_acts, sampled_vals, exp_logits,
+       exp_labels) = self._GenerateTestData(
+           num_classes=num_classes,
+           dim=10,
+           batch_size=batch_size,
+           num_true=num_true,
+           labels=labels,
+           sampled=[1, 0, 2, 3],
+           subtract_log_q=False)
+      logits_tensor, labels_tensor = _compute_sampled_logits(
+          weights=constant_op.constant(weights),
+          biases=constant_op.constant(biases),
+          labels=constant_op.constant(
+              labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
+          inputs=constant_op.constant(hidden_acts),
+          num_sampled=4,
+          num_classes=num_classes,
+          num_true=num_true,
+          sampled_values=sampled_vals,
+          subtract_log_q=False,
+          remove_accidental_hits=False,
+          partition_strategy="div",
+          name="sampled_logits_basic_num_true_%d" % num_true)
+      got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor])
+      self.assertEqual(exp_logits.shape, got_logits.shape, self._eps)
+      self.assertEqual(exp_labels.shape, got_labels.shape, self._eps)
 
   def testBasic(self):
     """Without accidental hit removal or subtract_log_q."""
     np.random.seed(0)
     num_classes = 5
     batch_size = 3
-    with self.cached_session() as sess:
-      for num_true in range(1, 5):
-        labels = np.random.randint(
-            low=0, high=num_classes, size=batch_size * num_true)
-        (weights, biases, hidden_acts, sampled_vals, exp_logits,
-         exp_labels) = self._GenerateTestData(
-             num_classes=num_classes,
-             dim=10,
-             batch_size=batch_size,
-             num_true=num_true,
-             labels=labels,
-             sampled=[1, 0, 2, 3],
-             subtract_log_q=False)
-        logits_tensor, labels_tensor = _compute_sampled_logits(
-            weights=constant_op.constant(weights),
-            biases=constant_op.constant(biases),
-            labels=constant_op.constant(
-                labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
-            inputs=constant_op.constant(hidden_acts),
-            num_sampled=4,
-            num_classes=num_classes,
-            num_true=num_true,
-            sampled_values=sampled_vals,
-            subtract_log_q=False,
-            remove_accidental_hits=False,
-            partition_strategy="div",
-            name="sampled_logits_basic_num_true_%d" % num_true)
-        got_logits, got_labels = sess.run([logits_tensor, labels_tensor])
-        self.assertAllClose(exp_logits, got_logits, self._eps)
-        self.assertAllClose(exp_labels, got_labels, self._eps)
+
+    for num_true in range(1, 5):
+      labels = np.random.randint(
+          low=0, high=num_classes, size=batch_size * num_true)
+      (weights, biases, hidden_acts, sampled_vals, exp_logits,
+       exp_labels) = self._GenerateTestData(
+           num_classes=num_classes,
+           dim=10,
+           batch_size=batch_size,
+           num_true=num_true,
+           labels=labels,
+           sampled=[1, 0, 2, 3],
+           subtract_log_q=False)
+      logits_tensor, labels_tensor = _compute_sampled_logits(
+          weights=constant_op.constant(weights),
+          biases=constant_op.constant(biases),
+          labels=constant_op.constant(
+              labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
+          inputs=constant_op.constant(hidden_acts),
+          num_sampled=4,
+          num_classes=num_classes,
+          num_true=num_true,
+          sampled_values=sampled_vals,
+          subtract_log_q=False,
+          remove_accidental_hits=False,
+          partition_strategy="div",
+          name="sampled_logits_basic_num_true_%d" % num_true)
+      got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor])
+      self.assertAllClose(exp_logits, got_logits, self._eps)
+      self.assertAllClose(exp_labels, got_labels, self._eps)
 
   def testAccidentalHitRemoval(self):
     """With accidental hit removal, no subtract_log_q."""
@@ -642,118 +635,118 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     num_classes = 5
     batch_size = 3
     sampled = [1, 0, 2, 3]
-    with self.cached_session():
-      for num_true in range(1, 5):
-        labels = np.random.randint(
-            low=0, high=num_classes, size=batch_size * num_true)
-        (weights, biases, hidden_acts, sampled_vals, _,
-         _) = self._GenerateTestData(
-             num_classes=num_classes,
-             dim=10,
-             batch_size=batch_size,
-             num_true=num_true,
-             labels=labels,
-             sampled=sampled,
-             subtract_log_q=False)
-        logits_tensor, _ = _compute_sampled_logits(
-            weights=constant_op.constant(weights),
-            biases=constant_op.constant(biases),
-            labels=constant_op.constant(
-                labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
-            inputs=constant_op.constant(hidden_acts),
-            num_sampled=len(sampled),
-            num_classes=num_classes,
-            num_true=num_true,
-            sampled_values=sampled_vals,
-            subtract_log_q=False,
-            remove_accidental_hits=True,
-            partition_strategy="div",
-            name="sampled_logits_accidental_hit_removal_num_true_%d" % num_true)
-        # Test that the exponentiated logits of accidental hits are near 0.
-        # First we need to find the hits in this random test run:
-        labels_reshape = labels.reshape((batch_size, num_true))
-        got_logits = self.evaluate(logits_tensor)
-        for row in xrange(batch_size):
-          row_labels = labels_reshape[row, :]
-          for col in xrange(len(sampled)):
-            if sampled[col] in row_labels:
-              # We need to add the num_true_test offset into logits_*
-              self.assertNear(
-                  np.exp(got_logits[row, col + num_true]), 0., self._eps)
+
+    for num_true in range(1, 5):
+      labels = np.random.randint(
+          low=0, high=num_classes, size=batch_size * num_true)
+      (weights, biases, hidden_acts, sampled_vals, _,
+       _) = self._GenerateTestData(
+           num_classes=num_classes,
+           dim=10,
+           batch_size=batch_size,
+           num_true=num_true,
+           labels=labels,
+           sampled=sampled,
+           subtract_log_q=False)
+      logits_tensor, _ = _compute_sampled_logits(
+          weights=constant_op.constant(weights),
+          biases=constant_op.constant(biases),
+          labels=constant_op.constant(
+              labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
+          inputs=constant_op.constant(hidden_acts),
+          num_sampled=len(sampled),
+          num_classes=num_classes,
+          num_true=num_true,
+          sampled_values=sampled_vals,
+          subtract_log_q=False,
+          remove_accidental_hits=True,
+          partition_strategy="div",
+          name="sampled_logits_accidental_hit_removal_num_true_%d" % num_true)
+      # Test that the exponentiated logits of accidental hits are near 0.
+      # First we need to find the hits in this random test run:
+      labels_reshape = labels.reshape((batch_size, num_true))
+      got_logits = self.evaluate(logits_tensor)
+      for row in xrange(batch_size):
+        row_labels = labels_reshape[row, :]
+        for col in xrange(len(sampled)):
+          if sampled[col] in row_labels:
+            # We need to add the num_true_test offset into logits_*
+            self.assertNear(
+                np.exp(got_logits[row, col + num_true]), 0., self._eps)
 
   def testSubtractLogQ(self):
     """With subtract_log_q, no accidental hit removal."""
     np.random.seed(0)
     num_classes = 5
     batch_size = 3
-    with self.cached_session() as sess:
-      for num_true in range(1, 5):
-        labels = np.random.randint(
-            low=0, high=num_classes, size=batch_size * num_true)
-        (weights, biases, hidden_acts, sampled_vals, exp_logits,
-         exp_labels) = self._GenerateTestData(
-             num_classes=num_classes,
-             dim=10,
-             batch_size=batch_size,
-             num_true=num_true,
-             labels=labels,
-             sampled=[1, 0, 2, 3],
-             subtract_log_q=True)
-        logits_tensor, labels_tensor = _compute_sampled_logits(
-            weights=constant_op.constant(weights),
-            biases=constant_op.constant(biases),
-            labels=constant_op.constant(
-                labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
-            inputs=constant_op.constant(hidden_acts),
-            num_sampled=4,
-            num_classes=num_classes,
-            num_true=num_true,
-            sampled_values=sampled_vals,
-            subtract_log_q=True,
-            remove_accidental_hits=False,
-            partition_strategy="div",
-            name="sampled_logits_subtract_log_q_num_true_%d" % num_true)
-        got_logits, got_labels = sess.run([logits_tensor, labels_tensor])
-        self.assertAllClose(exp_logits, got_logits, self._eps)
-        self.assertAllClose(exp_labels, got_labels, self._eps)
+
+    for num_true in range(1, 5):
+      labels = np.random.randint(
+          low=0, high=num_classes, size=batch_size * num_true)
+      (weights, biases, hidden_acts, sampled_vals, exp_logits,
+       exp_labels) = self._GenerateTestData(
+           num_classes=num_classes,
+           dim=10,
+           batch_size=batch_size,
+           num_true=num_true,
+           labels=labels,
+           sampled=[1, 0, 2, 3],
+           subtract_log_q=True)
+      logits_tensor, labels_tensor = _compute_sampled_logits(
+          weights=constant_op.constant(weights),
+          biases=constant_op.constant(biases),
+          labels=constant_op.constant(
+              labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
+          inputs=constant_op.constant(hidden_acts),
+          num_sampled=4,
+          num_classes=num_classes,
+          num_true=num_true,
+          sampled_values=sampled_vals,
+          subtract_log_q=True,
+          remove_accidental_hits=False,
+          partition_strategy="div",
+          name="sampled_logits_subtract_log_q_num_true_%d" % num_true)
+      got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor])
+      self.assertAllClose(exp_logits, got_logits, self._eps)
+      self.assertAllClose(exp_labels, got_labels, self._eps)
 
   def testSharded(self):
     """With sharded weights and sharded biases."""
     np.random.seed(0)
     num_classes = 5
     batch_size = 3
-    with self.cached_session() as sess:
-      for num_true in range(1, 5):
-        labels = np.random.randint(
-            low=0, high=num_classes, size=batch_size * num_true)
-        (weights, biases, hidden_acts, sampled_vals, exp_logits,
-         exp_labels) = self._GenerateTestData(
-             num_classes=num_classes,
-             dim=10,
-             batch_size=batch_size,
-             num_true=num_true,
-             labels=labels,
-             sampled=[1, 0, 2, 3],
-             subtract_log_q=False)
-        weight_shards, bias_shards = self._ShardTestEmbeddings(
-            weights, biases, num_shards=3)
-        logits_tensor, labels_tensor = _compute_sampled_logits(
-            weights=[constant_op.constant(shard) for shard in weight_shards],
-            biases=[constant_op.constant(shard) for shard in bias_shards],
-            labels=constant_op.constant(
-                labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
-            inputs=constant_op.constant(hidden_acts),
-            num_sampled=4,
-            num_classes=num_classes,
-            num_true=num_true,
-            sampled_values=sampled_vals,
-            subtract_log_q=False,
-            remove_accidental_hits=False,
-            partition_strategy="div",
-            name="sampled_logits_sharded_num_true_%d" % num_true)
-        got_logits, got_labels = sess.run([logits_tensor, labels_tensor])
-        self.assertAllClose(exp_logits, got_logits, self._eps)
-        self.assertAllClose(exp_labels, got_labels, self._eps)
+
+    for num_true in range(1, 5):
+      labels = np.random.randint(
+          low=0, high=num_classes, size=batch_size * num_true)
+      (weights, biases, hidden_acts, sampled_vals, exp_logits,
+       exp_labels) = self._GenerateTestData(
+           num_classes=num_classes,
+           dim=10,
+           batch_size=batch_size,
+           num_true=num_true,
+           labels=labels,
+           sampled=[1, 0, 2, 3],
+           subtract_log_q=False)
+      weight_shards, bias_shards = self._ShardTestEmbeddings(
+          weights, biases, num_shards=3)
+      logits_tensor, labels_tensor = _compute_sampled_logits(
+          weights=[constant_op.constant(shard) for shard in weight_shards],
+          biases=[constant_op.constant(shard) for shard in bias_shards],
+          labels=constant_op.constant(
+              labels, dtype=dtypes.int64, shape=(batch_size, num_true)),
+          inputs=constant_op.constant(hidden_acts),
+          num_sampled=4,
+          num_classes=num_classes,
+          num_true=num_true,
+          sampled_values=sampled_vals,
+          subtract_log_q=False,
+          remove_accidental_hits=False,
+          partition_strategy="div",
+          name="sampled_logits_sharded_num_true_%d" % num_true)
+      got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor])
+      self.assertAllClose(exp_logits, got_logits, self._eps)
+      self.assertAllClose(exp_labels, got_labels, self._eps)
 
   def testNCELoss(self):
     # A simple test to verify the numerics.
@@ -782,35 +775,34 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     exp_nce_loss = np.sum(
         _SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1)
 
-    with self.cached_session():
-      got_nce_loss = nn_impl.nce_loss(
-          weights=constant_op.constant(weights),
-          biases=constant_op.constant(biases),
-          labels=constant_op.constant(labels, shape=(batch_size, 1)),
-          inputs=constant_op.constant(hidden_acts),
-          num_sampled=4,
-          num_classes=num_classes,
-          num_true=1,
-          sampled_values=sampled_vals,
-          partition_strategy="div")
-
-      self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
-
-      # Test with sharded weights and sharded biases.
-      weight_shards, bias_shards = self._ShardTestEmbeddings(
-          weights, biases, num_shards=3)
-      got_nce_loss = nn_impl.nce_loss(
-          weights=[constant_op.constant(shard) for shard in weight_shards],
-          biases=[constant_op.constant(shard) for shard in bias_shards],
-          labels=constant_op.constant(labels, shape=(batch_size, 1)),
-          inputs=constant_op.constant(hidden_acts),
-          num_sampled=4,
-          num_classes=num_classes,
-          num_true=1,
-          sampled_values=sampled_vals,
-          partition_strategy="div")
-
-      self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
+    got_nce_loss = nn_impl.nce_loss(
+        weights=constant_op.constant(weights),
+        biases=constant_op.constant(biases),
+        labels=constant_op.constant(labels, shape=(batch_size, 1)),
+        inputs=constant_op.constant(hidden_acts),
+        num_sampled=4,
+        num_classes=num_classes,
+        num_true=1,
+        sampled_values=sampled_vals,
+        partition_strategy="div")
+
+    self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
+
+    # Test with sharded weights and sharded biases.
+    weight_shards, bias_shards = self._ShardTestEmbeddings(
+        weights, biases, num_shards=3)
+    got_nce_loss = nn_impl.nce_loss(
+        weights=[constant_op.constant(shard) for shard in weight_shards],
+        biases=[constant_op.constant(shard) for shard in bias_shards],
+        labels=constant_op.constant(labels, shape=(batch_size, 1)),
+        inputs=constant_op.constant(hidden_acts),
+        num_sampled=4,
+        num_classes=num_classes,
+        num_true=1,
+        sampled_values=sampled_vals,
+        partition_strategy="div")
+
+    self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
 
   def testSampledSoftmaxLoss(self):
     # A simple test to verify the numerics.
@@ -839,39 +831,38 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits(
         exp_logits, exp_labels)
 
-    with self.cached_session():
-      got_sampled_softmax_loss = nn_impl.sampled_softmax_loss(
-          weights=constant_op.constant(weights),
-          biases=constant_op.constant(biases),
-          labels=constant_op.constant(labels, shape=(batch_size, 1)),
-          inputs=constant_op.constant(hidden_acts),
-          num_sampled=4,
-          num_classes=num_classes,
-          num_true=1,
-          sampled_values=sampled_vals,
-          remove_accidental_hits=False,
-          partition_strategy="div")
-
-      self.assertAllClose(exp_sampled_softmax_loss,
-                          self.evaluate(got_sampled_softmax_loss), 1e-4)
-
-      # Test with sharded weights and sharded biases.
-      weight_shards, bias_shards = self._ShardTestEmbeddings(
-          weights, biases, num_shards=3)
-      got_sampled_softmax_loss = nn_impl.sampled_softmax_loss(
-          weights=[constant_op.constant(shard) for shard in weight_shards],
-          biases=[constant_op.constant(shard) for shard in bias_shards],
-          labels=constant_op.constant(labels, shape=(batch_size, 1)),
-          inputs=constant_op.constant(hidden_acts),
-          num_sampled=4,
-          num_classes=num_classes,
-          num_true=1,
-          sampled_values=sampled_vals,
-          remove_accidental_hits=False,
-          partition_strategy="div")
-
-      self.assertAllClose(exp_sampled_softmax_loss,
-                          self.evaluate(got_sampled_softmax_loss), 1e-4)
+    got_sampled_softmax_loss = nn_impl.sampled_softmax_loss(
+        weights=constant_op.constant(weights),
+        biases=constant_op.constant(biases),
+        labels=constant_op.constant(labels, shape=(batch_size, 1)),
+        inputs=constant_op.constant(hidden_acts),
+        num_sampled=4,
+        num_classes=num_classes,
+        num_true=1,
+        sampled_values=sampled_vals,
+        remove_accidental_hits=False,
+        partition_strategy="div")
+
+    self.assertAllClose(exp_sampled_softmax_loss,
+                        self.evaluate(got_sampled_softmax_loss), 1e-4)
+
+    # Test with sharded weights and sharded biases.
+    weight_shards, bias_shards = self._ShardTestEmbeddings(
+        weights, biases, num_shards=3)
+    got_sampled_softmax_loss = nn_impl.sampled_softmax_loss(
+        weights=[constant_op.constant(shard) for shard in weight_shards],
+        biases=[constant_op.constant(shard) for shard in bias_shards],
+        labels=constant_op.constant(labels, shape=(batch_size, 1)),
+        inputs=constant_op.constant(hidden_acts),
+        num_sampled=4,
+        num_classes=num_classes,
+        num_true=1,
+        sampled_values=sampled_vals,
+        remove_accidental_hits=False,
+        partition_strategy="div")
+
+    self.assertAllClose(exp_sampled_softmax_loss,
+                        self.evaluate(got_sampled_softmax_loss), 1e-4)
 
   def testSampledSoftmaxLossBf16(self):
     # A simple test to verify the numerics for bfloat16.
@@ -900,29 +891,30 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits(
         exp_logits, exp_labels)
 
-    with self.cached_session():
-      true_exp_bf16 = np.full(
-          [batch_size, 1], fill_value=0.5, dtype=dtypes.bfloat16.as_numpy_dtype)
-      sampled_exp_bf16 = np.full(
-          [len(sampled)], fill_value=0.5, dtype=dtypes.bfloat16.as_numpy_dtype)
-      sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16)
-
-      got_sampled_softmax_loss = math_ops.cast(
-          nn_impl.sampled_softmax_loss(
-              weights=constant_op.constant(weights, dtype=dtypes.bfloat16),
-              biases=constant_op.constant(biases, dtype=dtypes.bfloat16),
-              labels=constant_op.constant(
-                  labels, shape=(batch_size, 1), dtype=dtypes.bfloat16),
-              inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16),
-              num_sampled=4,
-              num_classes=num_classes,
-              num_true=1,
-              sampled_values=sampled_vals_bf16,
-              remove_accidental_hits=False,
-              partition_strategy="div"), dtypes.float32)
-
-      self.assertAllClose(exp_sampled_softmax_loss,
-                          self.evaluate(got_sampled_softmax_loss), 1e-1)
+    true_exp_bf16 = np.full([batch_size, 1],
+                            fill_value=0.5,
+                            dtype=dtypes.bfloat16.as_numpy_dtype)
+    sampled_exp_bf16 = np.full([len(sampled)],
+                               fill_value=0.5,
+                               dtype=dtypes.bfloat16.as_numpy_dtype)
+    sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16)
+
+    got_sampled_softmax_loss = math_ops.cast(
+        nn_impl.sampled_softmax_loss(
+            weights=constant_op.constant(weights, dtype=dtypes.bfloat16),
+            biases=constant_op.constant(biases, dtype=dtypes.bfloat16),
+            labels=constant_op.constant(
+                labels, shape=(batch_size, 1), dtype=dtypes.bfloat16),
+            inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16),
+            num_sampled=4,
+            num_classes=num_classes,
+            num_true=1,
+            sampled_values=sampled_vals_bf16,
+            remove_accidental_hits=False,
+            partition_strategy="div"), dtypes.float32)
+
+    self.assertAllClose(exp_sampled_softmax_loss,
+                        self.evaluate(got_sampled_softmax_loss), 1e-1)
 
 
 class CReluTest(test_lib.TestCase):
@@ -931,9 +923,9 @@ class CReluTest(test_lib.TestCase):
     np.random.seed(1)  # Make it reproducible.
     x = np.random.randn(3, 4).astype(np.float32)
     y = np.concatenate([x * (x > 0), -x * (x < 0)], axis=1)
-    with self.cached_session():
-      z = nn_ops.crelu(constant_op.constant(x)).eval()
-      self.assertAllClose(y, z, 1e-4)
+
+    z = self.evaluate(nn_ops.crelu(constant_op.constant(x)))
+    self.assertAllClose(y, z, 1e-4)
 
 
 class ReluTest(test_lib.TestCase):
@@ -942,9 +934,9 @@ class ReluTest(test_lib.TestCase):
     np.random.seed(1)  # Make it reproducible.
     x = np.random.randn(3, 4).astype(np.float32)
     y = np.maximum(x, 0.0)
-    with self.cached_session():
-      z = nn_ops.relu(constant_op.constant(x)).eval()
-      self.assertAllEqual(y, z)
+
+    z = self.evaluate(nn_ops.relu(constant_op.constant(x)))
+    self.assertAllEqual(y, z)
 
   def testNaNs(self):
     # Test that relu(nan) = nan for various sizes.
@@ -967,8 +959,9 @@ class LeakyReluTest(test_lib.TestCase):
 
     outputs = nn_ops.leaky_relu(inputs)
     self.assertEquals(inputs.shape, outputs.shape)
-    with self.cached_session() as sess:
-      inputs, outputs = sess.run([inputs, outputs])
+
+    inputs, outputs = self.evaluate([inputs, outputs])
+
     self.assertGreaterEqual(outputs.min(), 0.0)
     self.assertLessEqual(outputs.max(), 1.0)
     self.assertAllClose(inputs, outputs)
@@ -977,8 +970,9 @@ class LeakyReluTest(test_lib.TestCase):
     for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]:
       np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype)
       outputs = nn_ops.leaky_relu(constant_op.constant(np_values))
-      with self.cached_session() as sess:
-        outputs = self.evaluate(outputs)
+
+      outputs = self.evaluate(outputs)
+
       tol = 2e-3 if dtype == np.float16 else 1e-6
       self.assertAllClose(
           outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol)
@@ -1004,9 +998,10 @@ class SwishTest(test_lib.TestCase):
     tf_values = constant_op.constant(np_values)
     actual_tf_outputs = nn_impl.swish(tf_values)
     expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values)
-    with self.cached_session() as sess:
-      actual_outputs, expected_outputs = sess.run(
-          [actual_tf_outputs, expected_tf_outputs])
+
+    actual_outputs, expected_outputs = self.evaluate(
+        [actual_tf_outputs, expected_tf_outputs])
+
     self.assertAllClose(actual_outputs, expected_outputs)
 
   def testGradients(self):
@@ -1051,7 +1046,7 @@ class MomentsTest(test_lib.TestCase):
                 self.assertLess(err, 1e-3)
 
               # Evaluate.
-              [mean, variance] = sess.run([mean, variance])
+              [mean, variance] = self.evaluate([mean, variance])
               # Make sure that there are no NaNs
               self.assertFalse(np.isnan(mean).any())
               self.assertFalse(np.isnan(variance).any())
@@ -1094,9 +1089,9 @@ class DataFormatDimMapTest(test_lib.TestCase):
   def _test(self, x_val, y_val_expected):
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x)
-    with self.cached_session(use_gpu=test_lib.is_gpu_available()) as sess:
-      y_val = self.evaluate(y)
-      self.assertAllEqual(y_val, y_val_expected)
+
+    y_val = self.evaluate(y)
+    self.assertAllEqual(y_val, y_val_expected)
 
   def test(self):
     self._test(0, 0)
@@ -1117,7 +1112,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     y_val_expected = [2, 2, 3]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="NCHW")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
@@ -1126,7 +1121,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     y_val_expected = [2, 0, 1, 3, 2, 0, 1, 3]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="HWNC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
@@ -1135,7 +1130,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     y_val_expected = [3, 1, 0, 2, 3, 1, 0, 2]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="WHCN")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
@@ -1144,7 +1139,7 @@ class DataFormatDimMapTest(test_lib.TestCase):
     y_val_expected = [3, 2, 1, 0, 3, 2, 1, 0]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_dim_map(x, src_format="qwer", dst_format="rewq")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, y_val_expected)
 
@@ -1155,7 +1150,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [7, 4, 9, 3]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x)
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [7, 3, 4, 9])
 
@@ -1163,7 +1158,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [7, 4, 9, 3]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [7, 9, 3, 4])
 
@@ -1171,7 +1166,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [7, 4, 9, 3]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [4, 9, 7, 3])
 
@@ -1179,7 +1174,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [7, 4, 9, 3]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [9, 7, 4, 3])
 
@@ -1187,7 +1182,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [[7, 4], [9, 3], [4, 5], [5, 1]]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x)
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[7, 4], [5, 1], [9, 3], [4, 5]])
 
@@ -1195,7 +1190,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [[7, 4], [9, 3], [4, 5], [5, 1]]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[9, 3], [4, 5], [7, 4], [5, 1]])
 
@@ -1203,7 +1198,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [[7, 4], [9, 3], [4, 5], [5, 1]]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[4, 5], [7, 4], [9, 3], [5, 1]])
 
@@ -1211,7 +1206,7 @@ class DataFormatVectorPermuteTest(test_lib.TestCase):
     x_val = [[7, 4], [9, 3], [4, 5], [5, 1]]
     x = constant_op.constant(x_val)
     y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC")
-    with self.session(use_gpu=test_lib.is_gpu_available()) as sess:
+    with test_util.use_gpu():
       y_val = self.evaluate(y)
       self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]])
 
diff --git a/tensorflow/python/training/rmsprop_test.py b/tensorflow/python/training/rmsprop_test.py
index 9ec315f62d..a9b8954e39 100644
--- a/tensorflow/python/training/rmsprop_test.py
+++ b/tensorflow/python/training/rmsprop_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -92,7 +93,7 @@ class RMSPropOptimizerTest(test.TestCase):
     # TODO(yori): Use ParameterizedTest when available
     for (dtype, learning_rate, decay, momentum,
          epsilon, centered, use_resource) in _TESTPARAMS:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         # Initialize variables for numpy implementation.
         var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
         grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
@@ -115,7 +116,7 @@ class RMSPropOptimizerTest(test.TestCase):
             centered=centered)
 
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
 
         mg0 = opt.get_slot(var0, "mg")
         self.assertEqual(mg0 is not None, centered)
@@ -143,7 +144,7 @@ class RMSPropOptimizerTest(test.TestCase):
 
         # Run 4 steps of RMSProp
         for _ in range(1, 5):
-          update.run()
+          self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
               var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate,
@@ -176,11 +177,11 @@ class RMSPropOptimizerTest(test.TestCase):
             momentum=0.0,
             epsilon=0.0,
             centered=False).minimize(loss)
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
         # Fetch params to validate initial values
         self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
-        sgd_op.run()
+        self.evaluate(sgd_op)
         # Validate updated params
         self.assertAllCloseAccordingToType([[0., 1.]],
                                            self.evaluate(var0),
@@ -199,11 +200,11 @@ class RMSPropOptimizerTest(test.TestCase):
             momentum=0.0,
             epsilon=1.0,
             centered=True).minimize(loss)
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
         # Fetch params to validate initial values
         self.assertAllCloseAccordingToType([[1.0, 2.0]], self.evaluate(var0))
         # Run 1 step of sgd
-        sgd_op.run()
+        self.evaluate(sgd_op)
         # Validate updated params
         self.assertAllCloseAccordingToType([[-111, -138]],
                                            self.evaluate(var0),
@@ -213,7 +214,7 @@ class RMSPropOptimizerTest(test.TestCase):
     # TODO(yori): Use ParameterizedTest when available
     for (dtype, learning_rate, decay,
          momentum, epsilon, centered, _) in _TESTPARAMS:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         # Initialize variables for numpy implementation.
         var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
         grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype)
@@ -237,7 +238,7 @@ class RMSPropOptimizerTest(test.TestCase):
             epsilon=epsilon,
             centered=centered)
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
 
         mg0 = opt.get_slot(var0, "mg")
         self.assertEqual(mg0 is not None, centered)
@@ -265,7 +266,7 @@ class RMSPropOptimizerTest(test.TestCase):
 
         # Run 4 steps of RMSProp
         for _ in range(1, 5):
-          update.run()
+          self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
               var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np,
@@ -287,7 +288,7 @@ class RMSPropOptimizerTest(test.TestCase):
 
   def testWithoutMomentum(self):
     for dtype in [dtypes.half, dtypes.float32]:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         var0 = variables.Variable([1.0, 2.0], dtype=dtype)
         var1 = variables.Variable([3.0, 4.0], dtype=dtype)
         grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
@@ -295,7 +296,7 @@ class RMSPropOptimizerTest(test.TestCase):
         opt = rmsprop.RMSPropOptimizer(
             learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0)
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
 
         rms0 = opt.get_slot(var0, "rms")
         self.assertTrue(rms0 is not None)
@@ -311,7 +312,7 @@ class RMSPropOptimizerTest(test.TestCase):
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: the rms accumulators where 1. So we should see a normal
         # update: v -= grad * learning_rate
-        update.run()
+        self.evaluate(update)
         # Check the root mean square accumulators.
         self.assertAllCloseAccordingToType(
             np.array([0.901, 0.901]), self.evaluate(rms0))
@@ -329,7 +330,7 @@ class RMSPropOptimizerTest(test.TestCase):
                 4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0))
             ]), self.evaluate(var1))
         # Step 2: the root mean square accumulators contain the previous update.
-        update.run()
+        self.evaluate(update)
         # Check the rms accumulators.
         self.assertAllCloseAccordingToType(
             np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]),
@@ -355,7 +356,7 @@ class RMSPropOptimizerTest(test.TestCase):
 
   def testWithMomentum(self):
     for dtype in [dtypes.half, dtypes.float32]:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         var0 = variables.Variable([1.0, 2.0], dtype=dtype)
         var1 = variables.Variable([3.0, 4.0], dtype=dtype)
         grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
@@ -364,7 +365,7 @@ class RMSPropOptimizerTest(test.TestCase):
         opt = rmsprop.RMSPropOptimizer(
             learning_rate=2.0, decay=0.9, momentum=0.5, epsilon=1e-5)
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
 
         rms0 = opt.get_slot(var0, "rms")
         self.assertTrue(rms0 is not None)
@@ -380,7 +381,7 @@ class RMSPropOptimizerTest(test.TestCase):
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
         # Step 1: rms = 1, mom = 0. So we should see a normal
         # update: v -= grad * learning_rate
-        update.run()
+        self.evaluate(update)
         # Check the root mean square accumulators.
         self.assertAllCloseAccordingToType(
             np.array([0.901, 0.901]), self.evaluate(rms0))
@@ -409,7 +410,7 @@ class RMSPropOptimizerTest(test.TestCase):
             ]), self.evaluate(var1))
 
         # Step 2: the root mean square accumulators contain the previous update.
-        update.run()
+        self.evaluate(update)
         # Check the rms accumulators.
         self.assertAllCloseAccordingToType(
             np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]),
-- 
GitLab


From f5dd35d63187e697be79585d7bf8df6fe0436231 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Tue, 20 Nov 2018 17:56:24 -0800
Subject: [PATCH 0642/1554] TFTRT: fix pylint and some clang-format

There are more clang-format issues that need to be addressed.
---
 .../contrib/tensorrt/convert/convert_graph.cc | 21 +++++++-------
 .../contrib/tensorrt/convert/convert_nodes.cc | 28 +++++++++----------
 .../contrib/tensorrt/kernels/trt_engine_op.cc |  5 ++--
 .../tensorrt/python/trt_convert_test.py       |  3 +-
 .../tensorrt/resources/trt_resources.h        |  3 +-
 .../contrib/tensorrt/segment/segment.cc       | 17 ++++++-----
 .../tensorrt/test/reshape_transpose_test.py   |  2 +-
 7 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index bc82b3c0f0..fa36fa377b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -226,12 +226,12 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   item.fetch = output_names;
   item.graph = graph_def;
 
-  // TODO(aaroey): we should have used single machine cluster like the
-  // following, but the problem is then wrap_conversion will depend on
-  // direct_session and cause double linking problems. To fix this we need to
-  // fix or get rid of the swig dependency. Here we use VirtualCluster
-  // as a work around, and we need to create a session to initialize the
-  // underlying device before calling this method.
+// TODO(aaroey): we should have used single machine cluster like the
+// following, but the problem is then wrap_conversion will depend on
+// direct_session and cause double linking problems. To fix this we need to
+// fix or get rid of the swig dependency. Here we use VirtualCluster
+// as a work around, and we need to create a session to initialize the
+// underlying device before calling this method.
 #if 0
   // Create single machine cluster. Note that this will create a session and
   // initialize the gpu devices.
@@ -869,9 +869,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   tensorflow::tensorrt::segment::SegmentNodesVector initial_segments;
   TrtCandidateSelector candidate_selector(*params.graph_properties);
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
-      &graph,
-      std::bind(&TrtCandidateSelector::IsTensorRTCandidate, &candidate_selector,
-                std::placeholders::_1),
+      &graph, std::bind(&TrtCandidateSelector::IsTensorRTCandidate,
+                        &candidate_selector, std::placeholders::_1),
       // Input validation is already done by TrtCandidateSelector, so we don't
       // need to check the input edges.
       [](const Edge* edge) { return true; }, OutputEdgeValidator(),
@@ -970,8 +969,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                 &graph, alloc.get(), &engine_nodes);
     // If status is ok, we successfully added the node to the graph and can
     // remove segment ops. Otherwise graph is not modified.
-    string msg = StrCat("TensorRT node ", engine.engine_name, " added for segment ",
-                        i, " consisting of ",
+    string msg = StrCat("TensorRT node ", engine.engine_name,
+                        " added for segment ", i, " consisting of ",
                         converted_segments.at(i).first.size(), " nodes");
     if (status.ok()) {
       LOG(INFO) << msg << " succeeded.";
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 0e4dca5960..7c53c6b4bb 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -405,8 +405,8 @@ size_t TRT_ShapedWeights::size_bytes() const {
 
 string TRT_ShapedWeights::DebugString() const {
   return StrCat("TRT_ShapedWeights(shape=", convert::DebugString(shape_),
-                ", type=", DataTypeString(type_),
-                ", values=", reinterpret_cast<uintptr_t>(GetValues()), ")");
+                ", type=", DataTypeString(type_), ", values=",
+                reinterpret_cast<uintptr_t>(GetValues()), ")");
 }
 
 // A fake ITensor implementation used to check whether the TF-TRT converter can
@@ -651,11 +651,10 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
       break;
     }
     case tensorflow::DataType::DT_HALF: {
-      Reorder2(
-          {k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
-          istrides,
-          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
-          ostrides);
+      Reorder2({k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
+               istrides, static_cast<Eigen::half*>(
+                             const_cast<void*>(oweights->GetValues())),
+               ostrides);
       break;
     }
     default:
@@ -2008,9 +2007,9 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
           uint8* data = reinterpret_cast<uint8*>(temp_weights.data());
           std::copy(data, data + tensor.NumElements(), dst);
         } else {
-          return errors::FailedPrecondition(
-              "Unexpected data type: ", DataTypeString(dtype),
-              " at: ", node_def.name());
+          return errors::FailedPrecondition("Unexpected data type: ",
+                                            DataTypeString(dtype), " at: ",
+                                            node_def.name());
         }
       }
     }
@@ -2671,9 +2670,9 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
     op = nvinfer1::TopKOperation::kMAX;
     reducedAxes |= 1 << (nbDims - 1);
   } else {
-    return tensorflow::errors::Unimplemented(
-        "Operation: " + node_def.op() +
-        " not implemented, at: " + node_def.name());
+    return tensorflow::errors::Unimplemented("Operation: " + node_def.op() +
+                                             " not implemented, at: " +
+                                             node_def.name());
   }
 
   nvinfer1::ITopKLayer* layer = params->converter->network()->addTopK(
@@ -2982,7 +2981,8 @@ tensorflow::Status ConvertSegmentToGraphDef(
     }
   }
   *common_scope = local_scope;
-  VLOG(1) << "Converted TensorRT candidate segment @scope '" << local_scope << "' to a GraphDef";
+  VLOG(1) << "Converted TensorRT candidate segment @scope '" << local_scope
+          << "' to a GraphDef";
   return tensorflow::Status::OK();
 }
 
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 8037d47040..54f1d9b31f 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -252,9 +252,8 @@ int TRTEngineOp::GetEngineBatch(OpKernelContext* ctx) {
       cached_engine_batches_.push_back(num_batch);
       VLOG(1) << "Running with batch size " << num_batch;
     } else {
-      string msg =
-          StrCat("Engine buffer is full. buffer limit=", max_cached_engines_,
-                 ", current entries=");
+      string msg = StrCat("Engine buffer is full. buffer limit=",
+                          max_cached_engines_, ", current entries=");
       for (auto i : cached_engine_batches_) StrAppend(&msg, i, ",");
       StrAppend(&msg, " requested batch=", num_batch);
       LOG(WARNING) << msg;
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 7f1045c199..dbf8dd2614 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -189,7 +189,8 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     execute_engine_test_value = ("done" if expect_engine_is_run else "")
     execute_native_segment_test_value = ("" if expect_engine_is_run else "done")
     self.assertEqual(execute_engine_test_value,
-                     trt_convert.get_test_value("TRTEngineOp_0:ExecuteTrtEngine"))
+                     trt_convert.get_test_value(
+                         "TRTEngineOp_0:ExecuteTrtEngine"))
     self.assertEqual(
         execute_native_segment_test_value,
         trt_convert.get_test_value("TRTEngineOp_0:ExecuteNativeSegment"))
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 6b68f92d16..aac9e5c7bd 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -39,7 +39,8 @@ namespace tensorrt {
 class TRTCalibrationResource : public tensorflow::ResourceBase {
  public:
   ~TRTCalibrationResource() {
-    LOG(INFO) << "Destroying Calibration Resource " << std::endl << DebugString();
+    LOG(INFO) << "Destroying Calibration Resource " << std::endl
+              << DebugString();
     builder_.reset();
     engine_.reset();
     // We need to manually destroy the builder and engine before the allocator
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 917ff41fbf..d8f63779e6 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -434,11 +434,10 @@ tensorflow::Status SegmentGraph(
     }
     node_segments.emplace_back(node);
   }
-  string msg = StrCat("There are ", num_unsupported_ops,
-                      " ops of ", unsupported_ops.size(),
-                      " different types in the graph that",
-                      " are not converted to TensorRT: ");
-  for (const auto& elem: unsupported_ops) {
+  string msg = StrCat(
+      "There are ", num_unsupported_ops, " ops of ", unsupported_ops.size(),
+      " different types in the graph that", " are not converted to TensorRT: ");
+  for (const auto& elem : unsupported_ops) {
     StrAppend(&msg, elem, ", ");
   }
   LOG(INFO) << msg << "(For more information see "
@@ -461,7 +460,8 @@ tensorflow::Status SegmentGraph(
   std::vector<const SimpleNode*> order;
   order.reserve(graph->num_node_ids());
   StableDFS(*graph, /*reverse=*/false, {graph->source_node()},
-            /*enter=*/nullptr, [&order](const SimpleNode* n) {
+            /*enter=*/nullptr,
+            [&order](const SimpleNode* n) {
               order.push_back(n);
               return true;
             });
@@ -570,7 +570,7 @@ tensorflow::Status SegmentGraph(
     std::set<const tensorflow::Node*>& segment_nodes = itr.second;
     VLOG(1) << "Segment original size: " << segment_nodes.size();
     while (true) {
-      std::deque<const tensorflow::Node*> in_nodes_que, out_nodes_que;
+      std::deque<const tensorflow::Node *> in_nodes_que, out_nodes_que;
       // Find an input node that is not eligible and add it to the queue.
       // Nodes that has no incoming edges should not be treated as "input",
       // as there are really no inputs to them. Similar for output nodes.
@@ -616,8 +616,7 @@ tensorflow::Status SegmentGraph(
       // their outputs. In this way, for common cases the number of removed
       // nodes should be minimum.
       auto remove_nodes = [&segment_nodes](
-                              bool is_input_nodes,
-                              std::deque<const tensorflow::Node*>* que) {
+          bool is_input_nodes, std::deque<const tensorflow::Node*>* que) {
         // Run a BFS on the queue to find all the input/output nodes.
         std::set<const tensorflow::Node*> visited;
         std::set<const tensorflow::Node*> logged(que->begin(), que->end());
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
index 419e52b768..207944468a 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
@@ -80,7 +80,7 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return {
         "TRTEngineOp_0": ["reshape-%d" % i for i in range(7)] +
-                       ["reshape-%d/shape" % i for i in range(7)]
+                         ["reshape-%d/shape" % i for i in range(7)]
     }
 
   def ShouldRunTest(self, run_params):
-- 
GitLab


From bef3a3ffd7fba6839ab8233ea3710fa9fefa2d02 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 18:03:03 -0800
Subject: [PATCH 0643/1554] Add PickUnusedPortOrDie to c_api_experimental.

PiperOrigin-RevId: 222336760
---
 tensorflow/c/c_api_experimental.cc | 5 +++++
 tensorflow/c/c_api_experimental.h  | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 069de45e33..69de4cb711 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/platform/net.h"
 #include "tensorflow/core/platform/platform.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/tensorflow_server.pb.h"
@@ -8865,3 +8866,7 @@ int TF_OpIsStateful(const char* op_type, TF_Status* status) {
 void TF_InitMain(const char* usage, int* argc, char*** argv) {
   tensorflow::port::InitMain(usage, argc, argv);
 }
+
+int TF_PickUnusedPortOrDie() {
+  return tensorflow::internal::PickUnusedPortOrDie();
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 728445e8fd..c04cd441bf 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -237,6 +237,10 @@ TF_CAPI_EXPORT extern int TF_OpIsStateful(const char* op_type,
 // this to be called.
 TF_CAPI_EXPORT void TF_InitMain(const char* usage, int* argc, char*** argv);
 
+// Platform-specific implementation to return an unused port. (This should used
+// in tests only.)
+TF_CAPI_EXPORT int TF_PickUnusedPortOrDie();
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
-- 
GitLab


From 26800f6872e47478cc605bd6dc46330e6d027826 Mon Sep 17 00:00:00 2001
From: akikaaa <yuchunlong111@gmail.com>
Date: Wed, 21 Nov 2018 10:28:50 +0800
Subject: [PATCH 0644/1554] fix return type error

---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 932367f4dd..78c50fd8c5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2783,7 +2783,7 @@ def _export_output_to_tensors(export_output):
   elif isinstance(export_output, export_output_lib.RegressionOutput):
     return [export_output.value]
   elif isinstance(export_output, export_output_lib.PredictOutput):
-    return export_output.outputs.values()
+    return list(export_output.outputs.values())
   else:
     raise ValueError(
         '`export_output` must be have type `ClassificationOutput`, '
-- 
GitLab


From 3ffbbfb5a9e4cf0ae777453f0c70bc95c1e17d4e Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 20 Nov 2018 19:45:36 -0800
Subject: [PATCH 0645/1554] Move input spec checking logic to a separate file
 (part of a multi-step refactoring of base_layer.py to achieve better
 readability).

PiperOrigin-RevId: 222344969
---
 .../cudnn_rnn/python/layers/cudnn_rnn.py      |   3 +-
 .../keras/api/keras/layers/__init__.py        |   2 +-
 .../contrib/layers/python/layers/layers.py    |   5 +-
 .../python/layers/core_layers.py              |   9 +-
 tensorflow/contrib/rnn/python/ops/gru_ops.py  |   4 +-
 tensorflow/contrib/rnn/python/ops/lstm_ops.py |   5 +-
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |  14 +-
 tensorflow/python/keras/BUILD                 |   1 +
 tensorflow/python/keras/engine/__init__.py    |   2 +-
 tensorflow/python/keras/engine/base_layer.py  | 156 ++--------------
 tensorflow/python/keras/engine/input_spec.py  | 170 ++++++++++++++++++
 tensorflow/python/keras/layers/__init__.py    |   2 +-
 .../keras/layers/advanced_activations.py      |   2 +-
 .../python/keras/layers/convolutional.py      |   2 +-
 .../keras/layers/convolutional_recurrent.py   |   2 +-
 tensorflow/python/keras/layers/core.py        |   2 +-
 .../python/keras/layers/cudnn_recurrent.py    |   2 +-
 tensorflow/python/keras/layers/local.py       |   2 +-
 .../python/keras/layers/normalization.py      |   2 +-
 tensorflow/python/keras/layers/pooling.py     |   2 +-
 tensorflow/python/keras/layers/recurrent.py   |   2 +-
 .../python/keras/layers/unified_rnn_test.py   |   3 +-
 tensorflow/python/keras/layers/wrappers.py    |   2 +-
 tensorflow/python/layers/base.py              |   4 +-
 tensorflow/python/layers/base_test.py         |  13 +-
 tensorflow/python/layers/layers.py            |   2 +-
 tensorflow/python/ops/rnn_cell_impl.py        |   9 +-
 .../tensorflow.keras.layers.-input-spec.pbtxt |   2 +-
 .../v1/tensorflow.layers.-input-spec.pbtxt    |   2 +-
 .../tensorflow.keras.layers.-input-spec.pbtxt |   2 +-
 30 files changed, 235 insertions(+), 195 deletions(-)
 create mode 100644 tensorflow/python/keras/engine/input_spec.py

diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
index 8bbcc7cd03..8e25637ed9 100644
--- a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
+++ b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
@@ -21,6 +21,7 @@ from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -322,7 +323,7 @@ class _CudnnRNN(base_layer.Layer):
       raise ValueError("The last dimension of the inputs to `CudnnRNN` "
                        "should be defined. Found `None`.")
     self._input_size = input_shape[-1].value
-    self.input_spec = base_layer.InputSpec(ndim=3, axes={-1: self._input_size})
+    self.input_spec = input_spec.InputSpec(ndim=3, axes={-1: self._input_size})
 
     self._set_scope(None)
 
diff --git a/tensorflow/contrib/keras/api/keras/layers/__init__.py b/tensorflow/contrib/keras/api/keras/layers/__init__.py
index 3327a9f9a6..9e19884df8 100644
--- a/tensorflow/contrib/keras/api/keras/layers/__init__.py
+++ b/tensorflow/contrib/keras/api/keras/layers/__init__.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 # Generic layers.
 # pylint: disable=g-bad-import-order
-from tensorflow.python.keras.engine.base_layer import InputSpec
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py
index ac9561c769..403b522ce4 100644
--- a/tensorflow/contrib/layers/python/layers/layers.py
+++ b/tensorflow/contrib/layers/python/layers/layers.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.layers import base
 from tensorflow.python.layers import convolutional as convolutional_layers
 from tensorflow.python.layers import core as core_layers
@@ -1958,7 +1959,7 @@ class GDN(base.Layer):
     self._reparam_offset = reparam_offset
     self.data_format = data_format
     self._channel_axis()  # trigger ValueError early
-    self.input_spec = base.InputSpec(min_ndim=3, max_ndim=5)
+    self.input_spec = input_spec.InputSpec(min_ndim=3, max_ndim=5)
 
   def _channel_axis(self):
     try:
@@ -2015,7 +2016,7 @@ class GDN(base.Layer):
       raise ValueError('The channel dimension of the inputs to `GDN` '
                        'must be defined.')
     self._input_rank = input_shape.ndims
-    self.input_spec = base.InputSpec(
+    self.input_spec = input_spec.InputSpec(
         ndim=input_shape.ndims, axes={
             channel_axis: num_channels
         })
diff --git a/tensorflow/contrib/model_pruning/python/layers/core_layers.py b/tensorflow/contrib/model_pruning/python/layers/core_layers.py
index f0ce6fe039..1fa5c8cb48 100644
--- a/tensorflow/contrib/model_pruning/python/layers/core_layers.py
+++ b/tensorflow/contrib/model_pruning/python/layers/core_layers.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.layers import base
 from tensorflow.python.layers import utils
 from tensorflow.python.ops import array_ops
@@ -119,7 +120,7 @@ class _MaskedConv(base.Layer):
     self.bias_initializer = bias_initializer
     self.kernel_regularizer = kernel_regularizer
     self.bias_regularizer = bias_regularizer
-    self.input_spec = base.InputSpec(ndim=self.rank + 2)
+    self.input_spec = input_spec.InputSpec(ndim=self.rank + 2)
 
   def build(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape)
@@ -171,7 +172,7 @@ class _MaskedConv(base.Layer):
           dtype=self.dtype)
     else:
       self.bias = None
-    self.input_spec = base.InputSpec(
+    self.input_spec = input_spec.InputSpec(
         ndim=self.rank + 2, axes={channel_axis: input_dim})
     self.built = True
 
@@ -393,14 +394,14 @@ class MaskedFullyConnected(base.Layer):
     self.bias_initializer = bias_initializer
     self.kernel_regularizer = kernel_regularizer
     self.bias_regularizer = bias_regularizer
-    self.input_spec = base.InputSpec(min_ndim=2)
+    self.input_spec = input_spec.InputSpec(min_ndim=2)
 
   def build(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape)
     if tensor_shape.dimension_value(input_shape[-1]) is None:
       raise ValueError('The last dimension of the inputs to `Dense` '
                        'should be defined. Found `None`.')
-    self.input_spec = base.InputSpec(
+    self.input_spec = input_spec.InputSpec(
         min_ndim=2, axes={-1: tensor_shape.dimension_value(input_shape[-1])})
 
     self.kernel = self.add_variable(
diff --git a/tensorflow/contrib/rnn/python/ops/gru_ops.py b/tensorflow/contrib/rnn/python/ops/gru_ops.py
index b30ca7882f..251a933eae 100644
--- a/tensorflow/contrib/rnn/python/ops/gru_ops.py
+++ b/tensorflow/contrib/rnn/python/ops/gru_ops.py
@@ -21,7 +21,7 @@ from tensorflow.contrib.rnn.ops import gen_gru_ops
 from tensorflow.contrib.util import loader
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.layers import base as base_layer
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
@@ -165,7 +165,7 @@ class GRUBlockCell(LayerRNNCell):
       num_units = cell_size
     self._cell_size = num_units
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
   @property
   def state_size(self):
diff --git a/tensorflow/contrib/rnn/python/ops/lstm_ops.py b/tensorflow/contrib/rnn/python/ops/lstm_ops.py
index 4db431f85a..b043026bc5 100644
--- a/tensorflow/contrib/rnn/python/ops/lstm_ops.py
+++ b/tensorflow/contrib/rnn/python/ops/lstm_ops.py
@@ -25,6 +25,7 @@ from tensorflow.contrib.rnn.ops import gen_lstm_ops
 from tensorflow.contrib.util import loader
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -385,7 +386,7 @@ class LSTMBlockCell(LayerRNNCell):
         "scope": "lstm_cell"
     }
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
   @property
   def state_size(self):
@@ -628,7 +629,7 @@ class LSTMBlockFusedCell(LSTMBlockWrapper):
     self._use_peephole = use_peephole
 
     # Inputs must be 3-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=3)
+    self.input_spec = input_spec.InputSpec(ndim=3)
 
   @property
   def num_units(self):
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index e159dc9579..8a1c09f171 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -30,7 +30,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import initializers
-from tensorflow.python.layers import base as base_layer
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import gen_array_ops
@@ -2752,7 +2752,7 @@ class SRUCell(rnn_cell_impl.LayerRNNCell):
     self._activation = activation or math_ops.tanh
 
     # Restrict inputs to be 2-dimensional matrices
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
   @property
   def state_size(self):
@@ -3089,7 +3089,7 @@ class IndRNNCell(rnn_cell_impl.LayerRNNCell):
     super(IndRNNCell, self).__init__(_reuse=reuse, name=name, dtype=dtype)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     self._activation = activation or math_ops.tanh
@@ -3183,7 +3183,7 @@ class IndyGRUCell(rnn_cell_impl.LayerRNNCell):
     super(IndyGRUCell, self).__init__(_reuse=reuse, name=name, dtype=dtype)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     self._activation = activation or math_ops.tanh
@@ -3323,7 +3323,7 @@ class IndyLSTMCell(rnn_cell_impl.LayerRNNCell):
     super(IndyLSTMCell, self).__init__(_reuse=reuse, name=name, dtype=dtype)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     self._forget_bias = forget_bias
@@ -3444,7 +3444,7 @@ class MinimalRNNCell(rnn_cell_impl.LayerRNNCell):
     super(MinimalRNNCell, self).__init__(name=name, dtype=dtype, **kwargs)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self.units = units
     self.activation = activations.get(activation)
@@ -3558,7 +3558,7 @@ class CFNCell(rnn_cell_impl.LayerRNNCell):
     super(CFNCell, self).__init__(name=name, dtype=dtype, **kwargs)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self.units = units
     self.activation = activations.get(activation)
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 81ef941ba6..540dd03768 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -124,6 +124,7 @@ py_library(
         "engine/base_layer.py",
         "engine/distributed_training_utils.py",
         "engine/input_layer.py",
+        "engine/input_spec.py",
         "engine/network.py",
         "engine/saving.py",
         "engine/sequential.py",
diff --git a/tensorflow/python/keras/engine/__init__.py b/tensorflow/python/keras/engine/__init__.py
index 26aed34766..005f6462ff 100644
--- a/tensorflow/python/keras/engine/__init__.py
+++ b/tensorflow/python/keras/engine/__init__.py
@@ -20,10 +20,10 @@ from __future__ import print_function
 
 # TODO(fchollet): Remove hourglass imports once external code is done importing
 # non-public APIs.
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils.layer_utils import get_source_inputs
 
 del absolute_import
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 5dcbc4d04b..c8e964d1f7 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -36,6 +36,7 @@ from tensorflow.python.keras import backend
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import tf_utils
 # A module that only depends on `keras.layers` import these from here.
@@ -797,7 +798,8 @@ class Layer(checkpointable.CheckpointableBase):
     with ops.name_scope(self._name_scope()):
       if not self.built:
         # Check input assumptions set before layer building, e.g. input rank.
-        self._assert_input_compatibility(inputs)
+        input_spec.assert_input_compatibility(
+            self.input_spec, inputs, self.name)
         if input_list and self._dtype is None:
           try:
             self._dtype = input_list[0].dtype.base_dtype.name
@@ -822,7 +824,8 @@ class Layer(checkpointable.CheckpointableBase):
       if build_graph:
         # Symbolic execution on symbolic tensors. We will attempt to build
         # the corresponding TF subgraph inside `backend.get_graph()`
-        self._assert_input_compatibility(inputs)
+        input_spec.assert_input_compatibility(
+            self.input_spec, inputs, self.name)
         graph = backend.get_graph()
         with graph.as_default():
           if not executing_eagerly:
@@ -1492,101 +1495,6 @@ class Layer(checkpointable.CheckpointableBase):
     """Deprecated, do NOT use! Only for compatibility with external Keras."""
     return self._outbound_nodes
 
-  def _assert_input_compatibility(self, inputs):
-    """Checks compatibility between the layer and provided inputs.
-
-    This checks that the tensor(s) `inputs` verify the input assumptions
-    of the layer (if any). If not, a clear and actional exception gets raised.
-
-    Arguments:
-        inputs: input tensor or list of input tensors.
-
-    Raises:
-        ValueError: in case of mismatch between
-            the provided inputs and the expectations of the layer.
-    """
-    if not self.input_spec:
-      return
-    if not isinstance(self.input_spec, (list, tuple)):
-      input_spec = nest.flatten(self.input_spec)
-    else:
-      input_spec = self.input_spec
-    inputs = nest.flatten(inputs)
-    if len(inputs) != len(input_spec):
-      raise ValueError('Layer ' + self.name + ' expects ' +
-                       str(len(input_spec)) + ' inputs, '
-                       'but it received ' + str(len(inputs)) +
-                       ' input tensors. Inputs received: ' + str(inputs))
-    for input_index, (x, spec) in enumerate(zip(inputs, input_spec)):
-      if spec is None:
-        continue
-
-      if (spec.ndim is not None or
-          spec.min_ndim is not None or
-          spec.max_ndim is not None):
-        if x.shape.ndims is None:
-          raise ValueError('Input ' + str(input_index) + ' of layer ' +
-                           self.name + ' is incompatible with the layer: '
-                           'its rank is undefined, but the layer requires a '
-                           'defined rank.')
-
-      # Check ndim.
-      if spec.ndim is not None:
-        ndim = x.shape.ndims
-        if ndim != spec.ndim:
-          raise ValueError('Input ' + str(input_index) + ' of layer ' +
-                           self.name + ' is incompatible with the layer: '
-                           'expected ndim=' + str(spec.ndim) + ', found ndim=' +
-                           str(ndim) + '. Full shape received: ' +
-                           str(x.shape.as_list()))
-      if spec.max_ndim is not None:
-        ndim = x.shape.ndims
-        if ndim is not None and ndim > spec.max_ndim:
-          raise ValueError('Input ' + str(input_index) + ' of layer ' +
-                           self.name + ' is incompatible with the layer: '
-                           'expected max_ndim=' + str(spec.max_ndim) +
-                           ', found ndim=' + str(ndim))
-      if spec.min_ndim is not None:
-        ndim = x.shape.ndims
-        if ndim is not None and ndim < spec.min_ndim:
-          raise ValueError('Input ' + str(input_index) + ' of layer ' +
-                           self.name + ' is incompatible with the layer: '
-                           ': expected min_ndim=' + str(spec.min_ndim) +
-                           ', found ndim=' + str(ndim) +
-                           '. Full shape received: ' +
-                           str(x.shape.as_list()))
-      # Check dtype.
-      if spec.dtype is not None:
-        if x.dtype != spec.dtype:
-          raise ValueError('Input ' + str(input_index) + ' of layer ' +
-                           self.name + ' is incompatible with the layer: '
-                           'expected dtype=' + str(spec.dtype) +
-                           ', found dtype=' + str(x.dtype))
-      # Check specific shape axes.
-      if spec.axes:
-        shape = x.shape.as_list()
-        if shape is not None:
-          for axis, value in spec.axes.items():
-            if hasattr(value, 'value'):
-              value = value.value
-            if value is not None and shape[int(axis)] not in {value, None}:
-              raise ValueError(
-                  'Input ' + str(input_index) + ' of layer ' + self.name + ' is'
-                  ' incompatible with the layer: expected axis ' + str(axis) +
-                  ' of input shape to have value ' + str(value) +
-                  ' but received input with shape ' + str(shape))
-      # Check shape.
-      if spec.shape is not None:
-        shape = x.shape.as_list()
-        if shape is not None:
-          for spec_dim, dim in zip(spec.shape, shape):
-            if spec_dim is not None and dim is not None:
-              if spec_dim != dim:
-                raise ValueError('Input ' + str(input_index) +
-                                 ' is incompatible with layer ' + self.name +
-                                 ': expected shape=' + str(spec.shape) +
-                                 ', found shape=' + str(shape))
-
   def set_weights(self, weights):
     """Sets the weights of the layer, from Numpy arrays.
 
@@ -1687,55 +1595,6 @@ class Layer(checkpointable.CheckpointableBase):
     return self._call_is_graph_friendly
 
 
-@tf_export(
-    'keras.layers.InputSpec', v1=['keras.layers.InputSpec', 'layers.InputSpec'])
-class InputSpec(object):
-  """Specifies the ndim, dtype and shape of every input to a layer.
-
-  Every layer should expose (if appropriate) an `input_spec` attribute:
-  a list of instances of InputSpec (one per input tensor).
-
-  A None entry in a shape is compatible with any dimension,
-  a None shape is compatible with any shape.
-
-  Arguments:
-      dtype: Expected DataType of the input.
-      shape: Shape tuple, expected shape of the input
-          (may include None for unchecked axes).
-      ndim: Integer, expected rank of the input.
-      max_ndim: Integer, maximum rank of the input.
-      min_ndim: Integer, minimum rank of the input.
-      axes: Dictionary mapping integer axes to
-          a specific dimension value.
-  """
-
-  def __init__(self,
-               dtype=None,
-               shape=None,
-               ndim=None,
-               max_ndim=None,
-               min_ndim=None,
-               axes=None):
-    self.dtype = dtype
-    self.shape = shape
-    if shape is not None:
-      self.ndim = len(shape)
-    else:
-      self.ndim = ndim
-    self.max_ndim = max_ndim
-    self.min_ndim = min_ndim
-    self.axes = axes or {}
-
-  def __repr__(self):
-    spec = [('dtype=' + str(self.dtype)) if self.dtype else '',
-            ('shape=' + str(self.shape)) if self.shape else '',
-            ('ndim=' + str(self.ndim)) if self.ndim else '',
-            ('max_ndim=' + str(self.max_ndim)) if self.max_ndim else '',
-            ('min_ndim=' + str(self.min_ndim)) if self.min_ndim else '',
-            ('axes=' + str(self.axes)) if self.axes else '']
-    return 'InputSpec(%s)' % ', '.join(x for x in spec if x)
-
-
 class Node(object):
   """A `Node` describes the connectivity between two layers.
 
@@ -2037,3 +1896,8 @@ def default(method):
 
 def generate_placeholders_from_shape(shape):
   return array_ops.placeholder(shape=shape, dtype=backend.floatx())
+
+
+# Avoid breaking users who directly import this symbol from this file.
+# TODO(fchollet): remove this.
+InputSpec = input_spec.InputSpec  # pylint:disable=invalid-name
diff --git a/tensorflow/python/keras/engine/input_spec.py b/tensorflow/python/keras/engine/input_spec.py
new file mode 100644
index 0000000000..7277c16fe5
--- /dev/null
+++ b/tensorflow/python/keras/engine/input_spec.py
@@ -0,0 +1,170 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=protected-access
+"""Contains the InputSpec class."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from six.moves import zip  # pylint: disable=redefined-builtin
+
+from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export('keras.layers.InputSpec',
+           v1=['keras.layers.InputSpec', 'layers.InputSpec'])
+class InputSpec(object):
+  """Specifies the ndim, dtype and shape of every input to a layer.
+
+  Every layer should expose (if appropriate) an `input_spec` attribute:
+  a list of instances of InputSpec (one per input tensor).
+
+  A None entry in a shape is compatible with any dimension,
+  a None shape is compatible with any shape.
+
+  Arguments:
+      dtype: Expected DataType of the input.
+      shape: Shape tuple, expected shape of the input
+          (may include None for unchecked axes).
+      ndim: Integer, expected rank of the input.
+      max_ndim: Integer, maximum rank of the input.
+      min_ndim: Integer, minimum rank of the input.
+      axes: Dictionary mapping integer axes to
+          a specific dimension value.
+  """
+
+  def __init__(self,
+               dtype=None,
+               shape=None,
+               ndim=None,
+               max_ndim=None,
+               min_ndim=None,
+               axes=None):
+    self.dtype = dtype
+    self.shape = shape
+    if shape is not None:
+      self.ndim = len(shape)
+    else:
+      self.ndim = ndim
+    self.max_ndim = max_ndim
+    self.min_ndim = min_ndim
+    self.axes = axes or {}
+
+  def __repr__(self):
+    spec = [('dtype=' + str(self.dtype)) if self.dtype else '',
+            ('shape=' + str(self.shape)) if self.shape else '',
+            ('ndim=' + str(self.ndim)) if self.ndim else '',
+            ('max_ndim=' + str(self.max_ndim)) if self.max_ndim else '',
+            ('min_ndim=' + str(self.min_ndim)) if self.min_ndim else '',
+            ('axes=' + str(self.axes)) if self.axes else '']
+    return 'InputSpec(%s)' % ', '.join(x for x in spec if x)
+
+
+def assert_input_compatibility(input_spec, inputs, layer_name):
+  """Checks compatibility between the layer and provided inputs.
+
+  This checks that the tensor(s) `inputs` verify the input assumptions
+  of a layer (if any). If not, a clear and actional exception gets raised.
+
+  Arguments:
+      input_spec: An InputSpec instance, or None.
+      inputs: Input tensor or list of input tensors.
+      layer_name: String, name of the layer (for error message formatting).
+
+  Raises:
+      ValueError: in case of mismatch between
+          the provided inputs and the expectations of the layer.
+  """
+  if not input_spec:
+    return
+  if not isinstance(input_spec, (list, tuple)):
+    input_spec = nest.flatten(input_spec)
+
+  inputs = nest.flatten(inputs)
+  if len(inputs) != len(input_spec):
+    raise ValueError('Layer ' + layer_name + ' expects ' +
+                     str(len(input_spec)) + ' inputs, '
+                     'but it received ' + str(len(inputs)) +
+                     ' input tensors. Inputs received: ' + str(inputs))
+  for input_index, (x, spec) in enumerate(zip(inputs, input_spec)):
+    if spec is None:
+      continue
+
+    if (spec.ndim is not None or
+        spec.min_ndim is not None or
+        spec.max_ndim is not None):
+      if x.shape.ndims is None:
+        raise ValueError('Input ' + str(input_index) + ' of layer ' +
+                         layer_name + ' is incompatible with the layer: '
+                         'its rank is undefined, but the layer requires a '
+                         'defined rank.')
+
+    # Check ndim.
+    if spec.ndim is not None:
+      ndim = x.shape.ndims
+      if ndim != spec.ndim:
+        raise ValueError('Input ' + str(input_index) + ' of layer ' +
+                         layer_name + ' is incompatible with the layer: '
+                         'expected ndim=' + str(spec.ndim) + ', found ndim=' +
+                         str(ndim) + '. Full shape received: ' +
+                         str(x.shape.as_list()))
+    if spec.max_ndim is not None:
+      ndim = x.shape.ndims
+      if ndim is not None and ndim > spec.max_ndim:
+        raise ValueError('Input ' + str(input_index) + ' of layer ' +
+                         layer_name + ' is incompatible with the layer: '
+                         'expected max_ndim=' + str(spec.max_ndim) +
+                         ', found ndim=' + str(ndim))
+    if spec.min_ndim is not None:
+      ndim = x.shape.ndims
+      if ndim is not None and ndim < spec.min_ndim:
+        raise ValueError('Input ' + str(input_index) + ' of layer ' +
+                         layer_name + ' is incompatible with the layer: '
+                         ': expected min_ndim=' + str(spec.min_ndim) +
+                         ', found ndim=' + str(ndim) +
+                         '. Full shape received: ' +
+                         str(x.shape.as_list()))
+    # Check dtype.
+    if spec.dtype is not None:
+      if x.dtype != spec.dtype:
+        raise ValueError('Input ' + str(input_index) + ' of layer ' +
+                         layer_name + ' is incompatible with the layer: '
+                         'expected dtype=' + str(spec.dtype) +
+                         ', found dtype=' + str(x.dtype))
+    # Check specific shape axes.
+    if spec.axes:
+      shape = x.shape.as_list()
+      if shape is not None:
+        for axis, value in spec.axes.items():
+          if hasattr(value, 'value'):
+            value = value.value
+          if value is not None and shape[int(axis)] not in {value, None}:
+            raise ValueError(
+                'Input ' + str(input_index) + ' of layer ' + layer_name + ' is'
+                ' incompatible with the layer: expected axis ' + str(axis) +
+                ' of input shape to have value ' + str(value) +
+                ' but received input with shape ' + str(shape))
+    # Check shape.
+    if spec.shape is not None:
+      shape = x.shape.as_list()
+      if shape is not None:
+        for spec_dim, dim in zip(spec.shape, shape):
+          if spec_dim is not None and dim is not None:
+            if spec_dim != dim:
+              raise ValueError('Input ' + str(input_index) +
+                               ' is incompatible with layer ' + layer_name +
+                               ': expected shape=' + str(spec.shape) +
+                               ', found shape=' + str(shape))
diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py
index 7268040b02..49990b6bf4 100644
--- a/tensorflow/python/keras/layers/__init__.py
+++ b/tensorflow/python/keras/layers/__init__.py
@@ -22,7 +22,7 @@ from __future__ import print_function
 # pylint: disable=g-bad-import-order
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
-from tensorflow.python.keras.engine.base_layer import InputSpec
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
 
 # Advanced activations.
diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py
index a2385dfdbb..35ac7830b2 100644
--- a/tensorflow/python/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/layers/advanced_activations.py
@@ -22,8 +22,8 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py
index d1b03b8822..6564d6e8fd 100644
--- a/tensorflow/python/keras/layers/convolutional.py
+++ b/tensorflow/python/keras/layers/convolutional.py
@@ -26,8 +26,8 @@ from tensorflow.python.keras import backend
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 # imports for backwards namespace compatibility
 # pylint: disable=unused-import
 from tensorflow.python.keras.layers.pooling import AveragePooling1D
diff --git a/tensorflow/python/keras/layers/convolutional_recurrent.py b/tensorflow/python/keras/layers/convolutional_recurrent.py
index 100542129b..cf3861da21 100644
--- a/tensorflow/python/keras/layers/convolutional_recurrent.py
+++ b/tensorflow/python/keras/layers/convolutional_recurrent.py
@@ -26,8 +26,8 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.layers.recurrent import _generate_dropout_mask
 from tensorflow.python.keras.layers.recurrent import _standardize_args
 from tensorflow.python.keras.layers.recurrent import RNN
diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index 8031272e25..56dd70558c 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -34,8 +34,8 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import conv_utils
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import tf_utils
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent.py b/tensorflow/python/keras/layers/cudnn_recurrent.py
index beacdf2515..81f292817f 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent.py
@@ -25,7 +25,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.layers.recurrent import RNN
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_cudnn_rnn_ops
diff --git a/tensorflow/python/keras/layers/local.py b/tensorflow/python/keras/layers/local.py
index 33d09a1660..d2c4aaa125 100644
--- a/tensorflow/python/keras/layers/local.py
+++ b/tensorflow/python/keras/layers/local.py
@@ -23,8 +23,8 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import conv_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.util.tf_export import tf_export
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 7a91693128..aa8598d731 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -26,8 +26,8 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
diff --git a/tensorflow/python/keras/layers/pooling.py b/tensorflow/python/keras/layers/pooling.py
index 72a9c1d629..a0744cddad 100644
--- a/tensorflow/python/keras/layers/pooling.py
+++ b/tensorflow/python/keras/layers/pooling.py
@@ -22,8 +22,8 @@ import functools
 
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import conv_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index d04533815e..5d0efc2f16 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -28,8 +28,8 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import array_ops
diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index 22d29f191b..744d51824b 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -36,8 +36,7 @@ from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
 from tensorflow.python.keras import testing_utils
-from tensorflow.python.keras.engine.base_layer import \
-  InputSpec
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.layers.cudnn_recurrent import CuDNNLSTM
 from tensorflow.python.keras.layers.recurrent import RNN
 from tensorflow.python.keras.utils import tf_utils
diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index 27419a1ffc..67b154141e 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -23,8 +23,8 @@ import copy
 
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend as K
-from tensorflow.python.keras.engine.base_layer import InputSpec
 from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.layers.recurrent import _standardize_args
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import tf_utils
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index fccea484b0..42086e4c3e 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -30,10 +30,10 @@ from tensorflow.python.util import nest
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-
+# Avoid breaking users who directly import this symbol from this file.
+# TODO(fchollet): remove this.
 InputSpec = base_layer.InputSpec  # pylint: disable=invalid-name
 
-
 _KERAS_STYLE_SCOPE = False
 
 
diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py
index 90abf35e87..45099677e0 100644
--- a/tensorflow/python/layers/base_test.py
+++ b/tensorflow/python/layers/base_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import base_layer as keras_base_layer
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.layers import base as base_layers
 from tensorflow.python.layers import core as core_layers
 from tensorflow.python.ops import array_ops
@@ -251,7 +252,7 @@ class BaseLayerTest(test.TestCase):
 
       def __init__(self):
         super(CustomerLayer, self).__init__()
-        self.input_spec = base_layers.InputSpec(ndim=2)
+        self.input_spec = input_spec.InputSpec(ndim=2)
 
       def call(self, inputs):
         return inputs
@@ -278,7 +279,7 @@ class BaseLayerTest(test.TestCase):
 
       def __init__(self):
         super(CustomerLayer, self).__init__()
-        self.input_spec = base_layers.InputSpec(min_ndim=2)
+        self.input_spec = input_spec.InputSpec(min_ndim=2)
 
       def call(self, inputs):
         return inputs
@@ -306,7 +307,7 @@ class BaseLayerTest(test.TestCase):
 
       def __init__(self):
         super(CustomerLayer, self).__init__()
-        self.input_spec = base_layers.InputSpec(max_ndim=2)
+        self.input_spec = input_spec.InputSpec(max_ndim=2)
 
       def call(self, inputs):
         return inputs
@@ -334,7 +335,7 @@ class BaseLayerTest(test.TestCase):
 
       def __init__(self):
         super(CustomerLayer, self).__init__()
-        self.input_spec = base_layers.InputSpec(dtype='float32')
+        self.input_spec = input_spec.InputSpec(dtype='float32')
 
       def call(self, inputs):
         return inputs
@@ -354,7 +355,7 @@ class BaseLayerTest(test.TestCase):
 
       def __init__(self):
         super(CustomerLayer, self).__init__()
-        self.input_spec = base_layers.InputSpec(axes={-1: 2})
+        self.input_spec = input_spec.InputSpec(axes={-1: 2})
 
       def call(self, inputs):
         return inputs
@@ -376,7 +377,7 @@ class BaseLayerTest(test.TestCase):
 
       def __init__(self):
         super(CustomerLayer, self).__init__()
-        self.input_spec = base_layers.InputSpec(shape=(None, 3))
+        self.input_spec = input_spec.InputSpec(shape=(None, 3))
 
       def call(self, inputs):
         return inputs
diff --git a/tensorflow/python/layers/layers.py b/tensorflow/python/layers/layers.py
index 11a2ebc040..93eec38a08 100644
--- a/tensorflow/python/layers/layers.py
+++ b/tensorflow/python/layers/layers.py
@@ -24,7 +24,7 @@ from __future__ import print_function
 
 # Base objects.
 from tensorflow.python.layers.base import Layer
-from tensorflow.python.layers.base import InputSpec
+from tensorflow.python.keras.engine.input_spec import InputSpec
 
 # Core layers.
 from tensorflow.python.layers.core import Dense
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 2bc1ad4e04..85efd6a4f7 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -36,6 +36,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import initializers
+from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
@@ -410,7 +411,7 @@ class BasicRNNCell(LayerRNNCell):
                    "performance on GPU.", self)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     if activation:
@@ -507,7 +508,7 @@ class GRUCell(LayerRNNCell):
                    "Please use tf.contrib.cudnn_rnn.CudnnGRU for better "
                    "performance on GPU.", self)
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     if activation:
@@ -683,7 +684,7 @@ class BasicLSTMCell(LayerRNNCell):
                    "performance on GPU.", self)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     self._forget_bias = forget_bias
@@ -871,7 +872,7 @@ class LSTMCell(LayerRNNCell):
                    "performance on GPU.", self)
 
     # Inputs must be 2-dimensional.
-    self.input_spec = base_layer.InputSpec(ndim=2)
+    self.input_spec = input_spec.InputSpec(ndim=2)
 
     self._num_units = num_units
     self._use_peepholes = use_peepholes
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt
index 5fd0a47a68..bc3ceb67a4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.keras.layers.InputSpec"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.InputSpec\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.input_spec.InputSpec\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt
index fd02c919ae..80834e08f7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.layers.InputSpec"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.InputSpec\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.input_spec.InputSpec\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt
index 5fd0a47a68..bc3ceb67a4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.keras.layers.InputSpec"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.InputSpec\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.input_spec.InputSpec\'>"
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-- 
GitLab


From abdd39fa9e078d2642397055ef669936746b4196 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Nov 2018 19:56:41 -0800
Subject: [PATCH 0646/1554] Move the implementation of MirroredStrategy to
 tensorflow/python/distribute/. Note that this implementation is not yet
 exposed publicly throught the TensorFlow API.

PiperOrigin-RevId: 222345484
---
 tensorflow/contrib/distribute/python/BUILD    |  21 +-
 .../python/collective_all_reduce_strategy.py  |   5 +
 .../distribute/python/mirrored_strategy.py    | 791 +----------------
 .../distribute/python/one_device_strategy.py  |   5 +
 .../python/parameter_server_strategy.py       |   5 +
 .../contrib/distribute/python/tpu_strategy.py |   5 +
 tensorflow/python/distribute/BUILD            |  29 +
 .../python/distribute/mirrored_strategy.py    | 805 ++++++++++++++++++
 .../engine/distributed_training_utils.py      |  11 +-
 tensorflow/python/training/distribute.py      |   5 +
 10 files changed, 878 insertions(+), 804 deletions(-)
 create mode 100644 tensorflow/python/distribute/mirrored_strategy.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 2a595e7c87..91282a8c1d 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -49,28 +49,9 @@ py_library(
     srcs = ["mirrored_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:device",
-        "//tensorflow/python:device_util",
         "//tensorflow/python:distribute",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:pywrap_tensorflow",
-        "//tensorflow/python:tensor_util",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python:variables",
-        "//tensorflow/python/distribute:cross_device_ops",
-        "//tensorflow/python/distribute:multi_worker_util",
-        "//tensorflow/python/distribute:reduce_util",
-        "//tensorflow/python/distribute:shared_variable_creator",
+        "//tensorflow/python/distribute:mirrored_strategy",
         "//tensorflow/python/distribute:values",
-        "//tensorflow/python/eager:context",
-        "//tensorflow/python/eager:tape",
     ],
 )
 
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index f13cf26d36..906377b739 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -320,3 +320,8 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   @property
   def _num_replicas_in_sync(self):
     return len(self._devices) * self._num_workers
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return False
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index d671d1d562..a3bcc8db88 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -12,760 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Class MirroredStrategy implementing DistributionStrategy."""
+"""Contrib version of MirroredStrategy."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import contextlib
-from functools import partial
-import threading
+import functools
 
-from tensorflow.python import pywrap_tensorflow
-from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
-from tensorflow.python.distribute import multi_worker_util
-from tensorflow.python.distribute import reduce_util
-from tensorflow.python.distribute import shared_variable_creator
+from tensorflow.python.distribute import mirrored_strategy
 from tensorflow.python.distribute import values
-from tensorflow.python.eager import context
-from tensorflow.python.eager import tape
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import device as tf_device
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.training import coordinator
-from tensorflow.python.training import device_util
 from tensorflow.python.training import distribute as distribute_lib
-from tensorflow.python.util import nest
 
 
-# TODO(josh11b): Replace asserts in this file with if ...: raise ...
-
-
-@contextlib.contextmanager
-def _enter_graph(g):
-  if context.executing_eagerly():
-    with g.as_default(), context.eager_mode():
-      yield
-  else:
-    with g.as_default():
-      yield
-
-
-def _cpu_device(device):
-  cpu_device = tf_device.DeviceSpec.from_string(device)
-  cpu_device.merge_from(tf_device.DeviceSpec(device_type="CPU", device_index=0))
-  return cpu_device.to_string()
-
-
-class _RequestedStop(Exception):
-  pass
-
-
-# _call_for_each_replica and _reduce_non_distributed_value are not members of
-# MirroredStrategy so that they are generally not allowed to use anything
-# specific to MirroredStrategy and thus can be shared with other distribution
-# strategies.
-
-
-# TODO(yuefengz): maybe create a common class for those who need to call this
-# _call_for_each_replica.
-def _call_for_each_replica(distribution, fn, args, kwargs):
-  """Run `fn` in separate threads, once per replica/worker device.
-
-  Args:
-    distribution: the DistributionStrategy object.
-    fn: function to run (will be run once per device, each in its own thread).
-    args: positional arguments for `fn`
-    kwargs: keyword arguments for `fn`.
-
-  Returns:
-    Merged return value of `fn` across all replicas.
-
-  Raises:
-    RuntimeError: If fn() calls get_replica_context().merge_call() a different
-        number of times from the available devices.
-  """
-  # TODO(josh11b): Add this option once we add synchronization to variable
-  # creation. Until then, this is pretty unsafe to use.
-  run_concurrently = False
-  if not context.executing_eagerly():
-    # Needed for per-thread device, etc. contexts in graph mode.
-    ops.get_default_graph().switch_to_thread_local()
-
-  coord = coordinator.Coordinator(clean_stop_exception_types=(_RequestedStop,))
-
-  shared_variable_store = {}
-
-  # TODO(isaprykin): Create these threads once instead of during every run()
-  # call.
-  threads = []
-  for index, d in enumerate(distribution.extended.worker_devices):
-    variable_creator_fn = shared_variable_creator.make_fn(
-        shared_variable_store, index)
-    t = MirroredExtended._MirroredReplicaThread(  # pylint: disable=protected-access
-        distribution, coord, d, variable_creator_fn, fn,
-        *values.select_device(d, args), **values.select_device(d, kwargs))
-    threads.append(t)
-
-  for t in threads:
-    t.start()
-
-  # When `fn` starts `should_run` event is set on _MirroredReplicaThread
-  # (`MRT`) threads. The execution waits until
-  # `MRT.has_paused` is set, which indicates that either `fn` is
-  # complete or a `get_replica_context().merge_call()` is called.  If `fn` is
-  # complete, then `MRT.done` is set to True.  Otherwise, arguments
-  # of `get_replica_context().merge_call` from all paused threads are grouped
-  # and the `merge_fn` is performed.  Results of the
-  # `get_replica_context().merge_call` are then set to `MRT.merge_result`.
-  # Each such `get_replica_context().merge_call` call returns the
-  # `MRT.merge_result` for that thread when `MRT.should_run` event
-  # is reset again. Execution of `fn` resumes.
-
-  try:
-    with coord.stop_on_exception():
-      all_done = False
-      while not all_done and not coord.should_stop():
-        done = []
-        if run_concurrently:
-          for t in threads:
-            t.should_run.set()
-          for t in threads:
-            t.has_paused.wait()
-            t.has_paused.clear()
-            if coord.should_stop():
-              return None
-            done.append(t.done)
-        else:
-          for t in threads:
-            t.should_run.set()
-            t.has_paused.wait()
-            t.has_paused.clear()
-            if coord.should_stop():
-              return None
-            done.append(t.done)
-        if coord.should_stop():
-          return None
-        all_done = all(done)
-        if not all_done:
-          if any(done):
-            raise RuntimeError("Some replicas made a different number of "
-                               "replica_context().merge_call() calls.")
-          # get_replica_context().merge_call() case
-          merge_args = values.regroup({t.device: t.merge_args for t in threads})
-          merge_kwargs = values.regroup(
-              {t.device: t.merge_kwargs for t in threads})
-          # We capture the name_scope of the MRT when we call merge_fn
-          # to ensure that if we have opened a name scope in the MRT,
-          # it will be respected when executing the merge function. We only
-          # capture the name_scope from the first MRT and assume it is
-          # the same for all other MRTs.
-          mtt_captured_name_scope = threads[0].captured_name_scope
-          with ops.name_scope(mtt_captured_name_scope):
-            merge_result = threads[0].merge_fn(distribution, *merge_args,
-                                               **merge_kwargs)
-          for t in threads:
-            t.merge_result = values.select_device(t.device, merge_result)
-  finally:
-    for t in threads:
-      t.should_run.set()
-    coord.join(threads)
-
-  return values.regroup({t.device: t.main_result for t in threads})
-
-
-def _reduce_non_distributed_value(extended, reduce_op, value, destinations):
-  """Reduce a non-DistributedValue `value` to `destinations`."""
-  if isinstance(value, values.DistributedValues):
-    raise ValueError("You are passing a `DistributedValue` to "
-                     "`_reduce_non_distributed_value`, which is not allowed.")
-
-  # If the same value is present on all replicas then the PerReplica value will
-  # be a single value. We also handle the case when `value` is a single value
-  # and equal to 0.
-  if value == 0:
-    return 0
-  # If there is only a single value and the reduce op is MEAN,
-  # that value should be on all destinations.
-  if reduce_op == reduce_util.ReduceOp.MEAN:
-    return value
-
-  cross_device_ops_lib.validate_destinations(destinations)
-  # We do not support a reduce op of SUM if the value is the same across
-  # all replicas. We call this as part of assign functions for MirroredVariables
-  # and summing up identical values across replicas is not clearly defined.
-  if (len(extended.worker_devices) != 1 or
-      not cross_device_ops_lib.check_destinations(destinations)):
-    raise ValueError("A non-DistributedValues value %s cannot be reduced with "
-                     "the given reduce op %s." % (value, reduce_op))
-  # TODO(anjalisridhar): Moves these methods to a device utility file?
-  devices = cross_device_ops_lib.get_devices_from(destinations)
-  if len(devices) == 1:
-    with ops.device(devices[0]):
-      return array_ops.identity(value)
-  else:
-    value_updates = {}
-    for d in devices:
-      with ops.device(d):
-        value_updates[d] = array_ops.identity(value)
-    return values.Mirrored(value_updates)
-
-
-def _create_mirrored_variable(devices, real_mirrored_creator, *args, **kwargs):  # pylint: disable=g-missing-docstring
-  # Figure out what collections this variable should be added to.
-  # We'll add the MirroredVariable to those collections instead.
-  collections = kwargs.pop("collections", None)
-  if collections is None:
-    collections = [ops.GraphKeys.GLOBAL_VARIABLES]
-  kwargs["collections"] = []
-
-  # Get synchronization value
-  synchronization = kwargs.get("synchronization",
-                               variable_scope.VariableSynchronization.ON_WRITE)
-  if synchronization == variable_scope.VariableSynchronization.NONE:
-    raise ValueError("`NONE` variable synchronization mode is not "
-                     "supported with `Mirrored` distribution strategy. Please"
-                     " change the `synchronization` for variable: " +
-                     kwargs["name"])
-  elif synchronization == variable_scope.VariableSynchronization.ON_READ:
-    # Variables that are to be synced on read are replica local.
-    is_replica_local = True
-    kwargs["trainable"] = False
-  elif (synchronization == variable_scope.VariableSynchronization.ON_WRITE or
-        synchronization == variable_scope.VariableSynchronization.AUTO):
-    # `AUTO` synchronization for `MirroredStrategy` is `ON_WRITE`.
-    is_replica_local = False
-  else:
-    raise ValueError("Invalid variable synchronization mode: " +
-                     synchronization + " for variable: " + kwargs["name"])
-
-  # Get aggregation value
-  aggregation = kwargs.pop("aggregation",
-                           variable_scope.VariableAggregation.NONE)
-  if aggregation not in (
-      variable_scope.VariableAggregation.NONE,
-      variable_scope.VariableAggregation.SUM,
-      variable_scope.VariableAggregation.MEAN,
-      variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
-  ):
-    raise ValueError("Invalid variable aggregation mode: " + aggregation +
-                     " for variable: " + kwargs["name"])
-
-  # Ignore user-specified caching device, not needed for mirrored variables.
-  kwargs.pop("caching_device", None)
-
-  # TODO(josh11b,apassos): It would be better if variable initialization
-  # was never recorded on the tape instead of having to do this manually
-  # here.
-  with tape.stop_recording():
-    index = real_mirrored_creator(devices, *args, **kwargs)
-
-    if is_replica_local:
-      result = values.ReplicaLocalVariable(
-          index, index[devices[0]], aggregation)
-    else:
-      result = values.MirroredVariable(index, index[devices[0]], aggregation)
-
-  # Add the wrapped variable to the requested collections.
-  # The handling of eager mode and the global step matches
-  # ResourceVariable._init_from_args().
-  if not context.executing_eagerly():
-    g = ops.get_default_graph()
-    # If "trainable" is True, next_creator() will add the member variables
-    # to the TRAINABLE_VARIABLES collection, so we manually remove
-    # them and replace with the MirroredVariable. We can't set
-    # "trainable" to False for next_creator() since that causes functions
-    # like implicit_gradients to skip those variables.
-    if kwargs.get("trainable", True):
-      collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
-      l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
-      for v in index.values():
-        if v in l:
-          l.remove(v)
-    g.add_to_collections(collections, result)
-  elif ops.GraphKeys.GLOBAL_STEP in collections:
-    ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, result)
-
-  return result
-
-
-class CoreMirroredStrategy(distribute_lib.DistributionStrategy):
-  """Mirrors vars to distribute across multiple devices and machines.
-
-  *** core version ***
-
-  This strategy uses one replica per device and sync replication for its
-  multi-GPU version.
-
-  The multi-worker version will be added in the fture.
-
-  Args:
-    devices: a list of device strings.
-    num_gpus_per_worker: number of GPUs per worker.
-    cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not
-      set, nccl will be use by default.
-  """
-
-  def __init__(self,
-               devices=None,
-               num_gpus_per_worker=None,
-               cross_device_ops=None):
-    extended = CoreMirroredExtended(self, devices, num_gpus_per_worker,
-                                    cross_device_ops)
-    super(CoreMirroredStrategy, self).__init__(extended)
-
-
-class CoreMirroredExtended(distribute_lib.DistributionStrategyExtended):
-  """Implementation of CoreMirroredStrategy."""
-
-  def __init__(self,
-               container_strategy,
-               devices=None,
-               num_gpus_per_worker=None,
-               cross_device_ops=None):
-    super(CoreMirroredExtended, self).__init__(container_strategy)
-    self._cross_device_ops = cross_device_ops
-    # Remember num GPUs which might be needed by `configure` method.
-    self._num_gpus = num_gpus_per_worker
-
-    self._initialize_local(self._num_gpus, devices)
-
-  def _initialize_local(self, num_gpus, devices):
-    """Initializes the object for local training."""
-    self._cluster_spec = None
-    # Convert `num_gpus` into `devices`, shouldn't specify both.
-    if devices is None:
-      if num_gpus is None:
-        num_gpus = context.num_gpus()
-      if num_gpus == 0:
-        devices = ["/device:CPU:0"]
-      else:
-        devices = ["/device:GPU:%d" % d for d in range(num_gpus)]
-    elif num_gpus is not None:
-      raise ValueError("Must only specify one of `devices` and `num_gpus`.")
-    self._num_gpus = num_gpus
-    # TODO(yuefengz): consider setting the default device.
-
-    assert devices, "Must specify at least one device."
-    assert len(set(devices)) == len(devices), (
-        "No duplicates allowed in `devices` argument.")
-    # TODO(josh11b): Require at least 2 devices?
-    self._devices = [device_util.resolve(d) for d in devices]
-    self._canonical_device_set = set(self._devices)
-    self._device_index = values.PerReplica(
-        {d: i for i, d in enumerate(devices)})
-
-  def _initialize_multi_worker(self, num_gpus, cluster_spec):
-    """Initializes the object for multi-worker training."""
-    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-    self._cluster_spec = cluster_spec
-
-    self._workers = []
-    for job in ["chief", "worker"]:
-      for task in range(len(cluster_spec.as_dict().get(job, []))):
-        self._workers.append("/job:%s/task:%d" % (job, task))
-
-    if num_gpus is None:
-      raise ValueError("`num_gpus` is required if `cluster_spec` is given.")
-    if num_gpus > 0:
-      self._worker_devices = [
-          (worker, [
-              device_util.canonicalize(worker + "/device:GPU:%d" % gpu)
-              for gpu in range(num_gpus)
-          ]) for worker in self._workers
-      ]
-    else:
-      self._worker_devices = [
-          (worker, [device_util.canonicalize(worker, "/device:CPU:0")])
-          for worker in self._workers
-      ]
-
-    devices = nest.flatten([l for _, l in self._worker_devices])
-
-    # Setting `_default_device` will add a device scope in the
-    # distribution.scope. We set the default device to the first worker. When
-    # users specify device under distribution.scope by
-    #   with tf.device("/cpu:0"):
-    #     ...
-    # their ops will end up on the cpu device of its first worker, e.g.
-    # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode.
-    self._default_device = self._workers[0]
-
-    assert devices, "Must specify at least one device."
-    assert len(set(devices)) == len(devices), (
-        "No duplicates allowed in `devices` argument.")
-    # TODO(josh11b): Require at least 2 devices?
-    self._devices = [device_util.resolve(d) for d in devices]
-    self._canonical_device_set = set(self._devices)
-    self._device_index = values.PerReplica(
-        {d: i for i, d in enumerate(devices)})
-
-  def _create_variable(self, next_creator, *args, **kwargs):
-    """Create a mirrored variable. See `DistributionStrategy.scope`."""
-    colocate_with = kwargs.pop("colocate_with", None)
-    devices = self._get_devices_from(colocate_with)
-
-    def _real_mirrored_creator(devices, *args, **kwargs):  # pylint: disable=g-missing-docstring
-      index = {}
-      for i, d in enumerate(devices):
-        with ops.device(d):
-          if i > 0:
-            # Give replicas meaningful distinct names:
-            var0name = index[devices[0]].name.split(":")[0]
-            # We append a / to variable names created on replicas with id > 0 to
-            # ensure that we ignore the name scope and instead use the given
-            # name as the absolute name of the variable.
-            kwargs["name"] = "%s/replica_%d/" % (var0name, i)
-            # Initialize replicas with the same value:
-            def initial_value_fn(device=d):
-              if context.executing_eagerly():
-                init_value = index[devices[0]].value()
-                return array_ops.identity(init_value)
-              else:
-                with ops.device(device):
-                  init_value = index[devices[0]].initial_value
-                  return array_ops.identity(init_value)
-            kwargs["initial_value"] = initial_value_fn
-          with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
-            # Don't record operations (e.g. other variable reads) during
-            # variable creation.
-            with tape.stop_recording():
-              v = next_creator(*args, **kwargs)
-          assert not isinstance(v, values.DistributedVariable)
-          index[d] = v
-      return index
-
-    return _create_mirrored_variable(devices, _real_mirrored_creator, *args,
-                                     **kwargs)
-
-  def _distribute_dataset(self, dataset_fn):
-    if self._cluster_spec:
-      return values.MultiWorkerDataset(
-          partial(self._call_dataset_fn, dataset_fn),
-          self._worker_devices,
-          auto_shard=False)
-    else:
-      return values.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._devices)
-
-  def _make_dataset_iterator(self, dataset):
-    if self._cluster_spec:
-      worker_device_pairs = self._worker_devices
-    else:
-      worker_device_pairs = [("/job:localhost", self._devices)]
-    return values.DatasetIterator(dataset, worker_device_pairs,
-                                  self._num_replicas_in_sync)
-
-  def _make_input_fn_iterator(
-      self,
-      input_fn,
-      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    input_contexts = []
-    if self._cluster_spec:
-      num_workers = len(self._worker_devices)
-      worker_device_pairs = self._worker_devices
-    else:
-      num_workers = 1
-      worker_device_pairs = [("/job:localhost", self._devices)]
-    for i in range(num_workers):
-      input_contexts.append(distribute_lib.InputContext(
-          num_input_pipelines=num_workers,
-          input_pipeline_id=i,
-          num_replicas_in_sync=self._num_replicas_in_sync))
-    return values.InputFunctionIterator(
-        input_fn, worker_device_pairs, input_contexts)
-
-  # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
-  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
-                                          initial_loop_values=None):
-    if initial_loop_values is None:
-      initial_loop_values = {}
-    initial_loop_values = nest.flatten(initial_loop_values)
-
-    ctx = values.MultiStepContext()
-    def body(i, *args):
-      """A wrapper around `fn` to create the while loop body."""
-      del args
-      fn_inputs = iterator.get_next()
-      if not isinstance(fn_inputs, tuple):
-        fn_inputs = (fn_inputs,)
-      fn_result = fn(ctx, fn_inputs)
-      for (name, output) in ctx.last_step_outputs.items():
-        # Convert all outputs to tensors, potentially from `DistributedValues`.
-        ctx.last_step_outputs[name] = self._unwrap(output)
-      flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
-      with ops.control_dependencies([fn_result]):
-        return [i + 1] + flat_last_step_outputs
-
-    # We capture the control_flow_context at this point, before we run `fn`
-    # inside a while_loop. This is useful in cases where we might need to exit
-    # these contexts and get back to the outer context to do some things, for
-    # e.g. create an op which should be evaluated only once at the end of the
-    # loop on the host. One such usage is in creating metrics' value op.
-    self._outer_control_flow_context = (
-        ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
-
-    cond = lambda i, *args: i < iterations
-    i = constant_op.constant(0)
-    loop_result = control_flow_ops.while_loop(
-        cond, body, [i] + initial_loop_values, name="",
-        parallel_iterations=1, back_prop=False, swap_memory=False,
-        return_same_structure=True)
-    del self._outer_control_flow_context
-
-    ctx.run_op = control_flow_ops.group(loop_result)
-
-    # Convert the last_step_outputs from a list to the original dict structure
-    # of last_step_outputs.
-    last_step_tensor_outputs = loop_result[1:]
-    last_step_tensor_outputs_dict = nest.pack_sequence_as(
-        ctx.last_step_outputs, last_step_tensor_outputs)
-
-    for name, reduce_op in ctx._last_step_outputs_reduce_ops.items():  # pylint: disable=protected-access
-      output = last_step_tensor_outputs_dict[name]
-      # For outputs that have already been reduced, wrap them in a Mirrored
-      # container, else in a PerReplica container.
-      if reduce_op is None:
-        last_step_tensor_outputs_dict[name] = values.regroup(
-            {d: t for d, t in zip(self._devices, output)}, values.PerReplica)
-      else:
-        assert len(output) == 1
-        last_step_tensor_outputs_dict[name] = output[0]
-
-    ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
-    return ctx
-
-  def _broadcast_to(self, tensor, destinations):
-    # This is both a fast path for Python constants, and a way to delay
-    # converting Python values to a tensor until we know what type it
-    # should be converted to. Otherwise we have trouble with:
-    #   global_step.assign_add(1)
-    # since the `1` gets broadcast as an int32 but global_step is int64.
-    if isinstance(tensor, (float, int)):
-      return tensor
-    # TODO(josh11b): In eager mode, use one thread per device, or async mode.
-    return self._get_cross_device_ops().broadcast(
-        tensor, destinations or self._devices)
-
-  def _call_for_each_replica(self, fn, args, kwargs):
-    return _call_for_each_replica(self._container_strategy(), fn, args, kwargs)
-
-  def _configure(self,
-                 session_config=None,
-                 cluster_spec=None,
-                 task_type=None,
-                 task_id=None):
-    del task_type, task_id
-
-    if session_config:
-      session_config.isolate_session_state = True
-
-    if cluster_spec:
-      self._initialize_multi_worker(self._num_gpus, cluster_spec)
-
-    if self._cross_device_ops is None:
-      if self._cluster_spec:
-        # It currently cannot detect the toplogy of remote workers. So we
-        # hard-code the multi-worker all-reduce algorithm for now.
-        if len(self._workers) == 1:
-          # The default is "nccl".
-          self._cross_device_ops = (
-              cross_device_ops_lib.AllReduceCrossDeviceOps())
-        else:
-          # The default is hierarchical reduce and broadcast.
-          self._cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce(
-              self._workers, self._num_gpus)
-      else:
-        self._cross_device_ops = cross_device_ops_lib.choose_the_best(
-            self._devices, session_config=session_config)
-
-  def _get_cross_device_ops(self):
-    if self._cross_device_ops is None:
-      self._cross_device_ops = (
-          cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps())
-    return self._cross_device_ops
-
-  def _reduce_to(self, reduce_op, value, destinations):
-    assert not isinstance(value, values.Mirrored)
-    if not isinstance(value, values.DistributedValues):
-      # This function handles reducing values that are not PerReplica or
-      # Mirrored values. For example, the same value could be present on all
-      # replicas in which case `value` would be a single value or value could
-      # be 0.
-      return _reduce_non_distributed_value(self, reduce_op, value,
-                                           destinations)
-    return self._get_cross_device_ops().reduce(
-        reduce_op, value, destinations=destinations)
-
-  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    return self._get_cross_device_ops().batch_reduce(reduce_op,
-                                                     value_destination_pairs)
-
-  def _update(self, var, fn, args, kwargs, group):
-    # TODO(josh11b): In eager mode, use one thread per device.
-    assert isinstance(var, values.DistributedVariable)
-    updates = {}
-    for d, v in var._index.items():  # pylint: disable=protected-access
-      name = "update_%d" % self._device_index.get(d)
-      with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
-        # If args and kwargs are not mirrored, the value is returned as is.
-        updates[d] = fn(v,
-                        *values.select_device_mirrored(d, args),
-                        **values.select_device_mirrored(d, kwargs))
-    return values.update_regroup(self, updates, group)
-
-  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
-    assert isinstance(colocate_with, list)
-    # TODO(josh11b): In eager mode, use one thread per device.
-    updates = {}
-    for d in colocate_with:
-      name = "update_%d" % self._device_index.get(d)
-      with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
-        updates[d] = fn(*values.select_device_mirrored(d, args),
-                        **values.select_device_mirrored(d, kwargs))
-    return values.update_regroup(self, updates, group)
-
-  def read_var(self, replica_local_var):
-    """Read the aggregate value of a replica-local variable."""
-    if isinstance(replica_local_var, values.ReplicaLocalVariable):
-      return replica_local_var._get_cross_replica()  # pylint: disable=protected-access
-    assert isinstance(replica_local_var, values.Mirrored)
-    return array_ops.identity(replica_local_var.get())
-
-  def _unwrap(self, val):
-    if isinstance(val, values.DistributedValues):
-      # Return in a deterministic order.
-      if set(val.devices) == self._canonical_device_set:
-        return [val.get(device=d) for d in self._devices]
-      return [val.get(device=d) for d in sorted(val.devices)]
-    return [val]
-
-  def value_container(self, val):
-    return values.value_container(val)
-
-  @property
-  def _num_replicas_in_sync(self):
-    return len(self._devices)
-
-  @property
-  def worker_devices(self):
-    # Make a copy to prevent users from accidentally mutating our copy.
-    return list(self._devices)
-
-  @property
-  def parameter_devices(self):
-    return list(self._devices)
-
-  @property
-  def experimental_between_graph(self):
-    return False
-
-  @property
-  def experimental_should_init(self):
-    return True
-
-  @property
-  def should_checkpoint(self):
-    return True
-
-  @property
-  def should_save_summary(self):
-    return True
-
-  def non_slot_devices(self, var_list):
-    del var_list
-    return list(self._devices)
-
-  def _get_devices_from(self, colocate_with=None):
-    if colocate_with is None:
-      return self._devices
-    else:
-      return cross_device_ops_lib.get_devices_from(colocate_with)
-
-  class _MirroredReplicaThread(threading.Thread):
-    """A thread that runs() a function on a device."""
-
-    def __init__(self, dist, coord, device, variable_creator_fn, fn, *args,
-                 **kwargs):
-      super(CoreMirroredExtended._MirroredReplicaThread, self).__init__()  # pylint: disable=protected-access
-      self.coord = coord
-      self.distribution = dist
-      self.device = device
-      self.replica_id = dist.extended.worker_devices.index(device)
-      self.variable_creator_fn = variable_creator_fn
-      # State needed to run and return the results of `fn`.
-      self.main_fn = fn
-      self.main_args = args
-      self.main_kwargs = kwargs
-      self.main_result = None
-      self.done = False
-      # State needed to run the next merge_call() (if any) requested via
-      # ReplicaContext.
-      self.merge_fn = None
-      self.merge_args = None
-      self.merge_kwargs = None
-      self.merge_result = None
-      self.captured_name_scope = None
-      # We use a thread.Event for the main thread to signal when this
-      # thread should start running (`should_run`), and another for
-      # this thread to transfer control back to the main thread
-      # (`has_paused`, either when it gets to a
-      # `get_replica_context().merge_call` or when `fn` returns). In
-      # either case the event starts cleared, is signaled by calling
-      # set(). The receiving thread waits for the signal by calling
-      # wait() and then immediately clearing the event using clear().
-      self.should_run = threading.Event()
-      self.has_paused = threading.Event()
-      # These fields have to do with inheriting various contexts from the
-      # parent thread:
-      # pylint: disable=protected-access
-      self.context_mode = context.context()._eager_context.mode
-      if not context.context()._context_handle:
-        context.context()._initialize_handle_and_devices()
-      self.context_device_policy = (
-          pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy(
-              context.context()._context_handle))
-      self.graph = ops.get_default_graph()
-      self._variable_creator_stack = self.graph._variable_creator_stack[:]
-      self._captured_var_scope = variable_scope.get_variable_scope()
-      # Adding a "/" at end lets us re-enter this scope later.
-      self._name_scope = self.graph.get_name_scope()
-      if self._name_scope:
-        self._name_scope += "/"
-      if self.replica_id > 0:
-        if not self._name_scope:
-          self._name_scope = ""
-        self._name_scope += "replica_%d/" % self.replica_id
-
-    def run(self):
-      # pylint: disable=protected-access
-      self.graph._variable_creator_stack = self._variable_creator_stack
-      self.should_run.wait()
-      self.should_run.clear()
-      try:
-        if self.coord.should_stop():
-          return
-        with self.coord.stop_on_exception(), \
-            context.context()._mode(self.context_mode), \
-            context.context().device_policy(self.context_device_policy), \
-            _enter_graph(self.graph), \
-            MirroredReplicaContext(self.distribution, constant_op.constant(
-                self.replica_id, dtypes.int32)), \
-            ops.device(self.device), \
-            ops.name_scope(self._name_scope), \
-            variable_scope.variable_scope(
-                self._captured_var_scope, reuse=self.replica_id > 0), \
-            variable_scope.variable_creator_scope(self.variable_creator_fn):
-          self.main_result = self.main_fn(*self.main_args, **self.main_kwargs)
-          self.done = True
-      finally:
-        self.has_paused.set()
+# pylint: disable=protected-access,invalid-name
+_call_for_each_replica = mirrored_strategy._call_for_each_replica
+_reduce_non_distributed_value = mirrored_strategy._reduce_non_distributed_value
+_create_mirrored_variable = mirrored_strategy._create_mirrored_variable
+CoreMirroredStrategy = mirrored_strategy.MirroredStrategy
+CoreMirroredExtended = mirrored_strategy.MirroredExtended
+# pylint: enable=protected-access,invalid-name
 
 
 class MirroredStrategy(distribute_lib.DistributionStrategy):
@@ -873,43 +139,14 @@ class MirroredExtended(CoreMirroredExtended):
   def _distribute_dataset(self, dataset_fn):
     if self._cluster_spec:
       return values.MultiWorkerDataset(
-          partial(self._call_dataset_fn, dataset_fn),
+          functools.partial(self._call_dataset_fn, dataset_fn),
           self._worker_devices,
           auto_shard=self._auto_shard_dataset)
     else:
       return values.PerReplicaDataset(
           self._call_dataset_fn(dataset_fn), self._devices)
 
-
-class MirroredReplicaContext(distribute_lib.ReplicaContext):
-  """ReplicaContext used in MirroredStrategy.call_for_each_replica().
-
-  Opened in `_MirroredReplicaThread`, to allow the user to invoke
-  `MirroredStrategy`'s specific implementation of `merge_call()`,
-  which works by delegating the function and its arguments to
-  the main thread (the one that invoked
-  `MirroredStrategy.call_for_each_replica()`).
-  """
-
-  def _merge_call(self, fn, args, kwargs):
-    """Delegate to the main thread to actually perform merge_call()."""
-    t = threading.current_thread()  # a _MirroredReplicaThread
-    t.merge_fn = fn
-    t.merge_args = args
-    t.merge_kwargs = kwargs
-    t.captured_name_scope = t.graph.get_name_scope()
-    # Adding a "/" at end lets us re-enter this scope later.
-    if t.captured_name_scope:
-      t.captured_name_scope += "/"
-    t.has_paused.set()
-    t.should_run.wait()
-    t.should_run.clear()
-    if t.coord.should_stop():
-      raise _RequestedStop()
-    return t.merge_result
-
+  # TODO(priyag): Delete this once all strategies use global batch size.
   @property
-  def devices(self):
-    distribute_lib.require_replica_context(self)
-    replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
-    return [self._distribution_strategy.extended.worker_devices[replica_id]]
+  def _global_batch_size(self):
+    return False
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index f881c234ac..421507232a 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -194,6 +194,11 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
   def should_save_summary(self):
     return True
 
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
+
 
 class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
   """ReplicaContext for OneDeviceStrategy."""
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 6fc81a1e57..fc2d2b20c9 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -510,3 +510,8 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
   @property
   def should_save_summary(self):
     return self._is_chief
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return False
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 478e3accd7..f1115cb0c0 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -544,6 +544,11 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       if cluster_spec:
         session_config.cluster_def.CopyFrom(cluster_spec.as_cluster_def())
 
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
+
 
 class _TPUReplicaContext(distribute_lib.ReplicaContext):
   """Replication Context class for TPU Strategy."""
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 83c3901441..999543d71f 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -139,6 +139,35 @@ py_library(
     deps = [],
 )
 
+py_library(
+    name = "mirrored_strategy",
+    srcs = ["mirrored_strategy.py"],
+    deps = [
+        ":cross_device_ops",
+        ":multi_worker_util",
+        ":reduce_util",
+        ":shared_variable_creator",
+        ":values",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:device",
+        "//tensorflow/python:device_util",
+        "//tensorflow/python:distribute",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:pywrap_tensorflow",
+        "//tensorflow/python:tensor_util",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:tape",
+    ],
+)
+
 py_library(
     name = "multi_worker_util",
     srcs = [
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
new file mode 100644
index 0000000000..7ed096b863
--- /dev/null
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -0,0 +1,805 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class MirroredStrategy implementing DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import contextlib
+import functools
+import threading
+
+from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import reduce_util
+from tensorflow.python.distribute import shared_variable_creator
+from tensorflow.python.distribute import values
+from tensorflow.python.eager import context
+from tensorflow.python.eager import tape
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import coordinator
+from tensorflow.python.training import device_util
+from tensorflow.python.training import distribute as distribute_lib
+from tensorflow.python.util import nest
+
+
+# TODO(josh11b): Replace asserts in this file with if ...: raise ...
+
+
+@contextlib.contextmanager
+def _enter_graph(g):
+  if context.executing_eagerly():
+    with g.as_default(), context.eager_mode():
+      yield
+  else:
+    with g.as_default():
+      yield
+
+
+def _cpu_device(device):
+  cpu_device = tf_device.DeviceSpec.from_string(device)
+  cpu_device.merge_from(tf_device.DeviceSpec(device_type="CPU", device_index=0))
+  return cpu_device.to_string()
+
+
+class _RequestedStop(Exception):  # pylint: disable=g-bad-exception-name
+  pass
+
+
+# _call_for_each_replica and _reduce_non_distributed_value are not members of
+# MirroredStrategy so that they are generally not allowed to use anything
+# specific to MirroredStrategy and thus can be shared with other distribution
+# strategies.
+
+
+# TODO(yuefengz): maybe create a common class for those who need to call this
+# _call_for_each_replica.
+def _call_for_each_replica(distribution, fn, args, kwargs):
+  """Run `fn` in separate threads, once per replica/worker device.
+
+  Args:
+    distribution: the DistributionStrategy object.
+    fn: function to run (will be run once per device, each in its own thread).
+    args: positional arguments for `fn`
+    kwargs: keyword arguments for `fn`.
+
+  Returns:
+    Merged return value of `fn` across all replicas.
+
+  Raises:
+    RuntimeError: If fn() calls get_replica_context().merge_call() a different
+        number of times from the available devices.
+  """
+  # TODO(josh11b): Add this option once we add synchronization to variable
+  # creation. Until then, this is pretty unsafe to use.
+  run_concurrently = False
+  if not context.executing_eagerly():
+    # Needed for per-thread device, etc. contexts in graph mode.
+    ops.get_default_graph().switch_to_thread_local()
+
+  coord = coordinator.Coordinator(clean_stop_exception_types=(_RequestedStop,))
+
+  shared_variable_store = {}
+
+  # TODO(isaprykin): Create these threads once instead of during every run()
+  # call.
+  threads = []
+  for index, d in enumerate(distribution.extended.worker_devices):
+    variable_creator_fn = shared_variable_creator.make_fn(
+        shared_variable_store, index)
+    t = MirroredExtended._MirroredReplicaThread(  # pylint: disable=protected-access
+        distribution, coord, d, variable_creator_fn, fn,
+        *values.select_device(d, args), **values.select_device(d, kwargs))
+    threads.append(t)
+
+  for t in threads:
+    t.start()
+
+  # When `fn` starts `should_run` event is set on _MirroredReplicaThread
+  # (`MRT`) threads. The execution waits until
+  # `MRT.has_paused` is set, which indicates that either `fn` is
+  # complete or a `get_replica_context().merge_call()` is called.  If `fn` is
+  # complete, then `MRT.done` is set to True.  Otherwise, arguments
+  # of `get_replica_context().merge_call` from all paused threads are grouped
+  # and the `merge_fn` is performed.  Results of the
+  # `get_replica_context().merge_call` are then set to `MRT.merge_result`.
+  # Each such `get_replica_context().merge_call` call returns the
+  # `MRT.merge_result` for that thread when `MRT.should_run` event
+  # is reset again. Execution of `fn` resumes.
+
+  try:
+    with coord.stop_on_exception():
+      all_done = False
+      while not all_done and not coord.should_stop():
+        done = []
+        if run_concurrently:
+          for t in threads:
+            t.should_run.set()
+          for t in threads:
+            t.has_paused.wait()
+            t.has_paused.clear()
+            if coord.should_stop():
+              return None
+            done.append(t.done)
+        else:
+          for t in threads:
+            t.should_run.set()
+            t.has_paused.wait()
+            t.has_paused.clear()
+            if coord.should_stop():
+              return None
+            done.append(t.done)
+        if coord.should_stop():
+          return None
+        all_done = all(done)
+        if not all_done:
+          if any(done):
+            raise RuntimeError("Some replicas made a different number of "
+                               "replica_context().merge_call() calls.")
+          # get_replica_context().merge_call() case
+          merge_args = values.regroup({t.device: t.merge_args for t in threads})
+          merge_kwargs = values.regroup(
+              {t.device: t.merge_kwargs for t in threads})
+          # We capture the name_scope of the MRT when we call merge_fn
+          # to ensure that if we have opened a name scope in the MRT,
+          # it will be respected when executing the merge function. We only
+          # capture the name_scope from the first MRT and assume it is
+          # the same for all other MRTs.
+          mtt_captured_name_scope = threads[0].captured_name_scope
+          with ops.name_scope(mtt_captured_name_scope):
+            merge_result = threads[0].merge_fn(distribution, *merge_args,
+                                               **merge_kwargs)
+          for t in threads:
+            t.merge_result = values.select_device(t.device, merge_result)
+  finally:
+    for t in threads:
+      t.should_run.set()
+    coord.join(threads)
+
+  return values.regroup({t.device: t.main_result for t in threads})
+
+
+def _reduce_non_distributed_value(extended, reduce_op, value, destinations):
+  """Reduce a non-DistributedValue `value` to `destinations`."""
+  if isinstance(value, values.DistributedValues):
+    raise ValueError("You are passing a `DistributedValue` to "
+                     "`_reduce_non_distributed_value`, which is not allowed.")
+
+  # If the same value is present on all replicas then the PerReplica value will
+  # be a single value. We also handle the case when `value` is a single value
+  # and equal to 0.
+  if value == 0:
+    return 0
+  # If there is only a single value and the reduce op is MEAN,
+  # that value should be on all destinations.
+  if reduce_op == reduce_util.ReduceOp.MEAN:
+    return value
+
+  cross_device_ops_lib.validate_destinations(destinations)
+  # We do not support a reduce op of SUM if the value is the same across
+  # all replicas. We call this as part of assign functions for MirroredVariables
+  # and summing up identical values across replicas is not clearly defined.
+  if (len(extended.worker_devices) != 1 or
+      not cross_device_ops_lib.check_destinations(destinations)):
+    raise ValueError("A non-DistributedValues value %s cannot be reduced with "
+                     "the given reduce op %s." % (value, reduce_op))
+  # TODO(anjalisridhar): Moves these methods to a device utility file?
+  devices = cross_device_ops_lib.get_devices_from(destinations)
+  if len(devices) == 1:
+    with ops.device(devices[0]):
+      return array_ops.identity(value)
+  else:
+    value_updates = {}
+    for d in devices:
+      with ops.device(d):
+        value_updates[d] = array_ops.identity(value)
+    return values.Mirrored(value_updates)
+
+
+def _create_mirrored_variable(devices, real_mirrored_creator, *args, **kwargs):  # pylint: disable=g-missing-docstring
+  # Figure out what collections this variable should be added to.
+  # We'll add the MirroredVariable to those collections instead.
+  collections = kwargs.pop("collections", None)
+  if collections is None:
+    collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+  kwargs["collections"] = []
+
+  # Get synchronization value
+  synchronization = kwargs.get("synchronization",
+                               variable_scope.VariableSynchronization.ON_WRITE)
+  if synchronization == variable_scope.VariableSynchronization.NONE:
+    raise ValueError("`NONE` variable synchronization mode is not "
+                     "supported with `Mirrored` distribution strategy. Please"
+                     " change the `synchronization` for variable: " +
+                     kwargs["name"])
+  elif synchronization == variable_scope.VariableSynchronization.ON_READ:
+    # Variables that are to be synced on read are replica local.
+    is_replica_local = True
+    kwargs["trainable"] = False
+  elif (synchronization == variable_scope.VariableSynchronization.ON_WRITE or
+        synchronization == variable_scope.VariableSynchronization.AUTO):
+    # `AUTO` synchronization for `MirroredStrategy` is `ON_WRITE`.
+    is_replica_local = False
+  else:
+    raise ValueError("Invalid variable synchronization mode: " +
+                     synchronization + " for variable: " + kwargs["name"])
+
+  # Get aggregation value
+  aggregation = kwargs.pop("aggregation",
+                           variable_scope.VariableAggregation.NONE)
+  if aggregation not in (
+      variable_scope.VariableAggregation.NONE,
+      variable_scope.VariableAggregation.SUM,
+      variable_scope.VariableAggregation.MEAN,
+      variable_scope.VariableAggregation.ONLY_FIRST_REPLICA
+  ):
+    raise ValueError("Invalid variable aggregation mode: " + aggregation +
+                     " for variable: " + kwargs["name"])
+
+  # Ignore user-specified caching device, not needed for mirrored variables.
+  kwargs.pop("caching_device", None)
+
+  # TODO(josh11b,apassos): It would be better if variable initialization
+  # was never recorded on the tape instead of having to do this manually
+  # here.
+  with tape.stop_recording():
+    index = real_mirrored_creator(devices, *args, **kwargs)
+
+    if is_replica_local:
+      result = values.ReplicaLocalVariable(
+          index, index[devices[0]], aggregation)
+    else:
+      result = values.MirroredVariable(index, index[devices[0]], aggregation)
+
+  # Add the wrapped variable to the requested collections.
+  # The handling of eager mode and the global step matches
+  # ResourceVariable._init_from_args().
+  if not context.executing_eagerly():
+    g = ops.get_default_graph()
+    # If "trainable" is True, next_creator() will add the member variables
+    # to the TRAINABLE_VARIABLES collection, so we manually remove
+    # them and replace with the MirroredVariable. We can't set
+    # "trainable" to False for next_creator() since that causes functions
+    # like implicit_gradients to skip those variables.
+    if kwargs.get("trainable", True):
+      collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
+      l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
+      for v in index.values():
+        if v in l:
+          l.remove(v)
+    g.add_to_collections(collections, result)
+  elif ops.GraphKeys.GLOBAL_STEP in collections:
+    ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, result)
+
+  return result
+
+
+class MirroredStrategy(distribute_lib.DistributionStrategy):
+  """Mirrors vars to distribute across multiple devices and machines.
+
+  This strategy uses one replica per device and sync replication for its
+  multi-GPU version.
+
+  The multi-worker version will be added in the fture.
+
+  Args:
+    devices: a list of device strings.
+    num_gpus_per_worker: number of GPUs per worker.
+    cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not
+      set, nccl will be use by default.
+  """
+
+  def __init__(self,
+               devices=None,
+               num_gpus_per_worker=None,
+               cross_device_ops=None):
+    extended = MirroredExtended(self, devices, num_gpus_per_worker,
+                                cross_device_ops)
+    super(MirroredStrategy, self).__init__(extended)
+
+
+class MirroredExtended(distribute_lib.DistributionStrategyExtended):
+  """Implementation of MirroredStrategy."""
+
+  def __init__(self,
+               container_strategy,
+               devices=None,
+               num_gpus_per_worker=None,
+               cross_device_ops=None):
+    super(MirroredExtended, self).__init__(container_strategy)
+    self._cross_device_ops = cross_device_ops
+    # Remember num GPUs which might be needed by `configure` method.
+    self._num_gpus = num_gpus_per_worker
+
+    self._initialize_local(self._num_gpus, devices)
+
+  def _initialize_local(self, num_gpus, devices):
+    """Initializes the object for local training."""
+    self._cluster_spec = None
+    # Convert `num_gpus` into `devices`, shouldn't specify both.
+    if devices is None:
+      if num_gpus is None:
+        num_gpus = context.num_gpus()
+      if num_gpus == 0:
+        devices = ["/device:CPU:0"]
+      else:
+        devices = ["/device:GPU:%d" % d for d in range(num_gpus)]
+    elif num_gpus is not None:
+      raise ValueError("Must only specify one of `devices` and `num_gpus`.")
+    self._num_gpus = num_gpus
+    # TODO(yuefengz): consider setting the default device.
+
+    assert devices, "Must specify at least one device."
+    assert len(set(devices)) == len(devices), (
+        "No duplicates allowed in `devices` argument.")
+    # TODO(josh11b): Require at least 2 devices?
+    self._devices = [device_util.resolve(d) for d in devices]
+    self._canonical_device_set = set(self._devices)
+    self._device_index = values.PerReplica(
+        {d: i for i, d in enumerate(devices)})
+
+  def _initialize_multi_worker(self, num_gpus, cluster_spec):
+    """Initializes the object for multi-worker training."""
+    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
+    self._cluster_spec = cluster_spec
+
+    self._workers = []
+    for job in ["chief", "worker"]:
+      for task in range(len(cluster_spec.as_dict().get(job, []))):
+        self._workers.append("/job:%s/task:%d" % (job, task))
+
+    if num_gpus is None:
+      raise ValueError("`num_gpus` is required if `cluster_spec` is given.")
+    if num_gpus > 0:
+      self._worker_devices = [
+          (worker, [
+              device_util.canonicalize(worker + "/device:GPU:%d" % gpu)
+              for gpu in range(num_gpus)
+          ]) for worker in self._workers
+      ]
+    else:
+      self._worker_devices = [
+          (worker, [device_util.canonicalize(worker, "/device:CPU:0")])
+          for worker in self._workers
+      ]
+
+    devices = nest.flatten([l for _, l in self._worker_devices])
+
+    # Setting `_default_device` will add a device scope in the
+    # distribution.scope. We set the default device to the first worker. When
+    # users specify device under distribution.scope by
+    #   with tf.device("/cpu:0"):
+    #     ...
+    # their ops will end up on the cpu device of its first worker, e.g.
+    # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode.
+    self._default_device = self._workers[0]
+
+    assert devices, "Must specify at least one device."
+    assert len(set(devices)) == len(devices), (
+        "No duplicates allowed in `devices` argument.")
+    # TODO(josh11b): Require at least 2 devices?
+    self._devices = [device_util.resolve(d) for d in devices]
+    self._canonical_device_set = set(self._devices)
+    self._device_index = values.PerReplica(
+        {d: i for i, d in enumerate(devices)})
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    """Create a mirrored variable. See `DistributionStrategy.scope`."""
+    colocate_with = kwargs.pop("colocate_with", None)
+    devices = self._get_devices_from(colocate_with)
+
+    def _real_mirrored_creator(devices, *args, **kwargs):  # pylint: disable=g-missing-docstring
+      index = {}
+      for i, d in enumerate(devices):
+        with ops.device(d):
+          if i > 0:
+            # Give replicas meaningful distinct names:
+            var0name = index[devices[0]].name.split(":")[0]
+            # We append a / to variable names created on replicas with id > 0 to
+            # ensure that we ignore the name scope and instead use the given
+            # name as the absolute name of the variable.
+            kwargs["name"] = "%s/replica_%d/" % (var0name, i)
+            # Initialize replicas with the same value:
+            def initial_value_fn(device=d):
+              if context.executing_eagerly():
+                init_value = index[devices[0]].value()
+                return array_ops.identity(init_value)
+              else:
+                with ops.device(device):
+                  init_value = index[devices[0]].initial_value
+                  return array_ops.identity(init_value)
+            kwargs["initial_value"] = initial_value_fn
+          with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+            # Don't record operations (e.g. other variable reads) during
+            # variable creation.
+            with tape.stop_recording():
+              v = next_creator(*args, **kwargs)
+          assert not isinstance(v, values.DistributedVariable)
+          index[d] = v
+      return index
+
+    return _create_mirrored_variable(devices, _real_mirrored_creator, *args,
+                                     **kwargs)
+
+  def _distribute_dataset(self, dataset_fn):
+    if self._cluster_spec:
+      return values.MultiWorkerDataset(
+          functools.partial(self._call_dataset_fn, dataset_fn),
+          self._worker_devices,
+          auto_shard=False)
+    else:
+      return values.PerReplicaDataset(
+          self._call_dataset_fn(dataset_fn), self._devices)
+
+  def _make_dataset_iterator(self, dataset):
+    if self._cluster_spec:
+      worker_device_pairs = self._worker_devices
+    else:
+      worker_device_pairs = [("/job:localhost", self._devices)]
+    return values.DatasetIterator(dataset, worker_device_pairs,
+                                  self._num_replicas_in_sync)
+
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    input_contexts = []
+    if self._cluster_spec:
+      num_workers = len(self._worker_devices)
+      worker_device_pairs = self._worker_devices
+    else:
+      num_workers = 1
+      worker_device_pairs = [("/job:localhost", self._devices)]
+    for i in range(num_workers):
+      input_contexts.append(distribute_lib.InputContext(
+          num_input_pipelines=num_workers,
+          input_pipeline_id=i,
+          num_replicas_in_sync=self._num_replicas_in_sync))
+    return values.InputFunctionIterator(
+        input_fn, worker_device_pairs, input_contexts)
+
+  # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
+  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
+                                          initial_loop_values=None):
+    if initial_loop_values is None:
+      initial_loop_values = {}
+    initial_loop_values = nest.flatten(initial_loop_values)
+
+    ctx = values.MultiStepContext()
+    def body(i, *args):
+      """A wrapper around `fn` to create the while loop body."""
+      del args
+      fn_inputs = iterator.get_next()
+      if not isinstance(fn_inputs, tuple):
+        fn_inputs = (fn_inputs,)
+      fn_result = fn(ctx, fn_inputs)
+      for (name, output) in ctx.last_step_outputs.items():
+        # Convert all outputs to tensors, potentially from `DistributedValues`.
+        ctx.last_step_outputs[name] = self._unwrap(output)
+      flat_last_step_outputs = nest.flatten(ctx.last_step_outputs)
+      with ops.control_dependencies([fn_result]):
+        return [i + 1] + flat_last_step_outputs
+
+    # We capture the control_flow_context at this point, before we run `fn`
+    # inside a while_loop. This is useful in cases where we might need to exit
+    # these contexts and get back to the outer context to do some things, for
+    # e.g. create an op which should be evaluated only once at the end of the
+    # loop on the host. One such usage is in creating metrics' value op.
+    self._outer_control_flow_context = (
+        ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
+
+    cond = lambda i, *args: i < iterations
+    i = constant_op.constant(0)
+    loop_result = control_flow_ops.while_loop(
+        cond, body, [i] + initial_loop_values, name="",
+        parallel_iterations=1, back_prop=False, swap_memory=False,
+        return_same_structure=True)
+    del self._outer_control_flow_context
+
+    ctx.run_op = control_flow_ops.group(loop_result)
+
+    # Convert the last_step_outputs from a list to the original dict structure
+    # of last_step_outputs.
+    last_step_tensor_outputs = loop_result[1:]
+    last_step_tensor_outputs_dict = nest.pack_sequence_as(
+        ctx.last_step_outputs, last_step_tensor_outputs)
+
+    for name, reduce_op in ctx._last_step_outputs_reduce_ops.items():  # pylint: disable=protected-access
+      output = last_step_tensor_outputs_dict[name]
+      # For outputs that have already been reduced, wrap them in a Mirrored
+      # container, else in a PerReplica container.
+      if reduce_op is None:
+        last_step_tensor_outputs_dict[name] = values.regroup(
+            {d: t for d, t in zip(self._devices, output)}, values.PerReplica)
+      else:
+        assert len(output) == 1
+        last_step_tensor_outputs_dict[name] = output[0]
+
+    ctx._set_last_step_outputs(last_step_tensor_outputs_dict)  # pylint: disable=protected-access
+    return ctx
+
+  def _broadcast_to(self, tensor, destinations):
+    # This is both a fast path for Python constants, and a way to delay
+    # converting Python values to a tensor until we know what type it
+    # should be converted to. Otherwise we have trouble with:
+    #   global_step.assign_add(1)
+    # since the `1` gets broadcast as an int32 but global_step is int64.
+    if isinstance(tensor, (float, int)):
+      return tensor
+    # TODO(josh11b): In eager mode, use one thread per device, or async mode.
+    return self._get_cross_device_ops().broadcast(
+        tensor, destinations or self._devices)
+
+  def _call_for_each_replica(self, fn, args, kwargs):
+    return _call_for_each_replica(self._container_strategy(), fn, args, kwargs)
+
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
+    del task_type, task_id
+
+    if session_config:
+      session_config.isolate_session_state = True
+
+    if cluster_spec:
+      self._initialize_multi_worker(self._num_gpus, cluster_spec)
+
+    if self._cross_device_ops is None:
+      if self._cluster_spec:
+        # It currently cannot detect the toplogy of remote workers. So we
+        # hard-code the multi-worker all-reduce algorithm for now.
+        if len(self._workers) == 1:
+          # The default is "nccl".
+          self._cross_device_ops = (
+              cross_device_ops_lib.AllReduceCrossDeviceOps())
+        else:
+          # The default is hierarchical reduce and broadcast.
+          self._cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce(
+              self._workers, self._num_gpus)
+      else:
+        self._cross_device_ops = cross_device_ops_lib.choose_the_best(
+            self._devices, session_config=session_config)
+
+  def _get_cross_device_ops(self):
+    if self._cross_device_ops is None:
+      self._cross_device_ops = (
+          cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps())
+    return self._cross_device_ops
+
+  def _reduce_to(self, reduce_op, value, destinations):
+    assert not isinstance(value, values.Mirrored)
+    if not isinstance(value, values.DistributedValues):
+      # This function handles reducing values that are not PerReplica or
+      # Mirrored values. For example, the same value could be present on all
+      # replicas in which case `value` would be a single value or value could
+      # be 0.
+      return _reduce_non_distributed_value(self, reduce_op, value,
+                                           destinations)
+    return self._get_cross_device_ops().reduce(
+        reduce_op, value, destinations=destinations)
+
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
+    return self._get_cross_device_ops().batch_reduce(reduce_op,
+                                                     value_destination_pairs)
+
+  def _update(self, var, fn, args, kwargs, group):
+    # TODO(josh11b): In eager mode, use one thread per device.
+    assert isinstance(var, values.DistributedVariable)
+    updates = {}
+    for d, v in var._index.items():  # pylint: disable=protected-access
+      name = "update_%d" % self._device_index.get(d)
+      with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
+        # If args and kwargs are not mirrored, the value is returned as is.
+        updates[d] = fn(v,
+                        *values.select_device_mirrored(d, args),
+                        **values.select_device_mirrored(d, kwargs))
+    return values.update_regroup(self, updates, group)
+
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
+    assert isinstance(colocate_with, list)
+    # TODO(josh11b): In eager mode, use one thread per device.
+    updates = {}
+    for d in colocate_with:
+      name = "update_%d" % self._device_index.get(d)
+      with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
+        updates[d] = fn(*values.select_device_mirrored(d, args),
+                        **values.select_device_mirrored(d, kwargs))
+    return values.update_regroup(self, updates, group)
+
+  def read_var(self, replica_local_var):
+    """Read the aggregate value of a replica-local variable."""
+    if isinstance(replica_local_var, values.ReplicaLocalVariable):
+      return replica_local_var._get_cross_replica()  # pylint: disable=protected-access
+    assert isinstance(replica_local_var, values.Mirrored)
+    return array_ops.identity(replica_local_var.get())
+
+  def _unwrap(self, val):
+    if isinstance(val, values.DistributedValues):
+      # Return in a deterministic order.
+      if set(val.devices) == self._canonical_device_set:
+        return [val.get(device=d) for d in self._devices]
+      return [val.get(device=d) for d in sorted(val.devices)]
+    return [val]
+
+  def value_container(self, val):
+    return values.value_container(val)
+
+  @property
+  def _num_replicas_in_sync(self):
+    return len(self._devices)
+
+  @property
+  def worker_devices(self):
+    # Make a copy to prevent users from accidentally mutating our copy.
+    return list(self._devices)
+
+  @property
+  def parameter_devices(self):
+    return list(self._devices)
+
+  @property
+  def experimental_between_graph(self):
+    return False
+
+  @property
+  def experimental_should_init(self):
+    return True
+
+  @property
+  def should_checkpoint(self):
+    return True
+
+  @property
+  def should_save_summary(self):
+    return True
+
+  def non_slot_devices(self, var_list):
+    del var_list
+    return list(self._devices)
+
+  def _get_devices_from(self, colocate_with=None):
+    if colocate_with is None:
+      return self._devices
+    else:
+      return cross_device_ops_lib.get_devices_from(colocate_with)
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
+
+  class _MirroredReplicaThread(threading.Thread):
+    """A thread that runs() a function on a device."""
+
+    def __init__(self, dist, coord, device, variable_creator_fn, fn, *args,
+                 **kwargs):
+      super(MirroredExtended._MirroredReplicaThread, self).__init__()  # pylint: disable=protected-access
+      self.coord = coord
+      self.distribution = dist
+      self.device = device
+      self.replica_id = dist.extended.worker_devices.index(device)
+      self.variable_creator_fn = variable_creator_fn
+      # State needed to run and return the results of `fn`.
+      self.main_fn = fn
+      self.main_args = args
+      self.main_kwargs = kwargs
+      self.main_result = None
+      self.done = False
+      # State needed to run the next merge_call() (if any) requested via
+      # ReplicaContext.
+      self.merge_fn = None
+      self.merge_args = None
+      self.merge_kwargs = None
+      self.merge_result = None
+      self.captured_name_scope = None
+      # We use a thread.Event for the main thread to signal when this
+      # thread should start running (`should_run`), and another for
+      # this thread to transfer control back to the main thread
+      # (`has_paused`, either when it gets to a
+      # `get_replica_context().merge_call` or when `fn` returns). In
+      # either case the event starts cleared, is signaled by calling
+      # set(). The receiving thread waits for the signal by calling
+      # wait() and then immediately clearing the event using clear().
+      self.should_run = threading.Event()
+      self.has_paused = threading.Event()
+      # These fields have to do with inheriting various contexts from the
+      # parent thread:
+      # pylint: disable=protected-access
+      self.context_mode = context.context()._eager_context.mode
+      if not context.context()._context_handle:
+        context.context()._initialize_handle_and_devices()
+      self.context_device_policy = (
+          pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy(
+              context.context()._context_handle))
+      self.graph = ops.get_default_graph()
+      self._variable_creator_stack = self.graph._variable_creator_stack[:]
+      self._captured_var_scope = variable_scope.get_variable_scope()
+      # Adding a "/" at end lets us re-enter this scope later.
+      self._name_scope = self.graph.get_name_scope()
+      if self._name_scope:
+        self._name_scope += "/"
+      if self.replica_id > 0:
+        if not self._name_scope:
+          self._name_scope = ""
+        self._name_scope += "replica_%d/" % self.replica_id
+
+    def run(self):
+      # pylint: disable=protected-access
+      self.graph._variable_creator_stack = self._variable_creator_stack
+      self.should_run.wait()
+      self.should_run.clear()
+      try:
+        if self.coord.should_stop():
+          return
+        with self.coord.stop_on_exception(), \
+            context.context()._mode(self.context_mode), \
+            context.context().device_policy(self.context_device_policy), \
+            _enter_graph(self.graph), \
+            MirroredReplicaContext(self.distribution, constant_op.constant(
+                self.replica_id, dtypes.int32)), \
+            ops.device(self.device), \
+            ops.name_scope(self._name_scope), \
+            variable_scope.variable_scope(
+                self._captured_var_scope, reuse=self.replica_id > 0), \
+            variable_scope.variable_creator_scope(self.variable_creator_fn):
+          self.main_result = self.main_fn(*self.main_args, **self.main_kwargs)
+          self.done = True
+      finally:
+        self.has_paused.set()
+
+
+class MirroredReplicaContext(distribute_lib.ReplicaContext):
+  """ReplicaContext used in MirroredStrategy.call_for_each_replica().
+
+  Opened in `_MirroredReplicaThread`, to allow the user to invoke
+  `MirroredStrategy`'s specific implementation of `merge_call()`,
+  which works by delegating the function and its arguments to
+  the main thread (the one that invoked
+  `MirroredStrategy.call_for_each_replica()`).
+  """
+
+  def _merge_call(self, fn, args, kwargs):
+    """Delegate to the main thread to actually perform merge_call()."""
+    t = threading.current_thread()  # a _MirroredReplicaThread
+    t.merge_fn = fn
+    t.merge_args = args
+    t.merge_kwargs = kwargs
+    t.captured_name_scope = t.graph.get_name_scope()
+    # Adding a "/" at end lets us re-enter this scope later.
+    if t.captured_name_scope:
+      t.captured_name_scope += "/"
+    t.has_paused.set()
+    t.should_run.wait()
+    t.should_run.clear()
+    if t.coord.should_stop():
+      raise _RequestedStop()
+    return t.merge_result
+
+  @property
+  def devices(self):
+    distribute_lib.require_replica_context(self)
+    replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
+    return [self._distribution_strategy.extended.worker_devices[replica_id]]
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 41da3930e2..25685fb5cf 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -391,14 +391,11 @@ def validate_inputs(x, y, distribution_strategy):
               'Found unknown shape {} in input {}.'.format(s, i))
 
 
-# TODO(b/118776054): Currently we support global batch size for TPUStrategy
-# and CoreMirroredStrategy only. Remove this check when contrib MirroredStrategy
-# is no longer needed.
+# TODO(b/118776054): Currently we support global batch size for TPUStrategy and
+# core MirroredStrategy only. Remove this check when contrib MirroredStrategy is
+# no longer needed.
 def global_batch_size_supported(distribution_strategy):
-  strategy_name = distribution_strategy.__class__.__name__
-  # TODO(priyag): Change this to whatever condition makes sense when
-  # CoreMirroredStrategy is moved to core and renamed.
-  return strategy_name in ('TPUStrategy', 'CoreMirroredStrategy')
+  return distribution_strategy.extended._global_batch_size  # pylint: disable=protected-access
 
 
 # TODO(sourabhbajaj): Remove this once we use the same API for all strategies.
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 4ef784d121..f930a89f99 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1604,6 +1604,11 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
       else:
         return [self._iterator.initializer]
 
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
+
 
 # ------------------------------------------------------------------------------
 # We haven't yet implemented deserialization for DistributedVariables.
-- 
GitLab


From 38ec3338777fbd8605e45e19bb50f4e4c0055a62 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 20 Nov 2018 20:36:21 -0800
Subject: [PATCH 0647/1554] Change API for
 feature_column.categorical_column_with_vocabulary_file for TF 2.0.

PiperOrigin-RevId: 222348279
---
 .../feature_column/feature_column_v2.py       | 93 ++++++++++++++++++-
 .../golden/v2/tensorflow.feature_column.pbtxt |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  4 +
 3 files changed, 97 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 59828de840..2af2b9f254 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -1435,7 +1435,7 @@ def categorical_column_with_hash_bucket(key,
   return HashedCategoricalColumn(key, hash_bucket_size, dtype)
 
 
-@tf_export('feature_column.categorical_column_with_vocabulary_file')
+@tf_export(v1=['feature_column.categorical_column_with_vocabulary_file'])
 def categorical_column_with_vocabulary_file(key,
                                             vocabulary_file,
                                             vocabulary_size=None,
@@ -1513,6 +1513,97 @@ def categorical_column_with_vocabulary_file(key,
   Returns:
     A `CategoricalColumn` with a vocabulary file.
 
+  Raises:
+    ValueError: `vocabulary_file` is missing or cannot be opened.
+    ValueError: `vocabulary_size` is missing or < 1.
+    ValueError: `num_oov_buckets` is a negative integer.
+    ValueError: `num_oov_buckets` and `default_value` are both specified.
+    ValueError: `dtype` is neither string nor integer.
+  """
+  return categorical_column_with_vocabulary_file_v2(
+      key, vocabulary_file, vocabulary_size,
+      dtype, default_value,
+      num_oov_buckets)
+
+
+@tf_export('feature_column.categorical_column_with_vocabulary_file', v1=[])
+def categorical_column_with_vocabulary_file_v2(key,
+                                               vocabulary_file,
+                                               vocabulary_size=None,
+                                               dtype=dtypes.string,
+                                               default_value=None,
+                                               num_oov_buckets=0):
+  """A `CategoricalColumn` with a vocabulary file.
+
+  Use this when your inputs are in string or integer format, and you have a
+  vocabulary file that maps each value to an integer ID. By default,
+  out-of-vocabulary values are ignored. Use either (but not both) of
+  `num_oov_buckets` and `default_value` to specify how to include
+  out-of-vocabulary values.
+
+  For input dictionary `features`, `features[key]` is either `Tensor` or
+  `SparseTensor`. If `Tensor`, missing values can be represented by `-1` for int
+  and `''` for string, which will be dropped by this feature column.
+
+  Example with `num_oov_buckets`:
+  File '/us/states.txt' contains 50 lines, each with a 2-character U.S. state
+  abbreviation. All inputs with values in that file are assigned an ID 0-49,
+  corresponding to its line number. All other values are hashed and assigned an
+  ID 50-54.
+
+  ```python
+  states = categorical_column_with_vocabulary_file(
+      key='states', vocabulary_file='/us/states.txt', vocabulary_size=50,
+      num_oov_buckets=5)
+  columns = [states, ...]
+  features = tf.parse_example(..., features=make_parse_example_spec(columns))
+  linear_prediction = linear_model(features, columns)
+  ```
+
+  Example with `default_value`:
+  File '/us/states.txt' contains 51 lines - the first line is 'XX', and the
+  other 50 each have a 2-character U.S. state abbreviation. Both a literal 'XX'
+  in input, and other values missing from the file, will be assigned ID 0. All
+  others are assigned the corresponding line number 1-50.
+
+  ```python
+  states = categorical_column_with_vocabulary_file(
+      key='states', vocabulary_file='/us/states.txt', vocabulary_size=51,
+      default_value=0)
+  columns = [states, ...]
+  features = tf.parse_example(..., features=make_parse_example_spec(columns))
+  linear_prediction, _, _ = linear_model(features, columns)
+  ```
+
+  And to make an embedding with either:
+
+  ```python
+  columns = [embedding_column(states, 3),...]
+  features = tf.parse_example(..., features=make_parse_example_spec(columns))
+  dense_tensor = input_layer(features, columns)
+  ```
+
+  Args:
+    key: A unique string identifying the input feature. It is used as the
+      column name and the dictionary key for feature parsing configs, feature
+      `Tensor` objects, and feature columns.
+    vocabulary_file: The vocabulary file name.
+    vocabulary_size: Number of the elements in the vocabulary. This must be no
+      greater than length of `vocabulary_file`, if less than length, later
+      values are ignored. If None, it is set to the length of `vocabulary_file`.
+    dtype: The type of features. Only string and integer types are supported.
+    default_value: The integer ID value to return for out-of-vocabulary feature
+      values, defaults to `-1`. This can not be specified with a positive
+      `num_oov_buckets`.
+    num_oov_buckets: Non-negative integer, the number of out-of-vocabulary
+      buckets. All out-of-vocabulary inputs will be assigned IDs in the range
+      `[vocabulary_size, vocabulary_size+num_oov_buckets)` based on a hash of
+      the input value. A positive `num_oov_buckets` can not be specified with
+      `default_value`.
+
+  Returns:
+    A `CategoricalColumn` with a vocabulary file.
+
   Raises:
     ValueError: `vocabulary_file` is missing or cannot be opened.
     ValueError: `vocabulary_size` is missing or < 1.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
index f6e165bd7a..3aadd7dc34 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.feature_column.pbtxt
@@ -14,7 +14,7 @@ tf_module {
   }
   member_method {
     name: "categorical_column_with_vocabulary_file"
-    argspec: "args=[\'key\', \'vocabulary_file\', \'vocabulary_size\', \'num_oov_buckets\', \'default_value\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\', \"<dtype: \'string\'>\"], "
+    argspec: "args=[\'key\', \'vocabulary_file\', \'vocabulary_size\', \'dtype\', \'default_value\', \'num_oov_buckets\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'string\'>\", \'None\', \'0\'], "
   }
   member_method {
     name: "categorical_column_with_vocabulary_list"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index c60c6a4b32..0caf1743d7 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -205,6 +205,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "min_range", "max_range", "T", "mode", "name",
             "round_mode"
         ],
+        "tf.feature_column.categorical_column_with_vocabulary_file": [
+            "key", "vocabulary_file", "vocabulary_size",
+            "num_oov_buckets", "default_value", "dtype"
+        ],
         "tf.shape": ["input", "name", "out_type"],
         "tf.size": ["input", "name", "out_type"],
         "tf.sparse.concat": [
-- 
GitLab


From a911ecf5a94e037554f677670762b0122c062b41 Mon Sep 17 00:00:00 2001
From: Cibifang <cifang_bifang@hotmail.com>
Date: Wed, 21 Nov 2018 13:34:07 +0800
Subject: [PATCH 0648/1554] Raise error if trying to use Gradients when scope
 have device or control dependencies specified(Gradients doesn't currently
 support them).

---
 tensorflow/go/op/gradients.go      | 16 +++++++++++++++-
 tensorflow/go/op/gradients_test.go | 30 ++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/gradients.go b/tensorflow/go/op/gradients.go
index 2eaa7e70ab..9f892e1da6 100644
--- a/tensorflow/go/op/gradients.go
+++ b/tensorflow/go/op/gradients.go
@@ -16,7 +16,12 @@ limitations under the License.
 
 package op
 
-import tf "github.com/tensorflow/tensorflow/tensorflow/go"
+import (
+	"fmt"
+
+	tf "github.com/tensorflow/tensorflow/tensorflow/go"
+)
+
 
 // Gradients adds gradients computation ops to the graph according to scope.
 //
@@ -27,6 +32,15 @@ import tf "github.com/tensorflow/tensorflow/tensorflow/go"
 //
 //  return the partial derivatives
 func Gradients(scope *Scope, y []tf.Output, x []tf.Output, dx ...tf.Output) (output []tf.Output) {
+	if len(scope.controlDependencies) > 0 {
+		scope.UpdateErr("Gradients", fmt.Errorf("Gradients does not currently support control dependencies (via Scope.WithControlDependencies)."))
+		return
+	}
+	if scope.device != "" {
+		scope.UpdateErr("Gradients", fmt.Errorf("Gradients does not currently support device annotations (via Scope.WithDevice)."))
+		return
+	}
+
 	var err error
 	if output, err = scope.graph.AddGradients(scope.opName("Gradients"), y, x, dx); err != nil {
 		scope.UpdateErr("Gradients", err)
diff --git a/tensorflow/go/op/gradients_test.go b/tensorflow/go/op/gradients_test.go
index 1febd08366..3d1d57b77e 100644
--- a/tensorflow/go/op/gradients_test.go
+++ b/tensorflow/go/op/gradients_test.go
@@ -214,3 +214,33 @@ func TestValidateGradientsNames(t *testing.T) {
 		t.Error("Gradients should have failed if executed more than once for scope of the same namespace")
 	}
 }
+
+func TestAddGradientsWithControlDependencies(t *testing.T) {
+	var (
+		s        = NewScope()
+		zero     = Const(s.SubScope("zero"), int32(0))
+		x        = Placeholder(s.SubScope("x"), tf.Float)
+		y0       = Square(s.SubScope("y0"), x)
+		variable = VarHandleOp(s, tf.Int32, tf.ScalarShape())
+		init     = AssignVariableOp(s, variable, zero)
+		readDeps = []*tf.Operation{init}
+	)
+	s = s.WithControlDependencies(readDeps...)
+	Gradients(s, []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err == nil {
+		t.Error("Gradients should have failed when control dependencies are set")
+	}
+}
+
+func TestAddGradientsWithDevice(t *testing.T) {
+	var (
+		s  = NewScope()
+		x  = Placeholder(s.SubScope("x"), tf.Float)
+		y0 = Square(s.SubScope("y0"), x)
+	)
+	s = s.WithDevice("/device:GPU:0")
+	Gradients(s, []tf.Output{y0}, []tf.Output{x})
+	if err := s.Err(); err == nil {
+		t.Error("Gradients should have failed when device is set")
+	}
+}
-- 
GitLab


From abea54aa8c963ea6cd3edaa9a28acf4999eb6e70 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 20 Nov 2018 22:57:59 -0800
Subject: [PATCH 0649/1554] Adding V2 APIs for Mean and Accuracy metrics.

PiperOrigin-RevId: 222357974
---
 tensorflow/python/keras/metrics.py            | 153 ++++++++++++--
 tensorflow/python/keras/metrics_test.py       |  48 ++++-
 tensorflow/python/ops/metrics_impl.py         |   4 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  | 194 ++++++++++++++++++
 ...rflow.keras.metrics.-binary-accuracy.pbtxt | 194 ++++++++++++++++++
 ....keras.metrics.-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 .../v2/tensorflow.keras.metrics.-mean.pbtxt   | 192 +++++++++++++++++
 ...metrics.-sparse-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |  20 ++
 .../v2/tensorflow.metrics.-accuracy.pbtxt     | 194 ++++++++++++++++++
 .../tensorflow.metrics.-binary-accuracy.pbtxt | 194 ++++++++++++++++++
 ...orflow.metrics.-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 .../golden/v2/tensorflow.metrics.-mean.pbtxt  | 192 +++++++++++++++++
 ...metrics.-sparse-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 .../api/golden/v2/tensorflow.metrics.pbtxt    |  26 ++-
 15 files changed, 2163 insertions(+), 24 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 0793585047..de99d47d5e 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -401,7 +401,7 @@ def _update_confusion_matrix_variables(variables_to_update,
 class Metric(Layer):
   """Encapsulates metric logic and state.
 
-  Usage with eager execution:
+  Usage:
 
   ```python
   m = SomeMetric(...)
@@ -410,19 +410,6 @@ class Metric(Layer):
   print('Final result: ', m.result().numpy())
   ```
 
-  Usage with graph execution:
-
-  ```python
-  m = SomeMetric(...)
-  init_op = tf.variables_initializer(m.variables)  # Initialize variables
-  with tf.Session() as sess:
-    sess.run(init_op)
-    for input in ...:
-      update_op = m.update_state(input)
-      sess.run(update_op)
-    print('Final result: ', sess.run(m.result()))
-  ```
-
   Usage with tf.keras API:
 
   ```python
@@ -600,15 +587,35 @@ class Metric(Layer):
   ### End: For use by subclasses ###
 
 
+@tf_export('metrics.Mean', 'keras.metrics.Mean', v1=[])
 class Mean(Metric):
   """Computes the (weighted) mean of the given values.
 
+  For example, if values is [1, 3, 5, 7] then the mean is 4.
+  If the weights were specified as [1, 1, 0, 0] then the mean would be 2.
+
   This metric creates two variables, `total` and `count` that are used to
   compute the average of `values`. This average is ultimately returned as `mean`
   which is an idempotent operation that simply divides `total` by `count`.
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.Mean()
+  m.update_state([1, 3, 5, 7])
+  print('Final result: ', m.result().numpy())  # Final result: 4.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.add_metric(metrics_module.Mean(name='mean_1')(outputs))
+  model.compile('sgd', loss='mse')
+  ```
   """
 
   def __init__(self, name='mean', dtype=None):
@@ -721,9 +728,56 @@ class MeanMetricWrapper(Mean):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+@tf_export('metrics.Accuracy', 'keras.metrics.Accuracy', v1=[])
+class Accuracy(MeanMetricWrapper):
+  """Calculates how often predictions matches labels.
+
+  For example, if `y_true` is [1, 2, 3, 4] and `y_pred` is [0, 2, 3, 4]
+  then the accuracy is 3/4 or .75.  If the weights were specified as
+  [1, 1, 0, 0] then the accuracy would be 1/2 or .5.
+
+  This metric creates two local variables, `total` and `count` that are used to
+  compute the frequency with which `y_pred` matches `y_true`. This frequency is
+  ultimately returned as `binary accuracy`: an idempotent operation that simply
+  divides `total` by `count`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.Accuracy()
+  m.update_state([1, 2, 3, 4], [0, 2, 3, 4])
+  print('Final result: ', m.result().numpy())  # Final result: 0.75
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.Accuracy()])
+  ```
+  """
+
+  def __init__(self, name='accuracy', dtype=None):
+    super(Accuracy, self).__init__(accuracy, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(Accuracy, cls).from_config(config)
+
+
+@tf_export('metrics.BinaryAccuracy', 'keras.metrics.BinaryAccuracy', v1=[])
 class BinaryAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
+  For example, if `y_true` is [1, 1, 0, 0] and `y_pred` is [0.98, 1, 0, 0.6]
+  then the binary accuracy is 3/4 or .75.  If the weights were specified as
+  [1, 0, 0, 1] then the binary accuracy would be 1/2 or .5.
+
   This metric creates two local variables, `total` and `count` that are used to
   compute the frequency with which `y_pred` matches `y_true`. This frequency is
   ultimately returned as `binary accuracy`: an idempotent operation that simply
@@ -731,6 +785,21 @@ class BinaryAccuracy(MeanMetricWrapper):
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.BinaryAccuracy()
+  m.update_state([1, 1, 0, 0], [0.98, 1, 0, 0.6])
+  print('Final result: ', m.result().numpy())  # Final result: 0.75
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.BinaryAccuracy()])
+  ```
   """
 
   def __init__(self, name='binary_accuracy', dtype=None, threshold=0.5):
@@ -752,9 +821,16 @@ class BinaryAccuracy(MeanMetricWrapper):
     return super(BinaryAccuracy, cls).from_config(config)
 
 
+@tf_export(
+    'metrics.CategoricalAccuracy', 'keras.metrics.CategoricalAccuracy', v1=[])
 class CategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
+  For example, if `y_true` is [[0, 0, 1], [0, 1, 0]] and `y_pred` is
+  [[0.1, 0.9, 0.8], [0.05, 0.95, 0]] then the categorical accuracy is 1/2 or .5.
+  If the weights were specified as [0.7, 0.3] then the categorical accuracy
+  would be .3.
+
   This metric creates two local variables, `total` and `count` that are used to
   compute the frequency with which `y_pred` matches `y_true`. This frequency is
   ultimately returned as `categorical accuracy`: an idempotent operation that
@@ -765,6 +841,21 @@ class CategoricalAccuracy(MeanMetricWrapper):
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.CategoricalAccuracy()
+  m.update_state([[0, 0, 1], [0, 1, 0]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+  print('Final result: ', m.result().numpy())  # Final result: 0.5
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.CategoricalAccuracy()])
+  ```
   """
 
   def __init__(self, name='categorical_accuracy', dtype=None):
@@ -784,9 +875,18 @@ class CategoricalAccuracy(MeanMetricWrapper):
     return super(CategoricalAccuracy, cls).from_config(config)
 
 
+@tf_export(
+    'metrics.SparseCategoricalAccuracy',
+    'keras.metrics.SparseCategoricalAccuracy',
+    v1=[])
 class SparseCategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches integer labels.
 
+  For example, if `y_true` is [[2], [1]] and `y_pred` is
+  [[0.1, 0.9, 0.8], [0.05, 0.95, 0]] then the categorical accuracy is 1/2 or .5.
+  If the weights were specified as [0.7, 0.3] then the categorical accuracy
+  would be .3.
+
   This metric creates two local variables, `total` and `count` that are used to
   compute the frequency with which `y_pred` matches `y_true`. This frequency is
   ultimately returned as `sparse categorical accuracy`: an idempotent operation
@@ -794,6 +894,24 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.SparseCategoricalAccuracy()
+  m.update_state([[2], [1]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+  print('Final result: ', m.result().numpy())  # Final result: 0.5
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+      'sgd',
+      loss='mse',
+      metrics=[tf.metrics.SparseCategoricalAccuracy()])
+  ```
   """
 
   def __init__(self, name='sparse_categorical_accuracy', dtype=None):
@@ -1097,6 +1215,13 @@ class Recall(Metric):
         array_ops.zeros_like(self.thresholds))
 
 
+def accuracy(y_true, y_pred):
+  y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
+  if y_true.dtype != y_pred.dtype:
+    y_pred = math_ops.cast(y_pred, y_true.dtype)
+  return math_ops.cast(math_ops.equal(y_true, y_pred), K.floatx())
+
+
 @tf_export('keras.metrics.binary_accuracy')
 def binary_accuracy(y_true, y_pred, threshold=0.5):
   threshold = math_ops.cast(threshold, y_pred.dtype)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 535802a64a..eeade4f37d 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -49,7 +49,7 @@ class KerasMetricsTest(test.TestCase):
         output = metric(y_a, y_b)
         self.assertEqual(K.eval(output).shape, (6,))
 
-  def test_sparse_categorical_accuracy(self):
+  def test_sparse_categorical_accuracy_int(self):
     with self.cached_session():
       metric = metrics.sparse_categorical_accuracy
       y_true = K.variable(np.random.randint(0, 7, (6,)))
@@ -367,6 +367,28 @@ class KerasMetricsTest(test.TestCase):
     self.assertEqual(200., self.evaluate(restore_mean.result()))
     self.assertEqual(3, self.evaluate(restore_mean.count))
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_accuracy(self):
+    acc_obj = metrics.Accuracy(name='my acc')
+
+    # check config
+    self.assertEqual(acc_obj.name, 'my acc')
+    self.assertTrue(acc_obj.stateful)
+    self.assertEqual(len(acc_obj.variables), 2)
+    self.assertEqual(acc_obj.dtype, dtypes.float32)
+    self.evaluate(variables.variables_initializer(acc_obj.variables))
+
+    # verify that correct value is returned
+    update_op = acc_obj.update_state([[1], [2], [3], [4]], [[1], [2], [3], [4]])
+    self.evaluate(update_op)
+    result = self.evaluate(acc_obj.result())
+    self.assertEqual(result, 1)  # 2/2
+
+    # check with sample_weight
+    result_t = acc_obj([[2], [1]], [[2], [0]], sample_weight=[[0.5], [0.2]])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 0.96, 2)  # 4.5/4.7
+
   @test_util.run_in_graph_and_eager_modes
   def test_binary_accuracy(self):
     acc_obj = metrics.BinaryAccuracy(name='my acc')
@@ -437,6 +459,30 @@ class KerasMetricsTest(test.TestCase):
     result = self.evaluate(result_t)
     self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_sparse_categorical_accuracy(self):
+    acc_obj = metrics.SparseCategoricalAccuracy(name='my acc')
+
+    # check config
+    self.assertEqual(acc_obj.name, 'my acc')
+    self.assertTrue(acc_obj.stateful)
+    self.assertEqual(len(acc_obj.variables), 2)
+    self.assertEqual(acc_obj.dtype, dtypes.float32)
+    self.evaluate(variables.variables_initializer(acc_obj.variables))
+
+    # verify that correct value is returned
+    update_op = acc_obj.update_state([[2], [1]],
+                                     [[0.1, 0.1, 0.8], [0.05, 0.95, 0]])
+    self.evaluate(update_op)
+    result = self.evaluate(acc_obj.result())
+    self.assertEqual(result, 1)  # 2/2
+
+    # check with sample_weight
+    result_t = acc_obj([[2], [1]], [[0.1, 0.1, 0.8], [0.05, 0, 0.95]],
+                       [[0.5], [0.2]])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7
+
   @test_util.run_in_graph_and_eager_modes
   def test_invalid_result(self):
 
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 03de8d552a..0b91b8dde8 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -312,7 +312,7 @@ def _aggregate_across_replicas(metrics_collections, metric_value_fn, *args):
       fn, args=args)
 
 
-@tf_export('metrics.mean')
+@tf_export(v1=['metrics.mean'])
 def mean(values,
          weights=None,
          metrics_collections=None,
@@ -393,7 +393,7 @@ def mean(values,
     return mean_t, update_op
 
 
-@tf_export('metrics.accuracy')
+@tf_export(v1=['metrics.accuracy'])
 def accuracy(labels,
              predictions,
              weights=None,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
new file mode 100644
index 0000000000..2db07df523
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.Accuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
new file mode 100644
index 0000000000..904ad3a21a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.BinaryAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..17b74924fa
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.CategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
new file mode 100644
index 0000000000..40fe64bbd2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.keras.metrics.Mean"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..0c17452292
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.SparseCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index a296e13158..8a8fb97b96 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -1,5 +1,25 @@
 path: "tensorflow.keras.metrics"
 tf_module {
+  member {
+    name: "Accuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BinaryAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Mean"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "KLD"
     argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
new file mode 100644
index 0000000000..f8e12f8817
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.Accuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
new file mode 100644
index 0000000000..b9bc6a716a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.BinaryAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..0ef75d8756
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.CategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
new file mode 100644
index 0000000000..7fe6d6fda9
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.metrics.Mean"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..7bce43fbde
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.SparseCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
index e9b996c9f5..d82ce8b38a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
@@ -1,8 +1,24 @@
 path: "tensorflow.metrics"
 tf_module {
-  member_method {
-    name: "accuracy"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "Accuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BinaryAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Mean"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalAccuracy"
+    mtype: "<type \'type\'>"
   }
   member_method {
     name: "auc"
@@ -28,10 +44,6 @@ tf_module {
     name: "false_positives_at_thresholds"
     argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "mean"
-    argspec: "args=[\'values\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "mean_absolute_error"
     argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-- 
GitLab


From 186da29b491d43b3c02c692e1a64222ca57b0be4 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 21 Nov 2018 00:25:50 -0800
Subject: [PATCH 0650/1554] Disable conv_ops_test in msan

PiperOrigin-RevId: 222364783
---
 tensorflow/core/kernels/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ae76034b0b..4fe0d2e87e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1257,6 +1257,9 @@ tf_cc_test(
     name = "conv_ops_test",
     size = "medium",
     srcs = ["conv_ops_test.cc"],
+    tags = [
+        "nomsan",
+    ],
     deps = [
         ":conv_ops",
         ":image",
-- 
GitLab


From 777039302d52565ef838082cde12bdfa6a1e38e1 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 21 Nov 2018 00:27:16 -0800
Subject: [PATCH 0651/1554] Disable flaky stats_dataset_ops_test in tsan.

PiperOrigin-RevId: 222364901
---
 tensorflow/python/data/experimental/kernel_tests/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index c9b11a2c38..6a387f55bd 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -618,7 +618,10 @@ py_test(
     size = "medium",
     srcs = ["stats_dataset_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "notsan",
+    ],
     deps = [
         ":reader_dataset_ops_test_base",
         ":stats_dataset_test_base",
-- 
GitLab


From 2d580e7d04d5fa96cf2553ed041b6234ffe4f62d Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 21 Nov 2018 00:29:00 -0800
Subject: [PATCH 0652/1554] Bump the size of c_api_exoerimental_test. It
 occasionally times out when run with thread sanitizer.

PiperOrigin-RevId: 222364999
---
 tensorflow/c/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 71089a87c7..f653e581bf 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -264,7 +264,7 @@ tf_cuda_cc_test(
 
 tf_cc_test(
     name = "c_api_experimental_test",
-    size = "small",
+    size = "medium",
     srcs = ["c_api_experimental_test.cc"],
     data = ["testdata/tf_record"],
     linkopts = select({
-- 
GitLab


From f1b77734a76fe8ad6c1af21e7e748af93da88913 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 01:02:16 -0800
Subject: [PATCH 0653/1554] compat: Update forward compatibility horizon to
 2018-11-21

PiperOrigin-RevId: 222368181
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 1b8114dd26..75290f0613 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 20)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 21)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 7586bb3cfba2bbfffe8d74522a9a5a00c137f6e3 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 21 Nov 2018 02:19:56 -0800
Subject: [PATCH 0654/1554] Simplify TopK xla client op.

Instead of creating a rank 1 iota and broadcasting it, we can directly create
the iota we need.

PiperOrigin-RevId: 222377479
---
 tensorflow/compiler/xla/client/lib/BUILD      |  2 ++
 tensorflow/compiler/xla/client/lib/sorting.cc | 11 ++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index f833ddcd32..bbfe8712c5 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -179,7 +179,9 @@ cc_library(
     hdrs = ["sorting.h"],
     deps = [
         ":numeric",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
     ],
diff --git a/tensorflow/compiler/xla/client/lib/sorting.cc b/tensorflow/compiler/xla/client/lib/sorting.cc
index 0475fd9c94..d3980d5575 100644
--- a/tensorflow/compiler/xla/client/lib/sorting.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/sorting.h"
 #include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/util.h"
 
 namespace xla {
 
@@ -23,13 +25,12 @@ XlaOp TopK(XlaOp input, int64 k) {
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(Shape input_shape, builder->GetShape(input));
     int last_dim = input_shape.dimensions_size() - 1;
-    int last_dim_size = input_shape.dimensions(last_dim);
 
-    XlaOp iota_s32 = Iota(builder, S32, last_dim_size);
+    Shape iota_shape =
+        ShapeUtil::MakeShape(S32, AsInt64Slice(input_shape.dimensions()));
+    XlaOp iota_s32 = Iota(builder, iota_shape, last_dim);
     auto input_dims = input_shape.dimensions();
-    std::vector<int64> broadcast_dims(input_dims.begin(), input_dims.end() - 1);
-    XlaOp broadcast_s32 = Broadcast(iota_s32, broadcast_dims);
-    XlaOp sort_result = Sort(Neg(input), {broadcast_s32});
+    XlaOp sort_result = Sort(Neg(input), {iota_s32});
     std::vector<int64> start_indices(input_shape.dimensions_size(), 0);
     std::vector<int64> limit_indices(input_dims.begin(), input_dims.end());
     limit_indices[last_dim] = k;
-- 
GitLab


From 887e6042951aadaa247cc801ae5713a87dc7666f Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 21 Nov 2018 04:47:21 -0800
Subject: [PATCH 0655/1554] Make TopK stable on the GPU backend.

TopK op needs the guarantee that the sorting is stable. In that case, a
'values' parameter is passed which is a iota. We can use that to disambiguate
comparison ties to sort by original index, which makes the sorting stable.
Note that this still doesn't guarantee that the sort op is stable for all
inputs, just that it is stable when used by TopK.

PiperOrigin-RevId: 222390450
---
 tensorflow/compiler/xla/client/lib/BUILD      |  3 +-
 .../compiler/xla/client/lib/sorting_test.cc   |  8 ++
 .../xla/service/gpu/ir_emitter_unnested.cc    | 18 +++-
 .../compiler/xla/service/llvm_ir/sort_util.cc | 85 +++++++++++--------
 .../compiler/xla/service/llvm_ir/sort_util.h  |  5 +-
 5 files changed, 77 insertions(+), 42 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index bbfe8712c5..45a7b020b3 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -191,8 +191,7 @@ xla_test(
     name = "sorting_test",
     srcs = ["sorting_test.cc"],
     blacklisted_backends = [
-        "cpu",
-        "gpu",
+        "cpu",  # Fails with --config=asan.
     ],
     tags = ["enable_for_xla_interpreter"],
     deps = [
diff --git a/tensorflow/compiler/xla/client/lib/sorting_test.cc b/tensorflow/compiler/xla/client/lib/sorting_test.cc
index fef98c9923..ebb30d3acc 100644
--- a/tensorflow/compiler/xla/client/lib/sorting_test.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting_test.cc
@@ -56,5 +56,13 @@ XLA_TEST_F(SortingTest, TopKFullSort) {
   ComputeAndCompareR1<float>(&builder, inputs, {});
 }
 
+XLA_TEST_F(SortingTest, TopKFullSortWithDuplicates) {
+  XlaBuilder builder(TestName());
+  XlaOp a;
+  auto a_data = CreateR1Parameter<int>({1, 1, 2, 2, 1}, 0, "a", &builder, &a);
+  xla::GetTupleElement(xla::TopK(a, 5), 1);
+  ComputeAndCompareR1<int>(&builder, {2, 3, 0, 1, 4}, {a_data.get()});
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index efe335c1c1..ebd73f3a91 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -65,6 +65,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/buffer_assignment_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h"
@@ -2172,7 +2173,18 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) {
 Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
   std::vector<std::unique_ptr<Thunk>> thunks;
   Shape keys_shape = sort->operand(0)->shape();
+  int64 dimension_to_sort = sort->dimensions(0);
+  // In case there is a 'values' parameter that is a iota, we take note and use
+  // it later to ensure a stable sort. Otherwise, we don't guarantee a stable
+  // sort.
+  int64 iota_values_parameter_index = -1;
   for (int64 i = 0; i < sort->operand_count(); ++i) {
+    if (i > 0 && sort->operand(i)->opcode() == HloOpcode::kIota &&
+        ShapeUtil::ElementIsIntegral(sort->operand(i)->shape()) &&
+        Cast<HloIotaInstruction>(sort->operand(i))->iota_dimension() ==
+            dimension_to_sort) {
+      iota_values_parameter_index = i;
+    }
     ShapeIndex shape_index =
         sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({});
     // We assume that the layout of all involved operands and outputs is the
@@ -2197,7 +2209,6 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
     }
   }
 
-  int64 dimension_to_sort = sort->dimensions(0);
   uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort);
   int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound);
   CHECK_GE(1ULL << num_stages, dimension_to_sort_bound);
@@ -2299,8 +2310,9 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
       }
     }
     return llvm_ir::EmitSortInPlace(
-        dimension_to_sort, keys_array, values_arrays, IrName(sort), xor_masks,
-        &b_, launch_dimensions,
+        dimension_to_sort, keys_array, values_arrays,
+        iota_values_parameter_index, IrName(sort), xor_masks, &b_,
+        launch_dimensions,
         xor_masks.size() > 1 ? num_iterations_in_sort_dim
                              : standard_num_iterations_in_sort_dim,
         kTileSize);
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
index fd16af67fe..e22c2173c2 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
@@ -47,7 +47,8 @@ namespace {
 // Adds the inner comparison loop body where we compare elements.
 void EmitCompareLoopBody(
     int64 iteration_bound, PrimitiveType key_type, int64 num_values,
-    llvm::Value* element_pair_index, int64 xor_mask, llvm::Type* index_type,
+    int64 iota_values_parameter_index, llvm::Value* element_pair_index,
+    int64 xor_mask, llvm::Type* index_type,
     std::function<llvm::Value*(int64 operand, llvm::Value* index)> read_element,
     std::function<void(int64 operand, llvm::Value* index, llvm::Value* value)>
         write_element,
@@ -139,34 +140,42 @@ void EmitCompareLoopBody(
       is_signed_comparison = false;
     }
     // If key2 < key1
-    ksl.IfReturnVoid(
-        "is_smaller_than",
+    auto is_smaller_than =
         b->CreateICmp(is_signed_comparison ? llvm::ICmpInst::ICMP_SLT
                                            : llvm::ICmpInst::ICMP_ULT,
-                      compare_key2, compare_key1),
-        [&]() {
-          // Swap key1 with key2.
-          write_element(0, current_keys_index, key2);
-          write_element(0, compare_keys_index, key1);
-          for (int64 i = 1; i <= num_values; ++i) {
-            // Also swap the values.
-            auto value1 = read_element(i, current_keys_index);
-            auto value2 = read_element(i, compare_keys_index);
-            write_element(i, current_keys_index, value2);
-            write_element(i, compare_keys_index, value1);
-          }
-        });
+                      compare_key2, compare_key1);
+    if (iota_values_parameter_index >= 0) {
+      auto keys_equal = b->CreateICmpEQ(compare_key1, compare_key2);
+      auto key_index1 =
+          read_element(iota_values_parameter_index, current_keys_index);
+      auto key_index2 =
+          read_element(iota_values_parameter_index, compare_keys_index);
+      auto index_is_smaller_than =
+          b->CreateICmp(llvm::ICmpInst::ICMP_ULT, key_index2, key_index1);
+      is_smaller_than = b->CreateOr(
+          is_smaller_than, b->CreateAnd(keys_equal, index_is_smaller_than));
+    }
+    ksl.IfReturnVoid("is_smaller_than", is_smaller_than, [&]() {
+      // Swap key1 with key2.
+      write_element(0, current_keys_index, key2);
+      write_element(0, compare_keys_index, key1);
+      for (int64 i = 1; i <= num_values; ++i) {
+        // Also swap the values.
+        auto value1 = read_element(i, current_keys_index);
+        auto value2 = read_element(i, compare_keys_index);
+        write_element(i, current_keys_index, value2);
+        write_element(i, compare_keys_index, value1);
+      }
+    });
   });
 }
 
-void EmitTiledCompareLoop(const IrArray::Index& tiled_keys_index,
-                          int64 dimension_to_sort,
-                          int64 dimension_to_sort_bound,
-                          PrimitiveType keys_type,
-                          absl::Span<const int64> xor_masks,
-                          const std::vector<IrArray>& params,
-                          const std::vector<llvm::Value*>& param_shmem_buffers,
-                          int64 tile_size, llvm::IRBuilder<>* b) {
+void EmitTiledCompareLoop(
+    const IrArray::Index& tiled_keys_index, int64 dimension_to_sort,
+    int64 dimension_to_sort_bound, PrimitiveType keys_type,
+    absl::Span<const int64> xor_masks, const std::vector<IrArray>& params,
+    const std::vector<llvm::Value*>& param_shmem_buffers,
+    int64 iota_values_parameter_index, int64 tile_size, llvm::IRBuilder<>* b) {
   KernelSupportLibrary ksl(b);
   llvm::Value* thread_id = llvm_ir::EmitCallToIntrinsic(
       llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b);
@@ -253,20 +262,22 @@ void EmitTiledCompareLoop(const IrArray::Index& tiled_keys_index,
                   RoundDownToNearest(dimension_to_sort_bound, tile_size))),
           [&]() {
             EmitCompareLoopBody(dimension_to_sort_bound % tile_size, keys_type,
-                                params.size() - 1, element_pair_index, xor_mask,
+                                params.size() - 1, iota_values_parameter_index,
+                                element_pair_index, xor_mask,
                                 tiled_keys_index.GetType(), read_element,
                                 write_element, b);
           },
           [&]() {
-            EmitCompareLoopBody(
-                tile_size, keys_type, params.size() - 1, element_pair_index,
-                xor_mask, tiled_keys_index.GetType(), read_element,
-                write_element, b, /*needs_bounds_checks=*/false);
+            EmitCompareLoopBody(tile_size, keys_type, params.size() - 1,
+                                iota_values_parameter_index, element_pair_index,
+                                xor_mask, tiled_keys_index.GetType(),
+                                read_element, write_element, b,
+                                /*needs_bounds_checks=*/false);
           });
     } else {
       EmitCompareLoopBody(tile_size, keys_type, params.size() - 1,
-                          element_pair_index, xor_mask,
-                          tiled_keys_index.GetType(), read_element,
+                          iota_values_parameter_index, element_pair_index,
+                          xor_mask, tiled_keys_index.GetType(), read_element,
                           write_element, b, /*needs_bounds_checks=*/false);
     }
     // Wait until all comparisons have happened.
@@ -296,6 +307,7 @@ void EmitTiledCompareLoop(const IrArray::Index& tiled_keys_index,
 
 Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
                        const std::vector<IrArray>& values_arrays,
+                       int64 iota_values_parameter_index,
                        absl::string_view name,
                        absl::Span<const int64> xor_masks, llvm::IRBuilder<>* b,
                        const gpu::LaunchDimensions& launch_dimensions,
@@ -367,8 +379,8 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
     if (xor_masks.size() > 1) {
       EmitTiledCompareLoop(keys_index, dimension_to_sort,
                            dimension_to_sort_bound, keys_shape.element_type(),
-                           xor_masks, params, param_shmem_buffers, tile_size,
-                           b);
+                           xor_masks, params, param_shmem_buffers,
+                           iota_values_parameter_index, tile_size, b);
     } else {
       auto read_element = [&](int64 operand, llvm::Value* index) {
         keys_index[dimension_to_sort] = index;
@@ -380,9 +392,10 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
         params[operand].EmitWriteArrayElement(keys_index, value, b);
       };
       EmitCompareLoopBody(dimension_to_sort_bound, keys_shape.element_type(),
-                          values_arrays.size(), tiles_index[rank - 1],
-                          xor_masks[0], tiles_index.GetType(), read_element,
-                          write_element, b);
+                          values_arrays.size(), iota_values_parameter_index,
+                          tiles_index[rank - 1], xor_masks[0],
+                          tiles_index.GetType(), read_element, write_element,
+                          b);
     }
     return Status::OK();
   };
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
index 556a217322..685f9383ac 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
@@ -31,9 +31,12 @@ namespace llvm_ir {
 // Emits llvm IR to do pairwise comparisons/swaps in the 'dimension_to_sort'
 // dimension of 'keys_array'. All other dimensions are kept as-is. This
 // implements the inner loop of BitonicSort. It is assumed that 'xor_masks'
-// contains only powers of 2, or values 2^k - 1 (k > 0).
+// contains only powers of 2, or values 2^k - 1 (k > 0). If
+// 'iota_values_parameter_index' is >= 0, it points at a 'values_arrays' operand
+// that is a iota and can be used to make the sorting stable.
 Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
                        const std::vector<IrArray>& values_arrays,
+                       int64 iota_values_parameter_index,
                        absl::string_view name,
                        absl::Span<const int64> xor_masks, llvm::IRBuilder<>* b,
                        const gpu::LaunchDimensions& launch_dimensions,
-- 
GitLab


From 077fbf0a3518e6fadbf7fa671b134a420b52e759 Mon Sep 17 00:00:00 2001
From: Thomas Joerg <tjoerg@google.com>
Date: Wed, 21 Nov 2018 05:10:01 -0800
Subject: [PATCH 0656/1554] [XLA:GPU] Check explicitly for reduce input fusions
 in fusion_merger.

Now that Scatter kInput fusions have been introduced, checking just the fusion kind is not enough.

PiperOrigin-RevId: 222392637
---
 .../compiler/xla/service/gpu/fusion_merger.cc |  2 +-
 .../compiler/xla/service/gpu/gpu_fusible.cc   | 23 +++++++++---------
 .../compiler/xla/service/gpu/gpu_fusible.h    | 11 +++++----
 .../xla/service/gpu/gpu_fusible_test.cc       | 24 ++++++++++++-------
 4 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/fusion_merger.cc b/tensorflow/compiler/xla/service/gpu/fusion_merger.cc
index 30c1f90889..470457935a 100644
--- a/tensorflow/compiler/xla/service/gpu/fusion_merger.cc
+++ b/tensorflow/compiler/xla/service/gpu/fusion_merger.cc
@@ -229,7 +229,7 @@ Status FusionInstructionMerger::HandleFusion(HloInstruction* fusion) {
   if (!absl::c_all_of(fusion->users(), [&](const HloInstruction* user) {
         return user->opcode() == HloOpcode::kFusion &&
                (user->fusion_kind() == HloInstruction::FusionKind::kLoop ||
-                (user->fusion_kind() == HloInstruction::FusionKind::kInput &&
+                (IsReduceInputFusion(*user) &&
                  LayoutsAreReduceInputFusionFriendly(*fusion, *user)));
       })) {
     VLOG(3) << "Not merging " << fusion->name()
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
index 2d31fd5570..392b149abd 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
@@ -55,7 +55,7 @@ bool LayoutsAreReduceInputFusionFriendly(const HloInstruction& producer,
   });
 }
 
-bool IsInputFusibleReduction(const HloInstruction& instr) {
+bool IsReduceInputFusion(const HloInstruction& instr) {
   if (instr.IsMultiOutputFusion()) {
     for (const HloInstruction* operand :
          instr.fused_expression_root()->operands()) {
@@ -67,17 +67,18 @@ bool IsInputFusibleReduction(const HloInstruction& instr) {
         return true;
       }
     }
-    return false;
-  } else if (instr.opcode() == HloOpcode::kFusion) {
-    if (IsReductionToVector(*instr.fused_expression_root())) {
-      CHECK(instr.fusion_kind() == HloInstruction::FusionKind::kInput)
-          << " Fusion rooted at reduction-to-vector op must be of kind kInput: "
-          << instr.ToString();
-      return true;
-    }
-    return false;
+  } else if (instr.opcode() == HloOpcode::kFusion &&
+             IsReductionToVector(*instr.fused_expression_root())) {
+    CHECK(instr.fusion_kind() == HloInstruction::FusionKind::kInput)
+        << " Fusion rooted at reduction-to-vector op must be of kind kInput: "
+        << instr.ToString();
+    return true;
   }
-  return IsReductionToVector(instr);
+  return false;
+}
+
+bool IsInputFusibleReduction(const HloInstruction& instr) {
+  return IsReduceInputFusion(instr) || IsReductionToVector(instr);
 }
 
 }  // namespace gpu
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.h b/tensorflow/compiler/xla/service/gpu/gpu_fusible.h
index f7c24a0d5b..c0be354730 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.h
@@ -33,14 +33,17 @@ namespace gpu {
 bool LayoutsAreReduceInputFusionFriendly(const HloInstruction& producer,
                                          const HloInstruction& reduce);
 
-// Whether `instr` is fusible as root of a reduce input fusions, i.e. `instr`
-// is either an unfused reduction-to-vector op, an input fusion rooted at a
-// reduction-to-vector op, or a multi-output input fusion with at least one
-// reduction-to-vector op root.
 // Note that reduction ops are lowered in different ways. Reduce input fusions
 // are lowered by IrEmitterUnnested::EmitReductionToVector and must be rooted at
 // reduction-to-vector ops. Other reduction ops are lowered by
 // GpuElementalIrEmitter and fused like elementwise ops.
+
+// Whether `instr` is an input fusion rooted at a reduction-to-vector op or a
+// multi-output input fusion with at least one reduction-to-vector op root.
+bool IsReduceInputFusion(const HloInstruction& instr);
+
+// Whether `instr` is fusible as root of a reduce input fusions, i.e. `instr`
+// is either an unfused reduction-to-vector op or a reduce input fusion.
 bool IsInputFusibleReduction(const HloInstruction& instr);
 
 }  // namespace gpu
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
index d91b7bc61f..12222500ea 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
@@ -178,7 +178,7 @@ TEST_F(GpuFusibleTest,
   EXPECT_TRUE(LayoutsAreReduceInputFusionFriendly(*loop_fusion, *reduce));
 }
 
-TEST_F(GpuFusibleTest, IsInputFusibleReduction_ReductionToVector) {
+TEST_F(GpuFusibleTest, IsReduceInputFusion_ReductionToVector) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     ENTRY entry {
       c0 = f32[] parameter(0)
@@ -191,10 +191,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_ReductionToVector) {
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kReduce);
+  EXPECT_FALSE(IsReduceInputFusion(*reduce));
   EXPECT_TRUE(IsInputFusibleReduction(*reduce));
 }
 
-TEST_F(GpuFusibleTest, IsInputFusibleReduction_ElementalReduction) {
+TEST_F(GpuFusibleTest, IsReduceInputFusion_ElementalReduction) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     ENTRY entry {
       c0 = f32[] parameter(0)
@@ -207,10 +208,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_ElementalReduction) {
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kReduce);
+  EXPECT_FALSE(IsReduceInputFusion(*reduce));
   EXPECT_FALSE(IsInputFusibleReduction(*reduce));
 }
 
-TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputInputReduceFusion) {
+TEST_F(GpuFusibleTest, IsReduceInputFusion_SingleOutputInputReduceFusion) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     fused_reduction {
       c0 = f32[] parameter(0)
@@ -225,10 +227,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputInputReduceFusion) {
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion);
+  EXPECT_TRUE(IsReduceInputFusion(*reduce));
   EXPECT_TRUE(IsInputFusibleReduction(*reduce));
 }
 
-TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputLoopReduceFusion) {
+TEST_F(GpuFusibleTest, IsReduceInputFusion_SingleOutputLoopReduceFusion) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     fused_reduction {
       c0 = f32[] parameter(0)
@@ -243,10 +246,11 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_SingleOutputLoopReduceFusion) {
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion);
+  EXPECT_FALSE(IsReduceInputFusion(*reduce));
   EXPECT_FALSE(IsInputFusibleReduction(*reduce));
 }
 
-TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputInputReduceFusion) {
+TEST_F(GpuFusibleTest, IsReduceInputFusion_MultiOutputInputReduceFusion) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     fused_reduction {
       c0 = f32[] parameter(0)
@@ -263,11 +267,12 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputInputReduceFusion) {
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion);
+  EXPECT_TRUE(IsReduceInputFusion(*reduce));
   EXPECT_TRUE(IsInputFusibleReduction(*reduce));
 }
 
 TEST_F(GpuFusibleTest,
-       IsInputFusibleReduction_MultiOutputInputReduceFusionWithExtraOutputs) {
+       IsReduceInputFusion_MultiOutputInputReduceFusionWithExtraOutputs) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     fused_reduction {
       c0 = f32[] parameter(0)
@@ -284,10 +289,11 @@ TEST_F(GpuFusibleTest,
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion);
+  EXPECT_TRUE(IsReduceInputFusion(*reduce));
   EXPECT_TRUE(IsInputFusibleReduction(*reduce));
 }
 
-TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputLoopReduceFusion) {
+TEST_F(GpuFusibleTest, IsReduceInputFusion_MultiOutputLoopReduceFusion) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     fused_reduction {
       c0 = f32[] parameter(0)
@@ -304,11 +310,12 @@ TEST_F(GpuFusibleTest, IsInputFusibleReduction_MultiOutputLoopReduceFusion) {
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion);
+  EXPECT_FALSE(IsReduceInputFusion(*reduce));
   EXPECT_FALSE(IsInputFusibleReduction(*reduce));
 }
 
 TEST_F(GpuFusibleTest,
-       IsInputFusibleReduction_MultiOutputLoopFusionReduceAndElementwiseOp) {
+       IsReduceInputFusion_MultiOutputLoopFusionReduceAndElementwiseOp) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
     fused_reduction {
       c0 = f32[] parameter(0)
@@ -325,6 +332,7 @@ TEST_F(GpuFusibleTest,
   const HloInstruction* reduce =
       module->entry_computation()->root_instruction();
   ASSERT_EQ(reduce->opcode(), HloOpcode::kFusion);
+  EXPECT_FALSE(IsReduceInputFusion(*reduce));
   EXPECT_FALSE(IsInputFusibleReduction(*reduce));
 }
 
-- 
GitLab


From 434dbe38970ffc90a5b546780be702e0b5de9a0c Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 21 Nov 2018 05:19:07 -0800
Subject: [PATCH 0657/1554] Fix undefined behavior caused by integer overflow
 in custom float comparison.

PiperOrigin-RevId: 222393257
---
 tensorflow/compiler/xla/client/lib/BUILD         |  3 ---
 .../xla/service/cpu/runtime_key_value_sort.cc    | 16 +++++++++-------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 45a7b020b3..f8dce0db9f 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -190,9 +190,6 @@ cc_library(
 xla_test(
     name = "sorting_test",
     srcs = ["sorting_test.cc"],
-    blacklisted_backends = [
-        "cpu",  # Fails with --config=asan.
-    ],
     tags = ["enable_for_xla_interpreter"],
     deps = [
         ":sorting",
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
index c7fc101ec7..722aa3120e 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
@@ -51,19 +51,21 @@ void KeyValueSort(std::pair<KeyType, int64>* row_to_sort, int64 num_elements) {
 // then y is ordered as an int32 such that finite values have the
 // obvious order, -0 is ordered before 0, and -NaN and NaN appear at
 // the beginning and end of the ordering.
-template <typename CastType, typename KeyType>
+template <typename CastType, typename UnsignedCastType, typename KeyType>
 CastType Convert(KeyType value) {
   CastType casted_value;
   memcpy(&casted_value, &value, sizeof(CastType));
   if (casted_value < 0) {
-    return std::numeric_limits<CastType>::max() - casted_value;
+    return static_cast<UnsignedCastType>(std::numeric_limits<CastType>::max()) -
+           casted_value;
   }
   return casted_value;
 }
 
-template <typename CastType, typename KeyType>
+template <typename CastType, typename UnsignedCastType, typename KeyType>
 bool LessThan(KeyType lhs, KeyType rhs) {
-  return Convert<CastType>(lhs) < Convert<CastType>(rhs);
+  return Convert<CastType, UnsignedCastType>(lhs) <
+         Convert<CastType, UnsignedCastType>(rhs);
 }
 
 template <>
@@ -71,7 +73,7 @@ void KeyValueSort(std::pair<double, int64>* row_to_sort, int64 num_elements) {
   std::stable_sort(row_to_sort, row_to_sort + num_elements,
                    [](const std::pair<double, int64>& lhs,
                       const std::pair<double, int64>& rhs) -> bool {
-                     return LessThan<int64>(lhs.first, rhs.first);
+                     return LessThan<int64, uint64>(lhs.first, rhs.first);
                    });
 }
 
@@ -80,7 +82,7 @@ void KeyValueSort(std::pair<float, int64>* row_to_sort, int64 num_elements) {
   std::stable_sort(row_to_sort, row_to_sort + num_elements,
                    [](const std::pair<float, int64>& lhs,
                       const std::pair<float, int64>& rhs) -> bool {
-                     return LessThan<int32>(lhs.first, rhs.first);
+                     return LessThan<int32, uint32>(lhs.first, rhs.first);
                    });
 }
 
@@ -90,7 +92,7 @@ void KeyValueSort(std::pair<Eigen::half, int64>* row_to_sort,
   std::stable_sort(row_to_sort, row_to_sort + num_elements,
                    [](const std::pair<Eigen::half, int64>& lhs,
                       const std::pair<Eigen::half, int64>& rhs) -> bool {
-                     return LessThan<int32>(
+                     return LessThan<int32, uint32>(
                          Eigen::half_impl::half_to_float(lhs.first),
                          Eigen::half_impl::half_to_float(rhs.first));
                    });
-- 
GitLab


From e978c676341c3b047f90c18de74c15be24dd09f6 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 21 Nov 2018 05:59:55 -0800
Subject: [PATCH 0658/1554] Add deprecation warning for `tf_record_iterator`

PiperOrigin-RevId: 222396408
---
 tensorflow/python/lib/io/tf_record.py              | 9 +++++----
 tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt | 4 ----
 tensorflow/tools/compatibility/renames_v2.py       | 3 ++-
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py
index b7fae85295..43086ab18d 100644
--- a/tensorflow/python/lib/io/tf_record.py
+++ b/tensorflow/python/lib/io/tf_record.py
@@ -150,10 +150,11 @@ class TFRecordOptions(object):
     return options
 
 
-@tf_export(
-    "io.tf_record_iterator",
-    v1=["io.tf_record_iterator", "python_io.tf_record_iterator"])
-@deprecation.deprecated_endpoints("python_io.tf_record_iterator")
+@tf_export(v1=["io.tf_record_iterator", "python_io.tf_record_iterator"])
+@deprecation.deprecated(
+    date=None,
+    instructions=("Use eager execution and: \n"
+                  "`tf.data.TFRecordDataset(path)`"))
 def tf_record_iterator(path, options=None):
   """An iterator that read the records from a TFRecords file.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
index b27df178b5..caa207b022 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
@@ -120,10 +120,6 @@ tf_module {
     name: "serialize_tensor"
     argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "tf_record_iterator"
-    argspec: "args=[\'path\', \'options\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "write_file"
     argspec: "args=[\'filename\', \'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index f002418020..d71dce42b1 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -211,6 +211,7 @@ renames = {
     'tf.initializers.tables_initializer': 'tf.compat.v1.initializers.tables_initializer',
     'tf.initializers.variables': 'tf.compat.v1.initializers.variables',
     'tf.invert_permutation': 'tf.math.invert_permutation',
+    'tf.io.tf_record_iterator': 'tf.compat.v1.io.tf_record_iterator',
     'tf.is_finite': 'tf.math.is_finite',
     'tf.is_inf': 'tf.math.is_inf',
     'tf.is_nan': 'tf.math.is_nan',
@@ -350,7 +351,7 @@ renames = {
     'tf.python_io.TFRecordCompressionType': 'tf.io.TFRecordCompressionType',
     'tf.python_io.TFRecordOptions': 'tf.io.TFRecordOptions',
     'tf.python_io.TFRecordWriter': 'tf.io.TFRecordWriter',
-    'tf.python_io.tf_record_iterator': 'tf.io.tf_record_iterator',
+    'tf.python_io.tf_record_iterator': 'tf.compat.v1.python_io.tf_record_iterator',
     'tf.qr': 'tf.linalg.qr',
     'tf.quantize': 'tf.quantization.quantize',
     'tf.quantize_v2': 'tf.compat.v1.quantize_v2',
-- 
GitLab


From 3ff69a2fca5f16ba9b9e9b35da546f6066ec19c7 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 21 Nov 2018 06:29:04 -0800
Subject: [PATCH 0659/1554] Use stable_sort instead of sort in HloEvaluator
 implementation of the sort op.

We need to guarantee stable sorting when used by TopK.

PiperOrigin-RevId: 222399261
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc          | 8 ++++----
 .../compiler/xla/service/hlo_evaluator_typed_visitor.h    | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 7fcafafc09..9783f0574f 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1279,10 +1279,10 @@ StatusOr<Literal> EvaluateSortInternal(HloInstruction* sort,
           key_value_vector.push_back(
               std::make_pair(keys_data[i], values_data[i]));
         }
-        std::sort(key_value_vector.begin(), key_value_vector.end(),
-                  [](const kv_pair& a, const kv_pair& b) {
-                    return SafeLess<KeyType>(a.first, b.first);
-                  });
+        std::stable_sort(key_value_vector.begin(), key_value_vector.end(),
+                         [](const kv_pair& a, const kv_pair& b) {
+                           return SafeLess<KeyType>(a.first, b.first);
+                         });
         std::vector<KeyType> result_keys;
         // We use a InlinedVector here because we need to convert it to an
         // absl::Span later, and this would not work with std::vector<bool>.
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index f975ba63bb..b87fc3e340 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1553,10 +1553,10 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
           const auto& row_data = row_to_sort.data<NativeT>();
 
           std::vector<NativeT> result_data(row_data.begin(), row_data.end());
-          std::sort(result_data.begin(), result_data.end(),
-                    [](const NativeT& a, const NativeT& b) {
-                      return SafeLess<NativeT>(a, b);
-                    });
+          std::stable_sort(result_data.begin(), result_data.end(),
+                           [](const NativeT& a, const NativeT& b) {
+                             return SafeLess<NativeT>(a, b);
+                           });
           Literal sorted_row(ShapeUtil::MakeShape(keys->shape().element_type(),
                                                   {sort_dim_elements}));
           sorted_row.PopulateR1(absl::Span<const NativeT>(result_data));
-- 
GitLab


From 7c828171d40ed9019aeefca9d1ad783c6be66d5e Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 21 Nov 2018 07:01:44 -0800
Subject: [PATCH 0660/1554] Change API for nn.softmax and nn.log_softmax for TF
 2.0.

PiperOrigin-RevId: 222402226
---
 tensorflow/python/ops/nn_ops.py               | 54 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.math.pbtxt |  4 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  6 +++
 4 files changed, 62 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 21008fc392..755c8ffcd2 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2118,7 +2118,7 @@ def _softmax(logits, compute_op, dim=-1, name=None):
   return output
 
 
-@tf_export("nn.softmax", "math.softmax")
+@tf_export(v1=["nn.softmax", "math.softmax"])
 @deprecation.deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def softmax(logits, axis=None, name=None, dim=None):
   """Computes softmax activations.
@@ -2148,7 +2148,32 @@ def softmax(logits, axis=None, name=None, dim=None):
   return _softmax(logits, gen_nn_ops.softmax, axis, name)
 
 
-@tf_export("nn.log_softmax", "math.log_softmax")
+@tf_export("nn.softmax", "math.softmax", v1=[])
+def softmax_v2(logits, axis=None, name=None):
+  """Computes softmax activations.
+
+  This function performs the equivalent of
+
+      softmax = tf.exp(logits) / tf.reduce_sum(tf.exp(logits), axis)
+
+  Args:
+    logits: A non-empty `Tensor`. Must be one of the following types: `half`,
+      `float32`, `float64`.
+    axis: The dimension softmax would be performed on. The default is -1 which
+      indicates the last dimension.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type and shape as `logits`.
+
+  Raises:
+    InvalidArgumentError: if `logits` is empty or `axis` is beyond the last
+      dimension of `logits`.
+  """
+  return _softmax(logits, gen_nn_ops.softmax, axis, name)
+
+
+@tf_export(v1=["nn.log_softmax", "math.log_softmax"])
 @deprecation.deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def log_softmax(logits, axis=None, name=None, dim=None):
   """Computes log softmax activations.
@@ -2178,6 +2203,31 @@ def log_softmax(logits, axis=None, name=None, dim=None):
   return _softmax(logits, gen_nn_ops.log_softmax, axis, name)
 
 
+@tf_export("nn.log_softmax", "math.log_softmax", v1=[])
+def log_softmax_v2(logits, axis=None, name=None):
+  """Computes log softmax activations.
+
+  For each batch `i` and class `j` we have
+
+      logsoftmax = logits - log(reduce_sum(exp(logits), axis))
+
+  Args:
+    logits: A non-empty `Tensor`. Must be one of the following types: `half`,
+      `float32`, `float64`.
+    axis: The dimension softmax would be performed on. The default is -1 which
+      indicates the last dimension.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `logits`. Same shape as `logits`.
+
+  Raises:
+    InvalidArgumentError: if `logits` is empty or `axis` is beyond the last
+      dimension of `logits`.
+  """
+  return _softmax(logits, gen_nn_ops.log_softmax, axis, name)
+
+
 def _ensure_xent_args(name, sentinel, labels, logits):
   # Make sure that all arguments were passed as named arguments.
   if sentinel is not None:
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index e6b8fd225d..86df970514 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -230,7 +230,7 @@ tf_module {
   }
   member_method {
     name: "log_softmax"
-    argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "logical_and"
@@ -382,7 +382,7 @@ tf_module {
   }
   member_method {
     name: "softmax"
-    argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "softplus"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 19380337d8..e550b2d754 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -190,7 +190,7 @@ tf_module {
   }
   member_method {
     name: "log_softmax"
-    argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "lrn"
@@ -262,7 +262,7 @@ tf_module {
   }
   member_method {
     name: "softmax"
-    argspec: "args=[\'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "softmax_cross_entropy_with_logits"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 0caf1743d7..a4d307032d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -60,6 +60,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
         },
+        "tf.nn.log_softmax": {
+            "dim": "axis",
+        },
+        "tf.nn.softmax": {
+            "dim": "axis",
+        },
         "tf.debugging.assert_all_finite": {
             "t": "x",
             "msg": "message",
-- 
GitLab


From 2893986ce5ed2af52d2f2a72a093cab3734a9bab Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 21 Nov 2018 07:14:12 -0800
Subject: [PATCH 0661/1554] Remove Iota declaration from numeric.h

The same declaration exists in xla_builder.h, and it is also implemented in
xla_builder.cc

Remove the header include of numeric.h where it was only used to get access to
Iota. Add the xla_builder.h header instead if it was not already included.

PiperOrigin-RevId: 222403720
---
 tensorflow/compiler/tf2xla/BUILD                               | 1 -
 tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc          | 1 -
 tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc | 1 -
 tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc      | 1 -
 tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc       | 1 -
 tensorflow/compiler/tf2xla/kernels/random_ops.cc               | 1 -
 tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc      | 1 -
 tensorflow/compiler/tf2xla/kernels/sequence_ops.cc             | 2 +-
 tensorflow/compiler/tf2xla/xla_helpers.cc                      | 1 -
 tensorflow/compiler/xla/client/lib/BUILD                       | 2 --
 tensorflow/compiler/xla/client/lib/numeric.h                   | 3 ---
 tensorflow/compiler/xla/client/lib/prng.cc                     | 1 -
 tensorflow/compiler/xla/client/lib/sorting.cc                  | 2 +-
 13 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 486b4d8a8c..3458c7f1c4 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -211,7 +211,6 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/client/lib:arithmetic",
         "//tensorflow/compiler/xla/client/lib:constants",
-        "//tensorflow/compiler/xla/client/lib:numeric",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index c9a1be4940..b1046fcc00 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/core/framework/node_def_util.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc b/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc
index c68b0bfd79..29687c7b82 100644
--- a/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/extract_image_patches_op.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/util/tensor_format.h"
 
diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc
index 8dfd7de591..a99b74565d 100644
--- a/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 
diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc
index c0ca881ff8..4f980b6d14 100644
--- a/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matrix_set_diag_op.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 
 namespace tensorflow {
diff --git a/tensorflow/compiler/tf2xla/kernels/random_ops.cc b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
index 415ce9b77f..8822e29f7e 100644
--- a/tensorflow/compiler/tf2xla/kernels/random_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc b/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc
index 7ff3e91638..d7b38e86cc 100644
--- a/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reverse_sequence_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/tensor_shape.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc b/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc
index 60b011ba6d..b1fa2915d5 100644
--- a/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/sequence_ops.cc
@@ -18,7 +18,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/core/framework/op_kernel.h"
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc
index 9a34cd8c6a..af378bc95c 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.cc
+++ b/tensorflow/compiler/tf2xla/xla_helpers.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/types.h"
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index f8dce0db9f..c5733bc66d 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -164,7 +164,6 @@ cc_library(
     deps = [
         ":constants",
         ":math",
-        ":numeric",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -178,7 +177,6 @@ cc_library(
     srcs = ["sorting.cc"],
     hdrs = ["sorting.h"],
     deps = [
-        ":numeric",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
diff --git a/tensorflow/compiler/xla/client/lib/numeric.h b/tensorflow/compiler/xla/client/lib/numeric.h
index efd8cdc257..f62fdab4b0 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.h
+++ b/tensorflow/compiler/xla/client/lib/numeric.h
@@ -22,9 +22,6 @@ limitations under the License.
 
 namespace xla {
 
-// Returns a rank 1 tensor of `type` containing values [0, 1, 2, ...].
-XlaOp Iota(XlaBuilder* builder, PrimitiveType type, int64 size);
-
 // Returns an m x n matrix with 1s on the diagonal elements, zeros everywhere
 // else.
 XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, int64 n);
diff --git a/tensorflow/compiler/xla/client/lib/prng.cc b/tensorflow/compiler/xla/client/lib/prng.cc
index c6f68c8ee2..85b9e1827d 100644
--- a/tensorflow/compiler/xla/client/lib/prng.cc
+++ b/tensorflow/compiler/xla/client/lib/prng.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include "absl/base/casts.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/util.h"
 
diff --git a/tensorflow/compiler/xla/client/lib/sorting.cc b/tensorflow/compiler/xla/client/lib/sorting.cc
index d3980d5575..e8553a08bb 100644
--- a/tensorflow/compiler/xla/client/lib/sorting.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting.cc
@@ -14,7 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/client/lib/sorting.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/util.h"
 
-- 
GitLab


From 944efe54b6f566f5127e5122193b55db28ff9b4f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 07:40:44 -0800
Subject: [PATCH 0662/1554] Remove min_max_variable_partitioner from TF 2.0 API

PiperOrigin-RevId: 222406576
---
 tensorflow/python/ops/partitioned_variables.py  | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 tensorflow/tools/compatibility/renames_v2.py    | 1 +
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py
index 7743b634e8..6174979d5e 100644
--- a/tensorflow/python/ops/partitioned_variables.py
+++ b/tensorflow/python/ops/partitioned_variables.py
@@ -154,7 +154,7 @@ def variable_axis_size_partitioner(
   return _partitioner
 
 
-@tf_export("min_max_variable_partitioner")
+@tf_export(v1=["min_max_variable_partitioner"])
 def min_max_variable_partitioner(max_partitions=1, axis=0,
                                  min_slice_size=256 << 10,
                                  bytes_per_string_element=16):
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index ec2d72290e..c91aa43671 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -820,10 +820,6 @@ tf_module {
     name: "meshgrid"
     argspec: "args=[], varargs=args, keywords=kwargs, defaults=None"
   }
-  member_method {
-    name: "min_max_variable_partitioner"
-    argspec: "args=[\'max_partitions\', \'axis\', \'min_slice_size\', \'bytes_per_string_element\'], varargs=None, keywords=None, defaults=[\'1\', \'0\', \'262144\', \'16\'], "
-  }
   member_method {
     name: "minimum"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index d71dce42b1..088611dc6c 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -306,6 +306,7 @@ renames = {
     'tf.matrix_solve_ls': 'tf.linalg.lstsq',
     'tf.matrix_transpose': 'tf.linalg.transpose',
     'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
+    'tf.min_max_variable_partitioner': 'tf.compat.v1.min_max_variable_partitioner',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
     'tf.multinomial': 'tf.compat.v1.multinomial',
-- 
GitLab


From c20b4ba1e6d25368e08b9a36ee348f0864734a82 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 08:25:02 -0800
Subject: [PATCH 0663/1554] batch_to_space changes for TF 2.0 API

This change renames batch_to_space_nd to batch_to_space in TF 2.0. Arguments
are re-ordered and the the block_size int argument has been to block_shape of
type Tensor. Users may still use an int block_shape, for compatibility purposes
those are silently converted to a Tensor.

PiperOrigin-RevId: 222411473
---
 .../python_api/api_def_BatchToSpaceND.pbtxt   |   2 +
 tensorflow/python/ops/array_ops.py            | 147 +++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |   3 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  10 ++
 5 files changed, 160 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_BatchToSpaceND.pbtxt b/tensorflow/core/api_def/python_api/api_def_BatchToSpaceND.pbtxt
index 801dfbc285..94ffc7c068 100644
--- a/tensorflow/core/api_def/python_api/api_def_BatchToSpaceND.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_BatchToSpaceND.pbtxt
@@ -1,7 +1,9 @@
 op {
   graph_op_name: "BatchToSpaceND"
+  deprecation_message: "use batch_to_space"
   endpoint {
     name: "batch_to_space_nd"
+    deprecation_version: 2
   }
   endpoint {
     name: "manip.batch_to_space_nd"
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index ed050d740e..0f80a28d7f 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2675,7 +2675,7 @@ def depth_to_space(input, block_size, name=None, data_format="NHWC"):  # pylint:
 depth_to_space.__doc__ = gen_array_ops.depth_to_space.__doc__
 
 
-@tf_export("batch_to_space")
+@tf_export(v1=["batch_to_space"])
 def batch_to_space(input, crops, block_size, name=None):  # pylint: disable=redefined-builtin
   result = batch_to_space_nd(
       input,
@@ -2689,6 +2689,151 @@ def batch_to_space(input, crops, block_size, name=None):  # pylint: disable=rede
 batch_to_space.__doc__ = gen_array_ops.batch_to_space.__doc__
 
 
+@tf_export("batch_to_space", v1=[])
+def batch_to_space_v2(input, block_shape, crops, name=None):  # pylint: disable=redefined-builtin
+  """BatchToSpace for N-D tensors of type T.
+
+  This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of
+  shape `block_shape + [batch]`, interleaves these blocks back into the grid
+  defined by the spatial dimensions `[1, ..., M]`, to obtain a result with the
+  same rank as the input.  The spatial dimensions of this intermediate result
+  are then optionally cropped according to `crops` to produce the output.  This
+  is the reverse of SpaceToBatch.  See below for a precise description.
+
+  Args:
+    input: A `Tensor`.
+      N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
+      where spatial_shape has M dimensions.
+    block_shape: A `Tensor`. Must be one of the following types:
+      `int32`, `int64`. 1-D with shape `[M]`, all values must be >= 1.
+      For backwards compatibility with TF 1.0, this parameter may be an int, in
+      which case it is converted to
+      `numpy.array([block_shape, block_shape], dtype=numpy.int64)`.
+    crops: A `Tensor`. Must be one of the following types: `int32`, `int64`.
+      2-D with shape `[M, 2]`, all values must be >= 0.
+        `crops[i] = [crop_start, crop_end]` specifies the amount to crop from
+        input dimension `i + 1`, which corresponds to spatial dimension `i`.  It
+        is required that
+        `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
+
+      This operation is equivalent to the following steps:
+
+      1. Reshape `input` to `reshaped` of shape:
+           [block_shape[0], ..., block_shape[M-1],
+            batch / prod(block_shape),
+            input_shape[1], ..., input_shape[N-1]]
+
+      2. Permute dimensions of `reshaped` to produce `permuted` of shape
+           [batch / prod(block_shape),
+
+            input_shape[1], block_shape[0],
+            ...,
+            input_shape[M], block_shape[M-1],
+
+            input_shape[M+1], ..., input_shape[N-1]]
+
+      3. Reshape `permuted` to produce `reshaped_permuted` of shape
+           [batch / prod(block_shape),
+
+            input_shape[1] * block_shape[0],
+            ...,
+            input_shape[M] * block_shape[M-1],
+
+            input_shape[M+1],
+            ...,
+            input_shape[N-1]]
+
+      4. Crop the start and end of dimensions `[1, ..., M]` of
+         `reshaped_permuted` according to `crops` to produce the
+         output of shape:
+           [batch / prod(block_shape),
+
+            input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
+            ...,
+            input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
+
+            input_shape[M+1], ..., input_shape[N-1]]
+
+      Some examples:
+
+      (1) For the following input of shape `[4, 1, 1, 1]`,
+          `block_shape = [2, 2]`, and `crops = [[0, 0], [0, 0]]`:
+
+      ```
+      [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+      ```
+
+      The output tensor has shape `[1, 2, 2, 1]` and value:
+
+      ```
+      x = [[[[1], [2]], [[3], [4]]]]
+      ```
+
+      (2) For the following input of shape `[4, 1, 1, 3]`,
+          `block_shape = [2, 2]`, and `crops = [[0, 0], [0, 0]]`:
+
+      ```
+      [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+      ```
+
+      The output tensor has shape `[1, 2, 2, 3]` and value:
+
+      ```
+      x = [[[[1, 2, 3], [4, 5, 6]],
+            [[7, 8, 9], [10, 11, 12]]]]
+      ```
+
+      (3) For the following input of shape `[4, 2, 2, 1]`,
+          `block_shape = [2, 2]`, and `crops = [[0, 0], [0, 0]]`:
+
+      ```
+      x = [[[[1], [3]], [[9], [11]]],
+           [[[2], [4]], [[10], [12]]],
+           [[[5], [7]], [[13], [15]]],
+           [[[6], [8]], [[14], [16]]]]
+      ```
+
+      The output tensor has shape `[1, 4, 4, 1]` and value:
+
+      ```
+      x = [[[1],   [2],  [3],  [4]],
+           [[5],   [6],  [7],  [8]],
+           [[9],  [10], [11],  [12]],
+           [[13], [14], [15],  [16]]]
+      ```
+
+      (4) For the following input of shape `[8, 1, 3, 1]`,
+          `block_shape = [2, 2]`, and `crops = [[0, 0], [2, 0]]`:
+
+      ```
+      x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
+           [[[0], [2], [4]]], [[[0], [10], [12]]],
+           [[[0], [5], [7]]], [[[0], [13], [15]]],
+           [[[0], [6], [8]]], [[[0], [14], [16]]]]
+      ```
+
+      The output tensor has shape `[2, 2, 4, 1]` and value:
+
+      ```
+      x = [[[[1],   [2],  [3],  [4]],
+            [[5],   [6],  [7],  [8]]],
+           [[[9],  [10], [11],  [12]],
+            [[13], [14], [15],  [16]]]]
+      ```
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  if isinstance(block_shape, int):
+    block_shape = np.array([block_shape, block_shape], dtype=np.int64)
+
+  return batch_to_space_nd(input=input,
+                           block_shape=block_shape,
+                           crops=crops,
+                           name=name)
+
+
 @tf_export("one_hot")
 def one_hot(indices,
             depth,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index c91aa43671..0649ae267b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -510,10 +510,6 @@ tf_module {
   }
   member_method {
     name: "batch_to_space"
-    argspec: "args=[\'input\', \'crops\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "batch_to_space_nd"
     argspec: "args=[\'input\', \'block_shape\', \'crops\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 088611dc6c..a93b7d68bd 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -92,6 +92,7 @@ renames = {
     'tf.assign': 'tf.compat.v1.assign',
     'tf.assign_add': 'tf.compat.v1.assign_add',
     'tf.assign_sub': 'tf.compat.v1.assign_sub',
+    'tf.batch_to_space_nd': 'tf.compat.v1.batch_to_space_nd',
     'tf.betainc': 'tf.math.betainc',
     'tf.bincount': 'tf.math.bincount',
     'tf.ceil': 'tf.math.ceil',
@@ -287,7 +288,7 @@ renames = {
     'tf.logical_xor': 'tf.math.logical_xor',
     'tf.make_template': 'tf.compat.v1.make_template',
     'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto',
-    'tf.manip.batch_to_space_nd': 'tf.batch_to_space_nd',
+    'tf.manip.batch_to_space_nd': 'tf.compat.v1.manip.batch_to_space_nd',
     'tf.manip.gather_nd': 'tf.gather_nd',
     'tf.manip.reshape': 'tf.reshape',
     'tf.manip.reverse': 'tf.reverse',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index a4d307032d..f85b2f08dc 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -43,6 +43,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.expand_dims": {
             "dim": "axis",
         },
+        "tf.batch_to_space_nd": {
+            "block_size": "block_shape",
+        },
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
@@ -84,6 +87,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "m": "mean",
             "v": "variance",
         },
+        "tf.manip.batch_to_space_nd": {
+            "block_size": "block_shape",
+        },
         "tf.nn.conv3d": {
             "filter": "filters"
         },
@@ -117,6 +123,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # function_reorders or function_keyword_renames, use the OLD function name.
     # These renames happen after the arguments have been processed.
     self.symbol_renames.update({
+        "tf.batch_to_space_nd": "tf.batch_to_space",
         "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
         "tf.contrib.data.Counter": "tf.data.experimental.Counter",
         "tf.contrib.data.CheckpointInputPipelineHook": "tf.data.experimental.CheckpointInputPipelineHook",
@@ -161,6 +168,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.contrib.data.unique": "tf.data.experimental.unique",
         "tf.contrib.framework.sort": "tf.sort",
         "tf.contrib.framework.argsort": "tf.argsort",
+        "tf.manip.batch_to_space_nd": "tf.batch_to_space",
         "tf.quantize_v2": "tf.quantization.quantize",
         "tf.sparse_concat": "tf.sparse.concat",
         "tf.sparse_split": "tf.sparse.split",
@@ -186,6 +194,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.function_reorders = {
         "tf.argmax": ["input", "axis", "name", "dimension", "output_type"],
         "tf.argmin": ["input", "axis", "name", "dimension", "output_type"],
+        "tf.batch_to_space": ["input", "crops", "block_size", "name"],
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
         "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
         "tf.nn.convolution": [
@@ -200,6 +209,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "filter", "strides", "padding", "rate", "name",
             "data_format"
         ],
+        "tf.manip.batch_to_space_nd": ["input", "crops", "block_size", "name"],
         "tf.multinomial": [
             "logits", "num_samples", "seed", "name", "output_dtype"
         ],
-- 
GitLab


From 7b10cfd70094f619508f18b40c1fa60ab8d86083 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Wed, 21 Nov 2018 09:09:59 -0800
Subject: [PATCH 0664/1554] Aligned the reuse behaviour of plain and
 partitioned variables in eager

Previously, plain variables were only stored if EagerVariableStore was
active, whereas partitioned variables were always stored.

PiperOrigin-RevId: 222417188
---
 tensorflow/python/kernel_tests/variable_scope_test.py | 8 ++++++++
 tensorflow/python/ops/variable_scope.py               | 6 ++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 838838e0ac..de086860ea 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -1404,6 +1404,14 @@ class VariableScopeWithPartitioningTest(test.TestCase):
       v_reused = variable_scope.get_variable("name0")
     self.assertEqual(v, v_reused)
 
+  def testNoReuseInEagerByDefault(self):
+    with context.eager_mode():
+      with variable_scope.variable_scope(
+          "scope0", partitioner=axis0_into2_partitioner):
+        v1 = variable_scope.get_variable("name0", shape=(3, 1, 1))
+        v2 = variable_scope.get_variable("name0", shape=(3, 1, 1))
+        self.assertIsNot(v1, v2)
+
   @test_util.run_in_graph_and_eager_modes
   @run_inside_wrap_function_in_eager_mode
   def testPropagatePartitionerOnReopening(self):
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 077bb647ef..44bb62a5dc 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -799,15 +799,13 @@ class _VariableStore(object):
       vs.append(var)
       # pylint: enable=protected-access
 
-      # pylint: disable=protected-access
     partitioned_var = variables.PartitionedVariable(name=name,
                                                     shape=shape,
                                                     dtype=dtype,
                                                     variable_list=vs,
                                                     partitions=partitions)
-    # pylint: enable=protected-access
-
-    self._partitioned_vars[name] = partitioned_var
+    if not context.executing_eagerly() or self._store_eager_variables:
+      self._partitioned_vars[name] = partitioned_var
     return partitioned_var
 
   def _get_single_variable(self,
-- 
GitLab


From 9fe074be83373dd66f1711e7267da37b05341327 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 21 Nov 2018 09:11:24 -0800
Subject: [PATCH 0665/1554] Removes spurious cond name scope in contrib
 metrics.

PiperOrigin-RevId: 222417340
---
 tensorflow/contrib/eager/python/metrics_impl.py |  6 ++++--
 tensorflow/contrib/eager/python/metrics_test.py | 12 ------------
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index c88c0f52ee..566246de49 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -24,6 +24,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import smart_cond
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
@@ -354,9 +355,10 @@ class Mean(Metric):
     def write_summary_f():
       summary_ops.scalar(name=self.name, tensor=t)
       return t
-    control_flow_ops.cond(write_summary,
+    smart_cond.smart_cond(write_summary,
                           write_summary_f,
-                          lambda: t)
+                          lambda: t,
+                          name="")
     return t
 
 
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 9d2d172752..39e5957f5d 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -49,18 +49,6 @@ class MetricsTest(test.TestCase):
     self.assertEqual(dtypes.float64, m.dtype)
     self.assertEqual(dtypes.float64, m.result().dtype)
 
-  def testSummaryArg(self):
-    m = metrics.Mean()
-    m([1, 10, 100])
-    m(1000)
-    m([10000.0, 100000.0])
-    self.assertEqual(111111.0/6, m.result(write_summary=True).numpy())
-    self.assertEqual(111111.0/6, m.result(write_summary=False).numpy())
-    with self.assertRaises(ValueError):
-      m.result(write_summary=5)
-    with self.assertRaises(ValueError):
-      m.result(write_summary=[True])
-
   def testVariableCollections(self):
     with context.graph_mode(), ops.Graph().as_default():
       m = metrics.Mean()
-- 
GitLab


From 80f3b787e485d0d7dc775412d981a09e95090b78 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 09:19:22 -0800
Subject: [PATCH 0666/1554] Fix a typo

PiperOrigin-RevId: 222418257
---
 tensorflow/python/eager/def_function.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 52830d41bf..6bacd7a962 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -552,9 +552,9 @@ def function(func=None,
     return x + tf.to_float(c)
 
   assert int(c) == 0
-  assert f(1.0) == 3.0
+  assert f(1.0) == 2.0
   assert int(c) == 1
-  assert f(1.0) == 4.0
+  assert f(1.0) == 3.0
   assert int(c) == 2
   ```
 
-- 
GitLab


From 4f92a46fa8731fc7883888bce3f8a081293f8345 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 21 Nov 2018 09:43:16 -0800
Subject: [PATCH 0667/1554] Automated rollback of commit
 fd46ffb7bd04d0959294a09ae8a9d2cbd74a48ca

PiperOrigin-RevId: 222421495
---
 tensorflow/python/data/experimental/ops/BUILD |   2 +-
 .../python/data/experimental/ops/grouping.py  |  16 +-
 tensorflow/python/data/kernel_tests/BUILD     |   4 +
 .../data/kernel_tests/dataset_ops_test.py     |  61 ++++
 tensorflow/python/data/ops/dataset_ops.py     | 260 +++++++-----------
 tensorflow/python/data/ops/optional_ops.py    |  12 +-
 tensorflow/python/data/util/structure.py      |  26 +-
 tensorflow/python/data/util/structure_test.py |   9 +-
 8 files changed, 202 insertions(+), 188 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 170fda90b6..b6c1376b6a 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -165,7 +165,7 @@ py_library(
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index 80ca7104d8..db10ea3b7f 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -21,6 +21,7 @@ import numpy as np
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -448,7 +449,10 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
 
   def _make_reduce_func(self, reduce_func, input_dataset):
     """Make wrapping defun for reduce_func."""
-    nested_dataset = dataset_ops._NestedDatasetComponent(input_dataset)  # pylint: disable=protected-access
+    nested_dataset = dataset_ops.DatasetStructure(
+        structure.Structure._from_legacy_structure(  # pylint: disable=protected-access
+            input_dataset.output_types, input_dataset.output_shapes,
+            input_dataset.output_classes))
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
         reduce_func,
         self._transformation_name(),
@@ -456,11 +460,13 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
         input_shapes=(tensor_shape.scalar(), nested_dataset),
         input_types=(dtypes.int64, nested_dataset))
     if not isinstance(
-        wrapped_func.output_classes, dataset_ops._NestedDatasetComponent):  # pylint: disable=protected-access
+        wrapped_func.output_structure, dataset_ops.DatasetStructure):
       raise TypeError("`reduce_func` must return a `Dataset` object.")
-    self._output_classes = wrapped_func.output_classes.output_classes
-    self._output_types = wrapped_func.output_types.output_types
-    self._output_shapes = wrapped_func.output_shapes.output_shapes
+    # pylint: disable=protected-access
+    element_structure = wrapped_func.output_structure._element_structure
+    self._output_classes = element_structure._to_legacy_output_classes()
+    self._output_types = element_structure._to_legacy_output_types()
+    self._output_shapes = element_structure._to_legacy_output_shapes()
     self._reduce_func = wrapped_func.function
 
   @property
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 21eed2b070..fa1f6d701a 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -117,8 +117,12 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:optional_ops",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
index a5324af4d0..1f22a37c2e 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
@@ -24,10 +24,14 @@ import numpy as np
 from tensorflow.core.framework import graph_pb2
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import optional_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.platform import test
 
 
@@ -249,6 +253,63 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertTrue(ds.options().experimental_autotune)
     self.assertTrue(ds.options().experimental_filter_fusion)
 
+  # pylint: disable=g-long-lambda
+  @parameterized.named_parameters(
+      ("Tensor", lambda: constant_op.constant(37.0),
+       structure.TensorStructure(dtypes.float32, [])),
+      ("SparseTensor", lambda: sparse_tensor.SparseTensor(
+          indices=[[0]], values=constant_op.constant([0], dtype=dtypes.int32),
+          dense_shape=[1]),
+       structure.SparseTensorStructure(dtypes.int32, [1])),
+      ("Nest", lambda: {
+          "a": constant_op.constant(37.0),
+          "b": (constant_op.constant(["Foo"]), constant_op.constant("Bar"))},
+       structure.NestedStructure({
+           "a": structure.TensorStructure(dtypes.float32, []),
+           "b": (structure.TensorStructure(dtypes.string, [1]),
+                 structure.TensorStructure(dtypes.string, []))})),
+      ("Dataset", lambda: dataset_ops.Dataset.from_tensor_slices(
+          constant_op.constant([1, 2, 3])),
+       dataset_ops.DatasetStructure(
+           structure.TensorStructure(dtypes.int32, []))),
+      ("Optional", lambda: optional_ops.Optional.from_value(37.0),
+       optional_ops.OptionalStructure(
+           structure.TensorStructure(dtypes.float32, []))),
+  )
+  def testDatasetStructure(self, tf_value_fn, expected_element_structure):
+    dataset = dataset_ops.Dataset.from_tensors(0).map(lambda _: tf_value_fn())
+    dataset_structure = structure.Structure.from_value(dataset)
+    self.assertIsInstance(dataset_structure, dataset_ops.DatasetStructure)
+
+    # TODO(b/110122868): Add a public API to `tf.data.Dataset` for accessing
+    # the element structure.
+    self.assertTrue(expected_element_structure.is_compatible_with(
+        dataset_structure._element_structure))
+    self.assertTrue(dataset_structure._element_structure.is_compatible_with(
+        expected_element_structure))
+
+    self.assertEqual([dtypes.variant], dataset_structure._flat_types)
+    self.assertEqual([tensor_shape.scalar()], dataset_structure._flat_shapes)
+
+    # Assert that the `Dataset` survives a round-trip via _from_tensor_list()
+    # and _to_tensor_list().
+    round_trip_dataset = dataset_structure._from_tensor_list(
+        dataset_structure._to_tensor_list(dataset))
+
+    value = tf_value_fn()
+
+    if isinstance(value, dataset_ops.Dataset):
+      self.assertDatasetsEqual(value, dataset.flat_map(lambda x: x))
+    elif isinstance(value, optional_ops.Optional):
+      self.assertDatasetProduces(
+          round_trip_dataset.map(lambda opt: opt.get_value()),
+          [self.evaluate(value.get_value())],
+          requires_initialization=True)
+    else:
+      self.assertDatasetProduces(
+          round_trip_dataset, [self.evaluate(tf_value_fn())],
+          requires_initialization=True)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 4a11619112..5c0cfe994d 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -31,6 +31,7 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import random_seed
 from tensorflow.python.data.util import sparse
+from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -1868,57 +1869,6 @@ class SparseTensorSliceDataset(DatasetSource):
     return (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64)
 
 
-class _NestedDatasetComponent(object):
-  """The structure of a `Dataset` nested in a component of another `Dataset`.
-
-  A `StructuredFunctionWrapper` around a function that returns a `Dataset` as
-  one of its components will have a `NestedDatasetComponent` in the
-  corresponding position in the `output_classes`, `output_shapes`, and
-  `output_types` properties.
-
-  TODO(b/110122868): Add this class, or something equivalent, to the public API.
-  We are considering revising the public API for accessing Dataset structure
-  (`output_classes` etc.) based on experience with nested datasets and other
-  custom component types.
-  """
-
-  def __init__(self,
-               dataset=None,
-               output_shapes=None,
-               output_types=None,
-               output_classes=None):
-    if dataset is None:
-      if (output_classes is None or output_shapes is None or
-          output_types is None):
-        raise ValueError(
-            "Either `dataset`, or all of `output_classes`, "
-            "`output_shapes`, and `output_types` must be specified.")
-      self._output_classes = output_classes
-      self._output_shapes = output_shapes
-      self._output_types = output_types
-    else:
-      if not (output_classes is None and output_shapes is None and
-              output_types is None):
-        raise ValueError(
-            "Either `dataset`, or all of `output_classes`, "
-            "`output_shapes`, and `output_types` must be specified.")
-      self._output_classes = dataset.output_classes
-      self._output_shapes = dataset.output_shapes
-      self._output_types = dataset.output_types
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
-
-
 class _VariantDataset(DatasetV2):
   """A Dataset wrapper around a `tf.variant`-typed function argument."""
 
@@ -1935,15 +1885,73 @@ class _VariantDataset(DatasetV2):
 
   @property
   def output_classes(self):
-    return self._structure.output_classes
+    return self._structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    return self._structure.output_shapes
+    return self._structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._structure.output_types
+    return self._structure._to_legacy_output_types()  # pylint: disable=protected-access
+
+
+class DatasetStructure(structure_lib.Structure):
+  """Represents a `Dataset` of structured values."""
+
+  def __init__(self, element_structure):
+    self._element_structure = element_structure
+
+  @property
+  def _flat_shapes(self):
+    return [tensor_shape.scalar()]
+
+  @property
+  def _flat_types(self):
+    return [dtypes.variant]
+
+  def is_compatible_with(self, other):
+    # pylint: disable=protected-access
+    return (isinstance(other, DatasetStructure) and
+            self._element_structure.is_compatible_with(
+                other._element_structure))
+
+  def _to_tensor_list(self, value):
+    return [value._as_variant_tensor()]  # pylint: disable=protected-access
+
+  def _from_tensor_list(self, flat_value):
+    if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
+        not flat_value[0].shape.is_compatible_with(tensor_shape.scalar())):
+      raise ValueError(
+          "DatasetStructure corresponds to a single tf.variant scalar.")
+    return self._from_compatible_tensor_list(flat_value)
+
+  def _from_compatible_tensor_list(self, flat_value):
+    # pylint: disable=protected-access
+    return _VariantDataset(flat_value[0], self._element_structure)
+
+  @staticmethod
+  def from_value(value):
+    # TODO(b/110122868): We can simplify this when a `Dataset` object has a
+    # `Structure`-valued property.
+    element_structure = structure_lib.Structure._from_legacy_structure(
+        value.output_types, value.output_shapes, value.output_classes)
+    return DatasetStructure(element_structure)
+
+  def _to_legacy_output_types(self):
+    return self
+
+  def _to_legacy_output_shapes(self):
+    return self
+
+  def _to_legacy_output_classes(self):
+    return self
+
+
+# pylint: disable=protected-access
+structure_lib.Structure._register_custom_converter(DatasetV2,
+                                                   DatasetStructure.from_value)
+# pylint: enable=protected-access
 
 
 class StructuredFunctionWrapper(object):
@@ -2001,6 +2009,9 @@ class StructuredFunctionWrapper(object):
       self._input_types = dataset.output_types
       self._input_classes = dataset.output_classes
 
+    self._input_structure = structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
+        self._input_types, self._input_shapes, self._input_classes)
+
     self._transformation_name = transformation_name
     readable_transformation_name = transformation_name.replace(
         ".", "_")[:-2] if len(transformation_name) > 2 else ""
@@ -2008,35 +2019,18 @@ class StructuredFunctionWrapper(object):
         readable_transformation_name,
         function_utils.get_func_name(func),
         str(ops.uid())
-
     ])
 
     if defun_kwargs is None:
       defun_kwargs = {}
 
     @function.Defun(
-        *self._defun_args(), func_name=self._func_name, **defun_kwargs)
+        *self._input_structure._flat_types, func_name=self._func_name,  # pylint: disable=protected-access
+        **defun_kwargs)
     def tf_data_structured_function_wrapper(*args):
       """Wrapper for passing nested structures to and from tf.data functions."""
-      flat_args = []
-      for arg, arg_class, arg_shape, arg_type in zip(
-          args,
-          nest.flatten(self._input_classes),
-          nest.flatten(self._input_shapes),
-          nest.flatten(self._input_types)):
-        # TODO(b/110122868): Add a registration mechanism for new component
-        # types.
-        if arg_class is sparse_tensor_lib.SparseTensor:
-          arg = sparse.deserialize_sparse_tensors(
-              arg, arg_type, arg_shape, arg_class)
-          arg.indices.set_shape([None, arg_shape.ndims])
-          arg.dense_shape.set_shape([arg_shape.ndims])
-        elif isinstance(arg_class, _NestedDatasetComponent):
-          arg = _VariantDataset(arg, arg_class)
-        else:
-          arg.set_shape(arg_shape)
-        flat_args.append(arg)
-      nested_args = nest.pack_sequence_as(self._input_classes, flat_args)
+      # pylint: disable=protected-access
+      nested_args = self._input_structure._from_compatible_tensor_list(args)
       if not _should_unpack_args(nested_args):
         nested_args = (nested_args,)
 
@@ -2054,50 +2048,14 @@ class StructuredFunctionWrapper(object):
       if isinstance(ret, list):
         ret = tuple(ret)
 
-      # Convert any `SparseTensorValue`s to `SparseTensor`s and all other
-      # values to tensors.
-      flat_ret = []
-      flat_classes = []
-      flat_shapes = []
-      flat_types = []
-      for t in nest.flatten(ret):
-        # TODO(b/110122868): Add a registration mechanism for new component
-        # types.
-        if sparse_tensor_lib.is_sparse(t):
-          t = sparse_tensor_lib.SparseTensor.from_value(t)
-          flat_ret.append(sparse.serialize_sparse_tensors(t))
-          flat_classes.append(sparse_tensor_lib.SparseTensor)
-          flat_shapes.append(t.get_shape())
-          flat_types.append(t.dtype)
-        elif isinstance(t, DatasetV2):
-          flat_ret.append(t._as_variant_tensor())  # pylint: disable=protected-access
-          component = _NestedDatasetComponent(t)
-          flat_classes.append(component)
-          flat_shapes.append(component)
-          flat_types.append(component)
-          if t.options() != Options():
-            warnings.warn("Encountered a nested dataset with non-default "
-                          "options. These options will not be propagated to "
-                          "the outer dataset.")
-        else:
-          try:
-            t = ops.convert_to_tensor(t)
-          except (ValueError, TypeError):
-            raise TypeError("Unsupported return value from function passed to "
-                            "%s: %s." % (transformation_name, t))
-          flat_ret.append(t)
-          flat_classes.append(ops.Tensor)
-          flat_shapes.append(t.get_shape())
-          flat_types.append(t.dtype)
-
-      ret = nest.pack_sequence_as(ret, flat_ret)
-      self._output_classes = nest.pack_sequence_as(ret, flat_classes)
-      self._output_shapes = nest.pack_sequence_as(ret, flat_shapes)
-      self._output_types = nest.pack_sequence_as(ret, flat_types)
+      try:
+        self._output_structure = structure_lib.Structure.from_value(ret)
+      except (ValueError, TypeError):
+        raise TypeError("Unsupported return value from function passed to "
+                        "%s: %s." % (transformation_name, ret))
 
       _warn_if_collections(transformation_name)
-
-      return flat_ret
+      return self._output_structure._to_tensor_list(ret)
 
     self._function = tf_data_structured_function_wrapper
     if add_to_graph:
@@ -2108,32 +2066,21 @@ class StructuredFunctionWrapper(object):
       # in case (e.g.) we need to rerun the function.
       self._function._create_definition_if_needed()  # pylint: disable=protected-access
 
-  def _defun_args(self):
-    """Returns a flat list of `tf.DType` for the input element structure."""
-    ret = []
-    for input_type, input_class in zip(nest.flatten(self._input_types),
-                                       nest.flatten(self._input_classes)):
-      # TODO(b/110122868): Add a registration mechanism for new component types.
-      if input_class is sparse_tensor_lib.SparseTensor:
-        ret.append(dtypes.variant)
-      elif isinstance(input_class, _NestedDatasetComponent):
-        ret.append(dtypes.variant)
-      else:
-        assert isinstance(input_type, dtypes.DType)
-        ret.append(input_type)
-    return ret
+  @property
+  def output_structure(self):
+    return self._output_structure
 
   @property
   def output_classes(self):
-    return self._output_classes
+    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    return self._output_shapes
+    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._output_types
+    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
   @property
   def function(self):
@@ -2156,30 +2103,12 @@ def flat_structure(dataset):
     A dictionary of keyword arguments that can be passed to many Dataset op
     constructors.
   """
-  output_classes = []
-  output_shapes = []
-  output_types = []
-  for output_class, output_shape, output_type in zip(
-      nest.flatten(dataset.output_classes), nest.flatten(dataset.output_shapes),
-      nest.flatten(dataset.output_types)):
-    if isinstance(output_class, _NestedDatasetComponent):
-      output_classes.append(output_class.output_classes)
-      output_shapes.append(output_shape.output_shapes)
-      output_types.append(output_type.output_types)
-    else:
-      output_classes.append(output_class)
-      output_shapes.append(output_shape)
-      output_types.append(output_type)
-
-  output_classes = nest.pack_sequence_as(dataset.output_classes, output_classes)
-  output_shapes = nest.pack_sequence_as(dataset.output_shapes, output_shapes)
-  output_types = nest.pack_sequence_as(dataset.output_types, output_types)
-
+  # pylint: disable=protected-access
+  structure = structure_lib.Structure._from_legacy_structure(
+      dataset.output_types, dataset.output_shapes, dataset.output_classes)
   return {
-      "output_shapes":
-          nest.flatten(sparse.as_dense_shapes(output_shapes, output_classes)),
-      "output_types":
-          nest.flatten(sparse.as_dense_types(output_types, output_classes)),
+      "output_shapes": structure._flat_shapes,
+      "output_types": structure._flat_types,
   }
 
 
@@ -2902,11 +2831,13 @@ class FlatMapDataset(UnaryDataset):
 
     wrapped_func = StructuredFunctionWrapper(
         map_func, self._transformation_name(), dataset=input_dataset)
-    if not isinstance(wrapped_func.output_classes, _NestedDatasetComponent):
+    if not isinstance(wrapped_func.output_structure, DatasetStructure):
       raise TypeError("`map_func` must return a `Dataset` object.")
-    self._output_classes = wrapped_func.output_classes.output_classes
-    self._output_types = wrapped_func.output_types.output_types
-    self._output_shapes = wrapped_func.output_shapes.output_shapes
+    # pylint: disable=protected-access
+    element_structure = wrapped_func.output_structure._element_structure
+    self._output_classes = element_structure._to_legacy_output_classes()
+    self._output_types = element_structure._to_legacy_output_types()
+    self._output_shapes = element_structure._to_legacy_output_shapes()
     self._map_func = wrapped_func.function
 
   def _as_variant_tensor(self):
@@ -3048,10 +2979,9 @@ class WindowDataset(UnaryDataset):
     self._output_classes = nest.pack_sequence_as(
         input_dataset.output_classes,
         [
-            _NestedDatasetComponent(  # pylint: disable=protected-access
-                output_classes=output_class,
-                output_shapes=output_shape,
-                output_types=output_type)
+            DatasetStructure(
+                structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
+                    output_type, output_shape, output_class))
             for output_class, output_shape, output_type in zip(
                 nest.flatten(input_dataset.output_classes),
                 nest.flatten(input_dataset.output_shapes),
diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py
index 91cf883ce9..4113b7ed31 100644
--- a/tensorflow/python/data/ops/optional_ops.py
+++ b/tensorflow/python/data/ops/optional_ops.py
@@ -183,19 +183,13 @@ class OptionalStructure(structure.Structure):
     return OptionalStructure(value.value_structure)
 
   def _to_legacy_output_types(self):
-    raise NotImplementedError("The `output_types` property is not supported on "
-                              "structured objects containing an `Optional`. "
-                              "Use the corresponding `structure` property.")
+    return self
 
   def _to_legacy_output_shapes(self):
-    raise NotImplementedError("The `output_shapes` property is not supported on"
-                              " structured objects containing an `Optional`. "
-                              "Use the corresponding `structure` property.")
+    return self
 
   def _to_legacy_output_classes(self):
-    raise NotImplementedError("The `output_classes` property is not supported "
-                              "on structured objects containing an `Optional`. "
-                              "Use the corresponding `structure` property.")
+    return self
 
 
 # pylint: disable=protected-access
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 9a3118297d..3cf67b0745 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -208,14 +208,16 @@ class Structure(object):
     flat_ret = []
     for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes,
                                                  flat_classes):
-      if issubclass(flat_class, sparse_tensor_lib.SparseTensor):
+      if isinstance(flat_class, Structure):
+        flat_ret.append(flat_class)
+      elif issubclass(flat_class, sparse_tensor_lib.SparseTensor):
         flat_ret.append(SparseTensorStructure(flat_type, flat_shape))
       elif issubclass(flat_class, ops.Tensor):
         flat_ret.append(TensorStructure(flat_type, flat_shape))
       else:
         # NOTE(mrry): Since legacy structures produced by iterators only
-        # comprise Tensors, SparseTensors, and nests, we do not need to support
-        # all structure types here.
+        # comprise Tensors, SparseTensors, and nests, we do not need to
+        # support all structure types here.
         raise TypeError(
             "Could not build a structure for output class %r" % flat_type)
 
@@ -381,6 +383,13 @@ class TensorStructure(Structure):
     return self._from_compatible_tensor_list(flat_value)
 
   def _from_compatible_tensor_list(self, flat_value):
+    # TODO(b/112266545): It would be cleaner to create a new `ensure_shape()`
+    # op here and return that, instead of mutating the input's shape using
+    # `Tensor.set_shape()`. However, that would add extra ops on the arguments
+    # of each `tf.data` function, which could impact performance. When this
+    # bug is resolved, we should be able to add the `ensure_shape()` ops and
+    # optimize them away using contextual shape information.
+    flat_value[0].set_shape(self._shape)
     return flat_value[0]
 
   @staticmethod
@@ -406,7 +415,11 @@ class SparseTensorStructure(Structure):
 
   @property
   def _flat_shapes(self):
-    return [tensor_shape.vector(3)]
+    # NOTE(mrry): The default flat shape of a boxed `SparseTensor` is `(3,)`,
+    # but a `SparseTensorStructure` can also represent a batch of boxed
+    # `SparseTensor` objects with shape `(?, 3)` (and batches of batches, etc.),
+    # so the flat shape must be unknown.
+    return [tensor_shape.unknown_shape(None)]
 
   @property
   def _flat_types(self):
@@ -428,8 +441,11 @@ class SparseTensorStructure(Structure):
     return self._from_compatible_tensor_list(flat_value)
 
   def _from_compatible_tensor_list(self, flat_value):
-    return sparse_ops.deserialize_sparse(
+    ret = sparse_ops.deserialize_sparse(
         flat_value[0], dtype=self._dtype, rank=self._dense_shape.ndims)
+    ret.indices.set_shape([None, self._dense_shape.ndims])
+    ret.dense_shape.set_shape([self._dense_shape.ndims])
+    return ret
 
   @staticmethod
   def from_value(value):
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index 630a0c912b..65a41a50f1 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -44,7 +44,7 @@ class StructureTest(test.TestCase, parameterized.TestCase):
        [dtypes.float32], [[]]),
       (lambda: sparse_tensor.SparseTensor(
           indices=[[3, 4]], values=[-1], dense_shape=[4, 5]),
-       structure.SparseTensorStructure, [dtypes.variant], [[3]]),
+       structure.SparseTensorStructure, [dtypes.variant], [None]),
       (lambda: (constant_op.constant(37.0), constant_op.constant([1, 2, 3])),
        structure.NestedStructure, [dtypes.float32, dtypes.int32], [[], [3]]),
       (lambda: {
@@ -58,14 +58,17 @@ class StructureTest(test.TestCase, parameterized.TestCase):
                 sparse_tensor.SparseTensor(
                     indices=[[3, 4]], values=[-1], dense_shape=[4, 5]))
       }, structure.NestedStructure,
-       [dtypes.float32, dtypes.variant, dtypes.variant], [[], [3], [3]]))
+       [dtypes.float32, dtypes.variant, dtypes.variant], [[], None, None]))
   def testFlatStructure(self, value_fn, expected_structure, expected_types,
                         expected_shapes):
     value = value_fn()
     s = structure.Structure.from_value(value)
     self.assertIsInstance(s, expected_structure)
     self.assertEqual(expected_types, s._flat_types)
-    self.assertEqual(expected_shapes, s._flat_shapes)
+    for expected, actual in zip(expected_shapes, s._flat_shapes):
+      self.assertTrue(actual.is_compatible_with(expected))
+      self.assertTrue(
+          tensor_shape.as_shape(expected).is_compatible_with(actual))
 
   @parameterized.parameters(
       (lambda: constant_op.constant(37.0), lambda: [
-- 
GitLab


From 1799b11347253cd80408d15715fb443c7a1814e9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 09:52:26 -0800
Subject: [PATCH 0668/1554] Reduce ops v2 API changes

PiperOrigin-RevId: 222422608
---
 .../python/kernel_tests/monte_carlo_test.py   |   2 +-
 .../bayesflow/python/ops/monte_carlo_impl.py  |   4 +-
 .../boosted_trees/python/utils/losses.py      |   3 +-
 .../normal_conjugate_posteriors_test.py       |   2 +-
 .../python/kernel_tests/wishart_test.py       |   5 +-
 .../python/ops/bijectors/softmax_centered.py  |   2 +-
 tensorflow/contrib/eager/python/tfe_test.py   |   4 +-
 .../contrib/layers/python/layers/encoders.py  |   3 +-
 .../layers/python/layers/feature_column.py    |   3 +-
 .../layers/python/layers/layers_test.py       |   4 +-
 .../estimators/dynamic_rnn_estimator_test.py  |   6 +-
 .../contrib/losses/python/losses/loss_ops.py  |  18 +-
 .../contrib/metrics/python/ops/metric_ops.py  |   2 +-
 .../contrib/quantize/python/quant_ops.py      |   8 +-
 tensorflow/python/eager/backprop_test.py      |   2 +-
 .../python/grappler/cost_analyzer_test.py     |   4 +-
 tensorflow/python/ops/linalg/linalg_impl.py   |   2 +-
 .../python/ops/linalg/linear_operator.py      |   2 +-
 .../ops/linalg/linear_operator_circulant.py   |  10 +-
 .../python/ops/linalg/linear_operator_diag.py |   4 +-
 .../linalg/linear_operator_low_rank_update.py |   2 +-
 .../linear_operator_lower_triangular.py       |   4 +-
 tensorflow/python/ops/losses/losses_impl.py   |   9 +-
 tensorflow/python/ops/math_ops.py             | 584 +++++++++++++-----
 tensorflow/python/ops/math_ops_test.py        |   4 +-
 tensorflow/python/ops/metrics_impl.py         |   4 +-
 .../ops/parallel_for/control_flow_ops_test.py |   7 +-
 tensorflow/python/ops/sparse_grad.py          |   2 +-
 tensorflow/python/ops/special_math_ops.py     |   7 +-
 .../tools/api/golden/v1/tensorflow.math.pbtxt |   4 +-
 .../tools/api/golden/v2/tensorflow.math.pbtxt |  20 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  16 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 154 ++++-
 33 files changed, 659 insertions(+), 248 deletions(-)

diff --git a/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py b/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py
index 13215ffabf..8b6ed9f041 100644
--- a/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py
+++ b/tensorflow/contrib/bayesflow/python/kernel_tests/monte_carlo_test.py
@@ -81,7 +81,7 @@ class ExpectationImportanceSampleTest(test.TestCase):
       # Compute E_p[X_1 * X_2 > 0], with X_i the ith component of X ~ p(x).
       # Should equal 1/2 because p is a spherical Gaussian centered at (0, 0).
       def indicator(x):
-        x1_times_x2 = math_ops.reduce_prod(x, reduction_indices=[-1])
+        x1_times_x2 = math_ops.reduce_prod(x, axis=[-1])
         return 0.5 * (math_ops.sign(x1_times_x2) + 1.0)
 
       prob = mc.expectation_importance_sampler(
diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py
index 18d40fc1df..e83a548511 100644
--- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py
+++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py
@@ -353,12 +353,12 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True,
 
 def _sample_mean(values):
   """Mean over sample indices.  In this module this is always [0]."""
-  return math_ops.reduce_mean(values, reduction_indices=[0])
+  return math_ops.reduce_mean(values, axis=[0])
 
 
 def _sample_max(values):
   """Max over sample indices.  In this module this is always [0]."""
-  return math_ops.reduce_max(values, reduction_indices=[0])
+  return math_ops.reduce_max(values, axis=[0])
 
 
 def _get_samples(dist, z, n, seed):
diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py
index 7a99dccdd1..220e981618 100644
--- a/tensorflow/contrib/boosted_trees/python/utils/losses.py
+++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py
@@ -119,8 +119,7 @@ def per_example_maxent_loss(labels, weights, logits, num_classes, eps=1e-15):
     labels = array_ops.expand_dims(labels, 1)
   # Labels are indices of classes, convert them to one hot encodings.
   target_one_hot = array_ops.one_hot(indices=labels, depth=num_classes)
-  labels = math_ops.reduce_sum(
-      input_tensor=target_one_hot, reduction_indices=[1])
+  labels = math_ops.reduce_sum(input_tensor=target_one_hot, axis=[1])
   labels = math_ops.to_float(labels)
 
   # Calculate softmax probabilities for each class.
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py b/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py
index 29eeaf43c5..ab3c07172a 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/normal_conjugate_posteriors_test.py
@@ -82,7 +82,7 @@ class NormalTest(test.TestCase):
       x = constant_op.constant(
           [[-2.5, 2.5, 4.0, 0.0, -1.0, 2.0], [2.5, -2.5, -4.0, 0.0, 1.0, -2.0]],
           dtype=dtypes.float32)
-      s = math_ops.reduce_sum(x, reduction_indices=[1])
+      s = math_ops.reduce_sum(x, axis=[1])
       x = array_ops.transpose(x)  # Reshape to shape (6, 2)
       n = constant_op.constant([6] * 2)
       prior = distributions.Normal(loc=mu0, scale=sigma0)
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py
index a60056c444..cdee30bbc4 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py
@@ -147,14 +147,13 @@ class WishartCholeskyTest(test.TestCase):
       x = chol_w.sample(10000, seed=42)
       self.assertAllEqual((10000, 3, 3), x.get_shape())
 
-      moment1_estimate = math_ops.reduce_mean(x, reduction_indices=[0]).eval()
+      moment1_estimate = math_ops.reduce_mean(x, axis=[0]).eval()
       self.assertAllClose(chol_w.mean().eval(), moment1_estimate, rtol=0.05)
 
       # The Variance estimate uses the squares rather than outer-products
       # because Wishart.Variance is the diagonal of the Wishart covariance
       # matrix.
-      variance_estimate = (math_ops.reduce_mean(
-          math_ops.square(x), reduction_indices=[0]) -
+      variance_estimate = (math_ops.reduce_mean(math_ops.square(x), axis=[0]) -
                            math_ops.square(moment1_estimate)).eval()
       self.assertAllClose(
           chol_w.variance().eval(), variance_estimate, rtol=0.05)
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py
index 15c241d5d7..74765f19e5 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py
@@ -168,7 +168,7 @@ class SoftmaxCentered(bijector.Bijector):
     #   log_normalization = 1 + reduce_sum(exp(logits))
     #   -log_normalization + reduce_sum(logits - log_normalization)
     log_normalization = nn_ops.softplus(
-        math_ops.reduce_logsumexp(x, axis=-1, keep_dims=True))
+        math_ops.reduce_logsumexp(x, axis=-1, keepdims=True))
     return array_ops.squeeze(
         (-log_normalization + math_ops.reduce_sum(
             x - log_normalization, axis=-1, keepdims=True)), axis=-1)
diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index 4454abfb96..8c35dddb5a 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -87,8 +87,8 @@ class TFETest(test_util.TensorFlowTestCase):
       x += 1.
     # Without a device context, heuristics are used to place ops.
     # In this case, ops.reduce_mean runs on the GPU.
-    reduction_indices = range(x.shape.ndims)
-    m = math_ops.reduce_mean(x, reduction_indices)
+    axis = range(x.shape.ndims)
+    m = math_ops.reduce_mean(x, axis)
     # m is on GPU, bring it back to CPU and compare.
     self.assertEqual(3.5, m.cpu().numpy())
 
diff --git a/tensorflow/contrib/layers/python/layers/encoders.py b/tensorflow/contrib/layers/python/layers/encoders.py
index f42112206d..3671633c8d 100644
--- a/tensorflow/contrib/layers/python/layers/encoders.py
+++ b/tensorflow/contrib/layers/python/layers/encoders.py
@@ -84,8 +84,7 @@ def bow_encoder(ids,
       if isinstance(ids, sparse_tensor.SparseTensor):
         raise TypeError('ids are expected to be dense Tensor, got: %s', ids)
       return math_ops.reduce_mean(
-          embedding_ops.embedding_lookup(embeddings, ids),
-          reduction_indices=1)
+          embedding_ops.embedding_lookup(embeddings, ids), axis=1)
 
 
 def embed_sequence(ids,
diff --git a/tensorflow/contrib/layers/python/layers/feature_column.py b/tensorflow/contrib/layers/python/layers/feature_column.py
index 222404b19d..00d819ed0e 100644
--- a/tensorflow/contrib/layers/python/layers/feature_column.py
+++ b/tensorflow/contrib/layers/python/layers/feature_column.py
@@ -1015,8 +1015,7 @@ class _OneHotColumn(
         dense_id_tensor, depth=self.length, on_value=1.0, off_value=0.0)
 
     # Reduce to get a multi-hot per example.
-    return math_ops.reduce_sum(
-        one_hot_id_tensor, reduction_indices=[output_rank - 1])
+    return math_ops.reduce_sum(one_hot_id_tensor, axis=[output_rank - 1])
 
   @property
   def _variable_shape(self):
diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py
index 8ead6336a0..0a4d2c6d4c 100644
--- a/tensorflow/contrib/layers/python/layers/layers_test.py
+++ b/tensorflow/contrib/layers/python/layers/layers_test.py
@@ -3811,7 +3811,7 @@ class UnitNormTests(test.TestCase):
       image = random_ops.random_uniform((height, width, 3))
       output = _layers.unit_norm(image, dim=dim, epsilon=1e-6)
       norms = math_ops.sqrt(
-          math_ops.reduce_sum(math_ops.square(output), reduction_indices=dim))
+          math_ops.reduce_sum(math_ops.square(output), axis=dim))
 
       shape = [height, width, 3]
       del shape[dim]
@@ -3847,7 +3847,7 @@ class UnitNormTests(test.TestCase):
       image = array_ops.placeholder(dtypes.float32, (None, None, 3))
       output = _layers.unit_norm(image, dim=dim, epsilon=1e-6)
       norms = math_ops.sqrt(
-          math_ops.reduce_sum(math_ops.square(output), reduction_indices=dim))
+          math_ops.reduce_sum(math_ops.square(output), axis=dim))
 
       with self.cached_session():
         actual = norms.eval({image: placeholder_value})
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py
index 1d8a59281a..28c4964527 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator_test.py
@@ -668,7 +668,7 @@ class DynamicRNNEstimatorLearningTest(test.TestCase):
         sequences = centers + noise
 
         inputs = array_ops.expand_dims(sequences, 2)
-        labels = math_ops.reduce_mean(sequences, reduction_indices=[1])
+        labels = math_ops.reduce_mean(sequences, axis=[1])
         return {'inputs': inputs}, labels
 
       return input_fn
@@ -722,8 +722,8 @@ class DynamicRNNEstimatorLearningTest(test.TestCase):
         inputs = array_ops.expand_dims(math_ops.to_float(random_sequence), 2)
         labels = math_ops.to_int32(
             array_ops.squeeze(
-                math_ops.reduce_sum(
-                    inputs, reduction_indices=[1]) > (sequence_length / 2.0)))
+                math_ops.reduce_sum(inputs, axis=[1]) > (
+                    sequence_length / 2.0)))
         return {'inputs': inputs}, labels
 
       return input_fn
diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py
index d8ac4163b2..709a042bbc 100644
--- a/tensorflow/contrib/losses/python/losses/loss_ops.py
+++ b/tensorflow/contrib/losses/python/losses/loss_ops.py
@@ -59,9 +59,8 @@ def _scale_losses(losses, weights):
   """
   # First, compute the sum of the losses over all elements:
   start_index = max(0, weights.get_shape().ndims)
-  reduction_indices = list(range(start_index, losses.get_shape().ndims))
-  reduced_losses = math_ops.reduce_sum(
-      losses, reduction_indices=reduction_indices)
+  axis = list(range(start_index, losses.get_shape().ndims))
+  reduced_losses = math_ops.reduce_sum(losses, axis=axis)
   reduced_losses = math_ops.multiply(reduced_losses, weights)
   return math_ops.reduce_sum(reduced_losses)
 
@@ -158,10 +157,9 @@ def _num_present(losses, weights, per_batch=False):
 
   # First, count the number of nonzero weights:
   if weights.get_shape().ndims >= 1:
-    reduction_indices = list(range(1, weights.get_shape().ndims))
+    axis = list(range(1, weights.get_shape().ndims))
     num_nonzero_per_batch = math_ops.reduce_sum(
-        math_ops.to_float(math_ops.not_equal(weights, 0)),
-        reduction_indices=reduction_indices)
+        math_ops.to_float(math_ops.not_equal(weights, 0)), axis=axis)
 
   # Next, determine the number of elements that weights would broadcast to:
   broadcast_dims = array_ops.slice(
@@ -577,16 +575,16 @@ def mean_pairwise_squared_error(predictions,
     if weights.get_shape().ndims is None:
       raise ValueError("weights.get_shape().ndims cannot be None")
 
-    reduction_indices = list(range(1, diffs.get_shape().ndims))
+    axis = list(range(1, diffs.get_shape().ndims))
 
     sum_squares_diff_per_batch = math_ops.reduce_sum(
-        math_ops.square(diffs), reduction_indices=reduction_indices)
+        math_ops.square(diffs), axis=axis)
     num_present_per_batch = _num_present(diffs, weights, per_batch=True)
 
     term1 = 2.0 * math_ops.div_no_nan(
         sum_squares_diff_per_batch, num_present_per_batch, name="value")
 
-    sum_diff = math_ops.reduce_sum(diffs, reduction_indices=reduction_indices)
+    sum_diff = math_ops.reduce_sum(diffs, axis=axis)
     term2 = 2.0 * math_ops.div_no_nan(
         math_ops.square(sum_diff),
         math_ops.square(num_present_per_batch),
@@ -645,7 +643,7 @@ def cosine_distance(predictions,
 
     radial_diffs = math_ops.multiply(predictions, labels)
     losses = 1 - math_ops.reduce_sum(
-        radial_diffs, reduction_indices=[
+        radial_diffs, axis=[
             axis,
         ])
     return compute_weighted_loss(losses, weights, scope=scope)
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 09fe65b73f..7b432f8bd2 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -3416,7 +3416,7 @@ def streaming_mean_cosine_distance(predictions,
   predictions.get_shape().assert_is_compatible_with(labels.get_shape())
   radial_diffs = math_ops.multiply(predictions, labels)
   radial_diffs = math_ops.reduce_sum(
-      radial_diffs, reduction_indices=[
+      radial_diffs, axis=[
           dim,
       ], keepdims=True)
   mean_distance, update_op = streaming_mean(radial_diffs, weights, None, None,
diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py
index 6f659347fb..8619708cda 100644
--- a/tensorflow/contrib/quantize/python/quant_ops.py
+++ b/tensorflow/contrib/quantize/python/quant_ops.py
@@ -138,7 +138,7 @@ def LastValueQuantize(inputs,
     if per_channel:
       if input_dim >= 2:
         batch_min = math_ops.reduce_min(
-            inputs, reduction_indices=reduce_dims, name='BatchMin')
+            inputs, axis=reduce_dims, name='BatchMin')
       else:
         batch_min = inputs
     else:
@@ -147,7 +147,7 @@ def LastValueQuantize(inputs,
     if per_channel:
       if input_dim >= 2:
         batch_max = math_ops.reduce_max(
-            inputs, reduction_indices=reduce_dims, name='BatchMax')
+            inputs, axis=reduce_dims, name='BatchMax')
       else:
         batch_max = inputs
     else:
@@ -263,7 +263,7 @@ def MovingAvgQuantize(inputs,
     if per_channel:
       if input_dim >= 2:
         batch_min = math_ops.reduce_min(
-            inputs, reduction_indices=reduce_dims, name='BatchMin')
+            inputs, axis=reduce_dims, name='BatchMin')
       else:
         batch_min = inputs
     else:
@@ -272,7 +272,7 @@ def MovingAvgQuantize(inputs,
     if per_channel:
       if input_dim >= 2:
         batch_max = math_ops.reduce_max(
-            inputs, reduction_indices=reduce_dims, name='BatchMax')
+            inputs, axis=reduce_dims, name='BatchMax')
       else:
         batch_max = inputs
     else:
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 237b7f304e..8b85548e5c 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -74,7 +74,7 @@ class BackpropTest(test.TestCase):
       tf_g1 = embedding_ops.embedding_lookup(tf_var, tf_ind1)
       tf_g2 = embedding_ops.embedding_lookup(tf_var, tf_ind2)
       tf_g3 = embedding_ops.embedding_lookup(tf_var, tf_ind3)
-      tf_g4 = math_ops.reduce_sum(tf_var * 2.0, reduction_indices=(0, 1))
+      tf_g4 = math_ops.reduce_sum(tf_var * 2.0, axis=(0, 1))
       tf_y = tf_g1 * tf_g2 * tf_g3 * tf_g4
       tf_grad = gradients.gradients(tf_y, [tf_var])[0]
 
diff --git a/tensorflow/python/grappler/cost_analyzer_test.py b/tensorflow/python/grappler/cost_analyzer_test.py
index b8225b81a5..de80df1879 100644
--- a/tensorflow/python/grappler/cost_analyzer_test.py
+++ b/tensorflow/python/grappler/cost_analyzer_test.py
@@ -96,8 +96,8 @@ class CostAnalysisTest(test.TestCase):
     b_fc = variables.Variable(random_ops.truncated_normal([10], stddev=0.1))
     y_conv = nn_ops.softmax(math_ops.matmul(h_conv_flat, w_fc) + b_fc)
 
-    cross_entropy = math_ops.reduce_mean(-math_ops.reduce_sum(
-        label * math_ops.log(y_conv), reduction_indices=[1]))
+    cross_entropy = math_ops.reduce_mean(
+        -math_ops.reduce_sum(label * math_ops.log(y_conv), axis=[1]))
     _ = adam.AdamOptimizer(1e-4).minimize(cross_entropy)
 
     mg = meta_graph.create_meta_graph_def(graph=ops.get_default_graph())
diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
index 08d50ce622..2c9476a9bd 100644
--- a/tensorflow/python/ops/linalg/linalg_impl.py
+++ b/tensorflow/python/ops/linalg/linalg_impl.py
@@ -88,7 +88,7 @@ def logdet(matrix, name=None):
     chol = gen_linalg_ops.cholesky(matrix)
     return 2.0 * math_ops.reduce_sum(
         math_ops.log(math_ops.real(array_ops.matrix_diag_part(chol))),
-        reduction_indices=[-1])
+        axis=[-1])
 
 
 @tf_export('linalg.adjoint')
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 6fb7a57e4d..8efafda3a1 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -690,7 +690,7 @@ class LinearOperator(object):
         "  Requires conversion to a dense matrix and O(N^3) operations.")
     if self._can_use_cholesky():
       diag = array_ops.matrix_diag_part(linalg_ops.cholesky(self.to_dense()))
-      return 2 * math_ops.reduce_sum(math_ops.log(diag), reduction_indices=[-1])
+      return 2 * math_ops.reduce_sum(math_ops.log(diag), axis=[-1])
     _, log_abs_det = linalg.slogdet(self.to_dense())
     return log_abs_det
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_circulant.py b/tensorflow/python/ops/linalg/linear_operator_circulant.py
index 09f0c518e7..b74baa5dfd 100644
--- a/tensorflow/python/ops/linalg/linear_operator_circulant.py
+++ b/tensorflow/python/ops/linalg/linear_operator_circulant.py
@@ -418,15 +418,13 @@ class _BaseLinearOperatorCirculant(linear_operator.LinearOperator):
     return math_ops.cast(y, self.dtype)
 
   def _determinant(self):
-    reduction_indices = [-(i + 1) for i in range(self.block_depth)]
-    det = math_ops.reduce_prod(
-        self.spectrum, reduction_indices=reduction_indices)
+    axis = [-(i + 1) for i in range(self.block_depth)]
+    det = math_ops.reduce_prod(self.spectrum, axis=axis)
     return math_ops.cast(det, self.dtype)
 
   def _log_abs_determinant(self):
-    reduction_indices = [-(i + 1) for i in range(self.block_depth)]
-    lad = math_ops.reduce_sum(
-        math_ops.log(self._abs_spectrum), reduction_indices=reduction_indices)
+    axis = [-(i + 1) for i in range(self.block_depth)]
+    lad = math_ops.reduce_sum(math_ops.log(self._abs_spectrum), axis=axis)
     return math_ops.cast(lad, self.dtype)
 
   def _solve(self, rhs, adjoint=False, adjoint_arg=False):
diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py
index ed53decc00..be893c705c 100644
--- a/tensorflow/python/ops/linalg/linear_operator_diag.py
+++ b/tensorflow/python/ops/linalg/linear_operator_diag.py
@@ -228,11 +228,11 @@ class LinearOperatorDiag(linear_operator.LinearOperator):
     return diag_mat * x
 
   def _determinant(self):
-    return math_ops.reduce_prod(self._diag, reduction_indices=[-1])
+    return math_ops.reduce_prod(self._diag, axis=[-1])
 
   def _log_abs_determinant(self):
     log_det = math_ops.reduce_sum(
-        math_ops.log(math_ops.abs(self._diag)), reduction_indices=[-1])
+        math_ops.log(math_ops.abs(self._diag)), axis=[-1])
     if self.dtype.is_complex:
       log_det = math_ops.cast(log_det, dtype=self.dtype)
     return log_det
diff --git a/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py
index c4288ff8f8..aa0500aff0 100644
--- a/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py
+++ b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py
@@ -391,7 +391,7 @@ class LinearOperatorLowRankUpdate(linear_operator.LinearOperator):
     if self._use_cholesky:
       chol_cap_diag = array_ops.matrix_diag_part(self._chol_capacitance)
       log_abs_det_c = 2 * math_ops.reduce_sum(
-          math_ops.log(chol_cap_diag), reduction_indices=[-1])
+          math_ops.log(chol_cap_diag), axis=[-1])
     else:
       det_c = linalg_ops.matrix_determinant(self._capacitance)
       log_abs_det_c = math_ops.log(math_ops.abs(det_c))
diff --git a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
index ca6d3f5405..d33fe17e04 100644
--- a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
+++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
@@ -195,11 +195,11 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator):
         self._tril, x, adjoint_a=adjoint, adjoint_b=adjoint_arg)
 
   def _determinant(self):
-    return math_ops.reduce_prod(self._diag, reduction_indices=[-1])
+    return math_ops.reduce_prod(self._diag, axis=[-1])
 
   def _log_abs_determinant(self):
     return math_ops.reduce_sum(
-        math_ops.log(math_ops.abs(self._diag)), reduction_indices=[-1])
+        math_ops.log(math_ops.abs(self._diag)), axis=[-1])
 
   def _solve(self, rhs, adjoint=False, adjoint_arg=False):
     rhs = linalg.adjoint(rhs) if adjoint_arg else rhs
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index e8cadf931b..0a5b511f82 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -583,12 +583,10 @@ def mean_pairwise_squared_error(
 
       diffs = math_ops.subtract(predictions, labels)
 
-      reduction_indices = math_ops.range(1, array_ops.rank(diffs))
+      axis = math_ops.range(1, array_ops.rank(diffs))
 
       sum_squares_diff_per_batch = math_ops.reduce_sum(
-          math_ops.square(diffs),
-          reduction_indices=reduction_indices,
-          keepdims=True)
+          math_ops.square(diffs), axis=axis, keepdims=True)
       num_present_per_batch = _num_present(diffs, weights, per_batch=True)
 
       term1 = 2.0 * math_ops.div_no_nan(
@@ -596,8 +594,7 @@ def mean_pairwise_squared_error(
           math_ops.maximum(num_present_per_batch - 1, 0),
           name="value")
 
-      sum_diff = math_ops.reduce_sum(
-          diffs, reduction_indices=reduction_indices, keepdims=True)
+      sum_diff = math_ops.reduce_sum(diffs, axis=axis, keepdims=True)
       term2 = 2.0 * math_ops.div_no_nan(
           math_ops.square(sum_diff),
           math_ops.maximum(
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 952a2a1e79..73ca3d527a 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1314,7 +1314,7 @@ def range(start, limit=None, delta=1, dtype=None, name="range"):  # pylint: disa
 
 
 # Reduction operations
-def _ReductionDims(x, axis, reduction_indices):
+def _ReductionDims(x, axis, reduction_indices=None):  # pylint: disable=invalid-name
   """Returns range(0, rank(x)) if reduction_indices is None."""
   # TODO(aselle): Remove this after deprecation
   if reduction_indices is not None:
@@ -1337,23 +1337,23 @@ def _ReductionDims(x, axis, reduction_indices):
     return range(0, array_ops.rank(x))
 
 
-def _may_reduce_to_scalar(keepdims, axis, reduction_indices, output):
+def _may_reduce_to_scalar(keepdims, axis, output):
   """Set a reduction's output shape to be a scalar if we are certain."""
   if not common_shapes.has_fully_defined_shape(output) and (not keepdims) and (
-      axis is None) and (reduction_indices is None):
+      axis is None):
     output.set_shape(())
   return output
 
 
-@tf_export("math.reduce_sum", "reduce_sum")
+@tf_export(v1=["math.reduce_sum", "reduce_sum"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_sum(input_tensor,
-               axis=None,
-               keepdims=None,
-               name=None,
-               reduction_indices=None,
-               keep_dims=None):
+def reduce_sum_v1(input_tensor,
+                  axis=None,
+                  keepdims=None,
+                  name=None,
+                  reduction_indices=None,
+                  keep_dims=None):
   """Computes the sum of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1393,18 +1393,58 @@ def reduce_sum(input_tensor,
   int64 while tensorflow returns the same dtype as the input.
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
-  if keepdims is None:
-    keepdims = False
+  return reduce_sum(input_tensor, axis, keepdims, name)
+
 
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops._sum(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
+@tf_export("math.reduce_sum", "reduce_sum", v1=[])
+def reduce_sum(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes the sum of elements across dimensions of a tensor.
+
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
+
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  For example:
+
+  ```python
+  x = tf.constant([[1, 1, 1], [1, 1, 1]])
+  tf.reduce_sum(x)  # 6
+  tf.reduce_sum(x, 0)  # [2, 2, 2]
+  tf.reduce_sum(x, 1)  # [3, 3]
+  tf.reduce_sum(x, 1, keepdims=True)  # [[3], [3]]
+  tf.reduce_sum(x, [0, 1])  # 6
+  ```
+
+  Args:
+    input_tensor: The tensor to reduce. Should have numeric type.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor, of the same dtype as the input_tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.sum apart the fact that numpy upcast uint8 and int32 to
+  int64 while tensorflow returns the same dtype as the input.
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops._sum(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
 
 
 @tf_export(v1=["math.count_nonzero", "count_nonzero"])
@@ -1544,15 +1584,13 @@ def count_nonzero_v2(input,  # pylint: disable=redefined-builtin
         dtype=dtype)
 
 
-@tf_export("math.reduce_mean", "reduce_mean")
-@deprecation.deprecated_args(
-    None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_mean(input_tensor,
-                axis=None,
-                keepdims=None,
-                name=None,
-                reduction_indices=None,
-                keep_dims=None):
+@tf_export(v1=["math.reduce_mean", "reduce_mean"])
+def reduce_mean_v1(input_tensor,
+                   axis=None,
+                   keepdims=None,
+                   name=None,
+                   reduction_indices=None,
+                   keep_dims=None):
   """Computes the mean of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1602,22 +1640,72 @@ def reduce_mean(input_tensor,
 
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
+  return reduce_mean(input_tensor, axis, keepdims, name)
 
-  if keepdims is None:
-    keepdims = False
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops.mean(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
+
+@tf_export("math.reduce_mean", "reduce_mean", v1=[])
+def reduce_mean(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes the mean of elements across dimensions of a tensor.
+
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
+
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  For example:
+
+  ```python
+  x = tf.constant([[1., 1.], [2., 2.]])
+  tf.reduce_mean(x)  # 1.5
+  tf.reduce_mean(x, 0)  # [1.5, 1.5]
+  tf.reduce_mean(x, 1)  # [1.,  2.]
+  ```
+
+  Args:
+    input_tensor: The tensor to reduce. Should have numeric type.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.mean
+
+  Please note that `np.mean` has a `dtype` parameter that could be used to
+  specify the output type. By default this is `dtype=float64`. On the other
+  hand, `tf.reduce_mean` has an aggressive type inference from `input_tensor`,
+  for example:
+
+  ```python
+  x = tf.constant([1, 0, 1, 0])
+  tf.reduce_mean(x)  # 0
+  y = tf.constant([1., 0., 1., 0.])
+  tf.reduce_mean(y)  # 0.5
+  ```
+
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops.mean(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
 
 
 @tf_export("math.reduce_variance")
-def reduce_variance(input_tensor, axis=None, keepdims=None, name=None):
+def reduce_variance(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the variance of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1665,7 +1753,7 @@ def reduce_variance(input_tensor, axis=None, keepdims=None, name=None):
 
 
 @tf_export("math.reduce_std")
-def reduce_std(input_tensor, axis=None, keepdims=None, name=None):
+def reduce_std(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the standard deviation of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1710,15 +1798,8 @@ def reduce_std(input_tensor, axis=None, keepdims=None, name=None):
     return sqrt(variance)
 
 
-@tf_export("math.reduce_prod", "reduce_prod")
-@deprecation.deprecated_args(
-    None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_prod(input_tensor,
-                axis=None,
-                keepdims=None,
-                name=None,
-                reduction_indices=None,
-                keep_dims=None):
+@tf_export("math.reduce_prod", "reduce_prod", v1=[])
+def reduce_prod(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the product of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1736,6 +1817,48 @@ def reduce_prod(input_tensor,
       `[-rank(input_tensor), rank(input_tensor))`.
     keepdims: If true, retains reduced dimensions with length 1.
     name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.prod
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops.prod(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
+
+
+@tf_export(v1=["math.reduce_prod", "reduce_prod"])
+@deprecation.deprecated_args(
+    None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
+def reduce_prod_v1(input_tensor,
+                   axis=None,
+                   keepdims=None,
+                   name=None,
+                   reduction_indices=None,
+                   keep_dims=None):
+  """Computes the product of elements across dimensions of a tensor.
+
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
+
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  Args:
+    input_tensor: The tensor to reduce. Should have numeric type.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
     keep_dims: Deprecated alias for `keepdims`.
 
@@ -1746,29 +1869,22 @@ def reduce_prod(input_tensor,
   Equivalent to np.prod
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
+  return reduce_prod(input_tensor, axis, keepdims, name)
 
-  if keepdims is None:
-    keepdims = False
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops.prod(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
 
-
-@tf_export("math.reduce_min", "reduce_min")
+@tf_export(v1=["math.reduce_min", "reduce_min"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_min(input_tensor,
-               axis=None,
-               keepdims=None,
-               name=None,
-               reduction_indices=None,
-               keep_dims=None):
+def reduce_min_v1(input_tensor,
+                  axis=None,
+                  keepdims=None,
+                  name=None,
+                  reduction_indices=None,
+                  keep_dims=None):
   """Computes the minimum of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1781,9 +1897,9 @@ def reduce_min(input_tensor,
 
   Args:
     input_tensor: The tensor to reduce. Should have real numeric type.
-    axis: The dimensions to reduce. If `None` (the default),
-      reduces all dimensions. Must be in the range
-      `[-rank(input_tensor), rank(input_tensor))`.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
     keepdims: If true, retains reduced dimensions with length 1.
     name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
@@ -1796,28 +1912,57 @@ def reduce_min(input_tensor,
   Equivalent to np.min
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
-  if keepdims is None:
-    keepdims = False
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops._min(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
+  return reduce_min(input_tensor, axis, keepdims, name)
+
+
+@tf_export("math.reduce_min", "reduce_min", v1=[])
+def reduce_min(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes the minimum of elements across dimensions of a tensor.
 
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
+
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
 
-@tf_export("math.reduce_max", "reduce_max")
+  Args:
+    input_tensor: The tensor to reduce. Should have real numeric type.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.min
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops._min(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
+
+
+@tf_export(v1=["math.reduce_max", "reduce_max"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_max(input_tensor,
-               axis=None,
-               keepdims=None,
-               name=None,
-               reduction_indices=None,
-               keep_dims=None):
+def reduce_max_v1(input_tensor,
+                  axis=None,
+                  keepdims=None,
+                  name=None,
+                  reduction_indices=None,
+                  keep_dims=None):
   """Computes the maximum of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1845,28 +1990,57 @@ def reduce_max(input_tensor,
   Equivalent to np.max
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
-  if keepdims is None:
-    keepdims = False
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops._max(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
+  return reduce_max(input_tensor, axis, keepdims, name)
+
+
+@tf_export("math.reduce_max", "reduce_max", v1=[])
+def reduce_max(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes the maximum of elements across dimensions of a tensor.
 
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
 
-@tf_export("math.reduce_all", "reduce_all")
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  Args:
+    input_tensor: The tensor to reduce. Should have real numeric type.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.max
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops._max(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
+
+
+@tf_export(v1=["math.reduce_all", "reduce_all"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_all(input_tensor,
-               axis=None,
-               keepdims=None,
-               name=None,
-               reduction_indices=None,
-               keep_dims=None):
+def reduce_all_v1(input_tensor,
+                  axis=None,
+                  keepdims=None,
+                  name=None,
+                  reduction_indices=None,
+                  keep_dims=None):
   """Computes the "logical and" of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1888,9 +2062,9 @@ def reduce_all(input_tensor,
 
   Args:
     input_tensor: The boolean tensor to reduce.
-    axis: The dimensions to reduce. If `None` (the default),
-      reduces all dimensions. Must be in the range
-      `[-rank(input_tensor), rank(input_tensor))`.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
     keepdims: If true, retains reduced dimensions with length 1.
     name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
@@ -1903,28 +2077,66 @@ def reduce_all(input_tensor,
   Equivalent to np.all
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
-  if keepdims is None:
-    keepdims = False
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops._all(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
+  return reduce_all(input_tensor, axis, keepdims, name)
+
 
+@tf_export("reduce_all", "math.reduce_all", v1=[])
+def reduce_all(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes the "logical and" of elements across dimensions of a tensor.
+
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
 
-@tf_export("math.reduce_any", "reduce_any")
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  For example:
+
+  ```python
+  x = tf.constant([[True,  True], [False, False]])
+  tf.reduce_all(x)  # False
+  tf.reduce_all(x, 0)  # [False, False]
+  tf.reduce_all(x, 1)  # [True, False]
+  ```
+
+  Args:
+    input_tensor: The boolean tensor to reduce.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.all
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops._all(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
+
+
+@tf_export(v1=["math.reduce_any", "reduce_any"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_any(input_tensor,
-               axis=None,
-               keepdims=None,
-               name=None,
-               reduction_indices=None,
-               keep_dims=None):
+def reduce_any_v1(input_tensor,
+                  axis=None,
+                  keepdims=None,
+                  name=None,
+                  reduction_indices=None,
+                  keep_dims=None):
   """Computes the "logical or" of elements across dimensions of a tensor.
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -1946,9 +2158,9 @@ def reduce_any(input_tensor,
 
   Args:
     input_tensor: The boolean tensor to reduce.
-    axis: The dimensions to reduce. If `None` (the default),
-      reduces all dimensions. Must be in the range
-      `[-rank(input_tensor), rank(input_tensor))`.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
     keepdims: If true, retains reduced dimensions with length 1.
     name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
@@ -1961,28 +2173,66 @@ def reduce_any(input_tensor,
   Equivalent to np.any
   @end_compatibility
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
-  if keepdims is None:
-    keepdims = False
-  return _may_reduce_to_scalar(keepdims, axis, reduction_indices,
-                               gen_math_ops._any(
-                                   input_tensor,
-                                   _ReductionDims(input_tensor, axis,
-                                                  reduction_indices),
-                                   keepdims,
-                                   name=name))
+  return reduce_any(input_tensor, axis, keepdims, name)
+
+
+@tf_export("math.reduce_any", "reduce_any", v1=[])
+def reduce_any(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes the "logical or" of elements across dimensions of a tensor.
+
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
 
+  If `axis` is None, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  For example:
+
+  ```python
+  x = tf.constant([[True,  True], [False, False]])
+  tf.reduce_any(x)  # True
+  tf.reduce_any(x, 0)  # [True, True]
+  tf.reduce_any(x, 1)  # [True, False]
+  ```
 
-@tf_export("math.reduce_logsumexp", "reduce_logsumexp")
+  Args:
+    input_tensor: The boolean tensor to reduce.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+
+  @compatibility(numpy)
+  Equivalent to np.any
+  @end_compatibility
+  """
+  keepdims = False if keepdims is None else keepdims
+  return _may_reduce_to_scalar(
+      keepdims, axis,
+      gen_math_ops._any(
+          input_tensor, _ReductionDims(input_tensor, axis), keepdims,
+          name=name))
+
+
+@tf_export(v1=["math.reduce_logsumexp", "reduce_logsumexp"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
-def reduce_logsumexp(input_tensor,
-                     axis=None,
-                     keepdims=None,
-                     name=None,
-                     reduction_indices=None,
-                     keep_dims=None):
+def reduce_logsumexp_v1(input_tensor,
+                        axis=None,
+                        keepdims=None,
+                        name=None,
+                        reduction_indices=None,
+                        keep_dims=None):
   """Computes log(sum(exp(elements across dimensions of a tensor))).
 
   Reduces `input_tensor` along the dimensions given in `axis`.
@@ -2010,9 +2260,9 @@ def reduce_logsumexp(input_tensor,
 
   Args:
     input_tensor: The tensor to reduce. Should have numeric type.
-    axis: The dimensions to reduce. If `None` (the default),
-      reduces all dimensions. Must be in the range
-      `[-rank(input_tensor), rank(input_tensor))`.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
     keepdims: If true, retains reduced dimensions with length 1.
     name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
@@ -2021,16 +2271,57 @@ def reduce_logsumexp(input_tensor,
   Returns:
     The reduced tensor.
   """
+  axis = deprecation.deprecated_argument_lookup(
+      "axis", axis, "reduction_indices", reduction_indices)
   keepdims = deprecation.deprecated_argument_lookup("keepdims", keepdims,
                                                     "keep_dims", keep_dims)
-  if keepdims is None:
-    keepdims = False
+  return reduce_logsumexp(input_tensor, axis, keepdims, name)
+
+
+@tf_export("math.reduce_logsumexp", "reduce_logsumexp", v1=[])
+def reduce_logsumexp(input_tensor, axis=None, keepdims=False, name=None):
+  """Computes log(sum(exp(elements across dimensions of a tensor))).
+
+  Reduces `input_tensor` along the dimensions given in `axis`.
+  Unless `keepdims` is true, the rank of the tensor is reduced by 1 for each
+  entry in `axis`. If `keepdims` is true, the reduced dimensions
+  are retained with length 1.
+
+  If `axis` has no entries, all dimensions are reduced, and a
+  tensor with a single element is returned.
+
+  This function is more numerically stable than log(sum(exp(input))). It avoids
+  overflows caused by taking the exp of large inputs and underflows caused by
+  taking the log of small inputs.
+
+  For example:
+
+  ```python
+  x = tf.constant([[0., 0., 0.], [0., 0., 0.]])
+  tf.reduce_logsumexp(x)  # log(6)
+  tf.reduce_logsumexp(x, 0)  # [log(2), log(2), log(2)]
+  tf.reduce_logsumexp(x, 1)  # [log(3), log(3)]
+  tf.reduce_logsumexp(x, 1, keepdims=True)  # [[log(3)], [log(3)]]
+  tf.reduce_logsumexp(x, [0, 1])  # log(6)
+  ```
+
+  Args:
+    input_tensor: The tensor to reduce. Should have numeric type.
+    axis: The dimensions to reduce. If `None` (the default), reduces all
+      dimensions. Must be in the range `[-rank(input_tensor),
+      rank(input_tensor))`.
+    keepdims: If true, retains reduced dimensions with length 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    The reduced tensor.
+  """
+  keepdims = False if keepdims is None else keepdims
   input_tensor = ops.convert_to_tensor(input_tensor)
   with ops.name_scope(name, "ReduceLogSumExp", [input_tensor]) as name:
     raw_max = reduce_max(
         input_tensor,
         axis=axis,
-        reduction_indices=reduction_indices,
         keepdims=True)
     my_max = array_ops.stop_gradient(
         array_ops.where(
@@ -2040,12 +2331,11 @@ def reduce_logsumexp(input_tensor,
         reduce_sum(
             gen_math_ops.exp(gen_math_ops.sub(input_tensor, my_max)),
             axis,
-            keepdims=keepdims,
-            reduction_indices=reduction_indices))
+            keepdims=keepdims))
     if not keepdims:
       my_max = array_ops.reshape(my_max, array_ops.shape(result))
     result = gen_math_ops.add(result, my_max)
-    return _may_reduce_to_scalar(keepdims, axis, reduction_indices, result)
+    return _may_reduce_to_scalar(keepdims, axis, result)
 
 
 @tf_export("linalg.trace", v1=["linalg.trace", "trace"])
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index e0329f66ff..cd45b6f136 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -104,7 +104,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
     for dtype in [np.float16, np.float32, np.double]:
       x_np = np.random.rand(5, 5).astype(dtype)
       with self.cached_session(use_gpu=True):
-        y_tf = math_ops.reduce_logsumexp(x_np, reduction_indices=[0])
+        y_tf = math_ops.reduce_logsumexp(x_np, axis=[0])
         y_np = log(np.sum(exp(x_np), axis=0))
         self.assertShapeEqual(y_np, y_tf)
         y_tf_np = self.evaluate(y_tf)
@@ -114,7 +114,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
     for dtype in [np.float16, np.float32, np.double]:
       x_np = np.random.rand(5, 5).astype(dtype)
       with self.cached_session(use_gpu=True):
-        y_tf = math_ops.reduce_logsumexp(x_np, reduction_indices=0)
+        y_tf = math_ops.reduce_logsumexp(x_np, axis=0)
         y_np = log(np.sum(exp(x_np), axis=0))
         self.assertShapeEqual(y_np, y_tf)
         y_tf_np = self.evaluate(y_tf)
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 0b91b8dde8..27269c51c1 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -948,7 +948,7 @@ def mean_cosine_distance(labels,
       predictions=predictions, labels=labels, weights=weights)
   radial_diffs = math_ops.multiply(predictions, labels)
   radial_diffs = math_ops.reduce_sum(
-      radial_diffs, reduction_indices=[
+      radial_diffs, axis=[
           dim,
       ], keepdims=True)
   mean_distance, update_op = mean(radial_diffs, weights, None, None, name or
@@ -3045,7 +3045,7 @@ def _sparse_average_precision_at_top_k(labels, predictions_idx):
 
     # Reduce along k dimension to get the sum, yielding a [D1, ... DN] tensor.
     precision_sum = math_ops.reduce_sum(
-        relevant_precision_per_k, reduction_indices=(-1,), name='precision_sum')
+        relevant_precision_per_k, axis=(-1,), name='precision_sum')
 
     # Divide by number of relevant items to get average precision. These are
     # the "num_relevant_items" and "AveP" terms from the formula above.
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 4470c0b958..72db0952b4 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -1324,13 +1324,12 @@ class ControlFlowTest(PForTest):
     pfor_out, pfor_out_grad = pfor_control_flow_ops.pfor(loop_fn, 4)
     # Note that tf.while_loop does not work in the setup above. So we manually
     # construct the equivalent computation of the above loops here.
-    real_out = math_ops.reduce_sum(inp, reduction_indices=[0])
-    real_out = math_ops.reduce_prod(real_out, reduction_indices=[1])
+    real_out = math_ops.reduce_sum(inp, axis=[0])
+    real_out = math_ops.reduce_prod(real_out, axis=[1])
     # Note that gradients of real_out will accumulate the gradients across the
     # output value. Hence we do the same aggregation on pfor_out_grad.
     real_out_grad = gradient_ops.gradients(real_out, inp)[0]
-    sum_pfor_out_grad = math_ops.reduce_sum(
-        pfor_out_grad, reduction_indices=[0])
+    sum_pfor_out_grad = math_ops.reduce_sum(pfor_out_grad, axis=[0])
 
     with session.Session() as sess:
       v1, v2, v1_grad, v2_grad = sess.run(
diff --git a/tensorflow/python/ops/sparse_grad.py b/tensorflow/python/ops/sparse_grad.py
index 1223b290ff..2ca9c0c647 100644
--- a/tensorflow/python/ops/sparse_grad.py
+++ b/tensorflow/python/ops/sparse_grad.py
@@ -195,7 +195,7 @@ def _SparseTensorDenseMatMulGrad(op, grad):
   parts_a = array_ops.gather(grad, rows if not adj_a else cols)
   parts_b = array_ops.gather(b if not adj_b else array_ops.transpose(b),
                              cols if not adj_a else rows)
-  a_values_grad = math_ops.reduce_sum(parts_a * parts_b, reduction_indices=1)
+  a_values_grad = math_ops.reduce_sum(parts_a * parts_b, axis=1)
 
   # gradients w.r.t. (a_indices, a_values, a_shape, b)
   return (None, a_values_grad, None, b_grad)
diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py
index f44f694109..21f4996798 100644
--- a/tensorflow/python/ops/special_math_ops.py
+++ b/tensorflow/python/ops/special_math_ops.py
@@ -70,8 +70,7 @@ def lbeta(x, name=None):
     x = ops.convert_to_tensor(x, name='x')
 
     # Note reduce_sum([]) = 0.
-    log_prod_gamma_x = math_ops.reduce_sum(
-        math_ops.lgamma(x), reduction_indices=[-1])
+    log_prod_gamma_x = math_ops.reduce_sum(math_ops.lgamma(x), axis=[-1])
 
     # Note lgamma(0) = infinity, so if x = []
     # log_gamma_sum_x = lgamma(0) = infinity, and
@@ -264,11 +263,11 @@ def einsum(equation, *inputs, **kwargs):
 
     missing_indices = set(temp_axis_labels) - set(output_axis_labels)
     if missing_indices:
-      reduction_indices = [
+      axis = [
           i for i, a in enumerate(temp_axis_labels)
           if a not in output_axis_labels
       ]
-      temp = math_ops.reduce_sum(temp, reduction_indices=reduction_indices)
+      temp = math_ops.reduce_sum(temp, axis=axis)
       temp_axis_labels = ''.join(
           a for a in temp_axis_labels if a in output_axis_labels)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
index 67f348be21..b7a99caeb7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
@@ -318,7 +318,7 @@ tf_module {
   }
   member_method {
     name: "reduce_std"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_sum"
@@ -326,7 +326,7 @@ tf_module {
   }
   member_method {
     name: "reduce_variance"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "rint"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index 86df970514..5215cfbab0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -290,43 +290,43 @@ tf_module {
   }
   member_method {
     name: "reduce_all"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_any"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_logsumexp"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_max"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_mean"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_min"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_prod"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_std"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_sum"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_variance"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "rint"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 0649ae267b..873c41a390 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -898,35 +898,35 @@ tf_module {
   }
   member_method {
     name: "reduce_all"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_any"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_logsumexp"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_max"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_mean"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_min"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_prod"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "reduce_sum"
-    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\', \'reduction_indices\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input_tensor\', \'axis\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "register_tensor_conversion_function"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index f85b2f08dc..aab7aa8af5 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -73,6 +73,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "t": "x",
             "msg": "message",
         },
+        "tf.sparse.add": ["a", "b", "thresh"],
         "tf.sparse.split": {
             "split_dim": "axis",
         },
@@ -113,6 +114,73 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
+        "tf.sparse.concat": [
+            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
+        ],
+        "tf.reduce_all": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_all": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_any": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_any": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_min": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_min": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_max": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_max": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_sum": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_sum": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_mean": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_mean": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_prod": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_prod": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.reduce_logsumexp": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.math.reduce_logsumexp": {
+            "reduction_indices": "axis",
+            "keep_dims": "keepdims",
+        },
     }
 
     # Mapping from function to the new name of the function
@@ -199,7 +267,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
         "tf.nn.convolution": [
             "input", "filter", "padding", "strides", "dilation_rate", "name",
-            "data_format"],
+            "data_format"
+        ],
         "tf.nn.crelu": ["features", "name", "axis"],
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
@@ -218,19 +287,19 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         ],
         "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
         "tf.quantize_v2": [
-            "input", "min_range", "max_range", "T", "mode", "name",
-            "round_mode"
+            "input", "min_range", "max_range", "T", "mode", "name", "round_mode"
         ],
         "tf.feature_column.categorical_column_with_vocabulary_file": [
-            "key", "vocabulary_file", "vocabulary_size",
-            "num_oov_buckets", "default_value", "dtype"
+            "key", "vocabulary_file", "vocabulary_size", "num_oov_buckets",
+            "default_value", "dtype"
         ],
         "tf.shape": ["input", "name", "out_type"],
         "tf.size": ["input", "name", "out_type"],
+        "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
+        "tf.sparse.add": ["a", "b", "thresh"],
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
-        "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
         "tf.sparse.segment_mean": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
@@ -243,10 +312,75 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.strings.length": ["input", "name", "unit"],
         "tf.transpose": ["a", "perm", "name", "conjugate"],
         "tf.tuple": ["tensors", "name", "control_inputs"],
-        "tf.while_loop": ["cond", "body", "loop_vars", "shape_invariants",
-                          "parallel_iterations", "back_prop", "swap_memory",
-                          "name", "maximum_iterations",
-                          "return_same_structure"],
+        "tf.while_loop": [
+            "cond", "body", "loop_vars", "shape_invariants",
+            "parallel_iterations", "back_prop", "swap_memory", "name",
+            "maximum_iterations", "return_same_structure"
+        ],
+        "tf.reduce_all": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_all": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_any": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_any": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_min": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_min": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_max": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_max": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_sum": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_sum": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_mean": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_mean": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_prod": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_prod": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.reduce_logsumexp": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
+        "tf.math.reduce_logsumexp": [
+            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
+            "keep_dims"
+        ],
     }
 
     # Specially handled functions.
-- 
GitLab


From c59719cf1f29a98cfc027b9c56f2028dff887663 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 21 Nov 2018 10:03:25 -0800
Subject: [PATCH 0669/1554] Expose amsgrad argument for Adam, but error out if
 it is set to True.

PiperOrigin-RevId: 222424267
---
 tensorflow/python/keras/optimizer_v2/adam.py  | 56 +++++++++++++++----
 .../python/keras/optimizer_v2/adam_test.py    |  5 ++
 .../python/keras/optimizer_v2/adamax.py       |  8 ++-
 3 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 962680fad6..fd5918dbfa 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -35,9 +35,13 @@ class Adam(optimizer_v2.OptimizerV2):
   requirement, invariant to diagonal rescaling of gradients, and is well suited
   for problems that are large in terms of data/parameters'.
 
+  Note, amsgrad is currently not supported and the argument can only be False.
+
   # References
       See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980)
         ([pdf](http://arxiv.org/pdf/1412.6980.pdf)).
+      For AMSGrad see [Reddi et al., 2-18]
+        (https://openreview.net/pdf?id=ryQu7f-RZ)
   """
 
   def __init__(self,
@@ -45,26 +49,47 @@ class Adam(optimizer_v2.OptimizerV2):
                beta_1=0.9,
                beta_2=0.999,
                epsilon=1e-7,
+               amsgrad=False,
                name='Adam'):
     r"""Construct a new Adam optimizer.
 
-    Initialization:
+    If amsgrad = False:
+      Initialization:
+
+      $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$
+      $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
+      $$t := 0 \text{(Initialize timestep)}$$
+
+      The update rule for `variable` with gradient `g` uses an optimization
+      described at the end of section2 of the paper:
+
+      $$t := t + 1$$
+      $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+
+      $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+      $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+      $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$
+
+    If amsgrad = True:
+      Initialization:
 
-    $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$
-    $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
-    $$t := 0 \text{(Initialize timestep)}$$
+      $$m_0 := 0 \text{(Initialize initial 1st moment vector)}$$
+      $$v_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
+      $$v_hat_0 := 0 \text{(Initialize initial 2nd moment vector)}$$
+      $$t := 0 \text{(Initialize timestep)}$$
 
-    The update rule for `variable` with gradient `g` uses an optimization
-    described at the end of section2 of the paper:
+      The update rule for `variable` with gradient `g` uses an optimization
+      described at the end of section2 of the paper:
 
-    $$t := t + 1$$
-    $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+      $$t := t + 1$$
+      $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
 
-    $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
-    $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
-    $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$
+      $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+      $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+      $$v_hat_t := max(v_hat_{t-1}, v_t)
+      $$variable := variable - lr_t * m_t / (\sqrt{v_hat_t} + \epsilon)$$
 
-    The default value of 1e-8 for epsilon might not be a good default in
+    The default value of 1e-7 for epsilon might not be a good default in
     general. For example, when training an Inception network on ImageNet a
     current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the
     formulation just before Section 2.1 of the Kingma and Ba paper rather than
@@ -89,6 +114,8 @@ class Adam(optimizer_v2.OptimizerV2):
       epsilon: A small constant for numerical stability. This epsilon is
         "epsilon hat" in the Kingma and Ba paper (in the formula just before
         Section 2.1), not the epsilon in Algorithm 1 of the paper.
+      amsgrad: boolean. Whether to apply AMSGrad variant of this algorithm from
+        the paper "On the Convergence of Adam and beyond".
       name: Optional name for the operations created when applying gradients.
         Defaults to "Adam".  @compatibility(eager) When eager execution is
         enabled, `learning_rate`, `beta_1`, `beta_2`, and `epsilon` can each be
@@ -102,6 +129,10 @@ class Adam(optimizer_v2.OptimizerV2):
     self._set_hyper('beta_1', beta_1)
     self._set_hyper('beta_2', beta_2)
     self._set_hyper('epsilon', epsilon)
+    # TODO(tanzheny): create op for resource_apply_adam_with_amsgrad
+    if amsgrad:
+      raise ValueError('Amsgrad is currently not supported.')
+    self._amsgrad = amsgrad
 
   def _create_slots(self, var_list):
     # Create slots for the first and second moments.
@@ -173,5 +204,6 @@ class Adam(optimizer_v2.OptimizerV2):
         'beta_1': self._serialize_hyperparameter('beta_1'),
         'beta_2': self._serialize_hyperparameter('beta_2'),
         'epsilon': self._serialize_hyperparameter('epsilon'),
+        'amsgrad': self._amsgrad,
     })
     return config
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 46a45af224..20780ead9c 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -298,6 +298,11 @@ class AdamOptimizerTest(test.TestCase):
       # variables for v1 and v2 respectively.
       self.assertEqual(9, len(set(opt.variables())))
 
+  def testAmsgradWithError(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 "Amsgrad is currently not supported"):
+      adam.Adam(learning_rate=1., beta_1=0.9, beta_2=0.99, amsgrad=True)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 7530e629d1..67b678f862 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -89,7 +89,13 @@ class Adamax(adam.Adam):
         Defaults to "Adamax".
     """
     # pylint: disable=useless-super-delegation
-    super(Adamax, self).__init__(learning_rate, beta_1, beta_2, epsilon, name)
+    super(Adamax, self).__init__(
+        learning_rate=learning_rate,
+        beta_1=beta_1,
+        beta_2=beta_2,
+        epsilon=epsilon,
+        amsgrad=False,
+        name=name)
     # pylint: enable=useless-super-delegation
 
   def _resource_apply_dense(self, grad, var):
-- 
GitLab


From 42e2bb488ad7571317f23cbdd5851227140b1985 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Wed, 21 Nov 2018 10:07:11 -0800
Subject: [PATCH 0670/1554] Fix bug that resulted in unnecessary placeholders
 created during functional model cloning when input tensors were specified.

PiperOrigin-RevId: 222424883
---
 tensorflow/python/keras/engine/input_layer.py | 28 +++++++++---------
 tensorflow/python/keras/models.py             | 14 +++++----
 tensorflow/python/keras/models_test.py        | 29 +++++++++++++++++++
 3 files changed, 50 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index 590b935d40..9874efe2bc 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -19,12 +19,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.utils import tf_utils
-from tensorflow.python.ops import array_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -94,19 +92,19 @@ class InputLayer(base_layer.Layer):
       else:
         batch_input_shape = None
       graph = backend.get_graph()
-      with context.graph_mode():
-        with graph.as_default():
-          # In graph mode, create a graph placeholder to call the layer on.
-          if sparse:
-            input_tensor = array_ops.sparse_placeholder(
-                shape=batch_input_shape,
-                dtype=dtype,
-                name=self.name)
-          else:
-            input_tensor = array_ops.placeholder(
-                shape=batch_input_shape,
-                dtype=dtype,
-                name=self.name)
+      with graph.as_default():
+        # In graph mode, create a graph placeholder to call the layer on.
+        if sparse:
+          input_tensor = backend.placeholder(
+              shape=batch_input_shape,
+              dtype=dtype,
+              name=self.name,
+              sparse=True)
+        else:
+          input_tensor = backend.placeholder(
+              shape=batch_input_shape,
+              dtype=dtype,
+              name=self.name)
 
       self.is_placeholder = True
       self._batch_input_shape = batch_input_shape
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 3a0c51b497..4813b8061e 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -100,17 +100,19 @@ def _clone_functional_model(model, input_tensors=None):
       input_tensors = list(input_tensors)
     input_tensors = generic_utils.to_list(input_tensors)
     input_tensors_ = []
-    for i, x in enumerate(input_tensors):
-      if not K.is_keras_tensor(x):
-        name = model._input_layers[i].name
-        input_tensor = Input(tensor=x, name='input_wrapper_for_' + name)
+    for i in range(len(input_tensors)):
+      input_tensor = input_tensors[i]
+      if not K.is_keras_tensor(input_tensor):
+        original_input_layer = model._input_layers[i]
+        name = original_input_layer.name
+        input_tensor = Input(tensor=input_tensor,
+                             name='input_wrapper_for_' + name)
         input_tensors_.append(input_tensor)
         # Cache newly created input layer.
-        original_input_layer = x._keras_history[0]
         newly_created_input_layer = input_tensor._keras_history[0]
         layer_map[original_input_layer] = newly_created_input_layer
       else:
-        input_tensors_.append(x)
+        input_tensors_.append(input_tensor)
     input_tensors = input_tensors_
 
   for x, y in zip(model.inputs, input_tensors):
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 4b6bb74ef9..23321a2d16 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -26,10 +26,12 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import metrics
 from tensorflow.python.keras import models
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
@@ -219,6 +221,33 @@ class TestModelCloning(test.TestCase):
     with self.assertRaises(ValueError):
       keras.models._clone_sequential_model(seq_model, input_tensors=y)
 
+  def test_functional_cloning_does_not_create_unnecessary_placeholders(self):
+    with ops.Graph().as_default():
+      x = keras.Input((4,))
+      y = keras.layers.Dense(4)(x)
+      model = keras.models.Model(x, y)
+    graph = ops.Graph()
+    with graph.as_default():
+      x = array_ops.ones((10, 4))
+      _ = keras.models.clone_model(model, input_tensors=[x])
+      has_placeholder = _has_placeholder(graph)
+      self.assertFalse(has_placeholder)
+
+  def test_sequential_cloning_does_not_create_unnecessary_placeholders(self):
+    with ops.Graph().as_default():
+      model = keras.models.Sequential([keras.layers.Dense(4)])
+    graph = ops.Graph()
+    with graph.as_default():
+      x = array_ops.ones((10, 4))
+      _ = keras.models.clone_model(model, input_tensors=[x])
+      has_placeholder = _has_placeholder(graph)
+      self.assertFalse(has_placeholder)
+
+
+def _has_placeholder(graph):
+  ops_types = [op.type for op in graph.get_operations()]
+  return any('Placeholder' in s for s in ops_types)
+
 
 class CheckpointingTests(test.TestCase):
 
-- 
GitLab


From a38f4113eeef01da0d38f467ba36dc0eb86a73ba Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Wed, 21 Nov 2018 10:08:58 -0800
Subject: [PATCH 0671/1554] Marked create_partitioned_variables as @deprecated

It has been deprecated for a while, but did not include a relevant note
in the docs.

PiperOrigin-RevId: 222425114
---
 tensorflow/python/ops/partitioned_variables.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py
index 6174979d5e..5ba1d46a66 100644
--- a/tensorflow/python/ops/partitioned_variables.py
+++ b/tensorflow/python/ops/partitioned_variables.py
@@ -57,7 +57,7 @@ import math
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 __all__ = [
@@ -238,6 +238,9 @@ def fixed_size_partitioner(num_shards, axis=0):
 
 
 @tf_export("create_partitioned_variables")
+@deprecation.deprecated(
+    date=None,
+    instructions="Use tf.get_variable with a partitioner set.")
 def create_partitioned_variables(
     shape, slicing, initializer, dtype=dtypes.float32,
     trainable=True, collections=None, name=None, reuse=None):
@@ -282,11 +285,6 @@ def create_partitioned_variables(
   Raises:
     ValueError: If any of the arguments is malformed.
   """
-  logging.warn(
-      "create_partitioned_variables is deprecated.  Use "
-      "tf.get_variable with a partitioner set, or "
-      "tf.get_partitioned_variable_list, instead.")
-
   if len(shape) != len(slicing):
     raise ValueError("The 'shape' and 'slicing' of a partitioned Variable "
                      "must have the length: shape: %s, slicing: %s" %
-- 
GitLab


From 086e7c3474a247dffb3ddc179197217e38c103ed Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Wed, 21 Nov 2018 10:17:59 -0800
Subject: [PATCH 0672/1554] Upgrade ARM_NEON_2_x86_SSE to 1200fe9 to fix
 warnings

PiperOrigin-RevId: 222426343
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 101d0e4c7a..5c58bb1080 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -689,11 +689,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "arm_neon_2_x86_sse",
         build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
-        sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5",
-        strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d",
+        sha256 = "213733991310b904b11b053ac224fee2d4e0179e46b52fe7f8735b8831e04dcc",
+        strip_prefix = "ARM_NEON_2_x86_SSE-1200fe90bb174a6224a525ee60148671a786a71f",
         urls = [
-            "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
-            "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
+            "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/1200fe90bb174a6224a525ee60148671a786a71f.tar.gz",
+            "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/1200fe90bb174a6224a525ee60148671a786a71f.tar.gz",
         ],
     )
 
-- 
GitLab


From ed52111074869696825d3dbf5c8bba800f3cefe2 Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Wed, 21 Nov 2018 10:20:32 -0800
Subject: [PATCH 0673/1554] Update image.xxx api to make its main in io.
 Include: image.decode_bmp|gif|jpeg|image, image.is_jpeg,
 image.decode_and_crop_jpeg, image.encode_jpeg, image.extract_jpeg_shape

PiperOrigin-RevId: 222426690
---
 .../api_def_DecodeAndCropJpeg.pbtxt           |  4 +--
 .../python_api/api_def_DecodeBmp.pbtxt        |  4 +--
 .../python_api/api_def_DecodeGif.pbtxt        |  4 +--
 .../python_api/api_def_DecodeJpeg.pbtxt       |  4 +--
 .../python_api/api_def_DecodePng.pbtxt        |  4 +--
 .../python_api/api_def_EncodeJpeg.pbtxt       |  4 +--
 .../python_api/api_def_ExtractJpegShape.pbtxt |  4 +--
 tensorflow/python/ops/image_ops_impl.py       | 24 +++++++++++--
 .../tools/api/golden/v1/tensorflow.io.pbtxt   | 36 +++++++++++++++++++
 .../tools/api/golden/v2/tensorflow.io.pbtxt   | 36 +++++++++++++++++++
 10 files changed, 101 insertions(+), 23 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeAndCropJpeg.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeAndCropJpeg.pbtxt
index fbe9c88253..2c3857cc53 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeAndCropJpeg.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeAndCropJpeg.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "DecodeAndCropJpeg"
-  endpoint {
-    name: "image.decode_and_crop_jpeg"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeBmp.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeBmp.pbtxt
index 573d83f373..ffe19ca8dc 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeBmp.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeBmp.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "DecodeBmp"
-  endpoint {
-    name: "image.decode_bmp"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeGif.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeGif.pbtxt
index eed64df79c..ff68b997e1 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeGif.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeGif.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "DecodeGif"
-  endpoint {
-    name: "image.decode_gif"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeJpeg.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeJpeg.pbtxt
index 994bc4e1f4..97d262abe5 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodeJpeg.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeJpeg.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "DecodeJpeg"
-  endpoint {
-    name: "image.decode_jpeg"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_DecodePng.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodePng.pbtxt
index 309eec5ac3..3b9290a2c5 100644
--- a/tensorflow/core/api_def/python_api/api_def_DecodePng.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_DecodePng.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "DecodePng"
-  endpoint {
-    name: "image.decode_png"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_EncodeJpeg.pbtxt b/tensorflow/core/api_def/python_api/api_def_EncodeJpeg.pbtxt
index 5c31e9d0f3..054ffb997b 100644
--- a/tensorflow/core/api_def/python_api/api_def_EncodeJpeg.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_EncodeJpeg.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "EncodeJpeg"
-  endpoint {
-    name: "image.encode_jpeg"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_ExtractJpegShape.pbtxt b/tensorflow/core/api_def/python_api/api_def_ExtractJpegShape.pbtxt
index 6849a6d3fa..a57955c8a7 100644
--- a/tensorflow/core/api_def/python_api/api_def_ExtractJpegShape.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_ExtractJpegShape.pbtxt
@@ -1,6 +1,4 @@
 op {
   graph_op_name: "ExtractJpegShape"
-  endpoint {
-    name: "image.extract_jpeg_shape"
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 1618b79504..229393c970 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -1760,7 +1760,7 @@ def adjust_saturation(image, saturation_factor, name=None):
         orig_dtype)
 
 
-@tf_export('image.is_jpeg')
+@tf_export('io.is_jpeg', 'image.is_jpeg', v1=['io.is_jpeg', 'image.is_jpeg'])
 def is_jpeg(contents, name=None):
   r"""Convenience function to check if the 'contents' encodes a JPEG image.
 
@@ -1795,8 +1795,28 @@ def _is_png(contents, name=None):
     substr = string_ops.substr(contents, 0, 3)
     return math_ops.equal(substr, b'\211PN', name=name)
 
+tf_export('io.decode_and_crop_jpeg', 'image.decode_and_crop_jpeg',
+          v1=['io.decode_and_crop_jpeg', 'image.decode_and_crop_jpeg'])(
+              gen_image_ops.decode_and_crop_jpeg)
 
-@tf_export('image.decode_image')
+tf_export('io.decode_bmp', 'image.decode_bmp',
+          v1=['io.decode_bmp', 'image.decode_bmp'])(gen_image_ops.decode_bmp)
+tf_export('io.decode_gif', 'image.decode_gif',
+          v1=['io.decode_gif', 'image.decode_gif'])(gen_image_ops.decode_gif)
+tf_export('io.decode_jpeg', 'image.decode_jpeg',
+          v1=['io.decode_jpeg', 'image.decode_jpeg'])(gen_image_ops.decode_jpeg)
+tf_export('io.decode_png', 'image.decode_png',
+          v1=['io.decode_png', 'image.decode_png'])(gen_image_ops.decode_png)
+
+tf_export('io.encode_jpeg', 'image.encode_jpeg',
+          v1=['io.encode_jpeg', 'image.encode_jpeg'])(gen_image_ops.encode_jpeg)
+tf_export('io.extract_jpeg_shape', 'image.extract_jpeg_shape',
+          v1=['io.extract_jpeg_shape', 'image.extract_jpeg_shape'])(
+              gen_image_ops.extract_jpeg_shape)
+
+
+@tf_export('io.decode_image', 'image.decode_image',
+           v1=['io.decode_image', 'image.decode_image'])
 def decode_image(contents, channels=None, dtype=dtypes.uint8, name=None):
   """Convenience function for `decode_bmp`, `decode_gif`, `decode_jpeg`,
   and `decode_png`.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt
index 64b63ed1a4..fee12594ee 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt
@@ -44,10 +44,18 @@ tf_module {
     name: "VarLenFeature"
     mtype: "<type \'type\'>"
   }
+  member_method {
+    name: "decode_and_crop_jpeg"
+    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], "
+  }
   member_method {
     name: "decode_base64"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "decode_bmp"
+    argspec: "args=[\'contents\', \'channels\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], "
+  }
   member_method {
     name: "decode_compressed"
     argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], "
@@ -56,10 +64,26 @@ tf_module {
     name: "decode_csv"
     argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], "
   }
+  member_method {
+    name: "decode_gif"
+    argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "decode_image"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'uint8\'>\", \'None\'], "
+  }
+  member_method {
+    name: "decode_jpeg"
+    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], "
+  }
   member_method {
     name: "decode_json_example"
     argspec: "args=[\'json_examples\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "decode_png"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \"<dtype: \'uint8\'>\", \'None\'], "
+  }
   member_method {
     name: "decode_raw"
     argspec: "args=[\'bytes\', \'out_type\', \'little_endian\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
@@ -72,6 +96,18 @@ tf_module {
     name: "encode_base64"
     argspec: "args=[\'input\', \'pad\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
+  member_method {
+    name: "encode_jpeg"
+    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'95\', \'False\', \'False\', \'True\', \'in\', \'300\', \'300\', \'\', \'None\'], "
+  }
+  member_method {
+    name: "extract_jpeg_shape"
+    argspec: "args=[\'contents\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
+  }
+  member_method {
+    name: "is_jpeg"
+    argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "match_filenames_once"
     argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
index caa207b022..1de2fc9e5d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
@@ -48,10 +48,18 @@ tf_module {
     name: "gfile"
     mtype: "<type \'module\'>"
   }
+  member_method {
+    name: "decode_and_crop_jpeg"
+    argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], "
+  }
   member_method {
     name: "decode_base64"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "decode_bmp"
+    argspec: "args=[\'contents\', \'channels\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], "
+  }
   member_method {
     name: "decode_compressed"
     argspec: "args=[\'bytes\', \'compression_type\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'None\'], "
@@ -60,10 +68,26 @@ tf_module {
     name: "decode_csv"
     argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], "
   }
+  member_method {
+    name: "decode_gif"
+    argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "decode_image"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'uint8\'>\", \'None\'], "
+  }
+  member_method {
+    name: "decode_jpeg"
+    argspec: "args=[\'contents\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], "
+  }
   member_method {
     name: "decode_json_example"
     argspec: "args=[\'json_examples\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "decode_png"
+    argspec: "args=[\'contents\', \'channels\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \"<dtype: \'uint8\'>\", \'None\'], "
+  }
   member_method {
     name: "decode_raw"
     argspec: "args=[\'bytes\', \'out_type\', \'little_endian\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
@@ -76,6 +100,18 @@ tf_module {
     name: "encode_base64"
     argspec: "args=[\'input\', \'pad\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
+  member_method {
+    name: "encode_jpeg"
+    argspec: "args=[\'image\', \'format\', \'quality\', \'progressive\', \'optimize_size\', \'chroma_downsampling\', \'density_unit\', \'x_density\', \'y_density\', \'xmp_metadata\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'95\', \'False\', \'False\', \'True\', \'in\', \'300\', \'300\', \'\', \'None\'], "
+  }
+  member_method {
+    name: "extract_jpeg_shape"
+    argspec: "args=[\'contents\', \'output_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
+  }
+  member_method {
+    name: "is_jpeg"
+    argspec: "args=[\'contents\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "match_filenames_once"
     argspec: "args=[\'pattern\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 408597998261a335c2311f674a846b0dd160de2b Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Wed, 21 Nov 2018 10:36:21 -0800
Subject: [PATCH 0674/1554] Support DT_INT32 for Cumsum/Cumprod.

PiperOrigin-RevId: 222429176
---
 tensorflow/compiler/tests/scan_ops_test.py     | 4 ++--
 tensorflow/compiler/tf2xla/kernels/scan_ops.cc | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tests/scan_ops_test.py b/tensorflow/compiler/tests/scan_ops_test.py
index 897db384b7..17639bd8a7 100644
--- a/tensorflow/compiler/tests/scan_ops_test.py
+++ b/tensorflow/compiler/tests/scan_ops_test.py
@@ -71,7 +71,7 @@ def handle_options(func, x, axis, exclusive, reverse):
 
 class CumsumTest(xla_test.XLATestCase):
 
-  valid_dtypes = [np.float32]
+  valid_dtypes = [np.float32, np.int32]
 
   def axis_dtypes(self):
     return set(self.int_types).intersection([np.int32, np.int64])
@@ -149,7 +149,7 @@ class CumsumTest(xla_test.XLATestCase):
 
 class CumprodTest(xla_test.XLATestCase):
 
-  valid_dtypes = [np.float32]
+  valid_dtypes = [np.float32, np.int32]
 
   def axis_dtypes(self):
     return set(self.int_types).intersection([np.int32, np.int64])
diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
index b5fd7850bf..7f4fef146f 100644
--- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
@@ -39,8 +39,8 @@ namespace {
 
 // TODO(phawkins): implement double-sized windowed reductions in XLA and remove
 // the type constraint.
-constexpr std::array<DataType, 3> kScanOpTypes = {
-    {DT_HALF, DT_BFLOAT16, DT_FLOAT}};
+constexpr std::array<DataType, 4> kScanOpTypes = {
+    {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_INT32}};
 
 class ScanOp : public XlaOpKernel {
  public:
-- 
GitLab


From 95d7bbb2fcfda083ab69f8e73d0719013cd9c3d4 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Wed, 21 Nov 2018 10:40:10 -0800
Subject: [PATCH 0675/1554] Replaced list comprehensions inside all/any/sum
 with generator expressions

PiperOrigin-RevId: 222429778
---
 configure.py                                     |  2 +-
 tensorflow/compiler/tests/dense_layer_test.py    |  2 +-
 tensorflow/compiler/tests/jit_test.py            |  2 +-
 .../python/constrained_minimization_problem.py   |  4 ++--
 .../python/kernel_tests/cudnn_rnn_ops_test.py    |  6 ++----
 .../python/kernel_tests/cudnn_rnn_test.py        |  4 ++--
 .../distribute/python/minimize_loss_test.py      |  2 +-
 .../contrib/distribute/python/tpu_strategy.py    |  2 +-
 .../gan/python/losses/python/losses_impl.py      |  4 +---
 tensorflow/contrib/gan/python/train_test.py      |  2 +-
 .../kernel_methods/python/kernel_estimators.py   |  2 +-
 .../layers/python/layers/regularizers_test.py    |  2 +-
 .../contrib/learn/python/learn/estimators/dnn.py |  4 ++--
 .../learn/estimators/dnn_linear_combined.py      |  8 ++++----
 .../learn/python/learn/estimators/estimator.py   |  6 +++---
 .../learn/python/learn/estimators/linear.py      |  4 ++--
 tensorflow/contrib/quantize/python/quantize.py   |  4 ++--
 .../kernel_tests/attention_wrapper_test.py       |  4 ++--
 tensorflow/contrib/tpu/python/tpu/tpu.py         |  2 +-
 tensorflow/python/autograph/utils/type_check.py  |  2 +-
 .../optimization/map_vectorization_test.py       |  2 +-
 .../filter_dataset_serialization_test.py         |  6 ++----
 .../python/data/experimental/ops/parsing_ops.py  |  4 ++--
 .../python/data/kernel_tests/dataset_ops_test.py |  2 +-
 tensorflow/python/data/util/sparse.py            |  2 +-
 tensorflow/python/distribute/cross_device_ops.py | 16 ++++++++--------
 .../python/distribute/cross_device_utils.py      |  2 +-
 tensorflow/python/eager/backprop.py              |  6 +++---
 tensorflow/python/eager/function.py              |  4 ++--
 tensorflow/python/framework/function.py          |  4 ++--
 tensorflow/python/framework/meta_graph_test.py   |  4 ++--
 tensorflow/python/framework/op_def_library.py    |  2 +-
 tensorflow/python/framework/test_util.py         |  2 +-
 tensorflow/python/keras/backend.py               |  2 +-
 tensorflow/python/keras/engine/base_layer.py     |  7 +++----
 .../keras/engine/distributed_training_utils.py   |  6 +++++-
 tensorflow/python/keras/engine/network.py        |  4 ++--
 tensorflow/python/keras/engine/saving.py         |  2 +-
 tensorflow/python/keras/engine/training_utils.py |  6 +++---
 tensorflow/python/keras/layers/merge.py          |  6 +++---
 tensorflow/python/keras/utils/layer_utils.py     |  2 +-
 .../kernel_tests/control_flow_ops_py_test.py     |  6 +++---
 tensorflow/python/ops/bitwise_ops_test.py        |  2 +-
 tensorflow/python/ops/control_flow_ops.py        | 12 ++++++------
 tensorflow/python/ops/data_flow_ops.py           |  6 +++---
 tensorflow/python/ops/gradients_impl.py          |  8 ++++----
 tensorflow/python/ops/parallel_for/pfor.py       |  7 +++----
 tensorflow/python/ops/rnn.py                     |  2 +-
 tensorflow/python/ops/rnn_cell_impl.py           |  2 +-
 tensorflow/python/ops/variable_scope.py          |  2 +-
 tensorflow/python/ops/variables.py               |  8 ++++----
 tensorflow/python/ops/while_v2.py                |  2 +-
 tensorflow/python/tools/inspect_checkpoint.py    |  2 +-
 tensorflow/python/training/evaluation.py         |  2 +-
 tensorflow/python/training/warm_starting_util.py |  6 +++---
 .../tools/api/tests/api_compatibility_test.py    |  4 ++--
 56 files changed, 113 insertions(+), 117 deletions(-)

diff --git a/configure.py b/configure.py
index 234561d94a..0b16fe1314 100644
--- a/configure.py
+++ b/configure.py
@@ -859,7 +859,7 @@ def set_tf_cuda_version(environ_cp):
     cuda_toolkit_paths_full = [
         os.path.join(cuda_toolkit_path, x) for x in cuda_rt_lib_paths
     ]
-    if any([os.path.exists(x) for x in cuda_toolkit_paths_full]):
+    if any(os.path.exists(x) for x in cuda_toolkit_paths_full):
       break
 
     # Reset and retry
diff --git a/tensorflow/compiler/tests/dense_layer_test.py b/tensorflow/compiler/tests/dense_layer_test.py
index d1b90f098d..23c94cf245 100644
--- a/tensorflow/compiler/tests/dense_layer_test.py
+++ b/tensorflow/compiler/tests/dense_layer_test.py
@@ -42,7 +42,7 @@ def GetRunMetadataLabels(run_metadata):
 
 def InLabels(labels, substr):
   """Returns true iff one of the labels contains substr."""
-  return any([substr in x for x in labels])
+  return any(substr in x for x in labels)
 
 
 class DenseLayerTest(test.TestCase):
diff --git a/tensorflow/compiler/tests/jit_test.py b/tensorflow/compiler/tests/jit_test.py
index 6f51ae33a1..dbea9849e2 100644
--- a/tensorflow/compiler/tests/jit_test.py
+++ b/tensorflow/compiler/tests/jit_test.py
@@ -75,7 +75,7 @@ def RunMetadataLabels(run_metadata):
 
 def InLabels(labels, substr):
   """Returns true iff one of the labels contains substr."""
-  return any([substr in x for x in labels])
+  return any(substr in x for x in labels)
 
 
 def MetadataHasXlaRunOp(run_metadata):
diff --git a/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py b/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py
index 41258edd90..6926c0d03f 100644
--- a/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py
+++ b/tensorflow/contrib/constrained_optimization/python/constrained_minimization_problem.py
@@ -74,8 +74,8 @@ class ConstrainedMinimizationProblem(object):
 
     if (constraints_shape.ndims is None or
         proxy_constraints_shape.ndims is None or
-        any([ii is None for ii in constraints_shape.as_list()]) or
-        any([ii is None for ii in proxy_constraints_shape.as_list()])):
+        any(ii is None for ii in constraints_shape.as_list()) or
+        any(ii is None for ii in proxy_constraints_shape.as_list())):
       raise ValueError(
           "constraints and proxy_constraints must have fully-known shapes")
     if constraints_shape != proxy_constraints_shape:
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py
index 1e2c9121d6..a268415f0e 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_ops_test.py
@@ -778,8 +778,7 @@ class CudnnParamsFormatConverterTest(TensorFlowTestCase,
 
       # Test opaque_params size lower bound
       opaque_params_size_v = sess.run(opaque_params_size)
-      min_params_size = (
-          np.sum([x.size for x in ws]) + np.sum([x.size for x in bs]))
+      min_params_size = sum(x.size for x in ws) + np.sum(x.size for x in bs)
       logging.info("min_parm_size: %d vs actual_opaque_param_size: %d",
                    min_params_size, opaque_params_size_v)
       self.assertLessEqual(min_params_size, opaque_params_size_v)
@@ -853,8 +852,7 @@ class CudnnParamsFormatConverterTest(TensorFlowTestCase,
 
       # Test opaque_params size lower bound
       opaque_params_size_v = sess.run(opaque_params_size)
-      min_params_size = (
-          np.sum([x.size for x in ws]) + np.sum([x.size for x in bs]))
+      min_params_size = sum(x.size for x in ws) + sum(x.size for x in bs)
       logging.info("min_parm_size: %d vs actual_opaque_param_size: %d",
                    min_params_size, opaque_params_size_v)
       self.assertLessEqual(min_params_size, opaque_params_size_v)
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 6cc93dccb0..7e1b4062ce 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -1045,8 +1045,8 @@ class CudnnRNNTestParamsSize(test_util.TensorFlowTestCase):
 
     # Min param size estimate = sum(weights.size) + sum(biases.size)
     min_params_size = (
-        np.sum(list(map(np.prod, rnn.canonical_weight_shapes))) +
-        np.sum([sp[0] for sp in rnn.canonical_bias_shapes]))
+        sum(map(np.prod, rnn.canonical_weight_shapes)) +
+        sum(sp[0] for sp in rnn.canonical_bias_shapes))
 
     opaque_params = rnn.trainable_variables[0]
     with self.test_session(use_gpu=True, graph=ops.get_default_graph()):
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index e77d3d455b..129b394bb6 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -344,7 +344,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       run_step()
 
       v = all_vars[0]
-      self.assertTrue(all([v is vi for vi in all_vars[1:]]))
+      self.assertTrue(all(v is vi for vi in all_vars[1:]))
       weight = numpy.squeeze(self.evaluate(v))
       # Our model is:
       #   predict = x * w
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f1115cb0c0..94cf548cb4 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -254,7 +254,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       self, fn, multi_worker_iterator, iterations, initial_loop_values=None):
     output_shapes = multi_worker_iterator.output_shapes
     shapes = nest.flatten(output_shapes)
-    if any([not s.is_fully_defined() for s in shapes]):
+    if any(not s.is_fully_defined() for s in shapes):
       raise ValueError(
           "TPU currently requires fully defined shapes. Either use "
           "set_shape() on the input tensors or use "
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index df0342c80c..c91ce2c0f3 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -36,8 +36,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-
 from tensorflow.contrib.framework.python.ops import variables as contrib_variables_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -817,7 +815,7 @@ def _numerically_stable_global_norm(tensor_list):
   Returns:
     A scalar tensor with the global norm.
   """
-  if np.all([x is None for x in tensor_list]):
+  if all(x is None for x in tensor_list):
     return 0.0
 
   list_max = math_ops.reduce_max([math_ops.reduce_max(math_ops.abs(x)) for x in
diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index 31d9e82700..e8c24eea3d 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -759,7 +759,7 @@ class TensorPoolAdjusteModelTest(test.TestCase):
           # For [pool_size, ?), the pool is full, tensor2 must be equal to some
           # historical values of tensor1 (which is previously stored in the
           # pool).
-          self.assertTrue(any([(v == t2).all() for v in history_values]))
+          self.assertTrue(any((v == t2).all() for v in history_values))
 
   def _make_new_model_and_check(self, model, pool_size):
     pool_fn = lambda x: random_tensor_pool.tensor_pool(x, pool_size=pool_size)
diff --git a/tensorflow/contrib/kernel_methods/python/kernel_estimators.py b/tensorflow/contrib/kernel_methods/python/kernel_estimators.py
index de7530231d..1626e55b9b 100644
--- a/tensorflow/contrib/kernel_methods/python/kernel_estimators.py
+++ b/tensorflow/contrib/kernel_methods/python/kernel_estimators.py
@@ -90,7 +90,7 @@ def _update_features_and_columns(features, feature_columns,
     mapped_column_name = column_name + "_MAPPED"
     # Construct new feature columns based on provided kernel_mappers.
     column_kernel_mappers = kernel_mappers_dict[feature_column]
-    new_dim = sum([mapper.output_dim for mapper in column_kernel_mappers])
+    new_dim = sum(mapper.output_dim for mapper in column_kernel_mappers)
     mapped_columns.add(
         layers.feature_column.real_valued_column(mapped_column_name, new_dim))
 
diff --git a/tensorflow/contrib/layers/python/layers/regularizers_test.py b/tensorflow/contrib/layers/python/layers/regularizers_test.py
index 51faba30c7..5cb00b7684 100644
--- a/tensorflow/contrib/layers/python/layers/regularizers_test.py
+++ b/tensorflow/contrib/layers/python/layers/regularizers_test.py
@@ -141,7 +141,7 @@ class RegularizerTest(test.TestCase):
     dummy_regularizer = lambda x: math_ops.reduce_sum(2 * x)
     array_weights_list = [[1.5], [2, 3, 4.2], [10, 42, 666.6]]
     tensor_weights_list = [constant_op.constant(x) for x in array_weights_list]
-    expected = sum([2 * x for l in array_weights_list for x in l])
+    expected = sum(2 * x for l in array_weights_list for x in l)
     with self.cached_session():
       result = regularizers.apply_regularization(dummy_regularizer,
                                                  tensor_weights_list)
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn.py b/tensorflow/contrib/learn/python/learn/estimators/dnn.py
index 18ca4214a1..10fbd60ba2 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn.py
@@ -150,10 +150,10 @@ def _dnn_model_fn(features, labels, mode, params, config=None):
         "input_from_feature_columns",
         values=tuple(six.itervalues(features)),
         partitioner=input_layer_partitioner) as input_layer_scope:
-      if all([
+      if all(
           isinstance(fc, feature_column._FeatureColumn)  # pylint: disable=protected-access
           for fc in feature_columns
-      ]):
+      ):
         net = layers.input_from_feature_columns(
             columns_to_tensors=features,
             feature_columns=feature_columns,
diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py
index 7a3cc8bd98..2ade6b7b6c 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_linear_combined.py
@@ -236,10 +236,10 @@ def _dnn_linear_combined_model_fn(features, labels, mode, params, config=None):
           "input_from_feature_columns",
           values=tuple(six.itervalues(features)),
           partitioner=input_layer_partitioner) as dnn_input_scope:
-        if all([
+        if all(
             isinstance(fc, feature_column_lib._FeatureColumn)  # pylint: disable=protected-access
             for fc in dnn_feature_columns
-        ]):
+        ):
           net = layers.input_from_feature_columns(
               columns_to_tensors=features,
               feature_columns=dnn_feature_columns,
@@ -292,8 +292,8 @@ def _dnn_linear_combined_model_fn(features, labels, mode, params, config=None):
         linear_parent_scope,
         values=tuple(six.itervalues(features)),
         partitioner=linear_partitioner) as scope:
-      if all([isinstance(fc, feature_column_lib._FeatureColumn)  # pylint: disable=protected-access
-              for fc in linear_feature_columns]):
+      if all(isinstance(fc, feature_column_lib._FeatureColumn)  # pylint: disable=protected-access
+             for fc in linear_feature_columns):
         if joint_linear_weights:
           linear_logits, _, _ = layers.joint_weighted_sum_from_feature_columns(
               columns_to_tensors=features,
diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
index 8bc869db89..9132b2209b 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
@@ -1066,11 +1066,11 @@ class BaseEstimator(sklearn.BaseEstimator, evaluable.Evaluable,
       chief_hooks = []
       if (self._config.save_checkpoints_secs or
           self._config.save_checkpoints_steps):
-        saver_hook_exists = any([
+        saver_hook_exists = any(
             isinstance(h, basic_session_run_hooks.CheckpointSaverHook)
             for h in (all_hooks + model_fn_ops.training_hooks + chief_hooks +
                       model_fn_ops.training_chief_hooks)
-        ])
+        )
         if not saver_hook_exists:
           chief_hooks = [
               basic_session_run_hooks.CheckpointSaverHook(
@@ -1493,7 +1493,7 @@ class Estimator(BaseEstimator):
 # pylint: disable=protected-access
 class SKCompat(sklearn.BaseEstimator):
   """Scikit learn wrapper for TensorFlow Learn Estimator.
-  
+
   THIS CLASS IS DEPRECATED. See
   [contrib/learn/README.md](https://www.tensorflow.org/code/tensorflow/contrib/learn/README.md)
   for general migration instructions.
diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py
index 439b17e505..9ee8d8004b 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py
@@ -155,8 +155,8 @@ def _linear_model_fn(features, labels, mode, params, config=None):
       parent_scope,
       values=tuple(six.itervalues(features)),
       partitioner=partitioner) as scope:
-    if all([isinstance(fc, feature_column._FeatureColumn)  # pylint: disable=protected-access
-            for fc in feature_columns]):
+    if all(isinstance(fc, feature_column._FeatureColumn)  # pylint: disable=protected-access
+           for fc in feature_columns):
       if joint_weights:
         layer_fn = layers.joint_weighted_sum_from_feature_columns
       else:
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 338923f751..21d1b12130 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -160,7 +160,7 @@ def Quantize(graph,
       # shouldn't quantize it, since the activation will be Fused into the
       # Add at inference time.
       consumers = input_to_ops_map.ConsumerOperations(layer_match.bypass_op)
-      if any([consumer.type in _ACTIVATION_TYPES for consumer in consumers]):
+      if any(consumer.type in _ACTIVATION_TYPES for consumer in consumers):
         logging.info('Skipping %s, because its followed by an activation.',
                      layer_match.bypass_op.name)
       else:
@@ -195,7 +195,7 @@ def Quantize(graph,
       # Add at inference time.
       consumers = input_to_ops_map.ConsumerOperations(
           layer_match.post_activation_bypass_op)
-      if any([consumer.type in _RELU_TYPES for consumer in consumers]):
+      if any(consumer.type in _RELU_TYPES for consumer in consumers):
         logging.info('Skipping %s, because its followed by an activation.',
                      layer_match.post_activation_bypass_op.name)
       else:
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index 8668c67cf9..922f21b98b 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -154,8 +154,8 @@ class AttentionWrapperTest(test.TestCase):
 
     if attention_layer_sizes is not None:
       # Compute sum of attention_layer_sizes. Use encoder_output_depth if None.
-      attention_depth = sum([attention_layer_size or encoder_output_depth
-                             for attention_layer_size in attention_layer_sizes])
+      attention_depth = sum(attention_layer_size or encoder_output_depth
+                            for attention_layer_size in attention_layer_sizes)
     elif attention_layers is not None:
       # Compute sum of attention_layers output depth.
       attention_depth = sum(
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index a02361241c..def57da20d 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -1111,7 +1111,7 @@ def validate_inference_rewrite_for_variables(graph):
   Raises:
     RuntimeError: if validation failed.
   """
-  if not any([x.type == "GuaranteeConst" for x in graph.get_operations()]):
+  if not any(x.type == "GuaranteeConst" for x in graph.get_operations()):
     raise RuntimeError(
         "No GuaranteeConst ops found in the graph after running "
         "tpu.rewrite_for_inference(...). Please check that you are using "
diff --git a/tensorflow/python/autograph/utils/type_check.py b/tensorflow/python/autograph/utils/type_check.py
index 8748abc47b..ccef7dee03 100644
--- a/tensorflow/python/autograph/utils/type_check.py
+++ b/tensorflow/python/autograph/utils/type_check.py
@@ -30,4 +30,4 @@ def is_tensor(*args):
   Returns:
     True if any *args are TensorFlow types, False if none are.
   """
-  return any([tensor_util.is_tensor(a) for a in args])
+  return any(tensor_util.is_tensor(a) for a in args)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 470de580e8..18b3bc9424 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -459,7 +459,7 @@ class MapVectorizationBenchmark(test.Benchmark):
     return median_time
 
   def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id):
-    num_elems = int(np.sum([np.prod(x) for x in input_size]))
+    num_elems = sum(np.prod(x) for x in input_size)
     name_template = "{}__batch_size_{}_input_element_size_{}_{}"
     unoptimized = input_dataset.map(map_fn).batch(batch_size)
     unoptimized_op = unoptimized.make_one_shot_iterator().get_next()
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py
index 225f6cbac0..e3ba8ad231 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/filter_dataset_serialization_test.py
@@ -17,8 +17,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-
 from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import sparse_tensor
@@ -35,7 +33,7 @@ class FilterDatasetSerializationTest(
 
   def testFilterCore(self):
     div = 3
-    num_outputs = np.sum([x % 3 != 2 for x in range(100)])
+    num_outputs = sum(x % 3 != 2 for x in range(100))
     self.run_core_tests(lambda: self._build_filter_range_graph(div),
                         lambda: self._build_filter_range_graph(div * 2),
                         num_outputs)
@@ -47,7 +45,7 @@ class FilterDatasetSerializationTest(
                 lambda d: d["foo"] + d["bar"])
 
   def testFilterDictCore(self):
-    num_outputs = np.sum([(x**2) % 2 == 0 for x in range(10)])
+    num_outputs = sum((x**2) % 2 == 0 for x in range(10))
     self.run_core_tests(self._build_filter_dict_graph, None, num_outputs)
 
   def _build_sparse_filter(self):
diff --git a/tensorflow/python/data/experimental/ops/parsing_ops.py b/tensorflow/python/data/experimental/ops/parsing_ops.py
index 6615b9022a..a63eb8c516 100644
--- a/tensorflow/python/data/experimental/ops/parsing_ops.py
+++ b/tensorflow/python/data/experimental/ops/parsing_ops.py
@@ -138,10 +138,10 @@ def parse_example_dataset(features, num_parallel_calls=1):
   def _apply_fn(dataset):
     """Function from `Dataset` to `Dataset` that applies the transformation."""
     out_dataset = _ParseExampleDataset(dataset, features, num_parallel_calls)
-    if any([
+    if any(
         isinstance(feature, parsing_ops.SparseFeature)
         for _, feature in features.items()
-    ]):
+    ):
       # pylint: disable=protected-access
       # pylint: disable=g-long-lambda
       out_dataset = out_dataset.map(
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
index 1f22a37c2e..3454082f96 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
@@ -42,7 +42,7 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       graph = graph_pb2.GraphDef().FromString(
           sess.run(dataset._as_serialized_graph()))
-      self.assertTrue(any([node.op != "RangeDataset" for node in graph.node]))
+      self.assertTrue(any(node.op != "RangeDataset" for node in graph.node))
 
   @staticmethod
   def make_apply_fn(dataset):
diff --git a/tensorflow/python/data/util/sparse.py b/tensorflow/python/data/util/sparse.py
index 5e6d224709..f2e22fefd3 100644
--- a/tensorflow/python/data/util/sparse.py
+++ b/tensorflow/python/data/util/sparse.py
@@ -34,7 +34,7 @@ def any_sparse(classes):
   Returns:
     `True` if `classes` contains a sparse tensor type and `False` otherwise.
   """
-  return any([c is sparse_tensor.SparseTensor for c in nest.flatten(classes)])
+  return any(c is sparse_tensor.SparseTensor for c in nest.flatten(classes))
 
 
 def as_dense_shapes(shapes, classes):
diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index f55385eddc..de25b718bf 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -103,10 +103,10 @@ def _validate_value_destination_pairs(value_destination_pairs):
   # pylint: disable=g-missing-docstring
   if not value_destination_pairs: return False
   if not isinstance(value_destination_pairs, (list, tuple)): return False
-  if not all([isinstance(pair, tuple) for pair in value_destination_pairs]):
+  if not all(isinstance(pair, tuple) for pair in value_destination_pairs):
     return False
-  if not all([isinstance(v[0], value_lib.PerReplica)
-              for v in value_destination_pairs]):
+  if not all(isinstance(v[0], value_lib.PerReplica)
+             for v in value_destination_pairs):
     return False
   return True
 
@@ -132,10 +132,10 @@ def _devices_match(left, right):
 
 
 def _all_devices_match(value_destination_pairs):
-  if not all([_devices_match(v, d) for v, d in value_destination_pairs]):
+  if not all(_devices_match(v, d) for v, d in value_destination_pairs):
     return False
-  if not all([_devices_match(v, value_destination_pairs[0][0])
-              for v, _ in value_destination_pairs[1:]]):
+  if not all(_devices_match(v, value_destination_pairs[0][0])
+             for v, _ in value_destination_pairs[1:]):
     return False
   return True
 
@@ -401,7 +401,7 @@ class ConcatAndSplitPacker(object):
         # all gradient shapes are defined, we use another method to get the
         # total size.
         # TODO(yuefengz): move this logic to array_ops.size.
-        if all([g.shape.is_fully_defined() for g, _ in device_grads_and_vars]):
+        if all(g.shape.is_fully_defined() for g, _ in device_grads_and_vars):
           total_grad_size = sum(
               [g.shape.num_elements() for g, _ in device_grads_and_vars])
         else:
@@ -941,7 +941,7 @@ def choose_the_best(devices, session_config=None):
                     "TensorFlow sessions.")
     return ReductionToOneDeviceCrossDeviceOps()
 
-  if any([d.device_type.lower() != "gpu" for d in using_devices]):
+  if any(d.device_type.lower() != "gpu" for d in using_devices):
     logging.warning("Not all devices in DistributionStrategy are visible to "
                     "TensorFlow session.")
     return ReductionToOneDeviceCrossDeviceOps()
diff --git a/tensorflow/python/distribute/cross_device_utils.py b/tensorflow/python/distribute/cross_device_utils.py
index 7903992ac7..0faadd7e0c 100644
--- a/tensorflow/python/distribute/cross_device_utils.py
+++ b/tensorflow/python/distribute/cross_device_utils.py
@@ -420,7 +420,7 @@ def sum_gradients_all_reduce(dev_prefixes, replica_grads, num_workers, alg,
   Returns:
     list of reduced tensors
   """
-  alg_contains_shuffle = any([n in alg for n in ['pscpu', 'psgpu']])
+  alg_contains_shuffle = any(n in alg for n in ['pscpu', 'psgpu'])
   is_hierarchical = '/' in alg
   if 'pscpu' in alg:
     aux_devices = [prefix + '/cpu:0' for prefix in dev_prefixes]
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 84b61f47c1..99da422781 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -547,11 +547,11 @@ def _aggregate_grads(gradients):
 
   if len(gradients) == 1:
     return gradients[0]
-  if all([isinstance(g, ops.Tensor) for g in gradients]):
+  if all(isinstance(g, ops.Tensor) for g in gradients):
     return gen_math_ops.add_n(gradients)
   else:
-    assert all([isinstance(g, (ops.Tensor, ops.IndexedSlices))
-                for g in gradients])
+    assert all(isinstance(g, (ops.Tensor, ops.IndexedSlices))
+               for g in gradients)
     indexed_slices_list = []
     for grad in gradients:
       # TODO(xpan): Support nested IndexedSlices and core IndexedSlices
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index bc92a0c974..eff7a384b8 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -89,8 +89,8 @@ def _parse_func_attrs(attributes):
   """
   attrs = {}
   for key, value in attributes.items():
-    if not any([re.match(reg, key)
-                for reg in WHITELIST_FUNCTION_ATTRIBUTE_REGEX]):
+    if not any(re.match(reg, key)
+               for reg in WHITELIST_FUNCTION_ATTRIBUTE_REGEX):
       raise ValueError("Attribute name is not whitelisted. "
                        "Whitelisted: prefix %s, got: %s" %
                        (WHITELIST_FUNCTION_ATTRIBUTE_REGEX, key))
diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index 230a554641..622686ce00 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -874,7 +874,7 @@ def func_graph_from_py_func(func, arg_names, arg_types, name=None,
       # If func only returned one value, make it a tuple.
       if not isinstance(outputs, (list, tuple)):
         outputs = (outputs,)
-      if any([_ is None for _ in outputs]):
+      if any(_ is None for _ in outputs):
         raise ValueError("Function %s can not return None." % name)
     # Ensures each output is a Tensor in the function graph.
     outputs = [ops.convert_to_tensor(t) for t in outputs]
@@ -1190,7 +1190,7 @@ def get_extra_args():
 
 
 def _type_list_to_str(types):
-  if any([_ not in _DTYPE_TO_STR for _ in types]):
+  if any(_ not in _DTYPE_TO_STR for _ in types):
     raise ValueError("Unsupported dtypes: %s" % types)
   return "".join([_DTYPE_TO_STR[_] for _ in types])
 
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 3605ed7fa2..559a76bebb 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -600,11 +600,11 @@ class ScopedMetaGraphTest(test.TestCase):
     with graph.as_default():
       variables.Variable(initial_value=1.0, trainable=True)
     self.assertTrue(
-        all([
+        all(
             graph.get_collection(key)
             for key in
             [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.TRAINABLE_VARIABLES]
-        ]))
+        ))
     meta_graph.export_scoped_meta_graph(
         filename=meta_graph_filename, graph=graph)
 
diff --git a/tensorflow/python/framework/op_def_library.py b/tensorflow/python/framework/op_def_library.py
index 9955a9a2cd..2318b32ef1 100644
--- a/tensorflow/python/framework/op_def_library.py
+++ b/tensorflow/python/framework/op_def_library.py
@@ -570,7 +570,7 @@ class OpDefLibrary(object):
                   "than minimum length %d." %
                   (input_name, op_type_name, len(values), num_attr.minimum))
           # All tensors must have the same base type.
-          if any([bt != base_types[0] for bt in base_types]):
+          if any(bt != base_types[0] for bt in base_types):
             raise TypeError(
                 "All tensors passed to '%s' of '%s' Op "
                 "must have the same type." %
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index bf0ebaea99..7c486b2cbe 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1044,7 +1044,7 @@ def is_gpu_available(cuda_only=False, min_cuda_compute_capability=None):
         return True
     return False
   except errors_impl.NotFoundError as e:
-    if not all([x in str(e) for x in ["CUDA", "not find"]]):
+    if not all(x in str(e) for x in ["CUDA", "not find"]):
       raise e
     else:
       logging.error(str(e))
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 54421d9022..c7654642d0 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -2325,7 +2325,7 @@ def concatenate(tensors, axis=-1):
     else:
       axis = 0
 
-  if py_all([is_sparse(x) for x in tensors]):
+  if py_all(is_sparse(x) for x in tensors):
     return sparse_ops.sparse_concat(axis, tensors)
   else:
     return array_ops.concat([to_dense(x) for x in tensors], axis)
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index c8e964d1f7..5426965509 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -768,7 +768,7 @@ class Layer(checkpointable.CheckpointableBase):
 
     if context.executing_eagerly():
       # Accept NumPy inputs by converting to Tensors when executing eagerly.
-      if all([isinstance(x, (np.ndarray, float, int)) for x in input_list]):
+      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
         inputs = nest.map_structure(ops.convert_to_tensor, inputs)
         input_list = nest.flatten(inputs)
 
@@ -1442,8 +1442,7 @@ class Layer(checkpointable.CheckpointableBase):
                          ', but the layer isn\'t built. '
                          'You can build it manually via: `' + self.name +
                          '.build(batch_input_shape)`.')
-    weight_shapes = [w.shape.as_list() for w in self.weights]
-    return int(sum([np.prod(w) for w in weight_shapes]))
+    return int(sum(np.prod(w.shape.as_list()) for w in self.weights))
 
   @property
   def output_shape(self):
@@ -1758,7 +1757,7 @@ def have_all_keras_metadata(iterable_or_element):
     iterable = [iterable_or_element]
   else:
     iterable = nest.flatten(iterable_or_element)
-  return all([hasattr(x, '_keras_history') for x in iterable])
+  return all(hasattr(x, '_keras_history') for x in iterable)
 
 
 def collect_previous_mask(input_tensors):
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 25685fb5cf..7d915544fc 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -383,7 +383,11 @@ def validate_inputs(x, y, distribution_strategy):
     for i in [x, y]:
       if isinstance(i, dataset_ops.Dataset):
         shapes = nest.flatten(i.output_shapes)
-        if any([not s.is_fully_defined() for s in shapes]):
+        try:
+          s = next(s for s in shapes if not s.is_fully_defined())
+        except StopIteration:
+          continue
+        else:
           raise ValueError(
               'Using TPUs currently requires fully defined shapes. Either use '
               'set_shape() on the input tensors or use '
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 4163176483..f854cdd4e0 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -428,8 +428,8 @@ class Network(base_layer.Layer):
 
   @property
   def stateful(self):
-    return any([(hasattr(layer, 'stateful') and layer.stateful)
-                for layer in self.layers])
+    return any((hasattr(layer, 'stateful') and layer.stateful)
+               for layer in self.layers)
 
   def reset_states(self):
     for layer in self.layers:
diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 22c48e3f13..54d9e32fb2 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -917,7 +917,7 @@ def save_attributes_to_hdf5_group(group, name, data):
   chunked_data = np.array_split(data_npy, num_chunks)
 
   # This will never loop forever thanks to the test above.
-  while any([x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data]):
+  while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
     num_chunks += 1
     chunked_data = np.array_split(data_npy, num_chunks)
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 8669daf99e..1735db8b6b 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -58,10 +58,10 @@ def _map_nested(data, func):
 def _nested_all(data, cond_func):
   """Checks if all elements in a nested structure satisfy cond_func."""
   if isinstance(data, (tuple, list)):
-    return all([_nested_all(nested_data, cond_func) for nested_data in data])
+    return all(_nested_all(nested_data, cond_func) for nested_data in data)
   elif isinstance(data, dict):
     return all(
-        [_nested_all(nested_data, cond_func) for nested_data in data.values()])
+        _nested_all(nested_data, cond_func) for nested_data in data.values())
   else:
     return cond_func(data)
 
@@ -69,7 +69,7 @@ def _nested_all(data, cond_func):
 def _nested_any(data, cond_func):
   """Checks if any nested_elements in a nested structure satisfy cond_func."""
   if isinstance(data, (tuple, list)):
-    return any([_nested_any(nested_data, cond_func) for nested_data in data])
+    return any(_nested_any(nested_data, cond_func) for nested_data in data)
   elif isinstance(data, dict):
     return any(
         [_nested_any(nested_data, cond_func) for nested_data in data.values()])
diff --git a/tensorflow/python/keras/layers/merge.py b/tensorflow/python/keras/layers/merge.py
index f295af3fe0..45e705c696 100644
--- a/tensorflow/python/keras/layers/merge.py
+++ b/tensorflow/python/keras/layers/merge.py
@@ -212,7 +212,7 @@ class _Merge(Layer):
     if len(mask) != len(inputs):
       raise ValueError('The lists `inputs` and `mask` '
                        'should have the same length.')
-    if all([m is None for m in mask]):
+    if all(m is None for m in mask):
       return None
     masks = [array_ops.expand_dims(m, axis=0) for m in mask if m is not None]
     return K.all(K.concatenate(masks, axis=0), axis=0, keepdims=False)
@@ -378,7 +378,7 @@ class Concatenate(_Merge):
     if not isinstance(input_shape, list) or len(input_shape) < 2:
       raise ValueError('A `Concatenate` layer should be called '
                        'on a list of at least 2 inputs')
-    if all([shape is None for shape in input_shape]):
+    if all(shape is None for shape in input_shape):
       return
     reduced_inputs_shapes = [list(shape) for shape in input_shape]
     shape_set = set()
@@ -418,7 +418,7 @@ class Concatenate(_Merge):
     if len(mask) != len(inputs):
       raise ValueError('The lists `inputs` and `mask` '
                        'should have the same length.')
-    if all([m is None for m in mask]):
+    if all(m is None for m in mask):
       return None
     # Make a list of masks while making sure
     # the dimensionality of each mask
diff --git a/tensorflow/python/keras/utils/layer_utils.py b/tensorflow/python/keras/utils/layer_utils.py
index 158a9a5e76..60677be735 100644
--- a/tensorflow/python/keras/utils/layer_utils.py
+++ b/tensorflow/python/keras/utils/layer_utils.py
@@ -77,7 +77,7 @@ def count_params(weights):
   Returns:
       The total number of scalars composing the weights
   """
-  return int(np.sum([np.prod(p.get_shape().as_list()) for p in set(weights)]))
+  return int(sum(np.prod(p.get_shape().as_list()) for p in set(weights)))
 
 
 def print_summary(model, line_length=None, positions=None, print_fn=None):
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 3b8f917282..59966ebc84 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -863,13 +863,13 @@ class ControlFlowTest(test.TestCase):
       # Should just be [1, 1], but possibly a sparse representation
       gv, gi = sess.run([grad.values, grad.indices], feed_dict={c: 1})
       dense_gv = [
-          sum([y for (x, y) in zip(gi, gv) if x == i]) for i in range(2)
+          sum(y for (x, y) in zip(gi, gv) if x == i) for i in range(2)
       ]
       self.assertAllEqual(dense_gv, [1.0, 1.0])
       # Should be [0, 2], as the else forwards v1[1] twice
       gv, gi = sess.run([grad.values, grad.indices], feed_dict={c: 3})
       dense_gv = [
-          sum([y for (x, y) in zip(gi, gv) if x == i]) for i in range(2)
+          sum(y for (x, y) in zip(gi, gv) if x == i) for i in range(2)
       ]
       self.assertAllEqual(dense_gv, [0.0, 2.0])
 
@@ -2809,7 +2809,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose([156.0, 400.0], sess.run(r, feed_dict=feed_dict))
       name = "gradients/while/stopped_grad"
       all_ops = x.graph.get_operations()
-      self.assertFalse(any([name in op.name for op in all_ops]))
+      self.assertFalse(any(name in op.name for op in all_ops))
 
   @test_util.disable_control_flow_v2("b/117954949")
   def testWhileGradGradFail(self):
diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py
index dfb40db2d5..f6f35374c0 100644
--- a/tensorflow/python/ops/bitwise_ops_test.py
+++ b/tensorflow/python/ops/bitwise_ops_test.py
@@ -59,7 +59,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
                   2**31 - 1, 2**31, 2**32 - 1, 2**32, -2**32 + 1, -2**32,
                   -2**63 + 1, 2**63 - 1]
     def count_bits(x):
-      return sum([bin(z).count("1") for z in six.iterbytes(x.tobytes())])
+      return sum(bin(z).count("1") for z in six.iterbytes(x.tobytes()))
     for dtype in dtype_list:
       with self.cached_session(use_gpu=True) as sess:
         print("PopulationCount test: ", dtype)
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 4417632e69..a36a24ebb0 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -158,7 +158,7 @@ def Assert(condition, data, summarize=None, name=None):
 
   with ops.name_scope(name, "Assert", [condition, data]) as name:
     xs = ops.convert_n_to_tensor(data)
-    if all([x.dtype in {dtypes.string, dtypes.int32} for x in xs]):
+    if all(x.dtype in {dtypes.string, dtypes.int32} for x in xs):
       # As a simple heuristic, we assume that string and int32 are
       # on host to avoid the need to use cond. If it is not case,
       # we will pay the price copying the tensor to host memory.
@@ -457,19 +457,19 @@ def merge(inputs, name=None):
     ValueError: If any of the inputs is None, or inputs are IndexedSlices and
       some but not all have a dense_shape property.
   """
-  if any([inp is None for inp in inputs]):
+  if any(inp is None for inp in inputs):
     raise ValueError("At least one of the merge inputs is None: %s" % inputs)
   with ops.name_scope(name, "Merge", inputs) as name:
     inputs = [
         ops.internal_convert_to_tensor_or_indexed_slices(inp, as_ref=True)
         for inp in inputs
     ]
-    if all([isinstance(v, ops.Tensor) for v in inputs]):
-      if all([v.dtype._is_ref_dtype for v in inputs]):  # pylint: disable=protected-access
+    if all(isinstance(v, ops.Tensor) for v in inputs):
+      if all(v.dtype._is_ref_dtype for v in inputs):  # pylint: disable=protected-access
         return gen_control_flow_ops.ref_merge(inputs, name)
       else:
         return gen_control_flow_ops.merge(inputs, name)
-    elif all([isinstance(v, sparse_tensor.SparseTensor) for v in inputs]):
+    elif all(isinstance(v, sparse_tensor.SparseTensor) for v in inputs):
       # Only handle the case when all inputs are SparseTensor.
       values, _ = merge([inp.values for inp in inputs], name=name)
       indices, chosen_index = gen_control_flow_ops.merge(
@@ -557,7 +557,7 @@ def _SetShapeInvariants(input_vars, enter_vars, shapes):
   if shapes is None:
     return
   flat_shapes = nest.flatten(shapes)
-  if not all([isinstance(s, tensor_shape.TensorShape) for s in flat_shapes]):
+  if not all(isinstance(s, tensor_shape.TensorShape) for s in flat_shapes):
     raise ValueError("`shapes` must be a (possibly nested) list of shapes.")
   # Check that the shapes of the inputs are less than the shape invariants,
   # and set the shapes of `enter_vars` to the shape invariants.
diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py
index 0fac7994cb..bb08dbaea1 100644
--- a/tensorflow/python/ops/data_flow_ops.py
+++ b/tensorflow/python/ops/data_flow_ops.py
@@ -79,7 +79,7 @@ def _as_shape_list(shapes,
     shapes = [shapes]
   shapes = [tensor_shape.as_shape(shape) for shape in shapes]
   if not unknown_dim_allowed:
-    if any([not shape.is_fully_defined() for shape in shapes]):
+    if any(not shape.is_fully_defined() for shape in shapes):
       raise ValueError("All shapes must be fully defined: %s" % shapes)
   if not unknown_rank_allowed:
     if any([shape.dims is None for shape in shapes]):
@@ -198,11 +198,11 @@ class QueueBase(object):
       raise TypeError("A list of queues expected")
 
     dtypes = queues[0].dtypes
-    if not all([dtypes == q.dtypes for q in queues[1:]]):
+    if not all(dtypes == q.dtypes for q in queues[1:]):
       raise TypeError("Queues do not have matching component dtypes.")
 
     names = queues[0].names
-    if not all([names == q.names for q in queues[1:]]):
+    if not all(names == q.names for q in queues[1:]):
       raise TypeError("Queues do not have matching component names.")
 
     queue_shapes = [q.shapes for q in queues]
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index 53c0709e32..278008526c 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -895,7 +895,7 @@ def _HasAnyNotNoneGrads(grads, op):
     if isinstance(out_grad, (ops.Tensor, ops.IndexedSlices)):
       return True
     if out_grad and isinstance(out_grad, collections.Sequence):
-      if any([g is not None for g in out_grad]):
+      if any(g is not None for g in out_grad):
         return True
   return False
 
@@ -1110,11 +1110,11 @@ def _AggregatedGrads(grads,
         assert control_flow_util.IsLoopSwitch(op)
         continue
     # Grads have to be Tensors or IndexedSlices
-    if (isinstance(out_grad, collections.Sequence) and not all([
+    if (isinstance(out_grad, collections.Sequence) and not all(
         isinstance(g, (ops.Tensor, ops.IndexedSlices))
         for g in out_grad
         if g is not None
-    ])):
+    )):
       raise TypeError("gradients have to be either all Tensors "
                       "or all IndexedSlices")
     # Aggregate multiple gradients, and convert [] to None.
@@ -1122,7 +1122,7 @@ def _AggregatedGrads(grads,
       if len(out_grad) < 2:
         used = "nop"
         out_grads[i] = out_grad[0]
-      elif all([isinstance(g, ops.Tensor) for g in out_grad if g is not None]):
+      elif all(isinstance(g, ops.Tensor) for g in out_grad if g is not None):
         tensor_shape = _AccumulatorShape(out_grad)
         if (aggregation_method == AggregationMethod.EXPERIMENTAL_ACCUMULATE_N
             and len(out_grad) > 2 and tensor_shape.is_fully_defined()):
diff --git a/tensorflow/python/ops/parallel_for/pfor.py b/tensorflow/python/ops/parallel_for/pfor.py
index d789dc65b1..a22c1126c9 100644
--- a/tensorflow/python/ops/parallel_for/pfor.py
+++ b/tensorflow/python/ops/parallel_for/pfor.py
@@ -1152,9 +1152,8 @@ class PFor(object):
           continue
 
         converted_inputs = [self._conversion_map[inp] for inp in y_op.inputs]
-        some_input_converted = any(
-            [self._was_converted(x) for x in y_op.inputs])
-        some_input_stacked = any([x.is_stacked for x in converted_inputs])
+        some_input_converted = any(self._was_converted(x) for x in y_op.inputs)
+        some_input_stacked = any(x.is_stacked for x in converted_inputs)
 
         converted_control_ops = set()
         some_control_input_converted = False
@@ -1198,7 +1197,7 @@ class PFor(object):
           # All inputs are unstacked or uncoverted but some control inputs are
           # converted.
           # TODO(rachelim): Handle the case where some inputs are sparsely
-          # stacked (i.e. any([x.is_sparse_stacked for x in converted_inputs]))
+          # stacked (i.e. any(x.is_sparse_stacked for x in converted_inputs))
           new_op = _create_op(y_op.type, [x.t for x in converted_inputs],
                               [x.dtype for x in y_op.outputs],
                               y_op.node_def.attr)
diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index c23b85847c..ec48cab91d 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -117,7 +117,7 @@ def _infer_state_dtype(explicit_dtype, state):
     inferred_dtypes = [element.dtype for element in nest.flatten(state)]
     if not inferred_dtypes:
       raise ValueError("Unable to infer dtype from empty state.")
-    all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes])
+    all_same = all(x == inferred_dtypes[0] for x in inferred_dtypes)
     if not all_same:
       raise ValueError(
           "State has tensors of different inferred_dtypes. Unable to infer a "
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 85efd6a4f7..ffc45619a7 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -1456,7 +1456,7 @@ class MultiRNNCell(RNNCell):
     if self._state_is_tuple:
       return tuple(cell.state_size for cell in self._cells)
     else:
-      return sum([cell.state_size for cell in self._cells])
+      return sum(cell.state_size for cell in self._cells)
 
   @property
   def output_size(self):
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 44bb62a5dc..4f210e3b12 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -680,7 +680,7 @@ class _VariableStore(object):
             "Partitioner returned a partition list that does not match the "
             "Variable's rank: %s vs. %s" % (partitions, shape))
 
-      if any([p < 1 for p in partitions]):
+      if any(p < 1 for p in partitions):
         raise ValueError(
             "Partitioner returned zero partitions for some axes: %s" %
             partitions)
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 5bee522481..c8d12c8ecf 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -2482,21 +2482,21 @@ class PartitionedVariable(object):
           "variable_list is not a list or tuple: %s" % variable_list)
     if not isinstance(partitions, (list, tuple)):
       raise TypeError("partitions is not a list or tuple: %s" % partitions)
-    if not all([p >= 1 for p in partitions]):
+    if not all(p >= 1 for p in partitions):
       raise ValueError("partition values must be positive: %s" % partitions)
     if not variable_list:
       raise ValueError("variable_list may not be empty")
     # pylint: disable=protected-access
     for v in variable_list:
       # Sort the variable_list lexicographically according to var offset value.
-      if not all([v._get_save_slice_info() is not None for v in variable_list]):
+      if not all(v._get_save_slice_info() is not None for v in variable_list):
         raise ValueError(
             "All variables must have a save_slice_info available: %s"
             % [v.name for v in variable_list])
       if len(shape) != len(partitions):
         raise ValueError("len(shape) != len(partitions): %s vs. %s"
                          % (shape, partitions))
-      if not all([v._get_save_slice_info().full_shape == shape]):
+      if v._get_save_slice_info().full_shape != shape:
         raise ValueError(
             "All variables' full shapes must match shape: %s; "
             "but full shapes were: %s"
@@ -2523,7 +2523,7 @@ class PartitionedVariable(object):
     return len(self._variable_list)
 
   def _partition_axes(self):
-    if all([p == 1 for p in self._partitions]):
+    if all(p == 1 for p in self._partitions):
       return [0]
     else:
       return [i for i, p in enumerate(self._partitions) if p > 1]
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 5ab7bffedc..1252c7fb03 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -509,7 +509,7 @@ def _grad_fn(ys, xs, args, func_graph):
 
   # TODO(b/118712257): Handle the case when grad_outs has None's e.g. when there
   # is a tf.StopGradient in the loop body.
-  assert all([g is not None for g in grad_outs])
+  assert all(g is not None for g in grad_outs)
   counter = args[0]
   total_iters = args[1]
   return [counter + 1, total_iters] + grad_outs
diff --git a/tensorflow/python/tools/inspect_checkpoint.py b/tensorflow/python/tools/inspect_checkpoint.py
index 6504fbc107..ea1f6aa555 100644
--- a/tensorflow/python/tools/inspect_checkpoint.py
+++ b/tensorflow/python/tools/inspect_checkpoint.py
@@ -63,7 +63,7 @@ def print_tensors_in_checkpoint_file(file_name, tensor_name, all_tensors,
       print("It's likely that your checkpoint file has been compressed "
             "with SNAPPY.")
     if ("Data loss" in str(e) and
-        (any([e in file_name for e in [".index", ".meta", ".data"]]))):
+        any(e in file_name for e in [".index", ".meta", ".data"])):
       proposed_file = ".".join(file_name.split(".")[0:-1])
       v2_file_error_template = """
 It's likely that this is a V2 checkpoint and you need to provide the filename
diff --git a/tensorflow/python/training/evaluation.py b/tensorflow/python/training/evaluation.py
index 2c4eb02d53..a10178f8cf 100644
--- a/tensorflow/python/training/evaluation.py
+++ b/tensorflow/python/training/evaluation.py
@@ -230,7 +230,7 @@ def _evaluate_once(checkpoint_path,
   hooks = list(hooks or [])
 
   if eval_ops is not None:
-    if any([isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks]):
+    if any(isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks):
       steps_per_run_variable = \
           basic_session_run_hooks.get_or_create_steps_per_run_variable()
       update_eval_step = state_ops.assign_add(
diff --git a/tensorflow/python/training/warm_starting_util.py b/tensorflow/python/training/warm_starting_util.py
index 78dbb465b5..3649d313ae 100644
--- a/tensorflow/python/training/warm_starting_util.py
+++ b/tensorflow/python/training/warm_starting_util.py
@@ -248,7 +248,7 @@ def _warm_start_var_with_vocab(var,
     prev_tensor_name = _infer_var_name(var)
 
   # TODO(eddz): Fix functionality for rank-1 Variables (like FC biases).
-  total_v_first_axis = sum([v.get_shape().as_list()[0] for v in var])
+  total_v_first_axis = sum(v.get_shape().as_list()[0] for v in var)
   for v in var:
     v_shape = v.get_shape().as_list()
     slice_info = v._get_save_slice_info()
@@ -333,12 +333,12 @@ def _get_grouped_variables(vars_to_warm_start):
         ops.GraphKeys.TRAINABLE_VARIABLES,
         scope=vars_to_warm_start)
   elif isinstance(vars_to_warm_start, list):
-    if all([isinstance(v, str) for v in vars_to_warm_start]):
+    if all(isinstance(v, str) for v in vars_to_warm_start):
       list_of_vars = []
       for v in vars_to_warm_start:
         list_of_vars += ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES,
                                            scope=v)
-    elif all([checkpoint_utils._is_variable(v) for v in vars_to_warm_start]):  # pylint: disable=protected-access
+    elif all(checkpoint_utils._is_variable(v) for v in vars_to_warm_start):  # pylint: disable=protected-access
       list_of_vars = vars_to_warm_start
     else:
       raise ValueError("If `vars_to_warm_start` is a list, it must be all "
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index b0f3742af1..cba6246fef 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -126,9 +126,9 @@ def _FilterNonCoreGoldenFiles(golden_file_list):
   filtered_file_list = []
   filtered_package_prefixes = ['tensorflow.%s.' % p for p in _NON_CORE_PACKAGES]
   for f in golden_file_list:
-    if any([
+    if any(
         f.rsplit('/')[-1].startswith(pre) for pre in filtered_package_prefixes
-    ]):
+    ):
       continue
     filtered_file_list.append(f)
   return filtered_file_list
-- 
GitLab


From c16394423c5226d7633d7255c37df762e85f6584 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 21 Nov 2018 10:48:49 -0800
Subject: [PATCH 0676/1554] Refactor to allow creation of an object graph proto
 with no variable values

Will be useful for creating a SavedModel object proto

PiperOrigin-RevId: 222430965
---
 .../python/training/checkpointable/BUILD      |  2 +-
 .../python/training/checkpointable/util.py    | 80 ++++++++++++-------
 .../training/checkpointable/util_test.py      | 16 +++-
 3 files changed, 67 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index d26932c1aa..f97f42a659 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -152,7 +152,7 @@ py_test(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
-        "//tensorflow/python/eager:function",
+        "//tensorflow/python/eager:def_function",
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras:engine",
         "//tensorflow/python/keras:layers",
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index f45f7445f1..85844393f3 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -549,13 +549,11 @@ def _serialize_slot_variables(checkpointable_objects, node_ids, object_names):
   return slot_variables
 
 
-def _serialize_checkpointables(
-    checkpointable_objects, node_ids, object_names, slot_variables,
+def _add_attributes_to_object_graph(
+    checkpointable_objects, object_graph_proto, node_ids, object_names,
     saveables_cache, object_map):
-  """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
-  object_graph_proto = (
-      checkpointable_object_graph_pb2.CheckpointableObjectGraph())
-  named_saveables = []
+  """Create SaveableObjects and corresponding SerializedTensor protos."""
+  named_saveable_objects = []
   if saveables_cache is None:
     # No SaveableObject caching. Either we're executing eagerly, or building a
     # static save which is specialized to the current Python state.
@@ -564,10 +562,9 @@ def _serialize_checkpointables(
     # If we are caching SaveableObjects, we need to build up a feed_dict with
     # functions computing volatile Python state to be saved with the checkpoint.
     feed_additions = {}
-  for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
+  for checkpoint_id, (checkpointable, object_proto) in enumerate(
+      zip(checkpointable_objects, object_graph_proto.nodes)):
     assert node_ids[checkpointable] == checkpoint_id
-    object_proto = object_graph_proto.nodes.add()
-    object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
     object_name = object_names[checkpointable]
     if object_map:
       object_to_save = object_map.get(checkpointable, checkpointable)
@@ -645,14 +642,24 @@ def _serialize_checkpointables(
                      "value.")
                     % (checkpointable, new_feed_key))
             feed_additions.update(saveable_feed_dict)
-        named_saveables.append(saveable)
+        named_saveable_objects.append(saveable)
 
+  return named_saveable_objects, feed_additions
+
+
+def _make_object_graph_proto(checkpointable_objects, node_ids, slot_variables):
+  """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
+  object_graph_proto = (
+      checkpointable_object_graph_pb2.CheckpointableObjectGraph())
+  for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
+    assert node_ids[checkpointable] == checkpoint_id
+    object_proto = object_graph_proto.nodes.add()
+    object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
     for child in checkpointable._checkpoint_dependencies:  # pylint: disable=protected-access
       child_proto = object_proto.children.add()
       child_proto.node_id = node_ids[child.ref]
       child_proto.local_name = child.name
-
-  return named_saveables, object_graph_proto, feed_additions
+  return object_graph_proto
 
 
 def _serialize_gathered_objects(
@@ -668,13 +675,18 @@ def _serialize_gathered_objects(
       checkpointable_objects=checkpointable_objects,
       node_ids=node_ids,
       object_names=object_names)
-  return _serialize_checkpointables(
+  object_graph_proto = _make_object_graph_proto(
       checkpointable_objects=checkpointable_objects,
       node_ids=node_ids,
+      slot_variables=slot_variables)
+  named_saveable_objects, feed_additions = _add_attributes_to_object_graph(
+      checkpointable_objects=checkpointable_objects,
+      object_graph_proto=object_graph_proto,
+      node_ids=node_ids,
       object_names=object_names,
-      slot_variables=slot_variables,
       saveables_cache=saveables_cache,
       object_map=object_map)
+  return named_saveable_objects, object_graph_proto, feed_additions
 
 
 def _serialize_object_graph(root_checkpointable, saveables_cache):
@@ -716,6 +728,23 @@ def named_saveables(root_checkpointable):
   return _serialize_object_graph(root_checkpointable, None)[0]
 
 
+def _find_objects(root_checkpointable):
+  """Find and number objects which are dependencies of `root_checkpointable`."""
+  checkpointable_objects, path_to_root = (
+      _breadth_first_checkpointable_traversal(root_checkpointable))
+  object_names = _ObjectIdentityDictionary()
+  for obj, path in path_to_root.items():
+    object_names[obj] = _object_prefix_from_path(path)
+  node_ids = _ObjectIdentityDictionary()
+  for node_id, node in enumerate(checkpointable_objects):
+    node_ids[node] = node_id
+  slot_variables = _serialize_slot_variables(
+      checkpointable_objects=checkpointable_objects,
+      node_ids=node_ids,
+      object_names=object_names)
+  return checkpointable_objects, node_ids, slot_variables
+
+
 def list_objects(root_checkpointable):
   """Traverse the object graph and list all accessible objects.
 
@@ -730,23 +759,18 @@ def list_objects(root_checkpointable):
   Returns:
     A flat list of objects.
   """
-  # TODO(allenl): Extract out gathering logic so the naming logic doesn't have
-  # to run.
-  checkpointable_objects, path_to_root = (
-      _breadth_first_checkpointable_traversal(root_checkpointable))
-  object_names = _ObjectIdentityDictionary()
-  for obj, path in path_to_root.items():
-    object_names[obj] = _object_prefix_from_path(path)
-  node_ids = _ObjectIdentityDictionary()
-  for node_id, node in enumerate(checkpointable_objects):
-    node_ids[node] = node_id
-  _serialize_slot_variables(
-      checkpointable_objects=checkpointable_objects,
-      node_ids=node_ids,
-      object_names=object_names)
+  checkpointable_objects, _, _ = _find_objects(root_checkpointable)
   return checkpointable_objects
 
 
+def make_object_graph_without_attributes(root_checkpointable):
+  """Construct a CheckpointableObjectGraph proto with no variable values."""
+  checkpointable_objects, node_ids, slot_variables = _find_objects(
+      root_checkpointable)
+  return _make_object_graph_proto(
+      checkpointable_objects, node_ids, slot_variables)
+
+
 def gather_initializers(root_checkpointable):
   """Traverse the object graph and find initialization ops.
 
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 1995514012..de9cac0863 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -26,7 +26,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
-from tensorflow.python.eager import function
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -44,6 +44,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import adam
 from tensorflow.python.training import checkpoint_management
+from tensorflow.python.training import momentum
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.training import training_util
 from tensorflow.python.training.checkpointable import base
@@ -198,6 +199,17 @@ class InterfaceTests(test.TestCase):
     with self.assertRaises(NotImplementedError):
       checkpoint_reversed.save(prefix)
 
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def test_object_graph_no_attributes(self):
+    root = tracking.Checkpointable()
+    root.v = resource_variable_ops.ResourceVariable(1.)
+    root.opt = momentum.MomentumOptimizer(0.01, 0.5)
+    root.opt.minimize(root.v.read_value)
+    object_graph = checkpointable_utils.make_object_graph_without_attributes(
+        root)
+    # Four objects: Root, v, opt, and a slot variable for v
+    self.assertEqual(4, len(object_graph.nodes))
+
 
 class _MirroringSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
 
@@ -632,7 +644,7 @@ class CheckpointingTests(test.TestCase):
             checkpoint_directory)
         status = root.restore(save_path=checkpoint_path)
         def train_fn():
-          @function.defun
+          @def_function.function
           def _call_model(x):
             return model(x)
           with backprop.GradientTape() as tape:
-- 
GitLab


From f6ce9fd485e740f4ecded2d556a191a782b26808 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 21 Nov 2018 11:09:47 -0800
Subject: [PATCH 0677/1554] Automated rollback of commit
 1fdd7c7408aa1cb37729e76a6e9fbfe8daa0b1f5

PiperOrigin-RevId: 222434204
---
 .../compiler/tests/categorical_op_test.py     |  10 +-
 tensorflow/compiler/tests/concat_ops_test.py  |   8 +-
 tensorflow/compiler/tests/eager_test.py       |   2 +-
 tensorflow/compiler/tests/function_test.py    |   6 +-
 tensorflow/compiler/tests/lstm_test.py        |   4 +-
 tensorflow/compiler/tests/placeholder_test.py |   2 +-
 tensorflow/compiler/tests/random_ops_test.py  |  14 +-
 .../compiler/tests/tensor_array_ops_test.py   |   2 +-
 .../compiler/tests/variable_ops_test.py       |  30 +-
 .../autograph/integration_tests/keras_test.py |   2 +-
 .../integration_tests/list_literals_test.py   |   2 +-
 .../speech_commands/input_data_test.py        |   2 +-
 .../speech_commands/label_wav_test.py         |   2 +-
 .../speech_commands/wav_to_features_test.py   |   2 +-
 .../autograph/converters/call_trees_test.py   |   2 +-
 .../python/autograph/converters/lists_test.py |   6 +-
 .../converters/side_effect_guards_test.py     |  20 +-
 .../autograph/converters/slices_test.py       |   2 +-
 tensorflow/python/autograph/impl/api_test.py  |  46 +-
 .../autograph/lang/special_functions_test.py  |  12 +-
 .../autograph/operators/control_flow_test.py  |  14 +-
 .../operators/data_structures_test.py         |  16 +-
 .../autograph/operators/logical_test.py       |  14 +-
 .../autograph/operators/py_builtins_test.py   |  28 +-
 .../python/autograph/operators/slices_test.py |   8 +-
 .../python/autograph/utils/misc_test.py       |   4 +-
 .../python/autograph/utils/py_func_test.py    |  18 +-
 .../autograph/utils/tensor_list_test.py       |   4 +-
 .../client/session_clusterspec_prop_test.py   |   6 +-
 tensorflow/python/client/timeline_test.py     |   4 +-
 tensorflow/python/client/virtual_gpu_test.py  |   2 +-
 .../kernel_tests/batch_dataset_op_test.py     |  54 ++-
 .../bucket_by_sequence_length_test.py         |   2 +-
 .../kernel_tests/copy_to_device_test.py       |  84 ++--
 .../experimental/kernel_tests/counter_test.py |  12 +-
 .../dense_to_sparse_batch_test.py             |   8 +-
 .../directed_interleave_dataset_test.py       |   6 +-
 .../kernel_tests/enumerate_dataset_test.py    |   6 +-
 .../function_buffering_resource_test.py       |  70 +--
 .../kernel_tests/group_by_reducer_test.py     |   6 +-
 .../kernel_tests/group_by_window_test.py      |  52 +--
 .../kernel_tests/ignore_errors_test.py        |  16 +-
 .../kernel_tests/indexed_dataset_ops_test.py  |   6 +-
 .../make_batched_features_dataset_test.py     |   4 +-
 .../kernel_tests/make_csv_dataset_test.py     |   2 +-
 .../make_tf_record_dataset_test.py            |   6 +-
 .../kernel_tests/map_and_batch_test.py        |  38 +-
 .../kernel_tests/map_defun_op_test.py         |   2 +-
 .../kernel_tests/override_threadpool_test.py  |   2 +-
 .../kernel_tests/parallel_interleave_test.py  |   6 +-
 .../kernel_tests/prefetch_to_device_test.py   |  26 +-
 .../experimental/kernel_tests/scan_test.py    |   6 +-
 .../range_dataset_serialization_test.py       |  26 +-
 .../serialization_integration_test.py         |   4 +-
 .../kernel_tests/shuffle_and_repeat_test.py   |   2 +-
 .../experimental/kernel_tests/sleep_test.py   |   4 +-
 .../kernel_tests/sql_dataset_test.py          |  97 ++--
 .../kernel_tests/stats_dataset_ops_test.py    |  58 ++-
 .../experimental/kernel_tests/unbatch_test.py |  12 +-
 .../experimental/kernel_tests/unique_test.py  |   4 +-
 .../kernel_tests/batch_dataset_op_test.py     |  26 +-
 .../kernel_tests/cache_dataset_op_test.py     |  18 +-
 .../concatenate_dataset_op_test.py            |   8 +-
 .../dataset_constructor_op_test.py            |  44 +-
 .../dataset_from_generator_op_test.py         |  78 ++--
 .../kernel_tests/filter_dataset_op_test.py    |  26 +-
 .../kernel_tests/flat_map_dataset_op_test.py  |  20 +-
 .../interleave_dataset_op_test.py             |   8 +-
 .../kernel_tests/iterator_ops_cluster_test.py |  12 +-
 .../data/kernel_tests/iterator_ops_test.py    |  44 +-
 .../list_files_dataset_op_test.py             |   2 +-
 .../data/kernel_tests/map_dataset_op_test.py  | 126 +++---
 .../multi_device_iterator_test.py             |  68 +--
 .../data/kernel_tests/optional_ops_test.py    |   4 +-
 .../kernel_tests/prefetch_dataset_op_test.py  |   2 +-
 .../kernel_tests/range_dataset_op_test.py     | 116 ++---
 .../kernel_tests/reader_dataset_ops_test.py   | 112 ++---
 .../kernel_tests/reduce_dataset_op_test.py    |  11 +-
 .../kernel_tests/sequence_dataset_op_test.py  |  20 +-
 .../kernel_tests/shuffle_dataset_op_test.py   |  12 +-
 .../kernel_tests/window_dataset_op_test.py    |  24 +-
 .../data/kernel_tests/zip_dataset_op_test.py  |   6 +-
 tensorflow/python/data/util/convert_test.py   |   8 +-
 .../python/debug/cli/analyzer_cli_test.py     |   2 +-
 .../lib/debug_graph_reconstruction_test.py    |  14 +-
 .../debug/lib/dist_session_debug_grpc_test.py |   8 +-
 .../debug/lib/session_debug_multi_gpu_test.py |   2 +-
 .../python/debug/lib/source_utils_test.py     |   4 +-
 .../python/distribute/input_ops_test.py       |   7 +-
 tensorflow/python/eager/def_function_test.py  |   8 +-
 .../python/eager/function_gradients_test.py   |   2 +-
 tensorflow/python/eager/function_test.py      |   4 +-
 .../feature_column/feature_column_test.py     |   8 +-
 .../feature_column/feature_column_v2_test.py  |  11 +-
 tensorflow/python/framework/function_test.py  |  50 +--
 .../python/framework/graph_util_test.py       |  10 +-
 tensorflow/python/framework/importer_test.py  |  12 +-
 .../python/framework/meta_graph_test.py       |  12 +-
 tensorflow/python/framework/ops_test.py       |   6 +-
 .../python/framework/smart_cond_test.py       |   8 +-
 .../python/framework/sparse_tensor_test.py    |   2 +-
 .../python/framework/tensor_util_test.py      |   2 +-
 .../python/grappler/constant_folding_test.py  |   2 +-
 .../python/grappler/layout_optimizer_test.py  |  48 +-
 .../python/grappler/memory_optimizer_test.py  |  12 +-
 tensorflow/python/keras/backend_test.py       |   2 +-
 .../python/keras/layers/recurrent_test.py     |   4 +-
 .../python/kernel_tests/accumulate_n_test.py  |   2 +-
 .../python/kernel_tests/array_ops_test.py     |  14 +-
 .../python/kernel_tests/basic_gpu_test.py     |   8 +-
 .../boosted_trees/quantile_ops_test.py        |   4 +-
 .../boosted_trees/stats_ops_test.py           |  30 +-
 .../python/kernel_tests/bucketize_op_test.py  |   6 +-
 .../candidate_sampler_ops_test.py             |   2 +-
 .../python/kernel_tests/cast_op_test.py       |   2 +-
 .../python/kernel_tests/concat_op_test.py     |   6 +-
 .../conditional_accumulator_test.py           |   8 +-
 .../kernel_tests/control_flow_ops_py_test.py  |  48 +-
 .../python/kernel_tests/conv_ops_3d_test.py   |  10 +-
 .../python/kernel_tests/conv_ops_test.py      |  12 +-
 .../python/kernel_tests/cwise_ops_test.py     |   4 +-
 .../kernel_tests/decode_jpeg_op_test.py       |   2 +-
 .../dense_update_ops_no_tsan_test.py          |   8 +-
 .../kernel_tests/depthwise_conv_op_test.py    |   6 +-
 .../distributions/categorical_test.py         |   4 +-
 .../kernel_tests/dynamic_partition_op_test.py |  28 +-
 .../python/kernel_tests/fifo_queue_test.py    |  84 ++--
 .../kernel_tests/functional_ops_test.py       |  46 +-
 .../kernel_tests/gradient_correctness_test.py |   8 +-
 .../python/kernel_tests/init_ops_test.py      |   8 +-
 .../python/kernel_tests/lookup_ops_test.py    |   2 +-
 tensorflow/python/kernel_tests/losses_test.py |   4 +-
 .../python/kernel_tests/map_stage_op_test.py  |  22 +-
 .../kernel_tests/matrix_inverse_op_test.py    |   2 +-
 .../kernel_tests/matrix_solve_op_test.py      |   2 +-
 .../matrix_square_root_op_test.py             |   2 +-
 .../python/kernel_tests/metrics_test.py       | 419 +++++++++---------
 .../neon_depthwise_conv_op_test.py            |   6 +-
 .../python/kernel_tests/norm_op_test.py       |   2 +-
 .../kernel_tests/nth_element_op_test.py       |   2 +-
 .../kernel_tests/padding_fifo_queue_test.py   |  88 ++--
 .../parse_single_example_op_test.py           |   2 +-
 .../python/kernel_tests/parsing_ops_test.py   |   4 +-
 .../kernel_tests/pooling_ops_3d_test.py       |   2 +-
 .../kernel_tests/priority_queue_test.py       |  20 +-
 .../python/kernel_tests/py_func_test.py       |  14 +-
 tensorflow/python/kernel_tests/qr_op_test.py  |   2 +-
 .../random/multinomial_op_big_test.py         |   6 +-
 .../random/multinomial_op_test.py             |  12 +-
 .../kernel_tests/random/random_gamma_test.py  |   2 +-
 .../kernel_tests/random/random_ops_test.py    |  12 +-
 .../random/random_poisson_test.py             |   2 +-
 .../random/random_shuffle_queue_test.py       |  66 +--
 .../python/kernel_tests/reader_ops_test.py    |   2 +-
 .../python/kernel_tests/record_input_test.py  |  16 +-
 .../python/kernel_tests/reduction_ops_test.py |  18 +-
 .../resource_variable_ops_test.py             |   2 +-
 .../kernel_tests/scatter_nd_ops_test.py       |  14 +-
 .../kernel_tests/self_adjoint_eig_op_test.py  |   2 +-
 .../python/kernel_tests/session_ops_test.py   |  22 +-
 tensorflow/python/kernel_tests/sets_test.py   |   2 +-
 .../python/kernel_tests/shape_ops_test.py     |   4 +-
 .../signal/reconstruction_ops_test.py         |   8 +-
 .../python/kernel_tests/sparse_add_op_test.py |   8 +-
 .../kernel_tests/sparse_concat_op_test.py     |  14 +-
 .../sparse_conditional_accumulator_test.py    |  24 +-
 .../kernel_tests/sparse_cross_op_test.py      |  34 +-
 .../python/kernel_tests/sparse_ops_test.py    |  34 +-
 .../kernel_tests/sparse_reorder_op_test.py    |   4 +-
 .../kernel_tests/sparse_reshape_op_test.py    |   4 +-
 .../sparse_serialization_ops_test.py          |   2 +-
 .../sparse_tensors_map_ops_test.py            |  11 +-
 .../python/kernel_tests/stage_op_test.py      |  18 +-
 .../kernel_tests/string_length_op_test.py     |   2 +-
 .../kernel_tests/string_split_op_test.py      |  28 +-
 .../kernel_tests/string_strip_op_test.py      |   6 +-
 .../kernel_tests/summary_v1_audio_op_test.py  |   2 +-
 .../kernel_tests/summary_v1_image_op_test.py  |   4 +-
 .../kernel_tests/summary_v1_ops_test.py       |   6 +-
 .../kernel_tests/summary_v1_tensor_op_test.py |  12 +-
 tensorflow/python/kernel_tests/svd_op_test.py |   4 +-
 .../python/kernel_tests/template_test.py      |   8 +-
 .../kernel_tests/tensor_array_ops_test.py     |   6 +-
 .../kernel_tests/unicode_transcode_op_test.py |  46 +-
 .../kernel_tests/variable_scope_test.py       |  16 +-
 .../python/kernel_tests/while_v2_test.py      |  56 +--
 .../python/kernel_tests/xent_op_test.py       |   4 +-
 .../python/layers/convolutional_test.py       |  16 +-
 tensorflow/python/layers/core_test.py         |   2 +-
 .../python/layers/normalization_test.py       |  74 ++--
 .../python/ops/control_flow_ops_test.py       |  12 +-
 tensorflow/python/ops/gradients_test.py       |  28 +-
 tensorflow/python/ops/image_grad_test.py      |   8 +-
 tensorflow/python/ops/image_ops_test.py       |  30 +-
 tensorflow/python/ops/init_ops_test.py        |   4 +-
 tensorflow/python/ops/math_ops_test.py        |   4 +-
 .../python/ops/nn_fused_batchnorm_test.py     |   2 +-
 .../python/ops/parallel_for/gradients_test.py |   2 +-
 .../python/ops/quantized_conv_ops_test.py     |   2 +-
 tensorflow/python/ops/quantized_ops_test.py   |   4 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |   2 +-
 .../python/profiler/model_analyzer_test.py    |  30 +-
 .../python/profiler/profile_context_test.py   |   8 +-
 tensorflow/python/saved_model/loader_test.py  |   4 +-
 .../python/saved_model/saved_model_test.py    |  42 +-
 .../python/saved_model/simple_save_test.py    |   2 +-
 tensorflow/python/tools/strip_unused_test.py  |   4 +-
 .../training/basic_session_run_hooks_test.py  |  32 +-
 .../python/training/checkpoint_ops_test.py    |   2 +-
 tensorflow/python/training/input_test.py      |  54 +--
 .../python/training/monitored_session_test.py |  16 +-
 .../python/training/moving_averages_test.py   |  12 +-
 tensorflow/python/training/saver_test.py      |  62 +--
 .../training/server_lib_sparse_job_test.py    |   2 +-
 tensorflow/python/training/supervisor_test.py |  10 +-
 .../training/warm_starting_util_test.py       |  76 ++--
 216 files changed, 1980 insertions(+), 2004 deletions(-)

diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py
index 15108487cf..f17e84df13 100644
--- a/tensorflow/compiler/tests/categorical_op_test.py
+++ b/tensorflow/compiler/tests/categorical_op_test.py
@@ -61,7 +61,7 @@ class CategoricalTest(xla_test.XLATestCase):
       random_seed.set_random_seed(1618)
       op = random_ops.multinomial(logits, num_samples,
                                   output_dtype=dtypes.int32)
-      d = self.evaluate(op)
+      d = sess.run(op)
 
     batch_size, num_classes = logits.shape
     freqs_mat = []
@@ -86,9 +86,9 @@ class CategoricalTest(xla_test.XLATestCase):
 
       # The random-number generator, if working correctly, should produce the
       # same output multiple times with low probability.
-      y = self.evaluate(x)
-      z = self.evaluate(x)
-      w = self.evaluate(x)
+      y = sess.run(x)
+      z = sess.run(x)
+      w = sess.run(x)
 
       # We use exact equality here. If the random-number generator is producing
       # deterministic output, all three outputs will be bitwise identical.
@@ -113,7 +113,7 @@ class CategoricalTest(xla_test.XLATestCase):
             x = random_ops.multinomial(
                 array_ops.ones(shape=[1, 20], dtype=dtype), 1000,
                 output_dtype=output_dtype)
-          y = self.evaluate(x)
+          y = sess.run(x)
           self.assertTrue((y >= 0).sum() == 1000)
           self.assertTrue((y < 20).sum() == 1000)
 
diff --git a/tensorflow/compiler/tests/concat_ops_test.py b/tensorflow/compiler/tests/concat_ops_test.py
index deb9ac186e..30fbe6f701 100644
--- a/tensorflow/compiler/tests/concat_ops_test.py
+++ b/tensorflow/compiler/tests/concat_ops_test.py
@@ -337,7 +337,7 @@ class ConcatOffsetTest(xla_test.XLATestCase):
         s1 = constant_op.constant([2, 7, 5], dtypes.int32)
         s2 = constant_op.constant([2, 20, 5], dtypes.int32)
         off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-        ans = self.evaluate(off)
+        ans = sess.run(off)
         self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
 
@@ -350,7 +350,7 @@ class PackTest(xla_test.XLATestCase):
         s1 = constant_op.constant([2, 7, 5], dtypes.int32)
         s2 = constant_op.constant([2, 20, 5], dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = self.evaluate(packed)
+        ans = sess.run(packed)
         self.assertAllEqual(ans, [[2, 3, 5], [2, 7, 5], [2, 20, 5]])
 
   def testScalars(self):
@@ -360,7 +360,7 @@ class PackTest(xla_test.XLATestCase):
         s1 = constant_op.constant(3, dtypes.int32)
         s2 = constant_op.constant(5, dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = self.evaluate(packed)
+        ans = sess.run(packed)
         self.assertAllEqual(ans, [2, 3, 5])
 
   def testEmpty(self):
@@ -370,7 +370,7 @@ class PackTest(xla_test.XLATestCase):
         s1 = constant_op.constant([[]], dtypes.int32)
         s2 = constant_op.constant([[]], dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = self.evaluate(packed)
+        ans = sess.run(packed)
         self.assertAllEqual(ans, [[[]], [[]], [[]]])
 
 
diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index 76706ad40a..63cee550fd 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -106,7 +106,7 @@ class EagerTest(xla_test.XLATestCase):
         three = constant_op.constant(3)
         five = constant_op.constant(5)
         product = three * five
-        self.assertAllEqual(15, self.evaluate(product))
+        self.assertAllEqual(15, sess.run(product))
 
   def testDegenerateSlices(self):
     with self.test_scope():
diff --git a/tensorflow/compiler/tests/function_test.py b/tensorflow/compiler/tests/function_test.py
index dd9b7f30ef..b1891b918c 100644
--- a/tensorflow/compiler/tests/function_test.py
+++ b/tensorflow/compiler/tests/function_test.py
@@ -50,7 +50,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_f = Foo(a, b)
-      result = self.evaluate(call_f)
+      result = sess.run(call_f)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testNestedFunctions(self):
@@ -76,7 +76,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_g = Foo(a, b)
-      result = self.evaluate(call_g)
+      result = sess.run(call_g)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testFunctionMultipleRetvals(self):
@@ -100,7 +100,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_f = Foo(a, b)
-      result = self.evaluate(call_f)
+      result = sess.run(call_f)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testCompileTimeConstantsInDefun(self):
diff --git a/tensorflow/compiler/tests/lstm_test.py b/tensorflow/compiler/tests/lstm_test.py
index fd02a50aff..265c0b6d14 100644
--- a/tensorflow/compiler/tests/lstm_test.py
+++ b/tensorflow/compiler/tests/lstm_test.py
@@ -88,7 +88,7 @@ class LSTMTest(test.TestCase):
                  (basename, m_prev_scalar, c_prev_scalar, pad_scalar))
 
       # Initialize variables and run the unrolled LSTM step.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       return sess.run([m, c])
 
   def testLSTMCell(self):
@@ -173,7 +173,7 @@ class LSTMTest(test.TestCase):
                  (basename, m_init_scalar, c_init_scalar, pad_scalar))
 
       # Initialize variables and run the unrolled LSTM layer.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       return sess.run(out_seq)
 
   def testLSTMLayer(self):
diff --git a/tensorflow/compiler/tests/placeholder_test.py b/tensorflow/compiler/tests/placeholder_test.py
index 9671ae0ae9..77bb839409 100644
--- a/tensorflow/compiler/tests/placeholder_test.py
+++ b/tensorflow/compiler/tests/placeholder_test.py
@@ -33,7 +33,7 @@ class PlaceholderTest(xla_test.XLATestCase):
       ph = array_ops.placeholder_with_default(v, shape=[])
       out = ph * 2
       sess.run(variables.variables_initializer([v]))
-      self.assertEqual(8.0, self.evaluate(out))
+      self.assertEqual(8.0, sess.run(out))
 
   def test_placeholder_with_default_fed(self):
     with self.cached_session() as sess, self.test_scope():
diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py
index 1e91390945..36ef6ed5fe 100644
--- a/tensorflow/compiler/tests/random_ops_test.py
+++ b/tensorflow/compiler/tests/random_ops_test.py
@@ -46,9 +46,9 @@ class RandomOpsTest(xla_test.XLATestCase):
 
       # The random-number generator, if working correctly, should produce the
       # same output multiple times with low probability.
-      y = self.evaluate(x)
-      z = self.evaluate(x)
-      w = self.evaluate(x)
+      y = sess.run(x)
+      z = sess.run(x)
+      w = sess.run(x)
 
       # We use exact equality here. If the random-number generator is producing
       # deterministic output, all three outputs will be bitwise identical.
@@ -83,7 +83,7 @@ class RandomOpsTest(xla_test.XLATestCase):
         with self.test_scope():
           x = random_ops.random_uniform(
               shape=[1000], dtype=dtype, minval=-2, maxval=33)
-        y = self.evaluate(x)
+        y = sess.run(x)
         self.assertTrue((y >= -2).sum() == 1000)
         self.assertTrue((y < 33).sum() == 1000)
 
@@ -102,7 +102,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.cached_session() as sess:
         with self.test_scope():
           x = random_ops.truncated_normal(shape=[count], dtype=dtype)
-        y = self.evaluate(x)
+        y = sess.run(x)
 
         def normal_cdf(x):
           return .5 * math.erfc(-x / math.sqrt(2))
@@ -148,7 +148,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.test_scope():
         x = math_ops.range(1 << 16)
         shuffle = random_ops.random_shuffle(x)
-      result = self.evaluate(shuffle)
+      result = sess.run(shuffle)
       expected = range(1 << 16)
       # Compare sets to avoid randomness behavior changes but make sure still
       # have all the values.
@@ -159,7 +159,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.test_scope():
         x = array_ops.diag(math_ops.range(20))
         shuffle = random_ops.random_shuffle(x)
-      result = self.evaluate(shuffle)
+      result = sess.run(shuffle)
       expected = np.diag(range(20)).flatten()
       # Compare sets to avoid randomness behavior changes but make sure still
       # have all the values.
diff --git a/tensorflow/compiler/tests/tensor_array_ops_test.py b/tensorflow/compiler/tests/tensor_array_ops_test.py
index d7e26d79c4..c8208adb58 100644
--- a/tensorflow/compiler/tests/tensor_array_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_array_ops_test.py
@@ -505,7 +505,7 @@ class TensorArrayTest(xla_test.XLATestCase):
                 [-0.5, 1.5],  # read(0) gradient
                 [20.0, 30.0, 40.0, 50.0],  # concat gradient
             ])
-      grad_vals = self.evaluate(grad_r)  # 2 + 2 entries
+      grad_vals = sess.run(grad_r)  # 2 + 2 entries
 
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py
index e776c8a951..77cdeac816 100644
--- a/tensorflow/compiler/tests/variable_ops_test.py
+++ b/tensorflow/compiler/tests/variable_ops_test.py
@@ -229,7 +229,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_add(
               handle, [0], constant_op.constant([[2]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(self.evaluate(read), [[3], [7]])
+      self.assertAllEqual(sess.run(read), [[3], [7]])
 
   def testScatterSub(self):
     with self.test_session() as sess, self.test_scope():
@@ -242,7 +242,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_sub(
               handle, [1], constant_op.constant([[2]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(self.evaluate(read), [[4], [-1]])
+      self.assertAllEqual(sess.run(read), [[4], [-1]])
 
   def testScatterMul(self):
     with self.test_session() as sess, self.test_scope():
@@ -255,7 +255,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_mul(
               handle, [0], constant_op.constant([[5]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[5]])
+      self.assertEqual(sess.run(read), [[5]])
 
   def testScatterDiv(self):
     with self.test_session() as sess, self.test_scope():
@@ -268,7 +268,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_div(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(self.evaluate(read), [[2]])
+      self.assertAllEqual(sess.run(read), [[2]])
 
   def testScatterMin(self):
     with self.test_session() as sess, self.test_scope():
@@ -281,7 +281,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_min(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[3]])
+      self.assertEqual(sess.run(read), [[3]])
 
   def testScatterMax(self):
     with self.test_session() as sess, self.test_scope():
@@ -294,7 +294,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_max(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[6]])
+      self.assertEqual(sess.run(read), [[6]])
 
   def testScatterUpdate(self):
     with self.test_session() as sess, self.test_scope():
@@ -307,7 +307,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_update(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[3]])
+      self.assertEqual(sess.run(read), [[3]])
 
   def testScatterAddScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -320,7 +320,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_add(
               handle, [0], constant_op.constant(2, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[3]])
+      self.assertEqual(sess.run(read), [[3]])
 
   def testScatterSubScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -333,7 +333,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_sub(
               handle, [0], constant_op.constant(2, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[-1]])
+      self.assertEqual(sess.run(read), [[-1]])
 
   def testScatterMulScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -346,7 +346,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_mul(
               handle, [0], constant_op.constant(5, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[5]])
+      self.assertEqual(sess.run(read), [[5]])
 
   def testScatterDivScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -359,7 +359,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_div(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[2]])
+      self.assertEqual(sess.run(read), [[2]])
 
   def testScatterMinScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -372,7 +372,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_min(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[3]])
+      self.assertEqual(sess.run(read), [[3]])
 
   def testScatterMaxScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -385,7 +385,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_max(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(self.evaluate(read), [[6]])
+      self.assertEqual(sess.run(read), [[6]])
 
   def testScatterNdAddOps(self):
     with self.test_session() as sess, self.test_scope():
@@ -400,7 +400,7 @@ class VariableOpsTest(xla_test.XLATestCase):
       sess.run(gen_state_ops.resource_scatter_nd_add(handle, indices, updates))
       read = resource_variable_ops.read_variable_op(
           handle, dtype=dtypes.float32)
-      self.assertAllClose(expected, self.evaluate(read))
+      self.assertAllClose(expected, sess.run(read))
 
   def testScatterNdUpdateAddOps(self):
     with self.test_session() as sess, self.test_scope():
@@ -416,7 +416,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           gen_state_ops.resource_scatter_nd_update(handle, indices, updates))
       read = resource_variable_ops.read_variable_op(
           handle, dtype=dtypes.float32)
-      self.assertAllClose(expected, self.evaluate(read))
+      self.assertAllClose(expected, sess.run(read))
 
 
 class StridedSliceAssignChecker(object):
diff --git a/tensorflow/examples/autograph/integration_tests/keras_test.py b/tensorflow/examples/autograph/integration_tests/keras_test.py
index 9828ac34dc..dca7c07b47 100644
--- a/tensorflow/examples/autograph/integration_tests/keras_test.py
+++ b/tensorflow/examples/autograph/integration_tests/keras_test.py
@@ -96,7 +96,7 @@ class KerasTest(tf.test.TestCase):
           sess.run(init)
           sample_input = tf.random_uniform((1, 10, 10, 1))
           output = model(sample_input)  # pylint: disable=not-callable
-          self.assertEqual(self.evaluate(output).shape, (1, 3))
+          self.assertEqual(sess.run(output).shape, (1, 3))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/examples/autograph/integration_tests/list_literals_test.py b/tensorflow/examples/autograph/integration_tests/list_literals_test.py
index e85d4abcfc..917f5ff9d8 100644
--- a/tensorflow/examples/autograph/integration_tests/list_literals_test.py
+++ b/tensorflow/examples/autograph/integration_tests/list_literals_test.py
@@ -34,7 +34,7 @@ class ListLiteralsTest(tf.test.TestCase):
     result = converted()
 
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(result), [1, 2, 3])
+      self.assertAllEqual(sess.run(result), [1, 2, 3])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/examples/speech_commands/input_data_test.py b/tensorflow/examples/speech_commands/input_data_test.py
index 33b58b9d09..b766ba6de0 100644
--- a/tensorflow/examples/speech_commands/input_data_test.py
+++ b/tensorflow/examples/speech_commands/input_data_test.py
@@ -35,7 +35,7 @@ class InputDataTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([32000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = self.evaluate(wav_encoder)
+      wav_data = sess.run(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/examples/speech_commands/label_wav_test.py b/tensorflow/examples/speech_commands/label_wav_test.py
index 77a88f98e1..f0af2a4798 100644
--- a/tensorflow/examples/speech_commands/label_wav_test.py
+++ b/tensorflow/examples/speech_commands/label_wav_test.py
@@ -33,7 +33,7 @@ class LabelWavTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([1000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = self.evaluate(wav_encoder)
+      wav_data = sess.run(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/examples/speech_commands/wav_to_features_test.py b/tensorflow/examples/speech_commands/wav_to_features_test.py
index cb8ea912fa..87f2987693 100644
--- a/tensorflow/examples/speech_commands/wav_to_features_test.py
+++ b/tensorflow/examples/speech_commands/wav_to_features_test.py
@@ -33,7 +33,7 @@ class WavToFeaturesTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([32000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = self.evaluate(wav_encoder)
+      wav_data = sess.run(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 892f90e350..916c736fb4 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -113,7 +113,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.compiled(node, ns) as result:
       with self.cached_session() as sess:
         result_tensor = result.test_fn(constant_op.constant(1))
-        self.assertEquals(self.evaluate(result_tensor), 3)
+        self.assertEquals(sess.run(result_tensor), 3)
 
   def test_call_to_decorated_function(self):
 
diff --git a/tensorflow/python/autograph/converters/lists_test.py b/tensorflow/python/autograph/converters/lists_test.py
index 8c8135acef..f6da845fcc 100644
--- a/tensorflow/python/autograph/converters/lists_test.py
+++ b/tensorflow/python/autograph/converters/lists_test.py
@@ -68,7 +68,7 @@ class ListTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(self.evaluate(r), [1, 2, 3])
+        self.assertAllEqual(sess.run(r), [1, 2, 3])
 
   def test_list_pop(self):
 
@@ -91,8 +91,8 @@ class ListTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         ts, tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(self.evaluate(r), [1, 2])
-        self.assertAllEqual(self.evaluate(ts), 3)
+        self.assertAllEqual(sess.run(r), [1, 2])
+        self.assertAllEqual(sess.run(ts), 3)
 
   def test_double_list_pop(self):
 
diff --git a/tensorflow/python/autograph/converters/side_effect_guards_test.py b/tensorflow/python/autograph/converters/side_effect_guards_test.py
index e72b5eac32..cef3199169 100644
--- a/tensorflow/python/autograph/converters/side_effect_guards_test.py
+++ b/tensorflow/python/autograph/converters/side_effect_guards_test.py
@@ -48,12 +48,12 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        self.evaluate(v.initializer)
+        sess.run(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Add support for this use case.
         # Right now the variable `a` is not conditioned on the `assign` because
         # there's no way to add control dependencies to a variable object.
-        self.assertEqual(2, self.evaluate(v))
+        self.assertEqual(2, sess.run(v))
 
   def test_side_effect_on_used_variable(self):
 
@@ -69,11 +69,11 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        self.evaluate(v.initializer)
+        sess.run(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
         # Right now it's 3 or 4 based on whether the read is synchronized.
-        self.assertEqual(3, self.evaluate(v))
+        self.assertEqual(3, sess.run(v))
 
   def test_side_effect_on_tensor(self):
 
@@ -109,10 +109,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign_add) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        self.evaluate(v.initializer)
+        sess.run(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(4, self.evaluate(v))
+        self.assertEqual(4, sess.run(v))
 
   def test_multiline_nested_block(self):
 
@@ -130,10 +130,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign, ops.name_scope) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        self.evaluate(v.initializer)
+        sess.run(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(3, self.evaluate(v))
+        self.assertEqual(3, sess.run(v))
 
   def test_multiline_block_unsafe(self):
 
@@ -153,10 +153,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
                        state_ops.assign_add) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        self.evaluate(v.initializer)
+        sess.run(v.initializer)
         sess.run(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(4, self.evaluate(v))
+        self.assertEqual(4, sess.run(v))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py
index bd049afdfc..e190a7cfe8 100644
--- a/tensorflow/python/autograph/converters/slices_test.py
+++ b/tensorflow/python/autograph/converters/slices_test.py
@@ -49,7 +49,7 @@ class SliceTest(converter_testing.TestCase):
         tl = list_ops.tensor_list_from_tensor(
             [1, 2], element_shape=constant_op.constant([], dtype=dtypes.int32))
         y = result.test_fn(tl)
-        self.assertEqual(2, self.evaluate(y))
+        self.assertEqual(2, sess.run(y))
 
   def test_index_access_multiple_definitions(self):
 
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 44cb99d657..ef577568c4 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -63,7 +63,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+      self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_decorator_does_not_recurse(self):
 
@@ -83,7 +83,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+      self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_decorator_calls_unconverted_graph(self):
 
@@ -104,7 +104,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+      self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_decorator_calls_unconverted_py_func(self):
 
@@ -130,7 +130,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+      self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_decorator_calls_decorated(self):
 
@@ -153,7 +153,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+      self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_decorator_preserves_argspec(self):
 
@@ -192,7 +192,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], self.evaluate(x).tolist())
+      self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_converted_call_builtin(self):
     x = api.converted_call(range, None, converter.ConversionOptions(), 3)
@@ -208,7 +208,7 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       x = api.converted_call(test_fn, None, converter.ConversionOptions(),
                              constant_op.constant(-1))
-      self.assertEqual(1, self.evaluate(x))
+      self.assertEqual(1, sess.run(x))
 
   def test_converted_call_method_explicit_owner(self):
     # TODO(mdan): Implement.
@@ -234,7 +234,7 @@ class ApiTest(test.TestCase):
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(tc.test_method, None,
                              converter.ConversionOptions(), tc)
-      self.assertEqual(1, self.evaluate(x))
+      self.assertEqual(1, sess.run(x))
 
   def test_converted_call_method_by_class(self):
 
@@ -252,7 +252,7 @@ class ApiTest(test.TestCase):
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(TestClass.test_method, None,
                              converter.ConversionOptions(), tc)
-      self.assertEqual(1, self.evaluate(x))
+      self.assertEqual(1, sess.run(x))
 
   def test_converted_call_callable_object(self):
 
@@ -269,7 +269,7 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(tc, None, converter.ConversionOptions())
-      self.assertEqual(1, self.evaluate(x))
+      self.assertEqual(1, sess.run(x))
 
   def test_converted_call_constructor(self):
 
@@ -288,7 +288,7 @@ class ApiTest(test.TestCase):
                               constant_op.constant(-1))
       # tc is now a converted object.
       x = tc.test_method()
-      self.assertEqual(1, self.evaluate(x))
+      self.assertEqual(1, sess.run(x))
 
   def test_converted_call_already_converted(self):
 
@@ -298,12 +298,12 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       x = api.converted_call(f, None, converter.ConversionOptions(),
                              constant_op.constant(0))
-      self.assertTrue(self.evaluate(x))
+      self.assertTrue(sess.run(x))
 
       converted_f = api.to_graph(f)
       x = api.converted_call(converted_f, None, converter.ConversionOptions(),
                              constant_op.constant(0))
-      self.assertTrue(self.evaluate(x))
+      self.assertTrue(sess.run(x))
 
   def test_converted_call_no_user_code(self):
 
@@ -334,8 +334,8 @@ class ApiTest(test.TestCase):
                            constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+      sess.run(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
 
   def test_converted_call_whitelisted_method_extra_self(self):
 
@@ -349,8 +349,8 @@ class ApiTest(test.TestCase):
                            model, constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+      sess.run(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
 
   def test_converted_call_whitelisted_method_via_owner(self):
 
@@ -364,8 +364,8 @@ class ApiTest(test.TestCase):
                            constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
+      sess.run(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
 
   def test_converted_call_lambda(self):
 
@@ -376,8 +376,8 @@ class ApiTest(test.TestCase):
     x = api.converted_call(l, None, opts, constant_op.constant(0))
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      self.assertAllEqual(True, self.evaluate(x))
+      sess.run(variables.global_variables_initializer())
+      self.assertAllEqual(True, sess.run(x))
 
   def test_to_graph_basic(self):
 
@@ -390,7 +390,7 @@ class ApiTest(test.TestCase):
 
     with self.cached_session() as sess:
       x = compiled_fn(constant_op.constant([4, 8]), 4)
-      self.assertListEqual([1, 2], self.evaluate(x).tolist())
+      self.assertListEqual([1, 2], sess.run(x).tolist())
 
   def test_to_graph_with_defaults(self):
 
@@ -405,7 +405,7 @@ class ApiTest(test.TestCase):
 
     with self.cached_session() as sess:
       x = compiled_fn(constant_op.constant([4, 8]))
-      self.assertListEqual([1, 2], self.evaluate(x).tolist())
+      self.assertListEqual([1, 2], sess.run(x).tolist())
 
   def test_to_code_basic(self):
 
diff --git a/tensorflow/python/autograph/lang/special_functions_test.py b/tensorflow/python/autograph/lang/special_functions_test.py
index 8d40f4036c..123ee65b32 100644
--- a/tensorflow/python/autograph/lang/special_functions_test.py
+++ b/tensorflow/python/autograph/lang/special_functions_test.py
@@ -36,7 +36,7 @@ class SpecialFunctionsTest(test.TestCase):
     python_one = special_functions.match_staging_level(1, 1)
     with self.cached_session() as sess:
       self.assertTrue(tensor_util.is_tensor(tensor_one))
-      self.assertAllEqual(self.evaluate(tensor_one), 1)
+      self.assertAllEqual(sess.run(tensor_one), 1)
       self.assertEqual(python_one, 1)
 
   def test_tensor_list_empty_list(self):
@@ -45,21 +45,21 @@ class SpecialFunctionsTest(test.TestCase):
                                       element_shape=())
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(sl), [])
+      self.assertAllEqual(sess.run(sl), [])
 
     l = special_functions.tensor_list((),
                                       element_dtype=dtypes.int32,
                                       element_shape=())
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(sl), [])
+      self.assertAllEqual(sess.run(sl), [])
 
   def test_tensor_list_tensor(self):
     l = special_functions.tensor_list(
         constant_op.constant([], dtype=dtypes.int32))
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(sl), [])
+      self.assertAllEqual(sess.run(sl), [])
 
   def test_tensor_list_unsupported_initializer(self):
     with self.assertRaisesRegexp(ValueError, 'unknown type'):
@@ -76,7 +76,7 @@ class SpecialFunctionsTest(test.TestCase):
     l = special_functions.tensor_list(elements)
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]])
+      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
 
   def test_tensor_list_array_from_elements(self):
     elements = [constant_op.constant([1, 2]), constant_op.constant([3, 4])]
@@ -84,7 +84,7 @@ class SpecialFunctionsTest(test.TestCase):
     l = special_functions.tensor_list(elements, use_tensor_array=True)
     sl = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]])
+      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
 
   def test_stack(self):
     self.assertEqual(special_functions.stack(1, strict=False), 1)
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 05b5660941..2dea18dc5f 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -35,7 +35,7 @@ class ForLoopTest(test.TestCase):
         body=lambda i, s: (s + i,),
         init_state=(0,))
     with self.cached_session() as sess:
-      self.assertEqual((10,), self.evaluate(s))
+      self.assertEqual((10,), sess.run(s))
 
   def test_python(self):
     s = control_flow.for_stmt(
@@ -53,7 +53,7 @@ class ForLoopTest(test.TestCase):
         body=lambda i, s: (s + i,),
         init_state=(0,))
     with self.cached_session() as sess:
-      self.assertEqual((10,), self.evaluate(s))
+      self.assertEqual((10,), sess.run(s))
 
 
 class WhileLoopTest(test.TestCase):
@@ -66,7 +66,7 @@ class WhileLoopTest(test.TestCase):
         init_state=(0, 0),
         extra_deps=(n,))
     with self.cached_session() as sess:
-      self.assertEqual((5, 10), self.evaluate(results))
+      self.assertEqual((5, 10), sess.run(results))
 
   def test_python(self):
     n = 5
@@ -90,9 +90,9 @@ class IfStmtTest(test.TestCase):
   def test_tensor(self):
     with self.cached_session() as sess:
       t = self.single_return_if_stmt(constant_op.constant(True))
-      self.assertEqual(1, self.evaluate(t))
+      self.assertEqual(1, sess.run(t))
       t = self.single_return_if_stmt(constant_op.constant(False))
-      self.assertEqual(-1, self.evaluate(t))
+      self.assertEqual(-1, sess.run(t))
 
   def test_python(self):
     self.assertEqual(1, self.single_return_if_stmt(True))
@@ -101,9 +101,9 @@ class IfStmtTest(test.TestCase):
   def test_tensor_multiple_returns(self):
     with self.cached_session() as sess:
       t = self.multi_return_if_stmt(constant_op.constant(True))
-      self.assertAllEqual([1, 2], self.evaluate(t))
+      self.assertAllEqual([1, 2], sess.run(t))
       t = self.multi_return_if_stmt(constant_op.constant(False))
-      self.assertAllEqual([-1, -2], self.evaluate(t))
+      self.assertAllEqual([-1, -2], sess.run(t))
 
   def test_python_multiple_returns(self):
     self.assertEqual((1, 2), self.multi_return_if_stmt(True))
diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py
index dc50edb4c9..72476ccb6b 100644
--- a/tensorflow/python/autograph/operators/data_structures_test.py
+++ b/tensorflow/python/autograph/operators/data_structures_test.py
@@ -43,7 +43,7 @@ class ListTest(test.TestCase):
     l = data_structures.tf_tensor_list_new([3, 4, 5])
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
+      self.assertAllEqual(sess.run(t), [3, 4, 5])
 
   def test_tf_tensor_list_new_empty(self):
     l = data_structures.tf_tensor_list_new([],
@@ -51,13 +51,13 @@ class ListTest(test.TestCase):
                                            element_shape=())
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(t), [])
+      self.assertAllEqual(sess.run(t), [])
 
   def test_tf_tensor_list_new_from_tensor(self):
     l = data_structures.tf_tensor_list_new(constant_op.constant([3, 4, 5]))
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
+      self.assertAllEqual(sess.run(t), [3, 4, 5])
 
   def test_tf_tensor_list_new_illegal_input(self):
     with self.assertRaises(ValueError):
@@ -77,7 +77,7 @@ class ListTest(test.TestCase):
     l = data_structures.tf_tensor_array_new([3, 4, 5])
     t = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
+      self.assertAllEqual(sess.run(t), [3, 4, 5])
 
   def test_tf_tensor_array_new_illegal_input(self):
     with self.assertRaises(ValueError):
@@ -102,7 +102,7 @@ class ListTest(test.TestCase):
 
     t = list_ops.tensor_list_stack(l, element_dtype=x.dtype)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(t), [[1, 2, 3]])
+      self.assertAllEqual(sess.run(t), [[1, 2, 3]])
 
   def test_append_tensorarray(self):
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
@@ -131,10 +131,10 @@ class ListTest(test.TestCase):
 
     with self.cached_session() as sess:
       l, x = data_structures.list_pop(l, None, opts)
-      self.assertAllEqual(self.evaluate(x), [3, 4])
+      self.assertAllEqual(sess.run(x), [3, 4])
 
       t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
-      self.assertAllEqual(self.evaluate(t), [[1, 2]])
+      self.assertAllEqual(sess.run(t), [[1, 2]])
 
   def test_pop_python(self):
     l = [1, 2, 3]
@@ -152,7 +152,7 @@ class ListTest(test.TestCase):
 
     with self.cached_session() as sess:
       t = data_structures.list_stack(l, opts)
-      self.assertAllEqual(sess.run(t), self.evaluate(initial_list))
+      self.assertAllEqual(sess.run(t), sess.run(initial_list))
 
   def test_stack_tensor_list_empty(self):
     l = list_ops.empty_tensor_list(
diff --git a/tensorflow/python/autograph/operators/logical_test.py b/tensorflow/python/autograph/operators/logical_test.py
index ebf6458f01..d6649f7b2b 100644
--- a/tensorflow/python/autograph/operators/logical_test.py
+++ b/tensorflow/python/autograph/operators/logical_test.py
@@ -45,11 +45,11 @@ class LogicalOperatorsTest(test.TestCase):
   def test_and_tf(self):
     with self.cached_session() as sess:
       t = logical.and_(self._tf_true, self._tf_true)
-      self.assertEqual(self.evaluate(t), True)
+      self.assertEqual(sess.run(t), True)
       t = logical.and_(self._tf_true, lambda: True)
-      self.assertEqual(self.evaluate(t), True)
+      self.assertEqual(sess.run(t), True)
       t = logical.and_(self._tf_false, lambda: True)
-      self.assertEqual(self.evaluate(t), False)
+      self.assertEqual(sess.run(t), False)
       # TODO(mdan): Add a test for ops with side effects.
 
   def test_or_python(self):
@@ -63,11 +63,11 @@ class LogicalOperatorsTest(test.TestCase):
   def test_or_tf(self):
     with self.cached_session() as sess:
       t = logical.or_(self._tf_false, self._tf_true)
-      self.assertEqual(self.evaluate(t), True)
+      self.assertEqual(sess.run(t), True)
       t = logical.or_(self._tf_false, lambda: True)
-      self.assertEqual(self.evaluate(t), True)
+      self.assertEqual(sess.run(t), True)
       t = logical.or_(self._tf_true, lambda: True)
-      self.assertEqual(self.evaluate(t), True)
+      self.assertEqual(sess.run(t), True)
       # TODO(mdan): Add a test for ops with side effects.
 
   def test_not_python(self):
@@ -78,7 +78,7 @@ class LogicalOperatorsTest(test.TestCase):
   def test_not_tf(self):
     with self.cached_session() as sess:
       t = logical.not_(self._tf_false())
-      self.assertEqual(self.evaluate(t), True)
+      self.assertEqual(sess.run(t), True)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/operators/py_builtins_test.py b/tensorflow/python/autograph/operators/py_builtins_test.py
index 4d9eec77c3..443e30a475 100644
--- a/tensorflow/python/autograph/operators/py_builtins_test.py
+++ b/tensorflow/python/autograph/operators/py_builtins_test.py
@@ -38,29 +38,29 @@ class PyBuiltinsTest(test.TestCase):
     self.assertEqual(py_builtins.abs_(-1), 1)
     with self.cached_session() as sess:
       t = py_builtins.abs_(constant_op.constant(-1))
-      self.assertEqual(self.evaluate(t), 1)
+      self.assertEqual(sess.run(t), 1)
       t = py_builtins.abs_(constant_op.constant([-1, 2, -3]))
-      self.assertAllEqual(self.evaluate(t), [1, 2, 3])
+      self.assertAllEqual(sess.run(t), [1, 2, 3])
 
   def test_float(self):
     self.assertEqual(py_builtins.float_(10), 10.0)
     self.assertEqual(py_builtins.float_('10.0'), 10.0)
     with self.cached_session() as sess:
       t = py_builtins.float_(constant_op.constant(1, dtype=dtypes.int64))
-      self.assertEqual(self.evaluate(t), 1.0)
+      self.assertEqual(sess.run(t), 1.0)
       st = py_builtins.float_(constant_op.constant('1.0'))
-      self.assertEqual(self.evaluate(st), 1.0)
+      self.assertEqual(sess.run(st), 1.0)
 
   def test_int(self):
     self.assertEqual(py_builtins.int_(10.0), 10)
     self.assertEqual(py_builtins.int_('11', 2), 3)
     with self.cached_session() as sess:
       t = py_builtins.int_(constant_op.constant(1, dtype=dtypes.float64))
-      self.assertEqual(self.evaluate(t), 1)
+      self.assertEqual(sess.run(t), 1)
       st = py_builtins.int_(constant_op.constant('1'))
-      self.assertEqual(self.evaluate(st), 1)
+      self.assertEqual(sess.run(st), 1)
       st = py_builtins.int_(constant_op.constant('1'), 10)
-      self.assertEqual(self.evaluate(st), 1)
+      self.assertEqual(sess.run(st), 1)
 
   def test_int_unsupported_base(self):
     t = constant_op.constant(1, dtype=dtypes.float64)
@@ -73,9 +73,9 @@ class PyBuiltinsTest(test.TestCase):
       t = py_builtins.len_(constant_op.constant([[1], [2], [3]]))
       self.assertEqual(t, 3)
       ta = py_builtins.len_(tensor_array_ops.TensorArray(dtypes.int32, size=5))
-      self.assertEqual(self.evaluate(ta), 5)
+      self.assertEqual(sess.run(ta), 5)
       tl = py_builtins.len_(data_structures.tf_tensor_list_new([3, 4, 5]))
-      self.assertEqual(self.evaluate(tl), 3)
+      self.assertEqual(sess.run(tl), 3)
 
   def test_len_scalar(self):
     with self.assertRaises(ValueError):
@@ -120,18 +120,18 @@ class PyBuiltinsTest(test.TestCase):
   def test_range_tensor(self):
     with self.cached_session() as sess:
       r = py_builtins.range_(constant_op.constant(3))
-      self.assertAllEqual(self.evaluate(r), [0, 1, 2])
+      self.assertAllEqual(sess.run(r), [0, 1, 2])
       r = py_builtins.range_(1, constant_op.constant(3))
-      self.assertAllEqual(self.evaluate(r), [1, 2])
+      self.assertAllEqual(sess.run(r), [1, 2])
       r = py_builtins.range_(2, 0, constant_op.constant(-1))
-      self.assertAllEqual(self.evaluate(r), [2, 1])
+      self.assertAllEqual(sess.run(r), [2, 1])
 
   def test_range_tensor_empty_range(self):
     with self.session() as sess:
       r = py_builtins.range_(constant_op.constant(-3))
-      self.assertAllEqual(self.evaluate(r), [])
+      self.assertAllEqual(sess.run(r), [])
       r = py_builtins.range_(5, constant_op.constant(2))
-      self.assertAllEqual(self.evaluate(r), [])
+      self.assertAllEqual(sess.run(r), [])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/operators/slices_test.py b/tensorflow/python/autograph/operators/slices_test.py
index d444054fd7..9e4865b3c6 100644
--- a/tensorflow/python/autograph/operators/slices_test.py
+++ b/tensorflow/python/autograph/operators/slices_test.py
@@ -34,7 +34,7 @@ class SlicesTest(test.TestCase):
 
     with self.cached_session() as sess:
       t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
-      self.assertAllEqual(self.evaluate(t), [[5, 6], [3, 4]])
+      self.assertAllEqual(sess.run(t), [[5, 6], [3, 4]])
 
   def test_get_item_tensor_list(self):
     initial_list = constant_op.constant([[1, 2], [3, 4]])
@@ -44,7 +44,7 @@ class SlicesTest(test.TestCase):
         l, 1, slices.GetItemOpts(element_dtype=initial_list.dtype))
 
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(t), [3, 4])
+      self.assertAllEqual(sess.run(t), [3, 4])
 
   def test_get_item_tensor_string(self):
     initial_str = constant_op.constant('abcd')
@@ -52,14 +52,14 @@ class SlicesTest(test.TestCase):
                         slices.GetItemOpts(element_dtype=initial_str.dtype))
 
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(t), b'b')
+      self.assertEqual(sess.run(t), b'b')
 
     initial_list_str = constant_op.constant(['abcd', 'bcde'])
     t = slices.get_item(initial_list_str, 1,
                         slices.GetItemOpts(element_dtype=initial_str.dtype))
 
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(t), b'bcde')
+      self.assertEqual(sess.run(t), b'bcde')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py
index c813e0f5c9..8d2b0d6e13 100644
--- a/tensorflow/python/autograph/utils/misc_test.py
+++ b/tensorflow/python/autograph/utils/misc_test.py
@@ -32,7 +32,7 @@ class MiscTest(test.TestCase):
     new_a = alias_tensors(a)
     self.assertFalse(new_a is a)
     with self.cached_session() as sess:
-      self.assertEqual(1, self.evaluate(new_a))
+      self.assertEqual(1, sess.run(new_a))
 
   def test_alias_tensors(self):
     a = constant(1)
@@ -47,7 +47,7 @@ class MiscTest(test.TestCase):
     self.assertTrue(new_s is s)
     self.assertTrue(new_l is l)
     with self.cached_session() as sess:
-      self.assertEqual(1, self.evaluate(new_a))
+      self.assertEqual(1, sess.run(new_a))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/utils/py_func_test.py b/tensorflow/python/autograph/utils/py_func_test.py
index 28cefd8c3e..1c220d9492 100644
--- a/tensorflow/python/autograph/utils/py_func_test.py
+++ b/tensorflow/python/autograph/utils/py_func_test.py
@@ -34,13 +34,13 @@ class PyFuncTest(test.TestCase):
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (1, constant_op.constant(1), 1))
-      self.assertEqual(3, self.evaluate(result))
+      self.assertEqual(3, sess.run(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, 1, 1))
-      self.assertEqual(3, self.evaluate(result))
+      self.assertEqual(3, sess.run(result))
       result = py_func.wrap_py_func(
           test_fn, dtypes.int64,
           (constant_op.constant(1), 1, constant_op.constant(1)))
-      self.assertEqual(3, self.evaluate(result))
+      self.assertEqual(3, sess.run(result))
 
   def test_wrap_py_func_complex_args(self):
 
@@ -54,10 +54,10 @@ class PyFuncTest(test.TestCase):
 
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass()))
-      self.assertEqual(35, self.evaluate(result))
+      self.assertEqual(35, sess.run(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (constant_op.constant(7), TestClass()))
-      self.assertEqual(35, self.evaluate(result))
+      self.assertEqual(35, sess.run(result))
 
   def test_wrap_py_func_kwargs(self):
 
@@ -74,13 +74,13 @@ class PyFuncTest(test.TestCase):
           'c': 11,
           'd': TestClass(13)
       })
-      self.assertEqual(178, self.evaluate(result))
+      self.assertEqual(178, sess.run(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (constant_op.constant(7), TestClass(5)), {
                                         'c': constant_op.constant(11),
                                         'd': TestClass(13)
                                     })
-      self.assertEqual(178, self.evaluate(result))
+      self.assertEqual(178, sess.run(result))
 
   def test_wrap_py_func_dummy_return(self):
 
@@ -91,11 +91,11 @@ class PyFuncTest(test.TestCase):
 
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, None, (5,), use_dummy_return=True)
-      self.assertEqual(1, self.evaluate(result))
+      self.assertEqual(1, sess.run(result))
       self.assertEqual([1], side_counter)
       result = py_func.wrap_py_func(
           test_fn, None, (constant_op.constant(5),), use_dummy_return=True)
-      self.assertEqual(1, self.evaluate(result))
+      self.assertEqual(1, sess.run(result))
       self.assertEqual([2], side_counter)
 
 
diff --git a/tensorflow/python/autograph/utils/tensor_list_test.py b/tensorflow/python/autograph/utils/tensor_list_test.py
index a5bbd97cf9..697c166eb1 100644
--- a/tensorflow/python/autograph/utils/tensor_list_test.py
+++ b/tensorflow/python/autograph/utils/tensor_list_test.py
@@ -43,13 +43,13 @@ class TensorListTest(test.TestCase):
     l = tl.dynamic_list_append(l, 1)
     s = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(s), [1])
+      self.assertAllEqual(sess.run(s), [1])
 
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l = tl.dynamic_list_append(l, 1)
     s = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(s), [1])
+      self.assertAllEqual(sess.run(s), [1])
 
     l = tl.TensorList(self._shape(()), dtypes.int32)
     l = tl.dynamic_list_append(l, 1)
diff --git a/tensorflow/python/client/session_clusterspec_prop_test.py b/tensorflow/python/client/session_clusterspec_prop_test.py
index 224f880ed1..df020f88a8 100644
--- a/tensorflow/python/client/session_clusterspec_prop_test.py
+++ b/tensorflow/python/client/session_clusterspec_prop_test.py
@@ -62,7 +62,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
 
     const = constant_op.constant(17)
     sess = session.Session(server1.target, config=config)
-    output = self.evaluate(const)
+    output = sess.run(const)
     self.assertEqual(17, output)
 
   def testClusterSpecPropagationWorker2Placement(self):
@@ -106,7 +106,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
     with ops.Graph().as_default() as g, ops.device('/job:worker/task:0'):
       const = constant_op.constant(17)
     sess = session.Session(server1.target, config=config, graph=g)
-    output = self.evaluate(const)
+    output = sess.run(const)
     self.assertEqual(17, output)
 
   def testCanonicalDeviceNames(self):
@@ -208,7 +208,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
       with ops.device('/job:worker/task:0/cpu:0'):
         sum3 = sum1 + sum2
     sess = session.Session(server1.target, config=config, graph=g)
-    output = self.evaluate(sum3)
+    output = sess.run(sum3)
     self.assertEqual(40, output)
 
   def testLegacyDeviceNames(self):
diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index f9bd50957a..dfd0147643 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -147,7 +147,7 @@ class TimelineTest(test.TestCase):
         num2 = variables.Variable(2.0, name='num2')
       with ops.device('/cpu:2'):
         result = num1 + num2 + num1 * num2
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       sess.run(result, options=run_options, run_metadata=run_metadata)
 
     self.assertTrue(run_metadata.HasField('step_stats'))
@@ -176,7 +176,7 @@ class TimelineTest(test.TestCase):
         num2 = variables.Variable(2.0, name='num2')
       with ops.device('/cpu:2'):
         result = num1 + num2 + num1 * num2
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       sess.run(result, options=run_options, run_metadata=run_metadata)
     self.assertTrue(run_metadata.HasField('step_stats'))
     step_stats = run_metadata.step_stats
diff --git a/tensorflow/python/client/virtual_gpu_test.py b/tensorflow/python/client/virtual_gpu_test.py
index e82ee0666c..5892e0fc84 100644
--- a/tensorflow/python/client/virtual_gpu_test.py
+++ b/tensorflow/python/client/virtual_gpu_test.py
@@ -216,7 +216,7 @@ class VirtualGpuTest(test_util.TensorFlowTestCase):
       for d in self._util.devices:
         with ops.device(d):
           var = variables.Variable(random_ops.random_uniform(mat_shape))
-          self.evaluate(var.initializer)
+          sess.run(var.initializer)
           data.append(var)
       s = data[0]
       for i in range(1, len(data)):
diff --git a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
index dbb780c47d..e896752a26 100644
--- a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
@@ -53,10 +53,10 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
       for start in range(0, len(components), 4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual([[i, j]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)], results.indices)
@@ -81,10 +81,10 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
       for start in range(0, len(components), 4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual([[i, j, z]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)
@@ -141,7 +141,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]})
       for i in range(4):
-        self.assertEqual(i, self.evaluate(next_elem))
+        self.assertEqual(i, sess.run(next_elem))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_elem)
 
@@ -159,7 +159,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i,) * 3, self.evaluate(op))
+        self.assertEqual((i,) * 3, sess.run(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -179,7 +179,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i, compat.as_bytes(str(i)), i), self.evaluate(op))
+        self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -198,7 +198,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        st_row = self.evaluate(next_element)
+        st_row = sess.run(next_element)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
         self.assertEqual([10], st_row.dense_shape)
@@ -219,7 +219,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        dense_elem, st_row = self.evaluate(next_element)
+        dense_elem, st_row = sess.run(next_element)
         self.assertEqual(i, dense_elem)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
@@ -241,7 +241,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(((i,),) * 3, self.evaluate(op))
+        self.assertEqual(((i,),) * 3, sess.run(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -354,7 +354,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 28, batch_size: 14})
       num_batches = (28 * 7) // 14
       for i in range(num_batches):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range(14):
             self.assertAllEqual(component[(i * 14 + j) % 7]**2,
@@ -369,12 +369,12 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       # We expect (num_batches - 1) full-sized batches.
       num_batches = int(math.ceil((14 * 7) / 8))
       for i in range(num_batches - 1):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range(8):
             self.assertAllEqual(component[(i * 8 + j) % 7]**2,
                                 result_component[j])
-      result = self.evaluate(get_next)
+      result = sess.run(get_next)
       for component, result_component in zip(components, result):
         for j in range((14 * 7) % 8):
           self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2,
@@ -408,10 +408,10 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
       if not drop_remainder:
-        self.assertAllEqual([[64], [81]], self.evaluate(next_element))
+        self.assertAllEqual([[64], [81]], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -423,9 +423,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
-      self.assertAllEqual([[64], [81]], self.evaluate(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[64], [81]], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -439,7 +439,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(5):
-        got = self.evaluate(elements)
+        got = sess.run(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -459,7 +459,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(4):
-        got = self.evaluate(elements)
+        got = sess.run(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -480,9 +480,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(2):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -524,7 +524,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "number of elements does not match"):
         sess.run(get_next)
@@ -576,8 +576,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(threshold // 10):
-        self.assertAllEqual([i * 10 + j for j in range(10)],
-                            self.evaluate(get_next))
+        self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next))
       if threshold % 10 != 0:
         self.assertAllEqual(
             [threshold // 10 * 10 + j for j in range(threshold % 10)],
@@ -610,8 +609,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(10):
-        self.assertAllEqual([element for _ in range(10)],
-                            self.evaluate(get_next))
+        self.assertAllEqual([element for _ in range(10)], sess.run(get_next))
 
 
 class UnbatchDatasetBenchmark(test.Benchmark):
diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 4263a90f4c..3903ec49b9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -300,7 +300,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       with self.cached_session() as sess:
         with self.assertRaises(errors.OutOfRangeError):
           while True:
-            output = self.evaluate(batch)
+            output = sess.run(batch)
             sprs_tensor = (tuple([tuple(idx) for idx in output.indices]),
                            tuple(output.values))
             all_sparse_tensors.add(sprs_tensor)
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index 6d063ac9c8..cea8bd6f0b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -57,7 +57,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -82,7 +82,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
+      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -108,7 +108,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -134,7 +134,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -160,7 +160,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, self.evaluate(next_element))
+        self.assertEqual({"a": i}, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -186,7 +186,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, self.evaluate(next_element))
+        self.assertEqual({"a": i}, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -217,7 +217,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = self.evaluate(next_element)
+        actual = sess.run(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
@@ -251,7 +251,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = self.evaluate(next_element)
+        actual = sess.run(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
@@ -271,9 +271,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -290,9 +290,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -323,9 +323,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(10):
-        x, y, z = self.evaluate(next_element)
+        x, y, z = sess.run(next_element)
         self.assertEqual(i**2, x)
         self.assertEqual(float(i**2), y)
         self.assertEqual(util_compat.as_bytes(str(i)), z)
@@ -345,8 +345,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
-      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
+      sess.run(iterator.initializer)
+      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -363,8 +363,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
-      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
+      sess.run(iterator.initializer)
+      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -381,8 +381,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
-      self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
+      sess.run(iterator.initializer)
+      self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -399,8 +399,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
-      self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
+      sess.run(iterator.initializer)
+      self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -420,9 +420,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         next_element = iterator.get_next()
 
       with self.cached_session() as sess:
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
         for i in range(10):
-          self.assertEqual(i, self.evaluate(next_element))
+          self.assertEqual(i, sess.run(next_element))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
@@ -447,12 +447,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, self.evaluate(next_element))
-      self.evaluate(iterator.initializer)
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -477,12 +477,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, self.evaluate(next_element))
-      self.evaluate(iterator.initializer)
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -499,12 +499,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, self.evaluate(next_element))
-      self.evaluate(iterator.initializer)
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -521,12 +521,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, self.evaluate(next_element))
-      self.evaluate(iterator.initializer)
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -553,7 +553,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
       # For each element of the dataset, assert that the optional evaluates to
       # the expected value.
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(3):
         elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t])
         self.assertTrue(elem_has_value)
@@ -562,7 +562,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       # After exhausting the iterator, `next_elem.has_value()` will evaluate to
       # false, and attempting to get the value will fail.
       for _ in range(2):
-        self.assertFalse(self.evaluate(elem_has_value_t))
+        self.assertFalse(sess.run(elem_has_value_t))
         with self.assertRaises(errors.InvalidArgumentError):
           sess.run(elem_value_t)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/counter_test.py b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
index d1dd07a879..4e114ac479 100644
--- a/tensorflow/python/data/experimental/kernel_tests/counter_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
@@ -38,13 +38,13 @@ class CounterTest(test_base.DatasetTestBase):
     negative_get_next = negative_iterator.get_next()
 
     with self.cached_session() as sess:
-      self.assertEqual(3, self.evaluate(get_next))
-      self.assertEqual(3 + 4, self.evaluate(get_next))
-      self.assertEqual(3 + 2 * 4, self.evaluate(get_next))
+      self.assertEqual(3, sess.run(get_next))
+      self.assertEqual(3 + 4, sess.run(get_next))
+      self.assertEqual(3 + 2 * 4, sess.run(get_next))
 
-      self.assertEqual(0, self.evaluate(negative_get_next))
-      self.assertEqual(-1, self.evaluate(negative_get_next))
-      self.assertEqual(-2, self.evaluate(negative_get_next))
+      self.assertEqual(0, sess.run(negative_get_next))
+      self.assertEqual(-1, sess.run(negative_get_next))
+      self.assertEqual(-2, sess.run(negative_get_next))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
index 9fe2ee43ed..73be6cbcca 100644
--- a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
@@ -41,10 +41,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
       for start in range(0, len(components), 4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual([[i, j]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)], results.indices)
@@ -69,10 +69,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
       for start in range(0, len(components), 4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual([[i, j, z]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)
diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
index 234fd86bdd..796a692c56 100644
--- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
@@ -40,10 +40,10 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for _ in range(100):
         for i in range(10):
-          self.assertEqual(i, self.evaluate(next_element))
+          self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -107,7 +107,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in choice_array:
-        self.assertEqual(words[i], self.evaluate(next_element))
+        self.assertEqual(words[i], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
index 78805bb801..e54235d9f8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
@@ -44,9 +44,9 @@ class EnumerateDatasetTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next[1]])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertEqual((20, (b"a", 1, 37.0)), self.evaluate(get_next))
-      self.assertEqual((21, (b"b", 2, 38.0)), self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertEqual((20, (b"a", 1, 37.0)), sess.run(get_next))
+      self.assertEqual((21, (b"b", 2, 38.0)), sess.run(get_next))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
diff --git a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
index 860442571e..d38452e265 100644
--- a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
@@ -94,18 +94,18 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
                                                   device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [5.0])
-      self.evaluate(destroy_op)
+      sess.run(destroy_op)
 
   def testSameDeviceCPU(self):
     self._prefetch_fn_helper_one_shot("same_device_cpu",
@@ -135,35 +135,35 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
         ds, ds_iterator, "reinit", device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      self.evaluate(ds_iterator.initializer)
-      elem = self.evaluate(prefetch_op)
+      sess.run(ds_iterator.initializer)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [5.0])
       # Lets reset the function buffering resource and reinitialize the
       # iterator. Should be able to go through this again.
       self._event.clear()
-      self.evaluate(reset_op)
-      self.evaluate(ds_iterator.initializer)
-      elem = self.evaluate(prefetch_op)
+      sess.run(reset_op)
+      sess.run(ds_iterator.initializer)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = self.evaluate(prefetch_op)
+      elem = sess.run(prefetch_op)
       self.assertEqual(elem, [5.0])
-      self.evaluate(destroy_op)
+      sess.run(destroy_op)
 
   def testReinitializationOutOfRange(self):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
@@ -175,30 +175,30 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
         ds, ds_iterator, "reinit", device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      self.evaluate(ds_iterator.initializer)
+      sess.run(ds_iterator.initializer)
       for i in range(1, 10):
-        elem = self.evaluate(prefetch_op)
+        elem = sess.run(prefetch_op)
         self.assertEqual(elem, [float(i)])
       # Try fetching after its over twice to test out end of sequence.
       with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
+        sess.run(prefetch_op)
       with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
+        sess.run(prefetch_op)
 
       # Now reset everything and try it out again.
       self._event.clear()
-      self.evaluate(reset_op)
-      self.evaluate(ds_iterator.initializer)
+      sess.run(reset_op)
+      sess.run(ds_iterator.initializer)
       for i in range(1, 10):
-        elem = self.evaluate(prefetch_op)
+        elem = sess.run(prefetch_op)
         self.assertEqual(elem, [float(i)])
       # Try fetching after its over twice to test out end of sequence.
       with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
+        sess.run(prefetch_op)
       with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
+        sess.run(prefetch_op)
 
-      self.evaluate(destroy_op)
+      sess.run(destroy_op)
 
   def testStringsGPU(self):
     if not test_util.is_gpu_available():
@@ -235,13 +235,13 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
           buffer_resource_handle, ignore_lookup_error=True)
 
     with self.cached_session() as sess:
-      self.assertEqual([b"a"], self.evaluate(prefetch_op))
-      self.assertEqual([b"b"], self.evaluate(prefetch_op))
-      self.assertEqual([b"c"], self.evaluate(prefetch_op))
+      self.assertEqual([b"a"], sess.run(prefetch_op))
+      self.assertEqual([b"b"], sess.run(prefetch_op))
+      self.assertEqual([b"c"], sess.run(prefetch_op))
       with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
+        sess.run(prefetch_op)
 
-      self.evaluate(destroy_op)
+      sess.run(destroy_op)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
index 15396f329d..9030328593 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
@@ -39,7 +39,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
     get_next = dataset.make_one_shot_iterator().get_next()
     with self.cached_session() as sess:
       for expected in values:
-        got = self.evaluate(get_next)
+        got = sess.run(get_next)
         self.assertEqual(got, expected)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -127,7 +127,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       iterator = dataset.make_one_shot_iterator()
       get_next = iterator.get_next()
       with self.cached_session() as sess:
-        x, y = self.evaluate(get_next)
+        x, y = sess.run(get_next)
         self.assertAllEqual([0] * (2**i), x)
         self.assertAllEqual(np.array(1, ndmin=i), y)
         with self.assertRaises(errors.OutOfRangeError):
@@ -190,7 +190,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
             grouping.group_by_reducer(lambda x, y: np.int64(0), reducer))
     get_next = dataset.make_one_shot_iterator().get_next()
     with self.cached_session() as sess:
-      x, y = self.evaluate(get_next)
+      x, y = sess.run(get_next)
       self.assertAllEqual(x, np.asarray([x for x in range(10)]))
       self.assertEqual(y, 45)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
index cfc357ba13..557d56e8b9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
@@ -68,9 +68,9 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
-      which_bucket, bucketed_values = self.evaluate(get_next)
+      which_bucket, bucketed_values = sess.run(get_next)
 
       self.assertEqual(0, which_bucket)
 
@@ -103,11 +103,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
       # Get two minibatches (one containing even values, one containing odds)
-      which_bucket_even, bucketed_values_even = self.evaluate(get_next)
-      which_bucket_odd, bucketed_values_odd = self.evaluate(get_next)
+      which_bucket_even, bucketed_values_even = sess.run(get_next)
+      which_bucket_odd, bucketed_values_odd = sess.run(get_next)
 
       # Count number of bucket_tensors.
       self.assertEqual(3, len(bucketed_values_even))
@@ -174,11 +174,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
 
       # Get two minibatches ([0, 2, ...] and [64, 66, ...])
-      which_bucket0, bucketed_values_even0 = self.evaluate(get_next)
-      which_bucket1, bucketed_values_even1 = self.evaluate(get_next)
+      which_bucket0, bucketed_values_even0 = sess.run(get_next)
+      which_bucket1, bucketed_values_even1 = sess.run(get_next)
 
       # Ensure that bucket 1 was completely filtered out
       self.assertAllEqual(0, which_bucket0)
@@ -207,11 +207,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaises(errors.OutOfRangeError):
         batches = 0
         while True:
-          result = self.evaluate(get_next)
+          result = sess.run(get_next)
           is_even = all(x % 2 == 0 for x in result)
           is_odd = all(x % 2 == 1 for x in result)
           self.assertTrue(is_even or is_odd)
@@ -232,11 +232,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       counts = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-          result = self.evaluate(get_next)
+          result = sess.run(get_next)
           self.assertTrue(
               all(x % 2 == 0
                   for x in result) or all(x % 2 == 1)
@@ -259,16 +259,16 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       # The input is infinite, so this test demonstrates that:
       # 1. We produce output without having to consume the entire input,
       # 2. Different buckets can produce output at different rates, and
       # 3. For deterministic input, the output is deterministic.
       for _ in range(3):
-        self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
-        self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next))
-        self.assertAllEqual([2, 2, 2, 2], self.evaluate(get_next))
-        self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
+        self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
+        self.assertAllEqual([1, 1, 1, 1], sess.run(get_next))
+        self.assertAllEqual([2, 2, 2, 2], sess.run(get_next))
+        self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
 
   def testSmallGroups(self):
     components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64)
@@ -280,13 +280,13 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
-      self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
+      self.assertAllEqual([1, 1, 1, 1], sess.run(get_next))
       # The small outputs at the end are deterministically produced in key
       # order.
-      self.assertAllEqual([0, 0, 0], self.evaluate(get_next))
-      self.assertAllEqual([1], self.evaluate(get_next))
+      self.assertAllEqual([0, 0, 0], sess.run(get_next))
+      self.assertAllEqual([1], sess.run(get_next))
 
   def testEmpty(self):
     iterator = (
@@ -297,7 +297,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           "Window size must be greater than zero, but got 0."):
@@ -323,7 +323,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(get_next)
 
@@ -351,11 +351,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       counts = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-          tight_result, multiple_of_10_result = self.evaluate(get_next)
+          tight_result, multiple_of_10_result = sess.run(get_next)
           self.assertEqual(0, multiple_of_10_result.shape[1] % 10)
           self.assertAllEqual(tight_result,
                               multiple_of_10_result[:, :tight_result.shape[1]])
diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
index cb0fc13914..c0ec1486ab 100644
--- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
@@ -47,9 +47,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for x in [1., 2., 3., 5.]:
-        self.assertEqual(x, self.evaluate(get_next))
+        self.assertEqual(x, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -65,9 +65,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for x in [1., 2., 3., 5.]:
-        self.assertEqual(x, self.evaluate(get_next))
+        self.assertEqual(x, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -93,9 +93,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       # All of the files are present.
-      self.evaluate(init_op)
+      sess.run(init_op)
       for filename in filenames:
-        self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next))
+        self.assertEqual(compat.as_bytes(filename), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -104,9 +104,9 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
 
       # Attempting to read filenames[0] will fail, but ignore_errors()
       # will catch the error.
-      self.evaluate(init_op)
+      sess.run(init_op)
       for filename in filenames[1:]:
-        self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next))
+        self.assertEqual(compat.as_bytes(filename), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
index c4076daef2..c93a8353ce 100644
--- a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
@@ -53,7 +53,7 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
     ds = indexed_dataset_ops.IdentityIndexedDataset(16)
     materialized = ds.materialize()
     with self.cached_session() as sess:
-      self.evaluate(materialized.initializer)
+      sess.run(materialized.initializer)
       placeholder = array_ops.placeholder(dtypes.uint64, shape=[])
       for i in range(16):
         output = sess.run(
@@ -68,9 +68,9 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
     itr = ds.make_initializable_iterator()
     n = itr.get_next()
     with self.cached_session() as sess:
-      self.evaluate(itr.initializer)
+      sess.run(itr.initializer)
       for i in range(16):
-        output = self.evaluate(n)
+        output = sess.run(n)
         self.assertEqual(i, output)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(n)
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
index c6cefa7034..91ae8cb1bd 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
@@ -112,10 +112,10 @@ class MakeBatchedFeaturesDatasetTest(
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for file_batch, _, _, _, record_batch, _ in self._next_expected_batch(
           range(self._num_files), 2, 10):
-        actual_batch = self.evaluate(next_element)
+        actual_batch = sess.run(next_element)
         self.assertAllEqual(file_batch, actual_batch["file"])
         self.assertAllEqual(record_batch, actual_batch["record"])
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index 5486369462..e4bf089184 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -90,7 +90,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         batch_size,
         num_epochs,
     ):
-      actual_features = self.evaluate(nxt)
+      actual_features = sess.run(nxt)
 
       if label_name is not None:
         expected_labels = expected_features.pop(label_name)
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
index 404edf2fda..657cf3c00e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
@@ -105,7 +105,7 @@ class MakeTFRecordDatasetTest(
     for expected_batch in self._next_expected_batch(
         file_indices, batch_size, num_epochs, interleave_cycle_length,
         drop_final_batch, use_parser_fn):
-      actual_batch = self.evaluate(outputs)
+      actual_batch = sess.run(outputs)
       self.assertAllEqual(expected_batch, actual_batch)
 
   def _read_test(self, batch_size, num_epochs, file_index=None,
@@ -188,7 +188,7 @@ class MakeTFRecordDatasetTest(
         iterator = dataset.make_initializable_iterator()
         next_element = iterator.get_next()
 
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
         first_batches = []
         try:
           while True:
@@ -196,7 +196,7 @@ class MakeTFRecordDatasetTest(
         except errors.OutOfRangeError:
           pass
 
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
         second_batches = []
         try:
           while True:
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index b4bc4a617f..5ead6d1c75 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -89,7 +89,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 28, batch_size: 14})
       num_batches = (28 * 7) // 14
       for i in range(num_batches):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range(14):
             self.assertAllEqual(component[(i * 14 + j) % 7]**2,
@@ -104,12 +104,12 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       # We expect (num_batches - 1) full-sized batches.
       num_batches = int(math.ceil((14 * 7) / 8))
       for i in range(num_batches - 1):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range(8):
             self.assertAllEqual(component[(i * 8 + j) % 7]**2,
                                 result_component[j])
-      result = self.evaluate(get_next)
+      result = sess.run(get_next)
       for component, result_component in zip(components, result):
         for j in range((14 * 7) % 8):
           self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2,
@@ -152,10 +152,10 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
       if not drop_remainder:
-        self.assertAllEqual([[64], [81]], self.evaluate(next_element))
+        self.assertAllEqual([[64], [81]], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -177,9 +177,9 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
-      self.assertAllEqual([[64], [81]], self.evaluate(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[64], [81]], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -201,7 +201,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(5):
-        got = self.evaluate(elements)
+        got = sess.run(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -230,7 +230,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(4):
-        got = self.evaluate(elements)
+        got = sess.run(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
@@ -261,9 +261,9 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(2):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -321,7 +321,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "number of elements does not match"):
         sess.run(get_next)
@@ -393,8 +393,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(threshold // 10):
-        self.assertAllEqual([i * 10 + j for j in range(10)],
-                            self.evaluate(get_next))
+        self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next))
       if threshold % 10 != 0:
         self.assertAllEqual(
             [threshold // 10 * 10 + j for j in range(threshold % 10)],
@@ -443,8 +442,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(10):
-        self.assertAllEqual([element for _ in range(10)],
-                            self.evaluate(get_next))
+        self.assertAllEqual([element for _ in range(10)], sess.run(get_next))
 
   @parameterized.named_parameters(
       ("Identity", None, lambda x: x, None),
@@ -464,7 +462,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       else:
         expected = map_fn(
             sess.run(self.structuredElement(structure, shape=[10])))
-      self.assertAllEqual(expected, self.evaluate(get_next))
+      self.assertAllEqual(expected, sess.run(get_next))
 
   def testShortCircuitCapturedInput(self):
     captured_t = array_ops.placeholder(dtypes.int64, shape=[])
@@ -475,7 +473,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={captured_t: 42})
-      self.assertAllEqual([42] * 10, self.evaluate(get_next))
+      self.assertAllEqual([42] * 10, sess.run(get_next))
 
   @parameterized.named_parameters(
       ("Normal", False),
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
index 3cf3b89c3f..11694540fa 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
@@ -218,7 +218,7 @@ class MapDefunTest(test_base.DatasetTestBase):
 
   def _assert_op_cancelled(self, sess, map_defun_op):
     with self.assertRaisesRegexp(errors.CancelledError, "was cancelled"):
-      self.evaluate(map_defun_op)
+      sess.run(map_defun_op)
 
   def testMapDefunWithParentCancellation(self):
     # Checks that a cancellation of the parent graph is threaded through to
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index ca8bc5ff97..5e419a9b2f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -72,7 +72,7 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       thread_ids = []
       try:
         while True:
diff --git a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
index 91908f5582..90ac250df7 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
@@ -637,11 +637,11 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, self.evaluate(get_next))
+          self.assertAllEqual(expected, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -796,7 +796,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for _ in range(2):
         elements = []
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
         try:
           while True:
             elements.extend(sess.run(next_element))
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index 60c3741d32..f73725366c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -57,7 +57,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -87,7 +87,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -117,7 +117,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, self.evaluate(next_element))
+        self.assertEqual({"a": i}, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -150,7 +150,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = self.evaluate(next_element)
+        actual = sess.run(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
@@ -170,7 +170,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -199,12 +199,12 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, self.evaluate(next_element))
-      self.evaluate(iterator.initializer)
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -220,12 +220,12 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, self.evaluate(next_element))
-      self.evaluate(iterator.initializer)
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index 0e9bb462f3..0730455431 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -60,7 +60,7 @@ class ScanTest(test_base.DatasetTestBase):
                  feed_dict={start: start_val, step: step_val, take: take_val})
         for expected, _ in zip(
             itertools.count(start_val, step_val), range(take_val)):
-          self.assertEqual(expected, self.evaluate(next_element))
+          self.assertEqual(expected, sess.run(next_element))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
@@ -110,7 +110,7 @@ class ScanTest(test_base.DatasetTestBase):
                  feed_dict={start: start_val, step: step_val, take: take_val})
         for expected, _ in zip(
             itertools.count(start_val, step_val), range(take_val)):
-          self.assertEqual(expected, self.evaluate(next_element).values[0])
+          self.assertEqual(expected, sess.run(next_element).values[0])
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
@@ -136,7 +136,7 @@ class ScanTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(5):
-        (longer_vector_val, larger_rank_val), _ = self.evaluate(next_element)
+        (longer_vector_val, larger_rank_val), _ = sess.run(next_element)
         self.assertAllEqual([0] * (2**i), longer_vector_val)
         self.assertAllEqual(np.array(1, ndmin=i), larger_rank_val)
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
index 704a40721f..ef99d01c73 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
@@ -71,19 +71,19 @@ class RangeDatasetSerializationTest(
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
-        self.evaluate(restore_op)
+        sess.run(init_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -91,14 +91,14 @@ class RangeDatasetSerializationTest(
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
-        self.evaluate(restore_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
index 496fd45947..88d5c896c9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
@@ -62,7 +62,7 @@ class SerializationIntegrationTest(test.TestCase):
       with self.session(graph=g) as sess:
         sess.run(init_ops)
         for _ in range(break_point):
-          output = self.evaluate(get_next_ops)
+          output = sess.run(get_next_ops)
           for i in range(num_pipelines):
             all_outputs[i].append(output[i])
         saver.save(sess, self._ckpt_path())
@@ -73,7 +73,7 @@ class SerializationIntegrationTest(test.TestCase):
       with self.session(graph=g) as sess:
         saver.restore(sess, self._ckpt_path())
         for _ in range(num_outputs - break_point):
-          output = self.evaluate(get_next_ops)
+          output = sess.run(get_next_ops)
           for i in range(num_pipelines):
             all_outputs[i].append(output[i])
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
index 5f7d9051ec..c208963a86 100644
--- a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
@@ -108,7 +108,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
           shuffle_ops.shuffle_and_repeat(buffer_size=21))
       get_next_op = ds.make_one_shot_iterator().get_next()
       with self.session(graph=g) as sess:
-        self.evaluate(get_next_op)
+        sess.run(get_next_op)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
index f7d42bc5b3..bf53acc82a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
@@ -38,10 +38,10 @@ class SleepTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       start_time = time.time()
       for i in range(10):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       end_time = time.time()
       self.assertGreater(end_time - start_time, (10 * sleep_microseconds) / 1e6)
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
index e11bad7969..a2c1169638 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
@@ -39,9 +39,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                             "ORDER BY first_name DESC"
             })
         for _ in range(2):  # Dataset is repeated. See setUp.
-          self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next))
-          self.assertEqual((b"Jane", b"Moe", b"Hi again!"),
-                           self.evaluate(get_next))
+          self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next))
+          self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -59,8 +58,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ON students.first_name = people.first_name "
                   "AND students.last_name = people.last_name"
           })
-      self.assertEqual((b"John", b"California", b"Hi!"),
-                       self.evaluate(get_next))
+      self.assertEqual((b"John", b"California", b"Hi!"), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -77,9 +75,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, last_name, favorite_nonsense_word "
                   "FROM students ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"n\0nsense"), self.evaluate(get_next))
-      self.assertEqual((b"Jane", b"Moe", b"nonsense\0"),
-                       self.evaluate(get_next))
+      self.assertEqual((b"John", b"Doe", b"n\0nsense"), sess.run(get_next))
+      self.assertEqual((b"Jane", b"Moe", b"nonsense\0"), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -96,8 +93,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, last_name, motto FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next))
-      self.assertEqual((b"Jane", b"Moe", b"Hi again!"), self.evaluate(get_next))
+      self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next))
+      self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
       sess.run(
@@ -106,8 +103,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, last_name, state FROM people "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"California"),
-                       self.evaluate(get_next))
+      self.assertEqual((b"John", b"Doe", b"California"), sess.run(get_next))
       self.assertEqual((b"Benjamin", b"Franklin", b"Pennsylvania"),
                        sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
@@ -216,8 +212,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
+      self.assertEqual((b"John", 9), sess.run(get_next))
+      self.assertEqual((b"Jane", 127), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -234,7 +230,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students "
                           "WHERE first_name = 'John' ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0, -2), self.evaluate(get_next))
+      self.assertEqual((b"John", 0, -2), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -250,9 +246,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT desk_number, favorite_negative_number FROM students "
                   "ORDER BY first_name DESC"
           })
-      self.assertEqual((9, -2), self.evaluate(get_next))
+      self.assertEqual((9, -2), sess.run(get_next))
       # Max and min values of int8
-      self.assertEqual((127, -128), self.evaluate(get_next))
+      self.assertEqual((127, -128), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -267,8 +263,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
+      self.assertEqual((b"John", 9), sess.run(get_next))
+      self.assertEqual((b"Jane", 127), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -285,7 +281,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students "
                           "WHERE first_name = 'John' ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0, -2), self.evaluate(get_next))
+      self.assertEqual((b"John", 0, -2), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -301,9 +297,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students ORDER BY first_name DESC"
           })
       # Max value of int16
-      self.assertEqual((b"John", 32767), self.evaluate(get_next))
+      self.assertEqual((b"John", 32767), sess.run(get_next))
       # Min value of int16
-      self.assertEqual((b"Jane", -32768), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -32768), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -318,8 +314,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
+      self.assertEqual((b"John", 9), sess.run(get_next))
+      self.assertEqual((b"Jane", 127), sess.run(get_next))
 
   # Test that `SqlDataset` can read a negative or 0-valued integer from a
   # SQLite database table and place it in an `int32` tensor.
@@ -332,8 +328,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, income FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0), self.evaluate(get_next))
-      self.assertEqual((b"Jane", -20000), self.evaluate(get_next))
+      self.assertEqual((b"John", 0), sess.run(get_next))
+      self.assertEqual((b"Jane", -20000), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -349,9 +345,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Max value of int32
-      self.assertEqual((b"John", 2147483647), self.evaluate(get_next))
+      self.assertEqual((b"John", 2147483647), sess.run(get_next))
       # Min value of int32
-      self.assertEqual((b"Jane", -2147483648), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -2147483648), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -366,8 +362,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, school_id FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 123), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 1000), self.evaluate(get_next))
+      self.assertEqual((b"John", 123), sess.run(get_next))
+      self.assertEqual((b"Jane", 1000), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -382,8 +378,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
+      self.assertEqual((b"John", 9), sess.run(get_next))
+      self.assertEqual((b"Jane", 127), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -398,8 +394,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, income FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0), self.evaluate(get_next))
-      self.assertEqual((b"Jane", -20000), self.evaluate(get_next))
+      self.assertEqual((b"John", 0), sess.run(get_next))
+      self.assertEqual((b"Jane", -20000), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -416,9 +412,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ORDER BY first_name DESC"
           })
       # Max value of int64
-      self.assertEqual((b"John", 9223372036854775807), self.evaluate(get_next))
+      self.assertEqual((b"John", 9223372036854775807), sess.run(get_next))
       # Min value of int64
-      self.assertEqual((b"Jane", -9223372036854775808), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -9223372036854775808), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -433,8 +429,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
+      self.assertEqual((b"John", 9), sess.run(get_next))
+      self.assertEqual((b"Jane", 127), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -450,9 +446,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Min value of uint8
-      self.assertEqual((b"John", 0), self.evaluate(get_next))
+      self.assertEqual((b"John", 0), sess.run(get_next))
       # Max value of uint8
-      self.assertEqual((b"Jane", 255), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 255), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -467,8 +463,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), self.evaluate(get_next))
-      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
+      self.assertEqual((b"John", 9), sess.run(get_next))
+      self.assertEqual((b"Jane", 127), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -484,9 +480,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Min value of uint16
-      self.assertEqual((b"John", 0), self.evaluate(get_next))
+      self.assertEqual((b"John", 0), sess.run(get_next))
       # Max value of uint16
-      self.assertEqual((b"Jane", 65535), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 65535), sess.run(get_next))
     with self.assertRaises(errors.OutOfRangeError):
       sess.run(get_next)
 
@@ -503,8 +499,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, registration_complete FROM students "
                   "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", True), self.evaluate(get_next))
-      self.assertEqual((b"Jane", False), self.evaluate(get_next))
+      self.assertEqual((b"John", True), sess.run(get_next))
+      self.assertEqual((b"Jane", False), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -519,8 +515,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, favorite_medium_sized_number "
                           "FROM students ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", True), self.evaluate(get_next))
-      self.assertEqual((b"Jane", True), self.evaluate(get_next))
+      self.assertEqual((b"John", True), sess.run(get_next))
+      self.assertEqual((b"Jane", True), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -537,9 +533,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, last_name, victories FROM townspeople "
                   "ORDER BY first_name"
           })
-      self.assertEqual((b"George", b"Washington", 20.0),
-                       self.evaluate(get_next))
-      self.assertEqual((b"John", b"Adams", -19.95), self.evaluate(get_next))
+      self.assertEqual((b"George", b"Washington", 20.0), sess.run(get_next))
+      self.assertEqual((b"John", b"Adams", -19.95), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index 958c3f0038..83028937d3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -74,18 +74,18 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       expected_sum = 0.0
       for i in range(100):
         self.assertAllEqual(
             np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = self.evaluate(summary_t)
+        summary_str = sess.run(summary_t)
         self._assertSummaryHasCount(summary_str, "bytes_produced", float(i + 1))
         expected_sum += i * 8.0
         self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      summary_str = self.evaluate(summary_t)
+      summary_str = sess.run(summary_t)
       self._assertSummaryHasCount(summary_str, "bytes_produced", 100.0)
       self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
 
@@ -99,15 +99,14 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(
-          self.evaluate(summary_t), "record_latency", 100.0)
+      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0)
 
   def testPrefetchBufferUtilization(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -119,11 +118,11 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(100):
         self.assertAllEqual(
             np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = self.evaluate(summary_t)
+        summary_str = sess.run(summary_t)
         self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                     float(i + 1))
         self._assertSummaryContains(summary_str, "Prefetch::buffer_capacity")
@@ -132,7 +131,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
                                     0, 1)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      summary_str = self.evaluate(summary_t)
+      summary_str = sess.run(summary_t)
       self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                   100)
 
@@ -146,11 +145,11 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(10):
         self.assertAllEqual(
             np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = self.evaluate(summary_t)
+        summary_str = sess.run(summary_t)
         self._assertSummaryHasScalarValue(summary_str,
                                           "Prefetch::buffer_capacity", 0)
         self._assertSummaryHasScalarValue(summary_str, "Prefetch::buffer_size",
@@ -168,9 +167,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(34):
-        self.assertEqual(i * 3, self.evaluate(next_element))
+        self.assertEqual(i * 3, sess.run(next_element))
         if i is not 0:
           self._assertSummaryHasScalarValue(
               sess.run(summary_t), "Filter::dropped_elements", float(i * 2))
@@ -262,9 +261,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
     with self.cached_session() as sess:
       for j in range(5):
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
         for i in range(100):
-          self.assertEqual(i, self.evaluate(next_element))
+          self.assertEqual(i, sess.run(next_element))
           self._assertSummaryHasCount(
               sess.run(summary_t), "record_latency", float((j * 100) + i + 1))
         with self.assertRaises(errors.OutOfRangeError):
@@ -279,9 +278,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -296,17 +295,16 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(i + 1))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency_2", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(
-          self.evaluate(summary_t), "record_latency", 100.0)
+      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0)
       self._assertSummaryHasCount(
           sess.run(summary_t), "record_latency_2", 100.0)
 
@@ -321,15 +319,14 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, self.evaluate(next_element))
+        self.assertEqual(i, sess.run(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(2 * (i + 1)))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(
-          self.evaluate(summary_t), "record_latency", 200.0)
+      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0)
 
   def testMultipleIteratorsSameAggregator(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -344,13 +341,12 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     with self.cached_session() as sess:
       sess.run([iterator_0.initializer, iterator_1.initializer])
       for i in range(100):
-        self.assertEqual(i * 2, self.evaluate(next_element))
+        self.assertEqual(i * 2, sess.run(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "record_latency", float(2 * (i + 1)))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
-      self._assertSummaryHasCount(
-          self.evaluate(summary_t), "record_latency", 200.0)
+      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0)
 
   def testMultipleDatasetWithPrefixes(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -368,7 +364,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     with self.test_session() as sess:
       sess.run([iterator_0.initializer, iterator_1.initializer])
       for i in range(100):
-        self.assertEqual(i * 2, self.evaluate(next_element))
+        self.assertEqual(i * 2, sess.run(next_element))
         self._assertSummaryHasCount(
             sess.run(summary_t), "dataset1_record_latency", float(i + 1))
         self._assertSummaryHasCount(
@@ -425,7 +421,7 @@ class FeatureStatsDatasetTest(
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for _ in range(num_output):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index 755294ac45..0278a208cb 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -50,7 +50,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]})
       for i in range(4):
-        self.assertEqual(i, self.evaluate(next_elem))
+        self.assertEqual(i, sess.run(next_elem))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_elem)
 
@@ -68,7 +68,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i,) * 3, self.evaluate(op))
+        self.assertEqual((i,) * 3, sess.run(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -88,7 +88,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i, compat.as_bytes(str(i)), i), self.evaluate(op))
+        self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
@@ -107,7 +107,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        st_row = self.evaluate(next_element)
+        st_row = sess.run(next_element)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
         self.assertEqual([10], st_row.dense_shape)
@@ -128,7 +128,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        dense_elem, st_row = self.evaluate(next_element)
+        dense_elem, st_row = sess.run(next_element)
         self.assertEqual(i, dense_elem)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
@@ -150,7 +150,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(((i,),) * 3, self.evaluate(op))
+        self.assertEqual(((i,),) * 3, sess.run(op))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(op)
diff --git a/tensorflow/python/data/experimental/kernel_tests/unique_test.py b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
index 4b14a7e963..847cff26b0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unique_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
@@ -49,11 +49,11 @@ class UniqueTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for test_case, expected in test_cases:
         current_test_case = test_case
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
         for element in expected:
           if dtype == dtypes.string:
             element = compat.as_bytes(element)
-          self.assertAllEqual(element, self.evaluate(next_element))
+          self.assertAllEqual(element, sess.run(next_element))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(next_element)
 
diff --git a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
index 10a0427c7f..e8decb9ad0 100644
--- a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
@@ -93,13 +93,13 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           })
       num_full_batches = (count * 7) // batch_size
       for i in range(num_full_batches):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range(batch_size):
             self.assertAllEqual(component[(i * batch_size + j) % 7]**2,
                                 result_component[j])
       if not drop_remainder and (count * 7) % batch_size > 0:
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range((count * 7) % batch_size):
             self.assertAllEqual(
@@ -128,9 +128,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(2):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -155,9 +155,9 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(2):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         expected_indices = []
         expected_values = []
         for j in range(5):
@@ -185,8 +185,8 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      actual = self.evaluate(get_next)
+      sess.run(init_op)
+      actual = sess.run(get_next)
       expected = sparse_tensor.SparseTensorValue(
           indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0],
                    [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]],
@@ -211,7 +211,7 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           r'Cannot batch tensors with different shapes in component 0. '
@@ -271,7 +271,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       num_full_batches = len(seq_lens) // batch_size
 
       for i in range(num_full_batches):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         padded_len = padded_shapes[0]
         if padded_len is None or padded_len == -1:
           padded_len = np.max(result) if result.size > 0 else 0
@@ -283,7 +283,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
                               [0] * (padded_len - seq_len))
 
       if not drop_remainder and len(seq_lens) % batch_size > 0:
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         padded_len = np.max(result) if result.size > 0 else 0
         self.assertEqual((len(seq_lens) % batch_size, padded_len),
                          result.shape)
@@ -315,7 +315,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      result = self.evaluate(get_next)
+      result = sess.run(get_next)
       self.assertAllEqual([[], [], [], []], result)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -347,7 +347,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
               seq_lens: random_seq_lens
           })
       for i in range(8):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         padded_len = np.max(result[0])
         self.assertEqual((4, padded_len), result[0].shape)
         self.assertEqual((4, padded_len), result[1].shape)
diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
index 1f351279c6..63625fac03 100644
--- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
@@ -71,7 +71,7 @@ class FileCacheDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       # First run without caching to collect the "ground truth".
-      self.evaluate(init_fifo_op)
+      sess.run(init_fifo_op)
       elements = []
       for _ in range(20):
         elements.append(sess.run(get_next))
@@ -220,14 +220,14 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
 
       with self.cached_session() as sess:
 
-        self.evaluate(repeat_count.initializer)
-        self.evaluate(cached_iterator.initializer)
-        self.evaluate(uncached_iterator.initializer)
+        sess.run(repeat_count.initializer)
+        sess.run(cached_iterator.initializer)
+        sess.run(uncached_iterator.initializer)
 
         for i in range(3):
           for _ in range(10):
-            self.assertEqual(self.evaluate(cached_next), i)
-            self.assertEqual(self.evaluate(uncached_next), i)
+            self.assertEqual(sess.run(cached_next), i)
+            self.assertEqual(sess.run(uncached_next), i)
 
         sess.run(repeat_count.assign(0))
 
@@ -238,7 +238,7 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
         # The cached iterator replays from cache.
         for i in range(3):
           for _ in range(10):
-            self.assertEqual(self.evaluate(cached_next), i)
+            self.assertEqual(sess.run(cached_next), i)
 
         # The cached iterator should now be empty.
         with self.assertRaises(errors.OutOfRangeError):
@@ -280,7 +280,7 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
     i2 = d2.make_initializable_iterator()
 
     with self.cached_session() as sess:
-      self.evaluate(i1.initializer)
+      sess.run(i1.initializer)
 
       self.assertEqual(1, sess.run(i1.get_next()))
       self.assertEqual(2, sess.run(i1.get_next()))
@@ -307,7 +307,7 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i, expected in enumerate(expected_values):
-        self.assertEqual(expected, self.evaluate(n),
+        self.assertEqual(expected, sess.run(n),
                          "Unexpected value at index %s" % i)
 
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
index a0ef69f082..83af31f380 100644
--- a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
@@ -51,9 +51,9 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(9):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         if i < 4:
           for component, result_component in zip(input_components, result):
             self.assertAllEqual(component[i], result_component)
@@ -85,9 +85,9 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(9):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         if i < 4:
           for component, result_component in zip(input_components, result):
             self.assertAllEqual(component[i], result_component)
diff --git a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
index f7b500881c..bc6b36285a 100644
--- a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
@@ -52,8 +52,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      results = self.evaluate(get_next)
+      sess.run(init_op)
+      results = sess.run(get_next)
       for component, result_component in zip(components, results):
         self.assertAllEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -81,8 +81,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
         [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      results = self.evaluate(get_next)
+      sess.run(init_op)
+      results = sess.run(get_next)
       for component, result_component in zip(components, results):
         self.assertSparseValuesEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -112,8 +112,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     ], [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      results = self.evaluate(get_next)
+      sess.run(init_op)
+      results = sess.run(get_next)
       for component, result_component in zip(components, results):
         if sparse_tensor.is_sparse(component):
           self.assertSparseValuesEqual(component, result_component)
@@ -139,9 +139,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component[i], result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -169,7 +169,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
         [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       expected = [
           (sparse_tensor.SparseTensorValue(
               indices=np.array([[0]]),
@@ -197,7 +197,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                dense_shape=np.array([3]))),
       ]
       for i in range(3):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(expected[i], results):
           self.assertSparseValuesEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -229,7 +229,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     ], [shape for shape in iterator.output_shapes])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       expected = [
           (sparse_tensor.SparseTensorValue(
               indices=np.array([[0]]),
@@ -257,7 +257,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                dense_shape=np.array([3]))),
       ]
       for i in range(3):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(
             (list(zip(*components[:3]))[i] + expected[i]), results):
           if sparse_tensor.is_sparse(component):
@@ -280,9 +280,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     self.assertEqual((1,), iterator.output_shapes["bar"])
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(3):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertEqual(components["foo"][i], results["foo"])
         self.assertEqual(components["bar"][i], results["bar"])
       with self.assertRaises(errors.OutOfRangeError):
@@ -308,7 +308,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                                                     dense_shape)
       sess.run(init_op, feed_dict={st: sparse_feed})
       for i, s in enumerate(slices):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual(s, results.values)
         expected_indices = np.array(
             [[j] for j in range(len(slices[i]))]).reshape([-1, 1])
@@ -474,15 +474,15 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
       with ops.device("/cpu:0"):
         var_0 = resource_variable_ops.ResourceVariable(initial_value=0)
         dataset = dataset.map(lambda x: x + var_0.read_value())
-      self.evaluate(var_0.initializer)
+      sess.run(var_0.initializer)
 
       with ops.device("/cpu:1"):
         var_1 = resource_variable_ops.ResourceVariable(initial_value=0)
         dataset = dataset.map(lambda x: x + var_1.read_value())
-      self.evaluate(var_1.initializer)
+      sess.run(var_1.initializer)
 
       iterator = dataset.make_initializable_iterator()
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
 
       with self.assertRaisesRegexp(
           errors.FailedPreconditionError,
@@ -506,7 +506,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
         sess.run(next_element)
@@ -543,7 +543,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       get_next_element = sess.make_callable(next_element)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
@@ -582,7 +582,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       get_next_element = sess.make_callable(next_element)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
@@ -620,7 +620,7 @@ class DatasetConstructorBenchmark(test.Benchmark):
     next_element = iterator.get_next()
 
     with session.Session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       get_next_element = sess.make_callable(next_element)
       # Run one whole epoch to burn in the computation.
       for _ in range(input_size // batch_size):
diff --git a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
index 7087b4dd57..cb8cb9a77d 100644
--- a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
@@ -47,10 +47,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for _ in range(2):  # Run twice to test reinitialization.
-        self.evaluate(init_op)
+        sess.run(init_op)
         for _ in range(num_repeats):
           for elem in elem_sequence:
-            self.assertAllEqual(elem, self.evaluate(get_next))
+            self.assertAllEqual(elem, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -65,7 +65,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for _ in range(num_repeats):
         for elem in elem_sequence:
-          self.assertAllEqual(elem, self.evaluate(get_next))
+          self.assertAllEqual(elem, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -133,10 +133,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for _ in range(num_inner_repeats * num_outer_repeats):
         for elem in input_list:
-          val0, val1 = self.evaluate(get_next)
+          val0, val1 = sess.run(get_next)
           self.assertAllEqual(elem[0], val0)
           self.assertAllEqual(elem[1], val1)
       with self.assertRaises(errors.OutOfRangeError):
@@ -192,10 +192,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for elem in [0, 1]:
         for _ in range(num_parallel_iterators):
-          self.assertAllEqual(elem, self.evaluate(get_next))
+          self.assertAllEqual(elem, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -215,9 +215,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
       self.assertEqual(dtype, get_next.dtype)
 
       with self.cached_session() as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         for expected in [[1], [2], [3]]:
-          next_val = self.evaluate(get_next)
+          next_val = sess.run(get_next)
           self.assertEqual(dtype.as_numpy_dtype, next_val.dtype)
           self.assertAllEqual(expected, next_val)
         with self.assertRaises(errors.OutOfRangeError):
@@ -236,9 +236,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for expected in [b"foo", b"bar", b"baz"]:
-        next_val = self.evaluate(get_next)
+        next_val = sess.run(get_next)
         self.assertAllEqual(expected, next_val)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -257,12 +257,12 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
-      self.assertAllEqual([4, 5, 6], self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual([1, 2, 3], sess.run(get_next))
+      self.assertAllEqual([4, 5, 6], sess.run(get_next))
       with self.assertRaisesOpError("The expected type was int64"):
         sess.run(get_next)
-      self.assertAllEqual([7, 8, 9], self.evaluate(get_next))
+      self.assertAllEqual([7, 8, 9], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -280,12 +280,12 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
-      self.assertAllEqual([4, 5, 6], self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual([1, 2, 3], sess.run(get_next))
+      self.assertAllEqual([4, 5, 6], sess.run(get_next))
       with self.assertRaisesOpError(r"element of shape \(3,\) was expected"):
         sess.run(get_next)
-      self.assertAllEqual([11, 12, 13], self.evaluate(get_next))
+      self.assertAllEqual([11, 12, 13], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -304,16 +304,16 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertEqual((1, 2), self.evaluate(get_next))
-      self.assertEqual((3, 4), self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertEqual((1, 2), sess.run(get_next))
+      self.assertEqual((3, 4), sess.run(get_next))
       with self.assertRaisesOpError(
           r"The expected structure was \(tf\.int64, tf\.int64\)"):
         sess.run(get_next)
       with self.assertRaisesOpError(
           r"The expected structure was \(tf\.int64, tf\.int64\)"):
         sess.run(get_next)
-      self.assertEqual((9, 10), self.evaluate(get_next))
+      self.assertEqual((9, 10), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -329,9 +329,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual(1, self.evaluate(get_next))
-      self.assertAllEqual([2, 3], self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual(1, sess.run(get_next))
+      self.assertAllEqual([2, 3], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -349,9 +349,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual(0, self.evaluate(get_next))
-      self.assertAllEqual(1, self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual(0, sess.run(get_next))
+      self.assertAllEqual(1, sess.run(get_next))
 
   def testFromGeneratorDestructorCalled(self):
     # Use an `Event` to signal that the generator has been deleted.
@@ -378,9 +378,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with session.Session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual(42, self.evaluate(get_next))
-      self.assertAllEqual(42, self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual(42, sess.run(get_next))
+      self.assertAllEqual(42, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
       # Test that `GeneratorWrapper` object is destroyed when the
@@ -407,10 +407,10 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       expected = [1, 2, 2, 3, 3, 3, 4, 4, 4, 4]
       for x in expected:
-        self.assertEqual(x, self.evaluate(get_next))
+        self.assertEqual(x, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -436,13 +436,13 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       expected = [(0, b"Hi!"),
                   (0, b"Hi!"), (1, b"Hi!"),
                   (0, b"Hi!"), (1, b"Hi!"), (2, b"Hi!"),
                   (0, b"Hi!"), (1, b"Hi!"), (2, b"Hi!"), (3, b"Hi!")]
       for x in expected:
-        self.assertEqual(x, self.evaluate(get_next))
+        self.assertEqual(x, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -470,9 +470,9 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual(37, self.evaluate(get_next))
-      self.assertAllEqual(37, self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual(37, sess.run(get_next))
+      self.assertAllEqual(37, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
         self.assertTrue(event.is_set())
diff --git a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
index 5ddb22285f..a0c6b37a6d 100644
--- a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
@@ -67,7 +67,7 @@ class FilterDatasetTest(test_base.DatasetTestBase):
         sess.run(init_op, feed_dict={count: count_val, modulus: modulus_val})
         for _ in range(count_val):
           for i in [x for x in range(7) if x**2 % modulus_val == 0]:
-            result = self.evaluate(get_next)
+            result = sess.run(get_next)
             for component, result_component in zip(components, result):
               self.assertAllEqual(component[i]**2, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -86,9 +86,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.assertEqual(0, self.evaluate(get_next))
-      self.assertEqual(1, self.evaluate(get_next))
-      self.assertEqual(3, self.evaluate(get_next))
+      self.assertEqual(0, sess.run(get_next))
+      self.assertEqual(1, sess.run(get_next))
+      self.assertEqual(3, sess.run(get_next))
 
   def testFilterDict(self):
     iterator = (dataset_ops.Dataset.range(10)
@@ -100,10 +100,10 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
         if (i ** 2) % 2 == 0:
-          self.assertEqual(i * 2 + i**2, self.evaluate(get_next))
+          self.assertEqual(i * 2 + i ** 2, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -125,8 +125,8 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual(input_data[0], self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual(input_data[0], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -148,9 +148,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(5):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         self.assertTrue(isinstance(actual, sparse_tensor.SparseTensorValue))
         self.assertSparseValuesEqual(actual, _map_fn(i * 2)[0])
       with self.assertRaises(errors.OutOfRangeError):
@@ -166,9 +166,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual((i, True), self.evaluate(get_next))
+        self.assertEqual((i, True), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -178,7 +178,7 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     iterators = [dataset.make_one_shot_iterator() for _ in range(10)]
     next_elements = [iterator.get_next() for iterator in iterators]
     with self.cached_session() as sess:
-      self.assertEqual([0 for _ in range(10)], self.evaluate(next_elements))
+      self.assertEqual([0 for _ in range(10)], sess.run(next_elements))
 
 
 class FilterDatasetBenchmark(test.Benchmark):
diff --git a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
index 02979fc2c4..68038f9cfc 100644
--- a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
@@ -45,10 +45,10 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in repeats:
         for _ in range(i):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -64,11 +64,11 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for row in repeats:
         for i in row:
           for _ in range(i):
-            self.assertEqual(i, self.evaluate(get_next))
+            self.assertEqual(i, sess.run(get_next))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -94,12 +94,12 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
       with session.Session(server.target) as sess2:
         for _ in range(3):
           sess = random.choice([sess1, sess2])
-          self.evaluate(init_op)
+          sess.run(init_op)
           for row in repeats:
             for i in row:
               for _ in range(i):
                 sess = random.choice([sess1, sess2])
-                self.assertEqual(i, self.evaluate(get_next))
+                self.assertEqual(i, sess.run(get_next))
 
         with self.assertRaises(errors.OutOfRangeError):
           sess = random.choice([sess1, sess2])
@@ -115,10 +115,10 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
         for _ in range(i ** 2):
-          self.assertEqual(i * 2, self.evaluate(get_next))
+          self.assertEqual(i * 2, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
   # pylint: enable=g-long-lambda
@@ -139,11 +139,11 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, self.evaluate(get_next))
+          self.assertAllEqual(expected, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py b/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
index 56434d6e4c..b911c249ce 100644
--- a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
@@ -196,7 +196,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       for expected_element in _interleave(
           _repeat(input_values, count), cycle_length, block_length):
-        self.assertEqual(expected_element, self.evaluate(get_next))
+        self.assertEqual(expected_element, sess.run(get_next))
 
       for _ in range(2):
         with self.assertRaises(errors.OutOfRangeError):
@@ -231,7 +231,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           with self.assertRaises(errors.InvalidArgumentError):
             sess.run(get_next)
         else:
-          self.assertEqual(value, self.evaluate(get_next))
+          self.assertEqual(value, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -254,7 +254,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, self.evaluate(get_next))
+          self.assertAllEqual(expected, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -308,7 +308,7 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
       for element in elements:
         coordination_events[element].set()
-        self.assertEqual(element * element, self.evaluate(get_next))
+        self.assertEqual(element * element, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
index cb38728f23..bf5fd781d6 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
@@ -57,7 +57,7 @@ class IteratorClusterTest(test.TestCase):
 
     with session.Session(worker[0].target) as sess:
       with self.assertRaises(errors.InvalidArgumentError):
-        self.evaluate(get_next_op)
+        sess.run(get_next_op)
 
   def _testRemoteIteratorHelper(self, device0, device1, target):
     with ops.device(device1):
@@ -134,12 +134,12 @@ class IteratorClusterTest(test.TestCase):
     get_next = iterator.get_next()
 
     with session.Session(worker[0].target) as sess:
-      self.evaluate(table.initializer)
-      self.evaluate(init_op)
-      self.assertAllEqual([0, 0, -1, 1, 2], self.evaluate(get_next))
+      sess.run(table.initializer)
+      sess.run(init_op)
+      self.assertAllEqual([0, 0, -1, 1, 2], sess.run(get_next))
 
     with session.Session(worker[0].target) as sess:
-      self.assertAllEqual([2, 0], self.evaluate(get_next))
+      self.assertAllEqual([2, 0], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -166,7 +166,7 @@ class IteratorClusterTest(test.TestCase):
     get_next = iterator.get_next()
 
     with session.Session(worker[0].target) as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for _ in range(3):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
index 405d94d956..490ca813dc 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
@@ -97,7 +97,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       for _ in range(14):
         for i in range(7):
-          result = self.evaluate(get_next)
+          result = sess.run(get_next)
           for component, result_component in zip(components, result):
             self.assertAllEqual(component[i]**2, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -123,7 +123,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       for _ in range(14):
         for i in range(7):
-          result = self.evaluate(get_next)
+          result = sess.run(get_next)
           for component, result_component in zip(components, result):
             self.assertAllEqual(component[i]**2, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -159,7 +159,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
         for _ in range(14):
           for i in range(7):
-            result = self.evaluate(get_next)
+            result = sess.run(get_next)
             for component, result_component in zip(components, result):
               self.assertAllEqual(component[i]**2, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -175,7 +175,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     config = config_pb2.ConfigProto(
         inter_op_parallelism_threads=1, use_per_session_threads=True)
     with session.Session(config=config) as sess:
-      self.assertAllEqual([1, 4, 9], self.evaluate(next_element))
+      self.assertAllEqual([1, 4, 9], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -254,15 +254,15 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       get_next = iterator.get_next()
 
       with session.Session(server.target) as sess:
-        self.evaluate(init_op)
-        results = self.evaluate(get_next)
+        sess.run(init_op)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
         # Re-initialize the iterator in the first session.
-        self.evaluate(init_op)
+        sess.run(init_op)
 
     with ops.Graph().as_default():
       # Re-define the iterator manually, without defining any of the
@@ -277,7 +277,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
       with session.Session(server.target) as sess:
         # Use the iterator without re-initializing in the second session.
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -317,20 +317,20 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         sess.run(get_next)
 
       # Initialize with one dataset.
-      self.evaluate(dataset_3_init_op)
-      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
+      sess.run(dataset_3_init_op)
+      self.assertAllEqual([1, 2, 3], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
       # Initialize with a different dataset.
-      self.evaluate(dataset_4_init_op)
-      self.assertAllEqual([4, 5, 6, 7], self.evaluate(get_next))
+      sess.run(dataset_4_init_op)
+      self.assertAllEqual([4, 5, 6, 7], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
       # Reinitialize with the first dataset.
-      self.evaluate(dataset_3_init_op)
-      self.assertAllEqual([1, 2, 3], self.evaluate(get_next))
+      sess.run(dataset_3_init_op)
+      self.assertAllEqual([1, 2, 3], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -348,7 +348,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
           g, output_types=dtypes.int64)
       sess.run(iterator.make_initializer(dataset_1))
       for expected in range(10):
-        self.assertEqual(expected, self.evaluate(next_element))
+        self.assertEqual(expected, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -356,7 +356,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
           g, output_types=dtypes.int64)
       sess.run(iterator.make_initializer(dataset_2))
       for expected in range(10):
-        self.assertEqual(expected, self.evaluate(next_element))
+        self.assertEqual(expected, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -679,10 +679,10 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       n = itr.get_next()
 
     with session.Session(s3.target, config=config) as sess:
-      self.evaluate(itr.initializer)
+      sess.run(itr.initializer)
       expected_values = worker_devices
       for expected in expected_values:
-        self.assertEqual((compat.as_bytes(expected),), self.evaluate(n))
+        self.assertEqual((compat.as_bytes(expected),), sess.run(n))
 
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(n)
@@ -786,8 +786,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with ops.Graph().as_default() as g:
       init_op, _, save_op, _ = _build_range_dataset_graph()
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
-        self.evaluate(save_op)
+        sess.run(init_op)
+        sess.run(save_op)
 
     # Attempt to restore the saved iterator into an IteratorResource of
     # incompatible type. An iterator of RangeDataset has output type int64,
@@ -798,7 +798,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       _, _, _, restore_op = _build_reader_dataset_graph()
       with self.session(graph=g) as sess:
         with self.assertRaises(errors.InvalidArgumentError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
 
   def testRepeatedGetNextWarning(self):
     iterator = dataset_ops.Dataset.range(10).make_one_shot_iterator()
@@ -949,7 +949,7 @@ class IteratorCheckpointingTest(test.TestCase):
         checkpoint.restore(checkpoint_management.latest_checkpoint(
             checkpoint_directory)).initialize_or_restore(sess)
         for j in range(2):
-          self.assertEqual(i * 2 + j, self.evaluate(get_next))
+          self.assertEqual(i * 2 + j, sess.run(get_next))
         checkpoint.save(file_prefix=checkpoint_prefix)
 
 
diff --git a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
index ac6fbabcd5..b58c1444da 100644
--- a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
@@ -102,7 +102,7 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
       all_produced_filenames = []
       for _ in range(3):
         produced_filenames = []
-        self.evaluate(itr.initializer)
+        sess.run(itr.initializer)
         try:
           while True:
             produced_filenames.append(sess.run(next_element))
diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
index 8f7a19d7e1..187b9da14c 100644
--- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
@@ -114,7 +114,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 14})
       for _ in range(14):
         for i in range(7):
-          result = self.evaluate(get_next)
+          result = sess.run(get_next)
           for component, result_component in zip(components, result):
             self.assertAllEqual(component[i]**2, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -185,7 +185,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
             output_buffer_size: output_buffer_size_val})
         for _ in range(14):
           for i in range(7):
-            result = self.evaluate(get_next)
+            result = sess.run(get_next)
             for component, result_component in zip(components, result):
               self.assertAllEqual(component[i]**2, result_component)
         with self.assertRaises(errors.OutOfRangeError):
@@ -242,7 +242,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for _ in range(3):
         sess.run(get_next)
 
@@ -257,7 +257,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for _ in range(3):
         sess.run(get_next)
 
@@ -272,7 +272,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for _ in range(3):
         sess.run(get_next)
       # The 4th element is NaN, so `array_ops.check_numerics()` should fail.
@@ -293,7 +293,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for _ in range(3):
         sess.run(get_next)
       # The 4th element is NaN, so `array_ops.check_numerics()` should fail.
@@ -325,10 +325,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with ops.Graph().as_default() as g:
       captured_init_op, init_op, get_next = _build_graph()
       with self.session(graph=g) as sess:
-        self.evaluate(captured_init_op)
-        self.evaluate(init_op)
+        sess.run(captured_init_op)
+        sess.run(init_op)
         for i in range(10):
-          self.assertEqual(i * i, self.evaluate(get_next))
+          self.assertEqual(i * i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -353,8 +353,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(table.initializer)
-      self.evaluate(init_op)
+      sess.run(table.initializer)
+      sess.run(init_op)
       sess.run(get_next)
       sess.run(get_next)
       with self.assertRaises(errors.OutOfRangeError):
@@ -371,11 +371,11 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(enqueue_op)
-      self.evaluate(close_op)
-      self.evaluate(init_op)
+      sess.run(enqueue_op)
+      sess.run(close_op)
+      sess.run(init_op)
       for element in elements:
-        self.assertEqual(element, self.evaluate(get_next))
+        self.assertEqual(element, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -396,9 +396,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(enqueue_op)
-      self.evaluate(close_op)
-      self.evaluate(init_op)
+      sess.run(enqueue_op)
+      sess.run(close_op)
+      sess.run(init_op)
       for i in range(100):
         self.assertEqual(sorted([elements[i * 2], elements[i * 2 + 1]]),
                          sorted(sess.run(get_next)))
@@ -415,15 +415,15 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(counter_var.initializer)
-      self.evaluate(init_op)
+      sess.run(counter_var.initializer)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(counter_var))
-        self.assertEqual(i + 1, self.evaluate(get_next))
-      self.assertEqual(10, self.evaluate(counter_var))
+        self.assertEqual(i, sess.run(counter_var))
+        self.assertEqual(i + 1, sess.run(get_next))
+      self.assertEqual(10, sess.run(counter_var))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
-      self.assertEqual(10, self.evaluate(counter_var))
+      self.assertEqual(10, sess.run(counter_var))
 
   def testCaptureUninitializedVariableError(self):
     counter_var = variable_scope.get_variable(
@@ -435,7 +435,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaises(errors.NotFoundError):
         sess.run(get_next)
 
@@ -447,14 +447,14 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       random_values = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
           random_values.extend(sess.run(get_next))
       self.assertEqual(10, len(random_values))
       self.assertGreater(np.abs(np.diff(random_values)).max(), 1e-6)
-      self.evaluate(init_op)
+      sess.run(init_op)
       random_values_2 = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
@@ -473,8 +473,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      random_values = self.evaluate(get_next)
+      sess.run(init_op)
+      random_values = sess.run(get_next)
 
       # Assert that one of the next 99 batches yielded by the iterator is
       # different from the first.
@@ -500,15 +500,15 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(counter_var.initializer)
-      self.evaluate(init_op)
+      sess.run(counter_var.initializer)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual(i, self.evaluate(counter_var))
-        self.assertEqual(i, self.evaluate(get_next))
-      self.assertEqual(10, self.evaluate(counter_var))
+        self.assertEqual(i, sess.run(counter_var))
+        self.assertEqual(i, sess.run(get_next))
+      self.assertEqual(10, sess.run(counter_var))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
-      self.assertEqual(10, self.evaluate(counter_var))
+      self.assertEqual(10, sess.run(counter_var))
 
   def testMapDict(self):
     iterator = (dataset_ops.Dataset.range(10)
@@ -519,9 +519,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual(i * 2 + i**2, self.evaluate(get_next))
+        self.assertEqual(i * 2 + i ** 2, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -569,8 +569,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
-      self.assertAllEqual(row**2, self.evaluate(get_next))
+      sess.run(init_op)
+      self.assertAllEqual(row ** 2, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -611,7 +611,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       row = np.arange(6)
       for num in [2, 3, 4]:
         init_op, get_next = build_dataset(row, num)
-        self.evaluate(init_op)
+        sess.run(init_op)
         for i in range(6):
           self.assertEqual(
               (i // 2 if i % 2 else i * 2) if (num == 2 or num == 3) else i * 2,
@@ -652,7 +652,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       row = np.arange(6)
       for num in [2, 3, 4]:
         init_op, get_next = build_dataset(row, num)
-        self.evaluate(init_op)
+        sess.run(init_op)
         self.assertAllEqual(
             [x // 2 if (num == 2 or num == 3) else x * 2 for x in row],
             sess.run(get_next))
@@ -697,7 +697,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       self.assertAllEqual([(x // 2 if x % 2 else x * 2) if
                            (num == 2 or num == 3) else x * 2 for x in row],
                           sess.run(get_next))
@@ -735,7 +735,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       for buffer_size in [1, 10, 100, 1000]:
         sess.run(init_op, feed_dict={buffer_size_placeholder: buffer_size})
         for i in range(100):
-          self.assertEqual(i * i, self.evaluate(get_next))
+          self.assertEqual(i * i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -753,10 +753,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(init_op, feed_dict={buffer_size_placeholder: buffer_size})
         for i in range(event_will_be_set_after_consuming):
           self.assertFalse(ev.is_set())
-          self.assertEqual(i * i, self.evaluate(get_next))
+          self.assertEqual(i * i, sess.run(get_next))
         ev.wait()
         for i in range(event_will_be_set_after_consuming, 100):
-          self.assertEqual(i * i, self.evaluate(get_next))
+          self.assertEqual(i * i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -768,9 +768,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual((i, 37.0), self.evaluate(get_next))
+        self.assertEqual((i, 37.0), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -789,9 +789,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual((i, 37.0), self.evaluate(get_next))
+        self.assertEqual((i, 37.0), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -810,9 +810,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         self.assertIsInstance(actual, sparse_tensor.SparseTensorValue)
         self.assertSparseValuesEqual(actual, _sparse(i))
       with self.assertRaises(errors.OutOfRangeError):
@@ -837,9 +837,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         self.assertIsInstance(actual, sparse_tensor.SparseTensorValue)
         self.assertSparseValuesEqual(actual, _check(_sparse(i)).eval())
       with self.assertRaises(errors.OutOfRangeError):
@@ -861,9 +861,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(100):
-        self.assertEqual(i, self.evaluate(get_next))
+        self.assertEqual(i, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -875,9 +875,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       for i in range(10):
-        self.assertEqual((i, b"hello", 10), self.evaluate(get_next))
+        self.assertEqual((i, b"hello", 10), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -945,7 +945,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "BrokenConst"):
-        self.evaluate(iterator.initializer)
+        sess.run(iterator.initializer)
 
 # pylint: disable=g-long-lambda
   @parameterized.named_parameters(
@@ -972,7 +972,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      tids = self.evaluate(get_next)
+      tids = sess.run(get_next)
       self.assertTrue(all(tids[0] == tid for tid in tids))
 # pylint: enable=g-long-lambda
 
@@ -996,7 +996,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         expected = map_fn(*sess.run(self.structuredElement(structure)))
       else:
         expected = map_fn(sess.run(self.structuredElement(structure)))
-      self.assertEqual(expected, self.evaluate(get_next))
+      self.assertEqual(expected, sess.run(get_next))
 
   @parameterized.named_parameters(
       ("Sequential", None),
@@ -1011,7 +1011,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={captured_t: 42})
-      self.assertEqual(42, self.evaluate(get_next))
+      self.assertEqual(42, sess.run(get_next))
 
   @parameterized.named_parameters(
       ("1", 1, 1),
@@ -1030,7 +1030,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session(config=config) as sess:
       for i in range(num_elements):
         coordination_events[i].set()
-        self.assertEqual(i * i, self.evaluate(get_next))
+        self.assertEqual(i * i, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -1052,7 +1052,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
       for element in elements:
         coordination_events[element].set()
-        self.assertEqual(element * element, self.evaluate(get_next))
+        self.assertEqual(element * element, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index ea6828e575..42ee1e2186 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -40,7 +40,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
 
   def testBasic(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -50,10 +50,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -67,10 +67,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -85,12 +85,12 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 20, 4):
-        self.assertEqual(i, self.evaluate(elem_on_1))
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
-        self.assertEqual(i + 2, self.evaluate(elem_on_3))
-        self.assertEqual(i + 3, self.evaluate(elem_on_4))
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i + 2, sess.run(elem_on_3))
+        self.assertEqual(i + 3, sess.run(elem_on_4))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -105,11 +105,11 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
-      self.assertEqual(8, self.evaluate(elem_on_1))
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+      self.assertEqual(8, sess.run(elem_on_1))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -126,7 +126,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
         elem_on_1_has_value, elem_on_1_value = sess.run(
             [elem_on_1_has_value_t, elem_on_1_t])
@@ -140,8 +140,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
           [elem_on_1_has_value_t, elem_on_1_t])
       self.assertTrue(elem_on_1_has_value)
       self.assertEqual(8, elem_on_1_value)
-      self.assertFalse(self.evaluate(elem_on_1_has_value_t))
-      self.assertFalse(self.evaluate(elem_on_2_has_value_t))
+      self.assertFalse(sess.run(elem_on_1_has_value_t))
+      self.assertFalse(sess.run(elem_on_2_has_value_t))
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(elem_on_1_t)
       with self.assertRaises(errors.InvalidArgumentError):
@@ -155,11 +155,11 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i, sess.run(elem_on_1))
       for i in range(0, 10, 2):
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -192,10 +192,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -211,11 +211,11 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i, sess.run(elem_on_1))
       for i in range(0, 10, 2):
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
@@ -235,7 +235,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
         elem_on_1_has_value, elem_on_1_value = sess.run(
             [elem_on_1_has_value_t, elem_on_1_t])
@@ -249,8 +249,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
           [elem_on_1_has_value_t, elem_on_1_t])
       self.assertTrue(elem_on_1_has_value)
       self.assertEqual(8, elem_on_1_value)
-      self.assertFalse(self.evaluate(elem_on_1_has_value_t))
-      self.assertFalse(self.evaluate(elem_on_2_has_value_t))
+      self.assertFalse(sess.run(elem_on_1_has_value_t))
+      self.assertFalse(sess.run(elem_on_2_has_value_t))
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(elem_on_1_t)
       with self.assertRaises(errors.InvalidArgumentError):
@@ -272,10 +272,10 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      self.evaluate(multi_device_iterator.initializer)
+      sess.run(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, self.evaluate(elem_on_1))
-        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(elem_on_1)
         sess.run(elem_on_2)
diff --git a/tensorflow/python/data/kernel_tests/optional_ops_test.py b/tensorflow/python/data/kernel_tests/optional_ops_test.py
index 0981ff9651..604e3ad88e 100644
--- a/tensorflow/python/data/kernel_tests/optional_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_ops_test.py
@@ -227,7 +227,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
 
       # For each element of the dataset, assert that the optional evaluates to
       # the expected value.
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       for _ in range(3):
         elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t])
         self.assertTrue(elem_has_value)
@@ -236,7 +236,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       # After exhausting the iterator, `next_elem.has_value()` will evaluate to
       # false, and attempting to get the value will fail.
       for _ in range(2):
-        self.assertFalse(self.evaluate(elem_has_value_t))
+        self.assertFalse(sess.run(elem_has_value_t))
         with self.assertRaises(errors.InvalidArgumentError):
           sess.run(elem_value_t)
 
diff --git a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
index af326ec210..76e2697b29 100644
--- a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
@@ -40,7 +40,7 @@ class PrefetchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(init_op, feed_dict={buffer_size_t: buffer_size})
       for m in range(10):
-        self.assertEqual(m, self.evaluate(get_next))
+        self.assertEqual(m, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
index fcb025c8b8..9fc79707d0 100644
--- a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
@@ -124,19 +124,19 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
-        self.evaluate(restore_op)
+        sess.run(init_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -144,14 +144,14 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
-        self.evaluate(restore_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -175,14 +175,14 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for _ in range(break_epoch):
           for i in range(start, stop):
-            self.assertEqual(i, self.evaluate(get_next))
+            self.assertEqual(i, sess.run(get_next))
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       # Create an empty IteratorResource and restore the Iterator into it.
@@ -193,12 +193,12 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
       restore_op = self._restore_op(iterator._iterator_resource)
       get_next = iterator.get_next()
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         for _ in range(break_epoch + 1, num_epochs):
           for i in range(start, stop):
-            self.assertEqual(i, self.evaluate(get_next))
+            self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -221,20 +221,20 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       # Intentionally build a graph with a different value for stop to make sure
       # the original dataset graph is actually getting loaded.
       init_op, get_next, _, restore_op = _build_graph(start, stop_1)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -259,19 +259,19 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
-        self.evaluate(restore_op)
+        sess.run(init_op)
+        sess.run(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -294,27 +294,27 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         for i in range(start, break_point1):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for i in range(break_point1, break_point2):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     break_point2 = 7
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for i in range(break_point2, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -338,28 +338,28 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
       init_op, get_next, save_op, restore_op = _build_graph(
           start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for _ in range(break_epoch - 1):
           for i in range(start, stop):
-            self.assertEqual(i, self.evaluate(get_next))
+            self.assertEqual(i, sess.run(get_next))
         for i in range(start, break_range):
-          self.assertEqual(i, self.evaluate(get_next))
-        self.evaluate(save_op)
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for i in range(break_range, stop):
-          self.assertEqual(i, self.evaluate(get_next))
+          self.assertEqual(i, sess.run(get_next))
         for _ in range(break_epoch, num_epochs):
           for i in range(start, stop):
-            self.assertEqual(i, self.evaluate(get_next))
+            self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
@@ -381,23 +381,23 @@ class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
       init_op, get_next, save_op, restore_op = _build_graph(
           start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.evaluate(init_op)
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for _ in range(num_epochs):
           for i in range(start, stop):
-            self.assertEqual(i, self.evaluate(get_next))
+            self.assertEqual(i, sess.run(get_next))
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
-        self.evaluate(save_op)
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
index e26381e902..4fef4f30bf 100644
--- a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
@@ -107,7 +107,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[0]],
                               num_epochs: 1})
       for i in range(5):
-        self.assertEqual(self._lineText(0, i), self.evaluate(get_next))
+        self.assertEqual(self._lineText(0, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -116,7 +116,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[1]],
                               num_epochs: 1})
       for i in range(5):
-        self.assertEqual(self._lineText(1, i), self.evaluate(get_next))
+        self.assertEqual(self._lineText(1, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -124,7 +124,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1})
       for j in range(2):
         for i in range(5):
-          self.assertEqual(self._lineText(j, i), self.evaluate(get_next))
+          self.assertEqual(self._lineText(j, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -133,7 +133,7 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
       for _ in range(10):
         for j in range(2):
           for i in range(5):
-            self.assertEqual(self._lineText(j, i), self.evaluate(get_next))
+            self.assertEqual(self._lineText(j, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -267,7 +267,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[0]],
                               num_epochs: 1})
       for i in range(self._num_records):
-        self.assertEqual(self._record(0, i), self.evaluate(get_next))
+        self.assertEqual(self._record(0, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -276,7 +276,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           init_op, feed_dict={filenames: [test_filenames[1]],
                               num_epochs: 1})
       for i in range(self._num_records):
-        self.assertEqual(self._record(1, i), self.evaluate(get_next))
+        self.assertEqual(self._record(1, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -284,7 +284,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1})
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertEqual(self._record(j, i), self.evaluate(get_next))
+          self.assertEqual(self._record(j, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -293,7 +293,7 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       for _ in range(10):
         for j in range(self._num_files):
           for i in range(self._num_records):
-            self.assertEqual(self._record(j, i), self.evaluate(get_next))
+            self.assertEqual(self._record(j, i), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -405,19 +405,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                self.evaluate(save_op)
+                sess.run(save_op)
                 break
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
             else:
               continue
             break
@@ -426,13 +426,13 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            self.evaluate(get_next_op)
+            sess.run(get_next_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -441,9 +441,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
 
   def testInitThenRestore(self):
     # Note: Calling init_op before restore_op is redundant. This test just makes
@@ -458,19 +458,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                self.evaluate(save_op)
+                sess.run(save_op)
                 break
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
             else:
               continue
             break
@@ -479,14 +479,14 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            self.evaluate(get_next_op)
+            sess.run(get_next_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
-        self.evaluate(restore_op)
+        sess.run(init_op)
+        sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -495,9 +495,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
 
   def testRestoreInModifiedGraph(self):
     num_epochs = 10
@@ -510,19 +510,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                self.evaluate(save_op)
+                sess.run(save_op)
                 break
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
             else:
               continue
             break
@@ -531,13 +531,13 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            self.evaluate(get_next_op)
+            sess.run(get_next_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs_1)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -546,9 +546,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
 
   def testRestoreWithoutBuildingDatasetGraph(self):
     num_epochs = 10
@@ -560,19 +560,19 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
               if (epoch == epoch_break and f == file_break and
                   r == record_break):
-                self.evaluate(save_op)
+                sess.run(save_op)
                 break
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
             else:
               continue
             break
@@ -581,12 +581,12 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
           break
         else:
           with self.assertRaises(errors.OutOfRangeError):
-            self.evaluate(get_next_op)
+            sess.run(get_next_op)
 
     with ops.Graph().as_default() as g:
       restore_op, get_next_op = self._restore_iterator()
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for epoch in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
@@ -595,9 +595,9 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
                   (epoch == epoch_break and f == file_break and
                    r < record_break)):
                 continue
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
 
   def testRestoreUnusedIterator(self):
     num_epochs = 10
@@ -605,22 +605,22 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         # Save unused iterator.
-        self.evaluate(save_op)
+        sess.run(save_op)
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         for _ in range(num_epochs * self._num_files * self._num_records):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
 
   def testRestoreExhaustedIterator(self):
     num_epochs = 10
@@ -629,26 +629,26 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         # Note: There is no checkpoint saved currently so a NotFoundError is
         # raised.
         with self.assertRaises(errors.NotFoundError):
-          self.evaluate(restore_op)
+          sess.run(restore_op)
         for _ in range(num_epochs):
           for f in range(self._num_files):
             for r in range(self._num_records):
-              self.assertEqual(self._record(f, r), self.evaluate(get_next_op))
+              self.assertEqual(self._record(f, r), sess.run(get_next_op))
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
-        self.evaluate(save_op)
+          sess.run(get_next_op)
+        sess.run(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
           num_epochs=num_epochs)
       with self.session(graph=g) as sess:
-        self.evaluate(restore_op)
+        sess.run(restore_op)
         with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(get_next_op)
+          sess.run(get_next_op)
 
 
 class TFRecordDatasetTest(test_base.DatasetTestBase):
@@ -807,7 +807,7 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), self.evaluate(next_element))
+          self.assertAllEqual(self._record(j, i), sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -819,7 +819,7 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), self.evaluate(next_element))
+          self.assertAllEqual(self._record(j, i), sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py b/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
index d7f3988b1a..11e07300b9 100644
--- a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
@@ -36,7 +36,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.range(1, i + 1)
       result = ds.reduce(np.int64(0), lambda x, y: x + y)
       with self.cached_session() as sess:
-        self.assertEqual(((i + 1) * i) // 2, self.evaluate(result))
+        self.assertEqual(((i + 1) * i) // 2, sess.run(result))
 
   def testSumTuple(self):
 
@@ -49,7 +49,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.zip((ds, ds))
       result = ds.reduce(np.int64(0), reduce_fn)
       with self.cached_session() as sess:
-        self.assertEqual(((i + 1) * i), self.evaluate(result))
+        self.assertEqual(((i + 1) * i), sess.run(result))
 
   def testSumAndCount(self):
 
@@ -61,7 +61,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.range(1, i + 1)
       result = ds.reduce((np.int64(0), np.int64(0)), reduce_fn)
       with self.cached_session() as sess:
-        s, c = self.evaluate(result)
+        s, c = sess.run(result)
         self.assertEqual(((i + 1) * i) // 2, s)
         self.assertEqual(i, c)
 
@@ -93,8 +93,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.from_tensors(make_sparse_fn(i+1))
       result = ds.reduce(make_sparse_fn(0), reduce_fn)
       with self.cached_session() as sess:
-        self.assertSparseValuesEqual(
-            make_sparse_fn(i + 1), self.evaluate(result))
+        self.assertSparseValuesEqual(make_sparse_fn(i+1), sess.run(result))
 
   def testNested(self):
 
@@ -117,7 +116,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ds = dataset_ops.Dataset.range(1, i + 1).map(map_fn)
       result = ds.reduce(map_fn(0), reduce_fn)
       with self.cached_session() as sess:
-        result = self.evaluate(result)
+        result = sess.run(result)
         self.assertEqual(((i + 1) * i) // 2, result["dense"])
         self.assertSparseValuesEqual(make_sparse_fn(i), result["sparse"])
 
diff --git a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
index 946aa01f73..e86356dee7 100644
--- a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
@@ -49,7 +49,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Test a finite repetition.
       sess.run(init_op, feed_dict={count_placeholder: 3})
       for _ in range(3):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
 
@@ -59,7 +59,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Test a different finite repetition.
       sess.run(init_op, feed_dict={count_placeholder: 7})
       for _ in range(7):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -75,7 +75,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # actually is infinite.
       sess.run(init_op, feed_dict={count_placeholder: -1})
       for _ in range(17):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
 
@@ -95,7 +95,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Take fewer than input size
       sess.run(init_op, feed_dict={count_placeholder: 4})
       for i in range(4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
 
       with self.assertRaises(errors.OutOfRangeError):
@@ -104,7 +104,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Take more than input size
       sess.run(init_op, feed_dict={count_placeholder: 25})
       for i in range(10):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
 
       with self.assertRaises(errors.OutOfRangeError):
@@ -113,7 +113,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Take all of input
       sess.run(init_op, feed_dict={count_placeholder: -1})
       for i in range(10):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
 
       with self.assertRaises(errors.OutOfRangeError):
@@ -142,7 +142,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # the first 4 elements and then read the rest.
       sess.run(init_op, feed_dict={count_placeholder: 4})
       for i in range(4, 10):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -165,7 +165,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       # Skip nothing
       sess.run(init_op, feed_dict={count_placeholder: 0})
       for i in range(0, 10):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         self.assertAllEqual(results, components[0][i:i+1])
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
@@ -187,7 +187,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       sess.run(init_op, feed_dict={inner_count: 7, outer_count: 14})
       for _ in range(7 * 14):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(components, results):
           self.assertAllEqual(component, result_component)
       with self.assertRaises(errors.OutOfRangeError):
@@ -201,7 +201,7 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
index 990f4f212b..cad28f860e 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
@@ -66,7 +66,7 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       # First run without shuffling to collect the "ground truth".
-      self.evaluate(init_fifo_op)
+      sess.run(init_fifo_op)
       unshuffled_elements = []
       for _ in range(20):
         unshuffled_elements.append(sess.run(get_next))
@@ -159,7 +159,7 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={seed_placeholder: 0})
       for elem in elems:
-        self.assertEqual(elem, self.evaluate(get_next))
+        self.assertEqual(elem, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -188,9 +188,9 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      initial_permutation = self.evaluate(next_element)
-      self.assertAllEqual(initial_permutation, self.evaluate(next_element))
-      self.assertAllEqual(initial_permutation, self.evaluate(next_element))
+      initial_permutation = sess.run(next_element)
+      self.assertAllEqual(initial_permutation, sess.run(next_element))
+      self.assertAllEqual(initial_permutation, sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -261,7 +261,7 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.session(graph=g) as sess:
         for iterator in iterators:
           if initializable:
-            self.evaluate(iterator.initializer)
+            sess.run(iterator.initializer)
           next_element = iterator.get_next()
           run_results = []
           for _ in range(300):
diff --git a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
index 35adcddfe7..9d06781094 100644
--- a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
@@ -102,7 +102,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       num_full_batches = max(
           0, (count * 7 - ((size - 1) * stride + 1)) // shift + 1)
       for i in range(num_full_batches):
-        result = self.evaluate(get_next)
+        result = sess.run(get_next)
         for component, result_component in zip(components, result):
           for j in range(size):
             self.assertAllEqual(component[(i * shift + j * stride) % 7]**2,
@@ -111,7 +111,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         num_partial_batches = (count * 7) // shift + (
             (count * 7) % shift > 0) - num_full_batches
         for i in range(num_partial_batches):
-          result = self.evaluate(get_next)
+          result = sess.run(get_next)
           for component, result_component in zip(components, result):
             remaining = (count * 7) - ((num_full_batches + i) * shift)
             num_elements = remaining // stride + ((remaining % stride) > 0)
@@ -164,10 +164,10 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       num_batches = (10 - 5) // 3 + 1
       for i in range(num_batches):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4],
@@ -193,10 +193,10 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       num_batches = (10 - 5) // 3 + 1
       for i in range(num_batches):
-        actual = self.evaluate(get_next)
+        actual = sess.run(get_next)
         expected_indices = []
         expected_values = []
         for j in range(5):
@@ -227,9 +227,9 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(init_op)
+      sess.run(init_op)
       # Slide: 1st batch.
-      actual = self.evaluate(get_next)
+      actual = sess.run(get_next)
       expected = sparse_tensor.SparseTensorValue(
           indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
                    [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
@@ -239,7 +239,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertTrue(sparse_tensor.is_sparse(actual))
       self.assertSparseValuesEqual(actual, expected)
       # Slide: 2nd batch.
-      actual = self.evaluate(get_next)
+      actual = sess.run(get_next)
       expected = sparse_tensor.SparseTensorValue(
           indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
                    [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
@@ -265,7 +265,7 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
+      sess.run(iterator.initializer)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           r"Cannot batch tensors with different shapes in component 0. "
@@ -281,8 +281,8 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = dataset.make_one_shot_iterator().get_next()
 
     with self.cached_session() as sess:
-      self.assertAllEqual(np.float32([1., 2.]), self.evaluate(get_next))
-      self.assertAllEqual(np.float32([2., 3.]), self.evaluate(get_next))
+      self.assertAllEqual(np.float32([1., 2.]), sess.run(get_next))
+      self.assertAllEqual(np.float32([2., 3.]), sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
diff --git a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
index b60ec4ecce..9d76387a34 100644
--- a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
@@ -55,7 +55,7 @@ class ZipDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={ph: value for ph, value in zip(
           component_placeholders, equal_length_components)})
       for i in range(4):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(
             equal_length_components, results):
           self.assertAllEqual(component[i], result_component)
@@ -66,7 +66,7 @@ class ZipDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={ph: value for ph, value in zip(
           component_placeholders, variable_length_components)})
       for i in range(2):
-        results = self.evaluate(get_next)
+        results = sess.run(get_next)
         for component, result_component in zip(
             variable_length_components, results):
           self.assertAllEqual(component[i], result_component)
@@ -103,7 +103,7 @@ class ZipDatasetTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={ph: value for ph, value in zip(
           component_placeholders, equal_length_components)})
       for i in range(4):
-        result1, (result2, result3) = self.evaluate(get_next)
+        result1, (result2, result3) = sess.run(get_next)
         self.assertAllEqual(equal_length_components[0][i], result1)
         self.assertAllEqual(equal_length_components[1][i], result2)
         self.assertAllEqual(equal_length_components[2][i], result3)
diff --git a/tensorflow/python/data/util/convert_test.py b/tensorflow/python/data/util/convert_test.py
index 4a5b730381..89c3afb296 100644
--- a/tensorflow/python/data/util/convert_test.py
+++ b/tensorflow/python/data/util/convert_test.py
@@ -31,24 +31,24 @@ class ConvertTest(test.TestCase):
   def testInteger(self):
     resp = convert.optional_param_to_tensor("foo", 3)
     with self.cached_session() as sess:
-      self.assertEqual(3, self.evaluate(resp))
+      self.assertEqual(3, sess.run(resp))
 
   def testIntegerDefault(self):
     resp = convert.optional_param_to_tensor("foo", None)
     with self.cached_session() as sess:
-      self.assertEqual(0, self.evaluate(resp))
+      self.assertEqual(0, sess.run(resp))
 
   def testStringDefault(self):
     resp = convert.optional_param_to_tensor("bar", None, "default",
                                             dtypes.string)
     with self.cached_session() as sess:
-      self.assertEqual(compat.as_bytes("default"), self.evaluate(resp))
+      self.assertEqual(compat.as_bytes("default"), sess.run(resp))
 
   def testString(self):
     resp = convert.optional_param_to_tensor("bar", "value", "default",
                                             dtypes.string)
     with self.cached_session() as sess:
-      self.assertEqual(compat.as_bytes("value"), self.evaluate(resp))
+      self.assertEqual(compat.as_bytes("value"), sess.run(resp))
 
   def testPartialShapeToTensorKnownDimension(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index 5aa7d1bb4c..f197a9e4dc 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -1583,7 +1583,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
       x = variables.VariableV1([1, 3, 3, 7], name="x")
       _, idx = array_ops.unique(x, name="x_unique")
       idx_times_two = math_ops.multiply(idx, 2, name="idx_times_two")
-      self.evaluate(x.initializer)
+      sess.run(x.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(
diff --git a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
index 34030c0adc..1f67f8a0d4 100644
--- a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
+++ b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
@@ -126,8 +126,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       u = variables.Variable([12.0], name="u")
       v = variables.Variable([30.0], name="v")
       w = math_ops.add(u, v, name="w")
-      self.evaluate(u.initializer)
-      self.evaluate(v.initializer)
+      sess.run(u.initializer)
+      sess.run(v.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, w, expected_output=[42.0])
@@ -139,7 +139,7 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
         b = math_ops.add(a, a, name="b")
       with ops.control_dependencies([a, b]):
         c = math_ops.multiply(b, b, name="c")
-      self.evaluate(a.initializer)
+      sess.run(a.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, c, expected_output=400.0)
@@ -150,8 +150,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       y = variables.Variable(20.0, name="y")
       cond = control_flow_ops.cond(
           x > y, lambda: math_ops.add(x, 1), lambda: math_ops.add(y, 1))
-      self.evaluate(x.initializer)
-      self.evaluate(y.initializer)
+      sess.run(x.initializer)
+      sess.run(y.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, cond, expected_output=21.0)
@@ -173,8 +173,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       toy_loss = x * (u - v)
       train_op = gradient_descent.GradientDescentOptimizer(
           learning_rate=0.1).minimize(toy_loss, name="train_op")
-      self.evaluate(u.initializer)
-      self.evaluate(v.initializer)
+      sess.run(u.initializer)
+      sess.run(v.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(sess, train_op)
 
diff --git a/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py b/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
index b78c3d16d4..74498c8ea3 100644
--- a/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
+++ b/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
@@ -131,8 +131,8 @@ class DistributedSessionDebugTest(test_util.TensorFlowTestCase):
     with session.Session(
         config=self.session_config, graph=graph,
         target=self.server_target) as sess:
-      self.evaluate(self.a.initializer)
-      self.evaluate(self.b.initializer)
+      sess.run(self.a.initializer)
+      sess.run(self.b.initializer)
 
       run_options = config_pb2.RunOptions()
       debug_utils.watch_graph(
@@ -198,8 +198,8 @@ class DistributedSessionDebugTest(test_util.TensorFlowTestCase):
     with session.Session(
         config=self.session_config, graph=graph,
         target=self.server_target) as sess:
-      self.evaluate(self.a.initializer)
-      self.evaluate(self.b.initializer)
+      sess.run(self.a.initializer)
+      sess.run(self.b.initializer)
 
       def watch_fn(feeds, fetch_keys):
         del feeds, fetch_keys
diff --git a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
index 8eef45392f..b0dc25851c 100644
--- a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
+++ b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
@@ -67,7 +67,7 @@ class SessionDebugMultiGPUTest(test_util.TensorFlowTestCase):
         u1 = math_ops.multiply(v, v, name="u1")
       w = math_ops.subtract(u1, u0, name="w")
 
-      self.evaluate(v.initializer)
+      sess.run(v.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(run_options, sess.graph,
diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py
index a16d68329a..4a8d4eaa99 100644
--- a/tensorflow/python/debug/lib/source_utils_test.py
+++ b/tensorflow/python/debug/lib/source_utils_test.py
@@ -109,8 +109,8 @@ class SourceHelperTest(test_util.TensorFlowTestCase):
       self.w = math_ops.matmul(self.u, self.v, name="w")
       self.w_line_number = line_number_above()
 
-      self.evaluate(self.u.initializer)
-      self.evaluate(self.v.initializer)
+      sess.run(self.u.initializer)
+      sess.run(self.v.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(
diff --git a/tensorflow/python/distribute/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py
index 54f7c5d012..cbb93e8995 100644
--- a/tensorflow/python/distribute/input_ops_test.py
+++ b/tensorflow/python/distribute/input_ops_test.py
@@ -92,7 +92,7 @@ class AutoShardDatasetTest(test.TestCase):
     with self.cached_session() as sess:
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(record_fn(r, f), self.evaluate(next_element))
+          self.assertAllEqual(record_fn(r, f), sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
@@ -205,11 +205,10 @@ class AutoShardDatasetTest(test.TestCase):
     with self.cached_session() as sess:
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(self._record(r, f), self.evaluate(next_element))
+          self.assertAllEqual(self._record(r, f), sess.run(next_element))
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(
-              self._text_line(r, f), self.evaluate(next_element))
+          self.assertAllEqual(self._text_line(r, f), sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 54991344b7..f0f71a219e 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -149,9 +149,9 @@ class DefFunctionTest(test.TestCase):
 
       result = fn(3.0)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       self.assertAllEqual(sess.run(state[0]), 2.0)
-      self.assertAllEqual(self.evaluate(result), 6.0)
+      self.assertAllEqual(sess.run(result), 6.0)
 
   def testLegacyGraphModeVariablesNonTrivialInitializer(self):
     with ops.Graph().as_default(), self.test_session() as sess:
@@ -168,9 +168,9 @@ class DefFunctionTest(test.TestCase):
 
       result = fn(3.0)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       self.assertAllEqual(sess.run(state[0]), 6.0)
-      self.assertAllEqual(self.evaluate(result), 18.0)
+      self.assertAllEqual(sess.run(result), 18.0)
 
   def testLegacyGraphModeInputDependentInitializerFails(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py
index 1ba596573f..d4f8aaa7e3 100644
--- a/tensorflow/python/eager/function_gradients_test.py
+++ b/tensorflow/python/eager/function_gradients_test.py
@@ -78,7 +78,7 @@ class FunctionGradientsTest(test.TestCase, parameterized.TestCase):
       c = constant_op.constant([[2.]])
       f_c = f(c)
       g, = gradients_impl.gradients(f_c, c)
-      self.assertAllEqual(self.evaluate(g).values, [[1.0]])
+      self.assertAllEqual(sess.run(g).values, [[1.0]])
 
   def testNoSymGradNestedDefun(self):
 
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index a206b1f791..b58b09140d 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -564,7 +564,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       variables.global_variables_initializer().run()
       call = def_function.function(o.call)
       op = call()
-      self.assertAllEqual(self.evaluate(op), 2.0)
+      self.assertAllEqual(sess.run(op), 2.0)
 
   def testGraphModeManyFunctions(self):
     with ops.Graph().as_default(), self.cached_session():
@@ -1732,7 +1732,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
       function.register(cpu_boost, x)
       y = gpu_boost(x)
-      y_value = self.evaluate(y)
+      y_value = sess.run(y)
 
       if test.is_gpu_available():
         self.assertEqual(y_value, 5.0)
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 2c70d66810..e9b11c3960 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -1027,7 +1027,7 @@ class CrossedColumnTest(test.TestCase):
     outputs = _transform_features(features, [price_cross_wire])
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
-      output_val = self.evaluate(output)
+      output_val = sess.run(output)
       self.assertAllEqual(
           [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
       for val in output_val.values:
@@ -1886,8 +1886,7 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
-                          self.evaluate(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc._numeric_column('price')
@@ -2526,8 +2525,7 @@ class _LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
-                          self.evaluate(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc._numeric_column('price')
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 23131e22ed..115763f656 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -1188,7 +1188,7 @@ class CrossedColumnTest(test.TestCase):
     outputs = fc._transform_features_v2(features, [price_cross_wire], None)
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
-      output_val = self.evaluate(output)
+      output_val = sess.run(output)
       self.assertAllEqual(
           [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
       for val in output_val.values:
@@ -2088,8 +2088,7 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]],
-                          self.evaluate(net))
+      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]], sess.run(net))
 
       coord.request_stop()
       coord.join(threads)
@@ -2125,8 +2124,7 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
-                          self.evaluate(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
@@ -2845,8 +2843,7 @@ class OldLinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
-                          self.evaluate(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 90deb9765f..971219d5b0 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -102,7 +102,7 @@ class FunctionTest(test.TestCase):
       call = MyIdentityFunc([18.0])
       self.assertEqual("MyIdentity", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([18.0], self.evaluate(call))
+        self.assertAllEqual([18.0], sess.run(call))
 
   def testIdentityImplicitDeref(self):
 
@@ -116,8 +116,8 @@ class FunctionTest(test.TestCase):
       self.assertEqual("MyIdentity", call.op.name)
       for cfg in _OptimizerOptions():
         with session.Session(config=cfg) as sess:
-          self.evaluate(var.initializer)
-          self.assertAllEqual([18.0], self.evaluate(call))
+          sess.run(var.initializer)
+          self.assertAllEqual([18.0], sess.run(call))
 
   def testIdentityOutputName(self):
 
@@ -130,7 +130,7 @@ class FunctionTest(test.TestCase):
       call = MyIdentityFunc([18.0])
       self.assertEqual("MyIdentity", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([18.0], self.evaluate(call))
+        self.assertAllEqual([18.0], sess.run(call))
 
   def testTooManyOutputNames(self):
 
@@ -158,7 +158,7 @@ class FunctionTest(test.TestCase):
       call = APlus2B([1.0], [2.0])
       self.assertEqual("APlus2B", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([5.0], self.evaluate(call))
+        self.assertAllEqual([5.0], sess.run(call))
 
   def testFunctionWithNoOutput(self):
 
@@ -187,7 +187,7 @@ class FunctionTest(test.TestCase):
       call = APlus2B([1.0], [2.0])
       self.assertEqual("APlus2B", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([5.0], self.evaluate(call))
+        self.assertAllEqual([5.0], sess.run(call))
 
   def testDefineFunctionDuplicateOutputs(self):
 
@@ -224,8 +224,8 @@ class FunctionTest(test.TestCase):
       call_g = XSquarePlusOneGrad([2.0], [0.1])
 
       with session.Session() as sess:
-        self.assertAllClose([5.0], self.evaluate(call_f))
-        self.assertAllClose([0.4], self.evaluate(call_g))
+        self.assertAllClose([5.0], sess.run(call_f))
+        self.assertAllClose([0.4], sess.run(call_g))
 
   def testTanhSymGrad(self):
 
@@ -387,7 +387,7 @@ class FunctionTest(test.TestCase):
       call = AConstant()
       self.assertEqual("AConstant", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([42], self.evaluate(call))
+        self.assertAllEqual([42], sess.run(call))
 
   def testDefineFunctionNames(self):
 
@@ -468,7 +468,7 @@ class FunctionTest(test.TestCase):
 
       loop = control_flow_ops.while_loop(lambda x: x < 1e5, Body, [1.0])
 
-      ans = self.evaluate(loop)
+      ans = sess.run(loop)
       self.assertAllClose(ans, 131072.)
 
   def testControlFlowStrictness(self):
@@ -650,8 +650,8 @@ class FunctionTest(test.TestCase):
       # pylint: enable=unexpected-keyword-arg
       self.assertEqual("next", call2.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([1], self.evaluate(call1))
-        self.assertAllEqual([0], self.evaluate(call2))
+        self.assertAllEqual([1], sess.run(call1))
+        self.assertAllEqual([0], sess.run(call2))
 
   def testNestedFunction(self):
 
@@ -794,7 +794,7 @@ class FunctionTest(test.TestCase):
       y = Foo()
 
     with self.session(graph=g) as sess:
-      self.assertEqual(self.evaluate(y), 10)
+      self.assertEqual(sess.run(y), 10)
 
   def testCaptureInCond(self):
     g = ops.Graph()
@@ -809,8 +809,8 @@ class FunctionTest(test.TestCase):
       z = Foo(False)
 
     with self.session(graph=g) as sess:
-      self.assertEqual(self.evaluate(y), 1)
-      self.assertEqual(self.evaluate(z), 2)
+      self.assertEqual(sess.run(y), 1)
+      self.assertEqual(sess.run(z), 2)
 
   def testStableName(self):
 
@@ -900,7 +900,7 @@ class FunctionTest(test.TestCase):
     self.assertEqual(global_vars[0].name, "linear/w:0")
 
     with session.Session() as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       output_val = sess.run(
           output_op, feed_dict={input_op: np.random.rand(32, 100)})
       self.assertEqual(output_val.shape, (32, 100))
@@ -928,7 +928,7 @@ class FunctionTest(test.TestCase):
     self.assertEqual(global_vars[0].name, "vs1/var:0")
 
     with session.Session() as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       out1, out2 = sess.run(
           [out1_op, out2_op], feed_dict={input_op: np.linspace(1, 10, 10)})
       self.assertAllEqual(out1, np.linspace(2, 11, 10))
@@ -991,8 +991,8 @@ class FunctionTest(test.TestCase):
     result_2 = Bar(constant_op.constant(100, dtype=dtypes.int64))
 
     with session.Session() as sess:
-      self.assertEqual(4.0, self.evaluate(result_1))
-      self.assertEqual(100, self.evaluate(result_2))
+      self.assertEqual(4.0, sess.run(result_1))
+      self.assertEqual(100, sess.run(result_2))
       self.assertEqual((4.0, 100), sess.run((result_1, result_2)))
 
   def testStatefulFunction(self):
@@ -1052,8 +1052,8 @@ class FunctionTest(test.TestCase):
     for config in _OptimizerOptions():
       config.device_count["CPU"] = 2
       with session.Session(config=config) as sess:
-        self.assertEqual(42.0, self.evaluate(f_0))
-        self.assertEqual(44.0, self.evaluate(f_1))
+        self.assertEqual(42.0, sess.run(f_0))
+        self.assertEqual(44.0, sess.run(f_1))
         self.assertEqual((42.0, 44.0), sess.run((f_0, f_1)))
 
   def testGuaranteedConstsAreCaptured(self):
@@ -1076,7 +1076,7 @@ class FunctionTest(test.TestCase):
       return output
 
     with self.session(use_gpu=False) as sess:
-      self.evaluate(var.initializer)
+      sess.run(var.initializer)
       _ = sess.run(CapturesGuaranteedConst(), {also_not_const: 1.0})
 
   def testSameFunctionDifferentGrads(self):
@@ -1651,8 +1651,8 @@ class ModuleFunctionTest(test.TestCase):
       y = LinearWithCApi(a, b, c)
       z = Linear2WithCApi(a, b, c, d, e)
       with session.Session() as sess:
-        self.assertAllEqual([[1]], self.evaluate(y))
-        self.assertAllEqual([[5]], self.evaluate(z))
+        self.assertAllEqual([[1]], sess.run(y))
+        self.assertAllEqual([[5]], sess.run(z))
 
 
 class VariableHoistingTest(test.TestCase):
@@ -1704,7 +1704,7 @@ class VariableHoistingTest(test.TestCase):
     self.assertEqual("Foo/b", b.op.name)
 
     with self.session(graph=g) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       w, b, x, y0, loss, dw, db = sess.run([w, b, x, y0, loss, dw, db])
 
     self.assertAllEqual(w.shape, (64, 64))
diff --git a/tensorflow/python/framework/graph_util_test.py b/tensorflow/python/framework/graph_util_test.py
index 7a9f2e8d86..563a177dd0 100644
--- a/tensorflow/python/framework/graph_util_test.py
+++ b/tensorflow/python/framework/graph_util_test.py
@@ -211,7 +211,7 @@ class DeviceFunctionsTest(test.TestCase):
       with session.Session() as sess:
         init = variables.variables_initializer([variable_node])
         sess.run(init)
-        output = self.evaluate(output_node)
+        output = sess.run(output_node)
         self.assertNear(4.0, output, 0.00001)
         variable_graph_def = sess.graph.as_graph_def()
 
@@ -242,8 +242,8 @@ class DeviceFunctionsTest(test.TestCase):
         output_node = math_ops_lib.multiply(
             variable_node, 2.0, name="output_node")
         with session.Session() as sess:
-          self.evaluate(variable_node.initializer)
-          output = self.evaluate(output_node)
+          sess.run(variable_node.initializer)
+          output = sess.run(output_node)
           self.assertNear(2.0, output, 0.00001)
           variable_graph_def = sess.graph.as_graph_def()
           # First get the constant_graph_def when variable_names_whitelist is
@@ -256,7 +256,7 @@ class DeviceFunctionsTest(test.TestCase):
 
           # Then initialize the unused variable, and get another
           # constant_graph_def when variable_names_whitelist is not set.
-          self.evaluate(another_variable.initializer)
+          sess.run(another_variable.initializer)
           constant_graph_def_without_variable_whitelist = (
               graph_util.convert_variables_to_constants(
                   sess, variable_graph_def, ["output_node"]))
@@ -295,7 +295,7 @@ class DeviceFunctionsTest(test.TestCase):
             ["Variable", "VariableV2", "VarHandleOp", "ReadVariableOp"])
       with session.Session() as sess:
         output_node = sess.graph.get_tensor_by_name("output_node:0")
-        output = self.evaluate(output_node)
+        output = sess.run(output_node)
         self.assertNear(2.0, output, 0.00001)
 
   def create_node_def(self, op, name, inputs):
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index a57f0b3654..fc7367649e 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -398,10 +398,10 @@ class ImportGraphDefTest(test.TestCase):
       # TODO(b/76173421): make this work (currently DCHECKS)
       # with self.cached_session() as sess:
       #   sess.run(imported_init)
-      #   self.assertEqual(self.evaluate(imported_var), 1.0)
-      #   self.assertEqual(self.evaluate(imported_assign), 2.0)
-      #   self.assertEqual(list(self.evaluate(imported_shape)), [])
-      #   self.assertEqual(list(self.evaluate(new_var_shape)), [])
+      #   self.assertEqual(sess.run(imported_var), 1.0)
+      #   self.assertEqual(sess.run(imported_assign), 2.0)
+      #   self.assertEqual(list(sess.run(imported_shape)), [])
+      #   self.assertEqual(list(sess.run(new_var_shape)), [])
 
   def testWhileLoop(self):
     # Produce GraphDef containing while loop.
@@ -418,7 +418,7 @@ class ImportGraphDefTest(test.TestCase):
                                               return_elements=[r.name])
       self.assertEqual(imported_r.name, "import/" + r.name)
       with self.cached_session() as sess:
-        self.assertEqual(self.evaluate(imported_r), 10)
+        self.assertEqual(sess.run(imported_r), 10)
 
   def testImportWhileLoopInCond(self):
     # Produce GraphDef containing while loop.
@@ -458,7 +458,7 @@ class ImportGraphDefTest(test.TestCase):
           lambda i: i < 2, ImportFn, [0],
           shape_invariants=[tensor_shape.TensorShape(None)])
       with self.cached_session() as sess:
-        self.assertEqual(self.evaluate(out), 10)
+        self.assertEqual(sess.run(out), 10)
 
   def testTypeMismatchInGraphDef(self):
     # TODO(skyewm): improve error message
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 559a76bebb..84e7f361bb 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -492,8 +492,8 @@ class ScopedMetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
       grad = gradients_impl.gradients([output], [var])
       with session.Session() as sess:
-        self.evaluate(init_op)
-        expected_grad_value = self.evaluate(grad)
+        sess.run(init_op)
+        expected_grad_value = sess.run(grad)
 
     # Restore the MetaGraphDef into a new Graph with an import scope.
     with ops.Graph().as_default():
@@ -518,8 +518,8 @@ class ScopedMetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with session.Session() as sess:
-        self.evaluate(init_op)
-        actual_grad_value = self.evaluate(grad)
+        sess.run(init_op)
+        actual_grad_value = sess.run(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
   def testImportWhileLoopInWhileLoop(self):
@@ -544,7 +544,7 @@ class ScopedMetaGraphTest(test.TestCase):
       _, x = control_flow_ops.while_loop(lambda i, x: i < 2, body, [0, 0.0],
                                          name="")
       with session.Session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         sess.run(x)
 
   def testScopedImportUnderNameScope(self):
@@ -869,7 +869,7 @@ class MetaGraphWithVariableScopeTest(test.TestCase):
 
       initializer = variables.local_variables_initializer()
       sess.run(initializer)
-      self.evaluate(update_op)
+      sess.run(update_op)
 
     meta_graph.export_scoped_meta_graph(
         filename=meta_graph_filename, graph=graph)
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index b9c690849d..3957d1de53 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -517,21 +517,21 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEquals(x.consumers(), [])
     self.assertEquals(y.consumers(), [z.op, z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(self.evaluate(z), 4)
+      self.assertEquals(sess.run(z), 4)
 
     z.op._update_input(0, x)  # pylint: disable=protected-access
     self.assertEquals(list(z.op.inputs), [x, y])
     self.assertEquals(x.consumers(), [z.op])
     self.assertEquals(y.consumers(), [z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(self.evaluate(z), 3)
+      self.assertEquals(sess.run(z), 3)
 
     z.op._update_input(1, y)  # pylint: disable=protected-access
     self.assertEquals(list(z.op.inputs), [x, y])
     self.assertEquals(x.consumers(), [z.op])
     self.assertEquals(y.consumers(), [z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(self.evaluate(z), 3)
+      self.assertEquals(sess.run(z), 3)
 
   def testUpdateInputGraphError(self):
     g_0 = ops.Graph()
diff --git a/tensorflow/python/framework/smart_cond_test.py b/tensorflow/python/framework/smart_cond_test.py
index 174ada9fe1..b8a9672b06 100644
--- a/tensorflow/python/framework/smart_cond_test.py
+++ b/tensorflow/python/framework/smart_cond_test.py
@@ -109,8 +109,8 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
                               exclusive=True)
     with session.Session() as sess:
       # No feed_dict necessary
-      self.assertEqual(self.evaluate(y), 1)
-      self.assertEqual(self.evaluate(z), 1)
+      self.assertEqual(sess.run(y), 1)
+      self.assertEqual(sess.run(z), 1)
 
   def testFalse(self):
     conditions = [(False, raise_exception)]
@@ -121,8 +121,8 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
                               default=lambda: constant_op.constant(1),
                               exclusive=True)
     with session.Session() as sess:
-      self.assertEqual(self.evaluate(y), 1)
-      self.assertEqual(self.evaluate(z), 1)
+      self.assertEqual(sess.run(y), 1)
+      self.assertEqual(sess.run(z), 1)
 
   def testMix(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
diff --git a/tensorflow/python/framework/sparse_tensor_test.py b/tensorflow/python/framework/sparse_tensor_test.py
index 9ee1bd75a5..2f7591abbd 100644
--- a/tensorflow/python/framework/sparse_tensor_test.py
+++ b/tensorflow/python/framework/sparse_tensor_test.py
@@ -50,7 +50,7 @@ class SparseTensorTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(indices, value.indices)
         self.assertAllEqual(values, value.values)
         self.assertAllEqual(shape, value.dense_shape)
-        sess_run_value = self.evaluate(sp)
+        sess_run_value = sess.run(sp)
         self.assertAllEqual(sess_run_value.indices, value.indices)
         self.assertAllEqual(sess_run_value.values, value.values)
         self.assertAllEqual(sess_run_value.dense_shape, value.dense_shape)
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index 87d65c8c46..bdf759f220 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -771,7 +771,7 @@ class TensorUtilTest(test.TestCase):
     with self.cached_session() as sess:
       ma = MockArray(np.array([10, 20, 30]))
       t = ops.convert_to_tensor(ma)
-      a = self.evaluate(t)
+      a = sess.run(t)
       self.assertEquals(np.int64, a.dtype)
       self.assertAllClose(np.array([10, 20, 30], dtype=np.int64), a)
 
diff --git a/tensorflow/python/grappler/constant_folding_test.py b/tensorflow/python/grappler/constant_folding_test.py
index 30c1e14681..ab1d0ed25b 100644
--- a/tensorflow/python/grappler/constant_folding_test.py
+++ b/tensorflow/python/grappler/constant_folding_test.py
@@ -61,7 +61,7 @@ class ConstantFoldingTest(test.TestCase):
           back_prop=False,
           parallel_iterations=1)
       with session.Session() as sess:
-        y_v = self.evaluate(y)
+        y_v = sess.run(y)
         self.assertAllEqual(np.zeros([10, 20, 30]), y_v)
 
 
diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 55ccfbb93c..7b68d5e80d 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -241,7 +241,7 @@ class LayoutOptimizerTest(test.TestCase):
       if restore:
         saver.restore(sess, checkpoint_path)
       else:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
 
       np.random.seed(0)
       for _ in range(2):
@@ -262,7 +262,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _two_layer_model(x)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -365,7 +365,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(pad)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -396,7 +396,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -425,7 +425,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(cast)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -456,7 +456,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -486,7 +486,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -516,7 +516,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -545,7 +545,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -574,7 +574,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -603,7 +603,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -632,7 +632,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -662,7 +662,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -691,7 +691,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -724,7 +724,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(concat)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -835,7 +835,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reverse)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -905,7 +905,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(select)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -966,7 +966,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(select)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1179,7 +1179,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(s)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1214,7 +1214,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(s)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1347,7 +1347,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1374,7 +1374,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop_with_branch()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1398,7 +1398,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop_with_vec_and_4d()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1422,7 +1422,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _model_with_second_port()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = self.evaluate(output)
+        output_val_ref = sess.run(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py
index d233629cbb..98cbb1a4b6 100644
--- a/tensorflow/python/grappler/memory_optimizer_test.py
+++ b/tensorflow/python/grappler/memory_optimizer_test.py
@@ -231,10 +231,10 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
       train_op = graph.get_operation_by_name(train_op_name)
       loss_op = graph.get_tensor_by_name(loss_op_name)
       with session.Session(config=config, graph=graph) as sess:
-        self.evaluate(init_op)
-        self.evaluate(train_op)
-        self.evaluate(train_op)
-        return self.evaluate(loss_op)
+        sess.run(init_op)
+        sess.run(train_op)
+        sess.run(train_op)
+        return sess.run(loss_op)
 
   def testRecomputationRewritingNoErrors(self):
     """Tests that graph output is not significantly different with rewriting."""
@@ -295,8 +295,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
           rewrite_options=manual_memory_config)
       session_config = config_pb2.ConfigProto(graph_options=graph_options)
       with session.Session(config=session_config) as sess:
-        self.evaluate(init_op)
-        self.evaluate(train_op)
+        sess.run(init_op)
+        sess.run(train_op)
 
   def testHintDoesRewrite(self):
     graph = self._annotated_graph()[0]
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 48fdd56e9f..a727e99f66 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -136,7 +136,7 @@ class BackendUtilsTest(test.TestCase):
       x = keras.Input((3,))
       y = keras.layers.BatchNormalization()(x)
       if not context.executing_eagerly():
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
   def test_learning_phase_scope(self):
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index b1449069e3..7172571f7c 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -1013,8 +1013,8 @@ class RNNTest(test.TestCase):
         inputs, _ = cell(inputs, initial_state)
         output = inputs
         if not context.executing_eagerly():
-          self.evaluate(variables_lib.global_variables_initializer())
-          output = self.evaluate(output)
+          sess.run(variables_lib.global_variables_initializer())
+          output = sess.run(output)
         return output
 
     random_seed.set_random_seed(12345)
diff --git a/tensorflow/python/kernel_tests/accumulate_n_test.py b/tensorflow/python/kernel_tests/accumulate_n_test.py
index c7f11f854d..ae24cf8f14 100644
--- a/tensorflow/python/kernel_tests/accumulate_n_test.py
+++ b/tensorflow/python/kernel_tests/accumulate_n_test.py
@@ -65,7 +65,7 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase):
             for _ in range(0, num_inputs)
         ]
         accum_n = math_ops.accumulate_n(input_vars)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         accum_n_grad = gradients.gradients(accum_n, input_vars)
         self.assertAllEqual(
             np.repeat(1.0, num_inputs),  # d/dx (x + y + ...) = 1
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index afc158f697..d345138ec7 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -833,7 +833,7 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase):
       index = constant_op.constant(1, dtype=dtypes.int64)
       b = 2. * a[index]
       grad, = gradients_impl.gradients(b, a)
-      self.assertAllEqual(self.evaluate(grad), [0., 2., 0.])
+      self.assertAllEqual(sess.run(grad), [0., 2., 0.])
 
 
 class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
@@ -846,7 +846,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
               math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
               shape=(4, 1, 1)))
       varshape = variables.Variable([6, 4, 4], dtype=dtypes.int32)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0])
       end = constant_op.constant([4, 1, 1])
       strides = constant_op.constant([1, 1, 1])
@@ -859,7 +859,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
           math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
           shape=(4, 1, 1))
       original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0], dtype=dtypes.int64)
       end = constant_op.constant([4, 1, 1], dtype=dtypes.int64)
       strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64)
@@ -873,7 +873,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
           math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
           shape=(4, 1, 1))
       original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0], dtype=dtypes.int32)
       end = constant_op.constant([4, 1, 1], dtype=dtypes.int64)
       strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64)
@@ -1042,7 +1042,7 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     too_large_val = constant_op.constant([3, 4], dtype=dtypes.int64)
     v = resource_variable_ops.ResourceVariable(init_val)
     with self.cached_session() as sess:
-      self.evaluate(v.initializer)
+      sess.run(v.initializer)
       with self.assertRaises(ValueError):
         sess.run(v[:].assign(too_large_val))
       with self.assertRaises(ValueError):
@@ -1269,7 +1269,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
             initializer=init_ops.constant_initializer(10.0),
             use_resource=use_resource)
         guarantee_a = array_ops.guarantee_const(a)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertEqual(10.0, guarantee_a.eval())
 
   def testResourceRejection(self):
@@ -1279,7 +1279,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
           initializer=init_ops.constant_initializer(10.0),
           use_resource=True)
       guarantee_a = array_ops.guarantee_const(a.handle)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError,
                                                "cannot be a resource variable"):
         guarantee_a.eval()
diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py
index cd33048121..ac5cbc810a 100644
--- a/tensorflow/python/kernel_tests/basic_gpu_test.py
+++ b/tensorflow/python/kernel_tests/basic_gpu_test.py
@@ -44,13 +44,13 @@ class GPUBinaryOpsTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_gpu = self.evaluate(out)
+      tf_gpu = sess.run(out)
 
     with self.cached_session(use_gpu=False) as sess:
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_cpu = self.evaluate(out)
+      tf_cpu = sess.run(out)
 
     self.assertAllClose(tf_cpu, tf_gpu)
 
@@ -96,7 +96,7 @@ class MathBuiltinUnaryTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       inx = ops.convert_to_tensor(x)
       ofunc = tf_func(inx)
-      tf_out = self.evaluate(ofunc)
+      tf_out = sess.run(ofunc)
     self.assertAllClose(np_out, tf_out)
 
   def _inv(self, x):
@@ -148,7 +148,7 @@ class MathBuiltinUnaryTest(test.TestCase):
       iny = ops.convert_to_tensor(y + 0.1)
       ofunc = inx / iny
       out_func2 = math_ops.floor(ofunc)
-      tf_out = self.evaluate(out_func2)
+      tf_out = sess.run(out_func2)
 
     self.assertAllClose(np_out, tf_out)
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index adfb094971..12afb6a2ad 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -98,8 +98,8 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
           quantile_accumulator_handle, num_features=2)
       quantiles = boosted_trees_ops.boosted_trees_bucketize(
           [self._feature_0, self._feature_1], buckets)
-      self.evaluate(summary_op)
-      self.evaluate(flush_op)
+      sess.run(summary_op)
+      sess.run(flush_op)
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
index e4c5431c26..cc3984015d 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
@@ -65,10 +65,10 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
       self.assertAllClose([[0.004775, 0.41184], [0.02823, 0.41184]],
                           sess.run(gains_list))
-      self.assertAllEqual([[1, 1], [1, 1]], self.evaluate(thresholds_list))
+      self.assertAllEqual([[1, 1], [1, 1]], sess.run(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[-.416667], [.568966]], [[-.6], [-.75]]],
@@ -113,10 +113,10 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
       self.assertAllClose([[0., 0.33931375], [0.01879096, 0.33931375]],
                           sess.run(gains_list))
-      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
@@ -162,9 +162,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
 
-      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
       self.assertAllClose([[[0.0], [0.3965517]], [[-0.4], [-0.5]]],
                           sess.run(left_node_contribs_list))
 
@@ -214,12 +214,12 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
 
       self.assertAllClose([[-3., -2.66068625], [-2.98120904, -2.66068625]],
                           sess.run(gains_list))
 
-      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
@@ -266,9 +266,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
 
       # We can't split node 1 on feature 1 and node 2 on feature 2 because of
       # the min node weight.
-      self.assertAllEqual([[2], [1]], self.evaluate(node_ids_list))
-      self.assertAllClose([[0.384314], [0.098013]], self.evaluate(gains_list))
-      self.assertAllEqual([[1], [1]], self.evaluate(thresholds_list))
+      self.assertAllEqual([[2], [1]], sess.run(node_ids_list))
+      self.assertAllClose([[0.384314], [0.098013]], sess.run(gains_list))
+      self.assertAllEqual([[1], [1]], sess.run(thresholds_list))
       self.assertAllClose([[[0.4852941]], [[-.6]]],
                           sess.run(left_node_contribs_list))
       self.assertAllClose([[[-0.75]], [[-0.014925]]],
@@ -311,9 +311,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            max_splits=max_splits)
 
       # We can't split either of the nodes on the first feature
-      self.assertEqual(2, len(self.evaluate(node_ids_list)))
-      self.assertAllEqual([], self.evaluate(node_ids_list)[0])
-      self.assertAllEqual([1], self.evaluate(node_ids_list)[1])
+      self.assertEqual(2, len(sess.run(node_ids_list)))
+      self.assertAllEqual([], sess.run(node_ids_list)[0])
+      self.assertAllEqual([1], sess.run(node_ids_list)[1])
 
       # Now check when we can't split on any feature
       (node_ids_list, _, _, _,
@@ -325,7 +325,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            tree_complexity=0.0,
            min_node_weight=10,
            max_splits=max_splits)
-      self.assertAllEqual([[], []], self.evaluate(node_ids_list))
+      self.assertAllEqual([[], []], sess.run(node_ids_list))
 
   def testMakeStatsSummarySimple(self):
     """Simple test for MakeStatsSummary."""
diff --git a/tensorflow/python/kernel_tests/bucketize_op_test.py b/tensorflow/python/kernel_tests/bucketize_op_test.py
index 9575b28899..57413e6af5 100644
--- a/tensorflow/python/kernel_tests/bucketize_op_test.py
+++ b/tensorflow/python/kernel_tests/bucketize_op_test.py
@@ -32,7 +32,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0, 3, 8, 11])
     expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, self.evaluate(op))
+      self.assertAllEqual(expected_out, sess.run(op))
 
   def testFloat(self):
     op = math_ops._bucketize(
@@ -40,7 +40,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0., 3., 8., 11.])
     expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, self.evaluate(op))
+      self.assertAllEqual(expected_out, sess.run(op))
 
   def test2DInput(self):
     op = math_ops._bucketize(
@@ -48,7 +48,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0, 3, 8, 11])
     expected_out = [[0, 1, 1, 2, 2], [3, 3, 4, 4, 1]]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, self.evaluate(op))
+      self.assertAllEqual(expected_out, sess.run(op))
 
   def testInvalidBoundariesOrder(self):
     op = math_ops._bucketize(
diff --git a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
index 031accee55..46ab71537f 100644
--- a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
+++ b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
@@ -97,7 +97,7 @@ class RangeSamplerOpsTest(test.TestCase):
           true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True)
       accidental_hits = candidate_sampling_ops.compute_accidental_hits(
           true_classes, sampled_candidates, self.NUM_TRUE)
-      indices, ids, weights = self.evaluate(accidental_hits)
+      indices, ids, weights = sess.run(accidental_hits)
 
     self.assertEqual(1, accidental_hits[0].get_shape().ndims)
     self.assertEqual(1, accidental_hits[1].get_shape().ndims)
diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py
index cdeaf7b696..bc49cd5a04 100644
--- a/tensorflow/python/kernel_tests/cast_op_test.py
+++ b/tensorflow/python/kernel_tests/cast_op_test.py
@@ -187,7 +187,7 @@ class CastOpTest(test.TestCase):
       y = variables.Variable(True, dtype=dtypes.bool)
       cast = math_ops.cast(y, x.dtype)
       variables.global_variables_initializer().run()
-      self.assertEqual(1.0, self.evaluate(cast))
+      self.assertEqual(1.0, sess.run(cast))
 
   def testGradients(self):
     t = [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128]
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 6944d73c5f..149302831b 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -627,7 +627,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([2, 7, 5], dtypes.int32)
       s2 = constant_op.constant([2, 20, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = self.evaluate(off)
+      ans = sess.run(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
   def testNotVector(self):
@@ -679,7 +679,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([2, 7, 5], dtypes.int32)
       s2 = constant_op.constant([2, 20, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = self.evaluate(off)
+      ans = sess.run(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
       cdim = constant_op.constant(-3, dtypes.int32)
@@ -687,7 +687,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([1, 3, 5], dtypes.int32)
       s2 = constant_op.constant([3, 3, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = self.evaluate(off)
+      ans = sess.run(off)
       self.assertAllEqual(ans, [[0, 0, 0], [2, 0, 0], [3, 0, 0]])
 
 
diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
index 8388070c63..893cb7cce3 100644
--- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
@@ -424,7 +424,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_grad(1)
 
       def apply_grad(accum_op):
-        self.evaluate(accum_op)
+        sess.run(accum_op)
 
       threads = [
           self.checkedThread(
@@ -451,7 +451,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       def apply_grad():
         for accum_op in accum_ops:
           time.sleep(1.0)
-          self.evaluate(accum_op)
+          sess.run(accum_op)
 
       apply_grad_thread = self.checkedThread(target=apply_grad)
 
@@ -485,7 +485,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       def apply_grad():
         time.sleep(1.0)
         for accum_op in accum_ops:
-          self.evaluate(accum_op)
+          sess.run(accum_op)
 
       return_array = []
 
@@ -503,7 +503,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
   def _blocking_takeg(self, sess, takeg_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(takeg_op)
+      sess.run(takeg_op)
 
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 59966ebc84..9a198d445f 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -593,7 +593,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: [math_ops.add(x, 1), math_ops.add(x, 2)]
       fn2 = lambda: [y, y]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      self.assertAllEqual([11, 12], self.evaluate(r))
+      self.assertAllEqual([11, 12], sess.run(r))
 
   def testCondListOutput(self):
     with self.cached_session() as sess:
@@ -603,7 +603,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: [math_ops.add(x, y), math_ops.add(x, y)]
       fn2 = lambda: [y, y]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = self.evaluate(r)
+      test_result = sess.run(r)
       self.assertListEqual([210, 210], test_result)
 
   def testTupleOutput(self):
@@ -614,7 +614,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: (math_ops.add(x, y), math_ops.add(x, y))
       fn2 = lambda: (y, y)
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = self.evaluate(r)
+      test_result = sess.run(r)
       self.assertTupleEqual((210, 210), test_result)
 
   def testDictOutput(self):
@@ -625,7 +625,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
       fn2 = lambda: {"a": y, "b": y}
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = self.evaluate(r)
+      test_result = sess.run(r)
       self.assertDictEqual({"a": 210, "b": 210}, test_result)
 
   def testEmbeddedListOutput(self):
@@ -638,7 +638,7 @@ class ControlFlowTest(test.TestCase):
       # Pass strict=True flag as cond_v2 allows for tensors to be
       # in nested output structures as singletons
       r = control_flow_ops.cond(pred, fn1, fn2, strict=True)
-      test_result = self.evaluate(r)
+      test_result = sess.run(r)
       self.assertListEqual([[210, 210]], test_result)
 
   def testEmbeddedTupleOutput(self):
@@ -649,7 +649,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: ((math_ops.add(x, y), math_ops.add(x, y)))
       fn2 = lambda: ((y, y))
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = self.evaluate(r)
+      test_result = sess.run(r)
       self.assertTupleEqual(((210, 210)), test_result)
 
   def testEmbeddedDictOutput(self):
@@ -662,7 +662,7 @@ class ControlFlowTest(test.TestCase):
       fn2 = lambda: {"a": {"c": y},
                      "b": {"d": y}}
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = self.evaluate(r)
+      test_result = sess.run(r)
       self.assertDictEqual({"a": {"c": 210}, "b": {"d": 210}}, test_result)
 
   def testCheckNestedOutputStruct(self):
@@ -677,7 +677,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError, v2_msg if control_flow_ops.ENABLE_COND_V2 else v1_msg):
         r = control_flow_ops.cond(pred, fn1, fn2)
-        self.evaluate(r)
+        test_result = sess.run(r)
 
   def testCondRef(self):
 
@@ -1050,7 +1050,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(r[0].dtype, dtypes.int32)
       self.assertEqual(r[1].dtype, dtypes.int32_ref)
 
-      value_i, value_x = self.evaluate(r)
+      value_i, value_x = sess.run(r)
 
     self.assertEqual(100, value_i)
     self.assertEqual(0, value_x)
@@ -1642,7 +1642,7 @@ class ControlFlowTest(test.TestCase):
       with ops.control_dependencies([control_flow_ops.no_op()]):
         loop = control_flow_ops.while_loop(cond, body,
                                            (constant_op.constant(5),))
-      self.assertEqual(0, self.evaluate(loop))
+      self.assertEqual(0, sess.run(loop))
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
   def testWhileCondWithControl_1(self):
@@ -2055,7 +2055,7 @@ class ControlFlowTest(test.TestCase):
         self.assertFalse(gpu_dev_name in dev)
 
     with self.session(graph=graph) as sess:
-      self.assertAllClose(1024.0, self.evaluate(r))
+      self.assertAllClose(1024.0, sess.run(r))
 
   @test_util.disable_control_flow_v2("b/116351701 (colocation)")
   def testWhileGrad_ColocateGradients(self):
@@ -2547,8 +2547,8 @@ class ControlFlowTest(test.TestCase):
       res = outer_loop(inp)
       optimizer = adam.AdamOptimizer(learning_rate=0.001)
       train_op = optimizer.minimize(math_ops.reduce_mean(math_ops.square(res)))
-      self.evaluate(variables.global_variables_initializer())
-      self.evaluate(train_op)
+      sess.run(variables.global_variables_initializer())
+      sess.run(train_op)
       self.assertAllClose(2.999, self.evaluate(var))
 
   def _testWhileCondGrad_Simple(self, use_gpu):
@@ -2607,11 +2607,11 @@ class ControlFlowTest(test.TestCase):
           [i0.get_shape(), tensor_shape.TensorShape([None, 2])])
       s = math_ops.reduce_sum(h)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       optimizer = gradient_descent.GradientDescentOptimizer(0.01)
       op = optimizer.minimize(s)
       sess.run(op)
-      self.assertAllClose([[0.98000002, 1.98000002]], self.evaluate(x))
+      self.assertAllClose([[0.98000002, 1.98000002]], sess.run(x))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileWithRefsWithGradients_1(self):
@@ -2705,7 +2705,7 @@ class ControlFlowTest(test.TestCase):
 
       output_grad = control_flow_ops.while_loop(
           c, b, [i0, constant_op.constant(0.0)])
-      self.assertAllClose(600.0, self.evaluate(output_grad)[1])
+      self.assertAllClose(600.0, sess.run(output_grad)[1])
 
   def testWhileAndTensorArray(self):
     with self.cached_session() as sess:
@@ -2724,7 +2724,7 @@ class ControlFlowTest(test.TestCase):
 
       r = control_flow_ops.while_loop(c, b, [n0, y0], parallel_iterations=1)
       r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(107520.0, self.evaluate(r))
+      self.assertAllClose(107520.0, sess.run(r))
 
   def testWhileGrad_StopGrad(self):
     with self.cached_session():
@@ -2857,8 +2857,8 @@ class ControlFlowTest(test.TestCase):
     dy_dq, = gradients_impl.gradients(y, q)
     self.assertIsNotNone(dy_dq)
     with self.cached_session() as sess:
-      self.evaluate(q.initializer)
-      self.assertAllClose([0., 0.], self.evaluate(dy_dq))
+      sess.run(q.initializer)
+      self.assertAllClose([0., 0.], sess.run(dy_dq))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileGradientWithNontrainablePath2(self):
@@ -2875,8 +2875,8 @@ class ControlFlowTest(test.TestCase):
     dy_dq, = gradients_impl.gradients(y, q)
     self.assertIsNotNone(dy_dq)
     with self.cached_session() as sess:
-      self.evaluate(q.initializer)
-      self.assertAllClose([1., 1.], self.evaluate(dy_dq))
+      sess.run(q.initializer)
+      self.assertAllClose([1., 1.], sess.run(dy_dq))
 
   @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   def testIssue16504(self):
@@ -3295,7 +3295,7 @@ class ControlFlowTest(test.TestCase):
 
       result = control_flow_ops.while_loop(condition, body,
                                            [constant_op.constant(4)])
-      self.assertEqual(10, self.evaluate(result))
+      self.assertEqual(10, sess.run(result))
 
       # Ensure that we cannot run a tensor that escapes the loop body
       # accidentally.
@@ -3353,7 +3353,7 @@ class ControlFlowTest(test.TestCase):
           shape=[1], dtype=dtypes.qint8, name="v", container="", shared_name="")
       assign_op = state_ops.assign(
           var_qint, constant_op.constant(np.array([42]), dtypes.qint8))
-      self.evaluate(assign_op)
+      sess.run(assign_op)
 
       cond = constant_op.constant(True, dtypes.bool)
       v_f, v_t = control_flow_ops.ref_switch(var_qint, cond)
@@ -3682,7 +3682,7 @@ class WhileOpBenchmark(test.Benchmark):
     with session.Session() as sess, ops.device(default_device):
       # Get the initial id i, input x, and kernel.
       i, x, kernel = self._getInitVariables()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       if static_unroll:
         for _ in xrange(steps):
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index 3ec5c29df7..3924e13575 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -109,7 +109,7 @@ class Conv3DTest(test.TestCase):
         results.append(result)
 
       with self.cached_session() as sess:
-        values = self.evaluate(results)
+        values = sess.run(results)
         for value in values:
           print("expected = ", expected)
           print("actual = ", value)
@@ -184,8 +184,8 @@ class Conv3DTest(test.TestCase):
         computed_results.append(computed)
         tolerance = 1e-2 if use_gpu else 1e-5
         with self.cached_session() as sess:
-          expected_values = self.evaluate(expected_results)
-          computed_values = self.evaluate(computed_results)
+          expected_values = sess.run(expected_results)
+          computed_values = sess.run(computed_results)
           for e_value, c_value in zip(expected_values, computed_values):
             print("expected = ", e_value)
             print("actual = ", c_value)
@@ -715,8 +715,8 @@ class Conv3DTest(test.TestCase):
         expected_grad = gradients_impl.gradients(expected, t1
                                                  if mode == "input" else t2)[0]
         # "values" consists of two tensors for two backprops
-        actual_value = self.evaluate(actual_grad)
-        expected_value = self.evaluate(expected_grad)
+        actual_value = sess.run(actual_grad)
+        expected_value = sess.run(expected_grad)
         self.assertShapeEqual(actual_value, actual_grad)
         self.assertShapeEqual(expected_value, expected_grad)
       print("expected = ", expected_value)
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 2d21f6f4ae..835cc1504d 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -908,8 +908,8 @@ class Conv2DTest(test.TestCase):
         conv = gradients_impl.gradients(conv_forward, t1)[0]
         conv_2 = gradients_impl.gradients(conv_forward_2, t1)[0]
         # "values" consists of two tensors for two backprops
-        value = self.evaluate(conv)
-        value_2 = self.evaluate(conv_2)
+        value = sess.run(conv)
+        value_2 = sess.run(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
       tf_logging.info("expected = ", value_2)
@@ -961,8 +961,8 @@ class Conv2DTest(test.TestCase):
           conv_forward_2 = test_util.NCHWToNHWC(conv_forward_2)
         conv = gradients_impl.gradients(conv_forward, t2)[0]
         conv_2 = gradients_impl.gradients(conv_forward, t2)[0]
-        value = self.evaluate(conv)
-        value_2 = self.evaluate(conv_2)
+        value = sess.run(conv)
+        value_2 = sess.run(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
       tf_logging.info("expected = ", value_2)
@@ -1545,7 +1545,7 @@ class DepthwiseConv2DTest(test.TestCase):
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
       conv = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      value = self.evaluate(conv)
+      value = sess.run(conv)
     tf_logging.info("value = ", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
@@ -1667,7 +1667,7 @@ class SeparableConv2DTest(test.TestCase):
       if data_format == "NCHW":
         conv = array_ops.transpose(conv, [0, 2, 3, 1])
 
-      value = self.evaluate(conv)
+      value = sess.run(conv)
     tf_logging.info("value = ", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 87248bf9c8..d7dbf5ab9a 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -788,7 +788,7 @@ class RoundingTest(test.TestCase):
     y = np.rint(x) if y is None else np.asarray(y)
     with self.cached_session() as sess:
       tf_rint = math_ops.rint(x)
-      np_rint = self.evaluate(tf_rint)
+      np_rint = sess.run(tf_rint)
     self.assertAllEqual(y, np_rint)
     self.assertShapeEqual(y, tf_rint)
 
@@ -881,7 +881,7 @@ class ComplexMakeRealImagTest(test.TestCase):
         force_gpu=use_gpu and test_util.is_gpu_available()) as sess:
       inx = ops.convert_to_tensor(cplx)
       tf_angle = math_ops.angle(inx)
-      tf_angle_val = self.evaluate(tf_angle)
+      tf_angle_val = sess.run(tf_angle)
     self.assertAllEqual(np_angle, tf_angle_val)
     self.assertShapeEqual(np_angle, tf_angle)
 
diff --git a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
index 8c4ccbd88e..66b3e0f22f 100644
--- a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
@@ -80,7 +80,7 @@ class DecodeJpegBenchmark(test.Benchmark):
           initializer=image_ops.encode_jpeg(tiled_image))
 
     with session.Session() as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       images = []
       for _ in xrange(parallelism):
         if crop_window is None:
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
index 0676664685..3ed7dba966 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
@@ -43,7 +43,7 @@ class AssignOpTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       def run_add(add_op):
-        self.evaluate(add_op)
+        sess.run(add_op)
 
       threads = [
           self.checkedThread(
@@ -70,7 +70,7 @@ class AssignOpTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       def run_assign(assign_op):
-        self.evaluate(assign_op)
+        sess.run(assign_op)
 
       threads = [
           self.checkedThread(
@@ -103,7 +103,7 @@ class AssignOpTest(test.TestCase):
       p.initializer.run()
 
       def run_add(add_op):
-        self.evaluate(add_op)
+        sess.run(add_op)
 
       threads = [
           self.checkedThread(
@@ -131,7 +131,7 @@ class AssignOpTest(test.TestCase):
       p.initializer.run()
 
       def run_assign(assign_op):
-        self.evaluate(assign_op)
+        sess.run(assign_op)
 
       threads = [
           self.checkedThread(
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index f6d834c2f8..f65d0be367 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -162,7 +162,7 @@ class DepthwiseConv2DTest(test.TestCase):
         conv_native = array_ops.transpose(conv_native, [0, 2, 3, 1])
 
       try:
-        native_result = self.evaluate(conv_native)
+        native_result = sess.run(conv_native)
       except errors.InvalidArgumentError as e:
         # Grouped convolution kernel is only registered for cuDNN 7. Silently
         # return when we are running on an earlier version or without GPU.
@@ -174,7 +174,7 @@ class DepthwiseConv2DTest(test.TestCase):
 
       conv_interface = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      interface_result = self.evaluate(conv_interface)
+      interface_result = sess.run(conv_interface)
 
     tf_logging.info(
         "data_type: %r, use_gpu: %r, grouped_conv: %r, max diff = %f",
@@ -269,7 +269,7 @@ class DepthwiseConv2DTest(test.TestCase):
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
       conv = nn_ops.depthwise_conv2d_native(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      value = self.evaluate(conv)
+      value = sess.run(conv)
     tf_logging.info("value = %r", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/distributions/categorical_test.py b/tensorflow/python/kernel_tests/distributions/categorical_test.py
index 9c593d2737..f116c54bd1 100644
--- a/tensorflow/python/kernel_tests/distributions/categorical_test.py
+++ b/tensorflow/python/kernel_tests/distributions/categorical_test.py
@@ -287,7 +287,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     }
 
     with self.cached_session() as sess:
-      run_result = self.evaluate(to_run)
+      run_result = sess.run(to_run)
 
     self.assertAllEqual(run_result["cat_prob"].shape,
                         run_result["norm_prob"].shape)
@@ -462,7 +462,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
           b = categorical.Categorical(logits=b_logits)
 
           kl = kullback_leibler.kl_divergence(a, b)
-          kl_val = self.evaluate(kl)
+          kl_val = sess.run(kl)
           # Make sure KL(a||a) is 0
           kl_same = sess.run(kullback_leibler.kl_divergence(a, a))
 
diff --git a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
index 80da39dfde..07da855a01 100644
--- a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
@@ -40,7 +40,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant([0, 0, 2, 3, 2, 1])
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([0, 13], partition_vals[0])
@@ -62,7 +62,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant([0, 0, 2, 3, 2, 1])
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([[0, 1, 2], [3, 4, 5]], partition_vals[0])
@@ -87,7 +87,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual(part1, partition_vals[0])
@@ -109,7 +109,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=num_partitions)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(num_partitions, len(partition_vals))
     for i in range(num_partitions):
@@ -125,7 +125,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([3 + 4j, 7 + 8j], partition_vals[0])
@@ -138,7 +138,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = 3
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual(np.array([], dtype=np.float64).reshape(-1, 4),
@@ -164,7 +164,7 @@ class DynamicPartitionTest(test.TestCase):
             outputs = data_flow_ops.dynamic_partition(
                 data_t, partitions_t, num_partitions=n)
             self.assertEqual(n, len(outputs))
-            outputs_val = self.evaluate(outputs)
+            outputs_val = sess.run(outputs)
             for i, output in enumerate(outputs_val):
               self.assertAllEqual(output, data[partitions == i])
 
@@ -183,7 +183,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([], partition_vals[0])
@@ -199,7 +199,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=3)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(3, len(partition_vals))
     self.assertAllEqual([[]], partition_vals[0])
@@ -215,7 +215,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([], partition_vals[0])
@@ -236,7 +236,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([6], partition_vals[0])
@@ -257,7 +257,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=5)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(5, len(partition_vals))
     self.assertAllEqual([5], partition_vals[0])
@@ -281,7 +281,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=40)
-      partition_vals = self.evaluate(partitions)
+      partition_vals = sess.run(partitions)
 
     self.assertEqual(40, len(partition_vals))
     for i in range(40):
@@ -335,7 +335,7 @@ class DynamicPartitionTest(test.TestCase):
     self.assertEqual(len(inds), x.shape[0])
     partitioned = data_flow_ops.dynamic_partition(x, inds, 16)
     with self.cached_session() as sess:
-      res = self.evaluate(partitioned)
+      res = sess.run(partitioned)
     self.assertEqual(res[-1].shape[0], 192)
 
 
diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py
index c184b93c80..e3742f2e72 100644
--- a/tensorflow/python/kernel_tests/fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/fifo_queue_test.py
@@ -159,7 +159,7 @@ class FIFOQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -240,7 +240,7 @@ class FIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          self.evaluate(enqueue_op)
+          sess.run(enqueue_op)
 
       results = []
 
@@ -269,7 +269,7 @@ class FIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        x_val, y_val = self.evaluate(dequeued_t)
+        x_val, y_val = sess.run(dequeued_t)
         x, y = elems[i]
         self.assertEqual([x], x_val)
         self.assertEqual([y], y_val)
@@ -356,7 +356,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = self.evaluate(dequeued_t)
+        float_val, int_val = sess.run(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -399,17 +399,17 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
@@ -429,13 +429,13 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual([None], dequeued_t[0].get_shape().as_list())
       self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list())
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
@@ -529,7 +529,7 @@ class FIFOQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -596,11 +596,11 @@ class FIFOQueueTest(test.TestCase):
 
       def enqueue():
         for _ in xrange(100):
-          self.evaluate(enqueue_op)
+          sess.run(enqueue_op)
 
       def dequeue():
         for _ in xrange(100):
-          self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0))
+          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))
 
       enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
@@ -632,7 +632,7 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         for i in xrange(250):
-          self.assertEqual(i, self.evaluate(dequeued_t))
+          self.assertEqual(i, sess.run(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -663,7 +663,7 @@ class FIFOQueueTest(test.TestCase):
       dequeuemany_t = q.dequeue_many(count_placeholder)
 
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -701,7 +701,7 @@ class FIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -728,7 +728,7 @@ class FIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -797,7 +797,7 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         for elem in elems:
-          self.assertEqual([elem], self.evaluate(dequeued_t))
+          self.assertEqual([elem], sess.run(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -842,7 +842,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems, self.evaluate(dequeued_t))
+        self.assertAllEqual(elems, sess.run(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -867,7 +867,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -892,8 +892,8 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
-        self.assertAllEqual(elems[3:], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[3:], sess.run(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -913,16 +913,16 @@ class FIFOQueueTest(test.TestCase):
       cleanup_dequeue_t = q.dequeue()
 
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
-        self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[0:3], sess.run(dequeued_t))
         with self.assertRaises(errors_impl.OutOfRangeError):
           sess.run(dequeued_t)
-        self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t))
+        self.assertEqual(elems[3], sess.run(cleanup_dequeue_t))
 
       def close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -1051,7 +1051,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1074,7 +1074,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1103,7 +1103,7 @@ class FIFOQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # Expect the operation to succeed once the dequeue op runs.
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1113,7 +1113,7 @@ class FIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1138,7 +1138,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1148,7 +1148,7 @@ class FIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1266,19 +1266,19 @@ class FIFOQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_op)
+      sess.run(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_many_op)
+      sess.run(dequeue_many_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(enqueue_op)
+      sess.run(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(enqueue_many_op)
+      sess.run(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1331,14 +1331,14 @@ class FIFOQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(self.evaluate(size_op), 5)
+      self.assertEqual(sess.run(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(self.evaluate(size_op), 5)
+      self.assertEqual(sess.run(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1405,7 +1405,7 @@ class FIFOQueueTest(test.TestCase):
       q.enqueue_many(input_tuple).run()
 
       output_tuple_t = q.dequeue_many(32)
-      output_tuple = self.evaluate(output_tuple_t)
+      output_tuple = sess.run(output_tuple_t)
 
       for (input_elem, output_elem) in zip(input_tuple, output_tuple):
         self.assertAllEqual(input_elem, output_elem)
@@ -1507,7 +1507,7 @@ class FIFOQueueDictTest(test.TestCase):
       enqueue_op4 = q.enqueue_many({"f": [40.0, 50.0]})
       dequeue = q.dequeue()
       dequeue_2 = q.dequeue_many(2)
-      self.evaluate(enqueue_op)
+      sess.run(enqueue_op)
       sess.run(enqueue_op2)
       sess.run(enqueue_op3)
       sess.run(enqueue_op4)
@@ -1565,7 +1565,7 @@ class FIFOQueueDictTest(test.TestCase):
       })
       dequeue = q.dequeue()
       dequeue_2 = q.dequeue_many(2)
-      self.evaluate(enqueue_op)
+      sess.run(enqueue_op)
       sess.run(enqueue_op2)
       sess.run(enqueue_op3)
       sess.run(enqueue_op4)
@@ -1613,8 +1613,8 @@ class FIFOQueueWithTimeoutTest(test.TestCase):
                                    "Timed out waiting for notification"):
         sess.run(dequeued_t, options=config_pb2.RunOptions(timeout_in_ms=10))
 
-      self.evaluate(enqueue_op)
-      self.assertEqual(37, self.evaluate(dequeued_t))
+      sess.run(enqueue_op)
+      self.assertEqual(37, sess.run(dequeued_t))
 
 
 class QueueContainerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 0af32b048e..503569f3b1 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -567,8 +567,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:worker/replica:0/task:0/cpu:1")
 
     with session.Session(worker[0].target) as sess:
-      self.evaluate(variables.global_variables_initializer())
-      mul = self.evaluate(remote_op)
+      sess.run(variables.global_variables_initializer())
+      mul = sess.run(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionDirectSession(self):
@@ -591,8 +591,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/cpu:1")
 
     with self.test_session(config=worker_config) as sess:
-      self.evaluate(variables.global_variables_initializer())
-      mul = self.evaluate(remote_op)
+      sess.run(variables.global_variables_initializer())
+      mul = sess.run(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionSameDeviceDirectSession(self):
@@ -610,8 +610,8 @@ class FunctionalOpsTest(test.TestCase):
           args=[a, b], Tout=[dtypes.int32], f=_remote_fn, target="/cpu:0")
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      mul = self.evaluate(remote_op)
+      sess.run(variables.global_variables_initializer())
+      mul = sess.run(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionCPUGPU(self):
@@ -634,8 +634,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/device:GPU:0")[0] + 3.0
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      mul = self.evaluate(remote_op)
+      sess.run(variables.global_variables_initializer())
+      mul = sess.run(remote_op)
       self.assertEqual(mul, 9.0)
 
   def testRemoteFunctionGPUCPU(self):
@@ -658,8 +658,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/cpu:0")[0] + 3.0
 
     with self.cached_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      mul = self.evaluate(remote_op)
+      sess.run(variables.global_variables_initializer())
+      mul = sess.run(remote_op)
       self.assertEqual(mul, 9.0)
 
   def testRemoteFunctionGPUCPUStrings(self):
@@ -677,7 +677,7 @@ class FunctionalOpsTest(test.TestCase):
           args=[a], Tout=[dtypes.string], f=_remote_fn, target="/cpu:0")
 
     with self.cached_session() as sess:
-      ret = self.evaluate(remote_op)
+      ret = sess.run(remote_op)
       self.assertAllEqual(ret, [b"a"])
 
   def testRemoteFunctionCrossProcess(self):
@@ -699,8 +699,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:worker/replica:0/task:1/cpu:0")[0] + 3.0
 
     with session.Session(workers[0].target) as sess:
-      self.evaluate(variables.global_variables_initializer())
-      mul = self.evaluate(remote_op)
+      sess.run(variables.global_variables_initializer())
+      mul = sess.run(remote_op)
       self.assertEqual(mul, 9)
 
   def testIf(self):
@@ -857,11 +857,11 @@ class FunctionalOpsTest(test.TestCase):
           result_binary = functional_ops.While(
               [1.0, 0., 0.],
               function.Defun(*[dtypes.float32] * 3)(TestCond), TestBinary)
-          self.evaluate(variables.global_variables_initializer())
+          sess.run(variables.global_variables_initializer())
           assert len(result_unary) == 2
-          self.assertEqual([10.0, 54.0], self.evaluate(result_unary))
+          self.assertEqual([10.0, 54.0], sess.run(result_unary))
           assert len(result_binary) == 3
-          self.assertEqual([10.0, 54.0, 9.0], self.evaluate(result_binary))
+          self.assertEqual([10.0, 54.0, 9.0], sess.run(result_binary))
 
           def TestCondCapture(n, *args):
             del args
@@ -892,7 +892,7 @@ class FunctionalOpsTest(test.TestCase):
                 100, 0, -1, [0.], Body, rewrite_with_while=rewrite_with_while)
             [0],
         ]
-        xvals = self.evaluate(xs)
+        xvals = sess.run(xs)
       self.assertAllEqual(210, xvals[0])
       self.assertAllEqual(5050, xvals[1])
 
@@ -949,16 +949,16 @@ class FunctionalOpsTest(test.TestCase):
         result_binary = functional_ops.For(
             1, 10, 1, [0., 0.], TestBinary,
             rewrite_with_while=rewrite_with_while)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         assert not result_nullary
         # The nullary variant doesn't return anything so we can't easily run it.
         # As a total hack, fetch the operation by name and run it.
         sess.run(ops.get_default_graph().get_operation_by_name(
             "While" if rewrite_with_while else "For"))
         assert len(result_unary) == 1
-        self.assertEqual([54.0], self.evaluate(result_unary))
+        self.assertEqual([54.0], sess.run(result_unary))
         assert len(result_binary) == 2
-        self.assertEqual([54.0, 9.0], self.evaluate(result_binary))
+        self.assertEqual([54.0, 9.0], sess.run(result_binary))
 
   def _tfMLP(self, xval, wsval, bsval, rewrite_with_while):
     # On GPU, don't rewrite using a while loop.
@@ -1041,8 +1041,8 @@ class FunctionalOpsTest(test.TestCase):
       avals = [Poly(a), Grad(a)]
       b = constant_op.constant(1.)
       bvals = [Poly(b), Grad(b)]
-      self.assertAllEqual(self.evaluate(avals), [8., 4.])
-      self.assertAllEqual(self.evaluate(bvals), [17., 16.])
+      self.assertAllEqual(sess.run(avals), [8., 4.])
+      self.assertAllEqual(sess.run(bvals), [17., 16.])
 
 
 # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the
@@ -1193,7 +1193,7 @@ class PartitionedCallTest(test.TestCase):
             allow_soft_placement=False,
             log_device_placement=True,
             device_count={"CPU": 2})) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       expected = sess.run(sum_gather())
       result = sess.run(
           functional_ops.partitioned_call(
diff --git a/tensorflow/python/kernel_tests/gradient_correctness_test.py b/tensorflow/python/kernel_tests/gradient_correctness_test.py
index 12b8a4c8e3..291a69ebac 100644
--- a/tensorflow/python/kernel_tests/gradient_correctness_test.py
+++ b/tensorflow/python/kernel_tests/gradient_correctness_test.py
@@ -35,7 +35,7 @@ class GradientCorrectnessTest(test.TestCase):
       yexp = math_ops.exp(x)
       yexplog = math_ops.log(yexp)
       grads = gradients_impl.gradients([yexp, yexplog], [x])
-      grad_vals = self.evaluate(grads)
+      grad_vals = sess.run(grads)
       exp1_plus_one = (1.0 + np.exp(1.0)).astype(np.float32)
       # [dexp(x)/dx + d(log(exp(x)))/dx] @ x=1 == exp(1) + 1
       self.assertAllClose(grad_vals[0], exp1_plus_one)
@@ -44,13 +44,13 @@ class GradientCorrectnessTest(test.TestCase):
     x = constant_op.constant(3.)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
-      self.assertAllClose(1., self.evaluate(dx_dx))
+      self.assertAllClose(1., sess.run(dx_dx))
 
   def testIntegerIdentityGradient(self):
     x = constant_op.constant(3)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
-      self.assertAllClose(1, self.evaluate(dx_dx))
+      self.assertAllClose(1, sess.run(dx_dx))
 
   def testGradientWithIntegerPath(self):
     x = constant_op.constant([3.9, 4.1])
@@ -58,7 +58,7 @@ class GradientCorrectnessTest(test.TestCase):
     y = x * k
     dy_dx, = gradients_impl.gradients(y, x)
     with self.cached_session() as sess:
-      self.assertAllClose([3., 4.], self.evaluate(dy_dx))
+      self.assertAllClose([3., 4.], sess.run(dy_dx))
 
   def testNoIntegerGradient1(self):
     x = constant_op.constant([3.9, 4.1])
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 074985dd93..a3f2c0ddd7 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -709,7 +709,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
           t = self.evaluate(outputs)
           self.assertAllEqual(t.shape, outputs_shape)
           # Check isometry of the delta-orthogonal kernel.
-          self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
+          self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
   def testNonuniformity(self):
     value = 0
@@ -847,7 +847,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
@@ -942,7 +942,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
@@ -1067,7 +1067,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
 
 
 class IdentityInitializerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index ab4c9c730b..3efad4ea11 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -174,7 +174,7 @@ class HashTableOpTest(test.TestCase):
           constant_op.constant(sp_shape, dtypes.int64))
       output = table.lookup(input_tensor)
 
-      out_indices, out_values, out_shape = self.evaluate(output)
+      out_indices, out_values, out_shape = sess.run(output)
 
       self.assertAllEqual([0, 1, -1], out_values)
       self.assertAllEqual(sp_indices, out_indices)
diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py
index bda63bcaa9..d3a907852a 100644
--- a/tensorflow/python/kernel_tests/losses_test.py
+++ b/tensorflow/python/kernel_tests/losses_test.py
@@ -1046,9 +1046,9 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with self.cached_session() as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         for grad, _ in gradients_to_variables:
-          np_grad = self.evaluate(grad)
+          np_grad = sess.run(grad)
           self.assertFalse(np.isnan(np_grad).any())
 
   def testNonZeroLossWithPythonScalarWeight(self):
diff --git a/tensorflow/python/kernel_tests/map_stage_op_test.py b/tensorflow/python/kernel_tests/map_stage_op_test.py
index 4b5bd4059f..d503f3d7c9 100644
--- a/tensorflow/python/kernel_tests/map_stage_op_test.py
+++ b/tensorflow/python/kernel_tests/map_stage_op_test.py
@@ -148,7 +148,7 @@ class MapStageTest(test.TestCase):
       for i in range(n):
         self.assertTrue(sess.run(peek, feed_dict={gi: i})[0] == i)
 
-      self.assertTrue(self.evaluate(size) == 10)
+      self.assertTrue(sess.run(size) == 10)
 
   def testSizeAndClear(self):
     with ops.Graph().as_default() as G:
@@ -170,11 +170,11 @@ class MapStageTest(test.TestCase):
 
     with self.session(use_gpu=True, graph=G) as sess:
       sess.run(stage, feed_dict={x: -1, pi: 3})
-      self.assertEqual(self.evaluate(size), 1)
+      self.assertEqual(sess.run(size), 1)
       sess.run(stage, feed_dict={x: -1, pi: 1})
-      self.assertEqual(self.evaluate(size), 2)
+      self.assertEqual(sess.run(size), 2)
       sess.run(clear)
-      self.assertEqual(self.evaluate(size), 0)
+      self.assertEqual(sess.run(size), 0)
 
   def testCapacity(self):
     capacity = 3
@@ -231,13 +231,13 @@ class MapStageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(self.evaluate(size) == capacity)
+      self.assertTrue(sess.run(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
         sess.run(get)
 
-      self.assertTrue(self.evaluate(size) == 0)
+      self.assertTrue(sess.run(size) == 0)
 
   def testMemoryLimit(self):
     memory_limit = 512 * 1024  # 512K
@@ -295,13 +295,13 @@ class MapStageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(self.evaluate(size) == capacity)
+      self.assertTrue(sess.run(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
         sess.run(get)
 
-      self.assertTrue(self.evaluate(size) == 0)
+      self.assertTrue(sess.run(size) == 0)
 
   def testOrdering(self):
     import six
@@ -332,14 +332,14 @@ class MapStageTest(test.TestCase):
       for i in keys:
         sess.run(stage, feed_dict={pi: i, x: i})
 
-      self.assertTrue(self.evaluate(size) == n)
+      self.assertTrue(sess.run(size) == n)
 
       # Check that key, values come out in ascending order
       for i, k in enumerate(reversed(keys)):
-        get_key, values = self.evaluate(get)
+        get_key, values = sess.run(get)
         self.assertTrue(i == k == get_key == values)
 
-      self.assertTrue(self.evaluate(size) == 0)
+      self.assertTrue(sess.run(size) == 0)
 
   def testPartialDictInsert(self):
     with ops.Graph().as_default() as G:
diff --git a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
index 5cef4b79a3..434458721c 100644
--- a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
@@ -146,7 +146,7 @@ class InverseOpTest(test.TestCase):
         inv1 = linalg_ops.matrix_inverse(matrix1, adjoint=adjoint_)
         inv2 = linalg_ops.matrix_inverse(matrix2, adjoint=adjoint_)
         all_ops += [inv1, inv2]
-      inv = self.evaluate(all_ops)
+      inv = sess.run(all_ops)
       self.assertAllEqual(inv[0], inv[1])
       self.assertAllEqual(inv[2], inv[3])
 
diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
index 80badee896..1334d0c4ce 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
@@ -126,7 +126,7 @@ class MatrixSolveOpTest(test.TestCase):
         s1 = linalg_ops.matrix_solve(lhs1, rhs1, adjoint=adjoint_)
         s2 = linalg_ops.matrix_solve(lhs2, rhs2, adjoint=adjoint_)
         all_ops += [s1, s2]
-      val = self.evaluate(all_ops)
+      val = sess.run(all_ops)
       self.assertAllEqual(val[0], val[1])
       self.assertAllEqual(val[2], val[3])
 
diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 1f2144bdee..9212580313 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -108,7 +108,7 @@ class SquareRootOpTest(test.TestCase):
       sqrt1 = gen_linalg_ops.matrix_square_root(matrix1)
       sqrt2 = gen_linalg_ops.matrix_square_root(matrix2)
       all_ops = [sqrt1, sqrt2]
-      sqrt = self.evaluate(all_ops)
+      sqrt = sess.run(all_ops)
       self.assertAllEqual(sqrt[0], sqrt[1])
 
 
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index b68327105a..5dcdb9e420 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -203,10 +203,10 @@ class MeanTest(test.TestCase):
 
       mean, update_op = metrics.mean(values)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(4):
-        self.evaluate(update_op)
-      self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
+        sess.run(update_op)
+      self.assertAlmostEqual(1.65, sess.run(mean), 5)
 
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
@@ -220,14 +220,14 @@ class MeanTest(test.TestCase):
 
       mean, update_op = metrics.mean(values)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(0.5, self.evaluate(update_op), 5)
-      self.assertAlmostEqual(1.475, self.evaluate(update_op), 5)
-      self.assertAlmostEqual(12.4 / 6.0, self.evaluate(update_op), 5)
-      self.assertAlmostEqual(1.65, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(0.5, sess.run(update_op), 5)
+      self.assertAlmostEqual(1.475, sess.run(update_op), 5)
+      self.assertAlmostEqual(12.4 / 6.0, sess.run(update_op), 5)
+      self.assertAlmostEqual(1.65, sess.run(update_op), 5)
 
-      self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
+      self.assertAlmostEqual(1.65, sess.run(mean), 5)
 
   def testUnweighted(self):
     values = _test_values((3, 2, 4, 1))
@@ -370,10 +370,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(4):
-        self.evaluate(update_op)
-      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean))
+        sess.run(update_op)
+      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean))
 
   def testMultiDimensional(self):
     with self.cached_session() as sess:
@@ -391,11 +391,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(2):
-        self.evaluate(update_op)
-      self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]],
-                          self.evaluate(mean))
+        sess.run(update_op)
+      self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]], sess.run(mean))
 
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
@@ -409,14 +408,14 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
-      self.assertAllClose([[0, 1]], self.evaluate(update_op), 5)
-      self.assertAllClose([[-2.1, 5.05]], self.evaluate(update_op), 5)
-      self.assertAllClose([[2.3 / 3., 10.1 / 3.]], self.evaluate(update_op), 5)
-      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(update_op), 5)
+      self.assertAllClose([[0, 1]], sess.run(update_op), 5)
+      self.assertAllClose([[-2.1, 5.05]], sess.run(update_op), 5)
+      self.assertAllClose([[2.3 / 3., 10.1 / 3.]], sess.run(update_op), 5)
+      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(update_op), 5)
 
-      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean), 5)
+      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean), 5)
 
   def testBinaryWeighted1d(self):
     with self.cached_session() as sess:
@@ -440,10 +439,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(4):
-        self.evaluate(update_op)
-      self.assertAllClose([[3.25, 0.5]], self.evaluate(mean), 5)
+        sess.run(update_op)
+      self.assertAllClose([[3.25, 0.5]], sess.run(mean), 5)
 
   def testWeighted1d(self):
     with self.cached_session() as sess:
@@ -467,10 +466,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(4):
-        self.evaluate(update_op)
-      self.assertAllClose([[0.8, 3.52]], self.evaluate(mean), 5)
+        sess.run(update_op)
+      self.assertAllClose([[0.8, 3.52]], sess.run(mean), 5)
 
   def testWeighted2d_1(self):
     with self.cached_session() as sess:
@@ -494,10 +493,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(4):
-        self.evaluate(update_op)
-      self.assertAllClose([[-2.1, 0.5]], self.evaluate(mean), 5)
+        sess.run(update_op)
+      self.assertAllClose([[-2.1, 0.5]], sess.run(mean), 5)
 
   def testWeighted2d_2(self):
     with self.cached_session() as sess:
@@ -521,10 +520,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(4):
-        self.evaluate(update_op)
-      self.assertAllClose([[0, 0.5]], self.evaluate(mean), 5)
+        sess.run(update_op)
+      self.assertAllClose([[0, 0.5]], sess.run(mean), 5)
 
 
 class AccuracyTest(test.TestCase):
@@ -577,11 +576,11 @@ class AccuracyTest(test.TestCase):
     accuracy, update_op = metrics.accuracy(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_accuracy = accuracy.eval()
@@ -610,10 +609,10 @@ class AccuracyTest(test.TestCase):
 
       accuracy, update_op = metrics.accuracy(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in xrange(3):
-        self.evaluate(update_op)
-      self.assertEqual(0.5, self.evaluate(update_op))
+        sess.run(update_op)
+      self.assertEqual(0.5, sess.run(update_op))
       self.assertEqual(0.5, accuracy.eval())
 
   def testEffectivelyEquivalentSizes(self):
@@ -622,7 +621,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
@@ -632,7 +631,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions, weights=2.0)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
@@ -646,7 +645,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions, weights)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       # if streaming_accuracy does not flatten the weight, accuracy would be
       # 0.33333334 due to an intended broadcast of weight. Due to flattening,
       # it will be higher than .95
@@ -667,7 +666,7 @@ class AccuracyTest(test.TestCase):
       accuracy, update_op = metrics.accuracy(labels, predictions,
                                              weights_placeholder)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       # if streaming_accuracy does not flatten the weight, accuracy would be
       # 0.33333334 due to an intended broadcast of weight. Due to flattening,
       # it will be higher than .95
@@ -705,10 +704,10 @@ class AccuracyTest(test.TestCase):
 
       accuracy, update_op = metrics.accuracy(labels, predictions, weights)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in xrange(3):
-        self.evaluate(update_op)
-      self.assertEqual(1.0, self.evaluate(update_op))
+        sess.run(update_op)
+      self.assertEqual(1.0, sess.run(update_op))
       self.assertEqual(1.0, accuracy.eval())
 
 
@@ -748,11 +747,11 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_precision = precision.eval()
@@ -767,8 +766,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, sess.run(update_op))
       self.assertAlmostEqual(1, precision.eval())
 
   def testSomeCorrect_multipleInputDtypes(self):
@@ -780,7 +779,7 @@ class PrecisionTest(test.TestCase):
       precision, update_op = metrics.precision(labels, predictions)
 
       with self.cached_session() as sess:
-        self.evaluate(variables.local_variables_initializer())
+        sess.run(variables.local_variables_initializer())
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, precision.eval())
 
@@ -883,8 +882,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.evaluate(update_op)
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
       self.assertAlmostEqual(0, precision.eval())
 
   def testZeroTrueAndFalsePositivesGivesZeroPrecision(self):
@@ -893,8 +892,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.evaluate(update_op)
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
       self.assertEqual(0.0, precision.eval())
 
 
@@ -935,11 +934,11 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_recall = recall.eval()
@@ -954,8 +953,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.evaluate(update_op)
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
       self.assertEqual(1, recall.eval())
 
   def testSomeCorrect_multipleInputDtypes(self):
@@ -967,7 +966,7 @@ class RecallTest(test.TestCase):
       recall, update_op = metrics.recall(labels, predictions)
 
       with self.cached_session() as sess:
-        self.evaluate(variables.local_variables_initializer())
+        sess.run(variables.local_variables_initializer())
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, recall.eval())
 
@@ -978,7 +977,7 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       weighted_tp = 2.0 + 5.0
       weighted_t = (2.0 + 2.0) + (5.0 + 5.0)
       expected_precision = weighted_tp / weighted_t
@@ -992,7 +991,7 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       weighted_tp = 3.0 + 1.0
       weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
       expected_precision = weighted_tp / weighted_t
@@ -1007,8 +1006,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.evaluate(update_op)
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
       self.assertEqual(0, recall.eval())
 
   def testZeroTruePositivesAndFalseNegativesGivesZeroRecall(self):
@@ -1017,8 +1016,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.evaluate(update_op)
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
       self.assertEqual(0, recall.eval())
 
 
@@ -1057,11 +1056,11 @@ class AUCTest(test.TestCase):
     auc, update_op = metrics.auc(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_auc = auc.eval()
@@ -1079,8 +1078,8 @@ class AUCTest(test.TestCase):
       labels = constant_op.constant(inputs)
       auc, update_op = metrics.auc(labels, predictions, curve=curve)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(1, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(1, sess.run(update_op))
 
       self.assertEqual(1, auc.eval())
 
@@ -1094,8 +1093,8 @@ class AUCTest(test.TestCase):
             constant_op.constant([0, 1, 1, 0], shape=(1, 4)), dtype=label_dtype)
         auc, update_op = metrics.auc(labels, predictions)
 
-        self.evaluate(variables.local_variables_initializer())
-        self.assertAlmostEqual(0.5, self.evaluate(update_op))
+        sess.run(variables.local_variables_initializer())
+        self.assertAlmostEqual(0.5, sess.run(update_op))
 
         self.assertAlmostEqual(0.5, auc.eval())
 
@@ -1107,8 +1106,8 @@ class AUCTest(test.TestCase):
       weights = constant_op.constant([2], shape=(1, 1))
       auc, update_op = metrics.auc(labels, predictions, weights=weights)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.5, self.evaluate(update_op), 5)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, sess.run(update_op), 5)
 
       self.assertAlmostEqual(0.5, auc.eval(), 5)
 
@@ -1120,8 +1119,8 @@ class AUCTest(test.TestCase):
       weights = constant_op.constant([1, 2, 3, 4], shape=(1, 4))
       auc, update_op = metrics.auc(labels, predictions, weights=weights)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.7, self.evaluate(update_op), 5)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.7, sess.run(update_op), 5)
 
       self.assertAlmostEqual(0.7, auc.eval(), 5)
 
@@ -1135,10 +1134,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       # expected ~= 0.79726744594
       expected = 1 - math.log(1.5) / 2
-      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testCorrectAnotherAUCPRSpecialCase(self):
@@ -1151,10 +1150,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       # expected ~= 0.61350593198
       expected = (2.5 - 2 * math.log(4./3) - 0.25 * math.log(7./5)) / 3
-      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testThirdCorrectAUCPRSpecialCase(self):
@@ -1167,10 +1166,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       # expected ~= 0.90410597584
       expected = 1 - math.log(4./3) / 3
-      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testIncorrectAUCPRSpecialCase(self):
@@ -1181,8 +1180,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.79166, self.evaluate(update_op), delta=1e-3)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.79166, sess.run(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3)
 
@@ -1196,8 +1195,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.610317, self.evaluate(update_op), delta=1e-3)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.610317, sess.run(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3)
 
@@ -1211,8 +1210,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.90277, self.evaluate(update_op), delta=1e-3)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.90277, sess.run(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-3)
 
@@ -1224,8 +1223,8 @@ class AUCTest(test.TestCase):
       labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
       auc, update_op = metrics.auc(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0, sess.run(update_op))
 
       self.assertAlmostEqual(0, auc.eval())
 
@@ -1235,8 +1234,8 @@ class AUCTest(test.TestCase):
       labels = array_ops.zeros([4])
       auc, update_op = metrics.auc(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, self.evaluate(update_op), 6)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, sess.run(update_op), 6)
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
@@ -1246,8 +1245,8 @@ class AUCTest(test.TestCase):
       labels = array_ops.ones([4])
       auc, update_op = metrics.auc(labels, predictions, curve='PR')
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, self.evaluate(update_op), 6)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, sess.run(update_op), 6)
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
@@ -1318,9 +1317,9 @@ class AUCTest(test.TestCase):
                                      num_thresholds=500,
                                      weights=tf_weights)
 
-        self.evaluate(variables.local_variables_initializer())
+        sess.run(variables.local_variables_initializer())
         for i in range(num_batches):
-          self.evaluate(update_op)
+          sess.run(update_op)
 
         # Since this is only approximate, we can't expect a 6 digits match.
         # Although with higher number of samples/thresholds we should see the
@@ -1372,11 +1371,11 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.7)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_specificity = specificity.eval()
@@ -1392,8 +1391,8 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.7)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(1, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(1, sess.run(update_op))
       self.assertEqual(1, specificity.eval())
 
   def testSomeCorrectHighSensitivity(self):
@@ -1407,8 +1406,8 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.8)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(1.0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(1.0, sess.run(update_op))
       self.assertAlmostEqual(1.0, specificity.eval())
 
   def testSomeCorrectLowSensitivity(self):
@@ -1422,9 +1421,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.4)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(0.6, self.evaluate(update_op))
+      self.assertAlmostEqual(0.6, sess.run(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted1d_multipleLabelDtypes(self):
@@ -1441,9 +1440,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
           labels, predictions, weights=weights, sensitivity=0.4)
 
       with self.cached_session() as sess:
-        self.evaluate(variables.local_variables_initializer())
+        sess.run(variables.local_variables_initializer())
 
-        self.assertAlmostEqual(0.6, self.evaluate(update_op))
+        self.assertAlmostEqual(0.6, sess.run(update_op))
         self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted2d(self):
@@ -1459,9 +1458,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, weights=weights, sensitivity=0.4)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(8.0 / 15.0, self.evaluate(update_op))
+      self.assertAlmostEqual(8.0 / 15.0, sess.run(update_op))
       self.assertAlmostEqual(8.0 / 15.0, specificity.eval())
 
 
@@ -1509,11 +1508,11 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.7)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_sensitivity = sensitivity.eval()
@@ -1529,8 +1528,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.7)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(1, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(1, sess.run(update_op))
       self.assertEqual(1, specificity.eval())
 
   def testSomeCorrectHighSpecificity(self):
@@ -1544,8 +1543,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.8)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.8, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.8, sess.run(update_op))
       self.assertAlmostEqual(0.8, specificity.eval())
 
   def testSomeCorrectLowSpecificity(self):
@@ -1559,8 +1558,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.4)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.6, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.6, sess.run(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted_multipleLabelDtypes(self):
@@ -1578,8 +1577,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
           labels, predictions, weights=weights, specificity=0.4)
 
       with self.cached_session() as sess:
-        self.evaluate(variables.local_variables_initializer())
-        self.assertAlmostEqual(0.675, self.evaluate(update_op))
+        sess.run(variables.local_variables_initializer())
+        self.assertAlmostEqual(0.675, sess.run(update_op))
         self.assertAlmostEqual(0.675, specificity.eval())
 
 
@@ -1640,7 +1639,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
     rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates, then verify idempotency.
       sess.run([prec_op, rec_op])
@@ -1664,7 +1663,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertEqual(1, prec.eval())
@@ -1684,7 +1683,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                    thresholds)
 
-        self.evaluate(variables.local_variables_initializer())
+        sess.run(variables.local_variables_initializer())
         sess.run([prec_op, rec_op])
 
         self.assertAlmostEqual(0.5, prec.eval())
@@ -1702,7 +1701,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(0, prec.eval())
@@ -1730,7 +1729,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec_low = array_ops.reshape(rec_low, shape=())
       rec_high = array_ops.reshape(rec_high, shape=())
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(1.0, prec_low.eval(), places=5)
@@ -1760,7 +1759,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec_low = array_ops.reshape(rec_low, shape=())
       rec_high = array_ops.reshape(rec_high, shape=())
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(1.0, prec_low.eval(), places=5)
@@ -1784,7 +1783,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       [rec_low, rec_high] = array_ops.split(
           value=rec, num_or_size_splits=2, axis=0)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(0.75, prec_low.eval())
@@ -1802,7 +1801,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
 
       self.assertAlmostEqual(0, prec.eval(), 6)
@@ -1870,7 +1869,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(tf_labels, tf_predictions,
                                                  thresholds)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(int(num_samples / batch_size)):
         sess.run([prec_op, rec_op])
       # Since this is only approximate, we can't expect a 6 digits match.
@@ -2803,11 +2802,11 @@ class MeanAbsoluteErrorTest(test.TestCase):
     error, update_op = metrics.mean_absolute_error(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2824,8 +2823,8 @@ class MeanAbsoluteErrorTest(test.TestCase):
     error, update_op = metrics.mean_absolute_error(labels, predictions, weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(3, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(3, sess.run(update_op))
       self.assertEqual(3, error.eval())
 
 
@@ -2868,11 +2867,11 @@ class MeanRelativeErrorTest(test.TestCase):
                                                    normalizer)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2893,8 +2892,8 @@ class MeanRelativeErrorTest(test.TestCase):
         labels, predictions, normalizer=labels)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(expected_error, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(expected_error, sess.run(update_op))
       self.assertEqual(expected_error, error.eval())
 
   def testSingleUpdateNormalizedByZeros(self):
@@ -2909,8 +2908,8 @@ class MeanRelativeErrorTest(test.TestCase):
         labels, predictions, normalizer=array_ops.zeros_like(labels))
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(0.0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(0.0, sess.run(update_op))
       self.assertEqual(0.0, error.eval())
 
 
@@ -2947,11 +2946,11 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2965,8 +2964,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(0, sess.run(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithError(self):
@@ -2978,8 +2977,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(6, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(6, sess.run(update_op))
       self.assertEqual(6, error.eval())
 
   def testSingleUpdateWithErrorAndWeights(self):
@@ -2992,8 +2991,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions, weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(13, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(13, sess.run(update_op))
       self.assertEqual(13, error.eval())
 
   def testMultipleBatchesOfSizeOne(self):
@@ -3014,9 +3013,9 @@ class MeanSquaredErrorTest(test.TestCase):
 
       error, update_op = metrics.mean_squared_error(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.evaluate(update_op)
-      self.assertAlmostEqual(208.0 / 6, self.evaluate(update_op), 5)
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertAlmostEqual(208.0 / 6, sess.run(update_op), 5)
 
       self.assertAlmostEqual(208.0 / 6, error.eval(), 5)
 
@@ -3055,7 +3054,7 @@ class MeanSquaredErrorTest(test.TestCase):
       mse1, update_op1 = metrics.mean_squared_error(
           labels1, predictions1, name='msd1')
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([update_op0, update_op1])
       sess.run([update_op0, update_op1])
 
@@ -3082,7 +3081,7 @@ class MeanSquaredErrorTest(test.TestCase):
       mae, ma_update_op = metrics.mean_absolute_error(labels, predictions)
       mse, ms_update_op = metrics.mean_squared_error(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([ma_update_op, ms_update_op])
       sess.run([ma_update_op, ms_update_op])
 
@@ -3124,11 +3123,11 @@ class RootMeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.root_mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -3143,8 +3142,8 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(0, sess.run(update_op))
 
       self.assertEqual(0, rmse.eval())
 
@@ -3157,7 +3156,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAlmostEqual(math.sqrt(6), update_op.eval(), 5)
       self.assertAlmostEqual(math.sqrt(6), rmse.eval(), 5)
 
@@ -3172,8 +3171,8 @@ class RootMeanSquaredErrorTest(test.TestCase):
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions,
                                                         weights)
 
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(math.sqrt(13), self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(math.sqrt(13), sess.run(update_op))
 
       self.assertAlmostEqual(math.sqrt(13), rmse.eval(), 5)
 
@@ -3222,11 +3221,11 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=1)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -3244,8 +3243,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(0, sess.run(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithError1(self):
@@ -3260,8 +3259,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, self.evaluate(update_op), 5)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, sess.run(update_op), 5)
       self.assertAlmostEqual(1, error.eval(), 5)
 
   def testSingleUpdateWithError2(self):
@@ -3281,8 +3280,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertAlmostEqual(1.0, self.evaluate(update_op), 5)
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(1.0, sess.run(update_op), 5)
       self.assertAlmostEqual(1.0, error.eval(), 5)
 
   def testSingleUpdateWithErrorAndWeights1(self):
@@ -3300,8 +3299,8 @@ class MeanCosineDistanceTest(test.TestCase):
         labels, predictions, dim=2, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
-      self.assertEqual(0, self.evaluate(update_op))
+      sess.run(variables.local_variables_initializer())
+      self.assertEqual(0, sess.run(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithErrorAndWeights2(self):
@@ -3319,7 +3318,7 @@ class MeanCosineDistanceTest(test.TestCase):
         labels, predictions, dim=2, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertEqual(1.5, update_op.eval())
       self.assertEqual(1.5, error.eval())
 
@@ -3361,7 +3360,7 @@ class PcntBelowThreshTest(test.TestCase):
       pcnt1, update_op1 = metrics.percentage_below(values, 7, name='medium')
       pcnt2, update_op2 = metrics.percentage_below(values, 1, name='low')
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       sess.run([update_op0, update_op1, update_op2])
 
       pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2])
@@ -3383,7 +3382,7 @@ class PcntBelowThreshTest(test.TestCase):
       pcnt2, update_op2 = metrics.percentage_below(
           values, 1, weights=weights, name='low')
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertListEqual([1.0, 0.5, 0.0],
                            sess.run([update_op0, update_op1, update_op2]))
 
@@ -3447,11 +3446,11 @@ class MeanIOUTest(test.TestCase):
         labels, predictions, num_classes=num_classes)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_mean_iou = mean_iou.eval()
@@ -3483,9 +3482,9 @@ class MeanIOUTest(test.TestCase):
 
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(5):
-        self.evaluate(update_op)
+        sess.run(update_op)
       desired_output = np.mean([1.0 / 2.0, 1.0 / 4.0, 0.])
       self.assertEqual(desired_output, miou.eval())
 
@@ -3530,7 +3529,7 @@ class MeanIOUTest(test.TestCase):
 
       variables.local_variables_initializer().run()
       for _ in range(6):
-        self.evaluate(update_op)
+        sess.run(update_op)
       desired_output = np.mean([2.0 / 3.0, 1.0 / 2.0])
       self.assertAlmostEqual(desired_output, mean_iou.eval())
 
@@ -3564,9 +3563,9 @@ class MeanIOUTest(test.TestCase):
 
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(5):
-        self.evaluate(update_op)
+        sess.run(update_op)
       desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
@@ -3588,7 +3587,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       confusion_matrix = update_op.eval()
       self.assertAllEqual([[3, 0], [2, 5]], confusion_matrix)
       desired_miou = np.mean([3. / 5., 5. / 7.])
@@ -3600,7 +3599,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 1
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertEqual(40, update_op.eval()[0])
       self.assertEqual(1.0, miou.eval())
 
@@ -3610,7 +3609,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual([[0, 0], [40, 0]], update_op.eval())
       self.assertEqual(0., miou.eval())
 
@@ -3641,7 +3640,7 @@ class MeanIOUTest(test.TestCase):
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(
           labels, predictions, num_classes, weights=weights)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual([[2, 0], [2, 4]], update_op.eval())
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
@@ -3660,7 +3659,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 3
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
       self.assertAlmostEqual(
           1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
@@ -3672,7 +3671,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
       self.assertAlmostEqual(1, miou.eval())
 
@@ -3690,7 +3689,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 3
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
       self.assertAlmostEqual(
           1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
@@ -3753,11 +3752,11 @@ class MeanPerClassAccuracyTest(test.TestCase):
         labels, predictions, num_classes=num_classes)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        self.evaluate(update_op)
+        sess.run(update_op)
 
       # Then verify idempotency.
       initial_mean_accuracy = mean_accuracy.eval()
@@ -3789,9 +3788,9 @@ class MeanPerClassAccuracyTest(test.TestCase):
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(5):
-        self.evaluate(update_op)
+        sess.run(update_op)
       desired_output = np.mean([1.0, 1.0 / 3.0, 0.0])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3836,7 +3835,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
 
       variables.local_variables_initializer().run()
       for _ in range(6):
-        self.evaluate(update_op)
+        sess.run(update_op)
       desired_output = np.mean([2.0 / 2.0, 0.5 / 1.5])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3871,9 +3870,9 @@ class MeanPerClassAccuracyTest(test.TestCase):
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
 
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       for _ in range(5):
-        self.evaluate(update_op)
+        sess.run(update_op)
       desired_output = np.mean([1.0 / 2.0, 2.0 / 3.0, 0.])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3884,7 +3883,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval()[0])
       self.assertEqual(1.0, mean_accuracy.eval())
 
@@ -3895,7 +3894,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual([0.0, 0.0], update_op.eval())
       self.assertEqual(0., mean_accuracy.eval())
 
@@ -3914,7 +3913,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes, weights=weights)
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       desired_accuracy = np.array([2. / 2., 4. / 6.], dtype=np.float32)
       self.assertAllEqual(desired_accuracy, update_op.eval())
       desired_mean_accuracy = np.mean(desired_accuracy)
@@ -3946,7 +3945,7 @@ class FalseNegativesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
@@ -3965,7 +3964,7 @@ class FalseNegativesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(5., tn_update_op.eval())
       self.assertAllClose(5., tn.eval())
@@ -3995,7 +3994,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), fn.eval())
       self.assertAllEqual((0, 2, 3), fn_update_op.eval())
       self.assertAllEqual((0, 2, 3), fn.eval())
@@ -4014,7 +4013,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), fn.eval())
       self.assertAllEqual((0.0, 8.0, 11.0), fn_update_op.eval())
       self.assertAllEqual((0.0, 8.0, 11.0), fn.eval())
@@ -4045,7 +4044,7 @@ class FalsePositivesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
@@ -4064,7 +4063,7 @@ class FalsePositivesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(14., tn_update_op.eval())
       self.assertAllClose(14., tn.eval())
@@ -4094,7 +4093,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), fp.eval())
       self.assertAllEqual((7, 4, 2), fp_update_op.eval())
       self.assertAllEqual((7, 4, 2), fp.eval())
@@ -4115,7 +4114,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), fp.eval())
       self.assertAllEqual((125.0, 42.0, 12.0), fp_update_op.eval())
       self.assertAllEqual((125.0, 42.0, 12.0), fp.eval())
@@ -4146,7 +4145,7 @@ class TrueNegativesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
@@ -4165,7 +4164,7 @@ class TrueNegativesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(4., tn_update_op.eval())
       self.assertAllClose(4., tn.eval())
@@ -4195,7 +4194,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), tn.eval())
       self.assertAllEqual((2, 5, 7), tn_update_op.eval())
       self.assertAllEqual((2, 5, 7), tn.eval())
@@ -4214,7 +4213,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), tn.eval())
       self.assertAllEqual((5.0, 15.0, 23.0), tn_update_op.eval())
       self.assertAllEqual((5.0, 15.0, 23.0), tn.eval())
@@ -4245,7 +4244,7 @@ class TruePositivesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
@@ -4264,7 +4263,7 @@ class TruePositivesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(12., tn_update_op.eval())
       self.assertAllClose(12., tn.eval())
@@ -4294,7 +4293,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), tp.eval())
       self.assertAllEqual((3, 1, 0), tp_update_op.eval())
       self.assertAllEqual((3, 1, 0), tp.eval())
@@ -4311,7 +4310,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      self.evaluate(variables.local_variables_initializer())
+      sess.run(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), tp.eval())
       self.assertAllEqual((111.0, 37.0, 0.0), tp_update_op.eval())
       self.assertAllEqual((111.0, 37.0, 0.0), tp.eval())
diff --git a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
index 87f1991aa7..15e3826542 100644
--- a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
@@ -142,8 +142,8 @@ class DepthwiseConv2DTest(test.TestCase):
       conv_interface = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
 
-      native_result = self.evaluate(conv_native)
-      interface_result = self.evaluate(conv_interface)
+      native_result = sess.run(conv_native)
+      interface_result = sess.run(conv_interface)
 
     print("depthwise conv_2d: ", tensor_in_sizes, "*", filter_in_sizes,
           ", stride:", stride, ", padding: ", padding, ", max diff: ",
@@ -211,7 +211,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=filter_in_sizes)
         conv = nn_ops.depthwise_conv2d_native(
             t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        value = self.evaluate(conv)
+        value = sess.run(conv)
     print("value = ", value)
     self.assertAllClose(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py
index 5ff0c58bf1..e202b6e8a4 100644
--- a/tensorflow/python/kernel_tests/norm_op_test.py
+++ b/tensorflow/python/kernel_tests/norm_op_test.py
@@ -70,7 +70,7 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_):
         tf_matrix = constant_op.constant(matrix)
         tf_norm = linalg_ops.norm(
             tf_matrix, ord=ord_, axis=axis_, keepdims=keep_dims_)
-        tf_norm_val = self.evaluate(tf_norm)
+        tf_norm_val = sess.run(tf_norm)
       else:
         tf_matrix = array_ops.placeholder(dtype_)
         tf_norm = linalg_ops.norm(
diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py
index 6cd4974671..338b6cec01 100644
--- a/tensorflow/python/kernel_tests/nth_element_op_test.py
+++ b/tensorflow/python/kernel_tests/nth_element_op_test.py
@@ -35,7 +35,7 @@ class NthElementTest(test.TestCase):
     with self.cached_session(use_gpu=False) as sess:
       inputs_op = ops.convert_to_tensor(inputs, dtype=dtype)
       values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse)
-      values = self.evaluate(values_op)
+      values = sess.run(values_op)
 
       self.assertShapeEqual(np_expected_values, values_op)
       self.assertAllClose(np_expected_values, values)
diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
index 3696298132..520b663375 100644
--- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
@@ -126,7 +126,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -193,7 +193,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          self.evaluate(enqueue_op)
+          sess.run(enqueue_op)
 
       results = []
 
@@ -224,7 +224,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        x_val, y_val = self.evaluate(dequeued_t)
+        x_val, y_val = sess.run(dequeued_t)
         x, y = elems[i]
         self.assertEqual([x], x_val)
         self.assertEqual([y], y_val)
@@ -327,7 +327,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = self.evaluate(dequeued_t)
+        float_val, int_val = sess.run(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -344,7 +344,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = self.evaluate(dequeued_t)
+        float_val, int_val = sess.run(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -387,17 +387,17 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
@@ -418,7 +418,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertTrue(
@@ -428,11 +428,11 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertTrue(
@@ -459,7 +459,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       for enqueue_op in enqueue_ops:
         enqueue_op.run()
-      string_val, int_val = self.evaluate(dequeued_t)
+      string_val, int_val = sess.run(dequeued_t)
 
       self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""],
                            [b"abc", b"", b""], [b"abc", b"d", b""],
@@ -473,7 +473,7 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      string_val, int_val = self.evaluate(dequeued_single_t)
+      string_val, int_val = sess.run(dequeued_single_t)
       self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val)
       self.assertAllEqual([[1, 2, 3, 4]], int_val)
       self.assertTrue(
@@ -500,7 +500,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       for enqueue_op in enqueue_ops:
         enqueue_op.run()
-      string_val, int_val = self.evaluate(dequeued_t)
+      string_val, int_val = sess.run(dequeued_t)
 
       self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""],
                            [b"abc", b"", b""], [b"abc", b"d", b""],
@@ -514,7 +514,7 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      string_val, int_val = self.evaluate(dequeued_single_t)
+      string_val, int_val = sess.run(dequeued_single_t)
       self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val)
       self.assertAllEqual([[1, 2, 3, 4]], int_val)
       self.assertTrue(
@@ -633,7 +633,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -700,11 +700,11 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def enqueue():
         for _ in xrange(100):
-          self.evaluate(enqueue_op)
+          sess.run(enqueue_op)
 
       def dequeue():
         for _ in xrange(100):
-          self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0))
+          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))
 
       enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
@@ -736,7 +736,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         for i in xrange(250):
-          self.assertEqual(i, self.evaluate(dequeued_t))
+          self.assertEqual(i, sess.run(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -767,7 +767,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       dequeuemany_t = q.dequeue_many(count_placeholder)
 
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -805,7 +805,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -832,7 +832,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -901,7 +901,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         for elem in elems:
-          self.assertEqual([elem], self.evaluate(dequeued_t))
+          self.assertEqual([elem], sess.run(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -926,8 +926,8 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
-        self.assertAllEqual(elems[3:], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[3:], sess.run(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -968,7 +968,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems, self.evaluate(dequeued_t))
+        self.assertAllEqual(elems, sess.run(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -993,7 +993,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
@@ -1017,16 +1017,16 @@ class PaddingFIFOQueueTest(test.TestCase):
       cleanup_dequeue_t = q.dequeue()
 
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
-        self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[0:3], sess.run(dequeued_t))
         with self.assertRaises(errors_impl.OutOfRangeError):
           sess.run(dequeued_t)
-        self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t))
+        self.assertEqual(elems[3], sess.run(cleanup_dequeue_t))
 
       def close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -1155,7 +1155,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1178,7 +1178,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1207,7 +1207,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # Expect the operation to succeed once the dequeue op runs.
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1217,7 +1217,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1242,7 +1242,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1252,7 +1252,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1379,19 +1379,19 @@ class PaddingFIFOQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_op)
+      sess.run(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_many_op)
+      sess.run(dequeue_many_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(enqueue_op)
+      sess.run(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(enqueue_many_op)
+      sess.run(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1444,14 +1444,14 @@ class PaddingFIFOQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(self.evaluate(size_op), 5)
+      self.assertEqual(sess.run(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(self.evaluate(size_op), 5)
+      self.assertEqual(sess.run(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1517,7 +1517,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       q.enqueue_many(input_tuple).run()
 
       output_tuple_t = q.dequeue_many(32)
-      output_tuple = self.evaluate(output_tuple_t)
+      output_tuple = sess.run(output_tuple_t)
 
       for (input_elem, output_elem) in zip(input_tuple, output_tuple):
         self.assertAllEqual(input_elem, output_elem)
diff --git a/tensorflow/python/kernel_tests/parse_single_example_op_test.py b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
index 3f50087282..a84895a287 100644
--- a/tensorflow/python/kernel_tests/parse_single_example_op_test.py
+++ b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
@@ -107,7 +107,7 @@ class ParseExampleTest(test.TestCase):
         for result_dict in [out, out_with_example_name]:
           result = flatten_values_tensors_or_sparse(result_dict.values())
           # Check values.
-          tf_result = self.evaluate(result)
+          tf_result = sess.run(result)
           _compare_output_to_expected(self, result_dict, expected_values,
                                       tf_result)
 
diff --git a/tensorflow/python/kernel_tests/parsing_ops_test.py b/tensorflow/python/kernel_tests/parsing_ops_test.py
index d87adbfc2e..8f359bd32c 100644
--- a/tensorflow/python/kernel_tests/parsing_ops_test.py
+++ b/tensorflow/python/kernel_tests/parsing_ops_test.py
@@ -101,7 +101,7 @@ class ParseExampleTest(test.TestCase):
         out = parsing_ops.parse_example(**kwargs)
         result = flatten_values_tensors_or_sparse(out.values())
         # Check values.
-        tf_result = self.evaluate(result)
+        tf_result = sess.run(result)
         _compare_output_to_expected(self, out, expected_values, tf_result)
 
       # Check shapes; if serialized is a Tensor we need its size to
@@ -1614,7 +1614,7 @@ class DecodeJSONExampleTest(test.TestCase):
           shape=examples.shape,
           dtype=dtypes.string)
       binary_tensor = parsing_ops.decode_json_example(json_tensor)
-      binary_val = self.evaluate(binary_tensor)
+      binary_val = sess.run(binary_tensor)
 
       if examples.shape:
         self.assertShapeEqual(binary_val, json_tensor)
diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
index a8e962bc3a..e393c7a022 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
@@ -81,7 +81,7 @@ class PoolingTest(test.TestCase):
           data_format=data_format)
       if data_format == "NCDHW":
         t = test_util.NCHWToNHWC(t)
-      vals = self.evaluate(t)
+      vals = sess.run(t)
     # Verifies values.
     actual = vals.flatten()
     self.assertAllClose(expected, actual)
diff --git a/tensorflow/python/kernel_tests/priority_queue_test.py b/tensorflow/python/kernel_tests/priority_queue_test.py
index a510fccaaa..73a9c81638 100644
--- a/tensorflow/python/kernel_tests/priority_queue_test.py
+++ b/tensorflow/python/kernel_tests/priority_queue_test.py
@@ -50,7 +50,7 @@ class PriorityQueueTest(test.TestCase):
         enq.run()
 
       deq = q.dequeue_many(100)
-      deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq)
+      deq_elem, deq_value_0, deq_value_1 = sess.run(deq)
 
       allowed = {}
       missed = set()
@@ -81,7 +81,7 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       dequeue_op = q.dequeue_many(100)
 
@@ -93,7 +93,7 @@ class PriorityQueueTest(test.TestCase):
       for t in enqueue_threads:
         t.start()
 
-      deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op)
+      deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op)
 
       for t in enqueue_threads:
         t.join()
@@ -132,12 +132,12 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       dequeued = []
 
       def dequeue(dequeue_op):
-        (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op)
+        (dequeue_indices, dequeue_values) = sess.run(dequeue_op)
         self.assertAllEqual(dequeue_indices, dequeue_values)
         dequeued.extend(dequeue_indices)
 
@@ -184,10 +184,10 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue(dequeue_op, dequeued):
-        (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op)
+        (dequeue_indices, dequeue_values) = sess.run(dequeue_op)
         self.assertAllEqual(dequeue_indices, dequeue_values)
         dequeue_wait.acquire()
         dequeued.extend(dequeue_indices)
@@ -236,7 +236,7 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       dequeue_op = q.dequeue_many(100)
 
@@ -248,7 +248,7 @@ class PriorityQueueTest(test.TestCase):
       for t in enqueue_threads:
         t.start()
 
-      deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op)
+      deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op)
 
       for t in enqueue_threads:
         t.join()
@@ -276,7 +276,7 @@ class PriorityQueueTest(test.TestCase):
       side_value_1 = np.random.rand(1000).astype(bytes)
       q.enqueue_many((elem, side_value_0, side_value_1)).run()
       deq = q.dequeue_many(1000)
-      deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq)
+      deq_elem, deq_value_0, deq_value_1 = sess.run(deq)
 
       allowed = {}
       for e, v0, v1 in zip(elem, side_value_0, side_value_1):
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index c9cbe44a7f..b101da036e 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -307,9 +307,9 @@ class PyFuncTest(test.TestCase):
     with session_lib.Session() as sess:
       producer = iter(range(3))
       x, = script_ops.py_func(lambda: next(producer), [], [dtypes.int64])
-      self.assertEqual(self.evaluate(x), 0)
-      self.assertEqual(self.evaluate(x), 1)
-      self.assertEqual(self.evaluate(x), 2)
+      self.assertEqual(sess.run(x), 0)
+      self.assertEqual(sess.run(x), 1)
+      self.assertEqual(sess.run(x), 2)
 
   def testStateless(self):
     # Not using self.cached_session(), which disables optimization.
@@ -317,9 +317,9 @@ class PyFuncTest(test.TestCase):
       producer = iter(range(3))
       x, = script_ops.py_func(
           lambda: next(producer), [], [dtypes.int64], stateful=False)
-      self.assertEqual(self.evaluate(x), 0)
-      self.assertEqual(self.evaluate(x), 0)
-      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(sess.run(x), 0)
+      self.assertEqual(sess.run(x), 0)
+      self.assertEqual(sess.run(x), 0)
 
   def testGradientFunction(self):
     # Input to tf.py_func is necessary, otherwise get_gradient_function()
@@ -390,7 +390,7 @@ class PyFuncTest(test.TestCase):
     f = script_ops.py_func(
         do_nothing, [constant_op.constant(3, dtypes.int64)], [], stateful=False)
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(f), [])
+      self.assertEqual(sess.run(f), [])
 
   def _testExceptionHandling(self, py_exp, tf_exp, eager=False):
 
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 114481ed6a..617b724204 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -60,7 +60,7 @@ class QrOpTest(test.TestCase):
             q1, r1 = linalg_ops.qr(matrix1, full_matrices=full_matrices_)
             q2, r2 = linalg_ops.qr(matrix2, full_matrices=full_matrices_)
             all_ops += [q1, r1, q2, r2]
-      val = self.evaluate(all_ops)
+      val = sess.run(all_ops)
       for i in range(8):
         q = 4 * i
         self.assertAllEqual(val[q], val[q + 2])  # q1 == q2
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
index cab841741e..0023506b77 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
@@ -39,7 +39,7 @@ class MultinomialTest(test.TestCase):
           num_samples=1000000,
           seed=15)
       for _ in range(100):
-        x = self.evaluate(samples)
+        x = sess.run(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
@@ -57,7 +57,7 @@ class MultinomialTest(test.TestCase):
           num_samples=1000000,
           seed=15)
       for _ in range(100):
-        x = self.evaluate(samples)
+        x = sess.run(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
@@ -79,7 +79,7 @@ class MultinomialTest(test.TestCase):
       # we'll run out of memory if we try to draw 1e9 samples directly
       # really should fit in 12GB of memory...
       for _ in range(100):
-        x = self.evaluate(samples)
+        x = sess.run(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
index 8d2718c6d5..cfec4d08fb 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
@@ -70,8 +70,8 @@ class MultinomialTest(test.TestCase):
     with self.test_session(use_gpu=True) as sess:
       sample_op1, _ = self._make_ops(10)
       # Consecutive runs shouldn't yield identical output.
-      sample1a = self.evaluate(sample_op1)
-      sample1b = self.evaluate(sample_op1)
+      sample1a = sess.run(sample_op1)
+      sample1b = sess.run(sample_op1)
       self.assertFalse(np.equal(sample1a, sample1b).all())
 
   def testEagerOneOpMultipleStepsIndependent(self):
@@ -160,7 +160,7 @@ class MultinomialTest(test.TestCase):
     with self.test_session(use_gpu=True) as sess:
       random_seed.set_random_seed(1618)
       op = sampler(constant_op.constant(logits), num_samples)
-      d = self.evaluate(op)
+      d = sess.run(op)
 
     batch_size, num_classes = logits.shape
     freqs_mat = []
@@ -225,10 +225,8 @@ def native_op_vs_composed_ops(batch_size, num_classes, num_samples, num_iters):
     native_op = control_flow_ops.group(native_sampler(logits, num_samples))
     composed_op = control_flow_ops.group(composed_sampler(logits, num_samples))
 
-    native_dt = timeit.timeit(
-        lambda: sess.run(native_op), number=num_iters)
-    composed_dt = timeit.timeit(
-        lambda: sess.run(composed_op), number=num_iters)
+    native_dt = timeit.timeit(lambda: sess.run(native_op), number=num_iters)
+    composed_dt = timeit.timeit(lambda: sess.run(composed_op), number=num_iters)
     return native_dt, composed_dt
 
 
diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py
index d18e3feb04..606e8862c4 100644
--- a/tensorflow/python/kernel_tests/random/random_gamma_test.py
+++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py
@@ -48,7 +48,7 @@ class RandomGammaTest(test.TestCase):
             [num], alpha, beta=beta, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = self.evaluate(rng)
+          ret[i, :] = sess.run(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
index 76618316b2..6de894846b 100644
--- a/tensorflow/python/kernel_tests/random/random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/random_ops_test.py
@@ -49,9 +49,9 @@ class RandomOpTestCommon(test.TestCase):
         random_seed.set_random_seed(graph_seed)
       x = rng_func([num], min_or_mean, max_or_stddev, dtype=dtype, seed=op_seed)
 
-      y = self.evaluate(x)
-      z = self.evaluate(x)
-      w = self.evaluate(x)
+      y = sess.run(x)
+      z = sess.run(x)
+      w = sess.run(x)
 
       # We use exact equality here. If the random-number generator is producing
       # the same output, all three outputs will be bitwise identical.
@@ -69,7 +69,7 @@ class RandomNormalTest(RandomOpTestCommon):
             [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = self.evaluate(rng)
+          ret[i, :] = sess.run(rng)
       return ret
 
     return func
@@ -160,7 +160,7 @@ class TruncatedNormalTest(test.TestCase):
             [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = self.evaluate(rng)
+          ret[i, :] = sess.run(rng)
       return ret
 
     return func
@@ -256,7 +256,7 @@ class RandomUniformTest(RandomOpTestCommon):
             [num], minval=minv, maxval=maxv, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = self.evaluate(rng)
+          ret[i, :] = sess.run(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
index 47c0858db7..95e48101f6 100644
--- a/tensorflow/python/kernel_tests/random/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py
@@ -43,7 +43,7 @@ class RandomPoissonTest(test.TestCase):
         rng = random_ops.random_poisson(lam, [num], dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = self.evaluate(rng)
+          ret[i, :] = sess.run(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
index 5601b9864b..f3fcf1eff7 100644
--- a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
@@ -84,7 +84,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeue_t = q.dequeue()
       results = []
       for _ in range(2):
-        a, b = self.evaluate(dequeue_t)
+        a, b = sess.run(dequeue_t)
         results.append((a, b))
       a, b = sess.run(q.dequeue_many(3))
       for i in range(3):
@@ -101,7 +101,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -167,7 +167,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          self.evaluate(enqueue_op)
+          sess.run(enqueue_op)
 
       results = []
 
@@ -197,7 +197,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
       for _ in xrange(len(elems)):
-        x, y = self.evaluate(dequeued_t)
+        x, y = sess.run(dequeued_t)
         results.append((x, y))
       self.assertItemsEqual(elems, results)
 
@@ -321,7 +321,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
       for _ in range(8):
-        float_val, int_val = self.evaluate(dequeued_t)
+        float_val, int_val = sess.run(dequeued_t)
         results.append((float_val, [int_val[0], int_val[1]]))
       expected = list(zip(float_elems, int_elems)) * 2
       self.assertItemsEqual(expected, results)
@@ -368,20 +368,20 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       results = []
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape())
       results.append((float_val, int_val.tolist()))
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       results.append((float_val, int_val.tolist()))
 
       self.assertItemsEqual(zip(float_elems, int_elems), results)
@@ -402,21 +402,21 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       results = []
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       # dequeue_up_to has undefined shape.
       self.assertEqual([None], dequeued_t[0].get_shape().as_list())
       self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list())
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = self.evaluate(dequeued_t)
+      float_val, int_val = sess.run(dequeued_t)
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape())
       results.append((float_val, int_val.tolist()))
 
-      float_val, int_val = self.evaluate(dequeued_single_t)
+      float_val, int_val = sess.run(dequeued_single_t)
       results.append((float_val, int_val.tolist()))
 
       self.assertItemsEqual(zip(float_elems, int_elems), results)
@@ -442,7 +442,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -515,7 +515,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue(dequeue_op):
-        dequeued_elems.extend(self.evaluate(dequeue_op))
+        dequeued_elems.extend(sess.run(dequeue_op))
 
       threads = []
       for dequeue_op in dequeue_ops:
@@ -539,7 +539,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -566,7 +566,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        self.evaluate(enqueue_op)
+        sess.run(enqueue_op)
 
       def dequeue():
         dequeued_elems.extend(sess.run(dequeued_t).tolist())
@@ -727,7 +727,7 @@ class RandomShuffleQueueTest(test.TestCase):
       progress = []  # Must be mutable
 
       def dequeue():
-        self.assertItemsEqual(elems, self.evaluate(dequeued_t))
+        self.assertItemsEqual(elems, sess.run(dequeued_t))
         progress.append(1)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
@@ -922,7 +922,7 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -950,7 +950,7 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -987,11 +987,11 @@ class RandomShuffleQueueTest(test.TestCase):
       def blocking_enqueue():
         # Expect the operation to succeed since it will complete
         # before the queue is closed.
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"):
-          self.evaluate(blocking_enqueue_op)
+          sess.run(blocking_enqueue_op)
 
       thread1 = self.checkedThread(target=blocking_enqueue)
       thread1.start()
@@ -1001,7 +1001,7 @@ class RandomShuffleQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def blocking_close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       thread2 = self.checkedThread(target=blocking_close)
       thread2.start()
@@ -1032,7 +1032,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # This will block until the dequeue after the close.
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
       thread1 = self.checkedThread(target=blocking_enqueue)
       thread1.start()
@@ -1050,7 +1050,7 @@ class RandomShuffleQueueTest(test.TestCase):
         time.sleep(0.1)
 
       def blocking_close():
-        self.evaluate(close_op)
+        sess.run(close_op)
 
       thread2 = self.checkedThread(target=blocking_close)
       thread2.start()
@@ -1064,7 +1064,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # At this point the close operation will complete, so the next enqueue
       # will fail.
       with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"):
-        self.evaluate(blocking_enqueue_op)
+        sess.run(blocking_enqueue_op)
 
   def testSharedQueueSameSession(self):
     with self.cached_session():
@@ -1216,23 +1216,23 @@ class RandomShuffleQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_op)
+      sess.run(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_many_op)
+      sess.run(dequeue_many_op)
 
   def _blockingDequeueUpTo(self, sess, dequeue_up_to_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(dequeue_up_to_op)
+      sess.run(dequeue_up_to_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(enqueue_op)
+      sess.run(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(enqueue_many_op)
+      sess.run(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1393,14 +1393,14 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(self.evaluate(size_op), 5)
+      self.assertEqual(sess.run(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(self.evaluate(size_op), 5)
+      self.assertEqual(sess.run(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
diff --git a/tensorflow/python/kernel_tests/reader_ops_test.py b/tensorflow/python/kernel_tests/reader_ops_test.py
index 4d9b26f4eb..18a8a3d547 100644
--- a/tensorflow/python/kernel_tests/reader_ops_test.py
+++ b/tensorflow/python/kernel_tests/reader_ops_test.py
@@ -724,7 +724,7 @@ class AsyncReaderTest(test.TestCase):
         thread_data.append(thread_data_t(t, queue, output))
 
       # Start all readers. They are all blocked waiting for queue entries.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for d in thread_data:
         d.thread.start()
 
diff --git a/tensorflow/python/kernel_tests/record_input_test.py b/tensorflow/python/kernel_tests/record_input_test.py
index 74020667d9..ebb9872f22 100644
--- a/tensorflow/python/kernel_tests/record_input_test.py
+++ b/tensorflow/python/kernel_tests/record_input_test.py
@@ -54,7 +54,7 @@ class RecordInputOpTest(test.TestCase):
           batch_size=1,
           name="record_input").get_yield_op()
 
-      self.assertEqual(self.evaluate(yield_op), b"0000000000")
+      self.assertEqual(sess.run(yield_op), b"0000000000")
 
   def testRecordInputSimpleGzip(self):
     with self.cached_session() as sess:
@@ -73,7 +73,7 @@ class RecordInputOpTest(test.TestCase):
           compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op(
           )
 
-      self.assertEqual(self.evaluate(yield_op), b"0000000000")
+      self.assertEqual(sess.run(yield_op), b"0000000000")
 
   def testRecordInputSimpleZlib(self):
     with self.cached_session() as sess:
@@ -92,7 +92,7 @@ class RecordInputOpTest(test.TestCase):
           compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op(
           )
 
-      self.assertEqual(self.evaluate(yield_op), b"0000000000")
+      self.assertEqual(sess.run(yield_op), b"0000000000")
 
   def testRecordInputEpochs(self):
     files = 100
@@ -117,7 +117,7 @@ class RecordInputOpTest(test.TestCase):
       for _ in range(3):
         epoch_set = set()
         for _ in range(int(files * records_per_file / batches)):
-          op_list = self.evaluate(yield_op)
+          op_list = sess.run(yield_op)
           self.assertTrue(len(op_list) is batches)
           for r in op_list:
             self.assertTrue(r[0] not in epoch_set)
@@ -138,15 +138,15 @@ class RecordInputOpTest(test.TestCase):
 
         yield_op = records.get_yield_op()
         for _ in range(50):
-          self.evaluate(yield_op)
+          sess.run(yield_op)
 
   def testEmptyGlob(self):
     with self.cached_session() as sess:
       record_input = data_flow_ops.RecordInput(file_pattern="foo")
       yield_op = record_input.get_yield_op()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       with self.assertRaises(NotFoundError):
-        self.evaluate(yield_op)
+        sess.run(yield_op)
 
   def testBufferTooSmall(self):
     files = 10
@@ -171,7 +171,7 @@ class RecordInputOpTest(test.TestCase):
       for _ in range(3):
         epoch_set = set()
         for _ in range(int(files * records_per_file / batches)):
-          op_list = self.evaluate(yield_op)
+          op_list = sess.run(yield_op)
           self.assertTrue(len(op_list) is batches)
           for r in op_list:
             self.assertTrue(r[0] not in epoch_set)
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index 612b2c56a5..d1a295f42b 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -185,7 +185,7 @@ class SumReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -216,7 +216,7 @@ class SumReductionTest(BaseReductionTest):
       tf_arr = variables.Variable(arr)
       variables.global_variables_initializer().run()
       tf_mean = math_ops.reduce_mean(tf_arr, 0, False)
-      tf_out_mean = self.evaluate(tf_mean)
+      tf_out_mean = sess.run(tf_mean)
     self.assertAllClose(tf_out_mean, 1.)
 
   def testFloat32(self):
@@ -400,7 +400,7 @@ class MeanReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -473,7 +473,7 @@ class ProdReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -576,7 +576,7 @@ class MinReductionTest(test.TestCase):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -689,7 +689,7 @@ class MaxReductionTest(test.TestCase):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -817,7 +817,7 @@ class AllReductionTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         v = math_ops.reduce_all([True, True],
                                 constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, True)
 
   def testAll3D(self):
@@ -866,7 +866,7 @@ class AnyReductionTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         v = math_ops.reduce_any([True, True],
                                 constant_op.constant(0, dtype=dtype))
-        tf_v = self.evaluate(v)
+        tf_v = sess.run(v)
       self.assertAllEqual(tf_v, True)
 
   def testAll3D(self):
@@ -962,7 +962,7 @@ class CountNonzeroReductionTest(test.TestCase):
     # Test case for GitHub issue 18712
     with self.cached_session() as sess:
       v = math_ops.count_nonzero(constant_op.constant(["test"]))
-      self.assertAllClose(self.evaluate(v), 1)
+      self.assertAllClose(sess.run(v), 1)
 
   def testStringReduce1D(self):
     # Create a 1D array of strings
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index c351a18c8f..e85b04469b 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -153,7 +153,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
   def testCachedValueReadBeforeWrite(self):
     with self.cached_session() as sess:
       v = resource_variable_ops.ResourceVariable(0.0, caching_device="cpu:0")
-      self.evaluate(v.initializer)
+      sess.run(v.initializer)
       value, _ = sess.run([v, v.assign_add(1.0)])
       self.assertAllEqual(value, 0.0)
 
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index 1f1249727c..952ef34456 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -162,7 +162,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with self.session(use_gpu=True) as sess:
       sess.run(init)
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       self.assertAllClose(result, expected)
 
   def testSimpleResource(self):
@@ -190,7 +190,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with self.session(use_gpu=True) as sess:
       sess.run(init)
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       self.assertAllClose(result, expected)
 
   def testSimple3(self):
@@ -204,7 +204,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with self.session(use_gpu=True) as sess:
       sess.run(init)
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       self.assertAllClose(result, expected)
 
   def testVariableRankUpdate(self):
@@ -342,7 +342,7 @@ class StatefulScatterNdTest(test.TestCase):
 
     with session.Session() as sess:
       sess.run(init)
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       assert np.allclose(result, expected_result)
 
   # TODO(fpmc): Re-enable this test when gpu_pip test actually runs on a GPU.
@@ -421,7 +421,7 @@ class ScatterNdTest(test.TestCase):
                          b"", b"", b"seven"])
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       self.assertAllEqual(expected, result)
 
     # Same indice is updated twice by same value.
@@ -432,7 +432,7 @@ class ScatterNdTest(test.TestCase):
     expected = np.array([b"", b"", b"", b"bb", b"a", b"", b"", b"c"])
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       self.assertAllEqual(expected, result)
 
     # Same indice is updated twice by different value.
@@ -444,7 +444,7 @@ class ScatterNdTest(test.TestCase):
                 np.array([b"", b"", b"", b"cb", b"a", b"", b"", b"d"])]
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = self.evaluate(scatter)
+      result = sess.run(scatter)
       self.assertTrue(np.array_equal(result, expected[0]) or
                       np.array_equal(result, expected[1]))
 
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 8ca8e9dddf..85756b769d 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -63,7 +63,7 @@ class SelfAdjointEigTest(test.TestCase):
           e1 = linalg_ops.self_adjoint_eigvals(matrix1)
           e2 = linalg_ops.self_adjoint_eigvals(matrix2)
           all_ops += [e1, e2]
-      val = self.evaluate(all_ops)
+      val = sess.run(all_ops)
       self.assertAllEqual(val[0], val[2])
       # The algorithm is slightly different for compute_v being True and False,
       # so require approximate equality only here.
diff --git a/tensorflow/python/kernel_tests/session_ops_test.py b/tensorflow/python/kernel_tests/session_ops_test.py
index 73d85ddc07..03e1ae852f 100644
--- a/tensorflow/python/kernel_tests/session_ops_test.py
+++ b/tensorflow/python/kernel_tests/session_ops_test.py
@@ -37,7 +37,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = self.evaluate(h)
+      h = sess.run(h)
 
       # Feed a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -51,7 +51,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = self.evaluate(h)
+      h = sess.run(h)
 
       # Get the tensor from its handle.
       self.assertEqual(50, h.eval())
@@ -94,7 +94,7 @@ class SessionOpsTest(test.TestCase):
       # Initialize a handle.
       a = constant_op.constant(0)
       h = session_ops.get_session_handle(a)
-      h = self.evaluate(h)
+      h = sess.run(h)
 
       # Do some computation.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -111,7 +111,7 @@ class SessionOpsTest(test.TestCase):
       # Initialize a handle.
       a = constant_op.constant(0)
       h = session_ops.get_session_handle(a)
-      h = self.evaluate(h)
+      h = sess.run(h)
 
       # Do some computation.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -133,7 +133,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = self.evaluate(h)
+      h = sess.run(h)
 
       # Feed a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -144,7 +144,7 @@ class SessionOpsTest(test.TestCase):
       with ops.device(test.gpu_device_name()):
         a = constant_op.constant(10)
         h = session_ops.get_session_handle(a)
-        h = self.evaluate(h)
+        h = sess.run(h)
         self.assertEqual(100, sess.run(y, feed_dict={f: h.handle}))
 
   def testHandleDelete(self):
@@ -163,7 +163,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = self.evaluate(h)
+      h = sess.run(h)
 
       # Delete using a raw tensor handle.
       raw_h = h.get_raw_handle()
@@ -219,8 +219,8 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(2.0)
       b_handle_op = session_ops.get_session_handle(b)
 
-      a_handle = self.evaluate(a_handle_op)
-      b_handle = self.evaluate(b_handle_op)
+      a_handle = sess.run(a_handle_op)
+      b_handle = sess.run(b_handle_op)
 
       a_p, a_t = session_ops.get_session_tensor(a_handle.handle, dtypes.float32)
       b_p, b_t = session_ops.get_session_tensor(b_handle.handle, dtypes.float32)
@@ -288,10 +288,10 @@ class SessionOpsTest(test.TestCase):
       a = variables.Variable(12.0)
       inc_a = state_ops.assign_add(a, 2.0)
       b = math_ops.add(a, 5.0)
-      self.evaluate(a.initializer)
+      sess.run(a.initializer)
 
       h_a_read = sess.run(session_ops.get_session_handle(a.read_value()))
-      self.assertAllClose(12.0, self.evaluate(a))
+      self.assertAllClose(12.0, sess.run(a))
 
       self.assertAllClose(17.0, sess.run(b, feed_dict={a: h_a_read}))
       sess.run(inc_a)
diff --git a/tensorflow/python/kernel_tests/sets_test.py b/tensorflow/python/kernel_tests/sets_test.py
index e037f51e0f..8335e9c139 100644
--- a/tensorflow/python/kernel_tests/sets_test.py
+++ b/tensorflow/python/kernel_tests/sets_test.py
@@ -159,7 +159,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(None, op.get_shape().dims)
       self.assertEqual(dtypes.int32, op.dtype)
     with self.cached_session() as sess:
-      results = self.evaluate(ops)
+      results = sess.run(ops)
     self.assertAllEqual(results[0], results[1])
     return results[0]
 
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index a0506fbfc5..3e0eae326b 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -73,8 +73,8 @@ class ShapeOpsTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       tf_ans = array_ops.shape_n([x, x, x])
       tf_ans_64 = array_ops.shape_n([x, x, x], out_type=dtypes.int64)
-      result = self.evaluate(tf_ans)
-      result_64 = self.evaluate(tf_ans_64)
+      result = sess.run(tf_ans)
+      result_64 = sess.run(tf_ans_64)
     for i in range(3):
       self.assertAllEqual(np_ans, result[i])
       self.assertAllEqual(np_ans, result_64[i])
diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index de3351e543..c4e5b6f674 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -56,7 +56,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, 2)
 
     with self.session(use_gpu=True) as sess:
-      output = self.evaluate(reconstruction)
+      output = sess.run(reconstruction)
 
       expected_output = np.array([1, 1, 2, 2, 3, 2, 2, 1, 1])
 
@@ -99,7 +99,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = self.evaluate(reconstruction)
+      output = sess.run(reconstruction)
       string_output = [np.base_repr(x, self.bases[0]) for x in output]
 
       self.assertEqual(string_output, self.expected_string)
@@ -109,7 +109,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = self.evaluate(reconstruction)
+      output = sess.run(reconstruction)
 
       accumulator = True
       for i in range(self.batch_size):
@@ -125,7 +125,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = self.evaluate(reconstruction)
+      output = sess.run(reconstruction)
 
       string_output = [np.base_repr(int(x), self.bases[0]) for x in
                        np.squeeze(output)]
diff --git a/tensorflow/python/kernel_tests/sparse_add_op_test.py b/tensorflow/python/kernel_tests/sparse_add_op_test.py
index 845950bca7..a746830afb 100644
--- a/tensorflow/python/kernel_tests/sparse_add_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_add_op_test.py
@@ -91,7 +91,7 @@ class SparseAddTest(test.TestCase):
           sp_sum = sparse_ops.sparse_add(sp_a, sp_b)
           self.assertAllEqual((3, 3), sp_sum.get_shape())
 
-          sum_out = self.evaluate(sp_sum)
+          sum_out = sess.run(sp_sum)
 
           self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
           self.assertAllEqual(sum_out.indices, [[0, 1], [1, 0], [2, 0], [2, 1]])
@@ -104,7 +104,7 @@ class SparseAddTest(test.TestCase):
       sp_b = self._SparseTensor_3x3(negate=True)
 
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, 0.1)
-      sum_out = self.evaluate(sp_sum)
+      sum_out = sess.run(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, np.empty([0, 2]))
@@ -123,7 +123,7 @@ class SparseAddTest(test.TestCase):
 
       # two values should vanish: |.1| < .21, and |-.2| < .21
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.21)
-      sum_out = self.evaluate(sp_sum)
+      sum_out = sess.run(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0]])
@@ -132,7 +132,7 @@ class SparseAddTest(test.TestCase):
 
       # only .1 vanishes
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.11)
-      sum_out = self.evaluate(sp_sum)
+      sum_out = sess.run(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0], [2, 1]])
diff --git a/tensorflow/python/kernel_tests/sparse_concat_op_test.py b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
index a3d136c8d5..402c5eb4ea 100644
--- a/tensorflow/python/kernel_tests/sparse_concat_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
@@ -147,7 +147,7 @@ class SparseConcatTest(test.TestCase):
           self.assertEqual(sp_concat.values.get_shape(), [4])
           self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-          concat_out = self.evaluate(sp_concat)
+          concat_out = sess.run(sp_concat)
 
           self.assertAllEqual(concat_out.indices,
                               [[0, 2], [1, 0], [2, 0], [2, 2]])
@@ -169,7 +169,7 @@ class SparseConcatTest(test.TestCase):
             self.assertEqual(sp_concat.values.get_shape(), [8])
             self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-            concat_out = self.evaluate(sp_concat)
+            concat_out = sess.run(sp_concat)
 
             self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4],
                                                      [2, 0], [2, 2], [2, 3],
@@ -195,7 +195,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [7])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = self.evaluate(sp_concat)
+        concat_out = sess.run(sp_concat)
 
         self.assertAllEqual(
             concat_out.indices,
@@ -220,7 +220,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [10])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = self.evaluate(sp_concat)
+        concat_out = sess.run(sp_concat)
 
         self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [1, 8],
                                                  [2, 0], [2, 2], [2, 3], [2, 6],
@@ -244,7 +244,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [8])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = self.evaluate(sp_concat)
+        concat_out = sess.run(sp_concat)
 
         self.assertAllEqual(
             concat_out.indices,
@@ -302,8 +302,8 @@ class SparseConcatTest(test.TestCase):
           sp_concat_dim1 = sparse_ops.sparse_concat(
               concat_dim1, [sp_a, sp_b, sp_c, sp_d], expand_nonconcat_dim=True)
 
-          sp_concat_dim0_out = self.evaluate(sp_concat_dim0)
-          sp_concat_dim1_out = self.evaluate(sp_concat_dim1)
+          sp_concat_dim0_out = sess.run(sp_concat_dim0)
+          sp_concat_dim1_out = sess.run(sp_concat_dim1)
 
           self.assertAllEqual(sp_concat_dim0_out.indices,
                               [[0, 2], [1, 0], [2, 0], [2, 2], [4, 1], [5, 0],
diff --git a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
index 267275e771..a824d5c826 100644
--- a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
@@ -189,7 +189,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = self.evaluate(takeg_t)
+      val = sess.run(takeg_t)
       self.assertAllEqual([0, 1, 2], val.indices)
       self.assertAllEqual([[0.5, 0.5], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
@@ -209,7 +209,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = self.evaluate(takeg_t)
+      val = sess.run(takeg_t)
       self.assertAllEqual([0, 1, 2], val.indices)
       self.assertAllEqual([[1, 1], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
@@ -235,7 +235,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = self.evaluate(takeg_t)
+      val = sess.run(takeg_t)
       self.assertAllEqual(val.indices, [0, 1, 2])
       self.assertAllEqual(val.values, [[0.5, 0.5], [0, 2], [3, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
@@ -252,7 +252,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = self.evaluate(takeg_t)
+      val = sess.run(takeg_t)
       self.assertAllEqual(val.indices, [0, 1, 2])
       self.assertAllEqual(val.values, [[5, 5], [0, 20], [30, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
@@ -269,7 +269,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_indexed_slices_grad(1)
 
       def apply_indexed_slices_grad(accum_op):
-        self.evaluate(accum_op)
+        sess.run(accum_op)
 
       threads = [
           self.checkedThread(
@@ -281,7 +281,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = self.evaluate(takeg_t)
+      val = sess.run(takeg_t)
 
       expected_val = sum(elems) / len(elems)
       self._assertEqual_nparray(
@@ -303,7 +303,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_indexed_slices_grad(1)
 
       def apply_indexed_slices_grad(accum_op):
-        self.evaluate(accum_op)
+        sess.run(accum_op)
 
       threads = [
           self.checkedThread(target=apply_indexed_slices_grad, args=(o,))
@@ -315,7 +315,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = self.evaluate(takeg_t)
+      val = sess.run(takeg_t)
 
       expected_val = 550.0
       self._assertEqual_nparray(
@@ -338,13 +338,13 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       def apply_indexed_slices_grad():
         for accum_op in accum_ops:
           time.sleep(1.0)
-          self.evaluate(accum_op)
+          sess.run(accum_op)
 
       apply_indexed_slices_grad_thread = self.checkedThread(
           target=apply_indexed_slices_grad)
 
       def take_grad():
-        t = self.evaluate(takeg_t)
+        t = sess.run(takeg_t)
         results.append(t)
 
       threads = [self.checkedThread(target=take_grad) for _ in range(10)]
@@ -378,7 +378,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
       def apply_indexed_slices_grad():
         for accum_op in accum_ops:
-          self.evaluate(accum_op)
+          sess.run(accum_op)
 
       def take_grad():
         results.append(sess.run(takeg_t))
@@ -394,7 +394,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
   def _blocking_takeg(self, sess, takeg_op):
     with self.assertRaisesOpError("was cancelled"):
-      self.evaluate(takeg_op)
+      sess.run(takeg_op)
 
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
index 8451b96c56..17e867439a 100644
--- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
@@ -43,7 +43,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_dense(self):
     """Tests only dense inputs."""
@@ -63,7 +63,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_integer_mixed_string_sparse(self):
     """Tests mixed type."""
@@ -77,7 +77,7 @@ class SparseCrossOpTest(test.TestCase):
         '55555_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_integer_mixed_string_dense(self):
     """Tests mixed dense inputs."""
@@ -95,7 +95,7 @@ class SparseCrossOpTest(test.TestCase):
         '999999_X_batch2-FC2-F1', '999999_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_sparse_cross_dense(self):
     """Tests sparse and dense inputs."""
@@ -112,7 +112,7 @@ class SparseCrossOpTest(test.TestCase):
             'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
         ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_integer_sparse_input(self):
     """Tests mixed type sparse and dense inputs."""
@@ -128,7 +128,7 @@ class SparseCrossOpTest(test.TestCase):
             '5555_X_batch2-FC2-F1', '5555_X_batch2-FC2-F2'
         ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_permutation_3x3x3(self):
     """Tests 3x3x3 permutation."""
@@ -170,7 +170,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F3'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_permutation_3x1x2(self):
     """Tests 3x1x2 permutation."""
@@ -189,7 +189,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_large_batch(self):
     """Tests with large batch size to force multithreading."""
@@ -222,7 +222,7 @@ class SparseCrossOpTest(test.TestCase):
 
     expected_out = self._sparse_tensor(col_out)
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_one_column_empty(self):
     """Tests when one column is empty.
@@ -235,7 +235,7 @@ class SparseCrossOpTest(test.TestCase):
         self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']])
     ])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_empty(self.evaluate(op))
+      self._assert_sparse_tensor_empty(sess.run(op))
 
   def test_some_columns_empty(self):
     """Tests when more than one columns are empty.
@@ -254,7 +254,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2'
     ]], 2)
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_all_columns_empty(self):
     """Tests when all columns are empty.
@@ -267,7 +267,7 @@ class SparseCrossOpTest(test.TestCase):
         self._sparse_tensor([])
     ])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_empty(self.evaluate(op))
+      self._assert_sparse_tensor_empty(sess.run(op))
 
   def test_hashed_zero_bucket_no_hash_key(self):
     op = sparse_ops.sparse_cross_hashed([
@@ -278,7 +278,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[1971693436396284976]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_hashed_zero_bucket(self):
     op = sparse_ops.sparse_cross_hashed(
@@ -291,7 +291,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[4847552627144134031]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   # TODO(sibyl-Aix6ihai): Add benchmark to compare Hashed vs Non-hashed.
   def test_hashed_no_hash_key(self):
@@ -305,7 +305,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[83]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_hashed_output(self):
     op = sparse_ops.sparse_cross_hashed(
@@ -319,7 +319,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[31]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
+      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
 
   def test_hashed__has_no_collision(self):
     """Tests that fingerprint concatenation has no collisions."""
@@ -345,7 +345,7 @@ class SparseCrossOpTest(test.TestCase):
         ],
         num_buckets=1000)
     with self.cached_session() as sess:
-      out = self.evaluate(op)
+      out = sess.run(op)
       self.assertEqual(6, len(out.values))
       self.assertAllEqual([[0, i] for i in range(6)], out.indices)
       self.assertTrue(all(x < 1000 and x >= 0 for x in out.values))
diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index ad253595d2..db3f6c44e2 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -154,7 +154,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
                        sparse_tensor.SparseTensor.from_value(values_v)):
           sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-          output = self.evaluate(sp_output)
+          output = sess.run(sp_output)
           self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat32(self):
@@ -163,7 +163,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float32)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64(self):
@@ -172,7 +172,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt32AndFloat32NonCanonicalOrder(self):
@@ -182,7 +182,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size, already_sorted=True)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testInt64AndFloat32NonCanonicalOrder(self):
@@ -192,7 +192,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size, already_sorted=True)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testInt64AndFloat64NonCanonicalOrder(self):
@@ -203,7 +203,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size_tensor, already_sorted=True)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testShouldSetLastDimensionInDynamicShape(self):
@@ -261,7 +261,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float32)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64(self):
@@ -270,7 +270,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64Shape(self):
@@ -279,7 +279,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
 
@@ -302,7 +302,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
         to_retain = np.array([1, 0, 0, 1, 1, 0], dtype=np.bool)
         sp_output = sparse_ops.sparse_retain(sp_input, to_retain)
 
-        output = self.evaluate(sp_output)
+        output = sess.run(sp_output)
 
         self.assertAllEqual(output.indices, [[0, 0], [1, 4], [3, 2]])
         self.assertAllEqual(output.values, [0, 14, 32])
@@ -314,7 +314,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
       to_retain = np.zeros((6,), dtype=np.bool)
       sp_output = sparse_ops.sparse_retain(sp_input, to_retain)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
 
       self.assertAllEqual(output.indices, np.array([]).reshape((0, 2)))
       self.assertAllEqual(output.values, [])
@@ -365,7 +365,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       new_shape = np.array([3, 6, 7], dtype=np.int64)
       sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -378,7 +378,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       new_shape = np.array([3, 6, 7], dtype=np.int64)
       sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -404,7 +404,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       sp_input = self._SparseTensor_2x5x6()
       sp_output = sparse_ops.sparse_reset_shape(sp_input)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -416,7 +416,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       sp_input = self._SparseTensor_2x5x6_Empty()
       sp_output = sparse_ops.sparse_reset_shape(sp_input)
 
-      output = self.evaluate(sp_output)
+      output = sess.run(sp_output)
 
       self.assertAllEqual(output.indices.shape, [0, 3])
       self.assertAllEqual(output.values.shape, [0])
@@ -591,8 +591,8 @@ class SparseAddTest(test_util.TensorFlowTestCase):
     sp_output = sparse_ops.sparse_add(sp_input, sp_input)
 
     with self.session(use_gpu=False) as sess:
-      self.evaluate(variables.global_variables_initializer())
-      output = self.evaluate(sp_output)
+      sess.run(variables.global_variables_initializer())
+      output = sess.run(sp_output)
       self.assertAllEqual(output.values, [2])
 
 
diff --git a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
index bbf2f39202..7b83ae5177 100644
--- a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
@@ -60,7 +60,7 @@ class SparseReorderTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6(np.arange(6))
       sp_output = sparse_ops.sparse_reorder(input_val)
 
-      output_val = self.evaluate(sp_output)
+      output_val = sess.run(sp_output)
       self.assertAllEqual(output_val.indices, input_val.indices)
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
@@ -83,7 +83,7 @@ class SparseReorderTest(test.TestCase):
         input_val = self._SparseTensorValue_5x6(np.random.permutation(6))
         sp_output = sparse_ops.sparse_reorder(input_val)
 
-        output_val = self.evaluate(sp_output)
+        output_val = sess.run(sp_output)
         self.assertAllEqual(output_val.indices, expected_output_val.indices)
         self.assertAllEqual(output_val.values, expected_output_val.values)
         self.assertAllEqual(output_val.dense_shape,
diff --git a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
index 918af27091..f7be397c33 100644
--- a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
@@ -81,7 +81,7 @@ class SparseReshapeTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6()
       sp_output = sparse_ops.sparse_reshape(input_val, [5, 6])
 
-      output_val = self.evaluate(sp_output)
+      output_val = sess.run(sp_output)
       self.assertAllEqual(output_val.indices, input_val.indices)
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
@@ -151,7 +151,7 @@ class SparseReshapeTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6()
       sp_output = sparse_ops.sparse_reshape(input_val, [2, 3, 5])
 
-      output_val = self.evaluate(sp_output)
+      output_val = sess.run(sp_output)
       self.assertAllEqual(output_val.indices,
                           np.array([[0, 0, 0], [0, 1, 1], [0, 1, 4], [0, 2, 0],
                                     [1, 1, 0], [1, 1, 1]]))
diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
index 39a9ab9b49..b24a086969 100644
--- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
@@ -73,7 +73,7 @@ class SerializeSparseTest(test.TestCase):
       serialized = serialize_fn(sp_input, out_type=out_type)
       sp_deserialized = deserialize_fn(serialized, dtype=dtypes.int32)
 
-      indices, values, shape = self.evaluate(sp_deserialized)
+      indices, values, shape = sess.run(sp_deserialized)
 
       self.assertAllEqual(indices, sp_input[0])
       self.assertAllEqual(values, sp_input[1])
diff --git a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
index e63ba8f697..e08464a701 100644
--- a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
@@ -88,7 +88,7 @@ class SparseTensorsMapTest(test.TestCase):
       sp_out = take_many_sparse_from_tensors_map(
           sparse_map_op=handle0.op, sparse_handles=handles_concat)
 
-      combined_indices, combined_values, combined_shape = self.evaluate(sp_out)
+      combined_indices, combined_values, combined_shape = sess.run(sp_out)
 
       self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
       self.assertAllEqual(combined_indices[:6, 1:], sp_input0[0])
@@ -114,8 +114,7 @@ class SparseTensorsMapTest(test.TestCase):
       sp_roundtrip = take_many_sparse_from_tensors_map(
           sparse_map_op=handle.op, sparse_handles=sparse_handles)
 
-      combined_indices, combined_values, combined_shape = self.evaluate(
-          sp_roundtrip)
+      combined_indices, combined_values, combined_shape = sess.run(sp_roundtrip)
 
       self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
       self.assertAllEqual(combined_indices[:6, 1:], input0_val[0])
@@ -172,7 +171,7 @@ class SparseTensorsMapTest(test.TestCase):
     with self.session(use_gpu=False) as sess:
       input_val = self._SparseTensorValue_5x6(np.arange(6))
       handle = add_sparse_to_tensors_map(input_val)
-      handle_value = self.evaluate(handle)
+      handle_value = sess.run(handle)
       bad_handle = handle_value + 10
       sp_roundtrip = take_many_sparse_from_tensors_map(
           sparse_map_op=handle.op, sparse_handles=[handle_value, bad_handle])
@@ -213,8 +212,8 @@ class BenchmarkSparseTensorsMapVsSerialization(test.Benchmark):
 
         variables.global_variables_initializer().run()
 
-        st_roundtrip_values = self.evaluate(st_roundtrip)
-        st_deserialized_values = self.evaluate(st_deserialized)
+        st_roundtrip_values = sess.run(st_roundtrip)
+        st_deserialized_values = sess.run(st_deserialized)
         np.testing.assert_equal(st_roundtrip_values.values,
                                 st_deserialized_values.values)
         np.testing.assert_equal(st_roundtrip_values.indices,
diff --git a/tensorflow/python/kernel_tests/stage_op_test.py b/tensorflow/python/kernel_tests/stage_op_test.py
index b1e7ce5d62..b814843b86 100644
--- a/tensorflow/python/kernel_tests/stage_op_test.py
+++ b/tensorflow/python/kernel_tests/stage_op_test.py
@@ -152,11 +152,11 @@ class StageTest(test.TestCase):
 
     with self.session(use_gpu=True, graph=G) as sess:
       sess.run(stage, feed_dict={x: -1})
-      self.assertEqual(self.evaluate(size), 1)
+      self.assertEqual(sess.run(size), 1)
       sess.run(stage, feed_dict={x: -1})
-      self.assertEqual(self.evaluate(size), 2)
+      self.assertEqual(sess.run(size), 2)
       sess.run(clear)
-      self.assertEqual(self.evaluate(size), 0)
+      self.assertEqual(sess.run(size), 0)
 
   def testCapacity(self):
     capacity = 3
@@ -210,14 +210,14 @@ class StageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(self.evaluate(size) == capacity)
+      self.assertTrue(sess.run(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
-        self.assertTrue(self.evaluate(ret) == [i])
+        self.assertTrue(sess.run(ret) == [i])
 
       # It should now be empty
-      self.assertTrue(self.evaluate(size) == 0)
+      self.assertTrue(sess.run(size) == 0)
 
   def testMemoryLimit(self):
     memory_limit = 512 * 1024  # 512K
@@ -274,13 +274,13 @@ class StageTest(test.TestCase):
                                              capacity))
 
       # Should have capacity elements in the staging area
-      self.assertTrue(self.evaluate(size) == capacity)
+      self.assertTrue(sess.run(size) == capacity)
 
       # Clear the staging area completely
       for i in range(n):
-        self.assertTrue(np.all(self.evaluate(ret)[0] == i))
+        self.assertTrue(np.all(sess.run(ret)[0] == i))
 
-      self.assertTrue(self.evaluate(size) == 0)
+      self.assertTrue(sess.run(size) == 0)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/kernel_tests/string_length_op_test.py b/tensorflow/python/kernel_tests/string_length_op_test.py
index 0c68f0cadd..57db7302b1 100644
--- a/tensorflow/python/kernel_tests/string_length_op_test.py
+++ b/tensorflow/python/kernel_tests/string_length_op_test.py
@@ -29,7 +29,7 @@ class StringLengthOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       lengths = string_ops.string_length(strings)
-      values = self.evaluate(lengths)
+      values = sess.run(lengths)
       self.assertAllEqual(values, [[[1, 2], [3, 4], [5, 6]]])
 
   def testUnit(self):
diff --git a/tensorflow/python/kernel_tests/string_split_op_test.py b/tensorflow/python/kernel_tests/string_split_op_test.py
index 92e13db0f7..b968e885ed 100644
--- a/tensorflow/python/kernel_tests/string_split_op_test.py
+++ b/tensorflow/python/kernel_tests/string_split_op_test.py
@@ -34,7 +34,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
@@ -44,7 +44,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, delimiter="")
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4],
                                     [1, 0], [1, 1], [1, 2], [1, 3], [2, 0],
                                     [2, 1], [2, 2], [2, 3]])
@@ -62,7 +62,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(
           indices,
           [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
@@ -74,7 +74,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, delimiter=" .")
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(
           indices,
           [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
@@ -92,13 +92,13 @@ class StringSplitOpTest(test.TestCase):
           ValueError, string_ops.string_split, strings, delimiter=["a"])
 
       tokens = string_ops.string_split(strings, delimiter="|")
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
       self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
       self.assertAllEqual(shape, [2, 2])
 
       tokens = string_ops.string_split(strings, delimiter="| ")
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"])
       self.assertAllEqual(shape, [2, 2])
@@ -145,7 +145,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, "#", skip_empty=False)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1],
                                     [2, 0], [2, 1], [2, 2]])
@@ -154,7 +154,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, "#")
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(values, [b"a", b"b", b"c"])
       self.assertAllEqual(indices, [[0, 0], [1, 0], [2, 0]])
       self.assertAllEqual(shape, [3, 1])
@@ -167,7 +167,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
@@ -182,7 +182,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep="<>")
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(
           indices, [[0, 0], [0, 1], [0, 2],
                     [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6]])
@@ -200,7 +200,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep=',')
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2],
                                     [1, 0], [1, 1], [1, 2], [1, 3], [1, 4]])
       self.assertAllEqual(values, [b"1", b"2", b"3",
@@ -217,7 +217,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2],
                                     [1, 0], [1, 1], [1, 2]])
       self.assertAllEqual(values, [b"1", b"2", b"3", b"4", b"5", b"6"])
@@ -233,7 +233,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep=',', maxsplit=1)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"1", b"2,3", b"4", b"5,,6,"])
@@ -249,7 +249,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, maxsplit=1)
-      indices, values, shape = self.evaluate(tokens)
+      indices, values, shape = sess.run(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"1", b"2 3", b"4", b"5    6  "])
diff --git a/tensorflow/python/kernel_tests/string_strip_op_test.py b/tensorflow/python/kernel_tests/string_strip_op_test.py
index edff3862ff..1e404b7146 100644
--- a/tensorflow/python/kernel_tests/string_strip_op_test.py
+++ b/tensorflow/python/kernel_tests/string_strip_op_test.py
@@ -30,7 +30,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = self.evaluate(output)
+      output = sess.run(output)
       self.assertAllEqual(output, [b"pigs on the wing", b"animals"])
 
   def test_string_strip_2d(self):
@@ -39,7 +39,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = self.evaluate(output)
+      output = sess.run(output)
       self.assertAllEqual(output, [[b"pigs on the wing", b"animals"],
                                    [b"hello", b"world"]])
 
@@ -48,7 +48,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = self.evaluate(output)
+      output = sess.run(output)
       self.assertAllEqual(output, [b"hello", b"", b"world", b""])
 
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
index 1547c55f8b..63ce77b9d5 100644
--- a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
@@ -60,7 +60,7 @@ class SummaryV1AudioOpTest(test.TestCase):
         sample_rate = 8000
         summ = summary.audio(
             "snd", const, max_outputs=3, sample_rate=sample_rate)
-        value = self.evaluate(summ)
+        value = sess.run(summ)
         self.assertEqual([], summ.get_shape())
         audio_summ = self._AsSummary(value)
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
index e1b24756f3..094606944f 100644
--- a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
@@ -70,7 +70,7 @@ class SummaryV1ImageOpTest(test.TestCase):
 
           # Summarize
           summ = summary.image("img", const)
-          value = self.evaluate(summ)
+          value = sess.run(summ)
           self.assertEqual([], summ.get_shape())
           image_summ = self._AsSummary(value)
 
@@ -97,7 +97,7 @@ class SummaryV1ImageOpTest(test.TestCase):
 
         # Summarize
         summ = summary.image("img", tf_images)
-        value = self.evaluate(summ)
+        value = sess.run(summ)
         self.assertEqual([], summ.get_shape())
         image_summ = self._AsSummary(value)
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_ops_test.py b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
index 1206cb7013..6c4e106b11 100644
--- a/tensorflow/python/kernel_tests/summary_v1_ops_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
@@ -42,7 +42,7 @@ class SummaryV1OpsTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant([10.0, 20.0])
       summ = logging_ops.scalar_summary(["c1", "c2"], const, name="mysumm")
-      value = self.evaluate(summ)
+      value = sess.run(summ)
     self.assertEqual([], summ.get_shape())
     self.assertProtoEquals("""
       value { tag: "c1" simple_value: 10.0 }
@@ -53,7 +53,7 @@ class SummaryV1OpsTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant([10.0, 20.0])
       summ = logging_ops.scalar_summary(["c1", "c2"], const)
-      value = self.evaluate(summ)
+      value = sess.run(summ)
     self.assertEqual([], summ.get_shape())
     self.assertProtoEquals("""
       value { tag: "c1" simple_value: 10.0 }
@@ -66,7 +66,7 @@ class SummaryV1OpsTest(test.TestCase):
       summ1 = summary.histogram("h", const)
       summ2 = logging_ops.scalar_summary("c", const)
       merge = summary.merge([summ1, summ2])
-      value = self.evaluate(merge)
+      value = sess.run(merge)
     self.assertEqual([], merge.get_shape())
     self.assertProtoEquals("""
       value {
diff --git a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
index 71251f5602..34f771679a 100644
--- a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
@@ -68,7 +68,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(10.0)
       summ = summary_lib.tensor_summary("foo", const)
-      result = self.evaluate(summ)
+      result = sess.run(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -79,7 +79,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(s)
       summ = summary_lib.tensor_summary("foo", const)
-      result = self.evaluate(summ)
+      result = sess.run(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -89,7 +89,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = array_ops.ones([5, 5, 5])
       summ = summary_lib.tensor_summary("foo", const)
-      result = self.evaluate(summ)
+      result = sess.run(summ)
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
     self._AssertNumpyEq(n, np.ones([5, 5, 5]))
@@ -99,7 +99,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(strings)
       summ = summary_lib.tensor_summary("foo", const)
-      result = self.evaluate(summ)
+      result = sess.run(summ)
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
     self._AssertNumpyEq(n, strings)
@@ -109,7 +109,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(bools)
       summ = summary_lib.tensor_summary("foo", const)
-      result = self.evaluate(summ)
+      result = sess.run(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -119,7 +119,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
 
       def get_description(summary_op):
-        summ_str = self.evaluate(summary_op)
+        summ_str = sess.run(summary_op)
         summ = summary_pb2.Summary()
         summ.ParseFromString(summ_str)
         return summ.value[0].metadata
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index 589172e4b7..32c97a7b19 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -68,7 +68,7 @@ class SvdOpTest(test.TestCase):
             s2 = linalg_ops.svd(
                 matrix2, compute_uv=compute_uv_, full_matrices=full_matrices_)
             all_ops += [s1, s2]
-      val = self.evaluate(all_ops)
+      val = sess.run(all_ops)
       for i in range(2):
         s = 6 * i
         self.assertAllEqual(val[s], val[s + 3])  # s1 == s2
@@ -158,7 +158,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
         s_tf = linalg_ops.svd(
             x_tf, compute_uv=compute_uv_, full_matrices=full_matrices_)
         if use_static_shape_:
-          s_tf_val = self.evaluate(s_tf)
+          s_tf_val = sess.run(s_tf)
         else:
           s_tf_val = sess.run(s_tf, feed_dict={x_tf: x_np})
 
diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py
index a187fa115c..9dcdaa61ed 100644
--- a/tensorflow/python/kernel_tests/template_test.py
+++ b/tensorflow/python/kernel_tests/template_test.py
@@ -104,10 +104,10 @@ class TemplateTest(test.TestCase):
     train_op = optimizer.minimize(train_loss)
 
     with session.Session() as sess:
-      self.evaluate(variables.global_variables_initializer())
-      initial_test_loss = self.evaluate(test_loss)
-      self.evaluate(train_op)
-      final_test_loss = self.evaluate(test_loss)
+      sess.run(variables.global_variables_initializer())
+      initial_test_loss = sess.run(test_loss)
+      sess.run(train_op)
+      final_test_loss = sess.run(test_loss)
 
     # Parameters are tied, so the loss should have gone down when we trained it.
     self.assertLess(final_test_loss, initial_test_loss)
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 4ee1c27a87..7e8db8947b 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -751,7 +751,7 @@ class TensorArrayTest(test.TestCase):
                 [-0.5, 1.5],  # read(0) gradient
                 [20.0, 30.0, 40.0, 50.0]
             ])  # concat gradient
-      grad_vals = self.evaluate(grad_r)  # 2 + 2 entries
+      grad_vals = sess.run(grad_r)  # 2 + 2 entries
 
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
@@ -1286,7 +1286,7 @@ class TensorArrayTest(test.TestCase):
       r = w1.stack()
       self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
-      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
+      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
 
   @test_util.disable_control_flow_v2("b/117943489")
   def testSkipEagerTensorArrayUnpackDynamic(self):
@@ -1303,7 +1303,7 @@ class TensorArrayTest(test.TestCase):
       r = w1.concat()
       self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
-      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
+      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
 
   def _testTensorArrayEvalEmpty(self):
     with self.cached_session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
index d1c7b41c7b..4ad5ee4103 100644
--- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
@@ -42,7 +42,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, strings)
 
       outputs = string_ops.unicode_transcode(
@@ -52,7 +52,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, strings)
 
       outputs = string_ops.unicode_transcode(
@@ -62,7 +62,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, strings)
 
   def test_transcode_utf16_to_utf8(self):
@@ -77,7 +77,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, expected)
 
   def test_transcode_bad_utf8(self):
@@ -90,7 +90,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=True)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"  ")
 
       outputs = string_ops.unicode_transcode(
@@ -100,7 +100,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
   def test_transcode_bad_utf8_with_some_good(self):
@@ -113,7 +113,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"abc abcdefg")
 
   def test_transcode_bad_utf8_with_defaults(self):
@@ -121,7 +121,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"\x00\xef\xbf\xbd")
 
   def test_transcode_bad_utf8_with_space_replacement(self):
@@ -130,7 +130,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8",
           replacement_char=ord(" "))
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
   def test_transcode_bad_utf8_with_strict_errors(self):
@@ -165,7 +165,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           input_encoding="UTF-8",
           output_encoding="UTF-8",
           errors="ignore")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"\x00")
 
   def test_transcode_bad_utf8_with_elision_including_control_chars(self):
@@ -177,7 +177,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           errors="ignore",
           replace_control_characters=True)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"")
 
   def test_transcode_bad_utf8_termination_with_defaults(self):
@@ -185,7 +185,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"a\xef\xbf\xbd")   # 0xFFFD
 
   def test_transcode_utf8_with_replacement_char(self):
@@ -194,13 +194,13 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       outputs = string_ops.unicode_transcode(
           strings, input_encoding="UTF-8", output_encoding="UTF-8",
           errors="strict")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
 
       outputs = string_ops.unicode_transcode(
           strings, input_encoding="UTF-8", output_encoding="UTF-8",
           errors="replace", replacement_char=ord("?"))
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
 
   def test_transcode_utf8_to_utf16(self):
@@ -214,7 +214,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-16-BE",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       print("values=", values)
       self.assertAllEqual(values, expected)
 
@@ -230,7 +230,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, expected)
 
   def test_transcode_utf8_to_utf32(self):
@@ -243,7 +243,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-32-BE",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, expected)
 
   # Documentation in ICU suggests that getNextUChar may produce a different
@@ -258,7 +258,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, strings)
 
   def test_transcode_utf8_with_bom(self):
@@ -266,12 +266,12 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"\xef\xbb\xbfabcdefg")  # BOM preserved
 
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-8", output_encoding="UTF-16-BE")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       utf16expected = bom_string.decode("UTF-8").encode("UTF-16-BE")
       self.assertAllEqual(values, utf16expected)
 
@@ -280,20 +280,20 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-BE", output_encoding="UTF-8")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       # BOM is preserved in output
       self.assertAllEqual(values, b"\xef\xbb\xbfa")
 
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       # mangled BOM and value from (incorrect) LE encoding
       self.assertAllEqual(values, b"\xef\xbf\xbe\xe6\x84\x80")
 
       bom_string = b"\xff\xfe\x61\x00"  # Little-endian BOM with 'a' encoded
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
-      values = self.evaluate(outputs)
+      values = sess.run(outputs)
       self.assertAllEqual(values, b"\xef\xbb\xbfa")
 
   @parameterized.parameters(
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index de086860ea..a8a66a412d 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -438,15 +438,15 @@ class VariableScopeTest(test.TestCase):
         sess.run(v0)
       # We should be able to initialize and run v1 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      self.evaluate(v1.initializer)
-      self.assertEqual(1, self.evaluate(v1))
+      sess.run(v1.initializer)
+      self.assertEqual(1, sess.run(v1))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(v0)
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(add)
       # If we initialize v0 we should be able to run 'add'.
-      self.evaluate(v0.initializer)
+      sess.run(v0.initializer)
       sess.run(add)
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
@@ -490,10 +490,10 @@ class VariableScopeTest(test.TestCase):
       v2 = var_dict["v2"]
       # We should be able to initialize and run v1 and v2 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      self.evaluate(v1.initializer)
-      self.assertEqual([1], self.evaluate(v1))
-      self.evaluate(v2.initializer)
-      self.assertEqual([2], self.evaluate(v2))
+      sess.run(v1.initializer)
+      self.assertEqual([1], sess.run(v1))
+      sess.run(v2.initializer)
+      self.assertEqual([2], sess.run(v2))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(v0)
@@ -501,7 +501,7 @@ class VariableScopeTest(test.TestCase):
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
         sess.run(add)
       # If we initialize v0 we should be able to run 'add'.
-      self.evaluate(v0.initializer)
+      sess.run(v0.initializer)
       sess.run(add)
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 48b32f06aa..0634dfa2d8 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -48,8 +48,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., lambda v: v * v, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(ret), 16.)
-      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertEqual(sess.run(ret), 16.)
+      self.assertSequenceEqual(sess.run(grad), [32.])
 
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
@@ -65,8 +65,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # Note: This is simply d_ret[0]/d_x since d_ret[1]/d_x is 0.
     grad = gradients_impl.gradients(ret, [x])  # [2*x*y]
     with self.cached_session() as sess:
-      self.assertSequenceEqual(self.evaluate(ret), [45., 3.])
-      self.assertSequenceEqual(self.evaluate(grad), [9.])
+      self.assertSequenceEqual(sess.run(ret), [45., 3.])
+      self.assertSequenceEqual(sess.run(grad), [9.])
 
   def testMultipleLoopVars(self):
     x = constant_op.constant(5.)
@@ -88,13 +88,13 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grady_1 = gradients_impl.gradients(ret[1], [y])  # [x + 1]
     grady_2 = gradients_impl.gradients(ret, [y])  # [2*x*y + x**2 + x + 1]
     with self.cached_session() as sess:
-      self.assertSequenceEqual(self.evaluate(ret), [120., 23.])
-      self.assertSequenceEqual(self.evaluate(gradx_0), [39.])
-      self.assertSequenceEqual(self.evaluate(gradx_1), [4.])
-      self.assertSequenceEqual(self.evaluate(gradx_2), [43.])
-      self.assertSequenceEqual(self.evaluate(grady_0), [55.])
-      self.assertSequenceEqual(self.evaluate(grady_1), [6.])
-      self.assertSequenceEqual(self.evaluate(grady_2), [61.])
+      self.assertSequenceEqual(sess.run(ret), [120., 23.])
+      self.assertSequenceEqual(sess.run(gradx_0), [39.])
+      self.assertSequenceEqual(sess.run(gradx_1), [4.])
+      self.assertSequenceEqual(sess.run(gradx_2), [43.])
+      self.assertSequenceEqual(sess.run(grady_0), [55.])
+      self.assertSequenceEqual(sess.run(grady_1), [6.])
+      self.assertSequenceEqual(sess.run(grady_2), [61.])
 
   def testMultipleWhileLoops(self):
     x = constant_op.constant(2.)
@@ -103,8 +103,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret2, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
-      self.assertSequenceEqual(self.evaluate(grad), [32.])
-      self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
+      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertSequenceEqual(sess.run(grad_grad), [48.])
 
   def testDoubleDerivative(self):
     x = constant_op.constant(2.)
@@ -112,9 +112,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(ret), 16.)
-      self.assertSequenceEqual(self.evaluate(grad), [32.])
-      self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
+      self.assertEqual(sess.run(ret), 16.)
+      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertSequenceEqual(sess.run(grad_grad), [48.])
 
   def testPruning(self):
     x = constant_op.constant(1)
@@ -157,8 +157,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v + y < 9., lambda v: v * 3., [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(ret), 18.)
-      self.assertSequenceEqual(self.evaluate(grad), [9.])
+      self.assertEqual(sess.run(ret), 18.)
+      self.assertSequenceEqual(sess.run(grad), [9.])
 
   def testCaptureExternalTensorInBody(self):
     x = constant_op.constant(2.)
@@ -166,8 +166,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., lambda v: v * y, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(ret), 18.)
-      self.assertSequenceEqual(self.evaluate(grad), [9.])
+      self.assertEqual(sess.run(ret), 18.)
+      self.assertSequenceEqual(sess.run(grad), [9.])
 
   def testLoopWithTensorListPushBack(self):
     x = constant_op.constant(2.)
@@ -188,7 +188,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
-      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertSequenceEqual(sess.run(grad), [32.])
 
   def testDuplicateAccumulator(self):
     x = constant_op.constant(2.)
@@ -222,7 +222,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
-      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertSequenceEqual(sess.run(grad), [32.])
 
   @parameterized.named_parameters(
       ("UnknownShape", None),
@@ -315,9 +315,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
       # map_fn uses TensorArray internally.
       r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0)
-      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r))
+      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], sess.run(r))
       r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(21.0, self.evaluate(r))
+      self.assertAllClose(21.0, sess.run(r))
 
   def testNestedWhile(self):
     # Compute sum of geometric progression: n^0 + n^1 + ... + n^m
@@ -334,8 +334,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     result = while_loop_v2(lambda i, _: i >= 0, Body, [m, sum_of_powers])[1]
     grad = gradients_impl.gradients(result, [n])
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(result), 364.)
-      self.assertSequenceEqual(self.evaluate(grad), [547.])
+      self.assertEqual(sess.run(result), 364.)
+      self.assertSequenceEqual(sess.run(grad), [547.])
 
   def testIdentityNodeInBody(self):
 
@@ -348,8 +348,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., Body, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(self.evaluate(ret), 16.)
-      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertEqual(sess.run(ret), 16.)
+      self.assertSequenceEqual(sess.run(grad), [32.])
 
   def testNestedWhileAndTensorArray(self):
     n = constant_op.constant(3.0)
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index bd3142132c..c3c7f867a1 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -65,7 +65,7 @@ class XentTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       loss = nn_ops.softmax_cross_entropy_with_logits(
           labels=np_labels, logits=np_features, dim=dim)
-      tf_loss = self.evaluate(loss)
+      tf_loss = sess.run(loss)
     print("np_loss:", np_loss)
     print("tf_loss:", tf_loss)
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
@@ -280,7 +280,7 @@ class XentTest(test.TestCase):
     with self.session(use_gpu=True) as sess:
       loss = nn_ops.softmax_cross_entropy_with_logits(
           labels=labels, logits=features)
-      tf_loss = self.evaluate(loss)
+      tf_loss = sess.run(loss)
     self.assertAllEqual(np_loss, tf_loss)
 
 
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index d3200fa5b5..257fa27156 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -276,8 +276,8 @@ class ConvTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        self.evaluate(variables.global_variables_initializer())
-        weights = self.evaluate(weights)
+        sess.run(variables.global_variables_initializer())
+        weights = sess.run(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 32)))
         # Check that the bias still got initialized to zeros.
@@ -663,8 +663,8 @@ class SeparableConv2DTest(test.TestCase):
         self.assertTrue('depthwise_kernel' in weights[0].name)
         self.assertTrue('pointwise_kernel' in weights[1].name)
         self.assertTrue('bias' in weights[2].name)
-        self.evaluate(variables.global_variables_initializer())
-        weights = self.evaluate(weights)
+        sess.run(variables.global_variables_initializer())
+        weights = sess.run(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 1)))
         self.assertAllClose(weights[1], np.ones((1, 1, 3, 32)))
@@ -902,8 +902,8 @@ class Conv2DTransposeTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        self.evaluate(variables.global_variables_initializer())
-        weights = self.evaluate(weights)
+        sess.run(variables.global_variables_initializer())
+        weights = sess.run(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 32, 3)))
         # Check that the bias still got initialized to zeros.
@@ -1084,8 +1084,8 @@ class Conv3DTransposeTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        self.evaluate(variables.global_variables_initializer())
-        weights = self.evaluate(weights)
+        sess.run(variables.global_variables_initializer())
+        weights = sess.run(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 4, 32)))
         # Check that the bias still got initialized to zeros.
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index a61639b2db..0343bfa8bd 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -443,7 +443,7 @@ class DropoutTest(test.TestCase):
       dp = core_layers.Dropout(rate, name='dropout')
       inputs = array_ops.ones((5, 5))
       dropped = dp.apply(inputs, training=True)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_output = sess.run(dropped, feed_dict={rate: 0.5})
       self.assertAlmostEqual(0., np_output.min())
       np_output = sess.run(dropped, feed_dict={rate: 0.0})
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index febc3587fe..ba2bf10cf3 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -78,7 +78,7 @@ class BNTest(test.TestCase):
       if restore:
         saver.restore(sess, checkpoint_path)
       else:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
       np.random.seed(0)
       for _ in range(2):
         image_val = np.random.rand(*shape).astype(dtype.as_numpy_dtype)
@@ -321,7 +321,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 4, 1))
@@ -337,7 +337,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 2))
       std = np.std(np_inputs, axis=(0, 2))
       variance = np.square(std)
@@ -363,7 +363,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 3))
       np_beta = np.reshape(np_beta, (1, 1, 3))
@@ -377,7 +377,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 1))
       std = np.std(np_inputs, axis=(0, 1))
       variance = np.square(std)
@@ -404,7 +404,7 @@ class BNTest(test.TestCase):
 
       with self.session(use_gpu=True) as sess:
         # Test training with placeholder learning phase.
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
         np_gamma = np.reshape(np_gamma, (1, 4, 1, 1))
         np_beta = np.reshape(np_beta, (1, 4, 1, 1))
@@ -418,7 +418,7 @@ class BNTest(test.TestCase):
 
         # Verify that the statistics are updated during training.
         moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-        np_inputs = self.evaluate(inputs)
+        np_inputs = sess.run(inputs)
         mean = np.mean(np_inputs, axis=(0, 2, 3))
         std = np.std(np_inputs, axis=(0, 2, 3))
         variance = np.square(std)
@@ -444,7 +444,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 3, 1))
       np_beta = np.reshape(np_beta, (1, 1, 3, 1))
@@ -458,7 +458,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 3))
       std = np.std(np_inputs, axis=(0, 1, 3))
       variance = np.square(std)
@@ -484,7 +484,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -498,7 +498,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -524,7 +524,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -538,7 +538,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -565,7 +565,7 @@ class BNTest(test.TestCase):
 
       with self.cached_session() as sess:
         # Test training with placeholder learning phase.
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
         np_gamma = np.reshape(np_gamma, (1, 4, 1, 1))
         np_beta = np.reshape(np_beta, (1, 4, 1, 1))
@@ -579,7 +579,7 @@ class BNTest(test.TestCase):
 
         # Verify that the statistics are updated during training.
         moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-        np_inputs = self.evaluate(inputs)
+        np_inputs = sess.run(inputs)
         mean = np.mean(np_inputs, axis=(0, 2, 3))
         std = np.std(np_inputs, axis=(0, 2, 3))
         variance = np.square(std)
@@ -605,7 +605,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -620,7 +620,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -646,7 +646,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -659,7 +659,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -667,7 +667,7 @@ class BNTest(test.TestCase):
       self.assertAllClose(variance, moving_var, atol=1e-2)
 
       # Test inference with placeholder learning phase.
-      np_output = self.evaluate(outputs_infer)
+      np_output = sess.run(outputs_infer)
 
       # Verify that the axis is normalized during inference.
       normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
@@ -696,7 +696,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       np_gamma, np_beta = sess.run([gamma, beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
@@ -710,7 +710,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       np_mean = np.mean(np_inputs, axis=(0, 1, 2))
       np_std = np.std(np_inputs, axis=(0, 1, 2))
       np_variance = np.square(np_std)
@@ -758,14 +758,14 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(100):
         np_output, _, _ = sess.run([outputs2] + updates,
                                    feed_dict={training: True})
 
       # Verify that the statistics are updated during training.
       np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance])
-      np_inputs = self.evaluate(inputs2)
+      np_inputs = sess.run(inputs2)
       np_mean = np.mean(np_inputs, axis=(0, 1, 2))
       np_std = np.std(np_inputs, axis=(0, 1, 2))
       np_variance = np.square(np_std)
@@ -885,7 +885,7 @@ class BNTest(test.TestCase):
     renorm_mean = renorm_stddev = 0.
     renorm_weight = 0.
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -937,7 +937,7 @@ class BNTest(test.TestCase):
     moving_mean = 0.
     moving_variance = 1.
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
         yt_val_train, adj_scale_val, adj_bias_val = sess.run(
@@ -990,7 +990,7 @@ class BNTest(test.TestCase):
     renorm_mean = renorm_stddev = 0.
     renorm_weight = 0.
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
         yt_val_train, adj_scale_val, adj_bias_val = sess.run(
@@ -1040,7 +1040,7 @@ class BNTest(test.TestCase):
         out1.shape.as_list(), out2.shape.as_list())
 
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       x = np.random.random(shape)
       y1, y2 = sess.run([out1, out2], feed_dict={inp: x})
@@ -1062,7 +1062,7 @@ class BNTest(test.TestCase):
         inp, virtual_batch_size=2)
 
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       x = np.random.random(np_shape)
       y = sess.run(out, feed_dict={inp: x})
@@ -1093,7 +1093,7 @@ class BNTest(test.TestCase):
                     shape[1]])
 
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1146,7 +1146,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1200,7 +1200,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1256,7 +1256,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
 
@@ -1270,7 +1270,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=0, keepdims=True)
       std = np.std(np_inputs, axis=0, keepdims=True)
       variance = np.square(std)
@@ -1296,7 +1296,7 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
 
@@ -1310,7 +1310,7 @@ class BNTest(test.TestCase):
 
       # Verify that the statistics are updated during training.
       moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = self.evaluate(inputs)
+      np_inputs = sess.run(inputs)
       mean = np.mean(np_inputs, axis=(0, 4), keepdims=True)
       std = np.std(np_inputs, axis=(0, 4), keepdims=True)
       variance = np.square(std)
@@ -1350,7 +1350,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index 260af95a3b..47675d3f34 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -209,7 +209,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
       optimizer = momentum.MomentumOptimizer(0.1, 0.9)
       train_op = optimizer.minimize(cost)
       with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for _ in range(10):
           sess.run([train_op])
 
@@ -232,7 +232,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
           cond, body, [constant_op.constant(0),
                        constant_op.constant(0.0)])
       with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllEqual(10.0, self.evaluate(cost))
 
   def doTestIndexedSlicesGradientInCondInWhileLoop(self, use_resource=False):
@@ -269,7 +269,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
                                           static_grads.indices)
 
       with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllEqual(*sess.run([static_grads, dynamic_grads]))
 
   def testIndexedSlicesGradientInCondInWhileLoop(self):
@@ -398,9 +398,9 @@ class CondTest(test_util.TensorFlowTestCase):
             pred=bool_var,
             true_fn=lambda: state_ops.assign(bool_var, False),
             false_fn=lambda: True)
-        self.evaluate(bool_var.initializer)
-        self.assertEquals(self.evaluate(cond_on_bool_var), False)
-        self.assertEquals(self.evaluate(cond_on_bool_var), True)
+        sess.run(bool_var.initializer)
+        self.assertEquals(sess.run(cond_on_bool_var), False)
+        self.assertEquals(sess.run(cond_on_bool_var), True)
 
   def testCondMissingArg1(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index a9058c4a34..262b62e013 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -365,7 +365,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(
           [y], [x], unconnected_gradients="zero")
       with self.cached_session() as sess:
-        self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], self.evaluate(grads)[0])
+        self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], sess.run(grads)[0])
 
   def testUnconnectedGradientsZeroConnectedGradients(self):
     with ops.Graph().as_default():
@@ -374,7 +374,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grad = gradients.gradients(
           [y], [x], unconnected_gradients="zero")
       with self.cached_session() as sess:
-        self.assertEquals(3.0, self.evaluate(grad)[0])
+        self.assertEquals(3.0, sess.run(grad)[0])
 
   def testUnknownUnconnectedGradientsValueGiven(self):
     with ops.Graph().as_default():
@@ -438,8 +438,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(y, [x, b1])
 
       with self.cached_session() as sess:
-        self.assertAllEqual([40.0], self.evaluate(grads)[0])
-        self.assertAllEqual([10.0], self.evaluate(grads)[1])
+        self.assertAllEqual([40.0], sess.run(grads)[0])
+        self.assertAllEqual([10.0], sess.run(grads)[1])
 
   def testFunctionGradientsWithGradFunc(self):
     g = ops.Graph()
@@ -487,7 +487,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.assertEqual(self.evaluate(f), 2.0)
+        self.assertEqual(sess.run(f), 2.0)
 
   def testGradientOfCaptured(self):
     with ops.Graph().as_default():
@@ -501,7 +501,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.assertEqual(self.evaluate(f), 2.0)
+        self.assertEqual(sess.run(f), 2.0)
 
   def testCapturedResourceVariable(self):
     with ops.Graph().as_default():
@@ -515,8 +515,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.assertEqual(self.evaluate(f), 2.0)
+        sess.run(variables.global_variables_initializer())
+        self.assertEqual(sess.run(f), 2.0)
 
   def testCapturedNested(self):
     with ops.Graph().as_default():
@@ -541,9 +541,9 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
       x1_grad, x2_grad = Outer()
       with self.cached_session() as sess:
         # 1.0 + None + 2.0 + 1.0 = 4.0
-        self.assertEqual(self.evaluate(x1_grad), 4.0)
+        self.assertEqual(sess.run(x1_grad), 4.0)
         # None + 1.0 + 1.0 + None = 2.0
-        self.assertEqual(self.evaluate(x2_grad), 2.0)
+        self.assertEqual(sess.run(x2_grad), 2.0)
 
   def testCapturedFromFunction(self):
     with ops.Graph().as_default():
@@ -563,7 +563,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       z_grad = Outer()
       with self.cached_session() as sess:
-        self.assertEqual(self.evaluate(z_grad), 3.0)
+        self.assertEqual(sess.run(z_grad), 3.0)
 
   def testCapturedEagerTensors(self):
     # Test that we can handle captured eager tensors unrelated to the gradient
@@ -873,7 +873,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       y = MyMultiply(x1, x2)
       dy = gradients.gradients(y, [x1, x2])
       with session.Session() as sess:
-        self.assertAllEqual([3., 5.], self.evaluate(dy))
+        self.assertAllEqual([3., 5.], sess.run(dy))
 
   def testCustomGradientErrors(self):
 
@@ -914,7 +914,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       for g in grads:
         self.assertTrue(g is not None)
       with session.Session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         dw = sess.run(math_ops.reduce_sum(grads[1]))
         self.assertEqual(12., dw)
 
@@ -1074,7 +1074,7 @@ class TensorListGradientsTest(test_util.TensorFlowTestCase):
 
       grad = gradients.gradients(tl, a, grad_ys=grad_tl)[0]
       with self.cached_session() as sess:
-        self.assertEquals(self.evaluate(grad), 5.)
+        self.assertEquals(sess.run(grad), 5.)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py
index 0ea15b0d23..32c2f37c0b 100644
--- a/tensorflow/python/ops/image_grad_test.py
+++ b/tensorflow/python/ops/image_grad_test.py
@@ -44,7 +44,7 @@ class ResizeNearestNeighborOpTest(test.TestCase):
                                                        out_shape[1:3])
         self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-        resize_out = self.evaluate(resize_out)
+        resize_out = sess.run(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -113,7 +113,7 @@ class ResizeBilinearOpTest(test.TestCase):
       resize_out = image_ops.resize_bilinear(input_tensor, out_shape[1:3])
       self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-      resize_out = self.evaluate(resize_out)
+      resize_out = sess.run(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -196,7 +196,7 @@ class ResizeBicubicOpTest(test.TestCase):
                                               align_corners=align_corners)
         self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-        resize_out = self.evaluate(resize_out)
+        resize_out = sess.run(resize_out)
         self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -273,7 +273,7 @@ class CropAndResizeOpTest(test.TestCase):
           constant_op.constant(
               crop_size, shape=[2]))
       self.assertEqual(crops_shape, list(crops.get_shape()))
-      crops = self.evaluate(crops)
+      crops = sess.run(crops)
       self.assertEqual(crops_shape, list(crops.shape))
 
   def _randomUniformAvoidAnchors(self, low, high, anchors, radius, num_samples):
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index de82f4fc27..ac2d2698b6 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -488,11 +488,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.flip_left_right(inputs)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          self.evaluate(run_op)
+          sess.run(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -518,11 +518,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.random_flip_left_right(inputs)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          self.evaluate(run_op)
+          sess.run(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -548,11 +548,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.random_flip_left_right(inputs)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          self.evaluate(run_op)
+          sess.run(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -610,11 +610,11 @@ class AdjustHueBenchmark(test.Benchmark):
       delta = constant_op.constant(0.1, dtype=dtypes.float32)
       outputs = image_ops.adjust_hue(inputs, delta)
       run_op = control_flow_ops.group(outputs)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for i in xrange(warmup_rounds + benchmark_rounds):
         if i == warmup_rounds:
           start = time.time()
-        self.evaluate(run_op)
+        sess.run(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -653,12 +653,12 @@ class AdjustSaturationBenchmark(test.Benchmark):
       delta = constant_op.constant(0.1, dtype=dtypes.float32)
       outputs = image_ops.adjust_saturation(inputs, delta)
       run_op = control_flow_ops.group(outputs)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       for _ in xrange(warmup_rounds):
-        self.evaluate(run_op)
+        sess.run(run_op)
       start = time.time()
       for _ in xrange(benchmark_rounds):
-        self.evaluate(run_op)
+        sess.run(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -698,7 +698,7 @@ class ResizeBilinearBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -746,7 +746,7 @@ class ResizeBicubicBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -803,7 +803,7 @@ class ResizeAreaBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -4110,7 +4110,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
       images = [ops.convert_to_tensor(x, dtype=dtypes.float32) for x in images]
       msssim_ops = [image_ops.ssim_multiscale(x, y, 1.0)
                     for x, y in itertools.combinations(images, 2)]
-      msssim = self.evaluate(msssim_ops)
+      msssim = sess.run(msssim_ops)
       msssim = np.squeeze(msssim)
 
     self.assertTrue(np.all(msssim >= 0.0))
diff --git a/tensorflow/python/ops/init_ops_test.py b/tensorflow/python/ops/init_ops_test.py
index 1f22248004..5693c3caaf 100644
--- a/tensorflow/python/ops/init_ops_test.py
+++ b/tensorflow/python/ops/init_ops_test.py
@@ -45,8 +45,8 @@ class InitializersTest(test.TestCase):
       output = variable.numpy()
     else:
       sess = ops.get_default_session()
-      self.evaluate(variable.initializer)
-      output = self.evaluate(variable)
+      sess.run(variable.initializer)
+      output = sess.run(variable)
     lim = 3e-2
     if target_std is not None:
       self.assertGreater(lim, abs(output.std() - target_std))
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index cd45b6f136..adcaa7abff 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -373,7 +373,7 @@ class AddNTest(test_util.TensorFlowTestCase):
             for i in range(0, num_inputs)
         ]
         addn = math_ops.add_n(input_vars)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         add_n_grad = gradients.gradients(addn, input_vars)
         self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
                             [g.eval() for g in add_n_grad])
@@ -461,7 +461,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       a = variables.Variable(2.)
       b = variables.Variable(4.)
       with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         c_grad = gradients.gradients(math_ops.divide(a, b), [a, b])
         self.assertAllEqual([x.eval() for x in c_grad], [.25, -.125])
         c_grad = gradients.gradients(math_ops.div(a, b), [a, b])
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index 552b274b83..a6c582fcac 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -82,7 +82,7 @@ class BatchNormalizationTest(test.TestCase):
           epsilon=epsilon,
           data_format=data_format,
           is_training=False)
-      y_val = self.evaluate(y)
+      y_val = sess.run(y)
       y_ref = self._inference_ref(x, scale, offset, mean, var, epsilon,
                                   data_format)
     # An atol value of 1e-3 is too small for float16's, because some adjacent
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 545c482df8..bbb46539ea 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -485,7 +485,7 @@ class GradientsTest(test.TestCase):
     with session.Session() as sess:
       init = variables.global_variables_initializer()
       sess.run(init)
-      pfor = self.evaluate(pfor_jacobian)
+      pfor = sess.run(pfor_jacobian)
       for i in range(4):
         while_i = sess.run(while_gradients[i])
         self.assertAllClose(while_i, pfor[:, i, ...])
diff --git a/tensorflow/python/ops/quantized_conv_ops_test.py b/tensorflow/python/ops/quantized_conv_ops_test.py
index 6b469a954f..f7fa264461 100644
--- a/tensorflow/python/ops/quantized_conv_ops_test.py
+++ b/tensorflow/python/ops/quantized_conv_ops_test.py
@@ -73,7 +73,7 @@ class Conv2DTest(test.TestCase):
           max_input=x1_max,
           min_filter=x2_min,
           max_filter=x2_max)
-      value = self.evaluate(conv)
+      value = sess.run(conv)
     quantized_output = value[0]
     output_min = value[1]
     output_max = value[2]
diff --git a/tensorflow/python/ops/quantized_ops_test.py b/tensorflow/python/ops/quantized_ops_test.py
index b81843d174..0f3b04e4ad 100644
--- a/tensorflow/python/ops/quantized_ops_test.py
+++ b/tensorflow/python/ops/quantized_ops_test.py
@@ -41,7 +41,7 @@ class QuantizedOpsTest(test.TestCase):
       x_min = 0.0
       x_max = 255.0
       op = array_ops.quantize(x, x_min, x_max, dtypes.quint8, mode="MIN_FIRST")
-      value = self.evaluate(op)
+      value = sess.run(op)
       self.assertArrayNear(expected_output, value.output, 0.1)
 
   def testDequantizeOp(self):
@@ -52,7 +52,7 @@ class QuantizedOpsTest(test.TestCase):
       x_min = 0.0
       x_max = 255.0
       op = array_ops.dequantize(x, x_min, x_max, mode="MIN_FIRST")
-      value = self.evaluate(op)
+      value = sess.run(op)
       self.assertArrayNear(expected_output, value, 0.1)
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index c52db9e2a1..dcf1feaa69 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -190,7 +190,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
     with self.test_session() as sess:
       if hasattr(expected, 'tolist'):
         expected = expected.tolist()
-      self.assertEqual(self.evaluate(result).tolist(), expected)
+      self.assertEqual(sess.run(result).tolist(), expected)
 
   def testRaggedGatherNdUnknownRankError(self):
     params = ragged.constant([['a', 'b'], ['c', 'd']])
diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py
index 8648f0b514..94c685274a 100644
--- a/tensorflow/python/profiler/model_analyzer_test.py
+++ b/tensorflow/python/profiler/model_analyzer_test.py
@@ -93,10 +93,10 @@ class PrintModelAnalysisTest(test.TestCase):
           config=self._no_rewrite_session_config()) as sess, ops.device(dev):
         x = lib.BuildSmallModel()
 
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         pctx.trace_next_step()
         pctx.dump_next_step()
-        _ = self.evaluate(x)
+        _ = sess.run(x)
 
         pctx.profiler.profile_name_scope(options=opts)
 
@@ -160,7 +160,7 @@ class PrintModelAnalysisTest(test.TestCase):
                         ) as sess, ops.device('/device:CPU:0'):
       x = lib.BuildSmallModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -186,7 +186,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -220,9 +220,9 @@ class PrintModelAnalysisTest(test.TestCase):
       with session.Session(config=self._no_rewrite_session_config()) as sess:
         x = lib.BuildFullModel()
 
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         pctx.trace_next_step()
-        _ = self.evaluate(x)
+        _ = sess.run(x)
         tfprof_node = pctx.profiler.profile_python(options=opts)
 
         # pylint: disable=line-too-long
@@ -281,7 +281,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -309,7 +309,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -345,7 +345,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -391,7 +391,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -424,7 +424,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -490,7 +490,7 @@ class PrintModelAnalysisTest(test.TestCase):
 
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -555,7 +555,7 @@ class PrintModelAnalysisTest(test.TestCase):
 
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -587,10 +587,10 @@ class PrintModelAnalysisTest(test.TestCase):
   def _trainLoop(self, train_op, train_steps, time_dir, time_step,
                  memory_dir, memory_step, profile_dir, dump_step):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       # start from 1 because variable_initializer took one step.
       for i in range(1, train_steps + 1):
-        _ = self.evaluate(train_op)
+        _ = sess.run(train_op)
         if i in time_step:
           ret = gfile.ListDirectory(time_dir)
           self.assertEqual(len(ret), 1)
diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py
index abbeb8bedf..107ad443c3 100644
--- a/tensorflow/python/profiler/profile_context_test.py
+++ b/tensorflow/python/profiler/profile_context_test.py
@@ -48,7 +48,7 @@ class ProfilerContextTest(test.TestCase):
     with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
       pctx.add_auto_profiling("op", options=opts, profile_steps=[15, 50, 100])
       with session.Session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         total_steps = 101
         for i in range(total_steps):
           sess.run(x)
@@ -75,7 +75,7 @@ class ProfilerContextTest(test.TestCase):
 
     with profile_context.ProfileContext(test.get_temp_dir(), debug=True):
       with session.Session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for _ in range(10):
           sess.run(x)
           for f in gfile.ListDirectory(test.get_temp_dir()):
@@ -96,7 +96,7 @@ class ProfilerContextTest(test.TestCase):
     with profile_context.ProfileContext(test.get_temp_dir(),
                                         enabled=False) as pctx:
       with session.Session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for _ in range(10):
           sess.run(x)
       self.assertTrue(pctx.profiler is None)
@@ -105,7 +105,7 @@ class ProfilerContextTest(test.TestCase):
 
     with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
       with session.Session() as sess:
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for _ in range(10):
           sess.run(x)
       self.assertFalse(pctx.profiler is None)
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 0b97a73441..648c1c5928 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -50,7 +50,7 @@ class SavedModelLoaderTest(test.TestCase):
       x = variables.VariableV1(5, name="x")
       y = variables.VariableV1(11, name="y")
       z = x + y
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       foo_sig_def = signature_def_utils.build_signature_def(
           {"foo_input": utils.build_tensor_info(x)},
@@ -138,7 +138,7 @@ class SavedModelLoaderTest(test.TestCase):
       y = variables.VariableV1(0, name="y")
       z = x * y
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       # There are variables to restore, so a saver must be created.
       with self.assertRaises(ValueError):
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index e722b6ceae..a40ea7687f 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -61,7 +61,7 @@ class SavedModelTestBase(test.TestCase):
 
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.VariableV1(variable_value, name=variable_name)
-    self.evaluate(variables.global_variables_initializer())
+    sess.run(variables.global_variables_initializer())
     self.assertEqual(variable_value, self.evaluate(v))
 
   def _build_asset_collection(self, asset_file_name, asset_file_contents,
@@ -389,7 +389,7 @@ class SavedModelTest(SavedModelTestBase):
       a = ops.get_default_graph().get_tensor_by_name(constant_5_name)
       b = constant_op.constant(6.0)
       c = a * b
-      self.assertEqual(30.0, self.evaluate(c))
+      self.assertEqual(30.0, sess.run(c))
 
     # Restore the graph with tag "bar".
     with self.session(graph=ops.Graph()) as sess:
@@ -398,7 +398,7 @@ class SavedModelTest(SavedModelTestBase):
       a = ops.get_default_graph().get_tensor_by_name(constant_6_name)
       b = constant_op.constant(5.0)
       c = a * b
-      self.assertEqual(30.0, self.evaluate(c))
+      self.assertEqual(30.0, sess.run(c))
 
   def testNoOverwrite(self):
     export_dir = self._get_export_dir("test_no_overwrite")
@@ -464,7 +464,7 @@ class SavedModelTest(SavedModelTestBase):
     with self.session(graph=ops.Graph()) as sess:
       v = variables.VariableV1(42, name="v")
       ops.add_to_collection("foo_vars", v)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       self.assertEqual(42, self.evaluate(v))
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
@@ -474,7 +474,7 @@ class SavedModelTest(SavedModelTestBase):
     with self.session(graph=ops.Graph()) as sess:
       v = variables.VariableV1(43, name="v")
       ops.add_to_collection("bar_vars", v)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       self.assertEqual(43, self.evaluate(v))
       builder.add_meta_graph(["bar"])
 
@@ -802,7 +802,7 @@ class SavedModelTest(SavedModelTestBase):
         add_v1_v2 = math_ops.add(v1._ref(), v2._ref())
         custom_main_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2))
 
-      self.evaluate(custom_main_op)
+      sess.run(custom_main_op)
       builder.add_meta_graph_and_variables(
           sess, ["foo"], main_op=custom_main_op)
 
@@ -836,7 +836,7 @@ class SavedModelTest(SavedModelTestBase):
       assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2))
       legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op")
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(
           sess, ["foo"], legacy_init_op=legacy_init_op)
 
@@ -879,7 +879,7 @@ class SavedModelTest(SavedModelTestBase):
       assign_v2 = state_ops.assign(v2, v1)
       init_op = control_flow_ops.group(assign_v2, name="init_op")
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       ops.add_to_collection(key, control_flow_ops.no_op())
       # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
@@ -902,10 +902,10 @@ class SavedModelTest(SavedModelTestBase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       train_op = state_ops.assign_add(v1, v2)
 
-      self.evaluate(train_op)
+      sess.run(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph_and_variables(sess, ["foo"])
@@ -931,10 +931,10 @@ class SavedModelTest(SavedModelTestBase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       train_op = control_flow_ops.group()
 
-      self.evaluate(train_op)
+      sess.run(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph_and_variables(sess, ["foo"])
@@ -960,11 +960,11 @@ class SavedModelTest(SavedModelTestBase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["pre_foo"])
 
       train_op = state_ops.assign_add(v1, v2)
-      self.evaluate(train_op)
+      sess.run(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph(["foo"])
@@ -1090,7 +1090,7 @@ class SavedModelTest(SavedModelTestBase):
       ops.add_to_collection("v", v3)
       ops.add_to_collection("init_op", init_op)
 
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       self.assertEqual(1, ops.get_collection("v")[0].eval())
       self.assertEqual(2, ops.get_collection("v")[1].eval())
 
@@ -1145,7 +1145,7 @@ class SavedModelTest(SavedModelTestBase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       custom_saver = training.Saver(name="my_saver")
       builder.add_meta_graph_and_variables(sess, ["tag"], saver=custom_saver)
 
@@ -1167,7 +1167,7 @@ class SavedModelTest(SavedModelTestBase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       training.Saver(name="my_saver")
       builder.add_meta_graph_and_variables(sess, ["tag"])
 
@@ -1189,7 +1189,7 @@ class SavedModelTest(SavedModelTestBase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["tag_0"])
 
       saver_1 = training.Saver()
@@ -1298,7 +1298,7 @@ class SavedModelTest(SavedModelTestBase):
       real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
       imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
       math_ops.complex(real_num, imag_num, name="complex")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(
           sess, ["foo"], strip_default_attrs=True)
 
@@ -1308,7 +1308,7 @@ class SavedModelTest(SavedModelTestBase):
       real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
       imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
       math_ops.complex(real_num, imag_num, name="complex")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       builder.add_meta_graph(["bar"], strip_default_attrs=False)
 
     # Save the SavedModel to disk in text format.
@@ -1370,7 +1370,7 @@ class SavedModelTest(SavedModelTestBase):
     with session.Session(graph=ops.Graph()) as sess:
       variables.VariableV1(1.0, dtype=dtypes.float64, name="var")
       test_ops.test_attr(T=dtypes.float32, name="test_attr")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
     # Save the SavedModel to disk in text format.
diff --git a/tensorflow/python/saved_model/simple_save_test.py b/tensorflow/python/saved_model/simple_save_test.py
index 0d0665072a..2d404dcea4 100644
--- a/tensorflow/python/saved_model/simple_save_test.py
+++ b/tensorflow/python/saved_model/simple_save_test.py
@@ -33,7 +33,7 @@ class SimpleSaveTest(test.TestCase):
 
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.Variable(variable_value, name=variable_name)
-    self.evaluate(variables.global_variables_initializer())
+    sess.run(variables.global_variables_initializer())
     self.assertEqual(variable_value, self.evaluate(v))
     return v
 
diff --git a/tensorflow/python/tools/strip_unused_test.py b/tensorflow/python/tools/strip_unused_test.py
index e906ff94ba..7cf0c3e3ed 100644
--- a/tensorflow/python/tools/strip_unused_test.py
+++ b/tensorflow/python/tools/strip_unused_test.py
@@ -50,7 +50,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase):
           wanted_input_node, 2.0, name="output_node")
       math_ops.add(output_node, 2.0, name="later_node")
       sess = session.Session()
-      output = self.evaluate(output_node)
+      output = sess.run(output_node)
       self.assertNear(-4.0, output, 0.00001)
       graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
 
@@ -113,7 +113,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase):
           input_node1, input_node2, name="output_node")
       math_ops.add(output_node, 2.0, name="later_node")
       sess = session.Session()
-      output = self.evaluate(output_node)
+      output = sess.run(output_node)
       self.assertNear(6.0, output, 0.00001)
       graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
 
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 13c9e9aa67..2d469634e0 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -243,7 +243,7 @@ class LoggingTensorHookTest(test.TestCase):
           tensors=[t.name], at_end=True)
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       self.logged_message = ''
       for _ in range(3):
         mon_sess.run(train_op)
@@ -261,7 +261,7 @@ class LoggingTensorHookTest(test.TestCase):
         tensors=[t.name], every_n_iter=10, at_end=at_end)
     hook.begin()
     mon_sess = monitored_session._HookedSession(sess, [hook])
-    self.evaluate(variables_lib.global_variables_initializer())
+    sess.run(variables_lib.global_variables_initializer())
     mon_sess.run(train_op)
     self.assertRegexpMatches(str(self.logged_message), t.name)
     for _ in range(3):
@@ -308,7 +308,7 @@ class LoggingTensorHookTest(test.TestCase):
           tensors={'foo': t}, every_n_iter=1)
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess.run(train_op)
       self.assertRegexpMatches(str(self.logged_message), 'foo')
       # in first run, elapsed time is None.
@@ -322,7 +322,7 @@ class LoggingTensorHookTest(test.TestCase):
         tensors=[t.name], every_n_secs=1.0, at_end=at_end)
     hook.begin()
     mon_sess = monitored_session._HookedSession(sess, [hook])
-    self.evaluate(variables_lib.global_variables_initializer())
+    sess.run(variables_lib.global_variables_initializer())
 
     mon_sess.run(train_op)
     self.assertRegexpMatches(str(self.logged_message), t.name)
@@ -366,7 +366,7 @@ class LoggingTensorHookTest(test.TestCase):
           formatter=lambda items: 'qqq=%s' % items[t.name])
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess.run(train_op)
       self.assertEqual(self.logged_message[0], 'qqq=42.0')
 
@@ -921,7 +921,7 @@ class StepCounterHookTest(test.TestCase):
       hook = basic_session_run_hooks.StepCounterHook(
           summary_writer=summary_writer, every_n_steps=10)
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       with test.mock.patch.object(tf_logging, 'warning') as mock_log:
         for _ in range(30):
@@ -950,7 +950,7 @@ class StepCounterHookTest(test.TestCase):
           summary_writer=summary_writer, every_n_steps=None, every_n_secs=0.1)
 
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
       time.sleep(0.2)
@@ -987,7 +987,7 @@ class StepCounterHookTest(test.TestCase):
           summary_writer=summary_writer, every_n_steps=1, every_n_secs=None)
 
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
       mon_sess.run(train_op)
@@ -1007,7 +1007,7 @@ class StepCounterHookTest(test.TestCase):
     with ops.Graph().as_default(), session_lib.Session() as sess:
       variables.get_or_create_global_step()
       train_op = training_util._increment_global_step(0)  # keep same.
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       hook = basic_session_run_hooks.StepCounterHook(
           every_n_steps=1, every_n_secs=None)
       hook.begin()
@@ -1034,7 +1034,7 @@ class StepCounterHookTest(test.TestCase):
         summary_writer=self.summary_writer, every_n_steps=every_n_steps)
     self.hook._set_steps_per_run(steps_per_run)
     self.hook.begin()
-    self.evaluate(variables_lib.global_variables_initializer())
+    sess.run(variables_lib.global_variables_initializer())
     self.mon_sess = monitored_session._HookedSession(sess, [self.hook])
 
   def test_steps_per_run_less_than_every_n_steps(self):
@@ -1147,7 +1147,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(30):
         mon_sess.run(self.train_op)
@@ -1179,7 +1179,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(10):
         mon_sess.run(self.train_op)
@@ -1207,7 +1207,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(4):
         mon_sess.run(self.train_op)
@@ -1242,7 +1242,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(8):
         mon_sess.run(self.train_op)
@@ -1285,7 +1285,7 @@ class GlobalStepWaiterHookTest(test.TestCase):
       hook = basic_session_run_hooks.GlobalStepWaiterHook(wait_until_step=1000)
       hook.begin()
       with session_lib.Session() as sess:
-        self.evaluate(variables_lib.global_variables_initializer())
+        sess.run(variables_lib.global_variables_initializer())
         waiter = threading.Thread(
             target=hook.before_run,
             args=(session_run_hook.SessionRunContext(
@@ -1390,7 +1390,7 @@ class ResourceSummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      self.evaluate(variables_lib.global_variables_initializer())
+      sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(30):
         mon_sess.run(self.train_op)
diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py
index 21ad3df1c8..38d4acf85f 100644
--- a/tensorflow/python/training/checkpoint_ops_test.py
+++ b/tensorflow/python/training/checkpoint_ops_test.py
@@ -47,7 +47,7 @@ class LoadAndRemapWrappersTest(test.TestCase):
       with variable_scope.variable_scope('some_scope'):
         variable_scope.get_variable(name='embeddings', shape=[5, 16],
                                     initializer=initializer)
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       saver = saver_lib.Saver()
       saver.save(sess, checkpoint_prefix, global_step=5)
     self.checkpoint_file = '{}-5'.format(checkpoint_prefix)
diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py
index 31c2cc56c0..e5aac5da18 100644
--- a/tensorflow/python/training/input_test.py
+++ b/tensorflow/python/training/input_test.py
@@ -474,7 +474,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = self.evaluate(batched_fetch)
+        results = sess.run(batched_fetch)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -539,7 +539,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         expected_results = np.arange(i * batch_size, (i + 1) * batch_size)
         max_len = expected_results[-1]
         self.assertAllEqual(results[0], expected_results)
@@ -571,7 +571,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -610,7 +610,7 @@ class BatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertAllEqual(results[0], results[1].values)
@@ -651,7 +651,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -667,7 +667,7 @@ class BatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra_elements.
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       self.assertAllEqual(results[0],
                           np.arange(num_batches * batch_size,
                                     num_batches * batch_size + extra_elements))
@@ -709,7 +709,7 @@ class BatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertAllEqual(results[0], results[1].values)
@@ -721,7 +721,7 @@ class BatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra_elements.
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), extra_elements)
       self.assertAllEqual(results[0], results[1].values)
@@ -827,7 +827,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
@@ -1020,7 +1020,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = self.evaluate(batched_fetch)
+        results = sess.run(batched_fetch)
         self.assertEqual(3, len(results))
         self.assertEqual(batch_size, len(results[0]))
         self.assertEqual(batch_size, len(results[2]))
@@ -1116,7 +1116,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertEqual(2, len(results))
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[1]), batch_size)
@@ -1201,7 +1201,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -1221,7 +1221,7 @@ class BatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached the final batch with 2 * extra_elements.
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertEqual(len(results[2]), 2 * extra_elements)
@@ -1296,7 +1296,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[1]), batch_size)
@@ -1316,7 +1316,7 @@ class BatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached the final batch with 2 * extra_elements.
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertEqual(len(results[1]), 2 * extra_elements)
@@ -1410,7 +1410,7 @@ class BatchJoinTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertAllEqual(
             [0] * batch_size,
             np.mod(results[0], 2),)
@@ -1579,7 +1579,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = self.evaluate(batched_fetch)
+        results = sess.run(batched_fetch)
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
         self.assertAllEqual(
@@ -1634,7 +1634,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for _ in range(num_batches):
-        results = self.evaluate(batched_fetch)
+        results = sess.run(batched_fetch)
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
         self.assertAllEqual(
@@ -1645,7 +1645,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra elements.
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       self.assertAllEqual(results[1].dense_shape, [extra_elements, 1])
       self.assertAllEqual(results[2], [b"string"] * extra_elements)
       all_counts.extend(results[0])
@@ -1687,7 +1687,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
@@ -1737,7 +1737,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
@@ -1749,7 +1749,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra elements.
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       self.assertAllEqual(results[0].shape, [extra_elements])
       self.assertAllEqual(results[1].dense_shape, [extra_elements, 1])
       self.assertAllEqual(results[2], [b"string"] * extra_elements)
@@ -1817,7 +1817,7 @@ class ShuffleBatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
@@ -1990,7 +1990,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = self.evaluate(batched_fetch)
+        results = sess.run(batched_fetch)
         self.assertEqual(3, len(results))
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -2082,7 +2082,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -2102,7 +2102,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached end with 2 * extra_elements left
-      results = self.evaluate(batched)
+      results = sess.run(batched)
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertAllEqual(results[1].dense_shape, [2 * extra_elements, 1])
       self.assertEqual(len(results[2]), 2 * extra_elements)
@@ -2203,7 +2203,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = self.evaluate(batched)
+        results = sess.run(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 2ceb387ec3..b828be4499 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -1170,7 +1170,7 @@ class HookedSessionTest(test.TestCase):
       mock_run = FakeSession(sess)
       mon_sess = monitored_session._HookedSession(sess=mock_run, hooks=[])
       a_tensor = constant_op.constant([0], name='a_tensor')
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       output = mon_sess.run(fetches=a_tensor,
                             feed_dict='a_feed',
                             options='an_option',
@@ -1189,7 +1189,7 @@ class HookedSessionTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(
           sess=sess, hooks=[mock_hook, mock_hook2])
       a_tensor = constant_op.constant([0], name='a_tensor')
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       mon_sess.run(a_tensor)
 
       for hook in [mock_hook, mock_hook2]:
@@ -1214,7 +1214,7 @@ class HookedSessionTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(
           sess=sess, hooks=[mock_hook, mock_hook2])
       constant_op.constant([0], name='a_tensor')
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       mon_sess.run(fetches='a_tensor')
       self.assertFalse(mon_sess.should_stop())
@@ -1234,7 +1234,7 @@ class HookedSessionTest(test.TestCase):
       third_tensor = constant_op.constant([10], name='third_tensor')
       mock_hook.request = session_run_hook.SessionRunArgs([another_tensor])
       mock_hook2.request = session_run_hook.SessionRunArgs([third_tensor])
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       output = mon_sess.run(fetches=a_tensor)
       self.assertEqual(output, [0])
@@ -1254,7 +1254,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       self.assertEqual(mon_sess.run(fetches=add_tensor), [15])
 
@@ -1272,7 +1272,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       feed_dict = {c_tensor: [20]}
       self.assertEqual(
@@ -1293,7 +1293,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={a_tensor: [10]})
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'):
         mon_sess.run(fetches=add_tensor)
@@ -1311,7 +1311,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
 
       with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'):
         mon_sess.run(fetches=add_tensor, feed_dict={b_tensor: [10]})
diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py
index 41e9dcea84..8009e3c24e 100644
--- a/tensorflow/python/training/moving_averages_test.py
+++ b/tensorflow/python/training/moving_averages_test.py
@@ -274,14 +274,14 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual([], v1_avg.value().op.control_inputs)
       self.assertEqual([], v1_avg.value().op.control_inputs)
       # We should be able to initialize v1_avg before v0.
-      self.evaluate(v1_avg.initializer)
-      self.evaluate(v0.initializer)
-      self.assertEqual([10.0], self.evaluate(v1_avg))
+      sess.run(v1_avg.initializer)
+      sess.run(v0.initializer)
+      self.assertEqual([10.0], sess.run(v1_avg))
       # running ema_op should add to v0 (in addition to updating v1_avg)
       sess.run(assign_to_v1)
-      self.evaluate(ema_op)
-      self.assertEqual(1, self.evaluate(v0))
-      self.assertEqual([17.5], self.evaluate(v1_avg))
+      sess.run(ema_op)
+      self.assertEqual(1, sess.run(v0))
+      self.assertEqual([17.5], sess.run(v1_avg))
 
   @test_util.run_in_graph_and_eager_modes
   def testBasicEager(self):
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 7bc0a178a4..eb2690985d 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -227,7 +227,7 @@ class SaverTest(test.TestCase):
         w1 = resource_variable_ops.ResourceVariable(1.0, name="w1")
         w2 = resource_variable_ops.ResourceVariable(2.0, name="w2")
         graph_saver = saver_module.Saver([w1, w2])
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         graph_saver.save(sess, graph_ckpt_prefix)
 
     with context.eager_mode():
@@ -260,7 +260,7 @@ class SaverTest(test.TestCase):
         w3 = resource_variable_ops.ResourceVariable(0.0, name="w3")
         w4 = resource_variable_ops.ResourceVariable(0.0, name="w4")
         graph_saver = saver_module.Saver([w3, w4])
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         graph_saver.restore(sess, eager_ckpt_prefix)
         self.assertAllEqual(w3.eval(), 3.0)
         self.assertAllEqual(w4.eval(), 4.0)
@@ -326,7 +326,7 @@ class SaverTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initialize all variables
-      self.evaluate(init_all_op)
+      sess.run(init_all_op)
 
       # Check that the parameter nodes have been initialized.
       self.assertEqual(10.0, v0.eval())
@@ -376,7 +376,7 @@ class SaverTest(test.TestCase):
     with self.cached_session() as sess:
       tensor = sess.graph.get_tensor_by_name(
           save.saver_def.filename_tensor_name)
-      self.assertEqual(self.evaluate(tensor), filename)
+      self.assertEqual(sess.run(tensor), filename)
 
   def testInvalidPath(self):
     v0 = variables.VariableV1(0, name="v0")
@@ -742,7 +742,7 @@ class SaverTest(test.TestCase):
       try:
         with self.cached_session() as sess:
           # Initialize all variables
-          self.evaluate(init_all_op)
+          sess.run(init_all_op)
 
           # Check that the parameter nodes have been initialized.
           self.assertEqual(10.0, v0.eval())
@@ -777,7 +777,7 @@ class SaverTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initialize all variables
-      self.evaluate(init_all_op)
+      sess.run(init_all_op)
 
       # Check that the parameter nodes have been initialized.
       self.assertEqual(10.0, v0.eval())
@@ -824,11 +824,11 @@ class SaverTest(test.TestCase):
     save_graph = ops_lib.Graph()
     with save_graph.as_default(), self.session(graph=save_graph) as sess:
       orig_vars = _model()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       save = saver_module.Saver(max_to_keep=1)
       variables.global_variables_initializer().run()
       save.save(sess, save_dir)
-      orig_vals = self.evaluate(orig_vars)
+      orig_vals = sess.run(orig_vars)
 
     restore_graph = ops_lib.Graph()
     with restore_graph.as_default(), self.session(
@@ -836,7 +836,7 @@ class SaverTest(test.TestCase):
       restored_vars = _model()
       save = saver_module.Saver(max_to_keep=1)
       save.restore(sess, save_dir)
-      restored_vals = self.evaluate(restored_vars)
+      restored_vals = sess.run(restored_vars)
 
     for orig, restored in zip(orig_vals, restored_vals):
       self.assertAllEqual(orig, restored)
@@ -1949,7 +1949,7 @@ class MetaGraphTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initializes all the variables.
-      self.evaluate(init_all_op)
+      sess.run(init_all_op)
       # Runs to logit.
       sess.run(logits)
       # Creates a saver.
@@ -1991,7 +1991,7 @@ class MetaGraphTest(test.TestCase):
       ops_lib.add_to_collection("train_op", train_op)
 
       # Runs train_op.
-      self.evaluate(train_op)
+      sess.run(train_op)
 
       # Generates MetaGraphDef.
       saver_module.export_meta_graph(train_filename)
@@ -2005,7 +2005,7 @@ class MetaGraphTest(test.TestCase):
       # Restores from checkpoint.
       new_saver.restore(sess, saver0_ckpt)
       train_op = ops_lib.get_collection("train_op")[0]
-      self.evaluate(train_op)
+      sess.run(train_op)
 
   def testGraphExtension(self):
     test_dir = self._get_test_dir("graph_extension")
@@ -2037,7 +2037,7 @@ class MetaGraphTest(test.TestCase):
 
       # Generate a MetaGraphDef containing the while loop.
       with session.Session() as sess:
-        self.evaluate(init_op)
+        sess.run(init_op)
         sess.run(output)
         saver = saver_module.Saver()
         saver.save(sess, saver_ckpt)
@@ -2053,8 +2053,8 @@ class MetaGraphTest(test.TestCase):
       no_constfold_config.graph_options.rewrite_options.constant_folding = (
           rewriter_config_pb2.RewriterConfig.OFF)
       with session.Session(config=no_constfold_config) as sess:
-        self.evaluate(init_op)
-        expected_grad_value = self.evaluate(grad)
+        sess.run(init_op)
+        expected_grad_value = sess.run(grad)
 
     # Restore the MetaGraphDef into a new Graph.
     with ops_lib.Graph().as_default():
@@ -2070,8 +2070,8 @@ class MetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with session.Session(config=no_constfold_config) as sess:
-        self.evaluate(init_op)
-        actual_grad_value = self.evaluate(grad)
+        sess.run(init_op)
+        actual_grad_value = sess.run(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
   def _testWhileLoopAndGradientSerDes(self, outer_body_fn):
@@ -2209,7 +2209,7 @@ class MetaGraphTest(test.TestCase):
                                                       logits=logit, name="cost")
       adam.AdamOptimizer().minimize(cost, name="optimize")
       saver = saver_module.Saver()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       saver.save(sess, filename)
 
     graph = ops_lib.Graph()
@@ -2246,7 +2246,7 @@ class MetaGraphTest(test.TestCase):
 
       # Create a variable in graph_2 under scope "my_scope".
       variables.VariableV1(array_ops.zeros([10]), name="my_scope/my_var")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       # Restore the checkpoint into a different scope "subgraph_2".
       new_saver_2 = saver_module.import_meta_graph(
           filename + ".meta", graph=graph_2, import_scope="subgraph_2")
@@ -2279,7 +2279,7 @@ class MetaGraphTest(test.TestCase):
                                                       logits=logit, name="cost")
       adam.AdamOptimizer().minimize(cost, name="optimize")
       saver = saver_module.Saver()
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       saver.save(sess, filename)
 
     graph = ops_lib.Graph()
@@ -2316,12 +2316,12 @@ class MetaGraphTest(test.TestCase):
           meta_graph_def, clear_devices=False, import_scope="new_model")
       # Device refers to GPU, which is not available here.
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
 
     with session.Session(graph=ops_lib.Graph()) as sess:
       saver_module.import_meta_graph(
           meta_graph_def, clear_devices=True, import_scope="new_model")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       sess.run(["new_model/optimize"], {
           "new_model/image:0": np.random.random([1, 784]),
           "new_model/label:0": np.random.randint(
@@ -2348,7 +2348,7 @@ class MetaGraphTest(test.TestCase):
 
     with session.Session(graph=ops_lib.Graph()) as sess:
       saver_module.import_meta_graph(meta_graph_def, import_scope="new_model")
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       sess.run(["new_model/optimize"], {
           "new_model/image:0": np.random.random([1, 784]),
           "new_model/label:0": np.random.randint(
@@ -2374,7 +2374,7 @@ class MetaGraphTest(test.TestCase):
                            meta_graph_def_from_graph_def]:
       with session.Session(graph=ops_lib.Graph()) as sess:
         saver_module.import_meta_graph(meta_graph_def, import_scope="new_model")
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         for i in range(10):
           self.assertEqual(i * i, sess.run("new_model/output:0"))
         with self.assertRaises(errors.OutOfRangeError):
@@ -2400,7 +2400,7 @@ class CheckpointReaderTest(test.TestCase):
     save_path = os.path.join(self.get_temp_dir(),
                              "ckpt_for_debug_string" + str(self._WRITE_VERSION))
     with self.cached_session() as sess:
-      self.evaluate(init_all_op)
+      sess.run(init_all_op)
       # Saves a checkpoint.
       save.save(sess, save_path)
 
@@ -2546,7 +2546,7 @@ class ScopedGraphTest(test.TestCase):
       self.assertEqual(["biases:0", "weights:0"], sorted(var_list.keys()))
 
     with self.session(graph=graph) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       saver = saver_module.Saver(var_list=var_list, max_to_keep=1)
       saver.save(sess, os.path.join(test_dir, ckpt_filename), write_state=False)
 
@@ -2611,7 +2611,7 @@ class ScopedGraphTest(test.TestCase):
       # Verify that we have restored weights1 and biases1.
       sess.run([weights1, biases1])
       # Initialize the rest of the variables and run logits.
-      self.evaluate(init_rest_op)
+      sess.run(init_rest_op)
       sess.run(logits)
 
   # Verifies that we can save the subgraph under "hidden1" and restore it
@@ -2640,7 +2640,7 @@ class ScopedGraphTest(test.TestCase):
 
     # Run the graph and save scoped checkpoint.
     with self.session(graph=graph1) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       _, var_list_1 = meta_graph.export_scoped_meta_graph(
           export_scope="hidden1")
       saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1)
@@ -2696,7 +2696,7 @@ class ScopedGraphTest(test.TestCase):
 
     # Run the graph and save scoped checkpoint.
     with self.session(graph=graph1) as sess:
-      self.evaluate(variables.global_variables_initializer())
+      sess.run(variables.global_variables_initializer())
       _, var_list_1 = meta_graph.export_scoped_meta_graph(
           graph_def=graph1.as_graph_def(), export_scope="hidden1")
       saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1)
@@ -2964,7 +2964,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
     a_saver = saver_module.Saver([a])
     b_saver = saver_module.Saver([b])
     with self.cached_session() as sess:
-      self.evaluate(a.initializer)
+      sess.run(a.initializer)
       save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix)
       with self.assertRaisesRegexp(
           errors.NotFoundError, "Key b not found in checkpoint"):
@@ -2986,7 +2986,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
       a_saver = saver_module.Saver([a])
 
       with self.session(graph=g) as sess:
-        self.evaluate(a.initializer)
+        sess.run(a.initializer)
         save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix)
 
     with ops_lib.Graph().as_default() as g:
diff --git a/tensorflow/python/training/server_lib_sparse_job_test.py b/tensorflow/python/training/server_lib_sparse_job_test.py
index 8c2745b51a..1a6b44b90e 100644
--- a/tensorflow/python/training/server_lib_sparse_job_test.py
+++ b/tensorflow/python/training/server_lib_sparse_job_test.py
@@ -36,7 +36,7 @@ class SparseJobTest(test.TestCase):
       a = constant_op.constant(1.0)
 
     with session.Session(server.target) as sess:
-      self.assertEqual(1.0, self.evaluate(a))
+      self.assertEqual(1.0, sess.run(a))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py
index 9dc88d78cc..b734e9653e 100644
--- a/tensorflow/python/training/supervisor_test.py
+++ b/tensorflow/python/training/supervisor_test.py
@@ -100,7 +100,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir=logdir)
       sess = sv.prepare_or_wait_for_session("")
       for _ in xrange(10):
-        self.evaluate(my_op)
+        sess.run(my_op)
       sess.close()
       sv.stop()
 
@@ -111,7 +111,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir=logdir)
       with sv.managed_session("") as sess:
         for _ in xrange(10):
-          self.evaluate(my_op)
+          sess.run(my_op)
       # Supervisor has been stopped.
       self.assertTrue(sv.should_stop())
 
@@ -128,7 +128,7 @@ class SupervisorTest(test.TestCase):
             if step == 1:
               raise RuntimeError("failing here")
             else:
-              self.evaluate(my_op)
+              sess.run(my_op)
       # Supervisor has been stopped.
       self.assertTrue(sv.should_stop())
       self.assertEqual(1, last_step)
@@ -146,7 +146,7 @@ class SupervisorTest(test.TestCase):
             raise errors_impl.OutOfRangeError(my_op.op.node_def, my_op.op,
                                               "all done")
           else:
-            self.evaluate(my_op)
+            sess.run(my_op)
       # Supervisor has been stopped.  OutOfRangeError was not thrown.
       self.assertTrue(sv.should_stop())
       self.assertEqual(3, last_step)
@@ -335,7 +335,7 @@ class SupervisorTest(test.TestCase):
       sess = sv.prepare_or_wait_for_session(
           "", config=config_pb2.ConfigProto(device_count={"CPU": 2}))
       for _ in xrange(10):
-        self.evaluate(my_op)
+        sess.run(my_op)
       sess.close()
       sv.stop()
 
diff --git a/tensorflow/python/training/warm_starting_util_test.py b/tensorflow/python/training/warm_starting_util_test.py
index f1e719e6db..b575b8d364 100644
--- a/tensorflow/python/training/warm_starting_util_test.py
+++ b/tensorflow/python/training/warm_starting_util_test.py
@@ -49,7 +49,7 @@ class WarmStartingUtilTest(test.TestCase):
     return vocab_file
 
   def _write_checkpoint(self, sess):
-    self.evaluate(variables.global_variables_initializer())
+    sess.run(variables.global_variables_initializer())
     saver = saver_lib.Saver()
     ckpt_prefix = os.path.join(self.get_temp_dir(), "model")
     saver.save(sess, ckpt_prefix, global_step=0)
@@ -125,7 +125,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllClose(prev_val, fruit_weights.eval(sess))
 
   def testWarmStartVarPrevVarPartitioned(self):
@@ -143,7 +143,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllClose(prev_val, fruit_weights.eval(sess))
 
   def testWarmStartVarCurrentVarPartitioned(self):
@@ -162,7 +162,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         fruit_weights = fruit_weights._get_variable_list()
         new_val = np.concatenate(
             [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
@@ -189,7 +189,7 @@ class WarmStartingUtilTest(test.TestCase):
             fruit_weights, prev_tensor_name="old_scope/fruit_weights")
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         fruit_weights = fruit_weights._get_variable_list()
         new_val = np.concatenate(
             [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
@@ -211,7 +211,7 @@ class WarmStartingUtilTest(test.TestCase):
             "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5,
                                            self.get_temp_dir(), prev_vocab_path)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
 
@@ -236,7 +236,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.],
                              [2.3, 2., 0.]], fruit_output_layer.eval(sess))
 
@@ -261,7 +261,7 @@ class WarmStartingUtilTest(test.TestCase):
             self.get_temp_dir(),
             prev_vocab_path,
             previous_vocab_size=2)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Old vocabulary limited to ['apple', 'banana'].
         self.assertAllClose([[0.], [0.], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
@@ -285,7 +285,7 @@ class WarmStartingUtilTest(test.TestCase):
             "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5,
                                            self.get_temp_dir(), prev_vocab_path)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
 
@@ -312,7 +312,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.],
                              [2.3, 2., 0.]], fruit_output_layer.eval(sess))
 
@@ -340,7 +340,7 @@ class WarmStartingUtilTest(test.TestCase):
             self.get_temp_dir(),
             prev_vocab_path,
             current_oov_buckets=1)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_weights, variables.PartitionedVariable))
         fruit_weights_vars = fruit_weights._get_variable_list()
@@ -372,7 +372,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_output_layer, variables.PartitionedVariable))
         fruit_output_layer_vars = fruit_output_layer._get_variable_list()
@@ -404,7 +404,7 @@ class WarmStartingUtilTest(test.TestCase):
             partitioner=lambda shape, dtype: [2, 1])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 6,
                                            self.get_temp_dir(), prev_vocab_path)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_weights, variables.PartitionedVariable))
         fruit_weights_vars = fruit_weights._get_variable_list()
@@ -438,7 +438,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_output_layer, variables.PartitionedVariable))
         fruit_output_layer_vars = fruit_output_layer._get_variable_list()
@@ -463,7 +463,7 @@ class WarmStartingUtilTest(test.TestCase):
             shape=[10, 1],
             initializer=zeros())
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=[var])
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started (init overridden to ones).
         self.assertAllEqual(var.eval(), prev_int_val)
 
@@ -483,7 +483,7 @@ class WarmStartingUtilTest(test.TestCase):
             shape=[10, 1],
             initializer=zeros())
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=["v1"])
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started (init overridden to ones).
         self.assertAllEqual(var.eval(), prev_int_val)
 
@@ -519,7 +519,7 @@ class WarmStartingUtilTest(test.TestCase):
                            # This warm-starts both v1 and v1/Momentum, but only
                            # v2 (and not v2/Momentum).
                            vars_to_warm_start=["v1", "v2[^/]"])
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify the selection of weights were correctly warm-started (init
         # overridden to ones).
         self.assertAllEqual(v1.eval(), prev_v1_val)
@@ -542,7 +542,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_int], partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_int: [np.zeros([10, 1])]},
@@ -553,7 +553,7 @@ class WarmStartingUtilTest(test.TestCase):
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_int], partitioner)
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=".*sc_int.*")
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_int: [prev_int_val]}, sess)
 
@@ -571,7 +571,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_hash], partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_hash: [np.zeros([15, 1])]},
@@ -583,7 +583,7 @@ class WarmStartingUtilTest(test.TestCase):
         cols_to_vars = self._create_linear_model([sc_hash], partitioner)
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*sc_hash.*")
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_hash: [prev_hash_val]},
                                   sess)
@@ -605,7 +605,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
@@ -619,7 +619,7 @@ class WarmStartingUtilTest(test.TestCase):
         # vocab is assumed to be same as new vocab.
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*sc_vocab.*")
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
                                   sess)
@@ -641,7 +641,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
@@ -657,7 +657,7 @@ class WarmStartingUtilTest(test.TestCase):
             # Explicitly provide the file prefix instead of just the dir.
             os.path.join(self.get_temp_dir(), "model-0"),
             vars_to_warm_start=".*sc_vocab.*")
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
                                   sess)
@@ -686,7 +686,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([2, 1])]},
@@ -708,7 +708,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab/weights": vocab_info
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  'banana' isn't in the
         # first two entries of the old vocabulary, so it's newly initialized.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [[[1], [0]]]}, sess)
@@ -729,7 +729,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([real_bucket], partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars,
@@ -741,7 +741,7 @@ class WarmStartingUtilTest(test.TestCase):
         cols_to_vars = self._create_linear_model([real_bucket], partitioner)
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*real_bucketized.*")
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars,
                                   {real_bucket: [prev_bucket_val]}, sess)
@@ -800,7 +800,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model(all_linear_cols, partitioner)
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Without warm-starting, all weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {
@@ -826,7 +826,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab/weights": vocab_info
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {
             sc_int: [prev_int_val],
@@ -865,7 +865,7 @@ class WarmStartingUtilTest(test.TestCase):
             "linear_model/sc_vocab/weights",
             initializer=[[0.5], [1.], [2.], [3.]])
         self._write_checkpoint(sess)
-        prev_keys_val = self.evaluate(sc_keys_weights)
+        prev_keys_val = sess.run(sc_keys_weights)
 
     def _partitioner(shape, dtype):  # pylint:disable=unused-argument
       # Partition each var into 2 equal slices.
@@ -892,7 +892,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_hash should not be warm-started.  Var corresponding to sc_vocab
         # should be correctly warm-started after vocab remapping.
@@ -933,7 +933,7 @@ class WarmStartingUtilTest(test.TestCase):
             "linear_model/sc_vocab/weights",
             initializer=[[0.5], [1.], [2.], [3.]])
         self._write_checkpoint(sess)
-        prev_keys_val = self.evaluate(sc_keys_weights)
+        prev_keys_val = sess.run(sc_keys_weights)
 
     # New graph, new session with warm-starting.
     with ops.Graph().as_default() as g:
@@ -955,7 +955,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_hash should not be warm-started.  Var corresponding to sc_vocab
         # should be correctly warm-started after vocab remapping.
@@ -1024,7 +1024,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_vocab should be correctly warm-started after vocab remapping,
         # and neither of the other two should be warm-started..
@@ -1091,7 +1091,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[emb_vocab_column]):
                     vocab_info
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started. Var corresponding to
         # emb_vocab_column should be correctly warm-started after vocab
         # remapping. Missing values are filled in with the EmbeddingColumn's
@@ -1163,7 +1163,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab_embedding/embedding_weights": vocab_info
             })
-        self.evaluate(variables.global_variables_initializer())
+        sess.run(variables.global_variables_initializer())
         # Verify weights were correctly warm-started. Var corresponding to
         # emb_vocab should be correctly warm-started after vocab remapping.
         # Missing values are filled in with the EmbeddingColumn's initializer.
-- 
GitLab


From f22eec10b6c30b6ce1b6a05f0ac4dc69e9b237f4 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 21 Nov 2018 11:17:40 -0800
Subject: [PATCH 0678/1554] Replace ProgramShape proto with a C++ class. Rename
 the protobuf message ProgramShape to ProgramShapeProto and create a new
 ProgramShape C++ class with an interface which mirrors the protobuf generated
 code interface. This CL is a step toward replacing Shape proto with a C++
 class. ProgramShape needs to be migrated first because ProgramShape contains
 Shapes.

PiperOrigin-RevId: 222435461
---
 tensorflow/compiler/aot/codegen.cc            |  19 +--
 tensorflow/compiler/aot/codegen.h             |   2 +-
 tensorflow/compiler/aot/codegen_test.cc       |  16 +--
 tensorflow/compiler/aot/codegen_test_h.golden |  10 +-
 tensorflow/compiler/aot/codegen_test_o.golden | Bin 712 -> 720 bytes
 tensorflow/compiler/aot/compile.cc            |   4 +-
 tensorflow/compiler/aot/compile.h             |   6 +-
 .../compiler/aot/tests/tfcompile_test.cc      |   2 +-
 .../tf2xla/xla_compiled_cpu_function.h        |  10 +-
 .../tf2xla/xla_jit_compiled_cpu_function.cc   |   3 +-
 .../tf2xla/xla_jit_compiled_cpu_function.h    |   6 +-
 .../xla_jit_compiled_cpu_function_test.cc     |   2 +-
 tensorflow/compiler/xla/BUILD                 |  19 +++
 tensorflow/compiler/xla/client/BUILD          |   1 +
 tensorflow/compiler/xla/client/xla_builder.cc |   5 +-
 .../compiler/xla/client/xla_computation.cc    |   2 +-
 .../compiler/xla/client/xla_computation.h     |   1 +
 tensorflow/compiler/xla/layout_util.h         |   1 +
 .../xla/python/local_computation_builder.cc   |   9 +-
 .../xla/service/compile_only_service.cc       |   6 +-
 tensorflow/compiler/xla/service/hlo.proto     |   5 +-
 .../compiler/xla/service/hlo_computation.cc   |   2 +-
 tensorflow/compiler/xla/service/hlo_module.cc |   4 +-
 .../compiler/xla/service/local_service.cc     |   2 +-
 tensorflow/compiler/xla/service/service.cc    |  12 +-
 tensorflow/compiler/xla/shape.cc              |  62 ++++++++++
 tensorflow/compiler/xla/shape.h               | 108 +++++++++++++++++
 tensorflow/compiler/xla/shape_test.cc         | 112 ++++++++++++++++++
 tensorflow/compiler/xla/shape_util.cc         |  14 +++
 tensorflow/compiler/xla/shape_util.h          |   2 +
 tensorflow/compiler/xla/shape_util_test.cc    |  31 -----
 tensorflow/compiler/xla/tests/replay_test.cc  |   9 +-
 tensorflow/compiler/xla/xla_data.proto        |   2 +-
 .../compiler/xrt/kernels/xrt_compile_ops.cc   |  11 +-
 tensorflow/compiler/xrt/tests/raw_api_test.cc |  10 +-
 tensorflow/compiler/xrt/xrt.proto             |   4 +-
 36 files changed, 408 insertions(+), 106 deletions(-)
 create mode 100644 tensorflow/compiler/xla/shape.cc
 create mode 100644 tensorflow/compiler/xla/shape.h
 create mode 100644 tensorflow/compiler/xla/shape_test.cc

diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc
index b17bc658fa..697599f3bb 100644
--- a/tensorflow/compiler/aot/codegen.cc
+++ b/tensorflow/compiler/aot/codegen.cc
@@ -164,7 +164,8 @@ string RewriteWithName(const string& name, string code,
 }
 
 // Generate methods for args (inputs).
-Status GenArgMethods(const tf2xla::Config& config, const xla::ProgramShape& ps,
+Status GenArgMethods(const tf2xla::Config& config,
+                     const xla::ProgramShapeProto& ps,
                      const CompileResult& compile_result, string* methods) {
   size_t num_args = ps.parameters_size();
   if (config.feed_size() != num_args) {
@@ -204,7 +205,7 @@ Status GenArgMethods(const tf2xla::Config& config, const xla::ProgramShape& ps,
 
 // Generate methods for results (outputs).
 Status GenResultMethods(const tf2xla::Config& config,
-                        const xla::ProgramShape& ps, string* methods) {
+                        const xla::ProgramShapeProto& ps, string* methods) {
   if (ps.result().element_type() != xla::TUPLE) {
     // The XlaCompiler we use to build the xla computation always generates a
     // tuple result, and we rely on this to simplify code generation.
@@ -336,7 +337,7 @@ Status GenerateHeader(const CodegenOpts& opts, const tf2xla::Config& config,
       ExtractEntryParamBufferInfos(buffer_infos);
   std::vector<BufferInfo> buffer_infos_for_temps =
       ExtractTempBufferInfos(buffer_infos);
-  const xla::ProgramShape& ps = compile_result.program_shape;
+  const xla::ProgramShapeProto& ps = compile_result.program_shape;
   string methods_arg, methods_result;
   TF_RETURN_IF_ERROR(GenArgMethods(config, ps, compile_result, &methods_arg));
   TF_RETURN_IF_ERROR(GenResultMethods(config, ps, &methods_result));
@@ -548,8 +549,8 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction {
   static const char** StaticResultNames() {{RESULT_NAMES_CODE}}
 
   // Shape of the args and results.
-  static const xla::ProgramShape* StaticProgramShape() {
-    static const xla::ProgramShape* kShape = {{PROGRAM_SHAPE_SHIM_EXPRESSION}};
+  static const xla::ProgramShapeProto* StaticProgramShape() {
+    static const xla::ProgramShapeProto* kShape = {{PROGRAM_SHAPE_SHIM_EXPRESSION}};
     return kShape;
   }
 
@@ -615,11 +616,11 @@ static string CreateUniqueIdentifier(const CodegenOpts& opts,
 Status GenerateMetadata(const CodegenOpts& opts,
                         const CompileResult& compile_result,
                         MetadataResult* metadata_result) {
-  std::unique_ptr<xla::ProgramShape> program_shape;
+  std::unique_ptr<xla::ProgramShapeProto> program_shape;
 
   if (opts.gen_program_shape) {
     program_shape =
-        absl::make_unique<xla::ProgramShape>(compile_result.program_shape);
+        absl::make_unique<xla::ProgramShapeProto>(compile_result.program_shape);
 
     // The parameter names are currently meaningless, and redundant with the
     // rest of our metadata, so clear them out to avoid confusion and save
@@ -631,8 +632,8 @@ Status GenerateMetadata(const CodegenOpts& opts,
   // a shim that evaluates to nullptr, which is what we want.
 
   ProtobufToEmbed program_shape_protobuf{
-      CreateUniqueIdentifier(opts, "ProgramShape"), "xla::ProgramShape",
-      program_shape.get()};
+      CreateUniqueIdentifier(opts, "ProgramShapeProto"),
+      "xla::ProgramShapeProto", program_shape.get()};
 
   ProtobufToEmbed hlo_profile_printer_data_protobuf{
       CreateUniqueIdentifier(opts, "HloProfilePrinterData"),
diff --git a/tensorflow/compiler/aot/codegen.h b/tensorflow/compiler/aot/codegen.h
index 90410c46a8..9485e86b10 100644
--- a/tensorflow/compiler/aot/codegen.h
+++ b/tensorflow/compiler/aot/codegen.h
@@ -57,7 +57,7 @@ struct MetadataResult {
   std::vector<string> header_variable_decls;
 
   // program_shape_access_shim is a C++ expression that constructs the
-  // xla::ProgramShape instance for the CompileResult passed to
+  // xla::ProgramShapeProto instance for the CompileResult passed to
   // GenerateMetadata.
   string program_shape_access_shim;
 
diff --git a/tensorflow/compiler/aot/codegen_test.cc b/tensorflow/compiler/aot/codegen_test.cc
index bb288d2300..c1788ca32a 100644
--- a/tensorflow/compiler/aot/codegen_test.cc
+++ b/tensorflow/compiler/aot/codegen_test.cc
@@ -181,13 +181,15 @@ TEST(CodegenTest, Golden) {
        BufferInfo::MakeEntryParameter(/*size=*/96, /*param_number=*/1),
        BufferInfo::MakeTempBuffer(3), BufferInfo::MakeTempBuffer(120)},
       5, {}));
-  compile_result.program_shape = xla::ShapeUtil::MakeProgramShape(
-      {
-          xla::ShapeUtil::MakeShape(xla::F32, {1, 2}),
-          xla::ShapeUtil::MakeShape(xla::S64, {3, 4}),
-      },
-      xla::ShapeUtil::MakeTupleShape(
-          {xla::ShapeUtil::MakeShape(xla::U32, {5, 6})}));
+  compile_result.program_shape =
+      xla::ShapeUtil::MakeProgramShape(
+          {
+              xla::ShapeUtil::MakeShape(xla::F32, {1, 2}),
+              xla::ShapeUtil::MakeShape(xla::S64, {3, 4}),
+          },
+          xla::ShapeUtil::MakeTupleShape(
+              {xla::ShapeUtil::MakeShape(xla::U32, {5, 6})}))
+          .ToProto();
   compile_result.entry_point = "entry_point";
   compile_result.pointer_size = 8;
 
diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden
index e4d8a02877..a2cdab5d1a 100644
--- a/tensorflow/compiler/aot/codegen_test_h.golden
+++ b/tensorflow/compiler/aot/codegen_test_h.golden
@@ -22,7 +22,7 @@ extern "C" void entry_point(
     void* result, const xla::ExecutableRunOptions* run_options,
     const void** args, void** temps, tensorflow::int64* profile_counters);
 
-extern "C" char __tfcompile_foo_bar_MyClass_ProgramShape_protobuf_array_contents[];
+extern "C" char __tfcompile_foo_bar_MyClass_ProgramShapeProto_protobuf_array_contents[];
 
 
 namespace foo {
@@ -253,10 +253,10 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction {
   }
 
   // Shape of the args and results.
-  static const xla::ProgramShape* StaticProgramShape() {
-    static const xla::ProgramShape* kShape = []() {
-    xla::ProgramShape* proto = new xla::ProgramShape;
-    proto->ParseFromArray(&__tfcompile_foo_bar_MyClass_ProgramShape_protobuf_array_contents[0], 52);
+  static const xla::ProgramShapeProto* StaticProgramShape() {
+    static const xla::ProgramShapeProto* kShape = []() {
+    xla::ProgramShapeProto* proto = new xla::ProgramShapeProto;
+    proto->ParseFromArray(&__tfcompile_foo_bar_MyClass_ProgramShapeProto_protobuf_array_contents[0], 52);
     return proto;
   }();
     return kShape;
diff --git a/tensorflow/compiler/aot/codegen_test_o.golden b/tensorflow/compiler/aot/codegen_test_o.golden
index eb001c5d45bdfefc76629d7303d89f5480432235..ce8e5ec8c96a2c3696f14b8eea206d648182ecb5 100644
GIT binary patch
delta 82
zcmX@XdVzI<24lcP&2+}ti4)^k1B&uX@+ZEZ$-%(DP{hE%z&tsSQF!tKMh?cV$txM-
g8Ji|^GTF2GGB7YOf@KnzG+3(`7#Ntqyah}e0OJr9Qvd(}

delta 49
zcmcb>dV+O=2BXJB&2+|yi4)@{ewoRbJ9#3bJY(zRjg0Y(wUY&z>=`{K2Qt|+mQJ3?
GWDfwepb$g=

diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index 2b5f97b34c..3bc99ef7e6 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -56,8 +56,8 @@ Status CompileXla(xla::CompileOnlyClient* client,
     return errors::Unknown("Couldn't get XLA program shape: ",
                            pshape_or.status().error_message());
   }
-  compile_result->program_shape = *pshape_or.ValueOrDie();
-  xla::ProgramShape* pshape = &compile_result->program_shape;
+  compile_result->program_shape = pshape_or.ValueOrDie()->ToProto();
+  xla::ProgramShapeProto* pshape = &compile_result->program_shape;
   std::vector<const xla::Shape*> arg_layouts;
   arg_layouts.reserve(pshape->parameters_size());
   for (int i = 0; i < pshape->parameters_size(); ++i) {
diff --git a/tensorflow/compiler/aot/compile.h b/tensorflow/compiler/aot/compile.h
index e03c5b1aa7..ee7bb26fab 100644
--- a/tensorflow/compiler/aot/compile.h
+++ b/tensorflow/compiler/aot/compile.h
@@ -33,9 +33,9 @@ namespace tfcompile {
 struct CompileResult {
   // Contains object file and meta-info.
   std::unique_ptr<xla::cpu::CpuAotCompilationResult> aot;
-  xla::ProgramShape program_shape;  // Static shape of args and results.
-  string entry_point;               // Name of generated function.
-  int pointer_size = 0;             // Size of a pointer in bytes.
+  xla::ProgramShapeProto program_shape;  // Static shape of args and results.
+  string entry_point;                    // Name of generated function.
+  int pointer_size = 0;                  // Size of a pointer in bytes.
 };
 
 // CompileGraph compiles the graph_def into an object file containing a function
diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc
index f10852c785..711feed8f3 100644
--- a/tensorflow/compiler/aot/tests/tfcompile_test.cc
+++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc
@@ -526,7 +526,7 @@ TEST(TFCompileTest, ProgramShape) {
 
   // muladd has the program shape defined.
   MatMulAndAddComp muladd;
-  const xla::ProgramShape* muladd_shape = muladd.ProgramShape();
+  const xla::ProgramShapeProto* muladd_shape = muladd.ProgramShape();
   ASSERT_TRUE(muladd_shape != nullptr);
   ASSERT_EQ(muladd_shape->parameters_size(), 2);
   EXPECT_TRUE(ShapeUtil::Compatible(muladd_shape->parameters(0), f32_2x2));
diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index 66206909a9..a1d359e97c 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -26,7 +26,7 @@ limitations under the License.
 // Forward-declare, rather than include, to reduce code size for users that
 // never use this functionality.
 namespace xla {
-class ProgramShape;
+class ProgramShapeProto;
 class HloProfilePrinterData;
 }
 
@@ -84,7 +84,7 @@ class XlaCompiledCpuFunction {
     void set_result_names(const char** result_names) {
       result_names_ = result_names;
     }
-    void set_program_shape(const xla::ProgramShape* program_shape) {
+    void set_program_shape(const xla::ProgramShapeProto* program_shape) {
       program_shape_ = program_shape;
     }
     const xla::HloProfilePrinterData* hlo_profile_printer_data() const {
@@ -122,7 +122,7 @@ class XlaCompiledCpuFunction {
     const char** result_names_ = nullptr;
 
     // [Optional] Arg and result shapes.
-    const xla::ProgramShape* program_shape_ = nullptr;
+    const xla::ProgramShapeProto* program_shape_ = nullptr;
 
     // [Optional] Profile printer data.  Null if profiling is disabled.
     const xla::HloProfilePrinterData* hlo_profile_printer_data_ = nullptr;
@@ -264,7 +264,7 @@ class XlaCompiledCpuFunction {
 
   // Returns the shape of the args and results. May return nullptr if the
   // program shape isn't available.
-  const xla::ProgramShape* ProgramShape() const { return program_shape_; }
+  const xla::ProgramShapeProto* ProgramShape() const { return program_shape_; }
 
   bool hlo_profiling_enabled() const {
     return hlo_profile_printer_data_ != nullptr;
@@ -305,7 +305,7 @@ class XlaCompiledCpuFunction {
   // Optional metadata.
   const char** arg_names_ = nullptr;
   const char** result_names_ = nullptr;
-  const xla::ProgramShape* program_shape_ = nullptr;
+  const xla::ProgramShapeProto* program_shape_ = nullptr;
   const xla::HloProfilePrinterData* hlo_profile_printer_data_ = nullptr;
 };
 
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
index 86a78ee429..fabbcd04fe 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
@@ -133,7 +133,8 @@ XlaJitCompiledCpuFunction::Compile(
   jit->executable_ = std::move(executable);
   jit->buffer_infos_ = std::move(buffer_infos);
   jit->arg_index_table_ = std::move(arg_index_table);
-  jit->program_shape_ = std::move(program_shape);
+  jit->program_shape_ =
+      absl::make_unique<xla::ProgramShapeProto>(program_shape->ToProto());
   jit->static_data_.set_raw_function(raw_function);
   jit->static_data_.set_buffer_infos(jit->buffer_infos_.data());
   jit->static_data_.set_num_buffers(jit->buffer_infos_.size());
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h
index d3c8f22a80..a539205717 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.h
@@ -80,8 +80,10 @@ class XlaJitCompiledCpuFunction {
   std::vector<const char*> arg_names_;
   std::vector<const char*> result_names_;
 
-  // The backing data for the program shape.
-  std::unique_ptr<const xla::ProgramShape> program_shape_;
+  // The backing data for the program shape. The proto form of program shape is
+  // used because the program shape is serialized and embedded in the object
+  // file.
+  std::unique_ptr<const xla::ProgramShapeProto> program_shape_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
index 6d49298a6f..4496255d00 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
@@ -116,7 +116,7 @@ TEST(XlaJitCompiledCpuFunction, Sum) {
   // Check program shape.
   using xla::ShapeUtil;
   const xla::Shape s32 = ShapeUtil::MakeShape(xla::S32, {});
-  const xla::ProgramShape* program_shape = function.ProgramShape();
+  const xla::ProgramShapeProto* program_shape = function.ProgramShape();
   ASSERT_TRUE(program_shape != nullptr);
   ASSERT_EQ(program_shape->parameters_size(), 2);
   EXPECT_TRUE(ShapeUtil::Compatible(program_shape->parameters(0), s32));
diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index d914e97b6b..4360e08579 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -226,12 +226,14 @@ cc_library(
         "index_util.cc",
         "layout_util.cc",
         "primitive_util.cc",
+        "shape.cc",
         "shape_util.cc",
     ],
     hdrs = [
         "index_util.h",
         "layout_util.h",
         "primitive_util.h",
+        "shape.h",
         "shape_util.h",
     ],
     visibility = ["//visibility:public"],
@@ -254,6 +256,23 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "shape_test",
+    srcs = ["shape_test.cc"],
+    deps = [
+        ":shape_util",
+        ":status_macros",
+        ":test",
+        ":test_helpers",
+        ":types",
+        ":util",
+        ":xla_data_proto",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test_main",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 tf_cc_test(
     name = "shape_util_test",
     srcs = ["shape_util_test.cc"],
diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index 42da0ebf49..e1f3674c4f 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -191,6 +191,7 @@ cc_library(
     hdrs = ["xla_computation.h"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index f508ffb9c9..8a33b3930f 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -288,7 +288,8 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
 
   HloComputationProto entry;
   SetProtoIdAndName(&entry, name_, kNameSeparator, GetNextId());
-  TF_ASSIGN_OR_RETURN(*entry.mutable_program_shape(), GetProgramShape(root_id));
+  TF_ASSIGN_OR_RETURN(ProgramShape program_shape, GetProgramShape(root_id));
+  *entry.mutable_program_shape() = program_shape.ToProto();
   entry.set_root_id(root_id);
 
   for (auto& instruction : instructions_) {
@@ -2372,7 +2373,7 @@ StatusOr<XlaComputation> XlaBuilder::BuildConstantSubGraph(
   SetProtoIdAndName(&entry, StrCat(name_, "_compute_constant"), kNameSeparator,
                     GetNextId());
   entry.set_root_id(root->id());
-  ProgramShape* program_shape = entry.mutable_program_shape();
+  ProgramShapeProto* program_shape = entry.mutable_program_shape();
   *program_shape->mutable_result() = root->shape();
 
   // We use std::set to keep the instruction ids in ascending order (which is
diff --git a/tensorflow/compiler/xla/client/xla_computation.cc b/tensorflow/compiler/xla/client/xla_computation.cc
index c9870b65b9..f317892c12 100644
--- a/tensorflow/compiler/xla/client/xla_computation.cc
+++ b/tensorflow/compiler/xla/client/xla_computation.cc
@@ -25,7 +25,7 @@ namespace xla {
 
 StatusOr<ProgramShape> XlaComputation::GetProgramShape() const {
   TF_RET_CHECK(proto_.has_host_program_shape());
-  return proto_.host_program_shape();
+  return ProgramShape(proto_.host_program_shape());
 }
 
 StatusOr<std::unique_ptr<HloSnapshot>> XlaComputation::Snapshot() const {
diff --git a/tensorflow/compiler/xla/client/xla_computation.h b/tensorflow/compiler/xla/client/xla_computation.h
index 71598ef8b2..3ccbfb28bd 100644
--- a/tensorflow/compiler/xla/client/xla_computation.h
+++ b/tensorflow/compiler/xla/client/xla_computation.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
diff --git a/tensorflow/compiler/xla/layout_util.h b/tensorflow/compiler/xla/layout_util.h
index 6e0390763d..6c298e5725 100644
--- a/tensorflow/compiler/xla/layout_util.h
+++ b/tensorflow/compiler/xla/layout_util.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include <string>
 
 #include "absl/types/span.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/status.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 4d2a37cfac..2768ed618d 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -487,12 +487,13 @@ StatusOr<CompiledXrtComputation*> LocalComputation::CompileForXrt(
 
   xrt::XLAComputation c;
   auto config = c.mutable_config();
-  auto shapes = config->mutable_program_shape();
+  ProgramShape shapes;
   for (auto& shape : argument_shapes) {
-    *shapes->add_parameters() = shape;
+    *shapes.add_parameters() = shape;
   }
-  TF_ASSIGN_OR_RETURN(*shapes->mutable_result(), GetReturnValueShape());
-  LayoutUtil::SetToDefaultLayout(shapes);
+  TF_ASSIGN_OR_RETURN(*shapes.mutable_result(), GetReturnValueShape());
+  LayoutUtil::SetToDefaultLayout(&shapes);
+  *config->mutable_program_shape() = shapes.ToProto();
   auto snapshot = computation().Snapshot().ValueOrDie();
   *c.mutable_hlo_snapshot() = *snapshot;
 
diff --git a/tensorflow/compiler/xla/service/compile_only_service.cc b/tensorflow/compiler/xla/service/compile_only_service.cc
index 67132274c0..0237f16673 100644
--- a/tensorflow/compiler/xla/service/compile_only_service.cc
+++ b/tensorflow/compiler/xla/service/compile_only_service.cc
@@ -86,15 +86,15 @@ CompileOnlyService::CompileAheadOfTime(
           Executable::DumpToDirectory(per_host_path, filename, hlo_snapshot));
     }
 
-    const auto& program_shape = instance.computation.host_program_shape();
     ExecutionOptions execution_options;
     *execution_options.mutable_debug_options() = debug_options;
     *execution_options.mutable_shape_with_output_layout() =
         *instance.result_layout;
     TF_ASSIGN_OR_RETURN(
         std::unique_ptr<HloModuleConfig> module_config,
-        CreateModuleConfig(program_shape, instance.argument_layouts,
-                           &execution_options));
+        CreateModuleConfig(
+            ProgramShape(instance.computation.host_program_shape()),
+            instance.argument_layouts, &execution_options));
 
     TF_ASSIGN_OR_RETURN(
         std::unique_ptr<HloModule> hlo_module,
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 913d4c34b4..c62c935af7 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -205,7 +205,8 @@ message HloComputationProto {
   repeated HloInstructionProto instructions = 2;
 
   // The program shape (with layout) of this computation.
-  xla.ProgramShape program_shape = 4;
+
+  xla.ProgramShapeProto program_shape = 4;
 
   // The id of this computation.
   int64 id = 5;
@@ -297,7 +298,7 @@ message HloModuleProto {
   repeated HloComputationProto computations = 3;
 
   // The host program shape (with layout) of the entry computation.
-  xla.ProgramShape host_program_shape = 4;
+  xla.ProgramShapeProto host_program_shape = 4;
 
   // The id of this module.
   int64 id = 5;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 65bd251dd8..d06c2207cb 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -499,7 +499,7 @@ HloComputationProto HloComputation::ToProto() const {
     proto.add_instructions()->Swap(&instruction_proto);
   }
   proto.set_root_id(root_instruction()->unique_id());
-  *proto.mutable_program_shape() = ComputeProgramShape();
+  *proto.mutable_program_shape() = ComputeProgramShape().ToProto();
   return proto;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 59f44475df..a01853fe1f 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -240,7 +240,7 @@ HloModuleProto HloModule::ToProto() const {
     *proto.mutable_schedule() = schedule().ToProto().ValueOrDie();
   }
   *proto.mutable_host_program_shape() =
-      entry_computation_layout().ComputeProgramShape();
+      entry_computation_layout().ComputeProgramShape().ToProto();
   *proto.mutable_input_output_alias() = input_output_alias_config().ToProto();
   *proto.mutable_dynamic_parameter_binding() =
       dynamic_parameter_binding().ToProto();
@@ -371,7 +371,7 @@ StatusOr<HloModuleConfig> HloModule::CreateModuleConfigFromProto(
       << "No program shape found in the proto";
   const auto& program_shape = module.host_program_shape();
 
-  HloModuleConfig module_config(program_shape);
+  HloModuleConfig module_config(ProgramShape{program_shape});
   module_config.set_debug_options(debug_options);
 
   // The module config is constructed with default layouts regardless of what is
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 2180ac845d..5c105908f3 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -145,7 +145,7 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
     const ExecutableBuildOptions& build_options) {
   const HloModuleProto& proto = computation.proto();
   TF_RET_CHECK(proto.has_host_program_shape());
-  const ProgramShape& program_shape = proto.host_program_shape();
+  ProgramShape program_shape(proto.host_program_shape());
 
   // Validate incoming layouts.
   if (argument_layouts.size() != program_shape.parameters_size()) {
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 13fd6bc009..c4b0a5c080 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -658,9 +658,9 @@ Status Service::ExecuteGraphParallel(const ExecuteGraphParallelRequest* arg,
     // replica 0.
     TF_ASSIGN_OR_RETURN(
         std::unique_ptr<HloModuleConfig> module_config,
-        CreateModuleConfig(request.computation().host_program_shape(),
-                           replicated_arguments.front(),
-                           request.execution_options()));
+        CreateModuleConfig(
+            ProgramShape{request.computation().host_program_shape()},
+            replicated_arguments.front(), request.execution_options()));
     VLOG(3)
         << "ExecuteGraphParallel created HloModuleConfig computation layout: "
         << module_config->entry_computation_layout().ToString();
@@ -824,7 +824,7 @@ Status Service::Compile(const CompileRequest* arg, CompileResponse* result) {
                     [](const Shape& shape) { return &shape; });
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr<HloModuleConfig> module_config,
-      CreateModuleConfig(arg->computation().host_program_shape(),
+      CreateModuleConfig(ProgramShape{arg->computation().host_program_shape()},
                          argument_shapes, &arg->execution_options()));
   VLOG(3) << "Compile created HloModuleConfig computation layout: "
           << module_config->entry_computation_layout().ToString();
@@ -1072,7 +1072,7 @@ Status Service::ComputeConstantGraph(const ComputeConstantGraphRequest* arg,
         "constant computation may not depend on any parameters.");
   }
 
-  ProgramShape program_shape = arg->computation().host_program_shape();
+  ProgramShape program_shape(arg->computation().host_program_shape());
   TF_DCHECK_OK(ShapeUtil::ValidateShape(program_shape.result()));
   if (arg->has_output_layout()) {
     TF_RETURN_IF_ERROR(LayoutUtil::ValidateLayoutForShape(
@@ -1116,7 +1116,7 @@ Status Service::GetComputationGraphStats(
     return InvalidArgument("Program shape may not be empty.");
   }
 
-  HloModuleConfig config(arg->computation().host_program_shape());
+  HloModuleConfig config(ProgramShape{arg->computation().host_program_shape()});
   config.set_debug_options(arg->debug_options());
   TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
                       CreateModuleFromProto(arg->computation(), config));
diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
new file mode 100644
index 0000000000..d209389c74
--- /dev/null
+++ b/tensorflow/compiler/xla/shape.cc
@@ -0,0 +1,62 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/shape.h"
+
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+
+namespace xla {
+
+ProgramShape::ProgramShape(const ProgramShapeProto& program_shape_proto) {
+  for (const Shape& shape : program_shape_proto.parameters()) {
+    *add_parameters() = shape;
+  }
+  *mutable_result() = program_shape_proto.result();
+  for (const string& name : program_shape_proto.parameter_names()) {
+    add_parameter_names(name);
+  }
+}
+
+ProgramShapeProto ProgramShape::ToProto() const {
+  ProgramShapeProto proto;
+  for (const Shape& shape : parameters()) {
+    *proto.add_parameters() = shape;
+  }
+  *proto.mutable_result() = result();
+  for (const string& name : parameter_names()) {
+    proto.add_parameter_names(name);
+  }
+  return proto;
+}
+
+string ProgramShape::ToString() const {
+  std::vector<string> parameter_strings(parameters_size());
+  for (int i = 0; i < parameters_size(); ++i) {
+    parameter_strings[i] = absl::StrCat(
+        i < parameter_names_size() ? parameter_names(i) : "(unknown)", ": ",
+        ShapeUtil::HumanString(parameters(i)));
+  }
+  return absl::StrCat("(", absl::StrJoin(parameter_strings, ", "), ") -> ",
+                      ShapeUtil::HumanString(result()));
+}
+
+std::ostream& operator<<(std::ostream& out, const ProgramShape& program_shape) {
+  out << program_shape.ToString() << "\n";
+  return out;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
new file mode 100644
index 0000000000..c3aecb1736
--- /dev/null
+++ b/tensorflow/compiler/xla/shape.h
@@ -0,0 +1,108 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SHAPE_H_
+#define TENSORFLOW_COMPILER_XLA_SHAPE_H_
+
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+
+// Shape of the parameters and output of an XLA computation. This is analogous
+// to a traditional function signature.
+class ProgramShape {
+ public:
+  ProgramShape() = default;
+
+  // Creates a ProgramShape from a ProgramShapeProto protobuf.
+  explicit ProgramShape(const ProgramShapeProto& program_shape_proto);
+
+  // Returns a proto representation of the object.
+  ProgramShapeProto ToProto() const;
+
+  string ToString() const;
+
+  // The following methods mirror the protobuf generated code interface for the
+  // message ProgramShapeProto. This enabled easy migration of this data
+  // structure from a proto to a proper C++ class.
+  // TODO(b/29771030): Replace or augment these methods with a more ergonomic
+  // interface.
+
+  // Methods for accessing and manipulating the Shape of the parameters.
+  int parameters_size() const { return parameters_.size(); }
+  const Shape& parameters(int index) const { return parameters_.at(index); }
+  Shape* mutable_parameters(int index) { return &parameters_.at(index); }
+  Shape* add_parameters() {
+    parameters_.emplace_back();
+    return &parameters_.back();
+  }
+  void clear_parameters() { parameters_.clear(); }
+  const std::vector<Shape>& parameters() const { return parameters_; }
+  std::vector<Shape>* mutable_parameters() { return &parameters_; }
+
+  // Methods for accessing and manipulating the Shape of the result.
+  const Shape& result() const { return result_; }
+  Shape* mutable_result() { return &result_; }
+  void clear_result() { result_.Clear(); }
+
+  // Methods for accessing and manipulating the names of the parameters.
+  int parameter_names_size() const { return parameter_names_.size(); }
+  const string& parameter_names(int index) const {
+    return parameter_names_.at(index);
+  }
+  void set_parameter_names(int index, const string& value) {
+    parameter_names_.at(index) = value;
+  }
+  string* mutable_parameter_names(int index) {
+    return &parameter_names_.at(index);
+  }
+  void add_parameter_names(const string& value) {
+    parameter_names_.push_back(value);
+  }
+  string* add_parameter_names() {
+    parameter_names_.push_back("");
+    return &parameter_names_.back();
+  }
+  void clear_parameter_names() { parameter_names_.clear(); }
+  const std::vector<string>& parameter_names() const {
+    return parameter_names_;
+  }
+  std::vector<string>* mutable_parameter_names() { return &parameter_names_; }
+
+  string ShortDebugString() const { return ToProto().ShortDebugString(); }
+  string DebugString() const { return ToProto().DebugString(); }
+
+ private:
+  // The shapes of the parameters of the computation represented by this object.
+  std::vector<Shape> parameters_;
+
+  // The names of the parameters of the computation represented by this object.
+  std::vector<string> parameter_names_;
+
+  // The shape of the result of the computation represented by this object.
+  Shape result_;
+};
+
+std::ostream& operator<<(std::ostream& out, const ProgramShape& program_shape);
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SHAPE_H_
diff --git a/tensorflow/compiler/xla/shape_test.cc b/tensorflow/compiler/xla/shape_test.cc
new file mode 100644
index 0000000000..cc3a5eb1d6
--- /dev/null
+++ b/tensorflow/compiler/xla/shape_test.cc
@@ -0,0 +1,112 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/shape.h"
+
+#include <numeric>
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+#include "tensorflow/compiler/xla/layout_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+TEST(ShapeTest, ProgramShapeToFromProto) {
+  ProgramShape program_shape;
+  *program_shape.add_parameters() = ShapeUtil::MakeShape(F32, {1, 2, 3});
+  *program_shape.add_parameters() = ShapeUtil::MakeTokenShape();
+  *program_shape.add_parameters() = ShapeUtil::MakeShape(S64, {});
+  *program_shape.add_parameters() = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(S32, {}),
+       ShapeUtil::MakeTupleShape({ShapeUtil::MakeTokenShape()}),
+       ShapeUtil::MakeShape(F32, {42, 42})});
+
+  *program_shape.mutable_result() = ShapeUtil::MakeShape(F32, {7});
+
+  program_shape.add_parameter_names("foo");
+  program_shape.add_parameter_names("bar");
+  program_shape.add_parameter_names("baz");
+  program_shape.add_parameter_names("qux qux");
+
+  // Create a copy of the program shape by round-tripping through a proto.
+  ProgramShape program_shape_copy(program_shape.ToProto());
+  ASSERT_EQ(program_shape.parameters_size(),
+            program_shape_copy.parameters_size());
+  for (int i = 0; i < program_shape.parameters_size(); ++i) {
+    EXPECT_TRUE(ShapeUtil::Equal(program_shape.parameters(i),
+                                 program_shape_copy.parameters(i)));
+  }
+
+  EXPECT_TRUE(
+      ShapeUtil::Equal(program_shape.result(), program_shape_copy.result()));
+
+  ASSERT_EQ(program_shape.parameter_names_size(),
+            program_shape_copy.parameter_names_size());
+  for (int i = 0; i < program_shape.parameter_names_size(); ++i) {
+    EXPECT_EQ(program_shape.parameter_names(i),
+              program_shape_copy.parameter_names(i));
+  }
+}
+
+TEST(ShapeTest, ProgramShapeToString) {
+  Shape opaque = ShapeUtil::MakeOpaqueShape();
+  Shape token = ShapeUtil::MakeTokenShape();
+  Shape scalar = ShapeUtil::MakeShape(F32, {});
+  Shape matrix = ShapeUtil::MakeShape(U32, {1, 2});
+  Shape matrix2 = ShapeUtil::MakeShapeWithLayout(S32, {3, 4}, {0, 1});
+  Shape tuple = ShapeUtil::MakeTupleShape({opaque, scalar, matrix, matrix2});
+  Shape nested_tuple = ShapeUtil::MakeTupleShape({tuple, matrix, token});
+
+  ProgramShape prog = ShapeUtil::MakeProgramShape(
+      {opaque, scalar, matrix, matrix2, tuple, nested_tuple}, nested_tuple);
+  EXPECT_EQ(
+      "((unknown): opaque[], "
+      "(unknown): f32[], "
+      "(unknown): u32[1,2], "
+      "(unknown): s32[3,4], "
+      "(unknown): (opaque[], f32[], u32[1,2], s32[3,4]), "
+      "(unknown): ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])) "
+      "-> "
+      "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
+      ShapeUtil::HumanString(prog));
+
+  prog.add_parameter_names("arg0");
+  prog.add_parameter_names("scalar");
+  prog.add_parameter_names("matrix");
+  prog.add_parameter_names("matrix2");
+  prog.add_parameter_names("tuple");
+  prog.add_parameter_names("nested_tuple");
+  EXPECT_EQ(
+      "(arg0: opaque[], "
+      "scalar: f32[], "
+      "matrix: u32[1,2], "
+      "matrix2: s32[3,4], "
+      "tuple: (opaque[], f32[], u32[1,2], s32[3,4]), "
+      "nested_tuple: ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], "
+      "token[])) "
+      "-> "
+      "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
+      ShapeUtil::HumanString(prog));
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 7d011bfc65..b05ec209cc 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -563,6 +563,20 @@ StatusOr<PrimitiveType> StringToPrimitiveType(const string& name) {
                 HumanString(program_shape.result()));
 }
 
+/* static */ string ShapeUtil::HumanString(
+    const ProgramShapeProto& program_shape_proto) {
+  std::vector<string> parameters;
+  for (auto& shape : program_shape_proto.parameters()) {
+    const int i = parameters.size();
+    parameters.push_back(StrCat(i < program_shape_proto.parameter_names_size()
+                                    ? program_shape_proto.parameter_names(i)
+                                    : "(unknown)",
+                                ": ", HumanString(shape)));
+  }
+  return StrCat("(", absl::StrJoin(parameters, ", "), ") -> ",
+                HumanString(program_shape_proto.result()));
+}
+
 namespace {
 // Parses shapes with simple recursive descent structure -- consumes from the
 // front of s and passes that view recursively as required.
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index 7f72e57d00..3796c5be5d 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
@@ -239,6 +240,7 @@ class ShapeUtil {
   //
   // (param_name: f32[42x12], ...) -> f32[24x42]
   static string HumanString(const ProgramShape& program_shape);
+  static string HumanString(const ProgramShapeProto& program_shape_proto);
 
   // Parses a ShapeUtil::HumanString-format shape string back into a shape
   // object.
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 11b493323c..ce6330a0dc 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -575,37 +575,6 @@ TEST(ShapeUtilTest, HumanString) {
       "((opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1}), u32[1,2]{1,0}, "
       "token[])",
       ShapeUtil::HumanStringWithLayout(nested_tuple));
-
-  ProgramShape prog = ShapeUtil::MakeProgramShape(
-      {opaque, scalar, matrix, matrix2, tuple, nested_tuple}, nested_tuple);
-  EXPECT_EQ(
-      "((unknown): opaque[], "
-      "(unknown): f32[], "
-      "(unknown): u32[1,2], "
-      "(unknown): s32[3,4], "
-      "(unknown): (opaque[], f32[], u32[1,2], s32[3,4]), "
-      "(unknown): ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])) "
-      "-> "
-      "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
-      ShapeUtil::HumanString(prog));
-
-  prog.add_parameter_names("arg0");
-  prog.add_parameter_names("scalar");
-  prog.add_parameter_names("matrix");
-  prog.add_parameter_names("matrix2");
-  prog.add_parameter_names("tuple");
-  prog.add_parameter_names("nested_tuple");
-  EXPECT_EQ(
-      "(arg0: opaque[], "
-      "scalar: f32[], "
-      "matrix: u32[1,2], "
-      "matrix2: s32[3,4], "
-      "tuple: (opaque[], f32[], u32[1,2], s32[3,4]), "
-      "nested_tuple: ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], "
-      "token[])) "
-      "-> "
-      "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
-      ShapeUtil::HumanString(prog));
 }
 
 TEST(ShapeUtilTest, ForEachSubshapeArray) {
diff --git a/tensorflow/compiler/xla/tests/replay_test.cc b/tensorflow/compiler/xla/tests/replay_test.cc
index 5cf87e565b..34c7dc7c46 100644
--- a/tensorflow/compiler/xla/tests/replay_test.cc
+++ b/tensorflow/compiler/xla/tests/replay_test.cc
@@ -55,7 +55,8 @@ TEST_F(ReplayTest, TwoPlusTwoReplay) {
       client_->GetComputationShape(computation).ConsumeValueOrDie();
   std::unique_ptr<ProgramShape> replayed_shape =
       client_->GetComputationShape(replayed).ConsumeValueOrDie();
-  ASSERT_TRUE(protobuf_util::ProtobufEquals(*original_shape, *replayed_shape));
+  ASSERT_TRUE(protobuf_util::ProtobufEquals(original_shape->ToProto(),
+                                            replayed_shape->ToProto()));
 
   // Run it.
   Literal literal =
@@ -87,7 +88,8 @@ XLA_TEST_F(ReplayTest, XPlusYReplayWithParameters) {
       client_->GetComputationShape(computation).ConsumeValueOrDie();
   std::unique_ptr<ProgramShape> replayed_shape =
       client_->GetComputationShape(replayed).ConsumeValueOrDie();
-  ASSERT_TRUE(protobuf_util::ProtobufEquals(*original_shape, *replayed_shape));
+  ASSERT_TRUE(protobuf_util::ProtobufEquals(original_shape->ToProto(),
+                                            replayed_shape->ToProto()));
 
   // Run it.
   std::unique_ptr<GlobalData> x_data =
@@ -133,7 +135,8 @@ TEST_F(ReplayTest, MapPlusTwoOverR1) {
       client_->GetComputationShape(computation).ConsumeValueOrDie();
   std::unique_ptr<ProgramShape> replayed_shape =
       client_->GetComputationShape(replayed).ConsumeValueOrDie();
-  ASSERT_TRUE(protobuf_util::ProtobufEquals(*original_shape, *replayed_shape));
+  ASSERT_TRUE(protobuf_util::ProtobufEquals(original_shape->ToProto(),
+                                            replayed_shape->ToProto()));
 
   // Run it.
   Literal literal =
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 683ccc40f1..27ef86ab2e 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -183,7 +183,7 @@ message Shape {
 
 // Shape of the parameters and output of a computation (like a traditional
 // function signature).
-message ProgramShape {
+message ProgramShapeProto {
   repeated Shape parameters = 1;
   Shape result = 2;
   repeated string parameter_names = 3;
diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
index dc62cf7a6b..db43aeaafe 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
@@ -174,11 +174,12 @@ void XRTCompileOp::Compute(OpKernelContext* ctx) {
   ctx->set_output(0, handle_output);
 
   xla::LocalExecutable* executable = entry->get().get_executable();
-  xla::ProgramShape program_shape = executable->executable()
-                                        ->module()
-                                        .config()
-                                        .entry_computation_layout()
-                                        .ComputeProgramShape();
+  xla::ProgramShapeProto program_shape = executable->executable()
+                                             ->module()
+                                             .config()
+                                             .entry_computation_layout()
+                                             .ComputeProgramShape()
+                                             .ToProto();
   Tensor program_shape_output(DT_STRING, TensorShape({1}));
   program_shape_output.vec<string>()(0) = program_shape.SerializeAsString();
   ctx->set_output(1, program_shape_output);
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index 25464b5554..7e73db98f7 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -411,7 +411,7 @@ TEST(RawApiTest, CompileAndExecute) {
   auto expected = xla::LiteralUtil::CreateR1<float>({27.0f, 21.0f});
   EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response));
 
-  xla::ProgramShape program_shape;
+  xla::ProgramShapeProto program_shape;
   EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec<string>()(0)));
   EXPECT_EQ(program_shape.parameters_size(), 2);
 }
@@ -465,7 +465,7 @@ TEST(RawApiTest, CompileAndExecuteWithArgumentVector) {
   auto expected = xla::LiteralUtil::CreateR1<float>({27.0f, 21.0f});
   EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response));
 
-  xla::ProgramShape program_shape;
+  xla::ProgramShapeProto program_shape;
   EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec<string>()(0)));
   EXPECT_EQ(program_shape.parameters_size(), 2);
 }
@@ -510,7 +510,7 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) {
   TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(),
                            {c_handle.program_shape}, {release}, &outputs));
 
-  xla::ProgramShape program_shape;
+  xla::ProgramShapeProto program_shape;
   EXPECT_TRUE(program_shape.ParseFromString(outputs[0].vec<string>()(0)));
   EXPECT_EQ(program_shape.parameters_size(), 1);
 
@@ -520,7 +520,7 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) {
           << xla::ShapeUtil::HumanStringWithLayout(program_shape.result());
 
   xla::ProgramShape xla_program_shape =
-      XlaCompiledProgramShape(xla_computation, *shapes);
+      XlaCompiledProgramShape(xla_computation, xla::ProgramShape(*shapes));
   EXPECT_TRUE(xla::LayoutUtil::Equal(
       xla::ShapeUtil::GetSubshape(program_shape.parameters(0), {0}).layout(),
       xla::ShapeUtil::GetSubshape(xla_program_shape.parameters(0), {0})
@@ -739,7 +739,7 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   auto expected = xla::LiteralUtil::CreateR0<int64>(15123899);
   EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response));
 
-  xla::ProgramShape program_shape;
+  xla::ProgramShapeProto program_shape;
   EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec<string>()(0)));
   EXPECT_EQ(program_shape.parameters_size(), 2);
   EXPECT_TRUE(
diff --git a/tensorflow/compiler/xrt/xrt.proto b/tensorflow/compiler/xrt/xrt.proto
index 6ab77fbaaf..ae44f71740 100644
--- a/tensorflow/compiler/xrt/xrt.proto
+++ b/tensorflow/compiler/xrt/xrt.proto
@@ -36,11 +36,11 @@ message XLAComputationConfig {
   tensorflow.tf2xla.HostComputeMetadata host_compute_metadata = 3;
 
   // The arg/result shapes for the whole computation.
-  xla.ProgramShape program_shape = 4;
+  xla.ProgramShapeProto program_shape = 4;
   // The arg/result shapes for each core of a model-parallel
   // computation. per_core_args_and_result_shapes is optional for a
   // single-core computation.
-  repeated xla.ProgramShape per_core_program_shape = 5;
+  repeated xla.ProgramShapeProto per_core_program_shape = 5;
   // Describes how replicated computation instances should be assigned to
   // devices. There are num_cores_per_replica computations, and each one will be
   // sent and executed to the set of replica device numbers described in the
-- 
GitLab


From 722ef17a0ed80bad61baac084a8c8cfc7bddbe3f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 21 Nov 2018 11:19:37 -0800
Subject: [PATCH 0679/1554] Use a non-trivial benchmark for
 kernel_tests/benchmark_test.py

Otherwise XLA constant folds the computation resulting in
allocator_maximum_num_bytes_* getting a value of 0.  At least that's what I
think is happening, I have not dug beyond confirming that the failure goes away
with this change.

PiperOrigin-RevId: 222435753
---
 tensorflow/python/kernel_tests/benchmark_test.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/benchmark_test.py b/tensorflow/python/kernel_tests/benchmark_test.py
index 5777a5d097..bffa5e6e8f 100644
--- a/tensorflow/python/kernel_tests/benchmark_test.py
+++ b/tensorflow/python/kernel_tests/benchmark_test.py
@@ -21,9 +21,12 @@ import json
 import os
 import random
 
+import numpy as np
+
 from tensorflow.core.util import test_log_pb2
 from tensorflow.python.client import session
-from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import benchmark
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -64,11 +67,17 @@ class TestReportingBenchmark(test.Benchmark):
                 "other_key": "string"})
 
   def benchmark_times_an_op(self):
+    input_size = 5
     with session.Session(config=benchmark.benchmark_config()) as sess:
-      a = constant_op.constant(0.0)
+      a = array_ops.placeholder(dtype=dtypes.float32, shape=(input_size))
       a_plus_a = a + a
       return self.run_op_benchmark(
-          sess, a_plus_a, min_iters=1000, store_trace=True, name="op_benchmark")
+          sess,
+          a_plus_a,
+          feed_dict={a: np.arange(input_size)},
+          min_iters=1000,
+          store_trace=True,
+          name="op_benchmark")
 
 
 class BenchmarkTest(test.TestCase):
-- 
GitLab


From dde1c81ae4998fd521d082009cb4bb0ed56e2f5b Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 21 Nov 2018 11:40:12 -0800
Subject: [PATCH 0680/1554] Change API for io.parse_example and
 io.parse_single_example for TF 2.0.

PiperOrigin-RevId: 222439043
---
 tensorflow/python/ops/parsing_ops.py          | 264 +++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.io.pbtxt   |   4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |   6 +
 3 files changed, 269 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py
index 484caf0179..7a11096d4f 100644
--- a/tensorflow/python/ops/parsing_ops.py
+++ b/tensorflow/python/ops/parsing_ops.py
@@ -363,7 +363,7 @@ def _prepend_none_dimension(features):
     return features
 
 
-@tf_export("io.parse_example", v1=["io.parse_example", "parse_example"])
+@tf_export(v1=["io.parse_example", "parse_example"])
 def parse_example(serialized, features, name=None, example_names=None):
   # pylint: disable=line-too-long
   """Parses `Example` protos into a `dict` of tensors.
@@ -574,6 +574,223 @@ def parse_example(serialized, features, name=None, example_names=None):
   Returns:
     A `dict` mapping feature keys to `Tensor` and `SparseTensor` values.
 
+  Raises:
+    ValueError: if any feature is invalid.
+  """
+  return parse_example_v2(serialized, features, example_names, name)
+
+
+@tf_export("io.parse_example", v1=[])
+def parse_example_v2(serialized, features, example_names=None, name=None):
+  # pylint: disable=line-too-long
+  """Parses `Example` protos into a `dict` of tensors.
+
+  Parses a number of serialized [`Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+  protos given in `serialized`. We refer to `serialized` as a batch with
+  `batch_size` many entries of individual `Example` protos.
+
+  `example_names` may contain descriptive names for the corresponding serialized
+  protos. These may be useful for debugging purposes, but they have no effect on
+  the output. If not `None`, `example_names` must be the same length as
+  `serialized`.
+
+  This op parses serialized examples into a dictionary mapping keys to `Tensor`
+  and `SparseTensor` objects. `features` is a dict from keys to `VarLenFeature`,
+  `SparseFeature`, and `FixedLenFeature` objects. Each `VarLenFeature`
+  and `SparseFeature` is mapped to a `SparseTensor`, and each
+  `FixedLenFeature` is mapped to a `Tensor`.
+
+  Each `VarLenFeature` maps to a `SparseTensor` of the specified type
+  representing a ragged matrix. Its indices are `[batch, index]` where `batch`
+  identifies the example in `serialized`, and `index` is the value's index in
+  the list of values associated with that feature and example.
+
+  Each `SparseFeature` maps to a `SparseTensor` of the specified type
+  representing a Tensor of `dense_shape` `[batch_size] + SparseFeature.size`.
+  Its `values` come from the feature in the examples with key `value_key`.
+  A `values[i]` comes from a position `k` in the feature of an example at batch
+  entry `batch`. This positional information is recorded in `indices[i]` as
+  `[batch, index_0, index_1, ...]` where `index_j` is the `k-th` value of
+  the feature in the example at with key `SparseFeature.index_key[j]`.
+  In other words, we split the indices (except the first index indicating the
+  batch entry) of a `SparseTensor` by dimension into different features of the
+  `Example`. Due to its complexity a `VarLenFeature` should be preferred over a
+  `SparseFeature` whenever possible.
+
+  Each `FixedLenFeature` `df` maps to a `Tensor` of the specified type (or
+  `tf.float32` if not specified) and shape `(serialized.size(),) + df.shape`.
+
+  `FixedLenFeature` entries with a `default_value` are optional. With no default
+  value, we will fail if that `Feature` is missing from any example in
+  `serialized`.
+
+  Each `FixedLenSequenceFeature` `df` maps to a `Tensor` of the specified type
+  (or `tf.float32` if not specified) and shape
+  `(serialized.size(), None) + df.shape`.
+  All examples in `serialized` will be padded with `default_value` along the
+  second dimension.
+
+  Examples:
+
+  For example, if one expects a `tf.float32` `VarLenFeature` `ft` and three
+  serialized `Example`s are provided:
+
+  ```
+  serialized = [
+    features
+      { feature { key: "ft" value { float_list { value: [1.0, 2.0] } } } },
+    features
+      { feature []},
+    features
+      { feature { key: "ft" value { float_list { value: [3.0] } } }
+  ]
+  ```
+
+  then the output will look like:
+
+  ```python
+  {"ft": SparseTensor(indices=[[0, 0], [0, 1], [2, 0]],
+                      values=[1.0, 2.0, 3.0],
+                      dense_shape=(3, 2)) }
+  ```
+
+  If instead a `FixedLenSequenceFeature` with `default_value = -1.0` and
+  `shape=[]` is used then the output will look like:
+
+  ```python
+  {"ft": [[1.0, 2.0], [3.0, -1.0]]}
+  ```
+
+  Given two `Example` input protos in `serialized`:
+
+  ```
+  [
+    features {
+      feature { key: "kw" value { bytes_list { value: [ "knit", "big" ] } } }
+      feature { key: "gps" value { float_list { value: [] } } }
+    },
+    features {
+      feature { key: "kw" value { bytes_list { value: [ "emmy" ] } } }
+      feature { key: "dank" value { int64_list { value: [ 42 ] } } }
+      feature { key: "gps" value { } }
+    }
+  ]
+  ```
+
+  And arguments
+
+  ```
+  example_names: ["input0", "input1"],
+  features: {
+      "kw": VarLenFeature(tf.string),
+      "dank": VarLenFeature(tf.int64),
+      "gps": VarLenFeature(tf.float32),
+  }
+  ```
+
+  Then the output is a dictionary:
+
+  ```python
+  {
+    "kw": SparseTensor(
+        indices=[[0, 0], [0, 1], [1, 0]],
+        values=["knit", "big", "emmy"]
+        dense_shape=[2, 2]),
+    "dank": SparseTensor(
+        indices=[[1, 0]],
+        values=[42],
+        dense_shape=[2, 1]),
+    "gps": SparseTensor(
+        indices=[],
+        values=[],
+        dense_shape=[2, 0]),
+  }
+  ```
+
+  For dense results in two serialized `Example`s:
+
+  ```
+  [
+    features {
+      feature { key: "age" value { int64_list { value: [ 0 ] } } }
+      feature { key: "gender" value { bytes_list { value: [ "f" ] } } }
+     },
+     features {
+      feature { key: "age" value { int64_list { value: [] } } }
+      feature { key: "gender" value { bytes_list { value: [ "f" ] } } }
+    }
+  ]
+  ```
+
+  We can use arguments:
+
+  ```
+  example_names: ["input0", "input1"],
+  features: {
+      "age": FixedLenFeature([], dtype=tf.int64, default_value=-1),
+      "gender": FixedLenFeature([], dtype=tf.string),
+  }
+  ```
+
+  And the expected output is:
+
+  ```python
+  {
+    "age": [[0], [-1]],
+    "gender": [["f"], ["f"]],
+  }
+  ```
+
+  An alternative to `VarLenFeature` to obtain a `SparseTensor` is
+  `SparseFeature`. For example, given two `Example` input protos in
+  `serialized`:
+
+  ```
+  [
+    features {
+      feature { key: "val" value { float_list { value: [ 0.5, -1.0 ] } } }
+      feature { key: "ix" value { int64_list { value: [ 3, 20 ] } } }
+    },
+    features {
+      feature { key: "val" value { float_list { value: [ 0.0 ] } } }
+      feature { key: "ix" value { int64_list { value: [ 42 ] } } }
+    }
+  ]
+  ```
+
+  And arguments
+
+  ```
+  example_names: ["input0", "input1"],
+  features: {
+      "sparse": SparseFeature(
+          index_key="ix", value_key="val", dtype=tf.float32, size=100),
+  }
+  ```
+
+  Then the output is a dictionary:
+
+  ```python
+  {
+    "sparse": SparseTensor(
+        indices=[[0, 3], [0, 20], [1, 42]],
+        values=[0.5, -1.0, 0.0]
+        dense_shape=[2, 100]),
+  }
+  ```
+
+  Args:
+    serialized: A vector (1-D Tensor) of strings, a batch of binary
+      serialized `Example` protos.
+    features: A `dict` mapping feature keys to `FixedLenFeature`,
+      `VarLenFeature`, and `SparseFeature` values.
+    example_names: A vector (1-D Tensor) of strings (optional), the names of
+      the serialized protos in the batch.
+    name: A name for this operation (optional).
+
+  Returns:
+    A `dict` mapping feature keys to `Tensor` and `SparseTensor` values.
+
   Raises:
     ValueError: if any feature is invalid.
   """
@@ -764,8 +981,7 @@ def _process_raw_parameters(names, dense_defaults, sparse_keys, sparse_types,
           dense_shapes_as_proto, dense_shapes)
 
 
-@tf_export("io.parse_single_example",
-           v1=["io.parse_single_example", "parse_single_example"])
+@tf_export(v1=["io.parse_single_example", "parse_single_example"])
 def parse_single_example(serialized, features, name=None, example_names=None):
   """Parses a single `Example` proto.
 
@@ -795,6 +1011,48 @@ def parse_single_example(serialized, features, name=None, example_names=None):
   Returns:
     A `dict` mapping feature keys to `Tensor` and `SparseTensor` values.
 
+  Raises:
+    ValueError: if any feature is invalid.
+  """
+  return parse_single_example_v2_unoptimized(
+      serialized, features, example_names, name
+      )
+
+
+# TODO(b/70890287): Combine the implementation of this op and
+# `parse_single_example_v2()` after 1/10/2018.
+@tf_export("io.parse_single_example", v1=[])
+def parse_single_example_v2_unoptimized(
+    serialized, features, example_names=None, name=None
+    ):
+  """Parses a single `Example` proto.
+
+  Similar to `parse_example`, except:
+
+  For dense tensors, the returned `Tensor` is identical to the output of
+  `parse_example`, except there is no batch dimension, the output shape is the
+  same as the shape given in `dense_shape`.
+
+  For `SparseTensor`s, the first (batch) column of the indices matrix is removed
+  (the indices matrix is a column vector), the values vector is unchanged, and
+  the first (`batch_size`) entry of the shape vector is removed (it is now a
+  single element vector).
+
+  One might see performance advantages by batching `Example` protos with
+  `parse_example` instead of using this function directly.
+
+  Args:
+    serialized: A scalar string Tensor, a single serialized Example.
+      See `_parse_single_example_raw` documentation for more details.
+    features: A `dict` mapping feature keys to `FixedLenFeature` or
+      `VarLenFeature` values.
+    example_names: (Optional) A scalar string Tensor, the associated name.
+      See `_parse_single_example_raw` documentation for more details.
+    name: A name for this operation (optional).
+
+  Returns:
+    A `dict` mapping feature keys to `Tensor` and `SparseTensor` values.
+
   Raises:
     ValueError: if any feature is invalid.
   """
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
index 1de2fc9e5d..d32529876f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
@@ -122,7 +122,7 @@ tf_module {
   }
   member_method {
     name: "parse_example"
-    argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'serialized\', \'features\', \'example_names\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "parse_sequence_example"
@@ -130,7 +130,7 @@ tf_module {
   }
   member_method {
     name: "parse_single_example"
-    argspec: "args=[\'serialized\', \'features\', \'name\', \'example_names\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'serialized\', \'features\', \'example_names\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "parse_single_sequence_example"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index aab7aa8af5..f756bb80bc 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -312,6 +312,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.strings.length": ["input", "name", "unit"],
         "tf.transpose": ["a", "perm", "name", "conjugate"],
         "tf.tuple": ["tensors", "name", "control_inputs"],
+        "tf.io.parse_example": [
+            "serialized", "features", "name", "example_names"
+        ],
+        "tf.io.parse_single_example": [
+            "serialized", "features", "name", "example_names"
+        ],
         "tf.while_loop": [
             "cond", "body", "loop_vars", "shape_invariants",
             "parallel_iterations", "back_prop", "swap_memory", "name",
-- 
GitLab


From aaad174baa41472d996c91866883f16d67a844cf Mon Sep 17 00:00:00 2001
From: Jeremy Lau <lauj@google.com>
Date: Wed, 21 Nov 2018 11:51:09 -0800
Subject: [PATCH 0681/1554] TPUEstimator: Run tpu.initialize_system in a
 separate graph. This speeds up initialization by avoiding unnecessary graph
 processing.

PiperOrigin-RevId: 222440544
---
 tensorflow/contrib/tpu/BUILD                  |  1 +
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 31 +++++++++++++------
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index a0a9cb3f31..8264462a06 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -78,6 +78,7 @@ py_library(
         "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
+        "//tensorflow/python:session",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:summary",
         "//tensorflow/python:summary_ops_v2",
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 932367f4dd..9525121ebb 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -45,6 +45,7 @@ from tensorflow.contrib.training.python.training import hparam
 from tensorflow.core.framework import variable_pb2
 from tensorflow.core.framework.summary_pb2 import Summary
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.client import session as tf_session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest as data_nest
 from tensorflow.python.estimator import estimator as estimator_lib
@@ -412,12 +413,15 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
                enqueue_ops,
                dequeue_ops,
                run_infeed_loop_on_coordinator=True,
-               rendezvous=None):
+               rendezvous=None,
+               master=None,
+               session_config=None):
     self._master_job = ctx.master_job
     self._enqueue_ops = enqueue_ops
     self._dequeue_ops = dequeue_ops
     self._rendezvous = rendezvous
-
+    self._master = master
+    self._session_config = session_config
     self._run_infeed_loop_on_coordinator = run_infeed_loop_on_coordinator
     self._initial_infeed_sleep_secs = (
         ctx.config.tpu_config.initial_infeed_sleep_secs)
@@ -429,11 +433,10 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
   def begin(self):
     logging.info('TPU job name %s', self._master_job)
     self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
+    self._init_ops = []
     if self._should_initialize_tpu:
-      self._init_ops = [tpu.initialize_system(job=self._master_job)]
       self._finalize_ops = [tpu.shutdown_system(job=self._master_job)]
     else:
-      self._init_ops = []
       self._finalize_ops = []
 
     summary_writer_init_ops = contrib_summary.summary_writer_initializer_op()
@@ -475,11 +478,17 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
     return _OpQueueContext(name=name, target=target, args=args)
 
   def after_create_session(self, session, coord):
-    logging.info('Init TPU system')
-    start = time.time()
+    if self._should_initialize_tpu:
+      logging.info('Init TPU system')
+      start = time.time()
+      with ops.Graph().as_default():
+        with tf_session.Session(
+            self._master, config=self._session_config) as sess:
+          sess.run(tpu.initialize_system(job=self._master_job))
+      logging.info('Initialized TPU in %d seconds', time.time() - start)
+
     session.run(self._init_ops,
                 options=config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000))
-    logging.info('Initialized TPU in %d seconds', time.time() - start)
 
     self._infeed_controller = self._create_infeed_controller(
         name='InfeedController', target=self._run_infeed, args=(session,))
@@ -2564,6 +2573,8 @@ class TPUEstimator(estimator_lib.Estimator):
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator),
                   rendezvous=self._rendezvous[mode],
+                  master=self._config.master,
+                  session_config=self._session_config,
               ),
               InstallSignalHandlerHook()
           ])
@@ -2666,8 +2677,10 @@ class TPUEstimator(estimator_lib.Estimator):
                   eval_update_ops + host_ops,
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator),
-                  rendezvous=self._rendezvous[mode]),
-          ] + input_hooks
+                  rendezvous=self._rendezvous[mode],
+                  master=self._config.master,
+                  session_config=self._session_config,
+              )] + input_hooks
 
           if eval_hooks:
             hooks.extend(eval_hooks)
-- 
GitLab


From 5ce7e6476417a92a3c7455b898c5fb3632c9ccf7 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 21 Nov 2018 12:32:22 -0800
Subject: [PATCH 0682/1554] Add AddDependency HLO instruction. The
 AddDependency HLO instruction takes a data operand and a token operand. The
 output is the data operand. When used with AfterAll (which has been extended
 in the CL to accept non-token operands), this enables ordering of
 instructions which do produce or accept tokens (ie, non-side-effecting ops).
 This new functionality is intentionally not exposed in the XlaBuilder
 interface because the ordering of non-side-effecting operations is fragile to
 some optimizations. Rather this new op will be used to replace existing uses
 of control dependencies which are to be removed. The uses of these new
 ordering ops will be late in the optimization pipeline and will avoid the
 more problematic optimization interactions.

See go/xla-side-effect-semantics for further details.

PiperOrigin-RevId: 222445994
---
 tensorflow/compiler/xla/client/xla_builder.cc |   9 ++
 .../compiler/xla/client/xla_builder_test.cc   |   9 ++
 .../compiler/xla/g3doc/operation_semantics.md |  16 +++
 .../compiler/xla/service/cpu/ir_emitter.cc    |  13 +-
 .../compiler/xla/service/cpu/ir_emitter.h     |   3 +-
 .../compiler/xla/service/dfs_hlo_visitor.h    |   1 +
 .../service/dfs_hlo_visitor_with_default.h    |   3 +
 .../compiler/xla/service/gpu/ir_emitter.cc    |  12 ++
 .../compiler/xla/service/gpu/ir_emitter.h     |   1 +
 .../xla/service/gpu/ir_emitter_unnested.cc    |   2 +-
 .../xla/service/gpu/ir_emitter_unnested.h     |   2 +-
 .../compiler/xla/service/hlo_cost_analysis.cc |  15 +++
 .../compiler/xla/service/hlo_cost_analysis.h  |   1 +
 .../xla/service/hlo_dataflow_analysis.cc      |  18 +++
 .../xla/service/hlo_dataflow_analysis.h       |   1 +
 .../xla/service/hlo_dataflow_analysis_test.cc |  24 ++++
 .../compiler/xla/service/hlo_evaluator.cc     |  11 +-
 .../compiler/xla/service/hlo_evaluator.h      |   4 +-
 .../compiler/xla/service/hlo_graph_dumper.cc  |   1 +
 .../compiler/xla/service/hlo_instruction.cc   |  17 +++
 .../compiler/xla/service/hlo_instruction.h    |   3 +
 tensorflow/compiler/xla/service/hlo_opcode.h  |   3 +-
 tensorflow/compiler/xla/service/hlo_parser.cc |   9 ++
 .../compiler/xla/service/hlo_parser_test.cc   |  31 +++++
 .../compiler/xla/service/hlo_verifier.cc      |   8 +-
 .../compiler/xla/service/hlo_verifier.h       |   1 +
 .../xla/service/instruction_fusion.cc         |   6 +-
 .../compiler/xla/service/layout_assignment.cc |   1 +
 .../xla/service/logical_buffer_analysis.cc    |   7 ++
 .../xla/service/logical_buffer_analysis.h     |   1 +
 .../compiler/xla/service/shape_inference.cc   |  11 --
 .../compiler/xla/service/shape_inference.h    |   7 --
 .../xla/service/tuple_points_to_analysis.cc   |   7 ++
 .../xla/service/tuple_points_to_analysis.h    |   1 +
 .../service/tuple_points_to_analysis_test.cc  |  16 +++
 tensorflow/compiler/xla/tests/BUILD           |   1 +
 .../compiler/xla/tests/token_hlo_test.cc      | 111 ++++++++++++++----
 37 files changed, 336 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 8a33b3930f..f17bc456a6 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -1320,6 +1320,15 @@ XlaOp XlaBuilder::AfterAll(absl::Span<const XlaOp> tokens) {
     if (tokens.empty()) {
       return InvalidArgument("AfterAll requires at least one operand");
     }
+    for (int i = 0; i < tokens.size(); ++i) {
+      const XlaOp& operand = tokens[i];
+      TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+      if (!ShapeUtil::IsToken(operand_shape)) {
+        return InvalidArgument(
+            "All operands to AfterAll must be tokens; operand %d has shape %s",
+            i, ShapeUtil::HumanString(operand_shape));
+      }
+    }
     HloInstructionProto instr;
     *instr.mutable_shape() = ShapeUtil::MakeTokenShape();
     return AddInstruction(std::move(instr), HloOpcode::kAfterAll, tokens);
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index 8aa85c3cd6..e534fb67fd 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -446,5 +446,14 @@ TEST_F(XlaBuilderTest, ProtoMatches) {
   EXPECT_EQ(c0_string, c1_string);
 }
 
+TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
+  XlaBuilder b(TestName());
+  AfterAll(&b, {CreateToken(&b), ConstantR0<float>(&b, 1.0)});
+  Status status = b.Build().status();
+  ASSERT_IS_NOT_OK(status);
+  EXPECT_THAT(status.error_message(),
+              ::testing::HasSubstr("All operands to AfterAll must be tokens"));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 73a9db75f6..bc87a60c6e 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -13,6 +13,22 @@ arbitrary-dimensional array. For convenience, special cases have more specific
 and familiar names; for example a *vector* is a 1-dimensional array and a
 *matrix* is a 2-dimensional array.
 
+## AfterAll
+
+See also
+[`XlaBuilder::AfterAll`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/xla_builder.h).
+
+AfterAll takes a variadic number of tokens and produces a single token. Tokens
+are primitive types which can be threaded between side-effecting operations to
+enforce ordering. `AfterAll` can be used as a join of tokens for ordering a
+operation after a set operations.
+
+<b> `AfterAll(operands)` </b>
+
+Arguments  | Type    | Semantics
+---------- | ------- | -------------------------
+`operands` | `XlaOp` | variadic number of tokens
+
 ## AllToAll
 
 See also
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index cf97a8bde0..4032c2da2f 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -2565,10 +2565,17 @@ Status IrEmitter::HandleConditional(HloInstruction* conditional) {
   return Status::OK();
 }
 
-Status IrEmitter::HandleAfterAll(HloInstruction* gen_token) {
-  TF_RET_CHECK(ByteSizeOf(gen_token->shape()) == 0);
+Status IrEmitter::HandleAfterAll(HloInstruction* after_all) {
+  TF_RET_CHECK(ByteSizeOf(after_all->shape()) == 0);
   // No code to generate, but we need to emit an address for book-keeping.
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(gen_token));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(after_all));
+  return Status::OK();
+}
+
+Status IrEmitter::HandleAddDependency(HloInstruction* add_dependency) {
+  // AddDedendency just forwards its zero-th operand.
+  emitted_value_[add_dependency] =
+      GetEmittedValueFor(add_dependency->operand(0));
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index f529c613a3..559a8162a2 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -159,7 +159,8 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   Status HandleConcatenate(HloInstruction* concatenate) override;
   Status HandleConditional(HloInstruction* conditional) override;
   Status HandleScatter(HloInstruction* scatter) override;
-  Status HandleAfterAll(HloInstruction* gen_token) override;
+  Status HandleAfterAll(HloInstruction* after_all) override;
+  Status HandleAddDependency(HloInstruction* add_dependency) override;
   Status HandleRng(HloInstruction* rng) override;
   Status FinishVisit(HloInstruction* root) override;
 
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index d637128322..e84bf00153 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -251,6 +251,7 @@ class DfsHloVisitorBase {
 
   virtual Status HandleBatchNormGrad(HloInstructionPtr hlo) = 0;
 
+  virtual Status HandleAddDependency(HloInstructionPtr add_dependency) = 0;
   virtual Status HandleAfterAll(HloInstructionPtr token) = 0;
 
   // Invoked to inform the visitor that the traversal has completed, and that
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
index e57184f639..80ea5be298 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
@@ -206,6 +206,9 @@ class DfsHloVisitorWithDefaultBase
   Status HandleGetDimensionSize(HloInstructionPtr get_size) override {
     return DefaultAction(get_size);
   }
+  Status HandleAddDependency(HloInstructionPtr add_dependency) override {
+    return DefaultAction(add_dependency);
+  }
 
   // Invoked to inform the visitor that the traversal has completed, and that
   // the root was "root".
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index 7fcdd805ed..31591914cc 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -97,6 +97,18 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) {
   return Status::OK();
 }
 
+Status IrEmitter::HandleAddDependency(HloInstruction* add_dependency) {
+  VLOG(2) << "HandleAddDependency: " << add_dependency->ToString();
+  const HloInstruction* operand = add_dependency->operand(0);
+  // Add_Dependency is a no-op, but we still want to bind it to an llvm::Value
+  // sometimes, e.g., when it's operand is a constant or a bitcast of a
+  // constant.
+  if (bindings_.BoundToIrValue(*operand)) {
+    bindings_.BindHloToIrValue(*add_dependency, GetBasePointer(*operand));
+  }
+  return Status::OK();
+}
+
 Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) {
   auto operand = get_tuple_element->operand(0);
   CHECK(bindings_.BoundToIrValue(*operand));
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index 56c3f45200..2da46c0169 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -100,6 +100,7 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   Status HandleBatchNormInference(HloInstruction* batch_norm) override;
   Status HandleBatchNormTraining(HloInstruction* batch_norm) override;
   Status HandleBatchNormGrad(HloInstruction* batch_norm) override;
+  Status HandleAddDependency(HloInstruction* add_dependency) override;
 
   Status FinishVisit(HloInstruction* root) override { return Status::OK(); }
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index ebd73f3a91..52f0ba7aa7 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -2398,7 +2398,7 @@ Status IrEmitterUnnested::HandleCrossReplicaSum(HloInstruction* crs) {
   return Status::OK();
 }
 
-Status IrEmitterUnnested::HandleAfterAll(HloInstruction* gen_token) {
+Status IrEmitterUnnested::HandleAfterAll(HloInstruction* after_all) {
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index 97a1e10455..e09ed657a8 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -171,7 +171,7 @@ class IrEmitterUnnested : public IrEmitter {
   Status HandleSort(HloInstruction* sort) override;
   Status HandleTupleSelect(HloInstruction* tuple_select) override;
   Status HandleCrossReplicaSum(HloInstruction* crs) override;
-  Status HandleAfterAll(HloInstruction* gen_token) override;
+  Status HandleAfterAll(HloInstruction* after_all) override;
 
   Status EmitTargetElementLoop(
       const HloInstruction& hlo,
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index fdfb38b858..df7d3826db 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -419,6 +419,21 @@ Status HloCostAnalysis::HandleTranspose(const HloInstruction*) {
 }
 
 Status HloCostAnalysis::HandleAfterAll(const HloInstruction*) {
+  // This instruction is used to enforce ordering at compile time. No code is
+  // emitted.
+  current_should_compute_bottleneck_time_ = false;
+  current_properties_[kBytesAccessedKey] = 0;
+  current_properties_[kOptimalSecondsKey] = 0;
+  return Status::OK();
+}
+
+Status HloCostAnalysis::HandleAddDependency(
+    const HloInstruction* add_dependency) {
+  // This instruction is used to enforce ordering at compile time. No code is
+  // emitted.
+  current_should_compute_bottleneck_time_ = false;
+  current_properties_[kBytesAccessedKey] = 0;
+  current_properties_[kOptimalSecondsKey] = 0;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
index 8ced9d776e..33983119c9 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
@@ -101,6 +101,7 @@ class HloCostAnalysis : public ConstDfsHloVisitor {
   Status HandleBroadcast(const HloInstruction* broadcast) override;
   Status HandlePad(const HloInstruction* pad) override;
   Status HandleReshape(const HloInstruction* reshape) override;
+  Status HandleAddDependency(const HloInstruction* add_dependency) override;
   Status HandleAfterAll(const HloInstruction* token) override;
   Status HandleTranspose(const HloInstruction* transpose) override;
   Status HandleWhile(const HloInstruction* xla_while) override;
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
index 5dcf6bc985..3ed3d3c11c 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
@@ -466,6 +466,21 @@ bool HloDataflowAnalysis::UpdateDomainValueSet(HloInstruction* domain) {
   return changed;
 }
 
+bool HloDataflowAnalysis::UpdateAddDependencyValueSet(
+    HloInstruction* add_dependency) {
+  // AddDependency just forwards the value of its zero-th operand.
+  CHECK_EQ(add_dependency->opcode(), HloOpcode::kAddDependency);
+  const InstructionValueSet& operand_set =
+      GetInstructionValueSet(add_dependency->operand(0));
+  InstructionValueSet& add_dependency_set =
+      GetInstructionValueSet(add_dependency);
+  if (operand_set != add_dependency_set) {
+    add_dependency_set = operand_set;
+    return true;
+  }
+  return false;
+}
+
 bool HloDataflowAnalysis::UpdateGetTupleElementValueSet(HloInstruction* gte) {
   CHECK_EQ(gte->opcode(), HloOpcode::kGetTupleElement);
   bool changed = false;
@@ -622,6 +637,8 @@ bool HloDataflowAnalysis::UpdateInstructionValueSet(
     HloInstruction* instruction) {
   // Recompute from operands.
   switch (instruction->opcode()) {
+    case HloOpcode::kAddDependency:
+      return UpdateAddDependencyValueSet(instruction);
     case HloOpcode::kBitcast:
       return UpdateBitcastValueSet(instruction);
     case HloOpcode::kDomain:
@@ -795,6 +812,7 @@ Status HloDataflowAnalysis::InitializeInstructionValueSets() {
             define_all_values();
           }
           break;
+        case HloOpcode::kAddDependency:
         case HloOpcode::kWhile:
         case HloOpcode::kCall:
         case HloOpcode::kConditional:
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h
index abac398c04..ece17fc4c3 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h
@@ -193,6 +193,7 @@ class HloDataflowAnalysis {
   bool UpdateSendValueSet(HloInstruction* send);
   bool UpdateTupleValueSet(HloInstruction* tuple);
   bool UpdateWhileValueSet(HloInstruction* xla_while);
+  bool UpdateAddDependencyValueSet(HloInstruction* add_dependency);
 
   // Propagate the dataflow through the module.
   void Propagate();
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index e8eb7066f9..f7a1f19a6f 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -1877,6 +1877,30 @@ TEST_P(HloDataflowAnalysisTest, NestedConditionals) {
   }
 }
 
+TEST_P(HloDataflowAnalysisTest, AddDependency) {
+  string module_string = R"(
+HloModule AddDependency
+ENTRY %AddDependency (p: f32[3]) -> f32[3] {
+  %p = f32[3] parameter(0)
+  %token = token[] after-all()
+  ROOT %add_dep = f32[3] add-dependency(f32[3] %p, token[] %token)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::unique_ptr<HloModule> module,
+      ParseHloString(module_string, GetModuleConfigForTest()));
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloDataflowAnalysis> analysis,
+                          HloDataflowAnalysis::Run(*module));
+  const HloInstruction* root = module->entry_computation()->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kAddDependency);
+
+  // The after-all and parameter should define a value. Add-dependency should
+  // not.
+  EXPECT_EQ(analysis->values().size(), 2);
+  EXPECT_FALSE(analysis->ValueIsDefinedAt(root));
+}
+
 INSTANTIATE_TEST_CASE_P(HloDataflowAnalysisInstantiation,
                         HloDataflowAnalysisTest,
                         ::testing::Values(false, true));
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 9783f0574f..51a3fba176 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1046,8 +1046,15 @@ Status HloEvaluator::HandleBroadcast(HloInstruction* broadcast) {
   return Status::OK();
 }
 
-Status HloEvaluator::HandleAfterAll(HloInstruction* token) {
-  evaluated_[token] = LiteralUtil::CreateToken();
+Status HloEvaluator::HandleAfterAll(HloInstruction* after_all) {
+  evaluated_[after_all] = LiteralUtil::CreateToken();
+  return Status::OK();
+}
+
+Status HloEvaluator::HandleAddDependency(HloInstruction* add_dependency) {
+  // AddDedendency just forwards its zero-th operand.
+  evaluated_[add_dependency] =
+      GetEvaluatedLiteralFor(add_dependency->operand(0)).Clone();
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index d751f40fff..d847900010 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -180,7 +180,9 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
 
   Status HandleBroadcast(HloInstruction* broadcast) override;
 
-  Status HandleAfterAll(HloInstruction* token) override;
+  Status HandleAfterAll(HloInstruction* after_all) override;
+
+  Status HandleAddDependency(HloInstruction* add_dependency) override;
 
   Status HandleSort(HloInstruction* sort) override;
 
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 05cc1593e4..7e9e94ca5f 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -987,6 +987,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kGetTupleElement:
     case HloOpcode::kTrace:
     case HloOpcode::kAfterAll:
+    case HloOpcode::kAddDependency:
     case HloOpcode::kTuple:
       return kWhite;
     case HloOpcode::kBroadcast:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index cd95052580..1e3881c34f 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -855,6 +855,16 @@ HloInstruction::CreateCollectivePermute(
       new HloInstruction(HloOpcode::kAfterAll, ShapeUtil::MakeTokenShape()));
 }
 
+/* static */ std::unique_ptr<HloInstruction>
+HloInstruction::CreateAddDependency(HloInstruction* data_operand,
+                                    HloInstruction* token_operand) {
+  auto instruction = absl::WrapUnique(
+      new HloInstruction(HloOpcode::kAddDependency, data_operand->shape()));
+  instruction->AppendOperand(data_operand);
+  instruction->AppendOperand(token_operand);
+  return instruction;
+}
+
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateWhile(
     const Shape& shape, HloComputation* condition, HloComputation* body,
     HloInstruction* init) {
@@ -1394,6 +1404,10 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
         clone = CreateAfterAll(new_operands);
       }
       break;
+    case HloOpcode::kAddDependency:
+      CHECK_EQ(new_operands.size(), 2);
+      clone = CreateAddDependency(new_operands[0], new_operands[1]);
+      break;
   }
   // SetupDerivedInstruction will setup the precision_config_ field.
   SetupDerivedInstruction(clone.get());
@@ -1680,6 +1694,7 @@ bool HloInstruction::IdenticalSlowPath(
 
     // This opcode has complex or special behavior so just return false.
     case HloOpcode::kAfterAll:
+    case HloOpcode::kAddDependency:
       return false;
 
     // Remaining instructions with special values.
@@ -2467,6 +2482,8 @@ Status HloInstruction::Visit(DfsHloVisitorBase<HloInstructionPtr>* visitor) {
       return visitor->HandleDomain(this);
     case HloOpcode::kAfterAll:
       return visitor->HandleAfterAll(this);
+    case HloOpcode::kAddDependency:
+      return visitor->HandleAddDependency(this);
     case HloOpcode::kIota:
       return visitor->HandleIota(this);
     case HloOpcode::kGetDimensionSize:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 95ad29235a..87748a771a 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -770,6 +770,9 @@ class HloInstruction {
   static std::unique_ptr<HloInstruction> CreateGetDimensionSize(
       const Shape& shape, HloInstruction* operand, int64 dimension);
 
+  static std::unique_ptr<HloInstruction> CreateAddDependency(
+      HloInstruction* data_operand, HloInstruction* token_operand);
+
   // Returns the opcode for this instruction.
   HloOpcode opcode() const { return opcode_; }
 
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 70c7d70b41..127cfd165a 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -47,6 +47,8 @@ namespace xla {
 #define HLO_OPCODE_LIST(V)                                   \
   V(kAbs, "abs")                                             \
   V(kAdd, "add")                                             \
+  V(kAddDependency, "add-dependency")                        \
+  V(kAfterAll, "after-all", kHloOpcodeIsVariadic)            \
   V(kAllToAll, "all-to-all")                                 \
   V(kAtan2, "atan2")                                         \
   V(kBatchNormGrad, "batch-norm-grad")                       \
@@ -84,7 +86,6 @@ namespace xla {
   V(kGather, "gather")                                       \
   V(kGe, "greater-than-or-equal-to", kHloOpcodeIsComparison) \
   V(kGetDimensionSize, "get-dimension-size")                 \
-  V(kAfterAll, "after-all", kHloOpcodeIsVariadic)            \
   V(kGetTupleElement, "get-tuple-element")                   \
   V(kGt, "greater-than", kHloOpcodeIsComparison)             \
   V(kImag, "imag")                                           \
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 4bf287a9ed..9b5bb5d0bd 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -850,6 +850,15 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       }
       break;
     }
+    case HloOpcode::kAddDependency: {
+      if (!ParseOperands(&operands, /*expected_size=*/2) ||
+          !ParseAttributes(attrs)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateAddDependency(operands[0], operands[1]));
+      break;
+    }
     case HloOpcode::kSort: {
       optional<std::vector<tensorflow::int64>> dimensions;
       attrs["dimensions"] = {/*required=*/true, AttrTy::kBracedInt64List,
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 88682e55fb..f13f7504ee 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1241,7 +1241,38 @@ ENTRY Sort {
 }
 
 )"
+    },
+// AfterAll with multiple operands
+{
+"AfterAllWithMultipleOperands",
+R"(HloModule AfterAllWithMultipleOperands
+
+ENTRY AfterAllWithMultipleOperands {
+  p0 = f32[] parameter(0)
+  token0 = token[] after-all()
+  token1 = token[] after-all()
+  ROOT after-all = token[] after-all(p0, token0, token1)
 }
+
+)"
+},
+// AddDependency
+// A dependency chain is created from 'neg' to 'exp' using tokens.
+{
+"AddDependency",
+R"(HloModule AddDependency
+
+ENTRY AddDependency {
+  p = f32[] parameter(0)
+  neg = f32[] negate(p)
+  token = token[] after-all(neg)
+  p_after_token = f32[] add-dependency(p, token)
+  exp = f32[] exponential(p_after_token)
+  ROOT sum = f32[] add(neg, exp)
+}
+
+)"
+},
 });
   // clang-format on
 }
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 60d8a511b5..017549ce1b 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -753,7 +753,13 @@ Status ShapeVerifier::HandleAfterAll(HloInstruction* token) {
   for (const HloInstruction* operand : token->operands()) {
     operand_shapes.push_back(&operand->shape());
   }
-  return CheckShape(token, ShapeInference::InferAfterAllShape(operand_shapes));
+  return CheckShape(token, ShapeUtil::MakeTokenShape());
+}
+
+Status ShapeVerifier::HandleAddDependency(HloInstruction* add_dependency) {
+  TF_RETURN_IF_ERROR(CheckOperandCount(add_dependency, 2));
+  TF_RETURN_IF_ERROR(CheckIsTokenOperand(add_dependency, 1));
+  return CheckShape(add_dependency, add_dependency->operand(0)->shape());
 }
 
 Status ShapeVerifier::HandleGetDimensionSize(HloInstruction* get_size) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index 9fbfd6a21c..e4d0c3d695 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -95,6 +95,7 @@ class ShapeVerifier : public DfsHloVisitor {
   Status HandleScatter(HloInstruction* scatter) override;
   Status HandleAfterAll(HloInstruction* token) override;
   Status HandleGetDimensionSize(HloInstruction* get_size) override;
+  Status HandleAddDependency(HloInstruction* add_dependency) override;
 
   Status FinishVisit(HloInstruction*) override { return Status::OK(); }
 
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index 7f2d7e7cff..2297edcbe1 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -103,7 +103,6 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) {
     case HloOpcode::kShiftRightLogical:
     case HloOpcode::kSlice:
     case HloOpcode::kSubtract:
-    case HloOpcode::kAfterAll:
     case HloOpcode::kTranspose:
     case HloOpcode::kTuple:
     case HloOpcode::kTupleSelect:
@@ -116,7 +115,10 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) {
     case HloOpcode::kSin:
       return ShapeUtil::ElementIsComplex(instruction.shape());
 
-    // Expensive instructions.
+    // Expensive instructions or unusual instructions for which fusion is
+    // nonsensical.
+    case HloOpcode::kAddDependency:
+    case HloOpcode::kAfterAll:
     case HloOpcode::kAtan2:
     case HloOpcode::kBatchNormGrad:
     case HloOpcode::kBatchNormInference:
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index a904119222..eddef850cf 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -2000,6 +2000,7 @@ bool LayoutAssignment::InstructionCanChangeLayout(
   switch (instruction->opcode()) {
     case HloOpcode::kAbs:
     case HloOpcode::kAdd:
+    case HloOpcode::kAddDependency:
     case HloOpcode::kAnd:
     case HloOpcode::kAtan2:
     case HloOpcode::kBitcastConvert:
diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc
index ec52a24d78..972a5b9ced 100644
--- a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc
+++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc
@@ -113,6 +113,13 @@ Status LogicalBufferAnalysis::HandleGetTupleElement(HloInstruction*) {
   return Status::OK();
 }
 
+Status LogicalBufferAnalysis::HandleAddDependency(
+    HloInstruction* add_dependency) {
+  // AddDependency just forwards the value of its zero-th operand and does not
+  // create buffers.
+  return Status::OK();
+}
+
 Status LogicalBufferAnalysis::HandleCopy(HloInstruction* copy) {
   // The top-level buffer (index={}) for kCopy is newly created, but all other
   // buffers (in the case of a tuple shape) come from the operand
diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.h b/tensorflow/compiler/xla/service/logical_buffer_analysis.h
index 81f524d84a..7ffca943d0 100644
--- a/tensorflow/compiler/xla/service/logical_buffer_analysis.h
+++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.h
@@ -64,6 +64,7 @@ class LogicalBufferAnalysis : public DfsHloVisitorWithDefault {
   Status HandleRecvDone(HloInstruction* recv_done) override;
   Status HandleSend(HloInstruction* send) override;
   Status HandleTupleSelect(HloInstruction* tuple_select) override;
+  Status HandleAddDependency(HloInstruction* add_dependency) override;
 
   // A map from the buffer ID to the logical buffer
   std::vector<std::unique_ptr<LogicalBuffer>> logical_buffers_;
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 2bfc1676bd..528d5c0ecc 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -391,17 +391,6 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
   return ShapeUtil::MakeShape(element_type, new_dimensions);
 }
 
-/* static */ StatusOr<Shape> ShapeInference::InferAfterAllShape(
-    absl::Span<const Shape* const> arg_shapes) {
-  for (const Shape* arg_shape : arg_shapes) {
-    if (arg_shape->element_type() != TOKEN) {
-      return InvalidArgument(
-          "Operands of token instructions must be TOKEN types.");
-    }
-  }
-  return ShapeUtil::MakeTokenShape();
-}
-
 /* static */ StatusOr<Shape> ShapeInference::InferConvertShape(
     const Shape& operand_shape, PrimitiveType new_element_type) {
   auto old_element_type = operand_shape.element_type();
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index 31ef4b2e41..d94385a04d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -232,13 +232,6 @@ class ShapeInference {
   static StatusOr<Shape> InferConcatOpShape(
       absl::Span<const Shape* const> arg_shapes, int64 dimension);
 
-  // Infers the shape produced by a kAfterAll. Trivially this shape is always a
-  // TOKEN shape. However, ShapeInference serves two purposes: inferring shapes
-  // and checking operand shapes. This method verifies that the operand shapes
-  // are all TOKENs.
-  static StatusOr<Shape> InferAfterAllShape(
-      absl::Span<const Shape* const> arg_shapes);
-
   // Helper that validates the given operand shape can be converted to the
   // target output_shape via a convert instruction -- the requirement is that
   // the shape is identical except for the element type.
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
index 96f3055c98..50d51eaeb7 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
@@ -280,6 +280,13 @@ Status TuplePointsToAnalysis::HandleDomain(HloInstruction* domain) {
   return Status::OK();
 }
 
+Status TuplePointsToAnalysis::HandleAddDependency(
+    HloInstruction* add_dependency) {
+  // AddDependency just forwards the value of its zero-th operand.
+  CreateCopiedPointsToSet(add_dependency, add_dependency->operand(0));
+  return Status::OK();
+}
+
 Status TuplePointsToAnalysis::HandleRecvDone(HloInstruction* recv_done) {
   // RecvDone aliases its input (Recv) tuple element {0} to element {0} of its
   // output. The other indices ({} and {1}) define their own buffers.
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h
index bcfcb388f9..0a1d5649d6 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h
@@ -252,6 +252,7 @@ class TuplePointsToAnalysis : public DfsHloVisitorWithDefault {
   Status HandleRecvDone(HloInstruction* recv_done) override;
   Status HandleSend(HloInstruction* send) override;
   Status HandleTupleSelect(HloInstruction* tuple_select) override;
+  Status HandleAddDependency(HloInstruction* add_dependency) override;
 
   string ToString() const;
 
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
index 10ef2d38fa..561762b5d4 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
@@ -264,6 +264,22 @@ TEST_F(TuplePointsToAnalysisTest, GetTupleElement) {
               UnorderedElementsAre(inner_tuple));
 }
 
+TEST_F(TuplePointsToAnalysisTest, AddDependency) {
+  auto builder = HloComputation::Builder(TestName());
+  auto constant = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(1.0)));
+  auto token = builder.AddInstruction(HloInstruction::CreateToken());
+  auto add_dependency = builder.AddInstruction(
+      HloInstruction::CreateAddDependency(constant, token));
+  BuildModuleAndRunAnalysis(builder.Build());
+
+  auto& points_to_set = points_to_analysis_->GetPointsToSet(add_dependency);
+  EXPECT_EQ(1, points_to_set.size());
+  EXPECT_FALSE(points_to_set.IsAmbiguous());
+  EXPECT_TRUE(points_to_set.IsDistinct());
+  ExpectHasTopLevelBuffers(points_to_set.CreateFlattenedSet(), {constant});
+}
+
 TEST_F(TuplePointsToAnalysisTest, DuplicatedElement) {
   // Create a tuple which contains duplicate elements.
   auto builder = HloComputation::Builder(TestName());
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 20493a354c..0105e4a226 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1291,6 +1291,7 @@ xla_test(
         "enable_for_xla_interpreter",
     ],
     deps = [
+        "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/service:hlo_verifier",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/token_hlo_test.cc b/tensorflow/compiler/xla/tests/token_hlo_test.cc
index a2b7c26331..601c6b0693 100644
--- a/tensorflow/compiler/xla/tests/token_hlo_test.cc
+++ b/tensorflow/compiler/xla/tests/token_hlo_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include <array>
 
 #include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/service/hlo_verifier.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
@@ -108,26 +109,6 @@ XLA_TEST_F(TokenHloTest, InvalidTupleTokenShapedEntryParameter) {
       ::testing::HasSubstr("Entry parameter 0 is or contains a token shape"));
 }
 
-XLA_TEST_F(TokenHloTest, InvalidOperandToTokenInstruction) {
-  std::unique_ptr<HloModule> module = CreateNewUnverifiedModule();
-  auto builder = HloComputation::Builder(TestName());
-  auto param = builder.AddInstruction(
-      HloInstruction::CreateParameter(0, ShapeUtil::MakeShape(F32, {}), "p0"));
-  builder.AddInstruction(HloInstruction::CreateAfterAll({param}));
-  builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(123)));
-  module->AddEntryComputation(builder.Build());
-
-  Status status =
-      HloVerifier(/*layout_sensitive=*/false, /*allow_mixed_precision=*/false)
-          .Run(module.get())
-          .status();
-  ASSERT_IS_NOT_OK(status);
-  EXPECT_THAT(status.error_message(),
-              ::testing::HasSubstr(
-                  "Operands of token instructions must be TOKEN types"));
-}
-
 XLA_TEST_F(TokenHloTest, TokenInWhileLoop) {
   // Thread a token around a while loop. Token is created and consumed by a
   // AfterAll instruction in the while body.
@@ -220,5 +201,95 @@ ENTRY %TokenInConditional (param.3: pred[]) -> s32[] {
   }
 }
 
+XLA_TEST_F(TokenHloTest, AddDependency) {
+  string module_string = R"(
+HloModule AddDependency, is_scheduled=true
+
+// Computes (p0 + 42) * (-p1)
+// where there is a dependency from the add to the negation using a token
+// with after-all and add-dependency instructions.
+ENTRY %AddDependency (p0: f32[], p1: f32[]) -> f32[] {
+  %p0 = f32[] parameter(0)
+  %p1 = f32[] parameter(1)
+
+  %forty_two = f32[] constant(42.0)
+  %add = f32[] add(f32[] %p0, f32[] %forty_two)
+  %token = token[] after-all(f32[] %add)
+  %p1_after_token = f32[] add-dependency(f32[] %p1, token[] %token)
+  %neg = f32[] negate(f32[] %p1_after_token)
+  ROOT %product = f32[] multiply(f32[] %add, f32[] %neg)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::unique_ptr<HloModule> module,
+      ParseHloString(module_string, GetModuleConfigForTest()));
+  auto p0 = LiteralUtil::CreateR0<float>(10.0);
+  auto p1 = LiteralUtil::CreateR0<float>(3.0);
+  auto expected = LiteralUtil::CreateR0<float>(-156.0);
+  EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&p0, &p1}));
+}
+
+XLA_TEST_F(TokenHloTest, AddDependencyOfConstant) {
+  string module_string = R"(
+HloModule AddDependencyOfConstant, is_scheduled=true
+
+ENTRY %AddDependency (p0: f32[]) -> f32[] {
+  %p0 = f32[] parameter(0)
+  %forty_two = f32[] constant(42.0)
+  %token = token[] after-all(f32[] %p0)
+  %forty_two_after_token = f32[] add-dependency(f32[] %forty_two, token[] %token)
+  ROOT %product = f32[] multiply(f32[] %p0, f32[] %forty_two_after_token)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::unique_ptr<HloModule> module,
+      ParseHloString(module_string, GetModuleConfigForTest()));
+  auto p0 = LiteralUtil::CreateR0<float>(10.0);
+  auto expected = LiteralUtil::CreateR0<float>(420.0);
+  EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&p0}));
+}
+
+XLA_TEST_F(TokenHloTest, AddDependencyAsRoot) {
+  string module_string = R"(
+HloModule AddDependencyAsRoot, is_scheduled=true
+ENTRY %AddDependency (p: f32[3]) -> f32[3] {
+  %p = f32[3] parameter(0)
+  %neg = f32[3] negate(f32[3] %p)
+  %token = token[] after-all()
+  ROOT %add_dep = f32[3] add-dependency(f32[3] %neg, token[] %token)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::unique_ptr<HloModule> module,
+      ParseHloString(module_string, GetModuleConfigForTest()));
+  auto input = LiteralUtil::CreateR1<float>({1.0, 3.0, 7.0});
+  auto expected = LiteralUtil::CreateR1<float>({-1.0, -3.0, -7.0});
+  EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&input}));
+}
+
+XLA_TEST_F(TokenHloTest, TupleShapedAddDependency) {
+  string module_string = R"(
+HloModule TupleShapedAddDependency, is_scheduled=true
+ENTRY %TupleShapedAddDependency (p0: f32[3], p1: f32[3]) -> f32[3] {
+  %p0 = f32[3] parameter(0)
+  %p1 = f32[3] parameter(1)
+  %forty_two = f32[] constant(42.0)
+  %token = token[] after-all()
+  %tuple = (f32[3], token[], f32[3], f32[]) tuple(f32[3] %p0, token[] %token, f32[3] %p1, f32[] %forty_two)
+  %add_dep = (f32[3], token[], f32[3], f32[]) add-dependency((f32[3], token[], f32[3], f32[]) %tuple, token[] %token)
+  %elem0 = f32[3] get-tuple-element((f32[3], token[], f32[3], f32[]) %add_dep), index=0
+  %elem2 = f32[3] get-tuple-element((f32[3], token[], f32[3], f32[]) %add_dep), index=2
+  ROOT %diff = f32[3] subtract(f32[3] %elem0, f32[3] %elem2)
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::unique_ptr<HloModule> module,
+      ParseHloString(module_string, GetModuleConfigForTest()));
+  auto p0 = LiteralUtil::CreateR1<float>({3.0, 3.0, 47.0});
+  auto p1 = LiteralUtil::CreateR1<float>({1.0, -2.0, 2.0});
+  auto expected = LiteralUtil::CreateR1<float>({2.0, 5.0, 45.0});
+  EXPECT_EQ(expected, ExecuteNoHloPasses(std::move(module), {&p0, &p1}));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From b255e5c1dc458ea153c565abb228863754b4203b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 12:33:50 -0800
Subject: [PATCH 0683/1554] Clean up memory management in testing/tflite_driver

PiperOrigin-RevId: 222446170
---
 tensorflow/lite/testing/tflite_driver.cc | 11 +++++------
 tensorflow/lite/testing/tflite_driver.h  | 14 +++++++++++++-
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc
index 3a0febb780..27e3a3770b 100644
--- a/tensorflow/lite/testing/tflite_driver.cc
+++ b/tensorflow/lite/testing/tflite_driver.cc
@@ -147,9 +147,10 @@ TfLiteDriver::TfLiteDriver(bool use_nnapi, const string& delegate_name)
 }
 
 TfLiteDriver::~TfLiteDriver() {
-  for (TfLiteTensor* t : tensors_to_deallocate_) {
-    free(t->data.raw);
+  for (auto t : tensors_to_deallocate_) {
+    DeallocateStringTensor(t.second);
   }
+  interpreter_.reset();
 }
 
 void TfLiteDriver::AllocateTensors() {
@@ -242,12 +243,10 @@ void TfLiteDriver::SetInput(int id, const string& csv_values) {
     case kTfLiteString: {
       string s = absl::HexStringToBytes(csv_values);
 
-      tensor->data.raw = reinterpret_cast<char*>(malloc(s.size()));
-      tensor->bytes = s.size();
+      DeallocateStringTensor(tensors_to_deallocate_[id]);
+      AllocateStringTensor(id, s.size(), tensor);
       memcpy(tensor->data.raw, s.data(), s.size());
 
-      // We must remember to free the memory we allocated above.
-      tensors_to_deallocate_.push_back(tensor);
       break;
     }
     default:
diff --git a/tensorflow/lite/testing/tflite_driver.h b/tensorflow/lite/testing/tflite_driver.h
index d8b40565ba..1da0533c57 100644
--- a/tensorflow/lite/testing/tflite_driver.h
+++ b/tensorflow/lite/testing/tflite_driver.h
@@ -49,6 +49,18 @@ class TfLiteDriver : public TestRunner {
   string ReadOutput(int id) override { return "no-op"; }
 
  private:
+  void DeallocateStringTensor(TfLiteTensor* t) {
+    if (t) {
+      free(t->data.raw);
+      t->data.raw = nullptr;
+    }
+  }
+  void AllocateStringTensor(int id, size_t num_bytes, TfLiteTensor* t) {
+    t->data.raw = reinterpret_cast<char*>(malloc(num_bytes));
+    t->bytes = num_bytes;
+    tensors_to_deallocate_[id] = t;
+  }
+
   void ResetLSTMStateTensors();
 
   class Expectation;
@@ -59,7 +71,7 @@ class TfLiteDriver : public TestRunner {
   std::unique_ptr<Interpreter> interpreter_;
   std::map<int, std::unique_ptr<Expectation>> expected_output_;
   bool must_allocate_tensors_ = true;
-  std::vector<TfLiteTensor*> tensors_to_deallocate_;
+  std::map<int, TfLiteTensor*> tensors_to_deallocate_;
 };
 
 }  // namespace testing
-- 
GitLab


From 781a8f2e2093c38ff8e7dff94eb432cd948e54b0 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 21 Nov 2018 12:34:35 -0800
Subject: [PATCH 0684/1554] Remove pywrap_tensorflow from available symbols in
 Tensorflow v2

PiperOrigin-RevId: 222446280
---
 tensorflow/api_template.__init__.py             | 2 --
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 1 +
 3 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index 2efb8846c6..59b07e15b8 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -21,8 +21,6 @@ from __future__ import print_function as _print_function
 import os as _os
 
 # pylint: disable=g-bad-import-order
-from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
-
 from tensorflow.python.tools import component_api_helper as _component_api_helper
 _component_api_helper.package_hook(
     parent_package_str=__name__,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 873c41a390..1b496bde47 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -304,10 +304,6 @@ tf_module {
     name: "ones_initializer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "pywrap_tensorflow"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "qint16"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index f756bb80bc..c7bf73ba68 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -243,6 +243,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.multinomial": "tf.random.categorical",
         "tf.random.multinomial": "tf.random.categorical",
         "tf.load_file_system_library": "tf.load_library",
+        "tf.pywrap_tensorflow": "tf.compat.v1.pywrap_tensorflow",
     })
     # pylint: enable=line-too-long
 
-- 
GitLab


From bd258fe6d5761e864d07fbef52cd0835387be221 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Wed, 21 Nov 2018 12:37:15 -0800
Subject: [PATCH 0685/1554] Removed mentions of
 tf.get_partitioned_variable_list

PiperOrigin-RevId: 222446589
---
 tensorflow/python/ops/partitioned_variables.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py
index 5ba1d46a66..98a95f9f58 100644
--- a/tensorflow/python/ops/partitioned_variables.py
+++ b/tensorflow/python/ops/partitioned_variables.py
@@ -96,7 +96,7 @@ def variable_axis_size_partitioner(
 
   Returns:
     A partition function usable as the `partitioner` argument to
-    `variable_scope`, `get_variable`, and `get_partitioned_variable_list`.
+    `variable_scope` and `get_variable`.
 
   Raises:
     ValueError: If any of the byte counts are non-positive.
@@ -175,7 +175,7 @@ def min_max_variable_partitioner(max_partitions=1, axis=0,
 
   Returns:
     A partition function usable as the `partitioner` argument to
-    `variable_scope`, `get_variable`, and `get_partitioned_variable_list`.
+    `variable_scope` and `get_variable`.
 
   """
   def _partitioner(shape, dtype):
@@ -228,7 +228,7 @@ def fixed_size_partitioner(num_shards, axis=0):
 
   Returns:
     A partition function usable as the `partitioner` argument to
-    `variable_scope`, `get_variable`, and `get_partitioned_variable_list`.
+    `variable_scope` and `get_variable`.
   """
   def _partitioner(shape, **unused_args):
     partitions_list = [1] * len(shape)
-- 
GitLab


From a300c1de46dd71f07432d6368a1c2c48c4b69428 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 12:47:04 -0800
Subject: [PATCH 0686/1554] Remove the duplication of Convolution code, and
 rename TFLITE_USE_APPLE_ACCELERATE_FOR_CONV to TF_LITE_USE_CBLAS.

PiperOrigin-RevId: 222447851
---
 tensorflow/lite/kernels/conv.cc               |  16 +--
 tensorflow/lite/kernels/internal/BUILD        |   2 -
 .../kernels/internal/optimized/cblas_conv.h   | 109 ------------------
 .../internal/optimized/cblas_reference.h      |  69 -----------
 .../internal/optimized/optimized_ops.h        |  55 +++++++--
 .../lite/tools/make/targets/ios_makefile.inc  |   2 +-
 6 files changed, 47 insertions(+), 206 deletions(-)
 delete mode 100644 tensorflow/lite/kernels/internal/optimized/cblas_conv.h
 delete mode 100644 tensorflow/lite/kernels/internal/optimized/cblas_reference.h

diff --git a/tensorflow/lite/kernels/conv.cc b/tensorflow/lite/kernels/conv.cc
index 0c14b9eb65..1fd870be93 100644
--- a/tensorflow/lite/kernels/conv.cc
+++ b/tensorflow/lite/kernels/conv.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/kernels/eigen_support.h"
 #include "tensorflow/lite/kernels/gemm_support.h"
-#include "tensorflow/lite/kernels/internal/optimized/cblas_conv.h"
 #include "tensorflow/lite/kernels/internal/optimized/multithreaded_conv.h"
 #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
@@ -491,11 +490,10 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node,
   CalculateActivationRange(params->activation, &output_activation_min,
                            &output_activation_max);
   KernelType effective_kernel_type;
-  if ((kernel_type == kMultithreadOptimized ||
-       kernel_type == kCblasOptimized) &&
+  if ((kernel_type == kMultithreadOptimized) &&
       (params->dilation_width_factor != 1 ||
        params->dilation_height_factor != 1)) {
-    // kMultithreadOptimized and kCblasOptimized do not support dilation.
+    // kMultithreadOptimized does not support dilation.
     // Therefore, fallback to optimized.
     effective_kernel_type = kGenericOptimized;
   } else {
@@ -521,6 +519,7 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node,
                           GetTensorData<float>(im2col));
       break;
     }
+    case kCblasOptimized:
     case kGenericOptimized: {
       optimized_ops::Conv(op_params, GetTensorShape(input),
                           GetTensorData<float>(input), GetTensorShape(filter),
@@ -546,15 +545,6 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node,
           GetTensorData<float>(im2col));
       break;
     }
-    case kCblasOptimized: {
-      cblas_ops::Conv(op_params, GetTensorShape(input),
-                      GetTensorData<float>(input), GetTensorShape(filter),
-                      GetTensorData<float>(filter), GetTensorShape(bias),
-                      GetTensorData<float>(bias), GetTensorShape(output),
-                      GetTensorData<float>(output), GetTensorShape(im2col),
-                      GetTensorData<float>(im2col));
-      break;
-    }
   }
 }
 
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 6d9690ea46..7d2653f0a1 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -234,8 +234,6 @@ cc_library(
 cc_library(
     name = "optimized",
     hdrs = [
-        "optimized/cblas_conv.h",
-        "optimized/cblas_reference.h",
         "optimized/eigen_spatial_convolutions.h",
         "optimized/eigen_tensor_reduced_instantiations_oss.h",
         "optimized/multithreaded_conv.h",
diff --git a/tensorflow/lite/kernels/internal/optimized/cblas_conv.h b/tensorflow/lite/kernels/internal/optimized/cblas_conv.h
deleted file mode 100644
index 5377205050..0000000000
--- a/tensorflow/lite/kernels/internal/optimized/cblas_conv.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_CONV_H_
-#define TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_CONV_H_
-
-// The Conv implementation based on CBLAS interface. This is only used on iOS
-// for now, utilizing Apple's Accelerate framework.
-
-#if TFLITE_USE_APPLE_ACCELERATE_FOR_CONV
-#include <Accelerate/Accelerate.h>
-#else
-#include "tensorflow/lite/kernels/internal/optimized/cblas_reference.h"
-#endif
-
-#include "tensorflow/lite/kernels/internal/optimized/multithreaded_conv.h"
-#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
-
-namespace tflite {
-namespace cblas_ops {
-
-inline void Conv(const ConvParams& params, const RuntimeShape& input_shape,
-                 const float* input_data, const RuntimeShape& filter_shape,
-                 const float* filter_data, const RuntimeShape& bias_shape,
-                 const float* bias_data, const RuntimeShape& output_shape,
-                 float* output_data, const RuntimeShape& im2col_shape,
-                 float* im2col_data) {
-  const int stride_width = params.stride_width;
-  const int stride_height = params.stride_height;
-  const int pad_width = params.padding_values.width;
-  const int pad_height = params.padding_values.height;
-  const int dilation_width_factor = params.dilation_width_factor;
-  const int dilation_height_factor = params.dilation_height_factor;
-  const float output_activation_min = params.float_activation_min;
-  const float output_activation_max = params.float_activation_max;
-  TFLITE_DCHECK_EQ(input_shape.DimensionsCount(), 4);
-  TFLITE_DCHECK_EQ(filter_shape.DimensionsCount(), 4);
-  TFLITE_DCHECK_EQ(output_shape.DimensionsCount(), 4);
-  gemmlowp::ScopedProfilingLabel label("Conv/cblas");
-
-  const float* gemm_input_data = nullptr;
-  const RuntimeShape* gemm_input_shape = nullptr;
-  const int filter_width = filter_shape.Dims(2);
-  const int filter_height = filter_shape.Dims(1);
-  const bool need_im2col = stride_width != 1 || stride_height != 1 ||
-                           filter_width != 1 || filter_height != 1;
-  if (need_im2col) {
-    TFLITE_DCHECK(im2col_data);
-    ConvParams op_params;
-    op_params.padding_type = PaddingType::kSame;
-    op_params.padding_values.width = pad_width;
-    op_params.padding_values.height = pad_height;
-    op_params.stride_width = stride_width;
-    op_params.stride_height = stride_height;
-    op_params.dilation_width_factor = dilation_width_factor;
-    op_params.dilation_height_factor = dilation_height_factor;
-    optimized_ops::Im2col(op_params, filter_height, filter_width, 0,
-                          input_shape, input_data, im2col_shape, im2col_data);
-
-    gemm_input_data = im2col_data;
-    gemm_input_shape = &im2col_shape;
-  } else {
-    TFLITE_DCHECK(!im2col_data);
-    gemm_input_data = input_data;
-    gemm_input_shape = &input_shape;
-  }
-
-  // The following code computes matrix multiplication c = a * transponse(b)
-  // with CBLAS, where:
-  // * `a` is a matrix with dimensions (m, k).
-  // * `b` is a matrix with dimensions (n, k), so transpose(b) is (k, n).
-  // * `c` is a matrix with dimensions (m, n).
-  // The naming of variables are aligned with CBLAS specification here.
-  const float* a = gemm_input_data;
-  const float* b = filter_data;
-  float* c = output_data;
-  const int gemm_input_dims = gemm_input_shape->DimensionsCount();
-  int m = FlatSizeSkipDim(*gemm_input_shape, gemm_input_dims - 1);
-  int n = output_shape.Dims(3);
-  int k = gemm_input_shape->Dims(gemm_input_dims - 1);
-  // The stride of matrix a, b and c respectively.
-  int stride_a = k;
-  int stride_b = k;
-  int stride_c = n;
-
-  cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, m, n, k, 1.0f, a,
-              stride_a, b, stride_b, 0.0f, c, stride_c);
-
-  optimized_ops::AddBiasAndEvalActivationFunction(
-      output_activation_min, output_activation_max, bias_shape, bias_data,
-      output_shape, output_data);
-}
-
-}  // namespace cblas_ops
-}  // namespace tflite
-
-#endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_CONV_H_
diff --git a/tensorflow/lite/kernels/internal/optimized/cblas_reference.h b/tensorflow/lite/kernels/internal/optimized/cblas_reference.h
deleted file mode 100644
index fa07578612..0000000000
--- a/tensorflow/lite/kernels/internal/optimized/cblas_reference.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_REFERENCE_H_
-#define TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_REFERENCE_H_
-
-#include "tensorflow/lite/kernels/internal/compatibility.h"
-
-// The reference implementation for a small subset of CBLAS interface.
-// This is only used for testing CBLAS implementation, and should never be used
-// in production code.
-
-namespace tflite {
-namespace cblas_ops {
-
-// The following code follows the original CBLAS specification, and it might
-// conflict with the TensorFlow naming convention.
-// TODO(ycling): Find another way to test CBLAS with bazel, without writing
-// a reference implementation by ourselves.
-enum CBLAS_ORDER { CblasRowMajor = 0, CblasColMajor = 1 };
-
-enum CBLAS_TRANSPOSE { CblasNoTrans = 0, CblasTrans = 1, CblasConjTrans = 2 };
-
-// A reference implementation for matrix multiplication.
-// The following code computes, c = a * transponse(b) matrix multiplication
-// with CBLAS, where:
-// * `a` is a matrix with dimensions (m, k).
-// * `b` is a matrix with dimensions (n, k), so transpose(b) is (k, n).
-// * `c` is a matrix with dimensions (m, n).
-// The naming of variables is aligned with CBLAS specification here.
-void cblas_sgemm(const enum CBLAS_ORDER order,
-                 const enum CBLAS_TRANSPOSE trans_a,
-                 const enum CBLAS_TRANSPOSE trans_b, const int m, const int n,
-                 const int k, const float alpha, const float *a,
-                 const int stride_a, const float *b, const int stride_b,
-                 const float beta, float *c, const int stride_c) {
-  TFLITE_DCHECK(order == CblasRowMajor);
-  TFLITE_DCHECK(trans_a == CblasNoTrans);
-  TFLITE_DCHECK(trans_b == CblasTrans);
-  TFLITE_DCHECK(beta == 0.0f);
-  for (int row = 0; row < m; ++row) {
-    for (int col = 0; col < n; ++col) {
-      // If `beta` non-zero, multiple it with the original values in output.
-      // Otherwise, ignore the original value in output completely.
-      float value = 0.0f;
-      for (int idx = 0; idx < k; ++idx) {
-        value += alpha * a[stride_a * row + idx] * b[stride_b * col + idx];
-      }
-      c[stride_c * row + col] = value;
-    }
-  }
-}
-
-}  // namespace cblas_ops
-}  // namespace tflite
-
-#endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_OPTIMIZED_CBLAS_REFERENCE_H_
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index e2329c79c7..df335e9e92 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -25,6 +25,10 @@ limitations under the License.
 #include <tuple>
 #include <type_traits>
 
+#if defined(TF_LITE_USE_CBLAS) && defined(__APPLE__)
+#include <Accelerate/Accelerate.h>
+#endif
+
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "fixedpoint/fixedpoint.h"
@@ -1868,18 +1872,45 @@ inline void Conv(const ConvParams& params, const RuntimeShape& input_shape,
     gemm_input_shape = &input_shape;
   }
 
-  const auto im2col_matrix_map =
-      MapAsMatrixWithLastDimAsRows(gemm_input_data, *gemm_input_shape);
-  const auto filter_matrix_map =
-      MapAsMatrixWithFirstDimAsCols(filter_data, filter_shape);
-  auto output_matrix_map =
-      MapAsMatrixWithLastDimAsRows(output_data, output_shape);
-
-  Gemm(filter_matrix_map.transpose(), im2col_matrix_map, &output_matrix_map);
-
-  AddBiasAndEvalActivationFunction(output_activation_min, output_activation_max,
-                                   bias_shape, bias_data, output_shape,
-                                   output_data);
+  // The following code computes matrix multiplication c = a * transponse(b)
+  // with CBLAS, where:
+  // * `a` is a matrix with dimensions (m, k).
+  // * `b` is a matrix with dimensions (n, k), so transpose(b) is (k, n).
+  // * `c` is a matrix with dimensions (m, n).
+  // The naming of variables are aligned with CBLAS specification here.
+  const float* a = gemm_input_data;
+  const float* b = filter_data;
+  float* c = output_data;
+  const int gemm_input_dims = gemm_input_shape->DimensionsCount();
+  int m = FlatSizeSkipDim(*gemm_input_shape, gemm_input_dims - 1);
+  int n = output_shape.Dims(3);
+  int k = gemm_input_shape->Dims(gemm_input_dims - 1);
+
+#if defined(TF_LITE_USE_CBLAS) && defined(__APPLE__)
+  // The stride of matrix a, b and c respectively.
+  int stride_a = k;
+  int stride_b = k;
+  int stride_c = n;
+
+  cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, m, n, k, 1.0f, a,
+              stride_a, b, stride_b, 0.0f, c, stride_c);
+#else
+  // When an optimized CBLAS implementation is not available, fall back
+  // to using Eigen.
+  typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
+      Matrix;
+  typedef Eigen::Map<Matrix> MatrixRef;
+  typedef Eigen::Map<const Matrix> ConstMatrixRef;
+
+  MatrixRef matrix_c(c, m, n);
+  ConstMatrixRef matrix_a(a, m, k);
+  ConstMatrixRef matrix_b(b, n, k);
+  matrix_c.noalias() = matrix_a * matrix_b.transpose();
+#endif  //  defined(TF_LITE_USE_CBLAS) && defined(__APPLE__)
+
+  optimized_ops::AddBiasAndEvalActivationFunction(
+      output_activation_min, output_activation_max, bias_shape, bias_data,
+      output_shape, output_data);
 }
 
 inline void HybridConv(const ConvParams& params, float* scaling_factors_ptr,
diff --git a/tensorflow/lite/tools/make/targets/ios_makefile.inc b/tensorflow/lite/tools/make/targets/ios_makefile.inc
index 7f36b8ecef..ae9276f9a6 100644
--- a/tensorflow/lite/tools/make/targets/ios_makefile.inc
+++ b/tensorflow/lite/tools/make/targets/ios_makefile.inc
@@ -22,7 +22,7 @@ ifeq ($(TARGET), ios)
 	TARGET_ARCH := x86_64
 	CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \
 		-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \
-		-DTFLITE_USE_APPLE_ACCELERATE_FOR_CONV \
+		-DTF_LITE_USE_CBLAS \
 		-fembed-bitcode \
 		-Wno-c++11-narrowing \
 		-mno-thumb \
-- 
GitLab


From 32e55164f691ff1f61a57a7bf36d414cb7ac7968 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 12:54:21 -0800
Subject: [PATCH 0687/1554] Update Google Cloud Bigtable C++ Client to the
 v0.3.0 release.

PiperOrigin-RevId: 222448784
---
 .../test_kernels/bigtable_test_client.cc      | 33 +++++++++++++++++++
 .../test_kernels/bigtable_test_client.h       | 19 +++++++++++
 tensorflow/workspace.bzl                      |  8 ++---
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
index f083ce6f44..e95dc57718 100644
--- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
+++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
@@ -366,6 +366,39 @@ BigtableTestClient::MutateRows(
   return MakeUnique<MutateRowsResponse>(request.entries_size());
 }
 
+std::unique_ptr<grpc::ClientAsyncResponseReaderInterface<
+    google::bigtable::v2::MutateRowResponse>>
+BigtableTestClient::AsyncMutateRow(
+    grpc::ClientContext* context,
+    google::bigtable::v2::MutateRowRequest const& request,
+    grpc::CompletionQueue* cq) {
+  LOG(WARNING) << "Call to InMemoryDataClient::" << __func__
+               << "(); this will likely cause a crash!";
+  return nullptr;
+}
+
+std::unique_ptr<::grpc::ClientAsyncReaderInterface<
+    ::google::bigtable::v2::SampleRowKeysResponse>>
+BigtableTestClient::AsyncSampleRowKeys(
+    ::grpc::ClientContext* context,
+    const ::google::bigtable::v2::SampleRowKeysRequest& request,
+    ::grpc::CompletionQueue* cq, void* tag) {
+  LOG(WARNING) << "Call to InMemoryDataClient::" << __func__
+               << "(); this will likely cause a crash!";
+  return nullptr;
+}
+
+std::unique_ptr<::grpc::ClientAsyncReaderInterface<
+    ::google::bigtable::v2::MutateRowsResponse>>
+BigtableTestClient::AsyncMutateRows(
+    ::grpc::ClientContext* context,
+    const ::google::bigtable::v2::MutateRowsRequest& request,
+    ::grpc::CompletionQueue* cq, void* tag) {
+  LOG(WARNING) << "Call to InMemoryDataClient::" << __func__
+               << "(); this will likely cause a crash!";
+  return nullptr;
+}
+
 std::shared_ptr<grpc::Channel> BigtableTestClient::Channel() {
   LOG(WARNING) << "Call to InMemoryDataClient::Channel(); this will likely "
                   "cause a crash!";
diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
index dac2b16a21..c4a1f06bc5 100644
--- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
+++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
@@ -61,6 +61,25 @@ class BigtableTestClient : public ::google::cloud::bigtable::DataClient {
   MutateRows(grpc::ClientContext* context,
              google::bigtable::v2::MutateRowsRequest const& request) override;
 
+  std::unique_ptr<grpc::ClientAsyncResponseReaderInterface<
+      google::bigtable::v2::MutateRowResponse>>
+  AsyncMutateRow(grpc::ClientContext* context,
+                 google::bigtable::v2::MutateRowRequest const& request,
+                 grpc::CompletionQueue* cq) override;
+
+  std::unique_ptr<::grpc::ClientAsyncReaderInterface<
+      ::google::bigtable::v2::SampleRowKeysResponse>>
+  AsyncSampleRowKeys(
+      ::grpc::ClientContext* context,
+      const ::google::bigtable::v2::SampleRowKeysRequest& request,
+      ::grpc::CompletionQueue* cq, void* tag) override;
+
+  std::unique_ptr<::grpc::ClientAsyncReaderInterface<
+      ::google::bigtable::v2::MutateRowsResponse>>
+  AsyncMutateRows(::grpc::ClientContext* context,
+                  const ::google::bigtable::v2::MutateRowsRequest& request,
+                  ::grpc::CompletionQueue* cq, void* tag) override;
+
   std::shared_ptr<grpc::Channel> Channel() override;
 
  private:
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5c58bb1080..7ad094c507 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -179,15 +179,15 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "com_github_googlecloudplatform_google_cloud_cpp",
-        sha256 = "fdd3b3aecce60987e5525e55bf3a21d68a8695320bd5b980775af6507eec3944",
-        strip_prefix = "google-cloud-cpp-14760a86c4ffab9943b476305c4fe927ad95db1c",
+        sha256 = "3ade2072e6588ff56c0434abe6c63aa5f3f2d56be15a299bafc7e9cdf0a12c17",
+        strip_prefix = "google-cloud-cpp-0.3.0",
         system_build_file = clean_dep("//third_party/systemlibs:google_cloud_cpp.BUILD"),
         system_link_files = {
             "//third_party/systemlibs:google_cloud_cpp.google.cloud.bigtable.BUILD": "google/cloud/bigtable/BUILD",
         },
         urls = [
-            "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/14760a86c4ffab9943b476305c4fe927ad95db1c.tar.gz",
-            "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/14760a86c4ffab9943b476305c4fe927ad95db1c.tar.gz",
+            "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.3.0.tar.gz",
+            "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.3.0.tar.gz",
         ],
     )
 
-- 
GitLab


From dd3b98867f9809023ce975bcd3e96dba4021d7bb Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Wed, 21 Nov 2018 13:19:46 -0800
Subject: [PATCH 0688/1554] Re-apply buildifier

---
 tensorflow/core/BUILD | 114 +++++++++++++++++++++---------------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 41d3f6eab8..52d3f8f2b9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1349,63 +1349,63 @@ cc_library(
     name = "all_kernels_statically_linked",
     visibility = ["//visibility:private"],
     deps = [
-        "//tensorflow/core/kernels:array",
-        "//tensorflow/core/kernels:audio",
-        "//tensorflow/core/kernels:batch_kernels",
-        "//tensorflow/core/kernels:bincount_op",
-        "//tensorflow/core/kernels:boosted_trees_ops",
-        "//tensorflow/core/kernels:candidate_sampler_ops",
-        "//tensorflow/core/kernels:checkpoint_ops",
-        "//tensorflow/core/kernels:collective_ops",
-        "//tensorflow/core/kernels:control_flow_ops",
-        "//tensorflow/core/kernels:ctc_ops",
-        "//tensorflow/core/kernels:cudnn_rnn_kernels",
-        "//tensorflow/core/kernels:data_flow",
-        "//tensorflow/core/kernels:dataset_ops",
-        "//tensorflow/core/kernels:decode_proto_op",
-        "//tensorflow/core/kernels:encode_proto_op",
-        "//tensorflow/core/kernels:fake_quant_ops",
-        "//tensorflow/core/kernels:function_ops",
-        "//tensorflow/core/kernels:functional_ops",
-        "//tensorflow/core/kernels:grappler",
-        "//tensorflow/core/kernels:histogram_op",
-        "//tensorflow/core/kernels:image",
-        "//tensorflow/core/kernels:io",
-        "//tensorflow/core/kernels:linalg",
-        "//tensorflow/core/kernels:list_kernels",
-        "//tensorflow/core/kernels:lookup",
-        "//tensorflow/core/kernels:logging",
-        "//tensorflow/core/kernels:manip",
-        "//tensorflow/core/kernels:math",
-        "//tensorflow/core/kernels:multinomial_op",
-        "//tensorflow/core/kernels:nn",
-        "//tensorflow/core/kernels:parameterized_truncated_normal_op",
-        "//tensorflow/core/kernels:parsing",
-        "//tensorflow/core/kernels:partitioned_function_ops",
-        "//tensorflow/core/kernels:ragged_ops",
-        "//tensorflow/core/kernels:random_ops",
-        "//tensorflow/core/kernels:random_poisson_op",
-        "//tensorflow/core/kernels:remote_fused_graph_ops",
-        "//tensorflow/core/kernels:required",
-        "//tensorflow/core/kernels:resource_variable_ops",
-        "//tensorflow/core/kernels:rpc_op",
-        "//tensorflow/core/kernels:scoped_allocator_ops",
-        "//tensorflow/core/kernels:sdca_ops",
-        "//tensorflow/core/kernels:searchsorted_op",
-        "//tensorflow/core/kernels:set_kernels",
-        "//tensorflow/core/kernels:sparse",
-        "//tensorflow/core/kernels:state",
-        "//tensorflow/core/kernels:stateless_random_ops",
-        "//tensorflow/core/kernels:string",
-        "//tensorflow/core/kernels:summary_kernels",
-        "//tensorflow/core/kernels:training_ops",
-        "//tensorflow/core/kernels:word2vec_kernels",
-    ] + tf_additional_cloud_kernel_deps() +
-    select({
-        "//tensorflow:no_nccl_support": [],
-        "//tensorflow:windows": [],
-        "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"],
-    }) + if_not_windows([
+               "//tensorflow/core/kernels:array",
+               "//tensorflow/core/kernels:audio",
+               "//tensorflow/core/kernels:batch_kernels",
+               "//tensorflow/core/kernels:bincount_op",
+               "//tensorflow/core/kernels:boosted_trees_ops",
+               "//tensorflow/core/kernels:candidate_sampler_ops",
+               "//tensorflow/core/kernels:checkpoint_ops",
+               "//tensorflow/core/kernels:collective_ops",
+               "//tensorflow/core/kernels:control_flow_ops",
+               "//tensorflow/core/kernels:ctc_ops",
+               "//tensorflow/core/kernels:cudnn_rnn_kernels",
+               "//tensorflow/core/kernels:data_flow",
+               "//tensorflow/core/kernels:dataset_ops",
+               "//tensorflow/core/kernels:decode_proto_op",
+               "//tensorflow/core/kernels:encode_proto_op",
+               "//tensorflow/core/kernels:fake_quant_ops",
+               "//tensorflow/core/kernels:function_ops",
+               "//tensorflow/core/kernels:functional_ops",
+               "//tensorflow/core/kernels:grappler",
+               "//tensorflow/core/kernels:histogram_op",
+               "//tensorflow/core/kernels:image",
+               "//tensorflow/core/kernels:io",
+               "//tensorflow/core/kernels:linalg",
+               "//tensorflow/core/kernels:list_kernels",
+               "//tensorflow/core/kernels:lookup",
+               "//tensorflow/core/kernels:logging",
+               "//tensorflow/core/kernels:manip",
+               "//tensorflow/core/kernels:math",
+               "//tensorflow/core/kernels:multinomial_op",
+               "//tensorflow/core/kernels:nn",
+               "//tensorflow/core/kernels:parameterized_truncated_normal_op",
+               "//tensorflow/core/kernels:parsing",
+               "//tensorflow/core/kernels:partitioned_function_ops",
+               "//tensorflow/core/kernels:ragged_ops",
+               "//tensorflow/core/kernels:random_ops",
+               "//tensorflow/core/kernels:random_poisson_op",
+               "//tensorflow/core/kernels:remote_fused_graph_ops",
+               "//tensorflow/core/kernels:required",
+               "//tensorflow/core/kernels:resource_variable_ops",
+               "//tensorflow/core/kernels:rpc_op",
+               "//tensorflow/core/kernels:scoped_allocator_ops",
+               "//tensorflow/core/kernels:sdca_ops",
+               "//tensorflow/core/kernels:searchsorted_op",
+               "//tensorflow/core/kernels:set_kernels",
+               "//tensorflow/core/kernels:sparse",
+               "//tensorflow/core/kernels:state",
+               "//tensorflow/core/kernels:stateless_random_ops",
+               "//tensorflow/core/kernels:string",
+               "//tensorflow/core/kernels:summary_kernels",
+               "//tensorflow/core/kernels:training_ops",
+               "//tensorflow/core/kernels:word2vec_kernels",
+           ] + tf_additional_cloud_kernel_deps() +
+           select({
+               "//tensorflow:no_nccl_support": [],
+               "//tensorflow:windows": [],
+               "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"],
+           }) + if_not_windows([
         "//tensorflow/core/kernels:fact_op",
         "//tensorflow/core/kernels:array_not_windows",
         "//tensorflow/core/kernels:math_not_windows",
-- 
GitLab


From 12ef5a5fe9a95739f8cbc6c74f89b2959d2d2286 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Wed, 21 Nov 2018 13:22:44 -0800
Subject: [PATCH 0689/1554] Revert TFLite examples to use
 tensorflow/contrib/lite

The CocoaPod didn't catch up with the changes which moves
TFLite out of contrib/ directory. Reverting for now.
---
 .../examples/ios/camera/CameraExampleViewController.h     | 4 ++--
 .../examples/ios/camera/CameraExampleViewController.mm    | 8 ++++----
 .../lite/examples/ios/simple/RunModelViewController.mm    | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h
index 6bc94e9502..fb5800e86d 100644
--- a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h
+++ b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h
@@ -17,8 +17,8 @@
 
 #include <vector>
 
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/model.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/model.h"
 
 @interface CameraExampleViewController
     : UIViewController<UIGestureRecognizerDelegate, AVCaptureVideoDataOutputSampleBufferDelegate> {
diff --git a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm
index 1e6725592b..996cff2616 100644
--- a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm
+++ b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm
@@ -23,10 +23,10 @@
 #include <iostream>
 #include <queue>
 
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/model.h"
-#include "tensorflow/lite/string_util.h"
-#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/model.h"
+#include "tensorflow/contrib/lite/string_util.h"
+#include "tensorflow/contrib/lite/op_resolver.h"
 
 #define LOG(x) std::cerr
 
diff --git a/tensorflow/lite/examples/ios/simple/RunModelViewController.mm b/tensorflow/lite/examples/ios/simple/RunModelViewController.mm
index e5764944f6..650c73f732 100644
--- a/tensorflow/lite/examples/ios/simple/RunModelViewController.mm
+++ b/tensorflow/lite/examples/ios/simple/RunModelViewController.mm
@@ -22,10 +22,10 @@
 #include <sstream>
 #include <string>
 
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/model.h"
-#include "tensorflow/lite/string_util.h"
-#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/model.h"
+#include "tensorflow/contrib/lite/string_util.h"
+#include "tensorflow/contrib/lite/op_resolver.h"
 
 #include "ios_image_load.h"
 
-- 
GitLab


From 2dc24d1cf094d2a63fcf14a65d8f20691880f6a1 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Wed, 21 Nov 2018 13:20:44 -0800
Subject: [PATCH 0690/1554] [JAX] Support Python 3 in JAX.

Notable changes:
* __builtins__ -> six.moves.builtins
* xrange -> six.moves.xrange
* reduce -> six.moves.reduce
* map/filter return generators, so add explicit list() calls around many to avoid a semantic change. Probably too conservative, but it's easier to add them everywhere than work out which are strictly necessary.
* no long in Python 3
* raise TypeError, x -> raise TypeError(x)
* gen.next() -> next(gen)
* string / bytes
* no tuple destructuring in lambda arguments.
* / and np.divide changed semantics, use // and np.floor_divide.

PiperOrigin-RevId: 222451919
---
 tensorflow/compiler/xla/python/xla_client.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 92b0685dbb..995d2b64dc 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -26,6 +26,9 @@ import os
 
 import numpy as np
 
+import six
+from six.moves import xrange
+
 from tensorflow.compiler.xla import xla_data_pb2
 from tensorflow.compiler.xla.python import pywrap_xla as c_api
 from tensorflow.compiler.xla.service import hlo_pb2
@@ -322,6 +325,9 @@ class Shape(object):
   def __ne__(self, other):
     return not self == other
 
+  def __hash__(self):
+    return hash((self._dtype, self._dimensions, self._minor_to_major))
+
   def __repr__(self):
     return ('xla_client.Shape(_dtype={!r}, _dimensions={!r}, '
             '_is_tuple={!r}, _minor_to_major={!r})').format(
@@ -541,6 +547,8 @@ class LocalComputation(object):
       ]
       result_shape = result_shape.map_leaves(layout_fn)
 
+    argument_shapes = list(argument_shapes)
+
     compile_options = compile_options or CompileOptions()
     compile_options.result_shape = result_shape
     if self._backend.backend_type == BackendType.XRT:
@@ -1380,6 +1388,8 @@ def initialize_platform_name(platform_name):
   Raises:
     A runtime exception if the XLA service has already been initialized.
   """
+  if six.PY3:
+    platform_name = platform_name.encode('utf-8')
   c_api.InitializePlatformName(platform_name)
 
 
-- 
GitLab


From a2bda5a8b45a08dbd6c0580a0290777741c32d29 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 21 Nov 2018 13:27:53 -0800
Subject: [PATCH 0691/1554] Add Sigmoid, Tanh ops and unit tests.

---
 .../contrib/tensorrt/convert/convert_graph.cc |  2 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 40 +++++++++-
 .../tensorrt/convert/convert_nodes_test.cc    | 77 +++++++++++++++++++
 3 files changed, 115 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 21f505b7fe..589db56758 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -96,6 +96,8 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
       "MaxPool",
       "BiasAdd",
       "Relu",
+      "Sigmoid",
+      "Tanh",
       "Add",
       "Mul",
       "Sub",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 631c2575c0..b1fa6a51d4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1977,13 +1977,43 @@ tensorflow::Status ConvertPool(OpConverterParams* params) {
 }
 
 tensorflow::Status ConvertActivation(OpConverterParams* params) {
-  const nvinfer1::ITensor* tensor = params->inputs.at(0).tensor();
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        node_def.op(), " expects one input, at ", node_def.name());
+  }
+  if (!inputs.at(0).is_tensor()) {
+    return tensorflow::errors::Unimplemented(
+        node_def.op(), " is only implemented for tensors, at ", node_def.name());
+  }
+  static const std::unordered_map<string, nvinfer1::ActivationType> ops{
+      {"Relu", nvinfer1::ActivationType::kRELU},
+      {"Sigmoid", nvinfer1::ActivationType::kSIGMOID},
+      {"Tanh", nvinfer1::ActivationType::kTANH},
+  };
+  auto op_pair = ops.find(node_def.op());
+  if (op_pair == ops.end()) {
+    return tensorflow::errors::Unimplemented(
+        "Activation op: ", node_def.op(), " not supported at: ",
+        node_def.name());
+  }
+  if (params->validation_only) return tensorflow::Status::OK();
+
+  // Start conversion.
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   nvinfer1::IActivationLayer* layer =
       params->converter->network()->addActivation(
           *const_cast<nvinfer1::ITensor*>(tensor),
-          nvinfer1::ActivationType::kRELU);
-  TFTRT_RETURN_ERROR_IF_NULLPTR(layer, params->node_def.name());
+          op_pair->second);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  // Set quantization range for output of Sigmoid, Tanh.
+  if (node_def.op() == "Sigmoid") {
+    params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 1.0f);
+  } else if (node_def.op() == "Tanh") {
+    params->converter->ProvideQuantizationRange(output_tensor, -1.0f, 1.0f);
+  }
   params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
@@ -3093,6 +3123,9 @@ static void RegisterValidatableOpConverters(
        {"Add", "Mul", "Sub", "Div", "RealDiv", "Maximum", "Minimum"}) {
     (*registration)[binary_op_type] = ConvertBinary;
   }
+  for (auto activation_op_type : {"Relu", "Sigmoid", "Tanh"}) {
+    (*registration)[activation_op_type] = ConvertActivation;
+  }
 }
 
 void TrtNodeValidator::RegisterOpValidators() {
@@ -3104,7 +3137,6 @@ void Converter::RegisterOpConverters() {
 
   op_registry_["Conv2D"] = ConvertConv2D;
   op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
-  op_registry_["Relu"] = ConvertActivation;
   op_registry_["MaxPool"] = ConvertPool;
   op_registry_["AvgPool"] = ConvertPool;
   // TODO(ben,jie): this is a temp hack.
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 603c4f7b5e..92697e2f87 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1964,6 +1964,83 @@ TEST_F(OpConverterTest, ConvertRelu6) {
   }
 }
 
+TEST_F(OpConverterTest, ConvertActivation) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_act", "Relu", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Relu expects one input, at my_act");
+  }
+  {
+    // Input is weights, should fail.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto relu = ops::Relu(s.WithOpName("my_act"), input);
+    const NodeDef& node_def = relu.operation.node()->def();
+    AddTestWeights<int32>("input", {1, 2, 3}, {-3, -2, -1, 0, 1, 2});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Relu is only implemented for tensors, at my_act");
+  }
+
+  // Get nodedef for activation layer.
+  auto get_act_nodedef = [](std::string op_name) -> NodeDef {
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    if (op_name == "Relu") {
+      auto act = ops::Relu(s.WithOpName("my_act"), input);
+      return act.operation.node()->def();
+    } else if (op_name == "Sigmoid") {
+      auto act = ops::Sigmoid(s.WithOpName("my_act"), input);
+      return act.operation.node()->def();
+    } else if (op_name == "Tanh") {
+      auto act = ops::Tanh(s.WithOpName("my_act"), input);
+      return act.operation.node()->def();
+    }
+    EXPECT_TRUE(false);
+    return MakeNodeDef("my_act", "Relu", {});
+  };
+  // Get expected output for activation layer.
+  auto get_act_output = [](std::string op_name, float input) -> float {
+    if (op_name == "Relu") {
+      return (input > 0.0f) ? input : 0.0f;
+    } else if (op_name == "Sigmoid") {
+      return 1.0f / (1.0f + std::exp(-input));
+    } else if (op_name == "Tanh") {
+      return std::tanh(input);
+    }
+    EXPECT_TRUE(false);
+    return input;
+  };
+
+  {
+    // Ok.
+    for (std::string op_name : {"Relu", "Sigmoid", "Tanh"}) {
+      Reset();
+      NodeDef node_def = get_act_nodedef(op_name);
+      AddTestTensor("input", {1, 2, 3});
+      RunValidationAndConversion(node_def);
+      TRT_TensorOrWeights output;
+      TF_EXPECT_OK(GetTensorOrWeights("my_act", &output));
+      EXPECT_TRUE(output.is_tensor());
+      EXPECT_TRUE(TrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()))
+          << output.DebugString();
+
+      const std::vector<float> input_data = {-100, -2, -1, 0, 1, 100};
+      std::vector<float> output_data(6);
+      BuildAndRun("input", input_data, "my_act", &output_data);
+      for (int i = 0; i < input_data.size(); i++) {
+        const float expected_output = get_act_output(op_name, input_data[i]);
+        EXPECT_FLOAT_EQ(output_data[i], expected_output)
+          << op_name << "(" << input_data[i] << ") should be equal to "
+          << expected_output;
+      }
+    }
+  }
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From 92f59bf4c3d88b9553c7f0fae17d9fff6fbf13ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 13:45:54 -0800
Subject: [PATCH 0692/1554] Slightly improve toco error messages.

PiperOrigin-RevId: 222455209
---
 tensorflow/lite/toco/tooling_util.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 084169548e..611add9daf 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -1035,10 +1035,10 @@ void CheckEachArray(const Model& model) {
     if (colon_pos != string::npos) {
       CHECK_EQ(name.substr(colon_pos + 1).find_first_not_of("0123456789"),
                string::npos)
-          << "Array name must only have digits after colon";
+          << "Array '" << name << "' has non-digit characters after colon.";
     }
-    CHECK_GT(colon_pos, 0)
-        << "First character of array name must not be a colon.";
+    CHECK_GT(colon_pos, 0) << "Array '" << name
+                           << "' must not start with a colon.";
   }
 }
 
-- 
GitLab


From da2d9779e0e4706b5df96c3cc71f81132445a8af Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 21 Nov 2018 13:58:59 -0800
Subject: [PATCH 0693/1554] Add square op and unit tests

---
 .../contrib/tensorrt/convert/convert_graph.cc |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 43 +++++++++++++++++++
 .../tensorrt/convert/convert_nodes_test.cc    | 42 ++++++++++++++++++
 3 files changed, 86 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 21f505b7fe..74499a0298 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -129,6 +129,7 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
       "Max",
       "Min",
       "Relu6",
+      "Square",
   };
   bool is_supported_op_type =
       (candidate_ops.count(node->type_string()) ||
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 631c2575c0..6de3aba4d7 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2496,6 +2496,48 @@ tensorflow::Status ConvertUnary(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertSquare(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        "Square expects one input, at ", node_def.name());
+  }
+  if (inputs.at(0).is_weights()) {
+    return tensorflow::errors::Unimplemented(
+        "Square is only implemented for tensors, at ", node_def.name());
+  }
+  if (params->validation_only) return Status::OK();
+
+  // Constant 2 with same rank as input
+  nvinfer1::Dims dims = inputs.at(0).GetTrtDims();
+  for (int i = 0; i < dims.nbDims; i++) {
+    dims.d[i] = 1;
+  }
+  TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
+      tensorflow::DataType::DT_FLOAT, dims);
+  auto weights_ptr = static_cast<float*>(const_cast<void*>(
+      weights.GetValues()));
+  weights_ptr[0] = 2.f;
+  nvinfer1::IConstantLayer* const2_layer =
+      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const2_layer, node_def.name());
+
+  // ElementWise Pow Operation
+  const nvinfer1::ITensor* tensor_l = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor_r = const2_layer->getOutput(0);
+  nvinfer1::IElementWiseLayer* layer =
+      params->converter->network()->addElementWise(
+          *const_cast<nvinfer1::ITensor*>(tensor_l),
+          *const_cast<nvinfer1::ITensor*>(tensor_r),
+          nvinfer1::ElementWiseOperation::kPOW);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+
+  params->outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
 tensorflow::Status ConvertReduce(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
@@ -3083,6 +3125,7 @@ static void RegisterValidatableOpConverters(
   (*registration)["Reshape"] = ConvertReshape;
   (*registration)["MatMul"] = ConvertMatMul;
   (*registration)["Relu6"] = ConvertRelu6;
+  (*registration)["Square"] = ConvertSquare;
 
   for (auto quantization_op_type :
        {"QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 603c4f7b5e..312375a661 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1964,6 +1964,48 @@ TEST_F(OpConverterTest, ConvertRelu6) {
   }
 }
 
+TEST_F(OpConverterTest, ConvertSquare) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_square", "Square", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Square expects one input, at my_square");
+  }
+
+  // Get the NodeDef for Square.
+  Scope s = Scope::NewRootScope();
+  auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+  auto square = ops::Square(s.WithOpName("my_square"), input);
+  const NodeDef& node_def = square.operation.node()->def();
+
+  {
+    // Input is weights, should fail.
+    Reset();
+    AddTestWeights<int32>("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+        "Square is only implemented for tensors, at my_square");
+  }
+  {
+    // Input is tensor, Ok.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_square", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    std::vector<float> expected_output_data = {1, 4, 9, 16, 25, 36};
+    BuildAndRun("input", {1, 2, 3, 4, -5, 6}, "my_square", &output_data);
+    for (int i = 0; i < output_data.size(); i++) {
+      EXPECT_FLOAT_EQ(output_data[i], expected_output_data[i]);
+    }
+  }
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From 9c17bfba4885cf95ae7f2ec3ede96520d2bbfe59 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 21 Nov 2018 13:57:32 -0800
Subject: [PATCH 0694/1554] Change l2_normalize API for TF 2.0.

PiperOrigin-RevId: 222456684
---
 tensorflow/python/ops/nn_impl.py              | 29 +++++++++++++++++--
 .../api/golden/v2/tensorflow.linalg.pbtxt     |  2 +-
 .../tools/api/golden/v2/tensorflow.math.pbtxt |  2 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  9 ++++++
 5 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 9cf53f191a..6591da5be8 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -329,7 +329,7 @@ def swish(features):
   return features * math_ops.sigmoid(features)
 
 
-@tf_export("math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize")
+@tf_export(v1=["math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize"])
 @deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None):
   """Normalizes along dimension `axis` using an L2 norm.
@@ -350,11 +350,36 @@ def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None):
     name: A name for this operation (optional).
     dim: Deprecated alias for axis.
 
+  Returns:
+    A `Tensor` with the same shape as `x`.
+  """
+  axis = deprecated_argument_lookup("axis", axis, "dim", dim)
+  return l2_normalize_v2(x, axis, epsilon, name)
+
+
+@tf_export("math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize", v1=[])
+def l2_normalize_v2(x, axis=None, epsilon=1e-12, name=None):
+  """Normalizes along dimension `axis` using an L2 norm.
+
+  For a 1-D tensor with `axis = 0`, computes
+
+      output = x / sqrt(max(sum(x**2), epsilon))
+
+  For `x` with more dimensions, independently normalizes each 1-D slice along
+  dimension `axis`.
+
+  Args:
+    x: A `Tensor`.
+    axis: Dimension along which to normalize.  A scalar or a vector of
+      integers.
+    epsilon: A lower bound value for the norm. Will use `sqrt(epsilon)` as the
+      divisor if `norm < sqrt(epsilon)`.
+    name: A name for this operation (optional).
+
   Returns:
     A `Tensor` with the same shape as `x`.
   """
   with ops.name_scope(name, "l2_normalize", [x]) as name:
-    axis = deprecated_argument_lookup("axis", axis, "dim", dim)
     x = ops.convert_to_tensor(x, name="x")
     square_sum = math_ops.reduce_sum(math_ops.square(x), axis, keepdims=True)
     x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon))
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index d8259aa775..a3599bfa80 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -118,7 +118,7 @@ tf_module {
   }
   member_method {
     name: "l2_normalize"
-    argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\'], "
   }
   member_method {
     name: "logdet"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index 5215cfbab0..c10361f90d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -198,7 +198,7 @@ tf_module {
   }
   member_method {
     name: "l2_normalize"
-    argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\'], "
   }
   member_method {
     name: "lbeta"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index e550b2d754..035a6c76f6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -170,7 +170,7 @@ tf_module {
   }
   member_method {
     name: "l2_normalize"
-    argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'axis\', \'epsilon\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'1e-12\', \'None\'], "
   }
   member_method {
     name: "leaky_relu"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index c7bf73ba68..a018b5713f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -114,6 +114,15 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
+        "tf.linalg.l2_normalize": {
+            "dim": "axis",
+        },
+        "tf.math.l2_normalize": {
+            "dim": "axis",
+        },
+        "tf.nn.l2_normalize": {
+            "dim": "axis",
+        },
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
-- 
GitLab


From 4b4517491d0f8a3033b11e64cb57cbb5873f638e Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 21 Nov 2018 14:11:38 -0800
Subject: [PATCH 0695/1554] Remove fixed_size_partitioner(num_shards, axis)
 from the v2 API.

PiperOrigin-RevId: 222458768
---
 tensorflow/python/ops/partitioned_variables.py  | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 tensorflow/tools/compatibility/renames_v2.py    | 1 +
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py
index 98a95f9f58..816317da10 100644
--- a/tensorflow/python/ops/partitioned_variables.py
+++ b/tensorflow/python/ops/partitioned_variables.py
@@ -218,7 +218,7 @@ def min_max_variable_partitioner(max_partitions=1, axis=0,
   return _partitioner
 
 
-@tf_export("fixed_size_partitioner")
+@tf_export(v1=["fixed_size_partitioner"])
 def fixed_size_partitioner(num_shards, axis=0):
   """Partitioner to specify a fixed number of shards along given axis.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 1b496bde47..7f69a888bf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -660,10 +660,6 @@ tf_module {
     name: "fill"
     argspec: "args=[\'dims\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "fixed_size_partitioner"
-    argspec: "args=[\'num_shards\', \'axis\'], varargs=None, keywords=None, defaults=[\'0\'], "
-  }
   member_method {
     name: "floor"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index a93b7d68bd..0c1a4890be 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -167,6 +167,7 @@ renames = {
     'tf.fft': 'tf.signal.fft',
     'tf.fft2d': 'tf.signal.fft2d',
     'tf.fft3d': 'tf.signal.fft3d',
+    'tf.fixed_size_partitioner': 'tf.compat.v1.fixed_size_partitioner',
     'tf.floordiv': 'tf.math.floordiv',
     'tf.get_collection': 'tf.compat.v1.get_collection',
     'tf.get_collection_ref': 'tf.compat.v1.get_collection_ref',
-- 
GitLab


From 8f7007823cd337ab2721219ca8eb95b150d8617d Mon Sep 17 00:00:00 2001
From: RJ Ryan <rjryan@google.com>
Date: Wed, 21 Nov 2018 14:17:51 -0800
Subject: [PATCH 0696/1554] Internal change.

PiperOrigin-RevId: 222459567
---
 tensorflow/compiler/tf2xla/python/BUILD | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/python/BUILD b/tensorflow/compiler/tf2xla/python/BUILD
index c9f486edc8..fef97b98c3 100644
--- a/tensorflow/compiler/tf2xla/python/BUILD
+++ b/tensorflow/compiler/tf2xla/python/BUILD
@@ -1,11 +1,13 @@
 licenses(["notice"])  # Apache 2.0
 
+package_group(
+    name = "friends",
+    includes = ["//tensorflow:internal"],
+)
+
 package(
     default_visibility = [
-        "//learning/deepmind/public/wavenet/python:__subpackages__",
-        "//learning/deepmind/research/alphastar:__subpackages__",
-        "//learning/tfx:__subpackages__",
-        "//tensorflow:internal",
+        ":friends",
     ],
 )
 
-- 
GitLab


From 1f9055ceb744d18059ca4cb56d0be78226763857 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 21 Nov 2018 14:21:51 -0800
Subject: [PATCH 0697/1554] [tf.data] Properly guard access to shared state
 with a mutex.

PiperOrigin-RevId: 222460077
---
 tensorflow/core/kernels/data/map_and_batch_dataset_op.cc | 5 ++++-
 tensorflow/core/kernels/data/parallel_map_iterator.cc    | 5 ++++-
 tensorflow/python/data/experimental/kernel_tests/BUILD   | 1 -
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
index 72a401e99b..f389ff12c5 100644
--- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
@@ -566,7 +566,10 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         RecordStart(ctx.get());
         auto stop_cleanup =
             gtl::MakeCleanup([this, &ctx]() { RecordStop(ctx.get()); });
-        new_calls.reserve(num_parallel_calls_->value);
+        {
+          tf_shared_lock l(*mu_);  // mu_ == num_parallel_calls_->mu
+          new_calls.reserve(num_parallel_calls_->value);
+        }
         auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
           int64 num_parallel_calls = num_parallel_calls_->value;
           int64 max_batch_results =
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index ec1c923843..5d6c12ede9 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -252,7 +252,10 @@ class ParallelMapIterator : public DatasetBaseIterator {
     RecordStart(ctx.get());
     auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
     std::vector<std::shared_ptr<InvocationResult>> new_calls;
-    new_calls.reserve(num_parallel_calls_->value);
+    {
+      tf_shared_lock l(*mu_);  // mu_ == num_parallel_calls_->mu
+      new_calls.reserve(num_parallel_calls_->value);
+    }
     auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
       int64 num_parallel_calls = num_parallel_calls_->value;
       return num_calls_ >= num_parallel_calls ||
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 6a387f55bd..a7c306817c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -620,7 +620,6 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip",
-        "notsan",
     ],
     deps = [
         ":reader_dataset_ops_test_base",
-- 
GitLab


From a7f51164e664198d4e37563b33df34ef66e8f826 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Wed, 21 Nov 2018 14:22:00 -0800
Subject: [PATCH 0698/1554] Fixed AttributeError in variable_scope.__exit__

The error was triggered by testPartitionConcatenatesAlongCorrectAxisResource
when ran w/o wrapping.

PiperOrigin-RevId: 222460094
---
 tensorflow/python/kernel_tests/variable_scope_test.py | 5 ++++-
 tensorflow/python/ops/variable_scope.py               | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index a8a66a412d..6267b01a29 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -308,7 +308,6 @@ class VariableScopeTest(test.TestCase):
       self.evaluate(variables_lib.global_variables_initializer())
       self.assertAllEqual(self.evaluate(x.value()), self.evaluate(y.value()))
 
-  # TODO(alive): support variable partitioning/caching in eager mode.
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # InvalidArgumentError: /job:moo/replica:0/task:0/device:CPU:0 unknown device.
   def testVarScopeCachingDevice(self):
@@ -1467,6 +1466,10 @@ class VariableScopeWithPartitioningTest(test.TestCase):
   def testPartitionConcatenatesAlongCorrectAxisResource(self):
     self._testPartitionConcatenatesAlongCorrectAxis(use_resource=True)
 
+  def testPartitionConcatenatesAlongCorrectAxisResourceInEager(self):
+    with context.eager_mode():
+      self._testPartitionConcatenatesAlongCorrectAxis(use_resource=True)
+
 
 class VariableScopeWithCustomGetterTest(test.TestCase):
 
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 4f210e3b12..1b122ceffd 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -2231,8 +2231,8 @@ class variable_scope(object):
 
     try:
       return self._enter_scope_uncached()
-    except:
-      if not self._building_function:
+    except Exception:
+      if self._in_graph_mode and not self._building_function:
         if self._graph_context_manager is not None:
           self._graph_context_manager.__exit__(*sys.exc_info())
       raise
-- 
GitLab


From 78ef03f348fa4005890693f51349e714d9aeef5a Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Wed, 21 Nov 2018 14:36:21 -0800
Subject: [PATCH 0699/1554] Disable flaky performance test.

PiperOrigin-RevId: 222461831
---
 tensorflow/python/keras/layers/unified_rnn_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index 744d51824b..8dc6a070bd 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -288,7 +288,7 @@ class RNNTest(test.TestCase):
                    'Normal LSTM', (total_duration / epoch))
       return total_duration / epoch
 
-  def test_performance_with_standard_cudnn_impl(self):
+  def DISABLED_test_performance_with_standard_cudnn_impl(self):
     if not test.is_gpu_available():
       self.skipTest('performance test will only run on GPU')
 
-- 
GitLab


From fa5590b4ad9353a57289e2b537417ad4eee26c74 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 21 Nov 2018 14:45:09 -0800
Subject: [PATCH 0700/1554] [tf.data] Options-related changes.

This CL:
- refactors all options classes to use a shared options utility
- introduces `tf.data.experimental.ThreadingOptions` options for threading configuration and surfaces it through `experimental_threading` of `tf.data.Options`

PiperOrigin-RevId: 222462977
---
 ...rimentalMaxIntraOpParallelismDataset.pbtxt |  13 ++
 ...ExperimentalPrivateThreadPoolDataset.pbtxt |  13 ++
 .../experimental/threadpool_dataset_op.cc     | 217 +++++++++++++++++
 .../core/ops/experimental_dataset_ops.cc      |  16 ++
 .../python/data/experimental/__init__.py      |   2 +
 .../optimization/latency_all_edges_test.py    |   3 +-
 .../kernel_tests/override_threadpool_test.py  |  87 ++++---
 .../kernel_tests/stats_dataset_ops_test.py    |  18 +-
 tensorflow/python/data/experimental/ops/BUILD |  37 +--
 .../data/experimental/ops/stats_options.py    |  78 +++----
 .../experimental/ops/threading_options.py     |  50 ++++
 .../data/kernel_tests/dataset_ops_test.py     |   2 +-
 tensorflow/python/data/ops/BUILD              |   3 +
 tensorflow/python/data/ops/dataset_ops.py     | 221 ++++++++++--------
 tensorflow/python/data/util/BUILD             |  17 ++
 tensorflow/python/data/util/options.py        | 131 +++++++++++
 tensorflow/python/data/util/options_test.py   |  96 ++++++++
 .../golden/v1/tensorflow.data.-options.pbtxt  |   5 +
 ...low.data.experimental.-stats-options.pbtxt |   3 +-
 ...data.experimental.-threading-options.pbtxt |  18 ++
 .../v1/tensorflow.data.experimental.pbtxt     |   4 +
 .../golden/v2/tensorflow.data.-options.pbtxt  |   5 +
 ...low.data.experimental.-stats-options.pbtxt |   3 +-
 ...data.experimental.-threading-options.pbtxt |  18 ++
 .../v2/tensorflow.data.experimental.pbtxt     |   4 +
 25 files changed, 865 insertions(+), 199 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalMaxIntraOpParallelismDataset.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalPrivateThreadPoolDataset.pbtxt
 create mode 100644 tensorflow/python/data/experimental/ops/threading_options.py
 create mode 100644 tensorflow/python/data/util/options.py
 create mode 100644 tensorflow/python/data/util/options_test.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalMaxIntraOpParallelismDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalMaxIntraOpParallelismDataset.pbtxt
new file mode 100644
index 0000000000..a18aa378ff
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalMaxIntraOpParallelismDataset.pbtxt
@@ -0,0 +1,13 @@
+op {
+  graph_op_name: "ExperimentalMaxIntraOpParallelismDataset"
+  in_arg {
+    name: "max_intra_op_parallelism"
+    description: <<END
+Identifies the maximum intra-op parallelism to use.
+END
+  }
+  summary: <<END
+Creates a dataset that overrides the maximum intra-op parallelism.
+END
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalPrivateThreadPoolDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalPrivateThreadPoolDataset.pbtxt
new file mode 100644
index 0000000000..eaa49b7fa5
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalPrivateThreadPoolDataset.pbtxt
@@ -0,0 +1,13 @@
+op {
+  graph_op_name: "ExperimentalPrivateThreadPoolDataset"
+  in_arg {
+    name: "num_threads"
+    description: <<END
+Identifies the number of threads to use for the private threadpool.
+END
+  }
+  summary: <<END
+Creates a dataset that uses a custom thread pool to compute `input_dataset`.
+END
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 7bd393f0f4..139642d4a3 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -13,10 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <memory>
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/util/ptr_util.h"
 #include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
@@ -225,6 +227,221 @@ class ThreadPoolDatasetOp : public UnaryDatasetOpKernel {
   };
 };
 
+class MaxIntraOpParallelismDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit MaxIntraOpParallelismDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 max_intra_op_parallelism;
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument<int64>(ctx, "max_intra_op_parallelism",
+                                              &max_intra_op_parallelism));
+    OP_REQUIRES(
+        ctx, max_intra_op_parallelism >= 0,
+        errors::InvalidArgument("`max_intra_op_parallelism` must be >= 0"));
+    *output = new Dataset(ctx, input, max_intra_op_parallelism);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input,
+            int max_intra_op_parallelism)
+        : DatasetBase(DatasetContext(ctx)),
+          input_(input),
+          max_intra_op_parallelism_(max_intra_op_parallelism) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::MaxIntraOpParallelism")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
+    }
+
+    string DebugString() const override {
+      return "MaxIntraOpParallelismDatasetOp::Dataset";
+    }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
+      Node* max_intra_op_parallelism_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddScalar(max_intra_op_parallelism_,
+                                      &max_intra_op_parallelism_node));
+      TF_RETURN_IF_ERROR(b->AddDataset(
+          this, {input_graph_node, max_intra_op_parallelism_node}, output));
+      return Status::OK();
+    }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_);
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        IteratorContext::Params params(ctx);
+        auto max_parallelism = dataset()->max_intra_op_parallelism_;
+        params.runner = std::bind(
+            [max_parallelism](
+                const std::function<void(std::function<void()>)>& runner,
+                std::function<void()> fn) {
+              std::function<void()> scoped_fn = std::bind(
+                  [max_parallelism](const std::function<void()>& fn) {
+                    ScopedPerThreadMaxParallelism scope(max_parallelism);
+                    fn();
+                  },
+                  std::move(fn));
+              (runner)(std::move(scoped_fn));
+            },
+            std::move(*ctx->runner()), std::placeholders::_1);
+        return input_impl_->GetNext(IteratorContext{std::move(params)},
+                                    out_tensors, end_of_sequence);
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
+
+     private:
+      std::unique_ptr<IteratorBase> input_impl_;
+    };
+
+    const DatasetBase* const input_;
+    const int max_intra_op_parallelism_;
+  };
+};
+
+class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit PrivateThreadPoolDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 num_threads;
+    OP_REQUIRES_OK(
+        ctx, ParseScalarArgument<int64>(ctx, "num_threads", &num_threads));
+    OP_REQUIRES(ctx, num_threads >= 1,
+                errors::InvalidArgument("`num_threads` must be >= 1"));
+    *output = new Dataset(ctx, input, num_threads);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input, int num_threads)
+        : DatasetBase(DatasetContext(ctx)),
+          input_(input),
+          num_threads_(num_threads) {
+      thread_pool_ = MakeUnique<thread::ThreadPool>(
+          ctx->env(), ThreadOptions{}, "tf_data_private_threadpool",
+          num_threads,
+          /*low_latency_hint=*/false);
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(
+          new Iterator({this, strings::StrCat(prefix, "::PrivateThreadPool")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
+    }
+
+    string DebugString() const override {
+      return "PrivateThreadPoolDatasetOp::Dataset";
+    }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
+      Node* num_threads_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddScalar(num_threads_, &num_threads_node));
+      TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output));
+      return Status::OK();
+    }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_);
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        thread::ThreadPool* pool = dataset()->thread_pool_.get();
+        IteratorContext::Params params(ctx);
+        params.runner = [pool](std::function<void()> c) {
+          pool->Schedule(std::move(c));
+        };
+        params.runner_threadpool_size = dataset()->num_threads_;
+        return input_impl_->GetNext(IteratorContext{std::move(params)},
+                                    out_tensors, end_of_sequence);
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
+
+     private:
+      std::unique_ptr<IteratorBase> input_impl_;
+    };
+
+    const DatasetBase* const input_;
+    const int64 num_threads_;
+    std::unique_ptr<thread::ThreadPool> thread_pool_;
+  };
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalMaxIntraOpParallelismDataset").Device(DEVICE_CPU),
+    MaxIntraOpParallelismDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalPrivateThreadPoolDataset").Device(DEVICE_CPU),
+    PrivateThreadPoolDatasetOp);
 REGISTER_KERNEL_BUILDER(Name("ExperimentalThreadPoolHandle").Device(DEVICE_CPU),
                         ThreadPoolHandleOp);
 REGISTER_KERNEL_BUILDER(
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 9733cf2776..e9987f568c 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -140,6 +140,22 @@ REGISTER_OP("ExperimentalFunctionBufferingResourceReset")
     .Input("function_buffer_resource: resource")
     .SetShapeFn(shape_inference::UnknownShape);
 
+REGISTER_OP("ExperimentalMaxIntraOpParallelismDataset")
+    .Input("input_dataset: variant")
+    .Input("max_intra_op_parallelism: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("ExperimentalPrivateThreadPoolDataset")
+    .Input("input_dataset: variant")
+    .Input("num_threads: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalThreadPoolDataset")
     .Input("input_dataset: variant")
     .Input("thread_pool: resource")
diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 126c2be442..12aa0a09e1 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -32,6 +32,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@StatsAggregator
 @@StatsOptions
 @@TFRecordWriter
+@@ThreadingOptions
 
 @@bucket_by_sequence_length
 @@choose_from_datasets
@@ -101,6 +102,7 @@ from tensorflow.python.data.experimental.ops.shuffle_ops import shuffle_and_repe
 from tensorflow.python.data.experimental.ops.stats_aggregator import StatsAggregator
 from tensorflow.python.data.experimental.ops.stats_ops import latency_stats
 from tensorflow.python.data.experimental.ops.stats_options import StatsOptions
+from tensorflow.python.data.experimental.ops.threading_options import ThreadingOptions
 from tensorflow.python.data.experimental.ops.unique import unique
 from tensorflow.python.data.experimental.ops.writers import TFRecordWriter
 from tensorflow.python.data.ops.iterator_ops import get_next_as_optional
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
index c2665cf69a..fc65f52704 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
@@ -60,7 +60,8 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase):
             ["LatencyStats", "Map", "LatencyStats", "Prefetch",
              "LatencyStats"])).map(lambda x: x * x).prefetch(1)
     options = dataset_ops.Options()
-    options.experimental_stats = stats_options.StatsOptions(aggregator)
+    options.experimental_stats = stats_options.StatsOptions()
+    options.experimental_stats.aggregator = aggregator
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index 5e419a9b2f..497c011b92 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -22,6 +22,7 @@ import threading
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.experimental.ops import unique
 from tensorflow.python.data.kernel_tests import test_base
@@ -35,18 +36,7 @@ from tensorflow.python.platform import test
 class OverrideThreadpoolTest(test_base.DatasetTestBase,
                              parameterized.TestCase):
 
-  @parameterized.named_parameters(
-      ("1", 1, None),
-      ("2", 2, None),
-      ("3", 4, None),
-      ("4", 8, None),
-      ("5", 16, None),
-      ("6", 4, -1),
-      ("7", 4, 0),
-      ("8", 4, 1),
-      ("9", 4, 4),
-  )
-  def testNumThreads(self, num_threads, max_intra_op_parallelism):
+  def _testNumThreadsHelper(self, num_threads, override_threadpool_fn):
 
     def get_thread_id(_):
       # Python creates a dummy thread object to represent the current
@@ -60,14 +50,7 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
         dataset_ops.Dataset.range(1000).map(
             lambda x: script_ops.py_func(get_thread_id, [x], dtypes.int64),
             num_parallel_calls=32).apply(unique.unique()))
-
-    dataset = threadpool.override_threadpool(
-        dataset,
-        threadpool.PrivateThreadPool(
-            num_threads,
-            max_intra_op_parallelism=max_intra_op_parallelism,
-            display_name="private_thread_pool_%d" % num_threads))
-
+    dataset = override_threadpool_fn(dataset)
     iterator = dataset.make_initializable_iterator()
     next_element = iterator.get_next()
 
@@ -79,12 +62,64 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
           thread_ids.append(sess.run(next_element))
       except errors.OutOfRangeError:
         pass
-      self.assertEqual(len(thread_ids), len(set(thread_ids)))
-      self.assertGreater(len(thread_ids), 0)
-      # NOTE(mrry): We don't control the thread pool scheduling, and
-      # so cannot guarantee that all of the threads in the pool will
-      # perform work.
-      self.assertLessEqual(len(thread_ids), num_threads)
+      self.assertLen(thread_ids, len(set(thread_ids)))
+      self.assertNotEmpty(thread_ids)
+      if num_threads:
+        # NOTE(mrry): We don't control the thread pool scheduling, and
+        # so cannot guarantee that all of the threads in the pool will
+        # perform work.
+        self.assertLessEqual(len(thread_ids), num_threads)
+
+  @parameterized.named_parameters(
+      ("1", 1, None),
+      ("2", 2, None),
+      ("3", 4, None),
+      ("4", 8, None),
+      ("5", 16, None),
+      ("6", 4, -1),
+      ("7", 4, 0),
+      ("8", 4, 1),
+      ("9", 4, 4),
+  )
+  def testNumThreadsDeprecated(self, num_threads, max_intra_op_parallelism):
+
+    def override_threadpool_fn(dataset):
+      return threadpool.override_threadpool(
+          dataset,
+          threadpool.PrivateThreadPool(
+              num_threads,
+              max_intra_op_parallelism=max_intra_op_parallelism,
+              display_name="private_thread_pool_%d" % num_threads))
+
+    self._testNumThreadsHelper(num_threads, override_threadpool_fn)
+
+  @parameterized.named_parameters(
+      ("1", 1, None),
+      ("2", 2, None),
+      ("3", 4, None),
+      ("4", 8, None),
+      ("5", 16, None),
+      ("6", None, 0),
+      ("7", None, 1),
+      ("8", None, 4),
+      ("9", 4, 0),
+      ("10", 4, 1),
+      ("11", 4, 4),
+      ("12", None, None),
+  )
+  def testNumThreads(self, num_threads, max_intra_op_parallelism):
+
+    def override_threadpool_fn(dataset):
+      t_options = threading_options.ThreadingOptions()
+      if max_intra_op_parallelism is not None:
+        t_options.max_intra_op_parallelism = max_intra_op_parallelism
+      if num_threads is not None:
+        t_options.private_threadpool_size = num_threads
+      options = dataset_ops.Options()
+      options.experimental_threading = t_options
+      return dataset.with_options(options)
+
+    self._testNumThreadsHelper(num_threads, override_threadpool_fn)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index 83028937d3..d5f265d8a8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -45,22 +45,18 @@ def function_set_stats_aggregator(dataset,
 
 def function_apply_options(dataset, aggregator, prefix="", counter_prefix=""):
   options = dataset_ops.Options()
-  options.experimental_stats = stats_options.StatsOptions(aggregator)
+  options.experimental_stats = stats_options.StatsOptions()
+  options.experimental_stats.aggregator = aggregator
+  options.experimental_stats.prefix = prefix
+  options.experimental_stats.counter_prefix = counter_prefix
   options.experimental_stats.latency_all_edges = False
-  if prefix:
-    options.experimental_stats.prefix = prefix
-  if counter_prefix:
-    options.experimental_stats.counter_prefix = counter_prefix
   return dataset.with_options(options)
 
 
 @parameterized.named_parameters(
-    dict(
-        testcase_name="SetStatsAggregator",
-        dataset_transformation=function_set_stats_aggregator),
-    dict(
-        testcase_name="StatsOptions",
-        dataset_transformation=function_apply_options))
+    ("SetStatsAggregator", function_set_stats_aggregator),
+    ("StatsOptions", function_apply_options),
+)
 class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
   def testBytesProduced(self, dataset_transformation):
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index b6c1376b6a..33c493bf45 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -188,6 +188,17 @@ py_library(
     ],
 )
 
+py_library(
+    name = "map_defun",
+    srcs = ["map_defun.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:tensor_shape",
+    ],
+)
+
 py_library(
     name = "optimization",
     srcs = ["optimization.py"],
@@ -217,17 +228,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "map_defun",
-    srcs = ["map_defun.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/python:dataset_ops_gen",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:tensor_shape",
-    ],
-)
-
 py_library(
     name = "resampling",
     srcs = ["resampling.py"],
@@ -303,6 +303,18 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":stats_aggregator",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/util:options",
+    ],
+)
+
+py_library(
+    name = "threading_options",
+    srcs = ["threading_options.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/util:options",
     ],
 )
 
@@ -313,9 +325,8 @@ py_library(
     deps = [
         "//tensorflow/python:experimental_dataset_ops_gen",
         "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:util",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:sparse",
         "//tensorflow/python/eager:context",
     ],
 )
diff --git a/tensorflow/python/data/experimental/ops/stats_options.py b/tensorflow/python/data/experimental/ops/stats_options.py
index c088d3d888..cd7fdcb723 100644
--- a/tensorflow/python/data/experimental/ops/stats_options.py
+++ b/tensorflow/python/data/experimental/ops/stats_options.py
@@ -20,11 +20,12 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import stats_aggregator
+from tensorflow.python.data.util import options
 from tensorflow.python.util.tf_export import tf_export
 
 
 @tf_export("data.experimental.StatsOptions")
-class StatsOptions(object):
+class StatsOptions(options.OptionsBase):
   """Represents options for collecting dataset stats using `StatsAggregator`.
 
   To apply `StatsOptions` with a `tf.data.Dataset` object, use the following
@@ -52,52 +53,29 @@ class StatsOptions(object):
   ```
   """
 
-  for _name, _ty, _default, _docstring in [
-      ("aggregator", stats_aggregator.StatsAggregator, None,
-       "Associate the given statistics options with the dataset pipeline."),
-      ("prefix", str, "",
-       "Prefix to prepend all statistics recorded for the input `dataset` with."
-      ),
-      ("counter_prefix", str, "",
-       "Prefix for the statistics recorded as counter."),
-      ("latency_all_edges", bool, True,
-       "Whether to add latency measurements on all edges."),
-  ]:
-
-    def _make_getter(name):  # pylint: disable=no-self-argument
-
-      def getter(self):
-        return getattr(self, "_" + name)
-
-      return getter
-
-    def _make_setter(name, ty):  # pylint: disable=no-self-argument
-
-      def setter(self, value):
-        if not isinstance(value, ty):
-          raise TypeError(
-              "Attempting to set the option %s to incompatible value: %r when "
-              "it expects  %r" % (name, value, ty))
-        setattr(self, "_" + name, value)
-
-      return setter
-
-    vars()["_" + _name] = _default
-    vars()[_name] = property(
-        _make_getter(_name), _make_setter(_name, _ty), _default, _docstring)
-
-  def __init__(self, aggregator=None):
-    if aggregator:
-      self.aggregator = aggregator
-
-  def __eq__(self, other):
-    if isinstance(other, self.__class__):
-      return self.__dict__ == other.__dict__
-    else:
-      return False
-
-  def __ne__(self, other):
-    return not self.__eq__(other)
-
-  def __str__(self):
-    return str(self.__dict__)
+  aggregator = options.create_option(
+      name="aggregator",
+      ty=stats_aggregator.StatsAggregator,
+      docstring=
+      "Associates the given statistics aggregator with the dataset pipeline.")
+
+  prefix = options.create_option(
+      name="prefix",
+      ty=str,
+      docstring=
+      "Prefix to prepend all statistics recorded for the input `dataset` with.",
+      default="")
+
+  counter_prefix = options.create_option(
+      name="counter_prefix",
+      ty=str,
+      docstring=
+      "Prefix for the statistics recorded as counter.",
+      default="")
+
+  latency_all_edges = options.create_option(
+      name="latency_all_edges",
+      ty=bool,
+      docstring=
+      "Whether to add latency measurements on all edges.",
+      default=True)
diff --git a/tensorflow/python/data/experimental/ops/threading_options.py b/tensorflow/python/data/experimental/ops/threading_options.py
new file mode 100644
index 0000000000..98df371c25
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/threading_options.py
@@ -0,0 +1,50 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Experimental API for controlling threading in `tf.data` pipelines."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.data.util import options
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export("data.experimental.ThreadingOptions")
+class ThreadingOptions(options.OptionsBase):
+  """Represents options for dataset threading.
+
+  To apply `ThreadingOptions` to a `dataset` object, use the following pattern:
+
+  ```python
+  options = dataset_ops.Options()
+  options.experimental_threading = tf.data.experimental.ThreadingOptions()
+  options.experimental_threading.private_threadpool_size = 10
+  dataset = dataset.with_options(options)
+  ```
+  """
+
+  max_intra_op_parallelism = options.create_option(
+      name="max_intra_op_parallelism",
+      ty=int,
+      docstring=
+      "If set, it overrides the maximum degree of intra-op parallelism.")
+
+  private_threadpool_size = options.create_option(
+      name="private_threadpool_size",
+      ty=int,
+      docstring=
+      "If set, the dataset will use a private threadpool of the given size.",
+      default=None)
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
index 3454082f96..67d3b41d3e 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
@@ -239,7 +239,7 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
     options2 = dataset_ops.Options()
     options2.experimental_autotune = False
     with self.assertRaisesRegexp(ValueError,
-                                 "Cannot merge incompatible values of option"):
+                                 "Cannot merge incompatible values"):
       dataset_ops.Dataset.range(0).with_options(options1).with_options(options2)
 
   def testOptionsMergeOptionsFromMultipleInputs(self):
diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD
index 18edc0872d..dcbb0f1868 100644
--- a/tensorflow/python/data/ops/BUILD
+++ b/tensorflow/python/data/ops/BUILD
@@ -14,6 +14,7 @@ py_library(
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:experimental_dataset_ops_gen",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:function",
         "//tensorflow/python:math_ops",
@@ -26,7 +27,9 @@ py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python/data/experimental/ops:stats_options",
+        "//tensorflow/python/data/experimental/ops:threading_options",
         "//tensorflow/python/data/util:nest",
+        "//tensorflow/python/data/util:options",
         "//tensorflow/python/data/util:random_seed",
         "//tensorflow/python/data/util:sparse",
         "//tensorflow/python/data/util:structure",
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 5c0cfe994d..d040d17322 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -27,8 +27,10 @@ import six
 
 from tensorflow.python.compat import compat
 from tensorflow.python.data.experimental.ops import stats_options
+from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import options as options_lib
 from tensorflow.python.data.util import random_seed
 from tensorflow.python.data.util import sparse
 from tensorflow.python.data.util import structure as structure_lib
@@ -45,6 +47,7 @@ from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.ops import gen_io_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import script_ops
@@ -107,6 +110,14 @@ class DatasetV2(object):
 
     dataset = self
     options = self.options()
+    if options.experimental_threading is not None:
+      t_options = options.experimental_threading
+      if t_options.private_threadpool_size is not None:
+        dataset = _PrivateThreadPoolDataset(dataset,
+                                            t_options.private_threadpool_size)
+      if t_options.max_intra_op_parallelism is not None:
+        dataset = _MaxIntraOpParallelismDataset(
+            dataset, t_options.max_intra_op_parallelism)
     static_optimizations = options._static_optimizations()  # pylint: disable=protected-access
     if static_optimizations:
       dataset = _OptimizeDataset(dataset, static_optimizations)
@@ -1371,10 +1382,9 @@ class DatasetV2(object):
   def with_options(self, options):
     """Returns a new `tf.data.Dataset` with the given options set.
 
-    The options are "global" in the sense they apply to the entire input
-    pipeline in which the `with_options` transformation is used. If options are
-    set multiple times, they are merged if possible (see
-    `tf.data.Options.merge()` for details).
+    The options are "global" in the sense they apply to the entire dataset.
+    If options are set multiple times, they are merged as long as different
+    options do not use different non-default values.
 
     Args:
       options: A `tf.data.Options` that identifies the options the use.
@@ -1383,7 +1393,7 @@ class DatasetV2(object):
       Dataset: A `Dataset` with the given options.
 
     Raises:
-      ValueError: if options are set more than once
+      ValueError: when an option is set more than once to a non-default value
     """
     return _OptionsDataset(self, options)
 
@@ -1571,7 +1581,7 @@ class DatasetV1Adapter(DatasetV1):
 
 
 @tf_export("data.Options")
-class Options(object):
+class Options(options_lib.OptionsBase):
   """Represents options for tf.data.Dataset.
 
   An `Options` object can be for instance used to control which static
@@ -1579,69 +1589,81 @@ class Options(object):
   tune the parallelism of operations such as `tf.data.Dataset.map` or
   `tf.data.Dataset.interleave`.
   """
-  for _name, _ty, _docstring in [
-      ("experimental_autotune", bool,
-       "Whether to dynamically adjust the values of tunable parameters (e.g. "
-       "degrees of parallelism)."),
-      ("experimental_deterministic", bool,
-       "Whether the outputs need to be produced in deterministic order."),
-      ("experimental_filter_fusion", bool,
-       "Whether to fuse filter transformations."),
-      ("experimental_hoist_random_uniform", bool,
-       "Whether to hoist `tf.random_uniform()` ops out of map transformations."
-      ),
-      ("experimental_stats", stats_options.StatsOptions,
-       "Associate the given statistics options with the dataset pipeline."),
-      ("experimental_map_and_batch_fusion", bool,
-       "Whether to fuse map and batch transformations."),
-      ("experimental_map_and_filter_fusion", bool,
-       "Whether to fuse map and filter transformations."),
-      ("experimental_map_fusion", bool, "Whether to fuse map transformations."),
-      ("experimental_map_parallelization", bool,
-       "Whether to parallelize stateless map transformations."),
-      ("experimental_map_vectorization", bool,
-       "Whether to vectorize map transformations."),
-      ("experimental_noop_elimination", bool,
-       "Whether to eliminate no-op transformations."),
-      ("experimental_shuffle_and_repeat_fusion", bool,
-       "Whether to fuse shuffle and repeat transformations."),
-      ("experimental_numa_aware", bool,
-       "Whether to use NUMA-aware operations."),
-  ]:
-
-    def _make_getter(name):  # pylint: disable=no-self-argument
-
-      def getter(self):
-        return getattr(self, "_" + name)
-
-      return getter
-
-    def _make_setter(name, ty):  # pylint: disable=no-self-argument
-
-      def setter(self, value):
-        if not isinstance(value, ty):
-          raise TypeError(
-              "Attempting to set the option %s to incompatible value: %r when "
-              "it expects  %r" % (name, value, ty))
-        setattr(self, "_" + name, value)
-
-      return setter
-
-    vars()["_" + _name] = None
-    vars()[_name] = property(
-        _make_getter(_name), _make_setter(_name, _ty), None, _docstring)
-
-  def __init__(self):
-    pass
-
-  def __eq__(self, other):
-    if isinstance(other, self.__class__):
-      return self.__dict__ == other.__dict__
-    else:
-      return False
 
-  def __ne__(self, other):
-    return not self.__eq__(other)
+  experimental_autotune = options_lib.create_option(
+      name="experimental_autotune",
+      ty=bool,
+      docstring=
+      "Whether to dynamically adjust the values of tunable parameters (e.g. "
+      "degrees of parallelism).")
+
+  experimental_deterministic = options_lib.create_option(
+      name="experimental_deterministic",
+      ty=bool,
+      docstring=
+      "Whether to dynamically adjust the values of tunable parameters (e.g. "
+      "degrees of parallelism).")
+
+  experimental_filter_fusion = options_lib.create_option(
+      name="experimental_filter_fusion",
+      ty=bool,
+      docstring="Whether to fuse filter transformations.")
+
+  experimental_hoist_random_uniform = options_lib.create_option(
+      name="experimental_hoist_random_uniform",
+      ty=bool,
+      docstring=
+      "Whether to hoist `tf.random_uniform()` ops out of map transformations.")
+
+  experimental_map_and_batch_fusion = options_lib.create_option(
+      name="experimental_map_and_batch_fusion",
+      ty=bool,
+      docstring="Whether to fuse map and batch transformations.")
+
+  experimental_map_and_filter_fusion = options_lib.create_option(
+      name="experimental_map_and_filter_fusion",
+      ty=bool,
+      docstring="Whether to fuse map and filter transformations.")
+
+  experimental_map_fusion = options_lib.create_option(
+      name="experimental_map_and_filter_fusion",
+      ty=bool,
+      docstring="Whether to fuse map transformations.")
+
+  experimental_map_parallelization = options_lib.create_option(
+      name="experimental_map_parallelization",
+      ty=bool,
+      docstring="Whether to parallelize stateless map transformations.")
+
+  experimental_map_vectorization = options_lib.create_option(
+      name="experimental_map_vectorization",
+      ty=bool,
+      docstring="Whether to vectorize map transformations.")
+
+  experimental_noop_elimination = options_lib.create_option(
+      name="experimental_noop_elimination",
+      ty=bool,
+      docstring="Whether to eliminate no-op transformations.")
+
+  experimental_numa_aware = options_lib.create_option(
+      name="experimental_numa_aware",
+      ty=bool,
+      docstring="Whether to use NUMA-aware operations.")
+
+  experimental_shuffle_and_repeat_fusion = options_lib.create_option(
+      name="experimental_shuffle_and_repeat_fusion",
+      ty=bool,
+      docstring="Whether to fuse shuffle and repeat transformations.")
+
+  experimental_stats = options_lib.create_option(
+      name="experimental_stats",
+      ty=stats_options.StatsOptions,
+      docstring="Associates the given statistics options with the dataset.")
+
+  experimental_threading = options_lib.create_option(
+      name="experimental_threading",
+      ty=threading_options.ThreadingOptions,
+      docstring="Associates the given threading options with the dataset.")
 
   def _static_optimizations(self):
     """Produces the list of enabled static optimizations."""
@@ -1687,32 +1709,7 @@ class Options(object):
       New `tf.data.Options()` object which is the result of merging self with
       the input `tf.data.Options`.
     """
-    result = Options()
-    for other in [self, options]:
-      for name in [
-          "experimental_autotune",
-          "experimental_deterministic",
-          "experimental_filter_fusion",
-          "experimental_hoist_random_uniform",
-          "experimental_map_and_batch_fusion",
-          "experimental_map_and_filter_fusion",
-          "experimental_map_fusion",
-          "experimental_map_parallelization",
-          "experimental_map_vectorization",
-          "experimental_noop_elimination",
-          "experimental_numa_aware",
-          "experimental_shuffle_and_repeat_fusion",
-          "experimental_stats",
-      ]:
-        this = getattr(result, name)
-        that = getattr(other, name)
-        if that is not None:
-          if this is None:
-            setattr(result, name, that)
-          elif this != that:
-            raise ValueError(
-                "Cannot merge incompatible values of option: %s" % (name))
-    return result
+    return options_lib.merge_options(self, options)
 
 
 class DatasetSource(DatasetV2):
@@ -3065,7 +3062,7 @@ class _OptimizeDataset(UnaryUnchangedStructureDataset):
 
 
 class _SetStatsAggregatorDataset(UnaryUnchangedStructureDataset):
-  """A `Dataset` that acts as an identity, and sets stats aggregator."""
+  """A `Dataset` that acts as an identity, and sets a stats aggregator."""
 
   def __init__(self, input_dataset, aggregator, prefix, counter_prefix):
     super(_SetStatsAggregatorDataset, self).__init__(input_dataset)
@@ -3081,3 +3078,37 @@ class _SetStatsAggregatorDataset(UnaryUnchangedStructureDataset):
         self._prefix,
         self._counter_prefix,
         **flat_structure(self))
+
+
+class _MaxIntraOpParallelismDataset(UnaryUnchangedStructureDataset):
+  """A `Dataset` that acts as an identity, overriding intra-op parallelism."""
+
+  def __init__(self, input_dataset, max_intra_op_parallelism):
+    super(_MaxIntraOpParallelismDataset, self).__init__(input_dataset)
+    self._input_dataset = input_dataset
+    self._max_intra_op_parallelism = ops.convert_to_tensor(
+        max_intra_op_parallelism,
+        dtype=dtypes.int64,
+        name="max_intra_op_parallelism")
+
+  def _as_variant_tensor(self):
+    return ged_ops.experimental_max_intra_op_parallelism_dataset(
+        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        self._max_intra_op_parallelism,
+        **flat_structure(self))
+
+
+class _PrivateThreadPoolDataset(UnaryUnchangedStructureDataset):
+  """A `Dataset` that acts as an identity, setting a private threadpool."""
+
+  def __init__(self, input_dataset, num_threads):
+    super(_PrivateThreadPoolDataset, self).__init__(input_dataset)
+    self._input_dataset = input_dataset
+    self._num_threads = ops.convert_to_tensor(
+        num_threads, dtype=dtypes.int64, name="num_threads")
+
+  def _as_variant_tensor(self):
+    return ged_ops.experimental_private_thread_pool_dataset(
+        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        self._num_threads,
+        **flat_structure(self))
diff --git a/tensorflow/python/data/util/BUILD b/tensorflow/python/data/util/BUILD
index 39082ce370..f15ebc32a8 100644
--- a/tensorflow/python/data/util/BUILD
+++ b/tensorflow/python/data/util/BUILD
@@ -97,6 +97,23 @@ py_test(
     ],
 )
 
+py_library(
+    name = "options",
+    srcs = ["options.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_test(
+    name = "options_test",
+    size = "small",
+    srcs = ["options_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":options",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_library(
     name = "convert",
     srcs = ["convert.py"],
diff --git a/tensorflow/python/data/util/options.py b/tensorflow/python/data/util/options.py
new file mode 100644
index 0000000000..9badba8e56
--- /dev/null
+++ b/tensorflow/python/data/util/options.py
@@ -0,0 +1,131 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for tf.data options."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+def _internal_attr_name(name):
+  return "_" + name
+
+
+class OptionsBase(object):
+  """Base class for representing a set of tf.data options.
+
+  Attributes:
+    _options: Stores the option values.
+  """
+
+  def __init__(self):
+    self._options = {}
+
+  def __eq__(self, other):
+    if not isinstance(other, self.__class__):
+      return NotImplemented
+    for name in set(self._options) | set(other._options):  # pylint: disable=protected-access
+      if getattr(self, name) != getattr(other, name):
+        return False
+    return True
+
+  def __ne__(self, other):
+    if isinstance(other, self.__class__):
+      return not self.__eq__(other)
+    else:
+      return NotImplemented
+
+
+def create_option(name, ty, docstring, default=None):
+  """Creates a type-checked property.
+
+  Args:
+    name: the name to use
+    ty: the type to use
+    docstring: the docstring to use
+    default: the default value to use
+
+  Returns:
+    A type-checked property.
+  """
+
+  def get_fn(self):
+    return self._options.get(name, default)  # pylint: disable=protected-access
+
+  def set_fn(self, value):
+    if not isinstance(value, ty):
+      raise TypeError("Property \"%s\" must be of type %s, got: %r (type: %r)" %
+                      (name, ty, value, type(value)))
+    self._options[name] = value  # pylint: disable=protected-access
+
+  return property(get_fn, set_fn, None, docstring)
+
+
+def merge_options(*options_list):
+  """Merges the given options, returning the result as a new options object.
+
+  The input arguments are expected to have a matching type that derives from
+  `OptionsBase` (and thus each represent a set of options). The method outputs
+  an object of the same type created by merging the sets of options represented
+  by the input arguments.
+
+  The sets of options can be merged as long as there does not exist an option
+  with different non-default values.
+
+  If an option is an instance of `OptionsBase` itself, then this method is
+  applied recursively to the set of options represented by this option.
+
+  Args:
+    *options_list: options to merge
+
+  Raises:
+    TypeError: if the input arguments are incompatible or not derived from
+      `OptionsBase`
+    ValueError: if the given options cannot be merged
+
+  Returns:
+    A new options object which is the result of merging the given options.
+  """
+  if len(options_list) < 1:
+    raise ValueError("At least one options should be provided")
+  result_type = type(options_list[0])
+
+  for options in options_list:
+    if not isinstance(options, result_type):
+      raise TypeError("Incompatible options type: %r vs %r" % (type(options),
+                                                               result_type))
+
+  if not isinstance(options_list[0], OptionsBase):
+    raise TypeError("The inputs should inherit from `OptionsBase`")
+
+  default_options = result_type()
+  result = result_type()
+  for options in options_list:
+    # Iterate over all set options and merge the into the result.
+    for name in options._options:  # pylint: disable=protected-access
+      this = getattr(result, name)
+      that = getattr(options, name)
+      default = getattr(default_options, name)
+      if that == default:
+        continue
+      elif this == default:
+        setattr(result, name, that)
+      elif isinstance(this, OptionsBase):
+        setattr(result, name, merge_options(this, that))
+      elif this != that:
+        raise ValueError(
+            "Cannot merge incompatible values (%r and %r) of option: %s" %
+            (this, that, name))
+  return result
diff --git a/tensorflow/python/data/util/options_test.py b/tensorflow/python/data/util/options_test.py
new file mode 100644
index 0000000000..c5169835a3
--- /dev/null
+++ b/tensorflow/python/data/util/options_test.py
@@ -0,0 +1,96 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for dataset options utilities."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.util import options
+from tensorflow.python.platform import test
+
+
+class _TestOptions(options.OptionsBase):
+  x = options.create_option(
+      name="x", ty=int, docstring="the answer to everything", default=42)
+  y = options.create_option(
+      name="y", ty=float, docstring="a tasty pie", default=3.14)
+
+
+class _NestedTestOptions(options.OptionsBase):
+  opts = options.create_option(
+      name="opts", ty=_TestOptions, docstring="nested options")
+
+
+class OptionsTest(test.TestCase):
+
+  def testDocumentation(self):
+    self.assertEqual(_TestOptions.x.__doc__, "the answer to everything")
+    self.assertEqual(_TestOptions.y.__doc__, "a tasty pie")
+
+  def testCreateOption(self):
+    opts = _TestOptions()
+    self.assertEqual(opts.x, 42)
+    self.assertEqual(opts.y, 3.14)
+    self.assertIsInstance(opts.x, int)
+    self.assertIsInstance(opts.y, float)
+    opts.x = 0
+    self.assertEqual(opts.x, 0)
+    with self.assertRaises(TypeError):
+      opts.x = 3.14
+    opts.y = 0.0
+    self.assertEqual(opts.y, 0.0)
+    with self.assertRaises(TypeError):
+      opts.y = 42
+
+  def testMergeOptions(self):
+    options1, options2 = _TestOptions(), _TestOptions()
+    with self.assertRaises(ValueError):
+      options.merge_options()
+    merged_options = options.merge_options(options1, options2)
+    self.assertEqual(merged_options.x, 42)
+    self.assertEqual(merged_options.y, 3.14)
+    options1.x = 0
+    options2.y = 0.0
+    merged_options = options.merge_options(options1, options2)
+    self.assertEqual(merged_options.x, 0)
+    self.assertEqual(merged_options.y, 0.0)
+
+  def testMergeNestedOptions(self):
+    options1, options2 = _NestedTestOptions(), _NestedTestOptions()
+    merged_options = options.merge_options(options1, options2)
+    self.assertEqual(merged_options.opts, None)
+    options1.opts = _TestOptions()
+    merged_options = options.merge_options(options1, options2)
+    self.assertEqual(merged_options.opts, _TestOptions())
+    options2.opts = _TestOptions()
+    merged_options = options.merge_options(options1, options2)
+    self.assertEqual(merged_options.opts, _TestOptions())
+    options1.opts.x = 0
+    options2.opts.y = 0.0
+    merged_options = options.merge_options(options1, options2)
+    self.assertEqual(merged_options.opts.x, 0)
+    self.assertEqual(merged_options.opts.y, 0.0)
+
+  def testMergeOptionsInvalid(self):
+    with self.assertRaises(TypeError):
+      options.merge_options(0)
+    options1, options2 = _TestOptions(), _NestedTestOptions()
+    with self.assertRaises(TypeError):
+      options.merge_options(options1, options2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt
index 9d032d43de..024b4514ba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.data.Options"
 tf_class {
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Options\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "experimental_autotune"
@@ -54,6 +55,10 @@ tf_class {
     name: "experimental_stats"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "experimental_threading"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt
index f423eed42c..892f8c1fb8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-stats-options.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.data.experimental.StatsOptions"
 tf_class {
   is_instance: "<class \'tensorflow.python.data.experimental.ops.stats_options.StatsOptions\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "aggregator"
@@ -20,6 +21,6 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'aggregator\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt
new file mode 100644
index 0000000000..5b5ebf1080
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-threading-options.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.ThreadingOptions"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.threading_options.ThreadingOptions\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "max_intra_op_parallelism"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "private_threadpool_size"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index 244b24519c..7bc3faaedc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -40,6 +40,10 @@ tf_module {
     name: "TFRecordWriter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "ThreadingOptions"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "Counter"
     argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"<dtype: \'int64\'>\"], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt
index 9d032d43de..024b4514ba 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.data.Options"
 tf_class {
   is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.Options\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "experimental_autotune"
@@ -54,6 +55,10 @@ tf_class {
     name: "experimental_stats"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "experimental_threading"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt
index f423eed42c..892f8c1fb8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-stats-options.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.data.experimental.StatsOptions"
 tf_class {
   is_instance: "<class \'tensorflow.python.data.experimental.ops.stats_options.StatsOptions\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "aggregator"
@@ -20,6 +21,6 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'aggregator\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt
new file mode 100644
index 0000000000..5b5ebf1080
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-threading-options.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.ThreadingOptions"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.threading_options.ThreadingOptions\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "max_intra_op_parallelism"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "private_threadpool_size"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index 244b24519c..7bc3faaedc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -40,6 +40,10 @@ tf_module {
     name: "TFRecordWriter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "ThreadingOptions"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "Counter"
     argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"<dtype: \'int64\'>\"], "
-- 
GitLab


From 6ce2e53499922cc919771aa18f5b3180933fa48a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 14:49:42 -0800
Subject: [PATCH 0701/1554] Clarify percentage in error message.

PiperOrigin-RevId: 222463565
---
 tensorflow/python/keras/layers/unified_rnn_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index 8dc6a070bd..e28f9625b9 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -322,7 +322,7 @@ class RNNTest(test.TestCase):
     self.assertGreaterEqual(
         cudnn_vs_unified, 0.80,
         'Expect the performance of Unified LSTM is within 80% of CuDNN LSTM, '
-        'but got {}'.format(cudnn_vs_unified * 100))
+        'but got {}%'.format(cudnn_vs_unified * 100))
     # Assert the performance diff between CPU impl and GPU impl should be more
     # than 5 times.
     self.assertGreaterEqual(
-- 
GitLab


From 285f9d87f07e1d93363418ac0da2cb320cb8916e Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 21 Nov 2018 14:50:12 -0800
Subject: [PATCH 0702/1554] Do not run Grappler const folding in conv_ops_test

PiperOrigin-RevId: 222463615
---
 tensorflow/core/kernels/BUILD            |  3 ---
 tensorflow/core/kernels/conv_ops_test.cc | 30 +++++++++++++++++++++---
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4fe0d2e87e..ae76034b0b 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1257,9 +1257,6 @@ tf_cc_test(
     name = "conv_ops_test",
     size = "medium",
     srcs = ["conv_ops_test.cc"],
-    tags = [
-        "nomsan",
-    ],
     deps = [
         ":conv_ops",
         ":image",
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 87bbc30573..252467f27c 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/protobuf/rewriter_config.pb.h"
 #include "tensorflow/core/public/session.h"
 
 namespace tensorflow {
@@ -549,8 +550,16 @@ class FusedConv2DOpTest : public OpsTestBase {
     tensorflow::GraphDef graph;
     TF_ASSERT_OK(root.ToGraphDef(&graph));
 
+    // Disable Grappler constant folding for the test graphs.
+    tensorflow::SessionOptions session_options;
+    tensorflow::RewriterConfig* cfg =
+        session_options.config.mutable_graph_options()
+            ->mutable_rewrite_options();
+    cfg->set_constant_folding(tensorflow::RewriterConfig::OFF);
+
     std::unique_ptr<tensorflow::Session> session(
-        tensorflow::NewSession(tensorflow::SessionOptions()));
+        tensorflow::NewSession(session_options));
+
     TF_ASSERT_OK(session->Create(graph));
 
     std::vector<Tensor> unfused_tensors;
@@ -704,6 +713,7 @@ class FusedConv2DOpTest : public OpsTestBase {
     const int bias_size = filter_count;
     Tensor bias(dtype, {bias_size});
     bias.flat<T>() = bias.flat<T>().setRandom();
+    bias.flat<T>() += bias.flat<T>().constant(static_cast<T>(0.5f));
 
     Tensor conv_2d;
     Tensor fused_conv_2d;
@@ -714,7 +724,14 @@ class FusedConv2DOpTest : public OpsTestBase {
     ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
     ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
 
-    test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-5);
+    // NOTE(ezhulenev): When filter size is equal to the input image size, we
+    // effectevily do element-wise product and full sum reduction, and these
+    // operations intoroduce higher than "normal" numerical errors.
+    if (image_width == filter_size && image_height == filter_size) {
+      test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-3);
+    } else {
+      test::ExpectClose(conv_2d, fused_conv_2d);
+    }
   }
 
   void VerifyFusedBatchNormTensorsNear(int depth, int image_width,
@@ -754,7 +771,14 @@ class FusedConv2DOpTest : public OpsTestBase {
     ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
     ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
 
-    test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-3);
+    // NOTE(ezhulenev): When filter size is equal to the input image size, we
+    // effectevily do element-wise product and full sum reduction, and these
+    // operations intoroduce higher than "normal" numerical errors.
+    if (image_width == filter_size && image_height == filter_size) {
+      test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-3);
+    } else {
+      test::ExpectClose(conv_2d, fused_conv_2d);
+    }
   }
 
   // Verifies that computing Conv2D+BiasAdd in a graph is identical to
-- 
GitLab


From bb76a796c35c803611600285d26c83f580616cfe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 15:00:30 -0800
Subject: [PATCH 0703/1554] Expose tf.get_logger() for v2.

PiperOrigin-RevId: 222464879
---
 tensorflow/python/platform/tf_logging.py      | 25 ++++++++++---------
 .../tools/api/golden/v1/tensorflow.pbtxt      |  4 +++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 +++
 3 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/platform/tf_logging.py b/tensorflow/python/platform/tf_logging.py
index 9f00abb201..813bcb89be 100644
--- a/tensorflow/python/platform/tf_logging.py
+++ b/tensorflow/python/platform/tf_logging.py
@@ -37,7 +37,7 @@ import six
 
 from tensorflow.python.util.tf_export import tf_export
 
-# Don't use this directly. Use _get_logger() instead.
+# Don't use this directly. Use get_logger() instead.
 _logger = None
 _logger_lock = threading.Lock()
 
@@ -78,7 +78,8 @@ else:
       return '(unknown file)', 0, '(unknown function)'
 
 
-def _get_logger():
+@tf_export('get_logger')
+def get_logger():
   """Return TF logger instance."""
   global _logger
 
@@ -132,37 +133,37 @@ def _get_logger():
 
 @tf_export(v1=['logging.log'])
 def log(level, msg, *args, **kwargs):
-  _get_logger().log(level, msg, *args, **kwargs)
+  get_logger().log(level, msg, *args, **kwargs)
 
 
 @tf_export(v1=['logging.debug'])
 def debug(msg, *args, **kwargs):
-  _get_logger().debug(msg, *args, **kwargs)
+  get_logger().debug(msg, *args, **kwargs)
 
 
 @tf_export(v1=['logging.error'])
 def error(msg, *args, **kwargs):
-  _get_logger().error(msg, *args, **kwargs)
+  get_logger().error(msg, *args, **kwargs)
 
 
 @tf_export(v1=['logging.fatal'])
 def fatal(msg, *args, **kwargs):
-  _get_logger().fatal(msg, *args, **kwargs)
+  get_logger().fatal(msg, *args, **kwargs)
 
 
 @tf_export(v1=['logging.info'])
 def info(msg, *args, **kwargs):
-  _get_logger().info(msg, *args, **kwargs)
+  get_logger().info(msg, *args, **kwargs)
 
 
 @tf_export(v1=['logging.warn'])
 def warn(msg, *args, **kwargs):
-  _get_logger().warn(msg, *args, **kwargs)
+  get_logger().warn(msg, *args, **kwargs)
 
 
 @tf_export(v1=['logging.warning'])
 def warning(msg, *args, **kwargs):
-  _get_logger().warning(msg, *args, **kwargs)
+  get_logger().warning(msg, *args, **kwargs)
 
 
 _level_names = {
@@ -196,7 +197,7 @@ def flush():
 # Code below is taken from pyglib/logging
 @tf_export(v1=['logging.vlog'])
 def vlog(level, msg, *args, **kwargs):
-  _get_logger().log(level, msg, *args, **kwargs)
+  get_logger().log(level, msg, *args, **kwargs)
 
 
 def _GetNextLogCountPerToken(token):
@@ -299,13 +300,13 @@ def google2_log_prefix(level, timestamp=None, file_and_line=None):
 @tf_export(v1=['logging.get_verbosity'])
 def get_verbosity():
   """Return how much logging output will be produced."""
-  return _get_logger().getEffectiveLevel()
+  return get_logger().getEffectiveLevel()
 
 
 @tf_export(v1=['logging.set_verbosity'])
 def set_verbosity(v):
   """Sets the threshold for what messages will be logged."""
-  _get_logger().setLevel(v)
+  get_logger().setLevel(v)
 
 
 def _get_thread_id():
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 367f506b21..290cd2d9c5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1244,6 +1244,10 @@ tf_module {
     name: "get_local_variable"
     argspec: "args=[\'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'collections\', \'caching_device\', \'partitioner\', \'validate_shape\', \'use_resource\', \'custom_getter\', \'constraint\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'None\', \'None\', \'True\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
+  member_method {
+    name: "get_logger"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_seed"
     argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 7f69a888bf..f3099f7d82 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -692,6 +692,10 @@ tf_module {
     name: "gather_nd"
     argspec: "args=[\'params\', \'indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "get_logger"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "gradients"
     argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], "
-- 
GitLab


From eebf8d9e7dfb3c0c29fdba80f7495e4f1ce0f0d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 15:05:23 -0800
Subject: [PATCH 0704/1554] [TF:XLA] Remove two more redundant const_casts.

PiperOrigin-RevId: 222465810
---
 tensorflow/compiler/xla/service/hlo_matchers.cc          | 4 ----
 tensorflow/compiler/xla/service/hlo_matchers.h           | 1 -
 tensorflow/compiler/xla/service/hlo_sharding_metadata.cc | 2 +-
 3 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_matchers.cc b/tensorflow/compiler/xla/service/hlo_matchers.cc
index 5269cad94d..d28e79d41a 100644
--- a/tensorflow/compiler/xla/service/hlo_matchers.cc
+++ b/tensorflow/compiler/xla/service/hlo_matchers.cc
@@ -237,8 +237,4 @@ void PrintTo(const HloInstruction* inst, ::std::ostream* os) {
   *os << (inst ? inst->ToString() : "nullptr");
 }
 
-void PrintTo(HloInstruction* inst, ::std::ostream* os) {
-  PrintTo(const_cast<const HloInstruction*>(inst), os);
-}
-
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h
index 170ec93a33..235efb19ce 100644
--- a/tensorflow/compiler/xla/service/hlo_matchers.h
+++ b/tensorflow/compiler/xla/service/hlo_matchers.h
@@ -385,7 +385,6 @@ std::vector<const HloInstruction*> Pointers(const Container& container) {
 // Tell GMock to print HloInstruction* by value, so error messages are nice.
 // Has to be in the same namespace as 'HloInstruction'.
 void PrintTo(const HloInstruction* inst, ::std::ostream* os);
-void PrintTo(HloInstruction* inst, ::std::ostream* os);
 
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
index 88329c8997..f506130445 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
@@ -253,7 +253,7 @@ StatusOr<bool> ApplyShardingFromUsers(HloInstruction* instruction,
       instruction->shape(), HloSharding::AssignDevice(kUnassignedDevice));
   for (HloInstruction* user : instruction->users()) {
     if (user->opcode() == HloOpcode::kDomain &&
-        domain.exit_domains.count(const_cast<HloInstruction*>(user)) > 0) {
+        domain.exit_domains.count(user) > 0) {
       // If a user is a domain and it is registered in the domain exits, then
       // the instruction sharding is taken directly from the domain, and no
       // further users need to be visited.
-- 
GitLab


From 188d909b8162248049a6c9756ca1fa646d17e183 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 21 Nov 2018 15:10:03 -0800
Subject: [PATCH 0705/1554] [tf.data] Disable TSAN testing for
 checkpoint_input_pipeline_hook_test.

It uses V1 variables, which are not data-race-free, and this leads to TSAN failures.

PiperOrigin-RevId: 222466345
---
 .../data/experimental/kernel_tests/serialization/BUILD       | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 2cfb575903..079c5309a8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -74,7 +74,10 @@ py_test(
     size = "small",
     srcs = ["checkpoint_input_pipeline_hook_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "notsan",
+    ],
     deps = [
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
-- 
GitLab


From 0cf54c920ae58aaa216a26b6e866cc67b26244e2 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Wed, 21 Nov 2018 15:14:21 -0800
Subject: [PATCH 0706/1554] Make multioutput fuison test GPU only.

PiperOrigin-RevId: 222466808
---
 tensorflow/compiler/xla/tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 0105e4a226..d92255dc48 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1892,6 +1892,7 @@ xla_test(
 xla_test(
     name = "multioutput_fusion_test",
     srcs = ["multioutput_fusion_test.cc"],
+    backends = ["gpu"],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
-- 
GitLab


From e99b69d94f8eeb74f0672dd95808d156378ad3b0 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 21 Nov 2018 15:16:04 -0800
Subject: [PATCH 0707/1554] Update the name for `tf.train.piecewise_constant`
 to `tf.train.piecewise_constant_decay` and exporting in v1.

Updating the args for `tf.strings.substr` and `tf.strings.reduce_join`.

PiperOrigin-RevId: 222467016
---
 tensorflow/python/ops/string_ops.py           | 20 +++++++++++++++++--
 .../python/training/learning_rate_decay.py    |  2 +-
 .../python/training/learning_rate_decay_v2.py |  2 +-
 .../api/golden/v1/tensorflow.train.pbtxt      |  4 ++++
 .../api/golden/v2/tensorflow.strings.pbtxt    |  4 ++--
 .../api/golden/v2/tensorflow.train.pbtxt      |  2 +-
 tensorflow/tools/compatibility/renames_v2.py  |  4 ++++
 .../tools/compatibility/tf_upgrade_v2.py      | 17 ++++++++++++++++
 8 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py
index a20eec20b8..b2090da631 100644
--- a/tensorflow/python/ops/string_ops.py
+++ b/tensorflow/python/ops/string_ops.py
@@ -311,7 +311,7 @@ def _reduce_join_reduction_dims(x, axis, reduction_indices):
     return math_ops.range(array_ops.rank(x) - 1, -1, -1)
 
 
-@tf_export("strings.reduce_join", v1=["strings.reduce_join", "reduce_join"])
+@tf_export(v1=["strings.reduce_join", "reduce_join"])
 @deprecation.deprecated_endpoints("reduce_join")
 def reduce_join(inputs, axis=None,  # pylint: disable=missing-docstring
                 keep_dims=False,
@@ -329,6 +329,17 @@ def reduce_join(inputs, axis=None,  # pylint: disable=missing-docstring
       name=name)
 
 
+@tf_export("strings.reduce_join", v1=[])
+def reduce_join_v2(  # pylint: disable=missing-docstring
+    inputs,
+    axis=None,
+    keepdims=False,
+    separator="",
+    name=None):
+  return reduce_join(
+      inputs, axis, keep_dims=keepdims, separator=separator, name=name)
+
+
 reduce_join.__doc__ = deprecation.rewrite_argument_docstring(
     gen_string_ops.reduce_join.__doc__, "reduction_indices", "axis")
 reduce_join.__doc__ = reduce_join.__doc__.replace("tf.reduce_join(",
@@ -357,11 +368,16 @@ def substr_deprecated(input, pos, len, name=None, unit="BYTE"):
 substr_deprecated.__doc__ = gen_string_ops.substr.__doc__
 
 
-@tf_export("strings.substr")
+@tf_export(v1=["strings.substr"])
 def substr(input, pos, len, name=None, unit="BYTE"):
   return gen_string_ops.substr(input, pos, len, unit=unit, name=name)
 
 
+@tf_export("strings.substr", v1=[])
+def substr_v2(input, pos, len, unit="BYTE", name=None):
+  return substr(input, pos, len, name=name, unit=unit)
+
+
 substr.__doc__ = gen_string_ops.substr.__doc__
 
 
diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py
index 29b5465321..c52e89db1f 100644
--- a/tensorflow/python/training/learning_rate_decay.py
+++ b/tensorflow/python/training/learning_rate_decay.py
@@ -100,7 +100,7 @@ def exponential_decay(learning_rate,
   return decayed_lr
 
 
-@tf_export(v1=["train.piecewise_constant"])
+@tf_export(v1=["train.piecewise_constant_decay", "train.piecewise_constant"])
 def piecewise_constant(x, boundaries, values, name=None):
   """Piecewise constant from boundaries and interval values.
 
diff --git a/tensorflow/python/training/learning_rate_decay_v2.py b/tensorflow/python/training/learning_rate_decay_v2.py
index 9c5e144be6..eb69feb17d 100644
--- a/tensorflow/python/training/learning_rate_decay_v2.py
+++ b/tensorflow/python/training/learning_rate_decay_v2.py
@@ -117,7 +117,7 @@ def exponential_decay(learning_rate,
                            decay_rate, staircase, name)
 
 
-@tf_export("train.piecewise_constant", v1=[])
+@tf_export("train.piecewise_constant_decay", v1=[])
 def piecewise_constant(x, boundaries, values, name=None):
   """Piecewise constant from boundaries and interval values.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt
index 877c55c6b3..bdb3ea2197 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.pbtxt
@@ -396,6 +396,10 @@ tf_module {
     name: "piecewise_constant"
     argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "piecewise_constant_decay"
+    argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "polynomial_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'end_learning_rate\', \'power\', \'cycle\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0001\', \'1.0\', \'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
index 16b7f14ab2..2ccb3a4594 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
@@ -14,7 +14,7 @@ tf_module {
   }
   member_method {
     name: "reduce_join"
-    argspec: "args=[\'inputs\', \'axis\', \'keep_dims\', \'separator\', \'name\', \'reduction_indices\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\', \'None\'], "
+    argspec: "args=[\'inputs\', \'axis\', \'keepdims\', \'separator\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'\', \'None\'], "
   }
   member_method {
     name: "regex_full_match"
@@ -34,7 +34,7 @@ tf_module {
   }
   member_method {
     name: "substr"
-    argspec: "args=[\'input\', \'pos\', \'len\', \'name\', \'unit\'], varargs=None, keywords=None, defaults=[\'None\', \'BYTE\'], "
+    argspec: "args=[\'input\', \'pos\', \'len\', \'unit\', \'name\'], varargs=None, keywords=None, defaults=[\'BYTE\', \'None\'], "
   }
   member_method {
     name: "to_hash_bucket"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 89d9270b27..9c221ddea3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -269,7 +269,7 @@ tf_module {
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], "
   }
   member_method {
-    name: "piecewise_constant"
+    name: "piecewise_constant_decay"
     argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 0c1a4890be..474686d612 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -559,6 +559,10 @@ renames = {
     'tf.train.maybe_batch_join': 'tf.compat.v1.train.maybe_batch_join',
     'tf.train.maybe_shuffle_batch': 'tf.compat.v1.train.maybe_shuffle_batch',
     'tf.train.maybe_shuffle_batch_join': 'tf.compat.v1.train.maybe_shuffle_batch_join',
+    'tf.train.natural_exp_decay': 'tf.compat.v1.train.natural_exp_decay',
+    'tf.train.noisy_linear_cosine_decay': 'tf.compat.v1.train.noisy_linear_cosine_decay',
+    'tf.train.piecewise_constant': 'tf.train.piecewise_constant_decay',
+    'tf.train.polynomial_decay': 'tf.compat.v1.train.polynomial_decay',
     'tf.train.queue_runner.QueueRunner': 'tf.compat.v1.train.queue_runner.QueueRunner',
     'tf.train.queue_runner.add_queue_runner': 'tf.compat.v1.train.queue_runner.add_queue_runner',
     'tf.train.queue_runner.start_queue_runners': 'tf.compat.v1.train.queue_runner.start_queue_runners',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index a018b5713f..86c6caa6fe 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -190,6 +190,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "reduction_indices": "axis",
             "keep_dims": "keepdims",
         },
+        "tf.reduce_join": {
+            "keep_dims": "keepdims",
+            "reduction_indices": "axis"
+        },
+        "tf.strings.reduce_join": {
+            "keep_dims": "keepdims",
+            "reduction_indices": "axis"
+        },
     }
 
     # Mapping from function to the new name of the function
@@ -319,6 +327,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.segment_sum": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
+        "tf.strings.substr": ["input", "pos", "len", "name", "unit"],
+        "tf.strings.reduce_join": [
+            "input", "axis", "keep_dims", "separator", "name",
+            "reduction_indices"
+        ],
         "tf.strings.length": ["input", "name", "unit"],
         "tf.transpose": ["a", "perm", "name", "conjugate"],
         "tf.tuple": ["tensors", "name", "control_inputs"],
@@ -397,6 +410,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input_tensor", "axis", "keepdims", "name", "reduction_indices",
             "keep_dims"
         ],
+        "tf.reduce_join": [
+            "input", "axis", "keep_dims", "separator", "name",
+            "reduction_indices"
+        ],
     }
 
     # Specially handled functions.
-- 
GitLab


From 5f0b5c44407e570e5b9e531c0b4a88323f833346 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 21 Nov 2018 15:26:35 -0800
Subject: [PATCH 0708/1554] Adding V2 APIs for False positives, False
 negatives, True positives, True negatives , Precision, Recall metrics.

PiperOrigin-RevId: 222468145
---
 tensorflow/python/keras/metrics.py            | 135 +++++++++++-
 tensorflow/python/ops/metrics_impl.py         |  62 +++---
 .../tensorflow.keras.metrics.-accuracy.pbtxt  | 194 ++++++++++++++++++
 ...rflow.keras.metrics.-binary-accuracy.pbtxt | 194 ++++++++++++++++++
 ....keras.metrics.-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 ...rflow.keras.metrics.-false-negatives.pbtxt | 193 +++++++++++++++++
 ...rflow.keras.metrics.-false-positives.pbtxt | 193 +++++++++++++++++
 .../v1/tensorflow.keras.metrics.-mean.pbtxt   | 192 +++++++++++++++++
 .../tensorflow.keras.metrics.-precision.pbtxt | 192 +++++++++++++++++
 .../v1/tensorflow.keras.metrics.-recall.pbtxt | 192 +++++++++++++++++
 ...metrics.-sparse-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 ...orflow.keras.metrics.-true-negatives.pbtxt | 193 +++++++++++++++++
 ...orflow.keras.metrics.-true-positives.pbtxt | 193 +++++++++++++++++
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |  44 ++++
 .../v1/tensorflow.metrics.-accuracy.pbtxt     | 194 ++++++++++++++++++
 .../tensorflow.metrics.-binary-accuracy.pbtxt | 194 ++++++++++++++++++
 ...orflow.metrics.-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 .../tensorflow.metrics.-false-negatives.pbtxt | 193 +++++++++++++++++
 .../tensorflow.metrics.-false-positives.pbtxt | 193 +++++++++++++++++
 .../golden/v1/tensorflow.metrics.-mean.pbtxt  | 192 +++++++++++++++++
 .../v1/tensorflow.metrics.-precision.pbtxt    | 192 +++++++++++++++++
 .../v1/tensorflow.metrics.-recall.pbtxt       | 192 +++++++++++++++++
 ...metrics.-sparse-categorical-accuracy.pbtxt | 194 ++++++++++++++++++
 .../tensorflow.metrics.-true-negatives.pbtxt  | 193 +++++++++++++++++
 .../tensorflow.metrics.-true-positives.pbtxt  | 193 +++++++++++++++++
 .../api/golden/v1/tensorflow.metrics.pbtxt    |  44 ++++
 ...rflow.keras.metrics.-false-negatives.pbtxt | 193 +++++++++++++++++
 ...rflow.keras.metrics.-false-positives.pbtxt | 193 +++++++++++++++++
 .../tensorflow.keras.metrics.-precision.pbtxt | 192 +++++++++++++++++
 .../v2/tensorflow.keras.metrics.-recall.pbtxt | 192 +++++++++++++++++
 ...orflow.keras.metrics.-true-negatives.pbtxt | 193 +++++++++++++++++
 ...orflow.keras.metrics.-true-positives.pbtxt | 193 +++++++++++++++++
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |  24 +++
 .../tensorflow.metrics.-false-negatives.pbtxt | 193 +++++++++++++++++
 .../tensorflow.metrics.-false-positives.pbtxt | 193 +++++++++++++++++
 .../v2/tensorflow.metrics.-precision.pbtxt    | 192 +++++++++++++++++
 .../v2/tensorflow.metrics.-recall.pbtxt       | 192 +++++++++++++++++
 .../tensorflow.metrics.-true-negatives.pbtxt  | 193 +++++++++++++++++
 .../tensorflow.metrics.-true-positives.pbtxt  | 193 +++++++++++++++++
 .../api/golden/v2/tensorflow.metrics.pbtxt    | 140 ++-----------
 40 files changed, 6851 insertions(+), 158 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index de99d47d5e..c30b953c33 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -301,7 +301,9 @@ def _update_confusion_matrix_variables(variables_to_update,
   """
   if variables_to_update is None:
     return
-  y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
+  y_true = ops.convert_to_tensor(y_true)
+  y_pred = ops.convert_to_tensor(y_pred)
+  y_pred.shape.assert_is_compatible_with(y_true.shape)
 
   if not any(
       key for key in variables_to_update if key in list(_ConfusionMatrix)):
@@ -587,7 +589,7 @@ class Metric(Layer):
   ### End: For use by subclasses ###
 
 
-@tf_export('metrics.Mean', 'keras.metrics.Mean', v1=[])
+@tf_export('metrics.Mean', 'keras.metrics.Mean')
 class Mean(Metric):
   """Computes the (weighted) mean of the given values.
 
@@ -728,7 +730,7 @@ class MeanMetricWrapper(Mean):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@tf_export('metrics.Accuracy', 'keras.metrics.Accuracy', v1=[])
+@tf_export('metrics.Accuracy', 'keras.metrics.Accuracy')
 class Accuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
@@ -770,7 +772,7 @@ class Accuracy(MeanMetricWrapper):
     return super(Accuracy, cls).from_config(config)
 
 
-@tf_export('metrics.BinaryAccuracy', 'keras.metrics.BinaryAccuracy', v1=[])
+@tf_export('metrics.BinaryAccuracy', 'keras.metrics.BinaryAccuracy')
 class BinaryAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
@@ -822,7 +824,7 @@ class BinaryAccuracy(MeanMetricWrapper):
 
 
 @tf_export(
-    'metrics.CategoricalAccuracy', 'keras.metrics.CategoricalAccuracy', v1=[])
+    'metrics.CategoricalAccuracy', 'keras.metrics.CategoricalAccuracy')
 class CategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
@@ -877,8 +879,7 @@ class CategoricalAccuracy(MeanMetricWrapper):
 
 @tf_export(
     'metrics.SparseCategoricalAccuracy',
-    'keras.metrics.SparseCategoricalAccuracy',
-    v1=[])
+    'keras.metrics.SparseCategoricalAccuracy')
 class SparseCategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches integer labels.
 
@@ -975,15 +976,35 @@ class _ConfusionMatrixConditionCount(Metric):
     return ops.convert_to_tensor(self.accumulator)
 
 
+@tf_export('metrics.FalsePositives', 'keras.metrics.FalsePositives')
 class FalsePositives(_ConfusionMatrixConditionCount):
   """Calculates the number of false positives.
 
+  For example, if `y_true` is [0, 1, 0, 0] and `y_pred` is [0, 0, 1, 1]
+  then the false positives value is 2.  If the weights were specified as
+  [0, 0, 1, 0] then the false positives value would be 1.
+
   If `sample_weight` is given, calculates the sum of the weights of
   false positives. This metric creates one local variable, `accumulator`
   that is used to keep track of the number of false positives.
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.FalsePositives()
+  m.update_state([0, 1, 0, 0], [0, 0, 1, 1])
+  print('Final result: ', m.result().numpy())  # Final result: 2
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.FalsePositives()])
+  ```
   """
 
   def __init__(self, thresholds=None, name=None, dtype=None):
@@ -1005,15 +1026,35 @@ class FalsePositives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
+@tf_export('metrics.FalseNegatives', 'keras.metrics.FalseNegatives')
 class FalseNegatives(_ConfusionMatrixConditionCount):
   """Calculates the number of false negatives.
 
+  For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [0, 1, 0, 0]
+  then the false negatives value is 2.  If the weights were specified as
+  [0, 0, 1, 0] then the false negatives value would be 1.
+
   If `sample_weight` is given, calculates the sum of the weights of
   false negatives. This metric creates one local variable, `accumulator`
   that is used to keep track of the number of false negatives.
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.FalseNegatives()
+  m.update_state([0, 1, 1, 1], [0, 1, 0, 0])
+  print('Final result: ', m.result().numpy())  # Final result: 2
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.FalseNegatives()])
+  ```
   """
 
   def __init__(self, thresholds=None, name=None, dtype=None):
@@ -1035,15 +1076,35 @@ class FalseNegatives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
+@tf_export('metrics.TrueNegatives', 'keras.metrics.TrueNegatives')
 class TrueNegatives(_ConfusionMatrixConditionCount):
   """Calculates the number of true negatives.
 
+  For example, if `y_true` is [0, 1, 0, 0] and `y_pred` is [1, 1, 0, 0]
+  then the true negatives value is 2.  If the weights were specified as
+  [0, 0, 1, 0] then the true negatives value would be 1.
+
   If `sample_weight` is given, calculates the sum of the weights of
   true negatives. This metric creates one local variable, `accumulator`
   that is used to keep track of the number of true negatives.
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.TrueNegatives()
+  m.update_state([0, 1, 0, 0], [1, 1, 0, 0])
+  print('Final result: ', m.result().numpy())  # Final result: 2
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.TrueNegatives()])
+  ```
   """
 
   def __init__(self, thresholds=None, name=None, dtype=None):
@@ -1065,15 +1126,35 @@ class TrueNegatives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
+@tf_export('metrics.TruePositives', 'keras.metrics.TruePositives')
 class TruePositives(_ConfusionMatrixConditionCount):
   """Calculates the number of true positives.
 
+  For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [1, 0, 1, 1]
+  then the true positives value is 2.  If the weights were specified as
+  [0, 0, 1, 0] then the true positives value would be 1.
+
   If `sample_weight` is given, calculates the sum of the weights of
   true positives. This metric creates one local variable, `true_positives`
   that is used to keep track of the number of true positives.
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.TruePositives()
+  m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
+  print('Final result: ', m.result().numpy())  # Final result: 2
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.TruePositives()])
+  ```
   """
 
   def __init__(self, thresholds=None, name=None, dtype=None):
@@ -1095,9 +1176,14 @@ class TruePositives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
+@tf_export('metrics.Precision', 'keras.metrics.Precision')
 class Precision(Metric):
   """Computes the precision of the predictions with respect to the labels.
 
+  For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [1, 0, 1, 1]
+  then the precision value is 2/(2+1) ie. 0.66. If the weights were specified as
+  [0, 0, 1, 0] then the precision value would be 1.
+
   The metric creates two local variables, `true_positives` and `false_positives`
   that are used to compute the precision. This value is ultimately returned as
   `precision`, an idempotent operation that simply divides `true_positives`
@@ -1105,6 +1191,21 @@ class Precision(Metric):
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.Precision()
+  m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
+  print('Final result: ', m.result().numpy())  # Final result: 0.66
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.Precision()])
+  ```
   """
 
   def __init__(self, thresholds=None, name=None, dtype=None):
@@ -1155,9 +1256,14 @@ class Precision(Metric):
         array_ops.zeros_like(self.thresholds))
 
 
+@tf_export('metrics.Recall', 'keras.metrics.Recall')
 class Recall(Metric):
   """Computes the recall of the predictions with respect to the labels.
 
+  For example, if `y_true` is [0, 1, 1, 1] and `y_pred` is [1, 0, 1, 1]
+  then the recall value is 2/(2+1) ie. 0.66. If the weights were specified as
+  [0, 0, 1, 0] then the recall value would be 1.
+
   This metric creates two local variables, `true_positives` and
   `false_negatives`, that are used to compute the recall. This value is
   ultimately returned as `recall`, an idempotent operation that simply divides
@@ -1165,6 +1271,21 @@ class Recall(Metric):
 
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.Recall()
+  m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
+  print('Final result: ', m.result().numpy())  # Final result: 0.66
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.metrics.Recall()])
+  ```
   """
 
   def __init__(self, thresholds=None, name=None, dtype=None):
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 27269c51c1..cb42199011 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -625,7 +625,7 @@ def _aggregate_variable(v, collections):
   return _aggregate_across_replicas(collections, f, v)
 
 
-@tf_export('metrics.auc')
+@tf_export(v1=['metrics.auc'])
 def auc(labels,
         predictions,
         weights=None,
@@ -830,7 +830,7 @@ def auc(labels,
     return auc_value, update_op
 
 
-@tf_export('metrics.mean_absolute_error')
+@tf_export(v1=['metrics.mean_absolute_error'])
 def mean_absolute_error(labels,
                         predictions,
                         weights=None,
@@ -891,7 +891,7 @@ def mean_absolute_error(labels,
               updates_collections, name or 'mean_absolute_error')
 
 
-@tf_export('metrics.mean_cosine_distance')
+@tf_export(v1=['metrics.mean_cosine_distance'])
 def mean_cosine_distance(labels,
                          predictions,
                          dim,
@@ -965,7 +965,7 @@ def mean_cosine_distance(labels,
   return mean_distance, update_op
 
 
-@tf_export('metrics.mean_per_class_accuracy')
+@tf_export(v1=['metrics.mean_per_class_accuracy'])
 def mean_per_class_accuracy(labels,
                             predictions,
                             num_classes,
@@ -1069,7 +1069,7 @@ def mean_per_class_accuracy(labels,
     return mean_accuracy_v, update_op
 
 
-@tf_export('metrics.mean_iou')
+@tf_export(v1=['metrics.mean_iou'])
 def mean_iou(labels,
              predictions,
              num_classes,
@@ -1170,7 +1170,7 @@ def mean_iou(labels,
     return mean_iou_v, update_op
 
 
-@tf_export('metrics.mean_relative_error')
+@tf_export(v1=['metrics.mean_relative_error'])
 def mean_relative_error(labels,
                         predictions,
                         normalizer,
@@ -1239,7 +1239,7 @@ def mean_relative_error(labels,
               updates_collections, name or 'mean_relative_error')
 
 
-@tf_export('metrics.mean_squared_error')
+@tf_export(v1=['metrics.mean_squared_error'])
 def mean_squared_error(labels,
                        predictions,
                        weights=None,
@@ -1300,7 +1300,7 @@ def mean_squared_error(labels,
               name or 'mean_squared_error')
 
 
-@tf_export('metrics.mean_tensor')
+@tf_export(v1=['metrics.mean_tensor'])
 def mean_tensor(values,
                 weights=None,
                 metrics_collections=None,
@@ -1385,7 +1385,7 @@ def mean_tensor(values,
     return mean_t, update_op
 
 
-@tf_export('metrics.percentage_below')
+@tf_export(v1=['metrics.percentage_below'])
 def percentage_below(values,
                      threshold,
                      weights=None,
@@ -1485,7 +1485,7 @@ def _count_condition(values,
   return value_tensor, update_op
 
 
-@tf_export('metrics.false_negatives')
+@tf_export(v1=['metrics.false_negatives'])
 def false_negatives(labels,
                     predictions,
                     weights=None,
@@ -1537,7 +1537,7 @@ def false_negatives(labels,
                             updates_collections)
 
 
-@tf_export('metrics.false_negatives_at_thresholds')
+@tf_export(v1=['metrics.false_negatives_at_thresholds'])
 def false_negatives_at_thresholds(labels,
                                   predictions,
                                   thresholds,
@@ -1593,7 +1593,7 @@ def false_negatives_at_thresholds(labels,
     return fn_value, update_ops['fn']
 
 
-@tf_export('metrics.false_positives')
+@tf_export(v1=['metrics.false_positives'])
 def false_positives(labels,
                     predictions,
                     weights=None,
@@ -1646,7 +1646,7 @@ def false_positives(labels,
                             updates_collections)
 
 
-@tf_export('metrics.false_positives_at_thresholds')
+@tf_export(v1=['metrics.false_positives_at_thresholds'])
 def false_positives_at_thresholds(labels,
                                   predictions,
                                   thresholds,
@@ -1702,7 +1702,7 @@ def false_positives_at_thresholds(labels,
     return fp_value, update_ops['fp']
 
 
-@tf_export('metrics.true_negatives')
+@tf_export(v1=['metrics.true_negatives'])
 def true_negatives(labels,
                    predictions,
                    weights=None,
@@ -1755,7 +1755,7 @@ def true_negatives(labels,
                             updates_collections)
 
 
-@tf_export('metrics.true_negatives_at_thresholds')
+@tf_export(v1=['metrics.true_negatives_at_thresholds'])
 def true_negatives_at_thresholds(labels,
                                  predictions,
                                  thresholds,
@@ -1811,7 +1811,7 @@ def true_negatives_at_thresholds(labels,
     return tn_value, update_ops['tn']
 
 
-@tf_export('metrics.true_positives')
+@tf_export(v1=['metrics.true_positives'])
 def true_positives(labels,
                    predictions,
                    weights=None,
@@ -1864,7 +1864,7 @@ def true_positives(labels,
                             updates_collections)
 
 
-@tf_export('metrics.true_positives_at_thresholds')
+@tf_export(v1=['metrics.true_positives_at_thresholds'])
 def true_positives_at_thresholds(labels,
                                  predictions,
                                  thresholds,
@@ -1920,7 +1920,7 @@ def true_positives_at_thresholds(labels,
     return tp_value, update_ops['tp']
 
 
-@tf_export('metrics.precision')
+@tf_export(v1=['metrics.precision'])
 def precision(labels,
               predictions,
               weights=None,
@@ -2015,7 +2015,7 @@ def precision(labels,
     return p, update_op
 
 
-@tf_export('metrics.precision_at_thresholds')
+@tf_export(v1=['metrics.precision_at_thresholds'])
 def precision_at_thresholds(labels,
                             predictions,
                             thresholds,
@@ -2096,7 +2096,7 @@ def precision_at_thresholds(labels,
     return prec, update_op
 
 
-@tf_export('metrics.recall')
+@tf_export(v1=['metrics.recall'])
 def recall(labels,
            predictions,
            weights=None,
@@ -2447,7 +2447,7 @@ def _streaming_sparse_false_negative_at_k(labels,
     return var, state_ops.assign_add(var, batch_total_fn, name='update')
 
 
-@tf_export('metrics.recall_at_k')
+@tf_export(v1=['metrics.recall_at_k'])
 def recall_at_k(labels,
                 predictions,
                 k,
@@ -2540,7 +2540,7 @@ def recall_at_k(labels,
         name=scope)
 
 
-@tf_export('metrics.recall_at_top_k')
+@tf_export(v1=['metrics.recall_at_top_k'])
 def recall_at_top_k(labels,
                     predictions_idx,
                     k=None,
@@ -2624,7 +2624,7 @@ def recall_at_top_k(labels,
     return metric, update
 
 
-@tf_export('metrics.recall_at_thresholds')
+@tf_export(v1=['metrics.recall_at_thresholds'])
 def recall_at_thresholds(labels,
                          predictions,
                          thresholds,
@@ -2702,7 +2702,7 @@ def recall_at_thresholds(labels,
     return rec, update_op
 
 
-@tf_export('metrics.root_mean_squared_error')
+@tf_export(v1=['metrics.root_mean_squared_error'])
 def root_mean_squared_error(labels,
                             predictions,
                             weights=None,
@@ -2773,7 +2773,7 @@ def root_mean_squared_error(labels,
   return rmse, update_rmse_op
 
 
-@tf_export('metrics.sensitivity_at_specificity')
+@tf_export(v1=['metrics.sensitivity_at_specificity'])
 def sensitivity_at_specificity(labels,
                                predictions,
                                specificity,
@@ -3146,7 +3146,7 @@ def _streaming_sparse_average_precision_at_top_k(labels,
     return mean_average_precision, update
 
 
-@tf_export('metrics.sparse_average_precision_at_k')
+@tf_export(v1=['metrics.sparse_average_precision_at_k'])
 @deprecated(None, 'Use average_precision_at_k instead')
 def sparse_average_precision_at_k(labels,
                                   predictions,
@@ -3166,7 +3166,7 @@ def sparse_average_precision_at_k(labels,
       name=name)
 
 
-@tf_export('metrics.average_precision_at_k')
+@tf_export(v1=['metrics.average_precision_at_k'])
 def average_precision_at_k(labels,
                            predictions,
                            k,
@@ -3340,7 +3340,7 @@ def _streaming_sparse_false_positive_at_k(labels,
     return var, state_ops.assign_add(var, batch_total_fp, name='update')
 
 
-@tf_export('metrics.precision_at_top_k')
+@tf_export(v1=['metrics.precision_at_top_k'])
 def precision_at_top_k(labels,
                        predictions_idx,
                        k=None,
@@ -3429,7 +3429,7 @@ def precision_at_top_k(labels,
     return metric, update
 
 
-@tf_export('metrics.sparse_precision_at_k')
+@tf_export(v1=['metrics.sparse_precision_at_k'])
 @deprecated(None, 'Use precision_at_k instead')
 def sparse_precision_at_k(labels,
                           predictions,
@@ -3451,7 +3451,7 @@ def sparse_precision_at_k(labels,
       name=name)
 
 
-@tf_export('metrics.precision_at_k')
+@tf_export(v1=['metrics.precision_at_k'])
 def precision_at_k(labels,
                    predictions,
                    k,
@@ -3545,7 +3545,7 @@ def precision_at_k(labels,
         name=scope)
 
 
-@tf_export('metrics.specificity_at_sensitivity')
+@tf_export(v1=['metrics.specificity_at_sensitivity'])
 def specificity_at_sensitivity(labels,
                                predictions,
                                sensitivity,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
new file mode 100644
index 0000000000..2db07df523
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.Accuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
new file mode 100644
index 0000000000..904ad3a21a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.BinaryAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..17b74924fa
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.CategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
new file mode 100644
index 0000000000..49f577e136
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.FalseNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
new file mode 100644
index 0000000000..e8baf85866
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.FalsePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
new file mode 100644
index 0000000000..40fe64bbd2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.keras.metrics.Mean"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
new file mode 100644
index 0000000000..ae6a85026d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.keras.metrics.Precision"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
new file mode 100644
index 0000000000..31068a51d5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.keras.metrics.Recall"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..0c17452292
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.keras.metrics.SparseCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
new file mode 100644
index 0000000000..1b5eb8d0de
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.TrueNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
new file mode 100644
index 0000000000..5b9c470e32
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.TruePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index a296e13158..cc90d0e709 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -1,5 +1,49 @@
 path: "tensorflow.keras.metrics"
 tf_module {
+  member {
+    name: "Accuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BinaryAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalseNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalsePositives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Mean"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Precision"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Recall"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TrueNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TruePositives"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "KLD"
     argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt
new file mode 100644
index 0000000000..f8e12f8817
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.Accuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt
new file mode 100644
index 0000000000..b9bc6a716a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.BinaryAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..0ef75d8756
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.CategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt
new file mode 100644
index 0000000000..33226a2df6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.FalseNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt
new file mode 100644
index 0000000000..9953162ea3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.FalsePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt
new file mode 100644
index 0000000000..7fe6d6fda9
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.metrics.Mean"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt
new file mode 100644
index 0000000000..8c3271a109
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.metrics.Precision"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt
new file mode 100644
index 0000000000..840a68bbc7
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.metrics.Recall"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
new file mode 100644
index 0000000000..7bce43fbde
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
@@ -0,0 +1,194 @@
+path: "tensorflow.metrics.SparseCategoricalAccuracy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt
new file mode 100644
index 0000000000..83cd5b736b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.TrueNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt
new file mode 100644
index 0000000000..5b2502eafe
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.TruePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt
index e9b996c9f5..f5c267a166 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt
@@ -1,5 +1,49 @@
 path: "tensorflow.metrics"
 tf_module {
+  member {
+    name: "Accuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BinaryAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalseNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalsePositives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Mean"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Precision"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Recall"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SparseCategoricalAccuracy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TrueNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TruePositives"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "accuracy"
     argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
new file mode 100644
index 0000000000..49f577e136
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.FalseNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
new file mode 100644
index 0000000000..e8baf85866
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.FalsePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
new file mode 100644
index 0000000000..ae6a85026d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.keras.metrics.Precision"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
new file mode 100644
index 0000000000..31068a51d5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.keras.metrics.Recall"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
new file mode 100644
index 0000000000..1b5eb8d0de
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.TrueNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
new file mode 100644
index 0000000000..5b9c470e32
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.keras.metrics.TruePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 8a8fb97b96..cc90d0e709 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -12,14 +12,38 @@ tf_module {
     name: "CategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "FalseNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "FalsePositives"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Mean"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Precision"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Recall"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TrueNegatives"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "TruePositives"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "KLD"
     argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
new file mode 100644
index 0000000000..33226a2df6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.FalseNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
new file mode 100644
index 0000000000..9953162ea3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.FalsePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
new file mode 100644
index 0000000000..8c3271a109
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.metrics.Precision"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
new file mode 100644
index 0000000000..840a68bbc7
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
@@ -0,0 +1,192 @@
+path: "tensorflow.metrics.Recall"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
new file mode 100644
index 0000000000..83cd5b736b
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.TrueNegatives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
new file mode 100644
index 0000000000..5b2502eafe
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
@@ -0,0 +1,193 @@
+path: "tensorflow.metrics.TruePositives"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset_states"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "result"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_state"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
index d82ce8b38a..773efd03fc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
@@ -13,135 +13,35 @@ tf_module {
     mtype: "<type \'type\'>"
   }
   member {
-    name: "Mean"
+    name: "FalseNegatives"
     mtype: "<type \'type\'>"
   }
   member {
-    name: "SparseCategoricalAccuracy"
+    name: "FalsePositives"
     mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "auc"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'num_thresholds\', \'metrics_collections\', \'updates_collections\', \'curve\', \'name\', \'summation_method\'], varargs=None, keywords=None, defaults=[\'None\', \'200\', \'None\', \'None\', \'ROC\', \'None\', \'trapezoidal\'], "
-  }
-  member_method {
-    name: "average_precision_at_k"
-    argspec: "args=[\'labels\', \'predictions\', \'k\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "false_negatives"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "false_negatives_at_thresholds"
-    argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "false_positives"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "false_positives_at_thresholds"
-    argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_absolute_error"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_cosine_distance"
-    argspec: "args=[\'labels\', \'predictions\', \'dim\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_iou"
-    argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_per_class_accuracy"
-    argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_relative_error"
-    argspec: "args=[\'labels\', \'predictions\', \'normalizer\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_squared_error"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "mean_tensor"
-    argspec: "args=[\'values\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "percentage_below"
-    argspec: "args=[\'values\', \'threshold\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "precision"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "precision_at_k"
-    argspec: "args=[\'labels\', \'predictions\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "precision_at_thresholds"
-    argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "precision_at_top_k"
-    argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "recall"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "recall_at_k"
-    argspec: "args=[\'labels\', \'predictions\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "recall_at_thresholds"
-    argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "recall_at_top_k"
-    argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "root_mean_squared_error"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sensitivity_at_specificity"
-    argspec: "args=[\'labels\', \'predictions\', \'specificity\', \'weights\', \'num_thresholds\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'200\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sparse_average_precision_at_k"
-    argspec: "args=[\'labels\', \'predictions\', \'k\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "sparse_precision_at_k"
-    argspec: "args=[\'labels\', \'predictions\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "Mean"
+    mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "specificity_at_sensitivity"
-    argspec: "args=[\'labels\', \'predictions\', \'sensitivity\', \'weights\', \'num_thresholds\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'200\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "Precision"
+    mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "true_negatives"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "Recall"
+    mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "true_negatives_at_thresholds"
-    argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "SparseCategoricalAccuracy"
+    mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "true_positives"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "TrueNegatives"
+    mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "true_positives_at_thresholds"
-    argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  member {
+    name: "TruePositives"
+    mtype: "<type \'type\'>"
   }
 }
-- 
GitLab


From c9774910f1e24b964f0a31fdbf98df472d92501b Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Wed, 21 Nov 2018 15:48:01 -0800
Subject: [PATCH 0709/1554] Merge with upstream again

---
 tensorflow/contrib/tensorrt/BUILD             |  22 ------------------
 .../contrib/tensorrt/convert/convert_graph.cc |   1 -
 .../contrib/tensorrt/convert/convert_graph.h  |   3 ---
 .../contrib/tensorrt/python/trt_convert.py    |   2 +-
 tensorflow/contrib/tensorrt/segment/segment.h |   2 --
 .../test/quantization_mnist_test_data/BUILD   |  14 -----------
 .../quantization_mnist_test_data/checkpoint   |   2 --
 .../model.ckpt-14070.data-00000-of-00001      | Bin 686728 -> 0 bytes
 .../model.ckpt-14070.index                    | Bin 961 -> 0 bytes
 .../grappler/optimizers/layout_optimizer.cc   |   1 -
 10 files changed, 1 insertion(+), 46 deletions(-)
 delete mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD
 delete mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint
 delete mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
 delete mode 100644 tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index d304d72c6a..784acce444 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -513,28 +513,6 @@ cuda_py_tests(
     ],
 )
 
-cuda_py_tests(
-    name = "tf_trt_quantization_mnist_integration_test",
-    srcs = [
-        "test/quantization_mnist_test.py",
-    ],
-    additional_deps = [
-        ":tf_trt_integration_test_base",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python/keras:keras",
-        "//tensorflow/python/estimator:estimator",
-    ],
-    data = [
-        "//tensorflow/contrib/tensorrt/test/quantization_mnist_test_data:quantization_mnist_data",
-    ],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-)
-
 cuda_py_tests(
     name = "tf_trt_integration_test_no_oss",
     srcs = [
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index f6d44cb719..21f505b7fe 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -884,7 +884,6 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     segment_options.exclude_node_list.insert(node);
   }
   segment_options.minimum_segment_size = params.minimum_segment_size;
-  segment_options.precision_mode = params.precision_mode;
   tensorflow::tensorrt::segment::SegmentNodesVector initial_segments;
   TrtCandidateSelector candidate_selector(*params.graph_properties,
                                           params.precision_mode);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 2904e73abc..1f39f56f63 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -47,9 +47,6 @@ class TrtCandidateSelector {
   // supported. It will operate in validation-only mode.
   TrtNodeValidator validator_;
 
-  // Quantization ops are only converted when using quantized precisions.
-  int precision_mode_;
-
   // GraphProperties of the graph whose nodes are to be validated by
   // IsTensorRTCandidate().
   const grappler::GraphProperties& graph_properties_;
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index f0945087d9..203b2697ba 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -181,7 +181,7 @@ def create_inference_graph(input_graph_def,
     max_workspace_size_bytes: the maximum GPU temporary memory which the TRT
       engine can use at execution time. This corresponds to the 'workspaceSize'
       parameter of nvinfer1::IBuilder::setMaxWorkspaceSize().
-    precision_mode: one of TrtPrecisionMode.supported_precision_modes(). 
+    precision_mode: one of TrtPrecisionMode.supported_precision_modes().
     minimum_segment_size: the minimum number of nodes required for a subgraph to
       be replaced by TRTEngineOp.
     is_dynamic_op: whether to generate dynamic TRT ops which will build the TRT
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index b6ae1cf462..b9693aad1b 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -38,8 +38,6 @@ struct SegmentOptions {
   // Segment must contain at least this many nodes.
   int minimum_segment_size = 2;
   std::set<string> exclude_node_list;
-  // Quantization nodes are only included for quantized precisions
-  int precision_mode;
 };
 
 // Get the subgraphs of a graph that can be handled by TensorRT.
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD
deleted file mode 100644
index f38c2c5f4d..0000000000
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/BUILD
+++ /dev/null
@@ -1,14 +0,0 @@
-# Checkpoint files for quantization MNIST test.
-
-package(
-    default_visibility = ["//visibility:public"],
-)
-
-licenses(["notice"])  # Apache 2.0
-
-filegroup(
-    name = "quantization_mnist_data",
-    srcs = glob(
-        ["**"],
-    ),
-)
\ No newline at end of file
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint
deleted file mode 100644
index 4e69206e97..0000000000
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/checkpoint
+++ /dev/null
@@ -1,2 +0,0 @@
-model_checkpoint_path: "model.ckpt-14070"
-all_model_checkpoint_paths: "model.ckpt-14070"
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001 b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.data-00000-of-00001
deleted file mode 100644
index c1876b7a4c53a13696bd5a5aabce09a7455e36d7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 686728
zcmZQzU|`^_^D{jf6TV;U<%Iofp02W$`LSew<Mz1y_ZY72uX3KVpCib2KTph~edpe`
z?EmUAbH8=E>i#`_-TPUe&)okd-DKYq=B4{}W#;VvE#bZYl+pbC5;OhwC-nR87c2_g
z>-*GW|Ev_o1BJ5o`wxU2*ng<1b-zk&!TyqSbvDlJllMz~Zr^`%=g$4p9V@JF-oCuk
z_ebq6nbUH+G8VkCIyN=K`uNP%yB~Gl*md#YbF0(^+E!9Gy?3Q>S6DthSF|(a$)fE&
z=eO_NYWdruIV005>srsQ{6ivE%rDb+oo4Rc_4jd+wYNrqb(2Z_Zk5WTR+^#LcO@<m
zv<fou+{ty8!TO|bi-pW9)?FX;V|V{CS!20sB8R1`h`E)_v~?B+jwV*Z8#Y*VUlz8y
zo_XCeakhq4#Zhj{6W1PE$XK*l2L5|z@hG6z;&ZT@mA<^FWspLzMXRZu)gJZz7E^BW
zS+Y%fW+`KvVwF5i((2b`4NLEX@fOv>qLwDhR$BfF6SMjnC1I5_LDJG|_IC@>iQg?^
z*T1uUE|9$cke;#qg-i47xilv1Z^~!i-z@mT?$N)g`>v#C+OJ%}y<hd+&wbl1LhU0f
z%<NyC`e$EtirIeGBmI4>ma2A2*}M*0=9lba*m%s|ZBCJWU%^y+B_q52+fTi;_mZ};
z{rm68{+?@b`;B&+*>9Cg+^eLZvHylxn|<k<TANQ3=GgNnY46QnSY=z0Xl3_5=;FSb
z0I&V~KgrrBKY6#u@%<w^oqH1YyB`VcyM2-0zRcs@UIWqi{hLf3_kGzgY5&f`2K&o)
zzwKJ?@Y=t&5Vf6D@ND1NJ;wIQkCyDyF50}W<B-0Ma^jVJGgi#r*BCa#-o5Lkt=h&1
zHs@Bw+6nh8w7L6K;DEv7WA@F_Vh%prt@dxZq_OYjg0%f9(lPdiGC%B-Y902Sda~G_
zck=H2vv)S^PZK(1zcX&tz5@<Mb}3gF_jlb8w|BfOv`_cPDLbdLZFU=#UH31rXt7%p
zV!YoV?AiV`ZyNTQUuxKY<D#+cZ076xqf12h>r`E~JKWv6=l_bS`y*C(+g&!hweOnc
zyM3D#`t6yga_^4_RIs-wI<hbQ!67@AgyVMebJpxjzO-v!>B1NL_T`<ozoh!y?hYsC
zzB$HR4$k-b_swB5v+G=YX|Gl{k6nqnhh45@-Tq6{mhMaH^|gQV;=tb9?OFDbKOXF>
z*s^E8YkA+^2~&^l^IOWepO4AdUVF;|`<Q@7c7^Sy_s_q>YtInFXE(o3&(1~QrJbAG
zg?%$0uid+?hIPNV&ICJ)BPIK^rZw)paja*5gg*O$FFl*=yXMTbX*k-rFSpsu&gZ)2
ze&$Zb{lR{X`)kGYZSMwj?G@adyZ_oB9s6xRW9&7a+_#^xICj6<4H^5SXD@8eZSCLt
zCM(?9!ZgtSx~jvz!wlc{wd6dolWBOcZ*AXGyKK)UyLttc{T}-wZ67TXu@^B9*}K-o
z*lyzrW!rsc6z$~sn(a;U|L#jKUa)`qxs|r(JLUFnKX0=?{!{G!sU;fwzP?hio7Mh%
zpJ7;`eQ?$TJAJFyHo^{P_j~)VvAe${c;B?2PJ7=nDDE$*IAL#fWb(dKo)`8D$2#mc
zYx-fkV3Nar+l9GyX7w}e9c#Jm?4NksYfa|b@3*#QpRpT@{U0ZN2UCu_`}=n#+rPPU
zeQ)T(xBK#9?e=R6Ub6SHxMjnTGu19_L#nO!?2opmL^JLAxbE9u>vi7WG4G$<d#!`}
z9;?OgUvkOZeuKb;eWzVb>@=S-?o<EZZnxJ)+fMWN(!G(BboS5xSg^mwp<v%#=A?b;
zHBW4uga!6*Yb@R`JL9~abzg=ZU%`a^#z8aewVwRg=h(S)AODKneZpR|_bI)QwGV$(
zVK+Bb&EBPtd;c3j-u+9rU)sB}vU$(1ojvx=hx+zS)o9vR)F*GZGStX!_4YaY4g0I@
z9iCscKYZ-D{R_sG`=_4B*uR8v%6|SL=l$tlp6u?Ey=$lbaHoC#*M<A+#iZ@?SiabP
zc9!3#So(NhKiBI0*>5%XZ_rq58@74l-qT|7_R)-A?X2aW+RyVmU~iIq&~D1cIs30A
z{<70-yk~!;eVyGRt1bJc6*=2C2QgY(F)>?dv|H>x;M1|If3f=R3eNJKJ(beCd3Zdn
zR?azZm3jG$MVN%}&X#1xU7^=icc~rl-W6!svr|IJVuySDDa%`L!gqvdOYQu0qkY%R
z%TKI~7HjVIvA5c-`ts1OFR#|_I<;h;)zz?XR(oxltxU@`c1s>{vU*)`d)I~)dv|S{
z_GqV^ikIc}-rKv(f2`PbTa#_~;sq?b1?GD1Zei2g;n?DDJ>eC%RiD6VOMM>IUE2z0
z?&7`CwQG~{ja^r+*z9CHXKvN5xM3I5Z{3~jmQ#0KeCTPtj>&81EFp{C+n!XIcZZ(d
zWzOJXwO*mj+I8yYU7b%NcE2jCvzk#Yw5#D<yjAXjm%Es!6z%+QV)3r!4o<t??$p^;
zbJbz@MH9!}AJ(Yt`hG6aI{npj%XNK8R>~1NmQN-A?wZM*v5V&@pJm+sojX4ZUfj9o
z_`Mx7S>ty-VHevq*;dV(?}WZ}TByfv{k;#Z{z#m$klf>Gr8cM8x=KuG*LFv>T@fP6
zJ2}?gw%n=`V14@d9;@I21*>lTQ@d_R<nQ8``f8V0`te;q({5XcP2;luspVq5`(U4C
zS51iJx=_Pirym@)Vhpw4_4*m(E)^e{U6W5|TP=R1VEu&W!Hy8#f}J%#Pww2{*09^G
zWd80g)o*r1_b#=3yNqjB@mnD)zqO)PcZ#)lhn8;M=^d`Kt1C=>cd`6cD;5zctALud
zyXxhwcWvc=v@;;kW%ulxi+BAm6ttSnkZkR;DZ-L9Z@FdVt(oQl$39tJHQv3Wi%nve
zHoK`+);$NSxDClxwt`Z-Sf^~ZxNv`|)sbB)Ryz~>cZaR$-j#Eybl2=>6+36LJ6q@X
z&)<1}&(&QkSsg9Kf6m*v{k+buPxm9Oj&t3yJS(kgb=xy>mu8sD&eNB9b{jO!+O^rC
z*J@?4l(ln}f<=Vm2TPyExmM{*b$12rx8JE)I?HOJ?HsFH6ZY>&KKjOtiD~_=I}`FO
zO=nA5AIK=T7EQ0%eIaI-<+kVvJGt|}ncpf<wSK5_$m;pZ4OR@%>^qxt-L2-l<g;#)
zZQ12MQPc8|*z8@mrgd1>OxE6&SFE&KTQ=O<ZoaW~|4VD@?Ukh#zovb$ER^Tn<+xVD
zx_OJ!t}Q%<yS%n>?No}lwEDfh$J&v9%}%zZT2?2@j_*1>XVPw`-*b0YUbEWmD=KDn
zRqV+w-L7T3>@=S3=+RKvU9xfcu5FzBR=E!ZcQ3dmWK~dhc1Nai(yrUPEv)+9Yg>M<
zoUr@*MHXwTGYwWVxIL`TNwIE^<ImpZrEF$-NPD?ep3g1|yR8np=G9HN+Oj6nGViRE
z^~GN-yRK&zSta-Fws^B;rq%lP-ragD40r23ZQmu&|7h3Ga~+l)-i4NrxDM^oPW`ac
zBD26k%lPuHgOk}U*c)zJ?udS6#r1WLrKT98<pG|E-FmOMtzXHhTTjilwytm$vOM<K
zXxEu`3#+fc^{pyXwp;$1n6YzT#38Hoo=rQKE_rU*?<loPy`s<RRsK1vFSCoSmEP6v
zRyftP>%M37uJy}r>{Mv--6iPUWu;}Y){40x$<p_OsnwcKCoP{>a$D9W_gEP9U9)7Y
zwX$&Vd2V@ZzM@r7!&M8{D~B!IFK@T(58<?woEu}YlH-s?lj1&$wJg#Wymu~I7JWWs
zp%g4^aX09(WwzEgbEeHVEK-tLEN3(XS^l<rV#(vR(c<1x5sQsWk}YmBhFY53*0L0n
zoM`!E)e4Ka4|^?U{TH|RFLS}-o}Qz{b^hs=e-~L>WY?Urv_BJJ)zltqX??rfV(E<8
zmRp{mwfJ}<-6FzjwnfnSc@}3cJ~4m$_Kao8+c_3|9&Q#(7v8oEzRzW`L}Z>tji{96
z;z$|G$4vJu|E4xs%yHMX_}8jzaXZ=2vdX*Da%yv@rH|A)bN7$eE&8{xS!87|x0o*y
zWHE77f@S6bYm3ge$1S~*Cs+yUH(UNIzhW`TJ=5~Ssr?qkPjW48YVWmBI?!TKU}|o$
zv1p3r@21@rz5+rPvl)I^vbb1Tn6KJy5!pP&@<O$?<(!^XmW@*nTD;jf)55F8!eV})
zkmZV5n=E)r=UGZ^++vaYmfMnVcA3Riqk|Ufyjv_xf6cTkziepHYk%3&_tG!Rt`$2i
zB!o^{?4A*4xmM+sh3s`k3z53Z7QOl}ER0_KH)mdO&r<vHT8ov>MJ-;i9kMh|b+&M4
zXtbEyvd=<mdY5JX#ch_ZqK7PYewVSRT^(c*Qln?N`CX-j$L<=-ccn`$8ucezykWPt
zSlVW4nbUa4BHmWT@=a2S#nf%<EmIhxt@gf6v?!lF%i?jHfTeccWsCYH=ghTjwp(1T
znQYNk&TA2zb;xpcQ<+7Uou9>ryrY)OAO1BLeLcfMVKIy44INj@zLJ@ikABXy*sW!1
z@kpe`B7S{<Wng!;WovSfrAhe~bIDzsEY4hcZysdx&_bxH-9qVNi)H-*Pm8l+$1Jrc
z%(P;=Jln#yB*j8h+|1JC`9%v&!O!O2r*>PYPwTL#I-y{pxqhMLucsR=CX4e}OloJd
z`rES0{9oXDizh~nmZBWGmSuMHEYB_LwD|Kc*kaA!P78~@rj{-tnwEO=f-PU|JZ&*&
z_jwDmy(i7H_aC+}n<;47`0=#mqhrPvN!vbH9w>fb+55iKGDn}u^5{ByOGBUA7Dx86
zSTOqXT1Ia@Z{g!+YSEGN$}(8G(L!BD(jss3X-k{jITk1C4_drF&TCo2Jj1fAYrf^h
z?Bf=%7sy#y<ZQLLZBSup7t3JzA-UIbXLp&!ZjW;oC3|LCJkHo?AtU<EVi#AQW!^PE
zi<YXBmdoS4tW2lJTISX&SvJS<Sst|fW0AW_)j~FMf<>+33X6{Yb{2Jk2Q39|XIs?m
z^tC9+erP$ldXYuylV=tiN(C$%eU&U<q^`4E({<gV$641xdEpj|q!%HU=5^eb`E!yj
zmlW_=gqVD?a0!`jQGV`^h2q01i?Tu{s}6+%3y*16EX}4gSQS~NS$<omWI3@n&N40V
zkHuq=3X6H=JeJnH(=0MuJS@Ji-eH+!u*kybkd#HsEPksJ<_e30wwEjdS9w_0`*m2_
z@jtM<CU@N8%?fUd1osIR+l;&{KZ@B~?voC%3_BTR(KYqG#rD#0i<3Lo?c0{_y7xoe
z)P0jm-S&EhM(<^rTfaBuN9o>0H~4J+CYSA9=vT6rF?-LxNlI__ZB>}NPx{^8ef*c^
z@A;+vYR~zA_PthyllIE*E!~@TsCn<d*QI-p#s%-yI(Kv*Pkqha-CJGuv83hgP57{J
z-;-~T?C$8t*(Da1?R}Kqy020`XD^4F_1>$q*6$PiEwz7s%bI-&MXUCDzvJ5<^H|8v
zQbu8)`r}=DZPz#L6&2#%|K4ui-W?BC?JE^Lv3IUnz}`0R_4|@uDeui%5wice=fr)p
zxa{|7t4`h<9&5Ar-`mo?VoQ?un#9}fE&Nxy*F02p-<*H-dmoCI@3nYYx$nTU(!FKd
zJ@?Akvsi(~0`ji2?*h@t*vXG+CydX*S7GTjW6KU+^*5H+X9Vp4iEFTaMA4JixWN)6
z4#I2`WWnZs5N-#n*&cXcmj}C-rM-w0n9ssG6RsAd{^~ukUFIo=!E$__J9a(3aS}yd
zfblZeEfZF~MG*(-z4!SA*gYUNE<CTA59}Uv`3tlDnuXK`YKCu)Hh*_)ndVn3b<GuP
zXK2>D9?-C=S!O1YyH_o5XNTIKgFY4?wl6WSbXBr&6cIGrvwy43F*5;6&gciqXTSbd
z`=&3bmJ$_a9$vmu?ch;Ot&bl*n)Cc$ref(JXVJwbtNHTibMrrihUUg1f)*E6H>rxB
znPeUpn5kuE;H%kiE>1`2M7_Ctnwmu{`vvu{3%{A|3U4+S%+@nlt}s{QhP|D}*{mn#
zUv%0u?=7EbW_Tt=<@6dRtvfb5)HcjZS5cLjV!m&gw`$wQ6rCBo!5UGkY_y6_%BWSB
zODT5*?^C-uom+F)=40j;XNzfs$nt0yx!dV1IlD}=$1_LEx;o3AZ_%v1B?q$hpX*t(
zx0Q3%?vq9~dl^G#?(IC~WWQnSuf2z>kMGTKs@cECFJyo9{T{mojlT9KYR>zvBr)0Y
zz3Q{E7LnMO!_&F9_wx$-iMyEhDQ@_>JN1yO{oRfOdlQ{=_AkHpeeWT4mIH5t)a^rE
ztq+)~XIQWOy}>>)<mBGKbvk>v1ZV8?a^T<Z)lz6D`&VbL!B3C<U9KVan`X?~SD+-n
zmu>60y$9yWAMl^JcF()EtM)hMUfo-!-(W4rV!O}Y@7^BIhGTn$w=di)d^Kd>j%f*d
zUhu8hvx~`Y@3R%Wd*-b<xM!Ve*xoZM`t5J8;kSCbRB89t<D&a^l-%EYZN8LkLf3Zt
zwEN3!?6-C7>vJ{Od&W4`=4I{My&^>>dp|L}x7+b+-JZ**f9!eoIqv|^!NYcor|sV>
zq%vubx9+~ZWz(1KKjx5c<J&rQ@0lOR_Abn*v~!;@$HqypY|oM{Q*Ac-|JWPu@_pYT
zlNEdOCYIT6yTH4D5ASw6es+d^|Cna)6WbiQcVB|UUI|U#eOsP$9}um3v0pmDXs@}b
z?SA<cm3A)uhxf77KiIo!&yBs+?|<3~D<|wp-TQQZ*ugn_w?0<0nR&!~&x>8=Hh<=M
zAK>i^+w1hCYhO`G=idIGEc;Fr$?pDa+q+lg$i=;DO0Vx->}YTA;xTV;_{?J4@DH*3
z^lk0-UR%_=SM_`K{#`~Ud*uX*4phcm-|h3O%Ia>?r@a^dsO~)_8@vC+T&{f)%irwX
z-{Ee1d7<fkk=D}vhXot<&sApM$MJTjz53(Yz320-_8#A;y!TN%`~KbM=j=6C-fFXV
z*<ZWv>&y1q$Zps_q5J3FmrASlPg2RUfBngA-`UV5)&>nHZI!Pt+LLg2&fZ1&2ll<+
z`)~hhvG@CbtF!Liwxa!jjcB0#@5sb`CwN2l?)ueg6YFHKKkN3}z3H<w_Xn~G@AH|N
zzjynIx%(ap%(In0*=4`v=(&BHmAv;QxBT97>*&k9&lc<Lm3PnE>+pojHoN)4-igaw
z_A%;n?QQQpvPVg-Z=cwguX~T!TJF=B-(@dzuiR#DI?q0{1Dw|DOKNOBSKrucdS%^S
zt?52?{OKR}mYFo|z36>=zkR6H{<%jO4jlcWYCprPao^MV`S#OP*!FF?ps`zTne?8F
zuuFEgb?5IbOp@Gt!D+d@#?sck7ZTSW@RQWtbEVYvz?2!{`*nCv?0+Hqch9x6EcTpR
zqxSYIJlgwd(&T;CCDI2R-)ih@TC;rb(x*9gTt^q!sVqFU?@0R5JxWi{?cI2nb^oFe
zwS6&F@9Yw;Jm1?WaC7hP10VOEHrBA+X7|KSFLB{s!)?C%T2Bh>&Hk`{k5A{*y~fMm
z>^UvAWA6u**1d-NtoNOg7T)K^Gkb6RrU!ea4qe@Ane=7PN-?ed#a+z%RHV1q$<Jc9
zIWVnmFI%0=ZeNj~d$-1k9=Pym*WRD*jC<D|Wwv|FYG$|3?ET(ShVA=)ZN0HK=`Y)X
zYx}qCU6WqE*X0DmK5^IF{c{d3*ju)k-!@1)eSe!W!@jW4`F8i#%-y?U{n~wMGkNw;
zJfmV?FD<Z7YP!t<mE!(AR`0Iuea3TW-@h*t?caVpxNpbziF@aBAK0(7Yt#NWvU~UC
zi^uN`-g|WKhpU(OGRLa!J#>7oom9*7J*PO=+RS;hW}nNqO?x+F_wT<PGhrV?Z_GaX
ztjT+J@Z8;dQn+((WYy>0-nt_D-d=UJir@2k|K{qCd%IuB?)k#oxbFhX`Ms92f7)oL
zo!fu;p!?o}ESr7mnfGnnZtk|upT@JVA(MaKi%5<G&8z<Hbqsy8he1=`ZUwjZfm=qx
zdp)kr-!HZA{hn1yO8a$YKiw-bdHU|WzaOp3JiQJ?IC$+%ocv?=dhS>Io-s__v+>J(
zySs5~ZNA^uwN3FUw~xNMabK6!lfB!XZrrbt`D>5t9lgB*_pJ8yIUd`$-+STSyB~!1
z<(JOcXO#8U*6q6azJRxJdw4!I?tOXU?f&-Z68pFVpY5wnc)8d2ZQI_z#;f;jyFYVp
zeS76T>$*vMedkWzyYcdly}|jNHjYViY*OldZLV<q+B;o+-`<9hAiHDhH|@Q%eBs`O
zEp2-}6WMJfCSBNjSnAmRj+fng-w8JDS4z0O_u#Tidu=|f-zz6+zjw;=ko~{E|Jb{F
zmG*A+Qndrimi5|QR=BZ0{-1|^PqxZlhyA(sB|G$X-~ak_@1q{(eY4d1_ZMXC+WRFr
z*4p*c3%iR3ulMTMTOHVVD|YV*-aY#@zxCR)gdW<f{PxgZU(fG$k~>-Ug(Y3u>(kY_
z@9?YK0}hQ#_P>ZfvsZ|__5e$4^*)Uom-hX1Z{H(t_k8c)pxgUjbnn<}kkfBlZpOLK
z!(DOj@<mtos(JD4eag1aPMhcU-r3hm_bJ&e+#9~|sI^V|%e~GUuG_q-sN22G_R`*|
z4m<aYoj<?#?X~W`5|y_5d>{SUEBO1z-X}db?ey18uo2Kaz5n`ZKHJrL8~56nZQs4P
zt;4!-y7GRV#w?qSc}Mm>YT-MuX2G33-@9(@6Lq-0Z>zoAzTkQP_cDCHx94}Hm2I`O
z<=%Db9CqvX+3#^MHQp=L6ku=mEoHA+QrrI3Cu8=?DWBfAvee$*MeMpA^F@A};|@pc
zf{g@iwocC2+qvoVzQZ#+4+!m?y#Jf@fju4E$p`x8CE4fqFWBeLp|MXj#A5GvA-nw(
z)@NGvt8B0f-p0C@ui^Y&rxc%k-6m7^J~<p>pXPOT@2eN#`!4ZZ+q05MbRUB?<G!8y
znfDb(-`pd%Eo9F+>1q3?@$TJwkZs={m%N$#gsryj-T&<O-t5Ljc0Ziw?I~;L-_O%3
zv3Hti$DV1+7w;9xGu`_<;-Vc3@3P%2m23Cjn)75oV~*VZ$*W%4-_Kz`Ffs1--fu@2
z+g?|W+rwRRaaT*&roI19FxX33AK%L!eR%KYipBeT7Tw#+=Ko{=t<|sgGBiEkFEc@A
z|C5Bs{VB{x_V&as-rrg@Y44wBhjuM2n7{8pV$uFLQ<U~=nTzjf*nQ6~_8qJJi$fjz
zjD<A!I_^@~JN=HTo#|t~eP@(3_D|CKxQFW(mu*`b$KIVsr|&h*`@f%YiHgmpcfI>^
zoC^1zdCIxBXWIL{ZjzmQ3zp05b=W&?uWQB)JBzQ)HbKE$`=;ry-B)<7dGCbwn|mb_
z7w<b_BD7cBFYtiEV~%}}pKk08>6*V=L}KfnLlqPE-`W3t*NfMS_twVEuzSAb@&12$
zLi;DIo3!sCZ_3_h6Ry}@Uy-=CjbZ2B8m0Wb@mAdX*G;~^H~dQHUgr5;_HGH>`(Ad<
z+CS&X>b(=2rtDvNIo1AQ^KQGk|7LqFe!AK!zq`JdgY(ngmC_sbZ3yr_U~pCNz{`Y%
zdyDz@+sQ=R*ol{4-FM{k#J$ggMD`v~jokmkboZXrBck@J&k63`lu&J(dFJdstqn)_
z>`|<;w^^RF@5;=eeG|^rSv*eX-dpk3ZtsKJEc@2<s#!Y=f7s2x;F<L{>6X2VcQWr2
z+0eeP=i5e`#^$tr_a}4O*(NU7d*JqUJE_*Pz58lr>}_qe-5ccfY_Hb_wf)jl1owUN
zjoCZz(jGfs%|iQ2p^W=y*|XYxSuV71Qp246cYNmWE!uo)@7auJdr!2u?GHS-XV3KN
zoqN8d&$3(fdfMKMqW}BCc1hS=h_<rZDWQ8nmv`|#DY0ob3;}HR%Ze2C@Ez6Mljt*Z
z-<dp~{g;@7?DOOI?wy-;Vt1)>)V`-bm+ia3kg_|zfpZ_9^XdJ{YG3!}Wrf-?-f!6Z
zg>TZH)uyid4(d$ZE0DC`j_X3)-jI#E_FjJbeedQFseK<5zVDsBfPL>}Io-W{rgD2b
z6qfAg44AezYg5nO{~LPt71lBCOMBF1D{yIoz2B3@z0W55?f*Zy(MENB-`<3dm3x-3
z2JAI`E^q%?>890w{!e?g?3(uLKU%qW23zU=O=ffK`Cl6Bo~qbrFEEdFU##uUy$6KW
z*vQWG-CMk>eD8-H^Y=K%Ub0*Mb@rZlCG+-2vpVcuD0ggM-y*L4E&QwYbMv+A{dthX
zVaG)Necqe@?TrtZwr`f+G5cdBGIocjs_iZ1dB4vld)dC$+#~y3lP2wLJJhvDapuW=
z*=H_WF-_{;r^TYQ*UC_4@3qTJ``C9b+WXIG(*BhPSMHrsuCnj#H19q7&-d*;VQ_Zu
z!-y|?&n|nhSNz+py$8|+_WzDOvG>`Z^SgDc^7b7_ySI0pfvT-$s*An*1ZLad-xF*f
z1@79r*7o_{!k?S=zCFaV@6}08TfdJwd!nYy*c)Q;#a8L~BKuz=k_XhjYVX%Or?QXZ
zI@9i?zD;}oszmRd?>S>{cT)D=D&K^?oAqDY91pUy`*MMKuT}iD1Jf;3_PQ<Kc0lGQ
z_ugAOckDMh%)9RqyQ1Bt-_?7y_paV6ec|xFiX$Qp9uuzY>pQV<ue;mU{f=U7_Qo9x
z_nqvt*jryyws+&BqqfFuYxioo2_KmAMr@zl#eLR+j+^%0G_2a2b)eaPgZ-Jk1;uUq
zw!T}phflq0k7?<Ry?@<~?Y-UgdQT!Z>%MCCi~Egg*X>PM)wK6mV(&gJS=PM~(;BRE
zGXwWcp8I4k`xK@FNrgXcj(iE*+ik<TFDZSs&5DZ;?e&Y7?wK;9*T%PI|Nb4TBoEk3
zJiGtIvt4#_U;gebc$8qzdw;pD!_nz`HQrV1je0I`FU&k`uU6#Zy)pq0_NiZ%-uu`@
zZokILu-!TCa(1^5|F=DUNAiGZ_~gC+o=n`=apZ<I#{t*9aeUMFWr(ca|MTPZeQ&K7
z?@j2rX}?57-Tv(Ii~F=Q;;mKvcI-{dKeM;*)5ATXXV?xfg-+PpmO9l&ll%1^tzS#_
zG=AkgAXk*IH)OiUKAxO{y(f|iZL%FDcN_BFwc@gUwRhnb!9B8HrR_a`^6fcTTD@n<
z`F(rlHBQ-E-u}zx;#tN6#syJ(BaTYj>p51~7|fiocT>sTy^DNb@AcSTXdjUyyw|{d
z(_ZIWFYIJ5sM_y7U3Va<T*|)T<s}=LThr~zS?BDLa#^u=+3j6>l^nD7oB#T|<G?q@
zJ#%ws*<YB)xv&4M`vITVEc>FZWe>bg?6z;(60?8T>6*P?wr#TKf7-bBt<H_TF;ZRo
z9!RX--;>H_Uv6!+H@hdsp3R2Gp_Y5=z6cqgy|=fD>=XIBaG%eqKYKHT<?Ijh^6ae%
z>)l%sv|{hdU1EFoEi$v;_H)5rZMnjIi@)jYePyAu_uq$Wd)}!j?{(!jvH9|wZQml-
zW?Mna$$JDhb?#n$ifv!^+Q)kyipJP#eAu>s=@a?gQjeb6y?N5Jx96eaZa0~D8})xD
z_n!68wZGJyW%WDu{a$V@6MLm)x%PPn0}rTH``dr?mEJeu&T5;x?9=w1|FL230`^&Z
z6Yd4r@om4k_gtIFUKcTI`*~7|dzU+v9oW|OZEuMY<AHt0cG_ECp1kkoPQksd@mK9_
zEj0J|%@p2y!*9~QCC@SsTr0V5=X9`pkJ%Yd`?!*CcJcW)_T9VsWiOvZ@7^5~&g{Re
z@_6qef13ked!O#z#vZq)Tr_3Rnm&WQ`z<pZKD!^?+uNzX@8xQ_eG)FZd;OQN?7Jd)
zf3Jo7*FC%ox9>G>I=+7;`;NUUa<}be{^qrB-S(q<cc0GO`((*4ySL4WdtYw<YNxy`
z(dK6L{5@yC>F)jhmv`UGOTG36(*&&r8b9sf&fL2H;-9;ATX-JsFTT*T|Jv2td-Yx9
z?YCWR-g}gVYhUyezde=BjdrhoGVCq5(6Wckz|j7wEZ;tx+@J$K`}W%~o!V({bMunz
zmlI8P7x&8AOuR3%_vEbydnFr8_8xx~Vt4NH`Td`F$sJg#e|T@A1@Hc{xvh5d|2*Ay
zYmVt&HI??gg`WDhZ&Z!<eh-^`K;8M&?q_eE_Xvv#?u%7o-*^6x)PbwMQ}#OQsqTCD
z<LzF}ZOr?g-)yky-@0?}!{)U%d*WsH7(7~F_x(r1-l=x2d!8QIw%5f(Z%?G4%RcwC
zg|@P+<$I3&Vm;s-UcYz6X(n5{4J-G~W@g<hW^%!<nX`TOc7rE-S7x#wj9hobp83MF
zeKTgg-8ZZ6>)vVFjQe}0IPB$(Ke5+`=fz$b?`*plYgqRE(pbJHf!}9;r$y}^r!I#B
zS(?qZ#hX_iFx{B7w_b_=z{;E~>y-y5+r>2f-y4^6ey?QLvVGUOckCDZwrsy$nBd-X
zuY&hE@V?tSt@Gf%`Io!*{z+2ZcT+;qt|VjX-iIcU2UvAo_Wm}Rx3{4>+*ZA{VUK%;
z^1*qNHrVXpmD%^8IA*Uof5YAzJHPCiI_b||shOsG1AKk=%5HPncSg=+@3jSLdk?v8
z*tg$k%3hV9()%vHT(X~ScC5|U>r-tmO8(fJE^&455)q+&cj6`YCbI|cSFD`4CnRs~
z9<`17d)q4o_6Keg*mv&ZMZ3fuLVLy7wDz5A+PGK5bMM~AAC~TYSh#Tibhd>%tKWUv
zJ72%Vez7IvKIUt>`?v#i_TGLqZJ+AlSo`pUUu{pW{I&Nf+b+AA$9C>DJrJ<h>fGUd
z2O6I3dppa~cFRGzeU}$7@9X(IdAIO4#(j(a`0kzUd3bN$iv|1j=3n2t`Goj>%UiGZ
zs#euoKaAb8uVTX5z4={z`%Ye;vp0>2dtdmfC3~#Y4ffvg{<GI^XTzT8;@G_tI9m1w
z_pIM1_i^i9bDyhw*UA;{D?A*uXHo*ozP!78_FiJUw&%?;vje|MZ|*(4K5*}XBm4H8
zJ!x&j#HhUQj?2qEIk^>kH@sYRp!n)f`|n4d@AoJ$vTu5zzwe@6vYjx8g3YA6|Mu3*
zIKTJn-Mani2lV#{pP0ON%NuE%!z;q}b{?C&e}AR@-YB(S2gE!aZBMrw9QbjLXP=V%
z6Z?R*oO}05p4iJ}H+|pl_FV^>KY!RSu;7EumdT2CdULw%eyy9bPf?6<pLKlK-uxHG
z?7Zqi_jb80I<WES%w47%PTBlnw6wXkvf9Sie(r&@`M+$ud&>7!7@gY7ym#9k^Er!l
z9}$<@oBLbO_O|t-z3YFU+y8j+n!S^5*6&^CrLnJ_Wy_xV_m=Igjb6J?>Hj2~_bG4o
zW=Fo>%k+4vjfeL=tF~9C_wM?XZU5MG&R#7W=6x3{#P)kXTen~DUhsbXhu8OpPI<p4
z=o^#$gmatrvULjYVeDSKr>wARU-h1SyEki8?)|S7f57Cz5*y=v`3Iuw()L!&?K$8a
z_RB7qv+clfjYjMF&mH$$8FuUqFkP_sw#tot`);q<|B_c`pKGd;?OAJG`}>D__bDpv
z+&6b~(_W8%i8hDaoA%ip;ofV((YL?acf(#;gU-D&o#}fgw6yKLINN`JX=%{jpfjQS
zu9@)fWBDeqkHzr*-sit|@BJt&YWu7%aIef-Q~TZt+IzR#vRG9lsO*jU!Lu({hRb$J
z){=csVjk{Q5f$5Sx!lZ#>%hgmZ@MP#HDXw@H{T<`-s4KbUaq4Nd)PjD9k7+1Xzwgp
zxc{B{rG1XauiCb??6f!9w8-Y*s&ji^-?iDhHJ4%kGsPu)_4QZTL~Js!ul;*@PxGd{
z{i;`=?@9O@yML?s6#Eyo-TQxc_3f?EEVF;8v(HAs_37R$z0>=4zuL0z?|Ly?CYM>e
zjm5oe9Xj^dRo3L}dm?VN_n}Gl-nE8X>;%?r+Y>E#Z{N~;=l3psUS}&cJ$A3Z=lQ)4
zu72O|EaAA%-_CAd#EZVYyKZ*w@$5|9dn-)FCTQmxn<e{<_c~o}+_&nG@ZNiq8~1LU
zT)&SsAY|_^J}E1f?M3@_Z?o-H&|<M)9?`Rxd)}|T9!u8mN-WrI{Yd}d{-VBxdlx$D
z?9DfyXjd~e)c(0!!`|iR-S@p_;@?;A(Y?QWCgZ;2k68Ad%sIH1Q(63gQD*htqS7UM
z!vrSpfAp-!=IfD}c1mq>`#v9F-6!j>Vav8?f$foJNB1t-{%*gY*pxj&J=gYzp53rd
zYDI@#;a&&(<?}uF<UKLjf0k2uzrNM1y|xM8_FiNu-8)5L*?zgi-+OP~_-v;>t9!4c
z%_N&#<GuSj#dG%5ZPT{f=d{s^LqB3)vla8c^+ylyuE?FeCsLYwuVa4Q-Y4E%`;7Z1
z?YB*g-OCt!-bOObeBX;rMtcud@!ACDhS@J$I?+0rdD;GT=FhCZ{A}Et6}r=Up~(Kd
zF8?Rk-!l-}CpFb}uj14Vb_*VI960P?cfi?p;Xa2(nZ1)IY_|8%WZ4&4{eG{c&5^y%
zFCW;O1RmL2k$uA2c+MjG&Vw8GKG_m;AaUy!t6itH50qRzyYIh*!~uJu>w5yY{cP7|
zFW=q!VAb9Qnm6~kJXx{-(P1I`$$NbEdRZ$u2+j?&|NV8*-s2P4_pOiIwRgSu)cq?M
z{r0$Ad%s`xuhiaIU7z>tf55Z5li%I;rCq!I=Gr}b^BX<(g*us8Z)|+E=l`jtdzEKj
zwW%zaw)ejYr?vM>J^O9H>h=^`_U{#Vo3t;<M_}K&Sx@&cFn!zYvBPuk36;b996uTF
zamn!5YsTZUcZyWA4NKhreN(3FvH5a2Yj5n7ZFZJ%8|?nQ@874|*<@$o_;YW2Q|G>r
z**bfVdY<1aW}CM6Q^R6A^-zmFPu8xr>6wtUuetu+-u1`U@8j%SxaXoS@BW#>ru&UI
z$n9UUp2y}&o}4{ro9I499h<%X7fs){WrC*ND_0fU?XR@<Zc%=|m*aKl{=SJb_Q_9f
z-#bhA-rfMS7dAWn@9jNSG|5ghWr|I|$2%Kejl6xcp555%@~nHGhS8(F&$tu!2}qjn
zWqb2&&mI$xy?p)e_FmWL+k2y+eow9By#1*XeETc{&hAy#Y}+UQ+<lMg-FddbdSdp$
z=Fj(TbL_W!eMw>Oa=V?@;v&oT?oKY*bB^QM{`-ZxdtF*Xt$(CD+Arv+wJ)}Puz%VW
zNBcL=EcS{h3LOwWFncfioppPQ|18}5-(|}F(xiWT9C~)|75~g_@1Zt%@7x`G_FekM
zv~Pig?f&flS^F29UAq6%LC$?_vyR$lZ~wFR;qDDKH~+NmYjAzAf6DuZ_QE0D`)<DT
zwQqRx(T;27+I><o7xwzB5#D#dfqQ@KmOp#fan<a%{<O~e<n-@*uO(04^JdBKy-&_1
z+CS)G*{#CbvhT_7OM7LeweFo)yk_rf^IJAzAMfw&U$A2D)|=k@*WbFZ_i&lP-aq%{
z_WfEOZ?n;?ZEuRO{=SP7756@HPqueH)V9~#pknVk>vZd%xd(SYt9!aXY_Y`NPb)Kb
zNA$MXFOfWHKXY#1{^Kgr_DOw~dvEdZ9$0NBx95WFfxUSSy!#kj4(&ftG~Z@%bo<`O
z9gFM*mq*&BTJEstWaHbH^kA{=_DM(gF=q1b|9r1~ud4lid*7`*`=(5qwCCC>-hH{!
z{(EQtNw7C%*t@qYntR`mRY`ly-`v`_z*Kb~cWL+DtsOJ>J-GN~kMeO#`yQh!d()@H
z?bS@G-CH2J-{x5AM*D61c=o*)VBN=47P|YU=F+{XGJ$*lJHFo=d;5&dp{q>$608&V
zPv+gU_thkoJq7wR_c^clwO1`b-B#I2-mXUF&7PL?t_N~ly7zudWZiSES8(5}imQ9m
z7mC>ne{I|QcpbyOTeEHUM<=r!2#`LyFQiG%KKbawy@6kj+B-xa+jBYZ>P`-weS1DB
z-m!0x-m{nM%GNz!OaI!5)r9Zuy0>?~z@j~SCo@Ic)n1xsZ`-h9zfP3kzOpl__S?kf
z?|pOg++Lm60sFdikJxI8-m%M<ZrOW<-En_L{Vw}y@(cHsb8OyY(<8laPWIIO+&?S#
zdVi63*xARsZ~IjT+s^BT`|i&_Wg~uW#r{f@7MrEQ)Ap_8y}UQ#+x)$nCob;Tp4+{f
zk7NB_jvqJonjAS`|AXb#UWX0)_IB*^*mrM2(;mJv_pOX$ME0A!XWu*H0*}4-H0wQr
z5wUwc95(N*|9x@q$A#whyXHOF+h(+FZ<#>LzAYg??dB9Y?LV`8x1HhCavPr)Rr^+N
z-n{ooL4g(Xr<HqIQ~CE_4ZgegWw*~>t=&5Ix%0$#KR9Qy|4YP#z3y9!_s@5^ZfneE
zuzy0w-Mw;qwd{NfAMZ81m9_U(=gNI=n4R~fPTFM0dFkq2yZN^E?dp9trLWfP6V`6u
z`;F=H-j~mk_piU7y*J8=(f-<=lD&mp@AtHZite4%_+sz#Ijd~_Z*SW3;z!uNtcxG_
zCg|+m8^gMP?>V#Ly|It|_lk%t**kBC`QFpdukPh}v2O3$?aBMJj+*TK^>CHVjwzx0
z*&843JvNznzxnGud#mc)Y`*?@z1v*&`JUqT^ZTE4^X%IjbA0do)vIjREnaefS7?&$
z@3L+Ck7+O6yEUtL|0?SPdw+f6-?#7H`Ms&%9`AE(h}dg2m2+R~jf?xFRap0mW*)TL
zI6+}=&|D_F!+m%57Wwnpi!Iz{^B_a_Ko!T;y`SzZ*~^>OurFqVifw-;%YJnixxM0R
zdiPD-UcL8$$Ig8@?^^a)Pu;mU#cPFK-3pOCQ|J8No5|n1XNOkn-ke)Y_Wd}#fA6H-
zkM{-X-QD|pz5BkcJGJ-l+HmacYTvQvW>@vz$ECk)-iRpLoXk(%U&qn4cPGcYz3qYA
z`?8;(-)kcfxwqkt)BeeCKJA^FlX;*^hGn0WcY}>j@twUjZ^id|`F7YA%$dE%u)JYU
zoBX%^f7vhEPdR40KjQ5)`&jjhdjyZIw_7e`wdehp6;_-9|MxyvTDJd|@`1g-ch1@?
zp!v!+=)2P%$s@n_FH2azH*LPsfy*1T?af85?$7$lw69tEhW*@q+<WJm{I=1UJ#k-|
z@|*)}g>?6y7MQws>(O=l?O*@gr|#9VZ|&PTdrwZ)*>}g7b-&$A6WdrJodX%{oA-)u
zYq7l`GIL+U3whh1x&!+!>MQKM9O%67`7hJG6(;RA%lS6#<=pULZ@<RIy$^qU+3h$%
zeE+K^2J4`cEc+O~FWa}=+so$Hk0rZ3v+vrK#VFXkd)Bx&?ewSJnFn|7;hSx0b6)w{
z-eqezta;Y#+G~?(YQ3Su-u_#Hpnb>%?gR653iee7%k4X-pkROKdAseFos3q&)y;b~
zo2J+*KX=^Y!#&R?IeUrS<mNrQYuBva|8wKIy+0k?4s?i^*=6kHJ8;FMYwzUHmc8!1
z&3nw5w(d4NrnztN=7sxD`$pMso?x<<=R|{j)w!p3-_;lF`&J*n_gD(6O=LFTftvCO
zdrz;G*&lb5WnZsBtnL0|t8G=+tlqo+`wP2DCm|cb6qkL6>~#0qZrEs(cF1;*tsUDw
z*W~iOnHyeO9$Zjkcc^sDp1A27_K5tk+n3j3xc4nzjZMQWrM(8<-tKB=DBJH;_<D~^
z{GPo+lb7s`o!hrp`_mKq1!+(Bx*hPbd0#eRzk!pi{nZH!`yah(wexW|+9x%=*SdSh
z;obe87VSB#6~6a-))V{s@`W}ZODp#--Z0rd^&I;?n^~UwuXkMBd*n{xeufrShbQI@
z`%dZW?~Y+zW3S!YZ<Ay`f6v+<{rj#9Xz%;;iPzzc&EY+!trP7(sIRbZh)LVWTyoAj
zcTL;gc}(&L^jFQWKJ??Jy=jQvz9;FOdoQj!x_5r8!JZDQ9s6}no%Z#A>D?E{nZI|g
z8pqxjs#|R4uIja6tNy-MS<imA)vT}fQQ{MAmS(K9HW4n{x4B~R-m>3Pd*}Fs+oT&G
z-MdRb>A)%0h`kxtTK7auWZL7=aAfZ~mV^5=EUou4DJ|OTn6htQZ`+-H?f2*GpU^M3
zKPz<go==AFZEczM>`ByGwm1Fyy*+a!H|&e6nYlOi$IZPzuFc%bs1UMu%eD!2)f#Pk
z*CrU)fBJK0zsRI`yFVfa_Zo0Z+1+{ke{Yxgj=lQE6ZiZLy0O18Gs8|`&K;ZMd7JiW
zCTbmEom{x@MyBoFuz$aH2TV}4Uz&4!ZwT9z{r>+H_f>ju?(2`azwhnI$9r{ymmJ_U
zpS|yPt-`+DJiGS>Kd9Z?(iOZ{JFUXDa;f58{-#g6yHgtWZ$5BouZ0%>zQ9PseVyXZ
z_j226>^oO>b^lVX-Fxq5#~%0`bYgE;slJWE=0AIP@pbN%(=6Syai6QrWaqlQZP)S+
z<habW3w5vBzp~zJ|LOg6_xuUjx6j{;&u06T<9j8aUD)%z?#aF#)7I|UlohwP;jhMi
zwW_8)FOK!@-!ysq-Zc`<`)hgT*>4q0-p9fJc8}#D13L*Zraf&}w(W_OyR=Vf2IGO*
zrq%X`Bpdg7+HTr^&(y(wA6xW3-PKR_UNf}b6ZwD2zWj?!`)m?g_fL5JYp;FaoV|-~
zW$rum{Htwk&~p2!X)1eqf34f=^+aypY?fntohRSk+iBOocYBA;o|9SI_ln;zv%ASC
zyf6RE(!D-E!}jgecHJvl(7x~0Cx82WNjvxOY@J|VZNs$h)Z{rf*_Gn^lH^bB?K0}N
zXDrUzyJFj>-AZp+4qmyu!7k{9&HkD54(~nnUSwZeP59n9FO>EQx=8F<P|m-5h3f0Q
zz9+r*&Q_ka_qmPBzI2KBz562H>@Qp*zpvMfd%uK>qy5F$O?x%hcI?$&r)W2a(QB{!
z-w)PPj3(?8>07t|{6@9|{I)->^GuiTi*Ts33sgV4FMisoy$&LK_T(I$xW7qw_TKz%
z0ozm0+V)r%GVH7Wb7l8dqwc+G{rBuP+zZ-kdM<h2LB2zK+m)pDg;Y%5>v22UrfhY=
zUa6^VdxRG+vD=j(y)USA>z=-@N&6n26Wmvv%)Rfexvu^3E7`l(xf<<f-MQB0mbArQ
zg|K_p=@lIN+Wq+U&D9jxTaub))4rtFn*aW_eE|*E?c5oI_U~Ewc5jy7$^A$Ex7#ds
z&)fSVnsaYIqx$~sO^^3Vi>|e{Q%cwuym{lE<LW2v&MoWOJKL$#?un57e(rZVcJ^XF
z_Le{O*#BnxTdSPDKYPUH_w37zS!}nlWSiYGm;HP99Z%cORqMPz`pd3;ek*V7b^gG(
zx9>;O-U;(L_6hFcwB7sc<X%&&zP(K{tM{clEwcGKon`+ViKxB8d%N~s<Uh98f+2Y?
z*Sbf0PtE(b=fd{by|E7*_Acg<I^g<7d2i6!#e1H9FW#5cbY}0$|7-U$d@<k8exAo>
zR)x*p`B#efcHI=*TcX6WkB8fD?}WbY{f2s0Hf(H)dl}E}xBYv{eg9gg_4_hR-tD{l
zYmapSbHV;qGhgh@IeKsJox1yb9d}*WzvJOyt5wn4_b5Ko+ZX88VQpoqV4JA5WsjB)
z%l@50jt6qpV)ieZ&9;wKw0Zw={=B{Vhg<eO`QEqhoqX{=rmkMw4oQu@k2XKp`_gOY
zzUoJwd$-&@wwFcf!rsqEV-7t2-L|(c(`<kDn%=#SRrzc}z4q)|)y}%_;PyTHYX8>m
zosq!5@7*t<eeJuh?+q5axYuXWC!5|6&U-7BzVEgB9%$<^_n-A&jTL*77uW9d%wx6r
znJH^i%gVS<`4-RKCEu&}chBV6dsSY1Z>WU)o~gS`_LhCQynjc+@x4E<FW%euKIuSs
z?XkUq%jWO*<mR&D`O~;palu`?TjyJB>fKK7wXa>hH-%As&zA%_n-ulNy>jzD+qoZb
zun{}cy?>|s;XT&oGxj|Ri?tPcQnjz(q~Knm-M4MpRG#e(_n5HvE^Fz&(~b=X_SJ>#
z@9a?A`(Ur^fkpG5?VJDN#=g$nd3)Yp;NEBTdfonziuS$s%}#dCM;6)yJbk~XRchD1
zuelk!Lk&;aKd4*2_aUR<K3OHveSdOfZJ$`$*_^S}*~_z4W$!cL1$(ymKG{F{!aVEP
zyO*q_*3a0NUo(GC-ix4p?DInHE-!nq*IwcN{yY57_Zr>M+`EfMe^=&Ly}g>R7w_+R
z-@aG7c&5#*_?QEx`)}^;SaW+nr?T>XHV^ZC%=xnRNB)cM3tSVw_wS!ad)KG@w%d8+
z`CdM@w7mwCIqgI2Pg;e)58lsawPjDZ--f-3D~k5%Oq#s!V8@)js$Dzn9FlhJ6%Ur*
zo6od)-{buX2d))Q-M8uf+P!sGS`K8+J8l0yXv4meO$YbZ{t4JK+dIeh)+VNXt7^^m
z?aJG>yGH86-VXv__BC3)*t`9~zI}d+YW8+`{@MF<p~&9GyxF^&4t=rua%KJ=>2s%S
z;y*6ln{2^xV9ji%y%U(b_L%l8-?#d{>E5LhGxxdit+3m)<Ie6shnn|o|8{2YpYu(7
z`&yazi89sfT~^g?|CyO*-_x2)d$-y5?AH&@-2dvg&i<2kPwzi|<l5ec(^Bjf{N&qf
zH??H%+#9ufm%p>L|Hsd@?}6#lJ<s1iv0ccwX>aTHru~YFyZ4qPOt-i7>e$ct^OxP(
zyn}n|7M|Fj`c}mz%Z1naYSE>A78}F&KW$%aS2jyyk42cp{!Pq!`%j(Nx$oU<u6>>>
zF7DmbH{VVnm2sb~<(mD%y5@Vo@4agin#{LPDDvH2_sre<&0ZAjE1&AQ?=Qc`-b^L2
zy;hs+_6FQ}v?u6w_TKu_$M#-Z(Po=tb8K&J(xJW1O6L38Z~fad|KHBNx1+Z0O>$@0
zcjFl6fzqQNtdAwU+uaj)X75bRIX2bv_U(IpYQo-Ux}v)$s4d%jYlrIoZ7JXPai;y-
z|E@S<Pm>&nz1PW(-6i+x_I9qmyyyO9>3uEd`u1LNT(Nhv_+<N6LQnU`f9&4xCvbl6
zMxogKnNK#_CH^(vf6~x*@2o{Eb~<Z*?R_M0ZSQy9_I)Xx3<sD(yX=nLSid(@e5qaN
z<qP&VnlA0TzWeRo>fFbBc6{Ewr{&nJUAC*L_K9BQ-IsCl>23$z+I?d8Uf9$<F|hv=
z$F;AAS7hIjTSxb9vKO@xF45Yn<o|HbjFZf}`_!K8H9S3SzopRQz2%4A@A=HFw(o=J
z78|n+^?iSS*x9y}Y42U3roM09r_(kzQ{`<EkG$Bkt<+_2q)707Rojle%&psZ8ATe~
zB?_qQdzSOSPX6xIy)z#2?X$R5Xm{<-y1hA~1$)`OH`we{;<vwKGjngzy9qXS`U-o^
zv`hBpm>#yv&^)x4ZMl#Ak)6x;Z&N#BH!t(o?(U<N`*V)g?o~OvXOD?WnC*#4j{E*e
zHSG_zo3%IP!N0xRYMu8RJfFL7ox+b@t9mwA9n6W^-?#1fp1U61cHcgk?0s14WxG2~
zX5Z4O=l8zdIA^b?>k%6_|2g~0OF!=Qo~*gIb63*dJmJr~o5KC~RLb1iTP-ZH|AP4c
zz3P`Y*t~0q+gIZ{*XE>4ux;1UA9ngNclJI%Gtu5oqkOMJSK=Odt9N^^&T82EL2IVn
z>fUF&Yj<<+tNCraKh>INf8KL%yWMNv+L^C*wRN1sY5#hK*xpHZSMUAxK7OxxgRkB8
zV6J`5A7u7)F5|Ud`18wN+x~Ah8|FvuHJ@c~XA{e@-`q*We)7bpdx8!xwOjrC_ujed
zEB4mxUbrvZC)K`0X7B#Vg7<g5a?;<QIn8MgGh@R(({t<hDz^W%&VR$R-{Y*!-g%PO
z_Eab?*xTseviGQ2`#y;$OZHyhW4nK;*6+QqAE)fw`Cf5fU2FH=2b@dyoK3aZ`)lXg
z-7+EWd;RLS@2{QBx_2x0x4kxHHTxcLMebb|6}^XvbBkSb!K6LbY8eOSi3jb;(!RG>
z`uW_wy33dDRr~bL?w&NyzA%P@y=xmL9XPvfmR(ZNdmGKQAML#Eb?xQnZnu@F__<d+
zgk|4_+&_EUyx!ThI3L_yxx#q2r1#u?FHGlHC%C@Zue<!iUj63<`;}QV>@Qgu+neUE
zw3(#1VBbpT*gb0`y7xK;&)O$jJY&CciNJvuK2!JFi>*AcUa-~fLidh+EVYt*^*!hB
zowHnh@5D9p_AF~>wmB^szgMtj$6oHAXZLD}8SnkloOs|M_m<s!ZT9<?zB#>@KVaA1
z)9Y;ZhTOSn<CCMYJ9F)}-9=Yt?)^KFdEa)4>b)hCrtW+Hz{2MJU7>w!rt9q#p9$|_
z_#eLikmZcMO!p`5DRH^7*I#>^%}(*Xc3r(I_O9K3aj$Rm`uzc?*YB?|T(j?lFX#Ti
z?OJ;Tm)7p%iPztI&xU`m^TQ?^3$@GkIzJ!mk<4V>$FBNiZ{VAUdp0$t>{r#8Xp=W_
z-Tob(FYWk^kMDn7zH;vl(~SL3UNi08nUuXp=~w5zZjsCT+2l3$-+%jlZ@_oX{Y+aG
z_wsC7u<vMrsLd5d;eDN7rT6c<@NBPRRQf*q(w@BwqbBd26>YxzVinKcjBjoG<+}Ly
zEZX3@PlQu+Uyk!R8(G6|dwXU&?KSy$aBo@=<36b~(YBNJ^w@mSp0L*~Kz{GmsqK5Y
z>st2avmdi-pWeH7^0{Mnf(6|+mG(h<4!SSgyI?l+J`O`gd!wVr_H3WqxmR!d;(g&}
z+wHbgf8VEiXYRhYYI(MGJ5KI%^Ap-@k+EbC<DXr7zw8pVdo(3^FX#3b)>fhJ_DzB3
z_O5glwx2Khe)qTT$$MVcrrJNSzG-*2V#?lMi;C^ye*NE*&%I}F2y>v_#C8L_f)`dc
zazP$@BcIN(YYo)3pEL2~K7Rq`eJ3@e_V`s8?o~R|xOb1Wu>HGbA9gnvl-Xq6OW!NT
zer0cnoq*k;vrK!pEn?l5b@ImEyy+MAPE_Nv=?J*9$LdM(UWtk&dz0^8-m9~UXP?vb
zAG;kMmF>$Z;M~WavS@E-#*6*Rm0xV;+`Y4}_H*;z*UZ!Q-p-KOdtG+nUWxb1?KwId
z_RPEYcklAav-XE3_Sh{Av)X4L9%a|yv|;a5wrRF;ts*uNtq1qaVpHAw`-Q#Tg?T1>
zF9a{$+nU|H@2Nb)zN0=T_Q!3N*vs7~y{~nBxBWA(w*BXg*6rQSwSM2^;`e*Q?ycW_
z-{b$@CZ&1yuI-xk)kPonYR+ulw<Tot9^ckE`(_=vxTkeZ@80DWOZIzj2;WospT~~n
z{PDf-PHFAEyXW8DUA}Dl>SK89Yhqs6++aJsSD@j}p1+pT`y{zpZR}$C_a*lI*n6-w
z%KG%FAN!c6AF=ur`^fh2rJ{Wc#oG7!y<WX{Q)t8fXM!{L#(r$te<JhO-q|LKd(F1r
z-zysP-};Wza=ZKM>-P$(UD&M?&UQfXi1>b$TbcWPExGoqWK{26&8TR<;_b7&4^H*$
z{d|mRuMG#we({*!ds*eY_7?j!?e~g&wZ}0xc>mft*Y<RLIcpt~=de$0>Tx@Up51$X
zpIBkbv18-jCugkpein4Moo;1tKr86re)iXU_Re^-dH?jNqx-ej-r84FHG8i>eyere
zWD)zNi$wQ1TunaU`h;g+&hd&}&J*_ST{5?6Z%_QP18%oP_I-V*u<xwcg1wu#S@#-5
z80=NH{kYd#<-zWG^XBii-dt;Uc7^HQ|9b`Y-B>bt-#hD*d(S3m?n_9VV5jl;$KLLk
zvIG4V$M#k(Hr*T1%)Mvg--~;uWvAL5b6&UCdI9S`=GkBN+bvPCYfN?6U+o%Wx9P#L
zy$_aOxBFkqzwe}f&fZxkx7*CHFteX}f8m}wAx2wg@yUBd-g?<239R0CuxR7nx@1F#
z_LLR-ZM$>qZ)xq^lm5ni|B?7pd+%1}?A_qLc3+kLxBVNLW%pOfFW%c`cWeK@=Xv|~
zZoakepw8mGF8=fO8mgYLi-?!scXr8~{UMLP?yYMQ-Lq47j&0C91IxvA6A#!%o!D!p
zqq=VihtR&OO&xn}3mf*%T>WeB6|siB!jY@?a4Z(yue5N}o~q}ndrxzQ?0aOBx%XQ5
z;=Pmaz1w$4h;d)l-OIL8Q(x?z`lN2}A1=myD^B<9eXAF-KmJSG-p@~C_f}1Oy4PH2
z*}mZN&V7xKEcfeWPq*gmS+Q^0ks~&~1?)DjECX$(1U=f<d;8*EzNM%4u3o%sU&Vv3
zdnP<NWh-VPzBgRx>)vM^P5Ua^_U{X+xw==$`K#Tgrnfe6M=STL3FhqE|8@Vq-?24z
zDc5H1-MB+y-<`A<w%gV}-n(T!+rBTH+xJfSH*deQ_fhLB9<%qID3jg$Lj1wr$(L^K
zb74HRTX=i)KF898y-^H$`?ADe?dI@GvdOVOviHBnqdnf$T>EB6e%<RA!m+QmYsVfF
z32B?}n-llVNngD8ZhYY04}B}_+?h`7J$^*ufaLnnJxz1k_jnj3?5VYF-n(CA@xGk6
zjJ?rZoA#<6ZruNO^+LNfYrFTCbZ)WtvD>_Ni%yj7*|S&oh-ftL{onL^`|;a<_g-24
zbnl|jg?n!t>e^@Za;J5zcFO+EwY>YvYyR&G)O%;AF1Fp?_pY_gK3@g9f<LXhzaLv-
zvwTnYzFlV~?7#Ak@4%rKU3*=hN*u8NlD?ng#?^gd9aVd$CLiDXXiuXZV>_=+4%6oS
z*VixJ^Qe$(Uy9Aky(dGe_dNLFvd_)v>fZU6<@Q<V-rl?0<MiGKYd`OmKQFmg&ib&8
z*-??bKPz7D?>xJ9&&6M7Y*r~R-<LR9eJ{(iUwfT$-R&ap?%69gbIsoD)c1S0O262f
z62N3LS^dG@t-^Qg_NLw4yMABK-r}#N`?a2y?|<SaxBn|g)BcIic=l@8Pqpp+&a$un
z{P{i8_8IN1-q>t^!+y2RoP&(}5--@;q<>nux9OkPzEd?(dyn_M-xn}Jb^p6XP5ZWQ
z;o2vjle&+~plz?U{m;D|Ggj|=@@3_|oFaC6{rvX5uZkS^1=u$4Z*5+?PlDfUpYZcf
zJJ%{H?sqRR+iUE~w_lip(e{YuoV{(I*6*#HEwFd@3F-ZkI!E{HU0$%yTsdYh^XxBs
zI2&K=J#dhF@8P@C_ondf->bWN;@-c3lXuVfbIHd1oZ`Oc)1>wkY(KkKX|{rW{47@6
z=_bDR1+3pKn}6QgdmzAO?}nu%d-H4y_MNGEvFA_Z*S-Doy7vE&YTftwPr}|uLW}oz
zGA7tozj3j5zglU<<;1k_(UkXl-)kk?l~2vvD;4}}@6PvAY~>Wq_ExDp+P8FL^4`yD
z`t3}ckJ=gQ&9pCLow`?W#XsAt+vn}QbL{WlLv82wd1NoxAHBYPe`_b>Udii{``py-
z+ijbYw6C~+&)!?pIQQLkd1PmF{?=X(UmH8S{E$8A6)X1Mdp&QT+S_iM%U6^4olTS2
z8)mU{@2l=Rd$L+N_kHwn-P>lYxTn0}%^v2T=l1sbckMrUb@tvr(QEc5rY+vrpH#4S
zf2`s@#yFY%56<k_qvX+U*S%ZFR`cWby?#|@wvQid+w(&@e*YS^wR@M`J-t^!JA0eG
z|H^$Ep0nCt>)yT3xxsE9r_SX4iGN-9GM{m^771+FQ?~u!{s*a>_I_^8-do5#XMe()
zs=fD@KCr8NoU?bnd!e1L!R7snRUGZMZJV%HW*_r``(dhk`I|rPp2D(d-(9~tJN>Tz
zd+i=9+`F>>!@j4CJ^ON65A6Ntqq5gw?RM+KbH45G;k~tIZPG3~se?=Q*0#9q-Oj&Y
z-%{nNd-*2J-N$-s!CpPv+<klWpY3gaw`Q-M-;KT2&+qQ#tDU|#g6Z4dT@O0#YIgDL
zn|)ts-=9#EeRHC@_sT2t+dfa8y*K`n$(~Z7bo>82KW!GNZ``YL`^(-1FQ@IDHPgrb
zCCjG0y{R+z`nWFHAE3^@pSe|Ff4xNazHeS!du0<R*aRA!+8e?3ZST)j8GHYJxMHVv
z|Ips{O^5g15`Ss;Yxn-W!UwbVAC=|ZdvrU~0f&v6_P=K-94Is1wKwqV%>9=#GWXt6
zW84?l@3_||qH;g~f=~N?7A&yYa);YKr{lNX(+-w>nQC(T^p~3NJrOQ^;FSdX-X-fq
z_b<D|YNMB4wmUX%*Y3R=U+oT%6SJ3Eq;1=pYrgN9&*Ht=Cg1n`?fkyy)0f!2R<~~2
zl>cA0_sFSb`*O~;><TYc-ShZW)jr03|MsZ!F57eF@8A95KW6QX|EjUyrzULg2dzVU
zCpONvG3aUCdvk-<{(TPjc0Zr{bFay~EqfOpT4R&0E^GJsc7yE)p6z>gY0uoZ`fJ%<
zQQ6xzzZX^S_4G`%4>13`xAQcMt)fBQekY9ydoy%)*qvYVaPOvVy!HoHHSL#w)M2;d
zc<<gvH>~z8yu`LojqlIi(9Ki#EpsxpJHFts-FhXqeK%Y5?S9<7vF}FC;(gL8a{D;z
z_wK!6F0%je&nJ6o=Ca#n?_{u6`?u1j_mSDYZS{&a_o}Yi9%f<O$ElIEZ>ufez5~Tg
zyBA8V+-qgax%Wt?s_oqJMSCp{&$KhUSg`ldq}zKhE||Wrv|ehjo%M&^QTO-nJGk@h
zUO$#9`}_Ah_ZnOm*eiEu(q6B~i+e18*zDh>@nG*YXU;vRO5E%_(?j=7I@M(-Ti9(E
z_^xTM%T~ty>x>$#u74}CnlkJDUR%>2dpHcc_Ojg1-8(CtalfG8T&t;!GW(<IIc+x_
z?66PWcw)b3@nn0!DW~?7?cHbB%)4nXpWubP+ZQadjY)O1>wl?iS6#Vb?^B`qcJ4E~
z?6++?w~xDpW$&+0(S3aHU+wq#dS&nS-4}P?&h^?m=RLno!P<bmr;AVTwLR)@ANQAk
zpHSVmy&eMl_bl96wU>Wu!=4F$IrcrN_+uUMc)E>Rt;K<2-yM4;cCzlftI@nKvC?#p
zT6DTic+SneI~SVno%Gbn&g}=2ZO7x9y&slq?a9u(yVp*uYk$p^w|m)xeD`MFpK2$_
zZ@16Cl-VwQqNUxLnK65x$Xf3IyW{8Hs`#UO&nlnYD_{C~pa1>ty|cdt+iaCn-v1|{
z%vPqM$G%u*wT+gft-b8Td;4c?i?K1-#=3_=eB-{YuQ%>ZkUVYkeAfXx#x!kvpS2$L
zZ=VV8+p}K9eqW)({_2Wt`}B?D_a;rP-76E=YR~X_!QQMWZ##Y_i#<FKGxs*m=h-Xz
z_3d8ogl;>&zmxa*O26IvZ_ACn-1|lMPV4H}>ydtO?|Rw0HrZ(*dzQc1xZf>HaNn`I
zNxMZ)*6usEwsUXz&qKBsm6zLRN9?q5@O9tcKcjZ<)jJ&f)(5WNJL%H0z4MO0vP=HM
zx-X@1;@&m6dJdl_@z|B^%-=7}q`rUJ)46*E7Zln*t6XRE+4G*Yh|0>nzkN>GM;h1c
z4NBd-mus7j{e#}wdl!j3+kd>g!*<&Ct@~qx1Rb_@<l9DU{JYo4Z>9YlriFW6EN0)U
zcdT)r=z2%{6}%Jo-I7|j_rZz^`}-n4>`!V;+_&euuJ!#8w!Qxnx9=~|owoP){Nwgh
zzZ~9s@^Pundr{$iR-657Pj%@W(49JOpU3XfeRkJR?k!m6XZt7k!JeQ+)Aus{PPA#h
z)4BJ%+D&_dlP~seD-qo1``UHiiWh!+g(Q6U&VAQoXY#$>CQ;bNu0Hk7o_V`BSerac
z-&^zg$=;fKOZI+!!m!u3bJ^a`C$am&gktuEZTGVilnvacHR1hUHLDG~ci1krk$T~?
z_t=ed)}r4|*mB&rvEh2V!DiMOAv=W-IqRn)KkOdME#3P@`{bU^v(@_pHl481xc_(W
zggnpv0{cz&zF2i~@54#?`xb3}XS?#Q<o=3Ew!J0N&UU<d%J!ZvJN7Nh=GoWqRB&$y
zm+gKN=Z3wr@1@$+O{v&>OulOGOm^3O{^|Gj-e0P?@2{uI-mNuu`*yA2-d8Lrz0a=A
zaJMU;z0J2NntPZ2c(V7f#%ep$Q(ab?N7(kJ87uE=wqL*7M}%#kk;yc>oQgwxe6K3)
zKe<zS&nLI+y>o&m@715RfA3d~;Qb$uvhCfL^=)rj%WT_|&L{WRXV%+QE)hQvzcF`j
z<$10B&ArLC25Q^(-p^dQ_rvF>`=jsf-23n6T5GOnz4nPuuI_C(x5Vztmj1npj&b{x
zp8ndqOx<dq-rK;vE@^A_ixrCPTP@MOm)A95pKAL3ef4S%c0w{`dmdP~?t8#7ckcs<
zC;Ob<tlRUSLuH>c^WuFy4L9~m$(HV0G&OSX%Nu=rnN}RxXP@(K?*g9%`|K9`?p5+s
z-nV2H|32@1*Y_?zRBFRz`g`vgXY;+?{O|U@{C;y^tUl{L$-l4mPX9K0pFrQwy<q}s
zd&_mJ?a~#d@9~`(xMwZHslB(i%I}?*Ex32?`m()S!*=Z3o4IxGR2^}fR4-@ysB^M*
z?{;hN4>xPG=T_CSm04+Iw?F;yp3fqu_kK_}-?iq&<9!cW8~484cX981c~-l-ir4qr
z27KP1|6rAEAk*IcwadM1r&<WwJGedEdwfEhou-}G-WaREJ-(|r_uc1yyZ<Hgj{T=H
z*Y25bbZGyAGwu6!obTG_!lP=NvUuIzirhLIraKSzHidBRpY}R?Z{FYFy`T1P+V@Dp
z+?H?EXFHJ|k-d(U!uu9VvhI`kRJvE=YS5ldF-P_u`FX*nks*3-9ADFZ_v96O`>hx4
z5fj<H&)3&tkGBHTKIun+_CJ^D+OU23V)d_Rr_J+=f_vX+7VS2@5VnU|<=DPEx1FqK
zJbb-($Ba{Ue{Y=Ldxk@1e^~vKeVpbj`yRgiZ5JUdynBj+=)P%BrS?93f79;Q*#}m6
z8!lUOF6Fa-!+djZ`<CB!JB;7&J#u@2-M#b~_Pwk2?+;03-FHWv?ZBHqoA<hDZ?-us
zB4O8E_jYf<LgQVJ4o=$p=;3cW?c;s>z9#P3r+8R!-<dhPtiLSl-2b4;$hL)V!oDCO
z*?nu-oc3zlo!fJz`rqCuJ;k>3n3?xEt*Y5)e`mp7o6Eg>y*#@2YW!QVcWTbpJtp4E
z_deDV++Q!lzi;kU{(TB>z4yh2GucFJKVYk1!0*ug@AY1bcc1r9-)*&b%fd5zE_NmF
zjhk1xH(c1(p3CCv-Zvk1TC1hq+`lJymTjW!j(sAj4*PfCKE8J`@813O|3htZR7>}I
zo>{+_lkuv(B2)C<(gwRdk1qDvi}h~UbM&juftwcR_Q+3D*`IiDp*_QLZSysgXYG~Z
zRIzuR^=xn8-eY^W7Hr=arf}YN!Gfm!``i2W7PbiN+pw<G-j#pKK7qDLdwc$_+k0Z(
z+5K^!BKHQ^f3d&n!?RaKVdLH}kCyLi*|Kl%s<lZ6K3Zq(d%C=OU)T&an^d+<Ht+U!
z>{YqCVlT(+puH}_%zHxjcG)n;&Dwj}_50q)-8=U^{a3lSI_jIXsYvAkZU23{Tf7qv
z+-c|AyNYw>-q7%Ud;Qh^?Ol=}?vT5-b?=hpJ8S~PB@Sdg{JlHZ{?q<VwTta9_8i`;
z_Q}*fihK56yPLfG&cCkRYdv$-{*7DM_8shhzGt=Wj03+vbnN{e&21;jl(Dx+oW<c$
zM&thN4NL5ACq3PJTxGs(4JY5e-0hyWamwrVtu>u@K&MRF{_)Jty|cGO?6=AAKOmSo
ze;=2j>OP}A()(6Edv15=S<fEMK(>9ql4kBTdm*}~@kHmo^9{~>A8cHD!0)En-s#>7
z`&zA3_En|{?(KW?Y;Q{1$36GYBwEcbdbu~fuW-NcuQ_{7&&{y@SERCc(xWYViyryy
zYrijiaO%;&d$;Uea==5-ZLe`F%RaBo(KZEamV51TR@<KT<lD!5zkILL^`-lx$`;yR
zivGDjx`2DX^R1tIJK`?eU;ApcuSJ1x-%YVzo1oCc`wlg#?)ky`a_`%FuMT{+p0w8}
zT=_tt;E}!WYP=7ekeh4&Hjm|i<cqy~qK-(}FIvX3@4g4)zE4|@?^|Nfb70?$68khR
z9$V?p=KEuPw%T)h7w>C`7umP}>HfWcJ}lX*UOjv7HhK30D__pvdusBLy&vZ;-}htr
z#XU2YvD(+%IJT!(MRi|A+|<4L-GO@#Ge_EFJKeMieX(or?3Ii6a{pX%pl#x}y#@aE
z)`xa<?kj(>eD7hFP5Tmsr`Y@QblXH6<+Yz(9=-Pt@9{n6w<qttk(s}Dg1|{T<v`wj
zJ#+5vP1^eNK+^vecJt4i-0O36i{10B`)!=vpV+3YIkC539_PM$hk5sUPw&~^c7DZP
zjrNp1OM0f-eUjtayVw8R{(2ssefoRN52VkTXkR$3a(~3i&3glmPq$_F@7OE1e*fOp
z92xugy43f_#8&PXiJff|_cL+-^yiB9hkjq)=W=|--gSz{_Dq<rwO{i9*S=?xxA*vc
z`)*S@`{7;>(L4JTn%?Z~S|h%nS#YMUEt~5;AKOcNHSb^BTU@+$Pf0iDzV0&z_r@g&
z?t5EpwtrI4-o0nfaocoqr|t83b$9P`2dRAu)zkNL$gtYBxA`29oL6k~?$iCf9aFyV
z^(otC^Cp+e&T{F@y}NjJ?Nu$x+b=BPvwy{E7JDV*y?e~H*V_g~tk~~)@Acl6PL6%M
zeoxyIu<M$=^i;-u{#R}Hu5b+BKV4_OjrSJ^d-r#H_fBVFuwRk+b$=Icpxyp;6?->+
zp1Z%ZwZq1sddA*UGmh?am!7u2<>Mv04}aF~^?V<=|I3^v+m&y__SttW+Ph(I@7~R2
zh4x0#DSHp?*R$R0d}6QmoaK9WAFkcEw#mTuinQc`@{~V&9vgb@(+INPYq5U%9=^k}
zd%a3nY_GUI+5049-(K%83lF?Z{jf(P=kMOSRJMJ)mig}ez2W!XuU^b{JRy(w-r;Al
zYrM_Bcj|}7dwqW=@8#R~f3MXhuLI}K&f9zB`<cA~Ie+a|B(U$7kZs-fXm82^WyK47
zjeQ&U&zpSEdd6nOy^0wR_I}mp-tTa}d#~NB#=Vd4N!zJC?%rFJzkC16yokNqc1_){
z<)O0wLCf_0`!Ai`D{%A4{<q9od+&VXvze6Iu<vtkkL{}m)9eyVdiO5rTws?_a>6b{
zFlrxT!N<LeZs_a_OqqMYJeP6bOTokYi{~uZ>us}hZ|pC(y@Cnc`woe-?moF{`o7hF
zz4q-s7`eCi{ENNm=PaxP?ws8_W#%@U7Z+yl)-esT)7Uk|dgFA)eUmPv><h_Xzjs>W
zmc0RqdIuc)boaDfIle#b%@-SmHEw&)`-SY;9bvt<y>8L|UEj~`U9<bz-p?g@2kefO
z*-m&Ccwo}DU^~BLSsT0L4*TPW-S@`)JGgh_R;|4=9?##`|Lf%5AD&D0@>G7YS@}zH
zkNbq~z1v=9?CF}3Z*%Bv)c$}rcDv6KPxeN-a_;>jxoxj?=$E~ZFPZF{rSNb6#1BXJ
z|I~T6_bpS{0a=E*`!`+Nzpwf?lWn9c>%QD&Z|vA4%547UG3>iLpM7u5iqCsgbXfND
zrA@OfSt;zWwCdEJU$t-cYHh6BbMVu-Jsvhw_O9`%+grY#eV<6v7MmYdatE%=mfFkS
z`EIZHg9ZC~t}<F5;h49Nf8v4zb2VA^-QY|+;P~_U-pI$#cHb4)wfDgJ8GBFVp4`7a
zf@AN``yMtk#AOex+WB;!ZPU5~J5pQhBi;$^>)?=aNcMWXS9D$d-tCL_*qAXZ?)S-<
zV`H^YcCXpZcY7_?`R|>0kJ)~DZOWdtPg(8aSUC1~d^510*!6dB@UN|Q$>)~rU9<bc
z-mS;??>ji*+5vmZhx_HO&e>Z#b=d*+fSvnej+gICyM1Ku%hh}Le0uTR-bQWy-iq0-
z`{!sr+M_Jlxo66sZ~GWix9$x*)@^^|+$3whptpNX8in@BtZCX?E6l&|T2`XX(-X0K
zBb%1)Rdu&H;4L+Kx9zsAdrLOE@AGDUu$RTLYM-&#UAswhg7)xS{ku<7e(s(?mW`ID
zqipxGpK;j}uXuEy>*c+BJM@0<4KX~h|9|h({p;=f_aAMTzrQ)~)!y6ktM_p@Ufpw}
zZPi}yE6jTqCAZt3$jRJme=u}!PbQC@^i#FHoh#e-?{JLUyQgcG-JRI$`=)Z++3Uvc
z-FqstXFtyd>Ae#~+V|+(pSSPLFRlHz8dmMkdm+AW)|-v{ckcbV_pJJnea$Va_HO5R
zX!Aqry=@<(>|U=f!vlKUDz<8wEqlM%z1?Fy&2I1U13mj!yz<+3F28x-au)7=|JTUx
zbz8u)@5+e{d;j`p?JbI$y}LB(pq*2A@7~=hOZUFM>bUoOT)-X=7tOs@QpN|ezV+>0
zak=q;|GdJzE55Sq>*(Rwml88)@7v-(w)rpE_HFxcd~elC<Nf|7U+gw&$Q|g|%DO*t
z=M$TY`RDDg&)c!bW73<w?dMqcPG;iU=g<<cH)r#ez2Xll4n**C?+ZNnf4|}*16!Mp
z)B6opCfobFPv6fWv~};k*I(?6*8bSbvWDHpcc<sR+aBBp@)lg%z4=+v-j=UN_Pt(E
zW3SK@yl>K~7@G|Y2X~uBNZJRx_U&CUZ}GmRUflas!mjR_<88K2=v(LBO`;0+C!RW6
zXTDS2JInjz-j{FLY!bx(?QPh++vaiS)jc+Ia`)cMy1)NtLfl@Kvr&6XXNm61`!2S3
zcfH@fb5a}kUuDYNQ<8ggU-I5l*3(uT*}ExZkIk9@j(u$BJ@)?!ez~_^&CW(<&K|q(
z{0jRG%zEtmo^9M0u>a-WVBR@<bq-7Ky}DL-9}k<~UY+Tl_SW}q?U4&UVk11S%g%D&
z*1Z!VU)wYM?%lIN{hU3UK+JyC%Zv9-%}Lu^nAo&`+Ut(JMU1cZaP=SCx3_e`J}VFP
z13Rqk_AV6qx344Zs-4EtJ^S8I{kKP9-RZrYS$g)j#hUj1ie$4d_;O_T|JMS0cW2Mr
z*Y#-8-k6;`_8sZr-*+-7d|!m%$DQ_i%=>u#nfKjs-?3NUafww)=U$sYHq-rEYuon<
zes0@4*U^8Uz}d-rv&-gM7e8EPUoCfhZ`|a*{bAvU_U7h)+B@63cdv2xu|2<e!tG<q
zy7w&0E4I-M;6EUF*<#PVm5uf~Q|8;Bn|^VRDce!ILf<gkpF6kh`Ly)F-p&~-ZP+rk
z_IBQRws&*LQTvuW`8~VO?X!!hk=t7mddPl*K+C@EOBdU>=}p`#dheFqQ>SL@b761y
z7~VXx&uibO{h~bk_giJ2*i)`)Wqa^yfxVTV<31}vxqVq8oA<t2HQgp%(|E750h|3a
ztM_|V=kK-I#r0<2q1nIousZSDZ8-jWZ*)%DJ~ibVdu<=}?vYDp+H0_J-=3!9m-am2
zl-qkS$ZEgz<$ZgVKHS}FEfTh`>?XsW^WV1Y{h6S<|C&tQp7-a?4&0b1xi970?!9UY
zmhMfwKHZu_u4ez2l;3+<Os?<k=v{i?XTXzvUZ<7!>u53CKeg`L`}?woy;vF7-v0`r
zdxN)f?~4!Kw(pO9r%l}1&wD!b8|{vG?%eyVevf@q!p%KPx{updcKzSG*lnHNiT;Yc
z=YNOqyL@>0o`tV}?)87$vrp1k|G<yZ=KUeh-|YHkTXx`m?bH36o~_>Zb|&}UPdYF5
zZn2qY@1^<CdhunG{coRrx6U{DzjsE0{l5DfI`(q%oZRnolw%*$|L=Q`2>sprYFdoV
z)%R955*jWxe;>WvyHziHZ>OdH{>Vvx_Ds9_Z?A^)q<zU^toszsx$k=~FJ{O3KyOd?
zy88VgPb&6KeB8CSjInQz?2BHTsDxJgph*YzxTzo5d*!>|fi1tM+x?bau<w@Kw0&DN
zPwdgzsJH)Vis|0Qs8xG+e@xn2zo~O?%cQM)Q`VO3m3_3yuB^yquUW%}{qg_L?A>R5
zWB-}!qV}0byZ7zh-DLIKncsfpt<=3ISr_bm#HhJXXTz`k9(K?7zjO`So9N_VulzoD
ze+1*|eY1p{_D(#w-X=Zs$o`H8EB3@3+_A4eeCHmv1onN^{rmSiWj@}k)zY-T<oir(
zp|Z*Qj?Fl;_Z)-#-Y*{6HX3IR?6o<ua_^Lg_j}wf@YuT=z27@G!E`Unmo@vUrmx?7
z`Q|s9dDkTDUU1Fb<GX9Q-RvsXeMO2B_D-x~+dV<`>R!)@ar<^AaPDKZIkorSr`>xQ
z`8xM0#yZ=*^PID9VxRC{cFXpC<{Q55eYoh>-u)Wu_C#sa*-zNSZ~dnrX78%x&wC9x
zq-{PbU9mf08ekK(uX(qbpM?EanHBpC_&@LI&NQ^Ek}=tvwDi;7RmO|=72Mu$_qRgX
zUhsO^o}yrWyS#TycJZbc_ho0U+IxKEw7u_`W$mx5I=1K6UopFFA$@yKY__+(@MqWF
zF1NPb9T5j?ZneGJyU^Nx-_ad+_U35I?)&3@$*TBy{N7g_0k&eRT=tx{VX>DIt>3E~
zx6}HXT>U=n{La0mh82686PN9Op%ZUo*Sc<heU9?pZC%Uu8j2j-yJp2K8@nPCTZK)_
z_PSM-?k)atZvPavX?D}2+U&2_$k}gJ6W+^KzrybJsk3{P?&s|_uGwiLa71xG*PCCv
z)*tNM>t(Ur?kU^qy-M$l_8m%{v*+r`Y5TZu^w^(|+_lfa&}lF4yA?Jow>Q~b5a-!z
zVy3k3gX`&i{RhSD_w6{lx4`Q9K1tK6J#%ZP?>m2>X>V@V0-FNux%(YYZ{2&?bHDwq
z1@HFqiplShS?{s$(WS+E*^RdB-*;yDzUPO|@14NPy6^767kk~;9@&*8AG!C`SxH-w
zYmfH6>yfs9*1%+$W?a5!V@=CGK7qzPcavrI9WOe(U-s|bJ+8<4_NTcq?2Ej-YtQ9R
zhxguY?%O;6c$R%_dgtEi%}4eI7+&4K;enq0vDE2%FWboM`*QQ2joq=$`yO^P?z=7Y
z%v$%w-@O9-1^YJ_ZnLTTV7d3|MXm!cW=-8Ivr}$gqJ*w>(C)sy?|wVjUG31eFS0wl
zSBaU|e)ateyV)Gh?UlW{doQ<vz=6*mcJ^J8OnWb%S+Rdv$-({C_s!lHH{X7*;F9Ni
zN=upTw#r)X75VpM|AK7|d#f@<_r#sKw2#@7Y2O2f`3L40?AvXkCTFXhe|qn4ztekG
zrE~6^w<p9xrFZIHR<lZ*@UIj0U0<4K^JTq_&E6~1_ifXexhLUj$-e0u-rAn%dcW7`
zaomBoT#Ys^a};f|pX%&gyq3edo1uH(jeR{fm)0`veYVo(z&w_R`-}Ca?Y}0mZ=WLr
z$KJ?)i|iuU7Vb51eY2O@>cU<b`A7EY^M3D%dv<qk_zQl!#<-PxX0QHhcS7;9P0~Lt
zhYU{L14mjL?foxZ+1s$Wd;cz#t2R5ro%ZgIy|J%#>ihkLpLGwM{QY-t?l0Q|-_@n}
zzxc3dpT_$2d*clkS_y5ewGURGuvdTDrTzAIig!O``Mj&{<ElNnzc1`vv9aVpNMzN%
zNbMQ>E?ix)XOidZy$4%3_9;c)+jCzuYHwT4mc7eGT5XTt>)xBD61=y;p>*GZj|qF3
z51H-NH0|7%cJtWYAL|a<3M5KfZ|u6eck%lDd$Znt+QZ@IyuW0F^4?z?8TQ(ppKRy(
zGkt%UF3-OIX;%BoC!6m2-oAfdcISn?PY&JQv)yuy%`984{pP3B_ZoI+@7*RDXXn5r
zWnCuiX7?&j$EJYw>mKWb)BEL2XY5n?_H%FI2A_R1PtLbdi}BcdJFR@*$=LXPXWkv!
z_pDWW_cWi^dy8)I9@ysFwXY|%bFb|G^Lt*4IqiS6a^K#Si52^0mXz*2wIF48k6PQl
zRcROY_MCseXK`ub-UD0j?cI21_THmWPxjdVZnAl>ZsJ}eL38U+>7{$4D!6Ta|6|_0
zc(2UfNlS9}&7JaaZ@%d=Yw6MkJDrYQdzlZ;*w3@Cc~8=j6MO14`1c)lxVcxH?}goh
zrwsdiS^n%k(zty8m+LcZE31y#ud%nYT{~gU-oxKF*p*yTwBFKxYmdZ#|Gi>XR`yF6
zmhN>+$=w^6$!R~iLT%Sn&1H5ocD~))a@Eiwn_F=Inf6lqkJXFzo?Cl-UqQj*y>@PQ
z_qO$3*f;Bn@PWy*t@qF4oU})2qUwR%Sf%}3S~>efS-JOCJDlHn`ST%L(_4r4PSyIi
zPnhe(-gjaxHZHOo_MSHswyhGau>Z8bZErTm-@Ww$jQh^4XWbVfBf2-yE@jWhogI6p
z{Cu(Ze(>!5!d$+4xzeWY+EOrM-@B|cdxfuV*&7ogXdnByYVY?cuC`Y-)>v<!plefZ
zV`Q_}dh_0G8cX*@-o3l`z#Ng?sVkT5=lmM5FJ|uTeY1V}4scIr+FLk#?taxd^Y;7~
zPun}~_SU_bZHMe7N~Z2T^NVe-{+Fe8FQ<IpyVGXv{>F3b_Z~FazF)>g!v6Ymmi?-t
zxAt!D)7Y=}W3Tl~zstL~h?nnEZ%f;ERmtDJuE2WlM!}1AjIDq6-SU{X&*<sqy~}PI
z><#dow*S_wDSJcLwe0`2u+#RXaNM5X0$h9a_pjLN8jxqlcfw)cTh5k!%YQNL3uD>3
zCu%O+-cH`;y%yC;do#-IY`B6u?Hn8w_DoFvXB$~NdEZKwh`qa>h3_?Ap=0}X&k38Q
zEGqlMuKn5j+D&+GR|DT3Cw10+duA-%D?O`mPlUv38>?Ag_Q&Z-@6TR$$$nDg>iyp(
zENwkc@3G6Sys<a$mBha5k5=tE=pkZ%Ew*Vd>w~p>x7I$f3$)y@S1nk4pRxA)y@wmR
z?7x}k**d&mZQq&9U>z~d$j<oTwY}AfOKiMv?%v0E=EVNHmoDr-X4`M0xzE7v3wwk8
zeeXkipO*aH%fE2m-W%sW+wD5_YR?6|Ec;2hYJ0@CMecbSd3K+ZsjKb&n=9<C$}d~r
zpQycW-T8}q8zy`1F49-qE5dtx&q-Bl8`1AP``E909O$`udQb4Ow>IyV&Di&5^|HN7
ze{A3DeMWWPpPFlXKhBtHcRlLa-Wfk`?yb9PzUMX5Dx1!wllB*OezdOMx^S=3h3I{I
zr}6DqGwiWn;uOC>cT&{81k>gFosRbI%}t!V$L-<2y}rzM_t`h}@2yeu+<PMTo$bSn
zC3~&D&$i#bTYaB-ueH5ZlE{JM`x^G%(KWCx-M)PPzaQOuwY?N=(+Vf-`#C?#PH=Yc
z{+b&dd-gaM?K=_7Y4=QR&0b-hS$kjHT(mdu?~(m$<-+$`Z~S4$;J9lqgV}Q%zQe2c
zO@Ft@rgRnW{@qVn_qHFLwvSs%dawO-?tSKd_xBh*xoY!nMx*U#t=W73pO|X5k;B90
zS%>1@Ct)-9x#_>(TmRt6-W->6`zOa~?e*e0Wbe7--`*XyeEW(;x%Zv9wQTQJjc)tB
zR_uFE-#EJ0*?q!(?%ei0n~XQxZ8H?KuUfQyugrF1yT0godm{yv_k1nm-BrYLZ2vin
zJ9}e0j#++tywUd8*GYSMF81%gw(r8;ccrlhT9di=+VOtdpX|1MkG7rd{<|(`_Z~m~
zemD2r6?<8aPu+j5p8G)IvXeF%j_USGn|SwEn{V7VtL6G$1*hqI{4brg&D(W*Pt5)q
z`?mC#*v#eDwfWvRf8X<hX?v%1zuFg~yvjECz`ni5yE*r)p8s_3W4_kCycfRgc_emW
zuj$U?dz)-e?|(jlac{58bsP7GJN7y4+qyU6tH@s8B&Yqi#jC9KtRfC1T58#rPCK+m
zNRW5me9!fJBl~mh&du}N8~LrthQVUX{+pT;_kUa1Vjp26XMgo}=3e>3-1`$bcJG~W
zt!M8PzD0YzPO;g~-jrtZ|Jr8jO?I2?jCP*b+c&e_{&!cP&E5u0d&}6v`*}p(+Ql8+
zw6{H3VBgfe-21Zl-r7uVY1_A2+kgLK)hTxOIpg*oYk0gb_aTS9QC$7L7oj)zu5#_P
z{#88JcBRa_z0oqa?DFnk-YYy$(6&c>+rAt2Pxt)!zjObmmkahzSU7c`;K86hCtGUw
ze42Q5@AvWrd-;zZ-?Omt@@|J|dIz>{(B0$xzQLArdHz1j)u;AWaB=KAbFp#1Q{2?O
zs;&?0)N0uF9&(?%=To}8%|pwsJ-iXy_TBwIZLjm@@V%#ETI~NE>f68g_Z<7=u4{XC
zUVOLb#oQhH7JgZ`SN6TdUe9NRHcO>C>?bZ0+}D@dzo$j_*}j;WDtq^6EVs|UplGYP
zoX>%k>*#*R+j2I-%7J@@#hUlMTX|^D2C-RtFMPhTSJJ0!|I?#g`_1@v?`1WavX8q<
za=+<?Rr_Aq$nR}vGuW%mkhlMksL;M0{}}9-Pwd|NHSEq_)}$r-Y7cAfO?5tR_y6|{
zTm2^feT)+X_DxFMz5B?##d{0pB=0rLIK0Q@%hSE4lY|bq^0Vz@GP$$2s(0Z&k<?dv
zlNK!8Ya)KuZqCBLd(|pi?XvlQ?cL?pws*eUs=YmCfqSPf*VwOoHg#`K((=6?^6Tw{
zpJ(l}YUi?_vSyOq5(j=8mBli9_t(wXo29>J@1KSgo0bhrY)>+=?CU?qx%Y$6L_3W;
z^K3r4OWH*!aP2u<z20tb>b?DzR+sElPaNL!VegB5g&KW(ZM3fKITiNWn%B0*PQkZv
zzvL$AeXAt<_ImiL?QgohXWxQOhP{5N{CoLLn)f|&e6_daV)Ndqi+|YMz9VUSKjz)u
zd2JK-R=cs-&yV-rw^O=sUq<QEz2^+(*c`jJYp>LXNxPei%x!Ocy}vhV%6jX-o2545
zhV1)#zAWAM=l<tC{@hD!m$OW;H-0r?&#n_U_Um!)*z-9i-6qsDbFa*Q-n~sNo9y0X
z7w(lY?%pG*e{VmR%qF|TUuN&W_x#De29<w%{InG9cz&qvow=yT=8?$dy*@6R?UK^e
z_r&bGVk68r*Z$S!=)K)%WDh*Jt!`_lRkdHXAlZIFbIiVr6-|5fy1ng--wE&a$$PlB
zZN1yROD*O00iHMa#S3We-5)XAzNToO{VKL8`<zcs*ki@Gd#|Ly7yF#&5qnQ`&9yfW
zXWRF)S8?x^U%&UIeEz+gQLFDjZ%Xst{~}7ZUmhLZvoQCG&D$&Q_gY2z?XA3ba_`eO
z%=`5A$?ZSLAGlY7-FWYTYYX?;t-ilk?&J5paXY>D7tB1lmwRo~zOzp2_8f74Y9o?=
z%w|j3?7bC>IQK{UH1D<4e7pDb^2Yt4Dogeq(>rQE;ZuXn9$Bru`4^b?UR>I>>v;b2
zy=T`>*c-aM((cx*mc5S8-tVbhsA6v&_-e1~n>F_Bb7oj~?Yn1ZH*@yB1MKYkOIHZ*
zsf_m9=kkte-^VjNHWuvr_c^=i+2_?T9dIw`vRTHwYu}0otoDW6+xKQU&Dz^^D{(J_
zarS=O&vkpPEII6d{SV*G-L!r0Mg^&TbMNomJ$dnYyN54S_ii`o*e6iFcdy{qX?wPG
zaqjDAW!PuYE@yMSLU3QQ+=hLo_wMi2yne-YR@I_?nR<~ng<2tdS+;2H`(N~9uVBym
zy@sE^TgR2Y+`Fdd<zCtCSND1Z7~8H{8nairZ0lYtrn&pn{=V2Js>88Y)_~!FQpoi^
z5=R>MxzxAp=JHy;ccFaN-t_qv`<c^f_X-3t?UU8Gx9{<l>3h<|KknhZ>AUA?(UyI#
zH)rjSKl*r|<{{&~*K_9X->@Qc@47qewkAxr`-)~A*k}8Ct=(_dOMB-WowoP6>z=)K
z!87*ho||iPaKqQV@w(jm?z8AwAN<I_&!_y--i*2)o5$5t_8mEMZ*R8cnthdeGxuJY
z5VUV$)Q;WeeB8FP{7v@SoX*@^adp3qgb=?~Qh1)-DgmK=Zh^wP10U(`Yw+Z?d0@0>
zuV;>w{p2@+doAlK_K6)}**pK9v&|GKmVL^aZ2O*m4YN0Ju(iqBd~fgb_bcr<7c=Z<
zGFIB3BOkektDw(z?s8tc6wgn4nLYOGjjy%ZdrDE>UTvzl%?_`Iy<+oh?Spye?N!^=
zvF`)tJ=<AzN_!7C_3YpJgwH;GYu{dR?YezMWpnoOv0vP)d^>ty%EV>+59mDEf26N<
zZ#1vzeh!=d{TVO!?CTGDzjueN;oeVztL#J`Z{Pdp_niG(tNLtwye8NlPifzqb}Q93
zIN-nC?88_0O!*?WuggPW?+oS}HdnMtY_u-T+cURr^4_*x5qtOSsn{QG*s|y3T9$nd
z&x-e5U8KDC;Vl1sdR3Ec{XZ?<J8R#mees5Rw*FJ>cYpbQdr$b^^?SJ*z4re3&ay}2
z_0GLHIiKvD*k0`WQ1H|)jc57ZcYY@OKCiuKclsgc-ZxW^?Cm|gU@uGk%)MT+OZHka
z?%&&xywdLF=hVIDasusU-^$qgsM^PFLaXLpsiSNMdYBIHU0u6(zb@b7J$t4}?n#*_
zzOQS-{yp#F9_>#Q?A$v;vv{A~P1}81KJ)e!>=xU1`Mc2GbB=#(&jq#Y?Gf|a$9?Jb
z-i*!-dsG`g?&bQYvbVCk&F<|JBO9R@`+f6wi0-}ZJJF^#;f3`!pH+KLPT{q6tl4L?
zf9;ulqWsN!w@qHM*KuR?z9#oTt3Sd!ZMMA<v`_Zqw|#z4-?l`N)3zpl+ulCmS9=Zj
z?cKXYFwj1>mSeBu47PndhL`NZdL{Nd>{w~%DWYbdwM=;5%)?!FH>U9JD_DAFuiu=Q
z-B&A&><b%C?zN0xxp!e%?OrXHU3+u?wb*OAt=r3Eeq^5j+wQ$D3#1)(8m`-G(yimb
z<HWu9g!uivA`a{Kao+iE_h{;#z1<rQ@6B{MyKm}c1_!fmTlO(~Z`rf>*2O*6T~F;4
zf649@d8g+PG{bYx8cQMD^IkKoR#j@)l+25=|8a?XuZ%|EzWVcydy_ln?L9bUrOjro
zNqd@Dlx^LgEVJQb;I^AN;mTeYjjFxU2e|fetUte3Z|<bM>mGI4&ws{hvr^#mKF)Z7
zy%|Ld_ex|*?p<|@cb{d!?fsz@x_dnzDDG`Nyut4MwUm8Jsv_-JRv+1S`z`0beQLV<
z_0$FTe4b~r_t@Ksd;Rwt+b-C+bZ_Kp*1ffI|Mti2`L|d2>Y06Cjd=GO+HTpmqfE<w
z?rr0JvrUrs$~>66FJ<<+y?ZY+?h9dGzjyZ32ll$vul6m_Xtt^NDQcs#X{~+ZjFtOl
znTYO_wA{1T_SZx^hDoJ+k3`P0pJrsa_uJmCJ$1!adly>u?5(hQyq|qR%|4C!f&1=!
zI<oikc7;8)ee%0A4m{d3Wx@MB>&`apoqA};-s68J?zPrlytn%D)P16V8us|ze7SdS
zNBF+<Gg^CV9HjT|?BBMh>TKN}(VED;i<~+3$v@SyOP%y??|IL38|{bw_Nyfx+lq^A
z-(|h_`rZ>fU-xW!YGo&J=i8nWrZs!FSFPR?VYXo3r?T^V_k9Sp(fjwpPF?Nh-XkvV
z_BEgQ_wl{3v|B#0YTxyzEc^Z^{NFqA%t^cYpjI0O*IRp|y0fkKzH7CA8sV`w@V0@)
zmM44ne->F{7hC1N@Am)8dsnjy@4K;8V&Cd+m3^$8iTh`1P24--^z6MGN|*1edhozT
zU{SQa*`$U0qGmSj6PNhEw_~#6KJO<idrx<-v=KYWw0G8un|mcz81B>iblX}rPQ><1
z@sfSB4*lQT=k{ywqy78slA8VZRKGN|vr1y!mv~8L@2w>Qd-$5w_6C~x*{hXW?s-&K
zxz~T|dz+JU?Do&8%iBMRp?_a|ljuI@?acO18)NrQIxoI&>u&CSRfczMN@E!I8sFcx
zSLBtkeM<2CJ=5<@KA`ty`QF_3b_aCo9S>x)h3^0IYuesE7iGIUYbV?2^}gTx`uECx
z@_hyea$`E}w@BUH>!cfJ-(IlE&S=NceR0_~dzU%t?Y+wLWS{HFS9||$dB0D2wcx&Q
zcQ5ZfQmwm}{oAoU?R?AaEjLH)-7~Rq-@DBM`#$+^+?zh*{@yRUboZ{fU9~s-A@{!X
zYwPXbItA`!<@>y6s<_R*ZsUu4?Ut_FyZq9|{dLQ8_x@j4Wxwm<#yuOJ7TILUoU!g%
zUAm`@neD*Eb-a7a|E%51@YZO5>Zh*#?>?sQ6%Q}mf0b|cUcQ;z_XXwl?~Qm_uzTiS
z`@M~_p8HiA*6*$KW8TXnKiRJS=as!q+aFs`6kyyZb5dnL^E{h<7gtu;)tX-0`}cRi
z{#gMEHcwyl?2UQ0dS8uPfqm(MVB2Hw4(xr+@n!$6^P+YxmmBwO+w^Yl`8$922E@JD
z$2$Au-t@qm`@SoFvtC$xY|ol=;`^%0_U+Ye>D%WRZ?R8wSNYyo$}xN87Uk?&<ZyED
zTPenU=l1#8w5agxQ;2Qae=%5V@7Z(Sd-JTl_O(=0@7<bNv`1TJ`~D!M*1Zpc`u15j
zaO|zRlD_9s_b2PUCMWltUBb2hbsW>a!szXLR(-m&cj6hD{deDb?c3O|ao}Fc%Dpz-
zPwZq>t@mXq%I+4pX}njV^Zov%SI+GDCbZK=Y*zCAl`?zxMy!5qs}kH|v*MJj-My}N
z`>yBAvi0A!aL<RA;d_5?<KEj}#JzXxbK8Aa{n_`s%qiX1+_`;k_bTT7oBOWq+rjO-
zH%6$&M*Q57J)N%q_HC3tzBg^R{{H@5-*=zWG`7BaXv;p!32b}0-%0M*&RD)TwsFZ`
zvr6H;a<>oe&AxPPuR;TljoPEQy)V^{?LIPf%f8ueXZQY`I(2W1zvjNv`6u?gU*Nmv
z5v#_5hdfbx_vm)n2Q>e;+Gkd}_h01Ky$&;W?tS}xuidnbm-aG6^6y)ruw(ybwPSmI
zwNC66`?SICU~$8qqX~L;;-;-OpT!z%A{O!PyQ6NtpWVA_uimH6dpTBbv9o$J*`|K>
z#l4@J&e)uLaM*7BzE^h5n;+WsCvM;SxXH$D&L*S1Y{$0lovbRqkIRw!06)t#`&%EE
z?=d;*X#Y@>X@7s()qRaj;?|G&FYjG^Z}R?Asdl?E4ky@{+S=^7{A|@;b6dN82a1;L
z@ittwxBW5KJ~h^P`&xHS-}@xU#&+VF!oBK=Ec-a$?by?;DYVDY<;ebf_7ir$j(KX`
z9`bGP?-~2|a`Kz++d9GC{wzm{wQfb@zO}4(_Xa=n-|PB#>E35{e0z2N_3zV}HhWK^
z@AbVKE^OZa#H7GB>yMG$s*gGQJT`0Z+b*)(=E?CpdwbjMY~^M6_gxp2+o#|2X>Y_s
z*}ZdCi`sVxs_s3=Uu~zlG=1+!IiGz84F1_&zVh9sCEUzfefzK7@8-_hYdf=R@9x~d
zJ$I(G?e~e8WH-C~?p~L6Cwu2!6T3;F*Y_=E;oPU8rLvc4Der!nqPKhbKODCUkzlkH
zvg+6ySlPBO#*}4`bN$c#0*zgJzb;y}??gJ&K67E|ef)ik_HqZY?Q{6OXs;U!?>@C}
z+xD$DsMvewTZ^s6+tz(gME~rW&Mvl3q2R0ClAAC0{(W3yKV98#?>nwtHnsbw>=ktW
zxO<O6-2T3snKrx<SN6`Vw6ec`dz<Z5`>F%Gt2ho!ahSCC!HLHG^8^|9o#xTsTc60b
z@86yCcF*g1_hs1J+IylTaR1c)4|^Fsf)A{&V%s;l`t&|t8>@Y<{$1W%U%<U<GJD&8
zsgL~oTATa!uC3X>k7wm7`+23e>{ur6-rKFKXuqgQ$ewZazI_$jm+q<mrM@>kCDATV
zdgb0Ez8CwgbLZ~8xUh1s@lv&Y(IHcJw{hv(Ph1td+r)CxUV)%FHv5C`?`?Nw+I#kw
zxmCOGyS-oYIQI2$N!lNa`LlP0(3-tDGxhd)_|4jTZy(=2)$B?8cRcmkE3CO`|DAT$
zJx!CZ?m3~oc5iE}{_dqKRQE}9WZS4H&f6ojc#HkJr=9!7oEPnVy|vj+Fu-_kSK3Lt
zi6Z8E8!h+k)%IF%{qU>D{tff*?OAm<c<&r3d%NkA*6f)-ZGv5OfR61IL&1HjUk}*r
zogZqi$h2ec^2>|%ykUE{cmI|6y$NeB>`Rt<XeXl0wm;&s^uF>--21=YR@!%G#?5_?
zIG$TE7mM$!^}AwwXtk?N%N{m+pGo|CU;kjUO<UTwPm%NJUVfhm`@^5K?`yD~wohym
z%f9K?-|g0|p0L+N=FRTl#Kn8dcKq0@8XRF`e3@^LXlsGZK11Do4$eFG9@uZQZ?lim
zffX*ndlR-Y+FKR~?R|Z0l8v_ZsXh0<^X;2a#bs;!@xWgGV*-1R9yHiLf59%hg$L~R
z#~ADGXIt95SK)^Ez6hm#dw*Ck?(?cyus3ag^8W7gTXyRTUf(-&j>&<LQ@8ibHC5h!
zeE+OH+m>A3Cs`G2e@OJGEkC=?Ua`Xp2j*S>xR?2V!`_RIm-iigS+if^VV3>N_aggR
zdtK~5va#)-m414k`%0#LZ#fwD9o@Oot|rxG@3nbP?4B=Qvv(E0nvJ*CwY@?`fA`9q
z=CtOn<lK8z`GK{^HSWDRw^!|bT6|^CL^;NNOV&5<RW@+n`$1W0zxnS?do8?P?>Y5<
z#=hIts(UStHtux@zF^PrX2IS~f0gZzzNpz{^v`-1U)9V#xlZbPS#-?z+r=gBUA1w)
zjd7pfzIRXO*<OfSVaN4#kDd0<%Ds$2dHYY#J+QY%T77TEQr>;V8`SsBVK}h2N^^&e
zKrOd@|0ltHPL(cp7xwJiTUmR_E;BY`-vskdb}Xz8yXVOE*d?&6-fMG~b+4?#!F|G4
zlk6w*rrUe-?b&;(fq8#ORE#~p&CGq*+s^C>pD}rlKi8)HJK6PYxaW1))ts2R_g(%*
zo5q%hdu?py?ahi*vrlnj+xPR7^uATCr}sovDDK<;amHTlD-C;2U3j`zDEHJJ<41n>
zMez)KKWwbo>rz;_k2h}J9_>YU_d5FQv)g_)ZSNPZW_#wx7xu`tFWP&-qtB)xKxXfa
zDlfbBPK)=3uaw*yr6YA9jy1;q|BKuMPdt402hHW$$0D+8-@NHZZPwrTxHod=1e-=%
zGrOd9ulF8!W^ZlB&1jP}HFR&~<pcZe|F5*=yeDzsslwv@?9Eg5%T+AiJN5Hs+up2-
zy$L<GdnUKt*!O$w^<6=GC)!W{{&x5B0#*ALYrO1hejeU;q;TQhoXt^tX72g8ulv>7
zy(=bZACS3nXm9<KCwmz?^7hVq7-?g~&c9bV{M_F2-zV;yy6D(m6aD7Bav?AF3W#`H
zS-2+eO{gif`DK1?-{Ve$y?1Wj-20U=XWzaqfqh?&f8NV5vu^*p{`$S&mD%^7nAE;^
zU2(QeiT3wBB5`c{EIp+6w0z6oW5~d|H)rh{`((!Uecxvtw%zC3VO#szX7Ax5F8da~
zx3&&X59}45H+Appa~t=Str6Mv)@{+=GZI<{cE4fV`}N<N{fzfGZ0}Xa+i@wKv%6xr
zVL#I`&V7Z;KkV1vzhbY?nv%VeCYSaF#5eD+JE3aFF=g^z!{@E`ygh67DqY{T?}~oc
z-jhF+_kDT5ZojAB$9itL@`2AAB=)rie6VUr`LXwOlfhmK`Q7^-%s#$*p5x?whjw)B
zO=5bvSM<lky)BEG_N_TLb?-4dp1r%5wAoEft=SuvZ?nfH+kNjEqg{KpnFd<3|71L%
zAo_F9bKkIiR`YxJNHuKRYkvRj-i3d4_p%F~v{$y9w^zaQ<KD`9W&2Y$ZrSbiUF!fR
z`wIKe%%Hum{xsXm?eVgi?q9y=pMs~2*QO123l>h<o8s|&@8|Zm1F!X?_O|}-J`m(}
zVUI}ZtOGnxglsc&`}bc~e6*LTMC3qc<C8s=)~ohrPTsd~<>Z<Brq8Ojzy0N)&FQxx
z_H$0y>}T89voH9U(LU|}>-JuE<+6WsLwsM6?OOY%>Sy;Fd}`cl6nSO$gw2ch+9;p0
zn>kB)Z;nl??To(2z3XRe-y5|5^j`mM0eg)De%LI|khkG@x!kVrd7lmM^(q^_XyLu;
z4}R{o5_n?s{~5>rDa}lKE1$01zqRlFUM9Bld+)Oz+^b+KwKwDHQ+vzm`n{)f9`D)z
zch$Zx@6znsS5_R5*P3Nt>U?)kY4Ke9hkTFsZcN*^cdb^wjWqMjeG85)-#g3P!{)Zg
zUHg8<88!j@xA!kPHgWH?Z*}{<4=mb~5h{7$g4>(DHD_z=|MB$i;SpomJLA!ueP5Z)
z_D_k9->1s2v)5*l(SEyYXY4nAez32_YSvzdmo0lwyuH1zu<rb>McXIYPKf)ox1(*<
z-nOFNJ#*gA->s0OXm4MCc+W(s)_u26UEN)Ebn+hEHTU-DZ(`ZUn^tSfI$`48os$&z
zH$Q&2_e`XL)#MuCeJ^KM@42VLwr@wih<*LNb9<LYtg}}ZUA=cB&+R?d_x1OF?aJCK
z&01==;i9_Dl)cRR7#Ljbd%WH47cN_~pSfOd|F4fBdnX=#xX*rHgUxH@w!N_$4K|($
zZg!488TNY2HQ&2YX7m2DDf{;JyL{R|*>Lin&dpl;S(Ewp-CA0-Keg%op6rJ*`={od
z+cU?l(dLZCecPG`EB2ZtHSQOh{d8}Mi-di5N!Q-gooDwczKY)KTPC{ibg1<H{So5(
z?nK4ypS|zW-fI#!_w3{L*<Dh<W3S9p5BqIiJNHJO<lXlxn|JTV$BXxrPFk_I@=V`e
zw(5O*9UiZ@k#B!scTDN)p8EL(d-Ve*?Y(d;%*IRKXs_AI0=peImhbiFinIR|$Y;w^
zptILLF?7$bWoEn6q%-%ItejypdF9_dzY~|+F<g_j7jj;(f69vF{lA+n_MWSa*?-^m
z^<I7F`n{2>p6{_^NU^&t!?;f{K4VW|#mT+L^@a9TJUzGH?&t2km9te2<a=$f;h4^|
ze-7)8y>q2v?T+l-vPYO<skKI0^S-4~YwU%V#M$TRE9`S!=56<P8OQ#`XSVH=mN>rm
zxZ9$=%ip@}t1LI$yEy9L{{6jed(+Q1?G-5avDZj`pG~^~yWP<oW$XXF{QC~+ec!v|
zVDny6;rW(p8#e9Me7I{bzw?IOaVjNt6FxoID_Xj7@1X^g_8G)K+WTj(;y!Oq#sfcV
z3io=cbJ(>n*|WPhjCqfZtDQ}(F5^Cl!gF>X6j}FmG7Im!?UcW-NMq4H`(^WN8@!g;
zI9z+TN4!yb&jS{@eS6M6+<SLp``#w+jr(`BuG|}NO~~dE7r%XACd)qUg}HY9onCwO
zU#Qt0$Y8PevdXgEwVQdLd%`^111B!+ReOHg`p4b&eHrZT`%g_gWmoZndEdqInKrlm
zAMCB;mf6Qx*1Wgx;nBU5HgO(!DiFChOZvvXM$2h?e=a|`*QqIZU&e{1y$$!9_6yHm
zx3}%O-@f%NrPk_~?(K>Hae7bK_S<`BuaC50e(-(I9M`1%L30)NCja!?`^-jk-*<bV
zeSX*G?2!s#w?FN?V9!lI8T+8l4>q2PYj?k$nYGt6$9b>ljqH6}EKlw2l`OIeou<71
zM6;NERao!-H`Nw)n`Pha_1kH_zy3V;o(=aO?_EEIcb_dg>wcXtSNAUdT)cOe@MPO(
z>D_y_yKo<fV1B-rZ-d!^hVK{lX{%Tq@Jl_jXLW+G{T+q`D`(fr-KrX^_kB~Xw)&Rb
zY`^BQrFEZxhus;W9k$&p2lus3m)&RO#kg;d)ABvXmPPNC-QTo-N=)-!?|GZ|Ufj!U
zd*4aX_H9(Z{jVKo_FijSyw5FPeb3{GzxKZA@7&w@ZHA5bdO7P?FKYJeda`)`)MoL$
zZI7nzU98u)Z=!?x-jvF!Jy&fe?z><4ZSPMH1^a{?23x@eH}~>B_+&lf`RTpdLS1$e
zSC{Xa^7DX=Zf2j|TD$rCv#y`C6`x#bw?B|$@0(vqcCKri_8yl$zvsH)dYg5t|Jm$W
z_Fzx6EUS&`rLY5!(kAR>+%$9F*%HltT+K%IkE%Eh>|NAqU+l4J@4?p9`%~|R?Y+E_
z-8Mz&^ge|(E%xebFYR0D!Llb>ZM9v4<B$D|@=okaHSO5j`CM|ZzPYLWJsp<4`{zdO
zH|jRo8}pBK-=dw``#z_w+`CO$z`n|a-`4hZ;=ccZ{d-eRJlcEoP4}KQE$O{~*4^7X
z(^1vt<@DEfuNEKJYkt*muc4*p-m|}%_idh|yN6kSjs0te?R%A1tL!fiZ`*tP`r|!c
zzOwA&y?bu2*uVRB-18^dXx!x8cmK-!eH&QP?YFw@+2<H8Y~Q-*v+cS(>-~Z2toMGM
z{9y0qYVN%+%iHaumr3p|xxH#{?QaSD=ZjPK_TP=$r`<Ab@BK*OeTjO!_V+If@BhKE
z!RBaJsa+mp+Flo}DSMQ*w(YZClDS{SI>qj%<L148r>5_#T-0TMo;!VCk4MKIQ#sCk
za%U{}t1SDlS8*cu{*vjl_Zt4cy?6VS#C_cVnfJ}TXm3CL@SHsf6{7oE+!gowt`^>>
zzUs-|Sx?XGdCvZHFLUOmy}#;h>^*y*?Y8;z%4Qma^4`a%C+_t>WV82gRK|hLPcGQR
zWYybkxV^`QwdKOz{@Q7K1aB?dyZy1yew(MNdoDfZ+Q;->XTR(}#{G)+Z}+O#Pul-`
z1Jm9~Tj$#A?GxB{By#Cqg_+KK{vA*_a5$Z3pYL2g>#3FQ`>Sjk_C8eDZW|vivu}yk
z#C^AmI_!7-+h%+9(8-;jmR{bwT!U#}9P{bDwj3w+xi8e(XW_46Um@DHH~HSGed^(#
zZ5IAz+IOf!d0+Fq+j}pi&E3Z<-n-{g?mFA%)zbSMoDS}_-m_}otEt_4@4V>Tx8Xs~
zUSW|ETcLv0dliqg>`4!pu_r-#*4~~|TlVOkSiINRY1h8DmzVBQcqzJX!u$ArGpzjf
zI^Wr2Ej_=%&S?F<y@9I^?)x=MaBuC4JvM=hm+ftHtKH+yFJ~VU?QWyJxn|E*gZp+7
zS)%qIV-M_~r_H#hsK#mUZN4V^z2A52&F5IP$3MPe@5J@1_WGDL?G<Rfvs*us!~Xg6
zX?tBi3mn+vuw!q5?y-F>`=#s`U%s>7r}mrmmR;-jF8nC5H~43njm*X=`~04`AJEHr
zWH-<7&7O5xf9$8;T(nQ`{Ox_!yZZL>wn*%&St_+}PF2C)_rGTE7m4ZFYdUTI-j^%F
z_nuH%V#6X<vbT70nf1Nr4f{A>^z6;!d$IR~-@`q}n!oN1-Z*LRvYcgm((|_3bDw&*
z=NbQ#y;WIVd-F32ckAxlwRej9BD*W+IBn~5nD^&S`nk94gYn+aKcDTLrnG8rdy4jc
zp>p<pXXZET{gFCx-<=s<_GLwRHWfm__Es}H_Syyi+w1zN$cEV|Vs~pjr_JRLhioMl
zo!Ygw>ENztZTbBMPipsWoONQ~`j476cb}}?yQ%fC9bd`?+fZZuz1179*e=!Lwc%l5
z-K$r7dLP^IO}2}2gzdXR^!IX2y|AxL!_%(PzH;9_JB@u|3s&#>`fcmJc~h_LwF{cH
zZ=TK0y%u5&do-OY_I)WX-SacU*6vEP(_Zf<etXWS3GNenB)xY|L*d@KKIPrlc4qFq
zwfx=Qr0mmn!HKK({1#bYvqiXj-&J>U>m%FaYz@+K_bbaBw&@g_YHxHVYmd^#Hk*l_
z_Ir2x3GFMH6>V3(YwsTJ`QP?VJJGa1YC)pCTJo#?PV*<)CS_XLHn*<a@3Y{NO?%Mx
zy(!!_Hs(3{``12T+;@oM(H@@7V)ow_9^8Golk>pEYoUARXrHoEeyD3dRqw#w`Vf!3
zTlQ_=cj3p9y+_>ociU^E@0+W}e_&h0BRiKcj=eQ`_I9_rME0*fbYUNV_x!zk5)SV9
zx~X~p%sVG{Pka2+&ftvvzN+7ndsE`(+ur=Ob<g^D7IrDyZ|@D~__Wt<!K1yZ#_fAQ
zO)cJA64SA_w9L--*!-1yn<oF;zva-9y|PRxd!_oj_l5{Z*f^i}*t6jAF8fbbv3p~>
zt@p9jG3-@8m1*;=apB%9w#$207%A>IUlhJ~t;p-W-fg${Pc1!VFXwY^U(Kbd`zvdn
z?3NQ{-QQSaZ^O~sxVOE#-zM~&{(j%i6V}a5347c5Vh*gF?7a8dTcrcjJ~-}Ws_;7S
zJ-WmGQJUi3Q`Ox2-u&UUS2cRIcN+VSy(=4d_QhLWw&TwJX@5ehXYZ^gN&7PeFZTb|
zS+#HCJKeqS&lc}pnQyzlU2ex-Wr6edv)3~0i>_?6Ej-<2bLeQ+9z%OgdsElOeHkIo
z_x1_b>^0rOWvjuKyEpP#$KJEATdY&MzwJ$&&$jRL^8I^VwEpZ}e`(^rqEid@Ub(HY
zN0*<+zRdFZULjX6yNL|D_Z|}Q-;>wFyw9b6pUvJGYW5}fIQO_MX54qRXP$k@H}(U(
zQ3vdnIyu=$YEInyJ66-qxsPF=?(}tg%^chJ%yK$sUnR0&@7@D{_BLOhYCpG}f1lZ&
z9rm7@jeApPr`jJekKA`8?x+3Z3vBzUpRcl;#niSptNr5M56N@(MX*e^pD3zh&+uDd
z-)TpyeH!+^?KiM2-WQuceXm=Nz}`oP_uIeN*<mANZ0pb+G;Oc$jPkwJDtq>L6)xVp
z@t2bQ_bp+2W&FDKy>;T-Tao9m_vY+}yLnew?iI<OxA$iU>%Isj7kjBs)AmZLH0`x0
zdu5ZkNpJ7VV6MG=t+o57U*NM%VNl$A`01RzY8G}jW_)~m-#g#jd(k$~UioR)-q`Pt
z_s&c3+3zqVX`k!GIs2RTH`xodt+Tni^o*UM&Hp{_k52A6bYtV*5HbG!w=LRjo-F;j
zTdpJe08^~*-VCO>`~PWt+Z*<D+x`U$itMYHh4zX}V%sMy6TENoG}FC5R!`l1{7&0G
z(V7YSuYAk23w~CycXE=2y_@T5yGbSU_i5(*+q*78XRmIR<z8D^!+q`_*V!dVYVGsl
zY}sr3Zt5QORZDI1QUa}hRZO%|=DlH~7F@F@L23HlnHLhRW@_cz)N*F;UHfL~9vS_`
zw$DG`-<$h7Wba<?Uwb#C81H=)wRTUI%G>?=oICesEKAy_9(QD~*wu%7Zy#i^)?Bt@
z?^FTt10F%LdzEjm*qc|wVP~{K>Og#Twe2M@|NV=eU)al*am9}1TK?VwABDZYuLtf4
z>5AKb?OpTUe~Z8ES@}iFcAxUwy&SWC*dBh;x933k727`_JNBJ4e{QQgf88FQ1ke2p
z>~r@%7y7#Q^3hfMw*6JNllvUJZ;ABdz3Z8m?Em@Dci)k9EA|!r-@5zO7U8|s*Qe~=
z&ndd^)5CrH%T6}!y_y`g_x$I>`y6ko@12#&vj2YP%e^<6Z1x#cn(wJzcxG?As>)th
zKHt5c4)*NcY4&?B!wTDdM-RT<%l?da@3WgW`?lX)Wz%!^=ich~ZF`Fsy4a|1tgu@#
zJ$aYCEz92RZ~OO3PrSaDdy41Y$$pwP%-*?sr`_3SH|N@PyPezK?zPlzv-`3wbg%2h
zYx_8Feb_5&xOs1)qx9a-lNt9%CY|5=;?wTED;xstDyq2lElCjC-_^0$)*<=pKIe|7
zb{C#s-CxSL(^|aC#lFMi*Pa!vm3x0&n6l6Go8$qOkIU_Ec(m=gvOvJ@(yAx>*8Q8h
zFJ-#+-W_#H`wq>H*dOn@&Zcr_f!*!vdi&H@YVMul7q!nxwb1syx~9E@+@ZZ^l&9@u
zE@0c6ROG*Ba@XU%pML+{8(UYk_vrIAHe&Y4`<rjB-lO|FY_CXs!@kqpaW?BS&hJ&*
zdT#&TTPyb(o&9CE*`IT7eDTD+|C%oC-Sv`V-#Zo`JNvntZJ1wm?LB?7V&4-fG5cff
zyY}a5n(Xh|E3x<4*EIXh&9=4^j<4P8#r$XQ`e}CiB={5eL|hcv+cTm0z}2?hd#z{3
z9SB)+W{;2g%zaOn*V;Q}T->|tj=s&*<%S1X>-hJb<MrIr_GZ$)GNp(6|9^?L4-HY=
z_jq=m{e#Ds_G_KKvX4QKb>H%%OZRRN@UyoLxVyL1_Vzx7Z8>{6o=&q(QB}10dpd4!
zuJ6{p`qL-u<p|xhci#Vqz5d?2_B7o6xA(T;m%R%m^!BoyzOmP9ht|G#EHCyhe7j_?
zaFNhn|Jc}FY`+t1SyyN8Uw`+W%|~gy{hg_s_saf$XY*cekxfA=$G(M!4%x|w=j}DK
z$k?mWsj;`R^!&aj_UZe6ex15k^CtJc81<ue$9{S3t+08ySIJvo-}fK<4hpV!_O@^_
z?&F%8Y^SeQv}ZE&e)~J=mexkaHTG<Ce%kN9C2IdM!Ex`eC-dx2&1~6Qw1Z*aTf0;H
z6ck<UwtCArgk7%QTYsz3R%UmPebUaw`(}8h?v0Z&+H+awn2ku_9c#_<J@!Jc=Il|B
zy}IX*N&jA->-%@7>|)xV{&TmjKuP1i<CDtwR5m`}W2(-z_tJI7ebvsp_jtbAW22X=
zX4g=aV#D>Ieecs30sCxuoA+Mm-m$m-#+ALtD<1DX({*OQraspmo?YDgoRix2s=Yn6
zcgOX+dkbth_Zgm8WgUGg+5YW8<NbfXG23qs-nz#<%x<^5Dwkd8jT3w2blCUtT3+7U
zaQ^7NjkzcGuDTs=<932?|CX*-dtcr9U^kn~-}*v<>b?$Jd%LVX1$J|M%WcB$H179b
z%eD94nTa+vz3cW`+_1LKs+QkBVcX$7p_YRCO^zMd7d3g&KE34vdsC9N_B@d_*l*Np
zYBT$np`G_vu6+zk+xBYn2Jib{{&KI!x~KbNWoGUx{OGf<KdoU;LU7`qmHc1#*7DWb
z@Jr6KDKRwO)BUr1pLaX+KA{A*eLJ?_*lWRlZ11tE=DkjZ6ZZd3>D#kNx71GBka17Q
zTBd#bZMW=J4BEff>;<d69qR(?OFXxCe>k{x-!}p7eKK`x?DlA}?OWjKwzqt3$Nr5!
z-S%ykXWbXtH+4@%;D>##h5CE7&EMO+D!;X_=jxVSCE-T)JHD^md+bf5{Q{obdp8P*
z?9)rxZgb98VxLUgp1oTy#Oys9FJ~KXwqbAjB|ZC3&sg?($6wng95Z=;?6JxF6esN4
z+rF+~kMbH-`(lp^yLlg++!txoWD}-oxOYcO?Y`!(i*4_h_3wS<vwE+TMa{mLoXvZu
z{=C0;*72sjA)mJI^<8s+&%C=I_I~W0y8nU!)4nA2O?#g%(cKsHpl$E0O^5b2Z4cbP
zJHvZ#Tf-gulOC>njRd~#t=yWl*G}r`-o2k3><s_;?~V4~vbX=~u6?f=dF_&yneSV9
zN_qd`^ON`Tf4FM*y{*^g{j=h|D$9lTdff}yKR^AL)y9px_Hy$J*-cG5y|*fMrcIf_
zpWQZW$M<dgs$plh@4Nl)m1}L@efna1b0W{agL(RUB_Ge-*L=Euf5(eRdl&cld-NE1
z?bw<h+XeY9-#4d=f1mq@OM7=)8|>dI!@l=ZV9`E-rpCP@vGq2Rn>hEawc^_=6tA#<
zmDj61AGb`~xA}j|-bqo>d;D46?%|mAcyFLx&)!pQp!*3$_U~=ky!W1Wq0Q4p6ZUm&
zIbaj}BW$nB>KprO?%cI;Iyu$8FztZN_DfrB=Idwe{d|dU-<1r9{k2c0?Q!S7xR*o7
zcVGR2ZTlXTFxfwEZP`D`Uvlqdk@@?#a!=T6D06Y|(a+cSip-h0&saX!M)=^fy(ad1
z_nH+O+5|X$v0J<9%-#bJPT1K`lee4yp>$tr#In8a8=3Y$Kij*vf2!@C-`ef_#Nwsx
zQ@iKcZ9BoRZ^q~R{nM`;-a9S&*go;*6MGh~ytwzO+bp{cmV5TTT=I9{;wc~Y>^K#$
zS8B$}eI6IT?LE!fWOE_(#NLpbFZVwEqP6#e!jZiltq1lVITEpV?ZWGO7pS!E)m1m!
zr_8{xxA54yy-)2e_I>4U-D?=kw)g7BD|?q}H|+gj<FJp<t8=fc$>}}!&)e^<T)b&d
z$RlYRC5HYz`ZnzQqTg%oJ91?0K6|~R`woaL-rtzSx7S#q%XZhTRePH^pV|92%Vh5X
z^>h2*%d}dbWtO&nACa=3DKKxZoSV%4ZQV__OHA497DaL&&=owi?^EXin^Tv!*nMO)
z-Fs8+<lg0-UHc*{J@=>e#q49qaNP56UDdw8r(5^`NfX$&Re@`t!4;W3z1=eV1x&8*
z{a3Sj|96{4D`Vfjy%L^x_L*M2xmR|^!hIi3+U&b{&R}1z(dNC&_sp@W44z{1XQJHR
z)Yor!fAid8<ND>(z8LS%ds}Q-_sv-}cV8n5@4ihfclL%J?65oIa?oa5+H8A?EjM=@
z=P9+h;CXYeMVIbg-*0F3_In!cO*+3~ueHPN{XJJE?L8fAXD_)i#m->M`MtaN!}l`I
z;j-Jda^2pN?aTJEx<u_?A!J~a+qZM?+ePziIo4_Ky?=_=-nMbA&He>T?5f`0v5`7^
z%<kt@>%9v!ckN@|vwwHm?2w)3zXj~Q+;w`tebiUmz989sl2_K*+Lle(&!(|y-}078
z)(Qdt_Wp6+X1A+$_ujvkui5x-Ub*-CL~9#9j;4J*-i5Y|>KpgFDBs>IzcFNAYzON;
z<pqAWcQRG?^1s_-Q>s?6_x#BZ)-k`A+U=C9*vmM#W=~L(<h~k1xxMzAChhBCJYhHK
z_5>UD*}L|0epqbN8^f^gLEOVVZ<BoYKI^UAzv}Dsy_uR1cir66dSHv=mVGk3tL&Vs
zg!W0kk={2sBgs}>T4;CHR{gy~PAvNlzE9cr^&9g(8MRG&J@xn5&YL2*&q8{m-JBUa
zciq06vhStt+I;~(c<f`+{_HNldSKs`I<Gw$OW*GGeQ?l5PcmbF#Md{rnbqg_W}kez
zFI9K(e(Q*5`<ma1>}zdJ-P5J<V(+`fyZ7AObHHxa1=T%@F@}31D);Ow|JAT3z~uP;
z3D>&!iKQFwi`&|}X9}O?o(J27ZETk9whnl}xi9U=-95}rTkXFlitfv4pR%`DeC|Gz
zcT4w(FIL(&lkwOdZk1ho14CWy?=0%v6EA#V?|z>5drl|1Tdx$$+E=A9(S}Q>XYZVe
ztM}cqa@)(f`PV)-AM^eDcDmcthM3z<T$;0|&3naOque=rqE5@$r)(?T>+OGY?{$Wr
zefC8hds~E??JgO!?R%zdY@c*$*8UgsoAw!>xV+ofSn+_QmdW1e%t?C>x%KXo;<mBB
zJ^!P9z@!s<6&eNXulO9@e?j!vzONyjdv`x+-@Coxz1_Loi+lcUWjJu_z|*~Q>#S{M
z-%ITA7JI*E%BBcA+wBwgG3Lteiz#8)CEF9ZS9#Z+J=PN#_PvcgyIaA0=3b@k(`|+I
z@9zl@n!2~{`oet;?i~Bn7EIskA^UfKH206at_x=DpXu7Ow`IYv-SUs8>@7ZFyVpk9
zU?0b-w%reTdG@W;FxmU2h-v?hvU_%?*iY~473<wAoPTI<YiHu#`%`!CZT_`#ul1tf
z{fT?L_ihR@uy%alWB31h%wClbZ1x|O|L(oqGQ+N7=8S#G3x3(Ldz{>RN@LPK6ZtoL
zOLR8xJ^bqAz8|*b`#!8cVpqP?Yme#iCHoUsbnh<>KeO*uP}*LLjiGzFCg0d+v}O8U
zo8OLh0ymHB<+wk=R*kQ7-}Tt|Jr=%v2VNc2+}HhX`o2}q@9piHlwzy9!+)>I4x_zB
zOV;oGn6PxO#DCZQFQupKQCh{n&u*>7zP0zw_xcv{?Gr9F*srwe$S#gmM{ElOZtR_`
zxqNT$zK*^6f4}XK&iCD~%3HbDE9cYhjLqlv30<48uY+}#ZT)_S{r(Bd_rBQi)h@E(
z%igqY())a#9^Bh|p<(|<qkX%cuDZVW>K2{@Z<{0b7Wq`!ZS(oFx8|agU0~&r{q-kK
z*uBb>x6HPXKhU^ec~7u7tL-m-%Y8LUFZbCSUf6qD;`v_FHf3A0^i8%49!l<GRnpj(
z<;lC}OxZCzaV?g;>ASbtRi59m_w%(Wd)@{-+57Va)4syQ&HLUb^X#*(=-Hd7@nLU}
zUh5vq^St{mD{tMK=(}RC%qQ-BJER}k{W;gXhuN>!wn)8apVIT4HuYM9d#}$E-G4?>
zXzzV<VfzC6!o6)%#cdl_HSM*q4cxnGi>O@<W5wQke-G_tirKR7$XX@4btYHs+?;0a
zi~YxEThhA4j^)TIoBu{!`x5*Z_FcUB!0t7_$=*=K*EZdClk9xg*4c0h7umn9oVmwQ
zsoDN^qm=#c>2CHS9Ua!?Ti@-=^5fa7wXJ6FzL~-M1aG_9pP%Dpe_+k5y<7BW?+Xt7
zZa3|0<Gx!9r|!92cWv)U^J)8=&;Q%IT-?oGdd~UXD{V|{3b$(SeNh*&*KvydzM}{K
z?{$}z-nV0h;J!xfk9!@tZ|~i-<=bA@_?3Ixr%l~^L92bARP%$qE7mjbd-h=4K5K3D
zz2@$V_ZB^RvDf$dSDUGt9roHzPxm(NsI&1}#<VX}|H0mj$uWDU+~?U>s?4_UU)P6y
zh6Q5#!e$uQ`52hk)=xjPce{4M-t!V4_NsAo?Cq1^w)g()pSCWEZ>_%tF4^;&p}<!1
zY1f_wyZ7viVhY*YaIIwzL&yKUuU4?wudK7(+f%m7cKK!Ty|%mLt-Mz3-+N29#<n8+
z&^`-O?!D4C5882R#_Zqz=GneyZp-#;dvJ5_Y2|Z!Qr@iCTRhij|NG}__CA+6viH#^
zseOhn>3bA!-mooih_@;E8nTb;0HgIaotC|Imgn{|o2T!MxP57_o}|dG8A}e><ymy@
z{u|t2qh_tWPi+;i&9ZpG-LsUm?N2!G-kaLvZGV~XmQ6yV_TIZ;Z2ML-%j{jfdbXWu
zckP}<Zv^+vmov8)|KD$)_4k~u=SklEEt{J6hEyi*onW?Sk8-Q{zKEM=_r8B@yD!q~
z<X!>ozP&pQIqcWGTd}v|Zqt6Pg%kEn-@D4bbkmakPj-B<^WM_3H_73=y<PXcJu7Y~
z?rn^y-p8xfZqvJDgYA3u+PzBl@9mmiZr{J``h$JODIfRje$v0UEp6JKX~q2e;^n99
zTVdI5W3}68uXX5+J)GA2_bz`MxKEUUbszh@?tP26Ht%hFD{UJYw0W<OYS7-vyUX|b
z{Ic9@arpP%AGf9Uo;TUHXQk_>-E23T_I8<^vz8ZTu)8I%yZ3{}I=iA-du=`k@!9fL
zHQ3C_J7N86^%}e6TeEkc+}v%`v`u$^Lp9^RA1_1pHchOzXZdWgH{@y0-X{fL_dYZ`
zz4zm(qc+RFzOl7a6t&rAaK+~Kv-|sU;sW<BirThc>A{!XF;O@7S<0Wao7fs?Z#;)-
z-=4(Q{Xc9?_HI@^w6{!l!afbQnEjqdjBNi3+_aJHn_`!@Bg1aVlBN4rv@q{mmu9#(
zblS>&HCZBi!)|!)HBS4o$8Ng2?NXl3-9_$_drOY%*h~=2+-I6UW3S9hzJ2Pp5BGY%
zVcOd!ZMC;}3&Xx=m)Z82?f7eJ)|$KLsotGE#o3m7ovz*5>+hhvr+%I9ev|hbZ5DGk
z?3Z5ma&NK7oV``&@9%l9dSH+4>Q;N^#w&Zzq`%xd$#v?!(=STwlKOw|JIV6HZoO&h
z-gR=G_P?+5?0e<GXB)m}_Fk#CQ|$Pj%-MU?v1RY(4Nv!X%XjXbxItk5!iKoLQQJfJ
zuiyD%-;P-u?5+iM@40*++&+(c!rq{(2lsApXWiTM{g%D<opt;7tvkJQcE?0J)73}p
z=T<-2XPj_%?_Kr&J!0X%_A|)-wK<ml-0tMTvc3L#d-fiY72YSVeSL2#Gm~A*@w<CZ
z=@#xg>?mlJ&?c~#hwbEEuiD#tjh?scRY}~s=Z#zQey!t5Hug`O_kO(Cu&-nJlfA)y
zse4~&PTjxWFMRLfXG`tnA6(h{E8B7}v&*KvXMQs76A;^CyFc;YUfKCJyXQTAZa1@L
z+Wslr_4``qRPU2<T)cNi#<V?)IlA`FT6<x4<EqDd+n$*0>o`AouhqMYdwEZm?AKT+
zv`=ljgndwWn^o){^F4e0<o3tgOy0-oGi5LH_qY3;Tz}cz=~vqA?X=&<ZuU7lwV5mT
zvnOuayEjvEKilqByNdS{_PO+|-Fxa<=bn{281`$tJ++r5x?o?C<n6s>_rmu2{C&8W
z&9%igoypk#+hN&#&khCci@c??SAFThy?4%?+<UbCko6fWo_*fg0{iYu=<i#Sc4=?o
zEzW&`b5!@$AKA5cj+f}(`;%n$E!R4<H}7xz-pw{ayX!7C+5DRtwfEMlJ$p|Tp4fk}
zWb<DBGYWfm{a9sR>cP2xZ*hxl&6A_HE03?<IjK9|)@<qz>zSsP_uksHXK&@_4fZyM
zi*5cW`|rJ{Dz|s%V$HqUpIYrJpUvItk-=hjvP6HstYp}}$Ti%1&6E@O`*eM_=2{%L
zr=;}EKKDOO_Oa9a?W>&j?cG%K**5a_jQ#CB%lCOc)3Nz6{m$O`Iu~r4yF2!7&*a$u
z(nV^Y=T(`#)6(SkE&P6N?|a+%`zm%7>}!#lzi-Pw)_q@%g={xB-r9Rrxy$C#+4w!#
z(LDPmcr)*d*<!l4G3dzN(qPAZ&eIt8Z4hDH_w{4vKCjh3chCPDZCh+=vNt@&c`r-D
zi`~yfIQAaD{AUl_p^tm~S^RC{y;tomH@jeK6H~D7|LT|fiYFho`8+{k?<);~y(`n-
zS<5%%?wLN1eZRy9&V7A(5B9!N*Rr*HP`dZHL-}6+%%gib9cuU4-<H~^wn%R;U!2a~
z61Us7=c3d0w5(^_leu&6KADS(`(rol+<VbRW}n|PnSBlO-|zpC|7y?w&E0!{?O3_@
z%Z8Ty(tlX?tu;~J&&qaV??T()y?5VV+;>0BefOhuru|37IBl+R{oMQba>3r^pHJ?6
z?jyQa{_)Peg$wWQ6}_`{kI%;w`~GT7-Fx8kOq+~3UHjN{>-ReRda$=UaoxVe-Ft0v
z6VC4EyCP=ueflDstIHYo*e*}rTYqE4zRh{rdsx<e+q;e7$bN>Xj(xLK^6a<WtFXJF
zT(P%IH)v04`QyEY-N}2Wo{HTo^uW@dOL4)TQ+wEKY=5%bF1*0Eug<K=u3t`YZ?4XY
zefNS)>`$KAYbU`tYwt{>lzrX{y7ul4n_yEOd}tr@l&kiQ61(k+UA6aw&s@6Cf@AmI
z=F%zq;#c(V{kXGjkMol9eJU=z`!x4(+Nm=<-;>|gyXP|dp}mLLZreP6%e#NunJDXt
zKQs2Fi3IJP5@5T>Ft2g1_92eFZpQjHj7l^2&XP9T?_MvtPi@ORTagD#_SIBu-!r3?
zd!OBot$RABE!+F~jh=m0$)3GiBTiZ0_+4ooxu0pDOt65R%YW0oSMy}{mVEoO@5!a*
zb~?*$*&SPyxnFy`#NLvB%=Uf1KJU>{@7T*bVa47iffT#Ui)nk`+dSPXWoop~<HzGY
z*KAn#-@A5k?^PZ)dx<{h1GQh(_CFF3+w=aw61x|ykN3Wv-?aC;+Jt?N9JTFt^Q^Yd
z>-@2Iqrf^lKdamW)+NpR+$V0>YwN_buWm}|o~Mrl_Pxx1wtr3eguR{&XDzdKGVbea
zYS=CI;HK?^)id|SiaG2PKgO^xg=N>?saN0b{XIEj&uo_Ud)0-v+88h0v9GNqagWoY
z6??f9SMK#`yt~Ki=#9Nox?k)~ICpJ#!kumV+bTWwHXOaM=gbi=8@=7T_D-8=XPdpL
z+h)2)z+O|=6?PuQ$#w#BQ|$_0FSPyVw$s-2mF_;3z;v5DzNdTAQXKbgZt}KYEhw<B
z_erzO<_vMWb@S%!HNOyMduqMXzHd`2?cJ<Z?T$PX*{7lQb?=XN?E6+dlH8lx#Jumh
z{FZ$h%a!-<)Tpw1eudNaY{HiPw`=d(-FvOO&xLQx9{-~9y}y2H@6#yXvDbN~v;EfC
zV|%5)thG6$J9%G(S-7>&>PL23Y+r4ze1Eamw{_FrtYVJ6Vbj~K-xzJ&d%?AK?>b?w
zeOLM)*g3p3-z#*r-&RZI)81=2tv2b8{@P4Zn6Ur7-Tb{*Sx?%f6szrZUGQ=5W&ay{
z6n%f}x&8XvzO*fTd$0f1*?T%`(>`?$(fta#3+<Mvm+m*5&c3gNyVag$yM)ajeW6`x
ztKaN>xb4IKN4JjcRafEPGtK_hey;_|dl{xJvGZ1awYPoaSKF8R&+XzhlkMi`Gwe05
zU$u{m^YY$Knd-e4*t+(e;W)L|rCVfQ)ZdqTy|k3~9edzy=dakmxB0o!zC|az_x}Bp
zu=lvzrM(~4o!q<aw$#3(JH%{V`KIo(o3vnWJ<G{`acv82GE<-Kz5Oh3uaoQ2z46t|
z`zkCgZ1Q{N+pPS4ea{}v=XMz<7w)aAoNddoW9q)0g}?SrR_fX-czBx~<5Si>uIbu)
zKW)*pac(_tvn_1j-V4@?tY5tSW7p`_XtUg+&1UNV^R_cHS?tw$_Uzl}<GlY<w%OiY
z`AhA#-(cGJwxw_HgOqc7@6_e)|6|v*_in7mULTFec9(xQ?~RWw*njLztxa0_A=^8?
z*KE%gCD;od@!q>HFTyUP@#9|kW1M@ln2+uC6AakD@7@nv)92^+lm<+-tFfB8f5x5d
z`#6(d>~Znc+`IhVemn1D4ExrmPTXhf^LdY=jMg5};6K*7o6hX{bwGRnqa9UyIzPtj
zJ8{B%?~%nS`$`Tz+MCVTwCA@ikL^^Ut9wGT^X%?Ns@WVYceLF(Q+*!`+nGIY_y5@Q
zcki10H=cUi9Qh_~SAKGy&0U8ldk$4D*{gh*cVCF>wS8L-blU7(AYr3<e8D~~1u1(b
z+q-rO`z!1(KJMC+v`b|FVGHJc9upVa%nZ@nTYN-)e{$#1y=%J;?){z@xj$+1fj#f{
z%G#H{?z7Q;6K$XDYi(EAcGgZHntxwp$+>;C@}KvZY@f0B`L6zb2j3~#yT-n^E36l^
z{jiT`f665OeJ9c;?5nhh+S3yI*lO!s1v?!zrhT#XX?wm)=3CD#y=k*|&6a&3&!_EK
znk#2F%lWX)EpfJeM=f^mJzuW0Z=TSKy-t4)?X6hVW)nVl&EDOaHV(^_IQJPx7uzJK
zt=JdHEw?wg|H9tVN3Zv5%O2jFn%B0^r}gRX*Rt>T{<v^u_s_&5dqbEv?F+oWZ7+x7
z*1an8U+nw+U(lAZ$HaEy(=E39|1{eQ-;}cZCZ1;N@zQkf{0#^8E;*CAzbs|n?shH9
zy`RNR4=7CT+Vk9#eP6TBjlBYK`g^83PqR5Nk9WUsM)zLFJpud6j(^^hu#<V8L!+ae
z#jcI}OQn<Se5&s4jWjZ_+xwV#-$m)8`!;5<?iJ6Nzc;w<j-452z+R{4_wD8`7ut96
zD3eX7YRz5_%lCT^f79P*+84C9?z-f@m&Z=-QS!9fYqn8!?}16T_cs6G-urU}<37z@
z6YZrzL%=-y_AHvozjsl0r?uyVmc2e<XZA&j9ori_`|jRbXO8Wi;_YcIThqK(apnKL
zQR1chFQ3@Dx4kEO?>3hi`!A@p?)SUwX@9EwkX`qcIeRxgceGdM+`czOgx6-<f!ljI
zt%~+r9hkBA@6K0y4o>mezeIcb-V+il?Wct%+N9iy+b^~~+dkm3;_mH5oAz#xG&>M>
zWZhnohI@O}v+VXIt>3?|yWh|Lm%Y*6<>zGV>(bQieCJQy_xY^$-mju7taAm<?(^5S
z+*@UlWgF$7x-Ztjz$UlryUo-04Ey{&R#@(g&fAyJeP-|SK+oNCSI^iRHg)#iYKBdF
z&1Nj$`*_yzJ%={t?tAEaZSTcTe|KM4Rku&;h>Z<j%9*`KtzGS37TmD;+7WE$Jezr6
zbkIwyEB9CLeIeCgonWDBH|xcQy^f8)_L>==u*(o<w7+*taes^BoqgddKXx;{j<9bI
zda>t=&d0rSi|+2(yKbHRiN8;Fe_vF;SNA}H9dD5Jo<&pa?OIu;*a~p$vi}|P$X>!L
z-(JfiVz2VUo%Zu89`4OOcz$nSeCxi1LuPg{0Wa-8Yx?bd|77pJj)Q#mv59B*71iI^
z+mgC=PwZSy`;Aij_r|erv(GeoY~6g=a__{b(|c?W%iHcZ57}4MAhuWNQOdrij^@2b
zCb8|E*!Xyl!>`kO3)el`>&M7pb8M%)J+CkO-if>VZ6eIy?&UsIvBy1r>fU+hB<!W`
z^jfcf_{c7wOMdT}$_2a2{NrpK0{>dws9j;}v3Z3}oBy=Eih)V{E*?|Ze=1$T&V@1B
z{^_wzdowfo_r2Y;b+3`|w!I>Jy!&Rd9^SXXqH^z2k&`y>*5uiJzN)-e`a-td)R{c{
z`dQc7O>qv~^XWdD{W9lsdsZ)+urDY#%x2D&U3+(2KDcj|Puo6slbQR|`K@-x?-JQ}
z=!&e}pId$Vj(I=bn`^gvZ{5W*yUiO)_lj-Vwx`c<*51|sPVBAT#J4ZhgLB`psHuDV
zR<`XuzGms(rU;e2JjL?+c3AD*`}M+;-Bl@v_v-q!?yd4&V<+^r-R3VJ_udXSu6;%)
zTK4V>WZ5_K*{^*UeBSO2pFVHzw}hj6{i>MvO?Ozg_kQReo0@O^`xW=z+bb!Nu&e%U
zhn>$a^ZhGla@bkrxE|PXRD18t6`^~j{wdp9+nn0l;I?k>k85l97qs8q>r<k-N2XcW
zUc0Md&$5ijeU-{<Y|eRKv|bec%T7*a*WP-@mwPu(U27L2KV@%1f8*YzS$g|YL|*SJ
z7W-|pn(^=6UG>xU?b_nMhr6_YU%gV%-gdLBy)rF)`*M6Y?2*a$*{?L4ZO@gLZF|`~
z?(SQ)HDGT<*QtG5T#i_&NtEu}wOe$b1pnW?jXli!cB}o|Yt{R8uZPB)y;VxD_D!tc
zxtA#@&*s$ru6<ixbMDK3wsr3#E9U(Si+lENsa?K*-=YgPuOx5T7`^Y_`(|p@UjDZa
zZ0qgI_BNGYx4!$+-tOHE>3y<3q4xXw4%uB<!mxMBq)k>itVw&H>K@wr^xn=ruPV3L
zeQsa6*M{l9o`t{N>>dfT?OXg=+dgOUX<M@omUb(7x7t;nx?wkUU&7uuUCw*&-MY6o
zKPP_gf{)VsGPNey=gniYpZRpk-l<OUb_uML_Hmv)vv29r2R1zw`g@ZWhuXcLJbf?g
z)7^GIg4Oq4c<^q|Vd1^|?p<!OozxJ$@6eByz2@}`_6a)s?$uu@VcU|ucJEo!{5>h(
z1NUC6;@^8uF=9X8tdo0FH_x-V%R6u1-swO0GAGX4%k%TJUB~SEd!-eY+GTF;+8cM&
zZttCG%Wal3ckWe*aNqa6S8MO%L+7lWbgu8)KI^uv!RlvwF7z7gpC!1}Mo>%7KFNB8
z<+`V5b_)m`w>EznwO_Av@7@!^zxJ*?a$q0VTgH8l`(5^34v*N|8|P<V$$WKxNrJh3
z?PDgJE%#^a@3&6f<HmGuFaIX1eIG4Z_8Y!?VYiUIWp8QgMZ3qWi}z1VYu%^x``F&B
zxexbR$HduxeerpB(Nms%>*wCuE8SVV*SMu)&*N)ft^Y<`-Y3hxe6L~M#JvVer}s|n
znYi~XgYsUMdsp^6Rz6^J{LA&dIvL{oyz(Dg@0qi2uaB&%b?aTRy`CFO_wFg!Y_;hA
zBpYcqKKliJ-}VNbnz6@R=FZ;jYqsrGkE*u6>iTMLZ9CUK!FY`Wu}>D;#UF6n?{VhL
z{``IS_WBk^?w>RF_a4K(bJqV4Iqgkxsj`>*QLtCO`p(|>ieK%j7VX^o;#P|Nte99U
zs~UH^XJ2(3?74F6IR8!A`%xlt&z_A|dvC6<+naW$Vc(fNNxMKdrv0&#F7N$jmv8r5
zwAKFJvn~79d4An%5~;Xn&uW$Zh0!zj$|Se!XFGapZ=2F`+p3Vs`<kyt?7gVpV$Us=
zyU(xJbD!QZ*S&{ls_ZQ>IJEcH!XE4R8QFV3?E>9{H^uf;`r|#k9i02#?c2C_oBF@K
zhqyRxdaL60$KUDL8{cYqplxZI&Gz+5dwa6;_nx(2+j}VYh&?0I3hR^8U+wOHCAa@`
zfT#U~*8Teo9cS$`ZVK9Ka!Pyuok@IqA6$5|cll$secA^S_jb;DyZ6fuqrE}Wxpqyb
zdTi#rFtES6>gt}onuhkL{_F4G*wtl!>+}D;nG8SnIo}T48{=kayIzHJ&mQaI1G-Fn
z_KnXN_N;JM+Q+N3(Wdz6nSBzTm-b3-eYyAG{2rSW6`p;!t_axiHP`J;&Re(l<!_aJ
zt$o3JOV(~Zu+J}N&!k07`?A-~-n(QU`@VUfPwcH>71*sQ^l<Or^sjqOUT`0<b^E-x
zH6eX(bwAU-J8r-Bo}9(HuUV|%K&Vm3-t(X24m5dB+?yr&cki5bwtby4k$ZjCD(=(0
zx?%6kpW=ISa`x?8#aOiW$jz*ML6MU7#f{wizN+}zDtIs0JNeqLy%riA)@^M+_RD9y
z*b}~$bI<=J%m?I3`}W41FR=5I|GandiO78oCob8|J>+npn|Iru^=~BY5B~7ktIy4_
zPwdc{eU}Wf4mg&dv|~R#bI-d+-)-mY65Ri0srSCKn^*5G%hTO=-7C$$$%tv6iQRL%
zUmJz?Wghy!S8Qw5zS4}3HjQ`I958J)wE16Jxo^uVroA?rQu`iWv)@~?$Ia&ZvnJaY
ztqfLMau^RX{NUKTZgJV3SKlA(z4O0&udv6~y&u~Y?QG8+-uoub*goFx{ho%M$M)J)
zT(S^x6Shh8ooaWB|K8q^t>rdvm-FneUy*7nrqjA_cE*RjsY@PM>n;DYZ?gW5z3+22
z@10=ZxOd;riTjk8Htbn=|Ip4dg9CQE1y=3d)UwVl^Vf#GDXAOx%SWHKn|n`o{}qjb
zy{z9C+S&F#+50%dY_I+rQLF7GEcPm@G5as3tljOlwtcVfbZ@)%+Qa)UssG-~vgGkz
zBjzRhZiH;!yY=pRyML?qTd!d`y*Ktp+dipp$~JeU-|hRoBG=~BF~NO#V)OSZslDIx
z>(=Z&D<?$n_V8b0W5FlBhw)$UzSg)c){73i*r>l<zR%-!sg2&W&%1wnF0r@!eRMZh
zv*7`bDc!q26?5<Fb6>hw>977?Uwyv)3nV7&)e@+-@%#RHU)*Kgz3zMp_Ia(Y`#;KC
z?Q`w3-e1yrbMMA!*Y@5{W7zlUjOBhcw<$KVb9(l6Gq1K~ZnoL;p1IAAllku6YW7Gw
zQR&$ICVo5jUoASgcf#gbd+lxu?c4b=bWd6Jp?!NU>f1|)y6v5NnaOsA&b56R>znsp
z{NAzeg51$PJmLrUUY1{D_q|$k?}1_~JH>FZy<4Un+%sS0&^{OQq`f{O`}SEH_UujK
zsopoG;K1Is<yw1<55C!Zl96ejyk!60IM2|%M<z8LFnYgX@7k5i_MYnX+ZPnxvsd8A
z$-UBMZ*BemF53J3cC}sC?#{g#Qw8^SZcyLLS;M+d;$FgmWf|^!&-jY&_Df^7`}0Tg
zK(1q(?TOFI2e{^)-}`J<uH6Q~#kO)%pZ4m>zS!dzc5ZLe<vn}lLUZ>zTtB>b;_<q@
zlNLPMFJIiTr&VM2{vVO7`_*GN?-P15Z*Q`oz=4JHxcB{>`E1Xk#n$^|EkpLL2zS|c
z<0s#qvz&~3`QO&;U;6U#-amp;d#fbo@12sj*=B+A&)r)JSPygus_YF?+PP=T+BJKE
z&z;}H-l1*BqTjRcp1$F}?Dy~Xh}-_%Q~m7Hp2&EHy%S2W+2owo+!<l<VsArGn$5gD
zQZ~~zSMTGu2-*Al<9?e|Y0dqb{N_8^syFZF|FD101mEX-`YRvqS<}8__viAY{n<C(
z>}8*IeXkm?^MN~><@-|K?Y0;Bs$$=K^4^}Li%j;0TB3V*v2yPz;M}|SgptkulXt&b
zH!)nYIr)FV{tm%|d(E~VvR~LLxObLpj=ktc%l)ZO?Ckv+`SvC)p10qiLvhdE`@uHS
z)2{40*I|5M#q}+AFPEq8>E@5$KQBJuz-IBw`|`NBZQTmQ_wD1Uv8xqm*jsY*yxr1b
z-hGnuwpq^2n7(i6bHlynS6cUK9+%pCB4gUVYq1CRHcf4^@y_Mk7r&Zi?<u8!dtdG5
z-j~yTZC`p&%U&r~+a0oHqWcz~_SrkTi+7(nuik;YSIcZJ&*nXl9ldR@!84Y9M-P48
ztL%Jo@6!B`{rbo5?mg-|e=qyl!w2{;gxc5~Uu3r`Gs|8|<mld}Zw~f4CyV#m9AC2M
z!WPDT+H-XGuS)o{m;2A=y)z?R4pd2Z?R|T6xxLeu&Ryr{NZZRlQrOQCYG*&iV*lO<
zu7drhcZK#YTw=R7tL)4^L5J)Ee>P`Z%WY!ZyV!*3z|p9``*!_n+vm*Qw%4m>v5oVe
zZTs&si0!d*_p`T(P}^6!RCVvA^w52C+E4FIxBqYVWwyP|>yFZW59}XXiVCgX%W>uW
zF42hZdzMDO-TOBwY4^jV2m38jxc15=)Z1MD=(X>cj?g~MS7-MMe;2gN-?eyekI)Xg
z%WF^Xb&lC#<8<=i-n7eldlzvZ-+yMi&)&khr}zF`dUL<izn1;Wrmf!BzJArdU%VUl
zmK65votq-Cv*GZPz4u;C*em#TvE8D)nR_gL9Nin}BezdcjC<d+pJMxtPHfmKR{p}S
zXlca$x>a@d%L1p`L~J~`_uQ58J?%_oHtr@Cd*$j5*fGtRw!bF)*PeRLdHc;Xlx)u^
zZr)cEF1<H2_x9dB-P(2`(KGf&hUV?}G@ffS?=!>RqJ6CU<XVjP#@u$cZ8%WAZw80s
zzQ~Yedsj5y*c*27``(OGtozn`tlAsrJ!!9d&`&$teNuZ=**EQ7uR390*3OxGZ6h!4
z_3lmF&%3Q^PsofN`(N?=+<UqA#ojHcXZ92en(u9>mfOGZ(dNCTQyQ!}KCj(>O>U9h
z*U$z#p5<J2Eyez}EJqaVcfXV0`#{uuZ{_2adq2(g*>7-Tt(DWI*L#-D7eDap@Y_AF
z$}R0#y<YF-G;Xmk+|;!%E7fPeg*)rMXa6Se<0<2@w(i=u$5G(;KBiZa2R^N>w9DM1
zx^K@X$pe2DD(>GndF4K(XH2%K^XBed6|1y=_4mTP?UthU8$_M=UO2XJZ%TyfKCf`6
zJ+oQ)4@{Jwv?u<M#XgIS*1hMBRa!>~?bs8bS8B6VjA`Ff>GgYWGUy%9cHg#FQjcjL
zTXg5%S<jc-9Cw<r*OYJOe(T+D_b$nvvR}F5!XEXxU-nvSowqr9Rba2$nmPOZm#y5J
zZx+3`WGcu0u!*<#MBiX`*s7klKTYoU?qk;<+c;}I-1G5U*PgZW{#t9A%h}Jr#=Y-B
z+n&8=UsvqkIDOe(Sxr^@Np-UOzS~T&k1SF<VDJBA|7|v&eg1M!_O0OK*_%E2=H7}s
z7x$U2h_-L2UcNtr_xPUsN4oZ3HJZFHG34dmy(cH`oqtSxuf2oq{)6rt_Bt;Ruq$Gp
zx_95_+`Y{&&hAw>xO7kIyW6&=>>PWYR|o977JPE=u^poO{+C7X4Q~Crx5i}2o_!P5
z_GV4|xX;*~b)Vbh`FqkOyZ60GxV`t#NzQ#U{)+ErW74(t2;$y9qw$i>nuo`2+S{3J
z*FQJdd&~XBURK5Hd%dd<?Y-5XyPy4Cp8fviE&Er<sqOc)JiI$RwPD|Ct)#uHwM}+k
z2tT^l@AyRf)86~{#OW~Y3*=vA@9<4<pU}rW`~DS0>^(E{f?e+KmA0C#ZF>&+@$Yjm
zer<bDk98kA_uRdvk5=qc)|hIyU|zcI!*7#q<|uvLx2jIlPR?oh-akLr?TyI!w^xl#
zaQ`0Rz1FK)H`?<pe7tvayZYX(eoOXgzn{PNL_^(vOUu%I4Nb-S?(EpHH*d|`y)Ues
z_RcU(+naFm<evL~_4Y_gOxQcC>&M=Bo2I>2zkk~6+2dm^aA@CNj<9()OH#ku<bArb
zf5xBfd-eM|_DHU0+gG_MWABRF(zZ8NG41_&uye1>nz{StiwN%jvj3EQL7Sre-X-Gu
zEDrp!tDoJpTj#dmzT5j4_a)o%?9X&qx%b>k!#!7~?%aRR&)(MMcAI^PpwV8P5O({t
z2eYj<xtiJYU3S~Mn`4{ZYIU`}<~wBfKH{|BD^hjbTD8}IfAila8()Q2`^?QB?w`M6
z<39C(qkC`7X51J2;*I?kOS8SnyT98;_BHR-_PDq=cJ-QlTz#+inyq2nFT_7@&l$&n
zeU{Zsdz0Eu?44`Ny!ZIxqP^lPL-(c~jM&o^xXF%XlF?qir;GP~(W%}SHUFSZNdbfH
zjc2d+F~m>YYxV5>zOZFo*5V9P_BhtN+hi%7-}~sN^?vWaUN-g`b8Su(O|)C<5VJ4Y
zXpe1>%>I4O9A9l(gx&TZ6=T}_piXG-pR)}6R<7;0)!WRuFL-L9P5i&`eUfSkd*^Pi
zv%68eZ0}~5oc+Q5*Y?f(thxW8eXUKp>g|2IVmS8|e97Lcv;E=Tw>q{qu5&Kfv1f?v
zOWg5%ztWYQeat(S?PJsx-h1VI%ig8i<n4^lmh5r;y<z|IFFku3nrGW=Yn;8;?epTj
z0U^ou$ISfpz4_j<Pi(^Rz4ETF_e^-TZSSo_j(rB7m+ZZ|bnV`++t%)V!^5-BTV%=J
zhnHvUOYUE|yJp{!J!{*}+s*Kk+soPLXa6Q~gN?Us)n3l<4K{DW7VoWjXtOUpKxywJ
z<@eTU$4l*0H4FD2PF1u2(B@&!Q!2Ul=jB`b#B6r%bzh^tXSu?$J)X1O?FE0`-J3Gc
z(q@H(v>lJZp}o~=diIM`HrWI`Z?d<K`n9j`8Q&h`r5yXjjjr2e+<3fqL*|CP!V%~8
z9ZBWhx3*&6{x7opJFfM%?$e)s$ZqTIW&0j*DDQnOHs9t;O4~l>ixPXEeec|-EWN?{
z<+sl^fvwAJ4xMA!cfP7>um0N|`*y$T*gM7e&7Qz3Q}=3X6zy?H+iT-{Ud^`hR_vZd
zTV(AewJmlz{g}Mx(v3BHufLpcqiXqgug9++d+wZU*xULoW54tBK%2_Mo4fr=nQW3W
zn{2u+FSUQd>bp1kY^F^<U+{i$fi?R+J1(`CePwKaQl4S&7CAmUn-AyqdcJ4dn^XOI
z?}r@>_MR3i_kQDJvwd<{=m3}atG)TU>h|ulP4@_0eZ9|lr{8{;o96p77yQ_JzrEYO
zUvuMLkz(<^-(xcOy_k8#Uin_!{;gSldry=F*q8k2-G5^0u6=fG0{e{o3ih55F0o!0
zUby%B<px`)SpU8MpA_w#>BPCuGkeC~DbWu0_X|$iFvtAas~R(RuMGR$z3m$~_qi08
z@5xPAv*)4Sr9CPtTKmnGSncgt^?aAa|4I8c9O&4aJ*{@{q$3{t9z1xwHz975U0}HO
zo_LN|dtV%n-fO8h!{(4>kNs4()7BeO=k47YdT8%IJ=c9xzcAT{R21)LYgE}SRJCBg
z-p|E*?-rI?X;tsr`}uRgei>7aeS*g<_qvos+X=sUxzl4ts2$@rD_f2>=KUTDR{Pyf
z+_2lUE!1YIF!TNzZV8*iEc^E6_;K%(3NhVZThL_xaFXJ_t8A=$o^7<;zkT)VeMh@0
z_U<`;X>TXf+P%J0vi4@jp5Hs|e9PV^9*ujm-yGP-GiCc;{%P&|cfPv2_uH>OHWu>x
z_r@g)*~(9h-s`^Lw#_Ybg}spn{P$-5x7~l_!nwUl1p@nGCWP<v|9*Ind!WJIEx&H>
zl?+YXyI)>&|NM>Zdqh0+_OAJJeQyV!;J#_|m=9>o-EC8zkh3>)^~QZCZi^l`bS~O1
zxPikiAj*I5!Baczj?HD=+jdxSpU~}nd;7$-?0?_-wntLw%<i+L0{d=ETDJGj-0Ak#
z<()PPlI`}#_srhE%i6)7L6v8(z|#f$C1VcnO?G&_*Q|Q&zAKg-_UqRe?^obBv$wPM
z&c0Jlns)EkKHO)Zt+D6gx1)R0+Y;@5pT4~J_nbTX7EegG<vPH!H{<x+eaRPj_h~d`
z?Vta>ai8}6CHq{q80^jNn7Aj+`SG47frs}lo-bg#w7p<&webnNLn6X^ounA{+6PVA
zds41qFT*1Lz5Fto`<`B4-1mCX#6A3ld-k@+oZl<A(Qj}1G=_cZnI8KWWG~zMPM&>V
z2V14xG={1BkJ&2jKWietPySbd?d#{-b`zqt_pV>{X77Owb$jdg$nJ}ududPCsl|Kb
zyx#4TJJ7XPD@E0Q>9!Ml*9AoE`=7REzq#q@eacsV?M;f2J0Lv2XRpV{4|{oR_V4@h
z^XfiRG0T0;z9;u&rBv<vJY)HOrI15=OZq?V{bO{0Z<N;heNOYst@)C8_VsT%y!VRv
zzP;XNQ}#WQ;@;DK?w9QoTh_f(cjfG}mXz9)I+b^C%FBs+Pw>9p8}ic7CT`-Lz3x|+
z?Y%jXW#3DC;k_Ok68E_-H?%o;V*B1J+;jKx|Jb(sT>01i2W4CLN_WZbO?!1?@1&P)
zdlzm^-Oty~w9n^k(%z4^s`lsWCD@s5Hn+c~zQeAsICsx6)r)q`dMEaZIq>W|*}r?w
zhl8{1RCaM$TYBx^n`61$PInsLUIkG(`@q<zd*eMf+I_gadf&{lOM4@(uCcl2dVJqs
zyZXJI!P<NO9-p(X(6-(F!p?_wHEX8rovU!(?*5fi``8ae?2|n{XK%jlroC5gJhqeC
z_jRvIs+`@1v`2gUgf8y=r80S+SjzIfW^zIY#2R<(%f2{q-^q_hEw9fL-j~wCy3ed=
z;@&9tUwh5g^6%}qwa<p7P-XADt`oLTyK47|Ci2*DFuCrv2t2aKZHB{MrM+SH%qI=^
z{OfMnv$f7}Z_$$jduO=_9r&_}b#HF8i`DTB+52vtd9d$u(L}p7yO|GgX6x@&NtnJ*
z@y3@umi$`#e82D8lYM!ry^{8-z1&3)_nM!Yzt5y7V2|}o^ZgpK|MyP)Xu1E<yQ=*U
zR`BkxIr-VDp!~Sq&%Y1$)|@!9_jc0xy+($D_RL59_GKwN**oi&?EbW=Yxh0yII=Gz
z>C@ijYiI1;_Nir`r4Gx!`2i^hVwN1RFx>pl`pt8$eG=v6dv$f=?PD(|>|6acd7ro0
zC+jWoT6@JfYOM967Vg=0;+VC+{=dCnjZ^IQSmfErtvI%qnMG?~&NqR*pXLeJsFt$s
z&(=@h`%>WE{($Eed!HN#+M|&6VQ<pA&OI0R)$Ets%CPUm(iMAI{@=2juex|&(Dk1E
z-A!-zG~Lv)k>1a{FEXTa@0m=cy~|`8_wL;yWq*5JsEuH^=iZt?TYEQ4u6=&VU3)LB
zkJ}qw?`MBLXV$**h2^$7mx67g5;*r6&Azj@j!}DW#``1tY@)mD<fp#e_f%#5-izW<
z`~GN~+B;?b+9RI1Z0`~7S9^SNcG`6bX6@}yJ7m8}<ksHJf*<#0zFxLZP>$I)g-Leb
zzK+IypUvOyHLg6r_xt_Qy@G|@`*ws+-}^WH)ZU7J+x8}G*>9tJ#$@-+bIWW}x6j>I
z_;1tR3*7wsG&*GWzSUFSE8wMW7y0eY-s!AvdwCu<?|nSy#ojM>ZTGvVGw<1W!Ew*B
zI!jv}=LuGPPiyU^rd{0cc{6&S?mSuhog!2AzWL9$Z{93{eUDn|_s2<}-`%2<vG+|o
z$9^$2{(W0df3_?8+qZY_T3vh5Ji!CfCgS^6#oXI#Q2EQonfvnI+KA`Y2esPvZTHZ%
zo7kkjf482<-f5!Zd+#;+@BjAa*Ise=YkSWZ^6e9|pRjMQ%7eX%694S(y`Hj{#c9Ug
zTP}V3`d0DoEm^L&kKMIuuZgnjKHIZb_K0e}-P`uz+}@k}@9bq<>%6B)?A_j_>n_=@
zeV?$m<yQFK2<3VEa!#r2U9n)_UcaNa_UD_m?LD!%Z2$S1D>nD?KJ7Jn`F+o)qhI$n
zFFCUBc<GYemHfZ=s&?z_d*8KjuY#wxz2nU&`z#jzeHn91_ub5J-CHs-V|S_U`(4w|
znC$Odux0O7nW@%uw`lEcf6}$r`b~q~dP9YM)@KwBq{%$qdptE|f7G4XHUV1<_BXwK
zy|*ujao^#Gjr(})pV`H_{oAj1?$BNnfjj&EY1Hrc+O=_?uJiOgCO&<8H|1a6JN>1{
zp4IQ??BDy?elL4S-rncVv-hq}li53AN~V1kx2o+C-Vb}-b$IrzSd(GZ{%F$Pi#J=W
zr{~_^Q(1b-dSTw2{hL28-#O!*tL?e9wfo*hFW&nsn0uc_%&EOS887$zb&NQ`TX0~{
zHqkG8=V-p(+r3|KU(^ykdyBce`z9`bvFF{Ib$e63o!$G!@1|Y5e5U<YW{Z6XF23Kl
z>s|leiF@YlVOuwGufSTh{U#@)_hx@%+Q)OhYrnY4n!VTfGVCNwe(y2SDYN%qdU)Tm
zkTd(wc`)zu&i}se?XPKjj|*JhGs%3*KI>P<_SMC#vg`cEzR%D3#J-cS829Juzuo(!
zWW(NLc8~X7QCnblYWB{(raLy+os+QM`%3fI-g9mD_Zj`_*xM-NV(sq5v-iWtX?s`2
zPTk8m+jx(R%$dD2xZC$UUvP6b!}t4pbUjrL)bIbiCn@gpp6<Id_i28+xaZj2O?&&_
z+a2%-*4o7zxMhDtlId=nUp{*b%=>KCn`P|X7~ipf)0=5~x95J_6Kx*5?_x`hZA)RI
zy%(qK{`ss{)-M+O>^natdhfX&k$vktpX@nO-)UFybpGC@PrLSR*(Y>h@11FT*%n;2
z3;nZYucWYved5Gd`)8f+-M{AK%Dn<XGxsrPaP3=?{LuRHjJ|!%``!0hUzN6>bMO4#
z{NP=7PvSD|*d5R8OZYw4Ix4nf&!HC`d!=Vz+$(lz!>+9RNA~7Q*Vss2&E9wKtl+-c
z67jbD@zJ&lwT}BXKi;#Kw;_HHhmPT%{L<fh*W8V=JuiG<&zTH0d&Sl9dkeN*+<V+C
zYoEiNu01+3i}t>H+huRl|81{PubTbVL#BHZrrO#*b;#elD?xZ~!kTsan3aY0MK{jg
z`{T*qeLpUB+iD)&Y`6Pc^nOR<N4wJQ@3;H=pMM|!+zr;E&w2JRRZ8ru;#<DAvg!EV
z;*f)Ov2DtG!X{j_OZ#_h_i?$mb}Y3VcBlQ8*#~ld+uLt$zh5=fbnl5J+xIM*J$c_|
z<yJfP=zaUW#h&g?X=bu}zDe1xC3?obTw|4e3*R>G-On-2_SM5HyNqX@vK8F*de56R
zhPJyq=kM+02;Xb>YWKc%=a~1+uV>#U{(1M__e>LP7WQr4`#(y3@4RQvt=8{fVZHQ_
zuKoMB+It`0(%9>pvSFW)4D-H}4=?xHK2YB0R@t%lUZ|+Uw~PC&uiTh!lawa7S2rVX
zueaShyTTji_wH~owNVz)JCHcr&2DQ@;Qo7WS@ugYitn2*(6cZ4a-+?hO{ex+F+AM6
zVOO}lTguG60a=Uprk8Tq3r%>kcaKVy-Jy_`d-u(*v19po-2TdSFMFwoC)Pqs7VVu`
zw0DnZ-R8Y3m&n*`v6*DQeQlzBY=y?YR|`t^XZlarx8u#WeP3nR_vT&W+1IgWt-a8d
zn!UT+3hdNe_U|e6YO`JNf9Jl*QaXDRJw@#I&6vJNM^0zo8ovd*m9FUQ5xBJ9N}Er3
zug}(4n=@<9@9i<L+b_F0WcT!>hkLUtqxQW`W!opP;@{pil^pvk@1M7s<hF62@b!wl
zl|0^iOrELjo&WFP-h^n6{lYvecGq+|*+__OvEzAm$nNi#rS>X`-uBJ)iMHS8NAJt|
zpugvV(5AgtQq1;l;#0Mq73#f5j8%5;GOZ2!%1*4_d+GQ*yCui9_8oebYwz0|w_j@V
z-2F2vTK7hl%-+Y>FJsf@Tx%_OeA+&-)N6L?&od6Z4VBwloc(rxO`q2OHTlW=RNC|Q
z{x5m8XUBA>{nLJh?iJE{zi;P@mwRh=<?ppN<gq>eWy0Q>YkKXnK1uG4=-}F?Dz##7
z<J_&*YNw{}&AV=9^MGgd-qVhHd#jJg?+bRCu($X`psi`+)O|nR##*PX>eySgHgeCB
zj4OMyXPWLS>B-(Jx8%a!O_i+sB+l^cdsc92zqFXb-o|%-cHhcr-uLOX=Kk1^zwMq(
zl;7_!&S<-E`bj$>p6fOb6;JP)aC!0GmWgNfu_*rAt99kT-Zhdt?Iw!$SUsBbVXx5n
z6MG%^?Aezzxq1J=yXO1nIY;m1=~T9hI9#xIP5<=0o5Usdv7allE4FIi8<sU?@8gS~
z_6hxA*~iRWu#Yi$(cXtQHtcQ5?zG!*p?k0VqZ#`fuld{bN-fyaymQGu(Mf!JkFZGX
z<J`!!?`KW?zH@uBc5Ccgy|;PR`@N|T)c5{me6@Fe9Pi#43;ylx*;cmK@{*a2i%rQs
zm%{aXcbq!6H-EW?UFD^(dtaD2?eFpZxjTP$%iacUgS{;irtLLtR@|@IcWiH)cirBv
z=OgXsPI<nM>&ZX63452@JvBdJ{mntZ&XfP<?s@0$?)@EjXz!Qpl6LyGbNBjk&fd#g
zIN$Ea263z4%iHajJma#}mX^1FJHLJ3bx~EjvboIr-u|oE@3MZw-Y+bHdmjW=?>qM1
z*M7;pE&I5aoY_0a?7wY}?yr68m!IADL~GsNt2qXHP8|!f^X}Yb!&SA_u895Co|LlG
zJ!cD#?bH7^Z?DR+i2dj18SIna7rbxlkN11nt@i8<U$=Mnk=z)YpqstcIi8>P)JO&H
zuFqU+)7ib#rc_*cpYpejd;bQ%+S_vN?w<Rq{CmAWPu{=g=b^ogw|?)5ZPDN3e184j
zM>f;<KV<r`cYd^q4g1;aw)L4#`x-+e9M+wa+i$z}>|RbS4|~>4hxW2;d%8F1SH<4P
z=lbp37q#0kpVzW+o+7co>Tki`H5u&omJuKJa4u=I{dVuc{xb^U`@jD9vFA9;j{R%6
z=Ik{-%DQji@$h|B9Q*BRvRM!4)^Y4RaqRMb<*3#B9=x5mFFD}No?O!hdu>15-Me0O
z?q1p0`2Cx5<o9`Z)$jclIdkv53Z{KOJMY`QEuCxg?ZBPA{Na52?l!jDPW(UFI<}x=
z?{8&Q+Y9U1ZLYH4v(ffx*_-f`XP-=<wWVp^hrRC~?b++?5wkD#qx9ao3q||FU;W&Z
z^0Q^{i>3>Ew?{9sS#q(-F3fGFmDlN|*7G@++0I%kvhVFlZHKGty6mhY%=g-EKX1=b
zW4X6@+qyk#o6qb`nKXUBhOOV;<uS&4Rp++vpJ^(v@41lw{taQm`<}n$wg0NV(k>!E
z$$skBb9+rfgY7Szlh`}y&!xRDSCs5q!}#28X4D(I3lp~QJ*Dtr-wH)B2kSHU_SN;V
z?Auq-y=VH_t#*+oH2126ayz`wo4IGQ;@dq}C+*(D*70cXfj`~$mjhSq>Gjgt$9wg{
zo~UPzdn;p9_7r6M?6Fz@c<(CisJ$O%cJAAwdvNdZ_fz+p=NIpbket0cn=fFG;py}C
zS1Z}~F|yvV+g<!>uk6PkHc_*T_ew61+T$bIxPSY!pVpVY=<h!8y~9o|O=SO{PYV02
zR;wMzo)f$`ek+?j*S95m?6q0<?R<LD=Egm4dsmzJyR_4f?`1#E=x|1C)9&EIfA@>}
z9^PBw*0lfXkIQx?@21&)eayFSvsjJo5~*i<UxjAbD0WEg)fAm&uXEOGf7J6Gd(S_$
z*q@jF*xtnN>b_}nIrcSGKG<8E`DFjk{m=G>RLR&cIeu!-nV>6sdp+jwnR%++=A*8U
z{oNaR`|4kt?-R1Qw|DAZRhx$9p1p2Y-rKM(eqwd_{cLONy|VTi-wO8bNi5l0;$g7w
zq8#HsDdqmX3m*sVKRBb?hUas#eSsAFo?0i)eFqp;?hW3`zi;pHwRYvXg?p!H%-YL5
zeW~3E;oyB@`@YzvSU20LyxYGwOjXD(YsUP&(#IP1e4oO&H(8xy|KWdY_xyTNw)eTu
zV>`hT2J5L8zS=F@lE2sM1DE~!^5^?sIe*{VU(~qw*R7>?VTA>|+xso|F8`#rH)%$&
z{mSmS`(*#k-P0*Aus>ffWq<8~h5ObR_U?UqaLL}T$?^Mz`w#4mUy`@q>~y&8q-K@9
zGlJUoSvcL=%iDKl@BSHjdo?r7_igT4x#t9n?7q1>s`je)Ki^w;cg5aK8L#%p`<L5A
z?moWve2I!}|DCpdK`XcK4XDtv{o8hT|2n<*dyg-U*&jCV{+>qDJ$qwzI_z2H8)_4q
zG0pDZQ-*zO->urcC;XJ%(-5`&M)?u;D;O@@ey!KvHzR1Gy|3lpy>pGk_dbh0ZY?@v
z!M-DG4SSb7UA%X}`RV%;JSXinlSw>a^6<*u6HJHptJ)gxe`(cmV9&BQdzD|DxBoo(
z`(BO&mVG=6=I+Z_HqCzedsc^A33K<FEX%cHleMzz`!{{xEWt~ACm&wAC!ku;UTvem
zUNMuc`wt&ov**e_5nHdUg1zzG&-Si7D(!H*ZPwn3OC9^N1P<=m_W1tZ<3?F~JyxFE
zd-lV@y;8S+>{(QMW8aKhy!&1qG1_ao>Gj_B{h#*!ss6QlL7Vk{j)>)ZP0q{PmnvHA
zv1S#qoscwb@39rT_DZ-M+V68HZ%-8;>%PN@>umb8>-L|XHf`VT4+;ky?l<lI-0pKg
zv})$wjjW7&r!_Cw`#yHXe(S=Q)&W%~tvq}b?Cz(<?=82m*nj(i;NI<rR1WYpiR_!4
zR%|C&9<rA=c!qt-GVZ;6^Y_?9YV6r}Pcm))AvK-@JZoR>jXbww-`<n$c3TYQ?vp<`
zYwyD-m%SCCZ2SJt?b!S9p49#$#twS}e{}3wf1!8Zp~gg;D9v5_jypZyYsD;JE6t|9
z_w1u@yJc78SeI)2u<2=K-*<8CjXl$>rrIw2vS9DYSqt|nAD_Ojt?uI9v@O|t(<V6X
zzk5DxZ-Z^~zUG$SdslaL?d3ejxQ{cHci$?0x&8A_N!r|>%e^~mp6dQe8<G7BKMUDb
z?@`%%Cdg~goeLdy=Z{D3Tk>GlUdN4W`wG@A-sfm?Vo!4OuDxE5759DLJaMl%FO&Vo
zT}FFbvsi7n`10&4n6u2zm~Hc(r4@qvPh7dZr?8-5@7cHA`#$jY>}Qa>y-&=YVc+(o
zL;DtcUD$7ZtY@#d{LMW<)AFqE9dFuiWpj9Mi?!GOPibm<lzubr+g?z$?_9*py?$vP
z2TG>K@8uEyv)6J*>|P$8|2CEf_wIeUQgiSB$LsgT{k^=$xAL3qJFW}XiOFqy9Vflo
zyM}Y?UWqd|_BeW{9dKHA*yiLG!ToCt*6zJoaegmn&EGxjyeWI~9U|;B`2_ZwR9xFL
z`Ki&~hmSjKeBVypccin~K52`#ZCO#Y-KwG*n-AVg_p)uhz1KT4Y`>L#?_QS0;x=zy
zYV3D9T5loXGs$kp&!;wdT`~5H1mgDVE)hA<SNV7Ep$888@4LG1JvRHY^`eQB>{NWe
z?C0{oZ&#AIW>3!|3w!6ATlN>l2JTy?u+nPlxfy$$v_%gbnRjLHyeYr;9sN_V*LLf&
zy#{AnZKio|+IvM+z^*y@=^pLowfi=lJF>U6pw9N=E{45gUH|sp<M7$LXhPLqmVZC@
z&pGpYceX#vzCE=c_d1-OY4cd5(faU<D*H!KW_#?;dGCMoI)3lkf2n)Vg+1K+ayIkc
zJH>nU)izw(tIrv{SG41tt&-O)yWQJMZ5<MOZG!Kq?R$5|aUWX`*X~(0VS57=C+-dQ
zmEYGZux+>9q0@WgGRyZ1rO57WIrC%pspYfxE=%aMo%!&gts}eZz93KDeLYt=_wNo|
zz1Ql{%)QdBYJ2AXU%Y?*ocnucd|0%XGk31t+fNtw?{r?eFLp}co;~kE_a51CcYocL
zvwQg-MeeKFQndH5=;l4EzFxDQ*?M~KH8(E%XGa?M8BdYjH%X^qug1Bgz4B%&_Ra}C
zu{Um~zO~?k#d~*uW8Js8QG0LCw)uOKpU&Ku=pwjpSvBk4wZ?Jwe?QponSGMee$Jeu
zd;B8%_F9&2+Z&ghXMIUJZ~u~3^S$<3@Aj@yVzIMWlV{(ZVP${7iR-}4*Sq)bjLX`8
zZO*a1Wf9Nz{+8IkcUz8({cHKVdn5D~?>#H1x8HrX+1}q~M-D8!BEIi``?LeMX1na)
zzC6fYmT~`Hv&-M@(rmi-7Oz^q_r!yJ`xYM)vbR&s-nYAtZ|~$Y4E79(d-iu#Rqj*R
z<FGe%D&M}F%1iAOr}6CD@k?XhlX+r$Z$&=Z+x|RcZ^WFQz2^kk_a$p+?wdJPc5kVY
z#9nWOX?vF(oVWM0%=5i#|8Czs<D<de1KZEqFRoi)y?y?iz1mAB?3?gl!rrfvr}w<P
z^I-q=`>Sl`^*r8tM#yx}&BF$Jjb7Z@%OkdT&)j*;`*t6#-Sg(lmc5gFPTOx4p0vNV
zv&Jq}<;k9Xe;RGAXC1RGefWB>tfsq7;fZg1OuD@HAMKoBqki_n-YrX0_D?ojwnxaX
z!H#L$yFI3Fuh~t{`MmE=JCA*d!Qs7cCJFCnWBt4LqQ$not($uGC0TgdyY1uK&#IKL
zw=Xkdzxup+`*~|$?6W`Gv!`mN&_2IEb^Euh_u6a1#Js;`lA!ImX{@%<LgxEAboT7=
z4A$6Vqw&yYznbg5dHEmqa`7|lO=i5a_tdn>d!Jtj+{@9BwU<?s$L`d`?!9vlu<X0|
zZSLNw2NU)*noqYa=~A&}mfN=HT1x(Y)oZW!7U?MOwO_5fmqCMZUv0gieT^pTKC8v+
z_ePp2?(g<UwUc_aeZSobvwbPDcWk<stlY0xZD#Z3#O1vocP-jGktb@u(4u#H-m7ij
zQ*Lo{{}JhRd*3ny9GEhtZtv@<r}w)zY1wa7Zm?rKer4~jCI9VCtzq5UIl*mbhxv+q
z4&CAVzt-#78yyzfE3)~E-R;e1?R+(a_FcNkym!0juDv2pAKP{B5!)A1q+)l;X~o`_
z$xJp`6aMXSJ2Pi5-}g$pJpq6ADi+AtTHl*u{rnHtzDdH0dtXMc-`m{tYxhgxuDy><
z7VrCCVz_t8($YN>?>p^#cW3+F$dvbch2?wfJP+;K^XCxDz8~`&_Gr1N?L92iviJLs
zmwWlP71{lnRj~K7$ilt-5<PZJ8I|^mi+9@<cJ=J%Tqv|x#8_&-!{Zfu|4f~`H$3v!
zZdvoy`~Ro}>=kjDwRh1DDf|BijrLUeu^wPr*}nH0uigPcKFxiaDHHeeG&AlMTNiBi
zMd<0?3xVhMUMp+gXLoGH-rjq=?04BMxAFbyyKmc5CHsQw8}}Wr=G@nL@5*k0buVl_
z1zz84udT8#PNsWr^mFUIyN?y_-CQnj>+-wTPEOuz@6VM=`wnjr*mrSnz}_ofr8dd)
zMfScvcWke|Jjd=g>=*a3XRp|!y;OGZhgOSyf@16T${Ka<vE8xUe&-R!y?c(g*zYTA
z+<UoGeDAps?tMFj^7eNB{=3g_`NX|5yG!?2vAXa7^ZAm!qCoM1&6+;8NpGI*eLAgm
zUzzNly{4b__9`bY+2hF(wJ)=NolX4uzkAF7rS0cy`)K{@d*Fd-Tfgs($o#VZnbM5?
zX?&&!ypLJ#trfX#uQADPPvYYvd-GLv_Z^tuw}0bV1AAjt#(ffxSJ_A0Wk290?zGR$
z=-=KQFAwjD7L`44Zr}60+aD(G-*xlR-uB~9_q=JJym#g1p1s#*X4~!iHDm9^?JfKE
zGXB`x=rdujKGT+6)dxQAwPjqn_u4LweLt>k+rK<y>)x`3)Ap(_&foV>zHQHJ>qRzp
zNB-;&F4((QsYcNL?{`64b!%H|PP4yzr)=ilXEsIBZk5}Qy#e*2ds|vP?LAV(?W^zi
z?9UZkyT5JkslA<^TlQc2!(;a|x_j@3{-V9h>Wu6;Z>-<jrg&lRzFVCK?wPFLEBK3b
zpKaLQz2=*E>=h2Yw693e-*4h8xwoL~&i>;ayZ7Gb`DEj~_w2sibGO*n{XDh5Lb`MB
z@#MCByB}&h_^GYfC%A9xUS)|#dxMid+wF3>y0^{u(_Z#2{(Yu(+xEr>N$u0vW4EXG
zNu=HGcf0oficZ*9=J0XvmWa)J?CS*gWt?u`D|h+Lp8I#q_kRDRXTPg=<DTZN%6nh0
zW88gp{p`KkVh{I5+*!WQp!VY495Jy2+n0ClbvnCX@0yDGy$_0B?kWGt<G|~?$;Ny|
z?jDy2Ui&Y(vuzmfA3v~>DfPgq?9#mviE;-!U7Pk=ZL!>QFmavDm)d*w$EI`bQ&Etx
zy)o_1ewJxIHU@U`2Ny)|*c&0jb<p^5tzFc*1^er!dhLC7(t7_S&98f{85;H~a!lJN
z>1SlG>c@Yeg{^Qe(-W=zFXVpO6#CBEmtZ=7uTvu5-mA5e4m}GL_FYxkWao6!U~j^;
zGrKu8zVEfW(6yUIyURX3`i>3Dmg}~so=)G}DfM=5a`T@(YI1dZg?6mAxuNxW?`h4w
z_WLid-Mg!9+n&yNp1m`l$?UbU<FN5aE#9wjl+U&;Y}x*0A1BzXQ|sFs{^k4L(!UFA
z_)hiNAGdkAd&vX7eG~2+wcD%iV}EJ2&4EZw8HdwlAFbP6RqRgt|J!pW@XFq@E^Qm-
z-Oc;I&U?CNxj&=r??p=cpQioYtMXd>z@3u^_b8rg-d7p(#J=w1%>zd+u<Z5uCgfmR
z&9LWE6~jK`;+%co{&O5~d~n20{QAeedvZ40r7=%F@KSE+zRPzU_DtW<viB|XuKkJk
zBlhfHq`3Ed%(uPs^{j24=*d`Lu@BqhwoS&~+VRC+wG+Pk`nN9I)5iU5Z$qi|-u*k4
z?d|H`ve(arZ|}9p-UF$(*!ER@oww)E8ks%YX7KH6URb-QvrX4N+vJPQ$-hnZw=d7x
zJ3Eni-{<Qr`?$p(?v2#mv4{U&=-!{%EB1(OskIOM8D}$Tui<``O(F+E`2_bGFKgd#
z$K|p2{r3lZligYNncicySIX$!d;8~YYqpw=_WyW=Y&=&59H`mny4Uoe;sMX^1@=yF
z<?QYpUAFh3)9n2|yWZ`-V8?B{;a<$XcM{L-)<=pTh|FWz_mJIo|CXbXwmWBR-nU*q
zWAEf8ZF|Kml=sO!^xa(}8opOdX3uU;)dp*G&ux1}Q=aa9y?(Oo%H&o1(x05!yZemR
zUKhSuyHxko@7c~-wpWw)!`|0ln{6B#ob64-#r8IJEwIVc*kK*BL3eM~Ch@&h+j#Be
zX6&(9knLk{A0}oiq%nEVj3r|GnofV+8~*q5{_x<_d-p4z+T*-y?tX{u?e<rNBM;nb
zRo%~MsAv=5^J3r9?{D^+x1QN^VPU<^=^$U5)@2QQuNOVH;XfO_x4ocs&tA<N`;`yB
z+WSbO_JFUW{(hO0yKM^dzgT~~@X~(4fhwyswHJHWXgTjIYL?%>k)g`&qkra}M0;m@
zb+t+R&oS)Yw^BlD&y=g@_wHK}vR~xx@4c<ud+dB21omw&uiaa4P<r3asuO!(wH@AP
zux{BNhYQX7PH(%jm)Y2HuhWa=)=Py=_ZFts?fvkEYv1gxi|x|R@$XxaJ!7w!cgVgC
zLW}l3kZRbg8Tx+z8GWgp>U|aaI=0N)d;Zqty=zxB?ztt%x#vG$^*-a{;d`e)-Mg3R
zPVqj@cS`$jc^<P}&9TWYTz>PO@As7K;-1ald+C<2wO`7dy?kQ+`>)1qwk~Q}vbXc9
z=>9!=2lga$Fxs6JS!lh(R?u$F<@a{EmdkA%Dq{C~d{)?h<JCGFy`t8=duopCV_j0R
z-{oYCU8K&{-9-!E?x|+3+#hpk{oXf=;`Z9i7u{zkV|w6rf#^QZy8`<;PT$%)`RmTT
z&tI<Fm+$>*?}alW2aa&b>@j<swvYGt)II&NOZEypIk#u4E7LyH=~wp}thusR@5;CR
z(b-S;R@PYW6%%~EmupkA%_W_{J(Gmh?9Ugc@8-XrYG3r{n2mSl)7>()R(tz@sqFo@
zhjIUnEvNQu?R~blvwOz=jLj|fMajAcW_4Y*byv;b`{cdmK09W=y~h(YZKWLC_pWNR
z*?-cFb?=j9Gxq*he`gmMxogkjn!tV8d%E`;v#B1a5SKV`V7BvtquGi!`_`<m7ipTm
zcWU4jo7`<1_a$!LZ}%|6z`n8N-ComOkL}oA``9a*_3v|$<k`3V@{?W3?CbYW)VsU)
zkayg^igpLv9Z^*_QQuzfd6<81Z%61kyUi?>d)7Qs-M1i{Yu~S@BKxif#qB-AyKwJ{
zEE{W%F4x_BRY3>xLpJPTI+VNj%;KzlvwzIr>-pl?o--Y@_SvP+-WxA#XMex#>t6F-
z-MufoKkuFJ;p*OPbA;_)O-bGx-8<KM+5S8GqW;g{pI^P$ex2O813n!}dy@+D4(!^z
zcJKDP68oflPFUY!oMK=9@b2DIQ#b9Mon?PuN&@S?<ntHzuMrm5x3MJtK;~wf15>LH
z+poNSd9PtL%YkhQg8NpUX4|X0`M|!es%`dXTvqL`m@#p0l3v}O%jeqm|2#5bpXXi0
zeQ7p-_p<HZWq)Iu(4JeK>+RL9i|p%X?bti#Z~xwmF9Lf7*wpqZPJLsW*#3KO)aI7G
zh5}6c+E29XQC;_G_vtRieT=t+_nyCX!~X4BmVLh$Y~4F+PSjqu@O~TneK+?mVvx6A
zV)9_``LKQa8{bIm+Ys-(=h}ozd#=p>v?qxz&|d4*gT3;Wp?jwNJ8bXzug$(`amInx
zzqWRr-}-Hz&fwo)z^!egyo+VuEv6fL?amnOpZMHr?{a0^z2ZDs_P-t*?KxJs^uUb>
zi9JU*v>f=`=wTl`bJ_lLRm=A-x+7v=q_xqe#@Ki7;r3(uE*yEXk5@L+o=t~gpGoZ6
zeJPA9?SuX&?Gs-5VQ-yL$DX8xANSspytOxLx7YrDY1VynclYf5X285};?C;58@Fw@
zw>kc1uVBwA+k5ZV?Ol91cW>b5$h~b#+-zUw<nDEN%C+zIJH5TTCjQ?Wx<q>Kx<<==
zg4&z*-bgsL=fw<;{j*rr_8#o<*<WnTu<!QzM4RW^_wU_n`DX8qE1UQ1%IDn|pSf{w
z_DbJ<9(UdMyOxCPU%U6}UY`Q{y}K8x+cUpCyr=8T!reQ5|J+-DFnxcH&!oL+OP}q1
zw35Ri`kunx{zcXM+xxxt981gCZ}M!4{U2TaeZ1v*dmd~Ev-c98zPI09bg#3|u6?s?
z+4i5BWw!tA+_t?ZjfD>UO3<?x)pxe3Drwt0_r`f^3yBkZ->Mt$<^Ozk-@PYmcej;A
z@Ag*Rw=br#ea9>=#(n$`rrI#c-?ojDU9|Vgr4xIm9-OrIpk|d#=c{vjw>k3d%d_p@
zGnaX(O+%;lzPol?_KCVZ-+SrLkG*P+zxJ!EKHZ!4YR>)z3^{w}I7RI7;s3bT`ONXX
zNo(%f-ByvYIrgY)Z}`j|_OiU@_L<J}_Djukvj4N=+1_J+<@YeIW8C+2<@3Etl?;1>
zzWdn+rLWujRak2;`|@Q6*7>vV^N@JIcj7FWeG!7K_O0(P*uQEwIB>)8ueDI<pZ%U5
z-h1WGt=b!!b7h~z-DmdeCcoJ?w|~yw6K4+Xvrar~uX-VUpYZjzy%ih0_Q-JVw43W5
zuy@afQ~PhS$?TiE=f>VdCbzxOA)R}@`1adR3~SvZC?dDVq4)J()lFOW?0Rjww{A1b
zJ~NM=z3!K1+XU~lwNI3qvG?-#w!KZu-t8^(+_KknVej6(7A^<QGUnL$7CY^8QtsYc
zf4O|m<d4Gp4tjmvd;fXUzQU$Qd+)q`y7zCv65GHP9riooO!r?A65D@a$DO^qW<S~g
zaOJ1HM{g<HewpvPcbin~evu74`&NHBw%aj2&pzQb`#zClclUqcR@vwIWB>jKS_|!q
zw|ufb{Xcf^@p&xvPS*Q&?>e=7udG(jzFEK3_dC5$+rK>N;9e#p*#lqaD%f{fF5NeE
zjrZQukACl!f7QIt{G{$)wZHoNPb;eLd%GcHuj0|Pedp)C*jshlV&74vdwY-7ZnaIm
z!?5q*L-Bo#kCUuxp0n)p5V&joKjr-1mbqefuhy-$?(^Qi=e5J+ecUVb_N<!UyhqJp
zy?t!W^1az+NB2h-AK8=Z&$##D<p!I;+)sPAv@fySap2Y7pzF)69oHS&&%M)oKjVWR
z`vv~0*ne5FaIcAFxBZrXPi$36e0E<InzZ-3=92w&n|JKJ<h^#!>Q7Vb-(*Pa-6z_8
zp!m_6y_fZM4z$Hs*!xY`w(n!r^1TKfy7m*2rtjISF=Ov-qZ#`SznHy0(vsKynR5GH
z?)0bos<r;?3(H@z&nEcn-axtadsy0D?6v=?XcZG~Z@=oR@ZKY(3-&Cq_-+-aw_{Jd
zM3w!I%PIR(w!PkaO<ZQ5af-xV&*I6}>38Gy_D}q<_q6Aqy&KY%5Ads4@8x)Oaj(Yp
z<$F(w=I_;8p0n5NqLahXi2gmQ#bNueT@>1v;`_u#IZ$Sw^q=c{#T%>am#ZDy`{DT>
zoBa00`=d?I*nQu*YyUyR&I4VO7Te1IUAIpvfMcJ!>9@VhxDM~0)8c7gT->zh#*|E(
zdk-ocqWkXeea=7m!2KCh_Gqp=dVuRgw!O;quQnetH1?_#2G}ie?cAfAl(E;fpmpE-
z#C>*4KU}eYqpiMI^~tZj=T025XFR-cU!2pWJs!Nqd)Yh_?Ry^V+?&BQ&;H+u;=SiA
zC+ykFzjg1$2`_AB?)0;tS{ksAOC^5q*2}xC^qkh&B%ICNtMugE-lzAO_AL#*yl2L%
z&GyF_R_xW>)VJ6DS;M}nm8<u*9G|f#^URX{Z3eUV#`3P-^W=)(-pO1RdmF#6+I{|%
ztMw+uSo@zdzU=J^HQvi|c9ZR+^>_BaTldypFTdT+DpYpgw=^Yt?VDwLmO9_x>++*(
zkD!Ly-pi9#+XSud-aFyYce^#R0ybTHPuoA4AGUY)>*@Bp_VXU-dn;yt;hxGKLycGa
zS5DevlOkNTH|Jl<KKDgS?BfKd*}v{$*|%wS=l<ETruHZ9?%elKY5v|@sxEuHT*Vyf
zH_zBB$Fah0uASOG`@2i`zA5k8J89aDy??^}>;-hr@0njaV_$edz}~|dhpcC`EVfa|
zI<i-+qhasq^9%QuA3tVy^EI1|&{h6@=T=ACuGJRYJKJ{S-fOeX_G>ZCv2pn8eIQH7
za_>iW=e@pZ6*lSG0{c3qZL+_+{_CC(Zm0GxzH`TJ+5WVB*)=l`+`Z?q|Cxxm?XxES
zgW?qmHsZo__V)K|-Ya}%p51qa+TA|`PVW6#@n%1Z9{av`8%`haZ<w?vzq|c_$fd{j
z`{p0re}?tN-UWHP?e6bcV#9Oc<ert6r|vuLdBEmbO6!3t{a_ozPwV!-e5+~iF?;zw
zxzcZY6qM3;@8GVn-@UnW?}GB$eV);|d-!%Q-#htU*1qb*4>p&5wCoQ&-LkjaOV#fG
z5#D{qTO0Q}9i6;4hvVbk;IPKMM)uG4URt+r?|H@gy)GMHS+5ZD+jmP>aG&P(&o;9&
zZtT~-`PKT0NX|Z&ji+rot5f!#b>!W5wXS9F#HoMwD~Vg~ec#Kl?*VJ0U57--egn<d
zcBTf4_E|`s+8d*_aj%@R-oCAANB10GQNH)Nh5r83HS_nF-EXwXRhHiGfB)%TCB-9l
zr{2%r8@O!WUcYy@_O<6`+g!a=Y;)?}qJ5D{YxjP!+O;=aaK*mw`*zxuYc1Y4^``w^
zS+$7$Us(6<`@g?rA1~+9y~$;Q`<RZ_@3*nn-~CAB?tZsxEc@J;FYo>IYtuf~IrI0*
zE}FZa|D*7}b&-1e+<gx0Vq$u1{iyfeZXVI}y>GU%>}xyZX|wsTubs;KyL%L$Y}y<A
zh;`qi&NX{ywjSTx7SX)Vbsp>9(q!Yk$2RruO^VsEH)(3?-fuOp_MRy!+NU{f)86m9
zO!uDG->~OZ*=xJG1#J71w;#9jo#167*tpd;Rn5;f_Uoj*rRv}J&VCTS|KY-)du8fF
z_MSJLZ>L_(yzl9+KD+mf4SQ=#G;LWnC+?G*%X}bbf$MGw+X#C;J)V6_8Bf_XKJMRF
zF>T&{g^v^LHJt1AI<Q}~`*Oc?-}cML_GuYU*>myzs=a%5CmfjXv~=&&$hWqS)ZO;F
zoR`>JdcJR;9M9Ult9CH&-(qrNZ)EV~eeR3QY$jfMVRLVJ?A{%G4Exr!9^3m<mT}(+
z@3;f+`7`!zFM7LIA=7D}$fr$vkJKbv={FwS-@U(oZ|;ij{nJ;Z?tLniZBu9YVz0uM
zoZa%gh4xxOyZ7EySi4u3fq(y;f`#_QduHrwb-HfH%eQrRfRN7q_rE6C&Yu2jumAiT
zdrlTA*mW#Bx_4>ij=dK?a2@#6%e=2~v(SDnu1$MxeYUk%Wc{(PVqMVw^yoEvBczq~
z^98u<6;8L=d*y4`J`R>=cAXzn><g_v?)`ePaqpkHzWvYk`|R7e=-{54i3|6ZOI+BS
zn|XilZQtDe-*`p$EtM^?-QH1aGjoFH-g=%#cGu4r*ecjQ*_*QT&)$b|U3)d<zwK2!
z!fq3tw0^H_pXT0QlC^fLf)DJSlks!!jm47tHhcHjcpRU*_v}28{ok3HY_}b=uwQ7|
zwr9)2BQ`6Z&aydSvtaLOD;9eZ6ZU-{gwNSXtogX_)7>rm-Z|*lXMCHncfriFdsB@J
z?f0+m*=sM?zt`^GqrGMuC)-E7ys}r&?9U!eeTV(4eq7qiJ>{|e&2y%E%dM5|H-EG|
zz{d1!-~Fju_J%%i-7jdhaZkDbd7Jed@Ale6SlMg7d}MzseEZ%g0TH_-CT;uW{m1vk
zSh(%J@aM|jxyNVQmnMkq$^6l|?`m_(-h}}(_DVUJ?oCa0-P5*fuif<Ph5IHPDcbky
zr@+3&R-C)}N_XwOsLryF-E+%c)7ynMjNcyH);;gvE4y~P&5n{u`?}J3_Eybr-TSCY
z&LQe*$KE&lGWN}4V6{<Uy|H(Ptdh<DZ<Tursy5htddt6WiCy;IPlkK<=EU!}i+Nza
zr|5&^fqO2jd!4v;+ALS%+jHH9VV@7<uHEq}UHd+*yuP;}?S@UzoP+jzxZ7?1%Np+Q
z6}Yfx!?8pAHYx45OBL8?<373AditLkc1w32*n9uR={<ttjy8<vzu4I{EZ@&3T(|eq
zL*YFV{|)Rf%-Ol`k0|@Tf?Fr7zXs;oUuC+vH#YE;-H{!q_Jnvg?Wvn)xNm)6`rdUj
zx9oSXx7#aG^?&dBBSQP`d@A4D|Mu`+iOUgtS>8<A%R2SpUhbc)`@d~s+jqC=%HEA9
z-S(-dz1(ZIglF$7W`+YsE4S=re&uZ6Yf-<qJ%7EGh3xFTuh#tC>w7NF?n<lPZkC+>
zy)S3H*}IFCW&f(?J9d&LPxqNQN$(Yn(b%7M&17E~ul-(@b7?lb9$)vJQ()dV|D>oj
z>+T-A-vwOzm^7RA=3kq*_vu!aef6$B``20V+4<BR*xQ!kZI@dlyzlq=8k;Q-+V=f%
z6}CIlAhFNsci*1Eyjiw}yQc4F?$z4orqr-EcE^sr6QcX~8!eo<H}bj4{wmw3z3OiF
z_M|eU?K4}MzRRZ3ao^G%T>EA&PS|I_p|mG{=A^xgCa~_?G<E%+2b^E`Y+w=G=XsF9
zK4O!^zSLXq_v)-u+UFO!aPLH^`n^%zXZCA6d2M}4uxp=G{?5Hq4nEpj!Nj`n#TlEu
zn;w|j@jOf3YtZ;~Z@2CxJH6?S`%<g^?(N$?cmJLb%=_dgx$oyW*S>cjOV8eZw~u?S
zcMI%yXk_0Pyp?5NgoyusnXB!4o;_M>KUFVd?~$jjc42mx_uu;UZC`n4((X4}!uw~+
z7VHuFy=|{yc#o~=X+!(6mc90$tNz(6t9@=)xLoBxQ^@RnSs$bKs@5~@YwtK}A6fWm
z?~{iN2flw<x%Z`3-QLgEa{FYBuI!a^Rk2rpb9L|Eu8@6^W=#7;vzqs=K7M|$N76o<
zx{{N7Hv}x-yC|w*zm~$Ky+ucM?480qao;LFu6-tj-nN-Wk_UeJZrdw=>#lv##b<k+
zTK?`n;(TTAwVGLbSxevT+o%6>PnGzIy{wrB?5>@jW!GKMx^F*o!U3MM5_>N%Uv=P8
z0sFr0+Shxh9ay{fU&-nHO5b_*ZJE`umxEKo&L&{~-bv4l_6z*u*f&e0djEn~V*9<7
zg!WIqHh*{c5l*}B{rmUc?zwC;sdvS`xoN5URWl~-7il`Vcc0*#eY+IP_KB1w@B7E6
zyie<+l}*+esr{)Ny{y~&tPd<;Qr~-AjLUYZ=7fD0H>&Nu@<)8Xar?KucF%eD?s|D-
zPnDw4UYiqV_I~IN-n(z^$Gt|idu(DJ?zguQow1iOZ2R85ea-v)&WY_dGC#97%BpVP
zWAlZ3lYNu+-N{SXYjpkJ-oF+i`_>$=*(=#Sf4}1ou6-}GWA;9OB)GraFmIo5=+Zs5
zb9nX%vQFKrWOu`k{|Cq3oHGyi9+<b=+Nzjsf555!y@m3J_iW#0Xy<ew$NFVexZOmi
z6MHtKOta&?yl(&38lnBZo?g3qz9`wU>pa=JPWH~;b4R52O_MsaUr9f1-xt37dmWEI
z-e>;B<-i(Am3?+40{e2d8t>lV6u)04X7Ap6^=<nazFX`)P#d~;V)xm7rS1FoZk>H+
z|IU!keQ~_4`{o|;+2fw}bnlD<X?wK~YV8#~du6Y&=E}WAP1pAY1&8m=%6zlec&)*{
zwV!A0Ee|i)bGS#&PPg;o-um0S_D$khve&1Y!`5hW-`=<op?#~(+xOk^ox8U)>%(5J
z*)R8YHQ(8HdqRW#)fM~fzC@<)oyR1!&pR`3@7Zm9dw-lbv}bC|uf1X1e0v(?YW7-c
zG8}krc5+Yr-Z1+oXFGOt6e-v{|MA<~_-Ea|MNfP8{yn*6zi-Gx8`-<Md)Mzcxhwg1
z^!{*DkA2(L$nLpxqRH;urYrle<W1jqocY||^k4h;1}oj!-&g!@udt5Wo))t!dvmKl
z?77!xwJ%ry;9k*+zWpV<8}~k$R<zHkdFtNc86|rc-CemiMXhot?>_#$KV~i5J6Bl2
zPUp{xz4z{Xw-#5C-giHWXYa2)Df@WiUfA!96y2L8Yrens(=QwS`3(EG{1@A7I>oT>
z^~|<?eK+3j-NC+kPY?sc{x9=??6WBQV7EGge{WsShrJf=zxN5(a_{pi-fnaC!e*PZ
zzn$&gyKdYSa_`{Y+X)l)m*wu+J5l$h9phe}ecv@W_wm|&v0E*-%&to<Zm&c8`Tb#T
zs(ZB)boSP%<?L<QUSPNDii920Cw}X~n(4NyW-06!-gjzW$JyO`rrgWhYvnj$|DTDz
zdy~=^+f91awf98(jJ<c`i}n@%TW@py=Y;*WwM~1^>-O%uYRbEhFLu@5zRGWVGYyXJ
zy}ij|?~*GI_c*(<+i!o!w@>)@t-bo$i}!i_FSXt?W$K>r{#kYsMrUlU@k!V#{NJ|c
zNzl~20m|3+Zs@zRcgs?neY<_!Y(gdW?44~TZ?|>%x_xsN#M(I+7VdMI^?#28uk=2r
zxvhIQGVj?tLyvhMQ^h+wzVgS`huUNJ%KW@&xB2ATz4QO|?{AUcvgd%+ll_b1&FmlL
z<=A!0nC_LD(|Dl!lakFp=bw88Sg-BlH%hQG)PKHDajme8iQ-w?O?FoMxl*U@^JIOz
z*LOk6o~7S;_Ekrh*eK7*+57y>)V-Syuibmz`sSXVBk_A1w<qs^xvpVfKX1xD7Tptj
z`d4)AT|004-V=E<_wrQw?^T?tw0B!$tKItKtGid1H}5T*y<(s0&KWkh@6X+9aGdWz
zdwj*-O@A-ii3HELHrO=7rtwbw-nH!sdpf0;*kybD-aB79cJHFjWcz=`68kUwm$X;B
z;JH8E#@e<eLt;Por_Q~vbmr|n`u)t_b5f@Jf1ZB0mod6{kJfqD{afn~@0q&(x!ts_
z>-R3Xy}&MQw}{<|!`JP8{<PkE{?z4tTl32Hx_Nl*tvP;dU%Z>PeOP6go$(9nJr{pW
zw4LhXy<b(Pd!P0D=Dn9SrtMj9DQEvaqZxZkyI=1OW}dg_)9!-370TlKWTxNR%WO4g
zUz^0~y)}{(_no)t*~4xbU|aPsZf~Yc``*i`a<=DUb9ZMk1n%c5y0Ry?gJo|{rrkc%
zoauYiroGtfxzBe0YMCW_cO_rj|H7-wMnFAzuk?mTdyPD&+wk$9w=?S&+`Gi)>)zJ>
z7Q0i*m-a;$e%>Eax8C;aj|VoVPh8u-d}sUaP}_TZPkf%SSLWhYTS1+VdkzWv*v#Df
zdw-bulD&J=3=b^NIK8(Z`P}~6G7G!>-|uXhD%<w15PQ3?@JQ{RhybHKIaB8D<DMh5
zzv^Do{*;e9_HK)JJ#b)J@4jTe%zfJ1I`&GpJ=~i-|KL6oslR&?O&Jdu_x#;+pX=G)
zxTT5v?6a=yo!UIbrthNr-cQY1wh_8Z_Ud)b-aG%%hdtjTj_$d=zTbw|;o@GFJInWl
zUHY)+vVh0l1#>I*1*v53<=@Y5%Xo9ozCA4+ySdjc+c)oV)}BB6oA(?vT(Xx{B+Ta2
z+<<-We?QoJMRD8SqG(I|RC{Cl(4VRMmq#7izf7pzR^qPezUoO_`?e%+*n8o@^S#ec
z743W1)VcS&LBL)^Zhrfj&C9KSuyXG=44c09fOyFM$yr?XaVH<yJ%8J4W3cwbK9%DI
zyS>yF+ISc*+*h$Sald-3tNr|)bNBwyPqP1RT4(Rewsv2LIrqN0xVXI>Q%~9DyB^z{
zdHD2x@z(0S!s>pu^S>zAZc0tDat@ufU$1YuP3a-mee3hO_D(Ug-TUMIUK`u%`PSF1
zpYAPYNZdOwcHRC12RQa!3V*Wav3}dWhxaz_-Lhus-cpMt`(E|OTUBXa-@hn#{a!bf
zlY9PU-P)^suy=3Er%JmQX)=3fZ@jSg;k;B^ee(mhX<iBY*0#R1v+&<%bGrG%K9!g2
z_H3``*r#*m=3ceP8r#{HdV7kZefNf*>)F3iwAGfMWA(n}5AW`|X=P*|v39*(^HHJw
zeCers=AOE>@55*3J)RX!d$jWT?Q*B>-_M<Gx&LwN9jiyn>+OAuqz}kWx7zz7@WtLA
zB{%m@Ulepe@$vaR9g_?8wO#wWcl{ogz3o{ld!HT^+}qyUw~sqtbD#Wl|9xEB8TK7b
zl-kR3_3Yk9(Hr;jFthEgS83kUAf#kp`=4dsZ=wBr0xp{EOAold_g}@Q-FG77?SkXH
z_ReP8W4GAn>s~E~J$uUI|L(p0{O}&Ty|4C7bqw9hA1S!6J8sc_pQVfKoX=I*@39ow
zUsv*DulPID{Wo%w_I_K$wQqsmVjHKQdbTG!S@s%#>)(4$^3J{!i<|e#9e;24ll9PE
zy=;N~el?5iAI-kLmuc~$y*_Wh?f32G*vBHdVQ*~NvV9d_%IqU%+S(mFrn+~-{l@(|
zIoItw`x5tQ2$}Bv{6%EnvxkxPidU@nOm@7#zdPXm-jXVYeUqoC?0p=gW;>Phw%yc|
zulDYE&bqHr`sCh;&tmr;ea^P;cle9F%y*9NH7)1eC#h#_7bMBH@9xPO8)*TZJ@u_0
z_i|Qz*?XY-;r130yS;&i?fW0T=G<5ILuN0}grj?H1Saq8H<PpX+_!dbPW)1v2Q%yK
z_Z2?aTUpC_;K^Qt17DqM_O>1Td7#3tVsGjio_+4lJ$v^}tFo{Crnz^%|4EykV_)|x
zod3U<`QhsQ1(QVfMQz-0;Pxqh`zxQz_pLfFyLYob`~IqKMjL|z+iZMyU)i@`Q)!>`
zoPq<sQtEs6EQ>mjBVV&mXp`r@o9exLw<)pjWfppD$2L1@@BBAa`%*L;_Sm1>ytmH4
za^J+0p?h^(xa@Uq=I(PyD6$obW8BxAZ@5=}*V4UbwzOC;$z!mwnwzpWaL&(t6$;n(
z#If`2o8B0*=fT>jy-qug_hc{qw?FN~w7uqbFZMr;t=X%g61vwkf3J<<(wVz>*16j=
zX-(Qw)mLS6V6&8cp>xB&a$V;Ak^a4Qh10L@5jxMdZ;^c2-V6;M+fDKh_I^xyWSg5l
zWpDKB_C1dx6%R~2qrSKG%>MmTr6l)WTHR~!`Ok7+dqeL&nd?*c>Y1?b+joE0o^0n2
zd$r^b?49G`Yrl7)&i-$n`}fLik+P4tnY#aa{Gxr0jW_l_IKFj{$Yq8DlAgkQTScAs
z&F|9L^J~W4oqy-q@0IUAZeu)&W#8+QE&HNw`0eYeH`=>WPHEq#?3}$#6A$k7dh>4Y
z-Hz>hqGcQHmnwbPoAcRfZ(NnazVf7<d%r1i?oI7Ww_B!>wYOo0k$vO&n|q6O7TL7@
zUb9#6n8n`t9}n5NcJx~RXi3`pSYfH%QMtDL{aXv{vhz3Wuj$X*%jIaY|BPSQo`oEY
zdzi0k?Y*Zw%f@y!^S-(4t$VlLeqeWFUF+U6J_`E;JP+6?^Do~&)s5M1_IAmA>N)54
zYQI}-_kT)>&90((d+PKy?G1S$WG{OB)_yVWn!T|;+50ofnd}%&?%v16xOQ(r+os)-
zD{bxW-jdko`Mt(Y#9+&w=sI^B&rNst8ps*%Wt{H0@AH8q3;rp-d%M?X?Uk9TxA*=O
zrM+_7*Y0gUDYy48$Ctf@250RQ?i}26g-K}NLhr|WXRTXglN9)R&j&ddTLJq&d+ol8
z*+*JF*_->W(WWWy>E8Lre(zl}|INO8F`xD(x5(O@_T0X|X6fy{&(7}Kx9YQs{a&-~
zy%XNp*>-K2yVrY3&0d?0m-iYd2-))qOy5()G|Q%J*^hm_+=BZAo8Roqo_=XhW!KYv
zABETLQ@WdLCt$(7PhR!q{#gy-d(3y<u*ng)y3cl*-2S7rT>IB<oxR8Ub)VfNb*=s8
zPP6xEyMEYvM{DNZN82m+FMBd??=@C-dy^^5`;LXF?{(3CzW2}Vr+Yg(`u0zEirDA=
zXZAj=?~?oee-^YY<ea<b^#9j;f956bJ+NWj-keHdyEy-sdu#3-+3WR>W1o>|@!q_$
z(|g05V)rfye!jb=V$!}_+ve<*{XK2(y|nFn<$o*h{o)X@|D*ZJy#a>Y`*wD|+jqj-
za$lEPq^)+nmfgja)7G7{ByBg;SnTsX?X~Be^R2z{H--1lnDlb*L($!PCtO^;f7ug(
zeOFgT*lFq4?%h*pvG?6e5&KR5GxmP1?BB~&6SjY*+@-y%t557rXDQz&=aaUdC){@L
zum4Z>PE=#qw<Z7geq$l$y|aIw+?&wNz0Zte@%}F<$M#yLJhN4|+O&7oD&alfJ-6)(
z|9xby&?fE!%qNTY-rXU;Z%^F3z1k7mY)k`|?A5p+xM$)9DVy_!7p*qdui5u-tMcAG
zHo1FkGvxMVWZu}V!?j?qoWL@>N#%)qAJ3B7zjY?lzAO4O_bT_a?=4j2+Q+)Ob016N
z?LAw6JMX<GKhs7~U(-(8d#>GsXIl2Uck1`Ze@w7@b7I2Yr@J(5=ei%=yDx%upR7CU
zzTT-Vdmeb}?=O%t-m~)>$9{Fu%X=hL)$J6jHt*Y{o@V#qzsTO;W_|l@{U7(v$~d#P
ztnT<;Ay<ukXS*!+O5Odr*W^Htou$0OJ{#60``lt??-e`jvFE$fgZ($RFWG%edWzlI
zqLq8Uuexu|B-y*SRn&ej_iDrahipFVJ#sy5U*e<7dzs`T_ZoOD+`FtlXz$I!OEz5_
z{r3uNXxzsx-LRA4&Q@Eo?>YO-YWVi8nw!6OkLp|N4eCqxvV3K<fAVeXUdu}dt><hG
z*;Ar=ac`JT;{JE%ChXmGuYYfE-<5q^E*!JH7-VIC-!tvN`K9)|OC6`|-?e)8UX4#|
z`{X`U?EUbZ&Hh?^{hm#aqW7*2ZQfn?jd$M{i{_m|>t5{Dn6qWSv6%e<=dZl`jk%qz
zQYOzi;Ly2euZPFOz0!QW`#$~kw7YiWq^+0slfAFK?e-geT(f_^R^vXG$@lh7u}$2&
z+w}Tg7Sk_#c|ES~lWX6yN9_ii?Sxwr`}W_yyEjwFV85lz{(UT4;`V))?%un)Xy4wc
z^3(V7@pITd6Ybnv`0VT+8v)&YRz}&oKj|~=D?4tnXRpY=y_}6!dv7qY?0fO%#a@05
z#RI7UH>@tXJ-2DknQL=>!>qmU)->B{nOpA_J2ZLk7JX*>cZ*Ez4t^0j;2mnUuU{v9
zPxvP`JJ+*8dqdQx?VT#eV=XlC@IHkq?Y(!z7w@&vKW!(%yJc_2BH06x#)tM=*);9H
zBjK~}miuhGE%6O|xwgpJhnzIr%i}uT=GX2W`#K!Y+eu&Kv(5J1zlT95&5reRpMB9f
z-hF*{%=h)}f4(RB;9t8I0l&TS24ed!+|b>(x8~0ty>C7HxTYEGt*LuqtJc}MH#K|x
zULnpod#?V-+q;x;^4=Q^OZL{dbz5r{|K0oUU&6k>7UsK?ey-osvNLqwtg{>THd(Xp
zm6mSY&*a;;x2K$If7FX)o4~cl_lp1ivG)w?tv$t)ylnHbdiPqr?%5k<@Yn9RpX>hS
z<5Ty$?y$DmrXIZKiJ$ep7k{SPsGfVecge)tdzu6@_p9-$?AiIqX>Zx|d^_Qrt9QrE
zY1!Y?#<s8Kg694YB|GhQ%N^J^lV8GSPZXow`9%x%rmT^)E&5%z@Ae|D{mmOI_8(?Q
z*xSS}zQ6qC!hI=>J=WJuI`_^yBD(LL=9_&R&+gspW5ajg@?W-na?9TCy?bNDJ}I`v
zdz$On_B~s$aQB7g!?uw-MEB|`a_*bzU%EHzxznEW#|`#KC-U#RGw0!+-FBsW@0MrS
zTH1x~d%1ts-qWESdyhFE+oQYx%3fKq#QlE`x9zoTZrY<fFMe<Q|JQr-O>WxpSuyW>
zviQ)R#H*j}PUITxFWa4Bf8p!RePLdSdlx?N-mi8vY;Rk@{k^((KknV{-D;;`*JY)!
z=KJ2)??d+aHp%TR-n!CWEs$~V+ym41)gS%6Pkf8$zLY85d-I-K@9(hKVxzOOV9&%Y
z=k^vHV{$N(>D%9W@U``+gR=Ymw8HH2pEm6?jE>uT!cKT!ukv1d{k$`KxNl9{A9qJ%
z-{~i;dv7-~+Me^-vsbtA<Gu??-}YwBd$l*2Eoo1lbL<`wh1NY@(&zV_&bzwzZP~WH
z3eQdUrCnXUhquepHZOMZ-U%A#_HgZ5zV|6(&ViThH}+OMEwhR55#4)!&fdM9vt@0}
zm;c@CHb2O^BZ+_Cx7pM8?D^Yfr|R>@&b@l!e%`Zg`xhrN?G0JPd!V#!j^!k_i#Cp*
zp6@+&?yXJdq0M_|^|M%4OgU=r@vX!rrTEu=AvMN**NV67-}-ukolR`S{zm0~n|D*6
z>|-`6*_-;kXYY?WtM@%F-L=mp{NVo2JO1pwd#1)deceR+kUvfP(jLg|-Q4qPui}+k
zb_sgN_I@=A-tRD3YoBi0`aQfCcki22X}tIUMDu+)^L2JBRG+oZa$?vW_v_8xwKtOX
zFt@+o^Xq@bp5OCc?v9S+bNHlgxwp*QVeipb^Y%S@y<qPqgZ<VH&${<tQVZM5`Xg|^
z-Y&Pj6|%y6!~VMN6}bFtPn6(J+t1J6?_GC8a_`Tt(R&L#zVFk1HGRMH5;yzx?>FvA
z+QYj4|Gp)AxjJ6%)p=67cT?|5yE9oE_bM$|yw|hE>%h$uv3oNjXYYTerf0jkw|9SI
zQ<t6H4deYP$&2=W6`ExKA>D6pXS@EM3HN#TrQDde|I`a<+c1+4yTwy#_ZOQ@vO9k>
zYajDT=6%kQ;`=sKsqSaL%CV2BZN<Lg8J2tH(<kj|oxaG%l<Uu4rz`*Ms=e>-ow()r
z-V;%Z`?mkBut~}Jxp!{m<Gl*2PVX)CTE4eL%6xy%OSXO2t^M|FPPwvItgm^mSt9=)
z$*;fc8yQ;mMlapEztKc;Ur-R!o(*5M_ohTn+ndI_!|s&m`n{hPz1@2v)5Si>mDwRW
zWY+$>a%=Z3EvvV^CDLb?t#xyE!N#n;Z}%4LKDF_i?cRlbHZvXH?d6-R=CDSle9t{O
zkpunz^!8>&O*nA#WR?9QJ!X5(|I_#CnKj#gvf8mXrev1QcY&&XQi;a<6>lpZ5Zf-Z
zH#*h)K(C~o{Yu}meXWO%@7=JDVc(X|f&11!@7inrTE~8kq2XRunZmtqe=gqJ`8aZ~
zo=VpKB*Phd#h<L+l_-C3?`)k7dk<a--WwBhXwSp16ZX!Fe6rVcmdO6S2l@6%{eQRj
zL{HnkDY*>$zON74lNj5vcV@zM8)fb}_GgNB@12_cYwz1Nd3)pbueMpo#IP^uyROZQ
z^<jIC-8Qm)zG=0+!(1WzziSQl3)R2c%eMBZ?UDW$dsa0q+asItVDGYDefG*bO84Hl
zw0y7khSmG#az5U>{LVgmU#G^s;<pm*?Z0szu<enyJ9koI_a(Vq`_f-O*z0W2xi>Ja
z%C6Sl*xpvU*?z|YzJ2jmr|!><__VKO?fQM`AFFJ8!>8<3lje7Ll=yqkgiX`-shFMF
z+in_c&9rONZq8DUeFhT*?3higcS{tz?mKijeeag`+`V5Ej_kEaUcGmD%)-5_JD%?S
zv&ngn)(+Oarxf1oEzxh>*Y>P&ue?~7?VS}5_Jx?L>{WXuctE-G*)Ge)je8e&KG}Qw
z`Lo?q-u$)W56Im6<RQmipS#NTsU_X}&)#G_u({{z{u<4Hd*&@Ua$q_0v%R<b&+NT2
z&ByvzsFD4*nHTr==%3m9;-~-q#civ1OS!rnc)2BEZ}*M^2aM)!xAW`WwST`;fz4vx
z{rePb?DjhD`?~k~&b|9G#aZ`Go3rS^w#F5Eo?enX(Eq2xZtAbfeNnA)`y@9t?A>wm
zv|WGqf!&FI_x7vWYwb-f$lrUsZq;6XH}-ws{+zShyySq5rSd<U?SI$rDbN4Bx5<=y
zUv2!$y@t|)`=qu`-fMNWeSeL?>AjcvCR&Ra80<SegL8M(ragO<E068#Kl8-eD*O5F
zj|W@#F7s{MdtEJM&zD=9_v%WQ?B8H0u&?iv=$>O1t$X+VD&Mo=(0w~bnZx_MKjrQ{
z_r-PJ(OJv)D!*abXL0D#-YwcX`xO;t@9j8oYwz^kcKaq;t=T)fwA0pai|C%mb=<aR
z#3tD7<GH_&tATf)c7owvi_)gOw=-;QKFn*~7ueZnYczkEP4IK&eRaww_w77=X5TLR
zDf^;B9_`&DymaqRvqk&YPq@El{`cSe)lWUyBR;)juYCCYecygw*xPh(-F_LV-FrUY
z4caF#Z|dGjA^-Q@jg_<E=jPnU_~WhBCXu>5`dJEls+lGC7$&msjk{;MuQQ2b@8r|B
z_vS<d*?qG$->vv|`u>33OZIX_p4_YQaPD50NtgC|J8s)&vzBw;l!hQ{g;`2=N?#W4
z5u6cc&vZ@HzP)_C?au9fc0u1a?w#|yd5^!8%$^4q{_Oefe{!$w)JuEY6@2#Jy1=q;
z&ueje2g&_5f-$q~8fLHBS0S)!w`k3$-L8V_`^}Fn*du-O?cS-5Li>2)kJ|-JF|w2V
zE4z2+GERH(3EFnga`x{#=v}gB>xQX&(qBH^ZymG7`gfp&y*2kKn@Mb1dl}7+?bEkr
z+I#1c?f$pPJNMKro4wE7@zdU>FX?;#*;wvPKU=!D*^g`Qt@wYt*ygkB&-s|N_tD8K
zd-=rK_Fb!0-&^$kicN>5z3sZf>w7hHd-n@%oV7R9yKS%Meq(DjA-_E}5<l%OeCFH7
z7HPLP=2YljonF^{FW;8iS?HXwD|_CwH~qoi{f7_jwNicf%bM{>&t4J6RQr0p-Fx@?
z-n5w_Zo0p3{)W9wSLE%K<EHQN<<UG)_y7I=jm-l4B^f{NnQPd$-^%{b-XiNCdqWx|
z_C0xW%_d-m%z*&wyL;CuM;_Syh;#p}z|?)MMsM~uEjqh*Pmz|L{I8n5F9k2}-?Z@Z
zo?Jt--RG(!_GC4D+B<7+n4QvHrhRWVEwV9jnYDNQ*+g3{Htju^Wj5?Fxc*~rW&2`l
zn~rDuBwoGU)9}G)?+)KNd)&N*_KAK=w$&?lwo5TNwU;|%t9_Nq^u3)o18n4PR9HV=
z@n!Ga?nV3m=1tryFKo2eYwiZSQ+p2Dt$EF2FKO7Yf7{c}y}?W7*{wLVc<(;bOMCyx
zyxQ~MyTk6xv?*2x`OfW~HUHrLql;eeReSGe-`!NRSMvHv+n@RJ4i!BH_G@~d?7lg(
zb>DXP?mY(_vv%*>61^|3xP9NEcc%LvuhQA4c=7DMFM@yU9u@A|cfN~ZpKwdq-fisB
z_BU3@?G@zG-v99Am%aOLF0fg&{M+8;&f<I9Tbk^R-)-6(vxj-#3}>c&XT7G{I!x`{
z`ypwvwaohkdtL4<vS|%+wU_NWw0Ewbug$IpYxe1^nXp$panWA8W~BqKmgm`={oS<R
z;o<AOk=G9H4ZQGWulP5fJvC>U>=u4C*gNOZj=d&zTKi0;&+OsgQn%k6qP^d1gVny&
ze?0pWl|SxX^M`HU{%QaB&iSUczc}7#uT1IYy^($o_J6rhwWmCPnXTG`V|(-h8}`lO
zdA0A~s-nHi-9PQMYL~OuoYuE@_8eoIFA0+S{-(t5|I@f~Uq#hP>*!bA_U%?8_TtqS
z_Dyfs+c)Lv+`W^F&e^BdcI|EY*KhwXM|tn7qKkVK^xXH|nf_q!26LkW-#k|AbzgOO
zulzEJede0K_U^bKxL3vP#9sA!y}e6<m+jS;G&s-_y>@TZ+$noDe%0Ccud-wB$%3bQ
z&04|^T$}!2?@#BJ1B%T~dlfYE_wwESw0H4a=Di|vZ|$xo@$5@EJb!QfjHLa+G2iTb
zI#=4UrkL;J5<k3m#%6*2r5$qnI*fAno|-dlZ^wGEea5%N_G<bY?OhSnXfKibXK$8l
z;-2|kEA~pbB<}ZM>Ds$$a_E5q?z7gvcR1PKGZEY8ap}=s;p4OSdBv~U@7yO~r@e3c
z-Y4>$`;YV|@9$>v+2?F+vQL+%b&s9prM*(z?EB_#JiNc~*rC1ZeZP0-wHNI(zPWF&
zQzP4fkXJ>!o%rJRai_oCJLBn#y~gG}d#8LV+2bfXdymDmU3=fCmmXO4;P~Df^)h=6
zt&Z*8)f>F$(EM$CP1i>6(~;i3mp69X{tYec`{Jg~*xTf%Z}aJXhqZ4I^S*<2U-zsq
zVcffbrN{1LbiaLEyYYVJOq~NWK0n<X(Dq`#;S&2jN7~Qt)zfC!S90l|?b1hE_ey8K
z-Mj4Di+v|w+3tQQQDYywY|~zbPWOG&MLX;>drs{42=>`?@8|3Nm)5V@)3f;1-dWl^
z_nt~+-Yt>GWbgP_eD5MAuKmZDEe~Y#oZc5SPifz3pSycqj2ZSvm*3o5RCsqkQ^%UU
zshgrLXI+Tex0mJq-sN9g>_5Nr-s{*Kv9Chn@814xHa1T~U+noWcXw~3|NFfw*KzHW
zWjnS1@7qb%Yv)bhd;M<HK2NLH)_Zn2?7dfddjDE!Z<|eX75DS3db+n~$ECgBRXuD>
z9!=XDbj)CX{Pmf8RKD%l6Zpo>o|loyUP`EK|HS)m52RN9+nY4$>w)XZx_b}m&E3lu
z&c3(&tM!4z^Zk48&obP5y;XT1Pi3Xer|T>B?|Ob{?<U{v`;}vV+c_@jJD_7fd+$O6
zPy7B02lsw`)VB9^+0=crxlY;j9=W^!y5x@CoINY;86D#fd|KVV@0C0AzMSq)d-v}8
zxX-dYbMK>?rUSNni}$Wd{J2*m=fFPY|NMJ)bF}U2T*tH9Fu-fCgAwmOk(5(=uB@7B
zllmleFWcG+d-a+WY#NFM4`$u}vzP7jzP(Ofn)_ZnKfkw^?eX4v<!N@Z%5(P?u3obD
zaoxqe7u8?xabG-r@Af#Gy$PrG9B9%E**l$Y{oW<X*891cn-BQ6`0bD0)^Df2e2G<I
ztDODmBRsatl#KRrPd>Wa?%QKK2PN-4Ctq0a-JsxXpPW)`Bj<H|-;bYewtv|4_TN9n
zdSJ<P2D>j}clNT~ID6m>^V!|vX*2ik>P*^av+C%6Gar@x%gT4{UEr{O-(G+A{reby
z?S0L?ZqL3qu6q{#(%Cmll4GCNI{pLc-W&Ixohxs1#C_X7!SD8am!Cblf2-eyefF(6
z`>wR7?_KnL)}FY{ocpv_KHt0I)uz3kpR)Hxl`!qink2LL`J#E&Rk3~hra5WtwZH0U
z)%VcgfJ%Dp-b&kv`@N6M-plQEa!-Kj`@K`wT;E&&B*WHuJ=b23RU7w~xarv+_{+VY
z`6i>oqC-aZks9ZA^Uc+_eYfSVO`*EU-u3D2HZ@Am>>O=Z?o~~>w)cf!*ntTbCt5rG
z-ESWtroQ)A&@}sv&$9N5*i78NRE}YvcH+tX`{I}FUB~oeZ-Df<eY}gx><!X_?dA1f
z?Qtlzwb$N%XMfS0S^IQC7Vj-7m)$E}$#&q+wyid&MV0Ms8=Uq`X?C_%SLxYTcIon7
zzGdh4e|fcbujcKJeW4o;@7;3ch0V3rDSMxEz1VF&+ufGEYQx?Y;hOs$n~U~txhS<)
zrCVd)%jF06NEj&Zdy+4B;B3oJYv+P*`|s?SzIR8^%e_AoKkk{l;H$MM?-tt-S^2%b
zpN#e%5o@>K)Aipj^z$wI)s3n4pLcoe<9+|w_SBXmdqV^Q_Q>73X=C?2#{T%Ejy>~2
z9@}U&x*TAv*k&Ww)oFi}_l|Y?oTqlv-hJM;X@%te#0$Ij@+xs2m~J}JS}pw5t`d(8
z`*;%g_g%j;-+t0bxxIH<EbNqoJnZxL&)MfW``})eFpj<3*tPdRo4~#AkVUP1d{*0@
zm!2>8F8rLluPF4=o|lUR4lMeiyEi~Id!Nxw=6#{x7wp}5_{Z+{KHa_Z)PC)$|95n+
z<P+8dmv_|JOiM_%<$u0vpJmY2y)hSV>=n1&VpkMvxp!Nfo&BDvGxn@DSKoDKzVF_=
zW!ihUX3Fh1uHJ36an;woy3$?yU;bLZKlrV}ewX8{`%nMdzIS@NuHEzcAA9HWHt+rP
zFJ!Ofw2AvNpGxj+_LthbQ1iI;-7PHp7R{Qsf7+Lxy{hxo_wT#dv2Wd}8TPi<8*H3A
zAKPnhdbn5P7SG-grtACm{hDE;A7*jD{+Gz!(pwYvuaCO0_i<FgzE5vU_nvw0v}gV-
zMO(GGQ}*u3YPVxLx5D=0yV-k#UcTL%^mftShuMDnCointS77hE@4H*QwP5DSy=!L_
z?b-j!Z|}5kt8J!T)Y>aJ)4_hqy^g*44_EA+Et0U$leKA&-G<|PONH$BU+sCnx7qpV
z{_+{}wzm#E-y5mkyJt>>-rkpGyX~@9Iqbc1D#XUAXwrV)+adNh%+A{9vM}1unVn?o
zw@+nXjr%(r2gA%gW)oiQRj4txuMrO0dtlX<-A>gr_V2rMe0Nc_w!QuR*1Zfna_q9U
z_3W!#x?}&TG%lO&I}ZDgPSM<J_`TgG<m0k^|7vsW)ouLk)8zi_y}M!CzE7n`>{h>6
zvrl%-*S)KP7VX{gkY)dA_J%!8(sS%`nr80hN&dV0xz?+_oB`Tahh+}zyZZI)Uc0$c
z`}R9+-F4))#a=CeWGkMIn|r%Y+1k#N<KMeD``&)*eNXmGI#9M(GfZ>uRpWk}Fh!|-
z`}4l;pE>2vUe?6d`{fdU@4XSTeQ&aD)9%n0oA!u5KesPUR(P+t5TDI^iv#v5ri<)q
z8yW1+WgOb~X6m&)o8~nfXua}j?|NC~y&NZ|?$yb$-@8%Z)ZVb2RW>dnJ^RCtKid0R
zRLg#jxZ7@%C1>q@>-G1mpLo6hY{{=Z%nLs6Tl)HhjZ`fAUW319_NfPT*g0RHx1ZH!
z=iW^^O8Y}SJNL`#<?gd6o4?m1hHIbowDSGonOF8MdFOh-qfU6Ad#f0zJ-)m9hNW$d
z)G536t#7Qt_8s3lrCMw6uHFZG8>N@-;l8(I@1zHrd#4`n-M#4NpZ%ZKZ?_5jvdFe4
zq-meTJMMia5+Cf%u9dZ)FmcVE(pO#wPQ~`@J;fMhc|NMjCSm1do3js`_g@nX+;fIS
z$Hu(!h5gGtU+uf^Ox@p=C3K)k?xRgV|E>MfX-D>YhVbmW!O^w1VGY~<bm4`2-<@>W
zd)v-){{n+adk=36-1k^YVBhm8+;)4#`5oeU^!MAwciLPFdu4wj?DU@PD$n-L_`78v
ztM(*Y?$hS`Zj>wS4RO%lC;pN5fc=%j`>qGs?QKvK+1LD|XrDpk!M*#cm+kM`wtY|5
z&EI>BKgsWVG;`wK<JXk!d#@GjTWlw_?_|b=y_HQ>dyLYQZG22sY<{wf+pgE+*t_Xb
z_5L^7SN6DZ_wHrT6xb&taM0>i{d1cTvrYTX@ZPX_7XNvF#hRUaHyKRW+r8zbjpJMW
zy^qAt*)3UlZ_h++-MwkXUiSPymG*P+{kE&_+Gl5YR%`ELy}msarYrVZGyUIFRI+sM
z(e9acJ5)>eWSu;)=c%KMeUG_|txoZ5`{>{6ZKT%g+RqC#*th76#D42HEB1ObZreY9
zyNKnniHG-i8}jV?CNX{A@^4G_-<-s>Pc>w*-M*JB`@ihjy-z&y`JTnLeEVDlf9wxE
zqp^4UxjMUj;zGMFb~IV{eCXJ>?8JN<tAd98g6p{KjvsK?cdndy--J!|do%WM?+fmG
zzkAhmIorI=)q53ZhVPqe!n}{?u;kun|0nKKo4IO_j%L+fx5TgecF(ojd+u+?p5_DR
z_v)=|v##GMvhOYLx4l1RJlbdB%(8D*#?w8kj~VQj`^LPlt2@~~<UP;+v&9qkwk#Dn
zpk=kgX4Vn@z49VV`<Cd$?bZ3gwr{oQ*FBdSi}v5{+Oy|Kp0ItB%l17B1n=*=Wcgvg
zmfG8Wim`|Gw$Bx}lNBo2o20pDudDa{y$bhC>|VL0*{^!ew6E*S&;48P9^1D-t#sew
zQoFsf>woSxcp__muYUHPZmpyH=bU4<Ij+&NcbjGXUW4U}_8R&7+CQx~*cYr}w9imN
zcJJ*YaeH0$TK96t^zS{Gd42CQ$t`<Blo#0*Ki#mmS8w6omI>bbmJ9CQ`{mH{y>D1<
z+D834ySK$wZU5T6Y<q7l`LxH|)@@Is?&rO3m0J717IE&ID)V`-lJgV0a(UhZS7H|1
z_o;Cmh+gkz+q1j)fJsT!-fh<=?{W5(-&=iok$vHbJA2>9UbBA2SG)hAAm2W*WO=&>
z91HgHeVBe=y;R?R=?QK7|CccA$>h7e@8#69d+nE1?ydejZ=cN+pS^$HdmdOc>C|q|
z+v^W(GJ0rxEphq2jN0J6E3^7`8$_D!GxSl}^FwjNesP&4wnq+R?VWLy#YXbw%e_{u
zH|(Za{oi|DfM>6Y!~H$$#RB&ppChyP_?t$Xh?3I1#}$w6ZFEcAzx7b(-p0A@HkZOH
z_FmdqyLW%XguOPK&h3|oKD;-}o%KMB;e)-u9QW89zA0y$-P~&P?p>k%(mf~luz7dc
zTxl_~PjEBazkqd~{U%qfeY1J^_qtSGv~^g$ZLjX;{d?R}H}3uR^sN0((FxW8j~?t5
zTlZoA>{CnjCfshY*L(17Z_iUUdltju{ht~x?6bPpZBygJexRg?d7r95*<O)ZVf)tX
zinX6-b#&ix&4|4UCvxlykG$O9=)G^B=7ckQ(u6iyPt?4+-}0aMUQ3^peap7Q@2xsI
zbMN}nReM)BZ`gBoZkz3Up8mbbENuJs%W&J2u8gsMr!r~xJ=Qh5CHIu>k^Owgdg1Qb
z`(u+<?fqu6%2uhaYu_UlqrK1mf8EQe@@QYu36Z_t25$C?wwLX_)^ug>i+3OQHVVAl
zbHeo1-X*UD_wBhav-`nCC%ZcpQ}z{VEw)`?`e1*$sg!MLl+uCMZOr>T-)-GHSL*9t
ztNx(<8E2FBzIb+Zuj-TacB~(*_Ohkb?EhCeVXyJ;^!@ymQTv(B-LyNX_kQmm-!j{u
z1vmCo9kAW|vD9eq*XOLZZ{B3vf4lg8uknX?yY81dcB!Is_ciQw+<QWeecwjW0=vMb
z>3h!J`@7HW(bm0+vyWQO=321t>waDv3ti^@SGkS$e!FwScDwkmy|-jP?d9!UVUs?=
zcF#QR-o5;Ci}vm@ns0s0W656At!8^!<mLD6{#s%4*k|*e1^2vc^ORTa^;*zqudM#r
zM)AR#z414@Y^sH>?wuX-VNd=O+r71}?ECgDUBCa5nb(0+9_@DL7#Zxd1?2WN)E~6F
z-gIGaQpI)au<qr1Q}}1uE&aJ}uQq3lO=4{JzJeOoy*ZJL`?)TO?=71xWM|;rvaeF0
z&Hj7!R2%;Op#6ew>-W}s#qE7M=jc8*{*rxi@9yodxt+HsVpqammpDbcjypm76r>mI
z{oy-(Z^~lteVr!T_rA}0YbU0ryqE9cn!N(DYxb>`;MvEuapQjLUETXEW)$u#Qx(|z
zvf%gL4YL^bz3yDM_qt#AUYno2d*>***zXaS**h<becvm|+I_doIQQ9W74FqJ+qEy#
zXr>KIe!#w;OKbO-m0#SuPf=iR`^o)#j_>!{pL^o$-oGaV_Pu5g+s_oobf9Y4Nt+0H
zPy5+FrtPgQ<Fx;NRA8To3%kvOM5SG?COxuazrS$r9S#E<1&7G}0eUz0K4j*%ueN-!
zH*r<}-nkdo@2j4-#OB?k>b-v_pE$t$^T3|78G(B{xK8cUJ!`oC@jqd^|22(!n{N5;
zi~MeC*Ck!M*Y$(pp2YC!doyJc_6vTPyEl0ki@oJf>AgHtQ*9FK^7g4-lG$5lI(z@)
zwkLaM8cXgww4rI|apo_30+&ne{jM-|&ziJ(duvv{*mFspbq}9U``!i2oO>^c>F-;Y
zz_|CdwYYT>gZ92t-=Ev?u+HEARIGIG&hDnYwOohxCbS6dJ^AjG?Xfp6_pbO>u~+Cf
z%l@U=9JYJbGTWQfDcBd?U9{IKA={40L1&N7<i&eK9^Kq4WuR&=)6>1zBzNgvvGu9@
zRav&}c_-g)r|!tQZ?<rtoia<&-dU%$_BQ|P-g|M+hy8nf6s%QV|KAgxuyXG*Z!Y`a
zuNUvz&=RmG;>L5kQt3>4%ViAvTDDKwyKxueJ{x`J{ip5SZNgM1?49j6cW=)ry}eyC
ziue8eH`_XVyXpR&3p4j!UeUWxcY532X_J}v6@_x`d#xG2n`!=Gn;ki8_U?;Tv3ops
z>K?9+CR;C$uzgqmp4eNzq<F7!{T#c3TSa>v!gB2QCePiQ)nR25*!yJf>GU&u3!8)v
zum`{06I;r%&;Drfev`lR_q{n_Wgj3{wZCg$)*fZ&rTdrsUA|Y{@8+I`Gd1@L$lBZ8
z;J&<<PwdoQmPOwCgj_!F)tFMcf6uY&dpM8D98h@eyuT!Sn%$l{=6zQMX4>byy1DnM
z?#(?X#ZT{Zzwvb6)`he8F#J2b_gvt|{W0+mcdI!a+xPm*wLQKT-Fw4csqb4HCbUnp
zfa$=b8%Dcd%ZThP>*Lt>*lgL}Q=32USw5$2-<yOddw)#S*}HnrvpojR%=<WA@$W60
z^mos?yGw0$-#TlnDfeJ+$;8gRw_oho%PjR|5A)3#HWQZr*q5;E>t3TpiU)R<hws^;
zZobzng<~II)rY-4nQeRJ?I-T_NVD3zXWC2q;wL-o^kb*(KiT89zkJ0M+nE)*`wx4p
zw@Ef#wU>=OXOH~a&HG}O?AW{W=(;_(om=*woa?gJ=l|+`5e~2RT7=KF%SiRJKl3Be
zuEs`w@2>}k?N85{yLX4yyS@CI7VXQl$+hPY%h-SLXwKfY?~ePbvL@``*O<AFFI-};
z!Q1w|n^&&3Q(nrr&*ee){zjhkz2V&tc4o&N+*iGO!`@}5cG+K;uyuFuevy6N#|8JD
ze4wzmy`5*@M^Wy5Q9mB;?$-UdS83y}y$%5ld%sT=wNcJt+2?hAn$5#cvNqY-`TJ*u
z@7jChmD&E(J9qbL9J#gELrBzSR?xLQKTa|4cTzsHCpDITpFmQtJ(uH;eet=8`^}uM
z?pJTH+Gl<>ZGXMtioIsfKJWb~b#*Vp64U+n91C~ZKUB1xRLi~JV4?ZmDQ`FI=Ph&E
zYx-$|-IX1m_o`Vo?q_29w?}}faj)FFE}O%>3-@?x%-px2%H7WQ>ZbkPwj6t=S#sOw
z)Ri6RGCH{@+%|o$me|a_e(M$YE3ioKOIEnQfAP^~8<j)P_T)6`><cuVxwlzso1NYh
zM!U4mnY*VO&fGI|)z#gl0Z;ZmWlY(d_~X#t*0o7{{ECk3|9tYS^<S1iTke3sedhlj
z?=`tTXZMTh#rx07vD?0V^J@Pc*=2j*oH@34?>~2&s68k5F0L2bFU)moFXM+QE7kwq
z_OU;i?3*WkvNI8VyMMAl)80n$mIJ{zmsyGJnZL*C-t4`~0_J-U{a9#Yusq6o$${7V
zC0w`f{qQ8f{`cqPT{62I_U{P(wzpm9>VB<n0{iYer0!o_thqP)@`1e~t!wr*9^P%w
zw#4GVs&<xrn<E$xM72KHzqZ+KUwCZW-kz_?dyo8(wU;<*wb#!4$Nn!l>-RnqK4!yh
zxO8{Z1Q#2=f3xg&O?b2Cy!PikZ+>jvYc})w-pc<6_nxZ!xwr6c*xntrxAyLx!E@k)
zRH{w$|5n>M-zMyvdi%!Slh5|<@zUftz~*>m@7ohH2dt#O+kEi7xOalG|DK$s@Agj3
zK4kxE%b&fi&LKA5Z{zK+Px@+SrJud;aMIrc*T0<ETOG@H@Z!s9yBnjp_i^dH+<T*B
z_5SLZqkB(iFxp!G5jfDaf^TnouigRG$(?(9n|Ss+G;FoM_h0z{Yr}Ty6CcX#C#l`o
zE5r7BuU!1{eMbdU?Z3ahy}$HG_+CjaS9=eQKl|&HC+~ZFJ9KZ<vtN5wI<DNOyF_5$
zV*l>_`#3)DiT7N!_j5wqKH)fFTgCQoc17oQ@6B5zySGunaL;wUuX`^2zq@BoBImx+
zmHm5L{;=)K-@f3$5vkt2ZN*1+U%oSa-@ELid*j-C_FVFoIbit0V6UzB^8N4PChfZS
zr(<u>>9oBJTio_4y;j^G=&)t)&9XUrPi<AOj}nR7@A!MV-3r-Kd%sl4y&GAr_UE5|
zxcAPTrF-v09p5V(6lv$d6SMd1!8LmV{ATTcIcv&Z(;MyfTc35<Tyz$4V6SM}r+Y<l
zzlsyX-pO69wkIN4_qBchy*DOUZr?UJRr@9zTYKSJ>%Hb#Z1z5<rtIIpeEB}vA8LCy
z9N)0#{s+%}it~2w-Qu@nzjpGOJ)vj%_BOG+**o{;hrR1%^X#L0`)p^t=Giyr@Ta|V
zDtq@<T{qqPJ6Lz`?gG=jr>9-sd)0i~{yAG#?ft34u<z35Y5N%FP2DYfWBK0Cza#83
z&+^$Q-(%b_-YvebC3o`PSq@D5TxvG%VbC<%Z|5{)uXpp}J;Jt!_FPb(X#eHJv3(DA
zneNqFIDe1L=LP$Xp3d1LaeT?%EglT}3R@=bSD(MYI>2D7&4c+|_Gi^s?QOogZNI*e
z$DYU5yZ6smth0X{B5!{rC&0$;eu6#Ez0ke4nBLl4F0b4tza@JA)e~p-E6Y6CbE9w8
zUfK3XcB{4K?t3b=dhea)b$i9EFWAk_IBi{Np<=HuXRw!f$MU@!PU!3lEwtPFl)rNy
z&k6m#D+(U&iECTGw^3`=-j8Y9_sSX0*eg?Ub+7uZ*?W%OUcYbVa(9~rT&y-fxK{4_
z6S>hwIYY(Pi1WYQ(XU7M`XwK*H?pzWbMNV;y{;K7`?8-#*xcl`+pj$>XOCp%vAwm|
zSJ>;^jJE$SmcB>0UHE|Od9!_sCeGO(U$$s3<JFaWR@_qDd+A-Oox-lzy&-i>`<(YW
z@Be37xaSbRxqXwS*PatG()Rqj&e?Z7bhJBh>+as`Mx1tc!nf~{nEiV1N}Ier7yLB$
zu3YD6U&^7n_xqbU`!@vN+$YHTc;6|mg1z5fZ{IVsnEOD#p6A{Oqucv_{=Tud_@|8R
z-qOf@H%xl>1}WaQpCt5pPua85eL5%8ZB&Ji>^=DG=iZ2QaeKwMX4qV)Nw=Am8*i7v
zGGp(D4-@yEHnZJVTE@6<;cQb|wa6d)r?xHITV#EA|CZ8cdsG(9-Fy5++}`z1Htn5o
zaq-?Svy}Ii&-uQ0|M7?Rk3II-waoN(Si}=_;NZvkws{-)4u*Ps-t%=m|DL+uFMIF(
zx^3t7<oVvTeT@6EW3&!@SiO3$@BC2v#j`s1F8i}-|IyX0`&Hk4+b=y&VBfWTuDx+r
zm+U<gHEGW^lcW3o71`N`v2Q-GJCtc}XMfIt{|653Z{S_OPw6J>UdGV6y$?F<_UmsF
z+Gn{f`M}$K3-(5N^z6O3&c@bF-hA(UzZLd1ckb-1?R>n4mt)dijt`1EZ|+^WBdPoE
z?&cjc_BKmZ?Bxw$IPm4p`n^uU9ea6hPTeOwan@e{)I)nBJD1pOtMA{Fv6{nfdDpGI
zYCc={)ZFpe!x{a`dgby@`)^8K+cVK`p^fJJ1N#d3*!LD^iQC<_4BS7rdfi@GIcd8|
z?;q~fvEtjS=Ec9)Z-dhQ{@(7r^Jd6cPk%SbZb1y^ZZ?-2d-ZE8ZC5@`-<Lh-*FMXg
zANG|@>)QJ-F=8Lb@88x+$|W{M3<vf(t<$wv(JkJ$`@oF7-o2Oi={cny$kqJ1_t@Ua
zHo;Gp?m0WrY5$YrcY8VC@$9b+ZrJ--uxGExrqlbTf79B#<o>38+m4>vTfQf7-*wF;
zdrt(vv1v*=y*HBIcJB<m6?-dx9NoJ$`{rIArQ|)fooDt2)~W7WT>ED4oKS;3F5E13
z_j1_w+C`S{=l1E_``)_ODw|>U-sbzgdneoQ@B3`OZBO&Dvb_;&U+-hJowT3pM2P*@
zK6!h?xb=G*S47%{eo)!7cKcN8hiOxH?_Idk?#}t(z1%y_+1zPr+piiiY43vWF8j%R
z3-<E(uCi-l&E7XzB-Xb2%9lNQbE6I@rY+yA_{Ph|nQ`I10|DmtIUBV0FHwE7_w>hm
zcAx%N*>JyU+BbjNa~lrHjJ-ciQ|%7CU$}Rg>1Vt3lc(-&`_gP%*cGtv&z7xwTe7Y8
zw{eT_n>5p7pQpFbzHPFydul}Q?hRSIdT+rF&AkOqx2=!Xo7+9NyS?|8h|}IPBCdNg
z<+%58zgWE2EU<Q8Q6ish;O^}GT;?2mroQ;F=lh&{dymNVTgy83?GrBw+q*}=VQ;mA
zqx}|_b^9zip4hK{_|4YqxsGk>PnH8$U;W>!dHBxm)oW+%&D+OeuY3LYUJqUIz3WZB
z?N7UPc-NCfGWNedzubE+)b&7SvHpRmu08vGKK$GJimQFU;h!6O*^bNYed5@-@5asV
zb~*Nn2Q21j?W<bibU<&d+5VEuW&0#FH|<@ecW1BKlKcDg7!U7xWz)C++zZ~lAD_>&
z{lHkcx1sR#p4(zk_SqBW?q%Dt&UTwc=H5RpyZ1`z+3xkfcx*4X$z+?}4w=3GB(?Te
zv|ifV&GOI6RoioKwq5<+<!(WHC*OT$$-uzCpmfV~Cx}MI<z=hPVEn2xn|B;e`fZ;1
zx!r;#Xyta0_^NN_yFfGyC+S-6fU)l=-?9L)TaJ2y>AyQOEhJYPSStB7f%RtGYX#d2
zQUk)OEr)hiUz7uzSNUIN*Uj&7<{)tx?&4?%^WQBz1U3&Q4x;ynHo@%$@v&jcr<o}B
z!PF%0)&+}Cp1Tsm+@3wZ;QoJ|>}6i1+Oo5n+uMRiXwNo~{<QZuQNk?5=`h$VkQ{3g
z7udZ2+;U*IMVqYJA*Rp0^D(0qSWoBnRp4*{sRiMMNee6(GEKm8I?l^>ENuS>)(?^c
z;ZW&~V7c^*bHL#M5`*DuCvq^vU~({8#F!KAHkcTQ{x%~GEZ&w~h2kEN+~)LIU^_u<
z)`Z#Ht(%@(<lIiR43gOf_S0X^^>BHRnm3>3fz1N3-5;&D0@2Ke)xmLkVI{+^C2As8
zJPa;i{=el8V0n-l5Vrl_xby435U_cN!q@ItWhnw?gXBPX+NbAW{-m!`a63TaAl%lw
zbt{O*hv!Us4R$lSn%@&bz~(idngTW-S^iXs4qR-Zx9-jp&sLeKO)oN6efk6(|LY9r
z!R0~v^l#jQ#~E|Sb}-vRNeN7!uDfFq$uZm9*>D;-oH>jd!1jXlfv~LE&mA0IwP5qC
z`={*`;_LvkL2@9x@&F{vRntUK^nv6K+cTl41&N{K?=P$|^nui{Z`%cS-<2a77-sxT
zErF-2i^~`9)YJFdIcsB$CC6k5uv&?Tg<!oPb3oW@V;Wc-#NPa2CD=Y@iMinTKEJMQ
z*P(^lmO14zV7~v!Qm{NoEeP+oFxmC%Y$sTM%I5x^!E5iJ$k*)M0OrqKtBjIvKzhSU
z&w}-V*tl@cogS?5|6kV;Y;ODJD6qSb^;g-hh39|Yb5d5-dc79bPbXR45HbXZ3+t~M
z6g?JBE%5L-@;DQ$M%1+#tj6QO{#~l>%Pc1R$O5}#Ug-re4YRKxNOIThk6XZM{np>!
zwR_4!u)ARL3}0%%>Yq=<jQ`xFZ{haB)uS<LtYt954W#6I8l;^0uQVB)E<s{2{G-br
zY$u2xAt7h7bji6LpJta@o-FJIi&x4^f!zy|2jNvbo8WG)*L(+N#|s<*tKD^#ea8+5
z7OP0Xd*Hb0dUO>m57Gz1&G!#%pKNyntoF#&d0WK4cB1&{e9BC)KJUlNP{cucH%~^y
z6NrxuzivDa&a>!p8M}JG=3U<vj^Y2wvbk_Q%{PU%SDX{wX;>F(xrk{o*qs8mQ0u?E
zloq(3rc5*jhxfVDEnu~&I~MMg5n!{rbX^)8W;`A@;c7u{nq?fbeb<68uphUYF54!c
za2V_^kQ@km8?l4+CH#W87bFJ5jem?#)WXD(={=7E;eJ6DuQqK4yR+ky8`ykgxmGc2
zuzNsaej77)tap^!xiGqTyFxi5IIPz0tw7PkeYh9y4ly+uF#D&NiYZ8c)W!Vm>putY
zDBQFj93FApk|_SK&G6pozj7MbyyxK@yWby|Mv+f*z6*AXj&UP6UxCboVZ*?!Sp5c*
z2hpal7{UGov61m+r(&=>HBSYD-H9xxqjw5!?*4}&+k|!8&65{wvC#ip3id<hO9K?Q
zY_!?|=U<ZP2fKgmWLvPilc#*xKKFN+W#&~kF#qNr7qC3czO**Q9j?V?;BeK<Jh|;&
zsx6oelh=%fl>gScx?uOh#6k2w_9%Ec1>$4FfxHvIZbp}zwqY&UybyWJ^8eI>Lva6`
zUo5(FZ`QJ%*Y;{#XxRFI&8mKT5yjrGPU>(!h1typo4aPO7g+6PX|^4Fz00=k$lVI&
zuZq0^mWR1VCdYc`sZvP(SD3hM=Z;o=xLF`I!3S4^-LiQdYP|uHdpzS7*h~-`7e2?y
zhGHH}UG6jquz33G*%)TrK71c8_QrG74*i6SJNlnIvG{vn4p_XIj}x3;VP;wWlLp5v
zh<{ZGl7BfjegLa2n!0DlyUA}X%2grZyoEg+Y%fSH2y-ym>^#W|@xN&1ydAri^P<Ra
zoE!x<=j};b6mgJVM>{dNy&yg|Y+Ldb9De9>qFPhH=Ba6{1e=d6Z&L#)k3eEszmISG
zFEnlEjO_vz+4FXR&AJvDgW~_ter9+cdl;AjHg|UNBomN++w$^l+LD5nrJsAiZrxN4
zkq60vF#kMu3;PsEdviUf;La<Z87S%BD&+)N-wL6fV7ozjVEFUa#Ta5RIT&5gVGec=
zj1Qw@8<WA}9NSUbav*s{M+<nEEN9)j<4}6&_K=51wuTzSg5Bo+2USn96rvAtY)L9u
zjo(5Ja2b*wwbq2qV$%-6-B-c<vwIp)+}!wzcgHQJiC{l2-qf(m^aL+hKgb;*>|4_a
zcFP+D%=+)4?-H;aNF6S`SJfHpPjq=V=8a(Sd6An?;u558CxZdlJs|d@O-Ht^I?A*&
zdqUFo*!|Yvy!&q%q`wA|2jOSaPJzt=v3V9TgWXhra6Z_dra$Gke^}hH^R@glu=uK-
z6ToJG)Pk_K_VpcRdo00js8Cnk{i*FHioD9<HDI@}&+dZT0n!J;|3s&P%>=P=;i|<U
z@U)4n#<@liY~C^@d$9S)a-Of2z}-;E{?zPI<-;ASnzfeRp8vsqa$6$|b}vlN4aRz~
zSs;GJ+=*az_pz7Yc(gw4YW|4D-SXo~Ua&nf)`-3mNd0@2<{kb2UV`Q7^q%jSqzq}7
zg5*H>f|nN9k0}dK%T18novD>@cY^rX@NM@JtZFu8Lh}E56>F5T9ArlG3<a=zK<pEB
zE@olntUH)HbazasT?F=jio!#<JV;IUsp()pgV-+@7K8nL{aZ9Ry$5#hHLbXwwsYYm
zNLl@9f+pBMAhjSI^w8GaV8$Y_d8h6+?zDXYDPur#AUt933b35vEETZbATb!eEAks`
z2aJ!6o}CZr=cCKDuR04(7YhH_P~sA#?$n{xU^_wVY%j?j;vbxMR;lz@6m5tBhowll
z%WjYy49oB##x&-|d4bKXjg$bpH|9+HHhaC7mU(M;fX#fnObILx)BEe5-p)t2SAyLT
zC9k*Jra%QAmLN6ZM-9MohfHpR%>#+S@Q#gU;IM}AvC*j?_oA2wQ{&pV5ghMJ&c|Z(
zp*c^#1j~Wc@+nQ)n&kFoyG^B+g_Ev4IPQ!d%7Og?k_X|i)Ggp}0kLOW=z#4jbK?iA
zb#AUO=UOXjnKWG$tjG4zD!5vZ`s<&jn=aXD4UWeik;Mka$GTDcIaOi{*qj;c%_!m^
zy*JeM8iQzjxVQg3M*9w=J}}}u*qt}eVD^8EzeCz?Aazp~du%u7nX*G{<3@|3$xp%I
z@@YpKI8H$FAbhRYA8Zzg&8T?~%w9G1Avjz)OV69nI@WAie_asl{}%#h!SW!rAiU<s
zmu<0o&Vv28zhv^xj$QF6aVmJI9jvdsj0Yv$Kzg6fIS39n5E~a>5V8;>+(2qC9MA!$
z!=Kkyq4)(PC&9ZP>>d!?MJjg3AK~RY5+Yr;o!a^vY?fiTGK!x512SN7kov$cx54hu
z`JM_ED@^Cv{=7JB$EJPkU~#z`KNSC84yf39YI{A{yfQ`8-CvnpQRICEc)<RjnS2z(
z|7pjkf#U;YFD|@y9Y5Hg=<=_;o5A)-{hWzm?&+&3U^_v29yaXX{<Cr3cH!n6bM_}n
zU_S^{rh(lHk_TbM$%sCSwfjM^x%qNhV6_Q4p_W<`XIgl(27t|s{h9!l2k8UhCwC|8
zn0DS09NvFl`0dnvdIZHi&TTuv`fL{eL=gw+<t%>+)(2wa#Ln2;e^aM1fX#C}kqtH<
zS-;l$4zSxnV)Jetw2-ea-zm+$&tiUM1K2*t#0C^Sspdjpvq0+8qtAfj{RIaHIE=pS
zm)hl(*|@E-pa!gmcP?VA1f+i3uGu@Sc_3}P^eOLlCh6Zs37;c}A$>fbrKsZuAiZ~<
zH-gOsv2o#VT%9Q9!PK?;K+384di-E_!o)#z*gQ_S|Gzq=?VS5a&Rk{ciXAR1AA|E-
zm{T2!y%QE8#%v~TiUWtm-n3cZ{J)TIllkoh3wD^N@qxw9N}dGMAbUV~){#d$nroJV
z&6}X;vMb5@1UQUAav-d?=>ym;3tyt<8<3p;1ju+Ch>Z)Yh;G5^|0Dd(U~^TR=AqaJ
zGAGL7I9LwEw%xpL#|GBa9i<bbETZ<rfz`6+p9Iq&c@S<eLbRcRs*Z#GH$QeZ*iW~a
z+jj2PN;dnN25B>UW<bgXkUkI&ySjgeXb&G)?fY{FcPuhS9UomVO&e?%`|2JHe{k7D
z+EgHOaNz*%_$}Dv_+CKTe;z)laS2kp<-k0!A3*GugeyCCF2A=ke0PRL=)+*JTNW%z
z0@EOQ5Kd#;4R@PSmLZt^+~zDet=PuYm|u9baXTa9G_altA*lU-Rg=j(kNk)9e@-bL
z-nn4f9I$?nxgczQ_z^fRRQRsK-2@T`;rjoOF+~s?7hdb*g<>8|UBicSVDtWDCxXKs
zCJv%A{;UDZf!MPvUsz}?-?8(Lz1jA?uOVYvD}-H8^yF<egWI8Wrx~nwl@6q@lz60S
z`v(=d9Z&C<gUjl<XA8mQ1ITO;KH}Z9^P@^7*gVH49J|6#A;uCw@>%?lcEhgy$te0j
zatt+%;4lWWaS;4Yh_)HBlHD(7g5%?+T`QPImb=k_=o7zR^?FBW*->+D$56|iew|>o
z4jWPDe>Svug2h2*c+cwwhsB$tYG5(V!>`Rlr%$vLiO>i0ZSwlS@*uS!JYnOxostR5
z!2XZE9I{i)Q3cEf$${{B7A3I1dmizi=mW`3yMa2E3=%`fTC+ES(+|4%J0}sac|9*N
z$N!X5?||J3(&G?$Y3CHal{>#22;ITT1*uCGi>RQKnYMx1V6#B#mug%AyZ4J1WW3Mm
z<^%Kl<|}qw+Xk7(`r45MP9Gq(AiQePft}B9-Uj>m<-2veqU4QG>g%fK>%eY#QecZB
z4$}K}*)6a>5E~b6@R7&R2U6QV5i(}?!^jt0PJ_f?So_d7c$mF@#koCo&vWxtp5H9?
zy`2J1Z%)mTD1M0Hht%I7{VG|Id{uaJGT3drTN8H{*xt9;xmN=mE+_3SgK3aj5MC|B
zwA1p$3$VF5kN<Alk}m}IFGvoA6<eo*?aFgzg1ZSM4#J*|ZD4&MHZJ@#R29WMm^x!t
z$h;_zFB>>t!o)%JQ^|a|p3BR7&DZa)*?w&1T#KhSA>+UAjEhj>zs(IYmj=?CAqANW
z-InYN_GjDSds{p<lw0n5tO*W}#>QHBS^=p)UbbqR^#@2EJi1DGN9*0sDDKI+Z3ou(
z;j|Vw96)+t*lOc1jJSiz!Dz|tY*^KV99jxCZ~05iIizFrmxA36GD9um*N*7#A9l=f
z<=np4^%7Wo=8R47@Byir@D0&s4bjL1n|plL9<V<)v9XyyuGzmWXlW+c9nZddfaO8@
zK)CPIiyi$L;b1?m*m`KE^yvZ=dABae9PhOLHz?vDy`NtlCK&&LB9L|tNIf!k%WwzB
z!};B_Q2YW?Gi_xf+%E1N4Q3kRVLPAMEwl`hY6AP|N-rx){M!aR1cwVqugQ8#u(`6V
zJYaLzn;7qS)97Rwo&y=H6y5n7EDur(!d&t4JFC(Uf&K51GIPhH+iGAoNDhQ2i}Zuj
zeTyh$T>(f8hBrvXgYAIvvC)xO=1oEBn6lS`-ModP96U}55`*FPC@#3Uf+-iwXYp!U
zXgKb*FsOr!Vb3u}o&R3K`2p^yHSheudZptb;}(nWpV(QZ!M^j!%0FQ7T$e6**n;f)
z=P$macqXK*xZD%5Q(Jg1*uNk-5EgR34Gwd$cu2bfBnHDTZ{Hyp|8laJ?M9IP!w+_X
z-SPJ1I~4msax&3V;c;+G{JUAcrjw;d*>cO%Yxjcvu=(jlxI9RGO*ACWgV>2tap3ZH
zU$8DXjcnmh+_kvoou#g(J(wT4^cvV+kXjJl`1$mXIUbL}a;4G-c1#FE9n-8cgp3W=
z*PVffF-RW>FAS2w5Ce%L<IFG~aK1wp1F^f348VG2EZkB20+Ks?ung=^5ZilFpV^0e
z<sFRs=Uc8bVg!q4&f1NlC+QPnZNh>h&%ypny88sIw%>4<MTPiw3;qi$!0FApoedm5
zAhSWZ<-PCDb(21W)v}(F*y*benKuQ=f$$s?NMHQ5f-Km*ATbzb)ph~vgYmJ^v%C>)
zIb`_>_iuvZ{jLLM`_Fapbg&;l>hDa=-5K*GVTVNPBFiorNSkTKRdE#mpWh0p+d=A$
z<F&#5Tl#%9I8IA6UhR^RoM_?tKMBk?Jo_GQHb}kiwWyt&kG%uS{cW7H^Wdivuzrvn
z2+NgE0{gK(I|4-<B&QO!0mU64F?8&FW&$|e(Zx-F9s`Hx*~775yOHGs?s0?d1c_yq
zGg}=@Y~RN7c8ewV9wBgk+wt=tik{Yc5pX~81Z@MeIm;paU;WDHU5(SWS=`P#01j^n
zgL81RLFUUma@<w*Iv;G_Y^`UzG+#7<-35{ZVFT8QVD~qkJpracVlX^q^K}d{m>i7u
z{Gx{uZXh`!o(8ZxpG7Xl7()~C5rw-kcGY#`!1FJ5h#%#%jF(vm7T^8?b^QBXFrxpf
z$9WO#{xw}~V6~sFgzjADdfBq@1{2tiCtZ(%<w0%$;rUP1x2a}1gX1w`*DTXhZG~WW
zf#g6~BO8*}1Q;RV3KE0idCE>GYGLBY^xHratYI{>NE)nn1H)r*`HHM3imeFjc959O
zritbuj~RFVdM<8x_XDK=7cc4z_6tZJgg5sh#`MleLi(wz7D3vrhVG|!O1<7-88IKS
zW<t;v(N+TK{pNeyT<D@W*u20Uo@P(oA?wFLav<EC&H$G4{Epd=W806tZU^au;mh6^
z!2X2sVe}zh25`D~ZW#m)dzd(gR(^sQ|9`{yW_#)ZnVl~)ckkFAnE^ISWq%deO(3-(
zELW5TkJE*_6u|5`%0}Sy{;Jt?+xqh_wm&w5)R|RJW8q;7(%bcU?#^3Q8DR4`9xLyf
z+4&YFzUIhp1-s8IS{y|jq_;o*0!G|{#F6pkyLZ9vK^6nCjsBK^^{SW51m|gx7z}gI
zSps%Dh|lqL&sGb~4vS?@vn=upA!AWz+s>f)VY8<n+-;I|kouOr^fFjYpIz_HpTR7a
zk<j*!9)~$N3_)gt@Nbc$+jjVG0sGOvGkM$13I1SxAUP18UUnDEKdgh8lLCo@@V8%W
zC}x4g(6L?>Vk{q7PWo0ZSg%KLKiGU^IVas7xc{qL3U}_dox0QZGJ{2KfF0PMyGv)F
z=y{=vXtO?;23e<ay1WsrcKN}L+u1^9Zp#yZj1gTI+yu54W?xp6`3~N%c3|_eyJqfu
zAeM^~U%Q_|#v)B$PeTz0>5tk9=~IB%xUkeo>|u2HJY=1M`;xC<yOH%bM!$l)QE_$E
zHk;bU?Uz1wSW4#H1BZ+9qfOxWg6TOM_#Z3|;@f5}0K0R3Rts3oYL^{5`(0QpXNq!x
z`Iqip0n3Bbg0S;{=N&c8kTtk3pG@D8@)uHmg5*HBG)D=nFYIO`iawB>I5TQIfW*-8
z&usy4zaYygX)A)w(>%5koDY!YE|@Wc?F5OL-q~a#!4SXm-|2Z4-*&fy!>Z5T51d~?
z@*uq8hzPhW0kJPmfs9k1gRZ;0_P}LF*VRIcnX6BM&GaqnKyi<1f2;Z1*N|~sUN4m$
zC)ZV>$bVZ6S^s3537LBZ*#X1dw-1BU7>tjNUhv@zig_?KzO0aWwvS9Z!G3~?gJ{`3
zd*Nlm+$ysjOL|i+LKYs{@#6S5uzjXhsAI+@?zM0`++uUU=FO^n2o6{8i%ZReuhi~v
zF1!WS`;+k>m<HJc!n^nD?tH!%QpRQE-`}ZlcPczBf#mO9gN)B^`ep;S10)W@VGl#W
zc7fQqaAT!5IG)kv|K+U(i?jaVLy1d}x({C=Yn4H4>A5XCnX>|RS~+Q1EYxHL>p2^Y
zx@OW{4YB^iJM<QKzD+6ZDcHR$zI?a1wV`Kc<553w{BJ&ux@JS;slu)rCqu9s4hZkx
z6>=GMzH-m4Sg^h%<<(%jL2iQK_@7>2IT#-seaaGbJPD>Ii*py)eaak&v2K{$jtInD
zlclVTMW_AtofCdtu{<LIDfe5<EWqxCso%RFGUf;3-(S8H>`s4qHgMdOKQ-Q|zcbD9
znipj5R7v+TSRSMngu9uAcXFLO0QSedCDJ<-|3#z7-+p%(Y>tP)W3YQcdSLkDjQ#L*
z3KK)8V@x6CFuM4x#gKaH(md4l7$9|rZC%0c0kJz|7H&C}GtWX_b(dLjh85Ti#>SAj
ze2_c{8(drmce7?|0oc7JJh#F0!;pD9s`q*C^m+zq52YQ6MoHhFPcm2(H)Mj<eve<U
zBeV1&iu~?CNSm`qA_gVjfb138b_txvKx|w%>@cJ~f-djEG2IFz#!-x@>p*-EUK$H&
zbAs3(Htse*u0Lg`b4<;4S1V3%+SV7>LYe=M7D9}F>bv-Z&0Y0f4D1i1OP9_5>M2@T
z8%zSHnLQKB;b8?b-_L#0&d}ov!E!B!N_JU(>__qEr7iJbIp>2HFydO%kDs9bdHo>s
zks!N}ab;f{*q)OHT^Rji`PN5pyV4u^wtmSn+<uGUvw5k3CB#p&UxDKbW?nZBWQ-ET
z4>o|TQ<~D82X>plijzC)RG(T@XMF_g$#-uBry-D95Z>{TZR_iekao4LK<k$7$GgG$
zKyn~_&H*wWuzFJ~*u5Yz7`Au-nQMgcvC)jRh_W16US!Wqa6E7^$b$WZELRC#a|sfA
zsD9D(VO;ADF;z1Q-z`tU;ZoBg296VuJP5B+&;XkSVm~Nd1ZF#&(g6D--qhbhv3aRw
zy!Zleo|3bU2g`%hg7EZ|knN$;ieNd<%o95n?tO~lPxA&y-}+RlGD^6C^nM6S1=|5)
z<H9B1AZxbJ<<ELU*2TWP{0nS1viw?w3UFM3#H5*j?l`yb!j7`tt~<K<9KmjwyekaF
z|1vC)wiHPHg~O0?zcT_d?$EG)s>SCi*UX)-LFTHzu0)-0Z#^lt>)+o>uz3qUOx>kE
zrw}E4QbezV<45cAR<PY5J79PX*9N#dVPfcX-)bIkenc0KFouj%KAQ<iugLO>0+9VU
zATjT!$Ib5F3$-wIu(8NUo&k2t;{pk=dqMIb{Pe0P+|6-)&~Uxt1P)hM#|;+g5qXxD
zL2JSMhPr&XT9Dqf^A0<L3ktw-zD%EIr@o6Sio9I%Yp}i~yA%|0kY4c(*9iK*c|Hq9
zxPkQM*oc6|wREO~%?F9WaQL2Tus=b3Eqx)21euJTX_Ho4Jha;fPXCv3AoUeU9)w?S
zm4d6^lc5hbch;x>U~@xEq;^FFnwxK&*9_(tK05}M2k8UhXg9Z=?~g$8Ms=~ouDHw6
zDDlOw>JIi}e(fg={|n?S2b&2p2N(XNU5ud*r1sBiNZ<L7HlnQq5(nYL2?}sG+KV6E
zF)w8I_CL4TEOz;sfW^CO_M+I^>;_p&2hw}le-c=Ykv3%i*PJVY7U`#ScFs(PtkpXr
zybr}aKSgVIE?oo}|G0eP(ayd!MBf`^-UBwsSj;sE$ap<S42Dyld;^CKjE{}B$wp0Y
zFg5vMlfiMBI`tPg?ZL!BG-EDgog0Wfy;OEb-ruk0z5Hh_DztULdJe`jqWJ&rja_he
zoH_?t-+F7_9k7}oTg7)Vm^4}x^*Dn0uRd-<vG1Kr&yIqPT;Tj<l*qU9{8Isx_}7(C
z2b;~sz7-|hKyFKVgc?>LF?4)07&6a?E*|3onNP5o1(}yXmOt5!=rb(YV!X36W6Ms7
zx3L!6{5FH#68z=_JiUYT3GTOtyKU!ONWY@L>oHi(R>j60=N|-_wmyWc|6lHl+P*tb
zlDu=m@<4Fd{Css~=ch>2^`A$y=Yqp)OCjpI2#~!iRb;^WKx|ytd?I2j7g>$s9szJV
ztxhV%Fr#P|qK<YvsJ0_WPj07%Hq#D)38G-NLhAZp8fIRyY8%}D0*k!C{ubBH28Y!r
zs|!0!Iqq*aoyP{&!ygz7_76;NO`QEs9w|w%oQdh=og6z++iT3*8Ng;oyDNifm_87_
zDTQ$Px5yrmH$i$}IN~p4Tx?ehVy!<+j_)&KJ@?MrleWJR<KFq<-F3?}e#lzfPVRKD
zdtvGy>_Ute`*jP0({J2*$e4b&;rkuh;d3oN29|;K{4_xAkNk~d+_`%LWQ@OBlYNJb
zpBOmYU}lH3IfMDjj$-a(@H?admV>FqMyKlSLkVw~nm;a(`J|&p>%i`UiG%3lclyEk
z9K_z-`)TJdjh3Ca=RMf2-T<k)7sc9w^D9Umglmcs?J%|X%fa!#$o4+iTp6VaJFTB~
zn(vgj1m;H^MvN7M^j>^=Y3JLBUa*`-S@^EYkMhCc2a*Hf6S<JIamy|PMI0nI;VWcM
zA&89&o2#Jiw}Po#^-~$_PIb}kU>YV4qKg(ifVZpM@|(Bqne1<|$&<@+hvY@DT9(hv
zDB-TflMd&H=RoqpcX@xXpIWXQ+WAlXh}o$FEMR``3CQ|*kUb!L_4~CQ|E(b7f7NZr
zw(aYIw0A&qAiPLc0c`dr)(~)h1&P7%_1xF+c!G(c)2d|>aK9kSP5V~>&QFUxg)z+d
z)1C>o6QthtfYCNDmi!%`=Q3Mtd<QA_N?K6cKRo`3dAR)Vko}*cMt<P1N-z{LXRYeC
zydeUa5B)3{2(}kwHVA*yny}@pi7?o{9(9KuS8P^*^?~F-SY<n8-w12gAB=Py^iCM-
z+Afej7@mFPB8EPY92@IKaCr7wegfMK5`*E{a}n!1I8Vy#>^u5=$JU@J+a30-1DloJ
zzyOX*xcc7jV6#B{pYD)-T4vKB>$Q0=Z#6yoWY6{$v-W}g^)K=YTrEg_;<ev9Hi$k3
zyMgmz>dxLX(O^43av*$*p&cAQBK-m=;RcfH^haGQ2ogibTKSmk#z1QL&qDIQF|P`W
zdq8sH@ABYchY!p*XLM@bnX~JbWyl0ku(-w@)HdVQx%=RKNLf$F_(q~7q<)dlI=9nu
zL88S!F~~U1F+J4r-MeQ*cV1v*2b+6vhPcJYCwbto0J#H%#d<lw{$`0mtrtOZ8#OJ#
zVFP00!hO|{b?E5w2Ulx??ewTzieeu~9mj`#a52_PD|bkWI-2jCFxlcO7o^_%{6GgK
zPaPJB1G^EVx3Ya2*jz1c9&j1Cm*>dNrbEYfe4hv@|E3kxpv3Rjm2Nv8bPIvip1=5b
zr+@q@uzrwPAk3WwS^vU#<0XnXNKWuMVx9uT$A)W9W6m3c)crdh0d~h{nF5SF<nD)<
z6IvU$%Y1U|eao2VVmmi{g!I$X_o0@%jx!Kr^UVh!{gg@iv%!A4Y0bB5yS<{7s?IrZ
zp1HRWakc`;9fvGxwh8>;1COVKol@WVY}EyLSc2r=g+S(p_FEwG7)Tt1Pb?KiQ4117
z$Nf$HC~*aoS2)`Q4$ljVEx`E_CJv%aUP1PTg4oh7g=Tm5p4;KwbjEV2I4d|^Omch-
z_6tZJgv;h4*8i^(tpvMi(sRhX!r{pex2s&6y`xkbGFCt7BkKJB{rQqRn4=->s&5}>
zY;U%>g5sXyFi8L8`~gTl0NDq_udSbB^&3ncM87VFjKhG~$XN3h<U9`j$}q5>kmU*;
zc7wwPBo@{tyHjkF-Hyqgvn&n1a)Hy#xlT(I|2uae^2_Dpknw-x9xia4zF8!-TkoE>
z<*iMSc81@yKDgN+^KZY}Xu*>u16J#Kf8Ne{QK<F*(SFDr|IBcEjQam<17tlW$Q)dF
z%@#JWo6+SB-$T{~9^K;!PN&H7b?=PeVZy)ZspbDbm7N~@3@uBl=7Y!op1eJSqG$EO
zNVpv$^Dl$__u-8Xn4W(ucPD53MT@0xA?=uX<v!r_0kd!W#*$s8Htt~a!t`(Kn(c~O
zKC+xT3pTr+>nNCp=>ySI0*X-L0VIZwb$Z+|!VRRR%M&u6+LAR1>?e>I3^z4@fv3R;
zw=ZT{zpw0gvi6_(+il_Ca7nrJ0>xhQ2UFpG+Gh<Z3zpkJ&Zd|br?u01&1%b-WQZQ7
zc1XPgvrB01T$8s4A@d(4XJ>D#U+#_K&%9p9ypdTE<~Ys9#?#<1hM9$pKC@vrig_?K
z;-6H(;yRUi7<EX-c1f^5L28+vMVZf;^KVDivhUlL?pFr;`)oVv`qy9m+u;7+!3x<6
zcWpLgzk{`_*3M_Fo|fCiAY&#E`%%|y^QskXuel8wgA1CWvcn|+)x8%~-+}Fn$k>A6
z&JUb_2>O3>C8942@(&D4T^0nV3;oy_xL;v%c^;6lD-eIp%o#gW)R*jty)A9AN5&Uy
zpTQDHUIxj7aMCZtnr3Io)!?wGiG}QmetK%*&i8*mTL|uk?D5d36aveG^nvjApbb0g
zdLjF*PJJ-iF|QjkP6(0%VQy(hu=|#GqPB}bavB9yU^x&Q7tWoIef@XT3IEL?J<&V6
zFzUY>0aozzA5?E{?w9w_Vh!7j9p*>f!R77OgJ;2dLHa?MEBzl>9K;U0_6D4v6k{QK
zXqyX;?$Dj6Y-w8v*~hztdk%`5e;d5rbSMR~pUOWjW&1+0qj0l8=FNM-503v7ML}@b
zgT!Dsybv<R3gcs=-TzCXm<LnCT?bkBQ&)Tt#XTT-?Zhy+|CjL?@0=r7z4K=LAu|y#
z$ok(0TT#!{nY0?AzL-k~?9Q|&tHEZhoe;ls#)muGiozg!1`DqmgXKZ?fUs8DWDEcM
zYT&#$X|=`92s6Z3DoFlH!e_8KQ}|KG%|LSgijeUb5E~c1ev=o)KA5`9^=x2qzXUIg
zeARUcan{tPnOC+kZr`}0B5j6cNZB5+TcUMP&nSBH9<r7KWcJHE$Xvj4{_o&;o46o&
zmwtYr<!i;8;Be*=MT`}J)carI*uK^PvNwdiZQ1r0wg<uP0?C1Jco5|L6Shdyabb|$
z_7F{QxPjQX@Xjy%Sk=7Bhs@<Ye&+>myTi<QKTQDab`YPRrNON6blZ-?q-7RIHbc_1
zkAecYz5>aEFvkW+T?%5)KL^>T{ib+6*bIw1+B;XQaI$Eh0jb9&m?8V_L25y`=U}<{
zGyfT2wYtw9?YO`GA&UP6Tp;uR=ZcPi!ycpuhHF@>u=))q528PQ(nBe4V1VZyq|bVr
z<0OU|J4;T%?PAXf+E)KD&|>ngUi17ZmEgRz_?R!)y)d&9S0U=ADUy&qGTF`7!SVfe
z+tlq*MY=oK<JW-s2lP<S@nOE#wZn(!Em-bh-Lf5<Tp{g!nAs&8kAl<Zk6zSu!yx(Z
zU91>x2Z<x&(r83lMi!Sj2$`o|sR!A6jV!lAr3&u9L&r|;oV-h5XLo1hHq{-FG3*x+
z0x0n>unv)?cW;KQ$x4i@0GnYtcjAu8vYb1vErRU-T6p~sN_=p~NbmZy9y0!!<si6g
z{Q+hadEUQ}F_+($F~|RJZdnPo1KI89>=^$$DCWWBC+fw3<GAwPUW_m-p8?s!0aDw+
z>uM2w=*iA|aZfCBHKV}xJr}h`@&Ch6#F&PwFAKO{ZCM9dn`^GMaHp^C1WSWbPOx69
zr<YLNQ|%;SUVRpF2AQOu-gd*JFDU8Xs}!>5?w}&-xj7)W9ntLq`vJtpg-`xe!0;bP
z?YTI}{-4!1E5K<QBnHE}Ga=`Zf%r4{q|BAB)LAT;y}_c-o*8VP1K%7JH_kLcoas@0
zq#x|Rl?NbWT4F2Ob{4%Z-?4X|6gZ7fOX~pBAhSW(Xa4>jvnnBD!wVPj@4PAxsbfHL
zAbf@QH(0%)IOaSS)8QJ7vJ0dK8Sg|}Zx54;d1M8SryfSgJP=GAM6dQotO-9fug^?B
zlXa)`g{B?FXCQg-i2Nm#_;>hr3+!i*-g^F6u(`blSi$jK5U#z;GWCV!zhuZB|J`#T
z^P?cOAiU3d?T&D6$bJgV&A+xEUWPgzq1?g^wyVyh1H-Qs>5w%nAaiix^3yqRw;`*U
zU{wJwS7f#NG0c@&xe6=?QvZ=@ubJsNr%fxK1#S~zO9RJ$xh-P82c+g{-~o7gc*(a3
z%$|M;(uQW`s@ri-s@-CH+G%h+GHga&$Nbwx)NJ~5$o{EUr#Vd5|D1{99?u}i`ls-7
zM=;WH-JS%nT_88$!uO}%1GmS}<)0je?El%kvTq~09LJIMV1I(d=jOE-Pd>`9vzjA&
zyUzBxU~wO=2jI8`$%Ak~h8;YNKeM=j*%?BRb9<T#`*(a=Ji*fH_7$)mk<*ZKmq7YJ
zIC1Mkb5<ir8fTbuXX}e~ko`j-IS^LA4{0}FV1>*PgT!EX*@jOTVFHtb(FYRpv8p-P
z{1xoZgB9Ljcfr(y=nD^3!2SfWog*A}ZvTCCheW50#s4Xg^TsB1pFz>1aR8BLrPe~`
zccr-@>wi^q#deCUWwxBRf*Y*o?L^f5tyMqScP+8)2g{k<SKKK)X(J?^fGlTVV3=X+
z0Cwj$JqQgI22&mvBEkB=92`W9O*>dWy7cj_kh~nmidjBhvw`GYke>gMHD-^5W>`cT
zA2EOOX(~7^30gt=av*sS=6Zven`tbb54MkM=@GEomZ#3%X>f6_W$hQpe&{?#)bm)x
zO(S+(P<aRTbMCg6JASFnKuOb9%NW7=KOO0e36Py2ymtFJcszml*zijM?D5Z`57~!f
zz6NzIE69wDbDD5-=kiRn2rlf{X~80AX_He3?khb|V*tAeq!xr*rbG6&g4h#YLfZds
zADF>v*p)(dnOk?6Uz!eCQ#v~awLOw5W4rT2xdS+?RQK!e@|XAy4nL4tAY3I?3=SLN
zq6aAAAh{FUAm>Jc*tqcD1vMD@Kx&giA^uN%aslpFkT?kIu^^p6_<!PdnJ$T)jpnVE
zuhv1%(s<bkiA#`L5T0`zvHox36v%w#mrRJCgmO7{JnM4Uc_H%w*dJ|E-k`Y0WJ2$D
zq33#Fxz@MBJ6j?UXHS64F8FQ(mUGUjLeU43<BW2}nzuo6F#J&VAXYW=Jp#eyY+jEG
zysU@m$(R%bPlHlT96O=}inq_=4!1mLJ`o&NJ?l~D%qz|!&QP134mqcV`BDHl{-4EF
z?Q*}yWGO6m4XoE(9kQPvW|!WDj_s@wkh#+57dGsWnrDUL&&H)z;Iy&$oiB<wNdIop
zOJFlWY+TqlYC1;y2Bh}boKA4K7#4AW%?F9W@Y36}z;Yn|!j6QUr=lk8{FX4?Vvg7z
za9nJB0y(b*BoD&7s-ocG)nxJn%<j>FtZBc0f98&f)kzjc(uQDr`dGbD{1KRFyJPlM
z$XIc=?)IHuBM@Z?$m|WJCgAvSv3ic84<r}E3h6h1*tqcAe#n|-ba~$6kTYEG7)%1Y
z3t3*obq+jSt9m+jT0c+P+4*#z`Cf)AVE<c4L+U}89@}f?aP>kHmxKMk;|yfnLi^X3
z9cRLVw;NA|oMSW9b0ye6Fulrb>vs0>LfSkHOS*P;d_i3wu|cF9Y^RkzWWFAz4@5`Z
zL>-3)iJ{}Yt4uJ$4WwpTFk~)k|B99HxCV)X@P0o=updBdg=*GqpR(rfkehkJQj28`
z*zMifsCm?(BoXfa1OdqSe$t2UV6hvuCwKm5ZL&x?*aQyen;TL4zsGFEcPi*X_8_0I
z4Kn|D6EfZpvKxf=UxDN`#{|sv{}VpAU`^W~eK4%G*ct2&7#~K<v_Q^<yZs&W{Fl8C
zAZIRu)b3U9-x*T1VyDHA>6ZT=#Dl|SRs;(;Z-eAPn8T1497iBF?}i6pHP7}!&c7A7
z+`CJ2?gxwDF36dJ#x9V3cObPO%-_9vXJi^=4C=eb-JMErf1$*e-Fe9Vhx?f|;C3}g
z4-8k|{Rj?I7#|y5yvh{CJeV3QP6_ZBp4f^N7-9IN9C1cT=AM3&#_d)Xk}jXko%eqO
z$A6I%;(T0?d4K94XB&aolBKQSHbTLsYOvdmX*TS9K3l@VViu&0dn8l@9#$are0){g
zx4z5;hv%aL{p~!@AA!RUBnQH)CPLQuSFc4K7XrzZcFzUZ#~?N?yuElD*gfd-azD+$
z_GEeAz%YZ$M*{A~h?xsb9xUqIF(+t&8K1&$a9Caa23gApGjnAVWXuc1|Mnl!-s*lZ
z37qcOivI6VskE@v6o>2?7J76NY%fSH2<zKTH@{FX2)6Iv3^kMY?WpUuI1M3dL)wnn
zW8_m|amXAl$Q)dFpWSM3oT1DAyARoiGH<Cg*luKbi>runmH#j7cdR<_W=Ht#2^Jh5
z<H2rt--){KJjWl=A9mG$0^U!VI2$rv)l~6pyWfwA+qZT@_8A;=;X+A|S|u~K=XF5V
z%;^2zzGHh5V(kUUO#<(qf!*(M1k$eniNUaxGbByH_}FOQtBACWEdOabWIf@A`H*#*
z$Z}EF5oc!0E#ulbF<5G+baLMgS5?Rwl(Uk8;PeX9W4IBr=L^I?9KsBaU+2}A!1Rx-
z5Q{Hnyyh#u{R4+H_Xfy$CLpySd?w|{&X!Y<@g#OLhh1|NcA$ih@#3Fg|K~L!<|IM-
zK-lF!<g8&38yCLv1u`CqF28vLq#Q0?2H9VREbsksBG^ulSTvKBDgRIZ9Zz<MTFwzm
z1^c0YIqDcjry1ldO_2KR9n-;Tp00wd&3n!-x0CU}*ByHdAZ^tdst>^V0;Cp%6}Fw+
zQE?qo|B0{m-`V4r08V2dIS|fs*bP>HL=kdc8AuF<A3r0!{^Q42$U0}39vCg41DU7$
z_%j|{PQk=MbVVBCTq&^xlkMyU?mHz+WUQV}HwUY|SK){f|7%qcZSz{A$>4IDxikW-
z=HV5iT`$9rS~g@u`U8OyrC@oGJs>Q)oNHGnALI<iU&X>Zn{*-h5+n!0##xZ{i9Eij
z^QIuVS!W2(|5}?&0GC-H^~hNI4dgtHMs~>DBeGoG6ht4i<WS;{Ws}zKNG-Xzqr1WZ
zoGvW)L)J3E^b|EC#u!T<LiVq|QMLicqqlFXMTOX<9bsvZwFA%JqxSEP>^`vLgbd{T
z{QH0R>@=T(=u3dicen=`i#quS;$Dy#3{Sqh1DrNsd~CGab?oaOU+xV8>s|gIvZe)D
zugez1p17x9jLi3j9^URWck1>n!H~I0ZcEfNYFvXhgVP1bj8-Mc`R}En^T6@HMB(2~
zL7|lviY|I!{_At7{rDw^S$1Xg{RQV4kMO5Eu8LTqqz|9fkaN2H=g6aogY5d<0$Ec6
zV&lSl-$BNf(dC^rlECH6;zxF1cOlE$JVl&QcX;_K%ZUPCEvh@0TJAmzIdAID`r{~i
zWV;aU;m<Xb!R9`xg`9V=_SL~%oAk0Q@1{cbI38RCX+y*8yMJ}h&So#j{vW@#6T6B`
zQO|$&?S|Y3F}vh0xIBgF1JPVl(=fz9;>h@x0~1ELf#kcK)`9h&_j3S;JxC0OpK3tP
zss!;(qwblqUJBjmbF0UaW9mb&TKj`&``;%gg8d9qe_{fpUFmiK(*8f~(y+7Y`76s^
zJ0Nq$Ki)&;9YAV9c(tL?j_w^I;PAY6*nj8FLdgCBkQ@k0HI;zV=}+0SVE2N=VA#VL
zwXFaXN2aHo;l$7fl0V-l2Nvf^x&t;JBnHEgW{C4&TzA%*b2e|@*)J($wPWcVa5`W9
zC;{wVnEDtS#GbB6cOYZIBH?XdwMHyzyXw3DTIN_l&Oczx;zfy%4SZ*IOp}G~!ITu(
znZMBv>@JwuxrZQg@c}n6_i;`wgsjJcsl`SIbhE?b23dZqB;+2JzlZ8k`~p%l|25>E
z0}y-aRK*=KtX(@_&0*T!dk1p=N!M2;6g|&d5o?p&-a^`Gt77(S0_nH#7TWr$RA?t7
z%LH(EIJB38%SDhJ2)D%_-+BJU6tH=HbJ=$%Pv4B<|8;Mlfb~g=w4mq%={4)_Mo|kA
zL&vw2^ijeaCa>Mg3pQ`B8Vfj0!^A<f?=#5RMIbgSFPnJ+Q|xw0)k&5NZy@LYd^3jh
z*Fo|i{7QE#*enoRMg`K}%vXh+k*U3|Yv**u0!#ZVkT7LE2WcOJ)Piu%SH7LGA<w{R
zaJ8T7)-|h?;bwv4Bl#fv^d5ScV)(V$68qV1Abl{b^a9ehhVfzaI>QLCog&Pr>kUEj
zXMGTT6w`%!EcUvK?C`tKVbRR|5?udTc|q1Pfz*QVo@0l>W`Wq;J-*=j_JJ8>Ty^sL
zDht8;Oj~bpL(bIMFRz2r9}!*KzkTKg$Qe%>>&131sYM!lV_;xl-3u9uP5zBKZVa;P
z>1oKA0*H+Z%g8|XEThY3e1XjI1*p#jr!Qpr4B3-#|J-H_-flS6c8BD}-0dmXet`WS
zE)6NmVR{}$BhEkNoC#Uq62b+^w+|YFcg%W!%VOnM$eltf15w96c^QxH$ou{uoCa^l
z{ok>`262`;$o$_nufc8+nTEOklXpF2P7S0E7fzWRh>>nUYI{N;`+r;CLe^=6#9&zD
ziWS&BApW{pCFWD#J=po}*-i7A*^o6|)A%8GJAvdu__?wKJdd^NLDoWwKez`@2aWrc
zw?F@OYKO!N$o{Y6Hj*gm`+<emj^Gf;*-&baZ|#ixkC;mbnVs?ia&E*4Q_OW7a#e)a
zzv{l}z*_#T(}A>$KYjj+5&tFYg5hF8=OuPHJ8#&|^<|IQhUc>2_)q^*0+$E5amm&F
z@Gwrg3mH2}Sa8M=B==3vZ|g17c{{EvLi)p#Z^VPm0Lg*yt>vjZWU3+NO&0dsH8;i;
z9N!>05SC4X)FW$qN>RcMBxhg**&797<HE;oqMj!NQ>X0L3l5j2I@GhQLGmHqH{kyB
zW9P8i=P7J4L3;A`rIXdb{>&&o4VMS07ifc=c>`ikSq0fUbKN-|>`$%?<L%QPtl8O}
zw*s7|nra$Q{P*&I*3N?>kolxs7lz%TPa*jPWEKc#_11#xhSk3?&v99C1+q5)qz)H8
z7*-8Wo5*VXk3#yapEg3y-b0p~Ze<1cp9$y19n-_Acm8a<z3mOd7jPUjbwbYTh3R?N
zhq%)t>cJJT8rL0=^ELjhUc8<0F2BX%MUZ{Wsm`eDTasrU+EJtpspFphFx-{eyaF6A
zFtcSZL(cu?I&>Wz_Aqe}9oT|;mO4lb9ou9Ifb$``xS`f&usvDZe8KU8ET6y!Idcmn
zwrswZMT#rO4i_su^Oe6K<Dv_L5%M54Z=h$MfY_00SHb>Q%2@)Af6FJ$W_LoncCM7<
z1KSfZ6LOvvNG%ALADF!3dJyDp6oc<ec0SX%1=a_W1K}I%Am^yKMRtJA1&P7%H@RsT
zVlX)vZ9N}z+!Q1iG;tR=U1WW42d6KP7z{5qMU3wo7O3ys^;Too{otb(CgqSeBiA}f
z@OovKdU0OJoF9lk_s%PDxIVuJId_3^3B!(uQmZU0<o|%v)chxqbC*DBL0Id-p`8vb
zLJ;#RJ$JXvf}G0%k^|v#8%P~zwh~gVfW%;Uav=7+4U-4ayNn_IF%TOW^EvDThjX<(
z=J?PqPsF_%|B4T6eY|G!j)^BFEz57ofa6dq8Pb-6nOA024o{ovxsdj&v0DjPZO6CU
zJJgQITUz-;=1xR@pr-%UNBrA=GKzx3=8XsE4jI|?DD8uj`V3%qddkFN#5MQkHmqqI
z<}MKZ(G>NrB9IseNBn@C+4VIUan}_{9EA5=fZWvwVtaQ_*>RsCY3F>sW(!3Z$oZc?
zq#<`QfaF1V<$J_Fgd8DA`|qRhORzs`TNj$=SDRa&iiYenJgSR&{`0bG#%-CWB*AHL
zdFhRviT#Y=^aC;%gjr`p=5M;6Tf)N_Bo4xtqbe}OK;p=_k`r~kB}|T2lmndp%M%uW
z?S_eiXgxW|Tm^_dQ#{ggr|p6r1>f_{^{(51{p9uovR@k{55n9(A?;!i`(ZDnzxA;4
zt`UfzCiih$=#rWpJ(?H6>4E1v>U!kK|2FOPbB3HJRNtqvn^6gJZxl!`2#3Ce+>J85
zem;g@PuLOO{~vw82kcLfS;$z|5i%~wG6QnI0J2=45Twlk64Uu5Z8_uRs~t<<Y_j+p
z0y!(C+6%S)BYR;DJkB1qSA*F(FCb%6X-*F2Mps(Rm7F1E%8&nuvqC`T@93JnV^=F=
z{qyC6Dm%k}rlX|uMGqkRhGras+}8)v1H*@#&w}Fv#>YlSYX8OvH;_7q_mI0>f4g9&
zp*yD$eTF^jc&%c-Wm*Jnm}&9OX)ZXw1<!}}S3w3bFfdeUBF=PMTLn1_B-n`&oUZOZ
z<lQyt&^(J9?3v&^K2MnwoS&gu8H`vT?ofKP3M@C>igV}5ZIJb2F!}wSkTrKE&6w+d
zLj@rBCc)HVqnBnv#(B`?rhe`Rhs(8>khAxY<u_)<fZYQU+wQm4Tx<F49lDCuJKWvo
zf!i)y>QL8zaPIpEj&qRuGD*n(1+I0Fa|bQD>b6HM6WDpx64H<Bn<t3k|A(40b~K-X
ztPR{gUuu{7OHH_0AoDIgfvlT!WtswZFGviA-Gm@_qrmvs=(FwTP|Sm=d9>CY>=&J5
zkh{lV;vo95737>F5Iasj+LGhqe)F^U`7M5jS%TB1=z~?@ybO{DVNFRy{IB^5x!;22
z(-LqPDhO8X402+$2>-4Q=1bW^)?I=0f$*ESUpp4gfZTD%?k>6GhdkuIPmml4%N~H7
zHClQXbDiFUEp9080I5gE*ETqT-Ha|C^5HEw@63qD979NXh&WftWih*XrRts?JpSLe
zJQRi8;c%G+walndgPbJ{wxjL<IDQNDAoC!Gm7W%xTJyF?ibR9ms{U#ZihDHY=k2)m
zdp=lCH2=JvsZof#WI^U>W<$=Am*Io7*Fj=1Ja?53I80%DY&3rx;w(F4`N$uTb!f-<
zAp7)@<raFRz}@I|V3Wl$|D8LYSWGo9oCulQC^?5Z|MC6<q+A4<;d1#SI9)wegxmw;
z@3?AvPPoVRIz~OPTdSi`<BTJgb7x}{<c!1Vu6uWiOhH|*@^;mBaJbF-nu?K*XFIP2
z%Yp30g+EoU$Iu5-8*qdj?4HBYnEijJ#fQLt0I6eq9K3Tq|Dqi}K1<B^6}E!Qzby|T
z?RAhm2uE!^4_D899kNb;Wu_!JzZTXfSX{Ogw^&pI=~H}obqpT1Aia{Y%XV%Pg|tEB
zH8Xdf5J0`JZmEne*zC)UnEM}JdPB|z1DS&hFWimVo`9(fS_0W~d&CEG3_a~4WIaAe
zEss%?)vKoK+xaf0Z@>NGF4*ng7QP3E9Y`L8n~x*zY~)=R1Xkm^4Km*)`r*%x4W1`=
zBz=Rl@#iV528Rns9|#B7ckkRIZ45Tgi{Eb-??lM{agZDc$Js#k=VkG!VASLMMv%G@
zqz)IhNx{4~7^IfJ7;-Mc^_g$Mc@HE8!*5TS!_$?p-P9f1pPV<XQDND>$7Vh_E({+b
z&PNC7*|HC^<{!jfD*OTLKgkNnz75s8zB`*k=30CcfV5RiC*A;u4@fNtugf*w$>+HV
zoUU%3Tf6n)!|&jASs*zOUds)cN9}Zhw5vg4Fq}07GNu6IW228pLeADlmvcD)*|T>|
z6|)YBR%!$L6Qrm1z?bdI*ZkSRdUA$ENvR*$Z~f9%;JO_o55iyK5qC^?x<bZ(eu{~J
z{dv!&W_!O)xaIFVkabo)+K9aZAieYK9n3HPd<KsHsRhwH7sR5j50LY+0-IyL!T@YH
zNDmBeOhnw%0uw{0!}=lPyy)U$uOR0iDYIhs|9fsC?v+nsX|+gRthB@Bs`PdR?H+JE
z)z*K9rw@?19;+_G-6sC*4!B;i)QSVEy-<H(tApsK?fWdFz+qa`W`L63ezxmutJQ;y
z4J)0Ux%23sC18CZb3r&nL<k%<3Wlip1|+xrFKRgq5<|y}>kxMnAj_%G4hENtX;(1U
z{~8}f^tFuhn0Nl3{d&iw5JU6JD<NYWp`Rgpdtm0-O|Jm^4aCn}1BvJ05Xkxm9W%b2
z1)i>!Th~MOIGj<0qz{l<5bl}!c<1&Qh<RBb8F!jp>P89MC>3ikU;K&%*u5Y<F#O}W
zEm#i5$42X|n1!Jaq)ti@(w<`f1lgAj5`*DmYY}zgwtvk#K7M?%ZF=oG3)AnWVEZf{
zokod6Epx;g)5(2H!C}Fu51AwOd~LCFuT<BLd|SwV(6z;o^>Hw}0@I)FI41$w|4`)9
zzH^lm>i%fs?WW-PPu++ZV}R*xn*}+K493Ss{}RJI-vOk~up4qNM9UdS`wAom!*P!x
zbFLu%j#ru%<?W6;v)+1GRit!*-I9?3Ig=1155hVd_~2#p*3HYn>={c;z-2_I?%G`p
z`4hLVV~5OtiN&MN?^ec@@6`Cl1s?zC<B;CD^uvCX_=<9P1GaOT(0UYckiCj8QP+}!
z#L%(pS;#r(=;GJ;AmeFY&O`2AK$brz_YIz(UzADie0@Z4XVYZ<oo7Bm?mu(b3EA5K
z(_^v^F~*!a2{Nys=m%LtVRn1<4n<*!9X^_n_0L?PsO`8<RjYPhxdS=>FL<Zd&I6WH
zz-bI-_Oylb!Tj5bh&u*gawZLg&;L5sj@X+7(+8q2)j;-{iX=eB^g&`U?6nJWcQJ^6
z{Pt^$+P?D^x`)^;W3NEQzqWFCfa3%t55nvvkh6_I?00J*@jE4FEja#V4_w~0q`cR{
z<E$o_pHv7r9~q<;girrovGe=yV_-R>d6GMAuNHvQ4@eG#C%lF9e?DaY$B2LbGRWC@
zAa%HK0pAXc`#nHv7Yd#RoA)cx0c<x&42Bmqy$9DhAb#}Mo0i<aw_CV<Y_$yZ@dlU8
z%Ix1!^yo<<&P&^33h|%#H^_eX3w@EhE`69|vEV`;ILzGd-UG)c$ZQb)WBGcg2s31@
zK<$J3yC%yb_L6|)g|i^*1&TcF;b{^i4#KlU5&LdHd~CSnry)vs!qjYNxDWOV$46~&
zc?A;((UXoq>KqU|b5@+`-yDyfY8w=+wq_@T{UCl5lD9$fApEz@2^^Ln_U~hmv5;IX
z$h_VD2VZyY`1Zx}+H*)dySDxWTrEhw-FeF$(@P*@qsx`tc9z`tM)9ZJM95g=%OjZM
z7>lNF0oS!4b8z9~@!BZn!PJ$gK-T}Rzc3FT*D$%*zmT;HApV~(3TA>=qjqMVl(X`a
zg^YWu8=0cS|B8M@oAq`#<oufxT9A9oN@EOmHP$__tUGZP9Cq&w5pyLV^M9Hg+tIWY
za-UL6c-zhuA0cP#gXBOsN)|Hy>mW4&CEP%A-@m(p;}FEgh0Sj<Vdw*?Wo3r!vv|D@
z(oP47!SI2ON_ZM|OY1P7dQ5w#Ozwg0m2vODVYT2X<ZcU?dRb%0Tmp!%C<19CX5~WG
zkn4ngHH%X|v_s|w<oqk|^HN|lKx#pF$%|t<7g<B@cyU~9y=(p?#M&E>{67ZBJbH^5
zVow4{9E7X-7GaHBkQ@weZ<oQE{_SJ5!D%O-b0<dnkNEBk4_8KqwOeoVNSgnOSZFzI
z=2dXG7_Eck1DGAV-@SnQDO3g$7Q37wd#HbDAKq!6KGmXm9ptP+_7#Y{0@8m<;^2<f
zZb(1o$wWr8rQOfqW`X4A&xVY}M7%zM;ny|Or=s)~K<d%)vF15&zaYz<Eh_-~U(8V+
z!`#LnKfwM3sW<XDX7O_M)$Oa;`L<QYGJxGC)`Pn4{%8qgTnwbXz!Y-NnQbfN{AbUQ
zSKDM2-<a`<LH7S;v)h8r0I3CG{yCyM&gDV&mDUL;?{qCj-TxZ82y$OaX9ncFUXUIb
zo;(XOrwrp`qa|iS_9LRpy<Z4PXR98duH^u!)2&j6he>S33X3)_4vU=&b1aUW`3p`f
z-=OD!gVcg>7`rFjPZh0@yW@_iL)I<J^gY}eo#JXK^$5~d=U<C@9`ojw$vZD3Le4SY
z5Zkmp>P;z1e&z9mxKH(OHr!1hJ3+Xh4svc7h>Z)MS3tc_1g0*{2XddF_~s6jwk$~g
z{sKr|1+hysn#|`h`R@!|q-l9S9da+=1yj`fU%uW$+_C$J0dh{6#7xLJ9{>M5+3`JW
z$qw$Nkg{_nFY1|vsxM8q=Wl|n{}SH9x$FNk$XRe8yFu7r7}Dn4W5JK%|G41o;P?Qk
z!-d12V(<T){sLJ~zx8(nxLiiopLKu<p8kWK<9B$k+`i*hq5X~p+aTghel{q2D&rt)
zK0s!at%LMe!fFb^;hMg<Yr9u^_l|8FA!{d(8ltY9Omj@xIr{}<j(_=y{W~|6LfWMu
zvp`tj0Hkk_`$-m_#z5jAe5}+ELkuL2j6Xupc|;Ziu_Y&N2dA^G;nz|80+MrFgjf?i
z-AK=BrqPxi+OBODea6AyIFP!13`NgbM#P+Tb1mfTpjhKK;Bx2Fk+s_oF50wnQtM-I
zS+#J93)l>hJs>Q^Heu)eKajnWH-kHN1)Yt6hdoGsEf?e-v59|C`voAm2cD2J6~xAc
z|1^1GE&pGqK=xt&`~o@C2U$PIdqn*&w|DXmmA4CbehJ-U(fC>l?Dl1RkiHB|Pt9}0
z9J+U8AUIAo$U)ZmPMO2MbMaP1%d<U@I2QAY1)BlWyQM#8XXn&i;PBkG_W#bmji~+O
zOee@V!mk#{dKH*H5N-K=FN#?pF?4)Qt^uo>&y|pKfja~sYx<D&ebGUjtMc~7UlW0!
zQ!G}03bgpT^#j;%$;&pN<hwmKh%x(lM<Dwgnqwemr0nUuxBbGAww>%d7l7R%?uI%Z
zY1_MJN9Q*aaGbk_@$R(9L7mgz^_3GWXTyLvlNRPyFTRQ3GzQ~iqtj>RgY89^%V_lj
zrwemg)UzBx>gMW0*13b&|BrL-IF~K8YgWK*3q4WDcthkgZg77UBoD%e<gSDB5{S*a
zWe=ErYmYEEPS<{K-uds@d&|cUqQGrNDRxM`15yjZ)42tAS$y~emJ`jNv#YTs2gSV{
z>yCrXVenT5hXY6t4EwDq0ow)RW25ULM8W<<m%CuB3eJ!JD^cg$LFztFMC8Gu{q0*L
zezotY463%Y-3mFwU&R#CmjS5-;W7_I+(<8mtW$0jf$Smwu+)0zKP?5ze|?c)_n*;)
z%sYVefpBZJ_x3lLkTD}e^~c+{Z{b3T(<A&x!D;PCH0JpaGxEQI{Qxot7Z%>|0Nl<&
zm$&POoTuS*9<rtnS)S?R4!9fkiZGh~SW~_8;D)yrY}y`RKg?i;tXG5SnNy0`oAY(1
z4LDAI9)jE<_VLBB9da9^EZvf_!0{Nt0NEc5(`&)Ic*mFjko#m_$}8^FNkCoae`r4B
zY~b$``oMV)rVm6fnj(ghRzPCtIN$O)hCYy*^M@hpA6^Kf?&Sl?NneHB)d6DH^z~RW
zJo~eQJ<o54&YBtEI_=VANP86|55iSj6yfghPlBu+UAqW!Ue#9h>86fJ%eI#*K=wHQ
z*uM}QpCEl8yt(H4&QB|}!Rh}DN8_%VZyw<A1IdA~_Y%nZ{gf$?yGKD{Fg&?D4y)f_
z@*sL+u{GB8FTF|(Y>#jCEfn{F^ay4k_CKik*;^`!>h4U<{b-RIb{DMnPTnVQT!Pes
z@U7K|HeDF|Vz9Zo9*}#SxxF%W7BH<c7c7F@nX>f<>iExbF4vs`b0A|yrwZeCG)?^q
z4@;2Qe5T95c76U@f#KIiNl^?jkRD{5z5unnfyupCq6@ZX_J^O~Gz}96(WdSY_kh^j
zzp_~cuUcwRC9=WdBtPVA_vJjO=|7$kQHQ8C-2j`b7VrV=?niElyZ*4OvN*XOlE>p!
zEWq+0vq4yWuj@{84oF!kc=*mvA-_&=_<`g=xc@3-|AVY0W?9FVikPPYsl|pDZ}LMi
z52nVkB@yfn?W(z8cf!O$^nWYF{LgOPww;qMf8P=Iz-x!9E@Un4^FN6G5lFpo%`Uhd
z8h0S~sqTBW5ga$~!r692zGvHcvfvCj?O4x8oyT4oF=OZFvva|5kSM!&=QN2e;5-I0
z7lifKK+erDy^30Ig5)^T2+#j=9K}BV9bPF1F5^t}5qCF$%mm^2XCQSqh^=JmW_fkT
zGYfh99~N%4)4}EK%I}ad8IU{(AM&~h_A7{;VGlX?*WmUvaJUvN(B9Q~g2(bykOkO}
z0rGdi;SN#@!W()mcfS7zx&K!`?%j6F2-Nn^<uj1A)?)WmQS^cIZkYtxQwU<?!Zm&j
z;BZHm?>nytcJq}=%>BV<ej~<UKSb`>am0mjry}=ai&gK=f%RP1hLq(nGw)wQ?4hx_
ze-W%^Mu-mBzdIM7v#42JvHfxyB>i{YVny-C#nh`iI_(dF-MHXD<IXKhA>|UxZ1=*K
zU^&-k2jFQ8Cbx1QqMZ&CL#G8KFxL%%)F>vL1IND@-!iZ}L1Hl6l936v6U6_$Rn3w^
z>Hc=+nt&Y#g$=-Jk3Cia(;#^e=CDQFP1C0aIg`NN0&+*tikj|icTNgfY?}bNtG@gk
z>iqYe-1j>~FV=(Y%bc#UGeh((O85x1LC&15H-M}w0oez`)t;#5u*1ZW>3Ke=^NldM
zxq0is_5?&j=H6lAAUfc25Zr%O&dxiWIVbH1es;rRZxQ4yuJ<t<VE2O5g78^S$QT8P
zU3w2vzrV<bv>}`4a_qdmX|?5{Gv(m?pqhhv9(|tSvz?-Lkn?kwOTXSO@))s}24r4*
z+cvOW>scWEb&wbg%YFO;cPC5?o!-w0>F1)0{}X`Z)2h>my?Zb<cfRidyA{M|y`yin
zDrd6!F@ZFTf8mfZQ#L-xnmmv^2<L`F*1Un(E88Gy?%#lv%_%`IcJ`D>StP84o@<bd
zSStWhA6W8fr$Pc`Y~=Xit2>pRZ9z%rlbIl64R&gfab=Jm7~Zyo6Wnfq@v+f&W}}8T
zOwE?<+Te25j0G~M0}}_)exDbE!yLpG+wpq)e}(Gpr7gECE~`S$zn@_TDF;CEApCyb
zN_ad3M$QJipIt2-9EJ|9dw1HD>uxo5hn(Xu^Na(yd;sYK;lS*k9rJ|tfz|$PGuXcK
zkP<jug5*GW^IpjPR0+AO!1)y<2E%7^I^phwiJ{Y87F)q(HoEvHCdj@pLsrZ-l>bRY
zzM84<!ECv{$<C*j6f8C6A!qX!LGSj4nW?-Cv8P?(*<7&uub;{VhlyYn^G?RNM$6N6
zkanDC5M+D^rk5?Ed&fh@qhSB<h+47Zt`TJYC`^9Y;ay-quD=6WuK^PW(HVCkcld(X
zxbTL%sOv3Y>Xyu2362NnEXcZCm^g^8<y8aQ31XL+&)#vw{P+%sw<+dRxsAYmiIWPa
zz<vYCgD~I0#b7^!*dGmDz~<(*LDp-0+A-bS!CrOSTP_W7{7FCR1=Aq4Alx-`#f}pT
zAbmRt0fC*;7f{=4hK`Uu{_^>#^8z5fJESnjE<obQ_)oVuI6WeZf!Nm{K;{!1K0wYX
z1c|}$pBBhHpCEqs52<ZxE7f*pTw7_W^cAuP$0HVUmJLWAgg<_T+-(SAU;6-=3;)Cn
zsqb`nKku9;B5&Df0@+i#XwD22_jH}vxl>mWvKIAM(rWYW&8XvFKSLn%GfjWpQS^cA
zGS7#sn*_0OVeWk^;BkYjX2YV{;QSw*;slRtn7p<Wq6~ie`|6IZp<z2xH!xf3#>9cm
zDm90!?SZLh5k&0iI~xTl|7+ABXOmoR@!0w7?=1@!YsfzH<2k6~yQ2TLTL^eU#^A)J
zEZS+(`w+#SpG@k&d1K%7t0>|iyAL0J3{H<AHZB}+HyP|^bosS4hrsDD(AW**{Es77
z5PL1|?$)u`9{1itn`ee4+xc9uTb!1n&i_fQLafcr41$b>yl{c+kv^2LW!KH`1(uqr
zkTsrNv$mu7<L>lLW~sl#z~d>ZPANMdKS4dO`Sc3NTwB3=#2u3`zbrZdS&staW22Mi
zTY>$FE~nMG1RO5=6HxD40jXQF5HeQ*Vz0RVY1^||tQMc1cUao~gWM${sRy~+93&6I
z{}~`>K7rU@!monu6OMuG=~{nz`cAh=>v#ODg^VfQTghq#QU}A0bs9T=ctY9+$EHi|
zoV&OWrEEDG4moS&{z24z^C10sF$^ebL1O6mtd17gpXlPZ=Bk71h2!fGcNf9by!3$N
zT@YWaak_<7{29yp@$6P&`R~AbtgKPbo!AzJ=r4=wLe}XQ$2<nt&Hrc0?f#&)#qysW
z<Q`96?|I;S2QnLk?M`ppnX&F6SnYu&pLaYB5JCx`=?@`w{3Aisas#BdC<}5PF^G)|
zhsrz!#|OH+(<aEB#j(2HV7rm!|IUZZ;ef>ci?CR&kPh2%FXG<zzy!$rzvN^#aC!yF
zgYXVD$i0XlwoxslouWS%vL<v}v(HZ6E)I)$R!Dnj`b*UNzt5f3-*w6kvY$lE?8Yv+
z^{Dw{&V^oZ`LrqvaVHDN9uVHe^p9ZvZ{3G^Zvsf~KKov9J$1819V2`$-G;1<1*vng
zHQM?pMs=r(JCoI)8<6`3;<-`xe>b~A&Qb=cH*A^*&flLVT7b)nWch<TEmuCVY?%z%
zQz<(Gaz6-2EePw(^V+y05OM~CPgVZTxlNEYrXV>GZc>4)wQfp7oG}X$2Vt$pv$3x2
z0Lj7dOT7(Xf5P}Mdi7JtIiv^s7K6hVCJv(2cOd3>ZY56L@-n}9=Z?usE!VGw%+tM9
zh1^#MQVYV?dWf|-XT>1vlpjgF1^d7K<J+Ci#=KU$_1}Q=s)8cwIzGe7DcesqLiS+t
z#5V6#pMtv2$IBiv*Sz2X=KXJl(v<|`|E#ey*8HEo5pqwZy$I$WimvI1`~8-DjNkF9
zjd|z6lhPKi?ICv>u3V0KNBxqutH5y%3X63IAm%=dO9lHsb)WLiqEl%WPs1U7RnC6Y
z^DVym^6Z*r3)%lAmuR(fnGov!heyniwdGf5%*Kevt)~$CG(dJ^!}@8BaJM1L&rn$b
zKL6_}&lZgI|98>@co}7WzkKVzxR!0KzH==!86oW=!>77nzroC_>^FzUgFzApIJ{$-
zA^R4EU0Qc$%n-M{_No=^{-)*I!SXP@(a#TVvx<SNabBsLzOBk^7K;4E2uNIbyhIH*
zkp5L4Ofl+ikT^1a&tC_38?t!bJ;=SdS^u75r2pGBkoEW=^=W-8EhaKf+&*Q;+HLuo
z3&G{TN;B%Zv%~@!xET*z^}+14Sjf3O|Bi_6%)7JMJiiUnXI`0!y1#Rx%G@2d`61_x
zHvMJTxwINFmkP2!mVGri--L9=V#MQxCy??8qz)H8ZUH%u6kUF)2ju+6PY)4m7hr0V
z|0B*fPGI)6nD%R)Md{gZmKzfx?eyFisP#^$AmVOrwRMnj?{8-z=jv=Yq_Zo+`h!Jl
z9%PUI9!bdj5X>$C<5xRU+931l;;ZhPr?lMzrxTd`;(L&>xTgY;`($9^AX?|AElOGe
ziJ{|1CBj(Ml(&k5#SI%V@4w!70CB$QkMmErNqXKjKQ=4JGA9#~?yZ(U&US~{Rka9m
z<_U=3Q>X~ePsby6g7XOLC%>I5{Y5NguR->I{Ok?`+Y3?)!fp>%?>KQ7vIk-5T9I8`
zxsZJmAUP1e#R!>mx)zRlZZ=5H{|4gTF%TacUbP9eyoIUZkb?Ao^Y?y$=XIFeq%6oj
zR}eo{ZNiT39dZ_x(`>f?n79?3m-hZ&gc3(QsfaTjR;fbv$dpCT1gG1bt?zeSm>|3(
zO?nfUZ<G(|AA`&W<Nc01S4Uog$i?{W*k}qlHw-Mtz`!8?60$yriEkT<ogn!Mj}hlo
zgZS96e-ff^jx6uu2e}uFP4*Tzo{;5Q*mB@v6K-v?{NvVSaj>athv|CA86=$=ko|fv
zJ@rcweU_K^AbTU;<`sa=sChVNM@nDg&NXe2`(JtaAnk6L-eQNNJ1c4-cO@K=+PO3C
z4C;7Q-@*W}Io4Al=e5D~f#`c-gvY--uR!i&1gS^HoRN@xI^!T@Up}&&kS1j99!PBM
zqZMYo=2|<ymG)VF$bpO{i*rHNc7o(V*gp!f4q@AT$XRZgG0EV3P;_g;&X9X+E%%5)
z=7ZR+A>)o9eIR^m{q!BquOaJcmqm2%tl^YKDd%MR3c=;6r}SPFagg5KX^=CVKx|xi
z$~)Bh1g7pY8|0kHonEN-(SYQuUtR~F-2-Cpsp{JCxXOJ;?RS&yrvxDPT&=1S0H;@w
zJP2Q#H4W~zhB^1Z?4ws9dxY<stL%Jqw_~STGNc^)^BZ-4N2%_$9qXUI0Gl^INpk1c
zvxvLbKxQ950hx<(x`bK(8SX{g(*;tC4NEFdMk#M#YVN*u0rxY+65KHI)g)EK`lnX$
zIXf1F3h#Q@Io<Na5y&~^mpmY6`NPagaf6)s3gRETqzg7T$qjPfNfVRJu2oa5tSqi&
zg8LaW!co^FIYvnBdg2-fmebw6bf*|Q>iz)v63F?d^9!`V=?P>93|}mQ)Eh89Hrgu|
z5+CStJTbq(W!dT7xftf24EzaBb09r6%WSu1tbDnBgPgIYrz@mPlM)F9=Sh$}2zRCQ
zz{Ai5I{$Y%ax2)3)D?_7cYk2Fe0U-O9LIj2nc-?ddfC=KGfh*4tS4Yxld;1|;~Lmq
zAUP0T8wxoO({MiGj7^X@2><SxfKhgX#F26It9cmy1Ie%6aUAT9j|(CFAdnag*VaJp
z90&3DP2)6sxLR%J4zCK!L$Z*39y(hV9A6-L5MI%TxMRGn{WCbd<#9m971P+J?|d)S
zXgT@mVsMz9da4f2A0T}oydwAQwrxg`^*>q#D|XmaO$ECPBnQIF7$JSJJI0W)Cy*En
zpL@#&4r3S}8!es)xsMNB?wIOHusi;R%mTX$S$<NhCpfM^VnSB3miF(R&D>5qZ0CsH
z1aALj{tiLW)2ahG^97{-t2?AGXtk*t95);Dmhbq!!`4FR4rCwN%IYKF@Byg>;k0_b
zU5Bb6?WtKWKJ3W%V+6YkBnQHUD<JEa8I2KZGC|@XytCOI#T_6qbUa-Xb-o=YU+xV#
zXRb%a5F-q?twP+1s3T=<d40!;?ZpesEcg3C=J4x(K18wiAn#$go6Rml<}cp7hty3S
zc6_@Q@4UHPPXMyVf0sUFeE`fIS+Og3@N_}OioRt9ZohC4vG)L^X4xOeJ|fOF(^1j|
zNG|#<WIYOqjSD|6HAD$-m^%3%kn>b;uYt@{z{ElH19wPW1!7BUov_SfZLpa9+t<>n
zj}h#Ky&7&P>EGxAq%H-imn*pmwr|C{<=`;-#3#DTXVZF%K0nC5t{!L9{Y?|t*X-P|
z&<L!SMO0>&);q}gfgrO$*#9WxEC!8hnB&-2(lPt4AU()f_i8#kZji;-oQI6l#{8TG
zPwO!GL(35J`)%6HJ0}%9+;Q!PiAAA2WbNwRxu|Qj_h&%v`~>MY_zW56ydVi#BmM5Y
z=FZ*e0z2h*-vj4|4dRe|2T}{dCz(2S{$B;@<EVX;-udcX95^gMav)qQ1{t4s|7?Ka
z54%I!C~86K(edubka`|n{II|sa9(z?NJS}cLF#mRAY~4So%%A&qI8zFW&Gt+mdn;`
z0GFklYao6E$%F9xc*L0%0nugPG{Wr&nYY(-)!TJc{inqv7RX%%s~$k=0gyfrZaS*9
zGjO#xI1XI;nzsM-LT%F;%R&1MMqS|W1L=X`C)>^u9RE@Ge1f%)Y&aP*uYM*P^9->d
zImlWikUd{)7<an$GVNrHGu*1u%?S4Yr8khiJxCse*L6YWyg_VPm4{$8Po*Wnaib)=
z)S~#}E%Tz^kbaD^2V_4hNG%Aja}2gfuzCpg+kM8OotxIpwFHU7u#UoHaM%=`y@DbR
zl21*Bv`0W}T)1)%WS=*>e8&#RTIQgaka1~bd5Lw1HsMDmCrg9<Uv}(hbKh}!;cal5
z(dkFDk3ssjl?KAgP<Cs`y&98ldV>8QmN#d|dJ8_wyCJS%{-v{!JH$ZxK)7@MtQ~KI
zTfq8H?-bt|+%FDpKY`>xc=KaOyYb*z%<<o<-H3f^Ahp==>o<`47+vnjGsrr-{!Ylb
zWXST}j)=LoO2#si{>}Gy%*uXcDI5;jEAq?^(hh*>v0qvSF7rVABzH((SZ;$q*uQ&t
zwRRl|&af!6bp!Jm>JV#%K<Z;mdUpQwJr9<P4A-*gC`BFr6#D`hpV2Pk#)yBZqo`*o
zfXqY3&TpLHZbO#)(+p{cSFXh@|4a5A1iJ^MKDWBha!M?RB@@F^OWPty+r*G-H(Va1
zhIt)gOt-@l(y#u|0y)R+ua)mEW9=M^*Orho!~UBrL-B`GnC`AayCL&!s!I7gUDq7|
zrv;F?AiQY*Nw8gsD^Tyv0Leu!&_bz)Kw{{aMXCtJKA3!w#uft*`|DmUjPj3-`3<-X
z1IZg5mo)Qewz9Zzc;@!mYu1AEskuMu7=@f46F6Uh)C=m!gY{OfgzR@vyLV!zH(RI0
zB2FoAK7M72dS^*;{nzb0t6aeTDBQbq+uwTB`*8(JAnOzSO%ZEsK=y#Jh&yBs2gJsO
z59l(3$2rmEZ&yRo*_On)VD}=+`&>qhi3RUpvV}dsYv-CZ*_M~CKL`7vB@y-hN8!(i
z_;>cv1&8;MdyqBcPxvS7jErlx+;|(ZA7<+#)crl48Xeo2lOcO>8=jl(xN;r!KE@kA
zG{Ny7@d9%k??QwEhTD<-0b)Oy-i9^(S6_sjoxJFkDA-RRwJ^-J*#j=N=MMAEUwmtJ
zl-}(!f8Bc@9M8MuA!9r+^&U-Ka658}AomrfKY^TEzfSRmxza%a^9=?^!Fn?qtKn)v
z`hVR$zhg=c<X)YH_fGF{<>*F9|2vH!`zbEI#_a$6nF2Xe1Y`~_ylxW0ZOCdC2lIpD
zX+9fd?h#q8^ey5{2(IO4Ex+b@?c6)vX2;y=U%=^K^)2L1W|$tgnTWRe^l6vDakB9F
zZ7@Ca&?57X1v9qCt+N69>+TKIaZHZ*{+*KYBH;9IRcx}$x;_N#E|}Q|-$Kd}gKah_
z;Rcd_Q3Tn;4`SoO2d|*cr@_?qYf6CasmjGXA8qe62XLALsjXiuWKn3ac1L8n_4ZZ%
z8Q{8|<K2A}w?uwMoUyJP1}XPfXhZsczgqftNaqFY%yEaTGmFu@4h|QP*&zINTI<fD
zd)?r0H4G2kS+&gr#cb7QkoAIR1Q2`7LHa=WY#n4z35bmgcZ)lL-Ha~3>L+B)!t8h+
zaM&Zu_eMhcBOtL_-e)@*H}2la9sABgz!Wl1)5HfU+d%Rl?D!!79F`z9*9*w~?Zx&%
zU~}hmNbXv;$<K1<BvCM5=qNi%{eIb7ZkK(BJXr2@r`WDj4Ulu$KxTn(js|4k{MuEh
z;vl)%uL%1;$pLl0B}g9#zww8xv%KJU8J^cc;vjry(HVFgy>nb{8TqYl$B|u4J5J8`
z0LRl?6~ufuNWE$;;%o{j&5hvrE4=j)Y{u_9KHC|x-t2hREdl1Q=X(a02k8Uhd-I%k
zh9zZy)7viQ_?>?kW8h&4mbZb7qpdrSdDiD*XULhFAa%HK?~NBI=E2nMvxMAbd+Hoy
z?-fiOME__%1aC8M=IOV*b0OGV?P|J3X!Uz=S_%||+-(R_3&L_ii1K1h5aj;Ly6t`7
zI86zx-6?W@j)mKLDKOvdIO;jhOYd9n(%lSM^I&^y>Nb^m?I`JEb3A0-@a@-AG5o<U
z1es$1xd9iJJPNr#2VMSWHe@{Y_yZxZyO8B;IuLuNR^~HV{%2Tc(b#+5(q%8?Uf?gA
zQ0M<={zc3&^jUrYyYtr@$ew=gqKmu2ZI4=hUjx}Q!M$cXI32+3d#rtS$M3?$U^jG$
zZrgQtZU9R9Sh*O|$DOqqGOhyC2cl<4qplSMiJ{}3@cUr*po?2nLe9T^BL&&hj4aO^
z3n_C!Vr$EUc3wXCbjP&s+sxbUK-P@z+xr8YUP1C8?0pxp|7Gb_$hn6m>mloXOC}iY
zRG<0UqQC1AIG??$ftUf(2f~^Lr*^dd+y*v}ahmq7j$Q32{y)y_0IoM4*+IsEKzd-<
z?Gox(F-#npZrTNDk0OhK*!f*s!S*PzL&`Le7z}&pBF?{bDY&|0|7qWy(#u^f<*Fd}
zAJp=o&VPB&?1IPX)*Z=UHlzMJu%Cqgi0!&`XpW^EGo+1Qbrtm-gKg8F?5JwF2UdIQ
znc%L==BVrcH@89B8r}|R81d*P8VU9T%-z`NJ|;N~eIRwJ@oM0F_Ih?VIPHPNU|4gv
z8#t~&{P)i`?eHiu*eUc?%tBq211#>NfO=;fp9>;StDB^Q<EuBX5!|lsdKO|Xvd(O$
z$0W%42T~5G>&^GyTeb6vHsm}htv77D>s<U%{A8j5$>Y6!nD@Wsynhe23*=5*__(4M
z*q`X~sS%KMA+vHJ^EJrwn`eFk`x7LVe(>OqHJlYYDzr1VmOO==#kb-S>KXlf8zAF?
zAoVLUuYlce7tsoi(-Kp^op*MdSgulqtbdM$&O3tCf^fOC>CUg_kn&Ha_Suej^Q$O+
zlC6fEIlXV;8nC$_Juv*wjPUvYQ+>~Y>nxZa7=3WBEx0|ZdL40hD@^Y6;p1@s`{pjQ
zyts6w1^>si?V*Pu>jL>(QSUH19rh5M=0W;<tS*Ah4Xp?On~_~=w7r?rWIN+e$T*hJ
z1=RCK^Pd~<lu+3PcBAWr=IuKgQP<WT_zk%mCvGk3x)6|E7vG|;6$Oc*<H?^O=cl2I
z-|c{$WzOHF0gfkR`5E?zG9!NP8H+$4{+)c@p4(mS?*qq;Krv)&52ojn2IR~i5T8BV
z7M$OhHt>PXox-$rdtG+GwzoSVdk5IRc!K3YYC(9-(d3;b>mg~SW(nucZJQNP)2lq>
z{?F^4kh}-d1H)#ukhvik9~&*aN&&?_n3|Pwko%P698vr7Ao;nukn{**ClxF0Jo`+4
zXTqADW?b7Kd$;Y-?%m^27K6K4<Zv@Mo#v!K*1yFGecP5N*|0NC6mq7_mb1LzZ~>VO
z!V0Gj?NDogoTuva*LauJ0@U*{Wfnu$5#={-0GkWa1H*qzoxo`g#>Yl`_8-UUf0J&=
zc#7kde<<z&nQ>($VobPWx9g6tnNN0@_f9b7@`s$2aF!i%4>m|G2=mny!rkVT1j&;M
z6ZfwJ$yF=g-QpuYZ--Dn<lOs{g+Ec;oSQAZ)1M!*$Lr}KlU>{6Q1@%j`3>oxeOQFq
zPj@?sIHwL|7dE_553(NtUGCsz$e#Te^+1gB-%bWH9|h855p%>+AtliwYrg*u^{P^E
z`!9Uw8gM=U$%Am#7DV}<;{)lZf4UJ4R{QXwrg@wA)SVtzlEC5ot`yQf2I&Lg|Hfjw
zZe>H}f2XY9w=2K~(e?nzOMXxVyKlyR$hk@&F&MUyfb3z1@v+h6LSgW<i7YRr3%O^+
zJpr-S2_|oH7*YSPyY<(yblR~UOgAT+TV8{-UD(?o_qD^+S3KMTkAMAGE^yd{FhbV<
zTQ~~uh+Z{s$D(hZVDYUw^WkMBNPpSf1Lh`Q`oLjx?w<IrDb9<*;Rcce;qr%&_V3B7
zkaNXBVlaHo7jf=3Obngg_Y1S$0IA{lz66{v`rjV~m(w6I82+n=I9K(=!aZhc6PE5^
zaQtg7*u?=Zr?;MfjGe&Lo6JR=>BO4?Ip;E>$sFv@>uej$A|`k5{Q4OZ9+wtogT-Nb
zZDloga`r>!@t5*W+u8pUah3u|O?Pkx*v{jwnC-vVq-Pjz2kAk^4aYF|je_JG#323D
zHC{)+=?f$V!%AV0wOSzl#m`rE*d-|Jvdy?<Uc?S57cQ@coL2^t2Vv3UE#R;Ou@k;S
z>cZ+~$UO&KXIJgG_G6m)qf$u!XU+22;B){|3&L#QzwgY^=L4@NcI>L&EpP|2ei$SN
z!t?bY^C->}P}j|X<YE=E?`;Li!Eok0NIMV4htaPkpM%rci+03VA56}C5@fA5i0?N0
z>h`Cv`F2bx`(|GDk{|5$tvQJG5g;|&T2F!F2*lp513BO1c|T;WoG@GGj#~X`7QVHR
z`=4KTq3*Xg-t=!fLjYvGz5hh*ou2m~{bZ0?AS^9D1?<j6suA!!1`-Eh7WPvpX$d5T
zj=y_A&Z9*aXWtBIPdxvLdHz+e1Y`^sq$g~y<BpHki*|lKAhW%7tpYf`nM9zTe|0S%
zQkR0%H(!9<*I^|42<*1$f&x1(m|ilUc@DCsVAeFqI#7^W5MC;CeuvrxJ#ZTQ8mhHx
z_6*3l97qm?qxm3n4sW+j1&0Gj42HS(-T~VI<71<Jx-Vmd8%SN(rEB0cd;TZn?k$iQ
z41cKj4fZF9uOV$_;Z|p|Bh_QS+13I9aQxr?G7T;dQls|U8Y~WCueO5B<r%XxfybRs
zT{AbIld8O9!a2zJ*Y8K0P~7|||NhQtM<IJ<I#+P*me2hIHw$DQza(TG_0=_)`&fP(
z8-V=)Qiltx@z`K3|GiWp?J&Vm%<~UV)<D*yfy}H;pS?ruY28lF6UVl1Y=O)Ro?M7}
z{>`3sOX2QFiiViGh|?MD&+8I$JD4@EZvXRm2RO|7Gf~HPwgflq>^ArdF8}=&Zrkbi
z1oiy;O-+zJ=V9uQvv)vlg5ihDQO6-+;>h#_=0dpJkj0k^LGJ&}`1uJXyg_Q7col=)
z3Sz(M;I=Ay(q~at!eW`|Ee%%tvcnjhUP1C89ANMop2ql(L)Nb)owx(`Q~pVTU3IN;
zmVXr>{h!rOAm{FY^nq~5{|!5LvP1S$eKMT5_1pF^6!-S+t_7RZXa_MDqz8uQ-a?E!
z!^F_(N6o1FO=0qrcS7z3`?d#j4Y_ItqI{dUrpfY0P}dHzuL0X%tn3E2-BeT|>i}V9
zWn_lJ-JCFG5~SZ-v}GemeoN@HZT3|%JExz4%qeMXQv|0AkQ@l}h}!N{Ds=>h_p+3!
zJ169!uK#){4LKX6SFQ*i#vpwl94PS_tPjM-h1LH^V@?03*mc3~^z*olVa7rs$l73#
znV$;xSgcI;-7zQYp2fn&*5GkLjZf22%A#W-h_jjRU4xv{7I*-1|CvYnf*r3t4sBNw
zd=0ke>=o2|4AmQV?(A;%1Ix`Y6xgzN2WopgRveNRZ&lZT(;mo8Fs!@+wI2!-N2b@O
zWuk;POirqU1zcyPIOm}F1thN~iCF(3vV4_gqHLGNABNfH2i+j$ZPD8EVE2O5g7Abl
zh%+YV_AUe4H}Q!v*u8msjkf2lS+_%M0_04ci_Y3$d5}I3Zi|)LvF9e_j&>cr&>c^<
zLheEZ$$_wR3AAq%g?aw%)q8};zXRnFchi8(gW;ku$UX$|hEL#dfQf_XW6F?r4v0PN
z{2GfBUP?PWzP#M3XgLAwc8v&kFb$Fi;dLF5yaZxTFoWz3h?It$l^(@bXy#JEy5o@-
z7dRiQ%>IB9-nAk3b_nxB&bSxucHYVPO%Ww*4VWPHMq(7g4v;+{?8yx|8yUpLg$2GC
z!}BJx8qEh<V0#K1A!lzR%W0ih0S}YdLRZT*`agDT6I-?;o(-~A@US-Oy<-zncfs?f
z;O~WC_Q8P1;4oU4{Lidn?$Pa&T71Fceg=rS5|Eu29@p&5kcW)%u42vJ>5a6P5+u*N
z2eRgJW(zl1KS&IQyTwu062ruiX@ynT=f7MyAnPBh10ZYfkoBxP4moodB(}*r!s?0B
zar3XLi5AMs_`z{kdt?>3>;TDw@bqBB{jXg`kalXdIpn^Z&T9qRZz;O(m@q#O9H!+)
zsQZ{Svod!*Vz~~MbNjx0XWg6WDDky#9;B{u6IVhJ2ibc&5p}INNDLjfEw#d0{#iZZ
z2FFkAa?J6+H@=8-wezCR?3ixpxm~k)%l3OW*MQyG--oy#9%P?`*8y-^0<q({A^o&v
zw;^-di+^PA(A%}j{81m|oG~vq$hZ?oEeKCI_<e`D?OJfSZei`(w&f#a`~)Ni!r$Z|
z=blQv`U-Y0NDPM8)&;`tg^8il61@w+<q*2K`C~{sQ>QBt!;H1h5aSAebhnt*A5Px6
z`~|alxjv-Koc;p!{`X8j$ekA;JKo7a`neIUKf&&8J-%xDZ+=mWvaqe-u-jVzxkDbL
z7KA_N1npSjC<$&){C^;{>(D9Gb$&aKL)O2)Y=zvX1JVP-#~(rFFkpOa^uicOo<)~?
z^G5-kpC05EV)*~8;Uc)0W|E*~vDNc!a|AY;%N@7_j;93@sbCsrX0rog{m(Q<ac~{E
z$R=Y0Nd8`<zv<M;Av+H7-vaAd_!PD6aHaC&4#wxn;IIj9;M}RY19fj5k1b?uKu`^F
z#umt25Y`CZgEekJaxh%1g?;~9T_0p!DE~dk-gB6G5N)giS*r<RFTUGh{!Z)Mj&r9>
zw_E5z_Ww;@3K^3J$%F6>b;OwtW%nU%8G%`lJ5djHChVBHV76KGLdcrooHdo;bOF)_
z!un5^?=<L!td%P~t-LdBB5M2Ss{4Mh`k&`9&;K#zM%>d2G7B5_T#q?#0#f(u9^~v`
z`$WjSbRaPpHaUz~|5~-W*);ml-0d^Z?BBl1dj&WS-^_3Z*JUvEMRJI>ISzr!;Bv}w
z%|);p)AH_};(OO`yLunej=kZ6T8=$_VYV|y%^sZIw%L2{T&#jR&Q-7oa>nhdNto;Z
zT$&*JMquv1Mt8XfgToJ9ZudIKywKcO$htgadHp#%z<C2CCVuSQj&&d3?$9-`wt86(
zsoNjzTM70XNFIdO)mMQ13}P$T3xLbpMSmgViXIDB@7ld`uEl3v$e7vMW03n1L25x*
zjBCQqGuk}h@ekoB*<DRxf8l0<<Zp}igTqGI{yd5}NN#@_;q!0oWeebb0qKF^>)DVw
z-q*L+qxc0R|K>a-uY%ZnBBXbow#wZhC$N4;#xg5#e%rqvGL{XJ2Vt#$h&#q!&nf_`
z@z;Qy3E-fpy5q=JrkxTq<-qZ9CKGiYfBg!zo#uv+aJ_62wA17;<lZEZxgcD`0NLYV
z9FKYb;{!X$nsSgjTv*{E_Ws{Rj+@{z=UfY<?}e=2b5|BT%uJoOT3(cRW>GhBmgQOg
zli>8<za|At!}KhfG#joyyxa^N{|BNoz-9VWp1NKCL~dG4w1lJ?$uDT<xU#G5x>vgd
ztaectxA}bIV6c9e+51mI)+ZcOfvnetiGyfnX^5F1HZE+`Sq)A<=<=N{Kf&p#`?nm}
zZe)4iDUf-7kXY}h6_ziwF6`JGFvZg1lomJ+-+tu<r&o|X2p{Z3+|5*Bxf^WmoOKdl
zbC=Jm-X*c!!SaF8c5ph2(TDT{K>9#9!dP+V%ZN&_TGo44c6nu@o>w}-5;7KjVh`py
zUd}%e_wInq!iJ-&lu*JGrsli?q&_*mI1eLynu8$w+(Bv|*l_P)ELpL`ljo<!@=H^}
zY5T3G3)n9pc@XC8wSvdLhS4N&7=1{B>{~egpLv(JS+phVf5<xiJ0Iu4)q?bTm95==
z;S*%Psn!Y3U8PIjg7t&sK-hdY<i7Yz9hmDlSG6P3DM&3g9OQ*rZ-CS(hUbFaQF=og
zB`t&G{(kj^`!C0-#yn?-_V!=VPb@{JL+0@IRYC5n0jUMygZ_}b4q|g?LiSa-8@~aY
zk#VbK*S+I&Ex8sHgZZc4A?^wWsXyE>*Zf1K4><lMGnjVu2cour1W!QbgWU{vfYTL7
z4-7YiEkFq?m^d;$^Vv4AKas^i?EA~#fXx$~iaM7MlB<$G2u~B8|Fta+#_ihi>ZrmF
z;U38Szu)da=Bq$zL3rD`yI`|G>=kX0vjICRA@}MeR^K+CwTy44VKC&}kBnC@P{Lbz
z=Y^e_H>QKbrvAFaE`c48`BRWtAe?R@4KAmSq@&)80g{t-!E6(P#F6pT>2g@pf6omq
zu(@29QTHQ&^yUO0@@Vmir(5Iq_ibPQCf~BLWZn*t{0U3Q8K)pN2v2zmx!V-PHv9mI
z-{{Cfu%F&nJ=&QtLElon19BfmQ6cI)#vIS+9U<0``_w*5MepdcMjihVlY9iubJm+T
zW5lEF9K^nRklonu<%`(Q2vuIw1onU6bV&OeS?{yYki7~Zv3ZI!Ep1o5HD?Xt+L8Vk
z65eN6P}{H*lp%YRLFyI%Oa_}<5ReJ(Bi24C-*Lc3cl*od2f^{4G!t=FBuKsPi-w&V
zuWo|(aqMI)+A3Cu+CGe6gY2U+yA0V^0n!7*pJNE0e?3bN`}zM$6ZOG)hg}r0*BNGR
z=|Myp^`(E}jz9SYJ6P7(S?C3Xfa5}}A9c*IE*q&%Z-SiveLfp<hFWr2&d!>|Jr<o&
z*T88fB<(*+JcgEs@9fiq%z3uNs_qn(GXuK|<`%Co$lj=Z7a(J=FmVw5emUf92oM_=
zw*QN{ZWg50&aTc9M4yeoJpa~T8FE)ANIuHx$__ahy&Vr_c3UX^g6tU(`vEzVA0!XL
zm8lZ&JZ9?%nZNr}1=%C<B#nIsci_gIT5lltIDWW&1e^{)`asw?XP&w84oKTLFT!!x
zIwfy#`T@y-aMLcxzWLviQOC_ea-X$h!SMlN<HD0(pq3{vbzDc7!T$H(1UY*aCJv%k
zv{=L4@cv@*j;f@N9ov@e-hOf)C)mC<#*j5xAhjTDT($=ur^h)V^Z$y;kh0m#-@-I+
zOVf^S$%$Y!@p`XO{J+%6bf@cH$XStf4;SrR`gR#u9mre|HlGRE8|JVBa;_ps42Iv$
zK%7$q6GNxNb2fnei7x&i88T;KItlX(xib}zGj~CH?*4yg=@&6=d(^!7+a5gr1RnoB
zlJyTvgXBT@%6-UMN)X$2k~lb>D@;-X)8A~SnhUeF?KJa;-1~jQ3bO73q!xsqWxDVB
zy}S#oHv6Rfu9-Y9z+nuM17WWm$lMU4F6y`tNRIK=C9Hk}$-!{T)f#x(go%OZ!g|Pg
z^xNiO-k~vV<wkJX4N`Y`XQpNBW2v1-&YrLk5`>J+Y`SX;uCGDzAbc)a1@4Zwry%=0
z7S}@d?L1I=xI<bcYDbMHWR1U{Eb2W@_b%?)A-Ers{!fVU?rP(M>~{v43&K<GL&Ed#
z(Z49^0wg!pnhP9mAT}=CoCWE#qsx2NW`f;(p*#hi)?sRXE<>EFCt$6#gX!oebH+7F
z+xaB-fy?xtZ&Ok1?MOtNO_s6)GOm2)EaXnG8#D7aFTNtYbMuKE;BxKlIfy*WE@O+%
zonhgSIpcdT_w4w#05O&dQX_X7GRC#h4l%|E5(nYFWXPF<AT}<%T^hB%fvGE=0BIL{
z{DQ2@g^7b`_F0g<+93A(cXQ2)cE8-I?NDj)dIDr$NJIK=l=#p40Wk}t{z)n1yg_pn
z$h^w^gDyL7dIwp`Hp+taIDSE#Edx@&G+%3%`kd2Xxixp*?Apz~791}iIS^jA8?ruk
zRm&t4agf~A8q~X_Kw{|luaPa*^k4rAvWD@(xpI{J2GT#L6X8ZPA8)hCd%CvGU;Eu6
zi%AR|@7FfDgX?RMS`b!AgS3l5Z2zq#;PS2B0MaJ<bwGcoWZZWP^EZ%w*fl;TlyuBD
z`JQRW$9k~2DW^~E;C#{yPA4F<KzLXAHSl=azkbNLI!FwLZM7l$kzssn^qM7nDCWV`
zl!QRe8M>~FxrXj*MGn|KAhnZtmh5n5mfmT!qtQY*(gJK&?k~uhv><s9?pcjkW5<&V
z+2<=*&kD|0yLNot;s4svVooaLJnVz}P{*4mHkj<x%7?5$cT<+wWz&y1O9o{2{S}b0
zv3I{v&rJu(MK#o;ge6D}9ot_+oK1!-_eE(QI6bYjLZo$={FQ@VaQ~|wkF?wnx@<@D
z)7jfj3MPZoO4+xYaCwj(X*<YR5{T_}0#aY;zn2874WHX--o5k8)(uu!V7@Hl32+$#
zQVYV_O+vfwv_ST;URCkh^<p*ZdCe<C+`(nBhyv=l03f|*jzZenAT}<1c)mZ@^uOW+
z<Q$&mf~aFTAag3E9>D!)oF}%EpC@7`vrd+U=srjq=5a;6|9MRlV*gKTvlcj9Ym|P2
z{d-YTWas+QR`b<Ms=)sEU4VLzi=M`@ohwhD0L$fFSg`Zq#Tb<Ei4%jI2ejnbOpLhx
zeH63r4st&-E|Q0&b7V0P`}dbqVDr9LBj&n6;vl@x3bCJ~X(FrDE&)S}%PvfoQ|?{^
zuj%-ustzu%Kx#o)+6yrcb#EGEE#$Kt$hgujhYvfO-UwLAZib{8K}W=136Ner-HSWZ
zJ0WMh1Rh+vbLojQV0VG!KzKpoLU7n@-*6h7_dsGWY{idSmcYc3X|v;s;P^lm1F=_s
zgWTDE#29kF0!R#oqhb-`KVD%IOrOtX-oA3`j~&N@Ex~#E+NDNt83$9p^95r3^T-0o
z+M&0zAmQG#^TD>gtAutQS1$+q%QYb!#UF7$cJHWp0LdFC)-Twh8HUuCU|?WKTm{+x
zcsUy~4+OIlMBBGP)`o%DxUlWPLs<PU`*{l39p~Ip_Zx!D@w<iC6F)<^*h0so$-<eN
z-*WyRA8@;mbwWCd{}*;V0h<NVt3CyCc9xJF<h;WZS<XAB+sIqWyobzj+Zz1=hXY70
z2&>+mxHI!RB#zlQO}2|}K|NdRM+{_-<M&KiaM*+Nz_7&y$apM_kBydlc?j%gbUAC~
zt6=l&BrD;09i}EK12O)^mvzL_B4&ZbZH^n;_xP}Z-QJgk*dGAW^TxRm?*EckkiC5B
zsvp7m-><l3$6>Lz+gIk*g6;Vff_i6}VxQ}dd!L_z-6Hh3Y=_$q)HR*5Qx<{K|FZ_f
z8bgqsAe?jp^&D!D7&@;1gnj>~NV7lKeTfW+xl5Sd1*^=!c7phq!q!>7TDWyv-EyfN
z+B>g-!&1Hr_5SPkfkp82(6SUVf8RF|l5S=0v}~`oX0co>(GOPh)C_VyEXZsSKK>+c
zM-dyOZ@+Hk^&JP!@}Z>H$|sO^<eUu1d?iQ^3{NwKw2fhWZ1kzb^WZo`my3^q=v|j|
zAMRI}nq$Q)!SMm&Zwd9?dDl33XVwGZooVrqHSKCC>EO5o$%C-TFHX3dJ1Sbi?72T6
z^SvvS7<PJ_*;;7HLe77kJNq_@d-ibL-??X*9oW20f`4{gW<@=ZA)AK{EN8L^b>0+Y
zZ|Gij47Y>Ck#XC{eJJ+9<kW9`1E<;f5|HyXVB#Qp-WJF?e<1eVSamDjk2}oQI`7{R
zb(9<27Mpq6AFLN755k9a5$mw!R^I~4<xj8!r!RK)AEq9m4d!fDAoqIRQAC}On0YLB
z*Ym|eV70s0ig%n2LtW=_&5{pX*Zk?k-0NlOj+$3MZb8SBzCq@r(8aIx^@7u~*9*w_
z0<!#Vy`}K@@B3=I{lIgM9dSQ3ElO1<g7stvp9lL9rl;+yDmacn{7>f4^-qf+XH$Bl
zPTG0#@`LR~pCJ1lx||bG;^WE|#vRxHZUV>SbyH#UEj(V}^aC;rgtvcz)O&}&V6OjE
z<!T0p8%P~4%wDL0RZVmY<Sx6w)ob8xhnev^2ywpivel7hd7Lskedcs*dzk|{=Us0#
z7dTE}>hGLG?9Iw6fUFJgP(BQ{*URJ24(W0p%gs+9<Ds#um{II|uW@}xo$xoX|K|xG
z-FfjIVlD|}{*|d~!RnVQVeaGq&|HNQmLT=$_~~rK*bK7VoJUIFIOLj)80&_~f2?@|
zb~}jQepS@6XzCO58?m)JvX~sfW!U!>sORoJS_C;u45a?UBFK5u0S%CQjHm9}x2<~T
zk?oB))`RuVo&mYf45Su>d3N33@nY^FV-VZ0jA8rk`{5{l(p>{ti@M<OcW~MRse@sO
zDK2n#!o<+&g8kUrKQmuK+N>{4AnUS`_0>Fv+(!Zuvs$-y$H_X?ou)HoEd4Tiz+raw
zC*({DkUR*lO+l<7DgFl8SFmgHA+Xz)d_TN1>!`lPk8H@E^5-qyDCyf%;Ka@;{Ka6k
zb8c_h@o5iaUpB~G5H^_)xld4fPAS-KkQfYWs6pD8Fg`Zg?Fsh#&!yidfz7*dxd)uS
zkoCrEL&iBlVmBV7nIF8WwL@9?@OFb@9<VzD#WteoiQbMFo3FV8xxc7;lK?o6{JcJC
zXKdWv9Y_9N1E=v9#)!QVAoH((DA>Vh4Vi<#$>FuLmpKEazp`UKWZz^#Ip#TL3y3)X
z?Ro^{&R&pR$k<coGB`bQF2)>J?uhpWyA`C~IQG?!iO)4HS1B7=*)>7xo6eaUV84On
zLHL9vVvJQp0@A)~D!&L;!*~1ME~keREoW|rv@^P$QTsola$(z>E((C>Q`Pf-?esMc
z2D=MnE(pI`16ltlZflGZE?~Jh%sX*G;>h?@yC#ZxFuCK*koDxPTOoZqm^g@LTMxOr
z8pM7fXtDEBcG%93F9j^#GeO#Jccwzls{zS_@T|`f@HDOd@*ueXR{IXJSD+~P<WBRs
zH9MyYLiRXJQuu<Bzi-Gy?L4Y22M+H|9L+mjoFV5OfXoHqb@L(hhU*#BdyznLRokP$
z`ao=4IA41zihVG3zK$v2aOo7{MM=vb`Bg?HaC0mE-LX)c^vW{p$OFso-fCbyQ&>ab
z^&?2VhvEUSIEcNs5wd=v;7SKL{u@6$++}L9-Li=fa>kR>@m+AWAoV*v#ddu;bqK6B
zGiT5C#!sl@-~SjP^Zy6`L(ZNB>49MxMbvp{m^d<hD&!f8TVZmC-f@G|(+gI}-GeZ3
z5S?3uSpQ~_B4c%Az0VHUfD;xQ@2vux<sT{vPOl)fAbd#+QKm(6y$7cSvptag?i<>t
z?rgvH#$ulSb8whWnk@+~cR>0;cz@~IopCoI=YLQ8abXwFderl3k4$_Cj-SQMsOQ3i
z^j`47KDG~%gW-3j*w;UX>p||5e87*nhdOQ+<gOx+8K*hknTM{Lxg+=YG)tQakh5rn
z{ZY%^hx(AQ1d#gwl1AY4_EruuruFB8>n_1l$(9+#6Ttp3IcW|KACOuQZZf%IeqzpT
zaM)b2yts2hz6grEedYvknQQVHa;_ps4-8K$uSE$<m^d=M`!3`@a%3?OTjYQhI4(~l
zJqFVtF&O?Vk68bvo6%;zjL~)H%<0oDf4+d^m8DJ=;PeVpA1#I$|9embxmPslYztV8
z&g{Nj^UbzeI&6aM<I~%^6~%uxXRn%9^hSWydc5Dab4?)X`EN`%(%`T$Pv!)tSD2k3
z`lfvsIBY;{T-f&t>V7JiI&F^IV0TE&+X6NpCJv%w>=E&A&Zf4*@z|9eaqMzC+WtY-
zOfHis0mmgsEeN|wBi5lT6jcYibBzIH{fAz;ghlPP#}=2YAnQNawNdNipvh{x5>p^+
z{O3Q2+xgvA7_1*;E(m9A76rRg>2D;M28qG&>FRI{x5MOM^nPW?JQs`)qbpZ2g4<KO
z&+Y^FX<*_YddgYI*;pX<E2pk4=ecf~9bwS5{B`aQIK1UIpq`o5YOM+{)4n~0oPTYn
z2w5X}V#AZ2?iYSqh}azi>ot3ZI{&}XcJ{UvPatJb16Skr6IWPJ{AtWm1`cbrP}F`g
z$ZeTdTrkouNE{g-cg+ExbA~JiVz=e8f&H&Cc_Y|eATb#3<>&z03E~H|rC0`QeKTJW
zC~U!_s0+?-$JIB2^@8L<_}v}{aGnRTU+;sA$^3qA4HmobqHjlTG@scoZ7Z;zS!*Ek
zKp?dsyw7^=PQU6E;5Z0Ot2KME>^X{`_AQ6Z&ra%woL2zS1H<b)A$bGF$41|+oC0<S
zy4<B(koDA8-ec~oKYqIr9wq@s&vw{Ko!x2}bHU=tQpnkqIf0O~jbUbPFGifhvab-b
zMkFjt25j#<!?Qa#Y^}3cX=Vsct5IjoP~!JPHrLLV`4G7cmuz-ctP?@;|Dh{;z;W@-
z3iaLukln>cOW^JV@v-4Q`Pj#QwuC_L)40oux&AqIHRP-|ka?cWt9ICPNSJ58QP^=Q
z3bNj6qxe#=UqJF8{OmpAZpuS!ko(FK1;oH*+M@}YJ0}-0?2Ij51$Jwh8|pf?d-J#~
zxGzB3JW}@+b_K47+#d}x7la#o4uk#grh-^w1QG|~j?0j{20(0Fm@Q=jINhPkCj~*y
zAv^OFa}P=RbwvF3FdOU?cp9-SeTB)6Blj!7euz;--T!o?9kFI>p5%UT`)|!($i3zE
z3Kl!~Ei8Au@<;^d9j{ZU`@hdfCG3=io<aKNIO`6xL#TbLh)BrXsAO3bI6WczW7#&y
z-O$Kl=<F%Q*!#aFp%1|3P4;X+iA#|Fb%lsI@ip~(EuSw{vsmS`eTR=WWGwa3e$+MU
zd#w?3mUVuR^Lul`>cQrA8ar-dXjR&|(FJl=RZ<&bt{7yO6JO-6XK9c#_^-_F-O(im
znLh!^fw0wM$lUneAE@^NgXHcoL-v$`*tl@nEXa9G=<*9(A#2%A7NMSH3sQIU$OW+5
zLG1hG&n!d@BzG*Cu+}2Ou?$?!bI$w&mj|gSQbg=A44V#_-;28l+3ys^*|1YY-^^l?
z5+olt?Lp0t+N;=idd-HMLBN#ve&^g@ka<&(xgh-9G6S6cUtC44H$ZaLuh)UoBZ!R)
z-`^F7Rn06G$k{2sf-vX5xMUG?5C(Iu?r7kX-{CBfW8wT>7Mz#P%YH{OGqf4ern@~A
za!=hNUC4cRP2aX|pCxCy^MpNQj%&v}L|+Ny=2efxckc3p+~d3V#m60o@1Wk-S!V}1
z|20t<amN5i9|(Vmh3EsZap7QP$hsSJ`A1B8;C6=Xe$=}xK<akwWB`|4AogOP58F3y
znZI++!cCUF?vOZi_+t#02dQb&M&y4U|F7WkU1d5WSk0#EA9wD5YHxLNHDv!khYw<H
z8Az{Sl<zJfcgT3!fraL~{v753y9*=-!kcp-=V?|1VvheCEkvBt15%3(uZw`($Am6d
z;~@#IXC$p_z%;V_=InFu_?>s}rODo7vfF)m^ey>sc7VlCL_+SXhUvNe9I?k{huIHs
zIk$WrWNn6PU;EAt_MH|2;*fK#be^KF$2B{C+Pv!AJ#ZSAOq{mE$qzCv4KusNd^*^D
zYq}77jA3$4{Ayr17#|z0c>;3pCc0d(6lDL$#z4$4v_l&IxKgWSrmvg0<H-J>7S;h~
zV0YF8A;!l+=FZdM0LM9qJ;C}bSdIE;$T=9#Eb?}SXB1eTa)<QsbCe<T4j{E4?AUl^
zNAOL^-k5{u5AHPF135nhBnQIMO0&RzY<h}XE`sD5&SPKO2a<!~uk5Jv4KVSS=OF7z
z&i-19QkH?_7w(=6_5+ChNPdxJj!vgpv!&RMOKOlllh^&{;B)|z2jOLp(!q5Jh+X<>
zAvi3Aogn95b|m_3|M29VMYS-b?ZDyyX&-~sg7B*;M!Sp?A@>dV?|8QTza;8D*99+M
zfX(ha<qD=jdSKXU8f1<U#>Yk*a6tOw=yFHgA?v@R)*#NhfT{7Zgv>dC_#Y-j?65P}
z*xAp+VL5F&<V@QO+cQzZENe<O*v}yK3SN*oLSu1A9w}M2WGCbN7K`>9HDEm-5)gMq
zfz+Sq@7#Hz@(Nh)f#{l@Yi8xa!yY6*Wj5p-sW0J>ekDi@h8O=wj8nkG&}rx6*w257
z`SJ}Mp1DGZyY66mZ+A?DyJ6mk2bL|t_1n!uEG?Sv`~cT~y!l4pvJIwwiU{J4mE!e~
z^QPtHA$8Mq{j!~J_RhADNQ9i#5;7OEKNO~S#k$u!=d(lBzjkl>zq3RSa^D+FzEKr&
zK9{$?6xeQ<IEc>qih2$wNDLi!zKVv&4YFLrF-SaFEP=E=k>xJlfwV_JV!vY+?7S|j
zveQFGWXJE#)!_9H3tdsyD0b^Z=KMhF8I_yB;rdJ(a=##ZXUR^d2*aHkx7on%Kk);x
zRsy7c_3ECTkE$W-IAc?m?~qG{^pinyApAxQvj0o^I_CN(iT(%RbOTa{3$KaO!|MO*
znN48#gnvVn-7qsW|3J=e1Mx+@*>^PTW3<Tq>~0};-T_?yPdmLGTqc3!L3p#-5^$Lh
zVlS|SjPuOh3E2lQx3GBU>n&*(9>$P069V3lbwMDtAS|g@y_5ed<g6H@7c4uRJ7nQz
zf#h{0lfY@w$NVgcI7qJ5Y!z4z#Kwgs`yuD2pvxb8?+*6E*KMa!$}*6;NyR?!a9#c7
zkmbjn^DIIZRajiCd<k|#OAKne`wG(ipEpiJ_P0w*%><jvyMND)!b9d}-!=+^`G1(Z
z!Ql?F2ZWu^iS6{dQU`X+_LQLQxsiO}^aGLu;qFV2_PND-$h;az42GAxLfY6cJ~ny=
zBkFnkFg5doAoa{TSHxa_nB02laCo@>X542fn!0;OMVZphnxJ=Jcdm1}0+$ErIV5lZ
z94;XCW+4}Fdh_;ztg8z?IAe!b_($_~&md=Jx@4m6|Gxh<aA&VjB_wRNcI;R#ih5oV
z&+agATpV(|g%U0xyOv+WKDG^#gW(17nD>T(<R0II%rl#vhpg8HiNWy9IfygLIP)*<
zF!?XJGyOoX<qg+9usdbXU4#1<q^GwPvBvhdEu@~F5(t@Bec7(JE9}e~bDI>%eQ0$L
zQP(jCFI>KJ^&Dw%d0H@$VP}c%TCg1;b3u521!Vmz|5gbM|KE*=+^GpthYOc2+>fCT
zq;|bj3|QRs17z<NNDPKw-+Tv;-^AL>mO_18JL}!rcgn1j0{3lu)>fdTVKs3`oe$FQ
za|<#T=H12uHotsv+s+g3r|s}@hU|xkbb{QE3Q`NgeHm@L9xQ{L8LsaxxuaPEah4iL
zzC2?;IR3v)5CfYB5`*E;-)i7+gYmJ^*Y~1^H%v{HfDYI%UvFfB(=<#RM7Iky!Q<dj
z|4j3&8H+3~NqjNqop}eW_9Yu1xEutj1>xzp7l6e<Z1JBEe{YF{+<)^{Ok|gh?Mw^#
z3y^t;YiBIsW`oq*-Ra+9eNZ3lkN4^F<`bSl*3W_DK={92FE}pbOd$K9L1Hj0=c$Kc
z7EBzO7S@}oi_C|yAH0ItV{{*~_ZTJ)qM0_ygToxee!R%SZ0?EmJB((iSVpBo#szQX
zL(Z!P$%F8w1Bf%!KS)V}-Mdc*(%wz+=-+A9(qt(b*bO$b&wZ8=NDmCF9A@9K!ZaEz
zw}+*D=Yx-sG8QI(a9<4Ays5>AJ(e)J7HesYGy#)?(cP)o??0T!2sz7imMCWV=l;C_
z9<Fg*dKO0}<!tv^<G!QH2U3T>xCZGv!R#pUK&(U1YKHWGx=%y;IUz^2c06Zw+b+t;
z1}-yfDpkP#f$2Sb&tvDL-Xd^%I+M$=P4I;ril3M-LfU^ZXHKK!IgtJ*&oTEdfy9xq
zo8|!&^I&qXa%Y)ETS=&BF*2EQs>iF?c0W|^Yb{qf)Xkx4EaPrEb>=)J;Vc2gnwn$A
z>AU((S$7DV_D}p~oDpcP`f-Vz`G&If@_8K%%1=ZNDanREF+E}QOVO#UP~}U-8ndJY
z;)=Yp3QY_zGAn4Cahcqc{%i8fXNt+%zSRm0m+qR1D1@rYcvUE$6q~HR<ZqU#O6ej~
z2R>z$f9bxalMMPy?QH`z=id%ft_!=V+;E%SblcGiWt(sd69ZXe#oiTPRV>~LD~7C`
zBWK5d(!_&RPWG88tD2VRYL$sXs;atWJxW*J>&j1Ey+?8JhG5lI&L>TmzxGr~p6sil
z=&7Snef+-a(L_nrouxNT<&zv$FX{LiH>kO&u2251B7L@3^^$GBl4`$^X@q`+@|)Su
z6kkp$Hd;0z(2VEue3Q~RYtwnlt28+DnanSkwJYXxEmNHCoTBjd6Pww7Sv}>({kEzn
z7MwTh`Z7bl&$h*6#fSTfh7XNR_pH(|_3ExNXPvW5DXpc%)ZhA_>Vm?j%1*0em8NGt
zH#x8{-E?VJn~M4icN6RK8WY2LEE@KOuF5*1qN;y8{mf4??NnY}+h=@m!cm2mYgwxE
zouZYNGPNq&3vV^*kyKMKQaq|U|MzXBE59>ULtKt4y#MSXcioFm<#*3SW%tR+X2v><
z%5&9s%U&}-r}9FCOJ&n{ZZ%$0MYCL+pUS!^1|~&b_mz1VE-HWC?58qo`gcV;?f^4>
z%~OgEhKh=HbG?k0J9e6{^3^xLwSmXXcr~Y*mh*e_D{MFARSd5xC|)g<U$a5qJmBYh
zMb2s+m7PmNEHZ9tDfDGmn`eFBqA0lA!2Dh6PV*KvZi_>YmMW^*9x#8gomth<P)Rv#
z`vjFg7YfbhTZx)(ce|+kd1t4|9@9{BmZgieEUukanl4qZeB@Swd7;mJrC&T7P0bY)
z73SRcQTb!JOgUcVr+m&~J`)4aST&pV7uA+Vi79=3H%Ue4;7Yk=%a_XAx`-)d+n6c#
zxHg)lzj9IX{A?pv%6&kEaR-~Sn)_ze9|>zrjmkq*=kCil6?w8#wX#-7)pxtIs+(%E
zlHUPNv;GiW<pmrJibX0?CbOB|n`$OnnchCS%!J{Qn%YE*5c8Gn@(TM@_A2rJ7FRNk
z<Tm?c!lh*6E24Je_<u9+KNX6RVuy{NC@xU?`ozchx8Y6G+d_e+C%7gk-m&d7b15!T
z+o`Cos&z|Befvc(lZviiCNu7fs$|SHH{Dg;VwTXiNp<P`DJr+7@TfeT-ewZ|-A-l6
z%vmPpyOb4I$V^c6anVxbTi76PSe;>N)f1)Q_jRw@f00*87vcleb}r9U-2V58{0=2?
zrQ?sZmBSV;F#X-prF2o}nu7L@n+nfARVuH4wn_E2*-=wfTS>J&k9Ev?tyI-Y61r8t
z>`+x}ex|Q9)w<U7h;gOL%*TI~m92Zt*cktr{fJ1ka58*u8oH=gtw?vLS@IKmrK!s*
zlxD4#Qe0HB!t6@!Kjk|&@2RT1Z8R_b{9VCcH`pTPaFX(td3ol+pBc?tnA9vfd2E$$
z|7J70bnm$uYw!t`p8lI!KSIx$Zxrn`i*$`r)hxen%JoUs{L?C1E&b&$l<)DCs&xj-
zn9D9MQ~B^E)O_~SB}$7DMbr|PWvW!aTBLL|RNX9PmyjAm^9OaE?5~Ouaeb-=!5b9c
zFfW%c3~E+s(B7i_ntPU+f*p&>f{AbB_C(vLe^r%J*}t|#y*7Z?R9;R|J>>co^Mh;?
z)l1ba)UTLmsQ<mWMlCBc+jQQ~LZua(Cn&d<9kpONYh-$;<%0RtW4nz{>s{A6`P$Ip
zpm&4Z1IN`W@BU9w_R{JwUBAmqxjNQY<IQts(`Qfl6wh;gGgrO*OYIH+N^}3>g{C<h
zewwcMxKv^7+-@`OTV@&;pR}mIpH-*x-OJ8==k3WB)t%Q>TEw24C9Iok7Vf;qz%^{L
zn!DyJWw#~A%<AmC)kR;6nyxIiSG<34r~0(-X{z@hPgm%?eaqZ&>s<NM?ef|gYb@0=
zpF3zIq@7Uiv*wZCZg*R`q3xNPyV`Mc+1n>oXP61AS#MgVwNI~4{UFmhb&H1^O`Tj0
zt7cjMFyvqoQgaJBuUfXENHyt8vDzx}1XHG23`+7RCMnIg*{Of_kCpkhyfo9gB3`qD
ziy2fMHm^08Jg{9>g0)uV+jA@BBRQ+gbe?l7vr61nJAUN3Ip110#RKBPrYcig6qy#h
zH#I!nWmcKK*{t%MzM@CbJF|7BkJP5tv#GIc5K`?wpkQ>+=#6nUSDOm+><#+=k9V6c
zIbp8l#&=oe_{}#;n!EVTX5aQ#3Gps6Y2$TJ)Mm6)bF81F;1P6E){XO(p(~HBqDYO1
z+5^egs;r$X>IHno%6(_^<R=`RrZgpclA2%NZ{tflTU6g4IjYKj@Pp?5+pg-l=5y8N
z8XYq|H#Jl>^S6`HpU)+#YOD6CJkzRDT{xLd>1CdR>Cbr!l=G7&C{_jd8YcddGdJEp
z+w{z2X0tU0{pzJ0JI!wuGbpT_bx2{F^Z~`+j*HBWioH<U<;bn-H-*V!U&<QAJBKft
z&e_nfSmW~2?D(%;X8mVc%pW$lE1hT*H2;)luGXD?T-o~IS{0G)%*F*;J4~K+98=lo
zve+PAV2LS*@O!OvgKx^Q50<E^h+j8bF7#DdVapwpyXTK6p1T*TD#2~0yum0$G1kJ*
zIQ)^Xe9}y7wa;N$O5YpWRa+fA6?OVU<c~j7R-SIOSE=%Nr^%iDs>%{Vg7On?_NnCE
zKCGnpwO`G+bcK0Utc+@hjhy-Jk}TEOTt!un?KP^4U*1>AV&XLG{mZGcKHFBgWs{)k
zokT(N>#YpNQ}u#P-f&D;yOS$w@lr=q_Pp&v<t>3C%0{O`%q<fhC|%g>uNu3}*!=2?
z8H!8zl=NcLG89h+Wf&Dc<28LHC1&=sc!Q$9hk=>By}Mf9<TMqJ^YNN@!d1-n^)4}U
zxG+KG$aw>Eb(7tu6%}kc2}&_4r^}gCnHC;3zxk>}WqWwC5r+_;;;REMR2TEwDqBje
zmOnZ{&a`({pyH7xJ+0Qq8&qyEPg0#0TB&sW#9sOD4Lr(wqFhx&cFUW`S@NhXv~W?0
zT{BC`ylA0n`4w@sCz;*G=KI7|q9&~|4BzohIj!lg@^--_l?`n2s=JDWOr}dTC@z~~
ztJF~BVEpfox9P8}R#Vfb^Gy~SXs92|J7>Bs=#gB+ReR-*_eF}~>{6y#!39dW4ZSLn
z|2oa;i@(YLcl=^{rb|cR$LmnDvmV@LPyIKTP2vnu6l_#6dpuWORbq9R^2|6FHM@n*
zCf|QfH)e5ZP)?h~XfVOf*EHyspH}^66J`JN@yZjPwwgw6`>FJ(H{Im3;vf0L509xh
zf6Z5U>RBp3Ct;(}f%9FeZblqx7xJd59GR!5+TQt6(c}xOyu+_3CA~BHsvVUxOpg38
zP`UdxQKhX>Of7mUliK1af3=DxRueN_L)CMdH%$+Tg{f|PzFuYNTnp7>F=dKkOw&!I
zIf9kNT$xqwe+w{^Q(bPh#5UA4@y-F0@9jU-8dVpXd|G9zn0|{z$-KBtUh%xS=?|rJ
z<;rswsxd5AP293B$mx8_F=_3(rzCZHrpXuaZAMxlji!QGp-N|xPMCy*SgJjI>aX%C
zKU2wfLc6K0@B-5x!HZS4I&3oi`?}I(#?w7&JA<Q?6~E6?T|dRu_~z}U%Cc|Po7Qnj
zD~Xrbs`kJ9uCy~_vSO^x9aF!%xe9x3_G>JB{$9b+W3$SF*#!!UGuRY%n@>=V6l+!T
z>9IGF?hR0uJaAUb^F@TR(HAG>y_}KiO!=Co51&-2x+&~6iBbtx<?#wvWnq}9Dp$Tn
zsqDL|X-|!dO5V}{<#kWkOy2L?V!G*kyIJ4*55{3rJJmu?_?qWTD^!$g>{MvF;-YYK
zHJj;7P99~w)GF2G>ut<}s$&()^R&z!O=DE%Gz~O$7My6hKfu{+Q+~LzlZud8gGZv8
z`PWXB-xI6Ucjn$RwKtt>GD9d-Ro;osMBFsXEOo7~7RO~@<u^GSRqOMXn4CTGMR})f
zpJ}XHfl_%(pX%3b&dRCNt}0k=pJ!5}5T?lVpHX8&uDO!w2UAsF@3RWGPF+&)+cHJ@
z)2n3V!@(y^HgzvjzGY!9|M`u+^2396DvRQ-sj~Xjo6YuBR}pzyVtlA3U*+HPjmjly
z$tp^RUMf9N>@fXXYN<3uFIYbPS*O8+wZ7)@-)0!^u-<7}-?LujowB0ErbSET<uu)t
z&U^n;JRkPNtmxfbMNhRb6(Pl&=E2pk6xJnv*WVkzM*iO6#YTZ=WX+C#d2Fg3Ev0b8
zX|7qweLq!D*`oZ~QN1&t!+2RTlWDEtZRO4X=9rw0o@pAGETJiNbFQ-Z{ZGnUN|u_v
zcxt8GSHWm>Gi;;6rgu#$2|so#yh#5nKk2!UF++H|)WWsKYOEbB%716GtFn7qDPES{
zEmyoHTe1CEp7O-WY^E}7r<9(2-L2?&YrleQ>;+|wELYVD%MDC<E7aB9=N>fuSU5+$
zR$Ny-XOW(|^5fNNDNTW<OU$*EgYSM-da^0ZMC;2-lX<4mMnck4jn8eIsp+|Bm)V<*
zzH&mYhg1@C-ze)H{AS8CPf^*{)K$YOP|hT4s*qCmUp~WN{`YE+HSQRA?^$KMc8{U)
zi8=cfYJaXVVRo_5Sgl;G?%8=-y`%TJ@nPk!Cc#aoRZ5E4&F(JoF<pDMLvvTp3N=Ol
zt;+hbGfh=2gVmpQE-{WvUZPmZc|cwH(NjgG_8ao+*f*J+^7B_{C=k%FnQEt&wZ~QC
z*41yy4?1qkzfGT~tR(SS?SNN{scPK;)m|$>wGCXm6goS5)ML_4s3*6qGrj!ZU8U|w
zmBl)f0+k&y+$zmmid7`eaw&bA)M>V_qD^`A$*D>jtKH09cNdwjo9koxg00c?l8BGm
z>2pWSetn%JpLSxZ;-Q-h75CSkH@oZqQ7K!uMx`^h*kqbIi{dp&730i!6GaV=K9ijP
zBBqk<mrPw)7?g}JxtS)K=cs!8nW=nW)fcsSyn$x?ZkFa;s~#vHIkwq+o{Nc@)SMKp
zJ4HUqZ#L+wWce|f7x5`5&vkP#n>6K`{GmN7RsJeQD!p|Jmwz=+)_lK!jN&gRPK}9|
zB9$(`OH@gpYou7BeN^tn#zduA2No(dUNbZ22((a2GFq<CYNDcG;BTp9>=3N>*TB^D
zca)l{*Q`F%=yGS(k9LPuw)I=7`dUv?%DY%+T6ju8#pcr!rKSFlOqy5CG+V3v!(=K)
zgDIcZ0#(nox#sqc-U{qLf)&49bX79F>tUwOqpGZVKuMKvdbWA{J}xC&p$o>-bA*&W
z?EhxmEGl58m~`DF{@^vG*Ax28w9-t~eyq?^(cBcID($k-m}Mca+1{@#sy}|Dm_GAK
zHO+aqPh<504i$@?uBxAoikg(Xdatao`lIpIJ(m=Hd3{tJckwBE#8@c2c@k!7wlh`!
zgWGadR`FFz=cSWXbAq-gHg678$ol<O>3p4pa!IbINt9~0a<uG3>B~J=6%-52mAe<L
zRn_5IX{z3uulBI$nNgR(1+~9xD%A{2Jk>m&1*$gl9yVz|ze#D6{8FVY-JcBFr43A(
zIQfnLhQ2jERb8$oRNP{=dV9Wn&-|&%H$tW<otms<dY;u)sm7{HefReQ)7lJ21>eZ^
z2Haf?s%3wij5ingnjBi6Z<e*_roy(l^(M#fwWwP+Y*6cCvzFWL*ke-mv&rPn&a=w@
zPiL8Iy4q-xc=?y=WQHZGpJPra$MDQCn{#=Gs^{W(!v*076ke2`Q;TjAQaY*rOE%$f
zgHd9Ps^a2TCK`b+eyQ~7x2Vq$n5E?ZO<11in}+hL)_zq-z1yb0KGmztYS^#xK2Bbq
zf3cBT;nO8*yE&aKHm%54E0Aw6OaJ{;?dJV*H7Vl)HP@|Os>+OAW{cD(DCM*iD)tz7
zn_Tn`u(($sX!4}7+tgwITeZc)3oYONKO@8Tf1k2kpoFqAFSo_JV~k26DLv})?DZCZ
zu16_$i)9-1a_?0wTkdCk)_;bXla-2DlhGvkbIga#E34bp7YnUdOD;O5so}WKRA=Hm
zGrq<@%AXf>n!5(On&sNA)k)L+p{n$3wNebHsfAU4jM|L0W+ON23WZQLCiO#V1&Z89
zb>$wvXEc$To+01p)1}D}qpoW6xnKRG>jx#<j21b$ce|B%)32)9Ca9XfIk!or?9D9|
z{f%doW=ch=eM)<yHfQ2u<JrIXG@h7w8=Ww!(kOT;rg2kVL&HP&nEEk~yGGwKb(QT~
zI8+|V@|ajviJ0<Ue`~zO!oq~j$wp_Y-g48-m<jURx(}&VJYA@w^k|F8_WMdI-KJWa
zJZWD|5B`6k^ww{`@k0w~jjMA-O+P2hGC3WmXLiLmTuFM@SCeizS<RK3gEW-;C+JAH
z`5UKZs~UI59#FMAdEU^p&E9zKYF@P&JEy8=-(ghQ93f$1F(Fgq(YGB&JTmi@PVHW=
z;h5g6W)ZEWs8qygtR17Jcq>pyM@`U2y?COD<}Km7Dk=XrDCBUjP;vJAtzMzJ$4F)C
zO*QFv3hEYzO?94rU!<|@;1><!tYEXXrU|MiIQWdF=LD%*A6HPdSka?ee~VG6f?eNK
zWqr4@s+g(Lb+dbhi4Gd(Z#%Xb`;|{HY51P4wsEz;1;-^Bg=(gDh3(#k3X|HR%$nm1
zl`0i2Rc!=x%`eU`Q_R_L&ydU3NJ;7Y7sF15bEYS^y)Y^KHB;&4&wXakU-_ze-g8p1
zU`@~{nmNnx^ut2qDfQ~A95GU+(&o*kyw7%N`+c=l-f8q$bx)a>*=^=~$~Pw#85JK%
zR#LrQt~z^(lk%?qaK)PZtA-1LPbp|$jZ@=0aY2#UT|sq4oTLKRid701B#o4r-PM#-
zEhn1%ar&X8?%^o6_I;a1X}*_oe>9JpN!nAhAJ5}e-k;{MSW~!LW$Pmym5xW7Rg6>r
zDL!sdH8cF8t!y%tS;;D**6b2To0*)Pp7D`TK9i*zMKxZ}4KRO}`CgVOQB=um-c-f%
zoqx@mtF9_0KF(J;aOj}%_HG_U5!P=8C-|Nz&YRw4<o)rhvCgqaMo(*2DgIPVGF1v}
zQ=RoTS^0^IyT;;YN@mFuE}MN(IIp~U`9$-bWyefiVi#$Dy>d@kPnc2VY@ek0oIE}i
z(V1N)IX#jJwH3^&=VGdrL?iad{!{fd;|z$ByR?-<qiB-3(z^6VDwT>m6|D2j<R?1_
zDLJorrSz;o(VUC*zv5}Tg$fN38{|zjjg`W0>8P?VJZH*xJyK<r#5t1$wJMdB9D*uI
z*LqZ{l7GqXc{SJcSjlXqpQ;O$w2n+SmhcfUFLz`$Ti?=Xl2&j?ZS6^Ri)gkx@~8Vm
z6_@_lE$d#xX}0Bzk<yX*Whx%2zs;WIw8;I~^wgyM&m+ZOF?&qv7Acq=b(&_T|6z)v
zU&?&5?*}7Q=j1gj8!h^$9yiO=^!n-!lO};s<-Z@)Ory11%u4QS>!ugBDTQ@SQrRh!
zZkD-<RXO-Yn`u<mMg_;r-6|Dox=Mx};tJ;_EldwGD5|J6uTx*^-KcP;BU>f!#ZNi6
z@450R|IaBkJ=mZa@k`58B6y49ANjX(``ZpFnCnef^1kVzYW0NQM6uUa^;OnKvuAbH
zs`Ii|s7yHQq-x{;S>fXfZ_@`X{>oB)_mvC39W+-IxNc?`v($XcLj_Z1_LFMC4;GlE
zKekX5{q$bZD|xN_>10FGx#xV812bb)V?2(S^lTB9{p;swKK;~7rTU=vW|50LO*P-%
zG23y2U1_PXmFbMF4r-g3@>SklSJx5w_ruKU6NC8%XI+)<Te@aXKen4O>M9s0&8|_N
zz#FRSaALLT(;xSge`+i+d(G3OSS(PWs;|6X<)!mY#XDlA=2dzEitla)X>`wHQCzn~
zS#{d?HU+Jz?DCtXZz=~gZc}nTlwqdHtEgOc!CTh*0h4l2sju?&OaIikB|n(PZrr7!
z8lY%a^6je1sRv~$2d=zTapd<?>^2rMlQ%f46rM0q=}_GU)8N#_raPN0Od?YEn`BPS
zQ+3GKG~bggt5B?XU2(6outNUrD6{ehu}Wvx9Z~u2;9&NS_qT$g-9n>nNw*aL$GREG
z^)56C<!dp?Ouw(_f8n_4@rKLb@!wU8-zo3xTWex+qSVw{d$x-BuOFs!rCiKz1S%_f
zEn-u?VWF(*6*<S`*Qu$>7Rg^tSna+jmPxRxDlRNmN)O$yu+^E}tXF@XY|JfJwV8Z+
zisnT*DtAH-$h)0hqQJLtrqZ6}8cGpacT97_3YBWaROJsV?38zOKd9u&xInE#=a`Yi
zw*=L^dwWgp1|_Q=-tbFh-IF3!Db7kov5!$E=chI*o2=<pE;S7?5oPQ)-K9R)%;-9s
zNm;G5y12p@lee9B<jr*UDcY}@ARkk}VzTrwm$HR~o@)6`F4L4RM-?{CcQV_lxn60#
zONQw)aaL2qGXZ8t{<J7{3bUKsU+JoLWr2;#=i_b4LjDg-^NxKn*<Z6yW!tej6X$81
zX1i4kH4Y^5E6XhSsA{5aWFlC0NV%i`l4($po6`BpB-MS2Uz81{Rw(kPeKUC`u}R_a
z_9l%dOD`+rESaOCv5P^$y>+sJ?Hy0$=?Oa(8B*emb0@r3TH#+S_pZl8`Q4s#O8L8^
z)i>wdFqxM&T`l=2x0%~4R(09#xoYzLJ!+4g=czWUYM7oBZc?(CutZsF(>ycw5^3|e
z>x)d*30au-g>6t%VGS}}dGLq)x}0;$@0ZL~+WlDBY?+d|GIRe1^@T=~X3;)N6<%DF
zGMSh*S5+-)y~*^-oyIQ5ubaLqzoIbn!z0t=#7*iRde7A!MSCiBsIE1=JITpR#%Gzz
z-w8ZsMrJ2Xt~gv!KYj45s?F&MD$)AVrqhq=sdX#FnVvQXQQT@KsQzB9Q|ZvNc?v8G
zWz8BHq7>4OpHjQnVXS(jd78TX+mlLh#pmUD>z6Cn30zZU{mE<EefFJ-WD=8Vx$bKP
z-ji)=^J0|LC51apSEz4P`6wxGzI8H_YQxlY70zvMRc<FMs`M<VF)37$QVxBzK)I|+
z!EBzKyV?KZ9^>$}i%n)<{iANUBHy&@w1k3`9gDKlNdcv$LG7li|JW*fO;A#`^9VBg
zXR}PUt%cFp&PPzO&_=-I-OR1VUQ=hAB|QyQuo2}qRSDu#J2*2yWs-Y~YKj22S>d)i
zvxM8OD#2IS%@*%AH*ro()newDrY!IARpm=cv)Q%olgi6%-A!12U02xk-A#2*!g;0t
z84e0eM<1Jp_p~YgxxGO>=*S;sw;kN7O+8x`8Q1KRy}4XjnTsh%#cxTN+2g&FlqV`r
zS5E3)sqWD5T&0KUs#-(N9n<|rovMuXhfKKD?y1_wHma^L7FMnBwo>adoNe0WTC2n@
zQKF<|yW04&gsSOUCN8rJymcmRrrc_uR9>5P&tR6<vhq`T=j^Vm;NE8X?0Bw{5^Jy8
zqTM3qws+JNpE7SZz1aIe;owm@(|zZUn=-%sXF5ltL1F$o3$ubPQ`L^VT&_At)=vFK
z$}iKbliH@n^VC#U2DO?ny-_d=S{AGkpV_LiK;1%FKl#3i^GrUK$GdNtyvWZ{T+zX#
z_F<ubqUy~9vYTFCF*%Z9t9Xq)OZ5!b5mont+tgInrYc7-W|W^&WTsSKn60*Uo`hMB
zSETCp_ls03<bvd9S4ydGOxI9lk=bV&G2yMs{h|rxi<N(<lqu{}Ibb8M%D|$axM@zW
zS)K%ga+FV%^0uq8<~Cawncq#$HaA;+!PGA)MlFqv*?e37blI6XD;2YHl;oS|cAKsL
z?xmFVOkDN+<6hIyr+eiKb_AI{e|bak<9t@LWr;<mlb5NO|C@hZk@2aMX=SaD+U+!N
zm7{w_)m;8@n!98yHQRS^s!E{3e>0tHQs&DG3bd9?7FGVH8>?Ek@|dZ*=VIlZKKkZ@
z$G<D4U-43HJFled@Nc=oQSSir)@>Gws}5;tq;B9+jD8=X(%fkzud+5$!J?E~xoU@x
z!ZY4_Gp0ITB}G35g+GT+Dy~k-R<c=jNiF!4nepqV=hfe`FE&akH`7S}`dEFd;~Vw&
zojvNmU4@NjEBPz$I;5?V%${qUE^KThvpv_;HK5LT-bO?1d(w=is~Nt^OWx^F-EQKo
z!e(e?B9Wn|@~8Bl#@;t6rd9HWN`HzUm~PiTp&s~5+tg;?0Ta)kGfWr6>M6BNd12hq
z|5W4PHbsq;I&s~@o*zuMtyyi9d2p&~QT;cA!&9|PHr@zRV@h*X7paa{IlcI_$w3=Q
zjbN)a#^>jYDrIriY4oMOQl0gCregJyM5CxTlNIjny`i;&Sy25>`d^Lz8>XoQ{XHoE
z)NP^4oXb1Z58hs2B6D@Fn#!k_YT>HiwIU2NHMU)urEzG_QPcFqsj6~|z8Yj~{;m4?
zLYpf82L;td2Yl3W_iQ%N=ulO<zSc&`z(K&|M8^%YvgQyIqvn++kF&3-xCjVXyg%G1
zuiBiUqCCf6`AHw6S=fJO<<yLsYFk1?%xz~MQM@QJ!<f-<g@R_7jR~X49MiBRx~2>j
z&5EfP8_l$&mZ*ij+oziHxJ&nN1DBD<q8ekn?jn`PJRgj1A9`;xdEGf}v6_CBz%BgB
zNfS<)PP?tEVsPn{QE_LRqEH;4+J`g?HJ8t4<&Ll(F*>y(Qeoq4AGPVaXI1|Nol>)G
zZc(<f=2T#7ov6gSqg_oc^PX{NQ=Y2vf~BfVVGoqJob}X0@|0B#S9h5FYH3h8e|WX2
z+=e|WR=;Xhqy^@vJZp1N-7fv!<oAv?rA@7zDsD&4nSQrtGwYkJY@EO@Wb*m#FEyd}
zxu!QA1Y|i^bSuXkj#l#dlW01NMN;|PCVf@Lh>a$;PcO=UwCXXEc&4VPS@F|oqXvVK
z$I)L#<ts`R%68_PY%yU}(@3pVx#F^4HEvIhiMI%!Y4F@|l}WqpO{W_jHMuG?L2dCF
zdF5?4uc&kja~i9;E>m7}X`M;-Id{d262YprsVkI1KQ}3q@9r~YobW>7(x*TTwLkY&
zJOTw(cdRW|5?alzuqXP2Qb=OC>VicZP2%@=skqCpP~j`=RG9TdMs3o0YjrX0-KPHM
z>{SAf^q8zDk5=Jqf2N%NW3tM@mAtBJH^rMCIdNO*{kDG!PoJMNdYpI3^yK3yrnYaq
zOe#uERe!v>Y<8~KUNPA^TKQnwW+lVBTTK5gIj&T>x>-f{pS}6?sRtFDqy0_SZOf8V
z<@YkxS-rs2D<#QHU$H_l{3nlD(?V}mLya0`&N2lxJ)3w_Cxv?^Hw=|kT%x`k&&oYv
z`pj5RBjdA*vTTBw$`1ic(_;du%4~vBCccGM3im(!P;qVVSA3-CARm?DVsh>BCRMh>
z6I5AimZ(UrP*e3*U8%&s^}6ijoqbCA+agrA?s{XYs2H!J*yy6-^>4XS(*1j?24|&J
zl{<P&o5J-}uiV{WG%M6cwMuukN?mK0>c)+Kl{Ti#FuCt^M5%uAGsQqAWfMPT8ME0N
zcbG2YsWts@p<Uyl4YT?0hqn|AKWHkYDBhR*dCbtP+bv$1WvPPd+*vGUVz-VcWH4!&
z?O1ND^lrvW)6+{eO;d`U&5xFvE5&@)HeGh>lWOjUH_DPzMU-aUm|+^@z0*`DR#T;7
z;X)Htr<=xC?b$V^o~cj{zqV3k|4K{q;-y!V9cLSwswbY6Kk^_&wff9S#T&9siZj-!
z8eLs~LuK`n2dcB)H7WN<&Q;Alt)jrcs7Ow=>yh$_xpS2#+-5aXQoX3;67x>}c#WQ#
zYO=A4cC@Y9jwd@zmaE=WX4vrDL|uE6^7##&%726dR5+$7D!mTPGD~CIq<F`(UGdh5
zIHQD~3+B(ZrkP9jZ#UKZJ6Too;|cSpvzui_g!d^_?p-6lM0~MX<AVkz)p#itZ}~>^
zXP&YO)2wfqyOwq+o@JYEF1eN8g45=#c|{bT;<hp&i@mKsRGRFhl}(mvs2ZPSHf4Wx
z*!%<UO66Yfev{;!81oIzwmMH9@G0jr{#N?)?v44D7w*cFov)a#?QE8_>Tp*nSuU%#
zO2|;YVs^es;Y4wT-7=Eu#(O3!y|mt`()_<h{_EcWx$8$N6j?*<6kpD2H)(O>S9<WN
zL-vrsF*T8C>dN*^b!u@ZYfY-Br>mYUyKiDyvt4zYth{R3jA&K%9&M#vAqi$1=2s|h
zSa4cl|4T8$9=la$5zk(kZoJ86%GBVcy4cIt{EDZj;{6&=CEeF@O0#_h%>GB}E4lHC
zt1(UcVfLxnRdE~ZO4D!C{1rppGEMThv&>qaSel+R*HZd=|B_kg-eR@vawS!5HCEM#
zm~NBbv)fIw&)TSD+w3rDmaj46T3ex-QIez5cA;M7!JT9i4s$7$w~;SR3{?dbxh#5B
zg)TZN{q8N3|Dv?TBxs_g{Hog1Y7HlDD5bskRP((St0;atM1CjZUL}TYp2`nqs+iWq
zEmzv}(@?tAvRSFh(?QuT@vy3_?NsB1i$&BTWB;0#dRVE|%x+P=x?EB1QN$S~{w<1T
zQL*wWCu^Q63w4E<B^LfO<LKF8(Ue(WdfH)vT7G1f+1oG0ijr^Ylx}9;k@w7-U@GwI
zn{pr9Jym9>H)i)gE>*lAw9!H=e!g;>Cy#~E>_+oz0TLEb>q3-$4m>xjSn^P<{@{KU
ztt+3jHf*snFTV2DynA1$s_6Pfrt&w1EUFIe)8W2&TzQj&ml}tfuem{YxXPuyz2-~a
z&QVfR6;capo2mAy^s$m`+C9@+!E5qI_1QI)O(!a*Ub&*Gba}3V@!ER|>Axnbh}At-
zE_R$~{Dz%Pr6AW;GBatV`jSN*DoU3U)mt`AF*{%{t<J>p-dt(^MD;Dd&DEFAR8+sU
zX02NI(^ym1qj^f(+Uk@K|2b}<npbA}#81yWE_Awa(8{M;b1zM|cvHhHJL~@%l{F?i
zm9IO?nfZqZD(5NrYP8*EGjpluQndXdV{Yrkte(H}yV>-N&&H+)?wW4-rlqh!Th@&8
zi<!nk>sEE$<Yt}kPKD;ea^)80t@l*UrSq7l?pa~F>EZ?h7M;avn~z;r{&PIcOin0B
zJ)-4?@djNV#Y3sP)RV$}l)E0BmCvhxWv-|eCztw7THBw)Qf*bMy+&*GWo3rjALOL9
zCMi!k_(UyaIh#e>t&^&kEyUD5ADF3?tK6mDsC-(zZ~qq)MgBdiPak^epIaxSCi~&E
z>XA?7s$Ww|)OOZ5nFyqQSCrl}P3cnF7X5%^Uh|_45~d$7=$ai*`l2!|zt%kEq@>)*
z#s-z}=6YqpvKBK1(Va@H8*Zt6T-0r@y3Is!-;~9s&ziO=`mEh$=JbB1*|fjf<~o|!
z<n3ZEn{5wysCMZwv)Z04AC<`y#0~i$PB-Z)2vPa)MpEy>m3Jny8%4Bp_&%yU<?>Z>
zUEgH(=Dn}V6Z=LJ{<q1B2X|Pjg?TcloD}Smdtd#<sLWegas5<oHPv}PRp+L&s^1Q-
zSMF!XlG`k=rBr%zqFNh=fH7N6i|Up?r&RyA@Mz7*wN*dQuB$$&MblJ}BS$r(<&YuI
z>pIorF6ULmr?;wZ<6Wot^4(nH>TA1{J)CYS?nx*!%;djscClU6H2>vaQ{l;->THLm
znoE61RmfUWpjceCSK(_!gW2X2*2)#avZ|MUI-B#m$|^oijWhevlBOi^Qo!74^;)y<
zw~Nh%uT4|pzRYNzQ)r@Af9$d{vtWzrr>;rH)l6k(*_q2#<~~|xs9$=<<V(aUZHJUO
z%JN>DRplppm>=GMR5{-`!ep;ekK#(>IMr{bN>mj1_A73hY-i%v^Gspp1u?akvs;uB
z*B7cz@BFAx>aa;JUZ+p_q3{8vIQ7#e+q%4zjl-|XvhBA~JMh6x*)I0Hn$zP6<~Jg$
zRIT^FHC0%ery5`quKHuqTGa;0ttvOKc9{M(cUFma?^AyB)Z1i@y1H2jN4M#p?DZyG
z)4A33kKHtv*5X(4yPc?fZQ6FFwF~>rysF+PeQ_vKU3l)Hx!-?Q#l>6hn)Ju<EB)V)
zW7?>E(8T=3L-Q?Kdz8+!7npvt^;Yxvm9A3pV~SeU!(C>3PFtCkiEUT;+1zWoDyYNM
z<Jx0QO`jy?A1!lLU%dThKEo+MC1tXradppGMZT=Js+s?<Dfj&kR#>uC#dPQ2-zs^`
z>ojk&Oi<yRV4&)rTBWr0O`iOrt(TRnZhNaLmUf!QxSFYOb~335h`Y)!y|hwQQA$H?
z!Y>n}w9Py!Q*UiCN=#-^nQ?!f@+y&Nl~W1ws=u#i81pzgDXK7VDEWP`GUf<sF}-&x
z%XI&wLgO88HPu<IHkr=4FiAE`!$-L)?UrI-ps`6Sr<zjwgiaOyCv~RhlN=RFxYf+A
zIIdLK^FPAOUOCL{ua=3~dd@lWvVBEn(P46`G9R*)>)vv!PpT0&UM6j57O9}1eBl6x
z;iFl+rZaCY)!z9tNO{eV|4Mhg&CJC8M3hhGXqlAN+9~+IJgQO`)vVUP{*wITYuk;}
zS3Xo&Z@FBxEAoKK<h$moUH>?gtewr}_M075ObXXiP5-ZIe8xjd<?51hl_o()b<Tr~
zYF)yc)OfgKjL)#Bss1euGd(grTJ@siER~*x2C5NvekvAC{9>Y+ZKkYW@=$rRj)>W>
z;83$deQh(Ampe>ORjpQQO>r<?{LxrmZu@%0C2O|JZPIZxop~o#`BRmV>h%qmOvFU5
zD$L{7Hr>7XuF|dlGN%1;EylUmHkrP;ovoyCVS>r;4Mu8<--fBowhB-T`@PV#W4Ekn
ziQqDo1yYkuk8ZCs{Sua;k@hA~x%@%5>OZX$Mh~~`Q%>OeYg)b5Ua8yLN|iC6S4I8!
zK}D{V^``YT|K<4jYBlz@>M8zwDyX`{%v*u0E>fZLXMu9P*J35LvVLQhJ|*R&B2Q)J
z9=W3Oq40z<N2j}bf}?=x#>b_qulSiv4(W!duIYADjl42lRj0sAng7px)AHCO%B$0j
zl{KCmGfLQQY3A0k#Z2JMS>u;YwQ3i)%rU!iX^(<oP=~^|`8(zF@}`?AuZd9JvcFdK
zdu^YY!t^Y~7|l|%TTaWAjPEd-Wdu$&-Sk1rZ0F-hWyUwgW{;Ue)wH~uR6cuNQoVL0
z)bzF8caz;qgH@IMx{MEWKQ`OhP@%oSCs)~`t6J4<2aDN%g~!UXB2rA-Z4{L_DtlF%
z@~V_q^tmanvHNa(poUX!UiMS<%#}JyH3prkJaT3V@e?}~7F^z~+`o5$@*P$llhX_v
zl%uoDWaqs%REg1DsA8FSNp+_1SyN{nZxxe0IffH`3RGUkZ&Kc*Td(qz<%!axN?y~r
zPg|50)=!r2vyU{`RQ1;E^=4TU=CiV9;_sHLh$X%;e_yyrfqlYX#oSM46(7n@Gh1_G
zg3{xsnJSv=W|&{J?oepSFg9uBNtX-JP%(Y^pVjQr`|oDv{j!SgdmYWL33;kslXOwO
zGXIfs(7Bz)pKckM>fe8&e9~FVc!S$jlRJtmnicO_l*{JHsA$PDo3GobqikAy%xIS1
zGx<Z}O)5(mgBAHiycIUDjWmi1VV1wU+e_{EM=s?Ab0*bQPb(CQ-=30r#$lp#R@z@V
zMWWp_qWzANY<`ns&cp}G@2{;_-uf^|Rd*h%8GDJk`ssWgll!rA)P*wi)!%H<QI|Wu
zNbSRY3)9;VCMsXNBB3lTG}-9tqr)b)1|25rUQRXkyV|VDbSd1-ws5n&V8U^g2uVSe
zY%g!qMb3RniaM?uXZ$ajW*dK2eCm6^*pKCr8uPz0V?VD9lg-83On&8;D%z+XHBk<;
z)Oh=@QayLU9MvxKOC~d|&ziWOeWP+jN!)bKnl4k*b!XL9eO;mU(8*I}>-B2WXDSZr
zHVT|3sd|SLEz}OG8#S>i|9GP)zv=FDlY1v8DNLQ(pl*83TJ7UPI}NV?UzMG1PL#KP
zv{d=P+1F|vveQi!1NN!P@8ec0_EuB)W?!fNT<oMegUwwt>vL@?qM`@P>yj!|SP~pm
za?R(e+$t<reAC%!_ITeU<$`NJl(K#Pm`$7HZ!V;|%<OkVp6SbdZ&VAj_{^CjOcbYu
zrYS}^>nNn2+hjJG<)G4o%dIL?`97Ootd>{c-6Ur!m6)Z(#a?E*bj>T1iTn4MX)Y~S
z^4q9rdc&<iwVh#watQAMMcF(_^U}mqW~cc7E1&<#Z0^Ch+U)<DH|k54pHUJ&J71;!
z(FwC#Od`trm|D$FEqtU9(6UWsPj$OeQ=Y4Qp17s?@!t~_xj!9K|2c1sVqwB_<@n%L
z3f`J8<Y!#GqICN7Eu{({NAp=#vy~bb$;n5}oFM;MnMt{DRhe4fr}ZZHmpZBb32HG(
z^K?;Nl6pf$FDpRR?MSjx-R=Zah54E)5zgO~{#+F_c{pXWStwhIsm}F7ri)L^Q*AH{
zGM}-gS+UUel!C~ld5YXRerAzVXDROs(^2hkh&0bU7NU5r;F`$+o9RlX>6)gc0(xey
znNFq$+69%_(vr=@a*foo?`o>tH~y;>>gr_D;UsI;JL$j5o})ia>^%>eY;s(xF2uM^
znW-;FHDzj#$wl@@%KzJXOr+-CRqU(uRXwnWPkE)u0)^J3^(Lpi_bKE^EKvP+(qBop
zBvAG3q-ly5sv6}}(leAT|2ZhXX!>Pr{dl|b`E?iNmjAe^yfo>#a>j<!s*=LBrvGPU
zs?A=cXtd1og4&bj8nvJu{%Y-y!c}7nkC+tMO;GZGd|s)CbC<qykf7=QtA9)aw&<Iv
zFDq7ay<Tb7F7ZWS*XPN~%Wh3m5)LUfoe?Rhl(Df(efgRVrvJM66%;gYnpp4pr?TZh
zqe(>ZdXu?#7MaaEo}zee>n@WC-&@od>|Lv7-PtGiXTH4Yk)%9h_4e1w_s)GWwtZM(
z^6UFlRc*B;s@l5JDjaER%nTN6P_>yRV03oJOZhn`&Z+&9w^R~Ov6hdyDr#gtXN{uw
zr^o7i|DLLt-Ktm5JhV`WFDy(x<%gc~f3Z2L+r6aBJ_Sxv(NkNa(z)}sa#Wd#+O>Tr
z)OP)dvJl)_t~QxX&Sb}tr)tsc)oR}7iq)>~s#aZH_t4a+!a{kkOQRy^oHfQ0dJoKH
z8J$hPPP<{68of&`^_8IIA6<C`w&K0Y(}Q%C9cN!KzqZ;}X_IG<`h^3GmdZ_LitU$I
zoA9QtRCN-cVK!AH)66hF#3Iw@uflgnNAs+!9qQ9ItyimQP|{Ek?>AG}JkjiXoT5ta
zd>ON~g$GPer%GxwyFXP8)t{+6=XaTfxPi5r9=p16_vdN_)hZ@+#jl%{Y!d&=+>~N4
zsh?dS*Y}!N(|3ZhYLsP{`n=<Blw>aO$`u=ODeq`Ls4Ak>W1dpFTIG4!6_q?SdnJ>4
zdo}Nr?`j<C=ZsE1;m{Cs4KSMDQLQ19B&MO3FRwBC#zFPvMRJCjf=iV79rh@HDdsWB
zQao(3F*nTEoK4y!cZItS-^LQt>9&0edvp)0a`V4Znd{kYvSUY&vdu0{%_#jE(=GQ8
zDXrG~Y;2Mzr}0KC&-4KAZj-gVCT7ktvhsG*ubZ$G%W9?v1Z#LKnWi)QM1t|Zpcvz>
zoDHgW^|nUMu`7(c`pwnSW~@~I&2v%NN<Y(NQAvfys+K|{&$h!#Wv@4A%-8Kvy|s3}
z+@*u&#zEWD6^~36&`GNZRNozJqIuWqwTg$Sk9_N7MU`!J-_`G5<TSSZbVY5;DHZjL
z-nu%<)iX3&XMfO`nmXN#=W4ntvq+l(hi#PV=OSfQ{q|nfzIjSYV!t1n7<*<abDvRB
za#@*RXmg6koQ)&WxaQnlldWHM)sFA)G<V>;AXnG*Pkupgrb6twS!VpVCn=qBa96#*
zpvk=Oy|Q9iN0ZUz`tyn^D_Kk*-CS-exzy0C%xjL)T$vm*$zyJ6IYqWA6FR-sZ+y!&
zzVlDPWbFf8)lE#ICcI$}Ogy!VwKiEQDa&M>P`z%p#BA!dOUfFIiAI}0*ek^tm#F4m
zd#%Ly{D9(%mvx5XPJHs_kGj<!t^T8U#Lq!h_t#DNEgJ(Bq9)lY`xx{qCA`~V5|%Ho
z>^0+@tOI+A#wF>E%5$d+sIkZ;nBU*OP^DE#)cjK7ZWT}NK$X(ocPi&tf)$s!-#0bW
zu2KGaTS#f@lK-ZEjTFq9?oBp5{^gf(ydsB2Z_i(|8Hr~ULWAxraIIifWaUgS`~F8k
zDfvf@%A$_1X4(!J3g@2WnLJUNqFBE2ipi;v24f%F$7a>l%1YIa$4!)LdsGV#CMpZ4
z>Zv;MKQm*~Xf@+{#-Q@VJJxLd;diFO+axuMRG5^%zFeZBCoN@e$GAc{cHTjgV<)&3
zGwj$@k8NO4R_4%?bA6Lx<|GxPu;@#<x{3M`#cZ)Il@F7)DAZkAA(!dEt=v-mLP<-{
z&wOf#n-W{Ath@#L3)#aH?kg$0(N|ScWH39G7of7GUftx$&ny+a_be(awoOuTlr~oE
z__4>d%1uNmd)s!Usk~o}6ze6;({{(0H7++X;kmtEE%<eU`Tq%X<QE-hR$LSKLRyly
z#O%*vZ>63Er7E)?=$jYbog;sVZI<a2X-UPFz7M87w@u8FtUs9bXY5rxGJmDn)pzcy
z#!u>$&0>_*`~EhWili}^Or17M*>UqdlLPy{n016X=~{VxR=WJyTIG!28MDQuY|7rd
zTTSzmg%sB;Uaul%Ua!0}-Cp7OVI9+JEoO3i9h21SQ_~f0K3uNSK4-D~5v?8as<G#l
zxT{5!a##<VaJHOQ?49E%yY9BU(s_G6WveVz)dYd1#-E<4s`~POG5cs)s2cyFL&f)=
zv#K!XYehZnd8W~8o-4U6xu$Hney91<^>fW`E!t_mNrll=#%sOW(rq&4atEg<TuE4`
zIIE~#)~znb^vl0^<>{4ys*fdRm@Jzpr*OYm+x+kS4@x;DAIysOEH{}pf4$lAUD8Su
zl%z~Ivze(mH<hSNm~W)Rn9pSHWj(<xP$^JF=$WON)n`ug!yi-(Rv(?N{O6&J>Zgw#
zriyBuDq1ZI%&rUFQ%tMQQDy%1Tcv8*e8qfqOY^0|4DvB^wKN*iFDM)>N>ZKjGDyK~
zL#6zyZEKa+G2Bv8OQ|utc#1>$!--1Sk}0_=Jej+cCp>ts_AAoSH0S?rmE=1eri`i=
zRd(zzQdv;*LM8USw&K5C0cNw~#g$K<ja1tE@{md1EK{?)KhBt%3j8&xS)Q-D`mdw;
zg|07h4)fR*C%8;i5LupPmcG(oDM4Ysin{L$v$~W=3V%Pln0Btcr?B7{pNV=Er`a1W
zZBw7>Pl_VTi_P|FpHoeHbWb@U-&@fvPsx<mxyz(6Ypsf8>~7QU=?lz?g<dJ>7e7!E
zDp61kuKsV_8a-S2wxxsV#xQLq9X%%1gO{?EG7nu*INNa6bbqvoJjaZ5wJAwnikG<N
zs@$vKQkbsDrqK6puhO;N|B5?2mYZs5yDKTjewDra|GVOoMnUCmzsl97=Or6TyiQeJ
z`u4i9@#acZsf&!NN7fgq>i<bqJgj!tB>7i^veu<~W!I)z#v*TXO}QGLm~CD%&zP_5
zky_fV113$3Ewa+$QHs71`tm6rbB#GSGAO^BWUP8&W~lL;qu&%%q$|z#EZ?dmT`6W(
zvr)#ByG_Py*_S0s*GvzY{9kUbcKN%R%HEZ?lrNpHFwNh;-SlM1F_lH{qfGY1-8D0w
zpsW4+^E)N;_7AG7m)$dUP1&RD#j)O0A*@;HerJ^GzQ;>dqp}T@7Oy&Ol6P~8{DIdM
z8k794$Ol(nR#8q|CjZ(cL*Y|!s4}CVoKgX+ozdeO2IZ3}3#F?QR8^a2HYtDZ(NbUH
zr)MHFd$O8n<RsI!Fjn=utLCc31$C+|PnfT2z-(f=I<Z}8#r#FeYo0ALGp&1J_HO=j
z(?}&R(-5-_YS!GxOwTp($ZLGLr2ORnA*EIUE;9zP`%1Q+8`LkKnPnOz%dMEcN6z#{
z&OFtce1Frwxtu0_8+pw_e>y5|_giEtW3oy8o68fm#V0;0)UX{kEld<PYumY1WnQ+R
zS;&-kCIx3#tBKm(RJ|iMMdhctr%7ysg<9q=R<o-U*DCsD3#u19YE-%-wN}B$;gjhu
zHdguntdrEro|>sHE1RZ1f8!&iQ%CX@LQb_S+nQceRWdznyzs^YmH&lIsy3Ipl}y@N
z)LJ*GsXN9kGEJy{sS@41)2!?}kLtB0vsA=h38@~NF069gk-^0At*Y{ZGxwCAZDTW=
z@><eNB*D|v{K`8M-D3~cqyN1ztt=`~%&_>YR95D#=-2wvH2h$Ia#p66>XqiTrnmg&
z%7sM#F|j$St+eCXWs`o+Q$`PN?=<sVxkyoiKiy>BN&z*Mf+!WiI3>jy;ZbI{<rU0M
z)qANppX@iC-(qeev%W>6Q<6z}_ZcHqhPPE_M?;q@uhlg*ebUUNczbH3syF{(rTVo^
z3J>k?o38CQRJ(t}O+D!JRb}7PM^q**$x!r?nIUhvJzx2jeX5G;NguP>@>`VC^cs~F
znC7Y9cji*P%J)Gn+LPb(*~wPbES4S<r(O3{mz1@rhJII6U6*E~cJb9NQ&(<ZrK`r%
zlqU9c8Fy)Fn>DK&o4M6KHsO4-S=C|o5wmTDAqp=Zc&Y5QDOTRL@4ad5Q6;6bj(uu>
z8Dq@np7^0qm*j26!E!>O&t|u&$}wNFDFTsZ_q7fv@SmG!_ELMQn&JILs^=8g)b7>H
zG39-?)0nkIPQ}k5!T7IetJ%3jl^P-@y(%56B9*(O4NO8-epC*AFK+s1p}pds!z^mc
zUYaUa7k-kvcvsHEY}srz(fY?KyN(@F)sEh(7WpPe`LW9+xyxn~l+<hs)sEghZ2Ie2
zx@x)cT-Cr&|CD|-3aPI(vs9hTG0oKI1%v8b&xdC2kv~<Yw;xq0msM81mguSYefL%~
z#n;c3_E#1vCv87qcEP;Ie5u4w^DC>4m}XmBs!g|2G;evhOa4+*vcm2M_vCJ?^qA$}
z4^_I~C#~vmuEosla)SJn<09sBJeMh!o!x0RQ{cI2aEFKa1#Mv^4JJ`Dne)7A*|U9B
z7V14!$j|XI58lIL=2dc3CCh8QS+&M1^Z(x#X+M7rnt!ZQ{T$0{wrKer<-I5E&2<lC
zD8>IZRpm*`P~QLdi$e2sU-O*p@8r$nHmEOn&8qMyx?E+y%q#hmj5Fk)UsX^pJe8!#
zC@5>@uH&P$Icutny$XwR`bjb6s`<;+WJ59yuR0u4-?%*9c-CQajj2_S)YoR-Qx~=A
zQFkgmY&1(oN%_-HRux&+R%7#L9mWBBSC~57x@p8+;-vklZlZ};eye;=T(|0{eLX7K
ztIrv!ZQQOrf6rfymJVUlE|oB)Hz(E2j>}$BPx&;@bjGQC(<0A4(?Zra3a4|fnpk!{
z)yT0@(g@k1qWf>UsL9{NYSUWr$*Lje(hP6TEik#i`nV>id4hUWmypWM1xrl)B-J!t
zCW@I%IUA`IxTQ`bZbhbAYr=Yk9WVWj?mH(b7)IXLG71q^7oGT9<H+}gDoRJr$RFEa
zs1n+=U7hi#p3$NWbJUuCd{B#S`mUwRk)$EeKTBhZKd+hGlOEM8J?usg;(w@ak#1FG
z>rhZVZ{Vlq`_RC|*gai|`LTx5`KJOV(a-jniLf6vPT9$9YANTc(!rB#{^0UP84Kws
zl_d;a%F2ra%x1lssKgdNQ|+y<qxpj?OBHX<DKO4orJ`^nWs>R4mVQ&i!wzOT_r(>=
zwws&vu`X8olets1yro;WreD=)5$_G7Z@fM#ON}#)TQd?(CYfH<3dmoflCxl{(uZ1M
zvxG1ymFw|mjLiGP6n9wisdaeIRf!TgCii6HVWYYvVR?b#NHy)FS5$q~&#Q?DPf$K6
zF-LCqd>JKuop!Z6*N4X3Y?-QoYnQ9$om-(&ldG&Q=bEhgXTOFCpVk7E#qnIG7r!4@
zap-DSnYCcG%JsFDs=DkVrbSl#$}%rGRNlGvn`XSaVJf@eg^7lhgNfV!gKApw+fA3w
zu#;XOlBj%PMVV4bL9l7i;qOYd!iK6YUR9<C-!77G%;z*odh4dx{qKr#j{~;}!xU~4
z&(Qx0yW7s0Y~W&43%FgS(h@OC@mbt96Y<y>lmGSIDvl1yrt7+#OwG7?R1Yy<QsPZx
zQ#HDgV5GcmwX#+ZlWD?*`-)-Ld{w=ib|?w(?obfsY%#6fzFgsGv$h6P<^z=t%;KuE
z`8t(CES}3>dGtleczKd)z;<5~nUE$GwX+LU+?IwZPCYN7)-_X4JzhA=wEcje%C>cK
zCY*X{DlFyVD#v){sR%FTSLNE~Y%1J&L8&BTs$#QrjS=5L6|+Q-7p5O;C!1XDv{#*Q
zZH`%wu$!Xa+!W<)7YdY~#O*abc-K=|CuEX}$Wb5jg%>Z$|86TVO|R9GyI=RlH2kKr
znNxeJS+mLl#aAIoW^-n{sfzt-R&LN&Rq>lO&va(-Ig{)IKa{uc=`s#ZJ!yI>)K=q_
z=ToH#lkO=eoW5+ja-X)c<~ttKTuFU}iz)wA7Myla7T}DL&zr?<-0Z-iD)oMe${N;{
zDiLS&RmJUVmA>EUm!Bv2O^L%QL^VKcvS|i)vP$M6H<bgoR;UQ%-%*ue=v3W%I>1<H
znV9O&mHUjBHk+xocnGLu)fK3EE#pycT&iNaPq9^L_op(&pY=yf)_Ta8B^_8{7NFQ{
z`kudE<5=M%Gp+S)a{I0uDEZ`PDs=ALVp?0urQFKMtLkngXTo1_P?3Ma8#BR)eoFqg
z{mjxL_L^E)>zMnVTd4T|VT9@b%r~lq-5e^)ulST*yjB~ZwsJEQ6Dv{<=wdcqtKw|>
z<L4?(hOA@CW;Sjr`Mwcm31xLE=lrD162h1jdN(Jj3Z55HdDrKxSm6A@$W7?7;t5qA
z^_R<zD2M;pqFPpHq9E{mx!l!1zm-<KT&b)w<&eo2iOEV#(i;>{EIzMx+b>t8+w8xp
z=e7=`xl2zfmxNRsWKUbE+#R8>99kEo!g@zv>8*;EnP_^RBG>uViZ^d5n7p2L+5Boq
zy!qTS6HIv$c-8WEy)>7fJVVYcZ<~Vu!~%uA6@g|eWhW?wmrJS~I{U@cLivmQx<o+>
z9%*qUlX_<JD(9u<RZ`C8efHZG=Gn+ws4x4hBJhY$`G~iVx{JyLL!F;27GIxfDL=S3
z)5NGt(cDWSUU!~Xq_SuBDy6y@0SmQBxynZB_e^Bnc;!AXb5?nH&PDC{cW$|}Yp$8p
zUhh&oks_$}AV@}8%;KWT`HLb7QIp?FYsa=LFl9+8?wx+m@a`XDrHv<D6$C%!s&oF9
zQEphRt#;$9oN<I}gQ~-s??(DBcB#r5X{bK6s!_H6)U34hg|gYpNh!(&H#imf?*28{
z*w1Os=ILR!THnVs?vIt~>Wxa~eGjH9M)%xOoY|G2C{jApbbh$6vLKVJnyc_$v&&4g
z6*fh(n%Qc0D44OtnKTtlGqbsS!9-|LqmpR%A2X)kd1{+O)Kvpde^m6e_A;&$<1{T)
z@>4NtjxljK|KHT>NT+7w4nGwm>&Yqya+*yq+y7SH-tA_(AV*12K)6?xqqkK#aZ|K{
zrwXTW=^IW3=awk7j|qIrJuki0ly(Ox*45|Ay?)lLJZop1@|$WeqveZUDjmIfOwLWF
zTlIv~KILa-)@n;x9~)FEFsdE<Y-(I$tD&a-TT7K|vY=WIw~2C%Q?1$0j}Mja{W_w2
z_Wdrihf_Y7ZSnePv1&J`DP!nXwF~bB%w>Fk%KlT3RQfDXBHuZc+q7Wob!ErWL#l25
zH%!#RW+>VJW4080S)*LdS!j`S^_y8#c96OHLtW*LtPt~k$`{oRcU@Hx36t0EwhT7=
z%I$3-9c-n#Ah_1_#bQ&7S$scryB9F1aJC4k<*^<z7eBLCWv_*r`E>W`O7emNYL35E
z)n66eQ8LurWg2!cLjL3e1`XYLO-iD6Vrqq3)D-QHsw?PqiKrw$|EawBWT&B`<u2u=
zu|^82i<vai_|L1X*sP-d^@pZ;;1e13o2<+h@&6~PUtDjgKI?|O`sQUT)vB_5O&b&L
zE2&ycSMIf$WASYrquF6GcQcWSdB*?ZA876UG27yKvyoi6`xX`14VRVe>YB~U7u{9z
z-07pSlf}S1_~{dcgw|NoTx}-xZQd_UnGz40>^i#5tfxa?;TJQbnH!shM(oEH^)QP{
z9drE^<}1r%EN(|Ks%lh=n_qVMYWnWPT!W&trD_ppHB>gOsxgcG@2h@s`FG<BH47ED
zZ`!Rcz;sqgORZ5}@7D+OLvz&B*e6PBN7&k^1?O67xa_~IY+t~pkm)^F`DgH5wQa3N
z7MHmXsXja)q_#A?TdQ5RL0xm^DfN4OlZ>Cs?^Eq*>C`u=6;+eTIjhR|I#*R+JX6g)
z?55H0mwT0F=G7>r%-}RkUwOoA@&#!Vrpsz(soT|6<HJnNKXd8Iaq`!w$p4t8++R>^
zCboZ`(naN4YFB1nG&4^VP@EF*&9wV9v(l~N&t_6vgUw<dnwv-VZ<GJsI@jz&=R>up
z%URU~uK1~0+xO{zJJxRcw$xK4^lY^La?5ANsutWjvwi-m)ZJ)MN*3C0mV7)zrTg0~
z<Eat>idQx{s2wU6S5>h!k-Nb6-#GNuQblW~Kvh*!Cbchy%<8AQtCWo|3d=RjzomH0
zrd#b|o2K!}*^R1!=gz1WXiIAKhw7?N&0MW^cF_;x87_IM=AT{|F22*M`l{!+iusmy
z)z6BOO8d8}8Q)i_QQjKNuC!au#aOYS%k1=hQIpObX|o%bSE?_wU1nb9lB<w@X||%1
zdba#e(I&IJcUsDfb_%Ncd)v&Wx#lW9TA65OIQh5I2BS%4Yu_F>m7Qy1p3go<sp{fX
zv)6U{YE|BQm31Sz)Yo&i8@X-LFl*n?s$zEivC)iTD`UZo2|C5+I+a%z&QpEcBx!EE
zZjJJsEPj(y=2?nYw#TU2Y0p$K^^;Xvc)H1?g=vn0@uw)&isSp0w!Uak)m<*8py<vm
zA2nl#vTyW1B{zvSlX(*&mG614ku`VYR9~^6MOm2NNsZrH*?iKkX{x+S=a>pa^r*^z
znWif8<%+69`B@dM#57a6?J6o+?p?~9;fy9ZGd7rBIltWOv0{R$WF?Q9KHqM0?dE2s
z>0Zvt*H>vOt@n~PPgW~XmQbCh>h)LC;?Ks#^1ZFg%{aW4DHa(um>pYZYgWReZm~OG
zL^)@#zFDBXr&<krxr*o;U6tIGn&zK47)|%@xS(>0!^b2{dbXME2?@;;MF*6&sI61I
zd-{&~i#gwwuiAGS-%wkxl&<zvwQ0{E<v9y76z|T{G`%$^Nlhf_q()U_go+}sv+7C}
z3niu0EeefNkt&aQ^Hp2^%r|Eh$y9L_J)^wouBD32&9|zicTLoM>vkBJJZ4boiQ+MQ
zs>Y#W@wQ7@*UVVu#Y28o?br3jokwRW8Z@;l9!_gE&a9YkT9I+gB=gx5qg(P`>ecba
zW}?y@vWuQsE8o&KQVM05YTT)JN3nT%yNb<f10!E4HHGM8UbD`v7ZglZh?&mL>om;~
z5;Zk5Z<l}aG2AT5Nlx`|UXyaSn7+p51%3whLX~D-<*$@-roAxi_)%tZie;(J6`=~{
zNlNFHX1tX#i#lwo+`&+8^eNInVJFuym8T}fY6-g!$(J?$G1gxZrZ`#hy^5jS4i#Z}
zOVx8s+)8iLW#vAqF)Hf6Q&Tm(mSLP@q@hx_w?l=Yd#U;h!!N4S8t16S-AXi?#44d`
z!kJ{kY2vF|e7#V`!&^-?X>EYgz6M#-FGrY_1ty+WcBxop`pSdJ?231<nX1Pg6N7_S
z)C3MKGrigNNoMvpZN*|)cX@s_7t=@6?Uh$f(pQz?dt{P)X1?N{H5W|3?}=2}esPOw
z^)wa}iL<LsmoWw?`5o#pSz)NE_Rl9(W!h0om3e!2nJz4EH;LW6K&3M{-t<YfirK`o
zqFQGs_bD4r_Eh~~({8*^|Do~+8(uT<b^|5p6Go~x;ssQy4R<I?&yO`-aCx_!=wSnm
z>q#6+cD>=MXO#FApExX#=UuI)yg+|}()p!zM%782$}1Ta6$)?VtFD&$rCb=rp)Oiq
zY*KMMSyd~f!$4NfQT4jtPZckV397$@`<3ThwlY;+QK}r0S*Sd%SI+c#p}d){<2$p6
z{D;Q>8|u{7RYjYzFqp|KmseAKQnXjT?&m#|D;hJE+avN+KlQhmS?}joVqCh;Oj4mi
z>5{`Hv(9rTO#7uKn6iD{rnIcRz^rPnkD6Iwk4mFcm&$I>{l@m`*GvjcB~_jO>@{M1
zYi}05-&SXq?RMoY!a}O+=GvKE3=&eYyUlKTej=OV)TA!esrzm#U$g$KFiGgR@zE$Q
z#c=t}>dy~^DKE<3qk81WQH3p+{N>VS=_(tQtWcJ2bu~`$Xjb0s^GJTmhWn~MiHB64
z%b2KD@yVJh@};V*73MOSdZt1}`r|p}1-E)sat?h~@>y<W;yQJO((B)I6n-%W7<~QM
zZpMFgt%=aFDzhoPOI3<=PnrLSiC44~{;e1^pG#4#V!xR~x|H%h#a0#Hc3txwzuM(X
zXC5%s-!(;E>4UY|v$zkYA*E&JTz*Rw(=-{(_FZ&Ry`|=*ob|Xx(fZLE<M14RQ;)9I
z%J;b4jhAfXGP%IBN;4%mNco2ahYI6EE%VePoXW?Nn~XcRyqDkixIraU)J0MD#1w@O
zhOS1<4i+lY7YnE^StF;szwC!fuc?ya`HBZ}Hb&c(Ix3@-H_n@4`uP{9vi{<UiciW{
zsm5qrQ`Va=q89bZ#&ln_wt8jz0b_yfbJXWH8>_FFVWR$`evz6(p`|Ia<qze$W(#Gf
zfLVqw-l>}kT%T!rziOfJ4X-3kMPpMlCzW-I>^HZnY`M>-a@AAObVnDb^3RE`8e)rQ
zn5nogR^+`MV%og$n%cLhH73z1fu=_1mCbf9(o<3iv^8Z~Wu=kTQl-w#5ujw#D{K0H
z!%Sl{eL2+`EdnMB3>KMc@~u_vInuAz@#~+;jUz3lcN&e=t%4OzQZ~vf)=fC1e#Uc?
z^4n9|^4>;jCc=&Gitk_SR{LOYqQ*VNOk;lWALT7qRps6tkWl%U|4OYebgSvbGe=d|
zmNTjqe@{_<TUM-Iv;U;}e#WI{sb-xj3zmwQ2Z+?G2xg_J@K>x?IhequbX}m#Y|WuC
z<&KbJN{cI}nf0|hn&(!Rna^5QXu9g#JJn_L49$yvd{u}$y;I>+vZegAQ!Qp1h1-<Q
z9_UcfU65toBK}K$U+;P|kCPLX)*enayLn`b>CBn&=6Z4qlrpy)o2_TbR-HF{in1YV
znUZgJr+N3IFQ%DAqAJTD=a_Y`&oF<O?XAJ{cB#_tYa3Onc_x|tp7CF4mh)V*DF&|<
zzca5_$=Dj9bWFih;nyQubJyn?a$4fW>elPx6vQtIs;p&aQ(!zWLBZERU3rI{gYva>
zS#wYRQ%YO>Z%B!+5LdbobXYmN-byX+;xXf@OfOYU3wGlZKipL>`ae`D`4OWUV|QGM
z-Qc!K)VgQN53a~7TlgF?-uirl8PAc~rq{OfnjL7Guj=LPXujckzM{l}FvT^8d=<F=
zGMZWLJF2|i%~EySY94deliG^CliE%5HZm&znyX>fe#Xq~vz43Kr7{8KOg{~?W(Qq0
zX+w3D_ET?_oZ6*KrpF7Lh45ciso^+dq9DL!k}hqi;XGlk^7em$s;5n*O~tg=Dyv<Q
zH9ha4qj+Mux9XvssY)^@e<_5AT{PyNeOq;}g^XIvrV^zd|6)~b_azEub0*54nYdB;
z+pA{fwf$R-?ZeJ0Yh)ak-pF@XHJ-&-C1t9l+SlMoCR0CTtDUcVW_(ESlA2L&gIZ8?
zpxXMmajNYTxlJXl<|+L<`9P^?(>|lUtEQTka_=%#Uyx>UVQI12+sDCX`*$stKXhz@
za@*m4rH6BlnckP0q@>W$rCz)Ltm%w-UGmihf@aw#U#mRau4S55zQ@$SJlO25rKw_G
zS*fYV#8&n1A?wxBlse?EP5ohVImp9gsg;3B-$ZT`8>diHTjL384AHYyQ!mS?oLKtD
z^qBloRgG9t6X!;Ig%XAHYCrh&RHnKV%1{0?-RSVPOu6@Q%$lK+uT|>L)~oj#PgYX1
z+AS}y7o+?wd%EgdhZxgtjc%2D8Y@&3TY1$*{}`xs-jPz*<+x*>U)!Ws$1%^umi3w1
zgCmV<K~oyljyRU7UgzaA{aI6}tiM}HiT8c1anCU+^NdNEX2GIfW^+F8P!s;Q!{Un2
zEydv6^~%c|A1d{$Up6nFeM0H_s~&ZQ7FA0{*JTQ(zuL|2u3f0Ara#C0H|s33WoOwf
zUoCsCSme)WZu6~E{qTkLYNj`@tKH+3GJm^yv8luH0F@<ACmHXUS7R2>_C+ge^=8$B
zF-<Cm<>p$XS?j2|&K5F$ICqZ1CQ)YfWL+g?3DzdrS(=MXp7WnkyRW0J5#TSTdUi*R
zx{~EhrMbIQ<Q~N>QEs_-LRE*E*}_w8zsmEb{VIPvT~#v}9Mzc42&lh~e{RU|ltW`z
zUA)n`^jeKeN5wUEl*(xA;M=LbvgeS&^*k+=u8EtKWwkCF@9k<cVNTs=ln};jGLP3q
zXU<+<)7wX0%WE7uq`J+JOZ9wEoJr@oKIL62H8tNgeKzsm-J$e|^Ox~`VHu4L0&1o!
z%`X~Ddr6v2JT+I*U~0X|#wl`|wF#jbw<IU(yzMSE)?U<Nyl2Hx)yt9jMwcyA3|FUp
zP&2u;Ontr1Di!a5UK3aM91US#2E$p+_mt#!Y|^;ObzXV?F)kV2mzBo54dfL68S?4q
zeDG2Kc1u@NI^vaz)VE!-3Wriuo*jRyF3aU<>|}gRZS7Vi^%vUmI$5u}H5dXvXjq3`
zFg*~Mq8g`TsBf1Sqq=k(r|R5;ovKU4yOb7rTrk#h7F2#~+pIJpmBaA0SiTwm^bX^Y
zf83`0OZ3&2aB7;TcAb+o4(U=jb90Zp#5!HGsS_2H8$LR!GE}vjU)E4iWM8<_c(Kp{
z#mG6lX7^@KHqD;wWN!UDTFID?-^^r^jaue;2bDJy*wxyP+%tae^UT;hKuR?|x85L3
zC(I<(I7sU$Ym4$xv#F|y%+JlzSk5Vb+GAp5{c(n(9#f$zi@2atCKroRfci_L`tCjQ
zt_@sjH?6iQt^d}ndNeIXq55r>oS?gs@&~03r6Suz)BBTolts?Yk`t_H)@an)t-Q`a
zO-<)GySZV>NtJi$O=ia*98+OhK0#&Sd<oSCgLR78vMWt1WNnmx$4Mzo3+XW3_rTWd
zn9EKx_cJ0UCNaO&?_GOgcK=PjqEAtVLdtDs`O`5`W{gaEN}?Y?t?zf{8#H_6KV9=P
z>-w@nF^NIUO!4jkQ@)~U=B`4{O7lMmn?)vetETL)Rqj|NrS#x{x4GEt2-8iXoGRC?
z98Bl0-(=P!w?f@C%2g>;^1aHWUS9LFZBEMPQkhNNqPi5fTXU$|pMI{in=wOPzB|dx
zPee<tYtAIKm#cm&xGI*YtmNTRNa59x&z?MA`O7hD<sGjZ%;ntYDd{||l)2$HO{tin
zOZn{!BUP3aR;H6~S*y5LpEZ&?9i}2wc~F^EZGwuJ={m)4)90Gr>fEPjc6PCnQ-!Zd
zP~lm#$#L_{IxcFMbbdIdCXvr;F8V}Pwl3$RLQ>od*)0dv%+kMQDs_F&RuOEFH{Zqm
zT)uRokl92wCPmrXJ!bi(MrOUHZ04PYOp0;wTg-a+>{Q==ZCBp!epzkNZa32lEgB}i
zjT@9JenyxW9yx4w&%ImM;<=Ub%C85MU4^8~P26miYjdMb|6W+Cpy4$~<>9xP$~_mX
z<?W_8o5r7Rlzn*7Tz%s{XGI~m8!D$AEfr4o?3PnWX;S*CEU09(Ce9@9Q>o&WCMns1
zXQrwe()!B!SJ$d0#>twd#Br)>{@!a=X%?({`l6Z2d>L!i$Bz?~PWPTR{b)QvspHIQ
z<&F3Do8NKFHQVdG!#rZqCzIyZS875#?akRfzL(9r<fd4x&L!`0a<}O-R(<8KYdlqZ
z?<<=w(rQyIdM#&e`R%)s^8qFED@-OP$E0k{@}ll3p3)34ZD!S0+ap=8vihN=&Nm-+
zbJiDPW|Q>8RbFa^n|W2JnY%S98sy)<sl2N9g33{YX{HYULsbs#YcvZA{-_xFCPMY`
z^J^+^GBp(4XIYqU;W{t-<t~@TT(hf+8nV+>y+1Q5HVT}O7kZzed^z%=lJv<5X8pap
zlst5P%3m(AR$Z})UFB2JdNsX^tR`M5r&J>2OHE==o={mmFIwf-k3T9eezhs;#Ppi|
zSoumx$vjsnkh#u8^5splT?-n`9(fp<_RBS^e%z;N{(>!2fwl6ig4=gBg@|pLW^ya{
zDjk`5N=0jtgt@lgWQ9;sA+v;sF$#PaJWXZ=HkiFk;Ws@v_nG3mwHwU%-X2yB*!E6Y
zrzb)2{QLVRAH~H?#8<9TnS46i^o~udnU3)XHMuW5%2BI0RllBFZ1Sh6Q`x(@*R*Ew
z1V#4s|5Ty^9hB0TZYY#LXEIIF43z&ice&aYsjrF`9$!+KS+huCpU68|ADz`ot9p%<
z;(yhetVo`u)D{pZH@EG(${jBuWiBQMwM!Sb8N5Chr|R%X)3CIuMs;`oGZo##sjBr;
zIFz`Go|x1{$SG^jYgE<_X);y0scU*`lB9XX>aE7%>b&X?gk?>ebPvg@247MTd^B6`
z^S#@~#=qp0eJ>fSD&);E=7>mBWSnr>Ed4;R(ro|bX6!$-OdcpLHkosOw$g@fbJJ5s
zMrtk$E-DM!Yt{SM)|#Z2Z8ynny{%%xz-}`Ac&=H>B~zUlY`V&t-P=?j)U=x_D;`v4
zTw!F|^&?2hsV7i1{Ok->htfo)8#SLzE`4N@yIXFgv3<=m1$7M_)kAmE6&$($%d$<d
zQI_4Wro{Wa(eQZE7p2Us3o_gjcc?ymK0(=a@eQ?GtN4tKmP}ULy=1Kk%M}*&gKy@k
zJu2={OHf##dQdmhbh#m?a=7#q<y6l$Gp5gNW*+9V%}j(mO~YqyRI58-W_DJ)Ro;8)
zHf51-H<YSY95bCNy<KUY#|HJ*q;%76`OOOd&T*J&7|&OIymOD~tKepn{z4103}rdR
z(%mtpJ0dr!Z!UVQ7VmAUSTgCjY5Sftrg=@zRpL*4H#NV}YHII&S?!PYZPk$Gbt>j*
z_l-K5Ow=4^W}0~?-B(<eAgInUWu~$UgSvv%&xxk%U--#if1aU!@Ti3<`>g5e5tkX2
z<#s)l_Yj+)e1!F)>edr}Mo%K|sC+E@tD;z>ugdkgTCLntPW=a)vFW9|PgNMr-OTv%
zg;j&aH>i|4N~&sEDyalHOfd1$WmMi5_)&TK{&S{brD|rZR=H*cr@x!5Eq<;(sYt_2
zdCF9U%*2~YdEDHJ$8wjLK4~^q)-u#m%?*5S%CjU^cHahGvy)vaO3%Lbn7+Sp)8vrr
zI<w`xI!b3>Fq^&*;ZgI=^-|gPR!x3qRj}Da(N@#T{pl*%>!VDUvh|yuoUmH`%e&u7
zY`$8mRR>i~m)%&QeBSG`X^f?`(wkWks@C()E8P_~R5TJlYr4qtr{Y`FaP<W@Hz;4p
ze4`@T9H97RjjEz?M4R$o|5g<VgAUVVi#^Kk|A#7Dvrkl4{cEneZRHBJS9zu;3Qeu5
z>({cFq~Ca=x;Umob?rG7)kBLc)p|oFn&$1lqO?PLfs*UZ0K-#)lg++3B$#QcFEWX@
zUak5mZkAd884Y>q40n}+4{6HApWRK3BX%lH+uN&l{~?e0(i0sDN0w%oWj)YQT*_Q+
z=5yzs*_-3+W^DgIC_HTwGykSHRqbuea@8|Q94a4q&YHBbyP92Js-^NeM#fmx$llCa
zNkQwEQ>RMFOc~`OIZv|%+rBElzR_ly^wml6eFmpmL%*9!w)YhI#{!B*1s^ZST>a6e
zx^Bt|)%~VB)i$NXDf{>N%3n3zsU-U;RZZyNRb$O1F{(3IW~)9}wpZ2cIlFq!RRgtS
zHLFe7wU|}ag?Y>hgP2t>+3r^HnWC&J?)pRVqs|+%ZCiqs_P=#e4r{n<Rx+=}{K6x9
z3$qiqO#8KGswLYfncF9Z$@~9lRY)=ElDpJ+(5zc{y;9Q?B~{h{L37T+h4MU8=9>45
z98&0?Txss)>tWVwf7(pXm`zDfbG6xLO?I`qIbka2m>JY0w*{Erdu(R<=JGR@l@_&T
z%O~<%1f<N=nYI0iQu91l)xs7*v&?s0%K46F=81w#N;1n-RCQmoDhqCxP-I<eWNv<3
zQcgDJtoqtd&I)Hvyi*Zf|3jXY)l07Vsf4mdd4r;fZnWv&B0r^V!p_phE3;J&-}h3!
z^R`Xx=N>nMHmzgoN?pQ6njsb%Z$Ca(Uu1P%{fS$<`iF<I#%lgsl~4U(RWaM#V(ja0
zV6s4Tm+5S7MkD80&e|?jKTY;*+9KaFu}77sc#TT+EF;6o#<9xzwtqEVx3L?p=G9PY
zo0@5s$aO)zt9^-Sz}Al@54NbAmR^ljIFzhpYPI*72E!#KjrQwmx~3jajkdnZHLKaX
zP*r8Y2SXEv`6d$2s<gc~rK@}2uT$wPjx>G4q@uAP-_MwxuT4p&s6pd$pOqTt)>ipr
z22n=SD!(XfX1T9r^F&yEebyI^J6=mwx|8O}-)PWMS>3Tw{pm>`qq*;<su|RMP!s2S
ztHs41q;au-md2e^$)@QmJ5`Tu;4*%{{fFxFsjaGS@2RTZkoQ!JIizHqF!iHSC#Q`P
zm)sBI!;5Xre3x%A3jF=cq-24MO8P4Xb2%$#ITeRE72%`Hl_Mqynmu7zuGFhBQ%&Ti
zjk#=ors9i?7UP<Y^A%Kh)XYwv&oiym{%rR3XQ9Hf;5VkyrxvMQxwT!DWo@%=C&y=_
z1HH<|*N+>j$gSfr`dxg~c&_zVtz+-Esz}|<S6aXpZzi@_NX2$(iP6Hlg^C{21l8R1
zk`!kv@0WdbAkVn(Nr2qz%~w>H-oC9W#CAdL_0);VDzEuu7gVt+*>g6k`S|6U*x$=h
zHT=3nb(5*1THXW^^<Pf<YQ4MJjU8vMQCT}j#l$J^sLBMnxhlyjdsPIiOjKDnJDY~i
zWmDewNJM3+jjgFv`!&<jvx`g<87ocH7d=<An0d)m_jQRv+_NO*AE8Z3#``-=CmlYm
zwEBXv>VMxerk9Tk%Lje`YN8*;pm^c1kBOe*W0SD0nkLQ1H!5y-P%+KW{-zpPoTt+E
zF+)-Rp}1-Jn-#|9O+6}}&7VzDWUEd8>=IW`)q9{6eu+`luiew=<oBh@{-2Fai@A;{
z&d>E#%`oax@~9VAJe?zKs#bYd-bUnyy3>YhDrcGHRpTB`QJShRukh;ZXQhOaT-9{F
z4&w<AC#sx_?oye0SVCF$gtXfCipOfk(lKT#EE+1cD=p0Q=4Pv82=7zgRn(x;YtN->
z<)vVH{^kv(4K526WyQQqt*54$$}HYy;&@@P@f$}a)#CXkW((#olE2+qr~H<Eg_7oy
z^`?5ObCs5SYfy<>5^mPgxlZBZRaTR)tr~KRE0j!X!$ggF42w)x6rE96xcQ~Y8J8T@
zhK5$<H)-i=E8d(oWhyvhdcn;~Wj1@Z>A}ce(?B6}jY)q6lvn?;RarBCwQ1ynVrBWg
zvL>M~?<>qX$f~-rZ?<9>>vXv=k$zM2ipL7C@5HJ7Y+Is|Ey<udsdJ9fziW%-e5NNT
zRquCFm5Y}(v#+*MQT!OD67`TrZr0J8s&DkFRjpNana*PLRn>nLZz9*3ta{`gx9YS{
zJ*qd1qm>>o?=xv>EKzQ})2h@PcEd#3R>&-8%|g>hCml^yH)m*k(Aa2JrnOe_&zakb
zUc7VVpD0P0Z8*!TJZ*u!YHOmjsoS+v@=^&hrpfk2N*-(YOfP6oGi`WVZoX{KF{R6Q
zx0$qD<yD(=L|DaP$9csR<yWSQ9%YycChS%T?6)#iUfpUUd#X=Gkm0bBVqmzc(a9%f
zk^hz`cW2KrHDq0?C|X&fYAPPBbno3<#mF)t6L)QWwG)NwRdf2w6_@mys#d>>SBP0J
zD_?U+PUS_yEM?b~<z~g(d6Yx{x=6g3<FBIAD5mn@n3S6A)~lw|LWEQ%e%@gk9P&>&
zrsKZygUT3{3v9a-{pydHS+D=AxXCU;sYWf`G_P@rd3MnZbB(=~X5D80ReroMvRJj3
zUH;wPSqdx8uaN)o$k9Bo<eTD!o4P7zzSddfHc2RK{CCJ)W!X#x2Crvk1unD9ogVd?
z^F7?IsK>Cwyhv7FwXVTKx!jLcAuoTAnKHkbS@9b^mGim6rr-W{n!k=bqj^bXrqa5r
zUzD5o?K5kuy`tnL_1QGbR$lS(svwo?T+Yhj&nL?}e{L~dy8Mjt);Tq5cWhrOIO}Pt
z2o+f<>{NLm|LBsak^`TiVvTaV8P9<@r3dasa!+p_Q=W3|hf><L_p0Cb&os@Lo205b
z$=&SvjXkOYYeZCSw|c6cm2_8n#`)Vc?!BY3|2zq$txE4qw|!GFJ7l@i#OJWHN$0Nx
zs^_wM%%1!bljpInQqnZ=RQfPW!R*B0R3#o>2{oBdwPusfgeuAk_82qWEmv%t&TDew
zZiq=h-v$%y37v|63YpB_2sEkP&(Kg^Ai<=jT@-3^E;7{Alc_=_&^64|Ww({t+@JH*
zTjxzyx$`SVC4+67X~`2~6=xfDlTS`v3N@3btM;6osJQRmdihP#A543`rzmXd`lrU!
z!l3--oUK~v-KmNd#SQXpv3g2dwDpvysJ=0+S6isGXWJ(ExgWME<j=mP)W*-D7B|z`
zbZw=8n(!Mhv+mv6YCA)wtFF>kSF^i4RWb78MpOAar7F|j`lvL0*kN{4)5PrJv8fgv
zZ`(~y>Ks%H5o0t<$>mex{Vb}KJ}E@LMeUzy^9~u6n{v-o{lD~?pJ=(Fz_;1YLc0B#
zGP{MKc?~DO`Giy27WN9E${neb&3xPLsl}vSQAvt<sdX#hta(S6f!VPo?J5@^FEBBS
zx3V}TFQn`6fJwzOxk+t4d$M_rSg?xJ25oa)k4Z}JI91io&Ei#=z;{|{#w-Ezrk%G`
znsT@`84~mrr(T<^`ZVj9La2DUf<<MQ%38*2%0F0~%(C)sDwnIb%U$)=S7*DSqw-jI
zmm1HFV$%pQHTB&^ede=P^sBo~HCE5KqN5)Bc!OGP?;VpFzpa$6`(ITmvgxyM&NeaI
z7t?KCxI4>um&6q<2D`%+Gncl={jlAtviMZL@+-aNrk{!#mCLqyX_STunHfzDRGfbB
zrumnhEb2YAx6L^$*v)>n6q*Tso3HS8`!BQdIs=WBLUrnM`CD{+Z$CAwdER5;#kF0f
z{Mu==Y>p_?x0|;Z9N#ljEp^&ECHDHWW(T<a)R$klX!2o?vEqNW?drx~qg40YxFvr|
z!Owi!rBsDjZZ>Vbqqb^}g&rEa@(wB|wF$_(R5dF9Tl!FK&aK_%=2H);K07a_#;|0i
zR{xfX>Vk!*)!#)*o9?c?tr|aPvB?)+Ni{x|YpT0Ai&Z^2D%B3evYMS{Wm8&bHc#>W
zAq(Sk>Q~L;K3zBUbu~3x{P?5Fxie-K1@R$rH*a>TJhLuQE|TdqtG;(uX(!)3wb;mZ
z^GSy-74z%vnQjV5Rb2mkmg$vzJF^pcs%BDA{))EiSDC%+dZFg|mP<`iFj4+_o3P0_
zw=lD~%^y|foS19+KgHUVU%yY|%g2Q(Q5}<&+p-UuWUY!<G5^G6D#o{2p^MEz%_25e
zv2N}Sx!OCkOeS9Jm9JlCsLIv(LG{Hl2KAVvI^|zeC&)L>Xi*9+oT%pP(q?krrbkuu
z`(9Nw<HwqN`Q6l4bp2H2R4_8Vl2xFp<``nqx}`}~m*cF;x{xZ>m^aOe6-wt#CC~3x
z?wZW49J7wi`0Q&A^9z+RX3utYn1!56R%f$Nun<=|qtNWlq_l-kS8@HMnP%7SiYgav
z6I4A~+HBsjLS6AT|7NqPOPZCY`YkZKyM30~gYAaq49h!}rW>p?>ptnIw(1v)${f!S
zCDob#O}6>8nYk-3R9V|zX6jNWXqv;$uep9tvhw5D=c>PXgw4)Od#h~zc&=$*y`IvR
zBPpsS{)NhIx3?*(n4B?A==YS*Ypqu0yx*+or+iOEC;O|y133=)^9@bPi!~N2sl+>(
zzQ3WYyzbv(Sr`A^$_|_6Dsi>ds@>chU_M3hh02QQlg)p<PE%#okX9A-i&Hf;y05~X
zwa#?S1Syq6TnCjBd1jgJzCYKjx@M>0mi6CF)M}a3_jz=f+bIgkr?VF;bH6cHz8)fP
z{<Y<)QtzVx)rBFC%uaL8SM0cwq4y-TS0V1(WuxP92TeG#jLjeVl`57=wwS&wb5ZkO
zTB;K2(yfvH<CvL@^HVeHG6R)=x^m{1FTOUJbm@b(%;7$j7v)P-Qct}zU$iq<Woh#q
zqbyNH#kXr-s5-vSRc_S(EPo_xs%eYSMzJqV30m_HtW{xZxu`0#F<D8rDMrrTHbA*Z
z)<boTe~J0>GyE!DnJy}P2e=j0c&4bn=2lk|+QwnLakYU;W`T{d<{y5Q8S@2I9+&y6
zOna@O>N)q0iRQ{2#qUcOD|V;%8!!4f#q{)dVY5Uf4O5<V0_r7(9cDUr_Q-z-@Kx^M
z`=EGOKFXBk^lGIH|G6r+8J3&HNt~6R?B8P6lKfiA@|}p;o=6chJ5d3%a@)Cz;tT(o
zGM}+mEkC(b`C_?-QerT#sldKslLEc~711eg4b8jlO_h!oYp|5xP;%%#q`Y=Pvx(qF
zF6GOf`%E~apUdCeaa<*XdxOg4-S_0BuPigFdwWs&RGgaXybV1n(<Uja#!i+{VoLfc
zH?Mq>(rq0R)#OPVOc<j)RW`c&syw<hOSO@kS#6cZ3Ds3+EsU6#NU6FRl$ox)@2)D?
zzePp*m%i%huO}5|+wz+7Raz?tec@K&5M6G1;mI}A?*Eo%&z>-utj%1jzMw$Y^mBH)
zg2yRi#TrXz*~Oj(#%ba6l+UlXP+cxE%dk?zOrd$FtI3`_49Z&?^i48k!%d0==9z{#
zbt|Pani&0V*HHU;Fiz$9(HxcM*6ya$^>&$fDNR$py@lU2)BK@n#zJxJuWL>z%a=5&
z{`)Y?BxTM*<)j}QO=s;}qxeE9LA9bmM)~TDrHWQF#msKA?^g(T-=sO`XpX{WLv_{a
za7%@+y#@+%CUhuo@?N84waLUJz)Mtl|M@7{^>dn)x)yXO-^iC%Tl%-jbc<<~>Q2F4
zQzy+B)oj^RRgR>Esu_DZl#)JJo9>+Fp)%vRq>A2|7}L$&PfeBlW6dH>B#iysOVt*)
zhM85nC@B`L*HkQJpDyR4)M>ik*-TkMvO@KpO^aF110h9~hmTAT`a3Cyf8jQLwaCQO
za;}6~V1U1}>MS0!8jftWlcIA~l1o(7B1BqEAKwu-^)?bvZ50SL;R^e07ICjxJ4rcH
zd7=Lk)$jg{W}a>jm6N=0nG_#yQrZzYT~%*-r}FOnG{vN|yG)H@gq41kmT2S!-BFag
z->&i`=as_wC9I0Q1xuA@ho>q()nqmG&udY>KQB$5Z=<B*|8^^thc!P`*Y0{``eBy6
z%2oy)(`#QERPIk$qAb;zrlM5-M5(}ZyJ_RxP9^Wf{EB66SB%#l`DAwIwwv*dgde8b
zi<YaHC-GYNYiY_)Gd5Q;|NB?*vVFf<WkkACOMbdaNxp^o-RFxGmKfeOI@6mj;cl?S
z=s}N*>CuN1&G>Z}D=fU2X_me)UG>221m(pQ&8j=L)ti`ctT4M6rJ-`-Be&U!uj;0~
zwwpCX7RD>*MWm=)I_7AW>~E#KT#U(NlFMa<E!+E4dar~icFZ?bxH@N($(yf><+oVb
zsCl$pP&#e8N#)3i8H$q?S>$x1#Fg$Jj8d-tS!9~6ep^YvbBf|ppOXqL`m2;v-mF$#
z@Ws%yg<D5mI6K;GY3yuuw_O_QlS+)#^X{xub5yrCZQU)UTzW!UiGPij>6GoxrfjL3
z4fmgWXe4VqS#$Z@Bva`XSL7s2PO1b-GO2LR<}oXMGgIlpUnh;{7Zi;TzvNMBoff70
zbl+RGFD{LS|0Ly1&afRal6!GcUiX8N$*0-I8eit+sW-~1sdu=Po4B#xH(B(4yUNtV
zH%uotYnYxDJD~Z)c!OHXNk-)_0WoH`EyC4RF4h@&Y`U#rwtJttW<ZW2x3;C6vgZ|(
z`%euN)_tqixckFd&E45s<6`|Y<(aB)<<@V~QvO~2UhP8p8dKK#-Kz1rf@&XMXe#u&
zcc_2+bX47F*>qE(CSjEvw_D~j9%ZToWr?ZSNJOc8E_|ct%)8UfKDSpnyZ4OJugEfU
zD+gV(wNC72CvEqG7Tu}W9hqw;z0O!cLQqMOVST3J_smwaRV$g5=SkJ6OxRvy{MxWo
zf!}kT;RKzhipv7KjNg5@Vrp`n)$DxpWyQ9=M@&AvO;G(mYq>J(%RTB(9!xgd$lYVM
zXm7f*^lCNplV4|=u_$fP*;SOO?D>ybr7?7?d3d3g@`<wJrjh+$<g8|{P?@#FM#)5q
zSAGY-vAJE#OF5^<uQaOFwUj;tZC1Hhm8&2uvrF!J{4}NJV|$b)ZfP)odfi$n^gXZQ
zj?MB4hdD1OmQL<bEqFJ@WTk<w>VtR_v-?*(RTq8QuhKQaPPOd2u2OyWN>dX)eHEGf
zFUoQ|Jxv=^?wakhJ7MgkUT<paI9GMz8E*3zsa*<A$p%V09vxMjHgk(9Q(B+$BLNN7
z{Y@ffmx5gt=X3j+a2#E#q&SPyg!4zA@tw_vCKGy2Dix`IG`0V0rFOU5O69@QSQSm@
zB9oM@rDhA(FHp(cW@+~K#UfLcYpoimI}a#Jd}vZlP^vLDoA6k<TA;$TuKkhXr>k+Q
z2NK<t#ms6H6oihLYVB`O<Q8{Q+gP8XWbA!eWy!QE#rd(43gWfC%6&z5mF(7wnl$Jp
zDhEuISD3COsldi1t-N%Gq?(ND6w}#}1!^f5>P!?So>N=TU8z=b*iS9jJ6!d?!&Vdb
z@=Z!2hgp?A*!(r#ys_F;Dap+^{O)q&v)dcg%sMZdE)+MDf5taSIY4ZW((T9=(@R%$
zm5yX~swWjhnJj;JP+{3KCqveQ?5cAr42;h#tufj6e2$sP;^PX|Grt+vYBi~Ei(0Lg
zJAb#VxwxNcqo}dT)w+|)bJf~SHrxs^iMTXRb;;H3s{6mlEAzL`F;o1sLv?D-5u*Yb
zdBx1rC)KR_7bw1ZwpA`*&n@FEdi&*ruTIc#s1{YN`q802TWhfrV@#f0$Kj(&aSJA@
znu|-D6&&hSnJ>Fng-OOzzW$qm+EWfm^|J@eE!?f@)!dr4n9me_rnW{pUyb!ciJG5X
zkE%+dqZyyp3Z(*_1B#W7vrHw-nJi*GtWCHt<(vIKa8J!SRnt<&O-oKM>bUZ%cVfya
z@*);-k~<V-Y`WFGAFi{II4iCA%Hh6I_3e$SL4gO2pFc}CQ~d2=URe4>?uf3Vxi?>{
z`u~Z`)t(uB(Kybq$t=%jqxnycugZU395auoxMceBs<U?Erx&V{zK@jrrFbn49Ewug
z>f~d5Cw`us+Eqq%nY}%V@oyK)?0$d8^b3one89~mn%f;TRnr1`)laRzrDPn<A*UO$
zL}``(X;lNi6!Z4|^Hh2zAFAAIRZ(2w8l<+lM^L?4?x)e)U6LB}E2bJT=2mGesu9zO
z*U{DZS$bOiC&yc34<SC~?Up~4rq-S?_SU~(QZl8&M3%SL#Jk^6C*LW?Y|@Vgd5P+?
zs)>7#s#x!yWMVdtO-1{brsfyVHKv!JolyD~G~dKQPC~=(%vuw}c?(V6ZD2C{z~`t`
z=sd$@QH!)@0h7Om*WXDx+(n_rwQ7wfcee?sB`&No*7-TX=v}^y+LEe1^|<*;D)pxO
zj8rD3Yb@_-GIq3`rr001UZZGHr&^5ZQTZ&3`9{0bb|}t{`lPL$9-_X`(M0oE;%k*&
z-COd$i~UuuxcyXjNPchR@!_GG$UbTH?;_?pcV92mP+9&{!}4{#nO}Xfs(+E6spzB_
zRn<8vs*xM}RZmBUC@!}NFfCB*S5~+bsLX7^VDh`{xmkzjHsi~&Zl<cU4ApvPzc9aW
zVS{}AEK5bbI(`N1>KSGeT&5~*GPO``*6K2kpQNB@oBPi2hjftA-6$m^i>&FUJV$n!
z=txgds+pH<HbW>}E!QYTB}{ywI{(c-M(!6m%}hUhRSBE)#B{<oKhus+##&eSRFret
z1l1a%T1-njt||Xqde-RK!WN~(Q+2ApbNZA|Cf!k765MIBsai#GyL5zF#(Qf;r}M%p
ze%C`3`m(1fG_P_{-n~p)De3Vtla*T!D_!`%QrdBMy+*wqi}J_CHfoO>wwayWXruD2
z^Q$=*`)-vxkxD9xOiNTYq@Gk<wVK1M);>^qQGT7$%F8RvzP53gNxhzAw0uFl@xDYJ
zjjnADX2O~K<T#AKC>E`op*Y?9w3$$#xRN1ffl4-Coyqk}GZjp^J@t2OuT}hfh1F>0
zZ)=muveu@Xp4BSuTGwWh^SVuS{q169!>K3LPh9Rd+Yvd}?B7W{<<_Dg^BsmNrZT@f
zbpCt1P@d-cR9WxPG;^O0CKZ9X4aWE4dgb>oVORYVF<<G@+<CIfz5!-8?Y7Bm$PUpc
z3{_V8X8TZO;<<E%)T69&A^D9;%&pgzSU07aXIY+9d}(@Ip<B{kKIYXc#oO!jR3q0o
zn}wGtt3+S9VbYZqp|Z(9MrEIMt4iw27<pBJ@1_cK8kDB~OjEk?Xs2nL;Tf~IbE{3a
zc<GtuH*>0I{$6bUd=9TdU2lqlo?@eX#pE6{vH$u?67S1YPWLsLhpx3#xNYifB5A!q
z@v4W4iSKL`v(KBk%n~{0DSF!UnZ580P}M&@U3vNf2lZulW|$n_By1Y@#YH)RZK}!Q
z!c%6qGs<;$>clEPa|looQC?x@bx>HjzNF9Ow=k39h1HW(3U~P_J-1L-xKkixcDQ4^
z{O9sejSZ}3ib}U;sx0f3RygxYPQJzRg;KqUoYHYEM^oqi?~2R6td!OJX0C9+VU|)H
zhqUSi-!>z+*(|D^6W*Hg7sjeu{Me{+q{>wF+*)%5O`Ykc9&9Gc+c(Ts<~zU1eAW4#
zrkWme&BcH6o5Vkupw7BJ-h6McqM}WUnZkjSQ{}j~lpEjB$W=Z$HCmNx(|(iHB3I;-
zrwN&_?zK_&4moZ%fAwoq<2C2a93B}fecE!t^slI{+Tq4>mGC`gI`hBHH(RgRX_jlL
zt9&+u#q`-L1#_qSmIhbqWL2IlW>%d!af+Gnfj`Pl5$a~E`R^zQRL7~_T02{1=KP6@
z+AbmH=aRCOJ{<SgtWRFA&}h6>RVlJrf$I;0{P&};m3@D3Do?uWZ}y;2P<g?wBsrOt
zqDme7eaeC6tZFMmW|_7ZtW!CxwZ`n-oO3FU+bdOStDmT>&%U4#(!*``)bfJT$t8Ts
zW}l19-bomkc?emWH01v_p1Z0`Rp5rF8S}Bdassa|DCXTLS17vaWVUSC6eU%+gDU6N
z+&8KFwNGK;4OXMh8LEnNG^ZFJyxDGYR@cDfOw}<((ZC#&59}9IZ@m7kTrAI~Z0B>$
zB*^8niBB`X%96fyrt2ld%((mJYq&=gD8GCor+T~E%4~w=4COkNHj@Q+cPLsh3aEbc
zJFK+9iB)mZYXdX=UAYRsb@!@!Hzz78&5~8woUEZ>P$s6})4WpYsJ)KTQ&kJI`E9&P
zS59@xC3SWwocP+Nbhq)C>hw}(quNw2)prY%O<d1JsXnP^R4wDoQvH1}Pl0jm8I$u`
z4a#SpA6E`tHOq9{ifbmv!&}W-JRTYS-+e+|a)p73zon)8vC}b%Hx;eqPc6wdzTRl1
zoU3H0YVGmfWM}gp1^x>xW>#W%m2@1wnQT=yG%Y`-VCGObPwDwbMbq#v7HZdWgH*Q0
z8LPUrXPKUqUSOhK&!IBMX|7RC%tN!O$HaAt)F&!W|M67St6#?K=iwd7AOA9%`bp|5
zNd}gv?z^>4IckN5(rUxIrs@i!iaw`GHIHa-R9G{2j>`SlC*(_8r4@Akx+}k${7!L`
zNwe{_#_dXuY-Vz2U+F5v)b3Q;oT;m3{%x}HCbgMr>+@O69Ir8}Cv?nEJFnHJR<Uco
z>Ize3)2r4BN<6RYlp7Y_GTm|fky)0Hq}l$M>n4`(539vEDw`(kGm^dMcT@Se%S0tx
zv0{^#S1puPJlCmPU+XuTcJG+N^Q^hXZSo6M7ayBsvgqMb6C2O3rfyq;6xg~a8Z%_A
zR`1$$TP=n+QTgt^Leq;iC8nu*Ix0I=ADO;aQ#TFOs?zvS_e8a#l11g7UZTk<XDc=C
zdTq0W;<xfvcLda3?4^`u?e$W)TVrk(*dnVq<w~?h*lknQOW&5MyByxFbhwybA)&QS
z`OJ+AsufdanC#hcL1ohhc2(xWDGH&IooX!;eyCl!{@>)7N3P0I-)OU24&PJ+ONvy!
z?!KZD&!D05qdC~*%>qtkhi#e4vn88N*KFBpx_)-3>DBsqCY);=HRi0DYueFSrjW4V
zf>P`1H;T+G0;YFM-IQCGsi-RKe{ED&&LHO_C2xG@$0bGS7f*~||I0T~ZP{h^Vb2_e
zbEhX7YyM<Xn|n4<g<0*0T9yWbS-Ts9nbU<_WxpREP4zU|jGZeFY1hiVRyKIBNaa(d
zwV6@lE@i#vC8mE)S1TIa^jBrG%u(9@KUhItaii()*=?$g=Q}kr_^&F5?QBpjpJ=W)
ziAz%M{!4e|1&a$*IHH!BeQxYmK9svrIZJ1g`W^ZIDs%7uP*vY1Y<9(Hx~lbWRkJ5N
zk5#27wW}uH5LG?l?V#2!-(_m=)1-9#w3%W@gokNV$v>0Cil3&+cezX$&dyO~y4_~B
zY2iINkAeu5*@g|uVGp;M9$(w9lw8-Zb|><wnePEZ#k;=Eru#X1<eso_n(~UxGYQ~$
zY`R7)M&X6n8dC|+S!%oXtySG~dxxsr4M8)3`vIm~rK?ohc7HW_*r#hYtz)J-Yj=f8
z!ND%&zdR>R6Rdev64|>=GM?{KQ0HS%JCpW6L9geJT;I7hrZMl06^a*cR?XdeM)ls@
zt!k``s+AA?XO{ETlT}(3o}>2PBf`w}cAly(?`+j=_kPJek&;lio#CX~?x|&}vSY7`
zaXqv7d(KBH3-k}De0#>OT6a2H!S(oAvtNe)lxj)>l~<hIZq6_>)J!oU##~vN)zsQ}
zjygy8Br`(^2Dz+htct8c0t$gkPMX$TouR}aFQNL>%fKY+%VPQa2alPuTw188dotDZ
zMWmQnz~`@K&+7LoO4;<9PJ6?t7JV>SWm)7;wT_>2%o<+qGd(p=L`8DpX;Z0d2hCT-
zPSD++xlh?CSXK4b!rA7UyepK0xT?)2{S{P9f1{`RQ8!T8G`dDXqVcf#n>l|K=FITc
zbPBLj<X}CeBE67JLApOeUe58mQZ>T|g;@b}&1NQlSFANXATN9Nj6&>5Zl&Pcn^d>V
zeQ&h6;JW%OH&bKnPbM1o+8(HXcKWDZ-Z4QvSYV$~?Qb#Vy(`R>kG|e*oUJ2ZBpSET
z)HmXvQP=}L?Q7E;OqdTok>_@rpvt~CR^?Zom`Tx<ugbdw|7cVj7MorXZB%ld!C_`L
z`JDQ<Uy7!@J*_4mEoPerIlC%tbrvyEns{F$<*BsBqg%4NceD&lb9Wpw(&wL}TJY_+
z0bg>9$;v}M8eE&*)jvedQ@%cPn#osADUCmxtBl{u-BJ`NuhQ73a#3~dyUPk~`KOGG
zSo)N%DDTy>I3%iGeU3r%_0#Dp7yG0XYM#ZZ9Pr+*&Ziq^vh4OewKrd1s&U%=(7JOd
zSHrqvrp8PYF*CkB3sp;h&Nb0=`>#5wzg_iuzl!R8??APJR~)9!S(-|F1j-d(%Q=~J
zHO(+%yVhp1M8nk7Ox9DS>i=zXhKgocUF`xD@5$N9yzV|`p-z2DU8`rQ?XHS2=liuo
zarV?-#`{!0$yX#QnY8&=nx36-)<h)zmBLvE6|*a5%hhJ@+@rcKr$d)#<76Yw#Pz0D
z;`3GZtD2b}PCjX}d9$uo_moDJ^+zWv$1C-k@U7KXdA%XiIPdN!g^9O#)S7>KsonSE
zmG4pTGkGOgub{u-p(@XdOR7)04ydIU_bVU&y+poXzp0Y)(N?t$YY!XW5-wN$pFdyK
zHb+*mVy%w)ibs=Gx1ZQ+{5D5PMLB1asfWQbm9OjbR7~bHs+^E^Rh^&5YC4sxN9iH&
za^=%e+f4KB&oV78oMfVOq{8_1)9LCfLKsa?pJkHYvL;Wtwk}_3iCdk?^@!g}r%Vl0
z8|#D(*T~x`C_QO5DD81nkUJx7yy#tnvEZA(M&06H<uBU3H}JjmO7$X3gUT{yf3>1}
zb|%g9OigxK^eMl!xoq<P_HUCF|CVXR-v6RJ$0J5%&o2g3)>-qF?F5&aY&@K-=yN$=
z^+%nb((x&+3Jf#WnD$8DP-t}b)Z~o5tithdg6hRHZAzl|i{uxEKT-mXNB)$#X~MWA
zU!}Whok}6=e}&LU6}4`Kx2jgFx0)&!CaMU`NiqHVHCLtZ?^osA{nJ#q?(nH@(|%>*
zbm4;1-&yQRW>HH_rW+-h<{W7^wcPmFIE_P7HKRD$>|);(MZMxoW#-GNO0$}IOfPqy
zRhrH?StXA1lG!P(+w$zZOHI4g{>pF5tTeqP{mvv;E#GX9RHtI!%tNM8<teI1PE1ff
zwWd^g>wXclL$eJ`O^f|i;+kigWKUjhdYt=(>c`&(N=@^NRYLZyHhoi=seCWU+{Dhh
zOyR~1PSr<Ok196n-Id#Q)82H#>^}KLN<6BZ_S00RKK`td<XfZUC-qFWm;aH{;x(SC
zYi3HDWi>{soY4qS;d{xhSheGhDksAR)tR?BO%<*htBM|FHBnq2sVd^WTxCUDqw4KC
z7G*{u1=HB5*-D1nqm`0*4w}sGS#IiHxyMXNSlU!?LZQZqy+_T$+O-r8E)h^_H`pu}
z;>2S1tkyu8WuCIC`qk&AQFrYXc{<9?od0Vn{Wn`{YN@Vic1+mI{LtlArNypSP5ir=
z)p*;4RnAM_SB&vqWqNwLtm&WA^Hp@icuaSld}zApp^KXS31MYs%_}PWF*;_8v>z$A
z7$unfIKZjsw6;LC<M2_XyQ`Kfme?2?&ztg8)hp(p8Y`=((o*j})hWzt<lT3D1>a3n
zy>PX1MQ?}c)5-%%U(QO&|IHCsTX7>m<<YwdYEG56jrTwOuPn4q&vawb4P^_v<;sd#
z!72qoY)WVE9WhJSJF6JK)?G=yPTy3->AU%(<9E%AnR?C6Suar$d&XwrFWD-)bn9k?
zc4ICDA=Rs93D=sG_8Y0F<gI#XzBqZRy#H23i(~x73Rzuy%&iT-nj6eaH&?9;QM|I5
z%i`DzQB|uPYvrdrzvQPMsxu8U`fR>SQ%hyZPkz(qv1R6w9gI2+g_D(3D|?k!+?Z@W
zkAqS9%z9a~&@cbwv-bI@^xRicySm6l?$O3uChPgi<SWiiQJdH6p;*RRpz^2YuKe9!
z`{gP=?o|A3D5TiaWNo5vW}(#Z)J^(P^9;2jZ&l?i)&@0OwFZ-ypIcS89b0Y69==ai
z^q7L`F~20$;?HjsW#tsiZdJ`y{-+nIbZBY2>96wTX4AM%n#wMjWzw<CUUj!ir&)!1
zh~l2>I!byu`xU!e{+S8}Fe^V0l2$X$UT-GibV?yqcekmFON?TAnX&0Rk#(lLBJ!pW
z?#C+K{k6pGvw4$RzzRLpi<z4gY&}9v-!-<IUNxAjGS|z^^rNo6*;CsY>Z}*+RYd0A
zQQ7o7!SwS1NtFi;6HKJ<u_(^?I9;`4eUj3PzwhMpn>Lv4z4=sud-^1`+4bv{e!bUL
zo4kFc;u<3n`3L?A%E~EP%3@R8Oc!iAt0a43k*v+D&k8}s%F0#=s%l<ynT>9Wv#J#=
zziN7y$3QLkcD3p=DOokEk`5&uMmMt$b2wDe9X(aHp5!$Td(C6c{ARaB=hh3xS3|d{
zP1cYzwK}Y#V7J3pF}E;IHdErUN&io2m6qezRln8mHe)|!t8{t7WQ&!>UCKU3?wLz(
z$T2?^vCf=hRj2Z<B?ab}XJ1s4uf3!4sEk`X(4^3O`^mZHEFC*kmYGa6S@^Ko!Z?Ff
zubb0YrG2H7no`AOb564=mEW0{%+{^ArKBw&qZV-Wh3Y9cKIKJ|bj-i6X;;V!;?&g1
za93RE`a$)|Cq+e(e|m}`k~u0z<3B3jE%r0v|G7!I=G<DjeQ$cz!}e}baXZ|i*8O&~
z8H<{ndb845^FK=_s85bFR?qfVQQz};y;^qeRTGvMTuOpF_bZw9Hd<73NSTE!SY#H`
z`r62e?}65}s#g{df@EZ)pKMdfQ8}QTZJ%Mr;oPm%R^p}MV!YCfyWU%|-tv*z`%VsZ
zh6SAF_Qq399zU6FwtUT7`41u~X0}ZR8V?TFs;lR;>4bOuGh>%5wb<^vK}CHPgZae#
za+4JmD-85MuT?v{V4qUao6Tl_D#F#H4{SI7Q4^?m`TTZu4pnRAG<_?1R;y(5Lj{6L
ztdY#x>ps}11qHclcuE~p)|(+FKdnYu*}CbWn%+593+D$%R8?&y)n>k&qjkEdQ{A`q
zn7X*NvdKw}tE!d1;*1m5OQ@9wTvlC`lCR43rc&*#&>_=D+@BS#b}UfTa$alLTz=0?
zRK(LP_*a|R`IR44es9;aNDy(9<&<tw+1EBv`IUZ+S)?$hGTVx~YQ62h&8jabD#}^5
zn%ykVSKNNP#q70UyV=j?DQ0C`8WfgYPcb_n^-N7Mom)-zDyJOJ#UsZ5K1Z4Tcz#^v
z(wq*{ugkSe9>wm_c(v|;%2%yG<r8~2Oyi7#R5nkTZTctAPQjViMs4EK4wZXmKjcm<
zcQdvr;8%D(k4rUF;H&CCVP^H*c`eG10$OEl)oqkMv`$ouf5>lgf}u-w)#@Xv^A>Yy
zeGRlyFJxv>Pn$BsWM5&uYQZNq<8{#!RCm38s&b#FP4$YSrs4}1EmO((!^$sGn3Opz
z=NT5Zy)>KUveoQ+-~_XcZ==;4`xGp!ub)@2zxzt@>}Mtg@kjY)ucm7#|45Wpov6Fi
zY+*j9;_g-r^H-|`l#dm~ne#6`W;WeP+x(>e3?-kL4Q3oq?bYn-*j09{o3Hf$j)O_<
zw@+qm_B|@sd+JO#EaEa*cw(>Sk3Yi7zb5@tJ+{usEINC)@)A!;v%_viN_TcAsXl#h
zN`>29LrI&l!szdeURB*M4XT@8zE<$P`Cf%(n}k9Lv#$K6$@`V7raV##c6nzq|BSox
zVnrKi`&nXY0{eWF8w`2XSEwbK&k0LWm3y$+tRpC2HO(?xb)Lj>Rb!u}Dx6b=P3JoJ
zsR-O)Q=U{VZ+cvJxv7LwqiMgei|NS;nrerxPd5*3wo)qLHB;WnJVj|{!dkOQGTzEH
zhbvVJYJ<#g$Zt}R75HTG^98@s@hSYKv%CvTs^vH=taDZ=y=load7JO6HZiY7MN50C
z;_eH==J#2G%sz9rs+>*OXL@(?Zj-+WF<Q)Vhm;Q%cBv{$GF#l1(NO73xL{ngqEm6&
zqtB`Vd)_L0cGN4(+OymA0!N}+qw`_SKl>-BFdv9jUE?03v^+yV{+o=5%C?X|RXeG}
z=JVs6R2=3As7$WmSCn5oUzOP<LT%li9ftqyl~k;sav9eobEz1`omSSm?5XnKQBzfS
zONNQxGAqR&PG=M^J`ymN;5ln5q-<j5+$ChH-6f>1voF%D_7<n?#;QPN9W_~{#HJ*Z
z%K}G~&V)`^xp`d4Z1NKxg;JjFW@Z=TWEF!~nN9T#GK<#wXxb4GqG04|Y_=-SQk8AZ
zY-Kk)LzSxN1;*u4zGf>nvZ`!zoM*U>>9I+_@jNZnKcdR#(=(Loy^Brlrb#I4>}E3k
z-lHMEC+euma)t+LH*EjN%P(&?%nspD4x4#T<#@(Im6`8MRrk!5Q+m&MO7_}TDJ8Xk
zrmCHLxQtK#vsSrfo}yC!!BRc*36q-R&QEH0|8f~EzQe7Wv4z3(8n3l#kHRDs<2)VJ
zN%PhzR*Q?6&T-&Yc1#gek<8y=nm+M}sbhVU*@1;GjhjzQQP<$tF|9iKNj6ovNMYW^
zWwO0T-x%jy?pJ1aHBuGko@#WrEnCs}t)S_WLorI<556+l`TM3ZZ(5^iS6!|W#~~Tx
z8(a!%4T*Uw`o`ZCm^LgjjZ$82%p&tmd3p6|6O-yDGn3pdZO<dDDj#?~Rev=xn_fyk
ztlaZsiRn>$2Bj&jUaIE2-pbFXolsmf-OMcO<9+!>t70@cT3i)#jFMIN7V{|nkM5P%
zy%nT9Cvt_-Z=OTO6P(T|O>Op(>7RI4*;etNa`v3<s_hdenH&o)SA8wJ)Fjh9S=GZP
zNOf7meATq=-<9$sQ%&bD*r&`O#Hw=i{zlV9wL)fXH51H&s@sg89xhVzdBS3*ncbnl
zXfQ$HR(!c!{z4Yhv_x~|ActDjOPZZ#e?H7p46$)AlWyOnl$FMAcISMH=`OYe(-S9(
zmH$genu-5QR9jv+Q>F1)u2N;m5>uxMOr|1So~r#FON}K(Tg>+N%j+DuUab6l@@mzl
zG$yl@zqgh5#tEA&-+xvqf^mv!`vf=TV&QbfsQOK&QghbG<!1j<Kh!R$`2LEx>iS4t
z#Vs=@DA;}7qCDgLRAuo4{U$5dv@09bFvyh(Mk{f5EmILOVOA6Vn{HaMvs}gK9=8c+
zUAxLH(LKudRhv}iC4E$4)Shc9(7a2D^Waj&D?&ZSylb|baU7I0eR@mFY;XP&6&`7E
zi?j{aiu!R9N=qi!DEi)AY_=fiv(i!TMis_g24>wew#jQWaGDB-h${Hyd@?!N)L`0L
z%xkWb=AgLd)_2nxipi=9Rt3s2`>)E|ZC_`y$gj$*Lt8`T;BFzaRqy0X6kWA7y8W&z
zeJE^EnXAxf<`-h5{P8)9Y0Wt&1;(r%6=Aj>#bq;E6*f&cWPGVlP`+DhxoV#24kfn4
zc`6Noj*2QD4#_ru+p5&<Xs3KW`l4x+0H^Zr^)nTpuP{)S)L5&$_s1*MeC_L|m&Ekd
zCntNERyEI2Z#2|WZ;>}p-x9V&tt6b?lv8b$@&;~BrC|T_CPuS4O|4f}nC$6pF^VrQ
z(^RsxFim7yF8_Sr5fy)aJ{66upC%#Io0YD;b<(J<GBf=);hv(syOi;smk-p$K1iF$
z?OScE(4TI4^k}rAiamqLg_|ZC=M@UoR|`&8I5VTy)bi4K6A9Z(D(a%sO{OkqFg+8l
zr=dD~g__}24wXfZT+Ei)JE~WQ>@)g#k3;dp@dN4~mar)QPnMPDN`7i`-|~jSN9AMc
zvaR-NXN;XSj=H~7zTh!I_ECMKvUSZPweX7frd->0sJ4~!sYRCd%6IjYtFPO9Og(@v
z-%R#onhL|THgl`p`6}Tx@hY~ClT~C=mMcWH&o^8Ac)s%5Zy%K0ji#H|2mLS&dL3pK
zG2?*AWxq3O+b4CKwZA%{Q1<blg4F7@@;!EzX2}xF%GD|@Dr**hHQ7^tR{n1KZ(~N0
zeM+A)FPLns$TDpe++eo(LXJ{S^C=Vg%~`5*I+iJK@Vlup;m|p=bIx*RyshQRuF2A7
z%S+FhmHx`o@mMFReE(FF3QzPV^J*D3W$}f5rfgTV6~0W{pz?W9zf!u@8hHn+Uh~t7
zCa5||UeU<<wOVn8o2yEZ*=za7&8!OdCBG}3iD6T|WLR$A_;aGtcWxzx)6>iqWP+KM
z&n{6_m3(1ga!JQa^|P>_saB)E>VZ$+RXQ7zR0A~sDpsgpFs)!WQCV~CpK{aQPLn>x
zxn@tN8kp{Tzs_{YvU#co9a`p___ry#)NNPHyX>yu!gR*8X8LsH|LrEKrK!QDi}ub`
zQ2J$Pn!*;KoF1uYy3_o;$$1qS(~CCj%1tM4o64(Os-4gARCy~Os&quz(^RXD&&>SK
zDiu=^J+sQ?Hl})6-s;KItCin7Em8e=#KgGK<*f3&sy(JfMs`YG1#zlrdg{t2*1lF~
z-kf8qRJKB4eVCZqi-{?UhvN^cT*^MC(C)#Ykf=3NS#z?pvQr3)3HP^!%I0=AWh+Da
zl!|sdSGGBBuXcK2hpCToj#}UR+s2B`=hc+g)TkA24^oqR7OZ;D=a7kgf1gsoO?#zN
zUA`v!Lt;%$CI^^q|Gm;U>w1ye^0OyRWyRzbP1>d?hpMbly0B@T={+81rDqpA)lcyL
zHoYs>tf2mg)%2eFUzH^~k|ykucTF5ETFqEnrz%RESzxjwu1S5e$!fJ*M{mh>F?E?9
zf1_mF{Nu5*WfYr<QKqozvHh{CTU6Gm<}ZGw{7fay?Dg(Vs@$I1CPj0L6b{IpR!e($
zNzpE5rR?vH-NxIuPf|@>xJ+ZyJ0{g_8BOY{sq2))SJ%shoDfh}el%aT?B-|FAm%wL
zK1WulR9>x8?EI~(ruyDYT`oh?!kVK+%`d6VY(v6xHKnp5HP^N>H5<=HRhNta(<m`t
z<;Vl)6#MFLn~0crnkT0%Gi7BvX?87Qr<%1Ni={*OY=tSj+m%)8nUp6kSZ#iH+XN*J
zrf&7H?86orPWu#|GIf~tcq~(mnJ;Db*umROEwInx@%?6niPvVDu{yV@{|jHOw&te4
zYSCMB^9q4Z^8{Ocl`M&=W>YqBn=-BS)C~G|Qq@3ojq;iOkrwT{tkoV1w3zHHla@c*
zz@%<;aFLS3rx+REja{Y}@4Zu+c9~Bz$686XLZn0eaQQwZtI1|^*8aN6f3!ENMs&Y2
z|Fm_XiWko#l@b#PWp7&>HHB&F>ULYzjH;IkX}mY`H_BX5t)aC=R3qM7MI$2RxO)8m
zc;h6ORm!Ez$CViFFERcuc;4jD^EoDIddEy2-?z|-+UjSfG2w!|#r#XEHuu?7x6g|(
zS(5Wk`Cg-@=B`{7)6!pSlsHy0n@&lT)R-d^W4d1Fy2*1T0khST9~82Df=ry>NNd^{
z`D;|Zo~HAM-O2cm_jD8P6Sq`voy|0UpgrC2_+>@4dAAm-w|^E<iLq8S=Dt^?G0pj%
zvCE?miqo!a&|uwkMpcOSla#EtlhM4#vPulM-)Y}i5UE~cZmOBL=&g#5jJWKr!|#<f
z8o#P{T;wqJVYs9AJzr7%&jJIThpVS)+`Ru?qpF<GtT(z))viy}BxPH%D&uDjRrR1L
zs-_d?C@zxtWl}b^S=r#Pg|b_)nDI8%CNrB;g(gdXFEo7|tE=`=Wxe^cI#>DC6~`3j
zp6`~=bu%~Pmp!4R9_FOFaaN1D>d*HI6YMlh=IW^^t!wHtne=^@Y1FM^vu=egN-L9o
zo1WziR{NinreZqjwaN;Oy(YO?GG>!m#Z;fXerFPUAj!l!W}=3LIJ5G%c>-#CKg*d_
zsT@`gyxV8&r@LQi;iYob=CiKK^@rq@e&xI}Zu_aLaJpSuZJ)lA!amb}73q#+3exLa
z<!A0lRIZy}uJk6{!1UbDYf6u{&6az6wNPWP#XRMu3PEc1d1B^4f9I$)@AWZX=(Ag;
zK`c*Yd;ANP28s6y@rx&!P7rQTUMJL~bhu=`S!u>!ljWZjOpDeB7>jhUYH+!`nUyEG
zC|0j;RcLr<so*u?k7>JquTq0jv&xgz>1N66R?ADjoo;-Jk3)$~t;3|(w9Uv#N!C2T
zj8`e=mxyutmtIv_<9_APPyZCBon|x__+nu8cj7MPNq=6L>MPGTF<H1kyZi&Ia<T6+
z71!c^^Pj(_DOVqpG3Ky6s-SGbp=v22qFiDWEmv*5!0gZcGL`MqXQ<o%U8tx%wMa!X
z#!I0p<*saKkel+u$&Z!v`=**7Z1Pp=^q3>B*(4&zSGz}Pu7|Per=6{)f{el{_I6w*
zDgl8iMte9_4jK2V1aJSV5bwOt^#46}rKgtNN-_R&W|kpa%v@amo2KsRHf^imQlHeK
zZGJLLSfR~ng2J?&%<??9?9En-ZdH_*u27LGY%xEoqO8z<&eQbS{9wgRv9nG8glU`E
z{<JfTsGhF)Yt|gIpjt20?IshID}u9Cg%gWScL`~lx>vU-&#P!O7Uu6TTPHX{cS&re
z@}E?0m5iY4W}?pomAzx%8BclnMIq5_l1k=^eM+Y}0_E*f^v!;TE|L}gc}?AJ6|>^6
zoySyGo?55i*~uk8H}`;&thcYy`CDwJ1|?idg^jB6Z*>zDN{@b30$m@ieD{>`$`3bG
zwoXhpYispT-C|m$^5d|PD*KCViYwSwnBFU%ro`#FMtS3byXJG&|1vG*t21ZU`fSYl
zXrg+!N4vRm>VJ8e?jD6{Jfd>)S367&*;*(c$?{jtvzut5cF10FLd{V#sVa5lfTyd>
z&Uu=c-u`ZAHrL-&>GGU&rduV|)wqLeRkV&c>-42=Hd|Bw-qbS0S2;Da$F$p6$^1Hx
zoxwTdU=^+VeJV{FQ_Z4xDyST0TxUATR7WxCair?oa50sN6b;2m8yn5}*6Pb|e`BIq
zJ}XL5XTu%U?WGSDUjN=GZ~c0rvP%W4GG}Ct+4BD%m0BLHky9vjP-e*eu6)?<psMn>
zGE>>iT`InJna!G1FQ`0DDp1L-`>Nv0cv|81Zzi)nAu`H`p0O%RKUOu{e67z+-Rr68
zF6JA?tLmnxZjsV3%lNxo-l+4Kg1JhvLU6!4Q#Z|BN-=&1Rh-Mr%yvvjSJ;)PY`Q(`
zlmg3jZBtK=k0#4E|1jC8bzSkU!2;9#6K7R_2{Eat+Vd%VpR>xeXvQLwYyXo}=1oa9
z<<xL8<DKQGzMaWg`M_&K)x3ZFX7wT6%5hUSn1JTI?}o6c=JTIaO5$c#{GjM=_CKpq
zR%KP8y6rnj#dWpLD%O`aDTL&nmNR|(Q0f1-lZx(BUzjSc`LD<#6)j_Ed_ew9#v>({
z&x=(vosS#nIQppSon<r*oSUgSWe=08$Bh!z$=W>%my=$YJUAbxY_jWuvcL;{v)C9#
z)0j;&%)WiUX5@Q$hx&{&$Bh?y{gj)uy<frP<_x)g&&`cz@>?qBRvD{qYMo%*Z(*qT
z-(1*i{ikV4jF;}2o+}VD6%5WWiFMqeRQqd^>8&DTweII(D!nt9)IQ4=n#wkCng04`
zsZug~vXNy`zM1`m?K-k%3zXGYzEtHhbu?qSw^6y!`>pZv+%~1tt0Gk2y}hZzal2e;
z{sLh$-D^kWi#&ofXMERCaK3R-B~){r!lmOs<PPYCD1Ty6QhFPD#>jK+cBMr+hvlMs
z(o`OpRw^Gn`9pPnLWxmB^>nqX=EbJFSy<E?TxO_QXLqV~vdvR9t%x&iQ0i8yIo+i!
ze5B4)$aS^Zsi`N;mj2ppa!zZ%nrTm$Y2Nm!vM+kiC?{I3Q#x?A*F=KlmC|RPb?W)5
zY{oLVlN5xvS(}EkE>JzUm&x?~?`0;Qs#0e6m&}(JUY}$9E^D>=+tfR18?O5(+<SiC
zw8MkNboWbV73tnJrV1C_%;w0=(TM!_L)D{^TjkvgCX=7lR%+feE6sMa?@*X2B&dGI
zRzfK)Dn$NtiG|tvP*%nK1r8dlPfS$nKQB`MH}8{Dl2D=ix|S^E@Sanu&SeLUMc?dK
znc2pn`uD(m1qRI)waI30)T-BgG1>TSsfzaNP_wG=-zw*CEm66w%&59`2d7Hulg%aq
z@{-D>-*T1jc}+CkxbLY+K|z>V{m%lEBYKt^EYn%dL_W+>I5_c=lG+{*#ljjh)3xg|
zlrPHZsIF~JH$CnASMI?JP16)PKBaGNg(m*Xc}yP$t~OJ;xLa|V^+uyA8D=%<7r`nQ
zoyAm~HTcZ@Pf3`1?OUtN{8z{HH0xATzIj2~``ykc`~3K%!v1ltDf6dA$~lo&Os(XN
z6dx=NRDHf@fzryyHx%^SM9e<1>8o`e@6ZTLJgdBi$w#%s!c<XJxKW`pv|d@*zD=cP
zrnwo<)OE_Y?I$WfKDkgm@0Osdn#~l|xWA01`M0L4ewXMl^=N#oy60uLYWGVy)tO-q
zYA1#JO}D-XRB~Q7S<zAOyvezlUrkM(d70h5rEB88phs09v&u~P+hK*+_z0EL54x4r
zb7q>#?>?%O_iKV$z-bBdf`ESt{jSr@`omAlF-kL=na*M{Q(F1WG&8$VapS@1W<4P@
z)p!rAR{haGL6Q51huK!i2PQjsOI40`TA9?el$bfhUQ|23xLU<!-bUs8WgARP&;C%J
zx^9^XzsW|0j>{}+3wM1`;8>_9|2#g@^pr!J{9T3VDp~uEsTSYeq_&i=LOJYkwru*&
zbxLhaMQRmsA5AM>l&Ze0n67%CM_S=<C%^jtH;$^_?ct{HG;gW!^ei`f(fvZD@6<6B
zg;h$bbF4BH=CiFfTPt@?iR)3PvWhr^MdrihW;(s|%-7W1HE~<sq<;F2lbP2M4TXS!
zJ_Yrl0J*bm2TkWNWGH=MmQ|Ik?=}@mcaT5vz0RyS@~GlE^<1-`M>d#Fx$SQzVJELN
zZ?d%6$%Wi%x2$7T%2I?>RhKE5_v+p<@pR@@dBPTCI?-!``Ksh~x~uG(l^1`CQ?0z#
zXU?D1rR);%+4NIro}#Fuv8uyTTV-3m9)+2k*(^AkBNbD&b7|IDmMSP8-k=iPm!Z(7
ze@$+|Z87EG<Q7H6Y=1MGZMI5wQ>5e-4_sBS+3`hb%acW_KXRN5)<>LCU$}3Yao#;s
zjl1{nsoVa4tnR}wLES5n)A)PwbEQKDBFfhl78y6X8JQ$$T{Zo3-o{9e!CL!P+FN7q
zE7kIIttY7-37M_JU3koB?b~E!+o<0fj_FLMX}wKKDf>;#cv`NiZxEVe`tyyl>72v8
zrn)6Z6&!ZGGVz>oUt`^TX$|pTnz~nSRhvAqY%@)7YgPS}9B3$V?YhbFt5>xyolaM`
z4`)$+$Wd<+@>fG+w$KG*6B$XR^0(C*&SqL_MoZe|8DqnYPTmezbZp<J^}R?^UHc=0
z<`;(fDgtG5<-N;KE3dHGuKrL#+laA$j#|*x4{GmPerk2hOVrpsW2Q!X*i_RyaWhr-
z-CSnOR?DDfm)x#eI9ExvvM4~UW9J@|cEuc}Tdcnn)Z%^^e^}vW=9qolghgY%sk4}a
zO7w}9=H4?5<Te!Ns_cHWSh*=x-Yj&}45bSjXQ}nYc$jaBYF5m$KWEbKbVL4Euz=~*
z-_uMVOyxB5iQBH=nSIoBzR@zZ73+4Z#_4wGE;zo<I4)(E=||PsDiUEcOzO^@H~u8q
zqVeifr%KJUAZ4lj#-?g)Vk(XruZ^YI<|wq6^Qc*r^Qb--S|BH@({Fs*%~h6f??lz4
z=Bujd7fz_%wV$9Y!{#MxyMsq5=0b~_$Kn_h#f8PHp1g}x53akY%==efUDb1&YJ@GT
z$+vaRDy5sWOrP&rpki^cQ{{EcJe3umrm6v~k4<VAA1Lj5dsO*b=S0)>ftyX$a`H?s
zaa$QrPM)By+{R>TmnI}@?vt!MJ93ti+_5AR2FW){7mbWmXLd=N%>R8(eqYiiW0upO
z<by@DO;%ZLHhMbsjImKog96hz1EUGcUa96Es#bYio1|0~&uAL4GSOJe^@g%b;8c?p
z)sCib!lq~``TbK4aBNfQnS0vAhIP8KlEyBRiLFx<{oCVJ3&htb33IY3F1&o!bm@vp
zg=M!iHMhUHu41sXQFY>z9;MX<kLCV4KT|4eNm12snQKzJr$NQKX_3lafqM$YEfQ+2
z_li`tSlmt3qa#(kjLS^@FSV(B;nr5+Fq)~dT}n{(Y(b8xaom5UfJ#QC(`%L(YaS6Y
z+x-5YDf7oJ6Q_kbs%zP8%^v7}Rfv9&qr73Cno_2TxmmwDqw>^>DJsJ1Oy>K`v}C_6
z=Q4Y*W~A61`N{NI-g8rfBnvY~tuRIQGHtWxC*o8m>2xR`i8!g47p7|VRs4j>;X-p2
zjgz@1zt~=x=3G%$x7yRC_{hai<<_n+({xikW$%u6Cg-O!Di%obsIGdIs?<{aUH-|v
zdnU;;jH+J*Y*d&2?^VeyVp8>Ikx^RSvs_`{2T|o6xBOL`iZ+;Dz8|5YR1%`1)NH8|
zqI6Hy{n%4g9+_67o(tltUT^h{b*~t!P7csesXvmhx_`Qw@~y+4O;3pJP+I=iLMgFk
zqDjQ{_of1$ip<vtG@9I;SfUY8)N9sXeL=>R^P%F&!%_<62enQ8&%IDOe2q`_Mp}c(
z5$)58ZKVq4H#hE4JmR5b78&AZR`IpdyuK?$ajES=)8>}%s$EU0D))<aDL%H~Ghrx?
zGRs#tRZ+P1)@1%OJM+6qhqRuFZ&2pD9H#O<;ken}h-{U_^Q~s{c$X{849rnI8N5ry
zJW*6}^@IkazrSS_-<{p4t|XwRoT#-%mFewAh5Eo6Ila$Mlq$p4DgVD%VSN2-vr>>>
zsiO6U+iH)qT2(eD&sNKSBw<+X{!@A1&q~AW#8b-pqAAMPzIm$5_@bedb5Pgp`J@g-
z;XNHn3k3F>IxqWUUR-nA{O_z*v(hcfs?Cc8EYgy$$(0@Xrr@Qoq;Ngo#>`?%gOc}2
zWt9t3x6Ssa7|SaiY_L%CTBs-ycFw#vaiuxywl!w#ms%9=-%+)AXUL~&y~A4hzxqnG
zlPRZ-dHa+s{;Ynk{6s<7bW7JtGwYXpx{hAElte!zDaU`BYAzrkudJ?c%`|k=1G$!5
zUzO`c+tn<po#lR)XqikgUZrqY{JBcm!F!4-alI;TF)QV_><E&rx2aQHvQJUbN^rkH
z-SQ-*bR}(tuy+~i=l+T)3&wv?6HetajtHKp+MavR<ko_tsyEJ>t8NIdRXw$`N=agq
zyxGjz@ydz9c}nY4|Cr=;aG1N#v@<Kpdv8*F+d(yLO@rCj*uRRiw3L-5vusv)oz84l
zyFXbu_okwn_mmE^i@loiB6d|~+mcr+P6??tJ(Q$swm_%K<N)I&rNSLM%qF)rs4-15
zROL=wp!g%Z*7TU$6jQUqOH?AA-A%9VOfpsY8l~a#_mOhjy_YIK#GaX)m-?#wI9$QB
zAZoVaojFrg-CV4d<E7^*d@zwQ%|6pEzxP(D>how9C5c*PHN}tD6p{o#$iFH&qP*2D
zN4aX}Ok<V}lFI)BB4x!S?y7ispHbE;&Q<##$!?fae^)h^Rl-<XLPE`1MN>7iPf#tz
zPgVKtU4OIq@28cYc<ZTr|MuBj{iU{f`-dGCi+JuC>vnOfPqsg8x>e=844>L01!LPT
zdA`;~CYF4RDi1duRK4q~Z2B_&qEg3&1r~cHH!A&(v$nYTgxUPz<`~n|06Asbc^Az&
zj~rBUe)U#G{<@3y`p4|%-+i{4hqbY(#=Tf?(!TzIx!I~tJ;6t9Dr>6VsaAa5VLsP<
zx5_+qRkH<$b}32k<yTvC*Go<C>j9;o6?*2%sn_L-qYrDS^rb7=>T0VkOcPSFkhvvy
z(EO*eQRsW++#_;E(T75nopq<n&+s{=?q>B)Wl~9nn)QW1^A*MN>fR6jn5P_>sLr*|
zL_JeaQGMh5Rcg0HcA03DzEpbcwNxou)7;{w><QBro9Ct;53U;7u-(-9o%6wBZ^}`*
zw|_UQ<e6Vn{<3eqnNLrbk`jxTMk>n`^UIUv6+g|}ZnE2*Mg7vwpQe*p0!<RlEzI99
z+$F!Ubc$(DlcC0&TlMNOiWNE&`Ay7c=SEscRh?D&IEmeS%i7(hhn`O}n8mwVEkdwe
zxp3M)Ge5f!b<S@`jfEoH6*bFts(T&iP>S{{mr?O)HMcphr!ujbRr^qzwOYTHlZHS0
z3FXQ^>*Sm!1uKVbyrI^0)yzU<@h(*r8F96DkNUKJZ>d(lyX=^{)~zO^$6GF`*3bQF
z@c*-z+T5^<st2DHs$Q5?tmevWWy&{ui_+WfZHkP$?-?-t*kx8SW2%{1RD)T8{}+`D
zD>W=G-Kv+lzqv)lGjf6Q$t*du6?Os23L1CSEY53~?Va>mA?h8g`SXuyic`-DnQvS%
z*=(NGJ~Kbfg$kFau$!eXc&aA!jayAbXQI-1qy0uLxm(OW8qHKW^{2`7M3A*H=ca5e
zhPHbuhi^+M^W4=pUH&dhCB^%r>Fci=3T@Nv)oz3<sC~NeUhcHUJwtBx<%<8~GF8Ow
z|EdbVXI9sXs8!~=a#}W8S4OE-w_8p6e7f<st~OPPywj@7-^puD-=n3z`CPerOYaOL
z^S@Q9fm>b}a<xoXm1{krqT|`5dTio8#VNU;OwR0SP!7&wP*$07$1wEkDzn1*Z_Q?k
zd7HgHU9HacOvxg!I7|N7bz8+X9LjRX-*TEMcS$QdcFU-S3EG<$<z7{^ZJuZ@>A<Hv
zr83cc!rErD$aRY5e3i$P_Snmq<?J(6bCqUMQU5(bb)EE8qpY8L=B&}}DhxBKOn3a=
zYmzNKS-bzarE+EZ4OO|2L#CO$N0kq6QZQp#YN*8bF+r8{%01Pm21QC|=EWK@{Ev}8
zVR=<`-ZdLVRS65#K0Y0VsX_(v*~fP%ub#@QtlGq9lE2bg+31Il^tMC}_3taEDbMOr
zRCg5NG4E*XP_4V$Y__wiPxV>NBGnX*i>ew^H>q4%R%5Crpr-OQ%R-sgPQx^MTd?V_
zi=WK|KTI~&W*1RAU)yJ1@IPB=fwR7H<&WcvUA4;QZ%-yEZ;_v%dLWU};;?w5+}4JJ
zX7BjYl{S{kn{7E}Zr0T%YVqcdvT_!yw3*P#K()Qy4Jvyb$`#w*IGBIFZ)tk=Qj1Ej
z6_4p@!G~t6_XcWqFu5o<r)*MPpF7q3NctV+cHeB%=z>K`Dl6Wp-s*a$>{8LHSetv@
zbk)`gYLgOjG*_MRQF%JeQT2tHv{HMKmg1hv6I5!{%T#;1`pmVCRH;PnxT{?LJxS$Q
z?MKz!!qsX*8!`-+IE$<NxKwEnvVc)VL~?=hFMdZA=V|h)+dR^Y12b1EYDlhDRQmVW
z=%CSk(}10<W|1qWndm9#s{ht#H48ZGF6*l3t<3izTZzjh!`P#-TghHys!Dl5i%Hi4
zS%tjb7iM$KU&wO@&M?b1{A_w~qoC=sJ`MSECaz}R?wG3P1WZ<5YgDan`TeWDg6(m$
zHh~gl+soWW6D!<J*7Jzy$guh<^IsNKUUPSWso*9z<?PHLld5hl`O;-aRov}#)R&$U
zRtUWR*6<Xkgp%6r4XQr2TUFK^vsCTf&#%<q-X`nrtEIT<y@6_tw2$%3FIFl?E;p;N
zd+k$S^Wu~0Y^gtLj~B@qeiMGLVqv3bvi7rqYVPYol~sP~s?Tyflnz@6nM!Ouq_jVb
zU1dkKkXe!BHdD9usb((~9~xgivPQjBK*`ioXsX=zxETtq`$H7`uDF<_95Yk?=Ao~8
z^!+~LDSg`%*G06NR+XD5M!#EV8mrQ7yfU!CWZ(Y}iig&JH!ijkSIc)#QE_}QOLf7l
zE2g=59!4c?S}I!q7MLjIG@IR0`>1W}rK!UFk3}_t=eo(FiwBjrr<9o9xXY;2w#HcX
zlGR0}yK7q&@7YJ2ZT<00W>u)DX5zwMidJb8RlV&GDjxf^Q!YDPU0Lee93{RIdE@ZV
z{Yo{RymHf@O;Twr<yDy{HDA@?2b;;Uj#O2%o>b!ra^b39-w3GM2~APu+1sStP^@a|
z#bd9W9d|;xx^|*jKAVx5%k=`YADWgXH?u3%X6|@px}tuU?80^#MPuQo@>jZpO}DL_
zq&#m?zA8g&nwjcNRi%;*)@B+@ZYYYXx|kVzrkKX0s+-!)_@cD;dzBeOVz}BQmIW%;
z?_5>c;n`(+RsOQc;{G7jnHmd?@6~TG$z2$r^PsFuc`B2?szKO7)AgaODne^iOhS)+
zP~3W{S9Mibr1GZE@`|&(<4i?b9;)Q$=c~868!Kh-cc_Z^S|~1K`zouQ9-<t>I#v1h
z{@2DI_ZcduXzY=H!17DwXV(ptIU8iv=I)<va<8L7Wm)cAqd3JGD&OCKSFU>AsdB>g
zi_+(a8Kx81o+vGHe5v^7-#cT@g>hyzU)|08b-c{(pIol;(^cF;{i3Jhx(`}P*)u09
zBrRz(+Y<9v=~Qs1N++wd>C$C95}P?K%{B$EP+Y<G+thK|3)2F1P4he!cSY8lEv84V
z#Hud!i&ids7pABzRB57hfX6H?h*L$Ni_=W<hl0u7EmPFzZ*5jGdDNh?%Zk_RlRBI7
zw2hBVZ)L7h$dB$*G3jzwcAoc4;j_dwqo*^r%H85Rq0+OvRjIChs>*q5SB0{rVG4l@
z*p#2=RVq)ql4$C3LPgmyV4<ST-g;H}H}{ksB6QTYw-%WG++(E96jp0;dcs_F%_e<y
z#%X5iT{jl0o#E^^>E7^PdE=6)N*DHZm^3x<o4)wC!nB1k$SB}Yux9dkX;Z!~UL}Q_
z`&4oi9xCr>+-$<GZ?D{+?yR9-6mFLCGgxstYmw=m3)j>v47p6xB5F-8#qpRe-IA`<
z;&9ufIM_@h_GpQER;sW3b1QbUZ`aQnXTM-m4U-Kup0!ibbfKVv`p#>eYExtGsqpxj
znq_@9QICB(%ShdxO>tWCA$5+2OO!uOIWN1`>aoc+@m(r`it+0HU508W9-C_DyWCVh
z-NhuAJx@{PWX~ft2KOhXzE+1-jgPUaZ4EO~oTr?xzQFUiy7`9rX6&nrRBqpwGvD>T
zKt)nJOJ&yVl`5xrEfigUc$r=Q<)_TMUQM~NdA<3>A}KSC)new8Zq71U8*@%AG&b1m
z@S7}!$BXk6^509!_?8HoajaRZq@CBQl2IgT{@_xlLZ`Q}*`CG?N|S$mG!0qFY5FMi
zlbNXTTBXmQ=bBd7W~jQ|Sg5>eQ;>>eb-cNUm5gck-b&^Ca7k06Lo(*~=5^`xJeE-Y
zWi(&KM|QILU3XUHHFKp*4~54m&X(VzA`#o5^!c8gf^X~*^Do!utKMGIuF>N$K_RKj
zN~KrUT|pzwMM1D#ReADSZ{_a$2IgXw+m-rR9?Rb1l2XW1IiY+m`;E%V_tT6Sk9w)P
zzEm^O{}87-IqI`Ymv5S?!N!S7PQ}kmCuQAI&X2#S92Cr9e4|a>?A)}ErqcT_n!dQU
zP__DtuQ@x<bA?~qsuVBn7m-)1eq?%M=_2Laqc*AzllPfy+t92SbNi-gmw~PF$-e!j
zryi!6_D#q!J@s5q`RlPJ(<u%{YO}w4sZ7(^tyJsQV5)Yv-BiD8w~En1PO~!syG*9X
zF{)2pSEsyRe4^^geskmh(kqo`hBKLE^g1Xt@kFb-uI5)3R>@M#D_Lr?wBV>5=a+42
z|DsPRq#Tn_WtiuzP<(fn!mrO;m9NxvDl2*Nnatj^OxerSS}NtVp{kcypvsnY!D^dx
z*iE?SWvaPrhnYO;yr`zOvtBLvNU$2mmUz`e1^%Yh$L1-W`^u+uZ{-G)bd_VKD}E`P
zP1c-XoN~TcZSM6((>1Q93QHdKDZlZUq@*&r*_2&mjnednPIc!!k4zUA=*jO|s9<*F
z;v1Do_pY1t3LZ6OTHR$Pa(lm`_||QvlcJl|b1c@X?J&C|SJAiA<WJNNlfG0rl{kGa
zlbwIW&D0t7H7cFvsD>*tsTdbNHw{!>uIeEzZ_2+mRpEWXX*JngEtQ+C(eefX(~Xa{
zotN{DXw$Ta_^zVO*{tq<W2Vvvk8XL#^eSb6A2U=Z{k~$t@@0z3{58u}4$F3`{R=cu
zv)!(%KBIM!xsz{?+JaLuX5OAJ)RGU^s!g8OsAk_*s@ie(o5_kT4ax^SRFvZ9{xNnG
z-eq=T(*v`E9){*|YqzUyjQM4;UUQ$KROfc(|J~D+{x7*|uG4)<>Gk<;bzA)-7JK#P
z%Ig&fneWt{ubRXiV{R|<!t7w}Ckweh8x-B=x0~^5wyVc(T%(r6t*4ymRbbwq`_+tX
zSE`D@&o`#}YAeic3i+v<zn-mn;#a@QF##_NM^i1e^DhsY@@IZh2<&23S7>cgF3{O7
zyDiPx)Y>>&{ZHIJjl@)b)gQTq>e~6Ul=A2H%RdoVrF?VULDfzV9t*w0TU9oBpHsPi
zyFoQm)n3hMy1x3{8}AL@o)*?PIXm1)=v<9P^9wPJKLQFG^}qJ3x1N|}bo8{niU#Lh
zrMe?Q#;)79naG@)Wisu1r%Ava6P=BI7G@#I{EFJ=&Zs{6%&pox@4T_n0!Nk7C{0bX
z1|yRT!V{H_cLbRpF%#2p`LNGad;1!br6+mKf)0o%y{I^9T=rc?Gkj8j#?LE#I`^53
zjhTHIP2J+2t8#s8H2yX3u0icJeYJBh=BT&a*sOB)iHGr>Q@I)yuh~u9w=yWnwrtYS
zdX}JKUbt7PX-l}#YPMRXz=N-~rFVs?`<>I%OzF6#Vjm|h)0~&8^8Ed4^$zPm<5?ZI
z)r?hC)UWPS*17qnQ{(8_&l>#2g{G?`@>M@5G8>n!s8xN`E~<LEcY<orq%<Xl5^j?^
zx3fx@-4vA1?<_I&U!`Vdkn+TY>kpq<PpqCA!=|O?h5<?PY|&2?4t=;SKjovM*^>V<
z%Bw%Ss~Xk`nJ2G`RkTwKG@Ux{n4<l)Bc{hpmz(ChshYnnT&{FdyxY|9mxo&X)o2w{
z!z`tUzsF6q`yZRe@+qn=pEJ=UWbSJtQHg!(vwnV6`X_rz_1E+^GwBnDltbSHnWU%~
zE2Z}osv1AiR~AokQj*=<ZPd$=ud2M8Np*eXV};$5lvL-)=O|<({*pO*&qLWmsZ?o_
z%68LauNz7s?_bIYOE+r>9p0_%acY9vF)uE2g}7rXtx`|S7EL;>^4EEx%3MD&)#rQd
z75#;UOp{(~D|?scD}BqKV)k{bj;Z;J1!m%}a*aRV|E#{E%G1nT+FkLv$wUS3r59yV
zpO~9<+-XqaO<$=pEpffMpZGF)s|Gc*qlcB0_zSj}7DNb}ezGewSDuliRMpjFI$yS5
zRY|f_`R~;=ioKrc=2s+GO>ZyQq`cO%+GK7`r&+*%E$yv)>y#GCF{{d*jyIoap{QJX
z$G~`R-WtVgK3u8{W}lSyonI)Q?tj2+OO%J&^lM(~%baH`nEtm_ney_Y{99>dg|N`m
z$_~CB$}{*n&2uMASGx6nu1v<pE`@z_rYTn(Hdg((YN-iF&THlOTh1FDT<Wdza>@ne
zbJKcM&biK4)JQZpU7J2zF(Ic`N!&EaY-8wrGglTyvml0Zrl(v4)kPWqm>u}4E${Px
zo5DmfNd?g*dS<5%b1L01DpYwRt!*xPW~2Pxg?~)@<(QRDpKLMX&{HyNd%4!^$Z;n{
zXRRe>%yOQp#~*ho3%wLjKU~LdwqRzesi8}ya_!1m<2Bm(ru={A=&r8sQuaPSQF*ET
zU9)3*RFpNA-ZL)DsZcm!(5G_wi=48}0w=kX6E)04bjwuNZ7xt({gt73)reKq^yh4a
z&KyqpB~BVj?)$x!6b;syoOEDRVzUpBZ!7mzVLfc1oZ;=SnsdnAbkdnsD&JT-%$Q$T
zs20VWs4TJ3RsGW(rsVQszG<w*G$onUJCxI}3R=|nYnY{4shDSl95e3NvQyp2yTv@^
z+5$Na*$f5w2kHveAB#+V-oIB8+wY;e!KJ`dW7cZL{9I?V@1cdt$zN8PB{FB4_=bg>
z_MAyn+`QtVX=js?TE@<Ll|w(>bTlUYHY@txZ2D_bo$`YH7fk=?E;E}j-PFJ=rC%l3
zR#GK!TA&&03Ktd5R87+by5Wizk9}1IuE;BM$yzJ0s82CxxP4u&P>4lyx`D3JMaxgB
z?U$!1&VBkyZY_(D^5*0pN^RSwm?b-`QQCCfS5Da9ROROm6_rk>&8pIeo)~YseMrTL
zBha)(^`eUFiV~G97IxJ|Y+j1hA$(>TVbhfkG96Xw+OyA8*Xya-;aSVgrf-)u*;@8f
z^}X_A(>0MIiYE0eiiTY}3Q51s%(SMQRMKd=uHt5V#LPuYUSY}Y6f@t*T?#p#9j1kj
ztY$CzH=3AE=Te%;5@qJF^_c1k-#^OJg#Ib4nc-q4st{!|O}$a&^&EB6tY6Az9*wLT
z6Z0gMeRfK#cBsrTogkK~tlYw5dgH`Q#rF>wRcohpE1h%}Qmho#GwW~VQZV}yuD<W^
zWrh1I=c~+}IYS}qXqNnLky}b4=d_fxHEx;gkWW-<`(rA%Jt08(&rUw&-T&vP&Xn*p
z<h*2~dSAJ~AbnxFs!PW+m92A2R2|)KDKhTqH3|H7SE=juZsoa2pUqhFGfnT92bsBf
zw;5G0dZaFPc!}|h$sgs6mcEmBnG!3XcA&x7@`${0OS*+>`{68OVZIHDCwxDcz7t)g
z_<5zMS=RzTlafQL48MF|q2&MYqp8R_Ewzw)Au6$+z8X<nOHD(!?KajqQLAFsecY&C
zdWK1c=L?-X%eN`tc(zj2ZeNb+yoDQ;-~Qcb6e8%Zv}L!as!+I$%GFy{N;`^0&Dx&-
zmNV|>)x4{BM&Y}PgK8YRv0|0Cgxtw^O=VY2JtY;RWrn;r)+in4ik6?~#Grb4$s}bP
zji;*oXRa9L1WZ<2aIeE;zB!9}&Agdva|*lF828UnJ?_$Gx{GVIlF*LH%3@_4rdz(`
znSHSnFn`yx#pGi3NwuQreWvC&+vVeKUQm{iex>9Tdc#EY!%C&u((BZFms~b(u75A@
z`;^73cf%6ZeM=UZN^c1<SvH5wOe!~ALHjM2N!zE@>YGaMsVz#Fs}vZgVpcJy%VguK
z7M0eFR#O#gA#)?G4y{&+ud4N*JXDOz@0b`C8K~)$=bNo6`lgV`E1<qScAiqPl$5;V
z4s$b+tLJ3f1&TCYZP!tqcxs-y=H!1$s+V@l{k__vJWuMlsy4f#@%EV0DmQHYsPr%;
zE1$M4R|{M4LQVMQ3X@sWmaA;hFfu!L?Tg9;o+T=!Rh+8cjXWy1I73aHN>3^UHP2Oc
z`@X}JbtaSP-+m@@tDQ_HB~uMGPO9xQz4^0GeofDErTP!w6z*+jFs*VEQBHcGp(=kT
z$n<Z{0Xeo)tY*3m#}$=2B~5?dYBa69%V%zDETrV{|G0^nDuY^uOt?z_y-LMkm%XN2
zbvKwW9yqNWba}POi={l~Ue;H&%YMI6X0Cgr5<Y?5tgC0MvWXC%S@oMp#pq-o)n82a
zm3TV8Dg0<RH(RM8uee9EN#mjEPUR#ccU7ybrHa2FbSq?M<|%JG(yNkH-fVhu(OTt*
zB^k>0(<Z393fQPd=wzu1{J3Wv89G^2a=*6e>oreRm!0iVJ*_0Ks-ESb_T|WD)246V
zlw@yjP-OWq!#LV%nc3P?{bmMi6HU}6wyS1zS(=@dw@|R^4*=a6t(=z0WOm_poN~YL
z1U1`uZu7%CA{DAL51N@}x5yjaO)y(n(`@#7%PP~k@24s@Y<y(q8#Ghxp3`bo7a4Pf
zUC$nwa+<oEt<FtT`M6opWTUUR+5i6kn*DpTRAQr>l}k6coBln<tird`(lm!HK|#2Z
zU9HK=R(XBqYWd_>%S{R@&L~7*R93z5`j{%q{_ScR4H3$7xmD%ae10i;ZAn*??+7wU
z)JRu7uws^~ca)#%rY&shdKO8lotOGexFv6>_*zJq9aZ_EGQDiSigLJys`sZYicQyh
z&A46|D22LARDPYNV{!Veh53OGE6t0a%`*A8xk^2&g3;`v^aJ_)bMgun)kozlUn!c^
z9_v<`@<&NE@{o(!mpm?oXHValMeX!alv>GQ{&&YSQ>Nd^rk_nzlw=Inn`Oy!s-2RI
zRIykftuFp|rMbxee3PBMb}G)B{Y@7IYnmH{oY0M#)2KYr!cf)GipShBrcT*+Yq_cS
z?d6Jx*XpQBCJQOE7)dJ%-I26tGB;9C>3gp6ggZiE=4&2RMn-Oha{d?cAy?#;{c7ea
z_WfIKx;D{X$#SKde7RG&((O%Q$|9@gs!GeRF^K$jT0LVQm(jaGa}Ax}_tkxs?x^eL
zb*Y<dVloz5dPUjEf3ed3WzEJplM+pwGg-~JPA@Z<t!b^jq_EV4_2OH(JF@+%i92?y
zT=_8DaCfu4^3q4YG(t{IGP2SSR+{x9&8%e2MfLbMhfU2Ue==EiS-|vUN{a%=R(I3K
zuODd0?2^&=+^ePQ<6CIF!cyE!=Xkg3@<Ue)SH75JVszrRHp9Xk_18=fly7-Rn`Yft
z(+~@6HNN{wRmt3~PQ&r^Rn?VNX43t;*^T-xBq&O5*rV0QA*pWJ{72)s;5-#pRyMhg
zkN1_i7p_<T+2CySQDVAU*w0UDw>&;*o!jN7apc5I4Vw)QO!W0;s6O88YpiqgziJ;_
zhw4@%Wz~0=ebh8qHB2rX{h;(_!z{(;^YV;2r*WE1IKpH4|H3y@$GJ`_Gwx0|U$VzW
z;rG3C6`i=n%AwKUOr<;Zl}mbOsr{Pzz%2ETgW}(ZDW-J?1r#`Delv~pS!){6chIc$
zEvw=yOE=R>*`;bXZT716|E<@Z{ClTy%<46!YR|e<j#|W<+}eN8Xa~Eo2LFa0mEd?=
z<^In{Oed;Hscg3BFp)Q(uF$|LtY+oKsiNvuC|5SQ$7qxOJ7uK}aVl$*uBn=ro>DW+
z=ukd?xJOostw!lkOs!h@v8N`v^NLjk_bgErnkuOl>@B3euTxDeNc6rDD{H0-dw!CM
z%>4N(9oCCgBwwvpanjLIy>k45$=}>tN?{VOl;_^?HBH%a-*myGcczBdbc{JnR;dSc
zpEtQ(->dMVIzf3ZcbC#t#~UWPQ;sV!t}s)zyDx8AxJ6d}-FF_-Q=dM_%PH?PS@TcS
z<WORRvDdCh#bco|CK9ios^&*kswC%iDqVQE!1NV+s}V~PrwYG#l?n62Zc~={%~}V-
z8C2BH&QN*99A=^sG+Eg_<CMwKk0pwXvf-+JcjqYSviwl^HTjL{1-WGT`6}-;mOVeM
za%W<eYIECsr6BJF`4IKnO1b$tsvm#+F`mZJt#T!$PlZvlO(8a1Lha#vFI9c%lcsXj
z+A1?*JIsEs&Q#f#wqN;^OP0!>eSE61(eF(PHD4%c6-`wNV?Aa1SZ}k*oXB<)=k`g)
z@A$h^6?Z>0{Uvorfqiwg^527#lxD`Mm}cJMSAJmFpmJxYu<6AG+zP_nyNr%ryseOU
z$;L!<#wMeMzaE>u^*^ZK_bJerUo2ZSD{z{!l*=s@rGrOJ?<lM>?R$4!IV?im^wh&(
z6AL3L%@vDJDTPG_su(HQn+2(-DNns)Z(QJLub^khrTS^(8AU(SPjcE1=a`x##w&>b
z7f|2&eU6I%hVLq@HH}J|L5gxa^7ks)^&6|oHpZLn`mLhUE|;LP%WA(Y-@Z$#!Yu1m
z-|bsyy7rN)>WrlSCOlFZs%PGCsCxN!sirbYE2#vFnI>|iD?3aUQx-UT)#T`kg{Gf)
z9-77Ki<vy+4b<2^y})edDIdi+zr~6-Y64_aTiMJe9KNg6y3InB^>UzDTjWuNIltpg
zA6%QJq*nahWa4Xivn`yl=5K^<DV;sL(BzCQznX`pyvh!yK*eRH*=8@c#F*TB)u3`c
z=d#KEOP*%cIg09xMqJ8$JMvYd?tM1B(7s&x!Yq5UH4D!vR#eog&a_yl)V%kz;*x`p
zP1fd2Q}fdjQUBB>rTAMyRyDoPPC-CmqC(GdX_Y9~`O0as>Sk}&-c!;wtdUweK~=eC
zpN7iofKIhXkNQnt?~qgZuyLB%S1Umk@zguY|9w+cR4-R49#*|$Ci`Dq$#<!T@&~6#
zv&T;J%&YiUn}4YiG5dA=f{M<gr{;1`^5s%LF)EtzD#?eZbDOu-Dk&NIYO1U{{K-7k
zYqy--vIpiJ#~c->JMA{x@PgG`#n0RPV(lu$nQ@EF*X-6;72DvYd?;a}ywU+-^IKa!
zm?<3EtIWM4(Co#!<>t#(r)bTs$x=Fz6sPiZ*D^DoL+6!NY&0-CmB6I9va3RcW#1+x
zrK_*x8KPRu<Yx;g#u$35SBBOryuH_{-2X^b!S|B7!lzJSrF#bsDMTmxnf+J#qxe)q
zMK<{;gYvd*CzNjSKTvC%?qKTmB1JXpqovu63Hw#2{^V8lwRcce*{Gp3L1~Za_U>|J
zfv9sz9m~4RzA)>W?OJDN5|LJAY~}n<)%I(YX%A<l!pWE;O01JimC^&|npUxhD+|O+
zs43hLHPug8qVTX~ozcr(#}zy)RvBm18=CkwW|~Z!uud^3b-wAINlj`#E$XVxJ=`iG
z3R6t0SNt=X+nl1ZF!zYb{+?FT0~42NzI~Xdvf|=QmAOtWX1jQFR5qMlWvr`as_@TY
zf$Db0=ZcdyZkFGCir1`dmXPAgkhkiF<|mZae$`Wp{oSW%ymYM`r|AJDt3WR0OGR8}
z6Y^b^9;VkTXw8mS(4A$aWE$M3R#bT1M5Own>XDE7X8vwcYWcRaRWFIlsFmv*D_Gv-
zFyoaERPodMsl2kp)!awj)GYH@tc9iaD?`VZ8`U!f4j7vhO;TLH`Ja58K&pJk>&2!s
z+7(oe6+Ka1z|&>kw?R^g{h)wFn#CmL;EN(=2fwqK#~(Ou&bn-p@~3lM=Akc-sd>M?
zqw?|=t9DV!Hgm4@AEse<UMep?)MzNOvd4nm%1Q5R(jgU#TzxgxYhTPem?Kqc8=cK|
z&rwy%J1MRf6DXmwDek4x#!gm?;`m<*ote#=EW90xGv6wx9Wd8bTzB)4!g1RvDyz=$
zs?2DYHj9ycsQi+xN&e^;1N95Fg({~vl&P%@Pc-{-%~U;AUcfwXOTT(eypj45U1RkY
zrfq5)Ws6P3yS^#vT)C+DYK5~!kIODI=8_h3MI$HEKBHq=-<POZ2A7!287l5q`E0yg
z`Fv1=+2qxX%BR12YP`FiVcujkNl|3ZM{}FiEb0&Ac`YV~M46e|eln}N)S^(>^~p>#
zS65?=T7`O(U6W3i#9~uVpDqiD`~Ot_$u2aTlP+Om&%DTB)tiZGj(gaYGv8&HaX$1@
zU*V%*+P7+o!g2ep>PsG{sPfnympk#V#k67S4f$w;yIMc8-PNqsJT#QP99RC$^Iz`y
zH6i6~6CSAj+qc|&!m8t{2HS+yTxYG;TBS2def#gT>NBekn=~EzrdlLaYijXaMlEgO
zMO9w82GzB^wQ5(b&CCLpGb?SrbxBFQc9kjj(r0GR7@wLxurfBAI`_AVR3?i>Us;KC
zMR~7^iE)Lp?a_3zZxNiz`*iQCEhzeB+F+`vC~@Mm>0{n}#mgKzX6kVqrt{}=nf<p=
zQ}k~9WZL!UjhgTgZZ+1Xi}GR`Z%tg{+|8;56jeV2Rhu!o&NVURc&(x=*PwDhE=<Ms
z=0{`Cok=R{`yQG8j!9MQ?zL4*S~XwsTESs?;rK<StX9d2hf`;(9TRw}8oBp@n!)S|
z$^{8O<yBU9DybgsRcngBVj?Wqs=Bafr|O={tD3EWF6#AV!D_5m9-E{x7OI}|F)-Ep
z(W&Zo>$b{&#adP6MMo62y*4r1YkOWfGP^}tIMdBkzxJ7#O8gbG%dJ6Xat%J}t1hux
zoRa8K*t*U^@m=^A`TOBB&1_{{l?B-(R3Ed*nJrUXqM-2am07ZZkn&-(r)JU3=4Lj$
zZ_ET|Oi^MNk}#Wj+eS^fTSmpmCsT=m>zAp*T@SP34;Pfb&CE0F{eHw`G4mnS-dBmr
z{)xX-&mCQ5vO(s8vZ#Qo*}+?mN->}6RYm)1m3g+mS6tBPV7eoMMP=slRch`<cNNM^
zcc>)Hmr$^)Z<W)!F-y7s!hWT-@>5L}*GMYAJIN&9cTGXbeEKe>YBNdox`}_xT<&dA
z>H4<Se5ypEs>%Z)Rn-%&s?G{8RZLH$n@*cAqau5^N~!SEG1Dyv+RSb`@|g5;@tHQ)
zC97BTvzoV<&s4Z&V5U4PTvd6G+cz^aiC;>hC1I*d?)RB;l(i}PpIKrMbn=ekJ(KrF
zpCgu-d~RKB?%V98=stCs$=ywkYH14nD(}V1G}zzYHM`xp)9kP6A!WV$TIS1-ikZm#
z3DIc^n5Q!NwXe!ahDYZ2|2L@UyYDsOXEjuG{P0}Wc3zY6!tk&1b6!m_y^tQM`t*6R
zR)yjQmB0_*Rrk(yQ#!nQqAdU18Ojnnf>i@1tTJbdw^muOEm$Rc!%W4RsL85TeF18&
z_K%Ee|JbXjDCU}|De9=OJm*#UV;!v$cUDF9y{fdS;@xSA9x+ptiY=^7_U-XERg`r%
z%bB~=r1~(UdW&3_8B=ACe9W>?<yPspiXCAtrY}#KD?PDas*?3u!|YnYO8I3PE6mtA
zRAu$eB+NE%pKmI&eX;2R-#*2ks^?7gH%F+Nuiv43vHXVupUHI7>HmaGr*7y|x#ga2
z68<sF^vHpH^`|O76io_^RCu0qnPe;$RZiac$V9_jPw|TI9hC%!K9%Ptf(mnfIGgNx
z{Z-!XoxIwgXALTR9zRs1I;EAq7hRVBwRMrw!BAb*|DQIR9G+sM(kS7jBA<UmC4-+;
ztv_#pT1m)ELp||rDppFi#`?u-s!?(qRQ#Svs`_kgP}toOYkDl$NO^xsgEG^yNoLtk
zt4+F9O-&zIrx`6SsL%*CWHp|DNmJp)RV(?n$~^gdrzaanyLc<RFECTxRp4yYCZDDF
z^X6Hj)|gzSm$Dp2;ZN3?9O=1dVplm!X*<(ZljDEH)oPaKs~kSMT+MW<tXY%f4Wp;s
zD$4oxnue-NHO;)FPwC`cZC4S#&ZsIet<mg>-a6%=>)s}cO-4%puNbS|_|2zmd$LQh
z_rg5058j*Po@O1>{L`AQXjZ&lRbjHWqFK)o`Ea)i<u_Nkl(+buF%dK}Q4V@zCfDff
ztQ@4bUHR~-PSsTxZkilB5TUBewAa*TN2uxt!vfW*K{Hjmn&vAAHOiWvjnh{-?8>E5
zv_{aZqJ6LF_pfYbx8s<MU)`Chruo{_bc!aIVwG*NVzS#!`2_y&Ci=5Fm9uu0s?Pr>
zZYFCJs`$a;fQhrIrt+nllg8%VS4~z=bT&1T_fr<G(=~hNoT?VZI!C2aK~Ck%MLW~2
zxfP}xylPc`X8W03zOcaTg><OS{*7mq*EL^Oee@yKEc)$r<r01c<8@QED_+%`p}HmH
zxU$2;I7I=2ezOh1Zc3~3FK93xQc+U+^IK(yh>&8;r%4JOlh-Iqzpqzr@)9>S_RdxQ
zwZlg~ead}Br{lUR5qTnN*Un8f-680$vX5(>X&+mw%DdYOlr1VFRHUDMRGKRO%~a>Z
z7NxAPO-dh*Y%yKDQ{4RZuLCAAt(<1-{wb?A=4~+dwGC4Ewm4mhL2s7g*X%B{j^4{k
z%A2xP!pj;>9FKa*PfAKP&UZPdz&cyPbji`ZCQsJ9G&_G}k%Hj)>8ATcGgNIwOOz*E
zvr;}&bj9R*nyr~l)pF$&*V$%GwQr3VZ;4dmIQ2y-e^H-`!I7<|jF(K5f3&SKJ^s5@
z;mX6MD%L8C6#p)FREV^$Hg#zbR8XDcr>=K*qtX}I5S5AyL&XmLKsi^|1xh!jNh&+W
zF_`Vvo2q1=_)4+xeXIPmGZU0gaU`jUPvke<_}E;1x=^!Ojrkn)<n?Om+-pqLlXq-X
zGn=buy5hK`^2607iu>(aP5G)qOfOE{sDC%-kO9;GcFj*8MU0*A*2r>6T~?8OA*^z>
zYL@BZkB^kP4V^U3=XM#p8t^Ik-#n?q`RA@$&a<fobEim}yzS&QF^&#Y;N*Q`{7%YH
z<L9YNb?fac>Lt?0OppFpWg_+PiwggpF4N;GH%(4$mejW2u~1FwX_N9q#iM4tMS<!u
zkF<>#AMcg(64|F7w{@4|V$OR~9pPN2D$S+}aW~Fsygulzwk+LSW6rma%H^faGIETJ
z%4vCT)jl+zG`+HBtLhUsJ~jDYI`UimTh-5L9aRr^-)WkjeMC8BvbDK>Q=&?>i;_z2
zH!qdF_ctq=bo7{IF?K6^m|atn+j!AD{%wNU_pN#+>|NG|Y)g98dslcH>%7`0|JdGI
zVc8mG#p4OxW+%K4C`q1eQb}7i(<G*JvO?eLPQyIDcqN<W6vHd$xXem)rOdWxoL9Us
zUCC_uv@q3~Vq2B-t}M}*R?lj2!Qzn#Q%<u|%<7e9pQR?5eNgDpwg0h3S?0n@Wig8Y
zi`|cPmBZ$4H%Y6Vp<w2`N~KgySt-ilggoEuX!DmV7Rcw=iED0H$ENge-!qk+8kO?T
zUy8^xFsxN7{i~+@@S327*M*5nizaz1O1Uy9eApkQ)PMA%%25thlj_s*s&jo`n0|ZV
zs=EBiO_iQ$MyiLVZB!KB^1}3*ldZ~EeL0nxpAF1(ez=-(9`-Z6y>6oEjVqs3oo{KF
zSER5gK8mkXT$pd5*!x!6R6F3Rve<V`)w0(IjAS&YD3m4cG<M2yQm%PD&(tI?%w%0G
zi^&nei%QA1r%nDnGg7;HJzRyuHCcI$&{kvhkB>|@$lp|s$`UtI`*GGdJUT<uIDkPV
z+$TqM;iuguX;027ACV3=EilVh{CF){^=?X@GS`pg3R7I9%tFs@RJfY<Mm<KCU&;Hl
zw5s*dtqLCh&&s`;RH=NH?~l?c5f>BX8&1kx_O}%*`8O%39DS)Y@d=Mwa!I4<oV5*V
z{Kxf8KNp-+b7d)4le!kJwwg0u)!K5JiLz0j(y5K<ikB;Qm`py=X8NRQnhCeuX~QKs
zi`1TlJT_@~zghloO}}!_<ZVhU*4?Ily46bQEuHGRY@((v?due7wUruOKgX(CqOiu0
z-*KDCCev24ps%|Xc>eq``d`+lzVPsJwWn@B<qlR%GhOp@i?M)Lj7sd9NRuGjJtp~S
zGgQ<ZR;%uGDpY1T<YKmP@*&kk&O(#>8?_W<ZBDA)OpaCb;Fgm;_@&K6jzL1zzdc^F
zBtc%aOte#7UV5$4z87a?s_w5>s!y7zD%!wj_BCX#$}hE@Dii!}$yM&xQY(G<K<#Sa
z9&=aOezh~*eil6|pQ-IP&Q`m6x>Rk)tbWy7BIag`U5_cnzfDv+7PQE8#-Y{bS#O>h
z8&z&LYj*puW_n_mg?sXDd7eF2lwAykl%=+9G<W^frIcvXrM^scy?IyxtKx<pF_URO
zm#OaBx5i}OMlDnAX~GtI8MEb|*-tY&_`gY=cmGl~&HuI<UJcL96j+rlTq^lh4&|y?
zsIi%vCWIc<`Z@i$Di=qz@}v59=Brl)s@>(dX8L%lru^c+4C-N*uPPRo7s~dT+L>8Q
zSuAHIepvJMO<mP#8J+4ZVi%OYsn3;}Rytit=kZ}xrP2fDuNb;ix_uw3YzmWAl=cWv
zn`_LjzO8n)k(q_I#{2!|#=nl%X_y=o(KwoEta16@S@q>Y5hh2aq?P~ZhbXU|b;p=v
z=`s_yr87+Amo}Iv1S{%1_4PG-b+%YO>C1i9Y~cqgzhw(eD!GMK_H$@ys@~@|xtl#t
zDPr|e6PDS08mTW>OhcTm7+d6gG&P+Zqx8@DiSd2`8O@_6J{mja`gG=;TWM^0uExaq
zU5DCV&uWu{t6Gif)~c!noy}IaEjCxxi@apmQJkQWU}R)s_4S~l{^#`?Wly`+M88~<
z-+d?6Sm}zZ(u}av+6QX<)c3vB)V%uloeJx;K!p<)Dk{FFpVc2ooi@7Uc~h;XQ$k%g
z(p*O&b-sot?|+Seyr-u9sR61UA2iJlZAnz^NK;p3pWdx{CHt|w&Gb-H-kzz-Lh1p^
zLQO|a^1N4?IXoyd5}8qFdcoUQ?U;Il`NTOh<o8I<P<(W#K|%kBjG6Q!Hf7&dYt=@R
zrDpvVR}{S11dZFu-IZ4I`y0DXVKfPRd*0MuZ>o}M$}iJU4nH*`pG=i+2d1m>^Vgcp
zY&mMiHrGuh`+kJkpMQ@`Qm;E{HqYx<{xF|iO|3x7H2KMCW!}GYOlmmqDNYt@Q8mgu
zqim&gS@CSMu4(x7%Zf&P9qOAKJQbS_CaRPk-J#&tSRj94p|7%;r>W927dg|kW6PAh
zgIZ<P>gzO4uezwDS++tg$GODpLaK&}L;F7SohNswd=(K@`Oz{_MKW-z;!^wNro79G
zm7OI6lpJg?m<8(JH(k1ElfjY4^9}c{_^V!KRcj)&VyQes>obK8RVT%7MLlNr#o<a(
zZAB`JTGpF5TIMQ58_m|g#xhmW!})^2TX`1~{iq-_{^x~?ALqrHu<dA7t@}A$xtM`b
z<Jwkz^S*RVv$NOdDS1nYnXgzLZQ}GvOn2SVC(8Bx+mttH-!;EBg+WC$B+|Is_p$sw
zLvGdPz|BhMp01Rgo?~NXer36Q!RI~=1qFU3rvqH7fnW3#X6ndG&3b!TDYo;qQpC-Z
z<{!LRl^!~(E8hEUFF!9)T*<E1P<1=+PSXqDUMTxopEtd@#a~6po?qokQL~Ekg%tVC
zf9{&jX3SM;m^M{0eIA3^#tEy<`qHPE9FP<?<?2#Yw^}G;Uh!5|LAdLre78W3Jj*`?
zvwwvr6@{5<RMsc_G?P8(tYELO!gxubzY^;zIb-{~g{FFT>rEqFQx)~6Z8Ce39HzQ)
z(lq547tW~(EZlGQ+cnf|PnU`E&U1TA->B4@$zNHctHm-!d0M4`N=wx<GdE>jWr>M#
zCRTyd6_!~}QJM22Ug>X#n!Njy>t;Qi3l&Q5%+#15b5e1$`vH~t7yA@qPVmZ|OF68h
zTxg)Q{mc^6Dzg}+^+uOuKW?0(VEnaBNocWws%%HB3Cs3&l?zc%jKjhlRhQPSRq21B
zuIjGms$j}8!}MLYuJZYyFG@4NJT-Ty{$eV=P|n=hVwSPRXBCasxLM{EWychr_^p<U
ziPV(m|1NGSx<X1>=4F_wqB^VD0d8JJ;o5^{XM;{F&pETw%qrE`Y-WU)*}3y2O6EPg
z&HD6J)ofo@s`zEO>fHP%YyR+nw3#X6W2M<EBaL{sFqwy6aW~LW+NARKTB^$2WA^4M
zQHm<2qW?|at(H?-In7ITNy1{4AG%hG0n?|M=f19#zog!&$;7=uQPi7N?GW=D1-<h<
z@~7wBQMS2jt6a6V&#dsjr}E?ZA@T`n`pQeRk11cvSg*Qhs;+6-mns$UDUoJ=hWk{^
z3iDNhHBPH+;#XC$?&3DfS<a)}kv2)GC;6G#|K#(g&PIQX!e*^BI=Nx98sEJq##Z9l
za?kgADy~-ktq|~bqp8Ka7Nw-h11e%6>c%JcE>~E^;h>+8Ia%T6OdjLge>R!yyU1%|
z-PEb*J!PZu6^3J~=ac^`s~0*cAG>U1_GZFHW1F;A<#Nt^lijO!n|jEUXqRfeQFhaN
zq++QnWnOY%nsP}>m9cT0y~0ujcGdhzpOuRArYab+g_y0cu~KONdO*W7w^Q-|->WJ$
zSGFmvy?#lys&R_a!y+Rku^EYGKM%i9WWVVr^JtTpqD+>zlI^hsRgM1!hUK3HRPQ-{
zH!c@-SB=+TQ(eXyqk166QSQ0IGZP8hCS~(;4$2bWE|^;UGci3dg~==}T}1z;PL;;y
zDk}r;@CEYkSo{@Eyc1WLBfQO6szq2?X@-&NsoF;-OrP8o57|kZ798?bvivx~c=snO
z)7~4*rt^j8DlL~(G&`cHsdn8wS*7TQx_aQrRI?w(ZYEDST9gZn91QoLn{LKqBBd)m
z#YyGjsyV8yI_+kfPP>#lmpnG=sDGsBZXBxG{c)eN)B#4Ngw2P|p8r^{5ahZ+^P{Y)
zqTfM(Rl_g46ci3Fk?T)!P?mh|qco{vit*hCKb7vvvB~e|SfTX${8^=3!F#IZuHwc?
zx)as1S{+Sv-!iKEFwRi3-rl2@nY2iCMbm0i|JC~xZ*#s=TEDo@tfXz4$;EYQW^2ye
zG7em>uU@_`%=GsUOPSgYr<5D?k1MHi>@=1>cTVZD&06&o?v=*70#g(}H+C82ep#cs
z*LSMX!w;dRBL0s}57gSr|DG|y<QmU%^|>6^)V_UNtU7Cwx#@<8YLnywDV3h!Qe)le
z9cF%~e04tUWKxT5kx?#LCvP_8zoQzP?|M_O*ha<Rx7_N<F%63K4?5)CbY_}0&b_F-
zShhk_Zm*kaR_GRW!Ip1IOf25=Cn6b?yJYXG{taj}*{6L*#rh4is_~*l3jc+>)$08p
zsh#5fZo<tcsggU{(9~Y^p31g8RVw{1CsZz-wN&wbvC1T650kRS`e#Z>=WNWvRxUNU
zAFXaSJN%2$B7uI5D51lqhq_7?xI{N7weQ-iC@#QZI-QAAdEOOu)umg@OnZOoE1bO7
zY3y%4M`^dHvoVW~pXrHri_B`TnJRMc;WOdk`>uM{F-PS<#~$_E?<{5$^|qQ6ExMyr
zeqyG{^sOIFmxyfC>DrW}V&JBta*a{dEJX8=vR+1$X{f7*(y<gr)wek-6gTnoD@4y(
zY+C*0hT0aDn;PF%yi&H$SfeV+WTyCSTa`S|;&5fF;}cZaesP&Ujku^>KWnG5w9S5X
z{S(5fPAb`|$6lT=ZI)W9s<?BiS<HeLs!8WtRGkk<tIpDJQL|baY}z$7ROwlTq*82?
zsF|<gRTG{Y`%D8jCm3@oOjX_YVZAB8N`QRP-ei@;+ttc*?CVVBg87vn#7<P3Qz&hA
zD)o=TE>m68qCNLze-+L*Y4nmZ{=9Og>DuJ23aQr^P50`}ReN$`jcWPlCZ#M6CNmpT
zX;b4#dsPf=dQ7x`_nB@FKd5<phpEcirP3->idxOMWrS7s__r9_TXM-ibZ1t3b<I$b
z_d~J#omXE?6I!bk*9+vTIek5<`lx8R8q<U(<(}D7<@UbFR+2MIRZIQfY<B&BmTFPn
z4Atmu^JLd<5>dam=89^YC5P!tzEBnC%Il^^371u_pWLN#YxNJ6ZL62be=$F0)@~u7
z-1Bm$l9Z8vg?vVenY)Fg*-V-129lReHGJ4g44v+B%g?*NS)SeCsKT5>9A^BVl9kSS
zN~(S<n`W{)_J@3@$t>fb?0%&`MUM;*rly)bQaNCH=HWI)y-z>P45l%wO%P2|VQ4AW
z5c_Ur;i7C}tlVC%>~5@Va^Z%*`QZ!sdVVtQDqEcnsytN*w`iZXPWiyYWhMvwVimPA
zlvRI!Fj4-%DWY(1UZh3qoy`jO{~gy%+nJ`g+9OQ$Mvk7s_Xq!EjdB^3qj@JPa;-Kt
zKOvN*WE`%jpru`|=zQ(HqVlR9)uV?t8+9#ytKQ6e%{cS2na1|z_tcHA{ZNlAoveO-
z!)2pTjl)U{qy3e5{_Qh9r@GaMJM)sM!pEzIY*Q7rUBjOlAFel%FZY|Rs`;u><<-Ld
z#(9B<l<ju>*6`LZFf|o#RhllwZ|1f7q`I=fU6X(iL6g)AOH6~_>M2DyCYZRk-PPF4
zAgOVoTUOWpse|cFdtT$uKbh68uX<vzBw5Kcdq<|0{x2hS-*|hK50V`wZV8eaGQTQ}
zuL$pxKk87W@u=~HYH7NG!rOlnjU7rG6?${$X<a!Zp?+fhUyZKRSt^@5A{8`G@~ix)
z+@c;YX>M}6bb(s!zsG7?ec!bruI6ZHOq{7<HKX3t`u%EE>7AQQbrKoXF3xLJ4QkU+
zeQ_^HZKb(|X|T1MQe?lGQnUMIlb|h$W_8BmCe3$KOfya{QTa2g%bd}9mJHwDDwTyg
zZOT&`Z<!_@{irloW47AM=}l&VtLqde?{zfM=({BUm#yDqdH*S6wv0$q`3=7ne%5xF
zCOuf8b~bZ|>WtG3y6-1!G!|P`Zni~nv&wB*UbC06#U>TkK548v6Q;7&NJC{>^fKc&
z%<3wvnrlsZ7#=COa`LHJcL%7Mud0&YdVs^U<7u^`sepyL(blu7al6*4-Q6%l`FQhk
z`SZmxN~@+csU0xnH!;7LsrvTQEY*Ma<|`bmQ&+DEovJqFl&SH(;1=aF-W4VWeyu8-
z3QAOJD}q#9-utMEcsiNtiA-10(0QQr?WwleL^C-PEzX%nt1=E7I=&9jh-x@un0wV(
z=1)SP@_WB^N}X>MP1OC{l-Rc#sotLAsDH8Qv4ZaObRA=H38gbB=7zSdr;VA6a*W>I
zeJyXE<Zi5+a$mKkt6AlJYQFmV(le&UY-L8<&loCyV7_WJNAj*oN4bSg^yCf|kMFL^
z(SKFV!b=w@-)#D0<Z;bYso&RCRp5TJ;!^ic1-9@Hrl%OoWj8!npxMTLUnMs9tZLS~
z8A|4XA#&E+nw6Y?H>u|Dlrw!4)vFS?X|;+);~j;+9cpUnxrVAWPxhEv??_ddCMah1
zx4uM0>?FTR@4x9PDKGd`rE?2R4kWx#inM1`ek?S_RPBMhsq=+}rltos8lPM0sv4~>
zWG205u3}1CiE`5_btTp((x%mCxs?Uer>bafb~H1Y880U)*J?UT=#}C+hiX&y?>CK_
zHwK!0xn-xQnWt;QwK_v}4&zK^@m@&<xkJuoRZ)DVCVp3yGj3O!D9tlB&EIoZ-6=*;
zNx6E4%BwOpGci?1Wv7*X#yhO}6g6u_RFy4`C^G+BE8qTqzUf-cSk-O;Up3*HPL=6C
zhg75&m@8Rc?UTPEEuk!R!&jC4ldai~MrW1jogpfdY8VtYBwbT&OWms)<lAaed{15V
zNa$RXBS!*MgI+FGsW7NgmD2vMWbp2gsh{msrBvy2N|!1gn4Fuk!}Nr!pZVm!62|Ve
zp&E7P51UF_waagCou}BesZUmN*HqKUDn?~FWqH+|i;T<)KQ2_XX6`ksXZBaz&|zj;
zb*#@!?q!wv%<aueZ`JRcUbSXX+x$>hMY&?B;^w`tOpkBcWfD=mOXbT$OOunUTFfVv
zW@<^xmnh%yzMyiKW0F~yGmFZ(vUz63TfQph+$~nU_h^mM>%fVMW!aNWS{9efY}$WG
zUHbbq#eMgiRg>E8C<J_$Raor4K>1AjV&&7z?wK4<D^~hspdvqGN{4EWQNGHB4drSN
z4&@q4Je5-UTert_1v9HkfaoUW?F*7reyY4xjDO8(Zjk(1QL)!g`Juos)6frJ%(<<3
zEOf3YnC*DJMCIk|C*~JpMdYNG99Fo*$|tWUve9g(W4Y3|>uM@?R^H}fEpl?cmn|)(
zPHI#X+AzU9J-Off>w!yVp?VRD!kZtNKdRPLUHZvaIX|ge{_~_Fv&Ysi%s<DzRgUti
zGn?lWZ7zD;U*}GLz7oq}K9zZiugw;GKcjTKzSZni?k9!kKMGZ*ZGWpeC6i6Q<p`sh
zQQuX$O`@FYTWxw3&NjSMUa`Abo-sR4K5K=blFp%b3ZFjoo4mYtU2$J)g4~I2UA3pn
zg_M`_395U1Z!mGSpQL*JzM$FeZwFOZ#VV;Tn;fU=x%iOcypqMHm%278I~zMGKemiD
z`{TOOOsegiY4gc=<Hy^4R2Sd$HvKzmpJHHgp;FxA?+RbO&N9{Xa#t37B(1hbf3K<g
z-u-e<SI#k2ZBAB-{3m8QMKZ_4FOJ)EotT-@o$wc?f%jU});u><Ws&4p{4|x>EOwi_
zX=P)fN~f-{>BT-X(<TOC4X@hQ$`>>RRAnEjnPoHcsFciSGTHOtq{6)~%T;%ON>yU{
zEUxhKB7<3j{369Q@}cTW59BDZ%w|_ByvnO+rCT8{^{Yww;-w!-BGOY#`J9(4&G$Vm
zKeM4n;RhRo@?GJhYQou5464mHtGcaUZOXh}Sk3fwrmAD5m|F8vD<z&3U$d$2z9>gr
z{ioczXqEYnEwbiJ4Y?LvWseL!)%L6ZmYQue^T$c~!xBH_eWl&xdxhCdr)Td_7I!|c
zy6fLjvm1pQmG(B9SY)2rr1WEQr1`7I4(8_r_nC2I<tRV8@ZWsO-2H00i$1H&`KqmL
z@v_srUMbl8z!rNIGhH5Ihutz3+kc<cTYv3|%GQ0~RO7`?nj0`pP?_-5)g;C3tzxa7
zfEwS`xvKwvKTt}^{c7&9_n?CLmLkoc_FPI_*L>8b{1;YQU23XeeA!ZEmjjQA_=hDX
zio2I9FPC~Mzp=kvy<qoEmBzFTwHUE)W{ppD)CDe2F@Ib(K|S|`p?WF1mb%sYEo$qS
zUmE{WxuW>R|BvEKMr(_w`FUm+-pnxzY0)taWIC(0N>$x*NvVL0_k{y0HH&U2EASsM
zvksl1<hjFB<EHxz^A$QxiVKyr&5!(HQBPmnWv)HH$#nTIZF4I|DTRmj%4T<qbv05A
zD%5YXHtD?F-ETU7X064&&{HZ*H(#1v>bYyYX!kUOpp{G1-c5d^q?UNX?CsJ>_4SW+
zOfHB_Q~0`nn|hnJtV)P|zs&I+4Q8E7dQ`(pZ)rVLa8rwDa@TM@dRDnRkyB3l?S7^A
z{SVYwr?OZ~b3CTXQ!1wBb!dT>mPv=YTGT0Zt!dkh=iYgv%JR0;Wa&OxwMSphtDaa~
zuIlusPVK#Uq#4Ug7A3Q-+myV&S()~n-C=fn(OfgZJ`b~&^3N)jZOj%0?uTWKF7>K>
z@}HvIbEnkI!9riz_WwOK<(bu{FLs|+@b{T&*5_KR_|$5b*>0ErrY|hC%-k7&D@;*4
zY&x&tmD+53ZZ-Dga`}th!KSYR6wDqk6jFU<RA4r>H`=6X>MZrIUs_b+&hx6QZLl%S
zH;-14Pe?L5qs*@8a@$^Q>KRFuTeBnOr-X8uMt%-d<hvZE_Okkg>M@(QYP-{_m7mVP
zBKIM!O)2+#w_5WF8<W(YM%9REhg7}i|JD==uvF)J+^UwbT*bubQmN`jQ(uz}N;6f@
z-h8VP+0?EY*BGfF<Q-!+L4LjR`|n}OtB&k4d1kfBOgSsZTu{W=tm=rX`s~-N7D2)q
z3fmgqDTFScBG;r<XSR?hQu+HsIaRB$OQx~<1&UvexS6{*@hkfrbT_|xbgkK!rsrnT
zeCL!nbw8V4duyV$hgD3)=vKK>m*`2;(ArIAmTUhguX|`<ws=Rc=@ip;%?|Dy<t^8k
z)E>*<FutI;Re6E!c{9zu=}J!v3sp~*b*N04n6K2o)X2p3b(FmLO&0Y<u0P~GTbNb<
z?GRG<|2I!QaQ-%B^VclO9R_<$wyf1q{<-p@jGt(x>XS$P%I}_Qsq0D_ntP?0sw&@-
zG{3bsS9Q^cdez$%^HtgF7pU0sS(~aGC8<QJGb+{WIBohTro*(MPRHzM#xE27$~|h2
zE@_#Ey_Qv)Get_-W?Gh##*>?7cYJ%4f0#C?CeGwEKb5mYL1LSM=?;Tpr6~#%P4eDo
znks!~w2)Ptr*vwXj`4&GK5B>fdsK4osjJnp37YTKJ!e`Z=d1Fp^{;7<QN3xw^(gIi
zS)3{#%z{-PPT6FhULm1!!()=^gLiWk+v47++C<+{{`k{U;q@d7v%5Vz)I=wUYw2v8
zq2l#vvTD|3UZwLtWferOS*tMok5M&9y=lIZDPJY%h>nWF%;$=0g%_zBnJ1~4*KIQV
zuwPH*hv8~tt?9ZdPu4tAE?nrSvhanLs`)h=(>qTs70rd~l!WI;8h1`SZ#uiA)r>oI
zjmdT%CUsfGHnTew%~Dd{fy(>8NGZA9IB$|NHBKpX@&c9GRclQz+;or+)_ZAY_F%R=
zhxjD3UH1jde&o5EO3g4)*mM4`srfS>)tyy4mG@_aC^u;Enojv2U>0mtud;Htlu5_-
z&n7&r_p~nW_g0embyNA1t&7Q}mrBaQ+0LfFmv$=jox7$oO?rdc+dcjYuLHD<x4Q@^
zSjNv&)7aIkGNW8Tb;=rMrJDU*@~^}Ll=kcDt6Iqz7|)6{P;v53RFToWt=7@Ys8+J+
zvYLa&YQyF0Cae7GE;gR+qNp1Bp-YA9vb^fub3KYrCODWfGczf_GHy`5Eq%!B!?_72
z(FgQR@9qdUIy|RBL;m?S!}Z7C%E!n{E3m(tte~{<r18Ih*Ok6o8LLXi9yN4O-=wIs
z-O>29hn&({mIC8C^B5D(wR?^1cbh2LJ`*;*A0novnVzk}qVq-dTTP)^icpZj$5cDz
zUmqC^H+C#GRg-+9qrH5h%BAmzRB{gJn5FUVR<6t0U{n(DRx$Uhj%vK0nsUSfU&Z%Y
z=gejWm&x<xZP5Ji?W1DVz9XtGyACP}{3wxq94Df@a3Q1e!e<61%8s(iZxx&51I+Fy
zyMJL(N&M8IdfCp`BtAG^^#+rJ>0P%-)jWA$)!+N(tLm-oRr>dIyGd5t3gvy@O;x1z
z&CP=Uh?>ox|IbwDdAV_ocB7i)?M~C_lYAAHE_kJ2RO>Fc?!q6Fb2}xJCka%jt`gj9
zdX@W!BFnW^rU@dmlvtD;OuJS-Fu9kPY}&iIT6x1ISF>HyQq}BdELN#;NK+}`-D*1R
zcckh5KYc0*dkRcs(wI$oU+>TfPd=>tX!1tY-krNli(0QL|KeU^bi7VQN$1UU)fu}N
zDkolUQ7rScHd|+TSdnk8vBuj2%M|sTwNynl@)Z@s*D6GuTBw}tHcdHN{;^5>yKv>=
zGq2<rTwW-deqXMlG|fmYE=9ofr%kQO`X5hBcrH&+F@Jwv+4)Gj$_%zoN>4T^n_Vq>
zr?e^UuhPftWu_M_51F->q?z?cbC@NUK2S;F+HXF2nxmrFH(4camw5`#8B5HL$o)`C
zzSX9(IpD6z!8222`_$x3zfR;-x;%T6sax7?(<ag1W_5)Z6tpu=nVejhrfQ<qt~~E{
zg#v52fa&f2>888<mMLq!>NUGK@1TkORUV}eRc%Vlb-Pv6gjh_uC(0<#Ot3TypR28C
zymp1k{j@$sR@Rpa=3+9Yr^A1#FqocITYjrTX~JtEm8P`~3VFx9<PVj<Q0hP9sXYJs
zDboPyyGpBGpHvKWYF6y{H&^+DYp2?_uzJ&4cU$!y;mfA0o9C$WX=$kc4m430T(VN_
zIP(INb(ve0f2vMa{AVL$dNg~diB|9n;~fkB7=*6Q(RAIp$9Sp!1BDEcQz|BJcvND7
zo|u&UZ&CjA%u%E2y_kv1v@?oxjiwrH4!W-P>%&8%p!?p&_6EC6w{4%PXr7{NtlDLy
z(bScrE<L+J>Cg*jvty!DO)g(xQ1$WEG<hS*Z4xv8wWh%FDQYnZqAK=rbIl${Ijg5~
ziy4WUgvqni>{p+*lvnwV@?r7dbL^&^{jG|ey#*Q}mt54+t~qOnaNJdXDBUJgaeAV%
z`_<cO{@L1QrJmbVWdsD&Ci2M2P1;|qKJCR(^$CBp%@&DBs$8DRZGP}XoQm4oLY4A^
zO)9PR#qyh9b(v{s%~2M(oT{X|?47x?0*{&cp#qbiFT@PJYbL87+I_=F`0s1^pZc|O
zYqhx)UR1`LWf$=(zx8cZQT}9P>UZ<7g5`Gu<3GOLN@?6?M!Zw6nieLNnSK0IqV)Z?
zs9D#?7}a&hS1F4;-l*>1E^Bcw_=*Y3?_lLxsUXw(U3zAX&l`15OrD~=xJ5~2rkj(+
zcUEC#gGOPK@6SXPen~G=`Q6>9^wzpoUSj71bIIDn$`P^(njgcCDz0C>M<wh(v%*hn
z28E|b7?m9jOqF++UN>K}X{S<O=M)8rIlC0vq;!<s9vn~+-L}BEO595I5`U}dv5mf}
z+f%t!cT7!Ft+W+TTrrWuZ2c4^72XaFl@k**%nnaVHd`(D!!%<5MAMavuBm>1ZEBu&
zr&ZDX&R&HNIeBu@QgNobN;i}R`pi_{EeJABJHfB`@-vg^bK`d9s^`jPo0i3xSZ1-C
z&Yq{M{ITn#ajdnGT8&SPiu^`jrQ;D#OhS5^OtZPJDBs%8Z6+rWViKDqrzxy-TX|>G
zG}W|uIVKO{*D60upJ3{$aYZqGdWx!GLZmW_hp}RFqJdd$*B<5grI*yZ*{l>7vT>^x
zuUf3IQ$|+d<eZhtnVhxCskPHg@=x|EPyZnx_c-IPLPryW3h!DLwW-<pri(-y)fB3?
znwal6uNL;VQf-@enA*`bp{hNX&zeZiJEipD>Ul-U?Ym4;@|;aMlCGIvc-(KeWK)IO
ziraHdisfG`@Wn4sUSl*xDWbT>bm`YjCApWK>RmI{%tE=D6n?oInI1pNpvrz*$mq@q
zE7K>PHD-clClohl?KX*e*r+~5e7Twi*CYApM{`UAzM2|Wuo$Tr3g#Jqcq?GmS)-#G
z&bdJK+7)4y%1<v%*<-h=>PQ=#uKlU25W3-%THD1Liid^lWu~*wGP!xMRPku(d`+g~
zLaL2n_3Gc;=PBLDe;{|?M^xE%&kWVdJ#$U_S1eS?^INBKcbkd)VqI;ugKv(giEY|%
zzVYJ(wNvUP<}RGi)vis*S3AsItG1S<UG*=&f>}48netk`xk^_LbDBC=vzSi^&NtIN
z{MBsV)U#?arB5xiq%{<yS@tSlGPt93QgE?(x-zTs!oQvBi<Re^|9#i2uxvG>+5Ytl
zRL!0`n~BevU@FtO!eWnhkpi1<g{dfii@N8+m1>!s$5kd@_BAg`n`FK?Q(r~sMUr`;
zY=CJg(**UehgPaOncYy1{>NkSQpZ;9c3Fp+lv2EWc^sqqW!|Go=Y7A+rWnea>Df!E
zZ=PPR>9s;twaTnn{bs{_C6Os-WI1CDl%1rvsV;u6XmRP&Y?a`RPgD+V*r`;)Y_0Zx
zCxd$ZH4Y<(mFgP1?`$&?wyM{ddtX>%bFhZSapyDY6_;~OCU4%ZynjxO^7LnI#tQp#
zObQmiH~DMYZ*uUyicZRDZ!_U7v*f!rKUR&L$E>>ZVw}m{IvbUu4Vs#h)E5}5%GW7X
zs*9Qil<{h`s-~EB6vmnyKlKB2R)He(w%NwzQZkynY<?OJf)jMSE`Bh6;S_1o`pQr3
z%)LO9KTG_KTs~N+3BRpZ@0z5bYIiN!&^;kbLsN6LiHohdLZR^{joMaGwF^rR$n=G$
z8-2JZr8N1*ac#Hbf$CqrYH212-cvcX_KsYEfSSs;4{y|CSXqq~K3r1E;8s*`KWC)#
zdgm04#03nRi(C3l_Z&}AeX~W|bZK#kYJ0Gj>gEU2RCoToq%b$n$Mis4f^t*ZROJIc
z;wD$xe9g{tE;V^)cF^=yrLh{D-7ItFYB7bXRZ5B%SJ=ouT_9jquK8KXuGd-h?C;~I
z@0%DEpP4K-QI=~~`nl_giS?QS6Svo=%=TNIR`RrsFg>d4r}j83MP=i{>xvz3Ell0H
z63t>CS*YZtE1R*O{$hN3-bJ-7OZ}C7V)@i!S6?@E3*WB%MA*pmj30}V$>L_!dnZ>Y
zCoC{m5<9udWX91QijOmy)Vs?46^c|gs`N;<C_Lauk=NRlu6*}*pi=DhJ*FA!*D5Vp
zbVK&Smtu{l8+w&r9=@hFqwTs`SxcMB4>?EkojJQy99mmcen>r4QC1FC=qofgt&%QM
z7E{($YMK4s?2pqDlZi4tCYRo88g29Xrhe(Xn<?`*Q$^{UYvnD>kI5^amp7{joTH>_
z)unRgl(pIMulMCIr7keuc2ijCsZN0L{m@y)fvURZCmeZ|cD)oZes0*SDtd9QGP|mX
z+Jod|bM@w>rt__>l+8_hP0xq#HokIdf=*47uJY;yZ7MTME|~wH+Nx|fU)9)LFiT-w
z2dC<UkDSVyDP?lZ1^3M?k9w)?TrQ)bS-)CQfPbyZ`_M)5wRt{rG1`^NmLElx>o#0B
zKPuLz#P!-%p)*lbJ}>;85@V2=s>u~K)8AV6l!K}SO;?EqsZ>m1RtcEXu3~*;r9xQT
z0n@)eI!f&prxfq4oorgxH^(ezhq3A7{}HBVuSu)-hM1YJm|>#O`!h%3X~$Z5nGalM
zXJ5!G?&hvl`MV;{?0CZ|dG&v~rp1cgN_FoJn>4yjH<gl%F|%k}plHjr#cb2>P}L=e
zI+Xd0pR4&L|1+DTsckmpyS4J>j|)r}i_9>ccJHz7q!kmCeHFx2)K6|U+hQ!PEMK_C
zBq2amLB(&XN|MNWrTHzK@<rk&%yyabD46w$YIM(9r8q(DjLOyKr3wKZrt%za)0D*H
zvy~*66HVWkBq$wMJ}bB4;Zns<6&ICutu|AQoATD=w4I;I8~tL_wa1KA!!PElBqu4T
zO2?j5oLV-~bnW*=N=_b!m2OO8v~UyRG`l7}*KCV?t?}F%a}6!#S>}ZrObV9%>*b>+
znk(c#Ej8tOGePM^jK6Aw@qbgFrg@5gyfn<N=)6@v9Uf?Q-}Inq#qJ2xnzy1#SL(Nz
zeZHuqmg3!@@;f3(C)VG<{Lk$zrptQ0l?_%hnW}A+HDf&FVX%GkRh66C8<ib`ILt%i
ztyPrg{x|9^{;Fty%0;z4P*^2-`c#GQ+-uBroJwSOCpKy3q-iSY@JXmiZ=R%h`Gu7{
zKX;7smbnhf53K6UE@`SMdt96&r*Lt#atH&b$}jD8s&C@+Ob^WHRZ*Q_Vz#m1u!^Wc
zk;;F$-zpvtJQdRFSDD5ueNggLysPB0W53xPw^OFO`X`#o-(O<5JY=@otfVQ%sv_4E
zvX<;oIR1Ny!aCko(*T|2N_t<9s7!ZfH1%m=P}nMX$K+4Cui_r{1{2<M5+-NOPMKWP
zxu9r1$IL{a{<P}jb<8R%HU0_<Z%s1uzj@v`TdYjEeByrNntn6Wf9KC>@d+<gHf!Tl
z{TTkztlh3mIrwzC@o7hOh0I56s(MTLmH({Tpy2#p$}DqLi~^@_yN2161jVfGJt`*|
zrYd;F$ja?7I;V7F_g%&OOVVZ>zwDLXh`Y!>>#tC7(%q@_<6o5O&-7A5{xTlb$U7NE
zVe-MMT<QN+YNbn5XXR<jw=7y~@{`j``Hf$O@^2S@GyT(frYd@COc}55(C5EhtHGfb
zYH%wjP~qL@5QQC6*%fYu^%^fWzN>U;im9qwwy23Gcei4Wzq4tFwu+L{#`PvTO(Ldq
zT*{30FWaJYqetH?XSIe}&#N#Mc}@-u|NIWK{B282<mxk&E3~tWG<<ta_C!|b)<4iw
zc~`bvb<e-+rsu=gE4!pGF)(b>Q=HropgQH}Je8z0TcyBz`DT;dk`zVdr)u{13n-=s
z#;KNU(pTiJoFHv--BvkLD^=;^!Ac{w6->(U>z2r~Y|&DwJ6^2(xaF$q)VdVIs?rH+
zeMX;*6`7gTzh0WIwo0H&ZL!>Z)s0`)n(CjbRr;!YQK@|26*K-A2Gfg@GG<kuUmG{x
zS5;qbdcvg9zeaA_k5kHlrQFIBQZE^Mv`<h<{I*6tr~ZkNz5O+X#T#ar$Sm2YIze-*
z@oIBk)AzOArVatp^1n=(O;5a8u6{-Gy4w2le#+qywWbAA<{PKa;#0Y?U&46zrK4th
zBKvgm@;KD^u7)XlOO=@IIc=l1XS;`Ku|c6?|6y+Re4aCkwUZ{uPKw-T*3Kua5LH~F
z`R9>`>NVMo>LHzM%F@4-<qkbvtu*n_1=T-AXN;c-omSzpWmJ9h;*~;<NTb@in~&5S
zHqJ3oVOCbz@jcG;?e*I#$}CG%j{SP6lDn2$rR15S>CNq&%Kh&*D><*;X*N^gvdNYu
zrDl^X`i<uNoS`vUh0*lSwH5N!`=%<T7Plx)FXJ@zZsAilVbfOqvth64zbZ|I<4e>{
zC53vFWbfn}Z=N*I)Kz-DS-Ro|#qEy;P2<0PQ~mKVN~I+Hky?xT7SmN<RE-1A<S2(7
z4mA#7y=-Rk<Dt&_-B~JMd!tpVEH;}>oqI^x;pJXardu3J7IUps7x$ScJ^pW^uz1E?
zGp+MG<R?ww(3Hw}r+krhj_O<P7R6lcXoYCi24ze4=_(gkI?Q&@d#AiyY_9SR$))PT
zK~k#Lwh5{~&x)FcU0S8O?yZfPwZLoDO65+~UsL2&jaIs-g}x9ry_&gJY3-tVrPpU)
znoi&R-jt)O+|1}>i}8-uQdOQCGff4i>{56zJxgViRk5<%wHDLshJ4EAxs%jH^LCnv
z79}YBckDE)zxZ1IX!#w}!avVUOkQ6z6*|?Z_`r_Y%&cdQn)jB~s`Z<9D)>Zqn0d6?
zm_7^Gq_Vsr#YF#tf*GGPx8@RaNfos~N0oppYfU%%GOHY~4L6Rmc`t8h%B9xtsH-?%
zq(?p|h0%;LAWbpxXOAlTslBRizAjMveIQ1;#QUrq(~r|iH!h^9)r4`H89d8Uov>r3
z>Z+dY@-i2A)jtFYsC_=+Yr0~|Bo()$<ECoYZ>s!ybwb5HS44G2m%jY{xOrxuN}njn
zo<69g;`Pe>%n3I$;iuAO*9zYoT+^1*NV@N6SQs%=p+0+;e3s^Q`68W%rW&0dO7AM<
zRkN-Nn$F(OsW9{P4HJ)(OO-@*(~YK;OPKu*urfWmNKfhgTyb*&Q8u+ThT$r{YP=dz
zMiCa>oU4p{moz9ddrdWpXiYIYVY5*0-&!7(Re|!V>33Kx)@b%B|GB1REVJ5DarYio
z)%R)|$|mJi3iI=-ET&9vS4>h_pji^DrubP|OErTpR>753RX!zAMcMQyqf*MPWoCPY
z+LclbpUO@Y5>#4x@2pbPuP)VB#kB_Z#!uC`H=HoGt~1y8d*+V%1l3pSO=gqT^(|wJ
zC5@PrFYFaka@erSxG|#F#P<(_*=j2zgTfga+R|DFjGr9NllM`ZrK*;_QYBIKqtU6%
ze&rJR-x>k>zfDw5Cn|+xo15|1ol@U_c&6#rCtpmIjq6R>x}GQ)EM+&nnQ~VnBT`Z$
zJ6v7&fAVgVa;F;8OCOJ_p3{ytY&+&}`tk2X?V7#5>Yu84R6Gwwm_%@>X$0->G}1lr
zLEciXRwIg^P0g))j%;&+n(<koIf|^W=V~3vlvdZh_D`dqd7;YBP;I&U@_ot~&o`^L
zU2ijD)t;sHsq~GS&52)H_QLTR3WsNE#LJ7APTsRtwLtQ+iQ`*FwU-91s#}!QRHreA
zs@0dcnr?d6q_p0&Rw?qjrpfknL9<uCznXA&Z#LyR<FDdiG1ojpGgDqexj{wo%RJ>9
zS$9nxrb;Pi7|d4N&3WAPnYyE5{TfNr^$fxaT{>Z=#%r5RZb$~2`JWO|T)l$FbZg%V
zwQQB0s?7-vx@`NWnwYPWG%G&4L1p#(Tc$JQ{~0@~3aBl-;;V9jlTn3P_JT=Zp@fRY
zVk1-D|2Gt#xCp3)wk%YQuG5o0Y~5tS=(bm3hsp#sZ??0lEpl7bzOCv}KD76QT)>)G
zrH8BP)cl@(Gnuv_O_k%pBGtFvJCwEv$f=*4bVlt-=Obg=DXq$VFBh7K7q_bX5bIG{
zrr)J9Q^ivCuhB0PPVEg!p4uOjB6H2moXi#*pZzXs;xT)+;hD~8jnxGLhFSNH$e-^G
zSN`F8PRS;a!9?j~y3*n4rmB`Z&l*koRIl)7JF_ughNM!_*$SiYvo9NI?~pNG+uyD5
zt6bf9bLL~!vZe+VvvbaB@8>g{RYx-!YAGft$F2%7e3>F-GWmqK&K<{6m7m{tDK{(K
zHT`&Jit>vs(+rnwv`}LD>85(fg+oc~t(wAm)%B*Ik9ElHjhn0)wcx4BT*uw25oZ=E
zIbJZ8-66AHNyNHJRX2IF$=_Y`RQ#SURADZCufTm+RBgd}J=MSEOr}$X{8fZszczJ$
z-KKKgSzo1g;xv^Lmw8kJ`7fLJOk`Ky^8TrkRcwIiR6Tt&V?9~3TmJWq^=AaDN~Ya1
zHSU&IsJs=XywaIN$-@1f=@}<2<;fP^DosH(W(OGarDt!oGBet3qQv~d#VqK~3sWgp
zQ?uk56BYj+eqfrsC|=cb_eAB`&r%BWFIbpG?R{YK@Xj~o3*3Dsm%{Fw?%ecJ{q+1?
zrRx)$RgNVKnX0b0RsQsUhKbf01|_>k{Hn#bUn(yDW~E@wu+=ocD@rzg@dGtJ^G219
zhqqOP{z)sHJh4E5r&M2gvuud!<pYOIU%$3fdGOs|#dy+I<$C)^s`gXns2Z)GXz;p(
zMK$XCS0kN%L)E#ztyFT1@>O@QRZyNTC1y5RZIROK2NRX{C?7Cgq;G2WT)5TT%um#~
z+cjFlk;~Hb7LT9o3-8H_i~@5Mc3vzqxu0=ZDLR!;HTa8)=@hH^iiImqnr+wZP|Ufq
z&Ghxo9JA{0Jm#<J`4mmp%{Tks^jr1v9|e{6;_V7DmB&o>oVjGW?(;;IvrMukQC+{y
zxsHVBe0Dsiyq(Ei#g2cv`OQl<Di5cdn}w|5R@|qZquTm#k&5};UWNL!dnRubFUmM^
zDQnDGGfhdMa*gWAvNpw1)y=Y-cV1E2J$;Gt+{^+)wb!0XiLBy^%E^n>7;Ab|WFM5N
z%@i^*N;el%slCZ=T(8fevazpC*(fboMQf*m(q-i>X0yr~6klswDhC@DnmwHR-Mnf}
zpv9iGI%YSQaj1GcQ?mH5TTD*n;c*4Ez(9Es&Umv3mRU+awy3EDeoHsAFM1^RNOz%y
z*O3>BTWao_^X#rR-!^rnX{_fr1*6%F7QgwWR4)t1DgWQ9qP`%w-*i<6o5c@@o64OA
zQD#TQe9T_0d#7_$Fi)vB<-PLv?=#J29Ai?>lw4}2%ehNIzq?39Wc_NjPsgO?3saR%
zt-Iyq)GarwWj+3(a3n%j<z!c${QnaY@+;z*l_vcaRLuA5HFT~2qo|bUp&+Xyr{2>i
zt~|d-L0xX;1)~caXQ?h)#c#^OcTRQoI~&zC@^z|ao)SvCbLN`PjG3r>&?Hs)(=RTw
zt0Fq)2CmI!_2zqxxu)5vcJK5t{d{7wqUm}ErL8`j<#>CxnA(^cD~H4=s-01LXDX%Y
zDy?&So|)P0vr49VEM{$d7fl_i?wafuDNx!H!D}YFvPErDf`KX@M~Wi<AvQB*hiuc_
zk`k3^s*a}8ehntFCXDJ)zosj%@iSAk<(4t6{C{6rHEWN_i!K%=_fsoWy$&fTZOau@
zyyW%P^r{P+!ivw*>T*rf6@RXNt*X7_r+mzhdkSi`UzANhv?+H92%Cg5vnW>wFvz4_
zoUi;|>zwjCrt4~}{aB2hExJ^NoSlu29bi{mxm`syvRg#$ovN7fk&XOj_1$}vS6*UK
zNu1tk(K#*8T<N*7MUwSD!|HWk)Vaeh8h?qpDnCDBo<eMCkK(=qk4-AHvXv8~_o@E<
zdeGGA_FW~%P$`QD%UMdShD*#}FXlF1^(@Nh+Q&Ca7oV6~{A}H=W@mXv<!N)VHh*h_
zdG$JX^RLl)D#nWqP2OCyGF!4sL|?d*RdwgaUe&$FHO-wa&QpntTx;}q*)he7DeP*m
zoFA#aztOA|aG}SX^U!6v7B)rABa;p*75uJM>t&g!<p0P>PP1o$vfeQ!m6-B7hV}pZ
zl$U9}lg~VQN!{<>Z<UDbe6@<oRP*<&TIz4kTrj`dI8lAVTO)OkLRIyu|LfE)Bp)z7
z6vL`4xO=lAkMaU@!SoxZ=DZtBm)Gl>`s5wgnzKOO(%ojA>=ea)Dii!4E8B51n3rTR
zDBC-DY8*bqV6i&OSaDnREK`Sj4C))^A2PclaMDEo+C1}PD>o`|Y}jJ5qF+yAzgv~M
z8&i&sL#MNucUgi(fY42qeTv`BBy1*`e%{n+p!a!!+O*sb<&7uJ%$xJV)VZ=J8Q1$9
zQRw)zU46FoPNmSp9#Rvz70mB>o>G0Ma!+e@lCv6Lt+U3L1qYNf+~Z{B9Nm=HZN93e
zDPm!9s(!C3vy7NpL)9cLr56?IKF5!%yKkFelsDs%>iZWu##<R>)w0UYtCq7DsBYO^
zp=Oj5ZMK~6l@hPQMI|v-X4Cj<1!i{Z70gq<>zeIk;!vHN%Vlx)@E*At=XzE8XLT!o
zO0+V284;?yq2-<$hk%gjjmxnLx+_J@^ViN%{BV`W{B?l7*{k(FW?PMA6+ZJun(|G3
zp%&x8tJZ#nT`BlsovC(kzM0tMuPSGJjLga%)l46YywqgUnxL|{^PO`0tw%-^#4}VF
zr)@Rc_Rmg1*uzE5shC|&uKvB8mB>-!8ApQ@x6Ik1_PX_@YB=`?wZ##k$}=u{$}e2I
zS!v4VHnoLoZy8C=tyNXfJ*jFtK|ssBMN^%%hEbidvDYZ+NwR8HRin}Rjnh>hx1Ufs
z+g`6agK42+Y^b|gilCozyta+9<Kt^4%L7`?bn<qZE84T0RrrRgN9xF0?4EO5j%^yR
zLeqCPxlNme%-RlVC_niwqxv~j#x&!;o>I8j9rMz#B&EIICz(&IY&FZA#%o^ed{Jpf
z;#Jel-0ErvIpkF)muINXzo%l#Y+`7>bibtvL-{?^9_?&1ZP6s{*z-%2*Onhv{ph&I
zG(F+2a$n{zv(NlWO1ejqRUO{lRW**Aq$Djp&uH&~4{{=0{_0w3`xOK%qf~1*vnakk
zrzJnvty|f>m0fwwd`qLn>@3QQ4JOO}$X}!u%6&$;Zn}~Bj!O^B`cFrz{^fpYmYLb1
z+R-{kb%Nm@RnCdCRBo>QXS!~UhRVZYVWqnHH%xgwyiHrAqRe-5-87lDx>@Z}GN<|7
z15Xw221+YmxZbVkCGx`TPqnRb+Wk(|ES}Tm)2%Mb|C@i_OfK}F68ns7)0mHSX0;}A
z7T3kql%*A7&35?*s=1^VsjSG4SK(t;G5>hvwn=lTk4nx0XVcx;6U}-4e%9hxw@8_N
z!y?t0libbUJ^ra2c%sNGFEU!mB=w!@489%8tNEH0*J$^faqQJsUh~XC>)Mh?l|>&@
zREyu<Q530gP<&E0L#1O$lPc@xMDr>6T`Jd<m{n%1W>GnD_^YZ?uBDoA%LhX>M+FtT
z%RL4beX=UL6W1%x;<HrY`L3dRbLv5pcl!Gj%eVI`J?PqIlx23?bWimrGgp~*lO<ai
z)mQ&KXI2_sA)6H9qs(dLs&sj_nF%v%niAK(xhnN+TTS*T2+H>hnwd+wg(*y)vfC`%
z{*oy_<5lD921T;%lDuY&|D04Si%%&t^Chb{-(GC&6|mDRTuM%*CiJXvu!ptDj-Ri!
z?_IpEq!l|+xo!DglL@Kj%5(GVOslVnDg0e^Q|0qT2ldC_^%Yhq|1@A@u~eM#K~DYr
z{HZEGzDTLQmgG@tjS7`Bw|Jw-uc@i(HOI*K{2p@^H<d;egSuJjLhbKW_pjwwfBS8r
zfx^B_mDBQjj7qO7sIHPIP&s^6QI%`6ozh!-G1K>r=afX<=PCPsoNHFa`^IFluaIdC
z^E*SX!|58fC(8`~6fRImbdOf}yt7fUZPkBc*)@}tuE!auo`{n+y1AuY(JSkau}St~
zr30lNCjRW~CNHP=8mgC`Q~avwX3C%}sy6X_gi6LE7WI!TN6nlBUg)3FF;HH^HOFw{
zQXP}Wyn?!WEN7@ROZ2I<UfpTBZpTLDTH8nZa-j@L2b(lhZ|Uz;YU2@7Gz|D}R{FX}
zt}e1zvvOj%6335Qs=+BNN~#Ak<#|fiE9D72RuZ2qZk)1NTUlbgmwe!#AXP>^4wd5*
zyHu@v-W#tw?x#APS=_{JXQ1k?bK<JHZqrmtnM;*V&zNqK{n<cSbEA&RRM~@Ozb)&`
zE|u*z{eM>7Bsiu)ZQ0%hrdz*>%YRWVQ&_W9N<mi9(6mLbSb6E+T-8QvP1D*do=Vf-
zZ8E)85U!{ux7zfBV3BD!x0mUHEu6~N<x9<OhsCO0nz=wl@~4$*P4qs~?fmaeSWT{~
zSe%_}!r9ntlq&FDCsOBu^6qPas?Hl2OqXW8Qog0nW+Io&tW-X!Pn99iSGgj`UoqOZ
z*)-2rQTf@W-Rd_2l9U2u`&6s=+7w0X$`t-T@luX+o2q<&y}yY@B(JiQ!!h}*cFn4V
z^$%6-Wy00cISWkIq?fAPUwPix@9jL5<W6RlAG11D=I35g+QAuWc6r?|rAzO=D6N;+
zX4>{N(#-gjo_WU3ho+O~iKs5jTxV|A)1wgA^-)pk|1SCSZwkyTSooCLm|9i7gm##I
zn9(VBY<jR6r*gH@j??mH?6ba^`j_aNyROeuyt(|ViS+bH)vzOt%8vK^72@L>O%0dC
znCf2HtenhXZI<rfYZh>(LgRkM8l}ti8&rb$H<+ya#i7hox4|sx^EbsXm$@neKVB=v
zZ*x^Vz+7tb%>A_Nx|de!Z`NrkNi~|QR2cIrJfG98koA*a`R23^<u^gAP2R<^DNlR2
zM3F%|SVhO@z4HC%`D%OxQ%#Lqoz;aeo0=Xgn5+KPS5KXDx4HVhPm9!QAMY`4NEJ~r
z_~EFuYsDFp6qDs9(l-~HPC9eZ(EhNeroW$p$-N$arPZ@esvIsnqAY&ztw}`IV&$eH
zCk<N-K{Fo)ZN=d2MW)+hPOGJ=GMRjSZD;&i;;U(gRgBU@Hvto^PsSQ27<1J#LbVmc
zkCvG&$n-N_6)mRPd1#iAQucn6#CZ&wSpkJ=>kB@qT(_(-3#>6ySISB^a*xkdyzO~V
zJ(Xpa@|F9~W%AYwm_|gbSJK{Hq!D+^SS>ouP$T%jR^^-<ujHS{xu{It`be!d=Zoot
zU;9+6lQ`8J&*aMg63A6&`gu&f_s2xD_2z~uujH1R-Mts5!V==GvbAE0$}V*U1-`Ei
zW`_Rq%0)IUN<aCgn7?9gG|kxVZ}#_Xk|Bd?zq-Vp65~s3Dhe$}brlwrzLa}YY-*-A
zBUH&-zC*<{@3z?l#au<-qC=)!Ix$KNr!E>zXqjlbX~$f%C+e$}S^~4piu~hM?>cN$
zzQeDozJ8Ui#fLZL#xAc+l=+P38QHDOGqZR<LwEYaX63Lw$tn}4GFh<HiYos#J88@!
z@>#wiWVy=j7ZplBH}90MeC=&fv`tX)<D)woE`i00YmUrO;rsemVV#JTym=R|vi*e|
z<*hRk%=H6yD1~xIOKYXrDBj<CN%>UpZk4;2a*Y|Aj8vI}R+{iV4pTjS@RLf|ku+5&
zVF#sb{nw^xDch8bPphbCX9%0U?T$8kR##|N-_u}fsPS0!<P>@H!{2liR=)l(KkK}p
z?8$^)(~p6Bm2)<ksGfVTZ*t%$x6)hvqo&bjca%J)G@JeV^}~d3zP0K3HWy{n&vT7M
zPHL%zUdm8OH>^@(`E<fmH-yd9i&s`<Uf31WrN{M5gCz4bXN9q=aD~>YuI^zskrG{}
z+`90V>4BLBN}Eo`sK$4hDtn3aD86>GG4tHLS3#GHNu#GkPtoj@zv@&bbH&Xns})2<
zRw<u()2}?uE!N~)e1mfS%8#-yK73LxNoiDZ@p+&cFF(V0_Kg~~cRj03?s8sITe!VO
zZSK1uwLSadRRfHZP0i%bD-{JDP+T5)!{m(8anp0hQ_UQDIt`yst5-X><c#s(>Dmgd
zx$~5-%4I73UHZ<1e`<qL-keVL+VWSXYWe+g*SzG-%>I8=;b?kothV!@sq%@rW?ZXJ
zDsGv;VwQBZN&PMR3bo421_jGm+@=Sw|1p^t>#uSsai;N;uAgRl!m?Vex^1dWldM&K
z-B2<6R<T<38EdZT$7!7k^|wx`oqoZkvVkRE{w#ZoiN?uA3Xh}zX^O`&tLne3P@h^@
zpme==io9NbkMh(dGgOl}D@`ss&Qjr6xmM+eg`ApmyRMpF?mD%*qE+Tz8PnC)S8<v1
zOnRY~Tu`HyGN(yxV_ShLNAPsB;)-tNxvl+5(<3!ZC#=^nvrS%a{_8rkdEVN?YC)NA
zEJ~OX75C`wQhrv_q7-*@uX)9C9p%G|yVTdnYFOOs+9|I$)7IQ;!3<Sy4pnoLtp8@y
z7oWD6J8y;JVF!1!TOV80&#|viyYkUp`7>*$ITyoDGvC5AmE0r0&A#l(GS}TVQ{~aW
zF4b2evsH3JrkgLQ(N^2gX>8uAxnJS<9Y%G@g&r!#yf$*1j2X;KQw!CMxt3@e)C#If
zsAjA0=51A4u_H`@`TcoiznO<snYG`WTTEWBQj~L1W#OYfl^kYAHKwQU)ufkk8967b
zX<RcbF$%d|uffbFs*%24MPrxuG4+*9fhOlzGF0ZB-K*UD`nS=qNv0;9f;Of_2fa+Z
zS1IUNn8uo!|4CPT`ux6X)uAsc$vgB+B-iw))U|4ACOn>I+__jnsp)R5=`34L4H1u}
zrnv?sCKulSGG#sbPBFFrw$Z)gGMd}012ty3wCQLHT`_*9b=ibB(o)T6zl@3MDSl(q
znVxDahf3AgC?%>cwUae^B$utBx%{<>wqmb>Gw&9SJ24@u>@)Vt{oSZ<=v=*8X|3LI
zZLL#&>cx#Jn%c(uR3i5A%imS$R=H~bO#NlGhw=2Ht7;3csHh(eQP=UE(4p~Pn^ALv
zfwif7Y?A7_qC^wn#7b4Ra7opG^vSA@qLxZV@kypWRkxM2PtQ>Hlb>&V)c%BNbK)IS
zskZl~?G{FA$<x-EH%D)g{}v~pa8mcPobp0Fv&$D4l-aD^Rd<R_Fzt?=r<feN)b#uQ
z!-_>8Pne2|xSB3}c-L&E;szxzy=f-tF|KN>j-;tf;1X8)+g)N>TQ}eI`J7&rXDeTt
z9tpZ_yh&1AN#tX;a?+BMs)Y-SO_|iTC>K26WxD*On$oUQwW_rnRw{EHC{yzOe%$zj
z$zc^c;hSn^yCx{?>(x@779Xc@H^f@u_O3wXJDi0|0TL%ni#B&C#fZO^TFYFbv9{%i
zvb@hTHI_v`%%<wDQK?+|#jNf385QaJ<to+3IaR&SH7Z1H`((1gUR7D~ot%<7&q6aB
zr(R>PtVXjd?HtCJ-aJ<4`q*m9ynT+sE;}xTLo<HMgbK%+iSS1%-MTzW<<8MA^NoK~
z6jVgm&B~WCC?!m>HCgyX#Pq=$ck@|xiArx5^qCf3?Nyzl)~eihH(q7goL2L5J2Xx5
zIWv@xW@i|8u+1}bV+_<0?fa&*;r=_7!*e^#=bq9~7P1O8Va=&f)aK+;W!nB(>3;fF
z`HKlI=KC%9RPw4lG}7{z70*uYRLS<cA-_90O5s}KE#;J+XyyFH-R74h7AZZNW+TTQ
z{7k-c-&W;cXH8Uj(ke_6B=0Jhh6tN<)dr|^tKC%gc-Eq#U(%!)`Eb5zukLombhFip
zQf*bH4558yi#;XG?jD<L%Cc5c{hEr8`TVe}^2Ze{749EURLGmcW43Fzy`uSs3YA8e
zNv2_I`{n<2YnYwn+N>0?VZUh`3$vMONUK?eahT%6=-FoX?uM%J%xP2Bf4pA3Z21SX
z>A!E89(|mm+`_fc^t?fiNozfuo}~2xWsTtV%9B1CnWbElR{s6i$238tKw<HPDJspE
zf|SGE1LdS!e9dmpo~QU*cdxny!)?VD0T$Jq`Zk3r(e3iv&sZzHpHZoldw|FE)DjP+
zUqxS~pM*?S+3YE*9Q(>i^|eN)X^@46iuoczGpQ?js&P>kDz|bKRWHOwD=qw+V|wT5
zMkP+}RZ2?|1uVE<{4*^P*>0-)HqH14zpI7_(-d>TP(1~UeRt*eKk`;w{Bw<Ie37Qo
zfp%}z+aX3~u1A{{eI_)SEHzP6NoiVc%DB+o)bH+8Q?rlKih?o|%!-1P)fnGosF*GA
z*AZM4Z~jEF-1MNZvU1Y*BGZn`;U)&Awg&b=Z&X%)2~c*7Ut_lDzqQKVEB=NHj4ml&
zTW_nn^W;XQ+-nErUU4iipUV|29~5Y-xu0i&l8l;)T13Db#l=<=<)$uQt8`|HuJSIg
znP%ekqRM}K_sezb6sS%(BB`>hWt-~fIwliK_PHvrS9F@bk3FMOeV|#TUYlJt^Ysyh
zF7+hSv+LI>J?!RFo_Tk&nM>1jGqsnq%;aBp8T|dXMvY;QgYnHS1w|d_84AnAR?1IN
z*lik`GE>R_;T09<+51d=gdWH{)ij%(T|HIt){A7*HDOw&t`qf5xo&+@{Griqn(*hi
zs#NuV<+@8p6?-0EHQV0FW1{q7nlg(uk4cY+jA`}5P%YkwsmkoN?5gE23(Veq^iZC<
z<g&3s8o%O#N9?N0q_!xn@sv`u53@C6ROXc5u+3dVQu?0)yWlmIOb2cSoh#E77~X$U
z>i-n1q;%BU^u9!fQmqr0obunTihCMGl)bD&RZ}O;F*^O4LG{hP?*=iNL8`%W&sDOd
z%2X$K?^9$burc9y!m9isv|CyG?{l+j&u5xeA9pZa{w2=v-=0zpIU_kE8L7z%f0jAO
zf0C<GsAykq%%rzgiHqA*RXM8OSY^=;#m@(KnWUZls;IOn)1>ak6O+Fu{~GzJtWfHe
zN;F%{sH)a@I#}gd8mHz)|3)({Ggsp&#}+ED__xof!(gRBV$c>{MMp!G9*qXohaX)`
zfA3hK9MrqmaDAkP;-$S_s^yFvDryJfmCnt1X*yl%mcm>CH%(KAHpSe6iK=>8;flv%
zPRrlY7E+Guid4ER%WTB2^+KsK<fnX?Z@#MhxoOJB7%r+hyEYh{{XJ35wIRp2LWx<u
z&}XJvUwW6?snFS~L3UG2&vkYv^@SNIn|ZOB|4|Gw6=Tpf<Biy6JYj>P`ZUHw6YsBw
z6f`R?E1P)nDl5%VG?^RPujKZ2jryl3JtM=SXaz6x`zD9(ZB^w}d1>ObV1<d=mUG7I
zX6;s}ED1B!<Xx%0dHW5uqK^kvoOfuO>19Y6yU$iqX^ZzXzJ6qy8S}nfIyc!F)Ovf~
zDLX7|GRwBoSNp`TXri&cL-E`N9`)b-iAw4LDY6bn_|2z&7M3@h6RUYQ&|dYP$_jOl
zdCbZS=d#Nm%&}B%UU^nk{hYgTkkT!cji0`$v`IB6b(I&Zxkf)$t4>fi+3-+N<>x~?
z(|5HuR3eNgsob&tqH;BbN#)=2TGRE{Un=<omnxObJ8fp7lxez6bfVcAu?<E~O{Qy9
zmcBLqYOgHk@6)EVG`Usr*7vC<EVA87+8ec08C3t8%s+iX-ls;&bn^XqO77C)CSIOP
zO^em@%`{~`DMo+#Wx6W$r|Om!fhxl3YgH{9+f4NfZy8VhH%a+_{6?ccFCEQG$~1MC
zyC<p?>s6?{le9En-f>Y`tK7!4@I<U)HLIoSyO=pj($C}++UD7sZ(gCGXr%R7qqXg^
za!=x9)#qUk6>nJWk=v-~psde5M`h#k-KN)#?<;>_lcBthcY=D5p_%I2l}W1WKdmul
zO<kfo-+i;GR>XT%&h9Q%<t{nZW1C#mb{=drJ>Q_IY{J^9<a{I6G$Y`qS@(pMW&)r0
z8&}m9t9n$&o7z8<R`~9psN#IlLs`^&t?7bmuFB~*CaT3uY%|+vASo|Bm&07`;WowH
z>YL22sM(p#k=$ZB=XtH7!HrpFU1#Q~H4Cj%RcPo_2-I>n+jB77?3?svl{S}SCUbu`
zn6me)Yh8K3s#2YztdiqB&4hK8fJ)hwc#{`j8x=AqaH*Z|V^NxzsIL$=b+75sI%Y-R
z-My-Ze0Qq)@Gn;TH$hi<oqU13gC>`9nMktQhRY94a=yo_=02RII#c<xie><-dM>w)
znw;`hllu{KRi4QlG5uuzR7EuUkV^0dMOB-dxr%1*Q_S*bq$q7$%%J?Z_>1|=qchC{
z%$A#lcq<!vyD)1w7dsni{PR}W{As29)DH*b`a2lRdjCl&P1aFVwFsSS`fQS=Lcg)G
zncbs4r6bw5O@uEhm_6Tf#FX`nnbILsTl3#OY-&Q2!c;8s)iqYe<ytiGHyEF2X;fyd
z<~Lq2Yk}!?_Lq9W_kSyIj~7(^eZ0$jt!urq`Rb=evwjOH?l;y{)d}KMK6{%>QS6<C
zh2oBN3Zl0wHTOmfD85v6Q>~xruTW@QqVRC5j<S2!U&SW3Po}dhQk0r}3+20m_ozr-
z_E4T6*`+FS>zhIK{)g%_^(BnF)GRdO3-78sJb9#k>~Np@qAh&JXF1O+FX4QrxMjJC
z$^Go<CMizxW=l5Z8OYXaXwOu?YP@pmCAr&wXR1y=v{&U)FPG7txVy?J%YSKHsa<RQ
z-OF6*lU%gfv5UvlB~0F!noQqmBENyp^h>6_V)Dl*(`R$;Y51^7Yltzc>sqSvnn-=y
zX<GYrhiXgYYeV<_xyEz&Id!hP1gRS+1*#|o_?Wc1t7>G;+GV7`8mn-$vQC3FXTGYp
zf|W#jLAK%AKqjR`;n`XZWzy=exqfTRh?uVOJ8Y873(?ogj*M&6i;pN8)e6s0^OE|c
zmh|?Embaph#&^$I8Y*^^O@1z3t9tg0vB~QyCbfcB?W&?`YO4DygVZ|q#+lAPCa$dd
zbhi@E+y>)p|3#)%-`AN=sJURO7U!w5yl0vD(_`BeBE%|HmU8wet2KNv4e3r){--@#
zO=l~+Y59Y@3eJ5iO}}LMD)RaNHl6d-*7T7=h1v95-xO4(>P#42m#b}U*`wMMmai*v
z<BrLn@;Ro5Uu;m(ZrNzMSijf!`W{EsU{({AsdJtwFK1e6;x<=Vh40ilQ?aWv71jic
zsmUx!P?fC~kXJUGVr<LFs?2A$POWtHDb@T<Th#1M#3=9am6e}2W0%spBNb`|Hgcv#
zmMN-Na+ayKt_)OttRbi#F#Dcbv)LD;ji0tFPxf~<nHAWj(tB>1ilO;Jm8`uws<YH)
znk?JIt*peVq@2*LWOl4#j!C?dkLjfQ&kX!uhily3*J1Fz_mRRira)zlhs%`aKk_zt
zFmJw+|1ERXtd}>9{g39z_wL(e5)fjl)U<P=v3a3@@v7ca#v4D)Q1mK!Y#eU%LiMy;
zy^4Io3{{qWj%H>h4-DqX1u9>iw$yO$nF+=P{1Q54LQyKMMrtZsKlPcGDNa<rYk$k2
z<?Io~g0F$9iKPNc^CzV%>}kGidf#G-!W@YJO=g{YDt?a_sY+j+rewOiO+HS4yOM!<
zscK`YhKcc<r7BG`C#eWuEmtsb7Ex=Ttfy-DzjW`(S9kVqTA#ePL!)=!oZrcI3=9km
zM_5kpF_mfBmm9fwUr|2e-qI`H`x>uB?bSWJ&SqUu>)y=1`FnLfmF`ttV!f~Sz2e^P
zQ-OP*>-6ls>=Uy0oSu>mgX7Y@&Zk%H-B_z(Cpag0ukqqZwx=h@+E4#ky4QA1^4`Gp
zb#`GqclR>Y3+(&-xpZ&4XYzj6`t*Hf%X{}7tUSI)VpZYZ9gQ#c>bw`QNpw29_t5L)
zy~RB%_sVdo*nP8Hz1P1gWbav>y>`EEIqiR?@@L;_(+>N3=4pF7mrdEbC7^H5Uyqsl
zw2O0XgVuQ4f6Z94r}(PY{yPaudxhms?3HWn-?zt;f1k8x;NIWaNqd>5H{1K!y6)Rs
zuy!wJ*CxA|mcYH&WKQqRUVMD7^V<D;9j;E?*JXEYuau13{s$jQ_BN|e-+Sh3*WT{V
zmc4Pci}q$0s_!#gpS*W&dH=qBKQ8SRdR@A=@5`=zrJ_oE#k)K9K8|_4H+JRLy%Hzc
z_wV-a+pjPqWZ$xF+xIXXtJ=5Vwz~bU%&Gev=BL=-`&_zrcSGRb6?10oRf^uR?^%1n
zz99RF`>t{%?YmqqvVTiO^4=}-E;h3FOZV1HoU+&O`^kM08(Q{ju1Vgz`P#&Npm<}g
z7O}s5vD2>B^nhK+%Qbs1oshQIXNumN{Ht`Ytc0MwF?-S8y_Qe+CNNIlTTwdIPT93>
zUt5sUUiOdk_bSYqw(sTD*!?#i9olz-HGkg~!O45xJq_F&|GjkYw}qMZ$FINKyCZ1s
z{><wyY|k*9vW_^>W}PS8ySKHaW3T`56#ME`rF%<$mF`_E7qVAvQSx54HGO+hzLoBk
z6ZW$^vq)@T@vpmk+jsu9vzoWay7xfo-g)<@*{jw{?iVcV+uJ{@d+(GrrF$>i{I%Wt
z!)X7uV|@DpK9=s~%Fo~T`^j``b{XD%|0|B|^Wko^>nNDD@BSVBef-Dk_bOeTxX*7z
z^4?`r#rIxhSGMnZ6Sensy1DhTpQU^6=RDfe@*!}q_P5f#7t9Uzyp<^0t8_kR-!F%V
zeF9=Zdv`_j@9R3+wl}(KmfaPH<h@@ut=zkFqS0Q(yifZ|g@gC@nI-N$6tKv)X@8_W
zvzg@nay6cPpLwJ9p8tGx&)4&%d#x_5-FIb8^4@7(x_f5_FWg%#p1Jq=vVeX5wx{+!
zDi*X4jBd4`yky(n&?(CH786(Q&1jR~x0QA2zDbHo_Wqws_c}F-@4J|=Vy~I|+`W~d
z;`@r9#ew5+%TxA!xmKZjZH1ogJ6gDHH{%gid+DS-duOt*-FIa79sAY4O7~j0&)@q?
zcf$U^4iomR+3B>WtkZt4-H+0}AAgqa)dA&Y`^|eFpDo=R{p#7C`95iTxBV#HoA7e>
zzOXgPdtb)|?|CHKxljLF>E2i$<9*B?%l4VAOWs?0J$Rprk<R`-U*GL{8!>I4MOfdy
zDf3J1v%OaD)8_uNSCwz#o>0YUd%t*0-n;5_(cVu#O7~tWZ{4@mDtPbJrG|T!d@tS0
zmlCpf^X%}w^K5%;r#|D@CtIkwcg~Cn`_3Qxwr|#E+r1qhOZOVhZ`-Rm<E-7JFU9*d
z&obKQ^R9I7+;+CTN~sd|TmDAvWiPw9PerwUU$8{l9>dlfd#7!izPEy<eXpNbhn4@i
z_<fT;uD4s@e|;a@Tjjm68!Y#4kn7oNz2DB3nRE8Om>#v=29KBQdm7xi_t3nUy~e>S
z_s&{VyLa=dIeQ~iC+%e^ZQOh6n%7=XUNO;_zqf?LcHd4r{k?qE6Zd(#P2bI$R%R2W
zHfLYBUg6#gw|n<$J6G+iky*V*-PYVTdsT_OM1{y+QNO=?ttKh%jriZQ?@z;yz0xy8
z_oi)sY+EoRc<-iLGxoh$oxE4dI$+;t!Jxh6DbaRO$1-h0&5rF=lijyZZJFkNiMvsI
zC)}L6SG;%azFTFV_O4`Jy?5L9(!HziM(+C%K4;IX15*2@eQen4?ayqp%8PSv^sHig
z9^O-Xb8<QMirMepd+E*OegBW;?Nv-?+qeJJ#=ZT{wRStsf849Kn{nTy@8WwuuS?z=
zEdPJc;esf;RhyFcUj0$J_mc+O-h{)QdlM`x_Rjifuz&HI8auT*=DqD%F8j6!ZrYz3
zHE(b8+T^{5iaG4fz8CCM6=2?L`DeF%l%Ui;<)=(GT$TQNd2Da&bq$l;x3y5kcEYvp
zy+Mn#><>f-*}F*!?~m<svrjt2x1Up+d4KHon0>E)&)MrHFvTwXTGxI%&gi{w)+O&<
zuBv3mcB*i1X1dn?>FbjBKDavF&fP%Wj-g0r@A{3UdpBPa1*iYT{*(7qcm?k*=+@iY
zz}UU-|HQVv8D`@4+6sYt`_?7zUAKOl{o@~#_xAm~ws&Sw!Cott%Drd*Y}_k;Vai@_
z-D!K9Ji6_+H}>ysoteBh``mr|i5wSgS^MVi`?PGLb%TBJe(6hsdk=dn?RD|)+S|7v
zd2g|&;NF7^SMK|FYr)=-FQt2<LGibAm0e!>gni15Ec-sB1@FB!(Q5D7za{(sWNg^K
zxJrND%R<(DPfsxJeJxzF|5Ch-z3Q6ey}PG>vH#<ey)Uq*d9SkD&fO}~GJE@#FYLbY
zXw|-qSh>9poMC%ctvk85e|7R+`;36S-&ZE@JrTvZ-)MgQ-bpO;_APl|y0`XA)LzD`
z`TK<@TJN8(_it}?its+c7Zdg!`OmelUaE2L;-95^pSTO})e)B7+qzGB|D~(b?Yh2n
z?e+creNSK6GrNf1hJ90bKijp;TDwm&RDAE*n{W2pKdst(OTBOJueVWq4KJ|meR=iX
z-pqUR!Fd3b50J6O(HL`NenHM@3uHbq>@^!Kx4_h~r*m3?*il!QcgmfU+6`iZaKB3n
zn613+=qeC@XIB3%)#FEZg4p=5q~21iyVI1ngVcQQe6_1XrbQXV{>gY0Y!8SH!p_32
z)*!n2!F8~>*O~P?AU-yH@~9%*Ue;UrV0U3t_sEbBY!)^-V#OpR+`#TyqZ`MItcN4A
z6vN&!51CCcHSXyPEMaUIz4lx5&Ry-DC~i6;E45W|$K9QpS<As;)Hc%|Y#&I^q;Ild
z_MO}qaGXX8J8T1q!|;|yCAfVsF&KSz%N(#h9iJt4-wKq6$1hCY^U`As_rv5)IXi*%
zg7^zg%YoehVoPWpwz6(a+YDmE@M5zZt3x89C}JQvwS||!{<q5f38q0}=vX-G2u?Y4
zwaEMep1UaeiqH70hsjG;=kK}_zhj3>c??Q?f#jcaeqRluk@2)Evf#9&^3`I;yjwHD
z{{Apw3X1vq+6Peh$ojFdL-_mQVf&loHaz_7KJkLZAL-kH?Zl=RS!}`7nPBzEV(9Fb
z$}7QchKaZSY9Y#On0jLA$BUXU!r;S^T5#F(o!JjV4rZ=ayRYR6F@J*ofT<^kX7VZA
z1=Cl4VkVruuxSRI4U-4aW*bgftDf)#yUA(yxg8)eWUQF|7o6XAgkIgfH8TdQ8OZ7j
zWly8T!R~~0mZx}n%#hU}v;B{#fb9l}%~95dr;oG?f^arS4jFf6JcozzioZ&5HkayZ
zi`484F#qe!^H}3vP}B(QKEqq9cJ0p>1G6t1$y+v+w4$i*lvraGUymsB<E%o#;h?^p
z8(cQA&IkkBBQiq@%tv+~I=fCPA0BQ+0em?91=81X!C_7Bvo5f^Sc5)V-Ro(B>(%`5
z4V=b6>hR%?PxIkn3R71<b-iUlQXPuj?-%rI1&6-|+chiqCzyt*Cx%`qIRWgpL~~xa
zUhDsdH#+_eK(Pm#S=N?Zy8J;!TVZlPn)ZXs^EE-8yYxS#gWbQQ{U11tVQSFn^bi{@
z55Z+xeS5^T<StFriBw&03Q|*iuua=3)k<TdN{^+NqpPOQEg@~^a8A`wcO^}(FEX08
z)5SIGm&U5EkyO;aJDpiKtmUK1`w!iUOu4UBrn0t~%`x1i-k>C8sP3p_&TFruW*(?#
znZNC@#n&Icnvaq_HEyOIu#~*mq&cNJ-!S&?5_LabL5uboKP+}_s#SDxou}Eg>a6lI
z;SVZTF4gF8oeb389KTsxZDN3CiS{%L*#>_zJH24dWs@u|k8KV#&b`5@+cvq-GVDo^
zT9)lnm218WH3iisS*AR@ZqylLYT5QcOzUkEg9-bhH9E5rUA6PqUR1IUe5&owr>|-;
z!AvQ7L6Lbu{s}E%KON1FJG?E=iL0o8+%r{s?qwY<lcg@|&))A-_xO;g#oF`2JUr1w
z+jmckdYhJ%7T<O*&0Uw7)y=l|s#!ZO)|@@1P&=_dMoV3Dy4vjDm$c5z-mImu&P~&J
zwYtHJxIDFsj<!ZS5AaxBQ|eOYUcFLLDo{k_Uu=!hg$0dTmsbQ@a(tSv?k2rKZEcsA
z=F@;<28EKrrYZVN+D|q-(VpjDW!90hU&SE6U&sINMb((~teP#+#TK7k-&+2??xeBQ
zk5K~@Zyyq@&6+NMHJhyd)=bqvLgSu?m(|v+OPYdfZ)*PZh_#F~l~!Y!`Op0FMpw1R
zJ9<pZPEOXb6Ufkh@sm^YD(75deUU!XwHvl+@kCG8jcj7qT*Ysq_SZ63qq$|C#az{6
zT9wMvEn^<%n%1su(aS5EtSw<Hpeg=;md5)JRhF6lAu4|TPc_4XwyK)Ntx`LBlT}Me
zTt&mYb*cFqyX_kHK6`4nf8TAf_fv*e`uyW6x4xaVjOj5mWBt^lA%4?X(`-k)ir9}<
zi{n!l8y`ARr!95CTKz~zmg3F7HCo%*tt<pDxaqW2Ra;bMtkrEY@=?8U{hFrCk6r2y
zoy*jtiw|gUXic<~)Mrw&*6Y+XY>n2uT6<m7^z$T@XX`CAo5I61Rc(be=1-cXa^xM4
zhVye)HR(U$2D@HKs84y-Y~g<Ffb#Fk{hHUF3Tm!ewBE3$;I!)Vf&|S;+se$%rOhmE
z-8DAynUSpBcVvdfv8^4NBGUV{{x9k`eetSMqo~YHvuKBiT5d?4<}~x^=CYZ;r5NmQ
znmTWDw1}`z(0msmW%<rnMcd;;qWa}!XAHN>hFR83>QPsI#;F?h<%Uw^Bqepul3U83
zUM)2Q#o_zk;(BkcF3@sRG|_p#MpJwGGHc6&hg>z6cuQ;in19CNc$K);3-#;j4o1f;
zAN^jU!YDAqtS@Myj6=sJjp=S(nj!P&X--&gqsY{ys>-~hTEm=ogXYwl#~SI+n=B7=
z25Cg9|I#_WgHM5%Nm3)DDNDyKOjO5n37=+kR<ZU=$5R$OKQC*?TF0x(M$a+MGZC_s
zj#{jfV$iB+Jy}Kipa!#goAZ9{kmoFl89mIJ%gZX|Up2o|vD#@T?x)zQ!Mf?bYVVaZ
zD$}J^bS;e<HN6h((SGxIjap~z3*8lWuWH<7aZxWnSERa>dz$&NL_WP4N6u*Wi$7Bf
zw>_*Oq+DZm#Qm*$ZFP#-ET-2=6R%lo8gkDtSSx*2dtc6Uwa7_lwW|LMn18rG!Lq=G
zRga}uR`YIgmchxxmYPpj>FY}D>d*;J6xGQ8b6oE@Z-b_zjF?v6qYh1hA6#;GK2KN3
zw@KFQvUXJuoT#J;3g-n!deo;JG1a}R(yy6$Xqw8T@BcNI)uiZnyZKvOUgoGJ)6}B%
zWm2HU-E%i}j2yP;9y!)%`7c{a`--`T)^BEiEoR^Snm>Bv)KAWyZRubkU@8AlUrS?S
zqq*!deT}<4Ou8~@`&93UN9zmqDwz3u%~n&n|HG24Rnz>ou#M`j<DME<mh~7+Z@90e
z>{FxRmC>cKSKrGpKIfRGp#_uqn`9~NdwHw05_fLUcz*erdE3z~##dbSYxP>L(^w-n
zRo#towr2en3$+U$3v^dZ+iV$F{Y86|WTo~}TS3e4y^WgNHfd__KOATA^stuJq0(;}
z6Pd25FIlre<MxM5T8Zaw8lO<_uvlZZRqHgnke=eq`{t7pd$jl-Zr2FPdT+Y#1+&GZ
z-*Q@l;!7>grZ#ANWRW!b=E9(yKQ&P^#_FJEyqmD4^aBBnqV3x)pZjoI#rp6Vy?e=E
z@i^p&#ocZ(!@QLoTBb*Jbs~39vDAIzYpLmBuJP`}63r|>ar3MjPAXUSpETz?X{+hX
zywKulw}IjN?mm+pA@wSPZ`!p$@nX+9MeFD5Hr2`7{;BgU)YVO!`$;`l%E<DYkc!G3
zMsv+<|D%>0t39+^YFDW5E0j@BuU(^2moQtc)}B>+g1f0&X!K))*}r#K#4vHHC0xB_
zabroBh0zT`4VU{7n(P1XGmCbfsPX=htfrdnN2OAmZnf1~U$i*#4{FaoETru;b+fwq
z34V=;xAB&m!P8WoCuOMoc@v<;dUBpdyqkdeX~F51cYIhh3;DgZdJoLgIzCmy_~i*3
zOA}v5jk`jzma6C7wb}mHX{MR{()`viT~jb<u~vbvsy4&5ecImd(o~&fE~u?cm(n<u
zxlnQXuU+bzj!!IH6V26}?L{ot3G%9&t(>Oj#8ztjWy3j3&P%3hYaYp|bp<FZJ2^Dy
zOz&8#bwDXXbG`UP?U}wOHIF+nn#rtJG+pV;trHadQ9ERrqJ^Y=h1NGu9}6Yj4$I)b
zn%Zm~hb_g8q}5-l1!@NT5ms|<<JPoWFRu1^;$m<f2)ej!@9h|geGc0`?OA;7=)N__
z1MC+hv)Y7AJZS5;VckBx;#OPdKpETii!=9$t8464o3e55l|yd(euzHXJGs7R@3pUa
z`wkk-+FP~g>fW9EqW11`Y}|8sKhwVH#?SUf=il8oA(4My&&s{FTy>xJekwAx=QJs{
z)?0tjwpjVezViFC_kCIad2h$gzc!IePVSXi%C^6zVfwyrhnV&rbaL98?{{bK)X#2v
zXWu)z_r74rp2v#E_UemU@Ave&X)78pv`=B_QXBKHNB25bTJF1C^U3a7mGoZSWU0Le
zZYAyAeDJ$nxlQK2$9876dOON(Rot8QozUO4*Xzk5+bsK)`|KNj?UhMWvF%;$Y}*!o
z#P+}q=Y4Ccoc9*gdDyR#>D?O@qqooHP|?2b$N%<jjO*UFD)HCe%e#K;-Fij9hKrli
zChV2jz8w-1_6DYz+s^*WweS4x9eZ28&e&IS(tO{d$+k8E+-LVK{5WH8YvKRBI=5Et
z=~2tEou#pEuW0XW``6W7`|f1l+WYv;w0%Eb$?okcXxnGN_;{~v^_;!IJfi#N%k%GZ
zS+Q~N;or9V``q>R-D~~1cXf`)UXfKAwpN$V?-sCMu=oChlD!5CT<w2^z1a6{2d^#P
zirxEOgp1os^SkaBI&{bGEK|2#E2p5X%%$V|c;y!FRod`!?;Eq{`vSe&_ewbC+5RjI
z-d!eWw%`83YTFYjTKn8Gd-m>@Q?PA3P`ID_CHH=>!&COn>onPWg2~VJc*Cl_nSROk
zzt=9<6Pt2v|LYdReFvHu?JhhD*n51{rM-W;RQA33m1-a7Kh<{r)75*ACn)dL5$UuI
z-~DpW`93Qf&9Du&O$V#@{b<qL+tA}@e|%c%-kI`C_m(+^+rO^+y|48H+dj^=nR}=7
zneTPmu*m+%;$Qn-iz@GXp}u-gwL!-|`{Lkz`y6BKwe^<n%X#tNcF&RN`_^n*ws+#k
z)q58{*}hL@BCD<Q&L-Q9emeW7G)~>Kr>?=)Sft5D>1pHcH4oeNwuTz+Wn!Oh+jpLQ
zUyRn;eSOP2_eB&*?OWQ|y|?kvo87Xql6#N!?%Dez%zoeU2LXGx6h~N<DY5TethLo9
z+0V*;aj((-WfwH}u^zs&_pLeqULGlpy*geVwmk7}`>S_v*;CT}+vct=<KD)*SN4``
z7Vc{b>)6Anz;Az6y<uN*Wwh<QIeq(Y^w#WU{!_WHC!lrTW#M0Y{_lFUr%y6`-|>4=
z`<`TP+PiAE_`ZG0SN191nY%Y?7qgxBjzzYI|2FSq-DP20;L2)$VZO<})Vw|W;<(@1
zJ(&8+Mo5p-w!>nhz5KC*`@Wv?wo~Umy!XSjPu7}x_x4^DT)0PR&D_1U8!Yy+O0w*I
zlGC^M(xp9n7p=A0e`@vKz2S3X_MQ3HvhUC8UwanD`|X=mdV8ObLh9bP35)h+{O+*5
zQ2uuB6Yq$98<?lt+<!4;{|d1wdxgbU?%}s>+`DEri*2x0r)`hZw7n~?p0xQZ+qmyh
z_sYF7Yj*8jS~7oc&Zi}NK75z9ZOW9{r(?Kh?=K4*n?szw`#hN^?pyWz@ZR(CO18g)
znfGzEsO^tS;j^oi?%Eep-o2OiZ@}J;)9w2olw{d9Uv1hyRj+y9vVD^Ks{ZoW&B!$0
zr_v;}PkEO8-u3TZ?sMCAa_`^I&-StBHSE2}XSpxlcGkX_A3nCb4g~G($>!Pn_sp7o
z=6g5T#K`U6`)$iT%ZIzG_PSo4V)dnds?9d<&wCZP%xvc<?A^0UsCMt3|L6B?nAg7N
z+2r;6jxZSPlRLC_kKL8+dy^*T?0f$@dSA<ip1t2rs_vcn$Hg|-PTAfw?YQm5)aJdh
zr&sO$EbwKo^Q--P?Kzg&&l2?BSDNr;&*L`#eP^R=_bL?5+h_JIY;T%G)ZUxzOZUol
zth2oqy?cKmuY~<KRj0krHecQQqmkLJW{==LS?5AqHeW~E&nB|_TCO<k4SzXx|9lT-
z+e?=%Z56zC+jxBtKG3{q`M%;1>3w&<a_xKXo4-%tr|@2vJIVY09eum!;WhXD6TM|^
z881uk6Z+Y2YxDZ_-gc3~J$@dR_R>X+cIs9pdjoW3_GbETw-#UZV()@g`|N)4Y1>6M
zaqr>3>c21Waqa$`A3t|5no+d(yNaxRIJe8*-_|@fpF@QY95-Xw*OmTr-=!ist4@an
zdpY`g_MI0ow|CU?+9MrizrRf5udRGU$G%|pMf+Y9vfIyOUuetqJ#HV<&XRqa@6YX3
zGe2+d_q)wzW81^s&+c~EzInsHFF)e5&B1^zd#X~4_HQ~6zIU4Oq<t+qS8Udo2JO3$
zGJkLI8D=|{y={8~?tI=;@3U*4=r3FAg-0jYZE$0;zH!!JZ*P6*e*LAKdxLiE*t?N2
zZ=bon>)v~D)AwrX)$KEw%x;_4!*3H*wab1*=k~qF1*h5nId*n`-lmp)O)ppPIh9no
z?{Ib7UZ&D-w&(Kd_ht%)?dkDk-?w1VnSI3*j_<8m`C?Ci0h8U;B+Y$c90&HzVfnpx
z(pTTT*_ML)c(s`KtE`@CA9}N6pWoB^eH@-W*8G{b>}CZX-Y+0Ndr#j7Cp%fjP5azG
z8}2*0qR6)Fc#q914~~7?mT%r0XsEkyo~M>=KyJTH)j|2amOPnzbz7zOT7Ofsm#b9X
zYhL<$&x%`XZ7U<?_L*qiwGB9P(T-P7VBeHg5B8qWUAiwPhJByk6>r-D`(?Iv)(318
z`X=p^s(fQ-xi)66_GO2CO>ui{)sl1V?Zxf)-9Bx*pNk=7|8LV9*6+El*w^s2@3~O$
zd{1g4$KK{%j(rlzzxOUt?cT#GpT5^wYtr5W9Oiqqissq$Sn=38v9Gq-zOiiY!G*lG
zuR`bV-Q~b+J2iRTUg5<PY=5<6?EP?GY45{wx4ppzc6-A@I(D(~ud(Gz;j!cRa>HiT
z#L2dLVUzYQ-?7oQZe^Qo59=-4x8kX`R_2~Ir;Z%6Wq$a?=G6Mgy;oh-Y%Vm;*~3>F
zVf*9Q!o6qx`uFaAWoE1Q;=PT_qfdJt#Kl|pNY1eF(%G<g-@=+bPo%EfmYvzTm$QGp
z?Vf|PZIi4v?QOV`Y`Z{Z@$Tuahiyx*bJ;HJWV7us3E%tVpq{P&(OBya2SK~VYc}lZ
z-e9)(#D3=8zuZl1ucjT}op|7nt(oB&+j6J3w)Y!b_f9Zju|0OzX|LIe$9tVq<ZSFe
z$L)C#{%Q9CQ}MkA(`VWUP1v#b)r?kKQGq(^*-cS42CF-E@7cP1k5>}gzJJNVwl8)2
zc2_uZ@A)37xG(r?@!kvr9h*kQm^~Yq#BE&~vuvBB1ovfxKi)H6|HkfHS?Bi7c0FVp
zHlt=Qhf&ksrJN~y3!XUJPR`i9S75>6y)hizcK><~*>d)6wk=B(u~q15+B=!=&Yt@W
zDtiSC=k9jeEVyso>9e*Ojm>+LqYv&q_y4s`ZL6eR@=_JsLl2twmK~UDtGVczO#*MA
z&7Bnud&T$r*h(mN+5F3zy|-gC-@cpvy?d|u%I`h8sKD02`?J;lGaY-Arfu9C!Qi;(
zTHab4sTnu-yfQvzqZe?})`R`rUa?z`ZDMU=_tvsJw=JAEZ|{ZPAlu1?2lgr?ZLv98
zCcf83EXmfi`{dqVpH=p$ZhpG=$MTN7y{t#9ePj;p36XkX!|SoaM$V;R*XiZWw*B)?
z?>!>IX*<tI(Y9Xg%Wht$*L$X2{cCI8B5!-akI7c{xxrr32hn?1vK-$lY1_BwhgsB~
z(8U+`99VsN*PVmHd$&c2?tSpD!)Ep-@qIGfjeAWb_S@P`6}Gi05Zjv*acq~QYxiF9
zva@>~o(t^tEnwL<%Z<@?vFhQyd<N_H8dPNLv67f<v+VqGTlZH^d*{B`zxS<&>0X1C
zMs^OS?{`JVB-rYvt=@Yf-e`}-)46*XzIfO*u=MTuKT&cY^EJ<XmwGw&m8?>;yZQC&
zo`<$a_D28g*{jguv^(E>_uiQ~{d?!82k)Kzwsx<YyNk`?Co*<(SPb^sG@Z7M%sywM
z#x1^&`{t@W#l=teJXqFht*&ovD?59tZOet2JzK2a@3HLC-aEPdn2p2F+cwXq`R-nk
z6=b_7*?jl$pmnyA1v~b7g-Y6eusyX~MD&Pl%WG%bk1~7r_RajUcZ!*i?FNZ^HY?g!
z?45ID+g`)V+iZilkMC7F{MqIQ|MtBGcS~(_UvAqQ-nen^41fRKwX(5$?oDvBIhc~T
zTe9=1ZNsDMHW?Cg_NEuv+ishly7wlN=)NQ;*}bQdZ|_~$wQp~2jQHMo?j-B;YrJhF
z0(R^@a<kdCz@=|b-gLQr7xot0o=;n}H!0`Jt^}@@z2+Ns?q#_nY#ZL2y<>*#VVhGg
z+xK>sud=O}tzujAEy(tQ@ti&3VxMgvFE`mcmvx~{THk5gU5(#%Ta<9xCS2^cRoo=A
zx6`C=?+V4|d!v<4>=j=SzjwFJlD#*!aM-N7_|103Sst51Q3q_U73c3&TUfQX;ZC3J
zGij~8JLM(z3Ks3MZ95rh9T2v9uh_%$dm2CH?ah0<cyD*$GTT4D7Vj;XwQ=u?Qqw&a
z`?GANm?zo(zBSdxL1@V?0gqL-)e^h+GT866o$0t^?~xhdwhK=F*t_Var|qUDhP_M@
zU-mFWPTZ}qqsrD;z`<I1>RapFFFZE=!U}fR`X<>br5v!W-E)8E{Nm=l4NOM6=H5PM
zyJgKJ+oXEAy@9(mY&S73v|YniW&1~c{$BIT*86VsHty*u4%_>#<J<1X_ulOlN%_2&
zHzn6Lfrr8Nw(26=-EH=^?R^fm*N%7XWe#~_`^u+tFaP=_yR*A^ZKH4Z?$MFFzh_nZ
z>AkO|T=p)=d1mv2efFLY+pq7wc6F;wx|8wV5{?zNEmm`EZ~scOb(ti$FF)z)UhOcI
zJt7sdwt8vP_WpP4-gDwz<KFdZt86U{_SjCHbbjyrGn;o^U*TaZot$bj$z{@B$+nGq
zb)L=LJ1Jz}-pcmQy{fY6djk$k-m_cd<K6(qzk9A!blFBqPO#l<@pLbXq{7~ZG4i(U
zDy(}autx7a$iQkB=c!`z{9Tf5#D+CCEHWZCCRV%lhUs$KntX_|S=FRydq8oy?T<4y
zw%KOFb}SsVdw(jn?lsT(v!|zgvXy%2=G|XB#rOS)o?=_EY2F?mp@}vQInK5-<~+Aq
zkf31Wk~(kilZZ)sj&#KCvEtjfms_`g?|dTx+uf?)_O6}iy*I;bvhCl1rF&KCm+fWU
za(=H%&tltaQ$%bmCNtQ!nDp&=mS(b7fq`S+ud0YW5jns2T>ZekcSf$C&GD*^J@1q^
z?OD;kXm6fL$e!kq_Py5`5AWSjUAu=N$=doH^RB&114H)a&EVTtQzUAA#z)CcfqnY!
zC1-E$IgpfPJtbqdjljO=ySL6u+H2Q$a!=}lNw)pP{Co8#+}SJStiH!h)z<dg%ei|*
zoXc&08Jx4_V4k=~c-{BCGjhcDNS*&@EvB`=w!r70?SWaI)+x$Pd%rvq+UFyB*Y>8|
zwY~BekL?u_J8bLWC1uO{C~MEZvNg6^iAVMxdRAuZ_i2-DjPPCCqf@l)Sas&@{j*od
zcCPjIJslzw_s&*3W;@?7$@Wsm9NTmHclJsgW3Y88ow8@XP~4u3$J=(htykI`qH@*t
z3iHanclLPgE%+#~mouni_lM-x-S0%t@0IX#+B0ds*uFA<g?%iC)9nRsov_IjnrD6Q
z!I8bXZ<+R5bV=-cc#>lut9jSnCrRb|gyx^!tM9$aCe^5AZ<{3lKFi+=>@1k%_8xbB
zWLw#pzJD9Dj;$w~pMAyBsrEDH@7^Q9X}|X>+qS(^t+woAer&hzf^X#BuZw5xulUMn
zS1!=KFQK!;_6HxwzWnn7do$P~Y{mW=?%}qvwdIbvx7VkA;r`$V_Wj2fo7oEa8{7FU
z?X}Z6{bgUdMZLY)&*OVd?rhlm`s9_pO5a!5i#>9)+cT5j;n~^Wdyn1jwXM4-XQvlp
zzISK$ihTu3OZWODi0wN(?c-kI(hK|7DQw)AxA5#9&!2tv-(}ft_t}N-?SFo4kL^63
zeYRpPd+iP??E7W0*#7M-fxUw6jeA#zGVHtDxN>jIrwe<Xgv|Dxyue|zPOxX6cE_W=
zX7lFQ%$|8{-?82ed-)EQ*nD5`#g_Z8_`X#!`}Wz+wY2v?*}gAx(u}>v4UPNu{JCcL
z&SBv`xo^q#<;z%X1JhpYm1Q>A*SM`~@6!WJ`(x^6?R%>fuvaJG?fwta8?7~Eui3KC
zKel_H&eT0Cp3S#+|DJxpW^uEPa9i_UhEq&?-A??n`(9#X|JCB+-dpp&?-eecYR~*>
z`(Ekt7kl^pd$gZ1>)_ts3k4natL(5jcg)#7)o#N+L%mhICTFqk^L#74Pb1V}zq=66
z{ufKRY&Wfs+1q<EcCVUC&)x$wmhYcYF=KDQ!Os2FZu9Ii1g_g|n?HSTdXm__Kda>S
z-P#&rzb1%d@0nwwwiY-2_ogWy-F3f>XBRsMmn|p5hkbiF1@{@u`LMU`z0-j>(KmZd
z3M6dra|G`%Tdug*`ce2^KEJJdIXCm~%X7cJUp7)?zq4HLUV)smd-rG+?mNdTy*Jt3
z)qb~T>OQ+NS%>d@8~4?J<lVP*e%Iav`@Oc-^JDf3znQVOdHtOIEvt^~*-*@}SM6}k
zp1A2#_FAP0?3=eDdhe|j2lt-RUc5JWYti1UM-%r(XiM9i%6-1KV+Gqjy)3o;)>gcG
zw;Iarlm5cL*ZJ(MJ&D!-_B1V&-piS>aqs+$jrRF3Irg~+$M4(!bIIPd-P3GhKXmMw
za%Gd95kL37+L$9YWv8_Eb-OLy8@A10|K0-|_WssqvaA03*Zv>#vc2x9ZFZ91z4jX9
zv+TJRGJkLPl>7U;Do@*pe0{Wc<I<CR1N#~G8S`D+Ys?pBlOwy>)-J_!-@ojGdzZQ0
zv$dJUwC~95Tl*qDaofD%IknemO47b>F<15;<q+H3;`Ypb;^ck%3L0kGndpS?wYzhF
z?`-298@=nDd%ZWC?_0rjXK(QzcUzCUd-r5yPTd>fyTn#rqJ7Vy_i}sci^KM=e$2kF
z<GJrXj@Dy)Ul&Z@!yB__Z))+<{gMhfdslzIvG48++5H~^pV;%>U%M~l&AYuJFPH5t
zQ<}ZY@N3uJnQLWjHOhnbF5S3%FS}aP-e=QJ*m})$-COB=V{h#@zCGU@Qf+3hbJ^><
z>h#{vYP<Gj-`KTRc?a{}rI|*1AMTyL*PnUTKCR;)>;z+W?A2JjXs_gC`MtCLtl7K#
z;EKIXujA}x`j_vUa8<>&{V>a3Lzn11Gv1Wg8fvuKdbFO||H<l-^~Hzp_VO6++I!G_
z?Y^mspZ9uX&9gK7b9k@Dn*}z3>tgnO@@lnRR41{IXWqZPauGaxvzm7AopxSy-#v$u
zdsQ91c5$~G?u{(r-8Xfi#J=Y|l6&7?@7>$!{n++c@L%i1ZTt5HZ<X3x7Q?&OjEl)3
zh0nt7^mGwBJ-0b_tSg!J32n07x7>HSP35x{`<hh!_huOg9GE`mi|r*DyWP!d`}fTe
zjooWt`hM@}^RxC|zPQsaEZKE`;i`Rmr%u~!&tI@&-=@Q}ZB0bF_v$@6vG1?QgFRVm
zx9;(1@Yt&qJ!|jg;|ndmJdCjmYk6j)-QK!aRgTqmf7RDLN1xi*%|6JnZ>>Vnz6n!i
z?Dj4AWofocY@fo?=zV!dl<m(3`RqGB_r~6PW`g?_&)(mA_V4PwuRLb$57yjcvzkx9
zcEz?!*7wXa_PXqwzL#U={Czd2MEASO3GdT&PTkwulWH%y^w?gBd9U~SHU-&FTJqM;
zlK1f5_+20OcQ2ILH$!**9vLzFy+Y5n?%7iMZ|@<i5BptGR_$%?7CbQF*A)9(@2BlK
zzv+to!x{4S4QU+ve=Nw_=OK`8`%vWG{_FLT_F}$ndo`{l?`3A*ytk_V(cbLjc>6S+
zH+$1r%<a#c=-W-xoM#uA_<FB^|L467e-G@nydYxx<N6ZoRa3s~P56Fc&yD(J)=Q@*
z+A~$=?LBj7``%-bYxZ$XTxokDQFLG5@ASRDjBRbM&fT*&aB0I{k9pm@_1eVky4LsG
z#4i50S4Yopud*MD{qtvUtQ8X7c87%M?r-B2v<~9lxc9Tk0-MAfy}gS<rtgi6725w@
z)x&zK&Bndkde!$XoMvNh_BC-|I*0e(u<sXa^o^$3E)KCgz*h9#?!|VmeXSMV`x-vY
z-Q!pywU<fi&HhN!S9?{rb?;~YqPVwOpk+_#jsRQRvX6TwuYSCDgLU~n!)e{Nmiv$H
zU6F0K@2Gmy-Y1RDHoxjG?|sK|);>~XyNzL#&3=t52dw$(R@tULHQsME=fwUww?6OH
zuGQXWbb)8DbIIhr-JcTnh+KWQx07+Et?A8*J$5<|_fCCu%C6Qe$#&cQZMN>arS0Xv
zt=#9rD8JwRQN3Lpd-vW)65H&}`Aqkk*T(Lt@jAZubm6o;(i_|MW_?(=_rS{hy<Pfc
zd%ahk*?Z{9fxQhnTX$ccGiy)R?vB0t7I)aX|9obf{zKVT;LDFaYhrHhe%`)t&yDOC
zHcekw@0Ipb*}MGbl)W!1ci9;H4BRWQL10fo!^zz)*E08RUpL#<<U+FTvZ9l=)oYpT
z4z8GIBlcm2?W@ZfdjpwD_A)3m?fvm$>E5&NHtkIgP~P|9?Ww&iS37Jg8;{%aKQP(r
z&@<6iM`@nzt(J*<>%ubk3Lm{~ZBn++w!v)AUd_LZdtMqi?e*zXwS9Mf@!ndi3|s5(
zi}%hwxnWP>LT1|y<%YHwrJ441#7WxiwN=@7KSRQHT6)Rebt+POU!D4GE#n-vw`0m0
zn~Ec^tVQPg+HTsi(&o$2Gkc4x%xpKXP1$=Tl5<bOx&zjeYS!2)YRlWMx!G$wZBhN+
zyWFm}8|KE@&hFo1J3aR3-fe*|_U@GVuy?xMtUZ<mTlR__oxexdQDX0j83{Jm4*2i2
zYwO;d@_e3+cH|`6?6AVUBFD4#&d%@HEB#K(wjt5S=ECAmTZe`THV1Ytvw7a(y0_((
zj_o0?RePNNwAxxkEZh4pT;6WUboSl*)CFv(Y<94<_;=m*<mA#lA;)xWXZp?EoBZ{>
z4ga&*wtR)tZNF`?+}rau*mmO0*uC$cY}hMiz_M5J?BTu7)Z}eNthVks{!!g_;i6(2
z^^_|%4mb8%iLol~y|Ck?&8FgwdwcR*ZD!ruXS+_iz((Htk!{~AKbvg(1$&JSt+y3k
z%VK-PZTH^02c>LM+uQceYu;@8C^2&H`4u<!_O(siyC9)&w^G=ty>i_T_H1Kh-^ZW0
zXYa@9zxP(yweQt=^2H`&X_BqynUj0gMp*AvP~+J9LvgaL7~dP4kR#vsPW^XhPi>=z
zP1w^@wlh8S_szL_Z;zn<zrCBPHEg_m`1ZN1OtEEraAEI-g2%hH79Q9eVza{b%=CGC
zmuz{t_rgS8+Z`@T_jJlF-1|?V%eE)?^Im-mF`I-PTzgZ$9@rbHKY4Gg>IR#$Ev0*>
zO<%Hi-4f+}pM2NuO%qyfyC(Mf9uFstJu`lb?G5qIv~_ruZ!7CH*=B;Ct?dOrQCpKe
z&9*yIKie$$A-B8bJI_Ax10VN3IP}rFf8!S$!KRqKHy(xVk>fjRJ%L%#_WQR4+n}#(
zHhb<_*`C<2&{j=;?Ott<JGM3t&1?e_LTr744%*zZ;M=#>`QzU9X_IZgte3G>Khj})
zd;Z?NDfOJTQ}~%|Kg6%Kc^bm7Z_B><wrabk?NxAfwXR}gvOBx<lFfk)H}=-sN9;;F
z>AQD9%Ddf@^0w`*n4@R=#JqLSk5B!3d!9G#?Rvnl*L;fD-k3e+wsUW1?3H;OYh!eZ
zd7qO0dD|ary!R+94Yz&g&bY7iPMXbeFTs7mOE2!WT6=iUg6z+`ciM>VJN@M6?tqZ2
zJ(CVPT3X~y-8*rnkBwFJT3e<A*Y>&?^zT-F!eqy_OK0x^$K$*91a|M8@3m_$v*+2p
zeYd;!Zp+W#JE7Opmc2n^?~2)HZI}<Y>`f3pZ`*B|y;rVT-0ltkroAzOn`}Q^*=C!4
z^33i;hLbi^yj1sYaXY#9b0Vkhq!-iordIasEfG7iN22G*o&Ya)yMyN!?Y;9^(2k>$
zfA7V`%lE8Fm|-jJpS3sLxoVHzU3r_8WoCOAHqYF>c-{h=k6VoP>F6}tNT~<fzV?r?
zy;vu2BT)E%cd+jko7bi}dsk0lwOjI2be~7YoV^o5zV7+ae%iLJ{fbS-#lF2AI&8Me
zh4p*SSC{Vn$#-n;f+chIel+FUmk|DO_v@dN_DXDw+1qt){@$JWEVjl)yKM8F8}{t9
zG27d_M%<2x`=!ksoAb6S-v8Yz`S9yr)4er&xleW2%G!0?YANm8tK@%XPfW71ZCUYR
z+w%v+_no_|zi;o8#yz?n{(FtoB5muJ3GFjFyJGL3?az1bv6-{?nCeZNL%(!w-b`ZN
z_u|4#+qpG<w%%Xf+T4D)fA7jYGxlT^FWf79v2*X*@5lGfYCf{}AN!QO7SEpTG1K6-
z{dBQ$Z&zcVt&W(Woeqb+?W2hH-G!%@?2W#YYhxsL#YUvI!`j`cW{+#XhwYZ*TDHyB
zO|~nJoZj2gQ@{7#qlLCl8Xnqw*nNGEb@|afkxkBfd-LDzZC$0f=bx&_UgpyIdlvlp
zYjZJZ#oi@TH|^OVc4^O!jeK^;j&HM3?62Ot++M|Yv7dn5CcXK4O-@|1$vN@cn&rUn
zJ$IrxZQK9O*md2}Z||Lwk9*f`UTC{)*Y&+j%ckw^tUtE*r`rzOz1CcN>%A`QnYU+`
zZNlq&w(pJk_MY+Dv`5mvVeiwT@VySJ`E4(1SngeOd7&-OrPVf-SNZl8WLewJe-^(d
zt)|O1bDsL%^oIg_zswfj_u}f2Jsj2x_ndrk&341SwYF2N#P-STDYR|d@zZ9;jgPiJ
zH&xmme7D(VLWj`4JLdCjy||V4-an_ZC&5>G&mzxtdnIS)?Wvixb<d?$H}=T2Jh$T8
zsIX7vc%RLN$1ANrTx{JtPw&Xy*8xp7CKFn0<}<zDEwJCjrmU7@FGF;Y)jUgSyRNmZ
zwv*L__P*6Wv%6)A@*e(2-nO%|*lm1&b?)81{I<=l7o7V{b-wIb7Rb0aPC{j`kO#}Y
ziU1zlBM0u=cpkZEn^su6SG%Qr?}mR@ZEy1j?pa;JXLosFgH2TT(Y*qk*?YHyEZf7o
z_2Vuh-}$yJqSN<WnZ4ZBX2DvU=6S4kvrVP#may;Ho8lp0oA*zD?-}KNdlnW?v|W5(
zY>)CP2HWjUO}6%o+P3Y?3j0pYWZV~WQpkFaT8r)3d@;Kn-dZ*X_ORQ2*)U=6@#V2L
zEbNVYRz&LA&O5!vR$JoT9_EtGw&CqvHjaD~_geg)Z96S-&)#DBzP)dDw(R}G^k?sf
z{mnKH&dA#QE^)KHHF1OO6sL_g2@}rkww&-}&or;uwp|Q7Hvg9{wGM3iX5$t+V|Sa?
zo4paYSMJ?=X!7pz)Af7*7_Htr>9E1BA11T+Tv|DO&vhQgy*hW@ZPi{T?EPc?cFzO0
z!+RcyPT9+7tYIrLNyK)tLit|v3omRPOceL6db-p$M0on%jUjrrPoK@%yWQ-DE&tlZ
zdw0AwusO#ub?>&rlWgy>9^Mo4bFyv6OQpRZ-?{8ryyt=Sud{Z0iyln2ZMa)+!;-vW
z?@Y;?HU&;ocH1WJ*=s2%ye}eZwk_8&g}obIZ?|sKk+WfVw9x9s_QN)oPrleH-jcIh
zDl^%3g5RFqTCwf6S?A^UnNOA6*Dz1qM#YWCmSx$;y+0ls+G^-K+WMER-YfET-JVr-
zcX!Xf)wkDTZ_J*zdl&ABE}gS`rqrpu9KZhW(K^_&w=Sg7cGtYFy}!kJ_NK*K?7dKU
zV~>i5t?i=xdK(^={nq#S1omzAQP?MGsJBPv|CYTeZ~AN#@>Ol7O!>TLU*1vMFZZrl
zOXw}yd%<tXZke5Nwrnm^dpES4+I^x<U{Bz^Vw+5hHf!Na*0v14_wAmcAY@y>)3Qf6
zdy(z4du_H8WHtBRY@fMXqDIt~LA_|NWz-*A@m2x5jGxE%YS@SF`OUIoufR&aJ#~kw
z_sVq~-#dlJ({_dN4;w*&3wwU?{n<M=eDB^FQ7`s}nJnHLTAjD|L;gzJlUJwijWO7>
z=YgTq-tQ_CZ4y1E?$woK*=NOPxYs7C##Z~i|L$2==I;&5mEKzyq-d9`u-SIMTgP50
zk)^gfs;=%~J1V+2$RlL;2eWT`40yI#f2cdS_rA7<ZR+j=HbN&8_dam7-NSN6eOLa|
z$$KA&=j~RD^V)lB)}dV|6#Diu$4s`ppe?j_e!IlJE9xG$)katMzLAZ#HF*=c_u!3}
z)=Q=}?p?B<*Y>yeuDx3%X4<%ad1$M(^|bAaW2JioS6<q?;qe`tOXr&QuI-v*`=ZXs
z_HOsSz0OOg*!*5(Y5R$V#nw#XyDd|I{O)Im_-!qwGT5s7&Dy&r<=EbodEfW!Ugf^`
zVh@*1cKCVQDMqHY@7VY5mG)n4+q`7^o@0t@_XeKWZsm7=gYC2Frh7I1t=${4VWX{!
zN|UY5VJ}<BE<xK}o%{A4@VK_eBU@~5!I9Lx>hXJR4DIgj&DcHN##4UHUY*s=Ho9MQ
zZ8+GQ_tx!Z-`fzcX0_m@xot%2?7hVi)q4}xt=?0pv0(3P$EkaT+w856*~ISM@?e_n
zmogb!4+f3BI<dR=7V!SDnUJwyud(FGy?;)*+Dc8FYdhEIhAmV1eB0PB;`?-7oZNH1
zo^PMc!<xP4rth|W6|QL4|JB6ik2TZYBVVTMJ#^u)ZH*=G-j0gbHXjz8-|fER%-)8%
zFLwEwY}woOsBdq`{!M#!JmKD3Qa-`<bk-r;l#u>CKE9Lp28J)*yGW*eZ@h8io+U?j
z*~l}U*z0mZV)v@2R(lWZY1wnv^Wa{s^X;}#(;96iPJg&(dRvmMfu*wT=FQD}*T$T&
zS+GWD&&%4md+jY$_J&A_@Al}+v&pbd+f$#LxZC(Rr`-oxF1ww71oz!~$Yr}xk$K;T
zgq^kr4}|XJp7MI{>W(dYHZ)||-ut?LZ|0k>y{C^T+ivN7uqR`^sO_S~J+@1fKJI-b
z$+yqo>l&NfznnI_Z$kHeaQ$U-tI>P!lh)0)9k=J()+eypo%!6nXZ8ENwvrD5Y!8GS
z*(<Zj#@2t)#=Uhciu+oR)$HBbd(SqadX?3slT-HC-!s_Dxaz`QkIVCHvcCxJ6WcU>
zZ{|Y1y<Ml`_Z|>*w=J2w+D7bI^<KsP=X?L_I$5XpEVj9|a>Cw6_uXxtrXJe6DCFCo
zTP;&<cidIAx-|de9{(51Y)|Yh*z>O_dT+`9&3l(PS?*o<>g3*A{PXtm>}K9q<leE@
zxJkz@;109x{8{hzJmdJa_jHZ%-Yx$x*vR&>*m7hV?5P#y-Pa_zVsE!kmaW3-4SOfP
zi?g+~X0w~@xWM*m#eUn|ZYP`YgsV0`K2F|SIM3PUa&q$Cb8oKMtka3w>zQn^*P?EQ
zZC%S++p^2;w&CrU_vWnO-8*g4LEE5heS2aq_wH#wG=0xy?OV2s8K>EP*!|CXbLSS@
zHNhwMxY|Y77EXD*cgOK*dtVez-?L}KlD&@kdDbyX3-{JKpWfT}HF3A~p~bdUntiq{
zj}mQ*pZe}i?DMkOQ$1tP1eHTJ3D=%l=iOr2`*a2CzBJ$MduJJG+VGs>wte+^u5H>j
z{yl;By!QUTd3?7<kNMt5SD)-@i0IfmSCD_-C$@mSQe9rVUC)TyJ!_HO`$_k>ZA4D{
z-b|k})(%U~@6ErhzUKl{m+iqFvbI|$>e&iS>)Y$t>}y+fmTT{KebK!O?oHh5uw|yr
zi$z|x4Fch|<(;j2{_VSFbFoRv)<D*N&%1lOY<IYd+X}qDZfnzWW6#A*^Svv#1=^lS
z=CkEF`Df4jCEWXjxjOblcg)^<dt<0g!RKh(X<p5H8E$x3Go*an+mIn)GwJZ5y_|yU
zZFFYN+RM<#Z_9FQ-(IfgPJ0qx@!7QrXxMU?h}r~ASiLvk-K{-l%nWuq?n$<}t?he_
zScL7a2rSuq;qn68SO53f3N*IuJ!}!T_iy3U9cy<i+I!&og57iF^Y$+5S!SC$Eyb2W
zUSwaz-VJ*_E=cV4HS5_M#nZ4iEN$N2cQd{A7W-=LJ*`o>w>VH?H$#blZQx2XTfwxt
zz1jVjtrI$A_AU`>u(f$`*5)Id`rZRgM{MG2|Lk3|aK+xl+tM~cJ2>~reATo~=l^2+
z<GRw`-TQL)s#+f2yTbId&FjS_d#B(2VOwyi$u{UW|6Y0iHMZN5c=xVmd}ia3RI%52
z#`3+^GdJ0u_T092!keSEZ41iw{;Swx<Iu5t*QxuVdnMk^+<V{{+um2+yY?zrh1%X^
zZrl6yZ0_DQTpMgV8@cvH2{P<$Xt3XH;QZaTREl}uOu5f{rhPbVGsi$>ue4A5-e+-q
zcFkh1Y(r*6*m`y`?eXfKzqjz>X50K<llGb#cJAd~_s&Ldlj+`=70YbwBzpE%*L&GU
zua?;N;B~C6%Z?TshP+Oz#a}kq8uZVx-jNz)b8+Qlo5Mn-yFVH%vy};&X#22U+E%A#
z#~u;x_`UN(E%z=>*t^$Vdf(m*hS_`1J?GhX!|ar8ht@2ckWaJsZVHRpJHzhmp43j>
zeQ~Ea>}Cd@+-q=gzV(a)TWr3(x3iU6;JvrsE!B3Vpu#>~iH5xan<nm=d-R9xpG&=a
zTc`Bf%sVZzZ`mQ|y?-B_+v~7RdCvk>3+pwnSMD*Eonp&-<(UnOVfJ3WV;gP7?(eca
z@jQ92$W2w-C)VAzA-y+j88-;nE&1_g?~!-^_Fi1#Vrvs<Wh*u%+h)u4je9=7sNSQ#
zbQ-Adw%2(6?maU?t8K6PxZ3<Vr?c0?P<l@w)6qQ{4;*ZZr%l*fWj|xD1b>I^`p}Jg
z^JeYXJ7wPwn==Upwijnl+0)@<urH{_-{!`~?>2kXX4$L=H{4@Ysk67>^QyhRMg03-
zt!v-QP<Lr({H!y3FN<pJQCz)ruh!k_z1!Q~*jg-PvYWAPgH6i3LR;=L5B40)im*L&
zLuMaC(MsDY-kiO=qo(hDXa3$gGS6Wz$Kkm)7FS*OmfYamS8?2E@7@&*wj0vDY@a;l
z*xgaK%R0s5f{pkl&VAVu-FtZxzwKq|Shcr6b>g1Y;p_I6ujkv#pgL(!&7TW<GVW}%
z(Yd$KW^2n7+c_<t_B^|yWOIXk(cbDyEA|H5mfUO57HK<Y`DEL|mHYM{`T2g2=Cwp?
z%lcT`<Og=PinmR7`yD&CXWd@Uy#nVJ>`8vDvUkm8HE^3OyJgm1Z3$JojVo8~WwPCB
zE7x_xc4x1qjn`+(y|1FS*(x0J-Ls@<=H9o=)Ax4o|GSsR-Ng1ow$k2?dAxgTY_x0x
zKEB<(<g>7?hVvv_*UkKPCp+Kot#3}Uoy8lq_m<qWy#b1Q_ile7w{N~Z>%OY4<h?6)
ztM56JK4p)G*iKu)o5%N_IKXBTDn4g#dsY2jjkY+OHr1K7Qn%;q4QA!tckaxxJ&Y%J
z?=3zWvp2ie(^lb8w(ZZf#JzcwY;1a@+V}9MdhXr9WN8zyp38Rb9M8R0t|5Cxu5GhT
zf5W*~#f*KQg@wZ2xYee6tEMXK6KUSMS3-=zwy}@PR$-Td^%lM)+sciT_C}a)-?PuV
z-}X)Esy#C->TS(Dm)bsW-o96D(Mem+z;m|KRyFL6EZ)9%!Tbi>-;Z|fZFJjb^KDbx
z-r7~i_8x0C+N;C1Z*Tt-<-H}vvbG<KqxT+h`MSI7n!MeXyX>|}bBb*8<M;2qa+1Mr
ziRon9r!!0U+RYHy_nDb*Zw$w6TcwwiY}ToC+unK9VRPAKi*2KUmF=Wsj(aC({n;xO
zd)_u+nu={gP{r;eZ@%tzF>$kPP_EsRD7R~`-_JJNSqD$;QFwI2CO$RB=E@qQJ<Sud
z_T0+s-dk}=b5ExF7275c37ZqIOKlQ;@9ep>w$yfy?wdUssgL*k$a%Z>WsmAU4FiR}
z6VAumEL<V9cR_UTuB1h~Yzpe$Sbycy+1t9G*_yLTVBhOyjCMi-6ZWVr72JL0&`F!9
zJGb^a%(mVmThn2i!0vDJ=DUGSO{mM>FNO^^v;NfC=x>~4TXZDaR!njBUXBe*Z8?%E
zY=d1t?S55au{Z2C+g{xnfqOf6`fXY_E#GUAw`*^X?RDD#eqmdENddcAQ}^#xp7P$-
zB}Q`Z(#p%bb+*5>DcM@PSA4nQ-Wtunwh#V_?7rG`*w#(xw{7j6+k4`+?Ad)meYH*A
ziLyO2j0J4Z-u2l#U7c;;ORrNl*SS^pN;hA$y;VASuVq%sUh`)Wd$!GBvr*`3wDrEO
zYFG1)$xcCgo{i{{6Sk8|rR}mWl-nMNxMDTgMQP8TFlk$p=iByv`5?Eis+qy|T+bBS
zFXhIz*H3fW$uY)SZ!EvFcW>SU8|nTe+kIK<_a1q{wlB+X-`@F`qis9P_S&AhAF|gt
zJag|8H#b{zhvGdlU$@wbxtUrOlrY;KW4^Y#BJ`<k{FAP|%83bk@3AhieXrWKH-$6W
z_M-UnJrB~3?R}@Ve(#RhUAC`R|F(_0w9mTjw9dZcQw;W0FmUbr?DEG(;M@)y{i!qe
zEI8e1EAq9>_Rht5d$cV0_ZIPP*>ik<^4_}1y|!HortCfN_=xqs#|pLwx7qIPy<TO@
zAwFv_$Av<hHq-XKRaWYDLL7~@Cy&VNi(k*PcNX7+J(geMc0W2UzW149qV3M4S$kIm
zvfJ_=oMLPDPH?Zr?c6;+Gbio2X5O@SF5l`s+RGQ&3Z!J(P6_PUdn{IL-{kbedtc?~
z+QxEUu{rc>qHW7vb=y1D)AxL9E!wM4+_Xo~rEu@h^a`5~^S$=Ovk2HRFWzO_aJ<V(
zMx$X*h@#uxG}#q<8Nz&b-#w&kSO4vWZPGvHeIA{UtY>M=+v~Mabl=<p(Y+2WnY$Hc
z_SpVVe7$Ff`+8fIix#$8nM?LYJgl&d5a+VxD_CR8(|mbP&xdI?9+DM%5BPQM{^QbZ
zd*s0a+t<@Q_fGktZ2MuNpbcB$Y1<$9FKnV7tlnF_(P{5ACds{DZW-Ga{9R<dv88&i
z_DYq#es0tEhTL3YyQAinjap;jUZy3d_j366+cbPh+^aQbkFCc;ciU2-b9*;jXx!s=
zaGC9o@U6BC3U1aM_fl<?e%tJQGyU*hKhCDTw<=EU^(dIWchABjdkUts>|OO%VJ}Pi
zj=fG(&hFjthj$NK=v><+s(Q8)?mV>#cv)k^FrmfDXXzxHHy<DEiHVK16_H)Fr=^|2
z_RI7z+xSgu_9~n-w9N^qusxo%W$%p=ZCi`9eYTHk7VIs#CTlBW&u5#Z+h^<heB0g@
z{v~_PC2rWeC%Sg;n@{)mRPmm%&3G_<uS(8ATa{}Q_cmmeSzS<bwfXXRwe45y&3p3>
z@7-Hrw{lO&WAVK!o@Uuh>0`7NlWwr>NIhV)E^UWx#bIIFDPQ~c9PpfIW%YliEsL~@
z?LLF;whDPgdm6Gi_ug1#xz|W$kL@wrm9_@5&vyGqE!``Y=CJo%Z|B}OcUJA)le)q-
z^-$d22XpV)%$O#)mm_`Ko(pWhZQ}G+*&3dT*gGXIWUqzKIvay^bFA%TR@w%rKHMXB
zrGIb2@hP?@d@F2st?9SDvTpyLdokU%vleyiT@$;|=GpPRwhA4~tvlH7*@TI&u-*4?
z=H8H_-}gK_BC|KGT50bB^^<#-D6ikUBJYchZK~7WB}O}KSmy4v72un1-8OgY-ju#-
z+exld_fB}%y(j4WQQIe>?0cvDSzxOWRIvBKLJ=DW^;5Ppa=zO9@KfD;`Tp6xAMAfy
zbE$8$bvS;&CV|(%c7ZAP9+!+4dlE!vS@T@#+#_&Ye^1Mr<+cSIp6xj`({}HInG3Bu
zqTOv0rR(?lg&nb7$*^?q4Lv`b1@$Lw9p-+r`tWp%?a7S>d%u62z1J(UcyCF{0^111
zr8YdaCbk9zv#mA4<!!c<HrVc47PB{?bFcNBhbp#vRQB7x*j%yqK*g~=4eEt9QBEv-
zUEa*I{XFsH-s=*Z_CAeVv6tb)nH>yqJ9cZ#Txla9xnxfh5C7h{(+6yY4oB{FT{3^~
zv^AW27x8bf&E4#|cgn4B+Y?7t@2%kLwQcs>wD*&IgYA_6^Y)Z+_St^8-m&+{XGhx^
zzszkDvZwFaz)-x$AgazLu5ZHL2YVZ=A4Id->`-66NAJWITLFD$+dVIRY}LdU+A;*4
zwQe}N&bFbCXU~-@llRyxoMvl0H+t`f)-N^!Th>||1l8MC7+>16Z}EBCjt$IvH+W9i
z!?2BYPf7a;+n$?JwqlQ#@BOjZ##X|X*Y?bz&wEaY$nTYXHFqzQ&T3nMUtg_diZ||=
z_@rSkPobD?hs9=_wVC_&-p~=SU10BHE8)L?Pr<4Fy#-~5_AXhpcF&Km0k%Dhx7r5i
zy|;1Lwa50?f!@7$K6>u`_TP8!B2VGH7IHCrWVzPw?R(_0x9A?nULDUe+fQ2>_Re|x
zYfngKvu%mQZrc@WKJHQ3`)kiWpR=}Z{tdQor|R#$@_WjjODT(NWen%pDrlzfne+ef
z9)<f3);wV|Y(Hr9+Wxv;xc6^E|K4+(n)kLHblH3D+a6o4$+CNQ$j-F=6}`+hrT6IG
z=i6uRUGVCdZT^MLdwW)X+rx5hpY2SohP`(d?X=O-Ut-H^IA<?M$r9TK{VQ#@<gnS+
zXr9^QaAvmEi$@iEjx0O8$M44`+Xy!|n;8$YZM)7Nus!kJa*vJ7pFI}?*6-y#Hfe7}
zT%qle`P25axC_~~7>4c*`Ek_t2GcH^DNzl36V~O}C>YCHXNauWQ?O5D&x2XZ_TKpB
zy(f<+dGCqUg?kEqm)ai4DcsF5;lQ3h5;e9f6Xxy}P)pyd@anIPVyV&I57W2qT@jMM
zSL{*cUa74c_Wn4tf3HHv5!(aa*0v00DposETx?Gq*0$~Ywt26_jTT!4Q!d*DliylT
zxW=*Pj>UA_YnOud2HdFJyD8(e?Y#bO+aA?c+aJH2ZBMV7y4T?Of<1d?9=4S!Z?ILl
zonX5mc!KS_t0sHz>}9m=(%QFIDP+BE!ZlgzJH5?&4H^Y(J%bMH<xu0>b6e@i-h7eZ
zy*^(ZZI9eNvqxp~cAE${SDSA#n{0ncthZ&TnPN3#(li@J<|%t)j-0UFFhki^Bk$6l
z4GEj~9N>Sj$3$_CZO5|XwrRrrdlRnm?9KYNdoRm-1=}m1x7qHQ?qhpq&4s<W&vW-S
zomy_I(6Dt+*t#Wqd9HQs*^zm4PeZJ;t<^Qgy*ciTw!RZi?mZwWV;j=3(3W+v=iZiw
zVzxYNv9@2_3T$6I2;UpPnqZre+q>t(i#VI4iOcqKG)}a==-Ih9<v5FN!tr~17%o1y
z?kf@8yI_y7jnwxkwhXyHZ5ZBM*qz{i$);dljcrWC0o#fx!h7f4Yv0@Oal;;lfGyUa
zE;#IcvH79Ro<29*h`IZ14lHrBJ+{4Q@6WKpy%DzJwm<qpZ7w{TYWw0t$KF%ZPTO`I
z=&?Ov8e?<6NoTLu>vOgmr5d)!^5ypit!}bSuuZfM)84&z!WpH#Qvx$>XNct5_S`D6
zT@@a>_fd@d-T>dAy%RJmY+URV_uf%|x982*owfzb&+oQ-JKMHlmh+w^lV|SPBHwJI
zaaVayiJ_$J%0nmiDlOP+Td{h-O-5p_O>>3r-U}WdtPe;B+1z?D-Bu^2ZSSrp$M-%^
ze6jn)uT6U*3V8Rp{5ie%=ahuK8(y5+{n+ck-iD(6HVPFMw)eiDvVD@%vo~ha+dXW|
zLi<c*e(Z8zY`)J;O!xrj2D`l_YAg40w@uokE+1ys+H-Ml&)TNF!ao@Id9I$j=l<*E
z`_9Qv+$-_?@ot_t?tN)uuD0f1#r92T=dk5IWV-i#0M~w@Bc^-9-iz*6SZi-z!`!fE
z*3!u~N*yBmrFJOnGdQemGyVD*8+{gsecro@?dHytvpKx=soi>=AlqYWn)h9GnZ0+?
z6@`5~e_!w2q87O?;LSCg#ee77hIYy9)p>Y+UonU1elx-Kd*__nvd{6^p}kLU{Mr{a
zH+7%&^6h&WPqytbVLD(Vo;H7<R9NRe%WD<3SAQ$+b-r=mu9ZW4pZQ6jz26Py?K56H
z#nvJ1sEuLnwY@hki63aRY2TOgR(4-<z%hF^qfIs+gc$cNi&?SP<camZX#eMXOPRLW
zEED0d`!dCBFZaS08?*jt``(_Iw&%=~LwomRneY2nFny2Y)Tn)|6JG6|d1dY12Q3r!
zEmv0EYqyDK@6E-n`&R4h-}_+wq<!~h@7lNfZ`Ho_3iJ0~^v&PLda7aXefD$vcqIAk
zlbTs<&i#I1^WN;pUaJ^B+Xaz<`}^<q?z4_n-e;Pxw(sP&<UJOKjr%s8TeSCK_f0E5
z-bMTTR(0$%{qb?HCEvPz)4y%mTlV>)-DVDc`>&dPb|>;;_D;KUcE5z*mAyAY4p=SM
zf3u(GS?^xAC5P<V8FKda8@t;jYw+w3=lHw#v1IUG&-t_VRdpZQC%8XupYrv6d-o*9
z?_H5qzgM90iEXK@&VIM2s{1-(9qm$>AKJujnzY}sLTm5M6`%GV+a<Jb`=Q=FdYMu-
zp3j_Znf9L9dx;};uT<^gy*qo$_S$c0wda_0aPQ{tE_+r=N7(<Iad}S?=f%C+_fz&w
znBu-aWlhoEXNvuMKV<msVb)JSur6L@@AgA$_St$J+&%Zn;(fn=*w}vfr@cSMPig-v
z=i__7tvtE!nmhZxZqE39?=Cj)Rk&ccck!!()_L#F+p69>y?4)g?tN#I1#DLwUbj~#
z-fQnF)3<w54;<Nh;nRh^Zf1M;6>MzWd&I<M_i?@8eb;x%?@e-S*ndWVYu~EB+I!PK
z@z@#_E!)SqsBdq{{yTfW#~B=Gj6J>YV3M1iFJtLm+1assk3QIHGwp%uz7vV;_PZXh
z*}LuQ?>#H)@9ljjq-49ry54q&>&d-Uhb#6cH^|rth0NM}Y{ClrKWTTYEk*Y4+Y{8h
z?-OU!{$le(wjAQ0_G%qJx>t--c;B@=|9yH48*Db;*WP>0t!q!g$s2p`tvbFhlgW0k
zPlMq8CR4?|E~Yp3&P|%W&%jJ(-wuJLHdb4A?NfLdw2x^X%Yhxog!Ve<3t1~G>+kIn
z-Lcm!zIX4BY*Aaj*Fk%wp5ETOGkei)l_+DovY4-X4!QI1WfwfSH^_Lx-VB~kwy)or
z?sKfYxwq_}*S@Brdi%(-<9quo=IlEyp|F4N`|kax1*Yts*v7SwQ)Rxjz=J<F-WvA%
zgev>?iso;!ak!;s%cv*5$0Po(O`(LUb*j-HYbNPUHtSdg_AO#*+Go`N%f`uQ*}fKr
zZrksZb@z3dH|^c!d&T;wM&rKMn#=ZS$E5FzkICB`s{PSA`jOzCkXEI=d=EqSSFf7C
zuPVHLuVh@K?Zm46d-GpD+iNmabk8znUWX^=GxmD;FzzpBlHBKAKHZK%VAft)>9hNm
zHJ;wLd&`Tx*KK6?KjAR7;dBYIU7NOdZ>FmI-cIN0y}Q(&@BYH^*QRgY#=RLP413>1
zc<$NJWNIh6a{Au(-TC{%d<^z}e5bfi@VBHL$Lf1~t9;$}@SgIpU(}Vk@6M?o`;<9W
z9SB{VXT`s@abMNtWi}7uW%nt5wXjVU&)e_gY-Jx>d}8nPOWX&p*8kk|U24ic#u6{v
z)8?l8&19?WbNp{u3#(Y|*G)9B&yqT|H*ZU@?TnzMwsl|B_MP2vW^Y?&!`=!R=KZak
z1o!Uzba+4S%&5Itg%kI__ix&pWVUDDW7WF7_u^gm`Xs#DtADv~U&lekeeE+2?z`C^
zZ0rAwX@C9l!o6E}rR}>XdB*ld3G=@BTc!6M>M=afeP7>ZRVv@Uorj!llh*a@^-b7d
z%XNFUoz0Bu1H7?+_t;!qU~5u%YhT~98MZ$oKH02T;(6fW?tgag?@!*lz`=jNiTItp
z7vqffTQ|?x`(VG|zUHjZef|^F4xC)Sb8pK(E4whW#{KH+X6y<1nQpJYkZr%f?TEb<
zf9CA_boG&qMbFiJ6Pwldf5~#QmAmJ=x1_~)-;pQttT}f??!PCbXZzs_!-3#;Hv6|U
zl<v>YbFo&?y0%v-eaYUaFW2`z2t2UIeaVr%-n^gpcGcJIwYV;7pSG{VcH*P5y=Ltb
z_ld<Vx7}6hVcS;mf3NbS9eWGSO!xUZOyB#pKx*HCV~_R=`+3{PB_H1-V|;tx2F3aN
zS6ephJF!4$pGR1@{Ub-teJl3NwpIInVXsisjlCg4ul6q9pt`SZqU_#n)28k<^}279
zuk>-h#rduKY_6Q%TPnQW>d$MNy=GC|_7B+p@3md^Wbdyj-}bI))wj7Cuye1^H^%)(
zEoSVkEbrLAZu`Bx3T`a>;<tX^`$o)duXx(Ly&dV>_cho{?c=>TYcKD+vwM~{a_w`_
zQnWpMi)mkF{_cGZMI5#}C2H-M6nShHA3DA_G$>?$4DUQU>*|WVylcPgZCI6Sy<*Pm
zy?N<wwz(HYZ4M-aSr@1*-CHO1XWuhRvwcTp<Zbg=H1`D_^|D*eTfFbAz!Tf;HIMAR
z&z!cm*I?29vdJ^{G6^K^{cQ1n?<^Ioy$cvv_O|3k?-h_=u=jJ}@x9l$yY}+POxU|l
zWZGVtFWI&sUsY@uMAz-kpIW%L<IGi?B~L``Qj*MV1vdHFGFZObwS9fn-iniHw)c8?
z_Z4JavN5XD->c#Bde64M=WTfo9JZaI$7XvbsBv$@Eh!rdgAaR_HO{m>v0|ys(|;>$
z(nMAE`8-}=vt|9Gy}YbDY|A-2_w;R_Y%3(fxYuTDzU`4Wr}r8aTkbu<r)=w?U|{oW
zW8~h8j#F&y_V?Hd-R!iTwX9)p#smqQ?rn8@6>OVrJ0?lmnrs%d(FxkU_ebj<+qJW&
z>`ids-q&7MvsdVotet?Pkqy_xv$i$Qt88bUf3UZx_M`1F-7Z_c06E(k@7~+25udc@
zjdiW<iTfEgOXQ#07%a2e!|*0?uLJWY+e6JW_SU7Xwhb{dus#25)*i!y+;(Mi_u7hG
z>e=fqC}XF#<DM;F`a0W|>bZLtC~8{Q|9ZA}3v;_I$F$db``Fg+&C6W6H(yhE->;Xa
zZ2NXu+a{RowGCj-w=KD-W?Ln`)z;(N%soA&m9_=9mRX<p^=l7<&q<pIz8+fzzX`VM
zO1AG^v~GvZtE(q$7u>(Id(Nf^TZM}Ly+5up+e$<}wBd?j+soq7zjxAH-@Oh7J9h6+
zI<eOw!@}B*W0S3l;PSnPZnW6imd)O4VLor~mRsv>t^PgV<K-Q<cT#P?t;6K4HXZB%
zwmO=7_WVg$y62vX>fX3>{QLNAL#^+G{o7;kj?2ckdHdd&J$-w=2KMfK;i+RIptfso
zQ|D=0=g`8v>&3S1y<GKfub+$X-lF)Odt>(1@9i+Iv8^y`-Xqi2yZ1=3z3l;?e>Qbb
znD!k=_^|gt<<mW1G$+_vuqN+mU=Ojr*v4#Emae*YfzNiEJ3CbN9=qAK_oL&Ey*GX=
zw-H#zzIR7;hi%`9_PrU`KkwOa{ORrq|4!}Ubbhi|p?aE)mBt#|r74s5ez+!N8~5$R
z-WOS^wzvFT_dfr#d~fw~r#(@Yhi%vM{kB!V#<tI=<k+4aPnc|OI6tvj@{o0J&bP&T
zm${zV>%kXiTYhfB-jqf6_nZg{wY_-kwe8xYbN6myZQ5%U7`}IQ39sz|vzInA=4`a>
zou_Lv;Y80KhNN{?+V5F+GvtTvEn0nYcfh8^y-fQr+aBn#w&7VZdr!^E1GZnZg=`|9
z9NjAvwq@_>Hw*V3Iell3M@pw{K*s{xX+Iv>C<M>7ono@rcJ=2swnslR?OE`t!B%WR
z&)#gy(|c`hneX)wb+gThoo~}{`nS!8M=SPz`Q^X&UWxnO3-=G(OqdpLvt>=b?E@Xj
zT^ir&ZEvmEZJTkB$M#u%(cV*qzI&7Q^zIGlZm?Oe(Ybfpkq+BKpSRe?bR^p{xGdSV
zVDB1RpP7udli6eUS`-)B2DE9~wEX3=wYE{+`~J%{+c$|z_9zIZ?{T}a!S=?2qCEzk
z+xDg;H`+FZZ?zRt7v1OWzJG6q_&1v#tJOAZvX||-w#j_&1SOWe%bbmOH*DqGTcG)O
zH&5hQ+rw+k_Wo(<wN>%d*n7oT)0Xv&<K6}5rS`tib+?UK&~2MG$!G8J{d4v@++1b5
zZfDTm8{T{NR;hQ`{#wVo_sY+$Hd2cZ?v?Rp+j}AQ*`A4K*6)pH`fK}VX0GkC@?EwU
zmKyEdCwySf1jiM7Up$^<TM@@#b0Ou^UT+N<J7KX7oBaE#d#5ax+V{w)+4exSwDqP7
zp0-u2r)?b`P1!B<yxBJ5Rp6dBxe0q0I4bX!eY|$>A(3r+b#A@dbC+4luA*zsUIF{(
zyPX*??1{YfWzQ+o23rYlbKArDk$Zm>@$cd5Q{0n(+`{_H#%(qSc7L^c<G{N2SKHFP
z-zGHMrrwyjCpUA-UYEPwHcKsa_P*I2ZTsQof9tk(*S#HDj(cAOZninM?2^^Fq8+wp
zZgAUf`_N}wGHc%+1w9ShCmZB#qq7g&{Qu)^JL^)Z?SW!Wn-8T{d(M5EV|!!A+`a3H
zCG8~T!uDQRr?L0Jvr{&cHt)1ms6J_PVf!7M9)(@DrBmAW@;S`3Epu+MUDZFyRzmFJ
zo>NWTwja2++g?eowdH!*y|=+qagSZmqP+$Glx?3V?YA|`*lGJ!bNgO_IZAsjWHs$&
z_ix%;wS1<nLS&9@L)j*q4HvC!9P209D%^dtTkheEy*Yct_wL*&wda@11=}4xZ8m3=
z(``3&9k;GK*<d?CC(1@)iG=OH8Ha6UpLOi@&bht!@`bv+0hOzE?`Yn-_d$23%>xT&
zyC0`F?KNOqxkqQ;M%%xCwfD-cXtZUR=(+2p+U~tv$Di%JwQbAZim5yH2G0}Uce!4D
zZ^EJnw%al}_U<T5vvKgbWOL}~VcS1FtM+#6`?P0XX8+zvx4ZWWuT-}cYCmfmA=F|k
z_iC~244*GH;ZjO_pYT=MzDsS|%WJ%P@ASs}y&h+3Y){B*?3vftYP%ruy!CcJ)xGc3
zcG|}N@7;4?=L72xx?;9pET-<=F_+Kw(ds#SPfR^+>s}SHS3`1vZPZ5Yy*Iwx+0(P?
z@*ZxNMSGis_U_T^a@pH-eZt-|`DbkzST5|cF4|{nJ2!l<$3ua=Tl^2%_5^k8y%5`D
z^E1lF_Jw+%?Y3FxY?(DL*-m7ew#Vhg#626HZQ6SwV7~Ro8#a3<TtB&6exlXh@Ba$-
zYPkCCwfG@p`=`3wcHy3;y$jMLZ1ta=->cvwVY7^X!rl!{)_V`gO546WK7a2H*8{d~
z(>wN7J?^npJ+N-io!Xr?f_xEsW9zo<^(&gW_r)=1n==s)_mucgvz`8?Yj4R*7uyND
zs%$4%KHIHVBWyS4otSOF+pT*IJ{{a+!yvV9WBsJP-%iipyC$34)+%hKElbzgz26c~
z+gAKLWW!)*zW2ne(|eiSQ})i7!?LHWUv8g`>t5T8yBqhM`1ILEpzVmYguy(US^u`$
z{yVV2mV?dC_JO;%b&5CB-aAuy?H({U**G+;-hHT~cdtyFm|a=ICY#dl2lsk#H0(JT
zC}Niyd1X&x-K4#R3R`Vwu=?$plaXVarMcYJ;&{33jg1X9CpP8n<vby8XV5sqdXv{Q
z+XZ*?Y+V9Y>``M?+k2qzxOKPB;=Qe07j0P$=j>h8wa)g0hti%8d!+2<IA7i4EcVYf
z{>7xdX;*I8MuZ-+J&^Qt?;4-Ido3oi?QuyI-y^jBoNdLQ4BI4K-n~-6C-<s5w%Irs
z9o=JbCT7o!max4lL5;Qn?o(~1SgP6jJASm$x}3b%O+;g_ndI8NH~(|*+jCdgjwxn^
zZH-92ZJy~0+oHw%`~1YG?A7smzK6k^fA6Jz(`{q+tk@g1YN72vDaJk5<{Y*y;c&N|
zlX7_PyO)P-FZ-R|`)K>Vy&X0SZ3VXY?%DH8c%Q<Yy|$m$>+abl&|vHKRcQ}Lg__L~
z;q<+tEJyY}u=r=afn%cWALqus9$`;x9&obl*>QK{-jIWKwgzlrHW$(!*%<DfWUH~r
z#?~xQZSRgPS8O&EXxf-<p1(I<dZ&$9?mk<;Tl;J|n$q`t_}#SUz;6j#fn~Y2=kFcd
z`>gYb%`<<ceGIn`+q_vb+gAVWzP-xp3T+$&l&qD#+4glmYTR>y<E-u7z#V&cznZ_7
zMfHL0DVeRdH=Md{&g^<`bL3FAZ2-@c-9<C5?mZ(MZ2N4Y%wDU=URy2Sp1mIN?6wy2
zkM?x%EV6Z$ny|Mk_}-p3(!qNp>I7{pCM>WqeAm49!JO4LGxewM%{_T^Z&T0oJ*_{=
z_XbaPwyqMMVw*lC#CCyG%HBCqJ8hRvTeX)(WU}o6`_FssBsA{*5O&4J=M%r}Wh2&o
zrW+3JeX+~QMudrDpJdRKy|=X2?Vb0`WN%N#s=ZRdhi&bWw%Ar!Z`e~~x^M3V>pq)?
zYWqD57I$|m9yqb5gV*1dv3SwmDeo5TF%T5Ad-QejUV|gTdtZ0(?tUtHbZ<XLoUQv3
zS=*^T#Wpt{+_4o&(zE@zVyf->nzMVE&HDGoew)0v&mnD(PLi?h27OiAKPLnBKIBQ?
z%dmNY&62yDZS~mC?<u+RX3xI0b8IKxId04AacS?pl=HSHKWXnhao5>q(Y@Ba9J_68
zSFBsJr%_sU@5}=Mdl?fq?A=zd*H-Re$X-(uoxO98uiImtqqisGo9x~P$Nue!X|S@D
zdRw#C;=&J`bdEW;cP8uaO_q3MGcSvIuSzwC?VStr_p)Ve*}Ek@(l*Li!LHmn*@j_<
zscqhTzP(FbO7^Cu%iFBo{=~+{Wa{1-wJx?C%!h2Z=r--0>b-BTm*&j97h*+i?@H|0
zd!_f-9){DOZ8m(Bve_f}ZI7JiZrhyvleXfsgKbwV*}B(b`{BL3QIUIJC>+`wl(Bv9
zxw<oZ53Jo~b5vvDUX!?F+Z8?`dk!AYwb7a4XUoYXXuBX|ign6K9~;h9Vs<~?WZ9ls
zY_yjvexmK&n_u^aYp>Y*;o4!Ft~JVgJ<N6Xd`OsK6HwE+XU|My+kltbZ4W)1Zp-rd
z?4IRG$+lXZa{K(>_U=gw4&Srq%|Tn4NaMXzUrw@Jk=SArwCt5lgQ4{vZg*!}7maPU
z5xd=O%brc$Yj7pTmi6($y%w>HZ31>q+TEKfxYw-y_TEjLvuxR3<XJCQS!Q#wUC6HU
zc+cLH%^UVc=;Yg4J{R0K;e*(|l*RjOg*Z>x8q^%KWysmEw_)OKo6;EWeFa79_U@Wk
zyO$?xt?du@d3z_n+_yLG^YpzMJC5$@77^SV^Rjub>&_px3vNc+T5dGh%kZqyaz~!c
zo-b<cdjnRg*-n%HZhIx}hpk6o{T`K)6?=lE_S=-s*}JzlTW+sF)nS`gVr{!s(jsgV
z-cPm7nzP8Zbm79icGs%*E~wzJZk7MI$3%49UXdS(wmcpMwqc>0ZST}H?cs2UwmHEc
zYtvx!+D3bl!rtc&oOY|+!uN9V*X-R~9ke%X$28k_f?M|XNG!IkeK}+AoqyN%+_<~Z
z_Swgidq2wTwAJI;ZhJ0emaUTw%ij5t+%`Frj_hUlXSLhmXO(qA=hHn4)9Us{Xl2`k
zJ(_R(L$rF&ruL4#+Ur-^c*d;STd{DT?J|Yidynua*+#8DV!P(lfjw{LO||*6r^|*%
zYyF<JmyYcf*_UOjv8#J;wJ7htSq)9LFJoG5rZUdi`-4Bu=6~&3+mbRiTRG*`wj!$i
zdu)x5?@hWo$(F-KVDE<ToqHC{bg=CXzPXpzyk+mc#nboB{U*DYV=2?#^J@<8T{ml=
z?T`DGdzJS#?|rjCWN(GF%$^>;uX|kmFYKK@<NqFqgLO6n#*%xp_Y2r%+>f`tnx?b&
z+Q%if7vA5t>3I9V=5Bq!UW2`FZBBX0?mgv_xA)EVZF@S*_U(D2o^Sh3Y~7w$>3y~@
zZe`fA+-0<_XbG|XJvqgO{eAS_hOYs;Vh+pNHYhIHEAmmo*3Iy)txf-#y&9{Q?afV)
zv91?p*!QLBl<hz7skW*1)q73S#B3Xm^jh`!w(q?#ZTTLaRYrSH^z7No<g$5h0QcOz
zEPnI%G;Gz~CBgW}=DO$hy&`Np`-Jw$?u`?V-Ft1)^1b!WZF?8|lCX6u{;=mkP5JK6
z-)HSj?3`#*XWOuMQNTKz)7w04C)GFECVcR-Uhzl7_Sdg|TOXNydz^0_v;E}0&31wM
zqdn>Ev-aMdExK=dvc|qWU$^fOT`+UsgtNEyZ)j}a8$b8Jo(he^ee&Da+8@x$u-$W=
zXP;7O(w<m}4YnWG+_wooBC_wXzPgQiSIxfO%NqM82<PufQ4+J2o9Mpx-H}84QdVuV
z$xu0AXLm@$zFIHBW=g^Aed$Z~?*FB@Y_9{?v)%g~j@ciM*V-p>;n1Gs+TDBeKk)1i
zu#~oc_#<Xt*Vm1EK1V9-W4pV3Pl(doeU{1%dw<<i+_z1ici%A+?Y)2V;_M&!^6V4S
ze6sI4chA1X%RcQpFEV3ar0w0kr=z6zEv+xK{qLuJV1@PPy;_HN?{^F6-J3P@&OW|x
z5&N``{j(8Vo3Sr=pXOdpk2QM_E#bGjvF?mLm*ASczjcr8-T!I&{{N@0@6G*mWRLE}
zeS6{~Z1zcb&Dv*?%)9UIuI7E)C%m>%`7XTg=82VirTDw|8n*x2Q?k+Aws`%Uy+#L?
z?2~@FcHfMqzk91w)b{<H`(m%|C9b_3x$pO>T{^M%ppNGL-YUiY+&iZ1%g<=ryYS4E
zeQN2a_M4Qg-*@HNlKqExkL-P6eQmen8o7NUN`LpJSPI)8Jnz1bBjVt`6=sWVc<M#=
z@d*3c)rBzH_WoenXUBeVpNZsk+nV(*dnav)-{<gK)JD3zWlzzq$o(_krtUYm@^kOv
zd{6tm5A|$!yo%UcU%AeHit*XK)Arcx4apYY8}59}?u^dGeI;SCw(shC_a41cy<hq1
z&VAb=i}s25uH1Jr`Qe@e+P%BXUbODr(=%cJ>#3zSjjgKt+0@zgw@QB7S9NCj-bn6q
zd-vNl?-h(vurHc?d)NG~342*CAKCjeUuDlhIflLe1yuHNmF}=J=AW`x|FVw_=UwOh
z54uk7HQ30$?-GmlzP%T!_Am4F*t7B?=RQf^2YVMbt=_lzR@(lzmFxD^f2iLZX|B5O
zTtTpHWS*RTQu@MukK}Cj-}G6$@79vIeTh|P_fB0gZQs?!X?qTZ9N69dug`vq^4q=b
zMWL3Qx`*~2)mpdr?`O@uJDpW+AAD%rThcLO@1#XL_c3x-?9+SGv{$!~fB(-_?E6AB
zl<k{BIc-m0cDGsj;rrely;}RDrf%QMpfq7`BIgPFKSm2}TeKqgyKTH;>t?IG&)r*c
zx5FXpeg9s{?LXzY$Hw-8!rqdz9Q&?qxML;4UAFJ`@#A}=CN8sovm(Ntq2bfsN%lSV
zDj#m|U6$9kPn*ZYuB5feUNoL(-`*D=_qx74v{%-uac}C`?!9p;#ddm|n)k{~VcZ|J
z{PA9ae5-xlCYx+}(x2H+SfI2|UF_JtJ&#lE*L>~X`{0iBzV;j2_uZT|bD!gn5BsP8
zpJL-6&#>>IhUh-0hcovc@t(Q&_)^(@DcW^=PA9bOU8&n)^KH>V`>oBrd!MN{?ft-Z
z(w5oi;od`M_V3YcQrY*-#AEL%tH*o1tr#4xCuZ-<bez9WV3&gJvbj_3gl(nvbv=sQ
z%Qa=O&6dxxd-=bAv;A9Fy*I*!Yu^fOJ=<GRclX%vitW8%yl?M?b=UWlxX;`3e3|S%
zd(Y>4_u1Xu^C7o(->$3W`_8P-v@MBm-uvk5*}d-%OWSGat=V^VmH)o7(~`D%n^)|;
zahzlSvL*BPMqX6hpP3ZC@9XR4y?&qc_sbV**s5l8@0-*3bB|)=40}CkuKf{9=IopL
ze&XJH?|Jsl(JQm=6_VWdQ0~~i7h)4_+r-c8+H>>59<ECTdwDnW?pxJ5+g7~o@t%N-
z>iagy)a~t?bbYTq|D?T^Dam_}UVFJ$>)T_iTPK$9GhfKP_p=<|zA39#?@etC+#C98
zqwOxC%za1K-M9VY?`3y&af7{w<@UX^ek|R4#pT-Go!`RuT|N3`56fXQdl^I1{RbwT
z-g~{b(>}YBWnX)z+rH4<wzjFYx%&ikHS8DFd)bI(>D%w!GtK^O-Qm4I=85gIJD|G1
zi80RhlY)}fcJX(6^{+77x7aZ3m0#tyr)>Y<z5R~XdsVqxZ9Ymg>^pNRe*bIf#=Shc
zJNFh$UAs^G=gqy9Gd=d*<yW-}-tyM2oQ>1Y*KzK?4|Rv^dpBR*YqXPVZ<XXs`|Bn>
zd%rc!*!%B)i;elhm3!Jt4_UX(DcWx+w9Wo%-VEF7Ra0y>HM8zN5q{j(_<Q-@!?Rb~
zofek0HUHpY6Pf8?Co-peA1BWwJBD@VY;sRr+1nQ1v@dAEWP6t#SFOZV4D1s=FW7(H
zS8o6EjPLs*Qy18+u8iJW($>Du@cK5}3sX4lrwJ(TUsGJMXLa?Xy$3!S+OImtw{O$K
zS^M+?XYBc~&B;zRw!v<?h?5;Zqujou8~FA;5IeH><)M__kA79}UHwXU-&(gH`}A98
z?^PA&J}_a*!oAYXllSV_vFw-em~G2xR%BbuQ@yu7Y^}`!!`s$&m)UkJr#9HmTd>kz
zREK$gL`L?W)p9BJ*S)RmBNf{BE%cgW`|kGU{f?YW`!1(D+6FA^-o0$qzr8iJGp(D}
zwC$@~y=U+3MSpEX5B2Uj=^$k9KPhD257DH3ch++5dnDRud;WUte$9e;_8#vQ_s#3s
zuy=BW{N8)}WbE^j_wKKH&$MsZ^P~GUj>_!2tR!x`&uEj4`sK5G!)w!R-~HaOucuwo
z_DJH2JyU~k?)l)hbkA9#OMCBCyX?IfK4<^N%q4q;dASeFOv|)#mE5`a`?@x}%}3ex
zecIKsZ`yZ_eXpKO+qZu1C7aIatyUtJ<o7)+pR~7QLeQRj%^&SfdhXbty_apDqR_Uz
zv9E90XC0lqZ$j*gy<Soi?Isj;?R~t|Wgpjl>3yKN{_j_J?sd2Ry7z`Q(_WQ4t$nAw
zChUDOLEHB6r#pM!UNYUYcT()WTm9?@de6_;TOizEdvglQz8MOP`*|<3@8vB}+IK!_
z>)yN8-}Z(a>e}~#VfNmFD<w9!>;3krI|%J%h<CAFbHv<s(}MtOA?}6y1P|-)i;w$g
z*R*_%ec|DKdo`9Vx9^)`XXhod$o5hh$36#N`~3}<8}}*1t*|Zld1B8~jqA3SC43Ie
zk^%eXr9a&JN92!nS1jMYAB&Us*|9ydx!UKocV$cR-diyf_TRjbZr_*HxBE+T#Qr2{
zxxEWi|Lt4+eEWVi^-p`Hj;!1B=X>|wjwz??Z#nDlKlRqpK2t$tU&NE$d*i(P_wgC|
z?_tT)+V`h%!`==?Ra?7shJAeZ_8thB+rLlKLe|zU=<@z0=lS+6&U?ML_pX`ktzS#`
zRxF6$xAx8*E3>e3_8Eeqd$txv*nH$(wa<KZ{@!DI*IVl-PTPCr`OQ7%s`>kt7z^0U
zd9%t!d&}&7Ss^xij!iqV_ukz^JDV>_d-l6Z?tgnz-cCt6cVFZh7uz6{_I)9fSM7h?
zwQ*lj^^*M=ZBO^y7uMbL{^%c@Df?dT%Q|RbzyI*QeVh`tdo>#4?5E1=+va5U>~jb(
zwVt79y!VKdv7O=WXZF!Q81}_kcI=CZnz47<v>W@ZRvp+sfy;9Ld)+kac;C$Z5mUD8
zt;=NH`_=Y>z3D%jz4;f8?-%{wXzRI4-#+J##Qv=sZMJ7(WA=RPkG3=FIAni0^T58_
z2YL3{=&SCXwD`5nij{3PE2n=s@br=W-h*AI?GNYZ+bSz5?YGolY8yL2aBqgm8k^O(
zd-lJQKW953Y2BU~8`$?w-Nm(UrVi8IJz8zHkMuL`S8NI1m;X*`@1Fk2_E*@}?R9y=
zVO!JBY~L66V4rlp>)xd`m-bBPpJ@An>+}I0KOy^X#uN6rIc&H6wUl$;O%BkW^BwzN
zhws>DS9;WLkMp;ESGLFQUEXzJ?;VY{eQ}W+_D*1v-}fup(k^9J?q0d(lY7_fy|d@n
zJ?4F^PrLVcoLIQG@tpr&AJ#oKk7~E?ySbu#pRAXyZPdXGJ87P8R-EzH`&Wjq+k23g
z!!~tls?Bu<hyAQi8u#X=$?aW!@qv9Rqt?D^MnC(?3$A-b&x!7<7kadJO3F-Yk-z@?
z+VY?5wc8MEd+o!s-2t<g*t%?8v)|8rm+eLewF4R7-S-=xTfE1jvuB^*dj5TH+SB&k
zTA*QfgJHUTPFA<A+U%Hp5qi`1>Tf=_*W!-ezAJ9KZ7Y0RY&D%0?=_sAu($p+*M1%;
zt-S@BEC*!X9^T8&+qLiP>7oM;?l!i*w;u0(7wK}q>PX1G?apckWUS}ezOcD&BjMP;
zr}3wnt#e)P-U5**+o#&M_A>A(SVyhywQW;(+<#@((LG0ZKe9Q{cg*I&R0W&obKUmp
z_iOKyD1WlImnm^y?ddi9Po~=Jv%EK9-@4qL_QAm+_EL94_O;$s+t-kvWqaiS+rAgy
zI`%#&K55f2{m@>WhrPBkZ$Ix9cRIb7;aY-?H`_CtVt<W2k&@bbi>?Oklie_Vud2`F
zefM7O*()@?d2hfTNn5=(rG1K5*4Y2e`oA~lPv$;_uiy5}IS{;evgQJN+ah1v$Y0F%
zsk8j{UHT-xS8<z`Z4RsUUPieYdp|Exv^_fOz`j^Xq5bpY&+U6$tFq7BYx}<CDraoE
z67K8^TGg^I_VR{3pVMyaTVeQmZ)E(W-F`os_Z~cXX76TcfqjL>{(B<IjQ9TOFW&1X
zaKzU1)wjKe<Gc5HA6vY4!_k&~D{uYXJL_A{KK`jad;V=J->YA~WseWPtNpUAj{83J
zckjzJF5J&_F4k7(K-0budC9$Ap9J<z3_HA6s)T+2G*9(?A&;c@mj33oS6r}pkI5^Q
zea=n1`%?Jj_DN^)+Ka^hvbN@$V?Vd*<UU?GCENW^3-_+|`EDceS8MNP`~7<vWc;o7
z9`CZdIn!)!kdyX)_e2Yu81cTnqAfFQr^HIxTfE@kU)`a)@2A<qy^%eY`zl|ov*ET#
z-4}V|`rgZD{OnWJChtA?t#jWQ<Hh!z57yiC{MxYRMU~v%?bi?PUBKjIJ;lp+&)eyb
z_Lf}J-E-)&;=alEWF00KI`6%j_IeNFuGe<8ncdb;FUjrGTxEH{Y|WMZ?-_UOUAOJV
zK4nvfeHwudyLp2c_k83G+UMNcx%YSI{QdVIhV8C5`(&SOlVUHSHO2n<iA@J2z0K@G
z9PjS?^`m{CrCyKiO(rILgQo@u7GD+Kr!4Si?^91(d!~~1y{_#y_Qg%;*(?0*hyCq_
z&iyS0;`TLd6Zd-Qa_*CPwQujb-~6_`>o)H_kz}>|6nnwGY5~@L2^TH)e>i*D=EnU{
zn{N|D>`R#i_C8I^-pkt`xjSV0vOVj<W%pGdZ`^gWPj;V5{H*<T+t%)v+)=()+H>Xs
zyS}aaSHDf%mz5W?hb{iLy}Vw?-Y0v6_ca-`+s@d{xVNW9ai9Nbm3`T};%s+5IJS3_
zjl|xoZzJq(TkY7FHZ6MJWxj`d|L@n|TWa;n{%Xd<{fVAEdq0IY?)!Iz(?;*X;(fOE
zG7gtd9NfG7Q1|}Hwo>~hfB3fd?xxzkzc(J(d!DmvkDTr4eThLD`&gq^@12yVxbMg`
z%e{)Ti}!9ab=&)cFJ`~s9j?9K3q$QouSM^D$gq3QdY;$&R2X*Imp@h5wa3zUU;ET`
z`=+xU-s|kWV(*1Y?LBvHgzw#$ykhU}o~iq9rY^9{$#SzbIKsBiWUJnRX4Osm?)xYl
z;93^Ezbb#q-lq8d`<5x5+8Zu>a<Bf9D|@XCkL`)@-LrSUu%C_C?J2emvQKum=mhM$
z=E!CJYS-+2x2`GdZ*WrI6aIsBAIGbbePN1a_CFsA*p{&;?=v@*-N);~u~+C~z}|;%
zA6j^xd$7yZp?Tkl{PTO+dV2TV<dfU$w2*C|Z06j3^7k0_O{jC(rx3W-&g!Dx-nb(l
z?bnuW+i!Sfx;1~>nY~RD-`dsHY}jjfVe>u{brIYDz23G7p406b?o8kJ{6vSX$!z(3
z7p^JpJ0s*`n<m_{cftknJ?Fo2?$i3>yuan@&i(ex%lF*(NwnX1Aj>}Z8_WLgxi9Tv
z_G|2uwK=x8JkM-zxpb+$NxPK&MV2{swbK)<7l-88KM~2@d-qhyUazd$eG@8q_mzH?
zvQ@B-wl&zru(#7rbKkjhOKcJ~F4!p@Z`fOJYqc+CBfD)ww)@_H3?2JC=PujxzI3)d
zm)M^@Wy^K;evkaKS8_?z-VUKjd({~A_bG|;+IoEH+?QXevRBGn#da&#&V7!hzI&55
z$J<7^$nN{1b#3p36`S_DrGK;wX!x<Wmv!mhhYgGN^)Ip7H>-T!zN(FucFKn8whC|k
z_qJ(X*%NSXm0iRBx%+=;_UvnBt=YG8{>=R?M>g&K?=Zu*N2F%&H<OQhO?OG#e=b|K
zZ@&0-8~LKZeF1-8?B@E(wO_{U`aZ6t_xptQ_U*P#nq&Vu{L@}W3Gu!EqY~^^d+)XR
z>tkTAQMG*UJn!E9jeZOF@f>j7`!Zqrp7;H8?N%=cwQE!9*|&Dv&3(&kHt*S>d46xe
z#r-yUE9>q4o{QP{+QoFgbB^5p<*y&@^$!fU?H7pHoBuM|c8>C`{eQ2u?DY%e-K$l!
zZ*R<sroE;K*Y+&?;Jnu({K38y$8URIXlCw{xDmFWS2)G?P3-+W+85vK-?I0Z-QQVh
zdvhP{+&jal!N%n6F<XhBYpkDMP24y6z4_kc`3v?{{57!omF>4b>hyzsvtCW#_d-8%
z-|4uFeW$P8-y?UmWuMfcU3;F{JML^rK4iCZk-cr5pqp*ltG_neYj|yc-S)N)R$|__
zb?cNpod22kE!w|wpD*{Py%rC)?LAeuW-sr_Gka$QgxGIA`DCy3vS)i`lp^-@zmK+i
zDX@A^z0B`jx^o@(+c*gA*A7^?@9XNTdnyC=?e+BQv##;r-q+x{+%EY1w!OhO`1h@o
zZQ9$igK3X}htB@R+Z3!1wXEAWeF}rUK=Ut~bk)E6N`HCoP06a-w`}hU+o|Vg?XTW7
zWpDrK*nJ71JNMq0qq%S20r7oZZ5sPHc2C{=Z@bFA4YRWL`aV9g&t>+Cy<TB6_jgN8
z*mLdpo_&n}cka7#`}*Fyv*zsm*Ii}*a^Zx13l@m)uklK>J@nwKwfet?Jrdn3_C`%&
z-e-R0_g?*qs2xeMJNMf9c<p5jHMj5jow`S^X{v3V;;y}ikBQsPR@<@Ha<$4n*>xQI
zW>0V2H^H;g)+MOkHnaKIKEEvu`y0PI?YC8E-PaT?v5&*^fwhRn@jZU(qWgsO-tOJI
zcFJD63tsy)|8eZKSu?@5EN{u)tZ-`^(T~6Pem%x!_nP73-l~GOy?@f?@6(#SbYIJ^
z*EY=z3->nlNbQp-%-?f%xqw|_A;&%jk=ZsE9inW*cq{iUY3<*u_E%@$-<eDI85m68
z8?$(kZFz3v-ZP?Yd(Y?f+cxvh-v41&*#2W{cJI3^dvotG*QkAuyo2}p%;~Z{pmlU#
zLH+x^LG5k(rayVG+d5_S-k8Xyy%P?u*}G-A%igo61@_JTtY@1p&c1Jk#ai1QA)~#Q
zyIS}4t^KxlUh=$s=>i9A18wixxTPl8+r_WltGoB_-pGfX_A<KD_p&<s?k)Z?bN_pT
zb9;Lp2<(&U;oZ0Xi{8GX^S1kL7dPzxA8fV%t`Ez;IHe`~FGU#IiqEm#*LUFm9?5y!
zb{k$C+PCO@z}{@bN&B@}&e>&dkl6P_<%RXU+{ycX<hI+s`Vwp1Hkp5)z+HiTa_4kx
zr^__k{}MB@F$gTP)!5^;*Gfiwf2g{E{nZ6q_EsIex>w-kqJ6v1EZ=)};=w%?hZFY7
zKiIg>VXE;ymZf6*VwCLc12R=@{f=MWefZh?{Vok$dxO%g_jG*OWfSf>dvA20yshYl
zioIKQ*X%#a>9yDV=%u|ke|PUYxkzE}ynu^)?;2|E|Ln<Uf4ujBjmql^`)1|DeeWF_
z_BqC@?3uNdW#6O)ZTn6JC+=6`Pv1M?;qCpg7O(AHEg$aF>+Ij(`oh@WQ=emhI<MP4
zsSj87DjJCHpMUE3Ue)+jd&>+0_D*te*k9V(xwoe`&rW}dxow`r-o3Y#F4}!FH?WWP
zQQT*7M921pxS4Ilv1psW&3ku;Y+q}`u(@R4>Fl08caNU7ogd<Dv#WjD9{FSj`^=@Q
z_lnjnus(3>vd#Z^;eA%$#P-F%;oPenW3X?DTGsvquUY$j%U<tw{TQ^rO!KfkS4i_d
zr^Qlxr^XrEne$89ZYtupEs>kHx1c@JrX$98@7J$O_ig+6V{dYg-(HVrGW%4D-1aBh
z&)e7BX}eED<jmgqly`e&nv3?WjBc?na%<klE*Y`!rv1geddA^)TQ{w@yWJ_g?}k>`
z-q+`h_bvbZc5h6J+1_8pANGE@F>|lQoEQ7nOVr!wb4Kr-ke;#E;Df3y)8Q@~2c9>3
zZtyqQ_nLF<n;bjK*5k>%y*q?#to>fS*_W&ly3f4!(cU_d^nIn<SMEEx?El_H>u>F~
z5k0@>#og(9FMa&Cr+R{&z4zHO`+ldF+8WLe+AG%OWpDRBcVFC#Pxig*Sogck(zdlx
z6LB#ARk&B@&@-Ew<<k4E&S<b*_eg%<--FuwDxTKv%{cMdraph|o~v{C_N|>WYhPFO
zx&3t=d-j&SiP+1L>a}-6kgDC^wxxTRkFoCS(Qn<Gyr0+BJ!#dxm^+vEPB3Y=NfNf&
zcl61%JugrD+C2EJX&bnTb6?BHIr~0bJhu18qb2)R3D4O#>(R<RmrIZAo0bz`TRX*i
z?<`NIeNJv-`)wSS?(0nbv+qg!qP=T>I`7|CCALpDX3f4#`6JdcFYfOBnG(2fr(4Uu
zFXEni75C5Fy)2?`uK*9<{@Pi6d#z@x?v*s2Yk#ibhuuGeBl|u#b=xIO5ZK+hgn1v+
zgVlQ-IBV=nRp;3Le3Nhcp+$J#sT?Ld`G|e{Y94Rid%MADpR?Zy+Y0l}y|v}9_O6=f
zu>Z+VnSFv@tM=Z0G1=Zic-vln=DT|))p70n;q102hM{eraoN+ov;5U<FR$U=Cl+>h
z@ABU%`*gC*_i1xK-*aD1a-WAt%bu7!s(ZDIjQ84f@!Fm?f4nbydAF^PQ}tfg(m=a(
z{T`d0$<y{Oab0UK*4eR7<$vDZS*tqt?L6IV8xnYM-~8A8_D4jg*+vFx+g91w?Onu>
zZ!=x2z_u=Z`QAGLv-ic=U)pCIe|B%x_Z$0G?76e&wx!wL7aM%++4&yt53_l`xB9~V
zeHY%b?#-xsw~vFvcb~&~2m3kZFZZ6#&D_Ui#=K8tZ@jHR^@Dwh=RfZLynWR^*U9>O
zd0F}PAJ5yqZ^P0L)@OIH@BQlIWOrm!<o+#-<My%#t=g9oKi|gU*5tjf{?4@BtG8vJ
zeX`BIx6gL(dr@F-*ATR9|4+-Ty?zDRw)vHx_dYJ*wNLu7Yi~mHvAyh51@^tGJi9mR
z)2h8ieamb!d+qlzKdjs*BGzT=EMm0pu+EFU;T$ve<|(S$tDG>lQ*?N^*H(Sv-kd}2
zwv*C5cfX6AxbMWC#rrH)U$kKmU$k#}_|?5Ni9YrZ9FOmt{`$hcXUz-uWqc{v_s-$b
zUMZh7`^>b}_Nuw=-uomla39x{w!N)_zxE`=G~2WV@$5^Sz_!olz#Q9Tj~#o@dWr6P
zwr|Hiy?-ft<4hLp`Bbc6+kWlL-rF+1`<ef3+v~P|{ocb1^Y{3zdTY1$*3P{`J6ZPG
zaXTM~@A|MW`C<2-dj}Zz=_+sCTO_H!FJackz15Zr_Fm!S*n8E!)kc3!tL;4g#e4hj
zGwfwptGP${{igj&YmE2n&2iuB@JDgKn`rP}=lAROeSPwJpKUtB-c7v|_C9|mzfVnz
z*XB#H%-$7!Df?cDeBS4={fEt8m9%|v#j9<?c1_*qpT)k<te;`uX2WGRH$+$LJ1ZV=
zpvP{$O~$(`HuvKv+b7=k*>`~7VgJPW&U<x^oZtKG^CH`u{5pHYKJM6aBAvrlL4$wa
z@6&hphU|Q~=a%!H{j#rD@BKCHyiIjR+Fp&#O?!;;_F6AHd}r^5Wj40ae~kA1OBA+U
z7GJlolzZ#G58Eg1o4kAD{_L#`dzI~3_x_0L-n%(^_MSaGtoy!9-ENx|*l#<-D$90p
z&Jml22j$i_it&4A`s~^3yW4WFzo*!~n!B_2w$#S%?r;j+cS^r|-{LEG_ZWQZ*xO{o
zxvz2Vv%M2*ocD=t(%rX2#?N+Fg~{GM5p4SvB#-WOzx&p1QB=@A_pMiL0(y7sxx>8I
z`cDS$zLWa<_r^Zg+b1Q}V9%$qao?%FjD1|T4{Uczx9(#;z_I_@%Gvv}(xmq}dpzH_
z(?oI~ugw>mFQ2>iJ~+JHdde&Ry?-aG@B4NA^<JYd9Q%%?@7`ORXS(l=p@eOjjnKXV
z-p%_?S~>4kj%eJsVec0^snd~r-{jr5ZE^m*@7v->`!u@a_VdkJx&KgDoXz$2sr#x~
z=GvUHS+uWn(-9lH6K!^Vfd}{2dxY&h`f}#}{UZ7{9m$jSDeGI<Z0WJu-}^CcKi|Kk
zJ)U#d*`<H;*{88)tBu_1?!BRNkL{I~S+%!pozY&0voW?ie!kzQXWG8^)<XIHKOP+2
z71PpX#nydp@5G;n_r9FTYJZz)>t3Tm&VAcC_U{u_IJ<Yz>+SnC=l|c+>Ls&(2j`r<
zUzW%1JN)7NJ_lC=TRTb9y;px}?YnUN{NB94nYM}2m-qf_<=N+*8N2sq*mpa((h~<F
z+&=AXJMXZMtKpiR)nhGNg-+(ZKGzry#Lm{+zgzm(Uam*t_9>BB`*!*#?3*lCW!>Ll
zu<vD3^1i0Ym3G^@XW8seU2ga2a+S^bCAs_FTuMB!ic4U>(95WOvpMtjnP&Xod;ir5
zyZ7Hd+vPa+?Pc6^VXrOE7rV7AlKXsb2-q91e73jq&bxgdE_~S+@P7NgJTt|;x2}io
zK9siF_CQCIO&mv{ZQ_p4dy+Q2+NXWs?B1kvSL~l%Ilp)La?yRF=N{YMkCos1T3cw}
zqJ>iX=K5OiaXV$aFTHj0UZDr3d!1fy+kY)V%f5P-i0#yO-}ftXU9jWw$lB+A+GF49
zcXIpIYfJ5$^G<Xh!%PF4x%sd5EMk1HH{8{CU+KoqJts~Z?Geemzt^)NdOz3EN&D^x
zPv3j&)Xlvv!b^4+wuSC@=bdb?a*|_TnKk#mO%`8lHu-ee#(m@6e_%@I-i)Jl``?D^
z@8kSqzSlH2a<2#5qrGouaN6u>e7R4=g=gQ5mPva%uG;MN(TUo7ii2;TxXAXsvYlo2
za_fZl-70Ca-I9EL&k7!GTb82?`?F5)+D~%t-TUh5qrFBNOZRDCjor7OeX7llx>{QY
zrGI;)g`4*76pyl<6<uaGZJwrW3{UvpUvi2E81C)bm%2OI-Y50sKGwgJ_XvH6v0q?(
zY47)s%l4+a3)|j&rm}CFvgW?VQwwb_y>GDHz;SXfhpDx#ec1);JyF88{I@!6ax$#<
zmboq56Pwp=<GU)-Zq*H^z42>1Y;(QdSRE^4vHej$%f>Wr<z5bkG#eXP1KS`&XS?dF
zjQboW&9mK7m$0|#ZI|us?X&hOEZ%2ZlPGSJJC9`_v-wwBJHLyzbJ}LvR{mD8UBCa}
z-h|kg-Sc#`Z9mNYZY}jv!*0uMGuwtn%iWLGP2ZiPFm>-9_A|E3l^uI;y=vV%`J3e4
z_T?w`90*{yGqumy{pIqez0&PpY(+$+?0i$b_l9fr?48oT(dPYXmA(IV^6gX1++^c&
zuwu`{%cu4x?pwNN|E}{k_7mA`&x={@_0U?lr>$$sUTwCvz1nwL_IT);*$S?1wq4Rc
zb$8+BGTYl_d-i^~vv6-siP_#K_9nIwjfeK~X{Xwx`LNm@5M66~;>Y2=SEkIc{diPz
zuU7qGn+2g)ch57g+#A?zy7%nQV|(XbuHKt&AihtI%V1x1)Q-LTcJtYFPMx!t@4T*^
zsxFV6%8s*pdN$77^X_Bl-sG>>_A;2ww%Qwc(6&kGtnCSb$2Ls|)%OZHs@U0@ZM9iG
z<;ULeT7K*I#p3%Mv?TY1<jUJ^cz(*}BhT7B>St!}3h3Ou`=Ywb-gAFC_m)^qvTc^%
zZhPk#^S<H(du;9Qtls-NuG_Z7e#Pz-?q#;7kCbgV0_NFnEL&q^!0>XfzQfeLoGY02
z#4b|b+i_{5P1FWAyEjTpY%T8VfbQ7alUzK{#_Cz_-itwLdldv`?5XfSx%VVPfo*QO
zk<F8im3tGjF7Np#c5?5|=Lc*okNMkdc)Zqj!@Lc9XRZ|7cdjRXZ?xZ|y<IOB+3L=l
zv-ij01$*~=_P0H;c*WimMQitJJwI&A>N#o8-|2F8QjZtx_4F3lbH6!Yuc<fVp18U8
zdrf9U?G2w=zIUC$sl8gBbM~GryS4Y<wH>xU54YJ?Ih*XO+?&65*5?_v44XP_oV*nG
zJ(*pycdJR0t$l36-r7BD_l9xz@3j>7v7Te9xVPq9-=3s;wtdHH`}f|E*}eDQw8p(j
zaU6Sh+WGIPpUu9{aF428Xm8eDl|@HvAF&tPa`#@dF?hOW&#QpbHXE#S_F778+Ow(m
zqV1|%T{g>IKJFFXz_3r*?fG7px!Y};xt{O6uV%FO=A84kJA)%^Ba#p8a<IE-ooK$?
zc8(*v?UUBGdv&d=Z957?ZQpqL+B)qvwO0H8&?e>$yRAXvU+X^(clX4JKi>N?HE6G6
zjr?BG56AcZJGIGn#iN|PPx!d^ooRW$*K?_;ZQU34eLN?m_xer{-M5wF!rm$O7TZ+J
zGT$3h8*Mw!*>KN?mZiHBzCX8qKBssuuYd909R8Ji=RBUeXF{FbUcWPo_BIr^>}oN(
zZF42-^&U}GOS|S1r}loBy3OX1$E)3+j!fJ8@+jZFhB=j1ixgw_9&z5e_lg6z?c=r6
zZC|NZ@4dLCYwurQ>%CSRkM8Z2`MCES!-2i`EDzdtDB9XcY%#H&=j60^iH^1H9|cX@
zV{aPvYUJweIhVt{cl(Uyy??vA_uPBQz4ye?3pVciLiYM_&9^=1w`ni?&bc<x8z=1j
zu`k?KV`A2x>~{jT?JHi|=p5N=GyhrRo<%tmZ5a(7?74REf$bN)?!8qPAMHK!{Gbir
zF=x9H<qLc7_<z}ZcV_<HE!FC_F%KhcH6NYZo7CxL%Vx87Z%2^1t@aYeebHey)>BUO
z?5*(Kuve^O!Cu|R#e1XW&G+oRyJ4r&b@{z9CX@D*SR2@0thU<gk{oB-A-8-_-HMfa
zSNwCc-n_DXFT?$1mLESe?5+9wZ*TXJ4x2kH$85iK@$K{fA+>LfhUMPJcNq5l4t}{e
zI^fRUfID0FCNL`4K6<ZUTXZ&N?~+b-+q~8#dqou`_HB1b-R<};V(-m^={EBxaqP7)
z5#PHd)O)X})XhE1x#sLqUlO_3?$|op|2L=F&bj$$uW8SUy*FRJ+ACgazxQ)~p{?u9
z$+p!ezT5hwI@lg%mD~5H<LsWbNq6>YZR*|oY`d#nu>Re>PndITH7q~c?8#2BU9`Z=
zCR|Ws@5a~0R;!)o>{*knVf%5#p}pG~9c*8`l(ot2KW*##X8oQEmco10C+6A2aK5v3
zEkCo@Hfhz~nVs5u^<<9hy{ELphUxGl+Z8j9?OF6E(sr@XkG%yFBDQ9KZETI6e%>Qs
z-nh5<goBM(YNpM`7CF1#heoz*Z#3AN{$Fez!&GRit2$$k?HBjG8J1V~o+>KZ`~Q=n
z?P|s4d-vShwCC5nEW3lr`}dj`_t}1m`?=@IvKTvo#;Z09zW=e=QP^%P{UyixP|>=*
z29ft|SHv8(xoP%y@5?;dy$^g1Y>fn7TbnKZvAc18rR{y0`FlmV=j?sA>cw7@h9kDK
zSNmGOU%z(mX$8)G&lwx`o@-*;{V?5i?`*@#wsIzG_WEyHylaQX$~{_d5AJ!jOw{(n
z<QKcQFqH4zmAc&4c%9N-mn_A7FZe&~nI6PwE41;Gt-^d;YlX>wt*zHO*(zK5+rF;Q
z+Y`Do*lva7j6I)tX4?J@XWi$Wr)sCS)^e{#e(LVGjNkV5$!@R~UbcI0p~R8B|3#Z^
z>XR1P&hl2b_497C?NL3wH)8wZy}|kpd%u^h*qbtO*&fM{*KHH$1@2{>xO;bd-i5t=
zF*o=2C<|DN`mVF(Z2Pk(#!<sI;r(7~k5yuJ6T77LT~KYbKGZ3)&u>?}P2V?XyM4O?
z_P+lz*H*sc;NF;rO11}#rdaPt+-|Fp$86`aV5{wex&yW|?oGDlnZUd6-feMP`TVDQ
zBIG~s3BP=JZ;yz;-V=6mHvDV9?yZby*u%Tv#=ct1^8Gm>U3*m4f8DFjp}fx^W5xdM
z|9SQ<&`ICd^^$pS_;s#*4zDKcOA-;^7ZAPN*7c&oK6SP%+jaFxd(VfQ-aGx#wY}`#
zoc0=C3O0pDx%NHun!Mj9dih=prp-3|3+n9yOd9rVk80Rsn<jeTkmsy@5+z;xi@T!s
zUN6|Zzy9$nyU!)-_nl6a+jsgThpmi(<-Qn)wteEAFKpyZS$7{?u-$g++Ix2Dzt-EG
zt>@nNcYBJx)~S-c)7-`P2}?ZP@BQxFUa3T*y@wCW?w#?pc7N!h>V1cuW9_qYHtn6b
zVyUf|+1h=MMqjKyNzdQclrqCsNO-rcf`_WD=!*k3mpOv=*)Hnc>yqASU)s3H=Af|E
zz6sg$_Wo?M+PBwp-9CkPR{LB^{OxaRl<xUAW7QtVk5l#)-&nf0(J8_<XiL++b2h@Z
zGXwJXB@}7yW3X<sI=*J+zVf%pdlNd0Y&HwOwVQK+ZJ&YHAKPU+BlaIz*t9SFoRMwQ
z8NYpjs)F|Gyl3y5P}yiN5z4!#(4B4HEK`NO*VeS}PpaC#FCmm~pYp5DeX*Hrc7HOx
zY_A6I+?REUdGEP?M%(G@{q3d-wb`pVZQ6UwdhNagq5-zCvkvZ?D73?Fq33+t3G&+e
zv>x8xZys5=m*Hcm^{YKU_Z^D9xF?YF(EfrAZ*3lkrR~3VcgDWZmc@JjXiDz;AGdaY
zex9;Dn_~YyucW8@xHY8qO}Lx4Cu(ip-lPqjb`rHEdp7=kY4`6Tt9{ch&i#kCW$oSW
z@NVzz57YL#R9M(Q5H#8IYr|pNxi`i4rL-K~TR!3C-Va^7Y?pC}9++;rX7AQp@2xk!
z@ILU^Zkdhn^mBXdCDr!}NbIp$U7x+z_QJBg59;68T)W<H$6lpj<IZ`0AICQJJu4NI
z_62E)?_KpdWZ%)5FZZoElVX4N%cXsmuh;Ii(-z%#O>oIR4>gW`^SeTAj%*b?u)EoP
zZ^<jEeJcc|ZLTd!-)pXW+Q#D4vc2aX_S)nZGVQycvvuF+8SeYet)H?t==NPZ;myT+
zEk83Ic(6}wpWN=oy;A-c_wwI9w@;zFclQZnzP(NDZ2RV?`R!vlC}{7I&bRN)`RRN8
zFG}rubwP1ovj5aQd|Gn0ZA$xX_VR7o>oc`|?-zzY`&^uY_kLRN!tTdPfdgzYr}r8i
ze7|q%gV%fb-gNF$xh1*(kB+&W-~T?_Q%lYE)vCnqWo=2bjpU2lH<@S8{(=V)d+deJ
z?u*)>zfWmv^uDezrad`Dzil45Jlu2n?4*6>J~I1HOsTc~n`*H8XZ=yzpx@i}{gutJ
zTbkIl*CM=XPsq>1dlChY?M>)(*^~eH#NJ<b18h=7?(Ge@m2A6YbL0NU6Ta-poZ7Ni
zpli<VXoH@;5AyqM*&`qC{ZO!N@04Bn`?}uj-4|rpyiZRlY+t$Pn!QFUJM9{qxNN*q
z{r1`P>h2Yo%Ce7>L1>@efz7sdQB8aOtk&#RIkL;zK}d2h$K;+pYyB7OjaWEkuYX?q
zzQ~Ckdv`_f?NwagxzF%o&pxw9A^VO^KV*C0+Zmhulm?rG+tc=an|66`+7zL^Tr4a0
zu1x)7cRl^r-T<$FedV)P?Bjd8W4E-0z`lsj*Z0<Du<e_aqO)(O-K>4aLf`B+ChXoP
zWZ1G#_f)ryXK&lSr0CiE%=i}UtH_qv7n=TM-_dOud)4b@_MSYlb+2bh=H8pzHTE7k
zc6x7ks`%cAi9CA})~MO=y!yX)cH5Ud-!6UM`)K}*eQ$Dw_Gt>c?z6jb+4i1#=Dv5!
z0`{<Go7vygJiX6{d-uK%$@%-8yG8acDqFU1R#choa=sh;=D*^!{lL1=mZNBfee_Gk
zy)K~#_Xpan*?aIz|GsZaS@!+EDzJZPE9-$%*S_wR*FL_drc`17?O*$>WhXh>S}nR}
zXCQQ8@5iY8Jx{LBwQY)3+t(>4y62>v_5N8^RlD=dUH9$~TfKMb%IWr3HZHb3ym|HB
zI>nHETnbV9-dM=(y}x4JzO~H1Y@|BF_j#<juy@L}?tTAq*6hE0XyV>HrrN!y+Dh$Y
zzVPkcd@6bG+UOltqV=JByI(TyD~&y18~8-%K&A7By_|JZ_fFb#$8M7U8td(g{r5KS
z{$i(4tiS(e!^FMIV~^Xp%~IcIzg}+N;_~-<^<`W4F8=PZ@81uTeVH51?NvFaWUu>#
zZ{MshQ|x4>IqqW>RNpVSEPMZnACvcKr)lopTRz!deVf_-rd=YocbZ@9-6+bm|Fhe!
zJqPco??1Hn*<QzOEH?L6Pv8G>SLObao$a<kr8oAfyj0wqVY+qiojuKan`Rd8eX!)|
zzFS<=_deKu$=+&Vn{B+1g6)ot{`;mpX0|Qqoxk_^jMcm6njG3yuBWpv;^~(?vJQIt
zlImCNdsfM4KT+V`-tJr@`xSER_RXDb`x4q%_wK(@XYX{?Vz0|J`#lxiLN@G?tozov
zuHM7?_Um4wz_T_MQVM(dmhkR7yKSP~OO^xsCTUvkyZPki-fSsj+fai~b{6Nv><a%b
z-rHYnvd?Yn*1Zchytn2N+-1+4Ut=TonAcvg&~x7+)=PWMm>=zZI(PD3t4H^DJqum4
zZ?^Bpy_(ev_9mY_wO682$2Qbd%-Z!K>%Ljf3-+t-pJLs~#<ss}f7jk=M@08*v(?_O
zVp(m^T*<L-Wxw3M!xvrmRjm%(_sDGSUbZa@Y?uEs-op~OeD8gW&i&UFy!URNvfTDP
ztKdF!PN@SuCUf`wI&{}|-Lb8ELwV-e>Mj3h+xv6o-gOU}_o_xY+H!W?wQ<~_yO(GG
z=Uo?;H(EE8uiWji(8Jc?T$A-5<BmNKwy)g#!eQB-o(nT=yV_d!GDyW+3;$ZWHzrHo
z_RNxg+Y8^*_xxERU>pBQZ*NP{(>=R7I`;|$?6;YwIMtS+IdJa_;SGCdELgI)`n1|!
zj>upelUZ`MCRbkUS#a#G%?!D-wo>iuYz_Y2v|%{pzFVnlg6)<seKs57f^1eCS-5Ax
ze<@qDC!1_DSIpSk5%bW7q0-uxZAH)CGfqslHg+p*Wn%00o=9!o`y``juSweVJsrm%
z*vR;6+s<^}yLZO50NW{VIrg5rePnOys~LNLt}NQ?a(4Hgn!ZxotoBW|_abN83bfhU
ztWDXsS8(5wy)z=6ZH}CuX`A4#xQ9VHb<Yn`o;^=K2<;7!a<DZFFWLJb+}FlOr`_hj
z6z;tVxvaYzly~jQn0wXc(&lxxEd2BKeDkf|8}a@79;uaydrP=>?NR%Ge6Nf^)7~vB
z1#A<758F)0vD{-|?rxL9dV5b$a{S(el{aiS^uF#9m?^VIWMb>yeU2OVG6|ou-B2^d
zc0<j#JtB7lY|VN&_d1;9v)wcM!5&BM!+V#Qh1lMaJ78OJ`=w3C_qM$b=KWSnJofC3
zu<f+bSgc`tY0)v;5~F3d4A*3K&vRP1SL0vfo|bE`_xxFAZTsc?qCFms?R#tNT5O-C
zcJ6IR_S>`MWaZwe=I8d-xZ2waI8@o}D5$V45m;oKRlafWP1^-~S6$q{w_zfewb?9-
zy$3w>Y#HKq?JPdNXm501>0Y5*oqH2rIoe*>JjdqJvLm)XcDC+4Q`l;&64PPh8X33u
zg=C9O+5O43i&q5f-Lq%;o@pnF_j=^-*~_pX+A6}8*Vfx~)?OErwYCnv%6mLtMeY4@
zBzw=Ad}~{goGE)x$n3J+a_+2cieZm!gVTvU|0<hpr`&4UThAA@moe?o-X}e|wq}c2
z_A0dY@AWz4ZEKLQ$@)aw%snidx@<I-Otr0<WM=!|)VAI8=C#<`@NBl#@QK*-!TyQ0
z%t}7nr`Fr|&P$nMn{{!5?S};Ay?YvW+Pv_ZZNsH~%r@X{nC+=^XKk674%xm@J-$~)
zYmV)L7fiMZK?b{XQqJ2(ZCkvz@Jioa4lj;93I0L0G3n>_YAv3%*UVIGZ$YB$-VB!Y
zwh8}=tP^IQwRI3~-J7)djIHLkhP@06-tP_x*0<H@{$$g!ceYKJ(3-t(F4*tQ;C*M4
zuu0wahUx4*6Z$q;m$huRm9*^MYY@7|mSJbRP0ZbewldCqwo4D5-@Dy6ZSQCHjeA4p
z_St^mwcV@w?AYF^UJiSU-W=PjDxA2N;Wh8Be<gN%3$7irnU#3nHln-B=FsK+wksa~
zvw6YZwCBhof7=TzeKsQ3Vr<{B%-t(;%-;4y8=sAWc8YDsJqDX)DIK;ZhlFfz9G+u)
zn|I0H|1&1+{jr+UhHdA<z1-&$_8wWc)K=!PkL>}?342Z$huMBG60lA9skyg7tk`-_
z|Ajq9x~ulyu&c1We(Lz%f<`BsmVhi<2CopC;B6Uu*BBV>wZD2`ugJXJwl7vr-uq}h
z`(A-{ZF>*s^X}C+G{Ls%%SPLVLlgIGV12S@iDvm;1&!Hz70e23ei(4tKG3+g``MG(
zwh6rzduJ3M*)wHFx6P61>uld7ncBLEmG9+$AFx-0Z_nOKdJ}Cwu*=vy)7x*$CVS5I
z-&@JO+ke;X4G59m6Y_Pp?V{VqZI7&2vv)yIt#wEC6x&Ukn{8(pyx1ena&YgHtL=Mv
zX1&}qwfn$cgWv@=XNnf@{Sq*9?-uTldv;{0>@mr&vo(-z+H-?zyUi-m6Sh<L9Nf#S
zxoq#2+OWOvx^?z4Ok8f2vTFC<WnYfj@+^<pYvSLtw<7=0UWN&3mZB|IdrfBZ>`n99
zZd<lAWN+&8gL}_7T-~!{(l*-_KR9eJxcl0cC>^(T>RY&Xj)>e|32OlxhoDMZhhQsP
z4QAQBGDo-DEMGKnuY%<2z45z^_pa&QX7gf`l`X@MeYOEU-PYIGllL+#d9cT3&LP{1
zh>5l_XFP4EU8~>QqI_ac0aK=}&aGK{1(av)ePds1%W~|3?Uk+^+nkDXdnTkj-W_H>
zWiP||Q@a_i>)Xg&J#M=t>f|2oCz^W?_{_G6S$@WL(!$lY0bGl%1Ab4imO8!3_CvJE
z-fNFI_8!@`$2RAou<fhGhxf|qOy7I*k^SChPuA_d@{M<|!mf?B6-h0&0v`qTOyknq
zTk+-S9+TW{d;Z8}+J5BE-}_?LQJX4`)q8Ekr`v3q{B}=`MXc?DBkX%cer?;^v2CC2
zR{iCB*Sw!*J4b(|Z9wj9+Y3LIS--KU*c;M%e9t-0j=eV;pY3r_i?P;zU%fYNhQ;13
zyIc1Dh?`{dBe%)cxBt-IjviB+1m@t~##$D8)tYD6PWpMs_DfBTt=ZI7wll8j+B)nF
z+WVoH$>vc1E?d5+R@)6HJ8d(jN9`%F=(hc_D}4`x*zP@6AI$e&I=#)7eOCNlhV5%@
zZ^s?po7gyO@0@Efd$U~T*zS;Sux*(2aQ8m1)wXrd7TB(xQLs1Sm9g!L`{8@!<QLoK
zEZuKgQeA61VfvE2Kh&1l&XI|*Et|03w&7)g)pbj~y&}(xY!3)USzptiu$QZ)-&XF}
zSzBx8n7z02toAZ&@3r+yiP}3u_1NAMfiL%nq#v^Nv0Sz1_odLiT1$7^RwORjyFhii
zjl<)9Td92uZLe@x+7?K5?M-o1+dJQG?%oXyPi(Fnbhl+#`_p>Otw>w7PjhWc_7&T%
zyd1ID;^nnH)3@#4yW!SR+m=$%y#W(G+e|yvZaaaG&qhL|)%K3;`8^x@=k5($zI*S2
zCeA&Ve(tc9I=ITV?q1AZ4x#CLB)SgU<V6|p&8g|MefN3oUb(Z=_s)-!w^=sn_+EjD
zihEz`J=|+{Lv-)W|GW08I8U;T*b%a~JSB7Q10|{55!&8+{#Ku|ZBX4{n=*0Y-aETH
zZ0DUhY%9C!oXrL^Kb!a2JFJ<iKkv1j|IVgB=!zA;_?x{qHuKx=5IwzDc>99A9wDW+
zO3s{nlUB&vE#D@yFZ9@vy(b-a?p>I?d@qabj=d|)fA4uEy1>?PkJDaJiORhUW{3Ag
ztXyXk{iJ<w#1a9!1#6b?mf62)FKfN5oo4qG+qw+5y;(~=_Fh^!Z?BH#O552FbM_h}
zUEIBak<VtyenZ<ik^Q#YrQhv*ad^hw8%j&|?&{mQ*KF5<y*ya~wkHz5@2Q^KwmW$K
z`n?8OSNG)eZrkhC_ugisS%mEdHb>hXW!!sb*a+@%css*ZC1#oJ+o=clXuL48`Lm(g
zcE?wNeK~((Y$sb!+PmV!PFn-sLwk1B&#`^-ZLQ6m-j#b5o{R2TvVhTc{Y|aC>DL+d
z-HM&JSMJ4Rn?$cody~3Z?It*6+uod@X?vr%eeZ_mC3_c0RM<u&2<;UK*0(iS>uhu2
zqV*ml=a#*DW-YPV@HNC{Y0-MyKhg7TH;7c(+)UcN_gDIYy&P)`Y!i;Ywz+ppWzPr3
zragLlkMI7#c+UEbjlo`{2l9KjT|Kn-LA=nu=$`aF9HLRS8sdRApL>+|hNL{O6<B{~
zw|AhVZTGECTaTODY&BHQ+9>~3vQx-bw)?=j&_-(N{Jm_!&-TPRM(q|+VA~hY?Y;MQ
z<%Yc?O)7h>n{;dwRAu&tn6cR|h?~CWVef*yFII1`ot3@JmPH~F+{QPUlxri=k+bKO
zw#VMMS-iHZ@*DOl^$YI#^J~T4GY<~!brD{$x8wi8J?mcb?a`S3*5*pRq}^|;x3&jj
z+ik35=G$EBi{6{l8o2kS0N-A%zkGIA7A@KP;^<r3_wIFj+ZNxjUGSLA+Suxq^^1Tu
z+u4iD_wEQ-V|#<2$0p~P=w5CImA#986z>hYsIu4KV?AhZ{LZZbEB3nT^X!Y6a&qsc
zMXh^%oZs4d=>M>(mioNc^ZwnvBClob9&ifUDclOOcFA2}``b8s?~ylxw)Z9G?k(N_
zYtIIGKAVajPTLKwEw%zvC)<AU7T+tgPSaLiXvy9V6&srw;(dFTU16|0^QUsJL-)45
zKk^Irv^{9qd!J+0-sHl)d;eXE+IytBZ_hs24SPR+IkVS?U(8l#>Sk*pwcfoN%gp!s
z9@ny+%s$2X55sgD0rf|F78M8YjobLvb{F&J-3kZS?42-c&fYIi*Vvi_uHEaiYVlsF
z*37*HeLHN!<6rHWdz#J8;!4XNwHwQ9=bQ+#E&032Mxuag@0&GUdn(HNZ95n2+?)06
z++H@1Lwm2>KV{>x?#!Murs8&oSIx0Hpe|?|@$#z80?D5?^Q4#BdhbuQ*%!3JX33`8
zd!|$_vON(u+2;N1oV^E5&9qI3OW%8vU19GM*>0N|?o750Kf`SW?l9PXvwLYXP4T#O
zLhGTuwJMivyIK#~Rwp^{UC+5{?~FzLwmY+Q_hgvP+QXEyaqolwdA3>7?Y0aa%WN*h
zFR@l=J7VkSd&Ks{{9AiAY~tF}#jIq<ll{)NdBuC16t}B;F5LTPE4kZuPs3hzo0y7q
zHvbxa+aA4kdatG;ukDKTBep9d&e--H-oLjwV$a?m^B&v$_-<x%AlKd2ahKfQM*rh`
z-b{0}Ej#<xhAS`KR(+Y1jn}0Odt3B<Z5<Bx?ryQ<v6C>Xu-zbOyC;Qx`QG*&y4ELx
zX6zLgS+)10Dd#@p*XQ;+Yp>jUV&;DvDIx2<3ao2vDjMGJIk0)|-ixB|ZACoaS#!LR
z+}Ev;x%c)z?Y%B$Gi{lZP51uU617(?E!#Hv%hx?uYDIU2+?uoJgbn9j&ly{6)nk12
z-jlDiy)uJ&@2rNSd-)tJ_VP^Fyyr;lWSbSd3vCj_Dt8N9dSb(Tm(|XDo6X)20-QGL
ztuA{79NzE#S5mt7TI2q`EQif)D}qaHeO7SqHM<>Z+jPu#Z|9dSd!-&9v0YR$)%JwR
zoIN~~1#M+cZ`td!AaSoq@PfTQb9nb^eQ33jn^R=_bY8_?1Ct$lmwnx|E2MSdUXiT+
zJ?E5?_qJt-?|q<`VXdIjzIO(jsEwBR`MtmY_1gY3blGcCwsTLw-n+YdCvLEfc%Et-
z*6VL;puOI@N_Xzweabw08CLAs;T|8jm#6T?9tW>Ewh}juZ7=wLuwjff+aqJ4Xxs8y
z-1hV=8M`=!ti4y3_1n&w!e{&8j**Q+@uS^Sv}f;WD6q51I3aGkL@m^I)hXw_SK4d#
z#$7pRyCD6Fb#hGFUa5S^y^AjT+X^%%+VTW`-E(^Vti66QoqI(Jh4*fhnrQnvp>gkL
z9Ra(bw?5W=qC0J$FEHENCa}Qv((*&LF^W32^*Unvj+894onsYbTj6fLXTh4~)?6!?
z_g1jQ@6B7TV5jzdmhHjfiF<WsY_jc{kZ0@aqhNQ3|HGaH#$z@gKAhTX;;+1yA+de$
z)rU%U32Bwq4bzpa&P@=vQ{2dJ_vT5Rt(|x4UY3&Wdmo(V*lQ9JVLOLid+(&M*u5c#
zByD+LzumhnzJBkt$J6$HITvheaA1*508@f(*Ph6|Z<}xKG2z~B^P~LFt{08jwh~=W
z_w-b$?p@R&WLI~4ifz<{9eWq9V&4;Rn#pF`uUR&m?-%V|tCF=hqASN{MxU<DMb^o?
zYhtF@N`LOOJ)wEj_Cal}_0ieW_rBcew|B*ghCQlJj_z$zT4~E9dtmPkyAs>kmdp1_
zXujX$k&v{<G<b#W4c0CjpGH4hE$QpF^HYEBX34$3XU^q~dutw;@11ktjBVJpCfkVE
z{Wcrswb|Ir-C|?5Y`$%fGs9jE{wI4>a#!wE2$0^hf#;)b)Wkb`cjT+@$vgJPW>t>D
zz7wZ*@5;F4x%cQJ5!)kC=l9Og*R?&sxWv|IlhfXnUH*GFSt;7tJYw9}a4~C_#SahL
zHT?Vc-0%vwjVPS3S1V-k-sY@5d*gmh*=srHxoybD?|a^yUu~<Fp=4ttv~-U~5c58s
zB;LJe-X`z9dDwZc_`K<RR`MO*>+t=FwN^LtKHiO~dm9v<?m5ZTzPBc*z;*)1kG)gw
zEw`1Lq_X#pHm7aF0Xf^Aq6k|FhIU&<`9FIvcAeOJhpW)`g($Oa(sD`LXQvO_DlBc-
zn_^pGdw=boz19ov?+JL+y~k$N_dQy_T5UZhNbdQmr)1laH(}2;7cZMLj#u_r{Hd^!
z`5nHu;pE}H9uDVi|1H?I_rcn(y$=i<tQAb`Y%I9u?>(|e%0@1Dm+h{YB{o-LrERy~
zR@lRE?9Cnvu^ToQZZ6%MtHx&=!<(^JX6e$se?FYv6TrG=uR-{;Jr0L{@A}HIY|n|O
znzl+$C)gh04YjQ(pRni7f>v8ko!5KiuB^1xaDKPvL*Gr?Fp0Ug6=&sa*^c+_?F&3@
zdv<&1UU{7xdlw`)>|Sg$d#}mcb+&snf7+V$?%n&&I&|*|-vzci&-&~wTQ9m-|B|ci
zGk#_}2acWA59B&+4g4kcihS>~<v&omr^8HO-;5cZwl3<cZ8wH5ux*-pXs?&xn!Puo
z!}cC<xn<o~G1+!PAJ?9}^S15rQJKE?&f{f!!}7cKR&nmIT{!*O-qZc!wqo<!_6jKO
z+&guX>h3=KzP(1xQg#z#9`0p0VX$NSjRxCGU-sIZDcHT|PyDYvCw8^&nbDALbK6AT
zPNQ7Z_RG>;w)y)R_g?rfviFW(;?4v5YJ0ZbeZS}IoM*NH_UyLHo*V29xx%}5%I)8K
z_UQEQ(fK59yDVnfUh{=Ddmnhq?yX_>wf*hEyKjEH^IoC)leTlSFW5NFG}*g$pZ{J3
z>o0p3WX`u;xV*`xnLB&0{?WU(JKd-6jrlgwriwjiuiWHSdl<M|cJV%*y?4!$Mq7)6
z=l3ivwX(jE*l4qZyJOFf@=hBGnYKL<b_{kW@>BMd32n2TaJJM|G573VgVlDnR<lm-
zT`0DGFT=Al)+w1xwhT<)EaiDi_L|KSv6IM|xrbwp`kq4p8*D^h&D?vSIA)Il_nzJB
znymJ^JGR<Jth3o$w0_^-{WhU{6r!4J4?jP?_tIK6o1|mA_wISwV^gtxzKy)zb6bxd
zVcRx_jJ-$wPV9ZV^sr4Qo14wn!mhm!$|d)1XgXn?VYAHk$kC~L9$By2Tfy#a+oHp3
zm!)K6bwEIH_mh?3wld-~ZFA0Y*uLx%-e;@vX72%4-`zL37ujk$ckIn)5wtCHUb*+g
zku7^>94xYJ(5SX8ID2RJ8>hxSrOG?^+$cW0+tcvj-cu{L+xAAu*d2ZQ%~r?cg6-OG
zANF2&{@liBk*JN2mE~R;QNF!$BE5TCXZqP%8CULAxRh(nVE1zmQ*xE9glUA$4?8!T
zsG|>U_i=03K7KH1Z|JUjdu_Hz@4XZC&errqlWj?7myHeILR$%jp1r5L*!FSMGw&79
z@3Hx5oVwS5Q`pw0eYZ`_j&rt~ZZF$=Hcw{X8s@{cFZ34evFhGqn^rK#R=RYa?VokI
zwikNM_cGW>TdwMqu;ad9VLL<VkIjYlVA~m|ckD42owGMUq06>)LC@X=8q@bMd<n1$
zOjxyd$B}n?Uc6gu`<^RdFVD+!w*Ni(>|~w!?LJIcX`_+-U{A}e^?Mt-HrdV#pS1U2
zRo>n&hj{nJpVqXQv*`8S3GZ@jJXT${F?^A;TjQYg-v6x=Z8Jnu_8hek+b7PmXs-a{
z1?w4&VS5tHX4qI1&e*e1bGMCwL#Az5={8%3MlIWG&!^co{8P5Nsam_2L%G@}?cHkI
z7mmt%)FQsy_?b=F{rivPzM?Codk^rQxAwTfVynC5!QLXiGq#hu_SxQPonUh*{=i;`
zZ;`e)r`@u>wJ+HA?N0}ruHx*yvF4v_-+Aud`)6mGt<}Y)wwtycvVF8Cbng<~S+*?k
z$+mCOW%e?ByJ6#gfO}sUchcTn{jz(X?U%RPF#XUThhKSnIOe$TnXyU0_RA4R+r}2%
zy$UO??0#V+VH;e>W_RNGi9N@y*6y{_USp&Clf$+~b@QIx?yqb&=o#DGXjb0CcuaY3
z!jT!)E|aSFioI&uE236w`(TIS9-WgScEzi>_i9`_WAjj{&Gv#<jE%egfxQ=Q+uORF
z{AT5Q*nRI$y*+yyri<IMSRUE?Na6cl;qQO<a2#T_dG)B<+M9o(ZNl9h*1eC<><waR
z+Pk1Qa?gv3Nw&u&D))Yo*}C@y<K#Ux`}gl%G0)w0`}BsrGxA#Ze9Sc4<8<l3UXy!b
zdz)JNZU0~1y*J^$ke!#%^1TzX?%Mp$y0BNN_w?R3@vCi}A05~u);h_S&s%P9?S?p;
zY0o+Bq=Yi}ZhUuauf#<Mn+;PHY*>DD?kT)mu-CXCYwwHo-}jicq*^c7K5_4j2rk<X
z0;;wS!Fo0UCp2vrrJS?1ZT`6TH}j^wZ{L>e-L8CL@3w;Cy&|g)+m@I7w)K$~vwgC5
zj*aARfqlQcm)lxzSZO2T^VjB))9JlhGpE}6+}vX0-}-g0gJ_LSc5<$*5AU?SAKq`<
z%ka}>*NO8}ZB7ZY?Y(^G{a*9g^Y=|Zx7AL=M}6<H9@Tx<UOMi(Y(8_ZBWLzLR(Ym<
z9e+35Y6wK`yDQ_cZ{yshd&3`v?%maWYVRXM-hDd1&e;A_xw-eo!iT$3@+a+I5Fl(D
zzv99^aglZVLXwZ}U36>qezVV8?WWFHx>xXf-kwXjHg>FGuKWB&oAxdH9<sNx^_|^x
z^-257wTt%Mp1yML?PtDw;@A4^)%bIHkLWrb+rEo`_cEpx?EPfwZa+0*g{|l9O*Ua`
z&h6)){BYkj`_z5Wk#2T2xhrfhG49wC^-*=7c`N_^D?1wYl|K2hPv*j;eHYkI?yH+I
zaW4bE*gntA9s9yA%-L7LDZSU;vSn{gSp7cc15JC9qm%dQY%{Q%z4h^)<WA1L+xAc0
z+qixC-XxYu`<~DKv$yJX<o+GC7xyatP}{dSD0APY7Q1~jIwbZ!c%Z-U!;RLx6AhjB
z+0T~R=hN4>_tK^r`<(wK+g5bm*efI*V|%(XV((As-FweSwC~rl=Ghl^uF_`jx6plO
zzKZW#!h2xf?b*ru4&1-Fr#A56-VWbAdn!fd+3w0ev`^=q$3E7`rTZj8GVSwRw(nKf
zR@|qO_+t<6*S5V2^A7G)77Vc$OyS-q%&~XhkE%~LiDn!2x(B-2&${=(PGt8g+ln4(
z`x;$U+m1P>Z6;rl+V|&q=U!)>lzj}ocdb1<SoSlvIoqcAY3x;z-DR7ZuxIZ)$p~A)
z%eVGD*{-)QTzTT2*1hR_cQxAWb?9KU+b6YjufWWQ`;1K{+9joQ@AX`_$mX$6iOqSv
z^R|1Ae%)teG<9#tB0bxgDZP6+5~l1usM59fgZtIpBH01^co)~~6A5D9cdUTfUboA2
zZ^p0Lw$>FE_GgY8?U(vK&E`f%vCS#*ti9LU{O#K&<m~P2O5T^^nPr>J>9TJ|^1^+|
zo6Gm}8Eo3OW$LMYEPLbktz$CUTXuNi-c*Nq`+mirvgQ7{VsG{`*?sQDhxbZ6Qr>&)
z)O_37y(jjaGqvA4t+~$n-y(^9Go;q<{rt9l-yX4^eW4~gdy=1gu=Vz6-P?RwV*lai
zXZEJwp0eleVa2^?960tJdhpohqlm}8(zQbN2VC^*DwquS?qf)`mG~EGyJ3UZUV(6R
zTbafAdtZDJ*!S@E%Do~R;%!gsZ`?cgU8eoHSAX^{d-dGTc&EdD4skb|1=CvXq-V|F
zr+qeSU*MgW``pYH+e&XgzL#NZ+}`_hU+liWNX%|;LBrlPyH4!e{nTo&?;<I?bpiWr
zZQdQ<yTSQ_&6+(2?d0sv?q&MhwQo|$jJ-1Z#BDu3D(;iqU}zulwQnEa=EZhF+H-6l
zd<?Sw6gO?}j%iJMR|<0Py|S6X*5v#pn_1GgY(m`&_U_Abu{-;~V=w=d>3hX5OxpLT
z;OyRI*Gl#ozTL8~>SpBLV@_H7=4lAnzfnHB_nD5>K2O#Sb`ffdb_q{i_f-Z&?t7Nl
zvG>Bs7Mmw56Ko$W{<Y`U9Fx7eA9D8@)Yj~EyYIV~zv{x?f6Jw8PW%h7-C5ngcVlVh
zJ|>ChHcRE#@4K$QYG1`_uYD4WxNPm>o?3l6-oD>4;;8LykJ5eNhj#CsDAcfbP37Kw
zS3YaoPW4~9zw&t8KA)RB``Vd*?agPoymvu+scp-W&b_y+=kA+&+InxZR?oh~=fV39
zE#%&RP@-#}iE@$c(o6$e>H7xz7S$Zu*Ak<(tMk<5y#Z&#Y+mkty|-sx^xm~6D)vZT
ztk}0<v!reNviQBfr1kdd`tt3~5MHo1aZ}U2Ps!rD^A@}BU6NnCf8Fu0y$|i~*eqP`
zyZ^4b+Fo@V#(g@{I{Wgk`t9|LyR|ps-<-W6lP2xkzU=&79({TH^Z$+PR~>J%EjfGK
z_Q9Rm*5PkU_sNvc+<WoW&pqoa-tC>2veD-0zKq=m?;G!%Bg$o)?lIeX!~A?ZX`h8Q
zZ<H_XHI6vGw=DVSzJiQhd)Kpn-*?L9i}lCcvc1or1nymMpnd;|+K;=hobBHG;M0b^
zuhwtbXSb<+uTh=HKFQ$8`&g}P_HR=>XWRbmtWDD?mwg*51Z|W1ZTBuWZr^9_^T1|}
zo$cPHS!TAEHCF6>Yj<VO{C9h7-b%&W{V%z-_v!4l`?_*U_HSrfx%YYda$C9eo9yb`
z(`?PV*6%TYp1x1bop0X@Uh(}4cbC{qaIx5T-o<4f&(}G2E{nI>3eV2m|0{If{<MwV
zHhpoI_C^2Mx&QK%HFj?~rS~11I(uJ1mxKK|?lSu;S~snd&pqE89+kd-LCqqYCTGU|
z4T|l1r3>x%i%!<HJ+VV`e*@EYE2F#j_N30*y*J}6(>~$F$$KaA1nqX*x^&;My%~FB
zyT9yZh*h?AeZ;g+!}W!o*;2o~7X3l{w){!lx1MSGUXHw@d+)WS@4K<-oXw9LkN0@b
zs<1COxPAAw$m`Y>|6J_~m(JKXIZSe2v+?e||5c=HzDMk}x%=Sb-pm6J_sIRt*vEfu
z!@fK#E887wgl(oIWbbv2blrbZX4}563l#RTou9mShT@++O2?G##ByHk`EAj(?}6Q+
zy|<<;+;ir!@P4@$Z}#!ot=RjGFTpmacILiI)8E-#>b|sBqS0@!oAc7WxrbNmyRqAM
zUrbcv-n6w5dmEgkY~r76*~@Dsuve*!Z$E#}>Af81i}s}~-)$4|?7xjCue5z^gRfo8
znVNk}!6|!x7HrzL-v9F6%I$r7GgkHPotz}Jw}XqxcD|FGoo?=A+mhwq_imiM$!4yK
zi~TXB+I@u=-tBw1Hqrjr-<^9Kjq3La?c}suws?{4lNSQEchCOW>#E<ew=4FNjZSj7
z?KG49w%f{Q*cMDMw#j(9aBqP{jO`P-5L=ODdu+G8PTgDZcHVBexjuUtEMDwcB4=sa
z|9{8c3SQ2=Ee)&pCTQ)m7CO{t`yg%I-Us`-Y!19h-_6I{v3E<_CfjDV(|iAIWZavu
z@X($Kb?tj+#4X&jZpqfY9t<n?h8$SGhpTb#UY-|o_G;`Du-S3I%yz^2a+``JVm1x!
zzSc8zF52iYov^hyf5=9|S<rTN-^#ru(X(xyoZe*HyW{BIf@iDuMinvdJuB6<w}kod
zo)6KHdv{#mv}Mt&*(-2)x^={516%bIiF*qstlVRqc7Crzx#XT3d!qKTX1ncuwl#Qf
z!W0eb31w^cIxTM7d%)<P)dk<EJ(?<~_Xd2Jy~oNgcy9zlr0o%bskRFqT-(iH_R;3X
z+|#ztPI2zlur}ViRn2VgrrM;vPi7vootD?LH^bx5UK5rM+b2Aa_SDrG?ag@o(8g!w
zblV8)PMbeJOKcg=yV)=)&Dr~-ywG-$g6G~3b<=G#n5%8%;u-h4{Fu9^xp(8<M}b>y
zE2>0oH^j`^TT!*x_Jydd?Vr>0ZLf4}*yH!C$99+B4qHvm{d)ygsqDQWI?tBDsKYkm
z$&5WW)>hj-3u@iVA?La0iE*Co1Ll8wSVK4NJ+GLuH(+M3?T1arc1y@E*i+-uYx|<<
zj*Xsp-(Ee&Zd(DRt=3l*ckDSMGJ9{rWntSMa|_#w|1)h2YIAHZYUb}<vBlbU0?)F&
zD%I|`^H=Nay%D$8R$=Zjn;&1F?|BlUZ+mUhWLvSX2W?v#uI#b<dCWGZZ_eJDMKQKB
z*TwGLBYbXe2q&K{!~5-f*k<pw^~ztq_uR=zwqp5Adt;^?ux+@_Y;$GiY1<p>JhnD-
zHt*q|KV|Ql4K=p4=ML|+_{nDLRyld^%R;Ta7LShEp1t3^_rMH>Jr1gCck!wn+Pm|<
z{oeAEGxx@|?6hrorDCJ;#dnX)|Gj&R|4iH)z<X!U1eQ~_XWzT;-O$%zoh!3{Z^R1$
zTMonPHVn4)yQZp6-n*lH-=3_tPTP(tdbT^asO?R-)na|2(ASn}<`Ua4b`G}D=Qiz~
zanHlHM0VBQyt|FIJKi?$WyqgzrM8A^FGEwA<&J}_Hs5R8_HN;g+G`T2Vmrt3`<@p9
z`}f+_t+i#1ZrW?0cHD;hZrNV9O^kalIK|mM2vpgvaM|Cs=H!Mwj~b`ge)O8SH*}uX
zUWHID>xP|^Z2d&%>@Ar-VXyb;O?xveH{0HDIb+k)wqb8a^>Ley-j91kuCBAa#yihe
zV)a{_>%TqrhS?mq-E@5E-aBDy_w4hw-a9ilVsA>+bla!zw(hl9v&ObT_uuXrw-aoO
zvM$)htn0Mx_;b$Yiaww1iZ>T*7y`m~@hqOdx8z&2t<1G$dp-CM+0<Uo+FP)wWUoWr
zEUN_^&uu<@Y}j+WXxZMdH153&Q7twqv;XZ`^lPE*nRybn212p63L<tk990)=Do!Wb
zysBQXmq*OswzcBa-WRV8_fF6dwN(jgw%rlzY5R#|qV0Y!t-VV!yKHSvu-FP+TCrE-
z`J}zeOgGx<9y_>K%x=rxC1M-*l>Fkd4YZxMcT0)4ZPbFyy$qc*_B0$^Y!%_CxTj%N
z;+_zvvwIKt&)viIYR29z=X>@t+-2FTaxP_WY1pZ~D?UxP@oZhRm!Uy?`<mM=Hk;YI
z_ZEmG*|sj6viF_KQCp!S=k``z?$|rwmEE2zIjd~?>{r@WY&WvKu*ln1<7ScV1Foso
z5_Uzlk-tOs{;6Qv`!1w#uYiHR?Ex#5JraLU@2&6`vt6Rux#vL8;yrKQcJFn_Xy5B(
zzs&ZS<-WaovsUa~aV6E}iHzi4$+h$MdL&M@UD3F44_Dzt+c|9RwmZ%j?h%_^x|b*Q
z*q%kD>ur4=N7?@KX5Sm7w8i$ydp6q&(FZn8^NRL{m?Yb-xO{z&$7SEWJ7zQQ;XG5l
z*Q9Tr?UBSgd)6G<XWQ_j*7i^R)jd-RZtZzxJ<Ybpt7$L8!)w+rdyedt>gwH_n|W$)
zPxSV^8aD)OmR#RxW6&&X!;v&;Ps=Tty>c}Ydrx?+vNg*W-5a1e(Y7hC!Pb6%;$DW?
z$yUL9iF*$`y|GK+n~&{s^EG<~@(gSfth)Cu3%1>xwS0rEm|)M|d)^6qU*C<`+wiS@
z@0At<TZN=syMG+3v+;>x-@8L+wynm!xqEwrrrM<E9o%cKxqk13-wd`X3nlm7I1y)i
zPj!-QNL{wAjL~-68BWjlM7M0%YxAacufpFv8;(sUZQtxWW4l6`)wX4h(B4Z6CfQCo
zuw?JFuWM~}m|6Emtc|hN__K1ag~x)uEJxPcUOCijd!Xcn)fCBjHs6w&_vSrbZ@VC_
zZMUWE(!Edp9QGcVv~iDdZOGmW$**k=pO4=wG5L>8XGGWDT_-*FZd10}`(g93Jr7zg
zTKfp?vi-5+q)mwPDcfaz9eaQMZ?aXH&u#m}*2lIavu&@JZ{6Onx908*5h$~*SeIbC
zgu!(0k=GaZc;uGvJ#ePgrb=z4Z495LZ9&!}o93>SdmUt)ZJKPl_7)VD?9rbyfA5Ed
zOLj*YOtclalCtMR(6+q`c9>gFyOX?EL3-jIyQ1B;?8<ZZp73<G?eUJW{cXQ(uYr2A
zb<4#yHU~1->=ubvv^6Pew2jD$*b^Y8x%Ugt#yz|Ea`vtenYeewjwO2?pRU?#^P%77
z&;p6QJvuSAc9%r=-qCrt$Lev*UcCpq_gZl;*t>ymk4;9DzwMk;^0o`C8g@@8X0y!}
zvE6%M`wH7#`CWVSP9*Ksv+T81RGhPSRzTif2L{o-&dYT6Zo3$7vp~YcwnlM{&0X`>
zy-S{V*aRP`-D|LZfo-f9zulAS7Hj40hxeY%xV&eI*vh?g-URK<(6`-FA9`tT=NpcF
zZT~jdb_Y7|owtT#ufde{dk$Pu+GCm0Y1{F6h3ye3-o0uvTz1Y05;lKVi`%8lTfDa-
zRABGvAU@l&_Kv+-!pwX5<&$iVI)v|4m)5jZiaKiR*tBDBT=J^D_x5$}wNNy(ZR-o(
zZTa|wO~NnB-5-u<?&b5}x%Y*);6BeC)_e6b`S(6uTwp7D!Nzt*_C?z!(A^6CoO{$g
z_gEENWwtvox5+9^IAX767Rz2;$uHIskF9N&Np9P7<Ia}7Crqc?9uWI!<?r#(#$bEG
zp83=Jt(`c}?(UvlX`}Jv#GWl%5A7{Ec)?nXnQ`x*|3<b}ZPRuu7^d0OeNnPopipDG
z_``1-?^teIP8WfF584>_F0Wy=X}mSx#_YPwUIPt<eZ6V(_A<OXW#c;etIgXL6ZaO>
z@Y^j*R=3^%M$ES1sKj2ri9&mwS+?zM@LRp7<%-bWn{z5{i)KdcofCF&@80fg8^5=H
zw$J%$_e2=;?30P`+<V1m)84uTw!1wfWB2;Zj<Q|-OJtv&Ub(H-_AFaB*&BO~vG?w6
zdDUyX)LmlV{P#QeYFmZwl_<Tw*CO|)t=4&_ecKq?Y^*dCZN;7}vb8Cky@&1L3EN3K
z58JTHh}!Phns3XI?!9+yOq;F3g>c&_hGl!DF9z&g7w~%TjU~Q&7k!r7_eRylCP4Jy
z-Z?H^dy>!c?YnSV*LI0s%-;X|p4mK)5!@^6ePM6)<_mlO9ecXx)8|=xXFQSLcWNKI
z^>mT8-6_eN_pT_)*)wVO8r!7_`PTPje%YKUS+qw)U(W8Z538+V#OJ*euet5L5n5>L
zFgwrIvPOO1g;ia9BkCJ%E6S(X?z+IfkDuqpUXGAQHV=*(S<m6-vDH{FZfkxf-{zk9
zoV}}~JNNhti0u1Y+O@ZyZPDJ6JLP*rXMeG2Jo0EyA<vq<Q>4@OZZb@?*5LfHC;#cn
zy+`itv;Fq%n5|9VDccD*{_gIdSG-5bL)v!Pp%b<Ox5902<Z{|g*xI(YbSc}uyfUqQ
zEN_f#?|<UhyP1>I?rP$Wy;Gm0+kVL_u)SV5b8o^L%e{dN340IJ%I)^3Xx*E|p<%~3
zb<19fs{b}C{BGF<giNsENDkin>LrUU1H-ZH+?QEx#3uXiO*Wdk*MZ%B&xidDwiV3G
zds{xZ+1@>K(^lZp{N2w&-`d{Fon{lx_iXR8MG||T9O>A5Ks;#AFQMIgqYE$W`B%7h
z?~IVmdrZF6+dAl#+FTKGvK6>-anBpOJA38_<?nTkD%pF4qh*f*)2hA7Z613Mh1}n>
z@Z3!sp>53jBEDa>`H-k?%Q2hD)_wjV>y=AH?asNbv8~xCWIKWXgiYz=$<`r#%C-%a
zY}Wa~Yi$J#UH6`wJk{pT1JS(<*&UYObzE$CxhL*DFn@<NlhHM6pX&`aHHrSV2@&^q
ztJL=FVc<Ps`)$g`z1%fbJ5AS^?e$?eWE&xrv1g6JracMoukG2UJag||&;GqiqD?l2
z`y{Lrt}oqtpw?!0#r(OpZBMt@F1aMWk8{^y+Z*DsHf7>vwgtxz*iMpJy?6JWd3#Gj
z)oeGE{@<<j_n_^$U|HLSO>9<2`|NF(eYCI*+jVNMtkLSd<}!P1@}_HBuYT!l<9$nf
zANP)Td&}eZ?DbaOx!2|P?>(>Fe(wDfHgV4mc~P4sYm)b_XFX`UsbuHg+2<4Xu8k7d
zm&m(nw?%5*p7(!`TFnYLXnVTlt*vBZ+ukWN9QLq!v)hULNZM`{y|q`R|LdM_{q5F2
zW;EJzOc%0su`9HFBQ0e+FN<++Jh!kdgG{!y1K&xjLgfQ{^XjMU`NSk-mvFvw_qw`C
zw!Q)Cw#zIQ@BKGt^<I}O_T6Q%KlU=Jp0SPMZLnFNU%2<fOi|mVrQWt@><;gpD*0@0
z>oUE)6DH2vv+gP5zE!Ut?M;bo+H0{h*mlcb8{6Dj^K8Gw+}*P=_@S-DB-y=^D>vH8
zb$_=x9k$z6KS^kB;krqC9cHuc>(zU+M`Y3*o6wKfYz3y3*)s0_V5@AVZtG{fdQaw*
z)wTufKX-S%k=?gkdHUW53uoFW%~)d7ntI))Px+i}^dTPGT@RaWRW$$X=@T@!otAue
zkCe|W+xyi9wi!BaZ2hhnTD45;v)x-QW9OBy(I#un6kECK4Yn^FJ@+Pc)Y<B+Kf7D-
z)CcP(ci+7qmaf?2U9`{o_s=hT51T3Ni_<!_cmCxEHn(dsY~)TT?_06%ht1W?GxsXY
z{9|2mY>LgIy<2VX%!sp{a8tro<?pjSMui48o0xd_-rFN)E8VYTyZ70ty+`Y=?&Y|>
zWY2+D-g`9~rr9#~Y}q?U<lLUi9Ws00-T7-9{`|M?aYc*03#;z#*-^5{YOROQ-gohw
zdp}$<vDMQOw8`gsZLP61&i3!GMSBIfE?8fCRkv5-y4PM8<4D^-wfk+4@l4$tJ#qS;
zw6u3N;&!rj+dnSa%W&=LZUz&5D;qImyAz8XZKprbwaqro+B>Ocg6%%JNw&Nk8~1jI
z7wz?C$gnNl^=dD3wXRKHE%)9zJC5(Ya{B9D{i*BszWuAT_mf$*?U(r8y%Gx~_eyLJ
zvMsuDV(*3X%ywVq$6D&_+G+D(XS!{cNax;}j_G?JJ#*U2aA(Sn$M-MVUY2m&%eL*E
z&9o!`Y}F30wq0>0)@IK0FSfJxtgwB*M0f8^m)SN2>lEyIzwELp*?f5K4W205gqyQ%
z(p(?eByBxxyVG#>UIy9ud)F|g?sZ+U#pZ;y^<M7Y%X{A|x7y7q+rHOU(9)K}eb?TD
zk+${+?`iMr`eU|lrOdazEBs#UQ7D+PcdqQIy;4WE>^&r5y{~z#wjH~|Bb#j^6Zc+t
zA!hSya=QJSHGKQ_=6$r8(KLU*R_y$}xnHL4J1MDU7i1-|uliNt-rpOI_Ns>X@B5))
zx__x-<=#_<5A0(nckka}t-o*Q^Ame5MJMcS;TPV=k#D=t%=XgWfQdqTBp0*o-J6(h
ze}3_%{SPj@-TN+Qz5V0-6?<6^Fz$;M?zW%%H`Ipv47;^9W1sD{xmEV&?V9%8S%>%Q
zZTh`;`;AQ7b1_x>S}W)84wsy}Z;90Oy)RqY_R0P^x<}Wxdw=OauYGTl-t6&P&1{!a
zd2FwYoaR3M^@4j>vr6r~Be#3shsbAppZqei3jm!7mRq&w8B^1~=dDxrsy&>x_n~y-
zzN^dn_VRFb?)!AEanI90DLa0d3;X_aT;6;9(9^waoY(C{uWIaj()Q1edAqj#at_^n
zP80j~D6XHh??CF@eOH2K?K4|5Z;!-`<$E`s^WMATZTsGBDU0pcmHqcW5!<xSP+NE(
z+dY2!-Jf{&EoyVww`N_Dt!Dfuo1mbR_NjKS?X?AV?R95exKGNib+1W;;(?XBg!bo`
zGT6UQE8ZLa=7s%-)(2LcpgR1w`2NY#kM|1fKWn#qy4XISi!b+wxm)ksd|7L+c&yc4
z|HNkd2VRry<)2R6+cazWz5^<Z`=&%O?z`}Td*9^g-|e-F9QJ+Cdt-lbzn&fU*R%U_
zmTt7&m~nOQqe(CKR&M>hUt-xyn@T3>ee=EV@3m6S+dFmtCL2AQ)q8uRw(b9KYPv6D
z!PmVT9=O;(+I7lSknP*<%Gq^xYU?=c51MS*yS$svcD^9<zF#ZY?F+Z$+9wBj?UOtH
zd2h;^nthv2tlXPsv1va`&zybJo-!Z!z5Cd{z&E1%l)P2<9kE=sFDB^7o=ofZz1!+E
z4qSb&wKqVQe{bw$y}h<hdbSU;ChXDJGI8&kms|E;SoqFH%-eon=vJ0}A=g6o#-E;L
zw{gj58}k#r_Sv&T_7<;Mx%XpT>fYx9y7oC;zxM8!_R1zr<?a3pj4B5*m+I_g5O}aR
zr-Els2D|m1^Jg~iz1FyJpHbM)y(|{ydv)F(-0RIIwXf#LeEX9fhPD^OpYFGw$atVU
zwtKH0m+66ZOfPL5rv&Z$E&64jI;;GC!`N+mLzi;yGkht%Z-Hjb-cz$P_c_L<?oVOv
z*uz$(VRs`dZm;*{nfn%e{khv_=eGS~(F*&Xh&JrIq!e+$vFWyrZSaY`Oz%N^^``Cn
z$|z_%W3krWp8v(RT@kW-d9Fz9n`LXdulmNKy(<0zd%N!_>^u18hmFzt>-!IXTd;S@
z<LR~@d*<&wcY<Tz=QOK5x4m*~e;i%D$L5j3zMH#G?!T8Qw@=e5dEfp8zieAp+U;kT
z+q#$YqWHcTZ|%Lhex7>*rrPc7kaXMgzEIqD?bXD+=e28Xlw#NIZNJB{?`atS-kc1U
zeRXpd?$v5e*e50UZm;H~+53L#yx%)X!e`%Pt?0ckSgiNnjXAL2?lSlO=4;veE`?m!
z6P){PZ-SiK{zI!7_UUQ9vE$+D-)CmNc(2uEMY{`MjP_KyEwI^eN_nrmW&8eut%>`7
zUfj7)wBp$AR|gpOX9cJ1?S0d?FV@^<pX$jIdmV}<?7P{lysyA=*WR+d5qq>+(`>CD
z>Fnd&_Hu8deE1&2wr@5KMYVh0OIGdm6Q8|T`=!o4H~l^P5_Z__)%|y6@9C9a_uE}c
zvYKu#yw9?wb^jOUihVxwX76RI^tBIJX>aS`*u5`k@wC17uB7jK{IY6KC;!iVr}jUv
z?{tvZck*k~zKV=&>-W3u_T71Z(Pom0lHGwz&-X_i?%KP${^))U(Xaa+I33=5DyC!a
z$MP+EH{AKY_tq`By`M6)_V`Fm-52wu+=e;flU?)@aogz0GxsX%9@y)XX=*1g(`x^8
z6U*N3#R+?xSI*maLO*Y>{H`hcBsoNFXERkCII>J@-@8lS_grY2WIH8=b>AGB)q9+d
zhuO3HgzoKM_i?XolKj5)45#<b6IZoWJMq*`e<kz2+1gFJBmbY<+qdSF{pTI$_H1nr
z*=wrs#xCNbtew;UIeULi-)I-;YPWC4AK(3YLPz%;Icv6ev;F10vR;1nxBZm%9Z=*u
z@GzrqUy^#4T~*5x`{vG!y(tWi_HH_T`|j^3+G`o2V*jZ%YM-29{$Bs|RePVU^FGja
za_3&|gkF1Q)v&!2bqn`v1a7r^(k)^C@x9Dm2_><8C$#3+EIZM-m+#}XU2H#k_PXXb
z?K9k^z3*;+$o>x;EB0=GnPS~>NZ0<vqceN^rY+s$SRS)CIAPh|%v-$sbU!xRobsN&
zH|$>be!I<``yX%S+ou)ee4t^`3A>eFZ|!^iS$Utu3r72C5i9rl*WcS)H);0Xn7^O*
z`r7sEeRAs6UJsvtdwy&bv<ZFiac^86uYE~#+dj5ODf?C{8SM3S&fKe77h%8jy}zC2
z9?gAA&DZQ*`r)6o!1`zVjN>NT=Rd998)v<7U&m^vy<T2ht!4ME->d&*lWlKZ-X0;(
z=lg{9WcJOI-(n+YD6((j3XXkKwKZ)oK2_WMW_zT4hW4Vpr4L%{7rt6(J2lF8uWrg!
zyU8lg_bJwX*mL$1-#+DuyZ1>>XxPiXVfx+~WvVvkRl4?0F}Y@QT(HspWRJU@b^83h
zf%m5FP4QJda3to`zUy42_Fs1?IB;LswP)4@PWxxqf9>6@+Pb%=@}<p(m6!M2PPN`w
zEUL86=lKp>v87gf4W1pfIb9sJH}k@}y~UY3_wJa#c<)``6MMUtoZc(Sy~(C?@<!VS
z&m?z8+-2S~a~6kP&qYp~gpe0Dq4Jyebi^mycsx39+u&TYw<7QOo(IwGw*Ovj*t<cg
ze^1PFb~}&cGTRAkRyG>5E%vT747cUq{b3IWpQY_w^KUjw7kTe~=(^4}=yLsDGmdbZ
zsBMRBKa@G!PLc1lU2%SoZ4$>(Td8Bhb|sE{`w}jy?0J&WWy3dViY<%Z65EFQ6x##g
zMK+FgOnX>eIQC6en`)c0GtIWdgT;2k0oA>GR$SU+*12@A?6vMaV%s?FK1|uR_s`ai
zdwmNc_SWoc+`H#ryN$vmM%$Wc3vEB2TeNpcB%AGJ8KHd&y#~7je&z2u#6HP3B=7s4
z&->E%E(<wl>!g&q*Wd25%@L8-y_Z)_-YaH*U~g<@rmcX$+&yyy`fab6GTJhfi`%_;
zmTudjtg`RhkJEdfPm9`nYO2uQCCzJW8`78WU7eM%_rZpZ*2;fR?X{R{XUlNplchqM
z_nv7b$M#N<G_h?GSh?3mP-3sh94VVkYYy3Vb1B++H&yQ~I6ldC-ID3HH#ySwwrpjx
zz3nGys}UAsD<surE2q@H_l<?Ut=@}1+mLC!wr^7S>@H7Rymt?`nymx#MeC`XPVQae
zE^IrWS<}|Tb+xU5$iBUeG95N2BIeups4v<haz4iP!H#t6`zPi0*-P%;`=ou9txQyr
zZJl`c-oJ7O_AoIYwJk4~wmr7C)wb$$t8H8DDqGoN1>2Xtd^QZBr!56G%-s9<i;E5O
z@9sSfnkxGo(z^CendrN>;SS?&k>_GIMYr$meeuTFM#OC2o)tVPwjUTK+NN>W*~*1%
zvB_GtcJJRm6Zguj*4caM>x#W!8zlC<^l09D@9mDgOSZ1tGpS|UUXzKsw$u7MY*m|f
z+o)6@vb|;{ZTCX-<F0ioynEGITK7mDkKFrY%Vyi9CpGuPbS>MP(PCgTciCfG=@nD0
z%`z_RG1%j3dwBiDy%!Bt>|~O%Z0}9mus1;U+8%p{&3l);U9-2MKw^(tK=YnGI%T#l
zQC54qXWQ>pQjppku*2PE!~Ehs^-~|(<_8w-tvtoNuV`BL-jypD?e1jR+cGfDw0eGj
zne7cZ)je~xKJ4+meQ2-XpZdKgKCiZ6u)n?AVftg68+JGMxFnnH`PMnf*32c<Cc>{_
zug9Owdl&;|+OC|Gu~+84xvc?fg6*8WO16%Cm3#lr%HDhD<RaS!zU$V@)||B!;8C#c
z_tM#W-u9KP!#dSH=TjW_*4Vz-&0D~=uRbh#?}Vm>yRR%}+MA*I&UV2Y7n`#Mr)&(@
zeBL|r{>;4$n*?_XU0|_MaPP2f$THad?d`NZ>$Zs6ZFq9Q=7s-Fo8&uMdxid=+<jze
z?cRzOcbf?>_-s_pPT4zmUc}xAg`T}PIQQ7Z@(1qy*>AQ-Ldk4z+%)#RQma_@7ChN*
zbBRHD@5EU%_Xg`Q?-OyKw|58IU7O0)bN9adSiHOK<zCx)FZSEMd~|$Ip5n~CTpp8c
zO~mKgZr>=l@28xpt;(DAHZ0i_ZA+Xs+PpLtwmbQB-`*pux9!m>s<su_!fy9TL1OR3
zi7k7Ve4k{y>)#pMmXZkDJ@5Ez6KY@X;S^=GJz$(|TjJYmyT&fw_VBEUw#nyq@114F
zvhTp>^|l{98SJ{V`0NB8JhG8G)o;7OZrR?)O^^0w`xWn1TYhY>(?X8Df6C|D9$P22
zcR|7aJt22??S1X{bnnMImbNTTQ*CEGoMsyw|6$Ktft|J;5wd%K{F<~k<z14kPTdvT
z!bP)fWt7|Z9H@U`b7t-)TQ2JzdoRS=?b)BU%X-JnR@=p92lp2EmhCYx`oG5{;O(Ay
z!i#L*_{^|9ctz1}aig%EmaoV@Cyir!%dWZaZ8-gHce}gjz87^lHU&9{tey7$+q1)J
zfz6ak9@_$cJDXK2HrX6W+-jS@uFp2XOKZ2q+n0OqMi}g^5twHCz^!G^L8-#MhiaGa
z{nc>6W{SG6?U(g;_Ixn=wkPC(uPxJw4qJhhXZ9p~K4v><-BjDB@0|AXO|jam6U}Hh
zz2WlSD|0{Wd9d!c4TElp_1;Si`{s0Pwq3JD*EYo9jBS&J+};Z!hxhhRQnsyowZKO5
zmDs+MS~K>_oZhqNL~5X|WKrnej?l|{0^GCrRK#iRo%cgzuSBNI9tFeRJ;r-K+w_WX
z?Dgi}wRgz^HCvH?OKheT&))NcPs&DxKi;-3as6Ho+vj_(JlnW;frO0p(|4A8R~ZUg
zS3Z%mlek@HGtGMH-nvDXYy>Wt?OFPT#qL**=U#?wTze<P1ln|{PquZ5Tep|(U(epl
zU9o%FPIK=yu*<S7I{JI>15x(f(~ab97AV)+E-1dT`?!PJ-g$!4tV@2f>|OM&esAeL
z>%A2L9yVfWH}+Pqez9lzvhuy0ET?Shdmq}GxJ%exI=p7@Hp4BpPi`yQ-nr6kyY`^G
z?XH~pwtJ#C>}@+b#rBirBHMX9y|&Lj?zhd_eQS>e|9P8CL7TlQGO~LWCPiC+lzVBb
zW!!EX6U(*N<=Uq`7v?|O^UQUj?K3t-yF0VAZ5=)wv)#FSr>&-Ky=_?X5!;5zlWhg~
z>TJ0WZrS_OE_d(O_|<!lKDOL@t$3epf2s0535#boKK3cL5z9C2l~^RT$Drz{?VGZ@
zdz894ZCym#Y(G`+-g_);(OwxB4ck86$+lKDTy}3ZEw$}?w9ht~@!YOouUl<vo=x2|
z*Y&3j$H_*U2ySCruY(6{o!L(AWm8+TcizI^)=sV6w%(Gf_db}AZ@Z91e{bTW3wzT)
zDDR#2Dr?_44*va<k{0isyLpoB5=*zeVf#28HpI8suJUZ!x6o$kUa6bawk+zJdv8Ya
z?(6bR-D{J(Z?C;9^S;BvT{i1}c<!~_=W6@6%*THIEB1Zm59932mPXq}c%0npYkt7K
zXJ6NWK=sLch1;+0J=-?dK4#m_z0Hn7`{&!1@7cuGXm8Op%YOCwiTnPV-`=}o67z0J
zg-d%_)9={ipPaNOE@1NB*-hDd4_w~9fAym6`={7T?5mKPYCr38qy0jEj{V0b-?FoC
zh}nB&tKhyvF5mYux@qlCl1SgfzUlG)1>3LgVY1q8e_B!A#;5Yq-m+=4_C0lr*|YhF
z$UbI~TYHthwCuMw?%Q|l<Mh3ca;NTJ878{VW#7`hGtcwwt#iJ+_q*BVy}9>v_6B|1
zW^Xiq%U<Sk*}Y0UKWx+;AKE<O_u6}~o_pWR`UiWHFNf{>EU&juWsdSbhw`2Kx<oi^
zT0gYz{j|ShuR^EP-V;Il?U?>1*(X0O+$R^<xp!|@!oF1#*!Ia>S+Q^H-Ie=4*?ru5
zBb{mAHxDrzmCR3eF@1^qba!mum)<9^Z`Qxp`!;oP?X#JdV;i@najyc$l)WzV4eUFT
zr`WS@=(Xvc*tYLz-i5t(!Ta|mIw$P6eRqEULS@E%XH9+fUufXk_fE%h@2mbcyBdk7
zd+%Bwu-liuV9%kqSN2!=&)*lT&a>~wgl&7*2CTBZIYrmr<<9bbmNh5$-LtLT^W`J^
zz6sZ-*&KX$#6CO3eV^zXKl>Zk?(bhv=dthpvbMeVe1-RlC~4ZXey-dbeok%A9$&sa
z8nblw{95yL?>)WT-F&9ktZ#qZxi>??eji(<^}d*|*86_!|6&uc+F-B#nxF%>jF;F4
zYOUH^t<1MiJ5OX^U;^|0TF-R5cYS;JE%AJ|M_+ErKJG>HY^OMF+ONNP(Y`my0tXm&
z&)#?U-le_i7Hs>h3(oGn^VZ&G)*^3PcJF@si~Lvj-uX9a&u8BDy+Nr9Z9Y1R?DOif
z->Y;o$5!Rfq`hU|YxXIe_uYG!<Ne-_93i{J2h95fXDqgVCL^^^xl?}Mx*K=*t}wV~
zcW71j-h<Ki_h$e1-EZTfZ_im7zt^KsV&A3wwKne)a`)PvQP`I(nP7J*;kC^***AL=
zmvHTKIc;O>VeoICiSe7gSrV)FUzGN<*HG}VE@RuY?|v`qKCMaH_R0KMY$xq6c;LyJ
z9;<GlsC}|g7xo5B?B1&}BW~Z8drtclW<}c0_WZQhNi@NxaKqYtKmHfl9{Rq)Zpjtn
zed>&&``#qpwQAQ3vbph9Yu~N6oclNZU9m5JYS(`G9q;$5FLm6zb?4DN98=l%ET7M^
z@A%R1J+iuc_R9VJXcN{ev^!9CqdnX8#e4O5*V!C9%D>NVN6+qQDYN!+2wvQGp6SP4
z#|sJjRw#eA)qQek?->==eV-R+?mr)_bD(0A!#;~P)_vjCW_wLIRrb}+P}*n5Hq9m@
z_un4T&DHz(p55E~L^@|L<J}2+58jF0n_%+HCQL}fR{D|UzV#gY_68rBy>F%7nSD|T
z^L8m5<liYf;rw3fIw9NU_ic9hg|Yh%e4V!ErqZ&#{UuHNbS?_)S@Yh>zSOCI->Gk#
zY?7Z$*}JP<W#5jY;(KS;pWfrSI%MB3n>>5AOE&vzE^gk-8eqOp_EzJbuVUY==daD%
zXKtTq`}ha%KEuG@do5B!_L^-k+$*o4vp0N|*gjiPRvQMd%X{UtHt((4$+7oEjOpGp
z7Ss1$Kf}D&Ky=MM2Ep{*Yd2or<5n-=aP=Vn9?|o<`_?Wuv(J;=YO`%g)n4;sbM~9n
z7w_f%!M-m@K*;vo7WRD$yqc{NuWa16&(L?@()Wz}UjEFttG+6<kIi`XKF>eQ`%;%o
z+*kbb)xMO;bN5CiZQ3spwbwpn!S}rySrhhjs21$~E$h3-w6T59q#g3MhZh~&QySW`
zH{STXJ#&Do?F0TCo7KYC_tttl+Uv1gv5hy#w>`OL>t4S5{rfWGAMWkna(=J6waDHB
zUr+7pek`?b=ON~OGk<Qemu}m(H>`N!9=p8E{RT!N`?R;t*jwY)zW1)tWE<Y~!M2_!
zrS^$0PT0#Ywrg*0r2M|^a+&t+S55YP*fU}8grD2@m)9(`pI78#d+F{Q8&65beOgS%
z_vzm2-OKfrW8dX_=6mIrN9?<|Tx8##rp&!dR(RW`rqt}4@a@e$$>V?Qy>2YomtwqV
zALrBjy%iw~_Xi0o+V_ZTvr%Vb+IMW#!F@-!-`SfX7;Rs0>a>lC<fQ!vI->Xfl%8+5
z_`$`!CtkSi<=VPv@BKtC+mr@@y;qCQ?Y+FCb>FJ0w|g7T^xHk|`my(Zzwo|Qf20po
zF(0r#+j-W8`Mb?t|DO`J_T@kJ)|fEZPJ1(L?~Jyj{e~PI2VQwh+<WE7l>Hp`6Zdmy
z-`*oQPsMiiMX~*Csv=gq7PIVqtfsY3j$^;ITjYd2ew*d?acvOUmoQyvFZ((vTbGMX
z_Lq(>+_&qY{=V)-BKzJy?zHAp@!TgVxzsN8g6zKU8Djew-@V-1q03=kF8F7!x0&xA
z#~Wq#67hZeE@zzCYjDbXPv8%;y~b~!+Q!~%-<MOyVJo(6(cYdT_WQn0mfL5N$hNOs
z{M7E1>$clhe0#U&#I)o4gbLO6YHnxRx8dh*yC}iLeM{#o-Xm1VyYGho;=N&VO?!@H
z9^L!buzzp-qJHbIhOGPkJQlR?t(>)A)93D9&Bx_?Vl!0jx8!WtXMg;SUE>aRdw=_?
zy_2OT+D|cixkt2W*<R%tGxz@9TfBGe|4Do6r-tsm`o4AVo)5EZR?X<y>lV|vmtpR^
z-FNo<*rVZg$0ptOn(YUZAA8M>b!~6B9ko^avuUpc>k6AW_cd+hukYSFVdF;|yZbwA
zW8ztCAOC3C>vD7Qo`RkOdl@D^-}OLSeDA7VYxb&Zwc7OEF|*Bmad>a9<K#U|i@f$0
z?l`koCHl^u_x8JXUlVGx-My>P_Cf4J>rjORn}3G0_BJH@*|?Pb-=lVGt*u#T&))kB
zb#2qlYWAL(A-b2rkjLg_VYRL2oVm8!4s6}qH??uE>fbjumP*=t3%+i*v03S2tID=x
zuS4o?>l-T0wgQV>ZPVoU@7<8T(f06?vfUCgoO@)&Qf$m>&e@(hXlwgo)8;)3x)p3s
ztzEu1OzNYp%%ObS1J~Z|PC7qt?+>?3Tb>U$_Y}y5*hsMKwdqUUXUiABZP$2Pz*b_*
zADeHx6!tE;R%g2*q-*yLKS`TE1t<0dB)zg;Fj3I<Uhgd12f?v>8j2-%?YUIFC+mTT
z-Ti~@dySv(w!OFL@?N&nb8Kt=y6iQ$rfjP*JARMMu}s_WtWP%XHxl<6O_^jn$LZLf
z`P0nyaxCrJ>!sgd`)c2(Jr`@w?+rPfV(TEBX1#<f*H&}Jq}`jJOxz<MAZ$0`Hj9mq
zRHIFe4%^<sZyRj*g3s(tx;M>s>B0WJk*WOF0m<!q7)rvdB{W?3icQ&Ut8BH*c3;mb
z+pCHNwgolTdt`)`?VTIfyLY>c-`<ohzxKXhE4I1vY?E#Mz5{#jz3AQR;nB92XM)P!
z2bxYc3an1nu^P;_s~<G){q=HzZB`}U-U_1$du~7cZ=3T>bZ<!KYMVVfeQjB~7TYjt
z9opN|FKc`1<Q3Zwsu?yD<{R#H<8RwD@yw*Xg%@k~+PJCPu9s}sdm!wImF$_rd$%Z^
zw7tkuu-D+9k?pO9rM86yr}o||Te~+Q%zf{M+JAc_+TPkUuqj&!O_*f6PvFpA7Tw2t
z7?M_4b-g&ecSBBv?c%<9d){Rpx9#CgvF+Qi&30pImaX5<y|$V{m3tW!cJ8`myk@V#
z1j{`KzX<K)(9N^G@<ZQt(TAP3#-e7nb$^c7&a_P2+mYpAd-1sIUV-^sdp9UZ?M-+P
zV=LC@xaYNg!=8JLzpWIsg=|+gH13^g7Q0vH3d`O%|M_i~g`C{meOYa9MEYZ!OLwQ+
z?zy>ZkHES8)+-O3+dFGpudR~xUfY8{{B}C0<MwV`tYN#MZpoeoa~-P{b!~eS{IBep
zG54{J)*0Wua(AZM<^*@`ou{#LZzD&Q?GLeoHcNVU?RlAbZjXeUx=q7mY1@{SXKW%}
z_U%pBRcl@JZJup|h>w-`lB0Wjtak0?(VT2^W5Y8WAL|m^)qatCdrp<x#+6I%-J=_6
ztH5z-x2xxETb((wd+YK|Y(For-CH7Ty!X%VdwVQ8diGxMGTF1?!hzlOY&>?EoWXlN
z=1J^*{bS?a6(u)qf0o>_apOH>Td?qy^$e2-HtBu!d$+i%**ayNvu*h_#r8y(<(|a>
z>-SEXdv(tb;oiN=e>Crnys&hS@`0SaG5Z;852XLuqjPxG9)@=w)_0`0*#1ymyVvWu
zvh9ZtYPMXlr)>9?ueYt{wBE~5nQ3eE#CY$!o|k*r`d96}w)oSY+deOBoSge@C8Akv
zkA(8u3Y&eg-5}9x^S=JEtwZKn>$&fz?fvqCX|MHZe!EnrtG1g;diT!nwzakJJ7VL?
z6twrsQx4mjCL!BPmJ|2RiDR-YnR(LI@60?~mP;#b-9(%B&Uk3F_dsdg?k!b2_iSOm
zZM(zhl8xC0mVGyTME11A9^TtpddhZIDvRyMn9aL)bbPjMdG>6t6SMqYh2RU;DFUIk
zi{JF@ZF(}%_SXC3du91-ZF`oo+Lp*P+LX0!+sm~>e{TxU#ywkRx7uc}xMg+NefOS1
zp`5+z92VNH+QGk9=IL46Z5Qv_82e=Hb>Dbw?_;MUwj8(j?}_SC+oO1ib>G3y(`-Zz
zEVMne`H$_tn0>bHPHSwpEGXH_ki@k&MMG-u^p>T2tr)NFoue*b_kZt?y%FC}?MXY)
zy>|uU!oBTP%6s2k5Z*Uy{{FoY!7ps4&E0AHY--zH@2vrQ_iWf=Ybfw*Pgzx$t-OuK
z-Ud~Xef;w_*y_A#+VjDl)pq&54|`Um#n>!1yu0V(v<rKuH*)M{ZFkyx<)e+Q(^fIN
z$M4c@7pxH3E7*8&&;2Fe_gZ|ovZqyMrEOh7l=b4)=)L`wC-&B5FW9r<y_?Na?PA*n
zCo}eNl&!OkWnHkh`P8w!RXa}Eq{y$a{T8;-Heu2}n-Blq?%j4HVy_g(fxQjom3t<i
zIJZ~BUBvb`^X$DbOWO9VnA2l>aN}p2J(d-=4G(ATYOFiC*Xq3OUXkAtdqYf5?$uFS
zvX@~s?=Jhz7i^QBoUxT-EZjTcg_Ui`XJ^}n2b=c(3*EK1>u;~Ef~<h;<Ef@L|G8%G
zO<r?&Z)?8#UaMV4_R7wQvt8$-z4wgJ0^1$@?za6a&g`8RB)eB*$GW{ej9q(nWaZdw
zahKlb5qo>D$Je8KK9p+j-EO3@H|3aytx0Il-u~l<_UPP=u>HH6$M)N9C)+&@R(nDw
z?Ae<$_0t}{m2Yg8&F<ZMUC(0gjn`89v}{h=?ywKHy|=(=FNac(Et_=Uo}6lX+mI6q
zwk>SuZGGbB?QvD)*~fc4*``1^&DLhm?L85f>}^AyKi{h`YlDqUe%GFV7fbi9Zk@V!
z6Z`$WkDjgE+cB$gFGE_9t;2kYeHod$wg<#~_VUDw*k1Ravqya<tL@F>mbPuvzu5?E
z(6#vxeZ{6HHPyEGQu7{%qn~!Yc;LJD(L3RNr9o5ol%A{E$I{=pU!ZKNb>S_cy%p>;
z_gOYx+yDJk`(B$ph5MeY<=eOEciY}vsY81&1S#(0cp7C}vDbRvnwfKK4txr+t^6yy
zPvN19?dHl|_AFl}?M~kO$j<e_zWx7wp6{7!Rcmi;AameX+A7<rH@WuaRrc5`_4e-D
z|1oC2o(${Wigge7oAN%j3!OM;-|f`pd#`MYu(f{Rw(s)N<2HfQ%JymozTC^Wi)G&`
z+57vSJ{3E_H07W5*K{t2F4j<clQWF_9~36-PcjbO^CwK&c4FAwz0ro>_rI9EW-mwG
z+WlX<uI#m5A7H=4dydVIN~3+=DO2|)IBwdrS(tgBrQU=+>|0OltJY56_u*;j-d<DT
z{l~<2+Uiv6?|a{|Vz2T|xqaL&3-{@ZDp+mTnr`3jG{JVw;Wf6#S5oaPWnS&Q9I$Th
zE%U#76_wp>99!M?g{`dLyC`<wUhYkO`%bP?+;_&Pd(T22p}jAP-r9sOceXDP+p;g`
zdDXrnWwUJsjpy$ZiPyKu?A^L=gPHe!-R|DK72on~T5W{(&3m@dPV6^>1MkP!eG8uE
z?fZ0P^8Sa{+H8LO?6U27)wlQ1*?D^(ol4oyz}{^4J>u3LN%^9Eb8HUoo!z={&$<rX
z{qH*E56o&gYZE+i>3-Wu7xxAfJlieD@@xOIJ#BkU6s_$nj(pqm$DZ#%MDpi-*{+hd
z{S_;0f;89dU$a?dZ@GQLz7W?F`&PA@?p=F)*4_i}-L2&;#q5qublIn^x!C@*_JjSA
zE7bPw^G>(zzQMWYYfghr;7nJ$2Ca14DXLbs;Ub>40rhr!4}|mV)i7MLS4r*pKA|5$
z`*feo-D{M&YyScR<Gph-m+oowEjyqrq3-a`?bO~kk0<VVaB|0<3txNp|8v>7@A`tq
zeV-rn>~($LwQrqt_@3g$clSqe-`=MrZMOel@AZ8GNnLwy&bqMIo%`rMK|YtgQzu;B
z`?{RPfnnR8J+9lf?)6!*#OD2p<9jD1HttmtIkESW`2RiKa}VrQFE!lvSO3x84;_1L
zvcLG+<uI7oHcB10uS(?Fw{4roKD{PoTj5>P?d}{Dw0+>WXm8-oy8RO$l-rvqFR~4Y
z@3blG`?Z%nG<6TN>&?B-_Ilf$kW#eGeSO6y%9Lwgd1k+DlCt)JmD_ap_8TACKj+^n
z`yWQ?`*f%8v6ub4YA<JV*1kqIIeRYc`~v|A->jcq&D&R8#Ix^j$H%=k>pl1F*=)a0
zfbYPb;*gztV}Ek)dwqQSzWAV7do3&0@BJWUx7X3Oc^^~hguUmd)a{*FaBpwB>cst(
z+1vN=IbYwek*#REbpPhP*Yqp)hHf}-W8wdFuffM-Hd2+Ews(Bj+ZbGHw5^l6v0uHE
zXP*OaiS37;cYEY=&fC8JCc8KJ1mpgWM|#$4j+gG+s#Rpy9(;9=7hA%<%s1Eef0Z>l
zpkVcApXQvveW!1Iw^=9Kx>uiL&0dZTv9=3D*6yj1Jhqo(4gcO-9CK_L9j*3W^Y~+<
z_I~}|3jMRT29I_2Is^sno#_+4Pb}s3zIXfHS>O1&#&*^hQL7bgHTxESld${T*1E5u
zrr+lF_APr?c8KqrvoUe++g4_WiJI~IUe4j(r*wGVUf&hhto<7H?)~|7!QO%^GxlEp
z#pEz6u))S@qr^V$+N6D-4?nl9J@{d7&CQMbl#^fW{bYXB*7?HZy`2u`d->F#?0x-}
zd*8xG3-@~F-`Z=P*S$BOa*6eXSqp5}E>hiR)i1wKY7Ns~-D?i}-kiC<cgc&1d*l7h
z9Qykjb~7<5?AyA0$-X?3Z#La`d-vrD-rc`&Vv6;}7i;!?_<3OOo)0(n>NB3%`)~F7
z{k^6C_We*PwOLTF=5SfM-sbGQgL|*M@7=R<p74IA;^zJF?>hH({ZO@g7nEc7(BziQ
zbj!JWFTYUTr+?gUZ$!DKjohKwy}$U^+0Ea5X3wS*0uHLDckOLm%xmkJJi}J^&;fh5
zT!;Ni$CLM76N}s1bzs@P^W}GKwp^aSH!N}UKE+S1`z9tb?Q^X8Wc|n`;y}ZCbsOgs
zYJ0C}+wFfnY380YUJGsN%lY>;_=Vd(W$Cx+IP11Q%72ypkuU3QQ&f2OsqJd8fB*90
z-oL9`_7)%8zF&q>+&=cwrM&|G_U;$?GimRs#fp12O;Ovu%HhJE<BuzB+W+p}|E_fU
z-Z^z_`!2tWw?BO0=U$y~_5+s8fevS9+U|Fku-VsS@p!M+v+lhClV{p%iLc(XOSgIV
zhbRB`y_k64=Fg>D_OmxGwvLQ1*ne;P)qSr7Gxqf?zp|&q^@vT!-rakjm95<CahY%L
zw*4lyQ?gI*tF0EbRnsrDUl6!^ug<L{d#ApwI}l>F)b@uL``%DNu6?}L6*l)G%=Q`1
z%i43Hc<vsBqS*Z^#~$tPv{u;bYTUiQ>FmG#oAoO9J()RqueJF6{h!W0-zD(kwzasV
z^1h1yjW)`ng8S0GF5WBcaLtCt!q)cs39h}Z<q-}SV)FJ`1!V8#5G&jhvd+ME*8`*d
z+j$o5U%bL<-vy1NeGAk0_l7)(u)WJ9xQ}<qZX4l6Y_=(r`SyLgY;4<gZ~b0(M%KMw
zHaPB0NME+^_`$S2yZHC)QLT{NS7E(wufc~|)+zaicE<Jow%42f&1U)&g99lK`)$Nj
zoA#boXWRET_|kr3fr@<`>Rx+yTFC92eOBA1I^)LPw%mrjO|gsi1gG*^SM4s^ce?iS
zew`OBdspq-V3WJMeQ%nD*FM8lclI7W(`|oBLfh7YIndtdX7gUn-(ve_wtd=b7jx3)
zfu{66%a7N0UwEmo_e1=t-Ho1dHeVO}>{Ynlv`<@ei~WIw6MMejo@i}%EP3BV8*RId
zQNQ+<WOnYesF}6bNnzpMQ{|iX9b99y&)jOs-ZG>3ecjC>cKsL9_UfeV-5clXu;+T;
zoc+m8$$KO_c=so|Ufj2(hSk>e|CIf+Lap{O_|)v2dUopGoNH6<HtMh3x4%x%u5I`9
zy@9{p*e4X~@4q9Gyl+|G#JxJk`}aOkf3er_S<${nc^mic$vn9CSAXu_JG>6||NrdW
zSJ1g_x530^`&&2t-M4e)x_$A5ayIYd18hFePqr~md%yR@n*;VeE8O=p&-UH#y=&9n
zMT+A4;`cAvD|1Z9Hs<+`eg3C~c7NEwZX4Wpcke07Z~MGw>+XyE=4Kt0<hK8Pk=*W#
ze4)KM3Q>ElS#|e4IJ$4&Bi97mO8?9IMKhP|VR!tww_YiJ-@lj7_Eu)F?d>qrvz_pY
zaj*L4UHhJOdF**ryJc@g@q>L06D4e4vuW5W?%cn(KK-|ifQ0=1+3t7kUpx)jm$Qg_
z?~dLT`@TqI?O(V3%D%wUz55$v{#zfFxwJQD>!LjuYa?w0f^+vr=jrZCH}AAo{={T2
zvQWUTgYVZ~E{SH_>Cde8T6+oZTQO07uewR!-nEMy_Eq&w-&+~`b)TxJ^gipY;ro|u
zy<($OH+?_z&G&l+JX&r3u9&?)>ix&P*F`Pt56(QWcll-}`+lo)`?egIu)B64`#w<(
zYkR>Kwf&XMYxdoDZ{D{iVa=XTXZiPf%$c~SRK?a_v%YEXtJc-_l9%%Ki6y%3|8~{a
z*8K>hO|9*sy&Wxb2UI&Z?Y+46@ZMKnT=u4E*Y348VcXX+d-vXSnVoj;3#<2CUdgsE
z`*D-KNKfzHmdqV{d*W{ITje`zpYOKTy_^=Jdo@awZJk#n?2FsjV=rYtc^~`V=DiHu
zGxoI<Uf+A%H`G2)?a03O7sc)GrGMS0)cAX^)Xv0xmrHy0dH1>RP4Nk}O53#5Zl9mj
zUa_oNYo@a8dyVDJ+QctBzo%e#)1IxD{PsF1@7${qw0GZrC*^(nwk+Md=j_|PlM40r
zh&+<nCorjgugS)~y%O7E_A>upVAsJ@W%F~^*}azB;RpU7Sh1f~?&aS7#R;}ol#lFP
znIX8(@zAQ>M$xSMFFd}x*DB(<jmXWPd#*0kvOTs~)&8eQ&|ZEMr33Z~BKEw^I{PxS
z*X})$vTLttmC8Qh=xzHpvV7U+pg!AX@1mBy59gTgW4l#to${-2@4}7`wg)OSZA8Cq
zv`uB+zW4IS-TOSB-r2L_q~?KjVc+&nc-OYi#N_RM18Ym$+NT}1Px$KW_pIK#?~A$S
zK0D36y;as7dmZM7+nQZ%+k47<{@$k5A$z|G?XYTM=CJiDG~autaK62*y!PJZX3uOj
zjP}{OY1HhE<FdDz)3j#qk9BH$*{AaCtMuQ!zj$`f9@Wj=`(&1J?_VJrZ@=jELmQ1e
zrF|xwj@r&>duy}#B=4SppS3oB{>0fT>^`zP`^yZQ8}plOW|*?tL~>c})r$VMcdlZ@
zUT>{+dma5&+Ws!sv(IY()x9D%)Axn!K5g42E3+@oeAE6H3vcdSrvG_wcITPBHFmeH
zA3VIfk9XVEz4tx1?9J=e>??lYx_7~f7q+$^HTF#|e73js4!`Y6+n9Zy5?AlH$zj+Z
z_wd}_cPn1)eIswOk5}x#zMVxI_SI(b?)#APeBa5nwR_WxOZK#8Zr(e^PQ&*1(K~x<
z4`}W?k>a!W_rk|}8LGUk)wloN+Zw{W@AHpyd)oI*+qW{5b6<IP>^{Zc|M$vi|Jak-
z#kOxoh~)lSNrQc#gLm)CFAm>7xruFW*wxm3@h_%ZnQgniZ{n$jy<7fD+SX)8*`<eZ
z+nVSs*gHGo;hx@#nfvy?65i*ile14~Dyx;{kBNK!IqtMA-+a~XXyv!PFH%Kq*V>Ei
zy{R&N?-RN3y%(;V*k1m2Xy0yA=6zAd9QFYp6KyM#*V^1E`?q)L1kL?h-iYr1mOgWD
zhRlq8!dE=@dEJPzj^4L=-yZSxww)(aY+XJI?Mt2*ypQQllwGRkc55?Vhkg6^l<Ze|
zd0<ahLeJhuJNEB+;p%0Z!MteC6wR%687o@%*DQQvt<BZGH`RZp{j)oZY|>>V_deQJ
zzrWIRu6?@fv%SuJm-p{@oV71~0rUP7mXG(^U)`|xt9SOkIb}inw`-X1^<K4TkE8Oo
z{pmcC`_ldQ+yAV#urJ+u&+hhqw|#}|{(F<J&D$^Gy3Stg(1bnL_*U-Sn9;nCDN<=K
z<NJzzg>tO>W=-_p&;Q3{f8=4g{Wg=7ZTH*y?R{uJZSV0_SN4{qe%_<ER(h|<m$rR6
zOwoIucl_Ds@c#JT8YMg1I=?x48^xyYt(-Z3&zh8(HXEdb_wp1N?t2|S)4F-(tG#<R
zZ{NQyv~6GJecQd>zmxa--fP?!w6$rU1CQyxl=+hO<*%pNyt^j4@5WkBTT8oc+x5F<
z?bT*<*f&Au&R&gsF1Ej)Zrrn5Y~B9ASGoHHY(@66^)J{x=l{gLe>k4n9iEzEe@Q8A
z--NF9dt;Pj_x4ze?{BDPxBtI{d7nt$f&Hu!5&Mb+1NWA`T)MaA+=9Jz<}UlHHeKDP
z@lRl1)0ryUcXoYy-QsQc&Y#a}o8CEnul(oFcBZj%`|^$19Cic?+bo}CXdAZhk?mcT
zEp{G)_xCa{<=gl7M#jFtOAJ;)4NSHi=Jhs5wMA`@A6dQktp2inS6KAygpWk*-SDY!
z?-#u-`xjnI+NZx^$Nt+Q7x!BDEw#G;>GQt*#?jUu@g;lzeEYlCKVkBo#qP8AZa(&I
z4~Ne8Jtso;?e*HixzE3rVc!?s_4`a!?CtI!oxFF>t$F**dGq!yu(`MYW#yqg52WAk
zWl3MRSN*%`zJ9ia`}$&9_U>CYW3T1qS^Ju8x%a6_HrRHyPTad>O4FXFd~Wt0(id#b
zvYpv?qN#5mr`w#pzh^w!yLL&f{f?*wwjNFk_S`#Cu&@8|ynT1X#O)YX6zwc<`m#5)
zVz!<2t&)8o43F*JD&D>KT%zp0xq+hl1d_Jew)k@FGqsDh{l<LArgj>Sy}`mmw&!ja
z?f3YXzyHX!$i0b|FYoV|t843^b9uMkB$vI*KFQg4eze?QuUu|_`K158#@ws>lQzuW
zdwyBTKF#v%eaGY{?3?wyb#Jqi_@1?LllBF&Xzq)%GPQkV@XXFdh0pdY=Ul6x&0F^^
z%@y9a;n>7|JkIm?SSXn7_sg}n$vF6P&x84E?IzES-g`}R*WN=(llKb6$?f}OF1nAE
z^Ro?0{WjaIzjFIFtbA+}owaK3r=4r}Iq%Wlo9H)t|AeU@Y-`ut-Rr)rX7A@G5&Nc#
zb?@IlHF;m{j6C}-4a@i1$hYjBVNh)=bNlzcmCe0-ciK(g$2pt9E@8f?y}>?T+Xh~l
zy*)eM?Q#2e-{$%})_tEHI_wPv*Y3S;+PZJr-SvCFZ_=^9-M-r{w&}ZFnnM2GO}+j5
zAG~$4?s!wMC;h94UD!{)y#;$C_9tX=+CFi(z3=WrSv$7tbN4u9WZV8VSiEm#-_?Dt
z2e<D#sHDE{mBQA&7xr@5x|w&`W(%I%ub3va_g3F6I~LaSb~}4{_S&&N+2hIDXWMyy
z+1`ka>Go3?mhO@2*uQs~d#SA}(~P~m|4!L*^EU5QzPHnE-;(lueI9CiXRQ2gd*s^w
zy$jY}-5tysv;Xyyg8dqg&g?nPXSz3{P+;$N4sHA7t*&;BHlF+L6ffH=Ts&!C$>N56
zA2m<dZ*9D{Z;sm*d$09Y`#c$v_x(3~wzs!x?>?`mmV1RioU`@)oNOPeyl<~ZVZz=6
z*Z%DJey(=kf|&KYH{TK1`*X+Ty=&LY+ZuLF->3V<YhTaGAlnDqn)Y4!IK%eTi$i;t
zc&ype8+m!JLpk?e)5^#DGFHCa^Lo9^zB8Te_C7p+?1F9T_vQN)?z<;(Z7)mZ`@Qx3
zYI}K}Z?k=}ztcux!=ByVg=_Y{JGFeT>BmWSGiDX;eUPzeU$2F-{i~GYdleU6wBNQ*
zYwuOd=X*cLMeobA=iVn$Fn@2$->Z9@<r;TSoh@d&WtG~#)j~gPLq09ryRXLIZew5a
zzUt=Pdyi!5@0I@AWcUB|{JoF5+V?4~D%gJ}^4Q+o*)Dspu6k?NwPNnR;E5;gnd1`n
z+TC1jdn)3nZAX&y-gP3c_C}>`vOT=acwdH-$zE-q^1U&o8|~z_G43<Dy>#D;<Es0X
zyYAQ<@wj83;QG`19;9pTy~@3OpTxGq_T~K7Y{fJW?pr-Y+-}y2<#zHX*{!RpQ}<5c
zUts(0Yt7!Bys>+Bo{QakCF|FoZSVN^o!A>|YZCW-kHOA@Jtx|Z>?wG3&RWtmbYDlM
z#=a*97w-Mr@3ZfH(usX%5*YWL$n~?0xD~&*?5l<SwEb*$ta)wwGG@r_;p$mx8yTTz
zU$MN_wxD$0{#PF>_Idm~XYG2~YwypZ<$Dv(^zT;Yd}zb9kLN(`G&Orx4$pnRZoS)^
zGr4g89qY(_9l|a9{vW+>vu5Y)y(dIA?YZckV9RYcZLdYm%01;A;d>dxZte;FP`USz
z*4@1>d;aW^U_NPcZBO5xD~z4{ehXc<t-rWxpHq|N-Y+*q_sw1Tdw+$&{JneKo%dB-
z-EFtcBy#V^x6FI(OQzXvX;t61txIs<xv4$2*$aN}Z4p!0`^}};KFUPJzL7y~ufGT9
z{%cFt*?Jyby;noy-JXJD+WXlq%kB%Z(B6AD*?n(t+Q$71{AsppbGPqpeSE^M+3Wb8
z>-ved2Fd$uE@+*#_F&kw*RoY(|2d&uHjA|n?41zbVC}J?b-$N`?S8jqM|byah}hS4
z?eM-AB5k&b{#*CmxGcRl_0wYeb$dH)H$I-axBRB({(_nVwpQ!S_wdKRw3}hFc<-C+
zJ$q(aPutsLxWm>u^W@(9svGwl@yoJ%@|kJxRBzV(+R~c)18Pca4_f}*^X%eZyYHrZ
z_VL$C+WI`)zt1ALWv{xuymjei4cqj4X8W4gZP@2mH{D+S&4j&ua;5e<3=i#im^SQF
zDH1=hLw@~!j&5fA#`ClG`8WvgTj-u)|MBw`yXxynwu=9m>|`HLw-33~zn87sV1F!=
zq;2O%g?-`N4ErZZ_Uwz|KeTtR#D={#Hj`~8U7x?#=iyXamx6739?lQjx1;r<%>~Cj
zy8{hR?+tI-x3_!o5!<?pO#5oO9QWBwvf3A>_G7QcmxY$oc5vEBaZRzE{$}z%9`RQD
z^i{L>a(h?pGvi&e-&kSVJ|Tk{drt6s+CP1}!`A-5zC8?*NB2hDH?kE=`n&hy^W=R#
zvD5cvbXMDNALZD0mU+^?b5A<=30+RyYcea*w&uFEZAZM8{iHd92j+5j?elt}zAtft
z(Oxm}h5McKRSy&h-L-kEx_W=gyX<}5ySD93uHoHlq_D`=E~Cws`NWI8r*jqeHN0f8
z72%q?*K4Dh?ecl6_Ih5Iwq1X$WB-I-ANN?csoE#G)bIWBaKT>Bi&^$5cDL=5KKAX4
zv$fwBy>#-v3EMvH-E}&C--9H9z3+>;_L-_>?M>v#*}u1cwq5)T>3wTep6zvh#=ie?
z+w^^3<(v0Agj(%=sCCUYcju!07QPZTUmUyk9@r$j`?D+8zK+%F_h#&$yC<fHaqm5b
zw>At1<96pwX|iSLSZxz?XUg6=tL1E8c%|>zFlohZg-h4>%rHB*H>1C1@8u7B_LhCQ
zx;OMr?cRcIPi-zsTG}k@PT70qcGun&+b-A^UOBK=zH#>6E%}RVHN?_vI@@;bO}IJJ
zcGB|cd-sU9+w?wLVw=pvY0Ds>w!@n<a<6uupxx)CCDz9#EVZpWa>iy|K#Q$zX7Ao8
zzj=EPOsU@;w0VZD&)Uno8)BPmFIc?Tv%P1>-UE8ZyJeQj*%l;6+qUVl?+vLuW1BRY
z*DmPUo4pJU2dpl7>h4{cc4xPU`R+Z*ag%Jj_f6PqmdLYDdFQdcYpt*C*${nxuYji8
z-lln{_vA0^+AE=WV6O&4;9i|ilD3>SSNC>hSMIgZIAr@GcfL(n@^afB54PHvDXq7)
zis#)UpJig>CEaHm&@g$A%{%6OA|V@XAK#q3SD`B0`j`23+xD})drliK-Rr9KZx>5-
zn=ON?+^+2hIri;1wRDg1z8+hK2i%qntUIk<z4YCyHcMge`i+)*y)u{Brflf9;pqEr
zvmlINU$~yf-lpX@_vSLL+Z)h&-{yeFF55lrtM|_0$=jQ?Q+}_@8n(SlI@j9vIrZC$
zWX-k}X>hhx5SFuzxP0B_h4XQni7y@ZR{vMuD|4pHHfElS?GJBp+q1WF_Iii$**tk<
zY@7EaW3Rzkp1lvYZP`1ksc`RyYfXD6>{)A@&bnytzcV>|CVe*F8}V<!o(JJ(dslgX
zu_?LGvL{AS#O9K_p6#S-2lmcd*|*nNY2jWcYxaFTsz>)M*cZK9KvcnM(SpaeT?Tvh
z+O_BI_4s(m=5FEay#imJ?AfGn*d{6J{9ak%{d*sm@7nvS<;32lt(>+2MVG8g#A0k;
zxlFcwV%xFTws_%QyMH@vdSAY=DU*uaYrWWH?}y8My9IS8>}}<lv*$YJligc7?(Jca
zJhZp@Ud7&w;`cTz^Iz=!a%0Qhd++4-3M)$Ov3s?C@1KkcTjhZIy$!2Y?k$l0u~)}%
z>z<Op^Lz4CvuwlmAF%DT@!gxSW}nRshGN?n`n&foj&<C7^#_C9l=m98C$G-id#3sL
zo)0s`Y?o|Zyhr5E9^2LvO#8}&T=ss5uh^3>{dI57Cw@DniHdeEH%)C9oqVuof%)S-
z3RUN=ZTc;3|9!66dm+cuMq%FRy>08-Z2vFp+q*T=bZ?3-!`>6GqirR;j@Sx)o3!`!
zOkL~hWaGV4M4WAFI8$urpIo^2Lzu{(farXiKN_`r_uc8=%TOh0Ddly1ug|1ed%t%q
z-TUMJp1tMk;`WC9*lp8r-gfVUOw~OH1Rn32F)ee?ja}S(y}mc?-PbI&SNQ2Hn+NNU
z*fa?5u=d%&YJ0$9lFc5$*|rm`MQyA11?_!&J$LV$Y!=%m#oKMK=gqV+vRY`%p~||~
z=zHg0hU|Sd2|2ubEEcufPTM$j&o8bGd)eFNZ4aDTXtgD0wJpQk-_|irO12z#rrSzP
z>D~M8WbfWB+#0qEIQVRqz3kcR(mHF;^jPz~cMgcyPKkYGvp_U`&xIFLtbf@a-D|^P
zY{TufeecAF-)$0nYHW82i0(amQ`*iUzTB2atI2kcY`pE<(igiQboT7cx&3laLc$H}
z@Fh*Q8ioqCcLG&xzXU9^by_5EC&6N~ch9}awso(fc558r-XmJPY)_86;GUg1s<ukW
z)Ay=zFzw@+8EO02L(uL)V*g&22>!hqI=r?5UF-IyG?eevX_;ng&}?jTAwbgh_`fxK
zU+leOTlxHsErW{bu0PK=TQ6goX7h8ou-%P6%C@?9XYAb{o3nRLr|7;r`)}HQ(7j=k
z<T2Uyo1^;Pucdl>xvX~XT|1Y}&OxkiuY*F(o*mW9wgsP7>|ImCW~;##V7qV92HWe>
zANKweU%K~OL9UJW)Rw&sJ;!%1c*?Zr!c{4oG>+waU#=D0$Ckst*UeSJu0x4s&y4_e
z+c0hKT@B1@c60r|Vtemp`ra*fciDs_wb^>)a_u=QzJBik-^ASvXII(?<`(Uhsur{3
zDxA5uZu|7T4AVHR6*z5dYZbQaHQ`#jXX=WLds%|HZJGRU*f^|i-PIGZdhgZq2lwXu
z-?>NT?FHL=zm@lXYdCHjAU|Wzn-U${ly>&LA7WE&TF!^=`SK~qHstYn+XmaTU3wK~
z_IRi#*h<vR-*fQaqP+qwGprqwukHS3)U<cko@sk0I4IaGT03p8TlMB$2YjV=3u(^U
zYovN(&%wus_i|icW$ThOW$(PIeYSf%Pua@6=eKp(DZVG5VwJVf-syY1bj0?l$T#j)
z-0^7dN&Q&sN&jp2y6k1LJHYy2&xE<0wk&6-@4Yi)s*O%w@LqvuAvXKVllS_)USRvA
z`l_uatHj<<mk-;fE55hA5Pa6=&Vm_x)ikHr9#|{7_l2?J9+l4>dpRC$vXxlVVq2J|
zY!?(Fy7$2UT-#{L>$XypnCy<I-QKg;Xxd(esvgUNON(t52wLvBvH6P4jUT?Y3~Y;c
zcWCUjVVD$Uv*G_4Yn`q}Th`92wl5Y{?cKw&cyCq2VOt01TAK@kf_8JBtk}!5jlp)`
z(S>`Pe`M^{I(lKR_nNJH{z!(|YP{&OQ934X_fRL_HiWrmFT;ah=4oo@Y+Xvb_q+((
zxR*t+bMLRt!?r#ey?df2vG02nxn}Q|1g^acX7AjavCePL2cro#3skvm9Tv;m8f=_n
z%W(Us%>y%I+Y9x(Y%{d-_O57~VWXPrxmQH}=pLb|oqG+^rtHo>I&-g&J-eOB(dD)o
z3H^J2{9J0Y%wdsjROJ%eql<oR|L3%LFGI~`o3gHHdo#KV_bLR;*{k;J<z6Ns!+jkA
z$@@3$oVR!P#pQeNehb;R#($3emz&RRQtp=RTbL+s6R`K<-Z^Vi_c6$yvfd?ibnlAd
zvVD`c-?XtYY~H;@)6}*^Q)-`5?&AFsQ|Ili+PueZ!POb|^ZeQONuRv9cVEJ?{f1YI
z_kR8;v#(vS*?zmxrhNws*zG@fr`o0q+^{Rzu-N|5)%p8$d>`0+y0K&L>5>b3TlszV
z{l2?@?{?9$y%(4L*!f-CYCl&|jJ@wOmVMIQ?fY-=P2bmX<^8@r%~$uz-delY>$m(K
z@mbUNCVEuc{}$#wz_feW{sR}U?|mp`Yj??9XP@C8jXhsDD)(*waoF}v%%QzG>Dqhu
z-|sx&8+3E;r^(`b&u&saP&S=wuM<n#UeS`Ndn=cD?Mr-eXrHe$`(6&IgLVZ%NA@PA
zo!RqH%3vSchVZ>QO|5%ZGR)cQt6aDDd9mX@ro1ZKNfB%IMtPjvS0!Ar_x7rdHXo{l
z_f@K#-76CLW#8t<Tn@ip)bER0=(1NdU1{I*ruqBNxli1;HpKKm#CJ)X6)&gky(ZzX
zC&S;z&TrmQ`+qlF_7$|9*mF3x-!|d!kG;)#xAwNIkg!deuzqi}g3bOT3lsK#EWfnp
z)5A^sB3wiEPC4Ja&tm7~{ZGz3-v8~L_pY9~kN4mEIAQNHo(+2w=B>1s+w;d}=8QFa
zZy%Q2S9G?)K0=y#KjZw6y@z}cSbcfNW3RNT*>37==KXDtkMA{5oVNGK$GLkQ>OA*8
z4(B?s`mOiAz6a_KFK@otFAyxZznXv5-d+D#_q9)3zBhZt;XS9twD#_P;jy>XRN1!O
zFn#aG9dq_xSbxykeN(l)jFsj-o&SIL${n0+|H5y(ZNS03dp`y7*iE(zwPQZ_XK!u|
z%f1ag-nP=3R}Xx9HPJ3_&y;-`9^89Z-iq9(`;BYg6USNmEiZrGXSh*izhLRxeavgp
z_kFKc*=N0O>%RX%llLYsTDVu$Fvb3LztUbthLwA>Sey2)6>Ql%E!=nSKOeEZj0Riw
ze&Bw(=atFsefhlQ)>A9H_qK<N**;bf+`B^fnccmqclWAsi0l*VeX@7y3?ciR!*aH7
zRz9*eJ?3V=>&bPyA77K~O%f$+*D34mQ&}HoySG(*-@_;7`?E@q?k)MWbT2>im%YxV
z@AoLaKVciVeCghl$o2yZa(3HA|D3b8I^M}n$v|}9sogI7GJi_j-;j^9|7W_yc6&nc
zKHcaodziB#_VO|K?K^Z)!G6MSBOABqm3z~6?A-fs`{8}zlNj~{+<CIUzj?vlHy0P}
z+axr5|MG%ud*fKk_FgEpJK!7@wQsLU<pEo_V|y1Inzgs}{NcTm)b8z-y?<pde}?;B
z`542!nHLu9y)aqdmd&_rZ;{{Yy>=<_dtN==YP;*Ez+T1dBHK@Y5AW+yIB4^dw`!kD
zV!{3k&*Jx`x9#88pW$TpR;GLZwngT9y6#BqlYV`2@9%fJ_Xc!r+9kd9)L#C-bN3eN
zo!NWUI%99ga}S$4kN4SLySix4jq`{0@^P8%GYIV5n|J)<-jav=_pNgi+UFMfe6P`}
z4SPQ}UEaGUOwxX?{FA+_pT_Nbmvvz;;~vpHo=emBSKHp*EBEHB{gwaA_Wg_gwO97S
z&%MjT#rG!W&a#o(-MII0XNkRs&6|A=O*8h@-8Zz!y83=^d4|_MY4%6^Y!)uu7cIJc
zuWrEaeL5Rm_JwYawUv6Ay7#H=`Mtld-P(I`;orUIH97XZXZXMS%k9LyXF22dI;&0J
zXSnj--aPBheVSI~d)uG#?mK!))Ip~0_g>j1>3ypYYVBu@s^2$rb?Lt3yZQ%A8r*E{
zCePcq+3CXGWI3^Y+u0WG-FH>YUQ@7bztOCTd*+#NIh2Pn>?^tQbnhxBwY_gA)Y(mZ
zFKll$Y2V%pZLAJX-dql~Zu55kjB2*scRPF^^BJyvGPecxseCTB-N*mjuG4a|?WN9t
z`?U3V_sJ~?-YuQ{*LpvT=ziyn#ru=%jciyZtM2tv&)BDGIL{`av1Z@zyLS8RJs;Wc
z)!(zX`#|x&WIhgu&#kNXsva@hbD*YSU)$H&dv(I6?kQkgyQk_;_wFZcY<q9tG_co@
zadlX<G~afbXv<#X(=YAW>-23V9_`yZYl`jx?YqzRJ^Zm@@60Qw_OE!?x>w0IbZ^ZM
z;eAq}%==Q9rtW=I&V1mxV%e@^;h*ilmvZetcqQNd^U){!<;~e07^M>TxqXb?S9|37
zUOCf!`?dEK?OQNs^4^)NrtcL~m~8*ajB78u+cf*{n<wqP&N<&+#XWz2;;z?rykC#(
zeHe9dPsih3Hk^L{_wrRS*+h8w?fr3b^S;;T1FZkGa_x_l*u1x`cH-U--_P$CUwdlr
zN6qbfB3O-W4(cr4tGcjz?_IA6doE~fwYGV9Xg}i<z5QF5kMI5ddgg&T^NIs?pKJC#
z_F1rZo^as)WsGY3IyOJs`^9U)Ua`wfdo#ksZ8A^o+ZR72cW<BAHroT7EBAT|$=E+v
zSh|ma*?3<}mF(VAv1YcVJ1X{nvYcsWS`@W!T?x~^byk=5B%hwKk4LG0zqG@gy<aTz
z54ifY>|J3Vu*XV7ZQqq^X|@yYnA`lWW7x0G<hu9C!X}%h&-eBo^jy8O-08bbO5U}-
zWx1{PGY@at`?N&b{=(9ty>iM1wgPL__8<5mZh!6mrM=VEwC&A2RJ70c^PRn4HlN+g
zQChZV+SM#;hl%BT4=sw{Co_4<{tu>odn5ii>=ox0-oNYG;(eEIs~yOCduX3t=7K%1
zE_Uv@?xwdVbf(tcRXwe{SLsdM%gg(F?-C9vn+pf-+HCofvuDZOEjBCLRQJxiBC<C_
zm~&52w9ww8MoVlJ53=u$tvYJ0wtVj1pXct`JbD+ur{~Ygy@kDh_6Etk-!p653Y%&A
z6K%y_@3UoCre;08%X#k#mkWFUEO4@wl{&RIWZjxQdcyZ@^m{C9TLh-=J`=&dZ`Xb4
zeL4#B_xzN<Z2K%xZ%^N6nSC3U@7$Bo5VBYQwdCHA#|v#Lp0?S{mN~Ka(D|c#H|{!S
zv(dxM)+KhUt@>R3eHDI(Z2wEJ+U_}eVz0@91vYoCgj)Z7*JKqhbJF&R*LK@uc6;~w
z%A4(3$t-Fo9(Bk@JfzX4De?Xuqls&6*wx?I{=B5QM|{qly^mg~*glwZ-ex`5gT3!o
zwAhMWm$$iWf7GUm`M2$NXT!Y(&o1m0?Vh@KTF?dCu5HqL=k2bsS#S8o_6`rTom1dE
z+m(4Y_r9`UVv}}1b?@@@n!THT$n1S5cilRJEoCqJ-Z^{EUOTzBZU33QJ0#=w3bDT0
zyU3hr-yZpoyB|u3?#)PdvzceSc5lMj3)V+iT5UTx-`Pxiy?^fvsnop-VykVlDtq^&
z7q{B3+wgwR!aBXZv*h3IH7S?b>$&ciO+*`ujTrl!z31!x?!9sN+Md%7e%PK|an^=m
zf#e?HpjmtSB3SHPrmnJmTytXY)~KI*%0w7#@A<LqV~u{ZH+<zc+X)9=*l3jev0ZWE
zi0$oi*?q+-zI!Df9NYVH(Sg0D$4~8f|Fy@~<Dbahil$(j1425sU$&InT28yZH-5*V
zy#g04ZP&DU?zQ#3xhM0=+r4_TI&BY>^X>DV?X<V+$m_jN4^7(RGULcz4y~hmrn=7E
zTW3;l8+T81Z>MJe-o09SdpGFi?|r%Z!=5FdOKqpkli4?C@9sS+r`Y#;y^`Ixg72(N
z2KOAB`d|s$X>u3$UZ_~JM_%i)HRCqjz2>WQ_BPC#zqgQS!(QdXf_tU+{@d#yAigJ0
zHFU2^8keo;#BQ4h>t|ZO4*X+Nme8^HrN|wd)wWCaPLN>Rr?X_{-t|sX_f~a1*n8#5
z|2-2XFWYUhv1RXq-2pa%S1#EaSjF24?@+guI`6%g%X+TuQ}NEdAB}@;8<b`C^yq!J
zem!rk4X@Q?n-`UzZFVjHWZNw!vCpHZX>XnFWZQi+!fn4*@3CR<GPm~mb7Ie4-x+)5
zX0Ed}*x7Fz+q-)2qJ28HB>@Na*6QrpV-nuB_fPMBTb)Cn_a3@>Yp<9;ldXrJu1&z@
zGd8IvGB$4}>ev?TIkWc*<H@}j0(aOZ$Z^_k)l%5!$YHfdRabxSBNvvv6PNPa>b>c)
z*?WK2o>{jy+Md3>cJG<DGxsX7bJ$w=6z&ZOv)L=*)VcSkHiPYlvgv!pCvV=nscWk3
zrEpOj{zLM6Kg_M%<6^jOuh6nXwg!ip_Ws^mwD-rRO}0Dy?QLGn*s<qG{{)-Og%|fK
z-2T5?F5=pr`O5t^3oK6B=9SCZ7NqsroSD4aCf)Dao&{6#_C7M@+ZVk2@7_t8`)t2?
z?B1K^Fv+&6YvCS-jZ3ZB%IDa=37=v!!Emb0Dlz}P8x2nF^-nps*Gpr~Uf0Ru`(`{p
zVJptLb?-SA=DiHNFIczQ&fUA@XN9dfla}pOhUI&miWb}YESzTZ(bv<aph(K@g6KNy
z6R-d6eL0QQHn&mN;@;23z3m;1dn1aw_eOcv?9Dcty_bKl&tBP;@Ak&pp53$V5TD&q
z&m-1bx>oOzj;Y_1aR2=7fZjVccM{ob9Wq__zWZ=)?}Kw9HV&mJ)@*(o_w0<*v8`}i
zVRNE@VV_?Z?_Pzl8G8k;MA^L4JH0ogrhCuAoh-I94m9rxE7h^xS24*(p(@<gf=j?A
zE=zi!?`566J1#`q9_<m_6V`v-X3Yf^YfjdwdyDQY-}{11dr!%OsWz$lEjH`oO82(R
zy=`L{qPTb7$^1S1-Msr=haK2E{l|OT&B+t@N(gi9vp!I?_t5<{HZuyQ?zP{&VNb`Q
zW4phKEZ-Y?&2aCFwSs$@l9>0tY*V%^af!FJ=sdW$@4w*QQ_{Y+3YYw>Bg$sj^37ba
zH~5gWt^4xLdv(vPv3YXi-QJBckN0XX65bbcsncfbqK7t)9W!kGq80Z_C>*e@6ky)N
z=YQAs^;Q!bk2hOv-u&R-`zwNFuhJiWo7a}{ds)*K+U__OV{-&_=Ec{Ed$;XbYWu)^
zg3YBY*S#h=skU!7uCZj!>)(~IG01l7xi7ZQ{_Nay_@Cuo1+4{p%im3~jj&g-xj(tg
zw(pVa9`3KdcQ+f(-g`=?Yw!Qb*Y*^>)!p-5jnUTi-Y1)tLGpWjroP%+CiT(AASHgU
z!Oxp}7wFyHozTN-8~)?q-aebDdz14&?8)HMv0*#+!$u)|xou+&w_URH;XU`gp6{*o
zW!d}SO5Wb%cdzX^>fW|D<=n45ao&u33pY0HsnJfcy(;z9wm-pj@3F5}_8w^JvORNt
z>0Y0GhPEX>3VU^yE#JFC^tQFc@2Pt_4B~7g^Hld8{&{xqj-pMrXMbDmjmuEj%OGcE
z%XWSD-kB4%Y^Rv&?2S<WYEy7ec<&zFW}C?u?QI^ao!_hBoVa)L;kC90qGk8|`etg|
z@#L*d_1=CP(S@J(mN0zVV_LP?HeqJh9v;5sHd9RU_f|A<?6ID$WqT*)lWoIJcAE+E
z8*K|&&e$ZLk=ysfp>@x9zW%)me|2|1n%HHVak0^6@wH8Rj!)dNw@}e?&%etBd*3|l
zw|#O-+IDe`qMh5j$$JmD_U`q$>1C_HS7Dnocj{igekR)u^89w~UZ?gfnYnNetJ|@?
zetS9gSx>98Wm~w)wo|QkZ*lSBy>AjbZS>FU?X5W7Z!OY!(DsMEs2zjL3R{u0{d?be
z74D6f{$w+g!Funx-;b?k*xA{1Y*=CQD*BYIq124sK_>h6ZvVe|@43S%d*9?#>{WTc
z%ywE`-QF8+?`&_iFS8AGWw+DHTD51*6p?+_ZLE8u@37cf%;mQElKp?rt=>I*GZb&{
z(M#jnw?W+3_IuHyy{XoMw##Q{?KR*k+3V30y!S)(Uz_v}p?&Vh`0aRVWo&PIeYD-S
zymN2G+wZ%Z)s6Nph|%3Gw`iKJ&GYEJEhiV-P7x8a{d2uzk0KB6Ud}x~Y`5IpZyU67
z!Csf8d-e)??66&2TV&&$ziqEe|8!dg<14#)mh11`@}t(;?(1gTgbB`j?LW@nt5-eG
z*2nXL?e}{%dxH&nZ52u%SvPdC?3%cAmaSclrPXi7Kei38#P@#UjNNP4`fslt&&EB6
zrk>urSv|>SWz*WdKVD9=mGarRS17P#FN4`p>)8h{*(w;;?%sT8<KEeNyY~j%j<h{^
z>d;=L$p`lu6h5{&|JP)1$0V=4FaPY`d*%$E?X2|mwmJ@idvk<&?D#Zp+cauO?c3eK
zxUb;Ut=%E#boV;+-M4Yc*krq7O~qb^<g~pR(T8nKrE>Q0Fzm4{3GA^wFs*KP+{}r#
z5^cNol<=?LvvYgDtx935&9$jrd!MSWwAGOE+#_?n*+wL@eJ`K;GTT$#8hg%}O|tDj
zuypT(1RlFNf7k3e!xwFv<F<a!JWh9;e;vE`UfQm`ci(n_y;6soY!7TyvsRfQZ!5iv
zZJ*jYslBXAR#?j`7ubX)i|l*B_is<t+;z6iw<hmB5};r^P3g_vj(ZGyTsAw{23%{g
zVOYar^XS6dJ!+G~!S`K0m@I1dzUH6p2R+GsH{ST#p6`s_+oQ#?Z^k<<8;L_)HYrDL
z?+Fn9YIEv*<lg6UpKO<SH|#xW7P!}B*5thkeEPOeq9gV$unf00_N>|Cd{cdINBjoc
z%rM`*uRdPb`(RGf-b4Np`xxq{?{1i^Yo+pvd+*i_OZIw1W!k39XR)1W9I$uRyzSOI
z*ez{SQ~vH{JAQiaX<n<n9TMMcR?l2z>sb`Fx8zizZE=;lZNAphJti#8wnf4>ZH1(n
z_nzBwe@|4>aodA&XZ9Sd;NAD8`_Nv6gCA{vNBiv#ezn;4!{`3pb9zl}*>7{(y)m-c
zb0tgO*6N>(tw+t3J<P?Ny9*^5_eLB_wLP<;VQ&Sa-JYJizibwS*X^;J$hS8px77A#
zR=bT%{!|;i8A^MrR-Ck5t22GCMd8gomfPF*>MG^$y=HJ=?+^dkd!Fx0+glUAde0AY
zBij|fCfZKm=h|ym=d{<!eEQxWbI#gcaGtvR@59Y|8BU$CN!aVXTi2r4O8A4yo@*0o
z_HGcExc9(@16F%hez#r3xPPy9!*gr1lKFd21gO|X3oYFnk=nFp!8(5%+5W4xCL(<M
z@_PGhXGAsc@lPt<o3dn)tupV7y*^LlZ6$8`?QNJ|yKBQeC)-0illPtu%HL}herB)y
zt)e{)iyqmS+!3~2VjW=nq5q6c#+UZJbIpDCR+f71ZD?9=ouQPq*MWJ7?X)EaZ4bNd
z+B@^YYTJ<S5w=rj%Gmu#TxKJ3-FWW-2TfbfLm7LoyqRg+5pHJdd3VWPgP6&C<~l9j
zTTxeQ8(-XQo7?h#&knmuHaB}FT0iT&zxM;bpq<TcQ`<PNv$k$FUABJlJNLS9iSISn
ztKIu_@tr;8I+lB%aUZm`+O)*BAmYT{b3#jZU*4d+*Y2|1zTPCCJ(pfQwF#?bv7TvD
zzBhoo(RPt-maP{j+rBT?e($jrcG!D3Xqj~bm;9b<A7AXvJT!MN#}k%4i+;H7eGpK(
z`^1{hHmA;>+GFCHV5{c6)%O2^)Lrc5wR>9vW%tgRFJ{YrRBi7)!6|#=Ud*t~PhPpV
zRC}JSX~E6i24{}%4tg|Uud;BSZI-gq9?yywHs0@#+ITC9+Z|)ww|9zvkB$4vi?#xf
zjcxmPo!Kk2t$A-qMuqL%{$lH%?-jPjDoM8g7M{0NtEsg$KFVQht0`_*X7|eW%WQ^y
z6$TQv&u{GA+ni{%*JuChJrxCMdpGJe*hZS3uvMy>XuCag)ozE)ORNj0ZQ09kS=Z|I
zy}5gLGnv{JW+&Ueh!okQeR`pd0>>|#TYGu-_TA~VeQ-U0FAEp5ZTOZJ+kFYUY}I6M
z?TT1owr9oVPqx!OiSK<7bIj)d&!e^(x(PPlpBL|y5#zDddM3P2ZS_9e7_$X?P18^9
zt$*IIx3Fuy&6Q;qd$@(~+lCk|wN<#_VcYe7v+e4%^LrbnU9+C^BFa`HrP}sLQ->|H
z@YcNoiEg%gdgtvm=n%F!RCmVa-`)w9Lc;8O#W<PwSwu_QwoP-hWsp0+H^f)S)?(%!
zo2Np8`vhk9*gD$Yw_!N<)_P_5d7D|E?e<=;;@kcEPW+y3nW=jpoW8WT@tLgMgXQjf
zLQcurrnv614GI5k!!n(3FVm$3wlNwzY*^A=Y(Kouwpo0Axy@$2H+v@BZrpQXLe!pJ
zsrq}f^WWRXgk|hK7V5EAR6@?y$D(x)Lv{KtKBh-DPvh?G>GG_%xgc?JZ%jt5t()*(
z+lrYR_WW3Qey_w*g*_7UHt&|^ziI2~tFrg)&iuUxg-`Fj%C}{&-32DQkB%mLUTpQT
zS+sDqZKHX~-g%;%Z8NqP?vb^Q-OGQ4-|j&2?p-_VS?peId1^aF<ky}PzE^CdWj@+I
z^vT_;5FxNThkKE&uY{D{9i}z5E@i>CMtXgF^<E3?U0Wkyd!}2^c8e~@-irG3wog~C
zwV5%6$yT5+*fwv`ZaepXHv43rx7Y{>%(W>laNTFpy2`F}N!H$Z=VsVC<Zj;0Fim~e
ze5uC0KNc>sbvwLx?}z(&HV-c5?m79Rac}vGguQB0x7a@FJG{@Lzs=^yrA0QmVGHdF
zw&>dMt(d!SdR2;D>dlFJRRkDq7aa?;voM=wWAk;f-5IOdwl^0}+V{b1iCsmMg{_T%
znXSXT3sy6Z?6log+P9bGK+7J3U@cpny}S4N-^$zP(6Yep#6B<E^?KX(Ex1*-k1N=E
zuinj2y9OgYn-%)A_XwO?wD;T5eRgktIq$PhO4`S8<nUhIy(#+??#-~-Q6aNOB*xHI
z?Bf|*f$omo4<3l@7P>jZ?pfw~TZ74(HebDG?|X3Sz4eXeMB8Js+pRD7?%Bg-QN1r<
z<{s-z;oyB8f6wk&A+yhB)0-1}D{^+&Icz*_`{c{wy#mi(?TKmNw>uTM&#sVr!Cq!v
z1v?9!lY0*AvEDl)@`CNVz|FP}#lQBvSd_6(dhW5k7ruPjt?}5v_HwG<J_o1OHoR^-
z_N}TvX`8V++IGr$7uzca-M0A)7TO)K>99GmHrjS(=gNIQoL1~DIrMDL3G)V<ikyx;
z+Y7Sxbw2CgcVpLIn+vzY_Vs+@vCX(JY2U*oy>@~brTa{lOYJ>wIB8$g?8?12BpLQ{
z3FO%&eCXNh&bZ$$;Lnvk37?nQmH*J+>rgerD#6a)hIh|oyHop`_Z`1*YVWLvJM4~M
z-Lh9gvcr~XTfp8&eN*?%xO8GK&&;NM#uIn#J2KyCpU3@38xBEP+l1x1yKm}r?z?bI
z#`exp{=KZJ5_U2-XYBp+W5J#^yF6^0E?3$ytcl&_)9Am~N+{pX%yY}$tQ{+Djmp>9
zrOztgcSlg#cGp_jy$tV8TK<$P+*jV;v`_ertX)Bc*PaYR$-OtO9N$}zH_>iO?M|Dk
zDXZ-k9DB7#q-cXp+tP`9&zt(~`|y9~-c|Vv_s*F;#qPn&j6HKIvu!ih)Y}{~KDzgH
zboJi%_Uv|SrHOls{zdJzQCn~O;i&gMlY<Lvc?uf$t|=3;&HcW^&cR>O#z1bC?YqqR
zwk(ew_hqmgu<=<n-A3ElXP;B{-hHo5aO~|Y@88?S(7X4jr0U+ztSNi{o+{dBWD>tO
z!~2xY21&WSLb=QL{<7U=x21jiUeoZzeVgnP?5fyK?d5*B*Dk^1zjdSI4!c|SUAB`x
zb=m10eYwYCQp?_bD;Mltp=4(}TX)A^ugFDqIrAplvWM-nyR`I}ZPb^Zz3I|h?dE@X
z+NTt-)Yj&X)4s2>eD~#^nX>mx$T{2A%60ove$CkPCnv$~Ut{lHQ@P;10yDQ-N9f$y
zvm@HY)<H{RkHM!YHeVLZv27~dX*=iodb<xCXRO2f5A8KyV87SnIRD;n-N*J?nEtf6
z^mvM`g^aSTV#>mO45eo**LAk;oyu3bZ|3y5c8R;T?fsy%(00O()w?}(T=%YB*R}V`
zz4^8d-}-DF-kr4hcV*4q3-eak9$7Kf_T-%9dl?!Q?wru(Y7?;4*hWH7cJBkNwB06r
zS+*%T(zfRw<?X$+;)rd7>r}fS$&<ErUuWBHkpFB`;mT#p<x;Xw;l!)mfjef~@i_b0
zPKumvYm~fb@00bL_WpgbeD4)Ed)vl?yX@xp#qSGQywHaG)2zK6*_&<e2dC_vz_H9)
zL3M`p#jR8Ke)}=cRzZHp?n`@4@BJ}TeNR~OE?b7r2W>p;h4vnv+-}z_qQ6&Q*FNjd
z!5jA`9C5K->~v=D1LJ4bDmVRYGj@vF^jjzIlM-vTT~NQvS|D=izBj(!whnb#)^6`y
z_RWymzvsegjy+uadTr%ruiYDABxm!i*xK$&X~4dP=Ogw`)m>#LcUWbwz_J?KJhusZ
zmri%L+xd8(UBOZ=Th})$?ASlW+d0V0*d=yw)!uxQb9?)qo!cw9uXb-x%K_Ww6E%A`
zoNC`&;;?@&kG`qxg3}dy{?E+Z>tMThpZlAY`=-=0*rt@P+9NUTz}^Gz9@r$<_3c^p
z?t<<9>RLO-r)7JK?lJ7`*}lZ?L3_5X%hHv$&u*@^+r7=u_Dor{t&BU5t;T=nJq?=|
z?uxUj-?#Ep(Z0CW$+la*H`%-uUAu2VSdOj5p=2A`CoTJ`Z_lw?@wC|{ucL3@?YYf+
zi))+hq^$JpGPKHUe^m6@&a*sVr{l<G8*enj?w97meRGy=*vtQY=Dr^fJMH{z`s^4q
z6s_Xl>h2BXp1beb;u&^=&S&=qEjeLZcs0>Z!PVC$_iBjUtK#i@wRjiU9q(RlXIePZ
zZj$|yy~SVm@8wunW_#e3&F&1Y-aQkTT6RaWeYUk|Tf1ky_QZYCnkjpo)&%bT_;0FR
zjV_O^ME3?;l{XvqR?VDet7Eg(_DfifZPTkMw$GHB_Ld}1v72%Gl<nC|W&3vgnP)eD
zo$bEpb%}eg`YhY05wmFTf#ScrrdgicyQX2XO@y6<&8!{EY>!OO+k4<EpKYV|w!Ptp
zj_iA|ztLJ@LGRw9Y$kg<uCLs?Xxi3&95svW7`pjvW$jPyUBuVAw?=8!KC@|o`vS5L
z?PW>w-^XH_wwL8~o9#l5d3%d*JMVoI-nRGe_42)O8}{3B2(Ps%*w$dP>>lIZhV@F;
z0_NNI&Qa~O^Z0RP&-sf%b`0H4wocVsZDW(l?CuCB?pr$9dT)jK4%-P=z4o#29<n`f
zUd2}6b;jOLTXyX=a?;*gSTNmAICHk0$b@6IF?E`Do9!F-6;#gHqp^6?zHJ{J_Wdb6
zwdei*<9l~pl(d_uY`#~ar^sg3<ehsNlA5d^{O#RS9B;ALblSzeaxW!p-P{`Ys_fOU
zTX^-tKAu-0d$(~*+QjV-+{^dmsol5zIkw&lp6)H=lHAA6lDUWTuGYTirrLY=D6reP
zul}>QM(DZiA6>?MB}{>PfAZJb*zI6F!2Vur-;K%J?fBQ(?&q7edXLn(wEZjRuG;@!
zr+e=QE!lnXVcPcg`{wR*%wV=Vyk)*EW0<_nw6@g!zvQRy`=nQIvuxkCJxjNX?sGgo
zY2T{Ir+aU&wzQpp=F(oCMRWFFUd*vyXX$SnhGt)TZKd_Q_I={n_iD-J{q6Z%Y_?9;
z-Pg<7Zu{hP*#2FI+wJVve%ilJU(9xct<C<q5zYIA{7m*<PF}IE<Lf<}pls1S>NgZ^
zuf|CpkeEDmZ)^Px>*=h!_J1`L+Z(Q-w#RFu|K2k1j=dS5yKU6$&h1?-dwYLCfuL<y
znbf{{!fdtzs=xQHEz;i?DiN~pV3gFp42IBsAG#me6#Y24w@<urUv16(Jqtf9uyN^>
z+Apx+=HAWeX|`dHui0LG*uT%hvwZK`_$m8#J>RnbK=#aiYaY1o<&AaO;~e&HZ-H&Y
zUiH~m_dQ6xVJEy(Wglbu?Y*zcx9p!%Hh=HlXUuyWc<1lECT_6zN{iZoKU!}4XWod|
z``*TP-<gRjdpCWJvYYxbZ=dJcfc?Rn{_kC<RcQY-YpIRm;~9JVGL824gtzX_Dw?*h
z<LR}%l^yZ+;%8d+GK9XhkvQ{Z&(F;c`xTx|+HVtFW@q(d=DuYjtv2iO8u$Lq(6P-7
zX0$)`%EXqTd*}Y6t*7>JGI#DXVe{A%BDrU8bN$J^K1*2lw>}Wr<5Ik9x9_p9drT)a
z?)_cych75uS$m9T*w}|mH{EL=scJi|;luvC7qe~Ws0HtRe%)*TWqDQGY!&5wvnKHD
z+wj?8&%{eC2jq{Y?|%__X5W&$lsy+8Oxb6tD{gyW*0udJwj|octemo6{MF=roX*nw
z%<df9dv?jGy@wC7>|5<GyRS8!d%x$KlY32?WcK`tSKZgLR%36#g_ONpvK?)Vtq<-!
zHt)+`ne;F_`^%1dmz^};o3|@q|FZ{+_WbnOV&4+@%SNx)$5#HyPwUeuZu?xf9^1>M
z*1q@5Oqcy7hePdK?^x}BapI5d%WnRC_s&?_&bg?yFXWxWzRf3J@8#OsyZ4&b?Y&1d
zI`=N!QndecCzI`i{onSf35gw;ov?ZDgeRx%Hr?sod#<cwpK|pFJ3ha3`-8el`*zLo
z**Ep3uT_Ougmvy!t9>VSueA;N;<WdF(xiPAnUi<#u1eo~>vr7U3=Mz#OnHTUn%i0T
zsa#sO->YZxp0Mhfd*ggd9E^T1+1LDj>)s<xuC@=mg!V0pT)HPo{NwH~-^2C>83pXU
z*RQ<yN6Y6ueJi8)@D=gxWjUd+FVIoXCMa#i-nX-)_X%ft?|RUqZ*$@3pS=ecSnoT3
z$;1BlbCG?~*J}1D<z3kC!0~ebd)dalLOp$Z1797sy=Am~Pm2ZXKJ{k%y(V6qd$(>n
zx3}Y@)V_UwEBCU^*RZL09Jc4dhWUHfvexd?iD=u`p>}ugj=t^tidqEs+s@s(*Z<8G
zn=ZredsP<89(eS~#O`ro=f15aNA`ZaBCyY5$JhOn{LS~ua_R5iQ`odG-&SSMtP8dK
zjc-`(HEF(Mvqrdmuh>Vv11}3V?fbDJ+D0sP%iixtv-ZXPpSAaf%FBJl7W?;XT)*1p
z^qG=9+Rv}<{i^d}ulBZ-Jteo+*)Dv$Z?CZF);$xpSnYlL=cA3xU1QrH4r=@2rDgYR
zVdt@x)_2*rqkWal_B(&}9`UQQ&&$}cCnmbe_EtFef#6m7`;ytO*s1MWy|3(0|DLm1
z^Y*Q8oo>5EI(VOopvPWK^L@6b=il7#aB}Njjm7Wnc19=K9GB|fSCP`NS3lipzq8Nu
zeUfI|_RL$%<M8^XfW60JxqYc8T5LT(dGGhUwRi6m8)Msz^RMjHJ*8}~y|&MG%G?tB
zm=l-wij}GEJ^rR`uiTebdrP~F{noop_c{qn@B6f(c;BMTb$e4cJlUH*)6VX6MWo$)
z*>pRH+w%J!zU|q|StGR1$M?JKT!z#8ZknvN5&W-eyGD&;??n;$eUleR+OnBM?oSb^
z+VA;zg{|y~Pqs5G&+VPe-nKVwUAN7j>xcIxZm`&|?s9*x<|}Qx|JpnEif?ALe`YOZ
zJ9}@(o>ZAXd+JX+*>(Df+FZVPdjIN2N&BzM$n1BLS+j48c)$H`C!u{>&35}rxh?j_
zc(&Uexx(n6C%Is+x9Zxx*B%_&zw!9$y$P>F?7P`S_jTV&w9WmZvR@<nm|fk}rF$b>
z!uK86c5Qc5<m$a{YPa@!&6{m|)aA*(x9rpQ9%f+Oe{&_vp4njscF)Y}*nivSsm-iM
zoi-~BUhKIjH)ZdJzDIlZYo^<VaI)?Tzj4Oa$NQw+4VC?S&$jUI54*^3pTB<5-YwVX
z><(dlvws8MoV_g$wzl7P9o*Zureg2pDdBq+&n>hyl1bWoKJe$>OkTyk%S>hMkKSIt
zFJ<Ssy^OgpY_2VMu(w+C<o=>s1N#F<TJ{N@HrsdW!J<8TS+?7%H6O7TDEqQU?CrMw
zOPV?MNjR$R-Jf!5&xC4MTfPh8d(#fz+IOLP!tTTN=6lnYN$q=j=JDPmRVVgdyeP7-
zA?mPwmfK<*p-jpBr*7}qE&uza%_S?&{r%Hb4@lhc+q>$X`MwfXxxM>TBW#{N=Gb>X
z<l&wLQ+)RbzI5GdmDyy!)NJkkfDnegU%Y4T{h%te-$7u)zPL}mcFYSz>>b3|?0R;D
zJIr6aY41dfMSC0W2=3#1{C%(CRnvVjP4D(-I3Ki?(P`Ry^!Wb0>OX$jJecFP?^3Vq
zzKXzYw#RSj><umE-B)|?z&_>QSN7_jZQl2<gk$fuAnv{FOV;nx$jsV%<SyUd0~eV0
z$ur*B8_O%UM>M5pk88H$-oBNS_V3l)XuU^s*S_-STlV)So!z_rxxxN^J(2z0v(5Gf
zGqms3n)Js`^v%kBmv5BX+U`u<yHiNeeqE1=Eld9DeH{|>_O^N5-g{+&?mh*}X|@sN
zS$iUEB=$*`tL&-VUv59aUUh#t+q=CgH(U4J_HEc}!YH@@PSCG?Z~yPy8!}yEPn)NO
zEyspRyDw)&_MQ1vW<Ra)t(D4iCA+Kx^0pEojeFN#>EHKw?$y1*tDo9jK77-rc#+b+
zX|a;~cD>@>cjCr$JIyW@8=oig`&>7&?A!Rycki}?zWcgXy|D2JU$NhX<Imo!a}D-A
z`5V0NMxElm%A$>XLta$wi;Mog_q}2Nz6q1L_g(gzx%XSetbM;1&)@65XQl07W8-~o
z)0y{O$UL%dW7fX?4C^NCQ~S}d*Gs&4-&Du8eKWUA-S@0%!rrQb7xqS7<gsQHlHF%?
zp2;@&<zKtkU)JrLBYkK0m$|C@S^JjRFzQd*D`LHL@9B#4z3ZnQ-k<(q;{L{L#(jmR
znfo>bcUm{_CfeWL>TWkLEOfu+o5I}(-+AtN|MKCUfLwVS*~L!#`Fuq7CCNM5f1c*B
z=eO-X`%QYU_Bn^8?0pl|w%6cA#y-CHXZCwN>)Q7>yL(^tnTdNpE?sV;GW*XSn@jum
zMKL7rTd+IJc3qUl{&y>8?_q3zznf$4JR9HI-L~#qIQQ*HJhb=4i^H}n_4xL-Ee+WF
zekc3BY4dsa`ulb6bNq09@6E>x_WoLSdq3A=;eGL6Chtx8c4hxfJ?Z`Pb<%fjd$MHj
zmDVy_6}!{6@1DfkX<U!pXVA{Jx9juNeRmUdYz>Xp+5c>6-KQ~;%ONl4+&-V_f%~k#
zckQcsx_EEp-i>?m53kr8D4x0J;={bX{B~M<H72U=-RRh{M<a9V-Z$&!+b}e(+iN@Z
zyUh{Bj(r!yoUGz{>h~tip1*JZdcl1^=auX~wQRZV{LN3TWhctp@GUL2)jVutv&$lQ
z?^>%*`}$wbvHySm@4nx$QhQHE&ECtSdvkB}|JgRhLGJc30^9b!+GuWLkUhiZKxvt+
z(d8!ldW)lbSIu6uPd!*+?|=25)?5<V_94GA_S|>7wC`b+psnniIs3N=w%V>->a=gs
zLDPK?*X`XCHa~qIQ;Vuy!haR(DHHAX)&<Y9x}DOsZ^3;(+r^ur_BrhA-g|!H^1aFq
z%yw~&y?X;1754pFHrw{MV$;6RWk2mMv})T12q*2i!uo2@iD&$~x#~mqZed(%vueiH
zJqOj7@7*z<W$&~lx(<5;cJI}93Ei_OD|O%E3Dfo-4ZXhi>ErGDw0cGN{XLhzFXL{k
zo$=jTTd`#s`vQcI?PoUmZ7<xMypLOtbKmh>%r+A~^6k-VJGVC`UVCrK$}M|WnELEX
z_<wG%O5w3RhxRVmYob_b6IQu>uT|GK>jXB_eRGzt-K+m&+rIDpbN0paGwt(mIcpQ~
z^w*vP4>jz4zc=iA`c8b`2WBJNDcc(N1}i$+f8Mg#s_&VHJx|rNeQK+@_su9?VtZbb
zdtY~u=Dr;g@AsZz>)QX4)o=d^PS*WOoC*8xbWPv)yXnr}C-V%gpO?4n`=u_mhl^2j
zpK!DF-df+YyNi<#+H@Vu*ej@+Z*%PcpLMw5jlGAQ@7NSvf4RrnNM@hqgim{as25uo
z{Mos0p-#l!`FgRo^K{DXjaD@8J@wpZU*gYbd%?L=_nIp2vXYSbvG3N--`4C;_U^0a
zoV#b=ot68lz31%JssF$Kn~>AKxx$?`@2o!9t!#X}cd_UCeQI9E_RhSjVmGz%o89GY
zocoSjvpWcEo4x=1UBBJe9~|F%`-j~=AEQruubhzG7ohuYk4Ue?z6%bo_8L8$Zhvv-
z|GiUmIrsfEnY`EKhtz&mndSS`vi<g6`E_aUskX!W*6bG9=j~RzPyW&AJ$;u&9k}$m
z_f;vH?(_d*vHzrn-rk(AZF}<_bq}-@i0+FiS-98lo1Sgio$S5OGP(9{(2m}J#O9uD
zNZ%}5Hr<zdbLwB%S?^f9*F&ReuQsQ;O+wXOdrnLFedhB*_Aj4jw9irKvkgPMz}{0m
z5B9EO=iX;rdujg>wnuv$E1T@hloV{DeI)jWc{JObUEXXLU^#i;nYRn~vE+ZXIZ(1>
zf7>LB{fVox_pXn$wDIJtuy469vQN9TZ@<yJ6*lGhhwV2#V6od#+PHto?D>0CH!<wJ
zFuTh}_RjUai$y)HCH6J%-Td(5zJ=buY<e3RY#&rQ*sc_n+_z27+Rk^n!amo<JbSGL
z?e-Rb`nq>gAM4(O2hQx;a9wh5+_uB}tlzxezjblz-i0jg_OH|I?U$d3+jn<`$-eF1
zckUC)`@Ea~y!1Y?%71(PgBMu`ORCv&&pWVBNvLnni^ZO{rYrXE{;$Mu|NDCTzV{mL
z`#$mu?+Z@cYcoY@!+w|LUi;&|v+Wbk;@l?@esS+kYuWvtFFN<l`!2GNt75a=y%d$b
zpS;=k)pfqu%X*n<?+4yZwu+t4_UZkXvAuZ0)HYm2d0*73^?Tc8PwkDjXW8ek)w}=M
zIp%$TTGZ@q)M9NnNU`r*Ea<RbPi)6Njpu^<${8y6eTj&$o$PsV4_`6MzFDoi_P(65
zc<-D}@x86@rtaTxdG$WMzp;B43m@F;rM})az;gM%i&xz2G7GHi4+hTJ%l=+^U)SdY
zHhz=YZRNDt_Bzh+-m4LEXYcYS+xDK4Ilb3yQODjFYg}zl9+<n=hw<&6hA@>qPW{vN
zuJP{O6LPc5Hqp7w=GZHny+VJC_xhAD*tRI@*v_r}xbxHJ{kymR_TBq)M)zJNxdXO_
zzdHB!HLtRr!Y6L~&-mot0}b`IX$9uC*S)p(luW$1w=1#BW^UUY+dKI}`*OlqY*|h}
zwSBu!WuJD+G8>x|)_q+rhPDdgocj)h9o=hFH_hh2g*mnp!e{Q?9}r@zCo$8u!T;QD
zwi68dX8m2h_oz|n-jA#c_bzz<(#G{@$=;3M`uFt3SlSAIjNE&|>y)i8pR}E&shF({
z*TlW$zu4^-%QV_v>+Z7tcU^qnsk&!-e{Ip<`~JrEJqcICZ8urD+3XXr+bdwY)Mjgy
z?7j<1zBYH3@7z13e1px)V4i&uONDH|xV+i3yyx{^d4nT1?%gW3F;4_-{URprmI+*7
z`y_w!UY*VSw#61<dv7otvAy;rVQ-eowY>(nFWQ{=@zdst!wH*pzt-%n+jqpK+jRTh
zH_8`mH8mXe`sq6FmAG4Iv%vD6b;iG`do*Sow7KCXvo}Pk+xGCL9X3Ymh4wnDyV!2{
zczAd5TTdGWc9A{imCbwe>~`6fyUXp{eEs1b-zSsyHVAv$F0tIR_e{?IJq)JvcCNWC
zVJ8%tvsdZtcH0IMOWTL)_ifK9thLSEU%t2HCYx=>MOoV`>aT5g<k;CTK0Um5f8hE(
zmSU-U{k;zCo#OZ1CZu=M-Y5D(dl|kP+g4by>`nMqzPC8w@Lr|8ymk!L0X8g|%WYlE
z^z5>}XzVc-Ub$DifZa||IbrYFA8Tx<ta`kc<K{{071bMTW91t6GR#|RJ85OdUZtFq
zwhZS}ch^iS*!wE?udUQxf7>RbgVvHSE%uhm@Y$W;EVeJ|+z#8O&IB8QEgSZ}-)d(g
z(9>xB#Qgs*Y0dJzLGgEOq_>OP*>Gp=ee$H$wn3HEu3?ev?n;{#dl-eb?#(+Ux=&Ml
zg{_)*ylstevF*I@!!|o!p4+Q_<BKiRA%(r6Iv4jW-r#5Z<SV1CLUfGHj0an7E&mnm
z-FZW856k<ldwAq6Y+lY}x7$**&?ceY+Io6(sIA7o4SN_`TX$LS6}Dqu-D&Io#$YdF
z%*DOpEy8w__}1Gx_?+L<rz~kVLCbURmmP6?BNhnSE|yl^Bem|}9-k)hy&)dvwrl=N
z+WH%?>@x^lwb$&zCab8Rb+&sWKksFq%(7Q3-gWN}h7)^~o^QAPu5!)Rpn=(Tj*pEk
z!{nP*es4e88a=JryNNl)*2#)(Z>YE5-aT7Z@71vL+B5lDw5{s&CDs?d_3kxTw`+Hf
zZ<norfTG>r^!GNO<1X0rKH{_8C#twNMs1GmBVln{x4Z?m&oA5Be)ZV3TXXHUJ(^c6
z_6o5#+2$%Q-FrHte6PpBd3%_6nD*Kq`LUPh@@AVqT=Vt>-wxT^T)%nm-Z&Y%D^+c_
zO9E%@b(yl%#=zO$w&Nq;o|HukZTUn5_Fc(m+{e(UW;6NazP&dBl=kf2z_7QW_M6R$
zs36<U%iC?gM0MEmH6GkMNAb+w%!pQ-O>JU!4>pMItFF`AyUgU=o}PvYd*p?)_6j&X
z-+OV|>b-n5jJ9<)+ig!h*|2wuv5#%bq|7}BLR|N#wVm9%M{3JnTh~dp7tDUxgzVnA
z_y3i7yL%4E*>N(Q+S73=&&Ij(sBID7ecLOu7u()3zqD62kjKuTxXqSrU7*dE-F-Hh
zsR?^yJh*Iy8rIreb6;sI*KvAp#l-`*UbP)Iyz#7dVd)3<-rK-ww`p&qEx*yZJu{@P
z?up@5v<*49a_{5V*EZQJxa>3!3EAo!&f4oC&2ASEUT#}I_nb|UpqedHo66q((+}<4
zccO9c^O*B{HJ+*13SVEd_r)ABoBa4zThEt=ZG+PeTVL5)YI}n3$)3yh348PZ-`k_&
zu*jBS-M&3Lc{%rK9FN~?aOKTjcSUKtX$x6w%fe>weXljewjsZG&;55w`y|hOw{dcJ
zw|#W;<sOIZU%PK;mfEt!E#7<mz&hI>dEtBXCy3cD7M!^^Z0}p^g^%CbC~rBr_r}vD
zHY-oQ+^c8EWqUupeebdNH}|A0&$O{B*s<4qCI7ykFOqBuHr3i@D(l#Yyz;PR4STWs
zz^1u-7}l}wZCJo)_by_Gt<8rP8?)ogwnnSi_daSlVB2v1->!HiOWQANd~97dY}+GU
zXSdf+fYJ5{zq;*n_K!9au6nk<-3faQ!q{vzmU!FND!1EyG4$T+n!VW8Wpa^iZRhm8
z89BXs4K82bYp_h%F7_akZN|zIHs^0{+aq{c)$V|^-ku=6<<<^KRlDPIyY{lH&DcAq
ztj5;YK6USbhpo2d-Gckpth>JF7$1w>il03;7pL0nImcpU<MVd@-hDx{_SR&z+U}k-
zYwz5r+xM#Xir5|yGPiwPy3uCRm8W~VW+mHd_-?n&+PBBH^uDC6bh*P`_Pgx+jO8cW
zzMSZ{H{<rYy-Sw3?agZ1x;H>P%ElnGZ%@sF?mc0yMSCT@p4xiu<+I~u+F~2g&#=cb
zrE9O!rpdOQLh5!G;_up+PLtS|?~%4wM0=*KdRW}vNd}*6<CqTEUdz2<D=i&lyK3j<
zy`p_b_a=St*lWn_W^16df6s&i#<uGp9JbAv*0*PkZiQ|7A=^DAZ#URZFSfA>6|c26
zd)8`eV!z2&DxPicG@fbJ7drRsiR)*y)mm${cY=NI-p$PlwvuuOZF@{s?mfn^$o5r*
z*ItR|mbRAn?(MZLX|S<Wm$rL$h;Q$%!2Nq2mY?1GpnUzFMk6u174G7DmQC7g%kyBV
z4RgEUUJXgneQnE3_x}E|Zr^m{3;QJ8jrMFVXtcTddEP!H^YeR~j3(^m$>P|z?S%f`
zI{PzwX6@(Nm)Eg(kKj_Kz4L2J_vwCb-`mIaaPLe1m3xmEEZmzZHDh06)R)~R#xM4{
zU!1x>_tfjXXYFR#EoFPU@0f|wzWbSXZPqIB*suP?xbJ}7PFq7kwtXg{QFhWxpV%!w
z9ky?$N0tra`HtNO&W7z<rMbvv!<Wf>AFpJwy_Ge0uXvW{{sfiY{fx$o_Zm)8v!A<x
z)js(1?S0~c_WNb?MD_`azuEie?S7l(ArtI&Ht(}7=smdi(@l2U`bKu!{ae#*@Ao{|
z`{Z-_zHc&(dnKMZ+T5PJd++YIa{HHidG1^IrDAW&vZr?5OS$(&HTUejl=f>cXHLuB
zbIYCgB}{VO^OgVEKJ^)GyZ0^)-TSO^-kvRgzwa$b@3HpRdA0Z7ABBCFHQD#&tkJNJ
zJ<GYfIdH?i)d2x}eM&#?O`WT|&qH*^Ue?oIHfz4i>~mdLv2R=cI-A0o6ZU!kxV=we
z>*9Uywx-+X>=)kW(RpR>n}^){UQb}Q(`hWT&tQt%_p`QZ-;4>T_UU!s-#gJdaPKwN
zfIU1ZI{Th+Ki;=>TKxW90X+NSHq`7}6QjEK+G7WMrZ?yJP3U0VzkkA#U43~H_S^Sx
z@9Wo{YU5C`+L9-(XV128{dQ$GbM}gJZLkyDAhln8!;4+Z&#LcRY5i<()XmoYClr`%
z)c>#FCnI}d@9PQ0do@h{?`667e((0rM*Gaxez((BIJI|j;QhT#PCNH@{(ZbVZKcNE
zi9GT9SZ{Fe<0^I8`_O93Uj0L^d%iqu-1~1^-R?v7cJ}_+Qu{76p4&T9=hD8tsq6N*
zezM!EbIE^y)tt?C+BVnrF6UmmH}GwN4byfh`+q5~Y>W4A+xOAGd#`VR#=d92Htn^%
zmuoj6nsI+1>n)r4LS_4IFY(#OvrBj1DUCz>?ulI9n?Lo&?)*1)_F0Vo_v-9Bw^w7E
z@7{nkew%N#8~0WjoZOqadY%nO-@&~KT+8<{TrJvnQ)<E<-mV4vYV|bt22^j`XF2DV
z&BkLq`!c@V+*=>dX}5`S+TIJ-?(LP^GIxLJ6)}6y^@sNf{8rqj*vGv0+}Q<tt)^ez
z`%-(e{gqwK)>=0X?VWjCXV1CVioMyl1NRn|+~3P|Wa0j%*Npqu&5+pFecizRMRV%j
zS26ti0(IH!Oc^5f*PjaC>v^tXpZ)ixd-H{FTJv9z*|$z8!j^5p+Pw+(;x@nYl=f|3
zlCw|aQpaAK^SAAUa+UWfRTS@IF`cqscjbb;^&v|8e!p2``zI%RAFo}Uee$tL>)jLU
z_ii{WyU#nDV_#?8p}ofIK3R)ov+VowOKk7qC1!io+M4a7yq#@oJd*ZCWQFe8s`Jz4
z+>w)e8Dcv2)rGq6*>p#3pY6sR`x$!EY=c9$?E8Oo?|%Ek4Ez5VoY;5z)6P9Nt~J`W
zuZ-WzDJHp3qVNA+o{-!<tCg<r{i8W&FOSKpy$0E5ZGsLz+xsCzZSTn)&uy#MS?uFd
z|7`Q$c=5h-6SwY5Pz%{Bplr71p`hQMuDkE;-Yws-@7J^ZeRe|1dp&2n+s5=AvlICw
zy7$av9h*%P9QVyG?%T8Fh{Yb0m#_BzZRg%oWU^uJmy_N5SylenwcNSBuR+9bAH&iI
z``nJo?zOg?vR8r4ZXe&Bd3&cDo!#daSG4C~MYzq^?ML^jBy{ch`%hz^8QcE7svY0<
z_I1baZ8&#f&u6=Tdu4nA_dO3Q+bdc;ao^)V%Xdrt{AT0U-C!43YqXcGMQh*tZ9Vp3
zy^HsL@rd2`;X&cP<E_Rv;(zAv`(s?b_tJ@rd&}(Y_r5*-bN~09j{6S%Pu-jGYqg!d
z`HQ`CXB6(6UiNtJnU2N#?wcL3m7ZX-uR&DEVdby6`&VR2?7NVrxbN<!#C^5@t8Ell
zvhJJzP<Y>}xxD*s^<LaN<Li`tPcJ>)YvEUBTa>lL_Gb0hJ!kSb?R$8VY_l>&_ib3e
zabH*7LMx6><@+`UJMLSv`iuQW$JV{dx3}$W6Y$zU=ZM$dn@1n-S*87I|Hs$qdv`Zn
z-gCR--rk?$+xBW#Htr2u#JcZ8@xuLy3$E=BInTOpR?EbF`*ttcD>LEr-h~>=_f;&I
zWB27b_ukY+>ud~qCHGb4EVi5SE!ZaMg806kpqzb|YL4zpOfTD0!Tft4_m;PI9<>_#
zH~Tj3Psx9~FUNh#KD848`(#Tptd&bW_hqwi?bpA{vv2bv1>55*AKEdk?b`d1!`V)=
zCS<RE<Q)6*D=qsDDX!d?H=Ean=Vr@Zle#~9s%wwiNVT8YyEyUk9!}o-`_doj?Dbt|
zVt+%6f8W}DNA?IC?X>&y($LoJdV#H1N#ve_+^u_s^0fDC_IhnoWX7@g<DbU;#~ml_
zpI#xb&-Bsl{md@Cc8p!y_Fc?fwEMkx(B6OHO?$L^-tA@8v$M(m{Acf#mDBceHQDU*
zvMJl!`{>tR&6me*UYz~7XI<HWeL8Bq`_?8h?=xZfzW4UlnEgV&$+kVkpZ4-paqSD5
z5x(ctET#SG+f?nidE{)j{S(^Hv$=m?TbZxzr}=d@8={`>Dd{=4=l$<Z`~Ee_*$VzW
zu~++D(4OO*^Y_>n&9^yIczkc)epx%06oGyF#XsA%O|9R1v2)(uO}}OL?+*yGy~uNZ
zuY_XAz6U>h_vKiI?UtBRvG<up?w%VDR_&cvxMPpr>{t7=^40dud~|s4>O*a|c4x}>
zFE5|6FKb7sz39f3_Ir9H_sw|l!R`r@``)KNOZL9x=-n$+`+QHpKK*@7$KUT=e9V0B
z)x$scGM{DNH~Yr&y=8MP_ZbR5-QRUsbnjRG%3a@2>+M;0L&W~Sqr={Ri=y_0uRF81
z@%Q(=6AKsZdok&zO?!9e-kgOF`#y(D*?pZTxsPG?guUsSx_g)|{@K?ypJ(490mgkn
z68!ej@04uI?>@G@J(0&gqFZ;5)cWAP>>nQ5uI@O!H)xZ_{(@=mY?znNus4X@ZP%#3
zbl-8t2Yb~X&D`S<Be1XLmBPN98q>X*jVEl^gtl7spYONd9#~_4=&G~ry-!?r^?$?c
zQVpK$`#ZVU{%d&B-u(d+_kLr4yw~&mK70MD#rq$`3)_EIb=*6z_Mff&f!}*i)_3f6
z*e+;$debIbmV;;aoSeCSkNvk@`&HM6?px{Fy!W^7HapJN7kh2_Irmz5aqj)Nd;8w=
zd%E{5ox`>7^MPk}7kIR7AIB-|jdM`3GiBVncgB=;d-byf_t~$s*sECRx{p8k_1@;g
zr)@sP+}wBd(Ue`5mUs6qm#DDI*fHOBSDDd1=g)ubT|DRPOW1MHc3DF4KKYRLePO~^
z_f32BXrIpZAA48kD(vm(TxOfRL&;WJ{m}mIB>jC3at?b>J+iP@+$pqgcaPq_{|$@x
z`q)J5z2SMwo}u4w@3hqXy(x!-_Jyo@yk}W;?EbeWGwt2{MeWb8tGCq>6x*NrQe$u9
zUZ#CH9eMjLzEoQ??>J_c^+IXiX_3GC%56^XTi<eSuYHlw9?|a__SMBpZEfOD>{FLK
zvo|~D|K59^oA&OWv3+l^^Q`@Cr{wqjId#!KS#09ooSnM+i;cT&Q;l}n$p04J^Jk6u
z-Zl9X_D+~SVQ*2;%-w6smhJsw)wn0U{(y}ntL}b{y;A$;W?kMZ_{QG;>IsRxQ+&Ml
z9&|rwSNiOfolc?FUOi!1+kMTa_r`W=91uS`)voSl={|<}5BA=w&)63wsbj0bqHBL~
z@xOhlx}R-1zg*ro>7V7k+9R*_9#|i5^K{>Lo7IPx@BXsy{l21CuXY<*)!EJ|V%&GA
zqj7KP)K4}C-qzcGQsCKVp{BgAslRRChJ|bQMph*4{qK-qH^X@QUX3@6_VZlN?A6Om
zv0lNhWjkw+r~U0ziG9*4C-+4EP1@tYx@Z6Uk1F>6?WXTteq^gnv{0e#Lx$qL0Z;h$
z*S?nD<Lz>PZ(4q>?TUvJY@+Qd?O8-LZC@Ys-XF83!2Ya>$=-V9L;JE1eYKg+zi}V0
zL-#%})xLfIHtw}~z?`+WKc!-C(>wls7i5I@>7+>SSNXJgFKZ!(U7Q1tjgiQMeG<&l
zd)%`555#+1+`GcYV6VO4qP-C{fA>g4-`&f8LGwVO$l878Wq0>0ytA{3s4BDVnbW)X
z#um1Hs~tY>?eJjQoBx0PUTfX$d$YD|+<W9gg`JIK+MWr#Om-_aHQ0tYFSChQ)@S{p
z#%u30UKQJED<$_`kMP@XUwvyY!-W2Qx9>69Hy1s%+4-r$Hfx^yUfcPWdzO9^->0K!
zwNIo-#P)}!^j=v_*L};Hj_hISSz*KbZO7h8$G`49`Iu>6Y4+#6xnJt`aR_JF&fqK9
zXY6FO?=_3!K2dGIz3pdq?$Om+Wn&}Hw7-b`&E7RSITnZZ{@iQB%D7*@{Ni2-={)<h
zGyC_wc`jnpSh&gV7>o42nPsf|8ZS%k{eNG}J}<Ipe`NRGeOXf`@45Ei^j@yGjeE<j
zRrgs`iS7$}c5bhA>g|1Nt%CNN1zp)QZL`+CITt?c`cS&Xx^HirO~YTseZ_w-*hHG<
z?{!#lV(-m=T>Ciw#q18?Zr=CK=+0hE-hzF)m526gJbtq`yt&JIyY2(K1@F7|aq`vd
z`|zP+|GQ~7cF)Y+vG=m%E1P8@6Zd_+_I{65in878KlAP1>3`oV{Q1!S`3uBswT{{B
zQ#vKIuaYa>ra#wzf121#+wDJC4n()^wVN2?zPG~ftnK-_5Zj9{ocHawSiQ&Tm-yZV
zjOX?}UtYEMSCGH`S}j}aS(#k>B15;^T-i8j|E-XBcH3np?Ol3r^3GRki}waEZQ9E_
zJ8ItrzqPhO+>`cCu;ttLB-nSK!(4ZJr3GL1rcJ2aQ`~jW-tJD!-dZV@oqOk5+Ftiu
zYU^|+dhf|HDSMy9zk3oTCHKXYAG2Yv_-?<tyK#?(_{qKDfoJyXf5@>vIp21VxwzRr
z_HSSI+D-M@zuwT_X1^8VK7Tgvee8bm`|Hlk*gLCrg<Vl{)PA$oU+msU+}Y2pEopZt
zY1Ll&0*8Goc5v_Io!_^A$=a5E&Wpb7y;JdTPm|FdyAJz`y^aDM_P3tB-0M~xvyW$S
z#lA%eU3OJ=NA_+l6WsU2v%yBOj&U!Gz}dYTl7V~gHq`Fp>3g*&IH=Ly{hHg}(rjhh
zUra~#ZE6?Z_atoYUXLqXHgc~v?Rmp)vsY=>cN?pF%l8;qo!z&rV%t8YqQbqJuAld7
zexJ6#_gSSi%gvd4LtYfyexCh!Z+qU^y^6v+_G~jcw^#MhEt~n@SM9rZS84Bt<aFB)
zk1BS*Wtp?z=;5q=i9aIu={wA~x>NpfZ?*freahb@?L~4r_o+7X?F)J;xOaB7*S@B8
z=k{mnv+vFNw#MFeKF7Xes}}4P%4gmyyu4?xW@w|8rNfbZieI?*Uj4OvZ-sL3zK>~X
zHhX=h?p=23+TM2M4tp`db9?vh%(lOlxM!~p<GsBPI#=!6`R9WDshuh|nnlw4ZY12c
zae5(bBjGS%Z(rRVTh;JWwqIRl?v>CAvD@`%h24@{6K%g+AKZIO_tE~x+w%9-yFT6j
zy=C{_nOSRXCQn_zkL}9-y{g_5_Ez}t?wj=Nf{jeM#=ZqldTnmVGucSk&9^S)tlI0K
z(X#7@!Hhkp`Bv;(U>0Y4YsPHbg6Yw=)>Dt#F<8jiRtU}BQ_N<$cgeDF+XWSKcew_d
z?yc-_-`g^OhwYSz8e4{sGj@h~ckX3iO5F2jTg$#Bf!ww$W0P%EA5FJoNV45EJ2z$z
zqjUAXjRHM;y3_LaE(p4AcXjU}8v&;Mwp+Iz+{@r!YNx&E#XgPu+_r8St$SDORJFDL
zHEnN}S%U34chP+h{8elOl<(UZZVa+}{eZ<jWDAdN!^%yzqV{L)K7W0@x5@mRoxtH}
z8>an}_x4|vwR^gR*?yYSJ{t}1-hJVWD{MV$*V$RGTDW_;uk_v+v7LJhWTb7r^sKRc
z{%ObFX_gUo8pn6+oe;QjkD1ZUecyCS_hx)Qv4?+u-`-hg5AJ<nZ)UxcYs=mT$v5o|
z9iOw;b8o}m%pV{2Dx}KUP2ihmy<;VtjbP-Nz329e*s-tf+Iw`<p}l`^`Px0Xdwwr(
zB-h>zn-=a_{nNxI=j0mOxAW%Oo!w}=ciM`@d)ZbjvbC~%yjP=P!rlh$Y5UG4NZGCl
znY35#OZ8rXeDgi`-NX0w+-b78t*yGRH`{jKhTbK%Iulmy&AQaRcfr}K*0<K2vg7eP
zxv!v(Z|?%5E!HW2Lv3sS1@C*bbLPIE0v`JsqPE-K66~<sGIh7@R+g;271!JM@&B8@
zH#3^WKH);4^^w;_b`tNl?z#6&XYW$x4>oOEzuARq-QM>=Zt8B9q)yw6No#EzZXVhz
z+wp9#&ojS$c1O1ET^w?KUt->;y$k1w*anpE*nO?ebMJ#Qj=R4*Snex*`g70ogWUU~
zE>7S3<kHqX>{{(MC#&=Jh92E)$0*&ncjvdWdn25`+BL2cv$YpGw0Fn#n|68^?%O@N
zE3o&#lm|8wx1{V1Sk!Dc@#IFk13C9?E;;e+RcczZ_uDdQJD;sf_C1tM+1qE7vrlbX
z_g=o2b8J_$G})AV%-!3?Ah5UT*CU&=Ztb=T&$Mkti|*OBbgtgR#KCKuW81X1i}AqT
z%x&Lo*|ghjt)wjWPD<{#EqSnfuU=8x-YD6Ky<b1-*<F~pd(Rc=$$M-vzU+BXD`LCw
z%j~@y-p$z^eRku%^~WCXdpc$RzE8Jb?MXeww)fI{Uz?qpc6)=p+V^-oKe+cnz1i--
z%~ST?%KET(gQK|p=EdoI>x201&!lAAbvowkonF;#SF9Cg`6EHjR<2jiF6r4!+w_g9
zHmnMB_NJ|Dw!P1#vezzqtL-Aec3XqfS++9Fn`}ew@$EfzbN}8Y^)|NG68G#YxUqP5
zM698$+M)P;<*HNn_Uuu#6}MSpr_eTW?}e3|`=?KuZhLha)4ofK7VgRL(Y6tI$Y<NF
z^~A<bak=e^FpE86rr-81ef7}J-!6A=#9`JwUru`221z#WHIiVo?ci>-U7-7T_lnic
zwxN3a_O59RwzbvL-FrMG$4=txGrJi&s{0)BZ`e#wcw(&~GIP(7V1~W?AMfs+_*=+U
z#x7tl!}^-t3{~Q`%bVuzVbN&a!&0(zPpP1!UAfiKy&O4hwgOA9+sx?HvX#)6u={rO
z#hyPq8Ex0SG}|ZkN^0-hrR(-iiOSr)US{6j10jF*F7R-(W#?+xx2%1k?UoK6JDF=2
zY~>8L+NyLd+w*be2iw^ljdr<T=Iq;cRM^(FL&?tlvG<-Ghkn>_8_3&nSuEVs^y7=I
zP~Nh=FQ%TfT_Njl6XrW-N5I<nJrY%QwnZ1#?@f51Z|m@E(q50kpEe0!llF)fGw+>c
zt7MnMxO}fzTlt<P#TRXhtvA>OY)G?x_h`o6z?+Bm^)6XqJMYM4+mG|-*lJ1!?!Bn>
z$o5>$p}qPK4()aLyW7Uf{nS4G<c57uBzg9(-Q}@=P3zvhK1ba5GW<^3C3t<A^}bVl
z`|cQj+oQa&Xx|huCA*UQXZOD3y<`33X|HW(;fuWw6x(fB53ua*F#T@Buzc?Bn9ezN
z3KjXbc_mV|3X5mi`afyj7gr}|ck9%Iy-xyaZOd0r+k1dr*XFyH%sx%F^S0tEy7n>D
zl<ta|v25=R*C;!V9p*L(=U?n@cKg0h>9W|~%QGigwM((u&H6NR@0GJJ_N<!IyYJs;
z{{7F{5AE3?s$pZA-McR*bMBr4$|<({YnAP^zV5aYTB^0@QP`1vN9<4RZCkV0_CwPr
zn|BA!+umsIu$eIN%D%d}llIQxe7{e7ozuP}8;WhkOda<fJU7GkjX|Vs`qBM+%dUyr
zt^7ZGUt8V%eNWX~_i~vx?|t^PcVEKxUwbBBU1rA@yk>7C&+2{mUftT$xnh#tUEf!G
zqfUzNopg8aUOA2i+Yd~pd#=2U+$-=+*w!F?<KDP4-uo_aZm_v~#CorQxuU&DqWGR2
zpW61c$+PXfrF74B0mpP(h6%g&d=opc_r1$&+s7STb{8fcwBC06g<a#(d^?+L@qN;d
zcH2J74BB_O=gMx&*?+BHte?2oK+eN<PWzKRI>rla7Hw&?6?h<HyGpao_G^5v9m`!i
z+XoU%d(U*I+R6kU-{Y!3d7tBjIy;H{S+*7&^K6AwMECyv{m1Ucg=PEBq}ba2N`1XI
zWa%o~JC8kW|5=3CNGx4o<5xCy@BQg=cDiqc?HHU+?$LhQxwmk4(%xh7D{Z&^oWD1;
zY>J)FDdWA7ixuqceoozcp*hI5jIq~FWBuBFrvj_@&WYS+TcYuNkLFxXTZd`&dkP!X
zY~5Ba-+SfGPFn|ev)$fY8FqE)vu)4q<=Xo;`P)8+bD6eH*B{#%J>G7&qb0#sA@QXh
zgPz$=yJoh13(gnVNStrk(@+v>9mjQW@4WLudo`Cc?<@GYaCbn-t-bE8oA<i0McTfa
z*|9g{sgtegxz9FHmTGqK7t{B?;h(m*DV}woe?YbEzSuu|1AdF{dr%f()53IZ51Z8n
zn+F_fHYRcQw$m=C?KNMQU|VLrYH#i2ZMF?RXWMoraO|^}Z?`#_^>*)y7rXcB?G@R}
z*I{gz(JZvLKjF#VO-<kS#<_~@%@uLld&Kqpp3`+nd#m?v-^=q_dN0EPxjiP!TWlx%
z+P~NPT8Xt+;?cc2dw1@&KeS`7lIPaFGj6Hdx}ION_t&~g+kaPg?-eXNz1Lz>*Peo`
z7@OvL>3zPHb8R0bFzz$zXtq_VuD0dsIl5=_mEOHS_Fmq*g`t0s+PliVSHzC*U2ayp
z*C|MEU)1>m8-b*IHtJU$_eP)GVtuP($KDHD{#mCtNA6|xK4N=*-4xrbP5pb5&8F;K
z-p{)4o%x5oEv8+2_xxb84L$eOW{>#8J-7L;+5A7hd~Zj!*dF^8$M!VoMeH?Yo4beq
z$p@RM=>|6cv?tg$tT}62_HeK5jJLWr4PUfu!{*5DO^D9i`{PQn&HN8b_8wXBZBLwn
zj@`LW8hgUerR`OC=xBRw0>j=X4c2?pYU1}E*wVSzMncZ^i(#QP_uNA^B@<otR*1&!
zS;5I*=XQFDt%BEfo2~o`d(YqIwPXA_Z||<N3VRJc^V=~@yS7W@>m}Qzj0^UDWir^E
z_gi@18=WJz|5lprUEN`@w`h&Q&IOT9yZ_AHXY1&_YA?ezKbvWV{kyNN*V_A=b>Uu%
zvVuLyhn#F|Y;$d%bc@@1JifX&+fiZP!8KcKws=eL)p*HmD_K5k@1!TZ`{po**%%aS
z?KyKi%eLW3kj?Is6ZRhC(6^hs?BibX6WsgWy_#y9a_`NaYbVa_G5R&lmU-$5n*i?>
zd%7<@-Fq!+)86femG<)75!$=kGJY?&>$JTc$`@@4+1YH(B@Aru@I=`%uuk1O!&cSy
z)V5W7C-S!Lu1RLG(<r{Y_k5*;^`k`Tz4Iq-vgOFAu&L|Vu=mH*DZA&`$J&ahZLnRk
zZRVaIs%f?liznFr@|kRVRO-pzyz|0)dA}am>kwdMePsPIn}lmqZAD7w+pN6u!#1``
zcTd!{7q-@CKiYhf6R~yF%h<c3p?&X`LI+zrY1h5+&8PP!<aOB|Sul4`?VpL(+h3;b
z4R}1yCM;>LHT!KIo9Cq>`((G*+nQ}ZV!P?|n!RjgNqZHtRrU#<UANcAeYZ_33(LL<
ztF>%yiC?f4$u6}08|QADY;k<g)RtE^r|z$_{e7!t&xDVaHg?^@dv_d`w@oWZ-rIL*
zfh|w%2HOV)5B3QEd1<2+&%Ku=PRUmLm$GfvuKjy|$?vpPIV`hx$3Z*W*tQdUBbz4e
z{rqHxt(H8GO~$9Wdp7)AV|y#BWlzuqfxQgprtj6hX=+>I{dZ5>KHa@uQN6Z|ZfyJ9
z1!mYPKGd-DTQq&|EzS156=l7)>3U-OYNyTFdqwHto@q1J+gS85*z8N_*qh;Hy{E~{
zYp;KB{a%MzzxO;zy<%IyX=hupY|&mD)Be2^e^=U6@JsGxo4V5`VUhlBMvbR?X6c{W
z+riDc=aS{sy@?-{_s$VnWBc_q@4mMS>TQ=9EwFVCx83{Tzszp+3YEPUHXeKHZB1+y
z<5>50$!*^2prmcfdwaEQo7&esZe0DgMr(KORhf6edg+oMdwF)|?!9_p%HG%SZ|pTl
zTChhTvU!h-ZS&rM*Eek1^q23oI=0<*!tc{IB`ijJxn{hz5sFf=Ezmh<Qx(u)d+ci8
z-U&kMY{Nse_OiY0x6QZRV#~bvhOM>vS=+a!eYVzDR_#7=VY%&syW6dcr*GTC!YjA0
z;6|S<!zTk<U*n%PQk^=s=i(S`85GQSrg%@bkv+w4_jWmho$wOVz1j2R?CjsN?_=NY
zXMKFxguR^4X74SU&1Uz8V}bQP;Ut^6c9M3jZx8I1d%(GS{_cyma&z7GGOGyMt&QGk
zoA#`2Z~Q~HeOjEYd&_^-?R9uFaj%LVgKd%e8k+)#K3lPzy?ZV!S+yr)I=`Lh_Wiqe
zUie^ZS!iY((S6Zo*Y90>vmB=HomnfhPb#x;ub;yN+h@N5_kMS4wq0Yf(stQ)i@g)%
zJZ#+C9rp?^K4t4_6k?NnU)i=b`~03WndkS02hOzJ(6V|@^^x$s{x^d5h|45cc`To_
z_p^?goz6Bb8)aRwz01$Z+a=d5-jlIljcq}dg01$QyEZ9%S@)WT?cAfBE4TNa<aV1E
z3-;}mjyk$`q0w#IIKRfd8*Pv7Szk8O_W#Gbwx8sZ_Z(Y)aqog{&uw-#B<<ah(YVL$
z^PIhpdREw)uG_sg_QCeO6?d20-Y@xVv!gu1=I53>wmPe8ZH?k&_OiRa+v{b%$hLYP
zi=DxpQ#MhV5w-;p?RyO5TlOAbyUn&M_NDD{HRXNYX?OSfY?*4Kd#u*>mP!0xrqo4y
z|1S60d-<%--WhL?@3qO;X1k!{)$Ua~GW!Hxw(ngtZ^d3M`D3;P_ha@hx;E41z>^1i
zdu62TLRf`tZ(OXjVKCF)X}o%^ZPRkqy+>He_O=`e+k30XaqlH(DO-c^y}Q}&RomQB
zI<S|)P05xquxgLqH^sdg!rC?==O^0mSl-*SvHJSnVoBM3d%h^zitKK)T~vE&FMr&V
zy@e0C_dI&Suy5MuNqbWa8umu{x7hA{^W4_6c*<UP%XNDnZ|SvVnCW0WqwnUf?lZA_
z57>XTdN9#_Zwv1Sn+D#--M+Pa`|h#$?Y(0oy;nhvZQta8g1zmJKJJ~}-Lz+lwyf=j
zX<KdQ`_=9}k+Ej)G|h#!tEx8H&e`&Q?}U}pY<El-+T+3Awx_)E(e59mPiza_me^*P
z-`TTd0jn*;(ffNXH}dV<A%1yJq+`lnqln|S4EGIfl2+ZeJ&=^XTUY7e-VH%hZ3DMW
z+56?Y+}`PFt$U`}@$Z#){ckgEz4jjY%QyEPcqC)P_<(Jn=TXVM=CeHa7C#N$yI|UX
z8->NIY(8|?*f7i#wPtuAXfx+=*It9zZMKi5yxPsNi`mXXkZX^>*nHbp)^>Y+|A_Bv
zV6xxEb7q_E1BD&7hgUGzm~k-eb=?)ew?lI2o)!E_wpDqhwxZ_xdnepJX`@>3#U@$%
z&fb*mb-OKs)NOh>m+j36?6Zl9^|W2O^xfVUS{yd+Zy)dZ_V)XpV{032Uj$Uzo=n!?
z%f8Idw&B0|uBx8dws-b*+QxgHx3PGvuq#BWeDA)My|!6?-21Y(G3?#*bI+cc%}q89
zE05ZAG<)u~y0m!j2SvU;*@5Twmil|!-d$vD`@*=*_FpCA-U*j4+3xY=u#MR&x7SYU
zsBP@0iF+F=HrNP0+pxDm=CO5w`3su`23u?kZkO(va*1z`+s_%c4QFTXWvde3`{LeR
z+nBBmHXp<`TF>a0+V}O=FWUo?X4neO<J==I<GEKOXvbay=E*ifO<QbFJQUb-V9Qb4
zLR$vgE%RquhuiPjt8(Gf9uKjWJv^V*+5|*C+*5PX#kO?Oygdre276~{9o>6oL#A!m
zoK?0}$y@g_>^NbW*c`cA=KPgCY%dG<&g#^#o#VE7Z|=cEd)F#7TF+uv*r#&k>z+Bw
z{A`<Z4fc8bT(gHG+ui2FT@9ND;U8@ZswVB-`|R`H6^RZuU%oZ$6}h{1Z_0-N+bai;
z+4|PL-_vo*cuyX`qFuVyYg>l}>U*boRPL3@vap@3Ex1p}tiyIsqLQuL-gaA$znQkj
zrWNe{5!<%+(!+JO4Y55o>u&JwQF3b7yX}em-X9m5Y;2l3cZW?|x9fLdt*we@u`LHf
zs!h#}c{XOhx9>4gceTCYy4<#T%e1|LqUZLm;dr;l+w1h+h?P6`uDbrvrY~xutxoC*
z+f}@r`|eEdwrx4#vbTys+wR%L347o5v+v~%-M?4#&JLSZk^*)G*Sc-_Sz2w3>N@r&
zpQzq@_v(qg|J_dR4VkEByT$ap?Pr;ey#ij*wgxild%5n<+53i>ZJ$uwwLR~Bjcu!h
znd}l6cJJAllCt+}d)(dvM`;_5rk#5&+O=%gFZyDeGF5f2$EohUw%h0I-T$|F&z0Yk
z_f}YP@70Or-8(<>p^XgJraduUEqmj)YwdM7w9?jK`y-nj4obEg_D-{Yu>IC<fv8}c
z{kPrsUYg6gmuc&ty;~)U_nuMOXj7xzVB_z1&Sr*&ne9ZsD|^r0I%X5nB4GDt)|NdF
z%((X+e!tz?scYI^9><M)zw~+8&X()n-Ml1h@5B>__a>+xu(j9UxA*MhMSIm|Y_d(e
zI%jW0N%Ec#i8*^rzMI)DEm^zQ!N|mR30IPB2;1B}LKO%1Y9t=A6_8uKSFSX0FVBsR
zJva9p+Z*y#&(`Rp!`|A2|9c&tueC0)J+xceRdnCCS?$*Em8R{rXyMu$VY$kNXPSkr
zNv^Z4<Z{owF3nbZh0}iSO6qB}{cd`8uj(|meSg(sZPvu_*iGR7ZN0Zzb+5{ihqgDL
zG22Bf=CED7o@ZZG_whYBrHA&Od-%tuwt~y{)5elLe)8RWJUXY_*eNpXol!U6M#F0U
z-kjqJwk(~UHZk`8wtp6K?^bX)zxRO1ESns?@VyBR`)yu^H0-gN#lP1`{($Y*-Jh+3
zBX-$_m73U?ekil46JB8(oBiJA$!zAmo@OnUKW{MXdtrLsW~RBc-6P)Jwy8yD_R4H{
zxi{~0zpYQ)F<bxBs(agbrT1xEyuIhewIzG6c6;nyID^-A))oJ~tG*t!Em${guls&s
z>)nD~d!K0X?yKQ7+1vAe<6f>g_xG-wpk%A*xpmJ9N98>)?gZ~Smpsw-l|k^{b!|)c
zGHm<3Q)buNy>*&wc3y4sY-Kif+D=&i%Z4rDm+hJrORcBvnr^*NBVw=bP1$`rR(aUo
zdn95b71yzMul5dGF(cKz3W}k&%hNa6?z9lIJ5hDqcGsh&dt0n{ZAF?+>|v0Q-S_p&
zqdjf~40c)@PuX65Gs#v?m~HPIX;s_(8Q*P8&$!xro87o~OGkxmoKD+b-*db6JX>|r
zcJAz1wu*@?d-aOv+eV*!ws%X+sy(la7TUgQmEC)(#(hs=OX%JeH`#5sF&gjHDLi3q
zo4a7|y5%3OmMqKLqjBx8O$qnZJq!-4wnsR__MU5Mw2hg&(e_Vv)ZQz{G;9l$?(e;-
z(X@A|Me>fkXI*<wvaHxUfpwPc@^csWF4=W^ugCf<+lGw8whqlIyIbyZ?#pU)*jwxQ
z+9v769@~Vs>unV-dhS({7u@wy%VqDAauGY#xH)@&{I0cWJuJT0aq;Kf9GoBa91J$v
zdqDVsja@kNzIidKdpR!M+|#i0tnG<ajkf2%nc5!OvSM#fmB${NkEXWQ8WZ+zh`MEy
zoO5E&i3^2$_5T*^efso>EyJu|yR$`R?bYS#+FPUDXPXukV7toizV(OJlHEZ)etV~s
z?zC0E_rPXC$W!Yd{<8aGIWO8CcyZ50$1%^g>5Kf{SqJTGCl|f4ji?ddw}a!cjfJ<;
z9uLRZJ<{#`wmBi|Y&yOt*@zvIvU}kF&}PcMx3*tyS?xI>Jl~dM;U}A;X5xDt^k&)`
z7g_FI6TZ!M#bQp|=dJVh{7kX3>2*!p`yp9;U#;_`-5LL@Z5Zlq+4Qt+uwB0U-CkDj
zhP}PB;`esBtL@v+By^x8q2G3EjHK<UZI}1Bw9c~+d~tH`-^+FT`idvn*c(;as=C?i
zW$q8!E4c2=?!z^q`xGDEu<etcxOYz6)jb)D?Dl3qy0c%H<;~v2F4p~(Uv}-USAS)_
zdsY1Y4U4br`;aGQv$<J!kI?d|_KK1#_9c8jywBA8jqUaoVh*2s`}PMeuG<$S^uU_;
z|5}?B`da(;o@3v4=if7%%LVebt{*w}aj;#sf2DhAe*mBO-aR#z`*|c^*!3(tyYD;y
z8++CKr+elY?c8g2Alr7~j70nCZ{O@Su3fjk_%QPxK?WiF%0SV54-fy^qj6Z=mgyPm
z-bG>^d*}3<*y_CrwBKJGv+vznhJAA`FSI{A&(B(PV$j~CRONkPi9h%L*mY{}3aP_;
z=V(goKj(dJub&nFzBO;F_T6-@vYE=&xmR_i*S;hBLiSGaN!wQ|ZfX1C;MBcs;*0m4
z7c8<p=p<@ec6Ik2Z`-3*FTU)ud8s0AZ&mHHZ_du7z3qbj`yRWR?Ke-HV)ylo>VEdW
zrh5c*kJ$82n``|_EYz;w(cAvz8<Ty14deG+C@!!+bnKsv>s9Z4T$L4j7vK4``>wHs
z{i2Ec?O$2m++Ff|?LMBc+P!)o#q9#VWZB>BXR}}A*}ZrFJzj?geG~R7C+ym*x8?AD
z#rZq;`cJE~XAHc)cWKqq{Z;HQ_IW-Qv*liwy?6HiK6{f$n|*7-Pwabh=hL2&ISRW<
zq|WYDh}yR2&h5+l1g-_{?c2rfpnIa%PH1EF{<|Rpwub~*cIO<PxcBna<p+e%%--u>
zP`A5rQIE}^35qt=w}kf`^4hm|`-AQK|2@~-mpAYFj+a_acB=n`t*<!s@3rlBvuj;w
zy<a0ye4i~x(H@5)gFTY}+4fj#&a?NNF>jx5(R$mXx77B1HZb42p!3@PxKfw>UQJu~
zm$97QmlTz>Z}s<idv`yXv6mrk+uja)gMGS_9PN7L!}bP9mhO>!$+J(}IoC#F^Y%TQ
z_kwqeJKeJWWMQ|rRIh2@dcmuEJr~&Q5i|?kYooDh?;FpZ)}La|TbtCs+d1F${$2%n
z7JKGLQv2@fu<R>+Qnugcarb_a<NSMhOilJaebv2Z$pwzRRR%SCLVqu?cPU-D$NC@N
zzP}Da`yTn<u(>eph<!=S?>#-r&-d}G&e(6eP+;FSgTHpMM|*8vI9=E`|M{MMYHd;n
zQh4v}t>2Qf@3e-{KJEpcdvnAe>~-MQ+;>o{a_{m_fp+Ir1o!gh+}?N1mUoYVUeA8p
zwFma>HPznxv*_1;SsB@VkDgw#;f{>j-|n<^-?{b)`)y+QY>(|cw6{N3a9^&Fgw3nT
zTzmZl_wAj`x@PaUH8=N~HH+*Gl;d&Oy!PInFQ(=;zXMbEw)?2;TYGZBo|LXl`}`ca
zZG$FF+~@h@f&HA18vAD5E8CZ)t-U`wzTK92f6H#;8P5B@Z`->^O3%+GLXCZ&*u06h
znQzbT-SeJzubK|?-W`luwx@3j*}A@O+$#`$b1$cx>%Pi;v-Ymop=i6W*?*s|MejaQ
z(Tcss{7U;2`<(U)?>)Bv&yk}2?AzP+Nxop(r@Q6o-ldb3_uI{~-E(Xf*M4QOsC}wV
zH+P?$;$#<fDP!;Jm6P_`eRH$*QnlP4r@(aJfY0Q8DOH;L=I!U-CwF0^P5-HN`z$AI
z+;?sc^PZ`-zxMrN__XJaL4fVPPxEX#cNFZ+krdu1`d)LNn!|xTg}R6LZcv!8JN?H=
z>poVky&KLM+CDZf-B-iNxi9v_w7som2Y2sZ{B&>E>p6QHdwyB{b_m~h;-KTc^QR5=
z2c6x!SA1diK7lo#_j2@{+#6zbaPQ{p)(4b}SJ@wD^|i??v$hLSyu5eE^iBI%qtDy;
zhaB7c{ww$X+S4=l>YQuX7bv`HU&$Rs+t=C^d-r`^v}f@h*L{B{+uE-3o3l4N_U*pf
zPhqwdp*Q!<oV3tp6OX#>`}ZGq&uV5q;Len4f9Bq{J+-DM_cl#mzt3{-YU}5_^Y-l*
z+F?6I>($=Q=5u>5e&^Wt`|Y%S0Ss+><!u=Em7WSYaCrC9-7b5S?Y>^PxOZZ4$ewj#
zj(e}a`(}4YdCmT9e$#ETq>t?Va_{M0{Yq2Y)~FME7ELbLzoqVx{q6OS_Zqk;?u$NL
zyzkozw*9;tLiXlqtL~F3R@j#pu6iKAr^KdaI@1ATzrFj*WOMi3fB$Cx`P{z!VYk@r
zn->=B^FC>~hkHu%KDLrg`?ss6?LClwWp8!V>ixoPO?y`azuvpg$91n$nA!e`O?Gx6
z1%dlBm_F}e<UhOjO2~~p<{z7EKOCL6_phI-?eFl+eJhyw_fF^0w$oU*X7Ad4^Y%_?
zW!`r~PjH_?!x@`{d6V}FL|w2h_#(Ym(ec7wr%xQV78$bpr!W`U=a`jPv3ED_XQ^3k
zpY{Lj-adAdz2{!)*?%r$+4u0)CF>K>i}&VjcHDbi=iQ!kxAJ|j=eAkhS+s1gXeZyk
zCC{(z=S!Ql?`G+ry?2&vu*xit-s``3v%QnyCHr+dGxn8F%G+l${hjrv@7?<}&s^AN
zZntS~o?){6t)E7FH3W6{G4%58b6{9+^Zm5ZK4tlD`~G%E?^*LQ$%akq%-(m+oO`qH
z{Mq|(Ro`A0#`^tQ%T)KgFTQSHYsX?Iqs6`_G{$nTv~_}AsH*VZo0@X_UTh59v-z)x
z&EXc6-MZpm_Da}w><x6}+#9E$Wv@7^a&Oy~j6KI2PVJpP>)gJF`p5TW2(Z~7To&N4
zj;CwyjkgB-OsaVId^r2oMriZFy~i`w?=5@sX79%tqWeNF9pAI??Rwi8Tdnu#f9BZd
z$<we`;@Mpr7n${YUoC!Mqp<3Nt%WM@o~z4r_kNfgXZtC6!Y(E^!+jUGG1>aApKr5n
z+WI{f=X-2ftv>D9Q0!`>5OjC<j<yJ!QxXcc8JF(cG(1V)tK2Va=ldsR@9KknwqgN=
zdn0$U?>p7bW3$CY!#2x5a_`@4)qU}&1NMGvQm~ER6l{BE{{KA|`_I@e;MKL=&+5Ln
zc5bchq20S}L$qe>eYc@+ubS7!y}ZZx_8btd+LQ8hi;e7^b9>%zezSMi@?F-4pK|Wo
z)4F-@iT3Avgq7oL5+6UbN@$(BS0uW7_Z{Uew!ZPfw(sL6?|t!R-5w9&HrqrCtG%zC
z+4m_J_}g9(&fF_|?#<p~K5Tn8#0lEG5c9EZn`3Bmi+7Q&gE`Ocb;p+O5nest*7nwG
z+ZuVVy$vbSmc=RzwiVpjdpQ=(usyvod#|x%&0b@jlD&M}x%TEfoMM~ZAF_ARp{`wv
z-&^cW-mGSOvpII}_PHx;eIMKGm5Ecam0Pf5@2Y)!_BJf3vT0-NvsJEA-TNWpsP%_J
zg}oW4eXT!V6yCc#sBzCs$v#`-W(HgHqm%YN+I?;>%c8BeVavMrwids(QQ}{3Ya8jf
zH{vv-O~U4VyA^x`Y&_SV+RL0+zjs3v(;n^#%l0t8xW6aldYtXFN<mwOr?>aSP55uS
zr*F!hQ`VBUpC;ViyCP@zo{WSydpEA&wEMNWY|n~w3vCOkSohk!JiPbH$$xvIv=sMF
z`N6wy$_-)Lq-#g^&icr+Z%tDE?jPLyZLhp}vG>pg+r8U<>ew#0ZEP#_^4wkp<%&H*
zhm-eSS@w3XSNguaoy~mv0*c?+$oX#GyRF9Ft|EtjpF;NK-PVy|wp%o3*nZ$--Y3Ah
zbGKwX*FMLHwR==2ec8)!X4%dKW*vL~ykE6vQDK;kxafDAHSH$0+{+*D{kgetFBi)p
z+qoS#_O5A4vNd}&eebP9{I<Ju^!C2V;@l_xblKk4>}J~&S@-t@w%hK#_M>o5_vX(w
z9FnEB|K{(t4ZXH&Z{y$7dkt2<wvDh3vpsuI#WvR9wN1r_j=f^<Pw#EFS!jE~sdx{|
z{f~RBZynlu*00xg(b*X`h0C|uGJkustF3N>WkB;5+kYQ~_Ek+jzE`~W@Lq<UD!asa
zg!Y8yO}D+&Vzk$AP3K-7Z42Aq48QgqjoP(WolV5{jL#06ON%XR&uv}2*Q9Ii-ZMEJ
zwgG=DZEcr0+Pnx|xc7|t;ynuW?R%y)oZj<)Grx`5g3mVThOBn=vsw0i__uQJa$cw1
zLM12n_LPV2GP$|SHt53Hy&AF~_oS?PzWYEc|DNwoGxt1st!-<tc=O(e2UqT8=xf{i
zKH<pT^~tAge*EFH)612%oh5(CW>F2(-UC6iY%_QL-ur%Or>)M#KHFbbOtuRgxc8n>
z5#Be&Xv-csx7&Lbm2=p=U4G71>pIuI6Z<*$R4hNa_Y%Wen;$oOZ6^t9+q!04-IJs=
z-S)+qeA{h8tM)EBsA;>!NOG^yD~-L52gL2>XfWBX`0c+(BDct<P9n^PQ}&e2*UKmO
z7F;>9`|(uneaE=B?fvk2t+j}b%U-6KE?c29>~>}H_iUDx|Fw1ac+yHqu-hi|{|;+`
z&tLa=1#|8*ui@V7H;=*A`QwtklTW<2IcyNJSBSI4wm6p4E>e(hub|eAJ#Q`*TRjg>
zwn_c6e{b9;g?$NcCfa@v-L;oN=e|vj=%&4%*QVQ)#?0P3f%)E^N6h`Uq0a*L&S2x-
zr(b+w@7~>;_ptm6+`D1Rdg~V=K6{J8m+$pGtzj!C^wf6$b(_6+PG8ugRd#OoRNeo3
zx3@d)6?A2`)hxSi6LqO)FTc+Rn+<wdwzb!^Z6(ZW_RQFjXsa`S?%uYWulF2&*R%6%
zGt0hp`{nGk-d?sXxLUq<%`DG7#|8H8-6<ZpSLpfqy)4p8Z1O{1*iN_}v`6>ilfA0p
zC3|K5a_`+#_kJ&HZn8~ls?*-}htu{7KCrMocc9bOn9*VHgItT<8>YUsslH~ix7_fz
zt!@8>z2#58+B)8xy?0l=h+Wo6Z(HT#g8PgL_S<e;FR<tQx%D>k%oVnV971-+6*V@=
zhqv1r&+FKon)A~3hvsJ+Q(2q6Hi!T3sd`wsxB8-hEx*QMTZI?AdxPdp-osVUYg_R`
zc<;sLc-!OZg?r~*%&|T8#>&ps^1AJ(xV?Mcss6U<;QDVX%KUcEk3v4%%hz_;HW+E|
zReb$>@2P`-_x9O8-s3L&%SIqy#`bUjGMn@T)_c<x@7Q!~mE3(aD`HQ}mC3ft{|fDU
zpmD-Rne)P4-J2)(w&*RkEqT0ZZ$rkpJ@4i6Yy))H?vAyP-q&dpzxT*ey}hh2IPDHP
z9kTH+`@HwVCrMj_iM}@1y4m*L$z<8PrDwOz#L0a7)^A#9Yxwn|b>c|}TQ^R<eGwe&
zd%bK=?k(KSyJx3E$6kiS?mfv@H|)*w&E3m%RKl)utK`1#>u=lq_&ddBPvK_U4;fB-
zlX6w}-U+?GmxJ}(o(I!e?KXHs*c5EG-qUbZ)M{3As7?6e?A^<P+IP$D-eQ|(-Lyw4
z_Kl5s0@q&d!b5xa{SL6z`h0g!>yN^{7Viu9I_!RLlYUd)#^xvEUZ${Zd*1)rz4wCQ
zed~SkGi_PKWA^-sO0de|ld-#gNW)gC-)irLkB@A;7NqQrQQ5G^L+`uIYV~Z}LOn&>
zZD057&1WpK)%z8_*Ji5Y-WkWFZH?>~?J)>b+%xfVzwI9%M!U3qQoGvJxc6;oJGocT
zzH9H^i%ol<v}@SrbQJCtIJwM*A(qkVrZ~Udi><zU!#$MuB&T@pEAl+PpX1$b+ga?%
zdoNd**r=?N-FNtP=HBkqx%)KUitasc!LV<C_0oMUGiCQ>6#UwoswccJ_8_y3QeX34
zw=~whH)=)qsebO;ul$U2pUr)t{qj8i`{&oP?rT3e&6er5mL1bNX4_@nQv3XMXYW(_
zvv{ADDBJ!MFQ?nQ5@p=K<W1*3-4=&^&X>IQWLs?5YgM>#Pe7FazF!wocOMh*w_O%-
z-X?jjo4s=AUc1kE=l0BJ+-1Kl;qcxsx^?@M-WS_vvrO7M#edJ<QlAXltM8obCBD4a
zS7s(`7i2B7ujKN*eV5ZS_B9-K+cW>_)_wEYf7n)Rez51)9G$&tJ2>|5nbEW_MIn9f
zxmR2F>i6|nf8|!(w<}$9pW$D}ednI5?*02B$oBF+TidD1U-!=4z_f4k{7$=FYKHr`
zXD-{Db;5Dq*EE~mb#@>2iqCA`+dk>~-U;8A?Ax!cVD)oe?B0avHG7wTW!Ss>->rR@
zk16jBedlf?I&an9Cr795bCW)}e{Jl}eaZ8U_xkNH*~@Invv>O4$$Nv^-0i-*J=yL1
zR$`x5Y1zKDWeocRx+-n$pXTi|+1+HzB44z3Gl#^!oKD{T%MPC2J6CDbzO;w2d!-JX
z+0X9pv;QRPB0Jru^?O;S^4ZsYU$JL};io<Sw&>Vr)<4{9_T`$5(!pbUzw5r+XP<O#
z-z9@}HlZ6%>`7P=YG=&ZvER_5)RsAR?Y{C1P1~RY5;ji+m26u~=kIUvv)VgJk<WhB
ziH?0YLLK)%bZfO074x?}^00mHp@(k!?RgsZ_WW48m)mgr-hxl!dwn|R?%BV@)%Ktb
z%YofDc=j<y8SNEV_<7%3^GADU&NSFt_&32WGuGAKaB|n)meY@H0-EghMkRCaXEr>y
zPx|_%eOo<>_U6nK+&Am}FB|#(h4y>?DDLx9s@lEn)#QD%N=^6a?8w-sQ4((>qN2N}
zV*<~<>x=yMNBq;>b9dRkJy+Tv+upg$ZQB-bYwxGrKHI`I9=r46_w21a?y&E!JlDP%
zTNU@FhlcN4AEmPI%T^n^YfAU_wrYOa`|swwy{~Wd*gd(YWt+SI=3ZsiX?8(Z?G7|7
z?%B6?Gt<7s#uxYQ%vRc~e9wG$*<7{*>8h$Whb=_+mHs$kGjZNCYoDBpc30g?_9{Cx
z+f`Ju+ijdDw(p$J;r-lK_-(g_b?uuaC9>b(9;f{`A@+Um=Q!-MaIoCB&*_6rhL+&I
zijCd-8mv5bi3TUyXgt)}ExcBJU-7=?y_d{x?a!RWzsJqu<=*oKoAz^Dtg{XH%d~IV
zo!$HUC9C$aSoYh#;P>0Jberkk%%kCZgJ0>|Iy`XL_hp)(ZCZ%MUMYsiJx?-z?ODX_
zxIet)(B5PwRa+SoY1>Irv-YlTXSNGJ&|<4wq-;BDvG?9TytDTAvBd9VDxSA5fa`$m
z&%lYcJAWVCQyJ5~w_$VtUgcT(dsP)!_Zm)4-+Suge_L<i^?N^UpSf4`{@J}>{ATXm
zF8pM#1hb&+(%-rJUfxLBH!am+ul<UUeQi;n_WcVuy7!mIn!RWDaoY$sy|!zb%)H<8
z^PjyPs()<eaOLm4mlU}F*g~Ftp6^%KFHF+icmBt%y;EKK>^LoCcK>GmZu2Nvc5l**
zWc!X=JO?;VT-?WS&BL~|AaviHnj8CEnD6gfys&AXT_w-H{x6^REtH$KmwEkx-M=1(
z?)}lxxwk5^dC$$>p1qPx6ZV$h;@CTF)dbr}1L=J&ht&7kEV;9{;nq8=paA|o={$?~
zIRtg?e_<@Qk4=exU&o!r`z6GB_c`m`+P89U(tf?_-8NmF6ZbjK*R;(}uiYomd13Es
zFAW=+cFz3^!yoQVmwmCXv9NeAljGEVRdfAq?H(N1H$Uvy{=^%4wmXY#?V1V<_Rr_@
z+<W2Cmc2dZZ2P#Y+xKpp&tbP^=dwLK#;^9-2dM0|EuCP$RC&EkXqMmJxnJMz{a9kS
z|A2$&zHZ|`d#^0twzpkj%0Atnc6;3xmF#Plx?#)nSiw$7UT>eOjNiVNchmPRUVLzG
z*$<_C8;(ocdFW*BzHWAC54(x#UXvf2_Y|w^?A779yzfJT=l*%EpZC}mHtb8hx6I!4
zx$<5o{YO?G9RBPt<LB7tKWFA1pZTozU9+C;{qOeBdRpE5y{a7od!?tD?&F)f+-~RM
zReQfQpR@Co_p|>l`ea}H@uT}#ayRc|=9#k3Z(8lX#lPA1>NQ-m`!n^?{>8nst%IX1
z_tu>2woefHyXPR=KRes|Jp2Bu#oJ$s3$x2VbJF(7!?L|^7p>dt;=6W_R6YA{HirJa
zi4lwUn9RPl@06VV-W%7}@85Ul!EU9ESNF=7#M)2hlG~Th%DGQbgmbUA>aM-%D^Kiw
zdR}hd<S<Pe#b=-QMY*!=@0WYIC(Y#dK1Ty-`&kA}`@;8X?|XJ?)}Div5B8en2<?mf
zyWhq|R@Ig>mT6zl_b+?5E;wWp;V8FfS)%8j1kX?V)?9Acmt?<YFK^JsJ)6GV*`x5!
z!&V?9$3Af(!#?$Xv3&tMEcP;b`0kGi<FZ$N&tSW8g7$v7ZQu7+8uRV_zh=wcJ^B2$
ze?R){J-RD+-{l*d_PYDs*!y$evb`n`O>Iu-HtmgIwb{E#qS9_w_QJiK8Ort#wp_Pf
zSRlTyvfFuoae}lhYqE~5{H2_I#}}U8%bmAzFV|X0+l6Jz_V#v6usI<cZ!0xnn{9z3
zr)_dVk?onsocjdAXCHX;WcR)|o(%Tcr`GR}dDyjw?_`%<O6^*kxb3Zb%;pyEef6Zn
za>>yndmp`K*&Fe_Zcn^j=H5APT5L~D-L~h7^CH^~9uBrQsxtN{{PwrK(4n_?Lcy{<
zHi?#dXR!X=(~zXO`%b`A+da2?Y@_3*@4cio(e^-KrHyyQ!o6?ivDrE}v+ddN`|F;Z
zhA3N~dFi$-@=3N1?3vbkj#lmcrn-8sO5(n~d-7-5thuy!uj1FJy?cy%Y#F9{TTSV7
zwEfYMyZ3<6I@<&8kF4JtPu^>+Ghy%E*6O`$?ya+}V_#!ia@g3m!*$;tmf3x_4dueS
zQnjl0njSyD_le4)y$nf9c557%yl2rFuDvs=O!jDAJGPhM?ycP$0*>uv@RHlpV6|fR
zgvM^07b!lr7YcsctTExR<+?k?R)wW|?-%`zHau)g_arpf*d~7t*&DF6&)RY4q`f;f
zp0=4`TW-V3y=U*UbEj-g?@roVW3X~h!97n~*#}Ga&S{ggJy3GWTBYm59@!)N_j;I2
zv|VOryI1aH;9l*DS$ka~Lv44IJ+iql_4S@8hi$eu)S_(M^0(OrNPV^bxw>!fBj(b*
zj}OG}ec;@_$L{o6TY*z1dm}1J_v~Vuw|9?&%HFm!r)`r0R@!oz@3&1jc)>Q~%vM{4
zWDZ*nCr{gwNy}}`)@-nCVxDNr;QV-x*TY%1ubfueHXMntTCy`^uhN7awiz-ZdnRl+
zZJQt#V$+b^YipsvXOnSH%XS-A<K7jy^Y>gkwbr(S{jg1yS<BuhZd>*^Y`AUnZ6fd9
z4xbNuME;$#ReLyXuZ2(co)i9}dnERrwe4EVy7vK-fsHSV#@@QKqI(nOPq1b9*|ayI
zY_sizI}`S-IPYfLl;g8EsCR<xqh+nO4hd7Ndp4f5{dan@Er-MQy*E}}wb>H9Y449E
zH}@!|oZ7p8Z~Wd{Y%Y60v>mnGxoP&^hAlk11j5eSeyCkx>-%%#-op<)_8xr^w>N6h
zoV|+~Bln)zRkAn4lhf9r*w-fMS<l`Rf1~zlJUwY+r)9L)uKBQSnDF_%6?T92oSEKX
z%d_+2o)4cM*a)18+;iaU9_zsPj=e$Evut;K(c63Dgv6dJ)%|-W$>!}na=pt|LSN5T
zzhdg%Wjg2f9(golZ;+$VUWubu_8f>v-YwA5Z7U#_w&zF7gFPQYVr{G6o!iSM8@6{x
z>{;6@SJ&AxO>4G&<F?ZFhy2bxA$J9By|`O#O&kkt*&{dZwaNXqXWiHNdrPKw+orJ0
z-jlXaac_mFfNfM7|K1s$r8XMxcI<IcIlkw?sixi5LbLa7&zQaUOIft-#FT@3-QP^z
zt03rY?eaFxHsPa;?FN_UHdE|MZ2iqL_exZqw4Jaq#5U!fwC$^6nY|yf>unh<LTq<j
zYu?*oJ=-?n!=k+(me1aE)5Bx$#QENPcj_+P%VB@sW|2hy-h^B&TN~L`wl`WMY@h7$
zu$@p9V{<KKh3$u*ntSj47TkO9K%4C||LwMG4;<Y4Lng}B)@|9|3VnCmwI}!Q<yaA5
z!?nEER$;+TYq!EvwpX52+0NK~X73Ij|2-V0e0v(MTG`%sR%?@_w|B2a{~DWs72&oT
z^Um6aJZRXnL9561((~DSAH22PQ~b$uujwVty#g=IY*#G&Z^QG@eD58xg?n}UF6>Rw
z?zBB{lGFCB&z`*<SNGaHn7DQ?ixsDB<jm;3LH(Y4Zv<|zt<ssgcmIwfdtZF1vYEji
zV`~t)-{!)~uDuNBglwEnpWS=MYO3vbleE1nwtcg?;gn&k66Ijau<PE=2TRiS9^jX^
z-DbMPcE#aPn;#XMY;_;Z*?TtP(B2i-cH698zIX2@{-b+WEDo?`nSR!mMQFNhqWzw|
zCwNZVFkG5!+cGQ9b_Mrhn+f?|dnU|?uw8dhZZF5?LpC#-1Z^2^&$1CX^}u=?U+3P4
znJe~4lpe92!tSxxD)Yo%nao<-tts30ZaOe&@4K~2Y;R2Iv|adn?_RFM<$GO9oAzpd
z-@bRoT36ewJ?*w)8ZEXbE?ex$vEQ)Q#U{{JXMUFLjQdi1f4t(c-6gwm@9ThtdmUCy
z+~u(4myN~^Hrt9*rnWULS$peWuGt$GIA^cM-YQ!U&o?&FhZpXx5ZJRP;m-ow1(!bC
z%-XPbuTQ4;-kjwpZJCTW*;?5DvuQ|MwD-cq9$Pib^|n(_uihK*&SFnQm4WSo7+V_&
z(Rq6_Zspk;y-VCXL)zE2_lW!65Kl{6241s0Gq`WqxQn0Pt8lMvPqgddy$suYZC*GP
z*)Dk8VUxRV*4{tQJ8d83&#-04_`2uE?Uy!7pDy29v7vI$q6Oz{Bkij83Pj!8-Eh-?
zcg+b~+XGk9ch7m(VjHpKsEx$uX?uQMIcY1mv)R_Kwbk~?>#jW<kH6Y%;1}GZp|{mW
z;?hdn8MifUOAf8EnQ<u6*6UCAUV~>vdyhI#+S_>BV6RNVOxpz#Mb-j0l5H1+f8A5U
zZelCKc5Ba(p9lBu*zsx4hJPx1t|ngCs~}pwx28nEcGo=Sy%sMN_vFM~u)X_X``!x)
zA8hI(=Ghv(-MY7K$C|wd0-CLaZ+q>nRo%Ha=lYyI2~}0Q|NPdq^^mBtowK=luf<9)
z+e=1!_f~i&?q$zV+54epkM)bfk2V{2%-n0E?QZMfx_ZxtU-N7}{Oa5Nt9`!h5!>#)
zEtdOjbsRU^ept=3*C9I5dW-eay}o;n?Y-u1ytnen>b;yR&g|WMs&22JUBB%NZ8zID
zDZP7kWGUHBPO02mbNH0)fyIh@WGek_8@i;ecgRn(z0mgD=Fi3xdkgmV*d(Y6+Fo0>
zb?+<tm9`D?bF9{U>)-Ps{;18&Q%QT9p3Jv>Q8Cpf!#Q|Q!=3fED$O;vPi`98N;Men
zU6N3^*NVYm?|Orpy%%hI_ilVW+4f#e$KH8MWNlY|d9ioVWD~o96GD467VWpqUUz@b
z3+<!39~da?%hYVx>+kr^W`;?d&Ei5`+b1%%dp$m7+j^Ay?b+-;eYbhd@x9#>kK3MB
zn``^#u<$<q3%6}ka}{jor?A_4PvP9RrZ8l0$%f5)4doQ~{y({M4|id>?W=t=Z2Nh4
z>^*y{bngk(QX64^kG-Kv9=1j^kL`ZLrLvb%?%Lkp9ISgYCO@!Ah-KO%5;AemDxHnC
zVoB@vFvRuktul|=d%z~jwvmN--?uL(Y@@}a_GVYzvk7Bq+S@MKYg7GUq3y4ivusx{
zn7r5b<o&%SM`qYOWtz5kf`Q52nvm|jk_S0$oa^7)IK5fAw;_3-O$hr&+m2J8Y;MFH
zvZ;7lxcAcA`L+y=2lkYGi`@Gv@73N3jjelcoA0zqJJMmhM!j@zKi^Xu0~?vWGCj}s
z?6%&s*CKJs-m~5B_q>)1w6@-Uc5lLhf<0Z<2W<s9&e@t2`Pxd=?A^QPg^}$t|H8fX
z@4wjYS3kT*WvAZW>c`TyyutTvmtQ?++cKAZZyDEN+qSxQdxdZN*p&P*wtYB5Y2W)>
z=l3`~I=(l%ca=?AC+ofgvO;^VY~E(GWa<Q)GgsH_J+Ud$=F(@ieNRg7+Fnu>-V?EY
z!|pdzXV{)NQn6?MURPU%av9rwUV6LF^rY`y@p0>(1dr=B=eZW|U2?H*kII(mwh!iR
zv^o3t-`@MqW_#T_j_vj4udwZqc)mv=X~FJt-cXwv&AGNL?=x(@pa0tXEt|o1<KvBc
zB^9ph`G2K(&%KO>y$7`0_oi|d?WsJUxR>wHADb4f_Pq+hM%LHwtJsELpJMyrxS-9>
z)m1iAm1C@?cg@)IV4v*XCMP#*N2g<Z%2Jo@ee`(N-e2E%_I8wg*mJjV-QM{-bZoDw
z`|tHu<guG(Y+*Ym>)4(hwFP_Md{DMs$jH7ga4DCqdfk58kOt4ab??0Qa=P>EJ9;G9
zmcce}_aV+b)?a=b@4ex%c+cEpx_f8%i|+HtTx7ef$7ydz$Dut^f4i-!pWNKjuqI)*
zwWh<~Z<nlWYaCtozBqAWuUl!IZL^h*9p5C*y{i;?_RKgkd+!O=GTR>6MK<B`4mN9o
zT=!Zr9<g<q*<|Z-rfSb$gIBgM4qdZ(vT6U`lU3*UY%Th{m+RZiz3x$KZL_AH-dokw
zvDfH+sO`b`!uuZS8Sh;%JInTW$ke^pbms28wXf55xrN)_orw~A%x<r<?K{u1*U?UX
z&lBHOHf%yOZ1obl_D=q^+*T`f#a_L`du(@etM7f2nz46M@>Ls|mDl#psaDuGA@}^=
zxDQIUFT{Lof&{c}J?*FL<+J~|x4op-=IeEfy{#*E?e(4c-FC;%-Fv_LzulV<e%SU&
z$_Cr7j?4BQePp!vN2bU=pD!i0yTw%Y)t_V7bGp>mN?qyY9-CDxw%7CB_oh6u+3Rp{
zvn}Tu7TdUO0eh#W*zaxexxYv4Z=3BNjVXIP7tGkZg@t`DdrX0C-O`}F4Xk{7nNuqF
zu3oLY&-uTN?W=16dpj=K+IBXT*mT7S?PF<KY1<Smwzowz$+mdGJlj<9zP)07FKsUI
zEZw`Mu+_%v$XZ*T;#qrM*Sy<PZFFL<n)U=6wPh>zK0fopreNO5J(YH!_Z(L~V$+eP
zV#{<ucTd#XXM5~^J-7Ywl*jh9?cTkgZPfN{+%$8q!b~0O4-)*g?Th303U4^GH=()0
zdPBnG-FI|9?XmbOYy1C*!Cu8!nOzA+zxHmhkhFOtud!>w>Z3M!*_&;O44if=tX;OZ
z=`@R7r}(zLJN%B?I{0+#IvK~e?@yN5zWsMs>}7lU)FvjfW3P3>n%!BVk8J)+H|<r}
zcicMYW{>TZSXG-W=8JoU#bRyFRxaK9Gs0r;Ux`0^+h@4!oo=$qddDI4y{p5|?Vg%^
za?e43F57x8Z`-qWR(r2}u(my<*Jyj6>C+yq>c4xXdT-k#2lm-mWaRE?@ocl%nssIG
zyn^_>78h3Sxw%5#F2TmzMmI;^)}~9>Rxp*rcGbsEdjlHm_xheH-+O0rpzX?N(>+as
z8oO(!JM2BglWA)#)oi=}dB1IvNdKN$p5}Y+^{un{;oQBaSzmu|y2<Lj+1s|*+*s~t
z6Y^xO4d3aWy-llT+rEg3x4m)r{ho#tt=;l_AKM14QrYV&zGm-kUd`Qa@+a6#yVPJS
z@_F~3b%`Hs4&`j$J9AB+t!dX}n@z&A_eL*2wYSFc+MfQ(gSH71`1hKWYT3S+`pP!-
z+$!71xUYNfCB51^p@iLb_U0>lzZkvUqj<n#?~B@ZHq(Rc_eO7S-0Sef#P)#-!@ddy
zvAq@US+=M4n(UE1dC+<TbJt$KYl3^<hq><F`)=*t^tG;gr`(RSt($ei#$K7j_Q?8M
zwl<C1_L@58?77_e!0JwhvF(<vHG6B89oS=h;n^P10>-@z9v61%@_e*;zkbc$y}aD}
zqIjhCTG;HjN$Xj=w=A${?=D5%U0c*w+2{v7v{i|2vE@0?Xrt2`ZTr>Xw{6=Amc5@>
z9NinUaf|I7TYft)tHXQe&2_gey1v0?wO5dBi4?c(^dtQ?7Tvq{KJm)24b|1$yJF2j
zTPuM!o06-nd!Jf*?me?=${s$au)XKhx9_cbKiii1y0C3Qg0t-hy_+@%Vwr88S$whi
zHEpWxU%nlC|E*ZHXOGtHJ(9nhY|TT(_WpSoZ~G*`eXq9N#61DWw%AP1wccB}>CWEI
z5(jJ<+SK+2ic9S~Tex9wn9QTSf_5zX=5=r0d)Djo9>%Efy<sPx?NzaCw9W6@u;+rH
z=pJrszkN}QJNGVNyI^yxu5&Nnk&Sl$E-tmz2^8LY^Tmd}&&>60B(J*d+4wHO_T#ZL
zwj5otwo(G;Y!$wk+Ps@G#dc1fw(W=XIs2TpY_#q9*JaDHxO1PkyS?qR9dq^_n%BQ?
zliZrUU$^tv`q|3b8C+Xx8*f~<@2bdSTc5`#>}JZ=@9Xeov}JMHY;$6s>fV|wXKmA2
ztM`T<pJ3C#d&OqLykMKtqATshW}n>W5VdGm&6ehUtLyjL^?ZG8`#>+qZpMxqd;G$e
z*iL=uxu@}MnBC&bzIMT<4EM?RbK1=_Ib-*2_J_R>xuxwMOzF3cw-&YA5p-uy)v>30
zX6x139aQ4lC$8MKSLktw-Hf&Fd$!5;?R^lsd9TM^1KTI_m)a)%3)&kmux($|!4_ME
zH~hAXBiQ!7INP(g?_m30hnW3V*C(#F;r%MU?}Y2-y-T@1+a$Y%?zPZm+uQOz%=TB}
zB-^x1+uc>u&)W6~b?s~L6S7*f*4cK8a+vLx4a@9)e9y7H@PE#}snSdKxP0W<yZ)NZ
z9+4e0_Fg|6zjyJmhP_%xwd~e}Z`k)Bo^5x%P4r%#Vkz6RL439xx2M^O9n#nrD|69W
zrhJ0!niUy)#jZ`?H_;@|Zc_UhyNeY^_Hvf)w!3|E>)wR>NB3S%*|2YqslM&g#^rV)
z%4clLzRtAUv_H>ggX@z$W))KV61f)HY2T>d$1qWAPk_$T-5cDJY`rF)ww<OeW4BO$
zwataTZPq#shxSSDUbWBTPUqfJwFmZcegC^xj$ddWC%5Uo43*kF5@r&1=Unsm#wkA8
zyZiMy8*VG%ec#Rm*gg4jYL8d{9$O~gX?Ee;F4)-A2<&~tw9$^y<f@&N!;U@drG<O-
z56-oDURb&}_S|B-fWPy1=Utd*tMt3u_MXVz-R1JL_q~WK+mk7Hb)O7}y)8o$&z_qn
zX6>C3=WfHe!G7<0ZpOXRE5r942yNUGrp#xvD)Hl<6%!QqhBVFJGjaB<Jr`QjY$Z(Q
z?{P^lw$1ULvgd{4Gn+5(&e(2d^4_a&U1#^;?5w>%Qoigdc=*fuy3!Q88#U2;d<3rT
znG<?ycU0fbeRh8p?ZW02+3X8S-F^GjOPePpXKkOdwc8oQE#FhObD1qeuiY-K8_#!_
zFMDK{=euU_1;?YduDZ+Yy6ggNO<pwbn^}8u@4jnGY`^?f*?YvYa35>e9@~HXbN3#&
z%eN=|u(;hB1}$5SM>2a}e0yT!U({j~^7*Xoq{t_G0)mumPb%`;u6Vf4wqQZG&7Ao=
zY)`#Eu=k8f+uqMr$Lvl%oU`{vuI9e??vHIYh4t+Heg2}g#D*od8kvV|nFZNwH=NpN
zTV5Wz_d)g)>lt^Q>^@GuW_xBqh3$eSJ!=^+wtb!K0(OUHPO|0VTx*;0d%5k(5Bv8m
zU~;m~Kisfq%IoHRnn7v%tTZxhSS<JM(KzL9qhOY4^X2{Ny)8|9_T1dly7%C0|Gh0`
zTlOYs1ns#qW8a<=>AP&^)UMg{Gq-(@j^DC94s$#999Xu{=JN5oHf}BKwkD2Wtqmq}
z+OC!>-d7$vZEx49)V=q1G3-lHp0d}YciBEEkuKZMdmh-$X_2vY>iTTsS=PIEzMcDC
ziz+AE-{<DrM&H_G7rS%PzA)c!dwPA|@8ww8WYd~*Vy~u_mhIiW)AzPV9NL?u)nqGm
zZnf>ptMm5;{|(x;^`ZV=p5IEgb5z&u^@!NJcZY_(%^W2Y+qS3;dv82q+dEf)#qQ0m
zTYD-tsoLG)m~P9p^Rr!IcjKN5n%DO1FkZTEm+SpKsoO8^dA9SY9m9r~HjE#5?F^p2
zv*F15ZlnJ1)?Sup3-%S9-?%S<N!ZqD&H}r}fA?(y#7|iJt2OS`TDp4A4Ic{|ot_?B
zt#>hYtf!XSHSA8eoipqD-Yd)&wxYYg@72-eva^_W%ckW+if!S2XY0i+dA6HdKiKBH
z$=tK;kj~yL&slaEZf$nw4nNq{y>*X`lyZk{M#y&?{+^`0p8}Zd7H(K*ck>0W-LW$B
zz5jeC*=-44X?w<`YL7zt5j*||i}yAxWZ8G@xv)*K-HpA>)64cGRBYQ9f4j{tW}%*K
z&RdzieLMU2<xOp|z5Kt&rn05gZt2}#yUL&4d$-<Cw+rA2x79ZJW#h4g-!{Q{_nwTl
zBevYll6D=TLVFMAwC*!$IkNA<<$ZfUhql>Ooj7Nkn7PbOhk4;%wd|d?{ePnO{fgGG
zdm*)CZ-FN79)8)-eJR!kw#L7g+I<vUxo6MYP`e&hg*|NtL-svwyJ=%vvDk*=`34&{
zMi*O`zvpdv7}o98VqxFAp?jarncAgxudXlNtN!ty4a?%ycJ-6D@A<oo-B#kk*1bC}
zOYCc5(%8F?#n5(-+l9T`*k{=q?^tKYygki!_k?+NsY@o<Mc>@Dci(zun|S42``llz
zvpp{?VfSIf@;wP#j_pq5R<-@$XJ+GYah}bp_6K{iB@fwJ96o3Ffg!}^!r2SE|IApk
zx8m#;TN$PBeH>*o>@GO=?bh(D+!L{5-X5Qg{B}M{9d-@t>-NN*tlu}k==9z#;ybJl
zd^%+-(DHAu&PNWr=_cE4pK3MiNjt#5H-vfho`Xz!dn>AWZA6z}+3RrPklnGu$+l}d
zyY05P9kO+}zrf04^NoEz(mVGx2)gcy==``x^3lA#dB^4Us7X(=)wlY&_u&5Yy&C%)
z_cm`^X~%rkVV`9|-QJw`6*kfT`1Z+gY1$TCx^0&qw#sfXi=pjATh+bO-NN?j8%x>=
zsjS#L%jKL+$i>aJ0vD2PS3lC&D`0%m=I(~7y@j(=_G+z~Z~Ne|q3zqeb9?8VT4sCd
zs>t57RXg{xT|RGnsD@=PLo>%-6}LHiUa<%4ojz~nUXfz<y#XSxY#!Lv*h&^I+3Pgx
zgl)h=E1T);2lj3`p=0YJ$!41}|Jfe4&RKgIn3vi7I3i+u>dZ#l9i0q&GG<KN<8aNx
zcE^6Ny#bHb+w5<N-|Mg<)21O<+{U9K)h6UoxUI>OWZRy^1$#5jOx#=ZuibXT*IT<g
zDpuJx+`YGJy?NH&-wjLlPKY^Ub0$c@mf!!tUc)In_8M!r?yb;q+$#~|X!Al+aL<dW
zGxk)3{<NuRaIu+uK5}o^_13*TePOmwWRLH;p&?_-FuiY=#&0oOkJmaj`6(Ora<|p&
zO?hQvJ0+^$w!_EWrq%Ss-g&;ld&7Ax_G+BkX|v(B%ARBYllDHjrDi)vO~%&dVS?=y
z9wXa_=4baV(4K31dgb=L2g)OD@AO{Zv$%iG-U{oPwh1$z?YXdldr!~R`FmVsDs2q{
zi)|A8n5+f1p4vU9bgnJa^{Tx;WVhIGtkScMIg?=fLGQWEndOUY6rNjKPjg>un|5S|
zt@F%@d#^1$XZx_DdaoA$6x$5PQ#KqI_wV_0wbC|TW9wc6k7C;fi49f`9a?*M+M{i=
zz8tV+cwT3#GIfJ(#ournjy0vWSC%ZXiE9(w8*ppe-e)TAdl^bL*=%u{x3^o=WN)eL
z?7apiCv7w~Jh0JNv)!hlK6B54JJz=Atk>=>P}s6Jp=O%xRrRvH9upVteHLH5H)WTb
zZCZNx-tVQgd!;g`?N!Mtunnu;V%vMUes4$CC!3$FDSPkqWY|iqeZJ?6OVi$nnId~7
zO^@vTu(R5>)OzRM4~aKyv?NyA>dkGo^-Z(cdqmUQcF&xUz3Wt0*(S}IZ>wNzV0(eB
z&-zDex^4G^xqDy4f3Y#!-)H+_!YNzfmuvP~6pHQ13X#~`adXn%J2yA)t%z&fn{eQz
zO@hUqz4HQR*t*CZ+gs2zZMTf;0-GGqPkVSgJ#C)|FW$RnpM&kkxh;DGB5qkf={&Pn
zCcnovqKMnZmw)Hp4<QwMe%$?H6LF}|=12K*+nQZ_ZQpKRx7TMvfGy8HiM=lI^KCs6
z7;G{+U)%ioyWjR)u+82-8WZ*i+_c%lvAxPRXxdWS4=cQFBk!o~y^!{0kAg<eUWT8O
zb|rY;v3BA*yLZkf@x3aa8Ewm&X4(FT;@D$gJ8SQU&ueyTEOywV;Oc4H@M(qZi@UwH
z99rM@1i$LoyKKT%Tahap_JZ#E*4Wiz)Adnn?~P&)Td|wFY@@#Q?w#Pa&bBLKmu<vl
z<~=Q@O1420PueE<CG6qI*0Z^>_un3e$G+AEek!(iPOr3Gz`?xd)6`wIL3wL!WfWNU
z8WpzgUGj$A_8ZIOy*yd_Y#C~db{qWbunlz2-Mgf2imk`X*ESte&NgAUCfgba?cUvT
zeUr@(i+#2){J-1G3!b$%q-vYZgT|jW4sF@H1JdpH9)GN|ms4QR-UsWa+rHU9&vsAZ
z0b7Ahv-fT@n_!!8?(J@&#wE71^Ed7Nnr6S(AwSRN{M7S%f3cq4tFU{yZCdUE+f5U?
z_p10FwVfb2b$7t2dYfO{mhYXnVS%lX*V4Ub^E&rdJT9`?s*=9<!=`*&i%wJ96ABad
zwj^=v4Ra6KYc%KZUX~{jwx!jDdk<tz+Z&O)eeWu6j=dWkb!<JlKG^I{YTvu#{HeWp
z?<@BTEN8au*;Ti9NqwSi0PpqP0cACNikv6!eXn|Euf^@WJq6NPwg(T*+-q=Prp=|&
z4%?ovJ$sim_S&up;kA8mg<-E!BGX>C3#)CXFfQAhl(A~B>@N4cd)BVoQ?OORW=Gya
z+b7?r+P3x7>=jwKcJEE5U3)L&w^-**+OpT<!sfjK$Hex!)f()rIe)^|;j8HGD@K?1
zaQxe3t8rz*-l$0-dw1QNVC%72(N;@(jqSA4=l0H+@y}*L=z3d@aFM+yj<(ozeeJZ>
zaanD<W<j%UOTc#95B^MhcNp{S;km)S_dw%>UH3Qj?{z3Mv3?-B%r@%b3R^aP*}Vnf
zUV9_*f7>wJ$=TUp6=p5bxXrd;+S}a>&6BNaQf};#cxr5W^8ewz$5=Pm+U#kusW2DX
z+rpz}`{2njn`hcbZ6{?PvTcany?6RDhrJ<<9eZQsPwzR<o@^7)`M~D3iS=FwWlP%=
zjZ631&0N2C!<VDhjmFdVrq!IX{h;AzYxn%1?TH<qY*^O0+q#&|vi;OC-&Uot#P&r8
zukDX%@iqYylx%F8*6#iMeg0mbH5Y8pOkQGptEtI$!QTSwoEl@>(z^%uD$M-9=ketY
zd)2tj_qrX?+N-0)W}9-<Y_HjIzP%5AZnEt$nzN_dXvf|Ug6?~)wlCQ0^Kzk0f@+UV
z+vVQ96PHif`^E5(Er;*+y*K9A?BTJ?v_17?!rqKpKN}9|ew#h7nQSlHw(Kp4J!Ad4
z+;OkxkC44O-*?&WXpXYYxY=&2VB%=IgUiC^&wPH{_Jam{4fs-R6W-YDxhCJUcS|Xw
zt>C2{d&9o1vTaB>vOC7Mz;?@yW}CwQ?R!`Jm|?r()_$7>rxsW{6r8hhSh{3yMbj0V
z90~Wm1vkC-cG#!eo@k%6*TwPYo(*lLd*++X+ba<G#roX+xqAhU9@?uCWNgc<R=HPe
zck$jckL_(YNQBz%u+!d?@iM{Y@#n0)8x&{k{UY6IdnWY9o~HiVy&5S7HX-g)Y#J7c
z?k)T=d9PT<CfgIveKs80>U(6GYxeG9Yu@|tP1D{J_0w%{f2-WP?8Sk-6PDcHEiu2t
zc8m7rz3m5I?{&ELYcJ~>#eM67e(paoW&NHLRX_He{Uo_BYUNbh*B=e{KL7G>uifN>
zHo;4pEjDCY@3U2_vH5s_W8Z~h%Y6<K&umwvH0?D>b+uJib+Ik#+G0P)d$Em&|0LV}
z?>E?+XDrwoG5w<bk<ZHZ$p=2{ty;fmZ}#=Ob^%F`_tqSd-v7%|U~iuz+X2a{hP_=m
zjr%^^T(r3_v({o)Zp)roT*vlhc8c1p$W*fJzIbl$QosAQ*8MB@IlsNKcU|kReH*za
z?_0{XW?!bjf_>3`ZF|$jPwiYT(r=r+bhEu*q>cT3r?mYBZ~j@!nOxnQ`KfWA)0OnS
zhj{w->16%3xiTkwZ}{fLdr!0%+OsT=+1K#-?cNHvbNku0FSc<|ShCl#uWe7+oZEX(
zn@-#3@|n$MvURwf0oM+jJBjOU_SE$5^NW(SwLLv+uUzm(+wK($ckKwv-e-LN(B8j3
zvv#v>+P_aXX3<{H83*>T?z^*ZM_#e5k3{2sPxU{x8~b0{2LBb=`_fct-{PfS`*NM9
z>|?J{+xM&Y$?l6c@9b3!yS!J&a--d+-_rYy>t^iRP<M3SovP~nJl$=3W!_ipHEEf+
zCr_klPiJV0{R}JPeIkb0d&@;q_OadDYJ2z0{C)EN-}k574Bf~5|Nq|4igWEZte?4;
zW2NX`=3RGfrE0ltk6mQi|Kw=2EtmI<eYW!V_O)c?*d{ZY@BMB%*Y3IJs(m*(uGzTs
zf7n+b)N1pf<F<`6xAMLR#zJ=ba;x|9KJweI+VFJW(<N^EpKUp`_uPh;HYbk$+hdY2
z)xLmf{_g6qS9`g#KJLA6N@{QFs)oH;AJ*?_X6oD@8|=MLJ|f;WP?Tps*Iuo?bwLGt
zjY1aK*U#9#zdHBk-lW%BdnaXd?=_9wy^qa4XMdXcl6~jcX6|i1&9zU=P+*@zUy|L$
zWlsApn_t;i`EA=i_Fp}FnHJsKyY7YaJ~q{fd(UQE+FR&*&_2OBd#`@{q&;^fRQGMW
zr@lA8R(hX>TIZfqtLN;^@>*uSeEZscT4_J_*3^paJGA%9zB$!=dm|oCvwL)L!`|<W
zYxcIR2;O_{<dwa7AJ*?(!#Z!TUvK07s*S(xK1dbr|525(ckx^X8}nZpdp9Ot+w1)&
zWPjw$^u0C@KJ4Y{D&L!uX}$NBo2=ar|3!N`R3rCmy56^Yy~lO$`?+!lL{@k1z5UZ_
z--bn-_cci}+BZJ`y7%6Sb9<wudUtCc65r>4-e})sMPa);)=9Pz4fFQdy$iRV7sa-(
zPe6J1``5GWovNkwrETfn_hj+){nOQo_g+2t$ENRGtNlu?(0w~uKJTAwT)o%fneFcN
z-yZIrEN;Gc!uth#J&tSdNsQ#!7n}EBx7|5~eQVi!_v=QB?Ca<(*{kP#a<4{Cvvo*w
zgY7z7_q{W>oZRbp{?i^8i;x4-+x+&HZCSZ*eSg2*3?WUs`idX6DIYlYz1uIim;dO#
zz3Crk?A2)VwY|0B)!yF2{CgX&@7wdOc=g`8n^*TJ^55LcqtLWxtxV*;pYONs)!<vT
z_tDL{`vR>e?KNBYeeZRTcYCjwO7DGJE4ROU%j|t0KWFT_>?OH3B;nGY(`~zVJ8w_f
zJ1y*w-CG~seFlc+dtX>G@00eb-1~OIq`i}EYWCh=8fW{+!Qeni%A|dd(x&a@Td`nw
zVp{87+t%m%=KS&6mm(at*Yj1wzP!SVyC)nvZ}X&re@|tY{azsr_r0E%`)!uw3hm7@
z*uQtp>@#~eI2-LVULLep>zDAp0Hf9W${1buy6m2|Pj91&{X|BGy?J)5b~fiU_C+O4
z*=MJ<b|1$DFNa-IJ8WnB*6myKe%2l)CdPdZmsIxdaG7AA&7rdY%H=J)H%)J~zr@F|
z&%|o&zAsxJ?A`Kh^}ZOJsy*W8kL^7aEOp=pm)Sl=4j!9mY1Lf{yiEIh#f0`%SbOYU
z8yjW2Fh$PRdUAtpvJ8W*Rn&yN^A_6LUi4zv_b>O*zBe(;_qyn5?oLdS+M~UvVc)+e
z|MqgvTfNWgMugp!Z%q4tuUxtJm&LSwCss+?|5gv!%g=ejMwXk+fopZhUY5CQ_A*V{
zyZ2W|(cay$?6%LhPTz0;;;8+KA{AS<%~Ne%tk~=eoHF;W{L-;I>Qnpv=Kp8+trPCp
z>pP#@{-A^MJ`ui5+y0w|d+%f)*}L&e^gdSCB>TJ<to!ultg&tVz~}J&m(Kpmhu7?P
zG#;|Iws~VKCnvP;QEad6?mnJ<mg_gzUBC2suS40hy-kUU`<A@mvK4fHxZkajWuN7S
zb^8~KM%(9pUtw41J;A0;ym{|}a|L@IZm-+xG23bHp=N2DFu4``xaVKp6WM8V!0w9s
zUKUHsy=PA6*`LrCxBcSCW82noZ|?=2Xd4~Q*L(eq=I!+}pSrhApJBhiTe<y=;=A|0
zKhtIxee!}G@6Ln!`aV9{`@Bzc|Iyi|`!=$k*~`qFyVo!L`QGl}DSPU|v-WX$AKBCJ
zE6%2ThNA7(dM$^}$rk(M0{-pci?Ou58yjM4E}v#^)ws@1@xYV4o=h!!y?XWb7<gsb
zH{C7XU+e#1?^?m+eH)nm?2%ctb8nKzhux~FllMAY_uX6Zb@sl;#iIMxEzGo5ylS}b
zn@H8(w&4GJr=)z|JN0exzDBE+d%X75+b@V~*!#4#Y)_8L@;#nXllRBm-nF;gvTWa0
z{R#WBBn|f(?^(6?SHzCJ-S;hRC;t(!bxiBBW9qTDFXhv)J>bl-&xF&<ZZ?0<J}nRX
zeK87sHrr>;vkjIyZP&EFW3NHX+r25By0*Us)NKVmDeY5RwRUf$OWfXBJvDnbDB0`{
zaBJElkn_amm+1-Hd9`+X8Pp<dIj*hTbK~bu+jWs`wq6oKdqWm%wpqiTV0&oKLEA;W
z2lpCmow3Jn)oj}@e`WU`P~5OL<XD_7!zaJJ2H`(!Tmq-sc(lH?IeWrj?>wtS(7En=
zU6vQx7QEfOCo$e^uWH<xy&Ddn*<)aCWb>zs+xALdy6pyw3AXblZrl5$;Ln~0juqC6
zz5Vy<xJ=o@(|>61k11Ju8D8Aq<FM1-<{x*%-j2xIdyF!AY_I9e+k2&Q!rpn)y7p=m
z1=wCWBE9$8&vUjbm}+fHj_%$&qrA$7{fXn=1^&Bi7hOGT%WztLkA&!aTY-w}yC3L%
zvGFjRyw{@UyzL~GHMV>1ZM9i&Zj#OMC8~R;=*-%C{@R4Sjve#&T0Du^qmw_+mRD!;
z-UBJ0b_ezDvlTFYW}_t}y*KPjgY6XFAX}Mh6Zb5$oN2q_(IcCK|JwGtTwiK4bB_Ps
z3jHKo!4<3aS_E9&Gj;aby*csSHcm@U>@`lDzBi>TdT++WHMaMhH}AdVynXK))|oa8
ztQTz;9c<bgV4l4v$*|A1Lw(iW153|Xv2+}=4U_D*m0{by*TGM6Pt>%Pdn11G>?yf(
z%65e%v#o{lEgPSdWwx$thxS%YY_Xk^I%iM2(&4=iu0OO%S(|0MXP&U_8?GgLzwynq
zRo&#cSK&>N?fz5e_Hv0&-h08Wz_wuy>)s#nhinB_dF<_Am}0y3_^iEK_v-EC5j}1D
zNw#_KikbiRJeac6+Cpa09+k_l_i!v<y*F+1G+WIjYJ1n5X0@FmA#KwySh`p5SBtG<
z{i?kg33WC+Ol*4>oN(FApkK9Tn=bR-0*9Hq?RM_p+aTC!QzMpTdn2>Y=ESUDHaeUu
z_DHZb?LB1BVjHKRxwrdn=-xkz<7}^7EwMG2^w%cjp17^e9f`dmZU=4mm`=78;$O3u
zeW~}}JAW7NdA>Vf?~<*Ywrmm0Y>m&X+<Q(ld~X7Klx@f5342sIwf26veAsqI{Z5<o
z)Wdrn=2qEQd}Xk)ie9#N>z$)}TmQH1ZEBF%TU6U`>!7%7cSmxD?Eyh<>nd%by*<mF
zY-gUou-EPSxxHsj3foq=Ufc6VciSG3v|?L_6MuFkOu4zIXhD<h+3a(BXIvEDYgXE1
zdqHx~9<zvDd-n*=uz7Zne{asenYJ184fn|PRPW_@vccxW@+RA^%;mP34t{%WHq6<Z
z^3l!qa7V-5BE{If1_42~4y(@ZdFQlkZ^G7-dwKS*+1sMiVQZ7kV*9S6b?*`%^}XHQ
zOZO_=E3rNEnQ`wd(GJ_wXN~v1;Fh=DP`+yKDxU^hYwpE+RpOg%59(~&`|r+i+Xn46
zD~E`)HXP;;ZCpG`ZGUW@vB#yb&UXKCo4qR}<81p=SMN<w`)sp-m%(O774zOZs}I_`
z$4uKhvpr((U618^-596a2D}#7YZ111uZPMu+rt?Zdmnr(vAy6{xrd?0cF&JZd^Q|$
zQ*FiV&+JubxnSM<ZT;REPMd6V`1aZ!V9eNC?P0Z-`EBrC1||hthGGYs8?PtsiMX_I
z?~I&Pwjz#mYy;-)wb{wuv$vs5+4iv0+PxnbXWOKlF|^&me0s0SnO@r?K?Qr6U*_!9
z*>!EtFV`O16JO)^_MJIsD-(Xu_TjD*d%fmv-|Jwpch4Elep?2Eo7)XnH0>?;e%*%W
zb&Ktk&)K$1cx7z)b}h4Au&l&(P3(+4E(u%qzK~gCBX@7Z-WYo~+XMqITUW-ky$TjF
zdmLL5_X>6%-FwxnZ?DhvNwz)RJ+=;SBKI^G9o;Jztgv^<#x=H4=Op)jH`%gRsr;nv
z4)-3L2lFCr^n982T3lALwP9?xQ9HHP_Cb2}?wmd~+XYi<Y<k3|*-EU;*}H_H)|TPS
zZ0isI3~hU6^4NZO_}k_Sm;c^H+EZ;CLNa#G*_&YdBH*S?-|mgJGEpD)d}vYL+o!l|
z?}Rmhd*>h9vA4q`*merLwCxJfW}5?(C+~T)X5k(I#<_bM^jBDibSc}`e41^$Ug^|c
z+xKhs+PHPu$gwZj+oc}1cY*8N-7zXhY-dd4vNiJSvAvMLb?=1N&9+?G&3nyfFW%d6
z*Ua`+@n+i_c~Z7kpIP_HtbDm=S!}bdNave94Yu5S7>pm<ygl8$H}}=Fy#j4(_ndmR
z&Q^E-ioKh5dG1a6uDI7JscWx>rjqT7@RfTlcD%C@5PfRXu=JsIglw7Zg=uwrPP{a+
zeIYwx5690A+Zi|gZ2ryJvNz(?Pn&>0KWwgiIlp&}XszvmRS$MOd#AS7q;vV+2YEa8
zO7Fk0S5dZS@4GXLZPnJy-7AoIV^79>E87`WntK!iqj$3@pS5K%n6o!e?f71^vxR#<
zilptG@krD5Lx<L$4=;Q6Oj%=TYvYn>`;4o_Hr#6I-ZO6|+U|Z+zV}boxxJt6hVPYD
zbKNUr6l~k@=&Y3x_eonBcL&=|n~L^I{MWSg+taakK|qtO!bc@*1xr2aRXTmPJ<KV#
z5010!VfeJx>f<N>y$L_|SufaIZaeGYWLqWi4%-bY`)ou`9JBRN=(l}kyvDX5uX2yk
zx|4eq1ci2cm`m<izI)H!6Q`E#o#)uQH{s5?y*Ad<Y$oI^w$<IXaIe9Nw>F<vHSL}B
zXs+#ni8XsHO84)LRa>%m&z+TfcN7HJoTxl%JHzAfUX2O8dk$-~?Cm+CY`ezv@SX=z
zan=l_HMTs0yY|jZ@!7k8ZO-n5*_*9bL>;jW`qZ~KOJeojqUTfgI`l~GS<}{M%dnBj
zc7dAl?gesbdsN=!*)rVb*!|{m)!rpf+xNU^YOtN4>tPe|t7xyW>)O56tP}UnkQdtX
zpuNZ1nf3Ty#@#>nT&ru@Tis-_cR}+J>mUB-_dMcVVjC8_#`Z_R0$Yh$YI_XK=I&Xh
zb;|a{-i5XfJ0|aO(|u^uV6xF_TXd4G`=_40XAW$#W&Gl?*L3wETMnHadxW$aZM`C0
zZ9XV@+6qpYXZM5S$X*9W|Gis$C)#{juz!!W`<cCR94qz~$xYoGJ}JpgCu`lF8`tV=
z7i?m;-Y`#Q&%TgkTf4$u+q>dN_MB>1YV~XGGTZz5_IB!ej(cCMJh^AX-W(elkJ)?f
zT{f^y+s|s}`o3tdS4Hk#>60#Y-&XIjeeh7-cC*pRy$jx$?a>VC+}p!&)XwEav@L_m
z9Gfb`0-Fn66Lv58^n6c``=q@RGv4lbP?T&dHb-)=dvT-fM~A4r42?Z|lU`5UJK^fW
zy(z6Rdw;0(+eR3@w<$?wu$>uqZtq9-346-|!}co7Jz^u3w%FFi=z>kqf_7W&dvEtl
z&X~0KikggV&Pzqxie&vgdo`=<KFs#D70jEt?*}*M-m;HdZ21;2?wz7~VehxY!TT5_
z=Iu4GSYpF}F2VLe@%p{mPZroEEUDXLaB9}x&yxlAO?};Lb7WKZ-W~5!Y$If%Y@^tx
z?Nu(Hx_6gM(Vh&K@I4*Rr`qlC7Pb8_XOeYCw*6j%uGu!-E~5J;{y)6;=I>{=GWLSD
z7c?4d@AT}qO^#i*cau)(-go^wY!_^A-Sh36-d=`~^ES&&8f|B%I_!yxT4}rTy0x_x
z+ag=dO&4u`e4k+}mszm)fK0CK`I}BQXC4aJ@|`$ryYCI_UYA=_Y(+Lb*;7}xbnkbq
zceV>&<l20A?`V5JY?`f$rh@I`;)8n^PLkP|W~;ETs&~rX4Cxm(Q-r7NO`mPOcTT1B
zUWVV1yWgx?V>2hYV(&HOBYSUrF13x9%i8OFi_0!AvETNJ=4sm)p}%`nYBugsb7;10
z$Putf5iQ<h-x0UhV72L9C$q-AZ?2rTwRw1cZ^#`p+XH+1cLnU`*xMq1);6NCXD>(N
zESuRYJNNcjwc8%N#I}#2!^p;FYMiZ%Wr;1*F$G)E71Q@NI2KsPv8(J=-#^Wk%dKvY
zh4&7dh`HNswcMlj&M;!!8~9v$pMdFFYc|(4dl~##cAdzcZ)+d0ZqJ9krF+)Zd+qyl
zJI%Jou6FM_i@mnX*Q)JZFSu>*4^^GLTvn2{&Nc;m8GO=rwgzw6%kbW4PlL3Xt)zE_
z?T+V)wgq=q+j0x&@9mhmYww5CezrC<dF>KTt*}vO;@#8#=E`23j|*%ae6HE#9-nA;
zMY-AbT+L+L+6Dc48+-$7U)Y`88!<O%Psq+>+lK08yUyL{w7n6vY0rvqIa}|aZMKUy
z_wQAiu+4VO-AOiUPHx^C{A|nK4f$WKmxY|M&6+aH_G$I1y=(lZ+H5wRwfE-*0lRum
zHrqEc@_Q{xviCT=^0i*km9*EDXZ_wRwmr5H(_h-`5y-POahtIx<$cuN1GNu!^Q#u`
zHPP;{y|Yno-)y#ndt(-Fv1wS4xaU9$n{7^W>z)mD`>f~mUfz>?)Wgne1@qn&Ivlp|
z?)BPU_|9XKa(t!jl*D6u{njqGJ)!c(rYUXd-pYf!_x`Ek+-qkyWAC?ljkaR!KKowi
zv+VP8K4;7IaJj9ORo31kr3Y<|Tw3gIRGqSYa_^w6n%quXiJhf;JIY-5&JbU2TQOmw
z?G7_xo5S+Q_A;2*+3G~ku#IuqXq)AF()PysO4}8WuG%QbP2RIWBh*IS<c`h!$Sbx1
z{m=GH@|<kD)ud)$ub}Y08O{^-T%WkiZpUhNTb%+y+bs?YZQtGQ+#4d+xW{!OlkF=n
zVY?*(thPbP()*aodu&-$Lu~_G`uDIO`(!&w>Xxn2);3$!n~!Ww@>TXVU8>uAKqGUv
z@JgG#iOXB}3P>pLdBMc8=VWBm-lUJsdzqfc+GgDAvsvIOVci(Jc5gJZh3&2@Jo^;-
z6KrkSFYI;Tle9hODz`U>P0n^o@{7HaSEY7moENwK6|rmYe~-33nwbfEOCwhAy}kaQ
z?X0p{dpEqUwUv1~Z|{{Iakjghm)Y*`>f5VfEN!>7eA8aHu>E$k->L6Yxx;Deu<E!~
z%BPt&FP|6gQ@VM~wv2Dmo`43Yy)&AaZI2vTZM)$59@~O{d3(Nf1=wy7-(xKlqh)=1
z#cx~Zyrs5T@y&M0X8UZfU3y`w7G=73Pv#_>_$XPs<y8rGJbj#dZ7UM@TC*P8vqyBA
zP0cRv-7g+o+w<<*q&?MlOYL@CFxlH-x6)?+mc4r&-U--h>)P&})iB#uW7DfWXF?|0
z{BmmCTPF3vcES|zJui68_rAH^XyfvAu5H8dEqhbswQMt_|Jfuc8rqZuiSI43Pq7tu
z_O$cv%eI>`Y3JUCC_B5m`Cs-HWE$HxWXalIlz*~UspafmsiOyN8RGeO=@we;HTb@K
zH&?-KTaO3HwlB2**eLvF-gDeg%+{mkh)uOa^Iis#c-tIh23w2S8*G=X>faZz?1gp0
z)PK7#?0#h9+U~HgBHwY}v6UP4Dn6TN$93h9t&8>s+Zj%|wi+pBwhmmUca<=0v3a3<
z*v7G@Vy~3W%DoMW=WMmOC)xIV-C<jFVB6j<<Nda3v-I{}xIEdqLSDhvv8>1T6l3k4
zDV0Kd8y3H^*?FdQZ?Z<^z84N&d;99m_BQ<Hu=%quckkwB;roIo>e(@f$L-lV|A5UY
zu4T3vd+yt$ZqeU6J>&D<y!G2{HLREHy;iqr?*XlpJr2<m_C)BG*c|BpZT)8dLE9LW
zgSM7}%C?tgHtbFL-fsJ%&eHbV?Bcx@XN7D(AF<eL^p$7tr}quklSN+GG{ora*>c3g
zR;TE>?c}s`d+sb>Zj)P7yyp*Fk!|PS6ML)cPVP-!wtMfFX}9+DKRCA6A^*m1j`AzE
zZ;H5Vd*XR)EhL`oVXc#}TR*qUhE-?LUjGAPcDyP#_f#MHZ>#rF!Pcd^(`HYtm+h^f
z*|yeZ#d{+T8SFi>e44HP=fpj?_+QxU{L#9X<MVNwQ!n`UR@m*>D|i2tt(oU$TP-W*
zeHv@>Y$TSfv%Q@ayjNmsiLKvy2Ak7N2lmF#YO%Q))NK3gos!*!QpLU2jSYLf*UaA?
z(0_N2T~OBEJ8K&EOfuND_eL$hZT^i-dzq!2_OxxP+Pf&}%-+9eSM1gJ_Q{54g4o^+
z|J61dmn`00eTCEZM9<Q_Jb^uX*nOtjm}v{!F(@pxtO#DU=fnG3Hh=BSZ57s9Sw~wg
z-&<m9y*KURti9FCe^?i2N!ywpYTKh1;JWwY`jvZ4F3q&{Qn23ZbK}RJ8>>_Iep8uk
zo2}}%*HUqYO~cZ>-2#o8Hce}z_DZTV+5R$T+MB)Mv&|9JnR^z9rdq$8khS;5AvxO>
zB5bycGrabOq-EF^oT}bq!B%N&Gr8LK_sh$B3=VAD!_dyKS8PtVt;JLOJqkBWt(O)!
z?qQC<xo6V*3$`Nj9Bf1HJM9$+pS`D|;l9m@TdQrP*xB~7rFQO_aWQ=F{~fFLu9$ac
z&-_)NY*%Op+sb;m>@93$+8bs$)AmoO*WUUW>ulF&^4d+w7Pn>S*V=vU$EUp+!D4p0
z8vc7vWN+AegIC>l->hx6K|G2!3tcYk6$py4Ex7R3`UJP5%@560wzH)q?QXbA?OpMc
z$@W5R%-$Uy+<Sa<*X&U@WwVjVox3;tQT3h-+vB$3H=pb+NR-{XY$BUo3)|+s5fP$$
zW?YlAP1$<Hwxw0gmQi`L&5VGJdo9WhY#-hhvb!cJWSevKfbC2-UOSFoi*~J=%wu=*
zX4hVar)_&LK5n)C@#fr~6CR~|v<eT|@^P-RU3hWJ-Z1vgy%s4e_HNjeVq0)r)0XMd
z#J%sbx9*iW%D#8QRpmX2Ygp{2c!utsenM~Wf5)D^tO<fvOg^t|=A6v1{k=@Vj$y$~
zD}}9d_LMBj->WM$(PrQIMq7_t-}XFp@ZWo<b^o5sf`@wzHq5hK+}dh$Z_@m|8e*qy
z!en^uSk?>K&iGMo8<1(bXaAkfy#jn^Y!sf)-76s0XY=Oy?7e;3=WPF;Sh2UpW}Z#t
zvnRH1{Dk&y=$^C3H(g=xuBwE+M{2L^ap2(HbL7M9y+ucP?X(v?v$^zC+jj9Uw!IQZ
zt85)=9_*QSfn{$OYx3SOF6q7U8<*{EviNFae!+il^2xNl5xfd}Jlv1&y;xAUC!zO<
z&5QVqJ@+`y?w!%{*k*w!ll8yY`!>5KHrlQ-+GhKt_x7HQU+K0tBwp@W#?ZNUMt0#I
z0k^c>70H%+9SrYSO)x!Ry-xC|?RTwZHv3}B_a3NWv^mM$XS;4~$lg0Q`S-F&O|v-@
zp>8en_^FML$CJHJC#~J<R>W?nKEK#z<AFVU4ISt14cPg9&&d*A+Z>HeHVo<?cAlwk
z-J=lBU{^4O-{!%5bz7;YOKsD2X78C`^V&v3KVYxkUs1dF8g;gFthU?spSRx2A%EBA
z`NJgJkaku(zov%0niaQgS;8mSRKDxnqpGB{_rdy9+nIKo_bRMS-JNyt_}(KW5BG5E
zZ`gZP`t+WUxtw-t8TxxwoD}TVnMm5*PUf%^ey_gwz~(8|oz)Zft_-_mdwBgJ+W_tO
zy(OPYZKv2O+ch*ywX%7>XiqVp=iXVrHrk#LirSlSqr<jAqsP{5*POj<TwC@!r>@xh
z`~#b9=hlt3r+u6EGFW8Vy5CjW+wy7u-dS6o*?gX|Z7&1cL0iKsPi?;bbloeXb!*Q`
zgQ&gY-9mOAQ#S9dWq4#;EVzE}<;|aM3s$D?Wzt=`_vQ9OdzY*fvyCy^Z7VCxyf1a>
zZYz^@&bA9e4R<flK4`->U1aaHGON8SH*eYNVXd{thmCpfjaeFdQ(M*d?)av$*J^_G
z-v5=6dmlI*u?<cBzIVY-n>`N4-|lhQ{d-RU?@Jpi?~=W0YZGkm`!v}Gt(vfR!BJ~l
z3v&)Thk{c!8v@Vm)nV@1`y)QVcJ)@(y&iV5wgL$UZM!0-?Pc4nXs4_tyU(jPVegJT
zH*BmsZrCJh9NBwb!_xNmuETp1s(0?rSk$%0;pigUj-x)ddoH%^-QauJwyHVJHtTrb
z-tY_jwlkJV*)bSQv?ygNvz}1gV9T{-yRD$}g1rnocy}3m+_l$ZcZ|)I>TR}b_LtkL
zolvmrl3?D)z{<P3Prk(ViR^US$f)wYH`cteaZr9|TNEC!S7VCD9*#{{ZCI8**ux=P
zz2|8okKGGx7Tc!8t$SYlXWwU3mbEwIeUQzJ7cF}jR^GI737oaZLV3qtiI1&&TNYj4
zQy5*a*GSQP?~QjNdksP->~+*#xM#zJuQn|q2W(ILF0h?w;jnia^WHsmUJ82~{?6R3
z^>OQ7lgMp*4~SY=zh@J-yRk}i@4g=$dy5hk?YP}n+b;UhXS>7C*k;B1hc;`{yliXS
z6!tb0*IGH$w_1G&-D?v!@#XFf@~wLtE{9n+OuB8O*tBG?OaAmd4F}owJh^;%?+q2L
zJvH$m*3*ji>`i&XX!}J;c(2e6VLOT9e>P|S7VOoU*tM7Kvew@43DWx*r*Gez@g#81
z^$k6H>#EP$T0BzN^C8XDw!-oA9*51Jtcui__I~N;-?KmR(VhUE={D|XPVYT)fWcPz
z>?PY3*H75&ziPKPE|7ok{r{(J-?B{Id(xD5Pl#Ul-nmN_?K!IWWlw-zpEYyTmc0wq
z-`e=??b>~2ZmW&X$_aa)d@-_>3I1%WuuFHhR(HXkf4O`1<e7EuZ9G5E_UDWPdp8<B
zv*|n9XEUAE+V-af!@gw}{(D<|oA$;tzuB|U`pKS#<fXfmeK+pi$7s8kVW!9)saLvo
zEW3GaHlFg@ds2+eCZ>49-XA;X?pd*xd+(YD#e2UppRt`-x_R%SOQ&ojG^%V0)bjT{
zUa7g~apfBu;Uo8VAAZ7Ow>o9Q-jgvhdylU#w0%-zW_$X|`Mvq4KHD&CePrDvInnmT
zr>8atpP$-$-lAylgw$`gH$vrX1*Wat{WQDXHgx5>y&Z|0Y`?wXw4EID$7a*v#d~J(
z7TQ{0WZE~uBW&+VAB}zcpSkUQ@yEh;Z~Z%)T?!}m{?6pF{lT5QcZE)}ty71Ft&n1u
zwfL5K)@$NzZ0+9K+V(uq+8cCc-rgrqT5Te3T(>!M<?^2MpC8&@b2i_zZne=~nJ~e9
z&Mp&deTy1w^BkG>Y363`6;Rt?>(q4Lrs><^z3X?K-|KvFq0QHg2W{rQcx82GecRrO
zS8RLz{+zOPiet4~w|V~FzUF1NIdfU}ZmJisYcpKAm*Yj4b@~=V+Z!ya_ckyl?NL13
zx%aQ(7uz)FMK(Vxmha8bjJ0{Th-qI~@JE~X-z@j8n<BWEJMf*2fmW;ab`!_F_9ahk
zX3Fo`n{<55UeTB}dz~Dv**KRc@7<U=(?;8F_uj)FI`=YM>fNiB)w0*@Q^8)RCkyvJ
znBKI<^Y&7ktM}~oYIOPRP5Gl``^fD5?$--6_LjJX?fqbFU^8c?zHRE8JevTE1AA|N
z$lZH!mi^w!#XPouoO*0u&(p98XWhB?G4I*kyQZ$Ny=T<2*YaGeb-7}y?Wa@AZFl_J
zvbRHx(KaDn-u70<*}aOJO7{G8WZU<B!8IGHKGwajH!19G`2NZ|+FyE~pJ0%!!_KTd
z43=+pKZw+@O=?+YYx#u9cJK6adtY}N@3p<P(#D}?ja7hqx$Rtg>3s(r&e`z){J&?;
z)b)Fp{${pQRrs`LN$xvaorKSO=6g2pmB@~?o!g_o_tN`YdpyqU+k0+j^WNr)uQq-g
zPTLmT7qD$GU$x8W<5$}o55)InA9}L);nxLw7yE46dqZaF-h{am_GG@6*?ZK+c`t*0
z>t4gp-)%DFCHB1iC1PjL;Jjz<lVy7sEo<8=Y1zEz2*0V#WZyK~nk)LYo&4MPnlYcX
zowTWIZ_M81wxthN?Y*YeZOgXui_JC0`!<z}pYQ$nRDRz%)lGW^q88h>JeA(tP%zDQ
zZbX>%_lF$&c8095Wt|zgceOLKUBWqQ+nzGseP6St?VWR4#WqXS%+@7v>K?`%@qIED
zulH0)I_{O)#j$tZ3k|zXf(rZm8Mo|BiE7+4+w19Gjz`V5cB?+@4fEP=%~8B^&x7|f
z_s&ht+}mCKXs<#WqwNLFD%+PD6Kx8%<?Pw;?6?ib%bq>O$2RZI(3rB<^401+49*PJ
zlT6-PcS&*DW}iB|H{$j6y|K;Vdp#y~TKC-lx%Z4K|K0_24Qzd+I&HPCO}G7XX0@%&
zlh(Z^Z7H@hS8&_j<=wki^Uuw_IUWmaU(VQWYjJboUXc&-whxvY+je;zvgzd!w-c+p
zyZ6Znd)tGqO?ywo#M@4YtFXNgba+or$NRmRc^CF31kT)hLPczE@|hER8$|l{W~}13
zy>WDc)#>v|drx1t*`sx6&Yq0VoA<m4nYYKc*md`wQ@?F@FJ{=gA;)cRtZ|lYaP`8y
zY!bG6-@3N$o%7McR&#sX9*3U!dvsrK-*f-*j=hE42lleq=Gv}b8)v(3#xC1Wd5ib#
z-&SkuS-s6>%gc3pPG`K?`=UD7R_{od?dHGo`*KxV_8t$=+oRNCV{<{ydGDO%Q}^~R
z>al(r)oClFTf4_*-MhUEAuIMivt!<S<V>#Zhmh+wIU7Cpww5aGz4nE5FPCbkO%0Q#
zEuYnUn@2)Awqk}KZP$xy@13$bWAFEk_Iu0NC)>O@s=9a4hIE@go~5>pR|9P~{Zp~s
zlySlK-g`+~hVZR>z4aMwSN{0BhwV+W?YH2|dpFGgVZEbi>fY2mBiobf_wFs}*ku!T
z_^xfmeT}^z=APTTwXw@)e_f%iMKJf?+L^O#4_a@rJ->gJ?Xs^6Z4=jo*sNLCX8SYX
zz@CRwj_&p6-@4bM!Ov#F7E2q;$-iw3>YwdpooZ-Tlp<xPBvxa4^TNu#TPJMa8_}U=
z^I6*0*5r(k?Wb!Ud+(j|-8-Q}+{P`#aqp5?F53l5*!Hz0wb`~O^6m{hz`XaxE&&^<
z^Uc;jwtTcnSA4N2*W|+9(5KqAMka-Ot?Z<1S^wX&{oN#Q``Sy)hV?h=o)x*Wd)J<b
zwoTYE!zRZt$2La(+V1W4g?n%K?y-%L_TT$I=+544yl#6=>wD}Ku&CZMgJ0Pu?0w)~
zzgIrC5<ed8oy{S+H)Qin+lF<nyE4?B_SDL7?csS^X<KeoxhMSSMyokJ1-2}#NB8C{
zt=}us{daHs?n$;EB_nMV9a-%hPW;-Hk+5iw`nJv1Rt2YRtKt;*g_<hv%~PIg+aYzp
z_OEZ_-ia;}`|KJ_?0lu9_PzOkV$ZA<Q}#aS-)XbQv1d<%m$a?*m1-Mt(;8cbQ_QyU
z7j5=>o-?yyN^04ANocXHwg1H3Qx3e|6UuyiuS0CdUT1!0J06EH>$vwb_c9;+vNvSi
z$KAI$xNUA}o!t9i?kO9NnufhiJ}kBm%JX*LxM^o|c~aKi*Zec~YMyi4d+Wzi+Xm<3
zyH0)BWBXucg>8ejr)_}{k8RwXWwzb(d2H|Z`|O=P?fRZ|yQ}x=%s6FRP&s!m3rmNM
z%GU~8Kh^1b-~5uW{hG1L*5&riJ%PIq?kzLo+3UQ2lkLHIcWs_jUfScKBxaW^&Skr6
zqm12hOM!j;ig))~|8&}WcgMWFe#YK=CtN#eQ&7Ff=AO$d+x8a?HWtg8_q?!gwF$^e
z-?MMy8ruzD`)m(wJZ<}Flk(o3+;jI9=Ska6U#+_LbA#I6)pG)E!+tjI4g7U}FMC3+
zt$*GwTZ^3MHm}`O_nj(Z+>?~-u=nJhGd44uT=#lJZMM~V(Qlg&*uSUZ+}XXG)DPLJ
zJW;THcI&Kd4ukLBDFunPF;d5E#jK9(iFlW|S0b0m)}^-F_EPQdy?f3i*?LURwOMlG
zgsrmr!M(y-%60-8i8ikzf7&b;pR~6sCUI{squ{>g$+mlQtCsAwD_pcU!g<bKZ~HTQ
zxA?BwGa+r0?E#TRyJsx&+j~o!d9VM0#ytyKwp*vnW#7wiLVU;S&r!C^rFQL=Dv+|9
zpgMo=#|zzi7jUcDTBXn5JICSco`%SYwim1z_ez|&wdaGzNt<;cmV0BAIP4f|XYX3K
zka@5Bi&cA53fJzHIC#v~I9qS8@|Ep-S~yc|gHK+x{V|JUPrT5{y#~=QcmMcRZo4bw
zh0PAj_j`XwuHEayq;I=t!g<>X9X!@rE?u@lcNgvn`Xykq?A@%r3|_ab{SWKyy=nW*
zHf+)w+cldH+gi-z-+R$#_a504nSDo2-P}9lV3}=#(Y8G=O1x}+g!k+{%j>xJT!7x*
z19uPYiHu>h?fbIbHfm|e-X$9*?%BfLxwlM3W^V@9hdsLj&+L6|DY~yN@3D159rLb-
zS9oo?-kr8BnD=Ej<LeK56*#`y%&+IPlbE>6`o$D|+vlb?_FgIo-76xHvsZe;tKFiu
zjC(&Y8QQF=JYsw2;*Gs;Q@U)I+_17OzPx6yo!mp)gn*gWTdL34{^C{J%M!VIZ;1O0
z8!z>Rdl!5D-<whY+Qw*!(B6iR>-S!ns%R_W8E^CJ%Zj}}k|x?danQ0o?frA_ap#1+
zlea##o$Y#ZkHGr<yBT<Q?)ezFdGB1^3wzt_7uzmnE8Y9&{)s(e4_5Crm|$rmb1%lW
zy|Bw>j*<1=KzoLLDl&4m*Df~f%?^;ZyBjrmZ-de^>mJ^|y#i8QdncT|Z>_X>pDo9x
zLfaG4m-alE?PY6oYKhI<4O{l^T<B;map}$;4|jE2Umq2_>(_SgeP^-QR$>L`Uhmp-
zd$)<Q*|oE#+y4I1vL~@zb)Ui%*1a~Wm-fs*bz|@14=49-V7|NOmEy9!%jTT6Zg?eb
zyI;y~Z$a0Z-6uW&>~;UwvFA*-$lj=SpS>$i%-%CgXtAwE{07?uD^;s6XRhsuSh&@e
zfAhh;_t;O{I_#ZieWI|=)@c2fy>E7(+4JZoudT|uo;~OM<@QDtEZxJ`*t18apv5*)
z#(b}Nx13$a4_n(J`GmbEEl=;gI_KbCjf{`B_b0E|E2H3SJK^~<+b5lDb_q?FZ5maU
z?s+V>WbdI5KX+gJc6jfO>;{_&OZshY-b}FFyfS33S=m`zqmtCUEYB|3J~*eeXWilh
zd$YAZ+jOW)+Fi|=y;s@UeD9(8|Mr~qeq$ThebSbJgT-di^G~)x9KE)aS54kq^R#o%
ziNB6E(|@Y&wYZzQ_Ynu@-cL#ydpTnF+T2h7y(>7lcFz)n-Fw21ytIw{nrAEa@1(7T
zf2D0&#s*u4lfk=BG&k<<+TLq>tYxNcocZj%>DNVU55yj}-Qmk_+x}5@-vP#sU5nWE
z+a6yVx_8IZp1n+MEB7+IRoKOFmDM^_G-=Pp&u?tsD!kgOcju6;#n&Tyb{Wd-)oG62
z`{?{kTM_-)ds&ii@2POxVAE?+vUdr~jJ-_{)b@J&P1>tr{MP2&AIH7d4+`0||GK_c
z!Y0MW;&#xU9kv&248lL!d}@o>>!cxNXJD&h`^^5}-U4}JTf2iRY_)%s?%l9v-tJ%L
zm+Vc@inW$HzG`oXlZY+HUdeq8ZcMfcX9aC;`)2LEw6EBvZuvUfhMUE<52oy}ZrI(h
zmrb?9_Ik-y+l;$sY+wJ4-7{fvhxJW%cDpBqEqm{AcH6EgHnnv)SiL7EAkOx|n#DFO
zMaH%#>XdB0lyBL!;!Dw9jW~asmie-KZ%v(Vd%{@JmOow9j^V=_tA?GIcgvK_+WW^q
zX7302Rn|Y0FWB&<3foyc&)M@r{($Y;wTiY8F-Cj8{wUkaUJ+_r^JbOpJf4ZR8CC{+
z7=qMopY?3A4XN(5P5N_g@5hN7_8R_nvYqny>)uz_+4t(N{=H|#&(^(r?8<FV%u~0W
z`?Jx;LhFW&%-s;%^UiDc{tBOO8z0|j!)Rn{BY3-NZ>>bvo`e@swo?MKZC`8c-dlCH
z*S2S3(q0CU<+f8kS=t<6T(XzpxsMg|?p1rQF}2vL2qf5^$U40z%8AjAC8F16+oaUJ
z$5h<+D#Tr~ac3~LeN?%6@4i10dmC;@THRndxM$xasl8MBpYOU8HOKZq(=6+QK|Z#M
zhkn_Xtew3lqka3HZ7UjVvkGJO-cV7sU2<rHZB6b$+Xl6Tdz*g#-ur;xaSzMSn!OIL
z`)qIA4c_x-d(WOT5oUXvLNoXF>Fe+PsVBF0!P`39f<<5V{8`bsx1(i^?RM#;y#`@(
zY$tbg?Ukt8yVvoW(w^qm&3i4G>i4!hT4Nhhn_>INe)e7+#-p~=W`^5-S?g`P>eo@*
z1xyY$x#ElWR;V4fUGl<tZ^uh<n}7<by}w+K>`fM|*?VO1>AefiE!ex@xSs8Y150-&
zo;<bps@&1N9;K&j3fQ!4FYKSYN8pP4-WEgoy}M-X_dY1)*?S_r+E$_Nul3AB8~5rw
z@7X)UHgk_hd(hq^HoI&T;u>t_gbwdLz{9(D^Xrv+zrCHdcMY?utxd-vTZe}owmi4h
zZ8H{6*!$$?<vo!h=l5p(-(Wk(t;zOEScPqd6xW^|`%l@*EH1K@xc+!gOVQdrWui;=
zvS@AGd+5cXy&qDnZNv<x>}_~_z`BErbx*)C$-U{k%l6J%yT|ra``*1XPB_{;Nc7ty
zk#o`Jf{E*%lvsb;DO1ka{+RjM=F_X$wx@ng-}|C+pN)7;_TE)oU3<SBZm<o~?XbP`
zih1vz)9QP<)EoBhxV_j`L2~(?a+{{T)6^RGHq7hWyV>1yFGuu*y%ia<HXTp9Y%QKY
z-?Q{@>|P1oDYkXVZF?C+PVD43ZfN`Ai@a?^>sl+fqepB%)>`a!;oE8}aB$M@6(^N#
zzW7<$eq-Hft0dIBH*R&i?Swvi+qzJVy=%Dk?d|zDeb1p4J8U=XS-UqZXtyoHu0@s_
zrDZl;mCSn+)*iF5@&33c<Bi>(2`N)-XH06db(yAXTVs`FyQDnDw&i|>t>2H%y$K<g
zZ0_9MZOc=6!FHPd5?h81KW*X`FSaf4zGZXgN06<HO_wc?mXqzVDf?^-tZwa*7CXAv
z<l08tb2n%0wMewI*&?oEJK^HK-3;OHELj#$ww+;Px<}*pJliEd<o2d!wC<h3xY5?a
z)XA3R`D~k%9Tv8G{rdKLMKsw;7$(_nt2t(S#^BnX6Af9m9sg}@9QuW9qGd|<igeB2
zyTG<<w}();EyFXjy&E4!?B)5m#CG2W`@Jf8)AzjD)xGC|+MYc!9!vJ-ST3?jPddD}
zq(;M5dEffIAD*wb*^w8!XSG`2-UGW5cQ<Ujx9g1YB-@U3mOb1L_v}^Z2(p#1k+7AR
z!f9K7`PAMs+k0(9a?)*oZ0xeFv6J0<$M~G>mPu1>b$(vhlOQCr=R@*;n*h;$HZjIb
zwj$OgwtlzTZO@#ZVPkc1g>Bo@lY7rRQQ7;(cHN$LZF6lKK8EaZVVbhHKuF8Bf^Es(
z3qNygbS|^)ozQr2?}E>qHVkVcY%WZT+{3~?*~VteB3q8e&o)b%7w>ry(O}Cn|M(t`
zGEQ3-*{!xG6e?{`zAf1+^zNMPi3y8s6F&D@KXKe*v-8vGy#YFIHamXn?|Blq#I{06
z-p0s`VXwj+&b=}mQ*11*?%sRmY@F?e#S(jSUToT{^8L)7CGnDbIg(BH#$5KZ-M4dr
zE!Trhd#gMb?JY?@WV`J~m+j54g1r}e4(^?2f82J8a?c($snfQC%Nq6?)SB%%uv*VH
zV$tHg2h3LNaVT48+xlK<Z{rQqy&0=rZJxO;-+O0!neAPsy1gd$Ew(4rciA>9++=lS
z(l%R_>6*3*(Gq)h>nzzDqsd{bBj03e;+$gpg>S9Ri-%WjuFRig(|x9BZ^vaf+uu9V
z_U0Is*v?Q{YiqYQbZ^0i#=YMRrtf{?)Vqi0#Qr_=7Ob;nKUlYyBka44{hjrDZ)o4&
z^J3!Cy%x7P_b@Esu`O6Hw>Ks{ZST^&b9)&k7Vc&UnzD1-+!otw`<nJPaGkN57dvq;
z$GITeB}ba~Mu~Rq4RT(&S7}|x-h%Fvd!F>q-^2Fdgss%x3$_7;2W*%8y|SnEhQ;0u
zKQe5Do*%Gf<GirfM>*8?#9oa(32)7IhZOF&O}fyqcbC?-y&GD)ZO;5?x7~4SiR}l!
z6E;su+xD7SpV(_)f7Uvn@3xJG`)QkkQ~zu{EcVzkO!#KQ5OZda?!6;>?KmgcCUl!x
z&)6Wf_ls|nZNSb2Hp**f@BN!~Vy}py*xox2GHexC*!P@io^JcCEOBq2dYA22-r0NS
z_zBt;d`-5Upv$}Wn%nBVErKie#J+3Y%V*ZRH=?}5X2OrFHVJA@wp&CSY#mOY*mYy+
z7TW_&JNC}_d1CK2)0Vx_XLs)1FqzjTBdKz)PtPga1w4Okjx9Q{w_%mfo=N+H_b!=t
zZjZ#CzcwMAt$XBZrr0*{%Ua(%(Yn`WZ}{G*`PzHmSbf`LAiRDL$GyXQW~|-5mtjwt
z?G4`zd!)>k*{(M1*n2>w)HX=gY_Ek*fK9>5hc*maFDzLuEZ7rL;A{K%N!Z?~T@riu
z=v?39A@y&M&2LxRs+kUZEh1U=mKZA8%7jm_iP>agThcMfR-s~s?SmaEd$!J*wf9`+
zVcQoQD)!VxblX-kOxw$MW}z*ko6lYb<tNq)G7i`-xR|$xA*{&uM~dLyq@NpXGd2s_
zo;j&+8*9?EH=|W^FVEz*Jq`hhwi&XA_FQ3jwda7h+a96tJ$tQ+^7mfZpkN#GXOYc|
zi!HV)Nwu~mkNNj*`53bI&)da&3IrZm&ta9>d!t&*_QpRa+aLb(_BOETT6?f|*#6iy
z+15{%cW=y3Iom7K!}kU)Yujt`c$)1I!*E-Ljq7(i9ND>h#mfTQ$NCrc_8r@0D`4}%
zCSc(mn@0yl_WIn^-WwoRx93CPr#&I&&3g>DF5jDU>ik}fi1&LOlQ!&adDUQR@o>Xl
z6_W$DQ+Oxs={VcJ$LMOa?SdIqwhgA?yEk+f?=e{X(nf>(=$;8po;C)eul988+PSCb
znb_V34=(II^F3~F4|{=aU#R`w+zVlQk36-roueqY*Wow!Zjq4rwgC?&?$+ea+8bcJ
z$ofF_;@uAS-E0L6Kkt4sY3JUohP1sK7TMZLG)%JTklVU<htNdZZAxr=gB%v_^=AIR
z*WmUR8=VundmJu`TXz}GwB0sGXK&|(Wwu4%dG<(tSYoU1ooN#hz0SJHZOh*LY*$-{
z78YxhQ-^JNb3FGLHU{k7<2YgOmxKwnQ`_0>CVXwT*?D}TE$`W$-8?(l_bD_#vgNfo
zy4NGRchCEH2V3VGd3${|H|)*0&$#c%RXf`Rc2nCoUJq;&TDI?95n*dHd-;~V4r}My
z?Ckq*^GNE<?&IIi?EQB^c5jNx2HU*k_j?zG&a<5mHg%8D+Q+u>Q!H#toX**Lc~7+c
zvu)a*C53x-wYna&O$ch;ec;_ME9FyPZ00Yhv*CQQ-S*wYDSHzpChdv0H{AOrXz?E2
zNm8~2@Ap{e8QAZZIVfh^lO}6-Aa?d139cC1bxtpBUNmXh{@g!hZ*;cIo|AWP*e<=(
zXuI7aZSRS4Hd_}tee0)7Hti|ev}^CvHFs=&SMJ|Cg`>wd$(4O?QPYgQPaaLMx%G#`
z?sU44jYChfjob-g+p0QYyLAy~ZEfN^Z5iSZS>4@yVef%A%r?^=Zrf{Yw`K24hNF8I
zE7a|c+t0t({Ljg~ig%OusBPW1`?bx=z54|eY-K*o*n9l)q`iC8B=)5oG~JuVsc+ZC
z{bX<R?%=%+NnJJ@7!+)Ls@eAXByY4$>N;q<_CmUCNZ1Lh42|?X^LR}6Hf))?SK^qQ
zZNt8ewzHl)+MZ2P-)nI)%Jx{s-90{mbv6f;+4ufv6}L^0ow`@pOKM-r1WVgHe`oAT
z)2!WF;E--xp|X5$`P`efcTX?cdqGFqhOv-q-<^7S+qx|UHs;Is?A@}p*XGQeOxsF%
z-@P;C{@N_|_1+s2Zn*b}{E0m)zHZ&CqIG}o+`C14TX-~V7p(oWr;&ZStwZ5G+dFp|
z_WJyuWAioe(cTw<WqYGcj@ugOS=z{XOtD?^iPdIL+iIH^TMyYV$Y<CX?wYr^VBd#5
z(hZe+r|c}<tMb%%FV7y~ea$uVZDO8j+Qz?P-S>OxlD(T`|Jr1fPPAdDcC}WzaC~oA
zTEX5Ii)HqHk&D`^!@hY>U$v%<xL)yIhsM6W{!5MaUh&r7n{juB&B+ogn`OsZ_txZ8
z+DbIA?)&zXZJ+)ABHIch|DBy5m+o!rpT0Nt<D|W-{~p<Mfunctn!Qzf*BGC)<%`~7
z+x1r7PT=33Jt53Wwk{iGY*-DMZA{*M-RmUq)b@*Nv2AO_!MznNb8RN{EVOPqxpJ?T
zzv13Bb^`krOr2@FLZHBQN?+%moKw?mjebAaTNHO<&y2(Fdsz&`Y&j0C-t%~6@ZRaa
zm+k#?B;57@?|!Qd1(WtZUwUNkAKtRP7p5lcIm<kI?}nV#-HNLo>}^h1x%cO$$h{r~
zMYdv>x^_C`u<qSyC9|(~s;&+11vy)V-zRo=)IZ$Qvr2NW$?MB|LerG@I<<G$vU>is
zJ@C80mf@b4RTP_yZJ7TA+odLx_DXc^v+Wa@uy@w01AD(TE7;mB*=<|gue;Zy@}>>n
zt7h99Ga~o4bST*}Y<{$7$IU}FC9V7RhHRN(^EDuKZ_V{Rdt6>j-@A71k-Z&<>TUfp
zSK7vx9^LzKouI8l_?bNhYg%?Q_AJ>eDB`y_;O!NgG~e&GWmoswYHxbB$956BUBFT~
z+Y^G5_e${2-!tXIn!V*WLihIls@=Q4RA^szveDj@_sLfB2QKV+!)IW-R#|S}6HzYP
z^<w?D_s%`FF<2(DZ^4sxn_brXY?JQHwH3RsW81NwYtMrB8P+Vznf7Y13hYa~#J%Uj
z<ps7ZDzo;6IPvUD+Y@7Z@5|r42ex0`HA^nYwy0Ld>R(di-e+v0b`h@MY&k2=+dR6^
zV%wv(Z12Q1qP8*O^0sUellR==2(lHr!n|+MXKq^w(=ywFlmB+do%6TeQ?YjM{SNoN
zhn0Qzve(|)yTMd)-?RPaY@;m~?!CbJ&i1qacAJgIkL|r3_h)aJ)*ai>;?;W#Sr_dM
z6<EHfb*l0{i&HXtZ@oQa`*!P1+t0a&_v}q8+FLQ5#nxiwSDUTJ4Q<s;*X)f=JZ7V0
zx!U&btaG;PPGYv7!uaeCG8Wo~t8U!eu;rN*zuTL=9JP6SiY9aJU2wyBx8JJ_ThGID
z_Ed;YvK0(IZq41Oxi@*a<UW%dhxg1}v16~oOr|{%lPqi-&L6Y>YI13h#GX~Q9eI;%
z{$2Fk`{4Mly>D{rZGTrE*}H7{1nZtp#(RoBY})(#j`H5W_GPv`<qWnA7k}+Na;43-
z{)522BU%!+Dmshy*3@wCwYogncGvrLwjL=rcfN~Aw>h9;vR7`|n>{Jp`u8eanZ0*Q
z`;|S%i{I_N!=PX*Hv5RJ)i1TZ3J+uV%rUmIU2tvH9+49@dpEEL+Fq93zvsonQfrmC
z|9eGM-1Zt&Ufvxt=bNq9<>R(`M^5d%Rc608;|7CG?SJXLy?R&nYzTd9!;vGtZ|0^=
zd#{Qq+OGUGYwrxse|x5D^Vu!YxoxYwc=28b<)ga;!pdw?KJ3_2@xfpZPs&Q$-4EvP
zowT>vHv8)BJ;iQJ`%Wf)u-%g)VEb(GZrce<YHYI-j@THf?b$ow$E3Y$`P26pylb!x
zF}l0=@zx!Cc>V9~HBI5#*TKxNFN5*4Z3Kg=&61OM_dNICw)e@jg1zS>zSz9kB5(Jj
zBzez^o#i$h%A5B7IMTNF-t_%eevTo#D=b-US-fIx8NPMc*uGe8J0tY`p2~MRdxiEs
z-Lqz1^BxW@R=Z5?6ShYj8uuzp-LQN2oa45p@6Goz`^?@eA?{&YlDc5eihJ|->I=v0
zt$kRt_swi)n@P`^_tnU3-}^>wx@|)D37ZqALTtElckhj8%(mSS;k8F+^?utWc?<R|
znJaC}@w~^jJdn?>RiVx1Tu`fR)qBmoY<@y}`QJ(IJF-9CHZP0YR^jX}+Xmkdt1h!8
zw)s(RdskOY+`FY!dGDUxVYcNO40Z|U)ogkuPu%<9&)Gde`qOO_cd_j&nmyI_(Of>e
z`Wyz^s(lM>T57%big~K;Gn1cht8<ih?}ZiHY&VpI+cI=A+T0Lfu-he3yH{cJ2b-^Z
zX6+SY7T()2b*fDid%JB}fY#pU3l#SqfB$Ij4S)8%4>nA)t%=-kD>eCst--WhTUG;m
z8_o%f_w@f4w6oY^x##wCg?-u=U+pQ?sM^~u{ba9$z{}lJ+L!F@-RQD6UQ=st?BT+_
z^X_%o-uUWf`$MOFPm9Fly+ZZPwsWm^?RB2SY8(B;e#e(<%{Cf~-tIXs!)_~l^THnK
z)U$g(q}kevbZxfjU%ScHLnYrfE1%y+!a3j8N!D_&aKfs+5~t;DSR8)td0^vVqww3q
z=E3ib-5Qy^wm%vd?LD=z#a3t1?L98H*V?A73$yukbDiy-IYl-NHchr_5qx_XWW08!
zu92`?A);w}!AotA#Q6z(v_zNf*<p8h?@qIQdwCM&Y<DpA?&Y}CVr!tWes|4sF}u!9
zyY{}3*<{P$muGu|pWk*#Sm<8cH&ga3-@<P<BVK!t@d_T>3%1j2)7DSeD{@}d_VRJp
zy@g4vd*2jr*+y7<*j!lKYJKiw<KCq|x$G{5p0!ovwcc~~pQY^%O*PxwoICfPJS1!P
z(RQb8hDWX~!|Rhf!&_PR#pV3oEge70=Eov2+x2Gdd;f6y+kT&Uz;<Oz^`3W3lWl9d
z3T+=8w%q$uLu&8F=0;n^_mlT7SUu6Yrs?F~ioUb948b>c%@(cN%QAy&p9`OuEyJG4
zmaC>J?z<onZ_DG(W7oYlXD<uSESvXlkJ~nKaoE+}-Lg040o&d^u@`J*9oqMPm0`2n
zan;tQ<N0jcJ8A8^404<H-my7pduIj1-sPwN?@^m7W5;%J{@$9Zn!TFBUV9rfJa@Y>
zEZ$oqG}ETCRDR#9)2H_u2xr;c@Uyp#XJ4}Sz=!p=c6+z%U0f+?8za1aZxNg7-ogip
zwu&JSY%@16w|x<LZ;zJd4ci&YXKiHm9^Av?FJpT{-+#}o?-OhjeqFHnrD?ynM1GUa
z#n{(-b*66FqxoXNUWbLJZFTZnY%@ZB*)|{BYsrxNZqKHz3v6{BRM{SRp|H2@%*4GX
zLe|(Os7}~BYtl;F2R{OBFXZ31`E{7vHe#c#tvlE1y#<>O*j}mLxp&^0cH0X#R@px5
zJ!AXj;9J`WjcVJ%vz+_RZ8O^|5K^~ih2$LDH+MJgy>LIvHsGzvp2d>3d-oMB*wZty
z)V6~EpzXR(3v31bD)vV5Ywta_>8!25+(~<-o;2I8m@jCnwCU7d|8p1j9^g;hd+6);
zJq3@;_O_J$+B1VWZO?~El6%_jPuR=A^<wY4C0}h0_f6azlp?Y>ncH^n`AormPh?i@
z%}+YCm&KRg&S!F^tuKGUUY=v;Z9_fT_Z<`5yhqUW$KJ_`tM+bS^R{)mq-|AqeZgKW
z3FdvPRyWw5Dt%#7u|?R%Bk++8gD}(Hh3aB<F7IaBWJGMRQMk0p+N8+YW?k>%y}_~Z
zdsVDT_I#ZxW3$C~+FrK{NA_I17;O7%0izv{-<>@%XWRFDp0LXH-!ZklQZk8qpR=sl
zJFRT3t)sEpUZFKJZ9Di4Y=pKm?&ZmCw>8LFv?rY7?4ET?*6!`9W4Al3yxCSvzRh;S
z5l<VzUrY8D{N~%cXJ5aqxq;api32P4a-4{<)k#}y8*Oa4H*U>R+nQVF_V}2_?#;N<
zWb2<3vzO;}-(H0tX=|^_wR>YuzT0zpC71O&7wNrTYO#BH_w?@Njy<tg^3|ffuO}Yf
zJN0kZ-j^-Pwtwt@?G0;D-s@o7vX`xlckcqHS$hiRuC`ScP_#44+PatF*807&3QT(z
zoG-9B(4({`KwQt3C#lMI*2K$u3THmwbLGcT+b7xUY)wN}@AWI|-5XGSbZ<k8r%i~O
z^xlH2ExT8Jv)#is@wx2`sl9veD0J@eXufVE^+9^?<rOA-Sw76#>vE^i_V*v}y-Qwb
z*t(>h-OF0iV_V(jzvss6=Dh~C8a6dip|&P^&-NTRFSMtleTuCE&o!HbRW8;pm5#Oy
z+eCM#YHiwk=))!3=wEEMPh>4^^K@746*-`2=TIhNlRdr9w)BC+UXdyv+c^(pY*|j-
z+jF3z$EM9sVecW;^)_ja-)s*o)!+T>1*7%4ta-a5zOd~LTH9iKVBK2l4?n-`d2svp
zo@tg#ZTSOZ_NralWE=5s?;Z(7MH~4N0lTne+xA{KyV3TTm#UpY_)BXSp{IL%yt-_6
znOwDI*`{rKp=<5#n5nXRC;VEv_h*mV-t#lw*l^D(**o3gr|r%oH*Fo{EA~1m_U&!Z
z+i4@DHP6;!vf`eNRtxqlSn0KQ5_7NZlgW&>9IxAL8CD+K)pASPHZCrFXGQPwJ&z8Z
zww=%wX5+CU)7Dx++xB)?%HA6%5AW?*Gk325x0>w%?gh3LPXz1)s(<Wx#ks`x%~=+^
zQ#)7N9<Ua(P2tSld%*0J?H1{78?`;B_BwDs*%KhBu;)PduDyOadVAmXOYhwxpJXdB
zZ^E8+1+#2th}i52cRaYa^!9n1cN1Fn?$Ld-hy8KYUN`p(wh8t=wgLrPYzyA@@2OeE
zviI&(#(g$<Y_=z(ZtYE2=DJs4j?$i8i)ZYW@!GWaL;B-A2BtiFef?Byt#)kKdyj3l
z?Sv<ZHnuyo_nx`rz0c$P-MvTp7wnU*Uub_`J!<bW0|r~s3io}R7=PK7v$5<w{&32^
zIhUgL+Arp^Rg~YiSLLek-rWK!`-H3&_FbIsySHby*WRVpjQf_(-np0U=i5EKp}Y3Z
z+W*7u`)wcl!hmCYORt&Q+cwtiYnZ!auYdQ;y=Q0g>`&WTu`lPhmHpC3d-m?WvUxAV
z$C>u5UJd)^RBg6?GA(oOM0t&UiTMwAOWW}5J8$h^D_`+t&lOSI15<aX@4uPKZrk3@
zWxs6o-M#EbFYYVYJ#k;!+f#cZj_B;$I^)^idDZ&+s;0E>dsfl1uPx&2UX^t5{jyh@
zY_Hd4?EZSSci;O)m%UpKZrR&j*0Oij<%M?Z!|L}5t-iUpgD-Xe(ylvuMa9MUafY{9
ztMkh5`*6N}Uu>EBKF`93b_&-xZRZv6>`PXjvu}ssU2CHgJ=VA4%=byp+PrspPR%}x
z?>+netd!rkX8r4ZyI;xfJ0C4(t808_uhyjpdvAXExX<kEar=}6hka*OZM5-A2-?TY
zcWvK<<NNm=S=zaOiM#LK^$%X`jZih-m$rpz|HnH@c52gG_x<hqu&1)NXCM2OOE!5~
zv#mqgS@tb|I?cvQS81P_;bA+Oz($*<qs#X7xvt!ML3qag(3rdX_idlMf5ETyd!@J|
z_O&cgw7sS!yHA<*to^B1iu;_mOxmY0y=d>zgG%-cd^`4<?K!@8ca8SGnJ2{e^Y4(g
z|6$0yzxGA`zO{Cbtz%Wr?Y{Sa*4~A|O7>F)S!}e@{_N{A7ueq}>1S80P_Xyjy8U}4
z(;94jxo_-UWv{rec$dUJC6j43v&Em=sQ+)@D^>7xugR|c_JNXB`#RqF@5}w#XP?3`
z*>-w`?7qBN7WQK5681Sp(`{sOdG?(=`E9R*{RI2Z^X>P=C|U0_FPGY9b4O&K%2(EX
z+go?+Zz(;nPiW4=y_-KR+ZUT?u+QSH-M(j=TlYOvWZv7m`O;p+ck}jV_)oCe`<-DQ
zGwZ*-&9Bz)Rla<AZ<d^tZMOvXJ~Pgiy)(77_mwWx+jl}zV&6%*7CX+1Li?_8Y1yun
zX|VaSAYyN78`nN3wXgf4-m~o6+|6h!7u2(F#@BuOqCPm=9No5i?|*Y8TYU=~8_PLQ
z_eSRw?XL}(vUftIifv-@*}dJfJogr9+}wXDc=O)Iokshup6=a0ZQYi=J3s%~e}2vM
zJ@X1P_uac~WdHrZW&36W{k_|Io%h*zm+$@Rp|IOECS)Hw+YhT(9h>%UTc^KoiF(K0
z>2qf83)h*tx1l)8)}*?8?<9+D`z&tr+1qJ9*i)VLY_FSipuKR%(tYpp`0ST-rR?pC
z|FL(+oa=jSO&8hRc)4=#<EhFv0gvNsCv=_L-F&ZMud8R}{v~49_vHVm*!%C5=f2$M
zEB82MyxRMdrFWmyZEf4f0qgeN%Q&@v*Op~_7>}&l=lgN~-tJS4`-KFb?yKNe-FN-^
zq`mys;d@1wI@^BFS+{rIrkuSWcJl7aJ^pS_U+dYu>Dur2p3h#l_f6;`n_e-AJ-TgH
z`)2zk?oBA!w(p_NqkTe~<o4yRP~K;`U3^~z(`UP@dtTUHFKyi?dTG<%?H{9d-<Gx9
z|E;ugPeI)8{hKP4_r<?hy?05&pM6iQPVJQu5wU*uoPD=Jj?Vr+OT+Aa_%`iJn8>;B
zjzikMKYyh682hU4>nRi7_b+$iUO|C|eYw-B_r6`tv^S#MYmZ6Q+Py*_#r8e;$-OVq
zCV1}~R;hg&S0?XiaQC+1clY1-UT6F6*BM3oj%}6MSGxC~ZQiGZy?W0V?fu$hw{O4l
zj6Gsr9X28x()J!JZ?w-XVc92C+qG|N%&fhqHCy)`d;fdybHnNOr5o4m&&b}n+hf%f
zyL}0V_wF*(-dDGWWuNokdsgK?>g~^)n(bSlaMb>Lvw?l?cB#GJUme&RGyB%Q-Qqj<
zGM4b}W1FP3Z_QD8`@H9RdroPn9k`&UzW3*M1DpK)wR`P)H`{G_$hkjZ`EHx0xqAEN
z%-yo@=5f}2v3q9jd;Q9KUryb1JB4iay&)G4?`5BsZ+|_1@?Mrx@AsOSZ`{8-Z0g?M
zCbIjgM4R{Cvv|69&$TUkYqGA}y$_YNzv{Dp_Z@w{eS9rl`;(uD?>!jCzmFl+!2ZoO
z{sXD(ANHQ``nTVtF>0T@Cx?AWuh_l~Yzh0ev@F_tXG5KRhz8$2XRD^Yt*W~A*X|#+
z(|)L9pW-ZLnOEJo$38%G-``NxJ$c7G?5dNF*qeJh+veKt+pE^Wvp?S5$0qevl6|%#
z(>}xO?e+^i|Ljw_9Bv=^&tUJykf?oc<#+CV<zcutXKTV<-7BK|BBt!x=W*!y-gEh!
zc3e*9_C7i9yNBIBaNmyO3VW3w3G9oR@pLb1_>8^gS}ObQb06ND)B9>~cM0qMYrk^$
zTe-a5doGT7UsTTIJ#BaA?9=yNYa>1Niruy~qWhLL?by5bsOa7d0k(Z1Tc_+zZ#lH@
z?)Ue*XYUu>H_dF`-fa)8_bxd$eIJK#+&)$lrhVN`EB8FPCu&>rV)dTb6OH?x@@MZ`
zcK_1eA7P>UA}2k!R|^!}yKJSwUcb)Nef2-K*`ykA*@{%x+aBZS+TDMjbN^nWvwJ<0
z8}{}DpW3T=zRITT#L2x|KAYHH+cd*IVGGN?+SqNj>TmAt-LYWJ-v3`j?dGn#WjDuo
z-Cia4+<g~wTJ{~gXldIY#k;TXwa~su4qe-beqP(^xdwJKWt#SNseA3&U;M%*PiCuK
z!xW=^;*6&I8tvZfZ}F1Z`y<W8KEyR`FQ54#>)U2?_j)mI+LQ3WbD#TKhJ7v4fA-Y0
z9^aeAI&p7F$06ItCA;?qJbz=oD0Qyw0k(g;l+GO7dv?31ZR4|})*UJad-E6W*ef+{
zo$ZswjW#(R0(;%EZTC#LtFZUE-G#kNe$TcM3f^YRwxrvZMZ{)r$4-I0hI0?tPHNe>
z_lS<7t;51l+aHpXYzr>#+xy08v#rZZe!H#SANS7XKW1yQJ8AFUA0oDejv;&JbWN~b
zaekAnpQYp8FD;Aq{{9$m!@pL<=Fs{@dq34qwrzSg&9;H5c$dSMBepijwf4GBU$OU1
zq|n~_ehxdIO*{9xtew24;|-sk{+a&0YYcd7H>3*ey(@OxR;#{!?|x~~z1OUC_a@%V
z+`Dec>)kFI`)mu2ov?Lqxw>0GWV(%%^ieCDNj-bFGELjNr}NmJh8b6^CWeOZ<!N4T
zW6g1IkM_RUz0cCxY~8;&?{#(Ey|++B$5v*`X4?P`UE4LSOZRwvxo<Ne{eVrd&w{-c
z_Q!2w_8r;F%g?iK&4vzJuXyHt25)xR{%H%hE#>pw`{T_<+mgc%_p~s@*si!<Z+m54
zw=KKHhP^G$6ZXhWySVp&z(X66RZO;fgx1)KZvD6CMM|uVXk*G=2Zn!EE{Bx%J?UC$
z+c3A;HX-4-Z3APNt<SkE+Xc*wwr{uJ+*4DeVOx{lX;T)w!gfQwpzVf~dRyJt!`55(
z&fGQWQq^98m0h;C#18L`@Lz4~qTOSgz>#OYLZ4&LUddLQeOHd!N-!wcG6-MZ#c}Gc
z?VY-5Hh-4L*fLuv+pYDRzSrm2)V=b30(Pp|;&vbNHtjvywtg?O^uoOlKD@Bi{KvO<
zQIVr{f60ZtS2X!-PjB38E3#8zuSn|dy*zbhdl#4o+eTX6w!NvQYByo$gS{Nhyf*XJ
z@bA5_DAM-Hn=Q6qqzd=mmDkzp!?b5_S$>mk=RqSIfz{7#=0`BwEj^&WSLgY|J?!h-
z_H5ev%4WrN1KZ2T_^o5!@YsH+WVfCEgv-w7-lx5{kGJm?Tf@5dbcKd(;Qw~phK>ch
z8F&JA3$0pjn;{#sr(?R)-U$85J;&1~?QQ=ud+&qF|2F>?DD4xhJij+#s*uet2EV;(
z#<%wxXcz9eaq8!u^+u9*Je)^tqXJ*pNZ(Z5cjBMS-amKBZKEbGvHfPfWcQX}YukI)
z{(BRSF0|FkzO{Fc_8i+EH8*US_ix?H7SX>qj*-#U;$E|DjB47hmMwGl&e-<OrYxnx
z=K24vd&0#s_I5nGZ!@uI)?QETO?w;C<@OXDmat`5{m63f+oOAT$fVjd@IJR$lellU
zz=D%DfklV*?yj}pwbagb?-8>Dwg+Cg?op7BvFW-UZ5t@R(DpXp?L8Nltlk?D`NXEF
za`E1{tIPJT`f$#+Bhh~k_elm@&!>%6Ro55pm3ZH>_sIvDy_zEY`}(DSTiac3w0-3)
zyVqcTs4Y*twCx9$hP}UjE!jIiY3p7~AJ#p)t*`IB;xosF)l_fq0`ZF7JuQW{ds=4N
zto7P!Gi9Q<Z4+zv-rTH7d#fJp+VhBYldaqRO|~s>d~IyvChYxEt!umQrorAuM$Ua(
z+3(oK{k^%j=f&(jLY#N^%oA*|4!GQH({yT{wRw2dUag5-cK2$R+IILYw*AuEZTpYm
z%ARYw3vHM9%-qY*TC#VIMX{}q#p*pXGQZkHvDMg0{FSz=j^4W0QS;kgH|;~V+YDdZ
zT=c22WypWDTU|iWR(`p`UfCzx_E!A&v%O((dhb3*m%Xh%llC4-7v9^%<hHkjYuBEY
z2J-t}$fVga^q#ORnz7rq>t4q0CA}(p7Zi!wO4$1E)#)~{4Qjf&XS;v4EyKzUyVi*B
z-OKGGzBlDvzpb86lda2wX?qzyy4cF~`s}^3=fqyMxGj4pC`sA`sEFFM7;W6MROY<x
zEj2+q52gSclh=K=IWG71_%E8Zw@I~aufW?UyZ`=F+&eFA-`*8vl6xm4AKSA-+RVm#
z^Ca5>k@q%iFO=;>zRs{wRF~O1cgZYk6V@erWMp^R9`W|u`}<PbUae)}d+lGU*zM^$
zX#3^5)!rM^uG{47J!+dedDdRGkL|Wxw*7l8ez@()+f=_ty-C#W#NsULjzyDfxBL&Y
zePTFy?~jeYY!+>4vh}D}u>COq+MW)PYc|#^_U={So?<&S=bg>p5TSkXZ&&Vp#e3Se
z$9J<$gND4#o5X#4!`qJRz40@9&$dZIdjl3v+<THKYVQWg$+mNEWZDGGxN75BH)AjR
zx{5tMDj)aE&9<}sp?}g=B&%)jlj-yJ{OSC<C-`mqUa^3=wyPO`*i2(!vAz8ueDAz1
ze|DAK4zLmUP_t*sQ%;*`ZTGzk>^|=~_F}0m@22B>yPluhd%|wx-iA~)>o3dN_NLjd
zvULe$+&AO&>b<ic@$da`d4g?{A^%>6=PbK_nTy)qh?d{O5!bdiV9F1hoL^eDJFU|9
z9zFQn_V1)Ed$*XI*@|nh*!et7w3WZ*vo}g~kL>~RZ#FkvuGv`CDDUft*tMtV%t>3m
zzR9+G1orQ_bGgf=LV3N-B5z*12;&L14U+7(8f_PC@+$W1{kUM(9)T2ATidePdj-zr
z?OAcD*_P!<-QFmZJlpbbHG6F~ec9u>{*LXlswkV5g(q#50=C<(5c9Ep;xlE>w=ar&
zGpY{TyeXJu>kzhMx0%V}y+3C2*)kN?+pLLNy!Q^fuI(-B4Ynz=^|p<ixA*cJN9~QN
z(6V{LZML_lk9qF{!AHB>DmK_0*qmnTq4LRQ-JipI6f_L>y#9S^_wL2)whmj@?LF{x
zvek`=IkwezFWT^bDcieYC7WHrgpYeZROIbpGAiHep7Ge`%9AsDuO+zey|I#UkJsY3
zy{gi4_L@ZK?|s8IV~<+&+&%w<X782Yowc{ZreRN0HK*NPd8xhETyyujmdn~s*mHcZ
z%%0V@EX?M%5pO&9nmwJk_lnI%+wE5_><zxMz*_#%#=Qr8mRYZn-npk~={DQcrR@8(
zeD>Hnah<beIm2SRW%>6#8&gm1eZQDx--Gfx8-^8u);$$sdtby^*pwX3veli{V6FS@
z=-z^_XKYS%?Y31bYO?i_s<i!`>b*ClVvB9#%}aZhy<cV{lg4EC+`e<~bVlvH=br1>
zvIni+`@yu*MsVW>+saMPZI6FU-z$+g!6sNi$wt~}@7~oS`)vZML-tH|i`hG&x51|3
zk%$fR^=Y;ba?jW#FjQFgxTM=o2s&qbVNtZLn_ru4d_2S6l)2mYDlo6!ec;axE5*;A
zdzSlV*)mKCvgFulvPWj~&poRSvDqzKc7AWw0&}api(Gphw(r}W{Xu!}o+&469Woy8
z<v6x*kLUUcd-ERcvt64wZLf>)ioFtB*YCaKzkg51A&$LUuCCa#pu*Z#HEY9O4{^pl
z%sE_k4Kp^`@SIiJlfZgycSmIO9;T+lwwE^Cwz+a)gRRK5owgEEhi$EnuiVSBBHuRV
z0fVjfP7%AC8xQYY+upc$#vdkIUeD;gIY;?yJEJbxEMA~&`&f0y-i<t8Yy}P)+eR$j
zx0k6<c<*#3qrJ(e<@PP`^x9iqduXqQp@J<_Yo9HLm7<-@cALGI%=YYYl~Ay2aM!fe
z>YQoYUhJ{^SjV0{3??473qA?&@tmt*R~n>XXSLPLCYk-pUXeRn_A+F<@7`T@+cqwd
zd!Nj+J{yLfDSIF6-DNXJlFjZso5o(DllgmRDwXebc`s$VcKhPJcCDRz>m6M7-aK{7
z+GfSfy~`_i?49{{%AOmqAMDXl;<KBapu5LmF}Kaud{f(<4?kKc1YNM1@@t}vgxd_8
z2CHLxC(8)g`TqH1JEvFH)=S#Vc0K!N+aNRLy+3A6-<zPc*ILN5)@H_p1AEFAP1`%=
zPVnAhj{|!rM5o*8+<Lz!I`G_{36^_p*j#n?F6o+KbLP#FJ=R{edz;=ZwSBx%!!|&i
zWe-z$*Pf>qKWtX?zqj2c&S^WdcH!Qudl%czYMQq9+tOvWTop_9HhrADH|BDjZQk8A
zwhs<Zw|Qf|U@vbf_r4uL{(HLIe((LawrB6L2@7r8-f!3|(sRMuefO5V9Y?urFFZYM
zvyZpk_QKR`8-ag&t#^34@42V3Z*NK03|oaQZ8oyL(zahe81D5@aIme)m}T><QOcI(
zUZico&7U@18yWU?oLg+mr+a9xb{z9Q|MPr%dz&xposhC&Z<hSLy?0)p+7rREWpBXj
z(7g{8TWuA*B5Y5nbL`D%)7>L&@M`btPi=euGS}>N_`J_%OPb(bo{j5m`z~#@t*X?u
zUC9!<cY#lVO|O23O_B6O+XHtG?0%AUdhd$W$~Gz7Y<pLQ=<H1sT(|d&;O0F{R*idW
zg@pGlz1qCjVO_;;zZG(JGbiucE3l$zZ$qB<-sMRewj7%#@97ET+UIw`X|I&B_FiVg
z3$`{FTlcsWi0!*`{nVcONxyAHYEy03i5%Fw<Mq6~Z8zBW9=STf)}#A~O+k8#^$#T>
zyLXa@ZCi3S>@l=uvpZE%W1T%|i><%8fbEGNZ*AwDv)gOW-C-Mcu*sIOj%}AkaH8#J
z5s|%8O{;7@3e;>J9<kZ%I2vO6M_tBtUa{ofrq>O7Po*r`TT&=$_l)tJtu_OLox~@F
zy#|LeY#QA5ST$%&*_&3$u(!B~$4*mls%^uCw^n(})c3CZ^V`;8x2tWG<>I|TlN9&$
zZCh)5;A*aoM`DuA0p)6&2e0PXa6Q{*`y}b=9v!ZEdmhHs+Wwlsyq6=g%2rOKcW(eo
zwavxfEW4{Dy7nq2uC&z^Fy6gAj%%OQsa>`qeJl3<`JQKcWU{C2QP)1(Cx`p@sIm&!
zt*M^5_sN2ZwkwzP@0D8FWvg+o$M$2E_nsR)EB2nLUTdSV=#b4D_afWR42$-jn%TWK
z+OA@6R-gRd9bI$wtnX{xdsUptCY^iJUWQJky#|WwtxZf~ZTH>3v)5+R<~=3PbN4zd
zG~8prCt&L!{nX~>7Gc{y-mX25j{MoH^mMuHab6j_993c4FhT3RcdYYlr>(cz8+>d3
z-imVnJ!(>$ZP&OR+@r9#*QPM!(cUx1SMOP&nqkXhrnEQiV59A7>!r4q8^ZT)Fh5}9
zqW^GDQ}Y5_3-_&imfZZlr)?3NEwlJn+dtRV+M1kwv3C!b_#TzU+P%H{s&+amXKed?
zF4!{Mh}t=~cJ^MG)q=Jg_V3;!F-d#Rgh%PNtnse4{7=32zC5~n@9sq_Y>t%9vGv)0
z+%|>5)z)3X$9hK@|DK%d278s%C+wZFvD0>0?}6QEk*2l_)PL+Lt$enJ<(ayz!O5F@
z7^b`1e&F1+H==v-p1T`E_GTVu**l@CcW=hDm3x-IN!weuENbrrgGig6f*H1xCQY)<
z@Nl&;+u5`CfKk})U4@c+`zChUUhoRtlX3Rno(~f1Y|m=!-@9bq!o8Af_S-(t%(Z0*
zn__t)YPZc3)#FwNBA(iOaW$|tcqn39XSZhWn_H9idi3bn79729b3jPnde+v3d-yU}
z?=29EvC+v;v;CvgWSe*Nf~}O`AzS8c^7|wtui7YcNbUQwi__M2diLJ4VjK7VKJjgj
z&Z8T94lMa*^YhJ{JwKMcvDvp&b+4a9$lf+C>%Chv*lph=Z?HB0^4rGatneOL&V74l
zWX11|dnvg0g6IruhSD7C4(=1S407jfcFWf7RePFd_pu>i@3zMFy)Gx3ZCE9C?p2q{
z-OJ;&%eG-nrmew=WqW(x3EQqHnzff<r;*LF^pCbv3W99g>bC4%bkc3_v=do-f5x%z
zb6}~mabjt(S++`hU(Gb(y<R3i_KHgD?F+Epw3p+rwXM>2Cff#4J}W)XZd<?44ST!P
zx9^SnvwrW5zq&R{jvn1RW96?s61z0^i0qoW_r#$T+bb7YY%?t1*&IkMwDmFly4U5<
zLYvriqITwu`g=pP_uD#mZ`*qy^|p0~#7&#0r_%PWPFc2Z@!lr8ESDX&e^+bnJv8&Y
z?eeyMyH}6T?LB1MWXo2ew>Ke~d-oajd3#%w=GY~_v)HS)Z>g=s(KWUeHB)S_@h{%{
zifi89Tf9s6-f>%Cd+AT--i9ZO_nyh$wYNa4c5h1D-#wFj+ifc*_S@J@I<)u4hj%sy
zp7GlXION!DQ2cIV;mx&ojmF8nm$J{;9?zGtmipCd`*_kqyK576>?_gRXxmY$ZL_d*
z^IpY}Z+in8gl#7>v+q5i=DS-VLdeGCsDf>+!Fg-lBP(p#QYP8n4M^H+=i#uoC2)gn
z^|^z4XZ=ato8Vq<Ew5#}@0EA+-mLOoTY(7PJ&X6A*_-6nYB&9h)ZPI3&3iR_PVDtr
z*1K;5%R<`|`h9zzMN98(*j#1(I$@{nmQ(F}tTe;-9!!bd+sk!i?+fX}HVQFu`y8FU
z_az*hx3_M`B-?lHDtmL1cG)-y@b7I9QL=M5`eM&>wSD`{U9)W2FIw9@OEBEKM*O_3
z*B05m$@zJ98IFCnGY;t3CUAbUnKyUt-W*5sy$3WW+dg?|X8YyTmOTd~KU@DZsN3^E
zqkGSnJZIZ!8SnPmeObD9h9lpe#h?85?s#cnd**SE-Tc{Fwgvr(w%V}^_8$6Iv)7ih
zW3RU6`h5>Le_CgU|Jb7wmSg*3<BdHNR9p5w$XmKMAo`B=2J!WKKR2GRJ>tG{&zpBL
zwgy}NTN})oW>fYhX0N|e^WJ}34(xTA#jyALs_A=w9J;XA)Vp)9FjMs22G83zBI+4?
zTcox2E{ROFE!i`9?*#T;dmGdYY#TOBv~}S6V4GrXY<HHgWbbn0^?MDBOl?lZ3+z2}
zbpKu}^BTKFYaRCP@l&-8_<dmSo85i3{1GC1GHyBVu{h6Q=b4zlr^Twz*67ki+eunW
z_x^FtvejGt$fjo1gnb?s{x%y9?AW8AH(~Ev<pp~U_(g2i^=Iy7h-BV%MyJa5tn$RY
zLW!-mm-Kh-6>ZPA3z>S-R!C#v-ns}2yOozW@15X0*;Zc3Z*R^t#l0MP5qk~pi0u(@
zFSFgEG1vC;v&nmRoLp}E=6~<r0D}+KAvY%P)myRJmc2sT?f}PMYlUB&wq^`xY}s<=
z*}gt~Zf}slTHC3|^Y%ua<lp<9*TOE~OP1}9gn4^=oR98amGE-cpTNm`5>Hm|{lI0g
z_rkvYHhZGZ*xKAqw(YarXZs+kz-Gyu{yhwH#4R_RJh}J5$L>8hns@HKGR4tW!P46L
z1;=C?hR?ffY;MlpGb2*X=IGQzdnG1%+wy-)vdgO4X3HQk)yCp)i|s7o<+iK+tL*aP
z=IrG=;<qngqVis!dOO>-th@KV+o-tLVO_`G1--{@-2zzcgd@ZDC>%Ip&Ct8qa@Jjy
zeG<N<whdcT>=-L{?v*@lws%82gH3esslACyH|?EPm9<xBS;yXIPK#`pW%k;ZoSd<z
zp)Y1v!tLd|7wlQG_d(NBYZd?QJrnj!vJJS-X6yH8lbytuPqsVMX4p3PZnizMPtLAy
z;Y!>8jGOE#?%UdZxT;{Q!OgQrV$E%vBOj*hc_C7{$LG8MzP+~odu`$sY;Uzn*-bD@
zv|VPhz?Q>bVXrd7T06eI8!bJSUa;Xm#%9MYad(f`iRF7A{M&D>;jz&6N99TzVNJ`u
z0akBqo_VO*>PRoMEm6z0S@86oO@rd2T?ew(?KV3mw0BQ#u-%NdLwioTUD*3UEOJl5
zjXBmX{44i7s+w<iR%y{58|?+QI@3e<p0v%|Yq9t8o|3BmJ@NfYb|NmH_pohd-+Q5V
zrEP-h`#l}=pV<iP{kpf9mt)_(#k*`@Glkg+En2Z}!^&3M11taA2rBW|#fKT&t>P8k
zo4W1!9+9@ny;@Uw><;YJw28P~Z)0gWYj3UX<J}GSSM0gGDsH#G)akwRCr;a!vGJE}
zNz>H5xm?bBZ}=axox#Ip%M-Z8_Oj8XJ<m*<_Fl<7wkPt%);%k}w%CMzySwLuV9MSz
z7613#dM#?#&#1hQ$L!S}%h$nsPxZIi@&x_e!_&3e#_Pi}+XeaN)`As1d%f*W?%h$<
zzsIE2z;@4;YkQCU+iaU5lw@<Kve$OPq@q0zOYYgsIR9_YqD>2IyN{XLy;%HV@1D+b
z+usLP*d5C3-F?C)+IExOY1{V`S@(71SJ~`os<t&@+qx$v@AjTomsZ=ld_1#v%~AHf
zV%w+f{S|w`)_0k}?gjBvtX<O%@9kOHw)fynhrMQo8~3^^+^|i_54N>=plItk?a1EG
zF3W8E=P27<ykN0+$z5K%89{b?oTl&J8&n&%H_U@`?}Hx?Z7f!B+x|!_vzZZd+-Aem
zVB3a8jW$<gr`j&45wf)vvELi@KW=Y>yWPGAai{lQNH5&u<Ho!1jsp9>KMQ;IO!ykO
z=SkNt8-bZ;Z31R#?MbLRV8d|W-*y4hJ-c@?UEcd}am!wjsFJ<AmiE|MCN8xLm}6=4
zCZx~irROzUgJ1QwJ(mveec>{DZ-nbH8<&Pnwk@s+dr$rk-FxATjm?BaRogl*Wm~17
z>3e<7bnW5HUb@#lXz5;?)Jj_o`>i%>?ya|t`zvP`$Z~e)hsf<VZYx`CnSMRAoy2wA
z=0i5ao@ce~dtD|+*q%Aeve(Xi>fQx1{JV9$9BgAgYuZj?m}x6pv&s4y?=D*gonqU2
zp;PweW`DGOG9lF_V}_57wD?imGYhZnoqtWnPTQn;?-$q3z4yur_fF_2vGqDOVK2w!
z-8K&rT6P<<Fxu^6mDqPm<brL3&+1(cr(A4nGq>z@czVM2)Qir&&)!$=z4>PHUe{UD
zb{S$xwr$EYZOi7ru~iG=-McwY$#(CYUK@v84O=;{W42m%?%6yp7PYx?bD7PPFok^)
z`|NBxL{Hf;+`hDX#@?_!C)O?8tFa>3#=(C6ZVtbRd+U}Qx9u?0v7Pfqz&7-9_g>%h
z8GBc3UT7mA*|^tSMZm@@B70AU{Pw-_{ZsdTd9Y<q>7zq7AJ%NNy?c_?MlX4)?VfTT
z+pbrgHUW>O?0L|zd2im#j=dpWZng=GjeEDxU%PjK#avsSADXt8{_nDt_}sqtw86VQ
zE&p2g7M@tSH^fxP_Nc3>)tO`^+esVO+rGF}XS+#&ZEwMei@Rf51otH@`Mq1`kb~_K
z<4Jq|Ngc3t_<7GNXshJjO)|@E6Ph_}EN(2a>0z<4?btZUhT+nIT@6VB`#9I|*(hiw
zSwG=CY%4jjaIe<4je9LN=-4i(Hrt!xlC*c}Wud)ZT@!4%c6Zy(2|BgsRxpd*nn~;S
zKEHfw?^2Im+eH^u_ckb3+MHkSZu4=L?A}YOqxb$%jIy<<=GbR(zu9)uu@lyQRgrr{
zO6+XiV)yUOC}gygVd>fXZuJJ+)a(29R-{h0O=~%3TV}h+>dD-OJ?lO5_tu<Wym!V%
zrad|RdA5?rME6bd`DatUtY)vxihdh=VV%7)*N)mINyXW|(3aeHq|9*d`}Rk+84gXh
zN&GwZrnpt@37Mv5n<4yT&mQ;xw(7sUtk)b4vz1&}ZYw?c?w*TxkJxJ6+_X0@UfOo&
zGS0nrGUxVg;9=S0pm5n{N6!jdi@asFFUl6}S^coxX2T^P8;>L*+r?=vd*6Ouvd6=6
z^WKSTYxWw=vav1VJ!Na@&15TE_S42zGT+vrt!eLpXP<0-GOF5ryHdNC!{UXF2FE*_
z2E78CZ*#Wo{j)^Y_QcBPHjWB8d#eKv?^Rs5d9T-%Z+kUf814<pUbi>FhHG#4bK!lT
zsw?-pe{J0RWz!U!xIH^;FS#$<YqD~oZDq61-ZrP~y=(57@BOeqYtJ|1dfP8tytZu4
zYJ1;=2-z_`JZ972GQ}$6vC-ZO+ETVDI_b8)Iyrlv$BNndoH=RJ>wRRe;Y|I#HWwN9
zJ-M4?8|yV^@BZC$_O!g%YQxh1e@{oklRXwz?R!6QJhwJ!XS2QWWT}mZ`u@ENcP8w0
z$dB49qBzM`G*)5Xia)#e?v_|=dqP}vpX2L|dsB4-_kIabwKX`qW^cL20h_bJYI`|$
zy6;)yEw=Z^mi>FVA06C#V5P0~RmUTH7n~Hdy;81as}c8Z&pD$*wx{?e>`ExOVg10K
z)%Jk!$-T1oxb4&!Rrd-^`?ANeuX3-9dBxseQ&sI&L~pfaiMhY$!ISGYj=VQ*Gdp(f
zjW}}8#^wLoJ>t5G))haZ_cqx6+MV-|&5n1Eh+TW}^gWkydiUPXGPd2ZZI12rt$DjH
ze7RtI=?k~rmZ+(FR89Ww_37ibeX(xqo*6&(+RT`$Yilquc8`qoCR?lIqqYwP9@=zY
z{ktcItJ(JG-<f+#K6%;B`1!~7_{ybwWsTPEJyChX=DZ@~-pm)-woa34_9o1VwiS@!
zwu|_)&i29O8k-JVEt>-iD)&SwOtN*{Fkx@lgdMgjZqsb<{Qs~=_x9wyx|4bBzAxNk
zd*$U^8->M%dou3S*-U!4!dCX~M4K)D7VNn_O~|fs%2J!;$;|tj%a-r0I=Io+Nbb9B
zWbn4V*KYpVQ>C)eHs4!zUxz=tZG_q(8?~FKY&V#f?FqOUV#9j=?jHU$RXZO0^1UoS
za`xW*RJm9A+y1>jf;ji&i22%vrBB~0VRv=UXYcoWH>ES!aUSipVW>QB)9LQJcfrih
zHg6U`wpsD_!rnUt$8Dv$eE0q-pKUv@z+f+vBA4xx%bRS~>^b)xVcu;UC@*NIFtcD!
z&w+(|4{TdxBXoAxUIE58Hd}O}Y$f_i_DqP|u}Afyie0i}%wF|Fb$bKmF0vMB?YFkL
z`_Lx-_E}rGJ+o~yjEik1JbhsEDRzf#4Da<lHx}3LneHoMXCwZ{_MK7LULAw2HX<|E
z*}iFQ*~?pT(DqK-^u0Rb^KBW#CtJrbU*5C2ddpskN&{Q7Ny}_ssHxaSwVT>jOyS?>
z^M2=^2i(7GH<hLC<#=$>W<{{%9_!26wh6q8_8y4(yyu?|>)z+pxqG*GZL%%>_0o2M
zYr^hh7R|ODdV6;(wA`=>`0lnR{<h2Bd+QDN-u0ipw^7O8de$1=y$q*SZD$vr+}q&)
zYPa35RkjIdmhE|ZYL)eN%^iDFMUD0{?K!shz@z%zlcFv6SX`fDqaLwrui3_9wmvJj
z?^#fI#paU258E5RIrior=-un3rEhbv>8Y*8p$|40qRe~Po*vzM=d!ZxAFhXc92^5|
zH4eJj^2Ijn4VBZeUNAdp_k!t(dsO+2_PP|B?affzus0w?)#gmX?mg_LC42v-Uf8=Y
z?Zn;>|3B`zaA}2g@Qg>hBNp!18*|jbcGAj{J+^ly?cL<fv2UW$<-G?4e%VZ@_-k{5
z>%UF(-zJ+39#h-@wkK^@*sa)Oz#6niq++J+qK!R!OV%sfK9&~Wm!SCDW~I2NZN~hd
zJ(<z`b_Fd0HrGqU?0QuM_xT)~WE;YN)n=_T(>|jq6ZfjKJ=yCk!?w3cp=0l~_#3uT
zX-D^V_f6Y-AZnkDWjl-Yue{l|H?Jkyrh7lxQ<=7CZ~EF4+ur8~_J-Y`vv<)PLE8&m
zNw%!DNB0V3?A#;!Mt84$|G_=g5&5?B7q8jdV6$gW#6p3+7NRkG<d5+0-THXF^}7qp
zY~S;!?7QbFvM-0v#P-J=lRd8AXWHq-s@Yze(XoeP?G{@`^Idxv9Fnqgm#^IuB|XnZ
zrR$~b)+Zu%Cmm(>uIQM!w}hL|`e1#EO@NBbo>~i$Jp~y7wl6lz+rHuxv|EtlU@NyI
zbnnw&Tz2i&uWi^q^zJQCp1IdTRDQ2Qy14b>x6}8usGPMepYdYP8_vypE87n5-RAsv
zZ`?EGeVej*_VV12wza#-zwgkRroBDg9eZB{EVYTgac-}goxW}4_V?BdC*JQ(&OBf{
zk#WAwmZRHj>Uxja_B`Kb`%&caUI(EZ>-XO-*@h{e-OIUZg-u^jxQ(LCX`5}wI&7aT
zezx~i+(nyvIt%u4h>Pt#tuMN7_0t1;YL;;C+m|6@yO4jW<+o&(eaBe$+IpN@X``{>
zqV1u{bM~$b*tRD~a{perxh>YSIGOe?I{9Ib{lqhSHT;h65#7kQH~SR-KJ|b6wySS&
z*a}Fs?me?;{obM<cWrNS-M6i=G_^g_eAxEt2i|?1&76CSQvdCppTl9Z?8<c8)pyrg
z)!GN!Y^q4vtrmRW_KneOo6}!TSsQu9?BY6Izc-d=_1=;TPJ2)1ckVrQE6Vm4ck|vY
zZ%S>$g)H|5e>`r>XI*Fe)x^Ozw0+v%7aTYDT$-=G_wbzGHZ41t_g1f+v4`30yUlW*
z$u<oeYIkSmT(UXC_F|8Ya@gLfY6|;qb_egBVJBm2TY7J=a`EK7;%0_>cf0N0`ytGD
zuZ17uKIRL7Hu*Q?Y`5}nw3+r~)1KFRPV8~WX4^e)0sp=Y&sz7oZPDJlfKg{RbNkvo
z6At>=$oikPU3q@#o{8!Uc3*n>Z0}ovu)Pv05w=PKj&?KTp6xNOx81wgL}wQ_!wK6Z
z3l8mV_%~^{a+dgBlZYPM50+}SseI>bU;DkbQBi5#yUni4_5g$JUe)8ywk7*3ZLhSm
z+47#Z**mRMZjaW<#=UxfZrX19)nNO)GSpUH-)HYKRyo^$!jtxjXB^$@eD~4bhI5Iw
z8mcC?0lW9vRK1(E_or0j-VF~0_swO^uyvTZ!uH3v$9p^8%-j1mQp5Ia?tB~Gj5&Lw
zvli`*3OsGAmGW`#f&vfg^C?sIGPb?gyP<mR-WPA?*tQ%N+sAPCisgpcqPEV_*Y>W;
z*kJo5BGTGq|6|(*5e3_&ar5@-&75PSx#7fKck73Hl`m@U^<|5&RWK3U!zL=TueoFC
z-cmC;n*#BDdu942?Ooxr+V;KKguTmmt=VgG?DC#)`vrSh?0aoKYwX?otxj%lnf=tg
zFW9H=?aa>H+o-tBw!wXwZP>isdw-tF+Z!S9(S~pL?Y)(&_U`ql7O`C?es%Z#DFu5b
z1exryS5w-%b>5M^h1(l!6%~qY!ix{u9yrl!ZGLdEP3i1owokPBZNsBx?+N;Kf3KsQ
z?cPp9**)(Q?e^qtS!46mr_J_kg|Ut5?>V-HC2#laV`AFN6(YBnZEB9S_?<1b6PEQ`
zH&n;(@e=5?z2!1}@9c%~duM;Wy7z=noQ=Ys`!*&)EB4yo=GeE*WToxaZ3TP2)wJ$e
z#By}6$~oSBKN_0%{M9zF{ZRhZ=2r9)TgBiY+p8}`_gd`l-K*B`zxTE`<KD-Uy?3|j
zE#7;3VYZF*X$9MZZ@X;O#BcANr`l)h@o9w(Q}XJ)f>%2BKKRMLSIfQCR$F$J%_;xH
zy=Laz`%Zp7VEe$`aPJ?lq}_hUIc?u1itXW@6k@yN_9W|DC!Xy+<;=Oy)TG?@$Cv9?
z-Axj9oDa_1rWV}VD|BhnUW1$^HeE@(_ioHPVLRP)hApSc3EL1K@x3dg`R!EtChjeJ
z^~F~G_Kdx@i+1d(T`Fp`w*Kkf^V|OIJ$oi%Pt}YKdpXNb*ed<)u|4?5)Q-z5)ppO;
z?|YnO{cJ)Qb!|UAeYE$7YSZ2fo%Oa0!j*03_v`Fk^R#>KffDDvbz3*>{d_QSFGKZ{
zy_vRit=Feaw*6I5x3_x^pWXitTDJbNvUYcR&hA;x)?zzHHp15B=?t428`keV_PKda
zjS$~ngE=$zIx;BR76(M`Jz>ga)0Z{Z_Q(^7z3j>}_m&^7*fU|*g}tdq&1{d#?AZH4
z^2?qzfB)^-d^>z^)!mggl~tSe{y56M?+8=--h2U-y-shl_gdXLve)gO#9oJ)VteMl
zNwvvZq`kLm(waTLa_jas|2k%~Xv2g(?7GwU&b86n%k{2cuV%<T+hCT>HVal7?q%69
zcdzv$c3a*XUu?tNI`?)gl-=vvrD*GQ>EGVv_p|I4rFHJTu4b}VRbFN9*B#UM)E_L^
zt5+p$7r=DD`pu-?z50)0Y$v!2*e$=sZ8s~9&6dMVde7Qw8(WPE>AkJDX6|8)4BVR_
zyxz8x)!SCF(cR|Q=8U}$E~)O(-?L>;*_w%a)3+?$8@EZ>j^*}J8`dIOyFDkL+cxfU
zvz>JPv#rC0+TDHow(gah&}++JdUUt+T=_j`m_u#(@44^gceA#!OMkfc_Y_&X$>xV_
z=hkHJiBrzmdn9&&O@s0Iz1FOcZU1$**)=MQTQ3dUW?Q`D<sPF5>wU{?J#6l}#O`tU
zyL;DWPtko9Z|>}2Km)(R`?etqiF{Apfy^g{9h_Od8KzFWHQf@#PLY{o`BKYx7l;kQ
zzV}vw*|X}3SAzI$8}oPW=rpqcvGL)S(k{!LS`)x-+<Jz27sp38MUeU}#~z^Q^$J^K
z1rk5B`6$?a%N{pv5FZ<65xcV;CO*A=8Q5JgJ~F-XWFy!tWHD0Nd2-HRcQN_@<wn+P
zr__pJ?|&!ljW9Lh4k6|+HjFM(*R*(jzzA#}Oib-=;%4tA|DEc4?|{R|`b9ci{#t(>
zm~FIV4%n=pr>|^*=>yS^<^{v;1M!jZDj!9#-{;kq?z;PG3Os&cYF?%P$8bMPu1a7T
zSTBe_!CD>c1`vA}v%8hvrfHi%Y#7$NJJqs!-9-#{w6047$8T@iGMwpAJY5uSHq1O?
zXvvCsaQA`K-*9`s7DR*aZnJYcw@zuXD7K%-1L7m&EnlCnLgs_mdRkAx?(6!na(l$w
zwP16TE0==BK=L5`tD^vg4-&(NQ?*XO!}e$Zn>9>rf+i$CHSftF$~?1%ZE$m7W}wsJ
zT-(5IhKa`(eFUc?7@u17opbHr_(V3Zk+BJ!t~SV9V93GLwJ&J3cyo9bL4Uy1lS6xE
z_w0n}JFw~$oZWRa6|23PBKNI0=S=|1J#Li3h>OZ#O|X9^ull&F!?O@>9?TAOTF7n!
zN*rV!PBNeUEXWjH9gM&7RT0>35dYetMtB~G3krp^L2}4==~X7UoX*{i@ba(0`;+<X
zgOD;mo%0ihUqEK~Y7{}-*S2V9{rVWNIU<Yv&B`CFMDbgEq>feIWG}eeTD~!W!$EH2
zYp{LS%B;cWTq;Gab3k^W<I;P~aC5BRDS+)nR|Dg_9J#q#P$L0scF$KKE8fBaxVsjr
zT?5+<Q;Un<yz&y<KA4&(eT<fR{H-W<%iq1O@!DYr*segsXJ8tpo*4SWUQw_;ud`-@
z!v>^h-R#L5*+OQZ*aH$r$LDvO>+p%(fS2LtPE7{;EiAuk=Zh7Pw0hOzFx*^p{iFQR
z5Eu=C(GVC7fzc2c4S~@R0ENJcvOsVf0>lR4+f&L&Zqp>ncftE~+V>5>?GGE?DD%li
zhv4n7w4YDG?8>_$;QoJ^?wcJck-}Kpn;>_<@b4@U%Np0jtswq?2bG-xrlLw9_7#3l
z6nCEC69KoMHgEF*i>Kk~|2HhQ2Ac;mN8bDj*j*qtI$qdj1NIBLI4OK~ekriKmd*cz
z(*IvH^$do+AzyPgz|3wocm(dV!}uV2;nVzWJ8ELU{sZwr`2W}5&CiO9c9=cX0QcKc
zE;hjBGp2-r**_NOfz2wu$N_IVg7iLa?S$J0;v?hP?;!oP%*UoXEH-X|$1hAx&1?s7
zI~K-ArnfH=gooRS9YSC?faKNfr&+WrGN72X`OkB6<M0(2{eSz1Ibbt-g8IQUNIyDm
z{_chmhUjXr@r^qdp`^P2-{<iDzpa_gjt!GPn#ccFMTswvIVY`NVD$e_^TmL}HhE?1
zw&V5x!EUWF+mB-Y-n4~aagZ6r;D>t|;b9y4#s!>yK>E(+PXhb5<aH#9`5-xLc(bAy
z+#GE3AhEWm{P6ez@on8?QOX057&UO(?q}d|LN?2(e-F5iyIEp2h8#?t*cmtTqV%6&
zf5XJ6O-l&efQM&YSqq$fLZ}t1y-_Q8tnM^H#tfEdxMGCY;Z;mv|Gaw?w@c4O4{jbV
zcW=;-M2Q1el@Auxmcb^t)c<<s19z9#{+n>UPhtb%Y;?VsbuPp6zTwFk@bp;od*-%u
z^Z8)@<tclxy7jcdG_Zdb-1xiW{OmJeHs4Q0ixaGC!1`czEz)|ly=~cCxZBF>znFpK
z&c`hU)7u<9z<NWc=EBQgkUTcrbM7tNEeq68$EeWNr|3>xbvC>RY)7>6Pm61Z`QUE2
zdUz{F{GjUx@j0%rVC0p&xgr)bh1uZw52Ws!nUbbXX(QO*Es9HUhBrtpHe7OXIoMz3
zJUVc_OKK*s?|XO>&c~(?Bvz6asC{>;7Cat9_ql?@Go6cf=NzLpu%Bg@C=xUWWY#Di
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C
z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVCuA@IpZ1iXd-<|DUT>KN-FU~;%<r{e4Ib!7iG
z8-mv#ycJEdY-$yOo3;J~>uzNIaf0<brOl^;*HIvgqqA@IU$bCewh_EuWaE;ko$4PT
z`~Uhr_`%Hv>9Hv72Cwt*ova032T`M6p#u`fhL7tzg4KfLoYtL0F$W|D!>@Bt*BHXY
zNuhUCO$3{N`T1m&{bc>$(!t`$W-aU7ybi|y?R5(rjxaumR{q$r(^32Yig;4J+7{;X
zi*`ivMuXS?vA^wruUDDzJ_^iUFQ*HZTh4C|4o8q#Abhse9An)bNF0P&T;#!VBW~)n
zYa`Pl@IEb&I1FDDn1<ngnB3aCop3iA{5}qL14wTDhDla?i!4y=EIG8>;>m(Elr@1M
zwbGv^g6$7!LaZSLi6dhsRw=kW$l`?9j}q8W^mPX$!^6G3Q+MauVkHaj_xpL#%@MWY
zUX3nxFtiW6o~Qcc|E)KtUIe@OkS`0|&mi^M-@;J%ATfOS$Erqf8V0E;JkAT(8=DDP
zqf}+wjuMX`b=a`7)DoO=1XHi+$Om>aj4!rqKG;7nKDFp2TN_cz8IZY&xB0+pwu~03
zW5|KzL6~J)m8C-5Zko6WWIhOsd%l31&vtMvJj}eL&Y*-1NG%8l9MrO&#B>j=E__x1
zyet99f$*%{wP1b68@zY*UqD@74U$L4&UgJ#(&OxJ?q;`kHG|hiqN@e*eUE>Fr;h_W
zJmKl%#oHRNy&yH{SXVs;F2~n<7w+G+#s22zHUGhWI5@)*#T_s+eug)J^{xt<Y+-*=
z9;{~GG4`F}J{RHUfYhFyCSuK_xe8p?fcT~2=fQqZRZIk%AOBnvOmCU~0=|X~qz)Ut
zBXb$<$K4MOg6+hn4kmWg=QKDR+`WS=SXx@)?y6IG33fY79WL7Fh9WpVk>$SyJg~TW
zwG?bFO#Gu2yVl<HCt$JSP0xuk8>SaTt4+TH_SXmI&2an9Gk>1QzqlN1CP)n~eEr6C
zE!F1^@UVN<Wev7J{z2akpBHA}`ZZ;)Ey1wDW&Wu6Xb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2DQ41r9phu|?9nD1Jj$rBuhfT@Mi*Zda3$A?XoCxgcy!oxgw
zuAZ6%ALCFqf{aDRJcR83FYoQ$x#G<N@K^}U9Bg#aZ4s+Zz3JdFrHtUnUHMBCl|kxS
z7a{gVgZQWH7l7S;^Eo$|o*2u5bNqi#L<4+$NL~r|*<$)7^<ck%%qJILJoXvvp8FiN
zDC7U}CR5;H12bD<cMyDR{9@o`@YoJa4n*Heaxe?7S%VU0yIg;4E@|c7>66t94x=Q!
zqhR|$>O+%_!R*67SAfS^tv_Ffj~#*Jng40P?E~?V@u`ERV83rH3g30|<|KIh!qj*-
z*@MTMV0>g+u%I2R7bNz;<~^KkeAU=0dhuSAvHnm`Z%eyaRunOizKxIF!1@H=1flpB
zBnQK{O1*K)!PH=*FU|A@n+=m|)}Id_|BrG%X5J>EzteU9E|mBJsXh1w^ZY-af7`(E
zWXV*sJu+Z3nB6$94#j-Q$~h=}kXiU}(xgN1w6Wyz8E_ne)aSi&1&d#0dk7v!1&QIq
zVa0F3>hY<`X6AsK4^r3pAepQ%1?k0yCwWMM!v<MROQ!&M+&^&QEDSlAx=T)l7S7f4
z2>Ju2o*eqZj7xAof2cnSXR8O@!s@O@P8sWr7nNW))yONu%MzH~PnVws`|IjErrj0-
z&*0|4<k9H|>zz>IAbAbDrQ7dIDCIJ;p3}8`V7ozL-Y*xz^=5^4!`UD?WPIZQVhsJ?
z)Ms#i)!sU29<3|^PM@M?ld-y0c=1-S{BH)!UFqujV7qrSMOzwhSHjH!+4cNv#}0vs
zYvE=vpDX}}L;T07;Ihg6lQh_zld0ZdK1dHPEGLdQhcV)|Jy;(uwF<$DSFgOj6Ralv
z=u1oHPq*N12<g5Ib^|WGF!8O^_kjHc<AZ4JL{ZE5)hh7wH$h^P%xtwzmidFjSxt)_
zY$r&ZSp2)c8SJ(u;U>7d4$TnW*vA|UHWQyYH%j|-+T?lRW%xoH4RH7$jhEbY>*02=
z`~Bo137Ug!$9pwHxH~}NGTF=|`$^I5JiPp_-SQLcrbiQ|T1w4^oL2%e_eRofu)jfU
z5N5G<-nlU#608=)$A#A@vs$h#I1irh{GwaG6L$V@O0E@(JCC~l1@ohS3xnxjB};Lh
z|9fxCL}QqJ(%V?U;RfR)(;sKpfz3h|BbEL8lpNTP`!=AS{~L4T8cKM8%$-{;0I!di
z2rR%T<8}Z2Hm_Oq62(m{eD5}gmNxAWwi5)0QS8fiaCwc-ka_4i0hhsLiEhbVc-jHk
zw{(Rf+&&N=87JsCgU#WtP~PQ{k_(Stm>TI7o;btp+}kLypFwKf#96`SfY<_&Pb~d%
zby3XH`}Nww^#l)!7)WiAK_1wQMSFf>gab$(gmt)|!_5ZqiN#lC4x#AtH;{wZ|Jf2p
zcD|{$*b)DKDaQFf;-9-P&i~n2vlU$a=RNv0ZPF@ku({7-&Va>W?(rykO0c|xsRhwN
z^55WT<At6wJU@4dL&}cxQjFmE1gXV`?_V>-8AmXEN<NxkH^cZ@5#cCqhlx{4b4t%f
zDQ7_DH9s^1`{V4RV;FKEc@PfzP-H&Ikn}Z4Aicz5g@DU&KYMCr!^1xCE*D%J*(|4$
z0IO3EAoInAxqTSvQ*f&t*gxx!m+pEnQygv{vKh$i!y6E1c7eo2mD4Ti1uo+}|L5ly
zBY3=OtQCf*kKIZOP~s9~77Q0EBKk5OM&@vT$$Zu{|Fg#qoY#79UV)3l^nqykzmR$^
zA+>3z(7}CRJF*@y?s%2_9YsB-(JIT`+g#!PjXt;w-0%4RHXR&B%b)KD+moXpKrlb_
zI#s~UIr!iM*iMjJaAC`xQD8L|5fPRvcCLZDYl7Z<a9rTh2NS=ZWC0ITm{<$n8H?#j
zcJS~8iF<fNXgrt-Ib&|wFEg;6AaP>xvE38F{*pUZ3=R{JKF+$k8??mk5H#!kwo6(M
zf2zaFaFxpwz-FAe!nO0QXb{-_%3`_TFb0`R4CY)f1&$+_+7I7#NzxC}=aeY|?;mQP
zhnzXLciG<^uA3dfZUU)U$gmE~7F^8$KL5`?Zq3f6g4f{j1X6<ypGtjUd3+{2IK6n?
z(A@R<{C)8Gf5+||M+q-cwQ6uYZ7-|=i#xp(!P)<3+l)AqOL@&4u)DC??fMI{mJypA
zv0^(nLE7Y#*VLf&Cu?@JVAvb_Mi*Wm*Ox%nzaiW6(*Mwo*j<oz4ouwSm+zLF%kJ;k
zy~Y6?MvoW12Kyf*|NbasjKoL}(#D(mSpeKd0Lg){>+(RjeIPzE&e84yn=|j3&aT#w
zj~Mmyj-`Jw+z-=#XI>UqFNp8{SOXj{Aoii#QdY|s&O|XwU}Ci8mz%LDVj#6L`H*(x
z?f*jH`~ng~$H`MG38o)(Js|#3Nf&Uw0kJ{2_H_rm{x9TZ-znW8u~Ys34~+VM`f<$l
ze|`#G;QX`oBFFYGnN?tauhxQ;ry#RI_-<4jSRBMA22WdoNRy`gyW!#Y=?CQef308s
zMEQHB*Abj?1arqE6GXodB(^W@98qS2)RKaiK6S<@SH83Mfy3?u2V{L2HuH}Ek+Qh+
z9MaduCQl8qZ2t-HG<N#WG&o!I6ZY{XMqNSc-T&RdVf$J{0wcVvE*}8bIXhHu@4EAJ
z2i!bdZgtou4v%B7nupDnL7h9mZ6h!rjmVl447M92+{T^<*Lz})E1V6IL&jH5yTH@n
z*%A#nJ8kMB^Bdf+z;QEq+H|aL4Jf+@R{vpn{Z2>E_hA1zy}M|k^gtV|4`$EpOVccq
zjPJwESZMnZ9KW)KE5Kz_jKf-RxU`z5f%!0f*yyjPIpBW$w)HC5er)PMVlhW<uKN3-
z39RPo%gL5+IX=MMFzxUcczp>{iwj>myaH|?OpUXJuw_!uPLweHWW86jcOztsyzk&N
zu$?gV#L&BgA>$)Ik7vO3?wd4u!<8z;Jr20c*{qeNbLwaV+&>qV%m9~TXPB9G)w3)B
zyT6j97j7;tbCAVr{hxs22qu;Q9ruOtiJ?Q+=fUfL#>0@YCDv&Zc81=Q1iJ}j#=`r2
za5jj4apAum;+@;z@dOgXhK(QfSzh`46kKm@nJTl(@E8Yp{Qp*nDN5R_RWAbjyMD!Y
zu(&&(`u}t=WUm0ozRw(x@hlJ<9aq}&gUv!0Cxw6PJ7hd&-v-q2|Fy>jQNjacrqi6C
z@O>2TBlW=k0g1zK{y+Ae$0cu|xM@z~%}oby8E%(2&Iry^w}l~T6QpNC*lDmCcMdj!
z%aT{o^9(`aFf2Li9o#;c7>s_+Ee6(WU(>bohKesdeqr+a`!-;>A0{_tnE+TXh~JzK
zIsX{MwpIRZ$*ZW1QrGwU|1mH9H65cJ!YC#OHe)UWWIqQ;KRQ<2#YHgvpz8tgt@(Vx
zVFhA?u$#_ec>S*{@yk3V%4}!+e^r$B8c1F4vmF@yf7{UY;P{yERAie;>>_aa##1H>
zb~{K92>-u|R=$G-@!&k+KX89J@8W@*b*OtjSbS-&7>fIG8O2@t3alQNJV^Yw*#)rs
zL2MA-zfqr{9*{axu^)#pIBbyhZMJ}%5BrS60YeU^j(JgpMeZF$IRX=>Cf#r14EOVF
z&82Ym#~rBmM8WLTx%An}N`f1l{%3x(1DC-tIS~En2IO4UTOZ?gt*tPFn+KA^hMVhA
z&wjQ1_-;q;X597vf?XP5yFq5@#e~D_r?q#K;b{dVhm7^ol;H6!8=(uX!$I=DM2t+?
z8^ghJ@(m#vegUa@=>ciuoK)Mpv&s(A_E`JC&|;!>FWelEI{6nRmMzwZ^JlLef!s-!
zyyQQ){JFksIoSN@kA%<vRaBb=54UeMkUQl-cHqME<RJT{Y%Tt5cbu*O*DEah3TN6v
zHtXbjX?U2z#It*^STuS=%5RuhE8|{G=WxiKZmvd`Q2Y;5gO9%b%o6Oj({k-#8l*OQ
zrSOJ`8>szRkT^PSF)`7JT&o7J|7BGtfc;i}p?;?hM-$lnE$XYmafz;Xls_5*qaiRF
z0;3@?3PwX<Gz4HFz-{#trT@R>ZXvuq2~$Iee&DqL-apJaaUa|U)rv^lsV{T~+$I6(
zU+@}o7pa5O-W4EzPR!Mv@#47W-a%?WSUG{$>O8{~aDOMqa>}k%AKk$H|0f{=DDLd5
zGywAjLLv9X6fW+;IsUJ^vj^U8yE*wY+>Idnka5#x$eohNVx+SFRc`~k%R#{yWk0zL
z%R3BvXZgy(`@sP!n0HI)l(p^T2vdi<9pn~`Zq_YJ@)>s??D-51qcc2+`(kq<CxPuY
z*#qg1lwFAgkE4L}fw1@V7Px&NJ~D3N{{)W53x?Xe9yi{F$1hCHo#0rU>;IP4goE{h
z)Y@NY0=ogk{<1jIYIkQ7O8-A#hqR^lg<=#jklMXGkUNAsW<cgyL1O55QF$r64UH~`
zkAFNB_0F5?OgG{E|ArF1os0h{>~!64g%V#N^X6R-z}Wx)Ynn2+y?y;_?)G&H_JY|8
zS)af(NDl~K7MTtY0}!7Op3yNAo;G&*CByYr--opEi;R*`(f}^=x>s`Ij3byHKgQkg
zxB`h?l{rUNn1b};!=Eg}z<m{DHH!s(z+xdOJ}BXZtmgFNTNW=bBlb!o%TbFh?f3)k
z=d=eh@UWlIgjl<ZY}eVdwbpU9kTse|7cpbR#m$wM!2Xgmk=xC(n{XMYX_y9&V~{;u
z&GHt!=5i?IGDsYT!&&#i-Dq`|7rs7C^SBs_IWYAgTKVQZco-k_z6v*6ad+4Dl^0XN
zX~4r!0o*?asYk{?gCXlzqdgpUMax0fEkDV4ZE=Hb4~qH{1}>|WH>JVl5J>;Z8UMiL
zjmdn-*iU)?L2y`^yxt5J2dTk@8_s0I{djLu5l(-B^iR>9xXMr59;~MOp5b=gWv}7!
z`TwFQ+&v(*xbPQ^GjRK0YMOp&SULnd!^0OOemQ%#*3Q_+V7r2(e}Mf15+@eVRnh|6
zbKe|shQqqP9UETFUko-ApE=<snL5`mZGrdyWt9Ga(;4?#(Orx_%HX^{T>^D}7@v8g
za-$(I8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*
zAut*O!!rcRIHXYKaNJhOg6Ahl@$Hlai{SIaTNSy$^BAua94vaSvw-I$KxW-CDFw4l
z?&W~jQ0qB0?TYRuyslU0rnl9n5_Yf~x9Dx&brHJ%@3odBc+LuBHV7M=3WDcpOe7t_
z_OD_v!#V%IN_!T3-e#YIKiFL$GtjYhFe}(Bba7JnI}+xD%~<!_24z0E=CT8Zy?wj(
z!sp|Kgdp?p$ad+(7VT`k!2tFrO#Fq^xh=E$H|(_HDguX*L&86>eIWV7hr(dBY&=@v
zxj!YLzbNx=$5>=B=G{U1KzMHv<jnd|J?`CyCo99}oIr9QY&vZLhWkO{Ags*<IfDtr
zHuvNPy8*=3C}OsDTF4DgD<H9{27Fc-lY~&jK;lnJqQUXo-`D`3%L9ocV`T{ixL=XQ
z39+ZVI|4QvrY>-Tz<L-v-@tO$m*aVs@&8p(;tM3--hUKh{y#c)HQ0Tv-v7482U>vL
zQ}Hbr#r$pAh&9?Ea|q$fD!%Zr<(%3855IF&OknZ5HnLzl37K;%rU$GZpSmsG4PZCJ
z)L01F5oI<^J+ZWx$pdg2L{_h3bQU}h%@c<E%#mm<FUvV)khx7{z0_oTZnK1!dByW)
z!`V+2vF|%9R@iMFeMSo$wxz4r!pjnvo967>4fbo6>$2T1_UgdRgUO@Q?><GN#KEha
zI!l{Q3vk$=s|E4zK2!nQ4Pu`=!U0bol|8pm%mK;6@UA24;b~Gj$q?@5KO2M0xfyxE
zd<_N(ta0C@3pq#3-6(sPdNC*1jtqZg%c#jxDC#Ag6s%eIA@;65H0J<^gHP5QaM{FY
znF!`PzuQf4PJYv*I(WF5nX<tB2=fmzy^{am>U8ZDV0W>)r&-l0>B031oNEWqkt3_e
z#*W((4-ZqAd@RdeOMACS6ua|Pl(hcs{sFctm!$$s!_*T)udi7Kc3Z2{X?WP()4#IO
zPR|X+9&BdC7jM(~-d+wb!-LP<0;iF58UI}pvDskvw;A%m;~AS7qhg~WFd71*Aut*O
zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0wXU37J2ocjP1TS8$|N>!K2yt;p505S0U%U
zzDiKo8R>2g9_s*w>ct~>!Q&mpcc*~&|7~^Ix|7Wdb^HjV4jsQeG}Th|;}r0iheD6|
zE~j-B;Pd|?r=s4?d$uwK>~8;wUf}Vb?pPL_=l{;qI}9F=0@>9yO$hEr5FZ(PwjBhI
zS0Rg$%GTb?2Oc95<@$|so?KXxJ%+sk<?QhDcn-;6-i<6__h|?7w;r%RVeW9S-n+?o
zcli#R=aJwsHUIrT;PQgIA?J3xf3yOJN%;DSC~;l!z5_nr05cCp-zwn&yL;-VH9J2o
z+XRnan0(ce1Pu4X<RlOGf$anFoy?Qq?rCtJVOq4M0mUr6TW*#t@2pV7K>8x3A@_DV
z2(W_F4M+?f8+sjvn~g4qkFS5|G<dudrY6c{CVc<j>;lG}7td`mkN>ZV5?>&-9M;|7
zxhoJGhEF>|&ib9pGi}>WlXKwovQ9-C#a-!g+ri^>Fg=9mOZ{izeqJhxI2SZbd<9s1
zhMFXb`*E3f`EoeUID+X>W<%`N1c}`?VFTL%Vp9`WEVlrsPZTr1S%Smm^Rqf|xS_~F
zIWs!C%<U&}67&aDIa!nu&n<X({@k(w&fdNTvX>d%P*cv+R&&-u%8m7#Sm0#|Ox;(O
zKVW}#Xcg|7eA}8}c%I(@IlBg?|JTZ1i}Itm@Bh_*hBzO({I481pMvyGx^oF7-GRhl
z`16`6@G$ld34_Olx3~26iZktCxqf-<aW8Of2iRQ6%?ozEE%*uc&xNdW7E`aGmcdt)
zm@GcOnFY`LygM#~&sjfQ$^#CMSNX5NVU?Ir2QJTHZo@{07bC{UO>PK+?Z>7LBxa~9
zxQahc39LqC`YiKFRfw~LR205}?FOmCht;K4g8hZ8#@6JQ#dL`pc=&?kHv2?qKCg$I
zr991$0YeW+4L<A}0XYZ%{lh)rFbAo9zj*QnmJ|lC`$2qMSf6vAc4Vg^ybQmT$qWw9
zStq9Kyjc!8$8Orw%_O@Cmwlt+qaiRF0;3@?8UmvsFd72GGX(a(hupyh^Ih&YFM{n5
zm|7Tp>AwJc{Z^K_2v~n|sO$D)*?Zw_5$WmH;5K~d#|Uu$|7MxRPWQXt!R;5AIoRmX
z9ZM`H+fD|z7w?^XuoHIve~qOeO1q5bb2*sLd!z$wKfd#Sr(Q?gt&`!-0CpEPyJaUC
zfz84uN30mrO&hSgmRW0{oF{8|8*#oh%q;m@19*RYvgKEdu&J0BX7MUI3*J8g>0@I)
zxcNk6-;Uz08{jZH@+b>zA4qO_1LQ6swzI#$ZQ!E|C&1f|Ao=o%TyXn9d}J)L$^h*5
zv=o_LYMyx*{qw&a_i%<=R1RYO+sZqTvqL~;n@w)BL_hyu<Akf_t;y#x+_5*M7u-I+
zxceWN2I)t~KaI|!gax`hF23v{)VrfLd1u1=|FSc~cg8A4?ZkHfPYTm6jQ)S?PgQV#
zC*yzjHh)vdnZ+gIc_?m95C27yJAZzThllMy-ZprAs%+2!=O@k5>lkSRn_rt&^}@};
zCJz!b$e#j^XAm2NcWNQd#scxFg*i8Hql5{_-0SJw;B@(TdozX{NFIcfBIlVO3Q-~G
z50F}N@dBA8;BW=$f6!nAXHQT+jy2vk-3YS^xDGj6xlEP^L(R)}7jT*go!+vm-e4o#
zJdpX=@Z<B$@Hhs^Uvm+(l=vM4J~J1@N5?-c5oL$OZ^(K)kQ&{~xYsMMn1~o-G5vx#
zpXy=i4D-P49N;uiIuqe{ka;kiDjN!x|5QGAXP)RJu)9KE?XxJI^a{oA{f@FbW*tPF
zpX{)5EjWHJe-QzvSN&s<JH-}TrozV`V0NI>bB_Lpn|-<cDcF8=H86gP?!;A!8IXId
zS_G$C2F#uZ5A(=OHk@e-rXNJ_kU^|l1o1)Ga^7Q$-vW^J+aNv&KQ&&Y$$ha6?4Ffd
z)rc}1qz{JqceH`smc4H&*bI>P_o&znEJ<o8X#gaSj(0cu>rAdUh1a#dTOfC8eoN@z
zd9@C*hV}Z#M{sk|^^fvLLtr!nMnhmU1V%$(Gz3ON02BhjDYL<CG!PqvSM{Uz2S8$k
z@LMT8c$-90Y7)3jRGB}=qWgmwxUB=y@3;oC=Pnl7|CfIgy+bm!1gsV$j|=mK23bBe
z+z;-z{{7gtbJxdX;Q9YhYt%b4id}8N?Wg4#kUo1vj~mYU|D610aC;PF=YniVp9;iA
z#~KpY@Bc<uhmYT=tq69P|HS1e^U0kn;^AQf)Bmjk(f@bYuovF`g~?B`@3t^Ffw-#-
zB>uy-cQcQ$#g3q_kTnvo=QDuq1IgW41Gy7(f#*zc9QXYc28Sa^4up@hn(BjSY<S<N
z9pJc$;Q7AO*TrETx|+jHh&?vwasmyIb8tc8zFQ!3ejs+tp@$Yv*Uvy1O9;KqYWa8F
zd-%EskU9|lptuIyhjJ+11a=2V3>^o*G{h-~t`?cE;f=VT2qd1WylyRs2H~(<kvn`B
z9Ki^Ckle2?nD_sxSKk4<v*AbAw%3x7`=VS0UV+0Kqz8mAOmoACYmhiDeEjEmxcZ6e
z>hSn{<thVq*V2L}c>2X<21wlSLJQm+5FZ;}-I$5eUs&U?pD44jnT1bmWn>pPoRHNR
zp5F)VOD5ex^i7fFLF~1jlgz^13EvY2QcEs&I~E2HrzWogxc@nwEWqK7Ztm6MQ<n3V
zrGWKBcXYwk!_>Vz#0K_jE`!=Gerr*%I+z?bnk^ai-rf}r8!S#dyM*)pUk$@fxS8I!
z`r+{x*ZLFAMz_cD%Tah<$#@J|TLV&)*!j-v?qtXvQy=8Z;pT$mV7P6n2iROM_bEHm
zC)@#>vnK1D#RIlmDE)YwC+BzSG$GF9K2QnS-`RRS0vtyBnIUVmT&>@M$0}jwW1|Ze
ziNeGFixKYqp&-4W{=dNW8(=%uC&gHHEDwhJBQ&EG#f>2K*zo^Nf8lyy@|!ddm`|t?
zg6BVwc(tXH=JM@Jz;1K&oDU8QkT|h;i;^7J9zX8c;4lU0TbCZYp^{}a*h~-~7p{_-
zqrLT_Jb3&Zq$c^{ad0|IY&YAvfB6is{zO4exVa!Tgm8}B2b8jEqA0{~g!CcHA2^~8
zFVi_+Le_-rSbTHGp*F-grpdtu@EG1beogTGzwhTV?edYb0J|UAEOd6#!}XRiOH;w=
zWvj5(uKJJ16hUfEepLsDBZv*c{I`FD{e8jP0!*J-{2XWfKPB}VIIV#6`Q3%M3&cjp
z8N1(-9KYyh7=6?SyJynANhst0=T8@c!vk4;$(BZVJ*e!3xE~UxMkca<=jtR#I|nAV
zc_ZVNFKQEah)z`or*qLN#Cowic^=@jEBSc^IF3)V2!g{Arq^=%G>ma~n4H&w31D~E
z2)pde5&DOrZW6~z4EMwIe#|@r54X8NR$w=P<h(pvE&uG$ho=>=SU#WSJ<ruBVjyw8
z)_q_zcyoo|aRm}b#*;1jz-~kqBZj>{=silhJ7hKuKK`G{=d$yax$ahM_y2bET49X;
z>-j<M0Gr_Vb3@xI$lcS15C5aM+27|4N$zA}Hid_+w0k@}{EkSfg2j7hVZWCi-R}jZ
z)4=M{#gX~T7NXw0d+|F5O1?r?O9{KAg9#j;$Y!{&Hv^~3?d$Y0<Y4OL*YQ~#pNF{b
z9wtsrdX}m`+|MQ2GvRFOcw?;Y^6U|}=GRFFhi%6Hrzmj^au=`0IdJ+?3RtqMyacnI
z4^o4SiyKha|IO(<VG)mK{hz~a7qHzRGZK0;;OWC668Ac}Yv(3|-2~F(;=B@ESAf`2
z1&wCki)+CCw2zzxra|J!SpNc~J<qf_Xy-zMJK*q1=U!{^`lBq0`T~Cmt2teLV0n;!
z?_U$a@vGCb2W<Bm)_Y)kyA(H~ltUo3=(sQsaTjqIyC>X_=;~nnO|QR$^H>r8T#Kns
z4B>8Y3iLv8BTPLyy>v2WJrFIzZuu@(0>y6EpuL(84LiYNUk$f_?L;>VAK%Sj0oZM+
zD_g)}0a6>ia`JldmnmQ~L3~{Jhw26GOFd@rK3v7dhv2klxV>Vh(3wl%G+`Pt1#CVp
zGtkASFA{*e10?7AiS+pc36;(8{=b+Yr2egXy~@%ftpMyMkh#*66Txim*OK5eG~?5v
zoxiLIpJO6kdE4^$;k)3v?Z{o}T^FJCKif;xJE0#;WCEX`z|sw=Z#rXJv_SS^!~cYu
z;eI|Owgl`hZ0gRYkXOGToAq}87I3^~{alRF|L5Jk9>ZP<hbDMEyJ@BfMm!olX0dcy
z5Ct|5W{26**e%zZ>vwk8XoAziI<Gl!`M5vLVD>sz5pW(z(&9(Sce~E4hua4;4@Uni
zehE(7ioLUTS?`((k6)O43@_v?8W<m$wm3BltQRCU;Vu{04Is7&|6NOV!zh%x{^_4-
zmN$+|pooFgR#*K5yV-=x5^feq92v{H#=y-+7AM5M;kF#b%r0Xoc>VwT&xD<IHEXf1
z|34I$fpPxN4MrbuJWag+W_x7<WFEKqG-O^5<_<rr%V0GyK0$gNEBGuAkh+V@ZNTXo
z#C}`h2~IPo^IyW#FGw60&XdL6MuX`wI_QlOhN0QwM41iKPb~e|WDYn!k=1{j{1e>&
z@0kd>Lm62P#;$R3G<UlA2JCN`7`5pgs}SqOb6z9*KPRJRVYPRD<{K*;lX+k{!^_+l
z;idWD2e`hAc`mnmo(k^!t6~0lau0P3#xnbzg(RN-f6mc1u-zd23Ug<{<L|lZ0XQ2Z
zhm1Qy5as2ys0DC02mQHWR>_zM=C_-2Voj&H|M!6Ni{OSwJ0+gfg6#-DIBomRsmD;%
z|DJQz{2|v;xZ9R&fy}+fC~pUsO)ipq!RE}p(g-fkk==*Res{wGZccE&9oSBEH8B1z
zDULOd<MP37`DE*6^?X4nm<^KuBDx#wHxL^iPGWumw-2U{CoSGGCJYh2An}=Z+?LD<
zDF&OL`#u)0yFlvE@#b<vusx~b%5c56KcCuYe;9Iq5xRP0{;I(F+FNeu!OQT16D(lA
zb#avKe7c4g9JX%nA!EJB`bOEKAut*OqaiRF0;3@?8UmvsfE)sob{c})sxUDr6~tax
zm>412et9>%|90iGA-El)^KznP@J>x|I|-z}?hNW#zwuKwc5btR%oTv-abY(TU(2Y2
z>%ncFm7J@0ra}AvJr&bY`V(@U8sPTSsUwi_hqRt!ILH5kiZ+1F1KG*=<T2chAU-lq
z3GoK|1zC(#_N7m0U^6-uG*SBhyNmW?*!w_8AKo8lS;qrzGb7vA_m$1O_PH+DpD^(q
zKh|und)T_;yr3J{Pt6MJ;Nh|%TNtcX`l~v)z5V|W>i+-31)gyGVCKQ-V;dlI>+!~d
zyWXii$57X`sTkajg{eoT_xa9-yCLAN5uCkQ_MYXJLS1-T0qNOmf8E0DJ!1X{BnHCs
zbos#gK4?Jt5FjyhJdb%EIBe0y3Gw&&O+(Qq=9LZa|F^KM+L>oL1#ADmkP&nIf87Mg
zUA(1XtV{LJaf8jR{kaR=E{D0p_0Sd+K1>`JEz60>E9{Nd;5Y=SRTV4&rx{;~Jop$O
zNE{blu-X=99KrOg%=ZJk8OGOfo`I3>U~-huxf{2llrtc+{KJjG{r@xD%)oIC5<|!0
zf)!>a_51|=fv%4f{=H?c;BW=$nXIM{XTQjT%<+T7k?~?Pb*l|eZ-LddJ!Qj)+s;*&
z!Q<W!AEfRoN;C$mL)MSZmeTG*iG!nF`^<H%f^p9O-!b5YhsXW7hv0gHrRKxg=yqH)
z5QfW5bleS(WBC)+=6scU;P795#St!!tiLPdAz1$Dcj2A9BG<s??2D?hIBuwn;<xPy
zot7V$%!P|_y@sqU`?v<O-b!2A46Jvnq7Jw`Lbd~)oiXb^JX|lBe+BD9SA)z?Du(#4
zMLE{;T1O*XuNB91jC6;r55^Wc+6WI*nApTw=PjgmBIZ*;;$F%T8hK`r@xz7h=Mr=m
zNDnezkUj$(KC>S}_Q`?dHc#HMq4^l<I2=eE9hc+<YVY-34IhhDuU`oE+f^fuoqtzB
z&iZI}FUJ~(=w^-bM?+vV1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!n
zMnhmU1V%$(Gz3ONU^E1VZwQ=X{Eadf(--j(Jl8{tkI$Z24Bxl5WBwoToP=oqhV9Lh
zJi&7vAhVWt&jzzAq38b_)c@L^#h**iu8YqUEd}K>!1JAtUU=;cFck&w|F3@8iZb`|
zLHj0{?|<Vz*vuJt&i`LnnF*gGv1?lkb{EKv=s3q%9&8r6I4S(Jr()pwTj!ss=l|Dt
zrDNDz!N>*=$I^)pQQ`$;=eoR8CTDjsf&B^MgK*-lGn<#|sP9NVCJLVaZ!2$wo4YQo
z80>x@cTsShWE|xNha*TI2&ZpF+&2K?BV#8O$X%UB)&*Kr-F}ZT|Nlem9o!ydeIT|E
zH$*RpEf#^;Yyb7fO^Y*OhA3u5$uBa$5Hc6z9D;@{b+FqN{SoJ6gUmz5g3f>8^L)tS
zgxIXSr6^(BWh@2X|9_pOYKIEvOL+P~Hmi~W^ZY+1Q$Mgf(<8XIS-RZ=vv15|LUET0
zA3I8%A)ASfo#ECE58EYt=iz#9L__v{zghAZtNYN+nz<CQ4;oz##{aVN061;H_=l%X
zL|LB!6Q`8!{Mm$J56C=Qc}?*7@bYq>!1LuGF?77ZO3Un6cQ`ye(B-MY7kUC&M-4Ko
z>SX{tY){Twg%Xb-c@WOOaM5z=(Pv<FoV`6LX$d61s#X!~*YH<-yWX}WU^N4z4u(He
z)WPEzB*t;%nngR$3XD7fk^^DJv_`P`Ah!RVZn$2?_C?_I2NFlemLK+l-3t;w!SV>M
z?uq=7ZHrBWz-G8_d5_htdL4z}_{qE>w<Dyg6Ku}Pg>4p-zM<yL!%tmzGQ75g$JLW1
zv%vaHuPTAvd$G<GZ12+>nD>c*+=YxMT6n|F`K5yYOxh{Bf~yjAA!iJ3Kf!9zP^S&o
zJF9vL*uBVRV`B$~Anx3N$v+F3XJ%D`TF>mrx}ot%8#3>{FH9L6ZZP%4(9=~^z+vpY
z+ZL{O$;8R)x9<4>=i@TxhiZWKZl0I$`G1~I=3u|og&6OYG=`KJzh^E7n~%#3bn(|!
z@hIu{d{93)j?mTN<3A2zg1f0J!4|Avzn*_*P9owChpad+@cIh#V_hpjW~pj+?mRlZ
z10GKxF>H8ZVvOYvx7pyjjr~*R&e_82!1ezTQPjIG#Q%qa>)9W(A#1|$o&S>`fH>pC
zJN^kyH$<Gg3HA#%cM&V*kP6wOcG$88W&gkWV`*@BAe&PlC<d<^<NtAC*nQ_vgL$7R
zKT0}z&+NKo<?Q$!Tkphy<FR+U0XQ5$=JtGV2dmw{=m*Y!rqu%Aa0JPL@G<ed@b(6X
zkBndOIDqx0JbJhDeS9%Soe&Ve9K-!E{h^<?z}-HP1+gZnB-_+dTX-GZk0AB8nipBj
zyp4J$07(4XkyT(bzD;!jy8|SKj!XCv`5aviAHQ$T0<hUIHOTA#+#cQAemZc=c5L_m
zeBNn^vHy=fPz{{l_|~^=6Z@ABcB>8#H<*T*|Et{z9R4sqAv$lKFFb77Z~MaIb5`+7
za9XX?)dAZ{$ei+)cDOmX)IQQkKymABY0}mz;xZRqyo~1&Mi?-q#)IpJ|Lh(p@qw&2
zmgl^AvJvF29b|E8visv|z~KrqbBzLG{om|Kkh5Aq;>ftl)Zc1J<rT1=gGq<MWiYaQ
zQ5+<`*9*+s^>)i^f^qdM@d#=hl<c<n^91+#KPzAG!Q*w`hRJZfH@hdo*~snzu~QFU
zgu5e^eFHo$e6sd$oAj>?9RA0qDZtf$^uVz2;mKh6l?VKH9^U>Dtfnc&-Q4h)J(vws
zm;6k|@=DPqaJYcPn)S`V>pKr#nFh8`pmixYto-xSQOY5ZT68?CmKSbzeIDu=Jm_ja
z>k6*YT>b&<Mu~GvEI2(7ZHE*KJ+R&AdeHe5uQ2!jJ2W4(VBU?o_b|^~NmJ!o1UQ^!
zLrC8rt|GP-oaS2|BKmQc`j2S2xU<9aIl9|m{Fb|Z+TMH5!s}X%03C2RN<DJiSv3)|
zM%HiP8L-=6dPeEd5Eu=C(GVC7fzc2c4S~@RKo0?z942tT5hlK?I1SuBf$@o<xwmP;
z+a&#-kg-l%hc!DpPAJ0LsnHBoVD@RTUEuzIhgHJP>PE~n7t!sTab~ThNWgJ$J11t<
zmz^9J?t}aP;=3DA+BrKLAZJ8%euCVO+Zo$}bN^qCF5><lQ31p`6X<peZ(IRxQ=*HL
z!vBA8I=EfeV7~&T|Nq_;G9H3#*52tR@V@cZf|D3_Z#>wz<JsgVDE@Zs_SjOyHg(5~
zj~(E6T=x{R=MQA=<86@pKqR<VfZM<)6R)6*|LZ-`f!haCkBt30dcpSGe6n!ooOSOo
z`v2j}kAveASr3fe849@v2gFbM37P)@u@4;-vSM6~y8o{zLfmp^o-2m=jM0$$Fu&E*
zgZm61{pdLP1@`_Kx;k|JtizAN;Q<q0_o@@V|Bq=^!w$oqo*mfk|9QC!v;W@~FAO$o
zy7{qfSFT%w!)()64wSfK*|7pF4zrICU8hzF4_ijZ25|ZT>3MSW3%GCf>})o?Zv+y@
zh3Cu(gqwp)&4i7R`?6s2pD&yz%50c=V(Gi`m}mU$VK@LTYwA{91&1Lvvu1u5x7hbT
zhM+&N=_OVyFJTclTtVtTt%mHe1hG3}3bBSoh1D&qqT<(JJ*py@{r@ld*<k;uE;+yJ
zVdPt^W`N9x;dcwxqoj{LUp;o*NSFuiJHyl<)4ID`;NiiyQVO0vv@&<2n1ifG-Y^^-
z)*vyj>J#uhHS4JUcBP~<VE4v+e~i_wD^$h6=HAU;xbyWH$o;hwmo2l{x1$S1J!i=(
z%Vgn3aJYc%`Ob0!><7P-65#TC(|5?ZJU?C6f%zacxbTv`IdF5Xy}b>#AD3Ft`2Qh&
z$Q?yIuBw)@9PV(vm!ml0{>G&rB)&_!3?8N+J_t{~A!}Ka0I63%d=Q?@7oqv>?jo>z
zCd!b$|L@&aez4nqto#ml*PWu+4bfFP@OT8-gALC)%BYh#F$7+Q=PsBEj>ma(X6*Fa
zrU7>UoMq8qw_($dO-w!TG29&>d87L8C}kH&j1Z37cM4v9=OnHL>%VyPj#*>)L$F^#
z`m?Kc!r36c<?5iF%v!fm(ho=+9e)yCVJWlPA6&QP^uF79`GXty{NF3W0x15BR+j{a
zU#wgx*#2ld=l}Pg-2e_ZkX@n)cHnRWvC*+ffjo*G=<>wypIwHWOWc1_of}#G)&yY;
zd+%AUfRD%T(m|X<4%652^QgJ3s1}Nw46R}}&)0akW7a-Xa2Rd(L!4LnUiThY?MuBd
zuv}7RogvID5dF$74QKstc^Wc4^>4ECu6IQu@b(=@9|)&9J7KsVBo4wFTe`t<0Ag?B
z)`7FLCLFTdr5X%RD<FBP&lfEwFKfndM@^tPSReN-J(P3<(gVZWEm6-+g^3fQJD>jr
zn+;RfWjqUB{|j7~+EG*2Xo2ng-$F)6+ZCn{L_a#K3Qof*4{vW<ngcnjfZbp<xI6{P
zgRtafMX(wWn;8736|ukjy$E7HC+|4q>`<A)Y;d_s$gCNxnmFSKruQHh<V-dgKWM%Y
zNp6MdBSg>1WI+iNkovuBN5KAIO7umE50D%TZ(VTO{F9F}L4Uy1lSBWk<AbNM=kEgH
zY_?OS@Nxp#+)V+qttS5#0lVq+@*oT~2euc0{d%s;Vb}S^@^JHz%|K?`T`ULZQ<zx$
z#jWN=&v4)WcZ25?+|0gZkaIdgdbdu%eTE#{{AF-Eq>K>fpDqZGGQYWD8Q2dNHmzV9
zWEL`3h};R*`~J|eolk?hz;U(xcDlu?q!5&}V$-i>>AtBD?zYtLw%~fc$pUiTz&jPl
zStuVHwZY{PvK{E`2NHka=7>#(oCS`q2F8y$a&whZ8st0}Q$B0Uws()<?t1!3k)T~5
za|Dxq!0iL^L3mdEV+*^Bka=YgAB0U}BQ(BEod^ztyGAVFZ~=)Ei)YT<0Jg`gQwVGZ
zNMAzT-3@%vdIZhNT{}<v2ZtrR{x7f*1E;fFQ64)_r@erfyVn|SF39XrJQ@O{Aut*O
zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?@<O2f
z?pyF29L(qcpXq?-5ny~`=(N+C>tX7`c0kr`^*mZ){@A?%ZkD{8EO@@mi|H_U9(-@C
z%Fee(zr*KgVEWN%ty!KHsn31D^GFg$4R>;Ugv|d-iX+yUfb_&Y=>^Ywu&+D{HZum#
z{eOJN@5ASWeowy!cO%FgWNc8G12z*`j8yjRH$h<Y(a-;{zIq)jj%?nu+CupJ|B7Xh
zJ-Nv88GIHy>MtPn`-0?tr`BywSeCKl_05CeFtV75n3K{?^asx=>3i^k=ZVs9<e<#|
zbN2XQ%)5ik1mQ0=i@|<xj9t8Q;npIQIeU=&%VNZxsUSWw&ajIB+XoVpo-zmQ1`vC}
z<e8Sb7RylV+<SGR#p-KH80Nb^ybLzOZ22B=e1i0&<EQnId&$wo3Gt^e$fKBfIQTw%
z|No?2oIB>#AF#l-|4-p~H~8*KWb>KU=77y=SirE2rx<c(fwvqh$~@}ax>$G^AnV1&
zmU)TTtL3o-vYr;C_I$Q9IL*vGk&a?MNDdpWdiW2l9-ACYOjZ(g?_G(e3)nv}d1}$?
zI78ui4rb=pC@FB+<FHc_9M>>$Wcr|?q($4NJp}!MtcO%~`cgz1`!8t+x9jUW#5z`F
zGdX-GS=9)#g3GgQEo~TaQDdY6_Rj*xnL7o=R5ARGY!-;U?Zzc=UIVc~c;1C#bD7_`
z=l?a$+Tm_=ocar{_r=fKa5l&sWXvCj*w_8<oh&?#yIfPZm2IC6<}duC2d0trf!Gqq
zbiwJDzd?8BZXPwTe`YYN?x^!j1N#%C2825UK3UFv3ps}$#4jtT0jsStIRd8F9Zdn7
ze?99Vd=49=4jX1Jh=KdDcC#){f5G%l<$|mSchjA{qjlP1ust9-?+>?7+z1lGhL<L@
z!tH~}zZc$N@xN;W*zYj0I-?sJ5lm~rVygd1-z)o2^A6Z;vc>gq`|b+stuU&>ydM*0
zCyf4?GFSVR>mm4BtWvgB;Iwyo`}Q3>Ga>g1N-=CA7*;UzM(NQI7!85Z5Eu=C(GVC7
zfsq*k#vyLtwi(R#+6&de>*!&8V(6%B6?mIuS^?zVc7gMHJ0C|Hf!oL+GhX*s!P`3^
zzFc6)&a=*@C}JRSbZqVL!BS5UvbW7OMrs#q|6jWPNpKqsran6cvJRHh6tZT2#Wcr-
zFgbL(+TRk~1_O!T$`ypW5yVHv^_wB>USu&+*<Ts=fy48xy<{7*-t@QZ820Y{st4~I
zSFG!SyAh^Ob$`0~mQW#-FiU^Sx7qur{EqBkcW^xRajAms1L^U24Y}uT^ZeJ~u>r?-
zK?Wdk7(QYh54R5{2BRnKo&pZv>4#Z&34HN}$1hCY&Jy?8GS5$Y!P}rO9TDrFo6f{r
zZepoL>FeLM+hOr;LKB8N3WY4e_J4M1Lh&!meh{t3&jwZlVv~aT%^6YjIeET;um7*r
zs<xP%eR&7A{=cB<)0yb@6u+Ja?qA3;Jll3ELkJvavQtpcbGz_vD?B{V^<(3=Y($*z
z(qRng*MQVLn0Xi+7U!Nzg5w$_h7Xt2L)NI{Q=_B_*&_l|GxLuD*bW$<TC`IiH@L5Y
zY+g*oGVtCb>)sNa;q!Q%wwax+GDcoOHiuGn%yWdF!z`1*aR@Rq*QO06oq)tZxa#x*
zD~7i_z~Qvr8nKrSBnQIGwU9E^Iy7%r_k13(I*>Rnyw42v>=%VjQH#}<!QinGY<jLQ
zMeJF=c|IJT-)}IZo>haa_g82-+%Ntax4~%^B)>Ucb(=FYWUMb}h7(q|KD-1u!)qq@
zzn%IMAp3qSb9gMy3nI?21DSPPaKX+anv8HWI{OR3aV}%;08X#!Qtn_k_$Ld3`5-;G
z@Er}vS(hOB#e9A6I05m|v7zq7RnPD40o%3TdjEDu*EqNvjyN%b#nJVk^NT0Gf$M?E
zS-!q%u`GH$JbXdo%@4yhYCT%O?wO*D{k%kwI$ZdvyBOG>x^Iwr5~Oa^%E=piGS(1u
zXR0Ej&h3Ay@UfGFCuf1<@p<6Iol=^|!RGc#ZpN9HKz5Dd(GVC7fzc2c4S~@R7!85Z
z5Eu=C(GVC7fzc2c4S`WG8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF
z0;3@?8Un*F1ngxVqpZb9cp`$b-hmh&o6nmFUmxbMD;d1TW9^)loe%mv;Olno3YWv#
zATxFBYj<9Vokwu3!ZgLlmghEK1+QP@<uKaiwC)G^{Qt{~QP2OKn+LfEOM?$`C)1h5
z={g{Lu;JzX|KV#bs>S6{+=Z+@r^5v77i2L~*<#A~z~LFj70iRIx6L{Y!(Nvr2KYKS
z%_o@q|1xKW?({UN1g|}TnYH-s^v%78o^D^V?HV|Y4x}H0yIDc+A(*}2Y7_XL)wU^j
z;cG`g`VT%+hua6@BjX2bkhMAqm)UmQIr$Ak-7~%a819GZ_rA^pwim?D`u!iyo+7=`
zk}oX-#Vohodo3iZyinEzg7n>d3%P&L*^nK)t_389j&CfNLkSCXd0c!CSJZV&Oircn
z^?znjT02)LeF3fiLy0etS$}?|&Ii%RIH_PcI1R7;=D+F0Ye_KsP2*I!pFwK-xFF|+
zgV@Aiqf>}{bHf_4_6n@;o;Wznr`%~q$+KWNEX2jBtzh+7q~JoXL7%~Hh6$u?oQN~L
zVQR>sU6<B@*D4|FHREOji(PZiLU9AKnuaSj7C{TQ(8NtJdtRq_!OcIl9&*MqNN%1O
z>fHz+F%a%OsbC#{RT~_(D+HM_(x6MkRItAarCN5yHP3~c4bqDZM@+B;mk}_z1y{H%
zcR1oc|6hA6<SrGE`t3@P`$a%(4s*yE_#ieqek1A+cjxIE$oaJ(HGQ(}<_F%Ug464?
zdc>YPkQ@xhO@ORH{cy5%XUXNYV1K6{=&_t2h<e|k(Xov?RV=TA%>wD=o!190lN1kJ
z1*g~W_qyP)*sM%={U38ZVjW@G3lp$DkeS#pA7emQMD;nay-&aLT6Xmy(n~p$9?rCd
zY}Sf}ZD4;Pi*+twvCLpQgAzBd4yI}?HlGg`JCL##9;V3p@UcTe*um~mm@f<W)AhN+
z8*a?91)GUaFN0l%j?a5Wco{y^4sy17)t~=6&lkJ{yFcUBXK*;+HD^?EGz3ONU^E0q
zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(
zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!n
zMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmwg+R*M<={KG
zV2+xUTLC^t0LCYVmXN#zKflGzX)gHufnyxpI~o-b=TtoWudo|rmh+t)@Ez*<R>Hg9
z9bO7Q0|6w54QqOaT0Wn74t$P;`!}gwAJ<lZ&;Q|PgWL}X)5BD834DG+2A4FLc7G{?
z^ZXxQ0b%(0KOAo_;Ph{mkrLQ1FgwVhr?LJ5+qHIw4KK3(!@YYk><wAb4?i0xH6C(~
z5wf1dJCAqh3B3dR4<`OHOLmL6^41--pVooT*jRnQ3NG(faTLsEIi&zTbK=GYcCZ?l
z-WOVv;P%19VD$NRWw74f+l{-#?Zx2dJHg~b!fe3t3F9Nv^ZTzM!tDgYz1s69S{>be
z2cA|ydfo&bv79Ti8$}ExF8k;i*v*M<Tfj6(3>~-0%tY9OE{TV4{TKCYC4Yl?YhlXS
z!y|WfpV?=jYySk}{2y)Ez|}ByF#2%xGH@CWdVO@;{CNi8FpEFt2Nr{=sYsbck~?Qy
z_zO=PjZ>_xVdm@&nF$upXy!&SA104Z=Q8u*j3b!ZogchW-1h9@FYwtlFnMaxPtN?t
zh)11!^58q})Rp(*v{N%+uf@N*9bkV`%Pr{UEZnmV9-ccgec*QeWc7lxk@Z9;>sY5U
zZ34@2JvGKS1EzPQ6WFi&CE|Ch*zSd!jcf)oyGm;UN*s7hiZ_qdkpqVfvN{<1X2CbG
z-5~z-y4m0~2VyVLZ9*{zBnHFoXTN~m1ma)0C=PdXxTKD`ad0%4&$;nAm<Fjq#w#;j
z!Fo^Y{@l@M)C9Iq#d)Rqi9Ey|eIWI76#rV)*)_r4#!<`w)@N!H1CHPAH<`h5X$6GO
zuX5Figom5Qj!$4aL3UunDY_F^ZAqvGtFezYweneU2ClcyZw=0HL^kWTsV+uddG8@<
zxh0VwY%WZ#a`b0S*Tax7SMpqdA_r50k7m6X1P+5up1R<$0jce%vDg^!fe)U~L1Ngj
zouZiz%ce{4^MBsQRD<J-HEqJqD;kVo_b<303pO8{{!y{f5Eu=C(GVC7fzc2c4S~@R
z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7
zfl)9T0;3@?8UmvsFd71*Aut*OqaiRF0z)$dCeIB~zIk|q*6-gr3XZ!awO6-42j2}l
zEvH}O@L^ZgdtaWLU*7MlG`;eJmc`E53YvG#R6<%`D8A-iq?EAan&RGUHSMO0OSEpj
z?ogPsT2A(XnwYYY?m?3?hgg)S?6fuDy|EFz-g1u6eDjbrZ}ZnDS1E{v%PF?z>s#n=
zs#M#dGQj{8a`vJ#%+5?sHD9+YLB35YQ0<(spu*mjS5)pDbJ4EYJ3(WGsi&sdhL6hs
zAAU2FILl@#V9lVu(4x)!k=zwS+fNqSyBMyS2k7ln;+EZ@{A+=_Qo&7L3#%9Z4EW|B
zG2at=Sn0})Vxv~pChcUdCJl`}I~Afl9koFAUa1$Ca(c1FwDHhR&BoKym0w@KZvN>$
zo9fSXE!yqc{c6`Nx2rAA{G#$Dgi&L{Yd<qt(WM%uOXsP6Zn~n<$=jwH`+B>a>(l9K
z;cqMy&v3D5G468EkP_}z+Hg)*E3AK)`nJu|st5X}>l<cUsfhmDYmh6_XwfT|pzKg~
zMvhl|g))beykYi&uNoV4SAy5X^17{3vAn!pbxW|N!G*0mj9*DD(E7x;L*qxC7C0Pj
zOr~gq+#P*RO?ASDS7uxvtt})HaugZ&$*F?Gr##<a@_WN9Qy$ZP(}%OTlzrREE&Tqs
zs&3&CR{pht#iH={dL{8WVP@Yn+!aM9F`2w8a@Cel%hWh<X{KVbSCEnAtJlUnkLooS
zJaE_1EA3Xit8qrrX!biLxrbNHE;O`gE!Q}2E|AJ=l67aAZol6_jeeG7x%w&5Dl@x3
zg53{t!<Oy^;5`|?lDd?l_^g#~aao%gM>T4EHVe>vy{*h#k#)M7UU-s%<6LJ80mWdG
z*7@GbN!OmKf%M+}D`I{k($>gYN*f$Mp>@`Zp_le5wSS&u#&hAkmWr~d`TqQ)Iv33H
zRgF@)Rg|mWshC(ERoGLRs8G008oY*9MRBs?X_4Q`?v|g_+?D<-{r?#)rgg+oasS6<
z%HP(VSMJT9sB()>Sw+Njy?%&BoT{2V57_O_|G%j3O&3=G{A9YpnN?|uJmphW>s_MF
z!X9rmd*t-jpx#$fE9$kV(#P%os*ZJ%8V>uUOlES`DDuCRQ1gruQ@U93O1WQI(QKuC
zl+=lv8%>hixXi;k%oH2nE;ZL(9;?x8ZmsHdfy<DE>!f*e{wKw4!RD$@;<%J^<>o6C
z26L+@?6?Nb3zm~t=*m3i(`b`z(>nJ!U4vimta-`XyQ*goyQ*?1SAyMo?*D1^%Dp-k
zv*)EME%St=4N&;yaj`3|@D7qc{o$$th=1^$qsk;lV?_}A{QKwTGuiVL8CCkV>u&y$
zJ=iF$RJ!E5R^+|2T0ai^DGE=Q(9&X@VSY(9OzV8Dib}p^gVBs@-sWec<aDyXgeZQ$
z+b5~)<*B$V?g}_=R((}c{*=Qj`@+0k>7)G(5s)2C>@UD^e_yppd&Bc)&H1?<8tF6t
zDb>cT(YY6=shXPlU-9uWPL-=O<;+%gxalM?ztI5cQ;v*P*(mqU#I0bxl2mSuiT9J=
zO8a+Iss^kyFyOLYp{-pltl06DS^3>U8`Jfl9+|WHr0FE?by45*@{QhQ@r6pF_vY%X
zDSoOI>vu&_l(9p1jqOg=e=M`rdsZo{E}J-4lJ#qm!ZZ_cjqAUB72mIr1*f$Qav^GS
z%{_E7)cX{d+-Fk$K4YQkst+HuR_At_@tvBYtjEx=_2$%hGsmdU+Mi07=~Q3&WnR#c
zr14NuRFlnyQQh;}HjQ1n?#kP*7+M(iwVNMO(N*PI$7B|?;*-K;<()b&JXBRqd|#k9
z{nj)ScT-N4A0b*6mDiS<UbDZd@K~ryHF!CjLA$(&<^;Ji<u`X{DI`T-FwpnoQ#X|0
zH+}zJNkhYDw(=X{f67T~>P(mI@HG;;wO(D{Ls`xKZ>ZYb3k~W=X3kb#+Oc0}+H-A-
zNpJbIbEY<EFKX%qrw`K;yc%z8Ma|b52r4pbGAJu4y;Trn@>g-Wy<a`%>`WteOC$4i
zjw$M?MYg){p0b-=x~{3dr)q`tBHntFg}>s=yq)hV*lt^AwpQSsx|Ip9;j)5>@;$$O
zH7y-ysCheXFkjkbrFyHI(c+?EgN3T-S_4*5PP1LvyylJiAqJl!-fK=(eWrORkIO=#
z=brh=m&X*XAG-=)lRjjc_2!O3Se>rf<U_CJn+;jbXML(Q2skBaoN_=w!Q;>ma9S{#
zXROTnTSD<w)N0l0#|O0K${18iZ_F{@d*q6=<v|+-_B%2bLLYk6YXy&~m|bg<U#4KK
z!Zq`}%FTo&>gsFkmF(`^)Njmg1c$?xs7dCv>r&0)6jrNB-ttjxYSc7s`_rMgxz1Fn
z_i%@tRgj>fZ})q3k0ws7JR=#f{agK<6sP_<Y5u%WOKFB;vWh(0P4$#pHLB<DG@3rm
zd20U6Wwl02kAzn3S80vY;T?wWRn)<D{e5%U+^2P?hA&65s<hJ<<@*Oa)v}d))MstX
z*5Y=au5q}BU16`o0<}|a;ffyeJPOA5^Avf%>zMC*b6(|jR5REN+sj%?hbIRbS#fGw
zY}{d`((R+ArqUju&?6_V{q;<RIw-IFH#w)V@QtCmMnJyFhNct7&Bh&?qF2sq?<l-x
zwl>;A)2ueZ%=6V9bMATR8pXyu7LWhUQ|>7FrMT?Wa>X_G%~aJj4yoSS;Q%(vVVj!@
z$V@lO4lw_m;a+7BzbD#E?c`N0<wkK1HO<8>YV)3}s#)xut@PaN1h}5*wB%NM<GmfM
zSJ`7WSPw`a2uG$!f%Df>Uwbtc?)hr@4T>sjoGaBml_b<dG8U;A#hq1o&g81LO7yFe
zF!OS-TIPGq>YR?MU^jrwfZ^LlENYs<C15pv{~xKG`ua!V!rTL3H-Oa4dLpC#;p9(M
zR(2KD1n$Y;axQRBgqlfyf^uqCG`K7>OJP^H-{YsM-0!7!*u)QPmf6?2V7EVxa|Y`P
zSkt3E<!+gpU9KM7?57o;V6`B7@(%MV@!aZEGcI@q*0<|loC>egbTxAu8E`rK&*qET
z(|b$7?&s}V4K@>`FXFPR>iYX8$}F*u<n0Sx6&g63;AVp4k#X>E6|j5t7amvc;r*hP
zb$x@HXWkiwhp!sdj57RHd~bKj!_<pU*H?S7okvA|tA-j^&3(9=y<0-S?gObyE<K}E
za_xia%!;L|PTN<j+@4<yRwwZ0Em+O!WzwoY-)Vx`m&}aSdQ4}ioL1lls{!ekXpaKB
zbx!ki`A|=GFdHOZR=i9l(}4lZcR%<-WucLVn!;Ttu;2UGHNbxO^KzQX-2Ulmwr*0&
zv64k#J3xB3JyKR~wA=%(|5Y^YRO>IQs^9i+0Lv#=CCbg4p`dyynh(scY>ANr>0z^c
zqvlr`4HgS%JS*QHcOPu`lJ}wDboZ5Gi7I=Dsv6scPO#hO9V}N{q&PuUCBq5K2bmGo
zgBkAA;)B5Q6`B1iC*^j5-3?Ovfk6dqzERF|rFMS-wL^0bsn{!IsugXVq4xBiiCRH<
zCRlz#V~X0&NiV@>fb@M~TdyLh{S?l>?yyPanD+$LPUX9*8;-St#Z7LhtGZPjQQdt(
z80;UAz6s8TV1I~A$dd)}L#K7A-unMUrS)aAs;t*cRY&uWVEa@fWfUz%G}R;yR)f<V
zNG%MnI-(*0Vn6tFQrYN4A6Pwz55gahrl`3(%uza7k)W=*vsksqTLx?&sQwQQJ_&XQ
zr;avQ&u8t&V0X%z$SJrOPE`HRH&-Q2MguGdGOJap93D<zPO^f{0f~dKteY)3zCi5d
z_tL=TfY=}`ap|>E-R5w$c*lES{e3Lg)wG#rgU!u+yBF*qj=n<WEZqjBhNT5yz1L%k
zRat5y!0d^|6)Ku()6{pn%>&yxd8VJ54GXX8M>AJ-zS0XSDLt3emrZ-F_Cx2A>V_ms
zwHt?{RF>YfR5l2b0NVj_0|*;l?g0B2#0KG{-gU|wUY-G~S)_SEZDXaOdbh<~)qk%b
zX{<7AhH@wCIn{HLO6tA0V$}{Eo1nD(<{hw`y0-<WDTTiU`^RKjowAMLK4nSId205N
zVqo(?=C^qms3mxuR{i8}r<`EbsFL*ZyYe;rt4e#^ofLQvDXQ%Gw^Q|$yO!$XkWQuO
zqc4@c?1fd=i)5;`r~Oymd}EUO`T2)bZwCDV+gGceqI#tC2bc{q8-#B$iGb}o^I$QU
zp2U%&7N56LtxxESiqn%hs`EPf)VDn8S2!CAQFmj;4Y2<1?TTuCJmxBl&)+N6@yn>)
zT<50vaizPO`GUPl33^A>d*tlZI}5GBdHQ*w7MKRvSGt}_CCYp%m|w@XPB}qYR_(td
zi>mB`i%Oq(v(*$Ap9Q<)_53wpdp2HVQ@eDJQEvXL^<Z(38Ivz9Q>~nTRc$rn9JTAJ
zB5M6Uoa&(ZKTYd4IBx#kZ33GQQVYY8?_0rg=`lya=1eIyQCsuGRH>O=Qnjn~HQ25)
z{?}l+%SK#aT5oQ)a>tRgO6h7d)Ph$q$>pt{sCsP2eO0FbX0?L&Wneww7Ho1Q0)N11
z>azO{a5`MGrxL6mW;cj_AYP|BQ6gB~%#a1eZ6zm5!Te=^KdY{vxmoSJiN5MJg*~dt
zO8eCAEpS&YPdcI+P`*RWa~`Bl1evw=up8KJjc7GBr!zOfY>@bhZ-olMyZ0;0=w_?k
zo(V~RAaM{rdiVm|O#xe#!FtU1bE><$?*p?zYWj1vz-s<a3|9*7nWlVUTA-R==0~L?
z8+yR?(Xvy^mCt{F1Xln2OD4E33GKY9_Re#rR$E^eSZ~CX-9{j@Hst)&oNw%_w5_7c
ztbebnTH$haaKE#nVXNw;R~uCeRPt5$1AZ$<wMJ>L^p(@G=9E%OeN(P@tGrd|ezB9W
z`t=Ca--??J9<e<(zTtCNsrQ+v*@xOQrp#S0Rau|qtMbpEW>(mCRc(i9k3p`GwMwo7
zvuRw+ZPN{x`Q`UmFV|2Ho}~D9=}gskYd&bREwIr%a++29!NhCIFKQ}Hf36iXKEE_h
zJzd?{EUJB#!KGcv>hZOX;ByPU2=XiS%?(pyJ~PMcx#28>7w7EE+|TY;n<r*u7%wZY
zqaS3aS+_S=&c;JZJE7*7Qp*2xa+7zeo0OXPYORPptg3y_#jKQ-U*%~@wf6tVlU4U9
z+JWPD<@HPr_o5RfxB6Q&qmEgtbgfyZW@)0TdhWTpN?fX`+MnN?YD?E1P#64Lq>-GW
zt(>}`P`yj3Thr(0QdQS{8v`k2c4d1Oe?#RJjOJXvUdnMkXXQ>`7FWHgQ?9Q&=d9{`
z{TQ=pQKl*hbJ~>O6lkbfsJ$=<O%F7BAgH3feZGX2X_L6I>v=_Gjq_YO1rm8m3$89#
zF|N-sJ(#uJ%y}uI&0;4SYJ5J5+r-fQp>fr1Z{@bzrr>tv&vRvJcY8d|Zv1jpR#<-5
zWXkS6%9&gRM(5wksh#f#*5aF<p_=FQ)<E%>uhE2nx!|;*vu25^;<{AjZ#8VHcjmU6
z+8t5RD%C4Bi`yDz)W5P{H!ng;OP}MX>Z=Lbs^6Cfm|frbK_OeEN$vCw6P0z^j>?Dc
z&s3YkyGupAda+5+#R(c+B~4nB-qx8OeJ`l47oMc}cIyJO3rrTql24{7cbflJO<ws|
zA@F{O>4r7@hM+X){P~LV$^HI{?u{bqYqzSJs`OT?g?wu_jhO17wcJiwExxxM?9PZi
zn-#NKKB#2m1cK8*`VLXG;%7fpR~jBtlb@%iD4TIn)irj9>hXX3!RkR~CtjYe^dl_N
zATFF)<@I(~(-(@)3L5#>HQujcR+pGzV4!w@QAKb<tV-Z1d6Q?XmZtCLi5hymZqZWQ
z_CVR?vzwZAP?6@;`FoA{qtjK+2RW(j3O7^j*B4OxvZUT*Qt}olljc|>5e;@z>9gll
zZ}sb#C9X2m1lcS7f3o5GY+f^uWO)^ze=ce~|4J3D40M&YH7l#`dLIDxw|3%O-EUR$
znu&Q6)q>McYF<y%Hv21o6r2uiqgqYRA6Tsx@PSh!nCrQjT#}daN4xvRtB$Bhw&i-M
z#+{H+>#%cEyST1Ee(T2FinX6Nt6Lq@R-5p>LG|OTUbFR*n^jhM1cT4QSW>fArJN5k
zo|Nz~NHv1{iN=+`KTMy`FwkCqMNs+R=II7Z`V3~j<6mn}JJ_cvKWmqm{(ml&$E!@V
zuWpzoTcbV|T$a2rZwHr~=Rb+5227l$Qf`-_vUq=+j!JBUroZzOt(aH#O1E|ttL<E_
zuNv=oPx<jOPUZg9StdL3c4<jZpQX97uu1vu$xrI~cl5zyk1H0N8*67wQ(Bhjt?KNx
z*5E+yZgAPtR8*kebK$;m&#wt)wlgPdZ(F!sZS%`gy)}vVRRcpsb<3ZZYE6H7TBZB!
zY3=afO{%T>pVY(Cx2gyl-ILAp(vhFVcus>U@4j+Z;9eC_TzKf_sA>G&qjJh;C)gi(
zY{_abRPSi>Iq8~8=RQ~S7uv7IeZ<U^!@pB!1#hyh{3|mv&e&$nO3#Iwara#{tHM3i
z_Wuo6(s~hL7QK<d>|-pG%Jkww6Nk)c$`S>tx`!WzD+QKE=!^X4G@cm4uX6J%mw9{4
zYp@@0huW)de%@tJ=ohKwZfUE!=$(veV0NTo$_58@i&>Y91J2ISG|1LbKTyJ>qCfGf
z$w~Dv!y36|>di8X!EO~<F-JXi=MkmcC7QbK>zAAT&{EP~dfQbyJ(b(c$nl`6gwIaR
z%`=~yr0-v%rn<9R^*LvlvPF=GDp&q<jewn&hB|pprgzHSHOng+bXR!nHVNI;qfuNN
zplWO>WE`US+@z<yR?Q)$%XF$$v&I)`U&E}q%8GU?G&ECXc-5@if0))hJgU6qd5c+{
z!Vj~c6&nryNc=K!y1mnM@{NW1QtDz_2RIzHW5p899!lGqec5?KB=E7dTH&L2#<x2w
z6j!;QF{$B;P}Tdr%+%S$%kUigOv97|0*Xg=Jq72HJ)gAIS$|6?MfTnU`?c{hr)v3P
zeX}zhyA;hD&Z}+-+G>`QprG~lW0CUro0pZGt5>URU8bzIKsH-5mVs5}<JCsJs+^^!
zZz5(Zd7s~J%CsZjbXmbfW!W8EYL{EPjC(bot7sqErFuDKmXg_$5S4WKOpRY!G3vX;
zmumXW^H!O{tF7euz}u`%exb_$!U)xU57IO?+8HawIWIDHoziah<DY^iPq(X9?TYIf
z8-5EIh$x7f#YOK_X{`(}3yEsc?0l!A>{b)5cHvT^T6QG6`s7A$?QFKi;4w45^p&bw
zyH6`yvY5*~JJhXG`uU3Ki2@~+#BJhco}cG~+aZ6eS`996tTgjv->sa*^$%QTeOz}z
zn=NskCYQih)#a1-Yl^4Js5wu5X!P@`kdf0OS?#?uA8SVx$(g>~$)zE^(8Bb^!B(@M
zYpOL_C+q^3i9X>^z+w692#+$T|F0L84>s4PIteVkbKNO$`J!bv1)Mir9kNu{EwxpA
zf1gcFp80~BeVZPr|F8H<>455!!d=RKX)S7&FAKqK-!;cqfN4;k0%2}`Be0o%)0Nb&
zm26b|H(5eu>g|{6JqeH1J=Nq?au`_E!bA6|I<>K=JUZQ`mS$TeXa2HHy-&3c>;{k-
zFg)dmwd%1$VPLl=*>R|Zs{T^=(cS|#AEbu;UZMKyM-uAI;onuO)tSNV9EtZj>T~NE
zl(jsWRSI36Db86vMSaWf?J9a(->9wJ3mH53bg2(qW_52@1k<c1uB!j=Y*gER@gq16
zKxXSLUkbJt#Lm8JqZBH)U(H(gBG~RDKewo)anDuz+L{LrFY!HY>bqk9fZeZB-X#Ol
z1Hy_4KUM56MJuVDnWDIlZ?00zdJDLjAbDhbTx}EB{ao)9m5%LLtH!2mptkz<WW~CD
zb!yq+%N5y~BEjKYAdx5E7vHP4WbQ^qO^4;GdmC8cW?tgp1-lPqZt*2fl@EMZ)Oz=p
zs=PQ9q*`t+307BQJ_qc_zVoUotMno5K0m>gYQB@-DF@y$1^Wl2-*C!2uzGeeHRT3x
z8!#IruBAF#d5RM+n16j*o7$IMH&xGH{tM>Ey2*g`gZlrEO<UCG@a|Gp_;L{3)&%K&
zrTIlgdqpj%{#SV9bxkEJtV7)(|1MY$sQ>@$t-R_HO-T9d<@^Bb-&IFfs~=%?0kc<c
z4pZ8<<to^&C5FXdee!H?R3^+!R};wj1or#8vpdx0{ApHwx|9{nSGu@cIq~@$upW>b
zU^wY)3|Rl8JAEoaU#!9AgXAVee*lZ8H>^<lZ2ebt(N1-h-!E3GeM=Np*WI{O_1FAc
zV6%ROzECsyX9;G5^rf^ms7fA@0NV-TYlU^FPWsoaw!C?&>PK-wusS}2(<)CLW~+V9
zIuDL(kh)9EUf}ouu|Zg=>!|87odt?<3wNt=d-ki`l6ni)s~Rbzz-HH^`bQEmMgvlR
z!<bbO)c?2M{a7i)agNIVw=Upx2$BQgi#-PF8*7r3U-$;7p9$hsOKJ-T+r72BTIua$
zL$G@dH6VSbYI#R6AJqSs3X)YdRJftiYx@{%Kgg_Bsd8}GfY{l)KBKtX{vl*62_(1R
zHDX)@#E<B!QwiZUh4=rb=hv&LvDARwxw5bc9QMzzX(=C_SFZ5Q5z>#bYiCoPSQrJi
zkMXFGijbI;Mi+-LSgqR|MKxWanJO-vt?E-00#&tAY&1%5^r%a|@=}#Lvqvpg^MlG<
zuMTC&R!CnR<OUGdpRpKh9*7OXzvFe4W@re4-LW9~otkg&6!k`Jc@_8AOJK3Z`!iMB
zSLdtE+T5ka?|MwFI<i?IEJhq09?3J_sqK{r0f)hRAq6FCVKJqt=4;emsvZW1A;^65
zzgJaPPEJ*g{&z|4n(-a^#c9<l*X*w<HY?v!th$n~=;OnxHdU%mwXu7La_6#WrH8T8
zReDbSQd4u{QCq$Hw0h>cnW`&$7{T^2IqX*HbgcrjL1xG9Jfir<Q3=dH%#{wNZ4aMQ
z^DB{6FMqsD%_s4XO7GIc>XMmfl;@RvRDHI4q4M+%8DMoM_Xeo3cC1!aQ_53ti!D*z
zmo!(=C8bL(ncG~6d8wnu#)mu9+2^-{!$7fV9+(E%xm?jeewr04c$}QKCQLOTQ%y}U
zJX`I+pLRu0kxsRV$wpxFTU8)qYlm8wtJ!PDD(bmF#u7kgv_G~}^;v#ZZR?(?s*Jsh
zRYiAqgZuy2*D-_RX3at=u=yahFwD!b3mnd-s=vVIOF3Gp9WxSF?wz?!g;miOELP_8
z2Ta=~eglX9#$BI4{eKna8|`Y78vB$a=H6C)JdsCLTIaqhqcLP0!S`s8vS&*=*zD_(
zTjhTroU43U9x;Xov;T%;u1byaclCaUcz9TV<ScgEg87S-<keoAIH><L?NMnlbW|-&
zU{LqFqo~H*CaxO6qp7}lt1j4WAU!@Ex4`b3`|pw}mp?n04HB<^9jk16Sxfo<BVo0q
zlU-nQLE<3%+$sjle^zCu61T<=tVf5-RlVSu2AB;}QzNniY=@4gm%_xz?TUr(`P6IK
z(iL-Bs}*7uFDNZLwOolgN)W6b*8d0T17SJK?O^lwRJViU;GWeXRUVCrir+jps!F~(
zrRErNQPpDSY()=wNZq^dJ|Z1@zMKWllOS_I*nRJQupQG(tkhDU_o>aV+N#X*<B@vD
z^TS{}Bzj+|o`~5BR#WpQ0qka*htt#@uTBTcgUo>8%_^O$cEXWhHN5=SmCBwrNCuZN
zg7t#b%-qtVo>0c8ZmIEDwQ0LN*d6(YRn?0WcYxU-HRj8E)sO4VRb3HxR4sk-YPepI
zoW&b1IS|c!^^W@I>~ytf36MGgB<ABa9c(X%y~cn=>52aiwag2M@ftSmscK*4Il<v|
z;efe%aL_vp_iyo2QI*uU1-s?O+fHR)ZvnWOAaikH&#p?f<0e9C7k120l4?y;yPh-|
zUjKvis9*F{>kb!C+K{tAHDCs0%oZfRjbjNoErZxzd|Ha<f;Xz^y9R>&%+{s~wwrbT
zEU=p|2cA)Jl;{A9Z4mBOyPzVdvOx|pe{lE9B(NOFtmD5p;cQdCUgf5jTfk<1dEc()
zD0E1*Iq^7L9;AMqwxf#K_eOAd$jKUk>l=``ZET?m!{&6b_;s=MDuF7K)mMC(2zC!h
z-g)zWl}PJQ1rQsC7ya&5-*IOH+ziPJD^ct^euq(2wKze|Yd0iqinHzp%Lj=f>SDL7
ztza>jnIL-E$J1bY-IGtK+%RSW#}P<;`l3r<am8<|lyokxSJiPZQVF~~M@=$>L0x~H
ztLo2>kHBUnZ``RCbDIs!2I>3G|4`|qTQ)dcK>V3I4yYcGn5-rp^hiY@2a@jgExxXj
z>Dmd`15)R2!3efby*2{uN1=!%s?wYjlwS0nQG5DcQMI=hQvZVL|6|iO!_5cj#fHVU
zTY|$ECTAxL8N>SjS4;iwnn$W)u2o=rK=J~Pkovx-c`evYVkb@D=D(ghPsQoVB^6V@
z=U_RIdSQnuuv<ZFeE50eNw_&Mb?3Jm!_B#Iz(MVre=dqU4T3f+rReQe;y-5yRyX<P
zKb7N~`@r_yVY#dD=JY>xS0hevJOv)gQ9CLtrc(W;RsE|#n#!SxIvO8$Pf)k6icr1%
z?7CWRUzf^hy*%Zpe`P4{`098RY#z)_aSvORHZ6eouSjvL+Rlaw^=|EU)!PEM!ETG~
z3sH59^;a_psZrarYPZ^pdjX19PGo}36z{vI##wp~oK`!{)Re1ay%cY}S)!&Aa0M(6
zvO~8@PIb}CL*P8wQm&@*{<fSFsQw4ZMb?~?cl#o&_OuBSR>$ism8(8oRJq~xNbQk;
zfZE>YyVT#jcT|<%umEh{p6iQL4)A(_!v>@ugcqNJw9ET%iGsz;`6jEaloD1C^~zCO
z{^yU%THX_2f4m7V1e@VdQvg=C@UxEExvFZ_8?2EEom;t8-2#jhWVHO%<arnr553pZ
zXyjR={wlB*tXFv9d@v2NFT{PZ5~CsyxPHD<?yh?HbBx-9nXzhHIMS54%k9*{oBYA%
zvr9qJsX*2awY3-QmCsIfg`1&U9IYCm@JOxc*BjLfHRo03GxOkbAUnR@5dfPHV#Ba>
z+FNiKo~VPgy^p>rREzn}qx>soql%%O4A@SAH3DFBX8JOL`LU_r!1aHnMh}?p>(s8w
zT-vPq&3C$Ln&)A#SlDc1mFKT_g3T^|y8s-Qd!C*KhZW3j5PdfLp9-h_8+FaeHefqI
zVjwKJiyzFNr*m6XRYX~R!k%`Oq7Ck<0on)DoZm>Ry-8iG`uv!*`oG^TVEaINa)j1{
z&2RH7QeEzN3v4$?9E3sjzxpF}wQHZt!RCU*K)7?CIh_Cc7bjSMQJaeTw}LWwdhV3l
z0Jj6AXLtWywe6*eiiWAKV1I$?|6+4Q{syTz_1FolfA14<aG1OQKMdwO?{-w{dACOK
zpRku2o0*RKMu(|t_6GV&jgov|J67tRRx6*P16J$eribE({o5e(g$MjMs#Tc3QB&L&
ztr9%NSVK)!L?hQqQDwtQ1vMQ(9W|~11EpqTNMFuc@rC+<uixN)1i1}_eS%)7#=m<A
zw)3EwKG>dSp;ci2gXBdwo>f=c8>9YWnyT8-zv|#}P1dhfz0uPZKL0=Gz-4u>3pZ4S
z=SZqA7t;irrT$?X*sR@?c7Vesy+d8&%cA>gPB(erX8-=i26s<G>;XlG050`&k6pp~
zUBjj*mnnQuo3IbkA4=p|r2dV~8XSg(N>9LL6v%xUkGH8B&pM?9n*TpH^}M2wY5-V2
zNDdi){kcGK)ea%B`>*i7Q+#sox7wWT#cG@j-znMY-2m%pJf019Ge{qI+A_6-##E)m
z)Cje{-w;26#P@z(4_C9sW|P9|BuVvU%hl8vJw7XkDb|6_5oR_9`%(3yjB1PI5wKWx
z*f}-spe~i&xwc?AkXbtKA?@i8PLYcDO@hJdK;k^Y`YI1*L*^t8N*z$UP{^<L$<PcO
zR_jFj!G32Byr~k&a!CEEETUfx(i{0=waRjh9pL`|J}D>Fp7f9EFU2;4(-Elu@0XaR
z`Zbyx?gx;bS=*nePdm>7Rs-TcyT%Q+tLRrF*siUc7O8sqEK~cp@r!Et?8%A+{1WOO
zYV4{DHu=K+qI?aKPeJ-&`1~I`u)N(Pc~zDN24J&5a?VV%z~W-<;YtGSI%>y4dR4xa
zGphUS3Re&0{jK`B!USyp*O&V0Q>Dw`;UaJ@Q>D`XDp)Vb9M$*7RNeZ|s?9&epw=*7
z5v*?So;cM_doQVlYwm{oah=jtFx!mX51bay=P9TqF1(?{W+S0q;&onisY4{#KHn$n
z74DP;VV(a6*$2XVYwVRLO@OTB0r5fj)|4*w9*rKw`j#o`Y@MEJ-~VKR(-~;~Ut{%h
zuzT-|K*FbeyOkP952*j|#(qil)^{CMj%<6d{UGsWALqj3AvUB99%djpWXziP0BkNy
z%;`g=a@)g6YUQi0f%VK%->t@TcQf2vkh<%eM3h~Z9#`t~DgwK~iic6PL=Do`pKFn(
zBDN+#BQkp-SpVx?`_<T2s;Fv~JXB{i;!@?7oTM?M;<mb&=vCE~6SdWKE>BVMJt?C6
zge47Z7RY=MPV84iao44ttCj!UhOAw2;?+=}@${U!Z0KQCh6ELGUS9Khg=)`(^=guy
zg6bWr+Um16pHlGu3t9Ve$W2Xs*Ty(-Sl)W!r>wOxOJPc%u)4`V#9A4~Qa-iC_MxgD
ztNts^ob*e%S<yx1{qubA{6EM&FX9W-4$Kl$1I_=3vs$V2JW5uTlJrzR(>6<O#y2sI
zZEQ<aFQ!8Jj?aEfRZZgDj^dwv7a)DUyWP9M;zjxY)beVg)ZH47sNLz?t*X_ftRdPn
zUGeEoZm_)7*MnesF7I5m_jB*5`gFcjxYAGs4wGpfTh%yUn<;(JoS-4=#-{N#{24fY
z?(O*xra|t_`gc&dyEqseHoM~Is!GmYp!WOKCbjfq6I4W1`_v}vhm@I5oFQW*()rBl
z8E?CkZ(Qq#n{nWcq*_hCz4{S{nQGnwY-)_%$HD#o8&f>MX{)6z6Kp=nJ{X?R2k9eJ
z{4@od<E^ww?MiK$@*yq-)s%!nu%FhRs|L&6YKDw)RQ>l=@%zH2>O1|LnuS%C!jzcb
zs+Q}Qt1g`USC!+f0$A<s9y^slQv<NQFuU@)AmgMkK8QXgGGBFxmxIQF)l0zoKw=<#
zN^UZkAF6Dlc4BRh`XaL!m7VgBRp;BNsXskcp(eTQl`2zMnfgCQ#N6ZI0%%)}^{?vv
zwn=ciH5blOT&8<KdEer7YKLC!0h<f51BA<e%!KowHB14=m)fMs>IT>Nz-*8j=LR0Q
zn(a^5D{;SbP#3)WTxsj!mEdwk_=t`2nKh8H2ax(vJQ@O{Aut*OqaiRF0;3@?GDBcj
z&K#B0%}X@bcd}~uf8fxzT-66&gJi>&tu^!9eU*R*OcsCrb<|%RnyLxv|9?r>QL|WK
zs<`ZYrrIv|=PLi4{%RiCuwNUr{|^*`x5`^p6ge%y`~UtY%+_DYs9*+~|G&f2Wq$nZ
z3o~1(i7LnEKUU>AeZ<__RZ{)VtOxpadfOB$P4vtz{)jg}!)~Uq^g4@1^1dI+v!psy
z<}h|?3GVu?$tpELYahcgWhI?OX1BfbO&+Yw&{(@u!Cd~^Wy7b}9&4rN<e8gvU00IZ
z)1wk{-%q{tZ=$*H^os^klIrGjT8^qqAJ8ynjb_&tpTVOATL0%1!w+6l6wf?Y-v8YV
zlP}TxH1=1XRV&vmH~*<KUsW~diB@;8g8Ge)<tj^L8l{Bn(lzZfCYr5%FQ#>|r%P?3
z8iU$dt7~d+9!*#NxlTbzdqbD{DfvDP_Gkyq#jV;(Q$zFB9d!<AN=GrOy(?L%x2-T-
z^>!hjLDp?~i;J8x3Rx$(6s9FIE9obkF^FF}RYjY%&wR~OP31jq*{Uas8q}N9ChFfR
zT4pruDTmg_6KYz^mOe89&Hul7G7Y?rWXZy%YM<`sn-%HFSR7pJr}j*GF1U?%aQR#l
z>p705@+vH*f!f;@7v%6+sOFqgO?Y)prC_hLd0(`eT90PA+2VI13fYhSO%_ZK*KYDR
z*GfoWRG-SY&v5AuR+FN*Mvb0{`?dKw&Vk$24HIrF?&n-+R%a!vHK$>Zc|?Dwapan%
z+Q!kvT9f`SS3453L2dP05A*9gKgiF|o2xXjdx0`&{{Ob*boGGETa-&*Ofg$1v`C{e
zp<1hFyN|iM%{_4ce<N?Gg~+n~#;Y$KRi10O0^D{DQrv3F=KtO>QHx7!M$l>bw3jUM
z77nY`-yfP_s%ZUBGqByu?1kGJO_BEJD$9eJz-{M%po^+gMHj1OR6+L8EU=JPKVTrO
z7W2(p-B;?Z^30wB$%S5@)DE2PQ2m|6psd^~p?ogYNi`<qp59&_F15cMCrz)uJg$)X
zYokVPN}9&`ObhV-zkQPPRgEUPnnVctnel!SGu$e4Uh@ODrkdykYjqcapPH@{4NY#Z
z4pn1lX;Ry8^OB0WOP1<{D>K3SG)1Ck8(mz_Z5EJpRV`#qvw6~;m6}aTZmQ{KW(LzW
z+sz$y4k=!^Vxa<B{}<V=rBoQqt*rECzCI`qnMYsM5s93mslx4{b<fL8OGDq=d{>{m
zI;j6Y{aUaYgG`UoUUx6``&kwi5e|<O#exHj^_?z&_YRbF?o)nf5vD%d)(e~-ZdWO&
zJv~~ZzWh(B!r|sQ=BM)|)Rv#B0;hdY|Nrlt8QMz|R%lrXFHoD`lC8O$<APZ>le|{6
zWvlYc-K<9QmkXNrZ#k#ElUZCre%3B=(D?s6uYKS&n8|Mh?u&!=|AE|{e~C-ks(QV0
zxm|{Wyj8FE<R(3hn!8q7$I@0SZ@Vw5JK^p;HRESHl$;cDRhRD1G~FM0S$oHsNgC%`
zc$M64pHq9Xca2H+6;|a-`D`XPbuK7`O=VKs_FG5)r2BWR&*$V-X8Y@E^ez8s<|N%`
zu30r(d(WJ7mF2%?8tf|#Q!STd)}DF$rPhsgylPuaqO~Q8omJbGGHPxUf%JQy@<z#j
z+jw2Qki%cqs}!;a$nKA)ns&rf?ed+f>f5F*RN;!$P?wxMNo$t5kh#&szv{9VR5Z0-
z%bD-r&7fU-e4F;-#hhjx6PUF$zW!BzB6w0G=xUsL^0E8M?e#t8`cpm4eY=mTtl!mX
z;yB4hEjwYQcFA5_Rn;q+x;Fx!ni?9+SKZdV&wT#MATv<^|JDEVYSqfj`uCsx(O4CE
zO|?SXN9khbOMS~FrW*d~|BQdQoz*<GmrcWSrJHKSi3_G#yNZl-qvAD`R!maU7tvI=
zFu1G%+W+_dri+eQf3Eqj34L0}c(Sw>=*%^DoE)gm^d?wy<2*&v(l_E78Np4;Rrkcy
z7z5H&XLO5eO4vvlnf;P9tyVp)QM$B8r{mE>)Au~w8iK6H)udvrj5N1!m`z^&R$caO
zui2}>|LU<fSPZh}Dl2FNXKS2|2vS?%P--@DL9XibN*)X5R33|sYbF}#=X9H2e6+(X
zM|+aKOQ)U2oDOGghOQsx8s|OD&mVJEUTPDezGU_;li=$CDy?U9O&>pZS67)TZnnL2
zn*QVGLPntV{}N|J!R!Bo^Butbf96Sg>V>|t+J>u>l#Ts#!2AEQ)Q!}$)E}7(hxclF
zIh<8#f6uC#;9{zlY@Mp)nJBF(pnE{cfQwtVou%3wH2-gPVWnB@fi$x{J(HD`p0KNJ
zS6*YfI=@HlTRdbQp@h#!RVe+4hUMb9nh$zDfbEx&ic^~qf79%uMyv8&W<O=Y-Nu?n
z4~VFVSMZuG&f~WT-uqE~pUo!C$>vivHMhAK8FZdBmo+m~`*g(9TxutqX0nxl+Bz9=
z^*gcl>d)k?HQvvir^&j+6P#y)=2fW4h1IBbb!RK;G=5gSvu24|Q?H9!)vIE2*>9f8
zWv;EtGg2H4JJT+jtEQ`}o@RIg?%y+7?bOm;QmSdQ;-C`i89A+|fwt-<qDdw<*0>r!
zusf<HpZ{JfIzH1Zw`abFa;c`-j(MBS<!A2E3{P5W&h=6kJiZ9(|MT4Vuj;<GS@rPR
zHQ@5r$vFy~he7!l7hWg36FkO+E^j#50<YemKM`Z9AiZwS9w@fyLe}p*`rNIQ^?a40
z`W`N@87ULzfN79AWZd~$OyRc5Q}7%;vN(+GdiAS(Ur{WIUhcPZz-6C;@g%Ume>zfN
zx<PuDQW?WWuwQa5|A1+jIUu@LVH1iuATb!WWIv_&&H}P#5F{qK*+hYP^9ry%AU+6B
z<Gu-A=f?KsHkby9{eHU-Y)|u#WSnM$)M3NMzmI_RVv{3PZ1ah=c<uL@6|U5F>m=OW
z$mS;sZdH`uzX8RswplT7^His(VRh%?3(I8O**}289@#vn8&j1kSLUOb2a_*#d#U&)
z>ne&n8qSN!-<RZr+woEmGd(;}@lfzxkOp>BU+W^d9VaT3&z+ir)ee|lFgkVqWpLdI
z<3D_j7}pNzSqwMhf^v<**Jc5wJ;J$Qw`l1d1=BEnCp-}IJ}@yJB|)%X1lvU6Y8~us
z6p!~n_UFUY5li=VYrx%uO^@eBcDOz0^655HQS?UL2vuDE(+KQ7bhR*kN>wk49+)_Y
z){mHoQw}7LjEz(b!R}Jwa|XxJp{>DSKC+sfYD-YW3O;{DF+YsC0Y#p}WR>F9#q0`N
z9n+L%u2EGwy>>HP{qLl+c>M*lH|sv1QoxNUu-dQB85OTT7F1kkw+Oy|7o_g}QXz#1
z-N$gV0~fP^{Q#0%@gZFbMB~ES=lCcEzM8CLUd0X9gREBi8K>eYuJ1~q{r@oWa_bVX
z+TGbR!T!Cz)C0_i$s^MSGazfQk;Ty2&#sE#)Q_$f#Mk$k54IP?2H`Kh+lbOXJJAaq
zjvzf&XPdzJ-sjeL6th5bFwCVmMd{@nZ?JsUu@?&8)wRI(sPQ54A51Me?SCK)tPdof
z<fR0U8xWruY!ZT6XAq<3f+S}B@Ii|K>}GWRbyJh!?n@E8uju#sKS~*cuAUJ8e|{g_
zJY=<kHjuStATd{lCvY_z-CN;okQ@x3(_V-YR{D8sz<nr~JT{uyJqFx<0LeW%tqM2y
z@F@<wdQ*?Af~y7T-H?~3`l>e`Y}U+c7geUmoL4G1*ahZqP+y7FtXS5!N{w$^v4%0o
z%(H(aRa_>{RI%`T3%6^*v=d<Ujrx1RVv|(3z;@_4Xsht8e+`zKtdRm11DOHCN81y?
zZ7modnVxT)pwuPX052D|F&<E27L)~>1yh%z51D6CpPvLbJK%>EnC*Z10z7ZP)T7f6
zuZx4#o|kC`)65Y<Si=NeA2NSgNC?=^$YLP2h~;&#nIJYc?Cg3&u|$*~#Xn5H+`wkd
zt}avBv(pUBhUwii`yt#vFVl^bFYR&$^Vu($!p-=(;+b+#$1gY^rnm3ZM`h6YKQKNr
zt!mYv^w9#+uBx!+2HSaBG(si$%z3c;0~*iDo3MF+*?vk4(lEV=PZ$*BOm>6qYWp%z
zX;W~xYW3A6@U&j}S05|~Q;$psJz5J_2NGXw51DHOv9C)fg6%TD&8)a9JzJUAO;>64
z#u~6$ZzE9atMsQUFybpFY5_RkuW!@^>j9ZLOXisJ+C}wXb3uGU_))VNT<z9o#5xFM
zeW3OKApX3bXK;H!d>BrS?@(N|0<!NPCI;&Nm+iR?Hyb97OxyM=VfYCqzh;v<O1Qzq
zvC-iV6u{=d_Wz%lXQk}6Aq^Zh5h1SNcz&g2fZ`TY3COxRn0bPXf?zX1{O-+#iq~)5
zQQo|$Sng-eEahvvz9~($)RSEmYX>%~k=+<<R=OPOS`v_6WL$V;rZTMlFWV)gTy(Ki
z3BCT;+v%lZapEsHeS`G=OXz{S!`osz*glXrHq0GdtFp>#w(@^|JFt7u)jZHzr~E%<
zmI`S8A0}tJ#2>6yN>~j{AF!E@HBLe5pW36Iy#f+L#~L4}D^;xU!0S$AJ;BQx!S=$$
zp6E`6`xhpLjXrVh2wdOc3lqTXxm`EGG&c3OS>CJo%VmK52I~J$dt?o^C*ip?n8v0D
zCgyZp5AL6gLI#5BLHgFrT?9^VAa+LGWU##;HnI3qfF?%$@Zie^tadXqx53373`3QF
z|FFav4#e2Mwe}5G{mV2i!o|~D{)5@p4E>7Y7e8b5pMY;D++84h|JQn||LfAwEK_UM
zP*y6^-Wewa-k<uqYOgxiuRF@E9Cj9`C(KpfX59)N|F@G|t>!tyL+$S+TlIB&xKxEF
z@oL=;-3lHz0hxEJyj7t#OBp-{>deV#5aoWubaC5M6}Jft7CV(A&CLv3)J|<&uJ&+c
zvqf|2Bek;;Yz7BPE~%LP3pLXZIc+X&b4Pym;c{go)eME#83D@geWz(WG|o|f{db#s
zZFQ{5O7Rx+dMP{8ri>1i!=B&F&!$%!-B@r<>xq`QMN4;}GK06P!v5WV)Ey7<S*Y!2
zFk0~JkU8_Y7>(55-;Khu%{8YnFIV^8`#`aA(HZa<lTVSDLPNEw+3Z$P&1GA3)bDLz
zwCKxWSK40Hp!G!ii2A+g{VLNMBb3d|rmL=qJz@5)s7^h;K}`9sU9@^zN{o8pCdio;
z-(0vgRQa1U%kFZjU0y4qEd2AKrn^7C`jwPrYUV3G>v1_pDl;BeG<dW1uEp>9obnoe
zXXU3QGAr|#DHu9wG;8#qSG4$7)~j+sPe$pK(>ygtp(gzq6Q3IM)`)7Ub7g5crRbX8
zP*qcIejBHG_ARS&(sdE_yE0PdfyO&5zL+gl+Z(+D9QQR_T1>7wC7Eg@&Nh7_Z=|xS
z)YwwH?3Y@A;wQC-lJCty=l|KQn+x6($~K3^M7h*RTkTVq`u*Aj^@y7xMxl#6Ob+p^
z(?|}s*D=ru0*?`Oo%L1KXa8cxlrvl7#-Ab!pS$cP`%`3f9J_SYi~5t*ZD(v(o9mTt
z(U;M#c%M@gtS9WLu<8RoFAdqf-l{Vnd^0t_U#r2U(5vwwMasPPu!RO_|G&O$zr~|m
z4%7cGM^(M%NUM0Q?^EmxTV?j7-PK6+uz^<6#C#QvmZ|b5no2Yd6?vPVxoV=>BqV5V
z_aa><+IgdLOcS4a99xL;1BDx^tw&VVuWS#pD4#u9>B7-@>Xi>%)Oz~3LHqv|1g2)I
z9a-N4cGnS|AVt>J*~*ERrz^f<;L*2FTC3DGTg$v_!$hUm_a~`^+~!er`@a`_4&SVO
zt;($79A@Wzwwj5k{WmDN+N05&8>E)nzFB?Vl4SLX`phODV=t;@NH@qQFKktq73QdB
z=N@D>tG!Qr+O1NPq?ymnf?rpvi#l#L*V&k?o~7ci=6%l7;MHD5i;T7Jm7TqpDD(U)
zm9u3&qjF(GxzcG?Hn6?(iYj#;eEp%ux=ut(US+-duC$lt@639Y7f;%xT=bIBT-DN1
zJ<-)hy*Dw`VyEgSWhMt3lMP8bC9QWqR*iFBt2TcVyT<7cS7mrKf|U$k_sNyINovH3
z392Rdi(52a-l(?dPboOR8t_Z2{$JazWj*7C)|}79>gQkOYc81?WiA#!Tm5_Fb7jTN
ztwv5kXUwO`o!54JvQi=OgrKC8i;L>nwI-S}?V57)LLuv}Vf+6-Tv!P&A5FzrHNQor
zsQ+K7tKL{(p!!kky^c~gxB8)5GAfUkaVqv^IhZ{upQK$IA*6YKKct<j>f>$tcE%5t
z`%P7*&mV~@9`!X>3uY454~w3vwP)U9<$};F8ne%QG39%sX|d_)A#GuWM78_>oAtAw
z^r)@hFj0%^gP&HCiKg1yza={9JKw8vN1Lei?}e;)%k$EaxtkEKy7$al)z6C0!FgrT
zLN(=S^O|&i^>0_7`(8^qS#!R+?88$UPq)gO-_Bm4F^@%FQ(Q;E+(9!^bBEh*t(h0u
zE$#&=sT<XpXgt2`qxS1jl!o^-X5|&H>MYdEkC~fRN2{oayf%H1x=2lP-FEH655pBX
zwK#Q4X3jA2j{c?Wce~j_`41m>{QqlBxa#x2-}Gd|UDOkQuT`7Lv0L@=Qx*d$8AUaj
z5C$`&L(|l)4Hs&Zbt<Y&to&qF+c(YVf~v5Z-$DuX_+myikwRnD=x^(k8!O&w^T}pg
zd<zrSl&y8pikP<6{NUc_>X8@2)GMc_n<xC3rm=Igq}mo8eKiH;yQ*)D<JA2ELyh)I
z-7{1CKUK}@X}ZpfI7aizslqDv{wb+D@7imuermtjr=n(!-4}M5d)_~;e%p5eI9;CF
zzf8S^eW993<~{STF+Iv#ns!?3Pg-m7sA#o8olTxu>Ypv<rfOI9Jzdi^*mm5~*j=A)
z(HeNfocC0wYO+4R#-rjt;Pd}ZdFPu=EMBQT|GSj=>Y|nUHzJfk=l>~gXZix3Z>&wc
z2QHhpYD`yV<~aeLtEyyM13t%OQRRGfr+^L%UFoIj7yC_A<j=LM`ei`&$COU^t*+&+
zq##hXM4u@m%{*&TvPxdJnfc~}edg0dZ>p?kf2j7#GtRUtbCKG&6Mf+Ffa|@G>Y}b)
zs$INsTD=+(>J~yzRqR%@s;TOWSUfM(Qdx47S^1-(f@+21J=Gg~o|!&Voo<nTMO-6y
zTBYVsi7++u`{72s-mlHSo?Wh%=CRGZsEk>Cp;wmLC*d9HdAy(07&Er06$ek%JU?jz
z*uJ0>)yfY!+te&s%w<2MiK=ntPBFK8m8mR#X_k5CM`IPx`oF45JtNf#t1UKU&QUI1
z?WHnh-VQn8WDCtt&Fj@vZ+5EL^UY8TbULmebA!XwwQ!2DpQVJxp(o3<WVA}n?(FW=
zSg=^jJoB!l#qF>A)YYm_SX9MrQ5Gp|Py>zs?^bbE(~D(O3B11(-VO*l#I0%U=Af>{
z_gFJi_qHLZ9_JM_(VCNUN@ZjH6SJ}nd1}=++B87(|BHn=RYO`|$UQZI><3%U#G-ks
zqYOU(A2ugT#qryG<4sP>l@^7y>Fv3@#YAyiv@%D>I<wx#{NQ@0p|C+UuUf_Y&7@wH
zn^%wMf$WnF?KKs%Z#Dy+|Km9GnfjdFtCa=iBvmqxJ8MN<IH}njXsk6SG)jfHLeT6^
z1*ZwOwY$c(gB51>4j&AD-pkZ<{(sJF%ELA#i@PQ&CTF*)>FRwqtIF7+FX&!r78)$C
z?lQB>&}ZoZtzYf8K<EF+t3BKZo{!PKpCNz4Ow(A?;JUir_FJmAK6IE>)Z42Tn|Nwz
zFzr>#xYMn&hdV-fdWyB?>mQ3v6ByrWo_Z~-b}Ms>vX``)TH=>(a9L7pcR}r5bh|oZ
ziVV1Hl`<|+D=nF-xvKbr%8A6Oy4R(-RAZJt(DSP5HjnwKAn*6;H0b<a`I+*H`dzK8
z>h9|*!C|u}m0Pvwte?8#0du|1{kx4CPcPKaj^fg)xT<ZkLg|a*mOH#)ce51psxbub
zH2vZ<&Ah6!LN)N*WUxCHZhT^-^7o{1(q0o2owilVmL-Yi8jJbC<uYISDzo*PKU6KI
zC7aH6UMiom`JvI4r52id?r~`a{FGOl^&;0$(9P3WO3GFJ*P^vr&l|YaK<EF0;wMMq
zuZh#!1)8x^uI4k=)*DT$V9=hOVXc+*SVOIpAzPK9W-{3QAhn?Rf9AlO%5HIxb^7}b
zUp6^&`hfa(H8HIf?%8H%WIDllA@ov&x%|KN#z8vlDuv4+=TSCY$S~2pmtnZ|<2rB|
z;P#D0F6fM$x^tGcX=iDt#twspracF@X(coZsjTibQ3TchB?5C*%9G!#=D9PPgX|Yp
zvV)W(u^-hgXY_-^<&=~wc>bU5?0tpg^YfLxo_|%j$8=S<ZS_Re(DzSFo1T7E2(<Ij
zcphe@5h|Uke`ZyhV&UgDRhiY_Of=ruo90xq8%*#k)eM@~qWXR3M73ug2Q<aEZZneF
zu~hZyH#xPpGBZ`AjcS!%ZWRExeb24DYmmBPifQ9TZ8cV_0<*r!O`5GfGgT%%Qr2g#
z-U|+^?T5A~J&EH|+>olSd|^Ym;%4I=;JgrY%tJ>>qFA$5vPHu*hD}RGAi~UEpHm%F
z|JQbug3TAqFI4OAIdA4^eNj<4=qotSg8UA`^`WQKPJe*Rp~M9WsjZ#x03043XC|Aq
zeQi}edQA#krh?QOh}dc`iv6ac_}NsAqcvCK#*gc!#g|@bzC7rxbU-cLurzprndc*2
zQ2j3tn*Rr#|8qw@4V(r+`a$@Gc{{wX0h|B-wcc5~^J|Ct5jS?NzR)OT#_b;3`O_i$
zf*&vARNYggVJe%RskOKUvPbm%`J-xb6^tgKzeMCJ>v>IDtXt*oTQ{iu&STYUx!R%S
zllVvZyIGI=H2p-A$2V7)%`$h=(mZ}g<-!y$y{|5pl(TkpXzkWk(3*JbuF9j2F<M+Z
z(^NI4@7H*wHwm2fAG~0emy(;TzH(-@O6F(Cp1EV5*Hjh7j%#iF)~z=8y_V9$&yH&D
z2hFuo{$Dn2`c|f%+xJ;LCu6JW>Dm=qi~q!EH=D+q&3tuHQ(JbCn!)o48UYixs7cmb
zR8Dd`Z|1?=W47$pCsom&XC~`-6I5m8MYX=({iHHU&`oE8MT&8;eW~iDB?0D(iZ7dh
z=Ko(*&s0-Y=G5E9G)2SuViwqq&-m8p-F0Bmc#t7#GEHc)W?rnX`VvbU)rB)Ro6HVl
zGTiZzO+(W2o~pixrt-uCn>4OlpO$aoUZ!2ONzy!oO<T)fZk?udU8~u%+NEkc^7S-j
zf6XxEzYN(sqFlhJ8ek)&Dv<e6;}e^qq5R90rY^?wHN;Oc>3n$AVcO7lP+h6_m8zjd
zr=jXb9aArTd-bo|&zkzz|5nqQ>7x&t|2Kd0Mm^EUP%UC2yIE_fpX&b_4Rb-)Uh|@`
zL;AXEPA2>Q*_lNw?$=9XTCKMH_fM_OPo&M4n>Co7|F=VlEA^6EasM)tMMtDnQeVfJ
z=7<)lowz4!_GG1>-em(`@R@VZ^*4g^eE!ab;PT;HJcoMUHhHZ8#|-7)W(8&!iW}t>
z3NqEMR`HmxS>>a7z2m4#afiFg3h(8r6B(W=u2H$C!I9alq)@U$C$>e^?9pL8<(xI=
zOt!=~n&yZ1C?65GRh^r@$t3>oG*!ksZs0s|Z`MZTOFV@dY^-zDUnTAYm)(=n%~gN@
zIbf!GV87CtkSR*Le~W7h%ss2JH&fbF?iP>vq@;6dm+wB(DBql+sa3tmAi;BynaiXy
zRW2bJv%|fYG)_)pR9lf&re>9NR&8;>SM`lnjv6*)d|FAbnH2YPv8he@;-XqEr=qCQ
z_)bN;yTWYMWM(yf?;dcO0Gj{zRM=v$?5&D<I@fNMuD6}w`ag&-PwRuJHQ0R~qO4k;
z$GFt<6y1#<Mamf+x>u*Q$w^PEZ(_GeWmTv8m#0dm>v#Mz;|QCj`TC)&d2XYwa`D3+
zB~bl;CF7=wUM!oc`j=pE7-xKVr~cNBTPs>{uLh6&L(RV_Y2f~UJl`bEzjHUKcs#gm
zo)IRi!L~|73snC<_;W+;9>ZO=X-xg<t3PR~@+5d_z2V=c4LbktR|z-h{69r5^IayI
zcdb=7FI}L&|81h_#=L%&nVd7tcO}0!Tgoe{{yj26-6&qv;v1Kby0F@D{eJoRs<KDI
z%;GbH%t7`469q2t`G27Pf7rD3+H9&%G`?j|*Xq%cP;b#<F~2Q4%cOGtCN&O~+2&fo
z7Y$cx$Y{;%IBtGXTu<cy_c_J(Z*>}>*{94;-K#TvW|L)pef~X-*o(PF%M15vf7gAd
zQMWf&{_5fbTA=g)X6jVS8<)&AH7-!qI%Suw{;aswe34X-@@pdo?G3J)8p~uasiovb
zDnGq(MLpn7vYGr{PR&=`FO}{0K2^6io~>Ro_o3>~bqZ=9e|u|)e3_@&|L3CmGzL$l
zTdi!G=aShp?IyFS-+Lpi-_tuy`NFOo1C#1@i}&`1%D&&u%2ylMDypoVX|Tg-n#S!*
zE6q)QRI64*Co1c7>#IxOO4dKA-(p;pyHE4|1R<^JRhvyXUG-J$FRjupkjPW+d19dc
z$oZfdzg?6?O~_ld%adk;>$#7|TunGu3YlK~t76(?X{#2Tu+5^y@xHpIbdEZQ)+}?-
z`G3w;;bx%o|Fq^WFxI_pp|v9Rgr;>$pSq<itC2@Hn~7psi~9UYo3(sYJJl7}r7AaF
zGE&=p@4Q)CAcs~`6PJZMdyg?kj*ITg39B^O|EOv3B=D-=Fxv)N|EKuFWxM*XxeJs*
z{r`l}D;naRX{zdG$4x)(f3B`^Q&KCd__LYp&a)a#XVxg*JSl2Xbo8jnIk87-Hp|zj
zRxH#}%2_tqOzoqjk=ca`jhS1-RU3Y?$gh$7t#P?vj+x!F{TeqJmY5lf34{CpYA=?8
z`~Q<ZYpLEaFHk?}bjv)5ElK6bq)+MxC044{^Z!?yKA}m`E9{%vM3<xLxi_Y&#folM
zY~f;3{nHz!w71((|L%(Y%1H;Wnmye-P02Z}LgUkmGBwYe8U|-pr73?*^;WCOooAZs
zoNFd+GtEFNBVTLpatSr2Jsuid`8PBc^zAVAYrC#?yG%ii$#$p8j+h;48fu?SLFfPP
z+qciyO|!>LO-f87)+ogM<KxpBOKfdbO0|3qCfuHA?%uDfsu4X$xj4*5k@Kae8mRx@
zEByq#{_oypZ{7KJ{TfY2{WX_{Mro=ZDL1cvpaib}zv^u_(~CW*ethN?waYzx7LBZH
zm7J=JOcwOIC@xT)r`G4_q|O!As3D-D4m$r&NikSdU0-UIhQH=DwVZWF%uV)6s4G0+
z1fTymtL3(;*mE82A2r@uC98I;2SqY!W;n2#cW(KqwMY7|67R|*hH+mK%vs}FK<ocN
z=l@CkS|FlUe&eOq?2^}VT+bG%eahjLhV}o?e-cw;R6nNtv0$t62{8km6YXl6^DWc0
ztf%!TTdnuj@#@l1_e*@Ea{l%#B@2<qrV*S>+G0XMnr{uKsMw2|s#l)8WRj!9ujcgV
ziV3LypD^pAn$OB2eG`Xp?fiWWO859}G<(D~O@BrnFjr%6)^3|>r7rp5klu+m57mB!
zw(1<;kfkLsn?-GDT9wZ48aefQj%Dh1g$2|Xx)sW9zI;&5E;vg=_%(-GcR>KSzLnSJ
zRa=reNjrN5iw3CvKeXYO`m}@&?TqBDW+yk=Xf9&y)XbVW+05*(l@3ejES+<yUFOqg
zf6&N(m91fKv`K@##$98r{|V(=d#;(Eam_G4Uf8StQgfrJ<hLVgi@FSSK>h!X3y<si
zKDRW9mxxhm`>0{@|KvPVQ2k$@@lSQyG$Z{h%Vf3IRm@fY`o&x==L5gN>S9xMH{MoL
z`-S3~MhvW)JNM5}v*-*k<L+WHx_In_y71Qs^;u^7R2ulctKHyIQ0eAbrNj4&*`nsq
zQ!SpgZ?#h7z0G}>8)`gCcF>H=nPg_d^I2n3@)osIt=Cofm$a#UI66(EB|^+d-mcS3
zCuFaN<8NbK!R`XH1?vPg9ymv;yUy5Pd_^nEEadnFjih7h<}n=4)F<63HK@*?sIa8(
zj)rHph5ErgUgk$1ELS<JBxa$Tmu<m%vC<%73ZL1R``+d&3RCpGISMsAH+|9C7xBSd
zxMG?)4`YYgt!Gm-<Rel{LFfNHiq|xYi8`tNZ_^{QlZLJOU9DG*rr%hkID6?DaQ&9M
zI#4r0YK_wCqn4^X*PXRqiAbw1{u5;mI{$C~tmEneGAb5fOZI8LI`md$;`LV588&L_
zt}`C1a&5BI+>)8AeDVH0y@?uc&CVQOsd97XS+ig9|IGGl8>z^z^jF`gt!|pL>Au<*
z6-kYsw<VSSSngE)=OLxxB_gePYo3fI`}Z2vTTkRv{hzG_o&T>a>Rzd=eQ38j-(5kq
z<=1OX4OxUOQmiC2g!r~<imf!%u+vy(sGB<5T+wBfn)&BO^Lz(k&2}j{^=;Au8kf#b
zQa`u<w)!ohEX{&kH?7qtS(V?`_ozlMny&WtM6ldPzLlzG%;BK(|CKMjPB91F(_-+N
zNhR&ZOvCyHRSS#5=afz}JW&Dl{}*iQ)y_&4)7;0-s=nV+Q*&*Ko!XOiNhXQ;j>bn^
zHfx>YF3?ubb~m%_d!jDNyvFPl-(&NON&K2mGyE;;Kbxq$E>l(q)&E7)Z>#mMzNr$u
z&l%kQ0kz)+a#m}`2*{|tXrHNl{?ljhSWiXAT21X86I9Kz=9p{hTdUNZHw4}Pqh!dS
zscNypRO!yL>EJwcSL~FQ@DCMu|G)0_3gwr!3rs-f9x664NXTI}z2S3Mu}|C3{C&eB
za2x07?paDlg65d#yUMHGVHO43w|uX$X>61VsQy>fe(+vxfleB@|8FmSUUOQwtmcd*
zPc)169#Bm^w8iXKYo3XlK(G1?K3DUqe0`%HSu^cta=PZpEWed>?}@4;-94?6cu&Rr
ziO^}oynFY}uImM<=Irb@`hB8E>rk7GrpBI~irc&=g8MKj|If)Phv=GCa9L}v=B`)S
z^FhJfIF(m**`(K6*2dQA0vd~zFEmCd9}nu+(Eg!f=JCTo^ES^{)qq|d6-&0os)rr7
z%7fPb@7SBFs{gQ1%}%}u+=i_b%2yYf#;94JvQ(8>Gem#2#Q~MOuFnlN_v>1`I=5cI
z@6~Df#A{cSvOWGA1WYJVjXAmi+;_d&%&Ka9d4<~bY9WJMb~$5y?p<0(wtv;~m5Bw1
zX^~?t*xkaaZ&U-^)68DRGgv&CtgABl@C9)FA0=gGENR4R62pGor0GzQN;cCPix<h?
z!Ex2z$8Y{~^(<9YwNKza<j(09#vE^SwH5ZKYqI^mtny28kKv4W^NlA*IclyAv(owT
zc`kg6s8vPU?8N&mnkOz_HLqJ9YJA$bS9`nkGEHw=8^xotEULNBPlMeLiuZ}#3zXJr
zJ1Si|{sXLkMa&@+gJMC=RV6nyQ>A~J%81_sr@iMl?wQ}6S!q()wMaQ<n<ThB+NnIv
zG|+diA<w-Ous>7GH55u5)m8n9Pnn5s3DI)eDQ}j_-><bZ+g+7m-6~~J|35awSY`4<
z8CBKUkzhAW?A;Ew_wSDnYVB{W!Qqmm7Ne?r!CE!x7K?JpB}P?~6Z2FK$~WpCoK~)y
z>!)j`#&uB1Ur9@YNugJL=3B^kr#t%x)efm6#`dbaO~V!(HK;Z3)ZD=RN41wbL)A`B
zLNnUg%~)3IlFES;Hnlm&{K0n0c;th_*6oYEkw&_S>1`b|l?Lx7vp0JlXh<j*s&NOM
zGDr=q0lSm&(mj<Jy#opyJNuLrmaJ2Ln=T8^3qC=cb+$e|tD)`CqxJVhi>Bk-H)cAU
zGS#+i-J<OBXCgQqBy?_7_tEP&_o?SrwYdHb+<yXvJqYKEv8lrP|C99(sYvXCoOARe
zo7ue9bFK1?{_o(h1*!cf?XUexc)8Y#{bH(dR(4ub{63j##fF0V|0>yhR}598ILw8Y
zoYVep`y4#~4Z8nl`pOWvevlc+{eRf~KMHPFwEq5<)hHIM(KK7YuV!l8ty6VM8JrH!
z-=3v1WnHJK&4%eZ*E6!gW&XdgO{(U)2TV#N_bZF+Vm58DZdEiEE>qRIyIOy$$qlXY
z2X|HE9hua2<=LCY`Bs}Nh|bliUzM-gn-Ho0Z{c*6_lEA;pOklKC5kAiykFj+rPQ=i
z^@6#PhL{iJ?hw{lZE|fBJ=9s2-dEaq3UU_1ii_Q<_0Mf|c(#eEWp=kJ>GMxkGmK=^
zn(J3!#w@O=8uja;=B)E2W=qyLYgJ~?&|ax~-puB>oMz1RI~x6VFVvfAbW}IaovGq=
zb*VXT8=JXN%L6shO%*10vwc*5ws-0zu2`Ycza~}J`NkFFu%=5Y#+s+ir!x6~?*CCz
zix*MrGHub1pWdS(;KmQ0%l21qGf0zaSGUhpHw`hnpz-x%h8l<dJk{+FW|;j_w=nVz
z`K4afxIsl<L{oLA7K=v0;uOV|t&?^56(^Zj1u$t{`g>ICjWLt?l>LiT<BJ<LbXRzr
zDT+hR!VjEwPvw67NtNj7RvKxW+6;I7FEZg5V%9jVFkNR}Pm7uGqD*yr?_`xJiH(Mv
zB+i(w@?lZyWU?~<d8SdLb?tmm{V$(2(N3dSVX4|8O-b|PR*kCOZu%CiUrv~B;A%8D
zw>il4FROt0qsdMB>h%{i4lUZF_4ej6bEn`FW}yB57bY61F6h%YovD3Dxr=d~sXgBe
zRo>5bW?B<u44!c1g41tf@dj|&+0b`TmG!rT;&&AfwfOD3v^5Gclox6o2iG0u$HG)6
zY`kV(ac;GS`uyE04clE+bmmB@{;XN4$o|4fL*%oS64Mbo-QMgtvo{g56<4;LHI@CN
zWWK(gMJ02+k*deOGbW$%%2XTY7=g=$#k=*D&&KI%ELB{pb+C*doEF7bu2RigD`pNl
z|JO})o$^8RAdQPE*HynBGB>^Zn9Jg5(MyeqY(iQk>iaaxmcKN7eIwS~$1O!A@m#an
z@sLQ3=*A8;rH#C5?_KAq&5b{xE@pC7v-DiEX3}dWCBZAlRfX<vRMGbRu2Ay2Ri$Ar
zqq)4{eATxTZh-w)yn;zZ)4#y5PMF!^l~B7%>1r=EQ2lTF+g$70<jLUh`M2FpbEdSL
zddp=!lW*nEjeeez)%vXEul;4wVUzhzv6`Dx7MsnVRc>C_^-zQDPN>C`--lI-ANDAN
z&i_x1+oGx$%cd$l(*s=Ig7Pm2M?XCX<}dtP2)5&4`5*P~k6Ki0N)<Jlz3-}(W}H@I
z4E(Ib)$jtGhrQx9g3EA_-k0HPHIVlIDNGhyrC~Q;S%qn_xCYzPP>mZthm}jWozvLx
zav#|2;`KjN=WKxFd-a3{aF|>AZq%q)0*U{1R*16&&7UP|WGwouF0x)jUF=7kn*Ra`
zjd^tsH5iNDt7OP^s+-lcX*}jXr`Gi(P$OT-MTITeT47a~uzHi`LydqJJ?h*lC7RQu
zrl?z;*sNYrv0j5G>J`|Z^^P-@5^9bqru;uAFLCgQ`aw=Ujjz#RYJSptHF)?Scft4>
zuL7$D`SDi>yIRJ+YwG)E^nmC8L1Il!2i3MN`wuSnyO$WLpNwbJIJWN@IDGzdn<#R<
zQ&L}Z<GZTA?hXy<FGY$x^_Mh^Jv71o1?dOj%M&1H9>=hFfW<$qaaYPqQ&tvV(xc(7
zzh3jrUJ=Dc%XjLpn^V<6<Nq<{SHbqJ$Z1il<bJFX>+(kZ-9{C7T!iZ`(_C~ZRKv1O
zP$MSws%o@~x`yN1O<;d~s#pjP{|z@cY8K@!R@yZCjYe0Ks+vi|0`*X@9<@jH^EI5_
zMS$G_a$C9aRQUM+#Vv|py(+hLHMGNyYpBmm&{)Xjs~&IBp;mh&Ld_*%HC&(MbYDf#
z`G0Tk#ewZzVfO&67i8vL$Nd^l)gb57oc$xFsWa`S#;k`Y)m~{s>Zo*kZgt~Hd%^lZ
z>OpwUn<n+`HI`uWCr(JzWNTtXk&l-ZR@MAer#9jBUQHo>H5Flh$XS0rm)XJd{~$A5
z-a^hf0<k?3990r8Pgk5FZmbEK|9^K%T;uI2$o_}}QOi}<wTWwj#{XGMnAD!ATvl&5
ztDza<uBCY+B3OMUPnnwhS5bAYmFLxtPCTtK!;?$RgMGOQ*GmB<(D;9H<9@JTC$o5{
z9iFVJnQLpOUil|NbLC>lnS2|!+*FUra{;RXxdnvZPT>Kk8Qr6+z~XlbOf(j5{h)Rx
z(OF$--yQW8G6L$k8~&^Cc$X+fS*U^4fb?;29|4EMV}tc@F_5@m59BN+5c`E@kE%tP
zI#^x9jNO_wo~Kk-UGjpfS4iRmvx62a0;i#evotkj7eMx6GCh5-!Ri~Qc~ap&SPo>~
zcZ(Wu`UA0H7}oy>@jor9RWw`7r1bBZ5jehH?RgKj8zko<Sqe92S-6j;&5{VXT%GZL
zRpps3np0kgtNq_x3wD$0bQbj&yVii&lfErgF0f?LxW)EGBlEbkhTtz-b=Eik)MUL%
zG*_!%RgQW7RO48_oQ6;4N>$a8J?i^sH>$B$imA=#k%!v>asyM|eU($6&VkE*5T7k^
zp2~e6W3U?UE9&a%8!I&m-Z!iBbMFF+36>UWwo2~SSgahYX)kZ0vGD+p`hqR9RMkC9
z)Xd6us_AGiP|lQ5QUB6isjj9GqduQ+fyQiQNPYzM|KI(#(LBI#Q}YR%ms0h*Tur5&
zU(|okR8uy)5T#sq+gju1+aGG@r`Cbv-#A2BQ~$sswcE=Z)WsO@sJ{!ip}v08dbOUz
zGMX#*wQ58)SgW@lHUWp>s~*T1u!;dv8o^JmYHTW94R-H?6Yd%&@;d5s=FC)I&&IA<
zdEqQrT=V-BRd0iKMXfn8>Ls;J8k2KaHO#JascRG;*I2@7t66f&OznbExLT>Br@As*
zq^5e(D)rt^?clI~U~^EjI)5T~J%7-d>1xlbr8L!CmMDjxouRp=<hHt2^g#_KjTB9p
z@QdoJe;0uDgwK`*)6?p+G)kxEs%d@mP~ZA34V)&NUa)FD6#t_3@a;{_hUT9dd5d;}
z)q>&!g!eAx)&QOV@5BEPJg;>AG>eK=LaX{2j$I074d>N9+}@x*ar1Ze48bzxPyTkQ
zy!DVXxHJw5YVdTQRaD=fqH#zpTm4PMY-LB~m+Gw(7K6=q329c_a??vgK+r*LxisYN
zkXVir%8T}CgTts%#t2ORd!?t=dI@rtAjmz>tly|ED9uzWcz;pjc$pa34<PZ`QAKKg
z>(^+`O!};m!17IX21B7nlx4j7#TE(m<R8N7S5;lrcBLe%AN`=DeyU1SW4#At&7t3g
zeX5}O|7Pyvs()o5^?}^Vy<j)0l(wqyOi0vND$=8o$b#6L*M3i2<GU|pZ50Q5vHGiu
zB-Ow5@|s0$Y?`f7!eG5Tw*tU)p%9CDmm{~vt3S2s-~G8Yzs9Potg4k$$rEE%S^4uR
zIQ>X|w*aR(kUv&$+^%`imRpU@xLey%DHlBczxptX=7(<ys)@T!Se&_8q`F+hObb;1
zfBv{aHAh8QiQh#BtbgOJE-ja5%<%p{clQOQs}~o6`#>HhAqF8$wq~II|Fzj^787Rw
z0rz`2t*<FH?muo3aHm4eX{r^t&(rd7t696TlLct}|KGVIsy*H}lq^;<sMv4q(ad&s
z)e!n~P;=6+e=3|>H_WRyHJR>_oT_%YFy4Yy<)~4-NT;^ogE)&xkNA|!V-G6JPWY`d
zrKZG!p{dNM&_~lEwQ-5+%o!=h69NRa_RX0F8vj?|KfeOphXM8fKQCWw_I10OCexzB
zD%byqTgXJtQaPRbQcH_tu6pA4ROR3&k;)IPma6~HF*o0RHcMmsx>D8U;m1^i{s*hx
z_-Cf{bDe^6Sio<UCChs?;%{9Br=e_Cb4|Ar1q}~}pQ`>2A^KOCtW-WJ#u={Kqh~21
zy-MNoc0a}a9C6C8_-YJI4jflc-M`cV6c_(vCabJ8`K7wTfX6`oq^wEU{a&r;4~*K~
zulm7ZI<24z?Cxnh8PtAtb(tSq>S!s|bx)-=7_$EF!uNe9R=ulDmri6g^V!y-Y%*ho
zWq9WdaGaZ-3b636;Z%*-C=8B=o&#Mb1@EV5Ys{Riv2f=+l?{)U8$DybX0mW=pyomq
zcb(g3KElg-q0p)3Td%&>4EX-i!s&OR$&K|EI#*Xd)X2H_Na=IRBvt0@4Pf_!`~a>0
zl^$MQ4c7lEHOMT-lwUJff?dn8y~KQ<Gi2|rzmcG2rh2LAk0;KmHzv*oyU|ZV!(4yW
z79+?1*<gRNy=GBpl(?jF_unk@cK%+iUu|{f@&^rccCBYpWlDah1nU3yzqC^Mqq#w)
z$GQfz{!ek;LnUz9tXO#veE%=VP5&C#sI0ydt(<tKRC(pS5b*uK;q&|SV`8~fIp!vs
zSJcTWPU#NQu%6nb_N44FI8FLFn5jM3Rc-2;$zvWPR$$1pf1&0#R(`eK!+TXfYx!%u
zi*7MFAI_t!x3@|4^Ywk;I)!cBZ1DO&9WyzT4zGT*xn9nySJap-Dy;Q1Ub;4^^<|$j
zaJ@Sd9998pEGnS+e;wC2C7Y&$O7Gd#!S-73zOEy6%0eUQ;Ww?>%6yu!-m(^oYwA@~
zO1CRdpU?vCCkRYvQ=dKKrbYUd7Ue@6=fGnEAb)~z{?$^I;3RhN+Wf1Wr<89uL)NP+
zA1$^pPOw&<Z1@)(wji~eBAm6OAI52aNRCx5khIb)Q0_BdRdP?`3gbHEJ8Z{|mg@Ui
ze68Tq0qy^fc^e42|5vGeRv}zJ$c#1am%-&Uh;5Jp*%RDql&6)`q^_aARYddh4<=Pj
z?)y4yd^xHhHK6{#J<lYwm-+X#=Y8!4m#24wXQ&9J9x=UZ$Ez~aW*xZyzf#>x%|m3i
zeoy2#Ej#IjDp_w>)Y1$3&3x+STFm)&Ogp$gTlIoGvq7$Bib|@rzV@NbT-w_{?@}>j
znWkfrlC8RI*%bA@GYi4#YV+lTa?7)7)Klj^Qd%~l8!YxI_mFCW>vWx5hR>=??lY+r
zmEKdc{<%*x<6?_>>-I@1uG{u#uH{*2-jr#jbuTnPTXh?|#meb98m246G`F-%sk6<|
zQBS`#TZM<?v_;)tJqx9o?P?vB*G>1l_EcTxZ3e#ow>4#z?z|QQ(|RXC6^AM=%h%JY
z%s}=3!S7+J?Uu9kTi5T^c=hinIPIheRv6U1ex%mOE@CF&pro;TuAbU@t-q=&a&6`g
z;*7>e-b_+&kXBH=d*4pol~GPTVR4G`h8Jx*Tdg-*Y@Q{jrJAx%OF`VhqQ=chm3h+%
z^*Kd>=4rnmXW6CH7OH^m|1G>zuYUc{T%!%kvds=yg{aTsxvuj+Hr8BD_^I0N5;oO7
znRmw8@43vkwPdND(vz`RdOBNUn!_1T|6eY`t5W09H#_wwOY|*_+NY?x>{7LS{FcYE
zCqvUv)=|k^{1LxJuiF!S>7{2h;_TOI&D?$3!dFt?B5lcPl^6$p)&EC?&34StQLfvw
z#cT;@o~nj!uX#eTvBBF<j^H}yYV;Ox{bl037To{e=%A{4`RFa}xu)LAF6#W?JR=kt
zth(0eiN%GVf7LggXH~uSBuS<87_VxW3YXGTo%8Cfb6ga48Gq_c2-t59>i>T~o@0J_
zlCZ_Tm5Y=g$GEF5eJ5s?ux_@hnV<(a&GIc>q9k~DoqEHXOIkcPt-$t!#u(n*wy=0!
zsHF^A|F=6aRh@ahpW3A0GP7IPxGX=owreOIf2V05`b)hf^OTXuhuIbd3mQ~Doe{IR
zu+mAxHvWLx`b<u>RPzuur~HZPcG>D$N{dcvfad>84?j~4;02wd&!dpDu14j~Cm{<^
z|NqJg$lc}D91$umK?O!P_wZTP@?BOgP}`yk>i=I*RMo1Gg7o7SJ&V`4y3<nqn{|e%
zY`K)leLH?FOJ;5DITn3pZGPu8uBcBne|2c4h5R*j4GVT3%f&TYK>dFuQ2k%3=%=Pu
z<ET0@<1081fb!!g9u0xf5Eu=C(GVC7fzc2cSs~#3cQUxYip$q_TTjTXVto#7D}mIM
z*)XfA)R%+X4wpHY!R>jFI5x~8>i}-o!sIq^tE(MNTmsfRW%4D}yDEFsChICF2Y&EZ
zt@%-k)vPI(&Z%l2W<xOtW*&%6ytzpAv<R!pyaHjk8c%N{uvvfJaj3Ih5mvsL4!Mhr
zd9uFx(&v!71g8Fl+=Br!1BQLvA!{07d}MmgX+zZyKi`4f4idl2P^Qkic>~-|kl1yH
z8nC?8k@awKmQ2Xm)StIP_K}0s;lhG`Qeb!AxO@O?j_ylgaNNvO4+Hn3VfxO#Wrdp$
z6NAx~A{W7WVSE_<bC;;PYef{ePw4%W2SZJWzl6%J1Nvb5VS0XtLC)n5uwYh8dxE%g
zL1@EDMK2R^xY{c=k*dOX9o73X+7-Wj@&mgWWX4^wpQ?YI)8OhrVk@Q=tG>?-QtSEu
z1?(1(7z{5{y{ee%^Fg_Lu{+p|W7SK*{%*XZpk7x6xt}NQzJcob3wyxsUs?`13ml}M
z$uvsEcE2)sj`~uplA0S^np*QJ$Ufrk-Mwm7ldqt-1EdFrZ;Ob5?Sb+4{es+y2;#FV
zp90(U{b#spa;2d9#a1m9am5yJ`2RWo0mV$-$q0QQy&(J|)dcLO!v&sTb3kH87AdP#
zKe>ToCrBI@zE=R5s|3j#wTq~oo)85N9}pjeCtDs<2JQb-d}XNiWg_IR2#`2v{~w43
z;W;Zy)y~vF?zso?K{$)OQPtueq<;+JBjcIBHiGA*&sOYJ6WLd%ge(VQU)?ZGE;zdl
z?kA8K^8UXoxv1ytuXv`Tp4;f7dM(ii>~GgYtJS;?GlIhgw*N0(s20V%ypQj}YC&d$
z@afNmaDO{q+ON9Q+*~c~hN7ZIjk%gy!D98ahkS}h4d;XPSTVGM<7VEb!(cv0KQ=5~
zct=(6A|&1ansQn#NYYGwo!uh%K5>wqxtuSRTdpDQV^jTuC<k^PmItQ=kUDJmj+ddT
z+}$MAX*}0a)Xje12zFm}ae<=rYERYlo(*6=$c&=jkg`zaV++{7%q?e?p9Qv~=m)72
zoi|Nc=oMt$FNlu|uk2}5i`x58$;(g%C44>a_F%*-$PCx#V(Mv2SE#C8cm}p}!}jZN
zw}Irb;k*7NN{iDWcm9Fo(=#Dw#%%fqDMLVF==j;Y8>)L}$%EZgeIQ7|BnxtO2}g$k
zSPWe~GC$k4QB_bEa>tvFqy^ZWF!>l0$USf{K8U`ic2p^OA|wxTziL#ub{ul<97qls
zb29wJnO;^-4Z(2NPbbK|O2}q_*mErI!_^91y051Eh47h1)2b0?`Gd?r#ugH{!RDPV
zfQ;qB<PKh40QY;BKjdz!Yrlk55AT8Ofr6>~ShESNzwzI4aK46#hjt?FD}jk2)8-jQ
zvc8<ma64w}^r<*L-v|yP{(#?D(+^BP`uRWqt&XWMcfADbeJ4C!dG9lR)wm_Bin&78
z%1{4l!tH?R2hly83@SOFcv0L85{KbivV1B@;@=e?{G0%<Pn_mX1e@g|=&H)YBBD5B
zLN?gW#FGnE|NfT&r{$-9-C!}8IUss_pbLf=NF0Pe>$oUCcw!9J3*vXJS*MDA{!ewt
zY_R_16m598yxk6|!$9}{g7o3T)1pkl?tX1~5p0gmLx_D<S}Wn^f%F$h<lzht33td`
zGDsaV{&}uU)wLoDJcnnu8uPr2+~<##*7eK-+Yi$>Q#BXN|5n4M@;DGt2PIw=Q?Pe|
z-17rc`y;tQdH<(2)$2Py$Y-lG!NVwXQJf0s{$G$97>-=jqU?3)oC>z{f09@Jl?R>w
zvuXE4uo=#uuYmoXxqhPRbjJ_i_+{~6Qhqxf8i&2bV09p~rzEUZObh-24wsKYeJaJz
zeyHePfZQu9n{iPkZ5cZ_OhIbV@ynNI!TMn03kx9kBZBz%iy-Os!gnF%O<bC)*Tf~1
zj_SFC!{uPTDvFs8_wmEs1JVn^u=9Un-azsuNKE#_0VR)Q{PhM%9Sm#Fod<VcR-1;(
z`2&!<1Yz<&m(K>@|J#=9r4n}n5;h>Yts8fP!wkfp@V!B0vl0i2`<B$KRetp3ABrD9
zYC#xu{||`X`|h|(eB)biJq_Z+@YM~|WNOr>qUb&U3DJ&H6e<OW5lsE99mT4B)y>MV
z^MAr)MO9(v|G@75Wi&)RKWTHE1d92W&)<N%|E1$q<<;B&D>H@aD<lM*QPG|LRuy#q
zPiG@9*xs<{&0sn_eJhxTxdWZn_`65B?BO)98qFPYs-OOKtFA*k|F6CxUU6CVDY)IT
ziNSF8{=bmBl+n$F@uMC}DakZ1P@2xPA0CGwxk86!U^|!YIwK#nR!Vu%JIKA+AUTH<
zkb9J`ghKA2>wS1i@mXLyihhtf(EY#443ImrL3~_Tb!WE<==`5O5~$_dgPdRR^nt9G
z&8kUtzRXhPzII4GdWyFk>{eto=<GKUH|4oX&%)L1<=z8!uiM*Da2`fi58|)U=TW&Y
zmjI6Almh|^(W@cny!9sR0E>at;KI+}u2Z(%rVO?t@at6+e|&7lJhO6H!9=CvJ&^fG
zT~P*QYsU#-z3Arq&GUx)0bOo+5EDG!VRG_Zkh3dbd>H+-N*=D()K_1%W?Lv$`(Wxp
zwCCsfIK#^Q2Bc5RCGZs`Y?P}1!}SQ{8-m@wda|B!hnpUXI-l?-VE*rQTj6Kef$Yp`
zJ|gi?`@EJnzo1g6s)LT*OCj()v)ySf^#^|@sxh)US{|RfTmH#PKk%O6Co^v;zVBG7
zoc8`C*gx*ZQ?<=hu7T$R=d6~Ky;a_-c%$tB*bI{<A;Yl3^JY(f^C+f%d1kRMst-Je
z6Dc-Tv8g-L^6rcT4Yq4Vh9GyCF1v0X#=pek47-`4wp^e}hjp;LX<5B$@ShrOeE|b?
zPW@2uJRzr-v&C^13A2aJOH|#T9krOgj=^}V&LJI*Qcg>dIToQhN?#r?RCu@4)^egu
zoRR#ohZaVek&3TguQY!1X@hpfK6&u@KiN{t!EQc&Y_h`P@M~tv%GtFZmA+8;%gt!H
zuz!nU=lVI?XN8Wb^K=WSe+pQsW}-Dy?R3LLbGDBf>dF~G%DU{Sss@Qw;5OC4BU{w2
zTAo(WEe+R_zBmCqha~^5P3t`mx0>e7`Le}wRtC~viWL8g-7(zaxx$jGd5LljW45BR
zY?tb@wdzKf^tv@OkA4J)P4_2Lsk}60<tTGe!=UGPO`fY8XmfRJ(`+|S0MGkQS_3%`
zW9?!`g)?nj7Lzj;SZ>-XtypXiX>TU-oHgBAG0W`e=`~>Wr%#;$hYKidTqekY*-_EU
z!Snz5$=aqH``dMlHsq>L_;6N%@o|x{aEYhsiXTU`qB6>KKU`sir>%46BrSGsJ)t!t
z#L@Dfu8irICN;fwA3OE!nfv5=#RL=%m%jkJA7qdBbYI0ny&A>sc@E(AhI5y{c|-P9
zjeEzoXr={BvN$;ba^~cWH<K(EF|RcZ-aJKdZNf=#`us6P#$wB+24iKHSa5n1aE?;i
zcj>f}QR;H@>339g(kl%to`1>IU0B1V_NbFVm1)6JwJyua3R-NY3Xg0R!T$SNdk>t(
zH_ug2m7k}k@c(DD_}Z)<aC+VFltr~IOIi)I{*TeM#2}F0M``+QF|gY!)-`KP4Q^Bo
z4_ImlTK{+YPLN#hXEyUVkz?i`=ZG1#9h{}L>DWE_jsK@B=(c@UovdwdY9&^IQufSj
z3^cdzs*)<uu{Zt1dCFW@Y_0r0jqMiudezlGD$P?nar2^K*=k;}JButO)mE;Wt^BCl
zS@G(NFDhGi_JZ@mg3ru)w{PpKEj`Yoqh~xt?F-jAi=5zc6^2kw^;&Nguv--@6*Ugn
zHe2rReyAL6EDerpP}qa;tfR6@Q~YLt{o7jArBroB6C97V@|u=qHk)MYPd?R7{V4&i
z(;rQ<);`##qjRj@NhvnGO8eHL7K<slGqqF~y-+=CJ<E9Q#SIp}UoFt#`j)Mj$+!f3
z*6r@K3gEZ_+4X<tE^r)ww7($=;)B-zf#@f?v$dr(7HNu<yiohB&#J1=wnx`p3v!R$
z<7J#mD_-fEJ0u>}y;OZ4oL^@1O;MimNDAx^$=oBR(Lz<qS9(pA=4`rYVBc{NY@cFO
zn39#kF0-|#ge>P<Z`8ebw?^GJjNicJ4wvdgp>H|~+nsc-%AHY+(&p34E8~?tH}MR(
z3|=;IuH@f(UPYbbQ#JO#K&+{H%k@`HUd~FlTSY)|$$chO)%&SRT0I)tUtXU#pDDjk
z#ZvsM)^ZgI3*D7+I!}x3bVDlMSy%|AswHpRs41e$uk^C$D%kC3o(EV8bM6GkOGz20
z`GKxXg`9;Jx}f!cN6a$yqnDmDEjuEl6l~gW*?;RQ*pFPByOsBU+G${xxlwcHynWy>
zjLPjctZ~j%P5I?**3*7n^+YzL&)9uO(tJ){m$7o}R@EahGt@xy{~f<Y)eL_Jsz#Pc
z>2f^^w%lzSug$S}fp*O5hhTS<3R|g3-8g41@9qyC%V6tXt^iv9r+=|V&D(mWv6kjP
z^ABk|RCnpm*S%;}X1-JKxoSq&5e0UKIMY>s=a_eJ)G92SCT79FW2vFgb;+=`exh7`
zO`_%#)-Ls+w7p<E*e|$R#=GyZymsc2;m!;F=6mDpESw#640HFlYR$}7)QX)GU>S9t
z-Exaizr>^dJ&GbbH-Y_reRF`h?Sw|zEi*hV(x;~wW>v|7*RMDW%mA11ZhA@Jv^#O{
z7Ud4c-{8HfsUP-%?M!FyR=9gGz;a)!l#0;07?qGz5rs72=Zbzw?W%dx?9^J#J}D}M
z7aNp)>9e?az(xM>f;fxEZy7A?b%fM2**aAhB(5<FoSvfeuIjUV@8J%)h~%#dObb-i
z9^9U!J%3*dIK9l-)~~oM;E{z)Jrn5sAJwNDAomAfxyWa3@LboD`JJuiB(?~xj(4-w
zzM35}+Qs+?>^7y#-z*;buG7@syh5(X?2vr6gQ0Rq@KTj{7gg>5Um)}Ciqkc~V>+Pq
ze}5#ql~zk;TkLytUg=Eh8L%0kypymz#8_>8yk*7LXqDMFKB#$25l|BT*{&^e5OVM2
z4v7q~8FQnTnf~8=-Nb}-ispoPPMuJh?dJKXyS3_GZ8ZN`tZr#$a8!*aKEN_jOidDG
zE(j|%&QwekPFCA|G6dXK0Evy_(GVC7fzc2c4S~@R7!84u6#_jsAm<XR{#OFGW#@@O
z`eC<Z#ngJ<tx;s-jaAz?KS=#LM}r#Z{GaXnrNDjO33|uWu82U+w03dRLupHVGlaDB
zd8SWRb6>!xuDC5)Mc{jg#=#F}8hdUFDOZ09R7;%lO0~PmRjIa89Nd<9n!>DMev}pL
z2ILrM|2|i>!sim$&O2fU!S;Mjhyl0LK<cc1{#V!dlc4@-dZJoroieze^X6!`dby;6
zGVK1JcURaoc;uPXj?J`F&rycVH`TE~?zGUpp8@W_#hLnRxL4j(+p`=o208OQVh@PH
z{1r;oTg}z?D<y&5V(ZwbP;sA4y=4hxerw^JrRpia!@=&ant-^EVoTHmRpVKwl>AR$
zkl(K$rF<-ZDp)_r?a27+&jpH|eKz1e+0x4!6=NCs)F&O@tTt6$Luu#zZD2izI}!5$
zFArW*Q;EK$<S<EBZR_rOusq1TZ8JWB!vMtAj;mKX`CUxCMub%@q-CeF#hF&Hx;=Tx
z;PB(pPgPy@46+x$J5E6T;LLm#)7y?<IgtJw(NAFY?{>~rJnxeQW`o2xE#g+`KJ5eM
zOWpjhX2{y97S9Gbzpp<Ta&|v+;7t{!*jwtH7xLil0qOl_utVk2vQyyu|K6ngs#acQ
z)HwS84me&x@+)6Ss#bBL>Iqu@TRmnMA6(6RDM*@W3xR}3P+OC#O67UAvb;O0<+CR%
zI{y?`&zrJTW#jD<xL?-ULE7CQJ7742I|M9WRHLJs5MK;;=MuRFu=wqD%1S5AZmIfe
zJW^3;<5M@TidNru!$eKxNdP#_e(qCOpMDOq<^yEriF2kZZ6bTYdO`f|79F)#177tR
zAKt3M<}|NbyQm6De^Kjc+6cFAmQ*-+?8A)R51baK$?>Y)IjXL_D_%+caL;|!WqB@O
z`#|^qJvnCq4tJ3G*zlDD&y_YFUkX+aldrq5PF+YtM6vS2I(5#yQEK1+WP$rvp!5H&
z?@tH2ccXw1SdaNuRk-;aNAIZai8EHcZ-Uq>+%o?V*sUP5+1Rn)7X^|tKJO1V2gLtT
zWvBG%$vU+|TKB>JmW;TkX2X3KY$r$_gcZ-lD;p~cs<1tr57s;XP^jv;9>^II$Aq`5
z9M{Ozh|FFH)<0kMiW=|pc9pjsKh*zmR;xT;G+kp27o&z^{zcUjMb_#+R$W&~{a&Xc
zYNP-*3uHbB2hV}*TLQ5`ST6jH^311UV0WaNc&ewEJW^+l|E<biWDU;Cld|@zK3%#+
zZE_fcdWE&3I<t9?!bdsC-S*tG{M5f{c7ek(qTXLwYh#u|T(F4x28(#Ool~sltLjVm
zseY{bue931S4BO+QW<pq56Es1p2^;*#wg0K20H)W;O0h^a=tuOMK5c02j7WmCLCHC
z``_uS-kS%B^Hhh~sx0b|wLu`gAiVCwDzIG@*6YCZeGVb@rn(OGy+*s#b|0OfTC>zl
zBjeaB#j`UZYgLXy@4ef!VzS!bgGW_;I$tUrvef{m0dwwaYPEmoC@N2ytD(A0M&pkC
zJ#e^q=lloLAp6YEolyyntp|rqn#Cej-TZTEY5aTC{yy5LBHVXc?bbWUJoAfIL$G^-
z!j;uQ`~RaCZGoFHBUVzaoy%3-J!^~FVb9O1{h>G1LF50C8}@?p_`%OjVDmxt!LWm<
z4mjO?koE_g!*u4YTCG!za!Z<ws^wP5eh#PWko6(Eii*H?KRT?Zl6z1=bw?Uxzp#yo
zh??&6a@C@lZEDX-AZx4s7YC@Euv-N-8)nyXW?!&=7#~EZUOcCIeYw9zuEi3t{U9+A
zE@^@Aby%#_lszY?e@~EAnY(I>s>cBn_2phIYR~di)sB|6sOR(lg`4?wJtLTXZ~?2D
zaLz3hyFusw9nh>(+wtiMiakGEA$xK`cD8Qn1IL%w(V6OBQ^Ub*kodHd>~KBPTs@Q|
zoek6*PWvluJq)>*26X<<b-r0}^&mar3s$S773|eWesod&sE2@7lHXVGSq!HG(zIl4
z64h3m@i*t+wLz_mZLh{n6JZsta>$w=vu|?hi;W~yjIB~N_O3hyKK~D7UU0mR%I6GD
zllaNcl>_HY*OR^AZ34RgcjNqDX4gDh%)TAksGPyFSJhv?(0r%=T&2DG`@m<O-gt4&
zv{Y}E*}7c`@=aE!)fYwHQGT5fpzO7yS?k=Xdzz-L3$(6y%c$REE;pMN#%yA6L0{vR
z470hoCY#~q#m-u+zs1cro}Hi~CAv=K@)m1#6%}D~>Bl_=3=&(+-oC9;FM2n}aLfOn
zTD<3OYJ%?n-6blboltX3A@<{C1)taFP1aAb)nKaYRAX=1Z|1h`m}=;&%bHz<EgFp3
zRVoY?pH-ga8fd<qInDIW<L8>QZZ1*Nx;I66L1~}byI^t9`G3ksQ(V-geXKRi%>*@_
z3@6F{w#ZjM=~1J}a$i-QTRdHN!IDtb34y!yzv^5tj}j4<*LeICeE#nysb2lKZ53+q
zGnmcGuc^y_a$KOgtyfL`(gk-t4VP-8)1kN3Z3Jbt8fVyo&;L#6ng^b@c$*)u?yvL1
z)Z)Q4^V)weRl}ZZgX8s!qMEVXP6Ly!j~yoM+r`z+Y!tPKXN}fikA0|e+KkB@bpLOC
z!EV!yKi4bqt=?-?$)B#}DEm%x#=^~Nl5vv_llg0nPt?9wU-MK`yR3I5X#c;`qM%*M
zO_>H}k@wbX*8FHT&*Z;nG~F~oyR9optEaO_?f%Q-s*Q6mg2(n&`#)7o6l?<D|Lc;T
zqwe7(s(kb7A(KxZa@Ee~Owjt)=Wo{OVx<9^|6jeh(!B1Kyh+5S4{HA(JW)@sN|f`g
zyJp(u)nYjL(g}^%9lnZg-&kZL{?1T8QS-;N(?mp*VHJznhnOo`BJIzWrke98nzy`D
z@wZo1?XHeiWA^!G2HO9B^4bP9)~{WvQAKam84HC}{{M^?S#;u$n)~q$YGM*zN?n=!
zYMN`kRSlP})@?bXsdh`s-%MQNmSU{k8;#us)*5>hOTp*=&Ns1B*ScC^Qm&I?$|-YH
zUwYqkP2Xit)Mk4+tLtS=*KCd2X%rsdq2{{kyP9IxYn7SD6;(B#yO_>u?-NV<aNLM-
zeu?S2xH$E$WszoYoF{6AZJDDQ-@>D>w1M5+L*TFS&)hYNPvW?gj_oi}y|AHNp}Vvd
zoENnD&2;9aC1|<@wP{|SB(4>fQ)srQhee$w-AmP#IoNddGY+K}ZS1P-CXVKDcMX&t
zeBWm5wg7T3SBkg2@*G|*`2D}T^3JLLsNqr9$`X@bb(q6^NwBC|=Bj$|csi*5zvLgK
zZDn7qIde0k`t^Dq4b59uO?^Y&Xzr2zt5CWj(~vdel-cb%er>LA+2HekfAvmM+3UOm
zoCf!Oo&c_kUzoQmf3&|LwD8CpdD!{CbKXwV{#<RY{&bIqR_QMt<yI3l?Xc<kYKbTQ
zDV@JPOZ9q-oT;?V9j%p5veZHM|2`?LQWHopFlh^NSM$7`YSLoes?g}vrEcWOtrvPR
zSL>VCbk#5Mq8c4>Y9^8qU(HT^i_~JbH&s^=oupf~NI=yobcI%<Ub`0eJ$u!qJN{_#
zmW!+Fu{_ks4;9e>jsI`Hd{AEL2$x34?XxOV`ylni28YvXrX8(XHSX!4^Zylcp8i&6
zu3w||R&TQ@pYj@YUFR3-9=;Du4_;=_{wKOo+vnJEGi{z-nykh=>PH*uHCFt4rslJ#
zT~)Kg!@OqeKC^$m(i*!aT{XEsd7Em$FKaE({lBLF!gcI|q>PPMW~(Ndr<flxS#6Ru
z%~R#qi6v@m-|y@GQV`cX!!|`#{8^h4%i`I37L!6WWVZh{`M++uW<l*yb%m>@>becH
zOw(_f8Wyj5u5nf_QEi2Ii)!4#CE)Xa=kD35%_uEyzVo4#mYbKSrWETDvsH!4>edT<
zG%x1=HVv)Yt$weyTzQ&?i`tw`xvEDGh-=#9+Zl@TmYH(=y`*s?MP6q|>wDAQ=bjqr
z7X51N>z^9+ckM9kbKj_b!=%k@(Uv6D$v@QfC(l(@;B1rE&_1?QT_E9z*|pczYLyWM
z<`2X7n6Lk}T>qnaifPy8$z~Jv=Ih;^`%caBs<*a!Y`ppLw@=LaCrnV=u|QFMeR3!G
z{@=F5Nv01hZm7AZ+MDUl3e|hEOvMOP|NnZk16=Pno)FXko&V>R!me()dXHAXLI+hY
zzcjN?t8Ep_@_wk@5y&tPxL2qtEXJtzZ0au+Pw#_jk4|k>S|e?w`9s=5x$3H|Zjh6T
z*~VR;lozkxZ@RR^!|c|fH7Y#U=c+jyEHmL?@KakWdsGcH{{KcyQCYb{QuBq^aSi=%
z^TG2EY1h=%{y3j7Q$4UBwEs_0*Hu_kXluQ?<sK7LhxtC{bKGaD&34t**y5d{X~w?W
zpnol|xk#Uon(n=&W?RbsYdB9}(U|`^Up<z0gZc`FGa9?LIBNJkchfr4@1s0#_XM^4
zo9b#aZz(7S_`X*C?Y7g*?p3B*(UYBKe%arZ1Y=uO>)t3Dx@P8>i{wbD7N~7e>Ur+3
za9eP*me*b>O|w%i3RkmPwbXO;G?LQK8GBA^GFr0bnAWs22Q=AA|C`KBtygae_+ZK?
zt7QJ)f1zfd>U{Iopm5bXg*z(K=E|vtFt1g2U)!v*fkz!2)}!nHK&=)C9$o)8y8aK8
z5=PhmF))m-{{y822p(PkH@f~0loE#H`ahA#01b=3eVRrGcs0ERZfZoV<pZC^z`4mn
zbH<jtDzA3hnV)Km)aXu_t_eE-=gfi(_4_tH>PwkUsVlf9D<4SOqG_sK3OfHsF}QJ)
z0%-l8(U(pWpSMR<_GF0Z8}4N?{Wn=c^|64Ixq=g~nR1zv`rn`yb=$55=KnlTs!yLN
ztUved1697|J524*-7))gcA;GDfqwO5xfI2{E3YV-89QkUrcc%Ioa3z(!6>hu;hboe
zD)QNwvFN?3Xs52ZNo|v1Z15{huL<Gi(JXtx=l_MKEY(PPqh{XH{Kmkb=#bgVNurwC
z3pI^YvOTn$AFc<T|0C~N(4zIuJ6`F|u6gp+t7T0S8joli%C1(wbNaOz=={H94_md4
z7&&NI@#w3CUY;vp|H)9j{}H>Hd*XBr(EWb}nK~N0|KF-BTzam0@W>Wbp@an*!Hvb5
z`-&9QCArqCyw}OsxVW@R!?L(Wy~3nMZ$ew7%Eos}1`8t;Es8H}Q!wJRkmuE2A+Mgp
zX27#0SL2$6y7|)-W;M|Hf6OZL)E~{9qF4P*$#{)ljK<TBZp|!rB@@-j5-O_BPTFVR
zvZ@4rn4!M4HOeeH=ZpC|O$YT%D=+Yxx5wqLjhBR%n`{#ZHMx90Ld`hN(c(ghoJQQ=
z8|q#vx6MH3|5r}AXSz_QM!9yAlJNtLi5lmeT{HvkzEt-+RA%_8c$%?eZN0i`;S$Y%
zi_U@1|6jIrrdoAezFB#ApXMduUUSRrhemR159xIAd1<=oIcV(f)l~1f6K~#^(XN!1
zCZM6?^<3%C%W0|yZ5cIJsi>=!WT%_X`qZxeO5l;^RgE~a#nH@~n&nzbjdE?~LX)~o
z9>#oDn`rz4bpM}Hujmg`&z+MDcedB5doFfTJ@SV|uIZAVCSx+M*=K?E8r#J0m@e2m
zT^rQ@|Cq!KZo^hh7FLr9Sf^fayajyzzmb`?MxNjewUhdn!RP<!h2^UYh0AHIT~x1D
zzT~~a+>Wc@^Z$fuX6xA}tyS7mn`4%Ff1zR}gRzF6*izMx$7boP9bi=Ebmdfgw@%h{
z{vru8Hhy`7nmt@v#g87VEq>pm5#VpDalBs3*dQQTeSzi)RgYFL6|QUX>N>`rrV}F+
zB*Nq!jh_lAnnj#8*O;I1-0Yuwv&OGQ4-`+X-lFf^`rquIe66an-xu)xKli`fP_=0~
zsJK_%1$_R0_3v{!&Qo5iuN8LHoG#3!dGI5L`RsrmHPHEg=J%V-<Uc!UIOiv+Mg1u;
zm#J8z6#w?1vETLvsouc1YN~y+)sIhG1G@iDjz=R{x#sLDwJFKxH2!C(sjrRFH@CbU
zr=AkJUi;{!h0?p{8LH`K9o9O#*hMoVkyGProu5X*<|MPv@hw`H-*PISc))46gny6O
z+<))2rXB24ke{^+eEz>w^<*vO{b};K!fxuHa(HE6=l{)CU#nKWnoqghE<^e7gA3Yf
z3-@aXhH+?3>FiRrXKd6yIb)J~+A%&=(E0yIDvC_E&T7>vbUdkX>Fz_77vhK2rqwo?
z?Ax28cIxg-Q2$^4?!`iN=Oa7x3}Y{7^<N58QMK>We8h9b#OGp~`OMm7T4&$osQ)zY
z*Sl4qpzb90N&8N=pq7Kta<#SJu4)$)NULwY#G~$Vut$A@v!N{L{NHSSb&Xf{L26l-
zR)ELK@<nf`IyP+6y5b!LI{#n6ac!N(5e6k~754_Svi&KVF>Y*{6=A$)QpZkdv$^o=
zy!|d~&V4UjBXU)cMl`dUh9_gD#?5)9Du)<6%|-XUG0QL8p?>9AuBn|DyL$1?Gg_ea
zfAT_tx|3f1Gk&v6Th+&BlljpZiKd|Qe@-2BRTKN(tarQXpyr9=Thu#yj;I|d4A9>%
z>7t%`Ld$f<o&6e)B>|de{%ltBGY~MFGb73HN97jvn$qR!p!<KGID4z<Y>QR#>o(Gn
zmu)i-XGzz3Q=F@nIem`V(Vl9Jik)vY&U9*<>8WdL9$Uk$zG;%b+HvEjYFaGC8p~9s
z8djT`m??Y;)_CQ8NJmGT)lB$tlKNEtP3q6rx*Mm|e==QeBBtrle9CNRY>IkJ7!&CH
zANj@;J{qSiWYvRSFE{(yFh?b9{W5b;zuV@z7X=JrXU;T@UcqU8HFT!lvftO#nm#Yl
z;=WyKZkZrqe!pg(T9?;WjbkczOhD)VFL3;1`s#|3hQ}rsGg*-<di+kkM$){y6g8)s
zfcxM2$NIqM|4$HGsv^(lqj{HahDvu4llfWCKt=BxiW;eAJIvS4kJEg%K|nQQ&l^>j
z6;0}*Yz3;Cj(nOc72hd6tE<*?S9LJ^BKBORtRu|q)9EHN-s)#6zVp|sOJ}v2L`M6o
zXTLVo2-$d5aY55H)w#>IsUNw$NQ322pN56dQ&l-RU$sy%FLTiSKcMq}^3TbuU$p$B
zCa~(GNwC*?bIFI()#r72Y35CxuilmyU?`9$Yc3wCtj?{$YJR6+m4>~)r+RWwp~kDH
zo7IhcUaBipNog+J%>ugrNBM1$nQE`nM>Wv;zt@M_)b5-V1)u-_dfNrF1=CAaLFfNv
z9N;r7yuHV~D)YNi>1r?V{6P5w9qrhF9*s*})6_d!rfY~c#;M)w`eA%zahK7e1sPgu
zR$sNkOzxR}QDN11Tk3Ciqc`1LTluhtyyGA9FeV=5%7ty}p!0u%)Qr^CY8+MS_XvXL
zAVBRJWNai@2+kW*JvPZlDViz&a})*J&*r}qJU6NI9Wu^{tY`UmTkyO!vKWZ%&(EN;
zt|Aj`-t8&usu$A`cO}~tn}W>^*XaVwp9tVnl$o;+Je~;B3&RahPN*Ce+6PusB2c1w
z>gykc4!#1gUXYp!PQ+fV$+zBu-3St2#|*h69>fM=xfw^5PcK^_cXQ!cuvs86Wc<Pv
zvR|U_bPsqA3M5x*_6Tkdh<}HlUrF#M58Mur7$N*Pyh?d@y9YRYL=TlK$vEA^YX5={
zRVd<@mi>X-eXHL@wY4S^JkJQTuX^%6B@GWV<!igDP{NGS3Ub~#NZlkpFO?ZFCMf2D
z#4ntOoQJS-A;ixhF%Vw=#8~-+w;0_2ATbzTYB?Wn&XqGU7-3)-h?w()>4DMnJ2~Ng
ziH@^SUcYS<Siiy|PgNhzDH!$zw_@J24KoWLEny>slBS~<zd{Kcm|jao$lm0cCxXE4
ziZIv$rs48mEx~+8$$q7B%XunYwJ~Zl%Dh$2{@DQzgT)mMV0m}{GB6*e4@4UlL-xjk
z*pqWy!Rpi|#Hw(Ki-6T0D_^KIeU=`0y*Ee=GPbD~1&77vwRvjn9{ON*<^D5D|2`n@
z{=Od_2Np-xM=YCf6J-D20UgMmMr``zeGupSPI6^Y3ov$4t=!OpH5{?&hl%yC4gmWd
z#z&`1&5OZl09_o!U-8cjY+qKZl=8m&kT5>nQwC1|XIr*_<>Is^fz8ifaT^@2*Cw*5
zwn#ztct!}$P{}%aMcHe^2XMXvnMn*j=b5K?=u@#OqX*)=4W^C3YE4((E4`FeQPtia
zsUR_N8rWW#dAhOF!S1>BeYwh%I!m=6P7N?0CJ&-7ZxvCqNUE31a=oasbv|U@KS+GG
z<s~I~B}CY+=lY{^=6$tNkbRo6AI~P0JCc@QKd)i70^50E$7wJP(hI}+C+8~f*l|Jm
zv#&Ea9QK{R09GTaaY#vKD`Xu#Of86ZX3$q{Tz*>7a?&al!+-OY+T9l`J9N)cy|E2)
zhoh)us?vXc5wIU3)BV9|ZHdJdc)l`T{#$vP8Z+3A4^^V7Dmsuda<<xZ6;-!`ss*0R
zV7<RGxxns`ypGrp{9RHK>>uHukUg5S?=Y#IWmZ&CDM$g^19BfOtfJEo)(ewws&T{6
zBfJK(Rz&pH9t=5{zK$C{V0XdzAbQqUJFtJc4JU$WkQh2PRJfu1fH_1(D>n(ep8_Vo
za`{bFy(67qKmSxauDEa|pGu9Rl*-#p-xUvEx~F(K39|pYI9^@l?RpNdI+$J%Jv(7P
zhMitvYATM`%Td&-KS7)g5Nj0-_Mh%=XO$~AA?tTQW}P#tKnbHP)t@R8d%G1<&6X*G
z&i@Cgf#K(0GU0hc!9!Qm?!j91qgFRGCW@{B&;L8jWzzVxeW$Xbkh@v@Mrrk9(^54p
zcFtDF>eK><-{Iwu^K+8pS7}5l7=z~n=d6~Kzg6C<a5X~2MEIwy^7(o5^%fZNnD|P&
zD{tbpGGj~O1n&#<y3wH4@qz(#|BuSKH$~t%&xtPWCf!*Ap!<L2Vmx@k>lg&(B$Xbi
zZqPE8EmzNbp{*$?-KAQYH`VlLt2C(puOgw+Z`RfS(m;VFKy!WVY%@c>25`8XK2@&X
z`#Rbz$y&(Z-~t}AIY(}&Tl2UZ3ODT5n%)zr4!Zv@z*1WiH2<G9W2xM|=YA%a)v7g|
zm1We<v^at1vBf_qYY9C6ruI_cjA|Blgt9HaxY|E$bJNxtC%|qyu~S#EwsW@HZjW~=
zu1}|{N~gQ1E1!?oFqLou=aub^Y#N(xZB-B9X;j^zuL1U>uGcmF#Zwy0mtRy=jBv7*
z1NHy;mx$|ocZz5fR<wid1;r0&|DWIw2fbNGHXF%Ft<uQZXQbKKXKTz_ZKXWn(_wH}
zyG><LOTKZ=)F{}_T;~hB+UX^ez<xKr$7)>UG2htojKA?tiE`CEyJ_Ye@8*KtB+GCF
z9R3kA_)Xs!^eaAz5jW!6TCBOkZK=lTYfWn1Uuz8tkDoAlyF*)DYE!djdq5RfAIMFu
zp$(=jpF}ik58pN0b&SpEe3-6|QmVLyE5DKYy|;%|qgXD%-7h)aR|$0g?*+bN;IKN|
z+-)*VeU|$B*?%;CdAFNZctX|!-8x=iX69^QQejx9dTZ}`aN50V$7os-ven?gWKXbO
zXJ%H}n>Ws=d#W~?y7y1iD0wAhD$DT?JpaGC7jpma@dHbhJ+~*REt?OX|5xr--wk&6
zMlODES_QfP-rogav4fc}!1w=x#IkP%f#YS?&PS$ImnSGjJ?2o~Zq~07yYh?PnN?|u
zsRg3S(+(~)agDAqjbp0Pf0*K~*=6mj`f_TZ`V_xO>bVQDjP}2}q?+@JRb`WTyW*dZ
zHp-Uw%uPV^|Enx-86CLqZ7NeEqHdg$WF{lur7r!rLHS#9iT<TV5pY<6`u{xtO66}_
znSuNN4%rOg{(pvCrA|Stt9sS!7!5Y<XX+bxjLfEUtXF-dut9kO^ILEnE^G`{OW9Uu
zmND_9($ALd;5-ZR69~s8cBr5Ja1}frbTCIlZQs^q;IOpYt#9U)|4UVlVY?Qn{|{1I
zaqE+2CHoKc%Y5_I6gsD=AMo8~n#y(`bpMZ1LpPT}K;u2rxWzNI9xK=@#JmlZI3XmY
zT7Q3oCP*&`gZlp<8iqeDs#P*u%%psGy|{M4##oIB|JXH~*<F;+UG3B=5reE9e!PrR
zar*lsCckD1X}L%EgU4_twA!hy?Ym^W^rVt%#JdD=|G(Yfjq2Sw6?&Cnv$VDzTcUWN
zN<t%FHO|<$?X=l?el;zVA0ny;PWS8H%TQE(m)ENuVkWP-^@*_R>3xCPP8Id4igMy=
z?UBZ+6PyjD^SpFqo%qGnk8nWhC6IfM>UpSSI=t7sr2j*0?t3kT6s<<J{^<c)U2D&q
zN*=J)@bY`0abH}^^wTC^ZRwH}ZO((U&6HkFQ5Sl)N`0Pkn|iC8r~0C!Gn7{Um|>Rv
zT-7Xp!(r7;TYj4yb6BEk`?V9CPw#Rq*Xd!|Xq0p;PU(VxzWD)(Xp?L9R~3@(Z&SJ$
zq^TFmld17UcMUjgdadQsKhJbk^<3%;6YbN7)SvCTtllUrt7`5$)3kp=n!)XJTT~~N
zJF0>D|4b2(b-Hb=v$VhEJvA#2zpu$WTSKexL6d2ar;@t2m%YYr=kKQTd>(`2eyJZH
zxIQ^Bp<2Dgew9HN+Yi&*`**7Izc<ry@oF-aI=oy>Unp2jXw^)klNQ>h^A{aaFYa|S
zd!l5iej{PJKIr^^uC;s`m(u2`dIxZU)9<`#Eat%+@6C#@Md{m1+L(ky^_yj1W71PI
zyrY(HlAzg@yWY%{$J0!DO}gsM<x|unR4u{#{2qRuW_qz_zuIS?#ikytQ}h<_@PX5>
z-AM^>S~&2}15EEu{iU4vRbNv|#z$UJw9@Ph$1eF}%MYq?dN7%L?Tyt4=2)qG{~Djl
zqq(|jDmn(r?+T<ej_uV_oPV1`?`x2Psjtv6Wh<M_rpmcTOr@K9lvkbKqnZ=!W1PVE
zU)AjdtNPE|k_tixnN*iPD^ioU;?+nufZUa!zvH^H@m9qBzpLsT)m)}VsTNBdHlDft
zm)UF!W3^L?N}6vzv#YPIPcxX(XJnSibWru+TN^X>Og4>3^{uM@!h6&j<LcF-&l##c
z|6HWOCjJn7{_pqh>B_5qLG~G*5Uo`CdSi~+zBlI;7hSsp&eyh=wUq8n3o=+^r)ciw
zsiB~<j|Y7J-@Ef^S~Jsbg5$FLz%ljKb5ROA9w;0C4(~AJ&v(&O>|Uq!xBj$Ay4VbL
zp0op|N1~USxdxn5_b7}tk4Rmw1iJrs*{S7<^TmQyC;wTkj9mY(|0JaOefc@nU)CSA
zKG`dS$NwLQ%4w;@cd4wjSZyA-<g7}DhbXB2S9Hz&udH;sN%0n&CAe(;A2d}n+pGe-
zo(Pnu!scYDthpj#67N~7)Hc~q|E!W2`2N2c)1%CPd^iP8H)-))%9AQrm@hTGt9tRk
zWN?|c_Oz>+-#k9^GwfyxbK9P)|68+2dGE?A%C|n+YjGqW*E~MeO6ydzwX$UKVY5Z$
zRwg&+dugboTAJq_PB+|OKT(U1Gtzvg!BHiitF|h)`9)N_#XHTD99RtJ+-ERPTwJ2M
zfYZq+xMr_b*N@|%^?&j~k9LCdC+PhD+55#zzu)K7*q`a7qAxFRe%Mb|^}{Sbt=&3P
z)TFH?Rd#=mP!@Kw(#+5OZz{|DNOQ50n3@qAn`(Rjzv_>KU~oSsRViO}@~Y_?^ZVq%
zWn*e(z8cHwUd;-f1eI0Ej(V-Do~p1sw>AjS(YCnADWkA`y^sR0_6o&woZ1E{zptyi
zm3o5r<b&@2D?00^&cDuDKTnF$_{-cqnq|$;H4E97oA`!$DqP&H26ngBSy@%fa|UMp
zwjvhx%ZruUr6A{taB{9S*6z$VnQDE|BvDaVnN3R1Lgiv2IIiX&2{OO3>X@p@d&oZP
z`=SesWq&TwI<tpG>)_u#D%#V28m=+lZ2W<#NMp<V4cf0la^Y=L|NEQG8kuHlK3a9l
z{ON8NV{L|c+De}cv??qmRP=A$QRcjQ8XUKvG%&Gyfimd+zp3@bV0#y`WSBnv_(Z+u
znYdPj%XYJJzfa)uPU7MZ^DIjZlaHzEmG2*c+`)0KOW&+NNXF=w`e|@D_Pt?|FAHl{
zegEo->2ih#nl|NWX4~18YDGJ5R9?x~s|Y&(Uzh!na%<Eom3Im@;4t(R*a~*{$=0K4
za~tP^+eLmSzp6YA<5xXZd{yCz${J<4@T)544f^!{*LJJk4RkV-Dyxt`zpF&!*Lr7-
zxuTMw`+pVlPDHAzJF1$fyuW68Y+JIy>+}@Ob2n^N_uP_DbqU*|d2DW=vHP1Y6{)~w
zsudesRlf5-R4Dqq2b_<qqplgQT7ALvW9u{(^Y1-on?ILpw(gQp-Bd1MFxjCP?9T0n
zwt&a~m$Cg<ys)8Mu}(Y@oEHk~<8_{dq-b(wzShj{V9;Wjdcy3AH=p`s?v2WaGM9kO
z|00;KX2kT|e34MG(w@G%;5-Wodk}sarKAcw|KDZPZ&l6iRB$}hz0EdHJR_mP9wrUX
zuQR#LwM84swYpfJsf0Mr(X3!$FuQQ+iDtBAt8%jNPQxf2YjdHd0QmYpkomr1S>QAX
zVuSEEISFvx0%A9@zW|N@E8KQ*)}E>|N8QGiMa%Lilgjsa9i1Z_ki9jJmvO3on&f0^
z6|hsg?BPjpnZ~|#wOZ(>J`={aONy72i%o9oTu`WKov6Z_bY3rRO0rfRccF4_<OkKY
zd3L6fOw#5X+?Q%+I{sFE`}3h*-<{>k^S4dbdN#jLOG!#VdDrA1EepvYl{U}a8j;m)
zV81_j!7P6=rA>Wa%tNJ=+Qne89jw1px9d#QR&eQ3-8OBZa$%*B>gRKgT6H<`W>1>0
zs?PW_MML#xtl73FtF=~7uhn*OdS#|;eL%Cte6RY!yj~5***$70Ia`(Y$W1UWWRx-2
zOkAMyyG6~kldDAKcZ0U}G~qKU%&*yXJw92ReC#MtH7Px1{$NX}={5VSiiRmO)O2{a
z>wVp_SVO?F6`V%Dl=kQ!KmK3cO6s!7f|O3pciwMR*R8swveaXrsbBS7!v`#P)w7E?
zf$slPKYV+W#`0rwN-|IPXlwSgn2VfO(AsW$SM$4-i@B#-fa<Xg%9{UF7MmVUy#@B`
z94CGi)9L?JLifMcINSfz@TEtt>8HihH9l%;>BRcjnfWKIRWGg$Q2F1r&&Z{k*X%*d
zOI6UFEDd#>>c2kU(a)NztZ-HOmU>v<dA0aYkIZ`SeNkDtYnA!k`$iVKw>cWHT?sNf
zBl*f~ElZfbpbNi7<h8$AJ63Hrzw^T1{QSQiN-AnrswOU{Ozx%qRIV|bV!FN9ST#mz
zx|#4&C;dXk9B}>bShp2i?*|8QgUg3kZXD`dMXK5bY$nQQjShp$7qf=*D&0Bz%;mDp
zG+lmasBDh4Rz4His4|h^sp59GlNt>CdP+fDdvs1!FERt2|DSYU)GXX|rkV4eZsjF+
zm{n%|ax}ev?u5z&gO%Vig3+-_;p?nejppAQHLpKe1uk>eTyImE@2(2E|5xQMv!7C-
ztGMR5fK-+96?;wVX09_g^fFgB@_wliS(T`HFRsS0Cv=(l@@2v*YuH)M_hzwb{&k<K
zDr{Py+8Eib_E6`nx?w!4<`Ov$t;0R+3N>#8)yxlAsfaVJRrq{XQn_l&OS4sznbls$
zxq-_B+sj(Y_j6hebB;_jm%DocbpIc?{+Dqo))HF{xl`-lbr!9OZ)d5k2w^elQWP_8
z)NRq?j)~QpzNyUg_k>oBjcLkery6IO7bH*COs?E#e*U;CIDIWUwOq+->lzi%{eSIW
z^}+eBPdHXW>daKl6XiaNH+Ejvir&-&Zll#6U{*V8dP!Ar`e*Y4C2dO07nH$yzMQLF
z$zp}6B6q1GxZZQ{wb4p_$_thQo&N(W19+G7gUhIZ%`@~PRVSK)&i@ggwBI~3bvoFO
zdk=1v=P@j^cr&S2MSFu0_|B7__m|Dq>})ZYwz&fymk2Y=ln2fKAIn><)zM_4)ZKMR
zL!94KvAo9CTqe^5Y<|k=mFDcaU50EqJG5B~*x=_Ue4bLNaM`2QB3fa+!P#Tt7H5JB
zly<(lX!tYfo!0fLF4eldxr)tAOkg+bHR~!$F&{FW^Hp0@FXXji-XaCCIZwHKwdQ50
zsR=|ns-0f^8Jvgg+1$;}H9^KsYZV!ks$b6oi~n4wptkA6Ql*=}mT8)OYXpZY+q*Wf
z`HW4hN)PLw>w&_hb*-s^?2|SNH7$ANH(9Nq{=Z^M`DKHI<Nlg{bMAt}2BZ#z&rImo
z7d^~vGFkJ6rpL0+sug*WU_UeL`2;rCaQ#gA+*5Mq>yNiv+?gq;#4rJ}&#Cekzp2}^
zv!>sIZ-LccXqpZ756Dc*cdizNzt<}s*<B0HOW(>L8Y?^6X*;RERn44zMeep`o{?PB
zXXEfWd77Ckb#yA-5#{=1r!!^;8v8US?U%5a->h!3a-+BI`n~pQT&EKhxmW8c^2<Zc
zr2*L!_V=JT==`4}b*<ofVS`hM8N>G(YRO_6nyTrG%#(ku1do6BglAbWe(W}>XqyU-
zbCBL2S}x`u7i^45{zKyD#f;PPZR;JB3RQ2J9h$?W?XjfM{Ji@Wov+qS%By=#l*H@a
zslK_eK~dpIy+X#DSg>Dn=N^*=nbne+1J2_CmEXloI*O#VH!&#J?M+qDJsA&<x8p{<
z;QIDtXbd>)H?0x^mp?)>rr<Q`c>0lSF~fGV#R50Xw(j3%@W^I|X84UPIoqmP;IWEp
z7U3o#4_m<O<b|!^{NUG-YbK*_BUO{}#KiZZsd+UMk9^J{c8iBw7OORW3{#a(SZL6(
zj1lb4B|(zP7Aj)OTSD9wLG%C5^;U!JRhao)C-l^A#T$Q~XgTZ<R{N^M3RZu1pR3xW
z9V@_YeVcSo)#G)pMRst2V*Q($;J60)8H6<+vnZYZa24#Hb;sVwMIV8j5%*tR-QrQ}
z6ovO{656-UodL(ei+=)IUoRilD(4PTn6p4#OFsO!x!|jZ;Ql}7<|d<6RU6DLL}aub
zpR5GU|AWu}i7Jl)r$LZinZIJd<_LQfi-PztEc))eme%ud4Vk+f>VErPDc5b`(|NRL
zE;!sl=l{HCs5d*bW3kS@L;T<{2k8rb_8naIOiWm45^a>Fv{TkzDI}<0KmXzyuzlZ)
zOO(X6sF{5IV{CEo<a`|~vo^Kab=P!f1wT+)vi`3&kH3_5$Dj2|w!Z6im-IpQ34+XB
zHgT?$WZD#k7;PDi*7u#@_yUQ~f3`p^?e#J36&{rebKh$z>UhmkGCm%!)wVLg>_cw4
zvWxHyO>U`IW@W|x+NqiTI%no2S+FR*QDQD=1iRVe0;Im0z9ZYh-(U+kUTQbQm`yjn
zuJpF?AGnRRK_f*^@xB$rUo$6JXn5<v<I3hnz5Xs|F3nQ26<{}UIIDu=KaH=+^vKQ?
z3JxN7z~Xa{mz%C{_-nX-A)j1d@&#2;{~zRjQ2oD7n@MM3ZmUJSxQSLwSclg6(Ai*j
zOpX2{%XB}-+$^sLoDT9I-vgVK@@ujJ|EwuS?r&$CSFV&(D10ziXS*S!KTzGfNZ#Si
zc@vQd6U?kH@04|6++-%+$e{M-{t1ICdHoVU+e<YL1e{Y||C|M!9*@Z?TCC*WX3_V#
z&0xdnPbR8jeC8I30tU6ZR+_@KA(}s~saxFoDPpljs9*8`*BphqJ^W@_Z|=y8?Q1kM
z|M*nqNouuu&su(iD_2dx?bqL>hr!`0d{P%&Zh-vg8NUr2j=%5!2m33}X`iBokgdhP
z%IiwuQ<zk8?%tNyOsJQyT+O7K_5T9ck7rVa3^Fb+HRs=TPX6AXL*^P+`pl-PH>w?M
z>`_^IG~e_}sDzT!+|P2L_5UC{*xz#~-BNP}r!f%!Ztp+&BgV?0`+t<qgiL|EW$lYi
zrj8Yd&C5@vsz-hP40daW#}5NpbuF;l6eqA)Y<+rGYm4tO1($Vt3RlwX<#SdnROtD-
zNb|d4Hn>d_QIfA#eqkZloeTlzmFBbgg6IFYuQ~-b17!c^vy+S%Z{=H@QaYtvy4p+m
zO<;hc<+}q~Ir1Xl@+?zc2JEJm_e~}juHP{he*9X);KF*XEb-%J6DG20?%-%OJ8)aZ
zLe#5Uu`^Q8BJSoc36QxUtgovm51Ri6u|fE1^Z~G0AT};6d+k5Ge7<FZI1BW8Rs^_R
zflJ@dp9UyqKbUty@ypfuU_EYj?aI^oBb7G4TLM<&yfz#xj&7FbM}I}%%f}$$g)Yg!
zz!0mQDIeSOM={BU3$ErkBjjvH*#3X+Zf3BY=N}nm-$gNC_V(wEV0)2`v=2Rr)eRsu
z*J>{*O@50wgR}V78YT4ef2YrN0h{gY&H&DHAbp>mK7#FO-Chb81Bqk9=flmwYHIJV
z2Ad<G*bCN+O)aTnc{cdZ2|uYcL&;V!0c<bK?8+zcVEfPdPFC7Kn-T0bn4GF6<ZN9K
z-<!`!IWhqfN7gGLc@QLi{>%i0PU}}F`at5msdmbs{eK`fG6wDc1D*ftI&CeuPJzj#
zNkl1`+3~>RMy2*J*vx)a8@PK@+8bnN*Rz7%JL!U<;zWtn%5!`WXRa#k`wrFvGZUFE
z;_rf+0}?OLYKGg@)2IM8)9dyTg}l3o$}5ueWtxSw!0PK-x!~$R`c$PM^G_f)41ZdE
z9ITc_rwvSl#NA?#$v--7kD?bOP7L<XX_AvTSdG&!#l?^_o6*hc(dbdkySyAlA87v{
zyKx`bf9UFA>;G?cvtX46sTWb+0{0V$kBn8MQO~GOyR;Xq)^-;M*xx%Ye^GGTkcKjs
z8ukWqCI`$MvCLB_@*j-t;qLFyxuDRYctU<tlAh9qoLfq1EQb|OuidPm(OwCj_Xykj
z6>OiwMZ`JPFmsXVuNzJ)?MQ~CCw4wLrO!v-E25wOtDq30pzxCuUIw1u_!!PkUCs-(
z2iZ(y_O#u_O54^kE1OqwqlDugcPFr4IO@fe?mDxmfcF2x^!0sG0;^rKFdiIMtLmqM
zX_!1ReP)U*PC1ymr?NA^=E3-c=(VyLD1KUS98#VV()Z;jH$1+S=kSC5cW~8PobGE-
z;#R(TwhgQ<>(~p04hLVb+LZmr2)Z}^Qi0OlznoxqhDx0zsNTXV3v3U_5C8AV%Iw*R
zKi)v<(D74kez5zH#T(NR<sh<nHaDU^OZ{?BDMnZUrA$Ls3u9|8g!C(5d}Mm^?#E!i
zB8!381)sa&e*f`-3*0^ciQg(Uz^eXwxj2R&ZZFuRahW$--Eu3p)@oUP@K{Xh85XUL
zcIQ;j?b&X2JWN|{@wY=77CUDv=Z0&mg|xnqPcVg?H=$p_p*gKMP5b8S4zPXEVSkkO
z7F8LaImDtIs#mX9=DWf~aa*)<yZ!>Rj~S~?L2m2U@=(?O!Vez*7vkFr?i-(xY&KOb
zOaQO{Kjx*X{_)oy@c#dc3|}>k-!*B*C>d&X1)fy1_GUAykG*I7=GH~^QiTGutkB&C
z=bt^)ti650?7WGbirLpXmB7PG)T~49o1M*ms2|MbZKkBfrq1qAYIw1)O3QUcGkE;J
zW&>pWU;BQB{G(gJ#yiE?HKKo?Qhh!v)$E{Lo~pq#2CatYavBe!_bBh-j!>?$JEL*@
zaJ#8Xe3PbCd9d0Y-rq{cRdUo`{9XtSbBE<e)c#7WQEv)g2W}VgiRY`U8wzV4i@B;6
z@_m7B))z(9Caw>9H)nO4$Nf=N5OXq><<(xHq{jJBPuFs{x^RgjxKI9?#Y$CU?{&57
zRT+ABytf!VpRiKnPu&#Ff8CrW>`al0ZcQ5CcIt@__td(s&NSuv>Sb>HYME+)=VWla
znx7IdzL|H_SSXFhWZK^Ss+0H}%?r%lfyX2ox%<r4YyMCbk+L)Wrs1wApK``X@P~+2
zerB2Ggm5dhByBUpOnW0^nM3j#%knR2ZCPNbc30yJI9^w2Nt-gP=+YDwQZ#=s&DzLn
zKA*OM@(<03$3IoShaXZcIMD@mKPY}LMae3I=KnLIAa@ZtzdmWw;x<X0>!gyF;R0K;
z{+Bzz?kN>jH*f3SW?aG(rY33M2^M>N!`s9*(8loL-#g%NWcth^AK86cjipx0^wcFI
z&9&kUrmfrUw9+4%DBH-NRsfCvR~LLy*?n9_&0qAhncCuK%3_>x;I{UC&Sv!*&Nbk0
znei@BP3F`+RbDnOrT<Gdsxqf@sX82aqH7g(Th&7Aw5eV92}PEa1dZ&^*VWhd>w@?H
z=X~6*cHsDZla>X3rmJ7v)UW8v)~uhkNo`r@8MVn9O*O67&NIqdv`;lrbdK7~ycwYJ
zf5orTg5WgXQ4nls`fHi#ed&*CT=&Dx)}_g6&QhA9^5dwX{*?qsSpD<d3)=s$7%zWR
z<-&$?g;QJEz~d1`+EO~J_cCj0+I-bm-m*zE?3bUJgz0Iu|FMUao6?to&0nc<OEqlG
zakGe)rAq#?m%---gTfw!r+iqicKX9ra2@4yl3&&M17!Wju?J0N+PwU#1_dJEJ{3qU
zv!$%I+p@VD+@A#1yo?$(TDs1dX04CVyu!Fnsfk<7&||KHnJya#Z2VuWE1gHBsV@<n
z20{8k_?w&rIE+E;a>m(UHn*&V_B(zzb#=)-p!t6#cC~Wt$}kCVyd<PbtM1vvXDU)~
zLMx6DG7h=({ub3fx9i4J^)IQeFA@jO|Nm~0QD1*cKrcbwNo#>#u<B>ka&^vdH<MG%
zGt74M_G;zv?N|G}eU0v|NPp$5`vzKh{AyaR?d__<OG30VW-V0Pw5eC)dN<^JmIp7G
z<pa0XtGAX{t6VT-2B)>Xk44nfR=w5A?c-A4;Cn#HcwN7`8n3n1>CzjflUQ}sFT7XK
z=wIJr>Kxsvwaw5;`!Gw8*}c{f&Hat4>eb%d8s(3ct8Gi<QvJGZy_tu5rCFck4fPo1
zKPFXg?yD+J{H7_?!l%k{VxmrMLyECAlaT6K78Ub?Q|CeR|H@v^1J!b-DeATSo1t;K
z>=(EU**WK^-sU(Cjfo}(CO4JzHFftisynkBRm%&RWYXTf$$+z?O~Xy^5vc#K>TDbU
z+W)U`#HvsGP6CJdO}A&7KVL;^E?ebkmJ+y0EqmP=jRX67Ow%<WcSX&WEK}X8s-w!g
zYL&)&TQkG+{o70@9*Wk8cf6;)erkg$U&;&hZ-;o*M5fL+OgW)zI?d8q{m0Lnrib6p
zQ?q>K3Y!0y|9tMC`us>i^}S#2nwncbQe7_0ZO*~dX0FwDO5gJQ4wHlK&SqvI^?GYw
z1*<2V{h>82kI8%wcfHxQonmT@fjiWM<2RTrIwGxdTOrVN9`9B)O?C#efPxOaP91h|
zo%3%+KRC~4Uv5!n{Vk!iU$9c`_gM`sX7yPro6-}^_8z$+-M3(>T5AfIx%b`s8ceCp
zs*m3_sfY(9tAWP<U$Sk{Xiz+$RLrBI>)I_~_UN#l@-s#;Q;{A&)7`DVl<nUcsx7e3
zH}O8~p(faW4Lla*a9l*i({-yx-K<^><y(f}@&BAJMrxY^x0*dK)B^SY6_?5^(O500
zs6L7Hok`6~MsrycIrV<?evK!l+?q|IZ3c^M3e5I&URI6x#&4D-xlrS>z+Cm}!))po
z-W#av{rA@>vzx9lUHFt{(rYH=i4U06rg+$?_SRb|zL}Dun#dPsRy<2c?d;WNaQS&u
zP)t>5S+POLLU!}tWd~FW)V3&t?*Bb0;Hi~hzX}{a?W?9~CS6>rzLq!0_@fhpQC4WW
z*3&&VHLK%xo2<5(tN#C%w&}zVFU@wD8EZQ3kTO5%XRccOut#OtspTpjnKtT^|EyLy
z{&G7w55V^SZ{p(yn;qr~S=+2!vQrgw{$JJ6PpX35QtEn7{K4n{UE9Q|2D<;}dPTn)
zKlJ_|7qs*L+Byz{!?q;PP%Y;2TQ$)6fA2gcHM~}GXfSuZRJq@zt#)k33{}wmKfdD6
z)j;?EFbAwuKc_qw>?Tk-0>g(k`>I;EoCcR0g?}5tWsAd`Jh1s7HSFfw)g{B^)Sq08
zQp<>F1lLuYK1Qj7&i{km|Kq#&uzJ<c7*)YWb@e^cf5CcT_y0Uy2)Pd_`jWWDs~3H0
z>%uO9^CHM>9(KgKSJ(S7%E$kRtIsq157rO5|0jI%L$yZvYOs2qgI(%TCk()0h<yLg
zZnsGA`G27Me>!b@<*u<$1N#MJK02NjWeWBm=>DGx39r?rJFZasaVA$O<MLLp9(JZk
za9a;_|Ib>9ZndWe=P0<nSgRT{F9R$OGVjwQ$XQ+>HfPOw#myU^s4-ertAg(T$=_lG
zR;POyvM$c`_yOf<juXIQO;<mv39nC7*>MJPS3F4ng`!trInez-KbfV$Y>@b?)t{B8
zos|Rg&3Tro)nv!3ZHuv1N520DbpD_6#cAs3_x~8{tWvon+YdhfPe(>vHT>&l^?%PQ
zz+niw|L5%OCo1Uo|G53zrEdQA4LEK=_y0_@RspAzr@h7CxH!D3PBpt_zS^{Z)nGpJ
zT26J){Xg12d%%3zjEgE0f&{^OK<<a(3p&DJ`TuD`s)0J@;IIYBZBSVN7GLVdAZz~M
zy{gW7My0}=OzJDMt<^c)mDL2Q)WK%`P57(!<v3!$Md@)d)npM{uwIZkz1z}MSNU92
zbCwNM-7!HG9A0yE8&oxxEK@_i|7S*uIN0u~toy<Jr#*{h)c%z&QhJ)WUae@_Tb0@R
zI$--i=l>nO*Pwd-0RH=bj;qy!)x+-pNtBCF2i^bU=eI)re~SS8{6EnBKeLuT1iN?n
z;(K8Cy-o}T^FjT8ZLR$(M=gC-KZ+yH6>F6$2genm`+shJht&0;Z~$S@{Xgu>=c>W(
z|M@xbrdoC%<eWm-{XZ&cn^h8uw3U`*L(Zwpy>v-s$34hi4EqHFDzg?iY3%Qx3r+_z
zdwSIxufJEBvgn$6g0Z-&-^+51rPJ1_H}{IF?KXR*7SYqMGWSt|(j3qAVE=&J0K!E+
z$|(HlCod~cJY@k6uUE<}>PM~isY^M%P_fnd3l<Z!nyIROrCY7JcD-616EFDuKi?x(
z;II;6msLM;;sn?|b}#NJYHiF?I8iI4?tEGpEDy5pv3!r}?)xmNp!<Kk{!LbP3tkI8
z{||QmPw-N8H7);Ds-XLS@@F!ud`tVM!s#lk9#$=`)*vOJaZ)B+RpGUw8tDF?<w9Ro
zxZPL6%?927!xo+i=I;xEoXZwrdt9wc&tCm(Mu*yy1AeNiZ$&jg_x~(wP*wxo|FcGA
z7FfN6M~2$$h+gpdf3*ub!C`Wj<EYyE4V+3{->NlK-9M}EZrKM8gUGiVz%<A|QPWwK
zRh6B<<=mlWFIDctO=^rvZE8!T)+<MzT&<RO*cEL49!*Fds{eCH?fkzTN|l`Xa5MPi
ztW-AznW#Toxk2@Y{%X}t25Z3Q|H1D6xi8=bHXmdk49}I61ec9>`626sx;1vG9WxSF
z-srGZB_bJe$7-OyIoOU@zaVE%M)}I9bR0RWGVcbbI_UnNyH9?pUSDFanmF4>O{wDu
z*qw#dhgG)RO8}b<vI}(oPh?UNSPsO8;q%MasYG29)$sF~1GWPuenqSu%=c*ZQhUxF
zsXj?$r%L%rCsmELYU(G&ZPmU698yjF8LNK!EhK(G`o7P<4-W67Lk4QaYiz)5kT~f6
zAJF-K-2Zsg%2ZZ@%>{{ru;s}fFduaP&;1{eeMlXpE$V?T1#tc86}!OhhTZ?;_f#2t
z{@)pudGet9e?aH|&3N<^oPR-PBIC(7B*5lt@uS|`WmW!GWmyQTTDl-)-K?FXG}yl0
zOOUxdWHVsw9$Cnq5VKpos_mzlK=r?}GPj_5z<<d7qlYf<R#AQ61upA8U3dgG@9uGb
zwc>4vGiG6CmHpqZ!nyMfSPkFw3~)IqEb<VX=Rj)SvQ1Ds$}Xzr>nj9S0}?Zw=BcK;
z5^-nR&a7!_^W8I)HP&r~s{yG4;fJ*);C|AgyR7Q_?p{KXpU@ly=U+*cR`eEprD_><
z1|ELi+uy40$}<9|on?=fsBxXv1&1NXTo^XUgX~Fy@p}qU&tHUzqtgy6#no=#U!WR1
zzgJ23_aCr3K>dH1y1=iHbxr&`S(I-t)CJ4K<dEso6BpoezXBe@-40V@@X`q^FB$Vl
z^~oh^uwP(uDy@)nDnWdCDLJq@5POSnk}}&?PB5Qony#v+u8Jz_dC0y8kUR*3`u`{1
zmV?tHh!4XJE3%dU@}+^r-D`MMdI}b(J(?&7xA)0ac5ocS)PrcTNlVps+}QvhzbmVQ
zoQDdM1L4FIf~x=iPF8I?^ABte`(8--8=KP#_pj7OQLvpL^)URZp&2YM%Hyq^9w`Ag
z|Jf77op{T-<yAY&e<<ao{8Cl;<gdn?e?evLY)H7|q$sFqnJx#11IWzp{0|jn8kT_F
z0OH%esaBf)p&cyWxmXlT>u+AI^4}4%{uiVMgx#NPMe%c}{&%puRvgO)>#@IG4R^cT
zS#Pi$NFOfT!xsow4_3nsX?wVRJ)(A=T@7p|Se$`@!Tb9juzAnce@77q$+sl3C|_;4
zq&$KD6<n@Witt%5AiZ0f_`v3Z*f2a#Z5iC0UEVuYjhZ0!JWSpoXfrrpSM<4p)x{}F
zD8D+)4-XqHO*44@@=}I`Vd33ZYNA?I%5VGI)K9-{SKC=;qxv`bvZ`F*M0k9{%m>j*
zJ21nqhsg#UUhNG(Rm)=!s-<14P)c!xlvUbi0#*99JF9YVsHo1^DGOGobR!7tz8qUF
zFdt+t3^yOuQ}efc4fp$hu6-(*54Xb81WX-_uKY1qWl@<FSp7DpSmlKkx0P=_U{G7H
zS)uacg0TAA$kR&U-jIFM;&G+Q5qu}XcEj{DTR`d$TZThm@d^I>R4XD%)E3vrs%rBv
zD4TilgTri53}nsY`T3CaC^e5!)tU)%U#a$_Jh1($Um)w8)*dTY-=O$j?L^WOu>UM4
z)`4l5og2Sufz3Rm>8_IZAW}8`U@qA1fbWaIG)Nt9Viq{wPTrji7Q5T0iDCw`P=cCF
z-ACmqJKPjgCTvELE8GC7$3f<xW9yY+V71P7(^O-=^C;EoC#cL`8w<AMZ3bk238#EL
zm=EgzcN{sZJol*@SWiRYPOv{Knjfhga6SiClf%7Bd2f{}*sU<TULA()Nr&-4w9@X|
zN>3P{s9ks23pNWR2Ex3`>%si3o-<Xi@#%rttMinUO)Zqwx;xcW%T66saq00_o0236
zwhyF-?J?wz;np8*s)D<X!EOSH!?3j22^D3{DPXf;V)ly>cZ)mjg_KWGVyo2xq7T6J
zPrZ2pZU;!;(j$=jjzMe~zQ}+&*V(tL5xk}cB)?aeP2-eqEqrWnj>#qUrbnW>AURU-
z<m2<ecQ%3a?BD$p%x;~uQrlSQI(*!)R=8ddB)=}L6g=i~O;|(ouGJE-T97y{+`jI#
z=7OA3usSuHx7vJ+cfju56E+RS?=?#vX-&KAsC&ML8Lak!MF38JycW*{y8&cpDElI~
zI<_6P5+E_xrqAH?{_)Bkuvs865Pmgv6N;VD_gke8t~mvckC(3>!0%iFsfFP>J8igK
zAhG9>JYf5teYg##L1HlcIW-1sFNhy>MjOR$kl5Ch@jCe?iC}%7v{b-tW<|aC+9hv%
zwRY~x28)Yc4u{+4VQ>a)2IKPMV7ZG+{(xzaSs-j@d_W_lzZb0L-tj)TK9Jml&C9^z
z%&`-6oP#rT7H+SB$0bNU+a(&#H(UJ}#qS*dj)3)o)YV3s!`Wq8!rJrQ{J`-Ck^^Be
z)mPdvD^BT5&XfeN;RA_*uv@w;*sU7bu5kSz@$_2A+y;p4IwKOS2E<0j-^>N!?%Dq3
zzGU+8czC#i)ZJI@K+(5vGvZE9)!jR_r&uVV*a^}D!djXiF~l2_zk|c!QFptJJo{|0
z{M|W+!Sq`f$o*aG4s(FhKFADY?2w)Y4)d=p)8TplZ;-rN@kLLt`WJg$H6l03!`&+W
z><U=D>wOb&nnN}V8(Z-Ge7Ih0@*pu8=U#YSm0j+q`Stu=u-id$_^{cbK5=~V=yE2H
znNZ>#Bro*-BG|61<jG)vfyB_UbM05nO-mu?w4%!q!)K{B1@}Afsr9{|3U{07#(p?k
zb2;{OknL2atFJH@9JcE+AA#c^X8y}VYh?fG{{@Rhr&a5?<zS{6m|9%)zt!8+u81rK
zn~zOR=*(Mixy`C(aCWeIp!PL~<M6Z((|dOQdhM#~-f(|i%7m1mAo=S(M|Ez076HeD
z%*jn)HSsGZDpiNBfY%fI47<Vcb7b8#-Q5rFg4yon?piO-%4x?0_=5F-%qp+gsQb-t
zBABnbm`QB^QWmh=SnvM;hsT{??7Ax|XKH--yb^35NDn$bRU`$r_t(@nU~@q7Et}`U
z*&seP%<)?YY-Z<<4?2I&)v5hG#sn@O+kU(Ui(}J+EcW=sdJJ`sL(XgaE;2%~yCgb9
zQz>B;SS<4Ebg=)B%^}3TGWP;FEUeF*1cxI?{T%M2+I@nuU^78{iTD6`o&@n>Sj3z~
zCs%nNSYOtR<5JAaA>{}A+)LU!_w0tdWxwZba2Ue$A=CGH1XT(St7@*i(4ukJNlQI;
zt`K-G=0NN{jX5?eRE>X~Gu3vSr@l;~Q&Uy!sPez`>1xjW+{)g2s?|UEXejIIs%x<y
zT&ZQvDFtprJx~)<QchWEv_V)=#b)9o{q-j$jBofHR*@|4H5K2;W-2$kQYlY%mzsVd
zzuDgoXSJpM2leNg@G7Q@x0x(tbvFHUcA>mP-FC&d88?((X9OtU-zcH^Wa3PX%Y3Dp
z_Wic%JF8xpFo$#)9qHbxnz;Lb=^cGTgO@GW)oYJTG_6@0p_2O{P&sGkU-g^iyk@;q
z)eR1Yb(wx%m8`KQ@~}Z-^;*q3o?g)Te+u9C25BYK98<JwJSlT>eWNjBlZj?@tc-f?
z^gX8W{|i;*XLo4*E=$)4y`TWD<KMXLS391mWb$vDzJ~a7XVsjvh03cw6sU7~Xn@ZD
zS9(&$p&=1?NF$_ak=nYI-U@wXY#QB@Sk<*RaH${Y@zZycKCZl!qt)Q<mku*$6?XY)
zl^5kc9%xYht$9xGjE9!`FaGzYw{qtxxju_jS$A%=%Bxj;22$HM7<NWAYVr8xX{?;P
z(Wt(Ak@BT`jNtUbbGupnddM9Ut(&rDl^ILa7CqepUiWlCZJkk`LWJ>@J}G0#X)jgV
zO?Q|rWzy1^weGFTROdCO|99R~IpJe#!m&<C`RbZahM&*Is(VbyQr~YIqCQd4&|uoB
z`-U41HY)x7A*CTF@20M}E>(FFd!_Q^Oj*;#t3@>zGrlpkTyoR!DqElS-|Jo)3%#4v
zOEr4c_U%||dVS{y`SW~XijvcP71wDysvbM$p#d8IXEl&F>6tQLWAo-+n%Y@MP2Mgm
z)BvsjdoIOhcECg4xZml9T2uIPHPHG$lQLG*2Hkjr+2*rU`!&}p^u1wG=v|qkk=C7J
z()?txa?|A}Ci|rVH9`CTJiB&*^F`g=m8wZIwbV1_Y%~R({~vO&K>ZI}uxd+yf%;<Y
zCZ&#d-<55S_p5`}{~y`NAoK3$YE|wXtCVm5uGK&K_nwMQ&07<rszpjW=l@U-t=Oa5
z+Y8x;w@zPJeRe>Wu|4NvlfK<427P?`n!?@h)Z!0ZRZn}eN-f`A-^g{+eYF(DO0|7#
zYn3-RwW`VoUooE5-X~$J{myVdi@r%k=4y3$>8YkC&g*M%JmgWnZ?V<Df@y|n=3h3Y
z4?YW&pTuz~yxQLeI{#lu(Pbfc{kOSxoX)DYLiHJ(|I|f}9@m)WTxWXp(npmO4_uWd
z`))9a()g%S&7-Hzq~C5fYu_4WDce-zRYz1L+H$=V&mXZ<Ipf6zKK~Dt&lknE%L%@b
z)6gvtRmq9^VCp^LfLdNx6F3cl*8eH*e5mz$S)YbqUzd7EroYDGw_&C(%(vAY5?oYX
z1+W@8E|oF8U7`YB{}0;#zk9i-s@e--u>P`V-@ti(jr(OVeg2aeyd84tAdl9BX^rZ+
zp(ixN_9-eGYEIH(=KT#G|JO3tQxX3-*_eL<v(}fnqMD%d|23i|6knR|G<r9gLoNT{
zM<cKKD-?^D*sAYZK3`w6XMvVATc_&$juRTY*IOIw26C8jK6|5ed4q_$e)(D5CzcCT
z)hFK23fPvXc_;F%YUi@+S{&<T)K{)lRxjH88{F2*^U{&y<5W?Xa@?poW5ES*`--E~
zQ}uY_A<fzLFV#W&|DCkiH7rBsY3AizF`0gMjYgcFyypIg^Gx{Of7bkIHA8z*@m$lo
zHP6&j_ls#R4Q*3ZShrrCWeS&S;d@rI)q8wQ_ezLse4lmG_?!P`wM7{`I*BV*DBJD-
zr_1^x!l-YWta4}P1T(KQ@y0cu*i<-#r>J~h)Tb}i8LY8s))KW??U%}m_KgNxmnW&;
zEIe*3ot~xfpt4WnS**Ca_o~e%$vi6!gkBqJoH!`}I{#0_=eV<4Pseu!7a@6_CAOkw
zFCASpC(p9hGWzIkI&tc4b;$$f8taP^O%mo$(x`|zpnQPSK&{;MhVr>GAr0%Dtp?VU
z_nPQMA66In@<LnrUY&{hCM|X5JVkZIjUh(UKgF3SO>EXso0)5>_WZuO>Go&?(Ek62
zht8{i-)5v98r5#fX>wN8@cBJcP2qSm;rZ+ayMi-JPB7P-TK>JJcRl@u#)hm|t<E(s
zP4_BXFjYPst@dcOl!gF@j`7^H+{*qO9wwH5=BRVukv45rZ8Zo8;WbP-AfPaBn<==C
zUUvAtMugNFCE?YJmFM4TROc(_Q<bq?WO{~UmqL)xeDyzTJ<NV5`D*Yj%T^VZ$x*dU
zlTiOzvr=)g{!$IMQU}EimD;)yFIY^q{OeR+-83`}-N0+g@j5|e$)!(f9jo^meLi(Z
z&8(mcoGx^xXsB@AlF`_@?y=I7rrGLA>N6B2j0)6BJ5QT}=Kn$G|82b9sJ8R0y?S<_
zim|aYli8|vJ&k$FOEuqmN~m`qKCZ8H_nhgB&Yfx&zK2a`oqVGHtM7n%)#lUcFE}&R
z_q1AQh)61EC0;$K@l1q4+4{gOmE~DIYO%lj<RAHXt1Z_}FtvM?srn>fhH1gFQWeJ|
z4ys4%{_2bV<}z~^-UmPb@6sYmExGnQHT%F#%G<w6Y1sYlROf!yZgft4nxW;^6`E~F
zmue-XC7A@OGHK3v_t#`zHM3dZ_c)EVXBK8F;y6_^bT6ySjk%&C%+;k1TK~^=6mq@_
zsEo)kY1itx_DKEmhX^f+`#Rvg7=_lSwAwt+tKOARHQ(H%qyC3wl@{pyUpvJpRnYoB
zw^CVf_}M*wul4xsah({iwaULrxRrwAeUy*fd~CeU>WEt8E-r&vzsgN+_#9SFuRU*e
z?q?Brjl<dJ&1$L>Qp|-<MyN(Q{{^pk`l!Ca^s3nuGijSU^6%?(G><M8Q(iQmNA<|o
zGM%4A@>-$G=4yKf>1jm0&M=dH^}<+5kz3>N;|FE|@3jqo#J$$~5XWPlnlwXY(uNKt
zhNp|w6DB2@&-ZmSWMzG1_VV>#^{p@W8P@c*>g-pO*IKuWRdLJxb=q+YtyO|!73C#j
zADFy%Yt))(>8w^P!*2fMYqRQ~orT(44i;)mGigzOvrIrb_MwU<qkDnr<G&_a?z)Yt
ziy!?{I(P4=TH%B>s;*C`s|B5IR^NDUk%n%7I=DTy(UL=BW51BrrZ6A1KiZr1C!O1?
zT>j&P!9k-b<_{AVDt~%(UcTDERw-CZ%D^sch5F`fW^+(n_$>Oal9#5e=Ap`K;Br{U
zcnRAMt-tO2v{Fmo8?R9MqO>C?9^5XiJ$+B@t}c_=LB?kDS07KRN*~e&k9Xd=%4EW2
zJ=Nru>{63}i67O3r!ZJ}@T7qA)o0~jX8tvts+U=pnx2^*s~VeXX*|_&pB7)&9j#yG
zE7j5$Rv89P;4#iV(WG&+OF_G9eTSNIgFLwX(^#!%Hsk9ettYQz&F63CGQR(Lr>=hE
z2d#g?&FY_K&Qx;{`UZ|$kRNoso~x|Wc2qHDW(3=Nu2b4H{b;U6^h{yxs7+F4_A!wA
zz3yyGFn?!o%=pwjJN3>_nc(!qX)0}|tJP_E=#?xu9F@E^6l{2eH7qWRoAuo}q<ODI
z&8%atf_C@aC>0y|)8KkXVs)5`_lE{GTYo8YwZ+d=y1ZV3%lBE#ff}ItALJ&*>bGj2
z+}SleB-bePA5>FY;mN78?bRWD2S-oUwZ|FEOjga3sxz(A4BfR!UHiMBf!YB^RmQ7p
z)Tfj!Hn}5v%XIF8UV}GgirOopE!4yk=c!lta%w&KHqm(g341m9bwAVtReq{U8`UZ+
z>`phGl)Oc1W4EAjiokZ$iX~UoKkhwjw$qeV>-K$l)$ETC41{GagTu<aSxjYJijtD|
zy3eY9s;`uo6J5ad|1yiKx=T-O(VW?%q?PxtS?dM&9kY7VDh(d*5=Gqukh6oHZ1Ggz
zuxqM$>a=sp>no(e`^!N71mS#zsqp)Mw3H1rI!;{&hpp@85c7vC1J#n)e(S)_|0&%m
zs?!=Ot#v|<McqLyQd70Q-|URQPVFm<>y+C52pI;-JTx<?kky%Xuul<m{%^~AFV(-c
zYvFy9PdU8OFU%ol$ARo-&bCq&>AtO^5-+T)w|au+JmofR@4P80PJLzCwFh{?@$z^X
zr^+k=2h(G#gmhGR--F}X=1i(a-F9vh_n4__oFXMAD_`g<eQQrp=c{2gI8lF1JAHbq
zN~N2d#)QXJCZ`XZn9Iq3&@Kw<S8raqSx?m{L{)yLpsw{@EA4=dlT@7S6Ln^pacF$<
z-=*=}eTw?6j7B-Gs>kw`f-#!$%XL)0{1gF)he4CLx}`*<w%oQPQ2np;Uz$<9b#JiF
zw^n|$4K4*5ou@RkZkW9?{Sfy=Cw7&)?qMYk^Nk<0wcZ#BXg*zkNz>nPs=7=Quj(@e
zPV<v{H<~%h-`4nhU&b`9W~*wH$_{Y-Z+$IS?^^UB<C)ysRhE{VHlKaA#{_i#Z%^wS
zRptYy^cR16uhnw;5I9`Eo?mHT8R)OEvEYk|;s;x;mr<-5kK<0M&9rYZ)%!KiaK4_o
zrn6)(`2L^iD<eSb{}kU&&Cs=3*J8fYa*1~I>}2iND;Jwx<PTR@UK^(+ZZy|)e;Fq@
z5Av6<QK_-Mto-7~5>2Z|lMMe`-8JQJdZ+2ybx)VKbem~vM7pNMmoBwmJEM)w{~k1*
zcJ#OUTK-_O?h6hYQxeV^<k~$}^tlnKDZAZPV|Cb6vwl5sHIATg^VG`+%|()}3~Yqw
zm`>m+HZ$K_qi+|ZshP9rv$onjW%C(VpO`IC;#EJUGh1D-?U~7r89K_76hcjVFMLvK
z{!nQaTqtgEb2En#DDC*fLiS9r=&jXY{Vk#LDr2?UlfqqEZ(7!<9Xo#Ctiq^6S%l}Z
z+AQHL^EX+?w4Bzqt1|y+R5|CqTFu+{yxQu!!dkh<MO3-3r0acS@i03t(5cL>?_{>(
zwWnEmx1O?=<_5L$=~*UsZmOxRw{}n$nPj5;c+DM^8C!ZZkF65Ys$K2|ZrAE<ysG59
zh6l9&A9VkZ%Je2V&3`(Z)ebYSH#u2qYkos>od)A!6RnjuIyLp~>@Z++W-`}JJEVGA
zkl8%`vZ|Jy&2shSN%rcG#kn-T9TL}UK3<?%ekV-3)mL35w%AbZ74vyjQ2ie#y-%ek
zBHHXkfs$$p%PVl1aAoR3m93&2hOwJU%<U%Sf%gA{%imvp={gIJD1zPR#owl-_DM!V
z@wJZe-&5C(E}9+DF0{I@9oM?R^l3Dw#?8}QW(7_@=3k37wfLu{m>*+JQ*HCzqzXF!
z*W5NnU0vgls;dDXI9+hYI)n3qYYX-<#L)Roa%b{FluoU^04{4)j)tqc-qTPMy6C3F
zl^O(Z?}OqGhWS{&f#=kHPp(p0aHCcEhR<Qei~YUIFUl^eCfXUOrM`?)obdOh>f#H1
zYPxkFm9sfMsjU0ds(L5L9NeFSnFFGo7fn=^aDvRK-U(U*cEeeoV(>g6NWPq9y6WG@
zIx7CEh;w9@yR1?%<lt54X}F`dJ<maT_1aacp<%Vki%$PmJA4^(*49O-GH{+_zPAiq
zZ%$ZSq*#6fvbP^(E(r52-3*oku~lYBs-BPApmy2$I9P1Q9&vSz5GCa`yFY^YpJ!}Q
zV!0FqHuLoUKDa-$;~{5RychNYi>)a^?9l+3i;V4*-N11jvwVWu<;-RkK9`?h^LAf)
z1l9{vTOg6AG&w~JEVgO3h5TgY25`85<So2v!FoV!5Pl$f3LI8fBd4hyV`2j9S#n?k
z*dCC)YHc9c{o20@RTeB_gNuXYj!!ZN$8XH5{i=)JZGp>`@!V2kybGBl^L{3%<apUY
z#daDqX#Jl;9V2AE5p@2~(`o*yHTI}$4vw0Ms`~FeuBsQFq`3G%rP6;#E`{Igwu0U0
zwZ2a=U+uW^&H4Ype$cSU1lxb7bfuc|VaWV{<5v~cy=9GH`P^ss!St4GGnJ~gOj9{<
z`v%y5yAo!ru3ocQHGY{OSU-nyfLf|9=DLf6VU1wFPrWayqVBOBoMu7(4sJu->6@1@
zRV`yFi_*1&XH<T?->25KgkL$?4YEJ0q#|E+Vj~OKPLRH-Ti>f{NkP`AfcR~EH<Uk?
zp8(r&MZ5?cexK{7C~e*_OZm&pY_QwF>bzIM;}677e*sw!k@t-WtVVZcI+zB@X>HlB
z%3_3Aj|UR-`cS95>WGS@_3p>2MLW9G=GH9*hbNClFxXrfp#@3~vsBesGk2=Jeijdw
z1L-s1msY*9$_*Tba<ftQ0nYrjQ8AZORcTY>a<wbr@^C+b^ntKUyQW;scT2Ed5Fdn-
zpGtuJ2x4CkK+J)I_@Rn-)%2qjRO21*fz6tBzgF$VvnH@x*hH3t#ge8$<}3T>rh(()
z@%PEfQK}ok;RiB9bPwc?Vi21v#Xxm4i@Ms8r4yBd=jW?dzw=Yy@o9!q`V|%xRYxfW
zkDX><w|F=>!ov=v@0|i<KPHF`!e_2;R{i@{3as`KPq&(c)Dtz=?Lx{$*TTSV0;#bv
z;#HX;Xr=zLH5=^SWZ@ZLf6sozr*u;ovG)O_mi6BU<#tENdQK2OK`cR4y=9^5++~on
zC_rK$?Eh;e*iNrw+m(3Om#ey)6sm3&>Q#{VQK<CGI7sDrP^98sImkNKpD)Ul*=!_~
zKeU)Ar(9KnyK8qqtnw=MP2f1Ld<)6jcX_8Mw%jgNk|_xRyY<w|2TI$c4b=i7dQ|on
zTvt_8VgQG&y5lZ|sHwbQy%iP0U^B1rD1gJteBWX<{x6QIGwzmw`3qNF2D=sHZV>)C
z(GhGmh;8*UOTq1#hSH_x62%3~kh5Vy;{Q?&z-GFff!vRNPjM~SE|44u^H0oBw#nYD
zx^?MY)s>g#E2kU~P`u~>x!--Q6JkvbNG&q{R`L^UuA1F)m35Bml)fIwSNvH7IqN;o
zjuGsxes{>3UvDC2gZWz@v#We`RaQ>37XZiGj>=q>No;q)?r9A%RMJ`h8q5aS<9KX`
z3for5K6?-!hO0V^70h!wROc^(oLdhQ-{b8M4jZoDU0`>C#Mi9oQ_km_q^6xeUtzb)
zHraX0tJIiRBi5re$vy?A>2Hr#Do?nz8?NuT2&AmM9p|8YefJ-@n?Po|+jWB7zU)ku
z+B3D!U_Ek@T`G64W`WrtHIm^wz-+z8TUGY#FjTW-M4YRfQD+X;f8tgo*lv(|5Dx9U
zsupzPt7g~3W)1Ie3$^<C+rj<+_5Dm*-269Ggr=-F53PHqUMR$;WpALb)ZQHpTK})w
zc+6IvKj62rfET+KSD%~?X#79*O}XN&@>V66Z*xqNKWtPz;yzzL?a)V)c8@@nC5(aQ
z34M3Xn7dx8P8Q@+XH9!xZok7^?LyOT{ahhy6@wmTv*4|2=AiNa$BA_s4i9vdW=VA@
z-;F(|J?*o%=7k$<+EvPG>gK-=n)Rqhm@KNXQE&R5X}<i^GsBe+<h8bNRGJsvXHYq6
zw@Go+5jBnb@*B)0<1ZTKw3?aE<(#Hrf6US70Q-6E%dyUy8hdssY+%>c))16do)aOW
z@S4-gbfL>mO<#Ut^|XKt^RH+8RPtv()O!5qw|dCeb7~4ZBUC~kCuo?j5Hxe1R;=lK
zzCz{px^-$yX<6zQE1Oi$Tk@(oUY69zIm@s4!#7(aIYnDJT8Lf4<6)9!U|_5omokUG
zQY5qTeZe$?YpIqNE1s1patN3!fX@Fjw3}+cvigj=In#9WX;G#si!#nAzbVjAD~(Um
ze>Qi9v6kpN%{hkMS|8%uP0qdFuDpqDuQsUuFK1^|kDSzPc7ajCBA;uSTFmAd;JnIQ
z$ZvAiaHC0M)PIv1{_<+;*_T=H`|nl%d9F;YV_lNDe+{R~tpG8zas?NanFppDE2ea7
zUEZXp^}xbY{khdq!}lt?jX(Xpu0H!ko3`m)E%mz^XOzFyu&JJy*<^O4k6nvH^Mg6}
z!e-+JBRyShH$^Rutq;^EMIKTs6kBC}edh;-Y?UUprRlONp!NUHcv?Z@|0;56siv3j
z?or?Qp+n2wq~FZoXAfxnU$N`nXY)d*1{3$nwQ7mI4r<9O|0)P_N14su{lu`gsZrDU
z^A%;#`G4{?OEuV8)|#C*-K3GTzsHPSZH6}J{2yV761BX4!72h3d8&76rPPbNXPbk@
z|Gjs0g4X}5wycy<f4=^|qHM-R)sjsu>P$Zc)wlGzD=}{nQLWA8P|ntt*Z=LQsB+mm
z&+NO1isFF>Pc^<T_^c*e?q=|OmVwHSGA*^lf{P}shcwLgooX|<bY!B|hY8MVT1$8}
zI1>UibqqHe-w!*fmc3z_+P4?Zs=Jp;sQI((G6jwQt8}Uxn?_zRJ)3!4{q7<@^E5$q
z&7fm@RDX*)8XUV*VeXMEuM&}+ue@^AY{eI$w^R!j+bZ?4Zvc;l{m!t}4Pj%_%=p)$
zdA03`rr%UE^YZg+)z!*2s7Tito1H(fS}kJZW;L@PAI#r#sw;nSaxz(UL`AYK*Go0-
zgpArnId#zZzx>vXyA^-6H>)cwOVgO}y+L*Vsa|u*jUUuLeTW2)AuXv{t0MogN;`3S
zm)81cP3jgsW*Ua;)|owdsiduc;-HeE<~BniF$Z(@-ig}ZZJ#TAzuPCN)3!?W&Ic{+
zs~e`t)~HWah4%jyKHA?9KL1HfwZp(k<=1&bl|qwTo#)rMG)oR&(AvT(siOMJN=I<^
zBhdK2@=1|!<x5IyOiRVSYH^$~*IZfHq`YO@O!fFDt4%)q;a6MoXdmeOKgIR4=BU{Q
zyXvo~QPI|y=ux(u_erB?%L$VyJJ*;$WH;09m0?o<Wcxyo`Ky82<~5!=$IdL!x+u3-
z_09=i9lyOy>eX#;)j6EF)dh|2$v)+ck`K6aS7Ysz=c;#)2ZPJ(usv;RD<&V;z94%M
z-2Zo9)uV0{yh>a8ajV%Lzwa9F<2p6vgX+vUyRCIJ{<Z7;-&$n8L_J4y&dhX;$U-$u
zqbrOWk2h>q-n)3Q`Gq7k^B&Xb>bLh5nKtZfQkytqy7u9R;Y#7B=IfpneP`@gFQl^F
zz}kXmJ)7zK=lMz-M0He~C8YE>=$2{;zSmG+^iD=q=)phz%A<|yM;8c~{!i-Fd{94M
z!`3%I?dv5^vq$@+jjmo3)ljq%Q=etFPi02`1oc<G50z@(9oD(N=%;x^9fP(C$3`tN
zLuqqnwU_ES6K87{L>ZVpVqOJW|F0@^EJ0N!)>GB)<tL4g0xu2utVPYr>Qyvfo(t4{
zGd09)4quPP%uD*}9o7}bA&bS#++=$-%>6mdug1<%Uwya9VDemL#aFT`G@jb)soQPQ
zF+ciXxk{-buZ54Gy2YYvt_D%lCz~oBVliK%7^J^(Rk_CF(g5vOt=rAtnAw|ulrLA4
z{BEJKiOtz`#|#}M*Ryv`a~4lfm+?Dk_IGxV{w1AW(EY!PO3S~3$AVZ|_%uP||B4+C
zRQnl4v~8HQREw54nO}LfS}E_`1obU%zMAJ~C}_=%*`(4QZKL{a#SJyjLT|NX-!jc7
zUUDk@a<Y2Mqvn|%Rq0eY`6JgXbTym#3d;>Dn<xEIYdrJEBueU>+N_4%>X%byDYeXT
zRJAec)wo-qqN(RGNz-qhx5|dA+f-)nsR!TxbC=mqdGSGijfJ|*Y9=deO%46tn%DIU
zYRq5XteJLkw}!cWy&?bnN^`NqnQELjHO-fu?a|~r_Eqhh(Khv?{T1q4Bp;|hDo@cY
z&|aps`XsCJ@+Es!w}?cly<b)&oBaHQ>e6s-bGuiWD)$`C%{@QQS2lRfr1Fg=*RUs@
z--096PPyxCr;3TrPWio==e0j~H)(PSd{xZ|Tc;UtvP@0-mzK$y7)#?>{?gk1vvsuN
z_Z>3*_1a7$G+>F@$FS|@_BRVO!!{XMa2rLdTt3dM23r5$ynTaO|LU763dbS)A0A!k
zSKWGTlBU!l1r4_Ae>5AePXq5IHrhW$>)iAjRb72E^EiG{b(wolH0%xZl`FRRgZBT+
z$yYF_=Lt5dxJ}5?^bCFr+W!x3S9q_PrZQ(i3wZy3OV(XIo%d5sK>PpFHa;|SW~nxN
zqv@ygc&>!nzCCH?+7C5U>~opHWuE7$N2b~OvqAg+<v*CHX}sAlp<=O;LFL1|)mp24
zg|yag+@y8EyHBN8WP+K($9fa>rY?<R0cXr+9A`4r|HY~`yI$UWGH08zquy?nua}C{
zOX8)?D~x>%c-CAq>xw<8ZnC$^P-d^HcF|4+@EFSW8JgfR%*md66fAYDO!QT{G;W_b
zuI8C~(adezG1YB%w`qB0&QpIPus~&rOrzvtjaE(DCreGM!ws}VCT>=%Rh_JSPm)Xh
z^;$WVpX(Hqt`*NwXEJZoP`e+gxqRI$Irme$G%mM4((KepRm<RE({to2Ry9er)%SX~
z#++GCTz;LYjcm1ntzu6|l75h*nELzaC(SHsE-QU<T%g(=*{2@6^S$1rf+i!enbw+i
zGhDTb7!yo{A{Hq<@mvLVKWjIadQsXpQ~v5n=8v8`s1=(*`b;-%qK&uToo>?Bu+ilF
zoDSuw9UILjp17r^RWG0#mBL_t<Cm-2yVUvM{r{!f6OA5ag=v<3@z*+7e^>3wvi*ir
zA8;G*l#Nw)yrQCA|BMS%|10volU3bV7HPUZTv4m~L%R8%M;DDYHu-7$eVL&pzP3dz
z{jk3p!{Nu^@hhd-MoOoRpM&=Qt8nkwtsby>i*oAKwI)?Zgf!}B&D1Jd+GsZ6fHbK7
zSD5!8%AEJld*ctmma5)c@2P{@qd%4?n!3HnH00fPL_@Ffz0&jdEOO_Iuc|*S5jMME
zzg<IqK9||+AaCu2W+4?Rc1~~`+3LV7)mqlWYB|<2=H;^|D;{kTS6{lHLCrp5uDbj@
zHE_E~qgYKnCA?A1Mq!DPp>c-Ft54ag53iZ)?U}Eq_Hd@X+4qbNrKe)wHJ)=dX-t^2
z2)zIQ)U*V(pUV?WSG=_}jajE<@V5Dh<^zUeHM2Gyb={}jT2%=<j990%s8x0CR%`yt
zt!l41Pxbir4)Ff`v+u=>H(gdU6MV2!?Zv{qW<fJfXx@}xrV^dkqrXn~q*-=Wo#K3N
zPH_Eyb=72Lo2G+`zUqG9yui6)tImcSn>5#clF`&!8LlM}*=CkL)l=Q~$$G`hLYu+n
zuTfg8wjkrE*_v0Ym5L`Tfam`~;n#a_z9P@$F7=D+3c&esQKzYzS#++tN0XGYoTRw<
z{H+#h&eNxZ(>`eaKjrB>?VQ~gG_^h`s|PojX;?=TnKA2GX<cDlr=XfY#W3m3HnWgv
z-P${u#X;x)i3vMYsq~fJ&;r>to#mJ^==^`!{{OGMZxpSn*Mrahl{MU<U3x@JW2<qx
z))c)2Who|UZJ*C;K=r@U1&cn_Teg{|C+8i~n!lJ)12q4y8GcF4DwN+OBKLzb%M*Lh
z{(l7?M<ccA&sXZ*aJ-{+Ao8Qi)ucd;z$P1$*HNp@jJdqDzVC@rHGDZ$cU!QzDs$IM
zE$?;RT6c~-Q_WlbUi&oHU)7p=Db4LWh18c#oGWR4Z;^b2371Bpk&>$9nkV4C(Dzj{
z)Z*P(wR1kEg6e<8=%z{PN51dZ+AY^?y24jLW5T~+jq<IVO?#STwfAm1uf1nyznM+7
zsMfR7mFiy?KhwDG)~x>f#VwVxQ%dF=ckDE4C}LD=YP)KpTwtsA)9$L);fLX>ZyE0C
z)Xe^F<e0WuRVakR+)HA)$@}N|%9s0}t98q^=sk*((u_~#QG0WDmck|hbN$7~V>Ko%
zcxLi(%OlM$KNXGgS|2r6hubDrhPsB+zc1F9m;_mWuIl~`H2yFD?zgaxU6j1}9=#b_
zJmsQVhS6Kir1uu9&sp83S+^$2>`9f1#=9+Es$$;8YF`c7RS(LUXr6dxX87s+6Vu6z
z%$n?RnK~?v*G(^0*=aOBS)(S)XlCS=e8-gY61T?E8;)iVBVE;{_SAv)|0@*rWNY*<
z*{XKu?o_jlWj|GOkH(rCi5)fn`Rb;=Qdg<TuIDLcwz0SLLicB@|5wY=p3dTEKF_Al
zEPl7T^0T6e>T{Opn%?fLP^tf0V*2o*hkE1&IWw0WCVl1UYM}aG!BAWjT>l@5iUW=R
zD?TjnQ!n73tmV-ESJ`)6BV_;o6LWRnGF9^%4y!d8&*!S1uSiq%HCm|FI%$>CL4F&}
zOF_EIjh@wdVK<x1s{Z?`_{kqLJuQB~G{KrpValw(s!1<rn>bGpRl64_p$<C#ud>}s
zc}o_fW-v#whNco1I9*?RG(oNL^l3BD{ePhI|F$HDX$CH6Q`=q0Zu+WR!F;=%vbz2O
z8O=5O)@n}qSZg5R9cLD*Y@^nHW3}16)0&#!x&Eu2y5+8(6Wyks!rG{DtZItJnh!Nv
zNw1j{)x0{@?u7<`?*CIvNj6Y3b*VP9dzGoy_<BCL{48^ARgv^KWZ<*N+PufuQWaGH
zgYw!O<HuTFd3~A{Y0dJizLr{M8&<0qb*(ZkpPgr9x$CXg{TCCpj>tVSsW(`!&Z3`h
zIx|GTe4Xw+&HPYz^Jg7dDxmZKg^$>%1c^>mOB7C4{nW<;9{+#wCmUSug6sfct|be>
zZDSA{giBi98O+R@3@$@K{I?(F)f7}b!1;&KF$yjYl7r#bst3Vq4_Ra#lzFAp!E2X!
zZyD%jPGZtOQ`rR8`*6LX(UGXdYST|WR60;{NyTz`sQ#bdI^g+Fm^mQYn4wwqe#9B@
zI1Y%v+4PuBf8cX)`E@_9OJ7w_NKbR$Tjd)Cvfw!+keb^UlvO>Zsp*$GXsL6#D;eG|
z<56Q;-KV{b$xQu7{#vm8+jq!-X^{Fwg`D7Wx25f(>iW}VU^Yk`88>aZqf(g`2ktk4
z#QoK6)p<8>0FSqT_#iBKTN`X<eZd9g!<K#EaW=7SW?-?csp??0JEV`G_%Z3>Uhq11
z=7m!YPqRYy7HTJEgY_)8VgiSw>>C$t_w#;meP`dYg3V%)ZPb4z!T@fsf$Rt2&kc`s
zP0a4<t&A`Nhr8?lh2U^eS<I_`@WVZ@*h$?!u%4yn)4+U?J{_Y56^HmYI&aP0!FGbg
zBvM(yY2nqo1*&Cr_rc<+HB$|4+Md?E+%61{`_KXb)#)n&4f`_M6^~^g#%QZQepHRP
z;;UM-YdSpK8l5Bcf{KdNdj5Y=@V}R7*c`k7E(aR_Z!7iC`E#rlY**66e6St%PnFdr
zk8KD0|Iv(m{p7>v!D)B@e-3!stvnd6Ji}!nSYPW^d9~FYO=>4qnZf!6re>=-&z=GH
z3&>ySSmzWYN}Q|ZAoeP`PXDicL+P{r&&<nuW;*-S^0Qb~K3rx5o28aH5$<-7nIbGJ
z)!yewg2i5Xl&bh7I_bPfHIaXPV1b^RN(4AgJzwuq-67Vg`}EIou-icT2;olGS#bAN
zPdKNqYUv2}_rAhOU~whhER79GJHg}s@`)S3_JGWUjsJt#v0{R1!C{c|WkGxp7SnN7
z<yzMbR}T`?zo`TcL(up?_rC<iS;wRdj?QR>=Rc4d5C)C^Pg-G!qW8lE$e1=rUhx)W
ze<+9z!f%^6)m2^#tJZT?f&CqG{H1}{VMcHsxxXK>-(rhIJ2+i-EO!Q{OQV&DJ10SA
zv<DWecfS<Zi$B(HnD;7QHAQ*8>P`EjiaaY5)V`ZEtEWBWQ#@)oA8bCmK{z;$^h7Z4
zdjpvT!#?LcRL|{&+&5W$XuI0GJUR9DTT<X~1*vJ&Wi!~A{#$A3ghcgSCw76wtL8j{
z`+KXvZ>65U`}En0Ex_q>{e#uu_y_3)VM(r4$`d>qb!L2=r0i(wqAwVF1#Ctdhc(!J
zI#SaV&vq_RN$=SJ=7aPY-CPXzzwUtya6F5$s_8D<B8y@!NF8YWpZ!INifrr!uzHYK
znfo`e`5^YLrR-pKD6^be)ZT|m?`A{xQ72{|)B5K-P22Ns54in&|7a#y9Z1jHJ;I7t
z{*>xD9I#YPj1&f&+qp^r?lzE`>)TI*#h;d28GYN~Zg4bPO>U(px30mL_i%Gg*yJf^
zw<Q{`OAiIB1?khC8Vi<RZ4nO+EA`|~z0*8jRVPYF>Q^5KQfM!L?B8XL2!@9*NFNAC
z-F*et17c5OI-&Q_*hN)PcL&)0e|?p~G)NAFV@>3hF10s+?Ok_mn!fbH^#*(9DXW;9
zII8K)P0(eQn4t2p`w5C3kUkK$5PAwuKVfB+aQi^wApA9Po8Hso-QY9|;)C$MZJPSO
z*Vlo|*vSH`z;=Sfy+0%JVbtDZYWMtZVzux1-IHK@)N7daJ3m790zSR#qT!y?1{MdI
z1;YDHj)KEP^(JH<03-&&Uwsh!#tME3f%R-LlQgJ!2njDMu^(XfiWD}0^*0La1m`)B
zUJx$1{tYZHSO;0FbtwXJeu>)o9jf!TODZfn$f0JizDk`*R7(wX|KBZMHnoEPmsQP@
z`qi4f>r_Byc)mnEw=j?UIyk+A71gVKkei}5ziO+pAZv=oO5JRYi)P$vt$K&lOi#{I
zn_Z=$<|8#%t?pBj%6(H0jXVi0aJddL6NaN=F01B6F9y5u_)SP2$yuBV56gvb1vU21
z?N&F}Z&q_Y4>=1nlyisrl!r}9*V;4Hvf0uV{VOyy3}X7!y22N!yPYZkn`LANxi{3_
z&J!FSQfnhL_^<Y;{olD4>~@g-y^7M{{0Cy6IyF^=J0oBH)^W&w&zhohYU_hus#R$k
zfYnE2cdCco)&sjg;oSu#kRA~JxX(pRP25DWE$ojHvtyge>z9Z<vLJaF&XhD&UXsuV
z4$mFYB5Ip1>{ngj!K*HH#X+V0oDSH{do&Bc^*(6)-+rgJT9k9Ka^-~rRda2`e(&=s
zh_isWckEXAv?x^l+Jxn5YyFzkUiSR}n=|R~PB3l1h)-ELB^~TX^Ua&pw%$`#J=Nn4
z_7BKi3-S(w)wSA$s7`$i+4Bw(7o8oUDx3kiYu8GzRej|h8MWd^pVUC({}(MFXBC0g
z|7|!ds-buYvcDXp4}_g&ZdN@~%meEGEB+P~R*l-oqoF1Z*}Iusm8f`%<Dlw7BZwJS
z9*Tp*<GH`C#^b+`^W0Bo+o`;7y9IWaFC*kEC0or?%Jah(sCECbR}<n_Q@*lLMg1r1
z71iks!C=1#Tz;y$qZi_LkQ-n)?EPx6`4>5-sS1SNfY)&`sT^SQO(suPZr&=dCMU|M
zYA!FXUX)m;?rh($+QqUM9A?*kv8YGdBEsd#`}eAgCL`8XR9t9MGb{{H|B#!mB)1|R
zZ04<{|5W7f@vBepMBFjk&Djlhr|gO6U^m3c@TzfXO;q-rJXgJY36EMw>Km|L)kqon
z`ftu^jVB>vcQAjvlVk+X|6A<pQ(g1gRc-sc{b2WP-MAZU-Wto#>I)3_s($yeRqwcI
zt9J6i1#p_@vDmM&u^V#c2Fr3towoTGq^t+k{~2NHlsDC8sm<9n4{RRDJ|`C!uv<an
z|8BzDz-*8>2!qD|K{N=<Z4-m*0rA(r?N=+?R;UJ>|91(Tpk}8d0M0L<@qdt-r5o<4
zF4pZ<Sr&L6>|aH(<EnzrZ@_W%Y~L~EX_p!__P0UiG-cda)Hi<Gth#99BlS6_B-OaG
zXKV1UQqh>NQL1W@*{Cifdsp?3&~p{*3P?T&nGeE6Irb=S*8lFM8Wms-cI#SBL3I_4
z2kJ8=EYu8rJHc-Ad|jy~vhS6e!Z{W7xek8nFIdmWuYLkKw}5SZnYtuH1URq9-SAY_
z+L)!d;pG*zzc-k`?g!b&*t$ct-A7pU;_8b^NooI7ckH!OxMqJ<f%PtAeMH~sooaX9
zepCaE{~y~BqWVE~iK;_jmHId4OKR_Kn`_Kn<f*!<S_yRikJ7$Z3aVz$A!#0DHmLr;
zZ15dym+eVNd&leztGWhnr25<G*6OmMTx$9<IvVB%XH?G!Ur_25jaJp%2T4nHu{+hg
zrYfjSo#rpUBn)!@wRY`NwQ!jrr2zXy8ja~(8rB8}!Qm1v=mQQ5ke%n6*Qo66=>_wX
zib~a{E`F#s*@jpBR)?}$<yuEIf1P}=dmMK_*FveqsvqfWRuxsb12zj}MszTPT1p|S
zx{y<_nub${+O#W|K<ED`ewYDij|8on3N|047KUTIA!VupCuHr5S*M74>sCe;x4k7Q
zf6BtacHVddNvB7cO~L%MeZnf1U2|1ABd@CU9N|~izLuk=R2`<`nGvAYIKvg}9?;#J
zS)WS5X$)kSx^KJ61Gx@xISb;$@ZQp?s$1?((l}$?3)Tk{w>|?oH_P<YKUIzD^XlAP
zsmi-uwy7$8F;+ibvrf%(r@Wf}j>YOMT^ewGjir!#V~WqRt3FbOoK*la@AN8t6^jK2
zlrAhVQM;V_1Fjw<*XX$s%#Y3bqxw$?vQFS*w7a^)uP1P~w=ebpt8rZFp`!OUTXnmP
zhWe5R%F^~LBV<_EMASg{|AqeAg_8b3W`pn)Lk)14i6wV~%ZyXZg36%!U-^}#N=()@
zWu7~cDxmuRAfFT1zW0yB;O3y$|Ew4Pg2PXIvAC*wrkCmspTkP)Be$#S_(JYoc^<;3
zbfyJ*j@XiZuz9y<<*BUPqXRb+<R%a<I4-T)xavAs&5~xwS}WDfh_y<dy6mdw9A#Da
zKG>^hHWzY7bbatOl@+m&`(#08%#Qt_vOl(3<-s*o6}{4H;QR^_2Vs6iJ@7c`nd2)}
zH+DZ(k^9dBmoLp>0NV>vcl`Ne#YM4xD#j&<`yhDYf2thNc?k|Lt)6z(7ZIrIp+Nek
zMKh~_>VFWssG~&DaTVkoB@iDUzWlX8b^iWjRi}Mr3TKbygTv)(&rPsfLHg`Z>#9!7
zj8r_v^;-E{avM0jLE`B6+PgFrcF$#CH@>?DIU^XRro)t9C8whh>|e{%PgN8Rb(FKV
zodwqoAT`g|BhFM>e~uUI29UVTPDAB|{}{mh*#hy3Nu8`JF`?VRe2_c{uhVu^Hh%`$
zI|<^$aL1b83ZXSDV6`7UO;&FHnXPKEIu-7IArVwJg7m5P)q>T7*dY9;0dj^3hz-Ko
zd(>1uPF|(5>U}=gFFJEifa6W!3Su4R8D2;jg4DzCZf?kVl6$tPE5~f9gomwj?NPA)
zyG_yx!fR|*g{GD(*YKZFVG{3B{hAxBD1Fu$Y!8=5it38_kopN`=H-TZuwD>9Ext-Q
zwCszLsLCB>XRRc#_`S4WN*9Y-z-*8j5a!u;TLwhK@Z>H%m8Nss6&@P3DA!)xr{u2y
zIqwft|1%rELeUG;gG{r2`h=pU)leJEkFx)+8h+Pa1y=uq^qj0|0f&9*tPXG(zmk;)
z`wb*_=;kai`{s7UzCMsRF?iul%yWV|1-ilR0O`MB^;Ts|-Eue|BnHDuS<}I8Vqey(
zyh0ChzPZDuWs1*S<<z#C34+~pvOQ15QE7_8GtNHMS2j1~`?qzfNk8UL<(V~2xgtJ6
zW%YvHipKLME4i67fYrmyj`Y}$!oTuxx&jX;<P0p2cWtVkIgeF$Jh-4N=?rNHIsELA
z-BsA4to<%o_1>`(mG!zN@-IsK!0v8e6r~dR+zy;>49<6f&24B`R4upwSvL!^lYQ=9
z6`^Fv*dK`h?Lw+DsQw4BLHOKG3FRWypJ2Vb8ve@4buP-2T^6WlZ8lQglk-*es=1ib
ztmQ6X`%b(}RXq9#QeJ}ef^gxr|KPA<E>Hx=U%f7q>eW{|svbU?D!%#rN@u=7&R4VD
z7YCM)ei08g>!iqe<)4@KDue2O$Fw_OGwvOO)Ft1Qg4K!xAE~Zlhm02#uS3LJC@YIx
z*H^@uip}8{m7V(Mfc2lgFRl7^=5;Xp)bvg;t@E{1<@YyJ#g^rF!1jXNJW*0zdA+>9
zYKzB06+ia`<)lUHz-|M{f$+Cmkp3fx4Z|uA7{KOrOb!66{TuNY9PZ&I?aE$0%fM#4
zzu^bdT!DRH`c1@aaJYWef}FQg@G?=^{fnKFY3NynPq~M{YSw*XRBpe&8>|jy*PT<C
zV@Anhswz4{j;i}i4}$H7sV)8c2F!oVG*M;qBFK7O`TsYRk7?+u*6n<vyg;{6*)HBi
zwM|P5?0%5G@T-t<RhHU$Dl5`a+Zv$ze{=Sis9fmI0h<d_55h8En&5oFzeQlT^Jnf;
z{dyR2o>hGGcCa`|JqT|)+^Q<QAxNRhW+wRVfn}$bE3&Rf+-(9<^Yq?ol}~%-Yi`{3
zL;ZG2m{y=aWPIb2*k-NRm&vLd6$Q;F@?TdwZYZf~Z=kO{k0n{{_@6#SvGu#us?r`S
zwkY>&Zoez3bMtivI6k<$FQ_!c9XA4<|F6B3OFwd_BzXQ`h_}N`+}GC3B7d@S#L-CA
zZ?Y`r6Y8Q=OIG~U1MUCsn7hyPsiY2Q{$I{_R+<K=%o3E7R1sf%SBw9mrPlhaFIw`G
zmMQ0K5HvGYTVuSEb*sj^KjCKI%O@MO{Q9D4U$D^ZWSE!ozuYq_W^Nkl4_<FG`~6VB
zz<6)C*`2x`^^`eThQfAX+J^6)z;ir3G0EULgx7UnWDfmyGtszRrg1^_h#JeiJhQ?L
zKUHtmoYMLb60AOB@<bK8z6PoG`>Qm6`OBD^2S;nY=e?+AvbI{~j@D5%{yX~0p!xqy
z;U@KN4I2$c&pu7i{{NoL0ySm>4y|cZ3)C!UTIv1Dn628$P^m9;($)OAGp{`K{C`Eq
z-IDs7?B$j08C%S5-t$w+N#CaGWV=({d%cN%;7tRgYKc<KwNGobwom3ZaZ}(><jQ{z
zcE8IsPIXK5P}3cIP0dx#KT(Z28V;_v51gnno^;&8#KF<tWcdOO<)0VA%&Sgxs@+&5
zsM`Brl3D7+Dz#NtGQjiyz3fYk7ETn>dg|P&#Zdl2ZBj{qp|HqnqnY=W)idmqwF`D`
z0H6PFwZvc9GQY$$hS5;V>ou>r_kr0)?0?#|ooWtfRbAFq%Q>2%rm=TBcwRb9w@~R)
z6lDG1BjtMV`M;|#b(_?EQqo{fc&wFnO4jUW2?wbEuP`xH+5D@3pz*_&1{ED87LDYp
zM7bX?s!aoLdK;=Yx@by8+bLvpPnFxLw^+R(UBFZ+qd~)Gex>Q9{aRWH%|a@3qE>?Q
zlwI>v)gvCW)tYk4!E<OV_g|}hyLv$N!<+r;w`B}fTo)`<URBSjo>I`Eww-yl(tq=6
zl~%!CRjI{Ide8p*sd={)n6?{aDs|@RY0kW|RzvMtK6w5=T*Xsu&bvLv7KR$867Rq0
zr?Yctb@ybe&0~A1c0gJ~>!axlqm8}oYDKTksi_(%t8RI4SarUfEI1ry`Fj{y?zS{#
z3#(I`T%u?e)cIR;@s<V^NBL9w4z1Z{Wz0L|LG%Al;<%Kmw>v0-=Km*Jg@DILR~B5<
zG0VTMc_h+A^PhXO*5i2sW>z=5)IszA?YV1BUjBAh$(guR&7qgkJle5a@$O14`22qp
zBa4d69v$^Ijy>S~c#Ab!t#P`Yy43Xf${eql%!91-)pFd9gVR1}{{Kh#QSHm@6Ey=>
zzo}X5Nmbv^^3>GXx=>61#6iWgm*fl^%#zKV4OeMDR<H-p{~M|BseE%50P8=VzXqJ<
zzCq9b2krlVct=U4+%7}mndLj}4TAO>D|SuPVzUfX6cKUO?&vH7_5YRHj$cx(IJVy8
zQF*Ucj*u~U{{OC%jQU#kg~ls`?kESP_!>WdB&yhYH&5+o$RoW!pMPq7$_-TISS+id
z5@T(0hRe+CDaS!g)_8VRlkf|=GSA#pCngAM%NT9cdRIC@HD%FtEtXS%R7;iYHH{5k
zs)6SJAG}~zIF#k9aj23<^#Jr->k<hj^;ZkBw3g1YR-YUmp}au4SABiYPpvgqf=!vN
zkEjb>?9r&$Gtsnjy|=bd6pPNWdDl$;y)o08eg260RB0AXuL<wemKrTn@wq&~EYU60
zZ0pU7s+vxwCe3A9YC+d@v_H)0P-SoB)qV2V+IVa7bJhIlPiDWh1HkkDlFKF3?dNRP
z+ws6n)1-=5^}7gz3X@>7zJFDR#*2G%joXu%wHoieRDZiiN{#(-no0dbJHs23w`#DT
zn5L#LqN(!rv#REG>(dHOi!F3w!*-f&+PhUN{(z#^{<kyDYSY}+l_HjE&MR#+G28b;
zUD<lO(#AJsY9AbytJZBu)-+kKVrU#R*Cb3yPO~t&N9US~t*Q0wB#kWwv(y&Xg&6%}
zt}*2<eWae)was*YkE{CO#Hsq#`4bgvd)qaNzn)OzpZMDJFCUZI*ZVun^e0xC_urkU
zKVRdMNxaq`Qx%6@dSAaRR=-#`LwiTtH#4L8zTo-)soM?Ir81mMWZmbge2-mjqU>Ox
z?xMNU)Gp9nKj2Ctc>lk6uoHOxzk@*mJpcduaF4q8m)%-ZMK3D9$`l6A|645MQ?E@t
zV0Q4=dCmC=DXQ<zDyiQ67Nzzq^0N}7RkCKQu&MGo({`PQ?$Tz9&kL$-%C#{SxiH!E
z<VrD>TPt+cLLN*sNo}@J(_VF24K)9Msg7Uy`8sjUHL`pfHR-D0vQ;OuLv2&2B>4P)
zg{LQ#WT$s&F0)ytw#ALlbjGBkX1#_6>ZYpZn)jaF)>K^UZm=M(%gjYaLG55@u32i4
zou*OS5w&$WhU#A}`qbz6_iAXg$ZD=zFkee>Zk>#k#tpUVU~x5{*Yg#PbMLF>mR~ZJ
zH=M6loL^&>>_1xtH2<$&w%x!>akE*ocoKa6Urbh5d(QrKnr46GRq~BWw4Ptsr9R19
z+xUX9htZ6yGTN`#Ez~;sS<*DDvQz!tZZ%Vp_UmSM{W!Js%y*i_lnJOPPgGX|&Hra?
z^;dIW+pL;-R0=%Ca*hMCPh&He4VZ73XAEv<Y|dS&HgCJ6qJ&tqn(e+Sb)K(NLF505
zi+um79`)a>#6FQ(okw;Lxcv2e3F#+*+7lq0xd<{gY`=G_+C^hQb<q01eavkd=QgKl
zIPN;4+&-gMjmtz$Ey}W7VNxcO`l8rx3IVwW8ZT2cz-|DU0mB8nTUEcBLgsn9S3%lr
z{tp|#ZUm|6ydkgQr!!UE#$vu&zyW=*d%V<ks23b$Rz7rgGPvLS`h${&27|bo?a5^I
zD>2UCagH6oA!lxC-_HP-lg3+1G~yC&t6A|P&evQU;|Cs30oikxL0V~EV2FCf<5gh&
zl8kvO_6sD{C9NTQ233BXRBxJD2zLL>Cd3{K>D_s%YT_nJg};6(R#gb9HhhB2!+^|0
z#)ooGDgQ`;?Ac+d^Hk<`JFoWe!XGvL3);%bZ%?T;*1VBVWkK92)qnh`+O$b6%Aaj4
z)#|F6!EOSXXWWqmZv##;UZTR~8LG~kc26yn_n?AB=s~bL&6T^sbcyI%CG}V}upey}
zOR0<WZc%Ydn+Xrw+sk;sa$?V&m3KnV*aq3b_D)r0p(A9E-1a5r>TAEWsJTvsjBia*
zg52*0TK~W495*;T{!~<f;{~LbL)BU})RP%3{_j+Ys*;tW#^TkGvp7NZzZ-vxYV<$I
z*!aCoSHbc3Mp;3luPy=XujwaOD_-rB0=w(W#s^??*QGV7cGf>sJNZc+JpS(`V5`1N
znoV9vuN>@`GY@JMITzT1^?=*}!@jB}VEImachzX`Phh)1a$nXR1KU;mWu>w}@H^F{
zrT10VXX>l-T<%n7(-Kv4{M8RO>sg?i`d0_YUNDe8kzM{OuEK~jIQL72simxPSKp$m
zqw=cC9USH#h5A$@oW#`Y9w>v|4$`;j1mq5><)<Oz|HTh`)pS%+mFH<Ts(U|lQ!^1r
z2HU3^DWkv;ucelF*cGl9r2mUxG`Jpa%k@&K(?6-~HT4WwJxC6Om4Dn(KPz3XoUXoF
zosqK%H2x1x^Xo3g$h&Fm1BZdSy&hQ411HEi$+9ML3I<$xs<(p=sA|Nwf#pDEA&vjb
zRM}5JamQ63$XEhMUF(whVE=;HAiSSbS<&5eJ6!DOqXTL`xOu_p03;8>#f1lzr-h2E
zPL#X|Rwo|5P}SpQ7}$LodT$jq4)th!GM)vt^Hlr`wPLFes<ndh8iE{eRAv_L(s(&h
zP2;4{Q`O%aD%AJLG^oyw{jY3$sT6D$$b2x4sRo+|W-~A_e7(Y^@@E(1&h8BlRMiF6
zGiq%9rKVPOq#W$FC8oM+x=Yy9ubp5}-*Y=&-Rxbs!hfD8U^Dt}7OBsFe;b@`a##8&
z`+7Z4)-g0zFFN4@x6^mMuIl>i2Pz*fGATvYNU0?)W>LLne^s%8Rax=Cj5I~Lqg&KK
zb+ZNg3k6x3cPdIur&S|gma4O9{#HBJ?xx`}k6BIl3uGSd&8F?DPc?7C&5qr9L?P?x
zF0~7*H54k<A$L4&U8<$-Fk_xN|7l_M`e;kF?#baAX2t!=+c!K?z4((+{>cX(^~U9=
z6*k8mQd>D$3^e|)aIbw0*gs0WuhgoRxF}88aYUo*o4Ll;xsW~L_N-RmumFYG&tM}}
z^(}M2{I#0OYC4a2)lZbNsLx_nS7T~Bt|stB6`WQ+?SPzNsd~;&eapli)kh7N)W1tg
zg45m7E$M1`E0Wahw#lfO^fs&Qy}<%DuVwEha5`2?S_I~U?1SOI8*hQbS)l-OPRPtK
zOLhJ&i<Q5>HCFZV+XxPm%2vod1?pb?VE&`SdMc~emZ^HT7^-LQu~u3c@1VAJqNG~6
zeXiR62*_Bu=H!<u>f6P^{`ddFp`4^XL*ZSwI9NZ-ZV;`SFQOWn(5CS)_7vEEATbaw
z@a|LJx^|j^TW6*k!^-pO55I0!UAb9I%{?t#-RbyRwHqtr)%4#jQ2(8;r*1HX5A25n
z{*ZMu6}jJ371vJzmo*@>HcYvxY<pQt+1T`=+A=@L`aF<02(zuY2(~x!wz%^3U7TR|
zsRV9PSN4RAQ-RdTFxY_A^hbY^dvldt`5r^Qy7Y!1MW<3Vg;Nnbm6n}auJkqoaz-&o
zJqTYqRj(*>)KBw$gsIwU@yS{ma|^)h1wIRiY5g@QRn={3HxF8IU&T@Hx(4X{pDx!p
zRi)ES3NHH>tClRAqOjNKtHzJ)4*2?i(EWdQCxuL0Pc$gn9M;nJXbdtrte~oVR?W<O
z%bQJRvpr@hetdsiLD_AydF_r~m9}seaG%F(%?4AJkAKV_eQj5mvdCL~@!#{xdskjj
zUST7lRrjn~vu>e>*1Q=Zs)fcp=3B3?G10VqseUsp)O^Fje#2=lhqRt+<eHzpWTz~7
z-bCeNn2&1kiV*YlYa<OieYnkYUbCs*pLf#m`^+g?6WN+HwOGv*A2r_vxApc-yDp!-
zfY~&(Y?k`^=L)LTX0qm!T+&o`h<0dAJf^QMQ11YapFgW!Xt>O0HFJ+?)BN*SN6p4U
zT>0;-r7CX@PgVr=|0f<>ull5*Pi@QVBN{907!*PG|E(}E)(lMLP<yj0UN1f8nabsd
zCHgrZL@d_E%1YO@ESKQbUZKEVAEUpo)K-1}tY~vkeB3=Nrn=JPmzw!yWBvTC%*Jd!
zv6>|o*;=z?icLV{|JN)1!0s-a9ibMNt!c);_PM#e`3+@<W@T{NsuPkl){luX>2_l@
zm9yEZ?3vnZ;VdVpCiU)${PLHMp#6VpYZ;${`>3Ut*BND9?a<1+zfg0wwzKNIf42-*
zUo<xE@Yt@gaa*kR4L@yF(EWc$pJyn_Xgo00jz6K<oA=N>ZZ)5=@rz4Z@g~Ar@o_5^
zN?wJkwp?!kyC38S(EWdJmdscFk!PYhZ?=eHl02Kq^Mw)`D>_`Y^uJq~slVg}r_&Ql
zmYKJCWSZ>e(NHeR4F!wU+AlN>I@N7BKh_=G7pS|#BKy#!K(%1WN3)KMHq8Q;471e}
zT)^x9SMv3O{cOUNuOg&&N@d}^G_V_dA3s(pT+%9?-7cX%w-K@zSoC;<s+>r;n(otm
zO4}=IROG_1s!ZP0qxWR-301AiezR{kT$G-ynxWx8@2mR4h}YmWIaP>Nt$MeHX{DdB
zspd;#gJ}u9noVn`tMaX@RaO4#u9^0Ez0uFh?5e?w6xC)HuTz;k<GbAazgNIv+a1kn
z+_g~L%t(8N$_lPUX2Gh8nmICaRP7d?(04J3HP=zoRQ#E{MyWW=Mlor1hth=&<%*84
z62SJB{4Up7IfYl#ZXSo$ZQnzhOsDsmod`Xn20H&o#Zv=p{<YK&HAW3<bNOXXDn8QU
z;61ybuwN)3tC()1rz)VL4lZk(4CPgobfUp&xH8<){P5{!l}XFZ!2NfS`Ujyw+7lx#
zY1W%eRdx9!tGQ;Un^`UEbkO~OiXT_aH9W>`WbR?#2-^P#E}NCTJQX_=dcbLLBe$>u
zX#d|D==pyj^Zq4Y0N2lamZsWoc3o4Sx#+5<UBGlzxs7SsjtNV_?f~8YC$)N(>Fwn8
z+Rm#)z-7Ih>toeZQ}axkpQ<S-A5=BHsdGW$+LvzC!#>;fWV2Ri`Lk%L{_wU_OZy>W
zlIOu_{{8JT?Tq|WD(`pQ)jhTHpVEos2(5_@&05`}E0jCBm9!?hm8-fwZq%qcF%eu=
z=6UJJD?d7*zQ4ItNoPhISd4p{mRj3=U2VrBkaddzbyw8Bb$Mz1ulixSzeY{9ZkfG?
zoxh-2Ow<Cc5C1c?^`3n&OFf#V8D8n7vFneD#&TgswccH;Ro=)gH8-BeVg7aYM76XH
zuS_yDI8|h~9?+U5d`4xG>nxqsp_7bm-%wR`YMx;3#{)V4=XFA)+Pk|m_0-x9Y3!Aq
z2oAFa@y+_GbqCaMFI;XqjagK4%1chQdWme+dd{DwtP9%>r~eYwIN`+ty8lo4xK*kK
zsQ+)>a7&xZZlZa#p`;e~mCKq^7p|K{H<YQ?rtH`F{A8us;@)qn7ar6rYF#N(HJ$!n
zr7Cl>hJTTfq2l6B(;IW#G_)rg>O8BSY?dH2PhGv}qROAd2Zm>!Xqr6~6jDpyEM{JP
zAV%HT?Et9%FTd}%ng*9^pPEvgrnz3(a@CDHnJj!i3tH@2aaq6P*>sb`8K2FZ_HWV4
zwR)!(v`$Fdzkjy5?%8c-EPW4^s#Y<owwCpP_W!ALFt0I9{T-$1<JxXECm~z^y|ov3
z{hYAm5pY_#p8~nRV}{N`HM!j<v{v!VP<Hw@!R-B#mGTEJ#i;t+T5c|W?7N1EGMg%o
zik0&2JHo1ADqKpFG$(8Po0O}x*GN=PLbKWIb;4;S;|bqPUH)G%V|*a1q&s7WN_^8I
zljdGw)tc77Dxm)V<=MxTwv?u5Y!lYd%Kq&HE^|5_)vJDF(+BPUQ@+dWr&MINO+!@P
zT5bIW8B<}C&E`sCdg>R2q%@x}2xuPLdfuQ#pWVFpd%nsy{pDuI*xNOPO4h0UalEI>
z$8=3?;Wsvole3O$=my`^WZnBm{!ECB+T$C4Rm7RrD)1&Js)To*2jBnqX6=3O*d<51
zmWoUylVL{VVe_uQJu0yE|2uEiX{m0#4i2AlkxMn_Sj4M4-efVkX4z*XT-~De+1pP`
zRy5TlGGe>>;+{ye1*Z$mzvr22Rt9rhM4#;hum1;~|D)#mT~#lZP4)C^$oYq$JUfa<
zLtr!nMnhmU1V%$(Gz3Og2!PK20r|L<;V!ss5$u%!ZW}+nyj$I3zqiVnNA~Jh{)%Zx
z>$Iwa=Kr-Lc7xCVQMH*1KK}=#*NsIV+^z?)K{)sES+E%&1H05+0+y<4>V8+5{X}20
zf=5adbpDTxfvx(+&7tahpYf}$oIFh(bpB5>1Dj@L76-Wf1u_GMB~)Ll#mX^&-DcsO
zs&?w@AI1J~#C=iQ4qVqLp5v#XEM=yydlxePc==(8hB8~fii@DD8V`$zQj6Akjdkwg
z>c<>ZG$e1#0QWPW`a#xQ^BjkaN!`d6(ad^Psb2by4J;3`|Ktiiu)QGmVV)9|*>+qS
z+6=8=`K~qV)Ouw$t7|(?1&<x|nx<%&zhncuA9Vf?NDl~0)vZ;N);^=Gc4msw@(WW`
zpDlo#n+%de#ukOQR6*zefW%%~UR1qhS)#5ZbWq*Df<dKAs71ZCCQ?b@Gi04kfkd7X
z)2l4?%iQZ!{-ialXY7HTmj}|@n6ChKABdgduwJ!yrn<&N-Ou3jf0$>@1k2?#LdLLE
zlyp`79U)`!UKVrJ=V$P$HP4645rFi^d|m`r4?6!xM@SdU28(B1Q=N4fGR}ADEvtqT
z?^QKhHz^I!`hU>*KOi+y)g`JW0*Aoo|Lj*r?6uUe@=^;4*$JBeSM<zTqn1DMg~lS@
z$Kdn=I{zn7bFSL@zYza`&i?_KRhjfeqs>GE+(%;ZU{c9^4Y|AeX>T!DAHT*i)x1p=
z>h`l5)IsO}IPQ7{KL2O+L&(~E(D^?Hz9QBiTPntY)2rfH$bCGmadm3Bvv0%K0PVS=
z2sT%j=eo-Kr9$eHE*Pp8ah_A(Q(>VYxnrYRMX)6}UF=)LsG*e#xi=H!PUAgNYF8&H
z!^0`6%tW2tmr=v<b%2^hDKA)j<t;6>hY}amLFfN~^nmco@{8a!^x*<zADnUYY4vYe
z^Herm_R%<Ztz6ysI3$ih^Z%^JgVaIi|A6$t@XzJ5l~y60{}X3Y0(Kv4{eP6&WbpYv
z6+hH88vm)Q&tiHFwhwgv&y_&PeAX`RhhX=0Z-lIcmo<@7Og?@@Ei}JZO?tm8*gTMZ
zE+*%}dO_>|LFfN~#9$bC{eOXdB-lKd+^rqOY9W@U>ag|ymwNW8ADFTcY%Xm5zu1<Y
zs?$Qn)h_OX?0Yjh|5+`~><>6DjP-Y^1cjJro?CSr9Cy1J7OT&f)lrX(J*IJfX_Q)@
zY_Vo@<ui>wK7BR5wQd@h?(R_I%nw%OyN0@6JHUhoY#z*Ao*4nEc`hnocZe4-YLsqT
zt|9izUVXnb<Q(tFuu^rWf0xxgzj<nW-!7uj_&rJWapPOC8682Q8j>##gVUuQ<2lto
zPDj)}C7x3E-gyHo53<8#YO~tJNBwGfj0cqhd*sx^^pC27_Wu_;EK^yLFk9vLnmOus
z_#MIL|FD&stL<2PMlDFwQsY9Bkj7cZHyTsF^Qy<*UZegjuwCi2FQfX^JEib&joo=f
ziShJo^=`X6Do-3CW4~tB-_%_h?KFapzEWSft5Ut!<`4M%pS!%)p!<JS^O<jf#{U%$
zI&W2HtNW?8_U$qibB^n3H@3AaYdCeN->+g)-YQY6X|?={#uTeP;BZ-7E(0#tK>o>U
zGE@^^VhiR=9{i@3b?A)xvg7;J+l<uJOXj4iTg!%k_3UF@4|b<8JD-Nx@<g@i=`9-H
zB_+Y*bv2T6)o%oRRDb1oP<@lhEcGiFR%n3E|BGG?nb&$2;0-n(WFHK_OoZG^{^#H;
zu(>}NSTwHGrl}lfxT?DSh9y|+FhdvEZsul48czAKS0(qLf|}Epv+BzVCMqATv{qlS
zS3`ZN|2s9mN+Gb?^-Ee*&zo3-<7*?^9~IE~Kj-d9g84AJK{Q|DTD5LbXU+SQv%&g6
zVj#@TA)sLyU8$5e@2*<A%R&tcg-dD@9scTJCwVj?HWaB(JFTl8upmg|tuvQK(rYGg
z88b!X6xe(b|7taE8zpdl0+|Im|7V%*0hI}%H`TKYH>23Y-VN~^==`52fsiu#=}RXK
z&b70_W`XoO7_ox&te&i=(&473`uc*D#;Pyu$}atTlzmj+s)FYK8|@(Tw9mg}g2xBN
zpz91z`$5*u`>YlNyAfm_2p?&)R_~eWsM=U;0FJwdpYEz`;qH*B7hSHv!FdX721p$;
zjxD<e&ZD2tu2Q_QZLw<bqGt-ZlY7(_8}zF^sK2PZ_`Iq7`o>op2J4?GYJ57X2wMO5
zyxB-iRRS`{fo#@mRY#@gzKrlNs_5cZRjk<pj_)U2pOuVrUaBfuN2|0wM9fjiy`8N5
zQo~f~ytA`f?u0|?9eW<D*iVsB`}<Z*wWdoI>^AQ?I$(N}dbnJ8RH3~0&Dn~d-W^a4
z<Us5f`d;k<_A|&m*?wJ0nQD=0)1R&e-ybH)7^6}WT@DW`)$Iorgwr;H%{;c*7cTzP
z)(gy5J2M4r7RxcnUE3ftL0G?n9W3tUu~fY)F$5f*>@(hj%amd<h`+0H#gyXubl~v}
z()T2+P0?kxrwYR%PW2?47jhsm5We8(tDc*!tUBeXI>arXeu2}3`Bo9tqXy;RaNB+_
z5G*!vVUwzmoFLf#KmV5~O|^_uJ2@Y+_YGv$lns#mI6X5;6h6IytSyb4*QQo*D_xc8
zKRY;1SUxK%cC?*Re$w*{Y~S`cNIEiR=2K)XdZYq6|7TKE6xd%NGjC?FDy?vBQ`HMk
zQb;^^OLg`X7u5rzzrpgL`TuI3ex*kZkafrcJ$+#D-NzNwn4Uo9fsJ16SCviA1c%QL
z3&^>j;_=xE@#;M)f((!~{x>^wRgzTxseUzv#PQDqE0lvJ=7Qq^<OUe#;gto)Wy{Rx
zC}De^#S_lIAi=FD)ICj2w!c-8?F6IJ{`gr+2J)(4zgs@quBNvwAM8eunKsXVtGbsx
z2g|X)oFl_oov5t5=a&4`j9Nt|5y;+`I|=^s*W+Z>mp%Uu4tJ0~mkvmnf!HAId^#O0
z_mHUpY_@*a5`^28w<`s^bb<XZ`HUMJ=7)9V6jvQl0pC^WkojNvq@6!l-`0)0!StaT
zMn&hRiHb|NZdMW6aSy)N;9#S$Qrqj<U^TDz-3O2FzwomG+YOrkUzPG+?cwxv75x--
zuzeu6A!A1yHx%)vTF9CjkecFTIk-I_{?5;v72M8<D#Sb91N%kvE2jpN%4M+Kp!0t~
zYL-?<gVSf|Hprdkd8MYR-F6LN_kiRpE;FjhotmREQ_W7*$bdm<dZU(#w$3c2mv1;!
z-4z^F>#g_|bf$ADUG-d~c+kuRY|qT9RB#x9^n>svTXDE~AU=bikurC$J=h=H{w`5z
zlANfb@q|TrXMj0a-Ju5pYH{^fRi4(Ft1L`}?45eAJqaAgApOGUr^#)~YgC_63)%M>
z*qf>%^g0^sFOYnf#dQ@yYYP=`Rt=@BwhYzhfjMBi{N67D)4p~A%CBk?l^4B}0lPD-
zaH-<Nn=GnZe|?e<QnLd4;md<~<<cuV)Ry08QsCaA0MF|n^DjzBE30HOsyx501a|M0
zV~{-+y-TMn@FW~l`6&KC(RtMuB{8`fiVRJaiVr_p$v13mRCSydC$AJ42-Y*}3}l~A
zrP*c08m3hWQudk}0&S}mtzSUatpCkMy&F2G^90!K@2ak72xfbzzxGv;zsM0Gf8IM?
zWzH4Iy3^@~pA}30{ZOBpa|rAQkXv7g{|EcAa`txBx}R4SuiFQz%vrZY<-M)93h4fy
zJ(*{~<}UVv?2iNKfni-wQ*f9T&E6|@O_5t=KcA|y?Ka4qC!f6q*j&f!@4;%SVqYut
ziRh^;jQ3RZ5L&Hp@uPxTev*WWVH&II=Hwr0ciK6XoA?mtZ~UF23{LwXyKb$QQ4CnM
zNxtnl;#^&jI0y^8QB{9>a5XrMg7_dT*U_Ugo$o9-98F&>0MkZuWfXt9NvkBeUzI;l
zI9qNpXRPw_9%iuLKzd;KXi=u3qzdF5`VBtsRr?>9f&CF{P_5dQvkt``|09qxM~|~y
zMYLf(*u7exThtTl5N9Qu$QOe3-;undD&qQ3y?t{N*gYWoo=yD)4x6tvR$v+=Hi}0>
zU^E0qLtr!nMnhmU1V&~EOuL?_qV~{CqiNbx^;yx8TGG#T!D}(@&5zeKQa`0~a5b}q
z`tlEI3jbR*EOyRTe&-ynrgXYVku7kMnq%t`mDw$aG)|eX1g{4Ih2R4<G38HR8Nh3l
zx>j`R&oNwR+T3NW67YVrd8Y}RdA%X8N>;}X)k9yTEG*UMsd92EgV#LOB%Co#em>Fs
z47-`a!yTVh3!i!@&ywm;x;=|i)7@~MhPFzjX7ZkL6=`o}^NjR4CW|{itL-qDVs79p
zZ<P1$w${by1?E|20+bJ{+bMge`l#m>^q6~Xx@&myimthX>>PEc`29v|*CuGn^{)rr
z|Eu`>N-WsTcaLtDnc?uow9rdW!<4sI&G<^C`IE$V$`3Ej&^&6>tIjo*Ph~cDgmUB7
zAL`dv_L^z0?AB<zC9isvBTMCE#X7a`t03bE?*$6gmrvr=(AaQVU4E68a-~qdx_+FC
zM$@)UY9~6?^fKn}Q$DONZ*VVXokdfvlazo>pFHUN|2uqP2Bu9->KY54niti@Dc-j`
zqs(3USJk}xk>0Oc3C3pU5;V5I->fC-bHv0&`MY8aQx14+Y{r*&YR|9CFgyIX+oE9B
zbJbkuP2lz=TlaS3g&Q<Y6gq`Xd(>Vjt68#H>O=>rt4!IYqOBVTI{#PIFDVGT)+BJ(
zO5-ZN(^^5tCulsHeM;@3DudC~7C950`&^owwZ_`5Z7ae3;LvtqCH`mY%vgJlX&mjb
zvhWvbHqN^KK<n=oWliCxxoU^^c&i=q6SDxtzma9S{Ce5viZRbOs%)Iyr(XMEp0bW+
zjOqLjt?Jv~Y|>=EF~f{y@jP`<|9{iNAd86ot4+>JO;WX*0XcIoPm{-tCzjPnun{u;
zTC>AnzT?h(^=%$6%;v2*ruoLN(d^T=wb~->&y}WKYEp`03sHXFwLtZYPo-MSj=!Mu
zf0Y(|{iEi)VW#R_2VOPU{eP$D-&foJahdA!s5*szOC~9U?*Ba-9imtKXqk$5#%i;I
zZZ4(&51y!(bUs(xen}gAj#l|bE!AJL3rzeJlFS}Is4|$o_oGJW=3Qzbno8<Q3o0~L
zPfs)s>I+ocvGIV4z7wa)<*1h`!pDAq)4u!R%|^$s@|j&vyP|f>d8L_eyPU?gdI#0&
z(^K`WR9~4ZW#uc?-L_WZ`B$poY5Y*RFqm6${T_92URap8QQNLUR>Rz1P3!xTUmD93
zYRu1H*{b?K_OSAkL(jqHKkWFZ`qzoe0(Ac$w`Yz~WP&@mt$(oixst*MLv;Zab#NcS
zXcCXg`N?tWvUg<_9bMYZ4Qqtd9yGUt(>|#Gufu*qi+k;B%@r2g)XcuhYF_`k+4P2B
zhlbxrcID%{n2gS(^_tgMh-!0v%LbcwvMy5P#9j`t{-)H&N}%<BFU;G)cI&q>D}7p2
z3ubrzGt^=iUaZD`^PEPqXSJ$|Sd`Axx4vq-es2Wr|5G+!v)Oc$_HOOcw`bHr{r^fk
zd(|_XLZ*DaQmQ-p>`hy&TNS<vu2;MI%uP==f1T!q4+~URIE!k$KE22ES>0^&X%E(G
zPbjifTd&opUoqWXwMucXrdDZ=);fs=s{3E0X)<mvRjpR&Ro`$W72J<voz*5+P^6@;
z{zOgHL4*g~f46COSA9Hbs`fpmQ1#62R;BHnE7YIoCu!;=bC^q%yiq^yEvwP|?zUOu
z*Cm?1%4@ZZ_s=mu*HfsWnXy`ZRk50St0AkpwXdbhysOF<Y}<>>FU`89S{Yk!+OmsL
z&FEaRR^o~k%KqLuI!^peCLMPADk^zA7F(E~nO?KMsvM9Qs2bj2t#`5~Nu%2Mjq2l&
zPNf=EQ+=Ic3H5m<6{cTTrf8Uz-d4AE+pA{fc+Jf1gt3vXd7S$Eyk<4~zoBXqCKRbJ
zKPIO%P0m^S!&f$o)>WdKdi~!tEG9OZe@)X<UwG`W`m_`K%(4PMsqgezth~}*ObvAZ
zUt;+^^|aI*hK4y4OjDAL)f3x!bPl%PH(Pq9Pwigzay14gP9sMHH?vb$&#F6`%rtM9
zb3`q7swnvW-|fAh)q0~(srjF&GT+tbqH=d`mqn08q{WOEMF#bzxn|8$)67GraqIU?
zXi_(tQ>B%&U&^9yd#-s;HM=Uy2{ZMr#j&QhJ1dkozWZX@?5V1rf34rlQe9P_Z|!FA
z`M;M|>VU^3PkvVg)9OzORZl3L(K^5Mpd$Y(PVm~^vU!Z^pRRaVu&XO-6ms&a%$8rI
z61^^6?PtwO#o!_}4WG9bijNMs>Sh}pG1IucO8M)1Uo&T!QuDpFIm%v|^=f>F%uLnh
zcBo0q-UXL!t;^S|Z18tipTSbE<&qQ$w)b8=pBn$MMdlYZT2+?ZWLBEjy<L6(D<1Xs
zg^XtJCgxidE>u@PoK&x|X`z6|`aix#sqK%<YbC|hzHR$pCZo4bUE-;f`sX%Jb%Fei
zYJcPO)iWnrY5Hg1(*W)NlL`)2U3+;Y`2IgF-PNiLzaq^+_x}}H_=Ejdyn;zpUFMA8
zG2sLY5!*S+rK`P^r_9?SC;4cDX4$q=>W>+76kAr#(a_;BRm%?SH(B<B*Z8vc7R{=s
zahgea+fC!S4y%j)W;H+6^~YSgC|cut??Q`|y%H)NCBGDxom#FUQutF%FP2TEEyNY<
ze^8k{ibq3WGz3ONU^E0qLtr!nMs^6i_@RQ*F7Qj}0k?@r36+YZ1z<e~b=WmM+hoA(
z&+h7)TK9Ir%}({p1@~h`x9EcD(+-B3yi>WsY-GDG)?GxINBro-t-0p72Dtx`JIh*g
z_Rd7Go+lMM;AVsL7cu?N*!`hYGhP_7w=_(Bist3Zf#CTOkUTQ>o(Q=o79_T|W)q4z
zATb!;clQ9;FEBo_^y9=nu)e<U@>*qcE5PoB>DBL82eyy#0OU+jnD|$&ry9TiE`h84
zowpy%HtPKa?z@20iT-iaoc<>n+`j?wLHNPeiyFtc)HRukAmO^BBm%DfuT45Q+*DIs
zz-Dz!NdvnJqz{BsS0!owSlfXj1`-G1b(<jP*K0bmY2L^!*C<qq2D=+12f|NyAZs5%
zY;-)+PaJF?Nc?``QMmceyF4{khO@xsK=PmFZB_fV{xpghNc_8$16WLK`%V=9g5+RW
z*;Ej$2F52u8#oJr)!zttjbi58wPs*(kp9B=LMl_4&TAAcl7@?c#6h^}W+jGLX>$=+
z?8M0nDvKU21pBqsR1Pc#(lg&jR<rLI12`T)d}RF17Ii;7Os-@31GxM4eVGWKPc<`V
z2b({2&T_asOb<4C)~jV;_1NTKVnz4m;A-yhmTL%FErz=triK_g=Mv=nH(YAnmaoMy
zmv{0Ku-$oPv%oY+Z|1^<DC&w&%c`G{@`R_o$xM*-x+i-Cu;y)$8KmGVEge#_HV@(U
zA6jRl!Is{sl73%T`OHpscwRy_llgC@)>eKWaGZ8!Frv7(a?eAR5SPthF#!t`%~RJQ
zcQ=6a3-`{4+XLbof7=6A_b$c>MSN>_qvrEKDX{q<b;x+_={c%i{mfu}An~nBHQ{<|
zO*dh6Z^12fEjGEe;P6sq->nk1Ckw9s%gbEVf3?wIerhNyn2t>}N3mN$;RRTHxju)M
zto&h(YNjv^Yd2e!>>C%<Ehiz?$JsiHXzo^r?C%7*ZIadou-jg3_XF$SqQI<KvX4zY
zrM@354pM^)mj$BUQ-ez#NWAGdx5_6qbFi6z9$!~c<Z1@n2a@}Gv>na{@o`~yTSS=!
zQ&Vf=uF=X63pN)f79hr`S$hF;Kh6U4CGd0!lP5&C)RcnVV}7w3OoP<x1@mgQ=$6Cz
zAhE;AEbx2};)C#`_&N=r!;m!e==*0CF_m1f8l}%VT5PtD!1{e>&co}bZ_lUTEj!FJ
zjN}<sE(E6)ke!z%CxO=ggU!64e-ULZ3P>IqgYN%9=2yH^REw9L1=c(HV6WPHH81s3
zG3%9cg{)PXj`@Sdk=29P0b06hmlPmp4ukmUxFB$+YS-42s)0|Oz~kQ_dC>hoAUZ!j
zP<@v6OXZXEyufBx@13X~%^w2xukX@lU@?$77-s#^4)-Ta3`Q?*`Ju98p)Xi3NKECx
zY>fMVvhPC9E#IND4(^uSAFILk7%p867X#_XhTk1@1?N{UWyl)xr5`K7_LY6CM{!qy
zL>^AFJXJWsdeF^3te~pyS^-&yXSW)%?j2bTOLwmd(_Pg2R*tbj?ly?JwoujN_Z6_)
zKzf8WtW;dO{RNzF=_sLcbylnT^_?FS`CgX8?GOI3OZ8JI<P3h0J`j#v)S~+8jknr`
zh5x|g#2_&kPG0#}G1O5&#U%i8zJAWyBVd2mJL;;pUpfd@f6P-)^?ZvA*ben)iC`L}
zKb?P<N`VdJEX-0)XSGMqx2k^DHUzs-Hshk2)uOLpHb^Zx_US%~5}up;x#8wI>xqKd
z`P{myGc%>t7k^u?eE9=pkNXBA#5t%SGmShUXP<-EFbul?2gEi=PX(s~5Zm9^Nm*J9
zaz-(TPYC~;iD;kxuC7o!ow)++CYZjT%V#Tt?*D1a^-^2H0I3f_a$7ed&NZFfTc_4h
zFM^`aEM}5wn?2&bCXgN&mNk(BmnEBK$f|ENi$@6`keaI-rpdL7PJ#OgBzAER;@nn6
zp;DCaR?eBPe!sI9e*aJ6Pepau{XYj*=z`<9vk4LIAoH$GLOoNu;_O<u`^#1yRMoGI
zRx4Y5P`P-Gx7tjnt?IWKj>uipW&zvVQU|%4z^!9Cm=7`!8@~05SIs@M7p!KDvzxlp
z*CzFXHZzoRO>SA5>Xlh1;daY0e}MbjML`A}PT0%_iJ5mzSFL%}s`}*DEU+IzVjwK{
z7E(7qT(?sBb&8|ftbAR#nQyxx<-o#sN5OV}?VqmvEU+CT4q|s6QMOZ`1D=-wnTZQG
zONoQ;|7lf39rK0V|AVft=(U>qe3_-HeeLpKHy;<oUx$G7zlpf1sN<Cm4o48*?#>Lb
zd)J6L;#_aG!~Ue2_Y}z4M$>OBQfzvt4c1fB!2%BpkXhKUkJ4P#5MjuDFS^r?p~S(*
z=3W#&NdNcW3MyL5`oQL9-rJ{=x)`-g1*rjHzj@wpKY;kiIG!1?HX0_Lo0^4Hy_Lax
zuw5W^-cBOwOZ6s!!xF^Dg(Yv^$Ejwk804OZ&F>h&X#!@(^#4rYauLLj6-Wn%pY7Q&
zRs9E<aQi`WKH-o%FVmR0z-mBZAPn38r~iWq+^#Y6G6nmGrDuaG==^^ZAr`gWOpfZS
zG<8Ae|0!r#vxDk?Rmlg()%XK`gU#|pyZ=XM_hPUa1==lY&LSJZ_y0u9iO`U^9j^h}
z{};S_f|}H{P&Ls0zvMa1YM}jpFA6j@epHBo-2ieI3{SZ2p_;iHa=xAGsYtN7$9P}B
z!|BX~Z|VWQ^VJti#Hl6AL;6O_&sV53l{zZH_Wyk@7T4Gv&Y-5cp+lWH8j=TK`~Pko
z+yYLY%a*xl1g>AC#<df2PBzGX!S~wmuvX^|P}!#(sDA8r5;$Bz`~OY^o>Q}%^#JS^
z3&s|8ox3Jr_apEB+vD>ebpD?rX#d}qEkP<-=dOYM05TsLfBm^Y33>mYarb>y=9-;q
z=dS4~^ZY3R+iCe%2^`*_{eL$XO;yVjQ&*CCucW%P1hK!)`x!4-Ey!%q5LT5v_dL|?
z7-ZCbo-0#ZEDp(Qzm_<F)vVhtpc**`a_)|3_+7P$R`*mUibKxQ2I;rwg^Y!O_W#Ar
ziwElgiK~BCP%YUh4CdFUg{s?f7_04WdJew-CtC$_za!}WAJ(6u;QfCfJuo~Wx<NH<
z6$^O(-^Cx(Ra)<|YDi9k?1=#F|9jrHShd)DGT3a`{=X|P)itUr4}<eGX#ZcO#RafD
zZ2zD28y1xphU?X~ok&wFpFLTLp<e>L|L>v|A2_T*`~O<~1>tcCbIUyj$UeZu7x+{q
z6Ai)UgXFYUEd}d4ZI+?@;*O@;l$z<P%lTfZy`NOB-v8QCRh}QRKkexZarK1^h`mBj
zUka;5yf^?(4<K{Sy_lf(aD|R~nMsDsf%A~Pir?j4sob%aLfQY9ED2ffZpQ8h&dYP<
zJk?h0%vQ8GK2v?eH5=94N=#t4gU<ig+xtqD(EdN0>zlypVf+6y;$Nsw_}-vu<YcdI
zc`p)l{vWu`0PX)vs)O9+nKefotfyih<cuEB{=e>E8)b$*6E)k+2Cy8+PiI0Mz+ppZ
z{~u!^WDhV%4+w+y|NXt@4?6!BoQ5atPf<%a0@>HQvaks(HuvZT)!i*^Dw3xlcc(qm
zR#UY-x&$1ab}=iJRJK)UY=xfxug1DbO~8pqEq}vdb=DQ^YOD)pX_VSXY7`V)S83jq
zsJ`?uzuKftt;%<AJ^;HH<OUF)!zqu#pYCx{rRj(%*nd0nMb&#ZK2(2Xo3F~gupXS3
zO?Va5UX;F4`@e`&-Cti;9d!PG(R#=ko~<8))S1plfzwUyN*^WA{Xc;L=haS2$b-`-
z$i7ZdCsmPlCY9+-`<04*wy7S}y`cg+{~xsf4<vu*>uj~|hV|h4e=`4VQZ@|RuadYa
zN&WVm>1sWO3L0OcPN^Oc3kBc*BlPZ(O86ec{coWCe-+!Og6(2)>H+&p_6Co7{q9zE
zSw%f{L*CP>N59H|_y4Us6sxjF!bR0y7jn;&LDVv}scD*^^Z(>qH?@KF&S~gV+nKyg
z-ZOl$M(HyljgxP8gTqC1BH~`Q-3;%PwXz$)VWVRttj50JfZEodT<Qz{Oi-Qky-lrZ
z2V^hZs~fgpckOr-sDAShqiXf{3t+QAe(nxjs=B<4LEYuEvfADU)78upAou^k_WxZ{
z!W?tnKJ76$oL{eov=LAEJX7Q5-K@OY|EY?neGb^JHg-rlg^gj`d}mR4`eBMHU&=Z)
z(EdN+lpM8}p`TPX^YEzU_l1Go^ZVNZmHUh-;4}tu8)*OEEVi3)K1d9NAA`<ke$${K
zF{2Hv4<rV{9;T4}sb+2WRG+nNR+oG&uX_LXCRL?f#_AzDOVwJR?NW97F+;uVA2VFv
z<61}=J*j7=3gfTkaJxYJ|3K&eznc73^;*wQxO$L1a@pI!e9->CHHDD#8ht7v)Z2B;
zz;Ops^N<HpU%~eO9ShI}o&N_u|KBJ4i7M#)|1HWvaP=TPu=*dw#)i!^jO2VdA@`_)
z<Y(*jsX9L22sQ&&|AWM_;nX9mz;OnX`)_p&bWb-}?>ph?Dxms5ZV9UrsQ!QYR}&r%
zF!dn1hm%1y=MyiAIUsQuz9q}2nk4>R>A}wl;Ib1W?lgBI*eqE6KVw2R*v`b03)TMq
zht!p@`X8hohPMa0fYrhH$n<9&7nKK3jKS#)Bo3<onFS&3PY@r3t3zgk^&{8+Z?`W2
zi-GEYkQ#g#x&8;K(|HK7uSyGY&OJyBghBN`h{lE`+)?*8!_<K4f7gmAaJg@{8gh3J
zOb$fnK7Xvdu4f)N+(3K~o~fD(=7Z}0$ALy*Hb^}2s+gj^OFy{G2JwF+H>m9Y)P_?3
zgTylz#i`EW-38YN5{q2aqT+Sw9BBU^Jib8UAPlPiRjnG7H|>U;*#@isGuKa4o9_4l
ztR7VVznu<^L+Jj0klrZ?Yn9T1e}K~o=#1UsXFpVRFF@)QQ2n2_4ExwENbk#+XTkbF
zY!F^pFbf=KAol&DmtgjV??Nh@xHQ$SiAyLS)pG~?>0mu%PYy^O2tVA%4^JN;J_yG|
zEsz7#Z;pb^0kLI298gBy{|^$whqdR<gS#)QO@m<l4>Jo^|AY9T`hT+$YX1dP|37;2
z5AH6IIv57k{~-3>cgI!Z8{dN4$RIup6RQ6eg%EeDf%Jgt|Gw&G71;j&@K{kbw+(6F
zuz}V8jD{;v+_E`N0)>D1{0(^czI42*vU>Y}6{b*q#e{$}s=CwPs!g@jlk05c1;<TT
z^ky&}p1u`KgWP}(Yy91#Quc5fSdHcmIkiv!y4BX%EduWgg{iNvh*w%x4cW5|5|d3t
z+)umzZ#X<|Kyv6f>Y<dfO!ER|Q2mcA2dn>=?mD9ov{p(5bpIbr-r)q~KH@8(r@{X1
zeRxU<wErI_52Hc#zcK^l3~ZPfHd=LOw`#=-4<%6jk1YQn=NC9nA&Y_7Y*tNb^JSK*
z^tH=_^Y|%V$X#t9Ib8Tn#7zaR(zD=j1gY7}y$2jFZf`@uW`M-d@fv*|)%$V@;5bI!
z|KFRi11yHF9-06A?K&0PZOULf0>47e35UslZ0^PI7pVR(-UGQuOIMUZ#oBQKST9Tu
zj0V;JFg7~9JP2{GBTP=73$k|-#)r{Qs}OBKQ(t|xnr)$2?SrWY(V+StL}SD5H;lk(
z0aX7Z%PUnw?tcP_3FI4s!yi`vgT!DMRR4q6zt?R=N!uW?Q9K#~qaiRF0;3@?8Umvs
zFfv2H%o1{U?UE}u!F|M==Y<tO<NwA7*;L$Q8dT2PMuYGFJ=&xQZeu+Aff#o{AOGjS
zvKy=hbU$jMv<>+F-&KM<YMst0;C7wUnm-B;ro9BW3&VvV=gOqz?@~D=jJOLK77Is%
z7pk;JD1-GSWJ3CLJB*jY+jm!9)~H^;XsXI}s9njxN&+l?bBBoPv+K5CHprYQJnpIv
zH?)<et7U@S1rmc{qjf96eUR!PA+;^Bx~jqkZg72juS4#f0jXV{yGAkf>MoTz>bv0b
zQ8NWpHP}qReIwzDJ*rdNA$O^R)WdM_wwcPX`+w`!=_v&3U&U|(Of51!W2>HO)Q1e!
zyWWfn8FtNJ|FcLs!|lGdja_w`^%I3v=VmBfOqc==OJw~pb_{p8ieOJaI8OhnBF-xU
z$$Q_DR#<4n4c04s>bHux@fF1wxliEm0Lj1lhPaQ}&4&r>29S7I@iAr56LY}+cr0>Q
zamDwy$~)R8!sS6~K==PX)+s^}1Bq|_&ZgM&(h@BGTc}a_+nocdr*Hm&+nJ?R2X+rg
zEet=s(Fj%#<6rpTg~EU9wLm5R-6IuYzw=;!^1ssq%P-=C%t3<m?WwW?i-FiM{G%3f
zmrNV8l47pFS9sX+G9b<dnR!G|HRx7=!sDVND!xBDRb79mE1SQ019s!jEorJXdl7w}
z^8EP<PrgF-VS?;fb%0grQrHdUGc{`!r!;wj!^`8>9OW%(U0^my4G3#qTL?}=AT|gW
zFuJK2RnAj5lNX{g?ZX;5vxfU%w}Zz2ciyc*(F;<Kj-#~I!Rle+`RUwXzHRn@)$1!F
zQ11V&%HRT<C*}hg``Y2S033!Oy{jEpgZnI^5^Z2LATeTbA0K8swmmrmH|M2^sY)Cx
z_PeleS04h0Z}O|ZN`{t@y@m5_&MCZDte}=$E(VUL#5dnnKK++boTUFjRe!@Ih3Qgj
z)b`nht0w;ar|jOiRAs^JiwadT_hgj~n^D5Hs&o<9PcVPny6hyIUt|srpN4mGsuA0t
zs$LT0RepGH4p?ka-hX+Svc<|Np>tKzmhDpMSz{n)`FSqbA9WYL!0-Ppza|e}ryO+E
zOVzCv5x+0aA6L=*2070QWT(TTa|)pGe-ImlgXH^_W;~Ms_sJa3cPJ<SbX7LLeqH6y
z-i?a=?GkEn?0QNS7F)sLvGhB?lH)=|+63t@{l60|w<sTS|L<$N9V%hB6IBIN9h9&6
z?ois906918*)>QSP*R25jmu>;QTfjiJ@Ea%cZK(Z&9J(08tkvl`&!iE%0$$dsszA&
zPJ!cyJ8D37=Dt<~i-Xu7O{b~M*7yU~@8!s#+H4IubLP#;0&uvm%UuLEQ)>w%j6r&|
zymS;*Oy;Ymx6f6UdgrXz&9ea<)*v|$J};vHHy^}D#vyk0V71@3Emn^C&ZBtgXsWVe
zZ!_2qwMlQm^c<TDU>bD)?{3v{g-m;Eu%2IMT$SCT*GqPnsVY@ELhdu~+oz^f@Wv7)
zP3Uq$=D3jUefy<RNi4Zj^-|y-uvsv<lR1zxM$JPKRiu2R!S)tJH!21CD62}Qeo{7C
z+^E8plc~DK4{`7A(%Kzhb@}gARO)SSgToGH-t<M9iqHMoR4M`x_x*z8zUfVX+q-?9
z4Oo8z&tcW@|2bebNKJPu<oq8H8-%A@IjXLB?W4Fvc{eyugU0`vEFf#ZK=L3wI{ps|
z1qdD;{~sOy2c?A3@qY$}(eZy!N`T<e@&D2Be^5#oj^qEKomW*Wx72HPGu3La?mw-y
z>>Mlje34(F{904_a@4lZwy`)I)Tf^IR7^`%?5Og3*JWyXpEj#xKUoVp|6i&1qoY>G
z3<(`;PATQ+iyn#()Wnp2a#x#H$xl}8zQbmqJ6FwYaobdtw_WTOf-9TMnY&)9JQDn)
z_UulT#Ya15wZ>=p2Dw7k%1noO%{`x;Hs5fWU+!j}y+(WX1SLT^N#)ajYqU4~_R}=W
zd#}aQ|3Lj<`vtSKBPUD(PQ6sKs$;d#X!>dRKSfGgUqjJ?X+ff@yo`eK&vr?Tv&)(+
z3^IfbfBpYwKJ!$R2CLUKqc1iqw6&LIXlm@)so;D-Q(Hq&TKVYCQ2Bx*9A>e1`ZZ_l
z6jkRi=e02V%&jWiDXAsmc}`<@+zHiB5i3<sD}7f#G<k=am2Zk>Ab+E3K;K(c`7a*o
zZv$A>;!;i3(%yYlKi_^&bIn{&jpP(<B^&2_jjQnwH4?U}sh`T^*LRaXu6$Rr+VJz+
zdludExRuw>%2Pb}Lr%Gq-@@?rkywp_+CS#gqD+-*k9Vs~U~E(UFSJBIG%Unq-{Sk4
zemeIxgSHi$HfEWs=pO6UE|ADmoOmK%{ROADd4TwS3ohGMHNHFd!0p4Yi~gIae=0E*
zJKbyAI_0d|tM$h%WW%;={5)5ts=eObqGV5-ijLwQvjd;KRJ!ggGqHNkrd<}cSF^rb
zN8RqbrjeS^B$Jz_VVa8Dg>_sNgVYt*rK)_ZVN+?YSa0s_eOJRg!P`RZyMf8~uhVpE
zB?Pq`tX8OdZAnq<IsC```pysX*(yz{zveDb4tpxBn%S5FTK}i)+i}h8g~1~=u7&+t
z%_py#ZxCGsTK}iemUh5myELDv>fv_vP6I{t<duKrw;h>oZogo<(bcdon$DlEC^dDh
zSMcDe(^xb?!c23)IjuRT63w+<w(E$rKUWrfT%p=Mcd6?B{H-bvH?ye=Dp**kEq<o*
zETtH<{!gVU^u79G?ItDJjEgGa&8_O^68+St^;s%M+2^aRnte){BgabrqNRjt_$d=}
zg~WYIGaGm`m)z)5+s+$eaAsAS^0cZZ^*M5%O@FKrF;}|AV0iwunieaYj{4uJ-_>v2
zda9wmDa7P5*9JAusrBkgt9Pn=_E1&rXVNm8l)Ob!q>srY<i#?xC_e!Wql|^-cRj2%
z^@5UBt1>SbXlMCacqGd!U%k|-%<}NE(y4E}Y6XjJm3kwJz~>HqQNN(`Rc)<Chh?mm
zB-b*{ulM;Zct6XkZx!QG`#5Eb`S}B@RsZNLRD1jLo`vgrP9@)u(@jC=|F-3Nsl=U-
zQ9b!wNMp9GmjdYgUr%;h^|V)_8WX-ZsHlkZSlqK(tri<-tYgV*Cby(!tumjMhW6LC
zDy?@fr>JZD&d_{y&e2?4Y?szOD>vo)72QUydp4T4#%<Bs$t<oA^EOcO_T5FQb?fxB
zuWpzoQ=>jr<x>u?4CwwpVUJ?b^Pj|27}bv{i*(;su6d)W^ZV3i^@)cTXszFPU1?cD
ztIp>B?V$C4N+(6am5<xFnY}mX)_HShkNV2OCZ)5p3)J0N>`kZYUs97==w$Nzk*G5N
z)|2X&9J=(?G~Q^wy6vJGs(ni1kEn>*Op#Ixv9!<HqSvpgYv^v&f6>LH)_mPqM^=<s
z`{{($s_z&+>8$5oukPK*u8}$4QzP=>Mp?<UDGDndNocnGS*q$cSqZ$B>Ocvj`o&`h
zw3|i#sBiE+pyaqkNJIWWw|4j!YxCun`!v4ZIH4IN<7j>>=!<sg?iD&p5(ySfy84<=
z7GKiv`no{l?(6^Rv-ERRJI_>FlvtlK=li)_<5FUonV+Dh+M4U!I-v7^d=gUh&Rz2{
zwGw=<JcIR*MgM(6vzkwADlyigs&h|w=|}xj((-2i1zP{7GO5VaV52*;#_KgZOqNwu
zX*Mrw&{%bBin{#&0JA;QbByfm9%-!Fr=YGcqN%FsqpKd9^+)M+Z=8-|L8rwn6&0-?
z<`G)VNB^47>pP*oJflx@(T^H4|G-wz`ac!^Yp+z5ZckHr-tt6ayFR<|{<RV2V*KKo
zB^vCyPn8_ZI1dMF+zOUfk6-!FWYT(nv+L%$8ea@q%##G9RkanQ3}p%?DrD@8R~NaW
zqH#y&zWMu9PPGlv_7<rzH!V6c{~CxqQ#Vg5tTDgGC}F^%JYQ2K)K)t<zQjW0S)oN^
zl#+Uczk~*VYpE%->Pn@r3cJj<?X6PxiCbe{GndD}uD#tDbpG$HeQDr5Yu|4?)rgQ<
zqdeC@LT&e>Y1+(Hf@+Tvg)DZ42`W|bwX5ILnrC5YbXD{4!3nCqdn{Ft?X_0lHiKC$
z*)>Bmv0$t61H~VDt2AesXH800IUc^$T*^|+{KH2XHE(Hi^|UKjOyAkeSBsmpUA^~k
zhtd{BYt<bMUp0QKFl!ZkvDEaN=dEn`ZHlVwVg-xmg<48yLZ&D=@IO~Kep09I<0@#@
z>*a2-kME}XG@h>-dpHv{)U)}GPE850m>aQ1%}UF{V#Y*OEsF|0jY$XYsQ*biuRig^
z4GpzJj9O-|W@|lieWAQG*;-B5b+TIQ?>_nHX$)!$4qP_3dzGoO_m+r-=jZuK1;!Do
z<?H4cE&A1Fp?dF>vdTUlm7NC!6lZ^!t-WO49}O;nuPWv0MVgLN7&KDyub3RZ%Vpvo
z$gZtXQm%Eqf!kbv;vx;xoVR8bZ1xsRJ@%SkLz*lYrWL6ceLt!?H|C0p$=*-ulmDz%
zUaNQ<oCiSlKd5bk3!7&c$@+34&Muy<)2HJ29CFS*f52~)aT9biQje?xkJZ7%|63hX
z0oDH?{yX96%6p&jtHv#1RRq=lPycG7jDvvGfN&2dgG$aPUKDdc;xK$mmQN)~{JY|V
zpA*32O(1cnxsdxbT?AcKd00dgXH3WjkC7yvT&Vi@zYMsZeCpQ?76Yk=;q8Gg7-BHF
z&pIy351tr<^@7B@)~r)y7L-Nte|5-guzpbe4^jidZ?{9%FT>9N2Z`arp!y#q_uBFz
z*c_dQ5c{gMR-)Jgs{cXiuwe;z$k}G-a-jO(wIT{U&ThAw0c<|9n%w7)mDcslLy3c#
zs<~i3sQ!N(Xar`1^dw#tQ?PgGhr8)Va)a{zPi?B#q4hsV4G3p0ic^`xy9=%lBo?`-
zMcM1pITg_Pe;__G2G#$nRt-v<c0=Y8K=nUJK6Cv<)#;8O!12rC!KD0lIy4S@i{bGx
zC1I^%TJR5WI{7Hnr&9dvhl=inCa@bp^?%wjb__p*%m(3?FCpi5gV-RvumG`F>weKo
zu)Yi5g_JjOX{ufmmry#Y=MMJM!FtHqk03oD{BR#X+&v&Z2**S%0FUj)yg3Rs2P7u@
z;eZmT{s*y%!P;}@f&B$io7JYF0;>N(Y#9Ezd^Y&}e^~tw69?7*FgB?E->k%e;yzIQ
z|LDm-xVvC#K{V+6e-OR*-Eo!p#<$?{Ll7T^Vf8<Z528W!KZpilMWIq~7=hRzd}~Ls
zYF~A;GVJ{S@K{k*w+(6FumRQoAT^AJD^T3BIZgtFfBF0kxcgr^UR7Sb{l79(sJ=o%
zz!??Y>2Fo1TI$JmHu8e)4U66kro+=AW9K0Av0;tBdz8x_P6Mme+##p>>0h_%I=e;S
zv3!{N`igkPW!0zPcFQIr_O$N*8xD^fn0jP7>Y<d9O!ESz=}h}k)D=1`1KYWD*BSYs
zwNmi&|3P{kPC(W|UkN=8wzK!)DaB`jka-Z0JPgO~Jff(~@Bo~~VPe>5)t%ic6)QXx
z_eh|YZx3=HXTqcFVY6ydoiDRgxvyOwZ1ySMa<E&`)gklWMBJ3;Dm@EVx0ibl*uQRX
zL&0VstA(-G=<}%DmxIi)g6jY1)vjQ*y$O)<ahN)6^z*mt;OGAbe!U7V8(``_Hus|N
zLE_5_CMp&0Sq)B4x}prq){cnv;vjh#2G##CHafjL2r&l=lauFy+}Q==!|10~h<zre
zzWS;)+d{$N0h7l@gX({5;vg~i8%E%?0IL5%d>B@${tpf_5MLnQ5FGxiC+jJ9xaooY
z3K9olpK!>XD!<olMM>LhUPJblfz)8bp!)xLUJ=|5-uv|`-mi9o-BNzSA1;nfFM9nS
z`{9xDro)i^l4;zZR7GF8syWRxR{rxMRi*M*1=vh<bGlu*RSnt@d(zP5zc6a5hCA}B
zPVtAVHwDSV>i>v~@oI_JZIpMsgX{?|>Pb{9oCw*UR;%TPF~>A#1Ek#mG6NmI5Uf-w
zEBAxz36&^RL$CkeMq7a8&$wTMhl|=r$h?h=!V$O_y4f)PVT1MHID67%2sVf7^&T*7
zeE%|vUVQ8SeP&>;K?j-N$FW0|!1{ls=5GaH`$6hK_;e#=UBS15da6r1+Trm7s{f5u
z5NqVO+<mOLOTHPr{-1g21hAVy`d57GRK`;Ot5kDa;Hv)>K=uEAgKn@H`sR@G<aCyk
z>T#t;aQ^uwu}CHH#8<FeHr++Uhx@OqDku3k!Q$N^CseLDpO^ciJ00vFrmII(J$SI!
zb0BxdPM?huzDH*|g6#*%?Y;)tPm!_LQRVbeGc{#vRi#+JcCh--vryX<YXfe9^B+hb
z48!XG=6b{&@aKDSip$vzz;1)7!$-Sc_yqPlNX@bQFx3y&df@H@i7l|b1iJrEqAk};
z)nX0gE<up^*Wwg-IauX7Q#E9nGm5_B!o@0k0<rJ4hK>K%aD=K!T8m=%3$*^9F9EVH
z1!NZpC*4P!uce5#{+}hPTCG8HC20LWIPB^*?!oK-g|-*KV*L83=Ti7bLCzomnGM2f
zOCWREAhu!mC6%c&7O1}A(o__V<x|;kpcl0MUsgT`60V#Xkp0&2pQ<q24$=?9m7m>I
z{~S*P$HhCd532lo96{^<QOZcASC>?H+84mXZq{AMUOSK(a{u-4n%}|Jpqh|es``GG
z3fK;i9uUr+{2R>Q&bCyE>vg4S+Jb{%K1eR$Y6dvIWor+B?R@Pq73cbY?U@_EVF%KO
z3;zkp0ImO5obCjfJBO*mRsSn&pQgG;{i2HYr8!`;I}aej9i|tZ2Ce@;+sFg=!|ReS
zU_V?vI~8a6#CV=o?KpTBoM%DxzxvJquv$02S6~|54rIP^`&XsKlOcE2&ASFECt&i}
z>i_*pSxP@AK=uI1FfUg2d4V|72Br^0|K9*Pivq+($K~r$+i)QHo7%lt-FhtZ64+lL
zHS)*bsJ?p%xvL1o$Av$*L+WU3@+-6!fX%yEd=N~-)JQ6>f`|E9wkmLZ9e-J@vZDfW
zR~1a1@}D<geF|-TU^Ostbo$JWzu+_n6A#W^2_BPy@zLpxZ@z)!8Cm?t$ID<do_jHB
zHU~h?H)2?6srllx0(jpjNbS{b6|lM2mP5`qn`(ST{l;{5lyjy)>d<lNvU*LH4#=4-
z!afVtzIytD%}h+02c}`_^P6?l_jpcLzZ%a2wr6#LvW9QtU5tJVSUtBa*u5b3H;vg~
z^&mDn-f-d#c+3P{oDg3)>@!%O_eWNZI=eq$HcXF<8szTLJKI}O?A_=4R6YLm47iw*
z`dm00rZ+q#M&tLyBPiyD>{rs*w*91LZlC}-EW>A-!u=L=@DA9FZK{wn88^;AJ@+mo
zFjhUMVIDZ0!0dq0S`CU|drosqReKlSrFml|A3WWG)a=loixU1IF&Li8*#p)K;@?i}
zhqJfaG^+2P`xedz$(L`LrQ#@td5+(NQ;_vTN9KG(Q4i7s!yXq2pKk|K3!<;wmjJu@
z-<0FvZ~=*dFza;4o?Z|;@ZE8>38L@RS22o%)q=!ec*5Fh46%!cCBSCw`S4eLmwYi;
zzP8>7?&e*+Oj=jDC&KwKvq1EP<D6i<AT~1Iy=*nu9FUl0Hl$4mVrMRg+!5Mn2)XwY
zB!&;)7lh;md}?yE?!(i|LAiJ8H=qB)>Q;Pa5F=;V%K`TjO#PH4kaYRAuMqAA(PYS-
z)i5>M)fZKielx+tDeNF*pWhDug&5@!Ob;ovcae%xUDHpv{ns`@($u4AdsMWqBHV>+
zCRbdB#!gcYaDH3wx&ZDrkeZC=l4_ipkTc`n7e}d=$LGW4J^r3V34?FT^1yQDpU<L%
z`I;R)nv+;wqNoL#4Z@rgXQ>!*7=rbI_-ykJ!r3zFQ7Cy7BoD&z9nUrVc#6St6CWun
z<%U7degMgt@N89kv~dR5|Gtukz;^okw86~;sS%p=9$ZG;RXd?xRxYloZvRBh`&x_I
zqf70YDgpLzx0jcm)7)_%a<4o{pL_01uz4%){|D19Oo^KRqvohO%66id4^oScBkm*4
z?fX~;DKpX4!1x#TyjOYnzY%Pv!8-x9U~$O3k|6ml-?YKvAT~a{aysHZS(v(<HBZ&g
z=t0h}gNa>RDybeY8L~%s;-&()-(m8EXj4Wma9FH=zZb3-bdULwP&<O|bjrxks9-w>
z)(^60Nl`brZa??BN#kdEBUqf}GNSwksX@mI#}W6JqsxK#p#A?(vQNU>!}befR67qs
z`j`ukLe2^Ssl$fR_x~^JNmC7SH2|lrYO6lgNgut`4mn#Y-MqR(CH=G=I8C9Ob9z>*
zYJLjt^X^g?{#KdsVyd!)RwKL}2HXG7E;dOmv+|OX;8Dmr_K)Y6sa?*h1Ka<`3A5kh
zauN4lVJpsqDhbl*V7(wWCh#z-qwoKpzy>)xdG;wkxLfS=AbUoM+5dk=;1)Pf?!5#V
zqc{=17i{0GddPZnki8&`um3-@JqD}?qz)Nhv7ev@>;Er(1sQ`umIv+sS4-OjwjU<G
zw2TqVSBPDpDzp=FMhHkQ@v4}jZwX{f1;npz*`xBP?+Cd6-_daoZa>>5N6h{Istl^>
zC42Gi|2Go31U4gE2r?e=Z|y|2m;d6yX?xc3hbpWIkp8*if-_)sAUDLj*s1g`dJV1v
z+7Gm-vh3ZZy5<pNK4Zdi88xO<$QodfT6FyVb}&4?KyrUhAl5faJ%+@2!ls=nM<N!h
zJvsJONn~d#*eyIW`ccws%ymSYA7l;)!}kC0*)<og_M||yvc}mYaM*&>5W>7C5o?F{
zUO%8(8}S0}Z;;-e?Y7FRj;KiLs9jVo%qfD)A@Bcp3lmkV$c3C;1=0h;`?vC|ete@2
zR}T_{VOak^RHaxA+x~x<c1<~UbIAH|n4a^W#Nd8>a_1?E|AOvsQM;9M5!C+&hjmhj
zBz*rro5*son9NHF6t~P>2iXSzGk?>ihj95-y~e74IbN%t=MGa)5SycFUZbH7+W)`L
zk`rui!jV5<I(7dRu-Py(&}qk(D%Ii(kbJMWB2>-)+<s91A3QGsQ=7nlOPNy&aZch#
zw<x$fE;!laHGj&kGb*c?*;H@mw4kW_;SJf-q_p6xVsR?F>Y{g$IW3T#i!Vdg%+J>5
z0oMsGf9;gnY$Q<hgVgN~h*h5I`xBhzL3~`en@dCuwEsWZAN9OST>JlJ@+PS5F;-XI
zuapM1GxFDNc)a4W<5F{p;t6KNnU=8q|2OVefa3w1zQPras%~izH-Yy5n+QYZZcf-w
zBWPd3CP`J;{{R2|-C%dZ?7+7FU#jt$?6>V(z~;u?xTRdJZ~$vx@bx0xdtJJIpsvXW
z+1C^N8ms@VC8fgCNcg%`HK(2r;4%ZG2N(V&q6T&!HhEz^$oXr$+KBVUK<a9izX#9h
zg4j=!Amz21b(M<y<b^0^@s~mFS9_s86U7}3BDP?4ht^ty+Y4Ks+kw-Z)j?N{?CM#n
z)ALi+1@ptyMXeUAgVz5s=uTE+a=WK|tubAFqkjiD?QwkYRJ-}Q13uPxZ!P4EsZ`kp
z^|hf>)h%8bs<?^1(lGruUBi2=k;=Zim1?g|cB`NGzfmRh<Qw(5CcKJehi_<@dP2^d
z2Kfbs^<yTf&wX8_QOkTlrM~m6T70*(TK6_Xup2ccCa7%gJgD*c#$0u;DMIS|cFj`>
z^T<;AYQI-K%jUY;O1@szo68f`EUZ*C{_I|>&cRouJ~s#APnOCmaJ>CHEu{EA_OQ~0
z>(e!gcbr!X-=3&0$|ea8qi-Mc!C?Y&i#OXeB@gBW>gg-Cs{h(?M`?DEnrhik4fPpI
zYQW)Ftg~NzwMHRWz1vd6_<ye>ySn$zc2&^)KmSf9wZA<F!EOhc3Bm$F8^G!C`CNAO
zyGPE+TYXumSaKm&`9}9?b-^9Z8u}NsmG8*RSNC_SP+Ot&MX9^URRuKvznt4x{hCs<
z+Ovb|8d_>$@U(WiN)+sdY2p$p6=`m2{Qo#K`0L^|-rez5l~ITR+by!+T_gFeqGF!@
zEww^DJ+Rorw~Ff9ZeCK$b?yN3!EOs<P`|5jMv;GyhMeUKSFrfANDkG4HzsO3b7Ixy
z+ntq^Wdqa?O-fW-{&|rGX#W4^moRX+g6{wObqaj`pYkuM^<ce)EkUX`+OpJ-P0;|a
z|KoeMM}4{2HjRnzB-Evym#HLICCVi!T~JH6-Vau9#d{K*zTPcX*7zF=nOlE$F-Fbw
za1~gd_by~FR6=W=Dreyjwc|(CG|FdBRw<2jRxkY`uKK%O0~{`yhm2G>3Lx$W`HTI`
zCUBf7E;^#dWs|39x^#zn)6=hte}me<<w(iIYhZn~`l@O*o+9ebbMrLhx3Fn6=Dk%<
zV3$+Rl9~s$|A+f#^&-g`YHnhXGymQuCaWgjV+Mym$esz^9qL(@N7XN_;#Jqxft2N4
z^L^D)h27OBDXa(k4W#b(KFF9$M<rz6Q?cA^jnj=xY9419HJ1P9(`dL@qJHtNlER4x
zu1Z;Q&(u%yR)gIS(hI^%OMfVW*8f@Weyp_a(o~g)YKOt@gU$ax{`^&aUfCC=>!}yi
zRWH}6otW<omR}P3NGYiAKiI6FrI0+eDFM>o2hIOmZ$G2%qq;(k?bs5qdXQZ$>%CO#
zgCS?JW&ODV4lmgKf5`Ly&b42`VVF001z1hKyP;~h`g-vCKae?QMLHTCdXT=)%0kFG
zjkBh6)cU$7sD<8!^hx~Lrl`+X`VY37wKz)ksz8@U+p|e(+sq3TJx*;=e{!`_ox^Im
zM&0F7Rj0EbHRQVIXsDPhP`x4>pwYSepPKgJGitUQ|AO^{>;&Q2Y3-^VQi$_qJYQO?
zer`PmR--@drTVsA#u|r&|EoVTgY=(bj2$)1*4$H<HaV@aqP$&wmECu>>l~0dq5ntg
z)fY&92dDQbj>76+*x#%3-i}g#D%215JIKCMH`&zJgxyqs^Icr&+N6!@{2OPff!6;`
zj&4zW@Sspl*#3}u&;KUy`akQ1Eb4I{G3uwk+)(f1Y1D8K_R>&0?Wg`#Lsi4%N1#g5
z#MA1%;RoSngXaHhPajavsyU<f-@Y5{2k}FT)SuNZRevf~q5ebciu#+kVH&6M!<0Vj
z-cWtEd!cf0^<Q<?^}%X&vA5JCz2B*IzNuF`^=*N=11Gzh(Wj?sy1x&rq)z{#achyi
z#?>uV;I#aDwlBOddgktLHQnw*8e3~7s3?6hQ13qZL)}<IMg5J@HuW>#m#WX>h4l9)
ztab(KwYbKuaf9!&+JEgE>L0#A_URqpWT4T!w@AH#XRk))j0fuL4+?33#{VZPZU?88
z)M@iISbs}^%V1C#E}8yWSt1-Vw|8lEw7T8tnW{m#dKyLxDpZO)+*R{TeuKj#E~-{z
zvQD<5O;8*-ETZG&R1P!tsb1CJuAX=~UHP$<l=|YdJL+2HPt~||Ji%)2&DyAPoe6TL
zpa1JVHPHNjMUsKq_q9^scm>&SbpN&@e^R8n^1U}22VcGf>jQ~T?QT{SzxP}{^6(;6
zmq}%6+=-<c2HShoqf_px8|ST2-z_4f{`=Z~^<ToP)Vqc4G@gktfc+;?2kDcW)GtzH
zY2j1b_vXA}%O_^l9lT$`>;E<$&(mO*)CPwi$Q}?*exwfOzl+$ZA|7w7E^#qlRqe)A
z^%ZfK)lDb91iSTP2IL-)i9&rUrB$UWH(EEV*PilL-O<IbI#2PC3TXYGquo@n`@a}N
z`r9f6n)?&8*6&-^R%1VlZRY+~=WqL7-TZFXVZP2TA}(*=4&Ax74L9}es-AFI&o&dZ
z>=e`3HOG*_s={pJF4N2Jck)ii+4X<M&Rug^w(YvonP}~Lfzv`TIn`WM<gCTNa2bmQ
zev%f=XXjaXYHL_bzb|INTU2KuC)Q@+mT-TsR3GzRoib0GWxpp{Yvfp23%uvH<<h!t
zlVquE7uLsWE4ruOW><RN-hNJJyD8gV?bXz>-4`h(Vz+1V3R{1vn!V2s+UzU$xMK6R
z^s%*1tG6vfzOd~>v2QjpKIeDudZxBl^_7}Uys@usdOnxkhU%4j)SBn+b}j1Km)-ei
zU)A2}c5gS{+Z(WY)xMnd+BP!DjQjl0``J0zF1MR*T)*#ynWVktv@Clg*?|4?i`w^_
z&&t~OXF1RQ#ZhAWOWt_zm%A5a|1bQ)-djpCw%1HOZEYUT->XwOb+7i^bGELUEw+ck
z5^eXW?6!Hb@Y^1Tj2Tu5wY|3cKTg{F^a#K0n@MuE>N-qzQEc6NW$goOS8P?>dwG6^
z?P}K_dpcHH@A>-Na^I8%qPC}s^!9waA!|Fwn1A2x9lHBAb>6nUC&X+g-{xV%(3!XY
zK-RK-k2CuAomX$Szp;JUzEUSK`*kYv2aL+wY?d?f*z2_l*?ni-vzMdm^<Je&^?mW(
z@Aq&|u(B0cZLoLFk~Z7sO;>E=T)*yKw)VR1KF(Qt;<TD<=NakSt(mO6@4}bGc5c5N
zY~Gcy?A5;IVZBeIes4f=^S%unlWjP*Iq&_SK7Fr=;E}!D-=6O|?eK2zt;Z63vv*hR
zwUqVQeapUe&+_S?_68s3-|Jd+W$(w%m%CM3`u9$*HQl>jr_a`V-zwWRTNU?Ci<h>I
zT+g~s@XgKLG9T0S2=Csr_i@auJ*LrIdmo?avn}bkw72k==3ZTfg?k=&N9~<ZWMq3{
zh3DQArYCK+ncwZ{)@ro%<~qCgZQhYxPiN-aW?#48+u<;I@45GSR*N+j?RjUQY<rYJ
z)lT5%;k`x9f_6{RCG2K?W3Ud)4c*(p@O+QgwPstsps%*iAEfR*UGKcl&fnhdK*sF7
z@pU|VCCxAIExV9qCnaKJ+su%+FMxmfp6^dL*qX74?)wxn)%Lny$lkMar1u#uQnj1?
zKi@j}uk^n1>i2sE`FLz<CGXhSi5;+M*kZ7^+e~(E;AK18K40E_dLIm|rSD9z<$Tw)
zyO&qPmN`;lZ~e4;HtUsU+0Iqr-Lt%H%HE%)zim#cG+NJ@zI-oRsGRM}2Zwe^RtehP
zV0PQfT)1OTANP~JB8w;N{ilD;>Q{I7-f{z`eJ|R#?En5}quonGHT%lF>idIa>#Uis
z8toI}ziU@7c5Yu~pxyraqK5le+^_9+s<~(TgROh7;r+#XeLT<FYzo`7_pHFv-I^XV
z_eO5pwl{j&cbn`cg?)J)v39O^9@_aghU~lW>9ft!xu5rCv$EUeU%$Lh!_~xY&HvOr
zJNqZ@%TrU`FMf8C9an?w{)H_zcFu8k_j9`z*)wHru$#8e!2X=a&OMw=y*8ZLOnWvs
zwCs5=%DYeIYN;)U=BvH8H%jfbpSW}H_br09ae7<#`wBnb_b5Wp-v6qfed}$*{i_~K
zwqtxeXFp4rsclh2&i-TE)wY${$LxfrEwm5RU^yUJ_1EUsoYQ-^IZ5t!I=R9Aj=<Hu
z9kUttS6(;Rm%OcX&$4IryH+pl*&E05&*mUwsqM;grF|LOx7yDC%VJmW>~Aw`YRg`y
z)#A2Ff$aNkUpLx&effVIgY!DOI}f$ncHZgV8>s)sHbpya@8w{Yy%k@7@Bd}MvOjpj
z!u?Uc$M^aASK80);<29;vv&W!^p*B1R`&Ms)-UY}z6#o%s>-&#5>dXl_Ko)5GhbiX
z+?V6sW5U;K%RXtzZrLN#_D=ZCWSf6yqAlyf+qUKnnfu>V@$T2^VX?pLWVAnd!-0JV
zei_+D&zx&FH@VNQBtUZC<~s+hvR_8oaxDJ2JAcO6J%2Vz?c4F<+U}VTXYE<7YiE1s
zOpER9z#99LjJyZ>#Juc-jo9`J8Rzf&9sPO#zf+w1>-$aivk27LtIyTg&l0t6|B`je
z_HI7h_NTUK+5hbmvp;oo@jl`EEC&osO6)47`1YSVb<<9`&~RVt3aPzZc@g_=FN@jt
z%u3Vt#{Y(Wg%kYtX@$?;+kVg8Zqdq3`%c6z+T*inwN0;v;9mFNEPJAyd-nEZx7yD8
zG1K-<kix!AH?Hn!%T=)1awvCi_-w(wt&XjG>ux=>(f*aPS5^J<UVpijHp%+)_xk*h
z*~hY`(Z=-`|Gv+YF4(M)PT89j|8uX$oa1{_5>)qvik{wk<kNw@6}i#2@1}_F`&9FA
zZ<J`b-F@|wdz+*7?2T}EVe@{P$X=0s4SO^%-q_1Izi=;Sx15c*uCi_Z=RJE{->uu*
zDwS(9n~%-bS0dHc{Ft2WWtD|{FPTo*JL6=_9u}Y8J*!Jt_tp11?ve3vu`N5JYRh@P
z$>u}HLR)#Qh`kqgE!iExu*0^YaFy-5@40*B{_ES${?=^!AZn5ALjxAuAhpMP`Hh|S
z3hv?CyFNJB_WT`3dkX;;dk5Xg`=_opIv`th$UccPYroi&5_|hoeEY4}so9@fm$+ZN
zRMg(bGu%E-$Yg)U<-+~H6js{x9-O`ZxSEk&80!-|Hc!|6OCQABx@I|BuX0wm72R=o
z&zmE*HnXl~+H`L!v}F>SvG@4I$2KP}3E2kcOWUx{@w45k{Lwz%zsycZI>P?6b=Urh
z7cSa0CT-cj%KF_to0wPoUD@sT>+deuAG^lP?i<e%+qth7+Sy!J+&A0lh}|xZxBFH_
z+wN=nGSlwIgeV)8Yl(Kda~|yDFM4EKtTDlE+s-w6um8{AcQxYJo>`9jY+GYZ_T?_u
z-=9{t&~|Hi*WUH<jr+AYm)g&<Y~0`NnrOfK#WlO&5Bv5XUEXBRcRkW>?gQq%8~qjc
zZhZ0D_T+ip-Cwr!?d@pXv{%hUX`jBCjm`S<7q;B*|L?teRM1w(jdh>xP6OKn(Pul&
zPEEDZZEM+^_WhD=Zt$bMt16i6qKu#InJ7_g`~2OGy#*In*ak$z*w}9UyXVvDVB1w|
z%=TzZFt&9N-(s8me4b^n;iG+D=FLA4mOp7<_wQqN&zG<qP?)sW)??a6`?G&1?ynZJ
zwh!SCx0m^>vNv<4$o@+IHMYA0&g|vN%-)}tE@vMRe`wDNo1A^^I+OO!V3ykdD&y(C
z6K0J2h4)Uin-d{@pe5bj?(f9O`|`T?*^69Uv)}aq`~KysUiNF(iSBhtSJ->%(WAY~
zuZZn+5l^vYuCm%YVc|JjxwdV#dv8tN+n8LuS2FwUK5oezJD%ng+uYOY_B_`u_BYII
zwJR$#+|PD5*#6Gsj(r@@_Uv1{m}lRU`9XHCN{aS!EZ=Xv!$fp%tH4RC?29e7aU!es
zvb;&#6`=mw_KwO@n^dXFy>qmd?rCY5W2=>Cv*+~6-n|zNU$+fd9<hf(oWbtpf7`vj
zKa2Nm@D<v(z*2W#QOmczlerdHDXw?i_xN|H?FPjny9GBm_BGCK+o%5N(cYl=P1ao|
z7xvbFd1>3gyM9kz@j~0U?SXr3d8PKv-8aQnPuP5)j!WC#fa{EV-|?>A8$FkE-y)mv
zy^F0m_LZ<m>|5}K)AnA7fKAND(!Dp1O|bpJG;wc-^3y&0UjDJ}{w!@9aZ1Kc^re~Y
znFsf5cWgOt`@rm&Ew{b<{t53&>>s9m*yqV3dthpKti4F;5Boz;w%VOLvva?C=$-vj
zr~cdbR;tsEU1Y&t|0&D%$y~ABS95*R-p_N+>~d_7vs+ixxcA*+y?ye3lWZ?Y-`JaW
z)5CVN<@vpHjHcMG*mQaCf`wIkMejP>#zz*|2=TJ)(e&2eYdz`VzH}=NyM2-6cH8HA
z?qgY!X>+veg56@d1@=YH6%TaH$h3=iIB);EDf{=VUB6^c$(`M{`MqtnQx0FT@#S-|
zaX<avX5;&~y-w>^@4C29XK&WIM|=0Jkhe3vpKHtLv3{@2DHprL^XJ>>JNE8PRZ-ig
zE8S?zbWGDW>gPP$Mc=RQePB4v_Mxub-ax+}dn0deu|2B4&{n2+rEN=b*xvcpGpwyY
zaP7So<h?h0&WAk$``!2cys~!hjN7husy!=hC0(O;+v@$?+gqr!uh;X}-UyMLeLjw1
z`-B7!><PM)wr|=MrG4jKd+yUX=U{i4MQtC4^2@#7jaBz{z4EuY<v3|Cd%&8#C8uKc
zGG4o6yVv82t@P}rwy&md+$(l!<zCMT3-;goZ@2e<>ub9Ow^Zy+^y~Kj&ir7P8FFdg
zuhVU|tV?D0i{3Kce@1(X?c)QhZT$K)t&F{%TFv)3z1QwVq3uIW+r4MD+U$+KJ#X)V
zm+gD!Gn?B~Tv=^>P-Two!!I*!zh^A7WjDyRU3y7qk4O=-?JUo;HhR8hwzf%ocUzWj
z+GC;i-*%xJr|qxH^X)WQ4flQDvBNf}Ex_*0UyFU&pRVj>+0<n-U;o@*`CjgQ>n1Dh
zwd+2%$IFz#?(N5AwsH)M?NnC&*fVwAls#K6<m^p){CPJkXY1}4wVSOC<M{XKE6lgG
zQg7O8_jcZ%zTeaLcG%b0uz$(kEA?Qvt%d1ETiMA6_R9X;vG-#1VjET+?|s*mcI>?+
z-?sNi$g;hOvz7NHJk;LZyn6cHFjlGE#?2fyPWBbH59Z$4>-*{Wo@XzVY?*JIu+1^P
zx9^65*glCLta~lJC)$K&CfXU^ZQEORu-9(c0gZk7x-EMOl_%}(N|4^CFI&Hl!B*17
z#^c=HuZjovO^Tgu8~vckuEF=BO@6J4UBe-XeGvwEdyZGlv7Kx*+4ii&S}X6HEVe#;
zI(ur*J+lebH?sZ5^wU<`Mr&XD+x)%aZq<8gUT@ypXrr`O!Nk{Yx#7#bhi(?^`N_6%
zUzsN7KAWF6_7=Roy65HM6?=E@*=-}>_s=Ho>N4AEt0*hBWykmS>jdw4{`8Km_AI`A
z`TckHii>&dy1klz@42QToBh`p+0M{!+xu>@^PUG*y0#1F@$LPy#&*y4N$s`|J&SDP
zy%yUs->kNMHp6+}BB8jwhxMM=8ZG7BpZjLI?YuXYw#MGMHs`O(?OQLQzQ=Zr!M^=j
zOndLmY2RC8+i$Z$Vg25%`QL1(2OZwKKH<{d?$tf^3VKuQ1orN?%Z*uU=cL4HA8<i^
zzu`UU{e9B!ZEU`m?vHpZx&M2w!huD(N&7EedSO4`n%nL~qQrsEC+FEQP3JpcyL+#_
zgN%ax98R5mKYmH<YugdJ@3unH-s-qYyIIX@`<9rh?qhj1b)Wgo-hF8wy!Tc49NqiO
z__6h@`Fm|G?7eIXgF9@3%^Yl|Sf1V6!&z^e^hm{aj^VAnb7b7?+|FFytJ%HD`uT<v
zHpb`v>@|HfWzUb;xP2nKQ*75}%k5j4e`3$PlT-Ev_1f4yVk@^VzH;79G;7&@zhl4m
z6}acy7YT{&H`}w*RxjUUU&D>7dyRLe>^s@GZr@+`x%;Hwh1l&-D%w{vZRbAT+O~ba
zW}UQIdV9WIv(E)<ewj~pS?QMciWPbXVh^(K`*Og+ZdK+6d!d}r{kLbX*~hnd`u-wM
zj(wsv6K%FyHSWDAyMHg&J`tNuM;BO0+j`jUe7w_E&-~1ux3T(r+NRI3)zCDtS*IPe
zck9n-doI=n?ag_}wC~Ni#kQ|@vDo|%-@13n)cboEpI^5-K~8GV<HQV`km(%zf~|P$
z&M>iAD?LAB^J1!+-Fn62HlER4HVc-W*mwFM_kN2v3-*Qep0l$*cVVCAs{?lTin#XM
z98|MgcTC2<HLr2+sodKA$^XOb#bzD1t^X&u@6o2Sb{E5#?Iv<(?2nvjYv1s1^WLZz
z!TY-yC+=&>&#}9{hRrT<f`*;VgxR*O>052@y{z7MP9omUXbIQ8h%0aQns4FT*HTz#
z>w0a)zGo@(?S7uE-1j=O(oWqyZQnW8K--00JM65(PwZWmS#JMaD{Q~LbBEocd)oWE
z!~5)}ZaK3r^+t@{zX_A}?@t!6)#Ki>XHFmQ-fnYG8&379dy^!^_H}*SxA)gBKD){L
zr0wJ*IQK1)oxE3dhlXwcOetI0j&QrGV~jS7M0fA;ICW+3E>FkZx96<dQ<R`?ca#0l
zUbV&ZE#I7Vu(i7<ZL8USZI6Y@=Dj~oDewDnk6~ZtE3Z8+`}+5sUCX%7%Ab8t$+EMy
z`PZNB`4q6-D(aYmUB?D-TfRq~d;hDP+}pNFbl<J%3-`FJ+P6neUS!{nIE@2bH)h+p
zum5g$;mrR1x0bE5%X-9V=XBC>-^O=+b``>=`(z|+>}Lq|@4w0@>~Ma{(|s}<)%Q#N
znZ2J^H);RJXW9GoIU@I2=X2~giZb5Y$<DBE>FeEl8D^{5Hg&n~vhSI(C$##}-hQj+
zw$5uG?cJ$x#zyL^@IEdjvwh;>dHWQ<pWgdGd-A^GYfJZWFUsCmw<^(2gS*A<qfF$!
z6_YD$%$85v%W(XUwZT=kJzF}K?lGO&yjSkZj=c}&oZH(utJO9s*xojM=7jwX-e&t^
zzwF&tP^!6qdU%rkRW+vlvL9{i->&Pmzd3L5{&mJ#cJu$fvpXW2ZF}Fwe;>2B%Kit9
z5A8NgiQE7GDW5%WSnmGE;mh{5?Qh<<Gwr|KGQLv#>^xpOxm4x-byFVgjqtx=t7PhG
z-x|4l|GZDRc8$$b_iO(Y-&=L`!|vLR+WW2v8SI-L|Ig<7N9Vo#@7CJPSTANb`QmSz
zxL*Q$>&_S1xt^c0zu@Mo{mb6=?n|7rV*j2-2?yTeYxW)f&Ae}-s(^jjWKp}0B>p{<
zrit!b@SAn7o|K7g>9sF=64Ih=)vTZHZMxCESFo(wW^;7PUXho3_lBB_+D?!=XDhX+
z-6n@G&X$X3&)(P71~xX{jka$~tM_y&C)nJLSYRg`Be&<HQ<d#xF5|tA(}nGtCJEc!
z4RJW&u6W08L%8Su*;_j7>p#obMqX^PUG$}O&#I`~dpy<BY#*GCu=%lfp0&{_mVKUl
z;(M1^pWiFSB)z{i=jgt*J}UdUSlH}O7EQL>t+LHdGDm0M)gakD1#SE7f@@Rkw%_`(
zckQjDJwN&0+D>`9Zm%U5pY4peb$fptyRnx)j$@zB)5&{$JE!hV;#;%VCMjsY$ixHt
zzs{SxU(d$a-efxS{@Y*w@2gT5vHP?~#6DiS(tbi`lie$>$NQ3>a@dJR-rlRbLSmn6
zhvdGWE(vy=t*iH`?m4t~%|h$FkD?duU9?1XFKfb_y(|{LtXF)VWy8p`&Njy+eecam
zG27*#T6-5~_1iSgIBKI%B(V2V{Q}!%5=pkDKY46B@9XT{yjXCbP2oN34~v%Vy?1nq
z?bbi@>^AQ5-5*oB#^$usR=bcQ?)}Xgm3Fi5t=u=S-h01>9GiXEE(?2`Oqu;L5C7Pm
z@X)qzpCoKwcA&&&`8TV*mG9>6yWL;3UoJp*-`qC;JwLOK@0HptXSdI;-%h;EdH*GL
zzI~Z@SJ_QmH`Q(;1GoK(=N9`!!ld@ISNrUf`OmOl)vV4=$E9cA`fJ5|&&e;^C$(|T
z-n{u;b{lJF*&X)Wx#!u+FMFr9H(USNsb(|j*SbB4lke<xjm+NTUL9>Kq$6neW|`$)
zbJL)`J+}jFGv`O`{hP+SZ%QZA-q@3zdv8=u-Lub2%_eS+<KB(0X6<2NT5pw5!n`jq
zhI#Me+OxJ#!q@CId;7xX8`mV8%%-Kb&nq6=dY-h|`@3Y>9x3a$djr-q+Hbue=Fsu#
zy8UMBEBm<<v+S+nJoe9DC}V%%vf91{++GJ<-0#_Xcf{GH>gU;Bum8O#Oxb*obL;ax
z`zIIdd#5j9`>a{dX62GYd#&D1-^V`D;=nwPx%Rhfxb4>7U1i@IcW-}Sb@+avgO2tW
zf6cTvyS{dROywrK&_bsD42C8)lh+pR*N#4J%jB+TSN}-Q{;+55{?grCb{F57?wj%>
zX79DN!uFqj&)v6ma>qW&?(}`Ne4P8DZA9#L*?h1$B+a>hqiNs1MVF@9rkB|4d$#Yo
zmCI)-+qV(Ed!5C2to?5_*!p%B?_JY6Z})O`mpz^{_U$(2y0DieYth~d-Ku*(zsTCd
zwe^Tisp6@<%MY3DnlW?w-lyNqY>V!k*kjP<XRG^8Yj4=<TwBqr?Y5lCrF&1Wm}K#>
zTFp-PQ|;c#!B=eMM0pQP_}^{k7`)y7f{XV)!TBlsS^Zb+mwk2JR)6<L`*RkW2d0WH
z+dt*yeVc^%J$tilA6nmfX<<A0_0_$P*;Mxi^HlBq@3Um@RMS&7*}PJFH}&qfz9DC`
zH{=@QzE{(<Y-P4Cuyso}-XpB5Xm?$Gn@y}o{~pa(3->bngzVYa|IjAMn|t3vNtV6o
z`%c=-Z@q7GcFKi4KT<es8eSgR`z<19Z~Xh`dz^#~_hkJy+AEv0dpFzbiF^0G7PhsF
z=-+!Zc;cR7uIjy?pUGM;{h@BVvsYkmtjy#+_Y`;T5#GLhZ(Ew#Ua5)GZB;i0*=`gT
z*}aWv$=>tZ@9c3BeY2PUldtU+4%fY5YmeJbUuIx)aO!=VJD#n3*FNmD?fPGCbEm`7
zUixy_-udZ#2j=he+`oHPu>GD|&I4b!+wIr6#j!7_Bh7B+8ZG<P&$jOU%gedXcbcmm
z&zZ%xZ*FJo<4HWYw>D^+t<$CoyYl#jd#^uh+_y}S*{(U9ckhYA&-Q4xW!T;o{%@-_
zb*b%?vNwD0-D2J+_UqW*i>Zuz9@xCGzsdjC?x(NJf$hI$+WDSiwNq-^xc_;_#r^TW
zChTWh@XT&*h?0HN*({qCIdc2<<#p}7Is3xy$5kh6R8tH07W+rq=+DjCJNr$Ht#iTp
zy;V1*_WFl@wD#+6-o-SrXD`F)Hrw}iF6|9F>$P_ai;%6P$_886K<0frn(o?9uw~zG
zmaKO`bIQj3#{5hN{!e_cZ`#f_`|sDR><>TFwdWL#x7)kdb>A*!MSFF%jW*Y$bM4bA
zo9(=>?${&O^JibScY&Qn#o2v+HcR&|T<^HIJ0{Jx__eLAQD2G8g<iJ3%=cLAs{TB%
zKEJeM*QKd@_m)5Xy2q0Lh;^*A@IE7v#J%df<@YW7k+OHOe7KD{<IdfV($f1DmF?VH
zv;4{4&QjKW3~#M%{p@zxWI4Ut8}c=9?>xDNy(>h7_VLD_-u>WO^xo4)X75R{S!D0J
zWwL$Mw;KC|gA4YrxMF<Z*o7#EvWouwil6xQ?>Vl$-~Xr8fnAFO?LzLh??38qV1NE-
z=U$N`&-V)MS+lS9hL_!*>wEWY(0XE5EgH8kJmi(lp|7la^=z-(b$xZU`&GYmUzM@-
zzEk&h@2gGkv2$9;xbOC5hW*XqGJ8wJoNak_+}|s<Xx?70m<P6+^1`-#ZMC+dJ7(Gz
zE?c?xpM$JzTfdQ=SbERi>!n@$l73s*7Ft;C`;b>*6*G0gz5^|)dlMtS@3nf_w)Y5&
z<zC-KZ}v3r_`LUPeDdDxH(9oiC-~U@KFnaZPHoB_ACdOG`MV1DCcP-NOO;Tzi}2rN
z7ZrKh*5nwYecrNP`_rOC?aedn_idiNW`ESe?RMR6_WN2i66~IZpWGYmq`!amJloy!
zcbM$=pMGQ)eL~J&?Cr8$fwO1q{r-H~{wyQ2eHH>c_l3*e-hbJP$$s^Pxc!F~3)=ez
zSMS@NHO+1Xe}?_FxrTOKik16%_p9uSlV7~IT*}zy_sofQY_l2n-QOOwFYbe!otb2a
z-KCK2w*OuEY*~M2?RztUV_$jJx4qv4R@>;!+p)Ja^QP^=9Z&Xp>ay=EcCN7%`nO^C
zhfuM-?`FQ*+kWqZjewx^z6i5DyTc~1+v%@bxx3g+*LLa2*1h=$<M!p;$lEt>e~WEf
zjOV_;%e?mql$q|kcf-%FV-N3M)%R8VYOAgHKH9~+_f((6zQ|~Sz3b;b+WXx@(pn|^
z>RyLnuf29ps`tK3xo?|PW3+do-x^!z*rvVm^C#Hcnf-l_>U<{~RUNfG9ytqaZLgc{
zjZtpeo5W~-VD6gF`#3N6??1@fzyC=`hrL#Bhb_;7U;FnlAF@xlrm%lwL7)94FPpvZ
zuTQjQ%~@*wYSPlZ%25+-`WDaLdv-_B-jk;P_a4@sX7f+#<{m-CPkTkzN7<f}s<e4{
z>&)H_KbiKO3vb$+KH;aWXdZ*zE<sW2R|SVH(;`{+S>0S@yXK7MJ}I6rHVfL!ZJjn2
z?Uk~5Y%@27%Wl^qyFGu}G;M!wI%@OmvhCivvwZfNB&hEFBl%=6_ia<#k1IrM_&3e9
z{n31DPj!j;Ue{9Qy(NA;)*n8|@3X5|xA)zVLwjGp*SDSDx@oWDh2wj9{#~}=sQR?`
z=?ZyUkC5p$e3P>GKI~m%rBZU(`b2}It=V1Cy)&vxZ9835_c5w}xB6Ylu=j@L7h8Y(
zMw?5vT(;E$mbO2N*V#H+R@sUk7qD&ocV+L)N8Wp9UzFMx*r2*cwMyHz?v?J|yv=3S
zb`vh`y|3(O`^)~)-tF?rdv7Gx*kt5o*u}m)XLmN1XWuoZD>m!B`tA0tblO*+H)UTy
zP2xV@wJG~P>TcNQc$~|2o5R|@nfpR^Kk0aGoBjUto-0!q*)rzw+XkN3wA-99d+**(
zrG1GOtb0?FB=)VDuyW5NKG}UG0=|1xev9sX^1H(33Bz%lE%uyy9TxHK6KdLLyH+R1
z?xgrnJB$3Ob_a_M_szS$VDH8!+;+1$boT3IH`+ekKF!u-#Z;@aSEubU4Qbmm{pwEJ
z#wlWZ>$P^-xb^nf=9n1RPCYHT+lyz#UiT*)2et*b?F+e}v;XW78GEs{HFlh@t?mD;
zn!ewBh4jAPG7k24u8Y~vEx2scX*bc<!@7NM*Y@dqo96!Bv(Y5QMs1Sm-Wq)tyWjU-
z?ooEVvHR6~@qK-|Y5Sb6FWu{Tr`JZHX~Evp%9m|7l(_C)|44D~0gu+bNxyH{sGP{%
zy}+W=_H$3Yjr`~RHjU-W_WTanvU{VW*gnbsCwD*g_O{i#chEL-pX@&E-Ld<6u8Qru
zDs;+@@!b-uGh1HoZL~13^J6=>cl93zdv)a>wo|Vf@6}l?x>u&DaZin3u<g#x4Eq-4
zN$=fcP_b9erEbrzgll`^b}8(&TKRepuW_aA2J^N(b8lMio0Id~_KVM>Jqor>dk=-h
z@3lRAaF3#I+TL?nUiPVh()N3|_Smr;{Iu^$w8H+5=cW74rSIPtR3d6;Q@_#feF5iz
zV6zANRkMTbK0e%Pw?@5nzm2k<{YB$<`&Oql*#CN&U@uv*d4FT_DLbF7|7<-%cG$()
z9I-VKmawnr6}DY7Y4yH!q8IJf+`GST^Krg?Kcj!{y?U!*ANx*IYhIUQdml{buoZb7
zXX_f!Y8&>C->z!!HrwRR3pVTdb8ORe*4Wl@i&$@5FS@T?)@X0%GJD&MBTsF;H}mXs
zXnJQ`cVf$)6Z4MS%6BAMFJ1I#uNBwMy*_E$d-I%oY&j-s+V0=fxLd1)*Y2GBL|cvz
z*KFnFR@*qrxbG2@+q|cdcf#I_v#j<MORwMK<>I{OKZnv@j!X|*?N=Q81m^bbmwnN;
zFY)M=J$6=JyFZC~*<ae<Wyd&E$mUJn5j*>bUACrEV(pxHp6@+5k;^J5R(&rAw}I7V
z=dIRit<P-N{1M(~Yhb;Xb5;G`ed|nZw>~|vr`e#!Hc@=}-ct+j>^-<{<(^vevONps
z=UM-}uDdVtoa630HsZFbaiaUI;~K1=oH@PsL}cyWcfVHbNwV0vcctw@n+tjZcH6q{
zS_duswwFzTXJ3o?58Jq?NwzKC#y0Q2>FnWsYO^P@Ja&(|=5$-D57TTG2v4v{*)3?#
zlj>_<d#iDuXu`t%Pg<4jxx$$C&DgffPWrRt{<6MXb{9TJ?XSB2W=}Ymv|Zx0E?d2I
zOZWa?(zq{eOP=i+N%wtcWjXf!W8Ay<hOg>g`$gOL?&Qq3pB6C7es;m8{fly6?PLFN
zY+q!m(q5a|`}-S?2prJf=eD1rPV2xn#tHlGRQ=iO)3nz9f(y&On+&V$Dka7CSKSu2
zJ9^>s{uaOW_K&+)?0st@ZhOX=(>5h{vhB65jW*M#yI85#Fz?&`c-EfZS0nf6rLDF7
zu3~8acZz`Z31j;MoBpoce`4hs`=d=W_De2~+vn<X#C8ow?fx~3zu8??>$eS@H)~gu
ziLK27h1R{zYA^Rx?7m|&v$1UNgYcr=d7a$0vwU{${pG)R-%F#j`?B55>`t=G*vHSR
zW5@n_)4oFthxVzo>e>l6*4wR}7iec_FR|b3d8mCcSD0-;{C&H<wyW%FuK(UE^J>=q
zW5H&2?mjyE1-;(dg(a8Tx7_mCue^)x!2D;^_G|Jg*r)Q%+W+wLy#208UH0`yPTI!5
zXtQ_s-@f-nU7n5b`Bk>>%Kq#LXWh2v)g`5U>P2sD3j{CO%-$|$({+6I-UB<2+VZg9
z+xwo|Z(nUEpKaJ)`F-}Enr&8oxwqGA>Mq--Cj$0u-^#SFwARyh`=d>^oAu7x1oI!>
z^WEj{-og*x_cX8Cwf6|axxIolkN4L7SKSw;J8Ab1t*<uf#?SURos+N=3vbvPV!w9p
zS(oLuxqE-xJYZhDcV*BKn;#s@ZR=Q!ZO$Ev*jIG4%XZHWw|)1V)NG#bciT53w`nia
zru}<&)%)(d@-cH?R>f^wOFPT`g&ueJuJh`)GZCI;%Rf7Be`Sr3eYx{oyXr_@`vvzn
z_SQ(Z?z#E9-!^OZ+`SVrAMJ@y-M1%Vy1<_F)8}ljF}3ZzU?gOtV8CS;cX{r<>+hyo
zhwNRsS1szU&1~g)du_HZwo$w1weRiTiFU@vKUz<o>}vb>vF6_6hgxiH&R*Q>U-iKD
zxW2x1-obr)_s;meC+3FuUeVItJ$?s|?8$z7Zr>L6Cc6m&-F9lZi}p@>yldZ*^-XpY
z|K7CQw)Mikm2(C6Yph*qyElw$kMfGmwhBj=?!9oV$(HfMb{k=?T3g|MH``mLb++46
zs%>BYox5+XPMh5}rgr<QmTUGs>gKik<E>)<igoe+yON#u-_B08*R;B~fA5(+wzkX%
zcbn+Q@69SYyO*;{*fyCpc5jS8`QBHzFWClqPT14^d5*0&f9u{b;mUpAV$}Dfu>RQd
z=>Ph?%N3sOwO}^hSGbGaZkgp;8;^$(dskTqABdHhZXd?oW&d5<>_F??mVGteS_jzJ
zr0lDn$shO<(y%}0w53D;3;BI#)GPNG`X}u@HA8J*T5YQB@#xCE3+FG~Z7+3iPmO_y
zt^D`zduldauvNVqy*DLAckhAB^|s$OKHqzpMQfk5=kC25H%Hp-+BD5>fue<-v!IW?
z)1i&~IbN>WA6zkW|NA&5hegMJ><yjOWAB(YZGYVag#-P3LI)0V8}7^7YP)Ym*<!nY
z>9uw|OW5~bbZOeRo=I<C*_kQ(PNq-WXSQ8-pTBC8{lC9g?NeS&*zX^gvA^?U;QpVL
zuKUH`>+Wwm@zbunFlK+Srq=$*B!RugY9DNV?*3%^gzwW{>1AQITUYk)SrGDfujB;v
zeKoQhY)V2U_f<tN-}6x<XixiN&AlHJWo=X~rtWR!&f6RH(qQlQ)q8D}HJ<EUzNByO
zDUn;d16k(TTxKZTyF!l1mM>|Ct=5uhHW#IL+w8YnVtr@Jg}wC~Wo&l|J>R?7uWRp?
z9Pz#HIIh`F@v^n8WZ|_HN)^~Q#b%vtey4@)2JWyuzFc>0jn;14>yUMC?*~DVeb-m;
z?eqAcY4d6BM4NUS?mZE|o%XiBT4i%=jn=+*+XMIN#;e#K^AO&*^e+2el{-K8PHAM?
zyF|RhMked-UctzBw*Nx|ZG?^dY=fM+_JwX;w72);i(PSMoA<Jus<z$hs$=7E!QIwm
za^K#G-bS|l!K!=LKNjA%Z0_?t+tjb`nR?~<o_Rm!?&VhD*~{CnwRfUvs_oC*$9uc%
ziuPQrP_j*$rM6dIN7K%j_ngh<%bqrV+~Rh(mtEXDsmyxstsS%NKL7UJ-?!LlUo3ym
zKJ#U>?Rj~e4wO5w9XPmll6}g<K>KPheLDqKe%m>d|LtLjS+SRALX)jT_h0K9x3p~e
zITq}ZPkXUv(f7!`zq_{C7P6n;J1=Y9KH0e*w!12MZ9T-x_wG!U-y`qsx+lIVXpb;o
zg6-vFVtYINboQkmX|r3RF@JAkUDPf;NBeyXHiX-}a-6#F$S2NyZV3{5U!=(IOXHTb
zEwY?zyRGPhP3B+2J?AC#ZBP1H@71hcXxqY2VDtLn^}VM8*4vzrpKGg;eRuEKJ5%>M
zPTgbM)Ben6wh@n=rlziK?%}n2Lu1d|CdBaA+;bD!&(5-X-|Uks>~f=j@8N%!yYIMH
z%--!n^81W7ZrjJL`E~D<nQ!f87tgj`k|1Oo<bTtKLyysJOU^l)o)u?pAFJ-MX5(qJ
z$&$IX=eoV&Uc+w=cC7MK?VMk-?YCFnU^lI?*RJ~CBfIVeefv1$T<qc;g7&fLl<d=M
z`fqowc#Yl02%ZCbU!32+Wg??P?CrVx7fN2-zb5K{o%x#0b_*`n?0<Z8)!xMuMC^V|
z{<_yBrE~9uvn_jNf8DdWBOJRo=2nKy?(M7hKF|5Sr{!7wz8CxN+L+bn?)5TZvrP==
z-uFM_*WR@K)qB4;%j}!UJlU>c?WMhPv%>5j@0Q#5>}R*#p}i0H-SYgqZ-w@{eV-=#
z?ztmA$8OfG>HCaRzS%{E2=Ch@6}tCyjp+gATQUb;tkJUn*2%qJ-rCEaAu7mjD-*N5
z-^N(`)^DF}ADQ0WtNQlwUi;uRdw*}W+#6`EZL`hI*_LCS-=3X3*K7;Lm2Hc>tM<AV
z&fe>`k9TjJ#Jj!w4nEjhW@E7T#_vsgr^xTIRXyOj=fpFieU})1?lyd-Z!37>sZIQ4
z@4ebHTlPNH+ibi2^^HBUzdCKt=XTj%TK3+?u(Nxw)+`&FP%k#?_=+7i|5ml__S-&l
zZ{flhd&{~W+33|A-J7GrW%J06#g4Dsd0$ZEW!viXHM>LhMeZ$*^VzHXkHz}zi3PTk
zZ_e3UcKXL&78lihT>p}7H##!zdo8Q6*ZYaw-o1QNY+A2gwAnAWVDDK=-o3L8?(BV6
zq_J-&qmZq9zM5UhwGDgX{aNfLn+V#Z>|MR*dy%@`14(8(-FhwCMyXx<ycw(Pe*O;L
zx6knU-irUL_7!L?-p6@=*S>AnefPD!&D*yiV&lG_FMX|dzX{*9(U)i6dt(DzR;Hf4
zo1JIw?a%GrTk&qe9^tn2dk@(4?UA^hyf;EW*tYyJgJpjD*1e09uI_!NFxxhM5u+{7
z{BOIXSZ424S^Ca~FPV4Wvh}C;9#6Swdwxp89v%ZFo6ZO&+oetS_I#RbvA4Nq=iW>A
znD@p^nzQ$Vi~Zg|WsCP-o40e1o&ba0w&I4pJ4`3<71BzwW!3cGdx=ZHYI4@hJ@a-)
z?@hY>VDAyJe|uK@3GSOVf9KvB#mUyo-fXcsSnRRaYM0yI@QrHwR@BDWy<1?tuj!|-
zt=9~hy+@iK?>SN6YkRZDZm+~^cAEt(ZMO0)3vCWAI=T1DnT>nRk4o5mR&%$L`Q)>2
z_wE?m8~pnFI#SJSIdAyxJ^#(dwywo;-!k>Z)(d2h?Tt1&WAiojq^<jTQ`_?|SL~?@
ze!j=Iewxjlg_-+~T6XR;-0W?)_2B${+P`}CF6P|4Z$hByzLJ1R`&J3I?UQoay^l>~
z+TP2j{P#r&P1zIVeP{2*W7_r&9~RmD-YvKHNX+`ZcF#HXyfB%yZ;RNQJ;e`a+Mdnj
z+SeB+X3HBOwO3zO-nQuW>b=gL5AB@WefIJnJiNzg`VHF&n%y?9p6uSULHM=JIqpK+
zi;KJW-ZNQdV_79<^SzRJZxD~-KFRd1J*iik_bl1Cb8pe*pL;i|2ikt+IBMhi<ka4%
z?FGB^Ki=DYcGs`H%3rtdS$x37{#oT`JI}LE_D*)5WS4s5!=96p<@;YVsM?)pn!JCG
z@mss6$17|ng;?5tJZiUZUf2t}lO|K`PMdb@UG9`=x97%b+ogic`)eLB>}#9ZW~Z_-
zcHhK&Kf6<sUUnZ7|LiT&-mv%fgGu|s`V{sx7bx3(T7Jz|J!t2i^eI9%Zdavjwcqzy
z_iUJMJ^NCj%?8%H)+OEi`(oCH+lqBR*&7_TW6z0aDz-^_3459IIPBcnIQHfK`mje~
z%@J#%xxD+bcHXnSXTh`YYq0SCC+|1xV>%&audkG`f9>}|yIpGq?Ly}}+MVz#wzIwP
zdoOGL8@uFVA$!?G{A~`fe%`Ap{bY}*@;}>W-43=2JvVI31C{qInbK)v7-F<H*eb|Q
zGlBDfjoDh;GiUDG9h_Xb-zJ)2|H)IL`{yuk-M_SEg8ha4#WtHN^zAOQFSflP`^{#1
zrnK!_{@8s#dh%?0Hp=WP<xsInQxf0juVu3Dh4-619m@sx1$Q^uUOG8rPdCeqJs*ST
z*zS>vwB334vhBwNhP|q}X?t@V4({976uR$$r;;6KoQLf-)uMe8Yxml|UO2-}>Cnx6
zqBYm-+E=Z%HHe*JtK>S}cGl%{wzi!Y_D*@f%jWsN8au)JJ9kFUxo*?(=<eQ>0PDR`
zdz)=HcrD-aMSkkuea@A8PquQ{ee28HTN|ps?|<&ZJ@pT-?AZ|Lvo|-0ZJ(f0|K9kH
zp1lW3%=TW;@3!6kTyXE+Z_919=1jB6?ccFiEwFR%>b_?CO97J)JosC;fA`bdd!NK|
zA2_Nu!|pd{!Tt>*+V-c8n%iq_Szu?ba(eglW4ZRd&x7p`8A<Io{<zG3<D>igpL6=|
zjeoIdKl{;X`=@KJ-Oqn=njPc1oPBJ(#kLPW+t@EiSZ(L|y>{P@Wm5ZAv@_UmniRfo
zt`Fn>LMBN&(Wncy)8&@!espQq-Xy(=wi?r>?$vj?V5=#3Y_GF<_wEl?m-d#_y4twd
z&D%Sprp9iGR_b1*x+8m7x9RQaN>H?OKX-Z`XLZlMB3)Oz@P$uyuYXu=JE8rL?WDu|
zZSPfB*rv)|wfVP++16Nc;@)WiXZHU6$ZdN~u5gdi<Q%*6>gu)}@AdY5_j$ax*>}_4
z!jN#g0BOH{3dTbFuJU`@{AqXHJ6*bRU+^CVJMUS~ZLj#u-FM*G=e@J0RNJk0+-~>p
zQtQ4$tJm)12^QGrRbF6IC!4)@dMo!H%kRhc9J(;cHeJizhTmS>?#YL!J-^;Bwsli!
z*mp2tiEWM|$3CmGd-wf$x?rCP<5auWy|s4EmlF2H3EJ-~uFbaVoF8u+a>dN5Me5+*
z(+jxwN<`}K?UKD{bLofa9*(cpwmws)?tORl<lbZ+oxQs%0`{`c++zEqbFJ-*_8eR3
z78BckD?8ixRm!&Zy}q^wlvnKG_*ZJ<bB)QaUe9GObKVo19yVsX-tJJ_mogH2CM?R|
zYx%XrMsvcuJ+Z3-_O9YNZ_935XFEw~rmd%K`<{}!jka421ovM2)v%YBtJ7wZL#~bR
zw+!2Tu{&&KzRlcw@htbgJG<`fEtsUTcS%Rk-lqL*`-HX$?OReQzgJZG!=9u)eS1wN
zO72})p>AiU%DC&}fd<=)azFQKYuE32p!s)K(fz%9TPMx3eW1g-_abw{URibNJwd6}
zd$pZwYy*>b?9JXLu-E5K)83~n_iaNyFx#?q9N%+yDWA3Lrj>h+@0+mqT=Q)kMeQB7
z>oiZ=ieByAD;9ChmgV7(Jszvt_uAd4v;B~9Wbcugi*0VqSh)AzUp3pGsq1WYH7D-f
zm#S*(CO2o#M!)ZS73JpH+5}tg&XTyax1jIsF7KbOY>duK+51tG)z0_9m%VIHwfBkY
zx7f|IFWUF#<y2c<OGVqMoiptAt(<N5M@VR&%L;=%DJQe{@mlKI9^84(M*6<#-U%;y
z_O5VZvAtgN*Y<6FhV9iQn{4gFa`r0eHSb}KJh^xIG`YRn-j(}YUTw9BOx(TigNEh4
znVFOK?Z~yV{cKya@B51hc3qWX`yv8DY*USo?S1j4!}d2@zwN(YtM~ZnEVGq8%xU{z
z;?BKWm&fmYnEPn&Oa^5;rRc+Zi$XN^x^UR+ojG^rUY};By%IaF>`_1CVqI8Uy?43e
zrv0ak)$E?`4cs^Xv!{J<iTD1@c5}PLHp%_!8GG&Lb64#@p1fec!;G|jSwFVhDyAIW
z%Qi`7@4E`Ny{*Lodp1?_?cFs`Yu}>}UbgE*uJ75ufq8HEnSQ&^!h7tZR&U(bzNBmK
z!7g3f4_=(MhxX^&{(LI7Z_bI(eQ`&R@4tTg<UaNr0``|XtLz%=j@s!bFR^7b+`VsN
zzS+J7Mt5wdEZSn%d&a}|yU;{CtycMc|7>pT<q;Fz*SEiY@1@-{ZLaQ=+xPd`dK>Zg
zN3G8a-Pp4v>b9Lkz^r`%p(pkj2FvW*#CvXU{pBb7ekV<`TR#2#-sDMN_kNtVX8#|C
z&Gu_WTldFi{jk@QWIo{W;KIIeNA&}zGu<4Dh06Bda}YWpy<@WN&OdMWt~HalRm)go
ztFeCF-p?mY_87H(vTkSEv0FD~maW5yB->LZoAxzli0oUxf4QBY?D@UCiyHS$mr}Lc
z`ci%0o@d|ee5U@hJ1ca`c9r+Oy$|>P-+RJ2!&bm3%=Y)2PMiOSn{2+vp54pYB(Z1L
zB-4G1Q(E`#JTiUX>LW|`F<CL~En`#Pm$*IBuD4xcua{xb9y7bneUcy6?E9$HWA~)J
z)n=Wh`aaG_Uu=Z9)b_nAJG=MBK?&Ql&o|hfF5=zCzUIQ7#NW-<lW$(JZ(~@t@8^`3
z{YIr%_PIVO+IQAT<-mu8`u$(u9k&lIWZiH0?Y8YV4`aKAIVbi%<UVF+`y_Y&TjdS5
z>#j)e+r?UD-*I^2e%~qj_B)*8_V;akWG(icVeiMn1GYNHoA>66RqTzrcw_G;UWUDC
zbKlr{XC&G#<cQw2LZ)NC{~T_6RqHmpKZ^YJn+tOGi<K_5w@S0y&lU8-?#}IM`<2>L
z_jmnJw7V(PU^B-|aNpATI=ee>U)sBH$)i2txu^CVnsnJV<b&+KP#%uGYm>Zf*L8?n
zZ+LynR%u;_?cPb3_lgDl+1s=A;9l9K9Jbmy9CociM{SG##O;-Tym{}c7Z-O+OGw!o
ztDoBQbeZCA4*yUa5#O@C=9UL+mK)95E2mqv_pI9!yEnZVdv}#**r{HBv9JG&qwT&G
zCHo39!}py@RJE=8wA;>V;sHCQnpRuZ=;`~nzMruF&Ceb?iQ6purwD!7-)!^2?&6Jp
zJC@5K_A^+y_onmo*>+v|WSikXdC#ubYit&LTxgphcFB_G+Woy5vx01&S^eCT?=fS)
ztHUw-Vi|`6?GdU6?z4(Hlr0cCP`>}ye!<n!4mxj6?Ta`aYdbq%#E!?#-ge;^sr|_;
zN%q(7Htb{k>t|POIeC97bAkQR$HI2Cxy&}g_R_YCZZquj(S5qt(<XoKqb*K*?G6a;
zTj?xgvoF$h@0x#m_SpV<v3HA?!JguI%(f?<rSBEdir(8Yee+%sn@hHi57+Nqw^+#T
zWYcAv<<k87Can#!<KSf8A0nP&_pwP~-*SmiyR(fy_AC)OWcPE=-F>1V_I9!?OZFXL
z`C}^{Xk_E?ce-`Wk4d(f&u-Zo`X|_|^kKBinq6nxs<hWuVs6h~x7_qS+^Uv)zW2}E
z+unI`ugG(ry*)~+ZFh^y?EUqz)0Qi;&vtRqv%QUOEB16OW7_AqpMT$CUaoyRv$omP
zeb=+)zg=Ou`lR^2zBLJZ_nlYTyV3Hyt(@H~yObH0cF%Y^ZT)&`Y^@o#?fbCWY5x|5
zV|Ht7TlQZFG_~8)_hYw~weh|(rqg>aF)g=!y?o-{jTbxia`t)d^{M6D=g)M&#(u_!
zy*FoB?mx)5Wq%xh&3;h<5&JHW_jYT1jQ6KslHc$6M0!6@>jwM#H!s*9@tnMSzRuOX
zHIXlD*q$ld7WMM(JG@C`kIf?<o50lq`xd<_w|Q8?x7Ukhv(4I7zP2i>pYKU|oW9rm
zrMm4aoyE43goO59n<izaym^hSlgiY+f-hd!&ddI0%kZ?#_V>}&y%84#ZFdyDv{^Y>
zXz${*lD+=-J*;La3+(wbYwrGZ)hK&I-zNKQ(Q)=M=g!-Su6wY*dzI~erT;r^d!B9H
zduY;HyXuPf`*;+?_RWZ{vip9{-R{Z#<o)~77uvmNJFwUBoA-W^HYIyYZsmPlmv`AH
z@JQIcK7V2F9kWndyDb-OrwJO{&b-0Ach;nE+ts!qwzJ(8_qL^k?d6p4wcWe^%U;nh
z+O|s)|Jm56=UDs2D%dF8&ELxsGJVf#xA`_TpQh|<TpDbrz!|?!$JuJ12jkp*K6BRY
z+jL{kzJIqn_x1#u?rWR7Wnb2#4Yr~S-t67e`+ZOPW!Amkix%wNxA)<mdpVD7u3o-r
zb5^l+@An7ycFSgD+36{tvnju!yw~W_g+1YVcD9YSQhV(>X6@4{J-D~7Y2n^3BkR5T
zA#8S`E9UQ;8#`gQ`|_T>?~)bvy{l`tjhZyiHrC9+ws?DnZSd!7Hc^lH56nzkWbg9P
zVZWro)%{KV?e?qx3LdDpWId4SeahZIRb~I%*2(trZpUnQE9vijtNGNX(BPb{#}@9r
zS~*+yY8{Q<$HTDSb~Q8KUddeywue5?uoYV%Z}Uq$*Y4fLnbsR6h4<Ck)$ctfH+k>L
zHkEyU{ETfGUDoXFec7`wx$?ojSAI2nKOJ?qZz_Li+cx>2%^7#A13nYB+nR7pv-`hY
z+J5U{xqZUxxa>Y&>9%22yKGYyscvhw)nxBOyPUndPY2jqGx_g1y`yHw?_ZKOS4-3O
zR^Rz<{p)z2t!o>{KAU$7_TE}$xp(rHZkydktTxkH_uDcyi`$jvPqOz)wb-BW?)m-|
zPkrpwzrWc(QF_t-y6tifKL5S;U(VjLKS-p}R_p|aZQk`;dy{)O_A>W-@AW*Fyz5?)
zfSvMsGut(Fwzf+Z?%1B0z_yR^r}Ey^xCyrJy7%ol;C{mPoSv|4*RwKPrr#QNt7|{+
z(e0JpD;21?N9eMI&Ad~dw%eKIceAZ>+{^K|#nw1?*`CCOMteTjiR}B&SZDjLt$5G1
zqto{Ma8kGB-N?54W?JN)**44eW=iSrGo4;+6LwT$Z#k>rz7vY8_OkyuzW3v`roEqJ
z3-`n|Y~1^-f^+XC#dCYCb@X=kl+WGUZ1#K4rYkG$n4N|8*Br^Ud&n$rKaI=Bw)O9f
zeHZR;wwn>vy8n{@N;~<PH}}0Ne!YLkMP7&Hrp9(HPrdBZpE(|QxciX3=;ah!VKsI;
ztDoZgGfubfpLyfqo*hk^dy9DzcJDc4X1jsq{GR`JqxSMI%-QSRW^b!df6a#RUbJoG
zof)>%nR@p*^Jwp@*wbh$soS*gf%{Ke?(ns?ZjU+bs*iT>wT?`&i#Ptf@9aZ|13%u%
z*iHQ*WN+(Nwg1Dig#C$(s`gvBXYYS?d4c`snfCiP*I%+OeE)E-vS*jA^CnZ<;;LUZ
zAAk7TS{<skE&0K}k0te(O{Z(q-j<~ew(}>v-xKMgZ(Am=v3FzI#JvS)Yi+k*VY1E3
zoooBK;PPI9b`iUvw;p!3^8EHMBp=x4-CVSPVy)Hww>dZWSNhoOkAJ#npPKN~eR8aK
z_62Or+`o9&wEarW5&IWLgxFWESF=ym65M}sE8o87YuoJJ^Xb?dJIu5<UBkMU@0Xz6
z>Rh(HyMM{p>ZdKYdAa<Fb>0D<z3cag?fsf4y~k6vYj?WiVf$sX-tBk0z;1u#Cc}ZE
zkW+TNPqgh1XgJvA@@?6dFLu-3V3O{>J8Q)Dm71s6h00m)vt?x4$FVBa?vDE+TP?o>
zwzKy7+g)IpYn!d$y>Fp+%f4IJrrEK^9@{<ZRrjtbQqs2TWIgvjP5!&@-whu7tF9CG
zzuGj>uKKb3KApG1woC5c*ju)VZQqsaO13_01Z)fL&D?u)%><i^hga-nebu!$gP&*b
zg^Kq!+>A_iR?b3uJ?=*Cn^<CM=Uv0LZ}!Pawzc(c`!3tPvrV1nzt>uZ-R|O@v$ndY
zPwzeOcK6;7SM+xuExf;X)3XbEdOru-N-PYq&gt&7x9Ip~xBRS-T}Z38{Z!YkeXqo(
z?4Nq|!2YkIo%Ukcs{8)$+rO`BdbzEtz=6HJOV8LC1kc<P>a<}`@ryZol_#Xz#9YbW
zE2q)7x37H99;*qx_V<3b+M8Qv+OfV_w}1cQiS`Z2d<VK79o`?`ENP$pbk6>|4taaM
z$khE@?8W=SHrd(h>=3ixefsKtDLyWH*Mm`et)3d&DNT?&(AdUrZzs2EkDye~UIx*7
zcHb0}_x7E=VCR3rcc1>DAA1}YzS<{nEpH#=^}TjeCYtX5Vd~+q!d&sd4aZsgyXIBx
z*PX9%;MS&X`)~C;-#?|~lD(gI(|(?rQ}?gBW_7@A$LxJ+(X;L2<2&}t-nQMpF^}Ou
z-N#&ezv|cazgiaUR~0*Gmp1dkJ_)h=Hq!4p_bH_2@7?^oVBev-leQPm_1R5Zm2Njn
z_xj$hA8+<P{y)*~y=0HwAK8_5>wnJPcgWFuU(i)XJH<l7eYNbIcJg|)c5HrM_a0Qa
zy>~%Y@Lnfxr9IVEr}xf1xzl>%hGkYwCJeSmmc{Hn((-f9ji#CQg6e(yroOpgms+!K
z|1PeMeR0dJ>~1>Tvz_$cd;jgqru|<cqW70{9kpkD(Y8PN4$pz5#{K)JC|KC9N`JoJ
zB&OcpezBA-r{fPh=?$y)FDuElmCTdcv#foyt<vKbo0MZ&HhD@>dm9rpY^QHsv$s`D
z*>2L$&wE;muh{rsJiAAtjCo(@MH#yv3F~cA4*#_7SL)o$dQsE%UZ<}uSK^L6v-dFC
zc~`Et4-Z>tXEXJj9gAkDt>kUl{eJV~_qXkvX7?qC^?-2KNxRZ8i@oOe`)vci{MxIY
zrN2kU`>;*11j|0t<pTS9{`uGh{rR$c=MmX`e5UL7DedmH&zLx8pI&RFeRFQ4{fxWk
zZH_4)-q#%cU|+PY;l6iIckMCxqqcYcT_rp1aF)H-W(4hB81Q26R$pCPLD}lPMrx6J
z4jQ;yKYv+m8~bj~{<NQ|_Cmr@`{!@c*{>`x-G0W4mi^}!-`(HFkZOPS$K3rE?w$5g
zX8rr_L@n8!;ILqyEvtZ?fPee`WXELtx13FT%T@gCI;YCnyUo16_tvc?`$X&F_Wf9{
zv~OzEZ9BhkS3AC7O*@Wn?EBLd^X%9rM(@2L++>?CJ<;}D$^)B63wn3&|FB@M?#xfS
z=VS})^Rl?Ur>c)*-z1G)do-;l?LD;e@!poBCAO?rboY2iPqyVad(^tX(qw1K)%&)0
zU$fbIB^cR0xVU`pmAtz)f1bA3lpRsBRciL!yM3SI-Ycu_?J05jW;12R<~{p)8uw1%
z&)<7K%5>kx2Mv2m<DTy2>DJvlX}6iJ+w7RVSEk<Dof`LLk9FuTn~3V{J!*m>d;c%k
zxHFE+)z&=Zv&~$K-Il&n<@P@73$sm^TCsQC&MA9tu9MogE0=%o8nw@RZ<$@UG4q&a
zJLjM9KIKl)y{#+6_MCj*xBJ}hUR!U6aO-@|?Y3*rEZ*12pSn--<Gj5aRZ?yLeoNW+
z^ta{S(|r$ZzZ$aJm26mR=Nh%za_iZ>w&%@Q_Z`#nx8BFJ-`0om<lajUkMI35(Phu{
zd&~DG^NCo!_F>p(#bsqT<1w$j|Dt1i?Wfe*r5xG3@8S9;dwD<W16+Hv_n8#*?tjF%
ze{c7Vn>MCv|JmMs!nya9Q|;dA?|<wSG`eEr6T`cAw!?$HmlTfdp8Z*M@BS(^TjPuD
z`yLi#+5EA(w>SCH-aWOVM|RtqX6}8nLV8~W+ff_&Mlt)$or(vTcDC9rwh%us|Ag&+
zX~SrHF`sJt!z<_PKXhu1UFthkduvI7y)mogZGFr3?Tt$e+iiT|@ZRu@AlnrF#kMz2
zHrv*&{k3O9S>xVzmc@IG1D)+W^H15nVf?-Khr-dl!Q8C2<|)tjcFtL4%c%Neuc61D
zJr6et?vvWswfn@`*L!~(G}uO^m+Xz%anZ)WIB1{x#Ob!Le@@(Upg6`R-Co|-=z*%O
zyo#x9v22j-Q@)utlZCF@e!m~Jx9j*?o6|Zhwo8xA-22vc{@$C5O>Mq;SM80RICpQB
zaJRK$tAH)5vF0A3gnxU}!XEB9CZxFUzlPKvyXq&lTmN3ON$69vUADA#cb3g!TmG&a
zd#kcbY}c<iy0<|_a_{Fuk8BU}yx;Tv?kro6ua&!#+#PJ^?^wF`+2gIY>D|9=9EAT{
zU%tM>w(7@({afRf+e<zv+du#57TZ}j1o!#Oy}a*-2H*bqFD~z!x8?4>7e>8ydKXIV
z8x2bB4*Wf|KSE-Wz2`hrdoKQ?yY1@2?YCRM+c(EUYrk45!+{KEZQJYJ>-TcL+r4*@
zt+d@?y`{U;uYTNf;J?70br#CjC33%QyFV|nO@9?|;QAIOhwD}w_gAdww7=apeZTz!
z2mAByX4&7$aX;YiZfl<s_hVmY;sSe--zWD+ysg;Zs-?1DZ^>r6W4kQvCIA1itK;D~
zpc_5I{=ugh`+vT-cE33(X}dRf*50G{*=(0@*ttg|L}c%r-wbx9cf#zlt9h+Wqc`oH
zJY$K?Ti-lehLAaX=S^I>H+N#8%_f^Idv@Puwd+VKwrw=!*tcrS2OF!O%r*z|+xH1o
zcH4Kp{cESieP&<%p-<NDrpwsRJ-uPSlGUlb-=vrA+dMOL&kBjzwihPc-P@3qyjLTt
z#g?^gnazn86?<2FU159npy9r`+NyTbSQz(u^xxer`ANy<?{R0_>C4aT-QBOer$YX$
zE&u*hn?s)7w(Ne4yG`W$ZT*UqcXK{lYNhmR&mPM??`#ey1lv}<<F}PFy0lm4&6U09
z!nOADY9HGhnEYf9?=b`0*-_eid6%>8%QL9hqyB-{*5~<o+eHiwdlN4lx7|7`d+)c6
zXKfy>;M)5(TEUuQ;d2{d0Vi9l2QT-?_)oHBkC|e7^~6$}gEIeZu4O9NoiF#a+AqSf
ze_rbjyYsxxb~j7z?oqkpxzE;e+1`%byn9On%xvqVrrB1jw%A@yxL|AQ#<BO=IX~Ni
z6%TAL>Nf6Cxt(wO<;LmVhuDLxbm~oZE%>;=a%EHeUX@DreTG)w_C9v~W0NtHX`jP)
z*?oUBe(p(;IBm0+-N^Pv9+UMft}fead#~=Dx_kScy-BkBzI%SR-L&M#-b4=Rz1wC#
z-`BY7@xINE>g*nQ#oAx^D|uk)+l_YI7eDNCTBK!vxwhK+fUxiZvqRPUd|ol^6^Q8F
zXD~V5Zl?0$-S6hN?W=5@vZrru_P!(67w>!BV!CfpbJ*T}u0eY)HcZ}I5c_wJn?c^b
zgzW2k&uo~v*YE4Yy^?NEZJ(c?wRdGrtIa}liM=h+LVHbFm)LIg`?mYxlohrQ>bv%I
z)ywW%ec{_)9VPQUmb-2D@?VnJtL|{nCa7YP?U}^uw(^B{_ujc8zxUg-Xxo!rK6`(B
zYTB!QiOD{tdZx`{o>Y73=?3=F-!1o>>Gtj`loq#l;`Xw8W@xs*>+{{cvhqQ8zQVQp
zE0zZBPkZ&mKH{y~fuF}i>=*6o+3%F=e4zKpT|1Tz6Wb%Jd-l1s*zb#4`o!jJ5ucs+
zntgjYQd{<N+`VZVw%x_fTv>l#oJ`cd1#v?A%s<Yv-ReEbwr?imzQbNUcCJg_?p;~F
z*fwml^1i9bEc+4;x!H9toM&sq@otZQgp2KduC+GX6)mj!EBD!27v9=))S7SK<&4_B
z+b{pwd*}4uy(O1A_pcJswmWZ@W%q2uGP?!RtL)}|ShTOQ>x!+;Whq<l=_31uS?BK0
zz9(*9T*PayQCPgca?gMJpPkS5H|I{;fA=G^{hpbh_Ol#WY5!whrENomkgYo(x7|%E
zOY6n&SMRNRd&_pwJK4P%jPGsw-?Qv{^D1Dk)Z+X5#bjmdzo#48tE|$nUwKaWK#jPz
z?S*S9_M+E2_Y2yyI_S=yYdd*yt&QgqF+286i)}8o&ayRGld;#a_~2d<ne%(EJ1p9J
zQ2oGe>qF7DYbVIr1U35YmAqbT+qS=Z&ns8;Jtu#e+m^hQ*}HX_;@+PHXZE<y$*?*3
zf4Ysb)mhtz>>u~==dRy7_d28Pgu8QXi+|49v*@6)ZNQK0y>5$p_5^32uytI*vZuTK
zxs3#evdzQ`-))#P`fR_Oy6&CaA#E#neYdUULRs7T+v>Y>7qQxIUA}4W_Sh{p%O$4l
z?a!IH*RM`!pT65V8>vKpo6EwCc11s#?W7C@?HuHp?Zjim_Pw+bwc|)pvtyP`-uI@y
z)h<lbe&1&K{rf`Zy6$@-U1i&K@W<W`!=HQB?B?6OQE29#D{DAy`_iV_hDfpPsgU&9
z8_E!DTjsxN*OKRd_WqVNv)#P^wT%~tsIA5GlY9Bf&e$g1oVNG+1hu`9S!Z@H<z?P?
z<6@hw|He0a9~|SbdwP3=tpcNutvugF+sTTv_VzOFuvrl+xNny@r`>9;DK?jw|Jdf%
z`PwGTId1FH#I{e{)p^e<)q2YxE^K?hIk(sfS~BnJYq_zv@p0eYNm{G+E@9)gUGV(&
z-g~NY`zB{P*gkj6-s|z#axY(Yifu=H_}*z-JNHaWlC>#l{%Cvp>@^!PDP=pMo+W$r
zj&Ip@qj<O7Mg4X59p-!YGjey>EA%znudD3dSGup!zLovkzP9|CcB_K??T$qL+50qE
zZ_g)IEn5kHp}ku_UD#{T>a^El?|qv+SNd(-=j!dX_Wrdu;%xsuV<EKz!d2hx3fH&p
z_scuAf9J(WyGhgA?dP5q-JieD&7Nt&QM(JL!fm$X>+bzr=e>tF=lI^h6Kd8f_c{0N
zDV4JmWKy$Jd3bK`{qO7dmL52{r%ivC&A*=_`${kJ?Oor=VOwayzHh@Bfqmb3e(m*~
z?z;EI!!>(<I&j$QYl`l7eR#&s`(CEqySbP5uF6r^FLl^qZ$FpKzEuMA?RrYZ_Psr;
zV}JW}-~RfyA@&zLe(&Akq`BAGaqfPF4OVvN(j*Uvm=xLlvbV6SdCF?{{Y<A_qFVc2
zpFL8xEYrem@^m@&eb%{WonUf*kK5|`wmz4b?Q-%s_ige0Vk7AkYJ2&&<KFI5YWq0$
z%kMc9duVTP&b+;6zpL!KKErKa`MQLCPtPvfr?Pv(zIf9QHtGLt_g%TTY_Fo9-oBXA
z#dhtk4*NtEwe~$Um$DTvlG?ZMT)D0OnikvXzM^}VZ<4g>-?x9y!T6)Li`ysKUXA%+
z!w|{3r%;${-#0n8Jstewd-|E5?>?n@(Dtwh(_Z8AOncXD54U}Mq<Jrkl+3<+%M5H+
zAL7|7kYjKA_N4CKqz%XRR^8;<`^i_)_Ik0nO=;rOT?MuLc3Oou_8#5hv{&IVlkMKg
zbNAFOnQpszak6c5;?g~96Dw?Mm>ybj1{&EuyExG%v-_z{ognMJ_vul#E4K^V6jdkM
z>g#mduJHA@EtvnuHhJ@;J-JJc?7Q*nz`ohMqV`;MSL}W|@3PbQKW$&$Mw$IvjhpwG
zZf~~R`K{AdX&w9ikAnAXFTJnWx9Pm|{+Tl*?ape8*lSM_+b3llVAnG3zRjQIt$Ww*
zwA{zmd~(nAwR>zIvX}0)H%hRXxcsE;s}_d6rZ-jg%0>k2Er?%a`?9UVHp20Vt;g?|
zwpJ^eZDx9!?`tW}*!#$Wf3J1&U0bmUQ}-%LKHC4Ze3kwF_gnX#XL`4PLvWUT-mIMm
z*8D%cU*Z2#J7v?d{j7@@?!B~Q{XV~aXZBtftF<%qoVZWXp?_Z@-y%D9hj6=y71{gx
zPaD}?v7BWWcPnU*?~!|U=6^cuEO)8yf7o8SKe$B3KCGmEpY^@t`_Hzd+a~PTyl?6U
zrTyj>7xx>^RoW+hGid+1r>pEgm&w}4hnL&6`c>PP#_-$66+PJRbKltZ(B&6<JMS*p
z^SWKaPUpeTJ>t!0_iU?V-OG2F!`9U1hE2=i<+dvpTkb!eIM2S`S>!<3?fv%uCb-(4
z++%3pu}We8(ue!^|Fqm|cWc2h`<14>)<3sI+5X9p-gnI=*Xr!U1lxoiY<u3m*tB>0
znjJP@h4uFC;IG>AO|pJ3!`6>`W-Dp$%jJ}|wVJ=k_R!`z*1ebNZTFO@?|t=8Wgpuk
z`#m3RC-41jqG7Gi5Mg`nM$_KejxYB5OHbTm^Jc*w<>oNk$x$qO15Re_E!sKT#;5ME
zja1<}o5zWHwr4Je+3u(~W&39OsXhNchwpK%ve?t|XxU!V-pP9f`C9iLG2glO0W<%;
zBEz=bW_8<b8vjS{o%npq-mf#6_dQck-S>%s*>dA+=Y8jnZP;7&Ib_d?`dND!pU$wI
zK99?;%VNnMbN6i9NXZF%3wFouiNCse@5)z8_X?-%v|Z7yv-hUW9NTwi&g~V@-nZ8)
zE6(=bNsGOpGYtnbc6f5$RAj}<?SI?Xn@rpoV`%?H!<?N{&uE{ryz&3_?B)#n+>91T
za+?g4AFc^5wEEPpV7cIDf<?ro1(xpiX**fh8t$6-ZSD@C^ZL7*cNgrsay-mxpxifV
zFMUFw{}iw8UteFt`u7YbwwpQi58qbQ{<x;rz|Or%yU&Jo%j-ExS~hbH)=}Fv6S-$u
zb~iO!*e?8KAy=__$NE*`+oEQ#H}7?LwoNa5`VN5<y<H;P)>;y4@0M&q%h&rQEcd^X
z*>!1&tkqWTmYv1<=XTEHDcQNSx?q=Nhmz&itc_MueQz!GLSlA&7MyFbUhItV+*L9=
zpEk|1_<c`wSG_gY_KioDTCDhW*@9SmiB%WbG1EdhXPV_d+f6&qx?Z%L_l9TJl24Mm
zc$}wLa`1TX;(M6Avpstk(fLhbMwIILn0>2~^V1a)I-?n1R6kQUP05nAyd<eyWqnRd
z@|2Zw<iQJS#Jca%xAPXCJ^4*UjQ5#)Xx}p1J)?8G=_5Y#>5ng&@5p#>61Z>e&Kff(
zOQpXJJAUX!Sa$y2V6yAF+K#&qRCaM%*IFuEeQ7qGsejiCLA#x2Zk{6AAH=$Eq^Y~T
zoy9Vg^YONgm;Y~H+-<nE>qCqMV+yN<$UD8AH7OU&R4zW<Iz`@e`=rA5Z843KM&Z>_
zMhc3DbRWe2HC}un*udH>$?R!b@K%`v4-J)r*P9X>ejXxymh0C#?`#R4v}>A<t(EBH
zyA~&HZFau<KiNWi<&vGL&-ty)4VGFQe=|?$cYB(&hTD9FwL6|HUb?7CwA%cUa^mh@
zzSAx%WEM96k$z_2Ax^BlMmcjVjl{VvJKfcGID3ig3|*12b6q%xh3A79%g?!!ccfVw
z?b;{%o9Oa@SbK?8cb4Ug<)+ol+x2GVSmh{A*~uCDX@~IUBRe&ew0Eg~&)sRTTy&Rm
zeiG5?K66WeMGWKS?Zv|XE%Zwsn5t*Y*jc<TddCkl(Vdp76L+5ZB5aYqh>d9TwASUy
zF+7b`RyuV}CxL@?UHSI}CH@%|nrwb0GNrdK>&83_SCwE3QYY5G!$qB<cDreV3DdUI
zAC7N37TjZO5IxUiik`>z^N&2uZ?w3WMyu~JlWjU|Ol|*WZTMo9Fz51)UpbR^r0o;6
z+$+()J++^~N?xLDS60jki?!N!c7;{M4p@4owttB=kL&2S?Ke1%ZkxJzvjy7{8;g8*
zP7D8It9LlBUT>=Ydhd=>oA~X|T63sg{}SuIQFWss01AN&T|ujlD;q50OcJb|-V5zw
z_~f~(cTM!piR~+Q*4+=>DK)uxm&l~!M3>KgKNoTc-L~U(*A(Tta!recYprMRhWNKE
zjjYGn53pY3WPPE?>1=k43uN~|;I~ryEDy__+wp(l`W>2|Pw$Z0FSy;Sv(Yln;o8oF
zQ_L--l&|kNb4y~-msgKHe<~fhm1ppUNmy6+HM<_iv2MMUkK=XDri3YTKlr*qr?N*=
zcxSu;wZr(?JV{Flp(_S`Z*Q2Jvd=Or%~dtoZz*XSEO*z;q5hjuw#tN^cLnwh`f}~W
zgeMl+1yjsI8n5j1iTQ1@q)grHV*REa0@cjB?rci1NHJ=#JZkj8a>j%emLCi@Zohu}
zrX`=}(w&Ja=XO5&)M9BM8Msrb{fx!u__@2jHgyo4Kd2oae3y6Vxbn^4RC#Ene&m`K
zjkQ%A+v;i;>ug;eWn`ST(UkRsp23qW2UA1VJoCo}AGWvp{xnt0s57dNxMRNZd5;Op
zrTIJJxfIMdP6)HG{&~gJ^b*sy32naHQ`K)9zo}WN@3_<7bmMgnQ>RZ^<^^xtO%6q!
z+^SnpLUcPKIQ^03Qr(;#%0_xS16FxkiQjG8vC??ct_jb+n{S#ayQ41Vj^$jTE9TS=
z7tSar%O<AdmS+svtvW+~>~Oo+xa;c-i=8Jwzu8t)ad6l2PLZ8`C*BhsZw0S(O`Qbp
zZQE^q(KJ|GYKNIU*S3XRO6Jyz1!k-2cWuqT<Z1puXA#l<__tQs;MgZM!-$Cyx+j)3
zY243uG)c<7p;7+ph4vc7qs9(OxAo-LPu@c9a3R*bpZ6jSL25U;_Zei=xa)N{*=Xx-
zPSBA$)~a7q%%KtU{olqjPdvbCK<Y*co?(7>2iN8*i-gQ=mW`KoTLex1yyF`Cr=2VR
zueIR4w{>e{tfhrWYYoxmpL$I5PL8HGrh*w~EuAbT?U3EuwByM10&|u(fjdp#@3$zv
z9<@_lZKmapODm05U#!$y`6^WBkfyKlC+GY621mE+_MKU`$!~R`V(GohdInz?Xbkpv
z5Gguf-a4<-GHo-{&XBzm%qO+}-mZPr-Lmqzjz#6|^j)^5;^yA(8Hvu9gWbQxy3hE^
zMN9D`6D&n9zB3nVeZF;<_R^jI*GZTk%>HeDV7uF{Qp5K<lOz489xm~`IaY2$c9wYt
zPL?}!wwOmIaP5jY{?NQMO4hQtNMKjw>w~-IJN8rE-kkM})@!*FExt1ITZSwXwb;kS
zymOrf_fBOG{$0{nJ9e6^y|imm!dJ_`lOi>AO5OGUWwYwde6xOIzzipS?H!JWmwPhQ
zqmPv72NY%*lqIH7y`4jBylH!t?)bP{cBg6GQVVy%Wu_|*KC(#j+P3}J4qc1K+J<Jg
z|AlOuv+bDa#L!oIQ;un}sqr!!?MO4!(^HzI7bv2x^K-=#J=a?gwbKRSHeFt`deE1r
z1(W&ALRP=tF5JAt!hm7nwl8y5Se#C?+0p(;+d|IB!(waNOcTadlbytd-_A|vErM6o
zZdd5}WU)i&keR&tft~N?Y~GO?th&=hUTIhSw16F3r}t1j{SoWFfl_y5@7EpoR<E)=
zvb|%cw(e}p-}^7`6#0;_v;V?t^KuX6T>|q3cNTk19C6_?Q1LO??OndMXUAhv6U$&5
zww<>ZrCMD4X1p{1E6c7YHe1avUFft766Um=-%>na{aT?V8d{(*{PFOjPLp@9-qasj
zn#sjebfi|?(s^#QQiY3W*SbxgChLIIP!mV-|JbpeXR8I1VcgFBkG7fpYG^mRk#=>9
z)V+3#2)2WkznCJ;dQF3O-km;k$NQ3QOJ6Soi<*!mi&~zTou3z%?0i~fY;kFR=C0q?
z*<0`THCuUp;O^nww1P>XxRJf+;X>Bu?EajyJ{@7zv0&o?xqH<;Zcb6JZ_FU}V8&&F
zD-`z!ZqoeU`(3ek*8+*3XE&~_TDeZPO?swUb<Pg8>C?QFtIK|a?>rsM{@5F)V0lqT
zW*f(RCCdqgv$s9Snr^YzGi3+!`t+T5qH4^`w?*vyxa<hg?eX`o8cp2&CvR`%j4;;e
z&ec(uo@rbwCTtWO-fdo9EMZzKCt~FF<CMWi0YmfWetwpd`ptHVIo#dx?YhfOeQWj|
zxn1?<MQnZB*HpGxd|)ghI={XBQENG2(_4#+VdXo%njbgIXq&#%wO4V+KKH;K2lh@h
z_xMt?Giu2kqRpdroV^j$v1)E!V=?uPljR42OFO#v_U~N&<j?k<vli?$H5c9WuGf3#
zCNY80`iR;APbc#j53Dd<tEy_r!hL5)cEc+3>!)}ukCx7`nDijaBJ!W&F7ev8=7Fg#
zR8N0X6kE;Mb|+c(6^Cv=8})p%bvV=Z@<*$;<w-p@*W#1gWxV!;X^qZts@wZd{)VOK
z61kn5BVSuAiOI6?72dt0c<Sbz+PgOHe6dSrr*e|IrRYTx)@86;n_j6}wd4J_EoReR
zXz1Ruon_Hiv(Z%Rpx&0+EC$m#KF18b7wH($DLO~IHO3$Hx6OHeZM&4xI*V}4_8p0i
z96M7^oUpK#5VO?JO58c6^We^S!+%syS0gRpclJy$f3f$#cJ=_J?XR!dn!Jpcu!u`d
zHT%{gYw_fJoaL{Hemnfqa)|B^Qak)2EXyq#j&p3k`dQRs?g1u~qA0JO*XQ?deRk*8
zj>mB?Om43}xWmkJ{%AU&cEFFCHzGqo@@$uJ;L&Ca1|Fkrg~vIz8GYw6Q<yAiqPlvP
zao)wo9n<Hk8=L*yPV_phfeJ*2O+Ss-ygaw=V*eM@7pG6_WN>nC-Q32s?I`C5WBJ34
zCXBJ>CVUlF^oey}-3cDEb=6Oe-IdGDo}04nXezjC{Ce6Mvw!cojIt^?b}ZQtVNq8j
zFzCaDSpO2M?!*ZWEB?xbCM;8Ko6r8$Xnu1|t68S2mqoZp=#KA+EL#($rFYGo^2j_a
z`KslDGp0MfGIm&69S+|)$M*D27rt#4Umb&Y)<;Y4@?UMZ^DbZ9X#a!QfFD&i8Ul0-
zfdyjI&8~kwVs0dsx$S7#R}06m2^L(tjd$ugN|?@keq_hWH9}_nr=o}+ry<tq7XA}0
z7y71{+j~#m!6@KpzIW<Av$((6mYK)q?l`vX<90{IJv;O7-XnTF1F`mgReEUt@e!A$
z)24olr?USonj|wVq~@mV6!BhVuAuc|=K;4|^CG#7VbkthedVX+y{1M>pYV&8|Lmi7
zE`6G~OK$h^ok#lD?+n!o*!5>p$S%)?BGzk4KU=9SzrRB_%5Rs2iG}6F=XN`f3ou&g
zM9$pBXf@69L-L7TLa{YOm*XFLryF)~SD1Gg#F|EH+L$$e>NewZw>N$G!exUq)3@#Q
z8uBLZoQ|3i8*c-pPS!lkvSsO9^Jk)NMqlr_ZMib{xM|P|1(Of|ubFhEYwQrPj@rq*
z{4mk$OLQ$YETwP!-*)KdBa4@emOGkV6n5Eccx%DCd(w{7*`d31vvMqI^CnuwS>Lyu
zv8ThlBCW$bw!&>k(-KyTZ=PWmHg31JtDn-`5jt05m$Ze_ut}!_6~EfEmCaMtJWN|p
z-r5nAa?z5ly3+iK*2|q@mk*mCi`uv2Pho`RugtRT1LeLivkse@sO&V8mio2TmoLKH
zb^Gk?pRNmT_ZQi`)p{boS?4BgQ!Pto^ZfVQ482!ZYW>SHGgf)rr`x-oNw4Kg^VY6h
zA(L*oTFsOBVWxQtj_D1QKL*O)D{ib7#wDqit%hQjdv4a72HX+Z^-U_poNNCRi#IaY
zcOAdtzGK_B$3&NF#QLKpbAwg*v1H4qR&G{8E~YzqIQ(`U?Own0v9;UI(-O0HJr?@9
zE6-8TnppFQRi|dM-YkCNrESxXE!pA2y}|sv>3s9AJ=1s8<yxBW{NQ8R{`|K2R{y^{
zh&6AxsAE=R+@)VU%X044jvZU=&unM-q`EWqu-cAW&8?;xdhd5i8W-$1bg7Bz>A)*y
zgW3HKRukLh1v^S+tlcr|Oq}UU#YN`bpAMN_Nxrr%JM5}O#m7XV>kne%<GUWaX}?<V
zj`WmV^B;-_4X<2{vY5Af<JKFxH_Z0CnC)QF%Q5+$yVcxBGs^t5+F28=`1&0JRt-B8
z&Xid&?h7=RGh1Sjdv4MWqX$p7F(!x+-TvciPq7F;J=^s1{pOuvnR1p=cej~2tDW4j
z_LGoBlV`_{$*2BX$QGCqZ62}y9Vm6vYWU5Q-_EwESY@+QNp6>gZbGAle)xm!hBaR;
z%y=Sp>Kxm={pP&&MEh5|Yu8SX|8p%ip3b$rs^GmtEOOnBB}Y{(#J^lI*Hm1*^UB5B
zI~<qsSS9b0wo-p3Xc;MEwCnos0IO>YzwT79$=H=#FlVRwtu?!J7T&S+J#~-h^xSsR
zZ99hshxv8SeP&PMwwX#yHZxau-n}Dm1-FGg-<NIOj+_?%y*tgPJ+wA`7|d+0?B#C3
zzV3zPk@i#z>(_j{CYzg@ct5G!c}?Eh(wi%oX#Y|>-lQe2SSfw0+j;+`$*#Ev&stuo
zwc90l^2fH@k-fV%Oj)usCilV4pt%RFZq$FW+~8tk;p}8@$?|{s4uQ4<J8$^3?szc4
zb!U-u>8^QEu{)*8rxP81#QI~$`z6~B1*e;HOnhN}?Vg0$BGWpHL(&yy{82m17*@`(
z=%2URbklr$qQ~XMZca9w9hh#`e4xziThEd0|4%lU&yx@_n_Mbp6u2pJ+xzgZW_dq0
zZ6nrw_ES<U_WzZzylw1j;UlGQHtqS0ouvY77V%|Aw)Hb9?NV73XrWLkL3Fq@%-E`R
z+ttb->YkRNx1{p6Ek_-6Kfft5Hmv@m`MxgLc(KlU1C!0tgT5a?Z1@e7x>C7gR@#9L
zJL+dxSbkMw*s<up`L1Jbr!A+}d+pl$`O40d-s^W2oL)$Dyb<e<4T)OTeEQKABDchM
z#+7<oK6|ISOH4RwSA^@1UAB*Scm1zv-(|Pt3DxZ-)_sFroydYu3ZEpEI*j-4QBm5%
zEN!66rZLe~L3-i=T@|ZyYqb_tcgYp~U#u`!RoLpjr_2u7XDa494%nJqlz6Z``HGZ<
z3~SF$qeapdhYUCDJQ}co=ypUT*BP@fODio(R?pnFqV|l*xySwHD%@YTH_dmpaO@1)
zc|2Ife9qrv7QBonEYdm6w+HJ?v6Oh=Y;p9$o*g-f61!|vm+r{h_uXvYe0$4cCrPUJ
zLxpOXttzwb89KN<H{Vsh%Y?mBW9yQzS!N$tHtl%g<zle$?(dyFxvT?L&kXi-MQ!(e
z+ahb3%lX^<rBTVQkIUy;aheubN*$BmWwO1-;&e{ZuIazlSU&i2iR$Tq+Wy$R=GgYH
zvkMH)+r6~-6{T(7d@s@@QE-V_tI@yhi6UFJM~T1LG4pqW#WFs2t6&2ibI<f2JMyMn
zv^;f>!D327->y9$KkwXKXtwj{jn9^h_V)(Nzt0Z4E1KUok<*A1TcxUTS$WPzAvu9?
zF)dxMPcmn_XDD9W$|?W<=qKqH1}`n&F8E{d_A#5)QO(^u9Ym+?w9Q|;^SG^`g|~e6
z&Wl_ryOLO|h%TR9Xa6>z@Y&pQ(!~Ya-gT}p-?}Aar)1uGa|MNU7L`gRJLgHx+qvBL
z#(;$jwbRR}dDufha;cTY*9dcq?sa*VYdY<AT6JC8o*!tq^GTVB#eQGsorhw>ERQXj
zz7@M}OwpHXR$85?n!N2MSG$!uf5gt0N_)0Tzd5m!^?KB<24nG^<r9wVGPv$ZbUMYf
z0lP><<Jt9`EV0@R;TD=KA{v_4KONRsD*jNmW=D|PqgChh^Q>w#Rxm!$tl<%~{+yO+
zp4zm>Qs;pD&b8q?cI~`+*HV{b_O6>BJ9b7-liR&iUXJSHIoMr2d__;6nj`nHYK^=~
z$P@)r>zmr~k+O;h>u;?7y+l!7@~oBOxlg;73(NW|ue&a8;wy4t?ZGSddi<_eG^U<4
z(7C*Or9s$2g)Oyfe08!y&KT}^*Gl#MM`2~U=ApkzEg!ZwZdXh_zg@X}()N8{wwms}
z<hISWUuD<Jzp~~>FFzu>{JS}4x0&n5Rhu(>I?Y*i4_N$<c(*O!SL4pr=@u5U8?(3T
zu^+KG|7yCqB&(X$r1kH2?9&e2bxoAV>fgPo7D7z<JLThjch1X|-|=Z<#!f!o!$jM=
zxRl58Uihya!V|w(%seA(6)|g?#pWeBySjXzS@`5x@60H*wPc)iis<&_@J*4}edZeb
zM9s~7uX^sDnTn?zU#XiU|53}CdPv1_b+`JXy)zXh!UR>%uV_-gb|6)2uTj3%va)H~
z&*t3I+VnwFJ5qR^)^^MHT5|>VYAxLQK|8C~N&9ZFgwo`QdbKB7OloEu`qVcq*{trL
zd{q62#z~b8k&{&(|94cMy?ei!QSvtJ=`QSAjz5KUXR?Ot*vcHz_U6;j5ozMlRpemP
z*=Mpq=b-Zh9Sif_hPsDZjsNU=sGom1RNsqR$hhjvPW_@q>kY3P95rgXaou1-b}P~A
zUZYrIwLjHP)S7W|m)2}2Wvw}8shZj$`C9!e?`ZOAr)$l7BC9p&?=#J#77^+zXH8Js
zD8E(BNW)N-<qMbUne8`Kuh^|r-BX&O7IgZm>iHjzYU}RKHtV|`Wm>Rgrg`?k?dAdR
zlFc;C7MQb^shO?ym}7SM&K2`}Jq70TmR?d{%+;u|_t|uHH)aP-pVPjY9}BZIf92g$
zuh$PyJA9E<^ZfNzjkEu^Yo^&u(73uoKufT;Mx)`;A@%M%-!zUt?$SK`<D>?^cE6@j
zq=ly3o`s46ol#(Hn))72gTz7n)%Q-SfaI$Mrm5>oEYX--b5(t-@Ntb<mv?I@?-tY4
zRJf_WINd;9-guM7<(DVarDtTT-Dy~?5u1~(W-#@TYRbNuN+4P(TUWJc-%OS7*%0y2
zLrc}FI~X<ca)i`VcxP!;b*<Dm$a_rVoXr8X!n-nR|J$uJR_?#7zVhr1HD%8U>iv6X
zDyhUiQNN$VqP}3?Ocfc=3F;>MW-3{I(^2*9tWlp_z^}tnBBHg7aklo~4mF*OX|dWJ
zWs|hm*ZkBz?(eVtS#5^SNtV^x56+#}xu?HNYhUC69hIkgx`|6lwf!B9b$vKbYhSyk
zuVcDfOJ~#JKibDi=O{)@d#?Ch<+KWj2Jss+f)zm+nct?kUo174OQG_>Yf%txnPe@0
zshC&XewBjE1>?)ID;=sO+#H!?SvprK7xQz<ZDv+blx1~L;*4NbNEMo-pkQ_bOov`N
zr~=}@x0W{Es}*MKFsH<nhjW>UI+vS~V7;Mf=FDEB*Oxm?_V9Wdo!`%C?BI1ydCAE;
zYKxf+R9tFRRRW*CS3bu5L764vg7S;cB`SQXf-2Eb50pRsaMs_rT1D^mltzQ+s|pM{
zRgdXiS>$anb^S8EzdKnCqGE#e7ihNYFZeBMdS`E!@fGejCcZC1Oc#jznk3k&n5cCA
zHM;z5lIfZg`KC3WE}1U9-lDSPd6)9d_14P$>!*U*$YLOI%Yx}jCxT<l;-0pfJ!@{a
z5V@miao}8^8GC)0#l@RnOtTtRn}0tmW$u<U!Tf&sZL_NtQ%(9UCz~8+J#2VYV6y4f
zWm`@B|1g?QF)lPwe|5(6UD82QwR2qR%=IhO`)|srg|eMczi>rW{na-e<=b^D)OTlR
ztF&cjt7zX3QYW_DFmA6fZB3Fh^VxISOnkqFd71M=v(8)*^J5R{O_yKxHoyMV(p;l5
z%6!Ir1@+CHHR}J1<|;`~J*3|CO-DWPn~rMYHy!n)eKS?&$NQ-@|MgNYPZHFZ<(j5f
zE*P$7%T=YD!``o7!z`?O@)(Q$jSN@)zJi~6y$dGl@3s7(!oJp8T`Mkx7sTH9=@t)&
z&U3W_(=c{pn2V{<QB@=Tr<Y9<1364~muxXvu;qwRL!+wcJzpWCx%-vP0zavmu58_4
zy62UpnU&RM^KB2}%$*;uG2_zYHgB4}+vM&7Uh~jR6=nwQ+U5>Pt<nOCcZ8iXgXBQ?
z)XfQ^E9#!fb;%0Jq@-UHJ-X+J;+?OmisB`E)!vkT)|mL}m-4+I-Rf5qyH%auzEZCc
zy{NX}ZJxTBQ=D4F98>k5raej%d1i_^PghW#U>c~#$j_<3G|^mznZZu!^DGg?(+nxf
zAUdUep0Z!tQQ^K%ixlU+{4QTQ{hVC=rGK&lU5Az2HP*<M%kENf(1}sByz@?r|8SH}
z@OMY8wTIHRL|=Ajr%OK2I<U=1`$lS$_AK5Mt%Wiw+CJJ-HB;~KYKFX+q#5yHie|Rc
z6wN7W%v!momo%cEdTTTU9@LywsIIwe!(yG+uhwhd^Piz}D{8h*@3M2+%@aj+@@^m2
zo^rlZXVptK9pxfP9oesY)aCpcH41hcs%hSspuX-ct47atMGecJE$W`k+thk5a%i~m
z&rqMQoo#kobCr=~6~Eb+$bH6I%c@Kr^rFo?Uz{;(TQ|w%Qs*7BtoPxjCv-JTnf7oR
zU%mO$#QGw$iSKns6RwzvCK^^>O`Ko2nXGDSGg&_4w8@8q!s=pQxm9*5O;?Y3^hUMj
z_FWa9FX}3r{i;-=x5=nvT<uZWsk}g0DJaE^y=uDI$6KWq9qmud#sB;_wOP?=Arq}`
zHcel^e1^mnGrNU7X4BF)=(Bk68c5~G8*y<58g*MW87#e&WmMe3ZD4DoVp#G+*D(Cr
zD}x!P({<T8uV@vdSm?GYFzJS6`RK%)e4?XKyj)i&y-O!^rH-!s)d-!ONm6F!y3<S(
z(@M;){q8XfWw>r?J@Ja!(=~I<j`18ZmFF)q588g!?5xCXwb+L$s`;J&RW!Y7R6ouO
zSG`tpO!)&(i7NNzYSks>-&HqNWGGs3_ozMf2~}-5C#3o)e1__oJ<Mv$PA^n7TwAAV
z%P*)}Vyvb-GiIX7+ZK5>=?RO}gY|c-pS`D`VdO8T@zc~!Bj_EAS`{z1dQe`3dJDs0
zwcQc&=5rW!8Qt|=VeY+|-~8EPKC@JVA7*ShUglG)-k7eRB5W~j&qnjaY+3!jkHvb=
zQuz#b$n+Xi+o|dCU$isioPSmC#_!YmF-rpU-+x}GUo)Z0Y~>Ygv-5GOW-HoQP5fh&
z%v;;o&8+U~n5#{bH?z_<H@g-0$E<tW7qun1!Wu{2b=97oWY(-bu~kER#Z(P%-Kpv`
zjXKq;x->MrJ*TL<Ts>{Ts-Ix6y#2M&sTnVf7Tw%t5aIR0C}`INgUnC!3?G*m80EkH
zZ}72uhBos;d!7HYpKHnWU(s6S=Ag}3t)hLq+gf{}z!B|vTKBcC9M;#CH5S#ro|vh%
z=t#DfJ<~tUgoZB~oqyD|9u<UZ&R@)^W&Yk@ld)s9<~qSIGWz#_Nge*KrT8Z$TjH^X
zi}=)qCbE;;Sj6q-<S5SPyC`XLHclp|aH{t1atV!=*YmXhT%WAd`K43y`~E~7Ma8*V
z)Awa-ZCRtHowu+_t1~xIF;ir%3W(OawoDO(L2M9q|MX1}M1%0lbyjN2FSDyTv~O3n
z>Zno@h?h|P#5!3OB<7XOttxx+tJ25XyQ*c0Ka6&kCz-6v*Eb1&yxTx(9lP<0xIUAa
zrtgjUHl8qA?Gj<k|INW@Ui%%*t~UjmQzHsACWLZo3f$hQA$|U}#)D%FS|6jTG;Pdc
zH5YL|(y(>>q{eXOm^xb<kLnf~9gT?p2h^w9t7(+%o20%`T3S6!Qd@1df46%4&Lne}
zaAh-%rT*qp4985r?A9@7i3>63Ed5}<DBIT5$-T?`T}+Z0H{X1v*=aT^vW8r$@7~Q;
znpgcpMe~`t>i^@`sutXzl;4{jQ#l-AqLj_Vq1~3&q!oJRhBohaXYKbpJG6Z^`)f~d
zv(i2uqOE;GN=C<RO}}<>7Ket*J5dcqQ-1aIqvzGl#FRBQ8h_FVczRM@sP>uKouhj+
zR#mK0?<!oS^5tQ$s<!`qmB7vNDzW9cO8PlTs+%<3RaHMVss3ZTr4*_^Q(nkkT}EsV
zx7v?CPVy7p#)_t#50X70sx33qLt8;LR8+0Rc$svG5T|VQx?GJ*?*G))?yu3%brsgw
z$Gb`0&HayRx7ITCW)Wufw)^brN26QSfB%y<*_*n;q(M^9oLTvbnUhVn$;y-^W};k@
zCeQWvoASzYo95|Tn=aeAPmA9uQR~)ZCaq?!MosH)8#I=ypV55q!&`H@+&j(vtNS!}
z&R(o(VDnM+U8s|abG(4+cDoqW%ZHs+WgmK}Fjqt?Z%tjUx;I-`Rr95$vhP)O^%O5g
zjmHz-syeWjsPnojtM$c{DLbAxsh+S=Tg_@um--5w3u^1Oo>MDLxT1a}%uC%}?z{R^
zNnVY^@0&E{TOCy06LCsyyOzKD)M@9{e$4PzePGq2uB`Q6rC%XZE%)98HI4-hY5~h8
zs%5TeP}$V<NzK7%oywJFNksvP&nj^XzKF!$6BGFnazy2&C8OZ?TWjPM&hSZeYp~0-
z@_ESUJlrf_963#mwZ~2MP5MrC&tnJFe?;$5-!NTC^=T%jipJAG^@6%hY6h|Fnk{RY
zwcPI>R$Z$*QDem!ZOsI0cMYzHXsrahW$N`Q;u@=icr+ZgWoo{=@lvgNnzrWDHyP?K
z>9f?&98*>^Kf|f1x=lwtO>3EYdqb-lsGlV6IYG^B-%Q0MI}`OENq^K#_RUn(-#1e+
zzq3Y7#&d$2VeAt%zCR08M9#LU-b%WvWa#o<>B6M<njqTB!BBbg_t%=c7_KVS*?TE3
zQ@f_sahFre|J;8~&Rq_gitp<+*6`VD<whUYIC5S{i}|OLrtSwZ&47zR8dZfmHP-M6
zt4rT-*VuSGPlMs?JB=;kyc&yZB-Oco9n(nR=hv`&xL7~6I$ZZ6*Hir_p~d<(dzthj
z(>CdEG~A#sFn^-nx#IWw>rCDCIYeKomK|EH$bV5t_4G3ZRom{Hs)tUjR(X1lM=5cN
zi0b@?7b-GgLaG7fKKfgv`1IC<Of}F+@HY_rGGA}np=bl?3$prJyZ-1EO*o@})k;MF
z_P$GI-7glJ^i7R6Uwg33-1rBR*+p(=v!2~qW>4<?HVtuGX}+kv!JIKkLDo2ahVmJw
zRnk`j<c0b5<W!bDSSwku`mo|3=WDWm_vgw#wB9JcJ?ydUM3YI1vA*0g@pTh~ckG{_
zIQtN@=+%iH;)mIcWR>Sz$$I_0D$^gnS!ttdmQdn@Kq>1PrX03`ZzOI@EfLG!dRakF
zrHieS)kdl53A5mgJ@tCGpZM!Xmqh7*@?NCtp2(zc9QR)@Tex3e+>J}`!nT=u%u2R;
z9WOLw_lZPmlwRsp%B!ERsB6cls`OJt*&#wvZK=G8CZCF;lzsST&5EvdW?Vbon3~wf
zn@?F*ZtB6(ZI=I_&U_JrwYj;@O;g>|HfE9HM@`jNcdIKoy;48Df2Jacy)gEPT4N52
zTGf_KpuUspksKDa^GScy%3pP=w6UnG6iKX6taBAq+aUQ{#hde<QphJ4)yo~7s%&Y3
zN{g4QR0*G)sBU_WOJiQadbLUYMe6mp<kSmiu2l=^S*)J&-%IVNTal{&w6$ukAENZU
zJp^>;cqHn-)Xver<-nr%YpR!itW2Grcj`R7)Qd6t=i0yOxj4Vp7yiD^XxZjJMsGK+
zFk+S7WRzRlZ=`d0mVW7>C}Vc>TL!h#(~P9FytGbqPE;3s7pY~o`Lw1g<5Z0e|Ex7S
zR_szMJ<+Ewx9y&0OlGQPR!^{&{h~6h2b|w^SZgL~eXri2nWp(fhoNYuRxG!imcPm@
ztvKChnmXKGriX(Sj3jdxn{4EYFg7<3HxY5qGG23B*p#bRz<5Dfl4*U;3X_)tt5mAr
zUsk)c^S8?5Q>iL}9nX~5XE`Y?RO?i!*<+yu6A#$GOM7LJzs{kroI08{hqT2`^6T_%
z->Cihh>*@BZaJOD>rJ$ackI?q**8;Jcj_TE5RDJ}KQhv|8+=^-`$<7{zSxWEavPti
zZ&EYYNY4#a^Lr|*<|^*1vAkJSUAODK+5XptCh|or7Jsjln_oBmW72WxfO(bice9oR
zRkM$hedfm-In6J*i<pO{O)+s?)NUTTe3O~sxfiCs3VLQuUQ5g##L1d%pOa&rH=o6v
z*#6POV=ZzG@v>_7S-&bNIGf9C`I032T;jYck93EE@iI}hh!tXLT^`3}v5y8}ioSZr
zX(V2;(0Ge4tI^@cVncfk7ZU}SXNJqRtu$7W+-<zfX@k-4y4A*~vsRhBx$k5mx@4X4
zz4?3wue@qZ-Ocliw})RbU6b<7Xzy_~(@)2?8#BvPn6gA$7~2_?nXsIAXLx)fv&qyo
z*G&@SubZBq=4SM>ILLHrXPdG8l}&0tj&D(~Ik!_SamErgi)B9Q>6`l1bnSkqS|0na
zdi3WL)#8ULsvUAW&9V%>nkHJ=n{Vg)Y1UMyWw!Y64fDGBT;{i9)|s9C5oB(h^Vw`p
z#9FnhnQK+`j&!Q(?zySj__s$@@bgY}>C_t4^8Z(qJDC=#vsE8aCU)Gy#dM>YK;99v
zxNm&sOCx&CLh4?a?YE6K`{0~vx-ux++-$YJ*)QQ4=9hopRN7p(T?j-k535p<Fw&R(
zlFz2Ja@#KXtu{e&Zu}1vK=i6<qNZ+>-kZD)Tw^*@^snj3Ws+t$z17Ta<(@Y+TvuzV
zylREn_U9I64*yG){hzq0mE?w~eCOMy+&iIN`BP(yvib6S<x*#6l}qOqf!R^2vy5Wj
zSsK}{?Kfn$xM#5E=WOEwz5v5-1t~@yPc|AQvc56OXkBgG^}tCxe)k4V;l*KE^J-sc
zL_Xx!mfjYqDWMpq-S@dubLNUBt^ONTT3Sx$G>_XJ&^*^=qRGHsuW``ENyB(jqo#;L
ztY)(zlUDz!3mQCj-!+tsQ<Wkg9aP&<E2VbJcE0)*)642E?@p@E=|8P9{lHWe7x7u@
zj6WIF|Lk@$^g8id$4(&G&|^uv;iJ_H^^3VK80-pRGj!@?)PJ@}*XZG<07J*<Y~#0L
zyA4mijxhPoBWN=1@HC@vhYTadcY#J5ZP|=mC$O57%&9OgU81bDD|@Y~Tc4GdjhTX$
z<gfLbzj_0-Se*^Du4Rg8uI8-KTJ-ge#$HJ!-AmcWv|JUW^|ohi)BVjpP3PBrab32Z
zk9DtIc%xH!bE@u%DZaY$%xBem|0!!+aJsDeTIslY;Pj>H{Crc@7dJGjOI=x~dR<;z
zU9ccoU7BIKqWjHVDv}&NiW?j6D}wkSjLZkIiA`6FzWSJE-O@K>od4c*7XK1c!P6C{
z+%`sLuNg0ye!TI^MB{CjX`j;!lQ^wOI_}&ZTAOyf&^~zQm3GvxL><W$lXUDdj%t;a
zGw2vB7twJ!aY}pBg72oxN9G#MQJ7;M?!skW<UGmrey_UO$0$X!g$|LX`SDB4D+`yH
zeRnW0kuvZ#`ueQPc>lTehOW#>#%i`{#tSc3n+U&^GD<FdV7w(m&DgRlTq{@jw@#kV
zLaj}v&6?^lYjkd>UDurQqe8pgZ-@3xT{|ta!XEA9u;*I&U;H(lof@=6!a23F7jkQM
zTW!{KH2<jidlk3V<GW8a*G;&hY0tb!Wzv?1YRf*qQ>}`fq`p06t$JhjDfOxo+g0{E
z+*jGS%t~F_?}XYB4oCHVuZ!yE&RD50W4fbJ{OYB~<1i7;yLK1Vb%Vmxo><M%2)-qv
zA^AelbWXIg*?!SQW`2d*rcT{=O+6QCnVAT=nkJk*V|KC2!dTdSmubqrnTjy1JN1wX
zjEzq3&dyd}@z+bE?wgK^+|)yAiQjb8QvZ6X744g;4idAwDW~4JZ>DP6ne$51f1Okf
zc#)z6VsEmsQv$JlGFK{XKKM&1J-J88D|4k1d-YP4fA_yAUCUppRC9hKn7yJiUI``!
z63=86SGsz3p4tl)YmG3wsp_XAJT#ZR3DA6fCr<NM^lbGIzIL_yj88QF1hr~_+MJ;F
zC8+%gqCxG<BWG`@F3Dj5jpM4U$YD_fiEsKFX&SiZx9O(J<HkwntxYa&lQw%RR%`S(
zR@<bUpVjP2-eglrPiwQ>oNIEA6jmua*RNIc|EsNj_g0Q#O;v~bE7j|&zxT~o`gWs9
zO(%AuiqPK!npY;-sek8g)zVqIR(tOu4o#<2cCCc@r!<_F#cFCPFVvLRxTG0T5o7jr
zZL_Jvo_i)U7V(&N$jmk6Hd$%x5%bn$X~9p^$v!8{l9k_>c?TG4o{wbIRP8*Zku@<(
z)5)k@lU4MQW@YAbjk_i5G>+%))l}*d(v1J6qYT0qv$Iu@X%HKPVd7Uy_nE(Eb2K%z
zFg0Uh$uxa<`jaW+sxGtBYnsf1Rezc$Z0RxA+Tv~g<I_aLm+x6j{Hy{^mKDr2Io$Bh
zSi(%mWXn57qjr}U#*-$Q7-tEp8DH<*sJ>&SzWT4ITWSf@PpBuE2x=HJ3ut68nyES1
zNUQ7p71qeg;!vL|AESQg+B?;aGb7X<&dgG~P&ikuq%=?MPsmiYdxl<WVx|mgUwvk&
zg4}xLd6XK6#)l`f`k4yYvYM$0J~!c&>oz*H)XMZ<fTgKn`UBJLd(WGcJkB%OU>$1A
z_Hve?&+Ifks|j{Sa>`nU@2@BsUXwRB5D&j-=<hXIpZ(uPBd=TA3_EQ0Y3co0qp>Zv
zUuUiNYHiD(En3r>{j~jlXXq@wJXbqZakh3%o~ZW5E2`@4Y$wz|_pen2xwGxBm+G?T
zQEFNDgVY!O^#Y51&(2oatol%4UO=*7gHf}LO1B;FehVhq<JuENJBlXDw{tJzy|m<<
zyp-M-F)>4Sb&jq$4F$(e^)PcSO&L)iO^&8S&6a~x)n_?Isjb`eQKOu(LnHF!Nz=>g
z&KaA}nq_)Ey~#N2jiaelZHKAM{Trr>xA~ereZ9-%(T+ybXd5}*mj)*^tIjv-Tv@kB
z+cUC4XWcVH9f!krbS!zdX)}~s=qyRRqFvyA$y7O_+ce%#!1NJYfstJ3HM1#;CYzXc
zw3;2%+-JJrx0R{#(_XVZ>8V;1uBdBuEUVM1z8R`{c2A+E<>XGSibrZ1^U@l$%Znqm
z&TzcaeEN5q%ER|+g8zQ8D&A}jlwB^eOO7WtT>8#?2jx=}9txPA>Qmm)%Pafk36py8
zqZSQG&k1TTlK!YY?y6B+uy3ZKqRl@QhLug~o4$0bgT(!&K9=s{7f`!d>?m(JD^_&k
zLS31-`>HbW_D^IDLq*k6mQ9!5D8(UrBC$%f$VXWH-|;DGzmICEXZr}NUu#TPzwy09
zRp$6UwWF-=>VNOgQ!D%MS*^%<xq96NJ++<d)@jIWSfo)stw|%xdAa)k%h77*E@)^7
zncr0R*mm4x*Ft}@_|?127R8)1D>!(<Y`4jFv(PUpCh5gJ=B(EBW?|Bf=I1A-C>cFJ
zpbDlnS1Q5iT}i)`Kp4aaVH2&H>ep7sX%wahsAqkdufE{iGW9=xiW+ZktW#fegkSx2
z@&@%wZ&s_vvPh__O`E6w*7AdLnbKwTP8JDu9jgz@{L|*CPq6%;Tx0n`SyL=o{m#$7
z8YjD*G!<4AYpmluthq*KpXR2GoLWDtcr|l<k~9*2p4OD+)7Ly}&7@)_FQBlsWUnG4
zZ>2Jbo|d>$MfYF5Vpx~I(*L#7l|k&3eKXZQ^siNy4ii)Z;fvYXU>Y<RaOCU_P`e*g
zwyPFD{HrE%_<;I0b`!N4Wk!t$oZr>U-m7R_%IH>$xX-Ru@ApIfLfQ(o%?{y8vA0Fk
z|HY;#nb|Zdy}bKTr8hNKS!u;hWu-eRs)1hnl@{dXsoa}yrFySsic-@hFR4lQ<wRd~
zDl0CC*(24wz)SVeyX*3<E4$^lXL&01PZv>RdbnS$kda?)_romZHPtJWsy5}T9r?$n
zR#doKO{7p=byA_a>O4zN<uGABMTXL5CDEr$Dj@m{XS>S9`G!i)6+e`E`1UD-*vfYE
z^qJH2bYFj1q+ih&tG{LspFRVJr~cj-^Yjl|I_jnFx~{MG-(CNd(pHr<M`To*cKRtr
z7FnxCo&Kg0V7FD})pA?q+dA#4<}VJYSUXQsk_u`w>8pQXI<Z98?C6PeX5~|FnKryF
zHLKZGWs>xKx0!^ai&@|HF0+02)~j+Ge^T4ESz7J2jj-BQmba=WYJF7~e{WS2ndGVt
z61V1RR|`#1RX?V>Uj2BYkVfpPE1LclbsEC6g4KP!($#ref;FZ;O4aa~$Dn!Y-)0TZ
zh3pzL4a79qSSPAmZ+@osr0J8Iv|NdLD?_ZBa?xcqHMW`Rv)O;>-|^V0+jVQT0hdXO
zfyTiSeYU`610H`J1J5#9{d1A04SsGAGziz^Q`~cLk;=N9AjRa}v%wg|2H~KlXNn*i
zB;I0esXp`XVf8r%tJPKCo2lQ5n5q`x&Y`jN+e7tV=?of<9m~}n9N5)71GG%Ieoryo
zXpmx(C3Df}P$#SDwAkrJ^91EhEdM?-E!=qC*lV+_=`^1XWBJehW}@zgOhvDsG?~AB
zp4p8fmZqh*TTSz}oi#f+JJw{wEF-foQJV}3%>4CxbQuiQtSt@h-)J&8kS=a;t?-aR
z@UK?=kZvhMt|f^Ei<@|~HXo|g46w}7EI5{<+5K8Uv*Bs2meEu>&1jd)nsWS)G_@{H
z(A>i$ubz99S>uCCyZWcu8k$FQ9X02)g=se25LGW+G+Aw1^%srx_i{ACrD`=!TAtB(
zW!0?lm-oHqj1DF(gA2M^S3W6f>?|lyf2wm{^N59>=ERrnT2%ppTE(;4G|SY&G(q}Y
z4LUS+g+w&|uHK~ioJB|DfRDeXZ#9>Z%?=kM$>M3oiq)+~nF)uD_wY<Gw!D*Wyd&zX
z(c`P_#;j&PjnpKf)b{4{s?UD*Lhao$M~#{P`!r<TS8MQl#3)a7X;I^>wbY3Fx>emR
zgHJ6?jYs|b`(<h|t98`A?V7LlKh8!yuirs+&Fd1?=bvV&%FAC=`+C}5nSaGyHA(SK
zWee4XDhdy>Rm%+)sJyIVP~}=NQ?>6?jI!v)7b-f++#2S0R%oc)>{m(3J)t3{&Z@C?
zhM~I0!W9|}hdNbdub8NvkO<be!^dH!w``ixu|0XF$2=DrKTeP^^_y&K`h1#~*;Ix3
z#)>yv&4detO?M0Kl(e^fF89x3mPB5=B45H|d%2+2d18kgSY@WzxXSpnG>D%(!6T7;
zi{D7W!_e5_-&w=Id|q0Q_Z%^7xVY7jYnH5W^5IaU{QH51(R0fUrmqk-_KRvY-5q_?
zWM8GC>7{cAO_i8mnRHA!Va&QY#O(k2$0h-bUCfd^Y}KT5STqh>EmM2;>y!G+tKZf0
z6__+?G`_2zzn!CWl<AfFuPc0NGmli5ZhLBHb|t&YRC7jy$-#7I^Qli+O%I>QGE+%A
zZT57Nve~TDYs{M7HfSfD{iRj@?xR+QQKMGFwVj&%n$xrtQ=+skUs|YDZ@WyZ(>6pa
z>X)b5q^&Y)#`AMkIp#>HE^fW7_CqjFo#*UiRquWkHNT#0)%j|9>WRTC^nR_orJwnZ
z-{9REMuP{_*6D{EuhRd*=c6}wNtwZV8*%+s-#7a2i@41val4upDlae%Vcc!<>ovdG
z<ZWf93JRfS`mu&)i<2&z&M^FIMr=Py_fWKI;{FAq*R=HIHqQ=~v|M^gL3hf0af72w
zs;5u<5>A?XP3cmhoUGGx1(m<M4u~llotIZxxkiqi*+S8xrc6rl=|`o>b7kdRFUBZ=
z<b>ads&hLlsdsQsQ+s&ET7A{Jc6H%})76ugD5@w-nyt3SuR;CN-Q#L0_w7_tf)<FV
zhsnx@a{5XHu3Ii2_H(wF`|DTAVy{07r7JB~yez{kquZ#imw94^9{-L2z0LnE^m<%b
z^iRbc)veg9qNg0TSU=FSRnNXolj!qb<Uaq<4dH*QEs+?f>wl(0cYFLq-M<=Zbk$z8
z>0T@q)#YBWM0Z7cknZUPkG0qto3!pt5Yh6LpQm}?)I*KuhJ0E}xD7Rf7iefz2QJfO
zET5^VerJP@=EjLy-^4|9%wHYRHu!%++x#wz&N0LLI{A()I-ErfIyNVFX}3&})sdMq
zSDWv2i1xKVkF{J~RkT|C{Ivg0>ef24vqt;4OPv;{(G9KIwE<?Yu3R-+$CzXO`KgQ9
zBB32-hO2zctqXh19R8b{_t+mc%|Bvcw(fnL`r>a(HOx0ISL4>GR9|?iPtDM#PksB-
z?dl#{{ptel5*kmF=c}ijTBxYvCZHC*AWhLU^}S-lmMFy)mG2cz7c5eIb4FC{=)IFF
z9)i(|J+}XizC=lz*|N?wd6s+F@bzXv)2Y*VjYIZHn>H-yG2P`5YEqRaYbp^YsQ$8l
ztp<p`n4PU^^-V`5;+u|I(!QCh7qhd~uRM=Zf7icOZH4qnbrz+3d7COeRafuZY5``}
zYO3)|)srn3tDQf<p**`uLDk3hl*&aub90}AF=m~|UYp&jC^2)s!)`w9t+Dy7$ARWs
z=SiD=J;7lXbvVrIpY?RP#@4s8Z`k&!@^9Rw@+PQW@<W5G8n<|ayt&*;xid<o@?mrJ
z<Zpl9ZhWN3+2rY$tp=9O?K<V}3`~CS;xh7|<6?5UH`hqG-rjIsUAZBw|AS7W_c2oT
z%~a8ydPp5a<HIZdda0>*p9QH?1+hWl016jSpEqUSOwigORnWL;`%YPv&|)XGJ2UpG
zu1uMse&zlG^;H|@sq1p`sVw{_th#z*mHLPJYic6={mSPR=Lj!Zx=-=%*TeG59vR6W
z5EPX=$DyYPqCw)!DJ}Bo?#1RGkeJzmUNzq{`_$EAKB`KrKdPGM9;v1y>#y23*HBG5
zXQSHBw+<?^A9ky5*6mhn_S08)UA<gw`s5akbF*h@JY2R%BO_(0O4I!|RgpU$8v5OL
zz-{t(AIvovJtu(2-&dZ!p=uf?s9F*GL=D6S^|wJ|^TxMdnY1u*n{GRsY^w2biOGeo
z=VtH4(o7pv1<a;R;x_A^eb9`T>8Kg8<F-9I7j)|qu4(h{?9n~2+DOlK-zlB*+VOf}
zIm)_Q+LCmQ{(9)nPc_r^31E=jbyG&6-*l5y?Jax0Py0LMjUu?kLE<2>f(y4L>uz0@
z2BqOU4U5%5Gzdq0(*ccfD1&GaA2h}dqP2MRbR74c(6W<%r0s9utrN`BsdeM`DxFCS
z?6v17EY+Tpouj?*FPFANuAMet;WZ8E_DwnpC)R08hOE=x&aR{r%ClJK?c@ZVBg?#W
zCQdWgA=bYRS2t))lV(%>KEX--K&ps(j{i-~osIJ}nxCFjpYvmjdb{}wEvuJVTGxAd
zOhvEynO+vXZoJF0-|(DWlG((jEyf3JZ<_kBZZv(;6m2S1u48)U*K@55)hO+=J@#6L
z2i|D9R6A?U+qyt&L)(0<P%Ay{rgiHz=S^FnHPMD0JT8)3Gh0#P_Dn?(efae(#R>an
zD$WLt2`%PUQ{O*Rk$2Kh!vmAz^<*_O4V%oh3_Jrx4Ytm7GH7{MWSIX@#bEu#Zo_3u
zqYcAeSes>hon>OVIL*{jS;@%dbAefI_EpohXYQKyJ{LC){Itxhc=ANE2fG?{UuRls
zf4H$$XRX#5oupVr-4|6?bvKmd>%0od*9lx{sjIoTTZgOppw#2UIvK^9a=Eh_+vPws
zhmD^sh`p)mhSH4?ZrO=;SxU{^x4GVZQIW{`+0O$SwdXy*RPy=!-RxTJiv@-Euu3ZD
zK4-I4v=Cr^IZ^3D`AgN@89S83zcecKdNC=3#8&+hR0d&?xSYDHQsiEL9rLc;nvbV_
z)Moj-K&RJ4NN4?oSvvK*7<4>lZPPAi{-(Xk;W22QL=oS38EpKF*n0Z=y%Kr#qs*!T
z@@xthDtkmHeCSi^x7;d!s(rO`1Gko}*;E<j&lNWna%b#N`MfPg@o&Up#n;C_D#F+>
z8YB*4&pak8dP!-z<j0OoQTeT%oPFu%#qt$y3vCGMkch9B6bG?eF8>!jVfjIIujL1|
z54n^1LF{IQcD0R`A5@wZ+SNh)^Ohe}K>R=df|WPQbg9k^OIH?qI!F0_>mo4wlIdz?
zb4wl-kT{5aYaX|pMo5RU|NnA%5T0UxN8w!OcKPdko(h}#Hz|fEKa#t~psd)&dr+HC
z?TpUkiT2t)f8?~Ut^26$e>6usZT1dr&hr*J*C(lIZ@#lpd#Xdbx^8ib3b$H_y5S{b
z4WmgP)mmG>sr%i~QBU;wq@LS!R$c3#nTAnPsP@YDVOm?im}_rcuwLu7OR!d_`!+4X
zw0tdxPIqnN=0vTRd97N1zjmtZUb|G*)M%n=v|xkMvk&tnYocZ;8SB1ReB%0D(dy+X
zMfux43VAWBbU9-Tbl=Ob)(fn=rgv52rQU`sF?vk_Z}eF0H|f1I4b_*B*`l{?;y$%I
zX5t#pGL6+v=sT--ZriKQz;aOCf}c;V<nk?*!gn3&n}7aR1C1T6`0J&b_)SL{M5FWH
z^{)lhE2<WAwUq9$o#idxKU*=nZMw36@=9sNXS<Y6h%JzxW+o<^-G5SLpR$nrT;5C*
zA&GiZ$*A|H+pl~#ITm%$^t-pB>6b0MCUPlUX6DyrO!j*ynbm*j(GS(&(|MV7L{BM3
zOwaXpf_`oGdc6&5&U%X@HtC8U_t!fUEvmoa@KklrAB$8=^iwseXG~IGpYl>=!*yp(
zlM-dMAGJ@_CZBz-#`}DyDyZB9VNkh=jA3Fhy3X^7YOzaz#)l_4YSZ5<sps<xs?M6+
zpe}WoNj*Mis@gj#R<$<SHq|N5wi(}I?=iNzp=FwIbhEM6$qEzZB~MJ!tAb5_L=>3J
zTN`O2RNi5dJwaPtaPB`fn+Z{BKi7Ow{k4u)ebu{0HKCMFWf11^_fWYZ9ickOBgLq0
zO{w0?x_Kt+4)>Y7mcMMkmwMHt|M+SBr!I4h=bg|uiTVEAXu+0kicj-gg<kx9C%NMd
z4<FOA31EJ2D!=sG`ahy-Tk2%qm^_jd)~V45>06|sd*`3}vsQD>_B-L4Z=bbmwy6D6
zubL8}R$lL~nX_b$#`kxvdM!IH=&arKK+n{9pPtB3cRiDXXY?4nIrVj>pVjqU9jm`s
ze39Pr@T=M){FAjFXZPxyEbh_C5L3|Ef1*z()JIZBYr||E>1}B`Ce|CZ)s}8jn?9{c
z&FtGe)oSUVssaA9RoA8+RIf~#teP9HtYYH0Sbfp6R+VS=-_<mAmT2fsXVO@%FjsTl
z?d_U%-A6S=Hyu(xxqP15l|{~)Y_soc{GZHYCgZTd(6>&}^apc=sej&P6VLtLruMTY
znrO}4YVx?((Cp*3qbBdCo;T$Q5jFoh=d0O7**T`wjzZ@8(IRG*jy+~~es!2Fn9gAO
zhuzGq^_H0am6<)-`A_EQ?UobO)AUKwbLhy`bNHpLCzbnB*Y3z-J+~?B`Zk5Z>R&GG
zSI@harSfw9D)q^7mTFUr9o6eTma1`x=cv_+IB2lO)T&2tDe5!YYwB`Th8g5{ZPpJh
znxOB>5N@!l<GP+xTaw;Q*<X5RY?tcAb-h=Mvo+K>m}#Wy5`Io2=!UPx->cFZ^NoAe
z#qKsLyL{|akGj;RzPM(lw%)ImI*;9cY2D?$q%~)@jE<v@lXi8DkdCJIZ|$F3u4oAb
z&DUO$*`veTyi4n0V~zGt{UzGBtiNdAVEd`PPiU4-STmE3yt||h_xc{~8*Lxd92hLr
zcbcA1JF$7Ly2FY?>hn}is^7ivN9}0QPc`imP3mF$&#7H)Ijyy#_K^<fyN{YZ3>?}{
zKR@b(S>D%nWKht_*zcgDs6A1uWX2<%rI%J})Hb?kS|9(ZvH7v9R`y{Rt#*bsEwyLs
zH13%#&}h|C*IM8+L38J#t(sTaH8u5=R5f{Yo@jIzxM{qraMjdY!lG&Malb}In6u^`
z)8iWNZ+WOM?q$&E-?~6;yUPUihK|i@`~K`yukV|zF8WPd)u?~H`j@=tYKek3#m~IE
ztstlFCJw@}ulW_qPUcGFl$*&4@Gg?8UDGES`{=lA8Lx~=lh_+^-;L=C>cWolSFc$s
zZhfL4_bDS$$@58{Jczzlv09;UhK2h3n767wi{`66_TQs=-<?bSpGB>D@f3HBpYa#e
z-Uk0rU02nhoEke(Eqi9R`t^2iHQoKj>a))rQnR(0sLrOPr&4=QUM(O+Pi6WwKGg}E
z*Q+PK=}`an;hmayzoF`l&_`-I{xNEIQ+w6UDf6o@-I}Vd)N4WfeF=WaD$8ziDs5*~
zRV$h7sOpt!p{isVpz{Cbdew407PWP6GgV)(Oi*sy$)b0zP*r;t&vjip-&?vmE_ZY^
zHs<M6iTu}{#M`YMDY{HIElNzsxqg+J-|J86Gj8^)zj~mj8F|oNbFp)Q=7ORIb*oF$
z)U3U)Ypg%xsG-)WEcIcVr{t;Lw@M(kiENXy#~VA@eYYZ%es`S};L@p8{l_V;R{o<`
z{Xu}aMz`)Zb+za5n&<cRYgQDm(wrrGT0Nv*MqQ;KNK?+FQ)AlFUaeyBcpcMs{aVr!
zHfhPXMQMxMT-SPkNJ)G7p=9lHzfW40-+pQFE3=vOmGYVfIUP1LQD!#(wm{N!z2;-H
zOc_z*%(r1?q2h+7rrWJebGWTkX1|J4?K~N&R#RA^-WYO6y}Rh4`W2Zqs(JouipL9z
z)!qJ2RoltjtlgdCs=3LSPiMWeqt<eUO0Cm>g|%AVWNNRHDb#w=TBz;I@=dGi%Qn-C
zL32z$W`vp>Eblfeb=_>XnSsT8yP&t3WM{hBRQ-8o4}Y&RlMfWrem~1o=N*@%R^FsC
zt=_+0+IP-u*76tLqZL$;qb<@uL+hWxKdqaa8no?KHtT%5G)pV`{RXX-XZds@zA<U{
zi7wPuv1!oRZZDzDymp~>)aC|LX-`eFggyGE9SgUcq%`j^$*-_6^$l2U@~(4&8N>ci
zlbc7Tm@e~vt@UBwO3k#q6s_{e-I`8Ec{Np?|7ymxS857;tJbtVK3$XbtEc9(Ns~3M
zuVT{pw(F$E=LzrC9fTD%6>?=XTbFie_TRKow`gCZap~Z8^{tF;YEg6hRM(|7Y1GxZ
zsV83lqxRvSgGPb2l6q?WBeh_uT(uormZ@o~x2VZ?C29)KoU0VFZikx4pOc#MzgX2>
zPn)Y3FPNa#7wDjVW|M{bjGH>@-hyFj_saLGzR&bkJE9e+c2|6j%9%r3)!H^itF5(~
zr~YK+3AMGBCg$@uiI_P*t2e7`JZ(BhRKe_(eVJ+7;|8;L>))AG&X6-(ETUj8!EJ8t
zsvTh>u{+zmB>S9c-p)!hp(b|o#<laz3k2*<lWVt||Hx=I<9KbME)sNCy;|?P$~*bF
zs#_}L)bHM2uj;U*SUsh+LCvBsN%dQPmU@EKQX{5D0n_|#+e~KvFf&atR5Q5~HqC@@
zzlveB(Js?%d$>)Kxb~Rb4VF-z98s_SM2kt?Y(t;MrX`y-{F9Gr9ML$bx*>A1>f`^8
z8nburR~Inx(lxFU)h=r_)U|rDQMZiyf{q3A9-V1BB6L#@pV9I7QmmU$EutIt=%>;T
z1{V2mZEF>#$fPKN=n31-s#sKHDVU4fDW+dqp#)+_UKi9gx*@2iqdHe7ZL*L~RoQc0
z^{QB%KZ)mc6}E}#G1sipwYtAoSJ1G)<ig}>#xt@cjJH%98w)rzn=I0KXmsr72BXGh
zSti?^@0%=Vx@%(2yFu;xr`u}(<L;=cl<=s}k<U@}x6f7U`}RfU^OIgxxkd%G;O!b}
zx4t|#eX@J0+4eIcW?I=#O}1a$V1C~v)U-BN){G(ifthO1f74&x{pL)(A5HEuoHv@F
zeb2<|P`~NsJ4;L!+I5>S&eJj0mV9IqR5Qo)h>EExXx|cOzY++8#&cjaNF2n6?Pv1p
zXI5|IS*9`n?jyCZ)t^-#F!ZYYe0f<lt!=)_?|}7cKk6;iUm3Tko!e8Qe{b?h9d2=c
z{X34&ba%d2(x18Hwr+pzG<}XuRl2u=n)G|;KGl=j^g;8t;Cd~Na%HW_!U~!n-??ai
zj!V(@41T2bkkLr{$R{<e?MyD(8d;hed<>g44z1x-i}?OR<6*M1hQ`f1>ZKWzHEw=e
zt@`D0kUHDLG>yb>I%=SPHE4_kL>oZY3>-Oo1GG*+86@WMO-E&Q&Nn^BbA@{PFI)8Q
z&T-cF{b8n`U#Fz6J$I%azw2H7nEWQa>NQ^aZ>Py<ZVOe?s9NKxVL4Y`W1FJ5hSK4i
z8bapR)n9zQr=GH(LGy(~g8Gvf4kP}&UIUHGH6~?09+(`O>0|U^yRGSTb55h}U!NJ@
z`n=1e>&#!{<vNSCET&m#26ODw<aII8oE@}PBRrx(Gp(mjW5Xp!&F$yXHK!&m*5KGa
zQA5z|s)m=BtA=90BFz`87Hg`oywaTbi(SKU+hesS98)yUe!Z^&+8@U(*`}s5rBQXK
zgRI)|4^vdVryf>MFk7u^a%z#v{>e+!qlKO;Pio*Yy7lO+$=&&(rYrM~nY@qUGO6%p
zHQoK1!?@J2!GvG-h@syC9pf8WVum{l`t&a}_Zwxu-E6pjimAb6wq7H_*J}+^_p%u5
z&=fVA#j@2P$X!-5cdMA@v#Ap_R0KC^d~@y8P-tn<yi@y7qiyQ~4ZFUTnmMy~Xk6T5
zt)A9jt`_K^uJ-uyJmsRvOiEEH|5by+<WxT@>#DqQZ&bEh|4R9??I(qvKrOYEk9Mm5
zuRSlja$Soe2rI04trWV+S3J|*STR8<R^gXctor?j(rQMh*QhW2HAn4hu(kU4n<1+H
zhi%n7J*w4y99XEjG*48OKjZ{xd=0#h4z!nU!oHbGAo@n*R?s+%(vr-DV1CsJZ@n$Q
zG<2?QQP#^)o1?dQs-f<7=^J|SoPE0M)@{-KeesFztF*bgNsGfwyCqu9p6Yv<y(!l=
zeNj2l+~@H|)5W^`OugeQ%?}lwG`*-j&D{PhlTM&bi`LpNHQJ1?#IzS)SfIVR;+WR_
zee<;*L}_S0HVV_W$&b{&;B;B%%*{PoRt{Y{LW#^eA)oGQo3GN<d7-jFXIJ}VZR?+o
zI!k*rbxx=~Gi$h{ZhFMX(7e;h(A;EenAtYN>1MNd7MmKZ+ih0+YLWSI@nz;e%)`_a
zKTJ@X|Aj^6y8C0b{r$S??e#0vUg^$J+46Ils_|<{_1Kea>VIe1Y3oL_XzTy)(n@#q
z(2{*PU9;OeODlqpNlWx^m{z!Vv6g+Cp4MVMK2yU_q9)VMR+;Xrk}?UtaLA-+@oJN{
z?`<ZZ%r~2U?iMrAj(%nGU~!I;;m0Rx{jmlr;n(ZcHCDe;=Xu1S!S!f@VpF)2+U^v2
z_4(I4Kx=A=+K(uF>$*zn@@eXo5z6YF0Ua7epO<Mwxa`sBTdS-V`&&fygG7+VyDAZl
zoe@<k50o#f|GwL<Dx6@ZA}jbu`A)?~)mqCR%JHJCDmJqxsOBtaS2fXjsin8$uGa1}
zAuVBTD=menADRmD=W6ZVGDlNJTT;u5+ea(OT~sUPT9wgUty2BkW0FQ%eu;+Hyyh6b
z{^DZL+~a0+@{_oMY2-?yQ%-*j_syPS(wo*}*7ZBxbfQeHsjaKNDc?tLlL>3L8YTU_
zVs`xU2a}x_yv;uFC2B7**3`C0-mdlCtz2u5(m~B*{5@K3$4s?kJleH(yqu<0k(Z{m
z;KXmex;2M%ChJJ(N9~T+_t1T#yQg@C{`ZwZdg7-2dP-?m^#0ne(aYd;G}#@~Y2f?X
z-Yjaeq}kz!1fw}$nayS|us3o0BW;o=!)7*jcZZ2-yP?*e@79_=Vcc3SYg{$&$XwNU
z-OsN1oz+_N<^m4QB$Jhz1v^e?ut_tiG0qRvNN%-JJNz(H-FZ@m`hh3E)j#x0sI=&}
zt10kysMZ*sQ_ZzZGD_epG?{+1$f*0sC!HTI?~Ow*uQa^&YQOQ^i*m*ry(Snw*6cH!
z+PlP9$w|)mVUm#Po4tQce9xqsDt(`1TJzG}l-Xy8NxKWP*^vkhQ%{|C)y2D?sJ~y9
zpk{h%g8HT72i4a(>{S=t-=S(8+@|vVZjyTU+e>OzwTxyzUUQnbW(Jry-MwkL$Di9|
z!i{corq_2(4<DOn8q6qRrZ~&pG}vsjy6M*m>aSb`G`q9UsGG&MYjnr|)_5YJuW2YF
zuTlDUj@r?4+*&N|C-r7rJfP2~^hRIe(j0x6%DMW2<?Ho#CGXYKQz$T)yro(1@Y5ao
z&C}x*g<l;~0nv(8OBI!?mMVhiGt=W0zr6bbW`p=U_divaQxH-6Ue~WS$LFX<!1@WQ
zTPCn-CLdg&zB6Kh`rjNj<(%K|R0VG-YrJ(A(WusH*2v%4tG=b7M4kJylE&Pgd+Ns%
z_%t?2*lNV<K30F{bHS`>gSy#vjs~;Zd6lMj*IqDHKB8cD{)vOx_ms<KSC7h@>J~+r
zMb70`-zKs|HL&ZCiZSzjRVI$xs(-DwsG4{FRb5l<qbh#cSUKOQLN)4eq}h@+y{2jH
z=H{-&u4Z3$8Jd|mE;Msc>^D{2b<V7Dfsxs9ZU*zUADPToTbP-jZ&+*b>%Fpt;0puu
zeR4A_-Z@_}yJahG;e1`f{OBD9i)P-11~*#+^kzTbWw2N()<D(vgnm@bI{n0VQ3jVU
zT+lzhd#6DdN0h<J^G+t_;uR(d_QA$CjMR(^Ugnxa94$1~^JX(q@;Pkug;mt_dJU(^
z8`r-^IXiC~w@k=2TdbvGo_uVhv4G!Nv(_1^1|@zrW~<zqOg)56O;Z-P8~oe8NH^)9
ztG=kzCcSv6efo1c*6YomV`3nf{YCGxoW4QZgGc&Z;u|%WvdmMxdo52Zuw|Z#n1q+c
zr+eiZB@?tY?|+lhT0C`*y8Hdf8nv&!X`j8fR{QO(N7`?U>~y|f?a=N|d!_BL-$~o4
zPD<zIy&c--k}I@ZoLDTZ^wyiXaCun7t!Fp4yX|WJvO&Z`;PFfI{cA$a&wk7@kDff)
z+}7ugSq9@Ivkl9im^wKznLSi`Xr`;X%=G-i{buWxXP8~w>}qx)uiGrQs7Xz`qE5}?
z+jAw8K6|x$PdBRi-t|?ESlqAL^ovhrYJ#h3u2Q8+y+MO&=GikUrA>R4HD7b6*PUxt
zSBqY(9&z@J%8Qd#$~#j<)TN7$s7dMVG+y_N+r-<uz{FwKCF8sqLZ*v^e;CiWxWTxu
z=fBC6L`7qR>o%sKb<f1Ce@;C#%lJCu9g|h3mYV+D$89!um9*(@X?IgL^KHf-DsGxE
zybUzHkYZ&z;jEzAyki2YkykdTzA@UZK4IA_b^o6t8aeL|t5h(YRqV=HsjjuhO0D4X
zbd#E00%l=LolHAjicLN@oH65=+ilWnJ<H^Cm4n%_Yh5O8mJiL|7Aoq^%Z|~L65OY^
z(~v_?)nI|%>X!L>2Jcqr39L!e-_`s{@92YFdgT)H40q4wHL@vaG>YQfWOP?~qv7&D
zJVxE&jRx9Z9~!Z6d@($~>3|VvJQFl`1)7@z(YSCL*CF-KOe@u;woFysa8Xp_xRRrK
z#QKTqT(^8Q9D><23@=Ys6};7|4jS7h2AeW|R=eupp>edcL4Dn01I>-+JTwKHQZ?5f
zT&%vHp;T?Q(t8a<jfon2BsvryS?yH;(Z)q56+tu%BlG=R->b!cR#lT?VOO<}mr(1N
z9;4PF<E`vu$fTxz_JUHY54YMD10m(=&Mmqs^4(f74|eGWH}dKQeCp8ouIaA(+kA_z
zRoOf3$H_}|`L^xQG1!)%78kWcLss5J?fBJDHN}Xh%3mxtsOYxxss*huRXZdQspc7T
zT6Na5X0zu@*PGor_tI>p!8TJ)VMFuf2H|GEJ50=te=Ri??O$WYzh|=90i*TGcU%^$
z>1W8R%zw$Fa(e$hWsfKOltJRVCIzT$R{98Lchp`|xs<G@&M0QBDz?X29mM_@ze~M0
ztU;-M<1RHeF>BR|N!n`F0^An*k&<R>)^l1kZA&s+y1>Gmg;~mCqMN40<;MGF#@(_O
z?I-@4v!DN<ne)I_>ki91&7(iAXf9{{t9jR}RV(?Rp_am$KCMu3GcCc`|C&l4T~*!F
z3sl8}g;dQxMk#Ne7pwBj=!?qPBbh2kkDgLKtIDM^=^3BOV;>u}t3gbvQJg#~33Dc>
zN9*lUKii+8p8bhm@xZ}4)e|$*)Y;GNQq%s=uJP-JyM{olm-=h@!x~xKJ2hlwo@sn?
zU9Y~MGe<2fVx7jS59S)*B-YAV_Zg_Z)-#kTUG`rp%ez@+_CF5k;)KPD<$UjzBy>K?
zrpb0IKD@-IHTQ6_7N^S<&08P)HJ{5^YC66Z)2e*at;x}>q%}v2Nvrzk6itcsr<7ME
zYN-0wURSZb7OmVpr(Q+hUP|S(nw;7mE_Stfdm_}l&wf{(E!v}F_+3Y@+b%}u>D%R6
zJ{%^x606yDT7O^F`Jd0H_ok&ur>AqV&XY7j^OQyArpB-NOg9+mnQTn<HS4sJG(E>E
zZO*rt+qCmdf!PU7UsI+%j;abKA6347S)|fhy;oUFeTua2gdnik&BI$%UN#me`Fy#j
z{FP^y$&z=<X3FlIrdfe$#>;ngn##Y>HQjJY-t5VXEhc9DY^EI1LMC&zo;5XEAZk)`
ztkCROS(WKOiTNh8Zu~McD|}~av+$C!Yui23vSksbptYHAZ`El`u{Tlg>#R}BsbT@o
z`{ubhtH<yqs6SfKq<(n!A(iyK6V&%N>uBhyYO255vt0fCS8fe!7E6sJtsUypYmTVx
z%lxgLa!gL0@yG>}+W8!Y-%cGgS(V6cyoBesQDtX{k#YG{lLC7c!%K@)Omli1Ov=~(
zP(M-RtD(^9uU^BVsQEg^LvsbUz2+JYZnZBFy=sNJuQjH=P|<k!@`hU2W<!lA){W}d
z!`(HrCi!aCv}I_zn(R}LDO;jCb;Vna9j!AolJ_-BzJ1fLXw?2e0)#E6OjPI&6p;Ec
zgF{YbVUfIpzOMA@n!9rIcB`qId&Q~EHPcfS6`!T<sd-$z(v(Rf=5>Y2q5kE{EARbL
zUwJ)O?LAAArhfM)O?82H8jUYyHE+#r)vyzqtGUUYMN`)-LGw?*BF(wVlA7H;VcHkx
z%vS%H^H_^PC{Q!@cCO~BtTruYM?USDT#}l5{yx;4B^#~zPPtEeMd>%K-m24DmaA>F
ztY>v=-h6&QEBpNk%|}0;Xr({-qWPjwT}wJlP!$v>AR2@l_svuU(M9`aD)vFyAhGsu
zx0R~`^wna+!_+=rnyp?lf0nwv@eTEPo0ckDBuOg2eW9$rYu6#wJyTwqa^*FfT~sx-
zSkLjwT=?QQ)2(fm7CM*SnS5Tq**vLK!0g7FHnX3W52U>}_((Ud5>ZiSzoR0UW+$Y2
z?7Py$V0{Vu+6;wOL0N@IYyT;P-@U5H;_yQHj+L|G$<-e6&jJ~goh;Smtl!%zcTHlK
zKY8ViilTd#((H{=njvvD>WeRWYHn!wrG7$ELW40oPW_8%g!+^<tm@(d3)I<#q}5w>
zuBfx`{j5Iw#ve7-85h;uU!|*m=b5VUAbF{Jp3Ne)-OqY7+O-SSTP|EQF@I}iz&!W8
z$pMS)#;+c;85v)nZ0vnW$>hzuX-1X)Ql<`S_9iD&Kbq;hE;iv#JZjGJZ@PKnqrWCX
z-$X1PJXmS=bY7CF9Cy9B8rKdpW3P6#YwbdcPYmmnEk)zizZI=g3Txk@w&L3?)zk^I
z<c~zYR@UsAru4LsO?&gM$Li*HCu>!mn5cc<qCso&1}AORE+1_x2?H(0n_4>aG{v>~
z!uM+K{%}e?BW|tc!H~ro7OIRIM$68sH%{Z$s1jPPuC%5_<H7Z<>b1wr4fCDt^^cxq
zHfAlnYt$|kY@n3HVr<)(U{GAK(C~4ai{U(87Q>Z-drdv|Ih!rixn*W`Ny5x@N153(
zH7m2PDXUG7TbwZUk34HSm3xL+fmW`3gQ=q0AJ4a{?&})W>LWwcy|TZn?bQoVJ)%EL
zO<HEHoW9gFwFm$GOw&wfn0>hKWEPU*U@HIPp{Z&}h}r9T-DYPV+&A688f~_X?}n+|
zga)mHT~D-{j)Z6$%$%yp)ETas@&CH!*S(IKm2WR-=Bt!x3L9_GJh)a#LykjIBjuZs
zx)V>QM$5z&jn&QEn%QessqfjnO|8Y^p~laEx$37j*lU>C@M_*@*VQOZNYQ-6Hd9l!
zcaNr@e1t|#??QEj)s>pln;vUK&i}0;5O-d~>*rLBBcgno5fPU(L>&w?k5@Wq`1Isy
zNbrPdnkBbtEPC3cv7&2-M*FTM8YK~2nhMVUHJ+Ze)ZB0>Q^WX2rN*H~In9L*^E6f$
z+G!s#?9e>#(W&!1+fs{tca4sLqL7ZwEqR^xC`oPeE&H{-B<E^t`>xR1d~u8R#`7Xt
z%UnCO&McUywet1}t=UO4wf@V>X=nfK)@tcqqqXCUin5hqp$drJsac{d_xPR?jBobZ
z9V`x#TgdoHP4t(KGTZVN^~BE&>c^%oRp((fSO2w+SEc#NK9wq8E{(;^_f+mNhN!)G
z->wwj%&ziCrAgh6QBVEk1zvUL3G-BLPv5Mn=D0vTyL_hFSt8eaYu$OQ9KKFeC6rrK
z%`=Tz?TovKy4KMds&kn4C?C;eSNmH%NA1&vZE9Jo+)Um-YA`*z`ip7rlcQ#*m#3Om
zh%1;)E}3GoDq*rIuiQ1$meNh8OfF}X{wGc61<_M<#FbT<o=AQA{9Ez<w{`Lfw{}QJ
z7amht&FCsulqqZ8|MZs8TQ*;F#}#&F`YoSKdnQgcyC>ytesYVKsr%Ni=5A-o%r%^x
z)sue3Y4CVXPy?+glAd}<1+<3;RMvULK2g(L-K`E1uWy>Iw79oH>5_bql9b*vI4$O}
z56o|#eo7T2H#wip$mw6P!LBW-hT(=64c@e97)I-S(Le2-V`v)hVi;7i)9^&$Rzo8L
z6^l<IoMzv@n^;`CV`!c$vch~$t%P~cJ1z^~Di-tEw<0XAEOxT+Fx68{vC314iIG-~
zEdQjkPNGajX4!6)8Ik^~hFbeoqW5SjU-hk4TB3Sh@rwOLA#ocn)s43&%YyLSgl#gW
z>K)3}alJw^YN9GfZ{3n5cAtZQ@j9KqPmQ&|ernK3_x`W*`^Qf0=!?H~c4l$wh^Hs$
zoXeBZStuB+b0D)@%WH+OW`OY_&82c&nnxo_H4Z%B)H-BSt0C=pOS4Sxu!g9Ok%rs-
zAoX2S4K%JdEC!u#rS`Octt#lep_g?lKx_KcK<5&z|LdhXh2fND$0BLX)xV-N6R%rq
zzM3zh$;48n8JGJ?W0t(C#-YeI%?H<(YKX4CqHZLyRBi2s1ode1#j1BLxK*QN<<;xr
zvehP=SgXwCs8H`;c12CgINEr9jHpp$!&hV7n;VU#@(YZ<u`rs<sCjSXVl>If{M>Zo
zt7_@S%d{REU)&sJcr#(D@ya|UqoOU9Ms<ZQ#=qZln3xH_F<k!Tp0R;NqVbK%+q47r
zEYRv`n5(TK-KL#=X1<n?d4_i5{!LnUJsP!N>VMY?X5!Lzs8Z1l(s`~mLr79LE4x!i
zVOg4v^qM0&R}M((%62}~PS|%tM<;8h&h$meCf%nU%`_GrHt~>qYP|5@A=AR+tj3cq
z1WYgHiJM*e(P%p3-zignb0_UPk?d;X4Jw)@uYYLVJIb#m$1zjw%7Vq3(;XJ88MLm{
z673V$x+Rrv>QonEI=f%iT)4&1>}KX=lY6(b&G+RmG@az7VS1!{ohh?~lqvIy2D9gB
zvrH0QCYmq!b<)gt!!$GZ^abYAn|00AU5d<>-d=6a(ClXRP41r7y}glIH@ACfIn>|M
zoX_W~S$0NDD@7_@^JdQq&1+ZgYWnnU*VJw2HC+<SW~!61$n04muQ|`H>n8i>$(U!E
z%rkj%*3fj~$2il_9Sx=_3K{B)ge^4=Pq0<xYH3tmedB~m=h02dfxePzw|I=zWR{#%
zD>f2UW52ad?c*UU4G_(9+(0dDw~v}vm%f_fvw5nv=lsC(*LRhwg61Vaa}*#9nxBBt
zAaM{MHeV58IZd5+vaH6|o=Eke`12Z1^iFHs5U|qhSKXujKWwFHQYV{cle?M5{7vVT
zkKJ(*O8hUY^mQe(qV+RhS+9<NDi+og!0b$swMwVcloULjG&MHNI;ehmnvMEB_m66M
lT<z-K^=sA7EY(;4{eFSE7qgeTin^Q{F?~jmns3k(>jA%NpeO(U

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index b/tensorflow/contrib/tensorrt/test/quantization_mnist_test_data/model.ckpt-14070.index
deleted file mode 100644
index d338a78f056628943e0d7a59899e11fa5444b717..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 961
zcmZQzVB=tvV&Y(A;O0(BElD(tFUT)XE#hDlV$fhQT~;k<#lp^Ogb-9<(O@w>WoOgF
zz%7)VpI2s-qMwwRSj@pF#3IDRp`gH_(ZFb$JojM&H><F|V@hH!T&9B&Bz5_gR<8h)
zaJ(TxoCzcz+LL~aolQPFwJ0w&hl5c_K!}Nh8HyRfCU-DvG;o@(-(#uG%PJ2u8jCU>
zkg}UI);9?;$%C!IqDB;?CVPk2LpFBdctd0#I4E?;X*4jIx~>RW#f$0%2Zas=kd$AJ
zPr4A27aSBiltAKF-@j|*WK+QE1qC4{4hMw}6^(`iET%#23!?d16|wmNs_qF3SmB)<
zJ`NEkMGRj+RW0EJt2!33+KPcoG$l2!I2Gm<Rv~7N1{Q^mC%hUB0;Vji{LT#gNGf1f
z!<2miD=Yo`Jcgf16l@(#riBk=K)Qb~CmWj-+-sa5??A1X!l%*Dz+w85;b8?2s}#&+
z6cu7%6_f5y(Gg^l0vnGaEeDpqBbon?6+Jq*6gsA;X=s>sX;u{RphgFm!UnL^o<i40
zf=JQ9rEmi*p0fPuISv#zgMvheiGxd_VXB%&!vaQA->aVv^C7z#p<=5#SjC>p$sdH_
zZbnF7)c{M&RCmQPaPz0<<R>NO#21&O7I1J1F(@>A)zsiHozOD*J_7>-!%Id61||^W
y(DO~%Aes?G9Qd?w0|O&-2E%JM5Em@gvh&&A4{RKN+4w<1_;L8}hHjNo_uBx6l+IZI

diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 790b6955a5..f4653505f7 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -108,7 +108,6 @@ std::set<string> GetOpsFormatAgnostic() {
                                           "Conj",
                                           "Cos",
                                           "Cosh",
-                                          "Dequantize",
                                           "Digamma",
                                           "Div",
                                           "Elu",
-- 
GitLab


From 89deaf06c19a3eb1d5236d328b6b8cdde7238271 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 21 Nov 2018 15:48:04 -0800
Subject: [PATCH 0710/1554] Monkey patch _os_exit to make sure exit handlers
 are proper called.

PiperOrigin-RevId: 222470718
---
 tensorflow/compiler/tests/xla_test.py    | 5 +++++
 tensorflow/python/platform/googletest.py | 9 ++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py
index 98a41981cf..d15c073457 100644
--- a/tensorflow/compiler/tests/xla_test.py
+++ b/tensorflow/compiler/tests/xla_test.py
@@ -22,6 +22,7 @@ import contextlib
 import os
 import random
 import re
+import sys
 
 import numpy as np
 
@@ -38,6 +39,10 @@ from tensorflow.python.platform import flags
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
+# TODO(b/35678764): Disable monkeypatched exit handlers once we find a proper
+# solution to invoke exit handlers during program exit.
+test._googletest.SetOSExit(sys.exit)
+
 FLAGS = flags.FLAGS
 
 flags.DEFINE_string('test_device', None,
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index 8141cf92c5..c8312c153e 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -47,6 +47,13 @@ unittest_main = main
 # directory only once per test binary invocation.
 _googletest_temp_dir = ''
 
+_os_exit = sys.exit
+
+
+def SetOSExit(exit_func):
+  global _os_exit
+  _os_exit = exit_func
+
 
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
@@ -61,7 +68,7 @@ def g_main(argv):
       except IOError:
         sys.stderr.write('Error opening TEST_SHARD_STATUS_FILE (%s). Exiting.'
                          % os.environ['TEST_SHARD_STATUS_FILE'])
-        sys.exit(1)
+        _os_exit(1)
     finally:
       if f is not None: f.close()
 
-- 
GitLab


From f96b9ab51fbad077eb75af8595dcd7779a01b97b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 15:55:03 -0800
Subject: [PATCH 0711/1554] This CL marks the `ReduceDataset` op as not
 differentiable, to avoid crashes when it gets caught inside functions whose
 gradients are computed.

PiperOrigin-RevId: 222471432
---
 tensorflow/python/data/ops/dataset_ops.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index d040d17322..bd57c9f1d5 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -57,6 +57,9 @@ from tensorflow.python.util import function_utils
 from tensorflow.python.util.tf_export import tf_export
 
 
+ops.NotDifferentiable("ReduceDataset")
+
+
 @tf_export("data.Dataset", v1=[])
 @six.add_metaclass(abc.ABCMeta)
 class DatasetV2(object):
-- 
GitLab


From f51e2622b14c3e359396bdc33732813602f9af68 Mon Sep 17 00:00:00 2001
From: Taylor Robie <taylorrobie@google.com>
Date: Wed, 21 Nov 2018 15:59:30 -0800
Subject: [PATCH 0712/1554] remove deprecated `squeeze_dims` arg from
 `tf.squeeze`

PiperOrigin-RevId: 222471820
---
 tensorflow/python/ops/array_ops.py              | 8 +++++++-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 2 +-
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 3 +++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 0f80a28d7f..8e9b8950bc 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3057,7 +3057,7 @@ def sequence_mask(lengths, maxlen=None, dtype=dtypes.bool, name=None):
       return gen_math_ops.cast(result, dtype)
 
 
-@tf_export("squeeze")
+@tf_export(v1=["squeeze"])
 @deprecation.deprecated_args(None, "Use the `axis` argument instead",
                              "squeeze_dims")
 def squeeze(input, axis=None, name=None, squeeze_dims=None):
@@ -3107,6 +3107,12 @@ def squeeze(input, axis=None, name=None, squeeze_dims=None):
   return gen_array_ops.squeeze(input, axis, name)
 
 
+@tf_export("squeeze", v1=[])
+def squeeze_v2(input, axis=None, name=None):
+  # pylint: disable=redefined-builtin
+  return squeeze(input, axis, name)
+
+
 @tf_export("where")
 def where(condition, x=None, y=None, name=None):
   """Return the elements, either from `x` or `y`, depending on the `condition`.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index f3099f7d82..b4b4827584 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1058,7 +1058,7 @@ tf_module {
   }
   member_method {
     name: "squeeze"
-    argspec: "args=[\'input\', \'axis\', \'name\', \'squeeze_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'input\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "stack"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 86c6caa6fe..05e839aa9d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -198,6 +198,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "keep_dims": "keepdims",
             "reduction_indices": "axis"
         },
+        "tf.squeeze": {
+            "squeeze_dims": "axis",
+        },
     }
 
     # Mapping from function to the new name of the function
-- 
GitLab


From 7dcf314bcea609f552830122e9f94625c8687d7e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 16:18:17 -0800
Subject: [PATCH 0713/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 222473956
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 54 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 54 +++++++++++++++++++
 2 files changed, 108 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index dd1aaf966e..309154d74d 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21969,6 +21969,33 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalMaxIntraOpParallelismDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "max_intra_op_parallelism"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalNonSerializableDataset"
   input_arg {
@@ -22040,6 +22067,33 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalPrivateThreadPoolDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_threads"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalSleepDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index bc35ce7513..f690635558 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10432,6 +10432,33 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalMaxIntraOpParallelismDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "max_intra_op_parallelism"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalNonSerializableDataset"
   input_arg {
@@ -10503,6 +10530,33 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalPrivateThreadPoolDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_threads"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalSleepDataset"
   input_arg {
-- 
GitLab


From 8dcb3b69eb561be3d607a109041ed225395dad53 Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Wed, 21 Nov 2018 16:24:20 -0800
Subject: [PATCH 0714/1554] [XLA] Add tests for DotGeneral where LHS and RHS
 are not of equal rank.

PiperOrigin-RevId: 222474570
---
 .../compiler/xla/tests/dot_operation_test.cc  | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 6c0847a875..25091b8d5d 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -637,6 +637,76 @@ XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMul) {
       {x_data.get(), y_data.get()}, this->error_spec_);
 }
 
+#ifndef XLA_TEST_BACKEND_CPU
+// TODO(b/74459949): failed on CPU on 2018-10-29.
+XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMulR3LhsR2Rhs) {
+  using T = TypeParam;
+
+  XlaBuilder builder(this->TestName());
+  auto x =
+      Parameter(&builder, 0, ShapeUtil::MakeShapeWithType<T>({2, 2, 2}), "x");
+  auto y = Parameter(&builder, 1, ShapeUtil::MakeShapeWithType<T>({2, 2}), "y");
+
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(1);
+  dnums.add_rhs_contracting_dimensions(1);
+  dnums.add_lhs_batch_dimensions(0);
+  dnums.add_rhs_batch_dimensions(0);
+
+  DotGeneral(x, y, dnums);
+
+  auto x_data =
+      this->client_
+          ->TransferToServer(LiteralUtil::CreateR3FromArray3D<T>(
+              {{{1.0f, 2.0f}, {3.0f, 4.0f}}, {{5.0f, 6.0f}, {7.0f, 8.0f}}}))
+          .ConsumeValueOrDie();
+
+  auto y_data = this->client_
+                    ->TransferToServer(LiteralUtil::CreateR2FromArray2D<T>(
+                        {{1.0f, 0.0f}, {0.0f, 1.0f}}))
+                    .ConsumeValueOrDie();
+
+  this->template ComputeAndCompareR2<T>(
+      &builder,
+      /*expected=*/{{1.0f, 2.0f}, {7.0f, 8.0f}}, {x_data.get(), y_data.get()},
+      this->error_spec_);
+}
+
+// TODO(b/74459949): failed on CPU on 2018-10-29.
+XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMulR2LhsR3Rhs) {
+  using T = TypeParam;
+
+  XlaBuilder builder(this->TestName());
+  auto x = Parameter(&builder, 0, ShapeUtil::MakeShapeWithType<T>({2, 2}), "x");
+  auto y =
+      Parameter(&builder, 1, ShapeUtil::MakeShapeWithType<T>({2, 2, 2}), "y");
+
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(1);
+  dnums.add_rhs_contracting_dimensions(1);
+  dnums.add_lhs_batch_dimensions(0);
+  dnums.add_rhs_batch_dimensions(0);
+
+  DotGeneral(x, y, dnums);
+
+  auto x_data = this->client_
+                    ->TransferToServer(LiteralUtil::CreateR2FromArray2D<T>(
+                        {{1.0f, 0.0f}, {0.0f, 1.0f}}))
+                    .ConsumeValueOrDie();
+
+  auto y_data =
+      this->client_
+          ->TransferToServer(LiteralUtil::CreateR3FromArray3D<T>(
+              {{{1.0f, 2.0f}, {3.0f, 4.0f}}, {{5.0f, 6.0f}, {7.0f, 8.0f}}}))
+          .ConsumeValueOrDie();
+
+  this->template ComputeAndCompareR2<T>(
+      &builder,
+      /*expected=*/{{1.0f, 2.0f}, {7.0f, 8.0f}}, {x_data.get(), y_data.get()},
+      this->error_spec_);
+}
+#endif  // XLA_TEST_BACKEND_CPU
+
 XLA_TYPED_TEST(DotOperationTest_F16F32F64CF64, GeneralMatMulMultipleBatch) {
   using T = TypeParam;
 
-- 
GitLab


From 8b663936f830ae4223901753767fe41d5e87ee20 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 16:29:42 -0800
Subject: [PATCH 0715/1554] This CL switches isinstance(x, dataset_ops.Dataset)
 checks to compare against dataset_ops.DatasetV2

PiperOrigin-RevId: 222475047
---
 tensorflow/contrib/tpu/python/tpu/datasets.py          |  2 +-
 tensorflow/contrib/tpu/python/tpu/keras_support.py     | 10 +++++-----
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py     |  2 +-
 tensorflow/python/autograph/operators/control_flow.py  |  2 +-
 tensorflow/python/distribute/values.py                 |  2 +-
 .../python/keras/engine/distributed_training_utils.py  |  2 +-
 tensorflow/python/keras/engine/training.py             | 10 +++++-----
 tensorflow/python/keras/engine/training_generator.py   |  2 +-
 tensorflow/python/training/distribute.py               |  2 +-
 9 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/datasets.py b/tensorflow/contrib/tpu/python/tpu/datasets.py
index c694e9c1bc..d61c824eab 100644
--- a/tensorflow/contrib/tpu/python/tpu/datasets.py
+++ b/tensorflow/contrib/tpu/python/tpu/datasets.py
@@ -133,7 +133,7 @@ def StreamingFilesDataset(files,
   with ops.device('/job:%s' % file_reader_job):
     if isinstance(files, str):
       source_dataset = dataset_ops.Dataset.list_files(files)
-    elif isinstance(files, dataset_ops.Dataset):
+    elif isinstance(files, dataset_ops.DatasetV2):
       source_dataset = files
     else:
       raise ValueError('files was not a string or a dataset: %s' % files)
diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index 73753cd918..c4db4aa0c7 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -769,7 +769,7 @@ class TPUDatasetInfeedManager(TPUInfeedManager):
 
   def _verify_dataset_shape(self, dataset):
     """Verifies a dataset is of an appropriate shape for TPUs."""
-    if not isinstance(dataset, dataset_ops.Dataset):
+    if not isinstance(dataset, dataset_ops.DatasetV2):
       raise ValueError('The function passed as the `x` parameter did not '
                        'return a `tf.data.Dataset`.')
     if not isinstance(dataset.output_classes, tuple):
@@ -1465,7 +1465,7 @@ class KerasTPUModel(models.Model):
       assert not self._numpy_to_infeed_manager_list  # Ensure empty.
 
       infeed_managers = []  # Managers to clean up at the end of the fit call.
-      if isinstance(x, dataset_ops.Dataset):
+      if isinstance(x, dataset_ops.DatasetV2):
         # TODO(b/111413240): Support taking a tf.data.Dataset directly.
         raise ValueError(
             'Taking a Dataset directly is not yet supported. Please '
@@ -1491,7 +1491,7 @@ class KerasTPUModel(models.Model):
           y = infeed_manager.dummy_y
           infeed_managers.append((x, infeed_manager))
 
-      if isinstance(validation_data, dataset_ops.Dataset):
+      if isinstance(validation_data, dataset_ops.DatasetV2):
         # TODO(b/111413240): Support taking a tf.data.Dataset directly.
         raise ValueError(
             'Taking a Dataset directly is not yet supported. Please '
@@ -1550,7 +1550,7 @@ class KerasTPUModel(models.Model):
     with _tpu_session_context():
       # Managers to clean up at the end of the evaluate call.
       infeed_managers = []
-      if isinstance(x, dataset_ops.Dataset):
+      if isinstance(x, dataset_ops.DatasetV2):
         # TODO(b/111413240): Support taking a tf.data.Dataset directly.
         raise ValueError(
             'Taking a Dataset directly is not yet supported. Please '
@@ -1922,7 +1922,7 @@ class KerasTPUModel(models.Model):
     if validation_data:
       if (isinstance(validation_data, iterator_ops.Iterator) or
           isinstance(validation_data, iterator_ops.EagerIterator) or
-          isinstance(validation_data, dataset_ops.Dataset)):
+          isinstance(validation_data, dataset_ops.DatasetV2)):
         raise ValueError('KerasTPUModel cannot handle a Dataset or Iterator '
                          'for validation_data. Please instead pass a function '
                          'that returns a `tf.data.Dataset`.')
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 9525121ebb..2aaf65881b 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -3072,7 +3072,7 @@ class _Inputs(object):
   @staticmethod
   def from_input_fn(return_values):
     """Returns an `_Inputs` instance according to `input_fn` return value."""
-    if isinstance(return_values, dataset_ops.Dataset):
+    if isinstance(return_values, dataset_ops.DatasetV2):
       dataset = return_values
       return _Inputs(dataset=dataset)
 
diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index 6eedd695a7..1a35efedfa 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -61,7 +61,7 @@ def for_stmt(iter_, extra_test, body, init_state):
   """
   if tensor_util.is_tensor(iter_):
     return _known_len_for_stmt(iter_, extra_test, body, init_state)
-  elif isinstance(iter_, dataset_ops.Dataset):
+  elif isinstance(iter_, dataset_ops.DatasetV2):
     return _dataset_for_stmt(iter_, extra_test, body, init_state)
   else:
     return _py_for_stmt(iter_, extra_test, body, init_state)
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 5f69323bff..727e491a2f 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -1433,7 +1433,7 @@ class InputFunctionIterator(InputIteratorImpl):
       # TODO(priyag): We should probably explicitly specify CPU device on worker.
       with ops.device(worker):
         result = input_fn(ctx)
-        if not isinstance(result, dataset_ops.Dataset):
+        if not isinstance(result, dataset_ops.DatasetV2):
           raise ValueError("input_fn must return a tf.data.Dataset.")
         iterator = _SingleWorkerDatasetIterator(result, worker, devices)
         iterators.append(iterator)
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 7d915544fc..881f8b36ca 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -381,7 +381,7 @@ def validate_inputs(x, y, distribution_strategy):
 
   if is_tpu_strategy(distribution_strategy):
     for i in [x, y]:
-      if isinstance(i, dataset_ops.Dataset):
+      if isinstance(i, dataset_ops.DatasetV2):
         shapes = nest.flatten(i.output_shapes)
         try:
           s = next(s for s in shapes if not s.is_fully_defined())
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 56f069c057..888d8eb942 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1000,7 +1000,7 @@ class Model(Network):
     # TODO(anjalisridhar): Remove this check once we refactor the
     # _standardize_user_data code path. This check is already present elsewhere
     # in the codebase.
-    if check_steps and isinstance(x, dataset_ops.Dataset) and steps is None:
+    if check_steps and isinstance(x, dataset_ops.DatasetV2) and steps is None:
       raise ValueError('When using Datasets as input, '
                        'you should specify the `{steps_name}` argument.'
                        .format(steps_name=steps_name))
@@ -1043,7 +1043,7 @@ class Model(Network):
         x = dataset_ops.Dataset.from_tensor_slices(var_x)
         x = x.batch(batch_size, drop_remainder=drop_remainder)
 
-    assert isinstance(x, dataset_ops.Dataset)
+    assert isinstance(x, dataset_ops.DatasetV2)
 
     with self._distribution_strategy.scope():
       iterator = self._distribution_strategy.make_dataset_iterator(x)
@@ -1132,7 +1132,7 @@ class Model(Network):
           shuffle=shuffle)
       return iterator, None, None
 
-    if isinstance(x, dataset_ops.Dataset):
+    if isinstance(x, dataset_ops.DatasetV2):
       if context.executing_eagerly():
         x = x.make_one_shot_iterator()
       else:
@@ -1691,7 +1691,7 @@ class Model(Network):
     if validation_data:
       if (isinstance(validation_data, iterator_ops.Iterator) or
           isinstance(validation_data, iterator_ops.EagerIterator) or
-          isinstance(validation_data, dataset_ops.Dataset)):
+          isinstance(validation_data, dataset_ops.DatasetV2)):
         val_x = validation_data
         val_y = None
         val_sample_weight = None
@@ -2195,7 +2195,7 @@ class Model(Network):
     inputs, _, _ = self._standardize_user_data(x)
     if self.run_eagerly:
       if (isinstance(inputs, iterator_ops.EagerIterator) or
-          (isinstance(inputs, dataset_ops.Dataset))):
+          (isinstance(inputs, dataset_ops.DatasetV2))):
         inputs = training_utils.cast_if_floating_dtype(inputs)
       elif isinstance(inputs, collections.Sequence):
         inputs = [
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 45247a2751..988bed5170 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -67,7 +67,7 @@ def fit_generator(model,
     else:
       raise ValueError('Please specify the `steps_per_epoch` argument.')
 
-  if (isinstance(validation_data, dataset_ops.Dataset) and
+  if (isinstance(validation_data, dataset_ops.DatasetV2) and
       context.executing_eagerly()):
     validation_data = validation_data.make_one_shot_iterator()
   val_gen = (data_utils.is_generator_or_sequence(validation_data) or
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index f930a89f99..ba3334108b 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -991,7 +991,7 @@ class DistributionStrategyExtended(object):
   def _call_dataset_fn(self, dataset_fn):
     """Call the `dataset_fn` with `input_context` as argument."""
     result = dataset_fn()
-    if not isinstance(result, dataset_ops.Dataset):
+    if not isinstance(result, dataset_ops.DatasetV2):
       raise ValueError(
           "dataset_fn() must return a tf.data.Dataset when using a "
           "tf.distribute.Strategy.")
-- 
GitLab


From 2ecbae0e9cab99682187a6ffbce03e154fbf51bb Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 21 Nov 2018 16:35:40 -0800
Subject: [PATCH 0716/1554] Move gfile.* functions to io.gfile.* and change API
 for TF 2.0.

PiperOrigin-RevId: 222475688
---
 tensorflow/python/lib/io/file_io.py           | 239 +++++++++++++++---
 .../tools/api/generator/api_init_files.bzl    |   1 -
 .../tools/api/generator/api_init_files_v1.bzl |   1 +
 .../api/golden/v1/tensorflow.io.gfile.pbtxt   |  51 ++++
 .../tools/api/golden/v1/tensorflow.io.pbtxt   |   4 +
 .../api/golden/v2/tensorflow.gfile.pbtxt      |  47 ----
 .../api/golden/v2/tensorflow.io.gfile.pbtxt   |  44 ++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |  19 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  47 ++++
 10 files changed, 374 insertions(+), 83 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt

diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py
index c8aa5311d9..645cf8e95c 100644
--- a/tensorflow/python/lib/io/file_io.py
+++ b/tensorflow/python/lib/io/file_io.py
@@ -258,7 +258,7 @@ def file_exists(filename):
   return file_exists_v2(filename)
 
 
-@tf_export("io.gfile.exists", v1=[])
+@tf_export("io.gfile.exists")
 def file_exists_v2(path):
   """Determines whether a path exists or not.
 
@@ -280,7 +280,7 @@ def file_exists_v2(path):
   return True
 
 
-@tf_export("gfile.Remove")
+@tf_export(v1=["gfile.Remove"])
 def delete_file(filename):
   """Deletes the file located at 'filename'.
 
@@ -291,8 +291,22 @@ def delete_file(filename):
     errors.OpError: Propagates any errors reported by the FileSystem API.  E.g.,
     NotFoundError if the file does not exist.
   """
+  delete_file_v2(filename)
+
+
+@tf_export("io.gfile.remove")
+def delete_file_v2(path):
+  """Deletes the path located at 'path'.
+
+  Args:
+    path: string, a path
+
+  Raises:
+    errors.OpError: Propagates any errors reported by the FileSystem API.  E.g.,
+    NotFoundError if the path does not exist.
+  """
   with errors.raise_exception_on_not_ok_status() as status:
-    pywrap_tensorflow.DeleteFile(compat.as_bytes(filename), status)
+    pywrap_tensorflow.DeleteFile(compat.as_bytes(path), status)
 
 
 def read_file_to_string(filename, binary_mode=False):
@@ -331,7 +345,7 @@ def write_string_to_file(filename, file_content):
     f.write(file_content)
 
 
-@tf_export("gfile.Glob")
+@tf_export(v1=["gfile.Glob"])
 def get_matching_files(filename):
   """Returns a list of files that match the given pattern(s).
 
@@ -341,28 +355,44 @@ def get_matching_files(filename):
   Returns:
     A list of strings containing filenames that match the given pattern(s).
 
+  Raises:
+    errors.OpError: If there are filesystem / directory listing errors.
+  """
+  return get_matching_files_v2(filename)
+
+
+@tf_export("io.gfile.glob")
+def get_matching_files_v2(pattern):
+  """Returns a list of files that match the given pattern(s).
+
+  Args:
+    pattern: string or iterable of strings. The glob pattern(s).
+
+  Returns:
+    A list of strings containing filenames that match the given pattern(s).
+
   Raises:
     errors.OpError: If there are filesystem / directory listing errors.
   """
   with errors.raise_exception_on_not_ok_status() as status:
-    if isinstance(filename, six.string_types):
+    if isinstance(pattern, six.string_types):
       return [
           # Convert the filenames to string from bytes.
           compat.as_str_any(matching_filename)
           for matching_filename in pywrap_tensorflow.GetMatchingFiles(
-              compat.as_bytes(filename), status)
+              compat.as_bytes(pattern), status)
       ]
     else:
       return [
           # Convert the filenames to string from bytes.
           compat.as_str_any(matching_filename)
-          for single_filename in filename
+          for single_filename in pattern
           for matching_filename in pywrap_tensorflow.GetMatchingFiles(
               compat.as_bytes(single_filename), status)
       ]
 
 
-@tf_export("gfile.MkDir")
+@tf_export(v1=["gfile.MkDir"])
 def create_dir(dirname):
   """Creates a directory with the name 'dirname'.
 
@@ -373,14 +403,31 @@ def create_dir(dirname):
     The parent directories need to exist. Use recursive_create_dir instead if
     there is the possibility that the parent dirs don't exist.
 
+  Raises:
+    errors.OpError: If the operation fails.
+  """
+  create_dir_v2(dirname)
+
+
+@tf_export("io.gfile.mkdir")
+def create_dir_v2(path):
+  """Creates a directory with the name given by 'path'.
+
+  Args:
+    path: string, name of the directory to be created
+
+  Notes:
+    The parent directories need to exist. Use recursive_create_dir instead if
+    there is the possibility that the parent dirs don't exist.
+
   Raises:
     errors.OpError: If the operation fails.
   """
   with errors.raise_exception_on_not_ok_status() as status:
-    pywrap_tensorflow.CreateDir(compat.as_bytes(dirname), status)
+    pywrap_tensorflow.CreateDir(compat.as_bytes(path), status)
 
 
-@tf_export("gfile.MakeDirs")
+@tf_export(v1=["gfile.MakeDirs"])
 def recursive_create_dir(dirname):
   """Creates a directory and all parent/intermediate directories.
 
@@ -389,14 +436,29 @@ def recursive_create_dir(dirname):
   Args:
     dirname: string, name of the directory to be created
 
+  Raises:
+    errors.OpError: If the operation fails.
+  """
+  recursive_create_dir_v2(dirname)
+
+
+@tf_export("io.gfile.makedirs")
+def recursive_create_dir_v2(path):
+  """Creates a directory and all parent/intermediate directories.
+
+  It succeeds if path already exists and is writable.
+
+  Args:
+    path: string, name of the directory to be created
+
   Raises:
     errors.OpError: If the operation fails.
   """
   with errors.raise_exception_on_not_ok_status() as status:
-    pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(dirname), status)
+    pywrap_tensorflow.RecursivelyCreateDir(compat.as_bytes(path), status)
 
 
-@tf_export("gfile.Copy")
+@tf_export(v1=["gfile.Copy"])
 def copy(oldpath, newpath, overwrite=False):
   """Copies data from oldpath to newpath.
 
@@ -406,15 +468,31 @@ def copy(oldpath, newpath, overwrite=False):
     overwrite: boolean, if false its an error for newpath to be occupied by an
         existing file.
 
+  Raises:
+    errors.OpError: If the operation fails.
+  """
+  copy_v2(oldpath, newpath, overwrite)
+
+
+@tf_export("io.gfile.copy")
+def copy_v2(src, dst, overwrite=False):
+  """Copies data from src to dst.
+
+  Args:
+    src: string, name of the file whose contents need to be copied
+    dst: string, name of the file to which to copy to
+    overwrite: boolean, if false its an error for newpath to be occupied by an
+        existing file.
+
   Raises:
     errors.OpError: If the operation fails.
   """
   with errors.raise_exception_on_not_ok_status() as status:
     pywrap_tensorflow.CopyFile(
-        compat.as_bytes(oldpath), compat.as_bytes(newpath), overwrite, status)
+        compat.as_bytes(src), compat.as_bytes(dst), overwrite, status)
 
 
-@tf_export("gfile.Rename")
+@tf_export(v1=["gfile.Rename"])
 def rename(oldname, newname, overwrite=False):
   """Rename or move a file / directory.
 
@@ -424,12 +502,28 @@ def rename(oldname, newname, overwrite=False):
     overwrite: boolean, if false it's an error for `newname` to be occupied by
         an existing file.
 
+  Raises:
+    errors.OpError: If the operation fails.
+  """
+  rename_v2(oldname, newname, overwrite)
+
+
+@tf_export("io.gfile.rename")
+def rename_v2(src, dst, overwrite):
+  """Rename or move a file / directory.
+
+  Args:
+    src: string, pathname for a file
+    dst: string, pathname to which the file needs to be moved
+    overwrite: boolean, if false it's an error for `dst` to be occupied by
+        an existing file.
+
   Raises:
     errors.OpError: If the operation fails.
   """
   with errors.raise_exception_on_not_ok_status() as status:
     pywrap_tensorflow.RenameFile(
-        compat.as_bytes(oldname), compat.as_bytes(newname), overwrite, status)
+        compat.as_bytes(src), compat.as_bytes(dst), overwrite, status)
 
 
 def atomic_write_string_to_file(filename, contents, overwrite=True):
@@ -456,35 +550,61 @@ def atomic_write_string_to_file(filename, contents, overwrite=True):
     raise
 
 
-@tf_export("gfile.DeleteRecursively")
+@tf_export(v1=["gfile.DeleteRecursively"])
 def delete_recursively(dirname):
   """Deletes everything under dirname recursively.
 
   Args:
     dirname: string, a path to a directory
 
+  Raises:
+    errors.OpError: If the operation fails.
+  """
+  delete_recursively_v2(dirname)
+
+
+@tf_export("io.gfile.rmtree")
+def delete_recursively_v2(path):
+  """Deletes everything under path recursively.
+
+  Args:
+    path: string, a path
+
   Raises:
     errors.OpError: If the operation fails.
   """
   with errors.raise_exception_on_not_ok_status() as status:
-    pywrap_tensorflow.DeleteRecursively(compat.as_bytes(dirname), status)
+    pywrap_tensorflow.DeleteRecursively(compat.as_bytes(path), status)
 
 
-@tf_export("gfile.IsDirectory")
+@tf_export(v1=["gfile.IsDirectory"])
 def is_directory(dirname):
   """Returns whether the path is a directory or not.
 
   Args:
     dirname: string, path to a potential directory
 
+  Returns:
+    True, if the path is a directory; False otherwise
+  """
+  return is_directory_v2(dirname)
+
+
+@tf_export("io.gfile.isdir")
+def is_directory_v2(path):
+  """Returns whether the path is a directory or not.
+
+  Args:
+    path: string, path to a potential directory
+
   Returns:
     True, if the path is a directory; False otherwise
   """
   status = c_api_util.ScopedTFStatus()
-  return pywrap_tensorflow.IsDirectory(compat.as_bytes(dirname), status)
+  return pywrap_tensorflow.IsDirectory(compat.as_bytes(path), status)
 
 
-@tf_export("gfile.ListDirectory")
+@tf_export(v1=["gfile.ListDirectory"])
 def list_directory(dirname):
   """Returns a list of entries contained within a directory.
 
@@ -500,7 +620,26 @@ def list_directory(dirname):
   Raises:
     errors.NotFoundError if directory doesn't exist
   """
-  if not is_directory(dirname):
+  return list_directory_v2(dirname)
+
+
+@tf_export("io.gfile.listdir")
+def list_directory_v2(path):
+  """Returns a list of entries contained within a directory.
+
+  The list is in arbitrary order. It does not contain the special entries "."
+  and "..".
+
+  Args:
+    path: string, path to a directory
+
+  Returns:
+    [filename1, filename2, ... filenameN] as strings
+
+  Raises:
+    errors.NotFoundError if directory doesn't exist
+  """
+  if not is_directory(path):
     raise errors.NotFoundError(None, None, "Could not find directory")
   with errors.raise_exception_on_not_ok_status() as status:
     # Convert each element to string, since the return values of the
@@ -508,11 +647,11 @@ def list_directory(dirname):
     return [
         compat.as_str_any(filename)
         for filename in pywrap_tensorflow.GetChildren(
-            compat.as_bytes(dirname), status)
+            compat.as_bytes(path), status)
     ]
 
 
-@tf_export("gfile.Walk")
+@tf_export(v1=["gfile.Walk"])
 def walk(top, in_order=True):
   """Recursive directory tree generator for directories.
 
@@ -522,6 +661,27 @@ def walk(top, in_order=True):
 
   Errors that happen while listing directories are ignored.
 
+  Yields:
+    Each yield is a 3-tuple:  the pathname of a directory, followed by lists of
+    all its subdirectories and leaf files.
+    (dirname, [subdirname, subdirname, ...], [filename, filename, ...])
+    as strings
+  """
+  return walk_v2(top, in_order)
+
+
+@tf_export("io.gfile.walk")
+def walk_v2(top, topdown, onerror=None):
+  """Recursive directory tree generator for directories.
+
+  Args:
+    top: string, a Directory name
+    topdown: bool, Traverse pre order if True, post order if False.
+    onerror: optional handler for errors. Should be a function, it will be
+      called with the error as argument. Rethrowing the error aborts the walk.
+
+  Errors that happen while listing directories are ignored.
+
   Yields:
     Each yield is a 3-tuple:  the pathname of a directory, followed by lists of
     all its subdirectories and leaf files.
@@ -531,8 +691,11 @@ def walk(top, in_order=True):
   top = compat.as_str_any(top)
   try:
     listing = list_directory(top)
-  except errors.NotFoundError:
-    return
+  except errors.NotFoundError as err:
+    if onerror:
+      onerror(err)
+    else:
+      return
 
   files = []
   subdirs = []
@@ -545,18 +708,18 @@ def walk(top, in_order=True):
 
   here = (top, subdirs, files)
 
-  if in_order:
+  if topdown:
     yield here
 
   for subdir in subdirs:
-    for subitem in walk(os.path.join(top, subdir), in_order):
+    for subitem in walk_v2(os.path.join(top, subdir), topdown, onerror=onerror):
       yield subitem
 
-  if not in_order:
+  if not topdown:
     yield here
 
 
-@tf_export("gfile.Stat")
+@tf_export(v1=["gfile.Stat"])
 def stat(filename):
   """Returns file statistics for a given path.
 
@@ -566,12 +729,28 @@ def stat(filename):
   Returns:
     FileStatistics struct that contains information about the path
 
+  Raises:
+    errors.OpError: If the operation fails.
+  """
+  return stat_v2(filename)
+
+
+@tf_export("io.gfile.stat")
+def stat_v2(path):
+  """Returns file statistics for a given path.
+
+  Args:
+    path: string, path to a file
+
+  Returns:
+    FileStatistics struct that contains information about the path
+
   Raises:
     errors.OpError: If the operation fails.
   """
   file_statistics = pywrap_tensorflow.FileStatistics()
   with errors.raise_exception_on_not_ok_status() as status:
-    pywrap_tensorflow.Stat(compat.as_bytes(filename), file_statistics, status)
+    pywrap_tensorflow.Stat(compat.as_bytes(path), file_statistics, status)
     return file_statistics
 
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index b41a1bc8f6..3517c11cc9 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -14,7 +14,6 @@ TENSORFLOW_API_INIT_FILES = [
     "errors/__init__.py",
     "experimental/__init__.py",
     "feature_column/__init__.py",
-    "gfile/__init__.py",
     "io/gfile/__init__.py",
     "graph_util/__init__.py",
     "image/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 0fadec00ab..e35b9c4374 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -17,6 +17,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     "experimental/__init__.py",
     "feature_column/__init__.py",
     "gfile/__init__.py",
+    "io/gfile/__init__.py",
     "graph_util/__init__.py",
     "image/__init__.py",
     "io/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
new file mode 100644
index 0000000000..e5aba7eff9
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
@@ -0,0 +1,51 @@
+path: "tensorflow.io.gfile"
+tf_module {
+  member_method {
+    name: "copy"
+    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+  member_method {
+    name: "exists"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "glob"
+    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "isdir"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "listdir"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "makedirs"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mkdir"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "remove"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "rename"
+    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "rmtree"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "stat"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "walk"
+    argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt
index fee12594ee..b760ec3890 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.io.pbtxt
@@ -44,6 +44,10 @@ tf_module {
     name: "VarLenFeature"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "gfile"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "decode_and_crop_jpeg"
     argspec: "args=[\'contents\', \'crop_window\', \'channels\', \'ratio\', \'fancy_upscaling\', \'try_recover_truncated\', \'acceptable_fraction\', \'dct_method\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \'True\', \'False\', \'1\', \'\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
deleted file mode 100644
index 74d0a0579e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.gfile.pbtxt
+++ /dev/null
@@ -1,47 +0,0 @@
-path: "tensorflow.gfile"
-tf_module {
-  member_method {
-    name: "Copy"
-    argspec: "args=[\'oldpath\', \'newpath\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
-  }
-  member_method {
-    name: "DeleteRecursively"
-    argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Glob"
-    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "IsDirectory"
-    argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "ListDirectory"
-    argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MakeDirs"
-    argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "MkDir"
-    argspec: "args=[\'dirname\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Remove"
-    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Rename"
-    argspec: "args=[\'oldname\', \'newname\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
-  }
-  member_method {
-    name: "Stat"
-    argspec: "args=[\'filename\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Walk"
-    argspec: "args=[\'top\', \'in_order\'], varargs=None, keywords=None, defaults=[\'True\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
index 59652cb063..e5aba7eff9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
@@ -1,7 +1,51 @@
 path: "tensorflow.io.gfile"
 tf_module {
+  member_method {
+    name: "copy"
+    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
   member_method {
     name: "exists"
     argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "glob"
+    argspec: "args=[\'pattern\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "isdir"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "listdir"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "makedirs"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "mkdir"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "remove"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "rename"
+    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "rmtree"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "stat"
+    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "walk"
+    argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index b4b4827584..99751504ae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -220,10 +220,6 @@ tf_module {
     name: "float64"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
-  member {
-    name: "gfile"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "glorot_uniform_initializer"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 474686d612..dc47796a7c 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -155,7 +155,7 @@ renames = {
     'tf.erf': 'tf.math.erf',
     'tf.erfc': 'tf.math.erfc',
     'tf.expm1': 'tf.math.expm1',
-    'tf.extract_image_patches': 'tf.image.extract_image_patches',
+    'tf.extract_image_patches': 'tf.compat.v1.extract_image_patches',
     'tf.fake_quant_with_min_max_args': 'tf.quantization.fake_quant_with_min_max_args',
     'tf.fake_quant_with_min_max_args_gradient': 'tf.quantization.fake_quant_with_min_max_args_gradient',
     'tf.fake_quant_with_min_max_vars': 'tf.quantization.fake_quant_with_min_max_vars',
@@ -179,10 +179,21 @@ renames = {
     'tf.get_session_tensor': 'tf.compat.v1.get_session_tensor',
     'tf.get_variable': 'tf.compat.v1.get_variable',
     'tf.get_variable_scope': 'tf.compat.v1.get_variable_scope',
+    'tf.gfile.Copy': 'tf.compat.v1.gfile.Copy',
+    'tf.gfile.DeleteRecursively': 'tf.compat.v1.gfile.DeleteRecursively',
     'tf.gfile.Exists': 'tf.compat.v1.gfile.Exists',
     'tf.gfile.FastGFile': 'tf.compat.v1.gfile.FastGFile',
     'tf.gfile.GFile': 'tf.compat.v1.gfile.GFile',
+    'tf.gfile.Glob': 'tf.compat.v1.gfile.Glob',
+    'tf.gfile.IsDirectory': 'tf.compat.v1.gfile.IsDirectory',
+    'tf.gfile.ListDirectory': 'tf.compat.v1.gfile.ListDirectory',
+    'tf.gfile.MakeDirs': 'tf.compat.v1.gfile.MakeDirs',
+    'tf.gfile.MkDir': 'tf.compat.v1.gfile.MkDir',
     'tf.gfile.Open': 'tf.compat.v1.gfile.Open',
+    'tf.gfile.Remove': 'tf.compat.v1.gfile.Remove',
+    'tf.gfile.Rename': 'tf.compat.v1.gfile.Rename',
+    'tf.gfile.Stat': 'tf.compat.v1.gfile.Stat',
+    'tf.gfile.Walk': 'tf.compat.v1.gfile.Walk',
     'tf.global_norm': 'tf.linalg.global_norm',
     'tf.global_variables': 'tf.compat.v1.global_variables',
     'tf.global_variables_initializer': 'tf.compat.v1.global_variables_initializer',
@@ -308,12 +319,16 @@ renames = {
     'tf.matrix_solve_ls': 'tf.linalg.lstsq',
     'tf.matrix_transpose': 'tf.linalg.transpose',
     'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
+    'tf.metrics.accuracy': 'tf.compat.v1.metrics.accuracy',
+    'tf.metrics.mean': 'tf.compat.v1.metrics.mean',
     'tf.min_max_variable_partitioner': 'tf.compat.v1.min_max_variable_partitioner',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
     'tf.multinomial': 'tf.compat.v1.multinomial',
+    'tf.nn.bidirectional_dynamic_rnn': 'tf.compat.v1.nn.bidirectional_dynamic_rnn',
     'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
+    'tf.nn.ctc_loss_v2': 'tf.nn.ctc_loss',
     'tf.nn.depthwise_conv2d_native': 'tf.compat.v1.nn.depthwise_conv2d_native',
     'tf.nn.depthwise_conv2d_native_backprop_filter': 'tf.nn.depthwise_conv2d_backprop_filter',
     'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input',
@@ -328,7 +343,9 @@ renames = {
     'tf.nn.rnn_cell.BasicRNNCell': 'tf.compat.v1.nn.rnn_cell.BasicRNNCell',
     'tf.nn.rnn_cell.GRUCell': 'tf.compat.v1.nn.rnn_cell.GRUCell',
     'tf.nn.rnn_cell.LSTMCell': 'tf.compat.v1.nn.rnn_cell.LSTMCell',
+    'tf.nn.rnn_cell.MultiRNNCell': 'tf.compat.v1.nn.rnn_cell.MultiRNNCell',
     'tf.nn.softmax_cross_entropy_with_logits_v2': 'tf.nn.softmax_cross_entropy_with_logits',
+    'tf.nn.static_bidirectional_rnn': 'tf.compat.v1.nn.static_bidirectional_rnn',
     'tf.nn.static_rnn': 'tf.compat.v1.nn.static_rnn',
     'tf.nn.uniform_candidate_sampler': 'tf.random.uniform_candidate_sampler',
     'tf.nn.xw_plus_b': 'tf.compat.v1.nn.xw_plus_b',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 05e839aa9d..b8cf21b29c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -111,6 +111,41 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.gfile.Exists": {
             "filename": "path",
         },
+        "tf.gfile.Remove": {
+            "filename": "path",
+        },
+        "tf.gfile.Stat": {
+            "filename": "path",
+        },
+        "tf.gfile.Glob": {
+            "filename": "pattern",
+        },
+        "tf.gfile.MkDir": {
+            "dirname": "path",
+        },
+        "tf.gfile.MakeDirs": {
+            "dirname": "path",
+        },
+        "tf.gfile.DeleteRecursively": {
+            "dirname": "path",
+        },
+        "tf.gfile.IsDirectory": {
+            "dirname": "path",
+        },
+        "tf.gfile.ListDirectory": {
+            "dirname": "path",
+        },
+        "tf.gfile.Copy": {
+            "oldpath": "src",
+            "newpath": "dst",
+        },
+        "tf.gfile.Rename": {
+            "oldpath": "src",
+            "newpath": "dst",
+        },
+        "tf.gfile.Walk": {
+            "in_order": "topdown",
+        },
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
@@ -212,6 +247,18 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # These renames happen after the arguments have been processed.
     self.symbol_renames.update({
         "tf.batch_to_space_nd": "tf.batch_to_space",
+        "tf.gfile.Copy": "tf.io.gfile.Copy",
+        "tf.gfile.DeleteRecursively": "tf.io.gfile.DeleteRecursively",
+        "tf.gfile.Exists": "tf.io.gfile.Exists",
+        "tf.gfile.Glob": "tf.io.gfile.Glob",
+        "tf.gfile.IsDirectory": "tf.io.gfile.IsDirectory",
+        "tf.gfile.ListDirectory": "tf.io.gfile.ListDirectory",
+        "tf.gfile.MakeDirs": "tf.io.gfile.MakeDirs",
+        "tf.gfile.MkDir": "tf.io.gfile.MkDir",
+        "tf.gfile.Remove": "tf.io.gfile.Remove",
+        "tf.gfile.Rename": "tf.io.gfile.Rename",
+        "tf.gfile.Stat": "tf.io.gfile.Stat",
+        "tf.gfile.Walk": "tf.io.gfile.Walk",
         "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
         "tf.contrib.data.Counter": "tf.data.experimental.Counter",
         "tf.contrib.data.CheckpointInputPipelineHook": "tf.data.experimental.CheckpointInputPipelineHook",
-- 
GitLab


From afda578147523be4a302f5a785c1e8e32d6650f4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 16:46:58 -0800
Subject: [PATCH 0717/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 222476857

---
 tensorflow/go/op/wrappers.go | 142 +++++++++++++++++------------------
 1 file changed, 71 insertions(+), 71 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 9b59c03e55..ab7294e5a1 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5661,6 +5661,77 @@ func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size
 	return op.Output(0)
 }
 
+// MapUnstageAttr is an optional argument to MapUnstage.
+type MapUnstageAttr func(optionalAttr)
+
+// MapUnstageCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageCapacity(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageContainer(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageSharedName(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns the values associated with the key
+//
+// from the underlying container.   If the underlying container
+// does not contain this key, the op will block until it does.
+func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapUnstage",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstage", err)
+		return
+	}
+	return values
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -33926,74 +33997,3 @@ func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output
 	}
 	return scope.AddOperation(opspec)
 }
-
-// MapUnstageAttr is an optional argument to MapUnstage.
-type MapUnstageAttr func(optionalAttr)
-
-// MapUnstageCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageCapacity(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageContainer(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageSharedName(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the values associated with the key
-//
-// from the underlying container.   If the underlying container
-// does not contain this key, the op will block until it does.
-func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapUnstage",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstage", err)
-		return
-	}
-	return values
-}
-- 
GitLab


From 1a0e8eb4d6524d9381f495d13521b98fed8c5ca7 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 21 Nov 2018 18:03:17 -0800
Subject: [PATCH 0718/1554] Make {Stateless}Multinomial return a matrix even
 when num_samples=0

PiperOrigin-RevId: 222482965
---
 .../compiler/tests/categorical_op_test.py     | 21 +++++++++++++++++++
 .../compiler/tf2xla/kernels/categorical_op.cc |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py
index f17e84df13..f4918e50dc 100644
--- a/tensorflow/compiler/tests/categorical_op_test.py
+++ b/tensorflow/compiler/tests/categorical_op_test.py
@@ -169,6 +169,27 @@ class CategoricalTest(xla_test.XLATestCase):
           for s1, v1 in values:
             self.assertEqual(s0 == s1, np.all(v0 == v1))
 
+  def testEmpty(self):
+    with self.cached_session() as sess:
+      with self.test_scope():
+        x = random_ops.multinomial(
+            array_ops.zeros([42, 40]), 0, output_dtype=dtypes.int32)
+        y = sess.run(x)
+        self.assertEqual(y.shape, (42, 0))
+
+  def testEmptyStateless(self):
+    with self.cached_session() as sess:
+      with self.test_scope():
+        seed_t = array_ops.placeholder(dtypes.int32, shape=[2])
+        x = stateless_random_ops.stateless_multinomial(
+            array_ops.zeros([42, 40]),
+            0,
+            seed=seed_t,
+            output_dtype=dtypes.int32)
+        y = sess.run(x, {seed_t: [0x12345678, 0xabcdef12]})
+        self.assertEqual(y.shape, (42, 0))
+
+
 
 if __name__ == '__main__':
   googletest.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
index 3e398fff95..7199b9b6fe 100644
--- a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
@@ -62,7 +62,7 @@ class CategoricalOp : public XlaOpKernel {
 
     xla::Shape uniform_shape;
     int class_dimension;
-    if (num_samples > 1) {
+    if (num_samples != 1) {
       std::array<int64, 3> uniform_shape_array = {
           {batch_size, num_samples, num_classes}};
       xla::PrimitiveType uniform_xla_type;
-- 
GitLab


From 0dad8cfde287c4ffcff059f1df24dfd2aa5ae662 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 18:20:17 -0800
Subject: [PATCH 0719/1554] This CL updates the Keras model microbenchmarks in
 the eager benchmarks_test to reflect Keras's updated execution modes.

PiperOrigin-RevId: 222484074
---
 tensorflow/python/eager/benchmarks_test.py | 58 +++++++++-------------
 1 file changed, 24 insertions(+), 34 deletions(-)

diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 886715867c..31a7efca82 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -80,7 +80,6 @@ class SubclassedKerasModel(keras.Model):
 
   def __init__(self, initializer="ones"):
     super(SubclassedKerasModel, self).__init__()
-    self._can_use_graph_functions = True
     self.layer_a = keras.layers.Dense(
         64, kernel_initializer=initializer, bias_initializer="zeros")
     self.layer_b = keras.layers.Dense(
@@ -733,38 +732,38 @@ class MicroBenchmarks(test.Benchmark):
     assert np.equal(func(), make_keras_model()(data)).all()
     self._run(func, 30000)
 
-  def _benchmark_keras_model_fit(self, model):
+  def _benchmark_keras_model_fit(self, model, run_eagerly=False):
     data = random_ops.random_uniform((10, 10), minval=-1, maxval=1)
     labels = random_ops.random_uniform((10, 10), minval=-1, maxval=1)
     dataset = dataset_ops.Dataset.from_tensors((data, labels)).repeat()
     model.compile(
         gradient_descent.GradientDescentOptimizer(learning_rate=0.001),
-        loss="mse")
+        loss="mse", run_eagerly=run_eagerly)
     func = lambda: model.fit(dataset, epochs=1, steps_per_epoch=1000, verbose=0)
     # First call is more expensive (creates variables etc.), discount that.
     model.fit(dataset, epochs=1, steps_per_epoch=1, verbose=0)
 
     self._run(func, 1)
 
-  def _benchmark_keras_model_evaluate(self, model):
+  def _benchmark_keras_model_evaluate(self, model, run_eagerly=False):
     data = random_ops.random_uniform((10, 10), minval=-1, maxval=1)
     labels = random_ops.random_uniform((10, 10), minval=-1, maxval=1)
     dataset = dataset_ops.Dataset.from_tensors((data, labels)).repeat()
     model.compile(
         gradient_descent.GradientDescentOptimizer(learning_rate=0.001),
-        loss="mse")
+        loss="mse", run_eagerly=run_eagerly)
     func = lambda: model.evaluate(dataset, steps=1000, verbose=0)
     # First call is more expensive (creates variables etc.), discount that.
     model.evaluate(dataset, steps=1, verbose=0)
 
     self._run(func, 1)
 
-  def _benchmark_keras_model_predict(self, model):
+  def _benchmark_keras_model_predict(self, model, run_eagerly=False):
     data = random_ops.random_uniform((10, 10), minval=-1, maxval=1)
     dataset = dataset_ops.Dataset.from_tensors(tuple([data])).repeat()
     model.compile(
         gradient_descent.GradientDescentOptimizer(learning_rate=0.001),
-        loss="mse")
+        loss="mse", run_eagerly=run_eagerly)
     func = lambda: model.predict(dataset, steps=1000, verbose=0)
     # First call is more expensive (creates variables etc.), discount that.
     model.predict(dataset, steps=1, verbose=0)
@@ -780,10 +779,9 @@ class MicroBenchmarks(test.Benchmark):
       model = SubclassedKerasModel(initializer="glorot_uniform")
       self._benchmark_keras_model_fit(model)
 
-  def benchmark_keras_model_subclassed_fit_disable_defun(self):
+  def benchmark_keras_model_subclassed_fit_run_model_eagerly(self):
     model = SubclassedKerasModel(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_fit(model)
+    self._benchmark_keras_model_fit(model, run_eagerly=True)
 
   def benchmark_keras_model_functional_fit(self):
     model = make_keras_model(initializer="glorot_uniform")
@@ -794,10 +792,9 @@ class MicroBenchmarks(test.Benchmark):
       model = make_keras_model(initializer="glorot_uniform")
       self._benchmark_keras_model_fit(model)
 
-  def benchmark_keras_model_functional_fit_disable_defun(self):
+  def benchmark_keras_model_functional_fit_run_model_eagerly(self):
     model = make_keras_model(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_fit(model)
+    self._benchmark_keras_model_fit(model, run_eagerly=True)
 
   def benchmark_keras_model_sequential_fit(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
@@ -808,64 +805,57 @@ class MicroBenchmarks(test.Benchmark):
       model = make_sequential_keras_model(initializer="glorot_uniform")
       self._benchmark_keras_model_fit(model)
 
-  def benchmark_keras_model_sequential_fit_disable_defun(self):
+  def benchmark_keras_model_sequential_fit_run_model_eagerly(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_fit(model)
+    self._benchmark_keras_model_fit(model, run_eagerly=True)
 
   def benchmark_keras_model_subclassed_evaluate(self):
     model = SubclassedKerasModel(initializer="glorot_uniform")
     self._benchmark_keras_model_evaluate(model)
 
-  def benchmark_keras_model_subclassed_evaluate_disable_defun(self):
+  def benchmark_keras_model_subclassed_evaluate_run_model_eagerly(self):
     model = SubclassedKerasModel(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_evaluate(model)
+    self._benchmark_keras_model_evaluate(model, run_eagerly=True)
 
   def benchmark_keras_model_functional_evaluate(self):
     model = make_keras_model(initializer="glorot_uniform")
     self._benchmark_keras_model_evaluate(model)
 
-  def benchmark_keras_model_functional_evaluate_disable_defun(self):
+  def benchmark_keras_model_functional_evaluate_run_model_eagerly(self):
     model = make_keras_model(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_evaluate(model)
+    self._benchmark_keras_model_evaluate(model, run_eagerly=True)
 
   def benchmark_keras_model_sequential_evaluate(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
     self._benchmark_keras_model_evaluate(model)
 
-  def benchmark_keras_model_sequential_evaluate_disable_defun(self):
+  def benchmark_keras_model_sequential_evaluate_run_model_eagerly(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_evaluate(model)
+    self._benchmark_keras_model_evaluate(model, run_eagerly=True)
 
   def benchmark_keras_model_subclassed_predict(self):
     model = SubclassedKerasModel(initializer="glorot_uniform")
     self._benchmark_keras_model_predict(model)
 
-  def benchmark_keras_model_subclassed_predict_disable_defun(self):
+  def benchmark_keras_model_subclassed_predict_run_model_eagerly(self):
     model = SubclassedKerasModel(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_predict(model)
+    self._benchmark_keras_model_predict(model, run_eagerly=True)
 
   def benchmark_keras_model_functional_predict(self):
     model = make_keras_model(initializer="glorot_uniform")
     self._benchmark_keras_model_predict(model)
 
-  def benchmark_keras_model_functional_predict_disable_defun(self):
+  def benchmark_keras_model_functional_predict_run_model_eagerly(self):
     model = make_keras_model(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_predict(model)
+    self._benchmark_keras_model_predict(model, run_eagerly=True)
 
   def benchmark_keras_model_sequential_predict(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
     self._benchmark_keras_model_predict(model)
 
-  def benchmark_keras_model_sequential_predict_disable_defun(self):
+  def benchmark_keras_model_sequential_predict_run_model_eagerly(self):
     model = make_sequential_keras_model(initializer="glorot_uniform")
-    model._can_use_graph_functions = False
-    self._benchmark_keras_model_predict(model)
+    self._benchmark_keras_model_predict(model, run_eagerly=True)
 
   def benchmarkScan(self):
     elems = math_ops.range(1600)
-- 
GitLab


From 824bf4c67f01364b23be5c69fe5365ef35dc779a Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Wed, 21 Nov 2018 18:20:43 -0800
Subject: [PATCH 0720/1554] [TF2XLA] Conditionally include
 tf2xla/kernels:resampler_ops as contrib/resampler:resampler_ops_kernels'
 dependency based on whether XLA is enabled.

Also move compiler/tests/resampler_ops_test.py to contrib/resampler/xla/resampler_ops_xla_test.py instead.

PiperOrigin-RevId: 222484109
---
 tensorflow/compiler/tests/BUILD               | 21 ------------
 tensorflow/contrib/resampler/BUILD            | 32 +++++++++++++++++--
 .../resampler/xla/resampler_ops_xla_test.py}  |  0
 3 files changed, 30 insertions(+), 23 deletions(-)
 rename tensorflow/{compiler/tests/resampler_ops_test.py => contrib/resampler/xla/resampler_ops_xla_test.py} (100%)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 2b88a64fed..bc3d60b90e 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -375,27 +375,6 @@ tf_xla_py_test(
     ],
 )
 
-tf_xla_py_test(
-    name = "resampler_ops_test",
-    size = "small",
-    srcs = ["resampler_ops_test.py"],
-    disabled_backends = [
-        # TODO(b/74459949) Support BatchDot in CPU backend.
-        "cpu",
-        "cpu_ondemand",
-    ],
-    # TODO(b/112295522): figure out how to make OSS build pass.
-    tags = ["no_oss"],
-    deps = [
-        ":xla_test",
-        "//tensorflow/contrib/resampler:resampler_ops",
-        "//tensorflow/contrib/resampler:resampler_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
 tf_xla_py_test(
     name = "dynamic_stitch_test",
     size = "small",
diff --git a/tensorflow/contrib/resampler/BUILD b/tensorflow/contrib/resampler/BUILD
index 38fcca0311..bbf1099675 100644
--- a/tensorflow/contrib/resampler/BUILD
+++ b/tensorflow/contrib/resampler/BUILD
@@ -13,6 +13,7 @@ load(
 )
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
+load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
 
 tf_custom_op_py_library(
     name = "resampler_py",
@@ -50,10 +51,14 @@ tf_kernel_library(
     prefix = "resampler_ops",
     deps = [
         ":resampler_ops_op_lib",
-        "//tensorflow/compiler/tf2xla/kernels:resampler_ops",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-    ],
+    ] + select({
+        "//tensorflow:with_xla_support": [
+            "//tensorflow/compiler/tf2xla/kernels:resampler_ops",
+        ],
+        "//conditions:default": [],
+    }),
     alwayslink = 1,
 )
 
@@ -94,3 +99,26 @@ cuda_py_test(
         "//tensorflow/python:array_ops",
     ],
 )
+
+tf_xla_py_test(
+    name = "resampler_ops_xla_test",
+    size = "small",
+    srcs = ["xla/resampler_ops_xla_test.py"],
+    disabled_backends = [
+        # TODO(b/74459949) Support BatchDot in CPU backend.
+        "cpu",
+        "cpu_ondemand",
+    ],
+    # TODO(b/112295522): the OSS build will not likely work in the short to medium term, currently it is blocked by the fact that bazel does not allow py_library to depend on cc_library: https://github.com/bazelbuild/bazel/issues/701 which may not be resolvable.
+    tags = ["no_oss"],
+    deps = [
+        "//tensorflow/compiler/tests:xla_test",
+        "//tensorflow/compiler/tf2xla/kernels:resampler_ops",
+        "//tensorflow/contrib/resampler:resampler_ops",
+        "//tensorflow/contrib/resampler:resampler_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:platform_test",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/compiler/tests/resampler_ops_test.py b/tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py
similarity index 100%
rename from tensorflow/compiler/tests/resampler_ops_test.py
rename to tensorflow/contrib/resampler/xla/resampler_ops_xla_test.py
-- 
GitLab


From 1e4e6328c516ff9d5e861d64bc0d7626c579e670 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 21 Nov 2018 18:31:12 -0800
Subject: [PATCH 0721/1554] [tf.data] Moving the (experimental) MatchingFiles
 dataset implementation to the experimental source subtree.

Since the transformation is not exposed in the API, this CL has no effect on the API.

PiperOrigin-RevId: 222484670
---
 ...def_ExperimentalMatchingFilesDataset.pbtxt |   4 +
 .../api_def_MatchingFilesDataset.pbtxt        |   4 -
 tensorflow/core/kernels/data/BUILD            |  13 ---
 .../core/kernels/data/experimental/BUILD      |  13 +++
 .../matching_files_dataset_op.cc              |   5 +-
 .../core/ops/compat/ops_history.v1.pbtxt      |  12 --
 tensorflow/core/ops/dataset_ops.cc            |  12 --
 .../core/ops/experimental_dataset_ops.cc      |  12 ++
 .../python/data/experimental/benchmarks/BUILD |  17 +++
 .../benchmarks/matching_files_benchmark.py    | 101 +++++++++++++++++
 .../data/experimental/kernel_tests/BUILD      |  19 ++++
 .../kernel_tests/matching_files_test.py}      | 105 ++++--------------
 .../kernel_tests/serialization/BUILD          |   1 +
 ...tching_files_dataset_serialization_test.py |   4 +-
 tensorflow/python/data/experimental/ops/BUILD |  12 ++
 .../data/experimental/ops/matching_files.py   |  51 +++++++++
 tensorflow/python/data/kernel_tests/BUILD     |  44 +++++---
 tensorflow/python/data/ops/dataset_ops.py     |  24 ----
 18 files changed, 283 insertions(+), 170 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalMatchingFilesDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_MatchingFilesDataset.pbtxt
 rename tensorflow/core/kernels/data/{ => experimental}/matching_files_dataset_op.cc (99%)
 create mode 100644 tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
 rename tensorflow/python/data/{kernel_tests/matching_files_dataset_op_test.py => experimental/kernel_tests/matching_files_test.py} (62%)
 create mode 100644 tensorflow/python/data/experimental/ops/matching_files.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalMatchingFilesDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalMatchingFilesDataset.pbtxt
new file mode 100644
index 0000000000..993a798149
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalMatchingFilesDataset.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ExperimentalMatchingFilesDataset"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_MatchingFilesDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatchingFilesDataset.pbtxt
deleted file mode 100644
index ab2a33108d..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_MatchingFilesDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "MatchingFilesDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index b7ccf5f70e..7192684e2d 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -662,18 +662,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "matching_files_dataset_op",
-    srcs = ["matching_files_dataset_op.cc"],
-    deps = [
-        ":dataset",
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "model_dataset_op",
     srcs = ["model_dataset_op.cc"],
@@ -718,7 +706,6 @@ tf_kernel_library(
         ":map_and_batch_dataset_op",
         ":map_dataset_op",
         ":map_defun_op",
-        ":matching_files_dataset_op",
         ":model_dataset_op",
         ":multi_device_iterator_ops",
         ":optimize_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 1a18864ecf..958c42a22a 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -157,6 +157,18 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "matching_files_dataset_op",
+    srcs = ["matching_files_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:dataset",
+    ],
+)
+
 tf_kernel_library(
     name = "dataset_kernels",
     deps = [
@@ -166,6 +178,7 @@ tf_kernel_library(
         ":ignore_errors_dataset_op",
         ":indexed_dataset",
         ":lmdb_dataset_op",
+        ":matching_files_dataset_op",
         ":non_serializable_dataset_op",
         ":numa_map_and_batch_dataset_op",
         ":prefetching_kernels",
diff --git a/tensorflow/core/kernels/data/matching_files_dataset_op.cc b/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
similarity index 99%
rename from tensorflow/core/kernels/data/matching_files_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
index d36b9e7e78..aa27a13416 100644
--- a/tensorflow/core/kernels/data/matching_files_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
@@ -366,8 +366,9 @@ class MatchingFilesDatasetOp : public DatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("MatchingFilesDataset").Device(DEVICE_CPU),
-                        MatchingFilesDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalMatchingFilesDataset").Device(DEVICE_CPU),
+    MatchingFilesDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 309154d74d..efb2f5ed03 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -31544,18 +31544,6 @@ op {
     type: DT_STRING
   }
 }
-op {
-  name: "MatchingFilesDataset"
-  input_arg {
-    name: "patterns"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  is_stateful: true
-}
 op {
   name: "MatrixBandPart"
   input_arg {
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 8402f250f9..e7212b7004 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -622,18 +622,6 @@ REGISTER_OP("TextLineDataset")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("MatchingFilesDataset")
-    .Input("patterns: string")
-    .Output("handle: variant")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
-                      // stateful to inhibit constant folding.
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle unused;
-      // `patterns` must be a scalar or a vector.
-      TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused));
-      return shape_inference::ScalarShape(c);
-    });
-
 REGISTER_OP("SqlDataset")
     .Input("driver_name: string")
     .Input("data_source_name: string")
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index e9987f568c..aebe0bf984 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -86,6 +86,18 @@ REGISTER_OP("ExperimentalMapDataset")
     .Attr("use_inter_op_parallelism: bool = true")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalMatchingFilesDataset")
+    .Input("patterns: string")
+    .Output("handle: variant")
+    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+                      // stateful to inhibit constant folding.
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // `patterns` must be a scalar or a vector.
+      TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
 REGISTER_OP("ExperimentalNonSerializableDataset")
     .Input("input_dataset: variant")
     .Output("handle: variant")
diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index b89fbe7757..c48647a218 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -41,3 +41,20 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
+
+py_test(
+    name = "matching_files_benchmark",
+    size = "small",
+    srcs = ["matching_files_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/experimental/ops:matching_files",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
new file mode 100644
index 0000000000..d0d979dbd4
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
@@ -0,0 +1,101 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmark for the experimental `MatchingFilesDataset`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import shutil
+import tempfile
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import matching_files
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import test
+
+
+class MatchingFilesBenchmark(test.Benchmark):
+  """Benchmark for the experimental `MatchingFilesDataset`."""
+
+  def benchmarkNestedDirectories(self):
+    tmp_dir = tempfile.mkdtemp()
+    width = 500
+    depth = 10
+    for i in range(width):
+      for j in range(depth):
+        new_base = os.path.join(tmp_dir, str(i),
+                                *[str(dir_name) for dir_name in range(j)])
+        os.makedirs(new_base)
+        child_files = ['a.py', 'b.pyc'] if j < depth - 1 else ['c.txt', 'd.log']
+        for f in child_files:
+          filename = os.path.join(new_base, f)
+          open(filename, 'w').close()
+
+    patterns = [
+        os.path.join(tmp_dir, os.path.join(*['**'
+                                             for _ in range(depth)]), suffix)
+        for suffix in ['*.txt', '*.log']
+    ]
+
+    deltas = []
+    iters = 3
+    for _ in range(iters):
+      with ops.Graph().as_default():
+        dataset = matching_files.MatchingFilesDataset(patterns)
+        next_element = dataset.make_one_shot_iterator().get_next()
+
+        with session.Session() as sess:
+          sub_deltas = []
+          while True:
+            try:
+              start = time.time()
+              sess.run(next_element)
+              end = time.time()
+              sub_deltas.append(end - start)
+            except errors.OutOfRangeError:
+              break
+          deltas.append(sub_deltas)
+
+    median_deltas = np.median(deltas, axis=0)
+    print('Nested directory size (width*depth): %d*%d Median wall time: '
+          '%fs (read first filename), %fs (read second filename), avg %fs'
+          ' (read %d more filenames)' %
+          (width, depth, median_deltas[0], median_deltas[1],
+           np.average(median_deltas[2:]), len(median_deltas) - 2))
+    self.report_benchmark(
+        iters=iters,
+        wall_time=np.sum(median_deltas),
+        extras={
+            'read first file:':
+                median_deltas[0],
+            'read second file:':
+                median_deltas[1],
+            'avg time for reading %d more filenames:' %
+            (len(median_deltas) - 2):
+                np.average(median_deltas[2:])
+        },
+        name='benchmark_matching_files_dataset_nesteddirectory(%d*%d)' %
+        (width, depth))
+
+    shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index a7c306817c..ba5c6f7b17 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -371,6 +371,25 @@ py_test(
     ],
 )
 
+py_test(
+    name = "matching_files_test",
+    size = "small",
+    srcs = ["matching_files_test.py"],
+    srcs_version = "PY2AND3",
+    tags = ["no_pip"],
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/experimental/ops:matching_files",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "override_threadpool_test",
     size = "small",
diff --git a/tensorflow/python/data/kernel_tests/matching_files_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
similarity index 62%
rename from tensorflow/python/data/kernel_tests/matching_files_dataset_op_test.py
rename to tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
index 4d86ec4228..938dd4aff4 100644
--- a/tensorflow/python/data/kernel_tests/matching_files_dataset_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for the private `MatchingFilesDataset`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -20,20 +20,15 @@ from __future__ import print_function
 import os
 import shutil
 import tempfile
-import time
 
-import numpy as np
-
-from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import matching_files
 from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops.dataset_ops import MatchingFilesDataset
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
 
-class MatchingFilesDatasetTest(test_base.DatasetTestBase):
+class MatchingFilesTest(test_base.DatasetTestBase):
 
   def setUp(self):
     self.tmp_dir = tempfile.mkdtemp()
@@ -46,31 +41,34 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase):
       open(os.path.join(self.tmp_dir, filename), 'a').close()
 
   def testNonExistingDirectory(self):
-    """Test the MatchingFiles dataset with a non-existing directory"""
+    """Test the MatchingFiles dataset with a non-existing directory."""
 
     self.tmp_dir = os.path.join(self.tmp_dir, 'nonexistingdir')
-    dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*'))
+    dataset = matching_files.MatchingFilesDataset(
+        os.path.join(self.tmp_dir, '*'))
     with self.cached_session() as sess:
       next_element = dataset.make_one_shot_iterator().get_next()
       with self.assertRaises(errors.NotFoundError):
         sess.run(next_element)
 
   def testEmptyDirectory(self):
-    """Test the MatchingFiles dataset with an empty directory"""
+    """Test the MatchingFiles dataset with an empty directory."""
 
-    dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*'))
+    dataset = matching_files.MatchingFilesDataset(
+        os.path.join(self.tmp_dir, '*'))
     with self.cached_session() as sess:
       next_element = dataset.make_one_shot_iterator().get_next()
       with self.assertRaises(errors.NotFoundError):
         sess.run(next_element)
 
   def testSimpleDirectory(self):
-    """Test the MatchingFiles dataset with a simple directory"""
+    """Test the MatchingFiles dataset with a simple directory."""
 
     filenames = ['a', 'b', 'c']
     self._touchTempFiles(filenames)
 
-    dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*'))
+    dataset = matching_files.MatchingFilesDataset(
+        os.path.join(self.tmp_dir, '*'))
     with self.cached_session() as sess:
       next_element = dataset.make_one_shot_iterator().get_next()
 
@@ -86,12 +84,13 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase):
         sess.run(next_element)
 
   def testFileSuffixes(self):
-    """Test the MatchingFiles dataset using the suffixes of filename"""
+    """Test the MatchingFiles dataset using the suffixes of filename."""
 
     filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc']
     self._touchTempFiles(filenames)
 
-    dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, '*.py'))
+    dataset = matching_files.MatchingFilesDataset(
+        os.path.join(self.tmp_dir, '*.py'))
     with self.cached_session() as sess:
       next_element = dataset.make_one_shot_iterator().get_next()
       expected_filenames = []
@@ -106,12 +105,13 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase):
         sess.run(next_element)
 
   def testFileMiddles(self):
-    """Test the MatchingFiles dataset using the middles of filename"""
+    """Test the MatchingFiles dataset using the middles of filename."""
 
     filenames = ['aa.txt', 'bb.py', 'bbc.pyc', 'cc.pyc']
     self._touchTempFiles(filenames)
 
-    dataset = MatchingFilesDataset(os.path.join(self.tmp_dir, 'b*.py*'))
+    dataset = matching_files.MatchingFilesDataset(
+        os.path.join(self.tmp_dir, 'b*.py*'))
     with self.cached_session() as sess:
       next_element = dataset.make_one_shot_iterator().get_next()
       expected_filenames = []
@@ -126,7 +126,7 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase):
         sess.run(next_element)
 
   def testNestedDirectories(self):
-    """Test the MatchingFiles dataset with nested directories"""
+    """Test the MatchingFiles dataset with nested directories."""
 
     filenames = []
     width = 8
@@ -147,7 +147,7 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase):
                      suffix) for suffix in ['*.txt', '*.log']
     ]
 
-    dataset = MatchingFilesDataset(patterns)
+    dataset = matching_files.MatchingFilesDataset(patterns)
     with self.cached_session() as sess:
       next_element = dataset.make_one_shot_iterator().get_next()
       expected_filenames = [
@@ -165,70 +165,5 @@ class MatchingFilesDatasetTest(test_base.DatasetTestBase):
       self.assertItemsEqual(expected_filenames, actual_filenames)
 
 
-class MatchingFilesDatasetBenchmark(test.Benchmark):
-
-  def benchmarkNestedDirectories(self):
-    tmp_dir = tempfile.mkdtemp()
-    width = 500
-    depth = 10
-    for i in range(width):
-      for j in range(depth):
-        new_base = os.path.join(tmp_dir, str(i),
-                                *[str(dir_name) for dir_name in range(j)])
-        os.makedirs(new_base)
-        child_files = ['a.py', 'b.pyc'] if j < depth - 1 else ['c.txt', 'd.log']
-        for f in child_files:
-          filename = os.path.join(new_base, f)
-          open(filename, 'w').close()
-
-    patterns = [
-        os.path.join(tmp_dir, os.path.join(*['**'
-                                             for _ in range(depth)]), suffix)
-        for suffix in ['*.txt', '*.log']
-    ]
-
-    deltas = []
-    iters = 3
-    for _ in range(iters):
-      with ops.Graph().as_default():
-        dataset = MatchingFilesDataset(patterns)
-        next_element = dataset.make_one_shot_iterator().get_next()
-
-        with session.Session() as sess:
-          sub_deltas = []
-          while True:
-            try:
-              start = time.time()
-              sess.run(next_element)
-              end = time.time()
-              sub_deltas.append(end - start)
-            except errors.OutOfRangeError:
-              break
-          deltas.append(sub_deltas)
-
-    median_deltas = np.median(deltas, axis=0)
-    print('Nested directory size (width*depth): %d*%d Median wall time: '
-          '%fs (read first filename), %fs (read second filename), avg %fs'
-          ' (read %d more filenames)' %
-          (width, depth, median_deltas[0], median_deltas[1],
-           np.average(median_deltas[2:]), len(median_deltas) - 2))
-    self.report_benchmark(
-        iters=iters,
-        wall_time=np.sum(median_deltas),
-        extras={
-            'read first file:':
-                median_deltas[0],
-            'read second file:':
-                median_deltas[1],
-            'avg time for reading %d more filenames:' %
-            (len(median_deltas) - 2):
-                np.average(median_deltas[2:])
-        },
-        name='benchmark_matching_files_dataset_nesteddirectory(%d*%d)' %
-        (width, depth))
-
-    shutil.rmtree(tmp_dir, ignore_errors=True)
-
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 079c5309a8..c724987b24 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -361,6 +361,7 @@ py_test(
     deps = [
         ":dataset_serialization_test_base",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:matching_files",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py
index 7edb200d2e..c026e97835 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/matching_files_dataset_serialization_test.py
@@ -22,7 +22,7 @@ import shutil
 import tempfile
 
 from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
-from tensorflow.python.data.ops.dataset_ops import MatchingFilesDataset
+from tensorflow.python.data.experimental.ops import matching_files
 from tensorflow.python.platform import test
 
 
@@ -30,7 +30,7 @@ class MatchingFilesDatasetSerializationTest(
     dataset_serialization_test_base.DatasetSerializationTestBase):
 
   def _build_iterator_graph(self, test_patterns):
-    return MatchingFilesDataset(test_patterns)
+    return matching_files.MatchingFilesDataset(test_patterns)
 
   def testMatchingFilesCore(self):
     tmp_dir = tempfile.mkdtemp()
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 33c493bf45..f9544857a1 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -199,6 +199,17 @@ py_library(
     ],
 )
 
+py_library(
+    name = "matching_files",
+    srcs = ["matching_files.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:tensor_shape",
+    ],
+)
+
 py_library(
     name = "optimization",
     srcs = ["optimization.py"],
@@ -397,6 +408,7 @@ py_library(
         ":indexed_dataset_ops",
         ":interleave_ops",
         ":map_defun",
+        ":matching_files",
         ":optimization",
         ":prefetching_ops",
         ":readers",
diff --git a/tensorflow/python/data/experimental/ops/matching_files.py b/tensorflow/python/data/experimental/ops/matching_files.py
new file mode 100644
index 0000000000..8398f86e31
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/matching_files.py
@@ -0,0 +1,51 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Experimental API for matching input filenames."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
+
+
+class MatchingFilesDataset(dataset_ops.DatasetSource):
+  """A `Dataset` that list the files according to the input patterns."""
+
+  def __init__(self, patterns):
+    super(MatchingFilesDataset, self).__init__()
+    self._patterns = ops.convert_to_tensor(
+        patterns, dtype=dtypes.string, name="patterns")
+
+  def _as_variant_tensor(self):
+    return ged_ops.experimental_matching_files_dataset(self._patterns)
+
+  @property
+  def output_classes(self):
+    return ops.Tensor
+
+  @property
+  def output_shapes(self):
+    return tensor_shape.scalar()
+
+  @property
+  def output_types(self):
+    return dtypes.string
+
+
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index fa1f6d701a..dc89474f49 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -23,6 +23,8 @@ tf_py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:util",
@@ -42,6 +44,7 @@ tf_py_test(
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python:variables",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:iterator_ops",
@@ -100,8 +103,11 @@ tf_py_test(
         ":test_base",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:session",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:sparse",
@@ -116,8 +122,12 @@ tf_py_test(
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python/data/util:nest",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
@@ -137,8 +147,10 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python:functional_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
@@ -185,7 +197,10 @@ tf_py_test(
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python/data/util:nest",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
@@ -199,10 +214,12 @@ tf_py_test(
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
+        "//tensorflow/python:script_ops",
         "//tensorflow/python:session",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:sparse_tensor",
@@ -266,6 +283,7 @@ tf_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:function",
         "//tensorflow/python:functional_ops",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:session",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:iterator_ops",
@@ -288,6 +306,7 @@ tf_py_test(
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
@@ -301,30 +320,16 @@ tf_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:script_ops",
+        "//tensorflow/python:session",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
 
-tf_py_test(
-    name = "matching_files_dataset_op_test",
-    size = "small",
-    srcs = ["matching_files_dataset_op_test.py"],
-    additional_deps = [
-        ":test_base",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:dataset_ops",
-    ],
-)
-
 cuda_py_test(
     name = "multi_device_iterator_test",
     size = "medium",
@@ -396,6 +401,7 @@ tf_py_test(
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:io_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:parsing_ops",
@@ -413,6 +419,8 @@ tf_py_test(
     srcs = ["reader_dataset_ops_test.py"],
     additional_deps = [
         ":test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
@@ -499,10 +507,14 @@ py_library(
     name = "test_base",
     srcs = ["test_base.py"],
     deps = [
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
+        "//tensorflow/python/eager:context",
     ],
 )
 
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index bd57c9f1d5..843db18853 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -2769,30 +2769,6 @@ class MapDataset(UnaryDataset):
     return "Dataset.map()"
 
 
-class MatchingFilesDataset(DatasetSource):
-  """A `Dataset` that list the files according to the input patterns."""
-
-  def __init__(self, patterns):
-    super(MatchingFilesDataset, self).__init__()
-    self._patterns = ops.convert_to_tensor(
-        patterns, dtype=dtypes.string, name="patterns")
-
-  def _as_variant_tensor(self):
-    return gen_dataset_ops.matching_files_dataset(self._patterns)
-
-  @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.string
-
-
 class ParallelMapDataset(MapDataset):
   """A `Dataset` that maps a function over elements in its input in parallel."""
 
-- 
GitLab


From 18fe495e72bb711322ca7a361c8b0ffcc901100a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 18:46:38 -0800
Subject: [PATCH 0722/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 222485786

---
 tensorflow/go/op/wrappers.go | 48 ++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index ab7294e5a1..ad31c335e6 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -3487,30 +3487,6 @@ func BoostedTreesQuantileStreamResourceFlush(scope *Scope, quantile_stream_resou
 	return scope.AddOperation(opspec)
 }
 
-// Add the quantile summaries to each quantile stream resource.
-//
-// An op that adds a list of quantile summaries to a quantile stream resource. Each
-// summary Tensor is rank 2, containing summaries (value, weight, min_rank, max_rank)
-// for a single feature.
-//
-// Arguments:
-//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
-//	summaries: string; List of Rank 2 Tensor each containing the summaries for a single feature.
-//
-// Returns the created operation.
-func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_stream_resource_handle tf.Output, summaries []tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BoostedTreesQuantileStreamResourceAddSummaries",
-		Input: []tf.Input{
-			quantile_stream_resource_handle, tf.OutputList(summaries),
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Makes the summary of quantiles for the batch.
 //
 // An op that takes a list of tensors and outputs the quantile summaries for each tensor.
@@ -30165,6 +30141,30 @@ func ExperimentalDirectedInterleaveDataset(scope *Scope, selector_input_dataset
 	return op.Output(0)
 }
 
+// Add the quantile summaries to each quantile stream resource.
+//
+// An op that adds a list of quantile summaries to a quantile stream resource. Each
+// summary Tensor is rank 2, containing summaries (value, weight, min_rank, max_rank)
+// for a single feature.
+//
+// Arguments:
+//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
+//	summaries: string; List of Rank 2 Tensor each containing the summaries for a single feature.
+//
+// Returns the created operation.
+func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_stream_resource_handle tf.Output, summaries []tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BoostedTreesQuantileStreamResourceAddSummaries",
+		Input: []tf.Input{
+			quantile_stream_resource_handle, tf.OutputList(summaries),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Gets the next element from a FunctionBufferingResource.
 //
 // Arguments:
-- 
GitLab


From e679321d7829b6b7c06bc7b9ced4017ce492c2af Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Wed, 21 Nov 2018 18:53:31 -0800
Subject: [PATCH 0723/1554] Add back strategy.configure in
 non-distribute-coordinator path of Keras' configure_and_create_session.

PiperOrigin-RevId: 222486266
---
 tensorflow/python/keras/engine/distributed_training_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 881f8b36ca..fba557ff5b 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -350,6 +350,7 @@ def configure_and_create_session(distribution_strategy):
       session = session_module.Session(
           config=dc_session_config, target=worker_context.master_target)
     else:
+      distribution_strategy.configure(session_config)
       session = session_module.Session(config=session_config)
 
   K.set_session(session)
-- 
GitLab


From abcc0bbc0983c1699d357497441d34e249ef1d6c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 19:42:04 -0800
Subject: [PATCH 0724/1554] Automated rollback of commit
 209c7dae9368c99598d599cd7aea032417fc7cc2. Revert #22478.

PiperOrigin-RevId: 222489301
---
 tensorflow/python/keras/integration_test.py | 10 ----------
 tensorflow/python/keras/regularizers.py     | 15 ++++++---------
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index 25ca9e69e2..3c0f73b1c3 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.layers import core as tf_core_layers
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import rnn_cell
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
@@ -313,15 +312,6 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=0)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
-  def test_regularizers_with_get_variable(self):
-    # Test case for GitHub issue 22470.
-    with self.cached_session():
-      v = variable_scope.get_variable(
-          'v',
-          shape=[4, 4],
-          initializer=keras.initializers.glorot_uniform(),
-          regularizer=keras.regularizers.l2(0.))
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/regularizers.py b/tensorflow/python/keras/regularizers.py
index cbcdae214f..28b6ad4c65 100644
--- a/tensorflow/python/keras/regularizers.py
+++ b/tensorflow/python/keras/regularizers.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import six
 
-from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
@@ -55,14 +54,12 @@ class L1L2(Regularizer):
     self.l2 = K.cast_to_floatx(l2)
 
   def __call__(self, x):
-    if self.l1 or self.l2:
-      regularization = ops.convert_to_tensor(0., dtype=K.floatx())
-      if self.l1:
-        regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
-      if self.l2:
-        regularization += math_ops.reduce_sum(self.l2 * math_ops.square(x))
-      return regularization
-    return None
+    regularization = 0.
+    if self.l1:
+      regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
+    if self.l2:
+      regularization += math_ops.reduce_sum(self.l2 * math_ops.square(x))
+    return regularization
 
   def get_config(self):
     return {'l1': float(self.l1), 'l2': float(self.l2)}
-- 
GitLab


From e298f543b0a93d2c97323dd47d1816d688029065 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 21 Nov 2018 20:04:45 -0800
Subject: [PATCH 0725/1554] [TF] Better error message when single threaded
 executor fails.

PiperOrigin-RevId: 222490940
---
 .../core/kernels/data/single_threaded_executor.cc | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/data/single_threaded_executor.cc b/tensorflow/core/kernels/data/single_threaded_executor.cc
index 5b084a16f0..89e3881037 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor.cc
@@ -65,21 +65,28 @@ class SingleThreadedExecutorImpl : public Executor {
         if (IsRefType(dt)) {
           return errors::Unimplemented(
               "Single-threaded executor does not support reference-typed "
-              "edges.");
+              "edges.  But saw type ",
+              DataTypeString(dt), " in outputs of node ", n->name());
         }
       }
 
       if (n->IsControlFlow()) {
         return errors::Unimplemented(
-            "Single-threaded executor does not support control flow.");
+            "Single-threaded executor does not support control flow.  But saw "
+            "control flow node ",
+            n->name());
       }
       if (n->IsSend() || n->IsHostSend() || n->IsRecv() || n->IsHostRecv()) {
         return errors::Unimplemented(
-            "Single-threaded executor does not support partitioned graphs.");
+            "Single-threaded executor does not support partitioned graphs.  "
+            "But saw send/recv node ",
+            n->name());
       }
       if (n->IsCollective()) {
         return errors::Unimplemented(
-            "Single-threaded executor does not support collective ops.");
+            "Single-threaded executor does not support collective ops.  But "
+            "saw collective node ",
+            n->name());
       }
 
       KernelState& kernel_state = kernels_[i];
-- 
GitLab


From 4887e11e758a3887421a05dafab8b3ab4051d9c9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 20:18:58 -0800
Subject: [PATCH 0726/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 222491827
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 12 ++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 24 +++++++++----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index efb2f5ed03..c855f1c4a0 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21941,6 +21941,18 @@ op {
     }
   }
 }
+op {
+  name: "ExperimentalMatchingFilesDataset"
+  input_arg {
+    name: "patterns"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
 op {
   name: "ExperimentalMaterializedIndexDatasetHandle"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index f690635558..44cc64dc3b 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10404,6 +10404,18 @@ op {
     }
   }
 }
+op {
+  name: "ExperimentalMatchingFilesDataset"
+  input_arg {
+    name: "patterns"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
 op {
   name: "ExperimentalMaterializedIndexDatasetHandle"
   output_arg {
@@ -15991,18 +16003,6 @@ op {
     type: DT_STRING
   }
 }
-op {
-  name: "MatchingFilesDataset"
-  input_arg {
-    name: "patterns"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  is_stateful: true
-}
 op {
   name: "MatrixBandPart"
   input_arg {
-- 
GitLab


From 84b50a2c0096f570a18b0c7cd00ac45c18ffe57e Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 21 Nov 2018 21:40:29 -0800
Subject: [PATCH 0727/1554] Change API for io.serialize_sparse and
 io.serialize_many_sparse for TF 2.0.

PiperOrigin-RevId: 222496915
---
 tensorflow/python/ops/sparse_ops.py           | 54 +++++++++++++++++--
 .../tools/api/golden/v2/tensorflow.io.pbtxt   |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  2 +
 3 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 58cd8291e1..e8a0f31d9d 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -1849,8 +1849,7 @@ def sparse_fill_empty_rows(sp_input, default_value, name=None):
         dense_shape=sp_input.dense_shape), empty_row_indicator)
 
 
-@tf_export(
-    "io.serialize_sparse", v1=["io.serialize_sparse", "serialize_sparse"])
+@tf_export(v1=["io.serialize_sparse", "serialize_sparse"])
 @deprecation.deprecated_endpoints("serialize_sparse")
 def serialize_sparse(sp_input, name=None, out_type=dtypes.string):
   """Serialize a `SparseTensor` into a 3-vector (1-D `Tensor`) object.
@@ -1864,6 +1863,25 @@ def serialize_sparse(sp_input, name=None, out_type=dtypes.string):
     A 3-vector (1-D `Tensor`), with each column representing the serialized
     `SparseTensor`'s indices, values, and shape (respectively).
 
+  Raises:
+    TypeError: If `sp_input` is not a `SparseTensor`.
+  """
+  return serialize_sparse_v2(sp_input, out_type, name)
+
+
+@tf_export("io.serialize_sparse", v1=[])
+def serialize_sparse_v2(sp_input, out_type=dtypes.string, name=None):
+  """Serialize a `SparseTensor` into a 3-vector (1-D `Tensor`) object.
+
+  Args:
+    sp_input: The input `SparseTensor`.
+    out_type: The `dtype` to use for serialization.
+    name: A name prefix for the returned tensors (optional).
+
+  Returns:
+    A 3-vector (1-D `Tensor`), with each column representing the serialized
+    `SparseTensor`'s indices, values, and shape (respectively).
+
   Raises:
     TypeError: If `sp_input` is not a `SparseTensor`.
   """
@@ -1877,9 +1895,7 @@ def serialize_sparse(sp_input, name=None, out_type=dtypes.string):
       out_type=out_type)
 
 
-@tf_export(
-    "io.serialize_many_sparse",
-    v1=["io.serialize_many_sparse", "serialize_many_sparse"])
+@tf_export(v1=["io.serialize_many_sparse", "serialize_many_sparse"])
 @deprecation.deprecated_endpoints("serialize_many_sparse")
 def serialize_many_sparse(sp_input, name=None, out_type=dtypes.string):
   """Serialize `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor`.
@@ -1902,6 +1918,34 @@ def serialize_many_sparse(sp_input, name=None, out_type=dtypes.string):
     represents serialized `SparseTensor`'s indices, values, and shape
     (respectively).
 
+  Raises:
+    TypeError: If `sp_input` is not a `SparseTensor`.
+  """
+  return serialize_many_sparse_v2(sp_input, out_type, name)
+
+
+@tf_export("io.serialize_many_sparse", v1=[])
+def serialize_many_sparse_v2(sp_input, out_type=dtypes.string, name=None):
+  """Serialize `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor`.
+
+  The `SparseTensor` must have rank `R` greater than 1, and the first dimension
+  is treated as the minibatch dimension.  Elements of the `SparseTensor`
+  must be sorted in increasing order of this first dimension.  The serialized
+  `SparseTensor` objects going into each row of the output `Tensor` will have
+  rank `R-1`.
+
+  The minibatch size `N` is extracted from `sparse_shape[0]`.
+
+  Args:
+    sp_input: The input rank `R` `SparseTensor`.
+    out_type: The `dtype` to use for serialization.
+    name: A name prefix for the returned tensors (optional).
+
+  Returns:
+    A matrix (2-D `Tensor`) with `N` rows and `3` columns. Each column
+    represents serialized `SparseTensor`'s indices, values, and shape
+    (respectively).
+
   Raises:
     TypeError: If `sp_input` is not a `SparseTensor`.
   """
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
index d32529876f..98250df803 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
@@ -146,11 +146,11 @@ tf_module {
   }
   member_method {
     name: "serialize_many_sparse"
-    argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'string\'>\"], "
+    argspec: "args=[\'sp_input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'string\'>\", \'None\'], "
   }
   member_method {
     name: "serialize_sparse"
-    argspec: "args=[\'sp_input\', \'name\', \'out_type\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'string\'>\"], "
+    argspec: "args=[\'sp_input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'string\'>\", \'None\'], "
   }
   member_method {
     name: "serialize_tensor"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index b8cf21b29c..9b14c11614 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -328,6 +328,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # IMPORTANT: order here should correspond to OLD argument order.
     # We just prepend "arg_name=" to all arguments in function calls.
     self.function_reorders = {
+        "tf.io.serialize_sparse": ["sp_input", "name", "out_type"],
+        "tf.io.serialize_many_sparse": ["sp_input", "name", "out_type"],
         "tf.argmax": ["input", "axis", "name", "dimension", "output_type"],
         "tf.argmin": ["input", "axis", "name", "dimension", "output_type"],
         "tf.batch_to_space": ["input", "crops", "block_size", "name"],
-- 
GitLab


From 8be5ec06050a5084eec4f5ba3e2bc62b463e0044 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Nov 2018 22:22:51 -0800
Subject: [PATCH 0728/1554] Internal changes.

PiperOrigin-RevId: 222499496
---
 .../internal/reference/reference_ops.h        |  8 +-
 tensorflow/lite/kernels/reduce.cc             | 36 ++++++++-
 tensorflow/lite/testing/generate_examples.py  | 74 ++++++++++---------
 3 files changed, 76 insertions(+), 42 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 920f154049..be766ea452 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3663,8 +3663,10 @@ inline void Mean(const tflite::MeanParams& op_params,
                  const RuntimeShape& unextended_output_shape, T* output_data) {
   gemmlowp::ScopedProfilingLabel label("Mean");
 
-  TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4);
-  TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4);
+  // Current implementation only supports dimension equals 4 and simultaneous
+  // reduction over width and height.
+  TFLITE_CHECK_EQ(unextended_input_shape.DimensionsCount(), 4);
+  TFLITE_CHECK_LE(unextended_output_shape.DimensionsCount(), 4);
   const RuntimeShape input_shape =
       RuntimeShape::ExtendedShape(4, unextended_input_shape);
   const RuntimeShape output_shape =
@@ -3678,8 +3680,6 @@ inline void Mean(const tflite::MeanParams& op_params,
   const int input_height = input_shape.Dims(1);
   const int input_width = input_shape.Dims(2);
 
-  // The current implementation only supports simultaneous reduction over
-  // width and height.
   TFLITE_DCHECK_EQ(op_params.axis_count, 2);
   TFLITE_DCHECK((op_params.axis[0] == 1 && op_params.axis[1] == 2) ||
                 (op_params.axis[0] == 2 && op_params.axis[1] == 1));
diff --git a/tensorflow/lite/kernels/reduce.cc b/tensorflow/lite/kernels/reduce.cc
index ed2d475f6d..336e827ca4 100644
--- a/tensorflow/lite/kernels/reduce.cc
+++ b/tensorflow/lite/kernels/reduce.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include "tensorflow/lite/kernels/internal/quantization_util.h"
 #include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
 #include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/internal/tensor_ctypes.h"
+#include "tensorflow/lite/kernels/internal/types.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
 #include "tensorflow/lite/kernels/op_macros.h"
 
@@ -229,6 +231,17 @@ TfLiteStatus PrepareMeanOrSum(TfLiteContext* context, TfLiteNode* node) {
   return ResizeTempSum(context, &op_context, temp_sum);
 }
 
+void ResolveAxis(const int* axis_data, int axis_count,
+                 tflite::MeanParams* op_params) {
+  int i = 0;
+  for (; i < axis_count; ++i) {
+    op_params->axis[i] = static_cast<int16>(axis_data[i]);
+  }
+  for (; i < 4; ++i) {
+    op_params->axis[i] = 1;
+  }
+}
+
 template <KernelType kernel_type>
 TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) {
   OpContext op_context(context, node);
@@ -257,9 +270,23 @@ TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) {
 
   if (kernel_type == kReference) {
     switch (op_context.input->type) {
-      case kTfLiteFloat32:
-        TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, float, float));
-        break;
+      case kTfLiteFloat32: {
+        tflite::MeanParams op_params;
+        op_params.axis_count = num_axis;
+        ResolveAxis(GetTensorData<int>(op_context.axis), num_axis, &op_params);
+        const TfLiteTensor* input = op_context.input;
+        if (op_context.params->keep_dims && NumDimensions(input) == 4 &&
+            op_params.axis_count == 2 &&
+            ((op_params.axis[0] == 1 && op_params.axis[1] == 2) ||
+             (op_params.axis[0] == 2 && op_params.axis[1] == 1))) {
+          reference_ops::Mean(op_params, GetTensorShape(input),
+                              GetTensorData<float>(input),
+                              GetTensorShape(op_context.output),
+                              GetTensorData<float>(op_context.output));
+        } else {
+          TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, float, float));
+        }
+      } break;
       case kTfLiteInt32:
         TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, int, int64_t));
         break;
@@ -286,7 +313,8 @@ TfLiteStatus EvalMean(TfLiteContext* context, TfLiteNode* node) {
                   GetTensorData<int>(op_context.axis), num_axis,
                   op_context.params->keep_dims, GetTensorData<int>(temp_index),
                   GetTensorData<int>(resolved_axis),
-                  GetTensorData<int>(temp_sum), /*compute_sum=*/false));
+                  GetTensorData<int>(temp_sum),
+                  /*compute_sum=*/false));
         }
         break;
       default:
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index b7e549cc5c..5218844299 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -905,40 +905,46 @@ def make_reduce_tests(reduce_op,
   def f(zip_path):
     """Actual function that generates examples."""
 
-    test_parameters = [{
-        "input_dtype": [tf.float32, tf.int32, tf.int64],
-        "input_shape": [[3, 2, 4]],
-        "axis": [
-            0, 1, 2, [0, 1], [0, 2], [1, 2], [0, 1, 2], [1, 0], [2, 0],
-            [2, 1], [2, 1, 0], [2, 0, 1], -1, -2, -3, [1, -1], [0, -1], [-1, 0],
-            [-1, -2, -3], [0, 0, 0], [2, 2, 0], [1, 0, -3, -3]
-        ],
-        "const_axis": [True, False],
-        "keepdims": [True, False],
-    }, {
-        "input_dtype": [tf.float32],
-        "input_shape": [[1, 8, 8, 3]],
-        "axis": [
-            0, 1, 2, 3, [1, 2], [0, 3], [1, 2, 3], [0, 1, 2, 3],
-            [3, 2, 1, 0], [3, 1, 0, 2], [2, 0], [3, 0], [3, 1], [1, 0], -1, -2,
-            -3, -4, [0, -2], [2, 3, -1, 0], [3, 1, 2, -3], [3, -4], [2, 2, 2],
-            [2, 2, 3], [-3, -3, -4], [-3, 2, 1]
-        ],
-        "const_axis": [True, False],
-        "keepdims": [True, False],
-    }, {
-        "input_dtype": [tf.float32],
-        "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
-        "axis": [[]],   # shape is: [0]
-        "const_axis": [False],
-        "keepdims": [True, False],
-    }, {
-        "input_dtype": [tf.float32],
-        "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
-        "axis": [None],  # shape is: []
-        "const_axis": [True],
-        "keepdims": [True, False],
-    }]
+    test_parameters = [
+        {
+            "input_dtype": [tf.float32, tf.int32, tf.int64],
+            "input_shape": [[3, 3, 2, 4]],
+            "axis": [
+                0, 1, 2, [0, 1], [0, 2], [1, 2], [0, 1, 2], [1, 0], [2, 0],
+                [2, 1], [2, 1, 0], [2, 0, 1], -1, -2, -3, [1, -1], [0, -1],
+                [-1, 0], [-1, -2, -3], [0, 0, 0], [2, 2, 0], [1, 0, -3, -3]
+            ],
+            "const_axis": [True, False],
+            "keepdims": [True, False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[1, 8, 8, 3]],
+            "axis": [
+                0, 1, 2, 3, [1, 2], [0, 3], [1, 2, 3], [0, 1, 2,
+                                                        3], [3, 2, 1, 0],
+                [3, 1, 0, 2], [2, 0], [3, 0], [3, 1], [1, 0], -1, -2, -3, -4,
+                [0, -2], [2, 3, -1, 0], [3, 1, 2, -3], [3, -4], [2, 2, 2],
+                [2, 2, 3], [-3, -3, -4], [-3, 2, 1]
+            ],
+            "const_axis": [True, False],
+            "keepdims": [True, False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
+            "axis": [[]],  # shape is: [0]
+            "const_axis": [False],
+            "keepdims": [True, False],
+        },
+        {
+            "input_dtype": [tf.float32],
+            "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
+            "axis": [None],  # shape is: []
+            "const_axis": [True],
+            "keepdims": [True, False],
+        }
+    ]
 
     def build_graph(parameters):
       """Build the mean op testing graph."""
-- 
GitLab


From 5594c91a36a9df5c3ae6743ae4b6e763539634da Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 22 Nov 2018 00:27:41 -0800
Subject: [PATCH 0729/1554] Change return of the update op to be a no-op

PiperOrigin-RevId: 222506718
---
 tensorflow/python/keras/metrics.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index c30b953c33..668c56243b 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -676,7 +676,8 @@ class Mean(Metric):
     update_total_op = state_ops.assign_add(self.total, values)
     with ops.control_dependencies([update_total_op]):
       update_count_op = state_ops.assign_add(self.count, num_values)
-      return ops.convert_to_tensor(update_count_op)
+      with ops.control_dependencies([update_count_op]):
+        return control_flow_ops.no_op()
 
   def result(self):
     return math_ops.div_no_nan(self.total, self.count)
-- 
GitLab


From 80c7bf2f4b62ed58eab5cedbfe3f9dac3d3d6b62 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Nov 2018 01:02:30 -0800
Subject: [PATCH 0730/1554] compat: Update forward compatibility horizon to
 2018-11-22

PiperOrigin-RevId: 222510434
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 75290f0613..ebe3fdbf53 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 21)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 22)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 2007e1ba474030fcce840b0b8a599558e7d5998f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Nov 2018 06:18:43 -0800
Subject: [PATCH 0731/1554] Migrate losses to work with new
 tensorflow_probability.distributions.Distribution.

PiperOrigin-RevId: 222536654
---
 .../contrib/gan/python/losses/python/losses_impl.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index c91ce2c0f3..a0a86c6337 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -36,6 +36,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+
 from tensorflow.contrib.framework.python.ops import variables as contrib_variables_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -45,7 +46,6 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops.distributions import distribution as ds
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.ops.losses import util
 from tensorflow.python.summary import summary
@@ -738,11 +738,16 @@ def least_squares_discriminator_loss(
 def _validate_distributions(distributions):
   if not isinstance(distributions, (list, tuple)):
     raise ValueError('`distributions` must be a list or tuple. Instead, '
-                     'found %s.', type(distributions))
+                     'found %s.' % type(distributions))
   for x in distributions:
-    if not isinstance(x, ds.Distribution):
+    # We used to check with `isinstance(x, tf.distributions.Distribution)`.
+    # However, distributions have migrated to `tfp.distributions.Distribution`,
+    # which is a new code repo, so we can't check this way anymore until
+    # TF-GAN is migrated to a new repo as well.
+    # This new check is not sufficient, but is a useful heuristic for now.
+    if not callable(getattr(x, 'log_prob', None)):
       raise ValueError('`distributions` must be a list of `Distributions`. '
-                       'Instead, found %s.', type(x))
+                       'Instead, found %s.' % type(x))
 
 
 def _validate_information_penalty_inputs(
-- 
GitLab


From 72f193b5c9f306d258b289afd7a7977af7b8f5f7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Nov 2018 08:58:03 -0800
Subject: [PATCH 0732/1554] Automated rollback of commit
 54dc404a7d0cb0b39f6bbd92ff008414ce683480. Revert #23808.

PiperOrigin-RevId: 222548550
---
 configure.py          | 1 -
 tensorflow/BUILD      | 6 ------
 tensorflow/core/BUILD | 9 ++++-----
 tools/bazel.rc        | 1 -
 4 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/configure.py b/configure.py
index dfb87550b1..0b16fe1314 100644
--- a/configure.py
+++ b/configure.py
@@ -1694,7 +1694,6 @@ def main():
   config_info_line('nohdfs', 'Disable HDFS support.')
   config_info_line('noignite', 'Disable Apacha Ignite support.')
   config_info_line('nokafka', 'Disable Apache Kafka support.')
-  config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index fd4b94202a..17577afecb 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -246,12 +246,6 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-config_setting(
-    name = "no_nccl_support",
-    define_values = {"no_nccl_support": "true"},
-    visibility = ["//visibility:public"],
-)
-
 # Crosses between platforms and file system libraries not supported on those
 # platforms due to limitations in nested select() statements.
 config_setting(
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1caca0028e..2a8c2718ed 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -95,6 +95,7 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
 load("//tensorflow:tensorflow.bzl", "tf_cc_tests_gpu")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule")
+load("//tensorflow:tensorflow.bzl", "if_not_tx2_llvm_or_windows_cuda")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_only_cc_test")
 
 # For platform specific build config
@@ -1402,11 +1403,9 @@ cc_library(
         "//tensorflow/core/kernels:summary_kernels",
         "//tensorflow/core/kernels:training_ops",
         "//tensorflow/core/kernels:word2vec_kernels",
-    ] + tf_additional_cloud_kernel_deps() + select({
-        "//tensorflow:no_nccl_support": [],
-        "//tensorflow:windows": [],
-        "//conditions:default": ["//tensorflow/core/kernels:nccl_kernels"],
-    }) + if_not_windows([
+    ] + tf_additional_cloud_kernel_deps() + if_not_tx2_llvm_or_windows_cuda([
+        "//tensorflow/core/kernels:nccl_kernels",
+    ]) + if_not_windows([
         "//tensorflow/core/kernels:fact_op",
         "//tensorflow/core/kernels:array_not_windows",
         "//tensorflow/core/kernels:math_not_windows",
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 1fdf51f53e..8c2052ee8a 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -72,7 +72,6 @@ build:nogcp --define=no_gcp_support=true
 build:nohdfs --define=no_hdfs_support=true
 build:nokafka --define=no_kafka_support=true
 build:noignite --define=no_ignite_support=true
-build:nonccl --define=no_nccl_support=true
 
 build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
-- 
GitLab


From 4f40e8a5ec282163ea8978d8d83d5b96693c727a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Nov 2018 09:44:59 -0800
Subject: [PATCH 0733/1554] Improve performance of grouped convolutions

PiperOrigin-RevId: 222551854
---
 .../convolution_feature_group_converter.cc    | 148 +++++++++++++-----
 ...onvolution_feature_group_converter_test.cc |  20 +--
 .../compiler/xla/tests/convolution_test.cc    |  64 ++++++++
 3 files changed, 181 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
index 7f7f1503a0..10c53f15f5 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
@@ -142,16 +142,16 @@ std::vector<int32> GetMaskIds(int64 group_size, int64 group_count) {
 // Finally we use the Eq op of these two broadcasted constants and get the
 // desired mask.
 HloInstruction* GetExpandedFilterMask(
-    const Shape& filter_shape, int64 input_feature_dim,
-    int64 output_feature_dim, int64 group_count,
+    const Shape& filter_shape, int64 kernel_input_feature_dim,
+    int64 kernel_output_feature_dim, int64 group_count,
     const std::function<HloInstruction*(std::unique_ptr<HloInstruction>)>&
         add_instruction) {
   Shape expanded_filter_shape =
-      ExpandedFilterShape(filter_shape, group_count, input_feature_dim);
+      ExpandedFilterShape(filter_shape, group_count, kernel_input_feature_dim);
   Shape mask_shape = ShapeUtil::MakeShape(
       S32, AsInt64Slice(expanded_filter_shape.dimensions()));
-  int64 output_feature = filter_shape.dimensions(output_feature_dim);
-  int64 group_size = filter_shape.dimensions(input_feature_dim);
+  int64 output_feature = filter_shape.dimensions(kernel_output_feature_dim);
+  int64 group_size = filter_shape.dimensions(kernel_input_feature_dim);
 
   // Create a 'input_feature' sized linspace and 'output_feature' sized linspace
   // that will be broadcasted into perpendicular dimensions and compared.
@@ -159,15 +159,14 @@ HloInstruction* GetExpandedFilterMask(
       GetMaskIds(group_size, group_count);
   const std::vector<int32> output_feature_filter_mask =
       GetMaskIds(output_feature / group_count, group_count);
-
   auto mask1 = add_instruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<int32>(input_feature_filter_mask)));
-  auto broadcasted_mask1 = add_instruction(
-      HloInstruction::CreateBroadcast(mask_shape, mask1, {input_feature_dim}));
+  auto broadcasted_mask1 = add_instruction(HloInstruction::CreateBroadcast(
+      mask_shape, mask1, {kernel_input_feature_dim}));
   auto mask2 = add_instruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<int32>(output_feature_filter_mask)));
-  auto broadcasted_mask2 = add_instruction(
-      HloInstruction::CreateBroadcast(mask_shape, mask2, {output_feature_dim}));
+  auto broadcasted_mask2 = add_instruction(HloInstruction::CreateBroadcast(
+      mask_shape, mask2, {kernel_output_feature_dim}));
 
   // Compare the broadcasted output feature linspace to the input feature
   // linspace to create a diagonal predicate.
@@ -189,18 +188,20 @@ Status ConvolutionVisitor::HandleConvolution(HloInstruction* convolution) {
   };
 
   auto dim_numbers = convolution->convolution_dimension_numbers();
-  int64 input_feature_dim = dim_numbers.kernel_input_feature_dimension();
-  int64 group_size = filter->shape().dimensions(input_feature_dim);
-  int64 output_feature_dim = dim_numbers.kernel_output_feature_dimension();
-  auto expanded_filter_shape =
-      ExpandedFilterShape(filter->shape(), group_count, input_feature_dim);
-  HloInstruction* filter_mask = GetExpandedFilterMask(
-      filter->shape(), input_feature_dim, output_feature_dim, group_count, add);
+  int64 kernel_input_feature_dim = dim_numbers.kernel_input_feature_dimension();
+  int64 group_size = filter->shape().dimensions(kernel_input_feature_dim);
+  int64 kernel_output_feature_dim =
+      dim_numbers.kernel_output_feature_dimension();
+  auto expanded_filter_shape = ExpandedFilterShape(filter->shape(), group_count,
+                                                   kernel_input_feature_dim);
+  HloInstruction* filter_mask =
+      GetExpandedFilterMask(filter->shape(), kernel_input_feature_dim,
+                            kernel_output_feature_dim, group_count, add);
   HloInstruction* expanded_filter;
 
   if (group_size == 1) {
     bool depthwise_separable =
-        (group_count == filter->shape().dimensions(output_feature_dim));
+        (group_count == filter->shape().dimensions(kernel_output_feature_dim));
     // If the code generator handles depthwise separable convolutions
     // inherently, then no filter expansion is needed.
     if (!filter_expansion_ && depthwise_separable) {
@@ -241,39 +242,108 @@ Status ConvolutionVisitor::HandleConvolution(HloInstruction* convolution) {
     // We want to repeat 'filter' in the 'input_feature_dim' dimension
     // 'group_count' times.
     Shape reshaped_filter_shape =
-        ShapeUtil::DeleteDimension(input_feature_dim, filter->shape());
+        ShapeUtil::DeleteDimension(kernel_input_feature_dim, filter->shape());
     auto reshaped_filter =
         add(HloInstruction::CreateReshape(reshaped_filter_shape, filter));
     std::vector<int64> broadcast_dims;
     for (int64 i = 0; i < filter->shape().dimensions_size(); ++i) {
-      if (i == input_feature_dim) {
+      if (i == kernel_input_feature_dim) {
         continue;
       }
       broadcast_dims.push_back(i);
     }
     expanded_filter = add(HloInstruction::CreateBroadcast(
         expanded_filter_shape, reshaped_filter, broadcast_dims));
+
+    auto zero = add(HloInstruction::CreateConstant(
+        LiteralUtil::Zero(expanded_filter_shape.element_type())));
+    auto zero_filter =
+        add(HloInstruction::CreateBroadcast(expanded_filter_shape, zero, {}));
+    auto new_filter = add(HloInstruction::CreateTernary(
+        expanded_filter_shape, HloOpcode::kSelect, filter_mask, expanded_filter,
+        zero_filter));
+
+    auto new_convolution = HloInstruction::CreateConvolve(
+        convolution->shape(), convolution->mutable_operand(0), new_filter,
+        /*feature_group_count=*/1, convolution->window(), dim_numbers,
+        convolution->precision_config());
+    TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
+        convolution, std::move(new_convolution)));
   } else {
-    // We could possibly also use reshape, broadcast, reshape instead of concat
-    // here, but it would require more complex code, and for depthwise
-    // convolution we would never end up in this branch.
-    std::vector<HloInstruction*> concat_operands(group_count, filter);
-    expanded_filter = add(HloInstruction::CreateConcatenate(
-        expanded_filter_shape, concat_operands, input_feature_dim));
+    // The filter expansion mechanism adds zeroes in the kernel.
+    // For an OF = 12, IF = 6, and kernel IF = 2, the expanded filter mask
+    // would look like (IF on the Y-axis, OF on the X-axis)
+    // 1 1 1 1 0 0 0 0 0 0 0 0
+    // 1 1 1 1 0 0 0 0 0 0 0 0
+    // 0 0 0 0 1 1 1 1 0 0 0 0
+    // 0 0 0 0 1 1 1 1 0 0 0 0
+    // 0 0 0 0 0 0 0 0 1 1 1 1
+    // 0 0 0 0 0 0 0 0 1 1 1 1
+    //
+    // Instead of convolving the above with the input, we instead slice the
+    // kernel into three kernels, each containing islands of 1s from the filter
+    // above. We also slice the activations in the IF dimension with each slice
+    // of size = group_size. For each slice, we perform convolutions, and
+    // concatenate the generated outputs in the output OF dimension.
+
+    std::vector<HloInstruction*> sliced_convolutions;
+    auto activation = convolution->mutable_operand(0);
+    std::vector<int64> slice_strides(filter->shape().dimensions_size(), 1);
+    std::vector<int64> filter_slice_starts(filter->shape().dimensions_size(),
+                                           0);
+    std::vector<int64> filter_slice_limits(filter->shape().dimensions().begin(),
+                                           filter->shape().dimensions().end());
+    std::vector<int64> activation_slice_starts(
+        activation->shape().dimensions_size(), 0);
+    std::vector<int64> activation_slice_limits(
+        activation->shape().dimensions().begin(),
+        activation->shape().dimensions().end());
+
+    int64 output_feature =
+        filter->shape().dimensions(kernel_output_feature_dim);
+    auto output_feature_dim = dim_numbers.output_feature_dimension();
+    int64 filter_slice_width = output_feature / group_count;
+
+    int64 activation_input_feature_dim = dim_numbers.input_feature_dimension();
+
+    for (int64 i = 0; i < group_count; i++) {
+      filter_slice_starts[kernel_output_feature_dim] = i * filter_slice_width;
+      filter_slice_limits[kernel_output_feature_dim] =
+          (i + 1) * filter_slice_width;
+      auto filter_sliced_shape = filter->shape();
+      filter_sliced_shape.set_dimensions(kernel_output_feature_dim,
+                                         filter_slice_width);
+      auto filter_slice = add(HloInstruction::CreateSlice(
+          filter_sliced_shape, filter, filter_slice_starts, filter_slice_limits,
+          slice_strides));
+
+      activation_slice_starts[activation_input_feature_dim] = i * group_size;
+      activation_slice_limits[activation_input_feature_dim] =
+          (i + 1) * group_size;
+      auto activation_sliced_shape = activation->shape();
+      activation_sliced_shape.set_dimensions(activation_input_feature_dim,
+                                             group_size);
+      auto activation_slice = add(HloInstruction::CreateSlice(
+          activation_sliced_shape, activation, activation_slice_starts,
+          activation_slice_limits, slice_strides));
+
+      auto conv_slice_shape = convolution->shape();
+      conv_slice_shape.set_dimensions(output_feature_dim, filter_slice_width);
+
+      auto new_convolution = add(HloInstruction::CreateConvolve(
+          conv_slice_shape, activation_slice, filter_slice,
+          /*feature_group_count=*/1, convolution->window(), dim_numbers,
+          convolution->precision_config()));
+
+      sliced_convolutions.push_back(new_convolution);
+    }
+
+    auto new_conv = HloInstruction::CreateConcatenate(
+        convolution->shape(), sliced_convolutions, output_feature_dim);
+    TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
+        convolution, std::move(new_conv)));
   }
-  auto zero = add(HloInstruction::CreateConstant(
-      LiteralUtil::Zero(expanded_filter_shape.element_type())));
-  auto zero_filter =
-      add(HloInstruction::CreateBroadcast(expanded_filter_shape, zero, {}));
-  auto new_filter = add(
-      HloInstruction::CreateTernary(expanded_filter_shape, HloOpcode::kSelect,
-                                    filter_mask, expanded_filter, zero_filter));
-  auto new_convolution = HloInstruction::CreateConvolve(
-      convolution->shape(), convolution->mutable_operand(0), new_filter,
-      /*feature_group_count=*/1, convolution->window(), dim_numbers,
-      convolution->precision_config());
-  TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
-      convolution, std::move(new_convolution)));
+
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
index 28373ebf63..e6bf2143a2 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
+++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
@@ -82,18 +82,14 @@ ENTRY %Convolve1D1Window_0.v3 (input: f32[1,2,4], filter: f32[1,2,2]) -> f32[1,2
   ConvolutionFeatureGroupConverter converter;
   ASSERT_TRUE(converter.Run(module.get()).ValueOrDie());
   root = computation->root_instruction();
-  // Make sure the convolution is converted to one with feature_group_count = 1.
-  EXPECT_EQ(root->opcode(), HloOpcode::kConvolution);
-  EXPECT_EQ(root->feature_group_count(), 1);
-  // Verify that the filter operand has been replaced.
-  EXPECT_THAT(root->operand(1),
-              op::Select(op::Eq(op::Broadcast(op::Constant()),
-                                op::Broadcast(op::Constant())),
-                         // We expect to see Concatenate here instead of
-                         // Broadcast, because feature_group_count < input
-                         // feature dimension.
-                         op::Concatenate(op::Parameter(), op::Parameter()),
-                         op::Broadcast(op::Constant())));
+  // Make sure the convolution is replaced with a concatenate.
+  EXPECT_EQ(root->opcode(), HloOpcode::kConcatenate);
+  // And the operands of the concatenate are convolutions, each with a feature
+  // group count = 1.
+  EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kConvolution);
+  EXPECT_EQ(root->operand(1)->opcode(), HloOpcode::kConvolution);
+  EXPECT_EQ(root->operand(0)->feature_group_count(), 1);
+  EXPECT_EQ(root->operand(1)->feature_group_count(), 1);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index b52d30fd66..7e81905260 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -1346,6 +1346,70 @@ TYPED_TEST(Convolve2D_1x2x2x6_2x2x2x12_Grouped_Valid, Types) {
   this->RunTest();
 }
 
+template <typename T>
+class Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 2, 2, 1024};
+    std::vector<int64> filter_dims = {2, 2, 128, 512};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/8);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(512, static_cast<T>(1024));
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 = expected_r1.Reshape({1, 1, 1, 512}).ConsumeValueOrDie();
+
+    auto input_literal =
+        client_->TransferToServer(input_r4).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_);
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid, TestTypes);
+TYPED_TEST(Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid, Types) {
+  this->RunTest();
+}
+
 // Test fixture to run convolution tests with and without convolution
 // canonicalization enabled.
 class ConvolveWithAndWithoutCanonicalization
-- 
GitLab


From e320fba1e9349dee60ba1e06e1f6bbc08c2a85c1 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 22 Nov 2018 20:47:59 +0000
Subject: [PATCH 0734/1554] Update re2 library to 2018-10-01

This fix updates re2 library to the latest release of 2018-10-01

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7ad094c507..065a695453 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -168,12 +168,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "com_googlesource_code_re2",
-        sha256 = "803c7811146edeef8f91064de37c6f19136ff01a2a8cdb3230e940b2fd9f07fe",
-        strip_prefix = "re2-2018-07-01",
+        sha256 = "a31397714a353587413d307337d0b58f8a2e20e2b9d02f2e24e3463fa4eeda81",
+        strip_prefix = "re2-2018-10-01",
         system_build_file = clean_dep("//third_party/systemlibs:re2.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/google/re2/archive/2018-07-01.tar.gz",
-            "https://github.com/google/re2/archive/2018-07-01.tar.gz",
+            "https://mirror.bazel.build/github.com/google/re2/archive/2018-10-01.tar.gz",
+            "https://github.com/google/re2/archive/2018-10-01.tar.gz",
         ],
     )
 
-- 
GitLab


From ff2da7862e43aa624ac1f4689ce87eb929df8745 Mon Sep 17 00:00:00 2001
From: Rajagopal Ananthanarayanan <ananthr@google.com>
Date: Thu, 22 Nov 2018 13:18:15 -0800
Subject: [PATCH 0735/1554] Automated rollback of commit
 aaad174baa41472d996c91866883f16d67a844cf

PiperOrigin-RevId: 222563233
---
 tensorflow/contrib/tpu/BUILD                  |  1 -
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 31 ++++++-------------
 2 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 8264462a06..a0a9cb3f31 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -78,7 +78,6 @@ py_library(
         "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
-        "//tensorflow/python:session",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:summary",
         "//tensorflow/python:summary_ops_v2",
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 2aaf65881b..637bd30828 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -45,7 +45,6 @@ from tensorflow.contrib.training.python.training import hparam
 from tensorflow.core.framework import variable_pb2
 from tensorflow.core.framework.summary_pb2 import Summary
 from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import session as tf_session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest as data_nest
 from tensorflow.python.estimator import estimator as estimator_lib
@@ -413,15 +412,12 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
                enqueue_ops,
                dequeue_ops,
                run_infeed_loop_on_coordinator=True,
-               rendezvous=None,
-               master=None,
-               session_config=None):
+               rendezvous=None):
     self._master_job = ctx.master_job
     self._enqueue_ops = enqueue_ops
     self._dequeue_ops = dequeue_ops
     self._rendezvous = rendezvous
-    self._master = master
-    self._session_config = session_config
+
     self._run_infeed_loop_on_coordinator = run_infeed_loop_on_coordinator
     self._initial_infeed_sleep_secs = (
         ctx.config.tpu_config.initial_infeed_sleep_secs)
@@ -433,10 +429,11 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
   def begin(self):
     logging.info('TPU job name %s', self._master_job)
     self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
-    self._init_ops = []
     if self._should_initialize_tpu:
+      self._init_ops = [tpu.initialize_system(job=self._master_job)]
       self._finalize_ops = [tpu.shutdown_system(job=self._master_job)]
     else:
+      self._init_ops = []
       self._finalize_ops = []
 
     summary_writer_init_ops = contrib_summary.summary_writer_initializer_op()
@@ -478,17 +475,11 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
     return _OpQueueContext(name=name, target=target, args=args)
 
   def after_create_session(self, session, coord):
-    if self._should_initialize_tpu:
-      logging.info('Init TPU system')
-      start = time.time()
-      with ops.Graph().as_default():
-        with tf_session.Session(
-            self._master, config=self._session_config) as sess:
-          sess.run(tpu.initialize_system(job=self._master_job))
-      logging.info('Initialized TPU in %d seconds', time.time() - start)
-
+    logging.info('Init TPU system')
+    start = time.time()
     session.run(self._init_ops,
                 options=config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000))
+    logging.info('Initialized TPU in %d seconds', time.time() - start)
 
     self._infeed_controller = self._create_infeed_controller(
         name='InfeedController', target=self._run_infeed, args=(session,))
@@ -2573,8 +2564,6 @@ class TPUEstimator(estimator_lib.Estimator):
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator),
                   rendezvous=self._rendezvous[mode],
-                  master=self._config.master,
-                  session_config=self._session_config,
               ),
               InstallSignalHandlerHook()
           ])
@@ -2677,10 +2666,8 @@ class TPUEstimator(estimator_lib.Estimator):
                   eval_update_ops + host_ops,
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator),
-                  rendezvous=self._rendezvous[mode],
-                  master=self._config.master,
-                  session_config=self._session_config,
-              )] + input_hooks
+                  rendezvous=self._rendezvous[mode]),
+          ] + input_hooks
 
           if eval_hooks:
             hooks.extend(eval_hooks)
-- 
GitLab


From 0158a83cfb495de8a3d08dad8571ce75dc93d87b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Nov 2018 01:02:08 -0800
Subject: [PATCH 0736/1554] compat: Update forward compatibility horizon to
 2018-11-23

PiperOrigin-RevId: 222596771
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index ebe3fdbf53..9150d91409 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 22)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 23)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 5dbca226d9840a26178c53a509e87133bd1a73b9 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 23 Nov 2018 01:35:02 -0800
Subject: [PATCH 0737/1554] Add test case that shows TopK cannot handle minimum
 int value correctly.

PiperOrigin-RevId: 222599761
---
 .../compiler/xla/client/lib/sorting_test.cc   | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tensorflow/compiler/xla/client/lib/sorting_test.cc b/tensorflow/compiler/xla/client/lib/sorting_test.cc
index ebb30d3acc..27ff36c749 100644
--- a/tensorflow/compiler/xla/client/lib/sorting_test.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting_test.cc
@@ -14,6 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/client/lib/sorting.h"
+
+#include <limits>
+
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
@@ -41,6 +44,28 @@ XLA_TEST_F(SortingTest, TopK3From8Indices) {
   ComputeAndCompareR1<int>(&builder, {0, 1, 2}, {});
 }
 
+// TODO(b/119930279): enable this test.
+XLA_TEST_F(SortingTest, DISABLED_TopKFullSortMinInt) {
+  XlaBuilder builder(TestName());
+  auto x_rev = ConstantR1<int>(&builder, {std::numeric_limits<int>::min(),
+                                          std::numeric_limits<int>::min() + 1,
+                                          std::numeric_limits<int>::max()});
+  xla::GetTupleElement(xla::TopK(x_rev, 3), 1);
+  ComputeAndCompareR1<int>(&builder, {2, 1, 0}, {});
+}
+
+XLA_TEST_F(SortingTest, NOT_TopKFullSortMinInt) {
+  XlaBuilder builder(TestName());
+  auto x_rev = ConstantR1<int>(&builder, {std::numeric_limits<int>::min(),
+                                          std::numeric_limits<int>::min() + 1,
+                                          std::numeric_limits<int>::max()});
+  xla::GetTupleElement(xla::TopK(x_rev, 3), 1);
+  // TopK currently negates the keys, which doesn't work correctly for
+  // std::numeric_limits<int>::min(). Therefore, it will sort this key to the
+  // front instead of to the back.
+  ComputeAndCompareR1<int>(&builder, {0, 2, 1}, {});
+}
+
 XLA_TEST_F(SortingTest, TopKFullSort) {
   XlaBuilder builder(TestName());
   const int kSize = 16;
-- 
GitLab


From 9eb68e9f049b681fe448923da2205200410ea16d Mon Sep 17 00:00:00 2001
From: Sandip Giri <sgiri@us.ibm.com>
Date: Fri, 23 Nov 2018 15:33:08 +0530
Subject: [PATCH 0738/1554] Updating ppc64le CPU build status link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8af5370bef..b3133ccb25 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ The TensorFlow project strives to abide by generally accepted best practices in
 Build Type                                                                                                                                                                                      | Status                                                                                                                                                                                   | Artifacts
 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------
 **IBM s390x**                                                                                                                                                                                   | [![Build Status](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/badge/icon)](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/)                                                        | TBA
-**IBM ppc64le CPU**                                                                                                                                                                             | [![Build Status](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_CPU/badge/icon)](http://powerci.osuosl.org/job/TensorFlow_Ubuntu_16.04_CPU/)                                      | TBA
+**IBM ppc64le CPU**                                                                                                                                                                             | [![Build Status](http://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Build/badge/icon)](http://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Build/)                                      | TBA
 **IBM ppc64le GPU** Nightly                                                                                                                                                                     | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/)            | [Nightly](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Nightly_Artifact/)
 **IBM ppc64le GPU** Stable Release                                                                                                                                                              | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/)                  | [Release](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_GPU_Release_Build/)
 **Linux CPU with Intel® MKL-DNN** Nightly                                                                                                                                                       | [![Build Status](https://tensorflow-ci.intel.com/job/tensorflow-mkl-linux-cpu/badge/icon)](https://tensorflow-ci.intel.com/job/tensorflow-mkl-linux-cpu/)                                | [Nightly](https://tensorflow-ci.intel.com/job/tensorflow-mkl-build-whl-nightly/)
-- 
GitLab


From 520d5d5a66f9a0ca2004f4d4fc6698dbc531f9e8 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 23 Nov 2018 02:59:42 -0800
Subject: [PATCH 0739/1554] Merge slice operands of concatenate if they are
 "adjacent".

We sometimes have a concatenate that has slices as operands which are slicing
along the concatenate dimension and are also adjacent. This can be simplified
by combining those slices into one slice and make this the operand of the
concatenate.

PiperOrigin-RevId: 222606151
---
 .../xla/service/algebraic_simplifier.cc       | 81 +++++++++++++++++--
 .../xla/service/algebraic_simplifier_test.cc  | 70 ++++++++++++++++
 2 files changed, 144 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 56bf3a9f69..0b0502b4ff 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -96,6 +96,11 @@ bool ReshapeOrCopyIsBitcast(
          valid_bitcast_callback(operand->shape(), instr->shape());
 }
 
+bool IsUnstridedSlice(const HloInstruction* hlo) {
+  return absl::c_all_of(hlo->slice_strides(),
+                        [](int64 stride) { return stride == 1; });
+}
+
 // AlgebraicSimplifierVisitor traverses the HLO computation and reduces certain
 // algebraic expressions to simplified forms. Note: This only supports
 // simplifications that simply look at the operands of an instruction. For the
@@ -520,7 +525,74 @@ Status AlgebraicSimplifierVisitor::HandleConcatenate(
     VLOG(10) << "trying to replace " << concatenate->ToString() << " with "
              << replacement->ToString();
     ReplaceInstructionIfSameShape(concatenate, replacement);
-  } else if (operands.size() == 2) {
+    return Status::OK();
+  }
+
+  // Check if we can merge "adjacent" slice operands which take slices from the
+  // same other op. For simplicity we only merge unstrided slices.
+  int64 concatenate_dimension = concatenate->concatenate_dimension();
+  for (int64 i = 0; i < operands.size(); ++i) {
+    if (operands[i]->opcode() != HloOpcode::kSlice ||
+        !IsUnstridedSlice(operands[i])) {
+      continue;
+    }
+    int64 slice_end = operands[i]->slice_limits(concatenate_dimension);
+    HloInstruction* slice_operand = operands[i]->mutable_operand(0);
+    int64 j = i + 1;
+    while (j < operands.size() && operands[j]->opcode() == HloOpcode::kSlice &&
+           IsUnstridedSlice(operands[j]) &&
+           operands[j]->operand(0) == slice_operand &&
+           operands[j]->slice_starts(concatenate_dimension) == slice_end) {
+      // Check that all the slice_start values are the same in all other
+      // dimensions. This implies that the slice_limit values are also the same,
+      // because operands of concatenate need to have the same shape, and we
+      // already checked that the slices are unstrided.
+      bool same_other_starts = true;
+      for (int64 k = 0; k < operands[j]->slice_starts().size(); ++k) {
+        if (k == concatenate_dimension) {
+          continue;
+        }
+        if (operands[i]->slice_starts(k) != operands[j]->slice_starts(k)) {
+          same_other_starts = false;
+          break;
+        }
+      }
+      if (!same_other_starts) {
+        break;
+      }
+      slice_end = operands[j]->slice_limits(concatenate_dimension);
+      ++j;
+    }
+    if (j - i > 1) {
+      Shape new_slice_shape = operands[i]->shape();
+      new_slice_shape.set_dimensions(
+          concatenate_dimension,
+          slice_end - operands[i]->slice_starts(concatenate_dimension));
+      auto new_limit_indices = operands[i]->slice_limits();
+      new_limit_indices[concatenate_dimension] = slice_end;
+      auto new_slice_op =
+          computation_->AddInstruction(HloInstruction::CreateSlice(
+              new_slice_shape, slice_operand,
+              /*start_indices=*/operands[i]->slice_starts(),
+              /*limit_indices=*/new_limit_indices,
+              /*strides=*/operands[i]->slice_strides()));
+      std::vector<HloInstruction*> new_operands;
+      for (int64 k = 0; k < i; ++k) {
+        new_operands.push_back(operands[k]);
+      }
+      new_operands.push_back(new_slice_op);
+      for (int64 k = j; k < operands.size(); ++k) {
+        new_operands.push_back(operands[k]);
+      }
+      auto replacement =
+          computation_->AddInstruction(concatenate->CloneWithNewOperands(
+              concatenate->shape(), new_operands));
+      ReplaceInstructionIfSameShape(concatenate, replacement);
+      return Status::OK();
+    }
+  }
+
+  if (operands.size() == 2) {
     // A binary concat with a broadcasted scalar as an operand can be converted
     // into a pad which is simpler to fold into other operations.
     bool is_effective_low_pad = Match(
@@ -536,7 +608,7 @@ Status AlgebraicSimplifierVisitor::HandleConcatenate(
       padding_config_dim->set_edge_padding_high(0);
       padding_config_dim->set_edge_padding_low(0);
       padding_config_dim->set_interior_padding(0);
-      if (dim == concatenate->concatenate_dimension()) {
+      if (dim == concatenate_dimension) {
         if (is_effective_low_pad) {
           padding_config_dim->set_edge_padding_low(
               operands[0]->shape().dimensions(dim));
@@ -2010,11 +2082,6 @@ StatusOr<bool> AlgebraicSimplifierVisitor::TrySimplifyScalarSlice(
   return false;
 }
 
-bool IsUnstridedSlice(const HloInstruction* hlo) {
-  return absl::c_all_of(hlo->slice_strides(),
-                        [](int64 stride) { return stride == 1; });
-}
-
 StatusOr<bool> AlgebraicSimplifierVisitor::TryToReorderSliceAndReshape(
     HloInstruction* slice) {
   CHECK_EQ(slice->opcode(), HloOpcode::kSlice);
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 8b8ba2a77d..24c35464ad 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -1437,6 +1437,76 @@ TEST_F(AlgebraicSimplifierTest, ConcatenateOfBroadcastBecomesPad) {
   EXPECT_THAT(computation->root_instruction(), op::Pad(param0, param1));
 }
 
+TEST_F(AlgebraicSimplifierTest, SimplifyConcatenateOfSlices) {
+  auto m = CreateNewVerifiedModule();
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {100, 99});
+  Shape concat_shape = ShapeUtil::MakeShape(F32, {50, 80});
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r2f32, "param0"));
+  HloInstruction* param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r2f32, "param1"));
+
+  HloInstruction* slice0 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{0, 0},
+      /*limit_indices=*/{50, 10}, /*strides=*/{1, 1}));
+
+  // Cannot merge 'slice0' and 'slice1' because of different start indices in
+  // dimension 0.
+  HloInstruction* slice1 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 10},
+      /*limit_indices=*/{100, 20}, /*strides=*/{1, 1}));
+
+  // Cannot merge 'slice1' and 'slice2' because of stride in dimension 2.
+  HloInstruction* slice2 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 20},
+      /*limit_indices=*/{100, 40}, /*strides=*/{1, 2}));
+
+  // Cannot merge 'slice2' and 'slice3' because of stride in dimension 2.
+  HloInstruction* slice3 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 40},
+      /*limit_indices=*/{100, 50}, /*strides=*/{1, 1}));
+
+  // Can merge 'slice3' and 'slice4'.
+  HloInstruction* slice4 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 50},
+      /*limit_indices=*/{100, 60}, /*strides=*/{1, 1}));
+
+  // Can merge 'slice4' and 'slice5'.
+  HloInstruction* slice5 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 60},
+      /*limit_indices=*/{100, 70}, /*strides=*/{1, 1}));
+
+  // Cannot merge 'slice5' and 'slice6' because of overlap.
+  HloInstruction* slice6 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param0, /*start_indices=*/{50, 69},
+      /*limit_indices=*/{100, 79}, /*strides=*/{1, 1}));
+
+  // Cannot merge 'slice6' and 'slice7' because of slicing from a different
+  // parameter.
+  HloInstruction* slice7 = builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {50, 10}), param1, /*start_indices=*/{50, 79},
+      /*limit_indices=*/{100, 89}, /*strides=*/{1, 1}));
+
+  builder.AddInstruction(HloInstruction::CreateConcatenate(
+      concat_shape,
+      {slice0, slice1, slice2, slice3, slice4, slice5, slice6, slice7}, 1));
+  auto computation = m->AddEntryComputation(builder.Build());
+
+  AlgebraicSimplifier simplifier(default_options_);
+  ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  EXPECT_THAT(
+      computation->root_instruction(),
+      op::Concatenate(op::Slice(param0), op::Slice(param0), op::Slice(param0),
+                      op::Slice(param0), op::Slice(param0), op::Slice(param1)));
+  // The operand 3 should be a merge of 'slice3', 'slice4' and 'slice5', so its
+  // shape should have dimensions {50, 30}.
+  EXPECT_TRUE(
+      ShapeUtil::Equal(computation->root_instruction()->operand(3)->shape(),
+                       ShapeUtil::MakeShape(F32, {50, 30})));
+  EXPECT_EQ(computation->root_instruction()->operand(3)->slice_starts(1), 40);
+}
+
 // Test that a simplification which changes layouts is not performed if layout
 // sensitive is true.
 TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) {
-- 
GitLab


From 48809b87793882266f01b7b40bc9e4a6e0f18f57 Mon Sep 17 00:00:00 2001
From: AG Ramesh <ag.ramesh@intel.com>
Date: Fri, 23 Nov 2018 11:44:19 -0800
Subject: [PATCH 0740/1554] Fixed merge errors and clang format issues

---
 tensorflow/core/graph/mkl_layout_pass.cc      |   6 +-
 tensorflow/core/graph/mkl_layout_pass_test.cc |  47 +++--
 tensorflow/core/kernels/mkl_conv_ops.cc       | 180 +++++++++---------
 tensorflow/core/kernels/mkl_conv_ops.h        |  14 +-
 tensorflow/core/kernels/mkl_fused_ops_test.cc |   2 -
 tensorflow/core/ops/nn_ops.cc                 |   3 +-
 6 files changed, 121 insertions(+), 131 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 8de0fc6083..de1a982b9d 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2289,13 +2289,13 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // We handle control edges now.
   for (const Edge* e : pred->in_edges()) {
     if (e->IsControlEdge()) {
-      //Don't allow duplicate edge
+      // Don't allow duplicate edge
       (*g)->AddControlEdge(e->src(), new_node, false);
     }
   }
   for (const Edge* e : succ->in_edges()) {
     if (e->IsControlEdge()) {
-      //Don't allow duplicate edge
+      // Don't allow duplicate edge
       (*g)->AddControlEdge(e->src(), new_node, false);
     }
   }
@@ -2304,7 +2304,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // First, we will fix outgoing control edges from 'pred' node.
   for (const Edge* e : pred->out_edges()) {
     if (e->IsControlEdge()) {
-      //Don't allow duplicate edge
+      // Don't allow duplicate edge
       (*g)->AddControlEdge(new_node, e->dst(), false);
     }
   }
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 0c8d7f7dbb..fa059f1194 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -138,13 +138,8 @@ REGISTER_OP("_MklInput2")
     .Output("o: uint8")
     .Output("o1: uint8")
     .SetIsStateful();
-REGISTER_OP("Output2")
-    .Input("i: float")
-    .Input("i1: float")
-    .SetIsStateful();
-REGISTER_OP("Output")
-    .Input("i: float")
-    .SetIsStateful();
+REGISTER_OP("Output2").Input("i: float").Input("i1: float").SetIsStateful();
+REGISTER_OP("Output").Input("i: float").SetIsStateful();
 
 /////////////////////////////////////////////////////////////////////
 //  Unit tests related to node merge optiimization
@@ -163,7 +158,6 @@ TEST_F(MklLayoutPassTest, Basic) {
             "A->C;A->D;B->C:1;B->D:1");
 }
 
-
 // Test set 1: Conv2D + AddBias
 
 // C=Conv2D(A,B); E=BiasAdd(C,D); Z=Zeta(E,Y)
@@ -470,7 +464,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_ConvBpropInput_FilterFwd) {
             "E:3->G:4;F->G;F:control->DMT/_3:control;G->Z;X->Y:1;X->Z:1");
 }
 
-// Test set 3: Pad + Conv2D fusion 
+// Test set 3: Pad + Conv2D fusion
 // padding is VALID type
 // A = input(image), B = input(paddings), C= Pad = input of conv2D,
 // D=input(filter), E = Conv2D, Z = Zeta
@@ -508,10 +502,10 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
 }
 // Test if input control edges do not duplicate after merge.
 // If both the merging ops have input control edge from a common op
-// then, the merged op will have only one control edge from that 
+// then, the merged op will have only one control edge from that
 // common op.
 // padding is VALID type
-// A = input(image), A1 = input, B = input(paddings), 
+// A = input(image), A1 = input, B = input(paddings),
 // C= Pad = input of conv2D,
 // D=input(filter), E = Conv2D, Z = Zeta
 // C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
@@ -550,12 +544,14 @@ TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
   const Edge* edge_1 = graph_.AddControlEdge(a1, e);
   ASSERT_NE(edge, nullptr);
   ASSERT_NE(edge_1, nullptr);
-  EXPECT_EQ(DoMklLayoutOptimizationPass(),
-            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
-            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
-            "A1:control->E:control;A:control->DMT/_0:control;A:control->DMT/_1:control;"
-            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
-            "DMT/_2->E:5;E->Z;Y->Z:1");
+  EXPECT_EQ(
+      DoMklLayoutOptimizationPass(),
+      "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+      "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
+      "A1:control->E:control;A:control->DMT/_0:control;A:control->DMT/"
+      "_1:control;"
+      "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
+      "DMT/_2->E:5;E->Z;Y->Z:1");
 }
 // Test if output control edges does not duplicate after merge.
 // If both the merging ops have output control edge to a common op,
@@ -600,16 +596,17 @@ TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithConv2D_Positive) {
   const Edge* edge_1 = graph_.AddControlEdge(e, a1);
   ASSERT_NE(edge, nullptr);
   ASSERT_NE(edge_1, nullptr);
-  EXPECT_EQ(DoMklLayoutOptimizationPass(),
-            "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
-            "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
-            "A:control->DMT/_0:control;A:control->DMT/_1:control;"
-            "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
-            "DMT/_2->E:5;E->Z;E:control->A1:control;Y->Z:1");
+  EXPECT_EQ(
+      DoMklLayoutOptimizationPass(),
+      "A(Input);A1(Input);B(Int32Input);D(Input);DMT/_0(Const);DMT/_1(Const);"
+      "DMT/_2(Const);E(_MklPadWithConv2D);Y(Input);Z(Zeta)|A->E;"
+      "A:control->DMT/_0:control;A:control->DMT/_1:control;"
+      "A:control->DMT/_2:control;B->E:2;D->E:1;DMT/_0->E:3;DMT/_1->E:4;"
+      "DMT/_2->E:5;E->Z;E:control->A1:control;Y->Z:1");
 }
 // Pad + Conv2D fusion with padding is VALID,
 // Input node pointing to both Pad and Conv2D
-// A = input(image), B = input(paddings), C= Pad 
+// A = input(image), B = input(paddings), C= Pad
 // E = Conv2D, Z = Zeta
 // C=Pad(A,B); E=Conv2D(C,A); Z=Zeta(E,Y)
 // After layout pass
@@ -645,7 +642,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_Input) {
 // Pad + Conv2D with padding is VALID,
 // Input node pointing to both Pad and Conv2D
 // Output of both Pad and Conv2D feeds one node (Z as Output2)
-// A = input(as image), B = input(as paddings), C= Pad 
+// A = input(as image), B = input(as paddings), C= Pad
 // E = Conv2D, Z = Output2
 // C=Pad(A,B); E=Conv2D(C,A); Z=Output(C,E)
 // After layout pass - No merging, since Pad and Conv2D both
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index cfc36d1495..9193d00592 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -465,19 +465,18 @@ class MklConvOp : public OpKernel {
                                         filter.shape().DebugString()));
 
     for (int i = 0; i < 3; i++) {
-      OP_REQUIRES(
-          context,
-          FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()),
-          errors::InvalidArgument("filter too large"));
+      OP_REQUIRES(context, FastBoundsCheck(filter.dim_size(i),
+                                           std::numeric_limits<int>::max()),
+                  errors::InvalidArgument("filter too large"));
     }
 
     const int64 input_depth =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'C')
                             : GetTensorDim(input, data_format_, 'C');
-    OP_REQUIRES(context, input_depth == filter.dim_size(2),
-                errors::InvalidArgument(
-                    "input and filter must have the same depth: ", input_depth,
-                    " vs ", filter.dim_size(2)));
+    OP_REQUIRES(
+        context, input_depth == filter.dim_size(2),
+        errors::InvalidArgument("input and filter must have the same depth: ",
+                                input_depth, " vs ", filter.dim_size(2)));
     // The last dimension for filter is out_depth.
     const int out_depth = static_cast<int>(filter.dim_size(3));
 
@@ -486,10 +485,9 @@ class MklConvOp : public OpKernel {
     const int64 input_rows_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'H')
                             : GetTensorDim(input, data_format_, 'H');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_rows_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input rows too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_rows_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input rows too large"));
     const int input_rows = static_cast<int>(input_rows_raw);
     const int filter_rows = static_cast<int>(filter.dim_size(0));
 
@@ -498,10 +496,9 @@ class MklConvOp : public OpKernel {
     const int64 input_cols_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'W')
                             : GetTensorDim(input, data_format_, 'W');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_cols_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input cols too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_cols_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input cols too large"));
     const int input_cols = static_cast<int>(input_cols_raw);
     const int filter_cols = static_cast<int>(filter.dim_size(1));
 
@@ -509,10 +506,9 @@ class MklConvOp : public OpKernel {
     const int64 input_batch_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'N')
                             : GetTensorDim(input, data_format_, 'N');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_batch_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("batch is too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_batch_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("batch is too large"));
     const int batch = static_cast<int>(input_batch_raw);
 
     // For now we take the stride from the second and third dimensions only (we
@@ -850,8 +846,8 @@ REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias")
 
 // Base class for convolution forward operations
 template <typename Device, typename Tinput, typename Tfilter, typename Tbias,
-          typename Toutput, typename Ttemp_output, typename Tpadding, bool biasEnabled,
-          bool padEnabled>
+          typename Toutput, typename Ttemp_output, typename Tpadding,
+          bool biasEnabled, bool padEnabled>
 class MklConvOp : public OpKernel {
  public:
   ~MklConvOp() {}
@@ -894,17 +890,15 @@ class MklConvOp : public OpKernel {
       OP_REQUIRES(context, dilations_.size() == 5,
                   errors::InvalidArgument("Dilation rates field must "
                                           "specify 5 dimensions"));
-      OP_REQUIRES(context,
-                  (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
-                   GetTensorDim(dilations_, data_format_, 'C') == 1),
+      OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
+                            GetTensorDim(dilations_, data_format_, 'C') == 1),
                   errors::InvalidArgument(
                       "Current implementation does not yet support "
                       "dilations rates in the batch and depth dimensions."));
       OP_REQUIRES(
-          context,
-          (GetTensorDim(dilations_, data_format_, '0') > 0 &&
-           GetTensorDim(dilations_, data_format_, '1') > 0 &&
-           GetTensorDim(dilations_, data_format_, '2') > 0),
+          context, (GetTensorDim(dilations_, data_format_, '0') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '1') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '2') > 0),
           errors::InvalidArgument("Dilated rates should be larger than 0."));
     }
   }
@@ -940,9 +934,9 @@ class MklConvOp : public OpKernel {
       auto src_tf_shape = GetTfShape(context, kInputIndex_Src);
       auto filter_tf_shape = GetTfShape(context, kInputIndex_Filter);
       conv_utl.GetConvFwdSizesInMklOrder(
-          src_tf_shape, filter_tf_shape, &src_dims, &filter_dims,
-          &strides, &dilations, &dst_dims_tf_order, &dst_dims_mkl_order,
-          &padding_left, &padding_right, padEnabled);
+          src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, &strides,
+          &dilations, &dst_dims_tf_order, &dst_dims_mkl_order, &padding_left,
+          &padding_right, padEnabled);
       if (!context->status().ok()) return;
 
       // Check for corner case - if there is nothing to compute, return.
@@ -974,9 +968,10 @@ class MklConvOp : public OpKernel {
 
       bool isConv2D = (strides_.size() == 4);
       // TODO(Intel-tf) Add check to make sure padEnabled is true only for 2D
-      if(!isConv2D){
-        OP_REQUIRES(context, padEnabled,
-                errors::InvalidArgument("Pad+Conv fusion only works for 2D"));
+      if (!isConv2D) {
+        OP_REQUIRES(
+            context, !padEnabled,
+            errors::InvalidArgument("Pad+Conv fusion only works for 2D"));
       }
       // Create memory for user data.
       // Describe how the inputs and outputs of Convolution look like. Also
@@ -1211,7 +1206,6 @@ class MklConvOp : public OpKernel {
   const int kInputIndex_Pad = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
   const int kDilationH = 0, kDilationW = 1;
-  
 
   // Allocate filter output tensor.
   void AllocateFilterOutputTensor(
@@ -1282,7 +1276,7 @@ template <typename Device, typename Tbias, typename Toutput,
           typename Ttemp_output, bool biasEnabled>
 class MklQuantizedConv2DOp
     : public MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output,
-                       biasEnabled> {
+                       int32, biasEnabled, false> {
  public:
   virtual ~MklQuantizedConv2DOp() {
     if (this->input_bias_ != nullptr) {
@@ -1297,13 +1291,13 @@ class MklQuantizedConv2DOp
   }
 
   explicit MklQuantizedConv2DOp(OpKernelConstruction* context)
-      : MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output,
-                  biasEnabled>(context) {}
+      : MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output, int32,
+                  biasEnabled, false>(context) {}
 
   void Compute(OpKernelContext* context) override {
     // Compute int32 output tensor
-    MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output,
-              biasEnabled>::Compute(context);
+    MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output, int32,
+              biasEnabled, false>::Compute(context);
 
     // Compute additional outputs: min/max scalars.
     int bias_index_offset;
@@ -1349,8 +1343,8 @@ class MklQuantizedConv2DOp
  protected:
   void ExtendConvFwdParams(OpKernelContext* context,
                            MklConvFwdParams& params) override {
-    MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output,
-              biasEnabled>::ExtendConvFwdParams(context, params);
+    MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output, int32,
+              biasEnabled, false>::ExtendConvFwdParams(context, params);
 
     // When the output type is quint8, the output data id requantized
     // into quint8. A post_op "output_scale" is added to do the conversion.
@@ -1561,11 +1555,11 @@ class MklQuantizedConv2DSumReluOp
       }
     }
     // TODO(mdfaijul): Add cleaner code for non-mkl tensor
-    MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output,
-              biasEnabled>::AllocateOutputTensor(context, conv_prim_desc,
-                                                 output_dims_mkl_order,
-                                                 output_tf_format,
-                                                 output_tensor);
+    MklConvOp<Device, quint8, qint8, Tbias, Toutput, Ttemp_output, int32,
+              biasEnabled, false>::AllocateOutputTensor(context, conv_prim_desc,
+                                                        output_dims_mkl_order,
+                                                        output_tf_format,
+                                                        output_tensor);
     const Tensor& summand = MklGetInput(context, summand_idx);
     if (summand.dtype() != DT_FLOAT)
       TF_CHECK_OK(Status(error::Code::FAILED_PRECONDITION,
@@ -1583,8 +1577,8 @@ class MklQuantizedConv2DSumReluOp
     const float max_filter =
         context->input(5 + bias_index_offset).flat<float>()(0);
 
-    reorder_sum_scale = 255.0 * 127.0 /
-                        (std::max(std::abs(max_input), std::abs(min_input)) *
+    reorder_sum_scale =
+        255.0 * 127.0 / (std::max(std::abs(max_input), std::abs(min_input)) *
                          std::max(std::abs(max_filter), std::abs(min_filter)));
     std::vector<float> scales;
     scales.push_back(reorder_sum_scale);
@@ -1833,52 +1827,56 @@ REGISTER_KERNEL_BUILDER(
     MklQuantizedConv2DSumReluOp<CPUDevice, qint32, quint8, qint8, true>);
 #endif  // INTEL_MKL_ML
 
-  
 // Register 2D operations
-#define REGISTER_MKL_CPU_2D(T)                                              \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv2D")                             \
-                              .Device(DEVICE_CPU)                        \
-                              .TypeConstraint<T>("T")                    \
-                              .Label(mkl_op_registry::kMklOpLabel),      \
-                          MklConvOp<CPUDevice, float, float, float, float, float, int32, false, false>); \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias")                     \
-                              .Device(DEVICE_CPU)                        \
-                              .TypeConstraint<T>("T")                    \
-                              .Label(mkl_op_registry::kMklOpLabel),      \
-                          MklConvOp<CPUDevice, float, float, float, float, float, int32, true, false>);  \
-  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")               \
-                              .Device(DEVICE_CPU)                        \
-                              .TypeConstraint<T>("T")                    \
-                              .Label(mkl_op_registry::kMklOpLabel),      \
-                          MklDummyOp<CPUDevice, T>);                     \
-  REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                      \
-                              .Device(DEVICE_CPU)                        \
-                              .TypeConstraint<T>("T")                    \
-                              .TypeConstraint<int32>("Tpaddings")        \
-                              .Label(mkl_op_registry::kMklOpLabel),      \
-                          MklConvOp<CPUDevice, float, float, float, float, float, int32, false, true>);  \
-  REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                      \
-                              .Device(DEVICE_CPU)                        \
-                              .TypeConstraint<T>("T")                    \
-                              .TypeConstraint<int64>("Tpaddings")        \
-                              .Label(mkl_op_registry::kMklOpLabel),      \
-                          MklConvOp<CPUDevice, float, float, float, float, float, int64, false, true>);  \
-  REGISTER_KERNEL_BUILDER(Name("__MklDummyPadWithConv2D")                \
-                              .Device(DEVICE_CPU)                        \
-                              .TypeConstraint<T>("T")                    \
-                              .TypeConstraint<int32>("Tpaddings")        \
-                              .Label(mkl_op_registry::kMklOpLabel),      \
+#define REGISTER_MKL_CPU_2D(T)                                             \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv2D")                               \
+                              .Device(DEVICE_CPU)                          \
+                              .TypeConstraint<T>("T")                      \
+                              .Label(mkl_op_registry::kMklOpLabel),        \
+                          MklConvOp<CPUDevice, float, float, float, float, \
+                                    float, int32, false, false>);          \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias")                       \
+                              .Device(DEVICE_CPU)                          \
+                              .TypeConstraint<T>("T")                      \
+                              .Label(mkl_op_registry::kMklOpLabel),        \
+                          MklConvOp<CPUDevice, float, float, float, float, \
+                                    float, int32, true, false>);           \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")                 \
+                              .Device(DEVICE_CPU)                          \
+                              .TypeConstraint<T>("T")                      \
+                              .Label(mkl_op_registry::kMklOpLabel),        \
+                          MklDummyOp<CPUDevice, T>);                       \
+  REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                        \
+                              .Device(DEVICE_CPU)                          \
+                              .TypeConstraint<T>("T")                      \
+                              .TypeConstraint<int32>("Tpaddings")          \
+                              .Label(mkl_op_registry::kMklOpLabel),        \
+                          MklConvOp<CPUDevice, float, float, float, float, \
+                                    float, int32, false, true>);           \
+  REGISTER_KERNEL_BUILDER(Name("_MklPadWithConv2D")                        \
+                              .Device(DEVICE_CPU)                          \
+                              .TypeConstraint<T>("T")                      \
+                              .TypeConstraint<int64>("Tpaddings")          \
+                              .Label(mkl_op_registry::kMklOpLabel),        \
+                          MklConvOp<CPUDevice, float, float, float, float, \
+                                    float, int64, false, true>);           \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyPadWithConv2D")                  \
+                              .Device(DEVICE_CPU)                          \
+                              .TypeConstraint<T>("T")                      \
+                              .TypeConstraint<int32>("Tpaddings")          \
+                              .Label(mkl_op_registry::kMklOpLabel),        \
                           MklDummyOp<CPUDevice, T>);
 
-TF_CALL_float(REGISTER_MKL_CPU);
+TF_CALL_float(REGISTER_MKL_CPU_2D);
 
 // Register 3D operations
-#define REGISTER_MKL_CPU_3D(T)                                      \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv3D")                        \
-                              .Device(DEVICE_CPU)                   \
-                              .TypeConstraint<T>("T")               \
-                              .Label(mkl_op_registry::kMklOpLabel), \
-                          MklConvOp<CPUDevice, T, T, T, T, T, int32, false, false>);
+#define REGISTER_MKL_CPU_3D(T)                  \
+  REGISTER_KERNEL_BUILDER(                      \
+      Name("_MklConv3D")                        \
+          .Device(DEVICE_CPU)                   \
+          .TypeConstraint<T>("T")               \
+          .Label(mkl_op_registry::kMklOpLabel), \
+      MklConvOp<CPUDevice, T, T, T, T, T, int32, false, false>);
 TF_CALL_float(REGISTER_MKL_CPU_3D);
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 8c71c20879..963826a73a 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -17,8 +17,8 @@ limitations under the License.
 #define TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_
 
 #include <limits>
-#include <vector>
 #include <memory>
+#include <vector>
 
 #include "mkldnn.hpp"
 #include "tensorflow/core/framework/numeric_op.h"
@@ -85,7 +85,7 @@ class MklDnnConvUtil {
   }
 
   // Calculate Convolution dilations
-  virtual inline void GetDilationsInMklOrder(memory::dims *dilations) {
+  virtual inline void GetDilationsInMklOrder(memory::dims* dilations) {
     // For now we take the dilation from the second and third dimensions only
     // (we do not support dilation on the batch or depth dimension).
     CHECK_NOTNULL(dilations);
@@ -195,9 +195,8 @@ class MklDnnConvUtil {
                                         filter_shape.DebugString()));
 
     for (int i = 0; i < ((strides_.size() == 4) ? 3 : 5); i++) {
-      OP_REQUIRES(context_,
-                  FastBoundsCheck(filter_shape.dim_size(i),
-                                  std::numeric_limits<int>::max()),
+      OP_REQUIRES(context_, FastBoundsCheck(filter_shape.dim_size(i),
+                                            std::numeric_limits<int>::max()),
                   errors::InvalidArgument("filter too large"));
     }
 
@@ -463,8 +462,8 @@ class MklDnnConvUtil {
                                           input_tf_shape.DebugString()));
     }
 
-    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape,
-                                  strides, dilations, output_dims_tf_order,
+    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape, strides,
+                                  dilations, output_dims_tf_order,
                                   output_dims_mkl_order, pad_l, pad_r);
   }
 
@@ -556,7 +555,6 @@ class MklConvBackpropCommonOp : public OpKernel {
   TensorFormat data_format_;  // NCHW or NHWC
 };
 
-
 /////////////////////////////////////////////////////////////////////
 ///  Dummy Mkl op that is just used for operators that are intermediate
 ///  output of node fusion in the graph
diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl_fused_ops_test.cc
index 900325ac91..991fb08093 100644
--- a/tensorflow/core/kernels/mkl_fused_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_fused_ops_test.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #ifdef INTEL_MKL
-#ifndef INTEL_MKL_ML_ONLY  // We don't support fusion in MKL ML
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/image_ops.h"
 #include "tensorflow/cc/ops/nn_ops.h"
@@ -160,5 +159,4 @@ TEST_F(FusedPadConvOpTest, PaddingConvTestNchw) {
   Run<float>(DT_FLOAT, image, filter, padding, expected, "NCHW");
 }
 }  // namespace tensorflow
-#endif  // INTEL_MKL_ML_ONLY
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 8afbe0333a..0b99542c5c 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1649,7 +1649,7 @@ REGISTER_OP("_MklPadWithConv2D")
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Attr("Tpaddings: {int32, int64} = DT_INT32")
-    .SetShapeFn(shape_inference::Conv2DShape) 
+    .SetShapeFn(shape_inference::Conv2DShape)
     .Doc(R"doc(
 MKL version of Pad and Conv2D operator. Uses MKL DNN APIs to perform
 Pad and 2D convolution to the output of convolution.
@@ -2159,7 +2159,6 @@ NOTE Do not invoke this operator directly in Python. Graph rewrite pass is
 expected to invoke these operators.
 )doc");
 
-
 REGISTER_OP("_MklAvgPool3DGrad")
     .Input("orig_input_shape: int32")
     .Input("grad: T")
-- 
GitLab


From a1532717be531a75267a34c62566805aa68abf4e Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 23 Nov 2018 14:05:07 -0800
Subject: [PATCH 0741/1554] Automated rollback of commit
 89deaf06c19a3eb1d5236d328b6b8cdde7238271

PiperOrigin-RevId: 222644654
---
 tensorflow/compiler/tests/xla_test.py    | 5 -----
 tensorflow/python/platform/googletest.py | 9 +--------
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py
index d15c073457..98a41981cf 100644
--- a/tensorflow/compiler/tests/xla_test.py
+++ b/tensorflow/compiler/tests/xla_test.py
@@ -22,7 +22,6 @@ import contextlib
 import os
 import random
 import re
-import sys
 
 import numpy as np
 
@@ -39,10 +38,6 @@ from tensorflow.python.platform import flags
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
-# TODO(b/35678764): Disable monkeypatched exit handlers once we find a proper
-# solution to invoke exit handlers during program exit.
-test._googletest.SetOSExit(sys.exit)
-
 FLAGS = flags.FLAGS
 
 flags.DEFINE_string('test_device', None,
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index c8312c153e..8141cf92c5 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -47,13 +47,6 @@ unittest_main = main
 # directory only once per test binary invocation.
 _googletest_temp_dir = ''
 
-_os_exit = sys.exit
-
-
-def SetOSExit(exit_func):
-  global _os_exit
-  _os_exit = exit_func
-
 
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
@@ -68,7 +61,7 @@ def g_main(argv):
       except IOError:
         sys.stderr.write('Error opening TEST_SHARD_STATUS_FILE (%s). Exiting.'
                          % os.environ['TEST_SHARD_STATUS_FILE'])
-        _os_exit(1)
+        sys.exit(1)
     finally:
       if f is not None: f.close()
 
-- 
GitLab


From 809ed3c835403564333bfdc06fca512432db4ca1 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Fri, 23 Nov 2018 14:36:05 -0800
Subject: [PATCH 0742/1554] [Core]: Use unique_ptr in DeviceMgr

In order to take advantage of the type system to help enforce ownership, this
change refactors DeviceMgr to use std::unique_ptr<Device> instead of Device*'s.
It also updates all callers to use the new types.

PiperOrigin-RevId: 222645861
---
 tensorflow/c/eager/BUILD                      |  1 +
 tensorflow/c/eager/c_api.cc                   | 11 +++---
 .../compiler/jit/build_xla_ops_pass_test.cc   |  8 +---
 .../compiler/jit/create_xla_launch_op_test.cc |  6 +--
 .../mark_for_compilation_pass_test_helper.cc  |  8 +---
 .../jit/partially_decluster_pass_test.cc      |  6 +--
 tensorflow/compiler/jit/xla_cpu_device.cc     | 11 +++---
 tensorflow/compiler/jit/xla_gpu_device.cc     | 10 ++---
 .../compiler/jit/xla_interpreter_device.cc    |  7 ++--
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  2 +-
 tensorflow/core/BUILD                         |  4 ++
 .../collective_executor_mgr_test.cc           |  6 +--
 .../collective_param_resolver_local_test.cc   |  6 +--
 .../collective_rma_local_test.cc              |  6 +--
 .../core/common_runtime/device_factory.cc     | 17 +++++----
 .../core/common_runtime/device_factory.h      | 13 ++++---
 tensorflow/core/common_runtime/device_mgr.cc  | 37 +++++++++++--------
 tensorflow/core/common_runtime/device_mgr.h   | 15 ++++----
 .../device_resolver_local_test.cc             |  6 +--
 .../core/common_runtime/device_set_test.cc    |  2 +-
 .../core/common_runtime/direct_session.cc     |  4 +-
 tensorflow/core/common_runtime/eager/BUILD    |  1 +
 .../eager/kernel_and_device_test.cc           | 14 ++++---
 .../core/common_runtime/executor_test.cc      |  8 ++--
 .../core/common_runtime/function_test.cc      |  7 ++--
 .../function_threadpool_test.cc               |  6 +--
 .../core/common_runtime/gpu/gpu_device.cc     | 20 +++++-----
 .../core/common_runtime/gpu/gpu_device.h      | 18 ++++-----
 .../common_runtime/gpu/gpu_device_factory.cc  | 21 +++++------
 .../gpu/gpu_device_on_non_gpu_machine_test.cc |  2 +-
 .../common_runtime/gpu/gpu_device_test.cc     | 32 +++++++---------
 .../hierarchical_tree_broadcaster_test.cc     | 11 +++---
 .../kernel_benchmark_testlib.cc               |  8 ++--
 .../common_runtime/kernel_benchmark_testlib.h |  2 +-
 tensorflow/core/common_runtime/placer_test.cc |  2 +-
 .../process_function_library_runtime_test.cc  | 18 +++++----
 .../core/common_runtime/renamed_device.cc     | 14 +++----
 .../core/common_runtime/renamed_device.h      |  7 ++--
 .../core/common_runtime/ring_reducer_test.cc  | 11 +++---
 .../threadpool_device_factory.cc              | 17 +++++----
 tensorflow/core/distributed_runtime/BUILD     |  1 +
 ...lective_param_resolver_distributed_test.cc | 11 +++---
 .../collective_rma_distributed_test.cc        | 11 +++---
 .../device_resolver_distributed_test.cc       | 13 ++++---
 .../eager/eager_service_impl.cc               |  6 +--
 .../eager/eager_service_impl_test.cc          |  9 ++---
 .../rpc/grpc_server_lib.cc                    | 11 ++++--
 .../rpc_collective_executor_mgr_test.cc       |  6 +--
 .../core/distributed_runtime/session_mgr.cc   |  4 +-
 .../distributed_runtime/session_mgr_test.cc   |  8 ++--
 .../core/grappler/grappler_item_builder.cc    |  9 +++--
 tensorflow/core/grappler/optimizers/BUILD     |  2 +-
 .../grappler/optimizers/function_optimizer.cc |  9 +++--
 tensorflow/core/kernels/data/BUILD            |  1 +
 tensorflow/core/kernels/data/iterator_ops.cc  |  6 +--
 .../data/single_threaded_executor_test.cc     |  8 ++--
 tensorflow/lite/delegates/flex/BUILD          |  1 +
 .../lite/delegates/flex/delegate_data.cc      |  7 ++--
 tensorflow/lite/toco/import_tensorflow.cc     |  4 +-
 tensorflow/python/client/device_lib.i         |  7 +---
 tensorflow/python/grappler/tf_optimizer.i     |  5 +--
 61 files changed, 273 insertions(+), 271 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index ba3d8533db..5a0988ed31 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -50,6 +50,7 @@ tf_cuda_library(
         ],
         "//conditions:default": [],
     }) + [
+        "@com_google_absl//absl/memory",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/distributed_runtime/eager:eager_client",
         "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 192044915f..c9e730ef41 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/c_api_internal.h"
@@ -80,7 +81,7 @@ tensorflow::Status GetAllRemoteDevices(
     const std::vector<string>& remote_workers,
     tensorflow::WorkerCacheInterface* worker_cache,
     std::unique_ptr<tensorflow::DeviceMgr>* device_mgr) {
-  std::vector<tensorflow::Device*> remote_devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> remote_devices;
   tensorflow::Status status;
   // TODO(nareshmodi) do this in parallel instead of serially.
   for (const string& remote_worker : remote_workers) {
@@ -93,7 +94,7 @@ tensorflow::Status GetAllRemoteDevices(
           status = s;
           if (s.ok()) {
             for (tensorflow::Device* d : *devices) {
-              remote_devices.push_back(d);
+              remote_devices.emplace_back(d);
             }
           }
           n.Notify();
@@ -101,7 +102,7 @@ tensorflow::Status GetAllRemoteDevices(
     n.WaitForNotification();
   }
   std::unique_ptr<tensorflow::DeviceMgr> remote_device_mgr(
-      new tensorflow::DeviceMgr(remote_devices));
+      new tensorflow::DeviceMgr(std::move(remote_devices)));
 
   TF_RETURN_IF_ERROR(status);
 
@@ -262,13 +263,13 @@ TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx,
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
 
 TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   status->status = tensorflow::DeviceFactory::AddDevices(
       opts->session_options.options, "/job:localhost/replica:0/task:0",
       &devices);
   if (!status->status.ok()) return nullptr;
   std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(devices));
+      new tensorflow::DeviceMgr(std::move(devices)));
 
   tensorflow::Rendezvous* r =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
index 11df946cc1..48a23a4c17 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
@@ -42,14 +42,8 @@ class BuildXlaOpsTest : public ::testing::Test {
               .ok());
   }
 
-  void TearDown() override {
-    for (Device* device : devices_) {
-      delete device;
-    }
-  }
-
  private:
-  std::vector<Device*> devices_;
+  std::vector<std::unique_ptr<Device>> devices_;
 };
 
 using ::tensorflow::testing::FindNodeByName;
diff --git a/tensorflow/compiler/jit/create_xla_launch_op_test.cc b/tensorflow/compiler/jit/create_xla_launch_op_test.cc
index 7386660762..0f872a480f 100644
--- a/tensorflow/compiler/jit/create_xla_launch_op_test.cc
+++ b/tensorflow/compiler/jit/create_xla_launch_op_test.cc
@@ -59,8 +59,9 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 1});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices_));
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) {
@@ -69,7 +70,7 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
     lib_def_ = absl::make_unique<FunctionLibraryDefinition>(
         OpRegistry::Global(), proto);
     OptimizerOptions opts;
-    device_mgr_ = absl::make_unique<DeviceMgr>(devices_);
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
     pflr_ = absl::make_unique<ProcessFunctionLibraryRuntime>(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
@@ -77,7 +78,6 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
   }
 
   FunctionLibraryRuntime* flr_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
index d56d0f8ccf..64a3301745 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
@@ -34,15 +34,9 @@ namespace tensorflow {
   //
   // It may be worth refactoring out XlaOpRegistry::RegisterCompilationDevice to
   // make this more direct, but probably not worth it solely for this test.
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(*session_options, "", &devices));
 
-  auto delete_devices = gtl::MakeCleanup([&] {
-    for (Device* d : devices) {
-      delete d;
-    }
-  });
-
   GraphOptimizationPassOptions opt_options;
   opt_options.graph = graph;
   opt_options.session_options = session_options;
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index 1fc5da5071..38a54cc5ef 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -386,7 +386,7 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   TF_ASSERT_OK(s.ToGraph(graph.get()));
 
   // This is needed to register the XLA_GPU device.
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_ASSERT_OK(DeviceFactory::AddDevices(
       SessionOptions(), "/job:localhost/replica:0/task:0", &devices));
 
@@ -400,10 +400,6 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   TF_ASSERT_OK(PartiallyDecluster(&graph));
 
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0");
-
-  for (Device* d : devices) {
-    delete d;
-  }
 }
 
 TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) {
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 9006dd514b..7df898ad12 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -31,12 +31,12 @@ namespace tensorflow {
 class XlaCpuDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 };
 
-Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
-                                          const string& name_prefix,
-                                          std::vector<Device*>* devices) {
+Status XlaCpuDeviceFactory::CreateDevices(
+    const SessionOptions& session_options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   XlaDeviceFlags* flags = GetXlaDeviceFlags();
   bool compile_on_demand = flags->tf_xla_compile_on_demand;
 
@@ -63,8 +63,7 @@ Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_CPU_XLA_JIT;
   options.use_multiple_streams = false;
-  auto device = absl::make_unique<XlaDevice>(session_options, options);
-  devices->push_back(device.release());
+  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 4419701695..944f732b99 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -29,12 +29,12 @@ namespace tensorflow {
 class XlaGpuDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 };
 
-Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
-                                          const string& name_prefix,
-                                          std::vector<Device*>* devices) {
+Status XlaGpuDeviceFactory::CreateDevices(
+    const SessionOptions& session_options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   XlaOpRegistry::DeviceRegistration registration;
   registration.compilation_device_name = DEVICE_GPU_XLA_JIT;
   registration.autoclustering_policy =
@@ -70,7 +70,7 @@ Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
       return status;
     }
 
-    devices->push_back(device.release());
+    devices->push_back(std::move(device));
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index e828bae865..4007309ed1 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -33,12 +33,12 @@ constexpr std::array<DataType, 9> kExecAllTypes = {
 class XlaInterpreterDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 };
 
 Status XlaInterpreterDeviceFactory::CreateDevices(
     const SessionOptions& session_options, const string& name_prefix,
-    std::vector<Device*>* devices) {
+    std::vector<std::unique_ptr<Device>>* devices) {
   static XlaDeviceOpRegistrations* registrations = RegisterXlaDeviceKernels(
       DEVICE_XLA_INTERPRETER, DEVICE_INTERPRETER_XLA_JIT);
   (void)registrations;
@@ -61,8 +61,7 @@ Status XlaInterpreterDeviceFactory::CreateDevices(
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT;
   options.use_multiple_streams = false;
-  auto device = absl::make_unique<XlaDevice>(session_options, options);
-  devices->push_back(device.release());
+  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
 
   return Status::OK();
 }
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 8036bc6844..2230bfd7d9 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -380,7 +380,7 @@ XlaCompiler::XlaCompiler(XlaCompiler::Options options)
       initialization_status_(Status::OK()),
       next_step_id_(1),
       device_(new XlaCompilationDevice(SessionOptions(), options_.device_type)),
-      device_mgr_({device_}) {
+      device_mgr_(absl::WrapUnique(device_)) {
   CHECK(!options_.device_type.type_string().empty());
   if (options_.populate_resource_manager) {
     initialization_status_ =
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 2a8c2718ed..1b4371198a 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2963,6 +2963,7 @@ tf_cuda_library(
         ":lib_internal",
         ":proto_text",
         ":protos_all_cc",
+        "@com_google_absl//absl/memory",
         "//third_party/eigen3",
         "//tensorflow/core/grappler:grappler_item",
     ] + mkl_deps(),
@@ -3816,6 +3817,7 @@ tf_cc_tests_gpu(
         ":test",
         ":test_main",
         ":testlib",
+        "@com_google_absl//absl/memory",
     ],
 )
 
@@ -3844,6 +3846,7 @@ tf_cc_tests_gpu(
         ":test",
         ":test_main",
         ":testlib",
+        "@com_google_absl//absl/memory",
     ],
 )
 
@@ -4411,6 +4414,7 @@ tf_cc_test(
         "//tensorflow/core/kernels:random_ops",
         "//tensorflow/core/kernels:shape_ops",
         "//third_party/eigen3",
+        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
index 91994c5731..f3d86aa633 100644
--- a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
+++ b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
@@ -38,8 +38,9 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
     auto* device_count = options.config.mutable_device_count();
     string task_name = "/job:localhost/replica:0/task:0";
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     std::unique_ptr<DeviceResolverInterface> drl(
         new DeviceResolverLocal(device_mgr_.get()));
     std::unique_ptr<ParamResolverInterface> prl(
@@ -50,7 +51,6 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
   }
 
   std::unique_ptr<CollectiveExecutorMgr> cme_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
 };
 
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
index 9a501b3298..94d889c40d 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
@@ -37,8 +37,9 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     string task_name = "/job:localhost/replica:0/task:0";
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
     prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
                                                 task_name));
@@ -73,7 +74,6 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     }
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
   std::unique_ptr<CollectiveParamResolverLocal> prl_;
diff --git a/tensorflow/core/common_runtime/collective_rma_local_test.cc b/tensorflow/core/common_runtime/collective_rma_local_test.cc
index a931fe64bd..4263f3a4ad 100644
--- a/tensorflow/core/common_runtime/collective_rma_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_rma_local_test.cc
@@ -42,8 +42,9 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
     prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
                                                 kTaskName));
@@ -51,7 +52,6 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
                                                kStepId));
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
   std::unique_ptr<CollectiveParamResolverLocal> prl_;
diff --git a/tensorflow/core/common_runtime/device_factory.cc b/tensorflow/core/common_runtime/device_factory.cc
index b94900114c..0fad13fe1e 100644
--- a/tensorflow/core/common_runtime/device_factory.cc
+++ b/tensorflow/core/common_runtime/device_factory.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
@@ -89,9 +90,9 @@ DeviceFactory* DeviceFactory::GetFactory(const string& device_type) {
   return it->second.factory.get();
 }
 
-Status DeviceFactory::AddDevices(const SessionOptions& options,
-                                 const string& name_prefix,
-                                 std::vector<Device*>* devices) {
+Status DeviceFactory::AddDevices(
+    const SessionOptions& options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   // CPU first. A CPU device is required.
   auto cpu_factory = GetFactory("CPU");
   if (!cpu_factory) {
@@ -116,16 +117,16 @@ Status DeviceFactory::AddDevices(const SessionOptions& options,
   return Status::OK();
 }
 
-Device* DeviceFactory::NewDevice(const string& type,
-                                 const SessionOptions& options,
-                                 const string& name_prefix) {
+std::unique_ptr<Device> DeviceFactory::NewDevice(const string& type,
+                                                 const SessionOptions& options,
+                                                 const string& name_prefix) {
   auto device_factory = GetFactory(type);
   if (!device_factory) {
     return nullptr;
   }
   SessionOptions opt = options;
   (*opt.config.mutable_device_count())[type] = 1;
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(device_factory->CreateDevices(opt, name_prefix, &devices));
   int expected_num_devices = 1;
   auto iter = options.config.device_count().find(type);
@@ -133,7 +134,7 @@ Device* DeviceFactory::NewDevice(const string& type,
     expected_num_devices = iter->second;
   }
   DCHECK_EQ(devices.size(), static_cast<size_t>(expected_num_devices));
-  return devices[0];
+  return std::move(devices[0]);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/device_factory.h b/tensorflow/core/common_runtime/device_factory.h
index db50226fe8..b3cd7adca9 100644
--- a/tensorflow/core/common_runtime/device_factory.h
+++ b/tensorflow/core/common_runtime/device_factory.h
@@ -40,18 +40,19 @@ class DeviceFactory {
   // CPU devices are added first.
   static Status AddDevices(const SessionOptions& options,
                            const string& name_prefix,
-                           std::vector<Device*>* devices);
+                           std::vector<std::unique_ptr<Device>>* devices);
 
   // Helper for tests.  Create a single device of type "type".  The
   // returned device is always numbered zero, so if creating multiple
   // devices of the same type, supply distinct name_prefix arguments.
-  static Device* NewDevice(const string& type, const SessionOptions& options,
-                           const string& name_prefix);
+  static std::unique_ptr<Device> NewDevice(const string& type,
+                                           const SessionOptions& options,
+                                           const string& name_prefix);
 
   // Most clients should call AddDevices() instead.
-  virtual Status CreateDevices(const SessionOptions& options,
-                               const string& name_prefix,
-                               std::vector<Device*>* devices) = 0;
+  virtual Status CreateDevices(
+      const SessionOptions& options, const string& name_prefix,
+      std::vector<std::unique_ptr<Device>>* devices) = 0;
 
   // Return the device priority number for a "device_type" string.
   //
diff --git a/tensorflow/core/common_runtime/device_mgr.cc b/tensorflow/core/common_runtime/device_mgr.cc
index 470abc1431..1f7d7c4699 100644
--- a/tensorflow/core/common_runtime/device_mgr.cc
+++ b/tensorflow/core/common_runtime/device_mgr.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device_mgr.h"
 
+#include <memory>
 #include <vector>
 #include "tensorflow/core/common_runtime/local_device.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -24,32 +25,32 @@ limitations under the License.
 
 namespace tensorflow {
 
-DeviceMgr::DeviceMgr(const std::vector<Device*>& devices)
-    : name_backing_store_(128) {
-  for (Device* d : devices) {
+DeviceMgr::DeviceMgr(std::vector<std::unique_ptr<Device>> devices)
+    : devices_(std::move(devices)), name_backing_store_(128) {
+  for (auto& d : devices_) {
     CHECK(d->device_mgr_ == nullptr);
     d->device_mgr_ = this;
 
-    devices_.push_back(d);
-
     // Register under the (1) full name and (2) canonical name.
     for (const string& name :
          DeviceNameUtils::GetNamesForDeviceMappings(d->parsed_name())) {
-      device_map_[CopyToBackingStore(name)] = d;
+      device_map_[CopyToBackingStore(name)] = d.get();
     }
     // Register under the (3) local name and (4) legacy local name.
     for (const string& name :
          DeviceNameUtils::GetLocalNamesForDeviceMappings(d->parsed_name())) {
-      device_map_[CopyToBackingStore(name)] = d;
+      device_map_[CopyToBackingStore(name)] = d.get();
     }
     device_type_counts_[d->device_type()]++;
   }
 }
 
-DeviceMgr::~DeviceMgr() {
-  // TODO(b/37437134): Remove destructor after converting to std::unique_ptr.
-  for (Device* p : devices_) delete p;
-}
+DeviceMgr::DeviceMgr(std::unique_ptr<Device> device)
+    : DeviceMgr([&device] {
+        std::vector<std::unique_ptr<Device>> vector;
+        vector.push_back(std::move(device));
+        return vector;
+      }()) {}
 
 StringPiece DeviceMgr::CopyToBackingStore(StringPiece s) {
   size_t n = s.size();
@@ -61,18 +62,22 @@ StringPiece DeviceMgr::CopyToBackingStore(StringPiece s) {
 void DeviceMgr::ListDeviceAttributes(
     std::vector<DeviceAttributes>* devices) const {
   devices->reserve(devices_.size());
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     devices->emplace_back(dev->attributes());
   }
 }
 
 std::vector<Device*> DeviceMgr::ListDevices() const {
-  return std::vector<Device*>(devices_.begin(), devices_.end());
+  std::vector<Device*> devices(devices_.size());
+  for (size_t i = 0; i < devices_.size(); ++i) {
+    devices[i] = devices_[i].get();
+  }
+  return devices;
 }
 
 string DeviceMgr::DebugString() const {
   string out;
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     strings::StrAppend(&out, dev->name(), "\n");
   }
   return out;
@@ -80,7 +85,7 @@ string DeviceMgr::DebugString() const {
 
 string DeviceMgr::DeviceMappingString() const {
   string out;
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     if (!dev->attributes().physical_device_desc().empty()) {
       strings::StrAppend(&out, dev->name(), " -> ",
                          dev->attributes().physical_device_desc(), "\n");
@@ -107,7 +112,7 @@ Status DeviceMgr::LookupDevice(StringPiece name, Device** device) const {
 
 void DeviceMgr::ClearContainers(gtl::ArraySlice<string> containers) const {
   Status s;
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     if (containers.empty()) {
       s.Update(dev->resource_manager()->Cleanup(
           dev->resource_manager()->default_container()));
diff --git a/tensorflow/core/common_runtime/device_mgr.h b/tensorflow/core/common_runtime/device_mgr.h
index c1ff10d9b5..bf8694655a 100644
--- a/tensorflow/core/common_runtime/device_mgr.h
+++ b/tensorflow/core/common_runtime/device_mgr.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
 #define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
 
+#include <memory>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -34,15 +35,17 @@ class DeviceAttributes;
 
 class DeviceMgr {
  public:
-  // Takes ownership of each device in 'devices'.
+  // Constructs a DeviceMgr from a list of devices.
   // TODO(zhifengc): Other initialization information.
-  // TODO(b/37437134): Use std::unique_ptr's to track ownership.
-  explicit DeviceMgr(const std::vector<Device*>& devices);
-  ~DeviceMgr();
+  explicit DeviceMgr(std::vector<std::unique_ptr<Device>> devices);
+
+  // Constructs a DeviceMgr managing a single device.
+  explicit DeviceMgr(std::unique_ptr<Device> device);
 
   // Returns attributes of all devices.
   void ListDeviceAttributes(std::vector<DeviceAttributes>* devices) const;
 
+  // Returns raw pointers to the underlying devices.
   std::vector<Device*> ListDevices() const;
 
   // Returns a string listing all devices.
@@ -62,9 +65,7 @@ class DeviceMgr {
   int NumDeviceType(const string& type) const;
 
  private:
-  // TODO(b/37437134): Use std::unique_ptr's to track ownership.
-  typedef gtl::InlinedVector<Device*, 8> DeviceVec;
-  DeviceVec devices_;
+  const std::vector<std::unique_ptr<Device>> devices_;
 
   StringPiece CopyToBackingStore(StringPiece s);
 
diff --git a/tensorflow/core/common_runtime/device_resolver_local_test.cc b/tensorflow/core/common_runtime/device_resolver_local_test.cc
index f5a6471ff7..54f1119e13 100644
--- a/tensorflow/core/common_runtime/device_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/device_resolver_local_test.cc
@@ -36,12 +36,12 @@ class DeviceResolverLocalTest : public ::testing::Test {
     string task_name = "/job:localhost/replica:0/task:0";
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
 };
diff --git a/tensorflow/core/common_runtime/device_set_test.cc b/tensorflow/core/common_runtime/device_set_test.cc
index fd9c4222a7..6a8c3d14e5 100644
--- a/tensorflow/core/common_runtime/device_set_test.cc
+++ b/tensorflow/core/common_runtime/device_set_test.cc
@@ -57,7 +57,7 @@ class DeviceSetTest : public ::testing::Test {
 class DummyFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     return Status::OK();
   }
 };
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 40b7071f40..91717328b1 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -155,12 +155,12 @@ class DirectSessionFactory : public SessionFactory {
     if (options.config.graph_options().build_cost_model() > 0) {
       EnableCPUAllocatorFullStats(true);
     }
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
         options, "/job:localhost/replica:0/task:0", &devices));
 
     DirectSession* session =
-        new DirectSession(options, new DeviceMgr(devices), this);
+        new DirectSession(options, new DeviceMgr(std::move(devices)), this);
     {
       mutex_lock l(sessions_lock_);
       sessions_.push_back(session);
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index a7b618c18b..86890ba07d 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -181,6 +181,7 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
index 948bdbcaf5..3ffed3ce32 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/cc/client/client_session.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
@@ -37,12 +38,13 @@ namespace {
 class TestEnv {
  public:
   TestEnv() : flib_def_(OpRegistry::Global(), {}) {
-    Device* device =
-        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0");
-    device_mgr_.reset(new DeviceMgr({device}));
-    flib_runtime_ = NewFunctionLibraryRuntime(device_mgr_.get(), Env::Default(),
-                                              device, TF_GRAPH_DEF_VERSION,
-                                              &flib_def_, nullptr, {}, nullptr);
+    std::vector<std::unique_ptr<Device>> devices;
+    devices.push_back(
+        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0"));
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+    flib_runtime_ = NewFunctionLibraryRuntime(
+        device_mgr_.get(), Env::Default(), device_mgr_->ListDevices()[0],
+        TF_GRAPH_DEF_VERSION, &flib_def_, nullptr, {}, nullptr);
   }
 
   FunctionLibraryRuntime* function_library_runtime() const {
diff --git a/tensorflow/core/common_runtime/executor_test.cc b/tensorflow/core/common_runtime/executor_test.cc
index 7697103faf..c311b2533e 100644
--- a/tensorflow/core/common_runtime/executor_test.cc
+++ b/tensorflow/core/common_runtime/executor_test.cc
@@ -53,17 +53,17 @@ class ExecutorTest : public ::testing::Test {
     // when the test completes.
     CHECK(rendez_->Unref());
     delete exec_;
-    delete device_;
   }
 
   // Resets executor_ with a new executor based on a graph 'gdef'.
   void Create(std::unique_ptr<const Graph> graph) {
     const int version = graph->versions().producer();
     LocalExecutorParams params;
-    params.device = device_;
+    params.device = device_.get();
     params.create_kernel = [this, version](const NodeDef& ndef,
                                            OpKernel** kernel) {
-      return CreateNonCachedKernel(device_, nullptr, ndef, version, kernel);
+      return CreateNonCachedKernel(device_.get(), nullptr, ndef, version,
+                                   kernel);
     };
     params.delete_kernel = [](OpKernel* kernel) {
       DeleteNonCachedKernel(kernel);
@@ -83,7 +83,7 @@ class ExecutorTest : public ::testing::Test {
   }
 
   thread::ThreadPool* thread_pool_ = nullptr;
-  Device* device_ = nullptr;
+  std::unique_ptr<Device> device_;
   Executor* exec_ = nullptr;
   StepStatsCollector step_stats_collector_;
   StepStats step_stats_;
diff --git a/tensorflow/core/common_runtime/function_test.cc b/tensorflow/core/common_runtime/function_test.cc
index 13c189fb87..3b4c976685 100644
--- a/tensorflow/core/common_runtime/function_test.cc
+++ b/tensorflow/core/common_runtime/function_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <atomic>
 #include <utility>
 
+#include "absl/memory/memory.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
 #include "tensorflow/cc/ops/array_ops_internal.h"
@@ -147,14 +148,15 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 3});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices_));
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
     OptimizerOptions opts;
-    device_mgr_.reset(new DeviceMgr(devices_));
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
     pflr_.reset(new ProcessFunctionLibraryRuntime(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
@@ -358,7 +360,6 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   FunctionLibraryRuntime* flr0_;
   FunctionLibraryRuntime* flr1_;
   FunctionLibraryRuntime* flr2_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/core/common_runtime/function_threadpool_test.cc b/tensorflow/core/common_runtime/function_threadpool_test.cc
index 655a68cfc9..bdbe24a70d 100644
--- a/tensorflow/core/common_runtime/function_threadpool_test.cc
+++ b/tensorflow/core/common_runtime/function_threadpool_test.cc
@@ -54,14 +54,15 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 3});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices_));
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
     OptimizerOptions opts;
-    device_mgr_.reset(new DeviceMgr(devices_));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     pflr_.reset(new ProcessFunctionLibraryRuntime(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
@@ -194,7 +195,6 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   FunctionLibraryRuntime* flr0_;
   FunctionLibraryRuntime* flr1_;
   FunctionLibraryRuntime* flr2_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 81fea311e1..5152d97fde 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -907,9 +907,9 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
-Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options,
-                                           const string& name_prefix,
-                                           std::vector<Device*>* devices) {
+Status BaseGPUDeviceFactory::CreateDevices(
+    const SessionOptions& options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   TF_RETURN_IF_ERROR(ValidateGPUMachineManager());
   se::Platform* gpu_manager = GPUMachineManager();
   if (gpu_manager == nullptr) {
@@ -1073,12 +1073,10 @@ static string GetShortDeviceDescription(PlatformGpuId platform_gpu_id,
   // LINT.ThenChange(//tensorflow/python/platform/test.py)
 }
 
-Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
-                                             const string& name_prefix,
-                                             TfGpuId tf_gpu_id,
-                                             int64 memory_limit,
-                                             const DeviceLocality& dev_locality,
-                                             std::vector<Device*>* devices) {
+Status BaseGPUDeviceFactory::CreateGPUDevice(
+    const SessionOptions& options, const string& name_prefix, TfGpuId tf_gpu_id,
+    int64 memory_limit, const DeviceLocality& dev_locality,
+    std::vector<std::unique_ptr<Device>>* devices) {
   CHECK_GE(tf_gpu_id.value(), 0);
   const string device_name =
       strings::StrCat(name_prefix, "/device:GPU:", tf_gpu_id.value());
@@ -1108,7 +1106,7 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
   // different (which should be an error).
   //
   // TODO(laigd): report error if memory_limit doesn't match stats.bytes_limit.
-  BaseGPUDevice* gpu_device = CreateGPUDevice(
+  std::unique_ptr<BaseGPUDevice> gpu_device = CreateGPUDevice(
       options, device_name, static_cast<Bytes>(stats.bytes_limit), dev_locality,
       tf_gpu_id, GetShortDeviceDescription(platform_gpu_id, desc),
       gpu_allocator, ProcessState::singleton()->GetCPUAllocator(numa_node));
@@ -1116,7 +1114,7 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
             << (stats.bytes_limit >> 20) << " MB memory) -> physical GPU ("
             << GetShortDeviceDescription(platform_gpu_id, desc) << ")";
   TF_RETURN_IF_ERROR(gpu_device->Init(options));
-  devices->push_back(gpu_device);
+  devices->push_back(std::move(gpu_device));
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index 674e8384d5..d002d02c51 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -166,7 +166,7 @@ class BaseGPUDevice : public LocalDevice {
 class BaseGPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 
   struct InterconnectMap {
     // Name of interconnect technology, if known.
@@ -207,15 +207,13 @@ class BaseGPUDeviceFactory : public DeviceFactory {
   Status CreateGPUDevice(const SessionOptions& options,
                          const string& name_prefix, TfGpuId tf_gpu_id,
                          int64 memory_limit, const DeviceLocality& dev_locality,
-                         std::vector<Device*>* devices);
-
-  virtual BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
-                                         const string& name, Bytes memory_limit,
-                                         const DeviceLocality& dev_locality,
-                                         TfGpuId tf_gpu_id,
-                                         const string& physical_device_desc,
-                                         Allocator* gpu_allocator,
-                                         Allocator* cpu_allocator) = 0;
+                         std::vector<std::unique_ptr<Device>>* devices);
+
+  virtual std::unique_ptr<BaseGPUDevice> CreateGPUDevice(
+      const SessionOptions& options, const string& name, Bytes memory_limit,
+      const DeviceLocality& dev_locality, TfGpuId tf_gpu_id,
+      const string& physical_device_desc, Allocator* gpu_allocator,
+      Allocator* cpu_allocator) = 0;
 
   // Returns into 'ids' the list of valid platform GPU ids, in the order that
   // they should map to TF GPU ids "/device:GPU:0", "/device:GPU:1", etc,
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc b/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
index e1aaf95df6..8dc7197329 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
@@ -59,15 +59,14 @@ class GPUDevice : public BaseGPUDevice {
 
 class GPUDeviceFactory : public BaseGPUDeviceFactory {
  private:
-  BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
-                                 const string& name, Bytes memory_limit,
-                                 const DeviceLocality& locality,
-                                 TfGpuId tf_gpu_id,
-                                 const string& physical_device_desc,
-                                 Allocator* gpu_allocator,
-                                 Allocator* cpu_allocator) override {
-    return new GPUDevice(options, name, memory_limit, locality, tf_gpu_id,
-                         physical_device_desc, gpu_allocator, cpu_allocator);
+  std::unique_ptr<BaseGPUDevice> CreateGPUDevice(
+      const SessionOptions& options, const string& name, Bytes memory_limit,
+      const DeviceLocality& locality, TfGpuId tf_gpu_id,
+      const string& physical_device_desc, Allocator* gpu_allocator,
+      Allocator* cpu_allocator) override {
+    return absl::make_unique<GPUDevice>(options, name, memory_limit, locality,
+                                        tf_gpu_id, physical_device_desc,
+                                        gpu_allocator, cpu_allocator);
   }
 };
 
@@ -108,7 +107,7 @@ class GPUCompatibleCPUDevice : public ThreadPoolDevice {
 class GPUCompatibleCPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
     if (iter != options.config.device_count().end()) {
@@ -116,7 +115,7 @@ class GPUCompatibleCPUDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      devices->push_back(new GPUCompatibleCPUDevice(
+      devices->push_back(absl::make_unique<GPUCompatibleCPUDevice>(
           options, name, Bytes(256 << 20), DeviceLocality(), cpu_allocator()));
     }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
index 75be6d60b8..58656ec757 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
@@ -33,7 +33,7 @@ namespace {
 
 TEST(GPUDeviceOnNonGPUMachineTest, CreateGPUDevicesOnNonGPUMachine) {
   SessionOptions opts;
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, "/job:localhost/replica:0/task:0", &devices));
   EXPECT_TRUE(devices.empty());
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
index 36294094e9..ae623b2adb 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
@@ -88,7 +88,7 @@ class GPUDeviceTest : public ::testing::Test {
 
 TEST_F(GPUDeviceTest, FailedToParseVisibleDeviceList) {
   SessionOptions opts = MakeSessionOptions("0,abc");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -97,7 +97,7 @@ TEST_F(GPUDeviceTest, FailedToParseVisibleDeviceList) {
 
 TEST_F(GPUDeviceTest, InvalidGpuId) {
   SessionOptions opts = MakeSessionOptions("100");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -107,7 +107,7 @@ TEST_F(GPUDeviceTest, InvalidGpuId) {
 
 TEST_F(GPUDeviceTest, DuplicateEntryInVisibleDeviceList) {
   SessionOptions opts = MakeSessionOptions("0,0");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -117,7 +117,7 @@ TEST_F(GPUDeviceTest, DuplicateEntryInVisibleDeviceList) {
 
 TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithMemoryFractionSettings) {
   SessionOptions opts = MakeSessionOptions("0", 0.1, 1, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -129,7 +129,7 @@ TEST_F(GPUDeviceTest, GpuDeviceCountTooSmall) {
   // device_count is 0, but with one entry in visible_device_list and one
   // (empty) VirtualDevices messages.
   SessionOptions opts = MakeSessionOptions("0", 0, 0, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::UNKNOWN);
@@ -141,7 +141,7 @@ TEST_F(GPUDeviceTest, NotEnoughGpuInVisibleDeviceList) {
   // Single entry in visible_device_list with two (empty) VirtualDevices
   // messages.
   SessionOptions opts = MakeSessionOptions("0", 0, 8, {{}, {}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::UNKNOWN);
@@ -155,7 +155,7 @@ TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithVisibleDeviceList) {
   // Three entries in visible_device_list with two (empty) VirtualDevices
   // messages.
   SessionOptions opts = MakeSessionOptions("0,1", 0, 8, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -169,39 +169,36 @@ TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithVisibleDeviceList) {
 TEST_F(GPUDeviceTest, EmptyVirtualDeviceConfig) {
   // It'll create single virtual device when the virtual device config is empty.
   SessionOptions opts = MakeSessionOptions("0");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_GE(devices[0]->attributes().memory_limit(), 0);
-  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, SingleVirtualDeviceWithNoMemoryLimit) {
   // It'll create single virtual device for the gpu in question when
   // memory_limit_mb is unset.
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_GE(devices[0]->attributes().memory_limit(), 0);
-  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, SingleVirtualDeviceWithMemoryLimit) {
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{123}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_EQ(123 << 20, devices[0]->attributes().memory_limit());
-  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, MultipleVirtualDevices) {
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{123, 456}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(2, devices.size());
@@ -219,7 +216,6 @@ TEST_F(GPUDeviceTest, MultipleVirtualDevices) {
             devices[1]->attributes().locality().links().link(0).type());
   EXPECT_EQ(BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength,
             devices[1]->attributes().locality().links().link(0).strength());
-  gtl::STLDeleteElements(&devices);
 }
 
 // Enabling unified memory on pre-Pascal GPUs results in an initialization
@@ -236,7 +232,7 @@ TEST_F(GPUDeviceTest, UnifiedMemoryUnavailableOnPrePascalGpus) {
   opts.config.mutable_gpu_options()
       ->mutable_experimental()
       ->set_use_unified_memory(true);
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INTERNAL);
@@ -259,7 +255,7 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
   }
 
   SessionOptions opts = MakeSessionOptions("0", kGpuMemoryFraction);
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_ASSERT_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   ASSERT_EQ(1, devices.size());
@@ -278,8 +274,6 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
                                      (memory_limit >> 20) << 20);
   EXPECT_NE(ptr, nullptr);
   allocator->DeallocateRaw(ptr);
-
-  gtl::STLDeleteElements(&devices);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
index 2144eea84f..f0656ff533 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h"
 
 #include <algorithm>
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/base_collective_executor.h"
 #include "tensorflow/core/common_runtime/collective_rma_local.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -217,7 +218,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
             << " num_devices_per_worker=" << num_devices_per_worker;
     int total_num_devices = num_workers * num_devices_per_worker;
     device_type_ = device_type;
-    std::vector<Device*> local_devices;
+    std::vector<std::unique_ptr<Device>> local_devices;
     SessionOptions sess_opts;
     sess_opts.env = Env::Default();
     Bytes mem_limit(4 << 20);
@@ -227,7 +228,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
         if (device_type == DEVICE_CPU) {
           string dev_name = strings::StrCat("/job:worker/replica:0/task:", wi,
                                             "/device:CPU:", di);
-          local_devices.push_back(new ThreadPoolDevice(
+          local_devices.push_back(absl::make_unique<ThreadPoolDevice>(
               sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator()));
         } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) {
           int dev_idx = (wi * num_devices_per_worker) + di;
@@ -235,7 +236,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
             LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more "
                          "than one ring node.";
           } else {
-            local_devices.push_back(gpu_devices_[dev_idx]);
+            local_devices.push_back(std::move(gpu_devices_[dev_idx]));
           }
         } else {
           LOG(FATAL) << "Unsupported device_type " << device_type;
@@ -243,7 +244,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
       }
     }
     if (!dev_mgr_ || device_type == DEVICE_CPU) {
-      dev_mgr_.reset(new DeviceMgr(local_devices));
+      dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
     }
     if (!gpu_ring_order_) gpu_ring_order_.reset(new string());
     dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get()));
@@ -714,7 +715,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
   std::unique_ptr<DeviceResolverLocal> dev_resolver_;
   std::vector<DeviceInstance*> instances_;
   CollectiveParams col_params_;
-  std::vector<tensorflow::Device*> gpu_devices_;
+  std::vector<std::unique_ptr<tensorflow::Device>> gpu_devices_;
   std::unique_ptr<tensorflow::DeviceMgr> dev_mgr_;
   std::unique_ptr<string> gpu_ring_order_;
   mutex mu_;
diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
index 1f585a8c24..bdd6c0e87d 100644
--- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
+++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
@@ -75,12 +75,12 @@ Benchmark::Benchmark(const string& device, Graph* g,
   const int graph_def_version = g->versions().producer();
 
   LocalExecutorParams params;
-  params.device = device_;
+  params.device = device_.get();
   params.function_library = nullptr;
   params.create_kernel = [this, graph_def_version](const NodeDef& ndef,
                                                    OpKernel** kernel) {
-    return CreateNonCachedKernel(device_, nullptr, ndef, graph_def_version,
-                                 kernel);
+    return CreateNonCachedKernel(device_.get(), nullptr, ndef,
+                                 graph_def_version, kernel);
   };
   params.delete_kernel = [](OpKernel* kernel) {
     DeleteNonCachedKernel(kernel);
@@ -107,7 +107,7 @@ Benchmark::~Benchmark() {
     // run kernel destructors that may attempt to access state borrowed from
     // `device_`, such as the resource manager.
     exec_.reset();
-    delete device_;
+    device_.reset();
     delete pool_;
   }
 }
diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.h b/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
index 555b43f655..b1557c50b0 100644
--- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
+++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
@@ -55,7 +55,7 @@ class Benchmark {
 
  private:
   thread::ThreadPool* pool_ = nullptr;
-  Device* device_ = nullptr;
+  std::unique_ptr<Device> device_ = nullptr;
   Rendezvous* rendez_ = nullptr;
   std::unique_ptr<Executor> exec_;
 
diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc
index 009f905f10..04e77e55f6 100644
--- a/tensorflow/core/common_runtime/placer_test.cc
+++ b/tensorflow/core/common_runtime/placer_test.cc
@@ -92,7 +92,7 @@ class FakeDevice : public Device {
 class DummyFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     return Status::OK();
   }
 };
diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
index cce2308011..21cb62118a 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
@@ -62,9 +62,12 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 2});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(options, "/job:a/replica:0/task:0",
-                                          &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+                                          &devices));
+    device0_ = devices[0].get();
+    device1_ = devices[1].get();
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
@@ -138,8 +141,9 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
     return Status::OK();
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
+  Device* device0_ = nullptr;  // Not owned. (Owned by device_mgr_.)
+  Device* device1_ = nullptr;  // Not owned. (Owned by device_mgr_.)
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<TestClusterFLR> cluster_flr_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> proc_flr_;
@@ -165,16 +169,16 @@ TEST_F(ProcessFunctionLibraryRuntimeTest, Basic) {
   FunctionLibraryRuntime* flr =
       proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[0]);
+  EXPECT_EQ(flr->device(), device0_);
   flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/device:CPU:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[0]);
+  EXPECT_EQ(flr->device(), device0_);
   flr = proc_flr_->GetFLR("/device:CPU:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[0]);
+  EXPECT_EQ(flr->device(), device0_);
   flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:1");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[1]);
+  EXPECT_EQ(flr->device(), device1_);
   flr = proc_flr_->GetFLR("abc");
   EXPECT_EQ(flr, nullptr);
   rendezvous_->Unref();
diff --git a/tensorflow/core/common_runtime/renamed_device.cc b/tensorflow/core/common_runtime/renamed_device.cc
index 56766a8df4..45541c35fe 100644
--- a/tensorflow/core/common_runtime/renamed_device.cc
+++ b/tensorflow/core/common_runtime/renamed_device.cc
@@ -14,15 +14,14 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/common_runtime/renamed_device.h"
+#include "absl/memory/memory.h"
 
 namespace tensorflow {
 
-// TODO(saeta): Convert to returning a std::unique_ptr?
 /* static */
-Device* RenamedDevice::NewRenamedDevice(const string& new_base,
-                                        Device* underlying,
-                                        bool owns_underlying,
-                                        bool isolate_session_state) {
+std::unique_ptr<Device> RenamedDevice::NewRenamedDevice(
+    const string& new_base, Device* underlying, bool owns_underlying,
+    bool isolate_session_state) {
   DeviceNameUtils::ParsedName parsed_name;
   CHECK(DeviceNameUtils::ParseFullName(new_base, &parsed_name));
   DeviceNameUtils::ParsedName underlying_parsed_name =
@@ -36,8 +35,9 @@ Device* RenamedDevice::NewRenamedDevice(const string& new_base,
                                           parsed_name.id);
   DeviceAttributes attributes(underlying->attributes());
   attributes.set_name(name);
-  return new RenamedDevice(underlying, attributes, owns_underlying,
-                           isolate_session_state);
+  // Call absl::WrapUnique to access private constructor.
+  return absl::WrapUnique(new RenamedDevice(
+      underlying, attributes, owns_underlying, isolate_session_state));
 }
 
 RenamedDevice::RenamedDevice(Device* underlying,
diff --git a/tensorflow/core/common_runtime/renamed_device.h b/tensorflow/core/common_runtime/renamed_device.h
index c00789a556..6d24f496ff 100644
--- a/tensorflow/core/common_runtime/renamed_device.h
+++ b/tensorflow/core/common_runtime/renamed_device.h
@@ -28,9 +28,10 @@ namespace tensorflow {
 // session.
 class RenamedDevice : public Device {
  public:
-  static Device* NewRenamedDevice(const string& new_base, Device* underlying,
-                                  bool owns_underlying,
-                                  bool isolate_session_state);
+  static std::unique_ptr<Device> NewRenamedDevice(const string& new_base,
+                                                  Device* underlying,
+                                                  bool owns_underlying,
+                                                  bool isolate_session_state);
 
   ~RenamedDevice() override;
 
diff --git a/tensorflow/core/common_runtime/ring_reducer_test.cc b/tensorflow/core/common_runtime/ring_reducer_test.cc
index a271bf7b74..7feb29a6db 100644
--- a/tensorflow/core/common_runtime/ring_reducer_test.cc
+++ b/tensorflow/core/common_runtime/ring_reducer_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/ring_reducer.h"
 
 #include <algorithm>
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/base_collective_executor.h"
 #include "tensorflow/core/common_runtime/collective_rma_local.h"
 #include "tensorflow/core/common_runtime/device.h"
@@ -157,7 +158,7 @@ class RingReducerTest : public ::testing::Test {
     InitGPUDevices();
 #endif
     device_type_ = device_type;
-    std::vector<Device*> local_devices;
+    std::vector<std::unique_ptr<Device>> local_devices;
     SessionOptions sess_opts;
     sess_opts.env = Env::Default();
     Bytes mem_limit(4 << 20);
@@ -167,7 +168,7 @@ class RingReducerTest : public ::testing::Test {
         if (device_type == DEVICE_CPU) {
           string dev_name =
               strings::StrCat("/job:worker/replica:0/task:", wi, "/cpu:", di);
-          local_devices.push_back(new ThreadPoolDevice(
+          local_devices.push_back(absl::make_unique<ThreadPoolDevice>(
               sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator()));
         } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) {
           int dev_idx = (wi * num_devices) + di;
@@ -175,7 +176,7 @@ class RingReducerTest : public ::testing::Test {
             LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more "
                          "than one ring node.";
           } else {
-            local_devices.push_back(gpu_devices_[dev_idx]);
+            local_devices.push_back(std::move(gpu_devices_[dev_idx]));
           }
         } else {
           LOG(FATAL) << "Unsupported device_type " << device_type;
@@ -185,7 +186,7 @@ class RingReducerTest : public ::testing::Test {
     if (!dev_mgr_ || device_type == DEVICE_CPU) {
       LOG(ERROR) << "resetting dev_mgr for " << local_devices.size()
                  << " devices: ";
-      dev_mgr_.reset(new DeviceMgr(local_devices));
+      dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
     }
     if (!gpu_ring_order_) gpu_ring_order_.reset(new string());
     dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get()));
@@ -544,7 +545,7 @@ class RingReducerTest : public ::testing::Test {
   std::unique_ptr<DeviceResolverLocal> dev_resolver_;
   std::vector<DeviceInstance*> instances_;
   CollectiveParams col_params_;
-  std::vector<tensorflow::Device*> gpu_devices_;
+  std::vector<std::unique_ptr<tensorflow::Device>> gpu_devices_;
   std::unique_ptr<tensorflow::DeviceMgr> dev_mgr_;
   std::unique_ptr<string> gpu_ring_order_;
   mutex mu_;
diff --git a/tensorflow/core/common_runtime/threadpool_device_factory.cc b/tensorflow/core/common_runtime/threadpool_device_factory.cc
index c06a4035a7..f9cbb81749 100644
--- a/tensorflow/core/common_runtime/threadpool_device_factory.cc
+++ b/tensorflow/core/common_runtime/threadpool_device_factory.cc
@@ -13,12 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// Register a factory that provides CPU devices.
-#include "tensorflow/core/common_runtime/threadpool_device.h"
-
 #include <vector>
+
+// Register a factory that provides CPU devices.
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/process_state.h"
+#include "tensorflow/core/common_runtime/threadpool_device.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/public/session_options.h"
@@ -29,7 +30,7 @@ namespace tensorflow {
 class ThreadPoolDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     int num_numa_nodes = port::NUMANumNodes();
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
@@ -38,7 +39,7 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      ThreadPoolDevice* tpd = nullptr;
+      std::unique_ptr<ThreadPoolDevice> tpd;
       if (options.config.experimental().use_numa_affinity()) {
         int numa_node = i % num_numa_nodes;
         if (numa_node != i) {
@@ -49,15 +50,15 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
         }
         DeviceLocality dev_locality;
         dev_locality.set_numa_node(numa_node);
-        tpd = new ThreadPoolDevice(
+        tpd = absl::make_unique<ThreadPoolDevice>(
             options, name, Bytes(256 << 20), dev_locality,
             ProcessState::singleton()->GetCPUAllocator(numa_node));
       } else {
-        tpd = new ThreadPoolDevice(
+        tpd = absl::make_unique<ThreadPoolDevice>(
             options, name, Bytes(256 << 20), DeviceLocality(),
             ProcessState::singleton()->GetCPUAllocator(port::kNUMANoAffinity));
       }
-      devices->push_back(tpd);
+      devices->push_back(std::move(tpd));
     }
 
     return Status::OK();
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 818324746f..a47da0d6a0 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -624,6 +624,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 4eed856759..40b18d321a 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -29,7 +29,8 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-static Device* NewDevice(const string& type, const string& name) {
+static std::unique_ptr<Device> NewDevice(const string& type,
+                                         const string& name) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -40,7 +41,7 @@ static Device* NewDevice(const string& type, const string& name) {
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(3);  // a non-default value
-  return new FakeDevice(attr);
+  return absl::make_unique<FakeDevice>(attr);
 }
 
 class FakeWorker : public TestWorkerInterface {
@@ -156,16 +157,16 @@ class DeviceResDistTest : public ::testing::Test {
 
   void DefineWorker(const ConfigProto& config, const string& worker_name,
                     const string& device_type, int num_devices) {
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i)));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(devices);
+    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : devices) {
+    for (auto* d : dev_mgr->ListDevices()) {
       dv->push_back(d->name());
     }
     DeviceResolverDistributed* dev_res =
diff --git a/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
index 33e1c8f2c3..26f722a6bd 100644
--- a/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
@@ -41,7 +41,8 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-static Device* NewDevice(const string& type, const string& name) {
+static std::unique_ptr<Device> NewDevice(const string& type,
+                                         const string& name) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -52,7 +53,7 @@ static Device* NewDevice(const string& type, const string& name) {
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(3);  // a non-default value
-  return new FakeDevice(attr);
+  return absl::make_unique<FakeDevice>(attr);
 }
 
 static int64 kStepId = 123;
@@ -211,16 +212,16 @@ class CollRMADistTest : public ::testing::Test {
 
   void DefineWorker(const ConfigProto& config, const string& worker_name,
                     const string& device_type, int num_devices) {
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i)));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(devices);
+    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : devices) {
+    for (auto d : dev_mgr->ListDevices()) {
       dv->push_back(d->name());
     }
     DeviceResolverDistributed* dev_res =
diff --git a/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
index ae44b98bd5..842a2b3b05 100644
--- a/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/distributed_runtime/test_utils.h"
 #include "tensorflow/core/lib/core/notification.h"
@@ -41,8 +42,8 @@ class TestableDeviceResolverDistributed : public DeviceResolverDistributed {
 
 // Create a fake 'Device' whose only interesting attribute is a non-default
 // DeviceLocality.
-static Device* NewDevice(const string& type, const string& name,
-                         int numa_node) {
+static std::unique_ptr<Device> NewDevice(const string& type, const string& name,
+                                         int numa_node) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -53,7 +54,7 @@ static Device* NewDevice(const string& type, const string& name,
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(numa_node);
-  return new FakeDevice(attr);
+  return absl::make_unique<FakeDevice>(attr);
 }
 
 // Create a fake WorkerInterface that responds to requests without RPCs,
@@ -151,19 +152,19 @@ class DeviceResDistTest : public ::testing::Test {
 
   void DefineWorker(const string& worker_name, const string& device_type,
                     int num_devices) {
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i), i));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(devices);
+    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
     TestableDeviceResolverDistributed* dev_res =
         new TestableDeviceResolverDistributed(dev_mgr, &wc_, worker_name);
     resolvers_[worker_name] = dev_res;
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : devices) {
+    for (auto* d : dev_mgr->ListDevices()) {
       dv->push_back(d->name());
     }
     FakeWorker* fw = new FakeWorker(worker_name, dev_mgr, dev_res);
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
index 5b0a420fad..c66466c0a6 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
@@ -87,7 +87,7 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
     return tensorflow::errors::Internal(
         "invalid eager env_ or env_->rendezvous_mgr.");
   }
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
       // TODO(nareshmodi): Correctly set the SessionOptions.
@@ -97,12 +97,12 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
                       request->server_def().task_index()),
       &devices));
   response->mutable_device_attributes()->Reserve(devices.size());
-  for (auto& d : devices) {
+  for (const auto& d : devices) {
     *response->add_device_attributes() = d->attributes();
   }
 
   std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(devices));
+      new tensorflow::DeviceMgr(std::move(devices)));
 
   auto* r = env_->rendezvous_mgr->Find(request->rendezvous_id());
   auto session_name = strings::StrCat("eager_", request->rendezvous_id());
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
index 5ba522c2a2..7a1463e8f0 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
@@ -68,12 +68,9 @@ class EagerServiceImplTest : public ::testing::Test {
     worker_env_.rendezvous_mgr = &rendezvous_mgr_;
     worker_env_.session_mgr = session_mgr_.get();
 
-    Device* device = DeviceFactory::NewDevice(
-        "CPU", {}, "/job:localhost/replica:0/task:0/device:CPU:0");
-
-    worker_env_.local_devices = {device};
-
-    device_mgr_.reset(new DeviceMgr(worker_env_.local_devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(DeviceFactory::NewDevice(
+        "CPU", {}, "/job:localhost/replica:0/task:0/device:CPU:0"));
+    worker_env_.local_devices = device_mgr_->ListDevices();
     worker_env_.device_mgr = device_mgr_.get();
   }
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
index ae722fdfe9..cbd5cd927e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <vector>
 
 #include "grpc/support/alloc.h"
 #include "grpcpp/grpcpp.h"
@@ -156,10 +157,12 @@ Status GrpcServer::Init(
   string name_prefix =
       strings::StrCat("/job:", server_def_.job_name(), "/replica:0",
                       "/task:", server_def_.task_index());
-  TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(sess_opts, name_prefix,
-                                               &master_env_.local_devices));
-  worker_env_.local_devices = master_env_.local_devices;
-  worker_env_.device_mgr = new DeviceMgr(worker_env_.local_devices);
+  std::vector<std::unique_ptr<Device>> devices;
+  TF_RETURN_IF_ERROR(
+      DeviceFactory::AddDevices(sess_opts, name_prefix, &devices));
+  worker_env_.device_mgr = new DeviceMgr(std::move(devices));
+  master_env_.local_devices = worker_env_.device_mgr->ListDevices();
+  worker_env_.local_devices = worker_env_.device_mgr->ListDevices();
   worker_env_.rendezvous_mgr = rendezvous_mgr_func == nullptr
                                    ? new RpcRendezvousMgr(&worker_env_)
                                    : rendezvous_mgr_func(&worker_env_);
diff --git a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
index 0323300fdd..1c87fe9d92 100644
--- a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
@@ -42,8 +42,9 @@ class RpcCollectiveExecutorMgrTest : public ::testing::Test {
     WorkerCacheInterface* worker_cache = nullptr;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     std::unique_ptr<DeviceResolverDistributed> dr(new DeviceResolverDistributed(
         device_mgr_.get(), worker_cache, task_name));
     std::unique_ptr<CollectiveParamResolverDistributed> cpr(
@@ -57,7 +58,6 @@ class RpcCollectiveExecutorMgrTest : public ::testing::Test {
   }
 
   std::unique_ptr<RpcCollectiveExecutorMgr> cme_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
 };
 
diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc
index 38833bd202..29fe767e42 100644
--- a/tensorflow/core/distributed_runtime/session_mgr.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr.cc
@@ -78,13 +78,13 @@ Status SessionMgr::CreateSession(const string& session,
 
   if (isolate_session_state) {
     // Create a private copy of the DeviceMgr for the WorkerSession.
-    std::vector<Device*> renamed_devices;
+    std::vector<std::unique_ptr<Device>> renamed_devices;
     for (Device* d : worker_env_->local_devices) {
       renamed_devices.push_back(RenamedDevice::NewRenamedDevice(
           worker_name, d, false, isolate_session_state));
     }
 
-    auto device_mgr = MakeUnique<DeviceMgr>(renamed_devices);
+    auto device_mgr = MakeUnique<DeviceMgr>(std::move(renamed_devices));
     auto graph_mgr = MakeUnique<GraphMgr>(worker_env_, device_mgr.get());
     worker_session.reset(
         new WorkerSession(session, worker_name,
diff --git a/tensorflow/core/distributed_runtime/session_mgr_test.cc b/tensorflow/core/distributed_runtime/session_mgr_test.cc
index 99192119a6..1ab0d20f0b 100644
--- a/tensorflow/core/distributed_runtime/session_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr_test.cc
@@ -46,11 +46,9 @@ class SessionMgrTest : public ::testing::Test {
   SessionMgrTest()
       : mgr_(&env_, "/job:mnist/replica:0/task:0",
              std::unique_ptr<WorkerCacheInterface>(), factory_) {
-    Device* device =
-        FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0")
-            .release();
-    env_.local_devices = {device};
-    device_mgr_.reset(new DeviceMgr(env_.local_devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(
+        FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0"));
+    env_.local_devices = device_mgr_->ListDevices();
     env_.device_mgr = device_mgr_.get();
   }
 
diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index cf99f4908b..e69dfa79d1 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -102,10 +102,11 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   }
 
   // Instantiate all variables for function library runtime creation.
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
-  std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(devices));
+  Device* cpu_device = devices[0].get();
+  std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(std::move(devices)));
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              graph_def.library());
   Env* env = Env::Default();
@@ -124,7 +125,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
       new ProcessFunctionLibraryRuntime(dvc_mgr.get(), env,
                                         graph_def.versions().producer(),
                                         &function_library, *optimizer_opts));
-  FunctionLibraryRuntime* flr = pflr->GetFLR(devices[0]->name());
+  FunctionLibraryRuntime* flr = pflr->GetFLR(cpu_device->name());
 
   // Create the GraphOptimizer to optimize the graph def.
   GraphConstructorOptions graph_ctor_opts;
@@ -137,7 +138,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
 
   // Optimize the graph.
   ::tensorflow::GraphOptimizer optimizer(*optimizer_opts);
-  optimizer.Optimize(flr, env, devices[0], &graphptr, /*shape_map=*/nullptr);
+  optimizer.Optimize(flr, env, cpu_device, &graphptr, /*shape_map=*/nullptr);
   graphptr->ToGraphDef(output_graph_def);
 
   // The default values of attributes might have been stripped by the optimizer.
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index b6f989f2c9..8e6629565a 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -142,7 +142,6 @@ cc_library(
         ":graph_optimizer",
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -150,6 +149,7 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
+        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index f99826ddca..f8ddbeb659 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -16,7 +16,9 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 
 #include <unordered_map>
+#include <vector>
 
+#include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -343,14 +345,15 @@ class FunctionOptimizerContext {
       DeviceAttributes attr;
       attr.set_name("/device:CPU:0");
       attr.set_device_type("CPU");
-      Device* device = new FakeCPUDevice(env, attr);
-      device_mgr_.reset(new DeviceMgr({device}));
+      std::vector<std::unique_ptr<Device>> devices;
+      devices.push_back(absl::make_unique<FakeCPUDevice>(env, attr));
+      device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
       OptimizerOptions optimizer_opts;
       optimizer_opts.set_do_function_inlining(true);
       process_flr_.reset(new ProcessFunctionLibraryRuntime(
           device_mgr_.get(), env, graph_version_, &function_library_,
           optimizer_opts));
-      flr_ = process_flr_->GetFLR(device->name());
+      flr_ = process_flr_->GetFLR(device_mgr_->ListDevices()[0]->name());
     }
   }
 
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 7192684e2d..dcb6975669 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -600,6 +600,7 @@ tf_kernel_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:session_options",
         "//tensorflow/core/kernels:ops_util",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 93999dc095..98b67454d5 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/kernels/data/iterator_ops.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/common_runtime/renamed_device.h"
 #include "tensorflow/core/common_runtime/threadpool_device.h"
@@ -545,10 +546,9 @@ FunctionLibraryRuntime* IteratorHandleOp::CreatePrivateFLR(
   // in its resource manager. The existing device will outlive the
   // IteratorResource, because we are storing the IteratorResource
   // in that device's resource manager.
-  Device* wrapped_device = RenamedDevice::NewRenamedDevice(
+  *device_mgr = absl::make_unique<DeviceMgr>(RenamedDevice::NewRenamedDevice(
       ctx->device()->name(), down_cast<Device*>(ctx->device()),
-      false /* owns_underlying */, false /* isolate_session_state */);
-  device_mgr->reset(new DeviceMgr({wrapped_device}));
+      false /* owns_underlying */, false /* isolate_session_state */));
   flib_def->reset(new FunctionLibraryDefinition(
       *ctx->function_library()->GetFunctionLibraryDefinition()));
   pflr->reset(new ProcessFunctionLibraryRuntime(
diff --git a/tensorflow/core/kernels/data/single_threaded_executor_test.cc b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
index 6244e287bb..7bb51fb8b5 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor_test.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
@@ -51,17 +51,17 @@ class ExecutorTest : public ::testing::Test {
     // when the test completes.
     CHECK(rendez_->Unref());
     delete exec_;
-    delete device_;
   }
 
   // Resets executor_ with a new executor based on a graph 'gdef'.
   void Create(std::unique_ptr<const Graph> graph) {
     const int version = graph->versions().producer();
     LocalExecutorParams params;
-    params.device = device_;
+    params.device = device_.get();
     params.create_kernel = [this, version](const NodeDef& ndef,
                                            OpKernel** kernel) {
-      return CreateNonCachedKernel(device_, nullptr, ndef, version, kernel);
+      return CreateNonCachedKernel(device_.get(), nullptr, ndef, version,
+                                   kernel);
     };
     params.delete_kernel = [](OpKernel* kernel) {
       DeleteNonCachedKernel(kernel);
@@ -86,7 +86,7 @@ class ExecutorTest : public ::testing::Test {
     return exec_->Run(args);
   }
 
-  Device* device_ = nullptr;
+  std::unique_ptr<Device> device_;
   Executor* exec_ = nullptr;
   Executor::Args::Runner runner_;
   Rendezvous* rendez_ = nullptr;
diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD
index 222a043a88..63e86899da 100644
--- a/tensorflow/lite/delegates/flex/BUILD
+++ b/tensorflow/lite/delegates/flex/BUILD
@@ -116,6 +116,7 @@ cc_library(
     hdrs = ["delegate_data.h"],
     deps = [
         ":buffer_map",
+        "@com_google_absl//absl/memory",
         "//tensorflow/core/common_runtime/eager:context",
     ] + select({
         "//tensorflow:android": [
diff --git a/tensorflow/lite/delegates/flex/delegate_data.cc b/tensorflow/lite/delegates/flex/delegate_data.cc
index b62479a448..1483a53038 100644
--- a/tensorflow/lite/delegates/flex/delegate_data.cc
+++ b/tensorflow/lite/delegates/flex/delegate_data.cc
@@ -14,20 +14,21 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/delegates/flex/delegate_data.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tflite {
 namespace flex {
 tensorflow::Status DelegateData::Create(std::unique_ptr<DelegateData>* data) {
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
       tensorflow::SessionOptions(), "/job:localhost/replica:0/task:0",
       &devices));
 
-  std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(devices));
+  std::unique_ptr<tensorflow::DeviceMgr> device_mgr =
+      absl::make_unique<tensorflow::DeviceMgr>(std::move(devices));
   // Note that Rendezvous is ref-counted so it will be automatically deleted.
   tensorflow::Rendezvous* rendezvous =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 4c3a0717e7..dfeaebca26 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2012,13 +2012,13 @@ bool InlineAllFunctions(GraphDef* graphdef) {
   tensorflow::SessionOptions options;
   auto* device_count = options.config.mutable_device_count();
   device_count->insert({"CPU", 1});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   TF_CHECK_OK(tensorflow::DeviceFactory::AddDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
 
   tensorflow::FunctionLibraryDefinition fld(tensorflow::OpRegistry::Global(),
                                             graphdef_copy.library());
-  tensorflow::DeviceMgr device_mgr(devices);
+  tensorflow::DeviceMgr device_mgr(std::move(devices));
   tensorflow::OptimizerOptions o_opts;
   tensorflow::ProcessFunctionLibraryRuntime pflr(
       &device_mgr, tensorflow::Env::Default(), TF_GRAPH_DEF_VERSION, &fld,
diff --git a/tensorflow/python/client/device_lib.i b/tensorflow/python/client/device_lib.i
index 944e855cee..3e579152d5 100644
--- a/tensorflow/python/client/device_lib.i
+++ b/tensorflow/python/client/device_lib.i
@@ -48,17 +48,14 @@ static std::vector<string> ListDevicesWithSessionConfig(
   std::vector<string> output;
   SessionOptions options;
   options.config = config;
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::AddDevices(
       options, "" /* name_prefix */, &devices);
   if (!status.ok()) {
     Set_TF_Status_from_Status(out_status, status);
   }
 
-  std::vector<std::unique_ptr<Device>> device_holder(devices.begin(),
-                                                     devices.end());
-
-  for (const Device* device : devices) {
+  for (const std::unique_ptr<Device>& device : devices) {
     const DeviceAttributes& attr = device->attributes();
     string attr_serialized;
     if (!attr.SerializeToString(&attr_serialized)) {
diff --git a/tensorflow/python/grappler/tf_optimizer.i b/tensorflow/python/grappler/tf_optimizer.i
index daa5bc9444..b746c3ec26 100644
--- a/tensorflow/python/grappler/tf_optimizer.i
+++ b/tensorflow/python/grappler/tf_optimizer.i
@@ -74,13 +74,13 @@ limitations under the License.
 
 void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* device_map) {
   tensorflow::SessionOptions options;
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   tensorflow::Status status = tensorflow::DeviceFactory::AddDevices(options, "", &devices);
   if (!status.ok()) {
     return;
   }
 
-  for (const tensorflow::Device* device : devices) {
+  for (const std::unique_ptr<tensorflow::Device>& device : devices) {
     tensorflow::DeviceProperties& prop = (*device_map)[device->name()];
     prop = tensorflow::grappler::GetDeviceInfo(device->parsed_name());
 
@@ -88,7 +88,6 @@ void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* dev
     // available device memory.
     const tensorflow::DeviceAttributes& attr = device->attributes();
     prop.set_memory_size(attr.memory_limit());
-    delete device;
   }
 }
 
-- 
GitLab


From 50e5b015be1ead47f170013a640bf3028bb0560f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Nov 2018 14:53:40 -0800
Subject: [PATCH 0743/1554] Automated rollback of commit
 809ed3c835403564333bfdc06fca512432db4ca1

PiperOrigin-RevId: 222646553
---
 tensorflow/c/eager/BUILD                      |  1 -
 tensorflow/c/eager/c_api.cc                   | 11 +++---
 .../compiler/jit/build_xla_ops_pass_test.cc   |  8 +++-
 .../compiler/jit/create_xla_launch_op_test.cc |  6 +--
 .../mark_for_compilation_pass_test_helper.cc  |  8 +++-
 .../jit/partially_decluster_pass_test.cc      |  6 ++-
 tensorflow/compiler/jit/xla_cpu_device.cc     | 11 +++---
 tensorflow/compiler/jit/xla_gpu_device.cc     | 10 ++---
 .../compiler/jit/xla_interpreter_device.cc    |  7 ++--
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  2 +-
 tensorflow/core/BUILD                         |  4 --
 .../collective_executor_mgr_test.cc           |  6 +--
 .../collective_param_resolver_local_test.cc   |  6 +--
 .../collective_rma_local_test.cc              |  6 +--
 .../core/common_runtime/device_factory.cc     | 17 ++++-----
 .../core/common_runtime/device_factory.h      | 13 +++----
 tensorflow/core/common_runtime/device_mgr.cc  | 37 ++++++++-----------
 tensorflow/core/common_runtime/device_mgr.h   | 15 ++++----
 .../device_resolver_local_test.cc             |  6 +--
 .../core/common_runtime/device_set_test.cc    |  2 +-
 .../core/common_runtime/direct_session.cc     |  4 +-
 tensorflow/core/common_runtime/eager/BUILD    |  1 -
 .../eager/kernel_and_device_test.cc           | 14 +++----
 .../core/common_runtime/executor_test.cc      |  8 ++--
 .../core/common_runtime/function_test.cc      |  7 ++--
 .../function_threadpool_test.cc               |  6 +--
 .../core/common_runtime/gpu/gpu_device.cc     | 20 +++++-----
 .../core/common_runtime/gpu/gpu_device.h      | 18 +++++----
 .../common_runtime/gpu/gpu_device_factory.cc  | 21 ++++++-----
 .../gpu/gpu_device_on_non_gpu_machine_test.cc |  2 +-
 .../common_runtime/gpu/gpu_device_test.cc     | 32 +++++++++-------
 .../hierarchical_tree_broadcaster_test.cc     | 11 +++---
 .../kernel_benchmark_testlib.cc               |  8 ++--
 .../common_runtime/kernel_benchmark_testlib.h |  2 +-
 tensorflow/core/common_runtime/placer_test.cc |  2 +-
 .../process_function_library_runtime_test.cc  | 18 ++++-----
 .../core/common_runtime/renamed_device.cc     | 14 +++----
 .../core/common_runtime/renamed_device.h      |  7 ++--
 .../core/common_runtime/ring_reducer_test.cc  | 11 +++---
 .../threadpool_device_factory.cc              | 17 ++++-----
 tensorflow/core/distributed_runtime/BUILD     |  1 -
 ...lective_param_resolver_distributed_test.cc | 11 +++---
 .../collective_rma_distributed_test.cc        | 11 +++---
 .../device_resolver_distributed_test.cc       | 13 +++----
 .../eager/eager_service_impl.cc               |  6 +--
 .../eager/eager_service_impl_test.cc          |  9 +++--
 .../rpc/grpc_server_lib.cc                    | 11 ++----
 .../rpc_collective_executor_mgr_test.cc       |  6 +--
 .../core/distributed_runtime/session_mgr.cc   |  4 +-
 .../distributed_runtime/session_mgr_test.cc   |  8 ++--
 .../core/grappler/grappler_item_builder.cc    |  9 ++---
 tensorflow/core/grappler/optimizers/BUILD     |  2 +-
 .../grappler/optimizers/function_optimizer.cc |  9 ++---
 tensorflow/core/kernels/data/BUILD            |  1 -
 tensorflow/core/kernels/data/iterator_ops.cc  |  6 +--
 .../data/single_threaded_executor_test.cc     |  8 ++--
 tensorflow/lite/delegates/flex/BUILD          |  1 -
 .../lite/delegates/flex/delegate_data.cc      |  7 ++--
 tensorflow/lite/toco/import_tensorflow.cc     |  4 +-
 tensorflow/python/client/device_lib.i         |  7 +++-
 tensorflow/python/grappler/tf_optimizer.i     |  5 ++-
 61 files changed, 271 insertions(+), 273 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 5a0988ed31..ba3d8533db 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -50,7 +50,6 @@ tf_cuda_library(
         ],
         "//conditions:default": [],
     }) + [
-        "@com_google_absl//absl/memory",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/distributed_runtime/eager:eager_client",
         "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index c9e730ef41..192044915f 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include <string>
 #include <vector>
 
-#include "absl/memory/memory.h"
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/c_api_internal.h"
@@ -81,7 +80,7 @@ tensorflow::Status GetAllRemoteDevices(
     const std::vector<string>& remote_workers,
     tensorflow::WorkerCacheInterface* worker_cache,
     std::unique_ptr<tensorflow::DeviceMgr>* device_mgr) {
-  std::vector<std::unique_ptr<tensorflow::Device>> remote_devices;
+  std::vector<tensorflow::Device*> remote_devices;
   tensorflow::Status status;
   // TODO(nareshmodi) do this in parallel instead of serially.
   for (const string& remote_worker : remote_workers) {
@@ -94,7 +93,7 @@ tensorflow::Status GetAllRemoteDevices(
           status = s;
           if (s.ok()) {
             for (tensorflow::Device* d : *devices) {
-              remote_devices.emplace_back(d);
+              remote_devices.push_back(d);
             }
           }
           n.Notify();
@@ -102,7 +101,7 @@ tensorflow::Status GetAllRemoteDevices(
     n.WaitForNotification();
   }
   std::unique_ptr<tensorflow::DeviceMgr> remote_device_mgr(
-      new tensorflow::DeviceMgr(std::move(remote_devices)));
+      new tensorflow::DeviceMgr(remote_devices));
 
   TF_RETURN_IF_ERROR(status);
 
@@ -263,13 +262,13 @@ TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx,
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
 
 TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
-  std::vector<std::unique_ptr<tensorflow::Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   status->status = tensorflow::DeviceFactory::AddDevices(
       opts->session_options.options, "/job:localhost/replica:0/task:0",
       &devices);
   if (!status->status.ok()) return nullptr;
   std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(std::move(devices)));
+      new tensorflow::DeviceMgr(devices));
 
   tensorflow::Rendezvous* r =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
index 48a23a4c17..11df946cc1 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
@@ -42,8 +42,14 @@ class BuildXlaOpsTest : public ::testing::Test {
               .ok());
   }
 
+  void TearDown() override {
+    for (Device* device : devices_) {
+      delete device;
+    }
+  }
+
  private:
-  std::vector<std::unique_ptr<Device>> devices_;
+  std::vector<Device*> devices_;
 };
 
 using ::tensorflow::testing::FindNodeByName;
diff --git a/tensorflow/compiler/jit/create_xla_launch_op_test.cc b/tensorflow/compiler/jit/create_xla_launch_op_test.cc
index 0f872a480f..7386660762 100644
--- a/tensorflow/compiler/jit/create_xla_launch_op_test.cc
+++ b/tensorflow/compiler/jit/create_xla_launch_op_test.cc
@@ -59,9 +59,8 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 1});
-    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices));
+        options, "/job:localhost/replica:0/task:0", &devices_));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) {
@@ -70,7 +69,7 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
     lib_def_ = absl::make_unique<FunctionLibraryDefinition>(
         OpRegistry::Global(), proto);
     OptimizerOptions opts;
-    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(devices_);
     pflr_ = absl::make_unique<ProcessFunctionLibraryRuntime>(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
@@ -78,6 +77,7 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
   }
 
   FunctionLibraryRuntime* flr_;
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
index 64a3301745..d56d0f8ccf 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
@@ -34,9 +34,15 @@ namespace tensorflow {
   //
   // It may be worth refactoring out XlaOpRegistry::RegisterCompilationDevice to
   // make this more direct, but probably not worth it solely for this test.
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<Device*> devices;
   TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(*session_options, "", &devices));
 
+  auto delete_devices = gtl::MakeCleanup([&] {
+    for (Device* d : devices) {
+      delete d;
+    }
+  });
+
   GraphOptimizationPassOptions opt_options;
   opt_options.graph = graph;
   opt_options.session_options = session_options;
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index 38a54cc5ef..1fc5da5071 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -386,7 +386,7 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   TF_ASSERT_OK(s.ToGraph(graph.get()));
 
   // This is needed to register the XLA_GPU device.
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<Device*> devices;
   TF_ASSERT_OK(DeviceFactory::AddDevices(
       SessionOptions(), "/job:localhost/replica:0/task:0", &devices));
 
@@ -400,6 +400,10 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   TF_ASSERT_OK(PartiallyDecluster(&graph));
 
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0");
+
+  for (Device* d : devices) {
+    delete d;
+  }
 }
 
 TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) {
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 7df898ad12..9006dd514b 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -31,12 +31,12 @@ namespace tensorflow {
 class XlaCpuDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override;
+                       std::vector<Device*>* devices) override;
 };
 
-Status XlaCpuDeviceFactory::CreateDevices(
-    const SessionOptions& session_options, const string& name_prefix,
-    std::vector<std::unique_ptr<Device>>* devices) {
+Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
+                                          const string& name_prefix,
+                                          std::vector<Device*>* devices) {
   XlaDeviceFlags* flags = GetXlaDeviceFlags();
   bool compile_on_demand = flags->tf_xla_compile_on_demand;
 
@@ -63,7 +63,8 @@ Status XlaCpuDeviceFactory::CreateDevices(
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_CPU_XLA_JIT;
   options.use_multiple_streams = false;
-  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
+  auto device = absl::make_unique<XlaDevice>(session_options, options);
+  devices->push_back(device.release());
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 944f732b99..4419701695 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -29,12 +29,12 @@ namespace tensorflow {
 class XlaGpuDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override;
+                       std::vector<Device*>* devices) override;
 };
 
-Status XlaGpuDeviceFactory::CreateDevices(
-    const SessionOptions& session_options, const string& name_prefix,
-    std::vector<std::unique_ptr<Device>>* devices) {
+Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
+                                          const string& name_prefix,
+                                          std::vector<Device*>* devices) {
   XlaOpRegistry::DeviceRegistration registration;
   registration.compilation_device_name = DEVICE_GPU_XLA_JIT;
   registration.autoclustering_policy =
@@ -70,7 +70,7 @@ Status XlaGpuDeviceFactory::CreateDevices(
       return status;
     }
 
-    devices->push_back(std::move(device));
+    devices->push_back(device.release());
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index 4007309ed1..e828bae865 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -33,12 +33,12 @@ constexpr std::array<DataType, 9> kExecAllTypes = {
 class XlaInterpreterDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override;
+                       std::vector<Device*>* devices) override;
 };
 
 Status XlaInterpreterDeviceFactory::CreateDevices(
     const SessionOptions& session_options, const string& name_prefix,
-    std::vector<std::unique_ptr<Device>>* devices) {
+    std::vector<Device*>* devices) {
   static XlaDeviceOpRegistrations* registrations = RegisterXlaDeviceKernels(
       DEVICE_XLA_INTERPRETER, DEVICE_INTERPRETER_XLA_JIT);
   (void)registrations;
@@ -61,7 +61,8 @@ Status XlaInterpreterDeviceFactory::CreateDevices(
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT;
   options.use_multiple_streams = false;
-  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
+  auto device = absl::make_unique<XlaDevice>(session_options, options);
+  devices->push_back(device.release());
 
   return Status::OK();
 }
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 2230bfd7d9..8036bc6844 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -380,7 +380,7 @@ XlaCompiler::XlaCompiler(XlaCompiler::Options options)
       initialization_status_(Status::OK()),
       next_step_id_(1),
       device_(new XlaCompilationDevice(SessionOptions(), options_.device_type)),
-      device_mgr_(absl::WrapUnique(device_)) {
+      device_mgr_({device_}) {
   CHECK(!options_.device_type.type_string().empty());
   if (options_.populate_resource_manager) {
     initialization_status_ =
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1b4371198a..2a8c2718ed 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2963,7 +2963,6 @@ tf_cuda_library(
         ":lib_internal",
         ":proto_text",
         ":protos_all_cc",
-        "@com_google_absl//absl/memory",
         "//third_party/eigen3",
         "//tensorflow/core/grappler:grappler_item",
     ] + mkl_deps(),
@@ -3817,7 +3816,6 @@ tf_cc_tests_gpu(
         ":test",
         ":test_main",
         ":testlib",
-        "@com_google_absl//absl/memory",
     ],
 )
 
@@ -3846,7 +3844,6 @@ tf_cc_tests_gpu(
         ":test",
         ":test_main",
         ":testlib",
-        "@com_google_absl//absl/memory",
     ],
 )
 
@@ -4414,7 +4411,6 @@ tf_cc_test(
         "//tensorflow/core/kernels:random_ops",
         "//tensorflow/core/kernels:shape_ops",
         "//third_party/eigen3",
-        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
index f3d86aa633..91994c5731 100644
--- a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
+++ b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
@@ -38,9 +38,8 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
     auto* device_count = options.config.mutable_device_count();
     string task_name = "/job:localhost/replica:0/task:0";
     device_count->insert({"CPU", NUM_DEVS});
-    std::vector<std::unique_ptr<Device>> devices;
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
     std::unique_ptr<DeviceResolverInterface> drl(
         new DeviceResolverLocal(device_mgr_.get()));
     std::unique_ptr<ParamResolverInterface> prl(
@@ -51,6 +50,7 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
   }
 
   std::unique_ptr<CollectiveExecutorMgr> cme_;
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
 };
 
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
index 94d889c40d..9a501b3298 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
@@ -37,9 +37,8 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     string task_name = "/job:localhost/replica:0/task:0";
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    std::vector<std::unique_ptr<Device>> devices;
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
     prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
                                                 task_name));
@@ -74,6 +73,7 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     }
   }
 
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
   std::unique_ptr<CollectiveParamResolverLocal> prl_;
diff --git a/tensorflow/core/common_runtime/collective_rma_local_test.cc b/tensorflow/core/common_runtime/collective_rma_local_test.cc
index 4263f3a4ad..a931fe64bd 100644
--- a/tensorflow/core/common_runtime/collective_rma_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_rma_local_test.cc
@@ -42,9 +42,8 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    std::vector<std::unique_ptr<Device>> devices;
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices));
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
     prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
                                                 kTaskName));
@@ -52,6 +51,7 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
                                                kStepId));
   }
 
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
   std::unique_ptr<CollectiveParamResolverLocal> prl_;
diff --git a/tensorflow/core/common_runtime/device_factory.cc b/tensorflow/core/common_runtime/device_factory.cc
index 0fad13fe1e..b94900114c 100644
--- a/tensorflow/core/common_runtime/device_factory.cc
+++ b/tensorflow/core/common_runtime/device_factory.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
-#include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
@@ -90,9 +89,9 @@ DeviceFactory* DeviceFactory::GetFactory(const string& device_type) {
   return it->second.factory.get();
 }
 
-Status DeviceFactory::AddDevices(
-    const SessionOptions& options, const string& name_prefix,
-    std::vector<std::unique_ptr<Device>>* devices) {
+Status DeviceFactory::AddDevices(const SessionOptions& options,
+                                 const string& name_prefix,
+                                 std::vector<Device*>* devices) {
   // CPU first. A CPU device is required.
   auto cpu_factory = GetFactory("CPU");
   if (!cpu_factory) {
@@ -117,16 +116,16 @@ Status DeviceFactory::AddDevices(
   return Status::OK();
 }
 
-std::unique_ptr<Device> DeviceFactory::NewDevice(const string& type,
-                                                 const SessionOptions& options,
-                                                 const string& name_prefix) {
+Device* DeviceFactory::NewDevice(const string& type,
+                                 const SessionOptions& options,
+                                 const string& name_prefix) {
   auto device_factory = GetFactory(type);
   if (!device_factory) {
     return nullptr;
   }
   SessionOptions opt = options;
   (*opt.config.mutable_device_count())[type] = 1;
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<Device*> devices;
   TF_CHECK_OK(device_factory->CreateDevices(opt, name_prefix, &devices));
   int expected_num_devices = 1;
   auto iter = options.config.device_count().find(type);
@@ -134,7 +133,7 @@ std::unique_ptr<Device> DeviceFactory::NewDevice(const string& type,
     expected_num_devices = iter->second;
   }
   DCHECK_EQ(devices.size(), static_cast<size_t>(expected_num_devices));
-  return std::move(devices[0]);
+  return devices[0];
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/device_factory.h b/tensorflow/core/common_runtime/device_factory.h
index b3cd7adca9..db50226fe8 100644
--- a/tensorflow/core/common_runtime/device_factory.h
+++ b/tensorflow/core/common_runtime/device_factory.h
@@ -40,19 +40,18 @@ class DeviceFactory {
   // CPU devices are added first.
   static Status AddDevices(const SessionOptions& options,
                            const string& name_prefix,
-                           std::vector<std::unique_ptr<Device>>* devices);
+                           std::vector<Device*>* devices);
 
   // Helper for tests.  Create a single device of type "type".  The
   // returned device is always numbered zero, so if creating multiple
   // devices of the same type, supply distinct name_prefix arguments.
-  static std::unique_ptr<Device> NewDevice(const string& type,
-                                           const SessionOptions& options,
-                                           const string& name_prefix);
+  static Device* NewDevice(const string& type, const SessionOptions& options,
+                           const string& name_prefix);
 
   // Most clients should call AddDevices() instead.
-  virtual Status CreateDevices(
-      const SessionOptions& options, const string& name_prefix,
-      std::vector<std::unique_ptr<Device>>* devices) = 0;
+  virtual Status CreateDevices(const SessionOptions& options,
+                               const string& name_prefix,
+                               std::vector<Device*>* devices) = 0;
 
   // Return the device priority number for a "device_type" string.
   //
diff --git a/tensorflow/core/common_runtime/device_mgr.cc b/tensorflow/core/common_runtime/device_mgr.cc
index 1f7d7c4699..470abc1431 100644
--- a/tensorflow/core/common_runtime/device_mgr.cc
+++ b/tensorflow/core/common_runtime/device_mgr.cc
@@ -15,7 +15,6 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device_mgr.h"
 
-#include <memory>
 #include <vector>
 #include "tensorflow/core/common_runtime/local_device.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -25,32 +24,32 @@ limitations under the License.
 
 namespace tensorflow {
 
-DeviceMgr::DeviceMgr(std::vector<std::unique_ptr<Device>> devices)
-    : devices_(std::move(devices)), name_backing_store_(128) {
-  for (auto& d : devices_) {
+DeviceMgr::DeviceMgr(const std::vector<Device*>& devices)
+    : name_backing_store_(128) {
+  for (Device* d : devices) {
     CHECK(d->device_mgr_ == nullptr);
     d->device_mgr_ = this;
 
+    devices_.push_back(d);
+
     // Register under the (1) full name and (2) canonical name.
     for (const string& name :
          DeviceNameUtils::GetNamesForDeviceMappings(d->parsed_name())) {
-      device_map_[CopyToBackingStore(name)] = d.get();
+      device_map_[CopyToBackingStore(name)] = d;
     }
     // Register under the (3) local name and (4) legacy local name.
     for (const string& name :
          DeviceNameUtils::GetLocalNamesForDeviceMappings(d->parsed_name())) {
-      device_map_[CopyToBackingStore(name)] = d.get();
+      device_map_[CopyToBackingStore(name)] = d;
     }
     device_type_counts_[d->device_type()]++;
   }
 }
 
-DeviceMgr::DeviceMgr(std::unique_ptr<Device> device)
-    : DeviceMgr([&device] {
-        std::vector<std::unique_ptr<Device>> vector;
-        vector.push_back(std::move(device));
-        return vector;
-      }()) {}
+DeviceMgr::~DeviceMgr() {
+  // TODO(b/37437134): Remove destructor after converting to std::unique_ptr.
+  for (Device* p : devices_) delete p;
+}
 
 StringPiece DeviceMgr::CopyToBackingStore(StringPiece s) {
   size_t n = s.size();
@@ -62,22 +61,18 @@ StringPiece DeviceMgr::CopyToBackingStore(StringPiece s) {
 void DeviceMgr::ListDeviceAttributes(
     std::vector<DeviceAttributes>* devices) const {
   devices->reserve(devices_.size());
-  for (const auto& dev : devices_) {
+  for (Device* dev : devices_) {
     devices->emplace_back(dev->attributes());
   }
 }
 
 std::vector<Device*> DeviceMgr::ListDevices() const {
-  std::vector<Device*> devices(devices_.size());
-  for (size_t i = 0; i < devices_.size(); ++i) {
-    devices[i] = devices_[i].get();
-  }
-  return devices;
+  return std::vector<Device*>(devices_.begin(), devices_.end());
 }
 
 string DeviceMgr::DebugString() const {
   string out;
-  for (const auto& dev : devices_) {
+  for (Device* dev : devices_) {
     strings::StrAppend(&out, dev->name(), "\n");
   }
   return out;
@@ -85,7 +80,7 @@ string DeviceMgr::DebugString() const {
 
 string DeviceMgr::DeviceMappingString() const {
   string out;
-  for (const auto& dev : devices_) {
+  for (Device* dev : devices_) {
     if (!dev->attributes().physical_device_desc().empty()) {
       strings::StrAppend(&out, dev->name(), " -> ",
                          dev->attributes().physical_device_desc(), "\n");
@@ -112,7 +107,7 @@ Status DeviceMgr::LookupDevice(StringPiece name, Device** device) const {
 
 void DeviceMgr::ClearContainers(gtl::ArraySlice<string> containers) const {
   Status s;
-  for (const auto& dev : devices_) {
+  for (Device* dev : devices_) {
     if (containers.empty()) {
       s.Update(dev->resource_manager()->Cleanup(
           dev->resource_manager()->default_container()));
diff --git a/tensorflow/core/common_runtime/device_mgr.h b/tensorflow/core/common_runtime/device_mgr.h
index bf8694655a..c1ff10d9b5 100644
--- a/tensorflow/core/common_runtime/device_mgr.h
+++ b/tensorflow/core/common_runtime/device_mgr.h
@@ -16,7 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
 #define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
 
-#include <memory>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -35,17 +34,15 @@ class DeviceAttributes;
 
 class DeviceMgr {
  public:
-  // Constructs a DeviceMgr from a list of devices.
+  // Takes ownership of each device in 'devices'.
   // TODO(zhifengc): Other initialization information.
-  explicit DeviceMgr(std::vector<std::unique_ptr<Device>> devices);
-
-  // Constructs a DeviceMgr managing a single device.
-  explicit DeviceMgr(std::unique_ptr<Device> device);
+  // TODO(b/37437134): Use std::unique_ptr's to track ownership.
+  explicit DeviceMgr(const std::vector<Device*>& devices);
+  ~DeviceMgr();
 
   // Returns attributes of all devices.
   void ListDeviceAttributes(std::vector<DeviceAttributes>* devices) const;
 
-  // Returns raw pointers to the underlying devices.
   std::vector<Device*> ListDevices() const;
 
   // Returns a string listing all devices.
@@ -65,7 +62,9 @@ class DeviceMgr {
   int NumDeviceType(const string& type) const;
 
  private:
-  const std::vector<std::unique_ptr<Device>> devices_;
+  // TODO(b/37437134): Use std::unique_ptr's to track ownership.
+  typedef gtl::InlinedVector<Device*, 8> DeviceVec;
+  DeviceVec devices_;
 
   StringPiece CopyToBackingStore(StringPiece s);
 
diff --git a/tensorflow/core/common_runtime/device_resolver_local_test.cc b/tensorflow/core/common_runtime/device_resolver_local_test.cc
index 54f1119e13..f5a6471ff7 100644
--- a/tensorflow/core/common_runtime/device_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/device_resolver_local_test.cc
@@ -36,12 +36,12 @@ class DeviceResolverLocalTest : public ::testing::Test {
     string task_name = "/job:localhost/replica:0/task:0";
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    std::vector<std::unique_ptr<Device>> devices;
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
   }
 
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
 };
diff --git a/tensorflow/core/common_runtime/device_set_test.cc b/tensorflow/core/common_runtime/device_set_test.cc
index 6a8c3d14e5..fd9c4222a7 100644
--- a/tensorflow/core/common_runtime/device_set_test.cc
+++ b/tensorflow/core/common_runtime/device_set_test.cc
@@ -57,7 +57,7 @@ class DeviceSetTest : public ::testing::Test {
 class DummyFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override {
+                       std::vector<Device*>* devices) override {
     return Status::OK();
   }
 };
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 91717328b1..40b7071f40 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -155,12 +155,12 @@ class DirectSessionFactory : public SessionFactory {
     if (options.config.graph_options().build_cost_model() > 0) {
       EnableCPUAllocatorFullStats(true);
     }
-    std::vector<std::unique_ptr<Device>> devices;
+    std::vector<Device*> devices;
     TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
         options, "/job:localhost/replica:0/task:0", &devices));
 
     DirectSession* session =
-        new DirectSession(options, new DeviceMgr(std::move(devices)), this);
+        new DirectSession(options, new DeviceMgr(devices), this);
     {
       mutex_lock l(sessions_lock_);
       sessions_.push_back(session);
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 86890ba07d..a7b618c18b 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -181,7 +181,6 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
-        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
index 3ffed3ce32..948bdbcaf5 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
-#include "absl/memory/memory.h"
 #include "tensorflow/cc/client/client_session.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
@@ -38,13 +37,12 @@ namespace {
 class TestEnv {
  public:
   TestEnv() : flib_def_(OpRegistry::Global(), {}) {
-    std::vector<std::unique_ptr<Device>> devices;
-    devices.push_back(
-        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0"));
-    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
-    flib_runtime_ = NewFunctionLibraryRuntime(
-        device_mgr_.get(), Env::Default(), device_mgr_->ListDevices()[0],
-        TF_GRAPH_DEF_VERSION, &flib_def_, nullptr, {}, nullptr);
+    Device* device =
+        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0");
+    device_mgr_.reset(new DeviceMgr({device}));
+    flib_runtime_ = NewFunctionLibraryRuntime(device_mgr_.get(), Env::Default(),
+                                              device, TF_GRAPH_DEF_VERSION,
+                                              &flib_def_, nullptr, {}, nullptr);
   }
 
   FunctionLibraryRuntime* function_library_runtime() const {
diff --git a/tensorflow/core/common_runtime/executor_test.cc b/tensorflow/core/common_runtime/executor_test.cc
index c311b2533e..7697103faf 100644
--- a/tensorflow/core/common_runtime/executor_test.cc
+++ b/tensorflow/core/common_runtime/executor_test.cc
@@ -53,17 +53,17 @@ class ExecutorTest : public ::testing::Test {
     // when the test completes.
     CHECK(rendez_->Unref());
     delete exec_;
+    delete device_;
   }
 
   // Resets executor_ with a new executor based on a graph 'gdef'.
   void Create(std::unique_ptr<const Graph> graph) {
     const int version = graph->versions().producer();
     LocalExecutorParams params;
-    params.device = device_.get();
+    params.device = device_;
     params.create_kernel = [this, version](const NodeDef& ndef,
                                            OpKernel** kernel) {
-      return CreateNonCachedKernel(device_.get(), nullptr, ndef, version,
-                                   kernel);
+      return CreateNonCachedKernel(device_, nullptr, ndef, version, kernel);
     };
     params.delete_kernel = [](OpKernel* kernel) {
       DeleteNonCachedKernel(kernel);
@@ -83,7 +83,7 @@ class ExecutorTest : public ::testing::Test {
   }
 
   thread::ThreadPool* thread_pool_ = nullptr;
-  std::unique_ptr<Device> device_;
+  Device* device_ = nullptr;
   Executor* exec_ = nullptr;
   StepStatsCollector step_stats_collector_;
   StepStats step_stats_;
diff --git a/tensorflow/core/common_runtime/function_test.cc b/tensorflow/core/common_runtime/function_test.cc
index 3b4c976685..13c189fb87 100644
--- a/tensorflow/core/common_runtime/function_test.cc
+++ b/tensorflow/core/common_runtime/function_test.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <atomic>
 #include <utility>
 
-#include "absl/memory/memory.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
 #include "tensorflow/cc/ops/array_ops_internal.h"
@@ -148,15 +147,14 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 3});
-    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices));
+        options, "/job:localhost/replica:0/task:0", &devices_));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
     OptimizerOptions opts;
-    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+    device_mgr_.reset(new DeviceMgr(devices_));
     pflr_.reset(new ProcessFunctionLibraryRuntime(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
@@ -360,6 +358,7 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   FunctionLibraryRuntime* flr0_;
   FunctionLibraryRuntime* flr1_;
   FunctionLibraryRuntime* flr2_;
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/core/common_runtime/function_threadpool_test.cc b/tensorflow/core/common_runtime/function_threadpool_test.cc
index bdbe24a70d..655a68cfc9 100644
--- a/tensorflow/core/common_runtime/function_threadpool_test.cc
+++ b/tensorflow/core/common_runtime/function_threadpool_test.cc
@@ -54,15 +54,14 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 3});
-    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices));
+        options, "/job:localhost/replica:0/task:0", &devices_));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
     OptimizerOptions opts;
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+    device_mgr_.reset(new DeviceMgr(devices_));
     pflr_.reset(new ProcessFunctionLibraryRuntime(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
@@ -195,6 +194,7 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   FunctionLibraryRuntime* flr0_;
   FunctionLibraryRuntime* flr1_;
   FunctionLibraryRuntime* flr2_;
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 5152d97fde..81fea311e1 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -907,9 +907,9 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
-Status BaseGPUDeviceFactory::CreateDevices(
-    const SessionOptions& options, const string& name_prefix,
-    std::vector<std::unique_ptr<Device>>* devices) {
+Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options,
+                                           const string& name_prefix,
+                                           std::vector<Device*>* devices) {
   TF_RETURN_IF_ERROR(ValidateGPUMachineManager());
   se::Platform* gpu_manager = GPUMachineManager();
   if (gpu_manager == nullptr) {
@@ -1073,10 +1073,12 @@ static string GetShortDeviceDescription(PlatformGpuId platform_gpu_id,
   // LINT.ThenChange(//tensorflow/python/platform/test.py)
 }
 
-Status BaseGPUDeviceFactory::CreateGPUDevice(
-    const SessionOptions& options, const string& name_prefix, TfGpuId tf_gpu_id,
-    int64 memory_limit, const DeviceLocality& dev_locality,
-    std::vector<std::unique_ptr<Device>>* devices) {
+Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
+                                             const string& name_prefix,
+                                             TfGpuId tf_gpu_id,
+                                             int64 memory_limit,
+                                             const DeviceLocality& dev_locality,
+                                             std::vector<Device*>* devices) {
   CHECK_GE(tf_gpu_id.value(), 0);
   const string device_name =
       strings::StrCat(name_prefix, "/device:GPU:", tf_gpu_id.value());
@@ -1106,7 +1108,7 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(
   // different (which should be an error).
   //
   // TODO(laigd): report error if memory_limit doesn't match stats.bytes_limit.
-  std::unique_ptr<BaseGPUDevice> gpu_device = CreateGPUDevice(
+  BaseGPUDevice* gpu_device = CreateGPUDevice(
       options, device_name, static_cast<Bytes>(stats.bytes_limit), dev_locality,
       tf_gpu_id, GetShortDeviceDescription(platform_gpu_id, desc),
       gpu_allocator, ProcessState::singleton()->GetCPUAllocator(numa_node));
@@ -1114,7 +1116,7 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(
             << (stats.bytes_limit >> 20) << " MB memory) -> physical GPU ("
             << GetShortDeviceDescription(platform_gpu_id, desc) << ")";
   TF_RETURN_IF_ERROR(gpu_device->Init(options));
-  devices->push_back(std::move(gpu_device));
+  devices->push_back(gpu_device);
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index d002d02c51..674e8384d5 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -166,7 +166,7 @@ class BaseGPUDevice : public LocalDevice {
 class BaseGPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override;
+                       std::vector<Device*>* devices) override;
 
   struct InterconnectMap {
     // Name of interconnect technology, if known.
@@ -207,13 +207,15 @@ class BaseGPUDeviceFactory : public DeviceFactory {
   Status CreateGPUDevice(const SessionOptions& options,
                          const string& name_prefix, TfGpuId tf_gpu_id,
                          int64 memory_limit, const DeviceLocality& dev_locality,
-                         std::vector<std::unique_ptr<Device>>* devices);
-
-  virtual std::unique_ptr<BaseGPUDevice> CreateGPUDevice(
-      const SessionOptions& options, const string& name, Bytes memory_limit,
-      const DeviceLocality& dev_locality, TfGpuId tf_gpu_id,
-      const string& physical_device_desc, Allocator* gpu_allocator,
-      Allocator* cpu_allocator) = 0;
+                         std::vector<Device*>* devices);
+
+  virtual BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
+                                         const string& name, Bytes memory_limit,
+                                         const DeviceLocality& dev_locality,
+                                         TfGpuId tf_gpu_id,
+                                         const string& physical_device_desc,
+                                         Allocator* gpu_allocator,
+                                         Allocator* cpu_allocator) = 0;
 
   // Returns into 'ids' the list of valid platform GPU ids, in the order that
   // they should map to TF GPU ids "/device:GPU:0", "/device:GPU:1", etc,
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc b/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
index 8dc7197329..e1aaf95df6 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
@@ -59,14 +59,15 @@ class GPUDevice : public BaseGPUDevice {
 
 class GPUDeviceFactory : public BaseGPUDeviceFactory {
  private:
-  std::unique_ptr<BaseGPUDevice> CreateGPUDevice(
-      const SessionOptions& options, const string& name, Bytes memory_limit,
-      const DeviceLocality& locality, TfGpuId tf_gpu_id,
-      const string& physical_device_desc, Allocator* gpu_allocator,
-      Allocator* cpu_allocator) override {
-    return absl::make_unique<GPUDevice>(options, name, memory_limit, locality,
-                                        tf_gpu_id, physical_device_desc,
-                                        gpu_allocator, cpu_allocator);
+  BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
+                                 const string& name, Bytes memory_limit,
+                                 const DeviceLocality& locality,
+                                 TfGpuId tf_gpu_id,
+                                 const string& physical_device_desc,
+                                 Allocator* gpu_allocator,
+                                 Allocator* cpu_allocator) override {
+    return new GPUDevice(options, name, memory_limit, locality, tf_gpu_id,
+                         physical_device_desc, gpu_allocator, cpu_allocator);
   }
 };
 
@@ -107,7 +108,7 @@ class GPUCompatibleCPUDevice : public ThreadPoolDevice {
 class GPUCompatibleCPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override {
+                       std::vector<Device*>* devices) override {
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
     if (iter != options.config.device_count().end()) {
@@ -115,7 +116,7 @@ class GPUCompatibleCPUDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      devices->push_back(absl::make_unique<GPUCompatibleCPUDevice>(
+      devices->push_back(new GPUCompatibleCPUDevice(
           options, name, Bytes(256 << 20), DeviceLocality(), cpu_allocator()));
     }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
index 58656ec757..75be6d60b8 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
@@ -33,7 +33,7 @@ namespace {
 
 TEST(GPUDeviceOnNonGPUMachineTest, CreateGPUDevicesOnNonGPUMachine) {
   SessionOptions opts;
-  std::vector<std::unique_ptr<tensorflow::Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, "/job:localhost/replica:0/task:0", &devices));
   EXPECT_TRUE(devices.empty());
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
index ae623b2adb..36294094e9 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
@@ -88,7 +88,7 @@ class GPUDeviceTest : public ::testing::Test {
 
 TEST_F(GPUDeviceTest, FailedToParseVisibleDeviceList) {
   SessionOptions opts = MakeSessionOptions("0,abc");
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -97,7 +97,7 @@ TEST_F(GPUDeviceTest, FailedToParseVisibleDeviceList) {
 
 TEST_F(GPUDeviceTest, InvalidGpuId) {
   SessionOptions opts = MakeSessionOptions("100");
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -107,7 +107,7 @@ TEST_F(GPUDeviceTest, InvalidGpuId) {
 
 TEST_F(GPUDeviceTest, DuplicateEntryInVisibleDeviceList) {
   SessionOptions opts = MakeSessionOptions("0,0");
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -117,7 +117,7 @@ TEST_F(GPUDeviceTest, DuplicateEntryInVisibleDeviceList) {
 
 TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithMemoryFractionSettings) {
   SessionOptions opts = MakeSessionOptions("0", 0.1, 1, {{}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -129,7 +129,7 @@ TEST_F(GPUDeviceTest, GpuDeviceCountTooSmall) {
   // device_count is 0, but with one entry in visible_device_list and one
   // (empty) VirtualDevices messages.
   SessionOptions opts = MakeSessionOptions("0", 0, 0, {{}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::UNKNOWN);
@@ -141,7 +141,7 @@ TEST_F(GPUDeviceTest, NotEnoughGpuInVisibleDeviceList) {
   // Single entry in visible_device_list with two (empty) VirtualDevices
   // messages.
   SessionOptions opts = MakeSessionOptions("0", 0, 8, {{}, {}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::UNKNOWN);
@@ -155,7 +155,7 @@ TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithVisibleDeviceList) {
   // Three entries in visible_device_list with two (empty) VirtualDevices
   // messages.
   SessionOptions opts = MakeSessionOptions("0,1", 0, 8, {{}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -169,36 +169,39 @@ TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithVisibleDeviceList) {
 TEST_F(GPUDeviceTest, EmptyVirtualDeviceConfig) {
   // It'll create single virtual device when the virtual device config is empty.
   SessionOptions opts = MakeSessionOptions("0");
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_GE(devices[0]->attributes().memory_limit(), 0);
+  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, SingleVirtualDeviceWithNoMemoryLimit) {
   // It'll create single virtual device for the gpu in question when
   // memory_limit_mb is unset.
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_GE(devices[0]->attributes().memory_limit(), 0);
+  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, SingleVirtualDeviceWithMemoryLimit) {
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{123}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_EQ(123 << 20, devices[0]->attributes().memory_limit());
+  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, MultipleVirtualDevices) {
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{123, 456}});
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(2, devices.size());
@@ -216,6 +219,7 @@ TEST_F(GPUDeviceTest, MultipleVirtualDevices) {
             devices[1]->attributes().locality().links().link(0).type());
   EXPECT_EQ(BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength,
             devices[1]->attributes().locality().links().link(0).strength());
+  gtl::STLDeleteElements(&devices);
 }
 
 // Enabling unified memory on pre-Pascal GPUs results in an initialization
@@ -232,7 +236,7 @@ TEST_F(GPUDeviceTest, UnifiedMemoryUnavailableOnPrePascalGpus) {
   opts.config.mutable_gpu_options()
       ->mutable_experimental()
       ->set_use_unified_memory(true);
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INTERNAL);
@@ -255,7 +259,7 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
   }
 
   SessionOptions opts = MakeSessionOptions("0", kGpuMemoryFraction);
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_ASSERT_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   ASSERT_EQ(1, devices.size());
@@ -274,6 +278,8 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
                                      (memory_limit >> 20) << 20);
   EXPECT_NE(ptr, nullptr);
   allocator->DeallocateRaw(ptr);
+
+  gtl::STLDeleteElements(&devices);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
index f0656ff533..2144eea84f 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
@@ -15,7 +15,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h"
 
 #include <algorithm>
-#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/base_collective_executor.h"
 #include "tensorflow/core/common_runtime/collective_rma_local.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -218,7 +217,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
             << " num_devices_per_worker=" << num_devices_per_worker;
     int total_num_devices = num_workers * num_devices_per_worker;
     device_type_ = device_type;
-    std::vector<std::unique_ptr<Device>> local_devices;
+    std::vector<Device*> local_devices;
     SessionOptions sess_opts;
     sess_opts.env = Env::Default();
     Bytes mem_limit(4 << 20);
@@ -228,7 +227,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
         if (device_type == DEVICE_CPU) {
           string dev_name = strings::StrCat("/job:worker/replica:0/task:", wi,
                                             "/device:CPU:", di);
-          local_devices.push_back(absl::make_unique<ThreadPoolDevice>(
+          local_devices.push_back(new ThreadPoolDevice(
               sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator()));
         } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) {
           int dev_idx = (wi * num_devices_per_worker) + di;
@@ -236,7 +235,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
             LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more "
                          "than one ring node.";
           } else {
-            local_devices.push_back(std::move(gpu_devices_[dev_idx]));
+            local_devices.push_back(gpu_devices_[dev_idx]);
           }
         } else {
           LOG(FATAL) << "Unsupported device_type " << device_type;
@@ -244,7 +243,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
       }
     }
     if (!dev_mgr_ || device_type == DEVICE_CPU) {
-      dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
+      dev_mgr_.reset(new DeviceMgr(local_devices));
     }
     if (!gpu_ring_order_) gpu_ring_order_.reset(new string());
     dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get()));
@@ -715,7 +714,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
   std::unique_ptr<DeviceResolverLocal> dev_resolver_;
   std::vector<DeviceInstance*> instances_;
   CollectiveParams col_params_;
-  std::vector<std::unique_ptr<tensorflow::Device>> gpu_devices_;
+  std::vector<tensorflow::Device*> gpu_devices_;
   std::unique_ptr<tensorflow::DeviceMgr> dev_mgr_;
   std::unique_ptr<string> gpu_ring_order_;
   mutex mu_;
diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
index bdd6c0e87d..1f585a8c24 100644
--- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
+++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
@@ -75,12 +75,12 @@ Benchmark::Benchmark(const string& device, Graph* g,
   const int graph_def_version = g->versions().producer();
 
   LocalExecutorParams params;
-  params.device = device_.get();
+  params.device = device_;
   params.function_library = nullptr;
   params.create_kernel = [this, graph_def_version](const NodeDef& ndef,
                                                    OpKernel** kernel) {
-    return CreateNonCachedKernel(device_.get(), nullptr, ndef,
-                                 graph_def_version, kernel);
+    return CreateNonCachedKernel(device_, nullptr, ndef, graph_def_version,
+                                 kernel);
   };
   params.delete_kernel = [](OpKernel* kernel) {
     DeleteNonCachedKernel(kernel);
@@ -107,7 +107,7 @@ Benchmark::~Benchmark() {
     // run kernel destructors that may attempt to access state borrowed from
     // `device_`, such as the resource manager.
     exec_.reset();
-    device_.reset();
+    delete device_;
     delete pool_;
   }
 }
diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.h b/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
index b1557c50b0..555b43f655 100644
--- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
+++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
@@ -55,7 +55,7 @@ class Benchmark {
 
  private:
   thread::ThreadPool* pool_ = nullptr;
-  std::unique_ptr<Device> device_ = nullptr;
+  Device* device_ = nullptr;
   Rendezvous* rendez_ = nullptr;
   std::unique_ptr<Executor> exec_;
 
diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc
index 04e77e55f6..009f905f10 100644
--- a/tensorflow/core/common_runtime/placer_test.cc
+++ b/tensorflow/core/common_runtime/placer_test.cc
@@ -92,7 +92,7 @@ class FakeDevice : public Device {
 class DummyFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override {
+                       std::vector<Device*>* devices) override {
     return Status::OK();
   }
 };
diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
index 21cb62118a..cce2308011 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
@@ -62,12 +62,9 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 2});
-    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(options, "/job:a/replica:0/task:0",
-                                          &devices));
-    device0_ = devices[0].get();
-    device1_ = devices[1].get();
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+                                          &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
@@ -141,9 +138,8 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
     return Status::OK();
   }
 
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
-  Device* device0_ = nullptr;  // Not owned. (Owned by device_mgr_.)
-  Device* device1_ = nullptr;  // Not owned. (Owned by device_mgr_.)
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<TestClusterFLR> cluster_flr_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> proc_flr_;
@@ -169,16 +165,16 @@ TEST_F(ProcessFunctionLibraryRuntimeTest, Basic) {
   FunctionLibraryRuntime* flr =
       proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), device0_);
+  EXPECT_EQ(flr->device(), devices_[0]);
   flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/device:CPU:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), device0_);
+  EXPECT_EQ(flr->device(), devices_[0]);
   flr = proc_flr_->GetFLR("/device:CPU:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), device0_);
+  EXPECT_EQ(flr->device(), devices_[0]);
   flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:1");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), device1_);
+  EXPECT_EQ(flr->device(), devices_[1]);
   flr = proc_flr_->GetFLR("abc");
   EXPECT_EQ(flr, nullptr);
   rendezvous_->Unref();
diff --git a/tensorflow/core/common_runtime/renamed_device.cc b/tensorflow/core/common_runtime/renamed_device.cc
index 45541c35fe..56766a8df4 100644
--- a/tensorflow/core/common_runtime/renamed_device.cc
+++ b/tensorflow/core/common_runtime/renamed_device.cc
@@ -14,14 +14,15 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/common_runtime/renamed_device.h"
-#include "absl/memory/memory.h"
 
 namespace tensorflow {
 
+// TODO(saeta): Convert to returning a std::unique_ptr?
 /* static */
-std::unique_ptr<Device> RenamedDevice::NewRenamedDevice(
-    const string& new_base, Device* underlying, bool owns_underlying,
-    bool isolate_session_state) {
+Device* RenamedDevice::NewRenamedDevice(const string& new_base,
+                                        Device* underlying,
+                                        bool owns_underlying,
+                                        bool isolate_session_state) {
   DeviceNameUtils::ParsedName parsed_name;
   CHECK(DeviceNameUtils::ParseFullName(new_base, &parsed_name));
   DeviceNameUtils::ParsedName underlying_parsed_name =
@@ -35,9 +36,8 @@ std::unique_ptr<Device> RenamedDevice::NewRenamedDevice(
                                           parsed_name.id);
   DeviceAttributes attributes(underlying->attributes());
   attributes.set_name(name);
-  // Call absl::WrapUnique to access private constructor.
-  return absl::WrapUnique(new RenamedDevice(
-      underlying, attributes, owns_underlying, isolate_session_state));
+  return new RenamedDevice(underlying, attributes, owns_underlying,
+                           isolate_session_state);
 }
 
 RenamedDevice::RenamedDevice(Device* underlying,
diff --git a/tensorflow/core/common_runtime/renamed_device.h b/tensorflow/core/common_runtime/renamed_device.h
index 6d24f496ff..c00789a556 100644
--- a/tensorflow/core/common_runtime/renamed_device.h
+++ b/tensorflow/core/common_runtime/renamed_device.h
@@ -28,10 +28,9 @@ namespace tensorflow {
 // session.
 class RenamedDevice : public Device {
  public:
-  static std::unique_ptr<Device> NewRenamedDevice(const string& new_base,
-                                                  Device* underlying,
-                                                  bool owns_underlying,
-                                                  bool isolate_session_state);
+  static Device* NewRenamedDevice(const string& new_base, Device* underlying,
+                                  bool owns_underlying,
+                                  bool isolate_session_state);
 
   ~RenamedDevice() override;
 
diff --git a/tensorflow/core/common_runtime/ring_reducer_test.cc b/tensorflow/core/common_runtime/ring_reducer_test.cc
index 7feb29a6db..a271bf7b74 100644
--- a/tensorflow/core/common_runtime/ring_reducer_test.cc
+++ b/tensorflow/core/common_runtime/ring_reducer_test.cc
@@ -15,7 +15,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/ring_reducer.h"
 
 #include <algorithm>
-#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/base_collective_executor.h"
 #include "tensorflow/core/common_runtime/collective_rma_local.h"
 #include "tensorflow/core/common_runtime/device.h"
@@ -158,7 +157,7 @@ class RingReducerTest : public ::testing::Test {
     InitGPUDevices();
 #endif
     device_type_ = device_type;
-    std::vector<std::unique_ptr<Device>> local_devices;
+    std::vector<Device*> local_devices;
     SessionOptions sess_opts;
     sess_opts.env = Env::Default();
     Bytes mem_limit(4 << 20);
@@ -168,7 +167,7 @@ class RingReducerTest : public ::testing::Test {
         if (device_type == DEVICE_CPU) {
           string dev_name =
               strings::StrCat("/job:worker/replica:0/task:", wi, "/cpu:", di);
-          local_devices.push_back(absl::make_unique<ThreadPoolDevice>(
+          local_devices.push_back(new ThreadPoolDevice(
               sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator()));
         } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) {
           int dev_idx = (wi * num_devices) + di;
@@ -176,7 +175,7 @@ class RingReducerTest : public ::testing::Test {
             LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more "
                          "than one ring node.";
           } else {
-            local_devices.push_back(std::move(gpu_devices_[dev_idx]));
+            local_devices.push_back(gpu_devices_[dev_idx]);
           }
         } else {
           LOG(FATAL) << "Unsupported device_type " << device_type;
@@ -186,7 +185,7 @@ class RingReducerTest : public ::testing::Test {
     if (!dev_mgr_ || device_type == DEVICE_CPU) {
       LOG(ERROR) << "resetting dev_mgr for " << local_devices.size()
                  << " devices: ";
-      dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
+      dev_mgr_.reset(new DeviceMgr(local_devices));
     }
     if (!gpu_ring_order_) gpu_ring_order_.reset(new string());
     dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get()));
@@ -545,7 +544,7 @@ class RingReducerTest : public ::testing::Test {
   std::unique_ptr<DeviceResolverLocal> dev_resolver_;
   std::vector<DeviceInstance*> instances_;
   CollectiveParams col_params_;
-  std::vector<std::unique_ptr<tensorflow::Device>> gpu_devices_;
+  std::vector<tensorflow::Device*> gpu_devices_;
   std::unique_ptr<tensorflow::DeviceMgr> dev_mgr_;
   std::unique_ptr<string> gpu_ring_order_;
   mutex mu_;
diff --git a/tensorflow/core/common_runtime/threadpool_device_factory.cc b/tensorflow/core/common_runtime/threadpool_device_factory.cc
index f9cbb81749..c06a4035a7 100644
--- a/tensorflow/core/common_runtime/threadpool_device_factory.cc
+++ b/tensorflow/core/common_runtime/threadpool_device_factory.cc
@@ -13,13 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include <vector>
-
 // Register a factory that provides CPU devices.
-#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/threadpool_device.h"
+
+#include <vector>
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/process_state.h"
-#include "tensorflow/core/common_runtime/threadpool_device.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/public/session_options.h"
@@ -30,7 +29,7 @@ namespace tensorflow {
 class ThreadPoolDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<std::unique_ptr<Device>>* devices) override {
+                       std::vector<Device*>* devices) override {
     int num_numa_nodes = port::NUMANumNodes();
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
@@ -39,7 +38,7 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      std::unique_ptr<ThreadPoolDevice> tpd;
+      ThreadPoolDevice* tpd = nullptr;
       if (options.config.experimental().use_numa_affinity()) {
         int numa_node = i % num_numa_nodes;
         if (numa_node != i) {
@@ -50,15 +49,15 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
         }
         DeviceLocality dev_locality;
         dev_locality.set_numa_node(numa_node);
-        tpd = absl::make_unique<ThreadPoolDevice>(
+        tpd = new ThreadPoolDevice(
             options, name, Bytes(256 << 20), dev_locality,
             ProcessState::singleton()->GetCPUAllocator(numa_node));
       } else {
-        tpd = absl::make_unique<ThreadPoolDevice>(
+        tpd = new ThreadPoolDevice(
             options, name, Bytes(256 << 20), DeviceLocality(),
             ProcessState::singleton()->GetCPUAllocator(port::kNUMANoAffinity));
       }
-      devices->push_back(std::move(tpd));
+      devices->push_back(tpd);
     }
 
     return Status::OK();
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index a47da0d6a0..818324746f 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -624,7 +624,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 40b18d321a..4eed856759 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -29,8 +29,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-static std::unique_ptr<Device> NewDevice(const string& type,
-                                         const string& name) {
+static Device* NewDevice(const string& type, const string& name) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -41,7 +40,7 @@ static std::unique_ptr<Device> NewDevice(const string& type,
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(3);  // a non-default value
-  return absl::make_unique<FakeDevice>(attr);
+  return new FakeDevice(attr);
 }
 
 class FakeWorker : public TestWorkerInterface {
@@ -157,16 +156,16 @@ class DeviceResDistTest : public ::testing::Test {
 
   void DefineWorker(const ConfigProto& config, const string& worker_name,
                     const string& device_type, int num_devices) {
-    std::vector<std::unique_ptr<Device>> devices;
+    std::vector<Device*> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i)));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
+    DeviceMgr* dev_mgr = new DeviceMgr(devices);
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto* d : dev_mgr->ListDevices()) {
+    for (auto d : devices) {
       dv->push_back(d->name());
     }
     DeviceResolverDistributed* dev_res =
diff --git a/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
index 26f722a6bd..33e1c8f2c3 100644
--- a/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
@@ -41,8 +41,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-static std::unique_ptr<Device> NewDevice(const string& type,
-                                         const string& name) {
+static Device* NewDevice(const string& type, const string& name) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -53,7 +52,7 @@ static std::unique_ptr<Device> NewDevice(const string& type,
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(3);  // a non-default value
-  return absl::make_unique<FakeDevice>(attr);
+  return new FakeDevice(attr);
 }
 
 static int64 kStepId = 123;
@@ -212,16 +211,16 @@ class CollRMADistTest : public ::testing::Test {
 
   void DefineWorker(const ConfigProto& config, const string& worker_name,
                     const string& device_type, int num_devices) {
-    std::vector<std::unique_ptr<Device>> devices;
+    std::vector<Device*> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i)));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
+    DeviceMgr* dev_mgr = new DeviceMgr(devices);
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : dev_mgr->ListDevices()) {
+    for (auto d : devices) {
       dv->push_back(d->name());
     }
     DeviceResolverDistributed* dev_res =
diff --git a/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
index 842a2b3b05..ae44b98bd5 100644
--- a/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
@@ -15,7 +15,6 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
 
-#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/distributed_runtime/test_utils.h"
 #include "tensorflow/core/lib/core/notification.h"
@@ -42,8 +41,8 @@ class TestableDeviceResolverDistributed : public DeviceResolverDistributed {
 
 // Create a fake 'Device' whose only interesting attribute is a non-default
 // DeviceLocality.
-static std::unique_ptr<Device> NewDevice(const string& type, const string& name,
-                                         int numa_node) {
+static Device* NewDevice(const string& type, const string& name,
+                         int numa_node) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -54,7 +53,7 @@ static std::unique_ptr<Device> NewDevice(const string& type, const string& name,
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(numa_node);
-  return absl::make_unique<FakeDevice>(attr);
+  return new FakeDevice(attr);
 }
 
 // Create a fake WorkerInterface that responds to requests without RPCs,
@@ -152,19 +151,19 @@ class DeviceResDistTest : public ::testing::Test {
 
   void DefineWorker(const string& worker_name, const string& device_type,
                     int num_devices) {
-    std::vector<std::unique_ptr<Device>> devices;
+    std::vector<Device*> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i), i));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
+    DeviceMgr* dev_mgr = new DeviceMgr(devices);
     TestableDeviceResolverDistributed* dev_res =
         new TestableDeviceResolverDistributed(dev_mgr, &wc_, worker_name);
     resolvers_[worker_name] = dev_res;
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto* d : dev_mgr->ListDevices()) {
+    for (auto d : devices) {
       dv->push_back(d->name());
     }
     FakeWorker* fw = new FakeWorker(worker_name, dev_mgr, dev_res);
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
index c66466c0a6..5b0a420fad 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
@@ -87,7 +87,7 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
     return tensorflow::errors::Internal(
         "invalid eager env_ or env_->rendezvous_mgr.");
   }
-  std::vector<std::unique_ptr<tensorflow::Device>> devices;
+  std::vector<tensorflow::Device*> devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
       // TODO(nareshmodi): Correctly set the SessionOptions.
@@ -97,12 +97,12 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
                       request->server_def().task_index()),
       &devices));
   response->mutable_device_attributes()->Reserve(devices.size());
-  for (const auto& d : devices) {
+  for (auto& d : devices) {
     *response->add_device_attributes() = d->attributes();
   }
 
   std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(std::move(devices)));
+      new tensorflow::DeviceMgr(devices));
 
   auto* r = env_->rendezvous_mgr->Find(request->rendezvous_id());
   auto session_name = strings::StrCat("eager_", request->rendezvous_id());
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
index 7a1463e8f0..5ba522c2a2 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
@@ -68,9 +68,12 @@ class EagerServiceImplTest : public ::testing::Test {
     worker_env_.rendezvous_mgr = &rendezvous_mgr_;
     worker_env_.session_mgr = session_mgr_.get();
 
-    device_mgr_ = absl::make_unique<DeviceMgr>(DeviceFactory::NewDevice(
-        "CPU", {}, "/job:localhost/replica:0/task:0/device:CPU:0"));
-    worker_env_.local_devices = device_mgr_->ListDevices();
+    Device* device = DeviceFactory::NewDevice(
+        "CPU", {}, "/job:localhost/replica:0/task:0/device:CPU:0");
+
+    worker_env_.local_devices = {device};
+
+    device_mgr_.reset(new DeviceMgr(worker_env_.local_devices));
     worker_env_.device_mgr = device_mgr_.get();
   }
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
index cbd5cd927e..ae722fdfe9 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <cstring>
 #include <limits>
 #include <memory>
-#include <vector>
 
 #include "grpc/support/alloc.h"
 #include "grpcpp/grpcpp.h"
@@ -157,12 +156,10 @@ Status GrpcServer::Init(
   string name_prefix =
       strings::StrCat("/job:", server_def_.job_name(), "/replica:0",
                       "/task:", server_def_.task_index());
-  std::vector<std::unique_ptr<Device>> devices;
-  TF_RETURN_IF_ERROR(
-      DeviceFactory::AddDevices(sess_opts, name_prefix, &devices));
-  worker_env_.device_mgr = new DeviceMgr(std::move(devices));
-  master_env_.local_devices = worker_env_.device_mgr->ListDevices();
-  worker_env_.local_devices = worker_env_.device_mgr->ListDevices();
+  TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(sess_opts, name_prefix,
+                                               &master_env_.local_devices));
+  worker_env_.local_devices = master_env_.local_devices;
+  worker_env_.device_mgr = new DeviceMgr(worker_env_.local_devices);
   worker_env_.rendezvous_mgr = rendezvous_mgr_func == nullptr
                                    ? new RpcRendezvousMgr(&worker_env_)
                                    : rendezvous_mgr_func(&worker_env_);
diff --git a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
index 1c87fe9d92..0323300fdd 100644
--- a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
@@ -42,9 +42,8 @@ class RpcCollectiveExecutorMgrTest : public ::testing::Test {
     WorkerCacheInterface* worker_cache = nullptr;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    std::vector<std::unique_ptr<Device>> devices;
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
-    device_mgr_.reset(new DeviceMgr(std::move(devices)));
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
     std::unique_ptr<DeviceResolverDistributed> dr(new DeviceResolverDistributed(
         device_mgr_.get(), worker_cache, task_name));
     std::unique_ptr<CollectiveParamResolverDistributed> cpr(
@@ -58,6 +57,7 @@ class RpcCollectiveExecutorMgrTest : public ::testing::Test {
   }
 
   std::unique_ptr<RpcCollectiveExecutorMgr> cme_;
+  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
 };
 
diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc
index 29fe767e42..38833bd202 100644
--- a/tensorflow/core/distributed_runtime/session_mgr.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr.cc
@@ -78,13 +78,13 @@ Status SessionMgr::CreateSession(const string& session,
 
   if (isolate_session_state) {
     // Create a private copy of the DeviceMgr for the WorkerSession.
-    std::vector<std::unique_ptr<Device>> renamed_devices;
+    std::vector<Device*> renamed_devices;
     for (Device* d : worker_env_->local_devices) {
       renamed_devices.push_back(RenamedDevice::NewRenamedDevice(
           worker_name, d, false, isolate_session_state));
     }
 
-    auto device_mgr = MakeUnique<DeviceMgr>(std::move(renamed_devices));
+    auto device_mgr = MakeUnique<DeviceMgr>(renamed_devices);
     auto graph_mgr = MakeUnique<GraphMgr>(worker_env_, device_mgr.get());
     worker_session.reset(
         new WorkerSession(session, worker_name,
diff --git a/tensorflow/core/distributed_runtime/session_mgr_test.cc b/tensorflow/core/distributed_runtime/session_mgr_test.cc
index 1ab0d20f0b..99192119a6 100644
--- a/tensorflow/core/distributed_runtime/session_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr_test.cc
@@ -46,9 +46,11 @@ class SessionMgrTest : public ::testing::Test {
   SessionMgrTest()
       : mgr_(&env_, "/job:mnist/replica:0/task:0",
              std::unique_ptr<WorkerCacheInterface>(), factory_) {
-    device_mgr_ = absl::make_unique<DeviceMgr>(
-        FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0"));
-    env_.local_devices = device_mgr_->ListDevices();
+    Device* device =
+        FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0")
+            .release();
+    env_.local_devices = {device};
+    device_mgr_.reset(new DeviceMgr(env_.local_devices));
     env_.device_mgr = device_mgr_.get();
   }
 
diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index e69dfa79d1..cf99f4908b 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -102,11 +102,10 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   }
 
   // Instantiate all variables for function library runtime creation.
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<Device*> devices;
   TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
-  Device* cpu_device = devices[0].get();
-  std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(std::move(devices)));
+  std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(devices));
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              graph_def.library());
   Env* env = Env::Default();
@@ -125,7 +124,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
       new ProcessFunctionLibraryRuntime(dvc_mgr.get(), env,
                                         graph_def.versions().producer(),
                                         &function_library, *optimizer_opts));
-  FunctionLibraryRuntime* flr = pflr->GetFLR(cpu_device->name());
+  FunctionLibraryRuntime* flr = pflr->GetFLR(devices[0]->name());
 
   // Create the GraphOptimizer to optimize the graph def.
   GraphConstructorOptions graph_ctor_opts;
@@ -138,7 +137,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
 
   // Optimize the graph.
   ::tensorflow::GraphOptimizer optimizer(*optimizer_opts);
-  optimizer.Optimize(flr, env, cpu_device, &graphptr, /*shape_map=*/nullptr);
+  optimizer.Optimize(flr, env, devices[0], &graphptr, /*shape_map=*/nullptr);
   graphptr->ToGraphDef(output_graph_def);
 
   // The default values of attributes might have been stripped by the optimizer.
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 8e6629565a..b6f989f2c9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -142,6 +142,7 @@ cc_library(
         ":graph_optimizer",
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -149,7 +150,6 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
-        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index f8ddbeb659..f99826ddca 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -16,9 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 
 #include <unordered_map>
-#include <vector>
 
-#include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -345,15 +343,14 @@ class FunctionOptimizerContext {
       DeviceAttributes attr;
       attr.set_name("/device:CPU:0");
       attr.set_device_type("CPU");
-      std::vector<std::unique_ptr<Device>> devices;
-      devices.push_back(absl::make_unique<FakeCPUDevice>(env, attr));
-      device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+      Device* device = new FakeCPUDevice(env, attr);
+      device_mgr_.reset(new DeviceMgr({device}));
       OptimizerOptions optimizer_opts;
       optimizer_opts.set_do_function_inlining(true);
       process_flr_.reset(new ProcessFunctionLibraryRuntime(
           device_mgr_.get(), env, graph_version_, &function_library_,
           optimizer_opts));
-      flr_ = process_flr_->GetFLR(device_mgr_->ListDevices()[0]->name());
+      flr_ = process_flr_->GetFLR(device->name());
     }
   }
 
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index dcb6975669..7192684e2d 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -600,7 +600,6 @@ tf_kernel_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:session_options",
         "//tensorflow/core/kernels:ops_util",
-        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 98b67454d5..93999dc095 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/kernels/data/iterator_ops.h"
 
-#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/common_runtime/renamed_device.h"
 #include "tensorflow/core/common_runtime/threadpool_device.h"
@@ -546,9 +545,10 @@ FunctionLibraryRuntime* IteratorHandleOp::CreatePrivateFLR(
   // in its resource manager. The existing device will outlive the
   // IteratorResource, because we are storing the IteratorResource
   // in that device's resource manager.
-  *device_mgr = absl::make_unique<DeviceMgr>(RenamedDevice::NewRenamedDevice(
+  Device* wrapped_device = RenamedDevice::NewRenamedDevice(
       ctx->device()->name(), down_cast<Device*>(ctx->device()),
-      false /* owns_underlying */, false /* isolate_session_state */));
+      false /* owns_underlying */, false /* isolate_session_state */);
+  device_mgr->reset(new DeviceMgr({wrapped_device}));
   flib_def->reset(new FunctionLibraryDefinition(
       *ctx->function_library()->GetFunctionLibraryDefinition()));
   pflr->reset(new ProcessFunctionLibraryRuntime(
diff --git a/tensorflow/core/kernels/data/single_threaded_executor_test.cc b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
index 7bb51fb8b5..6244e287bb 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor_test.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
@@ -51,17 +51,17 @@ class ExecutorTest : public ::testing::Test {
     // when the test completes.
     CHECK(rendez_->Unref());
     delete exec_;
+    delete device_;
   }
 
   // Resets executor_ with a new executor based on a graph 'gdef'.
   void Create(std::unique_ptr<const Graph> graph) {
     const int version = graph->versions().producer();
     LocalExecutorParams params;
-    params.device = device_.get();
+    params.device = device_;
     params.create_kernel = [this, version](const NodeDef& ndef,
                                            OpKernel** kernel) {
-      return CreateNonCachedKernel(device_.get(), nullptr, ndef, version,
-                                   kernel);
+      return CreateNonCachedKernel(device_, nullptr, ndef, version, kernel);
     };
     params.delete_kernel = [](OpKernel* kernel) {
       DeleteNonCachedKernel(kernel);
@@ -86,7 +86,7 @@ class ExecutorTest : public ::testing::Test {
     return exec_->Run(args);
   }
 
-  std::unique_ptr<Device> device_;
+  Device* device_ = nullptr;
   Executor* exec_ = nullptr;
   Executor::Args::Runner runner_;
   Rendezvous* rendez_ = nullptr;
diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD
index 63e86899da..222a043a88 100644
--- a/tensorflow/lite/delegates/flex/BUILD
+++ b/tensorflow/lite/delegates/flex/BUILD
@@ -116,7 +116,6 @@ cc_library(
     hdrs = ["delegate_data.h"],
     deps = [
         ":buffer_map",
-        "@com_google_absl//absl/memory",
         "//tensorflow/core/common_runtime/eager:context",
     ] + select({
         "//tensorflow:android": [
diff --git a/tensorflow/lite/delegates/flex/delegate_data.cc b/tensorflow/lite/delegates/flex/delegate_data.cc
index 1483a53038..b62479a448 100644
--- a/tensorflow/lite/delegates/flex/delegate_data.cc
+++ b/tensorflow/lite/delegates/flex/delegate_data.cc
@@ -14,21 +14,20 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/delegates/flex/delegate_data.h"
 
-#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tflite {
 namespace flex {
 tensorflow::Status DelegateData::Create(std::unique_ptr<DelegateData>* data) {
-  std::vector<std::unique_ptr<tensorflow::Device>> devices;
+  std::vector<tensorflow::Device*> devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
       tensorflow::SessionOptions(), "/job:localhost/replica:0/task:0",
       &devices));
 
-  std::unique_ptr<tensorflow::DeviceMgr> device_mgr =
-      absl::make_unique<tensorflow::DeviceMgr>(std::move(devices));
+  std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
+      new tensorflow::DeviceMgr(devices));
   // Note that Rendezvous is ref-counted so it will be automatically deleted.
   tensorflow::Rendezvous* rendezvous =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index dfeaebca26..4c3a0717e7 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2012,13 +2012,13 @@ bool InlineAllFunctions(GraphDef* graphdef) {
   tensorflow::SessionOptions options;
   auto* device_count = options.config.mutable_device_count();
   device_count->insert({"CPU", 1});
-  std::vector<std::unique_ptr<tensorflow::Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   TF_CHECK_OK(tensorflow::DeviceFactory::AddDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
 
   tensorflow::FunctionLibraryDefinition fld(tensorflow::OpRegistry::Global(),
                                             graphdef_copy.library());
-  tensorflow::DeviceMgr device_mgr(std::move(devices));
+  tensorflow::DeviceMgr device_mgr(devices);
   tensorflow::OptimizerOptions o_opts;
   tensorflow::ProcessFunctionLibraryRuntime pflr(
       &device_mgr, tensorflow::Env::Default(), TF_GRAPH_DEF_VERSION, &fld,
diff --git a/tensorflow/python/client/device_lib.i b/tensorflow/python/client/device_lib.i
index 3e579152d5..944e855cee 100644
--- a/tensorflow/python/client/device_lib.i
+++ b/tensorflow/python/client/device_lib.i
@@ -48,14 +48,17 @@ static std::vector<string> ListDevicesWithSessionConfig(
   std::vector<string> output;
   SessionOptions options;
   options.config = config;
-  std::vector<std::unique_ptr<Device>> devices;
+  std::vector<Device*> devices;
   Status status = DeviceFactory::AddDevices(
       options, "" /* name_prefix */, &devices);
   if (!status.ok()) {
     Set_TF_Status_from_Status(out_status, status);
   }
 
-  for (const std::unique_ptr<Device>& device : devices) {
+  std::vector<std::unique_ptr<Device>> device_holder(devices.begin(),
+                                                     devices.end());
+
+  for (const Device* device : devices) {
     const DeviceAttributes& attr = device->attributes();
     string attr_serialized;
     if (!attr.SerializeToString(&attr_serialized)) {
diff --git a/tensorflow/python/grappler/tf_optimizer.i b/tensorflow/python/grappler/tf_optimizer.i
index b746c3ec26..daa5bc9444 100644
--- a/tensorflow/python/grappler/tf_optimizer.i
+++ b/tensorflow/python/grappler/tf_optimizer.i
@@ -74,13 +74,13 @@ limitations under the License.
 
 void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* device_map) {
   tensorflow::SessionOptions options;
-  std::vector<std::unique_ptr<tensorflow::Device>> devices;
+  std::vector<tensorflow::Device*> devices;
   tensorflow::Status status = tensorflow::DeviceFactory::AddDevices(options, "", &devices);
   if (!status.ok()) {
     return;
   }
 
-  for (const std::unique_ptr<tensorflow::Device>& device : devices) {
+  for (const tensorflow::Device* device : devices) {
     tensorflow::DeviceProperties& prop = (*device_map)[device->name()];
     prop = tensorflow::grappler::GetDeviceInfo(device->parsed_name());
 
@@ -88,6 +88,7 @@ void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* dev
     // available device memory.
     const tensorflow::DeviceAttributes& attr = device->attributes();
     prop.set_memory_size(attr.memory_limit());
+    delete device;
   }
 }
 
-- 
GitLab


From d6e0fd35b4baa11f1b3cc5a07c372322387fb2ad Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Sat, 24 Nov 2018 00:28:12 -0800
Subject: [PATCH 0744/1554] Add update_config_proto to distribution strategies.

PiperOrigin-RevId: 222667946
---
 .../python/collective_all_reduce_strategy.py  | 20 ++++++---
 .../collective_all_reduce_strategy_test.py    | 45 +++++++++++++------
 .../python/mirrored_strategy_multigpu_test.py | 10 +++++
 .../python/parameter_server_strategy.py       | 20 ++++++---
 .../python/parameter_server_strategy_test.py  | 27 +++++++++++
 .../contrib/distribute/python/tpu_strategy.py | 14 ++++--
 .../python/distribute/mirrored_strategy.py    |  8 +++-
 tensorflow/python/training/distribute.py      | 30 ++++++++++++-
 .../v1/tensorflow.distribute.-strategy.pbtxt  |  4 ++
 .../v2/tensorflow.distribute.-strategy.pbtxt  |  4 ++
 10 files changed, 149 insertions(+), 33 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 906377b739..17323e2741 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import copy
+
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
@@ -263,13 +265,15 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
           self._container_strategy(), self._num_gpus_per_worker, cluster_spec,
           task_type, task_id)
 
-    if not session_config:
-      return
+    if session_config:
+      session_config.CopyFrom(self._update_config_proto(session_config))
 
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
     # Enable the scoped allocator optimization for CollectiveOps.  This
     # optimization converts many small all-reduces into fewer larger
     # all-reduces.
-    rewrite_options = session_config.graph_options.rewrite_options
+    rewrite_options = updated_config.graph_options.rewrite_options
     rewrite_options.scoped_allocator_optimization = (
         rewriter_config_pb2.RewriterConfig.ON)
     # We turn on ScopedAllocator only for CollectiveReduce op, i.e. enable_op =
@@ -287,20 +291,22 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     # Collective group leader is needed for collective ops to coordinate
     # workers.
     if "chief" in self._cluster_spec.jobs:
-      session_config.experimental.collective_group_leader = (
+      updated_config.experimental.collective_group_leader = (
           "/job:chief/replica:0/task:0")
     else:
       if "worker" not in self._cluster_spec.jobs:
         raise ValueError(
             "You must have `chief` or `worker` jobs in the `cluster_spec`.")
-      session_config.experimental.collective_group_leader = (
+      updated_config.experimental.collective_group_leader = (
           "/job:worker/replica:0/task:0")
 
     # The device filters prevent communication between workers.
-    del session_config.device_filters[:]
-    session_config.device_filters.append(
+    del updated_config.device_filters[:]
+    updated_config.device_filters.append(
         "/job:%s/task:%d" % (self._task_type, self._task_id))
 
+    return updated_config
+
   @property
   def experimental_between_graph(self):
     return True
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index eb2b859aa5..09239ffc72 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -26,6 +26,7 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import cross_device_utils
@@ -56,9 +57,6 @@ class CollectiveAllReduceStrategyTestBase(
   collective_key_base = 0
 
   def setUp(self):
-    self._run_options = config_pb2.RunOptions()
-    self._run_options.experimental.collective_graph_key = 6
-
     # We use a different key_base for each test so that collective keys won't be
     # reused.
     # TODO(yuefengz, tucker): enable it to reuse collective keys in different
@@ -145,11 +143,10 @@ class CollectiveAllReduceStrategyTestBase(
       if context.num_gpus() < d.extended._num_gpus_per_worker:
         return True
 
-      sess.run(
-          variables.global_variables_initializer(), options=self._run_options)
+      sess.run(variables.global_variables_initializer())
 
       for i in range(10):
-        b, a = sess.run((before_out, after_out), options=self._run_options)
+        b, a = sess.run((before_out, after_out))
         if i == 0:
           before, = b
         after, = a
@@ -234,11 +231,9 @@ class CollectiveAllReduceStrategyTestBase(
               destinations='/cpu:0'))[0]
       x = distribution.unwrap(x)[0]
 
-      sess.run(
-          variables.global_variables_initializer(), options=self._run_options)
+      sess.run(variables.global_variables_initializer())
 
-      x_value, reduced_x_value = sess.run([x, reduced_x],
-                                          options=self._run_options)
+      x_value, reduced_x_value = sess.run([x, reduced_x])
       self.assertTrue(
           np.allclose(x_value, reduced_x_value, atol=1e-5),
           msg=('x_value = %r, reduced_x_value = %r' % (x_value,
@@ -342,6 +337,32 @@ class DistributedCollectiveAllReduceStrategyTest(
     self._test_input_fn_iterator('worker', 1, num_gpus,
                                  input_fn, expected_values)
 
+  def testUpdateConfigProto(self):
+    distribution = collective_all_reduce_strategy.CollectiveAllReduceStrategy(
+        num_gpus_per_worker=2)
+    distribution.configure(
+        cluster_spec=self._cluster_spec, task_type='worker', task_id=1)
+
+    config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden'])
+    rewrite_options = config_proto.graph_options.rewrite_options
+    rewrite_options.scoped_allocator_opts.enable_op.append('to_be_removed')
+
+    new_config = distribution.update_config_proto(config_proto)
+
+    # Verify group leader
+    self.assertEqual('/job:worker/replica:0/task:0',
+                     new_config.experimental.collective_group_leader)
+
+    # Verify device filters.
+    self.assertEqual(['/job:worker/task:1'], new_config.device_filters)
+
+    # Verify rewrite options.
+    new_rewrite_options = new_config.graph_options.rewrite_options
+    self.assertEqual(rewriter_config_pb2.RewriterConfig.ON,
+                     new_rewrite_options.scoped_allocator_optimization)
+    self.assertEqual(['CollectiveReduce'],
+                     new_rewrite_options.scoped_allocator_opts.enable_op)
+
 
 class DistributedCollectiveAllReduceStrategyTestWithChief(
     CollectiveAllReduceStrategyTestBase, parameterized.TestCase):
@@ -352,10 +373,6 @@ class DistributedCollectiveAllReduceStrategyTestWithChief(
     cls._cluster_spec = multi_worker_test_base.create_in_process_cluster(
         num_workers=3, num_ps=0, has_chief=True)
 
-  def setUp(self):
-    super(DistributedCollectiveAllReduceStrategyTestWithChief, self).setUp()
-    self._run_options.experimental.collective_graph_key = 7
-
   @combinations.generate(
       combinations.combine(mode=['graph'], num_gpus=[0, 1, 2], required_gpus=1))
   def testMinimizeLossGraph(self, num_gpus):
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 1027da857d..cf6c7f6879 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -1372,6 +1373,15 @@ class MultiWorkerMirroredStrategyTest(
       self._test_input_fn_iterator(
           iterator, distribution.extended.worker_devices, expected_values, sess)
 
+  def testUpdateConfigProto(self, distribution):
+    distribution.configure(cluster_spec={"worker": ["fake1", "fake2"]})
+
+    config_proto = config_pb2.ConfigProto()
+    new_config = distribution.update_config_proto(config_proto)
+
+    # Verify isolate_session_state
+    self.assertTrue(new_config.isolate_session_state)
+
 
 class MultiWorkerMirroredStrategyTestWithChief(
     multi_worker_test_base.MultiWorkerTestBase,
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index fc2d2b20c9..d127868525 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import copy
+
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import multi_worker_util
@@ -462,21 +464,27 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
       self._initialize_multi_worker(self._num_gpus_per_worker,
                                     self._cluster_spec, task_type, task_id)
 
-    if not session_config or not self._cluster_spec:
-      return
+    if session_config:
+      session_config.CopyFrom(self._update_config_proto(session_config))
+
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
+    if not self._cluster_spec:
+      updated_config.isolate_session_state = True
+      return updated_config
 
-    session_config.isolate_session_state = False
+    updated_config.isolate_session_state = False
 
-    assert self._cluster_spec
     assert self._task_type
     assert self._task_id is not None
 
     # The device filters prevent communication between workers.
     if self._task_type not in ["chief", "worker"]:
       return
-    del session_config.device_filters[:]
-    session_config.device_filters.extend(
+    del updated_config.device_filters[:]
+    updated_config.device_filters.extend(
         ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
+    return updated_config
 
   @property
   def _num_replicas_in_sync(self):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 1ada6a6ba4..3ea9b90f6f 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -656,6 +656,33 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         num_gpus_per_worker=context.num_gpus())
     self._test_global_step_update(strategy)
 
+  def testUpdateConfigProtoMultiWorker(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    distribution.configure(
+        cluster_spec=self._cluster_spec, task_type='worker', task_id=1)
+
+    config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden'])
+
+    new_config = distribution.update_config_proto(config_proto)
+
+    # Verify device filters.
+    self.assertEqual(['/job:worker/task:1', '/job:ps'],
+                     new_config.device_filters)
+
+    # Verify isolate_session_state
+    self.assertFalse(new_config.isolate_session_state)
+
+  def testUpdateConfigProtoLocal(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+
+    config_proto = config_pb2.ConfigProto()
+    new_config = distribution.update_config_proto(config_proto)
+
+    # Verify isolate_session_state
+    self.assertTrue(new_config.isolate_session_state)
+
 
 class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                                            parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 94cf548cb4..3e755242f7 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import copy
 import functools
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
@@ -539,10 +540,15 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
                  task_id=None):
     del cluster_spec, task_type, task_id
     if session_config:
-      session_config.isolate_session_state = True
-      cluster_spec = self._tpu_cluster_resolver.cluster_spec()
-      if cluster_spec:
-        session_config.cluster_def.CopyFrom(cluster_spec.as_cluster_def())
+      session_config.CopyFrom(self._update_config_proto(session_config))
+
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
+    updated_config.isolate_session_state = True
+    cluster_spec = self._tpu_cluster_resolver.cluster_spec()
+    if cluster_spec:
+      updated_config.cluster_def.CopyFrom(cluster_spec.as_cluster_def())
+    return updated_config
 
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 7ed096b863..7094ed6628 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import contextlib
+import copy
 import functools
 import threading
 
@@ -562,7 +563,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     del task_type, task_id
 
     if session_config:
-      session_config.isolate_session_state = True
+      session_config.CopyFrom(self._update_config_proto(session_config))
 
     if cluster_spec:
       self._initialize_multi_worker(self._num_gpus, cluster_spec)
@@ -583,6 +584,11 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
         self._cross_device_ops = cross_device_ops_lib.choose_the_best(
             self._devices, session_config=session_config)
 
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
+    updated_config.isolate_session_state = True
+    return updated_config
+
   def _get_cross_device_ops(self):
     if self._cross_device_ops is None:
       self._cross_device_ops = (
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index ba3334108b..a976062249 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -610,10 +610,35 @@ class DistributionStrategy(object):
                 cluster_spec=None,
                 task_type=None,
                 task_id=None):
-    """Configures the strategy class."""
+    # pylint: disable=g-doc-return-or-yield,g-doc-args
+    """DEPRECATED: use `update_config_proto` instead.
+
+    Configures the strategy class.
+
+    DEPRECATED: This method's functionality has been split into the strategy
+    constructor and `update_config_proto`. In the future, we will allow passing
+    cluster and config_proto to the constructor to configure the strategy. And
+    `update_config_proto` can be used to update the config_proto based on the
+    specific strategy.
+    """
     return self._extended._configure(  # pylint: disable=protected-access
         session_config, cluster_spec, task_type, task_id)
 
+  def update_config_proto(self, config_proto):
+    """Returns a copy of `config_proto` modified for use with this strategy.
+
+    The updated config has something needed to run a strategy, e.g.
+    configuration to run collective ops, or device filters to improve
+    distributed training performance.
+
+    Args:
+      config_proto: a `tf.ConfigProto` object.
+
+    Returns:
+      The updated copy of the `config_proto`.
+    """
+    return self._extended._update_config_proto(config_proto)  # pylint: disable=protected-access
+
   @property
   @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
   def should_init(self):
@@ -1345,6 +1370,9 @@ class DistributionStrategyExtended(object):
     """Configures the strategy class."""
     del session_config, cluster_spec, task_type, task_id
 
+  def _update_config_proto(self, config_proto):
+    return copy.deepcopy(config_proto)
+
   @property
   def experimental_should_init(self):
     """Whether initialization is needed."""
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 4fe035b474..f0b0cd0d38 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -122,6 +122,10 @@ tf_class {
     name: "update"
     argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
   }
+  member_method {
+    name: "update_config_proto"
+    argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "update_non_slot"
     argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 4fe035b474..f0b0cd0d38 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -122,6 +122,10 @@ tf_class {
     name: "update"
     argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
   }
+  member_method {
+    name: "update_config_proto"
+    argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "update_non_slot"
     argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-- 
GitLab


From 2395637465c0b58a7e62460f52029dee56f1538e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 24 Nov 2018 01:02:05 -0800
Subject: [PATCH 0745/1554] compat: Update forward compatibility horizon to
 2018-11-24

PiperOrigin-RevId: 222669714
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 9150d91409..969b95dd0d 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 23)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 24)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From e3a9d2a1eb42a63e6e9c704b3c71488720d1b640 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 24 Nov 2018 08:22:05 -0800
Subject: [PATCH 0746/1554] Clean up some text in cross_device_ops.

PiperOrigin-RevId: 222686318
---
 .../python/distribute/cross_device_ops.py      | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index de25b718bf..87b6f41eba 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -916,15 +916,15 @@ def _choose_all_reduce_algorithm(device_links):
 
 
 def choose_the_best(devices, session_config=None):
-  """Find the best subclass of CrossDeviceOps given a tensorflow session.
+  """Find the best subclass of CrossDeviceOps given a session config.
 
   Args:
-    devices: a list of devices passed for distribute strategy.
-    session_config: a tensorflow session config or None. If None, it will make
-      deciesion based on all local devices.
+    devices: a list of devices passed to `tf.distribute.Strategy`.
+    session_config: a `tf.ConfigProto` or `None`. If `None`, it will make
+      decision based on all local devices.
 
   Returns:
-    a subclass of CrossDeviceOps.
+    A subclass of `CrossDeviceOps`.
   """
   requested_devices = set([device_util.canonicalize(d) for d in devices])
   machine_devices = device_lib.list_local_devices(session_config=session_config)
@@ -937,13 +937,13 @@ def choose_the_best(devices, session_config=None):
           "Device is available but not used by distribute strategy: %s", d.name)
 
   if len(using_devices) != len(requested_devices):
-    logging.warning("Not all devices in distribute strategy are visible by "
-                    "TensorFlow sessions.")
+    logging.warning("Not all devices in `tf.distribute.Strategy` are visible "
+                    "to TensorFlow.")
     return ReductionToOneDeviceCrossDeviceOps()
 
   if any(d.device_type.lower() != "gpu" for d in using_devices):
-    logging.warning("Not all devices in DistributionStrategy are visible to "
-                    "TensorFlow session.")
+    logging.warning("Not all devices in `tf.distribute.Strategy` are visible "
+                    "to TensorFlow.")
     return ReductionToOneDeviceCrossDeviceOps()
 
   device_links = [[] for _ in range(len(using_devices))]
-- 
GitLab


From 2dfefe0b988bb1c6468ae0318825e96459fdfdba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 24 Nov 2018 10:06:30 -0800
Subject: [PATCH 0747/1554] Remove DNNEstimator from contrib

PiperOrigin-RevId: 222690226
---
 tensorflow/contrib/estimator/BUILD            | 13 -------
 tensorflow/contrib/estimator/__init__.py      |  1 -
 .../contrib/estimator/python/estimator/dnn.py | 32 -----------------
 .../python/estimator/dnn_linear_combined.py   | 34 -------------------
 4 files changed, 80 deletions(-)
 delete mode 100644 tensorflow/contrib/estimator/python/estimator/dnn.py
 delete mode 100644 tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 37f253d9c1..a8d3a7d9f3 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -16,7 +16,6 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":boosted_trees",
-        ":dnn",
         ":dnn_with_layer_annotations",
         ":early_stopping",
         ":expect_tensorflow_estimator_installed",
@@ -47,18 +46,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "dnn",
-    srcs = ["python/estimator/dnn.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":expect_tensorflow_estimator_installed",
-        "//tensorflow:tensorflow_py_no_contrib",
-        "//tensorflow/python/estimator",
-        "//tensorflow/python/estimator:dnn",
-    ],
-)
-
 py_library(
     name = "dnn_with_layer_annotations",
     srcs = ["python/estimator/dnn_with_layer_annotations.py"],
diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py
index 80d5962762..d8e13cb793 100644
--- a/tensorflow/contrib/estimator/__init__.py
+++ b/tensorflow/contrib/estimator/__init__.py
@@ -58,7 +58,6 @@ _allowed_symbols = [
     'multi_label_head',
     'poisson_regression_head',
     'regression_head',
-    'DNNEstimator',
     'LinearEstimator',
     'boosted_trees_classifier_train_in_memory',
     'boosted_trees_regressor_train_in_memory',
diff --git a/tensorflow/contrib/estimator/python/estimator/dnn.py b/tensorflow/contrib/estimator/python/estimator/dnn.py
deleted file mode 100644
index 10f657df8d..0000000000
--- a/tensorflow/contrib/estimator/python/estimator/dnn.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""dnn python module.
-
-Importing from tensorflow.python.estimator is unsupported
-and will soon break!
-"""
-# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow_estimator.contrib.estimator.python.estimator import dnn
-
-# Include attrs that start with single underscore.
-_HAS_DYNAMIC_ATTRIBUTES = True
-dnn.__all__ = [s for s in dir(dnn) if not s.startswith('__')]
-
-from tensorflow_estimator.contrib.estimator.python.estimator.dnn import *
diff --git a/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py b/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py
deleted file mode 100644
index 7894418c4a..0000000000
--- a/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""dnn_linear_combined python module.
-
-Importing from tensorflow.python.estimator is unsupported
-and will soon break!
-"""
-# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow_estimator.contrib.estimator.python.estimator import dnn_linear_combined
-
-# Include attrs that start with single underscore.
-_HAS_DYNAMIC_ATTRIBUTES = True
-dnn_linear_combined.__all__ = [
-    s for s in dir(dnn_linear_combined) if not s.startswith('__')
-]
-
-from tensorflow_estimator.contrib.estimator.python.estimator.dnn_linear_combined import *
-- 
GitLab


From e4149e99dd0767a3ce6f7c240b93eaf5e0843e22 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 24 Nov 2018 10:35:52 -0800
Subject: [PATCH 0748/1554] Remove LinearEstimator from contrib

PiperOrigin-RevId: 222691162
---
 tensorflow/contrib/estimator/BUILD            | 12 -------
 tensorflow/contrib/estimator/__init__.py      |  1 -
 .../estimator/python/estimator/linear.py      | 32 -------------------
 3 files changed, 45 deletions(-)
 delete mode 100644 tensorflow/contrib/estimator/python/estimator/linear.py

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index a8d3a7d9f3..a888379f13 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -24,7 +24,6 @@ py_library(
         ":extenders",
         ":head",
         ":hooks",
-        ":linear",
         ":logit_fns",
         ":multi_head",
         ":replicate_model_fn",
@@ -131,17 +130,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "linear",
-    srcs = ["python/estimator/linear.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":expect_tensorflow_estimator_installed",
-        "//tensorflow/python/estimator",
-        "//tensorflow/python/estimator:linear",
-    ],
-)
-
 py_library(
     name = "logit_fns",
     srcs = [
diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py
index d8e13cb793..7d61247e7e 100644
--- a/tensorflow/contrib/estimator/__init__.py
+++ b/tensorflow/contrib/estimator/__init__.py
@@ -58,7 +58,6 @@ _allowed_symbols = [
     'multi_label_head',
     'poisson_regression_head',
     'regression_head',
-    'LinearEstimator',
     'boosted_trees_classifier_train_in_memory',
     'boosted_trees_regressor_train_in_memory',
     'call_logit_fn',
diff --git a/tensorflow/contrib/estimator/python/estimator/linear.py b/tensorflow/contrib/estimator/python/estimator/linear.py
deleted file mode 100644
index b6a4444f66..0000000000
--- a/tensorflow/contrib/estimator/python/estimator/linear.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""linear python module.
-
-Importing from tensorflow.python.estimator is unsupported
-and will soon break!
-"""
-# pylint: disable=unused-import,g-bad-import-order,g-import-not-at-top,wildcard-import
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow_estimator.contrib.estimator.python.estimator import linear
-
-# Include attrs that start with single underscore.
-_HAS_DYNAMIC_ATTRIBUTES = True
-linear.__all__ = [s for s in dir(linear) if not s.startswith('__')]
-
-from tensorflow_estimator.contrib.estimator.python.estimator.linear import *
-- 
GitLab


From 31ac32eb03415c31066ce5b936c80934d83d84ed Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Sun, 25 Nov 2018 00:47:04 -0800
Subject: [PATCH 0749/1554] - Adds `MeanSquaredError` V2 loss implementation -
 Adds support for the V2 losses in Keras. With the new losses the default loss
 reduction function in Keras has been changed from `weighted_mean` to
 `sum_over_batch_size`.

PiperOrigin-RevId: 222720535
---
 .../contrib/keras/api/keras/utils/__init__.py |   1 +
 tensorflow/python/keras/BUILD                 |   1 +
 tensorflow/python/keras/engine/training.py    |  29 ++-
 .../python/keras/engine/training_eager.py     |  31 ++-
 .../python/keras/engine/training_test.py      |  32 ++-
 .../python/keras/engine/training_utils.py     |  20 +-
 tensorflow/python/keras/losses.py             | 103 +++++++++
 tensorflow/python/keras/losses_test.py        |  96 ++++++++
 tensorflow/python/keras/metrics.py            |  73 +-----
 tensorflow/python/keras/utils/__init__.py     |   1 +
 tensorflow/python/keras/utils/losses_utils.py | 213 ++++++++++++++++++
 tensorflow/python/ops/losses/losses_impl.py   |  29 +--
 .../v1/tensorflow.losses.-reduction.pbtxt     |   1 -
 .../tensorflow.keras.losses.-reduction.pbtxt  |  28 +++
 .../golden/v2/tensorflow.keras.losses.pbtxt   |   4 +
 .../v2/tensorflow.losses.-reduction.pbtxt     |   4 +-
 16 files changed, 547 insertions(+), 119 deletions(-)
 create mode 100644 tensorflow/python/keras/utils/losses_utils.py
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt

diff --git a/tensorflow/contrib/keras/api/keras/utils/__init__.py b/tensorflow/contrib/keras/api/keras/utils/__init__.py
index 47cd01b924..3b9fa1b230 100644
--- a/tensorflow/contrib/keras/api/keras/utils/__init__.py
+++ b/tensorflow/contrib/keras/api/keras/utils/__init__.py
@@ -30,6 +30,7 @@ from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.io_utils import HDF5Matrix
 from tensorflow.python.keras.utils.layer_utils import convert_all_kernels_in_model
+from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.keras.utils.np_utils import normalize
 from tensorflow.python.keras.utils.np_utils import to_categorical
 from tensorflow.python.keras.utils.vis_utils import plot_model
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 540dd03768..fa1cad2359 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -142,6 +142,7 @@ py_library(
         "regularizers.py",
         "utils/data_utils.py",
         "utils/io_utils.py",
+        "utils/losses_utils.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 888d8eb942..8c564ed61b 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -41,6 +41,8 @@ from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.engine.network import Network
 from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
+from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
@@ -568,16 +570,16 @@ class Model(Network):
               '" missing from loss dictionary. We assume '
               'this was done on purpose. The fit and evaluate APIs will not be '
               'expecting any data to be passed to "' + name + '".')
-        loss_functions.append(losses.get(loss.get(name)))
+        loss_functions.append(training_utils.get_loss_function(loss.get(name)))
     elif isinstance(loss, list):
       if len(loss) != len(self.outputs):
         raise ValueError('When passing a list as loss, '
                          'it should have one entry per model outputs. '
                          'The model has ' + str(len(self.outputs)) +
                          ' outputs, but you passed loss=' + str(loss))
-      loss_functions = [losses.get(l) for l in loss]
+      loss_functions = [training_utils.get_loss_function(l) for l in loss]
     else:
-      loss_function = losses.get(loss)
+      loss_function = training_utils.get_loss_function(loss)
       loss_functions = [loss_function for _ in range(len(self.outputs))]
     self.loss_functions = loss_functions
 
@@ -730,8 +732,21 @@ class Model(Network):
           mask = masks[i]
           loss_weight = loss_weights_list[i]
           with K.name_scope(self.output_names[i] + '_loss'):
-            weighted_loss = training_utils.weighted_masked_objective(loss_fn)
-            output_loss = weighted_loss(y_true, y_pred, sample_weight, mask)
+            if isinstance(loss_fn, losses.Loss):
+              if mask is not None:
+                mask = math_ops.cast(mask, y_pred.dtype)
+                # Update weights with mask.
+                if sample_weight is None:
+                  sample_weight = mask
+                else:
+                  # Update dimensions of weights to match with mask if possible.
+                  mask, _, sample_weight = squeeze_or_expand_dimensions(
+                      mask, None, sample_weight)
+                  sample_weight *= mask
+              output_loss = loss_fn(y_true, y_pred, sample_weight=sample_weight)
+            else:
+              weighted_loss = training_utils.weighted_masked_objective(loss_fn)
+              output_loss = weighted_loss(y_true, y_pred, sample_weight, mask)
 
           if len(self.outputs) > 1:
             # Keep track of the un-aggregated loss result tensor.
@@ -739,8 +754,10 @@ class Model(Network):
                                           '_loss'] = output_loss
 
             # Keep track of stateful result tensor and function for the loss.
+            loss_name = loss_fn.name if isinstance(
+                loss_fn, losses.Loss) else loss_fn.__name__
             mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-                loss_fn, name=loss_fn.__name__)
+                loss_fn, name=loss_name)
             result_tensor = training_utils.call_metric_function(
                 mean_wrapped_loss,
                 y_true,
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index b2dace84aa..cd85c365db 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -31,9 +31,11 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import callbacks as cbks
+from tensorflow.python.keras import losses as losses_module
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import generic_utils
+from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 
@@ -128,11 +130,24 @@ def _model_loss(model,
       else:
         weights = None
       mask = masks[i]
-
-      weighted_masked_fn = training_utils.weighted_masked_objective(loss_fn)
       with backend.name_scope(model.output_names[i] + '_loss'):
-        output_loss = weighted_masked_fn(
-            targets[i], outs[i], weights, mask=mask)
+        if isinstance(loss_fn, losses_module.Loss):
+          if mask is not None:
+            mask = math_ops.cast(mask, outs[i].dtype)
+            # Update weights with mask.
+            if weights is None:
+              weights = mask
+            else:
+              # Update dimensions of weights to match with mask if possible.
+              mask, _, weights = squeeze_or_expand_dimensions(
+                  mask, None, weights)
+              weights *= mask
+          output_loss = loss_fn(targets[i], outs[i], sample_weight=weights)
+        else:
+          weighted_masked_fn = training_utils.weighted_masked_objective(loss_fn)
+          output_loss = weighted_masked_fn(
+              targets[i], outs[i], weights, mask=mask)
+
       # If the number of outputs is 1 then we don't append the loss metric
       # associated with each model output. When there are multiple outputs
       # associated with a model, each output's loss is calculated and returned
@@ -351,8 +366,10 @@ def iterator_test_loop(model, inputs, steps, verbose=0):
   output_loss_metrics = []
   for i in range(len(model.outputs)):
     loss_fn = model.loss_functions[i]
+    loss_name = loss_fn.name if isinstance(
+        loss_fn, losses_module.Loss) else loss_fn.__name__
     mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-        loss_fn, name=loss_fn.__name__)
+        loss_fn, name=loss_name)
     output_loss_metrics.append(mean_wrapped_loss)
 
   num_samples = 0
@@ -744,8 +761,10 @@ def fit_loop(model,
     output_loss_metrics = []
     for i in range(len(model.outputs)):
       loss_fn = model.loss_functions[i]
+      loss_name = loss_fn.name if isinstance(
+          loss_fn, losses_module.Loss) else loss_fn.__name__
       mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-          loss_fn, name=loss_fn.__name__)
+          loss_fn, name=loss_name)
       output_loss_metrics.append(mean_wrapped_loss)
 
     callbacks.on_train_begin()
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 1009ef7138..97dfe6d900 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -600,6 +600,34 @@ class TrainingTest(test.TestCase):
           np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10)
     self.assertTrue('Epoch 5/10' in mock_stdout.getvalue())
 
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_training_with_loss_instance(self):
+    a = keras.layers.Input(shape=(3,), name='input_a')
+    b = keras.layers.Input(shape=(3,), name='input_b')
+
+    dense = keras.layers.Dense(4, name='dense')
+    c = dense(a)
+    d = dense(b)
+    e = keras.layers.Dropout(0.5, name='dropout')(c)
+
+    model = keras.models.Model([a, b], [d, e])
+    loss_weights = [1., 0.5]
+    model.compile(
+        RMSPropOptimizer(learning_rate=0.001),
+        loss=keras.losses.MeanSquaredError(),
+        metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
+        loss_weights=loss_weights)
+
+    input_a_np = np.random.random((10, 3))
+    input_b_np = np.random.random((10, 3))
+
+    output_d_np = np.random.random((10, 4))
+    output_e_np = np.random.random((10, 4))
+
+    model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
+              epochs=1,
+              batch_size=5)
+
 
 class TestExceptionsAndWarnings(test.TestCase):
 
@@ -1918,7 +1946,7 @@ class TestTrainingWithMetrics(test.TestCase):
 
     w = np.array([[3., 4.], [1., 2.]])
     outs = model.evaluate(x, y, sample_weight=w)
-    self.assertArrayNear(outs, [0.3, 0.7, 0.3], .001)
+    self.assertArrayNear(outs, [0.75, 0.7, 0.3], .001)
 
     # Verify that metric value is same with arbitrary weights and batch size.
     x = np.random.random((50, 2, 1))
@@ -1988,7 +2016,7 @@ class TestTrainingWithMetrics(test.TestCase):
       # verify that masking is combined with sample weights.
       w = np.array([3, 2, 4])
       scores = model.train_on_batch(x, y, sample_weight=w)
-      self.assertArrayNear(scores, [0.2, 0.8], 0.1)
+      self.assertArrayNear(scores, [0.3328, 0.8], 0.001)
 
   def test_add_metric_with_tensor_on_model_in_graph_mode(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 1735db8b6b..347582aa95 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -35,6 +35,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import losses
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
@@ -632,15 +633,14 @@ def weighted_masked_objective(fn):
         weights = mask
       else:
         # Update dimensions of weights to match with mask if possible.
-        mask, _, weights = metrics_module.squeeze_or_expand_dimensions(
-            mask, None, weights)
+        mask, _, weights = squeeze_or_expand_dimensions(mask, None, weights)
         weights *= mask
 
     # Apply sample weighting.
     if weights is not None:
 
       # Update dimensions of weights to match with values if possible.
-      score_array, _, weights = metrics_module.squeeze_or_expand_dimensions(
+      score_array, _, weights = squeeze_or_expand_dimensions(
           score_array, None, weights)
       try:
         # Broadcast weights if possible.
@@ -838,12 +838,22 @@ def call_metric_function(metric_fn, y_true, y_pred, weights=None, mask=None):
     return metric_fn(y_true, y_pred, sample_weight=mask)
 
   # Update dimensions of weights to match with mask.
-  mask, _, weights = metrics_module.squeeze_or_expand_dimensions(
-      mask, None, weights)
+  mask, _, weights = squeeze_or_expand_dimensions(mask, None, weights)
   weights *= mask
   return metric_fn(y_true, y_pred, sample_weight=weights)
 
 
+def get_loss_function(loss):
+  """Returns the loss function corresponding to the given loss input."""
+  if loss is None or isinstance(loss, losses.Loss):
+    return loss
+
+  # TODO(psv): After we have added all V2 losses, update this function.
+  if loss in ['mse', 'MSE', 'mean_squared_error']:
+    return losses.MeanSquaredError()
+  return losses.get(loss)
+
+
 def validate_iterator_input(x, y, sample_weight, validation_split=None):
   """Validates user input arguments when a dataset iterator is passed.
 
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index f871ee409e..0e274d4d50 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -19,17 +19,120 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import abc
+
 import six
 
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss
+from tensorflow.python.keras.utils.losses_utils import ReductionV2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.util.tf_export import tf_export
 
 
+class Loss(object):
+  """Loss base class.
+
+  To be implemented by subclasses:
+  * `call()`: Contains the logic for loss calculation using `y_true`, `y_pred`.
+
+  Example subclass implementation:
+  ```
+  class MeanSquaredError(Loss):
+    def call(self, y_true, y_pred):
+      y_pred = ops.convert_to_tensor(y_pred)
+      y_true = math_ops.cast(y_true, y_pred.dtype)
+      return K.mean(math_ops.square(y_pred - y_true), axis=-1)
+  ```
+
+  Args:
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self, reduction=ReductionV2.SUM_OVER_BATCH_SIZE, name=None):
+    self.reduction = reduction
+    self.name = name
+
+  def __call__(self, y_true, y_pred, sample_weight=None):
+    """Invokes the `Loss` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional `Tensor` whose rank is either 0, or the same rank
+        as `y_true`, or is broadcastable to `y_true`. `sample_weight` acts as a
+        coefficient for the loss. If a scalar is provided, then the loss is
+        simply scaled by the given value. If `sample_weight` is a tensor of size
+        `[batch_size]`, then the total loss for each sample of the batch is
+        rescaled by the corresponding element in the `sample_weight` vector. If
+        the shape of `sample_weight` matches the shape of `y_pred`, then the
+        loss of each measurable element of `y_pred` is scaled by the
+        corresponding value of `sample_weight`.
+
+    Returns:
+      Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same
+        shape as `y_true`; otherwise, it is scalar.
+
+    Raises:
+      ValueError: If the shape of `sample_weight` is invalid.
+    """
+    with ops.name_scope(self.name, format(self.__class__.__name__),
+                        (y_pred, y_true, sample_weight)):
+      losses = self.call(y_true, y_pred)
+      return compute_weighted_loss(
+          losses, sample_weight, reduction=self.reduction)
+
+  @classmethod
+  def from_config(cls, config):
+    """Instantiates a `Loss` from its config (output of `get_config()`).
+
+    Args:
+        config: Output of `get_config()`.
+
+    Returns:
+        A `Loss` instance.
+    """
+    return cls(**config)
+
+  def get_config(self):
+    return {'reduction': self.reduction, 'name': self.name}
+
+  @abc.abstractmethod
+  def call(self, y_true, y_pred):
+    """Invokes the `Loss` instance.
+
+    Args:
+      y_true: Ground truth values, with the same shape as 'y_pred'.
+      y_pred: The predicted values.
+    """
+    NotImplementedError('Must be implemented in subclasses.')
+
+
+class MeanSquaredError(Loss):
+  """Computes the mean of squares of errors between labels and predictions."""
+
+  def call(self, y_true, y_pred):
+    """Invokes the `MeanSquaredError` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Mean squared error losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return mean_squared_error(y_true, y_pred)
+
+
 @tf_export('keras.metrics.mean_squared_error',
            'keras.metrics.mse',
            'keras.metrics.MSE',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index c7015270ac..b056f920ab 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -24,6 +24,9 @@ import shutil
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 try:
@@ -138,5 +141,98 @@ class KerasLossesTest(test.TestCase):
         loaded_model.predict(np.random.rand(128, 2))
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MeanSquaredErrorTest(test.TestCase):
+
+  def test_config(self):
+    mse_obj = keras.losses.MeanSquaredError(
+        reduction=keras.losses.ReductionV2.SUM, name='mse_1')
+    self.assertEqual(mse_obj.name, 'mse_1')
+    self.assertEqual(mse_obj.reduction, keras.losses.ReductionV2.SUM)
+
+  def test_all_correct_unweighted(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3))
+    loss = mse_obj(y_true, y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mse_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 49.5, 3)
+
+  def test_scalar_weighted(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mse_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 113.85, 3)
+
+  def test_sample_weighted(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = mse_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 767.8 / 6, 3)
+
+  def test_timestep_weighted(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3, 1),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    loss = mse_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 587 / 6, 3)
+
+  def test_zero_weighted(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mse_obj(y_true, y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_invalid_sample_weight(self):
+    mse_obj = keras.losses.MeanSquaredError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3, 1))
+    sample_weight = constant_op.constant([3, 6, 5, 0], shape=(2, 2))
+    with self.assertRaisesRegexp(
+        ValueError, r'Shapes \(2, 2\) and \(2, 3\) are incompatible'):
+      mse_obj(y_true, y_pred, sample_weight=sample_weight)
+
+  def test_no_reduction(self):
+    mse_obj = keras.losses.MeanSquaredError(
+        reduction=keras.losses.ReductionV2.NONE)
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mse_obj(y_true, y_pred, sample_weight=2.3)
+    loss = self.evaluate(loss)
+    self.assertArrayNear(loss, [84.3333, 143.3666], 1e-3)
+
+  def test_sum_reduction(self):
+    mse_obj = keras.losses.MeanSquaredError(
+        reduction=keras.losses.ReductionV2.SUM)
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mse_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 227.69998, 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 668c56243b..1ddeb0bee7 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -48,9 +48,9 @@ from tensorflow.python.keras.losses import sparse_categorical_crossentropy
 from tensorflow.python.keras.losses import squared_hinge
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
@@ -172,77 +172,6 @@ def weakmethod(method):
   return inner
 
 
-def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
-  """Squeeze or expand last dimension if needed.
-
-  1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1
-  (using `confusion_matrix.remove_squeezable_dimensions`).
-  2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1
-  from the new rank of `y_pred`.
-  If `sample_weight` is scalar, it is kept scalar.
-
-  This will use static shape if available. Otherwise, it will add graph
-  operations, which could result in a performance hit.
-
-  Args:
-    y_pred: Predicted values, a `Tensor` of arbitrary dimensions.
-    y_true: Optional label `Tensor` whose dimensions match `y_pred`.
-    sample_weight: Optional weight scalar or `Tensor` whose dimensions match
-      `y_pred`.
-
-  Returns:
-    Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has
-    the last dimension squeezed,
-    `sample_weight` could be extended by one dimension.
-  """
-  if y_true is not None:
-    # squeeze last dim of `y_pred` or `y_true` if their rank differs by 1
-    y_true, y_pred = confusion_matrix.remove_squeezable_dimensions(
-        y_true, y_pred)
-
-  if sample_weight is None:
-    return y_pred, y_true, None
-
-  sample_weight = ops.convert_to_tensor(sample_weight)
-  weights_shape = sample_weight.get_shape()
-  weights_rank = weights_shape.ndims
-  if weights_rank == 0:  # If weights is scalar, do nothing.
-    return y_pred, y_true, sample_weight
-
-  y_pred_shape = y_pred.get_shape()
-  y_pred_rank = y_pred_shape.ndims
-  if (y_pred_rank is not None) and (weights_rank is not None):
-    # Use static rank.
-    if weights_rank - y_pred_rank == 1:
-      sample_weight = array_ops.squeeze(sample_weight, [-1])
-    elif y_pred_rank - weights_rank == 1:
-      sample_weight = array_ops.expand_dims(sample_weight, [-1])
-    return y_pred, y_true, sample_weight
-
-  # Use dynamic rank.
-  weights_rank_tensor = array_ops.rank(sample_weight)
-  rank_diff = weights_rank_tensor - array_ops.rank(y_pred)
-  maybe_squeeze_weights = lambda: array_ops.squeeze(sample_weight, [-1])
-
-  def _maybe_expand_weights():
-    return control_flow_ops.cond(
-        math_ops.equal(rank_diff,
-                       -1), lambda: array_ops.expand_dims(sample_weight, [-1]),
-        lambda: sample_weight)
-
-  def _maybe_adjust_weights():
-    return control_flow_ops.cond(
-        math_ops.equal(rank_diff, 1), maybe_squeeze_weights,
-        _maybe_expand_weights)
-
-  # squeeze or expand last dim of `sample_weight` if its rank differs by 1
-  # from the new rank of `y_pred`.
-  sample_weight = control_flow_ops.cond(
-      math_ops.equal(weights_rank_tensor, 0), lambda: sample_weight,
-      _maybe_adjust_weights)
-  return y_pred, y_true, sample_weight
-
-
 class _ConfusionMatrix(Enum):
   TRUE_POSITIVES = 'tp'
   FALSE_POSITIVES = 'fp'
diff --git a/tensorflow/python/keras/utils/__init__.py b/tensorflow/python/keras/utils/__init__.py
index 8939044f71..61940ad789 100644
--- a/tensorflow/python/keras/utils/__init__.py
+++ b/tensorflow/python/keras/utils/__init__.py
@@ -34,6 +34,7 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.io_utils import HDF5Matrix
 from tensorflow.python.keras.utils.layer_utils import convert_all_kernels_in_model
 from tensorflow.python.keras.utils.layer_utils import get_source_inputs
+from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.keras.utils.multi_gpu_utils import multi_gpu_model
 from tensorflow.python.keras.utils.np_utils import normalize
 from tensorflow.python.keras.utils.np_utils import to_categorical
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
new file mode 100644
index 0000000000..d11d785356
--- /dev/null
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -0,0 +1,213 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=protected-access
+"""Utilities related to loss functions."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend as K
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import confusion_matrix
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import weights_broadcast_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export('losses.Reduction', 'keras.losses.Reduction', v1=[])
+class ReductionV2(object):
+  """Types of loss reduction.
+
+  Contains the following values:
+  `NONE`: Un-reduced weighted losses with the same shape as input.
+  `SUM`: Scalar sum of weighted losses.
+  `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+  """
+
+  NONE = None
+  SUM = 'sum'
+  SUM_OVER_BATCH_SIZE = 'sum_over_batch_size'
+
+  @classmethod
+  def all(cls):
+    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
+
+  @classmethod
+  def validate(cls, key):
+    if key not in cls.all():
+      raise ValueError('Invalid Reduction Key %s.' % key)
+
+
+def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
+  """Squeeze or expand last dimension if needed.
+
+  1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1
+  (using `confusion_matrix.remove_squeezable_dimensions`).
+  2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1
+  from the new rank of `y_pred`.
+  If `sample_weight` is scalar, it is kept scalar.
+
+  This will use static shape if available. Otherwise, it will add graph
+  operations, which could result in a performance hit.
+
+  Args:
+    y_pred: Predicted values, a `Tensor` of arbitrary dimensions.
+    y_true: Optional label `Tensor` whose dimensions match `y_pred`.
+    sample_weight: Optional weight scalar or `Tensor` whose dimensions match
+      `y_pred`.
+
+  Returns:
+    Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has
+    the last dimension squeezed,
+    `sample_weight` could be extended by one dimension.
+  """
+  if y_true is not None:
+    # squeeze last dim of `y_pred` or `y_true` if their rank differs by 1
+    y_true, y_pred = confusion_matrix.remove_squeezable_dimensions(
+        y_true, y_pred)
+
+  if sample_weight is None:
+    return y_pred, y_true, None
+
+  sample_weight = ops.convert_to_tensor(sample_weight)
+  weights_shape = sample_weight.get_shape()
+  weights_rank = weights_shape.ndims
+  if weights_rank == 0:  # If weights is scalar, do nothing.
+    return y_pred, y_true, sample_weight
+
+  y_pred_shape = y_pred.get_shape()
+  y_pred_rank = y_pred_shape.ndims
+  if (y_pred_rank is not None) and (weights_rank is not None):
+    # Use static rank.
+    if weights_rank - y_pred_rank == 1:
+      sample_weight = array_ops.squeeze(sample_weight, [-1])
+    elif y_pred_rank - weights_rank == 1:
+      sample_weight = array_ops.expand_dims(sample_weight, [-1])
+    return y_pred, y_true, sample_weight
+
+  # Use dynamic rank.
+  weights_rank_tensor = array_ops.rank(sample_weight)
+  rank_diff = weights_rank_tensor - array_ops.rank(y_pred)
+  maybe_squeeze_weights = lambda: array_ops.squeeze(sample_weight, [-1])
+
+  def _maybe_expand_weights():
+    return control_flow_ops.cond(
+        math_ops.equal(rank_diff,
+                       -1), lambda: array_ops.expand_dims(sample_weight, [-1]),
+        lambda: sample_weight)
+
+  def _maybe_adjust_weights():
+    return control_flow_ops.cond(
+        math_ops.equal(rank_diff, 1), maybe_squeeze_weights,
+        _maybe_expand_weights)
+
+  # squeeze or expand last dim of `sample_weight` if its rank differs by 1
+  # from the new rank of `y_pred`.
+  sample_weight = control_flow_ops.cond(
+      math_ops.equal(weights_rank_tensor, 0), lambda: sample_weight,
+      _maybe_adjust_weights)
+  return y_pred, y_true, sample_weight
+
+
+def _safe_mean(losses, num_present):
+  """Computes a safe mean of the losses.
+
+  Args:
+    losses: `Tensor` whose elements contain individual loss measurements.
+    num_present: The number of measurable elements in `losses`.
+
+  Returns:
+    A scalar representing the mean of `losses`. If `num_present` is zero,
+      then zero is returned.
+  """
+  total_loss = math_ops.reduce_sum(losses)
+  return math_ops.div_no_nan(total_loss, num_present, name='value')
+
+
+def _num_elements(losses):
+  """Computes the number of elements in `losses` tensor."""
+  with ops.name_scope(None, 'num_elements', values=[losses]) as scope:
+    return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype)
+
+
+def _reduce_weighted_loss(weighted_losses,
+                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE):
+  """Reduces the individual weighted loss measurements."""
+  if reduction == ReductionV2.NONE:
+    loss = weighted_losses
+  else:
+    loss = math_ops.reduce_sum(weighted_losses)
+    if reduction == ReductionV2.SUM_OVER_BATCH_SIZE:
+      loss = _safe_mean(loss, _num_elements(weighted_losses))
+  return loss
+
+
+def compute_weighted_loss(losses,
+                          sample_weight=None,
+                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE,
+                          name=None):
+  """Computes the weighted loss.
+
+  Args:
+    losses: `Tensor` of shape `[batch_size, d1, ... dN]`.
+    sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
+      `losses`, or be broadcastable to `losses`.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+
+  Raises:
+    ValueError: If the shape of `sample_weight` is not compatible with `losses`.
+
+  Returns:
+    Weighted loss `Tensor` of the same type as `losses`. If `reduction` is
+    `NONE`, this has the same shape as `losses`; otherwise, it is scalar.
+  """
+  ReductionV2.validate(reduction)
+  if sample_weight is None:
+    sample_weight = 1.0
+  with ops.name_scope(name, 'weighted_loss', (losses, sample_weight)):
+    # Save the `reduction` argument for loss normalization when distributing
+    # to multiple replicas.
+    # TODO(josh11b): Associate it with the returned op for more precision.
+    ops.get_default_graph()._last_loss_reduction = reduction  # pylint: disable=protected-access
+
+    # Update dimensions of `sample_weight` to match with `losses` if possible.
+    losses, _, sample_weight = squeeze_or_expand_dimensions(
+        losses, None, sample_weight)
+    losses = ops.convert_to_tensor(losses)
+    input_dtype = losses.dtype
+    losses = math_ops.to_float(losses)
+    sample_weight = math_ops.to_float(sample_weight)
+
+    try:
+      # Broadcast weights if possible.
+      sample_weight = weights_broadcast_ops.broadcast_weights(
+          sample_weight, losses)
+    except ValueError:
+      # Reduce values to same ndim as weight array.
+      ndim = K.ndim(losses)
+      weight_ndim = K.ndim(sample_weight)
+      losses = K.mean(losses, axis=list(range(weight_ndim, ndim)))
+
+    sample_weight.get_shape().assert_is_compatible_with(losses.get_shape())
+    weighted_losses = math_ops.multiply(losses, sample_weight)
+    # Apply reduction function to the individual weighted losses.
+    loss = _reduce_weighted_loss(weighted_losses, reduction)
+    # Convert the result back to the input type.
+    loss = math_ops.cast(loss, input_dtype)
+    return loss
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 0a5b511f82..7c52b28b39 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -33,32 +33,8 @@ from tensorflow.python.util.deprecation import deprecated_argument_lookup
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("losses.Reduction", v1=[])
-class ReductionV2(object):
-  """Types of loss reduction.
-
-  Contains the following values:
-  `NONE`: Un-reduced weighted losses with the same shape as input.
-  `SUM`: Scalar sum of weighted losses.
-  `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
-  """
-
-  NONE = "none"
-  SUM = "weighted_sum"
-  SUM_OVER_BATCH_SIZE = "weighted_sum_over_batch_size"
-
-  @classmethod
-  def all(cls):
-    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
-
-  @classmethod
-  def validate(cls, key):
-    if key not in cls.all():
-      raise ValueError("Invalid Reduction Key %s." % key)
-
-
 @tf_export(v1=["losses.Reduction"])
-class Reduction(ReductionV2):
+class Reduction(object):
   """Types of loss reduction.
 
   Contains the following values:
@@ -71,6 +47,9 @@ class Reduction(ReductionV2):
   `SUM_BY_NONZERO_WEIGHTS`: Same as `SUM_OVER_NONZERO_WEIGHTS`.
   """
 
+  NONE = "none"
+  SUM = "weighted_sum"
+  SUM_OVER_BATCH_SIZE = "weighted_sum_over_batch_size"
   MEAN = "weighted_mean"
   SUM_BY_NONZERO_WEIGHTS = "weighted_sum_by_nonzero_weights"
   SUM_OVER_NONZERO_WEIGHTS = SUM_BY_NONZERO_WEIGHTS
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt
index b2adb52660..258ad5047e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.losses.-reduction.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.losses.Reduction"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.Reduction\'>"
-  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "MEAN"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
new file mode 100644
index 0000000000..031d9b171f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
@@ -0,0 +1,28 @@
+path: "tensorflow.keras.losses.Reduction"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.utils.losses_utils.ReductionV2\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "NONE"
+    mtype: "<type \'NoneType\'>"
+  }
+  member {
+    name: "SUM"
+    mtype: "<type \'str\'>"
+  }
+  member {
+    name: "SUM_OVER_BATCH_SIZE"
+    mtype: "<type \'str\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "all"
+    argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "validate"
+    argspec: "args=[\'cls\', \'key\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index eca6b91538..8618c6f1c7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "Reduction"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "KLD"
     argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
index 6a44e4ce66..ad72e3194a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
@@ -1,10 +1,10 @@
 path: "tensorflow.losses.Reduction"
 tf_class {
-  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
+  is_instance: "<class \'tensorflow.python.keras.utils.losses_utils.ReductionV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "NONE"
-    mtype: "<type \'str\'>"
+    mtype: "<type \'NoneType\'>"
   }
   member {
     name: "SUM"
-- 
GitLab


From 8a89ca38630943d8658bb1bee1526cdf1e5c615e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 25 Nov 2018 01:02:13 -0800
Subject: [PATCH 0750/1554] compat: Update forward compatibility horizon to
 2018-11-25

PiperOrigin-RevId: 222721347
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 969b95dd0d..216ab2c0ac 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 24)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 25)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 09a7d525c8556f6fd90c70f656d131559cf30fd4 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Sun, 25 Nov 2018 14:23:30 -0800
Subject: [PATCH 0751/1554] [XLA:GPU] Convert the reduction implementation to
 use tiling scheme.

Convert the implementation of scalar reduction, row reduction and column
reduction to use EmitTiledKernel, which is a more general kernel tiling
implementation that is based on the information defined by an object of
TilingScheme. For scalar reduction and row reduction, the new implementation
should generate the same optimized code as the old implementation.

For column reduction, the old implementation in routine
IrEmitterUnnested::EmitColumnReduction uses kTileWidth=2 so that one thread
computes the partial results for two elements in the output of each kReduce
instruction. The new implementation is equivalent to the old implementation
with kTileWidth=1 in this regard.
PiperOrigin-RevId: 222752674
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 1749 +++++++----------
 .../xla/service/gpu/ir_emitter_unnested.h     |  107 +-
 .../xla/service/llvm_ir/kernel_tiling.cc      |    2 +-
 .../xla/service/llvm_ir/kernel_tiling.h       |   17 +-
 4 files changed, 708 insertions(+), 1167 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 52f0ba7aa7..bbe1583c01 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h"
 
 #include "absl/algorithm/container.h"
-#include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/optional.h"
@@ -548,91 +547,7 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
           // TODO(b/112040122): Support variadic reduce.
           return Unimplemented("Variadic reduce is not supported on GPU");
         }
-        VLOG(3) << "Emitting fused reduction to vector: " << fusion->ToString();
-        std::vector<std::unique_ptr<Thunk>> thunks;
-        absl::Span<HloInstruction* const> output_instructions =
-            root->opcode() == HloOpcode::kTuple
-                ? root->operands()
-                : absl::Span<HloInstruction* const>(&root, 1);
-
-        // For multi-output fusion emit an initializer for each tuple element.
-        // Otherwise it's sufficient to just initialize the single output.
-        HloInstruction* first_reduce = nullptr;
-        for (int i = 0, e = output_instructions.size(); i != e; ++i) {
-          if (output_instructions[i]->opcode() == HloOpcode::kReduce) {
-            TF_ASSIGN_OR_RETURN(
-                std::unique_ptr<Thunk> initializer_thunk,
-                BuildInitializerThunk(fusion, output_instructions[i] == root
-                                                  ? ShapeIndex()
-                                                  : ShapeIndex({i})));
-            thunks.push_back(std::move(initializer_thunk));
-            first_reduce =
-                first_reduce == nullptr ? output_instructions[i] : first_reduce;
-          }
-        }
-        CHECK(first_reduce != nullptr);
-        std::unique_ptr<KernelThunk> kernel_thunk =
-            BuildKernelThunk(fusion, /*implements_whole_instruction=*/false);
-        GpuElementalIrEmitter elemental_emitter(
-            hlo_module_config_, ir_emitter_context_->llvm_module(), &b_,
-            GetNestedComputer());
-        FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(fusion),
-                                     &elemental_emitter);
-        TF_RETURN_IF_ERROR(root->Accept(&fused_emitter));
-
-        // For multi-output fusion CHECK the constraints and feed all the
-        // reduces into a single loop code generator. Single-output reduce
-        // fusion is a special case of that.
-        InlinedVector<llvm_ir::ElementGenerator, 1> input_gens;
-        InlinedVector<llvm_ir::ElementGenerator, 1> init_value_gens;
-        std::vector<std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-            extra_output_gens;
-        InlinedVector<HloComputation*, 1> reducers;
-        InlinedVector<ShapeIndex, 1> reduce_output_shapes;
-        for (int i = 0, e = output_instructions.size(); i != e; ++i) {
-          const HloInstruction* inst = output_instructions[i];
-          ShapeIndex output_shape_index;
-          if (root->opcode() == HloOpcode::kTuple) {
-            output_shape_index = {i};
-          }
-          if (inst->opcode() == HloOpcode::kReduce) {
-            CHECK(IsReductionToVector(*inst))
-                << "Only reductions to vector are supported";
-            // Shapes, layouts and dimensions must be the same for all reduces
-            // inside of this fusion.
-            CHECK(ShapeUtil::Equal(first_reduce->shape(), inst->shape()));
-            CHECK(ShapeUtil::Equal(first_reduce->operand(0)->shape(),
-                                   inst->operand(0)->shape()));
-            CHECK(ShapeUtil::Equal(first_reduce->operand(1)->shape(),
-                                   inst->operand(1)->shape()));
-            CHECK(first_reduce->dimensions() == inst->dimensions());
-            input_gens.push_back(fused_emitter.GetGenerator(inst->operand(0)));
-            init_value_gens.push_back(
-                fused_emitter.GetGenerator(inst->operand(1)));
-            reducers.push_back(inst->to_apply());
-            reduce_output_shapes.push_back(std::move(output_shape_index));
-          } else {
-            // For extra outputs we can relax shape equality to allow different
-            // types (with the same number of elements). Layouts still have to
-            // match.
-            CHECK(ShapeUtil::CompatibleIgnoringElementType(
-                first_reduce->operand(0)->shape(), inst->shape()));
-            CHECK(LayoutUtil::Equal(first_reduce->operand(0)->shape().layout(),
-                                    inst->shape().layout()));
-            extra_output_gens.emplace_back(fused_emitter.GetGenerator(inst),
-                                           std::move(output_shape_index));
-          }
-        }
-        const Shape& input_shape = first_reduce->operand(0)->shape();
-        TF_CHECK_OK(EmitReductionToVector(
-            kernel_thunk.get(), first_reduce, input_shape, input_gens,
-            init_value_gens, first_reduce->dimensions(), reducers,
-            reduce_output_shapes, extra_output_gens));
-        thunks.push_back(std::move(kernel_thunk));
-        std::unique_ptr<SequentialThunk> sequential_thunk =
-            absl::make_unique<SequentialThunk>(std::move(thunks), fusion);
-        AddThunkToThunkSequence(std::move(sequential_thunk));
-        return Status::OK();
+        return EmitReductionToVector(fusion);
       }
       default:
         LOG(FATAL) << "Bad opcode for input fusion: "
@@ -702,13 +617,12 @@ Status IrEmitterUnnested::HandleCopy(HloInstruction* copy) {
 }
 
 Status IrEmitterUnnested::EmitExtraOutputsForReduce(
-    const HloInstruction* reduce, const IrArray::Index& index,
+    const HloInstruction* unnested_hlo, const IrArray::Index& index,
     absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
         extra_output_gens) {
   for (int i = 0; i != extra_output_gens.size(); ++i) {
-    const HloInstruction* output = reduce->parent()->FusionInstruction();
     llvm::Value* extra_output_address =
-        GetIrArray(*output, *output, extra_output_gens[i].second)
+        GetIrArray(*unnested_hlo, *unnested_hlo, extra_output_gens[i].second)
             .EmitArrayElementAddress(index, &b_,
                                      "extra_output_element_address");
     TF_ASSIGN_OR_RETURN(llvm::Value* const extra_output_ir_value,
@@ -718,984 +632,13 @@ Status IrEmitterUnnested::EmitExtraOutputsForReduce(
   return Status::OK();
 }
 
-Status IrEmitterUnnested::EmitReductionToScalar(
-    KernelThunk* kernel_thunk, HloInstruction* reduce, const Shape& input_shape,
-    absl::Span<const llvm_ir::ElementGenerator> input_gens,
-    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-    absl::Span<HloComputation* const> reducers,
-    absl::Span<const ShapeIndex> reduce_output_shapes,
-    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-        extra_output_gens) {
-  // Number of elements processed by a single thread.
-  constexpr int64 kTileSize = 16;
-  int64 num_elems = ShapeUtil::ElementsIn(input_shape);
-
-  // Round up the number of tiles to a multiple of the warp size.  This is
-  // necessary for correctness.  We launch one thread per tile, and if the
-  // number of threads isn't a multiple of the number of the warp size, our
-  // shuffles will read from inactive threads, producing undefined values.
-  int64 num_tiles =
-      RoundUpToNearest(CeilOfRatio(num_elems, kTileSize), kWarpSize);
-
-  Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout(
-      reduce->shape().element_type(), {num_tiles}, {0});
-  LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
-      tiled_input_shape, ir_emitter_context_->device_description());
-
-  llvm::Type* index_ty =
-      GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_);
-
-  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
-    return llvm::ConstantInt::get(index_ty, c);
-  };
-
-  // Check whether every thread will process a full tile's worth of elements
-  // without reading outside the bounds of the input.  If this is true, we can
-  // skip some bounds checks in the final algorithm.
-  bool all_threads_in_bounds = num_tiles * kTileSize == num_elems;
-
-  // __global__ void full_reduce_kernel() {
-  //   x_in_tiles = threadIdx.x + blockIdx.x * blockDim.x;
-  //   x = x_in_tiles * kTileSize;
-  //
-  //   partial_result = init_value;
-  //   if (all_threads_in_bounds || x + kTileSize <= num_elems) {
-  //     for (i = 0; i < kTileSize; ++i) {
-  //       partial_result = Reducer(partial_result, input[x + i]);
-  //     }
-  //   } else {
-  //     for (i = 0; i < kTileSize; ++i) {
-  //       if (x + i < num_elems) {
-  //         partial_result = Reducer(partial_result, input[x + i]);
-  //       }
-  //     }
-  //   }
-  //   for (i = warpSize / 2; i > 0; i /= 2) {
-  //     partial_result = Reducer(partial_result,
-  //                              __shfl_down(partial_result, i));
-  //   }
-  //   if (lane_id == 0) {
-  //     AtomicReducer(&output[y], partial_result);
-  //   }
-  // }
-  //
-  // // Choose num_blocks and threads_per_block such that:
-  // //
-  // //   num_blocks * threads_per_block =
-  // //     RoundUpToNextMultipleOf(Ceil(num_elems / kTileSize), warpSize),
-  // //
-  // // and threads_per_block is a multiple of warpSize.
-  // reduce_kernel  //
-  auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status {
-    const int num_reduces = reducers.size();
-    llvm::Type* element_ir_type =
-        llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
-    std::vector<llvm::Value*> partial_reduction_result_addresses;
-    for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* partial_reduction_result_address =
-          Alloca(element_ir_type, /*ArraySize=*/nullptr,
-                 "partial_reduction_result." + llvm::Twine(i));
-      TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
-                          init_value_gens[i](IrArray::Index(index_ty)));
-      Store(init_ir_value, partial_reduction_result_address);
-      partial_reduction_result_addresses.push_back(
-          partial_reduction_result_address);
-    }
-
-    llvm::Value* x_in_tiles = tile_index[0];
-    x_in_tiles = ZExtOrTrunc(x_in_tiles, index_ty);
-
-    // Emit an inner for-loop that reduces the elements in the tile.
-    auto emit_tile_element_loop = [=](bool tile_in_bounds) -> Status {
-      std::unique_ptr<llvm_ir::ForLoop> tile_element_loop =
-          llvm_ir::ForLoop::EmitForLoop(
-              "element_id_in_tile", index_typed_constant(0),
-              index_typed_constant(kTileSize), index_typed_constant(1), &b_);
-
-      // Emit the body of the partial reduction loop.
-      llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
-                                     &b_);
-      llvm::Value* x =
-          NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileSize)),
-                 tile_element_loop->GetIndVarValue());
-      // Unless we know the tile is entirely in bounds, we have to emit a
-      // x-in-bounds check before reading from the input.
-      if (!tile_in_bounds) {
-        llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-            ICmpULT(x, index_typed_constant(num_elems)), "x_in_bounds", &b_);
-
-        // Emit code that reads the input element and accumulates it to
-        // the partial reduction result.
-        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
-      }
-
-      IrArray::Index input_index(
-          /*linear=*/x, input_shape, &b_);
-      llvm::Value* input_address = Alloca(element_ir_type);
-      for (int i = 0; i != num_reduces; ++i) {
-        TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
-                            input_gens[i](input_index));
-        Store(input_ir_value, input_address);
-        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
-            *reducers[i],
-            {partial_reduction_result_addresses[i], input_address},
-            partial_reduction_result_addresses[i]));
-      }
-      return EmitExtraOutputsForReduce(reduce, input_index, extra_output_gens);
-    };
-
-    // x_end = kTileSize + x_in_tiles * kTileSize, i.e., the location that's
-    // immediately beyond the tile.
-    llvm::Value* x_end =
-        NSWAdd(index_typed_constant(kTileSize),
-               NSWMul(x_in_tiles, index_typed_constant(kTileSize)));
-    // The tile is entirely in bound if all_threads_in_bounds or
-    // x_end <= num_elems.
-    llvm::Value* tile_in_bounds =
-        Or(ICmpULE(x_end, index_typed_constant(num_elems)),
-           b_.getInt1(all_threads_in_bounds));
-    llvm_ir::LlvmIfData if_tile_in_bounds_data =
-        llvm_ir::EmitIfThenElse(tile_in_bounds, "tile_in_bounds", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.true_block, &b_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.false_block, &b_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/false));
-
-    // After the if-then-else statement on tile_in_bounds, emit calls to
-    // shfl_down that accumulate the partial reduction results of all threads
-    // from the warp.
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.after_block, &b_);
-    int bit_width = llvm_ir::GetSizeInBits(element_ir_type);
-    // bitcast cannot be applied to aggregate types (even packed ones), so we
-    // instead bitcast addresses of load/store to intN* of the same bit-width.
-    llvm::Type* shuffle_ir_type = element_ir_type->isStructTy()
-                                      ? b_.getIntNTy(bit_width)
-                                      : element_ir_type;
-    for (int shuffle_distance = kWarpSize / 2; shuffle_distance >= 1;
-         shuffle_distance /= 2) {
-      llvm::Value* result_from_other_lane =
-          Alloca(element_ir_type, nullptr, "result_from_other_lane");
-      for (int i = 0; i != num_reduces; ++i) {
-        llvm::Value* partial_reduction_result =
-            Load(BitCast(partial_reduction_result_addresses[i],
-                         shuffle_ir_type->getPointerTo()),
-                 "partial_reduction_result");
-        CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0)
-            << "Requires block size a multiple of the warp size, otherwise we "
-               "will read undefined elements.";
-        Store(EmitFullWarpShuffleDown(partial_reduction_result,
-                                      b_.getInt32(shuffle_distance), &b_),
-              BitCast(result_from_other_lane, shuffle_ir_type->getPointerTo()));
-        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
-            *reducers[i],
-            {partial_reduction_result_addresses[i], result_from_other_lane},
-            partial_reduction_result_addresses[i]));
-      }
-    }
-
-    const HloInstruction* output =
-        reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
-
-    // Emit an atomic operation that accumulates the partial reduction result of
-    // lane 0 (which holds the partially accumulated result for its warp) to the
-    // output element.
-    llvm::Value* lane_id =
-        URem(x_in_tiles, index_typed_constant(kWarpSize), "lane_id");
-    llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
-        ICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
-
-    for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* output_address =
-          GetIrArray(*output, *output, reduce_output_shapes[i])
-              .EmitArrayElementAddress(
-                  IrArray::Index(
-                      /*linear=*/b_.getInt64(0),
-                      ShapeUtil::GetSubshape(output->shape(),
-                                             reduce_output_shapes[i]),
-                      &b_),
-                  &b_, "output_element_address");
-      TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
-          *reducers[i], output_address, partial_reduction_result_addresses[i]));
-    }
-    return Status::OK();
-  };
-
-  // Emit a parallel loop that iterates through all input tiles, one per thread.
-  UpdateLaunchDimensions(launch_dimensions, kernel_thunk,
-                         ir_emitter_context_->llvm_module());
-  return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
-                             launch_dimensions, &b_)
-      .EmitLoop(IrName(reduce), index_ty);
-}
-
-Status IrEmitterUnnested::EmitColumnReduction(
-    KernelThunk* kernel_thunk, int64 height, int64 width,
-    HloInstruction* reduce, const Shape& input_shape,
-    absl::Span<const llvm_ir::ElementGenerator> input_gens,
-    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-    absl::Span<HloComputation* const> reducers,
-    absl::Span<const ShapeIndex> reduce_output_shapes,
-    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-        extra_output_gens) {
-  // Divide the input matrix into tiles of size KxL. For example, when the
-  // input matrix is 4x4, K=2, and L=1 the tiled matrix looks like
-  //
-  //   0123
-  //   0123
-  //   4567
-  //   4567  // Numbers indicate tile IDs.
-  //
-  // Each tile is first partially reduced to a scalar by a thread, and then the
-  // scalar is accumulated to the output vector using atomic operations.
-  //
-  // We choose 128 as the tile size based on empirical evidence. It's big enough
-  // to reduce the amount of atomic adds in the end, maximizing the memory
-  // bandwidth. A tile width of 2 allows for high memory bandwidth utilization
-  // on 16b input data.
-  constexpr int64 kTileHeight = 128;
-  constexpr int64 kTileWidth = 2;
-
-  // If the height is not a multiple of kTileHeight, we pad the bottom of the
-  // input matrix.
-  const int64 height_in_tiles = CeilOfRatio(height, kTileHeight);
-  // If width is not a multiple of kTileWidth the rightmost thread will process
-  // fewer input elements.
-  const int64 width_in_tiles = CeilOfRatio(width, kTileWidth);
-  Shape tiled_input_shape =
-      ShapeUtil::MakeShapeWithLayout(reduce->shape().element_type(),
-                                     {height_in_tiles, width_in_tiles}, {1, 0});
-  LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
-      tiled_input_shape, ir_emitter_context_->device_description());
-
-  // TODO(b/110211620): Convert to use i32 index_type when it is possible.
-  llvm::Type* index_ty = b_.getInt64Ty();
-
-  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
-    return llvm::ConstantInt::get(index_ty, c);
-  };
-
-  // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
-  //      linear_index < height_in_tiles * width_in_tiles;
-  //      linear_index += blockDim.x * gridDim.x) {
-  //   y_in_tiles = linear_index / width_in_tiles;
-  //   x_in_tiles = linear_index % width_in_tiles;
-  //
-  //   partial_results[kTileWidth] = init_values;
-  //   tile_in_y_bounds = height % kTileHeight == 0 ||
-  //       y_in_tiles * kTileHeight + kTileHeight <= height;
-  //   tile_in_x_bounds = width % kTileWidth == 0 ||
-  //       x_in_tiles * kTileWidth + kTileWidth <= width;
-  //   // The implementation handles y and x bound checks separately.
-  //   if (tile_in_y_bounds && tile_in_x_bounds) {
-  //     for (y_offset : range(kTileHeight)) {
-  //       y = y_in_tiles * kTileHeight + y_offset;
-  //       for (x_offset : range(kTileWidth)) {
-  //         x = x_in_tiles * kTileWidth + x_offset;
-  //         partial_result = Reducer(partial_result[x_offset], input[y][x]);
-  //       }
-  //     }
-  //   } else {
-  //     for (y_offset : range(kTileHeight)) {
-  //       y = y_in_tiles * kTileHeight + y_offset;
-  //       for (y_offset : range(kTileHeight)) {
-  //         x = x_in_tiles * kTileWidth + x_offset;
-  //         if (y < height && x < width) {
-  //           partial_result = Reducer(partial_result, input[y][x]);
-  //         }
-  //       }
-  //     }
-  //   }
-  //   for (x_offset : range(kTileWidth)) {
-  //     AtomicReducer(&output[x + x_offset], partial_result[x_offset]);
-  //   }
-  // }
-  auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status {
-    const int num_reduces = reducers.size();
-    // Emit the loop body that reduces one tile.
-    llvm::Type* element_ir_type =
-        llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
-    std::vector<llvm::Value*> partial_reduction_result_addresses;
-    for (int i = 0; i != num_reduces; ++i) {
-      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
-        llvm::Value* partial_reduction_result_address =
-            Alloca(element_ir_type, /*ArraySize=*/nullptr,
-                   "partial_reduction_result." +
-                       llvm::Twine(i * kTileWidth + x_offset));
-        TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
-                            init_value_gens[i](IrArray::Index(index_ty)));
-        Store(init_ir_value, partial_reduction_result_address);
-        partial_reduction_result_addresses.push_back(
-            partial_reduction_result_address);
-      }
-    }
-
-    // Emit an inner for-loop that partially reduces the elements in the given
-    // tile.
-    llvm::Value* y_in_tiles = tile_index[0];
-    llvm::Value* x_in_tiles = tile_index[1];
-
-    y_in_tiles = ZExtOrTrunc(y_in_tiles, index_ty);
-    x_in_tiles = ZExtOrTrunc(x_in_tiles, index_ty);
-
-    auto emit_tile_element_loop = [=](bool tile_in_y_bounds,
-                                      bool tile_in_x_bounds) -> Status {
-      std::unique_ptr<llvm_ir::ForLoop> tile_element_loop =
-          llvm_ir::ForLoop::EmitForLoop(
-              "element_id_in_tile", index_typed_constant(0),
-              index_typed_constant(kTileHeight), index_typed_constant(1), &b_);
-
-      // Emit the body of the partial reduction loop.
-      llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
-                                     &b_);
-      llvm::Value* y =
-          NSWAdd(NSWMul(y_in_tiles, index_typed_constant(kTileHeight)),
-                 tile_element_loop->GetIndVarValue());
-
-      // Unless we know that y is in bounds, we have to emit a check before
-      // reading from the input.
-      if (!tile_in_y_bounds) {
-        llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-            ICmpULT(y, index_typed_constant(height)), "y_in_bounds", &b_);
-
-        // Emit code that reads the input element and accumulates it to
-        // the partial reduction result.
-        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
-      }
-      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
-        llvm::Value* x =
-            NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileWidth)),
-                   index_typed_constant(x_offset));
-        // Unless we know that x is in bounds, we have to emit a check before
-        // reading from the input.
-        if (!tile_in_x_bounds) {
-          llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-              ICmpULT(x, index_typed_constant(width)), "x_in_bounds", &b_);
-          llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
-        }
-        llvm::Value* input_address = Alloca(element_ir_type);
-        // {y,x} is an index to input_matrix_shape [height,width]. We need to
-        // convert that to an index to input_shape (the shape of the operand of
-        // "reduce"). This conversion is composed of a transposition from
-        // input_shape to normalized_input_shape and a reshape from
-        // normalized_input_shape to input_matrix_shape.
-        const Shape normalized_input_shape =
-            ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
-                input_shape);
-        auto input_shape_min2maj = LayoutUtil::MinorToMajor(input_shape);
-        const std::vector<int64> transpose_dimension_mapping(
-            input_shape_min2maj.rbegin(), input_shape_min2maj.rend());
-
-        const Shape input_matrix_shape =
-            ShapeUtil::MakeShapeWithDescendingLayout(input_shape.element_type(),
-                                                     {height, width});
-        const IrArray::Index input_matrix_index({y, x}, input_matrix_shape,
-                                                &b_);
-        const IrArray::Index input_index =
-            input_matrix_index
-                .SourceIndexOfReshape(input_matrix_shape,
-                                      normalized_input_shape, &b_)
-                .SourceIndexOfTranspose(normalized_input_shape, input_shape,
-                                        transpose_dimension_mapping, &b_);
-        for (int i = 0; i != num_reduces; ++i) {
-          TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
-                              input_gens[i](input_index));
-          Store(input_ir_value, input_address);
-          TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
-              *reducers[i],
-              {partial_reduction_result_addresses[i * kTileWidth + x_offset],
-               input_address},
-              partial_reduction_result_addresses[i * kTileWidth + x_offset]));
-          TF_RETURN_IF_ERROR(EmitExtraOutputsForReduce(reduce, input_index,
-                                                       extra_output_gens));
-        }
-      }
-      return Status::OK();
-    };
-
-    // y_end = kTileHeight + y_in_tiles * kTileHeight, i.e., the y location
-    // that's immediately beyond the tile.
-    llvm::Value* y_end =
-        NSWAdd(index_typed_constant(kTileHeight),
-               NSWMul(y_in_tiles, index_typed_constant(kTileHeight)));
-    // x_end = kTileWidth + x_in_tiles * kTileWidth, i.e., the x location
-    // that's immediately beyond the tile.
-    llvm::Value* x_end =
-        NSWAdd(index_typed_constant(kTileWidth),
-               NSWMul(x_in_tiles, index_typed_constant(kTileWidth)));
-    llvm::Value* tile_in_y_bounds =
-        Or(ICmpULE(y_end, index_typed_constant(height)),
-           b_.getInt1(height % kTileHeight == 0));
-    llvm::Value* tile_in_x_bounds =
-        Or(ICmpULE(x_end, index_typed_constant(width)),
-           b_.getInt1(width % kTileWidth == 0));
-    // The tile is in y bounds if "height" is a multiple of kTileHeight or
-    // y_end <= height.
-    llvm_ir::LlvmIfData if_tile_in_y_bounds_data =
-        llvm_ir::EmitIfThenElse(tile_in_y_bounds, "tile_in_y_bounds", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.true_block, &b_);
-    // The tile is in x bounds if "width" is a multiple of kTileWidth or
-    // x_end <= width.
-    llvm_ir::LlvmIfData if_tile_in_x_bounds_data =
-        llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
-                                              /*tile_in_x_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
-                                              /*tile_in_x_bounds=*/false));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.false_block, &b_);
-    if_tile_in_x_bounds_data =
-        llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
-                                              /*tile_in_x_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
-                                              /*tile_in_x_bounds=*/false));
-
-    // After the nested if-then-else statement on tile_in_y_bounds and
-    // tile_in_x_bounds, emit atomic operations to accumulate the partial
-    // reduction result to the output element.
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.after_block, &b_);
-    const HloInstruction* output =
-        reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
-    for (int i = 0; i != num_reduces; ++i) {
-      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
-        llvm::Value* x =
-            NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileWidth)),
-                   index_typed_constant(x_offset));
-        llvm::Value* output_address =
-            GetIrArray(*output, *output, reduce_output_shapes[i])
-                .EmitArrayElementAddress(
-                    IrArray::Index(
-                        x,
-                        ShapeUtil::GetSubshape(output->shape(),
-                                               reduce_output_shapes[i]),
-                        &b_),
-                    &b_, "output_element_address");
-        TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
-            *reducers[i], output_address,
-            partial_reduction_result_addresses[i * kTileWidth + x_offset]));
-      }
-    }
-    return Status::OK();
-  };
-
-  // Emit a parallel loop that iterate through all input tiles.
-  UpdateLaunchDimensions(launch_dimensions, kernel_thunk,
-                         ir_emitter_context_->llvm_module());
-  return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
-                             launch_dimensions, &b_)
-      .EmitLoop(IrName(reduce), index_ty);
-}
-
-static std::pair<int64, int64> ComputeKernelMappingSchemeForReduction(
-    int64 depth, int64 width, int64 kWarpSize) {
-  constexpr int64 kTargetNumElementsPerThread = 64;
-  int64 x_tile_size = kTargetNumElementsPerThread;
-  int64 z_tile_size = 1;
-
-  // Only tile along the x dimension with tile size kTargetNumElementsPerThread
-  // if doing so doesn't require a slow version of loop with bound check on each
-  // dimension. A more sophisticated heuristics is to enable tile along the
-  // x dimension with tile size kTargetNumElementsPerThread when either width is
-  // a factor of (kWarpSize * kTargetNumElementsPerThread) or width is big
-  // enough so that only a small fraction of the threads execute the slow
-  // version of loop with bound check.
-  if (width % (kWarpSize * kTargetNumElementsPerThread) != 0) {
-    x_tile_size = 8;
-    z_tile_size = 8;
-    while (depth % z_tile_size != 0) {
-      z_tile_size -= 1;
-    }
-  }
-
-  return std::pair<int64, int64>(x_tile_size, z_tile_size);
-}
-
-Status IrEmitterUnnested::EmitRowReduction(
-    KernelThunk* kernel_thunk, int64 depth, int64 height, int64 width,
-    HloInstruction* reduce, const Shape& input_shape,
-    absl::Span<const llvm_ir::ElementGenerator> input_gens,
-    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-    absl::Span<HloComputation* const> reducers,
-    absl::Span<const ShapeIndex> reduce_output_shapes,
-    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-        extra_output_gens) {
-  // A naive algorithm is:
-  // 1. Divide the x dimension of the input tensor into tiles of size 1x1xX.
-  // 2. Partially reduces each tile to a scalar using one thread.
-  // 3. Accumulates that scalar to the output vector using atomic operations.
-  //
-  // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
-  //      linear_index < depth * height * width_in_tiles;
-  //      linear_index += blockDim.x * gridDim.x) {
-  //   int x_in_tiles = linear_index % width_in_tiles;
-  //   int y = linear_index / width_in_tiles % height;
-  //   int z = linear_index / (height * width_in_tiles);
-  //   float partial_result = 0;
-  //   for (element_id_in_tile : range(x_tile_size)) {
-  //     int x = x_in_tiles * x_tile_size + element_id_in_tile;
-  //     if (x < width)
-  //       partial_result = reducer(partial_result, input[z][y][x]);
-  //   }
-  //   AtomicReducer(&output[y], partial_result);
-  // }
-  //
-  // Four optimizations are performed.
-  //
-  // 1. To coalesce global memory accesses, dilate the tile with a factor of 32
-  // (i.e. the warp size). For example, suppose the width is 8x32=256. Instead
-  // of making each tile consecutive, we let make tile 0 column
-  // [0,32,64,...,224], tile 1 column [1,33,65,...,225], and so on. This ensures
-  // that threads in a warp access consecutive memory in one iteration (i.e.
-  // coalesced). In the above example, the warp that contains thread 0-31
-  // accesses column 0-31 in the first iteration, and 32-63 in the second
-  // iteration, and so on.
-  //
-  // 2. Partially accumulate partial reduced results computed by threads in the
-  // same warp using shfl_down. Using shfl_down is faster than directly using
-  // atomic operations because shfl_down transfers the data between threads
-  // using shared memory and threads in the same warp run in lock step (thus no
-  // extra synchronization needed). See
-  // https://devblogs.nvidia.com/parallelforall/faster-parallel-reductions-kepler/
-  // for details. The downside is, to produce correct results when using
-  // shfl_down, we need to guarantee threads in the same warp work on input
-  // elements with the same y, so the number of tiles in each row must be a
-  // multiple of 32.
-  //
-  // 3. Specialize the case that the entire tile is in bounds. When that is
-  // true, we don't need to emit "if(x<width)" inside the loop on
-  // element_id_in_tile, which makes the code more friendly to optimizations
-  // such as LICM.
-  //
-  // 4. When the width is too small and x_tile_size is less than the target
-  //    number of elements per thread and use a small factor of depth as
-  //    z_tile_size to increase the number of elements calculated by each
-  //    partial sum. This can reduce the needed number of dynamic shfl_down and
-  //    atomic operations.
-  //
-  // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
-  //      linear_index < depth * height * width_in_tiles;
-  //      linear_index += blockDim.x * gridDim.x) {
-  //   int x_in_tiles = linear_index % width_in_tiles;
-  //   int y = linear_index / width_in_tiles % height;
-  //   int z_in_tiles = linear_index / (height * width_in_tiles);
-  //   int warp_id = x_in_tiles / warpSize;
-  //   int lane_id = x_in_tiles % warpSize;
-  //   float partial_result = 0;
-  //   int x = warp_id * kTileSize * warpSize + lane_id;
-  //   if (width % (x_tile_size * warpSize) == 0 ||
-  //       x + (x_tile_size - 1) * warpSize < width) {
-  //     // The entire x_tile is in bounds.
-  //     for (int element_id_in_z_tile = 0; element_id_in_z_tile < z_tile_size;
-  //          ++element_id_in_z_tile) {
-  //       z = z_in_tiles * z_tile_size + element_id_in_z_tile;
-  //       int tx = x;
-  //       for (int element_id_in_x_tile = 0;
-  //            element_id_in_x_tile < x_tile_size;
-  //            ++element_id_in_x_tile, tx += warpSize) {
-  //         partial_result = Reducer(partial_result, input[z][y][tx]);
-  //       }
-  //     }
-  //   } else {
-  //     // The tile is partially in bounds.
-  //     for (int element_id_in_z_tile = 0; element_id_in_z_tile < z_tile_size;
-  //          ++element_id_in_z_tile) {
-  //       z = z_in_tiles * z_tile_size + element_id_in_z_tile;
-  //       int tx = x;
-  //       for (int element_id_in_x_tile = 0; element_id_in_x_tile <
-  //            x_tile_size; ++element_id_in_tile, tx += warpSize) {
-  //         if (tx < width)
-  //           partial_result = Reducer(partial_result, input[z][y][tx]);
-  //       }
-  //     }
-  //   }
-  //   for (shuffle_distance = 16; shuffle_distance > 0; shuffle_distance /= 2)
-  //     partial_result = Reducer(
-  //         partial_result,
-  //         __shfl_down_sync(CUDA_WARP_ALL, partial_result, shuffle_distance));
-  //   if (lane_id == 0)
-  //     AtomicReducer(&output[y], partial_result);
-  // }
-  //
-
-  int64 x_tile_size;
-  int64 z_tile_size;
-  std::tie(x_tile_size, z_tile_size) =
-      ComputeKernelMappingSchemeForReduction(depth, width, kWarpSize);
-
-  // Round the width in tiles up to the nearest multiple of kWarpSize, so that
-  // the use of shfl_down is valid.
-  const int64 width_in_tiles =
-      RoundUpToNearest(CeilOfRatio(width, x_tile_size), kWarpSize);
-  Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout(
-      reduce->shape().element_type(),
-      {depth / z_tile_size, height, width_in_tiles}, {2, 1, 0});
-  LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
-      tiled_input_shape, ir_emitter_context_->device_description());
-  llvm::Type* index_ty =
-      GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_);
-
-  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
-    return llvm::ConstantInt::get(index_ty, c);
-  };
-
-  auto loop_body_emitter = [=](const IrArray::Index& tile_index) {
-    const int num_reduces = reducers.size();
-    llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType(
-        input_shape.element_type(), ir_emitter_context_->llvm_module());
-    std::vector<llvm::Value*> partial_reduction_result_addresses;
-    for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* partial_reduction_result_address =
-          Alloca(element_ir_type, /*ArraySize=*/nullptr,
-                 "partial_reduction_result." + llvm::Twine(i));
-      TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
-                          init_value_gens[i](IrArray::Index(index_ty)));
-      Store(init_ir_value, partial_reduction_result_address);
-      partial_reduction_result_addresses.push_back(
-          partial_reduction_result_address);
-    }
-
-    llvm::Value* z_tile = tile_index[0];
-    llvm::Value* y = tile_index[1];
-    llvm::Value* x_tile = tile_index[2];
-
-    x_tile = ZExtOrTrunc(x_tile, index_ty);
-
-    llvm::Value* warp_id =
-        UDiv(x_tile, index_typed_constant(kWarpSize), "warp_id");
-    llvm::Value* lane_id =
-        URem(x_tile, index_typed_constant(kWarpSize), "lane_id");
-
-    // The x-location of the last element in this z-x-tile.
-    // last_x = lane_id + warpSize * (x_tile_size - 1 + warp_id * x_tile_size);
-    llvm::Value* last_x = NSWAdd(
-        lane_id,
-        NSWMul(index_typed_constant(kWarpSize),
-               NSWAdd(index_typed_constant(x_tile_size - 1),
-                      NSWMul(warp_id, index_typed_constant(x_tile_size)))));
-
-    KernelSupportLibrary ksl(
-        &b_,
-        /*unroll_mode=*/xla::llvm_ir::UnrollMode::kFullyUnroll,
-        /*prevent_vectorization=*/false);
-
-    // Emit a for-loop that partially reduces the elements in the given
-    // z-x-tile.
-    auto emit_z_x_tile_element_loop = [&](bool x_tile_in_bounds,
-                                          int64 x_tile_loop_bound) -> Status {
-      auto emit_z_tile_element_loop = [&](llvm::Value* z_indvar) -> Status {
-        llvm::Value* z =
-            NSWAdd(z_indvar, NSWMul(index_typed_constant(z_tile_size), z_tile));
-        TF_RETURN_IF_ERROR(ksl.For(
-            "x_tile",
-            /*start=*/index_typed_constant(0),
-            /*end=*/index_typed_constant(x_tile_loop_bound),
-            /*step=*/1, [&](llvm::Value* x_indvar) -> Status {
-              // x = lane_id +
-              //     warpSize * (element_id_in_x_tile + warp_id * x_tile_size);
-              llvm::Value* x = NSWAdd(
-                  lane_id,
-                  NSWMul(index_typed_constant(kWarpSize),
-                         NSWAdd(x_indvar,
-                                NSWMul(warp_id, llvm::ConstantInt::get(
-                                                    index_ty, x_tile_size)))));
-
-              // Unless we know the x-tile is entirely in bounds, we have to
-              // emit a x-in-bounds check before reading from the input.
-              if (!x_tile_in_bounds) {
-                llvm_ir::LlvmIfData if_x_in_bounds_data =
-                    llvm_ir::EmitIfThenElse(
-                        ICmpULT(x, index_typed_constant(width)), "x_in_bounds",
-                        &b_);
-                // Points b_ to the then-block.
-                llvm_ir::SetToFirstInsertPoint(if_x_in_bounds_data.true_block,
-                                               &b_);
-              }
-
-              // Emit code that reads the input element and accumulates it
-              // to the partial reduction result.
-              llvm::Value* input_address = Alloca(element_ir_type);
-              {
-                // {z,y,x} is an index to input_3d_tensor_shape
-                // [depth,height,width]. We need to convert that to an index
-                // to input_shape (the shape of the operand of "reduce").
-                // This conversion is composed of a transposition from
-                // input_shape to normalized_input_shape and a reshape from
-                // normalized_input_shape to input_3d_tensor_shape.
-                const Shape normalized_input_shape = ShapeUtil::
-                    MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
-                        input_shape);
-                auto input_shape_min2maj =
-                    LayoutUtil::MinorToMajor(input_shape);
-                const std::vector<int64> transpose_dimension_mapping(
-                    input_shape_min2maj.rbegin(), input_shape_min2maj.rend());
-                const Shape input_3d_tensor_shape =
-                    ShapeUtil::MakeShapeWithDescendingLayout(
-                        input_shape.element_type(), {depth, height, width});
-                const IrArray::Index input_3d_tensor_index(
-                    {z, y, x}, input_3d_tensor_shape, &b_);
-                const IrArray::Index input_index =
-                    input_3d_tensor_index
-                        .SourceIndexOfReshape(input_3d_tensor_shape,
-                                              normalized_input_shape, &b_)
-                        .SourceIndexOfTranspose(
-                            normalized_input_shape, input_shape,
-                            transpose_dimension_mapping, &b_);
-
-                for (int i = 0; i != num_reduces; ++i) {
-                  TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
-                                      input_gens[i](input_index));
-                  Store(input_ir_value, input_address);
-                  TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
-                      *reducers[i],
-                      {partial_reduction_result_addresses[i], input_address},
-                      partial_reduction_result_addresses[i]));
-                }
-                return EmitExtraOutputsForReduce(reduce, input_index,
-                                                 extra_output_gens);
-              }
-            }));
-        return Status::OK();
-      };
-
-      return ksl.For("z_tile",
-                     /*start=*/index_typed_constant(0),
-                     /*end=*/index_typed_constant(z_tile_size),
-                     /*step=*/1, emit_z_tile_element_loop);
-    };
-
-    llvm::Value* tile_in_bounds =
-        Or(b_.getInt1(width % (x_tile_size * kWarpSize) == 0),
-           ICmpULT(last_x, index_typed_constant(width)));
-
-    TF_RETURN_IF_ERROR(
-        ksl.If(tile_in_bounds,
-               /*true_block_generator=*/
-               [&]() -> Status {
-                 return emit_z_x_tile_element_loop(/*x_tile_in_bounds=*/true,
-                                                   x_tile_size);
-               },
-               /*false_block_generator=*/
-               [&]() -> Status {
-                 return emit_z_x_tile_element_loop(
-                     /*x_tile_in_bounds=*/false,
-                     CeilOfRatio(width % (x_tile_size * kWarpSize), kWarpSize));
-               }));
-
-    // After accumulating the elements of the z_x_tile, emit calls to
-    // shfl_down that accumulate the partial reduction results of all
-    // threads in a warp.
-    int bit_width = llvm_ir::GetSizeInBits(element_ir_type);
-    // bitcast cannot be applied to aggregate types (even packed ones), so we
-    // instead bitcast addresses of load/store to intN* of the same bit-width.
-    llvm::Type* shuffle_ir_type = element_ir_type->isStructTy()
-                                      ? b_.getIntNTy(bit_width)
-                                      : element_ir_type;
-    for (int shuffle_distance = 16; shuffle_distance >= 1;
-         shuffle_distance /= 2) {
-      llvm::Value* result_from_other_lane =
-          Alloca(element_ir_type, nullptr, "result_from_other_lane");
-      for (int i = 0; i != num_reduces; ++i) {
-        llvm::Value* partial_reduction_result =
-            Load(BitCast(partial_reduction_result_addresses[i],
-                         shuffle_ir_type->getPointerTo()),
-                 "partial_reduction_result");
-        CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0)
-            << "Requires block size a multiple of the warp size, otherwise we "
-               "will read undefined elements.";
-        Store(EmitFullWarpShuffleDown(partial_reduction_result,
-                                      b_.getInt32(shuffle_distance), &b_),
-              BitCast(result_from_other_lane, shuffle_ir_type->getPointerTo()));
-        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
-            *reducers[i],
-            {partial_reduction_result_addresses[i], result_from_other_lane},
-            partial_reduction_result_addresses[i]));
-      }
-    }
-
-    const HloInstruction* output =
-        reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
-
-    // Emit an atomic operation that accumulates the partial reduction result of
-    // lane 0 (which holds the partially accumulated result for its warp) to the
-    // output element.
-    llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
-        ICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
-    for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* output_address =
-          GetIrArray(*output, *output, reduce_output_shapes[i])
-              .EmitArrayElementAddress(
-                  IrArray::Index(y,
-                                 ShapeUtil::GetSubshape(
-                                     output->shape(), reduce_output_shapes[i]),
-                                 &b_),
-                  &b_, "output_element_address");
-      // We don't need to emit atomic operations if there is only one tile of
-      // results. 'depth' is the z dimension, 'width' is the x dimension.
-      if (z_tile_size >= depth && x_tile_size >= width) {
-        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
-            *reducers[i],
-            {output_address, partial_reduction_result_addresses[i]},
-            output_address));
-      } else {
-        TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
-            *reducers[i], output_address,
-            partial_reduction_result_addresses[i]));
-      }
-    }
-    return Status::OK();
-  };
-
-  // Emit a parallel loop that iterates through every input tiles.
-  UpdateLaunchDimensions(launch_dimensions, kernel_thunk,
-                         ir_emitter_context_->llvm_module());
-  return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
-                             launch_dimensions, &b_)
-      .EmitLoop(IrName(reduce), index_ty);
-}
-
-// Figures out whether `reduce` is a row or column reduction, and which
-// dimensions to reduce, and calls either `EmitRowReduction` or
-// `EmitColumnReduction` as appropriate.
-// Prerequisite: all the dimensions to keep are contiguous in the input layout
-//               and, if `reduce` is fused, the fused subgraph is pure
-//               elementwise.
-Status IrEmitterUnnested::EmitReductionToVector(
-    KernelThunk* kernel_thunk, HloInstruction* reduce, const Shape& input_shape,
-    absl::Span<const llvm_ir::ElementGenerator> input_gens,
-    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-    absl::Span<const int64> dimensions_to_reduce,
-    absl::Span<HloComputation* const> reducers,
-    absl::Span<const ShapeIndex> reduce_output_shapes,
-    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-        extra_output_gens) {
-  // This emission requires "reduce" to have an input layout. It is either set
-  // by LayoutAssignment (for a top-level kReduce) or by InstructionFusion (for
-  // a fused kReduce).
-  CHECK(input_shape.has_layout()) << "LayoutAssignment or InstructionFusion "
-                                     "doesn't set the input layout of "
-                                  << reduce->ToString();
-
-  // Specialize multi-dimensional-array-to-vector reduction.
-  std::vector<int64> input_dims_to_keep;
-  for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape);
-       ++input_dim) {
-    if (std::find(dimensions_to_reduce.begin(), dimensions_to_reduce.end(),
-                  input_dim) == dimensions_to_reduce.end()) {
-      input_dims_to_keep.push_back(input_dim);
-    }
-  }
-
-  // Sort the dimensions to keep from minor to major, to facilitate checking
-  // whether another dimension is major or minor of them.
-  std::sort(input_dims_to_keep.begin(), input_dims_to_keep.end(),
-            [&input_shape](int64 dim_a, int64 dim_b) {
-              return PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
-                                         dim_a) <
-                     PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
-                                         dim_b);
-            });
-  // Now, if output rank is at least 1, `input_dims_to_keep.front()` is
-  // minormost and `input_dims_to_keep.back()` is majormost.
-
-  // If the dimensions to keep are minormost, emit a column reduction. As all
-  // the dimensions to keep are contiguous, by prerequisite of
-  // `EmitReductionToVector`, we only need to check whether the minormost
-  // dimension of the input is to keep.
-  if (ShapeUtil::IsEffectiveScalar(reduce->shape())) {
-    return EmitReductionToScalar(kernel_thunk, reduce, input_shape, input_gens,
-                                 init_value_gens, reducers,
-                                 reduce_output_shapes, extra_output_gens);
-  } else if (input_dims_to_keep.front() ==
-             LayoutUtil::Minor(input_shape.layout(), 0)) {
-    // Column reduction. Treat the result of "input" as a matrix whose width
-    // is the most minor dimension and height the product of other dimensions,
-    // and treat "reduce" as a column reduction of the input matrix.
-    const int64 width = ShapeUtil::ElementsIn(reduce->shape());
-    // "width" can be zero, so don't do
-    //   height = ShapeUtil::ElementsIn(input_shape) / width;
-    int64 height = 1;
-    for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape);
-         ++input_dim) {
-      if (!std::count(input_dims_to_keep.begin(), input_dims_to_keep.end(),
-                      input_dim)) {
-        height *= input_shape.dimensions(input_dim);
-      }
-    }
-    return EmitColumnReduction(kernel_thunk, height, width, reduce, input_shape,
-                               input_gens, init_value_gens, reducers,
-                               reduce_output_shapes, extra_output_gens);
-  } else {
-    // Reduce the row dimension of a matrix or reduce dimension 0 and 2 in a
-    // 3D tensor. The size of dimension 1 (the height) is the size of the
-    // dimension to keep, the size of dimension 0 (the depth) is the product
-    // of dimensions that are more major than the dimension to keep, and the
-    // size of dimension 2 (the width) is the product of more minor
-    // dimensions.
-    int64 depth = 1;
-    int64 width = 1;
-    for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape);
-         ++input_dim) {
-      if (PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
-                              input_dim) >
-          PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
-                              input_dims_to_keep.back())) {
-        depth *= input_shape.dimensions(input_dim);
-      } else if (PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
-                                     input_dim) <
-                 PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
-                                     input_dims_to_keep.front())) {
-        width *= input_shape.dimensions(input_dim);
-      }
-    }
-    const int64 height = ShapeUtil::ElementsIn(reduce->shape());
-    return EmitRowReduction(kernel_thunk, depth, height, width, reduce,
-                            input_shape, input_gens, init_value_gens, reducers,
-                            reduce_output_shapes, extra_output_gens);
-  }
-}
-
 Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) {
   // TODO(b/112040122): Support multi-output reduce.
   if (!ShapeUtil::IsArray(reduce->shape())) {
     return Unimplemented("Multi-output reduce is not supported on GPU");
   }
-  auto input = reduce->operand(0);
-  auto init_value = reduce->operand(1);
-  absl::Span<const int64> dimensions_to_reduce(reduce->dimensions());
-  HloComputation* reducer = reduce->to_apply();
-  // HandleReduce specializes reduction from a multi-dimensional array to a 1D
-  // array. The specialized version requires an initializer thunk that
-  // initializes the output array to the initial value of the reduce.
   if (IsReductionToVector(*reduce)) {
-    TF_ASSIGN_OR_RETURN(std::unique_ptr<Thunk> initializer_thunk,
-                        BuildInitializerThunk(reduce));
-    std::vector<std::unique_ptr<Thunk>> thunks;
-    thunks.push_back(std::move(initializer_thunk));
-    std::unique_ptr<KernelThunk> kernel_thunk =
-        BuildKernelThunk(reduce, /*implements_whole_instruction=*/false);
-
-    TF_CHECK_OK(EmitReductionToVector(
-        kernel_thunk.get(), reduce, input->shape(),
-        {[&](const IrArray::Index& index) {
-          return GetIrArray(*input, *reduce).EmitReadArrayElement(index, &b_);
-        }},
-        {[&](const IrArray::Index& index) {
-          return GetIrArray(*init_value, *reduce)
-              .EmitReadArrayElement(index, &b_);
-        }},
-        dimensions_to_reduce, {reducer}, {{}}, {}));
-
-    thunks.push_back(std::move(kernel_thunk));
-
-    std::unique_ptr<SequentialThunk> sequential_thunk =
-        absl::make_unique<SequentialThunk>(std::move(thunks), reduce);
-    AddThunkToThunkSequence(std::move(sequential_thunk));
-    return Status::OK();
+    return EmitReductionToVector(reduce);
   }
 
   return IrEmitter::HandleReduce(reduce);
@@ -1820,7 +763,7 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     // Create the inner loop to iterate over the window.
     llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "inner"), &b_,
                                       index_type);
-    std::vector<int64> window_size;
+    DimensionVector window_size;
     for (const auto& dim : window.dimensions()) {
       window_size.push_back(dim.size());
       CHECK_GT(dim.size(), 0);
@@ -3265,7 +2208,8 @@ void EmitPartialTile(
         builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
 
     ksl->IfReturnVoid(
-        "x_in_tile", builder->CreateICmpULT(x_loc, tile_width), [&] {
+        loop_name + "_x_in_tile", builder->CreateICmpULT(x_loc, tile_width),
+        [&] {
           // tile_height_bound =
           //   ceil(tile_height / num_threads_y) * num_threads_y
           llvm::Value* ceiling_of_ratio = builder->CreateUDiv(
@@ -3282,8 +2226,8 @@ void EmitPartialTile(
               [&](llvm::Value* y_indvar) {
                 llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
                 ksl->IfReturnVoid(
-                    "y_in_tile", builder->CreateICmpULT(y_loc, tile_height),
-                    [&] {
+                    loop_name + "_y_in_tile",
+                    builder->CreateICmpULT(y_loc, tile_height), [&] {
                       emit_elem_function(
                           source_idx.AddOffsetToDim(
                               y_indvar, KernelMappingScheme::DimY, builder),
@@ -3314,7 +2258,7 @@ void EmitTiledElementalCodeWithBoundsCheck(
   llvm::Type* index_ty = tile_width->getType();
 
   ksl->IfReturnVoid(
-      "full_tile",
+      loop_name + "_full_tile",
       builder->CreateAnd(
           builder->CreateICmpEQ(llvm::ConstantInt::get(index_ty, tile_size_x),
                                 tile_width),
@@ -3405,7 +2349,395 @@ void IrEmitterUnnested::EmitTileElementForFusion(
   }
 }
 
-// Emits a block of tiles, given a function object to emit one tile.
+// Information to support the code generation for a tiled reduction kernel.
+using AddressVector = InlinedVector<llvm::AllocaInst*, 1>;
+class ReductionCodegenInfo : public IrEmitterUnnested::KernelCodegenInfo {
+ public:
+  explicit ReductionCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme,
+                                bool is_row_reduction)
+      : KernelCodegenInfo(mapping_scheme),
+        current_output_linear_index_address_(nullptr),
+        current_output_inbound_address_(nullptr),
+        is_row_reduction_(is_row_reduction) {}
+
+  void SetCurrentOutputLinearIndexAddress(llvm::AllocaInst* a) {
+    current_output_linear_index_address_ = a;
+  }
+  // Returns the address of the memory that stores the linear index of the
+  // current output. Since we are processing reduction to contiguous physical
+  // dimensions, this linear index is the linear index of the 1D output array.
+  llvm::AllocaInst* GetCurrentOutputLinearIndexAddress() const {
+    return current_output_linear_index_address_;
+  }
+
+  void SetCurrentOutputInboundAddress(llvm::AllocaInst* a) {
+    current_output_inbound_address_ = a;
+  }
+
+  llvm::AllocaInst* GetCurrentOutputInboundAddress() const {
+    return current_output_inbound_address_;
+  }
+
+  AddressVector* GetMutablePartialResultAddresses() {
+    return &partial_result_addresses_;
+  }
+  const AddressVector& GetPartialResultAddresses() const {
+    return partial_result_addresses_;
+  }
+
+  AddressVector* GetMutableReductionInputAddresses() {
+    return &reduction_input_addresses_;
+  }
+  const AddressVector& GetReductionInputAddresses() const {
+    return reduction_input_addresses_;
+  }
+
+  InlinedVector<HloComputation*, 1>* GetMutableReducers() { return &reducers_; }
+  const InlinedVector<HloComputation*, 1>& GetReducers() const {
+    return reducers_;
+  }
+  int GetNumberOfReduces() const { return reducers_.size(); }
+
+  InlinedVector<ShapeIndex, 1>* GetMutableReductionOutputShapeIndices() {
+    return &reduction_output_shape_indices_;
+  }
+  const InlinedVector<ShapeIndex, 1>& GetReductionOutputShapeIndices() const {
+    return reduction_output_shape_indices_;
+  }
+
+  bool IsRowReduction() const { return is_row_reduction_; }
+
+  // Return the dimension that is being reduced between DimX and DimY.
+  int GetReducedDimensionEnum() const {
+    return IsRowReduction() ? llvm_ir::KernelMappingScheme::DimX
+                            : llvm_ir::KernelMappingScheme::DimY;
+  }
+
+  // Return the dimension that is being ketp between DimX and DimY.
+  int GetKeptDimensionEnum() const {
+    return IsRowReduction() ? llvm_ir::KernelMappingScheme::DimY
+                            : llvm_ir::KernelMappingScheme::DimX;
+  }
+
+ private:
+  AddressVector partial_result_addresses_;
+  AddressVector reduction_input_addresses_;
+  InlinedVector<HloComputation*, 1> reducers_;
+  InlinedVector<ShapeIndex, 1> reduction_output_shape_indices_;
+  llvm::AllocaInst* current_output_linear_index_address_;
+  llvm::AllocaInst* current_output_inbound_address_;
+  bool is_row_reduction_;
+};
+
+namespace {
+// Returns a group of instructions that generate the output for the kernel
+// containing the given HLO instruction. The result may be an unnested kReduce
+// HLO, a nested kReduce HLO of a kInput fusion, or the operands of the tuple
+// for a multiple output fusion.
+absl::Span<HloInstruction* const> GetOutputInstructions(
+    HloInstruction* const* reduce_or_tuple_pointer) {
+  HloOpcode opcode = (*reduce_or_tuple_pointer)->opcode();
+  CHECK(opcode == HloOpcode::kReduce || opcode == HloOpcode::kTuple);
+  return opcode == HloOpcode::kTuple
+             ? (*reduce_or_tuple_pointer)->operands()
+             : absl::Span<HloInstruction* const>(reduce_or_tuple_pointer, 1);
+}
+
+const HloInstruction* GetFirstReduceInstruction(
+    absl::Span<HloInstruction* const> instructions) {
+  auto first_reduce_iter =
+      absl::c_find_if(instructions, [](const HloInstruction* inst) {
+        return inst->opcode() == HloOpcode::kReduce;
+      });
+  CHECK_NE(first_reduce_iter, instructions.end());
+  return *first_reduce_iter;
+}
+
+};  // namespace
+
+void IrEmitterUnnested::EmitPrologueForOneReduction(
+    HloInstruction* unnested_hlo, HloInstruction* reduce_inst, int reduce_idx,
+    KernelCodegenInfo* kernel_info, GpuElementalIrEmitter* elemental_emitter,
+    ShapeIndex output_shape_index) {
+  ReductionCodegenInfo* reduction_info =
+      static_cast<ReductionCodegenInfo*>(kernel_info);
+
+  InlinedVector<HloComputation*, 1>* reducers =
+      reduction_info->GetMutableReducers();
+  CHECK(IsReductionToVector(*reduce_inst));
+  reducers->push_back(reduce_inst->to_apply());
+
+  InlinedVector<ShapeIndex, 1>* reduction_output_shape_indices =
+      reduction_info->GetMutableReductionOutputShapeIndices();
+  reduction_output_shape_indices->push_back(std::move(output_shape_index));
+
+  AddressVector* reduction_input_addresses =
+      reduction_info->GetMutableReductionInputAddresses();
+  llvm::Type* element_type = llvm_ir::PrimitiveTypeToIrType(
+      reduce_inst->shape().element_type(), ir_emitter_context_->llvm_module());
+  llvm::AllocaInst* reduction_input_address = Alloca(element_type);
+  reduction_input_addresses->push_back(reduction_input_address);
+
+  AddressVector* partial_result_addresses =
+      reduction_info->GetMutablePartialResultAddresses();
+  llvm::AllocaInst* partial_result_address =
+      Alloca(element_type, /*ArraySize=*/nullptr,
+             "partial_reduction_result." + llvm::Twine(reduce_idx));
+  partial_result_addresses->push_back(partial_result_address);
+
+  // Initialize the partial result with the initial value of the reduction.
+  llvm::Value* init_ir_value;
+  if (unnested_hlo->opcode() == HloOpcode::kFusion) {
+    HloInstruction* init_value_operand = reduce_inst->mutable_operand(1);
+    FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(unnested_hlo),
+                                 elemental_emitter);
+
+    TF_CHECK_OK(init_value_operand->Accept(&fused_emitter));
+    init_ir_value =
+        fused_emitter
+            .GetGenerator(init_value_operand)(IrArray::Index(b_.getInt32Ty()))
+            .ValueOrDie();
+  } else {
+    const HloInstruction* init_value = unnested_hlo->operand(1);
+    init_ir_value =
+        GetIrArray(*init_value, *unnested_hlo)
+            .EmitReadArrayElement(IrArray::Index(b_.getInt32Ty()), &b_);
+  }
+
+  Store(init_ir_value, partial_result_address);
+}
+
+void IrEmitterUnnested::EmitPrologueForReduction(
+    HloInstruction* unnested_hlo, KernelCodegenInfo* kernel_info) {
+  VLOG(10) << "Emit prologue for reduction " << unnested_hlo->ToString();
+  // Find the unnested kReduce or the tuple that contains a list of kReduce.
+  HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
+                                        ? unnested_hlo->fused_expression_root()
+                                        : unnested_hlo;
+  absl::Span<HloInstruction* const> output_instructions =
+      GetOutputInstructions(&reduce_or_tuple);
+  ReductionCodegenInfo* reduction_info =
+      static_cast<ReductionCodegenInfo*>(kernel_info);
+  GpuElementalIrEmitter elemental_emitter(hlo_module_config_,
+                                          ir_emitter_context_->llvm_module(),
+                                          &b_, GetNestedComputer());
+  const HloInstruction* first_reduce = nullptr;
+  for (int i = 0, e = output_instructions.size(); i != e; ++i) {
+    if (output_instructions[i]->opcode() != HloOpcode::kReduce) {
+      continue;
+    }
+    HloInstruction* reduce_inst = output_instructions[i];
+    if (first_reduce == nullptr) {
+      first_reduce = reduce_inst;
+    } else {
+      CHECK(first_reduce->dimensions() == reduce_inst->dimensions());
+    }
+    ShapeIndex output_shape_index;
+    if (reduce_or_tuple->opcode() == HloOpcode::kTuple) {
+      output_shape_index = {i};
+    }
+
+    EmitPrologueForOneReduction(unnested_hlo, reduce_inst, i, kernel_info,
+                                &elemental_emitter,
+                                std::move(output_shape_index));
+  }
+
+  // Allocate stack storage to store the current output linear index and record
+  // the address of the storage.
+  reduction_info->SetCurrentOutputLinearIndexAddress(
+      Alloca(reduction_info->GetIndexType()));
+
+  if (!reduction_info->IsRowReduction()) {
+    llvm::Type* bool_ty = b_.getInt1Ty();
+    llvm::AllocaInst* output_inbound_addr = Alloca(bool_ty);
+    Store(llvm::ConstantInt::get(bool_ty, 0), output_inbound_addr);
+    reduction_info->SetCurrentOutputInboundAddress(output_inbound_addr);
+  }
+}
+
+void IrEmitterUnnested::EmitFullWarpShuffleDownLoopForAllReduces(
+    const InlinedVector<HloComputation*, 1>& reducers,
+    const AddressVector& partial_result_addresses) {
+  for (int distance = 16; distance >= 1; distance /= 2) {
+    for (int i = 0; i != reducers.size(); ++i) {
+      llvm::Type* element_type =
+          partial_result_addresses[i]->getType()->getElementType();
+      int bit_width = llvm_ir::GetSizeInBits(element_type);
+      llvm::Value* result_from_other_lane = Alloca(
+          element_type, nullptr, "result_from_other_lane" + llvm::Twine(i));
+      // Bitcast cannot be applied to aggregate types (even packed ones), so
+      // we bitcast addresses of load/store to intN* of the same bit-width.
+      llvm::Type* shuffled_value_type =
+          element_type->isStructTy() ? b_.getIntNTy(bit_width) : element_type;
+      auto convert_pointer_for_shuffle = [&](llvm::Value* ptr) {
+        return BitCast(ptr, shuffled_value_type->getPointerTo());
+      };
+      llvm::Value* partial_result =
+          Load(convert_pointer_for_shuffle(partial_result_addresses[i]),
+               "partial_reduction_result");
+      Store(EmitFullWarpShuffleDown(partial_result, b_.getInt32(distance), &b_),
+            convert_pointer_for_shuffle(result_from_other_lane));
+      TF_CHECK_OK(EmitCallToNestedComputation(
+          *reducers[i], {partial_result_addresses[i], result_from_other_lane},
+          partial_result_addresses[i]));
+    }
+  }
+}
+
+void IrEmitterUnnested::EmitEpilogueForReduction(
+    HloInstruction* unnested_hlo, KernelCodegenInfo* kernel_info) {
+  ReductionCodegenInfo* reduction_info =
+      static_cast<ReductionCodegenInfo*>(kernel_info);
+  int num_reduces = reduction_info->GetNumberOfReduces();
+  const AddressVector& partial_result_addresses =
+      reduction_info->GetPartialResultAddresses();
+  const InlinedVector<HloComputation*, 1>& reducers =
+      reduction_info->GetReducers();
+  const InlinedVector<ShapeIndex, 1>& reduction_output_shape_indices =
+      reduction_info->GetReductionOutputShapeIndices();
+
+  if (reduction_info->IsRowReduction()) {
+    EmitFullWarpShuffleDownLoopForAllReduces(reducers,
+                                             partial_result_addresses);
+    llvm::Value* lane_id = reduction_info->GetLaneId();
+    llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
+        ICmpEQ(lane_id, llvm::ConstantInt::get(lane_id->getType(), 0)),
+        "lane_id_is_zero", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
+  } else {
+    llvm::Value* output_inbound_addr =
+        reduction_info->GetCurrentOutputInboundAddress();
+    llvm::Value* output_inbound = Load(output_inbound_addr);
+    llvm_ir::LlvmIfData if_output_inbound_data = llvm_ir::EmitIfThenElse(
+        ICmpEQ(output_inbound,
+               llvm::ConstantInt::get(output_inbound->getType(), 1)),
+        "output_inbound", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_output_inbound_data.true_block, &b_);
+  }
+
+  // Emit an atomic operation that accumulates the partial reduction to the
+  // output element. For row reduction, this is only for lane 0 due to the
+  // if-statement emitted above.
+  for (int i = 0; i != num_reduces; ++i) {
+    IrArray::Index element_index(
+        /*linear=*/Load(reduction_info->GetCurrentOutputLinearIndexAddress(),
+                        "output_linear_addr"),
+        ShapeUtil::GetSubshape(unnested_hlo->shape(),
+                               reduction_output_shape_indices[i]),
+        &b_);
+    llvm::Value* output_address =
+        GetIrArray(*unnested_hlo, *unnested_hlo,
+                   reduction_output_shape_indices[i])
+            .EmitArrayElementAddress(element_index, &b_,
+                                     "output_element_address");
+    // Do not emit atomic operations if each element in the reduction result is
+    // computed by one block, that is the dimension being reduced has only one
+    // block.
+    const llvm_ir::KernelMappingScheme* mapping_scheme =
+        reduction_info->GetKernelMappingScheme();
+    if (mapping_scheme->GetTileBlockSizeForDimension(
+            llvm_ir::KernelMappingScheme::DimZ) == 1 &&
+        mapping_scheme->GetTileBlockSizeForDimension(
+            reduction_info->GetReducedDimensionEnum()) == 1) {
+      TF_CHECK_OK(EmitCallToNestedComputation(
+          *reducers[i], {output_address, partial_result_addresses[i]},
+          output_address));
+    } else {
+      TF_CHECK_OK(EmitAtomicOperationForNestedComputation(
+          *reducers[i], output_address, partial_result_addresses[i]));
+    }
+  }
+}
+
+void IrEmitterUnnested::EmitTileElementForReduction(
+    HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index,
+    const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+    llvm::Value* x_loc) {
+  VLOG(10) << "Emit tile element for reduce " << unnested_hlo->ToString();
+  HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
+                                        ? unnested_hlo->fused_expression_root()
+                                        : unnested_hlo;
+  llvm_ir::TiledParameterInfo* tiled_param_info =
+      kernel_info->GetTiledParameterInfo();
+  tiled_param_info->set_y(y_loc);
+  tiled_param_info->set_x(x_loc);
+
+  // Record the linear address for the current reduction.
+  const ReductionCodegenInfo* reduction_info =
+      dynamic_cast<const ReductionCodegenInfo*>(kernel_info);
+  Store(index[reduction_info->GetKeptDimensionEnum()],
+        reduction_info->GetCurrentOutputLinearIndexAddress());
+  if (!reduction_info->IsRowReduction()) {
+    llvm::Type* bool_ty = b_.getInt1Ty();
+    llvm::AllocaInst* output_inbound_addr =
+        reduction_info->GetCurrentOutputInboundAddress();
+    Store(llvm::ConstantInt::get(bool_ty, 1), output_inbound_addr);
+  }
+
+  InlinedVector<llvm_ir::ElementGenerator, 1> input_gens;
+  std::vector<std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+      extra_output_gens;
+  GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_,
+                                     GetNestedComputer());
+  FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(unnested_hlo),
+                               &elem_emitter);
+  absl::Span<HloInstruction* const> output_instructions =
+      GetOutputInstructions(&reduce_or_tuple);
+  // Construct the ElementGenerator for each reduction and extra output in the
+  // the group of output instructions.
+  if (unnested_hlo->opcode() == HloOpcode::kFusion) {
+    fused_emitter.SetTiledParameterInfo(tiled_param_info);
+    TF_CHECK_OK(unnested_hlo->fused_expression_root()->Accept(&fused_emitter));
+
+    for (int i = 0, e = output_instructions.size(); i != e; ++i) {
+      const HloInstruction* inst = output_instructions[i];
+      ShapeIndex output_shape_index;
+      if (reduce_or_tuple->opcode() == HloOpcode::kTuple) {
+        output_shape_index = {i};
+      }
+      if (inst->opcode() == HloOpcode::kReduce) {
+        input_gens.push_back(fused_emitter.GetGenerator(inst->operand(0)));
+      } else {
+        extra_output_gens.emplace_back(fused_emitter.GetGenerator(inst),
+                                       std::move(output_shape_index));
+      }
+    }
+  } else {
+    input_gens.push_back([&](const IrArray::Index& index) {
+      return GetIrArray(*unnested_hlo->operand(0), *unnested_hlo)
+          .EmitReadArrayElement(index, &b_);
+    });
+  }
+
+  IrArray::Index input_index =
+      reduction_info->GetKernelMappingScheme()->GetUnnormalizedIndex(
+          index,
+          GetFirstReduceInstruction(output_instructions)->operand(0)->shape());
+  const AddressVector& partial_reduction_result_addresses =
+      reduction_info->GetPartialResultAddresses();
+  const AddressVector& reduction_input_addresses =
+      reduction_info->GetReductionInputAddresses();
+  const InlinedVector<HloComputation*, 1>& reducers =
+      reduction_info->GetReducers();
+
+  // Emit code to generate the input and perform the reduction computation for
+  // each reduction instruction.
+  for (int i = 0; i != reducers.size(); ++i) {
+    llvm::Value* const input_ir_value = input_gens[i](input_index).ValueOrDie();
+    Store(input_ir_value, reduction_input_addresses[i]);
+    TF_CHECK_OK(EmitCallToNestedComputation(
+        *reducers[i],
+        {partial_reduction_result_addresses[i], reduction_input_addresses[i]},
+        partial_reduction_result_addresses[i]));
+  }
+
+  // Emit code to generate the output for the non-reduction instructions in the
+  // fusion, if any.
+  TF_CHECK_OK(
+      EmitExtraOutputsForReduce(unnested_hlo, input_index, extra_output_gens));
+}
+
+// Emits a kernel for the hlo instruction using the given tiling scheme.
 void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
                                   const KernelCodegenInfo* kernel_info,
                                   KernelSupportLibrary& ksl,
@@ -3532,7 +2864,6 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
             << llvm_ir::DumpToString(*param_shmem_buffers[id]);
   }
 
-  CHECK_EQ(mapping_scheme->GetThreadsPerTile() % kWarpSize, 0);
   LaunchDimensions launch_dimensions = LaunchDimensions(
       mapping_scheme->GetNumberOfBlocks(), mapping_scheme->GetThreadsPerTile());
   llvm::Type* index_ty = GetIndexTypeForKernel(
@@ -3561,6 +2892,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
   kernel_info->SetLaneId(
       mapping_scheme->GetNumberOfThreadsForDimensionX() == kWarpSize ? x
                                                                      : nullptr);
+  kernel_info->SetIndexType(index_ty);
 
   KernelSupportLibrary ksl(&b_, llvm_ir::UnrollMode::kDefaultUnroll);
   // Curry a few parameters to EmitTiledElementalCodeWithBoundsCheck.
@@ -3585,29 +2917,31 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
         input_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
             .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
 
-    // Copy input parameter values to shared memory buffers:
-    // tile[y, x] = input[index]
-    // Note that tile_width and tile_height are flipped here because we are
-    // reading a transposed tile.
-    emit_tiled_elemental_code_with_bounds_check(
-        input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
-        [&](const IrArray::Index& index, llvm::Value* y_loc,
-            llvm::Value* x_loc) {
-          for (int64 id : tiled_param_ids) {
-            IrArray& input_in_logical_shape = param_in_reduced_shape_arrays[id];
-            llvm::Value* shmem_buffer = param_shmem_buffers[id];
-            // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
-            // global variables, so LLVM can't infer much about it.
-            Store(input_in_logical_shape.EmitReadArrayElement(index, &b_,
-                                                              "input_element"),
-                  GEP(shmem_buffer, {index_typed_constant(0), y_loc, x_loc}));
-          }
-        });
-
     // If shared memory transpose is needed, wait for all threads to reach this
     // point, lest we copy a value from tile to output before the other thread
     // copies it from input to tile. This is `__syncthreads` in CUDA.
     if (!tiled_param_ids.empty()) {
+      // Copy input parameter values to shared memory buffers:
+      // tile[y, x] = input[index]
+      // Note that tile_width and tile_height are flipped here because we are
+      // reading a transposed tile.
+      emit_tiled_elemental_code_with_bounds_check(
+          input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
+          [&](const IrArray::Index& index, llvm::Value* y_loc,
+              llvm::Value* x_loc) {
+            for (int64 id : tiled_param_ids) {
+              IrArray& input_in_logical_shape =
+                  param_in_reduced_shape_arrays[id];
+              llvm::Value* shmem_buffer = param_shmem_buffers[id];
+              // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
+              // global variables, so LLVM can't infer much about it.
+              Store(input_in_logical_shape.EmitReadArrayElement(
+                        index, &b_, "input_element"),
+                    GEP(shmem_buffer, {index_typed_constant(0), y_loc, x_loc}));
+            }
+          });
+
+      // Wait for all threads to reach this point using `__syncthreads` in CUDA.
       llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_);
     }
 
@@ -3627,6 +2961,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
           kernel_generator.GetTileElementGenerator()(unnested_hlo, index,
                                                      kernel_info, y_loc, x_loc);
         });
+
     // If a tile block contains multiple tiles and shared memory buffers are
     // used, we need to wait for all threads to finish using the shared memory
     // buffer for the current tile before we move on to process the next tile
@@ -3831,6 +3166,246 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
   return true;
 }
 
+namespace {
+// Checks that the outputs of a fusion with reduction are consistent.
+Status AreFusedReductionOutputsConsistent(
+    absl::Span<HloInstruction* const> output_instructions,
+    const HloInstruction* first_reduce) {
+  for (const HloInstruction* inst : output_instructions) {
+    if (inst->opcode() == HloOpcode::kReduce) {
+      // Shapes, layouts and dimensions must be the same for all reduces
+      // inside of this fusion.
+      TF_RET_CHECK(ShapeUtil::Equal(first_reduce->shape(), inst->shape()));
+      TF_RET_CHECK(ShapeUtil::Equal(first_reduce->operand(0)->shape(),
+                                    inst->operand(0)->shape()));
+      TF_RET_CHECK(ShapeUtil::Equal(first_reduce->operand(1)->shape(),
+                                    inst->operand(1)->shape()));
+      TF_RET_CHECK(first_reduce->dimensions() == inst->dimensions());
+    } else {
+      // For extra outputs we can relax shape equality to allow different
+      // types (with the same number of elements). Layouts still have to
+      // match.
+      TF_RET_CHECK(ShapeUtil::CompatibleIgnoringElementType(
+          first_reduce->operand(0)->shape(), inst->shape()));
+      TF_RET_CHECK(LayoutUtil::Equal(first_reduce->operand(0)->shape().layout(),
+                                     inst->shape().layout()));
+    }
+  }
+  return Status::OK();
+}
+
+// Finds the dimensions to keep for the reduction, sorts and returns the
+// dimensions from minor to major.
+DimensionVector GetDimensionsToKeepMinorToMajor(
+    const Shape& input_shape, absl::Span<const int64> dims_to_reduce) {
+  DimensionVector input_dims(ShapeUtil::Rank(input_shape), 0);
+  absl::c_iota(input_dims, 0);
+  DimensionVector input_dims_to_keep;
+  for (int input_dim : input_dims) {
+    auto it = absl::c_find_if(dims_to_reduce, [&](int64 dim_to_reduce) {
+      return dim_to_reduce == input_dim;
+    });
+    if (it == dims_to_reduce.end()) {
+      input_dims_to_keep.push_back(input_dim);
+    }
+  }
+
+  // Sort the dimensions to keep from minor to major.
+  absl::c_sort(input_dims_to_keep, [&input_shape](int64 dim_a, int64 dim_b) {
+    return PositionInContainer(LayoutUtil::MinorToMajor(input_shape), dim_a) <
+           PositionInContainer(LayoutUtil::MinorToMajor(input_shape), dim_b);
+  });
+
+  VLOG(10) << "dims to keep minor to major"
+           << absl::StrJoin(input_dims_to_keep, ",");
+  return input_dims_to_keep;
+}
+
+// Given the input shape and dimensions to reduce for the reduction to vector,
+// returns <num_reduced_major, num_kept, num_reduced_minor>:
+// num_kept: the number of elements in the contiguous dimensions to keep.
+// num_reduced_major: the number of elements in the dimensions to reduce that
+//   are more major than the dimensions to keep.
+// num_reduced_minor: the number of elements in the dimensions to reduce that
+//   are more minor than the dimensions to kept.
+std::tuple<int64, int64, int64> GetReductionToVectorDimensions(
+    const Shape& input_shape, absl::Span<const int64> dims_to_reduce) {
+  DimensionVector input_dims_to_keep_minor_to_major =
+      GetDimensionsToKeepMinorToMajor(input_shape, dims_to_reduce);
+  CHECK(LayoutUtil::AreDimensionsConsecutive(
+      input_shape.layout(), input_dims_to_keep_minor_to_major));
+  int num_reduced_major = 1, num_kept = 1, num_reduced_minor = 1;
+  if (input_dims_to_keep_minor_to_major.empty()) {
+    return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor);
+  }
+  DimensionVector input_dims(ShapeUtil::Rank(input_shape), 0);
+  absl::c_iota(input_dims, 0);
+  absl::Span<const int64> minor_to_major =
+      LayoutUtil::MinorToMajor(input_shape);
+  for (int input_dim : input_dims) {
+    int64 curr_dim_size = input_shape.dimensions(input_dim);
+    if (PositionInContainer(minor_to_major, input_dim) >
+        PositionInContainer(minor_to_major,
+                            input_dims_to_keep_minor_to_major.back())) {
+      num_reduced_major *= curr_dim_size;
+    } else if (PositionInContainer(minor_to_major, input_dim) <
+               PositionInContainer(minor_to_major,
+                                   input_dims_to_keep_minor_to_major.front())) {
+      num_reduced_minor *= curr_dim_size;
+    } else {
+      num_kept *= curr_dim_size;
+    }
+  }
+
+  return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor);
+}
+
+std::tuple<KernelMappingScheme, bool> ComputeMappingSchemeAndReductionKind(
+    const HloInstruction* first_reduce, llvm::IRBuilder<>* b) {
+  int64 depth = 1;
+  int64 height = 1;
+  int64 width = 1;
+  bool is_row_reduction = true;
+  int64 tile_size_x = 1;
+  int64 tile_size_y = 1;
+  int64 block_size_y = 1;
+  int64 block_size_z = 1;
+  int64 num_threads_x = 1;
+  int64 num_threads_y = 1;
+  const Shape& input_shape = first_reduce->operand(0)->shape();
+  int64 num_input_elems = ShapeUtil::ElementsIn(input_shape);
+  int64 num_output_elems = ShapeUtil::ElementsIn(first_reduce->shape());
+  int64 num_reduced_major, num_kept, num_reduced_minor;
+  std::tie(num_reduced_major, num_kept, num_reduced_minor) =
+      GetReductionToVectorDimensions(input_shape, first_reduce->dimensions());
+  CHECK_EQ(num_output_elems, num_kept);
+
+  if (num_kept == 1) {
+    // Scalar reduction is a special row reduction with depth = height = 1.
+    width = num_input_elems;
+    tile_size_x = kWarpSize * 16;
+    num_threads_x = kWarpSize;
+  } else if (num_reduced_minor == 1) {
+    // Column reduction reduces inputs with dimension [height, width], where
+    // width is the minor dimension, to dimension [width].
+    height = num_reduced_major;
+    width = num_kept;
+    is_row_reduction = false;
+    tile_size_x = std::min(kWarpSize, num_kept);
+    // The old Column reduction algorithm uses kTileHeight = 128. We choose
+    // tile_size_y * block_size_y = 128 to match the value of kTileHeight. Using
+    // a non-trivial block_size_y here is a way to avoid unrolling all the 128
+    // iterations.
+    tile_size_y = 32;
+    block_size_y = 4;
+    num_threads_x = tile_size_x;
+  } else {
+    // Row reduction reduces inputs with dimension [depth, height, width],
+    // where width is the most minor dimension, to dimension [height] .
+    depth = num_reduced_major;
+    height = num_kept;
+    width = num_reduced_minor;
+    num_threads_x = kWarpSize;
+    if (width % (kWarpSize * 64) == 0) {
+      tile_size_x = kWarpSize * 64;
+    } else {
+      tile_size_x = kWarpSize * 8;
+      block_size_z = 8;
+      while (depth % block_size_z != 0) {
+        block_size_z -= 1;
+      }
+    }
+  }
+  DCHECK_EQ(depth * height * width, num_input_elems);
+  VLOG(10) << "is_row_reduction " << is_row_reduction << depth << " " << height
+           << " " << width;
+
+  DimensionVector dims_in_elem{depth, height, width};
+  DimensionVector req_block_sizes{block_size_z, block_size_y, 1};
+  llvm_ir::KernelMappingScheme mapping_scheme(dims_in_elem, tile_size_y,
+                                              tile_size_x, req_block_sizes,
+                                              num_threads_y, num_threads_x, b);
+  return std::make_tuple(mapping_scheme, is_row_reduction);
+}
+
+}  // namespace
+
+Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) {
+  VLOG(10) << "Emitting reduction to vector " << unnested_hlo->ToString();
+
+  HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
+                                        ? unnested_hlo->fused_expression_root()
+                                        : unnested_hlo;
+  absl::Span<HloInstruction* const> output_instructions =
+      GetOutputInstructions(&reduce_or_tuple);
+  const HloInstruction* first_reduce =
+      GetFirstReduceInstruction(output_instructions);
+
+  if (output_instructions.size() > 1) {
+    TF_RETURN_IF_ERROR(
+        AreFusedReductionOutputsConsistent(output_instructions, first_reduce));
+  }
+
+  // Build an initializer thunk to initialize each reduction output.
+  std::vector<std::unique_ptr<Thunk>> thunks;
+  for (int i = 0, e = output_instructions.size(); i != e; ++i) {
+    if (output_instructions[i]->opcode() != HloOpcode::kReduce) {
+      continue;
+    }
+    TF_ASSIGN_OR_RETURN(
+        std::unique_ptr<Thunk> initializer_thunk,
+        BuildInitializerThunk(unnested_hlo,
+                              (output_instructions[i] == reduce_or_tuple)
+                                  ? ShapeIndex()
+                                  : ShapeIndex({i})));
+    thunks.push_back(std::move(initializer_thunk));
+  }
+
+  // Build a kernel thunk to compute all the outputs.
+  std::unique_ptr<KernelThunk> kernel_thunk =
+      BuildKernelThunk(unnested_hlo, /*implements_whole_instruction=*/false);
+
+  const Shape& input_shape = first_reduce->operand(0)->shape();
+  // The layout of a reduction input is either set by LayoutAssignment for
+  // unnested kReduce or by InstructionFusion for fused kReduce.
+  CHECK(input_shape.has_layout()) << "LayoutAssignment or InstructionFusion "
+                                     "doesn't set the input layout of "
+                                  << first_reduce->ToString();
+
+  bool is_row_reduction;
+  llvm_ir::KernelMappingScheme mapping_scheme;
+  std::tie(mapping_scheme, is_row_reduction) =
+      ComputeMappingSchemeAndReductionKind(first_reduce, &b_);
+  ReductionCodegenInfo reduction_info(&mapping_scheme, is_row_reduction);
+  KernelCodeGenerator kernel_generator(
+      /*tile_element_generator=*/
+      [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
+          const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+          llvm::Value* x_loc) {
+        EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc);
+      },
+      /*block_prologue_generator=*/
+      [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) {
+        EmitPrologueForReduction(hlo, kernel_info);
+      },
+      /*block_epilogue_generator*/
+      [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) {
+        EmitEpilogueForReduction(hlo, kernel_info);
+      });
+
+  LaunchDimensions launch_dimensions =
+      EmitKernel(unnested_hlo, {}, kernel_generator, &reduction_info);
+  UpdateLaunchDimensions(launch_dimensions, kernel_thunk.get(),
+                         ir_emitter_context_->llvm_module());
+
+  thunks.push_back(std::move(kernel_thunk));
+  std::unique_ptr<SequentialThunk> sequential_thunk =
+      absl::make_unique<SequentialThunk>(std::move(thunks), unnested_hlo);
+  AddThunkToThunkSequence(std::move(sequential_thunk));
+
+  return Status::OK();
+}
+
 Status IrEmitterUnnested::EmitConstantGlobals() {
   for (const BufferAllocation& allocation :
        ir_emitter_context_->buffer_assignment().Allocations()) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index e09ed657a8..85a0e5328c 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_
 #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_
 
+#include "absl/container/inlined_vector.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h"
 #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/thunk.h"
@@ -68,9 +69,12 @@ class IrEmitterUnnested : public IrEmitter {
     explicit KernelCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme)
         : mapping_scheme_(mapping_scheme),
           tiled_param_info_(nullptr),
-          lane_id_(nullptr) {}
+          lane_id_(nullptr),
+          index_ty_(nullptr) {}
+    virtual ~KernelCodegenInfo() {}
 
     void SetLaneId(llvm::Value* v) { lane_id_ = v; }
+    void SetIndexType(llvm::Type* t) { index_ty_ = t; }
     void SetTiledParamInfo(llvm_ir::TiledParameterInfo* tiled_param_info) {
       CHECK_EQ(tiled_param_info_, nullptr);
       tiled_param_info_ = tiled_param_info;
@@ -83,11 +87,13 @@ class IrEmitterUnnested : public IrEmitter {
     llvm_ir::TiledParameterInfo* GetTiledParameterInfo() const {
       return tiled_param_info_;
     }
+    llvm::Type* GetIndexType() const { return index_ty_; }
 
    private:
     llvm_ir::KernelMappingScheme* mapping_scheme_;
     llvm_ir::TiledParameterInfo* tiled_param_info_;
     llvm::Value* lane_id_;
+    llvm::Type* index_ty_;
   };
 
   // A function object to prepare for the code generation for a tile block.
@@ -200,82 +206,14 @@ class IrEmitterUnnested : public IrEmitter {
 
   // Helper for writing extra outputs from inside a reduce kernel.
   Status EmitExtraOutputsForReduce(
-      const HloInstruction* reduce, const llvm_ir::IrArray::Index& index,
+      const HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index,
       absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
           extra_output_gens);
 
-  // EmitColumnReduction and EmitRowReduction emit code for column and row
-  // reduction of a matrix and/or 3D tensor. Row and column reduction have
-  // different memory access pattern, so for performance their implementations
-  // are significantly different.
+  // Generates code for reduction to contiguous dimensions.
   //
-  // Emits code that reduces a matrix of shape [height x width] to a vector of
-  // [width]. Other parameters have the same meaning as those of
-  // `EmitReductionToVector`. Note that input shape might not be
-  // [height x width], but can be bitcast to [height x width] with "height"
-  // being the major dimension.
-  Status EmitColumnReduction(
-      KernelThunk* kernel_thunk, int64 height, int64 width,
-      HloInstruction* reduce, const Shape& input_shape,
-      absl::Span<const llvm_ir::ElementGenerator> input_gens,
-      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-      absl::Span<HloComputation* const> reducers,
-      absl::Span<const ShapeIndex> reduce_output_shapes,
-      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-          extra_output_gens);
-
-  // Emits code that reduces a 3D tensor of shape [depth x height x width] to a
-  // vector of shape [height]. Other parameters have the same meaning as those
-  // of `EmitReductionToVector`. Note that input shape might not be
-  // [depth x height x width], but can be bitcast to [depth x height x width]
-  // with "depth" being the most major dimension.
-  Status EmitRowReduction(
-      KernelThunk* kernel_thunk, int64 depth, int64 height, int64 width,
-      HloInstruction* reduce, const Shape& input_shape,
-      absl::Span<const llvm_ir::ElementGenerator> input_gens,
-      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-      absl::Span<HloComputation* const> reducers,
-      absl::Span<const ShapeIndex> reduce_output_shapes,
-      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-          extra_output_gens);
-
-  // Emits code that reduces a tensor of arbitrary rank to a scalar.
-  Status EmitReductionToScalar(
-      KernelThunk* kernel_thunk, HloInstruction* reduce,
-      const Shape& input_shape,
-      absl::Span<const llvm_ir::ElementGenerator> input_gens,
-      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-      absl::Span<HloComputation* const> reducers,
-      absl::Span<const ShapeIndex> reduce_output_shapes,
-      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-          extra_output_gens);
-
-  // Figures out whether `reduce` is a row or column reduction, and which
-  // dimensions to reduce, and calls either `EmitRowReduction` or
-  // `EmitColumnReduction` as appropriate. `input_shape` is the shape of the
-  // input array, which is the operand of the Reduce instruction if unfused or
-  // of the Fusion instruction if fused. `input_gen` and `init_value_gen`
-  // generate elements of the input and the initial value. Other parameters mean
-  // the same as for `HandleReduce`.
-  //
-  // Multiple reduces can be emitted in the same loop, assuming they have the
-  // same input and output shapes, and the same reduce dimensions.
-  //
-  // extra_output_gens can contain extra generators for intermediate outputs.
-  // These must have the same shape as the reduce input as they are computed
-  // when the reduce inputs are being read.
-  //
-  // Prerequisite: `IsReductionToVector(*reduce)`
-  Status EmitReductionToVector(
-      KernelThunk* kernel_thunk, HloInstruction* reduce,
-      const Shape& input_shape,
-      absl::Span<const llvm_ir::ElementGenerator> input_gens,
-      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
-      absl::Span<const int64> dimensions_to_reduce,
-      absl::Span<HloComputation* const> reducers,
-      absl::Span<const ShapeIndex> reduce_output_shapes,
-      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-          extra_output_gens);
+  // Prerequisite: `IsReductionToVector(*unnested_hlo)`
+  Status EmitReductionToVector(HloInstruction* unnested_hlo);
 
   // Emits code for an in-place scatter, modifying `thunk`s launch dimensions in
   // the process. `scatter` may be fused, scatter indices are taken from
@@ -314,6 +252,29 @@ class IrEmitterUnnested : public IrEmitter {
                                 const llvm_ir::IrArray::Index& index,
                                 const KernelCodegenInfo* kernel_info,
                                 llvm::Value* y_loc, llvm::Value* x_loc);
+  // Emits code to process a tensor element in a tile for the given input hlo
+  // that is either a unnested kReduce or a kInput fusion.
+  void EmitTileElementForReduction(HloInstruction* unnested_hlo,
+                                   const llvm_ir::IrArray::Index& index,
+                                   const KernelCodegenInfo* kernel_info,
+                                   llvm::Value* y_loc, llvm::Value* x_loc);
+  // Prepares for the code generation for a tile block of a reduction kernel.
+  void EmitPrologueForReduction(HloInstruction* unnested_hlo,
+                                KernelCodegenInfo* kernel_info);
+  void EmitPrologueForOneReduction(HloInstruction* unnested_hlo,
+                                   HloInstruction* reduce_inst, int reduce_idx,
+                                   KernelCodegenInfo* kernel_info,
+                                   GpuElementalIrEmitter* elemental_emitter,
+                                   ShapeIndex output_shape_index);
+  // Wraps up the code generation for a tile block of a reduction kernel.
+  void EmitEpilogueForReduction(HloInstruction* unnested_hlo,
+                                KernelCodegenInfo* kernel_info);
+  // For each reducer, emits the shuffle-down loop to accumulate the partial
+  // result to the global result.
+  void EmitFullWarpShuffleDownLoopForAllReduces(
+      const absl::InlinedVector<HloComputation*, 1>& reducers,
+      const absl::InlinedVector<llvm::AllocaInst*, 1>&
+          partial_result_addresses);
 
   // Generates the IrArray for each input of an hlo and returns a vector that
   // constains such IrArrays.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index c26711e526..1aa85eb8d2 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -120,7 +120,7 @@ KernelMappingScheme::KernelMappingScheme(
     absl::Span<const int64> req_block_sizes, int64 num_threads_y,
     int64 num_threads_x, llvm::IRBuilder<>* b)
     : b_(b),
-      dims_in_elems_(dims_in_elems),
+      dims_in_elems_(dims_in_elems.begin(), dims_in_elems.end()),
       tile_sizes_{1, tile_size_y, tile_size_x},
       num_threads_x_(num_threads_x),
       num_threads_y_(num_threads_y) {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index 06002d57b0..7277aeac8a 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -90,15 +90,16 @@ class KernelMappingScheme {
   enum { DimZ = 0, DimY, DimX, DimTot };
 
  public:
+  KernelMappingScheme() {}
   // dims_in_elems: the normalized tensor dimensions.
   // req_block_sizes: the requested block size in number of tiles for each
   //   dimension. The actual block size is set to min(req_block_size,
   //   dims_in_number_of_blocks).
-  explicit KernelMappingScheme(absl::Span<const int64> dims_in_elems,
-                               int64 tile_size_y, int64 tile_size_x,
-                               absl::Span<const int64> req_block_sizes,
-                               int64 num_threads_y, int64 num_threads_x,
-                               llvm::IRBuilder<>* b);
+  KernelMappingScheme(absl::Span<const int64> dims_in_elems, int64 tile_size_y,
+                      int64 tile_size_x,
+                      absl::Span<const int64> req_block_sizes,
+                      int64 num_threads_y, int64 num_threads_x,
+                      llvm::IRBuilder<>* b);
 
   absl::Span<const int64> GetDimensionsInElements() const {
     return dims_in_elems_;
@@ -133,6 +134,10 @@ class KernelMappingScheme {
   }
 
   absl::Span<const int64> GetBlockSizes() const { return block_sizes_; }
+  int64 GetTileBlockSizeForDimension(int d) const {
+    DCHECK(d >= DimZ && d <= DimX);
+    return dims_in_blocks_[d];
+  }
 
   int64 GetNumberOfThreadsForDimensionX() const { return num_threads_x_; }
   int64 GetNumberOfThreadsForDimensionY() const { return num_threads_y_; }
@@ -163,7 +168,7 @@ class KernelMappingScheme {
  private:
   llvm::IRBuilder<>* b_;
   // The number of elements in each dimension.
-  absl::Span<const int64> dims_in_elems_;
+  std::vector<int64> dims_in_elems_;
 
   // The number of elements for each dimension of a tile.
   std::vector<int64> tile_sizes_;
-- 
GitLab


From d21436b6fc251e84350efb2a0a1c181a05140a5c Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Sun, 25 Nov 2018 14:46:49 -0800
Subject: [PATCH 0752/1554] Add input processing to micro speech example

PiperOrigin-RevId: 222753537
---
 .../micro/examples/micro_speech/BUILD         |  79 ++++++++++++
 .../examples/micro_speech/audio_provider.cc   |  33 +++++
 .../examples/micro_speech/audio_provider.h    |  36 ++++++
 .../micro_speech/audio_provider_test.cc       |  44 +++++++
 .../examples/micro_speech/feature_provider.cc | 121 ++++++++++++++++++
 .../examples/micro_speech/feature_provider.h  |  48 +++++++
 .../micro_speech/feature_provider_test.cc     |  38 ++++++
 .../micro_speech/fixed_point/preprocessor.cc  |  44 +++----
 .../micro/examples/micro_speech/main.cc       | 112 ++++++++++++++++
 .../examples/micro_speech/model_settings.h    |  47 +++++++
 .../examples/micro_speech/preprocessor.cc     |  36 +++---
 .../examples/micro_speech/preprocessor.h      |   5 +
 .../micro/examples/micro_speech/timer.cc      |  22 ++++
 .../micro/examples/micro_speech/timer.h       |  31 +++++
 .../micro/examples/micro_speech/timer_test.cc |  46 +++++++
 .../experimental/micro/testing/micro_test.h   |  18 +++
 16 files changed, 715 insertions(+), 45 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 07fb876411..d48eb656b5 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -34,6 +34,7 @@ tflite_micro_cc_test(
 tflite_micro_cc_test(
     name = "preprocessor_reference_test",
     srcs = [
+        "model_settings.h",
         "no_30ms_sample_data.cc",
         "no_30ms_sample_data.h",
         "no_power_spectrum_data.cc",
@@ -57,6 +58,7 @@ tflite_micro_cc_test(
     name = "preprocessor_fixed_test",
     srcs = [
         "fixed_point/preprocessor.cc",
+        "model_settings.h",
         "no_30ms_sample_data.cc",
         "no_30ms_sample_data.h",
         "no_power_spectrum_data.cc",
@@ -74,3 +76,80 @@ tflite_micro_cc_test(
         "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
+
+tflite_micro_cc_test(
+    name = "audio_provider_test",
+    srcs = [
+        "audio_provider.cc",
+        "audio_provider.h",
+        "audio_provider_test.cc",
+        "model_settings.h",
+    ],
+    deps = [
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "feature_provider_test",
+    srcs = [
+        "audio_provider.cc",
+        "audio_provider.h",
+        "feature_provider.cc",
+        "feature_provider.h",
+        "feature_provider_test.cc",
+        "model_settings.h",
+        "preprocessor.cc",
+        "preprocessor.h",
+        "timer.cc",
+        "timer.h",
+    ],
+    deps = [
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "timer_test",
+    srcs = [
+        "timer.cc",
+        "timer.h",
+        "timer_test.cc",
+    ],
+    deps = [
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
+    ],
+)
+
+cc_binary(
+    name = "micro_speech",
+    srcs = [
+        "audio_provider.cc",
+        "audio_provider.h",
+        "feature_provider.cc",
+        "feature_provider.h",
+        "main.cc",
+        "model_settings.h",
+        "preprocessor.cc",
+        "preprocessor.h",
+        "timer.cc",
+        "timer.h",
+        "tiny_conv_model_data.cc",
+        "tiny_conv_model_data.h",
+        "yes_features_data.cc",
+        "yes_features_data.h",
+    ],
+    deps = [
+        "//tensorflow/lite:schema_fbs_version",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver",
+        "//tensorflow/lite/experimental/micro/kernels:micro_ops",
+        "//tensorflow/lite/schema:schema_fbs",
+    ],
+)
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
new file mode 100644
index 0000000000..c0365d5690
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
@@ -0,0 +1,33 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+namespace {
+int16_t g_dummy_audio_data[kMaxAudioSampleSize];
+}  // namespace
+
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
+    g_dummy_audio_data[i] = 0;
+  }
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_dummy_audio_data;
+  return kTfLiteOk;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
new file mode 100644
index 0000000000..7e2442a5e8
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
@@ -0,0 +1,36 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+
+// This is an abstraction around an audio source like a microphone, and is
+// expected to return 16-bit PCM sample data for a given point in time. The
+// sample data itself should be used as quickly as possible by the caller, since
+// to allow memory optimizations there are no guarantees that the samples won't
+// be overwritten by new data in the future. In practice, implementations should
+// ensure that there's a reasonable time allowed for clients to access the data
+// before any reuse.
+// The reference implementation can have no platform-specific dependencies, so
+// it just returns an array filled with zeros. For real applications, you should
+// ensure there's a specialized implementation that accesses hardware APIs.
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples);
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
new file mode 100644
index 0000000000..5f7c7605f0
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
@@ -0,0 +1,44 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(TestAudioProvider) {
+  tflite::MicroErrorReporter micro_error_reporter;
+  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
+
+  int audio_samples_size = 0;
+  int16_t* audio_samples = nullptr;
+  TfLiteStatus get_status =
+      GetAudioSamples(error_reporter, 0, kFeatureSliceDurationMs,
+                      &audio_samples_size, &audio_samples);
+  TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, get_status);
+  TF_LITE_MICRO_EXPECT_LE(audio_samples_size, kMaxAudioSampleSize);
+  TF_LITE_MICRO_EXPECT_NE(audio_samples, nullptr);
+
+  // Make sure we can read all of the returned memory locations.
+  int total = 0;
+  for (int i = 0; i < audio_samples_size; ++i) {
+    total += audio_samples[i];
+  }
+}
+
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
new file mode 100644
index 0000000000..c4c52ac0ff
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
@@ -0,0 +1,121 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+namespace {
+// Stores the timestamp for the previous fetch of audio data, so that we can
+// avoid recalculating all the features from scratch if some earlier timeslices
+// are still present.
+int32_t g_last_time_in_ms = 0;
+// Make sure we don't try to use cached information if this is the first call
+// into the provider.
+bool g_is_first_run = true;
+}  // namespace
+
+FeatureProvider::FeatureProvider(int feature_size, uint8_t* feature_data)
+    : feature_size_(feature_size), feature_data_(feature_data) {
+  // Initialize the feature data to default values.
+  for (int n = 0; n < feature_size_; ++n) {
+    feature_data_[n] = 0;
+  }
+}
+
+FeatureProvider::~FeatureProvider() {}
+
+TfLiteStatus FeatureProvider::PopulateFeatureData(
+    tflite::ErrorReporter* error_reporter, int* how_many_new_slices) {
+  if (feature_size_ != kFeatureElementCount) {
+    error_reporter->Report("Requested feature_data_ size %d doesn't match %d",
+                           feature_size_, kFeatureElementCount);
+    return kTfLiteError;
+  }
+
+  const int32_t time_in_ms = TimeInMilliseconds();
+  // Quantize the time into steps as long as each window stride, so we can
+  // figure out which audio data we need to fetch.
+  const int last_step = (g_last_time_in_ms / kFeatureSliceStrideMs);
+  const int current_step = (time_in_ms / kFeatureSliceStrideMs);
+  g_last_time_in_ms = time_in_ms;
+
+  int slices_needed = current_step - last_step;
+  // If this is the first call, make sure we don't use any cached information.
+  if (g_is_first_run) {
+    g_is_first_run = false;
+    slices_needed = kFeatureSliceCount;
+  }
+  if (slices_needed > kFeatureSliceCount) {
+    slices_needed = kFeatureSliceCount;
+  }
+  *how_many_new_slices = slices_needed;
+
+  const int slices_to_keep = kFeatureSliceCount - slices_needed;
+  const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
+  // If we can avoid recalculating some slices, just move the existing data
+  // up in the spectrogram, to perform something like this:
+  // last time = 80ms          current time = 120ms
+  // +-----------+             +-----------+
+  // | data@20ms |         --> | data@60ms |
+  // +-----------+       --    +-----------+
+  // | data@40ms |     --  --> | data@80ms |
+  // +-----------+   --  --    +-----------+
+  // | data@60ms | --  --      |  <empty>  |
+  // +-----------+   --        +-----------+
+  // | data@80ms | --          |  <empty>  |
+  // +-----------+             +-----------+
+  if (slices_to_keep > 0) {
+    for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
+      uint8_t* dest_slice_data =
+          feature_data_ + (dest_slice * kFeatureSliceSize);
+      const int src_slice = dest_slice + slices_to_drop;
+      const uint8_t* src_slice_data =
+          feature_data_ + (src_slice * kFeatureSliceSize);
+      for (int i = 0; i < kFeatureSliceSize; ++i) {
+        dest_slice_data[i] = src_slice_data[i];
+      }
+    }
+  }
+  // Any slices that need to be filled in with feature data have their
+  // appropriate audio data pulled, and features calculated for that slice.
+  if (slices_needed > 0) {
+    for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount;
+         ++new_slice) {
+      const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
+      const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
+      int16_t* audio_samples = nullptr;
+      int audio_samples_size = 0;
+      GetAudioSamples(error_reporter, slice_start_ms, kFeatureSliceDurationMs,
+                      &audio_samples_size, &audio_samples);
+      if (audio_samples_size < kMaxAudioSampleSize) {
+        error_reporter->Report("Audio data size %d  too small, want %d",
+                               audio_samples_size, kMaxAudioSampleSize);
+        return kTfLiteError;
+      }
+      uint8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
+      TfLiteStatus preprocess_status =
+          Preprocess(error_reporter, audio_samples, audio_samples_size,
+                     kFeatureSliceSize, new_slice_data);
+      if (preprocess_status != kTfLiteOk) {
+        return preprocess_status;
+      }
+    }
+  }
+  return kTfLiteOk;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
new file mode 100644
index 0000000000..a86c56ebf0
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+
+// Binds itself to an area of memory intended to hold the input features for an
+// audio-recognition neural network model, and fills that data area with the
+// features representing the current audio input, for example from a microphone.
+// The audio features themselves are a two-dimensional array, made up of
+// horizontal slices representing the frequencies at one point in time, stacked
+// on top of each other to form a spectrogram showing how those frequencies
+// changed over time.
+class FeatureProvider {
+ public:
+  // Create the provider, and bind it to an area of memory. This memory should
+  // remain accessible for the lifetime of the provider object, since subsequent
+  // calls will fill it with feature data. The provider does no memory
+  // management of this data.
+  FeatureProvider(int feature_size, uint8_t* feature_data);
+  ~FeatureProvider();
+
+  // Fills the feature data with information from audio inputs, and returns how
+  // many feature slices were updated.
+  TfLiteStatus PopulateFeatureData(tflite::ErrorReporter* error_reporter,
+                                   int* how_many_new_slices);
+
+ private:
+  int feature_size_;
+  uint8_t* feature_data_;
+};
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
new file mode 100644
index 0000000000..1e52aec8d2
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(TestFeatureProvider) {
+  tflite::MicroErrorReporter micro_error_reporter;
+  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
+
+  uint8_t feature_data[kFeatureElementCount];
+  FeatureProvider feature_provider(kFeatureElementCount, feature_data);
+
+  int how_many_new_slices = 0;
+  TfLiteStatus populate_status = feature_provider.PopulateFeatureData(
+      error_reporter, &how_many_new_slices);
+  TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, populate_status);
+  TF_LITE_MICRO_EXPECT_EQ(kFeatureSliceCount, how_many_new_slices);
+}
+
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
index de60c982f3..b623d8d11b 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
@@ -31,6 +31,8 @@ limitations under the License.
 
 #include <cmath>
 
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
 namespace {
 
 // q format notation: qx.y => 1 sign bit, x-1 integer bits, y fraction bits.
@@ -66,13 +68,6 @@ inline int32_t FloatToFixed_Q2_30(float input) {
   return static_cast<int32_t>(roundf(input * (1 << 30)));
 }
 
-// These constants allow us to allocate fixed-sized arrays on the stack for our
-// working memory.
-constexpr int kInputSize = 512;
-constexpr int kAverageWindowSize = 6;
-constexpr int kOutputSize =
-    ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
-
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
 // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft.
@@ -127,14 +122,14 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output) {
   // Ensure our input and output data arrays are valid.
-  if (input_size > kInputSize) {
+  if (input_size > kMaxAudioSampleSize) {
     error_reporter->Report("Input size %d larger than %d", input_size,
-                           kInputSize);
+                           kMaxAudioSampleSize);
     return kTfLiteError;
   }
-  if (output_size != kOutputSize) {
+  if (output_size != kFeatureSliceSize) {
     error_reporter->Report("Requested output size %d doesn't match %d",
-                           output_size, kOutputSize);
+                           output_size, kFeatureSliceSize);
     return kTfLiteError;
   }
 
@@ -142,18 +137,17 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
   // In a real application, we'd calculate this table once in an initialization
   // function and store it for repeated reuse.
   // q1.15 format.
-  int16_t window_function[kInputSize];
+  int16_t window_function[kMaxAudioSampleSize];
   CalculatePeriodicHann(input_size, window_function);
 
   // Apply the window function to our time series input, and pad it with zeroes
   // to the next power of two.
-  int32_t fixed_input[kInputSize];
-  for (int i = 0; i < kInputSize; ++i) {
+  int32_t fixed_input[kMaxAudioSampleSize];
+  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
     if (i < input_size) {
       // input is int16_t.  Treat as q1.15 fixed point value in range [-1,1)
       // window_function is also q1.15 fixed point number
-      fixed_input[i] =
-          Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]);
+      fixed_input[i] = Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]);
     } else {
       fixed_input[i] = 0;
     }
@@ -161,31 +155,31 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
 
   // Pull the frequency data from the time series sample.
   // Calculated in q10.22 format from q2.30 inputs.
-  int32_t fourier_values[kInputSize];
-  CalculateDiscreteFourierTransform(fixed_input, kInputSize, fourier_values);
+  int32_t fourier_values[kMaxAudioSampleSize];
+  CalculateDiscreteFourierTransform(fixed_input, kMaxAudioSampleSize,
+                                    fourier_values);
 
   // We have the complex numbers giving us information about each frequency
   // band, but all we want to know is how strong each frequency is, so calculate
   // the squared magnitude by adding together the squares of each component.
-  int32_t power_spectrum[kInputSize / 2];
-  for (int i = 0; i < (kInputSize / 2); ++i) {
+  int32_t power_spectrum[kMaxAudioSampleSize / 2];
+  for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) {
     const int32_t real = fourier_values[(i * 2) + 0];
     const int32_t imaginary = fourier_values[(i * 2) + 1];
     // q10.22 results
-    power_spectrum[i] =
-        Q10_22_FixedMultiply_Q10_22(real, real) +
-        Q10_22_FixedMultiply_Q10_22(imaginary, imaginary);
+    power_spectrum[i] = Q10_22_FixedMultiply_Q10_22(real, real) +
+                        Q10_22_FixedMultiply_Q10_22(imaginary, imaginary);
   }
 
   // Finally, reduce the size of the output by averaging together six adjacent
   // frequencies into each slot, producing an array of 43 values.
   // Power_spectrum numbers are q10.22.  Divide by kAverageWindowSize inside
   // loop to prevent overflow.
-  for (int i = 0; i < kOutputSize; ++i) {
+  for (int i = 0; i < kFeatureSliceSize; ++i) {
     int32_t average = 0;
     for (int j = 0; j < kAverageWindowSize; ++j) {
       const int index = (i * kAverageWindowSize) + j;
-      if (index < (kInputSize / 2)) {
+      if (index < (kMaxAudioSampleSize / 2)) {
         average += power_spectrum[index] / kAverageWindowSize;
       }
     }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
new file mode 100644
index 0000000000..1890c25cf2
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
@@ -0,0 +1,112 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h"
+#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/micro_interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/version.h"
+
+int main(int argc, char* argv[]) {
+  // Set up logging.
+  tflite::MicroErrorReporter micro_error_reporter;
+  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
+
+  // Map the model into a usable data structure. This doesn't involve any
+  // copying or parsing, it's a very lightweight operation.
+  const tflite::Model* model = ::tflite::GetModel(g_tiny_conv_model_data);
+  if (model->version() != TFLITE_SCHEMA_VERSION) {
+    error_reporter->Report(
+        "Model provided is schema version %d not equal "
+        "to supported version %d.\n",
+        model->version(), TFLITE_SCHEMA_VERSION);
+    return 1;
+  }
+
+  // This pulls in all the operation implementations we need.
+  tflite::ops::micro::AllOpsResolver resolver;
+
+  // Create an area of memory to use for input, output, and intermediate arrays.
+  // The size of this will depend on the model you're using, and may need to be
+  // determined by experimentation.
+  const int tensor_arena_size = 10 * 1024;
+  uint8_t tensor_arena[tensor_arena_size];
+  tflite::SimpleTensorAllocator tensor_allocator(tensor_arena,
+                                                 tensor_arena_size);
+
+  // Build an interpreter to run the model with.
+  tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator,
+                                       error_reporter);
+
+  // Get information about the memory area to use for the model's input.
+  TfLiteTensor* model_input = interpreter.input(0);
+  if ((model_input->dims->size != 4) || (model_input->dims->data[0] != 1) ||
+      (model_input->dims->data[1] != kFeatureSliceCount) ||
+      (model_input->dims->data[2] != kFeatureSliceSize) ||
+      (model_input->type != kTfLiteUInt8)) {
+    error_reporter->Report("Bad input tensor parameters in model");
+    return 1;
+  }
+
+  // Prepare to access the audio spectrograms from a microphone or other source
+  // that will provide the inputs to the neural network.
+  FeatureProvider feature_provider(kFeatureElementCount,
+                                   model_input->data.uint8);
+
+  // Keep reading and analysing audio data in an infinite loop.
+  while (true) {
+    // Fetch the spectrogram for the current time.
+    int how_many_new_slices = 0;
+    TfLiteStatus feature_status = feature_provider.PopulateFeatureData(
+        error_reporter, &how_many_new_slices);
+    if (feature_status != kTfLiteOk) {
+      error_reporter->Report("Feature generation failed");
+      return 1;
+    }
+    // If no new audio samples have been received since last time, don't bother
+    // running the network model.
+    if (how_many_new_slices == 0) {
+      continue;
+    }
+
+    // Run the model on the spectrogram input and make sure it succeeds.
+    TfLiteStatus invoke_status = interpreter.Invoke();
+    if (invoke_status != kTfLiteOk) {
+      error_reporter->Report("Invoke failed");
+      return 1;
+    }
+
+    // The output from the model is a vector containing the scores for each
+    // kind of prediction, so figure out what the highest scoring category was.
+    TfLiteTensor* output = interpreter.output(0);
+    uint8_t top_category_score = 0;
+    int top_category_index = 0;
+    for (int category_index = 0; category_index < kCategoryCount;
+         ++category_index) {
+      const uint8_t category_score = output->data.uint8[category_index];
+      if (category_score > top_category_score) {
+        top_category_score = category_score;
+        top_category_index = category_index;
+      }
+    }
+
+    error_reporter->Report("Heard %s", kCategoryLabels[top_category_index]);
+  }
+
+  return 0;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
new file mode 100644
index 0000000000..88369a07ee
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
@@ -0,0 +1,47 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
+
+// Keeping these as constant expressions allow us to allocate fixed-sized arrays
+// on the stack for our working memory.
+
+// The size of the input time series data we pass to the FFT to produce the
+// frequency information. This has to be a power of two, and since we're dealing
+// with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
+constexpr int kMaxAudioSampleSize = 512;
+
+// All of these values are derived from the values used during model training,
+// if you change your model you'll need to update these constants.
+constexpr int kAverageWindowSize = 6;
+constexpr int kFeatureSliceSize =
+    ((kMaxAudioSampleSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
+constexpr int kFeatureSliceCount = 49;
+constexpr int kFeatureElementCount = (kFeatureSliceSize * kFeatureSliceCount);
+constexpr int kFeatureSliceStrideMs = 20;
+constexpr int kFeatureSliceDurationMs = 30;
+
+constexpr int kCategoryCount = 4;
+constexpr int kSilenceIndex = 0;
+constexpr int kUnknownIndex = 1;
+constexpr char* kCategoryLabels[kCategoryCount] = {
+    "silence",
+    "unknown",
+    "yes",
+    "no",
+};
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
index 12f9e22038..f4a7f801cc 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
@@ -28,14 +28,9 @@ limitations under the License.
 
 #include <cmath>
 
-namespace {
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
 
-// These constants allow us to allocate fixed-sized arrays on the stack for our
-// working memory.
-constexpr int kInputSize = 512;
-constexpr int kAverageWindowSize = 6;
-constexpr int kOutputSize =
-    ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
+namespace {
 
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
@@ -78,27 +73,27 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output) {
   // Ensure our input and output data arrays are valid.
-  if (input_size > kInputSize) {
+  if (input_size > kMaxAudioSampleSize) {
     error_reporter->Report("Input size %d larger than %d", input_size,
-                           kInputSize);
+                           kMaxAudioSampleSize);
     return kTfLiteError;
   }
-  if (output_size != kOutputSize) {
+  if (output_size != kFeatureSliceSize) {
     error_reporter->Report("Requested output size %d doesn't match %d",
-                           output_size, kOutputSize);
+                           output_size, kFeatureSliceSize);
     return kTfLiteError;
   }
 
   // Pre-calculate the window function we'll be applying to the input data.
   // In a real application, we'd calculate this table once in an initialization
   // function and store it for repeated reuse.
-  float window_function[kInputSize];
+  float window_function[kMaxAudioSampleSize];
   CalculatePeriodicHann(input_size, window_function);
 
   // Apply the window function to our time series input, and pad it with zeroes
   // to the next power of two.
-  float float_input[kInputSize];
-  for (int i = 0; i < kInputSize; ++i) {
+  float float_input[kMaxAudioSampleSize];
+  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
     if (i < input_size) {
       float_input[i] =
           (input[i] * window_function[i]) / static_cast<float>(1 << 15);
@@ -108,14 +103,15 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
   }
 
   // Pull the frequency data from the time series sample.
-  float fourier_values[kInputSize];
-  CalculateDiscreteFourierTransform(float_input, kInputSize, fourier_values);
+  float fourier_values[kMaxAudioSampleSize];
+  CalculateDiscreteFourierTransform(float_input, kMaxAudioSampleSize,
+                                    fourier_values);
 
   // We have the complex numbers giving us information about each frequency
   // band, but all we want to know is how strong each frequency is, so calculate
   // the squared magnitude by adding together the squares of each component.
-  float power_spectrum[kInputSize / 2];
-  for (int i = 0; i < (kInputSize / 2); ++i) {
+  float power_spectrum[kMaxAudioSampleSize / 2];
+  for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) {
     const float real = fourier_values[(i * 2) + 0];
     const float imaginary = fourier_values[(i * 2) + 1];
     power_spectrum[i] = (real * real) + (imaginary * imaginary);
@@ -123,11 +119,11 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
 
   // Finally, reduce the size of the output by averaging together six adjacent
   // frequencies into each slot, producing an array of 43 values.
-  for (int i = 0; i < kOutputSize; ++i) {
+  for (int i = 0; i < kFeatureSliceSize; ++i) {
     float total = 0.0f;
     for (int j = 0; j < kAverageWindowSize; ++j) {
       const int index = (i * kAverageWindowSize) + j;
-      if (index < (kInputSize / 2)) {
+      if (index < (kMaxAudioSampleSize / 2)) {
         total += power_spectrum[index];
       }
     }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
index dede2a8642..adff790d6c 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
@@ -19,6 +19,11 @@ limitations under the License.
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
 
+// Converts audio sample data into a more compact form that's appropriate for
+// feeding into a neural network. There are reference implementations that use
+// both floating point and fixed point available, but because the calculations
+// involved can be time-consuming, it's recommended that you use or write
+// specialized versions for your platform.
 TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output);
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
new file mode 100644
index 0000000000..6c96a61ab5
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
@@ -0,0 +1,22 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+int32_t TimeInMilliseconds() {
+  static int current_time = 0;
+  current_time += 100;
+  return current_time;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
new file mode 100644
index 0000000000..162952844a
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
+
+#include <cstdint>
+
+// Returns the time in milliseconds. There's no contract about what time zero
+// represents, the accuracy, or the granularity of the result. Subsequent calls
+// will generally not return a lower value, but even that's not guaranteed if
+// there's an overflow  wraparound.
+// The reference implementation of this function just returns a constantly
+// incrementing value for each call, since it would need a non-portable platform
+// call to access time information. For real applications, you'll need to write
+// your own platform-specific implementation.
+int32_t TimeInMilliseconds();
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
new file mode 100644
index 0000000000..83a2dfcc65
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
@@ -0,0 +1,46 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+#include <limits>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(TestTimer) {
+  // Make sure that the technically-undefined overflow behavior we rely on below
+  // works on this platform. It's still not guaranteed, but at least this is a
+  // sanity check.
+  int32_t overflow_value = std::numeric_limits<int32_t>::max();
+  overflow_value += 1;
+  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
+
+  const int32_t first_time = TimeInMilliseconds();
+  const int32_t second_time = TimeInMilliseconds();
+
+  // It's possible that the timer may have wrapped around from +BIG_NUM to
+  // -BIG_NUM between the first and second calls, since we're storing
+  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
+  // would have taken more than 2^31 milliseconds though, so look at the
+  // difference and rely on integer overflow to ensure it's accurate.
+  const int32_t time_delta = (second_time - first_time);
+  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
+}
+
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/testing/micro_test.h b/tensorflow/lite/experimental/micro/testing/micro_test.h
index 10bab05fae..2f20dd5ac7 100644
--- a/tensorflow/lite/experimental/micro/testing/micro_test.h
+++ b/tensorflow/lite/experimental/micro/testing/micro_test.h
@@ -153,4 +153,22 @@ extern tflite::ErrorReporter* reporter;
     }                                                                        \
   } while (false)
 
+#define TF_LITE_MICRO_EXPECT_GE(x, y)                                         \
+  do {                                                                        \
+    if ((x) < (y)) {                                                          \
+      micro_test::reporter->Report(#x " >= " #y " failed at %s:%d", __FILE__, \
+                                   __LINE__);                                 \
+      micro_test::did_test_fail = true;                                       \
+    }                                                                         \
+  } while (false)
+
+#define TF_LITE_MICRO_EXPECT_LE(x, y)                                         \
+  do {                                                                        \
+    if ((x) > (y)) {                                                          \
+      micro_test::reporter->Report(#x " <= " #y " failed at %s:%d", __FILE__, \
+                                   __LINE__);                                 \
+      micro_test::did_test_fail = true;                                       \
+    }                                                                         \
+  } while (false)
+
 #endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_TESTING_MICRO_TEST_H_
-- 
GitLab


From a49887b0d24c9c1a97c5915147473bb09f854ebc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 25 Nov 2018 14:52:14 -0800
Subject: [PATCH 0753/1554] Automated rollback of commit
 d21436b6fc251e84350efb2a0a1c181a05140a5c

PiperOrigin-RevId: 222753714
---
 .../micro/examples/micro_speech/BUILD         |  79 ------------
 .../examples/micro_speech/audio_provider.cc   |  33 -----
 .../examples/micro_speech/audio_provider.h    |  36 ------
 .../micro_speech/audio_provider_test.cc       |  44 -------
 .../examples/micro_speech/feature_provider.cc | 121 ------------------
 .../examples/micro_speech/feature_provider.h  |  48 -------
 .../micro_speech/feature_provider_test.cc     |  38 ------
 .../micro_speech/fixed_point/preprocessor.cc  |  44 ++++---
 .../micro/examples/micro_speech/main.cc       | 112 ----------------
 .../examples/micro_speech/model_settings.h    |  47 -------
 .../examples/micro_speech/preprocessor.cc     |  36 +++---
 .../examples/micro_speech/preprocessor.h      |   5 -
 .../micro/examples/micro_speech/timer.cc      |  22 ----
 .../micro/examples/micro_speech/timer.h       |  31 -----
 .../micro/examples/micro_speech/timer_test.cc |  46 -------
 .../experimental/micro/testing/micro_test.h   |  18 ---
 16 files changed, 45 insertions(+), 715 deletions(-)
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index d48eb656b5..07fb876411 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -34,7 +34,6 @@ tflite_micro_cc_test(
 tflite_micro_cc_test(
     name = "preprocessor_reference_test",
     srcs = [
-        "model_settings.h",
         "no_30ms_sample_data.cc",
         "no_30ms_sample_data.h",
         "no_power_spectrum_data.cc",
@@ -58,7 +57,6 @@ tflite_micro_cc_test(
     name = "preprocessor_fixed_test",
     srcs = [
         "fixed_point/preprocessor.cc",
-        "model_settings.h",
         "no_30ms_sample_data.cc",
         "no_30ms_sample_data.h",
         "no_power_spectrum_data.cc",
@@ -76,80 +74,3 @@ tflite_micro_cc_test(
         "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
-
-tflite_micro_cc_test(
-    name = "audio_provider_test",
-    srcs = [
-        "audio_provider.cc",
-        "audio_provider.h",
-        "audio_provider_test.cc",
-        "model_settings.h",
-    ],
-    deps = [
-        "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
-        "//tensorflow/lite/experimental/micro/testing:micro_test",
-    ],
-)
-
-tflite_micro_cc_test(
-    name = "feature_provider_test",
-    srcs = [
-        "audio_provider.cc",
-        "audio_provider.h",
-        "feature_provider.cc",
-        "feature_provider.h",
-        "feature_provider_test.cc",
-        "model_settings.h",
-        "preprocessor.cc",
-        "preprocessor.h",
-        "timer.cc",
-        "timer.h",
-    ],
-    deps = [
-        "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
-        "//tensorflow/lite/experimental/micro/testing:micro_test",
-    ],
-)
-
-tflite_micro_cc_test(
-    name = "timer_test",
-    srcs = [
-        "timer.cc",
-        "timer.h",
-        "timer_test.cc",
-    ],
-    deps = [
-        "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
-        "//tensorflow/lite/experimental/micro/testing:micro_test",
-    ],
-)
-
-cc_binary(
-    name = "micro_speech",
-    srcs = [
-        "audio_provider.cc",
-        "audio_provider.h",
-        "feature_provider.cc",
-        "feature_provider.h",
-        "main.cc",
-        "model_settings.h",
-        "preprocessor.cc",
-        "preprocessor.h",
-        "timer.cc",
-        "timer.h",
-        "tiny_conv_model_data.cc",
-        "tiny_conv_model_data.h",
-        "yes_features_data.cc",
-        "yes_features_data.h",
-    ],
-    deps = [
-        "//tensorflow/lite:schema_fbs_version",
-        "//tensorflow/lite/experimental/micro:micro_framework",
-        "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver",
-        "//tensorflow/lite/experimental/micro/kernels:micro_ops",
-        "//tensorflow/lite/schema:schema_fbs",
-    ],
-)
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
deleted file mode 100644
index c0365d5690..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+++ /dev/null
@@ -1,33 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-
-namespace {
-int16_t g_dummy_audio_data[kMaxAudioSampleSize];
-}  // namespace
-
-TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
-                             int start_ms, int duration_ms,
-                             int* audio_samples_size, int16_t** audio_samples) {
-  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
-    g_dummy_audio_data[i] = 0;
-  }
-  *audio_samples_size = kMaxAudioSampleSize;
-  *audio_samples = g_dummy_audio_data;
-  return kTfLiteOk;
-}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
deleted file mode 100644
index 7e2442a5e8..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-
-// This is an abstraction around an audio source like a microphone, and is
-// expected to return 16-bit PCM sample data for a given point in time. The
-// sample data itself should be used as quickly as possible by the caller, since
-// to allow memory optimizations there are no guarantees that the samples won't
-// be overwritten by new data in the future. In practice, implementations should
-// ensure that there's a reasonable time allowed for clients to access the data
-// before any reuse.
-// The reference implementation can have no platform-specific dependencies, so
-// it just returns an array filled with zeros. For real applications, you should
-// ensure there's a specialized implementation that accesses hardware APIs.
-TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
-                             int start_ms, int duration_ms,
-                             int* audio_samples_size, int16_t** audio_samples);
-
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
deleted file mode 100644
index 5f7c7605f0..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
+++ /dev/null
@@ -1,44 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
-
-TF_LITE_MICRO_TESTS_BEGIN
-
-TF_LITE_MICRO_TEST(TestAudioProvider) {
-  tflite::MicroErrorReporter micro_error_reporter;
-  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
-
-  int audio_samples_size = 0;
-  int16_t* audio_samples = nullptr;
-  TfLiteStatus get_status =
-      GetAudioSamples(error_reporter, 0, kFeatureSliceDurationMs,
-                      &audio_samples_size, &audio_samples);
-  TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, get_status);
-  TF_LITE_MICRO_EXPECT_LE(audio_samples_size, kMaxAudioSampleSize);
-  TF_LITE_MICRO_EXPECT_NE(audio_samples, nullptr);
-
-  // Make sure we can read all of the returned memory locations.
-  int total = 0;
-  for (int i = 0; i < audio_samples_size; ++i) {
-    total += audio_samples[i];
-  }
-}
-
-TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
deleted file mode 100644
index c4c52ac0ff..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
+++ /dev/null
@@ -1,121 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-namespace {
-// Stores the timestamp for the previous fetch of audio data, so that we can
-// avoid recalculating all the features from scratch if some earlier timeslices
-// are still present.
-int32_t g_last_time_in_ms = 0;
-// Make sure we don't try to use cached information if this is the first call
-// into the provider.
-bool g_is_first_run = true;
-}  // namespace
-
-FeatureProvider::FeatureProvider(int feature_size, uint8_t* feature_data)
-    : feature_size_(feature_size), feature_data_(feature_data) {
-  // Initialize the feature data to default values.
-  for (int n = 0; n < feature_size_; ++n) {
-    feature_data_[n] = 0;
-  }
-}
-
-FeatureProvider::~FeatureProvider() {}
-
-TfLiteStatus FeatureProvider::PopulateFeatureData(
-    tflite::ErrorReporter* error_reporter, int* how_many_new_slices) {
-  if (feature_size_ != kFeatureElementCount) {
-    error_reporter->Report("Requested feature_data_ size %d doesn't match %d",
-                           feature_size_, kFeatureElementCount);
-    return kTfLiteError;
-  }
-
-  const int32_t time_in_ms = TimeInMilliseconds();
-  // Quantize the time into steps as long as each window stride, so we can
-  // figure out which audio data we need to fetch.
-  const int last_step = (g_last_time_in_ms / kFeatureSliceStrideMs);
-  const int current_step = (time_in_ms / kFeatureSliceStrideMs);
-  g_last_time_in_ms = time_in_ms;
-
-  int slices_needed = current_step - last_step;
-  // If this is the first call, make sure we don't use any cached information.
-  if (g_is_first_run) {
-    g_is_first_run = false;
-    slices_needed = kFeatureSliceCount;
-  }
-  if (slices_needed > kFeatureSliceCount) {
-    slices_needed = kFeatureSliceCount;
-  }
-  *how_many_new_slices = slices_needed;
-
-  const int slices_to_keep = kFeatureSliceCount - slices_needed;
-  const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
-  // If we can avoid recalculating some slices, just move the existing data
-  // up in the spectrogram, to perform something like this:
-  // last time = 80ms          current time = 120ms
-  // +-----------+             +-----------+
-  // | data@20ms |         --> | data@60ms |
-  // +-----------+       --    +-----------+
-  // | data@40ms |     --  --> | data@80ms |
-  // +-----------+   --  --    +-----------+
-  // | data@60ms | --  --      |  <empty>  |
-  // +-----------+   --        +-----------+
-  // | data@80ms | --          |  <empty>  |
-  // +-----------+             +-----------+
-  if (slices_to_keep > 0) {
-    for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
-      uint8_t* dest_slice_data =
-          feature_data_ + (dest_slice * kFeatureSliceSize);
-      const int src_slice = dest_slice + slices_to_drop;
-      const uint8_t* src_slice_data =
-          feature_data_ + (src_slice * kFeatureSliceSize);
-      for (int i = 0; i < kFeatureSliceSize; ++i) {
-        dest_slice_data[i] = src_slice_data[i];
-      }
-    }
-  }
-  // Any slices that need to be filled in with feature data have their
-  // appropriate audio data pulled, and features calculated for that slice.
-  if (slices_needed > 0) {
-    for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount;
-         ++new_slice) {
-      const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
-      const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
-      int16_t* audio_samples = nullptr;
-      int audio_samples_size = 0;
-      GetAudioSamples(error_reporter, slice_start_ms, kFeatureSliceDurationMs,
-                      &audio_samples_size, &audio_samples);
-      if (audio_samples_size < kMaxAudioSampleSize) {
-        error_reporter->Report("Audio data size %d  too small, want %d",
-                               audio_samples_size, kMaxAudioSampleSize);
-        return kTfLiteError;
-      }
-      uint8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
-      TfLiteStatus preprocess_status =
-          Preprocess(error_reporter, audio_samples, audio_samples_size,
-                     kFeatureSliceSize, new_slice_data);
-      if (preprocess_status != kTfLiteOk) {
-        return preprocess_status;
-      }
-    }
-  }
-  return kTfLiteOk;
-}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
deleted file mode 100644
index a86c56ebf0..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-
-// Binds itself to an area of memory intended to hold the input features for an
-// audio-recognition neural network model, and fills that data area with the
-// features representing the current audio input, for example from a microphone.
-// The audio features themselves are a two-dimensional array, made up of
-// horizontal slices representing the frequencies at one point in time, stacked
-// on top of each other to form a spectrogram showing how those frequencies
-// changed over time.
-class FeatureProvider {
- public:
-  // Create the provider, and bind it to an area of memory. This memory should
-  // remain accessible for the lifetime of the provider object, since subsequent
-  // calls will fill it with feature data. The provider does no memory
-  // management of this data.
-  FeatureProvider(int feature_size, uint8_t* feature_data);
-  ~FeatureProvider();
-
-  // Fills the feature data with information from audio inputs, and returns how
-  // many feature slices were updated.
-  TfLiteStatus PopulateFeatureData(tflite::ErrorReporter* error_reporter,
-                                   int* how_many_new_slices);
-
- private:
-  int feature_size_;
-  uint8_t* feature_data_;
-};
-
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
deleted file mode 100644
index 1e52aec8d2..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
-
-TF_LITE_MICRO_TESTS_BEGIN
-
-TF_LITE_MICRO_TEST(TestFeatureProvider) {
-  tflite::MicroErrorReporter micro_error_reporter;
-  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
-
-  uint8_t feature_data[kFeatureElementCount];
-  FeatureProvider feature_provider(kFeatureElementCount, feature_data);
-
-  int how_many_new_slices = 0;
-  TfLiteStatus populate_status = feature_provider.PopulateFeatureData(
-      error_reporter, &how_many_new_slices);
-  TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, populate_status);
-  TF_LITE_MICRO_EXPECT_EQ(kFeatureSliceCount, how_many_new_slices);
-}
-
-TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
index b623d8d11b..de60c982f3 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
@@ -31,8 +31,6 @@ limitations under the License.
 
 #include <cmath>
 
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-
 namespace {
 
 // q format notation: qx.y => 1 sign bit, x-1 integer bits, y fraction bits.
@@ -68,6 +66,13 @@ inline int32_t FloatToFixed_Q2_30(float input) {
   return static_cast<int32_t>(roundf(input * (1 << 30)));
 }
 
+// These constants allow us to allocate fixed-sized arrays on the stack for our
+// working memory.
+constexpr int kInputSize = 512;
+constexpr int kAverageWindowSize = 6;
+constexpr int kOutputSize =
+    ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
+
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
 // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft.
@@ -122,14 +127,14 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output) {
   // Ensure our input and output data arrays are valid.
-  if (input_size > kMaxAudioSampleSize) {
+  if (input_size > kInputSize) {
     error_reporter->Report("Input size %d larger than %d", input_size,
-                           kMaxAudioSampleSize);
+                           kInputSize);
     return kTfLiteError;
   }
-  if (output_size != kFeatureSliceSize) {
+  if (output_size != kOutputSize) {
     error_reporter->Report("Requested output size %d doesn't match %d",
-                           output_size, kFeatureSliceSize);
+                           output_size, kOutputSize);
     return kTfLiteError;
   }
 
@@ -137,17 +142,18 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
   // In a real application, we'd calculate this table once in an initialization
   // function and store it for repeated reuse.
   // q1.15 format.
-  int16_t window_function[kMaxAudioSampleSize];
+  int16_t window_function[kInputSize];
   CalculatePeriodicHann(input_size, window_function);
 
   // Apply the window function to our time series input, and pad it with zeroes
   // to the next power of two.
-  int32_t fixed_input[kMaxAudioSampleSize];
-  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
+  int32_t fixed_input[kInputSize];
+  for (int i = 0; i < kInputSize; ++i) {
     if (i < input_size) {
       // input is int16_t.  Treat as q1.15 fixed point value in range [-1,1)
       // window_function is also q1.15 fixed point number
-      fixed_input[i] = Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]);
+      fixed_input[i] =
+          Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]);
     } else {
       fixed_input[i] = 0;
     }
@@ -155,31 +161,31 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
 
   // Pull the frequency data from the time series sample.
   // Calculated in q10.22 format from q2.30 inputs.
-  int32_t fourier_values[kMaxAudioSampleSize];
-  CalculateDiscreteFourierTransform(fixed_input, kMaxAudioSampleSize,
-                                    fourier_values);
+  int32_t fourier_values[kInputSize];
+  CalculateDiscreteFourierTransform(fixed_input, kInputSize, fourier_values);
 
   // We have the complex numbers giving us information about each frequency
   // band, but all we want to know is how strong each frequency is, so calculate
   // the squared magnitude by adding together the squares of each component.
-  int32_t power_spectrum[kMaxAudioSampleSize / 2];
-  for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) {
+  int32_t power_spectrum[kInputSize / 2];
+  for (int i = 0; i < (kInputSize / 2); ++i) {
     const int32_t real = fourier_values[(i * 2) + 0];
     const int32_t imaginary = fourier_values[(i * 2) + 1];
     // q10.22 results
-    power_spectrum[i] = Q10_22_FixedMultiply_Q10_22(real, real) +
-                        Q10_22_FixedMultiply_Q10_22(imaginary, imaginary);
+    power_spectrum[i] =
+        Q10_22_FixedMultiply_Q10_22(real, real) +
+        Q10_22_FixedMultiply_Q10_22(imaginary, imaginary);
   }
 
   // Finally, reduce the size of the output by averaging together six adjacent
   // frequencies into each slot, producing an array of 43 values.
   // Power_spectrum numbers are q10.22.  Divide by kAverageWindowSize inside
   // loop to prevent overflow.
-  for (int i = 0; i < kFeatureSliceSize; ++i) {
+  for (int i = 0; i < kOutputSize; ++i) {
     int32_t average = 0;
     for (int j = 0; j < kAverageWindowSize; ++j) {
       const int index = (i * kAverageWindowSize) + j;
-      if (index < (kMaxAudioSampleSize / 2)) {
+      if (index < (kInputSize / 2)) {
         average += power_spectrum[index] / kAverageWindowSize;
       }
     }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
deleted file mode 100644
index 1890c25cf2..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
+++ /dev/null
@@ -1,112 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h"
-#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-#include "tensorflow/lite/experimental/micro/micro_interpreter.h"
-#include "tensorflow/lite/schema/schema_generated.h"
-#include "tensorflow/lite/version.h"
-
-int main(int argc, char* argv[]) {
-  // Set up logging.
-  tflite::MicroErrorReporter micro_error_reporter;
-  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
-
-  // Map the model into a usable data structure. This doesn't involve any
-  // copying or parsing, it's a very lightweight operation.
-  const tflite::Model* model = ::tflite::GetModel(g_tiny_conv_model_data);
-  if (model->version() != TFLITE_SCHEMA_VERSION) {
-    error_reporter->Report(
-        "Model provided is schema version %d not equal "
-        "to supported version %d.\n",
-        model->version(), TFLITE_SCHEMA_VERSION);
-    return 1;
-  }
-
-  // This pulls in all the operation implementations we need.
-  tflite::ops::micro::AllOpsResolver resolver;
-
-  // Create an area of memory to use for input, output, and intermediate arrays.
-  // The size of this will depend on the model you're using, and may need to be
-  // determined by experimentation.
-  const int tensor_arena_size = 10 * 1024;
-  uint8_t tensor_arena[tensor_arena_size];
-  tflite::SimpleTensorAllocator tensor_allocator(tensor_arena,
-                                                 tensor_arena_size);
-
-  // Build an interpreter to run the model with.
-  tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator,
-                                       error_reporter);
-
-  // Get information about the memory area to use for the model's input.
-  TfLiteTensor* model_input = interpreter.input(0);
-  if ((model_input->dims->size != 4) || (model_input->dims->data[0] != 1) ||
-      (model_input->dims->data[1] != kFeatureSliceCount) ||
-      (model_input->dims->data[2] != kFeatureSliceSize) ||
-      (model_input->type != kTfLiteUInt8)) {
-    error_reporter->Report("Bad input tensor parameters in model");
-    return 1;
-  }
-
-  // Prepare to access the audio spectrograms from a microphone or other source
-  // that will provide the inputs to the neural network.
-  FeatureProvider feature_provider(kFeatureElementCount,
-                                   model_input->data.uint8);
-
-  // Keep reading and analysing audio data in an infinite loop.
-  while (true) {
-    // Fetch the spectrogram for the current time.
-    int how_many_new_slices = 0;
-    TfLiteStatus feature_status = feature_provider.PopulateFeatureData(
-        error_reporter, &how_many_new_slices);
-    if (feature_status != kTfLiteOk) {
-      error_reporter->Report("Feature generation failed");
-      return 1;
-    }
-    // If no new audio samples have been received since last time, don't bother
-    // running the network model.
-    if (how_many_new_slices == 0) {
-      continue;
-    }
-
-    // Run the model on the spectrogram input and make sure it succeeds.
-    TfLiteStatus invoke_status = interpreter.Invoke();
-    if (invoke_status != kTfLiteOk) {
-      error_reporter->Report("Invoke failed");
-      return 1;
-    }
-
-    // The output from the model is a vector containing the scores for each
-    // kind of prediction, so figure out what the highest scoring category was.
-    TfLiteTensor* output = interpreter.output(0);
-    uint8_t top_category_score = 0;
-    int top_category_index = 0;
-    for (int category_index = 0; category_index < kCategoryCount;
-         ++category_index) {
-      const uint8_t category_score = output->data.uint8[category_index];
-      if (category_score > top_category_score) {
-        top_category_score = category_score;
-        top_category_index = category_index;
-      }
-    }
-
-    error_reporter->Report("Heard %s", kCategoryLabels[top_category_index]);
-  }
-
-  return 0;
-}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
deleted file mode 100644
index 88369a07ee..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
-
-// Keeping these as constant expressions allow us to allocate fixed-sized arrays
-// on the stack for our working memory.
-
-// The size of the input time series data we pass to the FFT to produce the
-// frequency information. This has to be a power of two, and since we're dealing
-// with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
-constexpr int kMaxAudioSampleSize = 512;
-
-// All of these values are derived from the values used during model training,
-// if you change your model you'll need to update these constants.
-constexpr int kAverageWindowSize = 6;
-constexpr int kFeatureSliceSize =
-    ((kMaxAudioSampleSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
-constexpr int kFeatureSliceCount = 49;
-constexpr int kFeatureElementCount = (kFeatureSliceSize * kFeatureSliceCount);
-constexpr int kFeatureSliceStrideMs = 20;
-constexpr int kFeatureSliceDurationMs = 30;
-
-constexpr int kCategoryCount = 4;
-constexpr int kSilenceIndex = 0;
-constexpr int kUnknownIndex = 1;
-constexpr char* kCategoryLabels[kCategoryCount] = {
-    "silence",
-    "unknown",
-    "yes",
-    "no",
-};
-
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
index f4a7f801cc..12f9e22038 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
@@ -28,10 +28,15 @@ limitations under the License.
 
 #include <cmath>
 
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
-
 namespace {
 
+// These constants allow us to allocate fixed-sized arrays on the stack for our
+// working memory.
+constexpr int kInputSize = 512;
+constexpr int kAverageWindowSize = 6;
+constexpr int kOutputSize =
+    ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
+
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
 // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft.
@@ -73,27 +78,27 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output) {
   // Ensure our input and output data arrays are valid.
-  if (input_size > kMaxAudioSampleSize) {
+  if (input_size > kInputSize) {
     error_reporter->Report("Input size %d larger than %d", input_size,
-                           kMaxAudioSampleSize);
+                           kInputSize);
     return kTfLiteError;
   }
-  if (output_size != kFeatureSliceSize) {
+  if (output_size != kOutputSize) {
     error_reporter->Report("Requested output size %d doesn't match %d",
-                           output_size, kFeatureSliceSize);
+                           output_size, kOutputSize);
     return kTfLiteError;
   }
 
   // Pre-calculate the window function we'll be applying to the input data.
   // In a real application, we'd calculate this table once in an initialization
   // function and store it for repeated reuse.
-  float window_function[kMaxAudioSampleSize];
+  float window_function[kInputSize];
   CalculatePeriodicHann(input_size, window_function);
 
   // Apply the window function to our time series input, and pad it with zeroes
   // to the next power of two.
-  float float_input[kMaxAudioSampleSize];
-  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
+  float float_input[kInputSize];
+  for (int i = 0; i < kInputSize; ++i) {
     if (i < input_size) {
       float_input[i] =
           (input[i] * window_function[i]) / static_cast<float>(1 << 15);
@@ -103,15 +108,14 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
   }
 
   // Pull the frequency data from the time series sample.
-  float fourier_values[kMaxAudioSampleSize];
-  CalculateDiscreteFourierTransform(float_input, kMaxAudioSampleSize,
-                                    fourier_values);
+  float fourier_values[kInputSize];
+  CalculateDiscreteFourierTransform(float_input, kInputSize, fourier_values);
 
   // We have the complex numbers giving us information about each frequency
   // band, but all we want to know is how strong each frequency is, so calculate
   // the squared magnitude by adding together the squares of each component.
-  float power_spectrum[kMaxAudioSampleSize / 2];
-  for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) {
+  float power_spectrum[kInputSize / 2];
+  for (int i = 0; i < (kInputSize / 2); ++i) {
     const float real = fourier_values[(i * 2) + 0];
     const float imaginary = fourier_values[(i * 2) + 1];
     power_spectrum[i] = (real * real) + (imaginary * imaginary);
@@ -119,11 +123,11 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
 
   // Finally, reduce the size of the output by averaging together six adjacent
   // frequencies into each slot, producing an array of 43 values.
-  for (int i = 0; i < kFeatureSliceSize; ++i) {
+  for (int i = 0; i < kOutputSize; ++i) {
     float total = 0.0f;
     for (int j = 0; j < kAverageWindowSize; ++j) {
       const int index = (i * kAverageWindowSize) + j;
-      if (index < (kMaxAudioSampleSize / 2)) {
+      if (index < (kInputSize / 2)) {
         total += power_spectrum[index];
       }
     }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
index adff790d6c..dede2a8642 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
@@ -19,11 +19,6 @@ limitations under the License.
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
 
-// Converts audio sample data into a more compact form that's appropriate for
-// feeding into a neural network. There are reference implementations that use
-// both floating point and fixed point available, but because the calculations
-// involved can be time-consuming, it's recommended that you use or write
-// specialized versions for your platform.
 TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output);
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
deleted file mode 100644
index 6c96a61ab5..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-int32_t TimeInMilliseconds() {
-  static int current_time = 0;
-  current_time += 100;
-  return current_time;
-}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
deleted file mode 100644
index 162952844a..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
-
-#include <cstdint>
-
-// Returns the time in milliseconds. There's no contract about what time zero
-// represents, the accuracy, or the granularity of the result. Subsequent calls
-// will generally not return a lower value, but even that's not guaranteed if
-// there's an overflow  wraparound.
-// The reference implementation of this function just returns a constantly
-// incrementing value for each call, since it would need a non-portable platform
-// call to access time information. For real applications, you'll need to write
-// your own platform-specific implementation.
-int32_t TimeInMilliseconds();
-
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
deleted file mode 100644
index 83a2dfcc65..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-#include <limits>
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
-
-TF_LITE_MICRO_TESTS_BEGIN
-
-TF_LITE_MICRO_TEST(TestTimer) {
-  // Make sure that the technically-undefined overflow behavior we rely on below
-  // works on this platform. It's still not guaranteed, but at least this is a
-  // sanity check.
-  int32_t overflow_value = std::numeric_limits<int32_t>::max();
-  overflow_value += 1;
-  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
-
-  const int32_t first_time = TimeInMilliseconds();
-  const int32_t second_time = TimeInMilliseconds();
-
-  // It's possible that the timer may have wrapped around from +BIG_NUM to
-  // -BIG_NUM between the first and second calls, since we're storing
-  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
-  // would have taken more than 2^31 milliseconds though, so look at the
-  // difference and rely on integer overflow to ensure it's accurate.
-  const int32_t time_delta = (second_time - first_time);
-  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
-}
-
-TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/testing/micro_test.h b/tensorflow/lite/experimental/micro/testing/micro_test.h
index 2f20dd5ac7..10bab05fae 100644
--- a/tensorflow/lite/experimental/micro/testing/micro_test.h
+++ b/tensorflow/lite/experimental/micro/testing/micro_test.h
@@ -153,22 +153,4 @@ extern tflite::ErrorReporter* reporter;
     }                                                                        \
   } while (false)
 
-#define TF_LITE_MICRO_EXPECT_GE(x, y)                                         \
-  do {                                                                        \
-    if ((x) < (y)) {                                                          \
-      micro_test::reporter->Report(#x " >= " #y " failed at %s:%d", __FILE__, \
-                                   __LINE__);                                 \
-      micro_test::did_test_fail = true;                                       \
-    }                                                                         \
-  } while (false)
-
-#define TF_LITE_MICRO_EXPECT_LE(x, y)                                         \
-  do {                                                                        \
-    if ((x) > (y)) {                                                          \
-      micro_test::reporter->Report(#x " <= " #y " failed at %s:%d", __FILE__, \
-                                   __LINE__);                                 \
-      micro_test::did_test_fail = true;                                       \
-    }                                                                         \
-  } while (false)
-
 #endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_TESTING_MICRO_TEST_H_
-- 
GitLab


From b8ac6cb24927e2688a514dcbd5ca3aeed97f7dfb Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Sun, 25 Nov 2018 16:39:24 -0800
Subject: [PATCH 0754/1554] Fix name of upgrade script in example.

PiperOrigin-RevId: 222758170
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9b14c11614..33e4f0f442 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -585,8 +585,8 @@ if __name__ == "__main__":
       description="""Convert a TensorFlow Python file to 2.0
 
 Simple usage:
-  tf_convert_v2.py --infile foo.py --outfile bar.py
-  tf_convert_v2.py --intree ~/code/old --outtree ~/code/new
+  tf_upgrade_v2.py --infile foo.py --outfile bar.py
+  tf_upgrade_v2.py --intree ~/code/old --outtree ~/code/new
 """)
   parser.add_argument(
       "--infile",
-- 
GitLab


From e14e62133cc72bafb2fd3251dc4e6b7116b2f4ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 25 Nov 2018 18:01:45 -0800
Subject: [PATCH 0755/1554] Move distribute.py,
 distribution_strategy_context.py, and device_util.py from training/ to
 distribute/.

PiperOrigin-RevId: 222761376
---
 tensorflow/contrib/distribute/python/BUILD    |   12 +-
 .../python/collective_all_reduce_strategy.py  |    2 +-
 .../contrib/distribute/python/combinations.py |    2 +-
 .../python/cross_device_ops_test.py           |    2 +-
 .../python/cross_device_utils_test.py         |    2 +-
 .../python/keras_optimizer_v2_test.py         |    2 +-
 .../distribute/python/mirrored_strategy.py    |    2 +-
 .../python/mirrored_strategy_multigpu_test.py |    4 +-
 .../distribute/python/one_device_strategy.py  |    2 +-
 .../python/parameter_server_strategy.py       |    4 +-
 .../python/parameter_server_strategy_test.py  |    4 +-
 .../distribute/python/strategy_test_lib.py    |    2 +-
 .../contrib/distribute/python/tpu_strategy.py |    4 +-
 .../contrib/distribute/python/values_test.py  |    4 +-
 tensorflow/python/BUILD                       |   35 +-
 tensorflow/python/distribute/BUILD            |   74 +-
 .../python/distribute/cross_device_ops.py     |    2 +-
 tensorflow/python/distribute/device_util.py   |   97 +
 .../device_util_test.py                       |    2 +-
 .../python/distribute/distribute_lib.py       | 1665 +++++++++++++++++
 .../distribute_lib_test.py}                   |    4 +-
 .../distribution_strategy_context.py          |  236 +++
 .../python/distribute/mirrored_strategy.py    |    4 +-
 tensorflow/python/distribute/values.py        |    6 +-
 .../python/tools/api/generator/doc_srcs.py    |    2 +-
 tensorflow/python/training/device_util.py     |   81 +-
 tensorflow/python/training/distribute.py      | 1649 +---------------
 .../training/distribution_strategy_context.py |  220 +--
 ...tensorflow.distribute.-input-context.pbtxt |    2 +-
 ...nsorflow.distribute.-replica-context.pbtxt |    2 +-
 ...orflow.distribute.-strategy-extended.pbtxt |    2 +-
 .../v1/tensorflow.distribute.-strategy.pbtxt  |    2 +-
 ...tensorflow.distribute.-input-context.pbtxt |    2 +-
 ...nsorflow.distribute.-replica-context.pbtxt |    2 +-
 ...orflow.distribute.-strategy-extended.pbtxt |    2 +-
 .../v2/tensorflow.distribute.-strategy.pbtxt  |    2 +-
 36 files changed, 2118 insertions(+), 2023 deletions(-)
 create mode 100644 tensorflow/python/distribute/device_util.py
 rename tensorflow/python/{training => distribute}/device_util_test.py (98%)
 create mode 100644 tensorflow/python/distribute/distribute_lib.py
 rename tensorflow/python/{training/distribute_test.py => distribute/distribute_lib_test.py} (98%)
 create mode 100644 tensorflow/python/distribute/distribution_strategy_context.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 91282a8c1d..b068ef029f 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -27,13 +27,13 @@ cuda_py_test(
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:device_util",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/distribute:device_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
@@ -49,7 +49,7 @@ py_library(
     srcs = ["mirrored_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        "//tensorflow/python:distribute",
+        "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:mirrored_strategy",
         "//tensorflow/python/distribute:values",
     ],
@@ -114,10 +114,10 @@ py_library(
     visibility = ["//tensorflow:internal"],
     deps = [
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
@@ -156,11 +156,11 @@ py_library(
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:layers",
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
@@ -181,10 +181,10 @@ py_library(
         ":tpu_strategy",
         "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
         "//tensorflow/contrib/optimizer_v2:training",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
+        "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/eager:context",
         "@absl_py//absl/testing:parameterized",
     ],
@@ -229,11 +229,11 @@ cuda_py_test(
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:layers",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 17323e2741..74a0550e81 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -24,6 +24,7 @@ from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
@@ -31,7 +32,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import collective_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribute as distribute_lib
 
 
 # TODO(yuefengz): support in-graph replication.
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index f3ce547f4d..c5ce29a436 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -53,11 +53,11 @@ from tensorflow.contrib.distribute.python import tpu_strategy as tpu_lib
 from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2
 from tensorflow.contrib.optimizer_v2 import adam as adam_v2
 from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.training import adagrad
 from tensorflow.python.training import adam
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import rmsprop
 from tensorflow.python.util import tf_inspect
diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 40410b90be..5d8690beb5 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -29,6 +29,7 @@ from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.eager import context
@@ -37,7 +38,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.training import device_util
 
 
 def _make_per_replica(values, devices, regroup=False):
diff --git a/tensorflow/contrib/distribute/python/cross_device_utils_test.py b/tensorflow/contrib/distribute/python/cross_device_utils_test.py
index 6086eba098..2303a31677 100644
--- a/tensorflow/contrib/distribute/python/cross_device_utils_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_utils_test.py
@@ -22,13 +22,13 @@ from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
-from tensorflow.python.training import device_util
 
 
 class IndexedSlicesUtilsTest(test.TestCase, parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index fba06283ce..6dfd85bcc4 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.estimator import run_config
 from tensorflow.python.estimator import training
 from tensorflow.python.estimator.canned import dnn_linear_combined
@@ -46,7 +47,6 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
-from tensorflow.python.training import distribution_strategy_context as ds_context
 
 
 class KerasOptimizerV2IntegrationTest(test.TestCase, parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index a3bcc8db88..7719715875 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -20,9 +20,9 @@ from __future__ import print_function
 
 import functools
 
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import mirrored_strategy
 from tensorflow.python.distribute import values
-from tensorflow.python.training import distribute as distribute_lib
 
 
 # pylint: disable=protected-access,invalid-name
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index cf6c7f6879..b304f63501 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -29,6 +29,8 @@ from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import backprop
@@ -48,8 +50,6 @@ from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import optimizer as optimizer_lib
 from tensorflow.python.training import server_lib
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 421507232a..7a2e225ca0 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -20,13 +20,13 @@ from __future__ import print_function
 
 import six
 
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import values
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index d127868525..5615abb751 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -22,6 +22,8 @@ import copy
 
 from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
@@ -32,8 +34,6 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import device_setter
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
 
 _LOCAL_CPU = "/device:CPU:0"
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 3ea9b90f6f..4debe72ca6 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -28,6 +28,8 @@ from tensorflow.contrib.distribute.python import parameter_server_strategy
 from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -46,8 +48,6 @@ from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import training_util
 
 CHIEF = run_config.TaskType.CHIEF
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 5a8e8ed0dd..756e5bdc1e 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import backprop
@@ -33,7 +34,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribution_strategy_context as ds_context
 from tensorflow.python.training import optimizer
 
 
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 3e755242f7..1f302fdde8 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -29,6 +29,8 @@ from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
@@ -41,8 +43,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 855b9c29ae..538b859f3d 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -25,6 +25,8 @@ from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
@@ -39,8 +41,6 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.util import nest
 
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 3def23bc44..5c9e7f5e89 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3596,9 +3596,7 @@ py_library(
     srcs = ["training/device_util.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":device",
-        ":framework_ops",
-        "//tensorflow/python/eager:context",
+        "//tensorflow/python/distribute:device_util",
     ],
 )
 
@@ -3610,35 +3608,7 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        ":array_ops",
-        ":constant_op",
-        ":control_flow_ops",
-        ":device_util",
-        ":dtypes",
-        ":framework_ops",
-        ":platform",
-        ":resource_variable_ops",
-        ":state_ops",
-        ":util",
-        ":variable_scope",
-        "//tensorflow/python/data",
-        "//tensorflow/python/distribute:reduce_util",
-        "//tensorflow/python/ops/losses",
-        "//tensorflow/tools/docs:doc_controls",
-    ],
-)
-
-py_test(
-    name = "distribute_test",
-    size = "small",
-    srcs = ["training/distribute_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":client_testlib",
-        ":constant_op",
-        ":distribute",
-        ":dtypes",
-        ":variable_scope",
+        "//tensorflow/python/distribute:distribute_lib",
     ],
 )
 
@@ -4627,7 +4597,6 @@ cuda_py_tests(
         "training/basic_loops_test.py",
         "training/coordinator_test.py",
         "training/device_setter_test.py",
-        "training/device_util_test.py",
         "training/ftrl_test.py",
         "training/gradient_descent_test.py",
         "training/learning_rate_decay_test.py",
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 999543d71f..5afbcec3a9 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -50,6 +50,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":cross_device_utils",
+        ":device_util",
         ":reduce_util",
         ":values",
         "//tensorflow/python:array_ops",
@@ -58,8 +59,6 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:context",
         "@six_archive//:six",
     ],
@@ -83,6 +82,67 @@ py_library(
     ],
 )
 
+py_library(
+    name = "device_util",
+    srcs = ["device_util.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:device",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
+cuda_py_test(
+    name = "device_util_test",
+    srcs = ["device_util_test.py"],
+    additional_deps = [
+        ":device_util",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+    ],
+)
+
+py_library(
+    name = "distribute_lib",
+    srcs = [
+        "distribute_lib.py",
+        "distribution_strategy_context.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":device_util",
+        ":reduce_util",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/data",
+        "//tensorflow/python/ops/losses",
+        "//tensorflow/tools/docs:doc_controls",
+    ],
+)
+
+py_test(
+    name = "distribute_lib_test",
+    size = "small",
+    srcs = ["distribute_lib_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":distribute_lib",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:variable_scope",
+    ],
+)
+
 py_library(
     name = "distribute_config",
     srcs = [
@@ -144,6 +204,8 @@ py_library(
     srcs = ["mirrored_strategy.py"],
     deps = [
         ":cross_device_ops",
+        ":device_util",
+        ":distribute_lib",
         ":multi_worker_util",
         ":reduce_util",
         ":shared_variable_creator",
@@ -153,8 +215,6 @@ py_library(
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:device",
-        "//tensorflow/python:device_util",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:pywrap_tensorflow",
@@ -195,12 +255,12 @@ cuda_py_test(
     additional_deps = [
         ":input_ops",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:readers",
         "//tensorflow/python:errors",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:io_ops",
-        "//tensorflow/python/data/ops:readers",
         "//tensorflow/python:util",
     ],
     tags = [
@@ -271,11 +331,11 @@ py_library(
     name = "values",
     srcs = ["values.py"],
     deps = [
+        ":device_util",
+        ":distribute_lib",
         ":input_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:device_util",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:training",
diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index 87b6f41eba..a88ed62533 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -23,6 +23,7 @@ import six
 
 from tensorflow.python.client import device_lib
 from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values as value_lib
 from tensorflow.python.eager import context
@@ -31,7 +32,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import device_util
 
 
 def check_destinations(destinations):
diff --git a/tensorflow/python/distribute/device_util.py b/tensorflow/python/distribute/device_util.py
new file mode 100644
index 0000000000..70e1ca4b5d
--- /dev/null
+++ b/tensorflow/python/distribute/device_util.py
@@ -0,0 +1,97 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Device-related support functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import ops
+
+
+def canonicalize(d, default=None):
+  """Canonicalize device string.
+
+  If d has missing components, the rest would be deduced from the `default`
+  argument or from '/replica:0/task:0/device:CPU:0'. For example:
+    If d = '/cpu:0', default='/job:worker/task:1', it returns
+      '/job:worker/replica:0/task:1/device:CPU:0'.
+    If d = '/cpu:0', default='/job:worker', it returns
+      '/job:worker/replica:0/task:0/device:CPU:0'.
+    If d = '/gpu:0', default=None, it returns
+      '/replica:0/task:0/device:GPU:0'.
+
+  Note: This uses "job:localhost" as the default if executing eagerly.
+
+  Args:
+    d: a device string.
+    default: a string for default device if d doesn't have all components.
+
+  Returns:
+    a canonicalized device string.
+  """
+  d = tf_device.DeviceSpec.from_string(d)
+  assert d.device_type is None or d.device_type == d.device_type.upper(), (
+      "Device type '%s' must be all-caps." % (d.device_type,))
+  # Fill in missing device fields using defaults.
+  result = tf_device.DeviceSpec(
+      replica=0, task=0, device_type="CPU", device_index=0)
+  if context.executing_eagerly():
+    result.job = "localhost"
+  if default:
+    result.merge_from(tf_device.DeviceSpec.from_string(default))
+  result.merge_from(d)
+  return result.to_string()
+
+
+def resolve(d):
+  """Canonicalize `d` with current device as default."""
+  return canonicalize(d, default=current())
+
+
+class _FakeNodeDef(object):
+  """A fake NodeDef for _FakeOperation."""
+
+  def __init__(self):
+    self.op = ""
+    self.name = ""
+
+
+class _FakeOperation(object):
+  """A fake Operation object to pass to device functions."""
+
+  def __init__(self):
+    self.device = ""
+    self.type = ""
+    self.name = ""
+    self.node_def = _FakeNodeDef()
+
+  def _set_device(self, device):
+    self.device = ops._device_string(device)  # pylint: disable=protected-access
+
+
+def current():
+  """Return a string (not canonicalized) for the current device."""
+  # TODO(josh11b): Work out how this function interacts with ops.colocate_with.
+  ctx = context.context()
+  if ctx.executing_eagerly():
+    d = ctx.device_name
+  else:
+    op = _FakeOperation()
+    ops.get_default_graph()._apply_device_functions(op)  # pylint: disable=protected-access
+    d = op.device
+  return d
diff --git a/tensorflow/python/training/device_util_test.py b/tensorflow/python/distribute/device_util_test.py
similarity index 98%
rename from tensorflow/python/training/device_util_test.py
rename to tensorflow/python/distribute/device_util_test.py
index cdbb08229d..baecd43c8e 100644
--- a/tensorflow/python/training/device_util_test.py
+++ b/tensorflow/python/distribute/device_util_test.py
@@ -18,10 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import device_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import test
-from tensorflow.python.training import device_util
 
 
 class DeviceUtilTest(test.TestCase):
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
new file mode 100644
index 0000000000..a1f03eab61
--- /dev/null
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -0,0 +1,1665 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for running a computation across multiple devices."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+import threading
+import weakref
+import enum
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import reduce_util
+from tensorflow.python.eager import context as eager_context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.losses import losses_impl
+from tensorflow.python.platform import tf_logging
+from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
+from tensorflow.tools.docs import doc_controls
+
+
+# ------------------------------------------------------------------------------
+# Context tracking whether in a strategy.update() or .update_non_slot() call.
+
+
+_update_device = threading.local()
+
+
+def get_update_device():
+  """Get the current device if in a `tf.distribute.Strategy.update()` call."""
+  try:
+    return _update_device.current
+  except AttributeError:
+    return None
+
+
+class UpdateContext(object):
+  """Context manager when you are in `update()` or `update_non_slot()`."""
+
+  def __init__(self, device):
+    self._device = device
+    self._old_device = None
+
+  def __enter__(self):
+    self._old_device = get_update_device()
+    _update_device.current = self._device
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    del exception_type, exception_value, traceback
+    _update_device.current = self._old_device
+
+
+# ------------------------------------------------------------------------------
+# Public utility functions.
+
+
+@tf_export("distribute.get_loss_reduction")
+def get_loss_reduction():
+  """`tf.distribute.ReduceOp` corresponding to the last loss reduction."""
+  loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
+  if loss_reduction == losses_impl.Reduction.SUM:
+    return reduce_util.ReduceOp.SUM
+  return reduce_util.ReduceOp.MEAN
+
+
+# ------------------------------------------------------------------------------
+# Internal API for validating the current thread mode
+
+
+def _require_cross_replica_context_extended(extended):
+  """Verify in cross-replica context."""
+  context = _get_per_thread_mode()
+  cross_replica = context.cross_replica_context
+  if cross_replica is not None and cross_replica.extended is extended:
+    return
+  strategy = extended._container_strategy()  # pylint: disable=protected-access
+  # We have an error to report, figure out the right message.
+  if context.distribution_strategy is not strategy:
+    _wrong_strategy_scope(strategy, context)
+  assert cross_replica is None
+  raise RuntimeError("Method requires being in cross-replica context, use "
+                     "get_replica_context().merge_call()")
+
+
+def _wrong_strategy_scope(strategy, context):
+  # Figure out the right error message.
+  if not distribution_strategy_context.has_distribution_strategy():
+    raise RuntimeError(
+        'Need to be inside "with strategy.scope()" for %s' %
+        (strategy,))
+  else:
+    raise RuntimeError(
+        "Mixing different tf.distribute.Strategy objects: %s is not %s" %
+        (context.distribution_strategy, strategy))
+
+
+def require_replica_context(replica_ctx):
+  """Verify in `replica_ctx` replica context."""
+  context = _get_per_thread_mode()
+  if context.replica_context is replica_ctx: return
+  # We have an error to report, figure out the right message.
+  if context.replica_context is None:
+    raise RuntimeError("Need to be inside `call_for_each_replica()`")
+  if context.distribution_strategy is replica_ctx.distribution_strategy:
+    # Two different ReplicaContexts with the same tf.distribute.Strategy.
+    raise RuntimeError("Mismatching ReplicaContext.")
+  raise RuntimeError(
+      "Mismatching tf.distribute.Strategy objects: %s is not %s." %
+      (context.distribution_strategy, replica_ctx.distribution_strategy))
+
+
+def _require_distribution_strategy_scope_strategy(strategy):
+  """Verify in a `strategy.scope()` in this thread."""
+  context = _get_per_thread_mode()
+  if context.distribution_strategy is strategy: return
+  _wrong_strategy_scope(strategy, context)
+
+
+def _require_distribution_strategy_scope_extended(extended):
+  """Verify in a `distribution_strategy.scope()` in this thread."""
+  context = _get_per_thread_mode()
+  if context.distribution_strategy.extended is extended: return
+  # Report error.
+  strategy = extended._container_strategy()  # pylint: disable=protected-access
+  _wrong_strategy_scope(strategy, context)
+
+
+# ------------------------------------------------------------------------------
+# Internal context managers used to implement the DistributionStrategy
+# base class
+
+
+class _CurrentDistributionContext(object):
+  """Context manager setting the current `tf.distribute.Strategy`.
+
+  Also: overrides the variable creator and optionally the current device.
+  """
+
+  def __init__(self,
+               strategy,
+               var_creator_scope,
+               var_scope=None,
+               default_device=None):
+    self._context = distribution_strategy_context._CrossReplicaThreadMode(  # pylint: disable=protected-access
+        strategy)
+    self._var_creator_scope = var_creator_scope
+    self._var_scope = var_scope
+    if default_device:
+      self._device_scope = ops.device(default_device)
+    else:
+      self._device_scope = None
+
+  def __enter__(self):
+    _push_per_thread_mode(self._context)
+    if self._var_scope:
+      self._var_scope.__enter__()
+    self._var_creator_scope.__enter__()
+    if self._device_scope:
+      self._device_scope.__enter__()
+    return self._context.distribution_strategy
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    if self._device_scope:
+      self._device_scope.__exit__(exception_type, exception_value, traceback)
+    self._var_creator_scope.__exit__(exception_type, exception_value, traceback)
+    if self._var_scope:
+      self._var_scope.__exit__(exception_type, exception_value, traceback)
+    _pop_per_thread_mode()
+
+
+class _SameScopeAgainContext(object):
+  """Trivial context manager when you are already in `scope()`."""
+
+  def __init__(self, strategy):
+    self._distribution_strategy = strategy
+
+  def __enter__(self):
+    return self._distribution_strategy
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    del exception_type, exception_value, traceback
+
+
+# TODO(yuefengz): add more replication modes.
+@tf_export("distribute.InputReplicationMode")
+class InputReplicationMode(enum.Enum):
+  """Replication mode for input function."""
+
+  # The input function will be called on each worker independently, creating as
+  # many input pipelines as number of workers. Replicas will dequeue from the
+  # local Dataset on their worker. Distribution Strategy doesn't manage any
+  # state sharing between such separate input pipelines.
+  PER_WORKER = "PER_WORKER"
+
+
+@tf_export("distribute.InputContext")
+class InputContext(object):
+  """A class wrapping information needed by an input function.
+
+  This is a context class that is passed to the user's input fn and contains
+  information about the compute replicas and input pipelines. The number of
+  compute replicas (in sync training) helps compute per input pipeline batch
+  size from the desired global batch size. Input pipeline information can be
+  used to return a different subset of the input in each input pipeline (for
+  e.g. shard the input pipeline, use a different input source etc).
+  """
+
+  def __init__(self,
+               num_input_pipelines=1,
+               input_pipeline_id=0,
+               num_replicas_in_sync=1):
+    """Initializes an InputContext object.
+
+    Args:
+      num_input_pipelines: the number of input pipelines in a cluster.
+      input_pipeline_id: the current input pipeline id, should be an int in
+        [0,`num_input_pipelines`).
+      num_replicas_in_sync: the number of replicas that are in sync.
+    """
+    self._num_input_pipelines = num_input_pipelines
+    self._input_pipeline_id = input_pipeline_id
+    self._num_replicas_in_sync = num_replicas_in_sync
+
+  @property
+  def num_replicas_in_sync(self):
+    """Returns the number of compute replicas in sync."""
+    return self._num_replicas_in_sync
+
+  @property
+  def input_pipeline_id(self):
+    """Returns the input pipeline ID."""
+    return self._input_pipeline_id
+
+  @property
+  def num_input_pipelines(self):
+    """Returns the number of input pipelines."""
+    return self._num_input_pipelines
+
+  def get_per_replica_batch_size(self, global_batch_size):
+    """Returns the per-replica batch size.
+
+    Args:
+      global_batch_size: the global batch size which should be divisible by
+        `num_replicas_in_sync`.
+
+    Returns:
+      the per-replica batch size.
+
+    Raises:
+      ValueError: if `global_batch_size` not divisible by
+        `num_replicas_in_sync`.
+    """
+    if global_batch_size % self._num_replicas_in_sync != 0:
+      raise ValueError("The `global_batch_size` %r is not divisible by "
+                       "`num_replicas_in_sync` %r " %
+                       (global_batch_size, self._num_replicas_in_sync))
+    return global_batch_size // self._num_replicas_in_sync
+
+
+# ------------------------------------------------------------------------------
+# Base classes for all distribution strategies.
+
+
+@tf_export("distribute.Strategy")
+class DistributionStrategy(object):
+  """A list of devices with a state & compute distribution policy.
+
+  See [tensorflow/contrib/distribute/README.md](
+  https://www.tensorflow.org/code/tensorflow/contrib/distribute/README.md)
+  for overview and examples.
+  """
+
+  # TODO(josh11b): Raise an exception if variable partitioning requested before
+  #   we add support.
+  # TODO(josh11b): Also `parameter_device_index` property?
+  # TODO(josh11b): `map()`
+  # TODO(josh11b): ClusterSpec/ClusterResolver
+  # TODO(josh11b): Partitioned computations, state; sharding
+  # TODO(josh11b): Model parallelism: "replicas" with multiple devices; shuffling
+  # TODO(josh11b): List of replicas with their worker and parameter devices
+  #   (where the parameter devices may overlap in the ps case).
+
+  def __init__(self, extended):
+    self._extended = extended
+
+  @property
+  def extended(self):
+    """`tf.distribute.StrategyExtended` with additional methods."""
+    return self._extended
+
+  def scope(self):
+    """Returns a context manager selecting this Strategy as current.
+
+    Inside a `with strategy.scope():` code block, this thread
+    will use a variable creator set by `strategy`, and will
+    enter its "cross-replica context".
+
+    Returns:
+      A context manager.
+    """
+    return self._extended._scope(self)  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def read_var(self, v):
+    """DEPRECATED: use extended.read_var() instead."""
+    return self._extended.read_var(v)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def colocate_vars_with(self, colocate_with_variable):
+    """DEPRECATED: use extended.colocate_vars_with() instead."""
+    return self._extended.colocate_vars_with(colocate_with_variable)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED
+  def distribute_dataset(self, dataset_fn):
+    """Return a `dataset` split across all replicas.  DEPRECATED.
+
+    DEPRECATED: Please use `make_dataset_iterator` or
+    `make_input_fn_iterator` instead.
+
+    Suitable for providing input to `extended.call_for_each_replica()` by
+    creating an iterator:
+
+    ```
+    def dataset_fn():
+      return tf.data.Dataset.from_tensors([[1.]]).repeat()
+
+    with strategy.scope():
+      distributed_dataset = strategy.distribute_dataset(dataset_fn)
+      iterator = distributed_dataset.make_initializable_iterator()
+      replica_results = strategy.extended.call_for_each_replica(
+          replica_fn, args=(iterator.get_next(),))
+    ```
+
+    Args:
+      dataset_fn: A function that returns a `tf.data.Dataset`.
+
+    Returns:
+      A `PerReplicaDataset` that will produce data for each replica.
+    """
+    return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
+
+  def make_dataset_iterator(self, dataset):
+    """Makes an iterator for input provided via input_dataset.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    global batch size. With this assumption, we will make a best effort to
+    divide each batch across all the replicas (one or more workers).
+    If this effort fails, an error will be thrown, and the user should instead
+    use `make_input_fn_iterator` which provides more control to the user, and
+    does not try to divide a batch across replicas.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return self._extended._make_dataset_iterator(dataset)  # pylint: disable=protected-access
+
+  def make_input_fn_iterator(self,
+                             input_fn,
+                             replication_mode=InputReplicationMode.PER_WORKER):
+    """Returns an iterator split across replicas created from an input function.
+
+    The `input_fn` should take an `tf.distribute.InputContext` object where
+    information about input sharding can be accessed:
+
+    ```
+    def input_fn(input_context):
+      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
+      return d.shard(input_context.num_input_pipelines,
+                     input_context.input_pipeline_id)
+    with strategy.scope():
+      iterator = strategy.make_input_fn_iterator(
+          input_fn)
+      replica_results = strategy.extended.call_for_each_replica(
+          replica_fn, iterator.get_next())
+    ```
+
+    Args:
+      input_fn: A function that returns a `tf.data.Dataset`. This function is
+        expected to take an `tf.distribute.InputContext` object.
+      replication_mode: an enum value of `tf.distribute.InputReplicationMode`.
+        Only `PER_WORKER` is supported currently.
+
+    Returns:
+      An iterator object that can be initialized and fetched next element.
+    """
+    if replication_mode != InputReplicationMode.PER_WORKER:
+      raise ValueError(
+          "Input replication mode not supported: %r" % replication_mode)
+    return self.extended._make_input_fn_iterator(  # pylint: disable=protected-access
+        input_fn, replication_mode=replication_mode)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def broadcast(self, tensor, destinations=None):
+    """DEPRECATED: use extended.broadcast_to() instead."""
+    return self._extended.broadcast_to(tensor, destinations)
+
+  @doc_controls.do_not_generate_docs  # Use experimental_initialize() instead.
+  def initialize(self):
+    """DEPRECATED: Use `experimental_initialize()` instead."""
+    return self._extended._initialize()  # pylint: disable=protected-access
+
+  def experimental_initialize(self):
+    """Any initialization to be done before running any computations.
+
+    In eager mode, it executes any initialization as a side effect.
+    In graph mode, it creates the initialization ops and returns them.
+
+    For example, TPU initialize_system ops.
+
+    Returns:
+      A list of ops to execute.
+    """
+    return self._extended._initialize()  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # Use experimental_finalize() instead.
+  def finalize(self):
+    """DEPRECATED: Use `experimental_finalize()` instead."""
+    return self._extended._finalize()  # pylint: disable=protected-access
+
+  def experimental_finalize(self):
+    """Any final actions to be done at the end of all computations.
+
+    In eager mode, it executes any finalize actions as a side effect.
+    In graph mode, it creates the finalize ops and returns them.
+
+    For example, TPU shutdown ops.
+
+    Returns:
+      A list of ops to execute.
+    """
+    return self._extended._finalize()  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def run_steps_on_dataset(self, fn, iterator, iterations=1,
+                           initial_loop_values=None):
+    """DEPRECATED: use extended.experimental_run_steps_on_iterator() instead."""
+    return self._extended.experimental_run_steps_on_iterator(
+        fn, iterator, iterations, initial_loop_values)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def call_for_each_replica(self, fn, *args, **kwargs):
+    """DEPRECATED: use extended.call_for_each_replica() instead."""
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to call_for_each_replica")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to call_for_each_replica")
+      kwargs = k
+    kwargs.pop("run_concurrently", None)  # Ignore old option.
+    return self._extended.call_for_each_replica(fn, args, kwargs)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def reduce(self, aggregation, value, destinations):
+    """DEPRECATED: use extended.reduce_to() instead."""
+    return self._extended.reduce_to(aggregation, value, destinations)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def batch_reduce(self, aggregation, value_destination_pairs):
+    """DEPRECATED: use extended.batch_reduce_to() instead."""
+    return self._extended.batch_reduce_to(aggregation, value_destination_pairs)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def update(self, var, fn, *args, **kwargs):
+    """DEPRECATED: use extended.update() instead."""
+    group = kwargs.pop("group", True)
+    # We temporarily support "grouped" in addition to "group" for backward-
+    # compatibility.
+    group = kwargs.pop("grouped", True) and group
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to update")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to update")
+      kwargs = k
+    return self._extended.update(var, fn, args, kwargs, group)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    """DEPRECATED: use extended.update_non_slot() instead."""
+    group = kwargs.pop("group", True)
+    # We temporarily support "grouped" in addition to "group" for backward-
+    # compatibility.
+    group = kwargs.pop("grouped", True) and group
+    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
+    # allow transition.
+    a = kwargs.pop("args", None)
+    if a is not None:
+      if args:
+        raise ValueError(
+            "Can't pass *args and args=... to update_non_slot")
+      args = a
+    k = kwargs.pop("kwargs", None)
+    if k is not None:
+      if kwargs:
+        raise ValueError(
+            "Can't pass **kwargs and kwargs=... to update_non_slot")
+      kwargs = k
+    return self._extended.update_non_slot(
+        colocate_with, fn, args, kwargs, group)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
+  def unwrap(self, value):
+    """Returns the list of all per-replica values contained in `value`.
+
+    Args:
+      value: A value returned by `extended.call_for_each_replica()` or a
+        variable created in `scope`.
+
+    Returns:
+      A list of values contained in `value`. If `value` represents a single
+      value, this returns `[value].`
+    """
+    return self._extended._unwrap(value)  # pylint: disable=protected-access
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def value_container(self, value):
+    """DEPRECATED: use extended.value_container() instead."""
+    return self._extended.value_container(value)
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
+  def group(self, value, name=None):
+    """Shortcut for `tf.group(self.unwrap(value))`."""
+    return self._extended._group(value, name)  # pylint: disable=protected-access
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def require_static_shapes(self):
+    """DEPRECATED: use extended.require_static_shapes instead."""
+    return self._extended.experimental_require_static_shapes
+
+  @property
+  def num_replicas_in_sync(self):
+    """Returns number of replicas over which gradients are aggregated."""
+    return self._extended._num_replicas_in_sync  # pylint: disable=protected-access
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def worker_devices(self):
+    """DEPRECATED: use extended.worker_devices instead."""
+    return self._extended.worker_devices
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def parameter_devices(self):
+    """DEPRECATED: use extended.parameter_devices instead."""
+    return self._extended.parameter_devices
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def non_slot_devices(self, var_list):
+    """DEPRECATED: use extended.non_slot_devices instead."""
+    return self._extended.non_slot_devices(var_list)
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def between_graph(self):
+    """DEPRECATED: use extended.experimental_between_graph instead."""
+    return self._extended.experimental_between_graph
+
+  @doc_controls.do_not_generate_docs  # DEPRECATED, being replaced by a new API.
+  def configure(self,
+                session_config=None,
+                cluster_spec=None,
+                task_type=None,
+                task_id=None):
+    # pylint: disable=g-doc-return-or-yield,g-doc-args
+    """DEPRECATED: use `update_config_proto` instead.
+
+    Configures the strategy class.
+
+    DEPRECATED: This method's functionality has been split into the strategy
+    constructor and `update_config_proto`. In the future, we will allow passing
+    cluster and config_proto to the constructor to configure the strategy. And
+    `update_config_proto` can be used to update the config_proto based on the
+    specific strategy.
+    """
+    return self._extended._configure(  # pylint: disable=protected-access
+        session_config, cluster_spec, task_type, task_id)
+
+  def update_config_proto(self, config_proto):
+    """Returns a copy of `config_proto` modified for use with this strategy.
+
+    The updated config has something needed to run a strategy, e.g.
+    configuration to run collective ops, or device filters to improve
+    distributed training performance.
+
+    Args:
+      config_proto: a `tf.ConfigProto` object.
+
+    Returns:
+      The updated copy of the `config_proto`.
+    """
+    return self._extended._update_config_proto(config_proto)  # pylint: disable=protected-access
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def should_init(self):
+    """DEPRECATED: use extended.should_init instead."""
+    return self._extended.experimental_should_init
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def should_checkpoint(self):
+    """DEPRECATED: use extended.should_checkpoint instead."""
+    return self._extended.should_checkpoint
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
+  def should_save_summary(self):
+    """DEPRECATED: use extended.should_save_summary instead."""
+    return self._extended.should_save_summary
+
+  def __deepcopy__(self, memo):
+    # First do a regular deepcopy of `self`.
+    cls = self.__class__
+    result = cls.__new__(cls)
+    memo[id(self)] = result
+    for k, v in self.__dict__.items():
+      setattr(result, k, copy.deepcopy(v, memo))
+    # One little fix-up: we want `result._extended` to reference `result`
+    # instead of `self`.
+    result._extended._container_strategy_weakref = weakref.ref(result)  # pylint: disable=protected-access
+    return result
+
+  def __copy__(self):
+    raise RuntimeError("Must only deepcopy DistributionStrategy.")
+
+
+@tf_export("distribute.StrategyExtended")
+class DistributionStrategyExtended(object):
+  """Additional APIs for algorithms that need to be distribution-aware.
+
+  The intent is that you can write an algorithm in a stylized way and
+  it will be usable with a variety of different
+  `tf.distribute.Strategy`
+  implementations. Each descendant will implement a different strategy
+  for distributing the algorithm across multiple devices/machines.
+  Furthermore, these changes can be hidden inside the specific layers
+  and other library classes that need special treatment to run in a
+  distributed setting, so that most users' model definition code can
+  run unchanged. The `tf.distribute.Strategy` API works the same way
+  with eager and graph execution.
+
+  First let's introduce a few high-level concepts:
+
+  * _Data parallelism_ is where we run multiple copies of the model
+    on different slices of the input data. This is in contrast to
+    _model parallelism_ where we divide up a single copy of a model
+    across multiple devices.
+    Note: we only support data parallelism for now, but
+    hope to add support for model parallelism in the future.
+  * A _replica_ is one copy of the model, running on one slice of the
+    input data.
+  * _Synchronous_, or more commonly _sync_, training is where the
+    updates from each replica are aggregated together before updating
+    the model variables. This is in contrast to _asynchronous_, or
+    _async_ training, where each replica updates the model variables
+    independently.
+  * Furthermore you might run your computation on multiple devices
+    on one machine (or "host"), or on multiple machines/hosts.
+    If you are running on multiple machines, you might have a
+    single master host that drives computation across all of them,
+    or you might have multiple clients driving the computation
+    asynchronously.
+
+  To distribute an algorithm, we might use some of these ingredients:
+
+  * Parameter servers: These are hosts that hold a single copy of
+    parameters/variables. All replicas that want to operate on a variable
+    retrieve it at the beginning of a step and send an update to be
+    applied at the end of the step. Can support either sync or async
+    training.
+  * Mirrored variables: These are variables that are copied to multiple
+    devices, where we keep the copies in sync by applying the same
+    updates to every copy. Normally would only be used with sync training.
+  * Reductions and Allreduce: A _reduction_ is some method of
+    aggregating multiple values into one value, like "sum" or
+    "mean". If doing sync training, we will perform a reduction on the
+    gradients to a parameter from all replicas before applying the
+    update. Allreduce is an algorithm for performing a reduction on
+    values from multiple devices and making the result available on
+    all of those devices.
+  * In the future we will have support for TensorFlow's partitioned
+    variables, where a single variable is split across multiple
+    devices.
+
+  We have then a few approaches we want to support:
+
+  * Code written (as if) with no knowledge of class `tf.distribute.Strategy`.
+    This code should work as before, even if some of the layers, etc.
+    used by that code are written to be distribution-aware. This is done
+    by having a default `tf.distribute.Strategy` that gives ordinary behavior,
+    and by default being in a single replica context.
+  * Ordinary model code that you want to run using a specific
+    `tf.distribute.Strategy`. This can be as simple as:
+
+    ```
+    with my_strategy.scope():
+      iterator = my_strategy.make_dataset_iterator(dataset)
+      session.run(iterator.initialize())
+      replica_train_ops = my_strategy.extended.call_for_each_replica(
+          replica_fn, args=(iterator.get_next(),))
+      train_op = my_strategy.group(replica_train_ops)
+    ```
+
+    This takes an ordinary `dataset` and `replica_fn` and runs it
+    distributed using a particular `tf.distribute.Strategy` in
+    `my_strategy`. Any variables created in `replica_fn` are created
+    using `my_strategy`'s policy, and library functions called by
+    `replica_fn` can use the `get_replica_context()` API to get enhanced
+    behavior in this case.
+
+  * If you want to write a distributed algorithm, you may use any of
+    the `tf.distribute.Strategy` APIs inside a
+    `with my_strategy.scope():` block of code.
+
+  Lower-level concepts:
+
+  * Wrapped values: In order to represent values parallel across devices
+    (either replicas or the devices associated with a particular value), we
+    wrap them in a "PerReplica" or "Mirrored" object that contains a map
+    from device to values. "PerReplica" is used when the value may be
+    different across replicas, and "Mirrored" when the value are the same.
+  * Unwrapping and merging: Consider calling a function `fn` on multiple
+    replicas, like `extended.call_for_each_replica(fn, args=[w])` with an
+    argument `w` that is a wrapped value. This means `w` will have a map taking
+    replica device `d0` to `w0`, replica device `d1` to `w1`,
+    etc. `extended.call_for_each_replica()` unwraps `w` before calling `fn`, so
+    it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc.  It then merges the return
+    values from `fn()`, which can possibly result in wrapped values. For
+    example, let's say `fn()` returns a tuple with three components: `(x, a,
+    v0)` from replica 0, `(x, b, v1)` on replica 1, etc. If the first component
+    is the same object `x` from every replica, then the first component of the
+    merged result will also be `x`. If the second component is different (`a`,
+    `b`, ...)  from each replica, then the merged value will have a wrapped map
+    from replica device to the different values. If the third component is the
+    members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to `v1`, etc.),
+    then the merged result will be that mirrored variable (`v`).
+  * Replica context vs. Cross-replica context: _replica context_ is when we
+    are in some function that is being called once for each replica.
+    Otherwise we are in cross-replica context, which is useful for
+    calling `tf.distribute.Strategy` methods which operate across the
+    replicas (like `reduce_to()`). By default you start in a replica context
+    (the default "single replica context") and then some methods can
+    switch you back and forth, as described below.
+  * Worker devices vs. parameter devices: Most replica computations will
+    happen on worker devices. Since we don't yet support model
+    parallelism, there will be one worker device per replica. When using
+    parameter servers (see above), the set of devices holding
+    variables may be different, otherwise the parameter devices might
+    match the worker devices.
+  * Non-slot devices are some subset of the parameter devices where we
+    put all the non-slot variables. We need to ensure that all
+    non-slot variables are allocated on the same device, or mirrored
+    across the same set of devices. If you have some variable you want
+    to colocate all the non-slot variables with, you can use
+    `colocate_vars_with()` to get the remaining non-slot variables on
+    the same device.  Otherwise you can use `non_slot_devices()` to
+    pick a consistent set of devices to pass to both
+    `colocate_vars_with()` and `update_non_slot()`.
+
+  When using a `tf.distribute.Strategy`, we have a new type dimension
+  called _locality_ that says what values are compatible with which
+  APIs:
+
+  * T: different value for each replica (e.g. a PerReplica-wrapped value).
+  * M: value is "mirrored" across replicas, i.e. there are copies with the
+    same value on each replica (e.g. a Mirrored-wrapped value).
+  * V(`v`): value is "mirrored" across all the devices which have a
+    copy of variable `v` (also a Mirrored-wrapped value, but over
+    parameter devices instead of worker devices).
+  * N: value is "mirrored" across all the "non-slot" devices
+
+  Rules for methods with respect to locality and single-replica vs.
+  cross-replica context:
+
+  * `with d.scope()`: default single-replica context -> cross-replica context
+    for `d`
+  * `with d.extended.colocate_vars_with(v)`: in replica/cross-replica context,
+    variables will be created with locality V(`v`). That is, if we write
+    `with d.extended.colocate_vars_with(v1): v2 = tf.get_variable(...)`,
+    then `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal
+    V(`v1`).
+  * `with d.extended.colocate_vars_with(d.extended.non_slot_devices(...))`: in
+    replica/cross-replica context, variables will be created with locality N
+  * `v = tf.get_variable(...)`: in replica/cross-replica context, creates
+    a variable (which by definition will have locality V(`v`), though
+    will match another locality if inside a `colocate_vars_with`
+    scope).
+  * `d.make_dataset_iterator(dataset)` (or the deprecated
+    `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
+    context, produces an iterator with locality T
+  * `d.extended.broadcast_to(t)`: in cross-replica context, produces a value
+    with locality M
+  * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
+    with locality V(`v`)
+  * `d.extended.call_for_each_replica(fn, ...)`: in cross-replica context, runs
+    `fn()` in a replica context (and so may call `get_replica_context()` and
+    use its API, including `merge_call()` to get back to cross-replica
+    context), once for each replica. May use values with locality T or
+    M, and any variable.
+  * `d.extended.reduce_to(m, t, t)`: in cross-replica context, accepts t with
+    locality T and produces a value with locality M.
+  * `d.extended.reduce_to(m, t, v)`: in cross-replica context, accepts t with
+    locality T and produces a value with locality V(`v`).
+  * `d.extended.batch_reduce_to(m, [(t, v)]): see `d.extended.reduce_to()`
+  * `d.extended.update(v, fn, ...)`: in cross-replica context, runs `fn()` once
+    for each device `v` is copied to, all inputs should have locality
+    V(`v`), output will have locality V(`v`) as well.
+  * `d.extended.update_non_slot(d.extended.non_slot_devices(), fn)`: in
+    cross-replica context, like `d.extended.update()` except with locality N.
+  * `d.extended.read_var(v)`: Gets the (read-only) value of the variable `v` (on
+    the device determined by the current device scope), aggregating
+    across replicas for replica-local variables. Frequently, this will be
+    done automatically when using `v` in an expression or fetching it in
+    a cross-replica context, but this function can be used to force that
+    conversion happens at a particular point in time (for example, to
+    add the result of the conversion to a graph collection).
+
+  The standard pattern for updating variables is to:
+
+  1. Create an input iterator with `d.make_dataset_iterator()`.
+  2. Define each replica `d.extended.call_for_each_replica()` up to the point of
+     getting a list of gradient, variable pairs.
+  3. Call `d.extended.reduce_to(VariableAggregation.SUM, t, v)` or
+     `d.extended.batch_reduce_to()` to sum the gradients (with locality T)
+     into values with locality V(`v`).
+  4. Call `d.extended.update(v)` for each variable to update its value.
+
+  Steps 3 and 4 are done automatically by class `Optimizer` if you call
+  its `apply_gradients` method in a replica context. Otherwise you can
+  manually call its `_distributed_apply` method in a cross-replica context.
+
+  Another thing you might want to do in the middle of your replica function is
+  an all-reduce of some intermediate value, using `d.extended.reduce_to()` or
+  `d.extended.batch_reduce_to()`. You simply provide the same tensor as the
+  input and destination.
+
+  Layers should expect to be called in a replica context, and can use
+  the `tf.distribute.get_replica_context` function to get a
+  `tf.distribute.ReplicaContext` object. The
+  `ReplicaContext` object has a `merge_call()` method for entering
+  cross-replica context where you can use `reduce_to()` (or
+  `batch_reduce_to()`) and then optionally `update()` to update state.
+
+  You may use this API whether or not a `tf.distribute.Strategy` is
+  being used, since there is a default implementation of
+  `ReplicaContext` and `tf.distribute.Strategy`.
+
+  NOTE for new `tf.distribute.Strategy` implementations: Please put all logic
+  in a subclass of `tf.distribute.StrategyExtended`. The only code needed for
+  the `tf.distribute.Strategy` subclass is for instantiating your subclass of
+  `tf.distribute.StrategyExtended` in the `__init__` method.
+  """
+
+  def __init__(self, container_strategy):
+    self._container_strategy_weakref = weakref.ref(container_strategy)
+    self._default_device = None
+    # This property is used to determine if we should set drop_remainder=True
+    # when creating Datasets from numpy array inputs.
+    self._require_static_shapes = False
+
+  def _container_strategy(self):
+    """Get the containing `DistributionStrategy`.
+
+    This should not generally be needed except when creating a new
+    `ReplicaContext` and to validate that the caller is in the correct
+    `scope()`.
+
+    Returns:
+      The `DistributionStrategy` such that `strategy.extended` is `self`.
+    """
+    container_strategy = self._container_strategy_weakref()
+    assert container_strategy is not None
+    return container_strategy
+
+  def _scope(self, strategy):
+    """Implementation of DistributionStrategy.scope()."""
+    if distribution_strategy_context.has_distribution_strategy():
+      _require_cross_replica_context_extended(self)
+      return _SameScopeAgainContext(strategy)
+
+    def creator_with_resource_vars(*args, **kwargs):
+      _require_distribution_strategy_scope_extended(self)
+      kwargs["use_resource"] = True
+      return self._create_variable(*args, **kwargs)
+
+    def distributed_getter(getter, *args, **kwargs):
+      if not self._allow_variable_partition():
+        if kwargs.pop("partitioner", None) is not None:
+          tf_logging.log_first_n(
+              tf_logging.WARN, "Partitioned variables are disabled when using "
+              "current tf.distribute.Strategy.", 1)
+      return getter(*args, **kwargs)
+
+    return _CurrentDistributionContext(
+        strategy,
+        variable_scope.variable_creator_scope(creator_with_resource_vars),
+        variable_scope.variable_scope(
+            variable_scope.get_variable_scope(),
+            custom_getter=distributed_getter), self._default_device)
+
+  def _allow_variable_partition(self):
+    return False
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    # Note: should support "colocate_with" argument.
+    raise NotImplementedError("must be implemented in descendants")
+
+  def read_var(self, v):
+    """Reads the value of a variable.
+
+    Returns the aggregate value of a replica-local variable, or the
+    (read-only) value of any other variable.
+
+    Args:
+      v: A variable allocated within the scope of this `tf.distribute.Strategy`.
+
+    Returns:
+      A tensor representing the value of `v`, aggregated across replicas if
+      necessary.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  def colocate_vars_with(self, colocate_with_variable):
+    """Scope that controls which devices variables will be created on.
+
+    No operations should be added to the graph inside this scope, it
+    should only be used when creating variables (some implementations
+    work by changing variable creation, others work by using a
+    tf.colocate_with() scope).
+
+    This may only be used inside `self.scope()`.
+
+    Example usage:
+
+    ```
+    with strategy.scope():
+      var1 = tf.get_variable(...)
+      with strategy.extended.colocate_vars_with(v1):
+        # var2 and var3 will be created on the same device(s) as var1
+        var2 = tf.get_variable(...)
+        var3 = tf.get_variable(...)
+
+      def fn(v1, v2, v3):
+        # operates on v1 from var1, v2 from var2, and v3 from var3
+
+      # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too.
+      strategy.extended.update(v1, fn, args=(v2, v3))
+    ```
+
+    Args:
+      colocate_with_variable: A created in `self.scope()`. Variables created
+        while in the returned context manager will be on the same set of
+        devices as `colocate_with_variable`.
+
+    Returns:
+      A context manager.
+    """
+    def create_colocated_variable(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope_extended(self)
+      kwargs["use_resource"] = True
+      kwargs["colocate_with"] = colocate_with_variable
+      return next_creator(*args, **kwargs)
+
+    _require_distribution_strategy_scope_extended(self)
+    return variable_scope.variable_creator_scope(create_colocated_variable)
+
+  def _call_dataset_fn(self, dataset_fn):
+    """Call the `dataset_fn` with `input_context` as argument."""
+    result = dataset_fn()
+    if not isinstance(result, dataset_ops.DatasetV2):
+      raise ValueError(
+          "dataset_fn() must return a tf.data.Dataset when using a "
+          "tf.distribute.Strategy.")
+    return result
+
+  # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
+  # Dataset API such as make_one_shot_iterator and make_initializable_iterator.
+  # Extend to implement more functionality of datasets.
+  def _distribute_dataset(self, dataset_fn):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def _make_dataset_iterator(self, dataset):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def _make_input_fn_iterator(self, input_fn, replication_mode):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def broadcast_to(self, tensor, destinations):
+    """Mirror a tensor on one device to all worker devices.
+
+    Args:
+      tensor: A Tensor value to broadcast.
+      destinations: A mirrored variable, device string, or list of device
+        strings, specifying the destination devices to copy `tensor` to.
+
+    Returns:
+      A value mirrored to `destinations` devices.
+    """
+    # TODO(josh11b): More docstring
+    _require_cross_replica_context_extended(self)
+    return self._broadcast_to(tensor, destinations)
+
+  def _broadcast_to(self, tensor, destinations):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def _initialize(self):
+    return []
+
+  def _finalize(self):
+    return []
+
+  def experimental_run_steps_on_iterator(self, fn, iterator, iterations=1,
+                                         initial_loop_values=None):
+    """Run `fn` with input from `iterator` for `iterations` times.
+
+    This method can be used to run a step function for training a number of
+    times using input from a dataset.
+
+    Args:
+      fn: function to run using this distribution strategy. The function must
+        have the following signature: `def fn(context, inputs)`.
+        `context` is an instance of `MultiStepContext` that will be passed when
+        `fn` is run. `context` can be used to specify the outputs to be returned
+        from `fn` by calling `context.set_last_step_output`. It can also be used
+        to capture non tensor outputs by `context.set_non_tensor_output`.
+        See `MultiStepContext` documentation for more information.
+        `inputs` will have same type/structure as `iterator.get_next()`.
+        Typically, `fn` will use `call_for_each_replica` method of the strategy
+        to distribute the computation over multiple replicas.
+      iterator: Iterator of a dataset that represents the input for `fn`. The
+        caller is responsible for initializing the iterator as needed.
+      iterations: (Optional) Number of iterations that `fn` should be run.
+        Defaults to 1.
+      initial_loop_values: (Optional) Initial values to be passed into the
+        loop that runs `fn`. Defaults to `None`. # TODO(priyag): Remove
+        initial_loop_values argument when we have a mechanism to infer the
+        outputs of `fn`.
+
+    Returns:
+      Returns the `MultiStepContext` object which has the following properties,
+      among other things:
+        - run_op: An op that runs `fn` `iterations` times.
+        - last_step_outputs: A dictionary containing tensors set using
+        `context.set_last_step_output`. Evaluating this returns the value of
+        the tensors after the last iteration.
+        - non_tensor_outputs: A dictionatry containing anything that was set by
+          `fn` by calling `context.set_non_tensor_output`.
+    """
+    _require_cross_replica_context_extended(self)
+    return self._experimental_run_steps_on_iterator(
+        fn, iterator, iterations, initial_loop_values)
+
+  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
+                                          initial_loop_values):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def call_for_each_replica(self, fn, args=(), kwargs=None):
+    """Run `fn` once per replica.
+
+    `fn` may call `tf.get_replica_context()` to access methods such as
+    `replica_id_in_sync_group` and `merge_call()`.
+
+    `merge_call()` is used to communicate between the replicas and
+    re-enter the cross-replica context. All replicas pause their execution
+    having encountered a `merge_call()` call. After that the
+    `merge_fn`-function is executed. Its results are then unwrapped and
+    given back to each replica call. After that execution resumes until
+    `fn` is complete or encounters another `merge_call()`.  Example:
+
+    ```python
+    # Called once in "cross-replica" context.
+    def merge_fn(distribution, three_plus_replica_id):
+      # sum the values across replicas
+      return sum(distribution.unwrap(three_plus_replica_id))
+
+    # Called once per replica in `distribution`, in a "replica" context.
+    def fn(three):
+      replica_ctx = tf.get_replica_context()
+      v = three + replica_ctx.replica_id_in_sync_group
+      # Computes the sum of the `v` values across all replicas.
+      s = replica_ctx.merge_call(merge_fn, args=(v,))
+      return s + v
+
+    with distribution.scope():
+      # in "cross-replica" context
+      ...
+      merged_results = distribution.call_for_each_replica(fn, args=[3])
+      # merged_results has the values from every replica execution of `fn`.
+      print(distribution.unwrap(merged_results))  # Prints a list
+    ```
+
+    Args:
+      fn: function to run (will be run once per replica).
+      args: Tuple or list with positional arguments for `fn`.
+      kwargs: Dict with keyword arguments for `fn`.
+
+    Returns:
+      Merged return value of `fn` across all replicas.
+    """
+    _require_cross_replica_context_extended(self)
+    if kwargs is None:
+      kwargs = {}
+    return self._call_for_each_replica(fn, args, kwargs)
+
+  def _call_for_each_replica(self, fn, args, kwargs):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def reduce_to(self, reduce_op, value, destinations):
+    """Combine (via e.g. sum or mean) values across replicas.
+
+    Args:
+      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+        DEPRECATED but still accepted values:
+        `tf.VariableAggregation.SUM`,
+        `tf.VariableAggregation.MEAN`,
+      value: A per-replica value with one value per replica.
+      destinations: A mirrored variable, a per-replica tensor, a device string,
+        or list of device strings. The return value will be copied to all
+        destination devices (or all the devices where the `destinations` value
+        resides). To perform an all-reduction, pass `value` to `destinations`.
+
+    Returns:
+      A value mirrored to `destinations`.
+    """
+    # TODO(josh11b): More docstring
+    # TODO(josh11b): Return an unwrapped value if colocate_with is a
+    # single device.
+    _require_cross_replica_context_extended(self)
+
+    # TODO(priyag): Remove this when all callers have been updated.
+    if isinstance(reduce_op, variable_scope.VariableAggregation):
+      assert reduce_op in [
+          variable_scope.VariableAggregation.SUM,
+          variable_scope.VariableAggregation.MEAN,
+      ]
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    return self._reduce_to(reduce_op, value, destinations)
+
+  def _reduce_to(self, reduce_op, value, destinations):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def batch_reduce_to(self, reduce_op, value_destination_pairs):
+    """Combine multiple `reduce_to` calls into one for faster execution.
+
+    Args:
+      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+        DEPRECATED but still accepted values:
+        `tf.VariableAggregation.SUM`,
+        `tf.VariableAggregation.MEAN`,
+      value_destination_pairs: A sequence of (value, destinations)
+        pairs. See `reduce_to()` for a description.
+
+    Returns:
+      A list of mirrored values, one per pair in `value_destination_pairs`.
+    """
+    # TODO(josh11b): More docstring
+    _require_cross_replica_context_extended(self)
+
+    # TODO(priyag): Remove this when all callers have been updated.
+    if isinstance(reduce_op, variable_scope.VariableAggregation):
+      assert reduce_op in [
+          variable_scope.VariableAggregation.SUM,
+          variable_scope.VariableAggregation.MEAN,
+      ]
+      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    return self._batch_reduce_to(reduce_op, value_destination_pairs)
+
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
+    return [
+        self.reduce_to(reduce_op, t, destinations=v)
+        for t, v in value_destination_pairs
+    ]
+
+  def update(self, var, fn, args=(), kwargs=None, group=True):
+    """Run `fn` to update `var` using inputs mirrored to the same devices.
+
+    If `var` is mirrored across multiple devices, then this implements
+    logic like:
+
+    ```
+    results = {}
+    for device, v in var:
+      with tf.device(device):
+        # args and kwargs will be unwrapped if they are mirrored.
+        results[device] = fn(v, *args, **kwargs)
+    return merged(results)
+    ```
+
+    Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`.
+
+    Neither `args` nor `kwargs` may contain per-replica values.
+    If they contain mirrored values, they will be unwrapped before
+    calling `fn`.
+
+    Args:
+      var: Variable, possibly mirrored to multiple devices, to operate on.
+      fn: Function to call. Should take the variable as the first argument.
+      args: Tuple or list. Additional positional arguments to pass to `fn()`.
+      kwargs: Dict with keyword arguments to pass to `fn()`.
+      group: Boolean. Defaults to True. If False, the return value will be
+        unwrapped.
+
+    Returns:
+      By default, the merged return value of `fn` across all replicas.  The
+      merged result has dependencies to make sure that if it is evaluated at
+      all, the side effects (updates) will happen on every replica. If instead
+      "group=False" is specified, this function will return a nest of lists
+      where each list has an element per replica, and the caller is responsible
+      for ensuring all elements are executed.
+    """
+    _require_cross_replica_context_extended(self)
+    if kwargs is None:
+      kwargs = {}
+    return self._update(var, fn, args, kwargs, group)
+
+  def _update(self, var, fn, args, kwargs, group):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def update_non_slot(
+      self, colocate_with, fn, args=(), kwargs=None, group=True):
+    """Runs `fn(*args, **kwargs)` on `colocate_with` devices.
+
+    Args:
+      colocate_with: The return value of `non_slot_devices()`.
+      fn: Function to execute.
+      args: Tuple or list. Positional arguments to pass to `fn()`.
+      kwargs: Dict with keyword arguments to pass to `fn()`.
+      group: Boolean. Defaults to True. If False, the return value will be
+        unwrapped.
+
+    Returns:
+      Return value of `fn`, possibly merged across devices.
+    """
+    _require_cross_replica_context_extended(self)
+    if kwargs is None:
+      kwargs = {}
+    return self._update_non_slot(colocate_with, fn, args, kwargs, group)
+
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def _unwrap(self, distributed_value):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def value_container(self, value):
+    """Returns the container that this per-replica `value` belongs to.
+
+    Args:
+      value: A value returned by `call_for_each_replica()` or a variable
+        created in `scope()`.
+
+    Returns:
+      A container that `value` belongs to.
+      If value does not belong to any container (including the case of
+      container having been destroyed), returns the value itself.
+      `value in unwrap(value_container(value))` will always be true.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  def _group(self, value, name=None):
+    """Shortcut for `tf.group(distribution.unwrap(value))`."""
+    value = nest.flatten(self._unwrap(value))
+
+    if len(value) != 1 or name is not None:
+      return control_flow_ops.group(value, name=name)
+    # Special handling for the common case of one op.
+    v, = value
+    if hasattr(v, "op"):
+      v = v.op
+    return v
+
+  @property
+  def experimental_require_static_shapes(self):
+    return self._require_static_shapes
+
+  @property
+  def _num_replicas_in_sync(self):
+    """Returns number of replicas over which gradients are aggregated."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def worker_devices(self):
+    """Returns the list of devices used to run `call_for_each_replica()` calls.
+    """
+    # TODO(josh11b): More docstring
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def parameter_devices(self):
+    """Returns the list of devices used for variable and `update` placement."""
+    # TODO(josh11b): More docstring
+    raise NotImplementedError("must be implemented in descendants")
+
+  def non_slot_devices(self, var_list):
+    """Device(s) for non-slot variables.
+
+    Create variables on these devices in a
+    `with colocate_vars_with(non_slot_devices(...)):` block.
+    Update those using `update_non_slot()`.
+
+    Args:
+      var_list: The list of variables being optimized, needed with the
+        default `tf.distribute.Strategy`.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def experimental_between_graph(self):
+    """Whether the strategy uses between-graph replication or not.
+
+      This is expected to return a constant value that will not be changed
+      throughout its life cycle.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
+    """Configures the strategy class."""
+    del session_config, cluster_spec, task_type, task_id
+
+  def _update_config_proto(self, config_proto):
+    return copy.deepcopy(config_proto)
+
+  @property
+  def experimental_should_init(self):
+    """Whether initialization is needed."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def should_checkpoint(self):
+    """Whether checkpointing is needed."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def should_save_summary(self):
+    """Whether saving summaries is needed."""
+    raise NotImplementedError("must be implemented in descendants")
+
+
+# A note about the difference between the context managers
+# `ReplicaContext` (defined here) and `_CurrentDistributionContext`
+# (defined above) used by `DistributionStrategy.scope()`:
+#
+# * a ReplicaContext is only present during a `call_for_each_replica()`
+#   call (except during a `merge_run` call) and in such a scope it
+#   will be returned by calls to `get_replica_context()`.  Implementers of new
+#   DistributionStrategy descendants will frequently also need to
+#   define a descendant of ReplicaContext, and are responsible for
+#   entering and exiting this context.
+#
+# * DistributionStrategy.scope() sets up a variable_creator scope that
+#   changes variable creation calls (e.g. to make mirrored
+#   variables). This is intended as an outer scope that users enter once
+#   around their model creation and graph definition. There is no
+#   anticipated need to define descendants of _CurrentDistributionContext.
+#   It sets the current DistributionStrategy for purposes of
+#   `get_strategy()` and `has_strategy()`
+#   and switches the thread mode to a "cross-replica context".
+@tf_export("distribute.ReplicaContext")
+class ReplicaContext(object):
+  """`tf.distribute.Strategy` API when in a replica context.
+
+  To be used inside your replicated step function, such as in a
+  `tf.distribute.StrategyExtended.call_for_each_replica` call.
+  """
+
+  def __init__(self, strategy, replica_id_in_sync_group):
+    self._distribution_strategy = strategy
+    self._thread_context = distribution_strategy_context._InReplicaThreadMode(  # pylint: disable=protected-access
+        self)
+    self._replica_id_in_sync_group = replica_id_in_sync_group
+
+  def __enter__(self):
+    _push_per_thread_mode(self._thread_context)
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    _pop_per_thread_mode()
+
+  def merge_call(self, merge_fn, args=(), kwargs=None):
+    """Merge args across replicas and run `merge_fn` in a cross-replica context.
+
+    This allows communication and coordination when there are multiple calls
+    to a model function triggered by a call to
+    `strategy.extended.call_for_each_replica(model_fn, ...)`.
+
+    See `tf.distribute.StrategyExtended.call_for_each_replica` for an
+    explanation.
+
+    If not inside a distributed scope, this is equivalent to:
+
+    ```
+    strategy = tf.distribute.get_strategy()
+    with cross-replica-context(strategy):
+      return merge_fn(strategy, *args, **kwargs)
+    ```
+
+    Args:
+      merge_fn: function that joins arguments from threads that are given as
+        PerReplica. It accepts `tf.distribute.Strategy` object as
+        the first argument.
+      args: List or tuple with positional per-thread arguments for `merge_fn`.
+      kwargs: Dict with keyword per-thread arguments for `merge_fn`.
+
+    Returns:
+      The return value of `merge_fn`, except for `PerReplica` values which are
+      unpacked.
+    """
+    require_replica_context(self)
+    if kwargs is None:
+      kwargs = {}
+    return self._merge_call(merge_fn, args, kwargs)
+
+  def _merge_call(self, merge_fn, args, kwargs):
+    """Default implementation for single replica."""
+    _push_per_thread_mode(  # thread-local, so not needed with multiple threads
+        distribution_strategy_context._CrossReplicaThreadMode(  # pylint: disable=protected-access
+            self._distribution_strategy))
+    try:
+      return merge_fn(self._distribution_strategy, *args, **kwargs)
+    finally:
+      _pop_per_thread_mode()
+
+  @property
+  def num_replicas_in_sync(self):
+    """Returns number of replicas over which gradients are aggregated."""
+    return self._distribution_strategy.num_replicas_in_sync
+
+  @property
+  def replica_id_in_sync_group(self):
+    """Which replica is being defined, from 0 to `num_replicas_in_sync - 1`."""
+    require_replica_context(self)
+    return self._replica_id_in_sync_group
+
+  @property
+  @doc_controls.do_not_generate_docs  # DEPRECATED, use `strategy`
+  def distribution_strategy(self):
+    """DEPRECATED: use `self.stratgey` instead."""
+    return self._distribution_strategy
+
+  @property
+  def strategy(self):
+    """The current `tf.distribute.Strategy` object."""
+    return self._distribution_strategy
+
+  @property
+  def devices(self):
+    """The devices this replica is to be executed on, as a list of strings."""
+    require_replica_context(self)
+    return [device_util.current()]
+
+  # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient
+  # all-reduce. It would return a function returning the result of reducing `t`
+  # across all replicas. The caller would wait to call this function until they
+  # needed the reduce result, allowing an efficient implementation:
+  # * With eager execution, the reduction could be performed asynchronously
+  #   in the background, not blocking until the result was needed.
+  # * When constructing a graph, it could batch up all reduction requests up
+  #   to that point that the first result is needed. Most likely this can be
+  #   implemented in terms of `merge_call()` and `batch_reduce_to()`.
+
+# ------------------------------------------------------------------------------
+
+
+class _DefaultDistributionStrategy(DistributionStrategy):
+  """Default `tf.distribute.Strategy` if none is explicitly selected."""
+
+  def __init__(self):
+    super(_DefaultDistributionStrategy, self).__init__(
+        _DefaultDistributionExtended(self))
+
+
+class _DefaultDistributionExtended(DistributionStrategyExtended):
+  """Implementation of _DefaultDistributionStrategy."""
+
+  def _scope(self, strategy):
+    """Context manager setting a variable creator and `self` as current."""
+    if distribution_strategy_context.has_distribution_strategy():
+      raise RuntimeError("Must not nest tf.distribute.Strategy scopes.")
+
+    def creator(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope_strategy(strategy)
+      return next_creator(*args, **kwargs)
+
+    return _CurrentDistributionContext(
+        strategy, variable_scope.variable_creator_scope(creator))
+
+  def colocate_vars_with(self, colocate_with_variable):
+    """Does not require `self.scope`."""
+    _require_distribution_strategy_scope_extended(self)
+    return ops.colocate_with(colocate_with_variable)
+
+  def _distribute_dataset(self, dataset_fn):
+    return self._call_dataset_fn(dataset_fn)
+
+  def _make_dataset_iterator(self, dataset):
+    return _DefaultDistributionExtended.DefaultInputIterator(dataset)
+
+  def _make_input_fn_iterator(self,
+                              input_fn,
+                              replication_mode=InputReplicationMode.PER_WORKER):
+    return input_fn(InputContext()).make_initializable_iterator()
+
+  def _broadcast_to(self, tensor, destinations):
+    if destinations is None:
+      return tensor
+    else:
+      raise NotImplementedError("TODO")
+
+  def _call_for_each_replica(self, fn, args, kwargs):
+    with ReplicaContext(
+        self._container_strategy(),
+        replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
+      return fn(*args, **kwargs)
+
+  def _reduce_to(self, reduce_op, value, destinations):
+    # TODO(josh11b): Use destinations?
+    del reduce_op, destinations
+    return value
+
+  def _update(self, var, fn, args, kwargs, group):
+    # The implementations of _update() and _update_non_slot() are identical
+    # except _update() passes `var` as the first argument to `fn()`.
+    return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
+
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, should_group):
+    # TODO(josh11b): Figure out what we should be passing to UpdateContext()
+    # once that value is used for something.
+    with ops.colocate_with(colocate_with), UpdateContext(colocate_with):
+      result = fn(*args, **kwargs)
+      if should_group:
+        return result
+      else:
+        return nest.map_structure(self._unwrap, result)
+
+  def read_var(self, replica_local_var):
+    return array_ops.identity(replica_local_var)
+
+  def _unwrap(self, distributed_value):
+    return [distributed_value]
+
+  def value_container(self, value):
+    return value
+
+  @property
+  def _num_replicas_in_sync(self):
+    return 1
+
+  @property
+  def worker_devices(self):
+    raise RuntimeError("worker_devices() method unsupported by default "
+                       "tf.distribute.Strategy.")
+
+  @property
+  def parameter_devices(self):
+    raise RuntimeError("parameter_devices() method unsupported by default "
+                       "tf.distribute.Strategy.")
+
+  def non_slot_devices(self, var_list):
+    return min(var_list, key=lambda x: x.name)
+
+  # TODO(priyag): This should inherit from `InputIterator`, once dependency
+  # issues have been resolved.
+  class DefaultInputIterator(object):
+    """Default implementation of `InputIterator` for default strategy."""
+
+    def __init__(self, dataset):
+      self._dataset = dataset
+      if eager_context.executing_eagerly():
+        self._iterator = dataset.make_one_shot_iterator()
+      else:
+        self._iterator = dataset.make_initializable_iterator()
+
+    def get_next(self):
+      return self._iterator.get_next()
+
+    def initialize(self):
+      if eager_context.executing_eagerly():
+        self._iterator = self._dataset.make_one_shot_iterator()
+        return []
+      else:
+        return [self._iterator.initializer]
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
+
+
+# ------------------------------------------------------------------------------
+# We haven't yet implemented deserialization for DistributedVariables.
+# So here we catch any attempts to deserialize variables
+# when using distribution strategies.
+# pylint: disable=protected-access
+_original_from_proto = resource_variable_ops._from_proto_fn
+
+
+def _from_proto_fn(v, import_scope=None):
+  if distribution_strategy_context.has_distribution_strategy():
+    raise NotImplementedError(
+        "Deserialization of variables is not yet supported when using a "
+        "tf.distribute.Strategy.")
+  else:
+    return _original_from_proto(v, import_scope=import_scope)
+
+resource_variable_ops._from_proto_fn = _from_proto_fn
+# pylint: enable=protected-access
+
+
+#-------------------------------------------------------------------------------
+# Shorthand for some methods from distribution_strategy_context.
+_push_per_thread_mode = distribution_strategy_context._push_per_thread_mode  # pylint: disable=protected-access
+_get_per_thread_mode = distribution_strategy_context._get_per_thread_mode  # pylint: disable=protected-access
+_pop_per_thread_mode = distribution_strategy_context._pop_per_thread_mode  # pylint: disable=protected-access
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/distribute/distribute_lib_test.py
similarity index 98%
rename from tensorflow/python/training/distribute_test.py
rename to tensorflow/python/distribute/distribute_lib_test.py
index 4758e3d3d4..d63d1fe3c3 100644
--- a/tensorflow/python/training/distribute_test.py
+++ b/tensorflow/python/distribute/distribute_lib_test.py
@@ -18,12 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
-from tensorflow.python.training import distribute as distribute_lib
-from tensorflow.python.training import distribution_strategy_context
 
 
 class _TestReplicaContext(distribute_lib.ReplicaContext):
diff --git a/tensorflow/python/distribute/distribution_strategy_context.py b/tensorflow/python/distribute/distribution_strategy_context.py
new file mode 100644
index 0000000000..78e096e286
--- /dev/null
+++ b/tensorflow/python/distribute/distribution_strategy_context.py
@@ -0,0 +1,236 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility to get distribution strategy related contexts."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.util.lazy_loader import LazyLoader
+from tensorflow.python.util.tf_export import tf_export
+
+
+# There is a circular dependency between this and `distribute` module. So we
+# load it lazily to workaround this.
+distribute_lib = LazyLoader(
+    "distribute_lib", globals(),
+    "tensorflow.python.distribute.distribute_lib")
+
+# ------------------------------------------------------------------------------
+# Internal API for setting the current thread mode as being either in a
+# replica or cross-replica context for a particular distribution strategy.
+
+
+class _ThreadMode(object):
+
+  def __init__(self, dist, cross, replica):
+    self.distribution_strategy = dist
+    self.cross_replica_context = cross
+    self.replica_context = replica
+
+
+class _CrossReplicaThreadMode(_ThreadMode):
+
+  def __init__(self, distribution_strategy):
+    _ThreadMode.__init__(
+        self, distribution_strategy, distribution_strategy, None)
+
+
+class _InReplicaThreadMode(_ThreadMode):
+
+  def __init__(self, replica_ctx):
+    _ThreadMode.__init__(
+        self, replica_ctx.distribution_strategy, None, replica_ctx)
+
+
+def _push_per_thread_mode(context):
+  ops.get_default_graph()._distribution_strategy_stack.append(context)  # pylint: disable=protected-access
+
+
+def _pop_per_thread_mode():
+  ops.get_default_graph()._distribution_strategy_stack.pop(-1)  # pylint: disable=protected-access
+
+
+class _DefaultReplicaThreadMode(_ThreadMode):
+  """Type of default value returned by `_get_per_thread_mode()`.
+
+  Used when the thread-local stack is empty.
+  """
+
+  def __init__(self):
+    _ThreadMode.__init__(self, _get_default_distribution_strategy(), None,
+                         _get_default_replica_context())
+
+
+def _get_per_thread_mode():
+  try:
+    return ops.get_default_graph()._distribution_strategy_stack[-1]  # pylint: disable=protected-access
+  except (AttributeError, IndexError):
+    return _get_default_replica_mode()
+
+
+# ------------------------------------------------------------------------------
+# Public API for accessing the current thread mode
+
+
+@tf_export("distribute.get_replica_context")
+def get_replica_context():
+  """Returns the current `tf.distribute.ReplicaContext` or `None`.
+
+  Returns `None` if in a cross-replica context.
+
+  Note that execution:
+
+  1. starts in the default (single-replica) replica context (this function
+     will return the default `ReplicaContext` object);
+  2. switches to cross-replica context (in which case this will return
+     `None`) when entering a `with tf.distribute.Strategy.scope():` block;
+  3. switches to a (non-default) replica context inside
+     `extended.call_for_each_replica(fn, ...)`;
+  4. if `fn` calls `get_replica_context().merge_call(merge_fn, ...)`, then
+     inside `merge_fn` you are back in the cross-replica context (and again
+     this function will return `None`).
+
+  Note that you can also go directly from step 1 to 4 to switch to a
+  cross-replica context for the default `tf.distribute.Strategy`. You may
+  also switch from the cross-replica context of 4 to a replica context by
+  calling `extended.call_for_each_replica()`, jumping back to step 3.
+
+  Most `tf.distribute.Strategy` methods may only be executed in
+  a cross-replica context, in a replica context you should use the
+  `ReplicaContext` API instead.
+
+  Returns:
+    The current `ReplicaContext` object when in a replica context scope,
+    else `None`.
+
+    Within a particular block, exactly one of these two things will be true:
+
+    * `get_replica_context()` returns non-`None`, or
+    * `tf.distribute.is_cross_replica_context()` returns True.
+  """
+  return _get_per_thread_mode().replica_context
+
+
+def get_cross_replica_context():
+  """Returns the current tf.distribute.Strategy if in a cross-replica context.
+
+  DEPRECATED: Please use `in_cross_replica_context()` and
+  `get_distribution_strategy()` instead.
+
+  Note that execution:
+
+  1. starts in the default (single-replica) replica context;
+  2. switches to cross-replica context when entering a
+     `with tf.distribute.Strategy.scope():` block;
+  3. switches to a (non-default) replica context inside
+     `call_for_each_replica(fn, ...)`;
+  4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then
+     inside `merge_fn` you are back in the cross-replica context.
+
+  Note that you can also go directly from step 1 to 4 to switch to a
+  cross-replica context for the default `tf.distribute.Strategy`. You may
+  also switch from the cross-replica context of 4 to a replica context by
+  calling `call_for_each_replica()`, jumping back to step 3.
+
+  Most `tf.distribute.Strategy` methods may only be executed in
+  a cross-replica context.
+
+  Returns:
+    Returns the current `tf.distribute.Strategy` object in a cross-replica
+    context, or `None`.
+
+    Exactly one of `get_replica_context()` and `get_cross_replica_context()`
+    will return `None` in a particular block.
+  """
+  return _get_per_thread_mode().cross_replica_context
+
+
+@tf_export("distribute.in_cross_replica_context")
+def in_cross_replica_context():
+  """Returns True if in a cross-replica context.
+
+  See `tf.distribute.get_replica_context` for details.
+
+  Returns:
+    True if in a cross-replica context (`get_replica_context()` returns
+    `None`), or False if in a replica context (`get_replica_context()` returns
+    non-`None`).
+  """
+  return _get_per_thread_mode().cross_replica_context is not None
+
+
+@tf_export("distribute.get_strategy")
+def get_distribution_strategy():
+  """Returns the current `tf.distribute.Strategy` object.
+
+  Typically only used in a cross-replica context:
+
+  ```
+  if tf.distribute.in_cross_replica_context():
+    strategy = tf.distribute.get_strategy()
+    ...
+  ```
+
+  Returns:
+    A `tf.distribute.Strategy` object. Inside a
+    `with distribution_strategy.scope()` block, it returns
+    `distribution_strategy`, otherwise it returns the default
+    (single-replica) `tf.distribute.Strategy` object.
+  """
+  return _get_per_thread_mode().distribution_strategy
+
+
+@tf_export("distribute.has_strategy")
+def has_distribution_strategy():
+  """Return if there is a current non-default `tf.distribute.Strategy`.
+
+  Returns:
+    True if inside a `with strategy.scope():`.
+  """
+  return get_distribution_strategy() is not _get_default_distribution_strategy()
+
+
+# ------------------------------------------------------------------------------
+# Defaults that are used when no distribution strategy is explicitly created.
+# We create them lazily in a function so that we can workaround the circular
+# dependency on distribute_lib. See lazy loader at the top of this file.
+
+_defaults = {
+    "distribution_strategy": None,
+    "replica_context": None,
+    "replica_mode": None
+}
+
+
+def _get_default_distribution_strategy():
+  if _defaults["distribution_strategy"] is None:
+    _defaults["distribution_strategy"] = (
+        distribute_lib._DefaultDistributionStrategy())  # pylint: disable=protected-access
+  return _defaults["distribution_strategy"]
+
+
+def _get_default_replica_context():
+  if _defaults["replica_context"] is None:
+    _defaults["replica_context"] = distribute_lib.ReplicaContext(
+        _get_default_distribution_strategy(), replica_id_in_sync_group=0)
+  return _defaults["replica_context"]
+
+
+def _get_default_replica_mode():
+  if _defaults["replica_mode"] is None:
+    _defaults["replica_mode"] = _DefaultReplicaThreadMode()
+  return _defaults["replica_mode"]
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 7094ed6628..402a94c2b7 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -25,6 +25,8 @@ import threading
 
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import shared_variable_creator
@@ -40,8 +42,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.training import coordinator
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 727e491a2f..7dd1062e38 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -30,6 +30,9 @@ import six
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import multi_device_iterator_ops
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.distribute import input_ops
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
@@ -42,9 +45,6 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribute as distribute_lib
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import saver
 from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.util import nest
diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py
index 9e211d172e..abb5886deb 100644
--- a/tensorflow/python/tools/api/generator/doc_srcs.py
+++ b/tensorflow/python/tools/api/generator/doc_srcs.py
@@ -37,7 +37,7 @@ _TENSORFLOW_DOC_SOURCES = {
     'app': DocSource(docstring_module_name='platform.app'),
     'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'),
     'compat': DocSource(docstring_module_name='util.compat'),
-    'distribute': DocSource(docstring_module_name='training.distribute'),
+    'distribute': DocSource(docstring_module_name='distribute.distribute_lib'),
     'distributions': DocSource(
         docstring_module_name='ops.distributions.distributions'),
     'errors': DocSource(docstring_module_name='framework.errors'),
diff --git a/tensorflow/python/training/device_util.py b/tensorflow/python/training/device_util.py
index 70e1ca4b5d..bf8e98052d 100644
--- a/tensorflow/python/training/device_util.py
+++ b/tensorflow/python/training/device_util.py
@@ -12,86 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Device-related support functions."""
+"""Deprecated, please use ../distribute/device_util.py."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.eager import context
-from tensorflow.python.framework import device as tf_device
-from tensorflow.python.framework import ops
-
-
-def canonicalize(d, default=None):
-  """Canonicalize device string.
-
-  If d has missing components, the rest would be deduced from the `default`
-  argument or from '/replica:0/task:0/device:CPU:0'. For example:
-    If d = '/cpu:0', default='/job:worker/task:1', it returns
-      '/job:worker/replica:0/task:1/device:CPU:0'.
-    If d = '/cpu:0', default='/job:worker', it returns
-      '/job:worker/replica:0/task:0/device:CPU:0'.
-    If d = '/gpu:0', default=None, it returns
-      '/replica:0/task:0/device:GPU:0'.
-
-  Note: This uses "job:localhost" as the default if executing eagerly.
-
-  Args:
-    d: a device string.
-    default: a string for default device if d doesn't have all components.
-
-  Returns:
-    a canonicalized device string.
-  """
-  d = tf_device.DeviceSpec.from_string(d)
-  assert d.device_type is None or d.device_type == d.device_type.upper(), (
-      "Device type '%s' must be all-caps." % (d.device_type,))
-  # Fill in missing device fields using defaults.
-  result = tf_device.DeviceSpec(
-      replica=0, task=0, device_type="CPU", device_index=0)
-  if context.executing_eagerly():
-    result.job = "localhost"
-  if default:
-    result.merge_from(tf_device.DeviceSpec.from_string(default))
-  result.merge_from(d)
-  return result.to_string()
-
-
-def resolve(d):
-  """Canonicalize `d` with current device as default."""
-  return canonicalize(d, default=current())
-
-
-class _FakeNodeDef(object):
-  """A fake NodeDef for _FakeOperation."""
-
-  def __init__(self):
-    self.op = ""
-    self.name = ""
-
-
-class _FakeOperation(object):
-  """A fake Operation object to pass to device functions."""
-
-  def __init__(self):
-    self.device = ""
-    self.type = ""
-    self.name = ""
-    self.node_def = _FakeNodeDef()
-
-  def _set_device(self, device):
-    self.device = ops._device_string(device)  # pylint: disable=protected-access
-
-
-def current():
-  """Return a string (not canonicalized) for the current device."""
-  # TODO(josh11b): Work out how this function interacts with ops.colocate_with.
-  ctx = context.context()
-  if ctx.executing_eagerly():
-    d = ctx.device_name
-  else:
-    op = _FakeOperation()
-    ops.get_default_graph()._apply_device_functions(op)  # pylint: disable=protected-access
-    d = op.device
-  return d
+# pylint: disable=wildcard-import
+from tensorflow.python.distribute.device_util import *
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index a976062249..ad27bc8a70 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -12,1654 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Library for running a computation across multiple devices."""
+"""Deprecated, please use ../distribute/distribute_lib.py."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
-import threading
-import weakref
-import enum
-
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.distribute import reduce_util
-from tensorflow.python.eager import context as eager_context
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops.losses import losses_impl
-from tensorflow.python.platform import tf_logging
-from tensorflow.python.training import device_util
-from tensorflow.python.training import distribution_strategy_context
-from tensorflow.python.util import nest
-from tensorflow.python.util.tf_export import tf_export
-from tensorflow.tools.docs import doc_controls
-
-
-# ------------------------------------------------------------------------------
-# Context tracking whether in a strategy.update() or .update_non_slot() call.
-
-
-_update_device = threading.local()
-
-
-def get_update_device():
-  """Get the current device if in a `tf.distribute.Strategy.update()` call."""
-  try:
-    return _update_device.current
-  except AttributeError:
-    return None
-
-
-class UpdateContext(object):
-  """Context manager when you are in `update()` or `update_non_slot()`."""
-
-  def __init__(self, device):
-    self._device = device
-    self._old_device = None
-
-  def __enter__(self):
-    self._old_device = get_update_device()
-    _update_device.current = self._device
-
-  def __exit__(self, exception_type, exception_value, traceback):
-    del exception_type, exception_value, traceback
-    _update_device.current = self._old_device
-
-
-# ------------------------------------------------------------------------------
-# Public utility functions.
-
-
-@tf_export("distribute.get_loss_reduction")
-def get_loss_reduction():
-  """`tf.distribute.ReduceOp` corresponding to the last loss reduction."""
-  loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
-  if loss_reduction == losses_impl.Reduction.SUM:
-    return reduce_util.ReduceOp.SUM
-  return reduce_util.ReduceOp.MEAN
-
-
-# ------------------------------------------------------------------------------
-# Internal API for validating the current thread mode
-
-
-def _require_cross_replica_context_extended(extended):
-  """Verify in cross-replica context."""
-  context = _get_per_thread_mode()
-  cross_replica = context.cross_replica_context
-  if cross_replica is not None and cross_replica.extended is extended:
-    return
-  strategy = extended._container_strategy()  # pylint: disable=protected-access
-  # We have an error to report, figure out the right message.
-  if context.distribution_strategy is not strategy:
-    _wrong_strategy_scope(strategy, context)
-  assert cross_replica is None
-  raise RuntimeError("Method requires being in cross-replica context, use "
-                     "get_replica_context().merge_call()")
-
-
-def _wrong_strategy_scope(strategy, context):
-  # Figure out the right error message.
-  if not distribution_strategy_context.has_distribution_strategy():
-    raise RuntimeError(
-        'Need to be inside "with strategy.scope()" for %s' %
-        (strategy,))
-  else:
-    raise RuntimeError(
-        "Mixing different tf.distribute.Strategy objects: %s is not %s" %
-        (context.distribution_strategy, strategy))
-
-
-def require_replica_context(replica_ctx):
-  """Verify in `replica_ctx` replica context."""
-  context = _get_per_thread_mode()
-  if context.replica_context is replica_ctx: return
-  # We have an error to report, figure out the right message.
-  if context.replica_context is None:
-    raise RuntimeError("Need to be inside `call_for_each_replica()`")
-  if context.distribution_strategy is replica_ctx.distribution_strategy:
-    # Two different ReplicaContexts with the same tf.distribute.Strategy.
-    raise RuntimeError("Mismatching ReplicaContext.")
-  raise RuntimeError(
-      "Mismatching tf.distribute.Strategy objects: %s is not %s." %
-      (context.distribution_strategy, replica_ctx.distribution_strategy))
-
-
-def _require_distribution_strategy_scope_strategy(strategy):
-  """Verify in a `strategy.scope()` in this thread."""
-  context = _get_per_thread_mode()
-  if context.distribution_strategy is strategy: return
-  _wrong_strategy_scope(strategy, context)
-
-
-def _require_distribution_strategy_scope_extended(extended):
-  """Verify in a `distribution_strategy.scope()` in this thread."""
-  context = _get_per_thread_mode()
-  if context.distribution_strategy.extended is extended: return
-  # Report error.
-  strategy = extended._container_strategy()  # pylint: disable=protected-access
-  _wrong_strategy_scope(strategy, context)
-
-
-# ------------------------------------------------------------------------------
-# Internal context managers used to implement the DistributionStrategy
-# base class
-
-
-class _CurrentDistributionContext(object):
-  """Context manager setting the current `tf.distribute.Strategy`.
-
-  Also: overrides the variable creator and optionally the current device.
-  """
-
-  def __init__(self,
-               strategy,
-               var_creator_scope,
-               var_scope=None,
-               default_device=None):
-    self._context = distribution_strategy_context._CrossReplicaThreadMode(  # pylint: disable=protected-access
-        strategy)
-    self._var_creator_scope = var_creator_scope
-    self._var_scope = var_scope
-    if default_device:
-      self._device_scope = ops.device(default_device)
-    else:
-      self._device_scope = None
-
-  def __enter__(self):
-    _push_per_thread_mode(self._context)
-    if self._var_scope:
-      self._var_scope.__enter__()
-    self._var_creator_scope.__enter__()
-    if self._device_scope:
-      self._device_scope.__enter__()
-    return self._context.distribution_strategy
-
-  def __exit__(self, exception_type, exception_value, traceback):
-    if self._device_scope:
-      self._device_scope.__exit__(exception_type, exception_value, traceback)
-    self._var_creator_scope.__exit__(exception_type, exception_value, traceback)
-    if self._var_scope:
-      self._var_scope.__exit__(exception_type, exception_value, traceback)
-    _pop_per_thread_mode()
-
-
-class _SameScopeAgainContext(object):
-  """Trivial context manager when you are already in `scope()`."""
-
-  def __init__(self, strategy):
-    self._distribution_strategy = strategy
-
-  def __enter__(self):
-    return self._distribution_strategy
-
-  def __exit__(self, exception_type, exception_value, traceback):
-    del exception_type, exception_value, traceback
-
-
-# TODO(yuefengz): add more replication modes.
-@tf_export("distribute.InputReplicationMode")
-class InputReplicationMode(enum.Enum):
-  """Replication mode for input function."""
-
-  # The input function will be called on each worker independently, creating as
-  # many input pipelines as number of workers. Replicas will dequeue from the
-  # local Dataset on their worker. Distribution Strategy doesn't manage any
-  # state sharing between such separate input pipelines.
-  PER_WORKER = "PER_WORKER"
-
-
-@tf_export("distribute.InputContext")
-class InputContext(object):
-  """A class wrapping information needed by an input function.
-
-  This is a context class that is passed to the user's input fn and contains
-  information about the compute replicas and input pipelines. The number of
-  compute replicas (in sync training) helps compute per input pipeline batch
-  size from the desired global batch size. Input pipeline information can be
-  used to return a different subset of the input in each input pipeline (for
-  e.g. shard the input pipeline, use a different input source etc).
-  """
-
-  def __init__(self,
-               num_input_pipelines=1,
-               input_pipeline_id=0,
-               num_replicas_in_sync=1):
-    """Initializes an InputContext object.
-
-    Args:
-      num_input_pipelines: the number of input pipelines in a cluster.
-      input_pipeline_id: the current input pipeline id, should be an int in
-        [0,`num_input_pipelines`).
-      num_replicas_in_sync: the number of replicas that are in sync.
-    """
-    self._num_input_pipelines = num_input_pipelines
-    self._input_pipeline_id = input_pipeline_id
-    self._num_replicas_in_sync = num_replicas_in_sync
-
-  @property
-  def num_replicas_in_sync(self):
-    """Returns the number of compute replicas in sync."""
-    return self._num_replicas_in_sync
-
-  @property
-  def input_pipeline_id(self):
-    """Returns the input pipeline ID."""
-    return self._input_pipeline_id
-
-  @property
-  def num_input_pipelines(self):
-    """Returns the number of input pipelines."""
-    return self._num_input_pipelines
-
-  def get_per_replica_batch_size(self, global_batch_size):
-    """Returns the per-replica batch size.
-
-    Args:
-      global_batch_size: the global batch size which should be divisible by
-        `num_replicas_in_sync`.
-
-    Returns:
-      the per-replica batch size.
-
-    Raises:
-      ValueError: if `global_batch_size` not divisible by
-        `num_replicas_in_sync`.
-    """
-    if global_batch_size % self._num_replicas_in_sync != 0:
-      raise ValueError("The `global_batch_size` %r is not divisible by "
-                       "`num_replicas_in_sync` %r " %
-                       (global_batch_size, self._num_replicas_in_sync))
-    return global_batch_size // self._num_replicas_in_sync
-
-
-# ------------------------------------------------------------------------------
-# Base classes for all distribution strategies.
-
-
-@tf_export("distribute.Strategy")
-class DistributionStrategy(object):
-  """A list of devices with a state & compute distribution policy.
-
-  See [tensorflow/contrib/distribute/README.md](
-  https://www.tensorflow.org/code/tensorflow/contrib/distribute/README.md)
-  for overview and examples.
-  """
-
-  # TODO(josh11b): Raise an exception if variable partitioning requested before
-  #   we add support.
-  # TODO(josh11b): Also `parameter_device_index` property?
-  # TODO(josh11b): `map()`
-  # TODO(josh11b): ClusterSpec/ClusterResolver
-  # TODO(josh11b): Partitioned computations, state; sharding
-  # TODO(josh11b): Model parallelism: "replicas" with multiple devices; shuffling
-  # TODO(josh11b): List of replicas with their worker and parameter devices
-  #   (where the parameter devices may overlap in the ps case).
-
-  def __init__(self, extended):
-    self._extended = extended
-
-  @property
-  def extended(self):
-    """`tf.distribute.StrategyExtended` with additional methods."""
-    return self._extended
-
-  def scope(self):
-    """Returns a context manager selecting this Strategy as current.
-
-    Inside a `with strategy.scope():` code block, this thread
-    will use a variable creator set by `strategy`, and will
-    enter its "cross-replica context".
-
-    Returns:
-      A context manager.
-    """
-    return self._extended._scope(self)  # pylint: disable=protected-access
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def read_var(self, v):
-    """DEPRECATED: use extended.read_var() instead."""
-    return self._extended.read_var(v)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def colocate_vars_with(self, colocate_with_variable):
-    """DEPRECATED: use extended.colocate_vars_with() instead."""
-    return self._extended.colocate_vars_with(colocate_with_variable)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED
-  def distribute_dataset(self, dataset_fn):
-    """Return a `dataset` split across all replicas.  DEPRECATED.
-
-    DEPRECATED: Please use `make_dataset_iterator` or
-    `make_input_fn_iterator` instead.
-
-    Suitable for providing input to `extended.call_for_each_replica()` by
-    creating an iterator:
-
-    ```
-    def dataset_fn():
-      return tf.data.Dataset.from_tensors([[1.]]).repeat()
-
-    with strategy.scope():
-      distributed_dataset = strategy.distribute_dataset(dataset_fn)
-      iterator = distributed_dataset.make_initializable_iterator()
-      replica_results = strategy.extended.call_for_each_replica(
-          replica_fn, args=(iterator.get_next(),))
-    ```
-
-    Args:
-      dataset_fn: A function that returns a `tf.data.Dataset`.
-
-    Returns:
-      A `PerReplicaDataset` that will produce data for each replica.
-    """
-    return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
-
-  def make_dataset_iterator(self, dataset):
-    """Makes an iterator for input provided via input_dataset.
-
-    Data from the given dataset will be distributed evenly across all the
-    compute replicas. We will assume that the input dataset is batched by the
-    global batch size. With this assumption, we will make a best effort to
-    divide each batch across all the replicas (one or more workers).
-    If this effort fails, an error will be thrown, and the user should instead
-    use `make_input_fn_iterator` which provides more control to the user, and
-    does not try to divide a batch across replicas.
-
-    The user could also use `make_input_fn_iterator` if they want to
-    customize which input is fed to which replica/worker etc.
-
-    Args:
-      dataset: `tf.data.Dataset` that will be distributed evenly across all
-        replicas.
-
-    Returns:
-      An `tf.distribute.InputIterator` which returns inputs for each step of the
-      computation.  User should call `initialize` on the returned iterator.
-    """
-    return self._extended._make_dataset_iterator(dataset)  # pylint: disable=protected-access
-
-  def make_input_fn_iterator(self,
-                             input_fn,
-                             replication_mode=InputReplicationMode.PER_WORKER):
-    """Returns an iterator split across replicas created from an input function.
-
-    The `input_fn` should take an `tf.distribute.InputContext` object where
-    information about input sharding can be accessed:
-
-    ```
-    def input_fn(input_context):
-      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
-      return d.shard(input_context.num_input_pipelines,
-                     input_context.input_pipeline_id)
-    with strategy.scope():
-      iterator = strategy.make_input_fn_iterator(
-          input_fn)
-      replica_results = strategy.extended.call_for_each_replica(
-          replica_fn, iterator.get_next())
-    ```
-
-    Args:
-      input_fn: A function that returns a `tf.data.Dataset`. This function is
-        expected to take an `tf.distribute.InputContext` object.
-      replication_mode: an enum value of `tf.distribute.InputReplicationMode`.
-        Only `PER_WORKER` is supported currently.
-
-    Returns:
-      An iterator object that can be initialized and fetched next element.
-    """
-    if replication_mode != InputReplicationMode.PER_WORKER:
-      raise ValueError(
-          "Input replication mode not supported: %r" % replication_mode)
-    return self.extended._make_input_fn_iterator(  # pylint: disable=protected-access
-        input_fn, replication_mode=replication_mode)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def broadcast(self, tensor, destinations=None):
-    """DEPRECATED: use extended.broadcast_to() instead."""
-    return self._extended.broadcast_to(tensor, destinations)
-
-  @doc_controls.do_not_generate_docs  # Use experimental_initialize() instead.
-  def initialize(self):
-    """DEPRECATED: Use `experimental_initialize()` instead."""
-    return self._extended._initialize()  # pylint: disable=protected-access
-
-  def experimental_initialize(self):
-    """Any initialization to be done before running any computations.
-
-    In eager mode, it executes any initialization as a side effect.
-    In graph mode, it creates the initialization ops and returns them.
-
-    For example, TPU initialize_system ops.
-
-    Returns:
-      A list of ops to execute.
-    """
-    return self._extended._initialize()  # pylint: disable=protected-access
-
-  @doc_controls.do_not_generate_docs  # Use experimental_finalize() instead.
-  def finalize(self):
-    """DEPRECATED: Use `experimental_finalize()` instead."""
-    return self._extended._finalize()  # pylint: disable=protected-access
-
-  def experimental_finalize(self):
-    """Any final actions to be done at the end of all computations.
-
-    In eager mode, it executes any finalize actions as a side effect.
-    In graph mode, it creates the finalize ops and returns them.
-
-    For example, TPU shutdown ops.
-
-    Returns:
-      A list of ops to execute.
-    """
-    return self._extended._finalize()  # pylint: disable=protected-access
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def run_steps_on_dataset(self, fn, iterator, iterations=1,
-                           initial_loop_values=None):
-    """DEPRECATED: use extended.experimental_run_steps_on_iterator() instead."""
-    return self._extended.experimental_run_steps_on_iterator(
-        fn, iterator, iterations, initial_loop_values)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def call_for_each_replica(self, fn, *args, **kwargs):
-    """DEPRECATED: use extended.call_for_each_replica() instead."""
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to call_for_each_replica")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to call_for_each_replica")
-      kwargs = k
-    kwargs.pop("run_concurrently", None)  # Ignore old option.
-    return self._extended.call_for_each_replica(fn, args, kwargs)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def reduce(self, aggregation, value, destinations):
-    """DEPRECATED: use extended.reduce_to() instead."""
-    return self._extended.reduce_to(aggregation, value, destinations)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def batch_reduce(self, aggregation, value_destination_pairs):
-    """DEPRECATED: use extended.batch_reduce_to() instead."""
-    return self._extended.batch_reduce_to(aggregation, value_destination_pairs)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def update(self, var, fn, *args, **kwargs):
-    """DEPRECATED: use extended.update() instead."""
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update")
-      kwargs = k
-    return self._extended.update(var, fn, args, kwargs, group)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
-    """DEPRECATED: use extended.update_non_slot() instead."""
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update_non_slot")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update_non_slot")
-      kwargs = k
-    return self._extended.update_non_slot(
-        colocate_with, fn, args, kwargs, group)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
-  def unwrap(self, value):
-    """Returns the list of all per-replica values contained in `value`.
-
-    Args:
-      value: A value returned by `extended.call_for_each_replica()` or a
-        variable created in `scope`.
-
-    Returns:
-      A list of values contained in `value`. If `value` represents a single
-      value, this returns `[value].`
-    """
-    return self._extended._unwrap(value)  # pylint: disable=protected-access
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def value_container(self, value):
-    """DEPRECATED: use extended.value_container() instead."""
-    return self._extended.value_container(value)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
-  def group(self, value, name=None):
-    """Shortcut for `tf.group(self.unwrap(value))`."""
-    return self._extended._group(value, name)  # pylint: disable=protected-access
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def require_static_shapes(self):
-    """DEPRECATED: use extended.require_static_shapes instead."""
-    return self._extended.experimental_require_static_shapes
-
-  @property
-  def num_replicas_in_sync(self):
-    """Returns number of replicas over which gradients are aggregated."""
-    return self._extended._num_replicas_in_sync  # pylint: disable=protected-access
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def worker_devices(self):
-    """DEPRECATED: use extended.worker_devices instead."""
-    return self._extended.worker_devices
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def parameter_devices(self):
-    """DEPRECATED: use extended.parameter_devices instead."""
-    return self._extended.parameter_devices
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def non_slot_devices(self, var_list):
-    """DEPRECATED: use extended.non_slot_devices instead."""
-    return self._extended.non_slot_devices(var_list)
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def between_graph(self):
-    """DEPRECATED: use extended.experimental_between_graph instead."""
-    return self._extended.experimental_between_graph
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, being replaced by a new API.
-  def configure(self,
-                session_config=None,
-                cluster_spec=None,
-                task_type=None,
-                task_id=None):
-    # pylint: disable=g-doc-return-or-yield,g-doc-args
-    """DEPRECATED: use `update_config_proto` instead.
-
-    Configures the strategy class.
-
-    DEPRECATED: This method's functionality has been split into the strategy
-    constructor and `update_config_proto`. In the future, we will allow passing
-    cluster and config_proto to the constructor to configure the strategy. And
-    `update_config_proto` can be used to update the config_proto based on the
-    specific strategy.
-    """
-    return self._extended._configure(  # pylint: disable=protected-access
-        session_config, cluster_spec, task_type, task_id)
-
-  def update_config_proto(self, config_proto):
-    """Returns a copy of `config_proto` modified for use with this strategy.
-
-    The updated config has something needed to run a strategy, e.g.
-    configuration to run collective ops, or device filters to improve
-    distributed training performance.
-
-    Args:
-      config_proto: a `tf.ConfigProto` object.
-
-    Returns:
-      The updated copy of the `config_proto`.
-    """
-    return self._extended._update_config_proto(config_proto)  # pylint: disable=protected-access
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_init(self):
-    """DEPRECATED: use extended.should_init instead."""
-    return self._extended.experimental_should_init
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_checkpoint(self):
-    """DEPRECATED: use extended.should_checkpoint instead."""
-    return self._extended.should_checkpoint
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_save_summary(self):
-    """DEPRECATED: use extended.should_save_summary instead."""
-    return self._extended.should_save_summary
-
-  def __deepcopy__(self, memo):
-    # First do a regular deepcopy of `self`.
-    cls = self.__class__
-    result = cls.__new__(cls)
-    memo[id(self)] = result
-    for k, v in self.__dict__.items():
-      setattr(result, k, copy.deepcopy(v, memo))
-    # One little fix-up: we want `result._extended` to reference `result`
-    # instead of `self`.
-    result._extended._container_strategy_weakref = weakref.ref(result)  # pylint: disable=protected-access
-    return result
-
-  def __copy__(self):
-    raise RuntimeError("Must only deepcopy DistributionStrategy.")
-
-
-@tf_export("distribute.StrategyExtended")
-class DistributionStrategyExtended(object):
-  """Additional APIs for algorithms that need to be distribution-aware.
-
-  The intent is that you can write an algorithm in a stylized way and
-  it will be usable with a variety of different
-  `tf.distribute.Strategy`
-  implementations. Each descendant will implement a different strategy
-  for distributing the algorithm across multiple devices/machines.
-  Furthermore, these changes can be hidden inside the specific layers
-  and other library classes that need special treatment to run in a
-  distributed setting, so that most users' model definition code can
-  run unchanged. The `tf.distribute.Strategy` API works the same way
-  with eager and graph execution.
-
-  First let's introduce a few high-level concepts:
-
-  * _Data parallelism_ is where we run multiple copies of the model
-    on different slices of the input data. This is in contrast to
-    _model parallelism_ where we divide up a single copy of a model
-    across multiple devices.
-    Note: we only support data parallelism for now, but
-    hope to add support for model parallelism in the future.
-  * A _replica_ is one copy of the model, running on one slice of the
-    input data.
-  * _Synchronous_, or more commonly _sync_, training is where the
-    updates from each replica are aggregated together before updating
-    the model variables. This is in contrast to _asynchronous_, or
-    _async_ training, where each replica updates the model variables
-    independently.
-  * Furthermore you might run your computation on multiple devices
-    on one machine (or "host"), or on multiple machines/hosts.
-    If you are running on multiple machines, you might have a
-    single master host that drives computation across all of them,
-    or you might have multiple clients driving the computation
-    asynchronously.
-
-  To distribute an algorithm, we might use some of these ingredients:
-
-  * Parameter servers: These are hosts that hold a single copy of
-    parameters/variables. All replicas that want to operate on a variable
-    retrieve it at the beginning of a step and send an update to be
-    applied at the end of the step. Can support either sync or async
-    training.
-  * Mirrored variables: These are variables that are copied to multiple
-    devices, where we keep the copies in sync by applying the same
-    updates to every copy. Normally would only be used with sync training.
-  * Reductions and Allreduce: A _reduction_ is some method of
-    aggregating multiple values into one value, like "sum" or
-    "mean". If doing sync training, we will perform a reduction on the
-    gradients to a parameter from all replicas before applying the
-    update. Allreduce is an algorithm for performing a reduction on
-    values from multiple devices and making the result available on
-    all of those devices.
-  * In the future we will have support for TensorFlow's partitioned
-    variables, where a single variable is split across multiple
-    devices.
-
-  We have then a few approaches we want to support:
-
-  * Code written (as if) with no knowledge of class `tf.distribute.Strategy`.
-    This code should work as before, even if some of the layers, etc.
-    used by that code are written to be distribution-aware. This is done
-    by having a default `tf.distribute.Strategy` that gives ordinary behavior,
-    and by default being in a single replica context.
-  * Ordinary model code that you want to run using a specific
-    `tf.distribute.Strategy`. This can be as simple as:
-
-    ```
-    with my_strategy.scope():
-      iterator = my_strategy.make_dataset_iterator(dataset)
-      session.run(iterator.initialize())
-      replica_train_ops = my_strategy.extended.call_for_each_replica(
-          replica_fn, args=(iterator.get_next(),))
-      train_op = my_strategy.group(replica_train_ops)
-    ```
-
-    This takes an ordinary `dataset` and `replica_fn` and runs it
-    distributed using a particular `tf.distribute.Strategy` in
-    `my_strategy`. Any variables created in `replica_fn` are created
-    using `my_strategy`'s policy, and library functions called by
-    `replica_fn` can use the `get_replica_context()` API to get enhanced
-    behavior in this case.
-
-  * If you want to write a distributed algorithm, you may use any of
-    the `tf.distribute.Strategy` APIs inside a
-    `with my_strategy.scope():` block of code.
-
-  Lower-level concepts:
-
-  * Wrapped values: In order to represent values parallel across devices
-    (either replicas or the devices associated with a particular value), we
-    wrap them in a "PerReplica" or "Mirrored" object that contains a map
-    from device to values. "PerReplica" is used when the value may be
-    different across replicas, and "Mirrored" when the value are the same.
-  * Unwrapping and merging: Consider calling a function `fn` on multiple
-    replicas, like `extended.call_for_each_replica(fn, args=[w])` with an
-    argument `w` that is a wrapped value. This means `w` will have a map taking
-    replica device `d0` to `w0`, replica device `d1` to `w1`,
-    etc. `extended.call_for_each_replica()` unwraps `w` before calling `fn`, so
-    it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc.  It then merges the return
-    values from `fn()`, which can possibly result in wrapped values. For
-    example, let's say `fn()` returns a tuple with three components: `(x, a,
-    v0)` from replica 0, `(x, b, v1)` on replica 1, etc. If the first component
-    is the same object `x` from every replica, then the first component of the
-    merged result will also be `x`. If the second component is different (`a`,
-    `b`, ...)  from each replica, then the merged value will have a wrapped map
-    from replica device to the different values. If the third component is the
-    members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to `v1`, etc.),
-    then the merged result will be that mirrored variable (`v`).
-  * Replica context vs. Cross-replica context: _replica context_ is when we
-    are in some function that is being called once for each replica.
-    Otherwise we are in cross-replica context, which is useful for
-    calling `tf.distribute.Strategy` methods which operate across the
-    replicas (like `reduce_to()`). By default you start in a replica context
-    (the default "single replica context") and then some methods can
-    switch you back and forth, as described below.
-  * Worker devices vs. parameter devices: Most replica computations will
-    happen on worker devices. Since we don't yet support model
-    parallelism, there will be one worker device per replica. When using
-    parameter servers (see above), the set of devices holding
-    variables may be different, otherwise the parameter devices might
-    match the worker devices.
-  * Non-slot devices are some subset of the parameter devices where we
-    put all the non-slot variables. We need to ensure that all
-    non-slot variables are allocated on the same device, or mirrored
-    across the same set of devices. If you have some variable you want
-    to colocate all the non-slot variables with, you can use
-    `colocate_vars_with()` to get the remaining non-slot variables on
-    the same device.  Otherwise you can use `non_slot_devices()` to
-    pick a consistent set of devices to pass to both
-    `colocate_vars_with()` and `update_non_slot()`.
-
-  When using a `tf.distribute.Strategy`, we have a new type dimension
-  called _locality_ that says what values are compatible with which
-  APIs:
-
-  * T: different value for each replica (e.g. a PerReplica-wrapped value).
-  * M: value is "mirrored" across replicas, i.e. there are copies with the
-    same value on each replica (e.g. a Mirrored-wrapped value).
-  * V(`v`): value is "mirrored" across all the devices which have a
-    copy of variable `v` (also a Mirrored-wrapped value, but over
-    parameter devices instead of worker devices).
-  * N: value is "mirrored" across all the "non-slot" devices
-
-  Rules for methods with respect to locality and single-replica vs.
-  cross-replica context:
-
-  * `with d.scope()`: default single-replica context -> cross-replica context
-    for `d`
-  * `with d.extended.colocate_vars_with(v)`: in replica/cross-replica context,
-    variables will be created with locality V(`v`). That is, if we write
-    `with d.extended.colocate_vars_with(v1): v2 = tf.get_variable(...)`,
-    then `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal
-    V(`v1`).
-  * `with d.extended.colocate_vars_with(d.extended.non_slot_devices(...))`: in
-    replica/cross-replica context, variables will be created with locality N
-  * `v = tf.get_variable(...)`: in replica/cross-replica context, creates
-    a variable (which by definition will have locality V(`v`), though
-    will match another locality if inside a `colocate_vars_with`
-    scope).
-  * `d.make_dataset_iterator(dataset)` (or the deprecated
-    `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
-    context, produces an iterator with locality T
-  * `d.extended.broadcast_to(t)`: in cross-replica context, produces a value
-    with locality M
-  * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
-    with locality V(`v`)
-  * `d.extended.call_for_each_replica(fn, ...)`: in cross-replica context, runs
-    `fn()` in a replica context (and so may call `get_replica_context()` and
-    use its API, including `merge_call()` to get back to cross-replica
-    context), once for each replica. May use values with locality T or
-    M, and any variable.
-  * `d.extended.reduce_to(m, t, t)`: in cross-replica context, accepts t with
-    locality T and produces a value with locality M.
-  * `d.extended.reduce_to(m, t, v)`: in cross-replica context, accepts t with
-    locality T and produces a value with locality V(`v`).
-  * `d.extended.batch_reduce_to(m, [(t, v)]): see `d.extended.reduce_to()`
-  * `d.extended.update(v, fn, ...)`: in cross-replica context, runs `fn()` once
-    for each device `v` is copied to, all inputs should have locality
-    V(`v`), output will have locality V(`v`) as well.
-  * `d.extended.update_non_slot(d.extended.non_slot_devices(), fn)`: in
-    cross-replica context, like `d.extended.update()` except with locality N.
-  * `d.extended.read_var(v)`: Gets the (read-only) value of the variable `v` (on
-    the device determined by the current device scope), aggregating
-    across replicas for replica-local variables. Frequently, this will be
-    done automatically when using `v` in an expression or fetching it in
-    a cross-replica context, but this function can be used to force that
-    conversion happens at a particular point in time (for example, to
-    add the result of the conversion to a graph collection).
-
-  The standard pattern for updating variables is to:
-
-  1. Create an input iterator with `d.make_dataset_iterator()`.
-  2. Define each replica `d.extended.call_for_each_replica()` up to the point of
-     getting a list of gradient, variable pairs.
-  3. Call `d.extended.reduce_to(VariableAggregation.SUM, t, v)` or
-     `d.extended.batch_reduce_to()` to sum the gradients (with locality T)
-     into values with locality V(`v`).
-  4. Call `d.extended.update(v)` for each variable to update its value.
-
-  Steps 3 and 4 are done automatically by class `Optimizer` if you call
-  its `apply_gradients` method in a replica context. Otherwise you can
-  manually call its `_distributed_apply` method in a cross-replica context.
-
-  Another thing you might want to do in the middle of your replica function is
-  an all-reduce of some intermediate value, using `d.extended.reduce_to()` or
-  `d.extended.batch_reduce_to()`. You simply provide the same tensor as the
-  input and destination.
-
-  Layers should expect to be called in a replica context, and can use
-  the `tf.distribute.get_replica_context` function to get a
-  `tf.distribute.ReplicaContext` object. The
-  `ReplicaContext` object has a `merge_call()` method for entering
-  cross-replica context where you can use `reduce_to()` (or
-  `batch_reduce_to()`) and then optionally `update()` to update state.
-
-  You may use this API whether or not a `tf.distribute.Strategy` is
-  being used, since there is a default implementation of
-  `ReplicaContext` and `tf.distribute.Strategy`.
-
-  NOTE for new `tf.distribute.Strategy` implementations: Please put all logic
-  in a subclass of `tf.distribute.StrategyExtended`. The only code needed for
-  the `tf.distribute.Strategy` subclass is for instantiating your subclass of
-  `tf.distribute.StrategyExtended` in the `__init__` method.
-  """
-
-  def __init__(self, container_strategy):
-    self._container_strategy_weakref = weakref.ref(container_strategy)
-    self._default_device = None
-    # This property is used to determine if we should set drop_remainder=True
-    # when creating Datasets from numpy array inputs.
-    self._require_static_shapes = False
-
-  def _container_strategy(self):
-    """Get the containing `DistributionStrategy`.
-
-    This should not generally be needed except when creating a new
-    `ReplicaContext` and to validate that the caller is in the correct
-    `scope()`.
-
-    Returns:
-      The `DistributionStrategy` such that `strategy.extended` is `self`.
-    """
-    container_strategy = self._container_strategy_weakref()
-    assert container_strategy is not None
-    return container_strategy
-
-  def _scope(self, strategy):
-    """Implementation of DistributionStrategy.scope()."""
-    if distribution_strategy_context.has_distribution_strategy():
-      _require_cross_replica_context_extended(self)
-      return _SameScopeAgainContext(strategy)
-
-    def creator_with_resource_vars(*args, **kwargs):
-      _require_distribution_strategy_scope_extended(self)
-      kwargs["use_resource"] = True
-      return self._create_variable(*args, **kwargs)
-
-    def distributed_getter(getter, *args, **kwargs):
-      if not self._allow_variable_partition():
-        if kwargs.pop("partitioner", None) is not None:
-          tf_logging.log_first_n(
-              tf_logging.WARN, "Partitioned variables are disabled when using "
-              "current tf.distribute.Strategy.", 1)
-      return getter(*args, **kwargs)
-
-    return _CurrentDistributionContext(
-        strategy,
-        variable_scope.variable_creator_scope(creator_with_resource_vars),
-        variable_scope.variable_scope(
-            variable_scope.get_variable_scope(),
-            custom_getter=distributed_getter), self._default_device)
-
-  def _allow_variable_partition(self):
-    return False
-
-  def _create_variable(self, next_creator, *args, **kwargs):
-    # Note: should support "colocate_with" argument.
-    raise NotImplementedError("must be implemented in descendants")
-
-  def read_var(self, v):
-    """Reads the value of a variable.
-
-    Returns the aggregate value of a replica-local variable, or the
-    (read-only) value of any other variable.
-
-    Args:
-      v: A variable allocated within the scope of this `tf.distribute.Strategy`.
-
-    Returns:
-      A tensor representing the value of `v`, aggregated across replicas if
-      necessary.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
-  def colocate_vars_with(self, colocate_with_variable):
-    """Scope that controls which devices variables will be created on.
-
-    No operations should be added to the graph inside this scope, it
-    should only be used when creating variables (some implementations
-    work by changing variable creation, others work by using a
-    tf.colocate_with() scope).
-
-    This may only be used inside `self.scope()`.
-
-    Example usage:
-
-    ```
-    with strategy.scope():
-      var1 = tf.get_variable(...)
-      with strategy.extended.colocate_vars_with(v1):
-        # var2 and var3 will be created on the same device(s) as var1
-        var2 = tf.get_variable(...)
-        var3 = tf.get_variable(...)
-
-      def fn(v1, v2, v3):
-        # operates on v1 from var1, v2 from var2, and v3 from var3
-
-      # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too.
-      strategy.extended.update(v1, fn, args=(v2, v3))
-    ```
-
-    Args:
-      colocate_with_variable: A created in `self.scope()`. Variables created
-        while in the returned context manager will be on the same set of
-        devices as `colocate_with_variable`.
-
-    Returns:
-      A context manager.
-    """
-    def create_colocated_variable(next_creator, *args, **kwargs):
-      _require_distribution_strategy_scope_extended(self)
-      kwargs["use_resource"] = True
-      kwargs["colocate_with"] = colocate_with_variable
-      return next_creator(*args, **kwargs)
-
-    _require_distribution_strategy_scope_extended(self)
-    return variable_scope.variable_creator_scope(create_colocated_variable)
-
-  def _call_dataset_fn(self, dataset_fn):
-    """Call the `dataset_fn` with `input_context` as argument."""
-    result = dataset_fn()
-    if not isinstance(result, dataset_ops.DatasetV2):
-      raise ValueError(
-          "dataset_fn() must return a tf.data.Dataset when using a "
-          "tf.distribute.Strategy.")
-    return result
-
-  # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
-  # Dataset API such as make_one_shot_iterator and make_initializable_iterator.
-  # Extend to implement more functionality of datasets.
-  def _distribute_dataset(self, dataset_fn):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def _make_dataset_iterator(self, dataset):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def _make_input_fn_iterator(self, input_fn, replication_mode):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def broadcast_to(self, tensor, destinations):
-    """Mirror a tensor on one device to all worker devices.
-
-    Args:
-      tensor: A Tensor value to broadcast.
-      destinations: A mirrored variable, device string, or list of device
-        strings, specifying the destination devices to copy `tensor` to.
-
-    Returns:
-      A value mirrored to `destinations` devices.
-    """
-    # TODO(josh11b): More docstring
-    _require_cross_replica_context_extended(self)
-    return self._broadcast_to(tensor, destinations)
-
-  def _broadcast_to(self, tensor, destinations):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def _initialize(self):
-    return []
-
-  def _finalize(self):
-    return []
-
-  def experimental_run_steps_on_iterator(self, fn, iterator, iterations=1,
-                                         initial_loop_values=None):
-    """Run `fn` with input from `iterator` for `iterations` times.
-
-    This method can be used to run a step function for training a number of
-    times using input from a dataset.
-
-    Args:
-      fn: function to run using this distribution strategy. The function must
-        have the following signature: `def fn(context, inputs)`.
-        `context` is an instance of `MultiStepContext` that will be passed when
-        `fn` is run. `context` can be used to specify the outputs to be returned
-        from `fn` by calling `context.set_last_step_output`. It can also be used
-        to capture non tensor outputs by `context.set_non_tensor_output`.
-        See `MultiStepContext` documentation for more information.
-        `inputs` will have same type/structure as `iterator.get_next()`.
-        Typically, `fn` will use `call_for_each_replica` method of the strategy
-        to distribute the computation over multiple replicas.
-      iterator: Iterator of a dataset that represents the input for `fn`. The
-        caller is responsible for initializing the iterator as needed.
-      iterations: (Optional) Number of iterations that `fn` should be run.
-        Defaults to 1.
-      initial_loop_values: (Optional) Initial values to be passed into the
-        loop that runs `fn`. Defaults to `None`. # TODO(priyag): Remove
-        initial_loop_values argument when we have a mechanism to infer the
-        outputs of `fn`.
-
-    Returns:
-      Returns the `MultiStepContext` object which has the following properties,
-      among other things:
-        - run_op: An op that runs `fn` `iterations` times.
-        - last_step_outputs: A dictionary containing tensors set using
-        `context.set_last_step_output`. Evaluating this returns the value of
-        the tensors after the last iteration.
-        - non_tensor_outputs: A dictionatry containing anything that was set by
-          `fn` by calling `context.set_non_tensor_output`.
-    """
-    _require_cross_replica_context_extended(self)
-    return self._experimental_run_steps_on_iterator(
-        fn, iterator, iterations, initial_loop_values)
-
-  def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
-                                          initial_loop_values):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def call_for_each_replica(self, fn, args=(), kwargs=None):
-    """Run `fn` once per replica.
-
-    `fn` may call `tf.get_replica_context()` to access methods such as
-    `replica_id_in_sync_group` and `merge_call()`.
-
-    `merge_call()` is used to communicate between the replicas and
-    re-enter the cross-replica context. All replicas pause their execution
-    having encountered a `merge_call()` call. After that the
-    `merge_fn`-function is executed. Its results are then unwrapped and
-    given back to each replica call. After that execution resumes until
-    `fn` is complete or encounters another `merge_call()`.  Example:
-
-    ```python
-    # Called once in "cross-replica" context.
-    def merge_fn(distribution, three_plus_replica_id):
-      # sum the values across replicas
-      return sum(distribution.unwrap(three_plus_replica_id))
-
-    # Called once per replica in `distribution`, in a "replica" context.
-    def fn(three):
-      replica_ctx = tf.get_replica_context()
-      v = three + replica_ctx.replica_id_in_sync_group
-      # Computes the sum of the `v` values across all replicas.
-      s = replica_ctx.merge_call(merge_fn, args=(v,))
-      return s + v
-
-    with distribution.scope():
-      # in "cross-replica" context
-      ...
-      merged_results = distribution.call_for_each_replica(fn, args=[3])
-      # merged_results has the values from every replica execution of `fn`.
-      print(distribution.unwrap(merged_results))  # Prints a list
-    ```
-
-    Args:
-      fn: function to run (will be run once per replica).
-      args: Tuple or list with positional arguments for `fn`.
-      kwargs: Dict with keyword arguments for `fn`.
-
-    Returns:
-      Merged return value of `fn` across all replicas.
-    """
-    _require_cross_replica_context_extended(self)
-    if kwargs is None:
-      kwargs = {}
-    return self._call_for_each_replica(fn, args, kwargs)
-
-  def _call_for_each_replica(self, fn, args, kwargs):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def reduce_to(self, reduce_op, value, destinations):
-    """Combine (via e.g. sum or mean) values across replicas.
-
-    Args:
-      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
-        DEPRECATED but still accepted values:
-        `tf.VariableAggregation.SUM`,
-        `tf.VariableAggregation.MEAN`,
-      value: A per-replica value with one value per replica.
-      destinations: A mirrored variable, a per-replica tensor, a device string,
-        or list of device strings. The return value will be copied to all
-        destination devices (or all the devices where the `destinations` value
-        resides). To perform an all-reduction, pass `value` to `destinations`.
-
-    Returns:
-      A value mirrored to `destinations`.
-    """
-    # TODO(josh11b): More docstring
-    # TODO(josh11b): Return an unwrapped value if colocate_with is a
-    # single device.
-    _require_cross_replica_context_extended(self)
-
-    # TODO(priyag): Remove this when all callers have been updated.
-    if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in [
-          variable_scope.VariableAggregation.SUM,
-          variable_scope.VariableAggregation.MEAN,
-      ]
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
-    return self._reduce_to(reduce_op, value, destinations)
-
-  def _reduce_to(self, reduce_op, value, destinations):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def batch_reduce_to(self, reduce_op, value_destination_pairs):
-    """Combine multiple `reduce_to` calls into one for faster execution.
-
-    Args:
-      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
-        DEPRECATED but still accepted values:
-        `tf.VariableAggregation.SUM`,
-        `tf.VariableAggregation.MEAN`,
-      value_destination_pairs: A sequence of (value, destinations)
-        pairs. See `reduce_to()` for a description.
-
-    Returns:
-      A list of mirrored values, one per pair in `value_destination_pairs`.
-    """
-    # TODO(josh11b): More docstring
-    _require_cross_replica_context_extended(self)
-
-    # TODO(priyag): Remove this when all callers have been updated.
-    if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in [
-          variable_scope.VariableAggregation.SUM,
-          variable_scope.VariableAggregation.MEAN,
-      ]
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
-    return self._batch_reduce_to(reduce_op, value_destination_pairs)
-
-  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    return [
-        self.reduce_to(reduce_op, t, destinations=v)
-        for t, v in value_destination_pairs
-    ]
-
-  def update(self, var, fn, args=(), kwargs=None, group=True):
-    """Run `fn` to update `var` using inputs mirrored to the same devices.
-
-    If `var` is mirrored across multiple devices, then this implements
-    logic like:
-
-    ```
-    results = {}
-    for device, v in var:
-      with tf.device(device):
-        # args and kwargs will be unwrapped if they are mirrored.
-        results[device] = fn(v, *args, **kwargs)
-    return merged(results)
-    ```
-
-    Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`.
-
-    Neither `args` nor `kwargs` may contain per-replica values.
-    If they contain mirrored values, they will be unwrapped before
-    calling `fn`.
-
-    Args:
-      var: Variable, possibly mirrored to multiple devices, to operate on.
-      fn: Function to call. Should take the variable as the first argument.
-      args: Tuple or list. Additional positional arguments to pass to `fn()`.
-      kwargs: Dict with keyword arguments to pass to `fn()`.
-      group: Boolean. Defaults to True. If False, the return value will be
-        unwrapped.
-
-    Returns:
-      By default, the merged return value of `fn` across all replicas.  The
-      merged result has dependencies to make sure that if it is evaluated at
-      all, the side effects (updates) will happen on every replica. If instead
-      "group=False" is specified, this function will return a nest of lists
-      where each list has an element per replica, and the caller is responsible
-      for ensuring all elements are executed.
-    """
-    _require_cross_replica_context_extended(self)
-    if kwargs is None:
-      kwargs = {}
-    return self._update(var, fn, args, kwargs, group)
-
-  def _update(self, var, fn, args, kwargs, group):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def update_non_slot(
-      self, colocate_with, fn, args=(), kwargs=None, group=True):
-    """Runs `fn(*args, **kwargs)` on `colocate_with` devices.
-
-    Args:
-      colocate_with: The return value of `non_slot_devices()`.
-      fn: Function to execute.
-      args: Tuple or list. Positional arguments to pass to `fn()`.
-      kwargs: Dict with keyword arguments to pass to `fn()`.
-      group: Boolean. Defaults to True. If False, the return value will be
-        unwrapped.
-
-    Returns:
-      Return value of `fn`, possibly merged across devices.
-    """
-    _require_cross_replica_context_extended(self)
-    if kwargs is None:
-      kwargs = {}
-    return self._update_non_slot(colocate_with, fn, args, kwargs, group)
-
-  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def _unwrap(self, distributed_value):
-    raise NotImplementedError("must be implemented in descendants")
-
-  def value_container(self, value):
-    """Returns the container that this per-replica `value` belongs to.
-
-    Args:
-      value: A value returned by `call_for_each_replica()` or a variable
-        created in `scope()`.
-
-    Returns:
-      A container that `value` belongs to.
-      If value does not belong to any container (including the case of
-      container having been destroyed), returns the value itself.
-      `value in unwrap(value_container(value))` will always be true.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
-  def _group(self, value, name=None):
-    """Shortcut for `tf.group(distribution.unwrap(value))`."""
-    value = nest.flatten(self._unwrap(value))
-
-    if len(value) != 1 or name is not None:
-      return control_flow_ops.group(value, name=name)
-    # Special handling for the common case of one op.
-    v, = value
-    if hasattr(v, "op"):
-      v = v.op
-    return v
-
-  @property
-  def experimental_require_static_shapes(self):
-    return self._require_static_shapes
-
-  @property
-  def _num_replicas_in_sync(self):
-    """Returns number of replicas over which gradients are aggregated."""
-    raise NotImplementedError("must be implemented in descendants")
-
-  @property
-  def worker_devices(self):
-    """Returns the list of devices used to run `call_for_each_replica()` calls.
-    """
-    # TODO(josh11b): More docstring
-    raise NotImplementedError("must be implemented in descendants")
-
-  @property
-  def parameter_devices(self):
-    """Returns the list of devices used for variable and `update` placement."""
-    # TODO(josh11b): More docstring
-    raise NotImplementedError("must be implemented in descendants")
-
-  def non_slot_devices(self, var_list):
-    """Device(s) for non-slot variables.
-
-    Create variables on these devices in a
-    `with colocate_vars_with(non_slot_devices(...)):` block.
-    Update those using `update_non_slot()`.
-
-    Args:
-      var_list: The list of variables being optimized, needed with the
-        default `tf.distribute.Strategy`.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
-  @property
-  def experimental_between_graph(self):
-    """Whether the strategy uses between-graph replication or not.
-
-      This is expected to return a constant value that will not be changed
-      throughout its life cycle.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
-  def _configure(self,
-                 session_config=None,
-                 cluster_spec=None,
-                 task_type=None,
-                 task_id=None):
-    """Configures the strategy class."""
-    del session_config, cluster_spec, task_type, task_id
-
-  def _update_config_proto(self, config_proto):
-    return copy.deepcopy(config_proto)
-
-  @property
-  def experimental_should_init(self):
-    """Whether initialization is needed."""
-    raise NotImplementedError("must be implemented in descendants")
-
-  @property
-  def should_checkpoint(self):
-    """Whether checkpointing is needed."""
-    raise NotImplementedError("must be implemented in descendants")
-
-  @property
-  def should_save_summary(self):
-    """Whether saving summaries is needed."""
-    raise NotImplementedError("must be implemented in descendants")
-
-
-# A note about the difference between the context managers
-# `ReplicaContext` (defined here) and `_CurrentDistributionContext`
-# (defined above) used by `DistributionStrategy.scope()`:
-#
-# * a ReplicaContext is only present during a `call_for_each_replica()`
-#   call (except during a `merge_run` call) and in such a scope it
-#   will be returned by calls to `get_replica_context()`.  Implementers of new
-#   DistributionStrategy descendants will frequently also need to
-#   define a descendant of ReplicaContext, and are responsible for
-#   entering and exiting this context.
-#
-# * DistributionStrategy.scope() sets up a variable_creator scope that
-#   changes variable creation calls (e.g. to make mirrored
-#   variables). This is intended as an outer scope that users enter once
-#   around their model creation and graph definition. There is no
-#   anticipated need to define descendants of _CurrentDistributionContext.
-#   It sets the current DistributionStrategy for purposes of
-#   `get_strategy()` and `has_strategy()`
-#   and switches the thread mode to a "cross-replica context".
-@tf_export("distribute.ReplicaContext")
-class ReplicaContext(object):
-  """`tf.distribute.Strategy` API when in a replica context.
-
-  To be used inside your replicated step function, such as in a
-  `tf.distribute.StrategyExtended.call_for_each_replica` call.
-  """
-
-  def __init__(self, strategy, replica_id_in_sync_group):
-    self._distribution_strategy = strategy
-    self._thread_context = distribution_strategy_context._InReplicaThreadMode(  # pylint: disable=protected-access
-        self)
-    self._replica_id_in_sync_group = replica_id_in_sync_group
-
-  def __enter__(self):
-    _push_per_thread_mode(self._thread_context)
-
-  def __exit__(self, exception_type, exception_value, traceback):
-    _pop_per_thread_mode()
-
-  def merge_call(self, merge_fn, args=(), kwargs=None):
-    """Merge args across replicas and run `merge_fn` in a cross-replica context.
-
-    This allows communication and coordination when there are multiple calls
-    to a model function triggered by a call to
-    `strategy.extended.call_for_each_replica(model_fn, ...)`.
-
-    See `tf.distribute.StrategyExtended.call_for_each_replica` for an
-    explanation.
-
-    If not inside a distributed scope, this is equivalent to:
-
-    ```
-    strategy = tf.distribute.get_strategy()
-    with cross-replica-context(strategy):
-      return merge_fn(strategy, *args, **kwargs)
-    ```
-
-    Args:
-      merge_fn: function that joins arguments from threads that are given as
-        PerReplica. It accepts `tf.distribute.Strategy` object as
-        the first argument.
-      args: List or tuple with positional per-thread arguments for `merge_fn`.
-      kwargs: Dict with keyword per-thread arguments for `merge_fn`.
-
-    Returns:
-      The return value of `merge_fn`, except for `PerReplica` values which are
-      unpacked.
-    """
-    require_replica_context(self)
-    if kwargs is None:
-      kwargs = {}
-    return self._merge_call(merge_fn, args, kwargs)
-
-  def _merge_call(self, merge_fn, args, kwargs):
-    """Default implementation for single replica."""
-    _push_per_thread_mode(  # thread-local, so not needed with multiple threads
-        distribution_strategy_context._CrossReplicaThreadMode(  # pylint: disable=protected-access
-            self._distribution_strategy))
-    try:
-      return merge_fn(self._distribution_strategy, *args, **kwargs)
-    finally:
-      _pop_per_thread_mode()
-
-  @property
-  def num_replicas_in_sync(self):
-    """Returns number of replicas over which gradients are aggregated."""
-    return self._distribution_strategy.num_replicas_in_sync
-
-  @property
-  def replica_id_in_sync_group(self):
-    """Which replica is being defined, from 0 to `num_replicas_in_sync - 1`."""
-    require_replica_context(self)
-    return self._replica_id_in_sync_group
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, use `strategy`
-  def distribution_strategy(self):
-    """DEPRECATED: use `self.stratgey` instead."""
-    return self._distribution_strategy
-
-  @property
-  def strategy(self):
-    """The current `tf.distribute.Strategy` object."""
-    return self._distribution_strategy
-
-  @property
-  def devices(self):
-    """The devices this replica is to be executed on, as a list of strings."""
-    require_replica_context(self)
-    return [device_util.current()]
-
-  # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient
-  # all-reduce. It would return a function returning the result of reducing `t`
-  # across all replicas. The caller would wait to call this function until they
-  # needed the reduce result, allowing an efficient implementation:
-  # * With eager execution, the reduction could be performed asynchronously
-  #   in the background, not blocking until the result was needed.
-  # * When constructing a graph, it could batch up all reduction requests up
-  #   to that point that the first result is needed. Most likely this can be
-  #   implemented in terms of `merge_call()` and `batch_reduce_to()`.
-
-# ------------------------------------------------------------------------------
-
-
-class _DefaultDistributionStrategy(DistributionStrategy):
-  """Default `tf.distribute.Strategy` if none is explicitly selected."""
-
-  def __init__(self):
-    super(_DefaultDistributionStrategy, self).__init__(
-        _DefaultDistributionExtended(self))
-
-
-class _DefaultDistributionExtended(DistributionStrategyExtended):
-  """Implementation of _DefaultDistributionStrategy."""
-
-  def _scope(self, strategy):
-    """Context manager setting a variable creator and `self` as current."""
-    if distribution_strategy_context.has_distribution_strategy():
-      raise RuntimeError("Must not nest tf.distribute.Strategy scopes.")
-
-    def creator(next_creator, *args, **kwargs):
-      _require_distribution_strategy_scope_strategy(strategy)
-      return next_creator(*args, **kwargs)
-
-    return _CurrentDistributionContext(
-        strategy, variable_scope.variable_creator_scope(creator))
-
-  def colocate_vars_with(self, colocate_with_variable):
-    """Does not require `self.scope`."""
-    _require_distribution_strategy_scope_extended(self)
-    return ops.colocate_with(colocate_with_variable)
-
-  def _distribute_dataset(self, dataset_fn):
-    return self._call_dataset_fn(dataset_fn)
-
-  def _make_dataset_iterator(self, dataset):
-    return _DefaultDistributionExtended.DefaultInputIterator(dataset)
-
-  def _make_input_fn_iterator(self,
-                              input_fn,
-                              replication_mode=InputReplicationMode.PER_WORKER):
-    return input_fn(InputContext()).make_initializable_iterator()
-
-  def _broadcast_to(self, tensor, destinations):
-    if destinations is None:
-      return tensor
-    else:
-      raise NotImplementedError("TODO")
-
-  def _call_for_each_replica(self, fn, args, kwargs):
-    with ReplicaContext(
-        self._container_strategy(),
-        replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
-      return fn(*args, **kwargs)
-
-  def _reduce_to(self, reduce_op, value, destinations):
-    # TODO(josh11b): Use destinations?
-    del reduce_op, destinations
-    return value
-
-  def _update(self, var, fn, args, kwargs, group):
-    # The implementations of _update() and _update_non_slot() are identical
-    # except _update() passes `var` as the first argument to `fn()`.
-    return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
-
-  def _update_non_slot(self, colocate_with, fn, args, kwargs, should_group):
-    # TODO(josh11b): Figure out what we should be passing to UpdateContext()
-    # once that value is used for something.
-    with ops.colocate_with(colocate_with), UpdateContext(colocate_with):
-      result = fn(*args, **kwargs)
-      if should_group:
-        return result
-      else:
-        return nest.map_structure(self._unwrap, result)
-
-  def read_var(self, replica_local_var):
-    return array_ops.identity(replica_local_var)
-
-  def _unwrap(self, distributed_value):
-    return [distributed_value]
-
-  def value_container(self, value):
-    return value
-
-  @property
-  def _num_replicas_in_sync(self):
-    return 1
-
-  @property
-  def worker_devices(self):
-    raise RuntimeError("worker_devices() method unsupported by default "
-                       "tf.distribute.Strategy.")
-
-  @property
-  def parameter_devices(self):
-    raise RuntimeError("parameter_devices() method unsupported by default "
-                       "tf.distribute.Strategy.")
-
-  def non_slot_devices(self, var_list):
-    return min(var_list, key=lambda x: x.name)
-
-  # TODO(priyag): This should inherit from `InputIterator`, once dependency
-  # issues have been resolved.
-  class DefaultInputIterator(object):
-    """Default implementation of `InputIterator` for default strategy."""
-
-    def __init__(self, dataset):
-      self._dataset = dataset
-      if eager_context.executing_eagerly():
-        self._iterator = dataset.make_one_shot_iterator()
-      else:
-        self._iterator = dataset.make_initializable_iterator()
-
-    def get_next(self):
-      return self._iterator.get_next()
-
-    def initialize(self):
-      if eager_context.executing_eagerly():
-        self._iterator = self._dataset.make_one_shot_iterator()
-        return []
-      else:
-        return [self._iterator.initializer]
-
-  # TODO(priyag): Delete this once all strategies use global batch size.
-  @property
-  def _global_batch_size(self):
-    return True
-
-
-# ------------------------------------------------------------------------------
-# We haven't yet implemented deserialization for DistributedVariables.
-# So here we catch any attempts to deserialize variables
-# when using distribution strategies.
-# pylint: disable=protected-access
-_original_from_proto = resource_variable_ops._from_proto_fn
-
-
-def _from_proto_fn(v, import_scope=None):
-  if distribution_strategy_context.has_distribution_strategy():
-    raise NotImplementedError(
-        "Deserialization of variables is not yet supported when using a "
-        "tf.distribute.Strategy.")
-  else:
-    return _original_from_proto(v, import_scope=import_scope)
-
-resource_variable_ops._from_proto_fn = _from_proto_fn
-# pylint: enable=protected-access
-
-
-#-------------------------------------------------------------------------------
-# Shorthand for some methods from distribution_strategy_context.
-_push_per_thread_mode = distribution_strategy_context._push_per_thread_mode  # pylint: disable=protected-access
-_get_per_thread_mode = distribution_strategy_context._get_per_thread_mode  # pylint: disable=protected-access
-_pop_per_thread_mode = distribution_strategy_context._pop_per_thread_mode  # pylint: disable=protected-access
+# pylint: disable=wildcard-import
+from tensorflow.python.distribute.distribute_lib import *
diff --git a/tensorflow/python/training/distribution_strategy_context.py b/tensorflow/python/training/distribution_strategy_context.py
index 0b3878de18..7391bf3b22 100644
--- a/tensorflow/python/training/distribution_strategy_context.py
+++ b/tensorflow/python/training/distribution_strategy_context.py
@@ -12,225 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utility to get distribution strategy related contexts."""
+"""Deprecated, please use ../distribute/distribution_strategy_context.py."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import ops
-from tensorflow.python.util.lazy_loader import LazyLoader
-from tensorflow.python.util.tf_export import tf_export
-
-
-# There is a circular dependency between this and `distribute` module. So we
-# load it lazily to workaround this.
-distribute_lib = LazyLoader(
-    "distribute_lib", globals(),
-    "tensorflow.python.training.distribute")
-
-# ------------------------------------------------------------------------------
-# Internal API for setting the current thread mode as being either in a
-# replica or cross-replica context for a particular distribution strategy.
-
-
-class _ThreadMode(object):
-
-  def __init__(self, dist, cross, replica):
-    self.distribution_strategy = dist
-    self.cross_replica_context = cross
-    self.replica_context = replica
-
-
-class _CrossReplicaThreadMode(_ThreadMode):
-
-  def __init__(self, distribution_strategy):
-    _ThreadMode.__init__(
-        self, distribution_strategy, distribution_strategy, None)
-
-
-class _InReplicaThreadMode(_ThreadMode):
-
-  def __init__(self, replica_ctx):
-    _ThreadMode.__init__(
-        self, replica_ctx.distribution_strategy, None, replica_ctx)
-
-
-def _push_per_thread_mode(context):
-  ops.get_default_graph()._distribution_strategy_stack.append(context)  # pylint: disable=protected-access
-
-
-def _pop_per_thread_mode():
-  ops.get_default_graph()._distribution_strategy_stack.pop(-1)  # pylint: disable=protected-access
-
-
-class _DefaultReplicaThreadMode(_ThreadMode):
-  """Type of default value returned by `_get_per_thread_mode()`.
-
-  Used when the thread-local stack is empty.
-  """
-
-  def __init__(self):
-    _ThreadMode.__init__(self, _get_default_distribution_strategy(), None,
-                         _get_default_replica_context())
-
-
-def _get_per_thread_mode():
-  try:
-    return ops.get_default_graph()._distribution_strategy_stack[-1]  # pylint: disable=protected-access
-  except (AttributeError, IndexError):
-    return _get_default_replica_mode()
-
-
-# ------------------------------------------------------------------------------
-# Public API for accessing the current thread mode
-
-
-@tf_export("distribute.get_replica_context")
-def get_replica_context():
-  """Returns the current `tf.distribute.ReplicaContext` or `None`.
-
-  Returns `None` if in a cross-replica context.
-
-  Note that execution:
-
-  1. starts in the default (single-replica) replica context (this function
-     will return the default `ReplicaContext` object);
-  2. switches to cross-replica context (in which case this will return
-     `None`) when entering a `with tf.distribute.Strategy.scope():` block;
-  3. switches to a (non-default) replica context inside
-     `extended.call_for_each_replica(fn, ...)`;
-  4. if `fn` calls `get_replica_context().merge_call(merge_fn, ...)`, then
-     inside `merge_fn` you are back in the cross-replica context (and again
-     this function will return `None`).
-
-  Note that you can also go directly from step 1 to 4 to switch to a
-  cross-replica context for the default `tf.distribute.Strategy`. You may
-  also switch from the cross-replica context of 4 to a replica context by
-  calling `extended.call_for_each_replica()`, jumping back to step 3.
-
-  Most `tf.distribute.Strategy` methods may only be executed in
-  a cross-replica context, in a replica context you should use the
-  `ReplicaContext` API instead.
-
-  Returns:
-    The current `ReplicaContext` object when in a replica context scope,
-    else `None`.
-
-    Within a particular block, exactly one of these two things will be true:
-
-    * `get_replica_context()` returns non-`None`, or
-    * `tf.distribute.is_cross_replica_context()` returns True.
-  """
-  return _get_per_thread_mode().replica_context
-
-
-def get_cross_replica_context():
-  """Returns the current tf.distribute.Strategy if in a cross-replica context.
-
-  DEPRECATED: Please use `in_cross_replica_context()` and
-  `get_distribution_strategy()` instead.
-
-  Note that execution:
-
-  1. starts in the default (single-replica) replica context;
-  2. switches to cross-replica context when entering a
-     `with tf.distribute.Strategy.scope():` block;
-  3. switches to a (non-default) replica context inside
-     `call_for_each_replica(fn, ...)`;
-  4. if `fn` calls `get_replica_context()->merge_call(merge_fn, ...)`, then
-     inside `merge_fn` you are back in the cross-replica context.
-
-  Note that you can also go directly from step 1 to 4 to switch to a
-  cross-replica context for the default `tf.distribute.Strategy`. You may
-  also switch from the cross-replica context of 4 to a replica context by
-  calling `call_for_each_replica()`, jumping back to step 3.
-
-  Most `tf.distribute.Strategy` methods may only be executed in
-  a cross-replica context.
-
-  Returns:
-    Returns the current `tf.distribute.Strategy` object in a cross-replica
-    context, or `None`.
-
-    Exactly one of `get_replica_context()` and `get_cross_replica_context()`
-    will return `None` in a particular block.
-  """
-  return _get_per_thread_mode().cross_replica_context
-
-
-@tf_export("distribute.in_cross_replica_context")
-def in_cross_replica_context():
-  """Returns True if in a cross-replica context.
-
-  See `tf.distribute.get_replica_context` for details.
-
-  Returns:
-    True if in a cross-replica context (`get_replica_context()` returns
-    `None`), or False if in a replica context (`get_replica_context()` returns
-    non-`None`).
-  """
-  return _get_per_thread_mode().cross_replica_context is not None
-
-
-@tf_export("distribute.get_strategy")
-def get_distribution_strategy():
-  """Returns the current `tf.distribute.Strategy` object.
-
-  Typically only used in a cross-replica context:
-
-  ```
-  if tf.distribute.in_cross_replica_context():
-    strategy = tf.distribute.get_strategy()
-    ...
-  ```
-
-  Returns:
-    A `tf.distribute.Strategy` object. Inside a
-    `with distribution_strategy.scope()` block, it returns
-    `distribution_strategy`, otherwise it returns the default
-    (single-replica) `tf.distribute.Strategy` object.
-  """
-  return _get_per_thread_mode().distribution_strategy
-
-
-@tf_export("distribute.has_strategy")
-def has_distribution_strategy():
-  """Return if there is a current non-default `tf.distribute.Strategy`.
-
-  Returns:
-    True if inside a `with strategy.scope():`.
-  """
-  return get_distribution_strategy() is not _get_default_distribution_strategy()
-
-
-# ------------------------------------------------------------------------------
-# Defaults that are used when no distribution strategy is explicitly created.
-# We create them lazily in a function so that we can workaround the circular
-# dependency on distribute_lib. See lazy loader at the top of this file.
-
-_defaults = {
-    "distribution_strategy": None,
-    "replica_context": None,
-    "replica_mode": None
-}
-
-
-def _get_default_distribution_strategy():
-  if _defaults["distribution_strategy"] is None:
-    _defaults["distribution_strategy"] = (
-        distribute_lib._DefaultDistributionStrategy())  # pylint: disable=protected-access
-  return _defaults["distribution_strategy"]
-
-
-def _get_default_replica_context():
-  if _defaults["replica_context"] is None:
-    _defaults["replica_context"] = distribute_lib.ReplicaContext(
-        _get_default_distribution_strategy(), replica_id_in_sync_group=0)
-  return _defaults["replica_context"]
-
-
-def _get_default_replica_mode():
-  if _defaults["replica_mode"] is None:
-    _defaults["replica_mode"] = _DefaultReplicaThreadMode()
-  return _defaults["replica_mode"]
+# pylint: disable=wildcard-import
+from tensorflow.python.distribute.distribution_strategy_context import *
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt
index c39ac5a20d..583cbc6654 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-input-context.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.InputContext"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.InputContext\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.InputContext\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "input_pipeline_id"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
index 3eda6c6036..df707e8920 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.ReplicaContext"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.ReplicaContext\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.ReplicaContext\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "devices"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
index 3b502b534b..77706e5713 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.StrategyExtended"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategyExtended\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategyExtended\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "experimental_between_graph"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index f0b0cd0d38..0fd9a3b42d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.Strategy"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategy\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "between_graph"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt
index c39ac5a20d..583cbc6654 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-input-context.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.InputContext"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.InputContext\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.InputContext\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "input_pipeline_id"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
index 3eda6c6036..df707e8920 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.ReplicaContext"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.ReplicaContext\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.ReplicaContext\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "devices"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
index 3b502b534b..77706e5713 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.StrategyExtended"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategyExtended\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategyExtended\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "experimental_between_graph"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index f0b0cd0d38..0fd9a3b42d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.distribute.Strategy"
 tf_class {
-  is_instance: "<class \'tensorflow.python.training.distribute.DistributionStrategy\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "between_graph"
-- 
GitLab


From 335a643d687026335c84fb83b2c0f8dd91fd2b19 Mon Sep 17 00:00:00 2001
From: Roy Frostig <frostig@google.com>
Date: Sun, 25 Nov 2018 18:53:48 -0800
Subject: [PATCH 0756/1554] [XLA] Expose the shape of XRT allocations in the
 Python XLA library. Also, pass in UTF-8 strings in XRT methods for py3
 compatibility.

PiperOrigin-RevId: 222763810
---
 .../xla/python/local_computation_builder.i    |  1 +
 tensorflow/compiler/xla/python/xla_client.py  | 20 +++++++++++++------
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index feabfdb889..03698d9ce8 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -992,6 +992,7 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::XrtAllocation;
 %unignore xla::swig::XrtAllocation::FromLiteral;
 %unignore xla::swig::XrtAllocation::ToLiteral;
+%unignore xla::swig::XrtAllocation::shape;
 %unignore xla::swig::XrtAllocationTuple;
 %unignore xla::swig::XrtAllocationTuple::Release;
 %unignore xla::swig::XrtAllocationTuple::size;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 995d2b64dc..5994e55387 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -78,6 +78,13 @@ def CurrentSourceInfoMetadata(op_type=None, op_name=None, skip_frames=1):
       source_line=lineno)
 
 
+def _maybe_encode_string(s):
+  if six.PY3:
+    return s.encode('utf-8')
+  else:
+    return s
+
+
 class PaddingType(enum.Enum):
   VALID = 1
   SAME = 2
@@ -228,7 +235,8 @@ class LocalBuffer(object):
     """Allocate and copy to XLA the given python value."""
     pyval = require_numpy_array_layout(pyval)
     if backend.backend_type == BackendType.XRT:
-      cbuf = c_api.XrtAllocation.FromLiteral(pyval, backend.target)
+      cbuf = c_api.XrtAllocation.FromLiteral(
+          pyval, _maybe_encode_string(backend.target))
     else:
       cbuf = c_api.LocalShapedBuffer.FromLiteral(pyval, None)
     return LocalBuffer(cbuf, backend)
@@ -248,8 +256,8 @@ class LocalBuffer(object):
     """Assuming a tuple buffer, unpack it into constituent tuple elements."""
     assert self.c_buffer is not None
     if self._backend.backend_type == BackendType.XRT:
-      result = c_api.DestructureXrtAllocationTuple(self.c_buffer,
-                                                   self._backend.target)
+      result = c_api.DestructureXrtAllocationTuple(
+          self.c_buffer, _maybe_encode_string(self._backend.target))
     else:
       result = c_api.DestructureLocalShapedBufferTuple(self.c_buffer)
     self.delete()
@@ -552,7 +560,8 @@ class LocalComputation(object):
     compile_options = compile_options or CompileOptions()
     compile_options.result_shape = result_shape
     if self._backend.backend_type == BackendType.XRT:
-      c = self.computation.CompileForXrt(argument_shapes, self._backend.target)
+      c = self.computation.CompileForXrt(
+          argument_shapes, _maybe_encode_string(self._backend.target))
     else:
       c = self.computation.Compile(argument_shapes, compile_options)
     return LocalComputation(c, is_compiled=True, backend=self._backend)
@@ -1388,8 +1397,7 @@ def initialize_platform_name(platform_name):
   Raises:
     A runtime exception if the XLA service has already been initialized.
   """
-  if six.PY3:
-    platform_name = platform_name.encode('utf-8')
+  platform_name = _maybe_encode_string(platform_name)
   c_api.InitializePlatformName(platform_name)
 
 
-- 
GitLab


From 6f9082773ea05e2be8f86ee97ada70889777b44a Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Sun, 25 Nov 2018 19:08:36 -0800
Subject: [PATCH 0757/1554] Fix parameter server strategy return None config.

PiperOrigin-RevId: 222764694
---
 .../contrib/distribute/python/collective_all_reduce_strategy.py | 2 +-
 .../contrib/distribute/python/parameter_server_strategy.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 74a0550e81..617a95f3c4 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -283,7 +283,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     rewrite_options.scoped_allocator_opts.enable_op.append("CollectiveReduce")
 
     if not self._cluster_spec:
-      return
+      return updated_config
 
     assert self._task_type
     assert self._task_id is not None
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 5615abb751..8c33d3a143 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -480,7 +480,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
 
     # The device filters prevent communication between workers.
     if self._task_type not in ["chief", "worker"]:
-      return
+      return updated_config
     del updated_config.device_filters[:]
     updated_config.device_filters.extend(
         ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
-- 
GitLab


From f6b80d5cd39d5764174a8e05f3b4f73d1ea70827 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 01:02:23 -0800
Subject: [PATCH 0758/1554] compat: Update forward compatibility horizon to
 2018-11-26

PiperOrigin-RevId: 222785233
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 216ab2c0ac..4e84b5ee48 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 25)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 26)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 38e94b3ee00738a2ad75663f6ad162e97abcdfcf Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Mon, 26 Nov 2018 18:45:15 +0900
Subject: [PATCH 0759/1554] fix typo

---
 tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 566701ec2a..e676323bf4 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -2409,7 +2409,7 @@ class ConvertPowStage : public ArithmeticOptimizerStage {
         ctx().graph_properties->GetInputProperties(node->name())[1];
     for (int i = 0; i < pow_props.shape().dim_size(); ++i) {
       if (pow_props.shape().dim(i).size() < 0) {
-        // skip if p is is not fully defined.
+        // skip if p is not fully defined.
         return Status::OK();
       }
     }
@@ -2461,7 +2461,7 @@ class ConvertPowStage : public ArithmeticOptimizerStage {
                  ShapesSymbolicallyEqual(value_props.shape(), output_shape)) {
         for (int i = 0; i < value_props.shape().dim_size(); ++i) {
           if (value_props.shape().dim(i).size() < 0) {
-            // skip if b is is not fully defined.
+            // skip if b is not fully defined.
             return Status::OK();
           }
         }
-- 
GitLab


From a055605ec5ac5cafdbae66aae5c96d0cedd0e55c Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Mon, 26 Nov 2018 18:46:01 +0900
Subject: [PATCH 0760/1554] fix typo

---
 tensorflow/compiler/xla/service/gpu/instruction_fusion.cc | 2 +-
 tensorflow/core/grappler/graph_analyzer/sig_node.h        | 2 +-
 tensorflow/lite/python/op_hint.py                         | 2 +-
 tensorflow/lite/toco/import_tensorflow.cc                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
index 43f43b50e4..985c0d411e 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
@@ -80,7 +80,7 @@ bool IsIEEEFloatingPointScalarConstant(const HloInstruction* constant) {
 // This function limits the maximum number of operands to a fusion.
 //
 // There's a cap on how many parameters we can pass to a CUDA kernel, but
-// exactly what that limit is is hazy, as it depends on (among other things) how
+// exactly what that limit is hazy, as it depends on (among other things) how
 // much GPU constant memory is in use for other purposes.
 //
 // Moreover, we don't even know at the point that we're running fusion how many
diff --git a/tensorflow/core/grappler/graph_analyzer/sig_node.h b/tensorflow/core/grappler/graph_analyzer/sig_node.h
index 45c0ed3162..66d290d88e 100644
--- a/tensorflow/core/grappler/graph_analyzer/sig_node.h
+++ b/tensorflow/core/grappler/graph_analyzer/sig_node.h
@@ -178,7 +178,7 @@ class SigNode {
   // computed.
   size_t GetTopoHash(int distance) const;
 
-  // The the hash value for the highest computed distance. It must be previously
+  // The hash value for the highest computed distance. It must be previously
   // computed.
   size_t GetHighTopoHash() const {
     CHECK(!topo_hash_.empty());
diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 3afce1baf2..718b23075d 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -403,7 +403,7 @@ class _LiteOperand(object):
       out_graphdef: A graphdef that is ready to have this input added.
 
     Returns:
-      The the output that the stub should use as an input for this operand.
+      The output that the stub should use as an input for this operand.
 
     Raises:
       RuntimeError: if the method is not implemented.
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 4c3a0717e7..c8d1e1bdc5 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1221,7 +1221,7 @@ void GetOutputNamesFromNodeDef(const NodeDef& node,
 void GetOutputTypesFromNodeDef(const NodeDef& node,
                                const tensorflow::OpDef& op_def,
                                TensorFlowUnsupportedOperator* op) {
-  // The the given type to the op, or clear the types if invalid.
+  // The given type to the op, or clear the types if invalid.
   auto add_type = [&node, op](tensorflow::DataType type) {
     if (type == tensorflow::DT_INVALID) {
       LOG(WARNING) << "Op node missing output type attribute: " << node.name();
-- 
GitLab


From a382d3e89188d9d8dc18ceccf842d1492bed0d1a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 06:51:35 -0800
Subject: [PATCH 0761/1554] Add cuda 10 config and update to latest cuda 10
 docker image.

PiperOrigin-RevId: 222817562
---
 .../preconfig/generate/containers.bzl         |    2 +-
 .../ubuntu14.04/cuda10.0-cudnn7/WORKSPACE     |    2 +
 .../ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD    | 1275 +++++++++++++++
 .../cuda10.0-cudnn7/cuda/build_defs.bzl       |   31 +
 .../cuda10.0-cudnn7/cuda/cuda/cuda_config.h   |   26 +
 .../ubuntu14.04/gcc-nvcc-cuda10.0/BUILD       |   87 +
 .../ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL   | 1431 +++++++++++++++++
 .../bin/crosstool_wrapper_driver_is_not_gcc   |  264 +++
 .../windows/msvc_wrapper_for_nvcc.bat         |   20 +
 .../windows/msvc_wrapper_for_nvcc.py          |  192 +++
 10 files changed, 3329 insertions(+), 1 deletion(-)
 create mode 100644 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/WORKSPACE
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/cuda/cuda_config.h
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.bat
 create mode 100755 third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py

diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 1f9e29d440..0309b8ffca 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,4 @@
 container_digests = {
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:c26138f4c38c754da2bad44a8a068523abf7fbd71d58a57ce92e5342c5431bf5",
-    "cuda10.0-cudnn7-ubuntu14.04": "sha256:34c4a55e2376b300cdc2b903775fc32e62352f6e33f927df5653743324378bfc",
+    "cuda10.0-cudnn7-ubuntu14.04": "sha256:7737d770599de8435115bfdf56977002319316a6735ab081f82506cb51443f9d",
 }
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/WORKSPACE b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/WORKSPACE
new file mode 100644
index 0000000000..b61f572d6d
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/WORKSPACE
@@ -0,0 +1,2 @@
+# DO NOT EDIT: automatically generated WORKSPACE file for cuda_configure rule
+workspace(name = "local_config_cuda")
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
new file mode 100755
index 0000000000..c813efccf9
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
@@ -0,0 +1,1275 @@
+licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
+
+package(default_visibility = ["//visibility:public"])
+
+config_setting(
+    name = "using_nvcc",
+    values = {
+        "define": "using_cuda_nvcc=true",
+    },
+)
+
+config_setting(
+    name = "using_clang",
+    values = {
+        "define": "using_cuda_clang=true",
+    },
+)
+
+# Equivalent to using_clang && -c opt.
+config_setting(
+    name = "using_clang_opt",
+    values = {
+        "define": "using_cuda_clang=true",
+        "compilation_mode": "opt",
+    },
+)
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "freebsd",
+    values = {"cpu": "freebsd"},
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cuda_headers",
+    hdrs = [
+        "cuda/cuda_config.h",
+        ":cuda-include",
+        ":cudnn-include",
+    ],
+    includes = [
+        ".",
+        "cuda/include",
+        "cuda/include/crt",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cudart_static",
+    srcs = ["cuda/lib/libcudart_static.a"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkopts = select({
+        ":freebsd": [],
+        "//conditions:default": ["-ldl"],
+    }) + [
+        "-lpthread",
+        "-lrt",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cuda_driver",
+    srcs = ["cuda/lib/libcuda.so"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cudart",
+    srcs = ["cuda/lib/libcudart.so.10.0"],
+    data = ["cuda/lib/libcudart.so.10.0"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cublas",
+    srcs = ["cuda/lib/libcublas.so.10.0"],
+    data = ["cuda/lib/libcublas.so.10.0"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cusolver",
+    srcs = ["cuda/lib/libcusolver.so.10.0"],
+    data = ["cuda/lib/libcusolver.so.10.0"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkopts = ["-lgomp"],
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cudnn",
+    srcs = ["cuda/lib/libcudnn.so.7"],
+    data = ["cuda/lib/libcudnn.so.7"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cudnn_header",
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cufft",
+    srcs = ["cuda/lib/libcufft.so.10.0"],
+    data = ["cuda/lib/libcufft.so.10.0"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "curand",
+    srcs = ["cuda/lib/libcurand.so.10.0"],
+    data = ["cuda/lib/libcurand.so.10.0"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    linkstatic = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cuda",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cublas",
+        ":cuda_headers",
+        ":cudart",
+        ":cudnn",
+        ":cufft",
+        ":curand",
+    ],
+)
+
+cc_library(
+    name = "cupti_headers",
+    hdrs = [
+        "cuda/cuda_config.h",
+        ":cuda-extras",
+    ],
+    includes = [
+        ".",
+        "cuda/extras/CUPTI/include/",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "cupti_dsos",
+    data = ["cuda/lib/libcupti.so.10.0"],
+    includes = [
+        ".",
+        "cuda/include",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "libdevice_root",
+    data = [":cuda-nvvm"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "cuda-include",
+    outs = [
+        "cuda/include/CL/cl.h",
+        "cuda/include/CL/cl.hpp",
+        "cuda/include/CL/cl_egl.h",
+        "cuda/include/CL/cl_ext.h",
+        "cuda/include/CL/cl_gl.h",
+        "cuda/include/CL/cl_gl_ext.h",
+        "cuda/include/CL/cl_platform.h",
+        "cuda/include/CL/opencl.h",
+        "cuda/include/builtin_types.h",
+        "cuda/include/channel_descriptor.h",
+        "cuda/include/common_functions.h",
+        "cuda/include/cooperative_groups.h",
+        "cuda/include/cooperative_groups_helpers.h",
+        "cuda/include/crt/common_functions.h",
+        "cuda/include/crt/device_double_functions.h",
+        "cuda/include/crt/device_double_functions.hpp",
+        "cuda/include/crt/device_functions.h",
+        "cuda/include/crt/device_functions.hpp",
+        "cuda/include/crt/func_macro.h",
+        "cuda/include/crt/host_config.h",
+        "cuda/include/crt/host_defines.h",
+        "cuda/include/crt/host_runtime.h",
+        "cuda/include/crt/math_functions.h",
+        "cuda/include/crt/math_functions.hpp",
+        "cuda/include/crt/mma.h",
+        "cuda/include/crt/mma.hpp",
+        "cuda/include/crt/nvfunctional",
+        "cuda/include/crt/sm_70_rt.h",
+        "cuda/include/crt/sm_70_rt.hpp",
+        "cuda/include/crt/storage_class.h",
+        "cuda/include/cuComplex.h",
+        "cuda/include/cublas.h",
+        "cuda/include/cublasXt.h",
+        "cuda/include/cublas_api.h",
+        "cuda/include/cublas_v2.h",
+        "cuda/include/cuda.h",
+        "cuda/include/cudaEGL.h",
+        "cuda/include/cudaGL.h",
+        "cuda/include/cudaProfiler.h",
+        "cuda/include/cudaVDPAU.h",
+        "cuda/include/cuda_device_runtime_api.h",
+        "cuda/include/cuda_egl_interop.h",
+        "cuda/include/cuda_fp16.h",
+        "cuda/include/cuda_fp16.hpp",
+        "cuda/include/cuda_gl_interop.h",
+        "cuda/include/cuda_occupancy.h",
+        "cuda/include/cuda_profiler_api.h",
+        "cuda/include/cuda_runtime.h",
+        "cuda/include/cuda_runtime_api.h",
+        "cuda/include/cuda_surface_types.h",
+        "cuda/include/cuda_texture_types.h",
+        "cuda/include/cuda_vdpau_interop.h",
+        "cuda/include/cudalibxt.h",
+        "cuda/include/cudart_platform.h",
+        "cuda/include/cufft.h",
+        "cuda/include/cufftXt.h",
+        "cuda/include/cufftw.h",
+        "cuda/include/curand.h",
+        "cuda/include/curand_discrete.h",
+        "cuda/include/curand_discrete2.h",
+        "cuda/include/curand_globals.h",
+        "cuda/include/curand_kernel.h",
+        "cuda/include/curand_lognormal.h",
+        "cuda/include/curand_mrg32k3a.h",
+        "cuda/include/curand_mtgp32.h",
+        "cuda/include/curand_mtgp32_host.h",
+        "cuda/include/curand_mtgp32_kernel.h",
+        "cuda/include/curand_mtgp32dc_p_11213.h",
+        "cuda/include/curand_normal.h",
+        "cuda/include/curand_normal_static.h",
+        "cuda/include/curand_philox4x32_x.h",
+        "cuda/include/curand_poisson.h",
+        "cuda/include/curand_precalc.h",
+        "cuda/include/curand_uniform.h",
+        "cuda/include/cusolverDn.h",
+        "cuda/include/cusolverRf.h",
+        "cuda/include/cusolverSp.h",
+        "cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h",
+        "cuda/include/cusolver_common.h",
+        "cuda/include/cusparse.h",
+        "cuda/include/cusparse_v2.h",
+        "cuda/include/device_atomic_functions.h",
+        "cuda/include/device_atomic_functions.hpp",
+        "cuda/include/device_double_functions.h",
+        "cuda/include/device_functions.h",
+        "cuda/include/device_launch_parameters.h",
+        "cuda/include/device_types.h",
+        "cuda/include/driver_functions.h",
+        "cuda/include/driver_types.h",
+        "cuda/include/fatBinaryCtl.h",
+        "cuda/include/fatbinary.h",
+        "cuda/include/host_config.h",
+        "cuda/include/host_defines.h",
+        "cuda/include/library_types.h",
+        "cuda/include/math_constants.h",
+        "cuda/include/math_functions.h",
+        "cuda/include/mma.h",
+        "cuda/include/npp.h",
+        "cuda/include/nppcore.h",
+        "cuda/include/nppdefs.h",
+        "cuda/include/nppi.h",
+        "cuda/include/nppi_arithmetic_and_logical_operations.h",
+        "cuda/include/nppi_color_conversion.h",
+        "cuda/include/nppi_compression_functions.h",
+        "cuda/include/nppi_computer_vision.h",
+        "cuda/include/nppi_data_exchange_and_initialization.h",
+        "cuda/include/nppi_filtering_functions.h",
+        "cuda/include/nppi_geometry_transforms.h",
+        "cuda/include/nppi_linear_transforms.h",
+        "cuda/include/nppi_morphological_operations.h",
+        "cuda/include/nppi_statistics_functions.h",
+        "cuda/include/nppi_support_functions.h",
+        "cuda/include/nppi_threshold_and_compare_operations.h",
+        "cuda/include/npps.h",
+        "cuda/include/npps_arithmetic_and_logical_operations.h",
+        "cuda/include/npps_conversion_functions.h",
+        "cuda/include/npps_filtering_functions.h",
+        "cuda/include/npps_initialization.h",
+        "cuda/include/npps_statistics_functions.h",
+        "cuda/include/npps_support_functions.h",
+        "cuda/include/nppversion.h",
+        "cuda/include/nvToolsExt.h",
+        "cuda/include/nvToolsExtCuda.h",
+        "cuda/include/nvToolsExtCudaRt.h",
+        "cuda/include/nvToolsExtMeta.h",
+        "cuda/include/nvToolsExtSync.h",
+        "cuda/include/nvblas.h",
+        "cuda/include/nvfunctional",
+        "cuda/include/nvgraph.h",
+        "cuda/include/nvjpeg.h",
+        "cuda/include/nvml.h",
+        "cuda/include/nvrtc.h",
+        "cuda/include/nvtx3/nvToolsExt.h",
+        "cuda/include/nvtx3/nvToolsExtCuda.h",
+        "cuda/include/nvtx3/nvToolsExtCudaRt.h",
+        "cuda/include/nvtx3/nvToolsExtOpenCL.h",
+        "cuda/include/nvtx3/nvToolsExtSync.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxImpl.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxImplCore.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxInit.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxInitDecls.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxInitDefs.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxLinkOnce.h",
+        "cuda/include/nvtx3/nvtxDetail/nvtxTypes.h",
+        "cuda/include/sm_20_atomic_functions.h",
+        "cuda/include/sm_20_atomic_functions.hpp",
+        "cuda/include/sm_20_intrinsics.h",
+        "cuda/include/sm_20_intrinsics.hpp",
+        "cuda/include/sm_30_intrinsics.h",
+        "cuda/include/sm_30_intrinsics.hpp",
+        "cuda/include/sm_32_atomic_functions.h",
+        "cuda/include/sm_32_atomic_functions.hpp",
+        "cuda/include/sm_32_intrinsics.h",
+        "cuda/include/sm_32_intrinsics.hpp",
+        "cuda/include/sm_35_atomic_functions.h",
+        "cuda/include/sm_35_intrinsics.h",
+        "cuda/include/sm_60_atomic_functions.h",
+        "cuda/include/sm_60_atomic_functions.hpp",
+        "cuda/include/sm_61_intrinsics.h",
+        "cuda/include/sm_61_intrinsics.hpp",
+        "cuda/include/sobol_direction_vectors.h",
+        "cuda/include/surface_functions.h",
+        "cuda/include/surface_functions.hpp",
+        "cuda/include/surface_indirect_functions.h",
+        "cuda/include/surface_indirect_functions.hpp",
+        "cuda/include/surface_types.h",
+        "cuda/include/texture_fetch_functions.h",
+        "cuda/include/texture_fetch_functions.hpp",
+        "cuda/include/texture_indirect_functions.h",
+        "cuda/include/texture_indirect_functions.hpp",
+        "cuda/include/texture_types.h",
+        "cuda/include/thrust/adjacent_difference.h",
+        "cuda/include/thrust/advance.h",
+        "cuda/include/thrust/binary_search.h",
+        "cuda/include/thrust/complex.h",
+        "cuda/include/thrust/copy.h",
+        "cuda/include/thrust/count.h",
+        "cuda/include/thrust/detail/adjacent_difference.inl",
+        "cuda/include/thrust/detail/advance.inl",
+        "cuda/include/thrust/detail/alignment.h",
+        "cuda/include/thrust/detail/allocator/allocator_traits.h",
+        "cuda/include/thrust/detail/allocator/allocator_traits.inl",
+        "cuda/include/thrust/detail/allocator/copy_construct_range.h",
+        "cuda/include/thrust/detail/allocator/copy_construct_range.inl",
+        "cuda/include/thrust/detail/allocator/default_construct_range.h",
+        "cuda/include/thrust/detail/allocator/default_construct_range.inl",
+        "cuda/include/thrust/detail/allocator/destroy_range.h",
+        "cuda/include/thrust/detail/allocator/destroy_range.inl",
+        "cuda/include/thrust/detail/allocator/fill_construct_range.h",
+        "cuda/include/thrust/detail/allocator/fill_construct_range.inl",
+        "cuda/include/thrust/detail/allocator/malloc_allocator.h",
+        "cuda/include/thrust/detail/allocator/malloc_allocator.inl",
+        "cuda/include/thrust/detail/allocator/no_throw_allocator.h",
+        "cuda/include/thrust/detail/allocator/tagged_allocator.h",
+        "cuda/include/thrust/detail/allocator/tagged_allocator.inl",
+        "cuda/include/thrust/detail/allocator/temporary_allocator.h",
+        "cuda/include/thrust/detail/allocator/temporary_allocator.inl",
+        "cuda/include/thrust/detail/binary_search.inl",
+        "cuda/include/thrust/detail/complex/arithmetic.h",
+        "cuda/include/thrust/detail/complex/c99math.h",
+        "cuda/include/thrust/detail/complex/catrig.h",
+        "cuda/include/thrust/detail/complex/catrigf.h",
+        "cuda/include/thrust/detail/complex/ccosh.h",
+        "cuda/include/thrust/detail/complex/ccoshf.h",
+        "cuda/include/thrust/detail/complex/cexp.h",
+        "cuda/include/thrust/detail/complex/cexpf.h",
+        "cuda/include/thrust/detail/complex/clog.h",
+        "cuda/include/thrust/detail/complex/clogf.h",
+        "cuda/include/thrust/detail/complex/complex.inl",
+        "cuda/include/thrust/detail/complex/cpow.h",
+        "cuda/include/thrust/detail/complex/cproj.h",
+        "cuda/include/thrust/detail/complex/csinh.h",
+        "cuda/include/thrust/detail/complex/csinhf.h",
+        "cuda/include/thrust/detail/complex/csqrt.h",
+        "cuda/include/thrust/detail/complex/csqrtf.h",
+        "cuda/include/thrust/detail/complex/ctanh.h",
+        "cuda/include/thrust/detail/complex/ctanhf.h",
+        "cuda/include/thrust/detail/complex/math_private.h",
+        "cuda/include/thrust/detail/complex/stream.h",
+        "cuda/include/thrust/detail/config.h",
+        "cuda/include/thrust/detail/config/compiler.h",
+        "cuda/include/thrust/detail/config/compiler_fence.h",
+        "cuda/include/thrust/detail/config/config.h",
+        "cuda/include/thrust/detail/config/debug.h",
+        "cuda/include/thrust/detail/config/device_system.h",
+        "cuda/include/thrust/detail/config/exec_check_disable.h",
+        "cuda/include/thrust/detail/config/forceinline.h",
+        "cuda/include/thrust/detail/config/global_workarounds.h",
+        "cuda/include/thrust/detail/config/host_device.h",
+        "cuda/include/thrust/detail/config/host_system.h",
+        "cuda/include/thrust/detail/config/simple_defines.h",
+        "cuda/include/thrust/detail/contiguous_storage.h",
+        "cuda/include/thrust/detail/contiguous_storage.inl",
+        "cuda/include/thrust/detail/copy.h",
+        "cuda/include/thrust/detail/copy.inl",
+        "cuda/include/thrust/detail/copy_if.h",
+        "cuda/include/thrust/detail/copy_if.inl",
+        "cuda/include/thrust/detail/count.inl",
+        "cuda/include/thrust/detail/cstdint.h",
+        "cuda/include/thrust/detail/device_delete.inl",
+        "cuda/include/thrust/detail/device_free.inl",
+        "cuda/include/thrust/detail/device_malloc.inl",
+        "cuda/include/thrust/detail/device_new.inl",
+        "cuda/include/thrust/detail/device_ptr.inl",
+        "cuda/include/thrust/detail/device_reference.inl",
+        "cuda/include/thrust/detail/device_vector.inl",
+        "cuda/include/thrust/detail/dispatch/is_trivial_copy.h",
+        "cuda/include/thrust/detail/distance.inl",
+        "cuda/include/thrust/detail/equal.inl",
+        "cuda/include/thrust/detail/execute_with_allocator.h",
+        "cuda/include/thrust/detail/execution_policy.h",
+        "cuda/include/thrust/detail/extrema.inl",
+        "cuda/include/thrust/detail/fill.inl",
+        "cuda/include/thrust/detail/find.inl",
+        "cuda/include/thrust/detail/for_each.inl",
+        "cuda/include/thrust/detail/function.h",
+        "cuda/include/thrust/detail/functional.inl",
+        "cuda/include/thrust/detail/functional/actor.h",
+        "cuda/include/thrust/detail/functional/actor.inl",
+        "cuda/include/thrust/detail/functional/argument.h",
+        "cuda/include/thrust/detail/functional/composite.h",
+        "cuda/include/thrust/detail/functional/operators.h",
+        "cuda/include/thrust/detail/functional/operators/arithmetic_operators.h",
+        "cuda/include/thrust/detail/functional/operators/assignment_operator.h",
+        "cuda/include/thrust/detail/functional/operators/bitwise_operators.h",
+        "cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h",
+        "cuda/include/thrust/detail/functional/operators/logical_operators.h",
+        "cuda/include/thrust/detail/functional/operators/operator_adaptors.h",
+        "cuda/include/thrust/detail/functional/operators/relational_operators.h",
+        "cuda/include/thrust/detail/functional/placeholder.h",
+        "cuda/include/thrust/detail/functional/value.h",
+        "cuda/include/thrust/detail/gather.inl",
+        "cuda/include/thrust/detail/generate.inl",
+        "cuda/include/thrust/detail/get_iterator_value.h",
+        "cuda/include/thrust/detail/host_vector.inl",
+        "cuda/include/thrust/detail/inner_product.inl",
+        "cuda/include/thrust/detail/integer_math.h",
+        "cuda/include/thrust/detail/integer_traits.h",
+        "cuda/include/thrust/detail/internal_functional.h",
+        "cuda/include/thrust/detail/logical.inl",
+        "cuda/include/thrust/detail/malloc_and_free.h",
+        "cuda/include/thrust/detail/merge.inl",
+        "cuda/include/thrust/detail/minmax.h",
+        "cuda/include/thrust/detail/mismatch.inl",
+        "cuda/include/thrust/detail/mpl/math.h",
+        "cuda/include/thrust/detail/numeric_traits.h",
+        "cuda/include/thrust/detail/overlapped_copy.h",
+        "cuda/include/thrust/detail/pair.inl",
+        "cuda/include/thrust/detail/partition.inl",
+        "cuda/include/thrust/detail/pointer.h",
+        "cuda/include/thrust/detail/pointer.inl",
+        "cuda/include/thrust/detail/preprocessor.h",
+        "cuda/include/thrust/detail/range/head_flags.h",
+        "cuda/include/thrust/detail/range/tail_flags.h",
+        "cuda/include/thrust/detail/raw_pointer_cast.h",
+        "cuda/include/thrust/detail/raw_reference_cast.h",
+        "cuda/include/thrust/detail/reduce.inl",
+        "cuda/include/thrust/detail/reference.h",
+        "cuda/include/thrust/detail/reference.inl",
+        "cuda/include/thrust/detail/reference_forward_declaration.h",
+        "cuda/include/thrust/detail/remove.inl",
+        "cuda/include/thrust/detail/replace.inl",
+        "cuda/include/thrust/detail/reverse.inl",
+        "cuda/include/thrust/detail/scan.inl",
+        "cuda/include/thrust/detail/scatter.inl",
+        "cuda/include/thrust/detail/seq.h",
+        "cuda/include/thrust/detail/sequence.inl",
+        "cuda/include/thrust/detail/set_operations.inl",
+        "cuda/include/thrust/detail/sort.inl",
+        "cuda/include/thrust/detail/static_assert.h",
+        "cuda/include/thrust/detail/static_map.h",
+        "cuda/include/thrust/detail/swap.h",
+        "cuda/include/thrust/detail/swap.inl",
+        "cuda/include/thrust/detail/swap_ranges.inl",
+        "cuda/include/thrust/detail/tabulate.inl",
+        "cuda/include/thrust/detail/temporary_array.h",
+        "cuda/include/thrust/detail/temporary_array.inl",
+        "cuda/include/thrust/detail/temporary_buffer.h",
+        "cuda/include/thrust/detail/transform.inl",
+        "cuda/include/thrust/detail/transform_reduce.inl",
+        "cuda/include/thrust/detail/transform_scan.inl",
+        "cuda/include/thrust/detail/trivial_sequence.h",
+        "cuda/include/thrust/detail/tuple.inl",
+        "cuda/include/thrust/detail/tuple_meta_transform.h",
+        "cuda/include/thrust/detail/tuple_transform.h",
+        "cuda/include/thrust/detail/type_traits.h",
+        "cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h",
+        "cuda/include/thrust/detail/type_traits/function_traits.h",
+        "cuda/include/thrust/detail/type_traits/has_member_function.h",
+        "cuda/include/thrust/detail/type_traits/has_nested_type.h",
+        "cuda/include/thrust/detail/type_traits/has_trivial_assign.h",
+        "cuda/include/thrust/detail/type_traits/is_call_possible.h",
+        "cuda/include/thrust/detail/type_traits/is_metafunction_defined.h",
+        "cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h",
+        "cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h",
+        "cuda/include/thrust/detail/type_traits/minimum_type.h",
+        "cuda/include/thrust/detail/type_traits/pointer_traits.h",
+        "cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h",
+        "cuda/include/thrust/detail/uninitialized_copy.inl",
+        "cuda/include/thrust/detail/uninitialized_fill.inl",
+        "cuda/include/thrust/detail/unique.inl",
+        "cuda/include/thrust/detail/use_default.h",
+        "cuda/include/thrust/detail/util/align.h",
+        "cuda/include/thrust/detail/util/blocking.h",
+        "cuda/include/thrust/detail/vector_base.h",
+        "cuda/include/thrust/detail/vector_base.inl",
+        "cuda/include/thrust/device_allocator.h",
+        "cuda/include/thrust/device_delete.h",
+        "cuda/include/thrust/device_free.h",
+        "cuda/include/thrust/device_malloc.h",
+        "cuda/include/thrust/device_malloc_allocator.h",
+        "cuda/include/thrust/device_new.h",
+        "cuda/include/thrust/device_new_allocator.h",
+        "cuda/include/thrust/device_ptr.h",
+        "cuda/include/thrust/device_reference.h",
+        "cuda/include/thrust/device_vector.h",
+        "cuda/include/thrust/distance.h",
+        "cuda/include/thrust/equal.h",
+        "cuda/include/thrust/execution_policy.h",
+        "cuda/include/thrust/extrema.h",
+        "cuda/include/thrust/fill.h",
+        "cuda/include/thrust/find.h",
+        "cuda/include/thrust/for_each.h",
+        "cuda/include/thrust/functional.h",
+        "cuda/include/thrust/gather.h",
+        "cuda/include/thrust/generate.h",
+        "cuda/include/thrust/host_vector.h",
+        "cuda/include/thrust/inner_product.h",
+        "cuda/include/thrust/iterator/constant_iterator.h",
+        "cuda/include/thrust/iterator/counting_iterator.h",
+        "cuda/include/thrust/iterator/detail/any_assign.h",
+        "cuda/include/thrust/iterator/detail/any_system_tag.h",
+        "cuda/include/thrust/iterator/detail/constant_iterator_base.h",
+        "cuda/include/thrust/iterator/detail/counting_iterator.inl",
+        "cuda/include/thrust/iterator/detail/device_system_tag.h",
+        "cuda/include/thrust/iterator/detail/discard_iterator_base.h",
+        "cuda/include/thrust/iterator/detail/distance_from_result.h",
+        "cuda/include/thrust/iterator/detail/host_system_tag.h",
+        "cuda/include/thrust/iterator/detail/is_iterator_category.h",
+        "cuda/include/thrust/iterator/detail/is_trivial_iterator.h",
+        "cuda/include/thrust/iterator/detail/iterator_adaptor_base.h",
+        "cuda/include/thrust/iterator/detail/iterator_category_to_system.h",
+        "cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h",
+        "cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h",
+        "cuda/include/thrust/iterator/detail/iterator_facade_category.h",
+        "cuda/include/thrust/iterator/detail/iterator_traits.inl",
+        "cuda/include/thrust/iterator/detail/iterator_traversal_tags.h",
+        "cuda/include/thrust/iterator/detail/join_iterator.h",
+        "cuda/include/thrust/iterator/detail/minimum_category.h",
+        "cuda/include/thrust/iterator/detail/minimum_system.h",
+        "cuda/include/thrust/iterator/detail/normal_iterator.h",
+        "cuda/include/thrust/iterator/detail/permutation_iterator_base.h",
+        "cuda/include/thrust/iterator/detail/retag.h",
+        "cuda/include/thrust/iterator/detail/reverse_iterator.inl",
+        "cuda/include/thrust/iterator/detail/reverse_iterator_base.h",
+        "cuda/include/thrust/iterator/detail/tagged_iterator.h",
+        "cuda/include/thrust/iterator/detail/transform_iterator.inl",
+        "cuda/include/thrust/iterator/detail/transform_output_iterator.inl",
+        "cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h",
+        "cuda/include/thrust/iterator/detail/universal_categories.h",
+        "cuda/include/thrust/iterator/detail/zip_iterator.inl",
+        "cuda/include/thrust/iterator/detail/zip_iterator_base.h",
+        "cuda/include/thrust/iterator/discard_iterator.h",
+        "cuda/include/thrust/iterator/iterator_adaptor.h",
+        "cuda/include/thrust/iterator/iterator_categories.h",
+        "cuda/include/thrust/iterator/iterator_facade.h",
+        "cuda/include/thrust/iterator/iterator_traits.h",
+        "cuda/include/thrust/iterator/permutation_iterator.h",
+        "cuda/include/thrust/iterator/retag.h",
+        "cuda/include/thrust/iterator/reverse_iterator.h",
+        "cuda/include/thrust/iterator/transform_iterator.h",
+        "cuda/include/thrust/iterator/transform_output_iterator.h",
+        "cuda/include/thrust/iterator/zip_iterator.h",
+        "cuda/include/thrust/logical.h",
+        "cuda/include/thrust/memory.h",
+        "cuda/include/thrust/merge.h",
+        "cuda/include/thrust/mismatch.h",
+        "cuda/include/thrust/pair.h",
+        "cuda/include/thrust/partition.h",
+        "cuda/include/thrust/random.h",
+        "cuda/include/thrust/random/detail/discard_block_engine.inl",
+        "cuda/include/thrust/random/detail/linear_congruential_engine.inl",
+        "cuda/include/thrust/random/detail/linear_congruential_engine_discard.h",
+        "cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl",
+        "cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h",
+        "cuda/include/thrust/random/detail/mod.h",
+        "cuda/include/thrust/random/detail/normal_distribution.inl",
+        "cuda/include/thrust/random/detail/normal_distribution_base.h",
+        "cuda/include/thrust/random/detail/random_core_access.h",
+        "cuda/include/thrust/random/detail/subtract_with_carry_engine.inl",
+        "cuda/include/thrust/random/detail/uniform_int_distribution.inl",
+        "cuda/include/thrust/random/detail/uniform_real_distribution.inl",
+        "cuda/include/thrust/random/detail/xor_combine_engine.inl",
+        "cuda/include/thrust/random/detail/xor_combine_engine_max.h",
+        "cuda/include/thrust/random/discard_block_engine.h",
+        "cuda/include/thrust/random/linear_congruential_engine.h",
+        "cuda/include/thrust/random/linear_feedback_shift_engine.h",
+        "cuda/include/thrust/random/normal_distribution.h",
+        "cuda/include/thrust/random/subtract_with_carry_engine.h",
+        "cuda/include/thrust/random/uniform_int_distribution.h",
+        "cuda/include/thrust/random/uniform_real_distribution.h",
+        "cuda/include/thrust/random/xor_combine_engine.h",
+        "cuda/include/thrust/reduce.h",
+        "cuda/include/thrust/remove.h",
+        "cuda/include/thrust/replace.h",
+        "cuda/include/thrust/reverse.h",
+        "cuda/include/thrust/scan.h",
+        "cuda/include/thrust/scatter.h",
+        "cuda/include/thrust/sequence.h",
+        "cuda/include/thrust/set_operations.h",
+        "cuda/include/thrust/sort.h",
+        "cuda/include/thrust/swap.h",
+        "cuda/include/thrust/system/cpp/detail/adjacent_difference.h",
+        "cuda/include/thrust/system/cpp/detail/assign_value.h",
+        "cuda/include/thrust/system/cpp/detail/binary_search.h",
+        "cuda/include/thrust/system/cpp/detail/copy.h",
+        "cuda/include/thrust/system/cpp/detail/copy_if.h",
+        "cuda/include/thrust/system/cpp/detail/count.h",
+        "cuda/include/thrust/system/cpp/detail/equal.h",
+        "cuda/include/thrust/system/cpp/detail/execution_policy.h",
+        "cuda/include/thrust/system/cpp/detail/extrema.h",
+        "cuda/include/thrust/system/cpp/detail/fill.h",
+        "cuda/include/thrust/system/cpp/detail/find.h",
+        "cuda/include/thrust/system/cpp/detail/for_each.h",
+        "cuda/include/thrust/system/cpp/detail/gather.h",
+        "cuda/include/thrust/system/cpp/detail/generate.h",
+        "cuda/include/thrust/system/cpp/detail/get_value.h",
+        "cuda/include/thrust/system/cpp/detail/inner_product.h",
+        "cuda/include/thrust/system/cpp/detail/iter_swap.h",
+        "cuda/include/thrust/system/cpp/detail/logical.h",
+        "cuda/include/thrust/system/cpp/detail/malloc_and_free.h",
+        "cuda/include/thrust/system/cpp/detail/memory.inl",
+        "cuda/include/thrust/system/cpp/detail/merge.h",
+        "cuda/include/thrust/system/cpp/detail/mismatch.h",
+        "cuda/include/thrust/system/cpp/detail/par.h",
+        "cuda/include/thrust/system/cpp/detail/partition.h",
+        "cuda/include/thrust/system/cpp/detail/reduce.h",
+        "cuda/include/thrust/system/cpp/detail/reduce_by_key.h",
+        "cuda/include/thrust/system/cpp/detail/remove.h",
+        "cuda/include/thrust/system/cpp/detail/replace.h",
+        "cuda/include/thrust/system/cpp/detail/reverse.h",
+        "cuda/include/thrust/system/cpp/detail/scan.h",
+        "cuda/include/thrust/system/cpp/detail/scan_by_key.h",
+        "cuda/include/thrust/system/cpp/detail/scatter.h",
+        "cuda/include/thrust/system/cpp/detail/sequence.h",
+        "cuda/include/thrust/system/cpp/detail/set_operations.h",
+        "cuda/include/thrust/system/cpp/detail/sort.h",
+        "cuda/include/thrust/system/cpp/detail/swap_ranges.h",
+        "cuda/include/thrust/system/cpp/detail/tabulate.h",
+        "cuda/include/thrust/system/cpp/detail/temporary_buffer.h",
+        "cuda/include/thrust/system/cpp/detail/transform.h",
+        "cuda/include/thrust/system/cpp/detail/transform_reduce.h",
+        "cuda/include/thrust/system/cpp/detail/transform_scan.h",
+        "cuda/include/thrust/system/cpp/detail/uninitialized_copy.h",
+        "cuda/include/thrust/system/cpp/detail/uninitialized_fill.h",
+        "cuda/include/thrust/system/cpp/detail/unique.h",
+        "cuda/include/thrust/system/cpp/detail/unique_by_key.h",
+        "cuda/include/thrust/system/cpp/detail/vector.inl",
+        "cuda/include/thrust/system/cpp/execution_policy.h",
+        "cuda/include/thrust/system/cpp/memory.h",
+        "cuda/include/thrust/system/cpp/vector.h",
+        "cuda/include/thrust/system/cuda/config.h",
+        "cuda/include/thrust/system/cuda/detail/adjacent_difference.h",
+        "cuda/include/thrust/system/cuda/detail/assign_value.h",
+        "cuda/include/thrust/system/cuda/detail/binary_search.h",
+        "cuda/include/thrust/system/cuda/detail/copy.h",
+        "cuda/include/thrust/system/cuda/detail/copy_if.h",
+        "cuda/include/thrust/system/cuda/detail/core/agent_launcher.h",
+        "cuda/include/thrust/system/cuda/detail/core/alignment.h",
+        "cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h",
+        "cuda/include/thrust/system/cuda/detail/core/util.h",
+        "cuda/include/thrust/system/cuda/detail/count.h",
+        "cuda/include/thrust/system/cuda/detail/cross_system.h",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/cub.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_device.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/util_type.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh",
+        "cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh",
+        "cuda/include/thrust/system/cuda/detail/equal.h",
+        "cuda/include/thrust/system/cuda/detail/error.inl",
+        "cuda/include/thrust/system/cuda/detail/execution_policy.h",
+        "cuda/include/thrust/system/cuda/detail/extrema.h",
+        "cuda/include/thrust/system/cuda/detail/fill.h",
+        "cuda/include/thrust/system/cuda/detail/find.h",
+        "cuda/include/thrust/system/cuda/detail/for_each.h",
+        "cuda/include/thrust/system/cuda/detail/gather.h",
+        "cuda/include/thrust/system/cuda/detail/generate.h",
+        "cuda/include/thrust/system/cuda/detail/get_value.h",
+        "cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h",
+        "cuda/include/thrust/system/cuda/detail/guarded_driver_types.h",
+        "cuda/include/thrust/system/cuda/detail/inner_product.h",
+        "cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h",
+        "cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h",
+        "cuda/include/thrust/system/cuda/detail/iter_swap.h",
+        "cuda/include/thrust/system/cuda/detail/logical.h",
+        "cuda/include/thrust/system/cuda/detail/malloc_and_free.h",
+        "cuda/include/thrust/system/cuda/detail/memory.inl",
+        "cuda/include/thrust/system/cuda/detail/merge.h",
+        "cuda/include/thrust/system/cuda/detail/mismatch.h",
+        "cuda/include/thrust/system/cuda/detail/par.h",
+        "cuda/include/thrust/system/cuda/detail/par_to_seq.h",
+        "cuda/include/thrust/system/cuda/detail/parallel_for.h",
+        "cuda/include/thrust/system/cuda/detail/partition.h",
+        "cuda/include/thrust/system/cuda/detail/reduce.h",
+        "cuda/include/thrust/system/cuda/detail/reduce_by_key.h",
+        "cuda/include/thrust/system/cuda/detail/remove.h",
+        "cuda/include/thrust/system/cuda/detail/replace.h",
+        "cuda/include/thrust/system/cuda/detail/reverse.h",
+        "cuda/include/thrust/system/cuda/detail/scan.h",
+        "cuda/include/thrust/system/cuda/detail/scan_by_key.h",
+        "cuda/include/thrust/system/cuda/detail/scatter.h",
+        "cuda/include/thrust/system/cuda/detail/sequence.h",
+        "cuda/include/thrust/system/cuda/detail/set_operations.h",
+        "cuda/include/thrust/system/cuda/detail/sort.h",
+        "cuda/include/thrust/system/cuda/detail/swap_ranges.h",
+        "cuda/include/thrust/system/cuda/detail/tabulate.h",
+        "cuda/include/thrust/system/cuda/detail/temporary_buffer.h",
+        "cuda/include/thrust/system/cuda/detail/terminate.h",
+        "cuda/include/thrust/system/cuda/detail/transform.h",
+        "cuda/include/thrust/system/cuda/detail/transform_reduce.h",
+        "cuda/include/thrust/system/cuda/detail/transform_scan.h",
+        "cuda/include/thrust/system/cuda/detail/uninitialized_copy.h",
+        "cuda/include/thrust/system/cuda/detail/uninitialized_fill.h",
+        "cuda/include/thrust/system/cuda/detail/unique.h",
+        "cuda/include/thrust/system/cuda/detail/unique_by_key.h",
+        "cuda/include/thrust/system/cuda/detail/util.h",
+        "cuda/include/thrust/system/cuda/detail/vector.inl",
+        "cuda/include/thrust/system/cuda/error.h",
+        "cuda/include/thrust/system/cuda/execution_policy.h",
+        "cuda/include/thrust/system/cuda/experimental/pinned_allocator.h",
+        "cuda/include/thrust/system/cuda/memory.h",
+        "cuda/include/thrust/system/cuda/vector.h",
+        "cuda/include/thrust/system/detail/adl/adjacent_difference.h",
+        "cuda/include/thrust/system/detail/adl/assign_value.h",
+        "cuda/include/thrust/system/detail/adl/binary_search.h",
+        "cuda/include/thrust/system/detail/adl/copy.h",
+        "cuda/include/thrust/system/detail/adl/copy_if.h",
+        "cuda/include/thrust/system/detail/adl/count.h",
+        "cuda/include/thrust/system/detail/adl/equal.h",
+        "cuda/include/thrust/system/detail/adl/extrema.h",
+        "cuda/include/thrust/system/detail/adl/fill.h",
+        "cuda/include/thrust/system/detail/adl/find.h",
+        "cuda/include/thrust/system/detail/adl/for_each.h",
+        "cuda/include/thrust/system/detail/adl/gather.h",
+        "cuda/include/thrust/system/detail/adl/generate.h",
+        "cuda/include/thrust/system/detail/adl/get_value.h",
+        "cuda/include/thrust/system/detail/adl/inner_product.h",
+        "cuda/include/thrust/system/detail/adl/iter_swap.h",
+        "cuda/include/thrust/system/detail/adl/logical.h",
+        "cuda/include/thrust/system/detail/adl/malloc_and_free.h",
+        "cuda/include/thrust/system/detail/adl/merge.h",
+        "cuda/include/thrust/system/detail/adl/mismatch.h",
+        "cuda/include/thrust/system/detail/adl/partition.h",
+        "cuda/include/thrust/system/detail/adl/reduce.h",
+        "cuda/include/thrust/system/detail/adl/reduce_by_key.h",
+        "cuda/include/thrust/system/detail/adl/remove.h",
+        "cuda/include/thrust/system/detail/adl/replace.h",
+        "cuda/include/thrust/system/detail/adl/reverse.h",
+        "cuda/include/thrust/system/detail/adl/scan.h",
+        "cuda/include/thrust/system/detail/adl/scan_by_key.h",
+        "cuda/include/thrust/system/detail/adl/scatter.h",
+        "cuda/include/thrust/system/detail/adl/sequence.h",
+        "cuda/include/thrust/system/detail/adl/set_operations.h",
+        "cuda/include/thrust/system/detail/adl/sort.h",
+        "cuda/include/thrust/system/detail/adl/swap_ranges.h",
+        "cuda/include/thrust/system/detail/adl/tabulate.h",
+        "cuda/include/thrust/system/detail/adl/temporary_buffer.h",
+        "cuda/include/thrust/system/detail/adl/transform.h",
+        "cuda/include/thrust/system/detail/adl/transform_reduce.h",
+        "cuda/include/thrust/system/detail/adl/transform_scan.h",
+        "cuda/include/thrust/system/detail/adl/uninitialized_copy.h",
+        "cuda/include/thrust/system/detail/adl/uninitialized_fill.h",
+        "cuda/include/thrust/system/detail/adl/unique.h",
+        "cuda/include/thrust/system/detail/adl/unique_by_key.h",
+        "cuda/include/thrust/system/detail/bad_alloc.h",
+        "cuda/include/thrust/system/detail/errno.h",
+        "cuda/include/thrust/system/detail/error_category.inl",
+        "cuda/include/thrust/system/detail/error_code.inl",
+        "cuda/include/thrust/system/detail/error_condition.inl",
+        "cuda/include/thrust/system/detail/generic/adjacent_difference.h",
+        "cuda/include/thrust/system/detail/generic/adjacent_difference.inl",
+        "cuda/include/thrust/system/detail/generic/advance.h",
+        "cuda/include/thrust/system/detail/generic/advance.inl",
+        "cuda/include/thrust/system/detail/generic/binary_search.h",
+        "cuda/include/thrust/system/detail/generic/binary_search.inl",
+        "cuda/include/thrust/system/detail/generic/copy.h",
+        "cuda/include/thrust/system/detail/generic/copy.inl",
+        "cuda/include/thrust/system/detail/generic/copy_if.h",
+        "cuda/include/thrust/system/detail/generic/copy_if.inl",
+        "cuda/include/thrust/system/detail/generic/count.h",
+        "cuda/include/thrust/system/detail/generic/count.inl",
+        "cuda/include/thrust/system/detail/generic/distance.h",
+        "cuda/include/thrust/system/detail/generic/distance.inl",
+        "cuda/include/thrust/system/detail/generic/equal.h",
+        "cuda/include/thrust/system/detail/generic/equal.inl",
+        "cuda/include/thrust/system/detail/generic/extrema.h",
+        "cuda/include/thrust/system/detail/generic/extrema.inl",
+        "cuda/include/thrust/system/detail/generic/fill.h",
+        "cuda/include/thrust/system/detail/generic/find.h",
+        "cuda/include/thrust/system/detail/generic/find.inl",
+        "cuda/include/thrust/system/detail/generic/for_each.h",
+        "cuda/include/thrust/system/detail/generic/gather.h",
+        "cuda/include/thrust/system/detail/generic/gather.inl",
+        "cuda/include/thrust/system/detail/generic/generate.h",
+        "cuda/include/thrust/system/detail/generic/generate.inl",
+        "cuda/include/thrust/system/detail/generic/inner_product.h",
+        "cuda/include/thrust/system/detail/generic/inner_product.inl",
+        "cuda/include/thrust/system/detail/generic/logical.h",
+        "cuda/include/thrust/system/detail/generic/memory.h",
+        "cuda/include/thrust/system/detail/generic/memory.inl",
+        "cuda/include/thrust/system/detail/generic/merge.h",
+        "cuda/include/thrust/system/detail/generic/merge.inl",
+        "cuda/include/thrust/system/detail/generic/mismatch.h",
+        "cuda/include/thrust/system/detail/generic/mismatch.inl",
+        "cuda/include/thrust/system/detail/generic/partition.h",
+        "cuda/include/thrust/system/detail/generic/partition.inl",
+        "cuda/include/thrust/system/detail/generic/reduce.h",
+        "cuda/include/thrust/system/detail/generic/reduce.inl",
+        "cuda/include/thrust/system/detail/generic/reduce_by_key.h",
+        "cuda/include/thrust/system/detail/generic/reduce_by_key.inl",
+        "cuda/include/thrust/system/detail/generic/remove.h",
+        "cuda/include/thrust/system/detail/generic/remove.inl",
+        "cuda/include/thrust/system/detail/generic/replace.h",
+        "cuda/include/thrust/system/detail/generic/replace.inl",
+        "cuda/include/thrust/system/detail/generic/reverse.h",
+        "cuda/include/thrust/system/detail/generic/reverse.inl",
+        "cuda/include/thrust/system/detail/generic/scalar/binary_search.h",
+        "cuda/include/thrust/system/detail/generic/scalar/binary_search.inl",
+        "cuda/include/thrust/system/detail/generic/scan.h",
+        "cuda/include/thrust/system/detail/generic/scan.inl",
+        "cuda/include/thrust/system/detail/generic/scan_by_key.h",
+        "cuda/include/thrust/system/detail/generic/scan_by_key.inl",
+        "cuda/include/thrust/system/detail/generic/scatter.h",
+        "cuda/include/thrust/system/detail/generic/scatter.inl",
+        "cuda/include/thrust/system/detail/generic/select_system.h",
+        "cuda/include/thrust/system/detail/generic/sequence.h",
+        "cuda/include/thrust/system/detail/generic/sequence.inl",
+        "cuda/include/thrust/system/detail/generic/set_operations.h",
+        "cuda/include/thrust/system/detail/generic/set_operations.inl",
+        "cuda/include/thrust/system/detail/generic/sort.h",
+        "cuda/include/thrust/system/detail/generic/sort.inl",
+        "cuda/include/thrust/system/detail/generic/swap_ranges.h",
+        "cuda/include/thrust/system/detail/generic/swap_ranges.inl",
+        "cuda/include/thrust/system/detail/generic/tabulate.h",
+        "cuda/include/thrust/system/detail/generic/tabulate.inl",
+        "cuda/include/thrust/system/detail/generic/tag.h",
+        "cuda/include/thrust/system/detail/generic/temporary_buffer.h",
+        "cuda/include/thrust/system/detail/generic/temporary_buffer.inl",
+        "cuda/include/thrust/system/detail/generic/transform.h",
+        "cuda/include/thrust/system/detail/generic/transform.inl",
+        "cuda/include/thrust/system/detail/generic/transform_reduce.h",
+        "cuda/include/thrust/system/detail/generic/transform_reduce.inl",
+        "cuda/include/thrust/system/detail/generic/transform_scan.h",
+        "cuda/include/thrust/system/detail/generic/transform_scan.inl",
+        "cuda/include/thrust/system/detail/generic/type_traits.h",
+        "cuda/include/thrust/system/detail/generic/uninitialized_copy.h",
+        "cuda/include/thrust/system/detail/generic/uninitialized_copy.inl",
+        "cuda/include/thrust/system/detail/generic/uninitialized_fill.h",
+        "cuda/include/thrust/system/detail/generic/uninitialized_fill.inl",
+        "cuda/include/thrust/system/detail/generic/unique.h",
+        "cuda/include/thrust/system/detail/generic/unique.inl",
+        "cuda/include/thrust/system/detail/generic/unique_by_key.h",
+        "cuda/include/thrust/system/detail/generic/unique_by_key.inl",
+        "cuda/include/thrust/system/detail/internal/decompose.h",
+        "cuda/include/thrust/system/detail/sequential/adjacent_difference.h",
+        "cuda/include/thrust/system/detail/sequential/assign_value.h",
+        "cuda/include/thrust/system/detail/sequential/binary_search.h",
+        "cuda/include/thrust/system/detail/sequential/copy.h",
+        "cuda/include/thrust/system/detail/sequential/copy.inl",
+        "cuda/include/thrust/system/detail/sequential/copy_backward.h",
+        "cuda/include/thrust/system/detail/sequential/copy_if.h",
+        "cuda/include/thrust/system/detail/sequential/count.h",
+        "cuda/include/thrust/system/detail/sequential/equal.h",
+        "cuda/include/thrust/system/detail/sequential/execution_policy.h",
+        "cuda/include/thrust/system/detail/sequential/extrema.h",
+        "cuda/include/thrust/system/detail/sequential/fill.h",
+        "cuda/include/thrust/system/detail/sequential/find.h",
+        "cuda/include/thrust/system/detail/sequential/for_each.h",
+        "cuda/include/thrust/system/detail/sequential/gather.h",
+        "cuda/include/thrust/system/detail/sequential/general_copy.h",
+        "cuda/include/thrust/system/detail/sequential/generate.h",
+        "cuda/include/thrust/system/detail/sequential/get_value.h",
+        "cuda/include/thrust/system/detail/sequential/inner_product.h",
+        "cuda/include/thrust/system/detail/sequential/insertion_sort.h",
+        "cuda/include/thrust/system/detail/sequential/iter_swap.h",
+        "cuda/include/thrust/system/detail/sequential/logical.h",
+        "cuda/include/thrust/system/detail/sequential/malloc_and_free.h",
+        "cuda/include/thrust/system/detail/sequential/merge.h",
+        "cuda/include/thrust/system/detail/sequential/merge.inl",
+        "cuda/include/thrust/system/detail/sequential/mismatch.h",
+        "cuda/include/thrust/system/detail/sequential/partition.h",
+        "cuda/include/thrust/system/detail/sequential/reduce.h",
+        "cuda/include/thrust/system/detail/sequential/reduce_by_key.h",
+        "cuda/include/thrust/system/detail/sequential/remove.h",
+        "cuda/include/thrust/system/detail/sequential/replace.h",
+        "cuda/include/thrust/system/detail/sequential/reverse.h",
+        "cuda/include/thrust/system/detail/sequential/scan.h",
+        "cuda/include/thrust/system/detail/sequential/scan_by_key.h",
+        "cuda/include/thrust/system/detail/sequential/scatter.h",
+        "cuda/include/thrust/system/detail/sequential/sequence.h",
+        "cuda/include/thrust/system/detail/sequential/set_operations.h",
+        "cuda/include/thrust/system/detail/sequential/sort.h",
+        "cuda/include/thrust/system/detail/sequential/sort.inl",
+        "cuda/include/thrust/system/detail/sequential/stable_merge_sort.h",
+        "cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl",
+        "cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h",
+        "cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl",
+        "cuda/include/thrust/system/detail/sequential/stable_radix_sort.h",
+        "cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl",
+        "cuda/include/thrust/system/detail/sequential/swap_ranges.h",
+        "cuda/include/thrust/system/detail/sequential/tabulate.h",
+        "cuda/include/thrust/system/detail/sequential/temporary_buffer.h",
+        "cuda/include/thrust/system/detail/sequential/transform.h",
+        "cuda/include/thrust/system/detail/sequential/transform_reduce.h",
+        "cuda/include/thrust/system/detail/sequential/transform_scan.h",
+        "cuda/include/thrust/system/detail/sequential/trivial_copy.h",
+        "cuda/include/thrust/system/detail/sequential/uninitialized_copy.h",
+        "cuda/include/thrust/system/detail/sequential/uninitialized_fill.h",
+        "cuda/include/thrust/system/detail/sequential/unique.h",
+        "cuda/include/thrust/system/detail/sequential/unique_by_key.h",
+        "cuda/include/thrust/system/detail/system_error.inl",
+        "cuda/include/thrust/system/error_code.h",
+        "cuda/include/thrust/system/omp/detail/adjacent_difference.h",
+        "cuda/include/thrust/system/omp/detail/assign_value.h",
+        "cuda/include/thrust/system/omp/detail/binary_search.h",
+        "cuda/include/thrust/system/omp/detail/copy.h",
+        "cuda/include/thrust/system/omp/detail/copy.inl",
+        "cuda/include/thrust/system/omp/detail/copy_if.h",
+        "cuda/include/thrust/system/omp/detail/copy_if.inl",
+        "cuda/include/thrust/system/omp/detail/count.h",
+        "cuda/include/thrust/system/omp/detail/default_decomposition.h",
+        "cuda/include/thrust/system/omp/detail/default_decomposition.inl",
+        "cuda/include/thrust/system/omp/detail/equal.h",
+        "cuda/include/thrust/system/omp/detail/execution_policy.h",
+        "cuda/include/thrust/system/omp/detail/extrema.h",
+        "cuda/include/thrust/system/omp/detail/fill.h",
+        "cuda/include/thrust/system/omp/detail/find.h",
+        "cuda/include/thrust/system/omp/detail/for_each.h",
+        "cuda/include/thrust/system/omp/detail/for_each.inl",
+        "cuda/include/thrust/system/omp/detail/gather.h",
+        "cuda/include/thrust/system/omp/detail/generate.h",
+        "cuda/include/thrust/system/omp/detail/get_value.h",
+        "cuda/include/thrust/system/omp/detail/inner_product.h",
+        "cuda/include/thrust/system/omp/detail/iter_swap.h",
+        "cuda/include/thrust/system/omp/detail/logical.h",
+        "cuda/include/thrust/system/omp/detail/malloc_and_free.h",
+        "cuda/include/thrust/system/omp/detail/memory.inl",
+        "cuda/include/thrust/system/omp/detail/merge.h",
+        "cuda/include/thrust/system/omp/detail/mismatch.h",
+        "cuda/include/thrust/system/omp/detail/par.h",
+        "cuda/include/thrust/system/omp/detail/partition.h",
+        "cuda/include/thrust/system/omp/detail/partition.inl",
+        "cuda/include/thrust/system/omp/detail/reduce.h",
+        "cuda/include/thrust/system/omp/detail/reduce.inl",
+        "cuda/include/thrust/system/omp/detail/reduce_by_key.h",
+        "cuda/include/thrust/system/omp/detail/reduce_by_key.inl",
+        "cuda/include/thrust/system/omp/detail/reduce_intervals.h",
+        "cuda/include/thrust/system/omp/detail/reduce_intervals.inl",
+        "cuda/include/thrust/system/omp/detail/remove.h",
+        "cuda/include/thrust/system/omp/detail/remove.inl",
+        "cuda/include/thrust/system/omp/detail/replace.h",
+        "cuda/include/thrust/system/omp/detail/reverse.h",
+        "cuda/include/thrust/system/omp/detail/scan.h",
+        "cuda/include/thrust/system/omp/detail/scan_by_key.h",
+        "cuda/include/thrust/system/omp/detail/scatter.h",
+        "cuda/include/thrust/system/omp/detail/sequence.h",
+        "cuda/include/thrust/system/omp/detail/set_operations.h",
+        "cuda/include/thrust/system/omp/detail/sort.h",
+        "cuda/include/thrust/system/omp/detail/sort.inl",
+        "cuda/include/thrust/system/omp/detail/swap_ranges.h",
+        "cuda/include/thrust/system/omp/detail/tabulate.h",
+        "cuda/include/thrust/system/omp/detail/temporary_buffer.h",
+        "cuda/include/thrust/system/omp/detail/transform.h",
+        "cuda/include/thrust/system/omp/detail/transform_reduce.h",
+        "cuda/include/thrust/system/omp/detail/transform_scan.h",
+        "cuda/include/thrust/system/omp/detail/uninitialized_copy.h",
+        "cuda/include/thrust/system/omp/detail/uninitialized_fill.h",
+        "cuda/include/thrust/system/omp/detail/unique.h",
+        "cuda/include/thrust/system/omp/detail/unique.inl",
+        "cuda/include/thrust/system/omp/detail/unique_by_key.h",
+        "cuda/include/thrust/system/omp/detail/unique_by_key.inl",
+        "cuda/include/thrust/system/omp/detail/vector.inl",
+        "cuda/include/thrust/system/omp/execution_policy.h",
+        "cuda/include/thrust/system/omp/memory.h",
+        "cuda/include/thrust/system/omp/vector.h",
+        "cuda/include/thrust/system/system_error.h",
+        "cuda/include/thrust/system/tbb/detail/adjacent_difference.h",
+        "cuda/include/thrust/system/tbb/detail/assign_value.h",
+        "cuda/include/thrust/system/tbb/detail/binary_search.h",
+        "cuda/include/thrust/system/tbb/detail/copy.h",
+        "cuda/include/thrust/system/tbb/detail/copy.inl",
+        "cuda/include/thrust/system/tbb/detail/copy_if.h",
+        "cuda/include/thrust/system/tbb/detail/copy_if.inl",
+        "cuda/include/thrust/system/tbb/detail/count.h",
+        "cuda/include/thrust/system/tbb/detail/equal.h",
+        "cuda/include/thrust/system/tbb/detail/execution_policy.h",
+        "cuda/include/thrust/system/tbb/detail/extrema.h",
+        "cuda/include/thrust/system/tbb/detail/fill.h",
+        "cuda/include/thrust/system/tbb/detail/find.h",
+        "cuda/include/thrust/system/tbb/detail/for_each.h",
+        "cuda/include/thrust/system/tbb/detail/for_each.inl",
+        "cuda/include/thrust/system/tbb/detail/gather.h",
+        "cuda/include/thrust/system/tbb/detail/generate.h",
+        "cuda/include/thrust/system/tbb/detail/get_value.h",
+        "cuda/include/thrust/system/tbb/detail/inner_product.h",
+        "cuda/include/thrust/system/tbb/detail/iter_swap.h",
+        "cuda/include/thrust/system/tbb/detail/logical.h",
+        "cuda/include/thrust/system/tbb/detail/malloc_and_free.h",
+        "cuda/include/thrust/system/tbb/detail/memory.inl",
+        "cuda/include/thrust/system/tbb/detail/merge.h",
+        "cuda/include/thrust/system/tbb/detail/merge.inl",
+        "cuda/include/thrust/system/tbb/detail/mismatch.h",
+        "cuda/include/thrust/system/tbb/detail/par.h",
+        "cuda/include/thrust/system/tbb/detail/partition.h",
+        "cuda/include/thrust/system/tbb/detail/partition.inl",
+        "cuda/include/thrust/system/tbb/detail/reduce.h",
+        "cuda/include/thrust/system/tbb/detail/reduce.inl",
+        "cuda/include/thrust/system/tbb/detail/reduce_by_key.h",
+        "cuda/include/thrust/system/tbb/detail/reduce_by_key.inl",
+        "cuda/include/thrust/system/tbb/detail/reduce_intervals.h",
+        "cuda/include/thrust/system/tbb/detail/remove.h",
+        "cuda/include/thrust/system/tbb/detail/remove.inl",
+        "cuda/include/thrust/system/tbb/detail/replace.h",
+        "cuda/include/thrust/system/tbb/detail/reverse.h",
+        "cuda/include/thrust/system/tbb/detail/scan.h",
+        "cuda/include/thrust/system/tbb/detail/scan.inl",
+        "cuda/include/thrust/system/tbb/detail/scan_by_key.h",
+        "cuda/include/thrust/system/tbb/detail/scatter.h",
+        "cuda/include/thrust/system/tbb/detail/sequence.h",
+        "cuda/include/thrust/system/tbb/detail/set_operations.h",
+        "cuda/include/thrust/system/tbb/detail/sort.h",
+        "cuda/include/thrust/system/tbb/detail/sort.inl",
+        "cuda/include/thrust/system/tbb/detail/swap_ranges.h",
+        "cuda/include/thrust/system/tbb/detail/tabulate.h",
+        "cuda/include/thrust/system/tbb/detail/temporary_buffer.h",
+        "cuda/include/thrust/system/tbb/detail/transform.h",
+        "cuda/include/thrust/system/tbb/detail/transform_reduce.h",
+        "cuda/include/thrust/system/tbb/detail/transform_scan.h",
+        "cuda/include/thrust/system/tbb/detail/uninitialized_copy.h",
+        "cuda/include/thrust/system/tbb/detail/uninitialized_fill.h",
+        "cuda/include/thrust/system/tbb/detail/unique.h",
+        "cuda/include/thrust/system/tbb/detail/unique.inl",
+        "cuda/include/thrust/system/tbb/detail/unique_by_key.h",
+        "cuda/include/thrust/system/tbb/detail/unique_by_key.inl",
+        "cuda/include/thrust/system/tbb/detail/vector.inl",
+        "cuda/include/thrust/system/tbb/execution_policy.h",
+        "cuda/include/thrust/system/tbb/memory.h",
+        "cuda/include/thrust/system/tbb/vector.h",
+        "cuda/include/thrust/system_error.h",
+        "cuda/include/thrust/tabulate.h",
+        "cuda/include/thrust/transform.h",
+        "cuda/include/thrust/transform_reduce.h",
+        "cuda/include/thrust/transform_scan.h",
+        "cuda/include/thrust/tuple.h",
+        "cuda/include/thrust/uninitialized_copy.h",
+        "cuda/include/thrust/uninitialized_fill.h",
+        "cuda/include/thrust/unique.h",
+        "cuda/include/thrust/version.h",
+        "cuda/include/vector_functions.h",
+        "cuda/include/vector_functions.hpp",
+        "cuda/include/vector_types.h",
+    ],
+    cmd = """
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp -f "/usr/local/cuda-10.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp -f "/usr/local/cuda-10.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp -f "/usr/local/cuda-10.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp -f "/usr/local/cuda-10.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp -f "/usr/local/cuda-10.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp -f "/usr/local/cuda-10.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp -f "/usr/local/cuda-10.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp -f "/usr/local/cuda-10.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp -f "/usr/local/cuda-10.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp -f "/usr/local/cuda-10.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp -f "/usr/local/cuda-10.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp -f "/usr/local/cuda-10.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp -f "/usr/local/cuda-10.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp -f "/usr/local/cuda-10.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp -f "/usr/local/cuda-10.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp -f "/usr/local/cuda-10.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp -f "/usr/local/cuda-10.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp -f "/usr/local/cuda-10.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp -f "/usr/local/cuda-10.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp -f "/usr/local/cuda-10.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp -f "/usr/local/cuda-10.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp -f "/usr/local/cuda-10.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp -f "/usr/local/cuda-10.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_egl_interop.h" "$(@D)/cuda/include/cuda_egl_interop.h" && cp -f "/usr/local/cuda-10.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp -f "/usr/local/cuda-10.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp -f "/usr/local/cuda-10.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp -f "/usr/local/cuda-10.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp -f "/usr/local/cuda-10.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp -f "/usr/local/cuda-10.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp -f "/usr/local/cuda-10.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp -f "/usr/local/cuda-10.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp -f "/usr/local/cuda-10.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp -f "/usr/local/cuda-10.0/include/cudart_platform.h" "$(@D)/cuda/include/cudart_platform.h" && cp -f "/usr/local/cuda-10.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp -f "/usr/local/cuda-10.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp -f "/usr/local/cuda-10.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp -f "/usr/local/cuda-10.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp -f "/usr/local/cuda-10.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp -f "/usr/local/cuda-10.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp -f "/usr/local/cuda-10.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp -f "/usr/local/cuda-10.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp -f "/usr/local/cuda-10.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp -f "/usr/local/cuda-10.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp -f "/usr/local/cuda-10.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp -f "/usr/local/cuda-10.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp -f "/usr/local/cuda-10.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp -f "/usr/local/cuda-10.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp -f "/usr/local/cuda-10.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp -f "/usr/local/cuda-10.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp -f "/usr/local/cuda-10.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp -f "/usr/local/cuda-10.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp -f "/usr/local/cuda-10.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp -f "/usr/local/cuda-10.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp -f "/usr/local/cuda-10.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp -f "/usr/local/cuda-10.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp -f "/usr/local/cuda-10.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp -f "/usr/local/cuda-10.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp -f "/usr/local/cuda-10.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp -f "/usr/local/cuda-10.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp -f "/usr/local/cuda-10.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp -f "/usr/local/cuda-10.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp -f "/usr/local/cuda-10.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp -f "/usr/local/cuda-10.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp -f "/usr/local/cuda-10.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp -f "/usr/local/cuda-10.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp -f "/usr/local/cuda-10.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp -f "/usr/local/cuda-10.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp -f "/usr/local/cuda-10.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp -f "/usr/local/cuda-10.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp -f "/usr/local/cuda-10.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp -f "/usr/local/cuda-10.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp -f "/usr/local/cuda-10.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp -f "/usr/local/cuda-10.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-10.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp -f "/usr/local/cuda-10.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp -f "/usr/local/cuda-10.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp -f "/usr/local/cuda-10.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp -f "/usr/local/cuda-10.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp -f "/usr/local/cuda-10.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp -f "/usr/local/cuda-10.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp -f "/usr/local/cuda-10.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp -f "/usr/local/cuda-10.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-10.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp -f "/usr/local/cuda-10.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp -f "/usr/local/cuda-10.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp -f "/usr/local/cuda-10.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp -f "/usr/local/cuda-10.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp -f "/usr/local/cuda-10.0/include/nvjpeg.h" "$(@D)/cuda/include/nvjpeg.h" && cp -f "/usr/local/cuda-10.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp -f "/usr/local/cuda-10.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExt.h" "$(@D)/cuda/include/nvtx3/nvToolsExt.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtCuda.h" "$(@D)/cuda/include/nvtx3/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvtx3/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtOpenCL.h" "$(@D)/cuda/include/nvtx3/nvToolsExtOpenCL.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtSync.h" "$(@D)/cuda/include/nvtx3/nvToolsExtSync.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImpl.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImpl.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCore.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCore.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInit.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInit.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInitDecls.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInitDecls.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInitDefs.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInitDefs.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxLinkOnce.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxLinkOnce.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxTypes.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxTypes.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp -f "/usr/local/cuda-10.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp -f "/usr/local/cuda-10.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp -f "/usr/local/cuda-10.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp -f "/usr/local/cuda-10.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp -f "/usr/local/cuda-10.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp -f "/usr/local/cuda-10.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp -f "/usr/local/cuda-10.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp -f "/usr/local/cuda-10.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/alignment.h" "$(@D)/cuda/include/thrust/detail/alignment.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/preprocessor.h" "$(@D)/cuda/include/thrust/detail/preprocessor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp -f "/usr/local/cuda-10.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp -f "/usr/local/cuda-10.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp -f "/usr/local/cuda-10.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp -f "/usr/local/cuda-10.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp -f "/usr/local/cuda-10.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
+   """,
+)
+
+genrule(
+    name = "cuda-nvvm",
+    outs = [
+        "cuda/nvvm/libdevice/libdevice.10.bc",
+    ],
+    cmd = """
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
+   """,
+)
+
+genrule(
+    name = "cuda-extras",
+    outs = [
+        "cuda/extras/CUPTI/include/GL/gl.h",
+        "cuda/extras/CUPTI/include/GL/glew.h",
+        "cuda/extras/CUPTI/include/GL/glext.h",
+        "cuda/extras/CUPTI/include/GL/glu.h",
+        "cuda/extras/CUPTI/include/GL/glut.h",
+        "cuda/extras/CUPTI/include/GL/glx.h",
+        "cuda/extras/CUPTI/include/GL/glxext.h",
+        "cuda/extras/CUPTI/include/GL/wglew.h",
+        "cuda/extras/CUPTI/include/GL/wglext.h",
+        "cuda/extras/CUPTI/include/cuda_stdint.h",
+        "cuda/extras/CUPTI/include/cupti.h",
+        "cuda/extras/CUPTI/include/cupti_activity.h",
+        "cuda/extras/CUPTI/include/cupti_callbacks.h",
+        "cuda/extras/CUPTI/include/cupti_driver_cbid.h",
+        "cuda/extras/CUPTI/include/cupti_events.h",
+        "cuda/extras/CUPTI/include/cupti_metrics.h",
+        "cuda/extras/CUPTI/include/cupti_nvtx_cbid.h",
+        "cuda/extras/CUPTI/include/cupti_result.h",
+        "cuda/extras/CUPTI/include/cupti_runtime_cbid.h",
+        "cuda/extras/CUPTI/include/cupti_version.h",
+        "cuda/extras/CUPTI/include/generated_cudaGL_meta.h",
+        "cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h",
+        "cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h",
+        "cuda/extras/CUPTI/include/generated_cuda_meta.h",
+        "cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h",
+        "cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h",
+        "cuda/extras/CUPTI/include/generated_nvtx_meta.h",
+        "cuda/extras/CUPTI/include/openacc/cupti_openacc.h",
+        "cuda/extras/CUPTI/include/openmp/cupti_openmp.h",
+        "cuda/extras/CUPTI/include/openmp/ompt.h",
+    ],
+    cmd = """
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openmp/cupti_openmp.h" "$(@D)/cuda/extras/CUPTI/include/openmp/cupti_openmp.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openmp/ompt.h" "$(@D)/cuda/extras/CUPTI/include/openmp/ompt.h"
+   """,
+)
+
+genrule(
+    name = "cuda-lib",
+    outs = [
+        "cuda/lib/libcuda.so",
+        "cuda/lib/libcudart.so.10.0",
+        "cuda/lib/libcudart_static.a",
+        "cuda/lib/libcublas.so.10.0",
+        "cuda/lib/libcusolver.so.10.0",
+        "cuda/lib/libcurand.so.10.0",
+        "cuda/lib/libcufft.so.10.0",
+        "cuda/lib/libcudnn.so.7",
+        "cuda/lib/libcupti.so.10.0",
+    ],
+    cmd = """
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart.so.10.0.130" "$(@D)/cuda/lib/libcudart.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcublas.so.10.0.130" "$(@D)/cuda/lib/libcublas.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcusolver.so.10.0.130" "$(@D)/cuda/lib/libcusolver.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcurand.so.10.0.130" "$(@D)/cuda/lib/libcurand.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcufft.so.10.0.145" "$(@D)/cuda/lib/libcufft.so.10.0" && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.3.1" "$(@D)/cuda/lib/libcudnn.so.7" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/lib64/libcupti.so.10.0.130" "$(@D)/cuda/lib/libcupti.so.10.0"
+   """,
+)
+
+genrule(
+    name = "cudnn-include",
+    outs = [
+        "cuda/include/cudnn.h",
+    ],
+    cmd = """
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/cudnn.h" "$(@D)/cudnn.h"
+   """,
+)
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
new file mode 100755
index 0000000000..a53c891d8b
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
@@ -0,0 +1,31 @@
+# Macros for building CUDA code.
+def if_cuda(if_true, if_false = []):
+    """Shorthand for select()'ing on whether we're building with CUDA.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with CUDA enabled.  Otherwise, the select statement evaluates to if_false.
+
+    """
+    return select({
+        "@local_config_cuda//cuda:using_nvcc": if_true,
+        "@local_config_cuda//cuda:using_clang": if_true,
+        "//conditions:default": if_false,
+    })
+
+def cuda_default_copts():
+    """Default options for all CUDA compilations."""
+    return if_cuda(["-x", "cuda", "-DGOOGLE_CUDA=1"] + [])
+
+def cuda_is_configured():
+    """Returns true if CUDA was enabled during the configure process."""
+    return True
+
+def if_cuda_is_configured(x):
+    """Tests if the CUDA was enabled during the configure process.
+
+    Unlike if_cuda(), this does not require that we are building with
+    --config=cuda. Used to allow non-CUDA code to depend on CUDA libraries.
+    """
+    if cuda_is_configured():
+        return x
+    return []
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/cuda/cuda_config.h b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/cuda/cuda_config.h
new file mode 100755
index 0000000000..0934618e0b
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/cuda/cuda_config.h
@@ -0,0 +1,26 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef CUDA_CUDA_CONFIG_H_
+#define CUDA_CUDA_CONFIG_H_
+
+#define TF_CUDA_CAPABILITIES CudaVersion("3.0")
+
+#define TF_CUDA_VERSION "10.0"
+#define TF_CUDNN_VERSION "7"
+
+#define TF_CUDA_TOOLKIT_PATH "/usr/local/cuda-10.0"
+
+#endif  // CUDA_CUDA_CONFIG_H_
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
new file mode 100755
index 0000000000..6442e7628a
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
@@ -0,0 +1,87 @@
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain(
+    name = "toolchain-linux-x86_64",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:linux",
+        "@bazel_tools//platforms:x86_64",
+    ],
+    toolchain = ":cc-compiler-local",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "local|compiler": ":cc-compiler-local",
+        "darwin|compiler": ":cc-compiler-darwin",
+        "x64_windows|msvc-cl": ":cc-compiler-windows",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-local",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    cpu = "local",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    # To support linker flags that need to go to the start of command line
+    # we need the toolchain to support parameter files. Parameter files are
+    # last on the command line and contain all shared libraries to link, so all
+    # regular options will be left of them.
+    supports_param_files = 1,
+)
+
+cc_toolchain(
+    name = "cc-compiler-darwin",
+    all_files = ":crosstool_wrapper_driver_is_not_gcc",
+    compiler_files = ":empty",
+    cpu = "darwin",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":crosstool_wrapper_driver_is_not_gcc",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 0,
+)
+
+cc_toolchain(
+    name = "cc-compiler-windows",
+    all_files = ":windows_msvc_wrapper_files",
+    compiler_files = ":empty",
+    cpu = "x64_windows",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":windows_msvc_wrapper_files",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "crosstool_wrapper_driver_is_not_gcc",
+    srcs = ["clang/bin/crosstool_wrapper_driver_is_not_gcc"],
+)
+
+filegroup(
+    name = "windows_msvc_wrapper_files",
+    srcs = glob(["windows/msvc_*"]),
+)
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL
new file mode 100755
index 0000000000..1c2e8bcae6
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/CROSSTOOL
@@ -0,0 +1,1431 @@
+major_version: "local"
+minor_version: ""
+default_target_cpu: "same_as_host"
+
+default_toolchain {
+  cpu: "k8"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "piii"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "arm"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "darwin"
+  toolchain_identifier: "local_darwin"
+}
+default_toolchain {
+  cpu: "ppc"
+  toolchain_identifier: "local_linux"
+}
+default_toolchain {
+  cpu: "x64_windows"
+  toolchain_identifier: "local_windows"
+}
+
+toolchain {
+  abi_version: "local"
+  abi_libc_version: "local"
+  compiler: "compiler"
+  host_system_name: "local"
+  needsPic: true
+  target_libc: "local"
+  target_cpu: "local"
+  target_system_name: "local"
+  toolchain_identifier: "local_linux"
+
+  feature {
+    name: "c++11"
+    flag_set {
+      action: "c++-compile"
+      flag_group {
+        flag: "-std=c++11"
+      }
+    }
+  }
+
+  feature {
+    name: "stdlib"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-lstdc++"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        flag: "-Wno-builtin-macro-redefined"
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  feature {
+    name: "alwayslink"
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-Wl,-no-as-needed"
+      }
+    }
+  }
+
+  # This feature will be enabled for builds that support pic by bazel.
+  feature {
+    name: "pic"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        expand_if_all_available: "pic"
+        flag: "-fPIC"
+      }
+      flag_group {
+        expand_if_none_available: "pic"
+        flag: "-fPIE"
+      }
+    }
+  }
+
+  # Security hardening on by default.
+  feature {
+    name: "hardening"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases.
+        # We need to undef it before redefining it as some distributions now
+        # have it enabled by default.
+        flag: "-U_FORTIFY_SOURCE"
+        flag: "-D_FORTIFY_SOURCE=1"
+        flag: "-fstack-protector"
+      }
+    }
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-Wl,-z,relro,-z,now"
+      }
+    }
+    flag_set {
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-pie"
+        flag: "-Wl,-z,relro,-z,now"
+      }
+    }
+  }
+
+  feature {
+    name: "warnings"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # All warnings are enabled. Maybe enable -Werror as well?
+        flag: "-Wall"
+        
+      }
+    }
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  feature {
+    name: "frame-pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-fno-omit-frame-pointer"
+      }
+    }
+  }
+
+  feature {
+    name: "build-id"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        # Stamp the binary with a unique identifier.
+        flag: "-Wl,--build-id=md5"
+        flag: "-Wl,--hash-style=gnu"
+      }
+    }
+  }
+
+  feature {
+    name: "no-canonical-prefixes"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-no-canonical-prefixes"
+        flag: "-fno-canonical-system-headers"
+      }
+    }
+  }
+
+  feature {
+    name: "disable-assertions"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "linker-bin-path"
+
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-B/usr/bin"
+      }
+    }
+  }
+
+  feature {
+    name: "common"
+    implies: "stdlib"
+    implies: "c++11"
+    implies: "determinism"
+    implies: "alwayslink"
+    implies: "hardening"
+    implies: "warnings"
+    implies: "frame-pointer"
+    implies: "build-id"
+    implies: "no-canonical-prefixes"
+    implies: "linker-bin-path"
+  }
+
+  feature {
+    name: "opt"
+    implies: "common"
+    implies: "disable-assertions"
+
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # No debug symbols.
+        # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt
+        # or even generally? However, that can't happen here, as it requires
+        # special handling in Bazel.
+        flag: "-g0"
+
+        # Conservative choice for -O
+        # -O3 can increase binary size and even slow down the resulting binaries.
+        # Profile first and / or use FDO if you need better performance than this.
+        flag: "-O2"
+
+        # Removal of unused code and data at link time (can this increase binary size in some cases?).
+        flag: "-ffunction-sections"
+        flag: "-fdata-sections"
+      }
+    }
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-Wl,--gc-sections"
+      }
+    }
+  }
+
+  feature {
+    name: "fastbuild"
+    implies: "common"
+  }
+
+  feature {
+    name: "dbg"
+    implies: "common"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-g"
+      }
+    }
+  }
+
+  # Set clang as a C/C++ compiler.
+  tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" }
+
+  # Use the default system toolchain for everything else.
+  tool_path { name: "ar" path: "/usr/bin/ar" }
+  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "cpp" path: "/usr/bin/cpp" }
+  tool_path { name: "dwp" path: "/usr/bin/dwp" }
+  tool_path { name: "gcov" path: "/usr/bin/gcov" }
+  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "nm" path: "/usr/bin/nm" }
+  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "objdump" path: "/usr/bin/objdump" }
+  tool_path { name: "strip" path: "/usr/bin/strip" }
+
+  # Enabled dynamic linking.
+  linking_mode_flags { mode: DYNAMIC }
+
+  cxx_builtin_include_directory: "/usr/include/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/c++/4.8/backward"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include-fixed"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/targets/x86_64-linux/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/extras/CUPTI/include"
+  cxx_builtin_include_directory: "/usr/include"
+}
+
+toolchain {
+  abi_version: "local"
+  abi_libc_version: "local"
+  compiler: "compiler"
+  host_system_name: "local"
+  needsPic: true
+  target_libc: "macosx"
+  target_cpu: "darwin"
+  target_system_name: "local"
+  toolchain_identifier: "local_darwin"
+  feature {
+    name: "c++11"
+    flag_set {
+      action: "c++-compile"
+      flag_group {
+        flag: "-std=c++11"
+      }
+    }
+  }
+
+  feature {
+    name: "stdlib"
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-lc++"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        flag: "-Wno-builtin-macro-redefined"
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  # This feature will be enabled for builds that support pic by bazel.
+  feature {
+    name: "pic"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        expand_if_all_available: "pic"
+        flag: "-fPIC"
+      }
+      flag_group {
+        expand_if_none_available: "pic"
+        flag: "-fPIE"
+      }
+    }
+  }
+
+  # Security hardening on by default.
+  feature {
+    name: "hardening"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases.
+        # We need to undef it before redefining it as some distributions now
+        # have it enabled by default.
+        flag: "-U_FORTIFY_SOURCE"
+        flag: "-D_FORTIFY_SOURCE=1"
+        flag: "-fstack-protector"
+      }
+    }
+    flag_set {
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-pie"
+      }
+    }
+  }
+
+  feature {
+    name: "warnings"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # All warnings are enabled. Maybe enable -Werror as well?
+        flag: "-Wall"
+        
+      }
+    }
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  feature {
+    name: "frame-pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-fno-omit-frame-pointer"
+      }
+    }
+  }
+
+  feature {
+    name: "no-canonical-prefixes"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag:"-no-canonical-prefixes"
+      }
+    }
+  }
+
+  feature {
+    name: "disable-assertions"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "linker-bin-path"
+
+    flag_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "-B/usr/bin"
+      }
+    }
+  }
+
+  feature {
+    name: "undefined-dynamic"
+    flag_set {
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-executable"
+      flag_group {
+        flag: "-undefined"
+        flag: "dynamic_lookup"
+      }
+    }
+  }
+
+  feature {
+    name: "common"
+    implies: "stdlib"
+    implies: "c++11"
+    implies: "determinism"
+    implies: "hardening"
+    implies: "warnings"
+    implies: "frame-pointer"
+    implies: "no-canonical-prefixes"
+    implies: "linker-bin-path"
+    implies: "undefined-dynamic"
+  }
+
+  feature {
+    name: "opt"
+    implies: "common"
+    implies: "disable-assertions"
+
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # No debug symbols.
+        # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt
+        # or even generally? However, that can't happen here, as it requires
+        # special handling in Bazel.
+        flag: "-g0"
+
+        # Conservative choice for -O
+        # -O3 can increase binary size and even slow down the resulting binaries.
+        # Profile first and / or use FDO if you need better performance than this.
+        flag: "-O2"
+
+        # Removal of unused code and data at link time (can this increase binary size in some cases?).
+        flag: "-ffunction-sections"
+        flag: "-fdata-sections"
+      }
+    }
+  }
+
+  feature {
+    name: "fastbuild"
+    implies: "common"
+  }
+
+  feature {
+    name: "dbg"
+    implies: "common"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "-g"
+      }
+    }
+  }
+
+  # Set clang as a C/C++ compiler.
+  tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" }
+
+  # Use the default system toolchain for everything else.
+  tool_path { name: "ar" path: "/usr/bin/libtool" }
+  tool_path { name: "compat-ld" path: "/usr/bin/ld" }
+  tool_path { name: "cpp" path: "/usr/bin/cpp" }
+  tool_path { name: "dwp" path: "/usr/bin/dwp" }
+  tool_path { name: "gcov" path: "/usr/bin/gcov" }
+  tool_path { name: "ld" path: "/usr/bin/ld" }
+  tool_path { name: "nm" path: "/usr/bin/nm" }
+  tool_path { name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path { name: "objdump" path: "/usr/bin/objdump" }
+  tool_path { name: "strip" path: "/usr/bin/strip" }
+
+  # Enabled dynamic linking.
+  linking_mode_flags { mode: DYNAMIC }
+
+  cxx_builtin_include_directory: "/usr/include/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.8"
+  cxx_builtin_include_directory: "/usr/include/c++/4.8/backward"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/lib/gcc/x86_64-linux-gnu/4.8/include-fixed"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/targets/x86_64-linux/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/include"
+  cxx_builtin_include_directory: "/usr/local/cuda-10.0/extras/CUPTI/include"
+  cxx_builtin_include_directory: "/usr/include"
+}
+
+toolchain {
+  toolchain_identifier: "local_windows"
+  host_system_name: "local"
+  target_system_name: "local"
+
+  abi_version: "local"
+  abi_libc_version: "local"
+  target_cpu: "x64_windows"
+  compiler: "msvc-cl"
+  target_libc: "msvcrt"
+
+
+
+  tool_path {
+    name: "ar"
+    path: ""
+  }
+  tool_path {
+    name: "ml"
+    path: ""
+  }
+  tool_path {
+    name: "cpp"
+    path: ""
+  }
+  tool_path {
+    name: "gcc"
+    path: ""
+  }
+  tool_path {
+    name: "gcov"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "ld"
+    path: ""
+  }
+  tool_path {
+    name: "nm"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objcopy"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objdump"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "strip"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  supports_interface_shared_objects: true
+
+  # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
+  compiler_flag: "/DCOMPILER_MSVC"
+
+  # Don't define min/max macros in windows.h.
+  compiler_flag: "/DNOMINMAX"
+
+  # Platform defines.
+  compiler_flag: "/D_WIN32_WINNT=0x0600"
+  # Turn off warning messages.
+  compiler_flag: "/D_CRT_SECURE_NO_DEPRECATE"
+  compiler_flag: "/D_CRT_SECURE_NO_WARNINGS"
+  compiler_flag: "/D_SILENCE_STDEXT_HASH_DEPRECATION_WARNINGS"
+
+  # Useful options to have on for compilation.
+  # Increase the capacity of object files to 2^32 sections.
+  compiler_flag: "/bigobj"
+  # Allocate 500MB for precomputed headers.
+  compiler_flag: "/Zm500"
+  # Use unsigned char by default.
+  compiler_flag: "/J"
+  # Use function level linking.
+  compiler_flag: "/Gy"
+  # Use string pooling.
+  compiler_flag: "/GF"
+  # Catch C++ exceptions only and tell the compiler to assume that functions declared
+  # as extern "C" never throw a C++ exception.
+  compiler_flag: "/EHsc"
+
+  # Globally disabled warnings.
+  # Don't warn about elements of array being be default initialized.
+  compiler_flag: "/wd4351"
+  # Don't warn about no matching delete found.
+  compiler_flag: "/wd4291"
+  # Don't warn about diamond inheritance patterns.
+  compiler_flag: "/wd4250"
+  # Don't warn about insecure functions (e.g. non _s functions).
+  compiler_flag: "/wd4996"
+
+  linker_flag: "/MACHINE:X64"
+
+  feature {
+    name: "no_legacy_features"
+  }
+
+  # Suppress startup banner.
+  feature {
+    name: "nologo"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      flag_group {
+        flag: "/nologo"
+      }
+    }
+  }
+
+  feature {
+    name: 'has_configured_linker_path'
+  }
+
+  # This feature indicates strip is not supported, building stripped binary will just result a copy of orignial binary
+  feature {
+    name: 'no_stripping'
+  }
+
+  # This feature indicates this is a toolchain targeting Windows.
+  feature {
+    name: 'targets_windows'
+    implies: 'copy_dynamic_libraries_to_binary'
+    enabled: true
+  }
+
+  feature {
+    name: 'copy_dynamic_libraries_to_binary'
+  }
+
+  action_config {
+    config_name: 'assemble'
+    action_name: 'assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'preprocess-assemble'
+    action_name: 'preprocess-assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'c-compile'
+    action_name: 'c-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-compile'
+    action_name: 'c++-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-link-executable'
+    action_name: 'c++-link-executable'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+  }
+
+  action_config {
+    config_name: 'c++-link-dynamic-library'
+    action_name: 'c++-link-dynamic-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'shared_flag'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+    implies: 'has_configured_linker_path'
+    implies: 'def_file'
+  }
+
+  action_config {
+      config_name: 'c++-link-nodeps-dynamic-library'
+      action_name: 'c++-link-nodeps-dynamic-library'
+      tool {
+        tool_path: ''
+      }
+      implies: 'nologo'
+      implies: 'shared_flag'
+      implies: 'linkstamps'
+      implies: 'output_execpath_flags'
+      implies: 'input_param_flags'
+      implies: 'user_link_flags'
+      implies: 'legacy_link_flags'
+      implies: 'linker_subsystem_flag'
+      implies: 'linker_param_file'
+      implies: 'msvc_env'
+      implies: 'no_stripping'
+      implies: 'has_configured_linker_path'
+      implies: 'def_file'
+    }
+
+  action_config {
+    config_name: 'c++-link-static-library'
+    action_name: 'c++-link-static-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'archiver_flags'
+    implies: 'input_param_flags'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+  }
+
+  # TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
+  # not used in this crosstool
+  feature {
+    name: 'legacy_compile_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'legacy_compile_flags'
+        flag: '%{legacy_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "PATH"
+        value: ""
+      }
+      env_entry {
+        key: "INCLUDE"
+        value: ""
+      }
+      env_entry {
+        key: "LIB"
+        value: ""
+      }
+      env_entry {
+        key: "TMP"
+        value: ""
+      }
+      env_entry {
+        key: "TEMP"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: 'include_paths'
+    flag_set {
+      action: "assemble"
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      flag_group {
+        iterate_over: 'quote_include_paths'
+        flag: '/I%{quote_include_paths}'
+      }
+      flag_group {
+        iterate_over: 'include_paths'
+        flag: '/I%{include_paths}'
+      }
+      flag_group {
+        iterate_over: 'system_include_paths'
+        flag: '/I%{system_include_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: "preprocessor_defines"
+    flag_set {
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-header-parsing"
+      action: "c++-module-compile"
+      flag_group {
+        flag: "/D%{preprocessor_defines}"
+        iterate_over: "preprocessor_defines"
+      }
+    }
+  }
+
+  # Tell Bazel to parse the output of /showIncludes
+  feature {
+    name: 'parse_showincludes'
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-module-compile'
+      action: 'c++-header-parsing'
+      flag_group {
+        flag: "/showIncludes"
+      }
+    }
+  }
+
+
+  feature {
+    name: 'generate_pdb_file'
+    requires: {
+      feature: 'dbg'
+    }
+    requires: {
+      feature: 'fastbuild'
+    }
+  }
+
+  feature {
+    name: 'shared_flag'
+    flag_set {
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/DLL'
+      }
+    }
+  }
+
+  feature {
+    name: 'linkstamps'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      expand_if_all_available: 'linkstamp_paths'
+      flag_group {
+        iterate_over: 'linkstamp_paths'
+        flag: '%{linkstamp_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: 'output_execpath_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'archiver_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'input_param_flags'
+    flag_set {
+      expand_if_all_available: 'interface_library_output_path'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/IMPLIB:%{interface_library_output_path}"
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libopts'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'libopts'
+        flag: '%{libopts}'
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libraries_to_link'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        iterate_over: 'libraries_to_link'
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file_group'
+          }
+          iterate_over: 'libraries_to_link.object_files'
+          flag_group {
+            flag: '%{libraries_to_link.object_files}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'interface_library'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'static_library'
+          }
+          flag_group {
+            expand_if_false: 'libraries_to_link.is_whole_archive'
+            flag: '%{libraries_to_link.name}'
+          }
+          flag_group {
+            expand_if_true: 'libraries_to_link.is_whole_archive'
+            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
+          }
+        }
+      }
+    }
+  }
+
+  # Since this feature is declared earlier in the CROSSTOOL than
+  # "user_link_flags", this feature will be applied prior to it anwyhere they
+  # are both implied. And since "user_link_flags" contains the linkopts from
+  # the build rule, this allows the user to override the /SUBSYSTEM in the BUILD
+  # file.
+  feature {
+    name: 'linker_subsystem_flag'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/SUBSYSTEM:CONSOLE'
+      }
+    }
+  }
+
+  # The "user_link_flags" contains user-defined linkopts (from build rules)
+  # so it should be defined after features that declare user-overridable flags.
+  # For example the "linker_subsystem_flag" defines a default "/SUBSYSTEM" flag
+  # but we want to let the user override it, therefore "link_flag_subsystem" is
+  # defined earlier in the CROSSTOOL file than "user_link_flags".
+  feature {
+    name: 'user_link_flags'
+    flag_set {
+      expand_if_all_available: 'user_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'user_link_flags'
+        flag: '%{user_link_flags}'
+      }
+    }
+  }
+  feature {
+    name: 'legacy_link_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'legacy_link_flags'
+        flag: '%{legacy_link_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'linker_param_file'
+    flag_set {
+      expand_if_all_available: 'linker_param_file'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '@%{linker_param_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'static_link_msvcrt'
+  }
+
+  feature {
+    name: 'static_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MT"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MD"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'static_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MTd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MDd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dbg'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+        flag: "/DDEBUG"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEBUG:FULL"
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'fastbuild'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+        flag: "/DDEBUG"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEBUG:FASTLINK"
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'opt'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/O2"
+        flag: "/DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: 'user_compile_flags'
+    flag_set {
+      expand_if_all_available: 'user_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'user_compile_flags'
+        flag: '%{user_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'sysroot'
+    flag_set {
+      expand_if_all_available: 'sysroot'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'sysroot'
+        flag: '--sysroot=%{sysroot}'
+      }
+    }
+  }
+
+  feature {
+    name: 'unfiltered_compile_flags'
+    flag_set {
+      expand_if_all_available: 'unfiltered_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'unfiltered_compile_flags'
+        flag: '%{unfiltered_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_output_flags'
+    flag_set {
+      action: 'assemble'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+        flag: '/Zi'
+      }
+    }
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_assembly_file'
+        flag: '/Fa%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_preprocess_file'
+        flag: '/P'
+        flag: '/Fi%{output_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_input_flags'
+    flag_set {
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'source_file'
+        flag: '/c'
+        flag: '%{source_file}'
+      }
+    }
+  }
+
+  feature {
+    name : 'def_file',
+    flag_set {
+      expand_if_all_available: 'def_file_path'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEF:%{def_file_path}"
+        # We can specify a different DLL name in DEF file, /ignore:4070 suppresses
+        # the warning message about DLL name doesn't match the default one.
+        # See https://msdn.microsoft.com/en-us/library/sfkk2fz7.aspx
+        flag: "/ignore:4070"
+      }
+    }
+  }
+
+  feature {
+    name: 'windows_export_all_symbols'
+  }
+
+  feature {
+    name: 'no_windows_export_all_symbols'
+  }
+
+  linking_mode_flags { mode: DYNAMIC }
+}
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
new file mode 100755
index 0000000000..7ae59e9967
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/clang/bin/crosstool_wrapper_driver_is_not_gcc
@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs.
+
+SYNOPSIS:
+  crosstool_wrapper_is_not_gcc [options passed in by cc_library()
+                                or cc_binary() rule]
+
+DESCRIPTION:
+  This script is expected to be called by the cc_library() or cc_binary() bazel
+  rules. When the option "-x cuda" is present in the list of arguments passed
+  to this script, it invokes the nvcc CUDA compiler. Most arguments are passed
+  as is as a string to --compiler-options of nvcc. When "-x cuda" is not
+  present, this wrapper invokes hybrid_driver_is_not_gcc with the input
+  arguments as is.
+
+NOTES:
+  Changes to the contents of this file must be propagated from
+  //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc to
+  //third_party/gpus/crosstool/v*/*/clang/bin/crosstool_wrapper_is_not_gcc
+"""
+
+from __future__ import print_function
+
+__author__ = 'keveman@google.com (Manjunath Kudlur)'
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('/usr/bin/gcc')
+GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc')
+
+NVCC_PATH = '/usr/local/cuda-10.0/bin/nvcc'
+PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
+NVCC_VERSION = '10.0'
+
+def Log(s):
+  print('gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from the argv list.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    option: The option whose value to extract, without the leading '-'.
+
+  Returns:
+    A list of values, either directly following the option,
+    (eg., -opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., -opt val1 -opt val2).
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-' + option, nargs='*', action='append')
+  args, _ = parser.parse_known_args(argv)
+  if not args or not vars(args)[option]:
+    return []
+  else:
+    return sum(vars(args)[option], [])
+
+
+def GetHostCompilerOptions(argv):
+  """Collect the -isystem, -iquote, and --sysroot option values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be used as the --compiler-options to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-isystem', nargs='*', action='append')
+  parser.add_argument('-iquote', nargs='*', action='append')
+  parser.add_argument('--sysroot', nargs=1)
+  parser.add_argument('-g', nargs='*', action='append')
+  parser.add_argument('-fno-canonical-system-headers', action='store_true')
+
+  args, _ = parser.parse_known_args(argv)
+
+  opts = ''
+
+  if args.isystem:
+    opts += ' -isystem ' + ' -isystem '.join(sum(args.isystem, []))
+  if args.iquote:
+    opts += ' -iquote ' + ' -iquote '.join(sum(args.iquote, []))
+  if args.g:
+    opts += ' -g' + ' -g'.join(sum(args.g, []))
+  if args.fno_canonical_system_headers:
+    opts += ' -fno-canonical-system-headers'
+  if args.sysroot:
+    opts += ' --sysroot ' + args.sysroot[0]
+
+  return opts
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    The string that can be passed directly to nvcc.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, _ = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return ' '.join(['--'+a for a in options])
+  return ''
+
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling os.system('nvcc ' + args)
+  """
+
+  host_compiler_options = GetHostCompilerOptions(argv)
+  nvcc_compiler_options = GetNvccOptions(argv)
+  opt_option = GetOptionValue(argv, 'O')
+  m_options = GetOptionValue(argv, 'm')
+  m_options = ''.join([' -m' + m for m in m_options if m in ['32', '64']])
+  include_options = GetOptionValue(argv, 'I')
+  out_file = GetOptionValue(argv, 'o')
+  depfiles = GetOptionValue(argv, 'MF')
+  defines = GetOptionValue(argv, 'D')
+  defines = ''.join([' -D' + define for define in defines])
+  undefines = GetOptionValue(argv, 'U')
+  undefines = ''.join([' -U' + define for define in undefines])
+  std_options = GetOptionValue(argv, 'std')
+  # currently only c++11 is supported by Cuda 7.0 std argument
+  nvcc_allowed_std_options = ["c++11"]
+  std_options = ''.join([' -std=' + define
+      for define in std_options if define in nvcc_allowed_std_options])
+
+  # The list of source files get passed after the -c option. I don't know of
+  # any other reliable way to just get the list of source files to be compiled.
+  src_files = GetOptionValue(argv, 'c')
+
+  # Pass -w through from host to nvcc, but don't do anything fancier with
+  # warnings-related flags, since they're not necessarily the same across
+  # compilers.
+  warning_options = ' -w' if '-w' in argv else ''
+
+  if len(src_files) == 0:
+    return 1
+  if len(out_file) != 1:
+    return 1
+
+  opt = (' -O2' if (len(opt_option) > 0 and int(opt_option[0]) > 0)
+         else ' -g -G')
+
+  includes = (' -I ' + ' -I '.join(include_options)
+              if len(include_options) > 0
+              else '')
+
+  # Unfortunately, there are other options that have -c prefix too.
+  # So allowing only those look like C/C++ files.
+  src_files = [f for f in src_files if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+  srcs = ' '.join(src_files)
+  out = ' -o ' + out_file[0]
+
+  supported_cuda_compute_capabilities = [ "3.0" ]
+  nvccopts = '-D_FORCE_INLINES '
+  for capability in supported_cuda_compute_capabilities:
+    capability = capability.replace('.', '')
+    nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s,compute_%s\" ' % (
+        capability, capability, capability)
+  nvccopts += ' ' + nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += std_options
+  nvccopts += m_options
+  nvccopts += warning_options
+
+  if depfiles:
+    # Generate the dependency file
+    depfile = depfiles[0]
+    cmd = (NVCC_PATH + ' ' + nvccopts +
+           ' --compiler-options "' + host_compiler_options + '"' +
+           ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+           ' -I .' +
+           ' -x cu ' + opt + includes + ' ' + srcs + ' -M -o ' + depfile)
+    if log: Log(cmd)
+    exit_status = os.system(cmd)
+    if exit_status != 0:
+      return exit_status
+
+  cmd = (NVCC_PATH + ' ' + nvccopts +
+         ' --compiler-options "' + host_compiler_options + ' -fPIC"' +
+         ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH +
+         ' -I .' +
+         ' -x cu ' + opt + includes + ' -c ' + srcs + out)
+
+  # TODO(zhengxq): for some reason, 'gcc' needs this help to find 'as'.
+  # Need to investigate and fix.
+  cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd
+  if log: Log(cmd)
+  return os.system(cmd)
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.bat b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.bat
new file mode 100755
index 0000000000..e896e654fd
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.bat
@@ -0,0 +1,20 @@
+:: Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+::
+:: Licensed under the Apache License, Version 2.0 (the "License");
+:: you may not use this file except in compliance with the License.
+:: You may obtain a copy of the License at
+::
+::     http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+:: =============================================================================
+
+:: Invoke msvc_wrapper_for_nvcc.py, which is located in the same directory.
+@echo OFF
+set arg0=%~0
+for %%F in ("%arg0%") do set DRIVER_BIN=%%~dpF
+"/usr/bin/python3" -B "%DRIVER_BIN%\msvc_wrapper_for_nvcc.py" %*
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
new file mode 100755
index 0000000000..00483951af
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Crosstool wrapper for compiling CUDA programs with nvcc on Windows.
+
+DESCRIPTION:
+  This script is the Windows version of //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc
+"""
+
+from __future__ import print_function
+
+from argparse import ArgumentParser
+import os
+import subprocess
+import re
+import sys
+import pipes
+
+# Template values set by cuda_autoconf.
+CPU_COMPILER = ('/usr/bin/gcc')
+GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc')
+
+NVCC_PATH = '/usr/local/cuda-10.0/bin/nvcc'
+NVCC_VERSION = '10.0'
+NVCC_TEMP_DIR = "C:\\Windows\\Temp\\nvcc_inter_files_tmp_dir"
+supported_cuda_compute_capabilities = [ "3.0" ]
+
+def Log(s):
+  print('gpus/crosstool: {0}'.format(s))
+
+
+def GetOptionValue(argv, option):
+  """Extract the list of values for option from options.
+
+  Args:
+    option: The option whose value to extract, without the leading '/'.
+
+  Returns:
+    1. A list of values, either directly following the option,
+    (eg., /opt val1 val2) or values collected from multiple occurrences of
+    the option (eg., /opt val1 /opt val2).
+    2. The leftover options.
+  """
+
+  parser = ArgumentParser(prefix_chars='/')
+  parser.add_argument('/' + option, nargs='*', action='append')
+  args, leftover = parser.parse_known_args(argv)
+  if args and vars(args)[option]:
+    return (sum(vars(args)[option], []), leftover)
+  return ([], leftover)
+
+def _update_options(nvcc_options):
+  if NVCC_VERSION in ("7.0",):
+    return nvcc_options
+
+  update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" }
+  return [ update_options[opt] if opt in update_options else opt
+                    for opt in nvcc_options ]
+
+def GetNvccOptions(argv):
+  """Collect the -nvcc_options values from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+
+  Returns:
+    1. The string that can be passed directly to nvcc.
+    2. The leftover options.
+  """
+
+  parser = ArgumentParser()
+  parser.add_argument('-nvcc_options', nargs='*', action='append')
+
+  args, leftover = parser.parse_known_args(argv)
+
+  if args.nvcc_options:
+    options = _update_options(sum(args.nvcc_options, []))
+    return (['--' + a for a in options], leftover)
+  return ([], leftover)
+
+
+def InvokeNvcc(argv, log=False):
+  """Call nvcc with arguments assembled from argv.
+
+  Args:
+    argv: A list of strings, possibly the argv passed to main().
+    log: True if logging is requested.
+
+  Returns:
+    The return value of calling os.system('nvcc ' + args)
+  """
+
+  src_files = [f for f in argv if
+               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+  if len(src_files) == 0:
+    raise Error('No source files found for cuda compilation.')
+
+  out_file = [ f for f in argv if f.startswith('/Fo') ]
+  if len(out_file) != 1:
+    raise Error('Please sepecify exactly one output file for cuda compilation.')
+  out = ['-o', out_file[0][len('/Fo'):]]
+
+  nvcc_compiler_options, argv = GetNvccOptions(argv)
+
+  opt_option, argv = GetOptionValue(argv, 'O')
+  opt = ['-g', '-G']
+  if (len(opt_option) > 0 and opt_option[0] != 'd'):
+    opt = ['-O2']
+
+  include_options, argv = GetOptionValue(argv, 'I')
+  includes = ["-I " + include for include in include_options]
+
+  defines, argv = GetOptionValue(argv, 'D')
+  defines = ['-D' + define for define in defines]
+
+  undefines, argv = GetOptionValue(argv, 'U')
+  undefines = ['-U' + define for define in undefines]
+
+  # The rest of the unrecongized options should be passed to host compiler
+  host_compiler_options = [option for option in argv if option not in (src_files + out_file)]
+
+  m_options = ["-m64"]
+
+  nvccopts = ['-D_FORCE_INLINES']
+  for capability in supported_cuda_compute_capabilities:
+    capability = capability.replace('.', '')
+    nvccopts += [r'-gencode=arch=compute_%s,"code=sm_%s,compute_%s"' % (
+        capability, capability, capability)]
+  nvccopts += nvcc_compiler_options
+  nvccopts += undefines
+  nvccopts += defines
+  nvccopts += m_options
+  nvccopts += ['--compiler-options="' + " ".join(host_compiler_options) + '"']
+  nvccopts += ['-x', 'cu'] + opt + includes + out + ['-c'] + src_files
+  # If we don't specify --keep-dir, nvcc will generate intermediate files under TEMP
+  # Put them under NVCC_TEMP_DIR instead, then Bazel can ignore files under NVCC_TEMP_DIR during dependency check
+  # http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-guiding-compiler-driver
+  # Different actions are sharing NVCC_TEMP_DIR, so we cannot remove it if the directory already exists.
+  if os.path.isfile(NVCC_TEMP_DIR):
+    os.remove(NVCC_TEMP_DIR)
+  if not os.path.exists(NVCC_TEMP_DIR):
+    os.makedirs(NVCC_TEMP_DIR)
+  nvccopts += ['--keep', '--keep-dir', NVCC_TEMP_DIR]
+  cmd = [NVCC_PATH] + nvccopts
+  if log:
+    Log(cmd)
+  proc = subprocess.Popen(cmd,
+                          stdout=sys.stdout,
+                          stderr=sys.stderr,
+                          env=os.environ.copy(),
+                          shell=True)
+  proc.wait()
+  return proc.returncode
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument('-x', nargs=1)
+  parser.add_argument('--cuda_log', action='store_true')
+  args, leftover = parser.parse_known_args(sys.argv[1:])
+
+  if args.x and args.x[0] == 'cuda':
+    if args.cuda_log: Log('-x cuda')
+    leftover = [pipes.quote(s) for s in leftover]
+    if args.cuda_log: Log('using nvcc')
+    return InvokeNvcc(leftover, log=args.cuda_log)
+
+  # Strip our flags before passing through to the CPU compiler for files which
+  # are not -x cuda. We can't just pass 'leftover' because it also strips -x.
+  # We not only want to pass -x to the CPU compiler, but also keep it in its
+  # relative location in the argv list (the compiler is actually sensitive to
+  # this).
+  cpu_compiler_flags = [flag for flag in sys.argv[1:]
+                             if not flag.startswith(('--cuda_log'))
+                             and not flag.startswith(('-nvcc_options'))]
+
+  return subprocess.call([CPU_COMPILER] + cpu_compiler_flags)
+
+if __name__ == '__main__':
+  sys.exit(main())
-- 
GitLab


From 92112c2ff39da4ebf37b518aabcbd48e3137cc9c Mon Sep 17 00:00:00 2001
From: Feiyang Chen <chenfeiyang@bjfu.edu.cn>
Date: Mon, 26 Nov 2018 23:08:28 +0800
Subject: [PATCH 0762/1554] add tensorFlow visualization toolkit links

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 8af5370bef..0d1e1ea2ea 100644
--- a/README.md
+++ b/README.md
@@ -127,6 +127,7 @@ Build Type
 * [TensorFlow Roadmap](https://www.tensorflow.org/community/roadmap)
 * [TensorFlow White Papers](https://www.tensorflow.org/about/bib)
 * [TensorFlow YouTube Channel](https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ)
+* [TensorFlow Visualization Toolkit](https://github.com/tensorflow/tensorboard)
 
 Learn more about the TensorFlow community at the [community page of tensorflow.org](https://www.tensorflow.org/community) for a few ways to participate.
 
-- 
GitLab


From 64e084b8cb27e8c53b15468c21f1b3471b4b9659 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 26 Nov 2018 07:35:53 -0800
Subject: [PATCH 0763/1554] Add cond_v2 and while_v2 as a dependency for tests

PiperOrigin-RevId: 222822446
---
 tensorflow/python/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 5c9e7f5e89..3fe381183a 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1081,10 +1081,12 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":client",
+        ":cond_v2",
         ":framework_test_lib",
         ":gradient_checker",
         ":platform_test",
         ":util",
+        ":while_v2",
     ],
 )
 
-- 
GitLab


From 8be3b4631892e10b18d0a4cc8cad6099c1f6803b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 08:24:04 -0800
Subject: [PATCH 0764/1554] Migrate ::StringPiece to absl::string_view.
 PiperOrigin-RevId: 222828687

---
 tensorflow/core/platform/regexp.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/regexp.h b/tensorflow/core/platform/regexp.h
index a4eedf3045..ca9ca1e244 100644
--- a/tensorflow/core/platform/regexp.h
+++ b/tensorflow/core/platform/regexp.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_PLATFORM_REGEXP_H_
 #define TENSORFLOW_PLATFORM_REGEXP_H_
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/core/platform/platform.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -23,7 +24,7 @@ limitations under the License.
     defined(GOOGLE_RE2)
 #include "tensorflow/core/platform/google/build_config/re2.h"
 namespace tensorflow {
-typedef ::StringPiece RegexpStringPiece;
+typedef absl::string_view RegexpStringPiece;
 }  // namespace tensorflow
 
 #else
-- 
GitLab


From b53a109aa242d8ef493f18d97f872494efa17082 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 26 Nov 2018 08:34:23 -0800
Subject: [PATCH 0765/1554] Change API for io.decode_csv for TF 2.0.

PiperOrigin-RevId: 222829999
---
 tensorflow/python/ops/parsing_ops.py          | 50 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.io.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  5 ++
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py
index 7a11096d4f..a84af6c5cf 100644
--- a/tensorflow/python/ops/parsing_ops.py
+++ b/tensorflow/python/ops/parsing_ops.py
@@ -1828,7 +1828,7 @@ def _parse_single_sequence_example_raw(serialized,
 
 
 # Swap `name` and `na_value` for backward compatibility.
-@tf_export("io.decode_csv", v1=["io.decode_csv", "decode_csv"])
+@tf_export(v1=["io.decode_csv", "decode_csv"])
 @deprecation.deprecated_endpoints("decode_csv")
 def decode_csv(records,
                record_defaults,
@@ -1867,6 +1867,54 @@ def decode_csv(records,
     A list of `Tensor` objects. Has the same type as `record_defaults`.
     Each tensor will have the same shape as records.
 
+  Raises:
+    ValueError: If any of the arguments is malformed.
+  """
+  return decode_csv_v2(
+      records, record_defaults,
+      field_delim, use_quote_delim,
+      na_value, select_cols, name
+      )
+
+
+@tf_export("io.decode_csv", v1=[])
+def decode_csv_v2(records,
+                  record_defaults,
+                  field_delim=",",
+                  use_quote_delim=True,
+                  na_value="",
+                  select_cols=None,
+                  name=None):
+  """Convert CSV records to tensors. Each column maps to one tensor.
+
+  RFC 4180 format is expected for the CSV records.
+  (https://tools.ietf.org/html/rfc4180)
+  Note that we allow leading and trailing spaces with int or float field.
+
+  Args:
+    records: A `Tensor` of type `string`.
+      Each string is a record/row in the csv and all records should have
+      the same format.
+    record_defaults: A list of `Tensor` objects with specific types.
+      Acceptable types are `float32`, `float64`, `int32`, `int64`, `string`.
+      One tensor per column of the input record, with either a
+      scalar default value for that column or an empty vector if the column is
+      required.
+    field_delim: An optional `string`. Defaults to `","`.
+      char delimiter to separate fields in a record.
+    use_quote_delim: An optional `bool`. Defaults to `True`.
+      If false, treats double quotation marks as regular
+      characters inside of the string fields (ignoring RFC 4180, Section 2,
+      Bullet 5).
+    na_value: Additional string to recognize as NA/NaN.
+    select_cols: Optional sorted list of column indices to select. If specified,
+      only this subset of columns will be parsed and returned.
+    name: A name for the operation (optional).
+
+  Returns:
+    A list of `Tensor` objects. Has the same type as `record_defaults`.
+    Each tensor will have the same shape as records.
+
   Raises:
     ValueError: If any of the arguments is malformed.
   """
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
index 98250df803..8906329742 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.pbtxt
@@ -66,7 +66,7 @@ tf_module {
   }
   member_method {
     name: "decode_csv"
-    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\', \'select_cols\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\', \'None\'], "
+    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'na_value\', \'select_cols\', \'name\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'\', \'None\', \'None\'], "
   }
   member_method {
     name: "decode_gif"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 33e4f0f442..ed29a7cb62 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -379,6 +379,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.segment_sum": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
+        "tf.io.decode_csv": [
+            "records", "record_defaults",
+            "field_delim", "use_quote_delim",
+            "name", "na_value", "select_cols",
+        ],
         "tf.strings.substr": ["input", "pos", "len", "name", "unit"],
         "tf.strings.reduce_join": [
             "input", "axis", "keep_dims", "separator", "name",
-- 
GitLab


From 47c8db116582633cfd898d2fab19fc57922e6a67 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Mon, 26 Nov 2018 08:38:45 -0800
Subject: [PATCH 0766/1554] Fix top TensorBoard link.

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0d1e1ea2ea..eaee8c274b 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,8 @@ data flow graphs.  The graph nodes represent mathematical operations, while
 the graph edges represent the multidimensional data arrays (tensors) that flow
 between them.  This flexible architecture enables you to deploy computation to one
 or more CPUs or GPUs in a desktop, server, or mobile device without rewriting
-code.  TensorFlow also includes [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard), a data visualization toolkit.
+code.  TensorFlow also includes [TensorBoard](https://github.com/tensorflow/tensorboard),
+a data visualization toolkit.
 
 TensorFlow was originally developed by researchers and engineers
 working on the Google Brain team within Google's Machine Intelligence Research
-- 
GitLab


From 0d2e879a162a6c92d5602a7a88ea4e92fd104ee2 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 26 Nov 2018 08:55:52 -0800
Subject: [PATCH 0767/1554] Refactor shared cond_v2 and _IfGrad logic into
 _build_cond helper method.

PiperOrigin-RevId: 222832725
---
 tensorflow/python/ops/cond_v2.py | 148 +++++++++++++++----------------
 1 file changed, 73 insertions(+), 75 deletions(-)

diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 0f08c611bc..4db59871d3 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -74,55 +74,14 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
             false_name, read_only_collections=False),
         add_control_dependencies=add_control_dependencies,
         op_return_value=pred)
-    _check_same_outputs(true_graph, false_graph)
-
-    # Add inputs to true_graph and false_graph to make them match. Note that
-    # this modifies true_graph and false_graph.
-    cond_inputs = _make_inputs_match(true_graph, false_graph,
-                                     true_graph.external_captures,
-                                     false_graph.external_captures)
-
-    # Add all intermediate tensors as function outputs so they're available for
-    # the gradient computation.
-
-    true_intermediates = _get_intermediates(true_graph)
-    false_intermediates = _get_intermediates(false_graph)
-
-    # Save the original number of outputs to return to the caller.
-    num_cond_outputs = len(true_graph.outputs)
-
-    # Make the number/type of new intermediate outputs match.
-    extra_true_outputs, extra_false_outputs = _pad_params(
-        true_graph, false_graph, true_intermediates, false_intermediates)
-
-    true_graph.outputs.extend(extra_true_outputs)
-    false_graph.outputs.extend(extra_false_outputs)
-
-    # Create the If op.
-    tensors = gen_functional_ops._if(  # pylint: disable=protected-access
-        pred,
-        cond_inputs, [t.dtype for t in true_graph.outputs],
-        util.create_new_tf_function(true_graph),
-        util.create_new_tf_function(false_graph),
-        output_shapes=_get_output_shapes(true_graph.outputs,
-                                         false_graph.outputs),
-        name=scope)
-
-    # TODO(b/110167197) this approach requires cond_v2 to have at least 1 output
-    util.maybe_set_lowering_attr(tensors[0].op)
-
-    # Return identities for each output of the If op, rather than the output of
-    # the If op directly. This makes pruning work if the output of cond() is
-    # fetched: the lowering pass converts the If outputs into IdentityN outputs,
-    # which if fetched will cause all ops in the taken branch to be run (since
-    # it takes all merge ops as input). After lowering, each output identity op
-    # will end up with only the appropriate merge op as input.
-    # TODO(b/79984175): this doesn't have to be a tuple once we covert to the
-    # correct output structure
-    tensors = tuple(array_ops.identity(t) for t in tensors)
+
+    outputs = _build_cond(pred, true_graph, false_graph,
+                          true_graph.external_captures,
+                          false_graph.external_captures,
+                          name=scope)
 
     return func_graph_module.pack_sequence_as(true_graph.structured_outputs,
-                                              tensors[:num_cond_outputs])
+                                              outputs)
 
 
 @ops.RegisterGradient("If")
@@ -150,44 +109,83 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   true_grad_inputs = _resolve_grad_inputs(true_graph, true_grad_graph)
   false_grad_inputs = _resolve_grad_inputs(false_graph, false_grad_graph)
 
-  # Make the inputs to true_grad_graph and false_grad_graph match. Note that
-  # this modifies true_grad_graph and false_grad_graph.
-  grad_inputs = _make_inputs_match(true_grad_graph, false_grad_graph,
-                                   true_grad_inputs, false_grad_inputs)
+  outputs = _build_cond(op.inputs[0], true_grad_graph, false_grad_graph,
+                        true_grad_inputs, false_grad_inputs)
+
+  # The predicate has no gradient.
+  return [None] + outputs
+
+
+def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
+                name=None):
+  """Creates an If op from the specified predicate, branch functions and inputs.
+
+  Note that this modifies true_graph and false_graph to make the inputs match,
+  and to output all intermediates values so they're available for the gradient
+  computation.
+
+  true_graph and false_graph need not have the same input types, but they must
+  have the same outpute types.
+
+  Args:
+    pred: boolean Tensor
+    true_graph: FuncGraph
+    false_graph: FuncGraph
+    true_inputs: a list of Tensors to be passed to true_graph as input.
+    false_inputs: a list of Tensors to be passed to false_graph as input.
+    name: the name for the If op.
+
+  Returns:
+    A list of Tensors which are the outputs of the If op. Does not include added
+    intermediate outputs.
+  """
+  _check_same_outputs(true_graph, false_graph)
+
+  # Add inputs to true_graph and false_graph to make them match. Note that
+  # this modifies true_graph and false_graph.
+  cond_inputs = _make_inputs_match(true_graph, false_graph,
+                                   true_inputs, false_inputs)
 
   # Add all intermediate tensors as function outputs so they're available for
-  # higher-order gradient computations.
+  # the gradient computation.
 
-  true_grad_intermediates = _get_intermediates(true_grad_graph)
-  false_grad_intermediates = _get_intermediates(false_grad_graph)
+  true_intermediates = _get_intermediates(true_graph)
+  false_intermediates = _get_intermediates(false_graph)
 
-  # Save the original number of gradient outputs to return.
-  num_grad_outputs = len(true_grad_graph.outputs)
+  # Save the original number of outputs to return to the caller.
+  num_cond_outputs = len(true_graph.outputs)
 
   # Make the number/type of new intermediate outputs match.
-  extra_true_grad_outputs, extra_false_grad_outputs = _pad_params(
-      true_grad_graph, false_grad_graph,
-      true_grad_intermediates, false_grad_intermediates)
-
-  true_grad_graph.outputs.extend(extra_true_grad_outputs)
-  false_grad_graph.outputs.extend(extra_false_grad_outputs)
-
-  # Create the gradient If op.
-  tensors = gen_functional_ops._if(
-      op.inputs[0],
-      grad_inputs, [t.dtype for t in true_grad_graph.outputs],
-      util.create_new_tf_function(true_grad_graph),
-      util.create_new_tf_function(false_grad_graph),
-      output_shapes=_get_output_shapes(true_grad_graph.outputs,
-                                       false_grad_graph.outputs))
-
+  extra_true_outputs, extra_false_outputs = _pad_params(
+      true_graph, false_graph, true_intermediates, false_intermediates)
+
+  true_graph.outputs.extend(extra_true_outputs)
+  false_graph.outputs.extend(extra_false_outputs)
+
+  # Create the If op.
+  tensors = gen_functional_ops._if(  # pylint: disable=protected-access
+      pred,
+      cond_inputs, [t.dtype for t in true_graph.outputs],
+      util.create_new_tf_function(true_graph),
+      util.create_new_tf_function(false_graph),
+      output_shapes=_get_output_shapes(true_graph.outputs,
+                                       false_graph.outputs),
+      name=name)
+
+  # TODO(b/110167197) this approach requires cond_v2 to have at least 1 output
   util.maybe_set_lowering_attr(tensors[0].op)
 
-  # See comment in cond_v2.
+  # Return identities for each output of the If op, rather than the output of
+  # the If op directly. This makes pruning work if the output of cond() is
+  # fetched: the lowering pass converts the If outputs into IdentityN outputs,
+  # which if fetched will cause all ops in the taken branch to be run (since
+  # it takes all merge ops as input). After lowering, each output identity op
+  # will end up with only the appropriate merge op as input.
+  # TODO(b/79984175): this doesn't have to be a tuple once we covert to the
+  # correct output structure
   tensors = [array_ops.identity(t) for t in tensors]
 
-  # The predicate has no gradient.
-  return [None] + tensors[:num_grad_outputs]
+  return tensors[:num_cond_outputs]
 
 
 def _get_func_graphs(if_op):
-- 
GitLab


From f0c122b979c6a0047eed25a22d2e7120a2ae258d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 09:00:23 -0800
Subject: [PATCH 0768/1554] Exposing CUDA 10 toolchain for image submitted in
 CL/222817562.

PiperOrigin-RevId: 222833255
---
 third_party/toolchains/BUILD | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index a7b4687c02..9da417fd5f 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -35,3 +35,16 @@ platform(
             value:"docker://gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04@%s"
         }""" % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
 )
+
+platform(
+    name = "rbe_cuda10.0-cudnn7-ubuntu14.04",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:linux",
+    ],
+    remote_execution_properties = """
+        properties: {
+            name: "container-image"
+            value:"docker://gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
+        }""" % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
+)
-- 
GitLab


From 284b2783c4676c9d00199a585db9d6d07d9e68be Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 09:10:04 -0800
Subject: [PATCH 0769/1554] Prevent executor accessing a deleted TraceCollector
 by making the latter a global singleton object. Currently, a device tracer
 can stop tracing at any point, while data pipiline is still running and uses
 a cached stale TraceCollector pointer.

PiperOrigin-RevId: 222834820
---
 tensorflow/core/common_runtime/executor.cc    | 16 +++---
 .../core/platform/default/device_tracer.cc    | 55 ++++++++++++++-----
 2 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 77b249c2b4..6b3284b84a 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1239,7 +1239,6 @@ class ExecutorState {
   // Step-local container.
   ScopedStepContainer* step_container_;
   StepStatsCollectorInterface* const stats_collector_;
-  const tracing::TraceCollector* const trace_collector_;
   const tracing::EventCollector* const event_collector_;
   Context context_;
 
@@ -1366,7 +1365,6 @@ ExecutorState::ExecutorState(const Executor::Args& args, ExecutorImpl* impl)
       tensor_store_(args.tensor_store),
       step_container_(args.step_container),
       stats_collector_(args.stats_collector),
-      trace_collector_(tracing::GetTraceCollector()),
       event_collector_(
           tracing::GetEventCollector(tracing::EventCategory::kCompute)),
       context_(ContextKind::kThread),
@@ -1565,7 +1563,6 @@ struct ExecutorState::AsyncState {
 // Returns true if `item` might be traced by the given trace and event
 // collectors. Returns false only if `item` definitely will not be traced.
 bool MightTrace(const NodeItem& item,
-                const tracing::TraceCollector* trace_collector,
                 const tracing::EventCollector* event_collector,
                 bool using_annotations) {
   // Tracing will only be enabled if either `event_collector` is non null,
@@ -1578,6 +1575,7 @@ bool MightTrace(const NodeItem& item,
   if (event_collector != nullptr) {
     return true;
   }
+  auto* trace_collector = tracing::GetTraceCollector();
   if (trace_collector) {
     if (using_annotations) {
       return trace_collector->IsEnabledForAnnotations();
@@ -1762,9 +1760,8 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
         OpKernelContext ctx(&params, item.num_outputs);
         nodestats::SetOpStart(stats);
 
-        if (TF_PREDICT_FALSE(MightTrace(item, trace_collector_,
-                                        event_collector_,
-                                        trace_using_annotations_))) {
+        if (TF_PREDICT_FALSE(
+                MightTrace(item, event_collector_, trace_using_annotations_))) {
           const string& op_name = op_kernel->name();
           tracing::ScopedRegion region(tracing::EventCategory::kCompute,
                                        op_name);
@@ -2048,13 +2045,14 @@ void ExecutorState::PropagateOutputs(const TaggedNode& tagged_node,
                                      TaggedNodeSeq* ready) {
   auto activity_handle =
       [&]() -> std::unique_ptr<tracing::TraceCollector::Handle> {
-    if (TF_PREDICT_FALSE(trace_collector_ != nullptr &&
-                         trace_collector_->IsEnabledForActivities(
+    auto* trace_collector = tracing::GetTraceCollector();
+    if (TF_PREDICT_FALSE(trace_collector != nullptr &&
+                         trace_collector->IsEnabledForActivities(
                              false /* is_expensive */))) {
       const string& op_name = item->kernel->name();
       // Intentionally using ExecutorPropagateOutputs as the first key so that
       // users are aware that it's not the op invocation.
-      return trace_collector_->CreateActivityHandle(
+      return trace_collector->CreateActivityHandle(
           "ExecutorPropagateOutputs",
           strings::StrCat(op_name, "#id=", step_id_, "#"),
           false /* is_expensive */);
diff --git a/tensorflow/core/platform/default/device_tracer.cc b/tensorflow/core/platform/default/device_tracer.cc
index cf8b477b83..8351362e05 100644
--- a/tensorflow/core/platform/default/device_tracer.cc
+++ b/tensorflow/core/platform/default/device_tracer.cc
@@ -297,19 +297,16 @@ CUPTIManager *GetCUPTIManager() {
 // for the duration of the CUPTI API callback.
 TF_STATIC_THREAD_LOCAL_POD(const char *, tls_current_annotation);
 
-class DeviceTracerImpl : public DeviceTracer,
-                         public CUPTIClient,
-                         public tracing::TraceCollector {
+class TraceCollectorImpl : public tracing::TraceCollector {
  public:
-  DeviceTracerImpl(CUPTIManager *cupti_manager);
-  ~DeviceTracerImpl() override;
+  TraceCollectorImpl() { tracing::SetTraceCollector(this); }
 
-  // DeviceTracer interface:
-  Status Start() override;
-  Status Stop() override;
-  Status Collect(StepStatsCollector *collector) override;
+  ~TraceCollectorImpl() override {
+    DCHECK(!active_trace_session_)
+        << "Unexpected active trace session detected. ";
+  }
 
-  // tracing::TraceCollector interface:
+  // Note the method can be called after a call to Stop().
   virtual std::unique_ptr<Handle> CreateAnnotationHandle(
       StringPiece name_part1, StringPiece name_part2) const {
     struct Impl : public tracing::TraceCollector::Handle {
@@ -332,8 +329,7 @@ class DeviceTracerImpl : public DeviceTracer,
   }
 
   bool IsEnabledForAnnotations() const override {
-    // We are always enabled for 'Annotations'.
-    return true;
+    return active_trace_session_.load(std::memory_order_relaxed);
   }
 
   bool IsEnabledForActivities(bool is_expensive) const override {
@@ -341,6 +337,36 @@ class DeviceTracerImpl : public DeviceTracer,
     return false;
   }
 
+  void Start() {
+    DCHECK(!active_trace_session_)
+        << "Unexpected active trace session detected. ";
+    active_trace_session_ = true;
+  }
+
+  void Stop() {
+    DCHECK(active_trace_session_) << "No active trace session detected. ";
+    active_trace_session_ = false;
+  }
+
+ private:
+  std::atomic<bool> active_trace_session_;
+};
+
+TraceCollectorImpl *GlobalDefaultTraceCollector() {
+  static auto *instance = new TraceCollectorImpl();
+  return instance;
+}
+
+class DeviceTracerImpl : public DeviceTracer, public CUPTIClient {
+ public:
+  DeviceTracerImpl(CUPTIManager *cupti_manager);
+  ~DeviceTracerImpl() override;
+
+  // DeviceTracer interface:
+  Status Start() override;
+  Status Stop() override;
+  Status Collect(StepStatsCollector *collector) override;
+
  protected:
   // This callback is used exclusively by CUPTIManager.
   friend class CUPTIManager;
@@ -430,7 +456,7 @@ Status DeviceTracerImpl::Start() {
   }
 
   // Register as a TraceEngine to receive ScopedAnnotations.
-  tracing::SetTraceCollector(this);
+  GlobalDefaultTraceCollector()->Start();
 
   // Intercept launch and memcpy calls to capture the Op name annotation.
   // TODO(pbar) Add callbacks for memcpy variants.
@@ -478,7 +504,8 @@ Status DeviceTracerImpl::Stop() {
     return Status::OK();
   }
   CUPTI_CALL(Unsubscribe(subscriber_));
-  tracing::SetTraceCollector(nullptr);
+  GlobalDefaultTraceCollector()->Stop();
+
   TF_RETURN_IF_ERROR(cupti_manager_->DisableTrace());
   end_walltime_us_ = NowInUsec();
   CUPTI_CALL(GetTimestamp(&end_timestamp_));
-- 
GitLab


From d9ba1a9e6d867a2e9b48558c566e96d062761c84 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 26 Nov 2018 09:42:25 -0800
Subject: [PATCH 0770/1554] Fix requested changes

---
 .../tensorrt/convert/convert_nodes_test.cc    | 51 +++++++++----------
 1 file changed, 23 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 92697e2f87..45e901bf5b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1986,7 +1986,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
   }
 
   // Get nodedef for activation layer.
-  auto get_act_nodedef = [](std::string op_name) -> NodeDef {
+  auto get_act_nodedef = [](string op_name) -> NodeDef {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     if (op_name == "Relu") {
@@ -1999,11 +1999,11 @@ TEST_F(OpConverterTest, ConvertActivation) {
       auto act = ops::Tanh(s.WithOpName("my_act"), input);
       return act.operation.node()->def();
     }
-    EXPECT_TRUE(false);
-    return MakeNodeDef("my_act", "Relu", {});
+    ASSERT_TRUE(false);
+    return NodeDef();
   };
   // Get expected output for activation layer.
-  auto get_act_output = [](std::string op_name, float input) -> float {
+  auto get_act_output = [](string op_name, float input) -> float {
     if (op_name == "Relu") {
       return (input > 0.0f) ? input : 0.0f;
     } else if (op_name == "Sigmoid") {
@@ -2011,32 +2011,27 @@ TEST_F(OpConverterTest, ConvertActivation) {
     } else if (op_name == "Tanh") {
       return std::tanh(input);
     }
-    EXPECT_TRUE(false);
-    return input;
+    ASSERT_TRUE(false);
+    return 0;
   };
 
-  {
-    // Ok.
-    for (std::string op_name : {"Relu", "Sigmoid", "Tanh"}) {
-      Reset();
-      NodeDef node_def = get_act_nodedef(op_name);
-      AddTestTensor("input", {1, 2, 3});
-      RunValidationAndConversion(node_def);
-      TRT_TensorOrWeights output;
-      TF_EXPECT_OK(GetTensorOrWeights("my_act", &output));
-      EXPECT_TRUE(output.is_tensor());
-      EXPECT_TRUE(TrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()))
-          << output.DebugString();
-
-      const std::vector<float> input_data = {-100, -2, -1, 0, 1, 100};
-      std::vector<float> output_data(6);
-      BuildAndRun("input", input_data, "my_act", &output_data);
-      for (int i = 0; i < input_data.size(); i++) {
-        const float expected_output = get_act_output(op_name, input_data[i]);
-        EXPECT_FLOAT_EQ(output_data[i], expected_output)
-          << op_name << "(" << input_data[i] << ") should be equal to "
-          << expected_output;
-      }
+  // Ok.
+  for (string op_name : {"Relu", "Sigmoid", "Tanh"}) {
+    Reset();
+    NodeDef node_def = get_act_nodedef(op_name);
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_act", &output));
+    EXPECT_TRUE(output.is_tensor());
+    ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions());
+
+    const std::vector<float> input_data = {-100, -2, -1, 0, 1, 100};
+    std::vector<float> output_data(6);
+    BuildAndRun("input", input_data, "my_act", &output_data);
+    for (int i = 0; i < input_data.size(); i++) {
+      const float expected_output = get_act_output(op_name, input_data[i]);
+      EXPECT_FLOAT_EQ(output_data[i], expected_output);
     }
   }
 }
-- 
GitLab


From 8e8f2a2a95d15021276cc6123e12066e42ef7f26 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 26 Nov 2018 10:02:59 -0800
Subject: [PATCH 0771/1554] Filter layers from checkpointable data structures
 on demand to avoid sync issues

Previously if a wrapper (e.g. an auto-generated _ListWrapper) got out of sync with the thing it was wrapping, it would return the wrapper's collection of layers instead of layers from the thing it was wrapping.

PiperOrigin-RevId: 222842576
---
 .../contrib/checkpoint/python/containers.py   |  4 ++
 .../checkpointable/data_structures.py         | 47 ++++++++++++-------
 .../checkpointable/data_structures_test.py    | 17 +++++++
 3 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/checkpoint/python/containers.py b/tensorflow/contrib/checkpoint/python/containers.py
index 242c1e8ba4..5418e2605b 100644
--- a/tensorflow/contrib/checkpoint/python/containers.py
+++ b/tensorflow/contrib/checkpoint/python/containers.py
@@ -46,6 +46,10 @@ class UniqueNameTracker(data_structures.CheckpointableDataStructure):
     self._maybe_initialize_checkpointable()
     self._name_counts = {}
 
+  @property
+  def _values(self):
+    return [dep.ref for dep in self._checkpoint_dependencies]
+
   def track(self, checkpointable, base_name):
     """Add a dependency on `checkpointable`.
 
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index c29e5db075..a46a4a1910 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -111,9 +111,6 @@ class CheckpointableDataStructure(base.CheckpointableBase):
   """Base class for data structures which contain checkpointable objects."""
 
   def __init__(self):
-    # An append-only ordered set
-    self._layers = []
-
     self.trainable = True
     self._extra_variables = []
 
@@ -128,21 +125,30 @@ class CheckpointableDataStructure(base.CheckpointableBase):
           ("Only checkpointable objects (such as Layers or Optimizers) may be "
            "stored in a List object. Got %s, which does not inherit from "
            "CheckpointableBase.") % (value,))
-    if (isinstance(value, CheckpointableDataStructure)
-        or layer_utils.is_layer(value)
-        or layer_utils.has_weights(value)):
-      # Check for object-identity rather than with __eq__ to avoid
-      # de-duplicating empty container types. Automatically generated list
-      # wrappers keep things like "[] == []" true, which means "[] in [[]]" is
-      # also true. This becomes not true once one of the lists is mutated.
-      if not any((layer is value for layer in self._layers)):
-        self._layers.append(value)
-        if hasattr(value, "_use_resource_variables"):
-          # In subclassed models, legacy layers (tf.layers) must always use
-          # resource variables.
-          value._use_resource_variables = True  # pylint: disable=protected-access
+    if hasattr(value, "_use_resource_variables"):
+      # In subclassed models, legacy layers (tf.layers) must always use
+      # resource variables.
+      value._use_resource_variables = True  # pylint: disable=protected-access
     return value
 
+  @property
+  def _values(self):
+    """An iterable/sequence which may contain checkpointable objects."""
+    raise NotImplementedError("Abstract method")
+
+  @property
+  def _layers(self):
+    """All Layers and Layer containers, including empty containers."""
+    # Filter objects on demand so that wrapper objects use values from the thing
+    # they're wrapping if out of sync.
+    collected = []
+    for obj in self._values:
+      if (isinstance(obj, CheckpointableDataStructure)
+          or layer_utils.is_layer(obj)
+          or layer_utils.has_weights(obj)):
+        collected.append(obj)
+    return collected
+
   @property
   def layers(self):
     return layer_utils.filter_empty_layer_containers(self._layers)
@@ -265,6 +271,10 @@ class List(CheckpointableDataStructure, collections.Sequence):
   def _name_element(self, index):
     return "%d" % (index,)
 
+  @property
+  def _values(self):
+    return self
+
   def append(self, value):
     """Add a new checkpointable value."""
     value = self._track_value(value, self._name_element(len(self._storage)))
@@ -479,6 +489,11 @@ class Mapping(CheckpointableDataStructure, collections.Mapping):
   def _make_storage(self, *args, **kwargs):
     return dict(*args, **kwargs)
 
+  @property
+  def _values(self):
+    # Sort items deterministically by key
+    return list(zip(*sorted(self.items(), key=lambda it: it[0])))[1]
+
   def _name_element(self, key):
     if not isinstance(key, six.string_types):
       raise TypeError(
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index ff7d1f1d2d..17cbe3192d 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -253,6 +253,13 @@ class ListTests(test.TestCase):
     l.append(1)
     self.assertEqual([1], l_wrapper)
 
+  def testLayerCollectionWithExternalMutation(self):
+    l = []
+    l_wrapper = data_structures._ListWrapper(l)
+    layer = core.Dense(1)
+    l.append(layer)
+    self.assertEqual([layer], l_wrapper.layers)
+
   def testHashing(self):
     has_sequences = set([data_structures.List(),
                          data_structures.List()])
@@ -324,6 +331,16 @@ class MappingTests(test.TestCase):
     with self.assertRaises(TypeError):
       mapping[1] = data_structures.List()
 
+  def testLayerCollectionWithExternalMutation(self):
+    d = {}
+    root = tracking.Checkpointable()
+    root.wrapper = d
+    layer1 = core.Dense(1)
+    layer2 = core.Dense(1)
+    d["a"] = layer1
+    d["b"] = layer2
+    self.assertEqual([layer1, layer2], root.wrapper.layers)
+
   def testHashing(self):
     has_mappings = set([data_structures.Mapping(),
                         data_structures.Mapping()])
-- 
GitLab


From ddb47e779808a3bd5cbd1948705c267548acc209 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 10:09:44 -0800
Subject: [PATCH 0772/1554] Publish cumulative graph run time and count to
 Streamz.

We record the time at the beginning and end of running DirectSession::RunInternal / GraphMgr::ExecuteAsync(for distributed learning) and calculate the difference. This way of publishing graph run time intends to work around the GPU/TPU OOM issue when calculate step time from Xprof StepStats.

PiperOrigin-RevId: 222843844
---
 tensorflow/core/BUILD                         | 10 +++++
 .../core/common_runtime/direct_session.cc     |  3 ++
 tensorflow/core/common_runtime/metrics.cc     | 40 +++++++++++++++++++
 tensorflow/core/common_runtime/metrics.h      | 27 +++++++++++++
 tensorflow/core/distributed_runtime/BUILD     |  1 +
 .../core/distributed_runtime/graph_mgr.cc     | 21 ++++++----
 6 files changed, 94 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/metrics.cc
 create mode 100644 tensorflow/core/common_runtime/metrics.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 2a8c2718ed..5e1d93cfa2 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3022,6 +3022,15 @@ cc_library(
     deps = [":lib_internal"],
 )
 
+tf_cuda_library(
+    name = "metrics",
+    srcs = ["common_runtime/metrics.cc"],
+    hdrs = ["common_runtime/metrics.h"],
+    deps = [
+        ":lib",
+    ],
+)
+
 tf_cuda_library(
     name = "direct_session_internal",
     srcs = ["common_runtime/direct_session.cc"],
@@ -3038,6 +3047,7 @@ tf_cuda_library(
         ":graph",
         ":lib",
         ":lib_internal",
+        ":metrics",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/debug:debug_graph_utils",
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 40b7071f40..178469db51 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/memory_types.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
@@ -462,6 +463,7 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
                                   CallFrameInterface* call_frame,
                                   ExecutorsAndKeys* executors_and_keys,
                                   RunMetadata* run_metadata) {
+  const uint64 start_time_usecs = Env::Default()->NowMicros();
   string session_id_meta = strings::StrCat("SessionRun #id=", step_id, "#");
   tracing::ScopedActivity activity(session_id_meta);
 
@@ -716,6 +718,7 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
       exec_and_lib.graph->ToGraphDef(partition_graph_def);
     }
   }
+  UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/metrics.cc b/tensorflow/core/common_runtime/metrics.cc
new file mode 100644
index 0000000000..f4c94ed7ec
--- /dev/null
+++ b/tensorflow/core/common_runtime/metrics.cc
@@ -0,0 +1,40 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/metrics.h"
+#include "tensorflow/core/lib/monitoring/counter.h"
+
+namespace tensorflow {
+
+namespace {
+
+auto* graph_runs = monitoring::Counter<0>::New(
+    "/tensorflow/core/graph_runs",
+    "The number of graph executions used to collect "
+    "/tensorflow/core/graph_run_time_usecs");
+
+auto* graph_run_time_usecs = monitoring::Counter<0>::New(
+    "/tensorflow/core/graph_run_time_usecs",
+    "The total time spent on executing graphs in microseconds.");
+}  // namespace
+
+void UpdateGraphExecTime(const uint64 running_time_usecs) {
+  if (running_time_usecs > 0) {
+    graph_runs->GetCell()->IncrementBy(1);
+    graph_run_time_usecs->GetCell()->IncrementBy(running_time_usecs);
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/metrics.h b/tensorflow/core/common_runtime/metrics.h
new file mode 100644
index 0000000000..d3430c9f03
--- /dev/null
+++ b/tensorflow/core/common_runtime/metrics.h
@@ -0,0 +1,27 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
+
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+void UpdateGraphExecTime(const uint64 running_time_usecs);
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 818324746f..cd9e58ac39 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -425,6 +425,7 @@ cc_library(
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:worker_proto_cc",
         "//tensorflow/core/debug",
diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc
index 3944668028..ee5823e314 100644
--- a/tensorflow/core/distributed_runtime/graph_mgr.cc
+++ b/tensorflow/core/distributed_runtime/graph_mgr.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/graph_mgr.h"
 
+#include <chrono>  // NOLINT(build/c++11)
 #include <vector>
 
 #include "tensorflow/core/common_runtime/build_graph_options.h"
@@ -25,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/memory_types.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/common_runtime/rendezvous_util.h"
@@ -386,6 +388,7 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id,
                             MutableRunGraphResponseWrapper* response,
                             CancellationManager* cancellation_manager,
                             const NamedTensors& in, StatusCallback done) {
+  const uint64 start_time_usecs = Env::Default()->NowMicros();
   // Lookup an item. Holds one ref while executing.
   Item* item = nullptr;
   {
@@ -443,14 +446,16 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id,
     return;
   }
 
-  StartParallelExecutors(handle, step_id, item, rendezvous, ce_handle,
-                         collector, cost_graph, cancellation_manager,
-                         [item, rendezvous, ce_handle, done](const Status& s) {
-                           done(s);
-                           rendezvous->Unref();
-                           item->Unref();
-                           delete ce_handle;
-                         });
+  StartParallelExecutors(
+      handle, step_id, item, rendezvous, ce_handle, collector, cost_graph,
+      cancellation_manager,
+      [item, rendezvous, ce_handle, done, start_time_usecs](const Status& s) {
+        done(s);
+        UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
+        rendezvous->Unref();
+        item->Unref();
+        delete ce_handle;
+      });
 }
 
 void GraphMgr::StartParallelExecutors(const string& handle, int64 step_id,
-- 
GitLab


From 7f2309d633a3ab43c4b78e7569fa987ba94f4192 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 10:29:22 -0800
Subject: [PATCH 0773/1554] Added training hooks to train ops.

PiperOrigin-RevId: 222846882
---
 .../estimator/python/gan_estimator_impl.py    | 15 ++--
 .../estimator/python/gan_estimator_test.py    | 35 ++++++++-
 tensorflow/contrib/gan/python/namedtuples.py  | 12 ++-
 tensorflow/contrib/gan/python/train.py        | 15 +++-
 tensorflow/contrib/gan/python/train_test.py   | 75 +++++++++++++++++++
 5 files changed, 140 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
index 219cc199d7..3593b501bb 100644
--- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
@@ -113,7 +113,8 @@ class GANEstimator(estimator.Estimator):
                add_summaries=None,
                use_loss_summaries=True,
                config=None,
-               warm_start_from=None):
+               warm_start_from=None,
+               is_chief=True):
     """Initializes a GANEstimator instance.
 
     Args:
@@ -154,6 +155,8 @@ class GANEstimator(estimator.Estimator):
       config: `RunConfig` object to configure the runtime settings.
       warm_start_from: A filepath to a checkpoint or saved model, or a
         WarmStartSettings object to configure initialization.
+      is_chief: Whether or not this Estimator is running on a chief or worker.
+        Needs to be set appropriately if using SyncReplicasOptimizers.
 
     Raises:
       ValueError: If loss functions aren't callable.
@@ -187,7 +190,7 @@ class GANEstimator(estimator.Estimator):
       return _get_estimator_spec(
           mode, gan_model, generator_loss_fn, discriminator_loss_fn,
           get_eval_metric_ops_fn, generator_optimizer, discriminator_optimizer,
-          get_hooks_fn, use_loss_summaries)
+          get_hooks_fn, use_loss_summaries, is_chief)
 
     super(GANEstimator, self).__init__(
         model_fn=_model_fn, model_dir=model_dir, config=config,
@@ -215,7 +218,7 @@ def _get_gan_model(
 def _get_estimator_spec(
     mode, gan_model, generator_loss_fn, discriminator_loss_fn,
     get_eval_metric_ops_fn, generator_optimizer, discriminator_optimizer,
-    get_hooks_fn=None, use_loss_summaries=True):
+    get_hooks_fn=None, use_loss_summaries=True, is_chief=True):
   """Get the EstimatorSpec for the current mode."""
   if mode == model_fn_lib.ModeKeys.PREDICT:
     estimator_spec = model_fn_lib.EstimatorSpec(
@@ -236,7 +239,7 @@ def _get_estimator_spec(
               else discriminator_optimizer)
       get_hooks_fn = get_hooks_fn or tfgan_train.get_sequential_train_hooks()
       estimator_spec = _get_train_estimator_spec(
-          gan_model, gan_loss, gopt, dopt, get_hooks_fn)
+          gan_model, gan_loss, gopt, dopt, get_hooks_fn, is_chief=is_chief)
 
   return estimator_spec
 
@@ -321,11 +324,11 @@ def _get_eval_estimator_spec(gan_model, gan_loss, get_eval_metric_ops_fn=None,
 
 def _get_train_estimator_spec(
     gan_model, gan_loss, generator_optimizer, discriminator_optimizer,
-    get_hooks_fn, train_op_fn=tfgan_train.gan_train_ops):
+    get_hooks_fn, train_op_fn=tfgan_train.gan_train_ops, is_chief=True):
   """Return an EstimatorSpec for the train case."""
   scalar_loss = gan_loss.generator_loss + gan_loss.discriminator_loss
   train_ops = train_op_fn(gan_model, gan_loss, generator_optimizer,
-                          discriminator_optimizer)
+                          discriminator_optimizer, is_chief=is_chief)
   training_hooks = get_hooks_fn(train_ops)
   return model_fn_lib.EstimatorSpec(
       loss=scalar_loss,
diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py
index 3d6bdab0ad..bc9021050b 100644
--- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py
@@ -48,6 +48,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import input as input_lib
 from tensorflow.python.training import learning_rate_decay
+from tensorflow.python.training import sync_replicas_optimizer
 from tensorflow.python.training import training
 from tensorflow.python.training import training_util
 
@@ -82,7 +83,7 @@ class GetGANModelTest(test.TestCase, parameterized.TestCase):
 
     self.assertEqual(generator_inputs, gan_model.generator_inputs)
     self.assertIsNotNone(gan_model.generated_data)
-    self.assertEqual(2, len(gan_model.generator_variables))  # 1 FC layer
+    self.assertLen(gan_model.generator_variables, 2)  # 1 FC layer
     self.assertIsNotNone(gan_model.generator_fn)
     if mode == model_fn_lib.ModeKeys.PREDICT:
       self.assertIsNone(gan_model.real_data)
@@ -95,7 +96,7 @@ class GetGANModelTest(test.TestCase, parameterized.TestCase):
       self.assertIsNotNone(gan_model.real_data)
       self.assertIsNotNone(gan_model.discriminator_real_outputs)
       self.assertIsNotNone(gan_model.discriminator_gen_outputs)
-      self.assertEqual(2, len(gan_model.discriminator_variables))  # 1 FC layer
+      self.assertLen(gan_model.discriminator_variables, 2)  # 1 FC layer
       self.assertIsNotNone(gan_model.discriminator_scope)
       self.assertIsNotNone(gan_model.discriminator_fn)
 
@@ -121,6 +122,7 @@ def get_dummy_gan_model():
 
 
 def dummy_loss_fn(gan_model, add_summaries=True):
+  del add_summaries
   return math_ops.reduce_sum(gan_model.discriminator_real_outputs -
                              gan_model.discriminator_gen_outputs)
 
@@ -168,6 +170,35 @@ class GetEstimatorSpecTest(test.TestCase, parameterized.TestCase):
       self.assertShapeEqual(np.array(0), spec.loss)  # must be a scalar
       self.assertIsNotNone(spec.eval_metric_ops)
 
+  def test_get_sync_estimator_spec(self):
+    """Make sure spec is loaded with sync hooks for sync opts."""
+
+    def get_sync_optimizer():
+      return sync_replicas_optimizer.SyncReplicasOptimizer(
+          training.GradientDescentOptimizer(learning_rate=1.0),
+          replicas_to_aggregate=1)
+
+    with ops.Graph().as_default():
+      self._gan_model = get_dummy_gan_model()
+      g_opt = get_sync_optimizer()
+      d_opt = get_sync_optimizer()
+
+      spec = estimator._get_estimator_spec(
+          model_fn_lib.ModeKeys.TRAIN,
+          self._gan_model,
+          generator_loss_fn=dummy_loss_fn,
+          discriminator_loss_fn=dummy_loss_fn,
+          get_eval_metric_ops_fn=get_metrics,
+          generator_optimizer=g_opt,
+          discriminator_optimizer=d_opt)
+
+      self.assertLen(spec.training_hooks, 4)
+      sync_opts = [
+          hook._sync_optimizer for hook in spec.training_hooks if
+          isinstance(hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)]
+      self.assertLen(sync_opts, 2)
+      self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt)))
+
 
 # TODO(joelshor): Add pandas test.
 class GANEstimatorIntegrationTest(test.TestCase):
diff --git a/tensorflow/contrib/gan/python/namedtuples.py b/tensorflow/contrib/gan/python/namedtuples.py
index b9ac1bf151..969b68449d 100644
--- a/tensorflow/contrib/gan/python/namedtuples.py
+++ b/tensorflow/contrib/gan/python/namedtuples.py
@@ -213,7 +213,8 @@ class GANTrainOps(
     collections.namedtuple('GANTrainOps', (
         'generator_train_op',
         'discriminator_train_op',
-        'global_step_inc_op'
+        'global_step_inc_op',
+        'train_hooks'
     ))):
   """GANTrainOps contains the training ops.
 
@@ -221,8 +222,17 @@ class GANTrainOps(
     generator_train_op: Op that performs a generator update step.
     discriminator_train_op: Op that performs a discriminator update step.
     global_step_inc_op: Op that increments the shared global step.
+    train_hooks: a list or tuple containing hooks related to training that need
+      to be populated when training ops are instantiated. Used primarily for
+      sync hooks.
   """
 
+  def __new__(cls, generator_train_op, discriminator_train_op,
+              global_step_inc_op, train_hooks=()):
+    return super(GANTrainOps, cls).__new__(cls, generator_train_op,
+                                           discriminator_train_op,
+                                           global_step_inc_op, train_hooks)
+
 
 class GANTrainSteps(
     collections.namedtuple('GANTrainSteps', (
diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index cf5b9d9476..4c7bee41b3 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -924,6 +924,7 @@ def gan_train_ops(
     generator_optimizer,
     discriminator_optimizer,
     check_for_unused_update_ops=True,
+    is_chief=True,
     # Optional args to pass directly to the `create_train_op`.
     **kwargs):
   """Returns GAN train ops.
@@ -939,6 +940,8 @@ def gan_train_ops(
     discriminator_optimizer: The optimizer for the discriminator updates.
     check_for_unused_update_ops: If `True`, throws an exception if there are
       update ops outside of the generator or discriminator scopes.
+    is_chief: Specifies whether or not the training is being run by the primary
+      replica during replica training.
     **kwargs: Keyword args to pass directly to
       `training.create_train_op` for both the generator and
       discriminator train op.
@@ -980,6 +983,9 @@ def gan_train_ops(
       kwargs, model.generator_scope.name, model.discriminator_scope.name,
       check_for_unused_update_ops)
 
+  # Get the sync hooks if these are needed.
+  sync_hooks = []
+
   generator_global_step = None
   if isinstance(generator_optimizer,
                 sync_replicas_optimizer.SyncReplicasOptimizer):
@@ -995,6 +1001,7 @@ def gan_train_ops(
         trainable=False,
         collections=[ops.GraphKeys.GLOBAL_VARIABLES])
     gen_update_ops += [generator_global_step.assign(global_step)]
+    sync_hooks.append(generator_optimizer.make_session_run_hook(is_chief))
   with ops.name_scope('generator_train'):
     gen_train_op = training.create_train_op(
         total_loss=loss.generator_loss,
@@ -1016,6 +1023,7 @@ def gan_train_ops(
         trainable=False,
         collections=[ops.GraphKeys.GLOBAL_VARIABLES])
     dis_update_ops += [discriminator_global_step.assign(global_step)]
+    sync_hooks.append(discriminator_optimizer.make_session_run_hook(is_chief))
   with ops.name_scope('discriminator_train'):
     disc_train_op = training.create_train_op(
         total_loss=loss.discriminator_loss,
@@ -1025,7 +1033,8 @@ def gan_train_ops(
         update_ops=dis_update_ops,
         **kwargs)
 
-  return namedtuples.GANTrainOps(gen_train_op, disc_train_op, global_step_inc)
+  return namedtuples.GANTrainOps(gen_train_op, disc_train_op, global_step_inc,
+                                 sync_hooks)
 
 
 # TODO(joelshor): Implement a dynamic GAN train loop, as in `Real-Time Adaptive
@@ -1066,7 +1075,7 @@ def get_sequential_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
                                      train_steps.generator_train_steps)
     discriminator_hook = RunTrainOpsHook(train_ops.discriminator_train_op,
                                          train_steps.discriminator_train_steps)
-    return [generator_hook, discriminator_hook]
+    return [generator_hook, discriminator_hook] + list(train_ops.train_hooks)
 
   return get_hooks
 
@@ -1126,7 +1135,7 @@ def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
     g_hook = RunTrainOpsHook(g_op, num_g_steps)
     d_hook = RunTrainOpsHook(d_op, num_d_steps)
 
-    return [joint_hook, g_hook, d_hook]
+    return [joint_hook, g_hook, d_hook] + list(train_ops.train_hooks)
 
   return get_hooks
 
diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index e8c24eea3d..841f25cd7f 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -836,6 +836,9 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase):
 
     self.assertIsInstance(train_ops, namedtuples.GANTrainOps)
 
+    # Make sure there are no training hooks populated accidentally.
+    self.assertEmpty(train_ops.train_hooks)
+
   # TODO(joelshor): Add a test to check that custom update op is run.
   @parameterized.named_parameters(
       ('gan', create_gan_model, False),
@@ -925,6 +928,14 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase):
     # No new trainable variables should have been added.
     self.assertLen(variables_lib.get_trainable_variables(), num_trainable_vars)
 
+    # Sync hooks should be populated in the GANTrainOps.
+    self.assertLen(train_ops.train_hooks, 2)
+    for hook in train_ops.train_hooks:
+      self.assertIsInstance(
+          hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)
+    sync_opts = [hook._sync_optimizer for hook in train_ops.train_hooks]
+    self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt)))
+
     g_sync_init_op = g_opt.get_init_tokens_op(num_tokens=1)
     d_sync_init_op = d_opt.get_init_tokens_op(num_tokens=1)
 
@@ -958,6 +969,32 @@ class GANTrainOpsTest(test.TestCase, parameterized.TestCase):
       coord.request_stop()
       coord.join(g_threads + d_threads)
 
+  @parameterized.named_parameters(
+      ('is_chief', True),
+      ('is_not_chief', False),
+  )
+  def test_is_chief_in_train_hooks(self, is_chief):
+    """Make sure is_chief is propagated correctly to sync hooks."""
+    model = create_gan_model()
+    loss = train.gan_loss(model)
+    g_opt = get_sync_optimizer()
+    d_opt = get_sync_optimizer()
+    train_ops = train.gan_train_ops(
+        model,
+        loss,
+        g_opt,
+        d_opt,
+        is_chief=is_chief,
+        summarize_gradients=True,
+        colocate_gradients_with_ops=True)
+
+    self.assertLen(train_ops.train_hooks, 2)
+    for hook in train_ops.train_hooks:
+      self.assertIsInstance(
+          hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)
+    is_chief_list = [hook._is_chief for hook in train_ops.train_hooks]
+    self.assertListEqual(is_chief_list, [is_chief, is_chief])
+
 
 class GANTrainTest(test.TestCase, parameterized.TestCase):
   """Tests for `gan_train`."""
@@ -1035,6 +1072,44 @@ class GANTrainTest(test.TestCase, parameterized.TestCase):
     self.assertTrue(np.isscalar(final_loss))
     self.assertEqual(17.0, final_loss)
 
+  @parameterized.named_parameters(
+      ('gan', create_gan_model),
+      ('callable_gan', create_callable_gan_model),
+      ('infogan', create_infogan_model),
+      ('callable_infogan', create_callable_infogan_model),
+      ('acgan', create_acgan_model),
+      ('callable_acgan', create_callable_acgan_model),
+  )
+  def test_train_hooks_exist_in_get_hooks_fn(self, create_gan_model_fn):
+    model = create_gan_model_fn()
+    loss = train.gan_loss(model)
+
+    g_opt = get_sync_optimizer()
+    d_opt = get_sync_optimizer()
+    train_ops = train.gan_train_ops(
+        model,
+        loss,
+        g_opt,
+        d_opt,
+        summarize_gradients=True,
+        colocate_gradients_with_ops=True)
+
+    sequential_train_hooks = train.get_sequential_train_hooks()(train_ops)
+    self.assertLen(sequential_train_hooks, 4)
+    sync_opts = [
+        hook._sync_optimizer for hook in sequential_train_hooks if
+        isinstance(hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)]
+    self.assertLen(sync_opts, 2)
+    self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt)))
+
+    joint_train_hooks = train.get_joint_train_hooks()(train_ops)
+    self.assertLen(joint_train_hooks, 5)
+    sync_opts = [
+        hook._sync_optimizer for hook in joint_train_hooks if
+        isinstance(hook, sync_replicas_optimizer._SyncReplicasOptimizerHook)]
+    self.assertLen(sync_opts, 2)
+    self.assertSetEqual(frozenset(sync_opts), frozenset((g_opt, d_opt)))
+
 
 class PatchGANTest(test.TestCase, parameterized.TestCase):
   """Tests that functions work on PatchGAN style output."""
-- 
GitLab


From 9fca5f0778c2a0b86f6ecd925256e8917c6030b3 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 26 Nov 2018 10:35:48 -0800
Subject: [PATCH 0774/1554] [tf.data] Adding eager coverage to dataset tests.

PiperOrigin-RevId: 222847973
---
 .../kernel_tests/batch_dataset_op_test.py     | 357 +++++++-----------
 .../kernel_tests/cache_dataset_op_test.py     | 357 +++++++-----------
 .../concatenate_dataset_op_test.py            |  62 ++-
 .../dataset_constructor_op_test.py            | 324 +++++++---------
 .../data/kernel_tests/dataset_ops_test.py     |  13 +-
 .../kernel_tests/filter_dataset_op_test.py    | 149 +++-----
 .../python/data/kernel_tests/inputs_test.py   |   2 +
 .../list_files_dataset_op_test.py             | 281 +++++---------
 .../data/kernel_tests/optional_ops_test.py    |  12 +-
 .../kernel_tests/prefetch_dataset_op_test.py  |  29 +-
 .../kernel_tests/sequence_dataset_op_test.py  | 217 ++++-------
 .../kernel_tests/shard_dataset_op_test.py     |  53 +--
 .../kernel_tests/shuffle_dataset_op_test.py   | 240 ++++++------
 .../python/data/kernel_tests/test_base.py     |  26 +-
 .../data/kernel_tests/zip_dataset_op_test.py  | 134 +++----
 15 files changed, 886 insertions(+), 1370 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
index e8decb9ad0..94c37a0f2c 100644
--- a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
@@ -26,11 +26,13 @@ import numpy as np
 from tensorflow.python.client import session
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import string_ops
@@ -38,6 +40,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
@@ -62,59 +65,42 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
                   np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
                   np.array(37.0) * np.arange(7))
 
-    count_t = array_ops.placeholder(dtypes.int64, shape=[])
-    batch_size_t = array_ops.placeholder(dtypes.int64, shape=[])
-    drop_remainder_t = array_ops.placeholder(dtypes.bool, shape=[])
-
     def _map_fn(x, y, z):
       return math_ops.square(x), math_ops.square(y), math_ops.square(z)
 
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn)
-        .repeat(count).batch(batch_size,
-                             drop_remainder).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
+        _map_fn).repeat(count).batch(batch_size, drop_remainder)
+    get_next = self.getNext(dataset)
 
     if drop_remainder:
       dim0 = batch_size
     else:
       dim0 = None
-    self.assertEqual([[dim0] + list(c.shape[1:]) for c in components],
-                     [t.shape.as_list() for t in get_next])
-
-    with self.cached_session() as sess:
-      sess.run(
-          init_op,
-          feed_dict={
-              count_t: count,
-              batch_size_t: batch_size,
-              drop_remainder_t: drop_remainder
-          })
-      num_full_batches = (count * 7) // batch_size
-      for i in range(num_full_batches):
-        result = sess.run(get_next)
-        for component, result_component in zip(components, result):
-          for j in range(batch_size):
-            self.assertAllEqual(component[(i * batch_size + j) % 7]**2,
-                                result_component[j])
-      if not drop_remainder and (count * 7) % batch_size > 0:
-        result = sess.run(get_next)
-        for component, result_component in zip(components, result):
-          for j in range((count * 7) % batch_size):
-            self.assertAllEqual(
-                component[(num_full_batches * batch_size + j) % 7]**2,
-                result_component[j])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertEqual(
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)],
+        [[dim0] + list(c.shape[1:]) for c in components])
+
+    num_full_batches = (count * 7) // batch_size
+    for i in range(num_full_batches):
+      result = self.evaluate(get_next())
+      for component, result_component in zip(components, result):
+        for j in range(batch_size):
+          self.assertAllEqual(component[(i * batch_size + j) % 7]**2,
+                              result_component[j])
+    if not drop_remainder and (count * 7) % batch_size > 0:
+      result = self.evaluate(get_next())
+      for component, result_component in zip(components, result):
+        for j in range((count * 7) % batch_size):
+          self.assertAllEqual(
+              component[(num_full_batches * batch_size + j) % 7]**2,
+              result_component[j])
+    with self.assertRaises(errors.OutOfRangeError):
+      result = self.evaluate(get_next())
 
   def testBatchDatasetInvalidBatchSize(self):
-    iterator = (dataset_ops.Dataset.range(10).batch(0).make_one_shot_iterator())
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(get_next)
+    dataset = (dataset_ops.Dataset.range(10).batch(0))
+    self.assertDatasetProduces(
+        dataset, expected_error=(errors.InvalidArgumentError, ''))
 
   def testBatchSparse(self):
 
@@ -122,23 +108,14 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return sparse_tensor.SparseTensorValue(
           indices=[[0]], values=(i * [1]), dense_shape=[1])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).batch(
-        5).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(2):
-        actual = sess.run(get_next)
-        expected = sparse_tensor.SparseTensorValue(
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5)
+    expected_output = [
+        sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
-            dense_shape=[5, 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+            dense_shape=[5, 1]) for i in range(2)
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testBatchSparseWithDifferentDenseShapes(self):
 
@@ -149,29 +126,21 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           values=array_ops.fill([math_ops.to_int32(i)], i),
           dense_shape=[i])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).batch(
-        5).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(2):
-        actual = sess.run(get_next)
-        expected_indices = []
-        expected_values = []
-        for j in range(5):
-          for k in range(i * 5 + j):
-            expected_indices.append([j, k])
-            expected_values.append(i * 5 + j)
-        expected = sparse_tensor.SparseTensorValue(
-            indices=expected_indices,
-            values=expected_values,
-            dense_shape=[5, (i + 1) * 5 - 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5)
+    expected_output = []
+    for i in range(2):
+      expected_indices = []
+      expected_outputs = []
+      for j in range(5):
+        for k in range(i * 5 + j):
+          expected_indices.append([j, k])
+          expected_outputs.append(i * 5 + j)
+      expected_output.append(
+          sparse_tensor.SparseTensorValue(
+              indices=expected_indices,
+              values=expected_outputs,
+              dense_shape=[5, (i + 1) * 5 - 1]))
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testNestedBatchSparse(self):
 
@@ -179,23 +148,15 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return sparse_tensor.SparseTensorValue(
           indices=[[0]], values=(i * [1]), dense_shape=[1])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).batch(5).batch(
-        2).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      actual = sess.run(get_next)
-      expected = sparse_tensor.SparseTensorValue(
-          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0],
-                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]],
-          values=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
-          dense_shape=[2, 5, 1])
-      self.assertTrue(sparse_tensor.is_sparse(actual))
-      self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5).batch(2)
+    expected_output = [
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0],
+                     [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]],
+            values=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
+            dense_shape=[2, 5, 1])
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testBatchShapeError(self):
 
@@ -204,25 +165,22 @@ class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       yield [4.0, 5.0, 6.0]
       yield [7.0, 8.0, 9.0, 10.0]
 
-    iterator = (
+    dataset = (
         dataset_ops.Dataset.from_generator(
-            generator, dtypes.float32, output_shapes=[None]).batch(3)
-        .make_initializable_iterator())
-    next_element = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          r'Cannot batch tensors with different shapes in component 0. '
-          r'First element had shape \[3\] and element 2 had shape \[4\].'):
-        sess.run(next_element)
+            generator, dtypes.float32, output_shapes=[None]).batch(3))
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            r'Cannot batch tensors with different shapes in component 0. First '
+            r'element had shape \[3\] and element 2 had shape \[4\].'))
 
 
 def _random_seq_lens(count):
   return np.random.randint(20, size=(count,)).astype(np.int32)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
@@ -243,125 +201,83 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         size does not divide number of inputs evenly
     """
 
-    seq_lens_t = array_ops.placeholder(dtypes.int32, shape=[None])
-    batch_size_t = array_ops.placeholder(dtypes.int64, shape=[])
-    padded_shapes_t = array_ops.placeholder(dtypes.int64, shape=[1])
-    drop_remainder_t = array_ops.placeholder(dtypes.bool, shape=[])
-
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(seq_lens_t)
-        .map(lambda x: array_ops.fill([x], x)).padded_batch(
-            batch_size=batch_size_t,
-            drop_remainder=drop_remainder_t,
-            padded_shapes=padded_shapes_t).make_initializable_iterator())
-
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(
-          init_op,
-          feed_dict={
-              seq_lens_t: seq_lens,
-              batch_size_t: batch_size,
-              padded_shapes_t: padded_shapes,
-              drop_remainder_t: drop_remainder,
-          })
-
-      num_full_batches = len(seq_lens) // batch_size
-
-      for i in range(num_full_batches):
-        result = sess.run(get_next)
-        padded_len = padded_shapes[0]
-        if padded_len is None or padded_len == -1:
-          padded_len = np.max(result) if result.size > 0 else 0
-        self.assertEqual((batch_size, padded_len), result.shape)
-        for j in range(batch_size):
-          seq_len = seq_lens[(i * batch_size) + j]
-          self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len)
-          self.assertAllEqual(result[j, seq_len:],
-                              [0] * (padded_len - seq_len))
-
-      if not drop_remainder and len(seq_lens) % batch_size > 0:
-        result = sess.run(get_next)
+    dataset = dataset_ops.Dataset.from_tensor_slices(seq_lens).map(
+        lambda x: array_ops.fill([x], x)).padded_batch(
+            batch_size=batch_size,
+            drop_remainder=drop_remainder,
+            padded_shapes=padded_shapes)
+
+    num_full_batches = len(seq_lens) // batch_size
+    get_next = self.getNext(dataset)
+    for i in range(num_full_batches):
+      result = self.evaluate(get_next())
+      padded_len = padded_shapes[0]
+      if padded_len is None or padded_len == -1:
         padded_len = np.max(result) if result.size > 0 else 0
-        self.assertEqual((len(seq_lens) % batch_size, padded_len),
-                         result.shape)
-        for j in range(len(seq_lens) % batch_size):
-          seq_len = seq_lens[num_full_batches * batch_size + j]
-          self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len)
-          self.assertAllEqual(result[j, seq_len:],
-                              [0] * (padded_len - seq_len))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+      self.assertEqual((batch_size, padded_len), result.shape)
+      for j in range(batch_size):
+        seq_len = seq_lens[(i * batch_size) + j]
+        self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len)
+        self.assertAllEqual(result[j, seq_len:], [0] * (padded_len - seq_len))
+
+    if not drop_remainder and len(seq_lens) % batch_size > 0:
+      result = self.evaluate(get_next())
+      padded_len = np.max(result) if result.size > 0 else 0
+      self.assertEqual((len(seq_lens) % batch_size, padded_len), result.shape)
+      for j in range(len(seq_lens) % batch_size):
+        seq_len = seq_lens[num_full_batches * batch_size + j]
+        self.assertAllEqual(result[j, :seq_len], [seq_len] * seq_len)
+        self.assertAllEqual(result[j, seq_len:], [0] * (padded_len - seq_len))
+
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testPaddedBatchShortPadding(self):
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices([6, 5, 5, 5, 5])
-        .map(lambda x: array_ops.fill([x], x)).padded_batch(
-            batch_size=4, padded_shapes=[5]).make_one_shot_iterator())
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      with self.assertRaises(errors.DataLossError):
-        sess.run(get_next)
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(
+            [6, 5, 5, 5, 5]).map(lambda x: array_ops.fill([x], x)).padded_batch(
+                batch_size=4, padded_shapes=[5]))
+    self.assertDatasetProduces(
+        dataset, expected_error=(errors.DataLossError, ''))
 
   def testPaddedBatchEmptyTensors(self):
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices([0, 0, 0, 0])
-        .map(lambda x: array_ops.fill([x], x)).padded_batch(
-            batch_size=4, padded_shapes=[-1]).make_one_shot_iterator())
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      result = sess.run(get_next)
-      self.assertAllEqual([[], [], [], []], result)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(
+            [0, 0, 0, 0]).map(lambda x: array_ops.fill([x], x)).padded_batch(
+                batch_size=4, padded_shapes=[-1]))
+    self.assertDatasetProduces(dataset, expected_output=[[[], [], [], []]])
 
   def testPaddedBatchDatasetNonDefaultPadding(self):
-    seq_lens = array_ops.placeholder(dtypes.int32, shape=[None])
-    padded_shape = array_ops.placeholder(dtypes.int64, shape=[1])
 
     def fill_tuple(x):
       filled = array_ops.fill([x], x)
       return (filled, string_ops.as_string(filled))
 
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(seq_lens).map(fill_tuple)
+    random_seq_lens = np.random.randint(20, size=(32,)).astype(np.int32)
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(random_seq_lens).map(fill_tuple)
         .padded_batch(
-            4,
-            padded_shapes=(padded_shape, padded_shape),
-            padding_values=(-1, '<end>')).make_initializable_iterator())
-
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # Test with random sequence lengths, and max padding.
-      random_seq_lens = np.random.randint(20, size=(32,)).astype(np.int32)
-      sess.run(
-          init_op, feed_dict={
-              padded_shape: [-1],
-              seq_lens: random_seq_lens
-          })
-      for i in range(8):
-        result = sess.run(get_next)
-        padded_len = np.max(result[0])
-        self.assertEqual((4, padded_len), result[0].shape)
-        self.assertEqual((4, padded_len), result[1].shape)
-        for j in range(4):
-          seq_len = random_seq_lens[(i * 4) + j]
-          self.assertAllEqual(result[0][j, :seq_len], [seq_len] * seq_len)
-          self.assertAllEqual(result[0][j, seq_len:],
-                              [-1] * (padded_len - seq_len))
-          self.assertAllEqual(result[1][j, :seq_len],
-                              [compat.as_bytes(str(seq_len))] * seq_len)
-          self.assertAllEqual(result[1][j, seq_len:],
-                              [b'<end>'] * (padded_len - seq_len))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+            4, padded_shapes=([-1], [-1]), padding_values=(-1, '<end>')))
+
+    get_next = self.getNext(dataset)
+    for i in range(8):
+      result = self.evaluate(get_next())
+      padded_len = np.max(result[0])
+      self.assertEqual((4, padded_len), result[0].shape)
+      self.assertEqual((4, padded_len), result[1].shape)
+      for j in range(4):
+        seq_len = random_seq_lens[(i * 4) + j]
+        self.assertAllEqual(result[0][j, :seq_len], [seq_len] * seq_len)
+        self.assertAllEqual(result[0][j, seq_len:],
+                            [-1] * (padded_len - seq_len))
+        self.assertAllEqual(result[1][j, :seq_len],
+                            [compat.as_bytes(str(seq_len))] * seq_len)
+        self.assertAllEqual(result[1][j, seq_len:],
+                            [b'<end>'] * (padded_len - seq_len))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testPaddedBatchDatasetUnicode(self):
     # See GitHub issue 16149
@@ -377,11 +293,10 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         (tensor_shape.TensorShape([None]), tensor_shape.TensorShape([None])))
     padded_dataset = dataset.padded_batch(
         2, padded_shapes=([None], [None]), padding_values=('', 0))
-    with self.cached_session() as sess:
-      next_element = padded_dataset.make_one_shot_iterator().get_next()
-      sess.run(next_element)
+    next_element = self.getNext(padded_dataset)
+    self.evaluate(next_element())
 
-  def testPaddedBatchDatasetShapeSpecifications(self):
+  def testSkipEagerPaddedBatchDatasetShapeSpecifications(self):
     int_placeholder = array_ops.placeholder(dtypes.int32)
     float_placeholder = array_ops.placeholder(dtypes.float32)
     string_placeholder = array_ops.placeholder(dtypes.string)
@@ -452,6 +367,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       _ = dataset_ops.Dataset.range(10).padded_batch(
           5, padded_shapes=shape_as_tensor)
 
+  def testSkipEagerPaddedBatchShapeError(self):
     with self.assertRaisesRegexp(
         ValueError,
         r'The padded shape \((\?|None), (\?|None)\) is not compatible with the '
@@ -461,6 +377,7 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           5, padded_shapes=shape_as_tensor)
 
 
+# TODO(b/119837791): Add eager benchmarks too.
 class BatchDatasetBenchmark(test.Benchmark):
 
   def benchmarkBatchSparse(self):
diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
index 63625fac03..06ce18a9b4 100644
--- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
@@ -25,16 +25,16 @@ import numpy as np
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class FileCacheDatasetTest(test_base.DatasetTestBase):
 
   def setUp(self):
@@ -48,159 +48,124 @@ class FileCacheDatasetTest(test_base.DatasetTestBase):
   def testCacheDatasetPassthrough(self):
     components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]),
                   np.array([9.0, 10.0, 11.0, 12.0]))
-    count_placeholder = array_ops.placeholder_with_default(
-        constant_op.constant(5, dtypes.int64), shape=[])
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
 
-    repeat_dataset = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .repeat(count_placeholder))
-
-    cache_dataset = repeat_dataset.cache(filename_placeholder)
+    def dataset_fn(count=5, filename=None):
+      repeat_dataset = (
+          dataset_ops.Dataset.from_tensor_slices(components).repeat(count))
+      if filename:
+        return repeat_dataset.cache(filename)
+      else:
+        return repeat_dataset
 
     self.assertEqual(
-        tuple([c.shape[1:] for c in components]), cache_dataset.output_shapes)
-
-    # Create initialization ops for iterators without and with
-    # caching, respectively.
-    iterator = iterator_ops.Iterator.from_structure(cache_dataset.output_types,
-                                                    cache_dataset.output_shapes)
-    init_fifo_op = iterator.make_initializer(repeat_dataset)
-    init_cache_op = iterator.make_initializer(cache_dataset)
-
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # First run without caching to collect the "ground truth".
-      sess.run(init_fifo_op)
-      elements = []
-      for _ in range(20):
-        elements.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Assert that the cached dataset has the same elements as the
-      # "ground truth".
-      sess.run(
-          init_cache_op, feed_dict={filename_placeholder: self.cache_prefix})
-      cached_elements = []
-      for _ in range(20):
-        cached_elements.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      self.assertAllEqual(elements, cached_elements)
-
-      # Re-initialize with an empty upstream (to throw errors.OutOfRangeError
-      # if we didn't use the cache).
-      sess.run(
-          init_cache_op,
-          feed_dict={
-              count_placeholder: 0,
-              filename_placeholder: self.cache_prefix
-          })
-      replayed_elements = []
-      for _ in range(20):
-        replayed_elements.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      self.assertEqual(cached_elements, replayed_elements)
-
-      # Re-initialize with an empty upstream and a missing cache file (should
-      # throw errors.OutOfRangeError immediately).
-      sess.run(
-          init_cache_op,
-          feed_dict={
-              count_placeholder: 0,
-              filename_placeholder: self.cache_prefix + "nonsense"
-          })
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        tuple([c.shape[1:] for c in components]),
+        dataset_fn().output_shapes)
+
+    get_next = self.getNext(dataset_fn())
+
+    # First run without caching to collect the "ground truth".
+    elements = []
+    for _ in range(20):
+      elements.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+    # Assert that the cached dataset has the same elements as the
+    # "ground truth".
+    get_next = self.getNext(dataset_fn(filename=self.cache_prefix))
+    cached_elements = []
+    for _ in range(20):
+      cached_elements.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    self.assertAllEqual(elements, cached_elements)
+
+    # Re-initialize with an empty upstream (to throw errors.OutOfRangeError
+    # if we didn't use the cache).
+    get_next = self.getNext(dataset_fn(count=0, filename=self.cache_prefix))
+    replayed_elements = []
+    for _ in range(20):
+      replayed_elements.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    self.assertEqual(cached_elements, replayed_elements)
+
+    # Re-initialize with an empty upstream and a missing cache file (should
+    # throw errors.OutOfRangeError immediately).
+    get_next = self.getNext(
+        dataset_fn(count=0, filename=self.cache_prefix + "nonsense"))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testConcurrentWriters(self):
     components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]),
                   np.array([9.0, 10.0, 11.0, 12.0]))
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
 
-    cache_dataset1 = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .cache(filename_placeholder))
-    cache_dataset2 = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .cache(filename_placeholder))
+    cache_dataset1 = (
+        dataset_ops.Dataset.from_tensor_slices(components).cache(
+            self.cache_prefix))
+    cache_dataset2 = (
+        dataset_ops.Dataset.from_tensor_slices(components).cache(
+            self.cache_prefix))
 
-    iterator1 = cache_dataset1.make_initializable_iterator()
-    iterator2 = cache_dataset2.make_initializable_iterator()
-    init_cache_op1 = iterator1.initializer
-    init_cache_op2 = iterator2.initializer
+    get_next1 = self.getNext(cache_dataset1)
+    get_next2 = self.getNext(cache_dataset2)
 
-    get_next1 = iterator1.get_next()
-    get_next2 = iterator2.get_next()
+    self.evaluate(get_next1())  # this should succeed
 
-    with self.cached_session() as sess:
-      sess.run(
-          init_cache_op1, feed_dict={filename_placeholder: self.cache_prefix})
-      sess.run(get_next1)  # this should succeed
+    with self.assertRaises(errors.AlreadyExistsError):
+      self.evaluate(get_next2())
 
-      sess.run(
-          init_cache_op2, feed_dict={filename_placeholder: self.cache_prefix})
-      with self.assertRaises(errors.AlreadyExistsError):
-        sess.run(get_next2)
-
-      sess.run(get_next1)  # this should continue to succeed
+    self.evaluate(get_next1())  # this should continue to succeed
 
   def testConcurrentReaders(self):
     components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]),
                   np.array([9.0, 10.0, 11.0, 12.0]))
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
-
-    cache_dataset1 = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .cache(filename_placeholder))
-    cache_dataset2 = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .cache(filename_placeholder))
-
-    iterator1 = cache_dataset1.make_initializable_iterator()
-    iterator2 = cache_dataset2.make_initializable_iterator()
-    init_cache_op1 = iterator1.initializer
-    init_cache_op2 = iterator2.initializer
-
-    get_next1 = iterator1.get_next()
-    get_next2 = iterator2.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(
-          init_cache_op1, feed_dict={filename_placeholder: self.cache_prefix})
-      elements = []
-      for _ in range(4):
-        elements.append(sess.run(get_next1))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next1)
-
-      # Re-initialize
-      sess.run(
-          init_cache_op1, feed_dict={filename_placeholder: self.cache_prefix})
-      sess.run(
-          init_cache_op2, feed_dict={filename_placeholder: self.cache_prefix})
-
-      # Reading concurrently should succeed.
-      elements_itr1 = []
-      elements_itr2 = []
-      elements_itr2.append(sess.run(get_next2))
-      elements_itr1.append(sess.run(get_next1))
-      elements_itr2.append(sess.run(get_next2))
-      elements_itr1.append(sess.run(get_next1))
-      # Intentionally reversing the order
-      elements_itr1.append(sess.run(get_next1))
-      elements_itr2.append(sess.run(get_next2))
-      elements_itr1.append(sess.run(get_next1))
-      elements_itr2.append(sess.run(get_next2))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next2)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next1)
-
-      self.assertAllEqual(elements, elements_itr1)
-      self.assertAllEqual(elements, elements_itr2)
-
 
+    cache_dataset1 = (
+        dataset_ops.Dataset.from_tensor_slices(components).cache(
+            self.cache_prefix))
+    cache_dataset2 = (
+        dataset_ops.Dataset.from_tensor_slices(components).cache(
+            self.cache_prefix))
+
+    get_next1 = self.getNext(cache_dataset1)
+    get_next2 = self.getNext(cache_dataset2)
+
+    elements = []
+    for _ in range(4):
+      elements.append(self.evaluate(get_next1()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next1())
+
+    # Re-initialize
+    get_next1 = self.getNext(cache_dataset1)
+    get_next2 = self.getNext(cache_dataset2)
+
+    # Reading concurrently should succeed.
+    elements_itr1 = []
+    elements_itr2 = []
+    elements_itr2.append(self.evaluate(get_next2()))
+    elements_itr1.append(self.evaluate(get_next1()))
+    elements_itr2.append(self.evaluate(get_next2()))
+    elements_itr1.append(self.evaluate(get_next1()))
+    # Intentionally reversing the order
+    elements_itr1.append(self.evaluate(get_next1()))
+    elements_itr2.append(self.evaluate(get_next2()))
+    elements_itr1.append(self.evaluate(get_next1()))
+    elements_itr2.append(self.evaluate(get_next2()))
+
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next2())
+
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next1())
+
+    self.assertAllEqual(elements, elements_itr1)
+    self.assertAllEqual(elements, elements_itr2)
+
+
+@test_util.run_all_in_graph_and_eager_modes
 class MemoryCacheDatasetTest(test_base.DatasetTestBase):
 
   def testCacheDatasetPassthrough(self):
@@ -212,106 +177,76 @@ class MemoryCacheDatasetTest(test_base.DatasetTestBase):
       cached_dataset = dataset.cache().repeat(2)
       uncached_dataset = dataset.repeat(2)
 
+      self.evaluate(repeat_count.initializer)
       # Needs to be initializable to capture the variable.
-      cached_iterator = cached_dataset.make_initializable_iterator()
-      cached_next = cached_iterator.get_next()
-      uncached_iterator = uncached_dataset.make_initializable_iterator()
-      uncached_next = uncached_iterator.get_next()
+      cached_next = self.getNext(cached_dataset, requires_initialization=True)
+      uncached_next = self.getNext(
+          uncached_dataset, requires_initialization=True)
+      for i in range(3):
+        for _ in range(10):
+          self.assertEqual(self.evaluate(cached_next()), i)
+          self.assertEqual(self.evaluate(uncached_next()), i)
 
-      with self.cached_session() as sess:
+      self.evaluate(repeat_count.assign(0))
 
-        sess.run(repeat_count.initializer)
-        sess.run(cached_iterator.initializer)
-        sess.run(uncached_iterator.initializer)
-
-        for i in range(3):
-          for _ in range(10):
-            self.assertEqual(sess.run(cached_next), i)
-            self.assertEqual(sess.run(uncached_next), i)
-
-        sess.run(repeat_count.assign(0))
-
-        # The uncached iterator should now be empty.
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(uncached_next)
+      # The uncached iterator should now be empty.
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(uncached_next())
 
-        # The cached iterator replays from cache.
-        for i in range(3):
-          for _ in range(10):
-            self.assertEqual(sess.run(cached_next), i)
+      # The cached iterator replays from cache.
+      for i in range(3):
+        for _ in range(10):
+          self.assertEqual(self.evaluate(cached_next()), i)
 
-        # The cached iterator should now be empty.
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(cached_next)
+      # The cached iterator should now be empty.
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(cached_next())
 
   def testEmptyCacheReading(self):
     components = (np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]),
                   np.array([9.0, 10.0, 11.0, 12.0]))
-    count_placeholder = array_ops.placeholder_with_default(
-        constant_op.constant(5, dtypes.int64), shape=[])
-
-    repeat_dataset = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .repeat(count_placeholder))
 
+    repeat_dataset = (
+        dataset_ops.Dataset.from_tensor_slices(components).repeat(0))
     cache_dataset = repeat_dataset.cache()
 
     # Create initialization ops for iterators without and with
     # caching, respectively.
-    iterator = cache_dataset.make_initializable_iterator()
-    init_cache_op = iterator.initializer
-
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # Initialize with an empty upstream and a missing cache file (should
-      # throw errors.OutOfRangeError immediately).
-      sess.run(init_cache_op, feed_dict={count_placeholder: 0})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(cache_dataset, expected_output=[])
 
   def testConcurrentReaders(self):
-    count_placeholder = array_ops.placeholder_with_default(
-        constant_op.constant(5, dtypes.int64), shape=[])
-    dataset = dataset_ops.Dataset.range(count_placeholder).cache()
+
+    dataset = dataset_ops.Dataset.range(5).cache()
     d1 = dataset.map(lambda x: x + 1)
     d2 = dataset.map(lambda x: x + 6)
 
-    i1 = d1.make_initializable_iterator()
-    i2 = d2.make_initializable_iterator()
+    get_next1 = self.getNext(d1)
 
-    with self.cached_session() as sess:
-      sess.run(i1.initializer)
+    self.assertEqual(1, self.evaluate(get_next1()))
+    self.assertEqual(2, self.evaluate(get_next1()))
+    self.assertEqual(3, self.evaluate(get_next1()))
 
-      self.assertEqual(1, sess.run(i1.get_next()))
-      self.assertEqual(2, sess.run(i1.get_next()))
-      self.assertEqual(3, sess.run(i1.get_next()))
+    get_next2 = self.getNext(d2)
 
-      sess.run(i2.initializer, feed_dict={count_placeholder: 3})
+    self.assertEqual(6, self.evaluate(get_next2()))
+    self.assertEqual(7, self.evaluate(get_next2()))
+    self.assertEqual(4, self.evaluate(get_next1()))  # interleave execution
+    self.assertEqual([8, 5],
+                     [self.evaluate(get_next2()),
+                      self.evaluate(get_next1())])
+    self.assertEqual(9, self.evaluate(get_next2()))
+    self.assertEqual(10, self.evaluate(get_next2()))
 
-      self.assertEqual(6, sess.run(i2.get_next()))
-      self.assertEqual(7, sess.run(i2.get_next()))
-      self.assertEqual(4, sess.run(i1.get_next()))  # interleave execution
-      self.assertEqual([8, 5], sess.run([i2.get_next(), i1.get_next()]))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(i1.get_next())
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(i2.get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next2())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next1())
 
   def testCacheTakeRepeat(self):
     dataset = dataset_ops.Dataset.range(10).cache().take(5).repeat(2)
-    itr = dataset.make_one_shot_iterator()
-    n = itr.get_next()
-
-    expected_values = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
 
-    with self.cached_session() as sess:
-      for i, expected in enumerate(expected_values):
-        self.assertEqual(expected, sess.run(n),
-                         "Unexpected value at index %s" % i)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    expected_output = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
index 83af31f380..3123e32d8e 100644
--- a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
@@ -24,9 +24,11 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ConcatenateDatasetTest(test_base.DatasetTestBase):
 
   def testConcatenateDataset(self):
@@ -46,23 +48,19 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase):
     self.assertEqual(concatenated.output_shapes, (tensor_shape.TensorShape(
         [20]), tensor_shape.TensorShape([15]), tensor_shape.TensorShape([])))
 
-    iterator = concatenated.make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(9):
-        result = sess.run(get_next)
-        if i < 4:
-          for component, result_component in zip(input_components, result):
-            self.assertAllEqual(component[i], result_component)
-        else:
-          for component, result_component in zip(to_concatenate_components,
-                                                 result):
-            self.assertAllEqual(component[i - 4], result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    get_next = self.getNext(concatenated)
+
+    for i in range(9):
+      result = self.evaluate(get_next())
+      if i < 4:
+        for component, result_component in zip(input_components, result):
+          self.assertAllEqual(component[i], result_component)
+      else:
+        for component, result_component in zip(to_concatenate_components,
+                                               result):
+          self.assertAllEqual(component[i - 4], result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testConcatenateDatasetDifferentShape(self):
     input_components = (
@@ -79,24 +77,18 @@ class ConcatenateDatasetTest(test_base.DatasetTestBase):
     self.assertEqual(
         [ts.as_list()
          for ts in nest.flatten(concatenated.output_shapes)], [[20], [None]])
-
-    iterator = concatenated.make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(9):
-        result = sess.run(get_next)
-        if i < 4:
-          for component, result_component in zip(input_components, result):
-            self.assertAllEqual(component[i], result_component)
-        else:
-          for component, result_component in zip(to_concatenate_components,
-                                                 result):
-            self.assertAllEqual(component[i - 4], result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    get_next = self.getNext(concatenated)
+    for i in range(9):
+      result = self.evaluate(get_next())
+      if i < 4:
+        for component, result_component in zip(input_components, result):
+          self.assertAllEqual(component[i], result_component)
+      else:
+        for component, result_component in zip(to_concatenate_components,
+                                               result):
+          self.assertAllEqual(component[i - 4], result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testConcatenateDatasetDifferentStructure(self):
     input_components = (
diff --git a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
index bc6b36285a..4a7a946576 100644
--- a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
@@ -31,33 +31,26 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class DatasetConstructorTest(test_base.DatasetTestBase):
 
   def testFromTensors(self):
     """Test a dataset that represents a single tuple of tensors."""
     components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
 
-    iterator = (dataset_ops.Dataset.from_tensors(components)
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.from_tensors(components)
 
     self.assertEqual([c.shape for c in components],
-                     [t.shape for t in get_next])
+                     nest.flatten(dataset.output_shapes))
 
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      results = sess.run(get_next)
-      for component, result_component in zip(components, results):
-        self.assertAllEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(dataset, expected_output=[components])
 
   def testFromTensorsSparse(self):
     """Test a dataset that represents a single tuple of tensors."""
@@ -70,23 +63,12 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                       values=np.array([-1, 1]),
                       dense_shape=np.array([2, 2])))
 
-    iterator = (
-        dataset_ops.Dataset.from_tensors(components)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.from_tensors(components)
 
     self.assertEqual(
         [tensor_shape.TensorShape(c.dense_shape) for c in components],
-        [shape for shape in iterator.output_shapes])
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      results = sess.run(get_next)
-      for component, result_component in zip(components, results):
-        self.assertSparseValuesEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        [shape for shape in dataset.output_shapes])
+    self.assertDatasetProduces(dataset, expected_output=[components])
 
   def testFromTensorsMixed(self):
     """Test an dataset that represents a single tuple of tensors."""
@@ -100,27 +82,13 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                       values=np.array([-1, 1]),
                       dense_shape=np.array([2, 2])))
 
-    iterator = (
-        dataset_ops.Dataset.from_tensors(components)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
+    dataset = dataset_ops.Dataset.from_tensors(components)
     self.assertEqual([
         tensor_shape.TensorShape(c.dense_shape)
         if sparse_tensor.is_sparse(c) else c.shape for c in components
-    ], [shape for shape in iterator.output_shapes])
+    ], [shape for shape in dataset.output_shapes])
 
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      results = sess.run(get_next)
-      for component, result_component in zip(components, results):
-        if sparse_tensor.is_sparse(component):
-          self.assertSparseValuesEqual(component, result_component)
-        else:
-          self.assertAllEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(dataset, expected_output=[components])
 
   def testFromTensorSlices(self):
     """Test a dataset that represents the slices from a tuple of tensors."""
@@ -130,22 +98,18 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
         np.array([37.0, 38.0, 39.0, 40.0])
     )
 
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+    get_next = self.getNext(dataset)
 
     self.assertEqual([c.shape[1:] for c in components],
-                     [t.shape for t in get_next])
+                     [shape for shape in dataset.output_shapes])
 
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(4):
-        results = sess.run(get_next)
-        for component, result_component in zip(components, results):
-          self.assertAllEqual(component[i], result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    for i in range(4):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(components, results):
+        self.assertAllEqual(component[i], result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      results = self.evaluate(get_next())
 
   def testFromTensorSlicesSparse(self):
     """Test a dataset that represents the slices from a tuple of tensors."""
@@ -158,50 +122,39 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                       values=np.array([1, 2, 3]),
                       dense_shape=np.array([3, 3])))
 
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
 
     self.assertEqual(
         [tensor_shape.TensorShape(c.dense_shape[1:]) for c in components],
-        [shape for shape in iterator.output_shapes])
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      expected = [
-          (sparse_tensor.SparseTensorValue(
-              indices=np.array([[0]]),
-              values=np.array([0]),
-              dense_shape=np.array([1])),
-           sparse_tensor.SparseTensorValue(
-               indices=np.array([[0]]),
-               values=np.array([1]),
-               dense_shape=np.array([3]))),
-          (sparse_tensor.SparseTensorValue(
-              indices=np.array([[0]]),
-              values=np.array([0]),
-              dense_shape=np.array([1])),
-           sparse_tensor.SparseTensorValue(
-               indices=np.array([[1]]),
-               values=np.array([2]),
-               dense_shape=np.array([3]))),
-          (sparse_tensor.SparseTensorValue(
-              indices=np.array([[0]]),
-              values=np.array([0]),
-              dense_shape=np.array([1])),
-           sparse_tensor.SparseTensorValue(
-               indices=np.array([[2]]),
-               values=np.array([3]),
-               dense_shape=np.array([3]))),
-      ]
-      for i in range(3):
-        results = sess.run(get_next)
-        for component, result_component in zip(expected[i], results):
-          self.assertSparseValuesEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        [shape for shape in dataset.output_shapes])
+
+    expected = [
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[0]]),
+             values=np.array([1]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[1]]),
+             values=np.array([2]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[2]]),
+             values=np.array([3]),
+             dense_shape=np.array([3]))),
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected)
 
   def testFromTensorSlicesMixed(self):
     """Test a dataset that represents the slices from a tuple of tensors."""
@@ -217,78 +170,68 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                       values=np.array([1, 2, 3]),
                       dense_shape=np.array([3, 3])))
 
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+    get_next = self.getNext(dataset)
     self.assertEqual([
         tensor_shape.TensorShape(c.dense_shape[1:])
         if sparse_tensor.is_sparse(c) else c.shape[1:] for c in components
-    ], [shape for shape in iterator.output_shapes])
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      expected = [
-          (sparse_tensor.SparseTensorValue(
-              indices=np.array([[0]]),
-              values=np.array([0]),
-              dense_shape=np.array([1])),
-           sparse_tensor.SparseTensorValue(
-               indices=np.array([[0]]),
-               values=np.array([1]),
-               dense_shape=np.array([3]))),
-          (sparse_tensor.SparseTensorValue(
-              indices=np.array([[0]]),
-              values=np.array([0]),
-              dense_shape=np.array([1])),
-           sparse_tensor.SparseTensorValue(
-               indices=np.array([[1]]),
-               values=np.array([2]),
-               dense_shape=np.array([3]))),
-          (sparse_tensor.SparseTensorValue(
-              indices=np.array([[0]]),
-              values=np.array([0]),
-              dense_shape=np.array([1])),
-           sparse_tensor.SparseTensorValue(
-               indices=np.array([[2]]),
-               values=np.array([3]),
-               dense_shape=np.array([3]))),
-      ]
-      for i in range(3):
-        results = sess.run(get_next)
-        for component, result_component in zip(
-            (list(zip(*components[:3]))[i] + expected[i]), results):
-          if sparse_tensor.is_sparse(component):
-            self.assertSparseValuesEqual(component, result_component)
-          else:
-            self.assertAllEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    ], [shape for shape in dataset.output_shapes])
+
+    expected = [
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[0]]),
+             values=np.array([1]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[1]]),
+             values=np.array([2]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[2]]),
+             values=np.array([3]),
+             dense_shape=np.array([3]))),
+    ]
+    for i in range(3):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(
+          (list(zip(*components[:3]))[i] + expected[i]), results):
+        if sparse_tensor.is_sparse(component):
+          self.assertSparseValuesEqual(component, result_component)
+        else:
+          self.assertAllEqual(component, result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testFromTensorSlicesWithDict(self):
     components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]}
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual(dtypes.int32, iterator.output_types["foo"])
-    self.assertEqual(dtypes.float32, iterator.output_types["bar"])
-    self.assertEqual((), iterator.output_shapes["foo"])
-    self.assertEqual((1,), iterator.output_shapes["bar"])
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(3):
-        results = sess.run(get_next)
-        self.assertEqual(components["foo"][i], results["foo"])
-        self.assertEqual(components["bar"][i], results["bar"])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testFromSparseTensorSlices(self):
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+    get_next = self.getNext(dataset)
+
+    self.assertEqual(dtypes.int32, dataset.output_types["foo"])
+    self.assertEqual(dtypes.float32, dataset.output_types["bar"])
+    self.assertEqual((), dataset.output_shapes["foo"])
+    self.assertEqual((1,), dataset.output_shapes["bar"])
+
+    for i in range(3):
+      results = self.evaluate(get_next())
+      self.assertEqual(components["foo"][i], results["foo"])
+      self.assertEqual(components["bar"][i], results["bar"])
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+  def testSkipEagerFromSparseTensorSlices(self):
     """Test a dataset based on slices of a `tf.SparseTensor`."""
     st = array_ops.sparse_placeholder(dtypes.float64)
     iterator = (dataset_ops.Dataset.from_sparse_tensor_slices(st)
@@ -389,8 +332,33 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
                           for s in nest.flatten(dataset.output_shapes)
                       ]))
 
-    iterator = dataset.make_one_shot_iterator()
-    (w, x), (y, z) = iterator.get_next()
+    # Define a separate set of components with matching leading
+    # dimension for the from-slices constructor.
+    components_for_slices = (np.array([1, 2, 3], dtype=np.int64),
+                             (np.array([4., 5., 6.]), np.array([7., 8., 9.])),
+                             np.array([10, 11, 12], dtype=np.int64))
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices)
+    self.assertEquals((dtypes.int64,
+                       (dtypes.float64, dtypes.float64), dtypes.int64),
+                      dataset.output_types)
+    self.assertEquals(([], ([], []), []), dataset.output_shapes)
+
+  # TODO(b/117581999): more specific shapes in eager mode.
+  def testSkipEagerNestedStructure(self):
+    components = (np.array([1, 2, 3], dtype=np.int64), (np.array([4., 5.]),
+                                                        np.array([6., 7.])),
+                  np.array([8, 9, 10], dtype=np.int64))
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+    dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1])))
+
+    dataset = dataset.flat_map(
+        lambda x, y: dataset_ops.Dataset.from_tensors(
+            ((x[0], x[1]), (y[0], y[1])))).batch(32)
+
+    get_next = self.getNext(dataset)
+    (w, x), (y, z) = get_next()
     self.assertEquals(dtypes.int64, w.dtype)
     self.assertEquals(dtypes.int64, x.dtype)
     self.assertEquals(dtypes.float64, y.dtype)
@@ -400,8 +368,8 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     self.assertEquals([None, 2], y.shape.as_list())
     self.assertEquals([None, 2], z.shape.as_list())
 
-    iterator = dataset.make_initializable_iterator()
-    (w, x), (y, z) = iterator.get_next()
+    get_next = self.getNext(dataset)
+    (w, x), (y, z) = get_next()
     self.assertEquals(dtypes.int64, w.dtype)
     self.assertEquals(dtypes.int64, x.dtype)
     self.assertEquals(dtypes.float64, y.dtype)
@@ -411,18 +379,6 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     self.assertEquals([None, 2], y.shape.as_list())
     self.assertEquals([None, 2], z.shape.as_list())
 
-    # Define a separate set of components with matching leading
-    # dimension for the from-slices constructor.
-    components_for_slices = (np.array([1, 2, 3], dtype=np.int64),
-                             (np.array([4., 5., 6.]),
-                              np.array([7., 8., 9.])),
-                             np.array([10, 11, 12], dtype=np.int64))
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices)
-    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
-                       dtypes.int64), dataset.output_types)
-    self.assertEquals(([], ([], []), []), dataset.output_shapes)
-
   def testNestedDict(self):
     components = {"a": {"aa": 1, "ab": [2.0, 2.0]}, "b": [3, 3, 3]}
     dataset = dataset_ops.Dataset.from_tensors(components)
@@ -454,12 +410,11 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
     self.assertEquals(dtypes.int64, dataset.output_types)
     self.assertEquals([3], dataset.output_shapes)
 
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-    self.assertEquals(dtypes.int64, get_next.dtype)
-    self.assertEquals([3], get_next.shape)
+    get_next = self.getNext(dataset)
+    self.assertEquals(dtypes.int64, get_next().dtype)
+    self.assertEquals([3], get_next().shape)
 
-  def testSplitPipelineFailsWithPlacementError(self):
+  def testSkipEagerSplitPipelineFailsWithPlacementError(self):
     with session.Session(
         target="",
         config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess:
@@ -490,6 +445,7 @@ class DatasetConstructorTest(test_base.DatasetTestBase):
         sess.run(iterator.get_next())
 
 
+# TODO(b/119837791): Add eager benchmarks as well.
 class DatasetConstructorBenchmark(test.Benchmark):
 
   def benchmarkSliceRepeatBatch(self):
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
index 67d3b41d3e..373cdc0a77 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_ops_test.py
@@ -32,17 +32,18 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testAsSerializedGraph(self):
     dataset = dataset_ops.Dataset.range(10)
-    with self.cached_session() as sess:
-      graph = graph_pb2.GraphDef().FromString(
-          sess.run(dataset._as_serialized_graph()))
-      self.assertTrue(any(node.op != "RangeDataset" for node in graph.node))
+    graph = graph_pb2.GraphDef().FromString(
+        self.evaluate(dataset._as_serialized_graph()))
+    self.assertTrue(any([node.op != "RangeDataset" for node in graph.node]))
 
   @staticmethod
   def make_apply_fn(dataset):
@@ -253,6 +254,7 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertTrue(ds.options().experimental_autotune)
     self.assertTrue(ds.options().experimental_filter_fusion)
 
+  # TODO(b/119882922): use-after-free bug in eager mode.
   # pylint: disable=g-long-lambda
   @parameterized.named_parameters(
       ("Tensor", lambda: constant_op.constant(37.0),
@@ -276,7 +278,8 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
        optional_ops.OptionalStructure(
            structure.TensorStructure(dtypes.float32, []))),
   )
-  def testDatasetStructure(self, tf_value_fn, expected_element_structure):
+  def testSkipEagerDatasetStructure(self, tf_value_fn,
+                                    expected_element_structure):
     dataset = dataset_ops.Dataset.from_tensors(0).map(lambda _: tf_value_fn())
     dataset_structure = structure.Structure.from_value(dataset)
     self.assertIsInstance(dataset_structure, dataset_ops.DatasetStructure)
diff --git a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
index a0c6b37a6d..fba474b97b 100644
--- a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
@@ -24,16 +24,17 @@ import numpy as np
 from tensorflow.python.client import session
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class FilterDatasetTest(test_base.DatasetTestBase):
 
   def testFilterDataset(self):
@@ -43,69 +44,43 @@ class FilterDatasetTest(test_base.DatasetTestBase):
             7, dtype=np.int64)[:, np.newaxis],
         np.array(37.0, dtype=np.float64) * np.arange(7)
     )
-    count = array_ops.placeholder(dtypes.int64, shape=[])
-    modulus = array_ops.placeholder(dtypes.int64)
-
     def _map_fn(x, y, z):
       return math_ops.square(x), math_ops.square(y), math_ops.square(z)
 
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn)
-        .repeat(count)
-        .filter(lambda x, _y, _z: math_ops.equal(math_ops.mod(x, modulus), 0))
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual([c.shape[1:] for c in components],
-                     [t.shape for t in get_next])
-
-    with self.cached_session() as sess:
-      # Test that we can dynamically feed a different modulus value for each
-      # iterator.
-      def do_test(count_val, modulus_val):
-        sess.run(init_op, feed_dict={count: count_val, modulus: modulus_val})
-        for _ in range(count_val):
-          for i in [x for x in range(7) if x**2 % modulus_val == 0]:
-            result = sess.run(get_next)
-            for component, result_component in zip(components, result):
-              self.assertAllEqual(component[i]**2, result_component)
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-      do_test(14, 2)
-      do_test(4, 18)
-
-      # Test an empty dataset.
-      do_test(0, 1)
+    def do_test(count, modulus):
+      dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
+          _map_fn).repeat(count).filter(
+              lambda x, _y, _z: math_ops.equal(math_ops.mod(x, modulus), 0))
+      self.assertEqual([c.shape[1:] for c in components],
+                       [shape for shape in dataset.output_shapes])
+      get_next = self.getNext(dataset)
+      for _ in range(count):
+        for i in [x for x in range(7) if x**2 % modulus == 0]:
+          result = self.evaluate(get_next())
+          for component, result_component in zip(components, result):
+            self.assertAllEqual(component[i]**2, result_component)
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(get_next())
+
+    do_test(14, 2)
+    do_test(4, 18)
+
+    # Test an empty dataset.
+    do_test(0, 1)
 
   def testFilterRange(self):
-    dataset = dataset_ops.Dataset.range(100).filter(
+    dataset = dataset_ops.Dataset.range(4).filter(
         lambda x: math_ops.not_equal(math_ops.mod(x, 3), 2))
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(get_next))
-      self.assertEqual(1, sess.run(get_next))
-      self.assertEqual(3, sess.run(get_next))
+    self.assertDatasetProduces(dataset, expected_output=[0, 1, 3])
 
   def testFilterDict(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(lambda x: {"foo": x * 2, "bar": x ** 2})
-                .filter(lambda d: math_ops.equal(d["bar"] % 2, 0))
-                .map(lambda d: d["foo"] + d["bar"])
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(10):
-        if (i ** 2) % 2 == 0:
-          self.assertEqual(i * 2 + i ** 2, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).map(
+        lambda x: {"foo": x * 2, "bar": x ** 2}).filter(
+            lambda d: math_ops.equal(d["bar"] % 2, 0)).map(
+                lambda d: d["foo"] + d["bar"])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[(i * 2 + i**2) for i in range(10) if not (i**2) % 2])
 
   def testUseStepContainerInFilter(self):
     input_data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64)
@@ -117,18 +92,9 @@ class FilterDatasetTest(test_base.DatasetTestBase):
       summed = math_ops.reduce_sum(squared_xs)
       return math_ops.equal(summed, 1 + 4 + 9)
 
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices([[1, 2, 3], [4, 5, 6]])
-        .filter(_predicate)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(input_data[0], sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.from_tensor_slices(
+        [[1, 2, 3], [4, 5, 6]]).filter(_predicate)
+    self.assertDatasetProduces(dataset, expected_output=[input_data[0]])
 
   def testSparse(self):
 
@@ -141,46 +107,29 @@ class FilterDatasetTest(test_base.DatasetTestBase):
     def _filter_fn(_, i):
       return math_ops.equal(i % 2, 0)
 
-    iterator = (
-        dataset_ops.Dataset.range(10).map(_map_fn).filter(_filter_fn).map(
-            lambda x, i: x).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(5):
-        actual = sess.run(get_next)
-        self.assertTrue(isinstance(actual, sparse_tensor.SparseTensorValue))
-        self.assertSparseValuesEqual(actual, _map_fn(i * 2)[0])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).map(_map_fn).filter(_filter_fn).map(
+        lambda x, i: x)
+    self.assertDatasetProduces(
+        dataset, expected_output=[_map_fn(i * 2)[0] for i in range(5)])
 
   def testShortCircuit(self):
-    iterator = (
-        dataset_ops.Dataset.zip(
-            (dataset_ops.Dataset.range(10),
-             dataset_ops.Dataset.from_tensors(True).repeat(None)))
-        .filter(lambda x, y: y).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(10):
-        self.assertEqual((i, True), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.zip(
+        (dataset_ops.Dataset.range(10),
+         dataset_ops.Dataset.from_tensors(True).repeat(None)
+        )).filter(lambda x, y: y)
+    self.assertDatasetProduces(
+        dataset, expected_output=[(i, True) for i in range(10)])
 
   def testParallelFilters(self):
     dataset = dataset_ops.Dataset.range(10).filter(
         lambda x: math_ops.equal(x % 2, 0))
-    iterators = [dataset.make_one_shot_iterator() for _ in range(10)]
-    next_elements = [iterator.get_next() for iterator in iterators]
-    with self.cached_session() as sess:
-      self.assertEqual([0 for _ in range(10)], sess.run(next_elements))
+    next_elements = [self.getNext(dataset) for _ in range(10)]
+    self.assertEqual([0 for _ in range(10)],
+                     self.evaluate(
+                         [next_element() for next_element in next_elements]))
 
 
+# TODO(b/119837791): Add eager benchmarks too.
 class FilterDatasetBenchmark(test.Benchmark):
 
   def _benchmark(self, predicate, name):
diff --git a/tensorflow/python/data/kernel_tests/inputs_test.py b/tensorflow/python/data/kernel_tests/inputs_test.py
index d089b49bcc..03df502d14 100644
--- a/tensorflow/python/data/kernel_tests/inputs_test.py
+++ b/tensorflow/python/data/kernel_tests/inputs_test.py
@@ -25,9 +25,11 @@ from tensorflow.python.data.ops import readers
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class InputsTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @staticmethod
diff --git a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
index b58c1444da..7efe3c692a 100644
--- a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
@@ -24,13 +24,13 @@ import tempfile
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.ops import array_ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ListFilesDatasetOpTest(test_base.DatasetTestBase):
 
   def setUp(self):
@@ -43,32 +43,23 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
     for filename in filenames:
       open(path.join(self.tmp_dir, filename), 'a').close()
 
-  def testEmptyDirectory(self):
+  # Note: eager mode fails in assertion error same as initializer in graph mode.
+  def testSkipEagerEmptyDirectory(self):
     dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*'))
-    with self.cached_session() as sess:
-      itr = dataset.make_one_shot_iterator()
-      next_element = itr.get_next()
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+    self.assertDatasetProduces(dataset, expected_output=[])
 
   def testSimpleDirectory(self):
     filenames = ['a', 'b', 'c']
     self._touchTempFiles(filenames)
 
     dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*'))
-    with self.cached_session() as sess:
-      itr = dataset.make_one_shot_iterator()
-      next_element = itr.get_next()
-
-      full_filenames = []
-      produced_filenames = []
-      for filename in filenames:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
-      self.assertItemsEqual(full_filenames, produced_filenames)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in filenames
+        ],
+        assert_items_equal=True)
 
   def testSimpleDirectoryNotShuffled(self):
     filenames = ['b', 'c', 'a']
@@ -76,15 +67,12 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
 
     dataset = dataset_ops.Dataset.list_files(
         path.join(self.tmp_dir, '*'), shuffle=False)
-    with self.cached_session() as sess:
-      itr = dataset.make_one_shot_iterator()
-      next_element = itr.get_next()
-
-      for filename in sorted(filenames):
-        self.assertEqual(compat.as_bytes(path.join(self.tmp_dir, filename)),
-                         sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in sorted(filenames)
+        ])
 
   def testFixedSeedResultsInRepeatableOrder(self):
     filenames = ['a', 'b', 'c']
@@ -92,120 +80,76 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
 
     dataset = dataset_ops.Dataset.list_files(
         path.join(self.tmp_dir, '*'), shuffle=True, seed=37)
-    with self.cached_session() as sess:
-      itr = dataset.make_initializable_iterator()
-      next_element = itr.get_next()
-
-      full_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename))
-                        for filename in filenames]
-
-      all_produced_filenames = []
-      for _ in range(3):
-        produced_filenames = []
-        sess.run(itr.initializer)
-        try:
-          while True:
-            produced_filenames.append(sess.run(next_element))
-        except errors.OutOfRangeError:
-          pass
-        all_produced_filenames.append(produced_filenames)
-
-      # Each run should produce the same set of filenames, which may be
-      # different from the order of `full_filenames`.
-      self.assertItemsEqual(full_filenames, all_produced_filenames[0])
-      # However, the different runs should produce filenames in the same order
-      # as each other.
-      self.assertEqual(all_produced_filenames[0], all_produced_filenames[1])
-      self.assertEqual(all_produced_filenames[0], all_produced_filenames[2])
-
-  def testEmptyDirectoryInitializer(self):
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
-    dataset = dataset_ops.Dataset.list_files(filename_placeholder)
-
-    with self.cached_session() as sess:
-      itr = dataset.make_initializable_iterator()
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError, 'No files matched pattern: '):
-        sess.run(
-            itr.initializer,
-            feed_dict={filename_placeholder: path.join(self.tmp_dir, '*')})
-
-  def testSimpleDirectoryInitializer(self):
-    filenames = ['a', 'b', 'c']
-    self._touchTempFiles(filenames)
-
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
-    dataset = dataset_ops.Dataset.list_files(filename_placeholder)
 
-    with self.cached_session() as sess:
-      itr = dataset.make_initializable_iterator()
-      next_element = itr.get_next()
-      sess.run(
-          itr.initializer,
-          feed_dict={filename_placeholder: path.join(self.tmp_dir, '*')})
+    full_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename))
+                      for filename in filenames]
 
-      full_filenames = []
+    all_produced_filenames = []
+    for _ in range(3):
       produced_filenames = []
-      for filename in filenames:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
+      next_element = self.getNext(dataset, requires_initialization=True)
+      try:
+        while True:
+          produced_filenames.append(self.evaluate(next_element()))
+      except errors.OutOfRangeError:
+        pass
+      all_produced_filenames.append(produced_filenames)
+
+    # Each run should produce the same set of filenames, which may be
+    # different from the order of `full_filenames`.
+    self.assertItemsEqual(full_filenames, all_produced_filenames[0])
+    # However, the different runs should produce filenames in the same order
+    # as each other.
+    self.assertEqual(all_produced_filenames[0], all_produced_filenames[1])
+    self.assertEqual(all_produced_filenames[0], all_produced_filenames[2])
+
+  # TODO(b/117581999): eager mode assertion fail wrapped, debug.
+  def tesSkipEagerEmptyDirectoryInitializer(self):
+    dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*'))
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(errors.InvalidArgumentError,
+                        'No files matched pattern'),
+        requires_initialization=True)
 
-      self.assertItemsEqual(full_filenames, produced_filenames)
+  def testSimpleDirectoryInitializer(self):
+    filenames = ['a', 'b', 'c']
+    self._touchTempFiles(filenames)
 
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*'))
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in filenames
+        ],
+        assert_items_equal=True)
 
   def testFileSuffixes(self):
     filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc']
     self._touchTempFiles(filenames)
 
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
-    dataset = dataset_ops.Dataset.list_files(filename_placeholder)
-
-    with self.cached_session() as sess:
-      itr = dataset.make_initializable_iterator()
-      next_element = itr.get_next()
-      sess.run(
-          itr.initializer,
-          feed_dict={filename_placeholder: path.join(self.tmp_dir, '*.py')})
-
-      full_filenames = []
-      produced_filenames = []
-      for filename in filenames[1:-1]:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
-      self.assertItemsEqual(full_filenames, produced_filenames)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*.py'))
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in filenames[1:-1]
+        ],
+        assert_items_equal=True)
 
   def testFileMiddles(self):
     filenames = ['a.txt', 'b.py', 'c.pyc']
     self._touchTempFiles(filenames)
 
-    filename_placeholder = array_ops.placeholder(dtypes.string, shape=[])
-    dataset = dataset_ops.Dataset.list_files(filename_placeholder)
-
-    with self.cached_session() as sess:
-      itr = dataset.make_initializable_iterator()
-      next_element = itr.get_next()
-      sess.run(
-          itr.initializer,
-          feed_dict={filename_placeholder: path.join(self.tmp_dir, '*.py*')})
-
-      full_filenames = []
-      produced_filenames = []
-      for filename in filenames[1:]:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
-
-      self.assertItemsEqual(full_filenames, produced_filenames)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*.py*'))
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in filenames[1:]
+        ],
+        assert_items_equal=True)
 
   def testNoShuffle(self):
     filenames = ['a', 'b', 'c']
@@ -222,21 +166,18 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
     # more meaningful.
     dataset = dataset_ops.Dataset.list_files(
         path.join(self.tmp_dir, '*'), shuffle=False).repeat(2)
-    with self.cached_session() as sess:
-      itr = dataset.make_one_shot_iterator()
-      next_element = itr.get_next()
-
-      full_filenames = []
-      produced_filenames = []
-      for filename in filenames * 2:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
-      self.assertItemsEqual(full_filenames, produced_filenames)
-      self.assertEqual(produced_filenames[:len(filenames)],
-                       produced_filenames[len(filenames):])
+    next_element = self.getNext(dataset)
+
+    full_filenames = []
+    produced_filenames = []
+    for filename in filenames * 2:
+      full_filenames.append(compat.as_bytes(path.join(self.tmp_dir, filename)))
+      produced_filenames.append(compat.as_bytes(self.evaluate(next_element())))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(next_element())
+    self.assertItemsEqual(full_filenames, produced_filenames)
+    self.assertEqual(produced_filenames[:len(filenames)],
+                     produced_filenames[len(filenames):])
 
   def testMultiplePatternsAsList(self):
     filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc']
@@ -244,47 +185,27 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
 
     patterns = [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']]
     dataset = dataset_ops.Dataset.list_files(patterns)
-    with self.cached_session() as sess:
-      itr = dataset.make_one_shot_iterator()
-      next_element = itr.get_next()
-
-      full_filenames = []
-      produced_filenames = []
-      for filename in filenames[:-1]:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
-      self.assertItemsEqual(full_filenames, produced_filenames)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in filenames[:-1]
+        ],
+        assert_items_equal=True)
 
   def testMultiplePatternsAsTensor(self):
     filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc']
     self._touchTempFiles(filenames)
 
-    filename_placeholder = array_ops.placeholder(
-        dtypes.string, shape=[
-            2,
-        ])
-    dataset = dataset_ops.Dataset.list_files(filename_placeholder)
-
-    with self.cached_session() as sess:
-      itr = dataset.make_initializable_iterator()
-      next_element = itr.get_next()
-      patterns = [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']]
-      sess.run(itr.initializer, feed_dict={filename_placeholder: patterns})
-
-      full_filenames = []
-      produced_filenames = []
-      for filename in filenames[:-1]:
-        full_filenames.append(
-            compat.as_bytes(path.join(self.tmp_dir, filename)))
-        produced_filenames.append(compat.as_bytes(sess.run(next_element)))
-      self.assertItemsEqual(full_filenames, produced_filenames)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(itr.get_next())
+    dataset = dataset_ops.Dataset.list_files(
+        [path.join(self.tmp_dir, pat) for pat in ['*.py', '*.txt']])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[
+            compat.as_bytes(path.join(self.tmp_dir, filename))
+            for filename in filenames[:-1]
+        ],
+        assert_items_equal=True)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/data/kernel_tests/optional_ops_test.py b/tensorflow/python/data/kernel_tests/optional_ops_test.py
index 604e3ad88e..5406a202a3 100644
--- a/tensorflow/python/data/kernel_tests/optional_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_ops_test.py
@@ -36,15 +36,14 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes
   def testFromValue(self):
     opt = optional_ops.Optional.from_value(constant_op.constant(37.0))
     self.assertTrue(self.evaluate(opt.has_value()))
     self.assertEqual(37.0, self.evaluate(opt.get_value()))
 
-  @test_util.run_in_graph_and_eager_modes
   def testFromStructuredValue(self):
     opt = optional_ops.Optional.from_value({
         "a": constant_op.constant(37.0),
@@ -56,7 +55,6 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
         "b": ([b"Foo"], b"Bar")
     }, self.evaluate(opt.get_value()))
 
-  @test_util.run_in_graph_and_eager_modes
   def testFromSparseTensor(self):
     st_0 = sparse_tensor.SparseTensorValue(
         indices=np.array([[0]]),
@@ -75,7 +73,6 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertAllEqual(expected.dense_shape,
                           self.evaluate(actual.dense_shape))
 
-  @test_util.run_in_graph_and_eager_modes
   def testFromNone(self):
     value_structure = structure.TensorStructure(dtypes.float32, [])
     opt = optional_ops.Optional.none_from_structure(value_structure)
@@ -90,7 +87,6 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(opt.get_value())
 
-  @test_util.run_in_graph_and_eager_modes
   def testCopyToGPU(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -151,7 +147,8 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
        optional_ops.OptionalStructure(
            structure.TensorStructure(dtypes.float32, []))),
   )
-  def testOptionalStructure(self, tf_value_fn, expected_value_structure):
+  def testSkipEagerOptionalStructure(self, tf_value_fn,
+                                     expected_value_structure):
     tf_value = tf_value_fn()
     opt = optional_ops.Optional.from_value(tf_value)
 
@@ -205,7 +202,8 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
                     indices=[[0, 1], [1, 0]], values=[37.0, 42.0],
                     dense_shape=[2, 2])}, False),
   )
-  def testIteratorGetNextAsOptional(self, np_value, tf_value_fn, works_on_gpu):
+  def testSkipEagerIteratorGetNextAsOptional(self, np_value, tf_value_fn,
+                                             works_on_gpu):
     if not works_on_gpu and test.is_gpu_available():
       self.skipTest("Test case not yet supported on GPU.")
     ds = dataset_ops.Dataset.from_tensors(np_value).repeat(3)
diff --git a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
index 76e2697b29..41b0689947 100644
--- a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
@@ -21,39 +21,24 @@ from absl.testing import parameterized
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.ops import array_ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class PrefetchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.parameters((-1), (0), (5))
   def testBufferSize(self, buffer_size):
-    buffer_size_t = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = dataset_ops.Dataset.range(10).prefetch(
-        buffer_size=buffer_size_t).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op, feed_dict={buffer_size_t: buffer_size})
-      for m in range(10):
-        self.assertEqual(m, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).prefetch(buffer_size=buffer_size)
+    self.assertDatasetProduces(dataset, expected_output=range(10))
 
   @parameterized.parameters((-2), (-42))
   def testInvalidBufferSize(self, buffer_size):
-    buffer_size_t = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = dataset_ops.Dataset.range(10).prefetch(
-        buffer_size=buffer_size_t).make_initializable_iterator()
-    init_op = iterator.initializer
-
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "buffer_size"):
-      with self.cached_session() as sess:
-        sess.run(init_op, feed_dict={buffer_size_t: buffer_size})
+    dataset = dataset_ops.Dataset.range(10).prefetch(buffer_size=buffer_size)
+    self.assertDatasetProduces(
+        dataset, expected_error=(errors.InvalidArgumentError, "buffer_size"))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
index e86356dee7..6da4e0dfca 100644
--- a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
@@ -21,12 +21,11 @@ import numpy as np
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.ops import array_ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class SequenceDatasetTest(test_base.DatasetTestBase):
 
   def testRepeatTensorDataset(self):
@@ -35,175 +34,101 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
     # This placeholder can be fed when dataset-definition subgraph
     # runs (i.e. `init_op` below) to configure the number of
     # repetitions used in a particular iterator.
-    count_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
 
-    iterator = (dataset_ops.Dataset.from_tensors(components)
-                .repeat(count_placeholder).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    def do_test(count):
+      dataset = dataset_ops.Dataset.from_tensors(components).repeat(count)
+      self.assertEqual([c.shape for c in components],
+                       [shape for shape in dataset.output_shapes])
+      self.assertDatasetProduces(dataset, [components] * count)
 
+    # Test a finite repetition.
+    do_test(3)
+
+    # test a different finite repetition.
+    do_test(7)
+
+    # Test an empty repetition.
+    do_test(0)
+
+    # Test an infinite repetition.
+    # NOTE(mrry): There's not a good way to test that the sequence
+    # actually is infinite.
+    dataset = dataset_ops.Dataset.from_tensors(components).repeat(-1)
     self.assertEqual([c.shape for c in components],
-                     [t.shape for t in get_next])
-
-    with self.cached_session() as sess:
-      # Test a finite repetition.
-      sess.run(init_op, feed_dict={count_placeholder: 3})
-      for _ in range(3):
-        results = sess.run(get_next)
-        for component, result_component in zip(components, results):
-          self.assertAllEqual(component, result_component)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test a different finite repetition.
-      sess.run(init_op, feed_dict={count_placeholder: 7})
-      for _ in range(7):
-        results = sess.run(get_next)
-        for component, result_component in zip(components, results):
-          self.assertAllEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test an empty repetition.
-      sess.run(init_op, feed_dict={count_placeholder: 0})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test an infinite repetition.
-      # NOTE(mrry): There's not a good way to test that the sequence
-      # actually is infinite.
-      sess.run(init_op, feed_dict={count_placeholder: -1})
-      for _ in range(17):
-        results = sess.run(get_next)
-        for component, result_component in zip(components, results):
-          self.assertAllEqual(component, result_component)
+                     [shape for shape in dataset.output_shapes])
+    get_next = self.getNext(dataset)
+    for _ in range(17):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(components, results):
+        self.assertAllEqual(component, result_component)
 
   def testTakeTensorDataset(self):
     components = (np.arange(10),)
-    count_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
 
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
-                .take(count_placeholder).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    def do_test(count):
+      dataset = dataset_ops.Dataset.from_tensor_slices(components).take(count)
+      self.assertEqual([c.shape[1:] for c in components],
+                       [shape for shape in dataset.output_shapes])
+      num_output = min(count, 10) if count != -1 else 10
+      self.assertDatasetProduces(
+          dataset, [tuple(components[0][i:i + 1]) for i in range(num_output)])
 
-    self.assertEqual([c.shape[1:] for c in components],
-                     [t.shape for t in get_next])
+    # Take fewer than input size
+    do_test(4)
 
-    with self.cached_session() as sess:
-      # Take fewer than input size
-      sess.run(init_op, feed_dict={count_placeholder: 4})
-      for i in range(4):
-        results = sess.run(get_next)
-        self.assertAllEqual(results, components[0][i:i+1])
+    # Take more than input size
+    do_test(25)
 
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    # Take all of input
+    do_test(-1)
 
-      # Take more than input size
-      sess.run(init_op, feed_dict={count_placeholder: 25})
-      for i in range(10):
-        results = sess.run(get_next)
-        self.assertAllEqual(results, components[0][i:i+1])
+    # Take nothing
+    do_test(0)
+
+  def testSkipTensorDataset(self):
+    components = (np.arange(10),)
 
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    def do_test(count):
+      dataset = dataset_ops.Dataset.from_tensor_slices(components).skip(count)
+      self.assertEqual([c.shape[1:] for c in components],
+                       [shape for shape in dataset.output_shapes])
+      start_range = min(count, 10) if count != -1 else 10
+      self.assertDatasetProduces(
+          dataset,
+          [tuple(components[0][i:i + 1]) for i in range(start_range, 10)])
 
-      # Take all of input
-      sess.run(init_op, feed_dict={count_placeholder: -1})
-      for i in range(10):
-        results = sess.run(get_next)
-        self.assertAllEqual(results, components[0][i:i+1])
+    # Skip fewer than input size, we should skip
+    # the first 4 elements and then read the rest.
+    do_test(4)
 
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    # Skip more than input size: get nothing.
+    do_test(25)
 
-      # Take nothing
-      sess.run(init_op, feed_dict={count_placeholder: 0})
+    # Skip exactly input size.
+    do_test(10)
 
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    # Set -1 for 'count': skip the entire dataset.
+    do_test(-1)
 
-  def testSkipTensorDataset(self):
-    components = (np.arange(10),)
-    count_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
-                .skip(count_placeholder).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual([c.shape[1:] for c in components],
-                     [t.shape for t in get_next])
-
-    with self.cached_session() as sess:
-      # Skip fewer than input size, we should skip
-      # the first 4 elements and then read the rest.
-      sess.run(init_op, feed_dict={count_placeholder: 4})
-      for i in range(4, 10):
-        results = sess.run(get_next)
-        self.assertAllEqual(results, components[0][i:i+1])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Skip more than input size: get nothing.
-      sess.run(init_op, feed_dict={count_placeholder: 25})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Skip exactly input size.
-      sess.run(init_op, feed_dict={count_placeholder: 10})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Set -1 for 'count': skip the entire dataset.
-      sess.run(init_op, feed_dict={count_placeholder: -1})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Skip nothing
-      sess.run(init_op, feed_dict={count_placeholder: 0})
-      for i in range(0, 10):
-        results = sess.run(get_next)
-        self.assertAllEqual(results, components[0][i:i+1])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    # Skip nothing
+    do_test(0)
 
   def testRepeatRepeatTensorDataset(self):
     """Test the composition of repeat datasets."""
     components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
-    inner_count = array_ops.placeholder(dtypes.int64, shape=[])
-    outer_count = array_ops.placeholder(dtypes.int64, shape=[])
-
-    iterator = (dataset_ops.Dataset.from_tensors(components).repeat(inner_count)
-                .repeat(outer_count).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    inner_count, outer_count = 7, 14
 
+    dataset = dataset_ops.Dataset.from_tensors(components).repeat(
+        inner_count).repeat(outer_count)
     self.assertEqual([c.shape for c in components],
-                     [t.shape for t in get_next])
-
-    with self.cached_session() as sess:
-      sess.run(init_op, feed_dict={inner_count: 7, outer_count: 14})
-      for _ in range(7 * 14):
-        results = sess.run(get_next)
-        for component, result_component in zip(components, results):
-          self.assertAllEqual(component, result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+                     [shape for shape in dataset.output_shapes])
+    self.assertDatasetProduces(dataset,
+                               [components] * (inner_count * outer_count))
 
   def testRepeatEmptyDataset(self):
     """Test that repeating an empty dataset does not hang."""
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10).skip(10)
-                .repeat(-1).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.from_tensors(0).repeat(10).skip(10).repeat(-1)
+    self.assertDatasetProduces(dataset, [])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py
index b9f3c79da5..7dd932d015 100644
--- a/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py
@@ -19,43 +19,26 @@ from __future__ import print_function
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ShardDatasetOpTest(test_base.DatasetTestBase):
 
   def testSimpleCase(self):
     dataset = dataset_ops.Dataset.range(10).shard(5, 2)
-    iterator = dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      self.assertEqual(2, sess.run(iterator.get_next()))
-      self.assertEqual(7, sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[2, 7])
 
   def testNestedData(self):
     dataset_a = dataset_ops.Dataset.range(10)
     dataset_b = dataset_ops.Dataset.range(10, 0, -1)
     dataset = dataset_ops.Dataset.zip((dataset_a, dataset_b)).shard(5, 2)
-    iterator = dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      self.assertEqual((2, 8), sess.run(iterator.get_next()))
-      self.assertEqual((7, 3), sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[(2, 8), (7, 3)])
 
   def testOffsetZero(self):
     dataset = dataset_ops.Dataset.range(10).shard(5, 0)
-    iterator = dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(iterator.get_next()))
-      self.assertEqual(5, sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[0, 5])
 
   def testOffsetGreaterNumShards(self):
     with self.assertRaises(ValueError):
@@ -75,37 +58,19 @@ class ShardDatasetOpTest(test_base.DatasetTestBase):
 
   def testIteratorEndsBeforeFirstElem(self):
     dataset = dataset_ops.Dataset.range(1).shard(5, 2)
-    iterator = dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[])
 
   def testLargerWorkerPool(self):
     dataset = dataset_ops.Dataset.range(10).shard(7, 5)
-    iterator = dataset.make_one_shot_iterator()
-    with self.cached_session() as sess:
-      self.assertEqual(5, sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[5])
 
   def testIndexEqualsNumShards(self):
     dataset = dataset_ops.Dataset.range(10).shard(5, 4)
-    iterator = dataset.make_one_shot_iterator()
-    with self.cached_session() as sess:
-      self.assertEqual(4, sess.run(iterator.get_next()))
-      self.assertEqual(9, sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[4, 9])
 
   def testIndexEqualsNumShards2(self):
     dataset = dataset_ops.Dataset.range(10).shard(4, 3)
-    iterator = dataset.make_one_shot_iterator()
-    with self.cached_session() as sess:
-      self.assertEqual(3, sess.run(iterator.get_next()))
-      self.assertEqual(7, sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    self.assertDatasetProduces(dataset, expected_output=[3, 7])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
index cad28f860e..bf8303ca6e 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
@@ -24,16 +24,17 @@ import numpy as np
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.framework import constant_op
+
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testShuffleDataset(self):
@@ -41,102 +42,80 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8]),
         np.array([9.0, 10.0, 11.0, 12.0])
     )
-    count_placeholder = array_ops.placeholder_with_default(
-        constant_op.constant(5, dtypes.int64), shape=[])
-    buffer_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-    seed_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-
-    repeat_dataset = (dataset_ops.Dataset.from_tensor_slices(components)
-                      .repeat(count_placeholder))
-
-    shuffle_dataset = repeat_dataset.shuffle(buffer_size_placeholder,
-                                             seed_placeholder)
-
-    self.assertEqual(tuple([c.shape[1:] for c in components]),
-                     shuffle_dataset.output_shapes)
-
-    # Create initialization ops for iterators without and with
-    # shuffling, respectively.
-    iterator = iterator_ops.Iterator.from_structure(
-        shuffle_dataset.output_types, shuffle_dataset.output_shapes)
-    init_fifo_op = iterator.make_initializer(repeat_dataset)
-    init_shuffle_op = iterator.make_initializer(shuffle_dataset)
-
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # First run without shuffling to collect the "ground truth".
-      sess.run(init_fifo_op)
-      unshuffled_elements = []
-      for _ in range(20):
-        unshuffled_elements.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Assert that the shuffled dataset has the same elements as the
-      # "ground truth".
-      sess.run(
-          init_shuffle_op,
-          feed_dict={buffer_size_placeholder: 100,
-                     seed_placeholder: 37})
-      shuffled_elements = []
-      for _ in range(20):
-        shuffled_elements.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      self.assertAllEqual(
-          sorted(unshuffled_elements), sorted(shuffled_elements))
-
-      # Assert that shuffling twice with the same seeds gives the same sequence.
-      sess.run(
-          init_shuffle_op,
-          feed_dict={buffer_size_placeholder: 100,
-                     seed_placeholder: 37})
-      reshuffled_elements_same_seed = []
-      for _ in range(20):
-        reshuffled_elements_same_seed.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      self.assertEqual(shuffled_elements, reshuffled_elements_same_seed)
-
-      # Assert that shuffling twice with a different seed gives a different
-      # permutation of the same elements.
-      sess.run(
-          init_shuffle_op,
-          feed_dict={buffer_size_placeholder: 100,
-                     seed_placeholder: 1037})
-      reshuffled_elements_different_seed = []
-      for _ in range(20):
-        reshuffled_elements_different_seed.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      self.assertNotEqual(shuffled_elements, reshuffled_elements_different_seed)
-      self.assertAllEqual(
-          sorted(shuffled_elements), sorted(reshuffled_elements_different_seed))
-
-      # Assert that the shuffled dataset has the same elements as the
-      # "ground truth" when the buffer size is smaller than the input
-      # dataset.
-      sess.run(
-          init_shuffle_op,
-          feed_dict={buffer_size_placeholder: 2,
-                     seed_placeholder: 37})
-      reshuffled_elements_small_buffer = []
-      for _ in range(20):
-        reshuffled_elements_small_buffer.append(sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-      self.assertAllEqual(
-          sorted(unshuffled_elements), sorted(reshuffled_elements_small_buffer))
 
-      # Test the case of shuffling an empty dataset.
-      sess.run(init_shuffle_op, feed_dict={buffer_size_placeholder: 2,
-                                           seed_placeholder: 37,
-                                           count_placeholder: 0})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    def dataset_fn(count=5, buffer_size=None, seed=0):
+      repeat_dataset = (
+          dataset_ops.Dataset.from_tensor_slices(components).repeat(count))
+      if buffer_size:
+        shuffle_dataset = repeat_dataset.shuffle(buffer_size, seed)
 
-  def testSeedZero(self):
+        self.assertEqual(
+            tuple([c.shape[1:] for c in components]),
+            shuffle_dataset.output_shapes)
+        return shuffle_dataset
+      else:
+        return repeat_dataset
+
+    # First run without shuffling to collect the "ground truth".
+    get_next = self.getNext(dataset_fn())
+    unshuffled_elements = []
+    for _ in range(20):
+      unshuffled_elements.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+    # Assert that the shuffled dataset has the same elements as the
+    # "ground truth".
+    get_next = self.getNext(dataset_fn(buffer_size=100, seed=37))
+    shuffled_elements = []
+    for _ in range(20):
+      shuffled_elements.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    self.assertAllEqual(sorted(unshuffled_elements), sorted(shuffled_elements))
+
+    # Assert that shuffling twice with the same seeds gives the same sequence.
+    get_next = self.getNext(dataset_fn(buffer_size=100, seed=37))
+    reshuffled_elements_same_seed = []
+    for _ in range(20):
+      reshuffled_elements_same_seed.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    self.assertEqual(shuffled_elements, reshuffled_elements_same_seed)
+
+    # Assert that shuffling twice with a different seed gives a different
+    # permutation of the same elements.
+    get_next = self.getNext(dataset_fn(buffer_size=100, seed=137))
+    reshuffled_elements_different_seed = []
+    for _ in range(20):
+      reshuffled_elements_different_seed.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    self.assertNotEqual(shuffled_elements, reshuffled_elements_different_seed)
+    self.assertAllEqual(
+        sorted(shuffled_elements), sorted(reshuffled_elements_different_seed))
+
+    # Assert that the shuffled dataset has the same elements as the
+    # "ground truth" when the buffer size is smaller than the input
+    # dataset.
+    get_next = self.getNext(dataset_fn(buffer_size=2, seed=37))
+    reshuffled_elements_small_buffer = []
+    for _ in range(20):
+      reshuffled_elements_small_buffer.append(self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    self.assertAllEqual(
+        sorted(unshuffled_elements), sorted(reshuffled_elements_small_buffer))
+
+    # Test the case of shuffling an empty dataset.
+    get_next = self.getNext(dataset_fn(count=0, buffer_size=100, seed=37))
+
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+  def testSkipEagerSeedZero(self):
     """Test for same behavior when the seed is a Python or Tensor zero."""
     iterator = (
         dataset_ops.Dataset.range(10).shuffle(10, seed=0)
@@ -165,52 +144,40 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testDefaultArguments(self):
     components = [0, 1, 2, 3, 4]
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components).shuffle(5)
-                .repeat().make_one_shot_iterator())
-
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      counts = collections.defaultdict(lambda: 0)
-      for _ in range(10):
-        for _ in range(5):
-          counts[sess.run(get_next)] += 1
+    dataset = dataset_ops.Dataset.from_tensor_slices(components).shuffle(
+        5).repeat()
+    get_next = self.getNext(dataset)
+    counts = collections.defaultdict(lambda: 0)
+    for _ in range(10):
+      for _ in range(5):
+        counts[self.evaluate(get_next())] += 1
 
     for i in range(5):
       self.assertEqual(10, counts[i])
 
   def testShuffleNoReshuffleEachIteration(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .shuffle(10, reshuffle_each_iteration=False)
-                .batch(10)
-                .repeat(3)
-                .make_one_shot_iterator())
-    next_element = iterator.get_next()
+    dataset = dataset_ops.Dataset.range(10).shuffle(
+        10, reshuffle_each_iteration=False).batch(10).repeat(3)
+    next_element = self.getNext(dataset)
 
-    with self.cached_session() as sess:
-      initial_permutation = sess.run(next_element)
-      self.assertAllEqual(initial_permutation, sess.run(next_element))
-      self.assertAllEqual(initial_permutation, sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+    initial_permutation = self.evaluate(next_element())
+    self.assertAllEqual(initial_permutation, self.evaluate(next_element()))
+    self.assertAllEqual(initial_permutation, self.evaluate(next_element()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(next_element())
 
   def testShuffleReshuffleEachIteration(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .shuffle(10, seed=3, reshuffle_each_iteration=True)
-                .batch(10)
-                .repeat(3)
-                .make_one_shot_iterator())
-    next_element = iterator.get_next()
+    dataset = dataset_ops.Dataset.range(10).shuffle(
+        10, seed=3, reshuffle_each_iteration=True).batch(10).repeat(3)
+    next_element = self.getNext(dataset)
 
-    with self.cached_session() as sess:
-      initial_permutation = list(sess.run(next_element))
-      for _ in range(2):
-        next_permutation = list(sess.run(next_element))
-        self.assertNotEqual(initial_permutation, next_permutation)
-        self.assertAllEqual(
-            sorted(initial_permutation), sorted(next_permutation))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+    initial_permutation = list(self.evaluate(next_element()))
+    for _ in range(2):
+      next_permutation = list(self.evaluate(next_element()))
+      self.assertNotEqual(initial_permutation, next_permutation)
+      self.assertAllEqual(sorted(initial_permutation), sorted(next_permutation))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(next_element())
 
   @parameterized.named_parameters(
       ("ReshuffleGraphLevelSeed", True, 38, None),
@@ -220,7 +187,8 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("NoReshuffleOpLevelSeed", False, None, 42),
       ("NoReshuffleGraphAndOpLevelSeed", False, 38, 42),
   )
-  def testShuffleSeed(self, reshuffle, graph_level_seed, op_level_seed):
+  def testSkipEagerShuffleSeed(self, reshuffle, graph_level_seed,
+                               op_level_seed):
     results = []
     for _ in range(2):
       with ops.Graph().as_default() as g:
@@ -241,13 +209,15 @@ class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     self.assertAllEqual(results[0], results[1])
 
+  # TODO(b/117581999): fails for eager mode with result[0] equal to result[1],
+  # debug.
   @parameterized.named_parameters(
       ("ReshuffleOneShot", True, False),
       ("ReshuffleInitializable", True, True),
       ("NoReshuffleOneShot", False, False),
       ("NoReshuffleInitializable", False, True),
   )
-  def testMultipleIterators(self, reshuffle, initializable):
+  def testSkipEagerMultipleIterators(self, reshuffle, initializable):
     with ops.Graph().as_default() as g:
       dataset = dataset_ops.Dataset.range(100).shuffle(
           10, reshuffle_each_iteration=reshuffle).repeat(3)
diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index ca853da3f6..af8e5e8a27 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -68,19 +68,29 @@ class DatasetTestBase(test.TestCase):
         iterator = dataset.make_one_shot_iterator()
       return iterator.get_next
 
-  def _compareOutputToExpected(self, result_values, expected_values):
+  def _compareOutputToExpected(self, result_values, expected_values,
+                               assert_items_equal):
+    if assert_items_equal:
+      # TODO(shivaniagrawal): add support for nested elements containing sparse
+      # tensors when needed.
+      self.assertItemsEqual(result_values, expected_values)
+      return
     for i in range(len(result_values)):
-      if sparse_tensor.is_sparse(result_values[i]):
-        self.assertSparseValuesEqual(result_values[i], expected_values[i])
-      else:
-        self.assertAllEqual(result_values[i], expected_values[i])
+      nest.assert_same_structure(result_values[i], expected_values[i])
+      for result_value, expected_value in zip(
+          nest.flatten(result_values[i]), nest.flatten(expected_values[i])):
+        if sparse_tensor.is_sparse(result_value):
+          self.assertSparseValuesEqual(result_value, expected_value)
+        else:
+          self.assertAllEqual(result_value, expected_value)
 
   def assertDatasetProduces(self,
                             dataset,
                             expected_output=None,
                             expected_error=None,
                             requires_initialization=False,
-                            num_test_iterations=2):
+                            num_test_iterations=1,
+                            assert_items_equal=False):
     """Asserts that a dataset produces the expected output / error.
 
     Args:
@@ -98,6 +108,8 @@ class DatasetTestBase(test.TestCase):
         dataset (e.g. when it contains stateful nodes). Defaults to False.
       num_test_iterations: Number of times `dataset` will be iterated. Defaults
         to 2.
+      assert_items_equal: Tests expected_output has (only) the same elements
+        regardless of order.
     """
     self.assertTrue(
         expected_error is not None or expected_output is not None,
@@ -120,7 +132,7 @@ class DatasetTestBase(test.TestCase):
       result = []
       for _ in range(len(expected_output)):
         result.append(self.evaluate(get_next()))
-      self._compareOutputToExpected(result, expected_output)
+      self._compareOutputToExpected(result, expected_output, assert_items_equal)
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next())
       with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
index 9d76387a34..c86765addc 100644
--- a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
@@ -21,94 +21,80 @@ import numpy as np
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.ops import array_ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ZipDatasetTest(test_base.DatasetTestBase):
 
   def testZipDataset(self):
-    component_placeholders = [
-        array_ops.placeholder(dtypes.int64),
-        array_ops.placeholder(dtypes.int64),
-        array_ops.placeholder(dtypes.float64)
-    ]
-
-    datasets = tuple([
-        dataset_ops.Dataset.from_tensor_slices(component_placeholder)
-        for component_placeholder in component_placeholders
-    ])
-    zipped = dataset_ops.Dataset.zip(datasets)
-
-    iterator = zipped.make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
 
-    with self.cached_session() as sess:
-      equal_length_components = [
-          np.tile(np.array([[1], [2], [3], [4]]), 20),
-          np.tile(np.array([[12], [13], [14], [15]]), 22),
-          np.array([37.0, 38.0, 39.0, 40.0])
-      ]
-      sess.run(init_op, feed_dict={ph: value for ph, value in zip(
-          component_placeholders, equal_length_components)})
-      for i in range(4):
-        results = sess.run(get_next)
-        for component, result_component in zip(
-            equal_length_components, results):
-          self.assertAllEqual(component[i], result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    def dataset_fn(components):
+      datasets = tuple([
+          dataset_ops.Dataset.from_tensor_slices(component)
+          for component in components
+      ])
+      return dataset_ops.Dataset.zip(datasets)
+
+    equal_length_components = [
+        np.tile(np.array([[1], [2], [3], [4]]), 20),
+        np.tile(np.array([[12], [13], [14], [15]]), 22),
+        np.array([37.0, 38.0, 39.0, 40.0])
+    ]
 
-      variable_length_components = [[1, 2, 3, 4], [1, 2, 3, 4, 5], [1.0, 2.0]]
-      sess.run(init_op, feed_dict={ph: value for ph, value in zip(
-          component_placeholders, variable_length_components)})
-      for i in range(2):
-        results = sess.run(get_next)
-        for component, result_component in zip(
-            variable_length_components, results):
-          self.assertAllEqual(component[i], result_component)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    get_next = self.getNext(dataset_fn(equal_length_components))
+    for i in range(4):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(equal_length_components, results):
+        self.assertAllEqual(component[i], result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+    variable_length_components = [[1, 2, 3, 4], [1, 2, 3, 4, 5], [1.0, 2.0]]
+    get_next = self.getNext(dataset_fn(variable_length_components))
+    for i in range(2):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(variable_length_components,
+                                             results):
+        self.assertAllEqual(component[i], result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testNestedZipDataset(self):
-    component_placeholders = [
-        array_ops.placeholder(dtypes.int64, shape=[4, 20]),
-        array_ops.placeholder(dtypes.int64, shape=[4, 22]),
-        array_ops.placeholder(dtypes.float64, shape=[4])
-    ]
 
+    equal_length_components = [
+        np.tile(np.array([[1], [2], [3], [4]]), 20),
+        np.tile(np.array([[12], [13], [14], [15]]), 22),
+        np.array([37.0, 38.0, 39.0, 40.0])
+    ]
     datasets = [
-        dataset_ops.Dataset.from_tensor_slices(component_placeholder)
-        for component_placeholder in component_placeholders
+        dataset_ops.Dataset.from_tensor_slices(component)
+        for component in equal_length_components
     ]
-    zipped = dataset_ops.Dataset.zip((datasets[0], (datasets[1], datasets[2])))
-
-    iterator = zipped.make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual([20], get_next[0].shape)
-    self.assertEqual([22], get_next[1][0].shape)
-    self.assertEqual([], get_next[1][1].shape)
-
-    with self.cached_session() as sess:
-      equal_length_components = [
-          np.tile(np.array([[1], [2], [3], [4]]), 20),
-          np.tile(np.array([[12], [13], [14], [15]]), 22),
-          np.array([37.0, 38.0, 39.0, 40.0])
-      ]
-      sess.run(init_op, feed_dict={ph: value for ph, value in zip(
-          component_placeholders, equal_length_components)})
-      for i in range(4):
-        result1, (result2, result3) = sess.run(get_next)
-        self.assertAllEqual(equal_length_components[0][i], result1)
-        self.assertAllEqual(equal_length_components[1][i], result2)
-        self.assertAllEqual(equal_length_components[2][i], result3)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.zip((datasets[0], (datasets[1], datasets[2])))
+
+    self.assertEqual(
+        dataset.output_shapes,
+        (tensor_shape.TensorShape([20]),
+         (tensor_shape.TensorShape([22]), tensor_shape.TensorShape([]))))
+
+    get_next = self.getNext(dataset)
+    for i in range(4):
+      result1, (result2, result3) = self.evaluate(get_next())
+      self.assertAllEqual(equal_length_components[0][i], result1)
+      self.assertAllEqual(equal_length_components[1][i], result2)
+      self.assertAllEqual(equal_length_components[2][i], result3)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
 
 if __name__ == "__main__":
-- 
GitLab


From 1d8c26069ce359be3d36f0a323993e919c0f6d73 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 26 Nov 2018 10:43:45 -0800
Subject: [PATCH 0775/1554] Fix error message and the checks for
 tf.get_variable applying the initializer to get the default value

PiperOrigin-RevId: 222849262
---
 tensorflow/python/ops/variable_scope.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 1b122ceffd..cf33c61b82 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -909,18 +909,20 @@ class _VariableStore(object):
         # Instantiate initializer if provided initializer is a type object.
         if isinstance(initializer, type(init_ops.Initializer)):
           initializer = initializer(dtype=dtype)
-        if shape and shape.is_fully_defined():
+        spec = tf_inspect.getargspec(initializer)
+        if shape is not None and shape.is_fully_defined():
           init_val = lambda: initializer(  # pylint: disable=g-long-lambda
               shape.as_list(), dtype=dtype, partition_info=partition_info)
-        elif not tf_inspect.getargspec(initializer).args:
+          variable_dtype = dtype.base_dtype
+        elif len(spec.args) == len(spec.defaults or []):
           init_val = initializer
+          variable_dtype = None
         else:
-          raise ValueError("You can only pass an initializer function that "
-                           "expects no arguments to its callable when the "
-                           "shape is not fully defined. The given initializer "
-                           "function expects the following args %s" %
-                           tf_inspect.getargspec(initializer).args)
-        variable_dtype = dtype.base_dtype
+          raise ValueError("The initializer passed is not valid. It should "
+                           "be a callable with no arguments and the "
+                           "shape should not be provided or an instance of "
+                           "`tf.keras.initializers.*' and `shape` should be "
+                           "fully defined.")
 
     # Create the variable.
     if use_resource is None:
-- 
GitLab


From a52d67777b0ac6f8f56ba17386ff7bafe1c2e4e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 10:46:58 -0800
Subject: [PATCH 0776/1554] update/remove symbols

PiperOrigin-RevId: 222849751
---
 tensorflow/python/framework/ops.py              |  2 +-
 tensorflow/python/ops/partitioned_variables.py  |  2 +-
 tensorflow/python/ops/random_ops.py             |  4 +++-
 tensorflow/python/ops/sparse_ops.py             |  2 +-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 16 ----------------
 tensorflow/tools/compatibility/renames_v2.py    |  4 ++++
 6 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index c465d2bc10..b8187ab20c 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5557,7 +5557,7 @@ def eager_run(main=None, argv=None):
   app.run(main, argv)
 
 
-@tf_export("reset_default_graph")
+@tf_export(v1=["reset_default_graph"])
 def reset_default_graph():
   """Clears the default graph stack and resets the global default graph.
 
diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py
index 816317da10..66153e33f2 100644
--- a/tensorflow/python/ops/partitioned_variables.py
+++ b/tensorflow/python/ops/partitioned_variables.py
@@ -68,7 +68,7 @@ __all__ = [
 ]
 
 
-@tf_export("variable_axis_size_partitioner")
+@tf_export(v1=["variable_axis_size_partitioner"])
 def variable_axis_size_partitioner(
     max_shard_bytes, axis=0, bytes_per_string_element=16, max_shards=None):
   """Get a partitioner for VariableScope to keep shards below `max_shard_bytes`.
diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py
index c893ef011b..f2df87cf2d 100644
--- a/tensorflow/python/ops/random_ops.py
+++ b/tensorflow/python/ops/random_ops.py
@@ -138,7 +138,9 @@ def parameterized_truncated_normal(shape,
     return rnd
 
 
-@tf_export("random.truncated_normal", "truncated_normal")
+@tf_export("random.truncated_normal",
+           v1=["random.truncated_normal", "truncated_normal"])
+@deprecation.deprecated_endpoints("truncated_normal")
 def truncated_normal(shape,
                      mean=0.0,
                      stddev=1.0,
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index e8a0f31d9d..91baa6f7b8 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -939,7 +939,7 @@ def sparse_slice(sp_input, start, size, name=None):
                                       output_shape)
 
 
-@tf_export("sparse_to_dense")
+@tf_export(v1=["sparse_to_dense"])
 @deprecation.deprecated(
     None,
     "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 99751504ae..6afe44a03f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -928,10 +928,6 @@ tf_module {
     name: "required_space_to_batch_paddings"
     argspec: "args=[\'input_shape\', \'block_shape\', \'base_paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "reset_default_graph"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reshape"
     argspec: "args=[\'tensor\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -1036,10 +1032,6 @@ tf_module {
     name: "sparse_concat"
     argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'concat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "sparse_to_dense"
-    argspec: "args=[\'sparse_indices\', \'output_shape\', \'sparse_values\', \'default_value\', \'validate_indices\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'True\', \'None\'], "
-  }
   member_method {
     name: "split"
     argspec: "args=[\'value\', \'num_or_size_splits\', \'axis\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \'split\'], "
@@ -1108,10 +1100,6 @@ tf_module {
     name: "truediv"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "truncated_normal"
-    argspec: "args=[\'shape\', \'mean\', \'stddev\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
-  }
   member_method {
     name: "truncatediv"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -1140,10 +1128,6 @@ tf_module {
     name: "unstack"
     argspec: "args=[\'value\', \'num\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'unstack\'], "
   }
-  member_method {
-    name: "variable_axis_size_partitioner"
-    argspec: "args=[\'max_shard_bytes\', \'axis\', \'bytes_per_string_element\', \'max_shards\'], varargs=None, keywords=None, defaults=[\'0\', \'16\', \'None\'], "
-  }
   member_method {
     name: "variable_creator_scope"
     argspec: "args=[\'variable_creator\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index dc47796a7c..c22b09df48 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -392,6 +392,7 @@ renames = {
     'tf.reduce_join': 'tf.strings.reduce_join',
     'tf.regex_replace': 'tf.strings.regex_replace',
     'tf.report_uninitialized_variables': 'tf.compat.v1.report_uninitialized_variables',
+    'tf.reset_default_graph': 'tf.compat.v1.reset_default_graph',
     'tf.resource_loader.get_data_files_path': 'tf.compat.v1.resource_loader.get_data_files_path',
     'tf.resource_loader.get_path_to_datafile': 'tf.compat.v1.resource_loader.get_path_to_datafile',
     'tf.resource_loader.get_root_dir_with_all_resources': 'tf.compat.v1.resource_loader.get_root_dir_with_all_resources',
@@ -499,6 +500,7 @@ renames = {
     'tf.sparse_split': 'tf.compat.v1.sparse_split',
     'tf.sparse_tensor_dense_matmul': 'tf.sparse.sparse_dense_matmul',
     'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense',
+    'tf.sparse_to_dense': 'tf.compat.v1.sparse_to_dense',
     'tf.sparse_to_indicator': 'tf.sparse.to_indicator',
     'tf.sparse_transpose': 'tf.sparse.transpose',
     'tf.spectral.dct': 'tf.signal.dct',
@@ -594,6 +596,7 @@ renames = {
     'tf.train.update_checkpoint_state': 'tf.compat.v1.train.update_checkpoint_state',
     'tf.train.write_graph': 'tf.io.write_graph',
     'tf.trainable_variables': 'tf.compat.v1.trainable_variables',
+    'tf.truncated_normal': 'tf.random.truncated_normal',
     'tf.uniform_unit_scaling_initializer': 'tf.initializers.uniform_unit_scaling',
     'tf.unsorted_segment_max': 'tf.math.unsorted_segment_max',
     'tf.unsorted_segment_mean': 'tf.math.unsorted_segment_mean',
@@ -601,6 +604,7 @@ renames = {
     'tf.unsorted_segment_prod': 'tf.math.unsorted_segment_prod',
     'tf.unsorted_segment_sqrt_n': 'tf.math.unsorted_segment_sqrt_n',
     'tf.unsorted_segment_sum': 'tf.math.unsorted_segment_sum',
+    'tf.variable_axis_size_partitioner': 'tf.compat.v1.variable_axis_size_partitioner',
     'tf.variable_op_scope': 'tf.compat.v1.variable_op_scope',
     'tf.variable_scope': 'tf.compat.v1.variable_scope',
     'tf.variables_initializer': 'tf.compat.v1.variables_initializer',
-- 
GitLab


From a70c22df20ffa56602d01be09187df76cb377f12 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 26 Nov 2018 10:48:30 -0800
Subject: [PATCH 0777/1554] [tf.data] Fix threadpool names.

The `ThreadPool` class adds the `tf_` prefix to the threadpool name in its constructor.

PiperOrigin-RevId: 222850033
---
 .../core/kernels/data/experimental/threadpool_dataset_op.cc    | 3 +--
 tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 139642d4a3..341ce08a57 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -359,8 +359,7 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
           input_(input),
           num_threads_(num_threads) {
       thread_pool_ = MakeUnique<thread::ThreadPool>(
-          ctx->env(), ThreadOptions{}, "tf_data_private_threadpool",
-          num_threads,
+          ctx->env(), ThreadOptions{}, "data_private_threadpool", num_threads,
           /*low_latency_hint=*/false);
       input_->Ref();
     }
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 985e197a99..23e6adc57a 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -1241,7 +1241,7 @@ class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
             element_in_use_(params.dataset->cycle_length_, false),
             thread_pool_(new thread::ThreadPool(
                 Env::Default(), ThreadOptions(),
-                "tf_data_parallel_interleave_worker_pool",
+                "data_parallel_interleave_worker_pool",
                 dataset()->cycle_length_ /* num_threads */,
                 false /* low_latency_hint */)) {
         std::vector<string> components =
-- 
GitLab


From 556b91e88fce84df4aca3302dc28c309a930546c Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Mon, 26 Nov 2018 10:50:59 -0800
Subject: [PATCH 0778/1554] Split conv GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile.

PiperOrigin-RevId: 222850419
---
 tensorflow/core/kernels/BUILD                 | 10 ++-
 tensorflow/core/kernels/conv_2d.h             |  2 +-
 .../{conv_2d_gpu.cu.cc => conv_2d_gpu.h}      | 86 ++-----------------
 .../core/kernels/conv_2d_gpu_double.cu.cc     | 50 +++++++++++
 .../core/kernels/conv_2d_gpu_float.cu.cc      | 63 ++++++++++++++
 .../core/kernels/conv_2d_gpu_half.cu.cc       | 57 ++++++++++++
 tensorflow/core/kernels/conv_2d_gpu_int.cu.cc | 38 ++++++++
 .../core/kernels/conv_2d_gpu_uint16.cu.cc     | 38 ++++++++
 .../core/kernels/conv_2d_gpu_uint32.cu.cc     | 38 ++++++++
 .../core/kernels/conv_2d_gpu_uint64.cu.cc     | 38 ++++++++
 .../core/kernels/conv_2d_gpu_uint8.cu.cc      | 38 ++++++++
 11 files changed, 377 insertions(+), 81 deletions(-)
 rename tensorflow/core/kernels/{conv_2d_gpu.cu.cc => conv_2d_gpu.h} (91%)
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_half.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_int.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint16.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint32.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint64.cu.cc
 create mode 100644 tensorflow/core/kernels/conv_2d_gpu_uint8.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ae76034b0b..1efce939a3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -199,8 +199,16 @@ tf_kernel_library(
     name = "conv_2d",
     hdrs = ["conv_2d.h"],
     gpu_srcs = [
-        "conv_2d_gpu.cu.cc",
         "conv_2d.h",
+        "conv_2d_gpu.h",
+        "conv_2d_gpu_double.cu.cc",
+        "conv_2d_gpu_float.cu.cc",
+        "conv_2d_gpu_half.cu.cc",
+        "conv_2d_gpu_int.cu.cc",
+        "conv_2d_gpu_uint16.cu.cc",
+        "conv_2d_gpu_uint32.cu.cc",
+        "conv_2d_gpu_uint64.cu.cc",
+        "conv_2d_gpu_uint8.cu.cc",
     ],
     deps = [
         ":eigen_helpers",
diff --git a/tensorflow/core/kernels/conv_2d.h b/tensorflow/core/kernels/conv_2d.h
index a6964b1aac..1bac2a18c3 100644
--- a/tensorflow/core/kernels/conv_2d.h
+++ b/tensorflow/core/kernels/conv_2d.h
@@ -162,7 +162,7 @@ struct TransformFilter {
     merged_dims[1] = in.dimension(NDIMS - 2);  // input filters
     merged_dims[2] = in.dimension(NDIMS - 1);  // output filters
 
-    CHECK(dst_filter_format == FORMAT_OIHW)
+    DCHECK(dst_filter_format == FORMAT_OIHW)
         << "Unsupported destination filter format: "
         << ToString(dst_filter_format);
     // Source filter format is FORMAT_HWIO and spatial dimensions HW are merged
diff --git a/tensorflow/core/kernels/conv_2d_gpu.cu.cc b/tensorflow/core/kernels/conv_2d_gpu.h
similarity index 91%
rename from tensorflow/core/kernels/conv_2d_gpu.cu.cc
rename to tensorflow/core/kernels/conv_2d_gpu.h
index c6adf9ebff..8d11757428 100644
--- a/tensorflow/core/kernels/conv_2d_gpu.cu.cc
+++ b/tensorflow/core/kernels/conv_2d_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_CONV_2D_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_CONV_2D_GPU_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -34,7 +37,7 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
-namespace {
+
 template <typename T, bool conjugate>
 struct maybe_conj {
   __device__ static __inline__ T run(T x) {
@@ -75,8 +78,6 @@ struct maybe_conj<double2, conjugate> {
   }
 };
 
-}  // namespace
-
 // TODO(mjanusz): Move this to a shared util file.
 // A simple array that contains data that can be passed between CPU and GPU.
 template <typename T, int IndexCount, T DefaultValue>
@@ -433,7 +434,7 @@ struct TransformFilter<GPUDevice, T, int, NDIMS> {
     combined_dims[2] = in.dimension(NDIMS - 1);  // output filters
     CudaLaunchConfig config = GetCudaLaunchConfig(out.size(), d);
 
-    DCHECK(dst_filter_format == FORMAT_OIHW)
+    CHECK(dst_filter_format == FORMAT_OIHW)
         << "Unsupported output layout: " << ToString(dst_filter_format);
 
     ShuffleInTensor3Simple<T, 2, 1, 0>
@@ -998,82 +999,9 @@ struct NCHWToNHWC<GPUDevice, T, NDIMS> {
   }
 };
 
-template struct ShuffleAndReverse<Eigen::GpuDevice, float, 4, int>;
-template struct ShuffleAndReverse<Eigen::GpuDevice, Eigen::half, 4, int>;
-
-template struct ShuffleAndReverse<Eigen::GpuDevice, float, 4,
-                                  Eigen::DenseIndex>;
-template struct ShuffleAndReverse<Eigen::GpuDevice, Eigen::half, 4,
-                                  Eigen::DenseIndex>;
-
-template struct TransformDepth<Eigen::GpuDevice, float, int>;
-template struct TransformDepth<Eigen::GpuDevice, Eigen::half, int>;
-
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint8>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint16>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint32>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint64>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, float4>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, float2,
-                                            /*conjugate=*/true>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, double2,
-                                            /*conjugate=*/true>;
-template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, Eigen::half>;
-
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint8>;
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint16>;
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint32>;
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint64>;
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, float4>;
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, float2,
-                                            /*conjugate=*/true>;
-template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, double2,
-                                            /*conjugate=*/true>;
-
-// For 2d ops.
-template struct TransformFilter<Eigen::GpuDevice, double, int, 4>;
-template struct TransformFilter<Eigen::GpuDevice, float, int, 4>;
-template struct TransformFilter<Eigen::GpuDevice, Eigen::half, int, 4>;
-
-template struct ReverseTransformFilter<Eigen::GpuDevice, double, 4>;
-template struct ReverseTransformFilter<Eigen::GpuDevice, float, 4>;
-template struct ReverseTransformFilter<Eigen::GpuDevice, Eigen::half, 4>;
-
-template struct NHWCToNCHW<Eigen::GpuDevice, double, 4>;
-template struct NHWCToNCHW<Eigen::GpuDevice, float, 4>;
-template struct NHWCToNCHW<Eigen::GpuDevice, Eigen::half, 4>;
-
-template struct NCHWToNHWC<Eigen::GpuDevice, double, 4>;
-template struct NCHWToNHWC<Eigen::GpuDevice, float, 4>;
-template struct NCHWToNHWC<Eigen::GpuDevice, Eigen::half, 4>;
-
-template struct PadInput<Eigen::GpuDevice, int, int, 4>;
-template struct PadInput<Eigen::GpuDevice, double, int, 4>;
-template struct PadInput<Eigen::GpuDevice, float, int, 4>;
-template struct PadInput<Eigen::GpuDevice, Eigen::half, int, 4>;
-
-// For 3d ops.
-template struct TransformFilter<Eigen::GpuDevice, double, int, 5>;
-template struct TransformFilter<Eigen::GpuDevice, float, int, 5>;
-template struct TransformFilter<Eigen::GpuDevice, Eigen::half, int, 5>;
-
-template struct ReverseTransformFilter<Eigen::GpuDevice, double, 5>;
-template struct ReverseTransformFilter<Eigen::GpuDevice, float, 5>;
-template struct ReverseTransformFilter<Eigen::GpuDevice, Eigen::half, 5>;
-
-template struct NHWCToNCHW<Eigen::GpuDevice, double, 5>;
-template struct NHWCToNCHW<Eigen::GpuDevice, float, 5>;
-template struct NHWCToNCHW<Eigen::GpuDevice, Eigen::half, 5>;
-
-template struct NCHWToNHWC<Eigen::GpuDevice, double, 5>;
-template struct NCHWToNHWC<Eigen::GpuDevice, float, 5>;
-template struct NCHWToNHWC<Eigen::GpuDevice, Eigen::half, 5>;
-
-template struct PadInput<Eigen::GpuDevice, double, int, 5>;
-template struct PadInput<Eigen::GpuDevice, float, int, 5>;
-template struct PadInput<Eigen::GpuDevice, Eigen::half, int, 5>;
-
 }  // namespace functor
 }  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_CONV_2D_GPU_H_
diff --git a/tensorflow/core/kernels/conv_2d_gpu_double.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_double.cu.cc
new file mode 100644
index 0000000000..353d6d1130
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_double.cu.cc
@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, double2,
+                                            /*conjugate=*/true>;
+
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, double2,
+                                            /*conjugate=*/true>;
+
+// For 2d ops.
+template struct TransformFilter<Eigen::GpuDevice, double, int, 4>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, double, 4>;
+template struct NHWCToNCHW<Eigen::GpuDevice, double, 4>;
+template struct NCHWToNHWC<Eigen::GpuDevice, double, 4>;
+template struct PadInput<Eigen::GpuDevice, double, int, 4>;
+
+// For 3d ops.
+template struct TransformFilter<Eigen::GpuDevice, double, int, 5>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, double, 5>;
+template struct NHWCToNCHW<Eigen::GpuDevice, double, 5>;
+template struct NCHWToNHWC<Eigen::GpuDevice, double, 5>;
+template struct PadInput<Eigen::GpuDevice, double, int, 5>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_float.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_float.cu.cc
new file mode 100644
index 0000000000..21030dd12b
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_float.cu.cc
@@ -0,0 +1,63 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct ShuffleAndReverse<Eigen::GpuDevice, float, 4, int>;
+template struct ShuffleAndReverse<Eigen::GpuDevice, float, 4,
+                                  Eigen::DenseIndex>;
+
+template struct TransformDepth<Eigen::GpuDevice, float, int>;
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, float4>;
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, float2,
+                                            /*conjugate=*/true>;
+
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, float4>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, float2,
+                                            /*conjugate=*/true>;
+
+// For 2d ops.
+template struct TransformFilter<Eigen::GpuDevice, float, int, 4>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, float, 4>;
+template struct NHWCToNCHW<Eigen::GpuDevice, float, 4>;
+template struct NCHWToNHWC<Eigen::GpuDevice, float, 4>;
+template struct PadInput<Eigen::GpuDevice, float, int, 4>;
+
+// For 3d ops.
+template struct TransformFilter<Eigen::GpuDevice, float, int, 5>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, float, 5>;
+template struct NHWCToNCHW<Eigen::GpuDevice, float, 5>;
+template struct NCHWToNHWC<Eigen::GpuDevice, float, 5>;
+template struct PadInput<Eigen::GpuDevice, float, int, 5>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_half.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_half.cu.cc
new file mode 100644
index 0000000000..948308651f
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_half.cu.cc
@@ -0,0 +1,57 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct ShuffleAndReverse<Eigen::GpuDevice, Eigen::half, 4, int>;
+template struct ShuffleAndReverse<Eigen::GpuDevice, Eigen::half, 4,
+                                  Eigen::DenseIndex>;
+
+template struct TransformDepth<Eigen::GpuDevice, Eigen::half, int>;
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, Eigen::half>;
+
+// For 2d ops.
+template struct TransformFilter<Eigen::GpuDevice, Eigen::half, int, 4>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, Eigen::half, 4>;
+template struct NHWCToNCHW<Eigen::GpuDevice, Eigen::half, 4>;
+template struct NCHWToNHWC<Eigen::GpuDevice, Eigen::half, 4>;
+template struct PadInput<Eigen::GpuDevice, Eigen::half, int, 4>;
+
+// For 3d ops.
+template struct TransformFilter<Eigen::GpuDevice, Eigen::half, int, 5>;
+template struct ReverseTransformFilter<Eigen::GpuDevice, Eigen::half, 5>;
+template struct NHWCToNCHW<Eigen::GpuDevice, Eigen::half, 5>;
+template struct NCHWToNHWC<Eigen::GpuDevice, Eigen::half, 5>;
+template struct PadInput<Eigen::GpuDevice, Eigen::half, int, 5>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_int.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_int.cu.cc
new file mode 100644
index 0000000000..901ce3e55d
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_int.cu.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+// For 2d ops.
+template struct PadInput<Eigen::GpuDevice, int, int, 4>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_uint16.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_uint16.cu.cc
new file mode 100644
index 0000000000..e47532a983
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_uint16.cu.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint16>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint16>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_uint32.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_uint32.cu.cc
new file mode 100644
index 0000000000..56cd5dd218
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_uint32.cu.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint32>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint32>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_uint64.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_uint64.cu.cc
new file mode 100644
index 0000000000..045a664e96
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_uint64.cu.cc
@@ -0,0 +1,38 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint64>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint64>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/conv_2d_gpu_uint8.cu.cc b/tensorflow/core/kernels/conv_2d_gpu_uint8.cu.cc
new file mode 100644
index 0000000000..215417860a
--- /dev/null
+++ b/tensorflow/core/kernels/conv_2d_gpu_uint8.cu.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <utility>
+
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_2d_gpu.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template struct SwapDimension1And2InTensor3<Eigen::GpuDevice, uint8>;
+template struct SwapDimension0And2InTensor3<Eigen::GpuDevice, uint8>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From bb425754adacc784c2ad50ed94307eaa03626b41 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 26 Nov 2018 10:57:38 -0800
Subject: [PATCH 0779/1554] [tf.data] Fix memory leak in
 `tf.data.experimental.unbatch()`.

Remember to `Unref()` the input dataset when the `UnbatchDatasetOp::Dataset` is destroyed.

Fixes #23924.

PiperOrigin-RevId: 222851427
---
 tensorflow/core/kernels/data/unbatch_dataset_op.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/data/unbatch_dataset_op.cc b/tensorflow/core/kernels/data/unbatch_dataset_op.cc
index b32ab8ba4f..af7f676370 100644
--- a/tensorflow/core/kernels/data/unbatch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/unbatch_dataset_op.cc
@@ -54,6 +54,8 @@ class UnbatchDatasetOp : public UnaryDatasetOpKernel {
       }
     }
 
+    ~Dataset() override { input_->Unref(); }
+
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       return std::unique_ptr<IteratorBase>(
-- 
GitLab


From 59d14c5d418849d7d6a8659c00677be27fd7f3cb Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Mon, 26 Nov 2018 11:13:09 -0800
Subject: [PATCH 0780/1554] Don't run done callback till all handles are
 scheduled, and the code breaks out of the loop.

PiperOrigin-RevId: 222854381
---
 tensorflow/core/kernels/partitioned_function_ops.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index 89b74495c7..6c90ffd75e 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -453,7 +453,7 @@ class PartitionedCallOp : public AsyncOpKernel {
         },
         rendez, std::move(done), std::placeholders::_1);
     auto* refcounted_done = new ReffedStatusCallback(std::move(callback));
-    for (int i = 1; i < handles->size(); ++i) {
+    for (int i = 0; i < handles->size(); ++i) {
       refcounted_done->Ref();
     }
 
@@ -507,6 +507,7 @@ class PartitionedCallOp : public AsyncOpKernel {
             });
       }
     }
+    refcounted_done->Unref();
   }
 
   string UniquifyFunctionName(const FunctionLibraryDefinition* function_library,
-- 
GitLab


From 32fb5465740ee99d6989c603643788b08a8a0461 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Mon, 26 Nov 2018 11:16:46 -0800
Subject: [PATCH 0781/1554] Enable generators in `model.predict` to yield
 tuples of size 1 (just the input data).

PiperOrigin-RevId: 222855043
---
 tensorflow/python/keras/BUILD                 |   1 +
 .../python/keras/engine/training_generator.py |   4 +-
 .../keras/engine/training_generator_test.py   | 388 ++++++++++--------
 3 files changed, 211 insertions(+), 182 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index fa1cad2359..bac961bb9d 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -746,6 +746,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 988bed5170..e7310a7bb9 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -388,7 +388,9 @@ def predict_generator(model,
       if isinstance(generator_output, tuple):
         # Compatibility with the generators
         # used for training.
-        if len(generator_output) == 2:
+        if len(generator_output) == 1:
+          x = generator_output[0]
+        elif len(generator_output) == 2:
           x, _ = generator_output
         elif len(generator_output) == 3:
           x, _, _ = generator_output
diff --git a/tensorflow/python/keras/engine/training_generator_test.py b/tensorflow/python/keras/engine/training_generator_test.py
index 88e8943424..42cfa3bc70 100644
--- a/tensorflow/python/keras/engine/training_generator_test.py
+++ b/tensorflow/python/keras/engine/training_generator_test.py
@@ -21,220 +21,269 @@ from __future__ import print_function
 import os
 import unittest
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import metrics as metrics_module
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
-class TestGeneratorMethods(test.TestCase):
+def custom_generator(mode=2):
+  batch_size = 10
+  num_samples = 50
+  arr_data = np.random.random((num_samples, 2))
+  arr_labels = np.random.random((num_samples, 4))
+  arr_weights = np.random.random((num_samples,))
+  i = 0
+  while True:
+    batch_index = i * batch_size % num_samples
+    i += 1
+    start = batch_index
+    end = start + batch_size
+    x = arr_data[start: end]
+    y = arr_labels[start: end]
+    w = arr_weights[start: end]
+    if mode == 1:
+      yield x
+    elif mode == 2:
+      yield x, y
+    else:
+      yield x, y, w
+
+
+@tf_test_util.run_all_in_graph_and_eager_modes
+class TestGeneratorMethods(test.TestCase, parameterized.TestCase):
 
   @unittest.skipIf(
       os.name == 'nt',
       'use_multiprocessing=True does not work on windows properly.')
-  def test_generator_methods(self):
-    arr_data = np.random.random((50, 2))
-    arr_labels = np.random.random((50,))
+  @parameterized.parameters('sequential', 'functional')
+  def test_fit_generator_method(self, model_type):
+    if model_type == 'sequential':
+      model = testing_utils.get_small_sequential_mlp(
+          num_hidden=3, num_classes=4, input_dim=2)
+    else:
+      model = testing_utils.get_small_functional_mlp(
+          num_hidden=3, num_classes=4, input_dim=2)
+    model.compile(
+        loss='mse',
+        optimizer='sgd',
+        metrics=['mae', metrics_module.CategoricalAccuracy()])
+
+    model.fit_generator(custom_generator(),
+                        steps_per_epoch=5,
+                        epochs=1,
+                        verbose=1,
+                        max_queue_size=10,
+                        workers=4,
+                        use_multiprocessing=True)
+    model.fit_generator(custom_generator(),
+                        steps_per_epoch=5,
+                        epochs=1,
+                        verbose=1,
+                        max_queue_size=10,
+                        use_multiprocessing=False)
+    model.fit_generator(custom_generator(),
+                        steps_per_epoch=5,
+                        epochs=1,
+                        verbose=1,
+                        max_queue_size=10,
+                        use_multiprocessing=False,
+                        validation_data=custom_generator(),
+                        validation_steps=10)
+    model.fit_generator(custom_generator(),
+                        steps_per_epoch=5,
+                        validation_data=custom_generator(),
+                        validation_steps=1,
+                        workers=0)
 
-    def custom_generator():
-      batch_size = 10
-      num_samples = 50
-      while True:
-        batch_index = np.random.randint(0, num_samples - batch_size)
-        start = batch_index
-        end = start + batch_size
-        x = arr_data[start: end]
-        y = arr_labels[start: end]
-        yield x, y
-
-    with self.cached_session():
-      x = keras.Input((2,))
-      y = keras.layers.Dense(1)(x)
-      fn_model = keras.models.Model(x, y)
-      fn_model.compile(
-          loss='mse',
-          optimizer='sgd',
-          metrics=['mae', metrics_module.CategoricalAccuracy()])
-
-      seq_model = keras.models.Sequential()
-      seq_model.add(keras.layers.Dense(1, input_shape=(2,)))
-      seq_model.compile(loss='mse', optimizer='sgd')
-
-      for model in [fn_model, seq_model]:
-        model.fit_generator(custom_generator(),
-                            steps_per_epoch=5,
-                            epochs=1,
-                            verbose=1,
+  @unittest.skipIf(
+      os.name == 'nt',
+      'use_multiprocessing=True does not work on windows properly.')
+  @parameterized.parameters('sequential', 'functional')
+  def test_evaluate_generator_method(self, model_type):
+    if model_type == 'sequential':
+      model = testing_utils.get_small_sequential_mlp(
+          num_hidden=3, num_classes=4, input_dim=2)
+    else:
+      model = testing_utils.get_small_functional_mlp(
+          num_hidden=3, num_classes=4, input_dim=2)
+    model.compile(
+        loss='mse',
+        optimizer='sgd',
+        metrics=['mae', metrics_module.CategoricalAccuracy()])
+    model.summary()
+
+    model.evaluate_generator(custom_generator(),
+                             steps=5,
+                             max_queue_size=10,
+                             workers=2,
+                             verbose=1,
+                             use_multiprocessing=True)
+    model.evaluate_generator(custom_generator(),
+                             steps=5,
+                             max_queue_size=10,
+                             use_multiprocessing=False)
+    model.evaluate_generator(custom_generator(),
+                             steps=5,
+                             max_queue_size=10,
+                             use_multiprocessing=False,
+                             workers=0)
+
+  @unittest.skipIf(
+      os.name == 'nt',
+      'use_multiprocessing=True does not work on windows properly.')
+  @parameterized.parameters('sequential', 'functional')
+  def test_predict_generator_method(self, model_type):
+    if model_type == 'sequential':
+      model = testing_utils.get_small_sequential_mlp(
+          num_hidden=3, num_classes=4, input_dim=2)
+    else:
+      model = testing_utils.get_small_functional_mlp(
+          num_hidden=3, num_classes=4, input_dim=2)
+    model.compile(
+        loss='mse',
+        optimizer='sgd',
+        metrics=['mae', metrics_module.CategoricalAccuracy()])
+
+    model.predict_generator(custom_generator(),
+                            steps=5,
                             max_queue_size=10,
-                            workers=4,
+                            workers=2,
                             use_multiprocessing=True)
-        model.fit_generator(custom_generator(),
-                            steps_per_epoch=5,
-                            epochs=1,
-                            verbose=1,
+    model.predict_generator(custom_generator(),
+                            steps=5,
                             max_queue_size=10,
                             use_multiprocessing=False)
-        model.fit_generator(custom_generator(),
-                            steps_per_epoch=5,
-                            epochs=1,
-                            verbose=1,
+    model.predict_generator(custom_generator(),
+                            steps=5,
+                            max_queue_size=10,
+                            workers=0)
+    # Test generator with just inputs (no targets)
+    model.predict_generator(custom_generator(mode=1),
+                            steps=5,
+                            max_queue_size=10,
+                            workers=2,
+                            use_multiprocessing=True)
+    model.predict_generator(custom_generator(mode=1),
+                            steps=5,
+                            max_queue_size=10,
+                            use_multiprocessing=False)
+    model.predict_generator(custom_generator(mode=1),
+                            steps=5,
                             max_queue_size=10,
-                            use_multiprocessing=False,
-                            validation_data=custom_generator(),
-                            validation_steps=10)
-        model.fit_generator(custom_generator(),
-                            steps_per_epoch=5,
-                            validation_data=custom_generator(),
-                            validation_steps=1,
                             workers=0)
-        model.predict_generator(custom_generator(),
-                                steps=5,
-                                max_queue_size=10,
-                                workers=2,
-                                use_multiprocessing=True)
-        model.predict_generator(custom_generator(),
-                                steps=5,
-                                max_queue_size=10,
-                                use_multiprocessing=False)
-        model.predict_generator(custom_generator(),
-                                steps=5,
-                                max_queue_size=10,
-                                workers=0)
-        model.evaluate_generator(custom_generator(),
-                                 steps=5,
-                                 max_queue_size=10,
-                                 workers=2,
-                                 verbose=1,
-                                 use_multiprocessing=True)
-        model.evaluate_generator(custom_generator(),
-                                 steps=5,
-                                 max_queue_size=10,
-                                 use_multiprocessing=False)
-        model.evaluate_generator(custom_generator(),
-                                 steps=5,
-                                 max_queue_size=10,
-                                 use_multiprocessing=False,
-                                 workers=0)
 
   def test_generator_methods_with_sample_weights(self):
-    arr_data = np.random.random((50, 2))
-    arr_labels = np.random.random((50,))
-    arr_sample_weights = np.random.random((50,))
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(4, input_shape=(2,)))
+    model.compile(
+        loss='mse',
+        optimizer='sgd',
+        metrics=['mae', metrics_module.CategoricalAccuracy()])
+
+    model.fit_generator(custom_generator(mode=3),
+                        steps_per_epoch=5,
+                        epochs=1,
+                        verbose=1,
+                        max_queue_size=10,
+                        use_multiprocessing=False)
+    model.fit_generator(custom_generator(mode=3),
+                        steps_per_epoch=5,
+                        epochs=1,
+                        verbose=1,
+                        max_queue_size=10,
+                        use_multiprocessing=False,
+                        validation_data=custom_generator(mode=3),
+                        validation_steps=10)
+    model.predict_generator(custom_generator(mode=3),
+                            steps=5,
+                            max_queue_size=10,
+                            use_multiprocessing=False)
+    model.evaluate_generator(custom_generator(mode=3),
+                             steps=5,
+                             max_queue_size=10,
+                             use_multiprocessing=False)
 
-    def custom_generator():
-      batch_size = 10
-      num_samples = 50
-      while True:
-        batch_index = np.random.randint(0, num_samples - batch_size)
-        start = batch_index
-        end = start + batch_size
-        x = arr_data[start: end]
-        y = arr_labels[start: end]
-        w = arr_sample_weights[start: end]
-        yield x, y, w
-
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(1, input_shape=(2,)))
-      model.compile(
-          loss='mse',
-          optimizer='sgd',
-          metrics=['mae', metrics_module.CategoricalAccuracy()])
+  def test_generator_methods_invalid_use_case(self):
 
-      model.fit_generator(custom_generator(),
+    def invalid_generator():
+      while 1:
+        yield 0
+
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(4, input_shape=(2,)))
+    model.compile(loss='mse', optimizer='sgd')
+
+    with self.assertRaises(ValueError):
+      model.fit_generator(invalid_generator(),
                           steps_per_epoch=5,
                           epochs=1,
                           verbose=1,
                           max_queue_size=10,
                           use_multiprocessing=False)
+    with self.assertRaises(ValueError):
       model.fit_generator(custom_generator(),
                           steps_per_epoch=5,
                           epochs=1,
                           verbose=1,
                           max_queue_size=10,
                           use_multiprocessing=False,
-                          validation_data=custom_generator(),
+                          validation_data=invalid_generator(),
                           validation_steps=10)
-      model.predict_generator(custom_generator(),
+    with self.assertRaises(AttributeError):
+      model.predict_generator(invalid_generator(),
                               steps=5,
                               max_queue_size=10,
                               use_multiprocessing=False)
-      model.evaluate_generator(custom_generator(),
+    with self.assertRaises(ValueError):
+      model.evaluate_generator(invalid_generator(),
                                steps=5,
                                max_queue_size=10,
                                use_multiprocessing=False)
 
-  def test_generator_methods_invalid_use_case(self):
+  def test_generator_input_to_fit_eval_predict(self):
+    val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
 
-    def custom_generator():
-      while 1:
-        yield 0
+    def ones_generator():
+      while True:
+        yield np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
 
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(1, input_shape=(2,)))
-      model.compile(loss='mse', optimizer='sgd')
+    inputs = keras.layers.Input(shape=(10,))
+    x = keras.layers.Dense(10, activation='relu')(inputs)
+    outputs = keras.layers.Dense(1, activation='sigmoid')(x)
+    model = keras.Model(inputs, outputs)
+
+    model.compile(RMSPropOptimizer(0.001), 'binary_crossentropy')
+    model.fit(
+        ones_generator(),
+        steps_per_epoch=2,
+        validation_data=val_data,
+        epochs=2)
+    model.evaluate(ones_generator(), steps=2)
+    model.predict(ones_generator(), steps=2)
 
-      with self.assertRaises(ValueError):
-        model.fit_generator(custom_generator(),
-                            steps_per_epoch=5,
-                            epochs=1,
-                            verbose=1,
-                            max_queue_size=10,
-                            use_multiprocessing=False)
-      with self.assertRaises(ValueError):
-        model.fit_generator(custom_generator(),
-                            steps_per_epoch=5,
-                            epochs=1,
-                            verbose=1,
-                            max_queue_size=10,
-                            use_multiprocessing=False,
-                            validation_data=custom_generator(),
-                            validation_steps=10)
-      with self.assertRaises(AttributeError):
-        model.predict_generator(custom_generator(),
-                                steps=5,
-                                max_queue_size=10,
-                                use_multiprocessing=False)
-      with self.assertRaises(ValueError):
-        model.evaluate_generator(custom_generator(),
-                                 steps=5,
-                                 max_queue_size=10,
-                                 use_multiprocessing=False)
+
+@tf_test_util.run_all_in_graph_and_eager_modes
+class TestGeneratorMethodsWithSequences(test.TestCase):
 
   def test_training_with_sequences(self):
 
     class DummySequence(keras.utils.Sequence):
 
       def __getitem__(self, idx):
-        return np.zeros([10, 2]), np.ones([10])
+        return np.zeros([10, 2]), np.ones([10, 4])
 
       def __len__(self):
         return 10
 
-    arr_data = np.random.random((50, 2))
-    arr_labels = np.random.random((50,))
-    arr_sample_weights = np.random.random((50,))
-
-    def custom_generator():
-      batch_size = 10
-      num_samples = 50
-      while True:
-        batch_index = np.random.randint(0, num_samples - batch_size)
-        start = batch_index
-        end = start + batch_size
-        x = arr_data[start: end]
-        y = arr_labels[start: end]
-        w = arr_sample_weights[start: end]
-        yield x, y, w
-
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(1, input_shape=(2,)))
-      model.compile(loss='mse', optimizer='sgd')
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(4, input_shape=(2,)))
+    model.compile(loss='mse', optimizer='sgd')
 
     model.fit_generator(DummySequence(),
                         steps_per_epoch=10,
@@ -251,29 +300,6 @@ class TestGeneratorMethods(test.TestCase):
                         workers=0,
                         use_multiprocessing=False)
 
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_generator_input_to_fit_eval_predict(self):
-    val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
-
-    def custom_generator():
-      while True:
-        yield np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
-
-    inputs = keras.layers.Input(shape=(10,))
-    x = keras.layers.Dense(10, activation='relu')(inputs)
-    outputs = keras.layers.Dense(1, activation='sigmoid')(x)
-    model = keras.Model(inputs, outputs)
-
-    model.compile(RMSPropOptimizer(0.001), 'binary_crossentropy')
-    model.fit(
-        custom_generator(),
-        steps_per_epoch=2,
-        validation_data=val_data,
-        epochs=2)
-    model.evaluate(custom_generator(), steps=2)
-    model.predict(custom_generator(), steps=2)
-
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_sequence_input_to_fit_eval_predict(self):
     val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
 
-- 
GitLab


From 17630415893203929f2b65a5df2b019bd4d3d040 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 26 Nov 2018 11:17:29 -0800
Subject: [PATCH 0782/1554] Make testConcurrentExecutesWithoutError resilient
 to non-deterministic autotuning

c1 and c2 may not be exactly equal if we pick different GEMM algorithms for
computing matrix1 and matrix2.  For some reason I don't understand this happens
fairly frequently with XLA enabled (via auto-clustering) but not without XLA.

PiperOrigin-RevId: 222855174
---
 tensorflow/python/kernel_tests/cholesky_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index e96b277266..fa41a03b54 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -184,7 +184,7 @@ class CholeskyOpTest(test.TestCase):
       c1 = linalg_ops.cholesky(matrix1)
       c2 = linalg_ops.cholesky(matrix2)
       c1_val, c2_val = sess.run([c1, c2])
-      self.assertAllEqual(c1_val, c2_val)
+      self.assertAllClose(c1_val, c2_val)
 
 
 class CholeskyGradTest(test.TestCase):
-- 
GitLab


From 243f9a78916be6f1f2edf27db5dd981ad33e1d44 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 26 Nov 2018 11:18:36 -0800
Subject: [PATCH 0783/1554] BEGIN_PUBLIC Rollback change of
 https://github.com/tensorflow/tensorflow/pull/23441 since its no longer
 needed. END_PUBLIC

Automated rollback of commit 8e048e840e4a0127b8f5bffab4ea5e649db4a0ab. Revert #23441.

PiperOrigin-RevId: 222855383
---
 tensorflow/python/keras/layers/wrappers.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index f6ae77a21f..67b154141e 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -435,15 +435,10 @@ class Bidirectional(Wrapper):
 
   @tf_utils.shape_type_conversion
   def compute_output_shape(self, input_shape):
-    forward_layer_output_shape = self.forward_layer.compute_output_shape(
-        input_shape)
-    if getattr(forward_layer_output_shape, 'as_list', None) is None:
-      output_shape = tuple(forward_layer_output_shape)
-    else:
-      output_shape = tuple(forward_layer_output_shape.as_list())
-
+    output_shape = tuple(self.forward_layer.compute_output_shape(
+        input_shape).as_list())
     if self.return_state:
-      state_shape = list(output_shape[1:])
+      state_shape = output_shape[1:]
       output_shape = output_shape[0]
 
     if self.merge_mode == 'concat':
-- 
GitLab


From e468989d1735d16850a34335aaff13b117660fed Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 26 Nov 2018 11:26:03 -0800
Subject: [PATCH 0784/1554] [tf.data] Fix serialization bug in
 PrivateThreadPoolDatasetOp.

PiperOrigin-RevId: 222856707
---
 .../core/kernels/data/experimental/threadpool_dataset_op.cc    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 341ce08a57..335f2b7a4b 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -391,7 +391,8 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
       TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
       Node* num_threads_node = nullptr;
       TF_RETURN_IF_ERROR(b->AddScalar(num_threads_, &num_threads_node));
-      TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output));
+      TF_RETURN_IF_ERROR(
+          b->AddDataset(this, {input_graph_node, num_threads_node}, output));
       return Status::OK();
     }
 
-- 
GitLab


From e802b5a236407ae916268979e35e1634c14355e2 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Mon, 26 Nov 2018 11:29:12 -0800
Subject: [PATCH 0785/1554] Symbol changes for Tf 2.0 API

Changes:
- Deprecate: create_partioned_variables
- Change args: gather, hessians
PiperOrigin-RevId: 222857296
---
 tensorflow/python/ops/array_ops.py              | 11 +++++++++--
 tensorflow/python/ops/gradients_impl.py         | 15 ++++++++++++++-
 tensorflow/python/ops/partitioned_variables.py  |  2 +-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt |  8 ++------
 tensorflow/tools/compatibility/renames_v2.py    |  1 +
 5 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 8e9b8950bc..496d3855b5 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3216,7 +3216,7 @@ reverse_sequence_v2.__doc__ = deprecation.rewrite_argument_docstring(
 # pylint: enable=redefined-builtin
 
 
-@tf_export("gather")
+@tf_export(v1=["gather"])
 def gather(params, indices, validate_indices=None, name=None, axis=0):
   del validate_indices
   if axis != 0:
@@ -3232,7 +3232,14 @@ def gather(params, indices, validate_indices=None, name=None, axis=0):
     return gen_array_ops.gather_v2(params, indices, axis, name=name)
 
 
-gather.__doc__ = gen_array_ops.gather_v2.__doc__
+@tf_export("gather", v1=[])
+def gather_v2(params, indices, validate_indices=None, axis=0, name=None):
+  return gather(params, indices, validate_indices=validate_indices, name=name,
+                axis=axis)
+
+
+gather.__doc__ = gather_v2.__doc__ = gen_array_ops.gather_v2.__doc__
+
 
 
 @tf_export("batch_gather")
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index 278008526c..27cee9113d 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -1239,7 +1239,7 @@ def _hessian_vector_product(ys, xs, v):
   return gradients(elemwise_products, xs)
 
 
-@tf_export("hessians")
+@tf_export(v1=["hessians"])
 def hessians(ys,
              xs,
              name="hessians",
@@ -1304,3 +1304,16 @@ def hessians(ys,
                                           array_ops.concat((_shape, _shape), 0))
     hessians.append(_reshaped_hessian)
   return hessians
+
+
+@tf_export("hessians", v1=[])
+def HessiansV2(ys,
+               xs,
+               gate_gradients=False,
+               aggregation_method=None,
+               name="hessians"):
+  return hessians(ys, xs, name=name, gate_gradients=gate_gradients,
+                  aggregation_method=aggregation_method)
+
+
+HessiansV2.__doc__ = hessians.__doc__
diff --git a/tensorflow/python/ops/partitioned_variables.py b/tensorflow/python/ops/partitioned_variables.py
index 66153e33f2..c1084c2559 100644
--- a/tensorflow/python/ops/partitioned_variables.py
+++ b/tensorflow/python/ops/partitioned_variables.py
@@ -237,7 +237,7 @@ def fixed_size_partitioner(num_shards, axis=0):
   return _partitioner
 
 
-@tf_export("create_partitioned_variables")
+@tf_export(v1=["create_partitioned_variables"])
 @deprecation.deprecated(
     date=None,
     instructions="Use tf.get_variable with a partitioner set.")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 6afe44a03f..92ebdb91c5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -580,10 +580,6 @@ tf_module {
     name: "cosh"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "create_partitioned_variables"
-    argspec: "args=[\'shape\', \'slicing\', \'initializer\', \'dtype\', \'trainable\', \'collections\', \'name\', \'reuse\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'True\', \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "cumsum"
     argspec: "args=[\'x\', \'axis\', \'exclusive\', \'reverse\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'False\', \'False\', \'None\'], "
@@ -682,7 +678,7 @@ tf_module {
   }
   member_method {
     name: "gather"
-    argspec: "args=[\'params\', \'indices\', \'validate_indices\', \'name\', \'axis\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0\'], "
+    argspec: "args=[\'params\', \'indices\', \'validate_indices\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], "
   }
   member_method {
     name: "gather_nd"
@@ -714,7 +710,7 @@ tf_module {
   }
   member_method {
     name: "hessians"
-    argspec: "args=[\'ys\', \'xs\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\'], varargs=None, keywords=None, defaults=[\'hessians\', \'False\', \'False\', \'None\'], "
+    argspec: "args=[\'ys\', \'xs\', \'gate_gradients\', \'aggregation_method\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'hessians\'], "
   }
   member_method {
     name: "histogram_fixed_width"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index c22b09df48..b02326278e 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -107,6 +107,7 @@ renames = {
     'tf.convert_to_tensor_or_sparse_tensor': 'tf.compat.v1.convert_to_tensor_or_sparse_tensor',
     'tf.count_nonzero': 'tf.compat.v1.count_nonzero',
     'tf.count_up_to': 'tf.compat.v1.count_up_to',
+    'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
     'tf.debugging.is_finite': 'tf.math.is_finite',
-- 
GitLab


From acafc054e1e080cf2a46d47ee6954816a01e4acc Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Mon, 26 Nov 2018 11:34:27 -0800
Subject: [PATCH 0786/1554] Fix diagram link.

PiperOrigin-RevId: 222858360
---
 tensorflow/lite/toco/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/toco/README.md b/tensorflow/lite/toco/README.md
index bd8f8282f0..fe98a90b38 100644
--- a/tensorflow/lite/toco/README.md
+++ b/tensorflow/lite/toco/README.md
@@ -26,4 +26,4 @@ to client devices, generally mobile devices, where the TensorFlow Lite
 interpreter handles them on-device.  This flow is represented in the diagram
 below.
 
-![drawing](g3doc/toco_landscape.svg)
+![drawing](../g3doc/images/convert/workflow.svg)
-- 
GitLab


From 3951cc626a924b3ebddfce7cfbb08c5d513c08d6 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 26 Nov 2018 11:37:16 -0800
Subject: [PATCH 0787/1554] [tf.data] Fixes leak of
 `OptimizeDatasetOp::Dataset` if optimization fails.

Previously, we were not calling `dataset->Unref()` when optimization failed, and the `Dataset` object would leak. If we *had* called `Unref()` we would have attempted to call `optimized_input_->Unref()`, but `optimized_input_` was uninitialized in the case that optimization fails.

PiperOrigin-RevId: 222858890
---
 .../core/kernels/data/optimize_dataset_op.cc      | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index f90dcb95e3..f5bb35d360 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item_builder.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+#include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/protobuf/meta_graph.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -56,8 +57,13 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
         ctx, ParseVectorArgument<string>(ctx, "optimizations", &optimizations));
     Dataset* dataset =
         new Dataset(ctx, input, optimizations, output_types_, output_shapes_);
-    OP_REQUIRES_OK(ctx, dataset->Optimize(ctx));
-    *output = dataset;
+    Status s = dataset->Optimize(ctx);
+    if (s.ok()) {
+      *output = dataset;
+    } else {
+      dataset->Unref();
+      OP_REQUIRES_OK(ctx, s);
+    }
   }
 
  private:
@@ -68,6 +74,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
             const DataTypeVector& output_types,
             const std::vector<PartialTensorShape>& output_shapes)
         : DatasetBase(DatasetContext(ctx)),
+          optimized_input_(nullptr),
           input_(input),
           optimizations_(optimizations),
           output_types_(output_types),
@@ -77,7 +84,9 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
 
     ~Dataset() override {
       input_->Unref();
-      optimized_input_->Unref();
+      if (optimized_input_) {
+        optimized_input_->Unref();
+      }
     }
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
-- 
GitLab


From c4553ea15d6489f3d2dba3e3b17e8cad5e3a9331 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 26 Nov 2018 11:52:44 -0800
Subject: [PATCH 0788/1554] [tf.data] Adding eager coverage to remaining core
 dataset tests.

PiperOrigin-RevId: 222861664
---
 tensorflow/python/data/kernel_tests/BUILD     |  34 +-
 .../kernel_tests/flat_map_dataset_op_test.py  |  99 +--
 .../interleave_dataset_op_test.py             | 192 +++--
 .../data/kernel_tests/iterator_ops_test.py    |  39 +-
 .../multi_device_iterator_test.py             |   1 +
 .../kernel_tests/range_dataset_op_test.py     | 333 --------
 .../kernel_tests/reader_dataset_ops_test.py   | 770 +++++-------------
 .../kernel_tests/reduce_dataset_op_test.py    |  40 +-
 .../save_restore_experimental_test.py         | 686 ++++++++++++++++
 .../kernel_tests/window_dataset_op_test.py    | 268 +++---
 10 files changed, 1196 insertions(+), 1266 deletions(-)
 create mode 100644 tensorflow/python/data/kernel_tests/save_restore_experimental_test.py

diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index dc89474f49..0cdb44a3b4 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -158,7 +158,7 @@ tf_py_test(
 
 tf_py_test(
     name = "flat_map_dataset_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["flat_map_dataset_op_test.py"],
     additional_deps = [
         ":test_base",
@@ -208,7 +208,7 @@ tf_py_test(
 
 tf_py_test(
     name = "interleave_dataset_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["interleave_dataset_op_test.py"],
     additional_deps = [
         ":test_base",
@@ -233,6 +233,7 @@ cuda_py_test(
     size = "small",
     srcs = ["iterator_ops_test.py"],
     additional_deps = [
+        ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python/data/ops:readers",
@@ -300,7 +301,7 @@ tf_py_test(
 
 tf_py_test(
     name = "map_dataset_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["map_dataset_op_test.py"],
     additional_deps = [
         ":test_base",
@@ -415,7 +416,7 @@ tf_py_test(
 
 tf_py_test(
     name = "reader_dataset_ops_test",
-    size = "small",
+    size = "medium",
     srcs = ["reader_dataset_ops_test.py"],
     additional_deps = [
         ":test_base",
@@ -456,6 +457,29 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "save_restore_experimental_test",
+    size = "small",
+    srcs = ["save_restore_experimental_test.py"],
+    additional_deps = [
+        ":reader_dataset_ops_test",
+        ":test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:variables",
+    ],
+)
+
 tf_py_test(
     name = "sequence_dataset_op_test",
     size = "small",
@@ -520,7 +544,7 @@ py_library(
 
 tf_py_test(
     name = "window_dataset_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["window_dataset_op_test.py"],
     additional_deps = [
         ":test_base",
diff --git a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
index 68038f9cfc..9292f20637 100644
--- a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
@@ -26,54 +26,41 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class FlatMapDatasetTest(test_base.DatasetTestBase):
 
   # pylint: disable=g-long-lambda
   def testFlatMapDataset(self):
     repeats = [1, 2, 3, 4, 5, 0, 1]
     components = np.array(repeats, dtype=np.int64)
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components)
-        .flat_map(lambda x: dataset_ops.Dataset.from_tensors([x]).repeat(x))
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in repeats:
-        for _ in range(i):
-          self.assertEqual(i, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.from_tensor_slices(components).flat_map(
+        lambda x: dataset_ops.Dataset.from_tensors([x]).repeat(x))
+    expected_output = []
+    for i in repeats:
+      expected_output.extend([[i]] * i)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testNestedFlatMapDataset(self):
     repeats = [[1, 2], [3, 4], [5, 0], [1, 7]]
     components = np.array(repeats, dtype=np.int64)
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components)
-        .flat_map(lambda x: dataset_ops.Dataset.from_tensor_slices(x)
-                  .flat_map(lambda y: dataset_ops.Dataset.from_tensors(y)
-                            .repeat(y))).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for row in repeats:
-        for i in row:
-          for _ in range(i):
-            self.assertEqual(i, sess.run(get_next))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testSharedResourceNestedFlatMapDataset(self):
+    dataset = dataset_ops.Dataset.from_tensor_slices(components).flat_map(
+        lambda x: dataset_ops.Dataset.from_tensor_slices(x).flat_map(
+            lambda y: dataset_ops.Dataset.from_tensors(y).repeat(y))
+    )
+    expected_output = []
+    for row in repeats:
+      for i in row:
+        expected_output.extend([i] * i)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  # Note: no eager mode coverage, session specific test.
+  def testSkipEagerSharedResourceNestedFlatMapDataset(self):
     repeats = [[1, 2], [3, 4], [5, 0], [1, 7]]
     components = np.array(repeats, dtype=np.int64)
     iterator = (
@@ -106,22 +93,16 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
           sess.run(get_next)
 
   def testMapDict(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(lambda x: {"foo": x * 2, "bar": x ** 2})
-                .flat_map(lambda d: dataset_ops.Dataset.from_tensors(d["foo"])
-                          .repeat(d["bar"]))
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(10):
-        for _ in range(i ** 2):
-          self.assertEqual(i * 2, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-  # pylint: enable=g-long-lambda
+    dataset = dataset_ops.Dataset.range(10).map(
+        lambda x: {"foo": x * 2, "bar": x ** 2}).flat_map(
+            lambda d: dataset_ops.Dataset.from_tensors(
+                d["foo"]).repeat(d["bar"]))
+    get_next = self.getNext(dataset)
+    for i in range(10):
+      for _ in range(i**2):
+        self.assertEqual(i * 2, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testSparse(self):
     def _map_fn(i):
@@ -132,20 +113,12 @@ class FlatMapDatasetTest(test_base.DatasetTestBase):
       return dataset_ops.Dataset.from_tensor_slices(
           sparse_ops.sparse_to_dense(x.indices, x.dense_shape, x.values))
 
-    iterator = (
-        dataset_ops.Dataset.range(10).map(_map_fn).flat_map(_flat_map_fn)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(10):
-        for j in range(2):
-          expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).map(_map_fn).flat_map(_flat_map_fn)
+    expected_output = []
+    for i in range(10):
+      for j in range(2):
+        expected_output.append([i, 0] if j % 2 == 0 else [0, -i])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py b/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
index b911c249ce..f0b16591f7 100644
--- a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import sparse_ops
@@ -115,9 +116,7 @@ def _make_coordinated_sloppy_dataset(input_values, cycle_length, block_length,
   dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
       2).interleave(interleave_fn, cycle_length, block_length,
                     num_parallel_calls).with_options(options)
-  iterator = dataset.make_one_shot_iterator()
-  get_next = iterator.get_next()
-  return get_next, coordination_events
+  return dataset, coordination_events
 
 
 def _repeat(values, count):
@@ -133,6 +132,7 @@ def _repeat(values, count):
   return [[value] * value for value in np.tile(values, count)]
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
@@ -191,16 +191,9 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         count).interleave(
             lambda x: dataset_ops.Dataset.from_tensors(x).repeat(x),
             cycle_length, block_length, num_parallel_calls)
-    get_next = dataset.make_one_shot_iterator().get_next()
-
-    with self.cached_session() as sess:
-      for expected_element in _interleave(
-          _repeat(input_values, count), cycle_length, block_length):
-        self.assertEqual(expected_element, sess.run(get_next))
-
-      for _ in range(2):
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
+    expected_output = [element for element in _interleave(
+        _repeat(input_values, count), cycle_length, block_length)]
+    self.assertDatasetProduces(dataset, expected_output)
 
   @parameterized.named_parameters(
       ("1", np.float32([1., np.nan, 2., np.nan, 3.]), 1, 3, None),
@@ -223,17 +216,16 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         lambda x: array_ops.check_numerics(x, "message")).interleave(
             dataset_ops.Dataset.from_tensors, cycle_length, block_length,
             num_parallel_calls)
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = self.getNext(dataset)
 
-    with self.cached_session() as sess:
-      for value in input_values:
-        if np.isnan(value):
-          with self.assertRaises(errors.InvalidArgumentError):
-            sess.run(get_next)
-        else:
-          self.assertEqual(value, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    for value in input_values:
+      if np.isnan(value):
+        with self.assertRaises(errors.InvalidArgumentError):
+          self.evaluate(get_next())
+      else:
+        self.assertEqual(value, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testInterleaveSparse(self):
 
@@ -245,72 +237,112 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return dataset_ops.Dataset.from_tensor_slices(
           sparse_ops.sparse_to_dense(x.indices, x.dense_shape, x.values))
 
-    iterator = (
-        dataset_ops.Dataset.range(10).map(_map_fn).interleave(
-            _interleave_fn, cycle_length=1).make_one_shot_iterator())
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.range(10).map(_map_fn).interleave(
+        _interleave_fn, cycle_length=1)
+    get_next = self.getNext(dataset)
+    for i in range(10):
+      for j in range(2):
+        expected = [i, 0] if j % 2 == 0 else [0, -i]
+        self.assertAllEqual(expected, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
 
-    with self.cached_session() as sess:
-      for i in range(10):
-        for j in range(2):
-          expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+class InterleaveDatasetTestWithConfig(test_base.DatasetTestBase,
+                                      parameterized.TestCase):
 
   @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1, 1),
-      ("2", np.int64([4, 5, 6]), 2, 1, 2),
-      ("3", np.int64([4, 5, 6]), 2, 3, 1),
-      ("4", np.int64([4, 5, 6]), 2, 3, 2),
-      ("5", np.int64([4, 5, 6]), 3, 2, 1),
-      ("6", np.int64([4, 5, 6]), 3, 2, 2),
-      ("7", np.int64([4, 5, 6]), 3, 2, 3),
-      ("8", np.int64([4, 0, 6]), 2, 3, 1),
-      ("9", np.int64([4, 0, 6]), 2, 3, 2),
+      ("1", np.int64([4, 5, 6]), 2, 1),
+      ("2", np.int64([4, 5, 6]), 2, 3),
+      ("3", np.int64([4, 5, 6]), 3, 2),
+      ("4", np.int64([4, 0, 6]), 2, 3),
   )
+  @test_util.run_in_graph_and_eager_modes(
+      config=config_pb2.ConfigProto(
+          inter_op_parallelism_threads=2, use_per_session_threads=True))
   def testSloppyInterleaveInOrder(self, input_values, cycle_length,
-                                  block_length, num_parallel_calls):
-    get_next, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls)
-    config = config_pb2.ConfigProto(
-        inter_op_parallelism_threads=num_parallel_calls + 1,
-        use_per_session_threads=True)
-    with self.cached_session(config=config) as sess:
-      for expected_element in _interleave(
-          _repeat(input_values, 2), cycle_length, block_length):
-        coordination_events[expected_element].set()
-        self.assertEqual(expected_element * expected_element,
-                         sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+                                  block_length):
+    dataset, coordination_events = _make_coordinated_sloppy_dataset(
+        input_values, cycle_length, block_length, num_parallel_calls=1)
+    get_next = self.getNext(dataset)
+    for expected_element in _interleave(
+        _repeat(input_values, 2), cycle_length, block_length):
+      coordination_events[expected_element].set()
+      self.assertEqual(expected_element * expected_element,
+                       self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+  @parameterized.named_parameters(
+      ("1", np.int64([4, 5, 6]), 2, 1),
+      ("2", np.int64([4, 5, 6]), 2, 3),
+      ("3", np.int64([4, 5, 6]), 3, 2),
+      ("4", np.int64([4, 0, 6]), 2, 3),
+  )
+  @test_util.run_in_graph_and_eager_modes(
+      config=config_pb2.ConfigProto(
+          inter_op_parallelism_threads=3, use_per_session_threads=True))
+  def testSloppyInterleaveInOrder_2(self, input_values, cycle_length,
+                                    block_length):
+    dataset, coordination_events = _make_coordinated_sloppy_dataset(
+        input_values, cycle_length, block_length, num_parallel_calls=2)
+    get_next = self.getNext(dataset)
+    for expected_element in _interleave(
+        _repeat(input_values, 2), cycle_length, block_length):
+      coordination_events[expected_element].set()
+      self.assertEqual(expected_element * expected_element,
+                       self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1, 2),
-      ("2", np.int64([4, 5, 6]), 2, 3, 2),
-      ("3", np.int64([4, 5, 6]), 3, 2, 3),
-      ("4", np.int64([4, 0, 6]), 2, 3, 2),
+      ("1", np.int64([4, 5, 6]), 2, 1),
+      ("2", np.int64([4, 5, 6]), 2, 3),
+      ("3", np.int64([4, 0, 6]), 2, 3),
   )
+  @test_util.run_in_graph_and_eager_modes(
+      config=config_pb2.ConfigProto(
+          inter_op_parallelism_threads=3, use_per_session_threads=True))
   def testSloppyInterleaveOutOfOrder(self, input_values, cycle_length,
-                                     block_length, num_parallel_calls):
-    get_next, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls)
-    config = config_pb2.ConfigProto(
-        inter_op_parallelism_threads=num_parallel_calls + 1,
-        use_per_session_threads=True)
-    with self.cached_session(config=config) as sess:
-      elements = [
-          x for x in _interleave(
-              _repeat(input_values, 2), cycle_length, block_length)
-      ]
-      for i in [1, 4, 7]:
-        elements[i], elements[i + 1] = elements[i + 1], elements[i]
-
-      for element in elements:
-        coordination_events[element].set()
-        self.assertEqual(element * element, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+                                     block_length):
+    dataset, coordination_events = _make_coordinated_sloppy_dataset(
+        input_values, cycle_length, block_length, num_parallel_calls=2)
+    get_next = self.getNext(dataset)
+    elements = [
+        x for x in _interleave(
+            _repeat(input_values, 2), cycle_length, block_length)
+    ]
+    for i in [1, 4, 7]:
+      elements[i], elements[i + 1] = elements[i + 1], elements[i]
+
+    for element in elements:
+      coordination_events[element].set()
+      self.assertEqual(element * element, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+  @test_util.run_in_graph_and_eager_modes(
+      config=config_pb2.ConfigProto(
+          inter_op_parallelism_threads=4, use_per_session_threads=True))
+  def testSloppyInterleaveOutOfOrder_2(self):
+    input_values, cycle_length, block_length = np.int64([4, 5, 6]), 3, 2
+    dataset, coordination_events = _make_coordinated_sloppy_dataset(
+        input_values, cycle_length, block_length, num_parallel_calls=3)
+    get_next = self.getNext(dataset)
+    elements = [
+        x for x in _interleave(
+            _repeat(input_values, 2), cycle_length, block_length)
+    ]
+    for i in [1, 4, 7]:
+      elements[i], elements[i + 1] = elements[i + 1], elements[i]
+
+    for element in elements:
+      coordination_events[element].set()
+      self.assertEqual(element * element, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
index 490ca813dc..3d4db80883 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
@@ -28,6 +28,7 @@ from tensorflow.core.protobuf import cluster_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.compat import compat as forward_compat
+from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
@@ -863,15 +864,16 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       self.assertEqual("overridden_name", next_element.op.name)
 
 
-class IteratorCheckpointingTest(test.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class IteratorCheckpointingTest(test_base.DatasetTestBase):
 
-  @test_util.run_in_graph_and_eager_modes
   def testSaveRestoreOneShotIterator(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
     dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map(
         math_ops.square).batch(2)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
     get_next = iterator.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator.get_next())
     checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
@@ -885,21 +887,23 @@ class IteratorCheckpointingTest(test.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       get_next()
 
-  @test_util.run_in_graph_and_eager_modes
   def testSaveRestoreMultipleIterator(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
     dataset = dataset_ops.Dataset.from_tensor_slices(
         [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
     dataset = dataset.map(math_ops.square).batch(2)
-    iterator_1 = dataset.make_one_shot_iterator()
+    iterator_1 = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
     get_next_1 = iterator_1.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator_1.get_next())
-    iterator_2 = dataset.make_one_shot_iterator()
+    iterator_2 = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
     get_next_2 = iterator_2.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator_2.get_next())
     dataset_2 = dataset_ops.Dataset.range(10)
-    iterator_3 = dataset_2.make_one_shot_iterator()
+    iterator_3 = iter(dataset_2) if context.executing_eagerly(
+    ) else dataset_2.make_one_shot_iterator()
     get_next_3 = iterator_3.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator_3.get_next())
     checkpoint = checkpointable_utils.Checkpoint(
@@ -917,12 +921,12 @@ class IteratorCheckpointingTest(test.TestCase):
     self.assertAllEqual([1, 4], get_next_2())
     self.assertAllEqual(3, get_next_3())
 
-  @test_util.run_in_graph_and_eager_modes
   def testRestoreExhaustedIterator(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
     dataset = dataset_ops.Dataset.range(3)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
     get_next = iterator.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator.get_next())
     checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
@@ -941,16 +945,17 @@ class IteratorCheckpointingTest(test.TestCase):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
     dataset = dataset_ops.Dataset.range(10)
-    iterator = dataset.make_initializable_iterator()
-    get_next = iterator.get_next()
+    iterator = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_initializable_iterator()
+    get_next = iterator.get_next
     checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
     for i in range(5):
-      with self.cached_session() as sess:
-        checkpoint.restore(checkpoint_management.latest_checkpoint(
-            checkpoint_directory)).initialize_or_restore(sess)
-        for j in range(2):
-          self.assertEqual(i * 2 + j, sess.run(get_next))
-        checkpoint.save(file_prefix=checkpoint_prefix)
+      checkpoint.restore(
+          checkpoint_management.latest_checkpoint(
+              checkpoint_directory)).initialize_or_restore()
+      for j in range(2):
+        self.assertEqual(i * 2 + j, self.evaluate(get_next()))
+      checkpoint.save(file_prefix=checkpoint_prefix)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 42ee1e2186..afb0939397 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
+# TODO(b/117581999): Add eager coverage for the following tests.
 class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
   def testNoGetNext(self):
diff --git a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
index 9fc79707d0..907cb59096 100644
--- a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
@@ -17,21 +17,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import gen_dataset_ops
-from tensorflow.python.ops import io_ops
-from tensorflow.python.ops import parsing_ops
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 
 
@@ -79,328 +69,5 @@ class RangeDatasetTest(test_base.DatasetTestBase):
     self.assertDatasetProduces(dataset, expected_output=range(10, 2, -1))
 
 
-class ExperimentalCheckpointDatasetTest(test_base.DatasetTestBase):
-
-  def tearDown(self):
-    # Remove all checkpoint files.
-    prefix = self._iterator_checkpoint_prefix()
-    pattern = prefix + "*"
-    files = gfile.Glob(pattern)
-    map(gfile.Remove, files)
-
-  def _iterator_checkpoint_prefix(self):
-    return os.path.join(self.get_temp_dir(), "iterator")
-
-  def _save_op(self, iterator_resource):
-    iterator_state_variant = gen_dataset_ops.serialize_iterator(
-        iterator_resource)
-    save_op = io_ops.write_file(
-        self._iterator_checkpoint_prefix(),
-        parsing_ops.serialize_tensor(iterator_state_variant))
-    return save_op
-
-  def _restore_op(self, iterator_resource):
-    iterator_state_variant = parsing_ops.parse_tensor(
-        io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant)
-    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
-                                                      iterator_state_variant)
-    return restore_op
-
-  def testSaveRestore(self):
-
-    def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    break_point = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-    # Saving and restoring in same session.
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testRestoreWithoutBuildingDatasetGraph(self):
-
-    def _build_graph(start, stop, num_epochs):
-      dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
-      iterator = dataset.make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    num_epochs = 5
-    break_point = 5
-    break_epoch = 3
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for _ in range(break_epoch):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      # Create an empty IteratorResource and restore the Iterator into it.
-      output_types = dtypes.int64
-      output_shapes = tensor_shape.scalar()
-      iterator = iterator_ops.Iterator.from_structure(output_types,
-                                                      output_shapes)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      get_next = iterator.get_next()
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        for _ in range(break_epoch + 1, num_epochs):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testRestoreInModifiedGraph(self):
-
-    def _build_graph(start, stop):
-      dataset = dataset_ops.Dataset.range(start, stop)
-      iterator = dataset.make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    stop_1 = 8
-    break_point = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      # Intentionally build a graph with a different value for stop to make sure
-      # the original dataset graph is actually getting loaded.
-      init_op, get_next, _, restore_op = _build_graph(start, stop_1)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testInitThenRestore(self):
-    # Note: Calling init_op before restore_op is redundant. This test just makes
-    # sure we do not fail if restore is called on an already initialized
-    # iterator resource.
-
-    def _build_graph(start, stop):
-      dataset = dataset_ops.Dataset.range(start, stop)
-      iterator = dataset.make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    break_point = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testMultipleSaves(self):
-
-    def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    start = 2
-    stop = 10
-    break_point1 = 5
-    break_point2 = 7
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point1):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point1, break_point2):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    break_point2 = 7
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point2, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testSaveRestoreWithRepeat(self):
-
-    def _build_graph(start, stop, num_epochs):
-      iterator = dataset_ops.Dataset.range(
-          start, stop).repeat(num_epochs).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    start = 2
-    stop = 10
-    num_epochs = 5
-    break_range = 5
-    break_epoch = 3
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(
-          start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for _ in range(break_epoch - 1):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        for i in range(start, break_range):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_range, stop):
-          self.assertEqual(i, sess.run(get_next))
-        for _ in range(break_epoch, num_epochs):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testSaveRestoreExhaustedIterator(self):
-
-    def _build_graph(start, stop, num_epochs):
-      iterator = dataset_ops.Dataset.range(
-          start, stop).repeat(num_epochs).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    start = 2
-    stop = 10
-    num_epochs = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(
-          start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for _ in range(num_epochs):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
index 4fef4f30bf..483a79513a 100644
--- a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
@@ -23,19 +23,11 @@ import zlib
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.lib.io import python_io
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_dataset_ops
-from tensorflow.python.ops import io_ops
-from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -47,6 +39,7 @@ except ImportError:
   psutil_import_succeeded = False
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class TextLineDatasetTest(test_base.DatasetTestBase):
 
   def _lineText(self, f, l):
@@ -88,66 +81,41 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
   def _testTextLineDataset(self, compression_type=None):
     test_filenames = self._createFiles(
         2, 5, crlf=True, compression_type=compression_type)
-    filenames = array_ops.placeholder(dtypes.string, shape=[None])
-    num_epochs = array_ops.placeholder(dtypes.int64, shape=[])
-    batch_size = array_ops.placeholder(dtypes.int64, shape=[])
-
-    repeat_dataset = readers.TextLineDataset(
-        filenames, compression_type=compression_type).repeat(num_epochs)
-    batch_dataset = repeat_dataset.batch(batch_size)
-
-    iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types)
-    init_op = iterator.make_initializer(repeat_dataset)
-    init_batch_op = iterator.make_initializer(batch_dataset)
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # Basic test: read from file 0.
-      sess.run(
-          init_op, feed_dict={filenames: [test_filenames[0]],
-                              num_epochs: 1})
-      for i in range(5):
-        self.assertEqual(self._lineText(0, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Basic test: read from file 1.
-      sess.run(
-          init_op, feed_dict={filenames: [test_filenames[1]],
-                              num_epochs: 1})
-      for i in range(5):
-        self.assertEqual(self._lineText(1, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Basic test: read from both files.
-      sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1})
-      for j in range(2):
-        for i in range(5):
-          self.assertEqual(self._lineText(j, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test repeated iteration through both files.
-      sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 10})
-      for _ in range(10):
-        for j in range(2):
-          for i in range(5):
-            self.assertEqual(self._lineText(j, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test batched and repeated iteration through both files.
-      sess.run(
-          init_batch_op,
-          feed_dict={filenames: test_filenames,
-                     num_epochs: 10,
-                     batch_size: 5})
-      for _ in range(10):
-        self.assertAllEqual([self._lineText(0, i) for i in range(5)],
-                            sess.run(get_next))
-        self.assertAllEqual([self._lineText(1, i) for i in range(5)],
-                            sess.run(get_next))
+
+    def dataset_fn(filenames, num_epochs, batch_size=None):
+      repeat_dataset = readers.TextLineDataset(
+          filenames, compression_type=compression_type).repeat(num_epochs)
+      if batch_size:
+        return repeat_dataset.batch(batch_size)
+      return repeat_dataset
+
+    # Basic test: read from file 0.
+    expected_output = [self._lineText(0, i) for i in range(5)]
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[0]], 1), expected_output=expected_output)
+
+    # Basic test: read from file 1.
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[1]], 1),
+        expected_output=[self._lineText(1, i) for i in range(5)])
+
+    # Basic test: read from both files.
+    expected_output = [self._lineText(0, i) for i in range(5)]
+    expected_output.extend([self._lineText(1, i) for i in range(5)])
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 1), expected_output=expected_output)
+
+    # Test repeated iteration through both files.
+    expected_output = [self._lineText(0, i) for i in range(5)]
+    expected_output.extend([self._lineText(1, i) for i in range(5)])
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 10), expected_output=expected_output * 10)
+
+    # Test batched and repeated iteration through both files.
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 10, 5),
+        expected_output=[[self._lineText(0, i) for i in range(5)],
+                         [self._lineText(1, i) for i in range(5)]] * 10)
 
   def testTextLineDatasetNoCompression(self):
     self._testTextLineDataset()
@@ -162,14 +130,10 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
     test_filenames = self._createFiles(2, 5, crlf=True)
 
     repeat_dataset = readers.TextLineDataset(test_filenames, buffer_size=10)
-    iterator = repeat_dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      for j in range(2):
-        for i in range(5):
-          self.assertEqual(self._lineText(j, i), sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    expected_output = []
+    for j in range(2):
+      expected_output.extend([self._lineText(j, i) for i in range(5)])
+    self.assertDatasetProduces(repeat_dataset, expected_output=expected_output)
 
   def testIteratorResourceCleanup(self):
     filename = os.path.join(self.get_temp_dir(), "text.txt")
@@ -200,10 +164,10 @@ class TextLineDatasetTest(test_base.DatasetTestBase):
       self.assertNotIn(filename, [open_file.path for open_file in open_files])
 
 
-class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
+class FixedLengthRecordReaderTestBase(test_base.DatasetTestBase):
 
   def setUp(self):
-    super(FixedLengthRecordReaderTest, self).setUp()
+    super(FixedLengthRecordReaderTestBase, self).setUp()
     self._num_files = 2
     self._num_records = 7
     self._header_bytes = 5
@@ -241,77 +205,64 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
 
     return filenames
 
+
+@test_util.run_all_in_graph_and_eager_modes
+class FixedLengthRecordReaderTest(FixedLengthRecordReaderTestBase):
+
   def _testFixedLengthRecordDataset(self, compression_type=None):
     test_filenames = self._createFiles(compression_type=compression_type)
-    filenames = array_ops.placeholder(dtypes.string, shape=[None])
-    num_epochs = array_ops.placeholder(dtypes.int64, shape=[])
-    batch_size = array_ops.placeholder(dtypes.int64, shape=[])
-
-    repeat_dataset = (
-        readers.FixedLengthRecordDataset(
-            filenames,
-            self._record_bytes,
-            self._header_bytes,
-            self._footer_bytes,
-            compression_type=compression_type).repeat(num_epochs))
-    batch_dataset = repeat_dataset.batch(batch_size)
-
-    iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types)
-    init_op = iterator.make_initializer(repeat_dataset)
-    init_batch_op = iterator.make_initializer(batch_dataset)
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # Basic test: read from file 0.
-      sess.run(
-          init_op, feed_dict={filenames: [test_filenames[0]],
-                              num_epochs: 1})
-      for i in range(self._num_records):
-        self.assertEqual(self._record(0, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Basic test: read from file 1.
-      sess.run(
-          init_op, feed_dict={filenames: [test_filenames[1]],
-                              num_epochs: 1})
-      for i in range(self._num_records):
-        self.assertEqual(self._record(1, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Basic test: read from both files.
-      sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 1})
+
+    def dataset_fn(filenames, num_epochs, batch_size=None):
+      repeat_dataset = readers.FixedLengthRecordDataset(
+          filenames,
+          self._record_bytes,
+          self._header_bytes,
+          self._footer_bytes,
+          compression_type=compression_type).repeat(num_epochs)
+      if batch_size:
+        return repeat_dataset.batch(batch_size)
+      return repeat_dataset
+
+    # Basic test: read from file 0.
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[0]], 1),
+        expected_output=[
+            self._record(0, i) for i in range(self._num_records)
+        ])
+
+    # Basic test: read from file 1.
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[1]], 1),
+        expected_output=[
+            self._record(1, i) for i in range(self._num_records)
+        ])
+
+    # Basic test: read from both files.
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 1), expected_output=expected_output)
+
+    # Test repeated iteration through both files.
+    get_next = self.getNext(dataset_fn(test_filenames, 10))
+    for _ in range(10):
       for j in range(self._num_files):
         for i in range(self._num_records):
-          self.assertEqual(self._record(j, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test repeated iteration through both files.
-      sess.run(init_op, feed_dict={filenames: test_filenames, num_epochs: 10})
-      for _ in range(10):
-        for j in range(self._num_files):
-          for i in range(self._num_records):
-            self.assertEqual(self._record(j, i), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test batched and repeated iteration through both files.
-      sess.run(
-          init_batch_op,
-          feed_dict={
-              filenames: test_filenames,
-              num_epochs: 10,
-              batch_size: self._num_records
-          })
-      for _ in range(10):
-        for j in range(self._num_files):
-          self.assertAllEqual(
-              [self._record(j, i) for i in range(self._num_records)],
-              sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+          self.assertEqual(self._record(j, i), self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+    # Test batched and repeated iteration through both files.
+    get_next = self.getNext(dataset_fn(test_filenames, 10, self._num_records))
+    for _ in range(10):
+      for j in range(self._num_files):
+        self.assertAllEqual(
+            [self._record(j, i) for i in range(self._num_records)],
+            self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   def testFixedLengthRecordDatasetNoCompression(self):
     self._testFixedLengthRecordDataset()
@@ -330,14 +281,11 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
         self._header_bytes,
         self._footer_bytes,
         buffer_size=10)
-    iterator = dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertEqual(self._record(j, i), sess.run(iterator.get_next()))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(iterator.get_next())
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testFixedLengthRecordDatasetWrongSize(self):
     test_filenames = self._createFiles()
@@ -347,310 +295,17 @@ class FixedLengthRecordReaderTest(test_base.DatasetTestBase):
         self._header_bytes,
         self._footer_bytes,
         buffer_size=10)
-    iterator = dataset.make_one_shot_iterator()
-
-    with self.cached_session() as sess:
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          r"Excluding the header \(5 bytes\) and footer \(2 bytes\), input "
-          r"file \".*fixed_length_record.0.txt\" has body length 21 bytes, "
-          r"which is not an exact multiple of the record length \(4 bytes\)."):
-        sess.run(iterator.get_next())
-
-  def _iterator_checkpoint_path(self):
-    return os.path.join(self.get_temp_dir(), "iterator")
-
-  def _save_op(self, iterator_resource):
-    iterator_state_variant = gen_dataset_ops.serialize_iterator(
-        iterator_resource)
-    save_op = io_ops.write_file(
-        self._iterator_checkpoint_path(),
-        parsing_ops.serialize_tensor(iterator_state_variant))
-    return save_op
-
-  def _restore_op(self, iterator_resource):
-    iterator_state_variant = parsing_ops.parse_tensor(
-        io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant)
-    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
-                                                      iterator_state_variant)
-    return restore_op
-
-  def _build_iterator_graph(self, num_epochs):
-    filenames = self._createFiles()
-    dataset = (readers.FixedLengthRecordDataset(
-        filenames, self._record_bytes, self._header_bytes, self._footer_bytes)
-               .repeat(num_epochs))
-    iterator = dataset.make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next_op = iterator.get_next()
-    save_op = self._save_op(iterator._iterator_resource)
-    restore_op = self._restore_op(iterator._iterator_resource)
-    return init_op, get_next_op, save_op, restore_op
-
-  def _restore_iterator(self):
-    output_types = dtypes.string
-    output_shapes = tensor_shape.scalar()
-    iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes)
-    get_next = iterator.get_next()
-    restore_op = self._restore_op(iterator._iterator_resource)
-    return restore_op, get_next
-
-  def testSaveRestore(self):
-    num_epochs = 10
-    epoch_break = 5
-    file_break = self._num_files // 2
-    record_break = self._num_records // 2
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch == epoch_break and f == file_break and
-                  r == record_break):
-                sess.run(save_op)
-                break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-            else:
-              continue
-            break
-          else:
-            continue
-          break
-        else:
-          with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch < epoch_break or
-                  (epoch == epoch_break and f < file_break) or
-                  (epoch == epoch_break and f == file_break and
-                   r < record_break)):
-                continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-
-  def testInitThenRestore(self):
-    # Note: Calling init_op before restore_op is redundant. This test just makes
-    # sure we do not fail if restore is called on an already initialized
-    # iterator resource.
-    num_epochs = 10
-    epoch_break = 5
-    file_break = self._num_files // 2
-    record_break = self._num_records // 2
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch == epoch_break and f == file_break and
-                  r == record_break):
-                sess.run(save_op)
-                break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-            else:
-              continue
-            break
-          else:
-            continue
-          break
-        else:
-          with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch < epoch_break or
-                  (epoch == epoch_break and f < file_break) or
-                  (epoch == epoch_break and f == file_break and
-                   r < record_break)):
-                continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-
-  def testRestoreInModifiedGraph(self):
-    num_epochs = 10
-    num_epochs_1 = 20
-    epoch_break = 5
-    file_break = self._num_files // 2
-    record_break = self._num_records // 2
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch == epoch_break and f == file_break and
-                  r == record_break):
-                sess.run(save_op)
-                break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-            else:
-              continue
-            break
-          else:
-            continue
-          break
-        else:
-          with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs_1)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch < epoch_break or
-                  (epoch == epoch_break and f < file_break) or
-                  (epoch == epoch_break and f == file_break and
-                   r < record_break)):
-                continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-
-  def testRestoreWithoutBuildingDatasetGraph(self):
-    num_epochs = 10
-    epoch_break = 5
-    file_break = self._num_files // 2
-    record_break = self._num_records // 2
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch == epoch_break and f == file_break and
-                  r == record_break):
-                sess.run(save_op)
-                break
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-            else:
-              continue
-            break
-          else:
-            continue
-          break
-        else:
-          with self.assertRaises(errors.OutOfRangeError):
-            sess.run(get_next_op)
-
-    with ops.Graph().as_default() as g:
-      restore_op, get_next_op = self._restore_iterator()
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for epoch in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              if (epoch < epoch_break or
-                  (epoch == epoch_break and f < file_break) or
-                  (epoch == epoch_break and f == file_break and
-                   r < record_break)):
-                continue
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-
-  def testRestoreUnusedIterator(self):
-    num_epochs = 10
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        # Save unused iterator.
-        sess.run(save_op)
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for _ in range(num_epochs * self._num_files * self._num_records):
-          sess.run(get_next_op)
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-
-  def testRestoreExhaustedIterator(self):
-    num_epochs = 10
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for _ in range(num_epochs):
-          for f in range(self._num_files):
-            for r in range(self._num_records):
-              self.assertEqual(self._record(f, r), sess.run(get_next_op))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next_op, save_op, restore_op = self._build_iterator_graph(
-          num_epochs=num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next_op)
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            r"Excluding the header \(5 bytes\) and footer \(2 bytes\), input "
+            r"file \".*fixed_length_record.0.txt\" has body length 21 bytes, "
+            r"which is not an exact multiple of the record length \(4 bytes\).")
+        )
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class TFRecordDatasetTest(test_base.DatasetTestBase):
 
   def setUp(self):
@@ -660,21 +315,17 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
 
     self.test_filenames = self._createFiles()
 
-    self.filenames = array_ops.placeholder(dtypes.string, shape=[None])
-    self.num_epochs = array_ops.placeholder_with_default(
-        constant_op.constant(1, dtypes.int64), shape=[])
-    self.compression_type = array_ops.placeholder_with_default("", shape=[])
-    self.batch_size = array_ops.placeholder(dtypes.int64, shape=[])
-
-    repeat_dataset = readers.TFRecordDataset(self.filenames,
-                                             self.compression_type).repeat(
-                                                 self.num_epochs)
-    batch_dataset = repeat_dataset.batch(self.batch_size)
+  def dataset_fn(self,
+                 filenames,
+                 compression_type="",
+                 num_epochs=1,
+                 batch_size=None):
 
-    iterator = iterator_ops.Iterator.from_structure(batch_dataset.output_types)
-    self.init_op = iterator.make_initializer(repeat_dataset)
-    self.init_batch_op = iterator.make_initializer(batch_dataset)
-    self.get_next = iterator.get_next()
+    repeat_dataset = readers.TFRecordDataset(
+        filenames, compression_type).repeat(num_epochs)
+    if batch_size:
+      return repeat_dataset.batch(batch_size)
+    return repeat_dataset
 
   def _record(self, f, r):
     return compat.as_bytes("Record %d of file %d" % (r, f))
@@ -691,71 +342,42 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
     return filenames
 
   def testReadOneEpoch(self):
-    with self.cached_session() as sess:
-      # Basic test: read from file 0.
-      sess.run(
-          self.init_op,
-          feed_dict={
-              self.filenames: [self.test_filenames[0]],
-              self.num_epochs: 1
-          })
-      for i in range(self._num_records):
-        self.assertAllEqual(self._record(0, i), sess.run(self.get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
-
-      # Basic test: read from file 1.
-      sess.run(
-          self.init_op,
-          feed_dict={
-              self.filenames: [self.test_filenames[1]],
-              self.num_epochs: 1
-          })
-      for i in range(self._num_records):
-        self.assertAllEqual(self._record(1, i), sess.run(self.get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
-
-      # Basic test: read from both files.
-      sess.run(
-          self.init_op,
-          feed_dict={self.filenames: self.test_filenames,
-                     self.num_epochs: 1})
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(self.get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
+    # Basic test: read from file 0.
+    dataset = self.dataset_fn(self.test_filenames[0])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[self._record(0, i) for i in range(self._num_records)])
+
+    # Basic test: read from file 1.
+    dataset = self.dataset_fn(self.test_filenames[1])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[self._record(1, i) for i in range(self._num_records)])
+
+    # Basic test: read from both files.
+    dataset = self.dataset_fn(self.test_filenames)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testReadTenEpochs(self):
-    with self.cached_session() as sess:
-      sess.run(
-          self.init_op,
-          feed_dict={self.filenames: self.test_filenames,
-                     self.num_epochs: 10})
-      for _ in range(10):
-        for j in range(self._num_files):
-          for i in range(self._num_records):
-            self.assertAllEqual(self._record(j, i), sess.run(self.get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
+    dataset = self.dataset_fn(self.test_filenames, num_epochs=10)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output * 10)
 
   def testReadTenEpochsOfBatches(self):
-    with self.cached_session() as sess:
-      sess.run(
-          self.init_batch_op,
-          feed_dict={
-              self.filenames: self.test_filenames,
-              self.num_epochs: 10,
-              self.batch_size: self._num_records
-          })
-      for _ in range(10):
-        for j in range(self._num_files):
-          values = sess.run(self.get_next)
-          self.assertAllEqual(
-              [self._record(j, i) for i in range(self._num_records)], values)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
+    dataset = self.dataset_fn(
+        self.test_filenames, num_epochs=10, batch_size=self._num_records)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.append(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output * 10)
 
   def testReadZlibFiles(self):
     zlib_files = []
@@ -767,17 +389,12 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
         with open(zfn, "wb") as f:
           f.write(cdata)
         zlib_files.append(zfn)
-
-    with self.cached_session() as sess:
-      sess.run(
-          self.init_op,
-          feed_dict={self.filenames: zlib_files,
-                     self.compression_type: "ZLIB"})
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(self.get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = self.dataset_fn(zlib_files, compression_type="ZLIB")
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testReadGzipFiles(self):
     gzip_files = []
@@ -787,59 +404,42 @@ class TFRecordDatasetTest(test_base.DatasetTestBase):
         with gzip.GzipFile(gzfn, "wb") as gzf:
           gzf.write(f.read())
         gzip_files.append(gzfn)
-
-    with self.cached_session() as sess:
-      sess.run(
-          self.init_op,
-          feed_dict={self.filenames: gzip_files,
-                     self.compression_type: "GZIP"})
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(self.get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.get_next)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = self.dataset_fn(gzip_files, compression_type="GZIP")
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testReadWithBuffer(self):
     one_mebibyte = 2**20
-    d = readers.TFRecordDataset(self.test_filenames, buffer_size=one_mebibyte)
-    iterator = d.make_one_shot_iterator()
-    next_element = iterator.get_next()
-    with self.cached_session() as sess:
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+    dataset = readers.TFRecordDataset(
+        self.test_filenames, buffer_size=one_mebibyte)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testReadFromDatasetOfFiles(self):
     files = dataset_ops.Dataset.from_tensor_slices(self.test_filenames)
-    d = readers.TFRecordDataset(files)
-    iterator = d.make_one_shot_iterator()
-    next_element = iterator.get_next()
-    with self.cached_session() as sess:
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertAllEqual(self._record(j, i), sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = readers.TFRecordDataset(files)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testReadTenEpochsFromDatasetOfFilesInParallel(self):
     files = dataset_ops.Dataset.from_tensor_slices(
         self.test_filenames).repeat(10)
-    d = readers.TFRecordDataset(files, num_parallel_reads=4)
-    iterator = d.make_one_shot_iterator()
-    next_element = iterator.get_next()
-    expected = []
-    actual = []
-    with self.cached_session() as sess:
-      for _ in range(10):
-        for j in range(self._num_files):
-          for i in range(self._num_records):
-            expected.append(self._record(j, i))
-            actual.append(sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      self.assertEqual(sorted(expected), sorted(actual))
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = readers.TFRecordDataset(files, num_parallel_reads=4)
+    self.assertDatasetProduces(
+        dataset, expected_output=expected_output * 10, assert_items_equal=True)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py b/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
index 11e07300b9..061f0d1343 100644
--- a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
@@ -22,21 +22,24 @@ import numpy as np
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testSum(self):
     for i in range(10):
       ds = dataset_ops.Dataset.range(1, i + 1)
-      result = ds.reduce(np.int64(0), lambda x, y: x + y)
-      with self.cached_session() as sess:
-        self.assertEqual(((i + 1) * i) // 2, sess.run(result))
+      result = ds.reduce(
+          constant_op.constant(0, dtype=dtypes.int64), lambda x, y: x + y)
+      self.assertEqual(((i + 1) * i) // 2, self.evaluate(result))
 
   def testSumTuple(self):
 
@@ -47,9 +50,8 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     for i in range(10):
       ds = dataset_ops.Dataset.range(1, i + 1)
       ds = dataset_ops.Dataset.zip((ds, ds))
-      result = ds.reduce(np.int64(0), reduce_fn)
-      with self.cached_session() as sess:
-        self.assertEqual(((i + 1) * i), sess.run(result))
+      result = ds.reduce(constant_op.constant(0, dtype=dtypes.int64), reduce_fn)
+      self.assertEqual(((i + 1) * i), self.evaluate(result))
 
   def testSumAndCount(self):
 
@@ -59,13 +61,14 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     for i in range(10):
       ds = dataset_ops.Dataset.range(1, i + 1)
-      result = ds.reduce((np.int64(0), np.int64(0)), reduce_fn)
-      with self.cached_session() as sess:
-        s, c = sess.run(result)
-        self.assertEqual(((i + 1) * i) // 2, s)
-        self.assertEqual(i, c)
-
-  def testSquareUsingPlaceholder(self):
+      result = ds.reduce((constant_op.constant(0, dtype=dtypes.int64),
+                          constant_op.constant(0, dtype=dtypes.int64)),
+                         reduce_fn)
+      s, c = self.evaluate(result)
+      self.assertEqual(((i + 1) * i) // 2, s)
+      self.assertEqual(i, c)
+
+  def testSkipEagerSquareUsingPlaceholder(self):
     delta = array_ops.placeholder(dtype=dtypes.int64)
 
     def reduce_fn(state, _):
@@ -92,8 +95,7 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     for i in range(10):
       ds = dataset_ops.Dataset.from_tensors(make_sparse_fn(i+1))
       result = ds.reduce(make_sparse_fn(0), reduce_fn)
-      with self.cached_session() as sess:
-        self.assertSparseValuesEqual(make_sparse_fn(i+1), sess.run(result))
+      self.assertSparseValuesEqual(make_sparse_fn(i + 1), self.evaluate(result))
 
   def testNested(self):
 
@@ -115,10 +117,10 @@ class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     for i in range(10):
       ds = dataset_ops.Dataset.range(1, i + 1).map(map_fn)
       result = ds.reduce(map_fn(0), reduce_fn)
-      with self.cached_session() as sess:
-        result = sess.run(result)
-        self.assertEqual(((i + 1) * i) // 2, result["dense"])
-        self.assertSparseValuesEqual(make_sparse_fn(i), result["sparse"])
+      result = self.evaluate(result)
+      self.assertEqual(((i + 1) * i) // 2, result["dense"])
+      self.assertSparseValuesEqual(make_sparse_fn(i), result["sparse"])
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/save_restore_experimental_test.py b/tensorflow/python/data/kernel_tests/save_restore_experimental_test.py
new file mode 100644
index 0000000000..c07d24ef6e
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/save_restore_experimental_test.py
@@ -0,0 +1,686 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the experimental saving and restoring input pipeline."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.python.data.kernel_tests import reader_dataset_ops_test
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import test
+
+
+class ExperimentalCheckpointFixedLengthRecordReaderTest(
+    reader_dataset_ops_test.FixedLengthRecordReaderTestBase):
+
+  def tearDown(self):
+    # Remove all checkpoint files.
+    prefix = self._iterator_checkpoint_prefix()
+    pattern = prefix + "*"
+    files = gfile.Glob(pattern)
+    map(gfile.Remove, files)
+
+  def _iterator_checkpoint_prefix(self):
+    return os.path.join(self.get_temp_dir(), "iterator")
+
+  def _save_op(self, iterator_resource):
+    iterator_state_variant = gen_dataset_ops.serialize_iterator(
+        iterator_resource)
+    save_op = io_ops.write_file(
+        self._iterator_checkpoint_prefix(),
+        parsing_ops.serialize_tensor(iterator_state_variant))
+    return save_op
+
+  def _restore_op(self, iterator_resource):
+    iterator_state_variant = parsing_ops.parse_tensor(
+        io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant)
+    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                      iterator_state_variant)
+    return restore_op
+
+  def testSaveRestore(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    break_point = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(init_op)
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+    # Saving and restoring in same session.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testRestoreWithoutBuildingDatasetGraph(self):
+
+    def _build_graph(start, stop, num_epochs):
+      dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
+      iterator = dataset.make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    num_epochs = 5
+    break_point = 5
+    break_epoch = 3
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for _ in range(break_epoch):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      # Create an empty IteratorResource and restore the Iterator into it.
+      output_types = dtypes.int64
+      output_shapes = tensor_shape.scalar()
+      iterator = iterator_ops.Iterator.from_structure(output_types,
+                                                      output_shapes)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      get_next = iterator.get_next()
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        for _ in range(break_epoch + 1, num_epochs):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testRestoreInModifiedGraph(self):
+
+    def _build_graph(start, stop):
+      dataset = dataset_ops.Dataset.range(start, stop)
+      iterator = dataset.make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    stop_1 = 8
+    break_point = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      # Intentionally build a graph with a different value for stop to make sure
+      # the original dataset graph is actually getting loaded.
+      init_op, get_next, _, restore_op = _build_graph(start, stop_1)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testInitThenRestore(self):
+    # Note: Calling init_op before restore_op is redundant. This test just makes
+    # sure we do not fail if restore is called on an already initialized
+    # iterator resource.
+
+    def _build_graph(start, stop):
+      dataset = dataset_ops.Dataset.range(start, stop)
+      iterator = dataset.make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    break_point = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(init_op)
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testMultipleSaves(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    start = 2
+    stop = 10
+    break_point1 = 5
+    break_point2 = 7
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point1):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point1, break_point2):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    break_point2 = 7
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point2, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreWithRepeat(self):
+
+    def _build_graph(start, stop, num_epochs):
+      iterator = dataset_ops.Dataset.range(
+          start, stop).repeat(num_epochs).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    start = 2
+    stop = 10
+    num_epochs = 5
+    break_range = 5
+    break_epoch = 3
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(
+          start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        # Note: There is no checkpoint saved currently so a NotFoundError is
+        # raised.
+        with self.assertRaises(errors.NotFoundError):
+          sess.run(restore_op)
+        for _ in range(break_epoch - 1):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        for i in range(start, break_range):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_range, stop):
+          self.assertEqual(i, sess.run(get_next))
+        for _ in range(break_epoch, num_epochs):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreExhaustedIterator(self):
+
+    def _build_graph(start, stop, num_epochs):
+      iterator = dataset_ops.Dataset.range(
+          start, stop).repeat(num_epochs).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    start = 2
+    stop = 10
+    num_epochs = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(
+          start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        # Note: There is no checkpoint saved currently so a NotFoundError is
+        # raised.
+        with self.assertRaises(errors.NotFoundError):
+          sess.run(restore_op)
+        for _ in range(num_epochs):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+
+class ExperimentalCheckpointRangeTest(test_base.DatasetTestBase):
+
+  def tearDown(self):
+    # Remove all checkpoint files.
+    prefix = self._iterator_checkpoint_prefix()
+    pattern = prefix + "*"
+    files = gfile.Glob(pattern)
+    map(gfile.Remove, files)
+
+  def _iterator_checkpoint_prefix(self):
+    return os.path.join(self.get_temp_dir(), "iterator")
+
+  def _save_op(self, iterator_resource):
+    iterator_state_variant = gen_dataset_ops.serialize_iterator(
+        iterator_resource)
+    save_op = io_ops.write_file(
+        self._iterator_checkpoint_prefix(),
+        parsing_ops.serialize_tensor(iterator_state_variant))
+    return save_op
+
+  def _restore_op(self, iterator_resource):
+    iterator_state_variant = parsing_ops.parse_tensor(
+        io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant)
+    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                      iterator_state_variant)
+    return restore_op
+
+  def testSaveRestore(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    break_point = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(init_op)
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+    # Saving and restoring in same session.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testRestoreWithoutBuildingDatasetGraph(self):
+
+    def _build_graph(start, stop, num_epochs):
+      dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
+      iterator = dataset.make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    num_epochs = 5
+    break_point = 5
+    break_epoch = 3
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for _ in range(break_epoch):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      # Create an empty IteratorResource and restore the Iterator into it.
+      output_types = dtypes.int64
+      output_shapes = tensor_shape.scalar()
+      iterator = iterator_ops.Iterator.from_structure(output_types,
+                                                      output_shapes)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      get_next = iterator.get_next()
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        for _ in range(break_epoch + 1, num_epochs):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testRestoreInModifiedGraph(self):
+
+    def _build_graph(start, stop):
+      dataset = dataset_ops.Dataset.range(start, stop)
+      iterator = dataset.make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    stop_1 = 8
+    break_point = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      # Intentionally build a graph with a different value for stop to make sure
+      # the original dataset graph is actually getting loaded.
+      init_op, get_next, _, restore_op = _build_graph(start, stop_1)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testInitThenRestore(self):
+    # Note: Calling init_op before restore_op is redundant. This test just makes
+    # sure we do not fail if restore is called on an already initialized
+    # iterator resource.
+
+    def _build_graph(start, stop):
+      dataset = dataset_ops.Dataset.range(start, stop)
+      iterator = dataset.make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    # Saving and restoring in different sessions.
+    start = 2
+    stop = 10
+    break_point = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(init_op)
+        sess.run(restore_op)
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testMultipleSaves(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    start = 2
+    stop = 10
+    break_point1 = 5
+    break_point2 = 7
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, _ = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point1):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point1, break_point2):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    break_point2 = 7
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_point2, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreWithRepeat(self):
+
+    def _build_graph(start, stop, num_epochs):
+      iterator = dataset_ops.Dataset.range(
+          start, stop).repeat(num_epochs).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    start = 2
+    stop = 10
+    num_epochs = 5
+    break_range = 5
+    break_epoch = 3
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(
+          start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        # Note: There is no checkpoint saved currently so a NotFoundError is
+        # raised.
+        with self.assertRaises(errors.NotFoundError):
+          sess.run(restore_op)
+        for _ in range(break_epoch - 1):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        for i in range(start, break_range):
+          self.assertEqual(i, sess.run(get_next))
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        for i in range(break_range, stop):
+          self.assertEqual(i, sess.run(get_next))
+        for _ in range(break_epoch, num_epochs):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreExhaustedIterator(self):
+
+    def _build_graph(start, stop, num_epochs):
+      iterator = dataset_ops.Dataset.range(
+          start, stop).repeat(num_epochs).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
+      return init_op, get_next, save_op, restore_op
+
+    start = 2
+    stop = 10
+    num_epochs = 5
+    with ops.Graph().as_default() as g:
+      init_op, get_next, save_op, restore_op = _build_graph(
+          start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        # Note: There is no checkpoint saved currently so a NotFoundError is
+        # raised.
+        with self.assertRaises(errors.NotFoundError):
+          sess.run(restore_op)
+        for _ in range(num_epochs):
+          for i in range(start, stop):
+            self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+        sess.run(save_op)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
+      with self.session(graph=g) as sess:
+        sess.run(restore_op)
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
index 9d06781094..7124cc75f1 100644
--- a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
@@ -22,14 +22,17 @@ import numpy as np
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
@@ -62,65 +65,50 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
                   np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
                   np.array(37.0) * np.arange(7))
 
-    count_t = array_ops.placeholder(dtypes.int64, shape=[])
-    size_t = array_ops.placeholder(dtypes.int64, shape=[])
-    shift_t = array_ops.placeholder(dtypes.int64, shape=[])
-    stride_t = array_ops.placeholder(dtypes.int64, shape=[])
-    drop_remainder_t = array_ops.placeholder(dtypes.bool, shape=[])
-
     def _map_fn(x, y, z):
       return math_ops.square(x), math_ops.square(y), math_ops.square(z)
 
     def _flat_map_fn(x, y, z):
-      return dataset_ops.Dataset.zip((x.batch(batch_size=size_t),
-                                      y.batch(batch_size=size_t),
-                                      z.batch(batch_size=size_t)))
+      return dataset_ops.Dataset.zip((x.batch(batch_size=size),
+                                      y.batch(batch_size=size),
+                                      z.batch(batch_size=size)))
 
-    iterator = dataset_ops.Dataset.from_tensor_slices(components).map(
+    dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
         _map_fn).repeat(count).window(
-            size=size_t,
-            shift=shift_t,
-            stride=stride_t,
-            drop_remainder=drop_remainder_t).flat_map(
-                _flat_map_fn).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual([[None] + list(c.shape[1:]) for c in components],
-                     [t.shape.as_list() for t in get_next])
-
-    with self.cached_session() as sess:
-      sess.run(
-          init_op,
-          feed_dict={
-              count_t: count,
-              size_t: size,
-              shift_t: shift,
-              stride_t: stride,
-              drop_remainder_t: drop_remainder
-          })
-      num_full_batches = max(
-          0, (count * 7 - ((size - 1) * stride + 1)) // shift + 1)
-      for i in range(num_full_batches):
-        result = sess.run(get_next)
+            size=size,
+            shift=shift,
+            stride=stride,
+            drop_remainder=drop_remainder).flat_map(_flat_map_fn)
+    get_next = self.getNext(dataset)
+
+    self.assertEqual(
+        [[None] + list(c.shape[1:]) for c in components],
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)])
+
+    num_full_batches = max(0,
+                           (count * 7 - ((size - 1) * stride + 1)) // shift + 1)
+    for i in range(num_full_batches):
+      result = self.evaluate(get_next())
+      for component, result_component in zip(components, result):
+        for j in range(size):
+          self.assertAllEqual(component[(i * shift + j * stride) % 7]**2,
+                              result_component[j])
+    if not drop_remainder:
+      num_partial_batches = (count * 7) // shift + (
+          (count * 7) % shift > 0) - num_full_batches
+      for i in range(num_partial_batches):
+        result = self.evaluate(get_next())
         for component, result_component in zip(components, result):
-          for j in range(size):
-            self.assertAllEqual(component[(i * shift + j * stride) % 7]**2,
-                                result_component[j])
-      if not drop_remainder:
-        num_partial_batches = (count * 7) // shift + (
-            (count * 7) % shift > 0) - num_full_batches
-        for i in range(num_partial_batches):
-          result = sess.run(get_next)
-          for component, result_component in zip(components, result):
-            remaining = (count * 7) - ((num_full_batches + i) * shift)
-            num_elements = remaining // stride + ((remaining % stride) > 0)
-            for j in range(num_elements):
-              self.assertAllEqual(
-                  component[((num_full_batches + i) * shift + j * stride) % 7]
-                  **2, result_component[j])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+          remaining = (count * 7) - ((num_full_batches + i) * shift)
+          num_elements = remaining // stride + ((remaining % stride) > 0)
+          for j in range(num_elements):
+            self.assertAllEqual(
+                component[((num_full_batches + i) * shift + j * stride) % 7]**2,
+                result_component[j])
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
   @parameterized.named_parameters(
       ("1", 14, 0, 3, 1),
@@ -128,28 +116,12 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("3", 14, 3, 3, 0),
   )
   def testWindowDatasetInvalid(self, count, size, shift, stride):
-    count_t = array_ops.placeholder(dtypes.int64, shape=[])
-    size_t = array_ops.placeholder(dtypes.int64, shape=[])
-    shift_t = array_ops.placeholder(dtypes.int64, shape=[])
-    stride_t = array_ops.placeholder(dtypes.int64, shape=[])
-
-    iterator = dataset_ops.Dataset.range(10).map(lambda x: x).repeat(
-        count_t).window(
-            size=size_t, shift=shift_t,
-            stride=stride_t).flat_map(lambda x: x.batch(batch_size=size_t)
-                                     ).make_initializable_iterator()
-    init_op = iterator.initializer
-
-    with self.cached_session() as sess:
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(
-            init_op,
-            feed_dict={
-                count_t: count,
-                size_t: size,
-                shift_t: shift,
-                stride_t: stride
-            })
+    dataset = dataset_ops.Dataset.range(10).map(lambda x: x).repeat(
+        count).window(
+            size=size, shift=shift,
+            stride=stride).flat_map(lambda x: x.batch(batch_size=size))
+    self.assertDatasetProduces(
+        dataset, expected_error=(errors.InvalidArgumentError, ""))
 
   def testWindowSparse(self):
 
@@ -157,25 +129,18 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return sparse_tensor.SparseTensorValue(
           indices=[[0]], values=(i * [1]), dense_shape=[1])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).window(
-        size=5, shift=3, drop_remainder=True).flat_map(
-            lambda x: x.batch(batch_size=5)).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).window(
+        size=5, shift=3,
+        drop_remainder=True).flat_map(lambda x: x.batch(batch_size=5))
 
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      num_batches = (10 - 5) // 3 + 1
-      for i in range(num_batches):
-        actual = sess.run(get_next)
-        expected = sparse_tensor.SparseTensorValue(
+    num_batches = (10 - 5) // 3 + 1
+    expected_output = [
+        sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4],
-            dense_shape=[5, 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+            dense_shape=[5, 1]) for i in range(num_batches)
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testWindowSparseWithDifferentDenseShapes(self):
 
@@ -186,31 +151,25 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           values=array_ops.fill([math_ops.to_int32(i)], i),
           dense_shape=[i])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).window(
-        size=5, shift=3, drop_remainder=True).flat_map(
-            lambda x: x.batch(batch_size=5)).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      num_batches = (10 - 5) // 3 + 1
-      for i in range(num_batches):
-        actual = sess.run(get_next)
-        expected_indices = []
-        expected_values = []
-        for j in range(5):
-          for k in range(i * 3 + j):
-            expected_indices.append([j, k])
-            expected_values.append(i * 3 + j)
-        expected = sparse_tensor.SparseTensorValue(
-            indices=expected_indices,
-            values=expected_values,
-            dense_shape=[5, i * 3 + 5 - 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).window(
+        size=5, shift=3,
+        drop_remainder=True).flat_map(lambda x: x.batch(batch_size=5))
+
+    expected_output = []
+    num_batches = (10 - 5) // 3 + 1
+    for i in range(num_batches):
+      expected_indices = []
+      expected_values = []
+      for j in range(5):
+        for k in range(i * 3 + j):
+          expected_indices.append([j, k])
+          expected_values.append(i * 3 + j)
+      expected_output.append(
+          sparse_tensor.SparseTensorValue(
+              indices=expected_indices,
+              values=expected_values,
+              dense_shape=[5, i * 3 + 5 - 1]))
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testNestedWindowSparse(self):
 
@@ -218,38 +177,27 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return sparse_tensor.SparseTensorValue(
           indices=[[0]], values=(i * [1]), dense_shape=[1])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).window(
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).window(
         size=4, shift=2,
         drop_remainder=True).flat_map(lambda x: x.batch(batch_size=4)).window(
-            size=3, shift=1, drop_remainder=True).flat_map(
-                lambda x: x.batch(batch_size=3)).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      # Slide: 1st batch.
-      actual = sess.run(get_next)
-      expected = sparse_tensor.SparseTensorValue(
-          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
-                   [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
-                   [2, 2, 0], [2, 3, 0]],
-          values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7],
-          dense_shape=[3, 4, 1])
-      self.assertTrue(sparse_tensor.is_sparse(actual))
-      self.assertSparseValuesEqual(actual, expected)
-      # Slide: 2nd batch.
-      actual = sess.run(get_next)
-      expected = sparse_tensor.SparseTensorValue(
-          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
-                   [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
-                   [2, 2, 0], [2, 3, 0]],
-          values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9],
-          dense_shape=[3, 4, 1])
-      self.assertTrue(sparse_tensor.is_sparse(actual))
-      self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+            size=3, shift=1,
+            drop_remainder=True).flat_map(lambda x: x.batch(batch_size=3))
+
+    expected_output = [
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
+                     [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
+                     [2, 2, 0], [2, 3, 0]],
+            values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7],
+            dense_shape=[3, 4, 1]),
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0],
+                     [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0],
+                     [2, 2, 0], [2, 3, 0]],
+            values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9],
+            dense_shape=[3, 4, 1])
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   def testWindowShapeError(self):
 
@@ -258,19 +206,15 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       yield [4.0, 5.0, 6.0]
       yield [7.0, 8.0, 9.0, 10.0]
 
-    iterator = dataset_ops.Dataset.from_generator(
+    dataset = dataset_ops.Dataset.from_generator(
         generator, dtypes.float32, output_shapes=[None]).window(
-            size=3, shift=1).flat_map(
-                lambda x: x.batch(batch_size=3)).make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          r"Cannot batch tensors with different shapes in component 0. "
-          r"First element had shape \[3\] and element 2 had shape \[4\]."):
-        sess.run(next_element)
+            size=3, shift=1).flat_map(lambda x: x.batch(batch_size=3))
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            r"Cannot batch tensors with different shapes in component 0. "
+            r"First element had shape \[3\] and element 2 had shape \[4\]."))
 
   def testWindowIgnoreErrors(self):
     input_values = np.float32([1., np.nan, 2., np.nan, 3.])
@@ -278,13 +222,9 @@ class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         lambda x: array_ops.check_numerics(x, "message")).window(
             size=2, shift=2, stride=2,
             drop_remainder=True).flat_map(lambda x: x.batch(batch_size=2))
-    get_next = dataset.make_one_shot_iterator().get_next()
-
-    with self.cached_session() as sess:
-      self.assertAllEqual(np.float32([1., 2.]), sess.run(get_next))
-      self.assertAllEqual(np.float32([2., 3.]), sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+    self.assertDatasetProduces(
+        dataset, expected_output=[np.float32([1., 2.]),
+                                  np.float32([2., 3.])])
 
 
 if __name__ == "__main__":
-- 
GitLab


From 34b20ffd2a0f370a4992c3dab915dbc832e374b5 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 26 Nov 2018 12:01:33 -0800
Subject: [PATCH 0789/1554] Re-enable previously disabled tests.

PiperOrigin-RevId: 222863222
---
 tensorflow/examples/autograph/integration_tests/BUILD  | 2 --
 tensorflow/python/autograph/converters/BUILD           | 2 --
 tensorflow/python/autograph/impl/BUILD                 | 2 --
 tensorflow/python/autograph/pyct/BUILD                 | 2 --
 tensorflow/python/autograph/pyct/static_analysis/BUILD | 2 --
 tensorflow/python/eager/BUILD                          | 6 ------
 6 files changed, 16 deletions(-)

diff --git a/tensorflow/examples/autograph/integration_tests/BUILD b/tensorflow/examples/autograph/integration_tests/BUILD
index d20c17b63b..2a4a0f75e7 100644
--- a/tensorflow/examples/autograph/integration_tests/BUILD
+++ b/tensorflow/examples/autograph/integration_tests/BUILD
@@ -22,7 +22,6 @@ py_test(
         "keras_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         "//tensorflow:tensorflow_py",
     ],
@@ -34,7 +33,6 @@ py_test(
         "list_literals_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/python/autograph/converters/BUILD b/tensorflow/python/autograph/converters/BUILD
index ced2e4796b..3ac446db02 100644
--- a/tensorflow/python/autograph/converters/BUILD
+++ b/tensorflow/python/autograph/converters/BUILD
@@ -63,7 +63,6 @@ py_test(
     name = "asserts_test",
     srcs = ["asserts_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":converters",
         "//tensorflow/python:client_testlib",
@@ -239,7 +238,6 @@ py_test(
     name = "error_handlers_test",
     srcs = ["error_handlers_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":converters",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/autograph/impl/BUILD b/tensorflow/python/autograph/impl/BUILD
index 2f9037c43b..201a888754 100644
--- a/tensorflow/python/autograph/impl/BUILD
+++ b/tensorflow/python/autograph/impl/BUILD
@@ -41,7 +41,6 @@ py_test(
     name = "api_test",
     srcs = ["api_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":impl",
         "//tensorflow/python:client_testlib",
@@ -54,7 +53,6 @@ py_test(
     name = "conversion_test",
     srcs = ["conversion_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":impl",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/autograph/pyct/BUILD b/tensorflow/python/autograph/pyct/BUILD
index ddadc6b96e..ba8ec27139 100644
--- a/tensorflow/python/autograph/pyct/BUILD
+++ b/tensorflow/python/autograph/pyct/BUILD
@@ -80,7 +80,6 @@ py_test(
     name = "compiler_test",
     srcs = ["compiler_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":pyct",
         "//tensorflow/python:client_testlib",
@@ -154,7 +153,6 @@ py_test(
     name = "transformer_test",
     srcs = ["transformer_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":pyct",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/autograph/pyct/static_analysis/BUILD b/tensorflow/python/autograph/pyct/static_analysis/BUILD
index 4a4ccdcbd1..5e260c5730 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/BUILD
+++ b/tensorflow/python/autograph/pyct/static_analysis/BUILD
@@ -38,7 +38,6 @@ py_test(
     name = "activity_test",
     srcs = ["activity_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":static_analysis",
         "//tensorflow/python:client_testlib",
@@ -51,7 +50,6 @@ py_test(
     name = "live_values_test",
     srcs = ["live_values_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":static_analysis",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index d3457ed241..5a18afa0fc 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -189,9 +189,6 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
     ],
     shard_count = 5,
-    tags = [
-        "no_windows",
-    ],
 )
 
 cuda_py_test(
@@ -214,9 +211,6 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
     ],
     shard_count = 15,
-    tags = [
-        "no_windows",
-    ],
 )
 
 py_library(
-- 
GitLab


From 968bb07ccc1871368addc2fbe07f61f1a91183a3 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Mon, 26 Nov 2018 12:04:09 -0800
Subject: [PATCH 0790/1554] Remove flags module from v2 API. Use argparse or
 absl if you need flag parsing.

PiperOrigin-RevId: 222863846
---
 tensorflow/api_template.__init__.py             | 2 --
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 4 ----
 2 files changed, 6 deletions(-)

diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index 59b07e15b8..f13623b0d5 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -28,8 +28,6 @@ _component_api_helper.package_hook(
 
 # API IMPORTS PLACEHOLDER
 
-from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-
 # Make sure directory containing top level submodules is in
 # the __path__ so that "from tensorflow.foo import bar" works.
 # We're using bitwise, but there's nothing special about that.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 92ebdb91c5..a1e2e7be6f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -204,10 +204,6 @@ tf_module {
     name: "feature_column"
     mtype: "<type \'module\'>"
   }
-  member {
-    name: "flags"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "float16"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
-- 
GitLab


From 107ae2e68de97434193070747e0fef8b597325e5 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 26 Nov 2018 12:12:51 -0800
Subject: [PATCH 0791/1554] Optimize mkldnn pack.

PiperOrigin-RevId: 222865206
---
 .../core/kernels/eigen_spatial_convolutions.h | 182 ++++++++++--------
 .../eigen_spatial_convolutions_test.cc        | 103 +++++++---
 2 files changed, 179 insertions(+), 106 deletions(-)

diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions.h b/tensorflow/core/kernels/eigen_spatial_convolutions.h
index 1f211b19b4..e8dea4763e 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions.h
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions.h
@@ -56,6 +56,7 @@ namespace internal {
 //
 // TODO(ezhulenev): Consolidate this part of the code with the image patch
 // extraction code since they are both very similar.
+
 template <typename NewDimension, Index Rows, Index Cols, typename ArgType,
           typename Device, typename Scalar_, typename Index,
           typename nocontract_t, typename contract_t, int Side, int packet_size,
@@ -70,6 +71,7 @@ class TensorContractionInputMapper<
     inner_dim_reordered, Alignment> {
  public:
   typedef Scalar_ Scalar;
+
   typedef TensorContractionInputMapper<
       Scalar, Index, Side,
       TensorEvaluator<
@@ -79,6 +81,7 @@ class TensorContractionInputMapper<
       nocontract_t, contract_t, packet_size, inner_dim_contiguous,
       inner_dim_reordered, Alignment>
       Self;
+
   typedef TensorContractionSubMapper<
       Scalar, Index, Side,
       TensorEvaluator<
@@ -88,6 +91,7 @@ class TensorContractionInputMapper<
       nocontract_t, contract_t, packet_size, inner_dim_contiguous,
       inner_dim_reordered, Alignment>
       SubMapper;
+
   typedef SubMapper VectorMapper;
   typedef SubMapper LinearMapper;
   typedef typename packet_traits<Scalar>::type Packet;
@@ -533,6 +537,7 @@ class TensorContractionSubMapper<
       nocontract_t, contract_t, packet_size, inner_dim_contiguous,
       inner_dim_reordered, Alignment>
       ParentMapper;
+
   typedef TensorContractionSubMapper<
       Scalar, Index, Side,
       TensorEvaluator<
@@ -542,6 +547,7 @@ class TensorContractionSubMapper<
       nocontract_t, contract_t, packet_size, inner_dim_contiguous,
       inner_dim_reordered, Alignment>
       Self;
+
   typedef Self LinearMapper;
 
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionSubMapper(
@@ -578,7 +584,6 @@ class TensorContractionSubMapper<
     return m_base_mapper.template loadPacket<Alignment>(i + m_depth_offset,
                                                         j + m_col_offset);
   }
-
   EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar
   loadCoeffStandard(Index i) const {
     return m_base_mapper.loadCoeffStandard(i + m_depth_offset, m_rowIndex,
@@ -611,18 +616,29 @@ class TensorContractionSubMapper<
   EIGEN_DEVICE_FUNC
   EIGEN_ALWAYS_INLINE Index maxCol(const Index peeled_k) const {
     const Index max_col =
-        fastPatchColStride().divide(m_depth_offset + peeled_k);
+        (m_depth_offset + (peeled_k == 0 ? 0 : peeled_k - 1)) /
+        fastPatchColStride();
     return std::min<Index>(1 + max_col, patchCols());
   }
 
   EIGEN_DEVICE_FUNC
   EIGEN_ALWAYS_INLINE Index maxRow(const Index peeled_k,
                                    const Index col) const {
-    const Index max_row = fastPatchRowStride().divide(
-        m_depth_offset + peeled_k - col * patchColStride());
+    const Index max_row = (m_depth_offset + (peeled_k == 0 ? 0 : peeled_k - 1) -
+                           col * patchColStride()) /
+                          fastPatchRowStride();
     return std::min<Index>(1 + max_row, patchRows());
   }
 
+  EIGEN_DEVICE_FUNC
+  EIGEN_ALWAYS_INLINE Index maxDepth(const Index peeled_k, const Index col,
+                                     Index row) const {
+    const Index max_depth = m_depth_offset + peeled_k -  //
+                            col * patchColStride() -     //
+                            row * patchRowStride();
+    return std::min<Index>(max_depth, patchDepth());
+  }
+
   // MaxDepth uses only the remaining number of elements in the peeled_k.
   EIGEN_DEVICE_FUNC
   EIGEN_ALWAYS_INLINE Index maxDepth(const Index num_elements,
@@ -692,6 +708,12 @@ class TensorContractionSubMapper<
     return r < 0 || r >= m_base_mapper.m_inputRows;
   }
   EIGEN_DEVICE_FUNC
+  EIGEN_ALWAYS_INLINE bool padAnyRow(const Index first_row,
+                                     const Index last_row) const {
+    return m_rowIndex + first_row < 0 ||
+           m_rowIndex + last_row >= m_base_mapper.m_inputRows;
+  }
+  EIGEN_DEVICE_FUNC
   EIGEN_ALWAYS_INLINE bool padCol(const Index col) const {
     const Index c = m_colIndex + col;
     return c < 0 || c >= m_base_mapper.m_inputCols;
@@ -738,9 +760,6 @@ class TensorContractionSubMapper<
   }
 
  private:
-  const ParentMapper m_base_mapper;  // Keeping a copy instead of a reference
-                                     // performs better in benchmarks.
-
   Index m_depth_offset;  // First row in the input matrix
   Index m_col_offset;    // First col in the input matrix
 
@@ -750,6 +769,9 @@ class TensorContractionSubMapper<
   Index m_rowIndex;
   Index m_colIndex;
   Index m_otherIndex;
+
+  const ParentMapper m_base_mapper;  // Keeping a copy instead of a reference
+                                     // performs better in benchmarks.
 };
 
 // Arrange a block of the right input matrix (in our case it's always a "virtual
@@ -1319,23 +1341,19 @@ struct mkldnn_gemm_pack<
   typedef typename packet_traits<Scalar>::type Packet;
 
   EIGEN_DONT_INLINE
-  void operator()(Scalar* block, const DataMapper& rhs, StorageIndex rows,
+  void operator()(Scalar* block, const DataMapper rhs, StorageIndex rows,
                   StorageIndex cols) {
     const bool standard_patches = !rhs.nonStandardPatches();
 
     if (standard_patches && (rhs.patchDepth() % packet_size == 0)) {
-      if (rhs.rowStride() == 1) {
-        packStandardPatches<true, /*squeeze*/ true>(block, rhs, rows, cols);
-      } else {
-        packStandardPatches<true, /*squeeze*/ false>(block, rhs, rows, cols);
-      }
+      // Single packet always belong to single patch (row, col).
+      packStandardPatches</*patch_depth_is_multiple_of_packet_size*/ true>(
+          block, rhs, rows, cols);
 
     } else if (standard_patches) {
-      if (rhs.rowStride() == 1) {
-        packStandardPatches<false, /*squeeze*/ true>(block, rhs, rows, cols);
-      } else {
-        packStandardPatches<false, /*squeeze*/ false>(block, rhs, rows, cols);
-      }
+      // Single packet can span across multiple patch rows or columns.
+      packStandardPatches</*patch_depth_is_multiple_of_packet_size*/ false>(
+          block, rhs, rows, cols);
 
     } else {
       // With non-standard patches we don't do any vectorized loads.
@@ -1357,72 +1375,64 @@ struct mkldnn_gemm_pack<
   // - patch_depth_is_multiple_of_packet_size=true: We are guaranteed to have
   //   depth dimension size to be a multiple of packet size, so we can skip all
   //   non vectorized loads and checks.
-  //
-  // - squeeze_reads=true: If stride along the `row` dimension is `1`, we can
-  //   squeeze reads along the `row` and `depth` dimensions, because they are
-  //   guaranteed to be contiguous in memory (two innermost dimensions).
-  //
-  template <bool patch_depth_is_multiple_of_packet_size, bool squeeze_reads>
+  template <bool patch_depth_is_multiple_of_packet_size>
   EIGEN_ALWAYS_INLINE void packStandardPatches(Scalar* block,
-                                               const DataMapper& rhs,
+                                               const DataMapper rhs,
                                                StorageIndex rows,
                                                StorageIndex cols) {
     eigen_assert(!rhs.nonStandardPatches());
 
     // Give vectorized_rows the name used in all other gemm_pack_rhs above.
-    const Index peeled_k = (rows / packet_size) * packet_size;
+    const StorageIndex peeled_k = (rows / packet_size) * packet_size;
 
-    const Index start_col = rhs.colOffset();
-    const Index max_col = rhs.maxCol(peeled_k);
+    const StorageIndex start_col = rhs.colOffset();
+    const StorageIndex max_col = rhs.maxCol(peeled_k);
 
     for (StorageIndex col = 0; col < cols; ++col) {
       SubMapper lm = rhs.getLinearMapper(0, col);
 
-      Index k = 0;
+      StorageIndex k = 0;
       for (Index c = start_col; c < max_col; ++c) {
         eigen_assert(k <= peeled_k);
 
-        const Index start_row = (c == start_col) ? rhs.rowOffset() : 0;
-        const Index max_row = rhs.maxRow(peeled_k, c);
+        const StorageIndex start_row = (c == start_col) ? rhs.rowOffset() : 0;
+        const StorageIndex max_row = rhs.maxRow(peeled_k, c);
         const bool pad_col = lm.padCol(c);
 
         // We can squeeze reads for all rows in [start_row, max_row) range.
-        if (squeeze_reads && !pad_col && !lm.padRow(start_row) &&
-            !lm.padRow(max_row - 1)) {
-          const Index start_depth = (c == start_col) ? rhs.depthOffset() : 0;
-
-          // Upper bound on the number of elements in the depth dimension that
-          // we can squeeze read.
-          const Index squeeze_length =
-              (max_row - start_row) * rhs.patchDepth() - start_depth;
+        if (!pad_col && !lm.padAnyRow(start_row, max_row - 1)) {
+          const StorageIndex start_depth =
+              (c == start_col) ? rhs.depthOffset() : 0;
 
-          // Do not overshoot beyond the block size.
-          const Index max_depth =
-              start_depth + std::min<Index>(peeled_k - k, squeeze_length);
+          const StorageIndex max_depth =
+              std::min<StorageIndex>(start_depth + (peeled_k - k),
+                                     (max_row - start_row) * rhs.patchDepth());
 
-          const Index base_idx = lm.baseIndex(start_row, c);
+          const StorageIndex base_idx = lm.baseIndex(start_row, c);
 
-          if (patch_depth_is_multiple_of_packet_size)
+          if (patch_depth_is_multiple_of_packet_size) {
+            // If patch depth is a multiple of packet size, it's guaranteed that
+            // we can process all values in depth dimension with packets.
             eigen_assert((max_depth - start_depth) % packet_size == 0);
+            StorageIndex d = start_depth;
 
-          // If patch depth is a multiple of packet size, it's guaranteed that
-          // we can process all values in depth dimension with packets.
-          const Index max_vectorized_depth =
-              patch_depth_is_multiple_of_packet_size ? max_depth
-                                                     : max_depth - packet_size;
-
-          Index d = start_depth;
+            for (; d < max_depth; d += packet_size) {
+              eigen_assert(k < peeled_k);
+              internal::pstoreu(block, rhs.packetNoPadding(d, base_idx));
+              block += packet_size;
+              k += packet_size;
+            }
 
-          // 1. Process depth dimension with vectorized instructions.
-          for (; d < max_vectorized_depth; d += packet_size) {
-            eigen_assert(k < peeled_k);
-            internal::pstoreu(block, rhs.packetNoPadding(d, base_idx));
-            block += packet_size;
-            k += packet_size;
-          }
+          } else {
+            StorageIndex d = start_depth;
+            const StorageIndex vectorized_depth = max_depth - packet_size;
 
-          // 2. Finish with coefficients.
-          if (!patch_depth_is_multiple_of_packet_size) {
+            for (; d <= vectorized_depth; d += packet_size) {
+              eigen_assert(k < peeled_k);
+              internal::pstoreu(block, rhs.packetNoPadding(d, base_idx));
+              block += packet_size;
+              k += packet_size;
+            }
             for (; d < max_depth; d++) {
               eigen_assert(k < peeled_k);
               *block = rhs.coeffNoPadding(d, base_idx);
@@ -1437,39 +1447,43 @@ struct mkldnn_gemm_pack<
 
         // If we are not allowed to squeeze reads along the `row` and `depth`
         // dimensions, we must process rows one by one.
-        for (Index r = start_row; r < max_row; ++r) {
+        for (StorageIndex r = start_row; r < max_row; ++r) {
           eigen_assert(k <= peeled_k);
 
-          const Index start_depth =
+          const StorageIndex start_depth =
               ((c == start_col) && (r == start_row)) ? rhs.depthOffset() : 0;
-          const Index max_depth = rhs.maxDepth(peeled_k - k, start_depth);
+          const StorageIndex max_depth =
+              rhs.maxDepth(peeled_k - k, start_depth);
 
           const bool pad = pad_col || lm.padRow(r);
-          const Index base_idx = lm.baseIndex(r, c);
+          const StorageIndex base_idx = lm.baseIndex(r, c);
 
-          if (patch_depth_is_multiple_of_packet_size)
+          if (patch_depth_is_multiple_of_packet_size) {
+            // If patch depth is a multiple of packet size, it's guaranteed that
+            // we can process all values in depth dimension with packets.
             eigen_assert((max_depth - start_depth) % packet_size == 0);
+            StorageIndex d = start_depth;
 
-          // If patch depth is a multiple of packet size, it's guaranteed that
-          // we can process all values in depth dimension with packets.
-          const Index max_vectorized_depth =
-              patch_depth_is_multiple_of_packet_size ? max_depth
-                                                     : max_depth - packet_size;
-
-          Index d = start_depth;
-
-          // 1. Process depth dimension with vectorized instructions.
-          for (; d < max_vectorized_depth; d += packet_size) {
-            eigen_assert(k < peeled_k);
-            const Packet p = pad ? pset1<Packet>(Scalar(0))
-                                 : rhs.packetNoPadding(d, base_idx);
-            internal::pstoreu(block, p);
-            block += packet_size;
-            k += packet_size;
-          }
+            for (; d < max_depth; d += packet_size) {
+              eigen_assert(k < peeled_k);
+              const Packet p = pad ? pset1<Packet>(Scalar(0))
+                                   : rhs.packetNoPadding(d, base_idx);
+              internal::pstoreu(block, p);
+              block += packet_size;
+              k += packet_size;
+            }
 
-          // 2. Finish with coefficients.
-          if (!patch_depth_is_multiple_of_packet_size) {
+          } else {
+            const StorageIndex max_vectorized_depth = max_depth - packet_size;
+            StorageIndex d = start_depth;
+            for (; d < max_vectorized_depth; d += packet_size) {
+              eigen_assert(k < peeled_k);
+              const Packet p = pad ? pset1<Packet>(Scalar(0))
+                                   : rhs.packetNoPadding(d, base_idx);
+              internal::pstoreu(block, p);
+              block += packet_size;
+              k += packet_size;
+            }
             for (; d < max_depth; d++) {
               eigen_assert(k < peeled_k);
               *block = pad ? Scalar(0) : rhs.coeffNoPadding(d, base_idx);
diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
index 8219fc9025..22f71d6260 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
@@ -1380,7 +1380,12 @@ static void PackRhsHelper(int iters,
                           /* Filter (kernel) dimensions: */
                           int filter_count, int filter_cols, int filter_rows,
                           /* Input strides: */
-                          int col_strides, int row_strides) {
+                          int col_strides, int row_strides,
+                          /* Block dimensions: */
+                          Index block_rows, Index block_cols) {
+  // Set random seed for benchmark repeatability.
+  srand(12345);
+
   tensorflow::testing::UseRealTime();
   tensorflow::testing::StopTiming();
 
@@ -1508,10 +1513,6 @@ static void PackRhsHelper(int iters,
 
   PackRhsImpl pack_rhs;
 
-  // This is the typical size of the rhs block used in Tensor contractions.
-  const Index default_depth = 320;  // must be multiple of 8
-  const Index default_cols = 280;
-
   const Index packed_total_size = input_dims.TotalSize();
 
   tensorflow::testing::StartTiming();
@@ -1520,11 +1521,14 @@ static void PackRhsHelper(int iters,
         num_inputs == 1 ? 1 : internal::random<int>(0, num_inputs - 1);
 
     // Depth offset must be a multiple of 8 (float packet size with AVX2).
-    Index depth_offset = (internal::random<Index>(0, patch_size - 10) / 8) * 8;
+    Index depth_offset =
+        (patch_size > block_rows)
+            ? (internal::random<Index>(0, patch_size - 10) / 8) * 8
+            : 0;
     Index col_offset = internal::random<Index>(0, num_patches - 10);
 
-    Index depth = std::min(default_depth, patch_size - depth_offset);
-    Index cols = std::min(default_cols, num_patches - col_offset);
+    Index depth = std::min(block_rows, patch_size - depth_offset);
+    Index cols = std::min(block_cols, num_patches - col_offset);
 
     // Write packed data to random memory location to emulate cold caches.
     Index packed_size = depth * cols;
@@ -1538,20 +1542,37 @@ static void PackRhsHelper(int iters,
   tensorflow::testing::StopTiming();
 
   std::ostringstream stringStream;
-  stringStream << "patch: depth=" << patch_depth << " rows=" << patch_rows
-               << " cols=" << patch_cols << " num_patches=" << num_patches
+  stringStream << "patch: " << patch_rows << "x" << patch_cols << " D"
+               << patch_depth << "; num_patches=" << num_patches
                << " patch_size=" << patch_size << " num_inputs=" << num_inputs;
   tensorflow::testing::SetLabel(stringStream.str());
 }
 
-#define BM_NAME(prefix, N, H, W, C, FC, FH, FW, SH, SW) \
-  BM_##prefix##_##N##_##H##x##W##_IC##C##_FC##FC##_##FH##x##FW##_s##SH##x##SW
-
-#define BM_PackRhs(N, H, W, C, FC, FH, FW, SH, SW)                          \
-  static void BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW)(int iters) { \
-    PackRhsHelper(iters, N, H, W, C, FC, FH, FW, SH, SW);                   \
-  }                                                                         \
-  BENCHMARK(BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW))
+// -------------------------------------------------------------------------- //
+// Macro argumentnames:
+//    N: batch size
+//    H: height
+//    W: width
+//    C: input channels
+//   FC: filter channles
+//   FH: filter height
+//   SH: stride in height dimensions
+//   SW: stride in width dimensions
+//   BR: block rows
+//   BC: block cols
+
+#define BM_CONCAT(a, b) a##b
+
+#define BM_NAME(prefix, N, H, W, C, FC, FH, FW, SH, SW, BR, BC)           \
+  BM_CONCAT(BM_##prefix##_##N##_##H##x##W##_IC##C##_FC##FC##_##FH##x##FW, \
+            _s##SH##x##SW##_B##BR##x##BC)
+
+#define BM_PackRhs(N, H, W, C, FC, FH, FW, SH, SW, BR, BC)         \
+  static void BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, \
+                      BC)(int iters) {                             \
+    PackRhsHelper(iters, N, H, W, C, FC, FH, FW, SH, SW, BR, BC);  \
+  }                                                                \
+  BENCHMARK(BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, BC))
 
 // Number of input channel (input depth) it equal to the number of patch
 // channels (patch depth).
@@ -1563,14 +1584,16 @@ BM_PackRhs(/*batch*/ 32,        //
            /*channels*/ 32,     //
            /*num_filters*/ 64,  //
            /*filter*/ 5, 5,     //
-           /*stride*/ 1, 1);
+           /*stride*/ 1, 1,     //
+           /*block*/ 256, 56);
 
 BM_PackRhs(/*batch*/ 32,        //
            /*image*/ 64, 64,    //
            /*channels*/ 32,     //
            /*num_filters*/ 64,  //
            /*filter*/ 5, 5,     //
-           /*stride*/ 2, 2);
+           /*stride*/ 2, 2,     //
+           /*block*/ 256, 56);
 
 // Slow path: input channel dimension is not the multiple of the packet size.
 BM_PackRhs(/*batch*/ 32,        //
@@ -1578,12 +1601,48 @@ BM_PackRhs(/*batch*/ 32,        //
            /*channels*/ 30,     //
            /*num_filters*/ 64,  //
            /*filter*/ 5, 5,     //
-           /*stride*/ 1, 1);
+           /*stride*/ 1, 1,     //
+           /*block*/ 256, 56);
 
 BM_PackRhs(/*batch*/ 32,        //
            /*image*/ 64, 64,    //
            /*channels*/ 30,     //
            /*num_filters*/ 64,  //
            /*filter*/ 5, 5,     //
-           /*stride*/ 2, 2);
+           /*stride*/ 2, 2,     //
+           /*block*/ 256, 56);
+
+// Slow path with input channel dimension smaller than the packet size.
+BM_PackRhs(/*batch*/ 32,        //
+           /*image*/ 256, 256,  //
+           /*channels*/ 4,      //
+           /*num_filters*/ 16,  //
+           /*filter*/ 8, 8,     //
+           /*stride*/ 1, 1,     //
+           /*block*/ 256, 56);
+
+BM_PackRhs(/*batch*/ 32,        //
+           /*image*/ 256, 256,  //
+           /*channels*/ 4,      //
+           /*num_filters*/ 16,  //
+           /*filter*/ 8, 8,     //
+           /*stride*/ 2, 4,     //
+           /*block*/ 256, 56);
+
+// Short and wide block with small input channel dimension.
+BM_PackRhs(/*batch*/ 32,        //
+           /*image*/ 64, 64,    //
+           /*channels*/ 4,      //
+           /*num_filters*/ 16,  //
+           /*filter*/ 3, 3,     //
+           /*stride*/ 1, 1,     //
+           /*block*/ 36, 432);
+
+BM_PackRhs(/*batch*/ 32,        //
+           /*image*/ 64, 64,    //
+           /*channels*/ 4,      //
+           /*num_filters*/ 16,  //
+           /*filter*/ 3, 3,     //
+           /*stride*/ 2, 2,     //
+           /*block*/ 36, 432);
 }  // namespace Eigen
-- 
GitLab


From f4b03c1c82906566852a117c2e452b0f7798874f Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 26 Nov 2018 12:44:55 -0800
Subject: [PATCH 0792/1554] Add negative offset to filter to properly test Relu
 activation.

PiperOrigin-RevId: 222869970
---
 tensorflow/core/kernels/conv_ops_test.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 252467f27c..ef27dfec54 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -707,8 +707,10 @@ class FusedConv2DOpTest : public OpsTestBase {
     Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
     image.flat<T>() = image.flat<T>().setRandom();
 
+    // Add some negative values to filter to properly test Relu.
     Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
     filter.flat<T>() = filter.flat<T>().setRandom();
+    filter.flat<T>() -= filter.flat<T>().constant(static_cast<T>(0.5f));
 
     const int bias_size = filter_count;
     Tensor bias(dtype, {bias_size});
@@ -744,8 +746,10 @@ class FusedConv2DOpTest : public OpsTestBase {
     Tensor image(dtype, {image_batch_count, image_height, image_width, depth});
     image.flat<T>() = image.flat<T>().setRandom();
 
+    // Add some negative values to filter to properly test Relu.
     Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
     filter.flat<T>() = filter.flat<T>().setRandom();
+    filter.flat<T>() -= filter.flat<T>().constant(static_cast<T>(0.5f));
 
     const int scale_size = filter_count;
 
-- 
GitLab


From e67c4e41c8d787240813a5533cb9a31465fb9ac6 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Mon, 26 Nov 2018 13:10:27 -0800
Subject: [PATCH 0793/1554] ArgMax custom call kernel should support negative
 dim argument

PiperOrigin-RevId: 222874198
---
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index 42bf4b06e5..dce9641f63 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -50,21 +50,24 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
     // overhead, when compiling ahead-of-time.
     int64 dim;
     OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &dim));
-    OP_REQUIRES(ctx, dim >= 0, errors::InvalidArgument("dim must be >= 0"));
-    OP_REQUIRES(
-        ctx, dim < input_shape.dims(),
-        errors::InvalidArgument("dim must be < input rank (",
-                                input_shape.dims(), "), but got: ", dim));
-    const int64 dim_size = input_shape.dim_size(dim);
-    OP_REQUIRES(ctx, dim_size > 0,
+
+    const int input_dims = input_shape.dims();
+    const int axis = dim < 0 ? dim + input_dims : dim;
+    OP_REQUIRES(ctx, axis >= 0 && axis < input_dims,
+                errors::InvalidArgument("Expected dimension in the range [",
+                                        -input_dims, ", ", input_dims,
+                                        "), but got ", dim));
+
+    const int64 axis_size = input_shape.dim_size(axis);
+    OP_REQUIRES(ctx, axis_size > 0,
                 errors::InvalidArgument(
                     "Reduction axis ", dim,
                     " is empty in shape: ", input_shape.DebugString()));
 
-    // The output shape is the input shape contracted along dim.
+    // The output shape is the input shape contracted along axis.
     TensorShape output_shape;
     for (int d = 0; d < input_shape.dims() - 1; ++d) {
-      output_shape.AddDim(input_shape.dim_size((d < dim) ? d : d + 1));
+      output_shape.AddDim(input_shape.dim_size((d < axis) ? d : d + 1));
     }
 
     // For now we use a custom-call, only for the 1d and 2d cases.
@@ -84,7 +87,7 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
       args.push_back(xla::ConstantLiteral(
           &b, xla::LiteralUtil::CreateR1<int64>(output_shape.dim_sizes())));
       args.push_back(
-          xla::ConstantLiteral(&b, xla::LiteralUtil::CreateR0<int32>(dim)));
+          xla::ConstantLiteral(&b, xla::LiteralUtil::CreateR0<int32>(axis)));
     }
 
     // The argmax function expects row-major layout.
-- 
GitLab


From 9e98765f10503ca9f4a60373966705345db39d39 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 13:14:14 -0800
Subject: [PATCH 0794/1554] Mock out calls to time.time() and time.sleep() in
 basic_session_run_hooks_test.py

This fixes race conditions in the unit tests that seems to be causing some flakiness.

PiperOrigin-RevId: 222874796
---
 .../training/basic_session_run_hooks_test.py  | 166 +++++++++++-------
 1 file changed, 98 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 2d469634e0..3fabb3e086 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import os.path
 import shutil
 import tempfile
-import threading
 import time
 
 from tensorflow.contrib.framework.python.framework import checkpoint_utils
@@ -52,6 +51,11 @@ from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 
 
+# Provide a realistic start time for unit tests where we need to mock out
+# calls to time.time().
+MOCK_START_TIME = 1484695987.209386
+
+
 class MockCheckpointSaverListener(
     basic_session_run_hooks.CheckpointSaverListener):
 
@@ -95,7 +99,9 @@ class SecondOrStepTimerTest(test.TestCase):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SecondOrStepTimer()
 
-  def test_every_secs(self):
+  @test.mock.patch.object(time, 'time')
+  def test_every_secs(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     timer = basic_session_run_hooks.SecondOrStepTimer(every_secs=1.0)
     self.assertTrue(timer.should_trigger_for_step(1))
 
@@ -103,7 +109,7 @@ class SecondOrStepTimerTest(test.TestCase):
     self.assertFalse(timer.should_trigger_for_step(1))
     self.assertFalse(timer.should_trigger_for_step(2))
 
-    time.sleep(1.0)
+    mock_time.return_value += 1.0
     self.assertFalse(timer.should_trigger_for_step(1))
     self.assertTrue(timer.should_trigger_for_step(2))
 
@@ -314,7 +320,7 @@ class LoggingTensorHookTest(test.TestCase):
       # in first run, elapsed time is None.
       self.assertEqual(str(self.logged_message).find('sec'), -1)
 
-  def _validate_print_every_n_secs(self, sess, at_end):
+  def _validate_print_every_n_secs(self, sess, at_end, mock_time):
     t = constant_op.constant(42.0, name='foo')
     train_op = constant_op.constant(3)
 
@@ -331,7 +337,7 @@ class LoggingTensorHookTest(test.TestCase):
     self.logged_message = ''
     mon_sess.run(train_op)
     self.assertEqual(str(self.logged_message).find(t.name), -1)
-    time.sleep(1.0)
+    mock_time.return_value += 1.0
 
     self.logged_message = ''
     mon_sess.run(train_op)
@@ -345,17 +351,21 @@ class LoggingTensorHookTest(test.TestCase):
       # assertNotRegexpMatches is not supported by python 3.1 and later
       self.assertEqual(str(self.logged_message).find(t.name), -1)
 
-  def test_print_every_n_secs(self):
+  @test.mock.patch.object(time, 'time')
+  def test_print_every_n_secs(self, mock_time):
     with ops.Graph().as_default(), session_lib.Session() as sess:
-      self._validate_print_every_n_secs(sess, at_end=False)
+      mock_time.return_value = MOCK_START_TIME
+      self._validate_print_every_n_secs(sess, at_end=False, mock_time=mock_time)
       # Verify proper reset.
-      self._validate_print_every_n_secs(sess, at_end=False)
+      self._validate_print_every_n_secs(sess, at_end=False, mock_time=mock_time)
 
-  def test_print_every_n_secs_and_end(self):
+  @test.mock.patch.object(time, 'time')
+  def test_print_every_n_secs_and_end(self, mock_time):
     with ops.Graph().as_default(), session_lib.Session() as sess:
-      self._validate_print_every_n_secs(sess, at_end=True)
+      mock_time.return_value = MOCK_START_TIME
+      self._validate_print_every_n_secs(sess, at_end=True, mock_time=mock_time)
       # Verify proper reset.
-      self._validate_print_every_n_secs(sess, at_end=True)
+      self._validate_print_every_n_secs(sess, at_end=True, mock_time=mock_time)
 
   def test_print_formatter(self):
     with ops.Graph().as_default(), session_lib.Session() as sess:
@@ -562,11 +572,8 @@ class CheckpointSaverHookTest(test.TestCase):
 
   @test.mock.patch.object(time, 'time')
   def test_save_secs_saves_periodically(self, mock_time):
-    # Let's have a realistic start time
-    current_time = 1484695987.209386
-
     with self.graph.as_default():
-      mock_time.return_value = current_time
+      mock_time.return_value = MOCK_START_TIME
       hook = basic_session_run_hooks.CheckpointSaverHook(
           self.model_dir, save_secs=2, scaffold=self.scaffold)
       hook.begin()
@@ -576,10 +583,10 @@ class CheckpointSaverHookTest(test.TestCase):
         sess.run(self.scaffold.init_op)
         mon_sess = monitored_session._HookedSession(sess, [hook])
 
-        mock_time.return_value = current_time
+        mock_time.return_value = MOCK_START_TIME
         mon_sess.run(self.train_op)  # Saved.
 
-        mock_time.return_value = current_time + 0.5
+        mock_time.return_value = MOCK_START_TIME + 0.5
         mon_sess.run(self.train_op)  # Not saved.
 
         self.assertEqual(1,
@@ -587,13 +594,13 @@ class CheckpointSaverHookTest(test.TestCase):
                                                         self.global_step.name))
 
         # Simulate 2.5 seconds of sleep.
-        mock_time.return_value = current_time + 2.5
+        mock_time.return_value = MOCK_START_TIME + 2.5
         mon_sess.run(self.train_op)  # Saved.
 
-        mock_time.return_value = current_time + 2.6
+        mock_time.return_value = MOCK_START_TIME + 2.6
         mon_sess.run(self.train_op)  # Not saved.
 
-        mock_time.return_value = current_time + 2.7
+        mock_time.return_value = MOCK_START_TIME + 2.7
         mon_sess.run(self.train_op)  # Not saved.
 
         self.assertEqual(3,
@@ -601,7 +608,7 @@ class CheckpointSaverHookTest(test.TestCase):
                                                         self.global_step.name))
 
         # Simulate 7.5 more seconds of sleep (10 seconds from start.
-        mock_time.return_value = current_time + 10
+        mock_time.return_value = MOCK_START_TIME + 10
         mon_sess.run(self.train_op)  # Saved.
         self.assertEqual(6,
                          checkpoint_utils.load_variable(self.model_dir,
@@ -609,11 +616,8 @@ class CheckpointSaverHookTest(test.TestCase):
 
   @test.mock.patch.object(time, 'time')
   def test_save_secs_calls_listeners_periodically(self, mock_time):
-    # Let's have a realistic start time
-    current_time = 1484695987.209386
-
     with self.graph.as_default():
-      mock_time.return_value = current_time
+      mock_time.return_value = MOCK_START_TIME
       listener = MockCheckpointSaverListener()
       hook = basic_session_run_hooks.CheckpointSaverHook(
           self.model_dir,
@@ -626,28 +630,28 @@ class CheckpointSaverHookTest(test.TestCase):
         sess.run(self.scaffold.init_op)
         mon_sess = monitored_session._HookedSession(sess, [hook])
 
-        mock_time.return_value = current_time + 0.5
+        mock_time.return_value = MOCK_START_TIME + 0.5
         mon_sess.run(self.train_op)  # hook runs here
 
-        mock_time.return_value = current_time + 0.5
+        mock_time.return_value = MOCK_START_TIME + 0.5
         mon_sess.run(self.train_op)
 
-        mock_time.return_value = current_time + 3.0
+        mock_time.return_value = MOCK_START_TIME + 3.0
         mon_sess.run(self.train_op)  # hook runs here
 
-        mock_time.return_value = current_time + 3.5
+        mock_time.return_value = MOCK_START_TIME + 3.5
         mon_sess.run(self.train_op)
 
-        mock_time.return_value = current_time + 4.0
+        mock_time.return_value = MOCK_START_TIME + 4.0
         mon_sess.run(self.train_op)
 
-        mock_time.return_value = current_time + 6.5
+        mock_time.return_value = MOCK_START_TIME + 6.5
         mon_sess.run(self.train_op)  # hook runs here
 
-        mock_time.return_value = current_time + 7.0
+        mock_time.return_value = MOCK_START_TIME + 7.0
         mon_sess.run(self.train_op)  # hook won't run here, so it does at end
 
-        mock_time.return_value = current_time + 7.5
+        mock_time.return_value = MOCK_START_TIME + 7.5
         hook.end(sess)  # hook runs here
       self.assertEqual({
           'begin': 1,
@@ -913,7 +917,9 @@ class StepCounterHookTest(test.TestCase):
   def tearDown(self):
     shutil.rmtree(self.log_dir, ignore_errors=True)
 
-  def test_step_counter_every_n_steps(self):
+  @test.mock.patch.object(time, 'time')
+  def test_step_counter_every_n_steps(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     with ops.Graph().as_default() as g, session_lib.Session() as sess:
       variables.get_or_create_global_step()
       train_op = training_util._increment_global_step(1)
@@ -925,7 +931,7 @@ class StepCounterHookTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(sess, [hook])
       with test.mock.patch.object(tf_logging, 'warning') as mock_log:
         for _ in range(30):
-          time.sleep(0.01)
+          mock_time.return_value += 0.01
           mon_sess.run(train_op)
         # logging.warning should not be called.
         self.assertIsNone(mock_log.call_args)
@@ -941,7 +947,9 @@ class StepCounterHookTest(test.TestCase):
         self.assertEqual('global_step/sec', summary_value.tag)
         self.assertGreater(summary_value.simple_value, 0)
 
-  def test_step_counter_every_n_secs(self):
+  @test.mock.patch.object(time, 'time')
+  def test_step_counter_every_n_secs(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     with ops.Graph().as_default() as g, session_lib.Session() as sess:
       variables.get_or_create_global_step()
       train_op = training_util._increment_global_step(1)
@@ -953,9 +961,9 @@ class StepCounterHookTest(test.TestCase):
       sess.run(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
-      time.sleep(0.2)
+      mock_time.return_value += 0.2
       mon_sess.run(train_op)
-      time.sleep(0.2)
+      mock_time.return_value += 0.2
       mon_sess.run(train_op)
       hook.end(sess)
 
@@ -1037,13 +1045,15 @@ class StepCounterHookTest(test.TestCase):
     sess.run(variables_lib.global_variables_initializer())
     self.mon_sess = monitored_session._HookedSession(sess, [self.hook])
 
-  def test_steps_per_run_less_than_every_n_steps(self):
+  @test.mock.patch.object(time, 'time')
+  def test_steps_per_run_less_than_every_n_steps(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     with ops.Graph().as_default() as g, session_lib.Session() as sess:
       self._setup_steps_per_run_test(10, 5, g, sess)
 
       # Logs at 15, 25
       for _ in range(5):
-        time.sleep(0.01)
+        mock_time.return_value += 0.01
         self.mon_sess.run(self.train_op)
 
       self.hook.end(sess)
@@ -1058,13 +1068,15 @@ class StepCounterHookTest(test.TestCase):
         self.assertEqual('global_step/sec', summary_value.tag)
         self.assertGreater(summary_value.simple_value, 0)
 
-  def test_steps_per_run_equal_every_n_steps(self):
+  @test.mock.patch.object(time, 'time')
+  def test_steps_per_run_equal_every_n_steps(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     with ops.Graph().as_default() as g, session_lib.Session() as sess:
       self._setup_steps_per_run_test(5, 5, g, sess)
 
       # Logs at 10, 15, 20, 25
       for _ in range(5):
-        time.sleep(0.01)
+        mock_time.return_value += 0.01
         self.mon_sess.run(self.train_op)
 
       self.hook.end(sess)
@@ -1080,13 +1092,15 @@ class StepCounterHookTest(test.TestCase):
         self.assertEqual('global_step/sec', summary_value.tag)
         self.assertGreater(summary_value.simple_value, 0)
 
-  def test_steps_per_run_greater_than_every_n_steps(self):
+  @test.mock.patch.object(time, 'time')
+  def test_steps_per_run_greater_than_every_n_steps(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     with ops.Graph().as_default() as g, session_lib.Session() as sess:
       self._setup_steps_per_run_test(5, 10, g, sess)
 
       # Logs at 20, 30, 40, 50
       for _ in range(5):
-        time.sleep(0.01)
+        mock_time.return_value += 0.01
         self.mon_sess.run(self.train_op)
 
       self.hook.end(sess)
@@ -1199,7 +1213,9 @@ class SummarySaverHookTest(test.TestCase):
             },
         })
 
-  def test_save_secs_saving_once_every_step(self):
+  @test.mock.patch.object(time, 'time')
+  def test_save_secs_saving_once_every_step(self, mock_time):
+    mock_time.return_value = MOCK_START_TIME
     hook = basic_session_run_hooks.SummarySaverHook(
         save_secs=0.5,
         summary_writer=self.summary_writer,
@@ -1211,7 +1227,7 @@ class SummarySaverHookTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(4):
         mon_sess.run(self.train_op)
-        time.sleep(0.5)
+        mock_time.return_value += 0.5
       hook.end(sess)
 
     self.summary_writer.assert_summaries(
@@ -1279,27 +1295,43 @@ class GlobalStepWaiterHookTest(test.TestCase):
             session_run_hook.SessionRunContext(
                 original_args=None, session=sess))
 
-  def test_wait_for_step(self):
+  @test.mock.patch.object(time, 'sleep')
+  def test_wait_for_step(self, mock_sleep):
     with ops.Graph().as_default():
       gstep = variables.get_or_create_global_step()
       hook = basic_session_run_hooks.GlobalStepWaiterHook(wait_until_step=1000)
       hook.begin()
+
       with session_lib.Session() as sess:
+        # Mock out calls to time.sleep() to update the global step.
+
+        class Context(object):
+          counter = 0
+
+        def mock_sleep_side_effect(seconds):
+          del seconds  # argument is ignored
+          Context.counter += 1
+          if Context.counter == 1:
+            # The first time sleep() is called, we update the global_step from
+            # 0 to 500.
+            sess.run(state_ops.assign(gstep, 500))
+          elif Context.counter == 2:
+            # The second time sleep() is called, we update the global_step from
+            # 500 to 1100.
+            sess.run(state_ops.assign(gstep, 1100))
+          else:
+            raise AssertionError(
+                'Expected before_run() to terminate after the second call to '
+                'time.sleep()')
+
+        mock_sleep.side_effect = mock_sleep_side_effect
+
+        # Run the mocked-out interaction with the hook.
         sess.run(variables_lib.global_variables_initializer())
-        waiter = threading.Thread(
-            target=hook.before_run,
-            args=(session_run_hook.SessionRunContext(
-                original_args=None, session=sess),))
-        waiter.daemon = True
-        waiter.start()
-        time.sleep(1.0)
-        self.assertTrue(waiter.is_alive())
-        sess.run(state_ops.assign(gstep, 500))
-        time.sleep(1.0)
-        self.assertTrue(waiter.is_alive())
-        sess.run(state_ops.assign(gstep, 1100))
-        time.sleep(1.2)
-        self.assertFalse(waiter.is_alive())
+        run_context = session_run_hook.SessionRunContext(
+            original_args=None, session=sess)
+        hook.before_run(run_context)
+        self.assertEqual(Context.counter, 2)
 
 
 class FinalOpsHookTest(test.TestCase):
@@ -1465,29 +1497,27 @@ class ProfilerHookTest(test.TestCase):
   @test.mock.patch.object(time, 'time')
   def test_save_secs_saves_periodically(self, mock_time):
     # Pick a fixed start time.
-    current_time = 1484863632.
-
     with self.graph.as_default():
-      mock_time.return_value = current_time
+      mock_time.return_value = MOCK_START_TIME
       hook = basic_session_run_hooks.ProfilerHook(
           save_secs=2, output_dir=self.output_dir)
       with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
         sess.run(self.train_op)  # Not saved.
         self.assertEqual(0, self._count_timeline_files())
         # Simulate 2.5 seconds of sleep.
-        mock_time.return_value = current_time + 2.5
+        mock_time.return_value = MOCK_START_TIME + 2.5
         sess.run(self.train_op)  # Saved.
         self.assertEqual(1, self._count_timeline_files())
 
         # Pretend some small amount of time has passed.
-        mock_time.return_value = current_time + 2.6
+        mock_time.return_value = MOCK_START_TIME + 2.6
         sess.run(self.train_op)  # Not saved.
         # Edge test just before we should save the timeline.
-        mock_time.return_value = current_time + 4.4
+        mock_time.return_value = MOCK_START_TIME + 4.4
         sess.run(self.train_op)  # Not saved.
         self.assertEqual(1, self._count_timeline_files())
 
-        mock_time.return_value = current_time + 4.5
+        mock_time.return_value = MOCK_START_TIME + 4.5
         sess.run(self.train_op)  # Saved.
         self.assertEqual(2, self._count_timeline_files())
 
-- 
GitLab


From 7d1710c55c552abe19ebbc5e999521678355b3aa Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Mon, 26 Nov 2018 13:23:56 -0800
Subject: [PATCH 0795/1554] Update TensorFlow Lite Converter documentation.

PiperOrigin-RevId: 222876388
---
 tensorflow/lite/g3doc/convert/index.md      |  14 +-
 tensorflow/lite/g3doc/convert/python_api.md |  29 ++--
 tensorflow/lite/g3doc/devguide.md           | 179 +++++++++-----------
 3 files changed, 105 insertions(+), 117 deletions(-)

diff --git a/tensorflow/lite/g3doc/convert/index.md b/tensorflow/lite/g3doc/convert/index.md
index bc92a1c1a1..60fa265c29 100644
--- a/tensorflow/lite/g3doc/convert/index.md
+++ b/tensorflow/lite/g3doc/convert/index.md
@@ -6,14 +6,20 @@ file used by the TensorFlow Lite interpreter.
 ## From model training to device deployment
 
 After a TensorFlow model is trained, the TensorFlow Lite converter uses that
-model to generate a TensorFlow Lite [FlatBuffer](https://google.github.io/flatbuffers/)
-file (`.tflite`). The converter supports as input:
+model to generate a TensorFlow Lite
+[FlatBuffer](https://google.github.io/flatbuffers/) file (`.tflite`). The
+converter supports as input:
 [SavedModels](https://www.tensorflow.org/guide/saved_model#using_savedmodel_with_estimators),
 frozen graphs (models generated by
 [freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)),
-and `tf.keras` models.  The TensorFlow Lite `FlatBuffer` file is deployed to a
-client device (generally a mobile or embedded device), and the TensorFlow Lite
+and `tf.keras` HDF5 models. The TensorFlow Lite `FlatBuffer` file is deployed to
+a client device (generally a mobile or embedded device), and the TensorFlow Lite
 interpreter uses the compressed model for on-device inference. This conversion
 process is shown in the diagram below:
 
 ![TFLite converter workflow](../images/convert/workflow.svg)
+
+The TensorFlow Lite Converter can be used either from [Python](python_api.md) or
+from the [command line](cmdline_examples.md). This allows you to integrate the
+conversion step into the model design workflow, ensuring the model is easy to
+convert to a mobile inference graph.
diff --git a/tensorflow/lite/g3doc/convert/python_api.md b/tensorflow/lite/g3doc/convert/python_api.md
index 4bdf0d8cbe..b914a34fa8 100644
--- a/tensorflow/lite/g3doc/convert/python_api.md
+++ b/tensorflow/lite/g3doc/convert/python_api.md
@@ -3,10 +3,9 @@
 This page provides examples on how to use the TensorFlow Lite Converter and the
 TensorFlow Lite interpreter using the Python API.
 
-Note: TFLite recently moved from `tf.contrib.lite` to `tf.lite`. If you are
-using tensorflow `r1.12` or earlier you will need to add `.contrib` to the
-commands below. `tf.lite` works with newer builds, like the nightly build,
-which can be installed with: `pip install tf-nightly`
+Note: These docs describe the converter in the TensorFlow nightly release,
+installed using `pip install tf-nightly`. For docs describing older versions
+reference ["Converting models from TensorFlow 1.12"](#pre_tensorflow_1.12).
 
 [TOC]
 
@@ -24,11 +23,6 @@ The API for converting TensorFlow models to TensorFlow Lite as of TensorFlow 1.9
 is `tf.lite.TFLiteConverter`. The API for calling the Python intepreter
 is `tf.lite.Interpreter`.
 
-Note: Reference "Additional Instructions" sections for converting TensorFlow
-models to TensorFlow Lite
-[in TensorFlow 1.9 to TensorFlow 1.11](#pre_tensorflow_1.11) and
-[prior to TensorFlow 1.9](#pre_tensorflow_1.9)
-
 `TFLiteConverter` provides class methods based on the original format of the
 model. `TFLiteConverter.from_session()` is available for GraphDefs.
 `TFLiteConverter.from_saved_model()` is available for SavedModels.
@@ -250,14 +244,13 @@ either install the nightly build with
 [Docker](https://www.tensorflow.org/install/docker), or
 [build the pip package from source](https://www.tensorflow.org/install/source).
 
-### Converting models in TensorFlow 1.9 to TensorFlow 1.11 <a name="pre_tensorflow_1.11"></a>
-
-To convert TensorFlow models to TensorFlow Lite in TensorFlow 1.9 through
-TensorFlow 1.11, use `TocoConverter`. `TocoConverter` is semantically
-identically to `TFLiteConverter`.
+### Converting models from TensorFlow 1.12 <a name="pre_tensorflow_1.12"></a>
 
-### Converting models prior to TensorFlow 1.9 <a name="pre_tensorflow_1.9"></a>
+Reference the following table to convert TensorFlow models to TensorFlow Lite in
+and before TensorFlow 1.12. Run `help()` to get details of each API.
 
-To convert TensorFlow models to TensorFlow Lite in TensorFlow 1.7 and TensorFlow
-1.8, use the `toco_convert` function. Run `help(tf.lite.toco_convert)`
-to get details about accepted parameters.
+TensorFlow Version | Python API
+------------------ | ---------------------------------
+1.12               | `tf.contrib.lite.TFLiteConverter`
+1.9-1.11           | `tf.contrib.lite.TocoConverter`
+1.7-1.8            | `tf.contrib.lite.toco_convert`
diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md
index 270cb8ce37..798bf4996a 100644
--- a/tensorflow/lite/g3doc/devguide.md
+++ b/tensorflow/lite/g3doc/devguide.md
@@ -35,7 +35,7 @@ by suggesting contextually relevant messages. The model is built specifically fo
 memory constrained devices, such as watches and phones, and has been successfully
 used in Smart Replies on Android Wear. Currently, this model is Android-specific.
 
-These pre-trained models are [available for download](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/models.md)
+These pre-trained models are [available for download](models.md).
 
 ### Re-train Inception-V3 or MobileNet for a custom data set
 
@@ -57,51 +57,58 @@ A developer may choose to train a custom model using Tensorflow (see the
 [TensorFlow tutorials](../tutorials/) for examples of building and training
 models). If you have already written a model, the first step is to export this
 to a `tf.GraphDef` file. This is required because some formats do not store the
-model structure outside the code, and we must communicate with other parts of the
-framework. See
-[Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md)
-to create .pb file for the custom model.
+model structure outside the code, and we must communicate with other parts of
+the framework. See
+[Exporting the Inference Graph](https://www.tensorflow.org/tutorials/keras/save_and_restore_models#save_the_entire_model)
+to create file for the custom model.
 
-TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to the
-[TensorFlow Lite & TensorFlow Compatibility Guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc/tf_ops_compatibility.md)
+TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to
+the [TensorFlow Lite & TensorFlow Compatibility Guide](tf_ops_compatibility.md)
 for supported operators and their usage. This set of operators will continue to
 grow in future Tensorflow Lite releases.
 
-
 ## 2. Convert the model format
 
-The model generated (or downloaded) in the previous step is a *standard*
-Tensorflow model and you should now have a .pb or .pbtxt `tf.GraphDef` file.
-Models generated with transfer learning (re-training) or custom models must be
-converted—but, we must first freeze the graph to convert the model to the
-Tensorflow Lite format. This process uses several model formats:
-
-* `tf.GraphDef` (.pb) —A protobuf that represents the TensorFlow training or
-  computation graph. It contains operators, tensors, and variables definitions.
-* *CheckPoint* (.ckpt) —Serialized variables from a TensorFlow graph. Since this
-  does not contain a graph structure, it cannot be interpreted by itself.
-* `FrozenGraphDef` —A subclass of `GraphDef` that does not contain
-  variables. A `GraphDef` can be converted to a `FrozenGraphDef` by taking a
-  CheckPoint and a `GraphDef`, and converting each variable into a constant
-  using the value retrieved from the CheckPoint.
-* `SavedModel` —A `GraphDef` and CheckPoint with a signature that labels
-  input and output arguments to a model. A `GraphDef` and CheckPoint can be
-  extracted from a `SavedModel`.
-* *TensorFlow Lite model* (.tflite) —A serialized
-  [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
-  Lite operators and tensors for the TensorFlow Lite interpreter, similar to a
-  `FrozenGraphDef`.
-
-### Freeze Graph
-
-To use the `GraphDef` .pb file with TensorFlow Lite, you must have checkpoints
-that contain trained weight parameters. The .pb file only contains the structure
-of the graph. The process of merging the checkpoint values with the graph
-structure is called *freezing the graph*.
-
-You should have a checkpoints folder or download them for a pre-trained model
-(for example,
-[MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
+The [TensorFlow Lite Converter](convert) accepts the following file formats:
+
+*   `SavedModel` — A `GraphDef` and checkpoint with a signature that labels
+    input and output arguments to a model. See the documentation for converting
+    SavedModels using [Python](convert/python_api#basic_savedmodel) or using the
+    [command line](convert/cmdline_examples#savedmodel).
+*   `tf.keras` - A HDF5 file containing a model with weights and input and
+    output arguments generated by `tf.Keras`. See [here] for converting HDF5
+    models. See the documentation for converting HDF5 models using
+    [Python](convert/python_api#basic_keras_file) or using the
+    [command line](convert/cmdline_examples#keras).
+*   `frozen tf.GraphDef` — A subclass of `tf.GraphDef` that does not contain
+    variables. A `GraphDef` can be converted to a `frozen GraphDef` by taking a
+    checkpoint and a `GraphDef`, and converting each variable into a constant
+    using the value retrieved from the checkpoint. Instructions on converting a
+    `tf.GraphDef` to a TensorFlow Lite model are described in the next
+    subsection.
+
+### Converting a tf.GraphDef
+
+TensorFlow models may be saved as a .pb or .pbtxt `tf.GraphDef` file. In order
+to convert the `tf.GraphDef` file to TensorFlow Lite, the model must first be
+frozen. This process invovles several file formats including the `frozen
+GraphDef`:
+
+*   `tf.GraphDef` (.pb or .pbtxt) — A protobuf that represents the TensorFlow
+    training or computation graph. It contains operators, tensors, and variables
+    definitions.
+*   *checkpoint* (.ckpt) — Serialized variables from a TensorFlow graph. Since
+    this does not contain a graph structure, it cannot be interpreted by itself.
+*   *TensorFlow Lite model* (.tflite) — A serialized
+    [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
+    Lite operators and tensors for the TensorFlow Lite interpreter.
+
+You must have checkpoints that contain trained weights. The `tf.GraphDef` file
+only contains the structure of the graph. The process of merging the checkpoint
+values with the graph structure is called *freezing the graph*.
+
+`tf.GraphDef` and checkpoint files for MobileNet models are available
+[here](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md).
 
 To freeze the graph, use the following command (changing the arguments):
 
@@ -113,69 +120,52 @@ freeze_graph --input_graph=/tmp/mobilenet_v1_224.pb \
   --output_node_names=MobileNetV1/Predictions/Reshape_1
 ```
 
-The `input_binary` flag must be enabled so the protobuf is read and written in
-a binary format. Set the `input_graph` and `input_checkpoint` files.
+Set the `input_binary` flag to `True` when reading a binary protobuf, a `.pb`
+file. Set to `False` for a `.pbtxt` file.
 
-The `output_node_names` may not be obvious outside of the code that built the
-model. The easiest way to find them is to visualize the graph, either with
-[TensorBoard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3)
-or `graphviz`.
+Set `input_graph` and `input_checkpoint` to the respective filenames. The
+`output_node_names` may not be obvious outside of the code that built the model.
+The easiest way to find them is to visualize the graph, either with
+[TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) or
+`graphviz`.
 
 The frozen `GraphDef` is now ready for conversion to the `FlatBuffer` format
-(.tflite) for use on Android or iOS devices. For Android, the Tensorflow
-Optimizing Converter tool supports both float and quantized models. To convert
-the frozen `GraphDef` to the .tflite format:
+(.tflite) for use on Android or iOS devices. For Android, the TensorFlow Lite
+Converter tool supports both float and quantized models. To convert the frozen
+`GraphDef` to the .tflite format use a command similar to the following:
 
 ```
-toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
-  --input_format=TENSORFLOW_GRAPHDEF \
-  --output_format=TFLITE \
+tflite_convert \
   --output_file=/tmp/mobilenet_v1_1.0_224.tflite \
-  --inference_type=FLOAT \
-  --input_type=FLOAT \
+  --graph_def_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \
   --input_arrays=input \
-  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
-  --input_shapes=1,224,224,3
+  --output_arrays=MobilenetV1/Predictions/Reshape_1
 ```
 
-The `input_file` argument should reference the frozen `GraphDef` file
-containing the model architecture. The [frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)
-file used here is available for download. `output_file` is where the TensorFlow
-Lite model will get generated. The `input_type` and `inference_type`
-arguments should be set to `FLOAT`, unless converting a
-<a href="https://www.tensorflow.org/performance/quantization">quantized model</a>.
-Setting the `input_array`, `output_array`, and `input_shape` arguments are not as
-straightforward. The easiest way to find these values is to explore the graph
-using Tensorboard. Reuse the arguments for specifying the output nodes for
-inference in the `freeze_graph` step.
-
-It is also possible to use the Tensorflow Optimizing Converter with protobufs
-from either Python or from the command line (see the 
-[toco_from_protos.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/toco/python/toco_from_protos.py)
-example). This allows you to integrate the conversion step into the model design
-workflow, ensuring the model is easily convertible to a mobile inference graph.
-For example:
-
-```python
-import tensorflow as tf
-
-img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
-val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
-out = tf.identity(val, name="out")
-
-with tf.Session() as sess:
-  tflite_model = tf.lite.toco_convert(sess.graph_def, [img], [out])
-  open("converteds_model.tflite", "wb").write(tflite_model)
-```
+The
+[frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)
+file used here is available for download. Setting the `input_array` and
+`output_array` arguments is not straightforward. The easiest way to find these
+values is to explore the graph using
+[TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard). Reuse
+the arguments for specifying the output nodes for inference in the
+`freeze_graph` step.
+
+### Full converter reference
 
-For usage, see the Tensorflow Optimizing Converter
-[command-line examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/toco/g3doc/cmdline_examples.md).
+The [TensorFlow Lite Converter](convert/) can be [Python](convert/python_api.md)
+or from the [command line](convert/cmdline_examples.md). This allows you to
+integrate the conversion step into the model design workflow, ensuring the model
+is easy to convert to a mobile inference graph.
 
-Refer to the
-[Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc/tf_ops_compatibility.md)
-for troubleshooting help, and if that doesn't help, please
+### Ops compatibility
+
+Refer to the [ops compatibility guide](tf_ops_compatibility.md) for
+troubleshooting help, and if that doesn't help, please
 [file an issue](https://github.com/tensorflow/tensorflow/issues).
 
+### Graph vizualization tool
+
 The [development repo](https://github.com/tensorflow/tensorflow) contains a tool
 to visualize TensorFlow Lite models after conversion. To build the
 [visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/visualize.py)
@@ -212,8 +202,8 @@ installing TensorFlow on Android and setting up `bazel` and Android Studio.
 ### iOS
 
 To integrate a TensorFlow model in an iOS app, see the
-[TensorFlow Lite for iOS](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc/ios.md)
-guide and <a href="./demo_ios.md">iOS demo</a> guide.
+[TensorFlow Lite for iOS](ios.md) guide and <a href="./demo_ios.md">iOS demo</a>
+guide.
 
 #### Core ML support
 
@@ -227,6 +217,5 @@ devices. To use the converter, refer to the
 ### Raspberry Pi
 
 Compile Tensorflow Lite for a Raspberry Pi by following the
-[RPi build instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/rpi.md)
-This compiles a static library file (`.a`) used to build your app. There are
-plans for Python bindings and a demo app.
+[RPi build instructions](rpi.md) This compiles a static library file (`.a`) used
+to build your app. There are plans for Python bindings and a demo app.
-- 
GitLab


From dbf34e95585a4c1f89bcc041e95752ae7dd0cacc Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 26 Nov 2018 13:34:22 -0800
Subject: [PATCH 0796/1554] Adding `MeanAbsoluteError`,
 `MeanAbsolutePercentageError`, `MeanSquaredLogarithmicError ` losses to the
 new loss module.

PiperOrigin-RevId: 222878057
---
 tensorflow/python/keras/losses.py      | 111 +++++++++++++
 tensorflow/python/keras/losses_test.py | 207 +++++++++++++++++++++++++
 2 files changed, 318 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 0e274d4d50..8afe2fadce 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -133,6 +133,117 @@ class MeanSquaredError(Loss):
     return mean_squared_error(y_true, y_pred)
 
 
+class MeanAbsoluteError(Loss):
+  """Computes the mean of absolute difference between labels and predictions.
+
+  For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
+  then the mean absolute error value is 3/4 (0.75).
+
+  Usage:
+
+  ```python
+  mae = tf.losses.MeanAbsoluteError()
+  loss = mae([0., 0., 1., 1.], [1., 1., 1., 0.])
+  print('Loss: ', loss.numpy())  # Loss: 0.75
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.MeanAbsoluteError())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    """Invokes the `MeanAbsoluteError` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Mean absolute error losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return mean_absolute_error(y_true, y_pred)
+
+
+class MeanAbsolutePercentageError(Loss):
+  """Computes the mean absolute percentage error between `y_true` and `y_pred`.
+
+  For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
+  then the mean absolute percentage error value is 5e+08.
+
+  Usage:
+
+  ```python
+  mape = tf.losses.MeanAbsolutePercentageError()
+  loss = mape([0., 0., 1., 1.], [1., 1., 1., 0.])
+  print('Loss: ', loss.numpy())  # Loss: 5e+08
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.MeanAbsolutePercentageError())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    """Invokes the `MeanAbsolutePercentageError` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Mean absolute percentage error losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return mean_absolute_percentage_error(y_true, y_pred)
+
+
+class MeanSquaredLogarithmicError(Loss):
+  """Computes the mean squared logarithmic error between `y_true` and `y_pred`.
+
+  For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
+  then the mean squared logarithmic error value is 0.36034.
+
+  Usage:
+
+  ```python
+  msle = tf.losses.MeanSquaredLogarithmicError()
+  loss = msle([0., 0., 1., 1.], [1., 1., 1., 0.])
+  print('Loss: ', loss.numpy())  # Loss: 0.36034
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.MeanSquaredLogarithmicError())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    """Invokes the `MeanSquaredLogarithmicError` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Mean squared logarithmic error losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return mean_squared_logarithmic_error(y_true, y_pred)
+
+
 @tf_export('keras.metrics.mean_squared_error',
            'keras.metrics.mse',
            'keras.metrics.MSE',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index b056f920ab..d80b272b12 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -234,5 +234,212 @@ class MeanSquaredErrorTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 227.69998, 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MeanAbsoluteErrorTest(test.TestCase):
+
+  def test_config(self):
+    mae_obj = keras.losses.MeanAbsoluteError(
+        reduction=keras.losses.ReductionV2.SUM, name='mae_1')
+    self.assertEqual(mae_obj.name, 'mae_1')
+    self.assertEqual(mae_obj.reduction, keras.losses.ReductionV2.SUM)
+
+  def test_all_correct_unweighted(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3))
+    loss = mae_obj(y_true, y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mae_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 5.5, 3)
+
+  def test_scalar_weighted(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mae_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 12.65, 3)
+
+  def test_sample_weighted(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = mae_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 81.4 / 6, 3)
+
+  def test_timestep_weighted(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3, 1),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    loss = mae_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 83 / 6, 3)
+
+  def test_zero_weighted(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mae_obj(y_true, y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_invalid_sample_weight(self):
+    mae_obj = keras.losses.MeanAbsoluteError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3], shape=(2, 3, 1))
+    sample_weight = constant_op.constant([3, 6, 5, 0], shape=(2, 2))
+    with self.assertRaisesRegexp(
+        ValueError, r'Shapes \(2, 2\) and \(2, 3\) are incompatible'):
+      mae_obj(y_true, y_pred, sample_weight=sample_weight)
+
+  def test_no_reduction(self):
+    mae_obj = keras.losses.MeanAbsoluteError(
+        reduction=keras.losses.ReductionV2.NONE)
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mae_obj(y_true, y_pred, sample_weight=2.3)
+    loss = self.evaluate(loss)
+    self.assertArrayNear(loss, [10.7333, 14.5666], 1e-3)
+
+  def test_sum_reduction(self):
+    mae_obj = keras.losses.MeanAbsoluteError(
+        reduction=keras.losses.ReductionV2.SUM)
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mae_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 25.29999, 3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class MeanAbsolutePercentageErrorTest(test.TestCase):
+
+  def test_config(self):
+    mape_obj = keras.losses.MeanAbsolutePercentageError(
+        reduction=keras.losses.ReductionV2.SUM, name='mape_1')
+    self.assertEqual(mape_obj.name, 'mape_1')
+    self.assertEqual(mape_obj.reduction, keras.losses.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    mape_obj = keras.losses.MeanAbsolutePercentageError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mape_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 211.8518, 3)
+
+  def test_scalar_weighted(self):
+    mape_obj = keras.losses.MeanAbsolutePercentageError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mape_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 487.259, 3)
+
+  def test_sample_weighted(self):
+    mape_obj = keras.losses.MeanAbsolutePercentageError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = mape_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 422.8888, 3)
+
+  def test_timestep_weighted(self):
+    mape_obj = keras.losses.MeanAbsolutePercentageError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3, 1),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    loss = mape_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 694.4445, 3)
+
+  def test_zero_weighted(self):
+    mape_obj = keras.losses.MeanAbsolutePercentageError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = mape_obj(y_true, y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class MeanSquaredLogarithmicErrorTest(test.TestCase):
+
+  def test_config(self):
+    msle_obj = keras.losses.MeanSquaredLogarithmicError(
+        reduction=keras.losses.ReductionV2.SUM, name='mape_1')
+    self.assertEqual(msle_obj.name, 'mape_1')
+    self.assertEqual(msle_obj.reduction, keras.losses.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    msle_obj = keras.losses.MeanSquaredLogarithmicError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = msle_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 1.4370, 3)
+
+  def test_scalar_weighted(self):
+    msle_obj = keras.losses.MeanSquaredLogarithmicError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = msle_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 3.3051, 3)
+
+  def test_sample_weighted(self):
+    msle_obj = keras.losses.MeanSquaredLogarithmicError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = msle_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 3.7856, 3)
+
+  def test_timestep_weighted(self):
+    msle_obj = keras.losses.MeanSquaredLogarithmicError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3, 1),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    loss = msle_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 2.6473, 3)
+
+  def test_zero_weighted(self):
+    msle_obj = keras.losses.MeanSquaredLogarithmicError()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = msle_obj(y_true, y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 8d71758444bb71ebb5bc6d851e8e1626bc512a8d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 26 Nov 2018 13:39:49 -0800
Subject: [PATCH 0797/1554] Simplify the error message for mismatched function
 source, since now it's an extremely rare possibility.

PiperOrigin-RevId: 222879086
---
 .../python/autograph/impl/conversion.py       | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 48a9307cab..3dfc12eb76 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -281,11 +281,10 @@ def function_to_graph(f,
   node, source = parser.parse_entity(f)
   node = node.body[0]
 
-  # In general, the output of inspect.getsource is inexact because it uses crude
-  # regex matching methods to search the source file. This is particularly
-  # problematic for lambda functions, where the entire containing lines are
-  # returned. Certain distributions of CPython may also return the enclosing
-  # function for local functions.
+  # In general, the output of inspect.getsource is inexact because it uses
+  # regex matching to adjust the exact location around the line number that
+  # CPython records. This is particularly problematic for lambda functions,
+  # where the entire containing lines are returned.
   nodes = ast_util.find_matching_definitions(node, f)
   if len(nodes) != 1:
     if f.__name__ == '<lambda>':
@@ -295,17 +294,11 @@ def function_to_graph(f,
           ' matching signature. To avoid ambiguity, define each lambda'
           ' in a separate expression.'.format(f, source))
     else:
-      # The inspect.getsource bug is currently known to occur in the Windows
-      # integration tests which run Python 3.6.
-      # TODO(mdan): Find out eaxctly which distribution of Python is that.
       raise ValueError(
           'Unable to identify source code of function {}. The source code'
           ' reported by Python did not include exactly one matching signature:'
-          '\n{}\nTo avoid ambiguity, use a unique name for each'
-          ' function.\nNote that some distributions of Python may report source'
-          ' code incorrectly. It may be possible to avoid that bug by'
-          ' organizing the code into smaller units (smaller files, functions or'
-          ' classes), or by turning AutoGraph off.'.format(f, source))
+          '\n{}\n. This is an extremely rare occurrence. Please report it to'
+          ' the TensorFlow team.'.format(f, source))
   node, = nodes
 
   # TODO(znado): Place inside standard_analysis.
-- 
GitLab


From c0d3a71e476d52345477727681d3acf3ae87a2eb Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 26 Nov 2018 13:44:56 -0800
Subject: [PATCH 0798/1554] Fix Layer/variable aggregation from empty
 checkpointable data structures

PiperOrigin-RevId: 222879879
---
 tensorflow/python/training/checkpointable/data_structures.py | 5 ++++-
 .../python/training/checkpointable/data_structures_test.py   | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index a46a4a1910..817552f326 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -492,7 +492,10 @@ class Mapping(CheckpointableDataStructure, collections.Mapping):
   @property
   def _values(self):
     # Sort items deterministically by key
-    return list(zip(*sorted(self.items(), key=lambda it: it[0])))[1]
+    ordered = list(zip(*sorted(self.items(), key=lambda it: it[0])))
+    if ordered:
+      return ordered[1]
+    return []
 
   def _name_element(self, key):
     if not isinstance(key, six.string_types):
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index 17cbe3192d..9cefd942ac 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -335,11 +335,15 @@ class MappingTests(test.TestCase):
     d = {}
     root = tracking.Checkpointable()
     root.wrapper = d
+    self.assertEqual([], root.wrapper.layers)
+    self.assertEqual([], root.wrapper.trainable_weights)
     layer1 = core.Dense(1)
     layer2 = core.Dense(1)
     d["a"] = layer1
     d["b"] = layer2
     self.assertEqual([layer1, layer2], root.wrapper.layers)
+    # The layers have still not created variables
+    self.assertEqual([], root.wrapper.trainable_weights)
 
   def testHashing(self):
     has_mappings = set([data_structures.Mapping(),
-- 
GitLab


From 0d81559010c5ab605a47d81b595b9bd36b894906 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 26 Nov 2018 13:55:29 -0800
Subject: [PATCH 0799/1554] Update unified_lstm in eager mode.

PiperOrigin-RevId: 222881522
---
 .../python/keras/layers/unified_rnn_test.py   | 253 +++++++++---------
 1 file changed, 129 insertions(+), 124 deletions(-)

diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index e28f9625b9..b08ff3cafc 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -24,12 +24,13 @@ import time
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
-from tensorflow.python.client import session
+from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import constraints
@@ -54,14 +55,19 @@ from tensorflow.python.training import gradient_descent
 
 class RNNTest(test.TestCase):
 
+  rewrites = rewriter_config_pb2.RewriterConfig()
+  rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
+  customer_optimizer = rewrites.custom_optimizers.add()
+  customer_optimizer.name = 'ExperimentalImplementationSelector'
+  rewrites.min_graph_nodes = -1
+  graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
+  config = config_pb2.ConfigProto(graph_options=graph_options)
+
   def setUp(self):
-    rewrites = rewriter_config_pb2.RewriterConfig()
-    rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
-    customer_optimizer = rewrites.custom_optimizers.add()
-    customer_optimizer.name = 'ExperimentalImplementationSelector'
-    rewrites.min_graph_nodes = -1
-    graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
-    self.config = config_pb2.ConfigProto(graph_options=graph_options)
+    self.config = RNNTest.config
+
+  def tearDown(self):
+    ops.reset_default_graph()
 
   def test_unifiedRNN(self):
     input_shape = 10
@@ -71,7 +77,7 @@ class RNNTest(test.TestCase):
     batch = 100
     epoch = 1
 
-    with ops.Graph().as_default(), session.Session(config=self.config) as sess:
+    with self.cached_session(config=self.config, use_gpu=True) as sess:
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=batch,
           test_samples=0,
@@ -107,32 +113,6 @@ class RNNTest(test.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
-  def test_keras_model_with_lstm(self):
-    input_shape = 10
-    rnn_state_size = 8
-    output_shape = 8
-    timestep = 4
-    batch = 100
-    epoch = 10
-
-    (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=batch,
-        test_samples=0,
-        input_shape=(timestep, input_shape),
-        num_classes=output_shape)
-    y_train = keras.utils.to_categorical(y_train, output_shape)
-
-    K.set_session(session.Session(config=self.config))
-    layer = UnifiedLSTM(rnn_state_size)
-
-    inputs = keras.layers.Input(
-        shape=[timestep, input_shape], dtype=dtypes.float32)
-
-    outputs, unused_runtime = layer(inputs)
-    model = keras.models.Model(inputs, outputs)
-    model.compile('rmsprop', loss='mse')
-    model.fit(x_train, y_train, epochs=epoch)
-
   def test_unifiedRNN_with_cond(self):
     # This test is to demonstrate the graph rewrite of grappler plugin under
     # the condition that the function returns different number of internal
@@ -144,7 +124,7 @@ class RNNTest(test.TestCase):
     batch = 100
     epoch = 1
 
-    with ops.Graph().as_default(), session.Session(config=self.config) as sess:
+    with self.cached_session(config=self.config, use_gpu=True) as sess:
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=batch,
           test_samples=0,
@@ -190,37 +170,63 @@ class RNNTest(test.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def test_keras_model_with_lstm(self):
+    input_shape = 10
+    rnn_state_size = 8
+    output_shape = 8
+    timestep = 4
+    batch = 100
+    epoch = 10
+
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=batch,
+        test_samples=0,
+        input_shape=(timestep, input_shape),
+        num_classes=output_shape)
+    y_train = keras.utils.to_categorical(y_train, output_shape)
+
+    layer = UnifiedLSTM(rnn_state_size)
+
+    inputs = keras.layers.Input(
+        shape=[timestep, input_shape], dtype=dtypes.float32)
+
+    outputs, unused_runtime = layer(inputs)
+    model = keras.models.Model(inputs, outputs)
+    model.compile('rmsprop', loss='mse')
+    model.fit(x_train, y_train, epochs=epoch)
+
+  def _measure_performance(self, test_config, model, x_train, y_train):
+    batch = test_config['batch']
+    epoch = test_config['epoch']
+    warmup_epoch = test_config['warmup_epoch']
+
+    # warm up the model
+    model.fit(x_train, y_train, batch_size=batch, epochs=warmup_epoch)
+    start_time = time.time()
+    model.fit(x_train, y_train, batch_size=batch, epochs=epoch - warmup_epoch)
+    end_time = time.time()
+    return (end_time - start_time) / (epoch - warmup_epoch)
+
   def _time_performance_run_cudnn_lstm(self, test_config, x_train, y_train):
     # Get the performance number for standard Cudnn LSTM
     input_shape = test_config['input_shape']
     rnn_state_size = test_config['rnn_state_size']
     timestep = test_config['timestep']
-    epoch = test_config['epoch']
-    warmup_epoch = test_config['warmup_epoch']
 
-    ops.reset_default_graph()
-    with self.test_session(use_gpu=True):
-      cudnn_lstm_layer = CuDNNLSTM(rnn_state_size)
-      inputs = keras.layers.Input(
-          shape=[timestep, input_shape], dtype=dtypes.float32)
-
-      outputs = cudnn_lstm_layer(inputs)
-      model = keras.models.Model(inputs, outputs)
-      model.compile('sgd', 'mse')
-
-      total_duration = 0
-      for i in range(epoch):
-        start_time = time.time()
-        model.fit(x_train, y_train)
-        end_time = time.time()
-        if i >= warmup_epoch:
-          duration_per_epoch = end_time - start_time
-          total_duration += duration_per_epoch
-          logging.vlog(2, '%s: Time consumed for epoch %d is: %s',
-                       'CuDNN LSTM', i, duration_per_epoch)
-      logging.info('Average performance for %s per epoch is: %s',
-                   'CuDNN LSTM', (total_duration / epoch))
-      return total_duration / epoch
+    cudnn_lstm_layer = CuDNNLSTM(rnn_state_size)
+    inputs = keras.layers.Input(
+        shape=[timestep, input_shape], dtype=dtypes.float32)
+
+    outputs = cudnn_lstm_layer(inputs)
+    model = keras.models.Model(inputs, outputs)
+    model.compile('sgd', 'mse')
+
+    sec_per_epoch = self._measure_performance(
+        test_config, model, x_train, y_train)
+    logging.info('Average performance for %s per epoch is: %s',
+                 'CuDNN LSTM', sec_per_epoch)
+    return sec_per_epoch
 
   def _time_performance_run_unifed_lstm_gpu(
       self, test_config, x_train, y_train):
@@ -228,11 +234,7 @@ class RNNTest(test.TestCase):
     input_shape = test_config['input_shape']
     rnn_state_size = test_config['rnn_state_size']
     timestep = test_config['timestep']
-    epoch = test_config['epoch']
-    warmup_epoch = test_config['warmup_epoch']
 
-    ops.reset_default_graph()
-    K.set_session(session.Session(config=self.config))
     layer = UnifiedLSTM(rnn_state_size)
     inputs = keras.layers.Input(
         shape=[timestep, input_shape], dtype=dtypes.float32)
@@ -241,19 +243,11 @@ class RNNTest(test.TestCase):
     model = keras.models.Model(inputs, outputs)
     model.compile('sgd', 'mse')
 
-    total_duration = 0
-    for i in range(epoch):
-      start_time = time.time()
-      model.fit(x_train, y_train)
-      end_time = time.time()
-      if i >= warmup_epoch:
-        duration_per_epoch = end_time - start_time
-        total_duration += duration_per_epoch
-        logging.vlog(2, '%s: Time consumed for epoch %d is: %s',
-                     'Unified LSTM', i, duration_per_epoch)
+    sec_per_epoch = self._measure_performance(
+        test_config, model, x_train, y_train)
     logging.info('Average performance for %s per epoch is: %s',
-                 'Unified LSTM', (total_duration / epoch))
-    return total_duration / epoch
+                 'Unified LSTM', sec_per_epoch)
+    return sec_per_epoch
 
   def _time_performance_run_normal_lstm(
       self, test_config, x_train, y_train):
@@ -261,49 +255,40 @@ class RNNTest(test.TestCase):
     input_shape = test_config['input_shape']
     rnn_state_size = test_config['rnn_state_size']
     timestep = test_config['timestep']
-    epoch = test_config['epoch']
-    warmup_epoch = test_config['warmup_epoch']
 
-    ops.reset_default_graph()
-    with self.test_session(use_gpu=True):
-      layer = keras.layers.LSTM(rnn_state_size)
-      inputs = keras.layers.Input(
-          shape=[timestep, input_shape], dtype=dtypes.float32)
-
-      outputs = layer(inputs)
-      model = keras.models.Model(inputs, outputs)
-      model.compile('sgd', 'mse')
-
-      total_duration = 0
-      for i in range(epoch):
-        start_time = time.time()
-        model.fit(x_train, y_train)
-        end_time = time.time()
-        if i >= warmup_epoch:
-          duration_per_epoch = end_time - start_time
-          total_duration += duration_per_epoch
-          logging.vlog(2, '%s: Time consumed for epoch %d is: %s',
-                       'Normal LSTM', i, duration_per_epoch)
-      logging.info('Average performance for %s per epoch is: %s',
-                   'Normal LSTM', (total_duration / epoch))
-      return total_duration / epoch
-
-  def DISABLED_test_performance_with_standard_cudnn_impl(self):
+    layer = keras.layers.LSTM(rnn_state_size)
+    inputs = keras.layers.Input(
+        shape=[timestep, input_shape], dtype=dtypes.float32)
+
+    outputs = layer(inputs)
+    model = keras.models.Model(inputs, outputs)
+    model.compile('sgd', 'mse')
+
+    sec_per_epoch = self._measure_performance(
+        test_config, model, x_train, y_train)
+    logging.info('Average performance for %s per epoch is: %s',
+                 'Normal LSTM', sec_per_epoch)
+    return sec_per_epoch
+
+  @test_util.run_in_graph_and_eager_modes(config=config, use_gpu=True)
+  def test_performance_with_standard_cudnn_impl(self):
     if not test.is_gpu_available():
       self.skipTest('performance test will only run on GPU')
 
+    batch = 64
+    num_batch = 10
     test_config = {
         'input_shape': 128,
         'rnn_state_size': 64,
         'output_shape': 64,
         'timestep': 50,
+        'batch': batch,
         'epoch': 20,
         # The performance for warmup epoch is ignored.
         'warmup_epoch': 1,
     }
-    batch = 64
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=batch,
+        train_samples=(batch * num_batch),
         test_samples=0,
         input_shape=(test_config['timestep'], test_config['input_shape']),
         num_classes=test_config['output_shape'])
@@ -318,17 +303,25 @@ class RNNTest(test.TestCase):
 
     cudnn_vs_unified = cudnn_duration / unified_lstm_gpu_duration
     unified_vs_normal = normal_lstm_duration / unified_lstm_gpu_duration
-    # Assert the performance diff should be within 80% of the native cudnn impl.
-    self.assertGreaterEqual(
-        cudnn_vs_unified, 0.80,
-        'Expect the performance of Unified LSTM is within 80% of CuDNN LSTM, '
-        'but got {}%'.format(cudnn_vs_unified * 100))
-    # Assert the performance diff between CPU impl and GPU impl should be more
-    # than 5 times.
-    self.assertGreaterEqual(
-        unified_vs_normal, 5,
-        'Expect the performance of Unified LSTM is more than 5 times of normal '
-        'LSTM, but got {}'.format(unified_vs_normal))
+
+    # TODO(scottzhu): reeanble the test after moving it to benchmark test suite.
+    # The current test has performance flakiness issue.
+    logging.info('Expect the performance of Unified LSTM is within 80% of '
+                 'CuDNN LSTM, got {0:.2f}%'.format(cudnn_vs_unified * 100))
+    logging.info('Expect the performance of Unified LSTM is more than 5 times'
+                 ' of normal LSTM, got {0:.2f}'.format(unified_vs_normal))
+
+    # Assert the performance diff should be within 80% of the native cudnn.
+    # self.assertGreaterEqual(
+    #     cudnn_vs_unified, 0.80,
+    #     'Expect the performance of Unified LSTM is within 80% of CuDNN LSTM, '
+    #     'but got {0:.2f}%'.format(cudnn_vs_unified * 100))
+    # # Assert the performance diff between CPU impl and GPU impl should be more
+    # # than 5 times.
+    # self.assertGreaterEqual(
+    #     unified_vs_normal, 5,
+    #     'Expect the performance of Unified LSTM is more than 5 times of '
+    #     'normal LSTM, but got {0:.2f}'.format(unified_vs_normal))
 
 
 class UnifiedLSTM(RNN):
@@ -454,13 +447,25 @@ class UnifiedLSTM(RNN):
       # Reverse time axis.
       inputs = K.reverse(inputs, 1)
 
-    outputs, [new_h, new_c], runtime = normal_lstm(
-        inputs, initial_state[0], initial_state[1], self.kernel,
-        self.recurrent_kernel, self.bias, self.units, self.activation,
-        self.recurrent_activation)
-
-    function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1],
-                      self.kernel, self.recurrent_kernel, self.bias, self.units)
+    if ops.executing_eagerly_outside_functions():
+      if context.num_gpus() > 0:
+        outputs, [new_h, new_c], runtime = cudnn_lstm(
+            inputs, initial_state[0], initial_state[1], self.kernel,
+            self.recurrent_kernel, self.bias, self.units)
+      else:
+        outputs, [new_h, new_c], runtime = normal_lstm(
+            inputs, initial_state[0], initial_state[1], self.kernel,
+            self.recurrent_kernel, self.bias, self.units, self.activation,
+            self.recurrent_activation)
+    else:
+      outputs, [new_h, new_c], runtime = normal_lstm(
+          inputs, initial_state[0], initial_state[1], self.kernel,
+          self.recurrent_kernel, self.bias, self.units, self.activation,
+          self.recurrent_activation)
+
+      function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1],
+                        self.kernel, self.recurrent_kernel, self.bias,
+                        self.units)
 
     states = [new_h, new_c]
 
-- 
GitLab


From 30344d07629fd350686046079fe1ff75c8d36fb5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 14:07:23 -0800
Subject: [PATCH 0800/1554] In TF2.x, use thread-local stacks for device
 functions, control dependencies, and colocation constraints.

PiperOrigin-RevId: 222883745
---
 tensorflow/python/BUILD            |  4 +++-
 tensorflow/python/framework/ops.py | 11 +++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 3fe381183a..4cfb90c486 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -861,6 +861,7 @@ py_library(
         ":platform",
         ":registry",
         ":tensor_shape",
+        ":tf2",
         ":traceable_stack",
         ":util",
         ":versions",
@@ -984,6 +985,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":dtypes",
+        ":tf2",
         ":util",
         "//tensorflow/core:protos_all_py",
     ],
@@ -2085,7 +2087,6 @@ py_library(
     srcs = ["ops/control_flow_ops.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "tensor_shape",
         ":array_ops",
         ":array_ops_gen",
         ":constant_op",
@@ -2100,6 +2101,7 @@ py_library(
         ":resource_variable_ops_gen",
         ":sparse_tensor",
         ":tensor_array_ops",
+        ":tensor_shape",
         ":tf2",
         ":tf_should_use",
         ":util",
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b8187ab20c..ec00778347 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -36,6 +36,7 @@ from tensorflow.core.framework import op_def_pb2
 from tensorflow.core.framework import versions_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import pywrap_tensorflow as c_api
+from tensorflow.python import tf2
 from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
@@ -2811,8 +2812,8 @@ class Graph(object):
     self._stack_state_is_thread_local = False
     self._thread_local = threading.local()
     # Functions that will be applied to choose a device if none is specified.
-    # After switch_to_thread_local(), self._thread_local._device_function_stack
-    # is used instead.
+    # In TF2.x or after switch_to_thread_local(),
+    # self._thread_local._device_function_stack is used instead.
     self._graph_device_function_stack = traceable_stack.TraceableStack()
     # Default original_op applied to new ops.
     self._default_original_op = None
@@ -2820,7 +2821,7 @@ class Graph(object):
     # WhileContext defined in ops/control_flow_ops.py
     self._control_flow_context = None
     # A new node will depend of the union of all of the nodes in the stack.
-    # After switch_to_thread_local(),
+    # In TF2.x or after switch_to_thread_local(),
     # self._thread_local._control_dependencies_stack is used instead.
     self._graph_control_dependencies_stack = []
     # Arbitrary collections of objects.
@@ -2844,7 +2845,7 @@ class Graph(object):
         producer=versions.GRAPH_DEF_VERSION,
         min_consumer=versions.GRAPH_DEF_VERSION_MIN_CONSUMER)
     self._building_function = False
-    # Stack of colocate_with ops. After switch_to_thread_local(),
+    # Stack of colocate_with ops. In TF2.x or after switch_to_thread_local(),
     # self._thread_local._colocation_stack is used instead.
     self._graph_colocation_stack = traceable_stack.TraceableStack()
     # Set of tensors that are dangerous to feed!
@@ -2877,6 +2878,8 @@ class Graph(object):
     # requirement (many custom ops do not have shape functions, and we don't
     # want to break these existing cases).
     c_api.SetRequireShapeInferenceFns(self._c_graph, False)
+    if tf2.enabled():
+      self.switch_to_thread_local()
 
   # Note: this method is private because the API of tf.Graph() is public and
   # frozen, and this functionality is still not ready for public visibility.
-- 
GitLab


From eb334c580a5763d83f5ca5e3bab69baf4a6919da Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 14:09:15 -0800
Subject: [PATCH 0801/1554] V2 changes for math.bincount, math.confusion_matrix

PiperOrigin-RevId: 222884085
---
 tensorflow/python/ops/confusion_matrix.py     |  77 ++++++-
 tensorflow/python/ops/math_ops.py             |  82 +++++--
 .../tools/api/golden/v2/tensorflow.math.pbtxt |   4 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 217 ++++++++++++------
 4 files changed, 280 insertions(+), 100 deletions(-)

diff --git a/tensorflow/python/ops/confusion_matrix.py b/tensorflow/python/ops/confusion_matrix.py
index b86b174afe..ccfe3b65c2 100644
--- a/tensorflow/python/ops/confusion_matrix.py
+++ b/tensorflow/python/ops/confusion_matrix.py
@@ -90,12 +90,13 @@ def remove_squeezable_dimensions(
     return labels, predictions
 
 
-@tf_export(
-    'math.confusion_matrix',
-    v1=['math.confusion_matrix', 'confusion_matrix'])
-@deprecation.deprecated_endpoints('confusion_matrix', 'train.confusion_matrix')
-def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32,
-                     name=None, weights=None):
+@tf_export('math.confusion_matrix', v1=[])
+def confusion_matrix(labels,
+                     predictions,
+                     num_classes=None,
+                     weights=None,
+                     dtype=dtypes.int32,
+                     name=None):
   """Computes the confusion matrix from predictions and labels.
 
   The matrix columns represent the prediction labels and the rows represent the
@@ -132,9 +133,9 @@ def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32,
     num_classes: The possible number of labels the classification task can
                  have. If this value is not provided, it will be calculated
                  using both predictions and labels array.
+    weights: An optional `Tensor` whose shape matches `predictions`.
     dtype: Data type of the confusion matrix.
     name: Scope name.
-    weights: An optional `Tensor` whose shape matches `predictions`.
 
   Returns:
     A `Tensor` of type `dtype` with shape `[n, n]` representing the confusion
@@ -193,3 +194,65 @@ def confusion_matrix(labels, predictions, num_classes=None, dtype=dtypes.int32,
     zero_matrix = array_ops.zeros(math_ops.to_int32(shape), dtype)
 
     return sparse_ops.sparse_add(zero_matrix, cm_sparse)
+
+
+@tf_export(v1=['math.confusion_matrix', 'confusion_matrix'])
+@deprecation.deprecated_endpoints('confusion_matrix', 'train.confusion_matrix')
+def confusion_matrix_v1(labels,
+                        predictions,
+                        num_classes=None,
+                        dtype=dtypes.int32,
+                        name=None,
+                        weights=None):
+  """Computes the confusion matrix from predictions and labels.
+
+  The matrix columns represent the prediction labels and the rows represent the
+  real labels. The confusion matrix is always a 2-D array of shape `[n, n]`,
+  where `n` is the number of valid labels for a given classification task. Both
+  prediction and labels must be 1-D arrays of the same shape in order for this
+  function to work.
+
+  If `num_classes` is `None`, then `num_classes` will be set to one plus the
+  maximum value in either predictions or labels. Class labels are expected to
+  start at 0. For example, if `num_classes` is 3, then the possible labels
+  would be `[0, 1, 2]`.
+
+  If `weights` is not `None`, then each prediction contributes its
+  corresponding weight to the total value of the confusion matrix cell.
+
+  For example:
+
+  ```python
+    tf.confusion_matrix([1, 2, 4], [2, 2, 4]) ==>
+        [[0 0 0 0 0]
+         [0 0 1 0 0]
+         [0 0 1 0 0]
+         [0 0 0 0 0]
+         [0 0 0 0 1]]
+  ```
+
+  Note that the possible labels are assumed to be `[0, 1, 2, 3, 4]`,
+  resulting in a 5x5 confusion matrix.
+
+  Args:
+    labels: 1-D `Tensor` of real labels for the classification task.
+    predictions: 1-D `Tensor` of predictions for a given classification.
+    num_classes: The possible number of labels the classification task can have.
+      If this value is not provided, it will be calculated using both
+      predictions and labels array.
+    dtype: Data type of the confusion matrix.
+    name: Scope name.
+    weights: An optional `Tensor` whose shape matches `predictions`.
+
+  Returns:
+    A `Tensor` of type `dtype` with shape `[n, n]` representing the confusion
+    matrix, where `n` is the number of possible labels in the classification
+    task.
+
+  Raises:
+    ValueError: If both predictions and labels are not 1-D vectors and have
+      mismatched shapes, or if `weights` is not `None` and its shape doesn't
+      match `predictions`.
+  """
+  return confusion_matrix(labels, predictions, num_classes, weights, dtype,
+                          name)
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 73ca3d527a..a4e3613079 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2927,13 +2927,64 @@ def tanh(x, name=None):
       return gen_math_ops.tanh(x, name=name)
 
 
-@tf_export("math.bincount", v1=["math.bincount", "bincount"])
-@deprecation.deprecated_endpoints("bincount")
+@tf_export("math.bincount", v1=[])
 def bincount(arr,
              weights=None,
              minlength=None,
              maxlength=None,
-             dtype=dtypes.int32):
+             dtype=dtypes.int32,
+             name=None):
+  """Counts the number of occurrences of each value in an integer array.
+
+  If `minlength` and `maxlength` are not given, returns a vector with length
+  `tf.reduce_max(arr) + 1` if `arr` is non-empty, and length 0 otherwise.
+  If `weights` are non-None, then index `i` of the output stores the sum of the
+  value in `weights` at each index where the corresponding value in `arr` is
+  `i`.
+
+  Args:
+    arr: An int32 tensor of non-negative values.
+    weights: If non-None, must be the same shape as arr. For each value in
+      `arr`, the bin will be incremented by the corresponding weight instead of
+      1.
+    minlength: If given, ensures the output has length at least `minlength`,
+      padding with zeros at the end if necessary.
+    maxlength: If given, skips values in `arr` that are equal or greater than
+      `maxlength`, ensuring that the output has length at most `maxlength`.
+    dtype: If `weights` is None, determines the type of the output bins.
+    name: A name scope for the associated operations (optional).
+
+  Returns:
+    A vector with the same dtype as `weights` or the given `dtype`. The bin
+    values.
+  """
+  name = "bincount" if name is None else name
+  with ops.name_scope(name):
+    arr = ops.convert_to_tensor(arr, name="arr", dtype=dtypes.int32)
+    array_is_nonempty = reduce_prod(array_ops.shape(arr)) > 0
+    output_size = cast(array_is_nonempty, dtypes.int32) * (reduce_max(arr) + 1)
+    if minlength is not None:
+      minlength = ops.convert_to_tensor(
+          minlength, name="minlength", dtype=dtypes.int32)
+      output_size = gen_math_ops.maximum(minlength, output_size)
+    if maxlength is not None:
+      maxlength = ops.convert_to_tensor(
+          maxlength, name="maxlength", dtype=dtypes.int32)
+      output_size = gen_math_ops.minimum(maxlength, output_size)
+    if weights is not None:
+      weights = ops.convert_to_tensor(weights, name="weights")
+      return gen_math_ops.unsorted_segment_sum(weights, arr, output_size)
+    weights = constant_op.constant([], dtype)
+    return gen_math_ops.bincount(arr, output_size, weights)
+
+
+@tf_export(v1=["math.bincount", "bincount"])
+@deprecation.deprecated_endpoints("bincount")
+def bincount_v1(arr,
+                weights=None,
+                minlength=None,
+                maxlength=None,
+                dtype=dtypes.int32):
   """Counts the number of occurrences of each value in an integer array.
 
   If `minlength` and `maxlength` are not given, returns a vector with length
@@ -2945,34 +2996,19 @@ def bincount(arr,
   Args:
     arr: An int32 tensor of non-negative values.
     weights: If non-None, must be the same shape as arr. For each value in
-        `arr`, the bin will be incremented by the corresponding weight instead
-        of 1.
+      `arr`, the bin will be incremented by the corresponding weight instead of
+      1.
     minlength: If given, ensures the output has length at least `minlength`,
-        padding with zeros at the end if necessary.
+      padding with zeros at the end if necessary.
     maxlength: If given, skips values in `arr` that are equal or greater than
-        `maxlength`, ensuring that the output has length at most `maxlength`.
+      `maxlength`, ensuring that the output has length at most `maxlength`.
     dtype: If `weights` is None, determines the type of the output bins.
 
   Returns:
     A vector with the same dtype as `weights` or the given `dtype`. The bin
     values.
   """
-  arr = ops.convert_to_tensor(arr, name="arr", dtype=dtypes.int32)
-  array_is_nonempty = reduce_prod(array_ops.shape(arr)) > 0
-  output_size = cast(array_is_nonempty, dtypes.int32) * (reduce_max(arr) + 1)
-  if minlength is not None:
-    minlength = ops.convert_to_tensor(
-        minlength, name="minlength", dtype=dtypes.int32)
-    output_size = gen_math_ops.maximum(minlength, output_size)
-  if maxlength is not None:
-    maxlength = ops.convert_to_tensor(
-        maxlength, name="maxlength", dtype=dtypes.int32)
-    output_size = gen_math_ops.minimum(maxlength, output_size)
-  if weights is not None:
-    weights = ops.convert_to_tensor(weights, name="weights")
-    return gen_math_ops.unsorted_segment_sum(weights, arr, output_size)
-  weights = constant_op.constant([], dtype)
-  return gen_math_ops.bincount(arr, output_size, weights)
+  return bincount(arr, weights, minlength, maxlength, dtype)
 
 
 @tf_export("math.cumsum", "cumsum")
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index c10361f90d..f5bbb77d32 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -78,7 +78,7 @@ tf_module {
   }
   member_method {
     name: "bincount"
-    argspec: "args=[\'arr\', \'weights\', \'minlength\', \'maxlength\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"<dtype: \'int32\'>\"], "
+    argspec: "args=[\'arr\', \'weights\', \'minlength\', \'maxlength\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \"<dtype: \'int32\'>\", \'None\'], "
   }
   member_method {
     name: "ceil"
@@ -86,7 +86,7 @@ tf_module {
   }
   member_method {
     name: "confusion_matrix"
-    argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'dtype\', \'name\', \'weights\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'int32\'>\", \'None\', \'None\'], "
+    argspec: "args=[\'labels\', \'predictions\', \'num_classes\', \'weights\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \"<dtype: \'int32\'>\", \'None\'], "
   }
   member_method {
     name: "conj"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index ed29a7cb62..eb4e62b895 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -246,71 +246,142 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # function_reorders or function_keyword_renames, use the OLD function name.
     # These renames happen after the arguments have been processed.
     self.symbol_renames.update({
-        "tf.batch_to_space_nd": "tf.batch_to_space",
-        "tf.gfile.Copy": "tf.io.gfile.Copy",
-        "tf.gfile.DeleteRecursively": "tf.io.gfile.DeleteRecursively",
-        "tf.gfile.Exists": "tf.io.gfile.Exists",
-        "tf.gfile.Glob": "tf.io.gfile.Glob",
-        "tf.gfile.IsDirectory": "tf.io.gfile.IsDirectory",
-        "tf.gfile.ListDirectory": "tf.io.gfile.ListDirectory",
-        "tf.gfile.MakeDirs": "tf.io.gfile.MakeDirs",
-        "tf.gfile.MkDir": "tf.io.gfile.MkDir",
-        "tf.gfile.Remove": "tf.io.gfile.Remove",
-        "tf.gfile.Rename": "tf.io.gfile.Rename",
-        "tf.gfile.Stat": "tf.io.gfile.Stat",
-        "tf.gfile.Walk": "tf.io.gfile.Walk",
-        "tf.contrib.data.AUTOTUNE": "tf.data.experimental.AUTOTUNE",
-        "tf.contrib.data.Counter": "tf.data.experimental.Counter",
-        "tf.contrib.data.CheckpointInputPipelineHook": "tf.data.experimental.CheckpointInputPipelineHook",
-        "tf.contrib.data.CsvDataset": "tf.data.experimental.CsvDataset",
-        "tf.contrib.data.Optional": "tf.data.experimental.Optional",
-        "tf.contrib.data.RandomDataset": "tf.data.experimental.RandomDataset",
-        "tf.contrib.data.Reducer": "tf.data.experimental.Reducer",
-        "tf.contrib.data.SqlDataset": "tf.data.experimental.SqlDataset",
-        "tf.contrib.data.StatsAggregator": "tf.data.experimental.StatsAggregator",
-        "tf.contrib.data.TFRecordWriter": "tf.data.experimental.TFRecordWriter",
-        "tf.contrib.data.assert_element_shape": "tf.data.experimental.assert_element_shape",
-        "tf.contrib.data.batch_and_drop_remainder": "tf.compat.v1.contrib.data.batch_and_drop_remainder",
-        "tf.contrib.data.bucket_by_sequence_length": "tf.data.experimental.bucket_by_sequence_length",
-        "tf.contrib.data.choose_from_datasets": "tf.data.experimental.choose_from_datasets",
-        "tf.contrib.data.copy_to_device": "tf.data.experimental.copy_to_device",
-        "tf.contrib.data.dense_to_sparse_batch": "tf.data.experimental.dense_to_sparse_batch",
-        "tf.contrib.data.enumerate_dataset": "tf.data.experimental.enumerate_dataset",
-        "tf.contrib.data.get_next_as_optional": "tf.data.experimental.get_next_as_optional",
-        "tf.contrib.data.get_single_element": "tf.data.experimental.get_single_element",
-        "tf.contrib.data.group_by_reducer": "tf.data.experimental.group_by_reducer",
-        "tf.contrib.data.group_by_window": "tf.data.experimental.group_by_window",
-        "tf.contrib.data.ignore_errors": "tf.data.experimental.ignore_errors",
-        "tf.contrib.data.latency_stats": "tf.data.experimental.latency_stats",
-        "tf.contrib.data.make_batched_features_dataset": "tf.data.experimental.make_batched_features_dataset",
-        "tf.contrib.data.make_csv_dataset": "tf.data.experimental.make_csv_dataset",
-        "tf.contrib.data.make_saveable_from_iterator": "tf.data.experimental.make_saveable_from_iterator",
-        "tf.contrib.data.map_and_batch": "tf.data.experimental.map_and_batch",
-        "tf.contrib.data.padded_batch_and_drop_remainder": "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
-        "tf.contrib.data.parallel_interleave": "tf.data.experimental.parallel_interleave",
-        "tf.contrib.data.parse_example_dataset": "tf.data.experimental.parse_example_dataset",
-        "tf.contrib.data.prefetch_to_device": "tf.data.experimental.prefetch_to_device",
-        "tf.contrib.data.read_batch_features": "tf.compat.v1.contrib.data.read_batch_features",
-        "tf.contrib.data.reduce_dataset": "tf.compat.v1.contrib.data.reduce_dataset",
-        "tf.contrib.data.rejection_resample": "tf.data.experimental.rejection_resample",
-        "tf.contrib.data.sample_from_datasets": "tf.data.experimental.sample_from_datasets",
-        "tf.contrib.data.scan": "tf.data.experimental.scan",
-        "tf.contrib.data.set_stats_aggregator": "tf.data.experimental.set_stats_aggregator",
-        "tf.contrib.data.shuffle_and_repeat": "tf.data.experimental.shuffle_and_repeat",
-        "tf.contrib.data.sliding_window_batch": "tf.compat.v1.contrib.data.sliding_window_batch",
-        "tf.contrib.data.sloppy_interleave": "tf.compat.v1.contrib.data.sloppy_interleave",
-        "tf.contrib.data.unbatch": "tf.data.experimental.unbatch",
-        "tf.contrib.data.unique": "tf.data.experimental.unique",
-        "tf.contrib.framework.sort": "tf.sort",
-        "tf.contrib.framework.argsort": "tf.argsort",
-        "tf.manip.batch_to_space_nd": "tf.batch_to_space",
-        "tf.quantize_v2": "tf.quantization.quantize",
-        "tf.sparse_concat": "tf.sparse.concat",
-        "tf.sparse_split": "tf.sparse.split",
-        "tf.multinomial": "tf.random.categorical",
-        "tf.random.multinomial": "tf.random.categorical",
-        "tf.load_file_system_library": "tf.load_library",
-        "tf.pywrap_tensorflow": "tf.compat.v1.pywrap_tensorflow",
+        "tf.batch_to_space_nd":
+            "tf.batch_to_space",
+        "tf.gfile.Copy":
+            "tf.io.gfile.Copy",
+        "tf.gfile.DeleteRecursively":
+            "tf.io.gfile.DeleteRecursively",
+        "tf.gfile.Exists":
+            "tf.io.gfile.Exists",
+        "tf.gfile.Glob":
+            "tf.io.gfile.Glob",
+        "tf.gfile.IsDirectory":
+            "tf.io.gfile.IsDirectory",
+        "tf.gfile.ListDirectory":
+            "tf.io.gfile.ListDirectory",
+        "tf.gfile.MakeDirs":
+            "tf.io.gfile.MakeDirs",
+        "tf.gfile.MkDir":
+            "tf.io.gfile.MkDir",
+        "tf.gfile.Remove":
+            "tf.io.gfile.Remove",
+        "tf.gfile.Rename":
+            "tf.io.gfile.Rename",
+        "tf.gfile.Stat":
+            "tf.io.gfile.Stat",
+        "tf.gfile.Walk":
+            "tf.io.gfile.Walk",
+        "tf.contrib.data.AUTOTUNE":
+            "tf.data.experimental.AUTOTUNE",
+        "tf.contrib.data.Counter":
+            "tf.data.experimental.Counter",
+        "tf.contrib.data.CheckpointInputPipelineHook":
+            "tf.data.experimental.CheckpointInputPipelineHook",
+        "tf.contrib.data.CsvDataset":
+            "tf.data.experimental.CsvDataset",
+        "tf.contrib.data.Optional":
+            "tf.data.experimental.Optional",
+        "tf.contrib.data.RandomDataset":
+            "tf.data.experimental.RandomDataset",
+        "tf.contrib.data.Reducer":
+            "tf.data.experimental.Reducer",
+        "tf.contrib.data.SqlDataset":
+            "tf.data.experimental.SqlDataset",
+        "tf.contrib.data.StatsAggregator":
+            "tf.data.experimental.StatsAggregator",
+        "tf.contrib.data.TFRecordWriter":
+            "tf.data.experimental.TFRecordWriter",
+        "tf.contrib.data.assert_element_shape":
+            "tf.data.experimental.assert_element_shape",
+        "tf.contrib.data.batch_and_drop_remainder":
+            "tf.compat.v1.contrib.data.batch_and_drop_remainder",
+        "tf.contrib.data.bucket_by_sequence_length":
+            "tf.data.experimental.bucket_by_sequence_length",
+        "tf.contrib.data.choose_from_datasets":
+            "tf.data.experimental.choose_from_datasets",
+        "tf.contrib.data.copy_to_device":
+            "tf.data.experimental.copy_to_device",
+        "tf.contrib.data.dense_to_sparse_batch":
+            "tf.data.experimental.dense_to_sparse_batch",
+        "tf.contrib.data.enumerate_dataset":
+            "tf.data.experimental.enumerate_dataset",
+        "tf.contrib.data.get_next_as_optional":
+            "tf.data.experimental.get_next_as_optional",
+        "tf.contrib.data.get_single_element":
+            "tf.data.experimental.get_single_element",
+        "tf.contrib.data.group_by_reducer":
+            "tf.data.experimental.group_by_reducer",
+        "tf.contrib.data.group_by_window":
+            "tf.data.experimental.group_by_window",
+        "tf.contrib.data.ignore_errors":
+            "tf.data.experimental.ignore_errors",
+        "tf.contrib.data.latency_stats":
+            "tf.data.experimental.latency_stats",
+        "tf.contrib.data.make_batched_features_dataset":
+            "tf.data.experimental.make_batched_features_dataset",
+        "tf.contrib.data.make_csv_dataset":
+            "tf.data.experimental.make_csv_dataset",
+        "tf.contrib.data.make_saveable_from_iterator":
+            "tf.data.experimental.make_saveable_from_iterator",
+        "tf.contrib.data.map_and_batch":
+            "tf.data.experimental.map_and_batch",
+        "tf.contrib.data.padded_batch_and_drop_remainder":
+            "tf.compat.v1.contrib.data.padded_batch_and_drop_remainder",
+        "tf.contrib.data.parallel_interleave":
+            "tf.data.experimental.parallel_interleave",
+        "tf.contrib.data.parse_example_dataset":
+            "tf.data.experimental.parse_example_dataset",
+        "tf.contrib.data.prefetch_to_device":
+            "tf.data.experimental.prefetch_to_device",
+        "tf.contrib.data.read_batch_features":
+            "tf.compat.v1.contrib.data.read_batch_features",
+        "tf.contrib.data.reduce_dataset":
+            "tf.compat.v1.contrib.data.reduce_dataset",
+        "tf.contrib.data.rejection_resample":
+            "tf.data.experimental.rejection_resample",
+        "tf.contrib.data.sample_from_datasets":
+            "tf.data.experimental.sample_from_datasets",
+        "tf.contrib.data.scan":
+            "tf.data.experimental.scan",
+        "tf.contrib.data.set_stats_aggregator":
+            "tf.data.experimental.set_stats_aggregator",
+        "tf.contrib.data.shuffle_and_repeat":
+            "tf.data.experimental.shuffle_and_repeat",
+        "tf.contrib.data.sliding_window_batch":
+            "tf.compat.v1.contrib.data.sliding_window_batch",
+        "tf.contrib.data.sloppy_interleave":
+            "tf.compat.v1.contrib.data.sloppy_interleave",
+        "tf.contrib.data.unbatch":
+            "tf.data.experimental.unbatch",
+        "tf.contrib.data.unique":
+            "tf.data.experimental.unique",
+        "tf.contrib.framework.sort":
+            "tf.sort",
+        "tf.contrib.framework.argsort":
+            "tf.argsort",
+        "tf.manip.batch_to_space_nd":
+            "tf.batch_to_space",
+        "tf.quantize_v2":
+            "tf.quantization.quantize",
+        "tf.sparse_concat":
+            "tf.sparse.concat",
+        "tf.sparse_split":
+            "tf.sparse.split",
+        "tf.multinomial":
+            "tf.random.categorical",
+        "tf.random.multinomial":
+            "tf.random.categorical",
+        "tf.load_file_system_library":
+            "tf.load_library",
+        "tf.pywrap_tensorflow":
+            "tf.compat.v1.pywrap_tensorflow",
+        "tf.bincount":
+            "tf.math.bincount",
+        "tf.confusion_matrix":
+            "tf.math.confusion_matrix",
+        "tf.train.confusion_matrix":
+            "tf.math.confusion_matrix",
     })
     # pylint: enable=line-too-long
 
@@ -380,9 +451,13 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
         "tf.io.decode_csv": [
-            "records", "record_defaults",
-            "field_delim", "use_quote_delim",
-            "name", "na_value", "select_cols",
+            "records",
+            "record_defaults",
+            "field_delim",
+            "use_quote_delim",
+            "name",
+            "na_value",
+            "select_cols",
         ],
         "tf.strings.substr": ["input", "pos", "len", "name", "unit"],
         "tf.strings.reduce_join": [
@@ -471,6 +546,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "axis", "keep_dims", "separator", "name",
             "reduction_indices"
         ],
+        "tf.confusion_matrix": [
+            "labels", "predictions", "num_classes", "dtype", "name", "weights"
+        ],
+        "tf.math.confusion_matrix": [
+            "labels", "predictions", "num_classes", "dtype", "name", "weights"
+        ]
     }
 
     # Specially handled functions.
-- 
GitLab


From 44d7919d0a46305d4eeaa63b84294a2e12aa97c1 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Mon, 26 Nov 2018 14:16:48 -0800
Subject: [PATCH 0802/1554] Add support for dynamic learning rate to
 MirroredStrategy.

PiperOrigin-RevId: 222885339
---
 .../contrib/distribute/python/keras_test.py   | 34 +++++++++++++++----
 .../engine/distributed_training_utils.py      | 24 +++++++++----
 tensorflow/python/keras/engine/training.py    |  7 ++--
 3 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 29d85fe971..07027bde8a 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import distributed_training_utils
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
 from tensorflow.python.ops.parsing_ops import gen_parsing_ops
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -42,7 +43,6 @@ from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import rmsprop
 
-
 _RANDOM_SEED = 1337
 _TRAIN_SIZE = 200
 _INPUT_SIZE = (10,)
@@ -973,6 +973,28 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       ref_output = np.ones((160, 1), dtype=np.float32)
       self.assertArrayNear(output, ref_output, 1e-1)
 
+  @combinations.generate(strategy_minus_tpu_combinations())
+  def testOptimizerWithCallbacks(self, distribution):
+    with self.cached_session():
+      model = get_model()
+
+      optimizer = gradient_descent_keras.SGD(0.01)
+      loss = 'mse'
+      model.compile(optimizer, loss, distribute=distribution)
+
+      dataset = get_dataset(distribution)
+
+      def schedule(_):
+        return 0.001
+
+      model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
+                callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
+      grouped_models = distribution.unwrap(model._grouped_model)
+      with distribution.scope():
+        for m in grouped_models:
+          self.assertAllClose(0.001, keras.backend.get_value(
+              m.optimizer.lr), atol=1e-05, rtol=1e-05)
+
 
 class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
 
@@ -1090,14 +1112,14 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
       def schedule(_):
         return 0.001
       with self.assertRaisesRegexp(ValueError,
-                                   'LearningRateScheduler callback is not '
-                                   'supported with DistributionStrategy.'):
+                                   'You must specify a Keras Optimizer V2 when '
+                                   'using'):
         model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
                   callbacks=[keras.callbacks.LearningRateScheduler(schedule)])
 
       with self.assertRaisesRegexp(ValueError,
-                                   'ReduceLROnPlateau callback is not '
-                                   'supported with DistributionStrategy.'):
+                                   'You must specify a Keras Optimizer V2 when '
+                                   'using'):
         model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
                   callbacks=[keras.callbacks.ReduceLROnPlateau()])
       with self.assertRaisesRegexp(ValueError,
@@ -1247,7 +1269,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         model.set_weights(initial_weights)
         model.compile(
             loss=keras.losses.mean_squared_error,
-            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+            optimizer=gradient_descent_keras.SGD(0.5),
             distribute=with_distribution)
 
         training_inputs, eval_inputs, predict_inputs = (
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index fba557ff5b..bb8174e26a 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
@@ -145,11 +146,14 @@ def flatten_perdevice_values(distribution_strategy, perdevice_values):
           for e in distribution_strategy.unwrap(flattened)]
 
 
-def validate_callbacks(input_callbacks):
+def validate_callbacks(input_callbacks, optimizer, current_strategy):
   """Validate whether given callbacks are supported by DistributionStrategy.
 
   Args:
     input_callbacks: List of callbacks passed by the user to fit.
+    optimizer: Optimizer instance used to train the model.
+    current_strategy: The DistributionStrategy used to distribute training
+      and validation.
 
   Raises:
     ValueError: If `LearningRateScheduler` or `ReduceLROnPlateau` is one of the
@@ -171,12 +175,18 @@ def validate_callbacks(input_callbacks):
                         'these attributes are not set. You can access each of '
                         'the individual distributed models using the '
                         '`_grouped_model` attribute of your original model.')
-      if isinstance(callback, callbacks.LearningRateScheduler):
-        raise ValueError('LearningRateScheduler callback is not supported with '
-                         'DistributionStrategy.')
-      if isinstance(callback, callbacks.ReduceLROnPlateau):
-        raise ValueError('ReduceLROnPlateau callback is not supported with '
-                         'DistributionStrategy.')
+      if isinstance(callback, (callbacks.LearningRateScheduler,
+                               callbacks.ReduceLROnPlateau)):
+        strategy_name = current_strategy.__class__.__name__
+        # TODO(anjalisridhar): We might need to add a condition for multi
+        # worker strategy when we support it in Keras.
+        if is_tpu_strategy(current_strategy):
+          raise ValueError('%s callback is not supported with %s.' %
+                           (callback, strategy_name))
+
+        if not isinstance(optimizer, optimizer_v2.OptimizerV2):
+          raise ValueError('You must specify a Keras Optimizer V2 when using '
+                           '%s callback with DistributionStrategy.' % callback)
 
       # If users want to use the TensorBoard callback they cannot use certain
       # features of the callback that involve accessing model attributes and
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 8c564ed61b..4d3fffb25a 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1676,7 +1676,8 @@ class Model(Network):
 
     # Validate and standardize user data.
     if self._distribution_strategy:
-      distributed_training_utils.validate_callbacks(callbacks)
+      distributed_training_utils.validate_callbacks(callbacks, self.optimizer,
+                                                    self._distribution_strategy)
 
       distributed_training_utils.validate_inputs(
           x, y, self._distribution_strategy)
@@ -2488,9 +2489,7 @@ class DistributedCallbackModel(Model):
 
   def __init__(self, model):
     super(DistributedCallbackModel, self).__init__()
-    # TODO(anjalisridhar): Right now the only attributes set are the layer and
-    # weights. We may need to set additional attributes as needed since we have
-    # not called compile on this model.
+    self.optimizer = model.optimizer
 
   def set_original_model(self, orig_model):
     self._original_model = orig_model
-- 
GitLab


From d530dd79e70820a6791f4f6158c225d3d2e23c46 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Mon, 26 Nov 2018 14:22:54 -0800
Subject: [PATCH 0803/1554] Allow HLO version of Argmax as fallback when
 CustomCall is not supported

PiperOrigin-RevId: 222886344
---
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  | 49 +++++++++----------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index dce9641f63..20b7176041 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -30,7 +30,9 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-// The logic below uses a custom-call to implement argmax.
+// The logic below uses a custom-call to implement argmax when possible. When
+// custom-call is not allowed or input shapes are not supported, this kernel
+// falls back to using XLA HLO native ArgMax.
 //
 // Also see b/29507024 for first-class XLA support for indexing ops.
 class ArgMaxCustomCallOp : public XlaOpKernel {
@@ -64,16 +66,26 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
                     "Reduction axis ", dim,
                     " is empty in shape: ", input_shape.DebugString()));
 
+    const DataType dtype = output_type(0);
+    xla::PrimitiveType output_type;
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dtype, &output_type));
+
+    // Fall back to XLA ArgMax HLO when CustomCall is not allowed or when input
+    // shape isn't supported.
+    if (!XlaContext::Get(ctx).allow_cpu_custom_calls() ||
+        (input_dims != 1 && input_dims != 2)) {
+      xla::XlaOp output = XlaHelpers::ArgMax(ctx->Input(0), output_type, axis);
+      ctx->SetOutput(0, output);
+      return;
+    }
+
+    xla::XlaOp output;
     // The output shape is the input shape contracted along axis.
     TensorShape output_shape;
     for (int d = 0; d < input_shape.dims() - 1; ++d) {
       output_shape.AddDim(input_shape.dim_size((d < axis) ? d : d + 1));
     }
 
-    // For now we use a custom-call, only for the 1d and 2d cases.
-    OP_REQUIRES(ctx, XlaContext::Get(ctx).allow_cpu_custom_calls(),
-                errors::InvalidArgument(
-                    "ArgMax implementation requires a CustomCall on CPU"));
     xla::XlaBuilder& b = *ctx->builder();
 
     // XLA passes <out> to the function, so it is not included here.
@@ -104,27 +116,14 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
     }
 
     // Tell XLA to call the custom code, defined in
-    // index_ops_kernel_argmax_float_1d.cc.
-    xla::XlaOp output;
-    switch (input_shape.dims()) {
-      case 1:
-        output = xla::CustomCallWithLayout(&b, "argmax_float_1d_xla_impl", args,
-                                           xla_shape, arg_shapes);
-        break;
-      case 2:
-        output = xla::CustomCallWithLayout(&b, "argmax_float_2d_xla_impl", args,
-                                           xla_shape, arg_shapes);
-        break;
-      default:
-        OP_REQUIRES(ctx, false,
-                    errors::Unimplemented(
-                        "Argmax is only implemented for 1d and 2d tensors"
-                        ", but got shape: ",
-                        input_shape.DebugString()));
+    // index_ops_kernel_argmax_float_{1, 2}d.cc.
+    if (input_dims == 1) {
+      output = xla::CustomCallWithLayout(&b, "argmax_float_1d_xla_impl", args,
+                                         xla_shape, arg_shapes);
+    } else {
+      output = xla::CustomCallWithLayout(&b, "argmax_float_2d_xla_impl", args,
+                                         xla_shape, arg_shapes);
     }
-    const DataType dtype = output_type(0);
-    xla::PrimitiveType output_type;
-    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dtype, &output_type));
     output = xla::ConvertElementType(output, output_type);
     ctx->SetOutput(0, output);
   }
-- 
GitLab


From 315336dafd057c494e015d7a7716573fa33a5a40 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Mon, 26 Nov 2018 14:27:47 -0800
Subject: [PATCH 0804/1554] Add input processing to micro speech example

PiperOrigin-RevId: 222887236
---
 .../micro/examples/micro_speech/BUILD         | 211 ++++++++++++++++--
 .../examples/micro_speech/audio_provider.cc   |  33 +++
 .../examples/micro_speech/audio_provider.h    |  36 +++
 .../micro_speech/audio_provider_test.cc       |  44 ++++
 .../examples/micro_speech/feature_provider.cc | 121 ++++++++++
 .../examples/micro_speech/feature_provider.h  |  48 ++++
 .../micro_speech/feature_provider_test.cc     |  38 ++++
 .../micro_speech/fixed_point/preprocessor.cc  |  44 ++--
 .../micro/examples/micro_speech/main.cc       | 112 ++++++++++
 .../examples/micro_speech/model_settings.cc   |  23 ++
 .../examples/micro_speech/model_settings.h    |  42 ++++
 .../examples/micro_speech/preprocessor.cc     |  36 ++-
 .../examples/micro_speech/preprocessor.h      |   5 +
 .../micro/examples/micro_speech/timer.cc      |  22 ++
 .../micro/examples/micro_speech/timer.h       |  31 +++
 .../micro/examples/micro_speech/timer_test.cc |  46 ++++
 .../experimental/micro/testing/micro_test.h   |  18 ++
 17 files changed, 843 insertions(+), 67 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 07fb876411..799b2e5a5d 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -10,18 +10,46 @@ load(
     "tflite_micro_cc_test",
 )
 
-tflite_micro_cc_test(
-    name = "micro_speech_test",
+cc_library(
+    name = "model_settings",
+    srcs = [
+        "model_settings.cc",
+    ],
+    hdrs = [
+        "model_settings.h",
+    ],
+)
+
+cc_library(
+    name = "tiny_conv_model_data",
     srcs = [
-        "micro_speech_test.cc",
-        "no_features_data.cc",
-        "no_features_data.h",
         "tiny_conv_model_data.cc",
+    ],
+    hdrs = [
         "tiny_conv_model_data.h",
+    ],
+)
+
+cc_library(
+    name = "features_test_data",
+    srcs = [
+        "no_features_data.cc",
         "yes_features_data.cc",
+    ],
+    hdrs = [
+        "no_features_data.h",
         "yes_features_data.h",
     ],
+)
+
+tflite_micro_cc_test(
+    name = "micro_speech_test",
+    srcs = [
+        "micro_speech_test.cc",
+    ],
     deps = [
+        ":features_test_data",
+        ":tiny_conv_model_data",
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver",
@@ -31,46 +59,185 @@ tflite_micro_cc_test(
     ],
 )
 
-tflite_micro_cc_test(
-    name = "preprocessor_reference_test",
+cc_library(
+    name = "preprocessor_test_data",
     srcs = [
         "no_30ms_sample_data.cc",
-        "no_30ms_sample_data.h",
         "no_power_spectrum_data.cc",
+        "yes_30ms_sample_data.cc",
+        "yes_power_spectrum_data.cc",
+    ],
+    hdrs = [
+        "no_30ms_sample_data.h",
         "no_power_spectrum_data.h",
+        "yes_30ms_sample_data.h",
+        "yes_power_spectrum_data.h",
+    ],
+)
+
+cc_library(
+    name = "preprocessor_reference",
+    srcs = [
         "preprocessor.cc",
+    ],
+    hdrs = [
         "preprocessor.h",
+    ],
+    deps = [
+        ":model_settings",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "preprocessor_reference_test",
+    srcs = [
         "preprocessor_test.cc",
-        "yes_30ms_sample_data.cc",
-        "yes_30ms_sample_data.h",
-        "yes_power_spectrum_data.cc",
-        "yes_power_spectrum_data.h",
     ],
     deps = [
+        ":model_settings",
+        ":preprocessor_reference",
+        ":preprocessor_test_data",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
 
-tflite_micro_cc_test(
-    name = "preprocessor_fixed_test",
+cc_library(
+    name = "preprocessor_fixed",
     srcs = [
         "fixed_point/preprocessor.cc",
-        "no_30ms_sample_data.cc",
-        "no_30ms_sample_data.h",
-        "no_power_spectrum_data.cc",
-        "no_power_spectrum_data.h",
+    ],
+    hdrs = [
         "preprocessor.h",
+    ],
+    deps = [
+        ":model_settings",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "preprocessor_fixed_test",
+    srcs = [
         "preprocessor_test.cc",
-        "yes_30ms_sample_data.cc",
-        "yes_30ms_sample_data.h",
-        "yes_power_spectrum_data.cc",
-        "yes_power_spectrum_data.h",
     ],
     deps = [
+        ":model_settings",
+        ":preprocessor_fixed",
+        ":preprocessor_test_data",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/experimental/micro/testing:micro_test",
     ],
 )
+
+cc_library(
+    name = "audio_provider",
+    srcs = [
+        "audio_provider.cc",
+    ],
+    hdrs = [
+        "audio_provider.h",
+    ],
+    deps = [
+        ":model_settings",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "audio_provider_test",
+    srcs = [
+        "audio_provider_test.cc",
+    ],
+    deps = [
+        ":audio_provider",
+        ":model_settings",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
+    ],
+)
+
+cc_library(
+    name = "feature_provider",
+    srcs = [
+        "feature_provider.cc",
+    ],
+    hdrs = [
+        "feature_provider.h",
+    ],
+    deps = [
+        ":audio_provider",
+        ":model_settings",
+        ":preprocessor_reference",
+        ":timer",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "feature_provider_test",
+    srcs = [
+        "feature_provider_test.cc",
+    ],
+    deps = [
+        ":audio_provider",
+        ":feature_provider",
+        ":model_settings",
+        ":timer",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
+    ],
+)
+
+cc_library(
+    name = "timer",
+    srcs = [
+        "timer.cc",
+    ],
+    hdrs = [
+        "timer.h",
+    ],
+)
+
+tflite_micro_cc_test(
+    name = "timer_test",
+    srcs = [
+        "timer_test.cc",
+    ],
+    deps = [
+        ":timer",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/testing:micro_test",
+    ],
+)
+
+cc_binary(
+    name = "micro_speech",
+    srcs = [
+        "main.cc",
+    ],
+    deps = [
+        ":audio_provider",
+        ":feature_provider",
+        ":features_test_data",
+        ":model_settings",
+        ":preprocessor_reference",
+        ":timer",
+        ":tiny_conv_model_data",
+        "//tensorflow/lite:schema_fbs_version",
+        "//tensorflow/lite/experimental/micro:micro_framework",
+        "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver",
+        "//tensorflow/lite/experimental/micro/kernels:micro_ops",
+        "//tensorflow/lite/schema:schema_fbs",
+    ],
+)
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
new file mode 100644
index 0000000000..c0365d5690
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
@@ -0,0 +1,33 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+namespace {
+int16_t g_dummy_audio_data[kMaxAudioSampleSize];
+}  // namespace
+
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
+    g_dummy_audio_data[i] = 0;
+  }
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_dummy_audio_data;
+  return kTfLiteOk;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
new file mode 100644
index 0000000000..7e2442a5e8
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
@@ -0,0 +1,36 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+
+// This is an abstraction around an audio source like a microphone, and is
+// expected to return 16-bit PCM sample data for a given point in time. The
+// sample data itself should be used as quickly as possible by the caller, since
+// to allow memory optimizations there are no guarantees that the samples won't
+// be overwritten by new data in the future. In practice, implementations should
+// ensure that there's a reasonable time allowed for clients to access the data
+// before any reuse.
+// The reference implementation can have no platform-specific dependencies, so
+// it just returns an array filled with zeros. For real applications, you should
+// ensure there's a specialized implementation that accesses hardware APIs.
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples);
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
new file mode 100644
index 0000000000..5f7c7605f0
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
@@ -0,0 +1,44 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(TestAudioProvider) {
+  tflite::MicroErrorReporter micro_error_reporter;
+  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
+
+  int audio_samples_size = 0;
+  int16_t* audio_samples = nullptr;
+  TfLiteStatus get_status =
+      GetAudioSamples(error_reporter, 0, kFeatureSliceDurationMs,
+                      &audio_samples_size, &audio_samples);
+  TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, get_status);
+  TF_LITE_MICRO_EXPECT_LE(audio_samples_size, kMaxAudioSampleSize);
+  TF_LITE_MICRO_EXPECT_NE(audio_samples, nullptr);
+
+  // Make sure we can read all of the returned memory locations.
+  int total = 0;
+  for (int i = 0; i < audio_samples_size; ++i) {
+    total += audio_samples[i];
+  }
+}
+
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
new file mode 100644
index 0000000000..c4c52ac0ff
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
@@ -0,0 +1,121 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+namespace {
+// Stores the timestamp for the previous fetch of audio data, so that we can
+// avoid recalculating all the features from scratch if some earlier timeslices
+// are still present.
+int32_t g_last_time_in_ms = 0;
+// Make sure we don't try to use cached information if this is the first call
+// into the provider.
+bool g_is_first_run = true;
+}  // namespace
+
+FeatureProvider::FeatureProvider(int feature_size, uint8_t* feature_data)
+    : feature_size_(feature_size), feature_data_(feature_data) {
+  // Initialize the feature data to default values.
+  for (int n = 0; n < feature_size_; ++n) {
+    feature_data_[n] = 0;
+  }
+}
+
+FeatureProvider::~FeatureProvider() {}
+
+TfLiteStatus FeatureProvider::PopulateFeatureData(
+    tflite::ErrorReporter* error_reporter, int* how_many_new_slices) {
+  if (feature_size_ != kFeatureElementCount) {
+    error_reporter->Report("Requested feature_data_ size %d doesn't match %d",
+                           feature_size_, kFeatureElementCount);
+    return kTfLiteError;
+  }
+
+  const int32_t time_in_ms = TimeInMilliseconds();
+  // Quantize the time into steps as long as each window stride, so we can
+  // figure out which audio data we need to fetch.
+  const int last_step = (g_last_time_in_ms / kFeatureSliceStrideMs);
+  const int current_step = (time_in_ms / kFeatureSliceStrideMs);
+  g_last_time_in_ms = time_in_ms;
+
+  int slices_needed = current_step - last_step;
+  // If this is the first call, make sure we don't use any cached information.
+  if (g_is_first_run) {
+    g_is_first_run = false;
+    slices_needed = kFeatureSliceCount;
+  }
+  if (slices_needed > kFeatureSliceCount) {
+    slices_needed = kFeatureSliceCount;
+  }
+  *how_many_new_slices = slices_needed;
+
+  const int slices_to_keep = kFeatureSliceCount - slices_needed;
+  const int slices_to_drop = kFeatureSliceCount - slices_to_keep;
+  // If we can avoid recalculating some slices, just move the existing data
+  // up in the spectrogram, to perform something like this:
+  // last time = 80ms          current time = 120ms
+  // +-----------+             +-----------+
+  // | data@20ms |         --> | data@60ms |
+  // +-----------+       --    +-----------+
+  // | data@40ms |     --  --> | data@80ms |
+  // +-----------+   --  --    +-----------+
+  // | data@60ms | --  --      |  <empty>  |
+  // +-----------+   --        +-----------+
+  // | data@80ms | --          |  <empty>  |
+  // +-----------+             +-----------+
+  if (slices_to_keep > 0) {
+    for (int dest_slice = 0; dest_slice < slices_to_keep; ++dest_slice) {
+      uint8_t* dest_slice_data =
+          feature_data_ + (dest_slice * kFeatureSliceSize);
+      const int src_slice = dest_slice + slices_to_drop;
+      const uint8_t* src_slice_data =
+          feature_data_ + (src_slice * kFeatureSliceSize);
+      for (int i = 0; i < kFeatureSliceSize; ++i) {
+        dest_slice_data[i] = src_slice_data[i];
+      }
+    }
+  }
+  // Any slices that need to be filled in with feature data have their
+  // appropriate audio data pulled, and features calculated for that slice.
+  if (slices_needed > 0) {
+    for (int new_slice = slices_to_keep; new_slice < kFeatureSliceCount;
+         ++new_slice) {
+      const int new_step = (current_step - kFeatureSliceCount + 1) + new_slice;
+      const int32_t slice_start_ms = (new_step * kFeatureSliceStrideMs);
+      int16_t* audio_samples = nullptr;
+      int audio_samples_size = 0;
+      GetAudioSamples(error_reporter, slice_start_ms, kFeatureSliceDurationMs,
+                      &audio_samples_size, &audio_samples);
+      if (audio_samples_size < kMaxAudioSampleSize) {
+        error_reporter->Report("Audio data size %d  too small, want %d",
+                               audio_samples_size, kMaxAudioSampleSize);
+        return kTfLiteError;
+      }
+      uint8_t* new_slice_data = feature_data_ + (new_slice * kFeatureSliceSize);
+      TfLiteStatus preprocess_status =
+          Preprocess(error_reporter, audio_samples, audio_samples_size,
+                     kFeatureSliceSize, new_slice_data);
+      if (preprocess_status != kTfLiteOk) {
+        return preprocess_status;
+      }
+    }
+  }
+  return kTfLiteOk;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
new file mode 100644
index 0000000000..a86c56ebf0
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+
+// Binds itself to an area of memory intended to hold the input features for an
+// audio-recognition neural network model, and fills that data area with the
+// features representing the current audio input, for example from a microphone.
+// The audio features themselves are a two-dimensional array, made up of
+// horizontal slices representing the frequencies at one point in time, stacked
+// on top of each other to form a spectrogram showing how those frequencies
+// changed over time.
+class FeatureProvider {
+ public:
+  // Create the provider, and bind it to an area of memory. This memory should
+  // remain accessible for the lifetime of the provider object, since subsequent
+  // calls will fill it with feature data. The provider does no memory
+  // management of this data.
+  FeatureProvider(int feature_size, uint8_t* feature_data);
+  ~FeatureProvider();
+
+  // Fills the feature data with information from audio inputs, and returns how
+  // many feature slices were updated.
+  TfLiteStatus PopulateFeatureData(tflite::ErrorReporter* error_reporter,
+                                   int* how_many_new_slices);
+
+ private:
+  int feature_size_;
+  uint8_t* feature_data_;
+};
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_FEATURE_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
new file mode 100644
index 0000000000..1e52aec8d2
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(TestFeatureProvider) {
+  tflite::MicroErrorReporter micro_error_reporter;
+  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
+
+  uint8_t feature_data[kFeatureElementCount];
+  FeatureProvider feature_provider(kFeatureElementCount, feature_data);
+
+  int how_many_new_slices = 0;
+  TfLiteStatus populate_status = feature_provider.PopulateFeatureData(
+      error_reporter, &how_many_new_slices);
+  TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, populate_status);
+  TF_LITE_MICRO_EXPECT_EQ(kFeatureSliceCount, how_many_new_slices);
+}
+
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
index de60c982f3..b623d8d11b 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
@@ -31,6 +31,8 @@ limitations under the License.
 
 #include <cmath>
 
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
 namespace {
 
 // q format notation: qx.y => 1 sign bit, x-1 integer bits, y fraction bits.
@@ -66,13 +68,6 @@ inline int32_t FloatToFixed_Q2_30(float input) {
   return static_cast<int32_t>(roundf(input * (1 << 30)));
 }
 
-// These constants allow us to allocate fixed-sized arrays on the stack for our
-// working memory.
-constexpr int kInputSize = 512;
-constexpr int kAverageWindowSize = 6;
-constexpr int kOutputSize =
-    ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
-
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
 // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft.
@@ -127,14 +122,14 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output) {
   // Ensure our input and output data arrays are valid.
-  if (input_size > kInputSize) {
+  if (input_size > kMaxAudioSampleSize) {
     error_reporter->Report("Input size %d larger than %d", input_size,
-                           kInputSize);
+                           kMaxAudioSampleSize);
     return kTfLiteError;
   }
-  if (output_size != kOutputSize) {
+  if (output_size != kFeatureSliceSize) {
     error_reporter->Report("Requested output size %d doesn't match %d",
-                           output_size, kOutputSize);
+                           output_size, kFeatureSliceSize);
     return kTfLiteError;
   }
 
@@ -142,18 +137,17 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
   // In a real application, we'd calculate this table once in an initialization
   // function and store it for repeated reuse.
   // q1.15 format.
-  int16_t window_function[kInputSize];
+  int16_t window_function[kMaxAudioSampleSize];
   CalculatePeriodicHann(input_size, window_function);
 
   // Apply the window function to our time series input, and pad it with zeroes
   // to the next power of two.
-  int32_t fixed_input[kInputSize];
-  for (int i = 0; i < kInputSize; ++i) {
+  int32_t fixed_input[kMaxAudioSampleSize];
+  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
     if (i < input_size) {
       // input is int16_t.  Treat as q1.15 fixed point value in range [-1,1)
       // window_function is also q1.15 fixed point number
-      fixed_input[i] =
-          Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]);
+      fixed_input[i] = Q1_15_FixedMultiply_Q2_30(input[i], window_function[i]);
     } else {
       fixed_input[i] = 0;
     }
@@ -161,31 +155,31 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
 
   // Pull the frequency data from the time series sample.
   // Calculated in q10.22 format from q2.30 inputs.
-  int32_t fourier_values[kInputSize];
-  CalculateDiscreteFourierTransform(fixed_input, kInputSize, fourier_values);
+  int32_t fourier_values[kMaxAudioSampleSize];
+  CalculateDiscreteFourierTransform(fixed_input, kMaxAudioSampleSize,
+                                    fourier_values);
 
   // We have the complex numbers giving us information about each frequency
   // band, but all we want to know is how strong each frequency is, so calculate
   // the squared magnitude by adding together the squares of each component.
-  int32_t power_spectrum[kInputSize / 2];
-  for (int i = 0; i < (kInputSize / 2); ++i) {
+  int32_t power_spectrum[kMaxAudioSampleSize / 2];
+  for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) {
     const int32_t real = fourier_values[(i * 2) + 0];
     const int32_t imaginary = fourier_values[(i * 2) + 1];
     // q10.22 results
-    power_spectrum[i] =
-        Q10_22_FixedMultiply_Q10_22(real, real) +
-        Q10_22_FixedMultiply_Q10_22(imaginary, imaginary);
+    power_spectrum[i] = Q10_22_FixedMultiply_Q10_22(real, real) +
+                        Q10_22_FixedMultiply_Q10_22(imaginary, imaginary);
   }
 
   // Finally, reduce the size of the output by averaging together six adjacent
   // frequencies into each slot, producing an array of 43 values.
   // Power_spectrum numbers are q10.22.  Divide by kAverageWindowSize inside
   // loop to prevent overflow.
-  for (int i = 0; i < kOutputSize; ++i) {
+  for (int i = 0; i < kFeatureSliceSize; ++i) {
     int32_t average = 0;
     for (int j = 0; j < kAverageWindowSize; ++j) {
       const int index = (i * kAverageWindowSize) + j;
-      if (index < (kInputSize / 2)) {
+      if (index < (kMaxAudioSampleSize / 2)) {
         average += power_spectrum[index] / kAverageWindowSize;
       }
     }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
new file mode 100644
index 0000000000..1890c25cf2
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
@@ -0,0 +1,112 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h"
+#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/micro_interpreter.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/version.h"
+
+int main(int argc, char* argv[]) {
+  // Set up logging.
+  tflite::MicroErrorReporter micro_error_reporter;
+  tflite::ErrorReporter* error_reporter = &micro_error_reporter;
+
+  // Map the model into a usable data structure. This doesn't involve any
+  // copying or parsing, it's a very lightweight operation.
+  const tflite::Model* model = ::tflite::GetModel(g_tiny_conv_model_data);
+  if (model->version() != TFLITE_SCHEMA_VERSION) {
+    error_reporter->Report(
+        "Model provided is schema version %d not equal "
+        "to supported version %d.\n",
+        model->version(), TFLITE_SCHEMA_VERSION);
+    return 1;
+  }
+
+  // This pulls in all the operation implementations we need.
+  tflite::ops::micro::AllOpsResolver resolver;
+
+  // Create an area of memory to use for input, output, and intermediate arrays.
+  // The size of this will depend on the model you're using, and may need to be
+  // determined by experimentation.
+  const int tensor_arena_size = 10 * 1024;
+  uint8_t tensor_arena[tensor_arena_size];
+  tflite::SimpleTensorAllocator tensor_allocator(tensor_arena,
+                                                 tensor_arena_size);
+
+  // Build an interpreter to run the model with.
+  tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator,
+                                       error_reporter);
+
+  // Get information about the memory area to use for the model's input.
+  TfLiteTensor* model_input = interpreter.input(0);
+  if ((model_input->dims->size != 4) || (model_input->dims->data[0] != 1) ||
+      (model_input->dims->data[1] != kFeatureSliceCount) ||
+      (model_input->dims->data[2] != kFeatureSliceSize) ||
+      (model_input->type != kTfLiteUInt8)) {
+    error_reporter->Report("Bad input tensor parameters in model");
+    return 1;
+  }
+
+  // Prepare to access the audio spectrograms from a microphone or other source
+  // that will provide the inputs to the neural network.
+  FeatureProvider feature_provider(kFeatureElementCount,
+                                   model_input->data.uint8);
+
+  // Keep reading and analysing audio data in an infinite loop.
+  while (true) {
+    // Fetch the spectrogram for the current time.
+    int how_many_new_slices = 0;
+    TfLiteStatus feature_status = feature_provider.PopulateFeatureData(
+        error_reporter, &how_many_new_slices);
+    if (feature_status != kTfLiteOk) {
+      error_reporter->Report("Feature generation failed");
+      return 1;
+    }
+    // If no new audio samples have been received since last time, don't bother
+    // running the network model.
+    if (how_many_new_slices == 0) {
+      continue;
+    }
+
+    // Run the model on the spectrogram input and make sure it succeeds.
+    TfLiteStatus invoke_status = interpreter.Invoke();
+    if (invoke_status != kTfLiteOk) {
+      error_reporter->Report("Invoke failed");
+      return 1;
+    }
+
+    // The output from the model is a vector containing the scores for each
+    // kind of prediction, so figure out what the highest scoring category was.
+    TfLiteTensor* output = interpreter.output(0);
+    uint8_t top_category_score = 0;
+    int top_category_index = 0;
+    for (int category_index = 0; category_index < kCategoryCount;
+         ++category_index) {
+      const uint8_t category_score = output->data.uint8[category_index];
+      if (category_score > top_category_score) {
+        top_category_score = category_score;
+        top_category_index = category_index;
+      }
+    }
+
+    error_reporter->Report("Heard %s", kCategoryLabels[top_category_index]);
+  }
+
+  return 0;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc
new file mode 100644
index 0000000000..b9b8fb37b1
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc
@@ -0,0 +1,23 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+const char* kCategoryLabels[kCategoryCount] = {
+    "silence",
+    "unknown",
+    "yes",
+    "no",
+};
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
new file mode 100644
index 0000000000..1d8f3123a5
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
@@ -0,0 +1,42 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
+
+// Keeping these as constant expressions allow us to allocate fixed-sized arrays
+// on the stack for our working memory.
+
+// The size of the input time series data we pass to the FFT to produce the
+// frequency information. This has to be a power of two, and since we're dealing
+// with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
+constexpr int kMaxAudioSampleSize = 512;
+
+// All of these values are derived from the values used during model training,
+// if you change your model you'll need to update these constants.
+constexpr int kAverageWindowSize = 6;
+constexpr int kFeatureSliceSize =
+    ((kMaxAudioSampleSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
+constexpr int kFeatureSliceCount = 49;
+constexpr int kFeatureElementCount = (kFeatureSliceSize * kFeatureSliceCount);
+constexpr int kFeatureSliceStrideMs = 20;
+constexpr int kFeatureSliceDurationMs = 30;
+
+constexpr int kCategoryCount = 4;
+constexpr int kSilenceIndex = 0;
+constexpr int kUnknownIndex = 1;
+extern const char* kCategoryLabels[kCategoryCount];
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_MODEL_SETTINGS_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
index 12f9e22038..f4a7f801cc 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
@@ -28,14 +28,9 @@ limitations under the License.
 
 #include <cmath>
 
-namespace {
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
 
-// These constants allow us to allocate fixed-sized arrays on the stack for our
-// working memory.
-constexpr int kInputSize = 512;
-constexpr int kAverageWindowSize = 6;
-constexpr int kOutputSize =
-    ((kInputSize / 2) + (kAverageWindowSize - 1)) / kAverageWindowSize;
+namespace {
 
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
@@ -78,27 +73,27 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output) {
   // Ensure our input and output data arrays are valid.
-  if (input_size > kInputSize) {
+  if (input_size > kMaxAudioSampleSize) {
     error_reporter->Report("Input size %d larger than %d", input_size,
-                           kInputSize);
+                           kMaxAudioSampleSize);
     return kTfLiteError;
   }
-  if (output_size != kOutputSize) {
+  if (output_size != kFeatureSliceSize) {
     error_reporter->Report("Requested output size %d doesn't match %d",
-                           output_size, kOutputSize);
+                           output_size, kFeatureSliceSize);
     return kTfLiteError;
   }
 
   // Pre-calculate the window function we'll be applying to the input data.
   // In a real application, we'd calculate this table once in an initialization
   // function and store it for repeated reuse.
-  float window_function[kInputSize];
+  float window_function[kMaxAudioSampleSize];
   CalculatePeriodicHann(input_size, window_function);
 
   // Apply the window function to our time series input, and pad it with zeroes
   // to the next power of two.
-  float float_input[kInputSize];
-  for (int i = 0; i < kInputSize; ++i) {
+  float float_input[kMaxAudioSampleSize];
+  for (int i = 0; i < kMaxAudioSampleSize; ++i) {
     if (i < input_size) {
       float_input[i] =
           (input[i] * window_function[i]) / static_cast<float>(1 << 15);
@@ -108,14 +103,15 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
   }
 
   // Pull the frequency data from the time series sample.
-  float fourier_values[kInputSize];
-  CalculateDiscreteFourierTransform(float_input, kInputSize, fourier_values);
+  float fourier_values[kMaxAudioSampleSize];
+  CalculateDiscreteFourierTransform(float_input, kMaxAudioSampleSize,
+                                    fourier_values);
 
   // We have the complex numbers giving us information about each frequency
   // band, but all we want to know is how strong each frequency is, so calculate
   // the squared magnitude by adding together the squares of each component.
-  float power_spectrum[kInputSize / 2];
-  for (int i = 0; i < (kInputSize / 2); ++i) {
+  float power_spectrum[kMaxAudioSampleSize / 2];
+  for (int i = 0; i < (kMaxAudioSampleSize / 2); ++i) {
     const float real = fourier_values[(i * 2) + 0];
     const float imaginary = fourier_values[(i * 2) + 1];
     power_spectrum[i] = (real * real) + (imaginary * imaginary);
@@ -123,11 +119,11 @@ TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
 
   // Finally, reduce the size of the output by averaging together six adjacent
   // frequencies into each slot, producing an array of 43 values.
-  for (int i = 0; i < kOutputSize; ++i) {
+  for (int i = 0; i < kFeatureSliceSize; ++i) {
     float total = 0.0f;
     for (int j = 0; j < kAverageWindowSize; ++j) {
       const int index = (i * kAverageWindowSize) + j;
-      if (index < (kInputSize / 2)) {
+      if (index < (kMaxAudioSampleSize / 2)) {
         total += power_spectrum[index];
       }
     }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
index dede2a8642..adff790d6c 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h
@@ -19,6 +19,11 @@ limitations under the License.
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
 
+// Converts audio sample data into a more compact form that's appropriate for
+// feeding into a neural network. There are reference implementations that use
+// both floating point and fixed point available, but because the calculations
+// involved can be time-consuming, it's recommended that you use or write
+// specialized versions for your platform.
 TfLiteStatus Preprocess(tflite::ErrorReporter* error_reporter,
                         const int16_t* input, int input_size, int output_size,
                         uint8_t* output);
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
new file mode 100644
index 0000000000..6c96a61ab5
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
@@ -0,0 +1,22 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+int32_t TimeInMilliseconds() {
+  static int current_time = 0;
+  current_time += 100;
+  return current_time;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
new file mode 100644
index 0000000000..162952844a
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
+
+#include <cstdint>
+
+// Returns the time in milliseconds. There's no contract about what time zero
+// represents, the accuracy, or the granularity of the result. Subsequent calls
+// will generally not return a lower value, but even that's not guaranteed if
+// there's an overflow  wraparound.
+// The reference implementation of this function just returns a constantly
+// incrementing value for each call, since it would need a non-portable platform
+// call to access time information. For real applications, you'll need to write
+// your own platform-specific implementation.
+int32_t TimeInMilliseconds();
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
new file mode 100644
index 0000000000..83a2dfcc65
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
@@ -0,0 +1,46 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+#include <limits>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
+#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
+
+TF_LITE_MICRO_TESTS_BEGIN
+
+TF_LITE_MICRO_TEST(TestTimer) {
+  // Make sure that the technically-undefined overflow behavior we rely on below
+  // works on this platform. It's still not guaranteed, but at least this is a
+  // sanity check.
+  int32_t overflow_value = std::numeric_limits<int32_t>::max();
+  overflow_value += 1;
+  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
+
+  const int32_t first_time = TimeInMilliseconds();
+  const int32_t second_time = TimeInMilliseconds();
+
+  // It's possible that the timer may have wrapped around from +BIG_NUM to
+  // -BIG_NUM between the first and second calls, since we're storing
+  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
+  // would have taken more than 2^31 milliseconds though, so look at the
+  // difference and rely on integer overflow to ensure it's accurate.
+  const int32_t time_delta = (second_time - first_time);
+  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
+}
+
+TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/testing/micro_test.h b/tensorflow/lite/experimental/micro/testing/micro_test.h
index 10bab05fae..2f20dd5ac7 100644
--- a/tensorflow/lite/experimental/micro/testing/micro_test.h
+++ b/tensorflow/lite/experimental/micro/testing/micro_test.h
@@ -153,4 +153,22 @@ extern tflite::ErrorReporter* reporter;
     }                                                                        \
   } while (false)
 
+#define TF_LITE_MICRO_EXPECT_GE(x, y)                                         \
+  do {                                                                        \
+    if ((x) < (y)) {                                                          \
+      micro_test::reporter->Report(#x " >= " #y " failed at %s:%d", __FILE__, \
+                                   __LINE__);                                 \
+      micro_test::did_test_fail = true;                                       \
+    }                                                                         \
+  } while (false)
+
+#define TF_LITE_MICRO_EXPECT_LE(x, y)                                         \
+  do {                                                                        \
+    if ((x) > (y)) {                                                          \
+      micro_test::reporter->Report(#x " <= " #y " failed at %s:%d", __FILE__, \
+                                   __LINE__);                                 \
+      micro_test::did_test_fail = true;                                       \
+    }                                                                         \
+  } while (false)
+
 #endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_TESTING_MICRO_TEST_H_
-- 
GitLab


From bc4638e250b39bd6ebfd3a9bded632ad0139d5e7 Mon Sep 17 00:00:00 2001
From: Sreeni Kesavarapu <sreenik@google.com>
Date: Mon, 26 Nov 2018 14:32:24 -0800
Subject: [PATCH 0805/1554] Adds round_mode to QuantizeAndDequantizeV2 op to
 select rounding algorithm.

PiperOrigin-RevId: 222888022
---
 tensorflow/compiler/tests/unary_ops_test.py   |  66 +++++++++
 .../kernels/quantize_and_dequantize_op.cc     |  43 +++++-
 .../api_def_QuantizeAndDequantizeV2.pbtxt     |  15 +-
 tensorflow/core/kernels/cwise_ops.h           |  21 +++
 .../kernels/quantize_and_dequantize_op.cc     |  29 +++-
 .../core/kernels/quantize_and_dequantize_op.h |  81 +++++++++--
 .../quantize_and_dequantize_op_gpu.cu.cc      |   4 +-
 .../quantize_and_dequantize_op_test.cc        | 136 +++++++++++++++++-
 tensorflow/core/ops/array_ops.cc              |   3 +
 .../golden/v1/tensorflow.quantization.pbtxt   |   2 +-
 .../golden/v2/tensorflow.quantization.pbtxt   |   2 +-
 11 files changed, 374 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index d612d3b32d..95c9e7ffd4 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -481,6 +481,72 @@ class UnaryOpsTest(xla_test.XLATestCase):
           np.array([-1, -0.5, 0, 0.3], dtype=dtype),
           expected=np.array([-1., -0.5, 0., 0.296875], dtype=dtype))
 
+      def quantize_and_dequantize_v2_round_half_up(x):
+        return array_ops.quantize_and_dequantize_v2(
+            x,
+            -1,
+            1.0,
+            signed_input=True,
+            num_bits=8,
+            range_given=True,
+            round_mode="HALF_UP")
+
+      self._assertOpOutputMatchesExpected(
+          quantize_and_dequantize_v2_round_half_up,
+          np.array([-0.8, -0.5, 0, 0.3, 0.8, -2, 33], dtype=dtype),
+          expected=np.array([
+              -102.0 / 127,
+              -63.0 / 127,
+              0,
+              38.0 / 127,
+              102.0 / 127,
+              -128.0 / 127,
+              1,
+          ],
+                            dtype=dtype))
+
+      def quantize_and_dequantize_v2_round_half_to_even(x):
+        return array_ops.quantize_and_dequantize_v2(
+            x,
+            -1.0,
+            1.0,
+            signed_input=True,
+            num_bits=8,
+            range_given=True,
+            round_mode="HALF_TO_EVEN")
+
+      self._assertOpOutputMatchesExpected(
+          quantize_and_dequantize_v2_round_half_to_even,
+          np.array(
+              [
+                  -0.8,
+                  # The -0.5 should become -63.5 after scaling and with
+                  # rounding this should become -64. But with the test
+                  # unary_ops_test_cpu_ondemand, this fails as the result
+                  # before scaling becomes -63.499996 and gets rounded to -63.
+                  # TODO(sreenik): Some one more familiar with this test needs
+                  # to take a look and resolve this. This works on all other
+                  # variations of the platform like cpu, and gpu.
+                  # -0.5,
+                  0,
+                  0.3,
+                  0.8,
+                  -2,
+                  33
+              ],
+              dtype=dtype),
+          expected=np.array(
+              [
+                  -102.0 / 127,
+                  # -64.0 / 127,
+                  0,
+                  38.0 / 127,
+                  102.0 / 127,
+                  -128.0 / 127,
+                  1,
+              ],
+              dtype=dtype))
+
       def quantize_and_dequantize_v3(x):
         return array_ops.quantize_and_dequantize_v3(
             x, -127, 127, num_bits=8, signed_input=True, range_given=False)
diff --git a/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc b/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc
index 6f4ed496a1..7fe102428d 100644
--- a/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/quantize_and_dequantize_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/core/platform/macros.h"
@@ -26,12 +27,26 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
+enum QuantizerRoundMode {
+  // Round half up: if the fraction of y is exactly 0.5, then
+  // round(y) = y + 0.5
+  // E.g., -5.5 gets rounded to -5, -5.4 goes to -5,
+  // 5.4 goes to 5, and 5.5 goes to 6.
+  ROUND_HALF_UP,
+  // Round half to even: if the fraction of y is exactly 0.5, then round(y) is
+  // the nearest even integer to y.
+  // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes
+  // -24, and -24.5 gets rounded to 24.
+  ROUND_HALF_TO_EVEN,
+};
+
 class QuantizeAndDequantizeOp : public XlaOpKernel {
  public:
   explicit QuantizeAndDequantizeOp(OpKernelConstruction* ctx)
       : XlaOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("signed_input", &signed_input_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("range_given", &range_given_));
+    round_mode_ = ROUND_HALF_TO_EVEN;
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
@@ -117,8 +132,17 @@ class QuantizeAndDequantizeOp : public XlaOpKernel {
       // in that case they were measured from the tensor.
       input = Clamp(min_range, input, max_range);
     }
-    xla::XlaOp result =
-        Floor((input - min_range) * scale + half) * inverse_scale + min_range;
+    xla::XlaOp result;
+    switch (round_mode_) {
+      case ROUND_HALF_TO_EVEN: {
+        result = xla::RoundToEven(input * scale) * inverse_scale;
+        break;
+      }
+      case ROUND_HALF_UP: {
+        result = Floor(input * scale + half) * inverse_scale;
+        break;
+      }
+    }
     ctx->SetOutput(0, result);
   }
 
@@ -126,6 +150,7 @@ class QuantizeAndDequantizeOp : public XlaOpKernel {
   int64 num_bits_ = -1;
   bool signed_input_;
   bool range_given_;
+  QuantizerRoundMode round_mode_;
 };
 
 class QuantizeAndDequantizeV2Op : public QuantizeAndDequantizeOp {
@@ -136,6 +161,20 @@ class QuantizeAndDequantizeV2Op : public QuantizeAndDequantizeOp {
     OP_REQUIRES(ctx, num_bits_ > 0 && num_bits_ < (signed_input_ ? 62 : 63),
                 errors::InvalidArgument("num_bits is out of range: ", num_bits_,
                                         " with signed_input_ ", signed_input_));
+    string round_mode_string;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string));
+    OP_REQUIRES(
+        ctx,
+        (round_mode_string == "HALF_UP" || round_mode_string == "HALF_TO_EVEN"),
+        errors::InvalidArgument("Round mode string must be "
+                                "'HALF_UP' or "
+                                "'HALF_TO_EVEN', is '" +
+                                round_mode_string + "'"));
+    if (round_mode_string == "HALF_UP") {
+      round_mode_ = ROUND_HALF_UP;
+    } else if (round_mode_string == "HALF_TO_EVEN") {
+      round_mode_ = ROUND_HALF_TO_EVEN;
+    }
   }
 };
 
diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizeAndDequantizeV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizeAndDequantizeV2.pbtxt
index c43142599b..dff7c8754f 100644
--- a/tensorflow/core/api_def/base_api/api_def_QuantizeAndDequantizeV2.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_QuantizeAndDequantizeV2.pbtxt
@@ -39,6 +39,19 @@ END
     name: "range_given"
     description: <<END
 Whether the range is given or should be determined from the `input` tensor.
+END
+  }
+  attr {
+    name: "round_mode"
+    description: <<END
+The 'round_mode' attribute controls which rounding tie-breaking algorithm is
+used when rounding float values to their quantized equivalents. The following
+rounding modes are currently supported:
+
+*   HALF_TO_EVEN: this is the default round_mode.
+*   HALF_UP: round towards positive. In this mode 7.5 rounds up to 8 and -7.5
+    rounds up to -7.
+
 END
   }
   summary: "Quantizes then dequantizes a tensor."
@@ -93,7 +106,7 @@ following to each value in the 'input' tensor.
 
 output = round(clamp(value, input_min, input_max) * scale_factor) / scale_factor.
 
-The above round function uses half to even rounding.
+The above round function rounds the value based on the given round_mode.
 
 END
 }
diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index 3f7aa0dc39..313def9a75 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -449,6 +449,27 @@ struct functor_traits<scalar_round_op_google<Scalar>> {
   enum { Cost = 4 * NumTraits<Scalar>::AddCost, PacketAccess = false };
 };
 
+template <typename Scalar>
+struct scalar_round_up_op {
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar
+  operator()(const Scalar& x) const {
+    EIGEN_STATIC_ASSERT((!NumTraits<Scalar>::IsComplex),
+                        NUMERIC_TYPE_MUST_BE_REAL)
+
+    Scalar round_val = Eigen::numext::floor(x);
+    const Scalar fraction = x - round_val;
+    if (fraction >= Scalar(.5)) {
+      round_val += Scalar(1.0);
+    }
+    return round_val;
+  }
+};
+
+template <typename Scalar>
+struct functor_traits<scalar_round_up_op<Scalar>> {
+  enum { Cost = 4 * NumTraits<Scalar>::AddCost, PacketAccess = false };
+};
+
 #undef ENABLE_FLOAT_EQUALITY_WARNING
 #undef DISABLE_FLOAT_EQUALITY_WARNING
 
diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.cc b/tensorflow/core/kernels/quantize_and_dequantize_op.cc
index dadc15b69e..f13341e0af 100644
--- a/tensorflow/core/kernels/quantize_and_dequantize_op.cc
+++ b/tensorflow/core/kernels/quantize_and_dequantize_op.cc
@@ -49,6 +49,21 @@ class QuantizeAndDequantizeV2Op : public OpKernel {
                 errors::InvalidArgument("num_bits is out of range: ", num_bits_,
                                         " with signed_input_ ", signed_input_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("range_given", &range_given_));
+
+    string round_mode_string;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string));
+    OP_REQUIRES(
+        ctx,
+        (round_mode_string == "HALF_UP" || round_mode_string == "HALF_TO_EVEN"),
+        errors::InvalidArgument("Round mode string must be "
+                                "'HALF_UP' or "
+                                "'HALF_TO_EVEN', is '" +
+                                round_mode_string + "'"));
+    if (round_mode_string == "HALF_UP") {
+      round_mode_ = ROUND_HALF_UP;
+    } else if (round_mode_string == "HALF_TO_EVEN") {
+      round_mode_ = ROUND_HALF_TO_EVEN;
+    }
   }
 
   void Compute(OpKernelContext* ctx) override {
@@ -76,13 +91,15 @@ class QuantizeAndDequantizeV2Op : public OpKernel {
 
     functor::QuantizeAndDequantizeOneScaleFunctor<Device, T> f;
     f(ctx->eigen_device<Device>(), input.flat<T>(), signed_input_, num_bits_,
-      range_given_, &input_min_tensor, &input_max_tensor, output->flat<T>());
+      range_given_, &input_min_tensor, &input_max_tensor, round_mode_,
+      output->flat<T>());
   }
 
  private:
   bool signed_input_;
   int num_bits_;
   bool range_given_;
+  QuantizerRoundMode round_mode_;
 };
 
 // Simulate quantization precision loss in a float tensor by:
@@ -135,7 +152,8 @@ class QuantizeAndDequantizeV3Op : public OpKernel {
 
     functor::QuantizeAndDequantizeOneScaleFunctor<Device, T> f;
     f(ctx->eigen_device<Device>(), input.flat<T>(), signed_input_, num_bits_val,
-      range_given_, &input_min_tensor, &input_max_tensor, output->flat<T>());
+      range_given_, &input_min_tensor, &input_max_tensor, ROUND_HALF_TO_EVEN,
+      output->flat<T>());
   }
 
  private:
@@ -180,7 +198,7 @@ class QuantizeAndDequantizeOp : public OpKernel {
     functor::QuantizeAndDequantizeOneScaleFunctor<Device, T> functor;
     functor(ctx->eigen_device<Device>(), input.flat<T>(), signed_input_,
             num_bits_, range_given_, &input_min_tensor, &input_max_tensor,
-            output->flat<T>());
+            ROUND_HALF_TO_EVEN, output->flat<T>());
   }
 
  private:
@@ -198,10 +216,11 @@ struct QuantizeAndDequantizeOneScaleFunctor<CPUDevice, T> {
   void operator()(const CPUDevice& d, typename TTypes<T>::ConstVec input,
                   const bool signed_input, const int num_bits,
                   const bool range_given, Tensor* input_min_tensor,
-                  Tensor* input_max_tensor, typename TTypes<T>::Vec out) {
+                  Tensor* input_max_tensor, QuantizerRoundMode round_mode,
+                  typename TTypes<T>::Vec out) {
     QuantizeAndDequantizeOneScaleImpl<CPUDevice, T>::Compute(
         d, input, signed_input, num_bits, range_given, input_min_tensor,
-        input_max_tensor, out);
+        input_max_tensor, round_mode, out);
   }
 };
 }  // namespace functor
diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op.h b/tensorflow/core/kernels/quantize_and_dequantize_op.h
index 6b0c5e5a46..a495e8b71f 100644
--- a/tensorflow/core/kernels/quantize_and_dequantize_op.h
+++ b/tensorflow/core/kernels/quantize_and_dequantize_op.h
@@ -22,6 +22,20 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops.h"
 
 namespace tensorflow {
+
+enum QuantizerRoundMode {
+  // Round half up: if the fraction of y is exactly 0.5, then
+  // round(y) = y + 0.5
+  // E.g., -5.5 gets rounded to -5, -5.4 goes to -5,
+  // 5.4 goes to 5, and 5.5 goes to 6.
+  ROUND_HALF_UP,
+  // Round half to even: if the fraction of y is exactly 0.5, then round(y) is
+  // the nearest even integer to y.
+  // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes
+  // -24, and -24.5 gets rounded to 24.
+  ROUND_HALF_TO_EVEN,
+};
+
 namespace functor {
 
 // TODO(pauldonnelly): 'signed_input' should really be called 'signed_output'.
@@ -31,15 +45,69 @@ struct QuantizeAndDequantizeOneScaleFunctor {
   void operator()(const Device& d, typename TTypes<T>::ConstVec input,
                   bool signed_input, int num_bits, bool range_given,
                   Tensor* input_min_tensor, Tensor* input_max_tensor,
-                  typename TTypes<T>::Vec out);
+                  QuantizerRoundMode round_mode, typename TTypes<T>::Vec out);
 };
 
+// The implementation below runs on both CPU and GPU.
+template <typename Device, typename T, typename Func>
+void ClampScaleAndRound(const Device& d, typename TTypes<T>::ConstVec input,
+                        T min_range, T max_range, T scale, T inverse_scale,
+                        Func round_func, typename TTypes<T>::Vec out) {
+  out.device(d) = (input.cwiseMin(max_range).cwiseMax(min_range) * scale)
+                      .unaryExpr(round_func) *
+                  inverse_scale;
+}
+
+// The implementation below runs on both CPU and GPU.
+template <typename Device, typename T>
+void ClampScaleAndRound(const Device& d, typename TTypes<T>::ConstVec input,
+                        T min_range, T max_range, T scale, T inverse_scale,
+                        QuantizerRoundMode round_mode,
+                        typename TTypes<T>::Vec out) {
+  switch (round_mode) {
+    case ROUND_HALF_TO_EVEN:
+      ClampScaleAndRound(d, input, min_range, max_range, scale, inverse_scale,
+                         Eigen::internal::scalar_round_op_google<T>(), out);
+      break;
+    case ROUND_HALF_UP:
+      ClampScaleAndRound(d, input, min_range, max_range, scale, inverse_scale,
+                         Eigen::internal::scalar_round_up_op<T>(), out);
+      break;
+  }
+}
+
+// The implementation below runs on both CPU and GPU.
+template <typename Device, typename T, typename Func>
+void ScaleAndRound(const Device& d, typename TTypes<T>::ConstVec input, T scale,
+                   T inverse_scale, Func round_func,
+                   typename TTypes<T>::Vec out) {
+  out.device(d) = (input * scale).unaryExpr(round_func) * inverse_scale;
+}
+
+// The implementation below runs on both CPU and GPU.
+template <typename Device, typename T>
+void ScaleAndRound(const Device& d, typename TTypes<T>::ConstVec input, T scale,
+                   T inverse_scale, QuantizerRoundMode round_mode,
+                   typename TTypes<T>::Vec out) {
+  switch (round_mode) {
+    case ROUND_HALF_TO_EVEN:
+      ScaleAndRound(d, input, scale, inverse_scale,
+                    Eigen::internal::scalar_round_op_google<T>(), out);
+      break;
+    case ROUND_HALF_UP:
+      ScaleAndRound(d, input, scale, inverse_scale,
+                    Eigen::internal::scalar_round_up_op<T>(), out);
+      break;
+  }
+}
+
 // The implementation below runs on both CPU and GPU.
 template <typename Device, typename T>
 struct QuantizeAndDequantizeOneScaleImpl {
   static void Compute(const Device& d, typename TTypes<T>::ConstVec input,
                       bool signed_input, int num_bits, bool range_given,
                       Tensor* input_min_tensor, Tensor* input_max_tensor,
+                      QuantizerRoundMode round_mode,
                       typename TTypes<T>::Vec out) {
     T min_range;
     T max_range;
@@ -89,15 +157,10 @@ struct QuantizeAndDequantizeOneScaleImpl {
       // The semantics of the op does not guarantee to clamp to the specified
       // min_range and max_range - because we may have changed either min_range
       // or max_range.
-      out.device(d) =
-          (input.cwiseMin(max_range).cwiseMax(min_range) * scale)
-              .unaryExpr(Eigen::internal::scalar_round_op_google<T>()) *
-          inverse_scale;
+      ClampScaleAndRound(d, input, min_range, max_range, scale, inverse_scale,
+                         round_mode, out);
     } else {
-      out.device(d) =
-          (input * scale)
-              .unaryExpr(Eigen::internal::scalar_round_op_google<T>()) *
-          inverse_scale;
+      ScaleAndRound(d, input, scale, inverse_scale, round_mode, out);
     }
   }
 };
diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc b/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc
index 61c79cf695..5745e418f3 100644
--- a/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/quantize_and_dequantize_op_gpu.cu.cc
@@ -32,10 +32,10 @@ struct QuantizeAndDequantizeOneScaleFunctor<GPUDevice, T> {
   void operator()(const GPUDevice& d, typename TTypes<T>::ConstVec input,
                   bool signed_input, int num_bits, bool range_given,
                   Tensor* input_min_tensor, Tensor* input_max_tensor,
-                  typename TTypes<T>::Vec out) {
+                  QuantizerRoundMode round_mode, typename TTypes<T>::Vec out) {
     QuantizeAndDequantizeOneScaleImpl<GPUDevice, T>::Compute(
         d, input, signed_input, num_bits, range_given, input_min_tensor,
-        input_max_tensor, out);
+        input_max_tensor, round_mode, out);
   }
 };
 }  // end namespace functor
diff --git a/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc b/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc
index cddabf8a99..b9e015c96b 100644
--- a/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc
+++ b/tensorflow/core/kernels/quantize_and_dequantize_op_test.cc
@@ -101,17 +101,51 @@ TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int8) {
           .Attr("range_given", false)
           .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
-  AddInputFromArray<float>(TensorShape({6}), {-1, -0.5, 0, 0.3, 0.8, 0.555});
+  AddInputFromArray<float>(TensorShape({7}),
+                           {-1, -0.5, 0, 0.3, 0.8, 0.555, 0.50390625});
   AddInputFromArray<float>(TensorShape({}), {0.0});  // Min
   AddInputFromArray<float>(TensorShape({}), {0.0});  // Max
 
-  // With int8, the tensor is quantized to {-128, -64, 0, 38, 102, 71}.
+  // With int8, the tensor is quantized to {-128, -64, 0, 38, 102, 71, 64}.
   // Scale is: 1/127
-  // Then it is dequantized to {-1, -0.5, 0, 38.0/128, 102.0/128, 71.0/128}
+  // Then it is dequantized to {-1, -0.5, 0, 38.0/128, 102.0/128, 71.0/128, 0.5}
   TF_ASSERT_OK(RunOpKernel());
-  Tensor expected(allocator(), DT_FLOAT, TensorShape({6}));
-  test::FillValues<float>(&expected,
-                          {-1, -0.5, 0, 38.0 / 128, 102.0 / 128, 71.0 / 128});
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({7}));
+  test::FillValues<float>(
+      &expected, {-1, -0.5, 0, 38.0 / 128, 102.0 / 128, 71.0 / 128, 0.5});
+  test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
+
+  // Ensure that the inputs haven't been changed.
+  EXPECT_EQ(inputs_[1]->scalar<float>()(), 0.0);
+  EXPECT_EQ(inputs_[2]->scalar<float>()(), 0.0);
+}
+
+// Convert a 1D tensor with signed 8 bits and round_mode half_up.
+TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int8_round_half_up) {
+  TF_ASSERT_OK(
+      NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2")
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Attr("signed_input", true)
+          .Attr("num_bits", 8)
+          .Attr("range_given", false)
+          .Attr("round_mode", "HALF_UP")
+          .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  AddInputFromArray<float>(TensorShape({7}),
+                           {-1, -0.5, 0, 0.3, 0.8, 0.555, 0.50390625});
+  AddInputFromArray<float>(TensorShape({}), {0.0});  // Min
+  AddInputFromArray<float>(TensorShape({}), {0.0});  // Max
+
+  // With int8, the tensor is quantized to {-128, -64, 0, 38, 102, 71, 65}.
+  // Scale is: 1/127
+  // Then it is dequantized to {-1, -0.5, 0, 38.0/128, 102.0/128, 71.0/128,
+  // 65.0 /128}
+  TF_ASSERT_OK(RunOpKernel());
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({7}));
+  test::FillValues<float>(&expected, {-1, -0.5, 0, 38.0 / 128, 102.0 / 128,
+                                      71.0 / 128, 65.0 / 128});
   test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
 
   // Ensure that the inputs haven't been changed.
@@ -162,7 +196,7 @@ TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4) {
           .Attr("range_given", false)
           .Finalize(node_def()));
   TF_ASSERT_OK(InitOp());
-  AddInputFromArray<float>(TensorShape({6}), {-1, -0.5, 0, 0.3, 0.8, 0.555});
+  AddInputFromArray<float>(TensorShape({6}), {-1, -0.5, 0, 0.3125, 0.8, 0.555});
   AddInputFromArray<float>(TensorShape({}), {0.0});  // Min
   AddInputFromArray<float>(TensorShape({}), {0.0});  // Max
 
@@ -178,6 +212,35 @@ TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4) {
   EXPECT_EQ(inputs_[2]->scalar<float>()(), 0.0);
 }
 
+// Convert a 1D tensor with signed 4 bits and round_mode hafl_up.
+TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4_round_half_up) {
+  TF_ASSERT_OK(
+      NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2")
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Attr("signed_input", true)
+          .Attr("num_bits", 4)
+          .Attr("range_given", false)
+          .Attr("round_mode", "HALF_UP")
+          .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  AddInputFromArray<float>(TensorShape({6}), {-1, -0.5, 0, 0.3125, 0.8, 0.555});
+  AddInputFromArray<float>(TensorShape({}), {0.0});  // Min
+  AddInputFromArray<float>(TensorShape({}), {0.0});  // Max
+
+  // With int4, the tensor is quantized to {-8, -4, 0, 3, 6, 4}.
+  // Scale is: 1/8
+  TF_ASSERT_OK(RunOpKernel());
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({6}));
+  test::FillValues<float>(&expected, {-1, -0.5, 0, 0.375, 0.75, 0.5});
+  test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
+
+  // Ensure that the inputs haven't been changed.
+  EXPECT_EQ(inputs_[1]->scalar<float>()(), 0.0);
+  EXPECT_EQ(inputs_[2]->scalar<float>()(), 0.0);
+}
+
 // Convert a 1D tensor with signed 4 bits.
 TEST_F(QuantizeAndDequantizeTest, Convert_1D_tensor_with_int4_V3) {
   TF_ASSERT_OK(
@@ -237,6 +300,38 @@ TEST_F(QuantizeAndDequantizeTest, Convert_2D_tensor_with_int8_range_given) {
   test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
 }
 
+// Convert a 2D tensor with signed 8 bits, given range and round_mode half_up.
+TEST_F(QuantizeAndDequantizeTest,
+       Convert_2D_tensor_with_int8_range_given_round_half_up) {
+  TF_ASSERT_OK(
+      NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2")
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Attr("signed_input", true)
+          .Attr("num_bits", 8)
+          .Attr("range_given", true)
+          .Attr("round_mode", "HALF_UP")
+          .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  // Note that the last two values are saturated.
+  AddInputFromArray<float>(TensorShape({2, 4}),
+                           {-0.8, -0.5, 0, 0.3, 0.8, 0.555, -2, 33});
+  AddInputFromArray<float>(TensorShape({}), {-1.0});  // Min
+  AddInputFromArray<float>(TensorShape({}), {1.0});   // Max
+
+  // Note that the range is given as [-1, 1].
+  // With int8, the tensor is quantized to {-102, -63, 0, 38, 102, 70, -128,
+  // 127}.
+  // Scale is: 1/127
+  TF_ASSERT_OK(RunOpKernel());
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({2, 4}));
+  test::FillValues<float>(
+      &expected, {-102.0 / 127, -63.0 / 127, 0, 38.0 / 127, 102.0 / 127,
+                  70.0 / 127, -128.0 / 127, 1});
+  test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
+}
+
 // Convert a 2D tensor with signed 8 bits with given range.
 TEST_F(QuantizeAndDequantizeTest, Convert_2D_tensor_with_int8_range_given_V3) {
   TF_ASSERT_OK(
@@ -293,6 +388,33 @@ TEST_F(QuantizeAndDequantizeTest, Convert_4D_tensor_with_uint8_range_given) {
   test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
 }
 
+// Convert a 4D tensor with unsigned 8 bits, given range and round_mode half_up.
+TEST_F(QuantizeAndDequantizeTest,
+       Convert_4D_tensor_with_uint8_range_given_round_half_up) {
+  TF_ASSERT_OK(
+      NodeDefBuilder("quantize_and_dequantize_op", "QuantizeAndDequantizeV2")
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Input(FakeInput(DT_FLOAT))
+          .Attr("signed_input", false)
+          .Attr("num_bits", 8)
+          .Attr("range_given", true)
+          .Attr("round_mode", "HALF_UP")
+          .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  AddInputFromArray<float>(TensorShape({2, 2, 1, 1}), {-0.5, 0, 0.3, 0.8});
+  AddInputFromArray<float>(TensorShape({}), {0.0});  // Min
+  AddInputFromArray<float>(TensorShape({}), {1.0});  // Max
+
+  // Note that the range is given as [0, 1].
+  // With int8, the tensor is quantized to {0, 0, 77, 204}
+  // Scale is: 1/255
+  TF_ASSERT_OK(RunOpKernel());
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({2, 2, 1, 1}));
+  test::FillValues<float>(&expected, {0, 0, 77.0 / 255, 204.0 / 255});
+  test::ExpectTensorNear<float>(expected, *GetOutput(0), 1e-5);
+}
+
 // Convert a 4D tensor with unsigned 8 bits with given range.
 TEST_F(QuantizeAndDequantizeTest, Convert_4D_tensor_with_uint8_range_given_V3) {
   TF_ASSERT_OK(
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index f55562ec99..e07a35a63d 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -2743,6 +2743,9 @@ REGISTER_OP("QuantizeAndDequantizeV2")
     .Attr("range_given: bool = false")
     .Output("output: T")
     .Attr("T: {bfloat16, half, float, double}")
+    .Attr(
+        "round_mode: {'HALF_TO_EVEN', 'HALF_UP'} = "
+        "'HALF_TO_EVEN'")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle unused;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt
index 2948b7318e..632c2f8f83 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.quantization.pbtxt
@@ -34,7 +34,7 @@ tf_module {
   }
   member_method {
     name: "quantize_and_dequantize"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'None\'], "
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'HALF_TO_EVEN\', \'None\'], "
   }
   member_method {
     name: "quantized_concat"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt
index 2948b7318e..632c2f8f83 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.quantization.pbtxt
@@ -34,7 +34,7 @@ tf_module {
   }
   member_method {
     name: "quantize_and_dequantize"
-    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'None\'], "
+    argspec: "args=[\'input\', \'input_min\', \'input_max\', \'signed_input\', \'num_bits\', \'range_given\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'8\', \'False\', \'HALF_TO_EVEN\', \'None\'], "
   }
   member_method {
     name: "quantized_concat"
-- 
GitLab


From c73e01176b69573d2495f833f939f285f0d8c0cc Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 26 Nov 2018 14:32:28 -0800
Subject: [PATCH 0806/1554] [tf.data] Avoid calling `Iterator.get_next()` in
 each iteration of graph mode.

PiperOrigin-RevId: 222888047
---
 tensorflow/python/data/kernel_tests/test_base.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index af8e5e8a27..03fc0da149 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -45,8 +45,8 @@ class DatasetTestBase(test.TestCase):
     ```python
     # In both graph and eager modes
     dataset = ...
-    nxt = self.getNext(dataset)
-    result = self.evaluate(nxt())
+    get_next = self.getNext(dataset)
+    result = self.evaluate(get_next())
     ```
 
     Args:
@@ -66,7 +66,8 @@ class DatasetTestBase(test.TestCase):
         self.evaluate(iterator.initializer)
       else:
         iterator = dataset.make_one_shot_iterator()
-      return iterator.get_next
+      get_next = iterator.get_next()
+      return lambda: get_next
 
   def _compareOutputToExpected(self, result_values, expected_values,
                                assert_items_equal):
-- 
GitLab


From db03fe99dfc31f083356310f41b998307e7dfe30 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 14:32:54 -0800
Subject: [PATCH 0807/1554] Enable a GRPC ClientSession to request compression
 by setting the RPCOptions fields of the SessionConfig.

PiperOrigin-RevId: 222888148
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  1 +
 .../distributed_runtime/rpc/grpc_channel.cc   | 44 ++++++++++++++----
 .../distributed_runtime/rpc/grpc_channel.h    |  8 +++-
 .../rpc/grpc_channel_test.cc                  | 45 ++++++++++++++-----
 .../distributed_runtime/rpc/grpc_session.cc   | 10 +++--
 .../rpc/grpc_session_test.cc                  | 27 +++++++++++
 tensorflow/core/protobuf/config.proto         |  7 +++
 7 files changed, 116 insertions(+), 26 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index d122016d3e..273709a01f 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -105,6 +105,7 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc b/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc
index 456c30ecf4..781b7d65cd 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_channel.cc
@@ -53,30 +53,58 @@ Status ValidateHostPortPair(const string& host_port) {
   }
   return Status::OK();
 }
-}  // namespace
 
-Status NewHostPortGrpcChannel(const string& target,
-                              SharedGrpcChannelPtr* channel_pointer) {
-  // Minimally ensure that the target is valid
-  TF_RETURN_IF_ERROR(ValidateHostPortPair(target));
+}  // namespace
 
+::grpc::ChannelArguments GetChannelArguments(const RPCOptions* rpc_options) {
   // TODO(mrry): Implement secure channels.
   ::grpc::ChannelArguments args;
   args.SetInt(GRPC_ARG_MAX_MESSAGE_LENGTH, std::numeric_limits<int32>::max());
   // NOTE(mrry): Some versions of gRPC use a 20-second minimum backoff
   // on connection failure, which makes our tests time out.
   args.SetInt("grpc.testing.fixed_reconnect_backoff_ms", 1000);
+  if (rpc_options != nullptr) {
+    if (rpc_options->compression_algorithm() == "deflate") {
+      args.SetCompressionAlgorithm(GRPC_COMPRESS_DEFLATE);
+      args.SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL,
+                  rpc_options->compression_level());
+      VLOG(5) << "Setting GRPC compression : algo='"
+              << rpc_options->compression_algorithm()
+              << "' level=" << rpc_options->compression_level();
+    } else if (rpc_options->compression_algorithm() == "gzip") {
+      args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);
+      args.SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL,
+                  rpc_options->compression_level());
+      VLOG(5) << "Setting GRPC compression : algo='"
+              << rpc_options->compression_algorithm()
+              << "' level=" << rpc_options->compression_level();
+    } else if (!rpc_options->compression_algorithm().empty()) {
+      LOG(ERROR) << "Invalid compression algorithm: "
+                 << rpc_options->compression_algorithm();
+    }
+  }
+  return args;
+}
+
+Status NewHostPortGrpcChannel(const string& target,
+                              const RPCOptions* rpc_options,
+                              SharedGrpcChannelPtr* channel_pointer) {
+  // Minimally ensure that the target is valid
+  TF_RETURN_IF_ERROR(ValidateHostPortPair(target));
+
+  ::grpc::ChannelArguments args = GetChannelArguments(rpc_options);
   *channel_pointer = ::grpc::CreateCustomChannel(
       "dns:///" + target, ::grpc::InsecureChannelCredentials(), args);
   return Status::OK();
 }
 
 ChannelCreationFunction ConvertToChannelCreationFunction(
-    const std::function<Status(string, SharedGrpcChannelPtr*)>&
-        new_channel_func_ptr) {
+    const std::function<Status(string, const RPCOptions*,
+                               SharedGrpcChannelPtr*)>& new_channel_func_ptr) {
   return [new_channel_func_ptr](const string& target) -> SharedGrpcChannelPtr {
     SharedGrpcChannelPtr channel_ptr;
-    if (new_channel_func_ptr(target, &channel_ptr).ok()) {
+    if (new_channel_func_ptr(target, /*rpc_options=*/nullptr, &channel_ptr)
+            .ok()) {
       return channel_ptr;
     } else {
       return nullptr;
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_channel.h b/tensorflow/core/distributed_runtime/rpc/grpc_channel.h
index 6fa99d7b14..57d16218e8 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_channel.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_channel.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "grpcpp/grpcpp.h"
 
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 
@@ -86,11 +87,14 @@ GrpcChannelCache* NewGrpcChannelCache(const GrpcChannelSpec& channel_spec,
 
 // Below here are internal-only functions.
 
+::grpc::ChannelArguments GetChannelArguments(const RPCOptions* rpc_options);
+
 ChannelCreationFunction ConvertToChannelCreationFunction(
-    const std::function<Status(string, SharedGrpcChannelPtr*)>&
-        new_channel_func_ptr);
+    const std::function<Status(string, const RPCOptions*,
+                               SharedGrpcChannelPtr*)>& new_channel_func_ptr);
 
 Status NewHostPortGrpcChannel(const string& target,
+                              const RPCOptions* rpc_options,
                               SharedGrpcChannelPtr* channel_pointer);
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_channel_test.cc b/tensorflow/core/distributed_runtime/rpc/grpc_channel_test.cc
index a814ef85e2..a6fae2286f 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_channel_test.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_channel_test.cc
@@ -184,18 +184,39 @@ TEST(GrpcChannelTest, SparseHostPorts) {
 TEST(GrpcChannelTest, NewHostPortGrpcChannelValidation) {
   SharedGrpcChannelPtr mock_ptr;
 
-  EXPECT_TRUE(NewHostPortGrpcChannel("127.0.0.1:2222", &mock_ptr).ok());
-  EXPECT_TRUE(NewHostPortGrpcChannel("example.com:2222", &mock_ptr).ok());
-  EXPECT_TRUE(NewHostPortGrpcChannel("fqdn.example.com.:2222", &mock_ptr).ok());
-  EXPECT_TRUE(NewHostPortGrpcChannel("[2002:a9c:258e::]:2222", &mock_ptr).ok());
-  EXPECT_TRUE(NewHostPortGrpcChannel("[::]:2222", &mock_ptr).ok());
-
-  EXPECT_FALSE(NewHostPortGrpcChannel("example.com/abc:2222", &mock_ptr).ok());
-  EXPECT_FALSE(NewHostPortGrpcChannel("127.0.0.1:2222/", &mock_ptr).ok());
-  EXPECT_FALSE(NewHostPortGrpcChannel("example.com/abc:", &mock_ptr).ok());
-  EXPECT_FALSE(NewHostPortGrpcChannel("[::]/:2222", &mock_ptr).ok());
-  EXPECT_FALSE(NewHostPortGrpcChannel("[::]:2222/", &mock_ptr).ok());
-  EXPECT_FALSE(NewHostPortGrpcChannel("[::]:", &mock_ptr).ok());
+  EXPECT_TRUE(NewHostPortGrpcChannel("127.0.0.1:2222", /*rpc_options=*/nullptr,
+                                     &mock_ptr)
+                  .ok());
+  EXPECT_TRUE(NewHostPortGrpcChannel("example.com:2222",
+                                     /*rpc_options=*/nullptr, &mock_ptr)
+                  .ok());
+  EXPECT_TRUE(NewHostPortGrpcChannel("fqdn.example.com.:2222",
+                                     /*rpc_options=*/nullptr, &mock_ptr)
+                  .ok());
+  EXPECT_TRUE(NewHostPortGrpcChannel("[2002:a9c:258e::]:2222",
+                                     /*rpc_options=*/nullptr, &mock_ptr)
+                  .ok());
+  EXPECT_TRUE(
+      NewHostPortGrpcChannel("[::]:2222", /*rpc_options=*/nullptr, &mock_ptr)
+          .ok());
+
+  EXPECT_FALSE(NewHostPortGrpcChannel("example.com/abc:2222",
+                                      /*rpc_options=*/nullptr, &mock_ptr)
+                   .ok());
+  EXPECT_FALSE(NewHostPortGrpcChannel("127.0.0.1:2222/",
+                                      /*rpc_options=*/nullptr, &mock_ptr)
+                   .ok());
+  EXPECT_FALSE(NewHostPortGrpcChannel(
+                   "example.com/abc:", /*rpc_options=*/nullptr, &mock_ptr)
+                   .ok());
+  EXPECT_FALSE(
+      NewHostPortGrpcChannel("[::]/:2222", /*rpc_options=*/nullptr, &mock_ptr)
+          .ok());
+  EXPECT_FALSE(
+      NewHostPortGrpcChannel("[::]:2222/", /*rpc_options=*/nullptr, &mock_ptr)
+          .ok());
+  EXPECT_FALSE(
+      NewHostPortGrpcChannel("[::]:", /*rpc_options=*/nullptr, &mock_ptr).ok());
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
index fdce1b10e0..1ad40fe297 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
@@ -52,8 +52,9 @@ Status GrpcSession::Create(const SessionOptions& options,
   }
   if (!master) {
     SharedGrpcChannelPtr master_channel;
-    TF_RETURN_IF_ERROR(NewHostPortGrpcChannel(
-        options.target.substr(kSchemePrefixLength), &master_channel));
+    TF_RETURN_IF_ERROR(
+        NewHostPortGrpcChannel(options.target.substr(kSchemePrefixLength),
+                               &options.config.rpc_options(), &master_channel));
     master.reset(NewGrpcMaster(master_channel));
   }
   session->SetRemoteMaster(std::move(master));
@@ -384,8 +385,9 @@ void GrpcSession::SetRemoteMaster(std::unique_ptr<MasterInterface> master) {
 Status GrpcSession::Reset(const SessionOptions& options,
                           const std::vector<string>& containers) {
   SharedGrpcChannelPtr master_channel;
-  TF_RETURN_IF_ERROR(NewHostPortGrpcChannel(
-      options.target.substr(kSchemePrefixLength), &master_channel));
+  TF_RETURN_IF_ERROR(
+      NewHostPortGrpcChannel(options.target.substr(kSchemePrefixLength),
+                             /*rpc_options=*/nullptr, &master_channel));
   auto master = NewGrpcMaster(master_channel);
   ResetRequest req;
   for (const auto& c : containers) req.add_container(c);
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session_test.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session_test.cc
index fc601991a2..ad0f8e5e2f 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session_test.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session_test.cc
@@ -1066,4 +1066,31 @@ TEST(SessionTest, RunTimeoutWithRunOptions) {
               error::INTERNAL == status.code());
 }
 
+TEST(SessionTest, TestCompression) {
+  std::unique_ptr<test::TestCluster> cluster;
+  TF_CHECK_OK(test::TestCluster::MakeTestCluster(Devices(1, 0), 1, &cluster));
+  SessionOptions options = Options(cluster->targets()[0], 100);
+  RPCOptions* rpc_options = options.config.mutable_rpc_options();
+  rpc_options->set_compression_algorithm("deflate");
+  rpc_options->set_compression_level(GRPC_COMPRESS_LEVEL_HIGH);
+
+  std::unique_ptr<Session> session(NewRemote(options));
+
+  static const float kTestValue = 409.1934f;
+  Graph graph(OpRegistry::Global());
+  Tensor tensor(DT_FLOAT, TensorShape({1, 1}));
+  tensor.flat<float>()(0) = kTestValue;
+  Node* b = test::graph::Constant(&graph, tensor);
+  GraphDef gdef;
+  graph.ToGraphDef(&gdef);
+  RunOptions run_options;
+  TF_CHECK_OK(session->Create(run_options, gdef));
+
+  std::vector<std::pair<string, Tensor>> inputs;
+  std::vector<Tensor> outputs;
+  TF_CHECK_OK(session->Run(inputs, {b->name()}, {}, &outputs));
+  ASSERT_EQ(1, outputs.size());
+  IsSingleFloatValue(outputs[0], kTestValue);
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 174b588661..b3dc5dccc0 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -291,6 +291,13 @@ message RPCOptions {
   // transport for client-master communication that avoids the RPC
   // stack. This option is primarily for used testing the RPC stack.
   bool use_rpc_for_inprocess_master = 1;
+
+  // The compression algorithm to be used. One of "deflate", "gzip".
+  string compression_algorithm = 2;
+
+  // If compression_algorithm is set, the compression level to be used.
+  // From 0 (no compression), up to 3.
+  int32 compression_level = 3;
 };
 
 // Session configuration parameters.
-- 
GitLab


From 6f2c7bebc8cef4806f8cbed422df256bb7b6bad4 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Mon, 26 Nov 2018 14:37:54 -0800
Subject: [PATCH 0808/1554] Add KubernetesClusterResolver and
 TFConfigClusterResolver to base init file

PiperOrigin-RevId: 222888923
---
 tensorflow/contrib/cluster_resolver/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/cluster_resolver/__init__.py b/tensorflow/contrib/cluster_resolver/__init__.py
index fd1263fe81..ab0746ab83 100644
--- a/tensorflow/contrib/cluster_resolver/__init__.py
+++ b/tensorflow/contrib/cluster_resolver/__init__.py
@@ -24,7 +24,9 @@ from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import
 from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver
 from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver
 from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver
+from tensorflow.contrib.cluster_resolver.python.training.kubernetes_cluster_resolver import KubernetesClusterResolver
 from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver
+from tensorflow.contrib.cluster_resolver.python.training.tfconfig_cluster_resolver import TFConfigClusterResolver
 from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver
 # pylint: enable=wildcard-import,unused-import
 
@@ -35,6 +37,8 @@ _allowed_symbols = [
     'SimpleClusterResolver',
     'UnionClusterResolver',
     'GceClusterResolver',
+    'KubernetesClusterResolver',
+    'TFConfigClusterResolver',
     'TPUClusterResolver',
     'SlurmClusterResolver',
 ]
-- 
GitLab


From 47131dbc19605e3754878d738dbfbe4e539c36da Mon Sep 17 00:00:00 2001
From: Taylor Robie <taylorrobie@google.com>
Date: Mon, 26 Nov 2018 14:42:08 -0800
Subject: [PATCH 0809/1554] Add a `name` arg to scalar_mul()

PiperOrigin-RevId: 222889631
---
 tensorflow/python/ops/math_ops.py               | 17 +++++++++++++----
 .../tools/api/golden/v1/tensorflow.math.pbtxt   |  2 +-
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt |  2 +-
 .../tools/api/golden/v2/tensorflow.math.pbtxt   |  2 +-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt |  2 +-
 5 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index a4e3613079..d6b2423c1b 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -440,8 +440,8 @@ def erf(x, name=None):
       return gen_math_ops.erf(x, name=name)
 
 
-@tf_export("math.scalar_mul", "scalar_mul")
-def scalar_mul(scalar, x):
+@tf_export(v1=["math.scalar_mul", "scalar_mul"])
+def scalar_mul(scalar, x, name=None):
   """Multiplies a scalar times a `Tensor` or `IndexedSlices` object.
 
   Intended for use in gradient code which might deal with `IndexedSlices`
@@ -451,6 +451,7 @@ def scalar_mul(scalar, x):
   Args:
     scalar: A 0-D scalar `Tensor`. Must have known shape.
     x: A `Tensor` or `IndexedSlices` to be scaled.
+    name: A name for the operation (optional).
 
   Returns:
     `scalar * x` of the same type (`Tensor` or `IndexedSlices`) as `x`.
@@ -463,13 +464,21 @@ def scalar_mul(scalar, x):
   shape = scalar.get_shape()
   if shape.ndims == 0:
     if isinstance(x, ops.IndexedSlices):
-      return ops.IndexedSlices(scalar * x.values, x.indices, x.dense_shape)
+      return ops.IndexedSlices(gen_math_ops.mul(scalar, x.values, name),
+                               x.indices, x.dense_shape)
     else:
-      return scalar * x
+      return gen_math_ops.mul(scalar, x, name)
   else:
     raise ValueError("Only scalar multiply works, got shape %s" % shape)
 
 
+@tf_export("math.scalar_mul", "scalar_mul", v1=[])
+@_set_doc(scalar_mul.__doc__)
+def scalar_mul_v2(scalar, x, name=None):
+  with ops.name_scope(name, "scalar_mul", [x]) as name:
+    return scalar_mul(scalar, x, name)
+
+
 @tf_export("math.pow", "pow")
 def pow(x, y, name=None):  # pylint: disable=redefined-builtin
   r"""Computes the power of one value to another.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
index b7a99caeb7..f34e2c2aa5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.math.pbtxt
@@ -342,7 +342,7 @@ tf_module {
   }
   member_method {
     name: "scalar_mul"
-    argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "segment_max"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 290cd2d9c5..656a52945c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1826,7 +1826,7 @@ tf_module {
   }
   member_method {
     name: "scalar_mul"
-    argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "scan"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index f5bbb77d32..979d77ea6b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -342,7 +342,7 @@ tf_module {
   }
   member_method {
     name: "scalar_mul"
-    argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "segment_max"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a1e2e7be6f..a380b0d99f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -946,7 +946,7 @@ tf_module {
   }
   member_method {
     name: "scalar_mul"
-    argspec: "args=[\'scalar\', \'x\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'scalar\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "scan"
-- 
GitLab


From eca1437dece6ed0379193d11dd3f3a94dc16681e Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 26 Nov 2018 14:47:18 -0800
Subject: [PATCH 0810/1554] Add extendable logging system.

PiperOrigin-RevId: 222890503
---
 tensorflow/core/BUILD                         | 17 +++++++
 .../core/platform/default/build_config.bzl    |  3 ++
 tensorflow/core/platform/default/logger.cc    | 34 +++++++++++++
 tensorflow/core/platform/logger.h             | 51 +++++++++++++++++++
 4 files changed, 105 insertions(+)
 create mode 100644 tensorflow/core/platform/default/logger.cc
 create mode 100644 tensorflow/core/platform/logger.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 5e1d93cfa2..781ada7b46 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -112,6 +112,7 @@ load(
     "tf_additional_device_tracer_test_flags",
     "tf_additional_gdr_lib_defines",
     "tf_additional_human_readable_json_deps",
+    "tf_additional_logger_deps",
     "tf_additional_lib_defines",
     "tf_additional_lib_deps",
     "tf_additional_lib_hdrs",
@@ -443,6 +444,18 @@ cc_library(
     ] + tf_additional_human_readable_json_deps(),
 )
 
+cc_library(
+    name = "logger",
+    srcs = tf_platform_srcs(["logger.cc"]),
+    hdrs = ["platform/logger.h"] + tf_platform_hdrs(["logger.h"]),
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":lib",
+        ":lib_internal",
+    ] + tf_additional_logger_deps(),
+)
+
 filegroup(
     name = "platform_env_hdrs",
     srcs = [
@@ -1594,6 +1607,8 @@ filegroup(
             "util/stats_calculator.*",
             "util/reporter.*",
             "platform/**/cuda_libdevice_path.*",
+            "platform/**/logger.cc",
+            "platform/**/logger.h",
             "platform/default/test_benchmark.*",
             "platform/cuda.h",
             "platform/google/**/*",
@@ -2206,6 +2221,7 @@ cc_library(
             "platform/**/env_time.cc",
             "platform/**/cuda_libdevice_path.cc",
             "platform/**/device_tracer.cc",
+            "platform/**/logger.cc",
             "platform/**/logging.cc",
             "platform/**/human_readable_json.cc",
             "platform/abi.cc",
@@ -2218,6 +2234,7 @@ cc_library(
             "platform/**/stream_executor.h",
             "platform/**/env_time.cc",
             "platform/**/device_tracer.cc",
+            "platform/**/logger.cc",
             "platform/**/logging.cc",
             "platform/**/human_readable_json.cc",
             "platform/abi.cc",
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index 3a4415f229..0428715130 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -543,6 +543,9 @@ def tf_additional_proto_srcs():
 def tf_additional_human_readable_json_deps():
     return []
 
+def tf_additional_logger_deps():
+    return []
+
 def tf_additional_all_protos():
     return ["//tensorflow/core:protos_all"]
 
diff --git a/tensorflow/core/platform/default/logger.cc b/tensorflow/core/platform/default/logger.cc
new file mode 100644
index 0000000000..54b1a1a67c
--- /dev/null
+++ b/tensorflow/core/platform/default/logger.cc
@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/logger.h"
+
+#include "tensorflow/core/platform/logging.h"
+
+namespace tensorflow {
+
+Logger* Logger::Singleton() {
+  class DefaultLogger : public Logger {
+   private:
+    void DoLogProto(google::protobuf::Any* proto) override {
+      VLOG(2) << proto->ShortDebugString();
+    }
+    void DoFlush() override {}
+  };
+  static Logger* instance = new DefaultLogger();
+  return instance;
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/logger.h b/tensorflow/core/platform/logger.h
new file mode 100644
index 0000000000..5d304bea63
--- /dev/null
+++ b/tensorflow/core/platform/logger.h
@@ -0,0 +1,51 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_LOGGER_H_
+#define TENSORFLOW_CORE_PLATFORM_LOGGER_H_
+
+#include "google/protobuf/any.pb.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+namespace tensorflow {
+
+// Abstract logging interface. Contrary to logging.h, this class describes an
+// interface, not a concrete logging mechanism. This is useful when we want to
+// log anything to a non-local place, e.g. a database.
+class Logger {
+ public:
+  static Logger* Singleton();
+
+  virtual ~Logger() = default;
+
+  // Logs a typed proto.
+  template <typename ProtoType>
+  void LogProto(const ProtoType& proto) {
+    google::protobuf::Any any;
+    any.PackFrom(proto);
+    DoLogProto(&any);
+  }
+
+  // Flushes any pending log. Blocks until everything is flushed.
+  void Flush() { DoFlush(); }
+
+ private:
+  virtual void DoLogProto(google::protobuf::Any* proto) = 0;
+  virtual void DoFlush() = 0;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PLATFORM_LOGGER_H_
-- 
GitLab


From f00608d7bc88aaa1cd6f1ce46fa28271860b3723 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 14:52:12 -0800
Subject: [PATCH 0811/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 222891342

---
 tensorflow/go/op/wrappers.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index ad31c335e6..02a1335149 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -463,6 +463,14 @@ func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
 	}
 }
 
+// QuantizeAndDequantizeV2RoundMode sets the optional round_mode attribute to value.
+// If not specified, defaults to "HALF_TO_EVEN"
+func QuantizeAndDequantizeV2RoundMode(value string) QuantizeAndDequantizeV2Attr {
+	return func(m optionalAttr) {
+		m["round_mode"] = value
+	}
+}
+
 // Quantizes then dequantizes a tensor.
 //
 // This op simulates the precision loss from the quantized forward pass by:
-- 
GitLab


From ac5311fcf66d0156b71ff17c0777b67dc0251448 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Mon, 26 Nov 2018 14:53:52 -0800
Subject: [PATCH 0812/1554] Switch to using Eigen::NumTraits instead of
 std::numeric_limits when checking bounds.

PiperOrigin-RevId: 222891638
---
 tensorflow/core/grappler/BUILD                |  1 +
 .../optimizers/arithmetic_optimizer_test.cc   | 27 +++++++++++++++++++
 tensorflow/core/grappler/utils.cc             |  4 +--
 tensorflow/core/grappler/utils_test.cc        | 23 ++++++++++++++++
 4 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 7b03ec38bf..7982b35853 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -41,6 +41,7 @@ tf_cc_test(
         "//tensorflow/core:all_kernels",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensor_testutil",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
     ],
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index b6286c425e..35d22898f6 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/arithmetic_optimizer.h"
+#include "tensorflow/cc/ops/math_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
@@ -3793,5 +3794,31 @@ TEST_F(ArithmeticOptimizerTest, RemoveStackStridedSliceSameAxis) {
                                  tensors[fCSlice2ToOut]);
 }
 
+TEST_F(ArithmeticOptimizerTest, SimplifyAggregationBFloat16) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2});
+  Output cast = ops::Cast(s.WithOpName("cast"), x, DT_BFLOAT16);
+  Output add = ops::AddN(s.WithOpName("add"), {cast, cast});
+  Output id = ops::Identity(s.WithOpName("id"), add);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  item.fetch = {"id"};
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlySimplifyAggregation(&optimizer);
+  OptimizeAndPrune(&optimizer, &item, &output);
+
+  // Extra node created for multiplier.
+  EXPECT_EQ(5, output.node_size());
+
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<bfloat16>(tensors_expected[0], tensors[0]);
+}
+
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index 9336c4df8b..2977544262 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -40,8 +40,8 @@ namespace {
 template <typename T>
 bool SafeSetScalarTensorValue(double value, Tensor* tensor) {
   using RealType = typename Eigen::NumTraits<T>::Real;
-  if (value > static_cast<double>(std::numeric_limits<RealType>::max()) ||
-      value < static_cast<double>(std::numeric_limits<RealType>::min())) {
+  if (value > static_cast<double>(Eigen::NumTraits<RealType>::highest()) ||
+      value < static_cast<double>(Eigen::NumTraits<RealType>::lowest())) {
     return false;
   }
   tensor->flat<T>()(0) = static_cast<T>(value);
diff --git a/tensorflow/core/grappler/utils_test.cc b/tensorflow/core/grappler/utils_test.cc
index 8cbff1c397..e993391b51 100644
--- a/tensorflow/core/grappler/utils_test.cc
+++ b/tensorflow/core/grappler/utils_test.cc
@@ -16,10 +16,13 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils.h"
 
 #include <unistd.h>
+#include <limits>
 #include <memory>
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/lib/bfloat16/bfloat16.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
@@ -441,6 +444,26 @@ BM_ParseNodeNameAsStringPiece("foo:123", foo123);
 BM_ParseNodeNameAsStringPiece("foo/bar/baz:123", foo_bar_baz_123);
 BM_ParseNodeNameAsStringPiece("^foo/bar/baz:123", foo_bar_baz_123_ctrl);
 
+TEST_F(UtilsTest, SetTensorValueBFloat16) {
+  Tensor t(DT_BFLOAT16, TensorShape({}));
+  TF_ASSERT_OK(SetTensorValue(t.dtype(), 2, &t));
+  test::ExpectTensorEqual<bfloat16>(Tensor(bfloat16(2)), t);
+}
+
+TEST_F(UtilsTest, SetTensorValueBFloat16IntMax) {
+  Tensor t(DT_BFLOAT16, TensorShape({}));
+  TF_ASSERT_OK(SetTensorValue(t.dtype(), std::numeric_limits<int>::max(), &t));
+  test::ExpectTensorEqual<bfloat16>(
+      Tensor(bfloat16(std::numeric_limits<int>::max())), t);
+}
+
+TEST_F(UtilsTest, SetTensorValueBFloat16IntMin) {
+  Tensor t(DT_BFLOAT16, TensorShape({}));
+  TF_ASSERT_OK(SetTensorValue(t.dtype(), std::numeric_limits<int>::min(), &t));
+  test::ExpectTensorEqual<bfloat16>(
+      Tensor(bfloat16(std::numeric_limits<int>::min())), t);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 11f77c8e22dc1040867356fd3475bc9264ae9e88 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Mon, 26 Nov 2018 14:54:26 -0800
Subject: [PATCH 0813/1554] Add TFLite specific template for select/missing
 ops.

PiperOrigin-RevId: 222891735
---
 .../ISSUE_TEMPLATE/40-tflite-op-request.md    | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/40-tflite-op-request.md

diff --git a/.github/ISSUE_TEMPLATE/40-tflite-op-request.md b/.github/ISSUE_TEMPLATE/40-tflite-op-request.md
new file mode 100644
index 0000000000..7b391279e4
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/40-tflite-op-request.md
@@ -0,0 +1,24 @@
+---
+name: TensorFlow Lite Op Request
+about: Use this template for reporting ops you are using or missing.
+
+---
+
+
+**System information**
+- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):
+- TensorFlow installed from (source or binary):
+- TensorFlow version (or github SHA if from source):
+
+
+**Provide the text output from tflite_convert**
+
+```
+# Copy and paste here
+```
+
+Also, please include a link to a GraphDef or the model if possible.
+
+**Any other info / logs**
+
+Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.
-- 
GitLab


From c81c9a4cf223149c5bda4f6bd9c8424272c6c83e Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 26 Nov 2018 15:05:10 -0800
Subject: [PATCH 0814/1554] [XLA] Don't create invalid output fusions

We were incorrectly allowing Ax+B fusions where B was also the `Ax` dot
operation.

PiperOrigin-RevId: 222893743
---
 .../xla/service/cpu/cpu_instruction_fusion.cc |  7 +++++-
 .../cpu/cpu_instruction_fusion_test.cc        | 22 ++++++++++++++++++
 .../xla/service/gpu/instruction_fusion.cc     |  3 ++-
 .../service/gpu/instruction_fusion_test.cc    | 23 +++++++++++++++++++
 4 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc
index f9cd61bea3..6f79ad7c14 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.cc
@@ -48,10 +48,15 @@ bool IsMatrixVectorDot(const HloInstruction* hlo) {
          (hlo_shape.dimensions(0) == 1 || hlo_shape.dimensions(1) == 1);
 }
 
+bool HasExactlyOneUse(const HloInstruction& hlo_instr) {
+  return hlo_instr.user_count() == 1 &&
+         absl::c_count(hlo_instr.users().front()->operands(), &hlo_instr) == 1;
+}
+
 bool CanBeOutputFused(const HloInstruction* producer,
                       const HloInstruction* consumer) {
   return consumer->opcode() == HloOpcode::kAdd && IsMatrixVectorDot(producer) &&
-         producer->user_count() == 1;
+         HasExactlyOneUse(*producer) == 1;
 }
 
 bool CanBeOutputFusedIntoSomeOperand(const HloInstruction* consumer) {
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index c77d5988ba..527df0bd1c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -763,6 +763,28 @@ TEST_F(OpcodeFusionTest, DotAddOutputFusion_19x50x1_multi_use) {
               Not(op::Fusion()));
 }
 
+TEST_F(InstructionFusionTest,
+       DotOperationFusion_DontOutputFuseDuplicateOperands) {
+  absl::string_view module_string = R"(
+HloModule module
+
+ENTRY main {
+  a = f32[50,60]{1,0} parameter(0)
+  b = f32[60,1]{1,0} parameter(1)
+  c = f32[50,1]{1,0} dot(a, b), lhs_contracting_dims={1}, rhs_contracting_dims={0}
+  ROOT d = f32[50,1]{1,0} add(c, c)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_string));
+  TF_ASSERT_OK_AND_ASSIGN(bool fused_something,
+                          CpuInstructionFusion().Run(module.get()));
+  EXPECT_FALSE(fused_something);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              Not(op::Fusion()));
+}
+
 struct GatherLoopFusionTestSpec {
   string test_name;
   string hlo_computation_text;
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
index 985c0d411e..6151dd8ff4 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
@@ -181,7 +181,8 @@ bool GpuInstructionFusion::ShouldFuse(HloInstruction* consumer,
         return true;
       }
     } else if (consumer->operand_count() == 2 &&
-               consumer->opcode() == HloOpcode::kAdd) {
+               consumer->opcode() == HloOpcode::kAdd &&
+               consumer->operand(other_operand_index) != producer) {
       // Fuse a bias add into the output of the dot.
       return true;
     }
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
index 2b060b03ce..688604cd36 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
@@ -358,6 +358,29 @@ TEST_F(InstructionFusionTest, DotOutputFusionBiasAdd) {
                       op::Parameter()));
 }
 
+TEST_F(InstructionFusionTest,
+       DotOperationFusion_DontOutputFuseDuplicateOperands) {
+  absl::string_view module_string = R"(
+HloModule module
+
+ENTRY main {
+  a = f32[50,60]{1,0} parameter(0)
+  b = f32[60,1]{1,0} parameter(1)
+  c = f32[50,1]{1,0} dot(a, b), lhs_contracting_dims={1}, rhs_contracting_dims={0}
+  ROOT d = f32[50,1]{1,0} add(c, c)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_string));
+  TF_ASSERT_OK_AND_ASSIGN(
+      bool fused_something,
+      GpuInstructionFusion(/*may_duplicate=*/false).Run(module.get()));
+  EXPECT_FALSE(fused_something);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              Not(op::Fusion()));
+}
+
 // Compute sum(1/p0), where p0 has type f32, twice.  Check that the division is
 // duplicated and fused into both reduces.
 TEST_F(InstructionFusionTest, FloatingPointDivIsCheap) {
-- 
GitLab


From 6a9b639385e48672345345adfc3a79f7ff49068e Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 26 Nov 2018 15:08:11 -0800
Subject: [PATCH 0815/1554] Remove train.LooperThread from TF 2.0 API.

PiperOrigin-RevId: 222894257
---
 tensorflow/python/training/coordinator.py     |  2 +-
 .../v2/tensorflow.train.-looper-thread.pbtxt  | 73 -------------------
 .../api/golden/v2/tensorflow.train.pbtxt      |  4 -
 tensorflow/tools/compatibility/renames_v2.py  |  1 +
 4 files changed, 2 insertions(+), 78 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt

diff --git a/tensorflow/python/training/coordinator.py b/tensorflow/python/training/coordinator.py
index 0ff97d85e3..b7e5c98c78 100644
--- a/tensorflow/python/training/coordinator.py
+++ b/tensorflow/python/training/coordinator.py
@@ -408,7 +408,7 @@ class Coordinator(object):
 
 
 # Threads for the standard services.
-@tf_export("train.LooperThread")
+@tf_export(v1=["train.LooperThread"])
 class LooperThread(threading.Thread):
   """A thread that runs code repeatedly, optionally on a timer.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt
deleted file mode 100644
index c61859004e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-looper-thread.pbtxt
+++ /dev/null
@@ -1,73 +0,0 @@
-path: "tensorflow.train.LooperThread"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.coordinator.LooperThread\'>"
-  is_instance: "<class \'threading.Thread\'>"
-  member {
-    name: "daemon"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "ident"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'coord\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "getName"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "isAlive"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "isDaemon"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "is_alive"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "join"
-    argspec: "args=[\'self\', \'timeout\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "loop"
-    argspec: "args=[\'coord\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "run"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "run_loop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "setDaemon"
-    argspec: "args=[\'self\', \'daemonic\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "setName"
-    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "start"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "start_loop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "stop_loop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 9c221ddea3..91e1e0582e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -112,10 +112,6 @@ tf_module {
     name: "LoggingTensorHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "LooperThread"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "MomentumOptimizer"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index b02326278e..e01360a060 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -549,6 +549,7 @@ renames = {
     'tf.to_int32': 'tf.compat.v1.to_int32',
     'tf.to_int64': 'tf.compat.v1.to_int64',
     'tf.trace': 'tf.linalg.trace',
+    'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread',
     'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession',
     'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader',
     'tf.train.ProfilerHook': 'tf.compat.v1.train.ProfilerHook',
-- 
GitLab


From 768b36822e0d5b988a697e0c9e3b65302b051630 Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 26 Nov 2018 15:45:08 -0800
Subject: [PATCH 0816/1554]  fixing the buildifer error of
 tensorflow/core/kernels/BUILD

---
 tensorflow/core/kernels/BUILD | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1759a7f790..efa571b23b 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -29,26 +29,26 @@ package_group(
 
 load(
     "//tensorflow:tensorflow.bzl",
+    "cc_header_only_library",
     "if_android",
+    "if_not_windows",
+    "tf_cc_binary",
     "tf_cc_test",
     "tf_cc_test_mkl",
     "tf_cc_tests",
-    "tf_cc_binary",
     "tf_copts",
     "tf_cuda_library",
-    "tf_opts_nortti_if_android",
     "tf_kernel_library",
     "tf_mkl_kernel_library",
-    "cc_header_only_library",
-    "if_not_windows",
+    "tf_opts_nortti_if_android",
 )
 load("@local_config_sycl//sycl:build_defs.bzl", "if_sycl")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_tests")
 load(
     "//tensorflow/core:platform/default/build_config.bzl",
-    "tf_proto_library",
     "tf_kernel_tests_linkstatic",
+    "tf_proto_library",
 )
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
@@ -6714,10 +6714,10 @@ tf_cc_test_mkl(
     srcs = ["mkl_fused_ops_test.cc"],
     linkstatic = 1,
     deps = [
-        ":mkl_conv_op",
-        ":mkl_tfconv_op",
         ":conv_ops",
         ":image",
+        ":mkl_conv_op",
+        ":mkl_tfconv_op",
         ":ops_testutil",
         ":ops_util",
         "//tensorflow/cc:cc_ops",
@@ -6730,8 +6730,9 @@ tf_cc_test_mkl(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-    ]
+    ],
 )
+
 tf_mkl_kernel_library(
     name = "mkl_transpose_op",
     srcs = [
-- 
GitLab


From f01c4b51d715a5fa903ac2876423b74d085f51f4 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 26 Nov 2018 15:35:38 -0800
Subject: [PATCH 0817/1554] Infer signatures for functional tf.keras.Models in
 tf.saved_model.save

Allows exporting functional Models without any @tf.function decoration.

PiperOrigin-RevId: 222898916
---
 tensorflow/python/saved_model/save.py      | 68 +++++++++++++++++-----
 tensorflow/python/saved_model/save_test.py | 57 ++++++++++++++++++
 2 files changed, 111 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 02c8dc7c13..d52251e49f 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -27,6 +27,7 @@ from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -42,17 +43,50 @@ from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
 
+def _check_for_functional_keras_model(root):
+  """Makes an export signature for `root` if it's a functional Keras Model."""
+  # If nothing is decorated yet but this is a functional Keras Model (duck
+  # typed), we'll try to make a signature ourselves.
+  try:
+    inputs = root.inputs
+    input_names = root.input_names
+  except AttributeError:
+    return None
+  input_signature = []
+  for input_tensor, input_name in zip(inputs, input_names):
+    input_signature.append(tensor_spec.TensorSpec(
+        shape=input_tensor.shape, dtype=input_tensor.dtype,
+        name=input_name))
+
+  @def_function.function(input_signature=input_signature)
+  def _wrapped_model(*args):
+    outputs_list = nest.flatten(root(inputs=list(args)))
+    return {name: output for name, output
+            in zip(root.output_names, outputs_list)}
+  return _wrapped_model
+
+
 def _find_function_to_export(root):
   """Iterate over `root`'s attributes, finding traced functions."""
-  functions = []
-  function_attribute_names = []
+  exported_function = None
+  previous_attribute_name = None
   for attribute_name in dir(root):
     attribute_value = getattr(root, attribute_name, None)
     if isinstance(attribute_value, def_function.PolymorphicFunction):
-      functions.append(attribute_value)
-      function_attribute_names.append(attribute_name)
-  # TODO(allenl): Automatically infer signatures for Keras functional models?
-  if not functions:
+      if exported_function is not None:
+        raise ValueError(
+            ("Exporting an object with no "
+             "tf.saved_model.save(..., signatures=...) "
+             "argument specified, and with more than one "
+             "@tf.function-decorated method attached to it: {}. The signature "
+             "keys for these functions are ambiguous. Specify signature "
+             "functions explicitly.").format(
+                 [previous_attribute_name, attribute_name]))
+      exported_function = attribute_value
+      previous_attribute_name = attribute_name
+  if exported_function is None:
+    exported_function = _check_for_functional_keras_model(root)
+  if exported_function is None:
     raise ValueError(
         ("Exporting an object with no tf.saved_model.save(..., signatures=...) "
          "argument specified, and with no @tf.function-decorated methods "
@@ -61,14 +95,7 @@ def _find_function_to_export(root):
          "signatures does not make sense, as the only consumers will expect "
          "signatures. Either decorate a method or specify a signature function "
          "explicitly."))
-  elif len(functions) > 1:
-    raise ValueError(
-        ("Exporting an object with no tf.saved_model.save(..., signatures=...) "
-         "argument specified, and with more than one @tf.function-decorated "
-         "method attached to it: {}. The signature keys for these functions "
-         "are ambiguous. Specify signature functions explicitly.").format(
-             function_attribute_names))
-  return functions[0]
+  return exported_function
 
 
 def _canonicalize_signatures(signatures):
@@ -451,6 +478,19 @@ def save(obj, export_dir, signatures=None):
           tf.TensorSpec(shape=[None, 3], dtype=tf.float32, name="inp")))
   ```
 
+  `tf.keras.Model` instances constructed from inputs and outputs already have a
+  signature and so do not require a `@tf.function` decorator or a `signatures`
+  argument. If neither are specified, the model's forward pass is exported.
+
+  ```python
+  x = input_layer.Input((4,), name="x")
+  y = core.Dense(5, name="out")(x)
+  model = training.Model(x, y)
+  tf.saved_model.save(model, '/tmp/saved_model/')
+  # The exported SavedModel takes "x" with shape [None, 4] and returns "out"
+  # with shape [None, 5]
+  ```
+
   Variables must be tracked by assigning them to an attribute of a tracked
   object or to an attribute of `obj` directly. TensorFlow objects (e.g. layers
   from `tf.keras.layers`, optimizers from `tf.train`) track their variables
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 04cd9d0683..8fb28039a4 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import os
 import sys
 
+import numpy
+
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
@@ -29,8 +31,11 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras.engine import input_layer
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
+from tensorflow.python.keras.layers import merge
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import loader
@@ -214,6 +219,19 @@ class SaveTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "call.*second_function"):
       save.save(model, save_dir)
 
+  def test_subclassed_no_signature(self):
+
+    class Subclassed(training.Model):
+
+      def call(self, inputs):
+        return inputs * 2.
+
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    model = Subclassed()
+    with self.assertRaisesRegexp(
+        ValueError, "no @tf.function-decorated methods"):
+      save.save(model, save_dir)
+
   def test_docstring(self):
 
     class Adder(util.Checkpoint):
@@ -254,6 +272,45 @@ class SaveTest(test.TestCase):
       self.assertNotIn("T", complex_node.attr)
       self.assertNotIn("Tout", complex_node.attr)
 
+  def test_export_functional_keras_model(self):
+    x = input_layer.Input((4,), name="x")
+    y = core.Dense(4, name="out")(x)
+    model = training.Model(x, y)
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    save.save(model, save_dir)
+    self.assertAllClose(
+        {"out": model(array_ops.ones([1, 4]))},
+        self._import_and_infer(save_dir, {"x": [[1., 1., 1., 1.]]}))
+
+  def test_export_functional_keras_model_after_fit(self):
+    x = input_layer.Input((1,))
+    y = core.Dense(1, name="y")(x)
+    model = training.Model(x, y)
+    model.compile(optimizer="sgd", loss="mse")
+    model.fit(x=numpy.array([[1.]]),
+              y=numpy.array([2.]), epochs=2)
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    save.save(model, save_dir)
+    self.assertAllClose(
+        {"y": model(constant_op.constant([[1.], [2.]]))},
+        self._import_and_infer(save_dir, {"input_1": [[1.], [2.]]}))
+
+  def test_export_multi_input_functional_keras_model(self):
+    x1 = input_layer.Input((2,), name="x1")
+    x2 = input_layer.Input((2,), name="x2")
+    y1 = core.Dense(4)(merge.Add()([x1, x2]))
+    y2 = core.Dense(4)(merge.Multiply()([x1, x2]))
+    model = training.Model([x1, x2], [y1, y2])
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    save.save(model, save_dir)
+    outputs = model([array_ops.ones([1, 2]), 2. * array_ops.ones([1, 2])])
+    self.assertAllClose(
+        {"dense": outputs[0], "dense_1": outputs[1]},
+        self._import_and_infer(
+            save_dir,
+            {"x1": [[1., 1.]],
+             "x2": [[2., 2.]]}))
+
 
 class MemoryTests(test.TestCase):
 
-- 
GitLab


From 04e95d6b4c67920cc7e442b89a150a88b86fea08 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 26 Nov 2018 15:36:50 -0800
Subject: [PATCH 0818/1554] [tf.data] Test and benchmark cleanup.

PiperOrigin-RevId: 222899136
---
 tensorflow/python/data/benchmarks/BUILD       |  55 ++
 .../python/data/benchmarks/batch_benchmark.py |  85 +++
 .../data/benchmarks/filter_benchmark.py       |  69 ++
 .../from_tensor_slices_benchmark.py           | 188 +++++
 .../python/data/benchmarks/map_benchmark.py   | 135 ++++
 .../python/data/experimental/benchmarks/BUILD |  63 +-
 .../benchmarks/autotune_benchmark.py          | 187 +++++
 .../benchmarks/csv_dataset_benchmark.py       | 129 ++++
 .../benchmarks/map_and_batch_benchmark.py     | 129 ++++
 .../experimental/benchmarks/map_benchmark.py  | 245 -------
 .../benchmarks/map_vectorization_benchmark.py | 194 +++++
 .../benchmarks/matching_files_benchmark.py    |   2 +-
 .../benchmarks/optimize_benchmark.py          | 120 +++
 .../data/experimental/kernel_tests/BUILD      |   4 -
 .../kernel_tests/batch_dataset_op_test.py     | 688 ------------------
 .../kernel_tests/csv_dataset_test.py          |  99 ---
 .../kernel_tests/optimization/BUILD           |   8 +-
 .../optimization/map_vectorization_test.py    | 101 ---
 .../optimization/model_dataset_test.py        | 164 -----
 tensorflow/python/data/kernel_tests/BUILD     | 413 +++++++----
 .../python/data/kernel_tests/batch_test.py    | 173 +++++
 ...cache_dataset_op_test.py => cache_test.py} |   6 +-
 ...dataset_op_test.py => concatenate_test.py} |   4 +-
 ...tal_test.py => dataset_checkpoint_test.py} | 331 +--------
 .../dataset_constructor_op_test.py            | 606 ---------------
 .../{dataset_ops_test.py => dataset_test.py}  |   7 +-
 ...lter_dataset_op_test.py => filter_test.py} |  45 +-
 .../fixed_length_record_dataset_test.py       | 171 +++++
 ...ap_dataset_op_test.py => flat_map_test.py} |   4 +-
 ...ator_op_test.py => from_generator_test.py} |   4 +-
 .../from_sparse_tensor_slices_test.py         |  85 +++
 .../kernel_tests/from_tensor_slices_test.py   | 177 +++++
 .../data/kernel_tests/from_tensors_test.py    | 258 +++++++
 .../python/data/kernel_tests/inputs_test.py   | 151 ----
 ..._dataset_op_test.py => interleave_test.py} | 151 ++--
 .../kernel_tests/iterator_checkpoint_test.py  | 129 ++++
 ...uster_test.py => iterator_cluster_test.py} |   2 +-
 ...{iterator_ops_test.py => iterator_test.py} | 100 +--
 ..._dataset_op_test.py => list_files_test.py} |   5 +-
 .../{map_dataset_op_test.py => map_test.py}   | 109 +--
 .../multi_device_iterator_test.py             |   4 +-
 ...{optional_ops_test.py => optional_test.py} |   2 +-
 ...ataset_op_test.py => padded_batch_test.py} | 196 +----
 ...ch_dataset_op_test.py => prefetch_test.py} |   5 +-
 ...range_dataset_op_test.py => range_test.py} |   5 +-
 .../kernel_tests/reader_dataset_ops_test.py   | 446 ------------
 ...duce_dataset_op_test.py => reduce_test.py} |   4 +-
 ...ence_dataset_op_test.py => repeat_test.py} |  55 +-
 ...shard_dataset_op_test.py => shard_test.py} |   5 +-
 ...fle_dataset_op_test.py => shuffle_test.py} |   4 +-
 .../python/data/kernel_tests/skip_test.py     |  62 ++
 .../python/data/kernel_tests/take_test.py     |  55 ++
 .../kernel_tests/text_line_dataset_test.py    | 165 +++++
 .../kernel_tests/tf_record_dataset_test.py    | 170 +++++
 ...ndow_dataset_op_test.py => window_test.py} |   4 +-
 .../{zip_dataset_op_test.py => zip_test.py}   |   4 +-
 56 files changed, 3150 insertions(+), 3632 deletions(-)
 create mode 100644 tensorflow/python/data/benchmarks/batch_benchmark.py
 create mode 100644 tensorflow/python/data/benchmarks/filter_benchmark.py
 create mode 100644 tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
 create mode 100644 tensorflow/python/data/benchmarks/map_benchmark.py
 create mode 100644 tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
 create mode 100644 tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
 delete mode 100644 tensorflow/python/data/experimental/benchmarks/map_benchmark.py
 create mode 100644 tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
 create mode 100644 tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
 delete mode 100644 tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
 create mode 100644 tensorflow/python/data/kernel_tests/batch_test.py
 rename tensorflow/python/data/kernel_tests/{cache_dataset_op_test.py => cache_test.py} (98%)
 rename tensorflow/python/data/kernel_tests/{concatenate_dataset_op_test.py => concatenate_test.py} (98%)
 rename tensorflow/python/data/kernel_tests/{save_restore_experimental_test.py => dataset_checkpoint_test.py} (51%)
 delete mode 100644 tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
 rename tensorflow/python/data/kernel_tests/{dataset_ops_test.py => dataset_test.py} (98%)
 rename tensorflow/python/data/kernel_tests/{filter_dataset_op_test.py => filter_test.py} (77%)
 create mode 100644 tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py
 rename tensorflow/python/data/kernel_tests/{flat_map_dataset_op_test.py => flat_map_test.py} (97%)
 rename tensorflow/python/data/kernel_tests/{dataset_from_generator_op_test.py => from_generator_test.py} (99%)
 create mode 100644 tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
 create mode 100644 tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
 create mode 100644 tensorflow/python/data/kernel_tests/from_tensors_test.py
 delete mode 100644 tensorflow/python/data/kernel_tests/inputs_test.py
 rename tensorflow/python/data/kernel_tests/{interleave_dataset_op_test.py => interleave_test.py} (69%)
 create mode 100644 tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
 rename tensorflow/python/data/kernel_tests/{iterator_ops_cluster_test.py => iterator_cluster_test.py} (98%)
 rename tensorflow/python/data/kernel_tests/{iterator_ops_test.py => iterator_test.py} (87%)
 rename tensorflow/python/data/kernel_tests/{list_files_dataset_op_test.py => list_files_test.py} (98%)
 rename tensorflow/python/data/kernel_tests/{map_dataset_op_test.py => map_test.py} (90%)
 rename tensorflow/python/data/kernel_tests/{optional_ops_test.py => optional_test.py} (99%)
 rename tensorflow/python/data/kernel_tests/{batch_dataset_op_test.py => padded_batch_test.py} (58%)
 rename tensorflow/python/data/kernel_tests/{prefetch_dataset_op_test.py => prefetch_test.py} (93%)
 rename tensorflow/python/data/kernel_tests/{range_dataset_op_test.py => range_test.py} (96%)
 delete mode 100644 tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
 rename tensorflow/python/data/kernel_tests/{reduce_dataset_op_test.py => reduce_test.py} (97%)
 rename tensorflow/python/data/kernel_tests/{sequence_dataset_op_test.py => repeat_test.py} (67%)
 rename tensorflow/python/data/kernel_tests/{shard_dataset_op_test.py => shard_test.py} (96%)
 rename tensorflow/python/data/kernel_tests/{shuffle_dataset_op_test.py => shuffle_test.py} (98%)
 create mode 100644 tensorflow/python/data/kernel_tests/skip_test.py
 create mode 100644 tensorflow/python/data/kernel_tests/take_test.py
 create mode 100644 tensorflow/python/data/kernel_tests/text_line_dataset_test.py
 create mode 100644 tensorflow/python/data/kernel_tests/tf_record_dataset_test.py
 rename tensorflow/python/data/kernel_tests/{window_dataset_op_test.py => window_test.py} (98%)
 rename tensorflow/python/data/kernel_tests/{zip_dataset_op_test.py => zip_test.py} (97%)

diff --git a/tensorflow/python/data/benchmarks/BUILD b/tensorflow/python/data/benchmarks/BUILD
index fd723e0d71..5b0500eae1 100644
--- a/tensorflow/python/data/benchmarks/BUILD
+++ b/tensorflow/python/data/benchmarks/BUILD
@@ -6,6 +6,61 @@ exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
+py_test(
+    name = "batch_benchmark",
+    srcs = ["batch_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:session",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "filter_benchmark",
+    srcs = ["filter_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "from_tensor_slices_benchmark",
+    srcs = ["from_tensor_slices_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "map_benchmark",
+    srcs = ["map_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "range_benchmark",
     srcs = ["range_benchmark.py"],
diff --git a/tensorflow/python/data/benchmarks/batch_benchmark.py b/tensorflow/python/data/benchmarks/batch_benchmark.py
new file mode 100644
index 0000000000..b61ac86eb5
--- /dev/null
+++ b/tensorflow/python/data/benchmarks/batch_benchmark.py
@@ -0,0 +1,85 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for `tf.data.Dataset.batch()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks.
+class BatchBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.Dataset.batch()`."""
+
+  def benchmarkBatchSparse(self):
+    non_zeros_per_row_values = [0, 1, 5, 10, 100]
+    batch_size_values = [1, 32, 64, 128, 1024]
+
+    sparse_placeholder = array_ops.sparse_placeholder(dtype=dtypes.int64)
+    batch_size_placeholder = array_ops.placeholder(dtype=dtypes.int64, shape=[])
+
+    dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat(
+        ).batch(batch_size_placeholder)
+    iterator = dataset.make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    for non_zeros_per_row in non_zeros_per_row_values:
+
+      sparse_value = sparse_tensor.SparseTensorValue(
+          indices=np.arange(non_zeros_per_row, dtype=np.int64)[:, np.newaxis],
+          values=np.arange(non_zeros_per_row, dtype=np.int64),
+          dense_shape=[1000])
+
+      for batch_size in batch_size_values:
+
+        with session.Session() as sess:
+          sess.run(iterator.initializer, feed_dict={
+              sparse_placeholder: sparse_value,
+              batch_size_placeholder: batch_size})
+          # Run five steps to warm up the session caches before taking the
+          # first measurement.
+          for _ in range(5):
+            sess.run(next_element.indices.op)
+          deltas = []
+          for _ in range(100):
+            start = time.time()
+            for _ in range(100):
+              sess.run(next_element.indices.op)
+            end = time.time()
+            deltas.append(end - start)
+
+        median_wall_time = np.median(deltas) / 100.0
+
+        print("Batch sparse dataset non-zeros per row: %d batch_size: %d "
+              "wall time: %f"
+              % (non_zeros_per_row, batch_size, median_wall_time))
+        self.report_benchmark(
+            iters=10000, wall_time=median_wall_time,
+            name="batch_sparse_dataset_nnz_%d_batch_size_%d" % (
+                non_zeros_per_row, batch_size))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/benchmarks/filter_benchmark.py b/tensorflow/python/data/benchmarks/filter_benchmark.py
new file mode 100644
index 0000000000..b9acdc7227
--- /dev/null
+++ b/tensorflow/python/data/benchmarks/filter_benchmark.py
@@ -0,0 +1,69 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for `tf.data.Dataset.filter()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks.
+class FilterBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.Dataset.filter()`."""
+
+  def _benchmark(self, predicate, name):
+    with ops.Graph().as_default():
+      dataset = (
+          dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
+      iterator = dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
+
+      with session.Session() as sess:
+        for _ in range(5):
+          sess.run(next_element.op)
+        deltas = []
+        for _ in range(100):
+          start = time.time()
+          for _ in range(100):
+            sess.run(next_element.op)
+          end = time.time()
+          deltas.append(end - start)
+
+        median_wall_time = np.median(deltas) / 100
+        print("Filter dataset using %s. Median wall time: %f" %
+              (name, median_wall_time))
+        self.report_benchmark(
+            iters=100,
+            wall_time=median_wall_time,
+            name=name)
+
+  def benchmarkSimpleFunction(self):
+    self._benchmark(array_ops.identity, "simple_function")
+
+  def benchmarkReturnComponentOptimization(self):
+    self._benchmark(lambda x: x, "return_component")
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
new file mode 100644
index 0000000000..74a2d271ad
--- /dev/null
+++ b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
@@ -0,0 +1,188 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for `tf.data.Dataset.from_tensor_slices()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import errors
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks.
+class FromTensorSlicesBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.Dataset.from_tensor_slices()`."""
+
+  def benchmarkSliceRepeatBatch(self):
+    input_size = 10000
+    batch_size = 100
+    num_epochs = 100
+
+    input_data = np.random.randn(input_size)
+
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(input_data)
+        .repeat(num_epochs + 1).batch(batch_size))
+    iterator = dataset.make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      sess.run(iterator.initializer)
+      # Run one whole epoch to burn in the computation.
+      for _ in range(input_size // batch_size):
+        sess.run(next_element)
+      deltas = []
+      try:
+        while True:
+          start = time.time()
+          sess.run(next_element)
+          deltas.append(time.time() - start)
+      except errors.OutOfRangeError:
+        pass
+
+    median_wall_time = np.median(deltas)
+    print("Slice/repeat/batch with sess.run() input size: %d batch size: %d "
+          "Median wall time per element: %f" % (input_size, batch_size,
+                                                median_wall_time))
+    self.report_benchmark(
+        iters=len(deltas),
+        wall_time=median_wall_time,
+        name="slice_repeat_batch_input_%d_batch_%d" % (input_size, batch_size))
+
+  def benchmarkSliceRepeatBatchCallable(self):
+    input_size = 10000
+    batch_size = 100
+    num_epochs = 100
+
+    input_data = np.random.randn(input_size)
+
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(input_data)
+        .repeat(num_epochs + 1).batch(batch_size))
+    iterator = dataset.make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      sess.run(iterator.initializer)
+      get_next_element = sess.make_callable(next_element)
+      # Run one whole epoch to burn in the computation.
+      for _ in range(input_size // batch_size):
+        get_next_element()
+      deltas = []
+      try:
+        while True:
+          start = time.time()
+          get_next_element()
+          deltas.append(time.time() - start)
+      except errors.OutOfRangeError:
+        pass
+
+    median_wall_time = np.median(deltas)
+    print(
+        "Slice/repeat/batch with callable input size: %d batch size: %d Median"
+        " wall time per element: %f" % (input_size, batch_size,
+                                        median_wall_time))
+    self.report_benchmark(
+        iters=len(deltas),
+        wall_time=median_wall_time,
+        name="slice_repeat_batch_callable_input_%d_batch_%d" %
+        (input_size, batch_size))
+
+  def benchmarkReshapeSliceRepeatCallable(self):
+    input_size = 10000
+    batch_size = 100
+    num_epochs = 100
+
+    input_data = np.random.randn(input_size)
+
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100))
+        .repeat(num_epochs + 1))
+    iterator = dataset.make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      sess.run(iterator.initializer)
+      get_next_element = sess.make_callable(next_element)
+      # Run one whole epoch to burn in the computation.
+      for _ in range(input_size // batch_size):
+        get_next_element()
+      deltas = []
+      try:
+        while True:
+          start = time.time()
+          get_next_element()
+          deltas.append(time.time() - start)
+      except errors.OutOfRangeError:
+        pass
+
+    median_wall_time = np.median(deltas)
+    print("Reshape/slice/repeat with callable input size: %d batch size: %d "
+          "Median wall time per element: %f" % (input_size, batch_size,
+                                                median_wall_time))
+    self.report_benchmark(
+        iters=len(deltas),
+        wall_time=median_wall_time,
+        name="reshape_slice_repeat_callable_input_%d_batch_%d" %
+        (input_size, batch_size))
+
+  def benchmarkSliceBatchCacheRepeatCallable(self):
+    input_size = 10000
+    batch_size = 100
+    num_epochs = 100
+
+    input_data = np.random.randn(input_size)
+
+    dataset = (
+        dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size)
+        .cache().repeat(num_epochs + 1))
+    iterator = dataset.make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      sess.run(iterator.initializer)
+      get_next_element = sess.make_callable(next_element)
+      # Run one whole epoch to burn in the computation.
+      for _ in range(input_size // batch_size):
+        get_next_element()
+      deltas = []
+      try:
+        while True:
+          start = time.time()
+          get_next_element()
+          deltas.append(time.time() - start)
+      except errors.OutOfRangeError:
+        pass
+
+    median_wall_time = np.median(deltas)
+    print(
+        "Slice/batch/cache/repeat with callable input size: %d batch size: %d "
+        "Median wall time per element: %f"
+        % (input_size, batch_size, median_wall_time))
+    self.report_benchmark(
+        iters=len(deltas),
+        wall_time=median_wall_time,
+        name="slice_batch_cache_repeat_callable_input_%d_batch_%d" %
+        (input_size, batch_size))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/benchmarks/map_benchmark.py b/tensorflow/python/data/benchmarks/map_benchmark.py
new file mode 100644
index 0000000000..48294eeb89
--- /dev/null
+++ b/tensorflow/python/data/benchmarks/map_benchmark.py
@@ -0,0 +1,135 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Bechmarks for `tf.data.Dataset.map()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks.
+class MapBenchmark(test.Benchmark):
+  """Bechmarks for `tf.data.Dataset.map()`."""
+
+  def benchmarkChainOfMaps(self):
+    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
+    for chain_length in chain_lengths:
+      for mode in ["general", "single-threaded", "short-circuit"]:
+        if mode == "general":
+          map_fn = lambda x: x + 1
+          use_inter_op_parallelism = True
+          print_label = ""
+          benchmark_label = ""
+        if mode == "single-threaded":
+          map_fn = lambda x: x + 1
+          use_inter_op_parallelism = False
+          print_label = " (single threaded mode)"
+          benchmark_label = "_single_threaded"
+        if mode == "short-circuit":
+          map_fn = lambda x: x
+          use_inter_op_parallelism = True  # should not have any significance
+          print_label = " (short circuit mode)"
+          benchmark_label = "_short_circuit"
+
+        with ops.Graph().as_default():
+          dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
+          for _ in range(chain_length):
+            dataset = dataset_ops.MapDataset(
+                dataset,
+                map_fn,
+                use_inter_op_parallelism=use_inter_op_parallelism)
+          iterator = dataset.make_one_shot_iterator()
+          next_element = iterator.get_next()
+
+          with session.Session() as sess:
+            for _ in range(5):
+              sess.run(next_element.op)
+            deltas = []
+            for _ in range(100):
+              start = time.time()
+              for _ in range(100):
+                sess.run(next_element.op)
+              end = time.time()
+              deltas.append(end - start)
+
+            median_wall_time = np.median(deltas) / 100
+            print("Map dataset chain length%s: %d Median wall time: %f" %
+                  (print_label, chain_length, median_wall_time))
+            self.report_benchmark(
+                iters=1000,
+                wall_time=median_wall_time,
+                name="map_dataset_chain_length_%d%s" % (chain_length,
+                                                        benchmark_label))
+
+  def benchmarkMapFanOut(self):
+    fan_outs = [1, 2, 5, 10, 20, 50, 100]
+    for fan_out in fan_outs:
+      for mode in ["general", "single-threaded", "short-circuit"]:
+        if mode == "general":
+          map_fn = lambda *xs: [x + 1 for x in xs]
+          use_inter_op_parallelism = True
+          print_label = ""
+          benchmark_label = ""
+        if mode == "single-threaded":
+          map_fn = lambda *xs: [x + 1 for x in xs]
+          use_inter_op_parallelism = False
+          print_label = " (single threaded mode)"
+          benchmark_label = "_single_threaded"
+        if mode == "short-circuit":
+          map_fn = lambda *xs: xs
+          use_inter_op_parallelism = True  # should not have any significance
+          print_label = " (short circuit mode)"
+          benchmark_label = "_short_circuit"
+
+        with ops.Graph().as_default():
+          dataset = dataset_ops.Dataset.from_tensors(
+              tuple(0 for _ in range(fan_out))).repeat(None)
+          dataset = dataset_ops.MapDataset(
+              dataset,
+              map_fn,
+              use_inter_op_parallelism=use_inter_op_parallelism)
+          iterator = dataset.make_one_shot_iterator()
+          next_element = iterator.get_next()
+
+          with session.Session() as sess:
+            for _ in range(5):
+              sess.run(next_element[0].op)
+            deltas = []
+            for _ in range(100):
+              start = time.time()
+              for _ in range(100):
+                sess.run(next_element[0].op)
+              end = time.time()
+              deltas.append(end - start)
+
+            median_wall_time = np.median(deltas) / 100
+            print("Map dataset fan out%s: %d Median wall time: %f" %
+                  (print_label, fan_out, median_wall_time))
+            self.report_benchmark(
+                iters=1000,
+                wall_time=median_wall_time,
+                name="map_dataset_fan_out_%d%s" % (fan_out, benchmark_label))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index c48647a218..075863d34f 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -7,32 +7,67 @@ exports_files(["LICENSE"])
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "py_test")
 
+py_test(
+    name = "csv_dataset_benchmark",
+    srcs = ["csv_dataset_benchmark.py"],
+    srcs_version = "PY2AND3",
+    tags = ["no_pip"],
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/experimental/ops:readers",
+        "//tensorflow/python/data/ops:readers",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "map_and_batch_benchmark",
-    size = "medium",
     srcs = ["map_and_batch_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:session",
         "//tensorflow/python/data/experimental/ops:batching",
-        "//tensorflow/python/data/experimental/ops:optimization",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
 )
 
 py_test(
-    name = "map_benchmark",
-    size = "medium",
-    srcs = ["map_benchmark.py"],
+    name = "map_vectorization_benchmark",
+    srcs = ["map_vectorization_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "autotune_benchmark",
+    srcs = ["autotune_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:session",
         "//tensorflow/python/data/experimental/ops:batching",
@@ -42,6 +77,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "optimize_benchmark",
+    srcs = ["optimize_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "matching_files_benchmark",
     size = "small",
diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
new file mode 100644
index 0000000000..b00e918338
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
@@ -0,0 +1,187 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for autotuning performance knobs."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+class AutotuneBenchmark(test.Benchmark):
+  """Benchmarks for autotuning performance knobs."""
+
+  def benchmarkMap(self):
+    k = 1024 * 1024
+    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
+                                                np.random.rand(4 * k,
+                                                               1))).repeat()
+    dataset = dataset.map(
+        math_ops.matmul, num_parallel_calls=optimization.AUTOTUNE)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    deltas = []
+    with session.Session() as sess:
+      for _ in range(5):
+        sess.run(get_next.op)
+      for _ in range(1000):
+        start = time.time()
+        sess.run(get_next.op)
+        end = time.time()
+        deltas.append(end - start)
+
+    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
+          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
+           np.max(deltas)))
+    self.report_benchmark(
+        iters=1000, wall_time=np.median(deltas), name="map_autotune")
+
+  def benchmarkMapAndBatch(self):
+    self._benchmarkMapAndBatch(numa_aware=False)
+    self._benchmarkMapAndBatch(numa_aware=True)
+
+  def _benchmarkMapAndBatch(self, numa_aware):
+    batch_size = 16
+    k = 1024 * 1024
+    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
+                                                np.random.rand(4 * k,
+                                                               1))).repeat()
+    dataset = dataset.apply(
+        batching.map_and_batch(
+            math_ops.matmul,
+            num_parallel_calls=optimization.AUTOTUNE,
+            batch_size=batch_size))
+    options = dataset_ops.Options()
+    options.experimental_numa_aware = numa_aware
+    dataset = dataset.with_options(options)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    deltas = []
+    with session.Session() as sess:
+      for _ in range(5):
+        sess.run(get_next.op)
+      for _ in range(100):
+        start = time.time()
+        sess.run(get_next.op)
+        end = time.time()
+        deltas.append(end - start)
+
+    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
+          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
+           np.max(deltas)))
+
+    self.report_benchmark(
+        iters=100,
+        wall_time=np.median(deltas),
+        name=("numa_" if numa_aware else "") + "map_and_batch_autotune")
+
+  def benchmarkInterleave(self):
+    k = 1024 * 1024
+    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
+                                                np.random.rand(4 * k,
+                                                               1))).repeat()
+    dataset = dataset.map(math_ops.matmul)
+    dataset = dataset_ops.Dataset.range(1).repeat().interleave(
+        lambda _: dataset,
+        cycle_length=10,
+        num_parallel_calls=optimization.AUTOTUNE)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    deltas = []
+    with session.Session() as sess:
+      for _ in range(5):
+        sess.run(get_next.op)
+      for _ in range(1000):
+        start = time.time()
+        sess.run(get_next.op)
+        end = time.time()
+        deltas.append(end - start)
+
+    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
+          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
+           np.max(deltas)))
+    self.report_benchmark(
+        iters=1000,
+        wall_time=np.median(deltas),
+        name="interleave_autotune")
+
+  def benchmarkMapAndInterleave(self):
+    k = 1024 * 1024
+    a = (np.random.rand(1, 8 * k), np.random.rand(8 * k, 1))
+    b = (np.random.rand(1, 4 * k), np.random.rand(4 * k, 1))
+    c = (np.random.rand(1, 2 * k), np.random.rand(2 * k, 1))
+    dataset = dataset_ops.Dataset.from_tensors((a, b, c)).repeat()
+
+    def f1(a, b, c):
+      x, y = a
+      return math_ops.matmul(x, y), b, c
+
+    def f2(a, b, c):
+      x, y = b
+      return a, math_ops.matmul(x, y), c
+
+    def f3(a, b, c):
+      x, y = c
+      return a, b, math_ops.matmul(x, y)
+
+    dataset = dataset.map(f1, num_parallel_calls=optimization.AUTOTUNE)
+    dataset = dataset_ops.Dataset.range(1).repeat().interleave(
+        lambda _: dataset,
+        num_parallel_calls=optimization.AUTOTUNE,
+        cycle_length=2)
+
+    dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE)
+    dataset = dataset_ops.Dataset.range(1).repeat().interleave(
+        lambda _: dataset,
+        num_parallel_calls=optimization.AUTOTUNE,
+        cycle_length=2)
+
+    dataset = dataset.map(f3, num_parallel_calls=optimization.AUTOTUNE)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    deltas = []
+    with session.Session() as sess:
+      for _ in range(5):
+        sess.run(get_next)
+      for _ in range(100):
+        start = time.time()
+        sess.run(get_next)
+        end = time.time()
+        deltas.append(end - start)
+
+    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
+          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
+           np.max(deltas)))
+    self.report_benchmark(
+        iters=100,
+        wall_time=np.median(deltas),
+        name="map_and_interleave_autotune")
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
new file mode 100644
index 0000000000..7eebf49c38
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
@@ -0,0 +1,129 @@
+#  Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for `tf.data.experimental.CsvDataset`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import string
+import tempfile
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import readers
+from tensorflow.python.data.ops import readers as core_readers
+from tensorflow.python.ops import parsing_ops
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import googletest
+from tensorflow.python.platform import test
+
+
+class CsvDatasetBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.experimental.CsvDataset`."""
+
+  FLOAT_VAL = '1.23456E12'
+  STR_VAL = string.ascii_letters * 10
+
+  def _setUp(self, str_val):
+    # Since this isn't test.TestCase, have to manually create a test dir
+    gfile.MakeDirs(googletest.GetTempDir())
+    self._temp_dir = tempfile.mkdtemp(dir=googletest.GetTempDir())
+
+    self._num_cols = [4, 64, 256]
+    self._num_per_iter = 5000
+    self._filenames = []
+    for n in self._num_cols:
+      fn = os.path.join(self._temp_dir, 'file%d.csv' % n)
+      with open(fn, 'wb') as f:
+        # Just write 100 rows and use `repeat`... Assumes the cost
+        # of creating an iterator is not significant
+        row = ','.join([str_val for _ in range(n)])
+        f.write('\n'.join([row for _ in range(100)]))
+      self._filenames.append(fn)
+
+  def _tearDown(self):
+    gfile.DeleteRecursively(self._temp_dir)
+
+  def _runBenchmark(self, dataset, num_cols, prefix):
+    dataset = dataset.skip(self._num_per_iter - 1)
+    deltas = []
+    for _ in range(10):
+      next_element = dataset.make_one_shot_iterator().get_next()
+      with session.Session() as sess:
+        start = time.time()
+        # NOTE: This depends on the underlying implementation of skip, to have
+        # the net effect of calling `GetNext` num_per_iter times on the
+        # input dataset. We do it this way (instead of a python for loop, or
+        # batching N inputs in one iter) so that the overhead from session.run
+        # or batch doesn't dominate. If we eventually optimize skip, this has
+        # to change.
+        sess.run(next_element)
+        end = time.time()
+      deltas.append(end - start)
+    # Median wall time per CSV record read and decoded
+    median_wall_time = np.median(deltas) / self._num_per_iter
+    print('%s num_cols: %d Median wall time: %f' % (prefix, num_cols,
+                                                    median_wall_time))
+    self.report_benchmark(
+        iters=self._num_per_iter,
+        wall_time=median_wall_time,
+        name='%s_with_cols_%d' % (prefix, num_cols))
+
+  def benchmarkMapWithFloats(self):
+    self._setUp(self.FLOAT_VAL)
+    for i in range(len(self._filenames)):
+      num_cols = self._num_cols[i]
+      kwargs = {'record_defaults': [[0.0]] * num_cols}
+      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
+      dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs))  # pylint: disable=cell-var-from-loop
+      self._runBenchmark(dataset, num_cols, 'csv_float_map_decode_csv')
+    self._tearDown()
+
+  def benchmarkMapWithStrings(self):
+    self._setUp(self.STR_VAL)
+    for i in range(len(self._filenames)):
+      num_cols = self._num_cols[i]
+      kwargs = {'record_defaults': [['']] * num_cols}
+      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
+      dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs))  # pylint: disable=cell-var-from-loop
+      self._runBenchmark(dataset, num_cols, 'csv_strings_map_decode_csv')
+    self._tearDown()
+
+  def benchmarkCsvDatasetWithFloats(self):
+    self._setUp(self.FLOAT_VAL)
+    for i in range(len(self._filenames)):
+      num_cols = self._num_cols[i]
+      kwargs = {'record_defaults': [[0.0]] * num_cols}
+      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
+      dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat()  # pylint: disable=cell-var-from-loop
+      self._runBenchmark(dataset, num_cols, 'csv_float_fused_dataset')
+    self._tearDown()
+
+  def benchmarkCsvDatasetWithStrings(self):
+    self._setUp(self.STR_VAL)
+    for i in range(len(self._filenames)):
+      num_cols = self._num_cols[i]
+      kwargs = {'record_defaults': [['']] * num_cols}
+      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
+      dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat()  # pylint: disable=cell-var-from-loop
+      self._runBenchmark(dataset, num_cols, 'csv_strings_fused_dataset')
+    self._tearDown()
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
index a90156cd33..1e8dd0f63d 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import hashlib
+import itertools
 import time
 
 import numpy as np
@@ -25,11 +27,15 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 
+_NUMPY_RANDOM_SEED = 42
+
 
 class MapAndBatchBenchmark(test.Benchmark):
   """Benchmarks for `tf.data.experimental.map_and_batch()`."""
@@ -89,6 +95,129 @@ class MapAndBatchBenchmark(test.Benchmark):
             name="benchmark_batch_dense_dataset_nnz_%d_batch_size_%d" % (
                 np.prod(shape), batch_size))
 
+  def benchmarkMapAndBatchChainingVersusFusing(self):
+    """Compares the performance of chaining and fusing map and batch.
+
+    NOTE: It is recommended to build the benchmark with
+    `-c opt --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-gmlt`
+    and execute it on a machine with at least 32 CPU cores.
+    """
+
+    # Sequential pipeline configurations.
+    seq_elem_size_series = itertools.product([1], [1], [1, 2, 4, 8], [16])
+    seq_batch_size_series = itertools.product([1], [1], [1], [8, 16, 32, 64])
+
+    # Parallel pipeline configuration.
+    par_elem_size_series = itertools.product([32], [32], [1, 2, 4, 8], [256])
+    par_batch_size_series = itertools.product([32], [32], [1],
+                                              [128, 256, 512, 1024])
+    par_num_calls_series = itertools.product([8, 16, 32, 64], [32], [1], [512])
+    par_inter_op_series = itertools.product([32], [8, 16, 32, 64], [1], [512])
+
+    def name(method, label, num_calls, inter_op, element_size, batch_size):
+      return ("%s_id_%s_num_calls_%d_inter_op_%d_elem_size_%d_batch_size_%d" % (
+          method,
+          hashlib.sha1(label).hexdigest()[:8],
+          num_calls,
+          inter_op,
+          element_size,
+          batch_size,
+      ))
+
+    def benchmark(label, series):
+      """Runs benchmark the given series."""
+
+      print("%s:" % label)
+
+      def make_base_dataset(element_size):
+        k = 1024 * 1024
+        x = constant_op.constant(np.random.rand(element_size, 4 * k))
+        y = constant_op.constant(np.random.rand(4 * k, 1))
+        return dataset_ops.Dataset.range(1000000000000).map(lambda _: (x, y))
+
+      for num_calls, inter_op, element_size, batch_size in series:
+
+        num_iters = 1024 // (
+            (element_size * batch_size) // min(num_calls, inter_op))
+        dataset = make_base_dataset(element_size)
+        chained_dataset = dataset.map(
+            math_ops.matmul,
+            num_parallel_calls=num_calls).batch(batch_size=batch_size)
+        chained_iterator = chained_dataset.make_one_shot_iterator()
+        chained_get_next = chained_iterator.get_next()
+
+        chained_deltas = []
+        with session.Session(
+            config=config_pb2.ConfigProto(
+                inter_op_parallelism_threads=inter_op,
+                use_per_session_threads=True)) as sess:
+          for _ in range(5):
+            sess.run(chained_get_next.op)
+          for _ in range(num_iters):
+            start = time.time()
+            sess.run(chained_get_next.op)
+            end = time.time()
+            chained_deltas.append(end - start)
+
+        fused_dataset = dataset.apply(
+            batching.map_and_batch(
+                math_ops.matmul,
+                num_parallel_calls=num_calls,
+                batch_size=batch_size))
+        fused_iterator = fused_dataset.make_one_shot_iterator()
+        fused_get_next = fused_iterator.get_next()
+
+        fused_deltas = []
+        with session.Session(
+            config=config_pb2.ConfigProto(
+                inter_op_parallelism_threads=inter_op,
+                use_per_session_threads=True)) as sess:
+
+          for _ in range(5):
+            sess.run(fused_get_next.op)
+          for _ in range(num_iters):
+            start = time.time()
+            sess.run(fused_get_next.op)
+            end = time.time()
+            fused_deltas.append(end - start)
+
+        print(
+            "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, "
+            "element size: %d, num iters: %d\nchained wall time: %f (median), "
+            "%f (mean), %f (stddev), %f (min), %f (max)\n  fused wall time: "
+            "%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n    "
+            "chained/fused:    %.2fx (median),    %.2fx (mean)" %
+            (batch_size, num_calls, inter_op, element_size, num_iters,
+             np.median(chained_deltas), np.mean(chained_deltas),
+             np.std(chained_deltas), np.min(chained_deltas),
+             np.max(chained_deltas), np.median(fused_deltas),
+             np.mean(fused_deltas), np.std(fused_deltas), np.min(fused_deltas),
+             np.max(fused_deltas),
+             np.median(chained_deltas) / np.median(fused_deltas),
+             np.mean(chained_deltas) / np.mean(fused_deltas)))
+
+        self.report_benchmark(
+            iters=num_iters,
+            wall_time=np.median(chained_deltas),
+            name=name("chained", label, num_calls, inter_op, element_size,
+                      batch_size))
+
+        self.report_benchmark(
+            iters=num_iters,
+            wall_time=np.median(fused_deltas),
+            name=name("fused", label, num_calls, inter_op, element_size,
+                      batch_size))
+
+      print()
+
+    np.random.seed(_NUMPY_RANDOM_SEED)
+    benchmark("Sequential element size evaluation", seq_elem_size_series)
+    benchmark("Sequential batch size evaluation", seq_batch_size_series)
+    benchmark("Parallel element size evaluation", par_elem_size_series)
+    benchmark("Parallel batch size evaluation", par_batch_size_series)
+    benchmark("Transformation parallelism evaluation", par_num_calls_series)
+    benchmark("Threadpool size evaluation", par_inter_op_series)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/benchmarks/map_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_benchmark.py
deleted file mode 100644
index ad253cffa5..0000000000
--- a/tensorflow/python/data/experimental/benchmarks/map_benchmark.py
+++ /dev/null
@@ -1,245 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for the experimental input pipeline ops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import hashlib
-import itertools
-import time
-
-import numpy as np
-
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import session
-from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.experimental.ops import optimization
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.platform import test
-
-_NUMPY_RANDOM_SEED = 42
-
-
-class MapDatasetBenchmark(test.Benchmark):
-
-  # The purpose of this benchmark is to compare the performance of chaining vs
-  # fusing of the map and batch transformations across various configurations.
-  #
-  # NOTE: It is recommended to build the benchmark with
-  # `-c opt --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-gmlt`
-  # and execute it on a machine with at least 32 CPU cores.
-  def benchmarkMapAndBatch(self):
-
-    # Sequential pipeline configurations.
-    seq_elem_size_series = itertools.product([1], [1], [1, 2, 4, 8], [16])
-    seq_batch_size_series = itertools.product([1], [1], [1], [8, 16, 32, 64])
-
-    # Parallel pipeline configuration.
-    par_elem_size_series = itertools.product([32], [32], [1, 2, 4, 8], [256])
-    par_batch_size_series = itertools.product([32], [32], [1],
-                                              [128, 256, 512, 1024])
-    par_num_calls_series = itertools.product([8, 16, 32, 64], [32], [1], [512])
-    par_inter_op_series = itertools.product([32], [8, 16, 32, 64], [1], [512])
-
-    def name(method, label, num_calls, inter_op, element_size, batch_size):
-      return ("%s_id_%s_num_calls_%d_inter_op_%d_elem_size_%d_batch_size_%d" % (
-          method,
-          hashlib.sha1(label).hexdigest(),
-          num_calls,
-          inter_op,
-          element_size,
-          batch_size,
-      ))
-
-    def benchmark(label, series):
-
-      print("%s:" % label)
-      for num_calls, inter_op, element_size, batch_size in series:
-
-        num_iters = 1024 // (
-            (element_size * batch_size) // min(num_calls, inter_op))
-        k = 1024 * 1024
-        dataset = dataset_ops.Dataset.from_tensors((np.random.rand(
-            element_size, 4 * k), np.random.rand(4 * k, 1))).repeat()
-
-        chained_dataset = dataset.map(
-            math_ops.matmul,
-            num_parallel_calls=num_calls).batch(batch_size=batch_size)
-        chained_iterator = chained_dataset.make_one_shot_iterator()
-        chained_get_next = chained_iterator.get_next()
-
-        chained_deltas = []
-        with session.Session(
-            config=config_pb2.ConfigProto(
-                inter_op_parallelism_threads=inter_op,
-                use_per_session_threads=True)) as sess:
-          for _ in range(5):
-            sess.run(chained_get_next.op)
-          for _ in range(num_iters):
-            start = time.time()
-            sess.run(chained_get_next.op)
-            end = time.time()
-            chained_deltas.append(end - start)
-
-        fused_dataset = dataset.apply(
-            batching.map_and_batch(
-                math_ops.matmul,
-                num_parallel_calls=num_calls,
-                batch_size=batch_size))
-        fused_iterator = fused_dataset.make_one_shot_iterator()
-        fused_get_next = fused_iterator.get_next()
-
-        fused_deltas = []
-        with session.Session(
-            config=config_pb2.ConfigProto(
-                inter_op_parallelism_threads=inter_op,
-                use_per_session_threads=True)) as sess:
-
-          for _ in range(5):
-            sess.run(fused_get_next.op)
-          for _ in range(num_iters):
-            start = time.time()
-            sess.run(fused_get_next.op)
-            end = time.time()
-            fused_deltas.append(end - start)
-
-        print(
-            "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, "
-            "element size: %d, num iters: %d\nchained wall time: %f (median), "
-            "%f (mean), %f (stddev), %f (min), %f (max)\n  fused wall time: "
-            "%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n    "
-            "chained/fused:    %.2fx (median),    %.2fx (mean)" %
-            (batch_size, num_calls, inter_op, element_size, num_iters,
-             np.median(chained_deltas), np.mean(chained_deltas),
-             np.std(chained_deltas), np.min(chained_deltas),
-             np.max(chained_deltas), np.median(fused_deltas),
-             np.mean(fused_deltas), np.std(fused_deltas), np.min(fused_deltas),
-             np.max(fused_deltas),
-             np.median(chained_deltas) / np.median(fused_deltas),
-             np.mean(chained_deltas) / np.mean(fused_deltas)))
-
-        self.report_benchmark(
-            iters=num_iters,
-            wall_time=np.median(chained_deltas),
-            name=name("chained", label, num_calls, inter_op, element_size,
-                      batch_size))
-
-        self.report_benchmark(
-            iters=num_iters,
-            wall_time=np.median(fused_deltas),
-            name=name("fused", label, num_calls, inter_op, element_size,
-                      batch_size))
-
-      print("")
-
-    np.random.seed(_NUMPY_RANDOM_SEED)
-    benchmark("Sequential element size evaluation", seq_elem_size_series)
-    benchmark("Sequential batch size evaluation", seq_batch_size_series)
-    benchmark("Parallel element size evaluation", par_elem_size_series)
-    benchmark("Parallel batch size evaluation", par_batch_size_series)
-    benchmark("Transformation parallelism evaluation", par_num_calls_series)
-    benchmark("Threadpool size evaluation", par_inter_op_series)
-
-  # This benchmark compares the performance of pipeline with multiple chained
-  # maps with and without map fusion.
-  def benchmarkChainOfMaps(self):
-    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
-    for chain_length in chain_lengths:
-      self._benchmarkChainOfMaps(chain_length, False)
-      self._benchmarkChainOfMaps(chain_length, True)
-
-  def _benchmarkChainOfMaps(self, chain_length, optimize_dataset):
-    with ops.Graph().as_default():
-      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
-      for _ in range(chain_length):
-        dataset = dataset.map(lambda x: x)
-      if optimize_dataset:
-        dataset = dataset.apply(optimization.optimize(["map_fusion"]))
-
-      iterator = dataset.make_one_shot_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for _ in range(5):
-          sess.run(next_element.op)
-        deltas = []
-        for _ in range(100):
-          start = time.time()
-          for _ in range(100):
-            sess.run(next_element.op)
-          end = time.time()
-          deltas.append(end - start)
-
-        median_wall_time = np.median(deltas) / 100
-        opt_mark = "opt" if optimize_dataset else "no-opt"
-        print("Map dataset {} chain length: {} Median wall time: {}".format(
-            opt_mark, chain_length, median_wall_time))
-        self.report_benchmark(
-            iters=1000,
-            wall_time=median_wall_time,
-            name="benchmark_map_dataset_chain_latency_{}_{}".format(
-                opt_mark, chain_length))
-
-
-class MapAndFilterBenchmark(test.Benchmark):
-
-  # This benchmark compares the performance of pipeline with multiple chained
-  # map + filter with and without map fusion.
-  def benchmarkMapAndFilter(self):
-    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
-    for chain_length in chain_lengths:
-      self._benchmarkMapAndFilter(chain_length, False)
-      self._benchmarkMapAndFilter(chain_length, True)
-
-  def _benchmarkMapAndFilter(self, chain_length, optimize_dataset):
-    with ops.Graph().as_default():
-      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
-      for _ in range(chain_length):
-        dataset = dataset.map(lambda x: x + 5).filter(
-            lambda x: math_ops.greater_equal(x - 5, 0))
-      if optimize_dataset:
-        dataset = dataset.apply(
-            optimization.optimize(["map_and_filter_fusion"]))
-
-      iterator = dataset.make_one_shot_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for _ in range(10):
-          sess.run(next_element.op)
-        deltas = []
-        for _ in range(100):
-          start = time.time()
-          for _ in range(100):
-            sess.run(next_element.op)
-          end = time.time()
-          deltas.append(end - start)
-
-        median_wall_time = np.median(deltas) / 100
-        opt_mark = "opt" if optimize_dataset else "no-opt"
-        print("Map and filter dataset {} chain length: {} Median wall time: {}".
-              format(opt_mark, chain_length, median_wall_time))
-        self.report_benchmark(
-            iters=1000,
-            wall_time=median_wall_time,
-            name="benchmark_map_and_filter_dataset_chain_latency_{}_{}".format(
-                opt_mark, chain_length))
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
new file mode 100644
index 0000000000..0c3ac8b371
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
@@ -0,0 +1,194 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for the `MapVectorization` optimization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.core.example import example_pb2
+from tensorflow.core.example import feature_pb2
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import parsing_ops
+from tensorflow.python.platform import test
+
+
+def _generate_csv_test_case():
+  """Generates a `decode_csv()` test case."""
+
+  def csv_factory():
+    return dataset_ops.Dataset.from_tensor_slices(["1.0:2:a",
+                                                   "2.4:5:c"]).repeat(5)
+
+  def decode_csv_fn(x):
+    return parsing_ops.decode_csv(
+        x,
+        record_defaults=[
+            constant_op.constant([], dtypes.float32),
+            constant_op.constant([], dtypes.int32),
+            constant_op.constant([], dtypes.string)
+        ],
+        field_delim=":")
+
+  return decode_csv_fn, csv_factory
+
+
+def _generate_parse_single_example_test_case():
+  """Generates a `parse_single_example()` test case."""
+
+  def parse_example_factory():
+    """Parse example factory."""
+
+    def _int64_feature(*values):
+      return feature_pb2.Feature(int64_list=feature_pb2.Int64List(value=values))
+
+    def _bytes_feature(*values):
+      return feature_pb2.Feature(
+          bytes_list=feature_pb2.BytesList(
+              value=[v.encode("utf-8") for v in values]))
+
+    return dataset_ops.Dataset.from_tensor_slices(
+        constant_op.constant([
+            example_pb2.Example(
+                features=feature_pb2.Features(
+                    feature={
+                        "dense_int": _int64_feature(i),
+                        "dense_str": _bytes_feature(str(i)),
+                        "sparse_int": _int64_feature(i, i * 2, i * 4, i * 8),
+                        "sparse_str": _bytes_feature(*["abc"] * i)
+                    })).SerializeToString() for i in range(10)
+        ]))
+
+  def parse_single_example_fn(x):
+    features = {
+        "dense_int": parsing_ops.FixedLenFeature((), dtypes.int64, 0),
+        "dense_str": parsing_ops.FixedLenFeature((), dtypes.string, ""),
+        "sparse_int": parsing_ops.VarLenFeature(dtypes.int64),
+        "sparse_str": parsing_ops.VarLenFeature(dtypes.string),
+    }
+    return parsing_ops.parse_single_example(x, features)
+
+  return parse_single_example_fn, parse_example_factory
+
+
+# TODO(rachelim): Add a benchmark for more expensive transformations, such as
+# vgg_preprocessing.
+class MapVectorizationBenchmark(test.Benchmark):
+  """Benchmarks for the `MapVectorization` optimization."""
+
+  def _run(self, x, num_iters=100, name=None):
+    deltas = []
+    with session.Session() as sess:
+      for _ in range(5):
+        # Warm up session...
+        sess.run(x)
+      for _ in range(num_iters):
+        start = time.time()
+        sess.run(x)
+        end = time.time()
+        deltas.append(end - start)
+    median_time = np.median(deltas)
+    self.report_benchmark(iters=num_iters, wall_time=median_time, name=name)
+    return median_time
+
+  def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id):
+    num_elems = int(np.sum([np.prod(x) for x in input_size]))
+    name_template = "{}__batch_size_{}_input_element_size_{}_{}"
+    unoptimized = input_dataset.map(map_fn).batch(batch_size)
+    unoptimized_op = unoptimized.make_one_shot_iterator().get_next()
+
+    optimized = input_dataset.map(map_fn).batch(batch_size)
+    options = dataset_ops.Options()
+    options.experimental_map_vectorization = True
+    optimized = optimized.with_options(options)
+    optimized_op = optimized.make_one_shot_iterator().get_next()
+
+    unoptimized_time = self._run(
+        unoptimized_op,
+        name=name_template.format(str_id, batch_size, num_elems, "unoptimized"))
+    optimized_time = self._run(
+        optimized_op,
+        name=name_template.format(str_id, batch_size, num_elems, "optimized"))
+
+    print("Batch size: {}\n"
+          "Input element size: {}\n"
+          "Transformation: {}\n"
+          "Speedup: {}\n".format(batch_size, input_size, str_id,
+                                 (unoptimized_time / optimized_time)))
+
+  # Known cheap functions
+  def benchmarkIdentity(self):
+    self._benchmark_helper(lambda *args: [array_ops.identity(x) for x in args],
+                           "identity")
+
+  def benchmarkAddConst(self):
+    self._benchmark_helper(lambda *args: [x + 1 for x in args], "add_const")
+
+  def benchmarkReturnConst(self):
+    self._benchmark_helper(lambda *args: [constant_op.constant(2)], "ret_const")
+
+  def benchmarkSelect(self):
+    self._benchmark_helper(lambda *args: args[0], "select")
+
+  def benchmarkCast(self):
+    self._benchmark_helper(
+        lambda *args: [math_ops.cast(x, dtypes.float64) for x in args], "cast")
+
+  def benchmarkReshape(self):
+    self._benchmark_helper(
+        lambda *args: [array_ops.reshape(x, (-1, 30)) for x in args], "reshape")
+
+  def benchmarkDecodeCSV(self):
+    csv_fn, csv_factory = _generate_csv_test_case()
+    self._benchmark_helper(csv_fn, "decode_csv", lambda: [csv_factory()])
+
+  def benchmarkParseSingleExample(self):
+    # NOTE: Since we haven't implemented a vectorizer for "SerializeSparse",
+    # this function is only naively vectorized.
+    parse_fn, parse_factory = _generate_parse_single_example_test_case()
+
+    self._benchmark_helper(parse_fn, "parse_single_example",
+                           lambda: [parse_factory()])
+
+  def _default_dataset_factory(self):
+    input_sizes = [(10, 10, 3), (10, 100, 300)]
+    for sz in input_sizes:
+      yield dataset_ops.Dataset.from_tensor_slices(np.random.rand(*sz))
+
+  def _benchmark_helper(self, map_fn, str_id, base_dataset_factory=None):
+    if base_dataset_factory is None:
+      base_dataset_factory = self._default_dataset_factory
+
+    batch_size = 1000
+    for base_dataset in base_dataset_factory():
+      base_dataset = base_dataset.repeat()
+      input_size = [
+          tuple(shape.as_list())
+          for shape in nest.flatten(base_dataset.output_shapes)
+      ]
+      self._compare(base_dataset, map_fn, batch_size, input_size, str_id)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
index d0d979dbd4..2eb5561b11 100644
--- a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
@@ -91,7 +91,7 @@ class MatchingFilesBenchmark(test.Benchmark):
             (len(median_deltas) - 2):
                 np.average(median_deltas[2:])
         },
-        name='benchmark_matching_files_dataset_nesteddirectory(%d*%d)' %
+        name='dataset_nested_directory(%d*%d)' %
         (width, depth))
 
     shutil.rmtree(tmp_dir, ignore_errors=True)
diff --git a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
new file mode 100644
index 0000000000..0eca97d26d
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
@@ -0,0 +1,120 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for static optimizations."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+class OptimizationBenchmark(test.Benchmark):
+  """Benchmarks for static optimizations."""
+
+  def benchmarkMapFusion(self):
+    """Evaluates performance map of fusion."""
+
+    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
+    for chain_length in chain_lengths:
+      self._benchmarkMapFusion(chain_length, False)
+      self._benchmarkMapFusion(chain_length, True)
+
+  def _benchmarkMapFusion(self, chain_length, optimize_dataset):
+    with ops.Graph().as_default():
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
+      for _ in range(chain_length):
+        dataset = dataset.map(lambda x: x)
+      if optimize_dataset:
+        options = dataset_ops.Options()
+        options.experimental_map_fusion = True
+        dataset = dataset.with_options(options)
+
+      iterator = dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
+
+      with session.Session() as sess:
+        for _ in range(5):
+          sess.run(next_element.op)
+        deltas = []
+        for _ in range(100):
+          start = time.time()
+          for _ in range(100):
+            sess.run(next_element.op)
+          end = time.time()
+          deltas.append(end - start)
+
+        median_wall_time = np.median(deltas) / 100
+        opt_mark = "opt" if optimize_dataset else "noopt"
+        print("Map dataset {} chain length: {} Median wall time: {}".format(
+            opt_mark, chain_length, median_wall_time))
+        self.report_benchmark(
+            iters=100,
+            wall_time=median_wall_time,
+            name="map_fusion_{}_chain_length_{}".format(
+                opt_mark, chain_length))
+
+  def benchmarkMapAndFilterFusion(self):
+    """Evaluates performance map of fusion."""
+
+    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
+    for chain_length in chain_lengths:
+      self._benchmarkMapAndFilterFusion(chain_length, False)
+      self._benchmarkMapAndFilterFusion(chain_length, True)
+
+  def _benchmarkMapAndFilterFusion(self, chain_length, optimize_dataset):
+    with ops.Graph().as_default():
+      dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
+      for _ in range(chain_length):
+        dataset = dataset.map(lambda x: x + 5).filter(
+            lambda x: math_ops.greater_equal(x - 5, 0))
+      if optimize_dataset:
+        options = dataset_ops.Options()
+        options.experimental_map_and_filter_fusion = True
+        dataset = dataset.with_options(options)
+      iterator = dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
+
+      with session.Session() as sess:
+        for _ in range(10):
+          sess.run(next_element.op)
+        deltas = []
+        for _ in range(100):
+          start = time.time()
+          for _ in range(100):
+            sess.run(next_element.op)
+          end = time.time()
+          deltas.append(end - start)
+
+        median_wall_time = np.median(deltas) / 100
+        opt_mark = "opt" if optimize_dataset else "noopt"
+        print("Map and filter dataset {} chain length: {} Median wall time: {}"
+              .format(opt_mark, chain_length, median_wall_time))
+        self.report_benchmark(
+            iters=100,
+            wall_time=median_wall_time,
+            name="map_and_filter_fusion_{}_chain_length_{}".format(
+                opt_mark, chain_length))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index ba5c6f7b17..6b22f9b8a8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -72,15 +72,11 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:session",
         "//tensorflow/python/data/experimental/ops:error_ops",
         "//tensorflow/python/data/experimental/ops:readers",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:readers",
         "//tensorflow/python/eager:context",
-        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
deleted file mode 100644
index e896752a26..0000000000
--- a/tensorflow/python/data/experimental/kernel_tests/batch_dataset_op_test.py
+++ /dev/null
@@ -1,688 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for the experimental input pipeline ops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-import time
-
-from absl.testing import parameterized
-import numpy as np
-
-from tensorflow.python.client import session
-from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import script_ops
-from tensorflow.python.ops import string_ops
-from tensorflow.python.platform import test
-from tensorflow.python.util import compat
-
-
-class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
-
-  def testDenseToSparseBatchDataset(self):
-    components = np.random.randint(12, size=(100,)).astype(np.int32)
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components)
-        .map(lambda x: array_ops.fill([x], x)).apply(
-            batching.dense_to_sparse_batch(4,
-                                           [12])).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-
-      for start in range(0, len(components), 4):
-        results = sess.run(get_next)
-        self.assertAllEqual([[i, j]
-                             for i, c in enumerate(components[start:start + 4])
-                             for j in range(c)], results.indices)
-        self.assertAllEqual(
-            [c for c in components[start:start + 4] for _ in range(c)],
-            results.values)
-        self.assertAllEqual([min(4,
-                                 len(components) - start), 12],
-                            results.dense_shape)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testDenseToSparseBatchDatasetWithUnknownShape(self):
-    components = np.random.randint(5, size=(40,)).astype(np.int32)
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components).map(
-            lambda x: array_ops.fill([x, x], x)).apply(
-                batching.dense_to_sparse_batch(
-                    4, [5, None])).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-
-      for start in range(0, len(components), 4):
-        results = sess.run(get_next)
-        self.assertAllEqual([[i, j, z]
-                             for i, c in enumerate(components[start:start + 4])
-                             for j in range(c)
-                             for z in range(c)], results.indices)
-        self.assertAllEqual([
-            c for c in components[start:start + 4] for _ in range(c)
-            for _ in range(c)
-        ], results.values)
-        self.assertAllEqual([
-            min(4,
-                len(components) - start), 5,
-            np.max(components[start:start + 4])
-        ], results.dense_shape)
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testDenseToSparseBatchDatasetWithInvalidShape(self):
-    input_tensor = array_ops.constant([[1]])
-    with self.assertRaisesRegexp(ValueError, "Dimension -2 must be >= 0"):
-      dataset_ops.Dataset.from_tensors(input_tensor).apply(
-          batching.dense_to_sparse_batch(4,
-                                         [-2])).make_initializable_iterator()
-
-  def testDenseToSparseBatchDatasetShapeErrors(self):
-    input_tensor = array_ops.placeholder(dtypes.int32)
-    iterator = (
-        dataset_ops.Dataset.from_tensors(input_tensor).apply(
-            batching.dense_to_sparse_batch(4,
-                                           [12])).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # Initialize with an input tensor of incompatible rank.
-      sess.run(init_op, feed_dict={input_tensor: [[1]]})
-      with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "incompatible with the row shape"):
-        sess.run(get_next)
-
-      # Initialize with an input tensor that is larger than `row_shape`.
-      sess.run(init_op, feed_dict={input_tensor: range(13)})
-      with self.assertRaisesRegexp(errors.DataLossError,
-                                   "larger than the row shape"):
-        sess.run(get_next)
-
-  def testUnbatchWithUnknownRankInput(self):
-    placeholder = array_ops.placeholder(dtypes.int32)
-    dataset = dataset_ops.Dataset.from_tensors(placeholder).apply(
-        batching.unbatch())
-    iterator = dataset.make_initializable_iterator()
-    next_elem = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]})
-      for i in range(4):
-        self.assertEqual(i, sess.run(next_elem))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_elem)
-
-  def testUnbatchScalarDataset(self):
-    data = tuple([math_ops.range(10) for _ in range(3)])
-    data = dataset_ops.Dataset.from_tensor_slices(data)
-    expected_types = (dtypes.int32,) * 3
-    data = data.batch(2)
-    self.assertEqual(expected_types, data.output_types)
-    data = data.apply(batching.unbatch())
-    self.assertEqual(expected_types, data.output_types)
-
-    iterator = data.make_one_shot_iterator()
-    op = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(10):
-        self.assertEqual((i,) * 3, sess.run(op))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
-
-  def testUnbatchDatasetWithStrings(self):
-    data = tuple([math_ops.range(10) for _ in range(3)])
-    data = dataset_ops.Dataset.from_tensor_slices(data)
-    data = data.map(lambda x, y, z: (x, string_ops.as_string(y), z))
-    expected_types = (dtypes.int32, dtypes.string, dtypes.int32)
-    data = data.batch(2)
-    self.assertEqual(expected_types, data.output_types)
-    data = data.apply(batching.unbatch())
-    self.assertEqual(expected_types, data.output_types)
-
-    iterator = data.make_one_shot_iterator()
-    op = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(10):
-        self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
-
-  def testUnbatchDatasetWithSparseTensor(self):
-    st = sparse_tensor.SparseTensorValue(
-        indices=[[i, i] for i in range(10)],
-        values=list(range(10)),
-        dense_shape=[10, 10])
-    data = dataset_ops.Dataset.from_tensors(st)
-    data = data.apply(batching.unbatch())
-    data = data.batch(5)
-    data = data.apply(batching.unbatch())
-    iterator = data.make_one_shot_iterator()
-    next_element = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(10):
-        st_row = sess.run(next_element)
-        self.assertEqual([i], st_row.indices)
-        self.assertEqual([i], st_row.values)
-        self.assertEqual([10], st_row.dense_shape)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-
-  def testUnbatchDatasetWithDenseAndSparseTensor(self):
-    st = sparse_tensor.SparseTensorValue(
-        indices=[[i, i] for i in range(10)],
-        values=list(range(10)),
-        dense_shape=[10, 10])
-    data = dataset_ops.Dataset.from_tensors((list(range(10)), st))
-    data = data.apply(batching.unbatch())
-    data = data.batch(5)
-    data = data.apply(batching.unbatch())
-    iterator = data.make_one_shot_iterator()
-    next_element = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(10):
-        dense_elem, st_row = sess.run(next_element)
-        self.assertEqual(i, dense_elem)
-        self.assertEqual([i], st_row.indices)
-        self.assertEqual([i], st_row.values)
-        self.assertEqual([10], st_row.dense_shape)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-
-  def testUnbatchSingleElementTupleDataset(self):
-    data = tuple([(math_ops.range(10),) for _ in range(3)])
-    data = dataset_ops.Dataset.from_tensor_slices(data)
-    expected_types = ((dtypes.int32,),) * 3
-    data = data.batch(2)
-    self.assertEqual(expected_types, data.output_types)
-    data = data.apply(batching.unbatch())
-    self.assertEqual(expected_types, data.output_types)
-
-    iterator = data.make_one_shot_iterator()
-    op = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(10):
-        self.assertEqual(((i,),) * 3, sess.run(op))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
-
-  def testUnbatchMultiElementTupleDataset(self):
-    data = tuple([(math_ops.range(10 * i, 10 * i + 10),
-                   array_ops.fill([10], "hi")) for i in range(3)])
-    data = dataset_ops.Dataset.from_tensor_slices(data)
-    expected_types = ((dtypes.int32, dtypes.string),) * 3
-    data = data.batch(2)
-    self.assertAllEqual(expected_types, data.output_types)
-    data = data.apply(batching.unbatch())
-    self.assertAllEqual(expected_types, data.output_types)
-
-    iterator = data.make_one_shot_iterator()
-    op = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(10):
-        self.assertEqual(((i, b"hi"), (10 + i, b"hi"), (20 + i, b"hi")),
-                         sess.run(op))
-
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
-
-  def testUnbatchEmpty(self):
-    data = dataset_ops.Dataset.from_tensors(
-        (constant_op.constant([]), constant_op.constant([], shape=[0, 4]),
-         constant_op.constant([], shape=[0, 4, 0])))
-    data = data.apply(batching.unbatch())
-    iterator = data.make_one_shot_iterator()
-    next_element = iterator.get_next()
-
-    with self.cached_session() as sess:
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-
-  def testUnbatchStaticShapeMismatch(self):
-    data = dataset_ops.Dataset.from_tensors((np.arange(7), np.arange(8),
-                                             np.arange(9)))
-    with self.assertRaises(ValueError):
-      data.apply(batching.unbatch())
-
-  def testUnbatchDynamicShapeMismatch(self):
-    ph1 = array_ops.placeholder(dtypes.int32, shape=[None])
-    ph2 = array_ops.placeholder(dtypes.int32, shape=None)
-    data = dataset_ops.Dataset.from_tensors((ph1, ph2))
-    data = data.apply(batching.unbatch())
-    iterator = data.make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    with self.cached_session() as sess:
-      # Mismatch in the 0th dimension.
-      sess.run(
-          iterator.initializer,
-          feed_dict={
-              ph1: np.arange(7).astype(np.int32),
-              ph2: np.arange(8).astype(np.int32)
-          })
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(next_element)
-
-      # No 0th dimension (i.e. scalar value) for one component.
-      sess.run(
-          iterator.initializer,
-          feed_dict={
-              ph1: np.arange(7).astype(np.int32),
-              ph2: 7
-          })
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(next_element)
-
-  @parameterized.named_parameters(
-      ("Default", None, None),
-      ("SequentialCalls", 1, None),
-      ("ParallelCalls", 2, None),
-      ("ParallelBatches", None, 10),
-  )
-  def testMapAndBatch(self, num_parallel_calls, num_parallel_batches):
-    """Test a dataset that maps a TF function across its input elements."""
-    # The pipeline is TensorSliceDataset ->
-    # RepeatDataset(count) -> MapAndBatchDataset(square_3, batch_size).
-    components = (np.arange(7),
-                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
-                  np.array(37.0) * np.arange(7))
-
-    count = array_ops.placeholder(dtypes.int64, shape=[])
-    batch_size = array_ops.placeholder(dtypes.int64, shape=[])
-
-    def _map_fn(x, y, z):
-      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
-
-    iterator = (
-        dataset_ops.Dataset.from_tensor_slices(components).repeat(count).apply(
-            batching.map_and_batch(
-                map_func=_map_fn,
-                batch_size=batch_size,
-                num_parallel_calls=num_parallel_calls,
-                num_parallel_batches=num_parallel_batches))
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual([[None] + list(c.shape[1:]) for c in components],
-                     [t.shape.as_list() for t in get_next])
-
-    with self.cached_session() as sess:
-      # Batch of a finite input, where the batch_size divides the
-      # total number of elements.
-      sess.run(init_op, feed_dict={count: 28, batch_size: 14})
-      num_batches = (28 * 7) // 14
-      for i in range(num_batches):
-        result = sess.run(get_next)
-        for component, result_component in zip(components, result):
-          for j in range(14):
-            self.assertAllEqual(component[(i * 14 + j) % 7]**2,
-                                result_component[j])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Batch of a finite input, where the batch_size does not
-      # divide the total number of elements.
-      sess.run(init_op, feed_dict={count: 14, batch_size: 8})
-
-      # We expect (num_batches - 1) full-sized batches.
-      num_batches = int(math.ceil((14 * 7) / 8))
-      for i in range(num_batches - 1):
-        result = sess.run(get_next)
-        for component, result_component in zip(components, result):
-          for j in range(8):
-            self.assertAllEqual(component[(i * 8 + j) % 7]**2,
-                                result_component[j])
-      result = sess.run(get_next)
-      for component, result_component in zip(components, result):
-        for j in range((14 * 7) % 8):
-          self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2,
-                              result_component[j])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Batch of an empty input should fail straight away.
-      sess.run(init_op, feed_dict={count: 0, batch_size: 8})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Empty batch should be an initialization time error.
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(init_op, feed_dict={count: 14, batch_size: 0})
-
-  @parameterized.named_parameters(
-      ("Even", False),
-      ("Uneven", True),
-  )
-  def testMapAndBatchPartialBatch(self, drop_remainder):
-    iterator = (
-        dataset_ops.Dataset.range(10).apply(
-            batching.map_and_batch(
-                lambda x: array_ops.reshape(x * x, [1]),
-                batch_size=4,
-                drop_remainder=drop_remainder)).make_one_shot_iterator())
-    if drop_remainder:
-      self.assertEqual([4, 1], iterator.output_shapes.as_list())
-    else:
-      self.assertEqual([None, 1], iterator.output_shapes.as_list())
-    next_element = iterator.get_next()
-    with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      if not drop_remainder:
-        self.assertAllEqual([[64], [81]], sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-
-  def testMapAndBatchYieldsPartialBatch(self):
-    iterator = (
-        dataset_ops.Dataset.range(10).apply(
-            batching.map_and_batch(lambda x: array_ops.reshape(x * x, [1]),
-                                   4)).make_one_shot_iterator())
-    self.assertEqual([None, 1], iterator.output_shapes.as_list())
-    next_element = iterator.get_next()
-    with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      self.assertAllEqual([[64], [81]], sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-
-  def testMapAndBatchParallelGetNext(self):
-    iterator = (
-        dataset_ops.Dataset.range(50000).apply(
-            batching.map_and_batch(lambda x: x,
-                                   batch_size=100)).make_one_shot_iterator())
-    elements = []
-    for _ in range(100):
-      elements.append(iterator.get_next())
-    with self.cached_session() as sess:
-      for i in range(5):
-        got = sess.run(elements)
-        got.sort(key=lambda x: x[0])
-        expected = []
-        for j in range(100):
-          expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100))
-        self.assertAllEqual(got, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elements)
-
-  def testMapAndBatchParallelGetNextDropRemainder(self):
-    iterator = (
-        dataset_ops.Dataset.range(49999).apply(
-            batching.map_and_batch(
-                lambda x: x, batch_size=100,
-                drop_remainder=True)).make_one_shot_iterator())
-    elements = []
-    for _ in range(100):
-      elements.append(iterator.get_next())
-    with self.cached_session() as sess:
-      for i in range(4):
-        got = sess.run(elements)
-        got.sort(key=lambda x: x[0])
-        expected = []
-        for j in range(100):
-          expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100))
-        self.assertAllEqual(got, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elements)
-
-  def testMapAndBatchSparse(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=[[0]], values=(i * [1]), dense_shape=[1])
-
-    iterator = dataset_ops.Dataset.range(10).apply(
-        batching.map_and_batch(_sparse, 5)).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      for i in range(2):
-        actual = sess.run(get_next)
-        expected = sparse_tensor.SparseTensorValue(
-            indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
-            values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
-            dense_shape=[5, 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testMapAndBatchFails(self):
-    """Test a dataset that maps a TF function across its input elements."""
-    dataset = dataset_ops.Dataset.from_tensors(
-        array_ops.check_numerics(
-            constant_op.constant(1.0) / constant_op.constant(0.0), "oops"))
-    batch_size = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = (
-        dataset.apply(batching.map_and_batch(
-            lambda x: x, batch_size)).make_initializable_iterator())
-    init_op = iterator.initializer
-    with self.cached_session() as sess:
-      with self.assertRaisesRegexp(errors.InvalidArgumentError, "oops"):
-        sess.run(init_op, feed_dict={batch_size: 14})
-
-  def testMapAndBatchShapeMismatch(self):
-    """Test a dataset that maps a TF function across its input elements."""
-
-    def generator():
-      yield [1]
-      yield [2]
-      yield [3]
-      yield [[4, 5, 6]]
-
-    dataset = dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.int32)
-    batch_size = 4
-    iterator = (
-        dataset.apply(batching.map_and_batch(
-            lambda x: x, batch_size)).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-    with self.cached_session() as sess:
-      sess.run(init_op)
-      with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "number of elements does not match"):
-        sess.run(get_next)
-
-  def testMapAndBatchImplicitDispose(self):
-    # Tests whether a map and batch dataset will be cleaned up correctly when
-    # the pipeline does not run it until exhaustion.
-    # The pipeline is TensorSliceDataset -> RepeatDataset(1000) ->
-    # MapAndBatchDataset(f=square_3, batch_size=100).
-    components = (np.arange(1000),
-                  np.array([[1, 2, 3]]) * np.arange(1000)[:, np.newaxis],
-                  np.array(37.0) * np.arange(1000))
-
-    def _map_fn(x, y, z):
-      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components).repeat(
-        1000).apply(batching.map_and_batch(_map_fn, batch_size=100))
-    dataset = dataset.prefetch(5)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for _ in range(3):
-        sess.run(get_next)
-
-  @parameterized.named_parameters(
-      ("1", 0),
-      ("2", 5),
-      ("3", 10),
-      ("4", 90),
-      ("5", 95),
-      ("6", 99),
-  )
-  def testMapAndBatchOutOfRangeError(self, threshold):
-
-    def raising_py_fn(i):
-      if i >= threshold:
-        raise StopIteration()
-      else:
-        return i
-
-    iterator = (
-        dataset_ops.Dataset.range(100).apply(
-            batching.map_and_batch(
-                lambda x: script_ops.py_func(raising_py_fn, [x], dtypes.int64),
-                batch_size=10)).make_one_shot_iterator())
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      for i in range(threshold // 10):
-        self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next))
-      if threshold % 10 != 0:
-        self.assertAllEqual(
-            [threshold // 10 * 10 + j for j in range(threshold % 10)],
-            sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  @parameterized.named_parameters(
-      ("1", False, dtypes.bool),
-      ("2", -42, dtypes.int8),
-      ("3", -42, dtypes.int16),
-      ("4", -42, dtypes.int32),
-      ("5", -42, dtypes.int64),
-      ("6", 42, dtypes.uint8),
-      ("7", 42, dtypes.uint16),
-      ("8", 42.0, dtypes.float16),
-      ("9", 42.0, dtypes.float32),
-      ("10", 42.0, dtypes.float64),
-      ("11", b"hello", dtypes.string),
-  )
-  def testMapAndBatchTypes(self, element, dtype):
-
-    def gen():
-      yield element
-
-    dataset = dataset_ops.Dataset.from_generator(gen, dtype).repeat(100).apply(
-        batching.map_and_batch(lambda x: x, batch_size=10))
-
-    get_next = dataset.make_one_shot_iterator().get_next()
-
-    with self.cached_session() as sess:
-      for _ in range(10):
-        self.assertAllEqual([element for _ in range(10)], sess.run(get_next))
-
-
-class UnbatchDatasetBenchmark(test.Benchmark):
-
-  def benchmarkNativeUnbatch(self):
-    batch_sizes = [1, 2, 5, 10, 20, 50]
-    elems_per_trial = 10000
-    with ops.Graph().as_default():
-      dataset = dataset_ops.Dataset.from_tensors("element").repeat(None)
-      batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-      dataset = dataset.batch(batch_size_placeholder)
-      dataset = dataset.apply(batching.unbatch())
-      dataset = dataset.skip(elems_per_trial)
-      iterator = dataset.make_initializable_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for batch_size in batch_sizes:
-          deltas = []
-          for _ in range(5):
-            sess.run(
-                iterator.initializer,
-                feed_dict={batch_size_placeholder: batch_size})
-            start = time.time()
-            sess.run(next_element.op)
-            end = time.time()
-            deltas.append((end - start) / elems_per_trial)
-
-          median_wall_time = np.median(deltas)
-          print("Unbatch (native) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
-          self.report_benchmark(
-              iters=10000,
-              wall_time=median_wall_time,
-              name="benchmark_unbatch_dataset_native_batch_size_%d" %
-              batch_size)
-
-  # Include a benchmark of the previous `unbatch()` implementation that uses
-  # a composition of more primitive ops. Eventually we'd hope to generate code
-  # that is as good in both cases.
-  def benchmarkOldUnbatchImplementation(self):
-    batch_sizes = [1, 2, 5, 10, 20, 50]
-    elems_per_trial = 10000
-    with ops.Graph().as_default():
-      dataset = dataset_ops.Dataset.from_tensors("element").repeat(None)
-      batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-      dataset = dataset.batch(batch_size_placeholder)
-      dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices)
-      dataset = dataset.skip(elems_per_trial)
-      iterator = dataset.make_initializable_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for batch_size in batch_sizes:
-          deltas = []
-          for _ in range(5):
-            sess.run(
-                iterator.initializer,
-                feed_dict={batch_size_placeholder: batch_size})
-            start = time.time()
-            sess.run(next_element.op)
-            end = time.time()
-            deltas.append((end - start) / elems_per_trial)
-
-          median_wall_time = np.median(deltas)
-          print("Unbatch (unfused) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
-          self.report_benchmark(
-              iters=10000,
-              wall_time=median_wall_time,
-              name="benchmark_unbatch_dataset_unfused_batch_size_%d" %
-              batch_size)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py
index fb75be1fbc..b2f1b43ecf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/csv_dataset_test.py
@@ -20,14 +20,8 @@ from __future__ import print_function
 
 import gzip
 import os
-import string
-import tempfile
-import time
 import zlib
 
-import numpy as np
-
-from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import error_ops
 from tensorflow.python.data.experimental.ops import readers
 from tensorflow.python.data.kernel_tests import test_base
@@ -38,8 +32,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import parsing_ops
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import googletest
 from tensorflow.python.platform import test
 
 
@@ -537,96 +529,5 @@ class CsvDatasetTest(test_base.DatasetTestBase):
           record_defaults=record_defaults)
 
 
-class CsvDatasetBenchmark(test.Benchmark):
-  """Benchmarks for the various ways of creating a dataset from CSV files.
-  """
-  FLOAT_VAL = '1.23456E12'
-  STR_VAL = string.ascii_letters * 10
-
-  def _setUp(self, str_val):
-    # Since this isn't test.TestCase, have to manually create a test dir
-    gfile.MakeDirs(googletest.GetTempDir())
-    self._temp_dir = tempfile.mkdtemp(dir=googletest.GetTempDir())
-
-    self._num_cols = [4, 64, 256]
-    self._num_per_iter = 5000
-    self._filenames = []
-    for n in self._num_cols:
-      fn = os.path.join(self._temp_dir, 'file%d.csv' % n)
-      with open(fn, 'wb') as f:
-        # Just write 100 rows and use `repeat`... Assumes the cost
-        # of creating an iterator is not significant
-        row = ','.join([str_val for _ in range(n)])
-        f.write('\n'.join([row for _ in range(100)]))
-      self._filenames.append(fn)
-
-  def _tearDown(self):
-    gfile.DeleteRecursively(self._temp_dir)
-
-  def _runBenchmark(self, dataset, num_cols, prefix):
-    dataset = dataset.skip(self._num_per_iter - 1)
-    deltas = []
-    for _ in range(10):
-      next_element = dataset.make_one_shot_iterator().get_next()
-      with session.Session() as sess:
-        start = time.time()
-        # NOTE: This depends on the underlying implementation of skip, to have
-        # the net effect of calling `GetNext` num_per_iter times on the
-        # input dataset. We do it this way (instead of a python for loop, or
-        # batching N inputs in one iter) so that the overhead from session.run
-        # or batch doesn't dominate. If we eventually optimize skip, this has
-        # to change.
-        sess.run(next_element)
-        end = time.time()
-      deltas.append(end - start)
-    # Median wall time per CSV record read and decoded
-    median_wall_time = np.median(deltas) / self._num_per_iter
-    print('%s num_cols: %d Median wall time: %f' % (prefix, num_cols,
-                                                    median_wall_time))
-    self.report_benchmark(
-        iters=self._num_per_iter,
-        wall_time=median_wall_time,
-        name='%s_with_cols_%d' % (prefix, num_cols))
-
-  def benchmarkMapWithFloats(self):
-    self._setUp(self.FLOAT_VAL)
-    for i in range(len(self._filenames)):
-      num_cols = self._num_cols[i]
-      kwargs = {'record_defaults': [[0.0]] * num_cols}
-      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
-      dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs))  # pylint: disable=cell-var-from-loop
-      self._runBenchmark(dataset, num_cols, 'csv_float_map_decode_csv')
-    self._tearDown()
-
-  def benchmarkMapWithStrings(self):
-    self._setUp(self.STR_VAL)
-    for i in range(len(self._filenames)):
-      num_cols = self._num_cols[i]
-      kwargs = {'record_defaults': [['']] * num_cols}
-      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
-      dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs))  # pylint: disable=cell-var-from-loop
-      self._runBenchmark(dataset, num_cols, 'csv_strings_map_decode_csv')
-    self._tearDown()
-
-  def benchmarkCsvDatasetWithFloats(self):
-    self._setUp(self.FLOAT_VAL)
-    for i in range(len(self._filenames)):
-      num_cols = self._num_cols[i]
-      kwargs = {'record_defaults': [[0.0]] * num_cols}
-      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
-      dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat()  # pylint: disable=cell-var-from-loop
-      self._runBenchmark(dataset, num_cols, 'csv_float_fused_dataset')
-    self._tearDown()
-
-  def benchmarkCsvDatasetWithStrings(self):
-    self._setUp(self.STR_VAL)
-    for i in range(len(self._filenames)):
-      num_cols = self._num_cols[i]
-      kwargs = {'record_defaults': [['']] * num_cols}
-      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
-      dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat()  # pylint: disable=cell-var-from-loop
-      self._runBenchmark(dataset, num_cols, 'csv_strings_fused_dataset')
-    self._tearDown()
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 1d0e6af649..121798ad3e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -221,15 +221,14 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:math_ops",
-        "//tensorflow/python:nn_ops",
+        "//tensorflow/python:nn",
         "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:session",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/experimental/ops:optimization",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
         "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
@@ -249,12 +248,9 @@ py_test(
     deps = [
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python/data/experimental/ops:batching",
         "//tensorflow/python/data/experimental/ops:optimization",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 18b3bc9424..4f05f02669 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -17,18 +17,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
-from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -439,102 +435,5 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
          ("IteratorGetNext", "IteratorGetNext_1", 1)])
 
 
-class MapVectorizationBenchmark(test.Benchmark):
-  # TODO(rachelim): Add a benchmark for more expensive transformations, such as
-  # vgg_preprocessing.
-
-  def _run(self, x, num_iters=100, name=None):
-    deltas = []
-    with session.Session() as sess:
-      for _ in range(5):
-        # Warm up session...
-        sess.run(x)
-      for _ in range(num_iters):
-        start = time.time()
-        sess.run(x)
-        end = time.time()
-        deltas.append(end - start)
-    median_time = np.median(deltas)
-    self.report_benchmark(iters=num_iters, wall_time=median_time, name=name)
-    return median_time
-
-  def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id):
-    num_elems = sum(np.prod(x) for x in input_size)
-    name_template = "{}__batch_size_{}_input_element_size_{}_{}"
-    unoptimized = input_dataset.map(map_fn).batch(batch_size)
-    unoptimized_op = unoptimized.make_one_shot_iterator().get_next()
-
-    optimized = input_dataset.map(map_fn).batch(batch_size)
-    options = dataset_ops.Options()
-    options.experimental_map_vectorization = True
-    optimized = optimized.with_options(options)
-    optimized_op = optimized.make_one_shot_iterator().get_next()
-
-    unoptimized_time = self._run(
-        unoptimized_op,
-        name=name_template.format(str_id, batch_size, num_elems, "unoptimized"))
-    optimized_time = self._run(
-        optimized_op,
-        name=name_template.format(str_id, batch_size, num_elems, "optimized"))
-
-    print("Batch size: {}\n"
-          "Input element size: {}\n"
-          "Transformation: {}\n"
-          "Speedup: {}\n".format(batch_size, input_size, str_id,
-                                 (unoptimized_time / optimized_time)))
-
-  # Known cheap functions
-  def benchmarkIdentity(self):
-    self._benchmark_helper(lambda *args: [array_ops.identity(x) for x in args],
-                           "identity")
-
-  def benchmarkAddConst(self):
-    self._benchmark_helper(lambda *args: [x + 1 for x in args], "add_const")
-
-  def benchmarkReturnConst(self):
-    self._benchmark_helper(lambda *args: [constant_op.constant(2)], "ret_const")
-
-  def benchmarkSelect(self):
-    self._benchmark_helper(lambda *args: args[0], "select")
-
-  def benchmarkCast(self):
-    self._benchmark_helper(
-        lambda *args: [math_ops.cast(x, dtypes.float64) for x in args], "cast")
-
-  def benchmarkReshape(self):
-    self._benchmark_helper(
-        lambda *args: [array_ops.reshape(x, (-1, 30)) for x in args], "reshape")
-
-  def benchmarkDecodeCSV(self):
-    csv_fn, csv_factory = _generate_csv_test_case()
-    self._benchmark_helper(csv_fn, "decode_csv", lambda: [csv_factory()])
-
-  def benchmarkParseSingleExample(self):
-    # NOTE: Since we haven't implemented a vectorizer for "SerializeSparse",
-    # this function is only naively vectorized.
-    parse_fn, parse_factory = _generate_parse_single_example_test_case()
-
-    self._benchmark_helper(parse_fn, "parse_single_example",
-                           lambda: [parse_factory()])
-
-  def _default_dataset_factory(self):
-    input_sizes = [(10, 10, 3), (10, 100, 300)]
-    for sz in input_sizes:
-      yield dataset_ops.Dataset.from_tensor_slices(np.random.rand(*sz))
-
-  def _benchmark_helper(self, map_fn, str_id, base_dataset_factory=None):
-    if base_dataset_factory is None:
-      base_dataset_factory = self._default_dataset_factory
-
-    batch_size = 1000
-    for base_dataset in base_dataset_factory():
-      base_dataset = base_dataset.repeat()
-      input_size = [
-          tuple(shape.as_list())
-          for shape in nest.flatten(base_dataset.output_shapes)
-      ]
-      self._compare(base_dataset, map_fn, batch_size, input_size, str_id)
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index f5a8399124..ea2737c3c7 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -17,182 +17,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 from absl.testing import parameterized
-import numpy as np
 
-from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
-from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 # TODO(b/117581999): Add eager coverage for the following tests.
 class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
-  def testModelMap(self):
-    k = 1024 * 1024
-    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
-                                                np.random.rand(4 * k,
-                                                               1))).repeat()
-    dataset = dataset.map(math_ops.matmul)
-    dataset = dataset_ops._ModelDataset(dataset)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    deltas = []
-    with self.cached_session() as sess:
-      for _ in range(5):
-        sess.run(get_next.op)
-      for _ in range(100):
-        start = time.time()
-        sess.run(get_next.op)
-        end = time.time()
-        deltas.append(end - start)
-
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
-  def testModelParallelMap(self):
-    k = 1024 * 1024
-    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
-                                                np.random.rand(4 * k,
-                                                               1))).repeat()
-    dataset = dataset.map(
-        math_ops.matmul, num_parallel_calls=optimization.AUTOTUNE)
-    dataset = dataset_ops._ModelDataset(dataset)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    deltas = []
-    with self.cached_session() as sess:
-      for _ in range(5):
-        sess.run(get_next.op)
-      for _ in range(100):
-        start = time.time()
-        sess.run(get_next.op)
-        end = time.time()
-        deltas.append(end - start)
-
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
-  @parameterized.named_parameters(
-      ("Default", False),
-      ("NUMA", True),
-  )
-  def testModelMapAndBatch(self, numa_aware):
-    batch_size = 16
-    k = 1024 * 1024
-    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
-                                                np.random.rand(4 * k,
-                                                               1))).repeat()
-    dataset = dataset.apply(
-        batching.map_and_batch(
-            math_ops.matmul,
-            num_parallel_calls=optimization.AUTOTUNE,
-            batch_size=batch_size))
-    dataset = dataset_ops._ModelDataset(dataset)
-    options = dataset_ops.Options()
-    options.experimental_numa_aware = numa_aware
-    dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    deltas = []
-    with self.cached_session() as sess:
-      for _ in range(5):
-        sess.run(get_next.op)
-      for _ in range(10):
-        start = time.time()
-        sess.run(get_next.op)
-        end = time.time()
-        deltas.append(end - start)
-
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
-  def testModelParallelInterleave(self):
-    k = 1024 * 1024
-    dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k),
-                                                np.random.rand(4 * k,
-                                                               1))).repeat()
-    dataset = dataset.map(math_ops.matmul)
-    dataset = dataset_ops.Dataset.range(1).repeat().interleave(
-        lambda _: dataset,
-        cycle_length=10,
-        num_parallel_calls=optimization.AUTOTUNE)
-    dataset = dataset_ops._ModelDataset(dataset)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    deltas = []
-    with self.cached_session() as sess:
-      for _ in range(5):
-        sess.run(get_next.op)
-      for _ in range(100):
-        start = time.time()
-        sess.run(get_next.op)
-        end = time.time()
-        deltas.append(end - start)
-
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
-  def testModelNested(self):
-    k = 1024 * 1024
-    a = (np.random.rand(1, 8 * k), np.random.rand(8 * k, 1))
-    b = (np.random.rand(1, 4 * k), np.random.rand(4 * k, 1))
-    c = (np.random.rand(1, 2 * k), np.random.rand(2 * k, 1))
-    dataset = dataset_ops.Dataset.from_tensors((a, b, c)).repeat()
-
-    def f1(a, b, c):
-      x, y = a
-      return math_ops.matmul(x, y), b, c
-
-    def f2(a, b, c):
-      x, y = b
-      return a, math_ops.matmul(x, y), c
-
-    def f3(a, b, c):
-      x, y = c
-      return a, b, math_ops.matmul(x, y)
-
-    dataset = dataset.map(f1, num_parallel_calls=optimization.AUTOTUNE)
-    dataset = dataset_ops.Dataset.range(1).repeat().interleave(
-        lambda _: dataset, cycle_length=2)
-
-    dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE)
-    dataset = dataset_ops.Dataset.range(1).repeat().interleave(
-        lambda _: dataset, cycle_length=2)
-
-    dataset = dataset.map(f3, num_parallel_calls=optimization.AUTOTUNE)
-    dataset = dataset_ops._ModelDataset(dataset)
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
-
-    deltas = []
-    with self.cached_session() as sess:
-      for _ in range(5):
-        sess.run(get_next)
-      for _ in range(100):
-        start = time.time()
-        sess.run(get_next)
-        end = time.time()
-        deltas.append(end - start)
-
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
   def testAutotuneOption(self):
     dataset = dataset_ops.Dataset.from_tensors(0)
     dataset = dataset.map(lambda x: x).apply(
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 0cdb44a3b4..0867471d74 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -10,35 +10,32 @@ load("//tensorflow:tensorflow.bzl", "tf_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 tf_py_test(
-    name = "batch_dataset_op_test",
+    name = "batch_test",
     size = "small",
-    srcs = ["batch_dataset_op_test.py"],
+    srcs = ["batch_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
-        "//tensorflow/python:session",
         "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:string_ops",
-        "//tensorflow/python:tensor_shape",
-        "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
 
 tf_py_test(
-    name = "cache_dataset_op_test",
+    name = "cache_test",
     size = "small",
-    srcs = ["cache_dataset_op_test.py"],
+    srcs = ["cache_test.py"],
     additional_deps = [
         ":test_base",
         "//third_party/py/numpy",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
@@ -46,15 +43,13 @@ tf_py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:variables",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
     ],
 )
 
 tf_py_test(
-    name = "concatenate_dataset_op_test",
+    name = "concatenate_test",
     size = "small",
-    srcs = ["concatenate_dataset_op_test.py"],
+    srcs = ["concatenate_test.py"],
     additional_deps = [
         ":test_base",
         "//third_party/py/numpy",
@@ -67,102 +62,94 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "dataset_constructor_op_test",
+    name = "dataset_checkpoint_test",
     size = "small",
-    srcs = ["dataset_constructor_op_test.py"],
+    srcs = ["dataset_checkpoint_test.py"],
     additional_deps = [
         ":test_base",
-        "//third_party/py/numpy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:random_ops",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:session",
-        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:platform",
         "//tensorflow/python:tensor_shape",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:sparse",
-    ],
-    tags = [
-        "manual",
-        "nomac",  # b/62040583
+        "//tensorflow/python:variables",
     ],
 )
 
 tf_py_test(
-    name = "dataset_from_generator_op_test",
-    size = "medium",
-    srcs = ["dataset_from_generator_op_test.py"],
+    name = "dataset_test",
+    size = "small",
+    srcs = ["dataset_test.py"],
     additional_deps = [
         ":test_base",
+        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python/data/util:nest",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:script_ops",
-        "//tensorflow/python:session",
-        "//tensorflow/python:tensor_shape",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python:sparse_tensor",
     ],
 )
 
 tf_py_test(
-    name = "dataset_ops_test",
+    name = "filter_test",
     size = "small",
-    srcs = ["dataset_ops_test.py"],
+    srcs = ["filter_test.py"],
     additional_deps = [
         ":test_base",
-        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python/data/ops:readers",
-        "//tensorflow/python/data/util:nest",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:functional_ops",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:optional_ops",
-        "//tensorflow/python/data/util:structure",
     ],
 )
 
 tf_py_test(
-    name = "filter_dataset_op_test",
+    name = "fixed_length_record_dataset_test",
     size = "small",
-    srcs = ["filter_dataset_op_test.py"],
+    srcs = ["fixed_length_record_dataset_test.py"],
     additional_deps = [
         ":test_base",
-        "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:functional_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:session",
-        "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/ops:iterator_ops",
+        "//tensorflow/python/data/ops:readers",
     ],
 )
 
 tf_py_test(
-    name = "flat_map_dataset_op_test",
+    name = "flat_map_test",
     size = "medium",
-    srcs = ["flat_map_dataset_op_test.py"],
+    srcs = ["flat_map_test.py"],
     additional_deps = [
         ":test_base",
         "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python/data/util:nest",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python:session",
@@ -175,65 +162,158 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "list_files_dataset_op_test",
+    name = "from_generator_test",
+    size = "medium",
+    srcs = ["from_generator_test.py"],
+    additional_deps = [
+        ":test_base",
+        "//third_party/py/numpy",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:session",
+    ],
+)
+
+tf_py_test(
+    name = "from_sparse_tensor_slices_test",
     size = "small",
-    srcs = ["list_files_dataset_op_test.py"],
+    srcs = ["from_sparse_tensor_slices_test.py"],
     additional_deps = [
         ":test_base",
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python:util",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
     ],
 )
 
 tf_py_test(
-    name = "inputs_test",
+    name = "from_tensors_test",
     size = "small",
-    srcs = ["inputs_test.py"],
+    srcs = ["from_tensors_test.py"],
     additional_deps = [
         ":test_base",
-        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
-        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
+    ],
+    tags = [
+        "nomac",  # b/62040583
+    ],
+)
+
+tf_py_test(
+    name = "from_tensor_slices_test",
+    size = "small",
+    srcs = ["from_tensor_slices_test.py"],
+    additional_deps = [
+        ":test_base",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
 
 tf_py_test(
-    name = "interleave_dataset_op_test",
+    name = "interleave_test",
     size = "medium",
-    srcs = ["interleave_dataset_op_test.py"],
+    srcs = ["interleave_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:script_ops",
-        "//tensorflow/python:session",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:training",
+    ],
+)
+
+tf_py_test(
+    name = "iterator_checkpoint_test",
+    size = "medium",
+    srcs = ["iterator_checkpoint_test.py"],
+    additional_deps = [
+        ":test_base",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/training/checkpointable:util",
+        "//tensorflow/python:checkpoint_management",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
     ],
+    grpc_enabled = True,
 )
 
-cuda_py_test(
-    name = "iterator_ops_test",
+tf_py_test(
+    name = "iterator_cluster_test",
     size = "small",
-    srcs = ["iterator_ops_test.py"],
+    srcs = ["iterator_cluster_test.py"],
+    additional_deps = [
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:function",
+        "//tensorflow/python:functional_ops",
+        "//tensorflow/python:lookup_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python:string_ops",
+    ],
+    grpc_enabled = True,
+    tags = [
+        "no_oss",  # Test flaky due to port collisions.
+        "no_windows",
+    ],
+)
+
+cuda_py_test(
+    name = "iterator_test",
+    size = "medium",
+    srcs = ["iterator_test.py"],
     additional_deps = [
-        ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python/data/ops:readers",
@@ -271,43 +351,30 @@ cuda_py_test(
 )
 
 tf_py_test(
-    name = "iterator_ops_cluster_test",
+    name = "list_files_test",
     size = "small",
-    srcs = ["iterator_ops_cluster_test.py"],
+    srcs = ["list_files_test.py"],
     additional_deps = [
-        "//tensorflow/core:protos_all_py",
+        ":test_base",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:function",
-        "//tensorflow/python:functional_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:session",
+        "//tensorflow/python:util",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:string_ops",
-        "//tensorflow/python:lookup_ops",
-    ],
-    grpc_enabled = True,
-    tags = [
-        "no_oss",  # Test flaky due to port collisions.
-        "no_windows",
     ],
 )
 
 tf_py_test(
-    name = "map_dataset_op_test",
+    name = "map_test",
     size = "medium",
-    srcs = ["map_dataset_op_test.py"],
+    srcs = ["map_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
@@ -327,7 +394,6 @@ tf_py_test(
         "//tensorflow/python:string_ops",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
 
@@ -355,9 +421,9 @@ cuda_py_test(
 )
 
 cuda_py_test(
-    name = "optional_ops_test",
+    name = "optional_test",
     size = "small",
-    srcs = ["optional_ops_test.py"],
+    srcs = ["optional_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
@@ -376,73 +442,58 @@ cuda_py_test(
 )
 
 tf_py_test(
-    name = "prefetch_dataset_op_test",
+    name = "padded_batch_test",
     size = "small",
-    srcs = ["prefetch_dataset_op_test.py"],
+    srcs = ["padded_batch_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:string_ops",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:util",
     ],
 )
 
 tf_py_test(
-    name = "range_dataset_op_test",
+    name = "prefetch_test",
     size = "small",
-    srcs = ["range_dataset_op_test.py"],
+    srcs = ["prefetch_test.py"],
     additional_deps = [
         ":test_base",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:tensor_shape",
-        "//tensorflow/python:variables",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
     ],
 )
 
 tf_py_test(
-    name = "reader_dataset_ops_test",
-    size = "medium",
-    srcs = ["reader_dataset_ops_test.py"],
+    name = "range_test",
+    size = "small",
+    srcs = ["range_test.py"],
     additional_deps = [
         ":test_base",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/eager:context",
-        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:dataset_ops_gen",
-        "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:lib",
-        "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:tensor_shape",
-        "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:iterator_ops",
-        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python:framework_test_lib",
     ],
 )
 
 tf_py_test(
-    name = "reduce_dataset_op_test",
+    name = "reduce_test",
     size = "small",
-    srcs = ["reduce_dataset_op_test.py"],
+    srcs = ["reduce_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
@@ -450,7 +501,6 @@ tf_py_test(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/ops:dataset_ops",
@@ -458,32 +508,56 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "save_restore_experimental_test",
+    name = "repeat_test",
     size = "small",
-    srcs = ["save_restore_experimental_test.py"],
+    srcs = ["repeat_test.py"],
     additional_deps = [
-        ":reader_dataset_ops_test",
         ":test_base",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
-        "//tensorflow/python/data/ops:readers",
+    ],
+)
+
+tf_py_test(
+    name = "shard_test",
+    size = "small",
+    srcs = ["shard_test.py"],
+    additional_deps = [
+        ":test_base",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:errors",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
+tf_py_test(
+    name = "shuffle_test",
+    size = "small",
+    srcs = ["shuffle_test.py"],
+    additional_deps = [
+        ":test_base",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:tensor_shape",
-        "//tensorflow/python:variables",
+        "//tensorflow/python:random_seed",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
     ],
 )
 
 tf_py_test(
-    name = "sequence_dataset_op_test",
+    name = "skip_test",
     size = "small",
-    srcs = ["sequence_dataset_op_test.py"],
+    srcs = ["skip_test.py"],
     additional_deps = [
         ":test_base",
         "//third_party/py/numpy",
@@ -496,34 +570,53 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "shard_dataset_op_test",
+    name = "take_test",
     size = "small",
-    srcs = ["shard_dataset_op_test.py"],
+    srcs = ["take_test.py"],
     additional_deps = [
         ":test_base",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
 
 tf_py_test(
-    name = "shuffle_dataset_op_test",
+    name = "text_line_dataset_test",
     size = "small",
-    srcs = ["shuffle_dataset_op_test.py"],
+    srcs = ["text_line_dataset_test.py"],
     additional_deps = [
         ":test_base",
-        "@absl_py//absl/testing:parameterized",
-        "//third_party/py/numpy",
+        "//tensorflow/python/data/ops:iterator_ops",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python/eager:context",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:random_seed",
+        "//tensorflow/python:util",
+    ],
+)
+
+tf_py_test(
+    name = "tf_record_dataset_test",
+    size = "small",
+    srcs = ["tf_record_dataset_test.py"],
+    additional_deps = [
+        ":test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:iterator_ops",
+        "//tensorflow/python/data/ops:readers",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:util",
     ],
 )
 
@@ -543,9 +636,9 @@ py_library(
 )
 
 tf_py_test(
-    name = "window_dataset_op_test",
+    name = "window_test",
     size = "medium",
-    srcs = ["window_dataset_op_test.py"],
+    srcs = ["window_test.py"],
     additional_deps = [
         ":test_base",
         "@absl_py//absl/testing:parameterized",
@@ -561,9 +654,9 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "zip_dataset_op_test",
+    name = "zip_test",
     size = "small",
-    srcs = ["zip_dataset_op_test.py"],
+    srcs = ["zip_test.py"],
     additional_deps = [
         ":test_base",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/data/kernel_tests/batch_test.py b/tensorflow/python/data/kernel_tests/batch_test.py
new file mode 100644
index 0000000000..5b035e5917
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/batch_test.py
@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.batch()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class BatchTest(test_base.DatasetTestBase, parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('even', 28, 14, False),
+      ('uneven_with_remainder', 28, 15, False),
+      ('uneven_without_remainder', 28, 15, True),
+      ('empty', 0, 14, False),
+  )
+  def testBatchDataset(self, count, batch_size, drop_remainder):
+    """Tests the batch dataset logic for various input configurations.
+
+    Args:
+      count: the number of input elements
+      batch_size: the batch size
+      drop_remainder: whether a smaller batch size should be produced if batch
+        size does not divide number of inputs evenly
+    """
+
+    # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
+    # RepeatDataset(count) -> BatchDataset(batch_size).
+    components = (np.arange(7),
+                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
+                  np.array(37.0) * np.arange(7))
+
+    def _map_fn(x, y, z):
+      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
+        _map_fn).repeat(count).batch(batch_size, drop_remainder)
+    get_next = self.getNext(dataset)
+
+    if drop_remainder:
+      dim0 = batch_size
+    else:
+      dim0 = None
+    self.assertEqual(
+        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)],
+        [[dim0] + list(c.shape[1:]) for c in components])
+
+    num_full_batches = (count * 7) // batch_size
+    for i in range(num_full_batches):
+      result = self.evaluate(get_next())
+      for component, result_component in zip(components, result):
+        for j in range(batch_size):
+          self.assertAllEqual(component[(i * batch_size + j) % 7]**2,
+                              result_component[j])
+    if not drop_remainder and (count * 7) % batch_size > 0:
+      result = self.evaluate(get_next())
+      for component, result_component in zip(components, result):
+        for j in range((count * 7) % batch_size):
+          self.assertAllEqual(
+              component[(num_full_batches * batch_size + j) % 7]**2,
+              result_component[j])
+    with self.assertRaises(errors.OutOfRangeError):
+      result = self.evaluate(get_next())
+
+  def testBatchDatasetInvalidBatchSize(self):
+    dataset = (dataset_ops.Dataset.range(10).batch(0))
+    self.assertDatasetProduces(
+        dataset, expected_error=(errors.InvalidArgumentError, ''))
+
+  def testBatchSparse(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5)
+    expected_output = [
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
+            values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
+            dense_shape=[5, 1]) for i in range(2)
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testBatchSparseWithDifferentDenseShapes(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=array_ops.expand_dims(
+              math_ops.range(i, dtype=dtypes.int64), 1),
+          values=array_ops.fill([math_ops.to_int32(i)], i),
+          dense_shape=[i])
+
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5)
+    expected_output = []
+    for i in range(2):
+      expected_indices = []
+      expected_outputs = []
+      for j in range(5):
+        for k in range(i * 5 + j):
+          expected_indices.append([j, k])
+          expected_outputs.append(i * 5 + j)
+      expected_output.append(
+          sparse_tensor.SparseTensorValue(
+              indices=expected_indices,
+              values=expected_outputs,
+              dense_shape=[5, (i + 1) * 5 - 1]))
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testNestedBatchSparse(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5).batch(2)
+    expected_output = [
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0],
+                     [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]],
+            values=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
+            dense_shape=[2, 5, 1])
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testBatchShapeError(self):
+
+    def generator():
+      yield [1.0, 2.0, 3.0]
+      yield [4.0, 5.0, 6.0]
+      yield [7.0, 8.0, 9.0, 10.0]
+
+    dataset = (
+        dataset_ops.Dataset.from_generator(
+            generator, dtypes.float32, output_shapes=[None]).batch(3))
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            r'Cannot batch tensors with different shapes in component 0. First '
+            r'element had shape \[3\] and element 2 had shape \[4\].'))
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/cache_test.py
index 06ce18a9b4..b561cd58ba 100644
--- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/cache_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.cache()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -35,7 +35,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class FileCacheDatasetTest(test_base.DatasetTestBase):
+class FileCacheTest(test_base.DatasetTestBase):
 
   def setUp(self):
     self.tmp_dir = tempfile.mkdtemp()
@@ -166,7 +166,7 @@ class FileCacheDatasetTest(test_base.DatasetTestBase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class MemoryCacheDatasetTest(test_base.DatasetTestBase):
+class MemoryCacheTest(test_base.DatasetTestBase):
 
   def testCacheDatasetPassthrough(self):
     with ops.device("cpu:0"):
diff --git a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/python/data/kernel_tests/concatenate_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/concatenate_test.py
index 3123e32d8e..5d8bfdc8f3 100644
--- a/tensorflow/python/data/kernel_tests/concatenate_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/concatenate_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.concatenate()."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -29,7 +29,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ConcatenateDatasetTest(test_base.DatasetTestBase):
+class ConcatenateTest(test_base.DatasetTestBase):
 
   def testConcatenateDataset(self):
     input_components = (
diff --git a/tensorflow/python/data/kernel_tests/save_restore_experimental_test.py b/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py
similarity index 51%
rename from tensorflow/python/data/kernel_tests/save_restore_experimental_test.py
rename to tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py
index c07d24ef6e..cdaa4fd4d5 100644
--- a/tensorflow/python/data/kernel_tests/save_restore_experimental_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental saving and restoring input pipeline."""
+"""Checkpoint tests for `tf.data.Dataset`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 import os
 
-from tensorflow.python.data.kernel_tests import reader_dataset_ops_test
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
@@ -35,331 +34,7 @@ from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 
 
-class ExperimentalCheckpointFixedLengthRecordReaderTest(
-    reader_dataset_ops_test.FixedLengthRecordReaderTestBase):
-
-  def tearDown(self):
-    # Remove all checkpoint files.
-    prefix = self._iterator_checkpoint_prefix()
-    pattern = prefix + "*"
-    files = gfile.Glob(pattern)
-    map(gfile.Remove, files)
-
-  def _iterator_checkpoint_prefix(self):
-    return os.path.join(self.get_temp_dir(), "iterator")
-
-  def _save_op(self, iterator_resource):
-    iterator_state_variant = gen_dataset_ops.serialize_iterator(
-        iterator_resource)
-    save_op = io_ops.write_file(
-        self._iterator_checkpoint_prefix(),
-        parsing_ops.serialize_tensor(iterator_state_variant))
-    return save_op
-
-  def _restore_op(self, iterator_resource):
-    iterator_state_variant = parsing_ops.parse_tensor(
-        io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant)
-    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
-                                                      iterator_state_variant)
-    return restore_op
-
-  def testSaveRestore(self):
-
-    def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    break_point = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-    # Saving and restoring in same session.
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testRestoreWithoutBuildingDatasetGraph(self):
-
-    def _build_graph(start, stop, num_epochs):
-      dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
-      iterator = dataset.make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    num_epochs = 5
-    break_point = 5
-    break_epoch = 3
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for _ in range(break_epoch):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      # Create an empty IteratorResource and restore the Iterator into it.
-      output_types = dtypes.int64
-      output_shapes = tensor_shape.scalar()
-      iterator = iterator_ops.Iterator.from_structure(output_types,
-                                                      output_shapes)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      get_next = iterator.get_next()
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        for _ in range(break_epoch + 1, num_epochs):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testRestoreInModifiedGraph(self):
-
-    def _build_graph(start, stop):
-      dataset = dataset_ops.Dataset.range(start, stop)
-      iterator = dataset.make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    stop_1 = 8
-    break_point = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      # Intentionally build a graph with a different value for stop to make sure
-      # the original dataset graph is actually getting loaded.
-      init_op, get_next, _, restore_op = _build_graph(start, stop_1)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testInitThenRestore(self):
-    # Note: Calling init_op before restore_op is redundant. This test just makes
-    # sure we do not fail if restore is called on an already initialized
-    # iterator resource.
-
-    def _build_graph(start, stop):
-      dataset = dataset_ops.Dataset.range(start, stop)
-      iterator = dataset.make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    # Saving and restoring in different sessions.
-    start = 2
-    stop = 10
-    break_point = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
-        for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testMultipleSaves(self):
-
-    def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    start = 2
-    stop = 10
-    break_point1 = 5
-    break_point2 = 7
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        for i in range(start, break_point1):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point1, break_point2):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    break_point2 = 7
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(start, stop)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_point2, stop):
-          self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testSaveRestoreWithRepeat(self):
-
-    def _build_graph(start, stop, num_epochs):
-      iterator = dataset_ops.Dataset.range(
-          start, stop).repeat(num_epochs).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    start = 2
-    stop = 10
-    num_epochs = 5
-    break_range = 5
-    break_epoch = 3
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(
-          start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for _ in range(break_epoch - 1):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        for i in range(start, break_range):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        for i in range(break_range, stop):
-          self.assertEqual(i, sess.run(get_next))
-        for _ in range(break_epoch, num_epochs):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testSaveRestoreExhaustedIterator(self):
-
-    def _build_graph(start, stop, num_epochs):
-      iterator = dataset_ops.Dataset.range(
-          start, stop).repeat(num_epochs).make_initializable_iterator()
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-      save_op = self._save_op(iterator._iterator_resource)
-      restore_op = self._restore_op(iterator._iterator_resource)
-      return init_op, get_next, save_op, restore_op
-
-    start = 2
-    stop = 10
-    num_epochs = 5
-    with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, restore_op = _build_graph(
-          start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
-        # Note: There is no checkpoint saved currently so a NotFoundError is
-        # raised.
-        with self.assertRaises(errors.NotFoundError):
-          sess.run(restore_op)
-        for _ in range(num_epochs):
-          for i in range(start, stop):
-            self.assertEqual(i, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-        sess.run(save_op)
-
-    with ops.Graph().as_default() as g:
-      init_op, get_next, _, restore_op = _build_graph(start, stop, num_epochs)
-      with self.session(graph=g) as sess:
-        sess.run(restore_op)
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-
-class ExperimentalCheckpointRangeTest(test_base.DatasetTestBase):
+class DatasetCheckpointTest(test_base.DatasetTestBase):
 
   def tearDown(self):
     # Remove all checkpoint files.
diff --git a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
deleted file mode 100644
index 4a7a946576..0000000000
--- a/tensorflow/python/data/kernel_tests/dataset_constructor_op_test.py
+++ /dev/null
@@ -1,606 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for the experimental input pipeline ops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import time
-
-import numpy as np
-
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import session
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.platform import test
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class DatasetConstructorTest(test_base.DatasetTestBase):
-
-  def testFromTensors(self):
-    """Test a dataset that represents a single tuple of tensors."""
-    components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
-
-    dataset = dataset_ops.Dataset.from_tensors(components)
-
-    self.assertEqual([c.shape for c in components],
-                     nest.flatten(dataset.output_shapes))
-
-    self.assertDatasetProduces(dataset, expected_output=[components])
-
-  def testFromTensorsSparse(self):
-    """Test a dataset that represents a single tuple of tensors."""
-    components = (sparse_tensor.SparseTensorValue(
-        indices=np.array([[0]]),
-        values=np.array([0]),
-        dense_shape=np.array([1])),
-                  sparse_tensor.SparseTensorValue(
-                      indices=np.array([[0, 0], [1, 1]]),
-                      values=np.array([-1, 1]),
-                      dense_shape=np.array([2, 2])))
-
-    dataset = dataset_ops.Dataset.from_tensors(components)
-
-    self.assertEqual(
-        [tensor_shape.TensorShape(c.dense_shape) for c in components],
-        [shape for shape in dataset.output_shapes])
-    self.assertDatasetProduces(dataset, expected_output=[components])
-
-  def testFromTensorsMixed(self):
-    """Test an dataset that represents a single tuple of tensors."""
-    components = (np.array(1), np.array([1, 2, 3]), np.array(37.0),
-                  sparse_tensor.SparseTensorValue(
-                      indices=np.array([[0]]),
-                      values=np.array([0]),
-                      dense_shape=np.array([1])),
-                  sparse_tensor.SparseTensorValue(
-                      indices=np.array([[0, 0], [1, 1]]),
-                      values=np.array([-1, 1]),
-                      dense_shape=np.array([2, 2])))
-
-    dataset = dataset_ops.Dataset.from_tensors(components)
-    self.assertEqual([
-        tensor_shape.TensorShape(c.dense_shape)
-        if sparse_tensor.is_sparse(c) else c.shape for c in components
-    ], [shape for shape in dataset.output_shapes])
-
-    self.assertDatasetProduces(dataset, expected_output=[components])
-
-  def testFromTensorSlices(self):
-    """Test a dataset that represents the slices from a tuple of tensors."""
-    components = (
-        np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile(
-            np.array([[12], [13], [14], [15]]), 22),
-        np.array([37.0, 38.0, 39.0, 40.0])
-    )
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components)
-    get_next = self.getNext(dataset)
-
-    self.assertEqual([c.shape[1:] for c in components],
-                     [shape for shape in dataset.output_shapes])
-
-    for i in range(4):
-      results = self.evaluate(get_next())
-      for component, result_component in zip(components, results):
-        self.assertAllEqual(component[i], result_component)
-    with self.assertRaises(errors.OutOfRangeError):
-      results = self.evaluate(get_next())
-
-  def testFromTensorSlicesSparse(self):
-    """Test a dataset that represents the slices from a tuple of tensors."""
-    components = (sparse_tensor.SparseTensorValue(
-        indices=np.array([[0, 0], [1, 0], [2, 0]]),
-        values=np.array([0, 0, 0]),
-        dense_shape=np.array([3, 1])),
-                  sparse_tensor.SparseTensorValue(
-                      indices=np.array([[0, 0], [1, 1], [2, 2]]),
-                      values=np.array([1, 2, 3]),
-                      dense_shape=np.array([3, 3])))
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components)
-
-    self.assertEqual(
-        [tensor_shape.TensorShape(c.dense_shape[1:]) for c in components],
-        [shape for shape in dataset.output_shapes])
-
-    expected = [
-        (sparse_tensor.SparseTensorValue(
-            indices=np.array([[0]]),
-            values=np.array([0]),
-            dense_shape=np.array([1])),
-         sparse_tensor.SparseTensorValue(
-             indices=np.array([[0]]),
-             values=np.array([1]),
-             dense_shape=np.array([3]))),
-        (sparse_tensor.SparseTensorValue(
-            indices=np.array([[0]]),
-            values=np.array([0]),
-            dense_shape=np.array([1])),
-         sparse_tensor.SparseTensorValue(
-             indices=np.array([[1]]),
-             values=np.array([2]),
-             dense_shape=np.array([3]))),
-        (sparse_tensor.SparseTensorValue(
-            indices=np.array([[0]]),
-            values=np.array([0]),
-            dense_shape=np.array([1])),
-         sparse_tensor.SparseTensorValue(
-             indices=np.array([[2]]),
-             values=np.array([3]),
-             dense_shape=np.array([3]))),
-    ]
-    self.assertDatasetProduces(dataset, expected_output=expected)
-
-  def testFromTensorSlicesMixed(self):
-    """Test a dataset that represents the slices from a tuple of tensors."""
-    components = (np.tile(np.array([[1], [2], [3]]), 20),
-                  np.tile(np.array([[12], [13], [14]]), 22),
-                  np.array([37.0, 38.0, 39.0]),
-                  sparse_tensor.SparseTensorValue(
-                      indices=np.array([[0, 0], [1, 0], [2, 0]]),
-                      values=np.array([0, 0, 0]),
-                      dense_shape=np.array([3, 1])),
-                  sparse_tensor.SparseTensorValue(
-                      indices=np.array([[0, 0], [1, 1], [2, 2]]),
-                      values=np.array([1, 2, 3]),
-                      dense_shape=np.array([3, 3])))
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components)
-    get_next = self.getNext(dataset)
-    self.assertEqual([
-        tensor_shape.TensorShape(c.dense_shape[1:])
-        if sparse_tensor.is_sparse(c) else c.shape[1:] for c in components
-    ], [shape for shape in dataset.output_shapes])
-
-    expected = [
-        (sparse_tensor.SparseTensorValue(
-            indices=np.array([[0]]),
-            values=np.array([0]),
-            dense_shape=np.array([1])),
-         sparse_tensor.SparseTensorValue(
-             indices=np.array([[0]]),
-             values=np.array([1]),
-             dense_shape=np.array([3]))),
-        (sparse_tensor.SparseTensorValue(
-            indices=np.array([[0]]),
-            values=np.array([0]),
-            dense_shape=np.array([1])),
-         sparse_tensor.SparseTensorValue(
-             indices=np.array([[1]]),
-             values=np.array([2]),
-             dense_shape=np.array([3]))),
-        (sparse_tensor.SparseTensorValue(
-            indices=np.array([[0]]),
-            values=np.array([0]),
-            dense_shape=np.array([1])),
-         sparse_tensor.SparseTensorValue(
-             indices=np.array([[2]]),
-             values=np.array([3]),
-             dense_shape=np.array([3]))),
-    ]
-    for i in range(3):
-      results = self.evaluate(get_next())
-      for component, result_component in zip(
-          (list(zip(*components[:3]))[i] + expected[i]), results):
-        if sparse_tensor.is_sparse(component):
-          self.assertSparseValuesEqual(component, result_component)
-        else:
-          self.assertAllEqual(component, result_component)
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-  def testFromTensorSlicesWithDict(self):
-    components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]}
-    dataset = dataset_ops.Dataset.from_tensor_slices(components)
-    get_next = self.getNext(dataset)
-
-    self.assertEqual(dtypes.int32, dataset.output_types["foo"])
-    self.assertEqual(dtypes.float32, dataset.output_types["bar"])
-    self.assertEqual((), dataset.output_shapes["foo"])
-    self.assertEqual((1,), dataset.output_shapes["bar"])
-
-    for i in range(3):
-      results = self.evaluate(get_next())
-      self.assertEqual(components["foo"][i], results["foo"])
-      self.assertEqual(components["bar"][i], results["bar"])
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-  def testSkipEagerFromSparseTensorSlices(self):
-    """Test a dataset based on slices of a `tf.SparseTensor`."""
-    st = array_ops.sparse_placeholder(dtypes.float64)
-    iterator = (dataset_ops.Dataset.from_sparse_tensor_slices(st)
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = sparse_tensor.SparseTensor(*iterator.get_next())
-
-    with self.cached_session() as sess:
-      slices = [[1., 2., 3.], [1.], [1.], [1., 2.], [], [1., 2.], [], [], []]
-
-      # Test with sparse tensor in the appropriate order.
-      indices = np.array(
-          [[i, j] for i in range(len(slices)) for j in range(len(slices[i]))])
-      values = np.array([val for s in slices for val in s])
-      dense_shape = np.array([len(slices), max(len(s) for s in slices) + 1])
-      sparse_feed = sparse_tensor.SparseTensorValue(indices, values,
-                                                    dense_shape)
-      sess.run(init_op, feed_dict={st: sparse_feed})
-      for i, s in enumerate(slices):
-        results = sess.run(get_next)
-        self.assertAllEqual(s, results.values)
-        expected_indices = np.array(
-            [[j] for j in range(len(slices[i]))]).reshape([-1, 1])
-        self.assertAllEqual(expected_indices, results.indices)
-        self.assertAllEqual(dense_shape[1:], results.dense_shape)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Test with sparse tensor in the reverse order, which is not
-      # currently supported.
-      reverse_order_indices = indices[::-1, :]
-      reverse_order_values = values[::-1]
-      sparse_feed = sparse_tensor.SparseTensorValue(
-          reverse_order_indices, reverse_order_values, dense_shape)
-      with self.assertRaises(errors.UnimplementedError):
-        sess.run(init_op, feed_dict={st: sparse_feed})
-
-      # Test with an empty sparse tensor.
-      empty_indices = np.empty((0, 4), dtype=np.int64)
-      empty_values = np.empty((0,), dtype=np.float64)
-      empty_dense_shape = [0, 4, 37, 9]
-      sparse_feed = sparse_tensor.SparseTensorValue(empty_indices, empty_values,
-                                                    empty_dense_shape)
-      sess.run(init_op, feed_dict={st: sparse_feed})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  # pylint: disable=g-long-lambda,unnecessary-lambda
-  def testNestedStructure(self):
-    components = (np.array([1, 2, 3], dtype=np.int64),
-                  (np.array([4., 5.]), np.array([6., 7.])),
-                  np.array([8, 9, 10], dtype=np.int64))
-
-    dataset = dataset_ops.Dataset.from_tensors(components)
-    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
-                       dtypes.int64), dataset.output_types)
-    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
-
-    dataset = dataset.shuffle(10, 10)
-    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
-                       dtypes.int64), dataset.output_types)
-    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
-
-    dataset = dataset.repeat(-1)
-    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
-                       dtypes.int64), dataset.output_types)
-    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
-
-    dataset = dataset.filter(lambda x, y, z: True)
-    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
-                       dtypes.int64), dataset.output_types)
-    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
-
-    dataset = dataset.take(5)
-    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
-                       dtypes.int64), dataset.output_types)
-    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
-
-    dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1])))
-    self.assertEquals(((dtypes.int64, dtypes.int64),
-                       (dtypes.float64, dtypes.float64)), dataset.output_types)
-    self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes)
-
-    dataset = dataset.flat_map(
-        lambda x, y: dataset_ops.Dataset.from_tensors(((x[0], x[1]),
-                                                       (y[0], y[1])))
-    )
-    self.assertEquals(((dtypes.int64, dtypes.int64),
-                       (dtypes.float64, dtypes.float64)), dataset.output_types)
-    self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes)
-
-    dataset = dataset.batch(32)
-    self.assertEquals(((dtypes.int64, dtypes.int64),
-                       (dtypes.float64, dtypes.float64)), dataset.output_types)
-    self.assertEquals((([None, 3], [None, 3]), ([None, 2], [None, 2])),
-                      nest.pack_sequence_as(dataset.output_shapes, [
-                          s.as_list()
-                          for s in nest.flatten(dataset.output_shapes)
-                      ]))
-
-    # Define a separate set of components with matching leading
-    # dimension for the from-slices constructor.
-    components_for_slices = (np.array([1, 2, 3], dtype=np.int64),
-                             (np.array([4., 5., 6.]), np.array([7., 8., 9.])),
-                             np.array([10, 11, 12], dtype=np.int64))
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices)
-    self.assertEquals((dtypes.int64,
-                       (dtypes.float64, dtypes.float64), dtypes.int64),
-                      dataset.output_types)
-    self.assertEquals(([], ([], []), []), dataset.output_shapes)
-
-  # TODO(b/117581999): more specific shapes in eager mode.
-  def testSkipEagerNestedStructure(self):
-    components = (np.array([1, 2, 3], dtype=np.int64), (np.array([4., 5.]),
-                                                        np.array([6., 7.])),
-                  np.array([8, 9, 10], dtype=np.int64))
-
-    dataset = dataset_ops.Dataset.from_tensors(components)
-    dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1])))
-
-    dataset = dataset.flat_map(
-        lambda x, y: dataset_ops.Dataset.from_tensors(
-            ((x[0], x[1]), (y[0], y[1])))).batch(32)
-
-    get_next = self.getNext(dataset)
-    (w, x), (y, z) = get_next()
-    self.assertEquals(dtypes.int64, w.dtype)
-    self.assertEquals(dtypes.int64, x.dtype)
-    self.assertEquals(dtypes.float64, y.dtype)
-    self.assertEquals(dtypes.float64, z.dtype)
-    self.assertEquals([None, 3], w.shape.as_list())
-    self.assertEquals([None, 3], x.shape.as_list())
-    self.assertEquals([None, 2], y.shape.as_list())
-    self.assertEquals([None, 2], z.shape.as_list())
-
-    get_next = self.getNext(dataset)
-    (w, x), (y, z) = get_next()
-    self.assertEquals(dtypes.int64, w.dtype)
-    self.assertEquals(dtypes.int64, x.dtype)
-    self.assertEquals(dtypes.float64, y.dtype)
-    self.assertEquals(dtypes.float64, z.dtype)
-    self.assertEquals([None, 3], w.shape.as_list())
-    self.assertEquals([None, 3], x.shape.as_list())
-    self.assertEquals([None, 2], y.shape.as_list())
-    self.assertEquals([None, 2], z.shape.as_list())
-
-  def testNestedDict(self):
-    components = {"a": {"aa": 1, "ab": [2.0, 2.0]}, "b": [3, 3, 3]}
-    dataset = dataset_ops.Dataset.from_tensors(components)
-    self.assertEquals(dtypes.int32, dataset.output_types["a"]["aa"])
-    self.assertEquals(dtypes.float32, dataset.output_types["a"]["ab"])
-    self.assertEquals(dtypes.int32, dataset.output_types["b"])
-    self.assertEquals([], dataset.output_shapes["a"]["aa"])
-    self.assertEquals([2], dataset.output_shapes["a"]["ab"])
-    self.assertEquals([3], dataset.output_shapes["b"])
-
-  def testNonSequenceNestedStructure(self):
-    components = np.array([1, 2, 3], dtype=np.int64)
-
-    dataset = dataset_ops.Dataset.from_tensors(components)
-    self.assertEquals(dtypes.int64, dataset.output_types)
-    self.assertEquals([3], dataset.output_shapes)
-
-    dataset = dataset.filter(
-        lambda x: math_ops.reduce_all(math_ops.equal(x, components)))
-    self.assertEquals(dtypes.int64, dataset.output_types)
-    self.assertEquals([3], dataset.output_shapes)
-
-    dataset = dataset.map(lambda x: array_ops.stack([x, x]))
-    self.assertEquals(dtypes.int64, dataset.output_types)
-    self.assertEquals([2, 3], dataset.output_shapes)
-
-    dataset = dataset.flat_map(
-        lambda x: dataset_ops.Dataset.from_tensor_slices(x))
-    self.assertEquals(dtypes.int64, dataset.output_types)
-    self.assertEquals([3], dataset.output_shapes)
-
-    get_next = self.getNext(dataset)
-    self.assertEquals(dtypes.int64, get_next().dtype)
-    self.assertEquals([3], get_next().shape)
-
-  def testSkipEagerSplitPipelineFailsWithPlacementError(self):
-    with session.Session(
-        target="",
-        config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess:
-
-      dataset = dataset_ops.Dataset.from_tensors(0)
-
-      # Define a pipeline that attempts to use variables on two
-      # different devices.
-      #
-      # Initialize the variables before creating to iterator, to avoid the
-      # placement algorithm overriding the DT_RESOURCE colocation constraints.
-      with ops.device("/cpu:0"):
-        var_0 = resource_variable_ops.ResourceVariable(initial_value=0)
-        dataset = dataset.map(lambda x: x + var_0.read_value())
-      sess.run(var_0.initializer)
-
-      with ops.device("/cpu:1"):
-        var_1 = resource_variable_ops.ResourceVariable(initial_value=0)
-        dataset = dataset.map(lambda x: x + var_1.read_value())
-      sess.run(var_1.initializer)
-
-      iterator = dataset.make_initializable_iterator()
-      sess.run(iterator.initializer)
-
-      with self.assertRaisesRegexp(
-          errors.FailedPreconditionError,
-          "Error while reading resource variable Variable"):
-        sess.run(iterator.get_next())
-
-
-# TODO(b/119837791): Add eager benchmarks as well.
-class DatasetConstructorBenchmark(test.Benchmark):
-
-  def benchmarkSliceRepeatBatch(self):
-    input_size = 10000
-    batch_size = 100
-    num_epochs = 100
-
-    input_data = np.random.randn(input_size)
-
-    dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data)
-        .repeat(num_epochs + 1).batch(batch_size))
-    iterator = dataset.make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        sess.run(next_element)
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          sess.run(next_element)
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    print("Slice/repeat/batch with sess.run() input size: %d batch size: %d "
-          "Median wall time per element: %f" % (input_size, batch_size,
-                                                median_wall_time))
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="benchmark_slice_repeat_batch_input_%d_batch_%d" % (input_size,
-                                                                 batch_size))
-
-  def benchmarkSliceRepeatBatchCallable(self):
-    input_size = 10000
-    batch_size = 100
-    num_epochs = 100
-
-    input_data = np.random.randn(input_size)
-
-    dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data)
-        .repeat(num_epochs + 1).batch(batch_size))
-    iterator = dataset.make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      get_next_element = sess.make_callable(next_element)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        get_next_element()
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          get_next_element()
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    print(
-        "Slice/repeat/batch with callable input size: %d batch size: %d Median"
-        " wall time per element: %f" % (input_size, batch_size,
-                                        median_wall_time))
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="benchmark_slice_repeat_batch_callable_input_%d_batch_%d" %
-        (input_size, batch_size))
-
-  def benchmarkReshapeSliceRepeatCallable(self):
-    input_size = 10000
-    batch_size = 100
-    num_epochs = 100
-
-    input_data = np.random.randn(input_size)
-
-    dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100))
-        .repeat(num_epochs + 1))
-    iterator = dataset.make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      get_next_element = sess.make_callable(next_element)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        get_next_element()
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          get_next_element()
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    print("Reshape/slice/repeat with callable input size: %d batch size: %d "
-          "Median wall time per element: %f" % (input_size, batch_size,
-                                                median_wall_time))
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="benchmark_reshape_slice_repeat_callable_input_%d_batch_%d" %
-        (input_size, batch_size))
-
-  def benchmarkSliceBatchCacheRepeatCallable(self):
-    input_size = 10000
-    batch_size = 100
-    num_epochs = 100
-
-    input_data = np.random.randn(input_size)
-
-    dataset = (
-        dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size)
-        .cache().repeat(num_epochs + 1))
-    iterator = dataset.make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    with session.Session() as sess:
-      sess.run(iterator.initializer)
-      get_next_element = sess.make_callable(next_element)
-      # Run one whole epoch to burn in the computation.
-      for _ in range(input_size // batch_size):
-        get_next_element()
-      deltas = []
-      try:
-        while True:
-          start = time.time()
-          get_next_element()
-          deltas.append(time.time() - start)
-      except errors.OutOfRangeError:
-        pass
-
-    median_wall_time = np.median(deltas)
-    print(
-        "Slice/batch/cache/repeat with callable input size: %d batch size: %d "
-        "Median wall time per element: %f"
-        % (input_size, batch_size, median_wall_time))
-    self.report_benchmark(
-        iters=len(deltas),
-        wall_time=median_wall_time,
-        name="benchmark_slice_batch_cache_repeat_callable_input_%d_batch_%d" %
-        (input_size, batch_size))
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/kernel_tests/dataset_ops_test.py b/tensorflow/python/data/kernel_tests/dataset_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/dataset_ops_test.py
rename to tensorflow/python/data/kernel_tests/dataset_test.py
index 373cdc0a77..7dbab60f9c 100644
--- a/tensorflow/python/data/kernel_tests/dataset_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the input pipeline ops."""
+"""Tests for `tf.data.Dataset`."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -37,7 +37,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
+class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testAsSerializedGraph(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -81,7 +81,7 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
        lambda: readers.FixedLengthRecordDataset("", 42)),
       ("FromGenerator",
        lambda: dataset_ops.Dataset.from_generator(
-           DatasetOpsTest.make_gen(), dtypes.int32),
+           DatasetTest.make_gen(), dtypes.int32),
        1),
       ("FromTensors", lambda: dataset_ops.Dataset.from_tensors([42])),
       ("FromTensorSlices", lambda: dataset_ops.Dataset.from_tensors([42])),
@@ -313,6 +313,5 @@ class DatasetOpsTest(test_base.DatasetTestBase, parameterized.TestCase):
           round_trip_dataset, [self.evaluate(tf_value_fn())],
           requires_initialization=True)
 
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/kernel_tests/filter_test.py
similarity index 77%
rename from tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/filter_test.py
index fba474b97b..afaf954cbc 100644
--- a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_test.py
@@ -12,30 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.filter()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 import numpy as np
 
-from tensorflow.python.client import session
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class FilterDatasetTest(test_base.DatasetTestBase):
+class FilterTest(test_base.DatasetTestBase):
 
   def testFilterDataset(self):
     components = (
@@ -129,41 +124,5 @@ class FilterDatasetTest(test_base.DatasetTestBase):
                          [next_element() for next_element in next_elements]))
 
 
-# TODO(b/119837791): Add eager benchmarks too.
-class FilterDatasetBenchmark(test.Benchmark):
-
-  def _benchmark(self, predicate, name):
-    with ops.Graph().as_default():
-      dataset = (
-          dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
-      iterator = dataset.make_one_shot_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for _ in range(5):
-          sess.run(next_element.op)
-        deltas = []
-        for _ in range(100):
-          start = time.time()
-          for _ in range(100):
-            sess.run(next_element.op)
-          end = time.time()
-          deltas.append(end - start)
-
-        median_wall_time = np.median(deltas) / 100
-        print("Filter dataset using %s. Median wall time: %f" %
-              (name, median_wall_time))
-        self.report_benchmark(
-            iters=100,
-            wall_time=median_wall_time,
-            name="benchmark_filter_dataset_%s" % name)
-
-  def benchmarkSimpleFunction(self):
-    self._benchmark(array_ops.identity, "simple_function")
-
-  def benchmarkReturnComponentOptimization(self):
-    self._benchmark(lambda x: x, "return_component")
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py b/tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py
new file mode 100644
index 0000000000..9503e57ca7
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/fixed_length_record_dataset_test.py
@@ -0,0 +1,171 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.FixedLengthRecordDataset`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gzip
+import os
+import zlib
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import readers
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.util import compat
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FixedLengthRecordDatasetTest(test_base.DatasetTestBase):
+
+  def setUp(self):
+    super(FixedLengthRecordDatasetTest, self).setUp()
+    self._num_files = 2
+    self._num_records = 7
+    self._header_bytes = 5
+    self._record_bytes = 3
+    self._footer_bytes = 2
+
+  def _record(self, f, r):
+    return compat.as_bytes(str(f * 2 + r) * self._record_bytes)
+
+  def _createFiles(self, compression_type=None):
+    filenames = []
+    for i in range(self._num_files):
+      fn = os.path.join(self.get_temp_dir(), "fixed_length_record.%d.txt" % i)
+      filenames.append(fn)
+
+      contents = []
+      contents.append(b"H" * self._header_bytes)
+      for j in range(self._num_records):
+        contents.append(self._record(i, j))
+      contents.append(b"F" * self._footer_bytes)
+      contents = b"".join(contents)
+
+      if not compression_type:
+        with open(fn, "wb") as f:
+          f.write(contents)
+      elif compression_type == "GZIP":
+        with gzip.GzipFile(fn, "wb") as f:
+          f.write(contents)
+      elif compression_type == "ZLIB":
+        contents = zlib.compress(contents)
+        with open(fn, "wb") as f:
+          f.write(contents)
+      else:
+        raise ValueError("Unsupported compression_type", compression_type)
+
+    return filenames
+
+  def _testFixedLengthRecordDataset(self, compression_type=None):
+    test_filenames = self._createFiles(compression_type=compression_type)
+
+    def dataset_fn(filenames, num_epochs, batch_size=None):
+      repeat_dataset = readers.FixedLengthRecordDataset(
+          filenames,
+          self._record_bytes,
+          self._header_bytes,
+          self._footer_bytes,
+          compression_type=compression_type).repeat(num_epochs)
+      if batch_size:
+        return repeat_dataset.batch(batch_size)
+      return repeat_dataset
+
+    # Basic test: read from file 0.
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[0]], 1),
+        expected_output=[
+            self._record(0, i) for i in range(self._num_records)
+        ])
+
+    # Basic test: read from file 1.
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[1]], 1),
+        expected_output=[
+            self._record(1, i) for i in range(self._num_records)
+        ])
+
+    # Basic test: read from both files.
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 1), expected_output=expected_output)
+
+    # Test repeated iteration through both files.
+    get_next = self.getNext(dataset_fn(test_filenames, 10))
+    for _ in range(10):
+      for j in range(self._num_files):
+        for i in range(self._num_records):
+          self.assertEqual(self._record(j, i), self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+    # Test batched and repeated iteration through both files.
+    get_next = self.getNext(dataset_fn(test_filenames, 10, self._num_records))
+    for _ in range(10):
+      for j in range(self._num_files):
+        self.assertAllEqual(
+            [self._record(j, i) for i in range(self._num_records)],
+            self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+  def testFixedLengthRecordDatasetNoCompression(self):
+    self._testFixedLengthRecordDataset()
+
+  def testFixedLengthRecordDatasetGzipCompression(self):
+    self._testFixedLengthRecordDataset(compression_type="GZIP")
+
+  def testFixedLengthRecordDatasetZlibCompression(self):
+    self._testFixedLengthRecordDataset(compression_type="ZLIB")
+
+  def testFixedLengthRecordDatasetBuffering(self):
+    test_filenames = self._createFiles()
+    dataset = readers.FixedLengthRecordDataset(
+        test_filenames,
+        self._record_bytes,
+        self._header_bytes,
+        self._footer_bytes,
+        buffer_size=10)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testFixedLengthRecordDatasetWrongSize(self):
+    test_filenames = self._createFiles()
+    dataset = readers.FixedLengthRecordDataset(
+        test_filenames,
+        self._record_bytes + 1,  # Incorrect record length.
+        self._header_bytes,
+        self._footer_bytes,
+        buffer_size=10)
+    self.assertDatasetProduces(
+        dataset,
+        expected_error=(
+            errors.InvalidArgumentError,
+            r"Excluding the header \(5 bytes\) and footer \(2 bytes\), input "
+            r"file \".*fixed_length_record.0.txt\" has body length 21 bytes, "
+            r"which is not an exact multiple of the record length \(4 bytes\).")
+        )
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/flat_map_test.py
similarity index 97%
rename from tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/flat_map_test.py
index 9292f20637..5f11c2e3a7 100644
--- a/tensorflow/python/data/kernel_tests/flat_map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/flat_map_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.flat_map()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -33,7 +33,7 @@ from tensorflow.python.training import server_lib
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class FlatMapDatasetTest(test_base.DatasetTestBase):
+class FlatMapTest(test_base.DatasetTestBase):
 
   # pylint: disable=g-long-lambda
   def testFlatMapDataset(self):
diff --git a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/data/kernel_tests/from_generator_test.py
similarity index 99%
rename from tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
rename to tensorflow/python/data/kernel_tests/from_generator_test.py
index cb8cb9a77d..4d82c2111c 100644
--- a/tensorflow/python/data/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/data/kernel_tests/from_generator_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for tf.data.Dataset.from_generator()."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -31,7 +31,7 @@ from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
 
-class DatasetConstructorTest(test_base.DatasetTestBase):
+class FromGeneratorTest(test_base.DatasetTestBase):
 
   def _testFromGenerator(self, generator, elem_sequence, num_repeats,
                          output_types=None):
diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
new file mode 100644
index 0000000000..d23ac0ebe9
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
@@ -0,0 +1,85 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.from_sparse_tensor_slices()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FromSparseTensorSlicesTest(test_base.DatasetTestBase):
+
+  def testSkipEagerFromSparseTensorSlices(self):
+    """Test a dataset based on slices of a `tf.SparseTensor`."""
+    st = array_ops.sparse_placeholder(dtypes.float64)
+    iterator = (dataset_ops.Dataset.from_sparse_tensor_slices(st)
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = sparse_tensor.SparseTensor(*iterator.get_next())
+
+    with self.cached_session() as sess:
+      slices = [[1., 2., 3.], [1.], [1.], [1., 2.], [], [1., 2.], [], [], []]
+
+      # Test with sparse tensor in the appropriate order.
+      indices = np.array(
+          [[i, j] for i in range(len(slices)) for j in range(len(slices[i]))])
+      values = np.array([val for s in slices for val in s])
+      dense_shape = np.array([len(slices), max(len(s) for s in slices) + 1])
+      sparse_feed = sparse_tensor.SparseTensorValue(indices, values,
+                                                    dense_shape)
+      sess.run(init_op, feed_dict={st: sparse_feed})
+      for i, s in enumerate(slices):
+        results = sess.run(get_next)
+        self.assertAllEqual(s, results.values)
+        expected_indices = np.array(
+            [[j] for j in range(len(slices[i]))]).reshape([-1, 1])
+        self.assertAllEqual(expected_indices, results.indices)
+        self.assertAllEqual(dense_shape[1:], results.dense_shape)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Test with sparse tensor in the reverse order, which is not
+      # currently supported.
+      reverse_order_indices = indices[::-1, :]
+      reverse_order_values = values[::-1]
+      sparse_feed = sparse_tensor.SparseTensorValue(
+          reverse_order_indices, reverse_order_values, dense_shape)
+      with self.assertRaises(errors.UnimplementedError):
+        sess.run(init_op, feed_dict={st: sparse_feed})
+
+      # Test with an empty sparse tensor.
+      empty_indices = np.empty((0, 4), dtype=np.int64)
+      empty_values = np.empty((0,), dtype=np.float64)
+      empty_dense_shape = [0, 4, 37, 9]
+      sparse_feed = sparse_tensor.SparseTensorValue(empty_indices, empty_values,
+                                                    empty_dense_shape)
+      sess.run(init_op, feed_dict={st: sparse_feed})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
new file mode 100644
index 0000000000..9a480e5678
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
@@ -0,0 +1,177 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.from_tensor_slices()."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FromTensorSlicesTest(test_base.DatasetTestBase):
+
+  def testFromTensorSlices(self):
+    """Test a dataset that represents the slices from a tuple of tensors."""
+    components = (
+        np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile(
+            np.array([[12], [13], [14], [15]]), 22),
+        np.array([37.0, 38.0, 39.0, 40.0])
+    )
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+    get_next = self.getNext(dataset)
+
+    self.assertEqual([c.shape[1:] for c in components],
+                     [shape for shape in dataset.output_shapes])
+
+    for i in range(4):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(components, results):
+        self.assertAllEqual(component[i], result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      results = self.evaluate(get_next())
+
+  def testSkipEagerFromTensorSlicesSparse(self):
+    """Test a dataset that represents the slices from a tuple of tensors."""
+    components = (sparse_tensor.SparseTensorValue(
+        indices=np.array([[0, 0], [1, 0], [2, 0]]),
+        values=np.array([0, 0, 0]),
+        dense_shape=np.array([3, 1])),
+                  sparse_tensor.SparseTensorValue(
+                      indices=np.array([[0, 0], [1, 1], [2, 2]]),
+                      values=np.array([1, 2, 3]),
+                      dense_shape=np.array([3, 3])))
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+
+    self.assertEqual(
+        [tensor_shape.TensorShape(c.dense_shape[1:]) for c in components],
+        [shape for shape in dataset.output_shapes])
+
+    expected = [
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[0]]),
+             values=np.array([1]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[1]]),
+             values=np.array([2]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[2]]),
+             values=np.array([3]),
+             dense_shape=np.array([3]))),
+    ]
+    self.assertDatasetProduces(dataset, expected_output=expected)
+
+  def testFromTensorSlicesMixed(self):
+    """Test a dataset that represents the slices from a tuple of tensors."""
+    components = (np.tile(np.array([[1], [2], [3]]), 20),
+                  np.tile(np.array([[12], [13], [14]]), 22),
+                  np.array([37.0, 38.0, 39.0]),
+                  sparse_tensor.SparseTensorValue(
+                      indices=np.array([[0, 0], [1, 0], [2, 0]]),
+                      values=np.array([0, 0, 0]),
+                      dense_shape=np.array([3, 1])),
+                  sparse_tensor.SparseTensorValue(
+                      indices=np.array([[0, 0], [1, 1], [2, 2]]),
+                      values=np.array([1, 2, 3]),
+                      dense_shape=np.array([3, 3])))
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+    get_next = self.getNext(dataset)
+    self.assertEqual([
+        tensor_shape.TensorShape(c.dense_shape[1:])
+        if sparse_tensor.is_sparse(c) else c.shape[1:] for c in components
+    ], [shape for shape in dataset.output_shapes])
+
+    expected = [
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[0]]),
+             values=np.array([1]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[1]]),
+             values=np.array([2]),
+             dense_shape=np.array([3]))),
+        (sparse_tensor.SparseTensorValue(
+            indices=np.array([[0]]),
+            values=np.array([0]),
+            dense_shape=np.array([1])),
+         sparse_tensor.SparseTensorValue(
+             indices=np.array([[2]]),
+             values=np.array([3]),
+             dense_shape=np.array([3]))),
+    ]
+    for i in range(3):
+      results = self.evaluate(get_next())
+      for component, result_component in zip(
+          (list(zip(*components[:3]))[i] + expected[i]), results):
+        if sparse_tensor.is_sparse(component):
+          self.assertSparseValuesEqual(component, result_component)
+        else:
+          self.assertAllEqual(component, result_component)
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+  def testFromTensorSlicesWithDict(self):
+    components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]}
+    dataset = dataset_ops.Dataset.from_tensor_slices(components)
+    get_next = self.getNext(dataset)
+
+    self.assertEqual(dtypes.int32, dataset.output_types["foo"])
+    self.assertEqual(dtypes.float32, dataset.output_types["bar"])
+    self.assertEqual((), dataset.output_shapes["foo"])
+    self.assertEqual((1,), dataset.output_shapes["bar"])
+
+    for i in range(3):
+      results = self.evaluate(get_next())
+      self.assertEqual(components["foo"][i], results["foo"])
+      self.assertEqual(components["bar"][i], results["bar"])
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/from_tensors_test.py b/tensorflow/python/data/kernel_tests/from_tensors_test.py
new file mode 100644
index 0000000000..2857817e14
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/from_tensors_test.py
@@ -0,0 +1,258 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.from_tensors()."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.client import session
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FromTensorsTest(test_base.DatasetTestBase):
+
+  def testFromTensors(self):
+    """Test a dataset that represents a single tuple of tensors."""
+    components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+
+    self.assertEqual([c.shape for c in components],
+                     nest.flatten(dataset.output_shapes))
+
+    self.assertDatasetProduces(dataset, expected_output=[components])
+
+  def testSkipEagerFromTensorsSparse(self):
+    """Test a dataset that represents a single tuple of tensors."""
+    components = (sparse_tensor.SparseTensorValue(
+        indices=np.array([[0]]),
+        values=np.array([0]),
+        dense_shape=np.array([1])),
+                  sparse_tensor.SparseTensorValue(
+                      indices=np.array([[0, 0], [1, 1]]),
+                      values=np.array([-1, 1]),
+                      dense_shape=np.array([2, 2])))
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+
+    self.assertEqual(
+        [tensor_shape.TensorShape(c.dense_shape) for c in components],
+        [shape for shape in dataset.output_shapes])
+    self.assertDatasetProduces(dataset, expected_output=[components])
+
+  def testFromTensorsMixed(self):
+    """Test an dataset that represents a single tuple of tensors."""
+    components = (np.array(1), np.array([1, 2, 3]), np.array(37.0),
+                  sparse_tensor.SparseTensorValue(
+                      indices=np.array([[0]]),
+                      values=np.array([0]),
+                      dense_shape=np.array([1])),
+                  sparse_tensor.SparseTensorValue(
+                      indices=np.array([[0, 0], [1, 1]]),
+                      values=np.array([-1, 1]),
+                      dense_shape=np.array([2, 2])))
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+    self.assertEqual([
+        tensor_shape.TensorShape(c.dense_shape)
+        if sparse_tensor.is_sparse(c) else c.shape for c in components
+    ], [shape for shape in dataset.output_shapes])
+
+    self.assertDatasetProduces(dataset, expected_output=[components])
+
+  # pylint: disable=g-long-lambda,unnecessary-lambda
+  def testNestedStructure(self):
+    components = (np.array([1, 2, 3], dtype=np.int64),
+                  (np.array([4., 5.]), np.array([6., 7.])),
+                  np.array([8, 9, 10], dtype=np.int64))
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
+                       dtypes.int64), dataset.output_types)
+    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
+
+    dataset = dataset.shuffle(10, 10)
+    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
+                       dtypes.int64), dataset.output_types)
+    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
+
+    dataset = dataset.repeat(-1)
+    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
+                       dtypes.int64), dataset.output_types)
+    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
+
+    dataset = dataset.filter(lambda x, y, z: True)
+    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
+                       dtypes.int64), dataset.output_types)
+    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
+
+    dataset = dataset.take(5)
+    self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
+                       dtypes.int64), dataset.output_types)
+    self.assertEquals(([3], ([2], [2]), [3]), dataset.output_shapes)
+
+    dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1])))
+    self.assertEquals(((dtypes.int64, dtypes.int64),
+                       (dtypes.float64, dtypes.float64)), dataset.output_types)
+    self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes)
+
+    dataset = dataset.flat_map(
+        lambda x, y: dataset_ops.Dataset.from_tensors(((x[0], x[1]),
+                                                       (y[0], y[1])))
+    )
+    self.assertEquals(((dtypes.int64, dtypes.int64),
+                       (dtypes.float64, dtypes.float64)), dataset.output_types)
+    self.assertEquals((([3], [3]), ([2], [2])), dataset.output_shapes)
+
+    dataset = dataset.batch(32)
+    self.assertEquals(((dtypes.int64, dtypes.int64),
+                       (dtypes.float64, dtypes.float64)), dataset.output_types)
+    self.assertEquals((([None, 3], [None, 3]), ([None, 2], [None, 2])),
+                      nest.pack_sequence_as(dataset.output_shapes, [
+                          s.as_list()
+                          for s in nest.flatten(dataset.output_shapes)
+                      ]))
+
+    # Define a separate set of components with matching leading
+    # dimension for the from-slices constructor.
+    components_for_slices = (np.array([1, 2, 3], dtype=np.int64),
+                             (np.array([4., 5., 6.]), np.array([7., 8., 9.])),
+                             np.array([10, 11, 12], dtype=np.int64))
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices)
+    self.assertEquals((dtypes.int64,
+                       (dtypes.float64, dtypes.float64), dtypes.int64),
+                      dataset.output_types)
+    self.assertEquals(([], ([], []), []), dataset.output_shapes)
+
+  # TODO(b/117581999): more specific shapes in eager mode.
+  def testSkipEagerNestedStructure(self):
+    components = (np.array([1, 2, 3], dtype=np.int64), (np.array([4., 5.]),
+                                                        np.array([6., 7.])),
+                  np.array([8, 9, 10], dtype=np.int64))
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+    dataset = dataset.map(lambda x, y, z: ((x, z), (y[0], y[1])))
+
+    dataset = dataset.flat_map(
+        lambda x, y: dataset_ops.Dataset.from_tensors(
+            ((x[0], x[1]), (y[0], y[1])))).batch(32)
+
+    get_next = self.getNext(dataset)
+    (w, x), (y, z) = get_next()
+    self.assertEquals(dtypes.int64, w.dtype)
+    self.assertEquals(dtypes.int64, x.dtype)
+    self.assertEquals(dtypes.float64, y.dtype)
+    self.assertEquals(dtypes.float64, z.dtype)
+    self.assertEquals([None, 3], w.shape.as_list())
+    self.assertEquals([None, 3], x.shape.as_list())
+    self.assertEquals([None, 2], y.shape.as_list())
+    self.assertEquals([None, 2], z.shape.as_list())
+
+    get_next = self.getNext(dataset)
+    (w, x), (y, z) = get_next()
+    self.assertEquals(dtypes.int64, w.dtype)
+    self.assertEquals(dtypes.int64, x.dtype)
+    self.assertEquals(dtypes.float64, y.dtype)
+    self.assertEquals(dtypes.float64, z.dtype)
+    self.assertEquals([None, 3], w.shape.as_list())
+    self.assertEquals([None, 3], x.shape.as_list())
+    self.assertEquals([None, 2], y.shape.as_list())
+    self.assertEquals([None, 2], z.shape.as_list())
+
+  def testNestedDict(self):
+    components = {"a": {"aa": 1, "ab": [2.0, 2.0]}, "b": [3, 3, 3]}
+    dataset = dataset_ops.Dataset.from_tensors(components)
+    self.assertEquals(dtypes.int32, dataset.output_types["a"]["aa"])
+    self.assertEquals(dtypes.float32, dataset.output_types["a"]["ab"])
+    self.assertEquals(dtypes.int32, dataset.output_types["b"])
+    self.assertEquals([], dataset.output_shapes["a"]["aa"])
+    self.assertEquals([2], dataset.output_shapes["a"]["ab"])
+    self.assertEquals([3], dataset.output_shapes["b"])
+
+  def testNonSequenceNestedStructure(self):
+    components = np.array([1, 2, 3], dtype=np.int64)
+
+    dataset = dataset_ops.Dataset.from_tensors(components)
+    self.assertEquals(dtypes.int64, dataset.output_types)
+    self.assertEquals([3], dataset.output_shapes)
+
+    dataset = dataset.filter(
+        lambda x: math_ops.reduce_all(math_ops.equal(x, components)))
+    self.assertEquals(dtypes.int64, dataset.output_types)
+    self.assertEquals([3], dataset.output_shapes)
+
+    dataset = dataset.map(lambda x: array_ops.stack([x, x]))
+    self.assertEquals(dtypes.int64, dataset.output_types)
+    self.assertEquals([2, 3], dataset.output_shapes)
+
+    dataset = dataset.flat_map(
+        lambda x: dataset_ops.Dataset.from_tensor_slices(x))
+    self.assertEquals(dtypes.int64, dataset.output_types)
+    self.assertEquals([3], dataset.output_shapes)
+
+    get_next = self.getNext(dataset)
+    self.assertEquals(dtypes.int64, get_next().dtype)
+    self.assertEquals([3], get_next().shape)
+
+  def testSkipEagerSplitPipelineFailsWithPlacementError(self):
+    with session.Session(
+        target="",
+        config=config_pb2.ConfigProto(device_count={"CPU": 2})) as sess:
+
+      dataset = dataset_ops.Dataset.from_tensors(0)
+
+      # Define a pipeline that attempts to use variables on two
+      # different devices.
+      #
+      # Initialize the variables before creating to iterator, to avoid the
+      # placement algorithm overriding the DT_RESOURCE colocation constraints.
+      with ops.device("/cpu:0"):
+        var_0 = resource_variable_ops.ResourceVariable(initial_value=0)
+        dataset = dataset.map(lambda x: x + var_0.read_value())
+      sess.run(var_0.initializer)
+
+      with ops.device("/cpu:1"):
+        var_1 = resource_variable_ops.ResourceVariable(initial_value=0)
+        dataset = dataset.map(lambda x: x + var_1.read_value())
+      sess.run(var_1.initializer)
+
+      iterator = dataset.make_initializable_iterator()
+      sess.run(iterator.initializer)
+
+      with self.assertRaisesRegexp(
+          errors.FailedPreconditionError,
+          "Error while reading resource variable Variable"):
+        sess.run(iterator.get_next())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/inputs_test.py b/tensorflow/python/data/kernel_tests/inputs_test.py
deleted file mode 100644
index 03df502d14..0000000000
--- a/tensorflow/python/data/kernel_tests/inputs_test.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl.testing import parameterized
-import numpy as np
-
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import readers
-from tensorflow.python.data.util import nest
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.framework import test_util
-from tensorflow.python.platform import test
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class InputsTest(test_base.DatasetTestBase, parameterized.TestCase):
-
-  @staticmethod
-  def make_apply_fn(dataset):
-
-    def apply_fn(dataset):
-
-      def _apply_fn(dataset):
-        return dataset.cache()
-
-      return dataset.apply(_apply_fn)
-
-    return apply_fn
-
-  @staticmethod
-  def make_gen():
-
-    def gen():
-      yield 42
-
-    return gen
-
-  @staticmethod
-  def make_interleave_fn(dataset, num_parallel_calls=None):
-
-    def interleave_fn(dataset):
-      return dataset.interleave(
-          lambda x: dataset_ops.Dataset.range(0),
-          cycle_length=2,
-          num_parallel_calls=num_parallel_calls)
-
-    return interleave_fn
-
-  @parameterized.named_parameters(
-      ("FixedLengthRecord", readers.FixedLengthRecordDataset("", 42)),
-      ("FromGenerator",
-       dataset_ops.Dataset.from_generator(make_gen.__func__(), dtypes.int32),
-       1),
-      ("FromSparseTensorSlices",
-       dataset_ops.Dataset.from_sparse_tensor_slices(
-           sparse_tensor.SparseTensor(
-               indices=np.array([[0, 0], [1, 0], [2, 0]]),
-               values=np.array([0, 0, 0]),
-               dense_shape=np.array([3, 1])))),
-      ("FromTensors", dataset_ops.Dataset.from_tensors([42])),
-      ("FromTensorSlices", dataset_ops.Dataset.from_tensors([42])),
-      ("Range", dataset_ops.Dataset.range(10)),
-      ("TextLine", readers.TextLineDataset("")),
-      ("TFRecord", readers.TFRecordDataset(""), 1),
-  )
-  def testDatasetSourceInputs(self, dataset, num_inputs=0):
-    self.assertEqual(num_inputs, len(dataset._inputs()))
-
-  @parameterized.named_parameters(
-      ("Apply", make_apply_fn.__func__(dataset_ops.Dataset.range(0)),
-       dataset_ops.Dataset.range(0)),
-      ("Batch", lambda x: x.batch(10), dataset_ops.Dataset.range(0)),
-      ("Cache", lambda x: x.cache(), dataset_ops.Dataset.range(0)),
-      ("Filter", lambda x: x.filter(lambda x: True),
-       dataset_ops.Dataset.range(0)),
-      ("FlatMap", lambda x: x.flat_map(lambda x: dataset_ops.Dataset.range(0)),
-       dataset_ops.Dataset.range(0)),
-      ("Interleave", make_interleave_fn.__func__(dataset_ops.Dataset.range(0)),
-       dataset_ops.Dataset.range(0)),
-      ("Map", lambda x: x.map(lambda x: x), dataset_ops.Dataset.range(0)),
-      ("PaddedBatch", lambda x: x.padded_batch(10, []),
-       dataset_ops.Dataset.range(0)),
-      ("ParallelInterleave",
-       make_interleave_fn.__func__(dataset_ops.Dataset.range(0), 2),
-       dataset_ops.Dataset.range(0)),
-      ("ParallelMap", lambda x: x.map(lambda x: x, num_parallel_calls=2),
-       dataset_ops.Dataset.range(0)),
-      ("Repeat", lambda x: x.repeat(), dataset_ops.Dataset.range(0)),
-      ("Shuffle", lambda x: x.shuffle(10), dataset_ops.Dataset.range(0)),
-      ("Skip", lambda x: x.skip(1), dataset_ops.Dataset.range(0)),
-      ("Take", lambda x: x.take(1), dataset_ops.Dataset.range(0)),
-      ("Window", lambda x: x.window(10), dataset_ops.Dataset.range(0)),
-  )
-  def testUnaryTransformationInputs(self, dataset_fn, input_dataset):
-    self.assertEqual([input_dataset], dataset_fn(input_dataset)._inputs())
-
-  @parameterized.named_parameters(
-      ("Concatenate", lambda x, y: x.concatenate(y),
-       dataset_ops.Dataset.range(0), dataset_ops.Dataset.range(1)))
-  def testBinaryTransformationInputs(self, dataset_fn, input1, input2):
-    self.assertEqual([input1, input2], dataset_fn(input1, input2)._inputs())
-
-  @parameterized.named_parameters(
-      ("ZipOne", dataset_ops.Dataset.zip, (dataset_ops.Dataset.range(0))),
-      ("ZipNest", dataset_ops.Dataset.zip,
-       (dataset_ops.Dataset.range(0),
-        (dataset_ops.Dataset.range(1), dataset_ops.Dataset.range(2)))),
-      ("ZipTuple", dataset_ops.Dataset.zip,
-       (dataset_ops.Dataset.range(0), dataset_ops.Dataset.range(1))))
-  def testVariadicTransformationInputs(self, dataset_fn, input_datasets):
-    self.assertEqual(
-        nest.flatten(input_datasets),
-        dataset_fn(input_datasets)._inputs())
-
-  def testCollectInputs(self):
-    ds1 = dataset_ops.Dataset.range(0)
-    ds2 = ds1.concatenate(ds1)
-    ds3 = dataset_ops.Dataset.zip((ds2, ds1, ds2))
-
-    inputs = []
-    queue = [ds3]
-    while queue:
-      ds = queue[0]
-      queue = queue[1:]
-      queue.extend(ds._inputs())
-      inputs.append(ds)
-
-    self.assertEqual(5, inputs.count(ds1))
-    self.assertEqual(2, inputs.count(ds2))
-    self.assertEqual(1, inputs.count(ds3))
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py b/tensorflow/python/data/kernel_tests/interleave_test.py
similarity index 69%
rename from tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/interleave_test.py
index f0b16591f7..cd1d850304 100644
--- a/tensorflow/python/data/kernel_tests/interleave_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.interleave()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -116,7 +116,9 @@ def _make_coordinated_sloppy_dataset(input_values, cycle_length, block_length,
   dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
       2).interleave(interleave_fn, cycle_length, block_length,
                     num_parallel_calls).with_options(options)
-  return dataset, coordination_events
+  iterator = dataset.make_one_shot_iterator()
+  get_next = iterator.get_next()
+  return get_next, coordination_events
 
 
 def _repeat(values, count):
@@ -133,7 +135,7 @@ def _repeat(values, count):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class InterleaveTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
       ("1", [4, 5, 6], 1, 1, [
@@ -191,8 +193,10 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         count).interleave(
             lambda x: dataset_ops.Dataset.from_tensors(x).repeat(x),
             cycle_length, block_length, num_parallel_calls)
-    expected_output = [element for element in _interleave(
-        _repeat(input_values, count), cycle_length, block_length)]
+    expected_output = [
+        element for element in _interleave(
+            _repeat(input_values, count), cycle_length, block_length)
+    ]
     self.assertDatasetProduces(dataset, expected_output)
 
   @parameterized.named_parameters(
@@ -249,100 +253,59 @@ class InterleaveDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
-
-class InterleaveDatasetTestWithConfig(test_base.DatasetTestBase,
-                                      parameterized.TestCase):
-
-  @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1),
-      ("2", np.int64([4, 5, 6]), 2, 3),
-      ("3", np.int64([4, 5, 6]), 3, 2),
-      ("4", np.int64([4, 0, 6]), 2, 3),
-  )
-  @test_util.run_in_graph_and_eager_modes(
-      config=config_pb2.ConfigProto(
-          inter_op_parallelism_threads=2, use_per_session_threads=True))
-  def testSloppyInterleaveInOrder(self, input_values, cycle_length,
-                                  block_length):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls=1)
-    get_next = self.getNext(dataset)
-    for expected_element in _interleave(
-        _repeat(input_values, 2), cycle_length, block_length):
-      coordination_events[expected_element].set()
-      self.assertEqual(expected_element * expected_element,
-                       self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
   @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1),
-      ("2", np.int64([4, 5, 6]), 2, 3),
-      ("3", np.int64([4, 5, 6]), 3, 2),
-      ("4", np.int64([4, 0, 6]), 2, 3),
+      ("1", np.int64([4, 5, 6]), 2, 1, 1),
+      ("2", np.int64([4, 5, 6]), 2, 1, 2),
+      ("3", np.int64([4, 5, 6]), 2, 3, 1),
+      ("4", np.int64([4, 5, 6]), 2, 3, 2),
+      ("5", np.int64([4, 5, 6]), 3, 2, 1),
+      ("6", np.int64([4, 5, 6]), 3, 2, 2),
+      ("7", np.int64([4, 5, 6]), 3, 2, 3),
+      ("8", np.int64([4, 0, 6]), 2, 3, 1),
+      ("9", np.int64([4, 0, 6]), 2, 3, 2),
   )
-  @test_util.run_in_graph_and_eager_modes(
-      config=config_pb2.ConfigProto(
-          inter_op_parallelism_threads=3, use_per_session_threads=True))
-  def testSloppyInterleaveInOrder_2(self, input_values, cycle_length,
-                                    block_length):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls=2)
-    get_next = self.getNext(dataset)
-    for expected_element in _interleave(
-        _repeat(input_values, 2), cycle_length, block_length):
-      coordination_events[expected_element].set()
-      self.assertEqual(expected_element * expected_element,
-                       self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
+  def testSkipEagerSloppyInterleaveInOrder(self, input_values, cycle_length,
+                                           block_length, num_parallel_calls):
+    get_next, coordination_events = _make_coordinated_sloppy_dataset(
+        input_values, cycle_length, block_length, num_parallel_calls)
+    config = config_pb2.ConfigProto(
+        inter_op_parallelism_threads=num_parallel_calls + 1,
+        use_per_session_threads=True)
+    with self.cached_session(config=config) as sess:
+      for expected_element in _interleave(
+          _repeat(input_values, 2), cycle_length, block_length):
+        coordination_events[expected_element].set()
+        self.assertEqual(expected_element * expected_element,
+                         self.evaluate(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
 
   @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1),
-      ("2", np.int64([4, 5, 6]), 2, 3),
-      ("3", np.int64([4, 0, 6]), 2, 3),
+      ("1", np.int64([4, 5, 6]), 2, 1, 2),
+      ("2", np.int64([4, 5, 6]), 2, 3, 2),
+      ("3", np.int64([4, 5, 6]), 3, 2, 3),
+      ("4", np.int64([4, 0, 6]), 2, 3, 2),
   )
-  @test_util.run_in_graph_and_eager_modes(
-      config=config_pb2.ConfigProto(
-          inter_op_parallelism_threads=3, use_per_session_threads=True))
-  def testSloppyInterleaveOutOfOrder(self, input_values, cycle_length,
-                                     block_length):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls=2)
-    get_next = self.getNext(dataset)
-    elements = [
-        x for x in _interleave(
-            _repeat(input_values, 2), cycle_length, block_length)
-    ]
-    for i in [1, 4, 7]:
-      elements[i], elements[i + 1] = elements[i + 1], elements[i]
-
-    for element in elements:
-      coordination_events[element].set()
-      self.assertEqual(element * element, self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-  @test_util.run_in_graph_and_eager_modes(
-      config=config_pb2.ConfigProto(
-          inter_op_parallelism_threads=4, use_per_session_threads=True))
-  def testSloppyInterleaveOutOfOrder_2(self):
-    input_values, cycle_length, block_length = np.int64([4, 5, 6]), 3, 2
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls=3)
-    get_next = self.getNext(dataset)
-    elements = [
-        x for x in _interleave(
-            _repeat(input_values, 2), cycle_length, block_length)
-    ]
-    for i in [1, 4, 7]:
-      elements[i], elements[i + 1] = elements[i + 1], elements[i]
-
-    for element in elements:
-      coordination_events[element].set()
-      self.assertEqual(element * element, self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
+  def testSkipEagerSloppyInterleaveOutOfOrder(self, input_values, cycle_length,
+                                              block_length, num_parallel_calls):
+    get_next, coordination_events = _make_coordinated_sloppy_dataset(
+        input_values, cycle_length, block_length, num_parallel_calls)
+    config = config_pb2.ConfigProto(
+        inter_op_parallelism_threads=num_parallel_calls + 1,
+        use_per_session_threads=True)
+    with self.cached_session(config=config) as sess:
+      elements = [
+          x for x in _interleave(
+              _repeat(input_values, 2), cycle_length, block_length)
+      ]
+      for i in [1, 4, 7]:
+        elements[i], elements[i + 1] = elements[i + 1], elements[i]
+
+      for element in elements:
+        coordination_events[element].set()
+        self.assertEqual(element * element, self.evaluate(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
new file mode 100644
index 0000000000..fc4164c81a
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
@@ -0,0 +1,129 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Checkpoint tests for `tf.data.Iterator`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+import os
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+from tensorflow.python.training import checkpoint_management
+from tensorflow.python.training.checkpointable import util as checkpointable_utils
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class IteratorCheckpointingTest(test_base.DatasetTestBase):
+
+  def testSaveRestoreOneShotIterator(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map(
+        math_ops.square).batch(2)
+    iterator = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
+    get_next = iterator.get_next if context.executing_eagerly(
+    ) else functools.partial(self.evaluate, iterator.get_next())
+    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
+    self.assertAllEqual([1, 4], get_next())
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.assertAllEqual([9, 16], get_next())
+    self.assertAllEqual([25, 36], get_next())
+    checkpoint.restore(save_path).run_restore_ops()
+    self.assertAllEqual([9, 16], get_next())
+    self.assertAllEqual([25, 36], get_next())
+    with self.assertRaises(errors.OutOfRangeError):
+      get_next()
+
+  def testSaveRestoreMultipleIterator(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    dataset = dataset_ops.Dataset.from_tensor_slices(
+        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
+    dataset = dataset.map(math_ops.square).batch(2)
+    iterator_1 = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
+    get_next_1 = iterator_1.get_next if context.executing_eagerly(
+    ) else functools.partial(self.evaluate, iterator_1.get_next())
+    iterator_2 = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
+    get_next_2 = iterator_2.get_next if context.executing_eagerly(
+    ) else functools.partial(self.evaluate, iterator_2.get_next())
+    dataset_2 = dataset_ops.Dataset.range(10)
+    iterator_3 = iter(dataset_2) if context.executing_eagerly(
+    ) else dataset_2.make_one_shot_iterator()
+    get_next_3 = iterator_3.get_next if context.executing_eagerly(
+    ) else functools.partial(self.evaluate, iterator_3.get_next())
+    checkpoint = checkpointable_utils.Checkpoint(
+        iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3)
+    self.assertAllEqual([1, 4], get_next_1())
+    self.assertAllEqual(0, get_next_3())
+    self.assertAllEqual(1, get_next_3())
+    self.assertAllEqual(2, get_next_3())
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.assertAllEqual([1, 4], get_next_2())
+    self.assertAllEqual([9, 16], get_next_2())
+    self.assertAllEqual(3, get_next_3())
+    checkpoint.restore(save_path).run_restore_ops()
+    self.assertAllEqual([9, 16], get_next_1())
+    self.assertAllEqual([1, 4], get_next_2())
+    self.assertAllEqual(3, get_next_3())
+
+  def testRestoreExhaustedIterator(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    dataset = dataset_ops.Dataset.range(3)
+    iterator = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_one_shot_iterator()
+    get_next = iterator.get_next if context.executing_eagerly(
+    ) else functools.partial(self.evaluate, iterator.get_next())
+    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
+    self.assertAllEqual(0, get_next())
+    self.assertAllEqual(1, get_next())
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.assertAllEqual(2, get_next())
+    checkpoint.restore(save_path).run_restore_ops()
+    self.assertAllEqual(2, get_next())
+    save_path = checkpoint.save(checkpoint_prefix)
+    checkpoint.restore(save_path).run_restore_ops()
+    with self.assertRaises(errors.OutOfRangeError):
+      get_next()
+
+  def testRestoreInReconstructedIteratorInitializable(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    dataset = dataset_ops.Dataset.range(10)
+    iterator = iter(dataset) if context.executing_eagerly(
+    ) else dataset.make_initializable_iterator()
+    get_next = iterator.get_next
+    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
+    for i in range(5):
+      checkpoint.restore(
+          checkpoint_management.latest_checkpoint(
+              checkpoint_directory)).initialize_or_restore()
+      for j in range(2):
+        self.assertEqual(i * 2 + j, self.evaluate(get_next()))
+      checkpoint.save(file_prefix=checkpoint_prefix)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
rename to tensorflow/python/data/kernel_tests/iterator_cluster_test.py
index bf5fd781d6..c1f856ec62 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops that need test_util."""
+"""Tests for `tf.data.Iterator` using distributed sessions."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_test.py
similarity index 87%
rename from tensorflow/python/data/kernel_tests/iterator_ops_test.py
rename to tensorflow/python/data/kernel_tests/iterator_test.py
index 3d4db80883..de95a53e57 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_test.py
@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Iterator`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
 import os
 import warnings
 
@@ -28,7 +27,6 @@ from tensorflow.core.protobuf import cluster_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.compat import compat as forward_compat
-from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
@@ -51,9 +49,7 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
-from tensorflow.python.training import checkpoint_management
 from tensorflow.python.training import server_lib
-from tensorflow.python.training.checkpointable import util as checkpointable_utils
 from tensorflow.python.util import compat
 
 
@@ -864,99 +860,5 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       self.assertEqual("overridden_name", next_element.op.name)
 
 
-@test_util.run_all_in_graph_and_eager_modes
-class IteratorCheckpointingTest(test_base.DatasetTestBase):
-
-  def testSaveRestoreOneShotIterator(self):
-    checkpoint_directory = self.get_temp_dir()
-    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map(
-        math_ops.square).batch(2)
-    iterator = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
-    get_next = iterator.get_next if context.executing_eagerly(
-    ) else functools.partial(self.evaluate, iterator.get_next())
-    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
-    self.assertAllEqual([1, 4], get_next())
-    save_path = checkpoint.save(checkpoint_prefix)
-    self.assertAllEqual([9, 16], get_next())
-    self.assertAllEqual([25, 36], get_next())
-    checkpoint.restore(save_path).run_restore_ops()
-    self.assertAllEqual([9, 16], get_next())
-    self.assertAllEqual([25, 36], get_next())
-    with self.assertRaises(errors.OutOfRangeError):
-      get_next()
-
-  def testSaveRestoreMultipleIterator(self):
-    checkpoint_directory = self.get_temp_dir()
-    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    dataset = dataset_ops.Dataset.from_tensor_slices(
-        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
-    dataset = dataset.map(math_ops.square).batch(2)
-    iterator_1 = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
-    get_next_1 = iterator_1.get_next if context.executing_eagerly(
-    ) else functools.partial(self.evaluate, iterator_1.get_next())
-    iterator_2 = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
-    get_next_2 = iterator_2.get_next if context.executing_eagerly(
-    ) else functools.partial(self.evaluate, iterator_2.get_next())
-    dataset_2 = dataset_ops.Dataset.range(10)
-    iterator_3 = iter(dataset_2) if context.executing_eagerly(
-    ) else dataset_2.make_one_shot_iterator()
-    get_next_3 = iterator_3.get_next if context.executing_eagerly(
-    ) else functools.partial(self.evaluate, iterator_3.get_next())
-    checkpoint = checkpointable_utils.Checkpoint(
-        iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3)
-    self.assertAllEqual([1, 4], get_next_1())
-    self.assertAllEqual(0, get_next_3())
-    self.assertAllEqual(1, get_next_3())
-    self.assertAllEqual(2, get_next_3())
-    save_path = checkpoint.save(checkpoint_prefix)
-    self.assertAllEqual([1, 4], get_next_2())
-    self.assertAllEqual([9, 16], get_next_2())
-    self.assertAllEqual(3, get_next_3())
-    checkpoint.restore(save_path).run_restore_ops()
-    self.assertAllEqual([9, 16], get_next_1())
-    self.assertAllEqual([1, 4], get_next_2())
-    self.assertAllEqual(3, get_next_3())
-
-  def testRestoreExhaustedIterator(self):
-    checkpoint_directory = self.get_temp_dir()
-    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    dataset = dataset_ops.Dataset.range(3)
-    iterator = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
-    get_next = iterator.get_next if context.executing_eagerly(
-    ) else functools.partial(self.evaluate, iterator.get_next())
-    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
-    self.assertAllEqual(0, get_next())
-    self.assertAllEqual(1, get_next())
-    save_path = checkpoint.save(checkpoint_prefix)
-    self.assertAllEqual(2, get_next())
-    checkpoint.restore(save_path).run_restore_ops()
-    self.assertAllEqual(2, get_next())
-    save_path = checkpoint.save(checkpoint_prefix)
-    checkpoint.restore(save_path).run_restore_ops()
-    with self.assertRaises(errors.OutOfRangeError):
-      get_next()
-
-  def testRestoreInReconstructedIteratorInitializable(self):
-    checkpoint_directory = self.get_temp_dir()
-    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    dataset = dataset_ops.Dataset.range(10)
-    iterator = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_initializable_iterator()
-    get_next = iterator.get_next
-    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
-    for i in range(5):
-      checkpoint.restore(
-          checkpoint_management.latest_checkpoint(
-              checkpoint_directory)).initialize_or_restore()
-      for j in range(2):
-        self.assertEqual(i * 2 + j, self.evaluate(get_next()))
-      checkpoint.save(file_prefix=checkpoint_prefix)
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py b/tensorflow/python/data/kernel_tests/list_files_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/list_files_test.py
index 7efe3c692a..26c536086b 100644
--- a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.list_files()`."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -31,7 +31,7 @@ from tensorflow.python.util import compat
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ListFilesDatasetOpTest(test_base.DatasetTestBase):
+class ListFilesTest(test_base.DatasetTestBase):
 
   def setUp(self):
     self.tmp_dir = tempfile.mkdtemp()
@@ -208,5 +208,6 @@ class ListFilesDatasetOpTest(test_base.DatasetTestBase):
         assert_items_equal=True)
 
 
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_test.py
similarity index 90%
rename from tensorflow/python/data/kernel_tests/map_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/map_test.py
index 187b9da14c..a9c4d79429 100644
--- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -19,7 +19,6 @@ from __future__ import print_function
 
 from collections import namedtuple
 import threading
-import time
 import warnings
 
 from absl.testing import parameterized
@@ -27,7 +26,6 @@ import numpy as np
 
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import session
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -521,7 +519,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(init_op)
       for i in range(10):
-        self.assertEqual(i * 2 + i ** 2, sess.run(get_next))
+        self.assertEqual(i * 2 + i**2, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -570,7 +568,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       sess.run(init_op)
-      self.assertAllEqual(row ** 2, sess.run(get_next))
+      self.assertAllEqual(row**2, sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
@@ -1057,108 +1055,5 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(get_next)
 
 
-class MapDatasetBenchmark(test.Benchmark):
-
-  def benchmarkChainOfMaps(self):
-    chain_lengths = [0, 1, 2, 5, 10, 20, 50]
-    for chain_length in chain_lengths:
-      for mode in ["general", "single-threaded", "short-circuit"]:
-        if mode == "general":
-          map_fn = lambda x: x + 1
-          use_inter_op_parallelism = True
-          print_label = ""
-          benchmark_label = ""
-        if mode == "single-threaded":
-          map_fn = lambda x: x + 1
-          use_inter_op_parallelism = False
-          print_label = " (single threaded mode)"
-          benchmark_label = "_single_threaded"
-        if mode == "short-circuit":
-          map_fn = lambda x: x
-          use_inter_op_parallelism = True  # should not have any significance
-          print_label = " (short circuit mode)"
-          benchmark_label = "_short_circuit"
-
-        with ops.Graph().as_default():
-          dataset = dataset_ops.Dataset.from_tensors(0).repeat(None)
-          for _ in range(chain_length):
-            dataset = dataset_ops.MapDataset(
-                dataset,
-                map_fn,
-                use_inter_op_parallelism=use_inter_op_parallelism)
-          iterator = dataset.make_one_shot_iterator()
-          next_element = iterator.get_next()
-
-          with session.Session() as sess:
-            for _ in range(5):
-              sess.run(next_element.op)
-            deltas = []
-            for _ in range(100):
-              start = time.time()
-              for _ in range(100):
-                sess.run(next_element.op)
-              end = time.time()
-              deltas.append(end - start)
-
-            median_wall_time = np.median(deltas) / 100
-            print("Map dataset chain length%s: %d Median wall time: %f" %
-                  (print_label, chain_length, median_wall_time))
-            self.report_benchmark(
-                iters=1000,
-                wall_time=median_wall_time,
-                name="benchmark_map_dataset_chain_latency_%d%s" %
-                (chain_length, benchmark_label))
-
-  def benchmarkMapFanOut(self):
-    fan_outs = [1, 2, 5, 10, 20, 50, 100]
-    for fan_out in fan_outs:
-      for mode in ["general", "single-threaded", "short-circuit"]:
-        if mode == "general":
-          map_fn = lambda *xs: [x + 1 for x in xs]
-          use_inter_op_parallelism = True
-          print_label = ""
-          benchmark_label = ""
-        if mode == "single-threaded":
-          map_fn = lambda *xs: [x + 1 for x in xs]
-          use_inter_op_parallelism = False
-          print_label = " (single threaded mode)"
-          benchmark_label = "_single_threaded"
-        if mode == "short-circuit":
-          map_fn = lambda *xs: xs
-          use_inter_op_parallelism = True  # should not have any significance
-          print_label = " (short circuit mode)"
-          benchmark_label = "_short_circuit"
-
-        with ops.Graph().as_default():
-          dataset = dataset_ops.Dataset.from_tensors(
-              tuple(0 for _ in range(fan_out))).repeat(None)
-          dataset = dataset_ops.MapDataset(
-              dataset,
-              map_fn,
-              use_inter_op_parallelism=use_inter_op_parallelism)
-          iterator = dataset.make_one_shot_iterator()
-          next_element = iterator.get_next()
-
-          with session.Session() as sess:
-            for _ in range(5):
-              sess.run(next_element[0].op)
-            deltas = []
-            for _ in range(100):
-              start = time.time()
-              for _ in range(100):
-                sess.run(next_element[0].op)
-              end = time.time()
-              deltas.append(end - start)
-
-            median_wall_time = np.median(deltas) / 100
-            print("Map dataset fan out%s: %d Median wall time: %f" %
-                  (print_label, fan_out, median_wall_time))
-            self.report_benchmark(
-                iters=1000,
-                wall_time=median_wall_time,
-                name="benchmark_map_dataset_fan_out_%d%s" % (fan_out,
-                                                             benchmark_label))
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index afb0939397..2ca9961585 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""MultiDeviceIterator tests."""
+"""Tests for `tf.data.MultiDeviceIterator`."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -31,7 +31,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-# TODO(b/117581999): Add eager coverage for the following tests.
+# TODO(b/117581999): Add eager coverage.
 class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
   def testNoGetNext(self):
diff --git a/tensorflow/python/data/kernel_tests/optional_ops_test.py b/tensorflow/python/data/kernel_tests/optional_test.py
similarity index 99%
rename from tensorflow/python/data/kernel_tests/optional_ops_test.py
rename to tensorflow/python/data/kernel_tests/optional_test.py
index 5406a202a3..856985e9fd 100644
--- a/tensorflow/python/data/kernel_tests/optional_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the Optional data type wrapper."""
+"""Tests for `tf.data.Optional`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
diff --git a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/padded_batch_test.py
similarity index 58%
rename from tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/padded_batch_test.py
index 94c37a0f2c..5f20d7b424 100644
--- a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/padded_batch_test.py
@@ -13,20 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.padded_batch()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.python.client import session
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -34,154 +30,17 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
 
-@test_util.run_all_in_graph_and_eager_modes
-class BatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
-
-  @parameterized.named_parameters(
-      ('even', 28, 14, False),
-      ('uneven_with_remainder', 28, 15, False),
-      ('uneven_without_remainder', 28, 15, True),
-      ('empty', 0, 14, False),
-  )
-  def testBatchDataset(self, count, batch_size, drop_remainder):
-    """Tests the batch dataset logic for various input configurations.
-
-    Args:
-      count: the number of input elements
-      batch_size: the batch size
-      drop_remainder: whether a smaller batch size should be produced if batch
-        size does not divide number of inputs evenly
-    """
-
-    # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
-    # RepeatDataset(count) -> BatchDataset(batch_size).
-    components = (np.arange(7),
-                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
-                  np.array(37.0) * np.arange(7))
-
-    def _map_fn(x, y, z):
-      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(components).map(
-        _map_fn).repeat(count).batch(batch_size, drop_remainder)
-    get_next = self.getNext(dataset)
-
-    if drop_remainder:
-      dim0 = batch_size
-    else:
-      dim0 = None
-    self.assertEqual(
-        [ts.as_list() for ts in nest.flatten(dataset.output_shapes)],
-        [[dim0] + list(c.shape[1:]) for c in components])
-
-    num_full_batches = (count * 7) // batch_size
-    for i in range(num_full_batches):
-      result = self.evaluate(get_next())
-      for component, result_component in zip(components, result):
-        for j in range(batch_size):
-          self.assertAllEqual(component[(i * batch_size + j) % 7]**2,
-                              result_component[j])
-    if not drop_remainder and (count * 7) % batch_size > 0:
-      result = self.evaluate(get_next())
-      for component, result_component in zip(components, result):
-        for j in range((count * 7) % batch_size):
-          self.assertAllEqual(
-              component[(num_full_batches * batch_size + j) % 7]**2,
-              result_component[j])
-    with self.assertRaises(errors.OutOfRangeError):
-      result = self.evaluate(get_next())
-
-  def testBatchDatasetInvalidBatchSize(self):
-    dataset = (dataset_ops.Dataset.range(10).batch(0))
-    self.assertDatasetProduces(
-        dataset, expected_error=(errors.InvalidArgumentError, ''))
-
-  def testBatchSparse(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=[[0]], values=(i * [1]), dense_shape=[1])
-
-    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5)
-    expected_output = [
-        sparse_tensor.SparseTensorValue(
-            indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
-            values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
-            dense_shape=[5, 1]) for i in range(2)
-    ]
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testBatchSparseWithDifferentDenseShapes(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=array_ops.expand_dims(
-              math_ops.range(i, dtype=dtypes.int64), 1),
-          values=array_ops.fill([math_ops.to_int32(i)], i),
-          dense_shape=[i])
-
-    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5)
-    expected_output = []
-    for i in range(2):
-      expected_indices = []
-      expected_outputs = []
-      for j in range(5):
-        for k in range(i * 5 + j):
-          expected_indices.append([j, k])
-          expected_outputs.append(i * 5 + j)
-      expected_output.append(
-          sparse_tensor.SparseTensorValue(
-              indices=expected_indices,
-              values=expected_outputs,
-              dense_shape=[5, (i + 1) * 5 - 1]))
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testNestedBatchSparse(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=[[0]], values=(i * [1]), dense_shape=[1])
-
-    dataset = dataset_ops.Dataset.range(10).map(_sparse).batch(5).batch(2)
-    expected_output = [
-        sparse_tensor.SparseTensorValue(
-            indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [0, 4, 0],
-                     [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [1, 4, 0]],
-            values=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
-            dense_shape=[2, 5, 1])
-    ]
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testBatchShapeError(self):
-
-    def generator():
-      yield [1.0, 2.0, 3.0]
-      yield [4.0, 5.0, 6.0]
-      yield [7.0, 8.0, 9.0, 10.0]
-
-    dataset = (
-        dataset_ops.Dataset.from_generator(
-            generator, dtypes.float32, output_shapes=[None]).batch(3))
-    self.assertDatasetProduces(
-        dataset,
-        expected_error=(
-            errors.InvalidArgumentError,
-            r'Cannot batch tensors with different shapes in component 0. First '
-            r'element had shape \[3\] and element 2 had shape \[4\].'))
-
-
 def _random_seq_lens(count):
   return np.random.randint(20, size=(count,)).astype(np.int32)
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
       ('default_padding', _random_seq_lens(32), 4, [-1], False),
@@ -377,56 +236,5 @@ class PaddedBatchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           5, padded_shapes=shape_as_tensor)
 
 
-# TODO(b/119837791): Add eager benchmarks too.
-class BatchDatasetBenchmark(test.Benchmark):
-
-  def benchmarkBatchSparse(self):
-    non_zeros_per_row_values = [0, 1, 5, 10, 100]
-    batch_size_values = [1, 32, 64, 128, 1024]
-
-    sparse_placeholder = array_ops.sparse_placeholder(dtype=dtypes.int64)
-    batch_size_placeholder = array_ops.placeholder(dtype=dtypes.int64, shape=[])
-
-    dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat(
-        ).batch(batch_size_placeholder)
-    iterator = dataset.make_initializable_iterator()
-    next_element = iterator.get_next()
-
-    for non_zeros_per_row in non_zeros_per_row_values:
-
-      sparse_value = sparse_tensor.SparseTensorValue(
-          indices=np.arange(non_zeros_per_row, dtype=np.int64)[:, np.newaxis],
-          values=np.arange(non_zeros_per_row, dtype=np.int64),
-          dense_shape=[1000])
-
-      for batch_size in batch_size_values:
-
-        with session.Session() as sess:
-          sess.run(iterator.initializer, feed_dict={
-              sparse_placeholder: sparse_value,
-              batch_size_placeholder: batch_size})
-          # Run five steps to warm up the session caches before taking the
-          # first measurement.
-          for _ in range(5):
-            sess.run(next_element.indices.op)
-          deltas = []
-          for _ in range(100):
-            start = time.time()
-            for _ in range(100):
-              sess.run(next_element.indices.op)
-            end = time.time()
-            deltas.append(end - start)
-
-        median_wall_time = np.median(deltas) / 100.0
-
-        print('Batch sparse dataset non-zeros per row: %d batch_size: %d '
-              'wall time: %f'
-              % (non_zeros_per_row, batch_size, median_wall_time))
-        self.report_benchmark(
-            iters=10000, wall_time=median_wall_time,
-            name='benchmark_batch_sparse_dataset_nnz_%d_batch_size_%d' % (
-                non_zeros_per_row, batch_size))
-
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/prefetch_test.py
similarity index 93%
rename from tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/prefetch_test.py
index 41b0689947..a143ba0ac6 100644
--- a/tensorflow/python/data/kernel_tests/prefetch_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/prefetch_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Test PrefetchDataset."""
+"""Tests for `tf.data.Dataset.prefetch()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -27,7 +27,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class PrefetchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class PrefetchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.parameters((-1), (0), (5))
   def testBufferSize(self, buffer_size):
@@ -40,6 +40,5 @@ class PrefetchDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertDatasetProduces(
         dataset, expected_error=(errors.InvalidArgumentError, "buffer_size"))
 
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py b/tensorflow/python/data/kernel_tests/range_test.py
similarity index 96%
rename from tensorflow/python/data/kernel_tests/range_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/range_test.py
index 907cb59096..3f5d25e7f3 100644
--- a/tensorflow/python/data/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/range_test.py
@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Test RangeDataset."""
+"""Tests for `tf.data.Dataset.range()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
@@ -26,7 +25,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class RangeDatasetTest(test_base.DatasetTestBase):
+class RangeTest(test_base.DatasetTestBase):
 
   def testStop(self):
     dataset = dataset_ops.Dataset.range(5)
diff --git a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
deleted file mode 100644
index 483a79513a..0000000000
--- a/tensorflow/python/data/kernel_tests/reader_dataset_ops_test.py
+++ /dev/null
@@ -1,446 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for the experimental input pipeline ops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import gzip
-import os
-import zlib
-
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import readers
-from tensorflow.python.eager import context
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import test_util
-from tensorflow.python.lib.io import python_io
-from tensorflow.python.platform import test
-from tensorflow.python.util import compat
-
-
-try:
-  import psutil  # pylint: disable=g-import-not-at-top
-  psutil_import_succeeded = True
-except ImportError:
-  psutil_import_succeeded = False
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TextLineDatasetTest(test_base.DatasetTestBase):
-
-  def _lineText(self, f, l):
-    return compat.as_bytes("%d: %d" % (f, l))
-
-  def _createFiles(self,
-                   num_files,
-                   num_lines,
-                   crlf=False,
-                   compression_type=None):
-    filenames = []
-    for i in range(num_files):
-      fn = os.path.join(self.get_temp_dir(), "text_line.%d.txt" % i)
-      filenames.append(fn)
-      contents = []
-      for j in range(num_lines):
-        contents.append(self._lineText(i, j))
-        # Always include a newline after the record unless it is
-        # at the end of the file, in which case we include it
-        if j + 1 != num_lines or i == 0:
-          contents.append(b"\r\n" if crlf else b"\n")
-      contents = b"".join(contents)
-
-      if not compression_type:
-        with open(fn, "wb") as f:
-          f.write(contents)
-      elif compression_type == "GZIP":
-        with gzip.GzipFile(fn, "wb") as f:
-          f.write(contents)
-      elif compression_type == "ZLIB":
-        contents = zlib.compress(contents)
-        with open(fn, "wb") as f:
-          f.write(contents)
-      else:
-        raise ValueError("Unsupported compression_type", compression_type)
-
-    return filenames
-
-  def _testTextLineDataset(self, compression_type=None):
-    test_filenames = self._createFiles(
-        2, 5, crlf=True, compression_type=compression_type)
-
-    def dataset_fn(filenames, num_epochs, batch_size=None):
-      repeat_dataset = readers.TextLineDataset(
-          filenames, compression_type=compression_type).repeat(num_epochs)
-      if batch_size:
-        return repeat_dataset.batch(batch_size)
-      return repeat_dataset
-
-    # Basic test: read from file 0.
-    expected_output = [self._lineText(0, i) for i in range(5)]
-    self.assertDatasetProduces(
-        dataset_fn([test_filenames[0]], 1), expected_output=expected_output)
-
-    # Basic test: read from file 1.
-    self.assertDatasetProduces(
-        dataset_fn([test_filenames[1]], 1),
-        expected_output=[self._lineText(1, i) for i in range(5)])
-
-    # Basic test: read from both files.
-    expected_output = [self._lineText(0, i) for i in range(5)]
-    expected_output.extend([self._lineText(1, i) for i in range(5)])
-    self.assertDatasetProduces(
-        dataset_fn(test_filenames, 1), expected_output=expected_output)
-
-    # Test repeated iteration through both files.
-    expected_output = [self._lineText(0, i) for i in range(5)]
-    expected_output.extend([self._lineText(1, i) for i in range(5)])
-    self.assertDatasetProduces(
-        dataset_fn(test_filenames, 10), expected_output=expected_output * 10)
-
-    # Test batched and repeated iteration through both files.
-    self.assertDatasetProduces(
-        dataset_fn(test_filenames, 10, 5),
-        expected_output=[[self._lineText(0, i) for i in range(5)],
-                         [self._lineText(1, i) for i in range(5)]] * 10)
-
-  def testTextLineDatasetNoCompression(self):
-    self._testTextLineDataset()
-
-  def testTextLineDatasetGzipCompression(self):
-    self._testTextLineDataset(compression_type="GZIP")
-
-  def testTextLineDatasetZlibCompression(self):
-    self._testTextLineDataset(compression_type="ZLIB")
-
-  def testTextLineDatasetBuffering(self):
-    test_filenames = self._createFiles(2, 5, crlf=True)
-
-    repeat_dataset = readers.TextLineDataset(test_filenames, buffer_size=10)
-    expected_output = []
-    for j in range(2):
-      expected_output.extend([self._lineText(j, i) for i in range(5)])
-    self.assertDatasetProduces(repeat_dataset, expected_output=expected_output)
-
-  def testIteratorResourceCleanup(self):
-    filename = os.path.join(self.get_temp_dir(), "text.txt")
-    with open(filename, "wt") as f:
-      for i in range(3):
-        f.write("%d\n" % (i,))
-    with context.eager_mode():
-      first_iterator = iter(readers.TextLineDataset(filename))
-      self.assertEqual(b"0", next(first_iterator).numpy())
-      second_iterator = iter(readers.TextLineDataset(filename))
-      self.assertEqual(b"0", next(second_iterator).numpy())
-      # Eager kernel caching is based on op attributes, which includes the
-      # Dataset's output shape. Create a different kernel to test that they
-      # don't create resources with the same names.
-      different_kernel_iterator = iter(
-          readers.TextLineDataset(filename).repeat().batch(16))
-      self.assertEqual([16], next(different_kernel_iterator).shape)
-      # Remove our references to the Python Iterator objects, which (assuming no
-      # reference cycles) is enough to trigger DestroyResourceOp and close the
-      # partially-read files.
-      del first_iterator
-      del second_iterator
-      del different_kernel_iterator
-      if not psutil_import_succeeded:
-        self.skipTest(
-            "psutil is required to check that we've closed our files.")
-      open_files = psutil.Process().open_files()
-      self.assertNotIn(filename, [open_file.path for open_file in open_files])
-
-
-class FixedLengthRecordReaderTestBase(test_base.DatasetTestBase):
-
-  def setUp(self):
-    super(FixedLengthRecordReaderTestBase, self).setUp()
-    self._num_files = 2
-    self._num_records = 7
-    self._header_bytes = 5
-    self._record_bytes = 3
-    self._footer_bytes = 2
-
-  def _record(self, f, r):
-    return compat.as_bytes(str(f * 2 + r) * self._record_bytes)
-
-  def _createFiles(self, compression_type=None):
-    filenames = []
-    for i in range(self._num_files):
-      fn = os.path.join(self.get_temp_dir(), "fixed_length_record.%d.txt" % i)
-      filenames.append(fn)
-
-      contents = []
-      contents.append(b"H" * self._header_bytes)
-      for j in range(self._num_records):
-        contents.append(self._record(i, j))
-      contents.append(b"F" * self._footer_bytes)
-      contents = b"".join(contents)
-
-      if not compression_type:
-        with open(fn, "wb") as f:
-          f.write(contents)
-      elif compression_type == "GZIP":
-        with gzip.GzipFile(fn, "wb") as f:
-          f.write(contents)
-      elif compression_type == "ZLIB":
-        contents = zlib.compress(contents)
-        with open(fn, "wb") as f:
-          f.write(contents)
-      else:
-        raise ValueError("Unsupported compression_type", compression_type)
-
-    return filenames
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class FixedLengthRecordReaderTest(FixedLengthRecordReaderTestBase):
-
-  def _testFixedLengthRecordDataset(self, compression_type=None):
-    test_filenames = self._createFiles(compression_type=compression_type)
-
-    def dataset_fn(filenames, num_epochs, batch_size=None):
-      repeat_dataset = readers.FixedLengthRecordDataset(
-          filenames,
-          self._record_bytes,
-          self._header_bytes,
-          self._footer_bytes,
-          compression_type=compression_type).repeat(num_epochs)
-      if batch_size:
-        return repeat_dataset.batch(batch_size)
-      return repeat_dataset
-
-    # Basic test: read from file 0.
-    self.assertDatasetProduces(
-        dataset_fn([test_filenames[0]], 1),
-        expected_output=[
-            self._record(0, i) for i in range(self._num_records)
-        ])
-
-    # Basic test: read from file 1.
-    self.assertDatasetProduces(
-        dataset_fn([test_filenames[1]], 1),
-        expected_output=[
-            self._record(1, i) for i in range(self._num_records)
-        ])
-
-    # Basic test: read from both files.
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    self.assertDatasetProduces(
-        dataset_fn(test_filenames, 1), expected_output=expected_output)
-
-    # Test repeated iteration through both files.
-    get_next = self.getNext(dataset_fn(test_filenames, 10))
-    for _ in range(10):
-      for j in range(self._num_files):
-        for i in range(self._num_records):
-          self.assertEqual(self._record(j, i), self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-    # Test batched and repeated iteration through both files.
-    get_next = self.getNext(dataset_fn(test_filenames, 10, self._num_records))
-    for _ in range(10):
-      for j in range(self._num_files):
-        self.assertAllEqual(
-            [self._record(j, i) for i in range(self._num_records)],
-            self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-  def testFixedLengthRecordDatasetNoCompression(self):
-    self._testFixedLengthRecordDataset()
-
-  def testFixedLengthRecordDatasetGzipCompression(self):
-    self._testFixedLengthRecordDataset(compression_type="GZIP")
-
-  def testFixedLengthRecordDatasetZlibCompression(self):
-    self._testFixedLengthRecordDataset(compression_type="ZLIB")
-
-  def testFixedLengthRecordDatasetBuffering(self):
-    test_filenames = self._createFiles()
-    dataset = readers.FixedLengthRecordDataset(
-        test_filenames,
-        self._record_bytes,
-        self._header_bytes,
-        self._footer_bytes,
-        buffer_size=10)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testFixedLengthRecordDatasetWrongSize(self):
-    test_filenames = self._createFiles()
-    dataset = readers.FixedLengthRecordDataset(
-        test_filenames,
-        self._record_bytes + 1,  # Incorrect record length.
-        self._header_bytes,
-        self._footer_bytes,
-        buffer_size=10)
-    self.assertDatasetProduces(
-        dataset,
-        expected_error=(
-            errors.InvalidArgumentError,
-            r"Excluding the header \(5 bytes\) and footer \(2 bytes\), input "
-            r"file \".*fixed_length_record.0.txt\" has body length 21 bytes, "
-            r"which is not an exact multiple of the record length \(4 bytes\).")
-        )
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TFRecordDatasetTest(test_base.DatasetTestBase):
-
-  def setUp(self):
-    super(TFRecordDatasetTest, self).setUp()
-    self._num_files = 2
-    self._num_records = 7
-
-    self.test_filenames = self._createFiles()
-
-  def dataset_fn(self,
-                 filenames,
-                 compression_type="",
-                 num_epochs=1,
-                 batch_size=None):
-
-    repeat_dataset = readers.TFRecordDataset(
-        filenames, compression_type).repeat(num_epochs)
-    if batch_size:
-      return repeat_dataset.batch(batch_size)
-    return repeat_dataset
-
-  def _record(self, f, r):
-    return compat.as_bytes("Record %d of file %d" % (r, f))
-
-  def _createFiles(self):
-    filenames = []
-    for i in range(self._num_files):
-      fn = os.path.join(self.get_temp_dir(), "tf_record.%d.txt" % i)
-      filenames.append(fn)
-      writer = python_io.TFRecordWriter(fn)
-      for j in range(self._num_records):
-        writer.write(self._record(i, j))
-      writer.close()
-    return filenames
-
-  def testReadOneEpoch(self):
-    # Basic test: read from file 0.
-    dataset = self.dataset_fn(self.test_filenames[0])
-    self.assertDatasetProduces(
-        dataset,
-        expected_output=[self._record(0, i) for i in range(self._num_records)])
-
-    # Basic test: read from file 1.
-    dataset = self.dataset_fn(self.test_filenames[1])
-    self.assertDatasetProduces(
-        dataset,
-        expected_output=[self._record(1, i) for i in range(self._num_records)])
-
-    # Basic test: read from both files.
-    dataset = self.dataset_fn(self.test_filenames)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testReadTenEpochs(self):
-    dataset = self.dataset_fn(self.test_filenames, num_epochs=10)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    self.assertDatasetProduces(dataset, expected_output=expected_output * 10)
-
-  def testReadTenEpochsOfBatches(self):
-    dataset = self.dataset_fn(
-        self.test_filenames, num_epochs=10, batch_size=self._num_records)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.append(
-          [self._record(j, i) for i in range(self._num_records)])
-    self.assertDatasetProduces(dataset, expected_output=expected_output * 10)
-
-  def testReadZlibFiles(self):
-    zlib_files = []
-    for i, fn in enumerate(self.test_filenames):
-      with open(fn, "rb") as f:
-        cdata = zlib.compress(f.read())
-
-        zfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.z" % i)
-        with open(zfn, "wb") as f:
-          f.write(cdata)
-        zlib_files.append(zfn)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    dataset = self.dataset_fn(zlib_files, compression_type="ZLIB")
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testReadGzipFiles(self):
-    gzip_files = []
-    for i, fn in enumerate(self.test_filenames):
-      with open(fn, "rb") as f:
-        gzfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.gz" % i)
-        with gzip.GzipFile(gzfn, "wb") as gzf:
-          gzf.write(f.read())
-        gzip_files.append(gzfn)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    dataset = self.dataset_fn(gzip_files, compression_type="GZIP")
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testReadWithBuffer(self):
-    one_mebibyte = 2**20
-    dataset = readers.TFRecordDataset(
-        self.test_filenames, buffer_size=one_mebibyte)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testReadFromDatasetOfFiles(self):
-    files = dataset_ops.Dataset.from_tensor_slices(self.test_filenames)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    dataset = readers.TFRecordDataset(files)
-    self.assertDatasetProduces(dataset, expected_output=expected_output)
-
-  def testReadTenEpochsFromDatasetOfFilesInParallel(self):
-    files = dataset_ops.Dataset.from_tensor_slices(
-        self.test_filenames).repeat(10)
-    expected_output = []
-    for j in range(self._num_files):
-      expected_output.extend(
-          [self._record(j, i) for i in range(self._num_records)])
-    dataset = readers.TFRecordDataset(files, num_parallel_reads=4)
-    self.assertDatasetProduces(
-        dataset, expected_output=expected_output * 10, assert_items_equal=True)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py b/tensorflow/python/data/kernel_tests/reduce_test.py
similarity index 97%
rename from tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/reduce_test.py
index 061f0d1343..d7b653961d 100644
--- a/tensorflow/python/data/kernel_tests/reduce_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.reduce()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -32,7 +32,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ReduceDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class ReduceTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testSum(self):
     for i in range(10):
diff --git a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py b/tensorflow/python/data/kernel_tests/repeat_test.py
similarity index 67%
rename from tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/repeat_test.py
index 6da4e0dfca..4ef2fc1bfc 100644
--- a/tensorflow/python/data/kernel_tests/sequence_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/repeat_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.repeat()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -26,7 +26,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class SequenceDatasetTest(test_base.DatasetTestBase):
+class RepeatTest(test_base.DatasetTestBase):
 
   def testRepeatTensorDataset(self):
     """Test a dataset that repeats its input multiple times."""
@@ -62,57 +62,6 @@ class SequenceDatasetTest(test_base.DatasetTestBase):
       for component, result_component in zip(components, results):
         self.assertAllEqual(component, result_component)
 
-  def testTakeTensorDataset(self):
-    components = (np.arange(10),)
-
-    def do_test(count):
-      dataset = dataset_ops.Dataset.from_tensor_slices(components).take(count)
-      self.assertEqual([c.shape[1:] for c in components],
-                       [shape for shape in dataset.output_shapes])
-      num_output = min(count, 10) if count != -1 else 10
-      self.assertDatasetProduces(
-          dataset, [tuple(components[0][i:i + 1]) for i in range(num_output)])
-
-    # Take fewer than input size
-    do_test(4)
-
-    # Take more than input size
-    do_test(25)
-
-    # Take all of input
-    do_test(-1)
-
-    # Take nothing
-    do_test(0)
-
-  def testSkipTensorDataset(self):
-    components = (np.arange(10),)
-
-    def do_test(count):
-      dataset = dataset_ops.Dataset.from_tensor_slices(components).skip(count)
-      self.assertEqual([c.shape[1:] for c in components],
-                       [shape for shape in dataset.output_shapes])
-      start_range = min(count, 10) if count != -1 else 10
-      self.assertDatasetProduces(
-          dataset,
-          [tuple(components[0][i:i + 1]) for i in range(start_range, 10)])
-
-    # Skip fewer than input size, we should skip
-    # the first 4 elements and then read the rest.
-    do_test(4)
-
-    # Skip more than input size: get nothing.
-    do_test(25)
-
-    # Skip exactly input size.
-    do_test(10)
-
-    # Set -1 for 'count': skip the entire dataset.
-    do_test(-1)
-
-    # Skip nothing
-    do_test(0)
-
   def testRepeatRepeatTensorDataset(self):
     """Test the composition of repeat datasets."""
     components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
diff --git a/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shard_test.py
similarity index 96%
rename from tensorflow/python/data/kernel_tests/shard_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/shard_test.py
index 7dd932d015..928550676d 100644
--- a/tensorflow/python/data/kernel_tests/shard_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/shard_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.shard()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -24,7 +24,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ShardDatasetOpTest(test_base.DatasetTestBase):
+class ShardTest(test_base.DatasetTestBase):
 
   def testSimpleCase(self):
     dataset = dataset_ops.Dataset.range(10).shard(5, 2)
@@ -72,6 +72,5 @@ class ShardDatasetOpTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).shard(4, 3)
     self.assertDatasetProduces(dataset, expected_output=[3, 7])
 
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py b/tensorflow/python/data/kernel_tests/shuffle_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/shuffle_test.py
index bf8303ca6e..49460a1a4e 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.shuffle()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -35,7 +35,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ShuffleDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testShuffleDataset(self):
     components = (
diff --git a/tensorflow/python/data/kernel_tests/skip_test.py b/tensorflow/python/data/kernel_tests/skip_test.py
new file mode 100644
index 0000000000..c22be57692
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/skip_test.py
@@ -0,0 +1,62 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.skip()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SkipTest(test_base.DatasetTestBase):
+
+  def testSkipTensorDataset(self):
+    components = (np.arange(10),)
+
+    def do_test(count):
+      dataset = dataset_ops.Dataset.from_tensor_slices(components).skip(count)
+      self.assertEqual([c.shape[1:] for c in components],
+                       [shape for shape in dataset.output_shapes])
+      start_range = min(count, 10) if count != -1 else 10
+      self.assertDatasetProduces(
+          dataset,
+          [tuple(components[0][i:i + 1]) for i in range(start_range, 10)])
+
+    # Skip fewer than input size, we should skip
+    # the first 4 elements and then read the rest.
+    do_test(4)
+
+    # Skip more than input size: get nothing.
+    do_test(25)
+
+    # Skip exactly input size.
+    do_test(10)
+
+    # Set -1 for 'count': skip the entire dataset.
+    do_test(-1)
+
+    # Skip nothing
+    do_test(0)
+
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/take_test.py b/tensorflow/python/data/kernel_tests/take_test.py
new file mode 100644
index 0000000000..03a7ece2d8
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/take_test.py
@@ -0,0 +1,55 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.Dataset.take()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TakeTest(test_base.DatasetTestBase):
+
+  def testTakeTensorDataset(self):
+    components = (np.arange(10),)
+
+    def do_test(count):
+      dataset = dataset_ops.Dataset.from_tensor_slices(components).take(count)
+      self.assertEqual([c.shape[1:] for c in components],
+                       [shape for shape in dataset.output_shapes])
+      num_output = min(count, 10) if count != -1 else 10
+      self.assertDatasetProduces(
+          dataset, [tuple(components[0][i:i + 1]) for i in range(num_output)])
+
+    # Take fewer than input size
+    do_test(4)
+
+    # Take more than input size
+    do_test(25)
+
+    # Take all of input
+    do_test(-1)
+
+    # Take nothing
+    do_test(0)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/text_line_dataset_test.py b/tensorflow/python/data/kernel_tests/text_line_dataset_test.py
new file mode 100644
index 0000000000..4db09a9808
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/text_line_dataset_test.py
@@ -0,0 +1,165 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.TextLineDataset`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gzip
+import os
+import zlib
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import readers
+from tensorflow.python.eager import context
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.util import compat
+
+
+try:
+  import psutil  # pylint: disable=g-import-not-at-top
+  psutil_import_succeeded = True
+except ImportError:
+  psutil_import_succeeded = False
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TextLineDatasetTest(test_base.DatasetTestBase):
+
+  def _lineText(self, f, l):
+    return compat.as_bytes("%d: %d" % (f, l))
+
+  def _createFiles(self,
+                   num_files,
+                   num_lines,
+                   crlf=False,
+                   compression_type=None):
+    filenames = []
+    for i in range(num_files):
+      fn = os.path.join(self.get_temp_dir(), "text_line.%d.txt" % i)
+      filenames.append(fn)
+      contents = []
+      for j in range(num_lines):
+        contents.append(self._lineText(i, j))
+        # Always include a newline after the record unless it is
+        # at the end of the file, in which case we include it
+        if j + 1 != num_lines or i == 0:
+          contents.append(b"\r\n" if crlf else b"\n")
+      contents = b"".join(contents)
+
+      if not compression_type:
+        with open(fn, "wb") as f:
+          f.write(contents)
+      elif compression_type == "GZIP":
+        with gzip.GzipFile(fn, "wb") as f:
+          f.write(contents)
+      elif compression_type == "ZLIB":
+        contents = zlib.compress(contents)
+        with open(fn, "wb") as f:
+          f.write(contents)
+      else:
+        raise ValueError("Unsupported compression_type", compression_type)
+
+    return filenames
+
+  def _testTextLineDataset(self, compression_type=None):
+    test_filenames = self._createFiles(
+        2, 5, crlf=True, compression_type=compression_type)
+
+    def dataset_fn(filenames, num_epochs, batch_size=None):
+      repeat_dataset = readers.TextLineDataset(
+          filenames, compression_type=compression_type).repeat(num_epochs)
+      if batch_size:
+        return repeat_dataset.batch(batch_size)
+      return repeat_dataset
+
+    # Basic test: read from file 0.
+    expected_output = [self._lineText(0, i) for i in range(5)]
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[0]], 1), expected_output=expected_output)
+
+    # Basic test: read from file 1.
+    self.assertDatasetProduces(
+        dataset_fn([test_filenames[1]], 1),
+        expected_output=[self._lineText(1, i) for i in range(5)])
+
+    # Basic test: read from both files.
+    expected_output = [self._lineText(0, i) for i in range(5)]
+    expected_output.extend([self._lineText(1, i) for i in range(5)])
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 1), expected_output=expected_output)
+
+    # Test repeated iteration through both files.
+    expected_output = [self._lineText(0, i) for i in range(5)]
+    expected_output.extend([self._lineText(1, i) for i in range(5)])
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 10), expected_output=expected_output * 10)
+
+    # Test batched and repeated iteration through both files.
+    self.assertDatasetProduces(
+        dataset_fn(test_filenames, 10, 5),
+        expected_output=[[self._lineText(0, i) for i in range(5)],
+                         [self._lineText(1, i) for i in range(5)]] * 10)
+
+  def testTextLineDatasetNoCompression(self):
+    self._testTextLineDataset()
+
+  def testTextLineDatasetGzipCompression(self):
+    self._testTextLineDataset(compression_type="GZIP")
+
+  def testTextLineDatasetZlibCompression(self):
+    self._testTextLineDataset(compression_type="ZLIB")
+
+  def testTextLineDatasetBuffering(self):
+    test_filenames = self._createFiles(2, 5, crlf=True)
+
+    repeat_dataset = readers.TextLineDataset(test_filenames, buffer_size=10)
+    expected_output = []
+    for j in range(2):
+      expected_output.extend([self._lineText(j, i) for i in range(5)])
+    self.assertDatasetProduces(repeat_dataset, expected_output=expected_output)
+
+  def testIteratorResourceCleanup(self):
+    filename = os.path.join(self.get_temp_dir(), "text.txt")
+    with open(filename, "wt") as f:
+      for i in range(3):
+        f.write("%d\n" % (i,))
+    with context.eager_mode():
+      first_iterator = iter(readers.TextLineDataset(filename))
+      self.assertEqual(b"0", next(first_iterator).numpy())
+      second_iterator = iter(readers.TextLineDataset(filename))
+      self.assertEqual(b"0", next(second_iterator).numpy())
+      # Eager kernel caching is based on op attributes, which includes the
+      # Dataset's output shape. Create a different kernel to test that they
+      # don't create resources with the same names.
+      different_kernel_iterator = iter(
+          readers.TextLineDataset(filename).repeat().batch(16))
+      self.assertEqual([16], next(different_kernel_iterator).shape)
+      # Remove our references to the Python Iterator objects, which (assuming no
+      # reference cycles) is enough to trigger DestroyResourceOp and close the
+      # partially-read files.
+      del first_iterator
+      del second_iterator
+      del different_kernel_iterator
+      if not psutil_import_succeeded:
+        self.skipTest(
+            "psutil is required to check that we've closed our files.")
+      open_files = psutil.Process().open_files()
+      self.assertNotIn(filename, [open_file.path for open_file in open_files])
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/tf_record_dataset_test.py b/tensorflow/python/data/kernel_tests/tf_record_dataset_test.py
new file mode 100644
index 0000000000..13a70aa88d
--- /dev/null
+++ b/tensorflow/python/data/kernel_tests/tf_record_dataset_test.py
@@ -0,0 +1,170 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.TFRecordDataset`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gzip
+import os
+import zlib
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import readers
+from tensorflow.python.framework import test_util
+from tensorflow.python.lib.io import python_io
+from tensorflow.python.platform import test
+from tensorflow.python.util import compat
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TFRecordDatasetTest(test_base.DatasetTestBase):
+
+  def setUp(self):
+    super(TFRecordDatasetTest, self).setUp()
+    self._num_files = 2
+    self._num_records = 7
+
+    self.test_filenames = self._createFiles()
+
+  def dataset_fn(self,
+                 filenames,
+                 compression_type="",
+                 num_epochs=1,
+                 batch_size=None):
+
+    repeat_dataset = readers.TFRecordDataset(
+        filenames, compression_type).repeat(num_epochs)
+    if batch_size:
+      return repeat_dataset.batch(batch_size)
+    return repeat_dataset
+
+  def _record(self, f, r):
+    return compat.as_bytes("Record %d of file %d" % (r, f))
+
+  def _createFiles(self):
+    filenames = []
+    for i in range(self._num_files):
+      fn = os.path.join(self.get_temp_dir(), "tf_record.%d.txt" % i)
+      filenames.append(fn)
+      writer = python_io.TFRecordWriter(fn)
+      for j in range(self._num_records):
+        writer.write(self._record(i, j))
+      writer.close()
+    return filenames
+
+  def testReadOneEpoch(self):
+    # Basic test: read from file 0.
+    dataset = self.dataset_fn(self.test_filenames[0])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[self._record(0, i) for i in range(self._num_records)])
+
+    # Basic test: read from file 1.
+    dataset = self.dataset_fn(self.test_filenames[1])
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[self._record(1, i) for i in range(self._num_records)])
+
+    # Basic test: read from both files.
+    dataset = self.dataset_fn(self.test_filenames)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testReadTenEpochs(self):
+    dataset = self.dataset_fn(self.test_filenames, num_epochs=10)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output * 10)
+
+  def testReadTenEpochsOfBatches(self):
+    dataset = self.dataset_fn(
+        self.test_filenames, num_epochs=10, batch_size=self._num_records)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.append(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output * 10)
+
+  def testReadZlibFiles(self):
+    zlib_files = []
+    for i, fn in enumerate(self.test_filenames):
+      with open(fn, "rb") as f:
+        cdata = zlib.compress(f.read())
+
+        zfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.z" % i)
+        with open(zfn, "wb") as f:
+          f.write(cdata)
+        zlib_files.append(zfn)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = self.dataset_fn(zlib_files, compression_type="ZLIB")
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testReadGzipFiles(self):
+    gzip_files = []
+    for i, fn in enumerate(self.test_filenames):
+      with open(fn, "rb") as f:
+        gzfn = os.path.join(self.get_temp_dir(), "tfrecord_%s.gz" % i)
+        with gzip.GzipFile(gzfn, "wb") as gzf:
+          gzf.write(f.read())
+        gzip_files.append(gzfn)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = self.dataset_fn(gzip_files, compression_type="GZIP")
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testReadWithBuffer(self):
+    one_mebibyte = 2**20
+    dataset = readers.TFRecordDataset(
+        self.test_filenames, buffer_size=one_mebibyte)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testReadFromDatasetOfFiles(self):
+    files = dataset_ops.Dataset.from_tensor_slices(self.test_filenames)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = readers.TFRecordDataset(files)
+    self.assertDatasetProduces(dataset, expected_output=expected_output)
+
+  def testReadTenEpochsFromDatasetOfFilesInParallel(self):
+    files = dataset_ops.Dataset.from_tensor_slices(
+        self.test_filenames).repeat(10)
+    expected_output = []
+    for j in range(self._num_files):
+      expected_output.extend(
+          [self._record(j, i) for i in range(self._num_records)])
+    dataset = readers.TFRecordDataset(files, num_parallel_reads=4)
+    self.assertDatasetProduces(
+        dataset, expected_output=expected_output * 10, assert_items_equal=True)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py b/tensorflow/python/data/kernel_tests/window_test.py
similarity index 98%
rename from tensorflow/python/data/kernel_tests/window_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/window_test.py
index 7124cc75f1..d083142ab6 100644
--- a/tensorflow/python/data/kernel_tests/window_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/window_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.window()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -33,7 +33,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class WindowDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class WindowTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(
       ("1", 20, 14, 7, 1),
diff --git a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py b/tensorflow/python/data/kernel_tests/zip_test.py
similarity index 97%
rename from tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
rename to tensorflow/python/data/kernel_tests/zip_test.py
index c86765addc..477c9fa7da 100644
--- a/tensorflow/python/data/kernel_tests/zip_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/zip_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for the experimental input pipeline ops."""
+"""Tests for `tf.data.Dataset.zip()`."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -28,7 +28,7 @@ from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ZipDatasetTest(test_base.DatasetTestBase):
+class ZipTest(test_base.DatasetTestBase):
 
   def testZipDataset(self):
 
-- 
GitLab


From b8e155c4a05dca07fcc5505cd773f849e93be7a4 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 26 Nov 2018 15:41:59 -0800
Subject: [PATCH 0819/1554] Place all nodes on CPU in conv_ops_test
 (FusedConv2D)

PiperOrigin-RevId: 222900089
---
 tensorflow/core/kernels/conv_ops_test.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index ef27dfec54..bf98acdecf 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -550,6 +550,12 @@ class FusedConv2DOpTest : public OpsTestBase {
     tensorflow::GraphDef graph;
     TF_ASSERT_OK(root.ToGraphDef(&graph));
 
+    // `FusedConv2D` is available only on CPU, and in this test we don't want to
+    // compare GPU vs CPU numbers, so place all nodes on CPU.
+    for (NodeDef& mutable_node : *graph.mutable_node()) {
+      mutable_node.set_device("/device:CPU:0");
+    }
+
     // Disable Grappler constant folding for the test graphs.
     tensorflow::SessionOptions session_options;
     tensorflow::RewriterConfig* cfg =
-- 
GitLab


From 9339900d3dad8f3cf81beb9924a94f237799057f Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Mon, 26 Nov 2018 15:51:36 -0800
Subject: [PATCH 0820/1554] Install converter as a command line utility in the
 pip package.

RELNOTES: Add a command line tool to convert to TF2.0
PiperOrigin-RevId: 222901543
---
 tensorflow/tools/pip_package/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index e164853428..001aa22227 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -107,6 +107,7 @@ CONSOLE_SCRIPTS = [
     # TensorBoard command, pip will inappropriately remove it during install,
     # even though the command is not removed, just moved to a different wheel.
     'tensorboard = tensorboard.main:run_main',
+    'tf_upgrade_v2 = tensorflow.python.tools.compatibility.tf_upgrade_v2:main',
 ]
 # pylint: enable=line-too-long
 
-- 
GitLab


From ea3b3dd2af17f96c88d357d993e851d9850908b4 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 26 Nov 2018 16:07:01 -0800
Subject: [PATCH 0821/1554] Add templated function to test other datatypes.

---
 .../tensorrt/convert/convert_nodes_test.cc    | 87 +++++++++++++------
 1 file changed, 62 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 312375a661..471382894c 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -151,6 +151,27 @@ void ExpectTrtDimsEqualsArray(const std::vector<int>& lhs,
       << "  actual: " << DebugString(rhs);
 }
 
+template <typename T>
+void ExpectArrayNear(const std::vector<T>& lhs,
+                     const std::vector<T>& rhs) {
+  ASSERT_EQ(lhs.size(), rhs.size());
+  for (int i = 0; i < lhs.size(); i++) {
+    EXPECT_FLOAT_EQ(lhs[i], rhs[i]);
+  }
+}
+
+// Eigen::half cannot implicitly convert to float which is required for
+// EXPECT_FLOAT_EQ.
+template <>
+void ExpectArrayNear(const std::vector<Eigen::half>& lhs,
+                     const std::vector<Eigen::half>& rhs) {
+  ASSERT_EQ(lhs.size(), rhs.size());
+  for (int i = 0; i < lhs.size(); i++) {
+    EXPECT_FLOAT_EQ(Eigen::half_impl::half_to_float(lhs[i]),
+                    Eigen::half_impl::half_to_float(rhs[i]));
+  }
+}
+
 bool TrtShapedWeightsEquals(const TRT_ShapedWeights& lhs,
                             const TRT_ShapedWeights& rhs) {
   return TrtDimsEquals(lhs.shape_, rhs.shape_) && lhs.type_ == rhs.type_ &&
@@ -1964,6 +1985,38 @@ TEST_F(OpConverterTest, ConvertRelu6) {
   }
 }
 
+template <DataType dtype>
+void TestConvertSquare(OpConverterTest* test) {
+  test->Reset();
+  typedef typename EnumToDataType<dtype>::Type CType;
+
+  Scope s = Scope::NewRootScope();
+  auto input = ops::Placeholder(s.WithOpName("input"), dtype);
+  auto square = ops::Square(s.WithOpName("my_square"), input);
+  NodeDef node_def = square.operation.node()->def();
+
+  test->AddTestTensor("input", {1, 20});
+  test->RunValidationAndConversion(node_def);
+  TRT_TensorOrWeights output;
+  TF_EXPECT_OK(test->GetTensorOrWeights("my_square", &output));
+  EXPECT_TRUE(output.is_tensor());
+  ExpectTrtDimsEqualsArray({1, 20}, output.tensor()->getDimensions());
+
+  const int num_inputs = 20;
+  std::vector<CType> input_data(num_inputs);
+  std::vector<CType> expected_output_data(num_inputs);
+  for (int i = 0; i < 20; i++) {
+    const CType value = CType(i - 9);
+    input_data[i] = value;
+    expected_output_data[i] = value * value;
+  }
+  std::vector<CType> output_data(num_inputs);
+  test->BuildAndRun<CType>(
+          {{"input", input_data}}, "my_square",
+          &output_data);
+  ExpectArrayNear(expected_output_data, output_data);
+}
+
 TEST_F(OpConverterTest, ConvertSquare) {
   {
     // Input list is empty, should fail.
@@ -1972,38 +2025,22 @@ TEST_F(OpConverterTest, ConvertSquare) {
         node_def, error::INVALID_ARGUMENT,
         "Square expects one input, at my_square");
   }
-
-  // Get the NodeDef for Square.
-  Scope s = Scope::NewRootScope();
-  auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
-  auto square = ops::Square(s.WithOpName("my_square"), input);
-  const NodeDef& node_def = square.operation.node()->def();
-
   {
     // Input is weights, should fail.
     Reset();
-    AddTestWeights<int32>("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6});
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto square = ops::Square(s.WithOpName("my_square"), input);
+    NodeDef node_def = square.operation.node()->def();
+    AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6});
     RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
         "Square is only implemented for tensors, at my_square");
   }
-  {
-    // Input is tensor, Ok.
-    Reset();
-    AddTestTensor("input", {1, 2, 3});
-    RunValidationAndConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_square", &output));
-    EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()))
-        << output.DebugString();
 
-    std::vector<float> output_data(6);
-    std::vector<float> expected_output_data = {1, 4, 9, 16, 25, 36};
-    BuildAndRun("input", {1, 2, 3, 4, -5, 6}, "my_square", &output_data);
-    for (int i = 0; i < output_data.size(); i++) {
-      EXPECT_FLOAT_EQ(output_data[i], expected_output_data[i]);
-    }
-  }
+  // OK. Note that kINT32 is not supported by IElementWiseLayer, so we don't
+  // test DT_INT32 type here.
+  TestConvertSquare<DT_FLOAT>(this);
+  TestConvertSquare<DT_HALF>(this);
 }
 
 }  // namespace convert
-- 
GitLab


From 22f789130667cd58cb33a6f63774a7b08be19dec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 16:05:07 -0800
Subject: [PATCH 0822/1554] nn.dilation2d changes for TF 2.0 API

Renames `filter` parameter to `filters`, `rates` to `dilations`, adds
data_format parameter.

PiperOrigin-RevId: 222903663
---
 .../python_api/api_def_Dilation2D.pbtxt       |  1 +
 tensorflow/python/ops/nn_ops.py               | 67 +++++++++++++++++++
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 tensorflow/tools/compatibility/renames_v2.py  | 46 ++++++++++---
 .../tools/compatibility/tf_upgrade_v2.py      |  4 ++
 5 files changed, 110 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt b/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt
index 6d73ecf1bb..1bd83d9061 100644
--- a/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Dilation2D.pbtxt
@@ -2,5 +2,6 @@ op {
   graph_op_name: "Dilation2D"
   endpoint {
     name: "nn.dilation2d"
+    deprecation_version: 2
   }
 }
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 755c8ffcd2..0e39dd0fa0 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -207,6 +207,73 @@ class _NonAtrousConvolution(object):
         name=self.name)
 
 
+@tf_export("nn.dilation2d", v1=[])
+def dilation2d_v2(
+    input,   # pylint: disable=redefined-builtin
+    filters,  # pylint: disable=redefined-builtin
+    strides,
+    padding,
+    data_format,
+    dilations,
+    name=None):
+  """Computes the grayscale dilation of 4-D `input` and 3-D `filters` tensors.
+
+  The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
+  `filters` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
+  input channel is processed independently of the others with its own
+  structuring function. The `output` tensor has shape
+  `[batch, out_height, out_width, depth]`. The spatial dimensions of the output
+  tensor depend on the `padding` algorithm. We currently only support the
+  default "NHWC" `data_format`.
+
+  In detail, the grayscale morphological 2-D dilation is the max-sum correlation
+  (for consistency with `conv2d`, we use unmirrored filters):
+
+      output[b, y, x, c] =
+         max_{dy, dx} input[b,
+                            strides[1] * y + rates[1] * dy,
+                            strides[2] * x + rates[2] * dx,
+                            c] +
+                      filters[dy, dx, c]
+
+  Max-pooling is a special case when the filter has size equal to the pooling
+  kernel size and contains all zeros.
+
+  Note on duality: The dilation of `input` by the `filters` is equal to the
+  negation of the erosion of `-input` by the reflected `filters`.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types: `float32`, `float64`,
+      `int32`, `uint8`, `int16`, `int8`, `int64`, `bfloat16`, `uint16`, `half`,
+      `uint32`, `uint64`.
+      4-D with shape `[batch, in_height, in_width, depth]`.
+    filters: A `Tensor`. Must have the same type as `input`.
+      3-D with shape `[filter_height, filter_width, depth]`.
+    strides: A list of `ints` that has length `>= 4`.
+      The stride of the sliding window for each dimension of the input
+      tensor. Must be: `[1, stride_height, stride_width, 1]`.
+    padding: A `string` from: `"SAME", "VALID"`.
+      The type of padding algorithm to use.
+    data_format: A `string`, only `"NCHW"` is currently supported.
+    dilations: A list of `ints` that has length `>= 4`.
+      The input stride for atrous morphological dilation. Must be:
+      `[1, rate_height, rate_width, 1]`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  if data_format != "NCHW":
+    raise ValueError("Data formats other than NCHW are not yet supported")
+
+  return gen_nn_ops.dilation2d(input=input,
+                               filter=filters,
+                               strides=strides,
+                               rates=dilations,
+                               padding=padding,
+                               name=name)
+
+
 @tf_export("nn.with_space_to_batch")
 def with_space_to_batch(
     input,  # pylint: disable=redefined-builtin
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 035a6c76f6..77b5887c1f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -122,7 +122,7 @@ tf_module {
   }
   member_method {
     name: "dilation2d"
-    argspec: "args=[\'input\', \'filter\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "dropout"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index e01360a060..fc4b19fb1c 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -321,7 +321,38 @@ renames = {
     'tf.matrix_transpose': 'tf.linalg.transpose',
     'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
     'tf.metrics.accuracy': 'tf.compat.v1.metrics.accuracy',
+    'tf.metrics.auc': 'tf.compat.v1.metrics.auc',
+    'tf.metrics.average_precision_at_k': 'tf.compat.v1.metrics.average_precision_at_k',
+    'tf.metrics.false_negatives': 'tf.compat.v1.metrics.false_negatives',
+    'tf.metrics.false_negatives_at_thresholds': 'tf.compat.v1.metrics.false_negatives_at_thresholds',
+    'tf.metrics.false_positives': 'tf.compat.v1.metrics.false_positives',
+    'tf.metrics.false_positives_at_thresholds': 'tf.compat.v1.metrics.false_positives_at_thresholds',
     'tf.metrics.mean': 'tf.compat.v1.metrics.mean',
+    'tf.metrics.mean_absolute_error': 'tf.compat.v1.metrics.mean_absolute_error',
+    'tf.metrics.mean_cosine_distance': 'tf.compat.v1.metrics.mean_cosine_distance',
+    'tf.metrics.mean_iou': 'tf.compat.v1.metrics.mean_iou',
+    'tf.metrics.mean_per_class_accuracy': 'tf.compat.v1.metrics.mean_per_class_accuracy',
+    'tf.metrics.mean_relative_error': 'tf.compat.v1.metrics.mean_relative_error',
+    'tf.metrics.mean_squared_error': 'tf.compat.v1.metrics.mean_squared_error',
+    'tf.metrics.mean_tensor': 'tf.compat.v1.metrics.mean_tensor',
+    'tf.metrics.percentage_below': 'tf.compat.v1.metrics.percentage_below',
+    'tf.metrics.precision': 'tf.compat.v1.metrics.precision',
+    'tf.metrics.precision_at_k': 'tf.compat.v1.metrics.precision_at_k',
+    'tf.metrics.precision_at_thresholds': 'tf.compat.v1.metrics.precision_at_thresholds',
+    'tf.metrics.precision_at_top_k': 'tf.compat.v1.metrics.precision_at_top_k',
+    'tf.metrics.recall': 'tf.compat.v1.metrics.recall',
+    'tf.metrics.recall_at_k': 'tf.compat.v1.metrics.recall_at_k',
+    'tf.metrics.recall_at_thresholds': 'tf.compat.v1.metrics.recall_at_thresholds',
+    'tf.metrics.recall_at_top_k': 'tf.compat.v1.metrics.recall_at_top_k',
+    'tf.metrics.root_mean_squared_error': 'tf.compat.v1.metrics.root_mean_squared_error',
+    'tf.metrics.sensitivity_at_specificity': 'tf.compat.v1.metrics.sensitivity_at_specificity',
+    'tf.metrics.sparse_average_precision_at_k': 'tf.compat.v1.metrics.sparse_average_precision_at_k',
+    'tf.metrics.sparse_precision_at_k': 'tf.compat.v1.metrics.sparse_precision_at_k',
+    'tf.metrics.specificity_at_sensitivity': 'tf.compat.v1.metrics.specificity_at_sensitivity',
+    'tf.metrics.true_negatives': 'tf.compat.v1.metrics.true_negatives',
+    'tf.metrics.true_negatives_at_thresholds': 'tf.compat.v1.metrics.true_negatives_at_thresholds',
+    'tf.metrics.true_positives': 'tf.compat.v1.metrics.true_positives',
+    'tf.metrics.true_positives_at_thresholds': 'tf.compat.v1.metrics.true_positives_at_thresholds',
     'tf.min_max_variable_partitioner': 'tf.compat.v1.min_max_variable_partitioner',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
@@ -352,8 +383,8 @@ renames = {
     'tf.nn.xw_plus_b': 'tf.compat.v1.nn.xw_plus_b',
     'tf.op_scope': 'tf.compat.v1.op_scope',
     'tf.orthogonal_initializer': 'tf.keras.initializers.Orthogonal',
-    'tf.parse_example': 'tf.io.parse_example',
-    'tf.parse_single_example': 'tf.io.parse_single_example',
+    'tf.parse_example': 'tf.compat.v1.parse_example',
+    'tf.parse_single_example': 'tf.compat.v1.parse_single_example',
     'tf.parse_single_sequence_example': 'tf.io.parse_single_sequence_example',
     'tf.parse_tensor': 'tf.io.parse_tensor',
     'tf.placeholder': 'tf.compat.v1.placeholder',
@@ -390,7 +421,7 @@ renames = {
     'tf.read_file': 'tf.io.read_file',
     'tf.real': 'tf.math.real',
     'tf.reciprocal': 'tf.math.reciprocal',
-    'tf.reduce_join': 'tf.strings.reduce_join',
+    'tf.reduce_join': 'tf.compat.v1.reduce_join',
     'tf.regex_replace': 'tf.strings.regex_replace',
     'tf.report_uninitialized_variables': 'tf.compat.v1.report_uninitialized_variables',
     'tf.reset_default_graph': 'tf.compat.v1.reset_default_graph',
@@ -461,8 +492,8 @@ renames = {
     'tf.segment_sum': 'tf.math.segment_sum',
     'tf.self_adjoint_eig': 'tf.linalg.eigh',
     'tf.self_adjoint_eigvals': 'tf.linalg.eigvalsh',
-    'tf.serialize_many_sparse': 'tf.io.serialize_many_sparse',
-    'tf.serialize_sparse': 'tf.io.serialize_sparse',
+    'tf.serialize_many_sparse': 'tf.compat.v1.serialize_many_sparse',
+    'tf.serialize_sparse': 'tf.compat.v1.serialize_sparse',
     'tf.serialize_tensor': 'tf.io.serialize_tensor',
     'tf.set_random_seed': 'tf.compat.v1.set_random_seed',
     'tf.setdiff1d': 'tf.compat.v1.setdiff1d',
@@ -580,10 +611,7 @@ renames = {
     'tf.train.maybe_batch_join': 'tf.compat.v1.train.maybe_batch_join',
     'tf.train.maybe_shuffle_batch': 'tf.compat.v1.train.maybe_shuffle_batch',
     'tf.train.maybe_shuffle_batch_join': 'tf.compat.v1.train.maybe_shuffle_batch_join',
-    'tf.train.natural_exp_decay': 'tf.compat.v1.train.natural_exp_decay',
-    'tf.train.noisy_linear_cosine_decay': 'tf.compat.v1.train.noisy_linear_cosine_decay',
-    'tf.train.piecewise_constant': 'tf.train.piecewise_constant_decay',
-    'tf.train.polynomial_decay': 'tf.compat.v1.train.polynomial_decay',
+    'tf.train.piecewise_constant': 'tf.compat.v1.train.piecewise_constant',
     'tf.train.queue_runner.QueueRunner': 'tf.compat.v1.train.queue_runner.QueueRunner',
     'tf.train.queue_runner.add_queue_runner': 'tf.compat.v1.train.queue_runner.add_queue_runner',
     'tf.train.queue_runner.start_queue_runners': 'tf.compat.v1.train.queue_runner.start_queue_runners',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index eb4e62b895..e8ea9784af 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -91,6 +91,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.manip.batch_to_space_nd": {
             "block_size": "block_shape",
         },
+        "tf.nn.dilation2d": {
+            "filter": "filters",
+            "rates": "dilations",
+        },
         "tf.nn.conv3d": {
             "filter": "filters"
         },
-- 
GitLab


From 5a08ce9bd17c2678c80b0af70996ebbc4879c937 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 26 Nov 2018 16:13:46 -0800
Subject: [PATCH 0823/1554] Log GPU and cuDNN version information.

PiperOrigin-RevId: 222904961
---
 tensorflow/stream_executor/BUILD              | 11 ++++++++
 tensorflow/stream_executor/dnn.h              |  7 +++--
 tensorflow/stream_executor/logging.proto      | 19 +++++++++++++
 .../stream_executor/stream_executor_pimpl.cc  | 28 ++++++++++++++++++-
 4 files changed, 61 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/stream_executor/logging.proto

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 2526e1adaa..74fec09d80 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -23,6 +23,14 @@ tf_proto_library(
     protodeps = tf_additional_all_protos(),
 )
 
+tf_proto_library(
+    name = "logging_proto",
+    srcs = ["logging.proto"],
+    cc_api_version = 2,
+    default_header = True,
+    protodeps = tf_additional_all_protos(),
+)
+
 cc_library(
     name = "stream_executor_impl",
     srcs = glob(
@@ -46,7 +54,9 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":dnn_proto_cc_impl",
+        ":logging_proto_cc_impl",
         "//tensorflow/core:lib",
+        "//tensorflow/core:logger",
         "//tensorflow/core:ptr_util",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/strings",
@@ -63,6 +73,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":dnn_proto_cc",
+        ":logging_proto_cc",
         "//tensorflow/core:lib",
         "//tensorflow/core:ptr_util",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index c044a356ef..43738d2d1d 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -906,9 +906,10 @@ class VersionInfo {
  public:
   VersionInfo(int major = 0, int minor = 0, int patch = 0)
       : major_(major), minor_(minor), patch_(patch) {}
-  int major_version() { return major_; }
-  int minor_version() { return minor_; }
-  int patch() { return patch_; }
+  int major_version() const { return major_; }
+  int minor_version() const { return minor_; }
+  int patch() const { return patch_; }
+
  private:
   int major_;
   int minor_;
diff --git a/tensorflow/stream_executor/logging.proto b/tensorflow/stream_executor/logging.proto
new file mode 100644
index 0000000000..2c75500cda
--- /dev/null
+++ b/tensorflow/stream_executor/logging.proto
@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package stream_executor;
+
+message CudnnVersion {
+  int32 major = 1;
+  int32 minor = 2;
+  int32 patch = 3;
+};
+
+message ComputeCapability {
+  int32 major = 1;
+  int32 minor = 2;
+}
+
+message CudaInfo {
+  CudnnVersion cudnn_version = 1;
+  ComputeCapability compute_capability = 2;
+}
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index d1d0bd9bc2..86bc4ab7d0 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include <utility>
 
 #include "absl/strings/str_cat.h"
+#include "tensorflow/core/platform/logger.h"
 #include "tensorflow/core/util/env_var.h"
 #include "tensorflow/stream_executor/blas.h"
 #include "tensorflow/stream_executor/fft.h"
@@ -33,6 +34,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
 #include "tensorflow/stream_executor/lib/threadpool.h"
+#include "tensorflow/stream_executor/logging.pb.h"
 #include "tensorflow/stream_executor/platform/port.h"
 #include "tensorflow/stream_executor/rng.h"
 #include "tensorflow/stream_executor/stream_executor_internal.h"
@@ -217,7 +219,31 @@ StreamExecutor::~StreamExecutor() {
 port::Status StreamExecutor::Init(int device_ordinal,
                                   DeviceOptions device_options) {
   device_ordinal_ = device_ordinal;
-  return implementation_->Init(device_ordinal, std::move(device_options));
+  TF_RETURN_IF_ERROR(
+      implementation_->Init(device_ordinal, std::move(device_options)));
+
+  if (platform_kind_ == PlatformKind::kCuda) {
+    CudaInfo info;
+
+    int cc_major, cc_minor;
+    GetDeviceDescription().cuda_compute_capability(&cc_major, &cc_minor);
+    info.mutable_compute_capability()->set_major(cc_major);
+    info.mutable_compute_capability()->set_minor(cc_minor);
+
+    if (auto *dnn = AsDnn()) {
+      port::StatusOr<dnn::VersionInfo> version_or = dnn->GetVersion();
+      if (version_or.ok()) {
+        const auto &version = version_or.ValueOrDie();
+        info.mutable_cudnn_version()->set_major(version.major_version());
+        info.mutable_cudnn_version()->set_minor(version.minor_version());
+        info.mutable_cudnn_version()->set_patch(version.patch());
+      }
+    }
+
+    tensorflow::Logger::Singleton()->LogProto(info);
+  }
+
+  return port::Status::OK();
 }
 
 port::Status StreamExecutor::Init() {
-- 
GitLab


From 08666e3eb80f144608347f4ffcc49ca51d58316d Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 26 Nov 2018 16:16:02 -0800
Subject: [PATCH 0824/1554] Adding `SensitivityAtSpecificity` and
 `SpecificityAtSensitivity` metrics to the new metrics module.

PiperOrigin-RevId: 222905327
---
 tensorflow/python/keras/BUILD           |   1 +
 tensorflow/python/keras/metrics.py      | 195 ++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 179 ++++++++++++++++++++++
 3 files changed, 375 insertions(+)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index bac961bb9d..697f591e35 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -306,6 +306,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 1ddeb0bee7..60632e5a25 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1266,6 +1266,201 @@ class Recall(Metric):
         array_ops.zeros_like(self.thresholds))
 
 
+@six.add_metaclass(abc.ABCMeta)
+class SensitivitySpecificityBase(Metric):
+  """Abstract base class for computing sensitivity and specificity.
+
+  For additional information about specificity and sensitivity, see the
+  following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity
+  """
+
+  def __init__(self, value, num_thresholds=200, name=None, dtype=None):
+    super(SensitivitySpecificityBase, self).__init__(name=name, dtype=dtype)
+    if num_thresholds <= 0:
+      raise ValueError('`num_thresholds` must be > 0.')
+    self.value = value
+    self.tp = self.add_weight(
+        'true_positives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.tn = self.add_weight(
+        'true_negatives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.fp = self.add_weight(
+        'false_positives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.fn = self.add_weight(
+        'false_negatives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+
+    # Compute `num_thresholds` thresholds in [0, 1]
+    if num_thresholds == 1:
+      self.thresholds = [0.5]
+    else:
+      thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
+                    for i in range(num_thresholds - 2)]
+      self.thresholds = [0.0] + thresholds + [1.0]
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates confusion matrix statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    return _update_confusion_matrix_variables({
+        _ConfusionMatrix.TRUE_POSITIVES: self.tp,
+        _ConfusionMatrix.TRUE_NEGATIVES: self.tn,
+        _ConfusionMatrix.FALSE_POSITIVES: self.fp,
+        _ConfusionMatrix.FALSE_NEGATIVES: self.fn,
+    }, y_true, y_pred, self.thresholds, sample_weight)
+
+
+class SensitivityAtSpecificity(SensitivitySpecificityBase):
+  """Computes the sensitivity at a given specificity.
+
+  `Sensitivity` measures the proportion of actual positives that are correctly
+  identified as such (tp / (tp + fn)).
+  `Specificity` measures the proportion of actual negatives that are correctly
+  identified as such (tn / (tn + fp)).
+
+  This metric creates four local variables, `true_positives`, `true_negatives`,
+  `false_positives` and `false_negatives` that are used to compute the
+  sensitivity at the given specificity. The threshold for the given specificity
+  value is computed and used to evaluate the corresponding sensitivity.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  For additional information about specificity and sensitivity, see the
+  following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity
+
+  Usage:
+
+  ```python
+  m = tf.metrics.SensitivityAtSpecificity(0.4, num_thresholds=1)
+  m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
+  print('Final result: ', m.result().numpy())  # Final result: 0.5
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+      'sgd',
+      loss='mse',
+      metrics=[tf.metrics.SensitivityAtSpecificity()])
+  ```
+  """
+
+  def __init__(self, specificity, num_thresholds=200, name=None, dtype=None):
+    """Creates a `SensitivityAtSpecificity` instance.
+
+    Args:
+      specificity: A scalar value in range `[0, 1]`.
+      num_thresholds: (Optional) Defaults to 200. The number of thresholds to
+        use for matching the given specificity.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    if specificity < 0 or specificity > 1:
+      raise ValueError('`specificity` must be in the range [0, 1].')
+    super(SensitivityAtSpecificity, self).__init__(
+        specificity, num_thresholds=num_thresholds, name=name, dtype=dtype)
+
+  def result(self):
+    # Calculate specificities at all the thresholds.
+    specificities = math_ops.div_no_nan(self.tn, self.tn + self.fp)
+
+    # Find the index of the threshold where the specificity is closest to the
+    # given specificity.
+    min_index = math_ops.argmin(
+        math_ops.abs(specificities - self.value), axis=0)
+    min_index = math_ops.cast(min_index, dtypes.int32)
+
+    # Compute sensitivity at that index.
+    return math_ops.div_no_nan(self.tp[min_index],
+                               self.tp[min_index] + self.fn[min_index])
+
+
+class SpecificityAtSensitivity(SensitivitySpecificityBase):
+  """Computes the specificity at a given sensitivity.
+
+  `Sensitivity` measures the proportion of actual positives that are correctly
+  identified as such (tp / (tp + fn)).
+  `Specificity` measures the proportion of actual negatives that are correctly
+  identified as such (tn / (tn + fp)).
+
+  This metric creates four local variables, `true_positives`, `true_negatives`,
+  `false_positives` and `false_negatives` that are used to compute the
+  specificity at the given sensitivity. The threshold for the given sensitivity
+  value is computed and used to evaluate the corresponding specificity.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  For additional information about specificity and sensitivity, see the
+  following: https://en.wikipedia.org/wiki/Sensitivity_and_specificity
+
+  Usage:
+
+  ```python
+  m = tf.metrics.SpecificityAtSensitivity(0.8, num_thresholds=1)
+  m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
+  print('Final result: ', m.result().numpy())  # Final result: 1.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+      'sgd',
+      loss='mse',
+      metrics=[tf.metrics.SpecificityAtSensitivity()])
+  ```
+  """
+
+  def __init__(self, sensitivity, num_thresholds=200, name=None, dtype=None):
+    """Creates a `SpecificityAtSensitivity` instance.
+
+    Args:
+      sensitivity: A scalar value in range `[0, 1]`.
+      num_thresholds: (Optional) Defaults to 200. The number of thresholds to
+        use for matching the given specificity.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    if sensitivity < 0 or sensitivity > 1:
+      raise ValueError('`sensitivity` must be in the range [0, 1].')
+    super(SpecificityAtSensitivity, self).__init__(
+        sensitivity, num_thresholds=num_thresholds, name=name, dtype=dtype)
+
+  def result(self):
+    # Calculate sensitivities at all the thresholds.
+    sensitivities = math_ops.div_no_nan(self.tp, self.tp + self.fn)
+
+    # Find the index of the threshold where the sensitivity is closest to the
+    # given specificity.
+    min_index = math_ops.argmin(
+        math_ops.abs(sensitivities - self.value), axis=0)
+    min_index = math_ops.cast(min_index, dtypes.int32)
+
+    # Compute specificity at that index.
+    return math_ops.div_no_nan(self.tn[min_index],
+                               self.tn[min_index] + self.fp[min_index])
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index eeade4f37d..74e5d4d4ce 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.eager import context
@@ -1023,5 +1024,183 @@ class RecallTest(test.TestCase):
                          1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
+
+  def test_config(self):
+    s_obj = metrics.SensitivityAtSpecificity(
+        0.4, num_thresholds=100, name='sensitivity_at_specificity_1')
+    self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')
+    self.assertLen(s_obj.variables, 4)
+    self.assertEqual(s_obj.value, 0.4)
+    self.assertLen(s_obj.thresholds, 100)
+
+  def test_value_is_idempotent(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.7)
+    y_pred = random_ops.random_uniform((10, 3),
+                                       maxval=1,
+                                       dtype=dtypes.float32,
+                                       seed=1)
+    y_true = random_ops.random_uniform((10, 3),
+                                       maxval=2,
+                                       dtype=dtypes.int64,
+                                       seed=1)
+    update_op = s_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_sensitivity = self.evaluate(s_obj.result())
+    for _ in range(10):
+      self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()),
+                             1e-3)
+
+  def test_unweighted_all_correct(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.7)
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
+    y_true = constant_op.constant(inputs)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+
+  def test_unweighted_high_specificity(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.8)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.8, self.evaluate(result))
+
+  def test_unweighted_low_specificity(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.6, self.evaluate(result))
+
+  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
+  def test_weighted(self, label_dtype):
+    s_obj = metrics.SensitivityAtSpecificity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = math_ops.cast(label_values, dtype=label_dtype)
+    weights = constant_op.constant(weight_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred, sample_weight=weights)
+    self.assertAlmostEqual(0.675, self.evaluate(result))
+
+  def test_invalid_specificity(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`specificity` must be in the range \[0, 1\].'):
+      metrics.SensitivityAtSpecificity(-1)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
+      metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
+
+  def test_config(self):
+    s_obj = metrics.SpecificityAtSensitivity(
+        0.4, num_thresholds=100, name='specificity_at_sensitivity_1')
+    self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')
+    self.assertLen(s_obj.variables, 4)
+    self.assertEqual(s_obj.value, 0.4)
+    self.assertLen(s_obj.thresholds, 100)
+
+  def test_value_is_idempotent(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.7)
+    y_pred = random_ops.random_uniform((10, 3),
+                                       maxval=1,
+                                       dtype=dtypes.float32,
+                                       seed=1)
+    y_true = random_ops.random_uniform((10, 3),
+                                       maxval=2,
+                                       dtype=dtypes.int64,
+                                       seed=1)
+    update_op = s_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_specificity = self.evaluate(s_obj.result())
+    for _ in range(10):
+      self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()),
+                             1e-3)
+
+  def test_unweighted_all_correct(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.7)
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
+    y_true = constant_op.constant(inputs)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+
+  def test_unweighted_high_sensitivity(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.8)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.4, self.evaluate(result))
+
+  def test_unweighted_low_sensitivity(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.6, self.evaluate(result))
+
+  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
+  def test_weighted(self, label_dtype):
+    s_obj = metrics.SpecificityAtSensitivity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = math_ops.cast(label_values, dtype=label_dtype)
+    weights = constant_op.constant(weight_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred, sample_weight=weights)
+    self.assertAlmostEqual(0.4, self.evaluate(result))
+
+  def test_invalid_sensitivity(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`sensitivity` must be in the range \[0, 1\].'):
+      metrics.SpecificityAtSensitivity(-1)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
+      metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 343734b56c2d9014293a2ed1e30ad4e1c7b6e07a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 26 Nov 2018 16:22:34 -0800
Subject: [PATCH 0825/1554] Use if_cuda instead of if_cuda_is_configured

if_cuda_is_configured is now a no-op.  For xla_gpu_jit this does not matter
since its deps are under if_cuda, but it does make a difference for
xla_gpu_device.

We should at some point refactor both xla_gpu_device and xla_gpu_jit to either
uniformly include an if_cuda or to not include it, but there are other hills I
want to die on at the moment.

PiperOrigin-RevId: 222906280
---
 tensorflow/compiler/jit/BUILD | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 682c0f0cb0..6b0d20bc18 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -23,7 +23,6 @@ package(
 load("//tensorflow:tensorflow.bzl", "cc_header_only_library")
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
-load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 
@@ -38,7 +37,7 @@ cc_library(
         ":xla_cpu_device",
         ":xla_cpu_jit",
         "//tensorflow/compiler/plugin",
-    ] + if_cuda_is_configured([
+    ] + if_cuda([
         ":xla_gpu_device",
         ":xla_gpu_jit",
     ]),
-- 
GitLab


From 98e8a01bc8f46f5ca0bff78017d00a15370e2ba1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 26 Nov 2018 16:24:56 -0800
Subject: [PATCH 0826/1554] Some cleanups involving tensorarray gradients and
 control flow v2.

Also mark a bunch of ops as not differentiable.

PiperOrigin-RevId: 222906549
---
 tensorflow/python/framework/function_def_to_graph.py | 4 +++-
 tensorflow/python/ops/cond_v2.py                     | 8 +++++++-
 tensorflow/python/ops/list_ops.py                    | 2 ++
 tensorflow/python/ops/resource_variable_ops.py       | 3 +++
 tensorflow/python/ops/while_v2.py                    | 4 ++++
 5 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/function_def_to_graph.py b/tensorflow/python/framework/function_def_to_graph.py
index 4d1aabde06..1803cb906d 100644
--- a/tensorflow/python/framework/function_def_to_graph.py
+++ b/tensorflow/python/framework/function_def_to_graph.py
@@ -174,7 +174,9 @@ def function_def_to_graph_def(fdef, input_shapes=None):
   # Update inputs of all nodes in graph.
   for node_def in graph_def.node:
     for i in range(len(node_def.input)):
-      node_def.input[i] = nested_to_flat_tensor_name[node_def.input[i]]
+      # TODO(apassos): how can it not be there?
+      if node_def.input[i] in nested_to_flat_tensor_name:
+        node_def.input[i] = nested_to_flat_tensor_name[node_def.input[i]]
 
   return graph_def, nested_to_flat_tensor_name
 
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 4db59871d3..927c64919d 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -25,12 +25,14 @@ from __future__ import print_function
 
 import collections
 
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import function_def_to_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import gen_functional_ops
+from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.util import nest
 
@@ -262,7 +264,11 @@ def _grad_fn(func_graph, grads):
   # both branches have zero gradient.
   for i in range(len(result)):
     if result[i] is None:
-      result[i] = array_ops.zeros_like(func_graph.inputs[i])
+      if func_graph.inputs[i].dtype == dtypes.resource:
+        result[i] = array_ops.zeros(
+            gen_resource_variable_ops.variable_shape(func_graph.inputs[i]))
+      else:
+        result[i] = array_ops.zeros_like(func_graph.inputs[i])
 
   return result
 
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index 515926002d..89ff48ebd9 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -31,6 +31,8 @@ from tensorflow.python.ops.gen_list_ops import *
 
 
 ops.NotDifferentiable("TensorListConcat")
+ops.NotDifferentiable("TensorListElementShape")
+ops.NotDifferentiable("TensorListLength")
 ops.NotDifferentiable("TensorListPushBackBatch")
 
 
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index c20f8fb938..f614c48485 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1525,3 +1525,6 @@ def copy_to_graph_uninitialized(var):
   new_variable._maybe_initialize_checkpointable()
   # pylint: enable=protected-access
   return new_variable
+
+ops.NotDifferentiable("VarIsInitializedOp")
+ops.NotDifferentiable("VariableShape")
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 1252c7fb03..6821b63d0e 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -839,6 +839,10 @@ def _is_tensor_array_handle(tensor):
   # TODO(b/118452219): add test coverage for this.
   tensor = func_graph_module.maybe_captured(tensor)
 
+  if isinstance(tensor, ops.EagerTensor):
+    # Eager execution doesn't quite support legacy tensorarray
+    return False
+
   return tensor.op.type in TENSOR_ARRAY_HANDLE_OPS
 
 
-- 
GitLab


From 09070cb9d3897db33f51bd7d3ac7807590290b8d Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Mon, 26 Nov 2018 16:29:18 -0800
Subject: [PATCH 0827/1554] Fix links in tflite_convert documentation.

PiperOrigin-RevId: 222907025
---
 tensorflow/lite/g3doc/devguide.md | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md
index 798bf4996a..d42729c7ea 100644
--- a/tensorflow/lite/g3doc/devguide.md
+++ b/tensorflow/lite/g3doc/devguide.md
@@ -69,17 +69,18 @@ grow in future Tensorflow Lite releases.
 
 ## 2. Convert the model format
 
-The [TensorFlow Lite Converter](convert) accepts the following file formats:
+The [TensorFlow Lite Converter](convert/index.md) accepts the following file
+formats:
 
 *   `SavedModel` — A `GraphDef` and checkpoint with a signature that labels
     input and output arguments to a model. See the documentation for converting
-    SavedModels using [Python](convert/python_api#basic_savedmodel) or using the
-    [command line](convert/cmdline_examples#savedmodel).
+    SavedModels using [Python](convert/python_api.md#basic_savedmodel) or using
+    the [command line](convert/cmdline_examples.md#savedmodel).
 *   `tf.keras` - A HDF5 file containing a model with weights and input and
     output arguments generated by `tf.Keras`. See [here] for converting HDF5
     models. See the documentation for converting HDF5 models using
-    [Python](convert/python_api#basic_keras_file) or using the
-    [command line](convert/cmdline_examples#keras).
+    [Python](convert/python_api.md#basic_keras_file) or using the
+    [command line](convert/cmdline_examples.md#keras).
 *   `frozen tf.GraphDef` — A subclass of `tf.GraphDef` that does not contain
     variables. A `GraphDef` can be converted to a `frozen GraphDef` by taking a
     checkpoint and a `GraphDef`, and converting each variable into a constant
@@ -153,10 +154,11 @@ the arguments for specifying the output nodes for inference in the
 
 ### Full converter reference
 
-The [TensorFlow Lite Converter](convert/) can be [Python](convert/python_api.md)
-or from the [command line](convert/cmdline_examples.md). This allows you to
-integrate the conversion step into the model design workflow, ensuring the model
-is easy to convert to a mobile inference graph.
+The [TensorFlow Lite Converter](convert/index.md) can be
+[Python](convert/python_api.md) or from the
+[command line](convert/cmdline_examples.md). This allows you to integrate the
+conversion step into the model design workflow, ensuring the model is easy to
+convert to a mobile inference graph.
 
 ### Ops compatibility
 
-- 
GitLab


From 76f1886556a552f22d533d5719eaaaaa1a91a66f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 16:32:23 -0800
Subject: [PATCH 0828/1554] Delete "device_util" from its old location in
 training/, all references are to its new location in distribute/.

PiperOrigin-RevId: 222907490
---
 tensorflow/python/BUILD                   |  9 ---------
 tensorflow/python/training/device_util.py | 22 ----------------------
 2 files changed, 31 deletions(-)
 delete mode 100644 tensorflow/python/training/device_util.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4cfb90c486..19d2af4515 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3595,15 +3595,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "device_util",
-    srcs = ["training/device_util.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/python/distribute:device_util",
-    ],
-)
-
 py_library(
     name = "distribute",
     srcs = [
diff --git a/tensorflow/python/training/device_util.py b/tensorflow/python/training/device_util.py
deleted file mode 100644
index bf8e98052d..0000000000
--- a/tensorflow/python/training/device_util.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Deprecated, please use ../distribute/device_util.py."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# pylint: disable=wildcard-import
-from tensorflow.python.distribute.device_util import *
-- 
GitLab


From 958859263d008b5c86a589b7574e5ef704e5176b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 16:35:19 -0800
Subject: [PATCH 0829/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 222907921
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 65 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 13 ++++
 2 files changed, 78 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index c855f1c4a0..ba0bf553d0 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -41786,6 +41786,71 @@ op {
     }
   }
 }
+op {
+  name: "QuantizeAndDequantizeV2"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_min"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_max"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "signed_input"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "num_bits"
+    type: "int"
+    default_value {
+      i: 8
+    }
+  }
+  attr {
+    name: "range_given"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_BFLOAT16
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "round_mode"
+    type: "string"
+    default_value {
+      s: "HALF_TO_EVEN"
+    }
+    allowed_values {
+      list {
+        s: "HALF_TO_EVEN"
+        s: "HALF_UP"
+      }
+    }
+  }
+}
 op {
   name: "QuantizeAndDequantizeV3"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 44cc64dc3b..bae50a7139 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -21090,6 +21090,19 @@ op {
       }
     }
   }
+  attr {
+    name: "round_mode"
+    type: "string"
+    default_value {
+      s: "HALF_TO_EVEN"
+    }
+    allowed_values {
+      list {
+        s: "HALF_TO_EVEN"
+        s: "HALF_UP"
+      }
+    }
+  }
 }
 op {
   name: "QuantizeAndDequantizeV3"
-- 
GitLab


From 35dcdd967ec15956145d7be7835da12ae74fc568 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 26 Nov 2018 16:51:20 -0800
Subject: [PATCH 0830/1554] - Adding V2 API for MeanSquaredError loss. -
 Deprecating V1 losses APIs.

PiperOrigin-RevId: 222910192
---
 tensorflow/python/keras/losses.py             | 22 ++++++++-
 tensorflow/python/ops/losses/losses_impl.py   | 22 ++++-----
 ...low.keras.losses.-mean-squared-error.pbtxt | 22 +++++++++
 .../golden/v1/tensorflow.keras.losses.pbtxt   |  4 ++
 ...ensorflow.losses.-mean-squared-error.pbtxt | 22 +++++++++
 .../api/golden/v1/tensorflow.losses.pbtxt     |  4 ++
 ...low.keras.losses.-mean-squared-error.pbtxt | 22 +++++++++
 .../golden/v2/tensorflow.keras.losses.pbtxt   |  4 ++
 ...ensorflow.losses.-mean-squared-error.pbtxt | 22 +++++++++
 .../api/golden/v2/tensorflow.losses.pbtxt     | 48 ++-----------------
 10 files changed, 136 insertions(+), 56 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 8afe2fadce..1bd9f729c5 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -115,8 +115,28 @@ class Loss(object):
     NotImplementedError('Must be implemented in subclasses.')
 
 
+@tf_export('losses.MeanSquaredError', 'keras.losses.MeanSquaredError')
 class MeanSquaredError(Loss):
-  """Computes the mean of squares of errors between labels and predictions."""
+  """Computes the mean of squares of errors between labels and predictions.
+
+  For example, if `y_true` is [0., 0., 1., 1.] and `y_pred` is [1., 1., 1., 0.]
+  then the mean squared error value is 3/4 (0.75).
+
+  Usage:
+
+  ```python
+  mse = tf.losses.MeanSquaredError()
+  loss = mse([0., 0., 1., 1.], [1., 1., 1., 0.])
+  print('Loss: ', loss.numpy())  # Loss: 0.75
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.MeanSquaredError())
+  ```
+  """
 
   def call(self, y_true, y_pred):
     """Invokes the `MeanSquaredError` instance.
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 7c52b28b39..1b470937d4 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -133,7 +133,7 @@ def _num_elements(losses):
     return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype)
 
 
-@tf_export("losses.compute_weighted_loss")
+@tf_export(v1=["losses.compute_weighted_loss"])
 def compute_weighted_loss(
     losses, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES,
     reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
@@ -203,7 +203,7 @@ def compute_weighted_loss(
       return loss
 
 
-@tf_export("losses.absolute_difference")
+@tf_export(v1=["losses.absolute_difference"])
 def absolute_difference(
     labels, predictions, weights=1.0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
@@ -256,7 +256,7 @@ def absolute_difference(
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
-@tf_export("losses.cosine_distance")
+@tf_export(v1=["losses.cosine_distance"])
 @deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def cosine_distance(
     labels, predictions, axis=None, weights=1.0, scope=None,
@@ -312,7 +312,7 @@ def cosine_distance(
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
-@tf_export("losses.hinge_loss")
+@tf_export(v1=["losses.hinge_loss"])
 def hinge_loss(labels, logits, weights=1.0, scope=None,
                loss_collection=ops.GraphKeys.LOSSES,
                reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
@@ -362,7 +362,7 @@ def hinge_loss(labels, logits, weights=1.0, scope=None,
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
-@tf_export("losses.huber_loss")
+@tf_export(v1=["losses.huber_loss"])
 def huber_loss(labels, predictions, weights=1.0, delta=1.0, scope=None,
                loss_collection=ops.GraphKeys.LOSSES,
                reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
@@ -440,7 +440,7 @@ def huber_loss(labels, predictions, weights=1.0, delta=1.0, scope=None,
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
-@tf_export("losses.log_loss")
+@tf_export(v1=["losses.log_loss"])
 def log_loss(labels, predictions, weights=1.0, epsilon=1e-7, scope=None,
              loss_collection=ops.GraphKeys.LOSSES,
              reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
@@ -497,7 +497,7 @@ def log_loss(labels, predictions, weights=1.0, epsilon=1e-7, scope=None,
 
 
 # TODO(b/37208492): Add reduction arg.
-@tf_export("losses.mean_pairwise_squared_error")
+@tf_export(v1=["losses.mean_pairwise_squared_error"])
 def mean_pairwise_squared_error(
     labels, predictions, weights=1.0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES):
@@ -593,7 +593,7 @@ def mean_pairwise_squared_error(
       return mean_loss
 
 
-@tf_export("losses.mean_squared_error")
+@tf_export(v1=["losses.mean_squared_error"])
 def mean_squared_error(
     labels, predictions, weights=1.0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
@@ -646,7 +646,7 @@ def mean_squared_error(
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
-@tf_export("losses.sigmoid_cross_entropy")
+@tf_export(v1=["losses.sigmoid_cross_entropy"])
 def sigmoid_cross_entropy(
     multi_class_labels, logits, weights=1.0, label_smoothing=0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
@@ -710,7 +710,7 @@ def sigmoid_cross_entropy(
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
-@tf_export("losses.softmax_cross_entropy")
+@tf_export(v1=["losses.softmax_cross_entropy"])
 def softmax_cross_entropy(
     onehot_labels, logits, weights=1.0, label_smoothing=0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
@@ -832,7 +832,7 @@ def _remove_squeezable_dimensions(
   return labels, predictions, weights
 
 
-@tf_export("losses.sparse_softmax_cross_entropy")
+@tf_export(v1=["losses.sparse_softmax_cross_entropy"])
 def sparse_softmax_cross_entropy(
     labels, logits, weights=1.0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..a571853350
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.MeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index eca6b91538..a0af6a29f0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "MeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "KLD"
     argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..a626d9c7e6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.losses.MeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt
index c1d190ae11..a198db1b35 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.losses"
 tf_module {
+  member {
+    name: "MeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Reduction"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..a571853350
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.MeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index 8618c6f1c7..cb156e2248 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "MeanSquaredError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Reduction"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
new file mode 100644
index 0000000000..a626d9c7e6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.losses.MeanSquaredError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
index c1d190ae11..87f5ef3491 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
@@ -1,25 +1,17 @@
 path: "tensorflow.losses"
 tf_module {
   member {
-    name: "Reduction"
+    name: "MeanSquaredError"
     mtype: "<type \'type\'>"
   }
-  member_method {
-    name: "absolute_difference"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
+  member {
+    name: "Reduction"
+    mtype: "<type \'type\'>"
   }
   member_method {
     name: "add_loss"
     argspec: "args=[\'loss\', \'loss_collection\'], varargs=None, keywords=None, defaults=[\'losses\'], "
   }
-  member_method {
-    name: "compute_weighted_loss"
-    argspec: "args=[\'losses\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "cosine_distance"
-    argspec: "args=[\'labels\', \'predictions\', \'axis\', \'weights\', \'scope\', \'loss_collection\', \'reduction\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\', \'None\'], "
-  }
   member_method {
     name: "get_losses"
     argspec: "args=[\'scope\', \'loss_collection\'], varargs=None, keywords=None, defaults=[\'None\', \'losses\'], "
@@ -36,36 +28,4 @@ tf_module {
     name: "get_total_loss"
     argspec: "args=[\'add_regularization_losses\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'total_loss\'], "
   }
-  member_method {
-    name: "hinge_loss"
-    argspec: "args=[\'labels\', \'logits\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "huber_loss"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'delta\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "log_loss"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'epsilon\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'1e-07\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "mean_pairwise_squared_error"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'scope\', \'loss_collection\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\'], "
-  }
-  member_method {
-    name: "mean_squared_error"
-    argspec: "args=[\'labels\', \'predictions\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "sigmoid_cross_entropy"
-    argspec: "args=[\'multi_class_labels\', \'logits\', \'weights\', \'label_smoothing\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "softmax_cross_entropy"
-    argspec: "args=[\'onehot_labels\', \'logits\', \'weights\', \'label_smoothing\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
-  member_method {
-    name: "sparse_softmax_cross_entropy"
-    argspec: "args=[\'labels\', \'logits\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
-  }
 }
-- 
GitLab


From 453335db8382418dc84593d044f63a995adc025a Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 26 Nov 2018 16:57:34 -0800
Subject: [PATCH 0831/1554]  applying proper coding style

---
 tensorflow/core/kernels/mkl_lrn_op.cc | 42 ++++++++++++---------------
 1 file changed, 19 insertions(+), 23 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc
index 4d46abb0a4..407ce5d653 100644
--- a/tensorflow/core/kernels/mkl_lrn_op.cc
+++ b/tensorflow/core/kernels/mkl_lrn_op.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 #include <vector>
 #include "mkldnn.hpp"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -32,6 +31,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/util/mkl_util.h"
 #include "tensorflow/core/util/tensor_format.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 #if !defined(IS_MOBILE_PLATFORM)
 #include "tensorflow/core/util/work_sharder.h"
@@ -71,11 +71,10 @@ class MklLRNOp : public OpKernel {
   explicit MklLRNOp(OpKernelConstruction* context) : OpKernel(context) {
     int64 depth_radius64;
     OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64));
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(depth_radius64, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("depth_radius = ", depth_radius64,
-                                " larger than int max"));
+    OP_REQUIRES(context, FastBoundsCheck(depth_radius64,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("depth_radius = ", depth_radius64,
+                                        " larger than int max"));
     depth_radius_ = static_cast<size_t>(depth_radius64);
 
     OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_));
@@ -161,9 +160,9 @@ class MklLRNOp : public OpKernel {
       PrepareAndExecuteNet(lrn_prim_desc, &src_dnn_data, &dst_dnn_data,
                            &workspace_dnn_data);
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -292,16 +291,14 @@ class MklLRNOp : public OpKernel {
     if (src_dnn_shape.IsMklTensor()) {
       OP_REQUIRES(context, src_dnn_shape.GetDimension() == 4,
                   errors::InvalidArgument("input must be 4-dimensional"));
-      OP_REQUIRES(context,
-                  FastBoundsCheck(src_tensor.NumElements(),
-                                  std::numeric_limits<int>::max()),
+      OP_REQUIRES(context, FastBoundsCheck(src_tensor.NumElements(),
+                                           std::numeric_limits<int>::max()),
                   errors::InvalidArgument("argument to LRN too large"));
     } else {
       OP_REQUIRES(context, src_tensor.dims() == 4,
                   errors::InvalidArgument("input must be 4-dimensional"));
-      OP_REQUIRES(context,
-                  FastBoundsCheck(src_tensor.NumElements(),
-                                  std::numeric_limits<int>::max()),
+      OP_REQUIRES(context, FastBoundsCheck(src_tensor.NumElements(),
+                                           std::numeric_limits<int>::max()),
                   errors::InvalidArgument("argument to LRN too large"));
     }
   }
@@ -321,11 +318,10 @@ class MklLRNGradOp : public OpKernel {
   explicit MklLRNGradOp(OpKernelConstruction* context) : OpKernel(context) {
     int64 depth_radius64;
     OP_REQUIRES_OK(context, context->GetAttr("depth_radius", &depth_radius64));
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(depth_radius64, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("depth_radius = ", depth_radius64,
-                                " larger than int max"));
+    OP_REQUIRES(context, FastBoundsCheck(depth_radius64,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("depth_radius = ", depth_radius64,
+                                        " larger than int max"));
     depth_radius_ = static_cast<int>(depth_radius64);
     OP_REQUIRES_OK(context, context->GetAttr("bias", &bias_));
     OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_));
@@ -432,9 +428,9 @@ class MklLRNGradOp : public OpKernel {
           memory::primitive_desc(target_diff_dst_md, cpu_engine),
           &workspace_dnn_data);
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
-- 
GitLab


From eea816869a0a85ab5d9427060a8b113fe709c623 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 26 Nov 2018 16:51:23 -0800
Subject: [PATCH 0832/1554] Automated rollback of commit
 72f193b5c9f306d258b289afd7a7977af7b8f5f7

PiperOrigin-RevId: 222910206
---
 configure.py              | 1 +
 tensorflow/BUILD          | 6 ++++++
 tensorflow/core/BUILD     | 8 ++++----
 tensorflow/tensorflow.bzl | 8 ++++++--
 tools/bazel.rc            | 1 +
 5 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/configure.py b/configure.py
index 0b16fe1314..dfb87550b1 100644
--- a/configure.py
+++ b/configure.py
@@ -1694,6 +1694,7 @@ def main():
   config_info_line('nohdfs', 'Disable HDFS support.')
   config_info_line('noignite', 'Disable Apacha Ignite support.')
   config_info_line('nokafka', 'Disable Apache Kafka support.')
+  config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 17577afecb..fd4b94202a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -246,6 +246,12 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "no_nccl_support",
+    define_values = {"no_nccl_support": "true"},
+    visibility = ["//visibility:public"],
+)
+
 # Crosses between platforms and file system libraries not supported on those
 # platforms due to limitations in nested select() statements.
 config_setting(
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 781ada7b46..bd5bbaa310 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -95,7 +95,7 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
 load("//tensorflow:tensorflow.bzl", "tf_cc_tests_gpu")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule")
-load("//tensorflow:tensorflow.bzl", "if_not_tx2_llvm_or_windows_cuda")
+load("//tensorflow:tensorflow.bzl", "if_nccl")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_only_cc_test")
 
 # For platform specific build config
@@ -1416,9 +1416,7 @@ cc_library(
         "//tensorflow/core/kernels:summary_kernels",
         "//tensorflow/core/kernels:training_ops",
         "//tensorflow/core/kernels:word2vec_kernels",
-    ] + tf_additional_cloud_kernel_deps() + if_not_tx2_llvm_or_windows_cuda([
-        "//tensorflow/core/kernels:nccl_kernels",
-    ]) + if_not_windows([
+    ] + tf_additional_cloud_kernel_deps() + if_not_windows([
         "//tensorflow/core/kernels:fact_op",
         "//tensorflow/core/kernels:array_not_windows",
         "//tensorflow/core/kernels:math_not_windows",
@@ -1443,6 +1441,8 @@ cc_library(
     ]) + if_cuda([
         "//tensorflow/core/grappler/optimizers:gpu_swapping_kernels",
         "//tensorflow/core/grappler/optimizers:gpu_swapping_ops",
+    ]) + if_nccl([
+        "//tensorflow/core/kernels:nccl_kernels",
     ]),
 )
 
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 2d67d1f466..4bc68445ac 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -203,8 +203,12 @@ def if_override_eigen_strong_inline(a):
         "//conditions:default": [],
     })
 
-def if_not_tx2_llvm_or_windows_cuda(a):
-    return if_not_windows_cuda(a)
+def if_nccl(a):
+    return select({
+        "//tensorflow:no_nccl_support": [],
+        "//tensorflow:windows": [],
+        "//conditions:default": a,
+    })
 
 def get_win_copts(is_external = False):
     WINDOWS_COPTS = [
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 8c2052ee8a..1fdf51f53e 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -72,6 +72,7 @@ build:nogcp --define=no_gcp_support=true
 build:nohdfs --define=no_hdfs_support=true
 build:nokafka --define=no_kafka_support=true
 build:noignite --define=no_ignite_support=true
+build:nonccl --define=no_nccl_support=true
 
 build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
-- 
GitLab


From 08ae34d7744b30a3837fae9c9114c80d55e89fdb Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Mon, 26 Nov 2018 17:05:46 -0800
Subject: [PATCH 0833/1554] Export InMemoryEvaluatorHook class to
 estimator.experimental

PiperOrigin-RevId: 222912301
---
 ...perimental.-in-memory-evaluator-hook.pbtxt | 30 +++++++++++++++++++
 .../tensorflow.estimator.experimental.pbtxt   |  4 +++
 ...perimental.-in-memory-evaluator-hook.pbtxt | 30 +++++++++++++++++++
 .../tensorflow.estimator.experimental.pbtxt   |  4 +++
 4 files changed, 68 insertions(+)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
new file mode 100644
index 0000000000..aba120218c
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.experimental.InMemoryEvaluatorHook"
+tf_class {
+  is_instance: "<class \'tensorflow_estimator.python.estimator.hooks.InMemoryEvaluatorHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'estimator\', \'input_fn\', \'steps\', \'hooks\', \'name\', \'every_n_iter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'100\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
index cabca3e883..4b287e1f80 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.estimator.experimental"
 tf_module {
+  member {
+    name: "InMemoryEvaluatorHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearSDCA"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
new file mode 100644
index 0000000000..aba120218c
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.experimental.InMemoryEvaluatorHook"
+tf_class {
+  is_instance: "<class \'tensorflow_estimator.python.estimator.hooks.InMemoryEvaluatorHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'estimator\', \'input_fn\', \'steps\', \'hooks\', \'name\', \'every_n_iter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'100\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
index cabca3e883..4b287e1f80 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.estimator.experimental"
 tf_module {
+  member {
+    name: "InMemoryEvaluatorHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearSDCA"
     mtype: "<type \'type\'>"
-- 
GitLab


From bd445e7e9d4ed703fc162a985751c160cb00f2af Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 26 Nov 2018 17:08:55 -0800
Subject: [PATCH 0834/1554] Expose decay in kwargs. A later design is required
 for scheduable decay objects.

PiperOrigin-RevId: 222912708
---
 .../python/keras/optimizer_v2/adadelta.py     |  25 +++--
 .../python/keras/optimizer_v2/adagrad.py      |  31 ++----
 .../python/keras/optimizer_v2/adagrad_test.py |  44 ++++++++
 tensorflow/python/keras/optimizer_v2/adam.py  |  29 +++--
 .../python/keras/optimizer_v2/adam_test.py    | 103 +++++++++++++-----
 .../python/keras/optimizer_v2/adamax.py       |  32 +++---
 .../python/keras/optimizer_v2/adamax_test.py  |  84 +++++++++++---
 tensorflow/python/keras/optimizer_v2/ftrl.py  |  45 ++++----
 .../keras/optimizer_v2/gradient_descent.py    |  35 +++---
 .../optimizer_v2/gradient_descent_test.py     |  68 +++++++++++-
 tensorflow/python/keras/optimizer_v2/nadam.py |  53 +++++++--
 .../python/keras/optimizer_v2/nadam_test.py   |  59 ++++++++--
 .../python/keras/optimizer_v2/optimizer_v2.py |  28 ++++-
 .../python/keras/optimizer_v2/rmsprop.py      |  37 ++++---
 .../python/keras/optimizer_v2/rmsprop_test.py |  67 ++++++++++++
 15 files changed, 565 insertions(+), 175 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index 21a3f06f4f..e1d7ecb558 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
-from tensorflow.python.ops import math_ops
 from tensorflow.python.training import training_ops
 
 
@@ -55,7 +54,8 @@ class Adadelta(optimizer_v2.OptimizerV2):
                learning_rate=0.001,
                rho=0.95,
                epsilon=1e-7,
-               name='Adadelta'):
+               name='Adadelta',
+               **kwargs):
     """Construct a new Adadelta optimizer.
 
     Adadelta is a more robust extension of Adagrad that adapts learning rates
@@ -73,6 +73,7 @@ class Adadelta(optimizer_v2.OptimizerV2):
                to better conditioning the grad update.
       name: Optional name prefix for the operations created when applying
         gradients.  Defaults to "Adadelta".
+      **kwargs: keyword arguments. Allowed to be {`decay`}
 
     @compatibility(eager)
     When eager execution is enabled, `learning_rate`, `rho`, and `epsilon` can
@@ -81,8 +82,9 @@ class Adadelta(optimizer_v2.OptimizerV2):
     invocations of optimizer functions.
     @end_compatibility
     """
-    super(Adadelta, self).__init__(name)
+    super(Adadelta, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', learning_rate)
+    self._set_hyper('decay', self._initial_decay)
     self._set_hyper('rho', rho)
     self._set_hyper('epsilon', epsilon)
 
@@ -92,28 +94,32 @@ class Adadelta(optimizer_v2.OptimizerV2):
       self.add_slot(v, 'accum_var')
 
   def _resource_apply_dense(self, grad, var):
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     accum_grad = self.get_slot(var, 'accum_grad')
     accum_var = self.get_slot(var, 'accum_var')
     return training_ops.resource_apply_adadelta(
         var.handle,
         accum_grad.handle,
         accum_var.handle,
-        math_ops.cast(self._get_hyper('learning_rate'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('rho'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('epsilon'), grad.dtype.base_dtype),
+        lr_t,
+        self._get_hyper('rho', var_dtype),
+        self._get_hyper('epsilon', var_dtype),
         grad,
         use_locking=self._use_locking)
 
   def _resource_apply_sparse(self, grad, var, indices):
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     accum_grad = self.get_slot(var, 'accum_grad')
     accum_var = self.get_slot(var, 'accum_var')
     return training_ops.resource_sparse_apply_adadelta(
         var.handle,
         accum_grad.handle,
         accum_var.handle,
-        math_ops.cast(self._get_hyper('learning_rate'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('rho'), grad.dtype.base_dtype),
-        math_ops.cast(self._get_hyper('epsilon'), grad.dtype.base_dtype),
+        lr_t,
+        self._get_hyper('rho', var_dtype),
+        self._get_hyper('epsilon', var_dtype),
         grad,
         indices,
         use_locking=self._use_locking)
@@ -122,6 +128,7 @@ class Adadelta(optimizer_v2.OptimizerV2):
     config = super(Adadelta, self).get_config()
     config.update({
         'learning_rate': self._serialize_hyperparameter('learning_rate'),
+        'decay': self._serialize_hyperparameter('decay'),
         'rho': self._serialize_hyperparameter('rho'),
         'epsilon': self._serialize_hyperparameter('epsilon'),
     })
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
index 7d090e8b84..0896f95f94 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -55,7 +54,8 @@ class Adagrad(optimizer_v2.OptimizerV2):
                learning_rate=0.001,
                initial_accumulator_value=0.1,
                epsilon=1e-7,
-               name='Adagrad'):
+               name='Adagrad',
+               **kwargs):
     """Construct a new Adagrad optimizer.
 
     Args:
@@ -66,6 +66,7 @@ class Adagrad(optimizer_v2.OptimizerV2):
         Starting value for the accumulators, must be positive.
       name: Optional name prefix for the operations created when applying
         gradients.  Defaults to "Adagrad".
+      **kwargs: keyword arguments. Allowed to be {`decay`}
 
     Raises:
       ValueError: If the `initial_accumulator_value` or `epsilon` is invalid.
@@ -82,8 +83,9 @@ class Adagrad(optimizer_v2.OptimizerV2):
                        initial_accumulator_value)
     if epsilon < 1e-7:
       raise ValueError('epsilon must be larger than 1e-7: %s' % epsilon)
-    super(Adagrad, self).__init__(name)
+    super(Adagrad, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', learning_rate)
+    self._set_hyper('decay', self._initial_decay)
     self._initial_accumulator_value = initial_accumulator_value
     self._set_hyper('epsilon', epsilon)
 
@@ -94,25 +96,16 @@ class Adagrad(optimizer_v2.OptimizerV2):
           self._initial_accumulator_value, dtype=dtype)
       self.add_slot(var, 'accumulator', init)
 
-  def _init_constant_op(self, v, dtype):
-    def init():
-      # Use a Tensor instead of initializer if variable does not have
-      # static shape.
-      init_constant = gen_array_ops.fill(array_ops.shape(v),
-                                         self._initial_accumulator_value)
-      return math_ops.cast(init_constant, dtype)
-    return init
-
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
-    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
-    epsilon = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    lr_t = self._decayed_lr(var_dtype)
+    epsilon = self._get_hyper('epsilon', var_dtype)
     acc = self.get_slot(var, 'accumulator')
 
     acc_t = state_ops.assign_add(
         acc, math_ops.square(grad), use_locking=self._use_locking)
     var_update = state_ops.assign_sub(
-        var, learning_rate * grad / (math_ops.sqrt(acc_t) + epsilon))
+        var, lr_t * grad / (math_ops.sqrt(acc_t) + epsilon))
     return var_update
 
   def _resource_apply_sparse(self, grad, var, indices):
@@ -123,21 +116,21 @@ class Adagrad(optimizer_v2.OptimizerV2):
         return x.value()
 
     var_dtype = var.dtype.base_dtype
-    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
-    epsilon = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    lr_t = self._decayed_lr(var_dtype)
+    epsilon = self._get_hyper('epsilon', var_dtype)
     acc = self.get_slot(var, 'accumulator')
 
     acc_t = _resource_scatter_add(acc, indices, math_ops.square(grad))
     acc_t_slice = array_ops.gather(acc_t, indices)
     var_update = _resource_scatter_add(
-        var, indices,
-        -learning_rate * grad / (math_ops.sqrt(acc_t_slice) + epsilon))
+        var, indices, -lr_t * grad / (math_ops.sqrt(acc_t_slice) + epsilon))
     return var_update
 
   def get_config(self):
     config = super(Adagrad, self).get_config()
     config.update({
         'learning_rate': self._serialize_hyperparameter('learning_rate'),
+        'decay': self._serialize_hyperparameter('decay'),
         'initial_accumulator_value': self._initial_accumulator_value,
         'epsilon': self._serialize_hyperparameter('epsilon'),
     })
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index 7d0f55c7d7..5ddeb1ad80 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -116,6 +116,50 @@ class AdagradOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_callable_params=True)
 
+  def testBasicWithLearningRateDecay(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = 3.0
+        decay = 0.5
+
+        ada_opt = adagrad.Adagrad(learning_rate, decay=decay)
+
+        accum0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+
+        if not context.executing_eagerly():
+          ada_update = ada_opt.apply_gradients(
+              zip([grads0, grads1], [var0, var1]))
+          self.evaluate(variables.global_variables_initializer())
+
+        # Fetch params to validate initial values
+        v0_val, v1_val = self.evaluate([var0, var1])
+        self.assertAllClose([1.0, 2.0], v0_val)
+        self.assertAllClose([3.0, 4.0], v1_val)
+
+        # Run 3 steps of adagrad
+        for t in range(3):
+          if not context.executing_eagerly():
+            self.evaluate(ada_update)
+          else:
+            ada_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+          lr_np = learning_rate / (1 + decay * t)
+          var0_np, accum0_np = adagrad_update_numpy(var0_np, accum0_np,
+                                                    grads0_np, lr_np)
+          var1_np, accum1_np = adagrad_update_numpy(var1_np, accum1_np,
+                                                    grads1_np, lr_np)
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index fd5918dbfa..a3f12909aa 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -50,7 +50,8 @@ class Adam(optimizer_v2.OptimizerV2):
                beta_2=0.999,
                epsilon=1e-7,
                amsgrad=False,
-               name='Adam'):
+               name='Adam',
+               **kwargs):
     r"""Construct a new Adam optimizer.
 
     If amsgrad = False:
@@ -122,10 +123,12 @@ class Adam(optimizer_v2.OptimizerV2):
         a callable that takes no arguments and returns the actual value to use.
         This can be useful for changing these values across different
         invocations of optimizer functions. @end_compatibility
+      **kwargs: keyword arguments. Allowed to be {`decay`}
     """
 
-    super(Adam, self).__init__(name)
+    super(Adam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', learning_rate)
+    self._set_hyper('decay', self._initial_decay)
     self._set_hyper('beta_1', beta_1)
     self._set_hyper('beta_2', beta_2)
     self._set_hyper('epsilon', epsilon)
@@ -141,12 +144,13 @@ class Adam(optimizer_v2.OptimizerV2):
       self.add_slot(var, 'v')
 
   def _resource_apply_dense(self, grad, var):
-    grad_dtype = grad.dtype.base_dtype
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     m = self.get_slot(var, 'm')
     v = self.get_slot(var, 'v')
-    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
-    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
-    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_t = self._get_hyper('beta_1', var_dtype)
+    beta_2_t = self._get_hyper('beta_2', var_dtype)
+    local_step = math_ops.cast(self.iterations + 1, var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
     beta_2_power = math_ops.pow(beta_2_t, local_step)
     return training_ops.resource_apply_adam(
@@ -155,22 +159,22 @@ class Adam(optimizer_v2.OptimizerV2):
         v.handle,
         beta_1_power,
         beta_2_power,
-        math_ops.cast(self._get_hyper('learning_rate'), grad_dtype),
+        lr_t,
         beta_1_t,
         beta_2_t,
-        math_ops.cast(self._get_hyper('epsilon'), grad_dtype),
+        self._get_hyper('epsilon', var_dtype),
         grad,
         use_locking=self._use_locking)
 
   def _resource_apply_sparse(self, grad, var, indices):
     var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
+    beta_1_t = self._get_hyper('beta_1', var_dtype)
+    beta_2_t = self._get_hyper('beta_2', var_dtype)
     local_step = math_ops.cast(self.iterations + 1, var_dtype)
-    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), var_dtype)
-    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
     beta_2_power = math_ops.pow(beta_2_t, local_step)
-    lr_t = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
-    epsilon_t = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    epsilon_t = self._get_hyper('epsilon', var_dtype)
     lr = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power))
 
     # m_t = beta1 * m + (1 - beta1) * g_t
@@ -201,6 +205,7 @@ class Adam(optimizer_v2.OptimizerV2):
     config = super(Adam, self).get_config()
     config.update({
         'learning_rate': self._serialize_hyperparameter('learning_rate'),
+        'decay': self._serialize_hyperparameter('decay'),
         'beta_1': self._serialize_hyperparameter('beta_1'),
         'beta_2': self._serialize_hyperparameter('beta_2'),
         'epsilon': self._serialize_hyperparameter('epsilon'),
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 20780ead9c..e2bc6a39f9 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -38,16 +38,16 @@ def adam_update_numpy(param,
                       t,
                       m,
                       v,
-                      alpha=0.001,
+                      lr=0.001,
                       beta1=0.9,
                       beta2=0.999,
                       epsilon=1e-7):
-  alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
+  lr_t = lr * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1))
 
   m_t = beta1 * m + (1 - beta1) * g_t
   v_t = beta2 * v + (1 - beta2) * g_t * g_t
 
-  param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon)
+  param_t = param - lr_t * m_t / (np.sqrt(v_t) + epsilon)
   return param_t, m_t, v_t
 
 
@@ -90,13 +90,13 @@ class AdamOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 3.0, 4.0], self.evaluate(var1))
 
-        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
-
+        beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
         # Run 3 steps of Adam
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
-          self.assertAllCloseAccordingToType(0.999**t,
-                                             self.evaluate(beta2_power))
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta_2_power))
           update.run()
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
@@ -177,21 +177,21 @@ class AdamOptimizerTest(test.TestCase):
           epsilon = epsilon()
 
         opt = adam.Adam(learning_rate=learning_rate)
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        if not context.executing_eagerly():
+          update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
         self.evaluate(variables.global_variables_initializer())
         # Run 3 steps of Adam
-        for t in range(1, 4):
-          if not context.executing_eagerly():
-            self.evaluate(update)
-          elif t > 1:
-            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-
+        for t in range(3):
           beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
           self.assertAllCloseAccordingToType(0.9**(t + 1),
                                              self.evaluate(beta_1_power))
           self.assertAllCloseAccordingToType(0.999**(t + 1),
                                              self.evaluate(beta_2_power))
+          if not context.executing_eagerly():
+            self.evaluate(update)
+          else:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
           var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
@@ -208,6 +208,52 @@ class AdamOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_callable_params=True)
 
+  def testBasicWithLearningRateDecay(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = 0.001
+        beta_1 = 0.9
+        beta_2 = 0.999
+        epsilon = 1e-7
+        decay = 0.5
+
+        opt = adam.Adam(
+            learning_rate=learning_rate,
+            beta_1=beta_1,
+            beta_2=beta_2,
+            epsilon=epsilon,
+            decay=decay)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+        self.evaluate(variables.global_variables_initializer())
+        # Run 3 steps of Adam
+        for t in range(3):
+          self.evaluate(update)
+          lr_np = learning_rate / (1 + decay * t)
+
+          var0_np, m0, v0 = adam_update_numpy(
+              var0_np, grads0_np, t, m0, v0, lr=lr_np)
+          var1_np, m1, v1 = adam_update_numpy(
+              var1_np, grads1_np, t, m1, v1, lr=lr_np)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -230,13 +276,13 @@ class AdamOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
-
+        beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
         # Run 3 steps of Adam
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
-          self.assertAllCloseAccordingToType(0.999**t,
-                                             self.evaluate(beta2_power))
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta_2_power))
           update.run()
 
           var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
@@ -265,17 +311,18 @@ class AdamOptimizerTest(test.TestCase):
         update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+        beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
 
         # Fetch params to validate initial values
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 3 steps of intertwined Adam1 and Adam2.
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, self.evaluate(beta1_power))
-          self.assertAllCloseAccordingToType(0.999**t,
-                                             self.evaluate(beta2_power))
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta_2_power))
           if t % 2 == 0:
             update1.run()
           else:
@@ -296,7 +343,7 @@ class AdamOptimizerTest(test.TestCase):
       opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
       # There should be iteration, hyper variables, and two unique slot
       # variables for v1 and v2 respectively.
-      self.assertEqual(9, len(set(opt.variables())))
+      self.assertEqual(10, len(set(opt.variables())))
 
   def testAmsgradWithError(self):
     with self.assertRaisesRegexp(ValueError,
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 67b678f862..ddd78584f8 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -44,7 +44,8 @@ class Adamax(adam.Adam):
                beta_1=0.9,
                beta_2=0.999,
                epsilon=1e-7,
-               name='Adamax'):
+               name='Adamax',
+               **kwargs):
     """Construct a new Adamax optimizer.
 
     Initialization:
@@ -87,6 +88,7 @@ class Adamax(adam.Adam):
       epsilon: A small constant for numerical stability.
       name: Optional name for the operations created when applying gradients.
         Defaults to "Adamax".
+      **kwargs: keyword arguments. Allowed to be {`decay`}
     """
     # pylint: disable=useless-super-delegation
     super(Adamax, self).__init__(
@@ -95,38 +97,40 @@ class Adamax(adam.Adam):
         beta_2=beta_2,
         epsilon=epsilon,
         amsgrad=False,
-        name=name)
+        name=name,
+        **kwargs)
     # pylint: enable=useless-super-delegation
 
   def _resource_apply_dense(self, grad, var):
-    grad_dtype = grad.dtype.base_dtype
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     m = self.get_slot(var, 'm')
     v = self.get_slot(var, 'v')
-    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
-    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
-    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_t = self._get_hyper('beta_1', var_dtype)
+    beta_2_t = self._get_hyper('beta_2', var_dtype)
+    local_step = math_ops.cast(self.iterations + 1, var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
     return training_ops.resource_apply_ada_max(
         var.handle,
         m.handle,
         v.handle,
         beta_1_power,
-        math_ops.cast(self._get_hyper('learning_rate'), grad_dtype),
+        lr_t,
         beta_1_t,
         beta_2_t,
-        math_ops.cast(self._get_hyper('epsilon'), grad_dtype),
+        self._get_hyper('epsilon', var_dtype),
         grad,
         use_locking=self._use_locking)
 
   def _resource_apply_sparse(self, grad, var, indices):
-    grad_dtype = grad.dtype.base_dtype
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
 
-    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
-    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
-    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_t = self._get_hyper('beta_1', var_dtype)
+    beta_2_t = self._get_hyper('beta_2', var_dtype)
+    local_step = math_ops.cast(self.iterations + 1, var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
-    lr_t = math_ops.cast(self._get_hyper('learning_rate'), grad_dtype)
-    epsilon_t = math_ops.cast(self._get_hyper('epsilon'), grad_dtype)
+    epsilon_t = self._get_hyper('epsilon', var_dtype)
 
     # m_t = beta1 * m + (1 - beta1) * g_t
     m = self.get_slot(var, 'm')
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index c6b45ccbe9..aa215b0faf 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -44,7 +44,7 @@ def adamax_update_numpy(param,
                         epsilon=1e-8):
   m_t = beta1 * m + (1 - beta1) * g_t
   v_t = np.maximum(beta2 * v, np.abs(g_t))
-  param_t = param - (alpha / (1 - beta1**t)) * (m_t / (v_t + epsilon))
+  param_t = param - (alpha / (1 - beta1**(t + 1))) * (m_t / (v_t + epsilon))
   return param_t, m_t, v_t
 
 
@@ -61,8 +61,8 @@ def adamax_sparse_update_numpy(param,
   m_t, v_t, param_t = np.copy(m), np.copy(v), np.copy(param)
   m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t
   v_t_slice = np.maximum(beta2 * v[indices], np.abs(g_t))
-  param_t_slice = param[indices] - ((alpha / (1 - beta1**t)) *
-                                    (m_t_slice / (v_t_slice + epsilon)))
+  param_t_slice = param[indices] - (
+      (alpha / (1 - beta1**(t + 1))) * (m_t_slice / (v_t_slice + epsilon)))
   m_t[indices] = m_t_slice
   v_t[indices] = v_t_slice
   param_t[indices] = param_t_slice
@@ -111,8 +111,8 @@ class AdamaxOptimizerTest(test.TestCase):
         beta1_power = get_beta_accumulators(opt, dtype)
 
         # Run 3 steps of Adamax
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
           update.run()
 
           var0_np, m0, v0 = adamax_sparse_update_numpy(
@@ -190,7 +190,8 @@ class AdamaxOptimizerTest(test.TestCase):
         grads1 = constant_op.constant(grads1_np)
 
         opt = adamax.Adamax()
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        if not context.executing_eagerly():
+          update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
         if not context.executing_eagerly():
           self.evaluate(variables.global_variables_initializer())
@@ -199,18 +200,71 @@ class AdamaxOptimizerTest(test.TestCase):
           self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
         # Run 3 steps of Adamax
-        for t in range(1, 4):
+        for t in range(3):
+          beta_1_power = get_beta_accumulators(opt, dtype)
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
           if not context.executing_eagerly():
             self.evaluate(update)
-          elif t > 1:
+          else:
             opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
+          var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(
+              var0_np, self.evaluate(var0), rtol=1e-2)
+          self.assertAllCloseAccordingToType(
+              var1_np, self.evaluate(var1), rtol=1e-2)
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testBasicWithLearningRateDecay(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        learning_rate = 0.001
+        decay = 0.002
+        opt = adamax.Adamax(learning_rate=learning_rate, decay=decay)
+        if not context.executing_eagerly():
+          update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+        if not context.executing_eagerly():
+          self.evaluate(variables.global_variables_initializer())
+          # Fetch params to validate initial values
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+        # Run 3 steps of Adamax
+        for t in range(3):
           beta_1_power = get_beta_accumulators(opt, dtype)
           self.assertAllCloseAccordingToType(0.9**(t + 1),
                                              self.evaluate(beta_1_power))
+          if not context.executing_eagerly():
+            self.evaluate(update)
+          else:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
-          var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
-          var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1)
+          lr = learning_rate / (1 + decay * t)
+
+          var0_np, m0, v0 = adamax_update_numpy(
+              var0_np, grads0_np, t, m0, v0, alpha=lr)
+          var1_np, m1, v1 = adamax_update_numpy(
+              var1_np, grads1_np, t, m1, v1, alpha=lr)
 
           # Validate updated params
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0),
@@ -243,8 +297,8 @@ class AdamaxOptimizerTest(test.TestCase):
         beta1_power = get_beta_accumulators(opt, dtype)
 
         # Run 3 steps of Adamax
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
           update.run()
 
           var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0)
@@ -280,8 +334,8 @@ class AdamaxOptimizerTest(test.TestCase):
         self.assertAllClose([3.0, 4.0], var1.eval())
 
         # Run 3 steps of intertwined Adamax1 and Adamax2.
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
           if t % 2 == 0:
             update1.run()
           else:
@@ -302,7 +356,7 @@ class AdamaxOptimizerTest(test.TestCase):
       opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
       # There should be iteration, hyper variables, and two unique slot
       # variables for v1 and v2 respectively.
-      self.assertEqual(9, len(set(opt.variables())))
+      self.assertEqual(10, len(set(opt.variables())))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl.py b/tensorflow/python/keras/optimizer_v2/ftrl.py
index 2faf65eab3..e278e352f5 100644
--- a/tensorflow/python/keras/optimizer_v2/ftrl.py
+++ b/tensorflow/python/keras/optimizer_v2/ftrl.py
@@ -40,7 +40,8 @@ class Ftrl(optimizer_v2.OptimizerV2):
                l1_regularization_strength=0.0,
                l2_regularization_strength=0.0,
                name='Ftrl',
-               l2_shrinkage_regularization_strength=0.0):
+               l2_shrinkage_regularization_strength=0.0,
+               **kwargs):
     r"""Construct a new FTRL optimizer.
 
     Args:
@@ -68,7 +69,8 @@ class Ftrl(optimizer_v2.OptimizerV2):
         w_{t+1} = w_t - lr_t / (1 + 2*L2*lr_t) * g_t -
                   2*L2_shrinkage*lr_t / (1 + 2*L2*lr_t) * w_t
         where lr_t is the learning rate at t.
-        When input is sparse shrinkage will only happen on the active weights.
+        When input is sparse shrinkage will only happen on the active weights.\
+      **kwargs: keyword arguments. Allowed to be {`decay`}
 
     Raises:
       ValueError: If one of the arguments is invalid.
@@ -77,7 +79,7 @@ class Ftrl(optimizer_v2.OptimizerV2):
       See [paper]
         (https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf)
     """
-    super(Ftrl, self).__init__(name)
+    super(Ftrl, self).__init__(name, **kwargs)
 
     if initial_accumulator_value < 0.0:
       raise ValueError(
@@ -100,6 +102,7 @@ class Ftrl(optimizer_v2.OptimizerV2):
           ' or zero' % l2_shrinkage_regularization_strength)
 
     self._set_hyper('learning_rate', learning_rate)
+    self._set_hyper('decay', self._initial_decay)
     self._set_hyper('learning_rate_power', learning_rate_power)
     self._set_hyper('l1_regularization_strength', l1_regularization_strength)
     self._set_hyper('l2_regularization_strength', l2_regularization_strength)
@@ -118,13 +121,12 @@ class Ftrl(optimizer_v2.OptimizerV2):
 
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
-    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
-    learning_rate_power = math_ops.cast(
-        self._get_hyper('learning_rate_power'), var_dtype)
-    l1_regularization_strength = math_ops.cast(
-        self._get_hyper('l1_regularization_strength'), var_dtype)
-    l2_regularization_strength = math_ops.cast(
-        self._get_hyper('l2_regularization_strength'), var_dtype)
+    lr_t = self._decayed_lr(var_dtype)
+    learning_rate_power = self._get_hyper('learning_rate_power', var_dtype)
+    l1_regularization_strength = self._get_hyper('l1_regularization_strength',
+                                                 var_dtype)
+    l2_regularization_strength = self._get_hyper('l2_regularization_strength',
+                                                 var_dtype)
     accum = self.get_slot(var, 'accumulator')
     linear = self.get_slot(var, 'linear')
     if self._l2_shrinkage_regularization_strength <= 0.0:
@@ -133,7 +135,7 @@ class Ftrl(optimizer_v2.OptimizerV2):
           accum.handle,
           linear.handle,
           grad,
-          learning_rate,
+          lr_t,
           l1_regularization_strength,
           l2_regularization_strength,
           learning_rate_power,
@@ -144,7 +146,7 @@ class Ftrl(optimizer_v2.OptimizerV2):
           accum.handle,
           linear.handle,
           grad,
-          learning_rate,
+          lr_t,
           l1_regularization_strength,
           l2_regularization_strength,
           math_ops.cast(self._l2_shrinkage_regularization_strength, var_dtype),
@@ -153,13 +155,12 @@ class Ftrl(optimizer_v2.OptimizerV2):
 
   def _resource_apply_sparse(self, grad, var, indices):
     var_dtype = var.dtype.base_dtype
-    learning_rate = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
-    learning_rate_power = math_ops.cast(
-        self._get_hyper('learning_rate_power'), var_dtype)
-    l1_regularization_strength = math_ops.cast(
-        self._get_hyper('l1_regularization_strength'), var_dtype)
-    l2_regularization_strength = math_ops.cast(
-        self._get_hyper('l2_regularization_strength'), var_dtype)
+    lr_t = self._decayed_lr(var_dtype)
+    learning_rate_power = self._get_hyper('learning_rate_power', var_dtype)
+    l1_regularization_strength = self._get_hyper('l1_regularization_strength',
+                                                 var_dtype)
+    l2_regularization_strength = self._get_hyper('l2_regularization_strength',
+                                                 var_dtype)
     accum = self.get_slot(var, 'accumulator')
     linear = self.get_slot(var, 'linear')
     if self._l2_shrinkage_regularization_strength <= 0.0:
@@ -169,7 +170,7 @@ class Ftrl(optimizer_v2.OptimizerV2):
           linear.handle,
           grad,
           indices,
-          learning_rate,
+          lr_t,
           l1_regularization_strength,
           l2_regularization_strength,
           learning_rate_power,
@@ -181,7 +182,7 @@ class Ftrl(optimizer_v2.OptimizerV2):
           linear.handle,
           grad,
           indices,
-          learning_rate,
+          lr_t,
           l1_regularization_strength,
           l2_regularization_strength,
           math_ops.cast(self._l2_shrinkage_regularization_strength, var_dtype),
@@ -193,6 +194,8 @@ class Ftrl(optimizer_v2.OptimizerV2):
     config.update({
         'learning_rate':
             self._serialize_hyperparameter('learning_rate'),
+        'decay':
+            self._serialize_hyperparameter('decay'),
         'initial_accumulator_value':
             self._initial_accumulator_value,
         'learning_rate_power':
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent.py b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
index 90106c941c..03e4515e02 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
@@ -19,7 +19,6 @@ from __future__ import print_function
 
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.training import training_ops
 
@@ -62,7 +61,8 @@ class SGD(optimizer_v2.OptimizerV2):
                learning_rate=0.001,
                momentum=0.0,
                nesterov=False,
-               name="SGD"):
+               name="SGD",
+               **kwargs):
     """Construct a new Stochastic Gradient Descent or Momentum optimizer.
 
     Arguments:
@@ -72,9 +72,11 @@ class SGD(optimizer_v2.OptimizerV2):
       nesterov: boolean. Whether to apply Nesterov momentum.
       name: Optional name prefix for the operations created when applying
         gradients.  Defaults to 'SGD'.
+      **kwargs: keyword arguments. Allowed to be {`decay`}
     """
-    super(SGD, self).__init__(name)
+    super(SGD, self).__init__(name, **kwargs)
     self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("decay", self._initial_decay)
 
     self._momentum = False
     if isinstance(momentum, ops.Tensor) or callable(momentum) or momentum > 0:
@@ -91,44 +93,44 @@ class SGD(optimizer_v2.OptimizerV2):
         self.add_slot(var, "momentum")
 
   def _resource_apply_dense(self, grad, var):
-    learning_rate = self._get_hyper("learning_rate")
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     if self._momentum:
       momentum_var = self.get_slot(var, "momentum")
       return training_ops.resource_apply_momentum(
           var.handle,
           momentum_var.handle,
-          math_ops.cast(learning_rate, grad.dtype.base_dtype),
+          lr_t,
           grad,
-          math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype),
+          self._get_hyper("momentum", var_dtype),
           use_locking=self._use_locking,
           use_nesterov=self._nesterov)
     else:
       return training_ops.resource_apply_gradient_descent(
-          var.handle,
-          math_ops.cast(learning_rate, grad.dtype.base_dtype),
-          grad,
-          use_locking=self._use_locking)
+          var.handle, lr_t, grad, use_locking=self._use_locking)
 
   def _resource_apply_sparse_duplicate_indices(self, grad, var, indices):
     if self._momentum:
       return super(SGD, self)._resource_apply_sparse_duplicate_indices(
           grad, var, indices)
     else:
-      return resource_variable_ops.resource_scatter_add(
-          var.handle, indices, -grad * math_ops.cast(
-              self._get_hyper("learning_rate"), grad.dtype.base_dtype))
+      var_dtype = var.dtype.base_dtype
+      lr_t = self._decayed_lr(var_dtype)
+      return resource_variable_ops.resource_scatter_add(var.handle, indices,
+                                                        -grad * lr_t)
 
   def _resource_apply_sparse(self, grad, var, indices):
     # This method is only needed for momentum optimization.
-    learning_rate = self._get_hyper("learning_rate")
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     momentum_var = self.get_slot(var, "momentum")
     return training_ops.resource_sparse_apply_momentum(
         var.handle,
         momentum_var.handle,
-        math_ops.cast(learning_rate, grad.dtype.base_dtype),
+        lr_t,
         grad,
         indices,
-        math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype),
+        self._get_hyper("momentum", var_dtype),
         use_locking=self._use_locking,
         use_nesterov=self._nesterov)
 
@@ -136,6 +138,7 @@ class SGD(optimizer_v2.OptimizerV2):
     config = super(SGD, self).get_config()
     config.update({
         "learning_rate": self._serialize_hyperparameter("learning_rate"),
+        "decay": self._serialize_hyperparameter("decay"),
         "momentum": self._serialize_hyperparameter("momentum"),
         "nesterov": self._nesterov,
     })
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index fa7cca1420..348d2728c8 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -47,7 +47,6 @@ class GradientDescentOptimizerTest(test.TestCase):
         grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
         grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
         sgd = gradient_descent.SGD(3.0)
-        # self.assertFalse(sgd._initial_decay)
         sgd_op = sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
         self.evaluate(variables.global_variables_initializer())
         # Run 1 step of sgd
@@ -58,6 +57,43 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
                                            self.evaluate(var1))
 
+  @test_util.run_in_graph_and_eager_modes
+  def testBasicWithLearningRateDecay(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
+        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        learning_rate = 3.0
+        decay = 0.5
+        sgd = gradient_descent.SGD(learning_rate=learning_rate, decay=decay)
+        if not context.executing_eagerly():
+          sgd_op = sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        self.evaluate(variables.global_variables_initializer())
+        # Run 2 steps of sgd
+        if not context.executing_eagerly():
+          self.evaluate(sgd_op)
+        else:
+          sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        # Validate updated params
+        self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
+                                           self.evaluate(var0))
+        self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
+                                           self.evaluate(var1))
+
+        if not context.executing_eagerly():
+          self.evaluate(sgd_op)
+        else:
+          sgd.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [1.0 - 3.0 * 0.1 - 2.0 * 0.1, 2.0 - 3.0 * 0.1 - 2.0 * 0.1],
+            self.evaluate(var0))
+        self.assertAllCloseAccordingToType(
+            [3.0 - 3.0 * 0.01 - 2.0 * 0.01, 4.0 - 3.0 * 0.01 - 2.0 * 0.01],
+            self.evaluate(var1))
+
   @test_util.run_in_graph_and_eager_modes
   def testBasicCallableParams(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -170,6 +206,36 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]],
                                            self.evaluate(var1))
 
+  def testSparseBasicWithLearningRateDecay(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        var0 = variables.Variable([[1.0], [2.0]], dtype=dtype)
+        var1 = variables.Variable([[3.0], [4.0]], dtype=dtype)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant([0.1], shape=[1, 1], dtype=dtype),
+            constant_op.constant([0]), constant_op.constant([2, 1]))
+        grads1 = ops.IndexedSlices(
+            constant_op.constant([0.01], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]), constant_op.constant([2, 1]))
+        sgd_op = gradient_descent.SGD(
+            3.0, decay=0.5).apply_gradients(
+                zip([grads0, grads1], [var0, var1]))
+        self.evaluate(variables.global_variables_initializer())
+        # Run 2 steps of sgd
+        self.evaluate(sgd_op)
+        # Validate updated params
+        self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]],
+                                           self.evaluate(var0))
+        self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]],
+                                           self.evaluate(var1))
+
+        self.evaluate(sgd_op)
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[1.0 - 3.0 * 0.1 - 2.0 * 0.1], [2.0]], self.evaluate(var0))
+        self.assertAllCloseAccordingToType(
+            [[3.0], [4.0 - 3.0 * 0.01 - 2.0 * 0.01]], self.evaluate(var1))
+
   def testCapturingInDefunWhileExecutingEagerly(self):
     with context.eager_mode():
       optimizer = gradient_descent.SGD(1.0)
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 4be421a73f..00b095e0dc 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -53,13 +53,46 @@ class Nadam(adam.Adam):
     See [Dozat, T., 2015](http://cs229.stanford.edu/proj2015/054_report.pdf).
   """
 
+  def __init__(self,
+               learning_rate=0.001,
+               beta_1=0.9,
+               beta_2=0.999,
+               epsilon=1e-7,
+               name='Nadam',
+               **kwargs):
+    """Construct a new Nadam optimizer.
+
+    Args:
+      learning_rate: A Tensor or a floating point value.  The learning rate.
+      beta_1: A float value or a constant float tensor. The exponential decay
+        rate for the 1st moment estimates.
+      beta_2: A float value or a constant float tensor. The exponential decay
+        rate for the exponentially weighted infinity norm.
+      epsilon: A small constant for numerical stability.
+      name: Optional name for the operations created when applying gradients.
+        Defaults to "Adamax".
+      **kwargs: keyword arguments. Allowed to be {`decay`}
+    """
+
+    # pylint: disable=useless-super-delegation
+    super(Nadam, self).__init__(
+        learning_rate=learning_rate,
+        beta_1=beta_1,
+        beta_2=beta_2,
+        epsilon=epsilon,
+        amsgrad=False,
+        name=name,
+        **kwargs)
+    # pylint: enable=useless-super-delegation
+
   def _resource_apply_dense(self, grad, var):
-    grad_dtype = grad.dtype.base_dtype
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     m = self.get_slot(var, 'm')
     v = self.get_slot(var, 'v')
-    local_step = math_ops.cast(self.iterations + 1, grad_dtype)
-    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), grad_dtype)
-    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), grad_dtype)
+    beta_1_t = self._get_hyper('beta_1', var_dtype)
+    beta_2_t = self._get_hyper('beta_2', var_dtype)
+    local_step = math_ops.cast(self.iterations + 1, var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
     beta_2_power = math_ops.pow(beta_2_t, local_step)
     return training_ops.resource_apply_adam(
@@ -68,23 +101,23 @@ class Nadam(adam.Adam):
         v.handle,
         beta_1_power,
         beta_2_power,
-        math_ops.cast(self._get_hyper('learning_rate'), grad_dtype),
+        lr_t,
         beta_1_t,
         beta_2_t,
-        math_ops.cast(self._get_hyper('epsilon'), grad_dtype),
+        self._get_hyper('epsilon', var_dtype),
         grad,
         use_locking=self._use_locking,
         use_nesterov=True)
 
   def _resource_apply_sparse(self, grad, var, indices):
     var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
+    beta_1_t = self._get_hyper('beta_1', var_dtype)
+    beta_2_t = self._get_hyper('beta_2', var_dtype)
     local_step = math_ops.cast(self.iterations + 1, var_dtype)
-    beta_1_t = math_ops.cast(self._get_hyper('beta_1'), var_dtype)
-    beta_2_t = math_ops.cast(self._get_hyper('beta_2'), var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
     beta_2_power = math_ops.pow(beta_2_t, local_step)
-    lr_t = math_ops.cast(self._get_hyper('learning_rate'), var_dtype)
-    epsilon_t = math_ops.cast(self._get_hyper('epsilon'), var_dtype)
+    epsilon_t = self._get_hyper('epsilon', var_dtype)
     lr = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power))
 
     # m_t = beta1 * m + (1 - beta1) * g_t
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
index 9cc81b1d11..b7132bbea7 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -48,7 +48,7 @@ def nadam_update_numpy(param,
                        beta1=0.9,
                        beta2=0.999,
                        epsilon=1e-8):
-  alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
+  alpha_t = alpha * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1))
 
   m_t = beta1 * m + (1 - beta1) * g_t
   v_t = beta2 * v + (1 - beta2) * g_t * g_t
@@ -97,9 +97,9 @@ class NadamOptimizerTest(test.TestCase):
         beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
 
         # Run 3 steps of Nadam
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power.eval())
           update.run()
 
           var0_np, m0, v0 = nadam_update_numpy(
@@ -146,9 +146,9 @@ class NadamOptimizerTest(test.TestCase):
         beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
 
         # Run 3 steps of Nadam
-        for t in range(1, 4):
-          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
-          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power.eval())
           update.run()
 
           var0_np, m0, v0 = nadam_update_numpy(var0_np, grads0_np, t, m0, v0)
@@ -158,12 +158,51 @@ class NadamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, var0.eval())
           self.assertAllCloseAccordingToType(var1_np, var1.eval())
 
-  def testBasic(self):
-    self.doTestBasic(use_resource=False)
-
   def testResourceBasic(self):
     self.doTestBasic(use_resource=True)
 
+  def testBasicWithLearningRateDecay(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        learning_rate = 0.001
+        decay = 0.5
+        opt = nadam.Nadam(learning_rate=learning_rate, decay=decay)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = get_beta_accumulators(opt, dtype)
+
+        # Run 3 steps of Nadam
+        for t in range(3):
+          self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power.eval())
+          update.run()
+
+          lr = learning_rate / (1 + decay * t)
+          var0_np, m0, v0 = nadam_update_numpy(
+              var0_np, grads0_np, t, m0, v0, alpha=lr)
+          var1_np, m1, v1 = nadam_update_numpy(
+              var1_np, grads1_np, t, m1, v1, alpha=lr)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index fa7cfa5b8a..ee0076703f 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -33,6 +33,7 @@ from tensorflow.python.keras import backend
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import distribution_strategy_context
@@ -114,7 +115,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
 
   """
 
-  def __init__(self, name):
+  def __init__(self, name, **kwargs):
     """Create a new Optimizer.
 
     This must be called by the constructors of subclasses.
@@ -128,6 +129,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
     Args:
       name: A non-empty string.  The name to use for accumulators created
         for the optimizer.
+      **kwargs: keyword arguments. Allowed to be {`decay`}
 
     Raises:
       ValueError: If name is malformed.
@@ -140,6 +142,12 @@ class OptimizerV2(optimizer_v1.Optimizer):
     # dict: {variable name : {slot name : variable}}
     self._slots = {}
     self._weights = []
+
+    decay = kwargs.pop("decay", 0.0)
+    if decay < 0.:
+      raise ValueError("decay cannot be less than 0: {}".format(decay))
+    self._initial_decay = decay
+
     self._prepared = False
 
   def minimize(self,
@@ -345,9 +353,14 @@ class OptimizerV2(optimizer_v1.Optimizer):
       else:
         backend.set_value(self._hyper[name], value)
 
-  def _get_hyper(self, name):
+  def _get_hyper(self, name, dtype=None):
     value = self._hyper[name]
-    return self._call_if_callable(value)
+    if callable(value):
+      value = value()
+    if dtype:
+      return math_ops.cast(value, dtype)
+    else:
+      return value
 
   def __getattribute__(self, name):
     """Overridden to support hyperparameter access."""
@@ -422,6 +435,15 @@ class OptimizerV2(optimizer_v1.Optimizer):
       self._prepare()
     return self._iterations
 
+  def _decayed_lr(self, var_dtype):
+    """Get decayed learning rate as a Tensor with dtype=var_dtype."""
+    lr_t = self._get_hyper("learning_rate", var_dtype)
+    if self._initial_decay > 0.:
+      local_step = math_ops.cast(self.iterations, var_dtype)
+      decay_t = self._get_hyper("decay", var_dtype)
+      lr_t = lr_t / (1. + decay_t * local_step)
+    return lr_t
+
   @abc.abstractmethod
   def get_config(self):
     """Returns the config of the optimimizer.
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index eae5620349..6a5b334fc4 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -19,7 +19,6 @@ from __future__ import print_function
 
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
-from tensorflow.python.ops import math_ops
 from tensorflow.python.training import training_ops
 
 
@@ -58,7 +57,8 @@ class RMSprop(optimizer_v2.OptimizerV2):
                momentum=0.0,
                epsilon=1e-7,
                centered=False,
-               name="RMSprop"):
+               name="RMSprop",
+               **kwargs):
     """Construct a new RMSprop optimizer.
 
     Note that in the dense implementation of this algorithm, variables and their
@@ -88,9 +88,11 @@ class RMSprop(optimizer_v2.OptimizerV2):
         `epsilon` can each be a callable that takes no arguments and returns the
         actual value to use. This can be useful for changing these values across
         different invocations of optimizer functions. @end_compatibility
+      **kwargs: keyword arguments. Allowed to be {`decay`}
     """
-    super(RMSprop, self).__init__(name)
+    super(RMSprop, self).__init__(name, **kwargs)
     self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("decay", self._initial_decay)
     self._set_hyper("rho", rho)
 
     self._momentum = False
@@ -111,13 +113,13 @@ class RMSprop(optimizer_v2.OptimizerV2):
         self.add_slot(var, "mg")
 
   def _resource_apply_dense(self, grad, var):
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     rms = self.get_slot(var, "rms")
     mom = self.get_slot(var, "momentum")
-    learning_rate = math_ops.cast(
-        self._get_hyper("learning_rate"), grad.dtype.base_dtype)
-    rho = math_ops.cast(self._get_hyper("rho"), grad.dtype.base_dtype)
-    momentum = math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype)
-    epsilon = math_ops.cast(self._get_hyper("epsilon"), grad.dtype.base_dtype)
+    rho = self._get_hyper("rho", var_dtype)
+    momentum = self._get_hyper("momentum", var_dtype)
+    epsilon = self._get_hyper("epsilon", var_dtype)
     if self._centered:
       mg = self.get_slot(var, "mg")
       return training_ops.resource_apply_centered_rms_prop(
@@ -125,7 +127,7 @@ class RMSprop(optimizer_v2.OptimizerV2):
           mg.handle,
           rms.handle,
           mom.handle,
-          learning_rate,
+          lr_t,
           rho,
           momentum,
           epsilon,
@@ -136,7 +138,7 @@ class RMSprop(optimizer_v2.OptimizerV2):
           var.handle,
           rms.handle,
           mom.handle,
-          learning_rate,
+          lr_t,
           rho,
           momentum,
           epsilon,
@@ -144,13 +146,13 @@ class RMSprop(optimizer_v2.OptimizerV2):
           use_locking=self._use_locking)
 
   def _resource_apply_sparse(self, grad, var, indices):
+    var_dtype = var.dtype.base_dtype
+    lr_t = self._decayed_lr(var_dtype)
     rms = self.get_slot(var, "rms")
     mom = self.get_slot(var, "momentum")
-    learning_rate = math_ops.cast(
-        self._get_hyper("learning_rate"), grad.dtype.base_dtype)
-    rho = math_ops.cast(self._get_hyper("rho"), grad.dtype.base_dtype)
-    momentum = math_ops.cast(self._get_hyper("momentum"), grad.dtype.base_dtype)
-    epsilon = math_ops.cast(self._get_hyper("epsilon"), grad.dtype.base_dtype)
+    rho = self._get_hyper("rho", var_dtype)
+    momentum = self._get_hyper("momentum", var_dtype)
+    epsilon = self._get_hyper("epsilon", var_dtype)
     if self._centered:
       mg = self.get_slot(var, "mg")
       return training_ops.resource_sparse_apply_centered_rms_prop(
@@ -158,7 +160,7 @@ class RMSprop(optimizer_v2.OptimizerV2):
           mg.handle,
           rms.handle,
           mom.handle,
-          learning_rate,
+          lr_t,
           rho,
           momentum,
           epsilon,
@@ -170,7 +172,7 @@ class RMSprop(optimizer_v2.OptimizerV2):
           var.handle,
           rms.handle,
           mom.handle,
-          learning_rate,
+          lr_t,
           rho,
           momentum,
           epsilon,
@@ -182,6 +184,7 @@ class RMSprop(optimizer_v2.OptimizerV2):
     config = super(RMSprop, self).get_config()
     config.update({
         "learning_rate": self._serialize_hyperparameter("learning_rate"),
+        "decay": self._serialize_hyperparameter("decay"),
         "rho": self._serialize_hyperparameter("rho"),
         "momentum": self._serialize_hyperparameter("momentum"),
         "epsilon": self._serialize_hyperparameter("epsilon"),
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 62b64d5cf9..a320cc0c49 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -158,6 +158,73 @@ class RMSpropOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  def testDenseWithLearningRateDecay(self):
+    var0_np = np.array([1.0, 2.0])
+    grads0_np = np.array([0.1, 0.2])
+    var1_np = np.array([3.0, 4.0])
+    grads1_np = np.array([0.01, 0.2])
+
+    var0 = resource_variable_ops.ResourceVariable(var0_np)
+    var1 = resource_variable_ops.ResourceVariable(var1_np)
+    grads0 = constant_op.constant(grads0_np)
+    grads1 = constant_op.constant(grads1_np)
+    learning_rate = 0.01
+    rho = 0.9
+    momentum = 0.0
+    epsilon = 1e-7
+    centered = False
+    decay = 0.5
+    opt = rmsprop.RMSprop(
+        learning_rate=learning_rate,
+        rho=rho,
+        momentum=momentum,
+        epsilon=epsilon,
+        centered=centered,
+        decay=decay)
+
+    update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+    self.evaluate(variables.global_variables_initializer())
+
+    rms0 = opt.get_slot(var0, "rms")
+    self.assertTrue(rms0 is not None)
+    rms1 = opt.get_slot(var1, "rms")
+    self.assertTrue(rms1 is not None)
+    mom0 = opt.get_slot(var0, "momentum")
+    self.assertTrue(mom0 is not None)
+    mom1 = opt.get_slot(var1, "momentum")
+    self.assertTrue(mom1 is not None)
+
+    mg0_np = np.array([0.0, 0.0])
+    mg1_np = np.array([0.0, 0.0])
+    rms0_np = np.array([0.0, 0.0])
+    rms1_np = np.array([0.0, 0.0])
+    mom0_np = np.array([0.0, 0.0])
+    mom1_np = np.array([0.0, 0.0])
+
+    # Fetch params to validate initial values
+    self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+    self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+    # Run 4 steps of RMSprop
+    for t in range(2):
+      self.evaluate(update)
+
+      lr = learning_rate / (1 + decay * t)
+      var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
+          var0_np, grads0_np, mg0_np, rms0_np, mom0_np, lr, rho, momentum,
+          epsilon, centered)
+      var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
+          var1_np, grads1_np, mg1_np, rms1_np, mom1_np, lr, rho, momentum,
+          epsilon, centered)
+
+      # Validate updated params
+      self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
+      self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
+      self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+      self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+      self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+      self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
-- 
GitLab


From cb74272050f4563f8eddee851470986ba6662d91 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 26 Nov 2018 17:32:18 -0800
Subject: [PATCH 0835/1554] Add `evaluate` to Benchmark to be compatible with
 TestCase

PiperOrigin-RevId: 222915339
---
 tensorflow/python/platform/benchmark.py             | 13 +++++++++++++
 .../api/golden/v1/tensorflow.test.-benchmark.pbtxt  |  4 ++++
 .../api/golden/v2/tensorflow.test.-benchmark.pbtxt  |  4 ++++
 3 files changed, 21 insertions(+)

diff --git a/tensorflow/python/platform/benchmark.py b/tensorflow/python/platform/benchmark.py
index 4f7abb311a..d6773d7b81 100644
--- a/tensorflow/python/platform/benchmark.py
+++ b/tensorflow/python/platform/benchmark.py
@@ -30,6 +30,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.core.util import test_log_pb2
 from tensorflow.python.client import timeline
+from tensorflow.python.framework import ops
 from tensorflow.python.platform import app
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
@@ -299,6 +300,18 @@ class TensorFlowBenchmark(Benchmark):
     benchmark_values["extras"].update(unreported_extras)
     return benchmark_values
 
+  def evaluate(self, tensors):
+    """Evaluates tensors and returns numpy values.
+
+    Args:
+      tensors: A Tensor or a nested list/tuple of Tensors.
+
+    Returns:
+      tensors numpy values.
+    """
+    sess = ops.get_default_session() or self.cached_session()
+    return sess.run(tensors)
+
 
 def _run_benchmarks(regex):
   """Run benchmarks that match regex `regex`.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt
index df528e26b6..6fc489c860 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.test.-benchmark.pbtxt
@@ -6,6 +6,10 @@ tf_class {
   member_method {
     name: "__init__"
   }
+  member_method {
+    name: "evaluate"
+    argspec: "args=[\'self\', \'tensors\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "is_abstract"
     argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt
index df528e26b6..6fc489c860 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.test.-benchmark.pbtxt
@@ -6,6 +6,10 @@ tf_class {
   member_method {
     name: "__init__"
   }
+  member_method {
+    name: "evaluate"
+    argspec: "args=[\'self\', \'tensors\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "is_abstract"
     argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From b84cc997427594a52200c8949c36928ba3aad123 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Mon, 26 Nov 2018 17:39:09 -0800
Subject: [PATCH 0836/1554] Improve readability of base_layer.py to improve the
 experience of users who use the code as a form of documentation (in
 particular people who subclass Layer). The general principle is to:

- Have the Layer class come first in the file
- Have subclassable methods come first in the list of methods
- Then other public methods, roughly in order of importance
- Then private methods
- Move auxiliary utilities to a dedicated file

PiperOrigin-RevId: 222916181
---
 tensorflow/contrib/eager/python/network.py    |    4 +-
 .../contrib/tpu/python/tpu/keras_support.py   |    7 +-
 tensorflow/python/keras/BUILD                 |    1 +
 tensorflow/python/keras/engine/base_layer.py  | 1894 ++++++++---------
 .../python/keras/engine/base_layer_utils.py   |  236 ++
 tensorflow/python/keras/engine/network.py     |   16 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |    4 +-
 tensorflow/python/layers/base.py              |   11 +-
 8 files changed, 1108 insertions(+), 1065 deletions(-)
 create mode 100644 tensorflow/python/keras/engine/base_layer_utils.py

diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index f801d9a47b..5cc0c4f23d 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -24,7 +24,7 @@ import weakref
 
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
-from tensorflow.python.keras.engine import base_layer as keras_base_layer
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.layers import base
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import tf_logging as logging
@@ -220,7 +220,7 @@ class Network(base.Layer):
         avoid_names = parent_network._owned_layers
         name_uid_map = parent_network._sub_layer_name_uids
       else:
-        name_uid_map = keras_base_layer.get_default_graph_uid_map()
+        name_uid_map = base_layer_utils.get_default_graph_uid_map()
         # Figure out which names we have to avoid based on which variable scope
         # we're nested in.
         strip_name = self._default_parent_variable_scope.name
diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index c4db4aa0c7..cf3b2e68e9 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -81,6 +81,7 @@ from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import models
 from tensorflow.python.keras import optimizers as keras_optimizers
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.keras.engine import training_arrays
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.layers import embeddings
@@ -438,7 +439,7 @@ class TPURewriteContext(object):
 
     self._default_placeholder = array_ops.placeholder
     self._default_name_scope = ops.name_scope
-    self._default_make_variable = base_layer.make_variable
+    self._default_make_variable = base_layer_utils.make_variable
     self._default_random_normal = random_ops.random_normal
     self._default_qr = gen_linalg_ops.qr
 
@@ -486,14 +487,14 @@ class TPURewriteContext(object):
     gen_linalg_ops.qr = qr
 
     ops.name_scope = _name_scope
-    base_layer.make_variable = variable_scope.get_variable
+    base_layer_utils.make_variable = variable_scope.get_variable
     logging.info('Overriding default placeholder.')
     return
 
   def __exit__(self, exc_type, exc_val, exc_tb):
     array_ops.placeholder = self._default_placeholder
     ops.name_scope = self._default_name_scope
-    base_layer.make_variable = self._default_make_variable
+    base_layer_utils.make_variable = self._default_make_variable
     random_ops.random_normal = self._default_random_normal
     gen_linalg_ops.qr = self._default_qr
 
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 697f591e35..a7f01817fb 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -122,6 +122,7 @@ py_library(
         "constraints.py",
         "engine/__init__.py",
         "engine/base_layer.py",
+        "engine/base_layer_utils.py",
         "engine/distributed_training_utils.py",
         "engine/input_layer.py",
         "engine/input_spec.py",
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 5426965509..8b795935c1 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections as collections_lib
-import enum  # pylint: disable=g-bad-import-order
 import functools
 import inspect  # Necessary supplement to tf_inspect to deal with variadic args.
 
@@ -36,6 +34,7 @@ from tensorflow.python.keras import backend
 from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.keras.engine import input_spec
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import tf_utils
@@ -43,7 +42,6 @@ from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.keras.utils.generic_utils import to_snake_case  # pylint: disable=unused-import
 from tensorflow.python.keras.utils.tf_utils import is_tensor_or_tensor_list  # pylint: disable=unused-import
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.training.checkpointable import base as checkpointable
@@ -55,28 +53,6 @@ from tensorflow.python.util.tf_export import tf_export
 from tensorflow.tools.docs import doc_controls
 
 
-class CallConvention(enum.Enum):
-  """Calling conventions for passing `Layer` inputs to `Layer.call`."""
-  # The Layer takes inputs as its first argument, named "inputs" for
-  # compatibility with the signature of Layer.__call__. This is the mode assumed
-  # for Layers which are not subclassed Models.
-  EXPLICIT_INPUTS_ARGUMENT = 1
-  # The Layer takes a single positional argument, not named "inputs". It's
-  # treated like an "inputs" argument.
-  SINGLE_POSITIONAL_ARGUMENT = 2
-  # The Layer has multiple positional arguments to which its inputs should be
-  # bound.
-  POSITIONAL_ARGUMENTS_ARE_INPUTS = 3
-
-
-def _create_mean_metric(value, name=None):
-  # TODO(psv): Remove this import when b/110718070 is fixed.
-  from tensorflow.python.keras import metrics as metrics_module  # pylint: disable=g-import-not-at-top
-  metric_obj = metrics_module.Mean(name=name)
-  result = metric_obj(value)
-  return metric_obj, result
-
-
 @tf_export('keras.layers.Layer')
 class Layer(checkpointable.CheckpointableBase):
   """Base layer class.
@@ -111,10 +87,6 @@ class Layer(checkpointable.CheckpointableBase):
     name: The name of the layer (string).
     dtype: Default dtype of the layer's weights (default of `None` means use the
       type of the first input).
-    trainable_variables: List of trainable variables.
-    non_trainable_variables: List of non-trainable variables.
-    variables: List of all variables of this layer, trainable and
-      non-trainable.
     updates: List of update ops of this layer.
     losses: List of losses added by this layer.
     trainable_weights: List of variables to be included in backprop.
@@ -159,9 +131,9 @@ class Layer(checkpointable.CheckpointableBase):
     self.built = False
     # Provides information about which inputs are compatible with the layer.
     self.input_spec = None
+    self.supports_masking = False
 
     self._init_set_name(name)
-
     self._activity_regularizer = kwargs.pop('activity_regularizer', None)
     self._trainable_weights = []
     self._non_trainable_weights = []
@@ -190,15 +162,14 @@ class Layer(checkpointable.CheckpointableBase):
     self._call_fn_args = function_utils.fn_args(self.call)
     self._compute_previous_mask = ('mask' in self._call_fn_args or
                                    hasattr(self, 'compute_mask'))
-    self._call_convention = CallConvention.EXPLICIT_INPUTS_ARGUMENT
+    self._call_convention = (base_layer_utils
+                             .CallConvention.EXPLICIT_INPUTS_ARGUMENT)
 
     # These lists will be filled via successive calls
     # to self._add_inbound_node().
     self._inbound_nodes = []
     self._outbound_nodes = []
 
-    self.supports_masking = False
-
     call_argspec = tf_inspect.getfullargspec(self.call)
     if 'training' in call_argspec.args:
       self._expects_training_arg = True
@@ -228,543 +199,292 @@ class Layer(checkpointable.CheckpointableBase):
     else:
       self._initial_weights = None
 
-  def _init_set_name(self, name, zero_based=True):
-    if not name:
-      self._name = unique_layer_name(
-          generic_utils.to_snake_case(self.__class__.__name__),
-          zero_based=zero_based)
-    else:
-      self._name = name
-
-  @property
-  def dtype(self):
-    return self._dtype
-
-  @property
-  def name(self):
-    return self._name
-
-  @property
-  def activity_regularizer(self):
-    """Optional regularizer function for the output of this layer."""
-    return self._activity_regularizer
-
-  @activity_regularizer.setter
-  def activity_regularizer(self, regularizer):
-    """Optional regularizer function for the output of this layer."""
-    self._activity_regularizer = self._no_dependency(regularizer)
-
-  @property
-  def trainable_weights(self):
-    return self._trainable_weights if self.trainable else []
-
-  @property
-  def non_trainable_weights(self):
-    if self.trainable:
-      return self._non_trainable_weights
-    else:
-      return self._trainable_weights + self._non_trainable_weights
-
-  @property
-  def trainable_variables(self):
-    return self.trainable_weights
+  def build(self, input_shape):
+    """Creates the variables of the layer."""
+    self.built = True
 
-  @property
-  def non_trainable_variables(self):
-    return self.non_trainable_weights
+  @doc_controls.for_subclass_implementers
+  def call(self, inputs, **kwargs):  # pylint: disable=unused-argument
+    """This is where the layer's logic lives.
 
-  @property
-  def weights(self):
-    """Returns the list of all layer variables/weights.
+    Arguments:
+        inputs: Input tensor, or list/tuple of input tensors.
+        **kwargs: Additional keyword arguments.
 
     Returns:
-      A list of variables.
+        A tensor or list/tuple of tensors.
     """
-    return self.trainable_weights + self.non_trainable_weights
+    return inputs
 
-  @property
-  def variables(self):
-    """Returns the list of all layer variables/weights.
+  @doc_controls.for_subclass_implementers
+  def add_weight(self,
+                 name,
+                 shape,
+                 dtype=None,
+                 initializer=None,
+                 regularizer=None,
+                 trainable=None,
+                 constraint=None,
+                 partitioner=None,
+                 use_resource=None,
+                 synchronization=tf_variables.VariableSynchronization.AUTO,
+                 aggregation=tf_variables.VariableAggregation.NONE,
+                 **kwargs):
+    """Adds a new variable to the layer, or gets an existing one; returns it.
 
-    Returns:
-      A list of variables.
-    """
-    return self.weights
+    Arguments:
+      name: variable name.
+      shape: variable shape.
+      dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
+      initializer: initializer instance (callable).
+      regularizer: regularizer instance (callable).
+      trainable: whether the variable should be part of the layer's
+        "trainable_variables" (e.g. variables, biases)
+        or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
+        Note, if the current variable scope is marked as non-trainable
+        then this parameter is ignored and any added variables are also
+        marked as non-trainable. `trainable` defaults to `True` unless
+        `synchronization` is set to `ON_READ`.
+      constraint: constraint instance (callable).
+      partitioner: Partitioner to be passed to the `Checkpointable` API.
+      use_resource: Whether to use `ResourceVariable`.
+      synchronization: Indicates when a distributed a variable will be
+        aggregated. Accepted values are constants defined in the class
+        `tf.VariableSynchronization`. By default the synchronization is set to
+        `AUTO` and the current `DistributionStrategy` chooses
+        when to synchronize. If `synchronization` is set to `ON_READ`,
+        `trainable` must not be set to `True`.
+      aggregation: Indicates how a distributed variable will be aggregated.
+        Accepted values are constants defined in the class
+        `tf.VariableAggregation`.
+      **kwargs: Additional keyword arguments. Accepted values are `getter` and
+        `collections`.
 
-  @property
-  def updates(self):
-    if not self.trainable and not self.stateful:
-      return []
-    return self._updates
+    Returns:
+      The created variable.  Usually either a `Variable` or `ResourceVariable`
+      instance.  If `partitioner` is not `None`, a `PartitionedVariable`
+      instance is returned.
 
-  @doc_controls.for_subclass_implementers
-  def add_update(self, updates, inputs=None):
-    """Add update op(s), potentially dependent on layer inputs.
+    Raises:
+      RuntimeError: If called with partioned variable regularization and
+        eager execution is enabled.
+      ValueError: When giving unsupported dtype and no initializer or when
+        trainable has been set to True with synchronization set as `ON_READ`.
+    """
+    # Validate optional keyword arguments.
+    for kwarg in kwargs:
+      if kwarg not in ['getter', 'collections']:
+        raise TypeError('Unknown keyword argument:', kwarg)
+    getter = kwargs.pop('getter', None)
+    collections = kwargs.pop('collections', None)
 
-    Weight updates (for instance, the updates of the moving mean and variance
-    in a BatchNormalization layer) may be dependent on the inputs passed
-    when calling a layer. Hence, when reusing the same layer on
-    different inputs `a` and `b`, some entries in `layer.updates` may be
-    dependent on `a` and some on `b`. This method automatically keeps track
-    of dependencies.
+    if dtype is None:
+      dtype = self.dtype or backend.floatx()
+    dtype = dtypes.as_dtype(dtype)
+    initializer = initializers.get(initializer)
+    regularizer = regularizers.get(regularizer)
+    constraint = constraints.get(constraint)
 
-    The `get_updates_for` method allows to retrieve the updates relevant to a
-    specific set of inputs.
+    if synchronization == tf_variables.VariableSynchronization.ON_READ:
+      if trainable:
+        raise ValueError(
+            'Synchronization value can be set to '
+            'VariableSynchronization.ON_READ only for non-trainable variables. '
+            'You have specified trainable=True and '
+            'synchronization=VariableSynchronization.ON_READ.')
+      else:
+        # Set trainable to be false when variable is to be synced on read.
+        trainable = False
+    elif trainable is None:
+      trainable = True
 
-    This call is ignored when eager execution is enabled (in that case, variable
-    updates are run on the fly and thus do not need to be tracked for later
-    execution).
+    # Initialize variable when no initializer provided
+    if initializer is None:
+      # If dtype is DT_FLOAT, provide a uniform unit scaling initializer
+      if dtype.is_floating:
+        initializer = initializers.glorot_uniform()
+      # If dtype is DT_INT/DT_UINT, provide a default value `zero`
+      # If dtype is DT_BOOL, provide a default value `FALSE`
+      elif dtype.is_integer or dtype.is_unsigned or dtype.is_bool:
+        initializer = initializers.zeros()
+      # NOTES:Do we need to support for handling DT_STRING and DT_COMPLEX here?
+      else:
+        raise ValueError('An initializer for variable %s of type %s is required'
+                         ' for layer %s' % (name, dtype.base_dtype, self.name))
 
-    Arguments:
-      updates: Update op, or list/tuple of update ops.
-      inputs: If anything other than None is passed, it signals the updates
-        are conditional on some of the layer's inputs,
-        and thus they should only be run where these inputs are available.
-        This is the case for BatchNormalization updates, for instance.
-        If None, the updates will be taken into account unconditionally,
-        and you are responsible for making sure that any dependency they might
-        have is available at runtime.
-        A step counter might fall into this category.
-    """
-    if context.executing_eagerly():
-      return  # Updates already applied when in eager mode.
+    variable = self._add_variable_with_custom_getter(
+        name=name,
+        shape=shape,
+        # TODO(allenl): a `make_variable` equivalent should be added as a
+        # `Checkpointable` method.
+        getter=getter or base_layer_utils.make_variable,
+        # Manage errors in Layer rather than Checkpointable.
+        overwrite=True,
+        initializer=initializer,
+        dtype=dtype,
+        constraint=constraint,
+        trainable=trainable and self.trainable,
+        partitioner=partitioner,
+        use_resource=use_resource,
+        collections=collections,
+        synchronization=synchronization,
+        aggregation=aggregation)
+    backend.track_variable(variable)
 
-    def process_update(x):
-      if isinstance(x, ops.Operation):
-        return x
-      elif hasattr(x, 'op'):
-        return x.op
-      else:
-        return ops.convert_to_tensor(x)
+    if regularizer is not None:
+      # TODO(fchollet): in the future, this should be handled at the
+      # level of variable creation, and weight regularization losses
+      # should be variable attributes.
+      self._handle_weight_regularization(name, variable, regularizer)
 
-    updates = generic_utils.to_list(updates)
-    updates = [process_update(x) for x in updates]
-    self._updates += updates
-    if inputs is None:
-      for u in updates:
-        u._unconditional_update = True  # pylint: disable=protected-access
+    if trainable:
+      self._trainable_weights.append(variable)
     else:
-      for u in updates:
-        u._unconditional_update = False  # pylint: disable=protected-access
+      self._non_trainable_weights.append(variable)
+    return variable
 
-  def get_updates_for(self, inputs):
-    """Retrieves updates relevant to a specific set of inputs.
+  def get_config(self):
+    """Returns the config of the layer.
 
-    Arguments:
-      inputs: Input tensor or list/tuple of input tensors.
+    A layer config is a Python dictionary (serializable)
+    containing the configuration of a layer.
+    The same layer can be reinstantiated later
+    (without its trained weights) from this configuration.
 
-    Returns:
-      List of update ops of the layer that depend on `inputs`.
+    The config of a layer does not include connectivity
+    information, nor the layer class name. These are handled
+    by `Network` (one layer of abstraction above).
 
-    Raises:
-      RuntimeError: If called in Eager mode.
+    Returns:
+        Python dictionary.
     """
-    # Updates disabled if layer is not trainable and not explicitly stateful.
-    if not self.trainable and not self.stateful:
-      return []
-
-    if inputs is None:
-      # Requesting unconditional updates.
-      return [x for x in self.updates if x._unconditional_update]  # pylint: disable=protected-access
+    config = {'name': self.name, 'trainable': self.trainable}
+    if hasattr(self, '_batch_input_shape'):
+      config['batch_input_shape'] = self._batch_input_shape
+    if hasattr(self, 'dtype'):
+      config['dtype'] = self.dtype
+    return config
 
-    # Requesting input-conditional updates.
-    inputs = nest.flatten(inputs)
-    reachable = tf_utils.get_reachable_from_inputs(inputs, self.updates)
-    updates = []
-    for update in self.updates:
-      if update in reachable:
-        updates.append(update)
-    return updates
+  @classmethod
+  def from_config(cls, config):
+    """Creates a layer from its config.
 
-  @property
-  def losses(self):
-    """Losses which are associated with this `Layer`.
+    This method is the reverse of `get_config`,
+    capable of instantiating the same layer from the config
+    dictionary. It does not handle layer connectivity
+    (handled by Network), nor weights (handled by `set_weights`).
 
-    Variable regularization tensors are created when this property is accessed,
-    so it is eager safe: accessing `losses` under a `tf.GradientTape` will
-    propagate gradients back to the corresponding variables.
+    Arguments:
+        config: A Python dictionary, typically the
+            output of get_config.
 
     Returns:
-      A list of tensors.
+        A layer instance.
     """
-    collected_losses = []
-    if context.executing_eagerly():
-      collected_losses.extend(self._eager_losses)
-    else:
-      collected_losses.extend(self._losses)
-    for regularizer in self._callable_losses:
-      loss_tensor = regularizer()
-      if loss_tensor is not None:
-        collected_losses.append(loss_tensor)
-    return collected_losses
-
-  @doc_controls.for_subclass_implementers
-  def add_loss(self, losses, inputs=None):
-    """Add loss tensor(s), potentially dependent on layer inputs.
-
-    Some losses (for instance, activity regularization losses) may be dependent
-    on the inputs passed when calling a layer. Hence, when reusing the same
-    layer on different inputs `a` and `b`, some entries in `layer.losses` may
-    be dependent on `a` and some on `b`. This method automatically keeps track
-    of dependencies.
+    return cls(**config)
 
-    The `get_losses_for` method allows to retrieve the losses relevant to a
-    specific set of inputs.
+  def compute_output_shape(self, input_shape):
+    """Computes the output shape of the layer.
 
-    Note that `add_loss` is not supported when executing eagerly. Instead,
-    variable regularizers may be added through `add_variable`. Activity
-    regularization is not supported directly (but such losses may be returned
-    from `Layer.call()`).
+    Assumes that the layer will be built
+    to match that input shape provided.
 
     Arguments:
-      losses: Loss tensor, or list/tuple of tensors. Rather than tensors, losses
-        may also be zero-argument callables which create a loss tensor.
-      inputs: Ignored when executing eagerly. If anything other than None is
-        passed, it signals the losses are conditional on some of the layer's
-        inputs, and thus they should only be run where these inputs are
-        available. This is the case for activity regularization losses, for
-        instance. If `None` is passed, the losses are assumed
-        to be unconditional, and will apply across all dataflows of the layer
-        (e.g. weight regularization losses).
-    """
-    losses = generic_utils.to_list(losses)
+        input_shape: Shape tuple (tuple of integers)
+            or list of shape tuples (one per output tensor of the layer).
+            Shape tuples can include None for free dimensions,
+            instead of an integer.
 
-    def _tag_unconditional(loss):
-      if callable(loss):
-        loss = loss()
-      if loss is None:
-        return None  # Will be filtered out when computing the .losses property
-      if not tensor_util.is_tensor(loss):
-        loss = ops.convert_to_tensor(loss, dtype=backend.floatx())
-      loss._unconditional_loss = (inputs is None)  # pylint: disable=protected-access
-      return loss
+    Returns:
+        An input shape tuple.
+    """
+    if context.executing_eagerly():
+      # In this case we build the model first in order to do shape inference.
+      # This is acceptable because the framework only calls
+      # `compute_output_shape` on shape values that the layer would later be
+      # built for. It would however cause issues in case a user attempts to
+      # use `compute_output_shape` manually (these users will have to
+      # implement `compute_output_shape` themselves).
+      self.build(input_shape)
+      with context.graph_mode():
+        graph = func_graph.FuncGraph('graph')
+        with graph.as_default():
+          if isinstance(input_shape, list):
+            inputs = [base_layer_utils.generate_placeholders_from_shape(shape)
+                      for shape in input_shape]
+          else:
+            inputs = base_layer_utils.generate_placeholders_from_shape(
+                input_shape)
 
-    for loss in losses:
-      if callable(loss):
-        self._callable_losses.append(
-            functools.partial(_tag_unconditional, loss))
+          try:
+            if self._expects_training_arg:
+              outputs = self(inputs, training=False)
+            else:
+              outputs = self(inputs)
+          except TypeError:
+            raise NotImplementedError('We could not automatically infer '
+                                      'the static shape of the layer\'s output.'
+                                      ' Please implement the '
+                                      '`compute_output_shape` method on your '
+                                      'layer (%s).' % self.__class__.__name__)
+      if isinstance(outputs, list):
+        return [output.shape for output in outputs]
       else:
-        if context.executing_eagerly():
-          self._eager_losses.append(_tag_unconditional(loss))
-        else:
-          self._losses.append(_tag_unconditional(loss))
+        return outputs.shape
+    raise NotImplementedError
 
-  @doc_controls.for_subclass_implementers
-  def add_metric(self, value, aggregation=None, name=None):
-    """Adds metric tensor to the layer.
+  def compute_mask(self, inputs, mask=None):  # pylint: disable=unused-argument
+    """Computes an output mask tensor.
 
-    Args:
-      value: Metric tensor.
-      aggregation: Sample-wise metric reduction function. If `aggregation=None`,
-        it indicates that the metric tensor provided has been aggregated
-        already. eg, `model.add_metric(BinaryAccuracy(name='acc')(y_true,
-        y_pred))`. If aggregation='mean', the given metric tensor will be
-        sample-wise reduced using `mean` function. eg, `model.add_metric(
-        tf.reduce_mean(outputs), name='output_mean', aggregation='mean')`.
-      name: String metric name.
+    Arguments:
+        inputs: Tensor or list of tensors.
+        mask: Tensor or list of tensors.
 
-    Raises:
-      ValueError: If `aggregation` is anything other than None or `mean`.
+    Returns:
+        None or a tensor (or list of tensors,
+            one per output tensor of the layer).
     """
-    if aggregation is not None and aggregation != 'mean':
-      raise ValueError(
-          'We currently support only `mean` sample-wise metric aggregation. '
-          'You provided aggregation=`%s`' % aggregation)
-
-    if tf_utils.is_symbolic_tensor(value):
-      self._symbolic_add_metric(value, aggregation, name)
-    else:
-      self._eager_add_metric(value, aggregation, name)
-
-  def _get_existing_metric(self, name=None):
-    match = [m for m in self._metrics if m.name == name]
-    if not match:
-      return
-    if len(match) > 1:
-      raise ValueError(
-          'Please provide different names for the metrics you have added. '
-          'We found {} metrics with the name: "{}"'.format(len(match), name))
-    return match[0]
-
-  def _eager_add_metric(self, value, aggregation=None, name=None):
-    # If the given metric is available in `metrics` list we just update state
-    # on it, otherwise we create a new metric instance and
-    # add it to the `metrics` list.
-    match = self._get_existing_metric(name)
-    if match:
-      match(value)  # Update the metric state.
-      return
-    else:
-      if aggregation is None:
-        raise ValueError('We do not support adding an aggregated metric tensor '
-                         'in `call` in eager execution.')
-      metric_obj, _ = _create_mean_metric(value, name)
-      self._metrics.append(metric_obj)
-
-  def _symbolic_add_metric(self, value, aggregation=None, name=None):
-    if aggregation is None:
-      # Iterate over the metrics and check if the given metric exists already.
-      # This can happen when a metric instance is created in subclassed model
-      # layer `__init__` and we have tracked that instance already in
-      # model.__setattr__.
-      match = self._get_existing_metric(name)
-      if match:
-        result_tensor = value
-        if match.name not in self._metrics_tensors:
-          self._metrics_tensors[match.name] = result_tensor
-          return
+    if not self.supports_masking:
+      if mask is not None:
+        if isinstance(mask, list):
+          if any(m is not None for m in mask):
+            raise TypeError('Layer ' + self.name + ' does not support masking, '
+                            'but was passed an input_mask: ' + str(mask))
         else:
-          raise ValueError(
-              'We currently do not support reusing a metric instance.')
-      else:
-        # We track the instance using the metadata on the result tensor.
-        result_tensor = value
-        metric_obj = result_tensor._metric_obj
-    else:
-      # If a non-aggregated tensor is given as input (ie. `aggregation` is
-      # explicitly set to `mean`), we wrap the tensor in `Mean` metric.
-      metric_obj, result_tensor = _create_mean_metric(value, name)
-    self._metrics.append(metric_obj)
-    self._metrics_tensors[metric_obj.name] = result_tensor
+          raise TypeError('Layer ' + self.name + ' does not support masking, '
+                          'but was passed an input_mask: ' + str(mask))
+      # masking not explicitly supported: return None as mask
+      return None
+    # if masking is explicitly supported, by default
+    # carry over the input mask
+    return mask
 
-  def get_losses_for(self, inputs):
-    """Retrieves losses relevant to a specific set of inputs.
+  def __call__(self, inputs, *args, **kwargs):
+    """Wraps `call`, applying pre- and post-processing steps.
 
     Arguments:
-      inputs: Input tensor or list/tuple of input tensors.
+      inputs: input tensor(s).
+      *args: additional positional arguments to be passed to `self.call`.
+      **kwargs: additional keyword arguments to be passed to `self.call`.
 
     Returns:
-      List of loss tensors of the layer that depend on `inputs`.
+      Output tensor(s).
+
+    Note:
+      - The following optional keyword arguments are reserved for specific uses:
+        * `training`: Boolean scalar tensor of Python boolean indicating
+          whether the `call` is meant for training or inference.
+        * `mask`: Boolean input mask.
+      - If the layer's `call` method takes a `mask` argument (as some Keras
+        layers do), its default value will be set to the mask generated
+        for `inputs` by the previous layer (if `input` did come from
+        a layer that generated a corresponding mask, i.e. if it came from
+        a Keras layer with masking support.
 
     Raises:
-      RuntimeError: If called in Eager mode.
+      ValueError: if the layer's `call` method returns None (an invalid value).
     """
-    if inputs is None:
-      # Requesting unconditional losses.
-      return [x for x in self.losses if x._unconditional_loss]  # pylint: disable=protected-access
-
-    # Requesting input-conditional losses.
-    inputs = nest.flatten(inputs)
-    # Retrieve the set of tensors in the TF graph that depend on `inputs`.
-    # The losses we want to return will be part of this set.
-    # To avoid unnecessary work, we stop the search in case all of
-    # `self.losses` have been retrieved.
-    reachable = tf_utils.get_reachable_from_inputs(inputs, self.losses)
-    losses = []
-    for loss in self.losses:
-      if loss in reachable:
-        losses.append(loss)
-    return losses
-
-  def _name_scope(self):
-    return self.name
-
-  def build(self, input_shape):
-    """Creates the variables of the layer."""
-    self.built = True
-
-  @doc_controls.for_subclass_implementers
-  def add_variable(self, *args, **kwargs):
-    """Alias for `add_weight`."""
-    return self.add_weight(*args, **kwargs)
-
-  @doc_controls.for_subclass_implementers
-  def add_weight(self,
-                 name,
-                 shape,
-                 dtype=None,
-                 initializer=None,
-                 regularizer=None,
-                 trainable=None,
-                 constraint=None,
-                 partitioner=None,
-                 use_resource=None,
-                 synchronization=tf_variables.VariableSynchronization.AUTO,
-                 aggregation=tf_variables.VariableAggregation.NONE,
-                 **kwargs):
-    """Adds a new variable to the layer, or gets an existing one; returns it.
-
-    Arguments:
-      name: variable name.
-      shape: variable shape.
-      dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
-      initializer: initializer instance (callable).
-      regularizer: regularizer instance (callable).
-      trainable: whether the variable should be part of the layer's
-        "trainable_variables" (e.g. variables, biases)
-        or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
-        Note, if the current variable scope is marked as non-trainable
-        then this parameter is ignored and any added variables are also
-        marked as non-trainable. `trainable` defaults to `True` unless
-        `synchronization` is set to `ON_READ`.
-      constraint: constraint instance (callable).
-      partitioner: Partitioner to be passed to the `Checkpointable` API.
-      use_resource: Whether to use `ResourceVariable`.
-      synchronization: Indicates when a distributed a variable will be
-        aggregated. Accepted values are constants defined in the class
-        `tf.VariableSynchronization`. By default the synchronization is set to
-        `AUTO` and the current `DistributionStrategy` chooses
-        when to synchronize. If `synchronization` is set to `ON_READ`,
-        `trainable` must not be set to `True`.
-      aggregation: Indicates how a distributed variable will be aggregated.
-        Accepted values are constants defined in the class
-        `tf.VariableAggregation`.
-      **kwargs: Additional keyword arguments. Accepted values are `getter` and
-        `collections`.
-
-    Returns:
-      The created variable.  Usually either a `Variable` or `ResourceVariable`
-      instance.  If `partitioner` is not `None`, a `PartitionedVariable`
-      instance is returned.
-
-    Raises:
-      RuntimeError: If called with partioned variable regularization and
-        eager execution is enabled.
-      ValueError: When giving unsupported dtype and no initializer or when
-        trainable has been set to True with synchronization set as `ON_READ`.
-    """
-    # Validate optional keyword arguments.
-    for kwarg in kwargs:
-      if kwarg not in ['getter', 'collections']:
-        raise TypeError('Unknown keyword argument:', kwarg)
-    getter = kwargs.pop('getter', None)
-    collections = kwargs.pop('collections', None)
-
-    if dtype is None:
-      dtype = self.dtype or backend.floatx()
-    dtype = dtypes.as_dtype(dtype)
-    initializer = initializers.get(initializer)
-    regularizer = regularizers.get(regularizer)
-    constraint = constraints.get(constraint)
-
-    if synchronization == tf_variables.VariableSynchronization.ON_READ:
-      if trainable:
-        raise ValueError(
-            'Synchronization value can be set to '
-            'VariableSynchronization.ON_READ only for non-trainable variables. '
-            'You have specified trainable=True and '
-            'synchronization=VariableSynchronization.ON_READ.')
-      else:
-        # Set trainable to be false when variable is to be synced on read.
-        trainable = False
-    elif trainable is None:
-      trainable = True
-
-    # Initialize variable when no initializer provided
-    if initializer is None:
-      # If dtype is DT_FLOAT, provide a uniform unit scaling initializer
-      if dtype.is_floating:
-        initializer = initializers.glorot_uniform()
-      # If dtype is DT_INT/DT_UINT, provide a default value `zero`
-      # If dtype is DT_BOOL, provide a default value `FALSE`
-      elif dtype.is_integer or dtype.is_unsigned or dtype.is_bool:
-        initializer = initializers.zeros()
-      # NOTES:Do we need to support for handling DT_STRING and DT_COMPLEX here?
-      else:
-        raise ValueError('An initializer for variable %s of type %s is required'
-                         ' for layer %s' % (name, dtype.base_dtype, self.name))
-
-    variable = self._add_variable_with_custom_getter(
-        name=name,
-        shape=shape,
-        # TODO(allenl): a `make_variable` equivalent should be added as a
-        # `Checkpointable` method.
-        getter=getter or make_variable,
-        # Manage errors in Layer rather than Checkpointable.
-        overwrite=True,
-        initializer=initializer,
-        dtype=dtype,
-        constraint=constraint,
-        trainable=trainable and self.trainable,
-        partitioner=partitioner,
-        use_resource=use_resource,
-        collections=collections,
-        synchronization=synchronization,
-        aggregation=aggregation)
-    backend.track_variable(variable)
-
-    if regularizer is not None:
-      # TODO(fchollet): in the future, this should be handled at the
-      # level of variable creation, and weight regularization losses
-      # should be variable attributes.
-      self._handle_weight_regularization(name, variable, regularizer)
-
-    if trainable:
-      self._trainable_weights.append(variable)
-    else:
-      self._non_trainable_weights.append(variable)
-    return variable
-
-  def _handle_weight_regularization(self, name, variable, regularizer):
-    """Create lambdas which compute regularization losses."""
-
-    def _loss_for_variable(v):
-      """Creates a regularization loss `Tensor` for variable `v`."""
-      with ops.colocate_with(v):
-        with ops.name_scope(name + '/Regularizer'):
-          regularization = regularizer(v)
-      return regularization
-
-    if isinstance(variable, tf_variables.PartitionedVariable):
-      for v in variable:
-        self.add_loss(functools.partial(_loss_for_variable, v))
-    else:
-      self.add_loss(functools.partial(_loss_for_variable, variable))
-
-  def _handle_activity_regularization(self, inputs, outputs):
-    # Apply activity regularization.
-    # Note that it should be applied every time the layer creates a new
-    # output, since it is output-specific.
-    if self._activity_regularizer:
-      output_list = nest.flatten(outputs)
-      with ops.name_scope('ActivityRegularizer'):
-        for output in output_list:
-          activity_loss = self._activity_regularizer(output)
-          batch_size = math_ops.cast(
-              array_ops.shape(output)[0], activity_loss.dtype)
-          # Make activity regularization strength batch-agnostic.
-          mean_activity_loss = activity_loss / batch_size
-          self.add_loss(mean_activity_loss, inputs=inputs)
-
-  @doc_controls.for_subclass_implementers
-  def call(self, inputs, **kwargs):  # pylint: disable=unused-argument
-    """This is where the layer's logic lives.
-
-    Arguments:
-        inputs: Input tensor, or list/tuple of input tensors.
-        **kwargs: Additional keyword arguments.
-
-    Returns:
-        A tensor or list/tuple of tensors.
-    """
-    return inputs
-
-  def __call__(self, inputs, *args, **kwargs):
-    """Wraps `call`, applying pre- and post-processing steps.
-
-    Arguments:
-      inputs: input tensor(s).
-      *args: additional positional arguments to be passed to `self.call`.
-      **kwargs: additional keyword arguments to be passed to `self.call`.
-
-    Returns:
-      Output tensor(s).
-
-    Note:
-      - The following optional keyword arguments are reserved for specific uses:
-        * `training`: Boolean scalar tensor of Python boolean indicating
-          whether the `call` is meant for training or inference.
-        * `mask`: Boolean input mask.
-      - If the layer's `call` method takes a `mask` argument (as some Keras
-        layers do), its default value will be set to the mask generated
-        for `inputs` by the previous layer (if `input` did come from
-        a layer that generated a corresponding mask, i.e. if it came from
-        a Keras layer with masking support.
-
-    Raises:
-      ValueError: if the layer's `call` method returns None (an invalid value).
-    """
-    input_list = nest.flatten(inputs)
+    input_list = nest.flatten(inputs)
 
     if context.executing_eagerly():
       # Accept NumPy inputs by converting to Tensors when executing eagerly.
@@ -783,7 +503,7 @@ class Layer(checkpointable.CheckpointableBase):
     previous_mask = None
     if build_graph and (not hasattr(self, '_compute_previous_mask') or
                         self._compute_previous_mask):
-      previous_mask = collect_previous_mask(inputs)
+      previous_mask = base_layer_utils.collect_previous_mask(inputs)
       if not hasattr(self, '_call_fn_args'):
         self._call_fn_args = self._no_dependency(
             function_utils.fn_args(self.call))
@@ -859,7 +579,7 @@ class Layer(checkpointable.CheckpointableBase):
                              '(layer: ' + self.name + ').')
           self._handle_activity_regularization(inputs, outputs)
           self._set_mask_metadata(inputs, outputs, previous_mask)
-          if have_all_keras_metadata(inputs):
+          if base_layer_utils.have_all_keras_metadata(inputs):
             inputs, outputs = self._set_connectivity_metadata_(
                 inputs, outputs, args, kwargs)
           if hasattr(self, '_set_inputs') and not self.inputs:
@@ -882,312 +602,299 @@ class Layer(checkpointable.CheckpointableBase):
         del self._initial_weights
     return outputs
 
-  def apply(self, inputs, *args, **kwargs):
-    """Apply the layer on a input.
+  @property
+  def dtype(self):
+    return self._dtype
+
+  @property
+  def name(self):
+    return self._name
 
-    This simply wraps `self.__call__`.
+  @property
+  def activity_regularizer(self):
+    """Optional regularizer function for the output of this layer."""
+    return self._activity_regularizer
 
-    Arguments:
-      inputs: Input tensor(s).
-      *args: additional positional arguments to be passed to `self.call`.
-      **kwargs: additional keyword arguments to be passed to `self.call`.
+  @activity_regularizer.setter
+  def activity_regularizer(self, regularizer):
+    """Optional regularizer function for the output of this layer."""
+    self._activity_regularizer = self._no_dependency(regularizer)
 
-    Returns:
-      Output tensor(s).
-    """
-    return self.__call__(inputs, *args, **kwargs)
+  @property
+  def trainable_weights(self):
+    return self._trainable_weights if self.trainable else []
 
-  def _set_mask_metadata(self, inputs, outputs, previous_mask):
-    # In some cases the mask of the outputs has already been computed by
-    # inner layers and does not need to be recomputed by this layer.
-    mask_already_computed = all(
-        hasattr(x, '_keras_mask') for x in generic_utils.to_list(outputs))
-    if hasattr(self, 'compute_mask') and not mask_already_computed:
-      output_mask = self.compute_mask(inputs, previous_mask)
+  @property
+  def non_trainable_weights(self):
+    if self.trainable:
+      return self._non_trainable_weights
     else:
-      output_mask = None
-    if isinstance(outputs, (list, tuple)):
-      if output_mask is None:
-        output_mask = [None for _ in range(len(outputs))]
-      for x, m in zip(outputs, output_mask):
-        try:
-          x._keras_mask = m  # pylint: disable=protected-access
-        except AttributeError:
-          pass  # C type such as dict. Masking not supported in this case.
-    else:
-      try:
-        outputs._keras_mask = output_mask  # pylint: disable=protected-access
-      except AttributeError:
-        pass  # C type such as dict. Masking not supported in this case.
+      return self._trainable_weights + self._non_trainable_weights
 
-  def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs):
-    call_convention = getattr(self, '_call_convention',
-                              CallConvention.EXPLICIT_INPUTS_ARGUMENT)
-    if args:
-      if call_convention == CallConvention.EXPLICIT_INPUTS_ARGUMENT:
-        raise TypeError(
-            'This layer ("{}") takes an `inputs` argument in `call()`, '
-            'and only the `inputs` argument may be specified as a positional '
-            'argument. Pass everything else as a keyword argument '
-            '(those arguments will not be tracked '
-            'as inputs to the layer).'.format(self.name))
-      elif call_convention == CallConvention.SINGLE_POSITIONAL_ARGUMENT:
-        raise TypeError(
-            'This layer ("{}") takes a single positional argument in `call()`,'
-            ' which is by convention the `inputs` argument, '
-            'and only this argument may be specified as a positional argument. '
-            'Pass everything else as a keyword argument '
-            '(those arguments will not be tracked '
-            'as inputs to the layer).'.format(self.name))
+  @property
+  def weights(self):
+    """Returns the list of all layer variables/weights.
 
-    # If the layer returns tensors from its inputs, unmodified,
-    # we copy them to avoid loss of tensor metadata.
-    output_ls = nest.flatten(outputs)
-    output_ls_copy = []
-    for x in output_ls:
-      if x in nest.flatten(inputs):
-        with ops.name_scope(self.name):
-          x = array_ops.identity(x)
-      output_ls_copy.append(x)
-    if len(output_ls_copy) == 1:
-      outputs = output_ls_copy[0]
-    else:
-      outputs = output_ls_copy
+    Returns:
+      A list of variables.
+    """
+    return self.trainable_weights + self.non_trainable_weights
 
-    inputs, kwargs = self._inputs_from_call_args(
-        call_args=(inputs,) + args, call_kwargs=kwargs)
-    # Add an inbound node to the layer, so it can keep track of this call.
-    # This updates the layer history of the output tensor(s).
-    kwargs.pop('mask', None)  # `mask` should not be serialized.
-    self._add_inbound_node(
-        input_tensors=inputs, output_tensors=outputs, arguments=kwargs)
-    return inputs, outputs
+  @property
+  def updates(self):
+    if not self.trainable and not self.stateful:
+      return []
+    return self._updates
 
-  def _inputs_from_call_args(self, call_args, call_kwargs):
-    """Get Layer inputs from __call__ *args and **kwargs.
+  @property
+  def losses(self):
+    """Losses which are associated with this `Layer`.
 
-    Args:
-      call_args: The positional arguments passed to __call__.
-      call_kwargs: The keyword argument dict passed to __call__.
+    Variable regularization tensors are created when this property is accessed,
+    so it is eager safe: accessing `losses` under a `tf.GradientTape` will
+    propagate gradients back to the corresponding variables.
 
     Returns:
-      A tuple of (inputs, non_input_kwargs). These may be the same objects as
-      were passed in (call_args and call_kwargs).
+      A list of tensors.
     """
-    call_convention = getattr(self, '_call_convention',
-                              CallConvention.EXPLICIT_INPUTS_ARGUMENT)
-    if (call_convention in (
-        CallConvention.EXPLICIT_INPUTS_ARGUMENT,
-        CallConvention.SINGLE_POSITIONAL_ARGUMENT)):
-      assert len(call_args) == 1  # TypeError raised earlier in __call__.
-      return call_args[0], call_kwargs
+    collected_losses = []
+    if context.executing_eagerly():
+      collected_losses.extend(self._eager_losses)
     else:
-      call_arg_spec = tf_inspect.getfullargspec(self.call)
-      # There is no explicit "inputs" argument expected or provided to
-      # call(). Arguments which have default values are considered non-inputs,
-      # and arguments without are considered inputs.
-      if call_arg_spec.defaults:
-        if call_arg_spec.varargs is not None:
-          raise TypeError(
-              'Layers may not accept both positional arguments and '
-              'arguments with default values (unable to determine which '
-              'are inputs to the layer). '
-              'Issue occurred with layer "%s"' % (self.name))
-        keyword_arg_names = set(
-            call_arg_spec.args[-len(call_arg_spec.defaults):])
+      collected_losses.extend(self._losses)
+    for regularizer in self._callable_losses:
+      loss_tensor = regularizer()
+      if loss_tensor is not None:
+        collected_losses.append(loss_tensor)
+    return collected_losses
+
+  @doc_controls.for_subclass_implementers
+  def add_loss(self, losses, inputs=None):
+    """Add loss tensor(s), potentially dependent on layer inputs.
+
+    Some losses (for instance, activity regularization losses) may be dependent
+    on the inputs passed when calling a layer. Hence, when reusing the same
+    layer on different inputs `a` and `b`, some entries in `layer.losses` may
+    be dependent on `a` and some on `b`. This method automatically keeps track
+    of dependencies.
+
+    The `get_losses_for` method allows to retrieve the losses relevant to a
+    specific set of inputs.
+
+    Note that `add_loss` is not supported when executing eagerly. Instead,
+    variable regularizers may be added through `add_variable`. Activity
+    regularization is not supported directly (but such losses may be returned
+    from `Layer.call()`).
+
+    Arguments:
+      losses: Loss tensor, or list/tuple of tensors. Rather than tensors, losses
+        may also be zero-argument callables which create a loss tensor.
+      inputs: Ignored when executing eagerly. If anything other than None is
+        passed, it signals the losses are conditional on some of the layer's
+        inputs, and thus they should only be run where these inputs are
+        available. This is the case for activity regularization losses, for
+        instance. If `None` is passed, the losses are assumed
+        to be unconditional, and will apply across all dataflows of the layer
+        (e.g. weight regularization losses).
+    """
+    losses = generic_utils.to_list(losses)
+
+    def _tag_unconditional(loss):
+      if callable(loss):
+        loss = loss()
+      if loss is None:
+        return None  # Will be filtered out when computing the .losses property
+      if not tensor_util.is_tensor(loss):
+        loss = ops.convert_to_tensor(loss, dtype=backend.floatx())
+      loss._unconditional_loss = (inputs is None)  # pylint: disable=protected-access
+      return loss
+
+    for loss in losses:
+      if callable(loss):
+        self._callable_losses.append(
+            functools.partial(_tag_unconditional, loss))
       else:
-        keyword_arg_names = set()
-        # Training is never an input argument name, to allow signatures like
-        # call(x, training).
-      keyword_arg_names.add('training')
-      _, unwrapped_call = tf_decorator.unwrap(self.call)
-      bound_args = inspect.getcallargs(
-          unwrapped_call, *call_args, **call_kwargs)
-      if call_arg_spec.varkw is not None:
-        var_kwargs = bound_args.pop(call_arg_spec.varkw)
-        bound_args.update(var_kwargs)
-        keyword_arg_names = keyword_arg_names.union(var_kwargs.keys())
-      all_args = call_arg_spec.args
-      if all_args and bound_args[all_args[0]] is self:
-        # Ignore the 'self' argument of methods
-        bound_args.pop(call_arg_spec.args[0])
-        all_args = all_args[1:]
-      non_input_arg_values = {}
-      input_arg_values = []
-      remaining_args_are_keyword = False
-      for argument_name in all_args:
-        if argument_name in keyword_arg_names:
-          remaining_args_are_keyword = True
-        else:
-          if remaining_args_are_keyword:
-            raise TypeError(
-                'Found a positional argument in a layer call after a non-input '
-                'argument. All arguments after "training" must be keyword '
-                'arguments, and are not tracked as inputs to the layer. '
-                'Issue occurred with layer "%s"' % (self.name))
-        if remaining_args_are_keyword:
-          non_input_arg_values[argument_name] = bound_args[argument_name]
+        if context.executing_eagerly():
+          self._eager_losses.append(_tag_unconditional(loss))
         else:
-          input_arg_values.append(bound_args[argument_name])
-      if call_arg_spec.varargs is not None:
-        input_arg_values.extend(bound_args[call_arg_spec.varargs])
-      return input_arg_values, non_input_arg_values
+          self._losses.append(_tag_unconditional(loss))
 
-  def compute_output_shape(self, input_shape):
-    """Computes the output shape of the layer.
+  @doc_controls.for_subclass_implementers
+  def add_metric(self, value, aggregation=None, name=None):
+    """Adds metric tensor to the layer.
 
-    Assumes that the layer will be built
-    to match that input shape provided.
+    Args:
+      value: Metric tensor.
+      aggregation: Sample-wise metric reduction function. If `aggregation=None`,
+        it indicates that the metric tensor provided has been aggregated
+        already. eg, `model.add_metric(BinaryAccuracy(name='acc')(y_true,
+        y_pred))`. If aggregation='mean', the given metric tensor will be
+        sample-wise reduced using `mean` function. eg, `model.add_metric(
+        tf.reduce_mean(outputs), name='output_mean', aggregation='mean')`.
+      name: String metric name.
 
-    Arguments:
-        input_shape: Shape tuple (tuple of integers)
-            or list of shape tuples (one per output tensor of the layer).
-            Shape tuples can include None for free dimensions,
-            instead of an integer.
+    Raises:
+      ValueError: If `aggregation` is anything other than None or `mean`.
+    """
+    if aggregation is not None and aggregation != 'mean':
+      raise ValueError(
+          'We currently support only `mean` sample-wise metric aggregation. '
+          'You provided aggregation=`%s`' % aggregation)
 
-    Returns:
-        An input shape tuple.
+    if tf_utils.is_symbolic_tensor(value):
+      self._symbolic_add_metric(value, aggregation, name)
+    else:
+      self._eager_add_metric(value, aggregation, name)
+
+  @doc_controls.for_subclass_implementers
+  def add_update(self, updates, inputs=None):
+    """Add update op(s), potentially dependent on layer inputs.
+
+    Weight updates (for instance, the updates of the moving mean and variance
+    in a BatchNormalization layer) may be dependent on the inputs passed
+    when calling a layer. Hence, when reusing the same layer on
+    different inputs `a` and `b`, some entries in `layer.updates` may be
+    dependent on `a` and some on `b`. This method automatically keeps track
+    of dependencies.
+
+    The `get_updates_for` method allows to retrieve the updates relevant to a
+    specific set of inputs.
+
+    This call is ignored when eager execution is enabled (in that case, variable
+    updates are run on the fly and thus do not need to be tracked for later
+    execution).
+
+    Arguments:
+      updates: Update op, or list/tuple of update ops.
+      inputs: If anything other than None is passed, it signals the updates
+        are conditional on some of the layer's inputs,
+        and thus they should only be run where these inputs are available.
+        This is the case for BatchNormalization updates, for instance.
+        If None, the updates will be taken into account unconditionally,
+        and you are responsible for making sure that any dependency they might
+        have is available at runtime.
+        A step counter might fall into this category.
     """
     if context.executing_eagerly():
-      # In this case we build the model first in order to do shape inference.
-      # This is acceptable because the framework only calls
-      # `compute_output_shape` on shape values that the layer would later be
-      # built for. It would however cause issues in case a user attempts to
-      # use `compute_output_shape` manually (these users will have to
-      # implement `compute_output_shape` themselves).
-      self.build(input_shape)
-      with context.graph_mode():
-        graph = func_graph.FuncGraph('graph')
-        with graph.as_default():
-          if isinstance(input_shape, list):
-            inputs = [generate_placeholders_from_shape(shape)
-                      for shape in input_shape]
-          else:
-            inputs = generate_placeholders_from_shape(input_shape)
+      return  # Updates already applied when in eager mode.
 
-          try:
-            if self._expects_training_arg:
-              outputs = self(inputs, training=False)
-            else:
-              outputs = self(inputs)
-          except TypeError:
-            raise NotImplementedError('We could not automatically infer '
-                                      'the static shape of the layer\'s output.'
-                                      ' Please implement the '
-                                      '`compute_output_shape` method on your '
-                                      'layer (%s).' % self.__class__.__name__)
-      if isinstance(outputs, list):
-        return [output.shape for output in outputs]
+    def process_update(x):
+      if isinstance(x, ops.Operation):
+        return x
+      elif hasattr(x, 'op'):
+        return x.op
       else:
-        return outputs.shape
-    raise NotImplementedError
+        return ops.convert_to_tensor(x)
 
-  def compute_mask(self, inputs, mask=None):  # pylint: disable=unused-argument
-    """Computes an output mask tensor.
+    updates = generic_utils.to_list(updates)
+    updates = [process_update(x) for x in updates]
+    self._updates += updates
+    if inputs is None:
+      for u in updates:
+        u._unconditional_update = True  # pylint: disable=protected-access
+    else:
+      for u in updates:
+        u._unconditional_update = False  # pylint: disable=protected-access
+
+  def set_weights(self, weights):
+    """Sets the weights of the layer, from Numpy arrays.
 
     Arguments:
-        inputs: Tensor or list of tensors.
-        mask: Tensor or list of tensors.
+        weights: a list of Numpy arrays. The number
+            of arrays and their shape must match
+            number of the dimensions of the weights
+            of the layer (i.e. it should match the
+            output of `get_weights`).
+
+    Raises:
+        ValueError: If the provided weights list does not match the
+            layer's specifications.
+    """
+    params = self.weights
+    if len(params) != len(weights):
+      raise ValueError('You called `set_weights(weights)` on layer "' +
+                       self.name + '" with a  weight list of length ' +
+                       str(len(weights)) + ', but the layer was expecting ' +
+                       str(len(params)) + ' weights. Provided weights: ' +
+                       str(weights)[:50] + '...')
+    if not params:
+      return
+    weight_value_tuples = []
+    param_values = backend.batch_get_value(params)
+    for pv, p, w in zip(param_values, params, weights):
+      if pv.shape != w.shape:
+        raise ValueError('Layer weight shape ' + str(pv.shape) +
+                         ' not compatible with '
+                         'provided weight shape ' + str(w.shape))
+      weight_value_tuples.append((p, w))
+    backend.batch_set_value(weight_value_tuples)
+
+  def get_weights(self):
+    """Returns the current weights of the layer.
 
     Returns:
-        None or a tensor (or list of tensors,
-            one per output tensor of the layer).
+        Weights values as a list of numpy arrays.
     """
-    if not self.supports_masking:
-      if mask is not None:
-        if isinstance(mask, list):
-          if any(m is not None for m in mask):
-            raise TypeError('Layer ' + self.name + ' does not support masking, '
-                            'but was passed an input_mask: ' + str(mask))
-        else:
-          raise TypeError('Layer ' + self.name + ' does not support masking, '
-                          'but was passed an input_mask: ' + str(mask))
-      # masking not explicitly supported: return None as mask
-      return None
-    # if masking is explicitly supported, by default
-    # carry over the input mask
-    return mask
+    params = self.weights
+    return backend.batch_get_value(params)
 
-  def _add_inbound_node(self,
-                        input_tensors,
-                        output_tensors,
-                        arguments=None):
-    """Internal method to create an inbound node for the layer.
+  def get_updates_for(self, inputs):
+    """Retrieves updates relevant to a specific set of inputs.
 
     Arguments:
-        input_tensors: list of input tensors.
-        output_tensors: list of output tensors.
-        arguments: dictionary of keyword arguments that were passed to the
-            `call` method of the layer at the call that created the node.
-    """
-    input_tensors = nest.flatten(input_tensors)
-    output_tensors = nest.flatten(output_tensors)
+      inputs: Input tensor or list/tuple of input tensors.
 
-    # Collect input tensor(s) coordinates.
-    inbound_layers = []
-    node_indices = []
-    tensor_indices = []
-    for x in input_tensors:
-      assert hasattr(x, '_keras_history')
-      inbound_layer, node_index, tensor_index = x._keras_history  # pylint: disable=protected-access
-      inbound_layers.append(inbound_layer)
-      node_indices.append(node_index)
-      tensor_indices.append(tensor_index)
+    Returns:
+      List of update ops of the layer that depend on `inputs`.
 
-    # Create node, add it to inbound nodes.
-    Node(
-        self,
-        inbound_layers=inbound_layers,
-        node_indices=node_indices,
-        tensor_indices=tensor_indices,
-        input_tensors=input_tensors,
-        output_tensors=output_tensors,
-        arguments=arguments)
+    Raises:
+      RuntimeError: If called in Eager mode.
+    """
+    # Updates disabled if layer is not trainable and not explicitly stateful.
+    if not self.trainable and not self.stateful:
+      return []
 
-    # Update tensor history metadata.
-    for i in range(len(output_tensors)):
-      # The metadata attribute consists of 1) a layer instance
-      # 2) a node index for the layer, 3) a tensor index for the node.
-      # The allows layer reuse (multiple nodes per layer) and multi-output
-      # or multi-input layers (e.g. a layer can return multiple tensors,
-      # and each can be sent to a different layer).
-      output_tensors[i]._keras_history = (self, len(self._inbound_nodes) - 1, i)  # pylint: disable=protected-access
+    if inputs is None:
+      # Requesting unconditional updates.
+      return [x for x in self.updates if x._unconditional_update]  # pylint: disable=protected-access
 
-  def _get_node_attribute_at_index(self, node_index, attr, attr_name):
-    """Private utility to retrieves an attribute (e.g. inputs) from a node.
+    # Requesting input-conditional updates.
+    inputs = nest.flatten(inputs)
+    reachable = tf_utils.get_reachable_from_inputs(inputs, self.updates)
+    updates = []
+    for update in self.updates:
+      if update in reachable:
+        updates.append(update)
+    return updates
 
-    This is used to implement the methods:
-        - get_input_shape_at
-        - get_output_shape_at
-        - get_input_at
-        etc...
+  def get_losses_for(self, inputs):
+    """Retrieves losses relevant to a specific set of inputs.
 
     Arguments:
-        node_index: Integer index of the node from which
-            to retrieve the attribute.
-        attr: Exact node attribute name.
-        attr_name: Human-readable attribute name, for error messages.
+      inputs: Input tensor or list/tuple of input tensors.
 
     Returns:
-        The layer's attribute `attr` at the node of index `node_index`.
+      List of loss tensors of the layer that depend on `inputs`.
 
     Raises:
-        RuntimeError: If the layer has no inbound nodes, or if called in Eager
-        mode.
-        ValueError: If the index provided does not match any node.
+      RuntimeError: If called in Eager mode.
     """
-    if not self._inbound_nodes:
-      raise RuntimeError('The layer has never been called '
-                         'and thus has no defined ' + attr_name + '.')
-    if not len(self._inbound_nodes) > node_index:
-      raise ValueError('Asked to get ' + attr_name + ' at node ' +
-                       str(node_index) + ', but the layer has only ' +
-                       str(len(self._inbound_nodes)) + ' inbound nodes.')
-    values = getattr(self._inbound_nodes[node_index], attr)
-    if len(values) == 1:
-      return values[0]
-    else:
-      return values
+    if inputs is None:
+      # Requesting unconditional losses.
+      return [x for x in self.losses if x._unconditional_loss]  # pylint: disable=protected-access
+
+    # Requesting input-conditional losses.
+    inputs = nest.flatten(inputs)
+    # Retrieve the set of tensors in the TF graph that depend on `inputs`.
+    # The losses we want to return will be part of this set.
+    # To avoid unnecessary work, we stop the search in case all of
+    # `self.losses` have been retrieved.
+    reachable = tf_utils.get_reachable_from_inputs(inputs, self.losses)
+    losses = []
+    for loss in self.losses:
+      if loss in reachable:
+        losses.append(loss)
+    return losses
 
   def get_input_mask_at(self, node_index):
     """Retrieves the input mask tensor(s) of a layer at a given node.
@@ -1482,99 +1189,379 @@ class Layer(checkpointable.CheckpointableBase):
                            'Use `get_output_shape_at(node_index)` '
                            'instead.' % self.name)
 
-  @property
-  @doc_controls.do_not_doc_inheritable
-  def inbound_nodes(self):
-    """Deprecated, do NOT use! Only for compatibility with external Keras."""
-    return self._inbound_nodes
+  @property
+  @doc_controls.do_not_doc_inheritable
+  def inbound_nodes(self):
+    """Deprecated, do NOT use! Only for compatibility with external Keras."""
+    return self._inbound_nodes
+
+  @property
+  @doc_controls.do_not_doc_inheritable
+  def outbound_nodes(self):
+    """Deprecated, do NOT use! Only for compatibility with external Keras."""
+    return self._outbound_nodes
+
+  ##############################################################################
+  # Methods & attributes below are public aliases of other methods.            #
+  ##############################################################################
+
+  def apply(self, inputs, *args, **kwargs):
+    """Apply the layer on a input.
+
+    This is an alias of `self.__call__`.
+
+    Arguments:
+      inputs: Input tensor(s).
+      *args: additional positional arguments to be passed to `self.call`.
+      **kwargs: additional keyword arguments to be passed to `self.call`.
+
+    Returns:
+      Output tensor(s).
+    """
+    return self.__call__(inputs, *args, **kwargs)
+
+  @doc_controls.for_subclass_implementers
+  def add_variable(self, *args, **kwargs):
+    """Alias for `add_weight`."""
+    return self.add_weight(*args, **kwargs)
+
+  @property
+  def variables(self):
+    """Returns the list of all layer variables/weights.
+
+    Alias of `self.weights`.
+
+    Returns:
+      A list of variables.
+    """
+    return self.weights
+
+  @property
+  def trainable_variables(self):
+    return self.trainable_weights
+
+  @property
+  def non_trainable_variables(self):
+    return self.non_trainable_weights
+
+  ##############################################################################
+  # Methods & attributes below are all private and only used by the framework. #
+  ##############################################################################
+
+  def _name_scope(self):
+    return self.name
+
+  def _init_set_name(self, name, zero_based=True):
+    if not name:
+      self._name = base_layer_utils.unique_layer_name(
+          generic_utils.to_snake_case(self.__class__.__name__),
+          zero_based=zero_based)
+    else:
+      self._name = name
+
+  def _get_existing_metric(self, name=None):
+    match = [m for m in self._metrics if m.name == name]
+    if not match:
+      return
+    if len(match) > 1:
+      raise ValueError(
+          'Please provide different names for the metrics you have added. '
+          'We found {} metrics with the name: "{}"'.format(len(match), name))
+    return match[0]
+
+  def _eager_add_metric(self, value, aggregation=None, name=None):
+    # If the given metric is available in `metrics` list we just update state
+    # on it, otherwise we create a new metric instance and
+    # add it to the `metrics` list.
+    match = self._get_existing_metric(name)
+    if match:
+      match(value)  # Update the metric state.
+      return
+    else:
+      if aggregation is None:
+        raise ValueError('We do not support adding an aggregated metric tensor '
+                         'in `call` in eager execution.')
+      metric_obj, _ = base_layer_utils.create_mean_metric(value, name)
+      self._metrics.append(metric_obj)
+
+  def _symbolic_add_metric(self, value, aggregation=None, name=None):
+    if aggregation is None:
+      # Iterate over the metrics and check if the given metric exists already.
+      # This can happen when a metric instance is created in subclassed model
+      # layer `__init__` and we have tracked that instance already in
+      # model.__setattr__.
+      match = self._get_existing_metric(name)
+      if match:
+        result_tensor = value
+        if match.name not in self._metrics_tensors:
+          self._metrics_tensors[match.name] = result_tensor
+          return
+        else:
+          raise ValueError(
+              'We currently do not support reusing a metric instance.')
+      else:
+        # We track the instance using the metadata on the result tensor.
+        result_tensor = value
+        metric_obj = result_tensor._metric_obj
+    else:
+      # If a non-aggregated tensor is given as input (ie. `aggregation` is
+      # explicitly set to `mean`), we wrap the tensor in `Mean` metric.
+      metric_obj, result_tensor = base_layer_utils.create_mean_metric(
+          value, name)
+    self._metrics.append(metric_obj)
+    self._metrics_tensors[metric_obj.name] = result_tensor
+
+  def _handle_weight_regularization(self, name, variable, regularizer):
+    """Create lambdas which compute regularization losses."""
+
+    def _loss_for_variable(v):
+      """Creates a regularization loss `Tensor` for variable `v`."""
+      with ops.colocate_with(v):
+        with ops.name_scope(name + '/Regularizer'):
+          regularization = regularizer(v)
+      return regularization
+
+    if isinstance(variable, tf_variables.PartitionedVariable):
+      for v in variable:
+        self.add_loss(functools.partial(_loss_for_variable, v))
+    else:
+      self.add_loss(functools.partial(_loss_for_variable, variable))
+
+  def _handle_activity_regularization(self, inputs, outputs):
+    # Apply activity regularization.
+    # Note that it should be applied every time the layer creates a new
+    # output, since it is output-specific.
+    if self._activity_regularizer:
+      output_list = nest.flatten(outputs)
+      with ops.name_scope('ActivityRegularizer'):
+        for output in output_list:
+          activity_loss = self._activity_regularizer(output)
+          batch_size = math_ops.cast(
+              array_ops.shape(output)[0], activity_loss.dtype)
+          # Make activity regularization strength batch-agnostic.
+          mean_activity_loss = activity_loss / batch_size
+          self.add_loss(mean_activity_loss, inputs=inputs)
+
+  def _set_mask_metadata(self, inputs, outputs, previous_mask):
+    # In some cases the mask of the outputs has already been computed by
+    # inner layers and does not need to be recomputed by this layer.
+    mask_already_computed = all(
+        hasattr(x, '_keras_mask') for x in generic_utils.to_list(outputs))
+    if hasattr(self, 'compute_mask') and not mask_already_computed:
+      output_mask = self.compute_mask(inputs, previous_mask)
+    else:
+      output_mask = None
+    if isinstance(outputs, (list, tuple)):
+      if output_mask is None:
+        output_mask = [None for _ in range(len(outputs))]
+      for x, m in zip(outputs, output_mask):
+        try:
+          x._keras_mask = m  # pylint: disable=protected-access
+        except AttributeError:
+          pass  # C type such as dict. Masking not supported in this case.
+    else:
+      try:
+        outputs._keras_mask = output_mask  # pylint: disable=protected-access
+      except AttributeError:
+        pass  # C type such as dict. Masking not supported in this case.
+
+  def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs):
+    call_convention = getattr(
+        self, '_call_convention',
+        base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT)
+    if args:
+      if call_convention == (base_layer_utils
+                             .CallConvention.EXPLICIT_INPUTS_ARGUMENT):
+        raise TypeError(
+            'This layer ("{}") takes an `inputs` argument in `call()`, '
+            'and only the `inputs` argument may be specified as a positional '
+            'argument. Pass everything else as a keyword argument '
+            '(those arguments will not be tracked '
+            'as inputs to the layer).'.format(self.name))
+      elif call_convention == (base_layer_utils
+                               .CallConvention.SINGLE_POSITIONAL_ARGUMENT):
+        raise TypeError(
+            'This layer ("{}") takes a single positional argument in `call()`,'
+            ' which is by convention the `inputs` argument, '
+            'and only this argument may be specified as a positional argument. '
+            'Pass everything else as a keyword argument '
+            '(those arguments will not be tracked '
+            'as inputs to the layer).'.format(self.name))
+
+    # If the layer returns tensors from its inputs, unmodified,
+    # we copy them to avoid loss of tensor metadata.
+    output_ls = nest.flatten(outputs)
+    output_ls_copy = []
+    for x in output_ls:
+      if x in nest.flatten(inputs):
+        with ops.name_scope(self.name):
+          x = array_ops.identity(x)
+      output_ls_copy.append(x)
+    if len(output_ls_copy) == 1:
+      outputs = output_ls_copy[0]
+    else:
+      outputs = output_ls_copy
 
-  @property
-  @doc_controls.do_not_doc_inheritable
-  def outbound_nodes(self):
-    """Deprecated, do NOT use! Only for compatibility with external Keras."""
-    return self._outbound_nodes
+    inputs, kwargs = self._inputs_from_call_args(
+        call_args=(inputs,) + args, call_kwargs=kwargs)
+    # Add an inbound node to the layer, so it can keep track of this call.
+    # This updates the layer history of the output tensor(s).
+    kwargs.pop('mask', None)  # `mask` should not be serialized.
+    self._add_inbound_node(
+        input_tensors=inputs, output_tensors=outputs, arguments=kwargs)
+    return inputs, outputs
 
-  def set_weights(self, weights):
-    """Sets the weights of the layer, from Numpy arrays.
+  def _inputs_from_call_args(self, call_args, call_kwargs):
+    """Get Layer inputs from __call__ *args and **kwargs.
 
-    Arguments:
-        weights: a list of Numpy arrays. The number
-            of arrays and their shape must match
-            number of the dimensions of the weights
-            of the layer (i.e. it should match the
-            output of `get_weights`).
+    Args:
+      call_args: The positional arguments passed to __call__.
+      call_kwargs: The keyword argument dict passed to __call__.
 
-    Raises:
-        ValueError: If the provided weights list does not match the
-            layer's specifications.
+    Returns:
+      A tuple of (inputs, non_input_kwargs). These may be the same objects as
+      were passed in (call_args and call_kwargs).
     """
-    params = self.weights
-    if len(params) != len(weights):
-      raise ValueError('You called `set_weights(weights)` on layer "' +
-                       self.name + '" with a  weight list of length ' +
-                       str(len(weights)) + ', but the layer was expecting ' +
-                       str(len(params)) + ' weights. Provided weights: ' +
-                       str(weights)[:50] + '...')
-    if not params:
-      return
-    weight_value_tuples = []
-    param_values = backend.batch_get_value(params)
-    for pv, p, w in zip(param_values, params, weights):
-      if pv.shape != w.shape:
-        raise ValueError('Layer weight shape ' + str(pv.shape) +
-                         ' not compatible with '
-                         'provided weight shape ' + str(w.shape))
-      weight_value_tuples.append((p, w))
-    backend.batch_set_value(weight_value_tuples)
+    call_convention = getattr(
+        self, '_call_convention',
+        base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT)
+    if (call_convention in (
+        base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT,
+        base_layer_utils.CallConvention.SINGLE_POSITIONAL_ARGUMENT)):
+      assert len(call_args) == 1  # TypeError raised earlier in __call__.
+      return call_args[0], call_kwargs
+    else:
+      call_arg_spec = tf_inspect.getfullargspec(self.call)
+      # There is no explicit "inputs" argument expected or provided to
+      # call(). Arguments which have default values are considered non-inputs,
+      # and arguments without are considered inputs.
+      if call_arg_spec.defaults:
+        if call_arg_spec.varargs is not None:
+          raise TypeError(
+              'Layers may not accept both positional arguments and '
+              'arguments with default values (unable to determine which '
+              'are inputs to the layer). '
+              'Issue occurred with layer "%s"' % (self.name))
+        keyword_arg_names = set(
+            call_arg_spec.args[-len(call_arg_spec.defaults):])
+      else:
+        keyword_arg_names = set()
+        # Training is never an input argument name, to allow signatures like
+        # call(x, training).
+      keyword_arg_names.add('training')
+      _, unwrapped_call = tf_decorator.unwrap(self.call)
+      bound_args = inspect.getcallargs(
+          unwrapped_call, *call_args, **call_kwargs)
+      if call_arg_spec.varkw is not None:
+        var_kwargs = bound_args.pop(call_arg_spec.varkw)
+        bound_args.update(var_kwargs)
+        keyword_arg_names = keyword_arg_names.union(var_kwargs.keys())
+      all_args = call_arg_spec.args
+      if all_args and bound_args[all_args[0]] is self:
+        # Ignore the 'self' argument of methods
+        bound_args.pop(call_arg_spec.args[0])
+        all_args = all_args[1:]
+      non_input_arg_values = {}
+      input_arg_values = []
+      remaining_args_are_keyword = False
+      for argument_name in all_args:
+        if argument_name in keyword_arg_names:
+          remaining_args_are_keyword = True
+        else:
+          if remaining_args_are_keyword:
+            raise TypeError(
+                'Found a positional argument in a layer call after a non-input '
+                'argument. All arguments after "training" must be keyword '
+                'arguments, and are not tracked as inputs to the layer. '
+                'Issue occurred with layer "%s"' % (self.name))
+        if remaining_args_are_keyword:
+          non_input_arg_values[argument_name] = bound_args[argument_name]
+        else:
+          input_arg_values.append(bound_args[argument_name])
+      if call_arg_spec.varargs is not None:
+        input_arg_values.extend(bound_args[call_arg_spec.varargs])
+      return input_arg_values, non_input_arg_values
 
-  def get_weights(self):
-    """Returns the current weights of the layer.
+  def _add_inbound_node(self,
+                        input_tensors,
+                        output_tensors,
+                        arguments=None):
+    """Internal method to create an inbound node for the layer.
 
-    Returns:
-        Weights values as a list of numpy arrays.
+    Arguments:
+        input_tensors: list of input tensors.
+        output_tensors: list of output tensors.
+        arguments: dictionary of keyword arguments that were passed to the
+            `call` method of the layer at the call that created the node.
     """
-    params = self.weights
-    return backend.batch_get_value(params)
-
-  def get_config(self):
-    """Returns the config of the layer.
+    input_tensors = nest.flatten(input_tensors)
+    output_tensors = nest.flatten(output_tensors)
 
-    A layer config is a Python dictionary (serializable)
-    containing the configuration of a layer.
-    The same layer can be reinstantiated later
-    (without its trained weights) from this configuration.
+    # Collect input tensor(s) coordinates.
+    inbound_layers = []
+    node_indices = []
+    tensor_indices = []
+    for x in input_tensors:
+      assert hasattr(x, '_keras_history')
+      inbound_layer, node_index, tensor_index = x._keras_history  # pylint: disable=protected-access
+      inbound_layers.append(inbound_layer)
+      node_indices.append(node_index)
+      tensor_indices.append(tensor_index)
 
-    The config of a layer does not include connectivity
-    information, nor the layer class name. These are handled
-    by `Network` (one layer of abstraction above).
+    # Create node, add it to inbound nodes.
+    Node(
+        self,
+        inbound_layers=inbound_layers,
+        node_indices=node_indices,
+        tensor_indices=tensor_indices,
+        input_tensors=input_tensors,
+        output_tensors=output_tensors,
+        arguments=arguments)
 
-    Returns:
-        Python dictionary.
-    """
-    config = {'name': self.name, 'trainable': self.trainable}
-    if hasattr(self, '_batch_input_shape'):
-      config['batch_input_shape'] = self._batch_input_shape
-    if hasattr(self, 'dtype'):
-      config['dtype'] = self.dtype
-    return config
+    # Update tensor history metadata.
+    for i in range(len(output_tensors)):
+      # The metadata attribute consists of 1) a layer instance
+      # 2) a node index for the layer, 3) a tensor index for the node.
+      # The allows layer reuse (multiple nodes per layer) and multi-output
+      # or multi-input layers (e.g. a layer can return multiple tensors,
+      # and each can be sent to a different layer).
+      output_tensors[i]._keras_history = (self, len(self._inbound_nodes) - 1, i)  # pylint: disable=protected-access
 
-  @classmethod
-  def from_config(cls, config):
-    """Creates a layer from its config.
+  def _get_node_attribute_at_index(self, node_index, attr, attr_name):
+    """Private utility to retrieves an attribute (e.g. inputs) from a node.
 
-    This method is the reverse of `get_config`,
-    capable of instantiating the same layer from the config
-    dictionary. It does not handle layer connectivity
-    (handled by Network), nor weights (handled by `set_weights`).
+    This is used to implement the methods:
+        - get_input_shape_at
+        - get_output_shape_at
+        - get_input_at
+        etc...
 
     Arguments:
-        config: A Python dictionary, typically the
-            output of get_config.
+        node_index: Integer index of the node from which
+            to retrieve the attribute.
+        attr: Exact node attribute name.
+        attr_name: Human-readable attribute name, for error messages.
 
     Returns:
-        A layer instance.
+        The layer's attribute `attr` at the node of index `node_index`.
+
+    Raises:
+        RuntimeError: If the layer has no inbound nodes, or if called in Eager
+        mode.
+        ValueError: If the index provided does not match any node.
     """
-    return cls(**config)
+    if not self._inbound_nodes:
+      raise RuntimeError('The layer has never been called '
+                         'and thus has no defined ' + attr_name + '.')
+    if not len(self._inbound_nodes) > node_index:
+      raise ValueError('Asked to get ' + attr_name + ' at node ' +
+                       str(node_index) + ', but the layer has only ' +
+                       str(len(self._inbound_nodes)) + ' inbound nodes.')
+    values = getattr(self._inbound_nodes[node_index], attr)
+    if len(values) == 1:
+      return values[0]
+    else:
+      return values
 
   @property
   def _static_graph_friendly(self):
@@ -1706,197 +1693,12 @@ class Node(object):
     }
 
 
-def unique_layer_name(name, name_uid_map=None, avoid_names=None, namespace='',
-                      zero_based=False):
-  """Makes a layer name (or arbitrary string) unique within a TensorFlow graph.
-
-  Arguments:
-    name: String name to make unique.
-    name_uid_map: An optional defaultdict(int) to use when creating unique
-      names. If None (default), uses a per-Graph dictionary.
-    avoid_names: An optional set or dict with names which should not be used. If
-      None (default) does not avoid any names.
-    namespace: Gets a name which is unique within the (graph, namespace). Layers
-      which are not Networks use a blank namespace and so get graph-global
-      names.
-    zero_based: If True, name sequences start with no suffix (e.g. "dense",
-      "dense_1"). If False, naming is one-based ("dense_1", "dense_2").
-
-  Returns:
-    Unique string name.
-
-  Example:
-
-  ```python
-  _unique_layer_name('dense')  # dense_1
-  _unique_layer_name('dense')  # dense_2
-  ```
-  """
-  if name_uid_map is None:
-    name_uid_map = get_default_graph_uid_map()
-  if avoid_names is None:
-    avoid_names = set()
-  proposed_name = None
-  while proposed_name is None or proposed_name in avoid_names:
-    name_key = (namespace, name)
-    if zero_based:
-      number = name_uid_map[name_key]
-      if number:
-        proposed_name = name + '_' + str(number)
-      else:
-        proposed_name = name
-      name_uid_map[name_key] += 1
-    else:
-      name_uid_map[name_key] += 1
-      proposed_name = name + '_' + str(name_uid_map[name_key])
-  return proposed_name
-
-
-def have_all_keras_metadata(iterable_or_element):
-  if not isinstance(iterable_or_element, (list, tuple)):
-    iterable = [iterable_or_element]
-  else:
-    iterable = nest.flatten(iterable_or_element)
-  return all(hasattr(x, '_keras_history') for x in iterable)
-
-
-def collect_previous_mask(input_tensors):
-  """Retrieves the output mask(s) of the previous node.
-
-  Arguments:
-      input_tensors: A tensor or list of tensors.
-
-  Returns:
-      A mask tensor or list of mask tensors.
-  """
-  input_tensors = nest.flatten(input_tensors)
-  masks = []
-  for x in input_tensors:
-    if hasattr(x, '_keras_mask'):
-      mask = x._keras_mask  # pylint: disable=protected-access
-      masks.append(mask)
-    else:
-      masks.append(None)
-  if len(masks) == 1:
-    return masks[0]
-  return masks
-
-
-def get_default_graph_uid_map():
-  # TODO(fchollet): refactor this into backend.
-  graph = ops.get_default_graph()
-  name_uid_map = backend.PER_GRAPH_LAYER_NAME_UIDS.get(graph, None)
-  if name_uid_map is None:
-    name_uid_map = collections_lib.defaultdict(int)
-    backend.PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map
-  return name_uid_map
-
-
-def make_variable(name,
-                  shape=None,
-                  dtype=dtypes.float32,
-                  initializer=None,
-                  partition_info=None,
-                  trainable=None,
-                  caching_device=None,
-                  validate_shape=True,
-                  constraint=None,
-                  use_resource=None,
-                  collections=None,
-                  synchronization=tf_variables.VariableSynchronization.AUTO,
-                  aggregation=tf_variables.VariableAggregation.NONE,
-                  partitioner=None):  # pylint: disable=unused-argument
-  """Temporary util to create a variable (relies on `variable_scope.variable`).
-
-  Some reuse-related technicalities prevent us from using
-  `variable_scope.get_variable()` directly, so we use a subcomponent
-  that has fewer constraints (`variable_scope.variable()`).
-
-  In the longer term, it seems like a similar "default variable creator" method
-  should exist in `CheckpointableBase` instead. When this happens, we can get
-  rid of this temporary solution.
-
-  TODO(fchollet): remove this method when no longer needed.
-  TODO(fchollet): handle `partitioner` argument.
-
-  Arguments:
-    name: Variable name.
-    shape: Variable shape.
-    dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
-    initializer: Initializer instance (callable).
-    partition_info: Not handled at this time.
-    trainable: Whether the variable should be part of the layer's
-      "trainable_variables" (e.g. variables, biases)
-      or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
-      Note, if the current variable scope is marked as non-trainable
-      then this parameter is ignored and any added variables are also
-      marked as non-trainable. `trainable` defaults to `True` unless
-      `synchronization` is set to `ON_READ`.
-    caching_device: Passed to `tf.Variable`.
-    validate_shape: Passed to `tf.Variable`.
-    constraint: Constraint instance (callable).
-    use_resource: Whether to use a `ResourceVariable`.
-    collections: List of graph collections keys. The new variable is added to
-      these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`.
-    synchronization: Indicates when a distributed a variable will be
-      aggregated. Accepted values are constants defined in the class
-      `tf.VariableSynchronization`. By default the synchronization is set to
-      `AUTO` and the current `DistributionStrategy` chooses
-      when to synchronize. If `synchronization` is set to `ON_READ`,
-      `trainable` must not be set to `True`.
-    aggregation: Indicates how a distributed variable will be aggregated.
-      Accepted values are constants defined in the class
-      `tf.VariableAggregation`.
-    partitioner: Not handled at this time.
-
-  Returns:
-    Variable instance.
-  """
-  initializing_from_value = False
-  if initializer is not None and not callable(initializer):
-    initializing_from_value = True
-
-  with ops.init_scope():
-    if initializing_from_value:
-      init_val = initializer
-      variable_dtype = None
-    else:
-      # Instantiate initializer if provided initializer is a type object.
-      if isinstance(initializer, type(init_ops.Initializer)):
-        initializer = initializer(dtype=dtype)
-      init_val = lambda: initializer(  # pylint: disable=g-long-lambda
-          shape, dtype=dtype, partition_info=partition_info)
-      variable_dtype = dtype.base_dtype
-  if use_resource is None:
-    use_resource = True
-
-  # TODO(apassos,rohanj) figure out how to remove collections from here so we
-  # can remove the V1.
-  v = tf_variables.VariableV1(
-      initial_value=init_val,
-      name=name,
-      trainable=trainable,
-      caching_device=caching_device,
-      dtype=variable_dtype,
-      validate_shape=validate_shape,
-      constraint=constraint,
-      use_resource=use_resource,
-      collections=collections,
-      synchronization=synchronization,
-      aggregation=aggregation)
-  return v
-
-
 def default(method):
   """Decorates a method to detect overrides in subclasses."""
   method._is_default = True
   return method
 
 
-def generate_placeholders_from_shape(shape):
-  return array_ops.placeholder(shape=shape, dtype=backend.floatx())
-
-
 # Avoid breaking users who directly import this symbol from this file.
 # TODO(fchollet): remove this.
 InputSpec = input_spec.InputSpec  # pylint:disable=invalid-name
diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
new file mode 100644
index 0000000000..d2f947f177
--- /dev/null
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -0,0 +1,236 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Contains private utilities used mainly by the base Layer class."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections as collections_lib
+import enum
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import variables as tf_variables
+from tensorflow.python.util import nest
+
+
+class CallConvention(enum.Enum):
+  """Calling conventions for passing `Layer` inputs to `Layer.call`."""
+  # The Layer takes inputs as its first argument, named "inputs" for
+  # compatibility with the signature of Layer.__call__. This is the mode assumed
+  # for Layers which are not subclassed Models.
+  EXPLICIT_INPUTS_ARGUMENT = 1
+  # The Layer takes a single positional argument, not named "inputs". It's
+  # treated like an "inputs" argument.
+  SINGLE_POSITIONAL_ARGUMENT = 2
+  # The Layer has multiple positional arguments to which its inputs should be
+  # bound.
+  POSITIONAL_ARGUMENTS_ARE_INPUTS = 3
+
+
+def create_mean_metric(value, name=None):
+  # TODO(psv): Remove this import when b/110718070 is fixed.
+  from tensorflow.python.keras import metrics as metrics_module  # pylint: disable=g-import-not-at-top
+  metric_obj = metrics_module.Mean(name=name)
+  result = metric_obj(value)
+  return metric_obj, result
+
+
+def make_variable(name,
+                  shape=None,
+                  dtype=dtypes.float32,
+                  initializer=None,
+                  partition_info=None,
+                  trainable=None,
+                  caching_device=None,
+                  validate_shape=True,
+                  constraint=None,
+                  use_resource=None,
+                  collections=None,
+                  synchronization=tf_variables.VariableSynchronization.AUTO,
+                  aggregation=tf_variables.VariableAggregation.NONE,
+                  partitioner=None):  # pylint: disable=unused-argument
+  """Temporary util to create a variable (relies on `variable_scope.variable`).
+
+  Some reuse-related technicalities prevent us from using
+  `variable_scope.get_variable()` directly, so we use a subcomponent
+  that has fewer constraints (`variable_scope.variable()`).
+
+  In the longer term, it seems like a similar "default variable creator" method
+  should exist in `CheckpointableBase` instead. When this happens, we can get
+  rid of this temporary solution.
+
+  TODO(fchollet): remove this method when no longer needed.
+  TODO(fchollet): handle `partitioner` argument.
+
+  Arguments:
+    name: Variable name.
+    shape: Variable shape.
+    dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
+    initializer: Initializer instance (callable).
+    partition_info: Not handled at this time.
+    trainable: Whether the variable should be part of the layer's
+      "trainable_variables" (e.g. variables, biases)
+      or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
+      Note, if the current variable scope is marked as non-trainable
+      then this parameter is ignored and any added variables are also
+      marked as non-trainable. `trainable` defaults to `True` unless
+      `synchronization` is set to `ON_READ`.
+    caching_device: Passed to `tf.Variable`.
+    validate_shape: Passed to `tf.Variable`.
+    constraint: Constraint instance (callable).
+    use_resource: Whether to use a `ResourceVariable`.
+    collections: List of graph collections keys. The new variable is added to
+      these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`.
+    synchronization: Indicates when a distributed a variable will be
+      aggregated. Accepted values are constants defined in the class
+      `tf.VariableSynchronization`. By default the synchronization is set to
+      `AUTO` and the current `DistributionStrategy` chooses
+      when to synchronize. If `synchronization` is set to `ON_READ`,
+      `trainable` must not be set to `True`.
+    aggregation: Indicates how a distributed variable will be aggregated.
+      Accepted values are constants defined in the class
+      `tf.VariableAggregation`.
+    partitioner: Not handled at this time.
+
+  Returns:
+    Variable instance.
+  """
+  initializing_from_value = False
+  if initializer is not None and not callable(initializer):
+    initializing_from_value = True
+
+  with ops.init_scope():
+    if initializing_from_value:
+      init_val = initializer
+      variable_dtype = None
+    else:
+      # Instantiate initializer if provided initializer is a type object.
+      if isinstance(initializer, type(init_ops.Initializer)):
+        initializer = initializer(dtype=dtype)
+      init_val = lambda: initializer(  # pylint: disable=g-long-lambda
+          shape, dtype=dtype, partition_info=partition_info)
+      variable_dtype = dtype.base_dtype
+  if use_resource is None:
+    use_resource = True
+
+  # TODO(apassos,rohanj) figure out how to remove collections from here so we
+  # can remove the V1.
+  v = tf_variables.VariableV1(
+      initial_value=init_val,
+      name=name,
+      trainable=trainable,
+      caching_device=caching_device,
+      dtype=variable_dtype,
+      validate_shape=validate_shape,
+      constraint=constraint,
+      use_resource=use_resource,
+      collections=collections,
+      synchronization=synchronization,
+      aggregation=aggregation)
+  return v
+
+
+def get_default_graph_uid_map():
+  # TODO(fchollet): refactor this into backend.
+  graph = ops.get_default_graph()
+  name_uid_map = backend.PER_GRAPH_LAYER_NAME_UIDS.get(graph, None)
+  if name_uid_map is None:
+    name_uid_map = collections_lib.defaultdict(int)
+    backend.PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map
+  return name_uid_map
+
+
+def unique_layer_name(name, name_uid_map=None, avoid_names=None, namespace='',
+                      zero_based=False):
+  """Makes a layer name (or arbitrary string) unique within a TensorFlow graph.
+
+  Arguments:
+    name: String name to make unique.
+    name_uid_map: An optional defaultdict(int) to use when creating unique
+      names. If None (default), uses a per-Graph dictionary.
+    avoid_names: An optional set or dict with names which should not be used. If
+      None (default) does not avoid any names.
+    namespace: Gets a name which is unique within the (graph, namespace). Layers
+      which are not Networks use a blank namespace and so get graph-global
+      names.
+    zero_based: If True, name sequences start with no suffix (e.g. "dense",
+      "dense_1"). If False, naming is one-based ("dense_1", "dense_2").
+
+  Returns:
+    Unique string name.
+
+  Example:
+
+  ```python
+  _unique_layer_name('dense')  # dense_1
+  _unique_layer_name('dense')  # dense_2
+  ```
+  """
+  if name_uid_map is None:
+    name_uid_map = get_default_graph_uid_map()
+  if avoid_names is None:
+    avoid_names = set()
+  proposed_name = None
+  while proposed_name is None or proposed_name in avoid_names:
+    name_key = (namespace, name)
+    if zero_based:
+      number = name_uid_map[name_key]
+      if number:
+        proposed_name = name + '_' + str(number)
+      else:
+        proposed_name = name
+      name_uid_map[name_key] += 1
+    else:
+      name_uid_map[name_key] += 1
+      proposed_name = name + '_' + str(name_uid_map[name_key])
+  return proposed_name
+
+
+def collect_previous_mask(input_tensors):
+  """Retrieves the output mask(s) of the previous node.
+
+  Arguments:
+      input_tensors: A tensor or list of tensors.
+
+  Returns:
+      A mask tensor or list of mask tensors.
+  """
+  input_tensors = nest.flatten(input_tensors)
+  masks = []
+  for x in input_tensors:
+    if hasattr(x, '_keras_mask'):
+      mask = x._keras_mask  # pylint: disable=protected-access
+      masks.append(mask)
+    else:
+      masks.append(None)
+  if len(masks) == 1:
+    return masks[0]
+  return masks
+
+
+def have_all_keras_metadata(iterable_or_element):
+  if not isinstance(iterable_or_element, (list, tuple)):
+    iterable = [iterable_or_element]
+  else:
+    iterable = nest.flatten(iterable_or_element)
+  return all(hasattr(x, '_keras_history') for x in iterable)
+
+
+def generate_placeholders_from_shape(shape):
+  return array_ops.placeholder(shape=shape, dtype=backend.floatx())
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index f854cdd4e0..1040fd8ea3 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -36,6 +36,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import layer_utils
@@ -162,7 +163,8 @@ class Network(base_layer.Layer):
 
   @checkpointable.no_automatic_dependency_tracking
   def _init_graph_network(self, inputs, outputs, name=None):
-    self._call_convention = base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT
+    self._call_convention = (base_layer_utils
+                             .CallConvention.EXPLICIT_INPUTS_ARGUMENT)
     # Normalize and set self.inputs, self.outputs.
     if isinstance(inputs, (list, tuple)):
       self.inputs = list(inputs)  # Tensor or list of tensors.
@@ -305,7 +307,7 @@ class Network(base_layer.Layer):
     return self._call_is_graph_friendly
 
   def _determine_call_convention(self, call_argspec):
-    """Decides how `self.call()` is invoked. See base_layer.CallConvention."""
+    """Decides how `self.call()` is invoked. See `CallConvention`."""
     if call_argspec.varargs:
       may_take_single_argument = False
     else:
@@ -337,11 +339,11 @@ class Network(base_layer.Layer):
               "Model.call() takes a single positional argument (to which "
               "inputs are passed by convention) and a separate 'inputs' "
               "argument. Unable to determine which arguments are inputs.")
-        return base_layer.CallConvention.SINGLE_POSITIONAL_ARGUMENT
+        return base_layer_utils.CallConvention.SINGLE_POSITIONAL_ARGUMENT
     if 'inputs' in call_argspec.args:
-      return base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT
+      return base_layer_utils.CallConvention.EXPLICIT_INPUTS_ARGUMENT
     else:
-      return base_layer.CallConvention.POSITIONAL_ARGUMENTS_ARE_INPUTS
+      return base_layer_utils.CallConvention.POSITIONAL_ARGUMENTS_ARE_INPUTS
 
   def _track_layers(self, layers):
     """Add Checkpointable dependencies on a list of Layers."""
@@ -807,10 +809,10 @@ class Network(base_layer.Layer):
         graph = func_graph.FuncGraph('graph')
         with graph.as_default():
           if isinstance(input_shape, list):
-            x = [base_layer.generate_placeholders_from_shape(shape)
+            x = [base_layer_utils.generate_placeholders_from_shape(shape)
                  for shape in input_shape]
           else:
-            x = base_layer.generate_placeholders_from_shape(input_shape)
+            x = base_layer_utils.generate_placeholders_from_shape(input_shape)
 
           kwargs = {}
           num_call_args = len(tf_inspect.getfullargspec(self.call).args)
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index ee0076703f..0101ea8b97 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -31,7 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import initializers
-from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.ops import gradients
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables as tf_variables
@@ -550,7 +550,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
     variable = self._add_variable_with_custom_getter(
         name=name,
         shape=shape,
-        getter=base_layer.make_variable,
+        getter=base_layer_utils.make_variable,
         overwrite=True,
         initializer=initializer,
         dtype=dtype,
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 42086e4c3e..bfe591f875 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -23,6 +23,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.util import function_utils
@@ -242,11 +243,11 @@ class Layer(base_layer.Layer):
   def _make_unique_name(self, name_uid_map=None, avoid_names=None,
                         namespace='', zero_based=False):
     base_name = base_layer.to_snake_case(self.__class__.__name__)
-    name = base_layer.unique_layer_name(base_name,
-                                        name_uid_map=name_uid_map,
-                                        avoid_names=avoid_names,
-                                        namespace=namespace,
-                                        zero_based=zero_based)
+    name = base_layer_utils.unique_layer_name(base_name,
+                                              name_uid_map=name_uid_map,
+                                              avoid_names=avoid_names,
+                                              namespace=namespace,
+                                              zero_based=zero_based)
     return (name, base_name)
 
   @property
-- 
GitLab


From 5d013e9d011a13bcfca8e013690b1f0375e8bf49 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 17:44:45 -0800
Subject: [PATCH 0837/1554] This CL avoids superfluous construction of
 backwards graphs for nested function calls. It delays construction of
 backwards graph to when a tape is actually used to compute a gradient.

PiperOrigin-RevId: 222916814
---
 tensorflow/python/eager/function.py | 115 ++++++++++++++++++++--------
 1 file changed, 82 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index eff7a384b8..68cdb1a871 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -424,7 +424,10 @@ class Function(object):
 
     if (tape.should_record(tensor_inputs) or
         tape.should_record(self._captured_inputs)):
-      return self._backprop_call(args)
+      if context.executing_eagerly():
+        return self._eager_backprop_call(args)
+      else:
+        return self._backprop_call_with_delayed_rewrite(args)
 
     # Only need to override the gradient in graph mode and when we have outputs.
     if context.executing_eagerly() or not self.outputs:
@@ -450,37 +453,40 @@ class Function(object):
       name: The name to register the gradient as.
     """
     @ops.RegisterGradient(name)
-    def grad_fn(op, *doutputs):  # pylint: disable=unused-variable
-      """Gradients of this function."""
-      if self._backward_graph_function is None:
-        self._construct_backprop_function()
+    def _registered_grad_fn(op, *doutputs):  # pylint: disable=unused-variable
+      return self._grad_fn(op, *doutputs)
 
-      # pylint: disable=protected-access
-      self._forward_function.add_to_graph(op.graph)
-      num_inference_outputs = self._inference_function._num_outputs
-
-      # Rewrite an inference call op to be a forward call op
-      if op.get_attr("f").name.encode() == self._inference_function.name:
-        func = attr_value_pb2.AttrValue(
-            func=attr_value_pb2.NameAttrList(
-                name=self._forward_function.name))
-        op._set_attr("f", func)
-        types = attr_value_pb2.AttrValue.ListValue(
-            type=self._forward_function._output_types)
-        op._set_attr("Tout", attr_value_pb2.AttrValue(list=types))
-        for i in range(
-            num_inference_outputs, len(self._forward_function._output_types)):
-          t = ops.Tensor(op, i, self._forward_function._output_types[i])
-          t.set_shape(self._forward_function._output_shapes[i])
-          func_graph_output = self._forward_function._func_graph_outputs[i]
-          custom_gradient.copy_handle_data(func_graph_output, t)
-          op._outputs.append(t)
-      # pylint: enable=protected-access
-      # Compute the gradients using the side outputs
-      side_outputs = op.outputs[num_inference_outputs:]
-      args = list(doutputs[:num_inference_outputs]) + list(side_outputs)
-      return self._backward_graph_function._call_flat(  # pylint: disable=protected-access
-          (a for a in args if a is not None))
+  def _grad_fn(self, op, *doutputs):
+    """Gradients of this function."""
+    if self._backward_graph_function is None:
+      self._construct_backprop_function()
+
+    # pylint: disable=protected-access
+    self._forward_function.add_to_graph(op.graph)
+    num_inference_outputs = self._inference_function._num_outputs
+
+    # Rewrite an inference call op to be a forward call op
+    if op.get_attr("f").name.encode() == self._inference_function.name:
+      func = attr_value_pb2.AttrValue(
+          func=attr_value_pb2.NameAttrList(
+              name=self._forward_function.name))
+      op._set_attr("f", func)
+      types = attr_value_pb2.AttrValue.ListValue(
+          type=self._forward_function._output_types)
+      op._set_attr("Tout", attr_value_pb2.AttrValue(list=types))
+      for i in range(
+          num_inference_outputs, len(self._forward_function._output_types)):
+        t = ops.Tensor(op, i, self._forward_function._output_types[i])
+        t.set_shape(self._forward_function._output_shapes[i])
+        func_graph_output = self._forward_function._func_graph_outputs[i]
+        custom_gradient.copy_handle_data(func_graph_output, t)
+        op._outputs.append(t)
+    # pylint: enable=protected-access
+    # Compute the gradients using the side outputs
+    side_outputs = op.outputs[num_inference_outputs:]
+    args = list(doutputs[:num_inference_outputs]) + list(side_outputs)
+    return self._backward_graph_function._call_flat(  # pylint: disable=protected-access
+        (a for a in args if a is not None))
 
   @property
   def name(self):
@@ -623,10 +629,13 @@ class Function(object):
         self._func_graph.outputs + backwards_graph_captures,
         forward_function_attr)
 
-  def _backprop_call(self, args):
+  def _eager_backprop_call(self, args):
     """Calls the forward function and records the result on a tape.
 
-    (Only records results on a tape if the function has outputs)
+    This method fully constructs the forward and backward functions before
+    calling the function and recording them on the tape.
+
+    (Only records results on a tape if the function has outputs).
 
     Args:
       args: All inputs to the function, including resolved captured inputs
@@ -668,6 +677,46 @@ class Function(object):
                           args, backward_function)
     return self._build_call_outputs(real_outputs)
 
+  def _backprop_call_with_delayed_rewrite(self, args):
+    """Calls the inference function and records the result on a tape.
+
+    The recorded backwards function will construct the backwards graph and
+    rewrite the inference function to the forward function. This only happens
+    if the recorded backwards function ends up being used to compute gradients.
+
+    This approach avoids constructing unnecessary graphs, but it only works if
+    we are calling this function when not executing eagerly.
+
+    (Only records results on a tape if the function has outputs)
+
+    Args:
+      args: All inputs to the function, including resolved captured inputs
+
+    Returns:
+      The call output.
+    """
+    ctx = context.context()
+
+    if not self._gradient_name:
+      self._gradient_name = "PartitionedCall-%s" % ops.uid()
+      self._register_gradient(self._gradient_name)
+    with ops.get_default_graph().gradient_override_map(
+        {"PartitionedCall": self._gradient_name,
+         "StatefulPartitionedCall": self._gradient_name}):
+      outputs = self._inference_function.call(ctx, args)
+
+    if isinstance(outputs, ops.Operation) or outputs is None:
+      return outputs
+
+    call_op = outputs[0].op
+
+    def backward_function(*args):
+      return self._grad_fn(call_op, *args)
+
+    tape.record_operation(self._inference_function.signature.name, outputs,
+                          args, backward_function)
+    return self._build_call_outputs(outputs)
+
   def _build_call_outputs(self, result):
     """Maps the fdef output list to actual output structure.
 
-- 
GitLab


From ab7cfa02796670d5061da81e683914b28218749f Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 26 Nov 2018 18:06:10 -0800
Subject: [PATCH 0838/1554] Add run_deprecated_v1 to test_util

PiperOrigin-RevId: 222919298
---
 tensorflow/python/framework/test_util.py      | 34 ++++++++++++++++++-
 tensorflow/python/framework/test_util_test.py |  2 +-
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 7c486b2cbe..a034032e39 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -960,7 +960,7 @@ def run_in_graph_and_eager_modes(func=None,
   def decorator(f):
     if tf_inspect.isclass(f):
       raise ValueError(
-          "`run_test_in_graph_and_eager_modes` only supports test methods. "
+          "`run_in_graph_and_eager_modes` only supports test methods. "
           "Did you mean to use `run_all_in_graph_and_eager_modes`?")
 
     def decorated(self, *args, **kwargs):
@@ -1005,6 +1005,38 @@ def run_in_graph_and_eager_modes(func=None,
   return decorator
 
 
+def run_deprecated_v1(func=None):
+  """Execute the decorated test in graph mode.
+
+  This function returns a decorator intended to be applied to tests that have
+  not been updated to a style that is compatible with both TensorFlow 1.x and
+  2.x. When this decorated is applied, the test body will be run in
+  an environment where API calls construct graphs instead of executing eagerly.
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+  Returns:
+    Returns a decorator that will run the decorated test method in graph mode.
+  """
+
+  def decorator(f):
+    if tf_inspect.isclass(f):
+      raise ValueError("`run_deprecated_v1` only supports test methods.")
+
+    def decorated(self, *args, **kwargs):
+      with context.graph_mode():
+        f(self, *args, **kwargs)
+
+    return decorated
+
+  if func is not None:
+    return decorator(func)
+
+  return decorator
+
+
 @tf_export("test.is_gpu_available")
 def is_gpu_available(cuda_only=False, min_cuda_compute_capability=None):
   """Returns whether TensorFlow can access a GPU.
diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py
index cbefe86481..2a37253db6 100644
--- a/tensorflow/python/framework/test_util_test.py
+++ b/tensorflow/python/framework/test_util_test.py
@@ -681,7 +681,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIsNone(test_util.get_node_def_from_graph("bar", graph_def))
 
   def test_run_in_eager_and_graph_modes_test_class(self):
-    msg = "`run_test_in_graph_and_eager_modes` only supports test methods.*"
+    msg = "`run_in_graph_and_eager_modes` only supports test methods.*"
     with self.assertRaisesRegexp(ValueError, msg):
       @test_util.run_in_graph_and_eager_modes()
       class Foo(object):
-- 
GitLab


From 290c330ee86ea8068d8a274d8202999f53eb2b35 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Mon, 26 Nov 2018 18:30:06 -0800
Subject: [PATCH 0839/1554] TF 2.0: Change the semantics of tf.nn.dropout to be
 consistent with tf.keras.layers.Dropout.

PiperOrigin-RevId: 222921458
---
 .../python/kernel_tests/core_rnn_cell_test.py |  6 +-
 .../rnn/python/kernel_tests/core_rnn_test.py  |  2 +-
 tensorflow/python/ops/nn_ops.py               | 68 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 27 +++++++-
 .../tools/compatibility/tf_upgrade_v2_test.py | 16 +++++
 6 files changed, 114 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
index 245fa68eae..7d57b0413a 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
@@ -906,7 +906,7 @@ class DropoutWrapperTest(test.TestCase):
 
   def testDropoutWrapperKeepNoOutput(self):
     keep_all = variable_scope.get_variable("all", initializer=1.0)
-    keep_none = variable_scope.get_variable("none", initializer=1e-10)
+    keep_none = variable_scope.get_variable("none", initializer=1e-6)
     res = self._testDropoutWrapper(
         input_keep_prob=keep_all,
         output_keep_prob=keep_none,
@@ -922,7 +922,7 @@ class DropoutWrapperTest(test.TestCase):
 
   def testDropoutWrapperKeepNoStateExceptLSTMCellMemory(self):
     keep_all = variable_scope.get_variable("all", initializer=1.0)
-    keep_none = variable_scope.get_variable("none", initializer=1e-10)
+    keep_none = variable_scope.get_variable("none", initializer=1e-6)
     # Even though we dropout state, by default DropoutWrapper never
     # drops out the memory ("c") term of an LSTMStateTuple.
     res = self._testDropoutWrapper(
@@ -943,7 +943,7 @@ class DropoutWrapperTest(test.TestCase):
 
   def testDropoutWrapperKeepNoInput(self):
     keep_all = variable_scope.get_variable("all", initializer=1.0)
-    keep_none = variable_scope.get_variable("none", initializer=1e-10)
+    keep_none = variable_scope.get_variable("none", initializer=1e-6)
     true_full_output = np.array(
         [[[0.751109, 0.751109, 0.751109]], [[0.895509, 0.895509, 0.895509]]],
         dtype=np.float32)
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
index 5cba54dd3d..ef372b947c 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
@@ -227,7 +227,7 @@ class RNNTest(test.TestCase):
   def testDropout(self):
     cell = Plus1RNNCell()
     full_dropout_cell = rnn_cell.DropoutWrapper(
-        cell, input_keep_prob=1e-12, seed=0)
+        cell, input_keep_prob=1e-6, seed=0)
     (name, dep), = full_dropout_cell._checkpoint_dependencies
     self.assertIs(dep, cell)
     self.assertEqual("cell", name)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 0e39dd0fa0..5655960aae 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2791,7 +2791,7 @@ def _get_noise_shape(x, noise_shape):
   return noise_shape
 
 
-@tf_export("nn.dropout")
+@tf_export(v1=["nn.dropout"])
 def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):  # pylint: disable=invalid-name
   """Computes dropout.
 
@@ -2850,8 +2850,74 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):  # pylint: di
       if tensor_util.constant_value(keep_prob) == 1:
         return x
 
+    rate = 1 - keep_prob
+
+  return dropout_v2(x, rate, noise_shape=noise_shape, seed=seed, name=name)
+
+
+@tf_export("nn.dropout", v1=[])
+def dropout_v2(x, rate, noise_shape=None, seed=None, name=None):  # pylint: disable=invalid-name
+  """Computes dropout.
+
+  With probability `rate`, drops elements of `x`. Input that are kept are
+  scaled up by `1 / (1 - rate)`, otherwise outputs `0`.  The scaling is so that
+  the expected sum is unchanged.
+
+  By default, each element is kept or dropped independently.  If `noise_shape`
+  is specified, it must be
+  [broadcastable](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+  to the shape of `x`, and only dimensions with `noise_shape[i] == shape(x)[i]`
+  will make independent decisions.  For example, if `shape(x) = [k, l, m, n]`
+  and `noise_shape = [k, 1, 1, n]`, each batch and channel component will be
+  kept independently and each row and column will be kept or not kept together.
+
+  Args:
+    x: A floating point tensor.
+    rate: A scalar `Tensor` with the same type as x. The probability
+      that each element is dropped. For example, setting rate=0.1 would drop
+      10% of input elements.
+    noise_shape: A 1-D `Tensor` of type `int32`, representing the
+      shape for randomly generated keep/drop flags.
+    seed: A Python integer. Used to create random seeds. See
+      `tf.set_random_seed`
+      for behavior.
+    name: A name for this operation (optional).
+
+  Returns:
+    A Tensor of the same shape of `x`.
+
+  Raises:
+    ValueError: If `keep_prob` is not in `(0, 1]` or if `x` is not a floating
+      point tensor.
+  """
+  with ops.name_scope(name, "dropout", [x]) as name:
+    x = ops.convert_to_tensor(x, name="x")
+    if not x.dtype.is_floating:
+      raise ValueError("x has to be a floating point tensor since it's going to"
+                       " be scaled. Got a %s tensor instead." % x.dtype)
+    if isinstance(rate, numbers.Real) and not 0 <= rate < 1:
+      raise ValueError("rate must be a scalar tensor or a float in the "
+                       "range [0, 1), got %g" % rate)
+
+    # Early return if nothing needs to be dropped.
+    if isinstance(rate, float) and rate == 0:
+      return x
+    if context.executing_eagerly():
+      if isinstance(rate, ops.EagerTensor):
+        if rate.numpy() == 0:
+          return x
+    else:
+      rate = ops.convert_to_tensor(
+          rate, dtype=x.dtype, name="rate")
+      rate.get_shape().assert_is_compatible_with(tensor_shape.scalar())
+
+      # Do nothing if we know rate == 0
+      if tensor_util.constant_value(rate) == 0:
+        return x
+
     noise_shape = _get_noise_shape(x, noise_shape)
 
+    keep_prob = 1 - rate
     # uniform [keep_prob, 1.0 + keep_prob)
     random_tensor = keep_prob
     random_tensor += random_ops.random_uniform(
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 77b5887c1f..0bfea74e40 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -126,7 +126,7 @@ tf_module {
   }
   member_method {
     name: "dropout"
-    argspec: "args=[\'x\', \'keep_prob\', \'noise_shape\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'rate\', \'noise_shape\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "elu"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index e8ea9784af..178521f6b3 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -559,7 +559,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     }
 
     # Specially handled functions.
-    self.function_handle = {}
+    self.function_handle = {
+        "tf.nn.dropout": self._dropout_handler,
+    }
 
     decay_function_comment = (
         "ERROR: <function name> has been changed to return a callable instead "
@@ -668,6 +670,29 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         if name not in self.function_warnings and name not in excluded_renames
     }
 
+  @staticmethod
+  def _dropout_handler(file_edit_recorder, node):
+    if len(node.args) < 2:
+      comment = ("ERROR: tf.nn.dropout did not take arguments, so automatic "
+                 "transformation was disabled. tf.nn.dropout has changed "
+                 "the semantics of the second argument.")
+      file_edit_recorder.add(
+          comment,
+          node.lineno,
+          node.col_offset,
+          "tf.nn.dropout",
+          "tf.nn.dropout",
+          error="tf.nn.dropout requires manual check.")
+    else:
+      comment = ("WARNING: tf.nn.dropout has changed the semantics of the "
+                 "second argument. Please check the transformation.\n")
+      file_edit_recorder.add(
+          comment,
+          node.args[1].lineno,
+          node.args[1].col_offset,
+          "",
+          "1 - ")
+
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 7baa1cafdd..65d77a2c9a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -116,6 +116,22 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
       self.assertIn("loss_reduction has been changed", report)
 
+  def testDropout(self):
+    text = "tf.nn.dropout(x, keep_prob, name=\"foo\")\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.nn.dropout(x, 1 - keep_prob, name=\"foo\")\n",
+    )
+
+    text = "tf.nn.dropout(x)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, text)
+    self.assertEqual(
+        errors,
+        ["test.py:1: tf.nn.dropout requires manual check."]
+    )
+
   def testCountNonZeroChanges(self):
     text = (
         "tf.math.count_nonzero(input_tensor=input, dtype=dtype, name=name, "
-- 
GitLab


From 9f3d9b54478359e9258895d3335731183bfd8e45 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 26 Nov 2018 18:41:51 -0800
Subject: [PATCH 0840/1554] Automated rollback of commit
 5a08ce9bd17c2678c80b0af70996ebbc4879c937

PiperOrigin-RevId: 222922435
---
 tensorflow/stream_executor/BUILD              | 11 --------
 tensorflow/stream_executor/dnn.h              |  7 ++---
 tensorflow/stream_executor/logging.proto      | 19 -------------
 .../stream_executor/stream_executor_pimpl.cc  | 28 +------------------
 4 files changed, 4 insertions(+), 61 deletions(-)
 delete mode 100644 tensorflow/stream_executor/logging.proto

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 74fec09d80..2526e1adaa 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -23,14 +23,6 @@ tf_proto_library(
     protodeps = tf_additional_all_protos(),
 )
 
-tf_proto_library(
-    name = "logging_proto",
-    srcs = ["logging.proto"],
-    cc_api_version = 2,
-    default_header = True,
-    protodeps = tf_additional_all_protos(),
-)
-
 cc_library(
     name = "stream_executor_impl",
     srcs = glob(
@@ -54,9 +46,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":dnn_proto_cc_impl",
-        ":logging_proto_cc_impl",
         "//tensorflow/core:lib",
-        "//tensorflow/core:logger",
         "//tensorflow/core:ptr_util",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/strings",
@@ -73,7 +63,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":dnn_proto_cc",
-        ":logging_proto_cc",
         "//tensorflow/core:lib",
         "//tensorflow/core:ptr_util",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 43738d2d1d..c044a356ef 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -906,10 +906,9 @@ class VersionInfo {
  public:
   VersionInfo(int major = 0, int minor = 0, int patch = 0)
       : major_(major), minor_(minor), patch_(patch) {}
-  int major_version() const { return major_; }
-  int minor_version() const { return minor_; }
-  int patch() const { return patch_; }
-
+  int major_version() { return major_; }
+  int minor_version() { return minor_; }
+  int patch() { return patch_; }
  private:
   int major_;
   int minor_;
diff --git a/tensorflow/stream_executor/logging.proto b/tensorflow/stream_executor/logging.proto
deleted file mode 100644
index 2c75500cda..0000000000
--- a/tensorflow/stream_executor/logging.proto
+++ /dev/null
@@ -1,19 +0,0 @@
-syntax = "proto3";
-
-package stream_executor;
-
-message CudnnVersion {
-  int32 major = 1;
-  int32 minor = 2;
-  int32 patch = 3;
-};
-
-message ComputeCapability {
-  int32 major = 1;
-  int32 minor = 2;
-}
-
-message CudaInfo {
-  CudnnVersion cudnn_version = 1;
-  ComputeCapability compute_capability = 2;
-}
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index 86bc4ab7d0..d1d0bd9bc2 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include <utility>
 
 #include "absl/strings/str_cat.h"
-#include "tensorflow/core/platform/logger.h"
 #include "tensorflow/core/util/env_var.h"
 #include "tensorflow/stream_executor/blas.h"
 #include "tensorflow/stream_executor/fft.h"
@@ -34,7 +33,6 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
 #include "tensorflow/stream_executor/lib/threadpool.h"
-#include "tensorflow/stream_executor/logging.pb.h"
 #include "tensorflow/stream_executor/platform/port.h"
 #include "tensorflow/stream_executor/rng.h"
 #include "tensorflow/stream_executor/stream_executor_internal.h"
@@ -219,31 +217,7 @@ StreamExecutor::~StreamExecutor() {
 port::Status StreamExecutor::Init(int device_ordinal,
                                   DeviceOptions device_options) {
   device_ordinal_ = device_ordinal;
-  TF_RETURN_IF_ERROR(
-      implementation_->Init(device_ordinal, std::move(device_options)));
-
-  if (platform_kind_ == PlatformKind::kCuda) {
-    CudaInfo info;
-
-    int cc_major, cc_minor;
-    GetDeviceDescription().cuda_compute_capability(&cc_major, &cc_minor);
-    info.mutable_compute_capability()->set_major(cc_major);
-    info.mutable_compute_capability()->set_minor(cc_minor);
-
-    if (auto *dnn = AsDnn()) {
-      port::StatusOr<dnn::VersionInfo> version_or = dnn->GetVersion();
-      if (version_or.ok()) {
-        const auto &version = version_or.ValueOrDie();
-        info.mutable_cudnn_version()->set_major(version.major_version());
-        info.mutable_cudnn_version()->set_minor(version.minor_version());
-        info.mutable_cudnn_version()->set_patch(version.patch());
-      }
-    }
-
-    tensorflow::Logger::Singleton()->LogProto(info);
-  }
-
-  return port::Status::OK();
+  return implementation_->Init(device_ordinal, std::move(device_options));
 }
 
 port::Status StreamExecutor::Init() {
-- 
GitLab


From 516e122b4c0f1ac83ab5abb5fb993277c8ebf2db Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 19:00:56 -0800
Subject: [PATCH 0841/1554] Close resource file after loading model file.

PiperOrigin-RevId: 222923990
---
 .../android/smartreply/SmartReplyClient.java  | 21 ++++++++-----------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java b/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java
index d5b1ac0ffb..fbd75051e7 100644
--- a/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java
+++ b/tensorflow/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java
@@ -90,29 +90,26 @@ public class SmartReplyClient implements AutoCloseable {
   }
 
   private MappedByteBuffer loadModelFile() throws IOException {
-    AssetFileDescriptor fileDescriptor = context.getAssets().openFd(MODEL_PATH);
-    FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
-    try {
+    try (AssetFileDescriptor fileDescriptor = context.getAssets().openFd(MODEL_PATH);
+        FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor())) {
       FileChannel fileChannel = inputStream.getChannel();
       long startOffset = fileDescriptor.getStartOffset();
       long declaredLength = fileDescriptor.getDeclaredLength();
       return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
-    } finally {
-      inputStream.close();
     }
   }
 
   private String[] loadBackoffList() throws IOException {
     List<String> labelList = new ArrayList<String>();
-    BufferedReader reader =
-        new BufferedReader(new InputStreamReader(context.getAssets().open(BACKOFF_PATH)));
-    String line;
-    while ((line = reader.readLine()) != null) {
-      if (!line.isEmpty()) {
-        labelList.add(line);
+    try (BufferedReader reader =
+        new BufferedReader(new InputStreamReader(context.getAssets().open(BACKOFF_PATH)))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        if (!line.isEmpty()) {
+          labelList.add(line);
+        }
       }
     }
-    reader.close();
     String[] ans = new String[labelList.size()];
     labelList.toArray(ans);
     return ans;
-- 
GitLab


From 75f6bf69fc4ef363a906ac5f958af1ae40a328d8 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Mon, 26 Nov 2018 19:03:46 -0800
Subject: [PATCH 0842/1554] Remove colocate_gradients_with_ops in TF 2.0 and
 default it to True.

PiperOrigin-RevId: 222924432
---
 .../contrib/optimizer_v2/optimizer_v2.py      |  16 +--
 tensorflow/python/ops/gradients_impl.py       | 115 +++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.pbtxt      |   2 +-
 tensorflow/tools/compatibility/ast_edits.py   |  11 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  24 ++++
 .../tools/compatibility/tf_upgrade_v2_test.py |  34 ++++++
 6 files changed, 184 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index a72db5e12f..747f5b9b8b 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -658,7 +658,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
                var_list=None,
                gate_gradients=GATE_OP,
                aggregation_method=None,
-               colocate_gradients_with_ops=False,
                name=None,
                grad_loss=None,
                stop_gradients=None,
@@ -681,8 +680,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
         `GATE_NONE`, `GATE_OP`, or  `GATE_GRAPH`.
       aggregation_method: Specifies the method used to combine gradient terms.
         Valid values are defined in the class `AggregationMethod`.
-      colocate_gradients_with_ops: If True, try colocating gradients with the
-        corresponding op.
       name: Optional name for the returned operation.
       grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.
       stop_gradients: Optional. A Tensor or list of tensors not to differentiate
@@ -705,8 +702,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
     Minimization (and gradient computation) is done with respect to the
     elements of `var_list` if not None, else with respect to any trainable
     variables created during the execution of the `loss` function.
-    `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
-    `grad_loss` are ignored when eager execution is enabled.
+    `gate_gradients`, `aggregation_method`, and `grad_loss` are ignored when
+    eager execution is enabled.
     @end_compatibility
     """
     grads_and_vars = self.compute_gradients(
@@ -714,7 +711,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
         var_list=var_list,
         gate_gradients=gate_gradients,
         aggregation_method=aggregation_method,
-        colocate_gradients_with_ops=colocate_gradients_with_ops,
         grad_loss=grad_loss,
         stop_gradients=stop_gradients,
         scale_loss_by_num_replicas=scale_loss_by_num_replicas)
@@ -734,7 +730,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
                         var_list=None,
                         gate_gradients=GATE_OP,
                         aggregation_method=None,
-                        colocate_gradients_with_ops=False,
                         grad_loss=None,
                         stop_gradients=None,
                         scale_loss_by_num_replicas=None):
@@ -757,8 +752,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
         `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`.
       aggregation_method: Specifies the method used to combine gradient terms.
         Valid values are defined in the class `AggregationMethod`.
-      colocate_gradients_with_ops: If True, try colocating gradients with the
-        corresponding op.
       grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.
       stop_gradients: Optional. A Tensor or list of tensors not to differentiate
         through.
@@ -777,8 +770,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
         not callable.
 
     @compatibility(eager)
-    When eager execution is enabled, `gate_gradients`, `aggregation_method`,
-    and `colocate_gradients_with_ops` are ignored.
+    When eager execution is enabled, `gate_gradients`, and `aggregation_method`
+    are ignored.
     @end_compatibility
     """
     # TODO(josh11b): Test that we handle weight decay in a reasonable way.
@@ -833,7 +826,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
         grad_ys=grad_loss,
         gate_gradients=(gate_gradients == optimizer_v1.Optimizer.GATE_OP),
         aggregation_method=aggregation_method,
-        colocate_gradients_with_ops=colocate_gradients_with_ops,
         stop_gradients=stop_gradients)
     if gate_gradients == optimizer_v1.Optimizer.GATE_GRAPH:
       grads = control_flow_ops.tuple(grads)
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index 27cee9113d..c8f5cb8349 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -539,7 +539,7 @@ def _Consumers(t, func_graphs):
   return consumers
 
 
-@tf_export("gradients")
+@tf_export(v1=["gradients"])
 def gradients(ys,
               xs,
               grad_ys=None,
@@ -655,6 +655,119 @@ def gradients(ys,
                             unconnected_gradients)
 
 
+@tf_export("gradients", v1=[])
+def gradients_v2(ys,  # pylint: disable=invalid-name
+                 xs,
+                 grad_ys=None,
+                 name="gradients",
+                 gate_gradients=False,
+                 aggregation_method=None,
+                 stop_gradients=None,
+                 unconnected_gradients=UnconnectedGradients.NONE):
+  """Constructs symbolic derivatives of sum of `ys` w.r.t. x in `xs`.
+
+  `ys` and `xs` are each a `Tensor` or a list of tensors.  `grad_ys`
+  is a list of `Tensor`, holding the gradients received by the
+  `ys`. The list must be the same length as `ys`.
+
+  `gradients()` adds ops to the graph to output the derivatives of `ys` with
+  respect to `xs`.  It returns a list of `Tensor` of length `len(xs)` where
+  each tensor is the `sum(dy/dx)` for y in `ys`.
+
+  `grad_ys` is a list of tensors of the same length as `ys` that holds
+  the initial gradients for each y in `ys`.  When `grad_ys` is None,
+  we fill in a tensor of '1's of the shape of y for each y in `ys`.  A
+  user can provide their own initial `grad_ys` to compute the
+  derivatives using a different initial gradient for each y (e.g., if
+  one wanted to weight the gradient differently for each value in
+  each y).
+
+  `stop_gradients` is a `Tensor` or a list of tensors to be considered constant
+  with respect to all `xs`. These tensors will not be backpropagated through,
+  as though they had been explicitly disconnected using `stop_gradient`.  Among
+  other things, this allows computation of partial derivatives as opposed to
+  total derivatives. For example:
+
+  ```python
+  a = tf.constant(0.)
+  b = 2 * a
+  g = tf.gradients(a + b, [a, b], stop_gradients=[a, b])
+  ```
+
+  Here the partial derivatives `g` evaluate to `[1.0, 1.0]`, compared to the
+  total derivatives `tf.gradients(a + b, [a, b])`, which take into account the
+  influence of `a` on `b` and evaluate to `[3.0, 1.0]`.  Note that the above is
+  equivalent to:
+
+  ```python
+  a = tf.stop_gradient(tf.constant(0.))
+  b = tf.stop_gradient(2 * a)
+  g = tf.gradients(a + b, [a, b])
+  ```
+
+  `stop_gradients` provides a way of stopping gradient after the graph has
+  already been constructed, as compared to `tf.stop_gradient` which is used
+  during graph construction.  When the two approaches are combined,
+  backpropagation stops at both `tf.stop_gradient` nodes and nodes in
+  `stop_gradients`, whichever is encountered first.
+
+  All integer tensors are considered constant with respect to all `xs`, as if
+  they were included in `stop_gradients`.
+
+  `unconnected_gradients` determines the value returned for each x in xs if it
+  is unconnected in the graph to ys. By default this is None to safeguard
+  against errors. MAthematically these gradients are zero which can be requested
+  using the `'zero'` option. `tf.UnconnectedGradients` provides the
+  following options and behaviors:
+
+  ```python
+  a = tf.ones([1, 2])
+  b = tf.ones([3, 1])
+  g1 = tf.gradients([b], [a], unnconnected_gradients='none')
+  sess.run(g1)  # [None]
+
+  g2 = tf.gradients([b], [a], unconnected_gradients='zero')
+  sess.run(g2)  # [array([[0., 0.]], dtype=float32)]
+  ```
+
+
+  Args:
+    ys: A `Tensor` or list of tensors to be differentiated.
+    xs: A `Tensor` or list of tensors to be used for differentiation.
+    grad_ys: Optional. A `Tensor` or list of tensors the same size as
+      `ys` and holding the gradients computed for each y in `ys`.
+    name: Optional name to use for grouping all the gradient ops together.
+      defaults to 'gradients'.
+    gate_gradients: If True, add a tuple around the gradients returned
+      for an operations.  This avoids some race conditions.
+    aggregation_method: Specifies the method used to combine gradient terms.
+      Accepted values are constants defined in the class `AggregationMethod`.
+    stop_gradients: Optional. A `Tensor` or list of tensors not to differentiate
+      through.
+    unconnected_gradients: Optional. Specifies the gradient value returned when
+      the given input tensors are unconnected. Accepted values are constants
+      defined in the class `tf.UnconnectedGradients` and the default value is
+      `none`.
+
+  Returns:
+    A list of `sum(dy/dx)` for each x in `xs`.
+
+  Raises:
+    LookupError: if one of the operations between `x` and `y` does not
+      have a registered gradient function.
+    ValueError: if the arguments are invalid.
+    RuntimeError: if called in Eager mode.
+
+  """
+  # Creating the gradient graph for control flow mutates Operations.
+  # _mutation_lock ensures a Session.run call cannot occur between creating and
+  # mutating new ops.
+  with ops.get_default_graph()._mutation_lock():  # pylint: disable=protected-access
+    return _GradientsHelper(ys, xs, grad_ys, name, True, gate_gradients,
+                            aggregation_method, stop_gradients,
+                            unconnected_gradients)
+
+
 def _GradientsHelper(ys,
                      xs,
                      grad_ys=None,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a380b0d99f..a48095eb91 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -686,7 +686,7 @@ tf_module {
   }
   member_method {
     name: "gradients"
-    argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], "
+    argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], "
   }
   member_method {
     name: "greater"
diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 56c67b8356..5edb4d4759 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -209,11 +209,11 @@ class _ASTCallVisitor(ast.NodeVisitor):
     items = []
     while not isinstance(curr, ast.Name):
       if not isinstance(curr, ast.Attribute):
-        return None
+        return None, None
       items.append(curr.attr)
       curr = curr.value
     items.append(curr.id)
-    return ".".join(reversed(items))
+    return ".".join(reversed(items)), items[0]
 
   def _find_true_position(self, node):
     """Return correct line number and column offset for a given node.
@@ -278,7 +278,7 @@ class _ASTCallVisitor(ast.NodeVisitor):
     """
 
     # Find a simple attribute name path e.g. "tf.foo.bar"
-    full_name = self._get_attribute_full_path(node.func)
+    full_name, name = self._get_attribute_full_path(node.func)
 
     # Make sure the func is marked as being part of a call
     node.func.is_function_for_call = True
@@ -286,6 +286,9 @@ class _ASTCallVisitor(ast.NodeVisitor):
     if full_name:
       # Call special handlers
       function_handles = self._api_change_spec.function_handle
+      glob_name = "*.{}".format(name)
+      if glob_name in function_handles:
+        function_handles[glob_name](self._file_edit, node)
       if full_name in function_handles:
         function_handles[full_name](self._file_edit, node)
 
@@ -358,7 +361,7 @@ class _ASTCallVisitor(ast.NodeVisitor):
     Args:
       node: Node that is of type ast.Attribute
     """
-    full_name = self._get_attribute_full_path(node)
+    full_name, _ = self._get_attribute_full_path(node)
     if full_name:
       self._rename_functions(node, full_name)
       self._print_warning_for_function(node, full_name)
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 178521f6b3..3588db395e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -561,6 +561,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Specially handled functions.
     self.function_handle = {
         "tf.nn.dropout": self._dropout_handler,
+        "tf.gradients": self._colocate_handler("tf.gradients"),
+        "*.minimize": self._colocate_handler("Optimizer.minimize"),
+        "*.compute_gradients":
+            self._colocate_handler("Optimizer.compute_gradients"),
     }
 
     decay_function_comment = (
@@ -693,6 +697,26 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
           "",
           "1 - ")
 
+  @staticmethod
+  def _colocate_handler(name):
+    def _helper(file_edit_recorder, node):
+      for keyword in node.keywords:
+        if keyword.arg == "colocate_gradients_with_ops":
+          # TODO(jhseu): Since ast_edit.py does string replacement, there's no
+          # straightforward way to remove the argument. Try to fix before 2.0 is
+          # final.
+          comment = ("For tf.gradients and tf.Optimizer.minimize, "
+                     "colocate_gradients_with_op has been removed and now "
+                     "defaults to True.")
+          file_edit_recorder.add(
+              comment,
+              node.lineno,
+              node.col_offset,
+              "",
+              "",
+              error="{} requires manual check.".format(name))
+    return _helper
+
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 65d77a2c9a..bda748f274 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -178,6 +178,40 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     )
     self.assertEqual(new_text, expected_text)
 
+  def testColocateGradientsWithOps(self):
+    text = "tf.gradients(a, foo=False)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(text, new_text)
+    self.assertEqual(errors, [])
+
+    text = "tf.gradients(a, colocate_gradients_with_ops=False)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(text, new_text)
+    self.assertEqual(errors, ["test.py:1: tf.gradients requires manual check."])
+
+    text = "optimizer.minimize(a, foo=False)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(text, new_text)
+    self.assertEqual(errors, [])
+
+    text = "optimizer.minimize(a, colocate_gradients_with_ops=False)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(text, new_text)
+    self.assertEqual(errors,
+                     ["test.py:1: Optimizer.minimize requires manual check."])
+
+    text = "optimizer.compute_gradients(a, foo=False)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(text, new_text)
+    self.assertEqual(errors, [])
+
+    text = "optimizer.compute_gradients(a, colocate_gradients_with_ops=False)\n"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(text, new_text)
+    self.assertEqual(errors,
+                     ["test.py:1: Optimizer.compute_gradients "
+                      "requires manual check."])
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 6d3d0de39f3eaecdbf7fbdb52a7a02b2539efeeb Mon Sep 17 00:00:00 2001
From: qiezi <qiezi@pinduoduo.com>
Date: Tue, 27 Nov 2018 11:18:09 +0800
Subject: [PATCH 0843/1554] #23987: LARSOptimizer initializes
 _learning_rate_tensor and _momentum_tensor in _prepare method

---
 .../contrib/opt/python/training/lars_optimizer.py    | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/contrib/opt/python/training/lars_optimizer.py b/tensorflow/contrib/opt/python/training/lars_optimizer.py
index a8dafd9a4c..205d6c3949 100644
--- a/tensorflow/contrib/opt/python/training/lars_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/lars_optimizer.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -162,3 +163,14 @@ class LARSOptimizer(optimizer.Optimizer):
         math_ops.cast(self._momentum_tensor, grad.dtype),
         use_locking=self._use_locking,
         use_nesterov=self._use_nesterov)
+
+  def _prepare(self):
+    learning_rate = self._learning_rate
+    if callable(learning_rate):
+      learning_rate = learning_rate()
+    self._learning_rate_tensor = ops.convert_to_tensor(learning_rate,
+                                                       name="learning_rate")
+    momentum = self._momentum
+    if callable(momentum):
+      momentum = momentum()
+    self._momentum_tensor = ops.convert_to_tensor(momentum, name="momentum")
\ No newline at end of file
-- 
GitLab


From 82fd59f0e4ba87e6410002ee5017b411568a0060 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 19:37:28 -0800
Subject: [PATCH 0844/1554] Add vmodule support to TF OSS logging.

Setting the environment variable TF_CPP_VMODULE=foo=7,bar=7 will cause the
"foo" and "bar" translation units (foo.{h.cc},bar.{h.cc}) to have a VLOG level
of 7, meaning VLOG(0) to VLOG(7) in those translation units will be logged to
stderr (as LOG(ERROR) does).

PiperOrigin-RevId: 222927031
---
 tensorflow/core/BUILD                       |   2 +
 tensorflow/core/platform/default/logging.cc | 119 +++++++++++++++++---
 tensorflow/core/platform/default/logging.h  |  44 ++++++--
 3 files changed, 143 insertions(+), 22 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index bd5bbaa310..437c368356 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3428,6 +3428,8 @@ tf_cc_tests(
         "platform/profile_utils/cpu_utils_test.cc",
         "platform/stacktrace_handler_test.cc",
         "platform/subprocess_test.cc",
+        "platform/vmodule_benchmark_test.cc",
+        "platform/vmodule_test.cc",
     ],
     deps = [
         ":lib",
diff --git a/tensorflow/core/platform/default/logging.cc b/tensorflow/core/platform/default/logging.cc
index 133ae45a55..26bd8542fd 100644
--- a/tensorflow/core/platform/default/logging.cc
+++ b/tensorflow/core/platform/default/logging.cc
@@ -21,18 +21,18 @@ limitations under the License.
 #include <android/log.h>
 #include <iostream>
 #include <sstream>
-#include <cstring>
 #endif
 
 #include <stdlib.h>
+#include <string.h>
 #include <time.h>
 
+#include <string>
+#include <unordered_map>
+
 namespace tensorflow {
 namespace internal {
 
-LogMessage::LogMessage(const char* fname, int line, int severity)
-    : fname_(fname), line_(line), severity_(severity) {}
-
 #if defined(PLATFORM_POSIX_ANDROID)
 void LogMessage::GenerateLogMessage() {
   int android_log_level;
@@ -94,24 +94,90 @@ void LogMessage::GenerateLogMessage() {
 
 namespace {
 
+int ParseInteger(const char* str, size_t size) {
+  // Ideally we would use env_var / safe_strto64, but it is
+  // hard to use here without pulling in a lot of dependencies,
+  // so we use std:istringstream instead
+  string integer_str(str, size);
+  std::istringstream ss(integer_str);
+  int level = 0;
+  ss >> level;
+  return level;
+}
+
 // Parse log level (int64) from environment variable (char*)
 int64 LogLevelStrToInt(const char* tf_env_var_val) {
   if (tf_env_var_val == nullptr) {
     return 0;
   }
+  return ParseInteger(tf_env_var_val, strlen(tf_env_var_val));
+}
 
-  // Ideally we would use env_var / safe_strto64, but it is
-  // hard to use here without pulling in a lot of dependencies,
-  // so we use std:istringstream instead
-  string min_log_level(tf_env_var_val);
-  std::istringstream ss(min_log_level);
-  int64 level;
-  if (!(ss >> level)) {
-    // Invalid vlog level setting, set level to default (0)
-    level = 0;
+// Using StringPiece breaks Windows build.
+struct StringData {
+  struct Hasher {
+    size_t operator()(const StringData& sdata) const {
+      // For dependency reasons, we cannot use hash.h here. Use DBJHash instead.
+      size_t hash = 5381;
+      const char* data = sdata.data;
+      for (const char* top = data + sdata.size; data < top; ++data) {
+        hash = ((hash << 5) + hash) + (*data);
+      }
+      return hash;
+    }
+  };
+
+  StringData() = default;
+  StringData(const char* data, size_t size) : data(data), size(size) {}
+
+  bool operator==(const StringData& rhs) const {
+    return size == rhs.size && memcmp(data, rhs.data, size) == 0;
   }
 
-  return level;
+  const char* data = nullptr;
+  size_t size = 0;
+};
+
+using VmoduleMap = std::unordered_map<StringData, int, StringData::Hasher>;
+
+// Returns a mapping from module name to VLOG level, derived from the
+// TF_CPP_VMOUDLE environment variable; ownership is transferred to the caller.
+VmoduleMap* VmodulesMapFromEnv() {
+  // The value of the env var is supposed to be of the form:
+  //    "foo=1,bar=2,baz=3"
+  const char* env = getenv("TF_CPP_VMODULE");
+  if (env == nullptr) {
+    // If there is no TF_CPP_VMODULE configuration (most common case), return
+    // nullptr so that the ShouldVlogModule() API can fast bail out of it.
+    return nullptr;
+  }
+  // The memory returned by getenv() can be invalidated by following getenv() or
+  // setenv() calls. And since we keep references to it in the VmoduleMap in
+  // form of StringData objects, make a copy of it.
+  const char* env_data = strdup(env);
+  VmoduleMap* result = new VmoduleMap();
+  while (true) {
+    const char* eq = strchr(env_data, '=');
+    if (eq == nullptr) {
+      break;
+    }
+    const char* after_eq = eq + 1;
+
+    // Comma either points at the next comma delimiter, or at a null terminator.
+    // We check that the integer we parse ends at this delimiter.
+    const char* comma = strchr(after_eq, ',');
+    const char* new_env_data;
+    if (comma == nullptr) {
+      comma = strchr(after_eq, '\0');
+      new_env_data = comma;
+    } else {
+      new_env_data = comma + 1;
+    }
+    (*result)[StringData(env_data, eq - env_data)] =
+        ParseInteger(after_eq, comma - after_eq);
+    env_data = new_env_data;
+  }
+  return result;
 }
 
 }  // namespace
@@ -146,10 +212,15 @@ int64 MinVLogLevelFromEnv() {
 #endif
 }
 
+LogMessage::LogMessage(const char* fname, int line, int severity)
+    : fname_(fname), line_(line), severity_(severity) {}
+
 LogMessage::~LogMessage() {
   // Read the min log level once during the first call to logging.
   static int64 min_log_level = MinLogLevelFromEnv();
-  if (TF_PREDICT_TRUE(severity_ >= min_log_level)) GenerateLogMessage();
+  if (severity_ >= min_log_level) {
+    GenerateLogMessage();
+  }
 }
 
 int64 LogMessage::MinVLogLevel() {
@@ -157,6 +228,24 @@ int64 LogMessage::MinVLogLevel() {
   return min_vlog_level;
 }
 
+bool LogMessage::VmoduleActivated(const char* fname, int level) {
+  if (level <= MinVLogLevel()) {
+    return true;
+  }
+  static VmoduleMap* vmodules = VmodulesMapFromEnv();
+  if (TF_PREDICT_TRUE(vmodules == nullptr)) {
+    return false;
+  }
+  const char* last_slash = strrchr(fname, '/');
+  const char* module_start = last_slash == nullptr ? fname : last_slash + 1;
+  const char* dot_after = strchr(module_start, '.');
+  const char* module_limit =
+      dot_after == nullptr ? strchr(fname, '\0') : dot_after;
+  StringData module(module_start, module_limit - module_start);
+  auto it = vmodules->find(module);
+  return it != vmodules->end() && it->second >= level;
+}
+
 LogMessageFatal::LogMessageFatal(const char* file, int line)
     : LogMessage(file, line, FATAL) {}
 LogMessageFatal::~LogMessageFatal() {
diff --git a/tensorflow/core/platform/default/logging.h b/tensorflow/core/platform/default/logging.h
index 08a692fff7..bb8735ed32 100644
--- a/tensorflow/core/platform/default/logging.h
+++ b/tensorflow/core/platform/default/logging.h
@@ -46,6 +46,17 @@ class LogMessage : public std::basic_ostringstream<char> {
   // but VLOG(3) will not. Defaults to 0.
   static int64 MinVLogLevel();
 
+  // Returns whether VLOG level lvl is activated for the file fname.
+  //
+  // E.g. if the environment variable TF_CPP_VMODULE contains foo=3 and fname is
+  // foo.cc and lvl is <= 3, this will return true. It will also return true if
+  // the level is lower or equal to TF_CPP_MIN_VLOG_LEVEL (default zero).
+  //
+  // It is expected that the result of this query will be cached in the VLOG-ing
+  // call site to avoid repeated lookups. This routine performs a hash-map
+  // access against the VLOG-ing specification provided by the env var.
+  static bool VmoduleActivated(const char* fname, int level);
+
  protected:
   void GenerateLogMessage();
 
@@ -55,6 +66,13 @@ class LogMessage : public std::basic_ostringstream<char> {
   int severity_;
 };
 
+// Uses the lower operator & precedence to voidify a LogMessage reference, so
+// that the ternary VLOG() implementation is balanced, type wise.
+struct Voidifier {
+  template <typename T>
+  void operator&(const T&)const {}
+};
+
 // LogMessageFatal ensures the process will exit in failure after
 // logging this message.
 class LogMessageFatal : public LogMessage {
@@ -77,18 +95,30 @@ class LogMessageFatal : public LogMessage {
 #define LOG(severity) _TF_LOG_##severity
 
 #ifdef IS_MOBILE_PLATFORM
+
 // Turn VLOG off when under mobile devices for considerations of binary size.
 #define VLOG_IS_ON(lvl) ((lvl) <= 0)
+
 #else
-// Otherwise, Set TF_CPP_MIN_VLOG_LEVEL environment to update minimum log level
-// of VLOG
-#define VLOG_IS_ON(lvl) \
-  ((lvl) <= ::tensorflow::internal::LogMessage::MinVLogLevel())
+
+// Otherwise, set TF_CPP_MIN_VLOG_LEVEL environment to update minimum log level
+// of VLOG, or TF_CPP_VMODULE to set the minimum log level for individual
+// translation units.
+#define VLOG_IS_ON(lvl)                                                     \
+  (([](int level, const char* fname) {                                      \
+    static const bool vmodule_activated =                                   \
+        ::tensorflow::internal::LogMessage::VmoduleActivated(fname, level); \
+    return vmodule_activated;                                               \
+  })(lvl, __FILE__))
+
 #endif
 
-#define VLOG(lvl)                        \
-  if (TF_PREDICT_FALSE(VLOG_IS_ON(lvl))) \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::INFO)
+#define VLOG(level)                                              \
+  TF_PREDICT_TRUE(!VLOG_IS_ON(level))                            \
+  ? (void)0                                                      \
+  : ::tensorflow::internal::Voidifier() &                        \
+          ::tensorflow::internal::LogMessage(__FILE__, __LINE__, \
+                                             tensorflow::INFO)
 
 // CHECK dies with a fatal error if condition is not true.  It is *not*
 // controlled by NDEBUG, so the check will be executed regardless of
-- 
GitLab


From 2d3c95d36766bd47a5df98ab07bf870c97758149 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 26 Nov 2018 20:06:56 -0800
Subject: [PATCH 0845/1554] [tf.data] Moving Unbatch benchmarks with others
 benchmarks.

PiperOrigin-RevId: 222929316
---
 .../python/data/experimental/benchmarks/BUILD |  40 +++++--
 .../benchmarks/unbatch_benchmark.py           | 107 ++++++++++++++++++
 .../experimental/kernel_tests/unbatch_test.py |  75 ------------
 3 files changed, 135 insertions(+), 87 deletions(-)
 create mode 100644 tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py

diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index 075863d34f..8175116c6e 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -7,6 +7,21 @@ exports_files(["LICENSE"])
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "py_test")
 
+py_test(
+    name = "autotune_benchmark",
+    srcs = ["autotune_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/experimental/ops:batching",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "csv_dataset_benchmark",
     srcs = ["csv_dataset_benchmark.py"],
@@ -63,15 +78,17 @@ py_test(
 )
 
 py_test(
-    name = "autotune_benchmark",
-    srcs = ["autotune_benchmark.py"],
+    name = "matching_files_benchmark",
+    size = "small",
+    srcs = ["matching_files_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:session",
-        "//tensorflow/python/data/experimental/ops:batching",
-        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/experimental/ops:matching_files",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
@@ -92,17 +109,16 @@ py_test(
 )
 
 py_test(
-    name = "matching_files_benchmark",
-    size = "small",
-    srcs = ["matching_files_benchmark.py"],
+    name = "unbatch_benchmark",
+    srcs = ["unbatch_benchmark.py"],
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:util",
-        "//tensorflow/python/data/experimental/ops:matching_files",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/experimental/ops:batching",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
diff --git a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
new file mode 100644
index 0000000000..c40d479823
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
@@ -0,0 +1,107 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental.unbatch()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+class UnbatchBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.experimental.unbatch()`."""
+
+  def benchmarkNativeUnbatch(self):
+    batch_sizes = [1, 2, 5, 10, 20, 50]
+    elems_per_trial = 10000
+    with ops.Graph().as_default():
+      dataset = dataset_ops.Dataset.from_tensors("element").repeat(None)
+      batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
+      dataset = dataset.batch(batch_size_placeholder)
+      dataset = dataset.apply(batching.unbatch())
+      dataset = dataset.skip(elems_per_trial)
+      iterator = dataset.make_initializable_iterator()
+      next_element = iterator.get_next()
+
+      with session.Session() as sess:
+        for batch_size in batch_sizes:
+          deltas = []
+          for _ in range(5):
+            sess.run(
+                iterator.initializer,
+                feed_dict={batch_size_placeholder: batch_size})
+            start = time.time()
+            sess.run(next_element.op)
+            end = time.time()
+            deltas.append((end - start) / elems_per_trial)
+
+          median_wall_time = np.median(deltas)
+          print("Unbatch (native) batch size: %d Median wall time per element:"
+                " %f microseconds" % (batch_size, median_wall_time * 1e6))
+          self.report_benchmark(
+              iters=10000,
+              wall_time=median_wall_time,
+              name="native_batch_size_%d" %
+              batch_size)
+
+  # Include a benchmark of the previous `unbatch()` implementation that uses
+  # a composition of more primitive ops. Eventually we'd hope to generate code
+  # that is as good in both cases.
+  def benchmarkOldUnbatchImplementation(self):
+    batch_sizes = [1, 2, 5, 10, 20, 50]
+    elems_per_trial = 10000
+    with ops.Graph().as_default():
+      dataset = dataset_ops.Dataset.from_tensors("element").repeat(None)
+      batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
+      dataset = dataset.batch(batch_size_placeholder)
+      dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices)
+      dataset = dataset.skip(elems_per_trial)
+      iterator = dataset.make_initializable_iterator()
+      next_element = iterator.get_next()
+
+      with session.Session() as sess:
+        for batch_size in batch_sizes:
+          deltas = []
+          for _ in range(5):
+            sess.run(
+                iterator.initializer,
+                feed_dict={batch_size_placeholder: batch_size})
+            start = time.time()
+            sess.run(next_element.op)
+            end = time.time()
+            deltas.append((end - start) / elems_per_trial)
+
+          median_wall_time = np.median(deltas)
+          print("Unbatch (unfused) batch size: %d Median wall time per element:"
+                " %f microseconds" % (batch_size, median_wall_time * 1e6))
+          self.report_benchmark(
+              iters=10000,
+              wall_time=median_wall_time,
+              name="unfused_batch_size_%d" %
+              batch_size)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index 0278a208cb..f9b800fe67 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -17,19 +17,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
 
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -224,77 +221,5 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(next_element)
 
 
-class UnbatchBenchmark(test.Benchmark):
-
-  def benchmarkNativeUnbatch(self):
-    batch_sizes = [1, 2, 5, 10, 20, 50]
-    elems_per_trial = 10000
-    with ops.Graph().as_default():
-      dataset = dataset_ops.Dataset.from_tensors("element").repeat(None)
-      batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-      dataset = dataset.batch(batch_size_placeholder)
-      dataset = dataset.apply(batching.unbatch())
-      dataset = dataset.skip(elems_per_trial)
-      iterator = dataset.make_initializable_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for batch_size in batch_sizes:
-          deltas = []
-          for _ in range(5):
-            sess.run(
-                iterator.initializer,
-                feed_dict={batch_size_placeholder: batch_size})
-            start = time.time()
-            sess.run(next_element.op)
-            end = time.time()
-            deltas.append((end - start) / elems_per_trial)
-
-          median_wall_time = np.median(deltas)
-          print("Unbatch (native) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
-          self.report_benchmark(
-              iters=10000,
-              wall_time=median_wall_time,
-              name="benchmark_unbatch_dataset_native_batch_size_%d" %
-              batch_size)
-
-  # Include a benchmark of the previous `unbatch()` implementation that uses
-  # a composition of more primitive ops. Eventually we'd hope to generate code
-  # that is as good in both cases.
-  def benchmarkOldUnbatchImplementation(self):
-    batch_sizes = [1, 2, 5, 10, 20, 50]
-    elems_per_trial = 10000
-    with ops.Graph().as_default():
-      dataset = dataset_ops.Dataset.from_tensors("element").repeat(None)
-      batch_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-      dataset = dataset.batch(batch_size_placeholder)
-      dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices)
-      dataset = dataset.skip(elems_per_trial)
-      iterator = dataset.make_initializable_iterator()
-      next_element = iterator.get_next()
-
-      with session.Session() as sess:
-        for batch_size in batch_sizes:
-          deltas = []
-          for _ in range(5):
-            sess.run(
-                iterator.initializer,
-                feed_dict={batch_size_placeholder: batch_size})
-            start = time.time()
-            sess.run(next_element.op)
-            end = time.time()
-            deltas.append((end - start) / elems_per_trial)
-
-          median_wall_time = np.median(deltas)
-          print("Unbatch (unfused) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
-          self.report_benchmark(
-              iters=10000,
-              wall_time=median_wall_time,
-              name="benchmark_unbatch_dataset_unfused_batch_size_%d" %
-              batch_size)
-
-
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From e0daeb19bde631543c0df691f37b06e2377a31fe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Nov 2018 20:28:44 -0800
Subject: [PATCH 0846/1554] Add data_format, rename Targmax for
 nn.max_pool_with_argmax for TF 2.0

PiperOrigin-RevId: 222930810
---
 .../api_def_MaxPoolWithArgmax.pbtxt           |  1 +
 tensorflow/python/ops/nn_ops.py               | 61 +++++++++++++++++++
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  3 +
 4 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt b/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt
index 7d8abca5f1..13a1a0b5df 100644
--- a/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_MaxPoolWithArgmax.pbtxt
@@ -2,5 +2,6 @@ op {
   graph_op_name: "MaxPoolWithArgmax"
   endpoint {
     name: "nn.max_pool_with_argmax"
+    deprecation_version: 2
   }
 }
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 5655960aae..2596f8e9bc 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2675,6 +2675,67 @@ def max_pool(value, ksize, strides, padding, data_format="NHWC", name=None):
         name=name)
 
 
+# pylint: disable=redefined-builtin
+@tf_export("nn.max_pool_with_argmax", v1=[])
+def max_pool_with_argmax_v2(input,
+                            ksize,
+                            strides,
+                            padding,
+                            data_format="NHWC",
+                            output_dtype=dtypes.int64,
+                            name=None):
+  """Performs max pooling on the input and outputs both max values and indices.
+
+  The indices in `argmax` are flattened, so that a maximum value at position
+  `[b, y, x, c]` becomes flattened index
+  `((b * height + y) * width + x) * channels + c`.
+
+  The indices returned are always in `[0, height) x [0, width)` before
+  flattening, even if padding is involved and the mathematically correct answer
+  is outside (either negative or too large).  This is a bug, but fixing it is
+  difficult to do in a safe backwards compatible way, especially due to
+  flattening.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types: `float32`, `float64`,
+      `int32`, `uint8`, `int16`, `int8`, `int64`, `bfloat16`, `uint16`, `half`,
+      `uint32`, `uint64`.
+      4-D with shape `[batch, height, width, channels]`.  Input to pool over.
+    ksize: A list of `ints` that has length `>= 4`.
+      The size of the window for each dimension of the input tensor.
+    strides: A list of `ints` that has length `>= 4`.
+      The stride of the sliding window for each dimension of the
+      input tensor.
+    padding: A `string` from: `"SAME", "VALID"`.
+      The type of padding algorithm to use.
+    data_format: An optional `string`, must be set to `"NHWC"`. Defaults to
+      `"NHWC"`.
+      Specify the data format of the input and output data.
+    output_dtype: An optional `tf.DType` from: `tf.int32, tf.int64`.
+      Defaults to `tf.int64`.
+      The dtype of the returned argmax tensor.
+    name: A name for the operation (optional).
+
+  Returns:
+    A tuple of `Tensor` objects (output, argmax).
+
+    output: A `Tensor`. Has the same type as `input`.
+    argmax: A `Tensor` of type `output_dtype`.
+  """
+
+  if data_format != "NHWC":
+    raise ValueError("Data formats other than 'NHWC' are not yet supported")
+
+  return gen_nn_ops.max_pool_with_argmax(input=input,
+                                         ksize=ksize,
+                                         strides=strides,
+                                         padding=padding,
+                                         Targmax=output_dtype,
+                                         name=name)
+
+# pylint: enable=redefined-builtin
+
+
 @ops.RegisterStatistics("Conv2D", "flops")
 def _calc_conv_flops(graph, node):
   """Calculates the compute resources needed for Conv2D."""
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 0bfea74e40..04b189784a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -206,7 +206,7 @@ tf_module {
   }
   member_method {
     name: "max_pool_with_argmax"
-    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'Targmax\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int64\'>\", \'None\'], "
+    argspec: "args=[\'input\', \'ksize\', \'strides\', \'padding\', \'data_format\', \'output_dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \"<dtype: \'int64\'>\", \'None\'], "
   }
   member_method {
     name: "moments"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3588db395e..7433e0ecbf 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -77,6 +77,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.split": {
             "split_dim": "axis",
         },
+        "tf.max_pool_with_argmax": {
+            "Targmax": "output_dtype",
+        },
         "tf.multinomial": {
             "output_dtype": "dtype",
         },
-- 
GitLab


From 4fbbeea283de57deb681699a93694871a297d286 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 26 Nov 2018 22:11:19 -0800
Subject: [PATCH 0847/1554] [TensorRT]: Define a shape function.

This will help address some of the issues encountered
when using TensorRT using other language APIs.
See context in
#23853#issuecomment-441867084

PiperOrigin-RevId: 222937978
---
 .../contrib/tensorrt/ops/trt_engine_op.cc     | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index ce04e5806e..92405906eb 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 
+#include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/shape_inference.h"
@@ -43,15 +44,15 @@ REGISTER_OP("TRTEngineOp")
     .Attr("calibration_data: string = ''")
     .Attr("use_calibration: bool = true")
     .Input("in_tensor: InT")
-    .Output("out_tensor: OutT");
-// TODO(jie): TF requires concrete output shape for concrete input shapes.
-// This is tricky for batch dimension, since we cannot ensure which input
-// would carry the correct batch dimension (for the current stage of the
-// implementation, we do require all input tensor to carry the same batch
-// size, but this could change in the future). Hence we disable shape
-// inference function as a workaround.
-// .SetShapeFn(shape_inference::TRTEngineOpShapeInference);
-
+    .Output("out_tensor: OutT")
+    // TODO(jie): TF requires concrete output shape for concrete input shapes.
+    // This is tricky for batch dimension, since we cannot ensure which input
+    // would carry the correct batch dimension (for the current stage of the
+    // implementation, we do require all input tensor to carry the same batch
+    // size, but this could change in the future). Hence we disable shape
+    // inference function as a workaround.
+    // .SetShapeFn(shape_inference::TRTEngineOpShapeInference);
+    .SetShapeFn(shape_inference::UnknownShape);
 }  // namespace tensorflow
 
 #endif  // GOOGLE_TENSORRT
-- 
GitLab


From 1899ac3d56e578b2b685f883bd1455dd0717ee02 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Tue, 27 Nov 2018 00:52:48 -0800
Subject: [PATCH 0848/1554] Make use of the Iota function that gets passed a
 shape.

This avoids manual broadcasts which are handled automatically by the Iota op.

PiperOrigin-RevId: 222949991
---
 .../tf2xla/kernels/conv_op_helpers.cc         | 77 ++++++++++---------
 .../tf2xla/kernels/matrix_band_part_op.cc     |  9 ++-
 2 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index b1046fcc00..641fefafb3 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -64,60 +64,63 @@ xla::Shape ExpandedFilterShapeForDepthwiseConvolution(const xla::Shape& shape) {
 //   0 0 1 1 0 0   0 0 1 1 0 0
 //   0 0 0 0 1 1   0 0 0 0 1 1
 //
-// The first step is to create a one tensor, A, that is [3]
-//   0 1 2
+// The first step is to create a iota A with iota_dimension = 2
+//   0 0 0 0 0 0   0 0 0 0 0 0
+//   1 1 1 1 1 1   1 1 1 1 1 1
+//   2 2 2 2 2 2   2 2 2 2 2 2
 //
-// and another tensor, B,  that is [3 * 2]
-//   0 1 2 3 4 5
+//   0 0 0 0 0 0   0 0 0 0 0 0
+//   1 1 1 1 1 1   1 1 1 1 1 1
+//   2 2 2 2 2 2   2 2 2 2 2 2
 //
-// and divide B it by 2 to get
-//   0 0 1 1 2 2
+// and another iota B with iota_dimension = 3
+//   0 1 2 3 4 5  0 1 2 3 4 5
+//   0 1 2 3 4 5  0 1 2 3 4 5
+//   0 1 2 3 4 5  0 1 2 3 4 5
 //
-// then we broadcast the B to [2, 2, 3, 3 * 2]
-//   0 0 1 1 2 2   0 0 1 1 2 2
-//   0 0 1 1 2 2   0 0 1 1 2 2
-//   0 0 1 1 2 2   0 0 1 1 2 2
+//   0 1 2 3 4 5  0 1 2 3 4 5
+//   0 1 2 3 4 5  0 1 2 3 4 5
+//   0 1 2 3 4 5  0 1 2 3 4 5
 //
-//   0 0 1 1 2 2   0 0 1 1 2 2
-//   0 0 1 1 2 2   0 0 1 1 2 2
-//   0 0 1 1 2 2   0 0 1 1 2 2
+// and divide B by 2 to get
+//   0 0 1 1 2 2  0 0 1 1 2 2
+//   0 0 1 1 2 2  0 0 1 1 2 2
+//   0 0 1 1 2 2  0 0 1 1 2 2
 //
-// Finally compare A and broadcasted B in dimension 2 amd return the result at
-// the beginning of the comment.
+//   0 0 1 1 2 2  0 0 1 1 2 2
+//   0 0 1 1 2 2  0 0 1 1 2 2
+//   0 0 1 1 2 2  0 0 1 1 2 2
+//
+// Finally compare A and B and return the result at the beginning of the
+// comment.
 xla::XlaOp CreateExpandedFilterMask(const xla::Shape& filter_shape,
                                     xla::XlaBuilder* builder) {
   xla::Shape expanded_filter_shape =
       ExpandedFilterShapeForDepthwiseConvolution(filter_shape);
   int64 depthwise_multiplier =
       filter_shape.dimensions(filter_shape.dimensions_size() - 1);
-  int64 input_feature =
-      filter_shape.dimensions(filter_shape.dimensions_size() - 2);
-
-  // Create a M sized linspace and an M*N sized linspace that will be
-  // broadcasted into perpendicular dimensions and compared.
-  xla::XlaOp input_feature_iota = xla::Iota(builder, xla::S32, input_feature);
-  xla::XlaOp expanded_feature_iota =
-      xla::Iota(builder, xla::S32, input_feature * depthwise_multiplier);
 
-  // Divide the M*N sized linspace by the depthwise_multiplier to create
-  // [0 0 1 1 2 2] in the example in the function comment.
+  // Create two iotas with the shape of the expanded filter, one of them with
+  // the iota dimension chosen as the feature dimension, and the other a iota
+  // with the iota dimension chosen as the expanded output feature dimension.
+  std::vector<int64> iota_dimensions(expanded_filter_shape.dimensions().begin(),
+                                     expanded_filter_shape.dimensions().end());
+  xla::Shape iota_shape = xla::ShapeUtil::MakeShape(xla::S32, iota_dimensions);
+  xla::XlaOp input_feature_iota = xla::Iota(
+      builder, iota_shape, /*iota_dimension=*/iota_dimensions.size() - 2);
+  xla::XlaOp expanded_feature_iota = xla::Iota(
+      builder, iota_shape, /*iota_dimension=*/iota_dimensions.size() - 1);
+
+  // Divide 'expanded_feature_iota' by the depthwise_multiplier to create
+  // [0 0 1 1 2 2] ... in the example in the function comment.
   expanded_feature_iota =
       xla::Div(expanded_feature_iota,
                XlaHelpers::IntegerLiteral(builder, DataType::DT_INT32,
                                           depthwise_multiplier));
 
-  // Broadcast the N*M linspace to [H, W, ..., M, M*N].
-  std::vector<int64> expanded_feature_broadcast_dims(
-      expanded_filter_shape.dimensions().begin(),
-      expanded_filter_shape.dimensions().end());
-  expanded_feature_broadcast_dims.pop_back();
-  auto broadcasted_expanded_feature_iota =
-      xla::Broadcast(expanded_feature_iota, expanded_feature_broadcast_dims);
-
-  // Compare the broadcasted linspace to the input feature linspace in the
-  // input feature dimension to create a diagonal predicate.
-  return xla::Eq(broadcasted_expanded_feature_iota, input_feature_iota,
-                 {expanded_filter_shape.dimensions_size() - 2});
+  // Compare 'input_feature_iota' with 'expanded_feature_iota' to create a
+  // diagonal predicate.
+  return xla::Eq(expanded_feature_iota, input_feature_iota);
 }
 
 // Reshapes a filter of shape [H, W, ..., M, N] to [H, W, ..., 1, M*N]. Used to
diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc
index a99b74565d..2dd0a710e4 100644
--- a/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matrix_band_part_op.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 
 namespace tensorflow {
@@ -60,11 +61,11 @@ class MatrixBandPartOp : public XlaOpKernel {
 
     // Compute 'offset', which is how many diagonals we are above/below the
     // diagonal.
-    xla::XlaOp iota_m = xla::Iota(builder, index_xla_type, m);
-    xla::XlaOp iota_n = xla::Iota(builder, index_xla_type, n);
+    xla::Shape iota_shape = xla::ShapeUtil::MakeShape(index_xla_type, {m, n});
+    xla::XlaOp iota_m = xla::Iota(builder, iota_shape, /*iota_dimension=*/0);
+    xla::XlaOp iota_n = xla::Iota(builder, iota_shape, /*iota_dimension=*/1);
 
-    auto offset = xla::Sub(xla::Broadcast(iota_n, {m}), iota_m,
-                           /*broadcast_dimensions=*/{0});
+    auto offset = xla::Sub(iota_n, iota_m);
 
     // If num_lower or num_upper are negative, include all lower/upper
     // diagonals.
-- 
GitLab


From ac167c39b44c3255848f671bfcece4fb50b264a9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 01:02:58 -0800
Subject: [PATCH 0849/1554] compat: Update forward compatibility horizon to
 2018-11-27

PiperOrigin-RevId: 222951259
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 4e84b5ee48..cb359cd084 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 26)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 27)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From b607192183e06ef18f79e7c6dd8812c043fc1ab3 Mon Sep 17 00:00:00 2001
From: Serge Panev <spanev@nvidia.com>
Date: Tue, 27 Nov 2018 10:57:57 +0100
Subject: [PATCH 0850/1554] Fix comparison between signed and unsigned integer
 expressions in tensor_format.h

---
 tensorflow/core/framework/dataset.h  | 2 +-
 tensorflow/core/util/tensor_format.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index 9b11449b30..e0041492ee 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -162,7 +162,7 @@ class GraphDefBuilderWrapper {
                     const std::vector<std::pair<StringPiece, AttrValue>>& attrs,
                     Node** output) {
     std::vector<std::pair<size_t, Node*>> enumerated_inputs(inputs.size());
-    for (int i = 0; i < inputs.size(); i++) {
+    for (int i = 0; static_cast<size_t>(i) < inputs.size(); i++) {
       enumerated_inputs[i] = std::make_pair(i, inputs[i]);
     }
     return AddDataset(dataset, enumerated_inputs, {}, attrs, output);
diff --git a/tensorflow/core/util/tensor_format.h b/tensorflow/core/util/tensor_format.h
index b0c349dd90..68674cb282 100644
--- a/tensorflow/core/util/tensor_format.h
+++ b/tensorflow/core/util/tensor_format.h
@@ -498,7 +498,7 @@ inline TensorShape ShapeFromFormat(TensorFormat format, int64 N,
   dim_sizes[GetTensorBatchDimIndex(dims, format)] = N;
   for (int dim = 0; static_cast<size_t>(dim) < spatial.size(); dim++) {
     auto dim_size = spatial[dim];
-    if (format == FORMAT_NHWC_VECT_W && dim == spatial.size() - 1) {
+    if (format == FORMAT_NHWC_VECT_W && static_cast<size_t>(dim) == spatial.size() - 1) {
       CHECK_EQ(0, dim_size % 4)
           << "FORMAT_NHWC_VECT_W requires W to be a multiple of 4, but W="
           << dim_size;
-- 
GitLab


From 6e436c4f058175c62dca5aa37b9b95c15c251ea9 Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Tue, 27 Nov 2018 19:08:47 +0900
Subject: [PATCH 0851/1554] fix typo

---
 tensorflow/compiler/xla/service/algebraic_simplifier_test.cc | 2 +-
 tensorflow/compiler/xla/shape_util.cc                        | 2 +-
 tensorflow/python/ops/ragged/ragged_tensor.py                | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 24c35464ad..054d518671 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -1617,7 +1617,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeOfTransposeOfRngToRng) {
       (AlgebraicSimplifierOptions(bitcasting_callback())));
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  // Verify that that reshape(transpose(rng)) is replace by a single rng of the
+  // Verify that reshape(transpose(rng)) is replace by a single rng of the
   // same shape as the reshape.
   EXPECT_THAT(computation->root_instruction(), op::Rng());
   EXPECT_TRUE(ShapeUtil::Equal(computation->root_instruction()->shape(),
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index b05ec209cc..c185b19687 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -1165,7 +1165,7 @@ Status ForEachMutableSubshapeHelper(
   // Let the argument `permutation` be P.  This is a permutation over `shape`'s
   // dimensions, so our return value will be a shape with dims P.I = P.  Our
   // goal is to construct a layout permutation L* that we can apply to P such
-  // that that the physical dimension ordering of the returned shape is the same
+  // that the physical dimension ordering of the returned shape is the same
   // as that of the original shape, namely L'.
   //
   // Our returned shape has dims P and layout L*, so its in-memory layout is
diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py
index abb27fc3c0..1d4bbd592d 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor.py
@@ -64,7 +64,7 @@ class RaggedTensor(object):
   a 3-D `RaggedTensor` that stores the fixed-size word embedding for each
   word in a sentence, for each sentence in a batch, could be written as
   `[num_sentences, (num_words), embedding_size]`.  The parentheses around
-  `(num_words)` indicate that that dimension is ragged, and that the length
+  `(num_words)` indicate that dimension is ragged, and that the length
   of each element list in that dimension may vary for each item.
 
   ### Component Tensors
-- 
GitLab


From b5b6628931a2db98a94c55e90c0df5db5cb4fa5b Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 27 Nov 2018 02:54:53 -0800
Subject: [PATCH 0852/1554] Replaced deprecated tf.create_partitioned_variables
 with tf.get_variable

PiperOrigin-RevId: 222963167
---
 .../python/layers/embedding_ops_test.py       | 51 ++++++++++-------
 .../contrib/tpu/python/tpu/tpu_embedding.py   | 11 ++--
 .../python/kernel_tests/embedding_ops_test.py | 10 ++--
 .../saver_large_partitioned_variable_test.py  |  9 ++-
 tensorflow/python/training/saver_test.py      | 56 ++++++-------------
 5 files changed, 66 insertions(+), 71 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py
index 8015a571e1..295c721fce 100644
--- a/tensorflow/contrib/layers/python/layers/embedding_ops_test.py
+++ b/tensorflow/contrib/layers/python/layers/embedding_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 import itertools
 import math
+import sys
 
 import numpy as np
 
@@ -36,6 +37,7 @@ from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import partitioned_variables
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -48,11 +50,13 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
     assert num_shards > 0
     assert num_shards <= vocab_size
 
-    embedding_weights = partitioned_variables.create_partitioned_variables(
+    initializer = init_ops.truncated_normal_initializer(
+        mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)
+    embedding_weights = list(variable_scope.get_variable(
+        "embedding_weights",
         shape=[vocab_size, embed_dim],
-        slicing=[num_shards, 1],
-        initializer=init_ops.truncated_normal_initializer(
-            mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32))
+        partitioner=partitioned_variables.fixed_size_partitioner(num_shards),
+        initializer=initializer))
     for w in embedding_weights:
       w.initializer.run()
     embedding_weights = [w.eval() for w in embedding_weights]
@@ -256,6 +260,13 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
                         embedding_weights, sparse_ids, sparse_weights)
 
 
+# pylint: disable=invalid-name
+def local_variable_scope():
+  """Create a variable scope named like the caller function."""
+  return variable_scope.variable_scope(sys._getframe(1).f_code.co_name)
+# pylint: enable=invalid-name
+
+
 class ScatteredEmbeddingLookupTest(test.TestCase):
 
   def setUp(self):
@@ -266,17 +277,18 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
     assert num_shards > 0
     assert num_shards <= size
 
-    embedding_weights = partitioned_variables.create_partitioned_variables(
+    embedding_weights = list(variable_scope.get_variable(
+        "embedding_weights",
         shape=[size],
-        slicing=[num_shards],
+        partitioner=partitioned_variables.fixed_size_partitioner(num_shards),
         initializer=init_ops.truncated_normal_initializer(
-            mean=0.0, stddev=1.0, dtype=dtypes.float32))
+            mean=0.0, stddev=1.0, dtype=dtypes.float32)))
     for w in embedding_weights:
       w.initializer.run()
     return embedding_weights
 
   def test_scattered_embedding_consistency(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedding_weights = self._random_weights()
       values = constant_op.constant(["foo", "foo"])
 
@@ -288,7 +300,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
                           embedding_lookup_result[1])
 
   def test_scattered_embedding_multiple_partition(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedding_weights = self._random_weights(num_shards=7)
       values = constant_op.constant([4, 4, 5])
 
@@ -304,7 +316,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
       self.assertGreater(embedding_diff, 0)
 
   def test_scattered_embedding_coverage(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       size = 8
       embedding_weights = self._random_weights(size=size, num_shards=3)
       values = constant_op.constant(["foo"])
@@ -316,7 +328,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
       self.assertEqual(len(np.unique(embedding_lookup_result[0])), size)
 
   def test_scattered_embedding_multi_dimension(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedding_weights = self._random_weights()
       values = constant_op.constant([["foo", "bar", "bar"],
                                      ["bar", "bar", "foo"]])
@@ -329,7 +341,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
                           embedding_lookup_result[1][2])
 
   def test_scattered_embedding_lookup_sparse(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedding_weights = self._random_weights(num_shards=3)
       sparse_tensor = sparse_tensor_lib.SparseTensor(
           values=["foo", "bar", "foo", "bar"],
@@ -358,7 +370,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
     embeds = np.random.randn(n_embed, d_embed)
     idx = np.random.randint(0, n_embed, idx_shape)
 
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedded_np = embeds[idx]
       embedded_tf = embedding_ops.embedding_lookup_unique(embeds, idx).eval()
 
@@ -370,7 +382,7 @@ class ScatteredEmbeddingLookupTest(test.TestCase):
     idx = np.random.randint(0, 5, 10)
     idx2d = np.random.randint(0, 5, (10, 2))
 
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedded_np = embeds[idx]
       embedded_np2d = embeds[idx2d]
       embedded_tf = embedding_ops.embedding_lookup_unique(embeds, idx).eval()
@@ -398,17 +410,18 @@ class SampledScatteredEmbeddingLookupTest(test.TestCase):
     assert num_shards > 0
     assert num_shards <= size
 
-    embedding_weights = partitioned_variables.create_partitioned_variables(
+    embedding_weights = list(variable_scope.get_variable(
+        "embedding_weights",
         shape=[size],
-        slicing=[num_shards],
+        partitioner=partitioned_variables.fixed_size_partitioner(num_shards),
         initializer=init_ops.truncated_normal_initializer(
-            mean=0.0, stddev=1.0, dtype=dtypes.float32))
+            mean=0.0, stddev=1.0, dtype=dtypes.float32)))
     for w in embedding_weights:
       w.initializer.run()
     return embedding_weights
 
   def test_hashed_embedding_consistency(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedding_weights = self._random_weights()
       values = constant_op.constant(["foo", "foo"])
       # The first three sampled_candidates are equal, so the first three
@@ -429,7 +442,7 @@ class SampledScatteredEmbeddingLookupTest(test.TestCase):
                           embedding_lookup_result[1][3])
 
   def test_hashed_embedding_multi_dimension(self):
-    with self.cached_session():
+    with self.cached_session(), local_variable_scope():
       embedding_weights = self._random_weights()
       values = constant_op.constant([["foo", "bar", "bar"],
                                      ["bar", "bar", "foo"]])
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index 3fe896426a..ccba8a46c7 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -1069,17 +1069,14 @@ def _create_partitioned_variables(name,
                      'As TPU embedding is not optimized for small tables, '
                      'please consider other ways for this embedding lookup.')
 
-  slicing = [num_hosts, 1]
-
-  # TODO(shizhiw): deprecated, use tf.get_variable()?
-  return partitioned_variables.create_partitioned_variables(
-      name=name,
-      slicing=slicing,
+  return list(variable_scope.get_variable(
+      name,
       shape=(vocabulary_size, embedding_dimension),
+      partitioner=partitioned_variables.fixed_size_partitioner(num_hosts),
       dtype=dtypes.float32,
       initializer=initializer,
       collections=collections,
-      trainable=False)
+      trainable=False))
 
 
 @ops.RegisterGradient('TPUEmbeddingActivations')
diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py
index 443f54a958..dba3409c9e 100644
--- a/tensorflow/python/kernel_tests/embedding_ops_test.py
+++ b/tensorflow/python/kernel_tests/embedding_ops_test.py
@@ -758,11 +758,13 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
     assert num_shards > 0
     assert num_shards <= vocab_size
 
-    embedding_weights = partitioned_variables.create_partitioned_variables(
+    initializer = init_ops.truncated_normal_initializer(
+        mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32)
+    embedding_weights = list(variable_scope.get_variable(
+        name="embedding_weights",
         shape=[vocab_size, embed_dim],
-        slicing=[num_shards, 1],
-        initializer=init_ops.truncated_normal_initializer(
-            mean=0.0, stddev=1.0 / math.sqrt(vocab_size), dtype=dtypes.float32))
+        partitioner=partitioned_variables.fixed_size_partitioner(num_shards),
+        initializer=initializer))
     for w in embedding_weights:
       w.initializer.run()
     embedding_weights = [w.eval() for w in embedding_weights]
diff --git a/tensorflow/python/training/saver_large_partitioned_variable_test.py b/tensorflow/python/training/saver_large_partitioned_variable_test.py
index 1a44511cfe..84458836d0 100644
--- a/tensorflow/python/training/saver_large_partitioned_variable_test.py
+++ b/tensorflow/python/training/saver_large_partitioned_variable_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import partitioned_variables
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import saver
@@ -44,8 +45,12 @@ class SaverLargePartitionedVariableTest(test.TestCase):
         # split into smaller sized variables.
         init = lambda shape, dtype, partition_info: constant_op.constant(
             True, dtype, shape)
-        partitioned_var = partitioned_variables.create_partitioned_variables(
-            [1 << 31], [4], init, dtype=dtypes.bool, name=var_name)
+        partitioned_var = list(variable_scope.get_variable(
+            var_name,
+            shape=[1 << 31],
+            partitioner=partitioned_variables.fixed_size_partitioner(4),
+            initializer=init,
+            dtype=dtypes.bool))
         variables.global_variables_initializer().run()
         save = saver.Saver(partitioned_var)
         val = save.save(sess, save_path)
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index eb2690985d..be49e6e715 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -998,19 +998,12 @@ class SaveRestoreShardedTest(test.TestCase):
 
     call_saver_with_dict = False  # updated by test loop below
 
-    def _save(slices=None, partitioner=None):
+    def _save(partitioner=None):
       with self.session(graph=ops_lib.Graph()) as sess:
         # Calls .eval() to return the ndarray that makes up the full variable.
         rnd = random_ops.random_uniform(var_full_shape).eval()
 
-        if slices:
-          assert not partitioner
-          # TODO(apassos): make create_partitioned_variables take use_resource
-          # option to make this test passable without creating a named
-          # variable_scope.
-          vs = partitioned_variables.create_partitioned_variables(
-              var_full_shape, slices, rnd, name=var_name)
-        elif partitioner:
+        if partitioner:
           vs = [
               variable_scope.get_variable(
                   var_name,
@@ -1027,7 +1020,7 @@ class SaveRestoreShardedTest(test.TestCase):
 
         variables.global_variables_initializer().run()
         if call_saver_with_dict:
-          saver = saver_module.Saver({var_name: (vs if slices else vs[0])})
+          saver = saver_module.Saver({var_name: vs[0]})
         else:
           saver = saver_module.Saver(vs)
         actual_path = saver.save(sess, saved_path)
@@ -1035,16 +1028,9 @@ class SaveRestoreShardedTest(test.TestCase):
 
         return rnd
 
-    def _restore(slices=None, partitioner=None):
+    def _restore(partitioner=None):
       with self.session(graph=ops_lib.Graph()) as sess:
-        if slices:
-          assert not partitioner
-          new_vs = partitioned_variables.create_partitioned_variables(
-              var_full_shape,
-              slices,
-              array_ops.zeros(var_full_shape),  # != original contents.
-              name=var_name)
-        elif partitioner:
+        if partitioner:
           new_vs = [
               variable_scope.get_variable(
                   var_name,
@@ -1063,7 +1049,7 @@ class SaveRestoreShardedTest(test.TestCase):
         variables.global_variables_initializer().run()
         if call_saver_with_dict:
           saver = saver_module.Saver({
-              var_name: (new_vs if slices else new_vs[0])
+              var_name: new_vs[0]
           })
         else:
           saver = saver_module.Saver(new_vs)
@@ -1071,11 +1057,7 @@ class SaveRestoreShardedTest(test.TestCase):
 
         if partitioner:
           return new_vs[0].as_tensor().eval()
-        elif slices and slices[0] != 1:
-          return array_ops.concat(new_vs, 0).eval()
-        elif slices and slices[1] != 1:
-          return array_ops.concat(new_vs, 1).eval()
-        else:  # Non-sliced.
+        else:
           return new_vs[0].eval()
 
     for call_saver_with_dict in {False, True}:
@@ -1086,27 +1068,23 @@ class SaveRestoreShardedTest(test.TestCase):
       restored_full = _restore()
       self.assertAllEqual(saved_full, restored_full)
 
-      # Saves 10 horizontal parts of a partitioned variable.
-      # Restores into a full variable, non-sliced.
-      saved_full = _save(slices=[10, 1])
-      restored_full = _restore()
-      self.assertAllEqual(saved_full, restored_full)
-
-      # Restores into a different number/orientation of slices.
-      restored_full = _restore(slices=[2, 1])  # 2 horizon parts.
-      self.assertAllEqual(saved_full, restored_full)
-      restored_full = _restore(slices=[1, 3])  # 3 vertical parts.
+      # Restores into the same number of partitions.
+      restored_full = _restore(
+          partitioner=partitioned_variables.fixed_size_partitioner(
+              num_shards=2))
       self.assertAllEqual(saved_full, restored_full)
 
-      # Restores into a PartitionedVariable
+      # Restores into a different number of partitions.
       restored_full = _restore(
           partitioner=partitioned_variables.fixed_size_partitioner(
-              num_shards=2))
+              num_shards=3))
       self.assertAllEqual(saved_full, restored_full)
 
-      # Now, saves a full variable and restores in slices.
+      # Now, saves a full variable and restores PartitionedVariable.
       saved_full = _save()
-      restored_full = _restore(slices=[1, 3])
+      restored_full = _restore(
+          partitioner=partitioned_variables.fixed_size_partitioner(
+              num_shards=3))
       self.assertAllEqual(saved_full, restored_full)
 
   def testPartitionedVariable(self):
-- 
GitLab


From 733ea8ac2c71989988cea7662a3acae4240362c7 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 27 Nov 2018 02:54:59 -0800
Subject: [PATCH 0853/1554] Removed duplicated _Overload* methods in variables

Prior to this change all variable classes (Variable, RefVariable and
ResourceVariable) had their own copy/pasted implementation of
_OverloadOperator and _OverloadAllOperators. This was unnecessary since
a single implementation in the base class would work just as well.

The change also fixes the __name__ and __module__ of the operator
wrapper functions.

PiperOrigin-RevId: 222963169
---
 .../python/kernel_tests/variables_test.py     |  7 ++
 .../python/ops/resource_variable_ops.py       | 35 --------
 tensorflow/python/ops/variables.py            | 82 ++++++-------------
 3 files changed, 31 insertions(+), 93 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 2bb75109b1..faa9f82067 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 import operator
 
 import numpy as np
@@ -309,6 +310,12 @@ class VariablesTestCase(test.TestCase):
       self.assertEqual([var_x], variables.trainable_variables("scope_1"))
       self.assertEqual([var_y], variables.trainable_variables("scope_2"))
 
+  def testOperatorWrapping(self):
+    for attr in functools.WRAPPER_ASSIGNMENTS:
+      self.assertEqual(
+          getattr(variables.Variable.__add__, attr),
+          getattr(ops.Tensor.__add__, attr))
+
   def testOperators(self):
     with self.cached_session():
       var_f = variables.Variable([2.0])
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index f614c48485..5c74dffb05 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -805,16 +805,6 @@ class ResourceVariable(variables.RefVariable):
     return ResourceVariable(
         variable_def=variable_def, import_scope=import_scope)
 
-  @staticmethod
-  def _OverloadAllOperators():  # pylint: disable=invalid-name
-    """Register overloads for all operators."""
-    for operator in ops.Tensor.OVERLOADABLE_OPERATORS:
-      ResourceVariable._OverloadOperator(operator)
-    # For slicing, bind getitem differently than a tensor (use SliceHelperVar
-    # instead)
-    # pylint: disable=protected-access
-    setattr(ResourceVariable, "__getitem__", array_ops._SliceHelperVar)
-
   def _AsTensor(self):
     return self.value()
 
@@ -826,30 +816,6 @@ class ResourceVariable(variables.RefVariable):
     """Unsupported."""
     raise NotImplementedError("ResourceVariable does not implement set_shape()")
 
-  @staticmethod
-  def _OverloadOperator(operator):  # pylint: disable=invalid-name
-    """Defer an operator overload to `ops.Tensor`.
-
-    We pull the operator out of ops.Tensor dynamically to avoid ordering issues.
-
-    Args:
-      operator: string. The operator name.
-    """
-
-    tensor_oper = getattr(ops.Tensor, operator)
-    def _run_op(a, *args):
-      # pylint: disable=protected-access
-      value = a._AsTensor()
-      return tensor_oper(value, *args)
-
-    # Propagate __doc__ to wrapper
-    try:
-      _run_op.__doc__ = tensor_oper.__doc__
-    except AttributeError:
-      pass
-
-    setattr(ResourceVariable, operator, _run_op)
-
   __array_priority__ = 100
 
   def is_initialized(self, name=None):
@@ -1435,7 +1401,6 @@ ops.register_tensor_conversion_function(
     variables.Variable, variables.Variable._TensorConversionFunction)  # pylint: disable=protected-access
 
 # pylint: disable=protected-access
-ResourceVariable._OverloadAllOperators()
 ops.register_dense_tensor_like_type(ResourceVariable)
 
 
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 438bc912c7..f72b19bcdd 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -18,7 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import enum  # pylint: disable=g-bad-import-order
-
+import functools
 import os
 import six
 
@@ -861,18 +861,18 @@ class Variable(six.with_metaclass(VariableMetaclass,
     else:
       return v.value()
 
-  @staticmethod
-  def _OverloadAllOperators():  # pylint: disable=invalid-name
+  @classmethod
+  def _OverloadAllOperators(cls):  # pylint: disable=invalid-name
     """Register overloads for all operators."""
     for operator in ops.Tensor.OVERLOADABLE_OPERATORS:
-      Variable._OverloadOperator(operator)
+      cls._OverloadOperator(operator)
     # For slicing, bind getitem differently than a tensor (use SliceHelperVar
     # instead)
     # pylint: disable=protected-access
-    setattr(Variable, "__getitem__", array_ops._SliceHelperVar)
+    setattr(cls, "__getitem__", array_ops._SliceHelperVar)
 
-  @staticmethod
-  def _OverloadOperator(operator):  # pylint: disable=invalid-name
+  @classmethod
+  def _OverloadOperator(cls, operator):  # pylint: disable=invalid-name
     """Defer an operator overload to `ops.Tensor`.
 
     We pull the operator out of ops.Tensor dynamically to avoid ordering issues.
@@ -880,17 +880,26 @@ class Variable(six.with_metaclass(VariableMetaclass,
     Args:
       operator: string. The operator name.
     """
+    tensor_oper = getattr(ops.Tensor, operator)
 
-    def _run_op(a, *args):
+    def _run_op(a, *args, **kwargs):
       # pylint: disable=protected-access
-      return getattr(ops.Tensor, operator)(a._AsTensor(), *args)
-    # Propagate __doc__ to wrapper
-    try:
-      _run_op.__doc__ = getattr(ops.Tensor, operator).__doc__
-    except AttributeError:
-      pass
+      return tensor_oper(a._AsTensor(), *args, **kwargs)
 
-    setattr(Variable, operator, _run_op)
+    functools.update_wrapper(_run_op, tensor_oper)
+    setattr(cls, operator, _run_op)
+
+  def __iter__(self):
+    """Dummy method to prevent iteration. Do not call.
+
+    NOTE(mrry): If we register __getitem__ as an overloaded operator,
+    Python will valiantly attempt to iterate over the variable's Tensor from 0
+    to infinity.  Declaring this method prevents this unintended behavior.
+
+    Raises:
+      TypeError: when invoked.
+    """
+    raise TypeError("'Variable' object is not iterable.")
 
   # NOTE(mrry): This enables the Variable's overloaded "right" binary
   # operators to run when the left operand is an ndarray, because it
@@ -1577,18 +1586,6 @@ class RefVariable(VariableV1):
     """
     return self._snapshot
 
-  def __iter__(self):
-    """Dummy method to prevent iteration. Do not call.
-
-    NOTE(mrry): If we register __getitem__ as an overloaded operator,
-    Python will valiantly attempt to iterate over the variable's Tensor from 0
-    to infinity.  Declaring this method prevents this unintended behavior.
-
-    Raises:
-      TypeError: when invoked.
-    """
-    raise TypeError("'Variable' object is not iterable.")
-
   def value(self):
     """Returns the last snapshot of this variable.
 
@@ -2124,37 +2121,6 @@ class RefVariable(VariableV1):
     else:
       return v.value()
 
-  @staticmethod
-  def _OverloadAllOperators():  # pylint: disable=invalid-name
-    """Register overloads for all operators."""
-    for operator in ops.Tensor.OVERLOADABLE_OPERATORS:
-      Variable._OverloadOperator(operator)  # pylint: disable=protected-access
-    # For slicing, bind getitem differently than a tensor (use SliceHelperVar
-    # instead)
-    # pylint: disable=protected-access
-    setattr(Variable, "__getitem__", array_ops._SliceHelperVar)
-
-  @staticmethod
-  def _OverloadOperator(operator):  # pylint: disable=invalid-name
-    """Defer an operator overload to `ops.Tensor`.
-
-    We pull the operator out of ops.Tensor dynamically to avoid ordering issues.
-
-    Args:
-      operator: string. The operator name.
-    """
-
-    def _run_op(a, *args):
-      # pylint: disable=protected-access
-      return getattr(ops.Tensor, operator)(a._AsTensor(), *args)
-    # Propagate __doc__ to wrapper
-    try:
-      _run_op.__doc__ = getattr(ops.Tensor, operator).__doc__
-    except AttributeError:
-      pass
-
-    setattr(Variable, operator, _run_op)
-
   def _gather_saveables_for_checkpoint(self):
     """For implementing `Checkpointable`. This object is saveable on its own."""
     return {checkpointable.VARIABLE_VALUE_KEY: self}
-- 
GitLab


From e4efa68f3ca39da953040990c45ab9435e6c5aee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 03:30:39 -0800
Subject: [PATCH 0854/1554] Visibility change for tensorflow contrib library.

PiperOrigin-RevId: 222966504
---
 tensorflow/contrib/tpu/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index a0a9cb3f31..999274018b 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -14,6 +14,7 @@ load("//tensorflow:tensorflow.bzl", "tf_py_test")
 package(
     default_visibility = [
         "//cloud/vmm/testing/tests/tpu:__subpackages__",
+        "//knowledge/cerebra/sense/im2query:__subpackages__",
         "//learning/brain:__subpackages__",
         "//learning/deepmind:__subpackages__",
         "//medical/pathology:__subpackages__",
-- 
GitLab


From 5bb5d6db8ac8f48ba2636ae18844cb584b837904 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 27 Nov 2018 03:34:09 -0800
Subject: [PATCH 0855/1554] Pushed Variable.from_proto implementation to
 RefVariable

It was not generic enough to be in the base class.

PiperOrigin-RevId: 222966856
---
 tensorflow/python/ops/resource_variable_ops.py | 2 +-
 tensorflow/python/ops/variables.py             | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 5c74dffb05..f9faa3e945 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1433,7 +1433,7 @@ def _from_proto_fn(v, import_scope=None):
   """Creates Variable or ResourceVariable from VariableDef as needed."""
   if v.is_resource:
     return ResourceVariable.from_proto(v, import_scope=import_scope)
-  return variables.Variable.from_proto(v, import_scope=import_scope)
+  return variables.RefVariable.from_proto(v, import_scope=import_scope)
 
 
 ops.register_proto_function(
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index f72b19bcdd..a001374e9a 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -968,8 +968,7 @@ class Variable(six.with_metaclass(VariableMetaclass,
   @staticmethod
   def from_proto(variable_def, import_scope=None):
     """Returns a `Variable` object created from `variable_def`."""
-    return RefVariable(variable_def=variable_def,
-                       import_scope=import_scope)
+    raise NotImplementedError
 
   class SaveSliceInfo(object):
     """Information on how to save this Variable as a slice.
@@ -2344,6 +2343,12 @@ class RefVariable(VariableV1):
     else:
       return None
 
+  @staticmethod
+  def from_proto(variable_def, import_scope=None):
+    """Returns a `Variable` object created from `variable_def`."""
+    return RefVariable(variable_def=variable_def,
+                       import_scope=import_scope)
+
   def __iadd__(self, other):
     logging.log_first_n(
         logging.WARN,
-- 
GitLab


From 956e75535d25d5fdabac4a3a25f85881762cbc0f Mon Sep 17 00:00:00 2001
From: Peter Buchlovsky <petebu@google.com>
Date: Tue, 27 Nov 2018 04:31:08 -0800
Subject: [PATCH 0856/1554] Fix incompatible device colocation warning.

PiperOrigin-RevId: 222971307
---
 tensorflow/python/distribute/mirrored_strategy.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 402a94c2b7..d6d40df5ce 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -457,7 +457,8 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     if self._cluster_spec:
       worker_device_pairs = self._worker_devices
     else:
-      worker_device_pairs = [("/job:localhost", self._devices)]
+      worker = device_util.canonicalize("/device:CPU:0")
+      worker_device_pairs = [(worker, self._devices)]
     return values.DatasetIterator(dataset, worker_device_pairs,
                                   self._num_replicas_in_sync)
 
@@ -471,7 +472,8 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
       worker_device_pairs = self._worker_devices
     else:
       num_workers = 1
-      worker_device_pairs = [("/job:localhost", self._devices)]
+      worker = device_util.canonicalize("/device:CPU:0")
+      worker_device_pairs = [(worker, self._devices)]
     for i in range(num_workers):
       input_contexts.append(distribute_lib.InputContext(
           num_input_pipelines=num_workers,
-- 
GitLab


From f56925c38e4fe67fba3e4da7ee8bbb72b1e69d38 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 05:38:43 -0800
Subject: [PATCH 0857/1554] update eigen to the commit:
 https://bitbucket.org/eigen/eigen/commits/efda481cbd7a

PiperOrigin-RevId: 222976986
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index d9d40874a4..0725bd0e8a 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "8fa7ba1af23f0320be05f4658061138d6eb8dd1f320669cbf305b3a034f9d1c2",
-        strip_prefix = "eigen-eigen-ea671884cc96",
+        sha256 = "fc0f871496cdaec892245afc9890e8267f73b0fcec5a7f75be0dc914e2972023",
+        strip_prefix = "eigen-eigen-efda481cbd7a",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/efda481cbd7a.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/efda481cbd7a.tar.gz",
         ],
     )
 
-- 
GitLab


From 84337310517914ca4b4d6eb35295a65758bc6d75 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 08:17:25 -0800
Subject: [PATCH 0858/1554] Unittest to show overflow characteristics of
 AveragePool.

PiperOrigin-RevId: 222993846
---
 tensorflow/lite/kernels/pooling_test.cc | 43 +++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/tensorflow/lite/kernels/pooling_test.cc b/tensorflow/lite/kernels/pooling_test.cc
index 80eef02509..98777f1c13 100644
--- a/tensorflow/lite/kernels/pooling_test.cc
+++ b/tensorflow/lite/kernels/pooling_test.cc
@@ -67,6 +67,10 @@ class QuantizedPoolingOpModel : public BasePoolingOpModel {
     QuantizeAndPopulate<uint8_t>(input_, data);
   }
 
+  void SetInput(const std::vector<float>& data) {
+    QuantizeAndPopulate<uint8_t>(input_, data);
+  }
+
   std::vector<uint8_t> GetOutput() { return ExtractVector<uint8_t>(output_); }
   std::vector<float> GetDequantizedOutput() {
     return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
@@ -106,6 +110,45 @@ TEST(QuantizedPoolingOpTest, AveragePool) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({44, 92}));
 }
 
+// Send in a white image, expect a white pixel.
+TEST(QuantizedPoolingOpTest, AveragePoolImageSize16) {
+  int image_size = 16;
+  QuantizedPoolingOpModel m(
+      BuiltinOperator_AVERAGE_POOL_2D,
+      /*input=*/{TensorType_UINT8, {1, image_size, image_size, 1}, 0, 16},
+      /*filter_width=*/image_size,
+      /*filter_height=*/image_size,
+      /*output=*/{TensorType_UINT8, {}, 0, 16});
+
+  std::vector<float> input(image_size * image_size, 16.f);
+  m.SetInput(input);
+  m.Invoke();
+
+  EXPECT_THAT(m.GetOutput(), ::testing::ElementsAre(255));
+  EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear({16})));
+}
+
+// Send in a white image, expect something other than a white pixel, due to
+// overflow.
+TEST(QuantizedPoolingOpTest, AveragePoolImageSize17) {
+  int image_size = 17;
+  QuantizedPoolingOpModel m(
+      BuiltinOperator_AVERAGE_POOL_2D,
+      /*input=*/{TensorType_UINT8, {1, image_size, image_size, 1}, 0, 16},
+      /*filter_width=*/image_size,
+      /*filter_height=*/image_size,
+      /*output=*/{TensorType_UINT8, {}, 0, 16});
+
+  std::vector<float> input(image_size * image_size, 16.f);
+  m.SetInput(input);
+  m.Invoke();
+
+  // Ordinarily we would see '255' here. However, the optimized version of
+  // AveragePool uses a uint16 accumulator which causes it to overflow for
+  // images this large.
+  EXPECT_THAT(m.GetOutput(), ::testing::ElementsAre(28));
+}
+
 TEST(FloatPoolingOpTest, MaxPool) {
   FloatPoolingOpModel m(BuiltinOperator_MAX_POOL_2D,
                         /*input=*/{TensorType_FLOAT32, {1, 2, 4, 1}},
-- 
GitLab


From 7a0d3c9dd52889a9424b4f9907ad55a5f76fa6df Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 27 Nov 2018 08:35:25 -0800
Subject: [PATCH 0859/1554] Link ArgMax custom call op in xla_cpu_jit

PiperOrigin-RevId: 222996011
---
 tensorflow/compiler/jit/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 6b0d20bc18..1f6dadf1ac 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -50,6 +50,7 @@ cc_library(
     deps = [
         ":jit_compilation_passes",
         "//tensorflow/compiler/jit/kernels:xla_ops",
+        "//tensorflow/compiler/tf2xla/kernels:xla_cpu_only_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_dummy_ops",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/xla/service:cpu_plugin",
-- 
GitLab


From b0cfc0bf65e8372d9cf6da078509e1988d5e2fad Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 27 Nov 2018 08:46:21 -0800
Subject: [PATCH 0860/1554] [TF:XLA] Small refactorings of XlaContext and
 XlaOpKernelContext. No functional changes intended.

* Add an XlaOpKernelContext::xla_context() method, and change most users of XlaContext::Get() to use XlaOpKernelContext methods instead.
* Remove XlaContext::CreateResource, which had a confusing API that mirror XlaResource's constructor. Replace it with XlaContext::AddResource, which takes ownership of a resource, and add helper methods to build XlaResources.
* Rename XlaResource::tensor_array_size to XlaResource::max_array_size, since it also applies to stacks as well as tensor arrays.
* Remove XlaContext::RepresentationShape and XlaContext::allow_cpu_custom_calls. XlaContext already has a reference to the XlaCompiler, and these can be determined from XlaCompiler::options(), and are therefore redundant.

PiperOrigin-RevId: 222997360
---
 tensorflow/compiler/tf2xla/kernels/arg_op.cc  |  2 +-
 tensorflow/compiler/tf2xla/kernels/if_op.cc   |  2 +-
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  |  2 +-
 .../compiler/tf2xla/kernels/retval_op.cc      |  3 +-
 .../compiler/tf2xla/kernels/sendrecv_ops.cc   |  4 +-
 .../compiler/tf2xla/kernels/stack_ops.cc      | 19 +++----
 .../tf2xla/kernels/tensor_array_ops.cc        | 33 +++++------
 .../compiler/tf2xla/kernels/while_op.cc       |  2 +-
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 40 +++++++-------
 tensorflow/compiler/tf2xla/xla_compiler.h     |  2 +-
 .../compiler/tf2xla/xla_compiler_test.cc      |  6 +-
 tensorflow/compiler/tf2xla/xla_context.cc     | 35 ++----------
 tensorflow/compiler/tf2xla/xla_context.h      | 32 +----------
 tensorflow/compiler/tf2xla/xla_op_kernel.cc   | 55 ++++++++++---------
 tensorflow/compiler/tf2xla/xla_op_kernel.h    |  2 +
 tensorflow/compiler/tf2xla/xla_resource.cc    | 35 +++++++++---
 tensorflow/compiler/tf2xla/xla_resource.h     | 19 +++++--
 17 files changed, 134 insertions(+), 159 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/arg_op.cc b/tensorflow/compiler/tf2xla/kernels/arg_op.cc
index 2db2514397..795ea09831 100644
--- a/tensorflow/compiler/tf2xla/kernels/arg_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/arg_op.cc
@@ -50,7 +50,7 @@ class XlaArgOp : public XlaOpKernel {
       return;
     }
 
-    const XlaExpression& arg = XlaContext::Get(ctx).args()[index_];
+    const XlaExpression& arg = ctx->xla_context()->args()[index_];
     OP_REQUIRES(ctx, arg.kind() != XlaExpression::Kind::kInvalid,
                 errors::InvalidArgument("Invalid/missing argument expression"));
     ctx->SetOutputExpression(0, arg);
diff --git a/tensorflow/compiler/tf2xla/kernels/if_op.cc b/tensorflow/compiler/tf2xla/kernels/if_op.cc
index 56da50f140..b5e0839125 100644
--- a/tensorflow/compiler/tf2xla/kernels/if_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/if_op.cc
@@ -72,7 +72,7 @@ void XlaIfOp::Compile(XlaOpKernelContext* ctx) {
       arg.shape = resource->shape();
       OP_REQUIRES(ctx, arg.initialized,
                   errors::Unimplemented("Uninitialized arguments: ", arg.name));
-      arg.tensor_array_size = resource->tensor_array_size();
+      arg.max_array_size = resource->max_array_size();
       for (const auto& gradient : resource->tensor_array_gradients()) {
         arg.tensor_array_gradients.insert(gradient.first);
       }
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index 20b7176041..e2c05b648b 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -72,7 +72,7 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
 
     // Fall back to XLA ArgMax HLO when CustomCall is not allowed or when input
     // shape isn't supported.
-    if (!XlaContext::Get(ctx).allow_cpu_custom_calls() ||
+    if (!ctx->compiler()->options().allow_cpu_custom_calls ||
         (input_dims != 1 && input_dims != 2)) {
       xla::XlaOp output = XlaHelpers::ArgMax(ctx->Input(0), output_type, axis);
       ctx->SetOutput(0, output);
diff --git a/tensorflow/compiler/tf2xla/kernels/retval_op.cc b/tensorflow/compiler/tf2xla/kernels/retval_op.cc
index 6970dd0a00..e4046c7955 100644
--- a/tensorflow/compiler/tf2xla/kernels/retval_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/retval_op.cc
@@ -47,8 +47,7 @@ class RetvalOp : public XlaOpKernel {
       // compilation.
       OP_REQUIRES_OK(ctx, frame->SetRetval(index_, input));
     } else {
-      XlaContext& xla_context = XlaContext::Get(ctx);
-      xla_context.SetRetval(index_, ctx->InputExpression(0));
+      ctx->xla_context()->SetRetval(index_, ctx->InputExpression(0));
     }
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc
index a7f5a8f169..84470b230d 100644
--- a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc
@@ -42,7 +42,7 @@ SendOp::SendOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
 }
 
 void SendOp::Compile(XlaOpKernelContext* ctx) {
-  XlaCompiler* compiler = XlaContext::Get(ctx).compiler();
+  XlaCompiler* compiler = ctx->compiler();
   xla::ChannelHandle channel;
   OP_REQUIRES_OK(ctx, compiler->GetChannelHandle(tensor_name_, &channel));
   xla::Send(ctx->Input(0), channel);
@@ -73,7 +73,7 @@ RecvOp::RecvOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
 }
 
 void RecvOp::Compile(XlaOpKernelContext* ctx) {
-  XlaCompiler* compiler = XlaContext::Get(ctx).compiler();
+  XlaCompiler* compiler = ctx->compiler();
   xla::ChannelHandle channel;
   OP_REQUIRES_OK(ctx, compiler->GetChannelHandle(tensor_name_, &channel));
   ctx->SetOutput(0, xla::Recv(ctx->builder(), shape_, channel));
diff --git a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
index 7b96b43ad8..8e9e4daf99 100644
--- a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
@@ -69,7 +69,7 @@ Status MaybeInitializeStack(xla::XlaBuilder* builder, XlaResource* resource,
   }
 
   TensorShape stack_shape;
-  stack_shape.AddDim(resource->tensor_array_size());
+  stack_shape.AddDim(resource->max_array_size());
   stack_shape.AppendShape(elem_shape);
 
   if (!resource->initialized()) {
@@ -97,10 +97,10 @@ class StackOp : public XlaOpKernel {
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    int64 size;
-    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(0, &size));
+    int64 max_size;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(0, &max_size));
     OP_REQUIRES(
-        ctx, size >= 0,
+        ctx, max_size >= 0,
         errors::InvalidArgument(
             "XLA compilation requires a fixed stack size upper bound. If "
             "you are using tf.while_loop, set the maximum_iterations parameter "
@@ -108,14 +108,9 @@ class StackOp : public XlaOpKernel {
 
     // We defer initializing the Stack resource until we see the first push.
     // Otherwise we do not know the shape of the stack elements.
-    xla::XlaOp value;
-    XlaContext& xc = XlaContext::Get(ctx);
-    XlaResource* resource;
-    string name = absl::StrCat("Stack: ", stack_name_);
-    OP_REQUIRES_OK(
-        ctx, xc.CreateResource(XlaResource::kStack, -1, std::move(name), dtype_,
-                               TensorShape(), value, /*tensor_array_size=*/size,
-                               /*tensor_array_gradients=*/{}, &resource));
+    XlaResource* resource =
+        ctx->xla_context()->AddResource(XlaResource::CreateStack(
+            /*name=*/absl::StrCat("Stack: ", stack_name_), dtype_, max_size));
     ctx->SetResourceOutput(0, resource);
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
index 252967a746..939d7e1951 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
@@ -61,8 +61,8 @@ Status MaybeInitializeTensorArray(xla::XlaBuilder* builder,
         " but op has dtype ", DataTypeString(dtype), ".");
   }
 
-  TF_RET_CHECK(resource->tensor_array_size() >= 0)
-      << resource->name() << " size " << resource->tensor_array_size();
+  TF_RET_CHECK(resource->max_array_size() >= 0)
+      << resource->name() << " size " << resource->max_array_size();
 
   if (!resource->initialized()) {
     TF_RETURN_IF_ERROR(resource->SetTypeAndShape(dtype, elem_shape));
@@ -78,7 +78,7 @@ Status MaybeInitializeTensorArray(xla::XlaBuilder* builder,
         XLAShapeToTensorShape(shape_or_status.ValueOrDie(), &shape));
 
     TensorShape ta_shape;
-    ta_shape.AddDim(resource->tensor_array_size());
+    ta_shape.AddDim(resource->max_array_size());
     ta_shape.AppendShape(elem_shape);
     if (ta_shape != shape) {
       return errors::InvalidArgument(
@@ -114,7 +114,7 @@ Status CheckTensorArrayIsInitialized(const string& op_name,
 Status GetTensorArrayShape(const XlaResource* resource,
                            xla::XlaBuilder* builder, TensorShape* shape) {
   *shape = resource->shape();
-  shape->InsertDim(0, resource->tensor_array_size());
+  shape->InsertDim(0, resource->max_array_size());
   return Status::OK();
 }
 
@@ -166,13 +166,10 @@ class TensorArrayOp : public XlaOpKernel {
       value = xla::Broadcast(zero, ta_shape.dim_sizes());
     }
 
-    XlaContext& xc = XlaContext::Get(ctx);
-    XlaResource* var;
-    string name = absl::StrCat("TensorArray: ", tensor_array_name_);
-    OP_REQUIRES_OK(
-        ctx, xc.CreateResource(XlaResource::kTensorArray, -1, std::move(name),
-                               dtype_, shape, value, /*tensor_array_size=*/size,
-                               /*tensor_array_gradients=*/{}, &var));
+    XlaResource* var =
+        ctx->xla_context()->AddResource(XlaResource::CreateTensorArray(
+            /*name=*/absl::StrCat("TensorArray: ", tensor_array_name_), dtype_,
+            shape, /*initial_value=*/value, /*max_array_size=*/size));
     ctx->SetResourceOutput(0, var);
 
     Tensor flow(DT_FLOAT, TensorShape({}));
@@ -517,14 +514,13 @@ class TensorArraySplitOp : public XlaOpKernel {
     xla::XlaOp ta = resource->value();
 
     TensorShape ta_shape;
-    ta_shape.AddDim(resource->tensor_array_size());
+    ta_shape.AddDim(resource->max_array_size());
     ta_shape.AppendShape(elem_shape);
 
-    OP_REQUIRES(
-        ctx, lengths.size() == resource->tensor_array_size(),
-        errors::InvalidArgument(
-            "TensorArray's size is not equal to the size of lengths (",
-            lengths.size(), " vs. ", resource->tensor_array_size(), ")"));
+    OP_REQUIRES(ctx, lengths.size() == resource->max_array_size(),
+                errors::InvalidArgument(
+                    "TensorArray's size is not equal to the size of lengths (",
+                    lengths.size(), " vs. ", resource->max_array_size(), ")"));
 
     const xla::XlaOp value = ctx->Input(1);
     const xla::XlaOp flow = ctx->Input(3);
@@ -562,8 +558,7 @@ class TensorArraySizeOp : public XlaOpKernel {
     XlaResource* var;
     OP_REQUIRES_OK(ctx, ctx->GetResourceInput(0, &var));
     Tensor size_tensor(DT_INT32, {});
-    size_tensor.scalar<int32>()() =
-        static_cast<int32>(var->tensor_array_size());
+    size_tensor.scalar<int32>()() = static_cast<int32>(var->max_array_size());
     ctx->SetConstantOutput(0, size_tensor);
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/while_op.cc b/tensorflow/compiler/tf2xla/kernels/while_op.cc
index 559414eeaa..ce007fc04a 100644
--- a/tensorflow/compiler/tf2xla/kernels/while_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/while_op.cc
@@ -64,7 +64,7 @@ Status MakeXlaCompilerArgumentsFromInputs(
       if (!arg.initialized) {
         *has_uninitialized_vars = true;
       }
-      arg.tensor_array_size = resource->tensor_array_size();
+      arg.max_array_size = resource->max_array_size();
       for (const auto& gradient : resource->tensor_array_gradients()) {
         arg.tensor_array_gradients.insert(gradient.first);
       }
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 8036bc6844..7fdd60145f 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -326,10 +326,10 @@ Status BuildComputation(
 
 bool XlaCompiler::Argument::operator==(
     const XlaCompiler::Argument& other) const {
-  if (std::tie(kind, resource_kind, type, name, initialized, tensor_array_size,
+  if (std::tie(kind, resource_kind, type, name, initialized, max_array_size,
                tensor_array_gradients) !=
       std::tie(other.kind, other.resource_kind, other.type, other.name,
-               other.initialized, other.tensor_array_size,
+               other.initialized, other.max_array_size,
                other.tensor_array_gradients)) {
     return false;
   }
@@ -359,8 +359,8 @@ string XlaCompiler::Argument::HumanString() const {
       string output = absl::StrCat("kind=resource", common, " resource_kind=",
                                    XlaResource::KindToString(resource_kind),
                                    " initialized=", initialized);
-      if (tensor_array_size >= 0) {
-        absl::StrAppend(&output, " tensor_array_size=", tensor_array_size);
+      if (max_array_size >= 0) {
+        absl::StrAppend(&output, " max_array_size=", max_array_size);
       }
       if (!tensor_array_gradients.empty()) {
         absl::StrAppend(&output, " tensor_array_gradients=",
@@ -567,12 +567,12 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg,
           return Status::OK();
         }
         case XlaResource::kTensorArray: {
-          if (arg.tensor_array_size < 0) {
+          if (arg.max_array_size < 0) {
             return errors::InvalidArgument(
-                "Negative tensor_array_size in XLAShapeForArgument");
+                "Negative max_array_size in XLAShapeForArgument");
           }
           TensorShape shape;
-          shape.AddDim(arg.tensor_array_size);
+          shape.AddDim(arg.max_array_size);
           shape.AppendShape(arg.shape);
           TF_RETURN_IF_ERROR(TensorShapeToXLAShape(arg.type, shape, xla_shape));
 
@@ -584,12 +584,12 @@ Status XlaCompiler::XLAShapeForArgument(const XlaCompiler::Argument& arg,
           return Status::OK();
         }
         case XlaResource::kStack: {
-          if (arg.tensor_array_size < 0) {
+          if (arg.max_array_size < 0) {
             return errors::InvalidArgument(
-                "Negative tensor_array_size in XLAShapeForArgument");
+                "Negative max_array_size in XLAShapeForArgument");
           }
           TensorShape shape;
-          shape.AddDim(arg.tensor_array_size);
+          shape.AddDim(arg.max_array_size);
           shape.AppendShape(arg.shape);
           xla::Shape buffer_shape;
           TF_RETURN_IF_ERROR(
@@ -635,21 +635,23 @@ Status XlaCompiler::BuildArguments(
     const XlaCompiler::Argument& arg = args[i];
     XlaExpression& arg_expression = (*arg_expressions)[i];
     switch (arg.kind) {
-      case XlaCompiler::Argument::kResource:
+      case XlaCompiler::Argument::kResource: {
         TF_RET_CHECK(arg.resource_kind != XlaResource::kInvalid);
         // TODO(phawkins): this code assumes that resource arguments do not
         // alias.
-        XlaResource* resource;
-        TF_RETURN_IF_ERROR(context->CreateResource(
-            arg.resource_kind, i, arg.name, arg.type, arg.shape, xla::XlaOp(),
-            /*tensor_array_size=*/arg.tensor_array_size,
-            /*tensor_array_gradients=*/arg.tensor_array_gradients, &resource));
+        XlaResource* resource =
+            context->AddResource(absl::make_unique<XlaResource>(
+                arg.resource_kind, i, arg.name, arg.type, arg.shape,
+                xla::XlaOp(),
+                /*max_array_size=*/arg.max_array_size,
+                /*tensor_array_gradients=*/arg.tensor_array_gradients,
+                /*tensor_array_multiple_writes_aggregate=*/true));
         arg_expression = XlaExpression::Resource(resource);
         if (arg.initialized) {
           input_mapping->push_back(i);
         }
-
         break;
+      }
       case XlaCompiler::Argument::kParameter:
       case XlaCompiler::Argument::kToken: {
         input_mapping->push_back(i);
@@ -923,9 +925,7 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
                                    options_.device_type, name));
 
   xla::XlaBuilder builder(name);
-  XlaContext* context =
-      new XlaContext(this, &builder, options_.allow_cpu_custom_calls,
-                     &options_.shape_representation_fn);
+  XlaContext* context = new XlaContext(this, &builder);
   core::ScopedUnref context_unref(context);
 
   std::vector<XlaCompiler::Argument> real_args(args.begin(), args.end());
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 6342612468..0d801b73a8 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -150,7 +150,7 @@ class XlaCompiler {
 
     // For a TensorArray or Stack resource, what is the array's declared size?
     // (Used for lazy initialization.)
-    int64 tensor_array_size = -1;
+    int64 max_array_size = -1;
 
     // TensorArray resource parameters are passed as (array, gradient array 0,
     // ..., gradient array k), where the gradient arrays are in the same order
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index eba5d77efa..fe2a5f5b0c 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -650,7 +650,7 @@ TEST_F(XlaCompilerTest, CanPassTensorArraysToAndFromComputation) {
   args[0].initialized = true;
   args[0].type = DT_INT32;
   args[0].shape = TensorShape({});
-  args[0].tensor_array_size = 2;
+  args[0].max_array_size = 2;
   args[0].tensor_array_gradients = {"grad2"};
 
   // Compiles the graph.
@@ -709,7 +709,7 @@ TEST_F(XlaCompilerTest, UnwrittenTensorArrayGradientsAreNotComputationOutputs) {
   args[0].initialized = true;
   args[0].type = DT_INT32;
   args[0].shape = TensorShape({});
-  args[0].tensor_array_size = 2;
+  args[0].max_array_size = 2;
   args[0].tensor_array_gradients = {"grad1"};
 
   // Compiles the graph.
@@ -741,7 +741,7 @@ TEST_F(XlaCompilerTest, NewTensorArrayGradientsAreComputationOutputs) {
   args[0].initialized = true;
   args[0].type = DT_INT32;
   args[0].shape = TensorShape({});
-  args[0].tensor_array_size = 2;
+  args[0].max_array_size = 2;
   args[0].tensor_array_gradients = {"grad1"};
 
   // Compiles the graph.
diff --git a/tensorflow/compiler/tf2xla/xla_context.cc b/tensorflow/compiler/tf2xla/xla_context.cc
index 43095fbb47..a69af70503 100644
--- a/tensorflow/compiler/tf2xla/xla_context.cc
+++ b/tensorflow/compiler/tf2xla/xla_context.cc
@@ -54,25 +54,14 @@ const char XlaContext::kXlaContextResourceName[] = "_xla_context";
   return *context;
 }
 
-/* static */ XlaContext& XlaContext::Get(const XlaOpKernelContext* ctx) {
-  return Get(ctx->op_kernel_context());
-}
-
 void XlaContext::set_args(std::vector<XlaExpression> args) {
   args_ = std::move(args);
 }
 
-XlaContext::XlaContext(
-    XlaCompiler* compiler, xla::XlaBuilder* builder,
-    bool allow_cpu_custom_calls,
-    const std::function<xla::StatusOr<xla::Shape>(
-        const TensorShape&, DataType)>* shape_representation_fn)
-    : compiler_(compiler),
-      builder_(builder),
-      allow_cpu_custom_calls_(allow_cpu_custom_calls),
-      shape_representation_fn_(shape_representation_fn) {}
+XlaContext::XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder)
+    : compiler_(compiler), builder_(builder) {}
 
-string XlaContext::DebugString() { return "TLA JIT context"; }
+string XlaContext::DebugString() { return "XLA JIT context"; }
 
 void XlaContext::SetRetval(int index, const XlaExpression& expression) {
   if (retvals_.size() <= index) {
@@ -81,21 +70,9 @@ void XlaContext::SetRetval(int index, const XlaExpression& expression) {
   retvals_[index] = expression;
 }
 
-Status XlaContext::CreateResource(
-    XlaResource::Kind kind, int arg_num, string name, DataType type,
-    TensorShape shape, const xla::XlaOp& handle, int64 tensor_array_size,
-    const std::set<string>& tensor_array_gradients, XlaResource** resource) {
-  resources_.emplace_back(
-      new XlaResource(kind, arg_num, std::move(name), type, std::move(shape),
-                      handle, tensor_array_size, tensor_array_gradients,
-                      /*tensor_array_multiple_writes_aggregate=*/false));
-  *resource = resources_.back().get();
-  return Status::OK();
-}
-
-xla::StatusOr<xla::Shape> XlaContext::RepresentationShape(
-    const TensorShape& shape, DataType type) const {
-  return (*shape_representation_fn_)(shape, type);
+XlaResource* XlaContext::AddResource(std::unique_ptr<XlaResource> resource) {
+  resources_.push_back(std::move(resource));
+  return resources_.back().get();
 }
 
 const xla::XlaComputation* XlaContext::GetOrCreateMax(const DataType type) {
diff --git a/tensorflow/compiler/tf2xla/xla_context.h b/tensorflow/compiler/tf2xla/xla_context.h
index dbfd344c9b..0767d1faac 100644
--- a/tensorflow/compiler/tf2xla/xla_context.h
+++ b/tensorflow/compiler/tf2xla/xla_context.h
@@ -41,14 +41,10 @@ class XlaContext : public ResourceBase {
  public:
   // Retrieves the XlaContext of the current compilation.
   static XlaContext& Get(const OpKernelContext* ctx);
-  static XlaContext& Get(const XlaOpKernelContext* ctx);
 
   // Creates a new XlaContext. See the documentation on the class data fields
   // for descriptions of the arguments.
-  XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder,
-             bool allow_cpu_custom_calls,
-             const std::function<xla::StatusOr<xla::Shape>(
-                 const TensorShape&, DataType)>* shape_representation_fn);
+  XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder);
 
   // Virtual method defined by ResourceBase.
   string DebugString() override;
@@ -58,8 +54,6 @@ class XlaContext : public ResourceBase {
   // Returns the XlaBuilder that Ops use for compiling new expressions.
   xla::XlaBuilder* builder() { return builder_; }
 
-  bool allow_cpu_custom_calls() const { return allow_cpu_custom_calls_; }
-
   const std::vector<XlaExpression>& args() const { return args_; }
   void set_args(std::vector<XlaExpression> args);
 
@@ -70,25 +64,13 @@ class XlaContext : public ResourceBase {
   // grows the return values vector to size index+1 if it is smaller.
   void SetRetval(int index, const XlaExpression& expression);
 
-  // Creates a resource with resource `kind` and initial value `handle`. `name`
-  // is a descriptive name for use in error messages. See the `XlaResource`
-  // constructor for a description of the remaining arguments.
-  // Fails if the resource already exists.
-  Status CreateResource(XlaResource::Kind kind, int arg_num, string name,
-                        DataType type, TensorShape shape,
-                        const xla::XlaOp& handle, int64 tensor_array_size,
-                        const std::set<string>& tensor_array_gradients,
-                        XlaResource** resource);
+  // Adds 'resource' to the set of resources owned by the context.
+  XlaResource* AddResource(std::unique_ptr<XlaResource> resource);
 
   const std::vector<std::unique_ptr<XlaResource>>& resources() {
     return resources_;
   }
 
-  // Returns the XLA shape to be used to represent a variable of TF `shape`
-  // and `type`, or of an argument or return value of a top-level computation.
-  xla::StatusOr<xla::Shape> RepresentationShape(const TensorShape& shape,
-                                                DataType type) const;
-
   // Get an XLA lambda to compute Max. This is cached in the
   // XlaContext since it may be used by multiple Ops. There is a
   // separate specialization of the computation for each DataType.
@@ -118,9 +100,6 @@ class XlaContext : public ResourceBase {
   // The XlaBuilder used to construct the subgraph's compiled representation.
   xla::XlaBuilder* builder_;
 
-  // Allow ops to emit CustomCall operations for CPU.
-  const bool allow_cpu_custom_calls_;
-
   // Arguments to the Tensorflow graph, indexed by _Arg index.
   // Includes both compile-time constant arguments and runtime parameters.
   std::vector<XlaExpression> args_;
@@ -131,11 +110,6 @@ class XlaContext : public ResourceBase {
   // Holds ownership of resources. The resources are not ordered.
   std::vector<std::unique_ptr<XlaResource>> resources_;
 
-  // Describes the on-host shapes of parameters and return values. Also see:
-  // XlaDevice::Options::shape_representation_fn.
-  const std::function<xla::StatusOr<xla::Shape>(const TensorShape&, DataType)>*
-      shape_representation_fn_;
-
   // Cache of prebuilt computations indexed by their type.
   using ComputationMap = std::map<DataType, xla::XlaComputation>;
 
diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc
index 8dd8def054..58808c76de 100644
--- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc
+++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc
@@ -36,8 +36,16 @@ bool XlaOpKernelContext::ValidateInputsAreSameShape(OpKernel* op) {
   return context_->ValidateInputsAreSameShape(op);
 }
 
+XlaContext* XlaOpKernelContext::xla_context() const {
+  return &XlaContext::Get(context_);
+}
+
 xla::XlaBuilder* XlaOpKernelContext::builder() const {
-  return XlaContext::Get(this).builder();
+  return xla_context()->builder();
+}
+
+XlaCompiler* XlaOpKernelContext::compiler() const {
+  return xla_context()->compiler();
 }
 
 // Retrieves an XlaExpression that was allocated by a previous Op.
@@ -338,8 +346,8 @@ Status XlaOpKernelContext::ConstantInputList(
 namespace {
 
 Status ReadVariableInputTensor(const Tensor& tensor, DataType type,
-                               const OpKernelContext* ctx, TensorShape* shape,
-                               xla::XlaOp* value) {
+                               const XlaOpKernelContext* ctx,
+                               TensorShape* shape, xla::XlaOp* value) {
   const XlaExpression* expression = CastExpressionFromTensor(tensor);
   XlaResource* variable = expression->resource();
   TF_RET_CHECK(variable != nullptr);
@@ -357,10 +365,9 @@ Status ReadVariableInputTensor(const Tensor& tensor, DataType type,
     *shape = variable->shape();
   }
 
-  XlaContext& xla_context = XlaContext::Get(ctx);
-  TF_ASSIGN_OR_RETURN(
-      xla::Shape representation_shape,
-      xla_context.RepresentationShape(variable->shape(), variable->type()));
+  TF_ASSIGN_OR_RETURN(xla::Shape representation_shape,
+                      ctx->compiler()->options().shape_representation_fn(
+                          variable->shape(), variable->type()));
   xla::Shape xla_shape;
   TF_RETURN_IF_ERROR(
       TensorShapeToXLAShape(variable->type(), variable->shape(), &xla_shape));
@@ -377,15 +384,15 @@ Status ReadVariableInputTensor(const Tensor& tensor, DataType type,
 Status XlaOpKernelContext::ReadVariableInput(int index, DataType type,
                                              TensorShape* shape,
                                              xla::XlaOp* value) {
-  return ReadVariableInputTensor(context_->input(index), type, context_, shape,
+  return ReadVariableInputTensor(context_->input(index), type, this, shape,
                                  value);
 }
 
 Status XlaOpKernelContext::ReadVariableInput(absl::string_view name,
                                              DataType type, TensorShape* shape,
                                              xla::XlaOp* value) {
-  return ReadVariableInputTensor(GetInputTensorByName(name), type, context_,
-                                 shape, value);
+  return ReadVariableInputTensor(GetInputTensorByName(name), type, this, shape,
+                                 value);
 }
 
 Status XlaOpKernelContext::GetVariableTypeAndShape(int index, DataType* type,
@@ -464,7 +471,7 @@ Status XlaOpKernelContext::GetResourceInput(int index, XlaResource** resource) {
 namespace {
 
 Status AssignVariableTensor(const Tensor& tensor, DataType type,
-                            const OpKernelContext* ctx, xla::XlaOp handle,
+                            const XlaOpKernelContext* ctx, xla::XlaOp handle,
                             xla::XlaBuilder* builder) {
   const XlaExpression* expression = CastExpressionFromTensor(tensor);
   XlaResource* variable = expression->resource();
@@ -481,9 +488,9 @@ Status AssignVariableTensor(const Tensor& tensor, DataType type,
 
   TF_RETURN_IF_ERROR(variable->SetTypeAndShape(type, shape));
 
-  XlaContext& xla_context = XlaContext::Get(ctx);
-  TF_ASSIGN_OR_RETURN(xla::Shape representation_shape,
-                      xla_context.RepresentationShape(shape, type));
+  TF_ASSIGN_OR_RETURN(
+      xla::Shape representation_shape,
+      ctx->compiler()->options().shape_representation_fn(shape, type));
   xla::Shape xla_shape;
   TF_RETURN_IF_ERROR(TensorShapeToXLAShape(type, shape, &xla_shape));
   if (!xla::ShapeUtil::Compatible(xla_shape, representation_shape)) {
@@ -498,19 +505,15 @@ Status AssignVariableTensor(const Tensor& tensor, DataType type,
 Status XlaOpKernelContext::AssignVariable(int input_index, DataType type,
                                           xla::XlaOp handle) {
   TF_RET_CHECK(handle.valid());
-  return AssignVariableTensor(context_->input(input_index), type, context_,
-                              handle, builder());
+  return AssignVariableTensor(context_->input(input_index), type, this, handle,
+                              builder());
 }
 
 Status XlaOpKernelContext::AssignVariable(absl::string_view name, DataType type,
                                           xla::XlaOp handle) {
   TF_RET_CHECK(handle.valid());
-  return AssignVariableTensor(GetInputTensorByName(name), type, context_,
-                              handle, builder());
-}
-
-XlaCompiler* XlaOpKernelContext::compiler() const {
-  return XlaContext::Get(context_).compiler();
+  return AssignVariableTensor(GetInputTensorByName(name), type, this, handle,
+                              builder());
 }
 
 void XlaOpKernelContext::CtxFailure(const Status& s) {
@@ -530,22 +533,22 @@ void XlaOpKernelContext::CtxFailureWithWarning(const char* file, int line,
 
 const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMax(
     const DataType type) {
-  return XlaContext::Get(context_).GetOrCreateMax(type);
+  return xla_context()->GetOrCreateMax(type);
 }
 
 const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMin(
     const DataType type) {
-  return XlaContext::Get(context_).GetOrCreateMin(type);
+  return xla_context()->GetOrCreateMin(type);
 }
 
 const xla::XlaComputation* XlaOpKernelContext::GetOrCreateAdd(
     const DataType type) {
-  return XlaContext::Get(context_).GetOrCreateAdd(type);
+  return xla_context()->GetOrCreateAdd(type);
 }
 
 const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMul(
     const DataType type) {
-  return XlaContext::Get(context_).GetOrCreateMul(type);
+  return xla_context()->GetOrCreateMul(type);
 }
 
 const Tensor& XlaOpKernelContext::GetInputTensorByName(absl::string_view name) {
diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.h b/tensorflow/compiler/tf2xla/xla_op_kernel.h
index c06efa2c47..1858844bc0 100644
--- a/tensorflow/compiler/tf2xla/xla_op_kernel.h
+++ b/tensorflow/compiler/tf2xla/xla_op_kernel.h
@@ -60,6 +60,8 @@ class XlaOpKernelContext {
  public:
   explicit XlaOpKernelContext(OpKernelContext* context);
 
+  XlaContext* xla_context() const;
+
   // Returns the XLA XlaBuilder containing the output of compilation.
   xla::XlaBuilder* builder() const;
 
diff --git a/tensorflow/compiler/tf2xla/xla_resource.cc b/tensorflow/compiler/tf2xla/xla_resource.cc
index a322eb9015..48a3c01272 100644
--- a/tensorflow/compiler/tf2xla/xla_resource.cc
+++ b/tensorflow/compiler/tf2xla/xla_resource.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <functional>
 #include <memory>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/sharding_util.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
@@ -39,9 +40,29 @@ namespace tensorflow {
   }
 }
 
+/*static*/ std::unique_ptr<XlaResource> XlaResource::CreateStack(
+    string name, DataType type, int64 max_size) {
+  return absl::make_unique<XlaResource>(
+      XlaResource::kStack, /*arg_num=*/-1, std::move(name), type, TensorShape(),
+      /*initial_value=*/xla::XlaOp(),
+      /*max_array_size=*/max_size,
+      /*tensor_array_gradients=*/std::set<string>{},
+      /*tensor_array_multiple_writes_aggregate=*/false);
+}
+
+/*static*/ std::unique_ptr<XlaResource> XlaResource::CreateTensorArray(
+    string name, DataType type, TensorShape shape, xla::XlaOp initial_value,
+    int64 max_array_size) {
+  return absl::make_unique<XlaResource>(
+      XlaResource::kTensorArray, /*arg_num=*/-1, std::move(name), type, shape,
+      initial_value, max_array_size,
+      /*tensor_array_gradients=*/std::set<string>{},
+      /*tensor_array_multiple_writes_aggregate=*/false);
+}
+
 XlaResource::XlaResource(Kind kind, int arg_num, string name, DataType type,
                          TensorShape shape, const xla::XlaOp& initial_value,
-                         int64 tensor_array_size,
+                         int64 max_array_size,
                          const std::set<string>& tensor_array_gradients,
                          bool tensor_array_multiple_writes_aggregate)
     : kind_(kind),
@@ -51,7 +72,7 @@ XlaResource::XlaResource(Kind kind, int arg_num, string name, DataType type,
       shape_(std::move(shape)),
       value_(initial_value),
       initial_value_(initial_value),
-      tensor_array_size_(tensor_array_size),
+      max_array_size_(max_array_size),
       tensor_array_multiple_writes_aggregate_(
           tensor_array_multiple_writes_aggregate) {
   CHECK(kind_ != kInvalid);
@@ -60,7 +81,7 @@ XlaResource::XlaResource(Kind kind, int arg_num, string name, DataType type,
     tensor_array_gradients_[gradient].reset(new XlaResource(
         /*kind=*/kTensorArray, /*arg_num=*/-1,
         /*name=*/absl::StrCat("TensorArrayGrad: ", name_), type_, shape_,
-        xla::XlaOp(), tensor_array_size_, /*tensor_array_gradients=*/{},
+        xla::XlaOp(), max_array_size_, /*tensor_array_gradients=*/{},
         /*tensor_array_multiple_writes_aggregate=*/true));
   }
 }
@@ -113,7 +134,7 @@ Status XlaResource::SetZeroValue(xla::XlaBuilder* builder) {
     }
     case kTensorArray: {
       TensorShape ta_shape;
-      ta_shape.AddDim(tensor_array_size_);
+      ta_shape.AddDim(max_array_size_);
       ta_shape.AppendShape(shape_);
       value_ = xla::Broadcast(XlaHelpers::Zero(builder, type_),
                               ta_shape.dim_sizes());
@@ -121,7 +142,7 @@ Status XlaResource::SetZeroValue(xla::XlaBuilder* builder) {
     }
     case kStack: {
       TensorShape ta_shape;
-      ta_shape.AddDim(tensor_array_size_);
+      ta_shape.AddDim(max_array_size_);
       ta_shape.AppendShape(shape_);
       value_ =
           xla::Tuple(builder, {xla::Broadcast(XlaHelpers::Zero(builder, type_),
@@ -146,14 +167,14 @@ Status XlaResource::GetOrCreateTensorArrayGradient(const string& source,
   std::unique_ptr<XlaResource>& gradient = tensor_array_gradients_[source];
   if (!gradient) {
     TensorShape ta_shape;
-    ta_shape.AddDim(tensor_array_size_);
+    ta_shape.AddDim(max_array_size_);
     ta_shape.AppendShape(shape_);
     xla::XlaOp gradient_value =
         xla::Broadcast(XlaHelpers::Zero(builder, type_), ta_shape.dim_sizes());
     gradient.reset(
         new XlaResource(/*kind=*/kTensorArray, /*arg_num=*/-1,
                         /*name=*/absl::StrCat("TensorArrayGrad: ", name_),
-                        type_, shape_, gradient_value, tensor_array_size_,
+                        type_, shape_, gradient_value, max_array_size_,
                         /*tensor_array_gradients=*/{},
                         /*tensor_array_multiple_writes_aggregate=*/true));
   }
diff --git a/tensorflow/compiler/tf2xla/xla_resource.h b/tensorflow/compiler/tf2xla/xla_resource.h
index 857b9a928b..736588bb8b 100644
--- a/tensorflow/compiler/tf2xla/xla_resource.h
+++ b/tensorflow/compiler/tf2xla/xla_resource.h
@@ -38,9 +38,18 @@ class XlaResource {
   };
   static absl::string_view KindToString(Kind kind);
 
+  // Creates a new Stack resource.
+  static std::unique_ptr<XlaResource> CreateStack(string name, DataType type,
+                                                  int64 max_size);
+
+  // Creates a new TensorArray resource.
+  static std::unique_ptr<XlaResource> CreateTensorArray(
+      string name, DataType type, TensorShape shape, xla::XlaOp initial_value,
+      int64 max_array_size);
+
   XlaResource(Kind kind, int arg_num, string name, DataType type,
               TensorShape shape, const xla::XlaOp& initial_value,
-              int64 tensor_array_size,
+              int64 max_array_size,
               const std::set<string>& tensor_array_gradients,
               bool tensor_array_multiple_writes_aggregate);
 
@@ -119,12 +128,12 @@ class XlaResource {
   // TODO(phawkins): refactor this code to use subclasses, rather than putting
   // kind-specific fields in XlaResource.
 
-  // 'tensor_array_size' stores the expected size of the TensorArray or Stack.
+  // 'max_array_size' stores the expected size of the TensorArray or Stack.
   // We need to store this since sometimes TensorArrays must be initialized
   // lazily since we do not know the element shape at construction time.
   // Used by both TensorArrays and Stacks.
-  int64 tensor_array_size() const { return tensor_array_size_; }
-  void set_tensor_array_size(int64 size) { tensor_array_size_ = size; }
+  int64 max_array_size() const { return max_array_size_; }
+  void set_max_array_size(int64 size) { max_array_size_ = size; }
 
   bool tensor_array_multiple_writes_aggregate() const {
     return tensor_array_multiple_writes_aggregate_;
@@ -151,7 +160,7 @@ class XlaResource {
   xla::XlaOp value_;
   xla::XlaOp initial_value_;
 
-  int64 tensor_array_size_ = -1;
+  int64 max_array_size_ = -1;
   bool tensor_array_multiple_writes_aggregate_ = false;
 
   std::map<string, std::unique_ptr<XlaResource>> tensor_array_gradients_;
-- 
GitLab


From f382bd6279141fe169e5aa8ab0867f18840ee037 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 27 Nov 2018 08:55:01 -0800
Subject: [PATCH 0861/1554] Remove train.VocabInfo from TF 2.0 API.

PiperOrigin-RevId: 222998560
---
 .../python/training/warm_starting_util.py     |  2 +-
 .../v2/tensorflow.train.-vocab-info.pbtxt     | 43 -------------------
 .../api/golden/v2/tensorflow.train.pbtxt      |  4 --
 tensorflow/tools/compatibility/renames_v2.py  |  1 +
 4 files changed, 2 insertions(+), 48 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt

diff --git a/tensorflow/python/training/warm_starting_util.py b/tensorflow/python/training/warm_starting_util.py
index 3649d313ae..19dc04e8fb 100644
--- a/tensorflow/python/training/warm_starting_util.py
+++ b/tensorflow/python/training/warm_starting_util.py
@@ -32,7 +32,7 @@ from tensorflow.python.training import saver
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.VocabInfo")
+@tf_export(v1=["train.VocabInfo"])
 class VocabInfo(
     collections.namedtuple("VocabInfo", [
         "new_vocab",
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt
deleted file mode 100644
index 39b946b82f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-vocab-info.pbtxt
+++ /dev/null
@@ -1,43 +0,0 @@
-path: "tensorflow.train.VocabInfo"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.warm_starting_util.VocabInfo\'>"
-  is_instance: "<class \'tensorflow.python.training.warm_starting_util.VocabInfo\'>"
-  is_instance: "<type \'tuple\'>"
-  member {
-    name: "axis"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "backup_initializer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "new_vocab"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "new_vocab_size"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "num_oov_buckets"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "old_vocab"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "old_vocab_size"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "count"
-  }
-  member_method {
-    name: "index"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 91e1e0582e..d47bc09e66 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -208,10 +208,6 @@ tf_module {
     name: "Supervisor"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "VocabInfo"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "WorkerSessionCreator"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index fc4b19fb1c..c5efc04d04 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -588,6 +588,7 @@ renames = {
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
     'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
+    'tf.train.VocabInfo': 'tf.compat.v1.train.VocabInfo',
     'tf.train.add_queue_runner': 'tf.compat.v1.train.add_queue_runner',
     'tf.train.assert_global_step': 'tf.compat.v1.train.assert_global_step',
     'tf.train.basic_train_loop': 'tf.compat.v1.train.basic_train_loop',
-- 
GitLab


From 7753011748cfd804b0ea48e66e36f3f3ed3805be Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 27 Nov 2018 09:44:57 -0800
Subject: [PATCH 0862/1554] Temporarily disable unit tests for half precision -
 investigating possible TRT bug

---
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 471382894c..c6cd765887 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2040,7 +2040,9 @@ TEST_F(OpConverterTest, ConvertSquare) {
   // OK. Note that kINT32 is not supported by IElementWiseLayer, so we don't
   // test DT_INT32 type here.
   TestConvertSquare<DT_FLOAT>(this);
-  TestConvertSquare<DT_HALF>(this);
+  // TODO(tmorris): Looks like there may be a bug with this layer for FP16
+  // inputs. Disabling for now.
+  // TestConvertSquare<DT_HALF>(this);
 }
 
 }  // namespace convert
-- 
GitLab


From 249bd81c3c3aa8dc89d8fcef8927df49fc8c42ec Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 27 Nov 2018 09:52:46 -0800
Subject: [PATCH 0863/1554] Automated rollback of commit
 5bb5d6db8ac8f48ba2636ae18844cb584b837904

PiperOrigin-RevId: 223007492
---
 tensorflow/python/ops/resource_variable_ops.py | 2 +-
 tensorflow/python/ops/variables.py             | 9 ++-------
 third_party/libxsmm.BUILD                      | 2 +-
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index f9faa3e945..5c74dffb05 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1433,7 +1433,7 @@ def _from_proto_fn(v, import_scope=None):
   """Creates Variable or ResourceVariable from VariableDef as needed."""
   if v.is_resource:
     return ResourceVariable.from_proto(v, import_scope=import_scope)
-  return variables.RefVariable.from_proto(v, import_scope=import_scope)
+  return variables.Variable.from_proto(v, import_scope=import_scope)
 
 
 ops.register_proto_function(
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index a001374e9a..f72b19bcdd 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -968,7 +968,8 @@ class Variable(six.with_metaclass(VariableMetaclass,
   @staticmethod
   def from_proto(variable_def, import_scope=None):
     """Returns a `Variable` object created from `variable_def`."""
-    raise NotImplementedError
+    return RefVariable(variable_def=variable_def,
+                       import_scope=import_scope)
 
   class SaveSliceInfo(object):
     """Information on how to save this Variable as a slice.
@@ -2343,12 +2344,6 @@ class RefVariable(VariableV1):
     else:
       return None
 
-  @staticmethod
-  def from_proto(variable_def, import_scope=None):
-    """Returns a `Variable` object created from `variable_def`."""
-    return RefVariable(variable_def=variable_def,
-                       import_scope=import_scope)
-
   def __iadd__(self, other):
     logging.log_first_n(
         logging.WARN,
diff --git a/third_party/libxsmm.BUILD b/third_party/libxsmm.BUILD
index ee49d281ab..dc7dcc9517 100644
--- a/third_party/libxsmm.BUILD
+++ b/third_party/libxsmm.BUILD
@@ -38,8 +38,8 @@ genrule(
         ":libxsmm_interface",
     ],
     visibility = [
-        "//third_party/eigen3:__pkg__",
         "//tensorflow/core/kernels:__pkg__",
+        "//third_party/eigen3:__pkg__",
     ],
 )
 
-- 
GitLab


From bfd9b3254a0e1a8782e63a36fe03ab748b7cbafe Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 27 Nov 2018 09:58:23 -0800
Subject: [PATCH 0864/1554] Fix clang-format issues

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 12 ++++++------
 .../contrib/tensorrt/convert/convert_nodes_test.cc   |  5 ++---
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index b1fa6a51d4..4238ffea79 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1985,7 +1985,8 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
   }
   if (!inputs.at(0).is_tensor()) {
     return tensorflow::errors::Unimplemented(
-        node_def.op(), " is only implemented for tensors, at ", node_def.name());
+        node_def.op(), " is only implemented for tensors, at ",
+        node_def.name());
   }
   static const std::unordered_map<string, nvinfer1::ActivationType> ops{
       {"Relu", nvinfer1::ActivationType::kRELU},
@@ -1994,9 +1995,9 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
   };
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end()) {
-    return tensorflow::errors::Unimplemented(
-        "Activation op: ", node_def.op(), " not supported at: ",
-        node_def.name());
+    return tensorflow::errors::Unimplemented("Activation op: ", node_def.op(),
+                                             " not supported at: ",
+                                             node_def.name());
   }
   if (params->validation_only) return tensorflow::Status::OK();
 
@@ -2004,8 +2005,7 @@ tensorflow::Status ConvertActivation(OpConverterParams* params) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   nvinfer1::IActivationLayer* layer =
       params->converter->network()->addActivation(
-          *const_cast<nvinfer1::ITensor*>(tensor),
-          op_pair->second);
+          *const_cast<nvinfer1::ITensor*>(tensor), op_pair->second);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   // Set quantization range for output of Sigmoid, Tanh.
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 45e901bf5b..724dc0e224 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1968,9 +1968,8 @@ TEST_F(OpConverterTest, ConvertActivation) {
   {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_act", "Relu", {});
-    RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Relu expects one input, at my_act");
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+                               "Relu expects one input, at my_act");
   }
   {
     // Input is weights, should fail.
-- 
GitLab


From edf88fcda80a19be56d396075a8fa2b4e822b9a4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 09:59:18 -0800
Subject: [PATCH 0865/1554] Remove ad-hoc vmodule test from tests.

PiperOrigin-RevId: 223008525
---
 tensorflow/core/BUILD | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 437c368356..6821ac7eb7 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3429,7 +3429,6 @@ tf_cc_tests(
         "platform/stacktrace_handler_test.cc",
         "platform/subprocess_test.cc",
         "platform/vmodule_benchmark_test.cc",
-        "platform/vmodule_test.cc",
     ],
     deps = [
         ":lib",
@@ -3443,6 +3442,20 @@ tf_cc_tests(
     ],
 )
 
+tf_cc_test(
+    name = "vmodule_test",
+    srcs = ["platform/vmodule_test.cc"],
+    tags = ["optonly"],
+    deps = [
+        ":lib",
+        ":lib_internal",
+        ":lib_test_internal",
+        ":protos_all_cc",
+        ":test",
+        "//third_party/eigen3",
+    ],
+)
+
 tf_cc_test(
     name = "lib_random_random_distributions_test",
     srcs = ["lib/random/random_distributions_test.cc"],
-- 
GitLab


From 3c4b9f19a9d85d899dd312e8c945b47bb2341e5a Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 27 Nov 2018 10:07:07 -0800
Subject: [PATCH 0866/1554] Fix compilation issue

---
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 724dc0e224..0096bf99cd 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1998,7 +1998,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
       auto act = ops::Tanh(s.WithOpName("my_act"), input);
       return act.operation.node()->def();
     }
-    ASSERT_TRUE(false);
+    EXPECT_TRUE(false);
     return NodeDef();
   };
   // Get expected output for activation layer.
@@ -2010,7 +2010,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
     } else if (op_name == "Tanh") {
       return std::tanh(input);
     }
-    ASSERT_TRUE(false);
+    EXPECT_TRUE(false);
     return 0;
   };
 
@@ -2027,7 +2027,7 @@ TEST_F(OpConverterTest, ConvertActivation) {
 
     const std::vector<float> input_data = {-100, -2, -1, 0, 1, 100};
     std::vector<float> output_data(6);
-    BuildAndRun("input", input_data, "my_act", &output_data);
+    BuildAndRun<float>({{"input", input_data}}, "my_act", &output_data);
     for (int i = 0; i < input_data.size(); i++) {
       const float expected_output = get_act_output(op_name, input_data[i]);
       EXPECT_FLOAT_EQ(output_data[i], expected_output);
-- 
GitLab


From b17d53c0cd07bbae46a55dc9c12d3f48a1c19604 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 27 Nov 2018 10:05:22 -0800
Subject: [PATCH 0867/1554] Replace a few calls of Session `run` with
 `evaluate`

In order to support tests running in eager mode we need to avoid
unnecessary use of Sessions in tests. This moves to remove some
of the uses of the `run` function in favor of `evaluate`.

PiperOrigin-RevId: 223009795
---
 .../compiler/tests/categorical_op_test.py     |  20 +-
 tensorflow/compiler/tests/concat_ops_test.py  |  20 +-
 tensorflow/compiler/tests/dense_layer_test.py |   6 +-
 tensorflow/compiler/tests/eager_test.py       |   4 +-
 tensorflow/compiler/tests/function_test.py    |  12 +-
 tensorflow/compiler/tests/listdiff_op_test.py |   4 +-
 tensorflow/compiler/tests/lstm_test.py        |   8 +-
 tensorflow/compiler/tests/placeholder_test.py |   2 +-
 tensorflow/compiler/tests/random_ops_test.py  |  16 +-
 .../tests/stateless_random_ops_test.py        |   2 +-
 .../compiler/tests/tensor_array_ops_test.py   |   2 +-
 .../compiler/tests/variable_ops_test.py       |  42 +-
 tensorflow/compiler/tests/xla_device_test.py  |   2 +-
 .../autograph/integration_tests/keras_test.py |   4 +-
 .../integration_tests/list_literals_test.py   |   2 +-
 .../speech_commands/input_data_test.py        |   2 +-
 .../speech_commands/label_wav_test.py         |   2 +-
 .../speech_commands/wav_to_features_test.py   |   2 +-
 .../lstm/unidirectional_sequence_lstm_test.py |   2 +-
 .../autograph/converters/asserts_test.py      |   2 +-
 .../autograph/converters/call_trees_test.py   |   4 +-
 .../python/autograph/converters/lists_test.py |   8 +-
 .../converters/side_effect_guards_test.py     |  30 +-
 .../autograph/converters/slices_test.py       |   2 +-
 .../python/autograph/core/errors_test.py      |   6 +-
 tensorflow/python/autograph/impl/api_test.py  |  46 +-
 .../autograph/lang/special_functions_test.py  |  12 +-
 .../autograph/operators/control_flow_test.py  |  14 +-
 .../operators/data_structures_test.py         |  20 +-
 .../autograph/operators/exceptions_test.py    |   6 +-
 .../autograph/operators/logical_test.py       |  14 +-
 .../autograph/operators/py_builtins_test.py   |  28 +-
 .../python/autograph/operators/slices_test.py |   8 +-
 .../python/autograph/utils/misc_test.py       |   4 +-
 .../python/autograph/utils/py_func_test.py    |  18 +-
 .../autograph/utils/tensor_list_test.py       |   8 +-
 .../client/session_clusterspec_prop_test.py   |   6 +-
 .../python/client/session_partial_run_test.py |   2 +-
 tensorflow/python/client/timeline_test.py     |   4 +-
 tensorflow/python/client/virtual_gpu_test.py  |   2 +-
 .../bucket_by_sequence_length_test.py         |  14 +-
 .../kernel_tests/copy_to_device_test.py       | 133 ++--
 .../experimental/kernel_tests/counter_test.py |  12 +-
 .../dense_to_sparse_batch_test.py             |  16 +-
 .../directed_interleave_dataset_test.py       |  14 +-
 .../kernel_tests/enumerate_dataset_test.py    |   8 +-
 .../kernel_tests/filter_dataset_op_test.py    |   4 +-
 .../function_buffering_resource_test.py       |  70 +-
 .../kernel_tests/group_by_reducer_test.py     |  10 +-
 .../kernel_tests/group_by_window_test.py      |  56 +-
 .../kernel_tests/ignore_errors_test.py        |  24 +-
 .../kernel_tests/indexed_dataset_ops_test.py  |  11 +-
 .../make_batched_features_dataset_test.py     |   6 +-
 .../kernel_tests/make_csv_dataset_test.py     |  12 +-
 .../make_tf_record_dataset_test.py            |  12 +-
 .../kernel_tests/map_and_batch_test.py        |  68 +-
 .../kernel_tests/map_defun_op_test.py         |   6 +-
 .../optimization/model_dataset_test.py        |   4 +-
 .../optimization/optimize_dataset_test.py     |   4 +-
 .../kernel_tests/override_threadpool_test.py  |   4 +-
 .../kernel_tests/parallel_interleave_test.py  |  70 +-
 .../kernel_tests/prefetch_to_device_test.py   |  40 +-
 .../experimental/kernel_tests/scan_test.py    |  12 +-
 .../range_dataset_serialization_test.py       |  30 +-
 .../serialization_integration_test.py         |   6 +-
 .../shuffle_dataset_serialization_test.py     |   4 +-
 .../kernel_tests/shuffle_and_repeat_test.py   |   6 +-
 .../experimental/kernel_tests/sleep_test.py   |   6 +-
 .../kernel_tests/sql_dataset_test.py          | 169 ++---
 .../kernel_tests/stats_dataset_ops_test.py    | 137 ++--
 .../experimental/kernel_tests/unbatch_test.py |  34 +-
 .../experimental/kernel_tests/unique_test.py  |   6 +-
 .../multi_device_iterator_test.py             | 115 ++--
 tensorflow/python/data/util/convert_test.py   | 146 ++--
 .../python/debug/cli/analyzer_cli_test.py     |   2 +-
 .../lib/debug_graph_reconstruction_test.py    |  14 +-
 .../debug/lib/session_debug_multi_gpu_test.py |   2 +-
 .../python/debug/lib/source_utils_test.py     |   4 +-
 .../python/distribute/input_ops_test.py       |  19 +-
 tensorflow/python/eager/def_function_test.py  |   8 +-
 .../python/eager/function_gradients_test.py   |   2 +-
 tensorflow/python/eager/function_test.py      |   4 +-
 .../feature_column/feature_column_test.py     |   8 +-
 .../feature_column/feature_column_v2_test.py  |  11 +-
 .../python/framework/file_system_test.py      |   2 +-
 tensorflow/python/framework/function_test.py  |  62 +-
 .../python/framework/graph_util_test.py       |  12 +-
 tensorflow/python/framework/importer_test.py  |  14 +-
 .../python/framework/meta_graph_test.py       |  18 +-
 tensorflow/python/framework/ops_test.py       |  16 +-
 .../python/framework/smart_cond_test.py       |   8 +-
 .../python/framework/sparse_tensor_test.py    |   2 +-
 tensorflow/python/framework/subscribe_test.py |  28 +-
 .../python/framework/tensor_util_test.py      |   2 +-
 .../python/grappler/constant_folding_test.py  |   2 +-
 .../python/grappler/layout_optimizer_test.py  |  48 +-
 .../python/grappler/memory_optimizer_test.py  |  12 +-
 tensorflow/python/keras/backend_test.py       |   2 +-
 .../python/keras/layers/recurrent_test.py     |   4 +-
 tensorflow/python/keras/metrics_test.py       |  10 +-
 .../python/keras/optimizer_v2/ftrl_test.py    |  32 +-
 .../python/kernel_tests/accumulate_n_test.py  |   2 +-
 .../python/kernel_tests/aggregate_ops_test.py |   2 +-
 .../python/kernel_tests/array_ops_test.py     |  14 +-
 .../python/kernel_tests/attention_ops_test.py |   2 +-
 .../python/kernel_tests/barrier_ops_test.py   |   6 +-
 .../python/kernel_tests/base64_ops_test.py    |   2 +-
 .../python/kernel_tests/basic_gpu_test.py     |   8 +-
 .../boosted_trees/quantile_ops_test.py        |  14 +-
 .../boosted_trees/stats_ops_test.py           |  58 +-
 .../python/kernel_tests/bucketize_op_test.py  |   8 +-
 .../candidate_sampler_ops_test.py             |   2 +-
 .../python/kernel_tests/cast_op_test.py       |   4 +-
 .../python/kernel_tests/cholesky_op_test.py   |   4 +-
 .../python/kernel_tests/concat_op_test.py     | 126 ++--
 .../conditional_accumulator_test.py           |  14 +-
 .../kernel_tests/control_flow_ops_py_test.py  |  92 +--
 .../python/kernel_tests/conv_ops_3d_test.py   |  10 +-
 .../python/kernel_tests/conv_ops_test.py      |  16 +-
 .../python/kernel_tests/ctc_loss_op_test.py   |  43 +-
 .../kernel_tests/cwise_ops_binary_test.py     |  14 +-
 .../python/kernel_tests/cwise_ops_test.py     |  14 +-
 .../kernel_tests/decode_image_op_test.py      |   8 +-
 .../kernel_tests/decode_jpeg_op_test.py       |   6 +-
 .../dense_update_ops_no_tsan_test.py          |   8 +-
 .../kernel_tests/depthtospace_op_test.py      |   2 +-
 .../kernel_tests/depthwise_conv_op_test.py    |   6 +-
 .../kernel_tests/determinant_op_test.py       |   2 +-
 .../distributions/categorical_test.py         |   4 +-
 .../distributions/special_math_test.py        |   4 +-
 .../kernel_tests/distributions/util_test.py   |   4 +-
 .../kernel_tests/division_future_test.py      |   2 +-
 .../python/kernel_tests/division_past_test.py |   2 +-
 .../kernel_tests/draw_bounding_box_op_test.py |   2 +-
 .../kernel_tests/dynamic_partition_op_test.py |  32 +-
 .../python/kernel_tests/embedding_ops_test.py |   4 +-
 .../python/kernel_tests/fifo_queue_test.py    | 134 ++--
 .../fractional_avg_pool_op_test.py            |   4 +-
 .../fractional_max_pool_op_test.py            |   4 +-
 .../kernel_tests/functional_ops_test.py       |  52 +-
 .../kernel_tests/gradient_correctness_test.py |   8 +-
 .../python/kernel_tests/init_ops_test.py      |  16 +-
 .../python/kernel_tests/inplace_ops_test.py   |   2 +-
 tensorflow/python/kernel_tests/io_ops_test.py |   4 +-
 .../linalg/linear_operator_circulant_test.py  |   4 +-
 .../linalg/linear_operator_diag_test.py       |   4 +-
 .../linalg/linear_operator_identity_test.py   |  12 +-
 .../linalg/linear_operator_util_test.py       |   4 +-
 .../python/kernel_tests/list_ops_test.py      |   2 +-
 .../python/kernel_tests/listdiff_op_test.py   |   2 +-
 .../python/kernel_tests/lookup_ops_test.py    |  10 +-
 tensorflow/python/kernel_tests/losses_test.py |   4 +-
 .../matrix_exponential_op_test.py             |   2 +-
 .../kernel_tests/matrix_inverse_op_test.py    |   2 +-
 .../kernel_tests/matrix_logarithm_op_test.py  |   2 +-
 .../kernel_tests/matrix_solve_op_test.py      |   2 +-
 .../matrix_square_root_op_test.py             |   2 +-
 .../python/kernel_tests/metrics_test.py       | 457 ++++++-------
 .../neon_depthwise_conv_op_test.py            |   6 +-
 .../python/kernel_tests/norm_op_test.py       |   2 +-
 .../kernel_tests/nth_element_op_test.py       |   2 +-
 .../kernel_tests/padding_fifo_queue_test.py   | 124 ++--
 .../parse_single_example_op_test.py           |   2 +-
 .../python/kernel_tests/parsing_ops_test.py   |   6 +-
 .../kernel_tests/pooling_ops_3d_test.py       |   2 +-
 .../python/kernel_tests/pooling_ops_test.py   |   2 +-
 .../kernel_tests/priority_queue_test.py       |  22 +-
 .../python/kernel_tests/py_func_test.py       |  14 +-
 tensorflow/python/kernel_tests/qr_op_test.py  |   4 +-
 .../random/multinomial_op_big_test.py         |   6 +-
 .../kernel_tests/random/random_gamma_test.py  |   2 +-
 .../kernel_tests/random/random_ops_test.py    |  12 +-
 .../random/random_poisson_test.py             |   2 +-
 .../random/random_shuffle_queue_test.py       | 118 ++--
 .../random/stateless_random_ops_test.py       |   2 +-
 .../python/kernel_tests/reader_ops_test.py    | 628 +++++++++---------
 .../python/kernel_tests/record_input_test.py  |  16 +-
 .../kernel_tests/reduce_benchmark_test.py     |   4 +-
 .../python/kernel_tests/reduction_ops_test.py |  22 +-
 .../python/kernel_tests/relu_op_test.py       |   2 +-
 .../resource_variable_ops_test.py             |   8 +-
 .../kernel_tests/scatter_nd_ops_test.py       |  26 +-
 .../kernel_tests/self_adjoint_eig_op_test.py  |   4 +-
 .../python/kernel_tests/session_ops_test.py   |  46 +-
 tensorflow/python/kernel_tests/sets_test.py   |   8 +-
 .../python/kernel_tests/shape_ops_test.py     |   4 +-
 .../signal/reconstruction_ops_test.py         |   8 +-
 .../kernel_tests/signal/spectral_ops_test.py  |   8 +-
 .../python/kernel_tests/slice_op_test.py      |   6 +-
 .../kernel_tests/spacetodepth_op_test.py      |   2 +-
 .../python/kernel_tests/sparse_add_op_test.py |  31 +-
 .../kernel_tests/sparse_concat_op_test.py     |  16 +-
 .../sparse_conditional_accumulator_test.py    |  36 +-
 .../kernel_tests/sparse_cross_op_test.py      |  34 +-
 .../python/kernel_tests/sparse_ops_test.py    |  34 +-
 .../kernel_tests/sparse_reorder_op_test.py    |   4 +-
 .../kernel_tests/sparse_reshape_op_test.py    |   4 +-
 .../sparse_serialization_ops_test.py          |   2 +-
 .../sparse_tensors_map_ops_test.py            |  15 +-
 .../kernel_tests/sparse_xent_op_test.py       |  10 +-
 .../python/kernel_tests/stack_ops_test.py     |  10 +-
 .../kernel_tests/string_length_op_test.py     |   6 +-
 .../kernel_tests/string_split_op_test.py      |  28 +-
 .../kernel_tests/string_strip_op_test.py      |   6 +-
 .../kernel_tests/summary_v1_audio_op_test.py  |   2 +-
 .../kernel_tests/summary_v1_image_op_test.py  |   4 +-
 .../kernel_tests/summary_v1_ops_test.py       |   6 +-
 .../kernel_tests/summary_v1_tensor_op_test.py |  14 +-
 tensorflow/python/kernel_tests/svd_op_test.py |   6 +-
 .../python/kernel_tests/template_test.py      |   8 +-
 .../kernel_tests/tensor_array_ops_test.py     |   8 +-
 .../python/kernel_tests/topk_op_test.py       |   2 +-
 .../kernel_tests/unicode_transcode_op_test.py |  56 +-
 .../python/kernel_tests/unique_op_test.py     |  24 +-
 .../python/kernel_tests/variable_ops_test.py  |   2 +-
 .../kernel_tests/variable_scope_test.py       |  34 +-
 .../python/kernel_tests/variables_test.py     |  18 +-
 .../python/kernel_tests/while_v2_test.py      |  56 +-
 .../python/kernel_tests/xent_op_test.py       |  10 +-
 .../python/layers/convolutional_test.py       |  16 +-
 tensorflow/python/layers/core_test.py         |   2 +-
 .../python/layers/normalization_test.py       | 139 ++--
 tensorflow/python/ops/bitwise_ops_test.py     |   5 +-
 tensorflow/python/ops/clip_ops_test.py        |   4 +-
 .../python/ops/control_flow_ops_test.py       |  16 +-
 tensorflow/python/ops/gradients_test.py       |  28 +-
 tensorflow/python/ops/image_grad_test.py      |   8 +-
 tensorflow/python/ops/image_ops_test.py       |  93 +--
 tensorflow/python/ops/init_ops_test.py        |   4 +-
 tensorflow/python/ops/math_ops_test.py        |   4 +-
 tensorflow/python/ops/nccl_ops_test.py        |   2 +-
 tensorflow/python/ops/nn_batchnorm_test.py    |  13 +-
 .../python/ops/nn_fused_batchnorm_test.py     |   8 +-
 .../ops/parallel_for/control_flow_ops_test.py |   6 +-
 .../python/ops/parallel_for/gradients_test.py |  10 +-
 .../python/ops/quantized_conv_ops_test.py     |   2 +-
 tensorflow/python/ops/quantized_ops_test.py   |   4 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |   2 +-
 .../ops/ragged/ragged_segment_op_test.py      |   8 +-
 .../ragged_tensor_bounding_shape_op_test.py   |  30 +-
 .../python/ops/ragged/ragged_tensor_test.py   | 110 +--
 .../python/profiler/model_analyzer_test.py    |  30 +-
 .../python/profiler/profile_context_test.py   |  20 +-
 tensorflow/python/saved_model/loader_test.py  |   8 +-
 .../python/saved_model/saved_model_test.py    |  42 +-
 .../python/saved_model/simple_save_test.py    |   2 +-
 tensorflow/python/tools/strip_unused_test.py  |   4 +-
 tensorflow/python/training/adagrad_da_test.py |  16 +-
 .../training/basic_session_run_hooks_test.py  |  32 +-
 .../python/training/checkpoint_ops_test.py    |   2 +-
 tensorflow/python/training/ftrl_test.py       |  36 +-
 tensorflow/python/training/input_test.py      | 108 +--
 .../training/learning_rate_decay_test.py      |  33 +-
 .../training/learning_rate_decay_v2_test.py   |  33 +-
 .../python/training/monitored_session_test.py |  16 +-
 .../python/training/moving_averages_test.py   |  14 +-
 .../python/training/proximal_adagrad_test.py  |  20 +-
 .../proximal_gradient_descent_test.py         |  16 +-
 .../python/training/quantize_training_test.py |   4 +-
 tensorflow/python/training/saver_test.py      |  78 +--
 ...lib_same_variables_clear_container_test.py |   8 +-
 .../training/server_lib_sparse_job_test.py    |   2 +-
 tensorflow/python/training/supervisor_test.py |  10 +-
 .../training/warm_starting_util_test.py       |  80 +--
 264 files changed, 3055 insertions(+), 2968 deletions(-)

diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py
index f4918e50dc..5d5e486f61 100644
--- a/tensorflow/compiler/tests/categorical_op_test.py
+++ b/tensorflow/compiler/tests/categorical_op_test.py
@@ -57,11 +57,11 @@ class CategoricalTest(xla_test.XLATestCase):
     Returns:
       Frequencies from sampled classes; shape [batch_size, num_classes].
     """
-    with self.cached_session() as sess, self.test_scope():
+    with self.cached_session(), self.test_scope():
       random_seed.set_random_seed(1618)
       op = random_ops.multinomial(logits, num_samples,
                                   output_dtype=dtypes.int32)
-      d = sess.run(op)
+      d = self.evaluate(op)
 
     batch_size, num_classes = logits.shape
     freqs_mat = []
@@ -80,15 +80,15 @@ class CategoricalTest(xla_test.XLATestCase):
 
   def _testRngIsNotConstant(self, rng, dtype, output_dtype):
     # Tests that 'rng' does not always return the same value.
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         x = rng(dtype, output_dtype)
 
       # The random-number generator, if working correctly, should produce the
       # same output multiple times with low probability.
-      y = sess.run(x)
-      z = sess.run(x)
-      w = sess.run(x)
+      y = self.evaluate(x)
+      z = self.evaluate(x)
+      w = self.evaluate(x)
 
       # We use exact equality here. If the random-number generator is producing
       # deterministic output, all three outputs will be bitwise identical.
@@ -108,12 +108,12 @@ class CategoricalTest(xla_test.XLATestCase):
   def testCategoricalIsInRange(self):
     for dtype in self.float_types:
       for output_dtype in self.output_dtypes():
-        with self.cached_session() as sess:
+        with self.cached_session():
           with self.test_scope():
             x = random_ops.multinomial(
                 array_ops.ones(shape=[1, 20], dtype=dtype), 1000,
                 output_dtype=output_dtype)
-          y = sess.run(x)
+          y = self.evaluate(x)
           self.assertTrue((y >= 0).sum() == 1000)
           self.assertTrue((y < 20).sum() == 1000)
 
@@ -170,11 +170,11 @@ class CategoricalTest(xla_test.XLATestCase):
             self.assertEqual(s0 == s1, np.all(v0 == v1))
 
   def testEmpty(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         x = random_ops.multinomial(
             array_ops.zeros([42, 40]), 0, output_dtype=dtypes.int32)
-        y = sess.run(x)
+        y = self.evaluate(x)
         self.assertEqual(y.shape, (42, 0))
 
   def testEmptyStateless(self):
diff --git a/tensorflow/compiler/tests/concat_ops_test.py b/tensorflow/compiler/tests/concat_ops_test.py
index 30fbe6f701..2187f57960 100644
--- a/tensorflow/compiler/tests/concat_ops_test.py
+++ b/tensorflow/compiler/tests/concat_ops_test.py
@@ -254,7 +254,7 @@ class ConcatTest(xla_test.XLATestCase):
   def DISABLED_testZeroSize(self):
     # Verify that concat doesn't crash and burn for zero size inputs
     np.random.seed(7)
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         for shape0 in (), (2,):
           axis = len(shape0)
@@ -270,7 +270,7 @@ class ConcatTest(xla_test.XLATestCase):
                 self.assertAllEqual(c.eval(), correct)
                 # Check gradients
                 dc = np.random.randn(*c.get_shape().as_list())
-                dxs = sess.run(gradients_impl.gradients(c, xs, dc))
+                dxs = self.evaluate(gradients_impl.gradients(c, xs, dc))
                 self.assertAllEqual(dc, np.concatenate(dxs, axis=axis))
 
   def testConcatTuple(self):
@@ -330,47 +330,47 @@ class ConcatTest(xla_test.XLATestCase):
 class ConcatOffsetTest(xla_test.XLATestCase):
 
   def testBasic(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         cdim = constant_op.constant(1, dtypes.int32)
         s0 = constant_op.constant([2, 3, 5], dtypes.int32)
         s1 = constant_op.constant([2, 7, 5], dtypes.int32)
         s2 = constant_op.constant([2, 20, 5], dtypes.int32)
         off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-        ans = sess.run(off)
+        ans = self.evaluate(off)
         self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
 
 class PackTest(xla_test.XLATestCase):
 
   def testBasic(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         s0 = constant_op.constant([2, 3, 5], dtypes.int32)
         s1 = constant_op.constant([2, 7, 5], dtypes.int32)
         s2 = constant_op.constant([2, 20, 5], dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = sess.run(packed)
+        ans = self.evaluate(packed)
         self.assertAllEqual(ans, [[2, 3, 5], [2, 7, 5], [2, 20, 5]])
 
   def testScalars(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         s0 = constant_op.constant(2, dtypes.int32)
         s1 = constant_op.constant(3, dtypes.int32)
         s2 = constant_op.constant(5, dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = sess.run(packed)
+        ans = self.evaluate(packed)
         self.assertAllEqual(ans, [2, 3, 5])
 
   def testEmpty(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       with self.test_scope():
         s0 = constant_op.constant([[]], dtypes.int32)
         s1 = constant_op.constant([[]], dtypes.int32)
         s2 = constant_op.constant([[]], dtypes.int32)
         packed = array_ops.stack([s0, s1, s2])
-        ans = sess.run(packed)
+        ans = self.evaluate(packed)
         self.assertAllEqual(ans, [[[]], [[]], [[]]])
 
 
diff --git a/tensorflow/compiler/tests/dense_layer_test.py b/tensorflow/compiler/tests/dense_layer_test.py
index 23c94cf245..bf5ea7b1fb 100644
--- a/tensorflow/compiler/tests/dense_layer_test.py
+++ b/tensorflow/compiler/tests/dense_layer_test.py
@@ -72,7 +72,7 @@ class DenseLayerTest(test.TestCase):
       x = array_ops.placeholder(shape=[None, None, 3], dtype=np.float32)
       y = layers.dense(x, 3)
 
-      sess.run(variables.initialize_all_variables())
+      self.evaluate(variables.initialize_all_variables())
       run_metadata = config_pb2.RunMetadata()
       test_utils.RunWithWarmup(
           sess,
@@ -97,7 +97,7 @@ class DenseLayerTest(test.TestCase):
       with jit_scope():
         y = layers.dense(x, 3)
 
-      sess.run(variables.initialize_all_variables())
+      self.evaluate(variables.initialize_all_variables())
       run_metadata = config_pb2.RunMetadata()
       test_utils.RunWithWarmup(
           sess,
@@ -126,7 +126,7 @@ class DenseLayerTest(test.TestCase):
       with jit_scope():
         y = layers.dense(x, 3)
 
-      sess.run(variables.initialize_all_variables())
+      self.evaluate(variables.initialize_all_variables())
       run_metadata = config_pb2.RunMetadata()
       test_utils.RunWithWarmup(
           sess,
diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index 63cee550fd..2af32b537b 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -101,12 +101,12 @@ class EagerTest(xla_test.XLATestCase):
       self.assertAllEqual(15, product)
 
     # Run some ops graphly
-    with context.graph_mode(), self.cached_session() as sess:
+    with context.graph_mode(), self.cached_session():
       with self.test_scope():
         three = constant_op.constant(3)
         five = constant_op.constant(5)
         product = three * five
-        self.assertAllEqual(15, sess.run(product))
+        self.assertAllEqual(15, self.evaluate(product))
 
   def testDegenerateSlices(self):
     with self.test_scope():
diff --git a/tensorflow/compiler/tests/function_test.py b/tensorflow/compiler/tests/function_test.py
index b1891b918c..a61827c2ae 100644
--- a/tensorflow/compiler/tests/function_test.py
+++ b/tensorflow/compiler/tests/function_test.py
@@ -40,7 +40,7 @@ class FunctionTest(xla_test.XLATestCase):
     bval = np.array([5, 6, 7, 8]).reshape([2, 2]).astype(np.float32)
     expected = APlus2B(aval, bval)
 
-    with self.cached_session() as sess:
+    with self.cached_session():
 
       @function.Defun(dtypes.float32, dtypes.float32)
       def Foo(a, b):
@@ -50,7 +50,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_f = Foo(a, b)
-      result = sess.run(call_f)
+      result = self.evaluate(call_f)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testNestedFunctions(self):
@@ -66,7 +66,7 @@ class FunctionTest(xla_test.XLATestCase):
     bval = np.array([4, 3, 2, 1]).reshape([2, 2]).astype(np.float32)
     expected = APlus2B(aval, bval)
 
-    with self.cached_session() as sess:
+    with self.cached_session():
 
       @function.Defun(dtypes.float32, dtypes.float32)
       def Foo(a, b):
@@ -76,7 +76,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_g = Foo(a, b)
-      result = sess.run(call_g)
+      result = self.evaluate(call_g)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testFunctionMultipleRetvals(self):
@@ -90,7 +90,7 @@ class FunctionTest(xla_test.XLATestCase):
     bval = np.array([5, 6, 7, 8]).reshape([2, 2]).astype(np.float32)
     expected = Func(aval, bval)
 
-    with self.cached_session() as sess:
+    with self.cached_session():
 
       @function.Defun(dtypes.float32, dtypes.float32)
       def Foo(a, b):
@@ -100,7 +100,7 @@ class FunctionTest(xla_test.XLATestCase):
       b = constant_op.constant(bval, name="b")
       with self.test_scope():
         call_f = Foo(a, b)
-      result = sess.run(call_f)
+      result = self.evaluate(call_f)
     self.assertAllClose(result, expected, rtol=1e-3)
 
   def testCompileTimeConstantsInDefun(self):
diff --git a/tensorflow/compiler/tests/listdiff_op_test.py b/tensorflow/compiler/tests/listdiff_op_test.py
index 58622114e4..0210201fa7 100644
--- a/tensorflow/compiler/tests/listdiff_op_test.py
+++ b/tensorflow/compiler/tests/listdiff_op_test.py
@@ -33,13 +33,13 @@ class ListDiffTest(xla_test.XLATestCase):
   def _testListDiff(self, x, y, out, idx):
     for dtype in [dtypes.int32, dtypes.int64]:
       for index_dtype in [dtypes.int32, dtypes.int64]:
-        with self.cached_session() as sess:
+        with self.cached_session():
           x_tensor = ops.convert_to_tensor(x, dtype=dtype)
           y_tensor = ops.convert_to_tensor(y, dtype=dtype)
           with self.test_scope():
             out_tensor, idx_tensor = array_ops.listdiff(
                 x_tensor, y_tensor, out_idx=index_dtype)
-            tf_out, tf_idx = sess.run([out_tensor, idx_tensor])
+            tf_out, tf_idx = self.evaluate([out_tensor, idx_tensor])
         self.assertAllEqual(out, tf_out)
         self.assertAllEqual(idx, tf_idx)
         self.assertEqual(1, out_tensor.get_shape().ndims)
diff --git a/tensorflow/compiler/tests/lstm_test.py b/tensorflow/compiler/tests/lstm_test.py
index 265c0b6d14..776ed899e6 100644
--- a/tensorflow/compiler/tests/lstm_test.py
+++ b/tensorflow/compiler/tests/lstm_test.py
@@ -88,8 +88,8 @@ class LSTMTest(test.TestCase):
                  (basename, m_prev_scalar, c_prev_scalar, pad_scalar))
 
       # Initialize variables and run the unrolled LSTM step.
-      sess.run(variables.global_variables_initializer())
-      return sess.run([m, c])
+      self.evaluate(variables.global_variables_initializer())
+      return self.evaluate([m, c])
 
   def testLSTMCell(self):
     # Run with all-0 weights, no padding.
@@ -173,8 +173,8 @@ class LSTMTest(test.TestCase):
                  (basename, m_init_scalar, c_init_scalar, pad_scalar))
 
       # Initialize variables and run the unrolled LSTM layer.
-      sess.run(variables.global_variables_initializer())
-      return sess.run(out_seq)
+      self.evaluate(variables.global_variables_initializer())
+      return self.evaluate(out_seq)
 
   def testLSTMLayer(self):
     # Run with all-0 weights, no padding.
diff --git a/tensorflow/compiler/tests/placeholder_test.py b/tensorflow/compiler/tests/placeholder_test.py
index 77bb839409..9671ae0ae9 100644
--- a/tensorflow/compiler/tests/placeholder_test.py
+++ b/tensorflow/compiler/tests/placeholder_test.py
@@ -33,7 +33,7 @@ class PlaceholderTest(xla_test.XLATestCase):
       ph = array_ops.placeholder_with_default(v, shape=[])
       out = ph * 2
       sess.run(variables.variables_initializer([v]))
-      self.assertEqual(8.0, sess.run(out))
+      self.assertEqual(8.0, self.evaluate(out))
 
   def test_placeholder_with_default_fed(self):
     with self.cached_session() as sess, self.test_scope():
diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py
index 36ef6ed5fe..97ffad34c0 100644
--- a/tensorflow/compiler/tests/random_ops_test.py
+++ b/tensorflow/compiler/tests/random_ops_test.py
@@ -46,9 +46,9 @@ class RandomOpsTest(xla_test.XLATestCase):
 
       # The random-number generator, if working correctly, should produce the
       # same output multiple times with low probability.
-      y = sess.run(x)
-      z = sess.run(x)
-      w = sess.run(x)
+      y = self.evaluate(x)
+      z = self.evaluate(x)
+      w = self.evaluate(x)
 
       # We use exact equality here. If the random-number generator is producing
       # deterministic output, all three outputs will be bitwise identical.
@@ -83,7 +83,7 @@ class RandomOpsTest(xla_test.XLATestCase):
         with self.test_scope():
           x = random_ops.random_uniform(
               shape=[1000], dtype=dtype, minval=-2, maxval=33)
-        y = sess.run(x)
+        y = self.evaluate(x)
         self.assertTrue((y >= -2).sum() == 1000)
         self.assertTrue((y < 33).sum() == 1000)
 
@@ -102,7 +102,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.cached_session() as sess:
         with self.test_scope():
           x = random_ops.truncated_normal(shape=[count], dtype=dtype)
-        y = sess.run(x)
+        y = self.evaluate(x)
 
         def normal_cdf(x):
           return .5 * math.erfc(-x / math.sqrt(2))
@@ -111,7 +111,7 @@ class RandomOpsTest(xla_test.XLATestCase):
           return math.exp(-(x**2) / 2.) / math.sqrt(2 * math.pi)
 
         def probit(x, sess=sess):
-          return sess.run(special_math.ndtri(x))
+          return self.evaluate(special_math.ndtri(x))
 
         a = -2.
         b = 2.
@@ -148,7 +148,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.test_scope():
         x = math_ops.range(1 << 16)
         shuffle = random_ops.random_shuffle(x)
-      result = sess.run(shuffle)
+      result = self.evaluate(shuffle)
       expected = range(1 << 16)
       # Compare sets to avoid randomness behavior changes but make sure still
       # have all the values.
@@ -159,7 +159,7 @@ class RandomOpsTest(xla_test.XLATestCase):
       with self.test_scope():
         x = array_ops.diag(math_ops.range(20))
         shuffle = random_ops.random_shuffle(x)
-      result = sess.run(shuffle)
+      result = self.evaluate(shuffle)
       expected = np.diag(range(20)).flatten()
       # Compare sets to avoid randomness behavior changes but make sure still
       # have all the values.
diff --git a/tensorflow/compiler/tests/stateless_random_ops_test.py b/tensorflow/compiler/tests/stateless_random_ops_test.py
index 21708aa158..ee7ca7e6f1 100644
--- a/tensorflow/compiler/tests/stateless_random_ops_test.py
+++ b/tensorflow/compiler/tests/stateless_random_ops_test.py
@@ -156,7 +156,7 @@ class StatelessRandomOpsTest(xla_test.XLATestCase):
           return math.exp(-(x**2) / 2.) / math.sqrt(2 * math.pi)
 
         def probit(x, sess=sess):
-          return sess.run(special_math.ndtri(x))
+          return self.evaluate(special_math.ndtri(x))
 
         a = -2.
         b = 2.
diff --git a/tensorflow/compiler/tests/tensor_array_ops_test.py b/tensorflow/compiler/tests/tensor_array_ops_test.py
index c8208adb58..d7e26d79c4 100644
--- a/tensorflow/compiler/tests/tensor_array_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_array_ops_test.py
@@ -505,7 +505,7 @@ class TensorArrayTest(xla_test.XLATestCase):
                 [-0.5, 1.5],  # read(0) gradient
                 [20.0, 30.0, 40.0, 50.0],  # concat gradient
             ])
-      grad_vals = sess.run(grad_r)  # 2 + 2 entries
+      grad_vals = self.evaluate(grad_r)  # 2 + 2 entries
 
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py
index 77cdeac816..fcd7ac5ba1 100644
--- a/tensorflow/compiler/tests/variable_ops_test.py
+++ b/tensorflow/compiler/tests/variable_ops_test.py
@@ -77,7 +77,7 @@ class VariableOpsTest(xla_test.XLATestCase):
         sess.run(variables.variables_initializer([v]))
         x = v.sparse_read(2)
         self.assertAllClose(
-            np.array([8j, 9, 10, 11]).astype(dtype), sess.run(x))
+            np.array([8j, 9, 10, 11]).astype(dtype), self.evaluate(x))
 
   def testSparseRead1DIndices(self):
     for dtype in self.numeric_types:
@@ -89,7 +89,7 @@ class VariableOpsTest(xla_test.XLATestCase):
         x = v.sparse_read([2, 1])
         self.assertAllClose(
             np.array([[8, 9, 10, 11], [4, 5, 6j, 7]]).astype(dtype),
-            sess.run(x))
+            self.evaluate(x))
 
   def testSparseRead2DIndices(self):
     for dtype in self.numeric_types:
@@ -102,7 +102,7 @@ class VariableOpsTest(xla_test.XLATestCase):
         self.assertAllClose(
             np.array([[[8, 9, 10, 11], [4, 5, 6, 7]],
                       [[0, 1, 2j, 3], [8, 9, 10, 11]]]).astype(dtype),
-            sess.run(x))
+            self.evaluate(x))
 
   def testSparseRead2DIndices3DTensor(self):
     for dtype in self.numeric_types:
@@ -115,9 +115,9 @@ class VariableOpsTest(xla_test.XLATestCase):
         x = v.sparse_read([[2, 1], [3, 0]])
         self.assertAllClose(
             np.array(
-                [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]
-                 ], [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]]
-                ],).astype(dtype), sess.run(x))
+                [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]],
+                 [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]]
+                ],).astype(dtype), self.evaluate(x))
 
   def testShape(self):
     for dtype in self.numeric_types:
@@ -229,7 +229,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_add(
               handle, [0], constant_op.constant([[2]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(sess.run(read), [[3], [7]])
+      self.assertAllEqual(self.evaluate(read), [[3], [7]])
 
   def testScatterSub(self):
     with self.test_session() as sess, self.test_scope():
@@ -242,7 +242,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_sub(
               handle, [1], constant_op.constant([[2]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(sess.run(read), [[4], [-1]])
+      self.assertAllEqual(self.evaluate(read), [[4], [-1]])
 
   def testScatterMul(self):
     with self.test_session() as sess, self.test_scope():
@@ -255,7 +255,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_mul(
               handle, [0], constant_op.constant([[5]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[5]])
+      self.assertEqual(self.evaluate(read), [[5]])
 
   def testScatterDiv(self):
     with self.test_session() as sess, self.test_scope():
@@ -268,7 +268,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_div(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertAllEqual(sess.run(read), [[2]])
+      self.assertAllEqual(self.evaluate(read), [[2]])
 
   def testScatterMin(self):
     with self.test_session() as sess, self.test_scope():
@@ -281,7 +281,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_min(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterMax(self):
     with self.test_session() as sess, self.test_scope():
@@ -294,7 +294,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_max(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[6]])
+      self.assertEqual(self.evaluate(read), [[6]])
 
   def testScatterUpdate(self):
     with self.test_session() as sess, self.test_scope():
@@ -307,7 +307,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_update(
               handle, [0], constant_op.constant([[3]], dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterAddScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -320,7 +320,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_add(
               handle, [0], constant_op.constant(2, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterSubScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -333,7 +333,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_sub(
               handle, [0], constant_op.constant(2, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[-1]])
+      self.assertEqual(self.evaluate(read), [[-1]])
 
   def testScatterMulScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -346,7 +346,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_mul(
               handle, [0], constant_op.constant(5, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[5]])
+      self.assertEqual(self.evaluate(read), [[5]])
 
   def testScatterDivScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -359,7 +359,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_div(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[2]])
+      self.assertEqual(self.evaluate(read), [[2]])
 
   def testScatterMinScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -372,7 +372,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_min(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[3]])
+      self.assertEqual(self.evaluate(read), [[3]])
 
   def testScatterMaxScalar(self):
     with self.test_session() as sess, self.test_scope():
@@ -385,7 +385,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           resource_variable_ops.resource_scatter_max(
               handle, [0], constant_op.constant(3, dtype=dtypes.int32)))
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
-      self.assertEqual(sess.run(read), [[6]])
+      self.assertEqual(self.evaluate(read), [[6]])
 
   def testScatterNdAddOps(self):
     with self.test_session() as sess, self.test_scope():
@@ -400,7 +400,7 @@ class VariableOpsTest(xla_test.XLATestCase):
       sess.run(gen_state_ops.resource_scatter_nd_add(handle, indices, updates))
       read = resource_variable_ops.read_variable_op(
           handle, dtype=dtypes.float32)
-      self.assertAllClose(expected, sess.run(read))
+      self.assertAllClose(expected, self.evaluate(read))
 
   def testScatterNdUpdateAddOps(self):
     with self.test_session() as sess, self.test_scope():
@@ -416,7 +416,7 @@ class VariableOpsTest(xla_test.XLATestCase):
           gen_state_ops.resource_scatter_nd_update(handle, indices, updates))
       read = resource_variable_ops.read_variable_op(
           handle, dtype=dtypes.float32)
-      self.assertAllClose(expected, sess.run(read))
+      self.assertAllClose(expected, self.evaluate(read))
 
 
 class StridedSliceAssignChecker(object):
diff --git a/tensorflow/compiler/tests/xla_device_test.py b/tensorflow/compiler/tests/xla_device_test.py
index 28d61fb07d..ef55292b1b 100644
--- a/tensorflow/compiler/tests/xla_device_test.py
+++ b/tensorflow/compiler/tests/xla_device_test.py
@@ -81,7 +81,7 @@ class XlaDeviceTest(xla_test.XLATestCase):
     with self.cached_session() as sess:
       with self.test_scope():
         x = gen_control_flow_ops.control_trigger()
-      sess.run(x)
+      self.evaluate(x)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/examples/autograph/integration_tests/keras_test.py b/tensorflow/examples/autograph/integration_tests/keras_test.py
index dca7c07b47..fc0b073696 100644
--- a/tensorflow/examples/autograph/integration_tests/keras_test.py
+++ b/tensorflow/examples/autograph/integration_tests/keras_test.py
@@ -93,10 +93,10 @@ class KerasTest(tf.test.TestCase):
         init = tf.global_variables_initializer()
 
         with tf.Session() as sess:
-          sess.run(init)
+          self.evaluate(init)
           sample_input = tf.random_uniform((1, 10, 10, 1))
           output = model(sample_input)  # pylint: disable=not-callable
-          self.assertEqual(sess.run(output).shape, (1, 3))
+          self.assertEqual(self.evaluate(output).shape, (1, 3))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/examples/autograph/integration_tests/list_literals_test.py b/tensorflow/examples/autograph/integration_tests/list_literals_test.py
index 917f5ff9d8..e85d4abcfc 100644
--- a/tensorflow/examples/autograph/integration_tests/list_literals_test.py
+++ b/tensorflow/examples/autograph/integration_tests/list_literals_test.py
@@ -34,7 +34,7 @@ class ListLiteralsTest(tf.test.TestCase):
     result = converted()
 
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(result), [1, 2, 3])
+      self.assertAllEqual(self.evaluate(result), [1, 2, 3])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/examples/speech_commands/input_data_test.py b/tensorflow/examples/speech_commands/input_data_test.py
index b766ba6de0..33b58b9d09 100644
--- a/tensorflow/examples/speech_commands/input_data_test.py
+++ b/tensorflow/examples/speech_commands/input_data_test.py
@@ -35,7 +35,7 @@ class InputDataTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([32000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = sess.run(wav_encoder)
+      wav_data = self.evaluate(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/examples/speech_commands/label_wav_test.py b/tensorflow/examples/speech_commands/label_wav_test.py
index f0af2a4798..77a88f98e1 100644
--- a/tensorflow/examples/speech_commands/label_wav_test.py
+++ b/tensorflow/examples/speech_commands/label_wav_test.py
@@ -33,7 +33,7 @@ class LabelWavTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([1000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = sess.run(wav_encoder)
+      wav_data = self.evaluate(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/examples/speech_commands/wav_to_features_test.py b/tensorflow/examples/speech_commands/wav_to_features_test.py
index 87f2987693..cb8ea912fa 100644
--- a/tensorflow/examples/speech_commands/wav_to_features_test.py
+++ b/tensorflow/examples/speech_commands/wav_to_features_test.py
@@ -33,7 +33,7 @@ class WavToFeaturesTest(test.TestCase):
     with self.cached_session() as sess:
       sample_data = tf.zeros([32000, 2])
       wav_encoder = contrib_audio.encode_wav(sample_data, 16000)
-      wav_data = sess.run(wav_encoder)
+      wav_data = self.evaluate(wav_encoder)
     return wav_data
 
   def _saveTestWavFile(self, filename, wav_data):
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
index eeb48d1231..9c00d0501a 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
@@ -111,7 +111,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
 
     # Initialize variables
     init = tf.global_variables_initializer()
-    sess.run(init)
+    self.evaluate(init)
     for _ in range(TRAIN_STEPS):
       batch_x, batch_y = self.mnist.train.next_batch(
           batch_size=self.batch_size, shuffle=False)
diff --git a/tensorflow/python/autograph/converters/asserts_test.py b/tensorflow/python/autograph/converters/asserts_test.py
index eef628aeb6..803b6a06da 100644
--- a/tensorflow/python/autograph/converters/asserts_test.py
+++ b/tensorflow/python/autograph/converters/asserts_test.py
@@ -41,7 +41,7 @@ class AssertsTest(converter_testing.TestCase):
         op = result.test_fn(constant_op.constant(False))
         with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                      'test message'):
-          sess.run(op)
+          self.evaluate(op)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 916c736fb4..9d760167a9 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -94,7 +94,7 @@ class CallTreesTest(converter_testing.TestCase):
                         dtypes.int64) as result:
       with self.cached_session() as sess:
         self.assertTrue(isinstance(result.test_fn(), ops.Tensor))
-        self.assertIn(sess.run(result.test_fn()), (0, 1, 2))
+        self.assertIn(self.evaluate(result.test_fn()), (0, 1, 2))
 
   def test_uncompiled_modules(self):
 
@@ -113,7 +113,7 @@ class CallTreesTest(converter_testing.TestCase):
     with self.compiled(node, ns) as result:
       with self.cached_session() as sess:
         result_tensor = result.test_fn(constant_op.constant(1))
-        self.assertEquals(sess.run(result_tensor), 3)
+        self.assertEquals(self.evaluate(result_tensor), 3)
 
   def test_call_to_decorated_function(self):
 
diff --git a/tensorflow/python/autograph/converters/lists_test.py b/tensorflow/python/autograph/converters/lists_test.py
index f6da845fcc..39843c7d74 100644
--- a/tensorflow/python/autograph/converters/lists_test.py
+++ b/tensorflow/python/autograph/converters/lists_test.py
@@ -68,7 +68,7 @@ class ListTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(sess.run(r), [1, 2, 3])
+        self.assertAllEqual(self.evaluate(r), [1, 2, 3])
 
   def test_list_pop(self):
 
@@ -91,8 +91,8 @@ class ListTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         ts, tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(sess.run(r), [1, 2])
-        self.assertAllEqual(sess.run(ts), 3)
+        self.assertAllEqual(self.evaluate(r), [1, 2])
+        self.assertAllEqual(self.evaluate(ts), 3)
 
   def test_double_list_pop(self):
 
@@ -123,7 +123,7 @@ class ListTest(converter_testing.TestCase):
 
     with self.compiled(node, {}, array_ops.stack, dtypes.int32) as result:
       with self.cached_session() as sess:
-        self.assertAllEqual(sess.run(result.test_fn()), [1, 2, 3])
+        self.assertAllEqual(self.evaluate(result.test_fn()), [1, 2, 3])
 
   # TODO(mdan): Add a test with tf.stack with axis kwarg.
 
diff --git a/tensorflow/python/autograph/converters/side_effect_guards_test.py b/tensorflow/python/autograph/converters/side_effect_guards_test.py
index cef3199169..f6d0f73b5b 100644
--- a/tensorflow/python/autograph/converters/side_effect_guards_test.py
+++ b/tensorflow/python/autograph/converters/side_effect_guards_test.py
@@ -48,12 +48,12 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
-        sess.run(result.test_fn(v))
+        self.evaluate(v.initializer)
+        self.evaluate(result.test_fn(v))
         # TODO(mdan): Add support for this use case.
         # Right now the variable `a` is not conditioned on the `assign` because
         # there's no way to add control dependencies to a variable object.
-        self.assertEqual(2, sess.run(v))
+        self.assertEqual(2, self.evaluate(v))
 
   def test_side_effect_on_used_variable(self):
 
@@ -69,11 +69,11 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
-        sess.run(result.test_fn(v))
+        self.evaluate(v.initializer)
+        self.evaluate(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
         # Right now it's 3 or 4 based on whether the read is synchronized.
-        self.assertEqual(3, sess.run(v))
+        self.assertEqual(3, self.evaluate(v))
 
   def test_side_effect_on_tensor(self):
 
@@ -109,10 +109,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign_add) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
-        sess.run(result.test_fn(v))
+        self.evaluate(v.initializer)
+        self.evaluate(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(4, sess.run(v))
+        self.assertEqual(4, self.evaluate(v))
 
   def test_multiline_nested_block(self):
 
@@ -130,10 +130,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
     with self.compiled(node, {}, state_ops.assign, ops.name_scope) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
-        sess.run(result.test_fn(v))
+        self.evaluate(v.initializer)
+        self.evaluate(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(3, sess.run(v))
+        self.assertEqual(3, self.evaluate(v))
 
   def test_multiline_block_unsafe(self):
 
@@ -153,10 +153,10 @@ class SideEffectGuardsTest(converter_testing.TestCase):
                        state_ops.assign_add) as result:
       with self.cached_session() as sess:
         v = variable_scope.get_variable('test', initializer=2)
-        sess.run(v.initializer)
-        sess.run(result.test_fn(v))
+        self.evaluate(v.initializer)
+        self.evaluate(result.test_fn(v))
         # TODO(mdan): Ensure the result of test_fn(v) is also deterministic.
-        self.assertEqual(4, sess.run(v))
+        self.assertEqual(4, self.evaluate(v))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py
index e190a7cfe8..bd049afdfc 100644
--- a/tensorflow/python/autograph/converters/slices_test.py
+++ b/tensorflow/python/autograph/converters/slices_test.py
@@ -49,7 +49,7 @@ class SliceTest(converter_testing.TestCase):
         tl = list_ops.tensor_list_from_tensor(
             [1, 2], element_shape=constant_op.constant([], dtype=dtypes.int32))
         y = result.test_fn(tl)
-        self.assertEqual(2, sess.run(y))
+        self.assertEqual(2, self.evaluate(y))
 
   def test_index_access_multiple_definitions(self):
 
diff --git a/tensorflow/python/autograph/core/errors_test.py b/tensorflow/python/autograph/core/errors_test.py
index aa6c293268..00c8a726ed 100644
--- a/tensorflow/python/autograph/core/errors_test.py
+++ b/tensorflow/python/autograph/core/errors_test.py
@@ -55,7 +55,7 @@ class RuntimeErrorsTest(test.TestCase):
     with self.assertRaises(errors.TfRuntimeError) as cm:
       with errors.improved_errors(zero_div_caller):
         with self.cached_session() as sess:
-          sess.run(ops)
+          self.evaluate(ops)
 
     for frame in cm.exception.custom_traceback:
       _, _, function_name, _ = frame
@@ -70,7 +70,7 @@ class RuntimeErrorsTest(test.TestCase):
     with self.assertRaises(errors.TfRuntimeError) as cm:
       with errors.improved_errors(zero_div_caller):
         with self.cached_session() as sess:
-          sess.run(ops)
+          self.evaluate(ops)
 
     all_function_names = set()
     for frame in cm.exception.custom_traceback:
@@ -87,7 +87,7 @@ class RuntimeErrorsTest(test.TestCase):
     with self.assertRaises(tf_errors.InvalidArgumentError):
       with errors.improved_errors(zero_div_caller):
         with self.cached_session() as sess:
-          sess.run(ops)
+          self.evaluate(ops)
 
   def test_improved_errors_validation(self):
     with self.assertRaisesRegexp(
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index ef577568c4..44cb99d657 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -63,7 +63,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_does_not_recurse(self):
 
@@ -83,7 +83,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_calls_unconverted_graph(self):
 
@@ -104,7 +104,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_calls_unconverted_py_func(self):
 
@@ -130,7 +130,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_calls_decorated(self):
 
@@ -153,7 +153,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_decorator_preserves_argspec(self):
 
@@ -192,7 +192,7 @@ class ApiTest(test.TestCase):
       x = tc.test_method(
           constant_op.constant([2, 4]), constant_op.constant(1),
           constant_op.constant(-2))
-      self.assertListEqual([0, 1], sess.run(x).tolist())
+      self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
   def test_converted_call_builtin(self):
     x = api.converted_call(range, None, converter.ConversionOptions(), 3)
@@ -208,7 +208,7 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       x = api.converted_call(test_fn, None, converter.ConversionOptions(),
                              constant_op.constant(-1))
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_explicit_owner(self):
     # TODO(mdan): Implement.
@@ -234,7 +234,7 @@ class ApiTest(test.TestCase):
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(tc.test_method, None,
                              converter.ConversionOptions(), tc)
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_method_by_class(self):
 
@@ -252,7 +252,7 @@ class ApiTest(test.TestCase):
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(TestClass.test_method, None,
                              converter.ConversionOptions(), tc)
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_callable_object(self):
 
@@ -269,7 +269,7 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       tc = TestClass(constant_op.constant(-1))
       x = api.converted_call(tc, None, converter.ConversionOptions())
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_constructor(self):
 
@@ -288,7 +288,7 @@ class ApiTest(test.TestCase):
                               constant_op.constant(-1))
       # tc is now a converted object.
       x = tc.test_method()
-      self.assertEqual(1, sess.run(x))
+      self.assertEqual(1, self.evaluate(x))
 
   def test_converted_call_already_converted(self):
 
@@ -298,12 +298,12 @@ class ApiTest(test.TestCase):
     with self.cached_session() as sess:
       x = api.converted_call(f, None, converter.ConversionOptions(),
                              constant_op.constant(0))
-      self.assertTrue(sess.run(x))
+      self.assertTrue(self.evaluate(x))
 
       converted_f = api.to_graph(f)
       x = api.converted_call(converted_f, None, converter.ConversionOptions(),
                              constant_op.constant(0))
-      self.assertTrue(sess.run(x))
+      self.assertTrue(self.evaluate(x))
 
   def test_converted_call_no_user_code(self):
 
@@ -334,8 +334,8 @@ class ApiTest(test.TestCase):
                            constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_extra_self(self):
 
@@ -349,8 +349,8 @@ class ApiTest(test.TestCase):
                            model, constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_whitelisted_method_via_owner(self):
 
@@ -364,8 +364,8 @@ class ApiTest(test.TestCase):
                            constant_op.constant([[0.0]]), training=True)
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual([[0.0, 0.0]], sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual([[0.0, 0.0]], self.evaluate(x))
 
   def test_converted_call_lambda(self):
 
@@ -376,8 +376,8 @@ class ApiTest(test.TestCase):
     x = api.converted_call(l, None, opts, constant_op.constant(0))
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      self.assertAllEqual(True, sess.run(x))
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual(True, self.evaluate(x))
 
   def test_to_graph_basic(self):
 
@@ -390,7 +390,7 @@ class ApiTest(test.TestCase):
 
     with self.cached_session() as sess:
       x = compiled_fn(constant_op.constant([4, 8]), 4)
-      self.assertListEqual([1, 2], sess.run(x).tolist())
+      self.assertListEqual([1, 2], self.evaluate(x).tolist())
 
   def test_to_graph_with_defaults(self):
 
@@ -405,7 +405,7 @@ class ApiTest(test.TestCase):
 
     with self.cached_session() as sess:
       x = compiled_fn(constant_op.constant([4, 8]))
-      self.assertListEqual([1, 2], sess.run(x).tolist())
+      self.assertListEqual([1, 2], self.evaluate(x).tolist())
 
   def test_to_code_basic(self):
 
diff --git a/tensorflow/python/autograph/lang/special_functions_test.py b/tensorflow/python/autograph/lang/special_functions_test.py
index 123ee65b32..8d40f4036c 100644
--- a/tensorflow/python/autograph/lang/special_functions_test.py
+++ b/tensorflow/python/autograph/lang/special_functions_test.py
@@ -36,7 +36,7 @@ class SpecialFunctionsTest(test.TestCase):
     python_one = special_functions.match_staging_level(1, 1)
     with self.cached_session() as sess:
       self.assertTrue(tensor_util.is_tensor(tensor_one))
-      self.assertAllEqual(sess.run(tensor_one), 1)
+      self.assertAllEqual(self.evaluate(tensor_one), 1)
       self.assertEqual(python_one, 1)
 
   def test_tensor_list_empty_list(self):
@@ -45,21 +45,21 @@ class SpecialFunctionsTest(test.TestCase):
                                       element_shape=())
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [])
+      self.assertAllEqual(self.evaluate(sl), [])
 
     l = special_functions.tensor_list((),
                                       element_dtype=dtypes.int32,
                                       element_shape=())
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [])
+      self.assertAllEqual(self.evaluate(sl), [])
 
   def test_tensor_list_tensor(self):
     l = special_functions.tensor_list(
         constant_op.constant([], dtype=dtypes.int32))
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [])
+      self.assertAllEqual(self.evaluate(sl), [])
 
   def test_tensor_list_unsupported_initializer(self):
     with self.assertRaisesRegexp(ValueError, 'unknown type'):
@@ -76,7 +76,7 @@ class SpecialFunctionsTest(test.TestCase):
     l = special_functions.tensor_list(elements)
     sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
+      self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]])
 
   def test_tensor_list_array_from_elements(self):
     elements = [constant_op.constant([1, 2]), constant_op.constant([3, 4])]
@@ -84,7 +84,7 @@ class SpecialFunctionsTest(test.TestCase):
     l = special_functions.tensor_list(elements, use_tensor_array=True)
     sl = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
+      self.assertAllEqual(self.evaluate(sl), [[1, 2], [3, 4]])
 
   def test_stack(self):
     self.assertEqual(special_functions.stack(1, strict=False), 1)
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 2dea18dc5f..05b5660941 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -35,7 +35,7 @@ class ForLoopTest(test.TestCase):
         body=lambda i, s: (s + i,),
         init_state=(0,))
     with self.cached_session() as sess:
-      self.assertEqual((10,), sess.run(s))
+      self.assertEqual((10,), self.evaluate(s))
 
   def test_python(self):
     s = control_flow.for_stmt(
@@ -53,7 +53,7 @@ class ForLoopTest(test.TestCase):
         body=lambda i, s: (s + i,),
         init_state=(0,))
     with self.cached_session() as sess:
-      self.assertEqual((10,), sess.run(s))
+      self.assertEqual((10,), self.evaluate(s))
 
 
 class WhileLoopTest(test.TestCase):
@@ -66,7 +66,7 @@ class WhileLoopTest(test.TestCase):
         init_state=(0, 0),
         extra_deps=(n,))
     with self.cached_session() as sess:
-      self.assertEqual((5, 10), sess.run(results))
+      self.assertEqual((5, 10), self.evaluate(results))
 
   def test_python(self):
     n = 5
@@ -90,9 +90,9 @@ class IfStmtTest(test.TestCase):
   def test_tensor(self):
     with self.cached_session() as sess:
       t = self.single_return_if_stmt(constant_op.constant(True))
-      self.assertEqual(1, sess.run(t))
+      self.assertEqual(1, self.evaluate(t))
       t = self.single_return_if_stmt(constant_op.constant(False))
-      self.assertEqual(-1, sess.run(t))
+      self.assertEqual(-1, self.evaluate(t))
 
   def test_python(self):
     self.assertEqual(1, self.single_return_if_stmt(True))
@@ -101,9 +101,9 @@ class IfStmtTest(test.TestCase):
   def test_tensor_multiple_returns(self):
     with self.cached_session() as sess:
       t = self.multi_return_if_stmt(constant_op.constant(True))
-      self.assertAllEqual([1, 2], sess.run(t))
+      self.assertAllEqual([1, 2], self.evaluate(t))
       t = self.multi_return_if_stmt(constant_op.constant(False))
-      self.assertAllEqual([-1, -2], sess.run(t))
+      self.assertAllEqual([-1, -2], self.evaluate(t))
 
   def test_python_multiple_returns(self):
     self.assertEqual((1, 2), self.multi_return_if_stmt(True))
diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py
index 72476ccb6b..0433e3f130 100644
--- a/tensorflow/python/autograph/operators/data_structures_test.py
+++ b/tensorflow/python/autograph/operators/data_structures_test.py
@@ -43,7 +43,7 @@ class ListTest(test.TestCase):
     l = data_structures.tf_tensor_list_new([3, 4, 5])
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4, 5])
+      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
   def test_tf_tensor_list_new_empty(self):
     l = data_structures.tf_tensor_list_new([],
@@ -51,13 +51,13 @@ class ListTest(test.TestCase):
                                            element_shape=())
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [])
+      self.assertAllEqual(self.evaluate(t), [])
 
   def test_tf_tensor_list_new_from_tensor(self):
     l = data_structures.tf_tensor_list_new(constant_op.constant([3, 4, 5]))
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4, 5])
+      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
   def test_tf_tensor_list_new_illegal_input(self):
     with self.assertRaises(ValueError):
@@ -77,7 +77,7 @@ class ListTest(test.TestCase):
     l = data_structures.tf_tensor_array_new([3, 4, 5])
     t = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4, 5])
+      self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
   def test_tf_tensor_array_new_illegal_input(self):
     with self.assertRaises(ValueError):
@@ -102,15 +102,15 @@ class ListTest(test.TestCase):
 
     t = list_ops.tensor_list_stack(l, element_dtype=x.dtype)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [[1, 2, 3]])
+      self.assertAllEqual(self.evaluate(t), [[1, 2, 3]])
 
   def test_append_tensorarray(self):
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l1 = data_structures.list_append(l, 1)
     l2 = data_structures.list_append(l1, 2)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(l1.stack()), [1])
-      self.assertAllEqual(sess.run(l2.stack()), [1, 2])
+      self.assertAllEqual(self.evaluate(l1.stack()), [1])
+      self.assertAllEqual(self.evaluate(l2.stack()), [1, 2])
 
   def test_append_python(self):
     l = []
@@ -131,10 +131,10 @@ class ListTest(test.TestCase):
 
     with self.cached_session() as sess:
       l, x = data_structures.list_pop(l, None, opts)
-      self.assertAllEqual(sess.run(x), [3, 4])
+      self.assertAllEqual(self.evaluate(x), [3, 4])
 
       t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
-      self.assertAllEqual(sess.run(t), [[1, 2]])
+      self.assertAllEqual(self.evaluate(t), [[1, 2]])
 
   def test_pop_python(self):
     l = [1, 2, 3]
@@ -152,7 +152,7 @@ class ListTest(test.TestCase):
 
     with self.cached_session() as sess:
       t = data_structures.list_stack(l, opts)
-      self.assertAllEqual(sess.run(t), sess.run(initial_list))
+      self.assertAllEqual(self.evaluate(t), self.evaluate(initial_list))
 
   def test_stack_tensor_list_empty(self):
     l = list_ops.empty_tensor_list(
diff --git a/tensorflow/python/autograph/operators/exceptions_test.py b/tensorflow/python/autograph/operators/exceptions_test.py
index 186535d05b..24d3f1bd35 100644
--- a/tensorflow/python/autograph/operators/exceptions_test.py
+++ b/tensorflow/python/autograph/operators/exceptions_test.py
@@ -30,7 +30,7 @@ class ExceptionsTest(test.TestCase):
     with self.cached_session() as sess:
       t = exceptions.assert_stmt(
           constant_op.constant(True), lambda: constant_op.constant('ignored'))
-      sess.run(t)
+      self.evaluate(t)
 
   def test_assert_tf_triggered(self):
     with self.cached_session() as sess:
@@ -40,7 +40,7 @@ class ExceptionsTest(test.TestCase):
 
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    'test message'):
-        sess.run(t)
+        self.evaluate(t)
 
   def test_assert_tf_multiple_printed_values(self):
     two_tensors = [
@@ -53,7 +53,7 @@ class ExceptionsTest(test.TestCase):
 
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    'test message.*another message'):
-        sess.run(t)
+        self.evaluate(t)
 
   def test_assert_python_untriggered(self):
     side_effect_trace = []
diff --git a/tensorflow/python/autograph/operators/logical_test.py b/tensorflow/python/autograph/operators/logical_test.py
index d6649f7b2b..ebf6458f01 100644
--- a/tensorflow/python/autograph/operators/logical_test.py
+++ b/tensorflow/python/autograph/operators/logical_test.py
@@ -45,11 +45,11 @@ class LogicalOperatorsTest(test.TestCase):
   def test_and_tf(self):
     with self.cached_session() as sess:
       t = logical.and_(self._tf_true, self._tf_true)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.and_(self._tf_true, lambda: True)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.and_(self._tf_false, lambda: True)
-      self.assertEqual(sess.run(t), False)
+      self.assertEqual(self.evaluate(t), False)
       # TODO(mdan): Add a test for ops with side effects.
 
   def test_or_python(self):
@@ -63,11 +63,11 @@ class LogicalOperatorsTest(test.TestCase):
   def test_or_tf(self):
     with self.cached_session() as sess:
       t = logical.or_(self._tf_false, self._tf_true)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.or_(self._tf_false, lambda: True)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       t = logical.or_(self._tf_true, lambda: True)
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
       # TODO(mdan): Add a test for ops with side effects.
 
   def test_not_python(self):
@@ -78,7 +78,7 @@ class LogicalOperatorsTest(test.TestCase):
   def test_not_tf(self):
     with self.cached_session() as sess:
       t = logical.not_(self._tf_false())
-      self.assertEqual(sess.run(t), True)
+      self.assertEqual(self.evaluate(t), True)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/operators/py_builtins_test.py b/tensorflow/python/autograph/operators/py_builtins_test.py
index 443e30a475..4d9eec77c3 100644
--- a/tensorflow/python/autograph/operators/py_builtins_test.py
+++ b/tensorflow/python/autograph/operators/py_builtins_test.py
@@ -38,29 +38,29 @@ class PyBuiltinsTest(test.TestCase):
     self.assertEqual(py_builtins.abs_(-1), 1)
     with self.cached_session() as sess:
       t = py_builtins.abs_(constant_op.constant(-1))
-      self.assertEqual(sess.run(t), 1)
+      self.assertEqual(self.evaluate(t), 1)
       t = py_builtins.abs_(constant_op.constant([-1, 2, -3]))
-      self.assertAllEqual(sess.run(t), [1, 2, 3])
+      self.assertAllEqual(self.evaluate(t), [1, 2, 3])
 
   def test_float(self):
     self.assertEqual(py_builtins.float_(10), 10.0)
     self.assertEqual(py_builtins.float_('10.0'), 10.0)
     with self.cached_session() as sess:
       t = py_builtins.float_(constant_op.constant(1, dtype=dtypes.int64))
-      self.assertEqual(sess.run(t), 1.0)
+      self.assertEqual(self.evaluate(t), 1.0)
       st = py_builtins.float_(constant_op.constant('1.0'))
-      self.assertEqual(sess.run(st), 1.0)
+      self.assertEqual(self.evaluate(st), 1.0)
 
   def test_int(self):
     self.assertEqual(py_builtins.int_(10.0), 10)
     self.assertEqual(py_builtins.int_('11', 2), 3)
     with self.cached_session() as sess:
       t = py_builtins.int_(constant_op.constant(1, dtype=dtypes.float64))
-      self.assertEqual(sess.run(t), 1)
+      self.assertEqual(self.evaluate(t), 1)
       st = py_builtins.int_(constant_op.constant('1'))
-      self.assertEqual(sess.run(st), 1)
+      self.assertEqual(self.evaluate(st), 1)
       st = py_builtins.int_(constant_op.constant('1'), 10)
-      self.assertEqual(sess.run(st), 1)
+      self.assertEqual(self.evaluate(st), 1)
 
   def test_int_unsupported_base(self):
     t = constant_op.constant(1, dtype=dtypes.float64)
@@ -73,9 +73,9 @@ class PyBuiltinsTest(test.TestCase):
       t = py_builtins.len_(constant_op.constant([[1], [2], [3]]))
       self.assertEqual(t, 3)
       ta = py_builtins.len_(tensor_array_ops.TensorArray(dtypes.int32, size=5))
-      self.assertEqual(sess.run(ta), 5)
+      self.assertEqual(self.evaluate(ta), 5)
       tl = py_builtins.len_(data_structures.tf_tensor_list_new([3, 4, 5]))
-      self.assertEqual(sess.run(tl), 3)
+      self.assertEqual(self.evaluate(tl), 3)
 
   def test_len_scalar(self):
     with self.assertRaises(ValueError):
@@ -120,18 +120,18 @@ class PyBuiltinsTest(test.TestCase):
   def test_range_tensor(self):
     with self.cached_session() as sess:
       r = py_builtins.range_(constant_op.constant(3))
-      self.assertAllEqual(sess.run(r), [0, 1, 2])
+      self.assertAllEqual(self.evaluate(r), [0, 1, 2])
       r = py_builtins.range_(1, constant_op.constant(3))
-      self.assertAllEqual(sess.run(r), [1, 2])
+      self.assertAllEqual(self.evaluate(r), [1, 2])
       r = py_builtins.range_(2, 0, constant_op.constant(-1))
-      self.assertAllEqual(sess.run(r), [2, 1])
+      self.assertAllEqual(self.evaluate(r), [2, 1])
 
   def test_range_tensor_empty_range(self):
     with self.session() as sess:
       r = py_builtins.range_(constant_op.constant(-3))
-      self.assertAllEqual(sess.run(r), [])
+      self.assertAllEqual(self.evaluate(r), [])
       r = py_builtins.range_(5, constant_op.constant(2))
-      self.assertAllEqual(sess.run(r), [])
+      self.assertAllEqual(self.evaluate(r), [])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/operators/slices_test.py b/tensorflow/python/autograph/operators/slices_test.py
index 9e4865b3c6..d444054fd7 100644
--- a/tensorflow/python/autograph/operators/slices_test.py
+++ b/tensorflow/python/autograph/operators/slices_test.py
@@ -34,7 +34,7 @@ class SlicesTest(test.TestCase):
 
     with self.cached_session() as sess:
       t = list_ops.tensor_list_stack(l, element_dtype=initial_list.dtype)
-      self.assertAllEqual(sess.run(t), [[5, 6], [3, 4]])
+      self.assertAllEqual(self.evaluate(t), [[5, 6], [3, 4]])
 
   def test_get_item_tensor_list(self):
     initial_list = constant_op.constant([[1, 2], [3, 4]])
@@ -44,7 +44,7 @@ class SlicesTest(test.TestCase):
         l, 1, slices.GetItemOpts(element_dtype=initial_list.dtype))
 
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(t), [3, 4])
+      self.assertAllEqual(self.evaluate(t), [3, 4])
 
   def test_get_item_tensor_string(self):
     initial_str = constant_op.constant('abcd')
@@ -52,14 +52,14 @@ class SlicesTest(test.TestCase):
                         slices.GetItemOpts(element_dtype=initial_str.dtype))
 
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(t), b'b')
+      self.assertEqual(self.evaluate(t), b'b')
 
     initial_list_str = constant_op.constant(['abcd', 'bcde'])
     t = slices.get_item(initial_list_str, 1,
                         slices.GetItemOpts(element_dtype=initial_str.dtype))
 
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(t), b'bcde')
+      self.assertEqual(self.evaluate(t), b'bcde')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py
index 8d2b0d6e13..c813e0f5c9 100644
--- a/tensorflow/python/autograph/utils/misc_test.py
+++ b/tensorflow/python/autograph/utils/misc_test.py
@@ -32,7 +32,7 @@ class MiscTest(test.TestCase):
     new_a = alias_tensors(a)
     self.assertFalse(new_a is a)
     with self.cached_session() as sess:
-      self.assertEqual(1, sess.run(new_a))
+      self.assertEqual(1, self.evaluate(new_a))
 
   def test_alias_tensors(self):
     a = constant(1)
@@ -47,7 +47,7 @@ class MiscTest(test.TestCase):
     self.assertTrue(new_s is s)
     self.assertTrue(new_l is l)
     with self.cached_session() as sess:
-      self.assertEqual(1, sess.run(new_a))
+      self.assertEqual(1, self.evaluate(new_a))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/autograph/utils/py_func_test.py b/tensorflow/python/autograph/utils/py_func_test.py
index 1c220d9492..28cefd8c3e 100644
--- a/tensorflow/python/autograph/utils/py_func_test.py
+++ b/tensorflow/python/autograph/utils/py_func_test.py
@@ -34,13 +34,13 @@ class PyFuncTest(test.TestCase):
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (1, constant_op.constant(1), 1))
-      self.assertEqual(3, sess.run(result))
+      self.assertEqual(3, self.evaluate(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, 1, 1))
-      self.assertEqual(3, sess.run(result))
+      self.assertEqual(3, self.evaluate(result))
       result = py_func.wrap_py_func(
           test_fn, dtypes.int64,
           (constant_op.constant(1), 1, constant_op.constant(1)))
-      self.assertEqual(3, sess.run(result))
+      self.assertEqual(3, self.evaluate(result))
 
   def test_wrap_py_func_complex_args(self):
 
@@ -54,10 +54,10 @@ class PyFuncTest(test.TestCase):
 
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass()))
-      self.assertEqual(35, sess.run(result))
+      self.assertEqual(35, self.evaluate(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (constant_op.constant(7), TestClass()))
-      self.assertEqual(35, sess.run(result))
+      self.assertEqual(35, self.evaluate(result))
 
   def test_wrap_py_func_kwargs(self):
 
@@ -74,13 +74,13 @@ class PyFuncTest(test.TestCase):
           'c': 11,
           'd': TestClass(13)
       })
-      self.assertEqual(178, sess.run(result))
+      self.assertEqual(178, self.evaluate(result))
       result = py_func.wrap_py_func(test_fn, dtypes.int64,
                                     (constant_op.constant(7), TestClass(5)), {
                                         'c': constant_op.constant(11),
                                         'd': TestClass(13)
                                     })
-      self.assertEqual(178, sess.run(result))
+      self.assertEqual(178, self.evaluate(result))
 
   def test_wrap_py_func_dummy_return(self):
 
@@ -91,11 +91,11 @@ class PyFuncTest(test.TestCase):
 
     with self.cached_session() as sess:
       result = py_func.wrap_py_func(test_fn, None, (5,), use_dummy_return=True)
-      self.assertEqual(1, sess.run(result))
+      self.assertEqual(1, self.evaluate(result))
       self.assertEqual([1], side_counter)
       result = py_func.wrap_py_func(
           test_fn, None, (constant_op.constant(5),), use_dummy_return=True)
-      self.assertEqual(1, sess.run(result))
+      self.assertEqual(1, self.evaluate(result))
       self.assertEqual([2], side_counter)
 
 
diff --git a/tensorflow/python/autograph/utils/tensor_list_test.py b/tensorflow/python/autograph/utils/tensor_list_test.py
index 697c166eb1..c655f773b0 100644
--- a/tensorflow/python/autograph/utils/tensor_list_test.py
+++ b/tensorflow/python/autograph/utils/tensor_list_test.py
@@ -43,13 +43,13 @@ class TensorListTest(test.TestCase):
     l = tl.dynamic_list_append(l, 1)
     s = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(s), [1])
+      self.assertAllEqual(self.evaluate(s), [1])
 
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l = tl.dynamic_list_append(l, 1)
     s = l.stack()
     with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(s), [1])
+      self.assertAllEqual(self.evaluate(s), [1])
 
     l = tl.TensorList(self._shape(()), dtypes.int32)
     l = tl.dynamic_list_append(l, 1)
@@ -92,7 +92,7 @@ class TensorListTest(test.TestCase):
     a2 = l.pop()
     c4 = l.count()
     with Session() as sess:
-      c1, c2, c3, c4, a, a2 = sess.run([c1, c2, c3, c4, a, a2])
+      c1, c2, c3, c4, a, a2 = self.evaluate([c1, c2, c3, c4, a, a2])
       self.assertEqual(c1, 1)
       self.assertEqual(c2, 2)
       self.assertEqual(c3, 1)
@@ -108,7 +108,7 @@ class TensorListTest(test.TestCase):
     l[0] = b
     l1 = l[0]
     with self.cached_session() as sess:
-      l0, l1, a, b = sess.run([l0, l1, a, b])
+      l0, l1, a, b = self.evaluate([l0, l1, a, b])
       self.assertEqual(l0, a)
       self.assertEqual(l1, b)
 
diff --git a/tensorflow/python/client/session_clusterspec_prop_test.py b/tensorflow/python/client/session_clusterspec_prop_test.py
index df020f88a8..224f880ed1 100644
--- a/tensorflow/python/client/session_clusterspec_prop_test.py
+++ b/tensorflow/python/client/session_clusterspec_prop_test.py
@@ -62,7 +62,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
 
     const = constant_op.constant(17)
     sess = session.Session(server1.target, config=config)
-    output = sess.run(const)
+    output = self.evaluate(const)
     self.assertEqual(17, output)
 
   def testClusterSpecPropagationWorker2Placement(self):
@@ -106,7 +106,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
     with ops.Graph().as_default() as g, ops.device('/job:worker/task:0'):
       const = constant_op.constant(17)
     sess = session.Session(server1.target, config=config, graph=g)
-    output = sess.run(const)
+    output = self.evaluate(const)
     self.assertEqual(17, output)
 
   def testCanonicalDeviceNames(self):
@@ -208,7 +208,7 @@ class SessionClusterSpecPropagationTest(test_util.TensorFlowTestCase):
       with ops.device('/job:worker/task:0/cpu:0'):
         sum3 = sum1 + sum2
     sess = session.Session(server1.target, config=config, graph=g)
-    output = sess.run(sum3)
+    output = self.evaluate(sum3)
     self.assertEqual(40, output)
 
   def testLegacyDeviceNames(self):
diff --git a/tensorflow/python/client/session_partial_run_test.py b/tensorflow/python/client/session_partial_run_test.py
index 92ca47efa9..a9bd5ab7e0 100644
--- a/tensorflow/python/client/session_partial_run_test.py
+++ b/tensorflow/python/client/session_partial_run_test.py
@@ -117,7 +117,7 @@ class PartialRunTest(test_util.TensorFlowTestCase):
     a = constant_op.constant(2.0, dtypes.float32)
     b = a * 2
     c = b * 3
-    r1 = sess.run([b, c])
+    r1 = self.evaluate([b, c])
     h = sess.partial_run_setup([b, c], [])
     r2 = sess.partial_run(h, [b, c])
     self.assertEqual(r1, r2)
diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index dfd0147643..f9bd50957a 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -147,7 +147,7 @@ class TimelineTest(test.TestCase):
         num2 = variables.Variable(2.0, name='num2')
       with ops.device('/cpu:2'):
         result = num1 + num2 + num1 * num2
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(result, options=run_options, run_metadata=run_metadata)
 
     self.assertTrue(run_metadata.HasField('step_stats'))
@@ -176,7 +176,7 @@ class TimelineTest(test.TestCase):
         num2 = variables.Variable(2.0, name='num2')
       with ops.device('/cpu:2'):
         result = num1 + num2 + num1 * num2
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(result, options=run_options, run_metadata=run_metadata)
     self.assertTrue(run_metadata.HasField('step_stats'))
     step_stats = run_metadata.step_stats
diff --git a/tensorflow/python/client/virtual_gpu_test.py b/tensorflow/python/client/virtual_gpu_test.py
index 5892e0fc84..e82ee0666c 100644
--- a/tensorflow/python/client/virtual_gpu_test.py
+++ b/tensorflow/python/client/virtual_gpu_test.py
@@ -216,7 +216,7 @@ class VirtualGpuTest(test_util.TensorFlowTestCase):
       for d in self._util.devices:
         with ops.device(d):
           var = variables.Variable(random_ops.random_uniform(mat_shape))
-          sess.run(var.initializer)
+          self.evaluate(var.initializer)
           data.append(var)
       s = data[0]
       for i in range(1, len(data)):
diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 3903ec49b9..af20e50fb9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -110,9 +110,9 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       with self.cached_session() as sess:
         batches = []
         for _ in range(4):
-          batches.append(sess.run(batch))
+          batches.append(self.evaluate(batch))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(batch)
+          self.evaluate(batch)
       batch_sizes_val = []
       lengths_val = []
       for batch in batches:
@@ -160,9 +160,9 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       batches = []
       for _ in range(3):
-        batches.append(sess.run(batch))
+        batches.append(self.evaluate(batch))
       with self.assertRaisesOpError("bucket_boundaries"):
-        sess.run(batch)
+        self.evaluate(batch)
     batch_sizes_val = []
     lengths_val = []
     for batch in batches:
@@ -197,9 +197,9 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       batches = []
       for _ in range(5):
-        batches.append(sess.run(batch))
+        batches.append(self.evaluate(batch))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(batch)
+        self.evaluate(batch)
 
     self.assertAllEqual(batches[0], [[1, 0],
                                      [1, 1]])
@@ -300,7 +300,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       with self.cached_session() as sess:
         with self.assertRaises(errors.OutOfRangeError):
           while True:
-            output = sess.run(batch)
+            output = self.evaluate(batch)
             sprs_tensor = (tuple([tuple(idx) for idx in output.indices]),
                            tuple(output.values))
             all_sparse_tensors.add(sprs_tensor)
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index cea8bd6f0b..7edaab81f4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -57,9 +57,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceInt32(self):
     host_dataset = dataset_ops.Dataset.from_tensors([0, 1, 2, 3])
@@ -82,9 +82,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToSameDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
@@ -108,9 +108,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceWithPrefetch(self):
     host_dataset = dataset_ops.Dataset.range(10)
@@ -134,9 +134,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyDictToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
@@ -160,9 +160,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, sess.run(next_element))
+        self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyDictToDeviceWithPrefetch(self):
     host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
@@ -186,9 +186,9 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, sess.run(next_element))
+        self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopySparseTensorsToDevice(self):
 
@@ -217,12 +217,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = sess.run(next_element)
+        actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopySparseTensorsToDeviceWithPrefetch(self):
 
@@ -251,12 +251,12 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = sess.run(next_element)
+        actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpu(self):
     if not test_util.is_gpu_available():
@@ -271,11 +271,11 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuWithPrefetch(self):
     if not test_util.is_gpu_available():
@@ -290,11 +290,11 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuWithMap(self):
     if not test_util.is_gpu_available():
@@ -323,14 +323,14 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        x, y, z = sess.run(next_element)
+        x, y, z = self.evaluate(next_element)
         self.assertEqual(i**2, x)
         self.assertEqual(float(i**2), y)
         self.assertEqual(util_compat.as_bytes(str(i)), z)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuInt32(self):
     if not test_util.is_gpu_available():
@@ -345,10 +345,10 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuInt32AndPrefetch(self):
     if not test_util.is_gpu_available():
@@ -363,10 +363,10 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([0, 1, 2, 3], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuStrings(self):
     if not test_util.is_gpu_available():
@@ -381,10 +381,10 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuStringsAndPrefetch(self):
     if not test_util.is_gpu_available():
@@ -399,10 +399,10 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
-      self.assertAllEqual([b"a", b"b", b"c"], sess.run(next_element))
+      self.evaluate(iterator.initializer)
+      self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDevicePingPongCPUGPU(self):
     if not test_util.is_gpu_available():
@@ -420,11 +420,11 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         next_element = iterator.get_next()
 
       with self.cached_session() as sess:
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         for i in range(10):
-          self.assertEqual(i, sess.run(next_element))
+          self.assertEqual(i, self.evaluate(next_element))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(next_element)
+          self.evaluate(next_element)
 
   def testCopyToDeviceWithReInit(self):
     host_dataset = dataset_ops.Dataset.range(10)
@@ -447,14 +447,14 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceWithReInitAndPrefetch(self):
     host_dataset = dataset_ops.Dataset.range(10)
@@ -477,14 +477,14 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuWithReInit(self):
     if not test_util.is_gpu_available():
@@ -499,14 +499,14 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testCopyToDeviceGpuWithReInitAndPrefetch(self):
     if not test_util.is_gpu_available():
@@ -521,14 +521,14 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testIteratorGetNextAsOptionalOnGPU(self):
     if not test_util.is_gpu_available():
@@ -547,24 +547,25 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       # Before initializing the iterator, evaluating the optional fails with
       # a FailedPreconditionError.
       with self.assertRaises(errors.FailedPreconditionError):
-        sess.run(elem_has_value_t)
+        self.evaluate(elem_has_value_t)
       with self.assertRaises(errors.FailedPreconditionError):
-        sess.run(elem_value_t)
+        self.evaluate(elem_value_t)
 
       # For each element of the dataset, assert that the optional evaluates to
       # the expected value.
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(3):
-        elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t])
+        elem_has_value, elem_value = self.evaluate(
+            [elem_has_value_t, elem_value_t])
         self.assertTrue(elem_has_value)
         self.assertEqual(i, elem_value)
 
       # After exhausting the iterator, `next_elem.has_value()` will evaluate to
       # false, and attempting to get the value will fail.
       for _ in range(2):
-        self.assertFalse(sess.run(elem_has_value_t))
+        self.assertFalse(self.evaluate(elem_has_value_t))
         with self.assertRaises(errors.InvalidArgumentError):
-          sess.run(elem_value_t)
+          self.evaluate(elem_value_t)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/counter_test.py b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
index 4e114ac479..d1dd07a879 100644
--- a/tensorflow/python/data/experimental/kernel_tests/counter_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
@@ -38,13 +38,13 @@ class CounterTest(test_base.DatasetTestBase):
     negative_get_next = negative_iterator.get_next()
 
     with self.cached_session() as sess:
-      self.assertEqual(3, sess.run(get_next))
-      self.assertEqual(3 + 4, sess.run(get_next))
-      self.assertEqual(3 + 2 * 4, sess.run(get_next))
+      self.assertEqual(3, self.evaluate(get_next))
+      self.assertEqual(3 + 4, self.evaluate(get_next))
+      self.assertEqual(3 + 2 * 4, self.evaluate(get_next))
 
-      self.assertEqual(0, sess.run(negative_get_next))
-      self.assertEqual(-1, sess.run(negative_get_next))
-      self.assertEqual(-2, sess.run(negative_get_next))
+      self.assertEqual(0, self.evaluate(negative_get_next))
+      self.assertEqual(-1, self.evaluate(negative_get_next))
+      self.assertEqual(-2, self.evaluate(negative_get_next))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
index 73be6cbcca..d9bbfb9c99 100644
--- a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
@@ -41,10 +41,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       for start in range(0, len(components), 4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual([[i, j]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)], results.indices)
@@ -56,7 +56,7 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
                             results.dense_shape)
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testDenseToSparseBatchDatasetWithUnknownShape(self):
     components = np.random.randint(5, size=(40,)).astype(np.int32)
@@ -69,10 +69,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       for start in range(0, len(components), 4):
-        results = sess.run(get_next)
+        results = self.evaluate(get_next)
         self.assertAllEqual([[i, j, z]
                              for i, c in enumerate(components[start:start + 4])
                              for j in range(c)
@@ -89,7 +89,7 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
         ], results.dense_shape)
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testDenseToSparseBatchDatasetWithInvalidShape(self):
     input_tensor = array_ops.constant([[1]])
@@ -111,13 +111,13 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
       sess.run(init_op, feed_dict={input_tensor: [[1]]})
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "incompatible with the row shape"):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
       # Initialize with an input tensor that is larger than `row_shape`.
       sess.run(init_op, feed_dict={input_tensor: range(13)})
       with self.assertRaisesRegexp(errors.DataLossError,
                                    "larger than the row shape"):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
index 796a692c56..768a8d774b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
@@ -40,12 +40,12 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for _ in range(100):
         for i in range(10):
-          self.assertEqual(i, sess.run(next_element))
+          self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def _normalize(self, vec):
     return vec / vec.sum()
@@ -71,9 +71,9 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       freqs = np.zeros([num_datasets])
       for _ in range(num_samples):
-        freqs[sess.run(next_element)] += 1
+        freqs[self.evaluate(next_element)] += 1
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
     return freqs
 
@@ -107,9 +107,9 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in choice_array:
-        self.assertEqual(words[i], sess.run(next_element))
+        self.assertEqual(words[i], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testErrors(self):
     with self.assertRaisesRegexp(ValueError,
diff --git a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
index e54235d9f8..f32d1d0a6f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
@@ -44,12 +44,12 @@ class EnumerateDatasetTest(test_base.DatasetTestBase):
                      [t.shape for t in get_next[1]])
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertEqual((20, (b"a", 1, 37.0)), sess.run(get_next))
-      self.assertEqual((21, (b"b", 2, 38.0)), sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertEqual((20, (b"a", 1, 37.0)), self.evaluate(get_next))
+      self.assertEqual((21, (b"b", 2, 38.0)), self.evaluate(get_next))
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py
index c6ee88c676..4f8cb1246f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py
@@ -52,12 +52,12 @@ class FilterBenchmark(test.Benchmark):
 
       with session.Session() as sess:
         for _ in range(10):
-          sess.run(next_element.op)
+          self.evaluate(next_element.op)
         deltas = []
         for _ in range(100):
           start = time.time()
           for _ in range(100):
-            sess.run(next_element.op)
+            self.evaluate(next_element.op)
           end = time.time()
           deltas.append(end - start)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
index d38452e265..860442571e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
@@ -94,18 +94,18 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
                                                   device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [5.0])
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
   def testSameDeviceCPU(self):
     self._prefetch_fn_helper_one_shot("same_device_cpu",
@@ -135,35 +135,35 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
         ds, ds_iterator, "reinit", device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      sess.run(ds_iterator.initializer)
-      elem = sess.run(prefetch_op)
+      self.evaluate(ds_iterator.initializer)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [5.0])
       # Lets reset the function buffering resource and reinitialize the
       # iterator. Should be able to go through this again.
       self._event.clear()
-      sess.run(reset_op)
-      sess.run(ds_iterator.initializer)
-      elem = sess.run(prefetch_op)
+      self.evaluate(reset_op)
+      self.evaluate(ds_iterator.initializer)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [1.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [2.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [3.0])
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [4.0])
       self._event.wait()
-      elem = sess.run(prefetch_op)
+      elem = self.evaluate(prefetch_op)
       self.assertEqual(elem, [5.0])
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
   def testReinitializationOutOfRange(self):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
@@ -175,30 +175,30 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
         ds, ds_iterator, "reinit", device0, device1)
 
     with self.test_session(config=worker_config) as sess:
-      sess.run(ds_iterator.initializer)
+      self.evaluate(ds_iterator.initializer)
       for i in range(1, 10):
-        elem = sess.run(prefetch_op)
+        elem = self.evaluate(prefetch_op)
         self.assertEqual(elem, [float(i)])
       # Try fetching after its over twice to test out end of sequence.
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
 
       # Now reset everything and try it out again.
       self._event.clear()
-      sess.run(reset_op)
-      sess.run(ds_iterator.initializer)
+      self.evaluate(reset_op)
+      self.evaluate(ds_iterator.initializer)
       for i in range(1, 10):
-        elem = sess.run(prefetch_op)
+        elem = self.evaluate(prefetch_op)
         self.assertEqual(elem, [float(i)])
       # Try fetching after its over twice to test out end of sequence.
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
 
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
   def testStringsGPU(self):
     if not test_util.is_gpu_available():
@@ -235,13 +235,13 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
           buffer_resource_handle, ignore_lookup_error=True)
 
     with self.cached_session() as sess:
-      self.assertEqual([b"a"], sess.run(prefetch_op))
-      self.assertEqual([b"b"], sess.run(prefetch_op))
-      self.assertEqual([b"c"], sess.run(prefetch_op))
+      self.assertEqual([b"a"], self.evaluate(prefetch_op))
+      self.assertEqual([b"b"], self.evaluate(prefetch_op))
+      self.assertEqual([b"c"], self.evaluate(prefetch_op))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(prefetch_op)
+        self.evaluate(prefetch_op)
 
-      sess.run(destroy_op)
+      self.evaluate(destroy_op)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
index 9030328593..f9856500c5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
@@ -39,10 +39,10 @@ class GroupByReducerTest(test_base.DatasetTestBase):
     get_next = dataset.make_one_shot_iterator().get_next()
     with self.cached_session() as sess:
       for expected in values:
-        got = sess.run(get_next)
+        got = self.evaluate(get_next)
         self.assertEqual(got, expected)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testSum(self):
     reducer = grouping.Reducer(
@@ -127,11 +127,11 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       iterator = dataset.make_one_shot_iterator()
       get_next = iterator.get_next()
       with self.cached_session() as sess:
-        x, y = sess.run(get_next)
+        x, y = self.evaluate(get_next)
         self.assertAllEqual([0] * (2**i), x)
         self.assertAllEqual(np.array(1, ndmin=i), y)
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
+          self.evaluate(get_next)
 
   def testTypeMismatch(self):
     reducer = grouping.Reducer(
@@ -190,7 +190,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
             grouping.group_by_reducer(lambda x, y: np.int64(0), reducer))
     get_next = dataset.make_one_shot_iterator().get_next()
     with self.cached_session() as sess:
-      x, y = sess.run(get_next)
+      x, y = self.evaluate(get_next)
       self.assertAllEqual(x, np.asarray([x for x in range(10)]))
       self.assertEqual(y, 45)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
index 557d56e8b9..d5a36e7cb1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
@@ -68,9 +68,9 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
-      which_bucket, bucketed_values = sess.run(get_next)
+      which_bucket, bucketed_values = self.evaluate(get_next)
 
       self.assertEqual(0, which_bucket)
 
@@ -103,11 +103,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       # Get two minibatches (one containing even values, one containing odds)
-      which_bucket_even, bucketed_values_even = sess.run(get_next)
-      which_bucket_odd, bucketed_values_odd = sess.run(get_next)
+      which_bucket_even, bucketed_values_even = self.evaluate(get_next)
+      which_bucket_odd, bucketed_values_odd = self.evaluate(get_next)
 
       # Count number of bucket_tensors.
       self.assertEqual(3, len(bucketed_values_even))
@@ -174,11 +174,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
 
       # Get two minibatches ([0, 2, ...] and [64, 66, ...])
-      which_bucket0, bucketed_values_even0 = sess.run(get_next)
-      which_bucket1, bucketed_values_even1 = sess.run(get_next)
+      which_bucket0, bucketed_values_even0 = self.evaluate(get_next)
+      which_bucket1, bucketed_values_even1 = self.evaluate(get_next)
 
       # Ensure that bucket 1 was completely filtered out
       self.assertAllEqual(0, which_bucket0)
@@ -207,11 +207,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaises(errors.OutOfRangeError):
         batches = 0
         while True:
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           is_even = all(x % 2 == 0 for x in result)
           is_odd = all(x % 2 == 1 for x in result)
           self.assertTrue(is_even or is_odd)
@@ -232,11 +232,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       counts = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-          result = sess.run(get_next)
+          result = self.evaluate(get_next)
           self.assertTrue(
               all(x % 2 == 0
                   for x in result) or all(x % 2 == 1)
@@ -259,16 +259,16 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       # The input is infinite, so this test demonstrates that:
       # 1. We produce output without having to consume the entire input,
       # 2. Different buckets can produce output at different rates, and
       # 3. For deterministic input, the output is deterministic.
       for _ in range(3):
-        self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
-        self.assertAllEqual([1, 1, 1, 1], sess.run(get_next))
-        self.assertAllEqual([2, 2, 2, 2], sess.run(get_next))
-        self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
+        self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
+        self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next))
+        self.assertAllEqual([2, 2, 2, 2], self.evaluate(get_next))
+        self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
 
   def testSmallGroups(self):
     components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64)
@@ -280,13 +280,13 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual([0, 0, 0, 0], sess.run(get_next))
-      self.assertAllEqual([1, 1, 1, 1], sess.run(get_next))
+      self.evaluate(init_op)
+      self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
+      self.assertAllEqual([1, 1, 1, 1], self.evaluate(get_next))
       # The small outputs at the end are deterministically produced in key
       # order.
-      self.assertAllEqual([0, 0, 0], sess.run(get_next))
-      self.assertAllEqual([1], sess.run(get_next))
+      self.assertAllEqual([0, 0, 0], self.evaluate(get_next))
+      self.assertAllEqual([1], self.evaluate(get_next))
 
   def testEmpty(self):
     iterator = (
@@ -297,11 +297,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           "Window size must be greater than zero, but got 0."):
-        print(sess.run(get_next))
+        print(self.evaluate(get_next))
 
   def testReduceFuncError(self):
     components = np.random.randint(100, size=(200,)).astype(np.int64)
@@ -323,9 +323,9 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testConsumeWindowDatasetMoreThanOnce(self):
     components = np.random.randint(50, size=(200,)).astype(np.int64)
@@ -351,11 +351,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       counts = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-          tight_result, multiple_of_10_result = sess.run(get_next)
+          tight_result, multiple_of_10_result = self.evaluate(get_next)
           self.assertEqual(0, multiple_of_10_result.shape[1] % 10)
           self.assertAllEqual(tight_result,
                               multiple_of_10_result[:, :tight_result.shape[1]])
diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
index c0ec1486ab..522b196060 100644
--- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
@@ -47,11 +47,11 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for x in [1., 2., 3., 5.]:
-        self.assertEqual(x, sess.run(get_next))
+        self.assertEqual(x, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testParallelMapIgnoreError(self):
     components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32)
@@ -65,11 +65,11 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for x in [1., 2., 3., 5.]:
-        self.assertEqual(x, sess.run(get_next))
+        self.assertEqual(x, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testReadFileIgnoreError(self):
 
@@ -93,22 +93,22 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       # All of the files are present.
-      sess.run(init_op)
+      self.evaluate(init_op)
       for filename in filenames:
-        self.assertEqual(compat.as_bytes(filename), sess.run(get_next))
+        self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
       # Delete one of the files.
       os.remove(filenames[0])
 
       # Attempting to read filenames[0] will fail, but ignore_errors()
       # will catch the error.
-      sess.run(init_op)
+      self.evaluate(init_op)
       for filename in filenames[1:]:
-        self.assertEqual(compat.as_bytes(filename), sess.run(get_next))
+        self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
index c93a8353ce..0a436034a8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
@@ -46,14 +46,14 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
         handle, index, output_types=[dtypes.uint64], output_shapes=[[]])
 
     with self.cached_session() as sess:
-      sess.run(materialize)
+      self.evaluate(materialize)
       self.assertEqual([3], sess.run(get_op, feed_dict={index: 3}))
 
   def testIdentityIndexedDataset(self):
     ds = indexed_dataset_ops.IdentityIndexedDataset(16)
     materialized = ds.materialize()
     with self.cached_session() as sess:
-      sess.run(materialized.initializer)
+      self.evaluate(materialized.initializer)
       placeholder = array_ops.placeholder(dtypes.uint64, shape=[])
       for i in range(16):
         output = sess.run(
@@ -68,12 +68,13 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
     itr = ds.make_initializable_iterator()
     n = itr.get_next()
     with self.cached_session() as sess:
-      sess.run(itr.initializer)
+      self.evaluate(itr.initializer)
       for i in range(16):
-        output = sess.run(n)
+        output = self.evaluate(n)
         self.assertEqual(i, output)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(n)
+        self.evaluate(n)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
index 91ae8cb1bd..109b3696b8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
@@ -112,14 +112,14 @@ class MakeBatchedFeaturesDatasetTest(
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for file_batch, _, _, _, record_batch, _ in self._next_expected_batch(
           range(self._num_files), 2, 10):
-        actual_batch = sess.run(next_element)
+        actual_batch = self.evaluate(next_element)
         self.assertAllEqual(file_batch, actual_batch["file"])
         self.assertAllEqual(record_batch, actual_batch["record"])
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testReadWithFusedShuffleRepeatDataset(self):
     num_epochs = 5
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index e4bf089184..1f509384d7 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -90,7 +90,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         batch_size,
         num_epochs,
     ):
-      actual_features = sess.run(nxt)
+      actual_features = self.evaluate(nxt)
 
       if label_name is not None:
         expected_labels = expected_features.pop(label_name)
@@ -102,7 +102,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         self.assertAllEqual(expected_features[k], actual_features[k])
 
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(nxt)
+      self.evaluate(nxt)
 
   def _test_dataset(self,
                     inputs,
@@ -607,8 +607,8 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           outputs1 = dataset1.make_one_shot_iterator().get_next()
           outputs2 = dataset2.make_one_shot_iterator().get_next()
           for _ in range(total_records // batch_size):
-            batch1 = nest.flatten(sess.run(outputs1))
-            batch2 = nest.flatten(sess.run(outputs2))
+            batch1 = nest.flatten(self.evaluate(outputs1))
+            batch2 = nest.flatten(self.evaluate(outputs2))
             for i in range(len(batch1)):
               self.assertAllEqual(batch1[i], batch2[i])
 
@@ -639,8 +639,8 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           outputs2 = dataset2.make_one_shot_iterator().get_next()
           all_equal = False
           for _ in range(total_records // batch_size):
-            batch1 = nest.flatten(sess.run(outputs1))
-            batch2 = nest.flatten(sess.run(outputs2))
+            batch1 = nest.flatten(self.evaluate(outputs1))
+            batch2 = nest.flatten(self.evaluate(outputs2))
             for i in range(len(batch1)):
               all_equal = all_equal and np.array_equal(batch1[i], batch2[i])
           self.assertFalse(all_equal)
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
index 657cf3c00e..0bb7b7c5f3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
@@ -105,7 +105,7 @@ class MakeTFRecordDatasetTest(
     for expected_batch in self._next_expected_batch(
         file_indices, batch_size, num_epochs, interleave_cycle_length,
         drop_final_batch, use_parser_fn):
-      actual_batch = sess.run(outputs)
+      actual_batch = self.evaluate(outputs)
       self.assertAllEqual(expected_batch, actual_batch)
 
   def _read_test(self, batch_size, num_epochs, file_index=None,
@@ -135,7 +135,7 @@ class MakeTFRecordDatasetTest(
             interleave_cycle_length=num_parallel_reads,
             drop_final_batch=drop_final_batch, use_parser_fn=parser_fn)
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(outputs)
+          self.evaluate(outputs)
 
   def testRead(self):
     for batch_size in [1, 2]:
@@ -188,19 +188,19 @@ class MakeTFRecordDatasetTest(
         iterator = dataset.make_initializable_iterator()
         next_element = iterator.get_next()
 
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         first_batches = []
         try:
           while True:
-            first_batches.append(sess.run(next_element))
+            first_batches.append(self.evaluate(next_element))
         except errors.OutOfRangeError:
           pass
 
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         second_batches = []
         try:
           while True:
-            second_batches.append(sess.run(next_element))
+            second_batches.append(self.evaluate(next_element))
         except errors.OutOfRangeError:
           pass
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index 5ead6d1c75..8449c0651d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -89,13 +89,13 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       sess.run(init_op, feed_dict={count: 28, batch_size: 14})
       num_batches = (28 * 7) // 14
       for i in range(num_batches):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(14):
             self.assertAllEqual(component[(i * 14 + j) % 7]**2,
                                 result_component[j])
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
       # Batch of a finite input, where the batch_size does not
       # divide the total number of elements.
@@ -104,23 +104,23 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       # We expect (num_batches - 1) full-sized batches.
       num_batches = int(math.ceil((14 * 7) / 8))
       for i in range(num_batches - 1):
-        result = sess.run(get_next)
+        result = self.evaluate(get_next)
         for component, result_component in zip(components, result):
           for j in range(8):
             self.assertAllEqual(component[(i * 8 + j) % 7]**2,
                                 result_component[j])
-      result = sess.run(get_next)
+      result = self.evaluate(get_next)
       for component, result_component in zip(components, result):
         for j in range((14 * 7) % 8):
           self.assertAllEqual(component[((num_batches - 1) * 8 + j) % 7]**2,
                               result_component[j])
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
       # Batch of an empty input should fail straight away.
       sess.run(init_op, feed_dict={count: 0, batch_size: 8})
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
       # Empty batch should be an initialization time error.
       with self.assertRaises(errors.InvalidArgumentError):
@@ -152,12 +152,12 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
       if not drop_remainder:
-        self.assertAllEqual([[64], [81]], sess.run(next_element))
+        self.assertAllEqual([[64], [81]], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -177,11 +177,11 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
+      self.assertAllEqual([[64], [81]], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -201,14 +201,14 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(5):
-        got = sess.run(elements)
+        got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
           expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100))
         self.assertAllEqual(got, expected)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elements)
+        self.evaluate(elements)
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -230,14 +230,14 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       elements.append(iterator.get_next())
     with self.cached_session() as sess:
       for i in range(4):
-        got = sess.run(elements)
+        got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
         expected = []
         for j in range(100):
           expected.append(range(i * 10000 + j * 100, i * 10000 + (j + 1) * 100))
         self.assertAllEqual(got, expected)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elements)
+        self.evaluate(elements)
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -261,9 +261,9 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(2):
-        actual = sess.run(get_next)
+        actual = self.evaluate(get_next)
         expected = sparse_tensor.SparseTensorValue(
             indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
             values=[i * 5, i * 5 + 1, i * 5 + 2, i * 5 + 3, i * 5 + 4],
@@ -271,7 +271,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
         self.assertTrue(sparse_tensor.is_sparse(actual))
         self.assertSparseValuesEqual(actual, expected)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -321,10 +321,10 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "number of elements does not match"):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -354,7 +354,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(3):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   @parameterized.named_parameters(
       ("1", 0, False),
@@ -393,13 +393,14 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(threshold // 10):
-        self.assertAllEqual([i * 10 + j for j in range(10)], sess.run(get_next))
+        self.assertAllEqual([i * 10 + j for j in range(10)],
+                            self.evaluate(get_next))
       if threshold % 10 != 0:
         self.assertAllEqual(
             [threshold // 10 * 10 + j for j in range(threshold % 10)],
-            sess.run(get_next))
+            self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   @parameterized.named_parameters(
       ("1", False, dtypes.bool, False),
@@ -442,7 +443,8 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(10):
-        self.assertAllEqual([element for _ in range(10)], sess.run(get_next))
+        self.assertAllEqual([element for _ in range(10)],
+                            self.evaluate(get_next))
 
   @parameterized.named_parameters(
       ("Identity", None, lambda x: x, None),
@@ -462,7 +464,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       else:
         expected = map_fn(
             sess.run(self.structuredElement(structure, shape=[10])))
-      self.assertAllEqual(expected, sess.run(get_next))
+      self.assertAllEqual(expected, self.evaluate(get_next))
 
   def testShortCircuitCapturedInput(self):
     captured_t = array_ops.placeholder(dtypes.int64, shape=[])
@@ -473,7 +475,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={captured_t: 42})
-      self.assertAllEqual([42] * 10, sess.run(get_next))
+      self.assertAllEqual([42] * 10, self.evaluate(get_next))
 
   @parameterized.named_parameters(
       ("Normal", False),
@@ -501,13 +503,13 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
         print("Case %d" % i)
         if i < 5:
           self.assertAllEqual([i * 10 + j + 1 for j in range(10)],
-                              sess.run(get_next))
+                              self.evaluate(get_next))
         else:
           self.assertAllEqual(
               [((i * 10) + j) * ((i * 10) + j) for j in range(10)],
-              sess.run(get_next))
+              self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
index 11694540fa..6042ca1c63 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_defun_op_test.py
@@ -218,7 +218,7 @@ class MapDefunTest(test_base.DatasetTestBase):
 
   def _assert_op_cancelled(self, sess, map_defun_op):
     with self.assertRaisesRegexp(errors.CancelledError, "was cancelled"):
-      sess.run(map_defun_op)
+      self.evaluate(map_defun_op)
 
   def testMapDefunWithParentCancellation(self):
     # Checks that a cancellation of the parent graph is threaded through to
@@ -260,10 +260,10 @@ class MapDefunBenchmark(test.Benchmark):
     with session.Session() as sess:
       # Warm up the session
       for _ in range(5):
-        sess.run(op)
+        self.evaluate(op)
       start = time.time()
       for _ in range(num_iters):
-        sess.run(op)
+        self.evaluate(op)
       end = time.time()
       mean_us = (end - start) * 1e6 / num_iters
       self.report_benchmark(
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index ea2737c3c7..d3c121491a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -41,9 +41,9 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(get_next))
+      self.assertEqual(0, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 510b197ddf..df26a2c0cd 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -51,7 +51,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # TODO(b/117581999): Add eager coverage for the following tests.
   def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
@@ -64,7 +64,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   def testOptimizationNestedDataset(self):
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index 497c011b92..1dfe854f18 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -55,11 +55,11 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       thread_ids = []
       try:
         while True:
-          thread_ids.append(sess.run(next_element))
+          thread_ids.append(self.evaluate(next_element))
       except errors.OutOfRangeError:
         pass
       self.assertLen(thread_ids, len(set(thread_ids)))
diff --git a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
index 90ac250df7..77f0dc8e81 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
@@ -195,9 +195,9 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
           [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 1):
         self.write_coordination_events[expected_element].set()
         self.assertEqual(expected_element * expected_element,
-                         sess.run(self.next_element))
+                         self.evaluate(self.next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testSingleThreaded(self):
     self._testSingleThreaded()
@@ -235,10 +235,10 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
       for expected_element in self._interleave(
           [[3] * 3, [7] * 7, [4] * 4] * self.repeat_count, 2, 1):
         self.write_coordination_events[expected_element].set()
-        output = sess.run(self.next_element)
+        output = self.evaluate(self.next_element)
         self.assertEqual(expected_element * expected_element, output)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def _testTwoThreadsNoContention(self, sloppy=False):
     # num_threads > 1.
@@ -262,7 +262,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
         self.write_coordination_events[expected_element].set()
         if done_first_event:  # First event starts the worker threads.
           self.read_coordination_events[expected_element].acquire()
-        actual_element = sess.run(self.next_element)
+        actual_element = self.evaluate(self.next_element)
         if not done_first_event:
           self.read_coordination_events[expected_element].acquire()
           done_first_event = True
@@ -270,7 +270,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                          "At index %s: %s expected, got: %s" %
                          (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testTwoThreadsNoContention(self):
     self._testTwoThreadsNoContention()
@@ -309,7 +309,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
         else:
           self.write_coordination_events[expected_element].set()
         time.sleep(0.5)  # Sleep to consistently "avoid" the race condition.
-        actual_element = sess.run(self.next_element)
+        actual_element = self.evaluate(self.next_element)
         if not done_first_event:
           done_first_event = True
           self.assertTrue(
@@ -318,7 +318,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                          "At index %s: %s expected, got: %s" %
                          (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testTwoThreadsNoContentionWithRaces(self):
     self._testTwoThreadsNoContentionWithRaces()
@@ -348,7 +348,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
         self.write_coordination_events[expected_element].set()
         if done_first_event:  # First event starts the worker threads.
           self.read_coordination_events[expected_element].acquire()
-        actual_element = sess.run(self.next_element)
+        actual_element = self.evaluate(self.next_element)
         if not done_first_event:
           done_first_event = True
           self.read_coordination_events[expected_element].acquire()
@@ -356,7 +356,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                          "At index %s: %s expected, got: %s" %
                          (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testTwoThreadsNoContentionBlockLength(self):
     self._testTwoThreadsNoContentionBlockLength()
@@ -396,7 +396,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
         else:
           self.write_coordination_events[expected_element].set()
         time.sleep(0.5)  # Sleep to consistently "avoid" the race condition.
-        actual_element = sess.run(self.next_element)
+        actual_element = self.evaluate(self.next_element)
         if not done_first_event:
           done_first_event = True
           self.assertTrue(
@@ -405,7 +405,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                          "At index %s: %s expected, got: %s" %
                          (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testTwoThreadsNoContentionWithRacesAndBlocking(self):
     self._testTwoThreadsNoContentionWithRacesAndBlocking()
@@ -428,7 +428,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
               self.prefetch_input_elements: 0,
           })
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testEmptyInput(self):
     self._testEmptyInput()
@@ -451,7 +451,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
               self.prefetch_input_elements: 0,
           })
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testNonEmptyInputIntoEmptyOutputs(self):
     self._testNonEmptyInputIntoEmptyOutputs()
@@ -484,7 +484,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
         # presence of finishing iterators.
         if done_first_event and not (sloppy and (i in race_indices)):
           self.read_coordination_events[expected_element].acquire()
-        actual_element = sess.run(self.next_element)
+        actual_element = self.evaluate(self.next_element)
         if not done_first_event or (sloppy and (i in race_indices)):
           done_first_event = True
           self.read_coordination_events[expected_element].acquire()
@@ -520,10 +520,10 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
       ]
       for element in mis_ordering:
         self.write_coordination_events[element].set()
-        self.assertEqual(element * element, sess.run(self.next_element))
+        self.assertEqual(element * element, self.evaluate(self.next_element))
         self.assertTrue(self.read_coordination_events[element].acquire(False))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testBlockLengthWithContentionSloppy(self):
     with self.cached_session() as sess:
@@ -549,7 +549,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
         self.write_coordination_events[expected_element].set()
         if done_first_event:  # First event starts the worker threads.
           self.read_coordination_events[expected_element].acquire()
-        actual_element = sess.run(self.next_element)
+        actual_element = self.evaluate(self.next_element)
         if not done_first_event:
           self.read_coordination_events[expected_element].acquire()
           done_first_event = True
@@ -557,7 +557,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                          "At index %s: %s expected, got: %s" %
                          (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def _testEarlyExit(self, sloppy=False):
     # Exiting without consuming all input should not block
@@ -575,7 +575,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
           })
       for i in range(4, 7):
         self.write_coordination_events[i].set()
-      elem = sess.run(self.next_element)  # Start all workers
+      elem = self.evaluate(self.next_element)  # Start all workers
       # Allow the one successful worker to progress beyond the py_func again.
       elem = int(math.sqrt(elem))
       self.write_coordination_events[elem].set()
@@ -608,7 +608,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       output_values = []
       for _ in range(30):
-        output_values.append(sess.run(iterator.get_next()))
+        output_values.append(self.evaluate(iterator.get_next()))
 
     expected_values = self._interleave(
         [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 2)
@@ -637,13 +637,13 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(init_op)
+      self.evaluate(init_op)
       for i in range(10):
         for j in range(2):
           expected = [i, 0] if j % 2 == 0 else [0, -i]
-          self.assertAllEqual(expected, sess.run(get_next))
+          self.assertAllEqual(expected, self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   def testErrorsInOutputFn(self):
     with self.cached_session() as sess:
@@ -668,15 +668,15 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
           self.error = ValueError()
           self.write_coordination_events[expected_element].set()
           with self.assertRaises(errors.InvalidArgumentError):
-            sess.run(self.next_element)
+            self.evaluate(self.next_element)
         else:
           self.write_coordination_events[expected_element].set()
-          actual_element = sess.run(self.next_element)
+          actual_element = self.evaluate(self.next_element)
           self.assertEqual(expected_element * expected_element, actual_element,
                            "At index %s: %s expected, got: %s" %
                            (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testErrorsInInputFn(self):
 
@@ -720,14 +720,14 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
           self._interleave([[4] * 4, [5], [6] * 6] * self.repeat_count, 2, 1)):
         if expected_element == 5:
           with self.assertRaises(errors.InvalidArgumentError):
-            sess.run(self.next_element)
+            self.evaluate(self.next_element)
         else:
-          actual_element = sess.run(self.next_element)
+          actual_element = self.evaluate(self.next_element)
           self.assertEqual(expected_element, actual_element,
                            "At index %s: %s expected, got: %s" %
                            (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testErrorsInInterleaveFn(self):
 
@@ -769,14 +769,14 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
           self._interleave([[4] * 4, [5], [6] * 6] * self.repeat_count, 2, 1)):
         if expected_element == 5:
           with self.assertRaises(errors.InvalidArgumentError):
-            sess.run(self.next_element)
+            self.evaluate(self.next_element)
         else:
-          actual_element = sess.run(self.next_element)
+          actual_element = self.evaluate(self.next_element)
           self.assertEqual(expected_element, actual_element,
                            "At index %s: %s expected, got: %s" %
                            (i, expected_element, actual_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(self.next_element)
+        self.evaluate(self.next_element)
 
   def testShutdownRace(self):
     dataset = dataset_ops.Dataset.range(20)
@@ -796,10 +796,10 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for _ in range(2):
         elements = []
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         try:
           while True:
-            elements.extend(sess.run(next_element))
+            elements.extend(self.evaluate(next_element))
         except errors.OutOfRangeError:
           pass
         results.append(elements)
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index f73725366c..8fc18e1ccd 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -57,9 +57,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testPrefetchToSameDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
@@ -87,9 +87,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testPrefetchDictToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
@@ -117,9 +117,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        self.assertEqual({"a": i}, sess.run(next_element))
+        self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testPrefetchSparseTensorsToDevice(self):
     def make_tensor(i):
@@ -150,12 +150,12 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
       for i in range(10):
-        actual = sess.run(next_element)
+        actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
         self.assertAllEqual([[0, 0]], actual.indices)
         self.assertAllEqual([2, 2], actual.dense_shape)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testPrefetchToDeviceGpu(self):
     if not test_util.is_gpu_available():
@@ -170,9 +170,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testPrefetchToDeviceWithReInit(self):
     host_dataset = dataset_ops.Dataset.range(10)
@@ -199,14 +199,14 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=worker_config) as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testPrefetchToDeviceGpuWithReInit(self):
     if not test_util.is_gpu_available():
@@ -220,14 +220,14 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(5):
-        self.assertEqual(i, sess.run(next_element))
-      sess.run(iterator.initializer)
+        self.assertEqual(i, self.evaluate(next_element))
+      self.evaluate(iterator.initializer)
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index 0730455431..dc8a7bca27 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -60,9 +60,9 @@ class ScanTest(test_base.DatasetTestBase):
                  feed_dict={start: start_val, step: step_val, take: take_val})
         for expected, _ in zip(
             itertools.count(start_val, step_val), range(take_val)):
-          self.assertEqual(expected, sess.run(next_element))
+          self.assertEqual(expected, self.evaluate(next_element))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(next_element)
+          self.evaluate(next_element)
 
   @test_util.run_in_graph_and_eager_modes
   def testFibonacci(self):
@@ -110,9 +110,9 @@ class ScanTest(test_base.DatasetTestBase):
                  feed_dict={start: start_val, step: step_val, take: take_val})
         for expected, _ in zip(
             itertools.count(start_val, step_val), range(take_val)):
-          self.assertEqual(expected, sess.run(next_element).values[0])
+          self.assertEqual(expected, self.evaluate(next_element).values[0])
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(next_element)
+          self.evaluate(next_element)
 
   def testChangingStateShape(self):
     # Test the fixed-point shape invariant calculations: start with
@@ -136,11 +136,11 @@ class ScanTest(test_base.DatasetTestBase):
 
     with self.cached_session() as sess:
       for i in range(5):
-        (longer_vector_val, larger_rank_val), _ = sess.run(next_element)
+        (longer_vector_val, larger_rank_val), _ = self.evaluate(next_element)
         self.assertAllEqual([0] * (2**i), longer_vector_val)
         self.assertAllEqual(np.array(1, ndmin=i), larger_rank_val)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testIncorrectStateType(self):
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
index ef99d01c73..aeb338dfd5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
@@ -71,36 +71,36 @@ class RangeDatasetSerializationTest(
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, _ = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
 
     with ops.Graph().as_default() as g:
       init_op, get_next, _, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(init_op)
-        sess.run(restore_op)
+        self.evaluate(init_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
+          self.evaluate(get_next)
 
     # Saving and restoring in same session.
     with ops.Graph().as_default() as g:
       init_op, get_next, save_op, restore_op = _build_graph(start, stop)
       with self.session(graph=g) as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(init_op)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(init_op)
         for i in range(start, break_point):
-          self.assertEqual(i, sess.run(get_next))
-        sess.run(save_op)
-        sess.run(restore_op)
+          self.assertEqual(i, self.evaluate(get_next))
+        self.evaluate(save_op)
+        self.evaluate(restore_op)
         for i in range(break_point, stop):
-          self.assertEqual(i, sess.run(get_next))
+          self.assertEqual(i, self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
+          self.evaluate(get_next)
 
   def _build_range_dataset(self, start, stop):
     return dataset_ops.Dataset.range(start, stop)
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
index 88d5c896c9..12fa0989d0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/serialization_integration_test.py
@@ -60,9 +60,9 @@ class SerializationIntegrationTest(test.TestCase):
       init_ops, get_next_ops, saver = self._build_graph(num_pipelines,
                                                         num_outputs)
       with self.session(graph=g) as sess:
-        sess.run(init_ops)
+        self.evaluate(init_ops)
         for _ in range(break_point):
-          output = sess.run(get_next_ops)
+          output = self.evaluate(get_next_ops)
           for i in range(num_pipelines):
             all_outputs[i].append(output[i])
         saver.save(sess, self._ckpt_path())
@@ -73,7 +73,7 @@ class SerializationIntegrationTest(test.TestCase):
       with self.session(graph=g) as sess:
         saver.restore(sess, self._ckpt_path())
         for _ in range(num_outputs - break_point):
-          output = sess.run(get_next_ops)
+          output = self.evaluate(get_next_ops)
           for i in range(num_pipelines):
             all_outputs[i].append(output[i])
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py
index a04f1ddafc..e753a7a15b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/shuffle_dataset_serialization_test.py
@@ -138,9 +138,9 @@ class ShuffleDatasetSerializationTest(
           saver = saver_lib.Saver(allow_empty=True)
           with self.session(graph=g) as sess:
             self._save(sess, saver)
-            expected = [sess.run(get_next_ops) for _ in range(num_outputs)]
+            expected = [self.evaluate(get_next_ops) for _ in range(num_outputs)]
             self._restore(saver, sess)
-            actual = [sess.run(get_next_ops) for _ in range(num_outputs)]
+            actual = [self.evaluate(get_next_ops) for _ in range(num_outputs)]
             self.match(expected, actual)
 
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
index c208963a86..2e8b93feaf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
@@ -38,10 +38,10 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
     outputs = []
     with self.cached_session() as sess:
       for _ in range(num_outputs):
-        outputs.append(sess.run(get_next))
+        outputs.append(self.evaluate(get_next))
       if verify_exhausted:
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
+          self.evaluate(get_next)
     return outputs
 
   def testCorrectOutput(self):
@@ -108,7 +108,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
           shuffle_ops.shuffle_and_repeat(buffer_size=21))
       get_next_op = ds.make_one_shot_iterator().get_next()
       with self.session(graph=g) as sess:
-        sess.run(get_next_op)
+        self.evaluate(get_next_op)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
index bf53acc82a..1a6d5522ef 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
@@ -38,14 +38,14 @@ class SleepTest(test_base.DatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       start_time = time.time()
       for i in range(10):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       end_time = time.time()
       self.assertGreater(end_time - start_time, (10 * sleep_microseconds) / 1e6)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
index a2c1169638..eb66927ee5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test.py
@@ -39,10 +39,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                             "ORDER BY first_name DESC"
             })
         for _ in range(2):  # Dataset is repeated. See setUp.
-          self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next))
-          self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next))
+          self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next))
+          self.assertEqual((b"Jane", b"Moe", b"Hi again!"),
+                           self.evaluate(get_next))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
+          self.evaluate(get_next)
 
   # Test that SqlDataset works on a join query.
   def testReadResultSetJoinQuery(self):
@@ -58,9 +59,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ON students.first_name = people.first_name "
                   "AND students.last_name = people.last_name"
           })
-      self.assertEqual((b"John", b"California", b"Hi!"), sess.run(get_next))
+      self.assertEqual((b"John", b"California", b"Hi!"),
+                       self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that SqlDataset can read a database entry with a null-terminator
   # in the middle of the text and place the entry in a `string` tensor.
@@ -75,10 +77,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, last_name, favorite_nonsense_word "
                   "FROM students ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"n\0nsense"), sess.run(get_next))
-      self.assertEqual((b"Jane", b"Moe", b"nonsense\0"), sess.run(get_next))
+      self.assertEqual((b"John", b"Doe", b"n\0nsense"), self.evaluate(get_next))
+      self.assertEqual((b"Jane", b"Moe", b"nonsense\0"),
+                       self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that SqlDataset works when used on two different queries.
   # Because the output types of the dataset must be determined at graph-creation
@@ -93,21 +96,22 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, last_name, motto FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"Hi!"), sess.run(get_next))
-      self.assertEqual((b"Jane", b"Moe", b"Hi again!"), sess.run(get_next))
+      self.assertEqual((b"John", b"Doe", b"Hi!"), self.evaluate(get_next))
+      self.assertEqual((b"Jane", b"Moe", b"Hi again!"), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
       sess.run(
           init_op,
           feed_dict={
               self.query: "SELECT first_name, last_name, state FROM people "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", b"Doe", b"California"), sess.run(get_next))
+      self.assertEqual((b"John", b"Doe", b"California"),
+                       self.evaluate(get_next))
       self.assertEqual((b"Benjamin", b"Franklin", b"Pennsylvania"),
-                       sess.run(get_next))
+                       self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that an `OutOfRangeError` is raised on the first call to
   # `get_next_str_only` if result set is empty.
@@ -122,7 +126,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "WHERE first_name = 'Nonexistent'"
           })
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that an error is raised when `driver_name` is invalid.
   def testReadResultSetWithInvalidDriverName(self):
@@ -151,7 +155,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ORDER BY first_name DESC"
           })
       with self.assertRaises(errors.UnknownError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that an error is raised when there is a syntax error in `query`.
   def testReadResultSetOfQueryWithSyntaxError(self):
@@ -166,7 +170,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ORDER BY first_name DESC"
           })
       with self.assertRaises(errors.UnknownError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that an error is raised when the number of columns in `query`
   # does not match the length of `output_types`.
@@ -181,7 +185,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that no results are returned when `query` is an insert query rather
   # than a select query. In particular, the error refers to the number of
@@ -199,7 +203,7 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "VALUES ('Foo', 'Bar', 'Baz'), ('Fizz', 'Buzz', 'Fizzbuzz')"
           })
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer from a SQLite database table and
   # place it in an `int8` tensor.
@@ -212,10 +216,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a negative or 0-valued integer from a
   # SQLite database table and place it in an `int8` tensor.
@@ -230,9 +234,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students "
                           "WHERE first_name = 'John' ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0, -2), sess.run(get_next))
+      self.assertEqual((b"John", 0, -2), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a large (positive or negative) integer from
   # a SQLite database table and place it in an `int8` tensor.
@@ -246,11 +250,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT desk_number, favorite_negative_number FROM students "
                   "ORDER BY first_name DESC"
           })
-      self.assertEqual((9, -2), sess.run(get_next))
+      self.assertEqual((9, -2), self.evaluate(get_next))
       # Max and min values of int8
-      self.assertEqual((127, -128), sess.run(get_next))
+      self.assertEqual((127, -128), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer from a SQLite database table and
   # place it in an `int16` tensor.
@@ -263,10 +267,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a negative or 0-valued integer from a
   # SQLite database table and place it in an `int16` tensor.
@@ -281,9 +285,9 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students "
                           "WHERE first_name = 'John' ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0, -2), sess.run(get_next))
+      self.assertEqual((b"John", 0, -2), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a large (positive or negative) integer from
   # a SQLite database table and place it in an `int16` tensor.
@@ -297,11 +301,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "FROM students ORDER BY first_name DESC"
           })
       # Max value of int16
-      self.assertEqual((b"John", 32767), sess.run(get_next))
+      self.assertEqual((b"John", 32767), self.evaluate(get_next))
       # Min value of int16
-      self.assertEqual((b"Jane", -32768), sess.run(get_next))
+      self.assertEqual((b"Jane", -32768), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer from a SQLite database table and
   # place it in an `int32` tensor.
@@ -314,8 +318,8 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
 
   # Test that `SqlDataset` can read a negative or 0-valued integer from a
   # SQLite database table and place it in an `int32` tensor.
@@ -328,10 +332,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, income FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0), sess.run(get_next))
-      self.assertEqual((b"Jane", -20000), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -20000), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a large (positive or negative) integer from
   # a SQLite database table and place it in an `int32` tensor.
@@ -345,11 +349,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Max value of int32
-      self.assertEqual((b"John", 2147483647), sess.run(get_next))
+      self.assertEqual((b"John", 2147483647), self.evaluate(get_next))
       # Min value of int32
-      self.assertEqual((b"Jane", -2147483648), sess.run(get_next))
+      self.assertEqual((b"Jane", -2147483648), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a numeric `varchar` from a SQLite database
   # table and place it in an `int32` tensor.
@@ -362,10 +366,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, school_id FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 123), sess.run(get_next))
-      self.assertEqual((b"Jane", 1000), sess.run(get_next))
+      self.assertEqual((b"John", 123), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 1000), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer from a SQLite database table
   # and place it in an `int64` tensor.
@@ -378,10 +382,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a negative or 0-valued integer from a
   # SQLite database table and place it in an `int64` tensor.
@@ -394,10 +398,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, income FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 0), sess.run(get_next))
-      self.assertEqual((b"Jane", -20000), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
+      self.assertEqual((b"Jane", -20000), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a large (positive or negative) integer from
   # a SQLite database table and place it in an `int64` tensor.
@@ -412,11 +416,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ORDER BY first_name DESC"
           })
       # Max value of int64
-      self.assertEqual((b"John", 9223372036854775807), sess.run(get_next))
+      self.assertEqual((b"John", 9223372036854775807), self.evaluate(get_next))
       # Min value of int64
-      self.assertEqual((b"Jane", -9223372036854775808), sess.run(get_next))
+      self.assertEqual((b"Jane", -9223372036854775808), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer from a SQLite database table and
   # place it in a `uint8` tensor.
@@ -429,10 +433,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read the minimum and maximum uint8 values from a
   # SQLite database table and place them in `uint8` tensors.
@@ -446,11 +450,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Min value of uint8
-      self.assertEqual((b"John", 0), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
       # Max value of uint8
-      self.assertEqual((b"Jane", 255), sess.run(get_next))
+      self.assertEqual((b"Jane", 255), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer from a SQLite database table
   # and place it in a `uint16` tensor.
@@ -463,10 +467,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, desk_number FROM students "
                           "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", 9), sess.run(get_next))
-      self.assertEqual((b"Jane", 127), sess.run(get_next))
+      self.assertEqual((b"John", 9), self.evaluate(get_next))
+      self.assertEqual((b"Jane", 127), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read the minimum and maximum uint16 values from a
   # SQLite database table and place them in `uint16` tensors.
@@ -480,11 +484,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                           "ORDER BY first_name DESC"
           })
       # Min value of uint16
-      self.assertEqual((b"John", 0), sess.run(get_next))
+      self.assertEqual((b"John", 0), self.evaluate(get_next))
       # Max value of uint16
-      self.assertEqual((b"Jane", 65535), sess.run(get_next))
+      self.assertEqual((b"Jane", 65535), self.evaluate(get_next))
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
+      self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a 0-valued and 1-valued integer from a
   # SQLite database table and place them as `True` and `False` respectively
@@ -499,10 +503,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, registration_complete FROM students "
                   "ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", True), sess.run(get_next))
-      self.assertEqual((b"Jane", False), sess.run(get_next))
+      self.assertEqual((b"John", True), self.evaluate(get_next))
+      self.assertEqual((b"Jane", False), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read an integer that is not 0-valued or 1-valued
   # from a SQLite database table and place it as `True` in a `bool` tensor.
@@ -515,10 +519,10 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
               self.query: "SELECT first_name, favorite_medium_sized_number "
                           "FROM students ORDER BY first_name DESC"
           })
-      self.assertEqual((b"John", True), sess.run(get_next))
-      self.assertEqual((b"Jane", True), sess.run(get_next))
+      self.assertEqual((b"John", True), self.evaluate(get_next))
+      self.assertEqual((b"Jane", True), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a float from a SQLite database table
   # and place it in a `float64` tensor.
@@ -533,10 +537,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "SELECT first_name, last_name, victories FROM townspeople "
                   "ORDER BY first_name"
           })
-      self.assertEqual((b"George", b"Washington", 20.0), sess.run(get_next))
-      self.assertEqual((b"John", b"Adams", -19.95), sess.run(get_next))
+      self.assertEqual((b"George", b"Washington", 20.0),
+                       self.evaluate(get_next))
+      self.assertEqual((b"John", b"Adams", -19.95), self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a float from a SQLite database table beyond
   # the precision of 64-bit IEEE, without throwing an error. Test that
@@ -555,13 +560,13 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
       self.assertEqual(
           (b"George", b"Washington",
            1331241.321342132321324589798264627463827647382647382643874),
-          sess.run(get_next))
+          self.evaluate(get_next))
       self.assertEqual(
           (b"John", b"Adams",
            1331241321342132321324589798264627463827647382647382643874.0),
-          sess.run(get_next))
+          self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
   # Test that `SqlDataset` can read a float from a SQLite database table,
   # representing the largest integer representable as a 64-bit IEEE float
@@ -579,11 +584,11 @@ class SqlDatasetTest(sql_dataset_test_base.SqlDatasetTestBase):
                   "ORDER BY first_name"
           })
       self.assertNotEqual((b"George", b"Washington", 9007199254740992.0),
-                          sess.run(get_next))
+                          self.evaluate(get_next))
       self.assertNotEqual((b"John", b"Adams", 9007199254740991.0),
-                          sess.run(get_next))
+                          self.evaluate(get_next))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
+        self.evaluate(get_next)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index d5f265d8a8..e816006933 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -70,18 +70,18 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       expected_sum = 0.0
       for i in range(100):
         self.assertAllEqual(
-            np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = sess.run(summary_t)
+            np.array([i] * i, dtype=np.int64), self.evaluate(next_element))
+        summary_str = self.evaluate(summary_t)
         self._assertSummaryHasCount(summary_str, "bytes_produced", float(i + 1))
         expected_sum += i * 8.0
         self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      summary_str = sess.run(summary_t)
+        self.evaluate(next_element)
+      summary_str = self.evaluate(summary_t)
       self._assertSummaryHasCount(summary_str, "bytes_produced", 100.0)
       self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
 
@@ -95,14 +95,15 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "record_latency", float(i + 1))
+            self.evaluate(summary_t), "record_latency", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0)
+        self.evaluate(next_element)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 100.0)
 
   def testPrefetchBufferUtilization(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -114,11 +115,11 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
         self.assertAllEqual(
-            np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = sess.run(summary_t)
+            np.array([i] * i, dtype=np.int64), self.evaluate(next_element))
+        summary_str = self.evaluate(summary_t)
         self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                     float(i + 1))
         self._assertSummaryContains(summary_str, "Prefetch::buffer_capacity")
@@ -126,8 +127,8 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         self._assertSummaryHasRange(summary_str, "Prefetch::buffer_utilization",
                                     0, 1)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      summary_str = sess.run(summary_t)
+        self.evaluate(next_element)
+      summary_str = self.evaluate(summary_t)
       self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                   100)
 
@@ -141,17 +142,17 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(10):
         self.assertAllEqual(
-            np.array([i] * i, dtype=np.int64), sess.run(next_element))
-        summary_str = sess.run(summary_t)
+            np.array([i] * i, dtype=np.int64), self.evaluate(next_element))
+        summary_str = self.evaluate(summary_t)
         self._assertSummaryHasScalarValue(summary_str,
                                           "Prefetch::buffer_capacity", 0)
         self._assertSummaryHasScalarValue(summary_str, "Prefetch::buffer_size",
                                           0)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testFilteredElementsStats(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -163,20 +164,21 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(34):
-        self.assertEqual(i * 3, sess.run(next_element))
+        self.assertEqual(i * 3, self.evaluate(next_element))
         if i is not 0:
           self._assertSummaryHasScalarValue(
-              sess.run(summary_t), "Filter::dropped_elements", float(i * 2))
+              self.evaluate(summary_t), "Filter::dropped_elements",
+              float(i * 2))
         self._assertSummaryHasScalarValue(
-            sess.run(summary_t), "Filter::filtered_elements", float(i + 1))
+            self.evaluate(summary_t), "Filter::filtered_elements", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
       self._assertSummaryHasScalarValue(
-          sess.run(summary_t), "Filter::dropped_elements", 67.0)
+          self.evaluate(summary_t), "Filter::dropped_elements", 67.0)
       self._assertSummaryHasScalarValue(
-          sess.run(summary_t), "Filter::filtered_elements", 34.0)
+          self.evaluate(summary_t), "Filter::filtered_elements", 34.0)
 
   def testMapBufferUtilization(self, dataset_transformation):
 
@@ -257,15 +259,16 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
     with self.cached_session() as sess:
       for j in range(5):
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         for i in range(100):
-          self.assertEqual(i, sess.run(next_element))
+          self.assertEqual(i, self.evaluate(next_element))
           self._assertSummaryHasCount(
-              sess.run(summary_t), "record_latency", float((j * 100) + i + 1))
+              self.evaluate(summary_t), "record_latency",
+              float((j * 100) + i + 1))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(next_element)
+          self.evaluate(next_element)
         self._assertSummaryHasCount(
-            sess.run(summary_t), "record_latency", (j + 1) * 100.0)
+            self.evaluate(summary_t), "record_latency", (j + 1) * 100.0)
 
   def testNoAggregatorRegistered(self, dataset_transformation):
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -274,11 +277,11 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testMultipleTags(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -291,18 +294,19 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "record_latency", float(i + 1))
+            self.evaluate(summary_t), "record_latency", float(i + 1))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "record_latency_2", float(i + 1))
+            self.evaluate(summary_t), "record_latency_2", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 100.0)
+        self.evaluate(next_element)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 100.0)
       self._assertSummaryHasCount(
-          sess.run(summary_t), "record_latency_2", 100.0)
+          self.evaluate(summary_t), "record_latency_2", 100.0)
 
   def testRepeatedTags(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -315,14 +319,15 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for i in range(100):
-        self.assertEqual(i, sess.run(next_element))
+        self.assertEqual(i, self.evaluate(next_element))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "record_latency", float(2 * (i + 1)))
+            self.evaluate(summary_t), "record_latency", float(2 * (i + 1)))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0)
+        self.evaluate(next_element)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 200.0)
 
   def testMultipleIteratorsSameAggregator(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -335,14 +340,15 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.cached_session() as sess:
-      sess.run([iterator_0.initializer, iterator_1.initializer])
+      self.evaluate([iterator_0.initializer, iterator_1.initializer])
       for i in range(100):
-        self.assertEqual(i * 2, sess.run(next_element))
+        self.assertEqual(i * 2, self.evaluate(next_element))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "record_latency", float(2 * (i + 1)))
+            self.evaluate(summary_t), "record_latency", float(2 * (i + 1)))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-      self._assertSummaryHasCount(sess.run(summary_t), "record_latency", 200.0)
+        self.evaluate(next_element)
+      self._assertSummaryHasCount(
+          self.evaluate(summary_t), "record_latency", 200.0)
 
   def testMultipleDatasetWithPrefixes(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
@@ -358,19 +364,19 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      sess.run([iterator_0.initializer, iterator_1.initializer])
+      self.evaluate([iterator_0.initializer, iterator_1.initializer])
       for i in range(100):
-        self.assertEqual(i * 2, sess.run(next_element))
+        self.assertEqual(i * 2, self.evaluate(next_element))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "dataset1_record_latency", float(i + 1))
+            self.evaluate(summary_t), "dataset1_record_latency", float(i + 1))
         self._assertSummaryHasCount(
-            sess.run(summary_t), "dataset2_record_latency", float(i + 1))
+            self.evaluate(summary_t), "dataset2_record_latency", float(i + 1))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
       self._assertSummaryHasCount(
-          sess.run(summary_t), "dataset1_record_latency", 100.0)
+          self.evaluate(summary_t), "dataset1_record_latency", 100.0)
       self._assertSummaryHasCount(
-          sess.run(summary_t), "dataset2_record_latency", 100.0)
+          self.evaluate(summary_t), "dataset2_record_latency", 100.0)
 
 
 @parameterized.named_parameters(
@@ -417,20 +423,21 @@ class FeatureStatsDatasetTest(
     summary_t = aggregator.get_summary()
 
     with self.test_session() as sess:
-      sess.run(iterator.initializer)
+      self.evaluate(iterator.initializer)
       for _ in range(num_output):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
       self._assertSummaryHasCount(
-          sess.run(summary_t), "record_stats_features", total_records)
+          self.evaluate(summary_t), "record_stats_features", total_records)
       self._assertSummaryHasCount(
-          sess.run(summary_t), "record_stats_feature-values", total_records)
+          self.evaluate(summary_t), "record_stats_feature-values",
+          total_records)
       self._assertSummaryHasSum(
-          sess.run(summary_t), "record_stats_features", total_records * 4)
+          self.evaluate(summary_t), "record_stats_features", total_records * 4)
       self._assertSummaryHasSum(
-          sess.run(summary_t), "record_stats_feature-values",
+          self.evaluate(summary_t), "record_stats_feature-values",
           self._sum_keywords(1) * num_epochs + 3 * total_records)
 
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index f9b800fe67..cb94bb4144 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -47,9 +47,9 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       sess.run(iterator.initializer, feed_dict={placeholder: [0, 1, 2, 3]})
       for i in range(4):
-        self.assertEqual(i, sess.run(next_elem))
+        self.assertEqual(i, self.evaluate(next_elem))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_elem)
+        self.evaluate(next_elem)
 
   def testUnbatchScalarDataset(self):
     data = tuple([math_ops.range(10) for _ in range(3)])
@@ -65,10 +65,10 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i,) * 3, sess.run(op))
+        self.assertEqual((i,) * 3, self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
+        self.evaluate(op)
 
   def testUnbatchDatasetWithStrings(self):
     data = tuple([math_ops.range(10) for _ in range(3)])
@@ -85,10 +85,10 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual((i, compat.as_bytes(str(i)), i), sess.run(op))
+        self.assertEqual((i, compat.as_bytes(str(i)), i), self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
+        self.evaluate(op)
 
   def testUnbatchDatasetWithSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
@@ -104,12 +104,12 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        st_row = sess.run(next_element)
+        st_row = self.evaluate(next_element)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
         self.assertEqual([10], st_row.dense_shape)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testUnbatchDatasetWithDenseAndSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
@@ -125,13 +125,13 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        dense_elem, st_row = sess.run(next_element)
+        dense_elem, st_row = self.evaluate(next_element)
         self.assertEqual(i, dense_elem)
         self.assertEqual([i], st_row.indices)
         self.assertEqual([i], st_row.values)
         self.assertEqual([10], st_row.dense_shape)
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testUnbatchSingleElementTupleDataset(self):
     data = tuple([(math_ops.range(10),) for _ in range(3)])
@@ -147,10 +147,10 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       for i in range(10):
-        self.assertEqual(((i,),) * 3, sess.run(op))
+        self.assertEqual(((i,),) * 3, self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
+        self.evaluate(op)
 
   def testUnbatchMultiElementTupleDataset(self):
     data = tuple([(math_ops.range(10 * i, 10 * i + 10),
@@ -168,10 +168,10 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.cached_session() as sess:
       for i in range(10):
         self.assertEqual(((i, b"hi"), (10 + i, b"hi"), (20 + i, b"hi")),
-                         sess.run(op))
+                         self.evaluate(op))
 
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(op)
+        self.evaluate(op)
 
   def testUnbatchEmpty(self):
     data = dataset_ops.Dataset.from_tensors(
@@ -183,7 +183,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     with self.cached_session() as sess:
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testUnbatchStaticShapeMismatch(self):
     data = dataset_ops.Dataset.from_tensors((np.arange(7), np.arange(8),
@@ -208,7 +208,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
               ph2: np.arange(8).astype(np.int32)
           })
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
       # No 0th dimension (i.e. scalar value) for one component.
       sess.run(
@@ -218,7 +218,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
               ph2: 7
           })
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/unique_test.py b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
index 847cff26b0..91f4bc84e9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unique_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
@@ -49,13 +49,13 @@ class UniqueTest(test_base.DatasetTestBase):
     with self.cached_session() as sess:
       for test_case, expected in test_cases:
         current_test_case = test_case
-        sess.run(iterator.initializer)
+        self.evaluate(iterator.initializer)
         for element in expected:
           if dtype == dtypes.string:
             element = compat.as_bytes(element)
-          self.assertAllEqual(element, sess.run(next_element))
+          self.assertAllEqual(element, self.evaluate(next_element))
         with self.assertRaises(errors.OutOfRangeError):
-          sess.run(next_element)
+          self.evaluate(next_element)
 
   def testSimpleInt(self):
     for dtype in [dtypes.int32, dtypes.int64]:
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 2ca9961585..886c9acc03 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -41,7 +41,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
 
   def testBasic(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -51,13 +51,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
   def testOneOnSameDevice(self):
     with ops.device("/cpu:0"):
@@ -68,13 +68,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
   def testRepeatDevices(self):
     with ops.device("/cpu:0"):
@@ -86,17 +86,17 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 20, 4):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
-        self.assertEqual(i + 2, sess.run(elem_on_3))
-        self.assertEqual(i + 3, sess.run(elem_on_4))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+        self.assertEqual(i + 2, self.evaluate(elem_on_3))
+        self.assertEqual(i + 3, self.evaluate(elem_on_4))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
-        sess.run(elem_on_3)
-        sess.run(elem_on_4)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
+        self.evaluate(elem_on_3)
+        self.evaluate(elem_on_4)
 
   def testNotFullyDivisible(self):
     dataset = dataset_ops.Dataset.range(9)
@@ -106,14 +106,14 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
-      self.assertEqual(8, sess.run(elem_on_1))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
+      self.assertEqual(8, self.evaluate(elem_on_1))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
   def testGetNextAsOptional(self):
     dataset = dataset_ops.Dataset.range(9)
@@ -127,7 +127,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
         elem_on_1_has_value, elem_on_1_value = sess.run(
             [elem_on_1_has_value_t, elem_on_1_t])
@@ -141,12 +141,12 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
           [elem_on_1_has_value_t, elem_on_1_t])
       self.assertTrue(elem_on_1_has_value)
       self.assertEqual(8, elem_on_1_value)
-      self.assertFalse(sess.run(elem_on_1_has_value_t))
-      self.assertFalse(sess.run(elem_on_2_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_1_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_2_has_value_t))
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(elem_on_1_t)
+        self.evaluate(elem_on_1_t)
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(elem_on_2_t)
+        self.evaluate(elem_on_2_t)
 
   def testUneven(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -156,14 +156,14 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i, self.evaluate(elem_on_1))
       for i in range(0, 10, 2):
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
   def testMultipleInitializations(self):
     with ops.device("/cpu:0"):
@@ -180,7 +180,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
     with self.test_session(config=config) as sess:
       for i in range(1000):
         sess.run(init_op, feed_dict={epoch: i})
-        self.assertEqual([(i, 0), (i, 1)], sess.run([elem_on_1, elem_on_2]))
+        self.assertEqual([(i, 0), (i, 1)], self.evaluate([elem_on_1,
+                                                          elem_on_2]))
 
   def testBasicGpu(self):
     if not test_util.is_gpu_available():
@@ -193,13 +194,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
   def testUnevenGpu(self):
     if not test_util.is_gpu_available():
@@ -212,14 +213,14 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i, self.evaluate(elem_on_1))
       for i in range(0, 10, 2):
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
   def testGetNextAsOptionalGpu(self):
     if not test_util.is_gpu_available():
@@ -236,7 +237,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 8, 2):
         elem_on_1_has_value, elem_on_1_value = sess.run(
             [elem_on_1_has_value_t, elem_on_1_t])
@@ -250,12 +251,12 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
           [elem_on_1_has_value_t, elem_on_1_t])
       self.assertTrue(elem_on_1_has_value)
       self.assertEqual(8, elem_on_1_value)
-      self.assertFalse(sess.run(elem_on_1_has_value_t))
-      self.assertFalse(sess.run(elem_on_2_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_1_has_value_t))
+      self.assertFalse(self.evaluate(elem_on_2_has_value_t))
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(elem_on_1_t)
+        self.evaluate(elem_on_1_t)
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(elem_on_2_t)
+        self.evaluate(elem_on_2_t)
 
   def testOptimization(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -273,13 +274,13 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config) as sess:
-      sess.run(multi_device_iterator.initializer)
+      self.evaluate(multi_device_iterator.initializer)
       for i in range(0, 10, 2):
-        self.assertEqual(i, sess.run(elem_on_1))
-        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i, self.evaluate(elem_on_1))
+        self.assertEqual(i + 1, self.evaluate(elem_on_2))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(elem_on_1)
-        sess.run(elem_on_2)
+        self.evaluate(elem_on_1)
+        self.evaluate(elem_on_2)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/util/convert_test.py b/tensorflow/python/data/util/convert_test.py
index 89c3afb296..3058e2b3f6 100644
--- a/tensorflow/python/data/util/convert_test.py
+++ b/tensorflow/python/data/util/convert_test.py
@@ -30,47 +30,52 @@ class ConvertTest(test.TestCase):
 
   def testInteger(self):
     resp = convert.optional_param_to_tensor("foo", 3)
-    with self.cached_session() as sess:
-      self.assertEqual(3, sess.run(resp))
+    self.assertEqual(3, self.evaluate(resp))
 
   def testIntegerDefault(self):
     resp = convert.optional_param_to_tensor("foo", None)
-    with self.cached_session() as sess:
-      self.assertEqual(0, sess.run(resp))
+    self.assertEqual(0, self.evaluate(resp))
 
   def testStringDefault(self):
     resp = convert.optional_param_to_tensor("bar", None, "default",
                                             dtypes.string)
-    with self.cached_session() as sess:
-      self.assertEqual(compat.as_bytes("default"), sess.run(resp))
+    self.assertEqual(compat.as_bytes("default"), self.evaluate(resp))
 
   def testString(self):
     resp = convert.optional_param_to_tensor("bar", "value", "default",
                                             dtypes.string)
-    with self.cached_session() as sess:
-      self.assertEqual(compat.as_bytes("value"), sess.run(resp))
+    self.assertEqual(compat.as_bytes("value"), self.evaluate(resp))
 
   def testPartialShapeToTensorKnownDimension(self):
-    with self.cached_session() as sess:
-      self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor(
-          tensor_shape.TensorShape([1]))))
-      self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor((1,))))
-      self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor([1])))
-      self.assertAllEqual([1], sess.run(convert.partial_shape_to_tensor(
-          constant_op.constant([1], dtype=dtypes.int64))))
+    self.assertAllEqual([1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                tensor_shape.TensorShape([1]))))
+    self.assertAllEqual([1], self.evaluate(
+        convert.partial_shape_to_tensor((1,))))
+    self.assertAllEqual([1], self.evaluate(
+        convert.partial_shape_to_tensor([1])))
+    self.assertAllEqual([1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                constant_op.constant([1], dtype=dtypes.int64))))
 
   def testPartialShapeToTensorUnknownDimension(self):
-    with self.cached_session() as sess:
-      self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor(
-          tensor_shape.TensorShape([None]))))
-      self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor(
-          (None,))))
-      self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor(
-          [None])))
-      self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor(
-          [-1])))
-      self.assertAllEqual([-1], sess.run(convert.partial_shape_to_tensor(
-          constant_op.constant([-1], dtype=dtypes.int64))))
+    self.assertAllEqual([-1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                tensor_shape.TensorShape([None]))))
+    self.assertAllEqual([-1],
+                        self.evaluate(convert.partial_shape_to_tensor((None,))))
+    self.assertAllEqual([-1],
+                        self.evaluate(convert.partial_shape_to_tensor([None])))
+    self.assertAllEqual([-1],
+                        self.evaluate(convert.partial_shape_to_tensor([-1])))
+    self.assertAllEqual([-1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                constant_op.constant([-1],
+                                                     dtype=dtypes.int64))))
 
     with self.assertRaisesRegexp(
         ValueError, r"The given shape .* must be a 1-D tensor of tf.int64 "
@@ -84,42 +89,63 @@ class ConvertTest(test.TestCase):
       convert.partial_shape_to_tensor(constant_op.constant([1., 1.]))
 
   def testPartialShapeToTensorMultipleDimensions(self):
-    with self.cached_session() as sess:
-      self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor(
-          tensor_shape.TensorShape([3, 6]))))
-      self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor(
-          (3, 6))))
-      self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor(
-          [3, 6])))
-      self.assertAllEqual([3, 6], sess.run(convert.partial_shape_to_tensor(
-          constant_op.constant([3, 6], dtype=dtypes.int64))))
-
-      self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor(
-          tensor_shape.TensorShape([3, None]))))
-      self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor(
-          (3, None))))
-      self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor(
-          [3, None])))
-      self.assertAllEqual([3, -1], sess.run(convert.partial_shape_to_tensor(
-          constant_op.constant([3, -1], dtype=dtypes.int64))))
-
-      self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor(
-          tensor_shape.TensorShape([None, None]))))
-      self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor(
-          (None, None))))
-      self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor(
-          [None, None])))
-      self.assertAllEqual([-1, -1], sess.run(convert.partial_shape_to_tensor(
-          constant_op.constant([-1, -1], dtype=dtypes.int64))))
+    self.assertAllEqual([3, 6],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                tensor_shape.TensorShape([3, 6]))))
+    self.assertAllEqual([3, 6],
+                        self.evaluate(convert.partial_shape_to_tensor((3, 6))))
+    self.assertAllEqual([3, 6],
+                        self.evaluate(convert.partial_shape_to_tensor([3, 6])))
+    self.assertAllEqual([3, 6],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                constant_op.constant([3, 6],
+                                                     dtype=dtypes.int64))))
+
+    self.assertAllEqual([3, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                tensor_shape.TensorShape([3, None]))))
+    self.assertAllEqual([3, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor((3, None))))
+    self.assertAllEqual([3, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor([3, None])))
+    self.assertAllEqual([3, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                constant_op.constant([3, -1],
+                                                     dtype=dtypes.int64))))
+
+    self.assertAllEqual([-1, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                tensor_shape.TensorShape([None, None]))))
+    self.assertAllEqual([-1, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor((None, None))))
+    self.assertAllEqual([-1, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor([None, None])))
+    self.assertAllEqual([-1, -1],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                constant_op.constant([-1, -1],
+                                                     dtype=dtypes.int64))))
 
   def testPartialShapeToTensorScalar(self):
-    with self.cached_session() as sess:
-      self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(
-          tensor_shape.TensorShape([]))))
-      self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(())))
-      self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor([])))
-      self.assertAllEqual([], sess.run(convert.partial_shape_to_tensor(
-          constant_op.constant([], dtype=dtypes.int64))))
+    self.assertAllEqual([],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                tensor_shape.TensorShape([]))))
+    self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor(())))
+    self.assertAllEqual([], self.evaluate(convert.partial_shape_to_tensor([])))
+    self.assertAllEqual([],
+                        self.evaluate(
+                            convert.partial_shape_to_tensor(
+                                constant_op.constant([], dtype=dtypes.int64))))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index f197a9e4dc..5aa7d1bb4c 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -1583,7 +1583,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
       x = variables.VariableV1([1, 3, 3, 7], name="x")
       _, idx = array_ops.unique(x, name="x_unique")
       idx_times_two = math_ops.multiply(idx, 2, name="idx_times_two")
-      sess.run(x.initializer)
+      self.evaluate(x.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(
diff --git a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
index 1f67f8a0d4..34030c0adc 100644
--- a/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
+++ b/tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
@@ -126,8 +126,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       u = variables.Variable([12.0], name="u")
       v = variables.Variable([30.0], name="v")
       w = math_ops.add(u, v, name="w")
-      sess.run(u.initializer)
-      sess.run(v.initializer)
+      self.evaluate(u.initializer)
+      self.evaluate(v.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, w, expected_output=[42.0])
@@ -139,7 +139,7 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
         b = math_ops.add(a, a, name="b")
       with ops.control_dependencies([a, b]):
         c = math_ops.multiply(b, b, name="c")
-      sess.run(a.initializer)
+      self.evaluate(a.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, c, expected_output=400.0)
@@ -150,8 +150,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       y = variables.Variable(20.0, name="y")
       cond = control_flow_ops.cond(
           x > y, lambda: math_ops.add(x, 1), lambda: math_ops.add(y, 1))
-      sess.run(x.initializer)
-      sess.run(y.initializer)
+      self.evaluate(x.initializer)
+      self.evaluate(y.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(
           sess, cond, expected_output=21.0)
@@ -173,8 +173,8 @@ class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
       toy_loss = x * (u - v)
       train_op = gradient_descent.GradientDescentOptimizer(
           learning_rate=0.1).minimize(toy_loss, name="train_op")
-      sess.run(u.initializer)
-      sess.run(v.initializer)
+      self.evaluate(u.initializer)
+      self.evaluate(v.initializer)
 
       self._compareOriginalAndReconstructedGraphDefs(sess, train_op)
 
diff --git a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
index b0dc25851c..8eef45392f 100644
--- a/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
+++ b/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
@@ -67,7 +67,7 @@ class SessionDebugMultiGPUTest(test_util.TensorFlowTestCase):
         u1 = math_ops.multiply(v, v, name="u1")
       w = math_ops.subtract(u1, u0, name="w")
 
-      sess.run(v.initializer)
+      self.evaluate(v.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(run_options, sess.graph,
diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py
index 4a8d4eaa99..a16d68329a 100644
--- a/tensorflow/python/debug/lib/source_utils_test.py
+++ b/tensorflow/python/debug/lib/source_utils_test.py
@@ -109,8 +109,8 @@ class SourceHelperTest(test_util.TensorFlowTestCase):
       self.w = math_ops.matmul(self.u, self.v, name="w")
       self.w_line_number = line_number_above()
 
-      sess.run(self.u.initializer)
-      sess.run(self.v.initializer)
+      self.evaluate(self.u.initializer)
+      self.evaluate(self.v.initializer)
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       debug_utils.watch_graph(
diff --git a/tensorflow/python/distribute/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py
index cbb93e8995..2689dbbec8 100644
--- a/tensorflow/python/distribute/input_ops_test.py
+++ b/tensorflow/python/distribute/input_ops_test.py
@@ -92,9 +92,9 @@ class AutoShardDatasetTest(test.TestCase):
     with self.cached_session() as sess:
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(record_fn(r, f), sess.run(next_element))
+          self.assertAllEqual(record_fn(r, f), self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testTFRecordDataset(self):
     dataset = readers.TFRecordDataset(self._createTFRecordFiles())
@@ -138,10 +138,10 @@ class AutoShardDatasetTest(test.TestCase):
       actual, expected = [], []
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          actual.append(sess.run(next_element))
+          actual.append(self.evaluate(next_element))
           expected.append(self._record(r, f))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
       self.assertAllEqual(expected, actual)
 
   def testComplexPipeline(self):
@@ -171,9 +171,9 @@ class AutoShardDatasetTest(test.TestCase):
       num_iterations = (self._num_files * self._num_records * num_epochs) // (
           self._num_shards * batch_size)
       for _ in range(num_iterations):
-        actual.extend(sess.run(next_element))
+        actual.extend(self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
       expected = []
       for f in range(0, self._num_files, self._num_shards):
@@ -205,12 +205,13 @@ class AutoShardDatasetTest(test.TestCase):
     with self.cached_session() as sess:
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(self._record(r, f), sess.run(next_element))
+          self.assertAllEqual(self._record(r, f), self.evaluate(next_element))
       for f in range(self._shard_index, self._num_files, self._num_shards):
         for r in range(self._num_records):
-          self.assertAllEqual(self._text_line(r, f), sess.run(next_element))
+          self.assertAllEqual(
+              self._text_line(r, f), self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
+        self.evaluate(next_element)
 
   def testTextLineReader(self):
     dataset = readers.TextLineDataset(self._createTextFiles())
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index f0f71a219e..54991344b7 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -149,9 +149,9 @@ class DefFunctionTest(test.TestCase):
 
       result = fn(3.0)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual(sess.run(state[0]), 2.0)
-      self.assertAllEqual(sess.run(result), 6.0)
+      self.assertAllEqual(self.evaluate(result), 6.0)
 
   def testLegacyGraphModeVariablesNonTrivialInitializer(self):
     with ops.Graph().as_default(), self.test_session() as sess:
@@ -168,9 +168,9 @@ class DefFunctionTest(test.TestCase):
 
       result = fn(3.0)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual(sess.run(state[0]), 6.0)
-      self.assertAllEqual(sess.run(result), 18.0)
+      self.assertAllEqual(self.evaluate(result), 18.0)
 
   def testLegacyGraphModeInputDependentInitializerFails(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py
index d4f8aaa7e3..1ba596573f 100644
--- a/tensorflow/python/eager/function_gradients_test.py
+++ b/tensorflow/python/eager/function_gradients_test.py
@@ -78,7 +78,7 @@ class FunctionGradientsTest(test.TestCase, parameterized.TestCase):
       c = constant_op.constant([[2.]])
       f_c = f(c)
       g, = gradients_impl.gradients(f_c, c)
-      self.assertAllEqual(sess.run(g).values, [[1.0]])
+      self.assertAllEqual(self.evaluate(g).values, [[1.0]])
 
   def testNoSymGradNestedDefun(self):
 
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index b58b09140d..a206b1f791 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -564,7 +564,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       variables.global_variables_initializer().run()
       call = def_function.function(o.call)
       op = call()
-      self.assertAllEqual(sess.run(op), 2.0)
+      self.assertAllEqual(self.evaluate(op), 2.0)
 
   def testGraphModeManyFunctions(self):
     with ops.Graph().as_default(), self.cached_session():
@@ -1732,7 +1732,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
       function.register(cpu_boost, x)
       y = gpu_boost(x)
-      y_value = sess.run(y)
+      y_value = self.evaluate(y)
 
       if test.is_gpu_available():
         self.assertEqual(y_value, 5.0)
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index e9b11c3960..2c70d66810 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -1027,7 +1027,7 @@ class CrossedColumnTest(test.TestCase):
     outputs = _transform_features(features, [price_cross_wire])
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
-      output_val = sess.run(output)
+      output_val = self.evaluate(output)
       self.assertAllEqual(
           [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
       for val in output_val.values:
@@ -1886,7 +1886,8 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc._numeric_column('price')
@@ -2525,7 +2526,8 @@ class _LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc._numeric_column('price')
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 115763f656..23131e22ed 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -1188,7 +1188,7 @@ class CrossedColumnTest(test.TestCase):
     outputs = fc._transform_features_v2(features, [price_cross_wire], None)
     output = outputs[price_cross_wire]
     with self.cached_session() as sess:
-      output_val = sess.run(output)
+      output_val = self.evaluate(output)
       self.assertAllEqual(
           [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
       for val in output_val.values:
@@ -2088,7 +2088,8 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]],
+                          self.evaluate(net))
 
       coord.request_stop()
       coord.join(threads)
@@ -2124,7 +2125,8 @@ class LinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
@@ -2843,7 +2845,8 @@ class OldLinearModelTest(test.TestCase):
       sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
       sess.run(bias.assign([5.]))
 
-      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          self.evaluate(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
diff --git a/tensorflow/python/framework/file_system_test.py b/tensorflow/python/framework/file_system_test.py
index 6901715e5d..066d34e781 100644
--- a/tensorflow/python/framework/file_system_test.py
+++ b/tensorflow/python/framework/file_system_test.py
@@ -42,7 +42,7 @@ class FileSystemTest(test.TestCase):
       queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
       queue.enqueue_many([["test://foo"]]).run()
       queue.close().run()
-      key, value = sess.run(reader.read(queue))
+      key, value = self.evaluate(reader.read(queue))
     self.assertEqual(key, compat.as_bytes("test://foo"))
     self.assertEqual(value, compat.as_bytes("AAAAAAAAAA"))
 
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 971219d5b0..1a17a48050 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -102,7 +102,7 @@ class FunctionTest(test.TestCase):
       call = MyIdentityFunc([18.0])
       self.assertEqual("MyIdentity", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([18.0], sess.run(call))
+        self.assertAllEqual([18.0], self.evaluate(call))
 
   def testIdentityImplicitDeref(self):
 
@@ -116,8 +116,8 @@ class FunctionTest(test.TestCase):
       self.assertEqual("MyIdentity", call.op.name)
       for cfg in _OptimizerOptions():
         with session.Session(config=cfg) as sess:
-          sess.run(var.initializer)
-          self.assertAllEqual([18.0], sess.run(call))
+          self.evaluate(var.initializer)
+          self.assertAllEqual([18.0], self.evaluate(call))
 
   def testIdentityOutputName(self):
 
@@ -130,7 +130,7 @@ class FunctionTest(test.TestCase):
       call = MyIdentityFunc([18.0])
       self.assertEqual("MyIdentity", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([18.0], sess.run(call))
+        self.assertAllEqual([18.0], self.evaluate(call))
 
   def testTooManyOutputNames(self):
 
@@ -158,7 +158,7 @@ class FunctionTest(test.TestCase):
       call = APlus2B([1.0], [2.0])
       self.assertEqual("APlus2B", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([5.0], sess.run(call))
+        self.assertAllEqual([5.0], self.evaluate(call))
 
   def testFunctionWithNoOutput(self):
 
@@ -187,7 +187,7 @@ class FunctionTest(test.TestCase):
       call = APlus2B([1.0], [2.0])
       self.assertEqual("APlus2B", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([5.0], sess.run(call))
+        self.assertAllEqual([5.0], self.evaluate(call))
 
   def testDefineFunctionDuplicateOutputs(self):
 
@@ -224,8 +224,8 @@ class FunctionTest(test.TestCase):
       call_g = XSquarePlusOneGrad([2.0], [0.1])
 
       with session.Session() as sess:
-        self.assertAllClose([5.0], sess.run(call_f))
-        self.assertAllClose([0.4], sess.run(call_g))
+        self.assertAllClose([5.0], self.evaluate(call_f))
+        self.assertAllClose([0.4], self.evaluate(call_g))
 
   def testTanhSymGrad(self):
 
@@ -365,7 +365,7 @@ class FunctionTest(test.TestCase):
       else:
         dx, dy = gradients_impl.gradients([z], [x, y])
       with session.Session() as sess:
-        dx_val, dy_val = sess.run([dx, dy])
+        dx_val, dy_val = self.evaluate([dx, dy])
         self.assertEqual([2.0], dx_val)
         self.assertEqual([0.0], dy_val)
 
@@ -387,7 +387,7 @@ class FunctionTest(test.TestCase):
       call = AConstant()
       self.assertEqual("AConstant", call.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([42], sess.run(call))
+        self.assertAllEqual([42], self.evaluate(call))
 
   def testDefineFunctionNames(self):
 
@@ -468,7 +468,7 @@ class FunctionTest(test.TestCase):
 
       loop = control_flow_ops.while_loop(lambda x: x < 1e5, Body, [1.0])
 
-      ans = sess.run(loop)
+      ans = self.evaluate(loop)
       self.assertAllClose(ans, 131072.)
 
   def testControlFlowStrictness(self):
@@ -650,8 +650,8 @@ class FunctionTest(test.TestCase):
       # pylint: enable=unexpected-keyword-arg
       self.assertEqual("next", call2.op.name)
       with session.Session() as sess:
-        self.assertAllEqual([1], sess.run(call1))
-        self.assertAllEqual([0], sess.run(call2))
+        self.assertAllEqual([1], self.evaluate(call1))
+        self.assertAllEqual([0], self.evaluate(call2))
 
   def testNestedFunction(self):
 
@@ -794,7 +794,7 @@ class FunctionTest(test.TestCase):
       y = Foo()
 
     with self.session(graph=g) as sess:
-      self.assertEqual(sess.run(y), 10)
+      self.assertEqual(self.evaluate(y), 10)
 
   def testCaptureInCond(self):
     g = ops.Graph()
@@ -809,8 +809,8 @@ class FunctionTest(test.TestCase):
       z = Foo(False)
 
     with self.session(graph=g) as sess:
-      self.assertEqual(sess.run(y), 1)
-      self.assertEqual(sess.run(z), 2)
+      self.assertEqual(self.evaluate(y), 1)
+      self.assertEqual(self.evaluate(z), 2)
 
   def testStableName(self):
 
@@ -854,7 +854,7 @@ class FunctionTest(test.TestCase):
       z = Bar(x)
 
     with self.session(graph=g) as sess:
-      v0, v1 = sess.run([y, z])
+      v0, v1 = self.evaluate([y, z])
       self.assertAllEqual(v0, 20.)
       self.assertAllEqual(v1, 20.)
 
@@ -900,7 +900,7 @@ class FunctionTest(test.TestCase):
     self.assertEqual(global_vars[0].name, "linear/w:0")
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       output_val = sess.run(
           output_op, feed_dict={input_op: np.random.rand(32, 100)})
       self.assertEqual(output_val.shape, (32, 100))
@@ -928,7 +928,7 @@ class FunctionTest(test.TestCase):
     self.assertEqual(global_vars[0].name, "vs1/var:0")
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       out1, out2 = sess.run(
           [out1_op, out2_op], feed_dict={input_op: np.linspace(1, 10, 10)})
       self.assertAllEqual(out1, np.linspace(2, 11, 10))
@@ -991,8 +991,8 @@ class FunctionTest(test.TestCase):
     result_2 = Bar(constant_op.constant(100, dtype=dtypes.int64))
 
     with session.Session() as sess:
-      self.assertEqual(4.0, sess.run(result_1))
-      self.assertEqual(100, sess.run(result_2))
+      self.assertEqual(4.0, self.evaluate(result_1))
+      self.assertEqual(100, self.evaluate(result_2))
       self.assertEqual((4.0, 100), sess.run((result_1, result_2)))
 
   def testStatefulFunction(self):
@@ -1052,8 +1052,8 @@ class FunctionTest(test.TestCase):
     for config in _OptimizerOptions():
       config.device_count["CPU"] = 2
       with session.Session(config=config) as sess:
-        self.assertEqual(42.0, sess.run(f_0))
-        self.assertEqual(44.0, sess.run(f_1))
+        self.assertEqual(42.0, self.evaluate(f_0))
+        self.assertEqual(44.0, self.evaluate(f_1))
         self.assertEqual((42.0, 44.0), sess.run((f_0, f_1)))
 
   def testGuaranteedConstsAreCaptured(self):
@@ -1076,7 +1076,7 @@ class FunctionTest(test.TestCase):
       return output
 
     with self.session(use_gpu=False) as sess:
-      sess.run(var.initializer)
+      self.evaluate(var.initializer)
       _ = sess.run(CapturesGuaranteedConst(), {also_not_const: 1.0})
 
   def testSameFunctionDifferentGrads(self):
@@ -1127,7 +1127,7 @@ class FunctionTest(test.TestCase):
       dx2, = gradients_impl.gradients(ys=[y2], xs=[x2])
 
     with self.session(graph=g) as sess:
-      v0, v1, v2 = sess.run([dx0, dx1, dx2])
+      v0, v1, v2 = self.evaluate([dx0, dx1, dx2])
 
     self.assertAllEqual(v0, 2.)
     self.assertAllEqual(v1, 101.)
@@ -1532,7 +1532,7 @@ class UnrollLSTMTest(test.TestCase):
       tf_logging.info("time: %f txt size: %d gdef bin size: %d", finish - start,
                       len(str(gdef)), len(gdef.SerializeToString()))
       with g.as_default(), session.Session(config=cfg) as sess:
-        return sess.run(m)
+        return self.evaluate(m)
 
     mv0 = RunForward("complete")
     for cfg in _OptimizerOptions():
@@ -1561,7 +1561,7 @@ class UnrollLSTMTest(test.TestCase):
       tf_logging.info("time: %f txt size: %d gdef bin size: %d", finish - start,
                       len(str(gdef)), len(gdef.SerializeToString()))
       with g.as_default(), session.Session(config=cfg) as sess:
-        return sess.run(dw)
+        return self.evaluate(dw)
 
     d0 = RunForwardBackward("complete")
     for cfg in _OptimizerOptions():
@@ -1651,8 +1651,8 @@ class ModuleFunctionTest(test.TestCase):
       y = LinearWithCApi(a, b, c)
       z = Linear2WithCApi(a, b, c, d, e)
       with session.Session() as sess:
-        self.assertAllEqual([[1]], sess.run(y))
-        self.assertAllEqual([[5]], sess.run(z))
+        self.assertAllEqual([[1]], self.evaluate(y))
+        self.assertAllEqual([[5]], self.evaluate(z))
 
 
 class VariableHoistingTest(test.TestCase):
@@ -1704,8 +1704,8 @@ class VariableHoistingTest(test.TestCase):
     self.assertEqual("Foo/b", b.op.name)
 
     with self.session(graph=g) as sess:
-      sess.run(variables.global_variables_initializer())
-      w, b, x, y0, loss, dw, db = sess.run([w, b, x, y0, loss, dw, db])
+      self.evaluate(variables.global_variables_initializer())
+      w, b, x, y0, loss, dw, db = self.evaluate([w, b, x, y0, loss, dw, db])
 
     self.assertAllEqual(w.shape, (64, 64))
     self.assertAllClose(np.sum(w), 2050.44)
diff --git a/tensorflow/python/framework/graph_util_test.py b/tensorflow/python/framework/graph_util_test.py
index 563a177dd0..10a01c71f2 100644
--- a/tensorflow/python/framework/graph_util_test.py
+++ b/tensorflow/python/framework/graph_util_test.py
@@ -210,8 +210,8 @@ class DeviceFunctionsTest(test.TestCase):
 
       with session.Session() as sess:
         init = variables.variables_initializer([variable_node])
-        sess.run(init)
-        output = sess.run(output_node)
+        self.evaluate(init)
+        output = self.evaluate(output_node)
         self.assertNear(4.0, output, 0.00001)
         variable_graph_def = sess.graph.as_graph_def()
 
@@ -242,8 +242,8 @@ class DeviceFunctionsTest(test.TestCase):
         output_node = math_ops_lib.multiply(
             variable_node, 2.0, name="output_node")
         with session.Session() as sess:
-          sess.run(variable_node.initializer)
-          output = sess.run(output_node)
+          self.evaluate(variable_node.initializer)
+          output = self.evaluate(output_node)
           self.assertNear(2.0, output, 0.00001)
           variable_graph_def = sess.graph.as_graph_def()
           # First get the constant_graph_def when variable_names_whitelist is
@@ -256,7 +256,7 @@ class DeviceFunctionsTest(test.TestCase):
 
           # Then initialize the unused variable, and get another
           # constant_graph_def when variable_names_whitelist is not set.
-          sess.run(another_variable.initializer)
+          self.evaluate(another_variable.initializer)
           constant_graph_def_without_variable_whitelist = (
               graph_util.convert_variables_to_constants(
                   sess, variable_graph_def, ["output_node"]))
@@ -295,7 +295,7 @@ class DeviceFunctionsTest(test.TestCase):
             ["Variable", "VariableV2", "VarHandleOp", "ReadVariableOp"])
       with session.Session() as sess:
         output_node = sess.graph.get_tensor_by_name("output_node:0")
-        output = sess.run(output_node)
+        output = self.evaluate(output_node)
         self.assertNear(2.0, output, 0.00001)
 
   def create_node_def(self, op, name, inputs):
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index fc7367649e..66e80b5585 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -397,11 +397,11 @@ class ImportGraphDefTest(test.TestCase):
       # Run the imported graph.
       # TODO(b/76173421): make this work (currently DCHECKS)
       # with self.cached_session() as sess:
-      #   sess.run(imported_init)
-      #   self.assertEqual(sess.run(imported_var), 1.0)
-      #   self.assertEqual(sess.run(imported_assign), 2.0)
-      #   self.assertEqual(list(sess.run(imported_shape)), [])
-      #   self.assertEqual(list(sess.run(new_var_shape)), [])
+      #   self.evaluate(imported_init)
+      #   self.assertEqual(self.evaluate(imported_var), 1.0)
+      #   self.assertEqual(self.evaluate(imported_assign), 2.0)
+      #   self.assertEqual(list(self.evaluate(imported_shape)), [])
+      #   self.assertEqual(list(self.evaluate(new_var_shape)), [])
 
   def testWhileLoop(self):
     # Produce GraphDef containing while loop.
@@ -418,7 +418,7 @@ class ImportGraphDefTest(test.TestCase):
                                               return_elements=[r.name])
       self.assertEqual(imported_r.name, "import/" + r.name)
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(imported_r), 10)
+        self.assertEqual(self.evaluate(imported_r), 10)
 
   def testImportWhileLoopInCond(self):
     # Produce GraphDef containing while loop.
@@ -458,7 +458,7 @@ class ImportGraphDefTest(test.TestCase):
           lambda i: i < 2, ImportFn, [0],
           shape_invariants=[tensor_shape.TensorShape(None)])
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(out), 10)
+        self.assertEqual(self.evaluate(out), 10)
 
   def testTypeMismatchInGraphDef(self):
     # TODO(skyewm): improve error message
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 84e7f361bb..cc93f8b1b8 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -492,8 +492,8 @@ class ScopedMetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
       grad = gradients_impl.gradients([output], [var])
       with session.Session() as sess:
-        sess.run(init_op)
-        expected_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        expected_grad_value = self.evaluate(grad)
 
     # Restore the MetaGraphDef into a new Graph with an import scope.
     with ops.Graph().as_default():
@@ -518,8 +518,8 @@ class ScopedMetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with session.Session() as sess:
-        sess.run(init_op)
-        actual_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        actual_grad_value = self.evaluate(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
   def testImportWhileLoopInWhileLoop(self):
@@ -544,8 +544,8 @@ class ScopedMetaGraphTest(test.TestCase):
       _, x = control_flow_ops.while_loop(lambda i, x: i < 2, body, [0, 0.0],
                                          name="")
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(x)
+        self.evaluate(variables.global_variables_initializer())
+        self.evaluate(x)
 
   def testScopedImportUnderNameScope(self):
     graph = ops.Graph()
@@ -868,8 +868,8 @@ class MetaGraphWithVariableScopeTest(test.TestCase):
       _, update_op = metrics.mean(values)
 
       initializer = variables.local_variables_initializer()
-      sess.run(initializer)
-      sess.run(update_op)
+      self.evaluate(initializer)
+      self.evaluate(update_op)
 
     meta_graph.export_scoped_meta_graph(
         filename=meta_graph_filename, graph=graph)
@@ -880,7 +880,7 @@ class MetaGraphWithVariableScopeTest(test.TestCase):
     with self.session(graph=graph) as sess:
       meta_graph.import_scoped_meta_graph(meta_graph_filename)
       initializer = variables.local_variables_initializer()
-      sess.run(initializer)
+      self.evaluate(initializer)
 
     # Verifies that importing an old meta_graph where "local_variables"
     # collection is of node_list type works, but cannot build initializer
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 3957d1de53..32a24521ad 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -503,7 +503,7 @@ class OperationTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           "Graph is invalid, contains a cycle with 2 nodes"):
-        sess.run(x)
+        self.evaluate(x)
 
   def testUpdateInput(self):
     g = ops.Graph()
@@ -517,21 +517,21 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEquals(x.consumers(), [])
     self.assertEquals(y.consumers(), [z.op, z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(sess.run(z), 4)
+      self.assertEquals(self.evaluate(z), 4)
 
     z.op._update_input(0, x)  # pylint: disable=protected-access
     self.assertEquals(list(z.op.inputs), [x, y])
     self.assertEquals(x.consumers(), [z.op])
     self.assertEquals(y.consumers(), [z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(sess.run(z), 3)
+      self.assertEquals(self.evaluate(z), 3)
 
     z.op._update_input(1, y)  # pylint: disable=protected-access
     self.assertEquals(list(z.op.inputs), [x, y])
     self.assertEquals(x.consumers(), [z.op])
     self.assertEquals(y.consumers(), [z.op])
     with session.Session(graph=g) as sess:
-      self.assertEquals(sess.run(z), 3)
+      self.assertEquals(self.evaluate(z), 3)
 
   def testUpdateInputGraphError(self):
     g_0 = ops.Graph()
@@ -557,7 +557,7 @@ class OperationTest(test_util.TensorFlowTestCase):
           errors.InvalidArgumentError,
           "Input 0 of node add was passed string from Const_1:0 incompatible "
           "with expected int32"):
-        sess.run(z)
+        self.evaluate(z)
 
   def testUpdateInputShapeError(self):
     g = ops.Graph()
@@ -2390,7 +2390,7 @@ class GraphTest(test_util.TensorFlowTestCase):
       c = math_ops.add(a, b)
     # Create a session we can delete
     with session.Session(graph=g) as sess:
-      sess.run(c)
+      self.evaluate(c)
     # Delete all references and trigger gc
     del g
     del a
@@ -2406,7 +2406,7 @@ class GraphTest(test_util.TensorFlowTestCase):
         math_ops.add([1, 2], [1, 2, 3])
       a = constant_op.constant(1)
       with session.Session() as sess:
-        sess.run(a)
+        self.evaluate(a)
 
   def testRunnableAfterInvalidShapeWithKernelLabelMap(self):
     g = ops.Graph()
@@ -2416,7 +2416,7 @@ class GraphTest(test_util.TensorFlowTestCase):
           test_ops.kernel_label_required(1)
       a = constant_op.constant(1)
       with session.Session() as sess:
-        sess.run(a)
+        self.evaluate(a)
 
 
 class AttrScopeTest(test_util.TensorFlowTestCase):
diff --git a/tensorflow/python/framework/smart_cond_test.py b/tensorflow/python/framework/smart_cond_test.py
index b8a9672b06..174ada9fe1 100644
--- a/tensorflow/python/framework/smart_cond_test.py
+++ b/tensorflow/python/framework/smart_cond_test.py
@@ -109,8 +109,8 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
                               exclusive=True)
     with session.Session() as sess:
       # No feed_dict necessary
-      self.assertEqual(sess.run(y), 1)
-      self.assertEqual(sess.run(z), 1)
+      self.assertEqual(self.evaluate(y), 1)
+      self.assertEqual(self.evaluate(z), 1)
 
   def testFalse(self):
     conditions = [(False, raise_exception)]
@@ -121,8 +121,8 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
                               default=lambda: constant_op.constant(1),
                               exclusive=True)
     with session.Session() as sess:
-      self.assertEqual(sess.run(y), 1)
-      self.assertEqual(sess.run(z), 1)
+      self.assertEqual(self.evaluate(y), 1)
+      self.assertEqual(self.evaluate(z), 1)
 
   def testMix(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
diff --git a/tensorflow/python/framework/sparse_tensor_test.py b/tensorflow/python/framework/sparse_tensor_test.py
index 2f7591abbd..9ee1bd75a5 100644
--- a/tensorflow/python/framework/sparse_tensor_test.py
+++ b/tensorflow/python/framework/sparse_tensor_test.py
@@ -50,7 +50,7 @@ class SparseTensorTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(indices, value.indices)
         self.assertAllEqual(values, value.values)
         self.assertAllEqual(shape, value.dense_shape)
-        sess_run_value = sess.run(sp)
+        sess_run_value = self.evaluate(sp)
         self.assertAllEqual(sess_run_value.indices, value.indices)
         self.assertAllEqual(sess_run_value.values, value.values)
         self.assertAllEqual(sess_run_value.dense_shape, value.dense_shape)
diff --git a/tensorflow/python/framework/subscribe_test.py b/tensorflow/python/framework/subscribe_test.py
index cab426844d..5322204ce6 100644
--- a/tensorflow/python/framework/subscribe_test.py
+++ b/tensorflow/python/framework/subscribe_test.py
@@ -66,9 +66,9 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertTrue(c.op in d.op.control_inputs)
 
     with self.cached_session() as sess:
-      c_out = sess.run([c])
-      n_out = sess.run([n])
-      d_out = sess.run([d])
+      c_out = self.evaluate([c])
+      n_out = self.evaluate([n])
+      d_out = self.evaluate([d])
 
     self.assertEqual(n_out, [-2])
     self.assertEqual(c_out, [2])
@@ -145,8 +145,8 @@ class SubscribeTest(test_util.TensorFlowTestCase):
                             lambda t: script_ops.py_func(sub, [t], [t.dtype]))
 
     with self.cached_session() as sess:
-      c_out = sess.run([c])
-      d_out = sess.run([d])
+      c_out = self.evaluate([c])
+      d_out = self.evaluate([d])
 
     self.assertEqual(c_out, [42])
     self.assertEqual(d_out, [11])
@@ -205,7 +205,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
 
     # Expect the three side effect graphs to have been evaluated.
     with self.cached_session() as sess:
-      sess.run([c_sub])
+      self.evaluate([c_sub])
     self.assertIn('graph1', shared)
     self.assertIn('graph2', shared)
     self.assertIn('graph3', shared)
@@ -229,20 +229,20 @@ class SubscribeTest(test_util.TensorFlowTestCase):
 
     with self.cached_session() as sess:
       # Initialize the variables first.
-      sess.run([v1.initializer])
-      sess.run([v2.initializer])
+      self.evaluate([v1.initializer])
+      self.evaluate([v2.initializer])
 
       # Expect the side effects to be triggered when evaluating the add op as
       # it will read the value of the variable.
-      sess.run([add])
+      self.evaluate([add])
       self.assertEqual(1, len(shared))
 
       # Expect the side effect not to be triggered when evaluating the assign
       # op as it will not access the 'read' output of the variable.
-      sess.run([assign_v1])
+      self.evaluate([assign_v1])
       self.assertEqual(1, len(shared))
 
-      sess.run([add])
+      self.evaluate([add])
       self.assertEqual(2, len(shared))
 
       # Make sure the values read from the variable match the expected ones.
@@ -273,7 +273,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertFalse(subscribe._is_subscribed_identity(tensor_array.handle))
 
     with self.cached_session() as sess:
-      sess.run([reader])
+      self.evaluate([reader])
     self.assertEqual(0, len(shared))
 
   def testMultipleOutputs(self):
@@ -304,7 +304,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
                         lambda t: script_ops.py_func(sub, [t], [t.dtype]))
 
     with self.cached_session() as sess:
-      sess.run([neg])
+      self.evaluate([neg])
 
     # All three ops have been processed.
     self.assertEqual(3, len(shared))
@@ -375,7 +375,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertIsNot(context(subscriptions[0]), context(subscriptions[1]))
 
     with self.cached_session() as sess:
-      sess.run(cond)
+      self.evaluate(cond)
 
     self.assertEqual(3, len(results))
 
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index bdf759f220..87d65c8c46 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -771,7 +771,7 @@ class TensorUtilTest(test.TestCase):
     with self.cached_session() as sess:
       ma = MockArray(np.array([10, 20, 30]))
       t = ops.convert_to_tensor(ma)
-      a = sess.run(t)
+      a = self.evaluate(t)
       self.assertEquals(np.int64, a.dtype)
       self.assertAllClose(np.array([10, 20, 30], dtype=np.int64), a)
 
diff --git a/tensorflow/python/grappler/constant_folding_test.py b/tensorflow/python/grappler/constant_folding_test.py
index ab1d0ed25b..30c1e14681 100644
--- a/tensorflow/python/grappler/constant_folding_test.py
+++ b/tensorflow/python/grappler/constant_folding_test.py
@@ -61,7 +61,7 @@ class ConstantFoldingTest(test.TestCase):
           back_prop=False,
           parallel_iterations=1)
       with session.Session() as sess:
-        y_v = sess.run(y)
+        y_v = self.evaluate(y)
         self.assertAllEqual(np.zeros([10, 20, 30]), y_v)
 
 
diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 7b68d5e80d..55ccfbb93c 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -241,7 +241,7 @@ class LayoutOptimizerTest(test.TestCase):
       if restore:
         saver.restore(sess, checkpoint_path)
       else:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
 
       np.random.seed(0)
       for _ in range(2):
@@ -262,7 +262,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _two_layer_model(x)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -365,7 +365,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(pad)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -396,7 +396,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -425,7 +425,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(cast)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -456,7 +456,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -486,7 +486,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -516,7 +516,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(squeeze)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -545,7 +545,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -574,7 +574,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -603,7 +603,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -632,7 +632,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -662,7 +662,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -691,7 +691,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reduce_sum)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -724,7 +724,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(concat)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -835,7 +835,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(reverse)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -905,7 +905,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(select)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -966,7 +966,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(select)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1179,7 +1179,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(s)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1214,7 +1214,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = array_ops.identity(s)
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1347,7 +1347,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1374,7 +1374,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop_with_branch()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1398,7 +1398,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _loop_with_vec_and_4d()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
@@ -1422,7 +1422,7 @@ class LayoutOptimizerTest(test.TestCase):
       output = _model_with_second_port()
 
       with session.Session(config=_get_config(False)) as sess:
-        output_val_ref = sess.run(output)
+        output_val_ref = self.evaluate(output)
 
       with session.Session(config=_get_config()) as sess:
         metadata = config_pb2.RunMetadata()
diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py
index 98cbb1a4b6..d233629cbb 100644
--- a/tensorflow/python/grappler/memory_optimizer_test.py
+++ b/tensorflow/python/grappler/memory_optimizer_test.py
@@ -231,10 +231,10 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
       train_op = graph.get_operation_by_name(train_op_name)
       loss_op = graph.get_tensor_by_name(loss_op_name)
       with session.Session(config=config, graph=graph) as sess:
-        sess.run(init_op)
-        sess.run(train_op)
-        sess.run(train_op)
-        return sess.run(loss_op)
+        self.evaluate(init_op)
+        self.evaluate(train_op)
+        self.evaluate(train_op)
+        return self.evaluate(loss_op)
 
   def testRecomputationRewritingNoErrors(self):
     """Tests that graph output is not significantly different with rewriting."""
@@ -295,8 +295,8 @@ class MemoryOptimizerRecomputeTest(test.TestCase):
           rewrite_options=manual_memory_config)
       session_config = config_pb2.ConfigProto(graph_options=graph_options)
       with session.Session(config=session_config) as sess:
-        sess.run(init_op)
-        sess.run(train_op)
+        self.evaluate(init_op)
+        self.evaluate(train_op)
 
   def testHintDoesRewrite(self):
     graph = self._annotated_graph()[0]
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index a727e99f66..48fdd56e9f 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -136,7 +136,7 @@ class BackendUtilsTest(test.TestCase):
       x = keras.Input((3,))
       y = keras.layers.BatchNormalization()(x)
       if not context.executing_eagerly():
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
   def test_learning_phase_scope(self):
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index 7172571f7c..b1449069e3 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -1013,8 +1013,8 @@ class RNNTest(test.TestCase):
         inputs, _ = cell(inputs, initial_state)
         output = inputs
         if not context.executing_eagerly():
-          sess.run(variables_lib.global_variables_initializer())
-          output = sess.run(output)
+          self.evaluate(variables_lib.global_variables_initializer())
+          output = self.evaluate(output)
         return output
 
     random_seed.set_random_seed(12345)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 74e5d4d4ce..f049b10721 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -322,19 +322,19 @@ class KerasMetricsTest(test.TestCase):
       m = metrics.Mean()
       v = array_ops.placeholder(dtypes.float32)
       w = array_ops.placeholder(dtypes.float32)
-      sess.run(variables.variables_initializer(m.variables))
+      self.evaluate(variables.variables_initializer(m.variables))
 
       # check __call__()
       result_t = m(v, sample_weight=w)
       result = sess.run(result_t, feed_dict=({v: 100, w: 0.5}))
-      self.assertEqual(sess.run(m.total), 50)
-      self.assertEqual(sess.run(m.count), 0.5)
+      self.assertEqual(self.evaluate(m.total), 50)
+      self.assertEqual(self.evaluate(m.count), 0.5)
       self.assertEqual(result, 50 / 0.5)
 
       # check update_state() and result()
       result = sess.run(result_t, feed_dict=({v: [1, 5], w: [1, 0.2]}))
-      self.assertAlmostEqual(sess.run(m.total), 52, 2)  # 50 + 1 + 5 * 0.2
-      self.assertAlmostEqual(sess.run(m.count), 1.7, 2)  # 0.5 + 1.2
+      self.assertAlmostEqual(self.evaluate(m.total), 52, 2)  # 50 + 1 + 5 * 0.2
+      self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2)  # 0.5 + 1.2
       self.assertAlmostEqual(result, 52 / 1.7, 2)
 
   @test_util.run_in_graph_and_eager_modes
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl_test.py b/tensorflow/python/keras/optimizer_v2/ftrl_test.py
index c14cf75c26..ca8c33dfa6 100644
--- a/tensorflow/python/keras/optimizer_v2/ftrl_test.py
+++ b/tensorflow/python/keras/optimizer_v2/ftrl_test.py
@@ -54,7 +54,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllClose([0.0, 0.0], v0_val)
         self.assertAllClose([0.0, 0.0], v1_val)
 
@@ -62,7 +62,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(3):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-2.60260963, -4.29698515]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -90,14 +90,14 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run 3 steps FTRL
         for _ in range(3):
           update.run()
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-2.55607247, -3.98729396]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -137,14 +137,14 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run 10 steps FTRL
         for _ in range(10):
           update.run()
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-7.66718769, -10.91273689]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -166,7 +166,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
@@ -174,7 +174,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(10):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.24059935, -0.46829352]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -203,7 +203,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
@@ -211,7 +211,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(10):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.22578995, -0.44345796]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -239,7 +239,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([[1.0], [2.0]], v0_val)
         self.assertAllCloseAccordingToType([[4.0], [3.0]], v1_val)
 
@@ -247,7 +247,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(10):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val)
         self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val)
 
@@ -275,7 +275,7 @@ class FtrlOptimizerTest(test.TestCase):
         update1 = opt1.apply_gradients([(grads1, var1)])
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([1.0, 2.0], v1_val)
 
@@ -284,7 +284,7 @@ class FtrlOptimizerTest(test.TestCase):
           update0.run()
           update1.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         # var0 is experiencing L2 shrinkage so it should be smaller than var1
         # in magnitude.
         self.assertTrue((v0_val**2 < v1_val**2).all())
@@ -313,7 +313,7 @@ class FtrlOptimizerTest(test.TestCase):
     variables.global_variables_initializer().run()
 
     sess = ops.get_default_session()
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     if is_sparse:
       self.assertAllCloseAccordingToType([[0.0], [0.0]], v0_val)
       self.assertAllCloseAccordingToType([[0.0], [0.0]], v1_val)
@@ -325,7 +325,7 @@ class FtrlOptimizerTest(test.TestCase):
     for _ in range(steps):
       update.run()
 
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     return v0_val, v1_val
 
   # When variables are initialized with Zero, FTRL-Proximal has two properties:
diff --git a/tensorflow/python/kernel_tests/accumulate_n_test.py b/tensorflow/python/kernel_tests/accumulate_n_test.py
index ae24cf8f14..c7f11f854d 100644
--- a/tensorflow/python/kernel_tests/accumulate_n_test.py
+++ b/tensorflow/python/kernel_tests/accumulate_n_test.py
@@ -65,7 +65,7 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase):
             for _ in range(0, num_inputs)
         ]
         accum_n = math_ops.accumulate_n(input_vars)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         accum_n_grad = gradients.gradients(accum_n, input_vars)
         self.assertAllEqual(
             np.repeat(1.0, num_inputs),  # d/dx (x + y + ...) = 1
diff --git a/tensorflow/python/kernel_tests/aggregate_ops_test.py b/tensorflow/python/kernel_tests/aggregate_ops_test.py
index 0f15319cb5..874d616658 100644
--- a/tensorflow/python/kernel_tests/aggregate_ops_test.py
+++ b/tensorflow/python/kernel_tests/aggregate_ops_test.py
@@ -61,7 +61,7 @@ class AddNTest(test.TestCase):
       for dtype in self._supported_types():
         for count in range(1, self._MAX_N + 1):
           data = [self._buildData((2, 2), dtype) for _ in range(count)]
-          actual = sess.run(math_ops.add_n(data))
+          actual = self.evaluate(math_ops.add_n(data))
           expected = np.sum(np.vstack(
               [np.expand_dims(d, 0) for d in data]), axis=0)
           tol = 5e-3 if dtype == dtypes.float16 else 5e-7
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index d345138ec7..afc158f697 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -833,7 +833,7 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase):
       index = constant_op.constant(1, dtype=dtypes.int64)
       b = 2. * a[index]
       grad, = gradients_impl.gradients(b, a)
-      self.assertAllEqual(sess.run(grad), [0., 2., 0.])
+      self.assertAllEqual(self.evaluate(grad), [0., 2., 0.])
 
 
 class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
@@ -846,7 +846,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
               math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
               shape=(4, 1, 1)))
       varshape = variables.Variable([6, 4, 4], dtype=dtypes.int32)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0])
       end = constant_op.constant([4, 1, 1])
       strides = constant_op.constant([1, 1, 1])
@@ -859,7 +859,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
           math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
           shape=(4, 1, 1))
       original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0], dtype=dtypes.int64)
       end = constant_op.constant([4, 1, 1], dtype=dtypes.int64)
       strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64)
@@ -873,7 +873,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
           math_ops.cast(math_ops.range(1, 5, 1), dtypes.float32),
           shape=(4, 1, 1))
       original_shape = constant_op.constant([6, 4, 4], dtype=dtypes.int64)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       begin = constant_op.constant([0, 0, 0], dtype=dtypes.int32)
       end = constant_op.constant([4, 1, 1], dtype=dtypes.int64)
       strides = constant_op.constant([1, 1, 1], dtype=dtypes.int64)
@@ -1042,7 +1042,7 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     too_large_val = constant_op.constant([3, 4], dtype=dtypes.int64)
     v = resource_variable_ops.ResourceVariable(init_val)
     with self.cached_session() as sess:
-      sess.run(v.initializer)
+      self.evaluate(v.initializer)
       with self.assertRaises(ValueError):
         sess.run(v[:].assign(too_large_val))
       with self.assertRaises(ValueError):
@@ -1269,7 +1269,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
             initializer=init_ops.constant_initializer(10.0),
             use_resource=use_resource)
         guarantee_a = array_ops.guarantee_const(a)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertEqual(10.0, guarantee_a.eval())
 
   def testResourceRejection(self):
@@ -1279,7 +1279,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
           initializer=init_ops.constant_initializer(10.0),
           use_resource=True)
       guarantee_a = array_ops.guarantee_const(a.handle)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       with self.assertRaisesWithPredicateMatch(errors.InvalidArgumentError,
                                                "cannot be a resource variable"):
         guarantee_a.eval()
diff --git a/tensorflow/python/kernel_tests/attention_ops_test.py b/tensorflow/python/kernel_tests/attention_ops_test.py
index 14db06b783..00dba9996d 100644
--- a/tensorflow/python/kernel_tests/attention_ops_test.py
+++ b/tensorflow/python/kernel_tests/attention_ops_test.py
@@ -85,7 +85,7 @@ class ExtractGlimpseTest(test.TestCase):
 
     # Evaluate the TensorFlow Graph.
     with self.cached_session() as sess:
-      value_rows, value_cols = sess.run([glimpse_rows, glimpse_cols])
+      value_rows, value_cols = self.evaluate([glimpse_rows, glimpse_cols])
 
     # Check dimensions of returned glimpse.
     self.assertEqual(value_rows.shape[1], glimpse_sizes[0])
diff --git a/tensorflow/python/kernel_tests/barrier_ops_test.py b/tensorflow/python/kernel_tests/barrier_ops_test.py
index 4d36b3a465..495bbe7b34 100644
--- a/tensorflow/python/kernel_tests/barrier_ops_test.py
+++ b/tensorflow/python/kernel_tests/barrier_ops_test.py
@@ -229,7 +229,7 @@ class BarrierTest(test.TestCase):
       insert_ops = [b.insert_many(0, [k], [v]) for k, v in zip(keys, values)]
       take_t = b.take_many(10)
 
-      sess.run(insert_ops)
+      self.evaluate(insert_ops)
       self.assertEquals(size_t.eval(), [10])
 
       indices_val, keys_val, values_val = sess.run(
@@ -491,9 +491,9 @@ class BarrierTest(test.TestCase):
       b = data_flow_ops.Barrier(
           (dtypes.float32, dtypes.float32), shapes=((), ()), name="B")
       take_t = b.take_many(1, allow_small_batch=True)
-      sess.run(b.close(cancel))
+      self.evaluate(b.close(cancel))
       with self.assertRaisesOpError("is closed and has insufficient elements"):
-        sess.run(take_t)
+        self.evaluate(take_t)
 
   def testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(self):
     self._testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(cancel=False)
diff --git a/tensorflow/python/kernel_tests/base64_ops_test.py b/tensorflow/python/kernel_tests/base64_ops_test.py
index 1b399942ef..bb903d827f 100644
--- a/tensorflow/python/kernel_tests/base64_ops_test.py
+++ b/tensorflow/python/kernel_tests/base64_ops_test.py
@@ -93,7 +93,7 @@ class Base64OpsTest(test_util.TensorFlowTestCase):
         decoded = string_ops.decode_base64(encoded)
 
         with self.cached_session() as sess:
-          encoded_value, decoded_value = sess.run([encoded, decoded])
+          encoded_value, decoded_value = self.evaluate([encoded, decoded])
 
         self.assertEqual(encoded_value.shape, msg.shape)
         self.assertEqual(decoded_value.shape, msg.shape)
diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py
index ac5cbc810a..cd33048121 100644
--- a/tensorflow/python/kernel_tests/basic_gpu_test.py
+++ b/tensorflow/python/kernel_tests/basic_gpu_test.py
@@ -44,13 +44,13 @@ class GPUBinaryOpsTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_gpu = sess.run(out)
+      tf_gpu = self.evaluate(out)
 
     with self.cached_session(use_gpu=False) as sess:
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
-      tf_cpu = sess.run(out)
+      tf_cpu = self.evaluate(out)
 
     self.assertAllClose(tf_cpu, tf_gpu)
 
@@ -96,7 +96,7 @@ class MathBuiltinUnaryTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       inx = ops.convert_to_tensor(x)
       ofunc = tf_func(inx)
-      tf_out = sess.run(ofunc)
+      tf_out = self.evaluate(ofunc)
     self.assertAllClose(np_out, tf_out)
 
   def _inv(self, x):
@@ -148,7 +148,7 @@ class MathBuiltinUnaryTest(test.TestCase):
       iny = ops.convert_to_tensor(y + 0.1)
       ofunc = inx / iny
       out_func2 = math_ops.floor(ofunc)
-      tf_out = sess.run(out_func2)
+      tf_out = self.evaluate(out_func2)
 
     self.assertAllClose(np_out, tf_out)
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index 12afb6a2ad..1a7b1a7e90 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -98,8 +98,8 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
           quantile_accumulator_handle, num_features=2)
       quantiles = boosted_trees_ops.boosted_trees_bucketize(
           [self._feature_0, self._feature_1], buckets)
-      sess.run(summary_op)
-      sess.run(flush_op)
+      self.evaluate(summary_op)
+      self.evaluate(flush_op)
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
@@ -132,8 +132,8 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
           quantile_accumulator_handle_1, num_features=1)
       quantiles = boosted_trees_ops.boosted_trees_bucketize(
           [self._feature_0, self._feature_1], bucket_0 + bucket_1)
-      sess.run([summary_op_0, summary_op_1])
-      sess.run([flush_op_0, flush_op_1])
+      self.evaluate([summary_op_0, summary_op_1])
+      self.evaluate([flush_op_0, flush_op_1])
       self.assertAllClose(self._feature_0_boundaries, bucket_0[0].eval())
       self.assertAllClose(self._feature_1_boundaries, bucket_1[0].eval())
 
@@ -158,7 +158,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
                                             self._example_weights)
       with ops.control_dependencies([summaries]):
         flush = accumulator.flush()
-      sess.run(flush)
+      self.evaluate(flush)
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
       save.save(sess, save_path)
@@ -185,12 +185,12 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
 
       summaries = accumulator.add_summaries([self._feature_0, self._feature_1],
                                             self._example_weights)
-      sess.run(summaries)
+      self.evaluate(summaries)
       buckets = accumulator.get_bucket_boundaries()
       self.assertAllClose([], buckets[0].eval())
       self.assertAllClose([], buckets[1].eval())
       save.save(sess, save_path)
-      sess.run(accumulator.flush())
+      self.evaluate(accumulator.flush())
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
index cc3984015d..e1036b0b75 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
@@ -65,16 +65,16 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
       self.assertAllClose([[0.004775, 0.41184], [0.02823, 0.41184]],
-                          sess.run(gains_list))
-      self.assertAllEqual([[1, 1], [1, 1]], sess.run(thresholds_list))
+                          self.evaluate(gains_list))
+      self.assertAllEqual([[1, 1], [1, 1]], self.evaluate(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[-.416667], [.568966]], [[-.6], [-.75]]],
-                          sess.run(left_node_contribs_list))
+                          self.evaluate(left_node_contribs_list))
       self.assertAllClose([[[-.592593], [-.75]], [[-.076923], [.568966]]],
-                          sess.run(right_node_contribs_list))
+                          self.evaluate(right_node_contribs_list))
 
   def testCalculateBestGainsWithL2(self):
     """Testing Gain calculation with L2."""
@@ -113,16 +113,16 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
       self.assertAllClose([[0., 0.33931375], [0.01879096, 0.33931375]],
-                          sess.run(gains_list))
-      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+                          self.evaluate(gains_list))
+      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
-                          sess.run(left_node_contribs_list))
+                          self.evaluate(left_node_contribs_list))
       self.assertAllClose([[[-.424658], [-.6]], [[-.043478], [.485294]]],
-                          sess.run(right_node_contribs_list))
+                          self.evaluate(right_node_contribs_list))
 
   def testCalculateBestGainsWithL1(self):
     """Testing Gain calculation with L1."""
@@ -162,18 +162,18 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
       self.assertAllClose([[[0.0], [0.3965517]], [[-0.4], [-0.5]]],
-                          sess.run(left_node_contribs_list))
+                          self.evaluate(left_node_contribs_list))
 
       self.assertAllClose([[[-0.3333333], [-0.5]], [[0.0], [0.396552]]],
-                          sess.run(right_node_contribs_list))
+                          self.evaluate(right_node_contribs_list))
 
       # Gain should also include an adjustment of the gradient by l1.
       self.assertAllClose([[0.0, 0.191207], [0.01, 0.191207]],
-                          sess.run(gains_list))
+                          self.evaluate(gains_list))
 
   def testCalculateBestGainsWithTreeComplexity(self):
     """Testing Gain calculation with L2."""
@@ -214,18 +214,18 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
           min_node_weight=0,
           max_splits=max_splits)
 
-      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllEqual([[1, 2], [1, 2]], self.evaluate(node_ids_list))
 
       self.assertAllClose([[-3., -2.66068625], [-2.98120904, -2.66068625]],
-                          sess.run(gains_list))
+                          self.evaluate(gains_list))
 
-      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      self.assertAllEqual([[0, 1], [1, 1]], self.evaluate(thresholds_list))
       # The left node contrib will be later added to the previous node value to
       # make the left node value, and the same for right node contrib.
       self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
-                          sess.run(left_node_contribs_list))
+                          self.evaluate(left_node_contribs_list))
       self.assertAllClose([[[-.424658], [-.6]], [[-.043478], [.485294]]],
-                          sess.run(right_node_contribs_list))
+                          self.evaluate(right_node_contribs_list))
 
   def testCalculateBestGainsWithMinNodeWeight(self):
     """Testing Gain calculation without any regularization."""
@@ -266,13 +266,13 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
 
       # We can't split node 1 on feature 1 and node 2 on feature 2 because of
       # the min node weight.
-      self.assertAllEqual([[2], [1]], sess.run(node_ids_list))
-      self.assertAllClose([[0.384314], [0.098013]], sess.run(gains_list))
-      self.assertAllEqual([[1], [1]], sess.run(thresholds_list))
+      self.assertAllEqual([[2], [1]], self.evaluate(node_ids_list))
+      self.assertAllClose([[0.384314], [0.098013]], self.evaluate(gains_list))
+      self.assertAllEqual([[1], [1]], self.evaluate(thresholds_list))
       self.assertAllClose([[[0.4852941]], [[-.6]]],
-                          sess.run(left_node_contribs_list))
+                          self.evaluate(left_node_contribs_list))
       self.assertAllClose([[[-0.75]], [[-0.014925]]],
-                          sess.run(right_node_contribs_list))
+                          self.evaluate(right_node_contribs_list))
 
   def testCalculateBestGainsWithMinNodeWeightNoSplitOnFeturePossible(self):
     """Testing Gain calculation without any regularization."""
@@ -311,9 +311,9 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            max_splits=max_splits)
 
       # We can't split either of the nodes on the first feature
-      self.assertEqual(2, len(sess.run(node_ids_list)))
-      self.assertAllEqual([], sess.run(node_ids_list)[0])
-      self.assertAllEqual([1], sess.run(node_ids_list)[1])
+      self.assertEqual(2, len(self.evaluate(node_ids_list)))
+      self.assertAllEqual([], self.evaluate(node_ids_list)[0])
+      self.assertAllEqual([1], self.evaluate(node_ids_list)[1])
 
       # Now check when we can't split on any feature
       (node_ids_list, _, _, _,
@@ -325,7 +325,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            tree_complexity=0.0,
            min_node_weight=10,
            max_splits=max_splits)
-      self.assertAllEqual([[], []], sess.run(node_ids_list))
+      self.assertAllEqual([[], []], self.evaluate(node_ids_list))
 
   def testMakeStatsSummarySimple(self):
     """Simple test for MakeStatsSummary."""
diff --git a/tensorflow/python/kernel_tests/bucketize_op_test.py b/tensorflow/python/kernel_tests/bucketize_op_test.py
index 57413e6af5..f40ca82527 100644
--- a/tensorflow/python/kernel_tests/bucketize_op_test.py
+++ b/tensorflow/python/kernel_tests/bucketize_op_test.py
@@ -32,7 +32,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0, 3, 8, 11])
     expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, sess.run(op))
+      self.assertAllEqual(expected_out, self.evaluate(op))
 
   def testFloat(self):
     op = math_ops._bucketize(
@@ -40,7 +40,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0., 3., 8., 11.])
     expected_out = [0, 1, 1, 2, 2, 3, 3, 4, 4]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, sess.run(op))
+      self.assertAllEqual(expected_out, self.evaluate(op))
 
   def test2DInput(self):
     op = math_ops._bucketize(
@@ -48,7 +48,7 @@ class BucketizationOpTest(test.TestCase):
         boundaries=[0, 3, 8, 11])
     expected_out = [[0, 1, 1, 2, 2], [3, 3, 4, 4, 1]]
     with self.session(use_gpu=True) as sess:
-      self.assertAllEqual(expected_out, sess.run(op))
+      self.assertAllEqual(expected_out, self.evaluate(op))
 
   def testInvalidBoundariesOrder(self):
     op = math_ops._bucketize(
@@ -56,7 +56,7 @@ class BucketizationOpTest(test.TestCase):
     with self.session(use_gpu=True) as sess:
       with self.assertRaisesRegexp(
           errors_impl.InvalidArgumentError, "Expected sorted boundaries"):
-        sess.run(op)
+        self.evaluate(op)
 
   def testBoundariesNotList(self):
     with self.assertRaisesRegexp(
diff --git a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
index 46ab71537f..031accee55 100644
--- a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
+++ b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
@@ -97,7 +97,7 @@ class RangeSamplerOpsTest(test.TestCase):
           true_classes, self.NUM_TRUE, self.NUM_SAMPLED, True)
       accidental_hits = candidate_sampling_ops.compute_accidental_hits(
           true_classes, sampled_candidates, self.NUM_TRUE)
-      indices, ids, weights = sess.run(accidental_hits)
+      indices, ids, weights = self.evaluate(accidental_hits)
 
     self.assertEqual(1, accidental_hits[0].get_shape().ndims)
     self.assertEqual(1, accidental_hits[1].get_shape().ndims)
diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py
index bc49cd5a04..2cfe084d95 100644
--- a/tensorflow/python/kernel_tests/cast_op_test.py
+++ b/tensorflow/python/kernel_tests/cast_op_test.py
@@ -187,7 +187,7 @@ class CastOpTest(test.TestCase):
       y = variables.Variable(True, dtype=dtypes.bool)
       cast = math_ops.cast(y, x.dtype)
       variables.global_variables_initializer().run()
-      self.assertEqual(1.0, sess.run(cast))
+      self.assertEqual(1.0, self.evaluate(cast))
 
   def testGradients(self):
     t = [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128]
@@ -229,7 +229,7 @@ class SaturateCastTest(test.TestCase):
               [lo, lo + 1, lo // 2, hi // 2, hi - 1, hi], dtype=in_type)
           y = math_ops.saturate_cast(x, dtype=out_type)
           self.assertEqual(y.dtype, out_type)
-          x, y = sess.run([x, y])
+          x, y = self.evaluate([x, y])
           correct = np.maximum(out_type.min, np.minimum(out_type.max, x))
           self.assertAllEqual(correct, y)
 
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index fa41a03b54..1a509a43d1 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -97,7 +97,7 @@ def TriAngInvCompositeGrad(l, grad):
 class CholeskyOpTest(test.TestCase):
 
   def _verifyCholeskyBase(self, sess, x, chol, verification):
-    chol_np, verification_np = sess.run([chol, verification])
+    chol_np, verification_np = self.evaluate([chol, verification])
     self.assertAllClose(x, verification_np)
     self.assertShapeEqual(x, chol)
     # Check that the cholesky is lower triangular, and has positive diagonal
@@ -183,7 +183,7 @@ class CholeskyOpTest(test.TestCase):
       matrix2 = math_ops.matmul(matrix2, matrix2, adjoint_a=True)
       c1 = linalg_ops.cholesky(matrix1)
       c2 = linalg_ops.cholesky(matrix2)
-      c1_val, c2_val = sess.run([c1, c2])
+      c1_val, c2_val = self.evaluate([c1, c2])
       self.assertAllClose(c1_val, c2_val)
 
 
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 149302831b..27137f76bd 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gradient_checker
@@ -65,7 +66,7 @@ class ConcatOpTest(test.TestCase):
     self.assertAllEqual(result[:, 4:], params[p2])
 
   def testInt32GPU(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       p1 = np.random.rand(2, 3).astype("i")
       p2 = np.random.rand(2, 3).astype("i")
       x1 = constant_op.constant(p1)
@@ -76,13 +77,13 @@ class ConcatOpTest(test.TestCase):
     self.assertAllEqual(result[2:, :], p2)
 
   def testRefType(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       p1 = np.random.rand(4, 4).astype("f")
       p2 = np.random.rand(4, 4).astype("f")
       v1 = variables.Variable(p1)
       v2 = variables.Variable(p2)
       c = array_ops.concat([v1, v2], 0)
-      variables.global_variables_initializer().run()
+      self.evaluate(variables.global_variables_initializer())
       result = self.evaluate(c)
 
     self.assertEqual(result.shape, c.get_shape())
@@ -172,7 +173,7 @@ class ConcatOpTest(test.TestCase):
     # Test both positive and negative concat axis.
     # -2 and 1 correspond to the same axis for 3-dimensional tensors.
     for axis in [-2, 1]:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         inp = []
         inp_tensors = []
         for x in [1, 2, 6]:
@@ -203,7 +204,7 @@ class ConcatOpTest(test.TestCase):
     self._testGradientsSimple(dtypes.complex64)
 
   def testGradientsFirstDim(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       inp = []
       inp_tensors = []
       for x in [1, 2, 6]:
@@ -230,7 +231,7 @@ class ConcatOpTest(test.TestCase):
     # Test both positive and negative concat axis.
     # -1 and 2 correspond to the same axis for 3-dimensional tensors.
     for axis in [-1, 2]:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         inp = []
         inp_tensors = []
         for x in [1, 2, 6]:
@@ -261,7 +262,7 @@ class ConcatOpTest(test.TestCase):
     # Random dim to concat on
     concat_dim = np.random.randint(5)
     concat_dim_sizes = np.random.randint(1, 5, size=num_tensors)
-    with self.cached_session(use_gpu=True):
+    with test_util.use_gpu():
       inp = []
       inp_tensors = []
       for x in concat_dim_sizes:
@@ -358,7 +359,7 @@ class ConcatOpTest(test.TestCase):
   def testZeroSize(self):
     # Verify that concat doesn't crash and burn for zero size inputs
     np.random.seed(7)
-    with self.session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       for shape0 in (), (2,):
         axis = len(shape0)
         for shape1 in (), (3,):
@@ -370,10 +371,10 @@ class ConcatOpTest(test.TestCase):
               # TODO(irving): Make tf.concat handle map, then drop list().
               xs = list(map(constant_op.constant, [x0, x1]))
               c = array_ops.concat(xs, axis)
-              self.assertAllEqual(c.eval(), correct)
+              self.assertAllEqual(self.evaluate(c), correct)
               # Check gradients
               dc = np.random.randn(*c.get_shape().as_list())
-              dxs = sess.run(gradients_impl.gradients(c, xs, dc))
+              dxs = self.evaluate(gradients_impl.gradients(c, xs, dc))
               self.assertAllEqual(dc, np.concatenate(dxs, axis=axis))
 
   def testTensorConcatDim0Grad(self):
@@ -473,18 +474,17 @@ class ConcatOpTest(test.TestCase):
   def testConcatTuple(self):
     c1 = np.random.rand(4, 4)
     c2 = np.random.rand(4, 4)
-    with self.cached_session():
-      concat_list_t = array_ops.concat([c1, c2], 0)
-      concat_tuple_t = array_ops.concat((c1, c2), 0)
-      self.assertAllEqual(concat_list_t.eval(), self.evaluate(concat_tuple_t))
+    concat_list_t = array_ops.concat([c1, c2], 0)
+    concat_tuple_t = array_ops.concat((c1, c2), 0)
+    self.assertAllEqual(
+        self.evaluate(concat_list_t), self.evaluate(concat_tuple_t))
 
   def testConcatNoScalars(self):
-    with self.cached_session():
-      scalar = constant_op.constant(7)
-      dim = array_ops.placeholder(dtypes.int32)
-      with self.assertRaisesRegexp(
-          ValueError, r"Can't concatenate scalars \(use tf\.stack instead\)"):
-        array_ops.concat([scalar, scalar, scalar], dim)
+    scalar = constant_op.constant(7)
+    dim = array_ops.placeholder(dtypes.int32)
+    with self.assertRaisesRegexp(
+        ValueError, r"Can't concatenate scalars \(use tf\.stack instead\)"):
+      array_ops.concat([scalar, scalar, scalar], dim)
 
   # important as gpu implementation could fail if
   # shared memory is not large for all the inputs
@@ -523,21 +523,21 @@ class ConcatOpTest(test.TestCase):
           self.assertAllEqual(result[index], params[p[i]])
 
   def testConcatEmpty(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       t1 = []
       t2 = []
-      output = gen_array_ops.concat_v2([t1, t2], 0).eval()
-      self.assertFalse(output)  # Checks that output is empty
+      output = gen_array_ops.concat_v2([t1, t2], 0)
+      self.assertFalse(self.evaluate(output))  # Checks that output is empty
 
   def testConcatInvalidAxis(self):
     with self.assertRaises(ValueError):
-      with self.session(use_gpu=True):
+      with test_util.use_gpu():
         t1 = [1]
         t2 = [2]
         gen_array_ops.concat_v2([t1, t2], 1).eval()
 
   def testConcatNegativeAxis(self):
-    with self.session(use_gpu=True):
+    with test_util.use_gpu():
       t1 = [[1, 2, 3], [4, 5, 6]]
       t2 = [[7, 8, 9], [10, 11, 12]]
 
@@ -608,7 +608,7 @@ class ConcatOpTest(test.TestCase):
 
   def testConcatAxisType(self):
     for dtype in [dtypes.int32, dtypes.int64]:
-      with self.cached_session(use_gpu=True):
+      with test_util.use_gpu():
         t1 = [[1, 2, 3], [4, 5, 6]]
         t2 = [[7, 8, 9], [10, 11, 12]]
 
@@ -621,65 +621,61 @@ class ConcatOpTest(test.TestCase):
 class ConcatOffsetTest(test.TestCase):
 
   def testBasic(self):
-    with self.session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       cdim = constant_op.constant(1, dtypes.int32)
       s0 = constant_op.constant([2, 3, 5], dtypes.int32)
       s1 = constant_op.constant([2, 7, 5], dtypes.int32)
       s2 = constant_op.constant([2, 20, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = sess.run(off)
+      ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
   def testNotVector(self):
-    with self.cached_session() as sess:
-      cdim = constant_op.constant(1, dtypes.int32)
-      s0 = constant_op.constant([[2, 3, 5]], dtypes.int32)
-      s1 = constant_op.constant([[2, 7, 5]], dtypes.int32)
-      off = gen_array_ops.concat_offset(cdim, [s0, s1])
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   r"should be a vector"):
-        sess.run(off)
+    cdim = constant_op.constant(1, dtypes.int32)
+    s0 = constant_op.constant([[2, 3, 5]], dtypes.int32)
+    s1 = constant_op.constant([[2, 7, 5]], dtypes.int32)
+    off = gen_array_ops.concat_offset(cdim, [s0, s1])
+    with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                 r"should be a vector"):
+      self.evaluate(off)
 
   def testConcatDimOutOfRange(self):
-    with self.cached_session() as sess:
-      cdim = constant_op.constant(4, dtypes.int32)
-      s0 = constant_op.constant([2, 3, 5], dtypes.int32)
-      s1 = constant_op.constant([2, 7, 5], dtypes.int32)
-      off = gen_array_ops.concat_offset(cdim, [s0, s1])
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   r"Concat dim is out of range: 4 vs. 3"):
-        sess.run(off)
+    cdim = constant_op.constant(4, dtypes.int32)
+    s0 = constant_op.constant([2, 3, 5], dtypes.int32)
+    s1 = constant_op.constant([2, 7, 5], dtypes.int32)
+    off = gen_array_ops.concat_offset(cdim, [s0, s1])
+    with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                 r"Concat dim is out of range: 4 vs. 3"):
+      self.evaluate(off)
 
   def testDimMismatch(self):
-    with self.cached_session() as sess:
-      cdim = constant_op.constant(1, dtypes.int32)
-      s0 = constant_op.constant([2, 3, 5], dtypes.int32)
-      s1 = constant_op.constant([2, 7, 5, 10], dtypes.int32)
-      off = gen_array_ops.concat_offset(cdim, [s0, s1])
-      with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
-                                   r"should contain 3 elem"):
-        sess.run(off)
+    cdim = constant_op.constant(1, dtypes.int32)
+    s0 = constant_op.constant([2, 3, 5], dtypes.int32)
+    s1 = constant_op.constant([2, 7, 5, 10], dtypes.int32)
+    off = gen_array_ops.concat_offset(cdim, [s0, s1])
+    with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                 r"should contain 3 elem"):
+      self.evaluate(off)
 
   def testSizeMismatch(self):
-    with self.cached_session() as sess:
-      cdim = constant_op.constant(1, dtypes.int32)
-      s0 = constant_op.constant([2, 3, 5], dtypes.int32)
-      s1 = constant_op.constant([2, 7, 10], dtypes.int32)
-      off = gen_array_ops.concat_offset(cdim, [s0, s1])
-      with self.assertRaisesRegexp(
-          errors_impl.InvalidArgumentError,
-          r"All dimensions except 1 must match. Input 1 has shape \[2 7 10\] "
-          r"and doesn't match input 0 with shape \[2 3 5\]."):
-        sess.run(off)
+    cdim = constant_op.constant(1, dtypes.int32)
+    s0 = constant_op.constant([2, 3, 5], dtypes.int32)
+    s1 = constant_op.constant([2, 7, 10], dtypes.int32)
+    off = gen_array_ops.concat_offset(cdim, [s0, s1])
+    with self.assertRaisesRegexp(
+        errors_impl.InvalidArgumentError,
+        r"All dimensions except 1 must match. Input 1 has shape \[2 7 10\] "
+        r"and doesn't match input 0 with shape \[2 3 5\]."):
+      self.evaluate(off)
 
   def testNegativeDim(self):
-    with self.session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       cdim = constant_op.constant(-2, dtypes.int32)
       s0 = constant_op.constant([2, 3, 5], dtypes.int32)
       s1 = constant_op.constant([2, 7, 5], dtypes.int32)
       s2 = constant_op.constant([2, 20, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = sess.run(off)
+      ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
       cdim = constant_op.constant(-3, dtypes.int32)
@@ -687,7 +683,7 @@ class ConcatOffsetTest(test.TestCase):
       s1 = constant_op.constant([1, 3, 5], dtypes.int32)
       s2 = constant_op.constant([3, 3, 5], dtypes.int32)
       off = gen_array_ops.concat_offset(cdim, [s0, s1, s2])
-      ans = sess.run(off)
+      ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [2, 0, 0], [3, 0, 0]])
 
 
diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
index 893cb7cce3..7ee1a4bc32 100644
--- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
@@ -111,7 +111,7 @@ class ConditionalAccumulatorTest(test.TestCase):
         for e in elems:
           q.apply_grad((e,)).run()
 
-        result = sess.run(q.take_grad(1))
+        result = self.evaluate(q.take_grad(1))
 
         self.assertEqual(sum(elems) / len(elems), result)
 
@@ -424,7 +424,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_grad(1)
 
       def apply_grad(accum_op):
-        sess.run(accum_op)
+        self.evaluate(accum_op)
 
       threads = [
           self.checkedThread(
@@ -451,14 +451,14 @@ class ConditionalAccumulatorTest(test.TestCase):
       def apply_grad():
         for accum_op in accum_ops:
           time.sleep(1.0)
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       apply_grad_thread = self.checkedThread(target=apply_grad)
 
       results = []
 
       def take_grad():
-        results.append(sess.run(takeg_t))
+        results.append(self.evaluate(takeg_t))
 
       threads = [self.checkedThread(target=take_grad) for _ in range(10)]
 
@@ -485,12 +485,12 @@ class ConditionalAccumulatorTest(test.TestCase):
       def apply_grad():
         time.sleep(1.0)
         for accum_op in accum_ops:
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       return_array = []
 
       def take_grad():
-        return_array.append(sess.run(takeg_t))
+        return_array.append(self.evaluate(takeg_t))
 
       accum_thread = self.checkedThread(target=apply_grad)
       takeg_thread = self.checkedThread(target=take_grad)
@@ -503,7 +503,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
   def _blocking_takeg(self, sess, takeg_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(takeg_op)
+      self.evaluate(takeg_op)
 
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 9a198d445f..37654abd18 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -593,7 +593,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: [math_ops.add(x, 1), math_ops.add(x, 2)]
       fn2 = lambda: [y, y]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      self.assertAllEqual([11, 12], sess.run(r))
+      self.assertAllEqual([11, 12], self.evaluate(r))
 
   def testCondListOutput(self):
     with self.cached_session() as sess:
@@ -603,7 +603,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: [math_ops.add(x, y), math_ops.add(x, y)]
       fn2 = lambda: [y, y]
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertListEqual([210, 210], test_result)
 
   def testTupleOutput(self):
@@ -614,7 +614,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: (math_ops.add(x, y), math_ops.add(x, y))
       fn2 = lambda: (y, y)
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertTupleEqual((210, 210), test_result)
 
   def testDictOutput(self):
@@ -625,7 +625,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: {"a": math_ops.add(x, y), "b": math_ops.add(x, y)}
       fn2 = lambda: {"a": y, "b": y}
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertDictEqual({"a": 210, "b": 210}, test_result)
 
   def testEmbeddedListOutput(self):
@@ -638,7 +638,7 @@ class ControlFlowTest(test.TestCase):
       # Pass strict=True flag as cond_v2 allows for tensors to be
       # in nested output structures as singletons
       r = control_flow_ops.cond(pred, fn1, fn2, strict=True)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertListEqual([[210, 210]], test_result)
 
   def testEmbeddedTupleOutput(self):
@@ -649,7 +649,7 @@ class ControlFlowTest(test.TestCase):
       fn1 = lambda: ((math_ops.add(x, y), math_ops.add(x, y)))
       fn2 = lambda: ((y, y))
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertTupleEqual(((210, 210)), test_result)
 
   def testEmbeddedDictOutput(self):
@@ -662,7 +662,7 @@ class ControlFlowTest(test.TestCase):
       fn2 = lambda: {"a": {"c": y},
                      "b": {"d": y}}
       r = control_flow_ops.cond(pred, fn1, fn2)
-      test_result = sess.run(r)
+      test_result = self.evaluate(r)
       self.assertDictEqual({"a": {"c": 210}, "b": {"d": 210}}, test_result)
 
   def testCheckNestedOutputStruct(self):
@@ -677,7 +677,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError, v2_msg if control_flow_ops.ENABLE_COND_V2 else v1_msg):
         r = control_flow_ops.cond(pred, fn1, fn2)
-        test_result = sess.run(r)
+        self.evaluate(r)
 
   def testCondRef(self):
 
@@ -731,7 +731,7 @@ class ControlFlowTest(test.TestCase):
       with ops.control_dependencies([v_t_op]):
         orig_v = array_ops.identity(v)
       merged_op = control_flow_ops.merge([assign_v, orig_v])
-      self.assertAllEqual([1.0], sess.run(merged_op.output))
+      self.assertAllEqual([1.0], self.evaluate(merged_op.output))
 
   def testCondSwitchIdentity(self):
     # Make sure the recv identity is not removed by optimization.
@@ -745,7 +745,7 @@ class ControlFlowTest(test.TestCase):
         return control_flow_ops.Assert(False, ["Wrong branch!!!"])
 
       r = control_flow_ops.cond(pred, fn1, fn2)
-      sess.run(r)
+      self.evaluate(r)
 
   def testCondRecvIdentity(self):
     # Make sure the switch identity is not removed by optimization.
@@ -761,7 +761,7 @@ class ControlFlowTest(test.TestCase):
           return control_flow_ops.Assert(False, ["Wrong branch!!!"])
 
       r = control_flow_ops.cond(pred, fn1, fn2)
-      sess.run(r)
+      self.evaluate(r)
 
   def testCondGrad_1(self):
     with self.cached_session():
@@ -1050,7 +1050,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(r[0].dtype, dtypes.int32)
       self.assertEqual(r[1].dtype, dtypes.int32_ref)
 
-      value_i, value_x = sess.run(r)
+      value_i, value_x = self.evaluate(r)
 
     self.assertEqual(100, value_i)
     self.assertEqual(0, value_x)
@@ -1642,7 +1642,7 @@ class ControlFlowTest(test.TestCase):
       with ops.control_dependencies([control_flow_ops.no_op()]):
         loop = control_flow_ops.while_loop(cond, body,
                                            (constant_op.constant(5),))
-      self.assertEqual(0, sess.run(loop))
+      self.assertEqual(0, self.evaluate(loop))
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
   def testWhileCondWithControl_1(self):
@@ -2055,7 +2055,7 @@ class ControlFlowTest(test.TestCase):
         self.assertFalse(gpu_dev_name in dev)
 
     with self.session(graph=graph) as sess:
-      self.assertAllClose(1024.0, sess.run(r))
+      self.assertAllClose(1024.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/116351701 (colocation)")
   def testWhileGrad_ColocateGradients(self):
@@ -2133,7 +2133,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [v], parallel_iterations=p_iters)
 
       grad_a, grad_v = gradients_impl.gradients(r, [a, v])
-      grad_a_val, grad_v_val = sess.run([grad_a, grad_v])
+      grad_a_val, grad_v_val = self.evaluate([grad_a, grad_v])
       self.assertAllClose(216.0, grad_a_val)
       self.assertAllClose(81.0, grad_v_val)
 
@@ -2264,7 +2264,7 @@ class ControlFlowTest(test.TestCase):
     i, x = control_flow_ops.while_loop(lambda i, x: i < 3, outer_body, [0, 0.0])
 
     with self.cached_session() as sess:
-      i_val, x_val = sess.run([i, x])
+      i_val, x_val = self.evaluate([i, x])
       self.assertEqual(i_val, 3)
       self.assertAllClose(x_val, 1.0)
 
@@ -2293,7 +2293,7 @@ class ControlFlowTest(test.TestCase):
 
       r_flattened = nest.flatten(r)
       self.assertEqual([100.0, 1.0, 102.0, 3.0, 4.0 + 100 * 2.0],
-                       sess.run(r_flattened))
+                       self.evaluate(r_flattened))
 
   def testWhile_NestedBadArityFails(self):
     with self.cached_session():
@@ -2547,8 +2547,8 @@ class ControlFlowTest(test.TestCase):
       res = outer_loop(inp)
       optimizer = adam.AdamOptimizer(learning_rate=0.001)
       train_op = optimizer.minimize(math_ops.reduce_mean(math_ops.square(res)))
-      sess.run(variables.global_variables_initializer())
-      sess.run(train_op)
+      self.evaluate(variables.global_variables_initializer())
+      self.evaluate(train_op)
       self.assertAllClose(2.999, self.evaluate(var))
 
   def _testWhileCondGrad_Simple(self, use_gpu):
@@ -2607,11 +2607,11 @@ class ControlFlowTest(test.TestCase):
           [i0.get_shape(), tensor_shape.TensorShape([None, 2])])
       s = math_ops.reduce_sum(h)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       optimizer = gradient_descent.GradientDescentOptimizer(0.01)
       op = optimizer.minimize(s)
-      sess.run(op)
-      self.assertAllClose([[0.98000002, 1.98000002]], sess.run(x))
+      self.evaluate(op)
+      self.assertAllClose([[0.98000002, 1.98000002]], self.evaluate(x))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileWithRefsWithGradients_1(self):
@@ -2705,7 +2705,7 @@ class ControlFlowTest(test.TestCase):
 
       output_grad = control_flow_ops.while_loop(
           c, b, [i0, constant_op.constant(0.0)])
-      self.assertAllClose(600.0, sess.run(output_grad)[1])
+      self.assertAllClose(600.0, self.evaluate(output_grad)[1])
 
   def testWhileAndTensorArray(self):
     with self.cached_session() as sess:
@@ -2724,7 +2724,7 @@ class ControlFlowTest(test.TestCase):
 
       r = control_flow_ops.while_loop(c, b, [n0, y0], parallel_iterations=1)
       r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(107520.0, sess.run(r))
+      self.assertAllClose(107520.0, self.evaluate(r))
 
   def testWhileGrad_StopGrad(self):
     with self.cached_session():
@@ -2857,8 +2857,8 @@ class ControlFlowTest(test.TestCase):
     dy_dq, = gradients_impl.gradients(y, q)
     self.assertIsNotNone(dy_dq)
     with self.cached_session() as sess:
-      sess.run(q.initializer)
-      self.assertAllClose([0., 0.], sess.run(dy_dq))
+      self.evaluate(q.initializer)
+      self.assertAllClose([0., 0.], self.evaluate(dy_dq))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
   def testWhileGradientWithNontrainablePath2(self):
@@ -2875,8 +2875,8 @@ class ControlFlowTest(test.TestCase):
     dy_dq, = gradients_impl.gradients(y, q)
     self.assertIsNotNone(dy_dq)
     with self.cached_session() as sess:
-      sess.run(q.initializer)
-      self.assertAllClose([1., 1.], sess.run(dy_dq))
+      self.evaluate(q.initializer)
+      self.assertAllClose([1., 1.], self.evaluate(dy_dq))
 
   @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   def testIssue16504(self):
@@ -3033,19 +3033,19 @@ class ControlFlowTest(test.TestCase):
           ((x > y, a), (x > y, b)), default=c, exclusive=True)
 
       variables.global_variables_initializer().run()
-      self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3)
+      self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1] * 3)
       self.assertEqual(2, self.evaluate(r2))
-      self.assertAllEqual(sess.run([v0, v1, v2]), [-1, -1, 2])
+      self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1, -1, 2])
 
       variables.global_variables_initializer().run()
-      self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3)
+      self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1] * 3)
       self.assertEqual(1, self.evaluate(r1))
-      self.assertAllEqual(sess.run([v0, v1, v2]), [-1, 1, -1])
+      self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1, 1, -1])
 
       variables.global_variables_initializer().run()
-      self.assertAllEqual(sess.run([v0, v1, v2]), [-1] * 3)
+      self.assertAllEqual(self.evaluate([v0, v1, v2]), [-1] * 3)
       self.assertEqual(0, self.evaluate(r0))
-      self.assertAllEqual(sess.run([v0, v1, v2]), [0, -1, -1])
+      self.assertAllEqual(self.evaluate([v0, v1, v2]), [0, -1, -1])
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
   def testOneOpCond(self):
@@ -3083,7 +3083,7 @@ class ControlFlowTest(test.TestCase):
 
       # Fetching v directly will result in an uninitialized error
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        sess.run([c, v])
+        self.evaluate([c, v])
 
       # Use a control dependency to ensure init_variable is run
       # while asking for c
@@ -3091,7 +3091,7 @@ class ControlFlowTest(test.TestCase):
           name="real_tensor",
           output_tensor=v._ref(),  # pylint: disable=protected-access
           dependencies=[v.initializer])
-      c_val, real_v_val = sess.run([c, real_v])
+      c_val, real_v_val = self.evaluate([c, real_v])
 
     # Ensure the result of 'real_c' is the same as 'c'
     self.assertAllEqual(10, c_val)
@@ -3184,7 +3184,7 @@ class ControlFlowTest(test.TestCase):
 
       # Runs "init" before fetching v1 and v2.
       init.run()
-      v1_val, v2_val = sess.run([v1, v2])
+      v1_val, v2_val = self.evaluate([v1, v2])
 
     # Ensure that v1 and v2 are initialized
     self.assertAllClose([0.0], v1_val)
@@ -3295,7 +3295,7 @@ class ControlFlowTest(test.TestCase):
 
       result = control_flow_ops.while_loop(condition, body,
                                            [constant_op.constant(4)])
-      self.assertEqual(10, sess.run(result))
+      self.assertEqual(10, self.evaluate(result))
 
       # Ensure that we cannot run a tensor that escapes the loop body
       # accidentally.
@@ -3345,7 +3345,7 @@ class ControlFlowTest(test.TestCase):
       cond = constant_op.constant(True, dtypes.bool)
       v_f, v_t = control_flow_ops.switch(constant_qint, cond)
       result = control_flow_ops.merge([v_f, v_t])
-      sess.run(result)
+      self.evaluate(result)
 
   def testQIntRefSwitchMerge(self):
     with self.cached_session(use_gpu=test.is_gpu_available()) as sess:
@@ -3353,12 +3353,12 @@ class ControlFlowTest(test.TestCase):
           shape=[1], dtype=dtypes.qint8, name="v", container="", shared_name="")
       assign_op = state_ops.assign(
           var_qint, constant_op.constant(np.array([42]), dtypes.qint8))
-      sess.run(assign_op)
+      self.evaluate(assign_op)
 
       cond = constant_op.constant(True, dtypes.bool)
       v_f, v_t = control_flow_ops.ref_switch(var_qint, cond)
       result = control_flow_ops.ref_merge([v_f, v_t])
-      sess.run(result)
+      self.evaluate(result)
 
   def testUInt64SwitchMerge(self):
     with self.cached_session(force_gpu=test.is_gpu_available()) as sess:
@@ -3366,7 +3366,7 @@ class ControlFlowTest(test.TestCase):
       cond = constant_op.constant(True, dtypes.bool)
       v_f, v_t = control_flow_ops.switch(constant_uint64, cond)
       result = control_flow_ops.merge([v_f, v_t])
-      sess.run(result)
+      self.evaluate(result)
 
   def testQIntArgAndRet(self):
 
@@ -3377,7 +3377,7 @@ class ControlFlowTest(test.TestCase):
     with self.cached_session(force_gpu=test.is_gpu_available()) as sess:
       qint = constant_op.constant(np.array([42]), dtypes.qint8)
       result = func(qint)
-      sess.run(result)
+      self.evaluate(result)
 
 
 class ControlFlowContextCheckTest(test.TestCase):
@@ -3682,7 +3682,7 @@ class WhileOpBenchmark(test.Benchmark):
     with session.Session() as sess, ops.device(default_device):
       # Get the initial id i, input x, and kernel.
       i, x, kernel = self._getInitVariables()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       if static_unroll:
         for _ in xrange(steps):
@@ -3701,11 +3701,11 @@ class WhileOpBenchmark(test.Benchmark):
 
       for _ in xrange(3):
         # exclude warm up time
-        sess.run(r)
+        self.evaluate(r)
 
       start_time = time.time()
       for _ in xrange(num_iters):
-        sess.run(r)
+        self.evaluate(r)
       return (time.time() - start_time) / num_iters
 
   def benchmarkWhileOpCrossDevicePlacement(self):
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index 3924e13575..3ec5c29df7 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -109,7 +109,7 @@ class Conv3DTest(test.TestCase):
         results.append(result)
 
       with self.cached_session() as sess:
-        values = sess.run(results)
+        values = self.evaluate(results)
         for value in values:
           print("expected = ", expected)
           print("actual = ", value)
@@ -184,8 +184,8 @@ class Conv3DTest(test.TestCase):
         computed_results.append(computed)
         tolerance = 1e-2 if use_gpu else 1e-5
         with self.cached_session() as sess:
-          expected_values = sess.run(expected_results)
-          computed_values = sess.run(computed_results)
+          expected_values = self.evaluate(expected_results)
+          computed_values = self.evaluate(computed_results)
           for e_value, c_value in zip(expected_values, computed_values):
             print("expected = ", e_value)
             print("actual = ", c_value)
@@ -715,8 +715,8 @@ class Conv3DTest(test.TestCase):
         expected_grad = gradients_impl.gradients(expected, t1
                                                  if mode == "input" else t2)[0]
         # "values" consists of two tensors for two backprops
-        actual_value = sess.run(actual_grad)
-        expected_value = sess.run(expected_grad)
+        actual_value = self.evaluate(actual_grad)
+        expected_value = self.evaluate(expected_grad)
         self.assertShapeEqual(actual_value, actual_grad)
         self.assertShapeEqual(expected_value, expected_grad)
       print("expected = ", expected_value)
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 835cc1504d..2f6f3bb383 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -908,8 +908,8 @@ class Conv2DTest(test.TestCase):
         conv = gradients_impl.gradients(conv_forward, t1)[0]
         conv_2 = gradients_impl.gradients(conv_forward_2, t1)[0]
         # "values" consists of two tensors for two backprops
-        value = sess.run(conv)
-        value_2 = sess.run(conv_2)
+        value = self.evaluate(conv)
+        value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
       tf_logging.info("expected = ", value_2)
@@ -961,8 +961,8 @@ class Conv2DTest(test.TestCase):
           conv_forward_2 = test_util.NCHWToNHWC(conv_forward_2)
         conv = gradients_impl.gradients(conv_forward, t2)[0]
         conv_2 = gradients_impl.gradients(conv_forward, t2)[0]
-        value = sess.run(conv)
-        value_2 = sess.run(conv_2)
+        value = self.evaluate(conv)
+        value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
       tf_logging.info("expected = ", value_2)
@@ -1545,7 +1545,7 @@ class DepthwiseConv2DTest(test.TestCase):
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
       conv = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     tf_logging.info("value = ", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
@@ -1667,7 +1667,7 @@ class SeparableConv2DTest(test.TestCase):
       if data_format == "NCHW":
         conv = array_ops.transpose(conv, [0, 2, 3, 1])
 
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     tf_logging.info("value = ", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
@@ -1774,10 +1774,10 @@ class DeepConv2DTest(test.TestCase):
       conv = nn_ops.conv2d(t1, t2, strides=strides, padding=padding)
 
       os.environ["TF_USE_DEEP_CONV2D"] = "0"
-      values_expect = sess.run([conv])
+      values_expect = self.evaluate([conv])
 
       os.environ["TF_USE_DEEP_CONV2D"] = "1"
-      values_test = sess.run([conv])
+      values_test = self.evaluate([conv])
 
       self.assertAllClose(values_expect, values_test, rtol=1e-5, atol=1e-5)
 
diff --git a/tensorflow/python/kernel_tests/ctc_loss_op_test.py b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
index b38776ec5b..36cae2846c 100644
--- a/tensorflow/python/kernel_tests/ctc_loss_op_test.py
+++ b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
@@ -98,12 +98,12 @@ class CTCLossTest(test.TestCase):
       self.assertShapeEqual(grad_truth, grad)
 
       if expected_err_re is None:
-        (tf_loss, tf_grad) = sess.run([loss, grad])
+        (tf_loss, tf_grad) = self.evaluate([loss, grad])
         self.assertAllClose(tf_loss, loss_truth, atol=1e-6)
         self.assertAllClose(tf_grad, grad_truth, atol=1e-6)
       else:
         with self.assertRaisesOpError(expected_err_re):
-          sess.run([loss, grad])
+          self.evaluate([loss, grad])
 
   def testBasic(self):
     """Test two batch entries."""
@@ -266,7 +266,7 @@ class CTCLossTest(test.TestCase):
           sequence_length=seq_lens,
           time_major=False)
 
-      (tf_loss, tf_loss_transposed) = sess.run([loss, loss_transposed])
+      (tf_loss, tf_loss_transposed) = self.evaluate([loss, loss_transposed])
       self.assertAllEqual(tf_loss, tf_loss_transposed)
 
   def testInvalidSecondGradient(self):
@@ -332,9 +332,10 @@ class CTCLossTestV2(test.TestCase):
 
     def assert_same_loss_and_grads(loss):
       with self.cached_session() as sess:
-        self.assertAllClose(*sess.run([loss, ref_loss]))
+        self.assertAllClose(*self.evaluate([loss, ref_loss]))
         grad = gradients_impl.gradients(loss, [logits])
-        self.assertAllClose(*sess.run([grad, ref_grad]), rtol=2e-06, atol=2e-06)
+        self.assertAllClose(
+            *self.evaluate([grad, ref_grad]), rtol=2e-06, atol=2e-06)
 
     assert_same_loss_and_grads(
         ctc_ops.ctc_loss_v2(
@@ -391,9 +392,11 @@ class CTCLossTestV2(test.TestCase):
 
       with self.cached_session() as sess:
         for _ in range(32):
-          self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
-          self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
-                              rtol=2e-06, atol=2e-06)
+          self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss]))
+          self.assertAllClose(
+              *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]),
+              rtol=2e-06,
+              atol=2e-06)
 
   def testCtcLossDenseUniqueFastPathIsSameAsCtcLoss(self):
     random_seed.set_random_seed(5)
@@ -442,9 +445,11 @@ class CTCLossTestV2(test.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(32):
-        self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
-        self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
-                            rtol=2e-06, atol=2e-06)
+        self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss]))
+        self.assertAllClose(
+            *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]),
+            rtol=2e-06,
+            atol=2e-06)
 
   def testCtcLossDenseWithBlankIndexIsSameAsCtcLoss(self):
     random_seed.set_random_seed(5)
@@ -496,9 +501,11 @@ class CTCLossTestV2(test.TestCase):
 
     with self.cached_session() as sess:
       for _ in range(32):
-        self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
-        self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
-                            rtol=2e-06, atol=2e-06)
+        self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss]))
+        self.assertAllClose(
+            *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]),
+            rtol=2e-06,
+            atol=2e-06)
 
   def testCtcLossDenseWithNegativeBlankIndexIsSameAsCtcLoss(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
@@ -542,9 +549,11 @@ class CTCLossTestV2(test.TestCase):
 
       with self.cached_session() as sess:
         for _ in range(32):
-          self.assertAllClose(*sess.run([ctc_loss, tf_nn_ctc_loss]))
-          self.assertAllClose(*sess.run([ctc_loss_grads, tf_nn_ctc_grads]),
-                              rtol=2e-06, atol=2e-06)
+          self.assertAllClose(*self.evaluate([ctc_loss, tf_nn_ctc_loss]))
+          self.assertAllClose(
+              *self.evaluate([ctc_loss_grads, tf_nn_ctc_grads]),
+              rtol=2e-06,
+              atol=2e-06)
 
   def testCollapseRepeated(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
diff --git a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
index df166b6101..fc7d4572e2 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
@@ -83,17 +83,17 @@ class BinaryOpTest(test.TestCase):
       out = tf_func(inx, iny)
       tf_cpu = self.evaluate(out)
       # Test that the op takes precedence over numpy operators.
-      np_left = tf_func(x, iny).eval()
-      np_right = tf_func(inx, y).eval()
+      np_left = self.evaluate(tf_func(x, iny))
+      np_right = self.evaluate(tf_func(inx, y))
 
       if also_compare_variables:
         var_x = variables.Variable(x)
         var_y = variables.Variable(y)
-        variables.global_variables_initializer().run()
+        self.evaluate(variables.global_variables_initializer())
         print(type(x), type(y), type(var_x), type(var_y))
         print(type(tf_func(x, var_y)), type(tf_func(var_x, y)))
-        np_var_left = tf_func(x, var_y).eval()
-        np_var_right = tf_func(var_x, y).eval()
+        np_var_left = self.evaluate(tf_func(x, var_y))
+        np_var_right = self.evaluate(tf_func(var_x, y))
 
     if np_ans.dtype != np.object:
       self.assertAllClose(np_ans, tf_cpu)
@@ -253,7 +253,7 @@ class BinaryOpTest(test.TestCase):
     var_x = variables.Variable(x)
     var_y = variables.Variable(y)
     with self.cached_session() as sess:
-      sess.run([var_x.initializer, var_y.initializer])
+      self.evaluate([var_x.initializer, var_y.initializer])
       left_result = (var_x * y).eval()
       right_result = (x * var_y).eval()
     np_result = x * y
@@ -385,7 +385,7 @@ class BinaryOpTest(test.TestCase):
     with self.test_session(use_gpu=False) as sess:
       cmp_eq = math_ops.equal(x, y)
       cmp_not_eq = math_ops.not_equal(x, y)
-      values = sess.run([cmp_eq, cmp_not_eq])
+      values = self.evaluate([cmp_eq, cmp_not_eq])
       self.assertAllEqual([[True, True], [False, False]], values[0])
       self.assertAllEqual([[False, False], [True, True]], values[1])
 
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index d7dbf5ab9a..ab116c400a 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -572,7 +572,7 @@ class MinMaxOpTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       omin, omax = math_ops.minimum(inx, iny), math_ops.maximum(inx, iny)
-      tf_min, tf_max = sess.run([omin, omax])
+      tf_min, tf_max = self.evaluate([omin, omax])
     self.assertAllEqual(np_min, tf_min)
     self.assertAllEqual(np_max, tf_max)
 
@@ -662,8 +662,8 @@ class MathOpsOverloadTest(test.TestCase):
   def _compareUnary(self, x, dtype, np_func, tf_func):
     np_ans = np_func(x).astype(dtype.as_numpy_dtype)
     with self.test_session(use_gpu=False):
-      self.assertAllClose(np_ans,
-                          tf_func(ops.convert_to_tensor(x, dtype=dtype)).eval())
+      self.assertAllClose(
+          np_ans, self.evaluate(tf_func(ops.convert_to_tensor(x, dtype=dtype))))
 
   def testOverload(self):
     dtypes = [
@@ -736,7 +736,7 @@ class IsFiniteInfNanTest(test.TestCase):
       inx = ops.convert_to_tensor(x)
       ofinite, oinf, onan = math_ops.is_finite(inx), math_ops.is_inf(
           inx), math_ops.is_nan(inx)
-      tf_finite, tf_inf, tf_nan = sess.run([ofinite, oinf, onan])
+      tf_finite, tf_inf, tf_nan = self.evaluate([ofinite, oinf, onan])
     self.assertAllEqual(np_inf, tf_inf)
     self.assertAllEqual(np_nan, tf_nan)
     self.assertAllEqual(np_finite, tf_finite)
@@ -788,7 +788,7 @@ class RoundingTest(test.TestCase):
     y = np.rint(x) if y is None else np.asarray(y)
     with self.cached_session() as sess:
       tf_rint = math_ops.rint(x)
-      np_rint = sess.run(tf_rint)
+      np_rint = self.evaluate(tf_rint)
     self.assertAllEqual(y, np_rint)
     self.assertShapeEqual(y, tf_rint)
 
@@ -797,7 +797,7 @@ class RoundingTest(test.TestCase):
     with self.cached_session() as sess:
       inx = ops.convert_to_tensor(x)
       ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx)
-      tf_floor, tf_ceil = sess.run([ofloor, oceil])
+      tf_floor, tf_ceil = self.evaluate([ofloor, oceil])
     self.assertAllEqual(np_floor, tf_floor)
     self.assertAllEqual(np_ceil, tf_ceil)
     self.assertShapeEqual(np_floor, ofloor)
@@ -881,7 +881,7 @@ class ComplexMakeRealImagTest(test.TestCase):
         force_gpu=use_gpu and test_util.is_gpu_available()) as sess:
       inx = ops.convert_to_tensor(cplx)
       tf_angle = math_ops.angle(inx)
-      tf_angle_val = sess.run(tf_angle)
+      tf_angle_val = self.evaluate(tf_angle)
     self.assertAllEqual(np_angle, tf_angle_val)
     self.assertShapeEqual(np_angle, tf_angle)
 
diff --git a/tensorflow/python/kernel_tests/decode_image_op_test.py b/tensorflow/python/kernel_tests/decode_image_op_test.py
index 7a8743e11f..267afdeb5e 100644
--- a/tensorflow/python/kernel_tests/decode_image_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_image_op_test.py
@@ -40,7 +40,7 @@ class DecodeImageOpTest(test.TestCase):
       bmp0 = io_ops.read_file(path)
       image0 = image_ops.decode_image(bmp0)
       image1 = image_ops.decode_bmp(bmp0)
-      bmp0, image0, image1 = sess.run([bmp0, image0, image1])
+      bmp0, image0, image1 = self.evaluate([bmp0, image0, image1])
       self.assertEqual(len(bmp0), 4194)
       self.assertAllEqual(image0, image1)
 
@@ -56,7 +56,7 @@ class DecodeImageOpTest(test.TestCase):
       gif0 = io_ops.read_file(path)
       image0 = image_ops.decode_image(gif0)
       image1 = image_ops.decode_gif(gif0)
-      gif0, image0, image1 = sess.run([gif0, image0, image1])
+      gif0, image0, image1 = self.evaluate([gif0, image0, image1])
 
       self.assertEqual(image0.shape, shape)
       self.assertAllEqual(image0, image1)
@@ -85,7 +85,7 @@ class DecodeImageOpTest(test.TestCase):
       jpeg0 = io_ops.read_file(path)
       image0 = image_ops.decode_image(jpeg0)
       image1 = image_ops.decode_jpeg(jpeg0)
-      jpeg0, image0, image1 = sess.run([jpeg0, image0, image1])
+      jpeg0, image0, image1 = self.evaluate([jpeg0, image0, image1])
       self.assertEqual(len(jpeg0), 3771)
       self.assertEqual(image0.shape, (256, 128, 3))
       self.assertAllEqual(image0, image1)
@@ -104,7 +104,7 @@ class DecodeImageOpTest(test.TestCase):
           png0 = io_ops.read_file(path)
           image0 = image_ops.decode_image(png0, channels=channels)
           image1 = image_ops.decode_png(png0, channels=channels)
-          png0, image0, image1 = sess.run([png0, image0, image1])
+          png0, image0, image1 = self.evaluate([png0, image0, image1])
           self.assertEqual(image0.shape, (26, 51, channels or channels_in))
           self.assertAllEqual(image0, image1)
 
diff --git a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
index 66b3e0f22f..f8fc28062f 100644
--- a/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_jpeg_op_test.py
@@ -80,7 +80,7 @@ class DecodeJpegBenchmark(test.Benchmark):
           initializer=image_ops.encode_jpeg(tiled_image))
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       images = []
       for _ in xrange(parallelism):
         if crop_window is None:
@@ -105,11 +105,11 @@ class DecodeJpegBenchmark(test.Benchmark):
 
       for _ in xrange(3):
         # Skip warm up time.
-        sess.run(r)
+        self.evaluate(r)
 
       start_time = time.time()
       for _ in xrange(num_iters):
-        sess.run(r)
+        self.evaluate(r)
       end_time = time.time()
     return end_time - start_time
 
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
index 3ed7dba966..0676664685 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
@@ -43,7 +43,7 @@ class AssignOpTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       def run_add(add_op):
-        sess.run(add_op)
+        self.evaluate(add_op)
 
       threads = [
           self.checkedThread(
@@ -70,7 +70,7 @@ class AssignOpTest(test.TestCase):
       variables.global_variables_initializer().run()
 
       def run_assign(assign_op):
-        sess.run(assign_op)
+        self.evaluate(assign_op)
 
       threads = [
           self.checkedThread(
@@ -103,7 +103,7 @@ class AssignOpTest(test.TestCase):
       p.initializer.run()
 
       def run_add(add_op):
-        sess.run(add_op)
+        self.evaluate(add_op)
 
       threads = [
           self.checkedThread(
@@ -131,7 +131,7 @@ class AssignOpTest(test.TestCase):
       p.initializer.run()
 
       def run_assign(assign_op):
-        sess.run(assign_op)
+        self.evaluate(assign_op)
 
       threads = [
           self.checkedThread(
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index c4bed11080..19f145865f 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -277,7 +277,7 @@ class DepthToSpaceTest(test.TestCase):
       actual = array_ops.depth_to_space(t, block_size, data_format=data_format)
 
     with self.session(use_gpu=use_gpu) as sess:
-      actual_vals, expected_vals = sess.run([actual, expected])
+      actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
   def testAgainstTranspose(self):
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index f65d0be367..f6d834c2f8 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -162,7 +162,7 @@ class DepthwiseConv2DTest(test.TestCase):
         conv_native = array_ops.transpose(conv_native, [0, 2, 3, 1])
 
       try:
-        native_result = sess.run(conv_native)
+        native_result = self.evaluate(conv_native)
       except errors.InvalidArgumentError as e:
         # Grouped convolution kernel is only registered for cuDNN 7. Silently
         # return when we are running on an earlier version or without GPU.
@@ -174,7 +174,7 @@ class DepthwiseConv2DTest(test.TestCase):
 
       conv_interface = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      interface_result = sess.run(conv_interface)
+      interface_result = self.evaluate(conv_interface)
 
     tf_logging.info(
         "data_type: %r, use_gpu: %r, grouped_conv: %r, max diff = %f",
@@ -269,7 +269,7 @@ class DepthwiseConv2DTest(test.TestCase):
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
       conv = nn_ops.depthwise_conv2d_native(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     tf_logging.info("value = %r", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py
index 602ceb6ebd..d6ef9e70b8 100644
--- a/tensorflow/python/kernel_tests/determinant_op_test.py
+++ b/tensorflow/python/kernel_tests/determinant_op_test.py
@@ -156,7 +156,7 @@ class DeterminantOpTest(test.TestCase):
       matrix2 = random_ops.random_normal([5, 5], seed=42)
       det1 = linalg_ops.matrix_determinant(matrix1)
       det2 = linalg_ops.matrix_determinant(matrix2)
-      det1_val, det2_val = sess.run([det1, det2])
+      det1_val, det2_val = self.evaluate([det1, det2])
       self.assertEqual(det1_val, det2_val)
 
 
diff --git a/tensorflow/python/kernel_tests/distributions/categorical_test.py b/tensorflow/python/kernel_tests/distributions/categorical_test.py
index f116c54bd1..9c593d2737 100644
--- a/tensorflow/python/kernel_tests/distributions/categorical_test.py
+++ b/tensorflow/python/kernel_tests/distributions/categorical_test.py
@@ -287,7 +287,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     }
 
     with self.cached_session() as sess:
-      run_result = sess.run(to_run)
+      run_result = self.evaluate(to_run)
 
     self.assertAllEqual(run_result["cat_prob"].shape,
                         run_result["norm_prob"].shape)
@@ -462,7 +462,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
           b = categorical.Categorical(logits=b_logits)
 
           kl = kullback_leibler.kl_divergence(a, b)
-          kl_val = sess.run(kl)
+          kl_val = self.evaluate(kl)
           # Make sure KL(a||a) is 0
           kl_same = sess.run(kullback_leibler.kl_divergence(a, a))
 
diff --git a/tensorflow/python/kernel_tests/distributions/special_math_test.py b/tensorflow/python/kernel_tests/distributions/special_math_test.py
index 6b6de8b139..0f800b95fa 100644
--- a/tensorflow/python/kernel_tests/distributions/special_math_test.py
+++ b/tensorflow/python/kernel_tests/distributions/special_math_test.py
@@ -448,7 +448,7 @@ class LogCDFLaplaceTest(test.TestCase):
       actual = sm.log_cdf_laplace(grid)
       grad = gradients_impl.gradients(actual, grid)[0]
 
-      actual_, grad_ = sess.run([actual, grad])
+      actual_, grad_ = self.evaluate([actual, grad])
 
       # isfinite checks for NaN and Inf.
       self.assertAllTrue(np.isfinite(actual_))
@@ -467,7 +467,7 @@ class LogCDFLaplaceTest(test.TestCase):
       actual = sm.log_cdf_laplace(grid)
       grad = gradients_impl.gradients(actual, grid)[0]
 
-      actual_, grad_ = sess.run([actual, grad])
+      actual_, grad_ = self.evaluate([actual, grad])
 
       # isfinite checks for NaN and Inf.
       self.assertAllTrue(np.isfinite(actual_))
diff --git a/tensorflow/python/kernel_tests/distributions/util_test.py b/tensorflow/python/kernel_tests/distributions/util_test.py
index f4e651b25b..d3fa513f05 100644
--- a/tensorflow/python/kernel_tests/distributions/util_test.py
+++ b/tensorflow/python/kernel_tests/distributions/util_test.py
@@ -805,7 +805,7 @@ class ReduceWeightedLogSumExp(test.TestCase):
       w = constant_op.constant(w_)
       actual, actual_sgn = du.reduce_weighted_logsumexp(
           logx, w, axis=-1, return_sign=True)
-      [actual_, actual_sgn_] = sess.run([actual, actual_sgn])
+      [actual_, actual_sgn_] = self.evaluate([actual, actual_sgn])
     self.assertAllEqual(expected, actual_)
     self.assertAllEqual([-1., -1, 1], actual_sgn_)
 
@@ -823,7 +823,7 @@ class ReduceWeightedLogSumExp(test.TestCase):
       w = constant_op.constant(w_)
       actual, actual_sgn = du.reduce_weighted_logsumexp(
           logx, w, axis=-1, return_sign=True, keep_dims=True)
-      [actual_, actual_sgn_] = sess.run([actual, actual_sgn])
+      [actual_, actual_sgn_] = self.evaluate([actual, actual_sgn])
     self.assertAllEqual(expected, actual_)
     self.assertAllEqual([[-1.], [-1], [1]], actual_sgn_)
 
diff --git a/tensorflow/python/kernel_tests/division_future_test.py b/tensorflow/python/kernel_tests/division_future_test.py
index e477bdc73b..85c85809d3 100644
--- a/tensorflow/python/kernel_tests/division_future_test.py
+++ b/tensorflow/python/kernel_tests/division_future_test.py
@@ -65,7 +65,7 @@ class DivisionTestCase(test.TestCase):
                 tf_floordiv = tf_x // tf_y
                 check(floordiv, tf_floordiv)
       # Do only one sess.run for speed
-      for f, (x, y) in zip(checks, sess.run(tensors)):
+      for f, (x, y) in zip(checks, self.evaluate(tensors)):
         f(x, y)
 
 
diff --git a/tensorflow/python/kernel_tests/division_past_test.py b/tensorflow/python/kernel_tests/division_past_test.py
index 63951b5b38..38bb18631a 100644
--- a/tensorflow/python/kernel_tests/division_past_test.py
+++ b/tensorflow/python/kernel_tests/division_past_test.py
@@ -64,7 +64,7 @@ class DivisionTestCase(test.TestCase):
                 tf_floordiv = tf_x // tf_y
                 check(floordiv, tf_floordiv)
       # Do only one sess.run for speed
-      for f, (x, y) in zip(checks, sess.run(tensors)):
+      for f, (x, y) in zip(checks, self.evaluate(tensors)):
         f(x, y)
 
 
diff --git a/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py b/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py
index c655876280..6aa757e293 100644
--- a/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py
+++ b/tensorflow/python/kernel_tests/draw_bounding_box_op_test.py
@@ -87,7 +87,7 @@ class DrawBoundingBoxOpTest(test.TestCase):
       image = array_ops.expand_dims(image, 0)
       image = image_ops.draw_bounding_boxes(image, bboxes)
       with self.cached_session(use_gpu=False) as sess:
-        op_drawn_image = np.squeeze(sess.run(image), 0)
+        op_drawn_image = np.squeeze(self.evaluate(image), 0)
         self.assertAllEqual(test_drawn_image, op_drawn_image)
 
   def testDrawBoundingBoxRGBColorCycling(self):
diff --git a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
index 07da855a01..3622fde3f3 100644
--- a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
@@ -40,7 +40,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant([0, 0, 2, 3, 2, 1])
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([0, 13], partition_vals[0])
@@ -62,7 +62,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant([0, 0, 2, 3, 2, 1])
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([[0, 1, 2], [3, 4, 5]], partition_vals[0])
@@ -87,7 +87,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual(part1, partition_vals[0])
@@ -109,7 +109,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=num_partitions)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(num_partitions, len(partition_vals))
     for i in range(num_partitions):
@@ -125,7 +125,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([3 + 4j, 7 + 8j], partition_vals[0])
@@ -138,7 +138,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = 3
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual(np.array([], dtype=np.float64).reshape(-1, 4),
@@ -164,7 +164,7 @@ class DynamicPartitionTest(test.TestCase):
             outputs = data_flow_ops.dynamic_partition(
                 data_t, partitions_t, num_partitions=n)
             self.assertEqual(n, len(outputs))
-            outputs_val = sess.run(outputs)
+            outputs_val = self.evaluate(outputs)
             for i, output in enumerate(outputs_val):
               self.assertAllEqual(output, data[partitions == i])
 
@@ -183,7 +183,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(4, len(partition_vals))
     self.assertAllEqual([], partition_vals[0])
@@ -199,7 +199,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=3)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(3, len(partition_vals))
     self.assertAllEqual([[]], partition_vals[0])
@@ -215,7 +215,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([], partition_vals[0])
@@ -236,7 +236,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=2)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(2, len(partition_vals))
     self.assertAllEqual([6], partition_vals[0])
@@ -257,7 +257,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=5)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(5, len(partition_vals))
     self.assertAllEqual([5], partition_vals[0])
@@ -281,7 +281,7 @@ class DynamicPartitionTest(test.TestCase):
       indices = constant_op.constant(indices_list, dtype=dtypes.int32)
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=40)
-      partition_vals = sess.run(partitions)
+      partition_vals = self.evaluate(partitions)
 
     self.assertEqual(40, len(partition_vals))
     for i in range(40):
@@ -295,7 +295,7 @@ class DynamicPartitionTest(test.TestCase):
       partitions = data_flow_ops.dynamic_partition(
           data, indices, num_partitions=4)
       with self.assertRaisesOpError(r"partitions\[2\] = 99 is not in \[0, 4\)"):
-        sess.run(partitions)
+        self.evaluate(partitions)
 
   def testScalarIndexOutOfRange(self):
     with self.cached_session() as sess:
@@ -303,7 +303,7 @@ class DynamicPartitionTest(test.TestCase):
       data = np.zeros(5)
       partitions = data_flow_ops.dynamic_partition(data, bad, num_partitions=7)
       with self.assertRaisesOpError(r"partitions = 17 is not in \[0, 7\)"):
-        sess.run(partitions)
+        self.evaluate(partitions)
 
   def testHigherRankIndexOutOfRange(self):
     with self.cached_session() as sess:
@@ -335,7 +335,7 @@ class DynamicPartitionTest(test.TestCase):
     self.assertEqual(len(inds), x.shape[0])
     partitioned = data_flow_ops.dynamic_partition(x, inds, 16)
     with self.cached_session() as sess:
-      res = sess.run(partitioned)
+      res = self.evaluate(partitioned)
     self.assertEqual(res[-1].shape[0], 192)
 
 
diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py
index dba3409c9e..39c0575cd5 100644
--- a/tensorflow/python/kernel_tests/embedding_ops_test.py
+++ b/tensorflow/python/kernel_tests/embedding_ops_test.py
@@ -294,7 +294,7 @@ class EmbeddingLookupTest(test.TestCase):
       variables.global_variables_initializer().run()
       params_values = [params[p_i.name] for p_i in p]
       # Test that the PartitionedVariable components equal the list in p
-      p_var_val = sess.run(list(p_variable))
+      p_var_val = self.evaluate(list(p_variable))
       # Actual test
       tf_result = embedding.eval(feed_dict=feed_dict)
     np_result, _, _ = _EmbeddingResult(params, id_vals, num_shards, vocab_size)
@@ -316,7 +316,7 @@ class EmbeddingLookupTest(test.TestCase):
       variables.global_variables_initializer().run()
       params_values = [params[p_i.name] for p_i in p]
       # Test that the PartitionedVariable components equal the list in p
-      p_var_val = sess.run(list(p_variable))
+      p_var_val = self.evaluate(list(p_variable))
       # Actual test
       print(ops.get_default_graph().as_graph_def())
       tf_result = self.evaluate(embedding)
diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py
index e3742f2e72..9655351a01 100644
--- a/tensorflow/python/kernel_tests/fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/fifo_queue_test.py
@@ -159,7 +159,7 @@ class FIFOQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -191,7 +191,7 @@ class FIFOQueueTest(test.TestCase):
       results = []
 
       def dequeue():
-        results.append(sess.run(dequeued_t))
+        results.append(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops]
       for thread in threads:
@@ -240,13 +240,13 @@ class FIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       results = []
 
       def dequeue():
         for _ in xrange(len(elems)):
-          results.append(sess.run(dequeued_t))
+          results.append(self.evaluate(dequeued_t))
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -269,7 +269,7 @@ class FIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        x_val, y_val = sess.run(dequeued_t)
+        x_val, y_val = self.evaluate(dequeued_t)
         x, y = elems[i]
         self.assertEqual([x], x_val)
         self.assertEqual([y], y_val)
@@ -356,7 +356,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -399,17 +399,17 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
@@ -429,13 +429,13 @@ class FIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual([None], dequeued_t[0].get_shape().as_list())
       self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list())
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
@@ -529,7 +529,7 @@ class FIFOQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -552,7 +552,7 @@ class FIFOQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t))
+        dequeued_elems.extend(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in range(10)]
       for thread in threads:
@@ -576,7 +576,7 @@ class FIFOQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t))
+        dequeued_elems.extend(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in range(10)]
       for thread in threads:
@@ -596,11 +596,11 @@ class FIFOQueueTest(test.TestCase):
 
       def enqueue():
         for _ in xrange(100):
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       def dequeue():
         for _ in xrange(100):
-          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))
+          self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0))
 
       enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
@@ -632,7 +632,7 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         for i in xrange(250):
-          self.assertEqual(i, sess.run(dequeued_t))
+          self.assertEqual(i, self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -663,7 +663,7 @@ class FIFOQueueTest(test.TestCase):
       dequeuemany_t = q.dequeue_many(count_placeholder)
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -701,10 +701,10 @@ class FIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t).tolist())
+        dequeued_elems.extend(self.evaluate(dequeued_t).tolist())
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -728,10 +728,10 @@ class FIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t).tolist())
+        dequeued_elems.extend(self.evaluate(dequeued_t).tolist())
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -797,11 +797,11 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         for elem in elems:
-          self.assertEqual([elem], sess.run(dequeued_t))
+          self.assertEqual([elem], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -821,7 +821,7 @@ class FIFOQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -842,11 +842,11 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems, sess.run(dequeued_t))
+        self.assertAllEqual(elems, self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -867,11 +867,11 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -892,8 +892,8 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
-        self.assertAllEqual(elems[3:], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[3:], self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -913,16 +913,16 @@ class FIFOQueueTest(test.TestCase):
       cleanup_dequeue_t = q.dequeue()
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        self.assertAllEqual(elems[0:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t))
         with self.assertRaises(errors_impl.OutOfRangeError):
-          sess.run(dequeued_t)
-        self.assertEqual(elems[3], sess.run(cleanup_dequeue_t))
+          self.evaluate(dequeued_t)
+        self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t))
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -955,7 +955,7 @@ class FIFOQueueTest(test.TestCase):
 
       def dequeue():
         with self.assertRaises(errors_impl.OutOfRangeError):
-          sess.run([dequeued_a_t, dequeued_b_t])
+          self.evaluate([dequeued_a_t, dequeued_b_t])
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -968,7 +968,7 @@ class FIFOQueueTest(test.TestCase):
       # Test that the elements in the partially-dequeued batch are
       # restored in the correct order.
       for elem_a, elem_b in zip(elems_a, elems_b):
-        val_a, val_b = sess.run([cleanup_dequeue_a_t, cleanup_dequeue_b_t])
+        val_a, val_b = self.evaluate([cleanup_dequeue_a_t, cleanup_dequeue_b_t])
         self.assertEqual(elem_a, val_a)
         self.assertEqual(elem_b, val_b)
       self.assertEqual(0, q.size().eval())
@@ -983,7 +983,7 @@ class FIFOQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -1003,7 +1003,7 @@ class FIFOQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -1051,7 +1051,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1074,7 +1074,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1103,7 +1103,7 @@ class FIFOQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # Expect the operation to succeed once the dequeue op runs.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1113,7 +1113,7 @@ class FIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1138,7 +1138,7 @@ class FIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1148,7 +1148,7 @@ class FIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1266,19 +1266,19 @@ class FIFOQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_op)
+      self.evaluate(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_many_op)
+      self.evaluate(dequeue_many_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_many_op)
+      self.evaluate(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1321,7 +1321,7 @@ class FIFOQueueTest(test.TestCase):
       def blocking_enqueue():
         enq_done.append(False)
         # This will fill the queue and then block until enough dequeues happen.
-        sess.run(enq)
+        self.evaluate(enq)
         enq_done.append(True)
 
       thread = self.checkedThread(target=blocking_enqueue)
@@ -1331,14 +1331,14 @@ class FIFOQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1364,7 +1364,7 @@ class FIFOQueueTest(test.TestCase):
 
       def blocking_dequeue():
         # Will only complete after 4 enqueues complete.
-        results.extend(sess.run(deq))
+        results.extend(self.evaluate(deq))
 
       thread = self.checkedThread(target=blocking_dequeue)
       thread.start()
@@ -1373,7 +1373,7 @@ class FIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         self.assertEqual(len(results), 0)
-        sess.run(enq)
+        self.evaluate(enq)
 
       # Enough enqueued to unblock the dequeue
       thread.join()
@@ -1405,7 +1405,7 @@ class FIFOQueueTest(test.TestCase):
       q.enqueue_many(input_tuple).run()
 
       output_tuple_t = q.dequeue_many(32)
-      output_tuple = sess.run(output_tuple_t)
+      output_tuple = self.evaluate(output_tuple_t)
 
       for (input_elem, output_elem) in zip(input_tuple, output_tuple):
         self.assertAllEqual(input_elem, output_elem)
@@ -1507,10 +1507,10 @@ class FIFOQueueDictTest(test.TestCase):
       enqueue_op4 = q.enqueue_many({"f": [40.0, 50.0]})
       dequeue = q.dequeue()
       dequeue_2 = q.dequeue_many(2)
-      sess.run(enqueue_op)
-      sess.run(enqueue_op2)
-      sess.run(enqueue_op3)
-      sess.run(enqueue_op4)
+      self.evaluate(enqueue_op)
+      self.evaluate(enqueue_op2)
+      self.evaluate(enqueue_op3)
+      self.evaluate(enqueue_op4)
       f = sess.run(dequeue["f"])
       self.assertEqual(10.0, f)
       f = sess.run(dequeue_2["f"])
@@ -1565,10 +1565,10 @@ class FIFOQueueDictTest(test.TestCase):
       })
       dequeue = q.dequeue()
       dequeue_2 = q.dequeue_many(2)
-      sess.run(enqueue_op)
-      sess.run(enqueue_op2)
-      sess.run(enqueue_op3)
-      sess.run(enqueue_op4)
+      self.evaluate(enqueue_op)
+      self.evaluate(enqueue_op2)
+      self.evaluate(enqueue_op3)
+      self.evaluate(enqueue_op4)
       i, f, s = sess.run([dequeue["i"], dequeue["f"], dequeue["s"]])
       self.assertEqual(123, i)
       self.assertEqual(10.0, f)
@@ -1597,7 +1597,7 @@ class FIFOQueueWithTimeoutTest(test.TestCase):
       # until operation_timeout_in_ms.
       with self.assertRaisesRegexp(errors_impl.DeadlineExceededError,
                                    "Timed out waiting for notification"):
-        sess.run(dequeued_t)
+        self.evaluate(dequeued_t)
 
   def testReusableAfterTimeout(self):
     with self.cached_session() as sess:
@@ -1613,8 +1613,8 @@ class FIFOQueueWithTimeoutTest(test.TestCase):
                                    "Timed out waiting for notification"):
         sess.run(dequeued_t, options=config_pb2.RunOptions(timeout_in_ms=10))
 
-      sess.run(enqueue_op)
-      self.assertEqual(37, sess.run(dequeued_t))
+      self.evaluate(enqueue_op)
+      self.assertEqual(37, self.evaluate(dequeued_t))
 
 
 class QueueContainerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
index cb7659a89a..272adecfb8 100644
--- a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
@@ -133,7 +133,7 @@ class FractionalAvgTest(test.TestCase):
           pseudo_random,
           overlapping,
           seed=self._SEED)
-      actual, row_seq, col_seq = sess.run([p, r, c])
+      actual, row_seq, col_seq = self.evaluate([p, r, c])
       expected = self._GetExpectedFractionalAvgPoolResult(input_tensor, row_seq,
                                                           col_seq, overlapping)
       self.assertShapeEqual(expected, p)
@@ -164,7 +164,7 @@ class FractionalAvgTest(test.TestCase):
             pseudo_random,
             overlapping,
             seed=self._SEED)
-        tensor_output, row_seq, col_seq = sess.run([p, r, c])
+        tensor_output, row_seq, col_seq = self.evaluate([p, r, c])
         expected_result = self._GetExpectedFractionalAvgPoolResult(
             rand_mat.astype(np.float32), row_seq, col_seq, overlapping)
         print("row sequence:")
diff --git a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
index 0427e34fc1..9b1e73b318 100644
--- a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
@@ -133,7 +133,7 @@ class FractionalMaxPoolTest(test.TestCase):
           pseudo_random,
           overlapping,
           seed=self._SEED)
-      actual, row_seq, col_seq = sess.run([p, r, c])
+      actual, row_seq, col_seq = self.evaluate([p, r, c])
       expected = self._GetExpectedFractionalMaxPoolResult(input_tensor, row_seq,
                                                           col_seq, overlapping)
       self.assertShapeEqual(expected, p)
@@ -164,7 +164,7 @@ class FractionalMaxPoolTest(test.TestCase):
             pseudo_random,
             overlapping,
             seed=self._SEED)
-        tensor_output, row_seq, col_seq = sess.run([p, r, c])
+        tensor_output, row_seq, col_seq = self.evaluate([p, r, c])
         expected_result = self._GetExpectedFractionalMaxPoolResult(rand_mat,
                                                                    row_seq,
                                                                    col_seq,
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 503569f3b1..23b3c7e1cc 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -458,7 +458,7 @@ class FunctionalOpsTest(test.TestCase):
     grad = gradients_impl.gradients(ys=[loss], xs=[a, b])
     with self.test_session(use_gpu=True) as sess:
       variables.global_variables_initializer().run()
-      sess.run(grad)
+      self.evaluate(grad)
 
   @test_util.run_in_graph_and_eager_modes
   def testFoldShape(self):
@@ -567,8 +567,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:worker/replica:0/task:0/cpu:1")
 
     with session.Session(worker[0].target) as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionDirectSession(self):
@@ -591,8 +591,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/cpu:1")
 
     with self.test_session(config=worker_config) as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionSameDeviceDirectSession(self):
@@ -610,8 +610,8 @@ class FunctionalOpsTest(test.TestCase):
           args=[a, b], Tout=[dtypes.int32], f=_remote_fn, target="/cpu:0")
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
   def testRemoteFunctionCPUGPU(self):
@@ -634,8 +634,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/device:GPU:0")[0] + 3.0
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9.0)
 
   def testRemoteFunctionGPUCPU(self):
@@ -658,8 +658,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:localhost/replica:0/task:0/cpu:0")[0] + 3.0
 
     with self.cached_session() as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9.0)
 
   def testRemoteFunctionGPUCPUStrings(self):
@@ -677,7 +677,7 @@ class FunctionalOpsTest(test.TestCase):
           args=[a], Tout=[dtypes.string], f=_remote_fn, target="/cpu:0")
 
     with self.cached_session() as sess:
-      ret = sess.run(remote_op)
+      ret = self.evaluate(remote_op)
       self.assertAllEqual(ret, [b"a"])
 
   def testRemoteFunctionCrossProcess(self):
@@ -699,8 +699,8 @@ class FunctionalOpsTest(test.TestCase):
           target="/job:worker/replica:0/task:1/cpu:0")[0] + 3.0
 
     with session.Session(workers[0].target) as sess:
-      sess.run(variables.global_variables_initializer())
-      mul = sess.run(remote_op)
+      self.evaluate(variables.global_variables_initializer())
+      mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9)
 
   def testIf(self):
@@ -769,7 +769,7 @@ class FunctionalOpsTest(test.TestCase):
           else:
             fetch = "my_while:1"
         with self.session(graph=g, use_gpu=use_gpu) as sess:
-          return sess.run(fetch)
+          return self.evaluate(fetch)
 
     self.assertAllEqual(Run(20., False), 210.)
     self.assertAllEqual(Run(20., True), 210.)
@@ -857,11 +857,11 @@ class FunctionalOpsTest(test.TestCase):
           result_binary = functional_ops.While(
               [1.0, 0., 0.],
               function.Defun(*[dtypes.float32] * 3)(TestCond), TestBinary)
-          sess.run(variables.global_variables_initializer())
+          self.evaluate(variables.global_variables_initializer())
           assert len(result_unary) == 2
-          self.assertEqual([10.0, 54.0], sess.run(result_unary))
+          self.assertEqual([10.0, 54.0], self.evaluate(result_unary))
           assert len(result_binary) == 3
-          self.assertEqual([10.0, 54.0, 9.0], sess.run(result_binary))
+          self.assertEqual([10.0, 54.0, 9.0], self.evaluate(result_binary))
 
           def TestCondCapture(n, *args):
             del args
@@ -892,7 +892,7 @@ class FunctionalOpsTest(test.TestCase):
                 100, 0, -1, [0.], Body, rewrite_with_while=rewrite_with_while)
             [0],
         ]
-        xvals = sess.run(xs)
+        xvals = self.evaluate(xs)
       self.assertAllEqual(210, xvals[0])
       self.assertAllEqual(5050, xvals[1])
 
@@ -949,16 +949,16 @@ class FunctionalOpsTest(test.TestCase):
         result_binary = functional_ops.For(
             1, 10, 1, [0., 0.], TestBinary,
             rewrite_with_while=rewrite_with_while)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         assert not result_nullary
         # The nullary variant doesn't return anything so we can't easily run it.
         # As a total hack, fetch the operation by name and run it.
         sess.run(ops.get_default_graph().get_operation_by_name(
             "While" if rewrite_with_while else "For"))
         assert len(result_unary) == 1
-        self.assertEqual([54.0], sess.run(result_unary))
+        self.assertEqual([54.0], self.evaluate(result_unary))
         assert len(result_binary) == 2
-        self.assertEqual([54.0, 9.0], sess.run(result_binary))
+        self.assertEqual([54.0, 9.0], self.evaluate(result_binary))
 
   def _tfMLP(self, xval, wsval, bsval, rewrite_with_while):
     # On GPU, don't rewrite using a while loop.
@@ -1041,8 +1041,8 @@ class FunctionalOpsTest(test.TestCase):
       avals = [Poly(a), Grad(a)]
       b = constant_op.constant(1.)
       bvals = [Poly(b), Grad(b)]
-      self.assertAllEqual(sess.run(avals), [8., 4.])
-      self.assertAllEqual(sess.run(bvals), [17., 16.])
+      self.assertAllEqual(self.evaluate(avals), [8., 4.])
+      self.assertAllEqual(self.evaluate(bvals), [17., 16.])
 
 
 # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the
@@ -1193,8 +1193,8 @@ class PartitionedCallTest(test.TestCase):
             allow_soft_placement=False,
             log_device_placement=True,
             device_count={"CPU": 2})) as sess:
-      sess.run(variables.global_variables_initializer())
-      expected = sess.run(sum_gather())
+      self.evaluate(variables.global_variables_initializer())
+      expected = self.evaluate(sum_gather())
       result = sess.run(
           functional_ops.partitioned_call(
               args=defined.captured_inputs, f=defined))
diff --git a/tensorflow/python/kernel_tests/gradient_correctness_test.py b/tensorflow/python/kernel_tests/gradient_correctness_test.py
index 291a69ebac..12b8a4c8e3 100644
--- a/tensorflow/python/kernel_tests/gradient_correctness_test.py
+++ b/tensorflow/python/kernel_tests/gradient_correctness_test.py
@@ -35,7 +35,7 @@ class GradientCorrectnessTest(test.TestCase):
       yexp = math_ops.exp(x)
       yexplog = math_ops.log(yexp)
       grads = gradients_impl.gradients([yexp, yexplog], [x])
-      grad_vals = sess.run(grads)
+      grad_vals = self.evaluate(grads)
       exp1_plus_one = (1.0 + np.exp(1.0)).astype(np.float32)
       # [dexp(x)/dx + d(log(exp(x)))/dx] @ x=1 == exp(1) + 1
       self.assertAllClose(grad_vals[0], exp1_plus_one)
@@ -44,13 +44,13 @@ class GradientCorrectnessTest(test.TestCase):
     x = constant_op.constant(3.)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
-      self.assertAllClose(1., sess.run(dx_dx))
+      self.assertAllClose(1., self.evaluate(dx_dx))
 
   def testIntegerIdentityGradient(self):
     x = constant_op.constant(3)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
-      self.assertAllClose(1, sess.run(dx_dx))
+      self.assertAllClose(1, self.evaluate(dx_dx))
 
   def testGradientWithIntegerPath(self):
     x = constant_op.constant([3.9, 4.1])
@@ -58,7 +58,7 @@ class GradientCorrectnessTest(test.TestCase):
     y = x * k
     dy_dx, = gradients_impl.gradients(y, x)
     with self.cached_session() as sess:
-      self.assertAllClose([3., 4.], sess.run(dy_dx))
+      self.assertAllClose([3., 4.], self.evaluate(dy_dx))
 
   def testNoIntegerGradient1(self):
     x = constant_op.constant([3.9, 4.1])
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index a3f2c0ddd7..87c7bbef3c 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -704,12 +704,12 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
         ratio = outputs_2norm / inputs_2norm
         my_ops = variables.global_variables_initializer()
         with self.session(use_gpu=True) as sess:
-          sess.run(my_ops)
+          self.evaluate(my_ops)
           # Check the shape of the outputs
           t = self.evaluate(outputs)
           self.assertAllEqual(t.shape, outputs_shape)
           # Check isometry of the delta-orthogonal kernel.
-          self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+          self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
   def testNonuniformity(self):
     value = 0
@@ -842,12 +842,12 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
       ratio = outputs_2norm / inputs_2norm
       my_ops = variables.global_variables_initializer()
       with self.session(use_gpu=True) as sess:
-        sess.run(my_ops)
+        self.evaluate(my_ops)
         # Check the shape of the outputs
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
@@ -937,12 +937,12 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
       ratio = outputs_2norm / inputs_2norm
       my_ops = variables.global_variables_initializer()
       with self.session(use_gpu=True) as sess:
-        sess.run(my_ops)
+        self.evaluate(my_ops)
         # Check the shape of the outputs
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
 
 class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
@@ -1062,12 +1062,12 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
       ratio = outputs_2norm / inputs_2norm
       my_ops = variables.global_variables_initializer()
       with self.cached_session(use_gpu=True) as sess:
-        sess.run(my_ops)
+        self.evaluate(my_ops)
         # Check the shape of the outputs
         t = self.evaluate(outputs)
         self.assertAllEqual(t.shape, outputs_shape)
         # Check isometry of the orthogonal kernel.
-        self.assertAllClose(sess.run(ratio), gain, rtol=tol, atol=tol)
+        self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
 
 class IdentityInitializerTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/inplace_ops_test.py b/tensorflow/python/kernel_tests/inplace_ops_test.py
index 51d16861dd..e0c36d3d2e 100644
--- a/tensorflow/python/kernel_tests/inplace_ops_test.py
+++ b/tensorflow/python/kernel_tests/inplace_ops_test.py
@@ -149,7 +149,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
       y = inplace_ops.alias_inplace_add(x, [0], [[1, 2, 3]])
       with ops.control_dependencies([y]):
         z = array_ops.identity(x)
-        _, vy, vz = sess.run([x, y, z])
+        _, vy, vz = self.evaluate([x, y, z])
       self.assertAllClose(vy, vz)
 
   def testError(self):
diff --git a/tensorflow/python/kernel_tests/io_ops_test.py b/tensorflow/python/kernel_tests/io_ops_test.py
index afa24195cb..a6b477062e 100644
--- a/tensorflow/python/kernel_tests/io_ops_test.py
+++ b/tensorflow/python/kernel_tests/io_ops_test.py
@@ -53,7 +53,7 @@ class IoOpsTest(test.TestCase):
         pass
       with self.cached_session() as sess:
         w = io_ops.write_file(temp.name, contents)
-        sess.run(w)
+        self.evaluate(w)
         with open(temp.name, 'rb') as f:
           file_contents = f.read()
         self.assertEqual(file_contents, contents)
@@ -67,7 +67,7 @@ class IoOpsTest(test.TestCase):
       filepath = os.path.join(subdir, 'subdir2', 'filename')
       with self.cached_session() as sess:
         w = io_ops.write_file(filepath, contents)
-        sess.run(w)
+        self.evaluate(w)
         with open(filepath, 'rb') as f:
           file_contents = f.read()
         self.assertEqual(file_contents, contents)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
index d5580d0e88..09867435a7 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
@@ -557,7 +557,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
       self.assertEqual(matrix_tensor.dtype,
                        linear_operator_circulant._DTYPE_COMPLEX)
       matrix_h = linalg.adjoint(matrix_tensor)
-      matrix, matrix_h = sess.run([matrix_tensor, matrix_h])
+      matrix, matrix_h = self.evaluate([matrix_tensor, matrix_h])
       self.assertAllClose(matrix, matrix_h, atol=0)
 
   def test_assert_non_singular_fails_for_singular_operator(self):
@@ -631,7 +631,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
                        linear_operator_circulant._DTYPE_COMPLEX)
       matrix_h = linalg.adjoint(matrix_tensor)
 
-      matrix, matrix_h = sess.run([matrix_tensor, matrix_h])
+      matrix, matrix_h = self.evaluate([matrix_tensor, matrix_h])
       self.assertAllEqual((2, 2 * 3 * 5, 2 * 3 * 5), matrix.shape)
       self.assertAllClose(matrix, matrix_h)
 
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index 91f4097438..80889a162f 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -147,12 +147,12 @@ class LinearOperatorDiagTest(
       operator_matmul = operator.matmul(x)
       mat_matmul = math_ops.matmul(mat, x)
       self.assertAllEqual(operator_matmul.get_shape(), mat_matmul.get_shape())
-      self.assertAllClose(*sess.run([operator_matmul, mat_matmul]))
+      self.assertAllClose(*self.evaluate([operator_matmul, mat_matmul]))
 
       operator_solve = operator.solve(x)
       mat_solve = linalg_ops.matrix_solve(mat, x)
       self.assertAllEqual(operator_solve.get_shape(), mat_solve.get_shape())
-      self.assertAllClose(*sess.run([operator_solve, mat_solve]))
+      self.assertAllClose(*self.evaluate([operator_solve, mat_solve]))
 
   def test_diag_matmul(self):
     operator1 = linalg_lib.LinearOperatorDiag([2., 3.])
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 522213e26b..e9fd91c6cf 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -170,7 +170,7 @@ class LinearOperatorIdentityTest(
       expected = x
 
       self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape())
-      self.assertAllClose(*sess.run([operator_matmul, expected]))
+      self.assertAllClose(*self.evaluate([operator_matmul, expected]))
 
   def test_default_batch_shape_broadcasts_with_everything_dynamic(self):
     # These cannot be done in the automated (base test class) tests since they
@@ -207,7 +207,7 @@ class LinearOperatorIdentityTest(
 
       operator_matmul = operator.matmul(x)
       self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape())
-      self.assertAllClose(*sess.run([operator_matmul, expected]))
+      self.assertAllClose(*self.evaluate([operator_matmul, expected]))
 
   def test_broadcast_matmul_dynamic_shapes(self):
     # These cannot be done in the automated (base test class) tests since they
@@ -403,13 +403,13 @@ class LinearOperatorScaledIdentityTest(
       expected = x * 2.2 + zeros
       operator_matmul = operator.matmul(x)
       self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape())
-      self.assertAllClose(*sess.run([operator_matmul, expected]))
+      self.assertAllClose(*self.evaluate([operator_matmul, expected]))
 
       # Test solve
       expected = x / 2.2 + zeros
       operator_solve = operator.solve(x)
       self.assertAllEqual(operator_solve.get_shape(), expected.get_shape())
-      self.assertAllClose(*sess.run([operator_solve, expected]))
+      self.assertAllClose(*self.evaluate([operator_solve, expected]))
 
   def test_broadcast_matmul_and_solve_scalar_scale_multiplier(self):
     # These cannot be done in the automated (base test class) tests since they
@@ -429,13 +429,13 @@ class LinearOperatorScaledIdentityTest(
       expected = x * 2.2
       operator_matmul = operator.matmul(x)
       self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape())
-      self.assertAllClose(*sess.run([operator_matmul, expected]))
+      self.assertAllClose(*self.evaluate([operator_matmul, expected]))
 
       # Test solve
       expected = x / 2.2
       operator_solve = operator.solve(x)
       self.assertAllEqual(operator_solve.get_shape(), expected.get_shape())
-      self.assertAllClose(*sess.run([operator_solve, expected]))
+      self.assertAllClose(*self.evaluate([operator_solve, expected]))
 
   def test_is_x_flags(self):
     operator = linalg_lib.LinearOperatorScaledIdentity(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
index 5ce2616972..f12714677e 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
@@ -119,7 +119,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertAllEqual(x_bc_expected.shape, x_bc.get_shape())
       self.assertAllEqual(y_bc_expected.shape, y_bc.get_shape())
-      x_bc_, y_bc_ = sess.run([x_bc, y_bc])
+      x_bc_, y_bc_ = self.evaluate([x_bc, y_bc])
       self.assertAllClose(x_bc_expected, x_bc_)
       self.assertAllClose(y_bc_expected, y_bc_)
 
@@ -138,7 +138,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertAllEqual(x_bc_expected.shape, x_bc.get_shape())
       self.assertAllEqual(y_bc_expected.shape, y_bc.get_shape())
-      x_bc_, y_bc_ = sess.run([x_bc, y_bc])
+      x_bc_, y_bc_ = self.evaluate([x_bc, y_bc])
       self.assertAllClose(x_bc_expected, x_bc_)
       self.assertAllClose(y_bc_expected, y_bc_)
 
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 09cb5cf0ba..1d9f4032d1 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -806,7 +806,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     l_read2 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
     grad = gradients_impl.gradients([l_read1, l_read2], [x])
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(grad), [2.])
+      self.assertSequenceEqual(self.evaluate(grad), [2.])
 
   def testSkipEagerBuildElementShape(self):
     fn = list_ops._build_element_shape
diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py
index baeb40dd63..2865710798 100644
--- a/tensorflow/python/kernel_tests/listdiff_op_test.py
+++ b/tensorflow/python/kernel_tests/listdiff_op_test.py
@@ -47,7 +47,7 @@ class ListDiffTest(test.TestCase):
             y_tensor = ops.convert_to_tensor(y, dtype=dtype)
             out_tensor, idx_tensor = diff_func(x_tensor, y_tensor,
                                                index_dtype=index_dtype)
-            tf_out, tf_idx = sess.run([out_tensor, idx_tensor])
+            tf_out, tf_idx = self.evaluate([out_tensor, idx_tensor])
           self.assertAllEqual(tf_out, out)
           self.assertAllEqual(tf_idx, idx)
           self.assertEqual(1, out_tensor.get_shape().ndims)
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 3efad4ea11..79961d8dd1 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -137,7 +137,7 @@ class HashTableOpTest(test.TestCase):
       output2 = table2.lookup(input_string)
       output3 = table3.lookup(input_string)
 
-      out1, out2, out3 = sess.run([output1, output2, output3])
+      out1, out2, out3 = self.evaluate([output1, output2, output3])
       self.assertAllEqual([0, 1, -1], out1)
       self.assertAllEqual([0, 1, -1], out2)
       self.assertAllEqual([0, 1, -1], out3)
@@ -174,7 +174,7 @@ class HashTableOpTest(test.TestCase):
           constant_op.constant(sp_shape, dtypes.int64))
       output = table.lookup(input_tensor)
 
-      out_indices, out_values, out_shape = sess.run(output)
+      out_indices, out_values, out_shape = self.evaluate(output)
 
       self.assertAllEqual([0, 1, -1], out_values)
       self.assertAllEqual(sp_indices, out_indices)
@@ -995,7 +995,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       output2 = table2.lookup(input_string)
       output3 = table3.lookup(input_string)
 
-      out1, out2, out3 = sess.run([output1, output2, output3])
+      out1, out2, out3 = self.evaluate([output1, output2, output3])
       self.assertAllEqual([0, 1, -1], out1)
       self.assertAllEqual([0, 1, -1], out2)
       self.assertAllEqual([0, 1, -1], out3)
@@ -1313,7 +1313,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       out1 = table1.lookup(input_string)
       out2 = table2.lookup(input_string)
 
-      out1, out2 = sess.run([out1, out2])
+      out1, out2 = self.evaluate([out1, out2])
       self.assertAllEqual([5, 0, 1, 2, 5], out1)
       self.assertAllEqual([5, 0, 1, 2, 3], out2)
       self.assertEquals(vocab_size + oov_buckets, table1.size().eval())
@@ -1396,7 +1396,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       out1 = table1.lookup(input_string_1)
       out2 = table2.lookup(input_string_2)
 
-      out1, out2 = sess.run([out1, out2])
+      out1, out2 = self.evaluate([out1, out2])
       self.assertAllEqual([0, 1, 2, -1], out1)
       self.assertAllEqual([-2, 1, -2], out2)
       self.assertEquals(vocab_size + oov_buckets, table1.size().eval())
diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py
index d3a907852a..bda63bcaa9 100644
--- a/tensorflow/python/kernel_tests/losses_test.py
+++ b/tensorflow/python/kernel_tests/losses_test.py
@@ -1046,9 +1046,9 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with self.cached_session() as sess:
-        sess.run(init_op)
+        self.evaluate(init_op)
         for grad, _ in gradients_to_variables:
-          np_grad = sess.run(grad)
+          np_grad = self.evaluate(grad)
           self.assertFalse(np.isnan(np_grad).any())
 
   def testNonZeroLossWithPythonScalarWeight(self):
diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
index d41b449a1f..83f4216e4d 100644
--- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
@@ -151,7 +151,7 @@ class ExponentialOpTest(test.TestCase):
       matrix2 = random_ops.random_normal([5, 5], seed=42)
       expm1 = linalg_impl.matrix_exponential(matrix1)
       expm2 = linalg_impl.matrix_exponential(matrix2)
-      expm = sess.run([expm1, expm2])
+      expm = self.evaluate([expm1, expm2])
       self.assertAllEqual(expm[0], expm[1])
 
 
diff --git a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
index 434458721c..5cef4b79a3 100644
--- a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py
@@ -146,7 +146,7 @@ class InverseOpTest(test.TestCase):
         inv1 = linalg_ops.matrix_inverse(matrix1, adjoint=adjoint_)
         inv2 = linalg_ops.matrix_inverse(matrix2, adjoint=adjoint_)
         all_ops += [inv1, inv2]
-      inv = sess.run(all_ops)
+      inv = self.evaluate(all_ops)
       self.assertAllEqual(inv[0], inv[1])
       self.assertAllEqual(inv[2], inv[3])
 
diff --git a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
index 81c0b5a772..b0bce6a1b9 100644
--- a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
@@ -129,7 +129,7 @@ class LogarithmOpTest(test.TestCase):
           random_ops.random_normal([5, 5], seed=42), dtypes.complex64)
       logm1 = gen_linalg_ops.matrix_logarithm(matrix1)
       logm2 = gen_linalg_ops.matrix_logarithm(matrix2)
-      logm = sess.run([logm1, logm2])
+      logm = self.evaluate([logm1, logm2])
       self.assertAllEqual(logm[0], logm[1])
 
 
diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
index 1334d0c4ce..80badee896 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
@@ -126,7 +126,7 @@ class MatrixSolveOpTest(test.TestCase):
         s1 = linalg_ops.matrix_solve(lhs1, rhs1, adjoint=adjoint_)
         s2 = linalg_ops.matrix_solve(lhs2, rhs2, adjoint=adjoint_)
         all_ops += [s1, s2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       self.assertAllEqual(val[0], val[1])
       self.assertAllEqual(val[2], val[3])
 
diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 9212580313..1f2144bdee 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -108,7 +108,7 @@ class SquareRootOpTest(test.TestCase):
       sqrt1 = gen_linalg_ops.matrix_square_root(matrix1)
       sqrt2 = gen_linalg_ops.matrix_square_root(matrix2)
       all_ops = [sqrt1, sqrt2]
-      sqrt = sess.run(all_ops)
+      sqrt = self.evaluate(all_ops)
       self.assertAllEqual(sqrt[0], sqrt[1])
 
 
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index 5dcdb9e420..eb5f99582c 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -203,10 +203,10 @@ class MeanTest(test.TestCase):
 
       mean, update_op = metrics.mean(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAlmostEqual(1.65, sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
 
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
@@ -220,14 +220,14 @@ class MeanTest(test.TestCase):
 
       mean, update_op = metrics.mean(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(0.5, sess.run(update_op), 5)
-      self.assertAlmostEqual(1.475, sess.run(update_op), 5)
-      self.assertAlmostEqual(12.4 / 6.0, sess.run(update_op), 5)
-      self.assertAlmostEqual(1.65, sess.run(update_op), 5)
+      self.assertAlmostEqual(0.5, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(1.475, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(12.4 / 6.0, self.evaluate(update_op), 5)
+      self.assertAlmostEqual(1.65, self.evaluate(update_op), 5)
 
-      self.assertAlmostEqual(1.65, sess.run(mean), 5)
+      self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
 
   def testUnweighted(self):
     values = _test_values((3, 2, 4, 1))
@@ -370,10 +370,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean))
+        self.evaluate(update_op)
+      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean))
 
   def testMultiDimensional(self):
     with self.cached_session() as sess:
@@ -391,10 +391,11 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(2):
-        sess.run(update_op)
-      self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]], sess.run(mean))
+        self.evaluate(update_op)
+      self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]],
+                          self.evaluate(mean))
 
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
@@ -408,14 +409,14 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAllClose([[0, 1]], sess.run(update_op), 5)
-      self.assertAllClose([[-2.1, 5.05]], sess.run(update_op), 5)
-      self.assertAllClose([[2.3 / 3., 10.1 / 3.]], sess.run(update_op), 5)
-      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(update_op), 5)
+      self.assertAllClose([[0, 1]], self.evaluate(update_op), 5)
+      self.assertAllClose([[-2.1, 5.05]], self.evaluate(update_op), 5)
+      self.assertAllClose([[2.3 / 3., 10.1 / 3.]], self.evaluate(update_op), 5)
+      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(update_op), 5)
 
-      self.assertAllClose([[-0.9 / 4., 3.525]], sess.run(mean), 5)
+      self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean), 5)
 
   def testBinaryWeighted1d(self):
     with self.cached_session() as sess:
@@ -439,10 +440,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[3.25, 0.5]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[3.25, 0.5]], self.evaluate(mean), 5)
 
   def testWeighted1d(self):
     with self.cached_session() as sess:
@@ -466,10 +467,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[0.8, 3.52]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[0.8, 3.52]], self.evaluate(mean), 5)
 
   def testWeighted2d_1(self):
     with self.cached_session() as sess:
@@ -493,10 +494,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[-2.1, 0.5]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[-2.1, 0.5]], self.evaluate(mean), 5)
 
   def testWeighted2d_2(self):
     with self.cached_session() as sess:
@@ -520,10 +521,10 @@ class MeanTensorTest(test.TestCase):
 
       mean, update_op = metrics.mean_tensor(values, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(4):
-        sess.run(update_op)
-      self.assertAllClose([[0, 0.5]], sess.run(mean), 5)
+        self.evaluate(update_op)
+      self.assertAllClose([[0, 0.5]], self.evaluate(mean), 5)
 
 
 class AccuracyTest(test.TestCase):
@@ -576,11 +577,11 @@ class AccuracyTest(test.TestCase):
     accuracy, update_op = metrics.accuracy(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_accuracy = accuracy.eval()
@@ -609,10 +610,10 @@ class AccuracyTest(test.TestCase):
 
       accuracy, update_op = metrics.accuracy(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in xrange(3):
-        sess.run(update_op)
-      self.assertEqual(0.5, sess.run(update_op))
+        self.evaluate(update_op)
+      self.assertEqual(0.5, self.evaluate(update_op))
       self.assertEqual(0.5, accuracy.eval())
 
   def testEffectivelyEquivalentSizes(self):
@@ -621,7 +622,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
@@ -631,7 +632,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions, weights=2.0)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
@@ -645,7 +646,7 @@ class AccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       accuracy, update_op = metrics.accuracy(labels, predictions, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # if streaming_accuracy does not flatten the weight, accuracy would be
       # 0.33333334 due to an intended broadcast of weight. Due to flattening,
       # it will be higher than .95
@@ -666,7 +667,7 @@ class AccuracyTest(test.TestCase):
       accuracy, update_op = metrics.accuracy(labels, predictions,
                                              weights_placeholder)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # if streaming_accuracy does not flatten the weight, accuracy would be
       # 0.33333334 due to an intended broadcast of weight. Due to flattening,
       # it will be higher than .95
@@ -704,10 +705,10 @@ class AccuracyTest(test.TestCase):
 
       accuracy, update_op = metrics.accuracy(labels, predictions, weights)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in xrange(3):
-        sess.run(update_op)
-      self.assertEqual(1.0, sess.run(update_op))
+        self.evaluate(update_op)
+      self.assertEqual(1.0, self.evaluate(update_op))
       self.assertEqual(1.0, accuracy.eval())
 
 
@@ -747,11 +748,11 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_precision = precision.eval()
@@ -766,8 +767,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op))
       self.assertAlmostEqual(1, precision.eval())
 
   def testSomeCorrect_multipleInputDtypes(self):
@@ -779,7 +780,7 @@ class PrecisionTest(test.TestCase):
       precision, update_op = metrics.precision(labels, predictions)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, precision.eval())
 
@@ -882,8 +883,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertAlmostEqual(0, precision.eval())
 
   def testZeroTrueAndFalsePositivesGivesZeroPrecision(self):
@@ -892,8 +893,8 @@ class PrecisionTest(test.TestCase):
     precision, update_op = metrics.precision(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(0.0, precision.eval())
 
 
@@ -934,11 +935,11 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_recall = recall.eval()
@@ -953,8 +954,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(1, recall.eval())
 
   def testSomeCorrect_multipleInputDtypes(self):
@@ -966,7 +967,7 @@ class RecallTest(test.TestCase):
       recall, update_op = metrics.recall(labels, predictions)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, recall.eval())
 
@@ -977,7 +978,7 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       weighted_tp = 2.0 + 5.0
       weighted_t = (2.0 + 2.0) + (5.0 + 5.0)
       expected_precision = weighted_tp / weighted_t
@@ -991,7 +992,7 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       weighted_tp = 3.0 + 1.0
       weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
       expected_precision = weighted_tp / weighted_t
@@ -1006,8 +1007,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(0, recall.eval())
 
   def testZeroTruePositivesAndFalseNegativesGivesZeroRecall(self):
@@ -1016,8 +1017,8 @@ class RecallTest(test.TestCase):
     recall, update_op = metrics.recall(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
       self.assertEqual(0, recall.eval())
 
 
@@ -1056,11 +1057,11 @@ class AUCTest(test.TestCase):
     auc, update_op = metrics.auc(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_auc = auc.eval()
@@ -1078,8 +1079,8 @@ class AUCTest(test.TestCase):
       labels = constant_op.constant(inputs)
       auc, update_op = metrics.auc(labels, predictions, curve=curve)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(1, self.evaluate(update_op))
 
       self.assertEqual(1, auc.eval())
 
@@ -1093,8 +1094,8 @@ class AUCTest(test.TestCase):
             constant_op.constant([0, 1, 1, 0], shape=(1, 4)), dtype=label_dtype)
         auc, update_op = metrics.auc(labels, predictions)
 
-        sess.run(variables.local_variables_initializer())
-        self.assertAlmostEqual(0.5, sess.run(update_op))
+        self.evaluate(variables.local_variables_initializer())
+        self.assertAlmostEqual(0.5, self.evaluate(update_op))
 
         self.assertAlmostEqual(0.5, auc.eval())
 
@@ -1106,8 +1107,8 @@ class AUCTest(test.TestCase):
       weights = constant_op.constant([2], shape=(1, 1))
       auc, update_op = metrics.auc(labels, predictions, weights=weights)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.5, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, self.evaluate(update_op), 5)
 
       self.assertAlmostEqual(0.5, auc.eval(), 5)
 
@@ -1119,8 +1120,8 @@ class AUCTest(test.TestCase):
       weights = constant_op.constant([1, 2, 3, 4], shape=(1, 4))
       auc, update_op = metrics.auc(labels, predictions, weights=weights)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.7, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.7, self.evaluate(update_op), 5)
 
       self.assertAlmostEqual(0.7, auc.eval(), 5)
 
@@ -1134,10 +1135,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # expected ~= 0.79726744594
       expected = 1 - math.log(1.5) / 2
-      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testCorrectAnotherAUCPRSpecialCase(self):
@@ -1150,10 +1151,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # expected ~= 0.61350593198
       expected = (2.5 - 2 * math.log(4./3) - 0.25 * math.log(7./5)) / 3
-      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testThirdCorrectAUCPRSpecialCase(self):
@@ -1166,10 +1167,10 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='careful_interpolation')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       # expected ~= 0.90410597584
       expected = 1 - math.log(4./3) / 3
-      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
   def testIncorrectAUCPRSpecialCase(self):
@@ -1180,8 +1181,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.79166, sess.run(update_op), delta=1e-3)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.79166, self.evaluate(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3)
 
@@ -1195,8 +1196,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.610317, sess.run(update_op), delta=1e-3)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.610317, self.evaluate(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3)
 
@@ -1210,8 +1211,8 @@ class AUCTest(test.TestCase):
       auc, update_op = metrics.auc(labels, predictions, curve='PR',
                                    summation_method='trapezoidal')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.90277, sess.run(update_op), delta=1e-3)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.90277, self.evaluate(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-3)
 
@@ -1223,8 +1224,8 @@ class AUCTest(test.TestCase):
       labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
       auc, update_op = metrics.auc(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0, self.evaluate(update_op))
 
       self.assertAlmostEqual(0, auc.eval())
 
@@ -1234,8 +1235,8 @@ class AUCTest(test.TestCase):
       labels = array_ops.zeros([4])
       auc, update_op = metrics.auc(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op), 6)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op), 6)
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
@@ -1245,8 +1246,8 @@ class AUCTest(test.TestCase):
       labels = array_ops.ones([4])
       auc, update_op = metrics.auc(labels, predictions, curve='PR')
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op), 6)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op), 6)
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
@@ -1317,9 +1318,9 @@ class AUCTest(test.TestCase):
                                      num_thresholds=500,
                                      weights=tf_weights)
 
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
         for i in range(num_batches):
-          sess.run(update_op)
+          self.evaluate(update_op)
 
         # Since this is only approximate, we can't expect a 6 digits match.
         # Although with higher number of samples/thresholds we should see the
@@ -1371,11 +1372,11 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_specificity = specificity.eval()
@@ -1391,8 +1392,8 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(1, self.evaluate(update_op))
       self.assertEqual(1, specificity.eval())
 
   def testSomeCorrectHighSensitivity(self):
@@ -1406,8 +1407,8 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.8)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1.0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1.0, self.evaluate(update_op))
       self.assertAlmostEqual(1.0, specificity.eval())
 
   def testSomeCorrectLowSensitivity(self):
@@ -1421,9 +1422,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, sensitivity=0.4)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(0.6, sess.run(update_op))
+      self.assertAlmostEqual(0.6, self.evaluate(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted1d_multipleLabelDtypes(self):
@@ -1440,9 +1441,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
           labels, predictions, weights=weights, sensitivity=0.4)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
+        self.evaluate(variables.local_variables_initializer())
 
-        self.assertAlmostEqual(0.6, sess.run(update_op))
+        self.assertAlmostEqual(0.6, self.evaluate(update_op))
         self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted2d(self):
@@ -1458,9 +1459,9 @@ class SpecificityAtSensitivityTest(test.TestCase):
         labels, predictions, weights=weights, sensitivity=0.4)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
-      self.assertAlmostEqual(8.0 / 15.0, sess.run(update_op))
+      self.assertAlmostEqual(8.0 / 15.0, self.evaluate(update_op))
       self.assertAlmostEqual(8.0 / 15.0, specificity.eval())
 
 
@@ -1508,11 +1509,11 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_sensitivity = sensitivity.eval()
@@ -1528,8 +1529,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.7)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(1, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(1, self.evaluate(update_op))
       self.assertEqual(1, specificity.eval())
 
   def testSomeCorrectHighSpecificity(self):
@@ -1543,8 +1544,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.8)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.8, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.8, self.evaluate(update_op))
       self.assertAlmostEqual(0.8, specificity.eval())
 
   def testSomeCorrectLowSpecificity(self):
@@ -1558,8 +1559,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
         labels, predictions, specificity=0.4)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.6, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.6, self.evaluate(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
   def testWeighted_multipleLabelDtypes(self):
@@ -1577,8 +1578,8 @@ class SensitivityAtSpecificityTest(test.TestCase):
           labels, predictions, weights=weights, specificity=0.4)
 
       with self.cached_session() as sess:
-        sess.run(variables.local_variables_initializer())
-        self.assertAlmostEqual(0.675, sess.run(update_op))
+        self.evaluate(variables.local_variables_initializer())
+        self.assertAlmostEqual(0.675, self.evaluate(update_op))
         self.assertAlmostEqual(0.675, specificity.eval())
 
 
@@ -1639,14 +1640,14 @@ class PrecisionRecallThresholdsTest(test.TestCase):
     rec, rec_op = metrics.recall_at_thresholds(labels, predictions, thresholds)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates, then verify idempotency.
-      sess.run([prec_op, rec_op])
+      self.evaluate([prec_op, rec_op])
       initial_prec = prec.eval()
       initial_rec = rec.eval()
       for _ in range(10):
-        sess.run([prec_op, rec_op])
+        self.evaluate([prec_op, rec_op])
         self.assertAllClose(initial_prec, prec.eval())
         self.assertAllClose(initial_rec, rec.eval())
 
@@ -1663,8 +1664,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([prec_op, rec_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([prec_op, rec_op])
 
       self.assertEqual(1, prec.eval())
       self.assertEqual(1, rec.eval())
@@ -1683,8 +1684,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                    thresholds)
 
-        sess.run(variables.local_variables_initializer())
-        sess.run([prec_op, rec_op])
+        self.evaluate(variables.local_variables_initializer())
+        self.evaluate([prec_op, rec_op])
 
         self.assertAlmostEqual(0.5, prec.eval())
         self.assertAlmostEqual(0.5, rec.eval())
@@ -1701,8 +1702,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([prec_op, rec_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([prec_op, rec_op])
 
       self.assertAlmostEqual(0, prec.eval())
       self.assertAlmostEqual(0, rec.eval())
@@ -1729,8 +1730,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec_low = array_ops.reshape(rec_low, shape=())
       rec_high = array_ops.reshape(rec_high, shape=())
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([prec_op, rec_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([prec_op, rec_op])
 
       self.assertAlmostEqual(1.0, prec_low.eval(), places=5)
       self.assertAlmostEqual(0.0, prec_high.eval(), places=5)
@@ -1759,8 +1760,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec_low = array_ops.reshape(rec_low, shape=())
       rec_high = array_ops.reshape(rec_high, shape=())
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([prec_op, rec_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([prec_op, rec_op])
 
       self.assertAlmostEqual(1.0, prec_low.eval(), places=5)
       self.assertAlmostEqual(0.0, prec_high.eval(), places=5)
@@ -1783,8 +1784,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       [rec_low, rec_high] = array_ops.split(
           value=rec, num_or_size_splits=2, axis=0)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([prec_op, rec_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([prec_op, rec_op])
 
       self.assertAlmostEqual(0.75, prec_low.eval())
       self.assertAlmostEqual(0.0, prec_high.eval())
@@ -1801,8 +1802,8 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(labels, predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([prec_op, rec_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([prec_op, rec_op])
 
       self.assertAlmostEqual(0, prec.eval(), 6)
       self.assertAlmostEqual(0, rec.eval(), 6)
@@ -1869,9 +1870,9 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.recall_at_thresholds(tf_labels, tf_predictions,
                                                  thresholds)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(int(num_samples / batch_size)):
-        sess.run([prec_op, rec_op])
+        self.evaluate([prec_op, rec_op])
       # Since this is only approximate, we can't expect a 6 digits match.
       # Although with higher number of samples/thresholds we should see the
       # accuracy improving
@@ -2802,11 +2803,11 @@ class MeanAbsoluteErrorTest(test.TestCase):
     error, update_op = metrics.mean_absolute_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2823,8 +2824,8 @@ class MeanAbsoluteErrorTest(test.TestCase):
     error, update_op = metrics.mean_absolute_error(labels, predictions, weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(3, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(3, self.evaluate(update_op))
       self.assertEqual(3, error.eval())
 
 
@@ -2867,11 +2868,11 @@ class MeanRelativeErrorTest(test.TestCase):
                                                    normalizer)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2892,8 +2893,8 @@ class MeanRelativeErrorTest(test.TestCase):
         labels, predictions, normalizer=labels)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(expected_error, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(expected_error, self.evaluate(update_op))
       self.assertEqual(expected_error, error.eval())
 
   def testSingleUpdateNormalizedByZeros(self):
@@ -2908,8 +2909,8 @@ class MeanRelativeErrorTest(test.TestCase):
         labels, predictions, normalizer=array_ops.zeros_like(labels))
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0.0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0.0, self.evaluate(update_op))
       self.assertEqual(0.0, error.eval())
 
 
@@ -2946,11 +2947,11 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -2964,8 +2965,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithError(self):
@@ -2977,8 +2978,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(6, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(6, self.evaluate(update_op))
       self.assertEqual(6, error.eval())
 
   def testSingleUpdateWithErrorAndWeights(self):
@@ -2991,8 +2992,8 @@ class MeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.mean_squared_error(labels, predictions, weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(13, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(13, self.evaluate(update_op))
       self.assertEqual(13, error.eval())
 
   def testMultipleBatchesOfSizeOne(self):
@@ -3013,9 +3014,9 @@ class MeanSquaredErrorTest(test.TestCase):
 
       error, update_op = metrics.mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertAlmostEqual(208.0 / 6, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate(update_op)
+      self.assertAlmostEqual(208.0 / 6, self.evaluate(update_op), 5)
 
       self.assertAlmostEqual(208.0 / 6, error.eval(), 5)
 
@@ -3054,11 +3055,11 @@ class MeanSquaredErrorTest(test.TestCase):
       mse1, update_op1 = metrics.mean_squared_error(
           labels1, predictions1, name='msd1')
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([update_op0, update_op1])
-      sess.run([update_op0, update_op1])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([update_op0, update_op1])
+      self.evaluate([update_op0, update_op1])
 
-      mse0, mse1 = sess.run([mse0, mse1])
+      mse0, mse1 = self.evaluate([mse0, mse1])
       self.assertAlmostEqual(208.0 / 6, mse0, 5)
       self.assertAlmostEqual(79.0 / 6, mse1, 5)
 
@@ -3081,9 +3082,9 @@ class MeanSquaredErrorTest(test.TestCase):
       mae, ma_update_op = metrics.mean_absolute_error(labels, predictions)
       mse, ms_update_op = metrics.mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([ma_update_op, ms_update_op])
-      sess.run([ma_update_op, ms_update_op])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([ma_update_op, ms_update_op])
+      self.evaluate([ma_update_op, ms_update_op])
 
       self.assertAlmostEqual(32.0 / 6, mae.eval(), 5)
       self.assertAlmostEqual(208.0 / 6, mse.eval(), 5)
@@ -3123,11 +3124,11 @@ class RootMeanSquaredErrorTest(test.TestCase):
     error, update_op = metrics.root_mean_squared_error(labels, predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -3142,8 +3143,8 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
 
       self.assertEqual(0, rmse.eval())
 
@@ -3156,7 +3157,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAlmostEqual(math.sqrt(6), update_op.eval(), 5)
       self.assertAlmostEqual(math.sqrt(6), rmse.eval(), 5)
 
@@ -3171,8 +3172,8 @@ class RootMeanSquaredErrorTest(test.TestCase):
       rmse, update_op = metrics.root_mean_squared_error(labels, predictions,
                                                         weights)
 
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(math.sqrt(13), sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(math.sqrt(13), self.evaluate(update_op))
 
       self.assertAlmostEqual(math.sqrt(13), rmse.eval(), 5)
 
@@ -3221,11 +3222,11 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=1)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_error = error.eval()
@@ -3243,8 +3244,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithError1(self):
@@ -3259,8 +3260,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1, self.evaluate(update_op), 5)
       self.assertAlmostEqual(1, error.eval(), 5)
 
   def testSingleUpdateWithError2(self):
@@ -3280,8 +3281,8 @@ class MeanCosineDistanceTest(test.TestCase):
     error, update_op = metrics.mean_cosine_distance(labels, predictions, dim=2)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(1.0, sess.run(update_op), 5)
+      self.evaluate(variables.local_variables_initializer())
+      self.assertAlmostEqual(1.0, self.evaluate(update_op), 5)
       self.assertAlmostEqual(1.0, error.eval(), 5)
 
   def testSingleUpdateWithErrorAndWeights1(self):
@@ -3299,8 +3300,8 @@ class MeanCosineDistanceTest(test.TestCase):
         labels, predictions, dim=2, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertEqual(0, sess.run(update_op))
+      self.evaluate(variables.local_variables_initializer())
+      self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
   def testSingleUpdateWithErrorAndWeights2(self):
@@ -3318,7 +3319,7 @@ class MeanCosineDistanceTest(test.TestCase):
         labels, predictions, dim=2, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.5, update_op.eval())
       self.assertEqual(1.5, error.eval())
 
@@ -3360,10 +3361,10 @@ class PcntBelowThreshTest(test.TestCase):
       pcnt1, update_op1 = metrics.percentage_below(values, 7, name='medium')
       pcnt2, update_op2 = metrics.percentage_below(values, 1, name='low')
 
-      sess.run(variables.local_variables_initializer())
-      sess.run([update_op0, update_op1, update_op2])
+      self.evaluate(variables.local_variables_initializer())
+      self.evaluate([update_op0, update_op1, update_op2])
 
-      pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2])
+      pcnt0, pcnt1, pcnt2 = self.evaluate([pcnt0, pcnt1, pcnt2])
       self.assertAlmostEqual(1.0, pcnt0, 5)
       self.assertAlmostEqual(0.75, pcnt1, 5)
       self.assertAlmostEqual(0.0, pcnt2, 5)
@@ -3382,11 +3383,11 @@ class PcntBelowThreshTest(test.TestCase):
       pcnt2, update_op2 = metrics.percentage_below(
           values, 1, weights=weights, name='low')
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertListEqual([1.0, 0.5, 0.0],
-                           sess.run([update_op0, update_op1, update_op2]))
+                           self.evaluate([update_op0, update_op1, update_op2]))
 
-      pcnt0, pcnt1, pcnt2 = sess.run([pcnt0, pcnt1, pcnt2])
+      pcnt0, pcnt1, pcnt2 = self.evaluate([pcnt0, pcnt1, pcnt2])
       self.assertAlmostEqual(1.0, pcnt0, 5)
       self.assertAlmostEqual(0.5, pcnt1, 5)
       self.assertAlmostEqual(0.0, pcnt2, 5)
@@ -3446,11 +3447,11 @@ class MeanIOUTest(test.TestCase):
         labels, predictions, num_classes=num_classes)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_mean_iou = mean_iou.eval()
@@ -3482,9 +3483,9 @@ class MeanIOUTest(test.TestCase):
 
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0 / 2.0, 1.0 / 4.0, 0.])
       self.assertEqual(desired_output, miou.eval())
 
@@ -3529,7 +3530,7 @@ class MeanIOUTest(test.TestCase):
 
       variables.local_variables_initializer().run()
       for _ in range(6):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([2.0 / 3.0, 1.0 / 2.0])
       self.assertAlmostEqual(desired_output, mean_iou.eval())
 
@@ -3563,9 +3564,9 @@ class MeanIOUTest(test.TestCase):
 
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
@@ -3587,7 +3588,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       confusion_matrix = update_op.eval()
       self.assertAllEqual([[3, 0], [2, 5]], confusion_matrix)
       desired_miou = np.mean([3. / 5., 5. / 7.])
@@ -3599,7 +3600,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 1
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(40, update_op.eval()[0])
       self.assertEqual(1.0, miou.eval())
 
@@ -3609,7 +3610,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[0, 0], [40, 0]], update_op.eval())
       self.assertEqual(0., miou.eval())
 
@@ -3640,7 +3641,7 @@ class MeanIOUTest(test.TestCase):
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(
           labels, predictions, num_classes, weights=weights)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[2, 0], [2, 4]], update_op.eval())
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
@@ -3659,7 +3660,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 3
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
       self.assertAlmostEqual(
           1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
@@ -3671,7 +3672,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 2
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
       self.assertAlmostEqual(1, miou.eval())
 
@@ -3689,7 +3690,7 @@ class MeanIOUTest(test.TestCase):
     num_classes = 3
     with self.cached_session() as sess:
       miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
       self.assertAlmostEqual(
           1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
@@ -3752,11 +3753,11 @@ class MeanPerClassAccuracyTest(test.TestCase):
         labels, predictions, num_classes=num_classes)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
 
       # Run several updates.
       for _ in range(10):
-        sess.run(update_op)
+        self.evaluate(update_op)
 
       # Then verify idempotency.
       initial_mean_accuracy = mean_accuracy.eval()
@@ -3788,9 +3789,9 @@ class MeanPerClassAccuracyTest(test.TestCase):
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0, 1.0 / 3.0, 0.0])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3835,7 +3836,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
 
       variables.local_variables_initializer().run()
       for _ in range(6):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([2.0 / 2.0, 0.5 / 1.5])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3870,9 +3871,9 @@ class MeanPerClassAccuracyTest(test.TestCase):
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
 
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       for _ in range(5):
-        sess.run(update_op)
+        self.evaluate(update_op)
       desired_output = np.mean([1.0 / 2.0, 2.0 / 3.0, 0.])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
@@ -3883,7 +3884,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertEqual(1.0, update_op.eval()[0])
       self.assertEqual(1.0, mean_accuracy.eval())
 
@@ -3894,7 +3895,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual([0.0, 0.0], update_op.eval())
       self.assertEqual(0., mean_accuracy.eval())
 
@@ -3913,7 +3914,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     with self.cached_session() as sess:
       mean_accuracy, update_op = metrics.mean_per_class_accuracy(
           labels, predictions, num_classes, weights=weights)
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       desired_accuracy = np.array([2. / 2., 4. / 6.], dtype=np.float32)
       self.assertAllEqual(desired_accuracy, update_op.eval())
       desired_mean_accuracy = np.mean(desired_accuracy)
@@ -3945,7 +3946,7 @@ class FalseNegativesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
@@ -3964,7 +3965,7 @@ class FalseNegativesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(5., tn_update_op.eval())
       self.assertAllClose(5., tn.eval())
@@ -3994,7 +3995,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), fn.eval())
       self.assertAllEqual((0, 2, 3), fn_update_op.eval())
       self.assertAllEqual((0, 2, 3), fn.eval())
@@ -4013,7 +4014,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), fn.eval())
       self.assertAllEqual((0.0, 8.0, 11.0), fn_update_op.eval())
       self.assertAllEqual((0.0, 8.0, 11.0), fn.eval())
@@ -4044,7 +4045,7 @@ class FalsePositivesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
@@ -4063,7 +4064,7 @@ class FalsePositivesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(14., tn_update_op.eval())
       self.assertAllClose(14., tn.eval())
@@ -4093,7 +4094,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), fp.eval())
       self.assertAllEqual((7, 4, 2), fp_update_op.eval())
       self.assertAllEqual((7, 4, 2), fp.eval())
@@ -4114,7 +4115,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), fp.eval())
       self.assertAllEqual((125.0, 42.0, 12.0), fp_update_op.eval())
       self.assertAllEqual((125.0, 42.0, 12.0), fp.eval())
@@ -4145,7 +4146,7 @@ class TrueNegativesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
@@ -4164,7 +4165,7 @@ class TrueNegativesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(4., tn_update_op.eval())
       self.assertAllClose(4., tn.eval())
@@ -4194,7 +4195,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), tn.eval())
       self.assertAllEqual((2, 5, 7), tn_update_op.eval())
       self.assertAllEqual((2, 5, 7), tn.eval())
@@ -4213,7 +4214,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), tn.eval())
       self.assertAllEqual((5.0, 15.0, 23.0), tn_update_op.eval())
       self.assertAllEqual((5.0, 15.0, 23.0), tn.eval())
@@ -4244,7 +4245,7 @@ class TruePositivesTest(test.TestCase):
         labels=labels, predictions=predictions)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
@@ -4263,7 +4264,7 @@ class TruePositivesTest(test.TestCase):
         labels=labels, predictions=predictions, weights=weights)
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllClose(0., tn.eval())
       self.assertAllClose(12., tn_update_op.eval())
       self.assertAllClose(12., tn.eval())
@@ -4293,7 +4294,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         predictions=predictions, labels=labels, thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0, 0, 0), tp.eval())
       self.assertAllEqual((3, 1, 0), tp_update_op.eval())
       self.assertAllEqual((3, 1, 0), tp.eval())
@@ -4310,7 +4311,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
 
     with self.cached_session() as sess:
-      sess.run(variables.local_variables_initializer())
+      self.evaluate(variables.local_variables_initializer())
       self.assertAllEqual((0.0, 0.0, 0.0), tp.eval())
       self.assertAllEqual((111.0, 37.0, 0.0), tp_update_op.eval())
       self.assertAllEqual((111.0, 37.0, 0.0), tp.eval())
diff --git a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
index 15e3826542..87f1991aa7 100644
--- a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
@@ -142,8 +142,8 @@ class DepthwiseConv2DTest(test.TestCase):
       conv_interface = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
 
-      native_result = sess.run(conv_native)
-      interface_result = sess.run(conv_interface)
+      native_result = self.evaluate(conv_native)
+      interface_result = self.evaluate(conv_interface)
 
     print("depthwise conv_2d: ", tensor_in_sizes, "*", filter_in_sizes,
           ", stride:", stride, ", padding: ", padding, ", max diff: ",
@@ -211,7 +211,7 @@ class DepthwiseConv2DTest(test.TestCase):
         t2 = constant_op.constant(x2, shape=filter_in_sizes)
         conv = nn_ops.depthwise_conv2d_native(
             t1, t2, strides=[1, stride, stride, 1], padding=padding)
-        value = sess.run(conv)
+        value = self.evaluate(conv)
     print("value = ", value)
     self.assertAllClose(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py
index e202b6e8a4..5ff0c58bf1 100644
--- a/tensorflow/python/kernel_tests/norm_op_test.py
+++ b/tensorflow/python/kernel_tests/norm_op_test.py
@@ -70,7 +70,7 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_):
         tf_matrix = constant_op.constant(matrix)
         tf_norm = linalg_ops.norm(
             tf_matrix, ord=ord_, axis=axis_, keepdims=keep_dims_)
-        tf_norm_val = sess.run(tf_norm)
+        tf_norm_val = self.evaluate(tf_norm)
       else:
         tf_matrix = array_ops.placeholder(dtype_)
         tf_norm = linalg_ops.norm(
diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py
index 338b6cec01..6cd4974671 100644
--- a/tensorflow/python/kernel_tests/nth_element_op_test.py
+++ b/tensorflow/python/kernel_tests/nth_element_op_test.py
@@ -35,7 +35,7 @@ class NthElementTest(test.TestCase):
     with self.cached_session(use_gpu=False) as sess:
       inputs_op = ops.convert_to_tensor(inputs, dtype=dtype)
       values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse)
-      values = sess.run(values_op)
+      values = self.evaluate(values_op)
 
       self.assertShapeEqual(np_expected_values, values_op)
       self.assertAllClose(np_expected_values, values)
diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
index 520b663375..b4818360d5 100644
--- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
@@ -126,7 +126,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -158,7 +158,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       results = []
 
       def dequeue():
-        results.append(sess.run(dequeued_t))
+        results.append(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops]
       for thread in threads:
@@ -193,13 +193,13 @@ class PaddingFIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       results = []
 
       def dequeue():
         for _ in xrange(len(elems)):
-          results.append(sess.run(dequeued_t))
+          results.append(self.evaluate(dequeued_t))
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -224,7 +224,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         enqueue_op.run()
 
       for i in xrange(len(elems)):
-        x_val, y_val = sess.run(dequeued_t)
+        x_val, y_val = self.evaluate(dequeued_t)
         x, y = elems[i]
         self.assertEqual([x], x_val)
         self.assertEqual([y], y_val)
@@ -327,7 +327,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -344,7 +344,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       for i in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         self.assertEqual(float_elems[i % 4], float_val)
         self.assertAllEqual(int_elems[i % 4], int_val)
 
@@ -387,17 +387,17 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
@@ -418,7 +418,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       enqueue_op.run()
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[0:4], float_val)
       self.assertAllEqual(int_elems[0:4], int_val)
       self.assertTrue(
@@ -428,11 +428,11 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertAllEqual(float_elems[4:8], float_val)
       self.assertAllEqual(int_elems[4:8], int_val)
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual(float_elems[8], float_val)
       self.assertAllEqual(int_elems[8], int_val)
       self.assertTrue(
@@ -459,7 +459,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       for enqueue_op in enqueue_ops:
         enqueue_op.run()
-      string_val, int_val = sess.run(dequeued_t)
+      string_val, int_val = self.evaluate(dequeued_t)
 
       self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""],
                            [b"abc", b"", b""], [b"abc", b"d", b""],
@@ -473,7 +473,7 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      string_val, int_val = sess.run(dequeued_single_t)
+      string_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val)
       self.assertAllEqual([[1, 2, 3, 4]], int_val)
       self.assertTrue(
@@ -500,7 +500,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       for enqueue_op in enqueue_ops:
         enqueue_op.run()
-      string_val, int_val = sess.run(dequeued_t)
+      string_val, int_val = self.evaluate(dequeued_t)
 
       self.assertAllEqual([[b"a", b"", b""], [b"ab", b"", b""],
                            [b"abc", b"", b""], [b"abc", b"d", b""],
@@ -514,7 +514,7 @@ class PaddingFIFOQueueTest(test.TestCase):
           tensor_shape.TensorShape(int_val.shape).is_compatible_with(dequeued_t[
               1].get_shape()))
 
-      string_val, int_val = sess.run(dequeued_single_t)
+      string_val, int_val = self.evaluate(dequeued_single_t)
       self.assertAllEqual([b"abc", b"d", b"e", b"f"], string_val)
       self.assertAllEqual([[1, 2, 3, 4]], int_val)
       self.assertTrue(
@@ -633,7 +633,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -656,7 +656,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t))
+        dequeued_elems.extend(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in range(10)]
       for thread in threads:
@@ -680,7 +680,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t))
+        dequeued_elems.extend(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in range(10)]
       for thread in threads:
@@ -700,11 +700,11 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def enqueue():
         for _ in xrange(100):
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       def dequeue():
         for _ in xrange(100):
-          self.assertTrue(sess.run(dequeued_t) in (10.0, 20.0))
+          self.assertTrue(self.evaluate(dequeued_t) in (10.0, 20.0))
 
       enqueue_threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       dequeue_threads = [self.checkedThread(target=dequeue) for _ in range(10)]
@@ -736,7 +736,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         for i in xrange(250):
-          self.assertEqual(i, sess.run(dequeued_t))
+          self.assertEqual(i, self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -767,7 +767,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       dequeuemany_t = q.dequeue_many(count_placeholder)
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -805,10 +805,10 @@ class PaddingFIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t).tolist())
+        dequeued_elems.extend(self.evaluate(dequeued_t).tolist())
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -832,10 +832,10 @@ class PaddingFIFOQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t).tolist())
+        dequeued_elems.extend(self.evaluate(dequeued_t).tolist())
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -901,11 +901,11 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         for elem in elems:
-          self.assertEqual([elem], sess.run(dequeued_t))
+          self.assertEqual([elem], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -926,8 +926,8 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
-        self.assertAllEqual(elems[3:], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
+        self.assertAllEqual(elems[3:], self.evaluate(dequeued_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -947,7 +947,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -968,11 +968,11 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems, sess.run(dequeued_t))
+        self.assertAllEqual(elems, self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -993,11 +993,11 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def dequeue():
-        self.assertAllEqual(elems[:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[:3], self.evaluate(dequeued_t))
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -1017,16 +1017,16 @@ class PaddingFIFOQueueTest(test.TestCase):
       cleanup_dequeue_t = q.dequeue()
 
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        self.assertAllEqual(elems[0:3], sess.run(dequeued_t))
+        self.assertAllEqual(elems[0:3], self.evaluate(dequeued_t))
         with self.assertRaises(errors_impl.OutOfRangeError):
-          sess.run(dequeued_t)
-        self.assertEqual(elems[3], sess.run(cleanup_dequeue_t))
+          self.evaluate(dequeued_t)
+        self.assertEqual(elems[3], self.evaluate(cleanup_dequeue_t))
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       enqueue_thread = self.checkedThread(target=enqueue)
       enqueue_thread.start()
@@ -1059,7 +1059,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def dequeue():
         with self.assertRaises(errors_impl.OutOfRangeError):
-          sess.run([dequeued_a_t, dequeued_b_t])
+          self.evaluate([dequeued_a_t, dequeued_b_t])
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -1072,7 +1072,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       # Test that the elements in the partially-dequeued batch are
       # restored in the correct order.
       for elem_a, elem_b in zip(elems_a, elems_b):
-        val_a, val_b = sess.run([cleanup_dequeue_a_t, cleanup_dequeue_b_t])
+        val_a, val_b = self.evaluate([cleanup_dequeue_a_t, cleanup_dequeue_b_t])
         self.assertEqual(elem_a, val_a)
         self.assertEqual(elem_b, val_b)
       self.assertEqual(0, q.size().eval())
@@ -1087,7 +1087,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -1107,7 +1107,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -1155,7 +1155,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1178,7 +1178,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -1207,7 +1207,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # Expect the operation to succeed once the dequeue op runs.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1217,7 +1217,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1242,7 +1242,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       enqueue_thread = self.checkedThread(target=blocking_enqueue)
       enqueue_thread.start()
@@ -1252,7 +1252,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       close_thread = self.checkedThread(target=close)
       close_thread.start()
@@ -1379,19 +1379,19 @@ class PaddingFIFOQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_op)
+      self.evaluate(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_many_op)
+      self.evaluate(dequeue_many_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_many_op)
+      self.evaluate(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1434,7 +1434,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       def blocking_enqueue():
         enq_done.append(False)
         # This will fill the queue and then block until enough dequeues happen.
-        sess.run(enq)
+        self.evaluate(enq)
         enq_done.append(True)
 
       thread = self.checkedThread(target=blocking_enqueue)
@@ -1444,14 +1444,14 @@ class PaddingFIFOQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1477,7 +1477,7 @@ class PaddingFIFOQueueTest(test.TestCase):
 
       def blocking_dequeue():
         # Will only complete after 4 enqueues complete.
-        results.extend(sess.run(deq))
+        results.extend(self.evaluate(deq))
 
       thread = self.checkedThread(target=blocking_dequeue)
       thread.start()
@@ -1486,7 +1486,7 @@ class PaddingFIFOQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         self.assertEqual(len(results), 0)
-        sess.run(enq)
+        self.evaluate(enq)
 
       # Enough enqueued to unblock the dequeue
       thread.join()
@@ -1517,7 +1517,7 @@ class PaddingFIFOQueueTest(test.TestCase):
       q.enqueue_many(input_tuple).run()
 
       output_tuple_t = q.dequeue_many(32)
-      output_tuple = sess.run(output_tuple_t)
+      output_tuple = self.evaluate(output_tuple_t)
 
       for (input_elem, output_elem) in zip(input_tuple, output_tuple):
         self.assertAllEqual(input_elem, output_elem)
diff --git a/tensorflow/python/kernel_tests/parse_single_example_op_test.py b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
index a84895a287..3f50087282 100644
--- a/tensorflow/python/kernel_tests/parse_single_example_op_test.py
+++ b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
@@ -107,7 +107,7 @@ class ParseExampleTest(test.TestCase):
         for result_dict in [out, out_with_example_name]:
           result = flatten_values_tensors_or_sparse(result_dict.values())
           # Check values.
-          tf_result = sess.run(result)
+          tf_result = self.evaluate(result)
           _compare_output_to_expected(self, result_dict, expected_values,
                                       tf_result)
 
diff --git a/tensorflow/python/kernel_tests/parsing_ops_test.py b/tensorflow/python/kernel_tests/parsing_ops_test.py
index 8f359bd32c..1f677103dc 100644
--- a/tensorflow/python/kernel_tests/parsing_ops_test.py
+++ b/tensorflow/python/kernel_tests/parsing_ops_test.py
@@ -101,7 +101,7 @@ class ParseExampleTest(test.TestCase):
         out = parsing_ops.parse_example(**kwargs)
         result = flatten_values_tensors_or_sparse(out.values())
         # Check values.
-        tf_result = sess.run(result)
+        tf_result = self.evaluate(result)
         _compare_output_to_expected(self, out, expected_values, tf_result)
 
       # Check shapes; if serialized is a Tensor we need its size to
@@ -1614,7 +1614,7 @@ class DecodeJSONExampleTest(test.TestCase):
           shape=examples.shape,
           dtype=dtypes.string)
       binary_tensor = parsing_ops.decode_json_example(json_tensor)
-      binary_val = sess.run(binary_tensor)
+      binary_val = self.evaluate(binary_tensor)
 
       if examples.shape:
         self.assertShapeEqual(binary_val, json_tensor)
@@ -1700,7 +1700,7 @@ class DecodeJSONExampleTest(test.TestCase):
       json_tensor = constant_op.constant(["{]"])
       binary_tensor = parsing_ops.decode_json_example(json_tensor)
       with self.assertRaisesOpError("Error while parsing JSON"):
-        sess.run(binary_tensor)
+        self.evaluate(binary_tensor)
 
 
 class ParseTensorOpTest(test.TestCase):
diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
index e393c7a022..a8e962bc3a 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
@@ -81,7 +81,7 @@ class PoolingTest(test.TestCase):
           data_format=data_format)
       if data_format == "NCDHW":
         t = test_util.NCHWToNHWC(t)
-      vals = sess.run(t)
+      vals = self.evaluate(t)
     # Verifies values.
     actual = vals.flatten()
     self.assertAllClose(expected, actual)
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 61628c4756..81222719f2 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -826,7 +826,7 @@ class PoolingTest(test.TestCase):
           strides=[1, 1, 1, 1],
           Targmax=dtypes.int64,
           padding="VALID")
-      out, argmax = sess.run([out_op, argmax_op])
+      out, argmax = self.evaluate([out_op, argmax_op])
       self.assertShapeEqual(out, out_op)
       self.assertShapeEqual(argmax, argmax_op)
       self.assertAllClose(out.ravel(), [1.0, 1.0, 1.0, 1.0])
diff --git a/tensorflow/python/kernel_tests/priority_queue_test.py b/tensorflow/python/kernel_tests/priority_queue_test.py
index 73a9c81638..9be682ea52 100644
--- a/tensorflow/python/kernel_tests/priority_queue_test.py
+++ b/tensorflow/python/kernel_tests/priority_queue_test.py
@@ -50,7 +50,7 @@ class PriorityQueueTest(test.TestCase):
         enq.run()
 
       deq = q.dequeue_many(100)
-      deq_elem, deq_value_0, deq_value_1 = sess.run(deq)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq)
 
       allowed = {}
       missed = set()
@@ -81,7 +81,7 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       dequeue_op = q.dequeue_many(100)
 
@@ -93,7 +93,7 @@ class PriorityQueueTest(test.TestCase):
       for t in enqueue_threads:
         t.start()
 
-      deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op)
 
       for t in enqueue_threads:
         t.join()
@@ -132,12 +132,12 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       dequeued = []
 
       def dequeue(dequeue_op):
-        (dequeue_indices, dequeue_values) = sess.run(dequeue_op)
+        (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op)
         self.assertAllEqual(dequeue_indices, dequeue_values)
         dequeued.extend(dequeue_indices)
 
@@ -184,10 +184,10 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue(dequeue_op, dequeued):
-        (dequeue_indices, dequeue_values) = sess.run(dequeue_op)
+        (dequeue_indices, dequeue_values) = self.evaluate(dequeue_op)
         self.assertAllEqual(dequeue_indices, dequeue_values)
         dequeue_wait.acquire()
         dequeued.extend(dequeue_indices)
@@ -215,7 +215,7 @@ class PriorityQueueTest(test.TestCase):
 
       # We can't guarantee full sorting because we can't guarantee
       # that the dequeued.extend() call runs immediately after the
-      # sess.run() call.  Here we're just happy everything came out.
+      # self.evaluate() call.  Here we're just happy everything came out.
       self.assertAllEqual(set(dequeued), set(all_enqueued_values))
 
   def testRoundTripInsertManyMultiThreadedReadOnceSorts(self):
@@ -236,7 +236,7 @@ class PriorityQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       dequeue_op = q.dequeue_many(100)
 
@@ -248,7 +248,7 @@ class PriorityQueueTest(test.TestCase):
       for t in enqueue_threads:
         t.start()
 
-      deq_elem, deq_value_0, deq_value_1 = sess.run(dequeue_op)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(dequeue_op)
 
       for t in enqueue_threads:
         t.join()
@@ -276,7 +276,7 @@ class PriorityQueueTest(test.TestCase):
       side_value_1 = np.random.rand(1000).astype(bytes)
       q.enqueue_many((elem, side_value_0, side_value_1)).run()
       deq = q.dequeue_many(1000)
-      deq_elem, deq_value_0, deq_value_1 = sess.run(deq)
+      deq_elem, deq_value_0, deq_value_1 = self.evaluate(deq)
 
       allowed = {}
       for e, v0, v1 in zip(elem, side_value_0, side_value_1):
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index b101da036e..c9cbe44a7f 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -307,9 +307,9 @@ class PyFuncTest(test.TestCase):
     with session_lib.Session() as sess:
       producer = iter(range(3))
       x, = script_ops.py_func(lambda: next(producer), [], [dtypes.int64])
-      self.assertEqual(sess.run(x), 0)
-      self.assertEqual(sess.run(x), 1)
-      self.assertEqual(sess.run(x), 2)
+      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(self.evaluate(x), 1)
+      self.assertEqual(self.evaluate(x), 2)
 
   def testStateless(self):
     # Not using self.cached_session(), which disables optimization.
@@ -317,9 +317,9 @@ class PyFuncTest(test.TestCase):
       producer = iter(range(3))
       x, = script_ops.py_func(
           lambda: next(producer), [], [dtypes.int64], stateful=False)
-      self.assertEqual(sess.run(x), 0)
-      self.assertEqual(sess.run(x), 0)
-      self.assertEqual(sess.run(x), 0)
+      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(self.evaluate(x), 0)
+      self.assertEqual(self.evaluate(x), 0)
 
   def testGradientFunction(self):
     # Input to tf.py_func is necessary, otherwise get_gradient_function()
@@ -390,7 +390,7 @@ class PyFuncTest(test.TestCase):
     f = script_ops.py_func(
         do_nothing, [constant_op.constant(3, dtypes.int64)], [], stateful=False)
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(f), [])
+      self.assertEqual(self.evaluate(f), [])
 
   def _testExceptionHandling(self, py_exp, tf_exp, eager=False):
 
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 617b724204..305b5aa364 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -60,7 +60,7 @@ class QrOpTest(test.TestCase):
             q1, r1 = linalg_ops.qr(matrix1, full_matrices=full_matrices_)
             q2, r2 = linalg_ops.qr(matrix2, full_matrices=full_matrices_)
             all_ops += [q1, r1, q2, r2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       for i in range(8):
         q = 4 * i
         self.assertAllEqual(val[q], val[q + 2])  # q1 == q2
@@ -129,7 +129,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
       q_tf, r_tf = linalg_ops.qr(x_tf, full_matrices=full_matrices_)
 
       if use_static_shape_:
-        q_tf_val, r_tf_val = sess.run([q_tf, r_tf])
+        q_tf_val, r_tf_val = self.evaluate([q_tf, r_tf])
       else:
         q_tf_val, r_tf_val = sess.run([q_tf, r_tf], feed_dict={x_tf: x_np})
 
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
index 0023506b77..cab841741e 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
@@ -39,7 +39,7 @@ class MultinomialTest(test.TestCase):
           num_samples=1000000,
           seed=15)
       for _ in range(100):
-        x = sess.run(samples)
+        x = self.evaluate(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
@@ -57,7 +57,7 @@ class MultinomialTest(test.TestCase):
           num_samples=1000000,
           seed=15)
       for _ in range(100):
-        x = sess.run(samples)
+        x = self.evaluate(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
@@ -79,7 +79,7 @@ class MultinomialTest(test.TestCase):
       # we'll run out of memory if we try to draw 1e9 samples directly
       # really should fit in 12GB of memory...
       for _ in range(100):
-        x = sess.run(samples)
+        x = self.evaluate(samples)
         indices, counts = np.unique(x, return_counts=True)
         for index, count in zip(indices, counts):
           if index in counts_by_indices.keys():
diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py
index 606e8862c4..d18e3feb04 100644
--- a/tensorflow/python/kernel_tests/random/random_gamma_test.py
+++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py
@@ -48,7 +48,7 @@ class RandomGammaTest(test.TestCase):
             [num], alpha, beta=beta, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
index 6de894846b..76618316b2 100644
--- a/tensorflow/python/kernel_tests/random/random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/random_ops_test.py
@@ -49,9 +49,9 @@ class RandomOpTestCommon(test.TestCase):
         random_seed.set_random_seed(graph_seed)
       x = rng_func([num], min_or_mean, max_or_stddev, dtype=dtype, seed=op_seed)
 
-      y = sess.run(x)
-      z = sess.run(x)
-      w = sess.run(x)
+      y = self.evaluate(x)
+      z = self.evaluate(x)
+      w = self.evaluate(x)
 
       # We use exact equality here. If the random-number generator is producing
       # the same output, all three outputs will be bitwise identical.
@@ -69,7 +69,7 @@ class RandomNormalTest(RandomOpTestCommon):
             [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
@@ -160,7 +160,7 @@ class TruncatedNormalTest(test.TestCase):
             [num], mean=mu, stddev=sigma, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
@@ -256,7 +256,7 @@ class RandomUniformTest(RandomOpTestCommon):
             [num], minval=minv, maxval=maxv, dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
index 95e48101f6..47c0858db7 100644
--- a/tensorflow/python/kernel_tests/random/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py
@@ -43,7 +43,7 @@ class RandomPoissonTest(test.TestCase):
         rng = random_ops.random_poisson(lam, [num], dtype=dtype, seed=seed)
         ret = np.empty([10, num])
         for i in xrange(10):
-          ret[i, :] = sess.run(rng)
+          ret[i, :] = self.evaluate(rng)
       return ret
 
     return func
diff --git a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
index f3fcf1eff7..ed4f5434d9 100644
--- a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
@@ -84,9 +84,9 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeue_t = q.dequeue()
       results = []
       for _ in range(2):
-        a, b = sess.run(dequeue_t)
+        a, b = self.evaluate(dequeue_t)
         results.append((a, b))
-      a, b = sess.run(q.dequeue_many(3))
+      a, b = self.evaluate(q.dequeue_many(3))
       for i in range(3):
         results.append((a[i], b[i]))
       self.assertItemsEqual([(1, [5]), (2, [6]), (3, [7]), (4, [8]), (9, [10])],
@@ -101,7 +101,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       # Run one producer thread for each element in elems.
       def enqueue(enqueue_op):
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [
           self.checkedThread(
@@ -133,7 +133,7 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
 
       def dequeue():
-        results.append(sess.run(dequeued_t))
+        results.append(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in enqueue_ops]
       for thread in threads:
@@ -167,13 +167,13 @@ class RandomShuffleQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         for enqueue_op in enqueue_ops:
-          sess.run(enqueue_op)
+          self.evaluate(enqueue_op)
 
       results = []
 
       def dequeue():
         for _ in xrange(len(elems)):
-          results.append(sess.run(dequeued_t))
+          results.append(self.evaluate(dequeued_t))
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -197,7 +197,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
       for _ in xrange(len(elems)):
-        x, y = sess.run(dequeued_t)
+        x, y = self.evaluate(dequeued_t)
         results.append((x, y))
       self.assertItemsEqual(elems, results)
 
@@ -321,7 +321,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
       for _ in range(8):
-        float_val, int_val = sess.run(dequeued_t)
+        float_val, int_val = self.evaluate(dequeued_t)
         results.append((float_val, [int_val[0], int_val[1]]))
       expected = list(zip(float_elems, int_elems)) * 2
       self.assertItemsEqual(expected, results)
@@ -368,20 +368,20 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       results = []
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       self.assertEqual(float_val.shape, dequeued_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_t[1].get_shape())
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape())
       results.append((float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       results.append((float_val, int_val.tolist()))
 
       self.assertItemsEqual(zip(float_elems, int_elems), results)
@@ -402,21 +402,21 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       results = []
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       # dequeue_up_to has undefined shape.
       self.assertEqual([None], dequeued_t[0].get_shape().as_list())
       self.assertEqual([None, 2], dequeued_t[1].get_shape().as_list())
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_t)
+      float_val, int_val = self.evaluate(dequeued_t)
       results.extend(zip(float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       self.assertEqual(float_val.shape, dequeued_single_t[0].get_shape())
       self.assertEqual(int_val.shape, dequeued_single_t[1].get_shape())
       results.append((float_val, int_val.tolist()))
 
-      float_val, int_val = sess.run(dequeued_single_t)
+      float_val, int_val = self.evaluate(dequeued_single_t)
       results.append((float_val, int_val.tolist()))
 
       self.assertItemsEqual(zip(float_elems, int_elems), results)
@@ -442,7 +442,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       # Enqueue 100 items in parallel on 10 threads.
       def enqueue():
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       threads = [self.checkedThread(target=enqueue) for _ in range(10)]
       for thread in threads:
@@ -466,7 +466,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t))
+        dequeued_elems.extend(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in range(10)]
       for thread in threads:
@@ -489,7 +489,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t))
+        dequeued_elems.extend(self.evaluate(dequeued_t))
 
       threads = [self.checkedThread(target=dequeue) for _ in range(10)]
       for thread in threads:
@@ -515,7 +515,7 @@ class RandomShuffleQueueTest(test.TestCase):
       dequeued_elems = []
 
       def dequeue(dequeue_op):
-        dequeued_elems.extend(sess.run(dequeue_op))
+        dequeued_elems.extend(self.evaluate(dequeue_op))
 
       threads = []
       for dequeue_op in dequeue_ops:
@@ -539,10 +539,10 @@ class RandomShuffleQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t).tolist())
+        dequeued_elems.extend(self.evaluate(dequeued_t).tolist())
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -566,10 +566,10 @@ class RandomShuffleQueueTest(test.TestCase):
         # The enqueue_op should run after the dequeue op has blocked.
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
-        sess.run(enqueue_op)
+        self.evaluate(enqueue_op)
 
       def dequeue():
-        dequeued_elems.extend(sess.run(dequeued_t).tolist())
+        dequeued_elems.extend(self.evaluate(dequeued_t).tolist())
 
       enqueue_thread = self.checkedThread(target=enqueue)
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -665,18 +665,18 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
 
       # Manually dequeue until we hit min_size.
-      results.append(sess.run(dequeued_t))
-      results.append(sess.run(dequeued_t))
+      results.append(self.evaluate(dequeued_t))
+      results.append(self.evaluate(dequeued_t))
 
       def blocking_dequeue():
-        results.append(sess.run(dequeued_t))
-        results.append(sess.run(dequeued_t))
+        results.append(self.evaluate(dequeued_t))
+        results.append(self.evaluate(dequeued_t))
 
         self.assertItemsEqual(elems, results)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=blocking_dequeue)
       dequeue_thread.start()
@@ -701,7 +701,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
         finished.append(True)
 
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -727,12 +727,12 @@ class RandomShuffleQueueTest(test.TestCase):
       progress = []  # Must be mutable
 
       def dequeue():
-        self.assertItemsEqual(elems, sess.run(dequeued_t))
+        self.assertItemsEqual(elems, self.evaluate(dequeued_t))
         progress.append(1)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
         progress.append(2)
 
       self.assertEqual(len(progress), 0)
@@ -763,9 +763,9 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
 
       def dequeue():
-        results.extend(sess.run(dequeued_t))
+        results.extend(self.evaluate(dequeued_t))
         self.assertEquals(3, len(results))
-        results.extend(sess.run(dequeued_t))
+        results.extend(self.evaluate(dequeued_t))
         self.assertEquals(4, len(results))
 
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -794,11 +794,11 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
 
       def dequeue():
-        results.extend(sess.run(dequeued_t))
+        results.extend(self.evaluate(dequeued_t))
         self.assertEquals(3, len(results))
         # min_after_dequeue is 2, we ask for 3 elements, and we end up only
         # getting the remaining 1.
-        results.extend(sess.run(dequeued_t))
+        results.extend(self.evaluate(dequeued_t))
         self.assertEquals(4, len(results))
 
       dequeue_thread = self.checkedThread(target=dequeue)
@@ -824,16 +824,16 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
 
       def dequeue():
-        results.extend(sess.run(dequeued_t))
+        results.extend(self.evaluate(dequeued_t))
         self.assertEqual(len(results), 3)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
         # While the last dequeue failed, we want to insure that it returns
         # any elements that it potentially reserved to dequeue. Thus the
         # next cleanup should return a single element.
-        results.extend(sess.run(cleanup_dequeue_t))
+        results.extend(self.evaluate(cleanup_dequeue_t))
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -854,7 +854,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -874,7 +874,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
-          sess.run(dequeued_t)
+          self.evaluate(dequeued_t)
 
       dequeue_thread = self.checkedThread(target=dequeue)
       dequeue_thread.start()
@@ -922,7 +922,7 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -950,7 +950,7 @@ class RandomShuffleQueueTest(test.TestCase):
       enqueue_op.run()
 
       def blocking_enqueue():
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread = self.checkedThread(target=blocking_enqueue)
       thread.start()
@@ -987,11 +987,11 @@ class RandomShuffleQueueTest(test.TestCase):
       def blocking_enqueue():
         # Expect the operation to succeed since it will complete
         # before the queue is closed.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"):
-          sess.run(blocking_enqueue_op)
+          self.evaluate(blocking_enqueue_op)
 
       thread1 = self.checkedThread(target=blocking_enqueue)
       thread1.start()
@@ -1001,7 +1001,7 @@ class RandomShuffleQueueTest(test.TestCase):
       time.sleep(0.1)
 
       def blocking_close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       thread2 = self.checkedThread(target=blocking_close)
       thread2.start()
@@ -1032,7 +1032,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       def blocking_enqueue():
         # This will block until the dequeue after the close.
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
       thread1 = self.checkedThread(target=blocking_enqueue)
       thread1.start()
@@ -1050,7 +1050,7 @@ class RandomShuffleQueueTest(test.TestCase):
         time.sleep(0.1)
 
       def blocking_close():
-        sess.run(close_op)
+        self.evaluate(close_op)
 
       thread2 = self.checkedThread(target=blocking_close)
       thread2.start()
@@ -1064,7 +1064,7 @@ class RandomShuffleQueueTest(test.TestCase):
       # At this point the close operation will complete, so the next enqueue
       # will fail.
       with self.assertRaisesRegexp(errors_impl.CancelledError, "closed"):
-        sess.run(blocking_enqueue_op)
+        self.evaluate(blocking_enqueue_op)
 
   def testSharedQueueSameSession(self):
     with self.cached_session():
@@ -1216,23 +1216,23 @@ class RandomShuffleQueueTest(test.TestCase):
 
   def _blockingDequeue(self, sess, dequeue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_op)
+      self.evaluate(dequeue_op)
 
   def _blockingDequeueMany(self, sess, dequeue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_many_op)
+      self.evaluate(dequeue_many_op)
 
   def _blockingDequeueUpTo(self, sess, dequeue_up_to_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(dequeue_up_to_op)
+      self.evaluate(dequeue_up_to_op)
 
   def _blockingEnqueue(self, sess, enqueue_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_op)
+      self.evaluate(enqueue_op)
 
   def _blockingEnqueueMany(self, sess, enqueue_many_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(enqueue_many_op)
+      self.evaluate(enqueue_many_op)
 
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
@@ -1383,7 +1383,7 @@ class RandomShuffleQueueTest(test.TestCase):
       def blocking_enqueue():
         enq_done.append(False)
         # This will fill the queue and then block until enough dequeues happen.
-        sess.run(enq)
+        self.evaluate(enq)
         enq_done.append(True)
 
       thread = self.checkedThread(target=blocking_enqueue)
@@ -1393,14 +1393,14 @@ class RandomShuffleQueueTest(test.TestCase):
       results = []
       results.append(deq.eval())  # Will only complete after the enqueue starts.
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       for _ in range(3):
         results.append(deq.eval())
 
       time.sleep(0.1)
       self.assertEqual(len(enq_done), 1)
-      self.assertEqual(sess.run(size_op), 5)
+      self.assertEqual(self.evaluate(size_op), 5)
 
       # This dequeue will unblock the thread.
       results.append(deq.eval())
@@ -1426,7 +1426,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       def blocking_dequeue():
         # Will only complete after 4 enqueues complete.
-        results.extend(sess.run(deq))
+        results.extend(self.evaluate(deq))
 
       thread = self.checkedThread(target=blocking_dequeue)
       thread.start()
@@ -1435,7 +1435,7 @@ class RandomShuffleQueueTest(test.TestCase):
         # TODO(mrry): Figure out how to do this without sleeping.
         time.sleep(0.1)
         self.assertEqual(len(results), 0)
-        sess.run(enq)
+        self.evaluate(enq)
 
       # Enough enqueued to unblock the dequeue
       thread.join()
diff --git a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
index 13f97a9367..d80bea955e 100644
--- a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
@@ -62,7 +62,7 @@ class StatelessOpsTest(test.TestCase):
         for stateless_op, stateful_op in cases:
           stateful = stateful_op(seed=seed[1])
           pure = stateless_op(seed=preseed)
-          self.assertAllEqual(stateful.eval(), self.evaluate(pure))
+          self.assertAllEqual(self.evaluate(stateful), self.evaluate(pure))
 
   def _test_determinism(self, cases):
     # Stateless values should be equal iff the seeds are equal (roughly)
diff --git a/tensorflow/python/kernel_tests/reader_ops_test.py b/tensorflow/python/kernel_tests/reader_ops_test.py
index 18a8a3d547..a4a18c5219 100644
--- a/tensorflow/python/kernel_tests/reader_ops_test.py
+++ b/tensorflow/python/kernel_tests/reader_ops_test.py
@@ -140,147 +140,143 @@ class TFCompressionTestCase(test.TestCase):
 
 class IdentityReaderTest(test.TestCase):
 
-  def _ExpectRead(self, sess, key, value, expected):
-    k, v = sess.run([key, value])
+  def _ExpectRead(self, key, value, expected):
+    k, v = self.evaluate([key, value])
     self.assertAllEqual(expected, k)
     self.assertAllEqual(expected, v)
 
   def testOneEpoch(self):
-    with self.cached_session() as sess:
-      reader = io_ops.IdentityReader("test_reader")
-      work_completed = reader.num_work_units_completed()
-      produced = reader.num_records_produced()
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      queued_length = queue.size()
-      key, value = reader.read(queue)
+    reader = io_ops.IdentityReader("test_reader")
+    work_completed = reader.num_work_units_completed()
+    produced = reader.num_records_produced()
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    queued_length = queue.size()
+    key, value = reader.read(queue)
 
-      self.assertAllEqual(0, self.evaluate(work_completed))
-      self.assertAllEqual(0, self.evaluate(produced))
-      self.assertAllEqual(0, self.evaluate(queued_length))
+    self.assertAllEqual(0, self.evaluate(work_completed))
+    self.assertAllEqual(0, self.evaluate(produced))
+    self.assertAllEqual(0, self.evaluate(queued_length))
 
-      queue.enqueue_many([["A", "B", "C"]]).run()
-      queue.close().run()
-      self.assertAllEqual(3, self.evaluate(queued_length))
+    self.evaluate(queue.enqueue_many([["A", "B", "C"]]))
+    self.evaluate(queue.close())
+    self.assertAllEqual(3, self.evaluate(queued_length))
 
-      self._ExpectRead(sess, key, value, b"A")
-      self.assertAllEqual(1, self.evaluate(produced))
+    self._ExpectRead(key, value, b"A")
+    self.assertAllEqual(1, self.evaluate(produced))
 
-      self._ExpectRead(sess, key, value, b"B")
+    self._ExpectRead(key, value, b"B")
 
-      self._ExpectRead(sess, key, value, b"C")
-      self.assertAllEqual(3, self.evaluate(produced))
-      self.assertAllEqual(0, self.evaluate(queued_length))
+    self._ExpectRead(key, value, b"C")
+    self.assertAllEqual(3, self.evaluate(produced))
+    self.assertAllEqual(0, self.evaluate(queued_length))
 
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        sess.run([key, value])
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      self.evaluate([key, value])
 
-      self.assertAllEqual(3, self.evaluate(work_completed))
-      self.assertAllEqual(3, self.evaluate(produced))
-      self.assertAllEqual(0, self.evaluate(queued_length))
+    self.assertAllEqual(3, self.evaluate(work_completed))
+    self.assertAllEqual(3, self.evaluate(produced))
+    self.assertAllEqual(0, self.evaluate(queued_length))
 
   def testMultipleEpochs(self):
-    with self.cached_session() as sess:
-      reader = io_ops.IdentityReader("test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      enqueue = queue.enqueue_many([["DD", "EE"]])
-      key, value = reader.read(queue)
-
-      enqueue.run()
-      self._ExpectRead(sess, key, value, b"DD")
-      self._ExpectRead(sess, key, value, b"EE")
-      enqueue.run()
-      self._ExpectRead(sess, key, value, b"DD")
-      self._ExpectRead(sess, key, value, b"EE")
-      enqueue.run()
-      self._ExpectRead(sess, key, value, b"DD")
-      self._ExpectRead(sess, key, value, b"EE")
-      queue.close().run()
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        sess.run([key, value])
+    reader = io_ops.IdentityReader("test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    enqueue = queue.enqueue_many([["DD", "EE"]])
+    key, value = reader.read(queue)
+
+    self.evaluate(enqueue)
+    self._ExpectRead(key, value, b"DD")
+    self._ExpectRead(key, value, b"EE")
+    self.evaluate(enqueue)
+    self._ExpectRead(key, value, b"DD")
+    self._ExpectRead(key, value, b"EE")
+    self.evaluate(enqueue)
+    self._ExpectRead(key, value, b"DD")
+    self._ExpectRead(key, value, b"EE")
+    self.evaluate(queue.close())
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      self.evaluate([key, value])
 
   def testSerializeRestore(self):
-    with self.cached_session() as sess:
-      reader = io_ops.IdentityReader("test_reader")
-      produced = reader.num_records_produced()
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      queue.enqueue_many([["X", "Y", "Z"]]).run()
-      key, value = reader.read(queue)
-
-      self._ExpectRead(sess, key, value, b"X")
-      self.assertAllEqual(1, self.evaluate(produced))
-      state = reader.serialize_state().eval()
-
-      self._ExpectRead(sess, key, value, b"Y")
-      self._ExpectRead(sess, key, value, b"Z")
-      self.assertAllEqual(3, self.evaluate(produced))
-
-      queue.enqueue_many([["Y", "Z"]]).run()
-      queue.close().run()
-      reader.restore_state(state).run()
-      self.assertAllEqual(1, self.evaluate(produced))
-      self._ExpectRead(sess, key, value, b"Y")
-      self._ExpectRead(sess, key, value, b"Z")
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        sess.run([key, value])
-      self.assertAllEqual(3, self.evaluate(produced))
-
-      self.assertEqual(bytes, type(state))
-
-      with self.assertRaises(ValueError):
-        reader.restore_state([])
-
-      with self.assertRaises(ValueError):
-        reader.restore_state([state, state])
-
-      with self.assertRaisesOpError(
-          "Could not parse state for IdentityReader 'test_reader'"):
-        reader.restore_state(state[1:]).run()
-
-      with self.assertRaisesOpError(
-          "Could not parse state for IdentityReader 'test_reader'"):
-        reader.restore_state(state[:-1]).run()
-
-      with self.assertRaisesOpError(
-          "Could not parse state for IdentityReader 'test_reader'"):
-        reader.restore_state(state + b"ExtraJunk").run()
-
-      with self.assertRaisesOpError(
-          "Could not parse state for IdentityReader 'test_reader'"):
-        reader.restore_state(b"PREFIX" + state).run()
-
-      with self.assertRaisesOpError(
-          "Could not parse state for IdentityReader 'test_reader'"):
-        reader.restore_state(b"BOGUS" + state[5:]).run()
+    reader = io_ops.IdentityReader("test_reader")
+    produced = reader.num_records_produced()
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    self.evaluate(queue.enqueue_many([["X", "Y", "Z"]]))
+    key, value = reader.read(queue)
+
+    self._ExpectRead(key, value, b"X")
+    self.assertAllEqual(1, self.evaluate(produced))
+    state = self.evaluate(reader.serialize_state())
+
+    self._ExpectRead(key, value, b"Y")
+    self._ExpectRead(key, value, b"Z")
+    self.assertAllEqual(3, self.evaluate(produced))
+
+    self.evaluate(queue.enqueue_many([["Y", "Z"]]))
+    self.evaluate(queue.close())
+    self.evaluate(reader.restore_state(state))
+    self.assertAllEqual(1, self.evaluate(produced))
+    self._ExpectRead(key, value, b"Y")
+    self._ExpectRead(key, value, b"Z")
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      self.evaluate([key, value])
+    self.assertAllEqual(3, self.evaluate(produced))
+
+    self.assertEqual(bytes, type(state))
+
+    with self.assertRaises(ValueError):
+      reader.restore_state([])
+
+    with self.assertRaises(ValueError):
+      reader.restore_state([state, state])
+
+    with self.assertRaisesOpError(
+        "Could not parse state for IdentityReader 'test_reader'"):
+      self.evaluate(reader.restore_state(state[1:]))
+
+    with self.assertRaisesOpError(
+        "Could not parse state for IdentityReader 'test_reader'"):
+      self.evaluate(reader.restore_state(state[:-1]))
+
+    with self.assertRaisesOpError(
+        "Could not parse state for IdentityReader 'test_reader'"):
+      self.evaluate(reader.restore_state(state + b"ExtraJunk"))
+
+    with self.assertRaisesOpError(
+        "Could not parse state for IdentityReader 'test_reader'"):
+      self.evaluate(reader.restore_state(b"PREFIX" + state))
+
+    with self.assertRaisesOpError(
+        "Could not parse state for IdentityReader 'test_reader'"):
+      self.evaluate(reader.restore_state(b"BOGUS" + state[5:]))
 
   def testReset(self):
-    with self.cached_session() as sess:
-      reader = io_ops.IdentityReader("test_reader")
-      work_completed = reader.num_work_units_completed()
-      produced = reader.num_records_produced()
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      queued_length = queue.size()
-      key, value = reader.read(queue)
+    reader = io_ops.IdentityReader("test_reader")
+    work_completed = reader.num_work_units_completed()
+    produced = reader.num_records_produced()
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    queued_length = queue.size()
+    key, value = reader.read(queue)
 
-      queue.enqueue_many([["X", "Y", "Z"]]).run()
-      self._ExpectRead(sess, key, value, b"X")
-      self.assertLess(0, self.evaluate(queued_length))
-      self.assertAllEqual(1, self.evaluate(produced))
+    self.evaluate(queue.enqueue_many([["X", "Y", "Z"]]))
+    self._ExpectRead(key, value, b"X")
+    self.assertLess(0, self.evaluate(queued_length))
+    self.assertAllEqual(1, self.evaluate(produced))
 
-      self._ExpectRead(sess, key, value, b"Y")
-      self.assertLess(0, self.evaluate(work_completed))
-      self.assertAllEqual(2, self.evaluate(produced))
+    self._ExpectRead(key, value, b"Y")
+    self.assertLess(0, self.evaluate(work_completed))
+    self.assertAllEqual(2, self.evaluate(produced))
 
-      reader.reset().run()
-      self.assertAllEqual(0, self.evaluate(work_completed))
-      self.assertAllEqual(0, self.evaluate(produced))
-      self.assertAllEqual(1, self.evaluate(queued_length))
-      self._ExpectRead(sess, key, value, b"Z")
+    self.evaluate(reader.reset())
+    self.assertAllEqual(0, self.evaluate(work_completed))
+    self.assertAllEqual(0, self.evaluate(produced))
+    self.assertAllEqual(1, self.evaluate(queued_length))
+    self._ExpectRead(key, value, b"Z")
 
-      queue.enqueue_many([["K", "L"]]).run()
-      self._ExpectRead(sess, key, value, b"K")
+    self.evaluate(queue.enqueue_many([["K", "L"]]))
+    self._ExpectRead(key, value, b"K")
 
 
 class WholeFileReaderTest(test.TestCase):
@@ -301,44 +297,42 @@ class WholeFileReaderTest(test.TestCase):
       os.remove(fn)
     super(WholeFileReaderTest, self).tearDown()
 
-  def _ExpectRead(self, sess, key, value, index):
-    k, v = sess.run([key, value])
+  def _ExpectRead(self, key, value, index):
+    k, v = self.evaluate([key, value])
     self.assertAllEqual(compat.as_bytes(self._filenames[index]), k)
     self.assertAllEqual(self._content[index], v)
 
   def testOneEpoch(self):
-    with self.cached_session() as sess:
-      reader = io_ops.WholeFileReader("test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      queue.enqueue_many([self._filenames]).run()
-      queue.close().run()
-      key, value = reader.read(queue)
+    reader = io_ops.WholeFileReader("test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    self.evaluate(queue.enqueue_many([self._filenames]))
+    self.evaluate(queue.close())
+    key, value = reader.read(queue)
 
-      self._ExpectRead(sess, key, value, 0)
-      self._ExpectRead(sess, key, value, 1)
-      self._ExpectRead(sess, key, value, 2)
+    self._ExpectRead(key, value, 0)
+    self._ExpectRead(key, value, 1)
+    self._ExpectRead(key, value, 2)
 
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        sess.run([key, value])
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      self.evaluate([key, value])
 
   def testInfiniteEpochs(self):
-    with self.cached_session() as sess:
-      reader = io_ops.WholeFileReader("test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      enqueue = queue.enqueue_many([self._filenames])
-      key, value = reader.read(queue)
-
-      enqueue.run()
-      self._ExpectRead(sess, key, value, 0)
-      self._ExpectRead(sess, key, value, 1)
-      enqueue.run()
-      self._ExpectRead(sess, key, value, 2)
-      self._ExpectRead(sess, key, value, 0)
-      self._ExpectRead(sess, key, value, 1)
-      enqueue.run()
-      self._ExpectRead(sess, key, value, 2)
-      self._ExpectRead(sess, key, value, 0)
+    reader = io_ops.WholeFileReader("test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    enqueue = queue.enqueue_many([self._filenames])
+    key, value = reader.read(queue)
+
+    self.evaluate(enqueue)
+    self._ExpectRead(key, value, 0)
+    self._ExpectRead(key, value, 1)
+    self.evaluate(enqueue)
+    self._ExpectRead(key, value, 2)
+    self._ExpectRead(key, value, 0)
+    self._ExpectRead(key, value, 1)
+    self.evaluate(enqueue)
+    self._ExpectRead(key, value, 2)
+    self._ExpectRead(key, value, 0)
 
 
 class TextLineReaderTest(test.TestCase):
@@ -366,22 +360,21 @@ class TextLineReaderTest(test.TestCase):
     return filenames
 
   def _testOneEpoch(self, files):
-    with self.cached_session() as sess:
-      reader = io_ops.TextLineReader(name="test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
+    reader = io_ops.TextLineReader(name="test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
 
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(self._num_lines):
-          k, v = sess.run([key, value])
-          self.assertAllEqual("%s:%d" % (files[i], j + 1), compat.as_text(k))
-          self.assertAllEqual(self._LineText(i, j), v)
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(self._num_lines):
+        k, v = self.evaluate([key, value])
+        self.assertAllEqual("%s:%d" % (files[i], j + 1), compat.as_text(k))
+        self.assertAllEqual(self._LineText(i, j), v)
 
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
   def testOneEpochLF(self):
     self._testOneEpoch(self._CreateFiles(crlf=False))
@@ -391,22 +384,21 @@ class TextLineReaderTest(test.TestCase):
 
   def testSkipHeaderLines(self):
     files = self._CreateFiles()
-    with self.cached_session() as sess:
-      reader = io_ops.TextLineReader(skip_header_lines=1, name="test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
+    reader = io_ops.TextLineReader(skip_header_lines=1, name="test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
 
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(self._num_lines - 1):
-          k, v = sess.run([key, value])
-          self.assertAllEqual("%s:%d" % (files[i], j + 2), compat.as_text(k))
-          self.assertAllEqual(self._LineText(i, j + 1), v)
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(self._num_lines - 1):
+        k, v = self.evaluate([key, value])
+        self.assertAllEqual("%s:%d" % (files[i], j + 2), compat.as_text(k))
+        self.assertAllEqual(self._LineText(i, j + 1), v)
 
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
 
 class FixedLengthRecordReaderTest(TFCompressionTestCase):
@@ -522,55 +514,53 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase):
   # gap_bytes=hop_bytes-record_bytes
   def _TestOneEpoch(self, files, num_records, gap_bytes, encoding=None):
     hop_bytes = 0 if gap_bytes == 0 else self._record_bytes + gap_bytes
-    with self.cached_session() as sess:
-      reader = io_ops.FixedLengthRecordReader(
-          header_bytes=self._header_bytes,
-          record_bytes=self._record_bytes,
-          footer_bytes=self._footer_bytes,
-          hop_bytes=hop_bytes,
-          encoding=encoding,
-          name="test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
-
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(num_records):
-          k, v = sess.run([key, value])
-          self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k))
-          self.assertAllEqual(self._Record(i, j), v)
-
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    reader = io_ops.FixedLengthRecordReader(
+        header_bytes=self._header_bytes,
+        record_bytes=self._record_bytes,
+        footer_bytes=self._footer_bytes,
+        hop_bytes=hop_bytes,
+        encoding=encoding,
+        name="test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
+
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(num_records):
+        k, v = self.evaluate([key, value])
+        self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k))
+        self.assertAllEqual(self._Record(i, j), v)
+
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
   def _TestOneEpochWithHopBytes(self,
                                 files,
                                 num_overlapped_records,
                                 encoding=None):
-    with self.cached_session() as sess:
-      reader = io_ops.FixedLengthRecordReader(
-          header_bytes=self._header_bytes,
-          record_bytes=self._record_bytes,
-          footer_bytes=self._footer_bytes,
-          hop_bytes=self._hop_bytes,
-          encoding=encoding,
-          name="test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
-
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(num_overlapped_records):
-          k, v = sess.run([key, value])
-          self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k))
-          self.assertAllEqual(self._OverlappedRecord(i, j), v)
-
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    reader = io_ops.FixedLengthRecordReader(
+        header_bytes=self._header_bytes,
+        record_bytes=self._record_bytes,
+        footer_bytes=self._footer_bytes,
+        hop_bytes=self._hop_bytes,
+        encoding=encoding,
+        name="test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
+
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(num_overlapped_records):
+        k, v = self.evaluate([key, value])
+        self.assertAllEqual("%s:%d" % (files[i], j), compat.as_text(k))
+        self.assertAllEqual(self._OverlappedRecord(i, j), v)
+
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
   def testOneEpoch(self):
     for num_records in [0, 7]:
@@ -621,84 +611,80 @@ class TFRecordReaderTest(TFCompressionTestCase):
 
   def testOneEpoch(self):
     files = self._CreateFiles()
-    with self.cached_session() as sess:
-      reader = io_ops.TFRecordReader(name="test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
-
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(self._num_records):
-          k, v = sess.run([key, value])
-          self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
-          self.assertAllEqual(self._Record(i, j), v)
-
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    reader = io_ops.TFRecordReader(name="test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
+
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(self._num_records):
+        k, v = self.evaluate([key, value])
+        self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
+        self.assertAllEqual(self._Record(i, j), v)
+
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
   def testReadUpTo(self):
     files = self._CreateFiles()
-    with self.cached_session() as sess:
-      reader = io_ops.TFRecordReader(name="test_reader")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      batch_size = 3
-      key, value = reader.read_up_to(queue, batch_size)
-
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      num_k = 0
-      num_v = 0
-
-      while True:
-        try:
-          k, v = sess.run([key, value])
-          # Test reading *up to* batch_size records
-          self.assertLessEqual(len(k), batch_size)
-          self.assertLessEqual(len(v), batch_size)
-          num_k += len(k)
-          num_v += len(v)
-        except errors_impl.OutOfRangeError:
-          break
-
-      # Test that we have read everything
-      self.assertEqual(self._num_files * self._num_records, num_k)
-      self.assertEqual(self._num_files * self._num_records, num_v)
+    reader = io_ops.TFRecordReader(name="test_reader")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    batch_size = 3
+    key, value = reader.read_up_to(queue, batch_size)
+
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    num_k = 0
+    num_v = 0
+
+    while True:
+      try:
+        k, v = self.evaluate([key, value])
+        # Test reading *up to* batch_size records
+        self.assertLessEqual(len(k), batch_size)
+        self.assertLessEqual(len(v), batch_size)
+        num_k += len(k)
+        num_v += len(v)
+      except errors_impl.OutOfRangeError:
+        break
+
+    # Test that we have read everything
+    self.assertEqual(self._num_files * self._num_records, num_k)
+    self.assertEqual(self._num_files * self._num_records, num_v)
 
   def testReadZlibFiles(self):
     options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB)
     files = self._CreateFiles(options)
 
-    with self.cached_session() as sess:
-      reader = io_ops.TFRecordReader(name="test_reader", options=options)
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
-
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(self._num_records):
-          k, v = sess.run([key, value])
-          self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
-          self.assertAllEqual(self._Record(i, j), v)
+    reader = io_ops.TFRecordReader(name="test_reader", options=options)
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
+
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(self._num_records):
+        k, v = self.evaluate([key, value])
+        self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
+        self.assertAllEqual(self._Record(i, j), v)
 
   def testReadGzipFiles(self):
     options = tf_record.TFRecordOptions(TFRecordCompressionType.GZIP)
     files = self._CreateFiles(options)
 
-    with self.cached_session() as sess:
-      reader = io_ops.TFRecordReader(name="test_reader", options=options)
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
+    reader = io_ops.TFRecordReader(name="test_reader", options=options)
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
 
-      queue.enqueue_many([files]).run()
-      queue.close().run()
-      for i in range(self._num_files):
-        for j in range(self._num_records):
-          k, v = sess.run([key, value])
-          self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
-          self.assertAllEqual(self._Record(i, j), v)
+    self.evaluate(queue.enqueue_many([files]))
+    self.evaluate(queue.close())
+    for i in range(self._num_files):
+      for j in range(self._num_records):
+        k, v = self.evaluate([key, value])
+        self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
+        self.assertAllEqual(self._Record(i, j), v)
 
 
 class AsyncReaderTest(test.TestCase):
@@ -724,7 +710,7 @@ class AsyncReaderTest(test.TestCase):
         thread_data.append(thread_data_t(t, queue, output))
 
       # Start all readers. They are all blocked waiting for queue entries.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for d in thread_data:
         d.thread.start()
 
@@ -733,7 +719,7 @@ class AsyncReaderTest(test.TestCase):
         fname = os.path.join(self.get_temp_dir(), "deadlock.%s.txt" % i)
         with open(fname, "wb") as f:
           f.write(("file-%s" % i).encode())
-        d.queue.enqueue_many([[fname]]).run()
+        self.evaluate(d.queue.enqueue_many([[fname]]))
         d.thread.join()
         self.assertEqual([[("file-%s" % i).encode()]], d.output)
 
@@ -752,22 +738,21 @@ class LMDBReaderTest(test.TestCase):
     shutil.copy(path, self.db_path)
 
   def testReadFromFile(self):
-    with self.cached_session() as sess:
-      reader = io_ops.LMDBReader(name="test_read_from_file")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
-
-      queue.enqueue([self.db_path]).run()
-      queue.close().run()
-      for i in range(10):
-        k, v = sess.run([key, value])
-        self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i)))
-        self.assertAllEqual(
-            compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i))))
-
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    reader = io_ops.LMDBReader(name="test_read_from_file")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
+
+    self.evaluate(queue.enqueue([self.db_path]))
+    self.evaluate(queue.close())
+    for i in range(10):
+      k, v = self.evaluate([key, value])
+      self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i)))
+      self.assertAllEqual(
+          compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i))))
+
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
   def testReadFromSameFile(self):
     with self.cached_session() as sess:
@@ -782,29 +767,28 @@ class LMDBReaderTest(test.TestCase):
       threads = queue_runner_impl.start_queue_runners(sess, coord=coord)
       for _ in range(3):
         for _ in range(10):
-          k1, v1, k2, v2 = sess.run([key1, value1, key2, value2])
+          k1, v1, k2, v2 = self.evaluate([key1, value1, key2, value2])
           self.assertAllEqual(compat.as_bytes(k1), compat.as_bytes(k2))
           self.assertAllEqual(compat.as_bytes(v1), compat.as_bytes(v2))
       coord.request_stop()
       coord.join(threads)
 
   def testReadFromFolder(self):
-    with self.cached_session() as sess:
-      reader = io_ops.LMDBReader(name="test_read_from_folder")
-      queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
-      key, value = reader.read(queue)
-
-      queue.enqueue([self.db_path]).run()
-      queue.close().run()
-      for i in range(10):
-        k, v = sess.run([key, value])
-        self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i)))
-        self.assertAllEqual(
-            compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i))))
-
-      with self.assertRaisesOpError("is closed and has insufficient elements "
-                                    "\\(requested 1, current size 0\\)"):
-        k, v = sess.run([key, value])
+    reader = io_ops.LMDBReader(name="test_read_from_folder")
+    queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
+    key, value = reader.read(queue)
+
+    self.evaluate(queue.enqueue([self.db_path]))
+    self.evaluate(queue.close())
+    for i in range(10):
+      k, v = self.evaluate([key, value])
+      self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i)))
+      self.assertAllEqual(
+          compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i))))
+
+    with self.assertRaisesOpError("is closed and has insufficient elements "
+                                  "\\(requested 1, current size 0\\)"):
+      k, v = self.evaluate([key, value])
 
   def testReadFromFileRepeatedly(self):
     with self.cached_session() as sess:
@@ -819,7 +803,7 @@ class LMDBReaderTest(test.TestCase):
       for _ in range(3):
         # Go over all 10 records each time.
         for j in range(10):
-          k, v = sess.run([key, value])
+          k, v = self.evaluate([key, value])
           self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(j)))
           self.assertAllEqual(
               compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + j))))
diff --git a/tensorflow/python/kernel_tests/record_input_test.py b/tensorflow/python/kernel_tests/record_input_test.py
index ebb9872f22..74020667d9 100644
--- a/tensorflow/python/kernel_tests/record_input_test.py
+++ b/tensorflow/python/kernel_tests/record_input_test.py
@@ -54,7 +54,7 @@ class RecordInputOpTest(test.TestCase):
           batch_size=1,
           name="record_input").get_yield_op()
 
-      self.assertEqual(sess.run(yield_op), b"0000000000")
+      self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
   def testRecordInputSimpleGzip(self):
     with self.cached_session() as sess:
@@ -73,7 +73,7 @@ class RecordInputOpTest(test.TestCase):
           compression_type=tf_record.TFRecordCompressionType.GZIP).get_yield_op(
           )
 
-      self.assertEqual(sess.run(yield_op), b"0000000000")
+      self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
   def testRecordInputSimpleZlib(self):
     with self.cached_session() as sess:
@@ -92,7 +92,7 @@ class RecordInputOpTest(test.TestCase):
           compression_type=tf_record.TFRecordCompressionType.ZLIB).get_yield_op(
           )
 
-      self.assertEqual(sess.run(yield_op), b"0000000000")
+      self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
   def testRecordInputEpochs(self):
     files = 100
@@ -117,7 +117,7 @@ class RecordInputOpTest(test.TestCase):
       for _ in range(3):
         epoch_set = set()
         for _ in range(int(files * records_per_file / batches)):
-          op_list = sess.run(yield_op)
+          op_list = self.evaluate(yield_op)
           self.assertTrue(len(op_list) is batches)
           for r in op_list:
             self.assertTrue(r[0] not in epoch_set)
@@ -138,15 +138,15 @@ class RecordInputOpTest(test.TestCase):
 
         yield_op = records.get_yield_op()
         for _ in range(50):
-          sess.run(yield_op)
+          self.evaluate(yield_op)
 
   def testEmptyGlob(self):
     with self.cached_session() as sess:
       record_input = data_flow_ops.RecordInput(file_pattern="foo")
       yield_op = record_input.get_yield_op()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       with self.assertRaises(NotFoundError):
-        sess.run(yield_op)
+        self.evaluate(yield_op)
 
   def testBufferTooSmall(self):
     files = 10
@@ -171,7 +171,7 @@ class RecordInputOpTest(test.TestCase):
       for _ in range(3):
         epoch_set = set()
         for _ in range(int(files * records_per_file / batches)):
-          op_list = sess.run(yield_op)
+          op_list = self.evaluate(yield_op)
           self.assertTrue(len(op_list) is batches)
           for r in op_list:
             self.assertTrue(r[0] not in epoch_set)
diff --git a/tensorflow/python/kernel_tests/reduce_benchmark_test.py b/tensorflow/python/kernel_tests/reduce_benchmark_test.py
index 3a2fb81157..ef9c4c350f 100644
--- a/tensorflow/python/kernel_tests/reduce_benchmark_test.py
+++ b/tensorflow/python/kernel_tests/reduce_benchmark_test.py
@@ -81,7 +81,7 @@ class ReduceBenchmarks(test.Benchmark):
       grad, = gradients_impl.gradients(reduction, tensor)
 
       def fn():
-        sess.run(grad.op)
+        self.evaluate(grad.op)
 
       self._run(fn, 10000)
 
@@ -98,7 +98,7 @@ class ReduceBenchmarks(test.Benchmark):
         grad, = gradients_impl.gradients(reduction, tensor)
 
       def fn():
-        sess.run(grad.op)
+        self.evaluate(grad.op)
 
       self._run(fn, 10000)
 
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index d1a295f42b..4eb329796e 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -185,7 +185,7 @@ class SumReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -216,7 +216,7 @@ class SumReductionTest(BaseReductionTest):
       tf_arr = variables.Variable(arr)
       variables.global_variables_initializer().run()
       tf_mean = math_ops.reduce_mean(tf_arr, 0, False)
-      tf_out_mean = sess.run(tf_mean)
+      tf_out_mean = self.evaluate(tf_mean)
     self.assertAllClose(tf_out_mean, 1.)
 
   def testFloat32(self):
@@ -238,7 +238,7 @@ class SumReductionTest(BaseReductionTest):
       with self.session(graph=ops.Graph(), use_gpu=True) as sess:
         tf_row_sum = self._tf_reduce(arr, 1, False)
         tf_col_sum = self._tf_reduce(arr, 0, False)
-        tf_out_row, tf_out_col = sess.run([tf_row_sum, tf_col_sum])
+        tf_out_row, tf_out_col = self.evaluate([tf_row_sum, tf_col_sum])
       self.assertAllClose(col_sum, tf_out_col)
       self.assertAllClose(row_sum, tf_out_row)
 
@@ -252,7 +252,7 @@ class SumReductionTest(BaseReductionTest):
           with self.session(graph=ops.Graph(), use_gpu=True) as sess:
             tf_sum_xz = self._tf_reduce(arr, [0, 2], False)
             tf_sum_y = self._tf_reduce(arr, 1, False)
-            tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y])
+            tf_out_sum_xz, tf_out_sum_y = self.evaluate([tf_sum_xz, tf_sum_y])
           self.assertAllClose(sum_y, tf_out_sum_y)
           self.assertAllClose(sum_xz, tf_out_sum_xz)
 
@@ -400,7 +400,7 @@ class MeanReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -473,7 +473,7 @@ class ProdReductionTest(BaseReductionTest):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -576,7 +576,7 @@ class MinReductionTest(test.TestCase):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -689,7 +689,7 @@ class MaxReductionTest(test.TestCase):
     for dtype in [dtypes.int64, dtypes.int32]:
       with self.cached_session(use_gpu=True) as sess:
         v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
   def testInfinity(self):
@@ -817,7 +817,7 @@ class AllReductionTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         v = math_ops.reduce_all([True, True],
                                 constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, True)
 
   def testAll3D(self):
@@ -866,7 +866,7 @@ class AnyReductionTest(test.TestCase):
       with self.session(use_gpu=True) as sess:
         v = math_ops.reduce_any([True, True],
                                 constant_op.constant(0, dtype=dtype))
-        tf_v = sess.run(v)
+        tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, True)
 
   def testAll3D(self):
@@ -962,7 +962,7 @@ class CountNonzeroReductionTest(test.TestCase):
     # Test case for GitHub issue 18712
     with self.cached_session() as sess:
       v = math_ops.count_nonzero(constant_op.constant(["test"]))
-      self.assertAllClose(sess.run(v), 1)
+      self.assertAllClose(self.evaluate(v), 1)
 
   def testStringReduce1D(self):
     # Create a 1D array of strings
diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index 68243f27c0..30cef90885 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -147,7 +147,7 @@ class ReluTest(test.TestCase):
       # Repeat the experiment for 100 times. All tensor shapes and its tensor
       # values are randomly generated for each run.
       for _ in xrange(100):
-        dx_f32_v, dx_f16_v = sess.run([dx_f32, dx_f16])
+        dx_f32_v, dx_f16_v = self.evaluate([dx_f32, dx_f16])
         self.assertAllClose(dx_f32_v, dx_f16_v, atol=3e-4)
 
   def testGradientFloat64(self):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index e85b04469b..13b39926ec 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -153,7 +153,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
   def testCachedValueReadBeforeWrite(self):
     with self.cached_session() as sess:
       v = resource_variable_ops.ResourceVariable(0.0, caching_device="cpu:0")
-      sess.run(v.initializer)
+      self.evaluate(v.initializer)
       value, _ = sess.run([v, v.assign_add(1.0)])
       self.assertAllEqual(value, 0.0)
 
@@ -590,11 +590,11 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     with ops.Graph().as_default(), self.cached_session() as sess:
       # v describes a VariableDef-based variable without an initial value.
       v = resource_variable_ops.ResourceVariable(variable_def=v_def)
-      self.assertEqual(3.0, sess.run(v.initialized_value()))
+      self.assertEqual(3.0, self.evaluate(v.initialized_value()))
 
       # initialized_value should not rerun the initializer_op if the variable
       # has already been initialized elsewhere.
-      sess.run(v.assign(1.0))
+      self.evaluate(v.assign(1.0))
       self.assertEqual(1.0, v.initialized_value().eval())
 
     v_def.ClearField("initial_value_name")
@@ -606,7 +606,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertProtoEquals(v_def, v.to_proto())
       # But attempts to use initialized_value will result in errors.
       with self.assertRaises(ValueError):
-        sess.run(v.initialized_value())
+        self.evaluate(v.initialized_value())
 
   def testTrainableInProto(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index 952ef34456..c388121982 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -161,8 +161,8 @@ class StatefulScatterNdTest(test.TestCase):
     init = variables.global_variables_initializer()
 
     with self.session(use_gpu=True) as sess:
-      sess.run(init)
-      result = sess.run(scatter)
+      self.evaluate(init)
+      result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
   def testSimpleResource(self):
@@ -175,8 +175,8 @@ class StatefulScatterNdTest(test.TestCase):
     init = variables.global_variables_initializer()
 
     with self.session(use_gpu=True) as sess:
-      sess.run(init)
-      sess.run(scatter)
+      self.evaluate(init)
+      self.evaluate(scatter)
       self.assertAllClose(ref.eval(), expected)
 
   def testSimple2(self):
@@ -189,8 +189,8 @@ class StatefulScatterNdTest(test.TestCase):
     init = variables.global_variables_initializer()
 
     with self.session(use_gpu=True) as sess:
-      sess.run(init)
-      result = sess.run(scatter)
+      self.evaluate(init)
+      result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
   def testSimple3(self):
@@ -203,8 +203,8 @@ class StatefulScatterNdTest(test.TestCase):
     init = variables.global_variables_initializer()
 
     with self.session(use_gpu=True) as sess:
-      sess.run(init)
-      result = sess.run(scatter)
+      self.evaluate(init)
+      result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
   def testVariableRankUpdate(self):
@@ -341,8 +341,8 @@ class StatefulScatterNdTest(test.TestCase):
     init = variables.global_variables_initializer()
 
     with session.Session() as sess:
-      sess.run(init)
-      result = sess.run(scatter)
+      self.evaluate(init)
+      result = self.evaluate(scatter)
       assert np.allclose(result, expected_result)
 
   # TODO(fpmc): Re-enable this test when gpu_pip test actually runs on a GPU.
@@ -421,7 +421,7 @@ class ScatterNdTest(test.TestCase):
                          b"", b"", b"seven"])
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllEqual(expected, result)
 
     # Same indice is updated twice by same value.
@@ -432,7 +432,7 @@ class ScatterNdTest(test.TestCase):
     expected = np.array([b"", b"", b"", b"bb", b"a", b"", b"", b"c"])
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertAllEqual(expected, result)
 
     # Same indice is updated twice by different value.
@@ -444,7 +444,7 @@ class ScatterNdTest(test.TestCase):
                 np.array([b"", b"", b"", b"cb", b"a", b"", b"", b"d"])]
     scatter = self.scatter_nd(indices, updates, shape=(8,))
     with self.cached_session() as sess:
-      result = sess.run(scatter)
+      result = self.evaluate(scatter)
       self.assertTrue(np.array_equal(result, expected[0]) or
                       np.array_equal(result, expected[1]))
 
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 85756b769d..42577f7e42 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -63,7 +63,7 @@ class SelfAdjointEigTest(test.TestCase):
           e1 = linalg_ops.self_adjoint_eigvals(matrix1)
           e2 = linalg_ops.self_adjoint_eigvals(matrix2)
           all_ops += [e1, e2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       self.assertAllEqual(val[0], val[2])
       # The algorithm is slightly different for compute_v being True and False,
       # so require approximate equality only here.
@@ -81,7 +81,7 @@ class SelfAdjointEigTest(test.TestCase):
     self.assertEqual(matrix.shape, (32, 32))
     matrix_tensor = constant_op.constant(matrix)
     with self.session(use_gpu=True) as sess:
-      (e, v) = sess.run(linalg_ops.self_adjoint_eig(matrix_tensor))
+      (e, v) = self.evaluate(linalg_ops.self_adjoint_eig(matrix_tensor))
       self.assertEqual(e.size, 32)
       self.assertAllClose(
           np.matmul(v, v.transpose()), np.eye(32, dtype=np.float32), atol=2e-3)
diff --git a/tensorflow/python/kernel_tests/session_ops_test.py b/tensorflow/python/kernel_tests/session_ops_test.py
index 03e1ae852f..dc663cb091 100644
--- a/tensorflow/python/kernel_tests/session_ops_test.py
+++ b/tensorflow/python/kernel_tests/session_ops_test.py
@@ -37,7 +37,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Feed a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -51,7 +51,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Get the tensor from its handle.
       self.assertEqual(50, h.eval())
@@ -64,7 +64,7 @@ class SessionOpsTest(test.TestCase):
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
       v = math_ops.multiply(a, c)
-      h, v = sess.run([h, v])
+      h, v = self.evaluate([h, v])
 
       self.assertEqual(50, h.eval())
       self.assertEqual(500, v)
@@ -77,7 +77,7 @@ class SessionOpsTest(test.TestCase):
       p = math_ops.less(a, b)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      p, h = sess.run([p, h])
+      p, h = self.evaluate([p, h])
 
       # Run by feeding a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -94,7 +94,7 @@ class SessionOpsTest(test.TestCase):
       # Initialize a handle.
       a = constant_op.constant(0)
       h = session_ops.get_session_handle(a)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Do some computation.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -111,7 +111,7 @@ class SessionOpsTest(test.TestCase):
       # Initialize a handle.
       a = constant_op.constant(0)
       h = session_ops.get_session_handle(a)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Do some computation.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -133,7 +133,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Feed a tensor handle.
       f, x = session_ops.get_session_tensor(h.handle, dtypes.int32)
@@ -144,7 +144,7 @@ class SessionOpsTest(test.TestCase):
       with ops.device(test.gpu_device_name()):
         a = constant_op.constant(10)
         h = session_ops.get_session_handle(a)
-        h = sess.run(h)
+        h = self.evaluate(h)
         self.assertEqual(100, sess.run(y, feed_dict={f: h.handle}))
 
   def testHandleDelete(self):
@@ -154,7 +154,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      sess.run(h).delete()
+      self.evaluate(h).delete()
 
   def testHandleDeleteRaw(self):
     with self.cached_session() as sess:
@@ -163,7 +163,7 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(5)
       c = math_ops.multiply(a, b)
       h = session_ops.get_session_handle(c)
-      h = sess.run(h)
+      h = self.evaluate(h)
 
       # Delete using a raw tensor handle.
       raw_h = h.get_raw_handle()
@@ -174,10 +174,10 @@ class SessionOpsTest(test.TestCase):
     with self.cached_session() as sess:
       with ops.device(test.gpu_device_name()):
         a = constant_op.constant(1.0)
-        a_handle = sess.run(session_ops.get_session_handle(a))
+        a_handle = self.evaluate(session_ops.get_session_handle(a))
       with ops.device("/cpu:0"):
         b = constant_op.constant(2.0)
-        b_handle = sess.run(session_ops.get_session_handle(b))
+        b_handle = self.evaluate(session_ops.get_session_handle(b))
 
       a_p, a_t = session_ops.get_session_tensor(a_handle.handle, dtypes.float32)
       b_p, b_t = session_ops.get_session_tensor(b_handle.handle, dtypes.float32)
@@ -193,8 +193,8 @@ class SessionOpsTest(test.TestCase):
       # initial values live on CPU
       with ops.device("/cpu:0"):
         one = constant_op.constant(1, dtype=dtypes.float32)
-        one_handle = sess.run(session_ops.get_session_handle(one))
-        x_handle = sess.run(session_ops.get_session_handle(one))
+        one_handle = self.evaluate(session_ops.get_session_handle(one))
+        x_handle = self.evaluate(session_ops.get_session_handle(one))
 
       # addition lives on GPU
       with ops.device(test.gpu_device_name()):
@@ -219,8 +219,8 @@ class SessionOpsTest(test.TestCase):
       b = constant_op.constant(2.0)
       b_handle_op = session_ops.get_session_handle(b)
 
-      a_handle = sess.run(a_handle_op)
-      b_handle = sess.run(b_handle_op)
+      a_handle = self.evaluate(a_handle_op)
+      b_handle = self.evaluate(b_handle_op)
 
       a_p, a_t = session_ops.get_session_tensor(a_handle.handle, dtypes.float32)
       b_p, b_t = session_ops.get_session_tensor(b_handle.handle, dtypes.float32)
@@ -239,7 +239,7 @@ class SessionOpsTest(test.TestCase):
       c = math_ops.multiply(a, b)
       d = math_ops.multiply(c, c)
 
-      h_c = sess.run(session_ops.get_session_handle(c))
+      h_c = self.evaluate(session_ops.get_session_handle(c))
 
       self.assertAllClose(2500.0, sess.run(d, feed_dict={c: h_c}))
 
@@ -248,7 +248,7 @@ class SessionOpsTest(test.TestCase):
       a = constant_op.constant(10.0)
       b = constant_op.constant(5.0)
       c = math_ops.multiply(a, b)
-      h_c = sess.run(session_ops.get_session_handle(c))
+      h_c = self.evaluate(session_ops.get_session_handle(c))
       d = array_ops.identity(c)
 
       c_val = sess.run(c, feed_dict={c: h_c})
@@ -277,8 +277,8 @@ class SessionOpsTest(test.TestCase):
       d = math_ops.div(a, b)
       e = math_ops.subtract(c, d)
 
-      h_c = sess.run(session_ops.get_session_handle(c))
-      h_d = sess.run(session_ops.get_session_handle(d))
+      h_c = self.evaluate(session_ops.get_session_handle(c))
+      h_d = self.evaluate(session_ops.get_session_handle(d))
 
       self.assertAllClose(48.0, sess.run(e, feed_dict={c: h_c, d: h_d}))
       self.assertAllClose(-48.0, sess.run(e, feed_dict={c: h_d, d: h_c}))
@@ -288,13 +288,13 @@ class SessionOpsTest(test.TestCase):
       a = variables.Variable(12.0)
       inc_a = state_ops.assign_add(a, 2.0)
       b = math_ops.add(a, 5.0)
-      sess.run(a.initializer)
+      self.evaluate(a.initializer)
 
       h_a_read = sess.run(session_ops.get_session_handle(a.read_value()))
-      self.assertAllClose(12.0, sess.run(a))
+      self.assertAllClose(12.0, self.evaluate(a))
 
       self.assertAllClose(17.0, sess.run(b, feed_dict={a: h_a_read}))
-      sess.run(inc_a)
+      self.evaluate(inc_a)
       self.assertAllClose(19.0, sess.run(b, feed_dict={a: h_a_read}))
 
 
diff --git a/tensorflow/python/kernel_tests/sets_test.py b/tensorflow/python/kernel_tests/sets_test.py
index 8335e9c139..ba3d32b192 100644
--- a/tensorflow/python/kernel_tests/sets_test.py
+++ b/tensorflow/python/kernel_tests/sets_test.py
@@ -159,7 +159,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(None, op.get_shape().dims)
       self.assertEqual(dtypes.int32, op.dtype)
     with self.cached_session() as sess:
-      results = sess.run(ops)
+      results = self.evaluate(ops)
     self.assertAllEqual(results[0], results[1])
     return results[0]
 
@@ -534,7 +534,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
   def _set_intersection_count(self, a, b):
     op = sets.set_size(sets.set_intersection(a, b))
     with self.cached_session() as sess:
-      return sess.run(op)
+      return self.evaluate(op)
 
   def test_set_difference_multirow_2d(self):
     for dtype in _DTYPES:
@@ -972,7 +972,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
   def _set_difference_count(self, a, b, aminusb=True):
     op = sets.set_size(sets.set_difference(a, b, aminusb))
     with self.cached_session() as sess:
-      return sess.run(op)
+      return self.evaluate(op)
 
   def test_set_union_multirow_2d(self):
     for dtype in _DTYPES:
@@ -1221,7 +1221,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
   def _set_union_count(self, a, b):
     op = sets.set_size(sets.set_union(a, b))
     with self.cached_session() as sess:
-      return sess.run(op)
+      return self.evaluate(op)
 
   def _assert_set_operation(self, expected_indices, expected_values,
                             expected_shape, sparse_tensor_value, dtype):
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index 3e0eae326b..a0506fbfc5 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -73,8 +73,8 @@ class ShapeOpsTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       tf_ans = array_ops.shape_n([x, x, x])
       tf_ans_64 = array_ops.shape_n([x, x, x], out_type=dtypes.int64)
-      result = sess.run(tf_ans)
-      result_64 = sess.run(tf_ans_64)
+      result = self.evaluate(tf_ans)
+      result_64 = self.evaluate(tf_ans_64)
     for i in range(3):
       self.assertAllEqual(np_ans, result[i])
       self.assertAllEqual(np_ans, result_64[i])
diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index c4e5b6f674..de3351e543 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -56,7 +56,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, 2)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
 
       expected_output = np.array([1, 1, 2, 2, 3, 2, 2, 1, 1])
 
@@ -99,7 +99,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
       string_output = [np.base_repr(x, self.bases[0]) for x in output]
 
       self.assertEqual(string_output, self.expected_string)
@@ -109,7 +109,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
 
       accumulator = True
       for i in range(self.batch_size):
@@ -125,7 +125,7 @@ class ReconstructionOpsTest(test.TestCase):
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction)
+      output = self.evaluate(reconstruction)
 
       string_output = [np.base_repr(int(x), self.bases[0]) for x in
                        np.squeeze(output)]
diff --git a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
index 7583c4d8fc..7b9748c7f2 100644
--- a/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/spectral_ops_test.py
@@ -235,7 +235,8 @@ class SpectralOpsTest(test.TestCase):
       inverse_window = inverse_window_fn(frame_length, dtype=dtypes.float32)
 
       with self.cached_session(use_gpu=True) as sess:
-        hann_window, inverse_window = sess.run([hann_window, inverse_window])
+        hann_window, inverse_window = self.evaluate(
+            [hann_window, inverse_window])
 
       # Expect unit gain at each phase of the window.
       product_window = hann_window * inverse_window
@@ -263,7 +264,8 @@ class SpectralOpsTest(test.TestCase):
       inverse_window = inverse_window_fn(frame_length, dtype=dtypes.float32)
 
       with self.cached_session(use_gpu=True) as sess:
-        hann_window, inverse_window = sess.run([hann_window, inverse_window])
+        hann_window, inverse_window = self.evaluate(
+            [hann_window, inverse_window])
 
       self.assertAllClose(hann_window, inverse_window * 1.5)
 
@@ -293,7 +295,7 @@ class SpectralOpsTest(test.TestCase):
       # the sum of the magnitude STFT.
       sinusoid = math_ops.sin(
           2 * np.pi * math_ops.linspace(0.0, 1.0, signal_length))
-      sinusoid_gradient = sess.run(self._compute_stft_gradient(sinusoid))
+      sinusoid_gradient = self.evaluate(self._compute_stft_gradient(sinusoid))
       self.assertFalse((sinusoid_gradient == 0.0).all())
 
   def test_gradients_numerical(self):
diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py
index 5bb34a632d..ee48c6eb0e 100644
--- a/tensorflow/python/kernel_tests/slice_op_test.py
+++ b/tensorflow/python/kernel_tests/slice_op_test.py
@@ -207,7 +207,7 @@ class SliceTest(test.TestCase):
           dtype=dtypes.float32)
       slice_t = array_ops.slice(a, [0, 0], [2, 2])
       slice2_t = a[:2, :2]
-      slice_val, slice2_val = sess.run([slice_t, slice2_t])
+      slice_val, slice2_val = self.evaluate([slice_t, slice2_t])
     self.assertAllEqual(slice_val, inp[:2, :2])
     self.assertAllEqual(slice2_val, inp[:2, :2])
     self.assertEqual(slice_val.shape, slice_t.get_shape())
@@ -247,7 +247,7 @@ class SliceTest(test.TestCase):
                    + sizes[3], indices[4]:indices[4] + sizes[4], indices[5]:
                    indices[5] + sizes[5]]
 
-      slice_val, slice2_val = sess.run([slice_t, slice2_t])
+      slice_val, slice2_val = self.evaluate([slice_t, slice2_t])
 
     expected_val = inp[indices[0]:indices[0] + sizes[0], indices[1]:indices[
         1] + sizes[1], indices[2]:indices[2] + sizes[2], indices[3]:indices[
@@ -313,7 +313,7 @@ class SliceTest(test.TestCase):
       g1 = gradients_impl.gradients(loss1, x)[0]
       g2 = gradients_impl.gradients(loss2, x)[0]
 
-      g1_val, g2_val = sess.run([g1, g2])
+      g1_val, g2_val = self.evaluate([g1, g2])
     self.assertAllEqual(g1_val, g2_val)
 
   def testGradientsAll(self):
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index 8ac98a198c..c9aaa68971 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -273,7 +273,7 @@ class SpaceToDepthTest(test.TestCase):
       actual = array_ops.space_to_depth(t, block_size, data_format=data_format)
 
     with self.cached_session(use_gpu=use_gpu) as sess:
-      actual_vals, expected_vals = sess.run([actual, expected])
+      actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
   def testAgainstTranspose(self):
diff --git a/tensorflow/python/kernel_tests/sparse_add_op_test.py b/tensorflow/python/kernel_tests/sparse_add_op_test.py
index a746830afb..c61f863355 100644
--- a/tensorflow/python/kernel_tests/sparse_add_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_add_op_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
@@ -85,13 +86,13 @@ class SparseAddTest(test.TestCase):
         constant_op.constant(shape, dtypes.int64))
 
   def testAddSelf(self):
-    with self.session(use_gpu=False) as sess:
+    with test_util.force_cpu():
       for sp_a in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()):
         for sp_b in (self._SparseTensorValue_3x3(), self._SparseTensor_3x3()):
           sp_sum = sparse_ops.sparse_add(sp_a, sp_b)
           self.assertAllEqual((3, 3), sp_sum.get_shape())
 
-          sum_out = sess.run(sp_sum)
+          sum_out = self.evaluate(sp_sum)
 
           self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
           self.assertAllEqual(sum_out.indices, [[0, 1], [1, 0], [2, 0], [2, 1]])
@@ -99,12 +100,12 @@ class SparseAddTest(test.TestCase):
           self.assertAllEqual(sum_out.dense_shape, [3, 3])
 
   def testAddSelfAndNegation(self):
-    with self.session(use_gpu=False) as sess:
+    with test_util.force_cpu():
       sp_a = self._SparseTensor_3x3()
       sp_b = self._SparseTensor_3x3(negate=True)
 
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, 0.1)
-      sum_out = sess.run(sp_sum)
+      sum_out = self.evaluate(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, np.empty([0, 2]))
@@ -112,7 +113,7 @@ class SparseAddTest(test.TestCase):
       self.assertAllEqual(sum_out.dense_shape, [3, 3])
 
   def testSmallValuesShouldVanish(self):
-    with self.session(use_gpu=False) as sess:
+    with test_util.force_cpu():
       sp_a = self._SparseTensor_3x3()
       sp_b = self._SparseTensor_3x3_v2()
 
@@ -123,7 +124,7 @@ class SparseAddTest(test.TestCase):
 
       # two values should vanish: |.1| < .21, and |-.2| < .21
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.21)
-      sum_out = sess.run(sp_sum)
+      sum_out = self.evaluate(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0]])
@@ -132,7 +133,7 @@ class SparseAddTest(test.TestCase):
 
       # only .1 vanishes
       sp_sum = sparse_ops.sparse_add(sp_a, sp_b, thresh=0.11)
-      sum_out = sess.run(sp_sum)
+      sum_out = self.evaluate(sp_sum)
 
       self.assertEqual(sp_sum.dense_shape.get_shape(), [2])
       self.assertAllEqual(sum_out.indices, [[0, 1], [2, 0], [2, 1]])
@@ -147,7 +148,7 @@ class SparseAddTest(test.TestCase):
           sp_a, nnz_a = self._randomTensor([n, m], np.float32)
           sp_b, nnz_b = self._randomTensor([n, m], np.float32)
           sp_sum = sparse_ops.sparse_add(sp_a, sp_b)
-          nnz_sum = len(sp_sum.values.eval())
+          nnz_sum = len(self.evaluate(sp_sum.values))
 
           err = gradient_checker.compute_gradient_error(
               [sp_a.values, sp_b.values], [(nnz_a,), (nnz_b,)], sp_sum.values,
@@ -162,16 +163,16 @@ class SparseAddTest(test.TestCase):
         rand_vals_np = np.random.randn(n, m).astype(dtype)
         dense_np = np.random.randn(n, m).astype(dtype)
 
-        with self.cached_session(use_gpu=False):
+        with test_util.force_cpu():
           sparse, unused_nnz = _sparsify(rand_vals_np, index_dtype=index_dtype)
-          s = sparse_ops.sparse_add(sparse,
-                                    constant_op.constant(dense_np)).eval()
+          s = self.evaluate(
+              sparse_ops.sparse_add(sparse, constant_op.constant(dense_np)))
           self.assertAllEqual(dense_np + rand_vals_np, s)
           self.assertTrue(s.dtype == dtype)
 
           # check commutativity
-          s = sparse_ops.sparse_add(constant_op.constant(dense_np),
-                                    sparse).eval()
+          s = self.evaluate(
+              sparse_ops.sparse_add(constant_op.constant(dense_np), sparse))
           self.assertAllEqual(dense_np + rand_vals_np, s)
           self.assertTrue(s.dtype == dtype)
 
@@ -191,7 +192,7 @@ class SparseAddTest(test.TestCase):
       self.assertLess(err, 1e-3)
 
   def testInvalidSparseTensor(self):
-    with self.session(use_gpu=False) as sess:
+    with test_util.force_cpu():
       shape = [2, 2]
       val = [0]
       dense = constant_op.constant(np.zeros(shape, dtype=np.int32))
@@ -205,7 +206,7 @@ class SparseAddTest(test.TestCase):
 
         with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                      "invalid index"):
-          sess.run(s)
+          self.evaluate(s)
 
 ######################## Benchmarking code
 
diff --git a/tensorflow/python/kernel_tests/sparse_concat_op_test.py b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
index 402c5eb4ea..368a533e56 100644
--- a/tensorflow/python/kernel_tests/sparse_concat_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
@@ -147,7 +147,7 @@ class SparseConcatTest(test.TestCase):
           self.assertEqual(sp_concat.values.get_shape(), [4])
           self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-          concat_out = sess.run(sp_concat)
+          concat_out = self.evaluate(sp_concat)
 
           self.assertAllEqual(concat_out.indices,
                               [[0, 2], [1, 0], [2, 0], [2, 2]])
@@ -169,7 +169,7 @@ class SparseConcatTest(test.TestCase):
             self.assertEqual(sp_concat.values.get_shape(), [8])
             self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-            concat_out = sess.run(sp_concat)
+            concat_out = self.evaluate(sp_concat)
 
             self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4],
                                                      [2, 0], [2, 2], [2, 3],
@@ -195,7 +195,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [7])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = sess.run(sp_concat)
+        concat_out = self.evaluate(sp_concat)
 
         self.assertAllEqual(
             concat_out.indices,
@@ -220,7 +220,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [10])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = sess.run(sp_concat)
+        concat_out = self.evaluate(sp_concat)
 
         self.assertAllEqual(concat_out.indices, [[0, 2], [1, 0], [1, 4], [1, 8],
                                                  [2, 0], [2, 2], [2, 3], [2, 6],
@@ -244,7 +244,7 @@ class SparseConcatTest(test.TestCase):
         self.assertEqual(sp_concat.values.get_shape(), [8])
         self.assertEqual(sp_concat.dense_shape.get_shape(), [2])
 
-        concat_out = sess.run(sp_concat)
+        concat_out = self.evaluate(sp_concat)
 
         self.assertAllEqual(
             concat_out.indices,
@@ -287,7 +287,7 @@ class SparseConcatTest(test.TestCase):
 
         # Shape mismatches can only be caught when the op is run
         with self.assertRaisesOpError("Input shapes must match"):
-          sess.run(sp_concat)
+          self.evaluate(sp_concat)
 
   def testMismatchedShapesExpandNonconcatDim(self):
     with self.session(use_gpu=False) as sess:
@@ -302,8 +302,8 @@ class SparseConcatTest(test.TestCase):
           sp_concat_dim1 = sparse_ops.sparse_concat(
               concat_dim1, [sp_a, sp_b, sp_c, sp_d], expand_nonconcat_dim=True)
 
-          sp_concat_dim0_out = sess.run(sp_concat_dim0)
-          sp_concat_dim1_out = sess.run(sp_concat_dim1)
+          sp_concat_dim0_out = self.evaluate(sp_concat_dim0)
+          sp_concat_dim1_out = self.evaluate(sp_concat_dim1)
 
           self.assertAllEqual(sp_concat_dim0_out.indices,
                               [[0, 2], [1, 0], [2, 0], [2, 2], [4, 1], [5, 0],
diff --git a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
index a824d5c826..66589fa315 100644
--- a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
@@ -140,7 +140,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
           t = _indexedslice(mat_to_add)
           q.apply_indexed_slices_grad(t).run()
 
-        result = sess.run(q.take_indexed_slices_grad(1))
+        result = self.evaluate(q.take_indexed_slices_grad(1))
 
         self._assertEqual_nparray(sum_elems / len(elems), result, sess)
 
@@ -189,7 +189,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual([0, 1, 2], val.indices)
       self.assertAllEqual([[0.5, 0.5], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
@@ -209,7 +209,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual([0, 1, 2], val.indices)
       self.assertAllEqual([[1, 1], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
@@ -235,7 +235,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual(val.indices, [0, 1, 2])
       self.assertAllEqual(val.values, [[0.5, 0.5], [0, 2], [3, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
@@ -252,7 +252,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
 
       takeg_t = q.take_indexed_slices_grad(1)
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
       self.assertAllEqual(val.indices, [0, 1, 2])
       self.assertAllEqual(val.values, [[5, 5], [0, 20], [30, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
@@ -269,7 +269,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_indexed_slices_grad(1)
 
       def apply_indexed_slices_grad(accum_op):
-        sess.run(accum_op)
+        self.evaluate(accum_op)
 
       threads = [
           self.checkedThread(
@@ -281,7 +281,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
 
       expected_val = sum(elems) / len(elems)
       self._assertEqual_nparray(
@@ -303,7 +303,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       takeg_t = q.take_indexed_slices_grad(1)
 
       def apply_indexed_slices_grad(accum_op):
-        sess.run(accum_op)
+        self.evaluate(accum_op)
 
       threads = [
           self.checkedThread(target=apply_indexed_slices_grad, args=(o,))
@@ -315,7 +315,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       for thread in threads:
         thread.join()
 
-      val = sess.run(takeg_t)
+      val = self.evaluate(takeg_t)
 
       expected_val = 550.0
       self._assertEqual_nparray(
@@ -338,13 +338,13 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       def apply_indexed_slices_grad():
         for accum_op in accum_ops:
           time.sleep(1.0)
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       apply_indexed_slices_grad_thread = self.checkedThread(
           target=apply_indexed_slices_grad)
 
       def take_grad():
-        t = sess.run(takeg_t)
+        t = self.evaluate(takeg_t)
         results.append(t)
 
       threads = [self.checkedThread(target=take_grad) for _ in range(10)]
@@ -378,10 +378,10 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
       def apply_indexed_slices_grad():
         for accum_op in accum_ops:
-          sess.run(accum_op)
+          self.evaluate(accum_op)
 
       def take_grad():
-        results.append(sess.run(takeg_t))
+        results.append(self.evaluate(takeg_t))
 
       accum_thread = self.checkedThread(target=apply_indexed_slices_grad)
       takeg_thread = self.checkedThread(target=take_grad)
@@ -394,7 +394,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
   def _blocking_takeg(self, sess, takeg_op):
     with self.assertRaisesOpError("was cancelled"):
-      sess.run(takeg_op)
+      self.evaluate(takeg_op)
 
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
@@ -585,7 +585,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
                     np.float32)).run()
 
       # After take grad, constraints on accumulated gradient are removed
-      sess.run(q.take_grad(1))
+      self.evaluate(q.take_grad(1))
 
       # First successful gradient imposes new constraints.
       # Hereafter, shape will additionally constrained to [None,2,2,3]
@@ -615,7 +615,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
           grad_values=np.array(
               [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]).astype(np.float32)).run()
 
-      val = sess.run(q.take_indexed_slices_grad(1))
+      val = self.evaluate(q.take_indexed_slices_grad(1))
       self.assertAllEqual(val.dense_shape, [2, 2, 2, 2])
 
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -627,7 +627,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
               [[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]]).astype(
                   np.float32)).run()
 
-      val = sess.run(q.take_indexed_slices_grad(1))
+      val = self.evaluate(q.take_indexed_slices_grad(1))
       self.assertAllEqual(val.dense_shape, [-1, 2, 2, 3])
 
   def testApplyGradtInt32IndicesAndShape(self):
@@ -653,7 +653,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
       self.assertEqual(q.num_accumulated().eval(), 2)
 
-      val = sess.run(q.take_indexed_slices_grad(1))
+      val = self.evaluate(q.take_indexed_slices_grad(1))
       self.assertAllEqual(val.indices, [0, 2])
       self.assertAllEqual(val.values, [[0, 0, 1], [3, 0, 4]])
       self.assertAllEqual(val.dense_shape, [3, 3])
diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
index 17e867439a..8451b96c56 100644
--- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
@@ -43,7 +43,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_dense(self):
     """Tests only dense inputs."""
@@ -63,7 +63,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_integer_mixed_string_sparse(self):
     """Tests mixed type."""
@@ -77,7 +77,7 @@ class SparseCrossOpTest(test.TestCase):
         '55555_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_integer_mixed_string_dense(self):
     """Tests mixed dense inputs."""
@@ -95,7 +95,7 @@ class SparseCrossOpTest(test.TestCase):
         '999999_X_batch2-FC2-F1', '999999_X_batch2-FC2-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_sparse_cross_dense(self):
     """Tests sparse and dense inputs."""
@@ -112,7 +112,7 @@ class SparseCrossOpTest(test.TestCase):
             'batch2-FC1-F2_X_batch2-FC2-F1', 'batch2-FC1-F2_X_batch2-FC2-F2'
         ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_integer_sparse_input(self):
     """Tests mixed type sparse and dense inputs."""
@@ -128,7 +128,7 @@ class SparseCrossOpTest(test.TestCase):
             '5555_X_batch2-FC2-F1', '5555_X_batch2-FC2-F2'
         ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_permutation_3x3x3(self):
     """Tests 3x3x3 permutation."""
@@ -170,7 +170,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F3_X_batch1-FC2-F3_X_batch1-FC3-F3'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_permutation_3x1x2(self):
     """Tests 3x1x2 permutation."""
@@ -189,7 +189,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F3_X_batch1-FC2-F1_X_batch1-FC3-F2'
     ]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_large_batch(self):
     """Tests with large batch size to force multithreading."""
@@ -222,7 +222,7 @@ class SparseCrossOpTest(test.TestCase):
 
     expected_out = self._sparse_tensor(col_out)
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_one_column_empty(self):
     """Tests when one column is empty.
@@ -235,7 +235,7 @@ class SparseCrossOpTest(test.TestCase):
         self._sparse_tensor([['batch1-FC3-F1', 'batch1-FC3-F2']])
     ])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_empty(sess.run(op))
+      self._assert_sparse_tensor_empty(self.evaluate(op))
 
   def test_some_columns_empty(self):
     """Tests when more than one columns are empty.
@@ -254,7 +254,7 @@ class SparseCrossOpTest(test.TestCase):
         'batch1-FC1-F2_X_batch1-FC2-F1_X_batch1-FC3-F2'
     ]], 2)
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_all_columns_empty(self):
     """Tests when all columns are empty.
@@ -267,7 +267,7 @@ class SparseCrossOpTest(test.TestCase):
         self._sparse_tensor([])
     ])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_empty(sess.run(op))
+      self._assert_sparse_tensor_empty(self.evaluate(op))
 
   def test_hashed_zero_bucket_no_hash_key(self):
     op = sparse_ops.sparse_cross_hashed([
@@ -278,7 +278,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[1971693436396284976]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_hashed_zero_bucket(self):
     op = sparse_ops.sparse_cross_hashed(
@@ -291,7 +291,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[4847552627144134031]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   # TODO(sibyl-Aix6ihai): Add benchmark to compare Hashed vs Non-hashed.
   def test_hashed_no_hash_key(self):
@@ -305,7 +305,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[83]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_hashed_output(self):
     op = sparse_ops.sparse_cross_hashed(
@@ -319,7 +319,7 @@ class SparseCrossOpTest(test.TestCase):
     # Check actual hashed output to prevent unintentional hashing changes.
     expected_out = self._sparse_tensor([[31]])
     with self.cached_session() as sess:
-      self._assert_sparse_tensor_equals(expected_out, sess.run(op))
+      self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   def test_hashed__has_no_collision(self):
     """Tests that fingerprint concatenation has no collisions."""
@@ -345,7 +345,7 @@ class SparseCrossOpTest(test.TestCase):
         ],
         num_buckets=1000)
     with self.cached_session() as sess:
-      out = sess.run(op)
+      out = self.evaluate(op)
       self.assertEqual(6, len(out.values))
       self.assertAllEqual([[0, i] for i in range(6)], out.indices)
       self.assertTrue(all(x < 1000 and x >= 0 for x in out.values))
diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index db3f6c44e2..ad253595d2 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -154,7 +154,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
                        sparse_tensor.SparseTensor.from_value(values_v)):
           sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-          output = sess.run(sp_output)
+          output = self.evaluate(sp_output)
           self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat32(self):
@@ -163,7 +163,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float32)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64(self):
@@ -172,7 +172,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt32AndFloat32NonCanonicalOrder(self):
@@ -182,7 +182,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size, already_sorted=True)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testInt64AndFloat32NonCanonicalOrder(self):
@@ -192,7 +192,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size, already_sorted=True)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testInt64AndFloat64NonCanonicalOrder(self):
@@ -203,7 +203,7 @@ class SparseMergeTest(test_util.TensorFlowTestCase):
       sp_output = sparse_ops.sparse_merge(
           indices, values, vocab_size_tensor, already_sorted=True)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsNotSorted(output, vocab_size)
 
   def testShouldSetLastDimensionInDynamicShape(self):
@@ -261,7 +261,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float32)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64(self):
@@ -270,7 +270,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
   def testInt64AndFloat64Shape(self):
@@ -279,7 +279,7 @@ class SparseMergeHighDimTest(test_util.TensorFlowTestCase):
       indices, values = self._SparseTensor_3x50(np.int64, np.float64)
       sp_output = sparse_ops.sparse_merge(indices, values, vocab_size)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
       self._AssertResultsSorted(output, vocab_size)
 
 
@@ -302,7 +302,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
         to_retain = np.array([1, 0, 0, 1, 1, 0], dtype=np.bool)
         sp_output = sparse_ops.sparse_retain(sp_input, to_retain)
 
-        output = sess.run(sp_output)
+        output = self.evaluate(sp_output)
 
         self.assertAllEqual(output.indices, [[0, 0], [1, 4], [3, 2]])
         self.assertAllEqual(output.values, [0, 14, 32])
@@ -314,7 +314,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
       to_retain = np.zeros((6,), dtype=np.bool)
       sp_output = sparse_ops.sparse_retain(sp_input, to_retain)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, np.array([]).reshape((0, 2)))
       self.assertAllEqual(output.values, [])
@@ -365,7 +365,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       new_shape = np.array([3, 6, 7], dtype=np.int64)
       sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -378,7 +378,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       new_shape = np.array([3, 6, 7], dtype=np.int64)
       sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -404,7 +404,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       sp_input = self._SparseTensor_2x5x6()
       sp_output = sparse_ops.sparse_reset_shape(sp_input)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices, [[0, 0, 0], [0, 1, 0], [0, 1, 3],
                                            [1, 1, 4], [1, 3, 2], [1, 3, 3]])
@@ -416,7 +416,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       sp_input = self._SparseTensor_2x5x6_Empty()
       sp_output = sparse_ops.sparse_reset_shape(sp_input)
 
-      output = sess.run(sp_output)
+      output = self.evaluate(sp_output)
 
       self.assertAllEqual(output.indices.shape, [0, 3])
       self.assertAllEqual(output.values.shape, [0])
@@ -591,8 +591,8 @@ class SparseAddTest(test_util.TensorFlowTestCase):
     sp_output = sparse_ops.sparse_add(sp_input, sp_input)
 
     with self.session(use_gpu=False) as sess:
-      sess.run(variables.global_variables_initializer())
-      output = sess.run(sp_output)
+      self.evaluate(variables.global_variables_initializer())
+      output = self.evaluate(sp_output)
       self.assertAllEqual(output.values, [2])
 
 
diff --git a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
index 7b83ae5177..bbf2f39202 100644
--- a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
@@ -60,7 +60,7 @@ class SparseReorderTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6(np.arange(6))
       sp_output = sparse_ops.sparse_reorder(input_val)
 
-      output_val = sess.run(sp_output)
+      output_val = self.evaluate(sp_output)
       self.assertAllEqual(output_val.indices, input_val.indices)
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
@@ -83,7 +83,7 @@ class SparseReorderTest(test.TestCase):
         input_val = self._SparseTensorValue_5x6(np.random.permutation(6))
         sp_output = sparse_ops.sparse_reorder(input_val)
 
-        output_val = sess.run(sp_output)
+        output_val = self.evaluate(sp_output)
         self.assertAllEqual(output_val.indices, expected_output_val.indices)
         self.assertAllEqual(output_val.values, expected_output_val.values)
         self.assertAllEqual(output_val.dense_shape,
diff --git a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
index f7be397c33..918af27091 100644
--- a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
@@ -81,7 +81,7 @@ class SparseReshapeTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6()
       sp_output = sparse_ops.sparse_reshape(input_val, [5, 6])
 
-      output_val = sess.run(sp_output)
+      output_val = self.evaluate(sp_output)
       self.assertAllEqual(output_val.indices, input_val.indices)
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
@@ -151,7 +151,7 @@ class SparseReshapeTest(test.TestCase):
       input_val = self._SparseTensorValue_5x6()
       sp_output = sparse_ops.sparse_reshape(input_val, [2, 3, 5])
 
-      output_val = sess.run(sp_output)
+      output_val = self.evaluate(sp_output)
       self.assertAllEqual(output_val.indices,
                           np.array([[0, 0, 0], [0, 1, 1], [0, 1, 4], [0, 2, 0],
                                     [1, 1, 0], [1, 1, 1]]))
diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
index b24a086969..39a9ab9b49 100644
--- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
@@ -73,7 +73,7 @@ class SerializeSparseTest(test.TestCase):
       serialized = serialize_fn(sp_input, out_type=out_type)
       sp_deserialized = deserialize_fn(serialized, dtype=dtypes.int32)
 
-      indices, values, shape = sess.run(sp_deserialized)
+      indices, values, shape = self.evaluate(sp_deserialized)
 
       self.assertAllEqual(indices, sp_input[0])
       self.assertAllEqual(values, sp_input[1])
diff --git a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
index e08464a701..538e7c69b5 100644
--- a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
@@ -88,7 +88,7 @@ class SparseTensorsMapTest(test.TestCase):
       sp_out = take_many_sparse_from_tensors_map(
           sparse_map_op=handle0.op, sparse_handles=handles_concat)
 
-      combined_indices, combined_values, combined_shape = sess.run(sp_out)
+      combined_indices, combined_values, combined_shape = self.evaluate(sp_out)
 
       self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
       self.assertAllEqual(combined_indices[:6, 1:], sp_input0[0])
@@ -114,7 +114,8 @@ class SparseTensorsMapTest(test.TestCase):
       sp_roundtrip = take_many_sparse_from_tensors_map(
           sparse_map_op=handle.op, sparse_handles=sparse_handles)
 
-      combined_indices, combined_values, combined_shape = sess.run(sp_roundtrip)
+      combined_indices, combined_values, combined_shape = self.evaluate(
+          sp_roundtrip)
 
       self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
       self.assertAllEqual(combined_indices[:6, 1:], input0_val[0])
@@ -165,19 +166,19 @@ class SparseTensorsMapTest(test.TestCase):
       with self.assertRaisesOpError(
           r"Inconsistent rank across SparseTensors: rank prior to "
           r"SparseTensor\[1\] was: 3 but rank of SparseTensor\[1\] is: 4"):
-        sess.run(sp_roundtrip)
+        self.evaluate(sp_roundtrip)
 
   def testTakeManyFailsWrongInputOp(self):
     with self.session(use_gpu=False) as sess:
       input_val = self._SparseTensorValue_5x6(np.arange(6))
       handle = add_sparse_to_tensors_map(input_val)
-      handle_value = sess.run(handle)
+      handle_value = self.evaluate(handle)
       bad_handle = handle_value + 10
       sp_roundtrip = take_many_sparse_from_tensors_map(
           sparse_map_op=handle.op, sparse_handles=[handle_value, bad_handle])
 
       with self.assertRaisesOpError(r"Unable to find SparseTensor: 10"):
-        sess.run(sp_roundtrip)
+        self.evaluate(sp_roundtrip)
 
 
 class BenchmarkSparseTensorsMapVsSerialization(test.Benchmark):
@@ -212,8 +213,8 @@ class BenchmarkSparseTensorsMapVsSerialization(test.Benchmark):
 
         variables.global_variables_initializer().run()
 
-        st_roundtrip_values = sess.run(st_roundtrip)
-        st_deserialized_values = sess.run(st_deserialized)
+        st_roundtrip_values = self.evaluate(st_roundtrip)
+        st_deserialized_values = self.evaluate(st_deserialized)
         np.testing.assert_equal(st_roundtrip_values.values,
                                 st_deserialized_values.values)
         np.testing.assert_equal(st_roundtrip_values.indices,
diff --git a/tensorflow/python/kernel_tests/sparse_xent_op_test.py b/tensorflow/python/kernel_tests/sparse_xent_op_test.py
index 3f91131dab..cc8c7c238f 100644
--- a/tensorflow/python/kernel_tests/sparse_xent_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_xent_op_test.py
@@ -66,7 +66,7 @@ class SparseXentTest(test.TestCase):
     with self.cached_session(use_gpu=True) as sess:
       loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
           np_features, np_labels)
-      tf_loss, tf_backprop = sess.run([loss, backprop])
+      tf_loss, tf_backprop = self.evaluate([loss, backprop])
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
     self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
 
@@ -76,7 +76,7 @@ class SparseXentTest(test.TestCase):
         loss, backprop = gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
             np.array([[1.], [-1.], [0.]]).astype(np.float32),
             np.array([0, 0, 0]).astype(label_dtype))
-        tf_loss, tf_backprop = sess.run([loss, backprop])
+        tf_loss, tf_backprop = self.evaluate([loss, backprop])
       self.assertAllClose([0.0, 0.0, 0.0], tf_loss)
       self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop)
 
@@ -90,7 +90,7 @@ class SparseXentTest(test.TestCase):
         loss, backprop = (
             gen_nn_ops.sparse_softmax_cross_entropy_with_logits(
                 features, labels))
-        tf_loss, tf_backprop = sess.run([loss, backprop])
+        tf_loss, tf_backprop = self.evaluate([loss, backprop])
         self.assertAllClose(
             [[np.nan] * 4, [0.25, 0.25, 0.25, -0.75],
              [-0.968, 0.087, 0.237, 0.6439], [np.nan] * 4],
@@ -104,7 +104,7 @@ class SparseXentTest(test.TestCase):
       loss, backprop = (
           gen_nn_ops.sparse_softmax_cross_entropy_with_logits(features, labels))
       with self.assertRaisesOpError("Received a label value of"):
-        sess.run([loss, backprop])
+        self.evaluate([loss, backprop])
 
   def testNpXent(self):
     # We create 2 batches of logits for testing.
@@ -226,7 +226,7 @@ class SparseXentTest(test.TestCase):
       loss = nn_ops.sparse_softmax_cross_entropy_with_logits(
           labels=labels, logits=features)
       backprop = loss.op.inputs[0].op.outputs[1]
-      tf_loss, tf_backprop = sess.run([loss, backprop])
+      tf_loss, tf_backprop = self.evaluate([loss, backprop])
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
     self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
 
diff --git a/tensorflow/python/kernel_tests/stack_ops_test.py b/tensorflow/python/kernel_tests/stack_ops_test.py
index 6c6fe8aba4..dffb260b5f 100644
--- a/tensorflow/python/kernel_tests/stack_ops_test.py
+++ b/tensorflow/python/kernel_tests/stack_ops_test.py
@@ -131,7 +131,7 @@ class StackOpTest(test.TestCase):
         pop1 = gen_data_flow_ops.stack_pop_v2(h1, dtypes.float32)
         pop2 = gen_data_flow_ops.stack_pop_v2(h2, dtypes.float32)
 
-      out1, out2 = sess.run([pop1, pop2])
+      out1, out2 = self.evaluate([pop1, pop2])
       self.assertAllClose(out1, 4.0)
       self.assertAllClose(out2, 5.0)
 
@@ -144,7 +144,7 @@ class StackOpTest(test.TestCase):
       h = gen_data_flow_ops.stack_v2(
           -1, elem_type=dtypes.float32, stack_name="foo")
       c1 = gen_data_flow_ops.stack_close_v2(h)
-      sess.run(c1)
+      self.evaluate(c1)
 
   def testCloseStack(self):
     self._testCloseStack(use_gpu=False)
@@ -157,7 +157,7 @@ class StackOpTest(test.TestCase):
       c = gen_data_flow_ops.stack_push_v2(h, [[4.0, 5.0]])
       with ops.control_dependencies([c]):
         c1 = gen_data_flow_ops.stack_close_v2(h)
-      sess.run(c1)
+      self.evaluate(c1)
 
   def testPushCloseStack(self):
     self._testPushCloseStack(use_gpu=False)
@@ -263,7 +263,7 @@ class StackOpRefTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       h = gen_data_flow_ops._stack(dtypes.float32, stack_name="foo")
       c1 = gen_data_flow_ops.stack_close(h)
-      sess.run(c1)
+      self.evaluate(c1)
 
   def testCloseStack(self):
     self._testCloseStack(use_gpu=False)
@@ -275,7 +275,7 @@ class StackOpRefTest(test.TestCase):
       c = gen_data_flow_ops.stack_push(h, [[4.0, 5.0]])
       with ops.control_dependencies([c]):
         c1 = gen_data_flow_ops.stack_close(h)
-      sess.run(c1)
+      self.evaluate(c1)
 
   def testPushCloseStack(self):
     self._testPushCloseStack(use_gpu=False)
diff --git a/tensorflow/python/kernel_tests/string_length_op_test.py b/tensorflow/python/kernel_tests/string_length_op_test.py
index 57db7302b1..06bf28ebce 100644
--- a/tensorflow/python/kernel_tests/string_length_op_test.py
+++ b/tensorflow/python/kernel_tests/string_length_op_test.py
@@ -29,7 +29,7 @@ class StringLengthOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       lengths = string_ops.string_length(strings)
-      values = sess.run(lengths)
+      values = self.evaluate(lengths)
       self.assertAllEqual(values, [[[1, 2], [3, 4], [5, 6]]])
 
   def testUnit(self):
@@ -43,9 +43,9 @@ class StringLengthOpTest(test.TestCase):
       utf8_char_lengths = string_ops.string_length(
           utf8_strings, unit="UTF8_CHAR")
       self.assertAllEqual(
-          sess.run(utf8_byte_lengths), expected_utf8_byte_lengths)
+          self.evaluate(utf8_byte_lengths), expected_utf8_byte_lengths)
       self.assertAllEqual(
-          sess.run(utf8_char_lengths), expected_utf8_char_lengths)
+          self.evaluate(utf8_char_lengths), expected_utf8_char_lengths)
       with self.assertRaisesRegexp(
           ValueError, "Attr 'unit' of 'StringLength' Op passed string 'XYZ' "
           'not in: "BYTE", "UTF8_CHAR"'):
diff --git a/tensorflow/python/kernel_tests/string_split_op_test.py b/tensorflow/python/kernel_tests/string_split_op_test.py
index b968e885ed..92e13db0f7 100644
--- a/tensorflow/python/kernel_tests/string_split_op_test.py
+++ b/tensorflow/python/kernel_tests/string_split_op_test.py
@@ -34,7 +34,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
@@ -44,7 +44,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, delimiter="")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4],
                                     [1, 0], [1, 1], [1, 2], [1, 3], [2, 0],
                                     [2, 1], [2, 2], [2, 3]])
@@ -62,7 +62,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(
           indices,
           [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
@@ -74,7 +74,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, delimiter=" .")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(
           indices,
           [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
@@ -92,13 +92,13 @@ class StringSplitOpTest(test.TestCase):
           ValueError, string_ops.string_split, strings, delimiter=["a"])
 
       tokens = string_ops.string_split(strings, delimiter="|")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
       self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
       self.assertAllEqual(shape, [2, 2])
 
       tokens = string_ops.string_split(strings, delimiter="| ")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"])
       self.assertAllEqual(shape, [2, 2])
@@ -145,7 +145,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, "#", skip_empty=False)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1],
                                     [2, 0], [2, 1], [2, 2]])
@@ -154,7 +154,7 @@ class StringSplitOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split(strings, "#")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(values, [b"a", b"b", b"c"])
       self.assertAllEqual(indices, [[0, 0], [1, 0], [2, 0]])
       self.assertAllEqual(shape, [3, 1])
@@ -167,7 +167,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
@@ -182,7 +182,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep="<>")
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(
           indices, [[0, 0], [0, 1], [0, 2],
                     [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6]])
@@ -200,7 +200,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep=',')
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2],
                                     [1, 0], [1, 1], [1, 2], [1, 3], [1, 4]])
       self.assertAllEqual(values, [b"1", b"2", b"3",
@@ -217,7 +217,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2],
                                     [1, 0], [1, 1], [1, 2]])
       self.assertAllEqual(values, [b"1", b"2", b"3", b"4", b"5", b"6"])
@@ -233,7 +233,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, sep=',', maxsplit=1)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"1", b"2,3", b"4", b"5,,6,"])
@@ -249,7 +249,7 @@ class StringSplitV2OpTest(test.TestCase):
 
     with self.cached_session() as sess:
       tokens = string_ops.string_split_v2(strings, maxsplit=1)
-      indices, values, shape = sess.run(tokens)
+      indices, values, shape = self.evaluate(tokens)
       self.assertAllEqual(indices, [[0, 0], [0, 1],
                                     [1, 0], [1, 1]])
       self.assertAllEqual(values, [b"1", b"2 3", b"4", b"5    6  "])
diff --git a/tensorflow/python/kernel_tests/string_strip_op_test.py b/tensorflow/python/kernel_tests/string_strip_op_test.py
index 1e404b7146..edff3862ff 100644
--- a/tensorflow/python/kernel_tests/string_strip_op_test.py
+++ b/tensorflow/python/kernel_tests/string_strip_op_test.py
@@ -30,7 +30,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = sess.run(output)
+      output = self.evaluate(output)
       self.assertAllEqual(output, [b"pigs on the wing", b"animals"])
 
   def test_string_strip_2d(self):
@@ -39,7 +39,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = sess.run(output)
+      output = self.evaluate(output)
       self.assertAllEqual(output, [[b"pigs on the wing", b"animals"],
                                    [b"hello", b"world"]])
 
@@ -48,7 +48,7 @@ class StringStripOpTest(test.TestCase):
 
     with self.cached_session() as sess:
       output = string_ops.string_strip(strings)
-      output = sess.run(output)
+      output = self.evaluate(output)
       self.assertAllEqual(output, [b"hello", b"", b"world", b""])
 
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
index 63ce77b9d5..1547c55f8b 100644
--- a/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_audio_op_test.py
@@ -60,7 +60,7 @@ class SummaryV1AudioOpTest(test.TestCase):
         sample_rate = 8000
         summ = summary.audio(
             "snd", const, max_outputs=3, sample_rate=sample_rate)
-        value = sess.run(summ)
+        value = self.evaluate(summ)
         self.assertEqual([], summ.get_shape())
         audio_summ = self._AsSummary(value)
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
index 094606944f..e1b24756f3 100644
--- a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
@@ -70,7 +70,7 @@ class SummaryV1ImageOpTest(test.TestCase):
 
           # Summarize
           summ = summary.image("img", const)
-          value = sess.run(summ)
+          value = self.evaluate(summ)
           self.assertEqual([], summ.get_shape())
           image_summ = self._AsSummary(value)
 
@@ -97,7 +97,7 @@ class SummaryV1ImageOpTest(test.TestCase):
 
         # Summarize
         summ = summary.image("img", tf_images)
-        value = sess.run(summ)
+        value = self.evaluate(summ)
         self.assertEqual([], summ.get_shape())
         image_summ = self._AsSummary(value)
 
diff --git a/tensorflow/python/kernel_tests/summary_v1_ops_test.py b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
index 6c4e106b11..1206cb7013 100644
--- a/tensorflow/python/kernel_tests/summary_v1_ops_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
@@ -42,7 +42,7 @@ class SummaryV1OpsTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant([10.0, 20.0])
       summ = logging_ops.scalar_summary(["c1", "c2"], const, name="mysumm")
-      value = sess.run(summ)
+      value = self.evaluate(summ)
     self.assertEqual([], summ.get_shape())
     self.assertProtoEquals("""
       value { tag: "c1" simple_value: 10.0 }
@@ -53,7 +53,7 @@ class SummaryV1OpsTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant([10.0, 20.0])
       summ = logging_ops.scalar_summary(["c1", "c2"], const)
-      value = sess.run(summ)
+      value = self.evaluate(summ)
     self.assertEqual([], summ.get_shape())
     self.assertProtoEquals("""
       value { tag: "c1" simple_value: 10.0 }
@@ -66,7 +66,7 @@ class SummaryV1OpsTest(test.TestCase):
       summ1 = summary.histogram("h", const)
       summ2 = logging_ops.scalar_summary("c", const)
       merge = summary.merge([summ1, summ2])
-      value = sess.run(merge)
+      value = self.evaluate(merge)
     self.assertEqual([], merge.get_shape())
     self.assertProtoEquals("""
       value {
diff --git a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
index 34f771679a..b8e5b5b882 100644
--- a/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_tensor_op_test.py
@@ -50,7 +50,7 @@ class SummaryV1TensorOpTest(test.TestCase):
         with ops.name_scope("zod"):
           s3 = summary_lib.tensor_summary("s3", c)
           s4 = summary_lib.tensor_summary("TensorSummary", c)
-      summ1, summ2, summ3, summ4 = sess.run([s1, s2, s3, s4])
+      summ1, summ2, summ3, summ4 = self.evaluate([s1, s2, s3, s4])
 
     v1 = self._SummarySingleValue(summ1)
     self.assertEqual(v1.tag, "s1")
@@ -68,7 +68,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(10.0)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -79,7 +79,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(s)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -89,7 +89,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = array_ops.ones([5, 5, 5])
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
     self._AssertNumpyEq(n, np.ones([5, 5, 5]))
@@ -99,7 +99,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(strings)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
     self._AssertNumpyEq(n, strings)
@@ -109,7 +109,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
       const = constant_op.constant(bools)
       summ = summary_lib.tensor_summary("foo", const)
-      result = sess.run(summ)
+      result = self.evaluate(summ)
 
     value = self._SummarySingleValue(result)
     n = tensor_util.MakeNdarray(value.tensor)
@@ -119,7 +119,7 @@ class SummaryV1TensorOpTest(test.TestCase):
     with self.cached_session() as sess:
 
       def get_description(summary_op):
-        summ_str = sess.run(summary_op)
+        summ_str = self.evaluate(summary_op)
         summ = summary_pb2.Summary()
         summ.ParseFromString(summ_str)
         return summ.value[0].metadata
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index 32c97a7b19..97a280ef51 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -68,7 +68,7 @@ class SvdOpTest(test.TestCase):
             s2 = linalg_ops.svd(
                 matrix2, compute_uv=compute_uv_, full_matrices=full_matrices_)
             all_ops += [s1, s2]
-      val = sess.run(all_ops)
+      val = self.evaluate(all_ops)
       for i in range(2):
         s = 6 * i
         self.assertAllEqual(val[s], val[s + 3])  # s1 == s2
@@ -150,7 +150,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
         s_tf, u_tf, v_tf = linalg_ops.svd(
             x_tf, compute_uv=compute_uv_, full_matrices=full_matrices_)
         if use_static_shape_:
-          s_tf_val, u_tf_val, v_tf_val = sess.run([s_tf, u_tf, v_tf])
+          s_tf_val, u_tf_val, v_tf_val = self.evaluate([s_tf, u_tf, v_tf])
         else:
           s_tf_val, u_tf_val, v_tf_val = sess.run(
               [s_tf, u_tf, v_tf], feed_dict={x_tf: x_np})
@@ -158,7 +158,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
         s_tf = linalg_ops.svd(
             x_tf, compute_uv=compute_uv_, full_matrices=full_matrices_)
         if use_static_shape_:
-          s_tf_val = sess.run(s_tf)
+          s_tf_val = self.evaluate(s_tf)
         else:
           s_tf_val = sess.run(s_tf, feed_dict={x_tf: x_np})
 
diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py
index 9dcdaa61ed..a187fa115c 100644
--- a/tensorflow/python/kernel_tests/template_test.py
+++ b/tensorflow/python/kernel_tests/template_test.py
@@ -104,10 +104,10 @@ class TemplateTest(test.TestCase):
     train_op = optimizer.minimize(train_loss)
 
     with session.Session() as sess:
-      sess.run(variables.global_variables_initializer())
-      initial_test_loss = sess.run(test_loss)
-      sess.run(train_op)
-      final_test_loss = sess.run(test_loss)
+      self.evaluate(variables.global_variables_initializer())
+      initial_test_loss = self.evaluate(test_loss)
+      self.evaluate(train_op)
+      final_test_loss = self.evaluate(test_loss)
 
     # Parameters are tied, so the loss should have gone down when we trained it.
     self.assertLess(final_test_loss, initial_test_loss)
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 7e8db8947b..bb8645e2d5 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -751,7 +751,7 @@ class TensorArrayTest(test.TestCase):
                 [-0.5, 1.5],  # read(0) gradient
                 [20.0, 30.0, 40.0, 50.0]
             ])  # concat gradient
-      grad_vals = sess.run(grad_r)  # 2 + 2 entries
+      grad_vals = self.evaluate(grad_r)  # 2 + 2 entries
 
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
@@ -1286,7 +1286,7 @@ class TensorArrayTest(test.TestCase):
       r = w1.stack()
       self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
-      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
+      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
   @test_util.disable_control_flow_v2("b/117943489")
   def testSkipEagerTensorArrayUnpackDynamic(self):
@@ -1303,7 +1303,7 @@ class TensorArrayTest(test.TestCase):
       r = w1.concat()
       self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), self.evaluate(r))
       grad = gradients_impl.gradients(ys=[r], xs=[x])
-      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0])
+      self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
   def _testTensorArrayEvalEmpty(self):
     with self.cached_session(use_gpu=True):
@@ -1583,7 +1583,7 @@ class TensorArrayTest(test.TestCase):
       # wrap it in the correct name scope.
       dx, = gradients_impl.gradients(ys=[y], xs=[x], grad_ys=[dy])
       with self.cached_session(use_gpu=True) as sess:
-        vdx, vdy = sess.run([dx, dy])
+        vdx, vdy = self.evaluate([dx, dy])
       self.assertAllClose(vdx, vdy)
 
   def testSkipEagerTensorArrayInt64GPU(self):
diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py
index d9f340de6b..a72888c256 100644
--- a/tensorflow/python/kernel_tests/topk_op_test.py
+++ b/tensorflow/python/kernel_tests/topk_op_test.py
@@ -48,7 +48,7 @@ class TopKTest(test.TestCase):
     np_expected_indices = np.array(expected_indices)
     with self.cached_session(use_gpu=True) as sess:
       values_op, indices_op = nn_ops.top_k(inputs, k, sorted=sorted)
-      values, indices = sess.run([values_op, indices_op])
+      values, indices = self.evaluate([values_op, indices_op])
 
       self.assertShapeEqual(np_expected_values, values_op)
       self.assertShapeEqual(np_expected_indices, indices_op)
diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
index 4ad5ee4103..037ecd104b 100644
--- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
@@ -42,7 +42,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
       outputs = string_ops.unicode_transcode(
@@ -52,7 +52,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
       outputs = string_ops.unicode_transcode(
@@ -62,7 +62,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
   def test_transcode_utf16_to_utf8(self):
@@ -77,7 +77,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, expected)
 
   def test_transcode_bad_utf8(self):
@@ -90,7 +90,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=True)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"  ")
 
       outputs = string_ops.unicode_transcode(
@@ -100,7 +100,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
   def test_transcode_bad_utf8_with_some_good(self):
@@ -113,7 +113,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="replace",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"abc abcdefg")
 
   def test_transcode_bad_utf8_with_defaults(self):
@@ -121,7 +121,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00\xef\xbf\xbd")
 
   def test_transcode_bad_utf8_with_space_replacement(self):
@@ -130,7 +130,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8",
           replacement_char=ord(" "))
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
   def test_transcode_bad_utf8_with_strict_errors(self):
@@ -143,7 +143,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="strict")
       with self.assertRaisesOpError(
           "Invalid formatting on input string"):
-        sess.run(outputs)
+        self.evaluate(outputs)
 
   def test_transcode_bad_utf8_start_with_strict_errors(self):
     bad_string = b"\xffabcd"
@@ -155,7 +155,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           errors="strict")
       with self.assertRaisesOpError(
           "Invalid formatting on input string"):
-        sess.run(outputs)
+        self.evaluate(outputs)
 
   def test_transcode_bad_utf8_with_elision_of_malformatting(self):
     bad_string = b"\x00\xff"
@@ -165,7 +165,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           input_encoding="UTF-8",
           output_encoding="UTF-8",
           errors="ignore")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00")
 
   def test_transcode_bad_utf8_with_elision_including_control_chars(self):
@@ -177,7 +177,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           errors="ignore",
           replace_control_characters=True)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"")
 
   def test_transcode_bad_utf8_termination_with_defaults(self):
@@ -185,7 +185,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bad_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"a\xef\xbf\xbd")   # 0xFFFD
 
   def test_transcode_utf8_with_replacement_char(self):
@@ -194,13 +194,13 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       outputs = string_ops.unicode_transcode(
           strings, input_encoding="UTF-8", output_encoding="UTF-8",
           errors="strict")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
 
       outputs = string_ops.unicode_transcode(
           strings, input_encoding="UTF-8", output_encoding="UTF-8",
           errors="replace", replacement_char=ord("?"))
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, [b"a\xef\xbf\xbd"])
 
   def test_transcode_utf8_to_utf16(self):
@@ -214,7 +214,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-16-BE",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       print("values=", values)
       self.assertAllEqual(values, expected)
 
@@ -230,7 +230,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, expected)
 
   def test_transcode_utf8_to_utf32(self):
@@ -243,7 +243,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-32-BE",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, expected)
 
   # Documentation in ICU suggests that getNextUChar may produce a different
@@ -258,7 +258,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           output_encoding="UTF-8",
           replacement_char=ord(" "),
           replace_control_characters=False)
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, strings)
 
   def test_transcode_utf8_with_bom(self):
@@ -266,12 +266,12 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-8", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\xef\xbb\xbfabcdefg")  # BOM preserved
 
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-8", output_encoding="UTF-16-BE")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       utf16expected = bom_string.decode("UTF-8").encode("UTF-16-BE")
       self.assertAllEqual(values, utf16expected)
 
@@ -280,20 +280,20 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-BE", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       # BOM is preserved in output
       self.assertAllEqual(values, b"\xef\xbb\xbfa")
 
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       # mangled BOM and value from (incorrect) LE encoding
       self.assertAllEqual(values, b"\xef\xbf\xbe\xe6\x84\x80")
 
       bom_string = b"\xff\xfe\x61\x00"  # Little-endian BOM with 'a' encoded
       outputs = string_ops.unicode_transcode(
           bom_string, input_encoding="UTF-16-LE", output_encoding="UTF-8")
-      values = sess.run(outputs)
+      values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\xef\xbb\xbfa")
 
   @parameterized.parameters(
@@ -336,7 +336,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           replace_control_characters=False)
       with self.assertRaisesOpError(
           "Could not create converter for input encoding: invalid"):
-        sess.run(outputs)
+        self.evaluate(outputs)
 
     with self.assertRaisesRegexp(ValueError, "Op passed string 'invalid'"):
       with self.cached_session() as sess:
@@ -347,7 +347,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
             errors="replace",
             replacement_char=ord(" "),
             replace_control_characters=False)
-        sess.run(outputs)
+        self.evaluate(outputs)
 
   def test_invalid_error_policy_causes_errors(self):
     strings = [[b"a", b"abc"], [b"ABC", b"DEF"]]
@@ -362,7 +362,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
             errors="invalid",
             replacement_char=ord(" "),
             replace_control_characters=False)
-        sess.run(outputs)
+        self.evaluate(outputs)
 
   def test_forwarding(self):
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/unique_op_test.py b/tensorflow/python/kernel_tests/unique_op_test.py
index 316570e13e..f203263e0c 100644
--- a/tensorflow/python/kernel_tests/unique_op_test.py
+++ b/tensorflow/python/kernel_tests/unique_op_test.py
@@ -32,7 +32,7 @@ class UniqueTest(test.TestCase):
     x = np.random.randint(2, high=10, size=7000)
     with self.cached_session() as sess:
       y, idx = array_ops.unique(x)
-      tf_y, tf_idx = sess.run([y, idx])
+      tf_y, tf_idx = self.evaluate([y, idx])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -43,7 +43,7 @@ class UniqueTest(test.TestCase):
     x = np.random.randint(2, high=10, size=7000)
     with self.cached_session() as sess:
       y, idx = array_ops.unique(x, out_idx=dtypes.int64)
-      tf_y, tf_idx = sess.run([y, idx])
+      tf_y, tf_idx = self.evaluate([y, idx])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -55,7 +55,7 @@ class UniqueTest(test.TestCase):
     x = [chr(i) for i in indx]
     with self.cached_session() as sess:
       y, idx = array_ops.unique(x)
-      tf_y, tf_idx = sess.run([y, idx])
+      tf_y, tf_idx = self.evaluate([y, idx])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -67,9 +67,9 @@ class UniqueTest(test.TestCase):
       x = np.array([[1, 0, 0], [1, 0, 0], [2, 0, 0]])
       with self.cached_session() as sess:
         y0, idx0 = gen_array_ops.unique_v2(x, axis=np.array([0], dtype))
-        tf_y0, tf_idx0 = sess.run([y0, idx0])
+        tf_y0, tf_idx0 = self.evaluate([y0, idx0])
         y1, idx1 = gen_array_ops.unique_v2(x, axis=np.array([1], dtype))
-        tf_y1, tf_idx1 = sess.run([y1, idx1])
+        tf_y1, tf_idx1 = self.evaluate([y1, idx1])
       self.assertAllEqual(tf_y0, np.array([[1, 0, 0], [2, 0, 0]]))
       self.assertAllEqual(tf_idx0, np.array([0, 0, 1]))
       self.assertAllEqual(tf_y1, np.array([[1, 0], [1, 0], [2, 0]]))
@@ -81,7 +81,7 @@ class UniqueTest(test.TestCase):
     x = np.random.randint(2, high=10, size=7000)
     with self.cached_session() as sess:
       y, idx = gen_array_ops.unique_v2(x, axis=np.array([], np.int32))
-      tf_y, tf_idx = sess.run([y, idx])
+      tf_y, tf_idx = self.evaluate([y, idx])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -95,7 +95,7 @@ class UniqueWithCountsTest(test.TestCase):
     x = np.random.randint(2, high=10, size=7000)
     with self.cached_session() as sess:
       y, idx, count = array_ops.unique_with_counts(x)
-      tf_y, tf_idx, tf_count = sess.run([y, idx, count])
+      tf_y, tf_idx, tf_count = self.evaluate([y, idx, count])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -108,7 +108,7 @@ class UniqueWithCountsTest(test.TestCase):
     x = np.random.randint(2, high=10, size=7000)
     with self.cached_session() as sess:
       y, idx, count = array_ops.unique_with_counts(x, out_idx=dtypes.int64)
-      tf_y, tf_idx, tf_count = sess.run([y, idx, count])
+      tf_y, tf_idx, tf_count = self.evaluate([y, idx, count])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -123,7 +123,7 @@ class UniqueWithCountsTest(test.TestCase):
 
     with self.cached_session() as sess:
       y, idx, count = array_ops.unique_with_counts(x)
-      tf_y, tf_idx, tf_count = sess.run([y, idx, count])
+      tf_y, tf_idx, tf_count = self.evaluate([y, idx, count])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
@@ -139,10 +139,10 @@ class UniqueWithCountsTest(test.TestCase):
       with self.cached_session() as sess:
         y0, idx0, count0 = gen_array_ops.unique_with_counts_v2(
             x, axis=np.array([0], dtype))
-        tf_y0, tf_idx0, tf_count0 = sess.run([y0, idx0, count0])
+        tf_y0, tf_idx0, tf_count0 = self.evaluate([y0, idx0, count0])
         y1, idx1, count1 = gen_array_ops.unique_with_counts_v2(
             x, axis=np.array([1], dtype))
-        tf_y1, tf_idx1, tf_count1 = sess.run([y1, idx1, count1])
+        tf_y1, tf_idx1, tf_count1 = self.evaluate([y1, idx1, count1])
       self.assertAllEqual(tf_y0, np.array([[1, 0, 0], [2, 0, 0]]))
       self.assertAllEqual(tf_idx0, np.array([0, 0, 1]))
       self.assertAllEqual(tf_count0, np.array([2, 1]))
@@ -157,7 +157,7 @@ class UniqueWithCountsTest(test.TestCase):
     with self.cached_session() as sess:
       y, idx, count = gen_array_ops.unique_with_counts_v2(
           x, axis=np.array([], np.int32))
-      tf_y, tf_idx, tf_count = sess.run([y, idx, count])
+      tf_y, tf_idx, tf_count = self.evaluate([y, idx, count])
 
     self.assertEqual(len(x), len(tf_idx))
     self.assertEqual(len(tf_y), len(np.unique(x)))
diff --git a/tensorflow/python/kernel_tests/variable_ops_test.py b/tensorflow/python/kernel_tests/variable_ops_test.py
index 769bbba47b..cdfd805a93 100644
--- a/tensorflow/python/kernel_tests/variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/variable_ops_test.py
@@ -220,7 +220,7 @@ class VariableOpTest(test.TestCase):
     with self.test_session(use_gpu=True):
       # The variable and an op to increment it are on the GPU.
       var = state_ops.variable_op([1], dtypes.float32)
-      state_ops.assign(var, [1.0]).eval()
+      self.evaluate(state_ops.assign(var, [1.0]))
       increment = state_ops.assign_add(var, [1.0])
       with ops.control_dependencies([increment]):
         with ops.device("/cpu:0"):
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 6267b01a29..37012af299 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -434,19 +434,19 @@ class VariableScopeTest(test.TestCase):
         add = v1 + v0
       # v0 should be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
-        sess.run(v0)
+        self.evaluate(v0)
       # We should be able to initialize and run v1 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      sess.run(v1.initializer)
-      self.assertEqual(1, sess.run(v1))
+      self.evaluate(v1.initializer)
+      self.assertEqual(1, self.evaluate(v1))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
-        sess.run(v0)
+        self.evaluate(v0)
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
-        sess.run(add)
+        self.evaluate(add)
       # If we initialize v0 we should be able to run 'add'.
-      sess.run(v0.initializer)
-      sess.run(add)
+      self.evaluate(v0.initializer)
+      self.evaluate(add)
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # AssertionError: True is not false (last assertFalse)
@@ -489,19 +489,19 @@ class VariableScopeTest(test.TestCase):
       v2 = var_dict["v2"]
       # We should be able to initialize and run v1 and v2 without initializing
       # v0, even if the variable was created with a control dep on v0.
-      sess.run(v1.initializer)
-      self.assertEqual([1], sess.run(v1))
-      sess.run(v2.initializer)
-      self.assertEqual([2], sess.run(v2))
+      self.evaluate(v1.initializer)
+      self.assertEqual([1], self.evaluate(v1))
+      self.evaluate(v2.initializer)
+      self.assertEqual([2], self.evaluate(v2))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
-        sess.run(v0)
+        self.evaluate(v0)
       # We should not be able to run 'add' yet.
       with self.assertRaisesRegexp(errors.OpError, "uninitialized"):
-        sess.run(add)
+        self.evaluate(add)
       # If we initialize v0 we should be able to run 'add'.
-      sess.run(v0.initializer)
-      sess.run(add)
+      self.evaluate(v0.initializer)
+      self.evaluate(add)
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # TypeError: Expected tf.group() expected Tensor arguments not 'None' with
@@ -1580,7 +1580,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase):
     self.assertEqual("custom_getter/add:0", v.name)
     with self.cached_session() as sess:
       variables_lib.global_variables_initializer().run()
-      np_vars, np_v = sess.run([true_vars, v])
+      np_vars, np_v = self.evaluate([true_vars, v])
       self.assertAllClose(np_v, sum(np_vars))
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
@@ -1625,7 +1625,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase):
 
     with self.cached_session() as sess:
       variables_lib.global_variables_initializer().run()
-      np_vars, np_v = sess.run([true_vars, v])
+      np_vars, np_v = self.evaluate([true_vars, v])
       # take products of sums of products
       self.assertAllClose(
           np_v, (((np_vars[0] * np_vars[1]) + (np_vars[2] * np_vars[3])) + (
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index faa9f82067..14ec46dcb2 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -228,13 +228,13 @@ class VariablesTestCase(test.TestCase):
       self.assertEqual([2], self.evaluate(v2))
       # v0 should still be uninitialized.
       with self.assertRaisesRegexp(errors_impl.OpError, "uninitialized"):
-        sess.run(v0)
+        self.evaluate(v0)
       # We should not be able to run 'add' yet.
       with self.assertRaisesRegexp(errors_impl.OpError, "uninitialized"):
-        sess.run(add)
+        self.evaluate(add)
       # If we initialize v0 we should be able to run 'add'.
       self.evaluate(v0.initializer)
-      sess.run(add)
+      self.evaluate(add)
 
   def testControlFlowInitialization(self):
     """Expects an error if an initializer is in a control-flow scope."""
@@ -476,11 +476,11 @@ class VariablesTestCase(test.TestCase):
     with ops.Graph().as_default(), self.cached_session() as sess:
       # v describes a VariableDef-based variable without an initial value.
       v = variables.Variable(variable_def=v_def)
-      self.assertEqual(3.0, sess.run(v.initialized_value()))
+      self.assertEqual(3.0, self.evaluate(v.initialized_value()))
 
       # initialized_value should not rerun the initializer_op if the variable
       # has already been initialized elsewhere.
-      sess.run(v.assign(1.0))
+      self.evaluate(v.assign(1.0))
       self.assertEqual(1.0, v.initialized_value().eval())
 
     v_def.ClearField("initial_value_name")
@@ -492,7 +492,7 @@ class VariablesTestCase(test.TestCase):
       self.assertProtoEquals(v_def, v.to_proto())
       # But attempts to use initialized_value will result in errors.
       with self.assertRaises(ValueError):
-        sess.run(v.initialized_value())
+        self.evaluate(v.initialized_value())
 
   def testTrainableInProto(self):
     with ops.Graph().as_default():
@@ -579,7 +579,7 @@ class IsInitializedTest(test.TestCase):
       variables.global_variables_initializer().run()
       do_opt = gradient_descent.GradientDescentOptimizer(0.1).minimize(
           objective)
-      sess.run([do_opt])
+      self.evaluate([do_opt])
       self.assertAllClose([[0.9, 0.9], [0.9, 0.9]], self.evaluate(b))
 
 
@@ -596,9 +596,9 @@ class ObsoleteIsInitializedTest(test.TestCase):
       _ = v, w
       inited = variables.assert_variables_initialized()
       with self.assertRaisesOpError("Attempting to use uninitialized value"):
-        sess.run(inited)
+        self.evaluate(inited)
       variables.global_variables_initializer().run()
-      sess.run(inited)
+      self.evaluate(inited)
 
   def testVariableList(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 0634dfa2d8..48b32f06aa 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -48,8 +48,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., lambda v: v * v, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertEqual(self.evaluate(ret), 16.)
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
@@ -65,8 +65,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # Note: This is simply d_ret[0]/d_x since d_ret[1]/d_x is 0.
     grad = gradients_impl.gradients(ret, [x])  # [2*x*y]
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(ret), [45., 3.])
-      self.assertSequenceEqual(sess.run(grad), [9.])
+      self.assertSequenceEqual(self.evaluate(ret), [45., 3.])
+      self.assertSequenceEqual(self.evaluate(grad), [9.])
 
   def testMultipleLoopVars(self):
     x = constant_op.constant(5.)
@@ -88,13 +88,13 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grady_1 = gradients_impl.gradients(ret[1], [y])  # [x + 1]
     grady_2 = gradients_impl.gradients(ret, [y])  # [2*x*y + x**2 + x + 1]
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(ret), [120., 23.])
-      self.assertSequenceEqual(sess.run(gradx_0), [39.])
-      self.assertSequenceEqual(sess.run(gradx_1), [4.])
-      self.assertSequenceEqual(sess.run(gradx_2), [43.])
-      self.assertSequenceEqual(sess.run(grady_0), [55.])
-      self.assertSequenceEqual(sess.run(grady_1), [6.])
-      self.assertSequenceEqual(sess.run(grady_2), [61.])
+      self.assertSequenceEqual(self.evaluate(ret), [120., 23.])
+      self.assertSequenceEqual(self.evaluate(gradx_0), [39.])
+      self.assertSequenceEqual(self.evaluate(gradx_1), [4.])
+      self.assertSequenceEqual(self.evaluate(gradx_2), [43.])
+      self.assertSequenceEqual(self.evaluate(grady_0), [55.])
+      self.assertSequenceEqual(self.evaluate(grady_1), [6.])
+      self.assertSequenceEqual(self.evaluate(grady_2), [61.])
 
   def testMultipleWhileLoops(self):
     x = constant_op.constant(2.)
@@ -103,8 +103,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret2, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
-      self.assertSequenceEqual(sess.run(grad), [32.])
-      self.assertSequenceEqual(sess.run(grad_grad), [48.])
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
   def testDoubleDerivative(self):
     x = constant_op.constant(2.)
@@ -112,9 +112,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
-      self.assertSequenceEqual(sess.run(grad_grad), [48.])
+      self.assertEqual(self.evaluate(ret), 16.)
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
   def testPruning(self):
     x = constant_op.constant(1)
@@ -157,8 +157,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v + y < 9., lambda v: v * 3., [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 18.)
-      self.assertSequenceEqual(sess.run(grad), [9.])
+      self.assertEqual(self.evaluate(ret), 18.)
+      self.assertSequenceEqual(self.evaluate(grad), [9.])
 
   def testCaptureExternalTensorInBody(self):
     x = constant_op.constant(2.)
@@ -166,8 +166,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., lambda v: v * y, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 18.)
-      self.assertSequenceEqual(sess.run(grad), [9.])
+      self.assertEqual(self.evaluate(ret), 18.)
+      self.assertSequenceEqual(self.evaluate(grad), [9.])
 
   def testLoopWithTensorListPushBack(self):
     x = constant_op.constant(2.)
@@ -188,7 +188,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   def testDuplicateAccumulator(self):
     x = constant_op.constant(2.)
@@ -222,7 +222,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   @parameterized.named_parameters(
       ("UnknownShape", None),
@@ -315,9 +315,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
       # map_fn uses TensorArray internally.
       r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0)
-      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], sess.run(r))
+      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r))
       r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(21.0, sess.run(r))
+      self.assertAllClose(21.0, self.evaluate(r))
 
   def testNestedWhile(self):
     # Compute sum of geometric progression: n^0 + n^1 + ... + n^m
@@ -334,8 +334,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     result = while_loop_v2(lambda i, _: i >= 0, Body, [m, sum_of_powers])[1]
     grad = gradients_impl.gradients(result, [n])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(result), 364.)
-      self.assertSequenceEqual(sess.run(grad), [547.])
+      self.assertEqual(self.evaluate(result), 364.)
+      self.assertSequenceEqual(self.evaluate(grad), [547.])
 
   def testIdentityNodeInBody(self):
 
@@ -348,8 +348,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     ret = while_loop_v2(lambda v: v < 8., Body, [x])
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
-      self.assertEqual(sess.run(ret), 16.)
-      self.assertSequenceEqual(sess.run(grad), [32.])
+      self.assertEqual(self.evaluate(ret), 16.)
+      self.assertSequenceEqual(self.evaluate(grad), [32.])
 
   def testNestedWhileAndTensorArray(self):
     n = constant_op.constant(3.0)
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index c3c7f867a1..77669f08cc 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -56,7 +56,7 @@ class XentTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
           np_features, np_labels)
-      tf_loss, tf_backprop = sess.run([loss, backprop])
+      tf_loss, tf_backprop = self.evaluate([loss, backprop])
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
     self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
 
@@ -65,7 +65,7 @@ class XentTest(test.TestCase):
     with self.cached_session(use_gpu=use_gpu) as sess:
       loss = nn_ops.softmax_cross_entropy_with_logits(
           labels=np_labels, logits=np_features, dim=dim)
-      tf_loss = sess.run(loss)
+      tf_loss = self.evaluate(loss)
     print("np_loss:", np_loss)
     print("tf_loss:", tf_loss)
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
@@ -80,7 +80,7 @@ class XentTest(test.TestCase):
         loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
             np.array([[1.], [-1.], [0.]]).astype(dtype),
             np.array([[-1.], [0.], [1.]]).astype(dtype))
-        tf_loss, tf_backprop = sess.run([loss, backprop])
+        tf_loss, tf_backprop = self.evaluate([loss, backprop])
       self.assertAllClose([0.0, 0.0, 0.0], tf_loss)
       self.assertAllClose([[2.0], [1.0], [0.0]], tf_backprop)
 
@@ -148,7 +148,7 @@ class XentTest(test.TestCase):
       with self.cached_session(use_gpu=use_gpu) as sess:
         loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
             tf_f, tf_l)
-        tf_loss, tf_backprop = sess.run([loss, backprop])
+        tf_loss, tf_backprop = self.evaluate([loss, backprop])
       self.assertAllCloseAccordingToType(np_loss, tf_loss)
       self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
 
@@ -280,7 +280,7 @@ class XentTest(test.TestCase):
     with self.session(use_gpu=True) as sess:
       loss = nn_ops.softmax_cross_entropy_with_logits(
           labels=labels, logits=features)
-      tf_loss = sess.run(loss)
+      tf_loss = self.evaluate(loss)
     self.assertAllEqual(np_loss, tf_loss)
 
 
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index 257fa27156..d3200fa5b5 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -276,8 +276,8 @@ class ConvTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 32)))
         # Check that the bias still got initialized to zeros.
@@ -663,8 +663,8 @@ class SeparableConv2DTest(test.TestCase):
         self.assertTrue('depthwise_kernel' in weights[0].name)
         self.assertTrue('pointwise_kernel' in weights[1].name)
         self.assertTrue('bias' in weights[2].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 1)))
         self.assertAllClose(weights[1], np.ones((1, 1, 3, 32)))
@@ -902,8 +902,8 @@ class Conv2DTransposeTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 32, 3)))
         # Check that the bias still got initialized to zeros.
@@ -1084,8 +1084,8 @@ class Conv3DTransposeTest(test.TestCase):
         # Check the names of weights in order.
         self.assertTrue('kernel' in weights[0].name)
         self.assertTrue('bias' in weights[1].name)
-        sess.run(variables.global_variables_initializer())
-        weights = sess.run(weights)
+        self.evaluate(variables.global_variables_initializer())
+        weights = self.evaluate(weights)
         # Check that the kernel weights got initialized to ones (from scope)
         self.assertAllClose(weights[0], np.ones((3, 3, 3, 4, 32)))
         # Check that the bias still got initialized to zeros.
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index 0343bfa8bd..a61639b2db 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -443,7 +443,7 @@ class DropoutTest(test.TestCase):
       dp = core_layers.Dropout(rate, name='dropout')
       inputs = array_ops.ones((5, 5))
       dropped = dp.apply(inputs, training=True)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       np_output = sess.run(dropped, feed_dict={rate: 0.5})
       self.assertAlmostEqual(0., np_output.min())
       np_output = sess.run(dropped, feed_dict={rate: 0.0})
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index ba2bf10cf3..cc3badbde1 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -78,7 +78,7 @@ class BNTest(test.TestCase):
       if restore:
         saver.restore(sess, checkpoint_path)
       else:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
       np.random.seed(0)
       for _ in range(2):
         image_val = np.random.rand(*shape).astype(dtype.as_numpy_dtype)
@@ -321,9 +321,9 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 4, 1))
       np_beta = np.reshape(np_beta, (1, 4, 1))
 
@@ -336,8 +336,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 2))
       std = np.std(np_inputs, axis=(0, 2))
       variance = np.square(std)
@@ -363,8 +364,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 3))
       np_beta = np.reshape(np_beta, (1, 1, 3))
       for _ in range(100):
@@ -376,8 +377,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1))
       std = np.std(np_inputs, axis=(0, 1))
       variance = np.square(std)
@@ -404,8 +406,8 @@ class BNTest(test.TestCase):
 
       with self.session(use_gpu=True) as sess:
         # Test training with placeholder learning phase.
-        sess.run(variables.global_variables_initializer())
-        np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+        self.evaluate(variables.global_variables_initializer())
+        np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
         np_gamma = np.reshape(np_gamma, (1, 4, 1, 1))
         np_beta = np.reshape(np_beta, (1, 4, 1, 1))
         for _ in range(100):
@@ -417,8 +419,9 @@ class BNTest(test.TestCase):
           self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
         # Verify that the statistics are updated during training.
-        moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-        np_inputs = sess.run(inputs)
+        moving_mean, moving_var = self.evaluate(
+            [bn.moving_mean, bn.moving_variance])
+        np_inputs = self.evaluate(inputs)
         mean = np.mean(np_inputs, axis=(0, 2, 3))
         std = np.std(np_inputs, axis=(0, 2, 3))
         variance = np.square(std)
@@ -444,8 +447,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 3, 1))
       np_beta = np.reshape(np_beta, (1, 1, 3, 1))
       for _ in range(100):
@@ -457,8 +460,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 3))
       std = np.std(np_inputs, axis=(0, 1, 3))
       variance = np.square(std)
@@ -484,8 +488,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
       for _ in range(100):
@@ -497,8 +501,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -524,8 +529,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
       for _ in range(100):
@@ -537,8 +542,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -565,8 +571,8 @@ class BNTest(test.TestCase):
 
       with self.cached_session() as sess:
         # Test training with placeholder learning phase.
-        sess.run(variables.global_variables_initializer())
-        np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+        self.evaluate(variables.global_variables_initializer())
+        np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
         np_gamma = np.reshape(np_gamma, (1, 4, 1, 1))
         np_beta = np.reshape(np_beta, (1, 4, 1, 1))
         for _ in range(100):
@@ -578,8 +584,9 @@ class BNTest(test.TestCase):
           self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
         # Verify that the statistics are updated during training.
-        moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-        np_inputs = sess.run(inputs)
+        moving_mean, moving_var = self.evaluate(
+            [bn.moving_mean, bn.moving_variance])
+        np_inputs = self.evaluate(inputs)
         mean = np.mean(np_inputs, axis=(0, 2, 3))
         std = np.std(np_inputs, axis=(0, 2, 3))
         variance = np.square(std)
@@ -605,8 +612,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
       for _ in range(100):
@@ -619,8 +626,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -646,8 +654,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
       for _ in range(100):
@@ -658,8 +666,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 1, 2))
       std = np.std(np_inputs, axis=(0, 1, 2))
       variance = np.square(std)
@@ -667,7 +676,7 @@ class BNTest(test.TestCase):
       self.assertAllClose(variance, moving_var, atol=1e-2)
 
       # Test inference with placeholder learning phase.
-      np_output = sess.run(outputs_infer)
+      np_output = self.evaluate(outputs_infer)
 
       # Verify that the axis is normalized during inference.
       normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
@@ -696,8 +705,8 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
-      np_gamma, np_beta = sess.run([gamma, beta])
+      self.evaluate(variables.global_variables_initializer())
+      np_gamma, np_beta = self.evaluate([gamma, beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
       for _ in range(100):
@@ -709,8 +718,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance])
-      np_inputs = sess.run(inputs)
+      np_moving_mean, np_moving_var = self.evaluate(
+          [moving_mean, moving_variance])
+      np_inputs = self.evaluate(inputs)
       np_mean = np.mean(np_inputs, axis=(0, 1, 2))
       np_std = np.std(np_inputs, axis=(0, 1, 2))
       np_variance = np.square(np_std)
@@ -758,14 +768,15 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(100):
         np_output, _, _ = sess.run([outputs2] + updates,
                                    feed_dict={training: True})
 
       # Verify that the statistics are updated during training.
-      np_moving_mean, np_moving_var = sess.run([moving_mean, moving_variance])
-      np_inputs = sess.run(inputs2)
+      np_moving_mean, np_moving_var = self.evaluate(
+          [moving_mean, moving_variance])
+      np_inputs = self.evaluate(inputs2)
       np_mean = np.mean(np_inputs, axis=(0, 1, 2))
       np_std = np.std(np_inputs, axis=(0, 1, 2))
       np_variance = np.square(np_std)
@@ -773,7 +784,7 @@ class BNTest(test.TestCase):
       self.assertAllClose(np_variance, np_moving_var, atol=1e-2)
 
       # Verify that the axis is normalized during training.
-      np_gamma, np_beta = sess.run([gamma, beta])
+      np_gamma, np_beta = self.evaluate([gamma, beta])
       np_gamma = np.reshape(np_gamma, (1, 1, 1, 6))
       np_beta = np.reshape(np_beta, (1, 1, 1, 6))
       normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
@@ -885,7 +896,7 @@ class BNTest(test.TestCase):
     renorm_mean = renorm_stddev = 0.
     renorm_weight = 0.
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -937,7 +948,7 @@ class BNTest(test.TestCase):
     moving_mean = 0.
     moving_variance = 1.
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
         yt_val_train, adj_scale_val, adj_bias_val = sess.run(
@@ -990,7 +1001,7 @@ class BNTest(test.TestCase):
     renorm_mean = renorm_stddev = 0.
     renorm_weight = 0.
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
         yt_val_train, adj_scale_val, adj_bias_val = sess.run(
@@ -1040,7 +1051,7 @@ class BNTest(test.TestCase):
         out1.shape.as_list(), out2.shape.as_list())
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       x = np.random.random(shape)
       y1, y2 = sess.run([out1, out2], feed_dict={inp: x})
@@ -1062,7 +1073,7 @@ class BNTest(test.TestCase):
         inp, virtual_batch_size=2)
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       x = np.random.random(np_shape)
       y = sess.run(out, feed_dict={inp: x})
@@ -1093,7 +1104,7 @@ class BNTest(test.TestCase):
                     shape[1]])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1146,7 +1157,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1200,7 +1211,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
@@ -1256,9 +1267,9 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
 
       for _ in range(100):
         np_output, _, _ = sess.run([outputs] + bn.updates,
@@ -1269,8 +1280,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=0, keepdims=True)
       std = np.std(np_inputs, axis=0, keepdims=True)
       variance = np.square(std)
@@ -1296,9 +1308,9 @@ class BNTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Test training with placeholder learning phase.
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
-      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+      np_gamma, np_beta = self.evaluate([bn.gamma, bn.beta])
 
       for _ in range(100):
         np_output, _, _ = sess.run([outputs] + bn.updates,
@@ -1309,8 +1321,9 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
       # Verify that the statistics are updated during training.
-      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
-      np_inputs = sess.run(inputs)
+      moving_mean, moving_var = self.evaluate(
+          [bn.moving_mean, bn.moving_variance])
+      np_inputs = self.evaluate(inputs)
       mean = np.mean(np_inputs, axis=(0, 4), keepdims=True)
       std = np.std(np_inputs, axis=(0, 4), keepdims=True)
       variance = np.square(std)
@@ -1350,7 +1363,7 @@ class BNTest(test.TestCase):
                    shape[1:])
 
     with self.session(use_gpu=True) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in range(5):
         x = np.random.random(shape)
 
diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py
index f6f35374c0..739278273b 100644
--- a/tensorflow/python/ops/bitwise_ops_test.py
+++ b/tensorflow/python/ops/bitwise_ops_test.py
@@ -66,7 +66,8 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
         inputs = np.array(raw_inputs, dtype=dtype.as_numpy_dtype)
         truth = [count_bits(x) for x in inputs]
         input_tensor = constant_op.constant(inputs, dtype=dtype)
-        popcnt_result = sess.run(gen_bitwise_ops.population_count(input_tensor))
+        popcnt_result = self.evaluate(
+            gen_bitwise_ops.population_count(input_tensor))
         self.assertAllEqual(truth, popcnt_result)
 
   def testInvertOp(self):
@@ -89,7 +90,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(not_a_or_a, [not_0] * 4)
         # For unsigned dtypes let's also check the result directly.
         if dtype.is_unsigned:
-          inverted = sess.run(bitwise_ops.invert(input_tensor))
+          inverted = self.evaluate(bitwise_ops.invert(input_tensor))
           expected = [dtype.max - x for x in inputs]
           self.assertAllEqual(inverted, expected)
 
diff --git a/tensorflow/python/ops/clip_ops_test.py b/tensorflow/python/ops/clip_ops_test.py
index 8aa9c4ffb3..e9f7941b42 100644
--- a/tensorflow/python/ops/clip_ops_test.py
+++ b/tensorflow/python/ops/clip_ops_test.py
@@ -35,7 +35,7 @@ class ClipOpsTest(test.TestCase):
       input_op = constant_op.constant(inputs)
       clipped = clip_ops.clip_by_norm(input_op, max_norm)
       check_op = numerics.add_check_numerics_ops()
-      result, _ = sess.run([clipped, check_op])
+      result, _ = self.evaluate([clipped, check_op])
     self.assertAllClose(result, expected)
 
   def _testClipIndexedSlicesByNorm(self, values, indices, shape, max_norm,
@@ -54,7 +54,7 @@ class ClipOpsTest(test.TestCase):
       # Tensor mode
       dense_tensor = ops.convert_to_tensor(indixed_slices)
       dense_clipped = clip_ops.clip_by_norm(dense_tensor, max_norm, axes)
-      result, expected = sess.run([clipped, dense_clipped])
+      result, expected = self.evaluate([clipped, dense_clipped])
     self.assertAllClose(result, expected)
 
   def testClipTensorByNorm(self):
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index 47675d3f34..c020189ad6 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -209,9 +209,9 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
       optimizer = momentum.MomentumOptimizer(0.1, 0.9)
       train_op = optimizer.minimize(cost)
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
-          sess.run([train_op])
+          self.evaluate([train_op])
 
   def testResourceReadInLoop(self):
     with ops.Graph().as_default():
@@ -232,7 +232,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
           cond, body, [constant_op.constant(0),
                        constant_op.constant(0.0)])
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllEqual(10.0, self.evaluate(cost))
 
   def doTestIndexedSlicesGradientInCondInWhileLoop(self, use_resource=False):
@@ -269,8 +269,8 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
                                           static_grads.indices)
 
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
-        self.assertAllEqual(*sess.run([static_grads, dynamic_grads]))
+        self.evaluate(variables.global_variables_initializer())
+        self.assertAllEqual(*self.evaluate([static_grads, dynamic_grads]))
 
   def testIndexedSlicesGradientInCondInWhileLoop(self):
     self.doTestIndexedSlicesGradientInCondInWhileLoop(use_resource=False)
@@ -398,9 +398,9 @@ class CondTest(test_util.TensorFlowTestCase):
             pred=bool_var,
             true_fn=lambda: state_ops.assign(bool_var, False),
             false_fn=lambda: True)
-        sess.run(bool_var.initializer)
-        self.assertEquals(sess.run(cond_on_bool_var), False)
-        self.assertEquals(sess.run(cond_on_bool_var), True)
+        self.evaluate(bool_var.initializer)
+        self.assertEquals(self.evaluate(cond_on_bool_var), False)
+        self.assertEquals(self.evaluate(cond_on_bool_var), True)
 
   def testCondMissingArg1(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index 262b62e013..a9058c4a34 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -365,7 +365,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(
           [y], [x], unconnected_gradients="zero")
       with self.cached_session() as sess:
-        self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], sess.run(grads)[0])
+        self.assertAllEqual([[0.0, 0.0], [0.0, 0.0]], self.evaluate(grads)[0])
 
   def testUnconnectedGradientsZeroConnectedGradients(self):
     with ops.Graph().as_default():
@@ -374,7 +374,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grad = gradients.gradients(
           [y], [x], unconnected_gradients="zero")
       with self.cached_session() as sess:
-        self.assertEquals(3.0, sess.run(grad)[0])
+        self.assertEquals(3.0, self.evaluate(grad)[0])
 
   def testUnknownUnconnectedGradientsValueGiven(self):
     with ops.Graph().as_default():
@@ -438,8 +438,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(y, [x, b1])
 
       with self.cached_session() as sess:
-        self.assertAllEqual([40.0], sess.run(grads)[0])
-        self.assertAllEqual([10.0], sess.run(grads)[1])
+        self.assertAllEqual([40.0], self.evaluate(grads)[0])
+        self.assertAllEqual([10.0], self.evaluate(grads)[1])
 
   def testFunctionGradientsWithGradFunc(self):
     g = ops.Graph()
@@ -487,7 +487,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(f), 2.0)
+        self.assertEqual(self.evaluate(f), 2.0)
 
   def testGradientOfCaptured(self):
     with ops.Graph().as_default():
@@ -501,7 +501,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(f), 2.0)
+        self.assertEqual(self.evaluate(f), 2.0)
 
   def testCapturedResourceVariable(self):
     with ops.Graph().as_default():
@@ -515,8 +515,8 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       f = Foo()
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
-        self.assertEqual(sess.run(f), 2.0)
+        self.evaluate(variables.global_variables_initializer())
+        self.assertEqual(self.evaluate(f), 2.0)
 
   def testCapturedNested(self):
     with ops.Graph().as_default():
@@ -541,9 +541,9 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
       x1_grad, x2_grad = Outer()
       with self.cached_session() as sess:
         # 1.0 + None + 2.0 + 1.0 = 4.0
-        self.assertEqual(sess.run(x1_grad), 4.0)
+        self.assertEqual(self.evaluate(x1_grad), 4.0)
         # None + 1.0 + 1.0 + None = 2.0
-        self.assertEqual(sess.run(x2_grad), 2.0)
+        self.assertEqual(self.evaluate(x2_grad), 2.0)
 
   def testCapturedFromFunction(self):
     with ops.Graph().as_default():
@@ -563,7 +563,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
       z_grad = Outer()
       with self.cached_session() as sess:
-        self.assertEqual(sess.run(z_grad), 3.0)
+        self.assertEqual(self.evaluate(z_grad), 3.0)
 
   def testCapturedEagerTensors(self):
     # Test that we can handle captured eager tensors unrelated to the gradient
@@ -873,7 +873,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       y = MyMultiply(x1, x2)
       dy = gradients.gradients(y, [x1, x2])
       with session.Session() as sess:
-        self.assertAllEqual([3., 5.], sess.run(dy))
+        self.assertAllEqual([3., 5.], self.evaluate(dy))
 
   def testCustomGradientErrors(self):
 
@@ -914,7 +914,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       for g in grads:
         self.assertTrue(g is not None)
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         dw = sess.run(math_ops.reduce_sum(grads[1]))
         self.assertEqual(12., dw)
 
@@ -1074,7 +1074,7 @@ class TensorListGradientsTest(test_util.TensorFlowTestCase):
 
       grad = gradients.gradients(tl, a, grad_ys=grad_tl)[0]
       with self.cached_session() as sess:
-        self.assertEquals(sess.run(grad), 5.)
+        self.assertEquals(self.evaluate(grad), 5.)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py
index 32c2f37c0b..0ea15b0d23 100644
--- a/tensorflow/python/ops/image_grad_test.py
+++ b/tensorflow/python/ops/image_grad_test.py
@@ -44,7 +44,7 @@ class ResizeNearestNeighborOpTest(test.TestCase):
                                                        out_shape[1:3])
         self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-        resize_out = sess.run(resize_out)
+        resize_out = self.evaluate(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -113,7 +113,7 @@ class ResizeBilinearOpTest(test.TestCase):
       resize_out = image_ops.resize_bilinear(input_tensor, out_shape[1:3])
       self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-      resize_out = sess.run(resize_out)
+      resize_out = self.evaluate(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -196,7 +196,7 @@ class ResizeBicubicOpTest(test.TestCase):
                                               align_corners=align_corners)
         self.assertEqual(out_shape, list(resize_out.get_shape()))
 
-        resize_out = sess.run(resize_out)
+        resize_out = self.evaluate(resize_out)
         self.assertEqual(out_shape, list(resize_out.shape))
 
   def testGradFromResizeToLargerInBothDims(self):
@@ -273,7 +273,7 @@ class CropAndResizeOpTest(test.TestCase):
           constant_op.constant(
               crop_size, shape=[2]))
       self.assertEqual(crops_shape, list(crops.get_shape()))
-      crops = sess.run(crops)
+      crops = self.evaluate(crops)
       self.assertEqual(crops_shape, list(crops.shape))
 
   def _randomUniformAvoidAnchors(self, low, high, anchors, radius, num_samples):
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index ac2d2698b6..71a574e0a0 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -70,7 +70,8 @@ class RGBToHSVTest(test_util.TensorFlowTestCase):
         split2 = list(map(image_ops.hsv_to_rgb, split1))
         join1 = array_ops.stack(split1)
         join2 = array_ops.stack(split2)
-        batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2])
+        batch1, batch2, join1, join2 = self.evaluate(
+            [batch1, batch2, join1, join2])
 
       # Verify that processing batch elements together is the same as separate
       self.assertAllClose(batch1, join1)
@@ -109,7 +110,8 @@ class RGBToYIQTest(test_util.TensorFlowTestCase):
         split2 = list(map(image_ops.yiq_to_rgb, split1))
         join1 = array_ops.stack(split1)
         join2 = array_ops.stack(split2)
-        batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2])
+        batch1, batch2, join1, join2 = self.evaluate(
+            [batch1, batch2, join1, join2])
 
       # Verify that processing batch elements together is the same as separate
       self.assertAllClose(batch1, join1, rtol=1e-4, atol=1e-4)
@@ -138,7 +140,8 @@ class RGBToYUVTest(test_util.TensorFlowTestCase):
         split2 = list(map(image_ops.yuv_to_rgb, split1))
         join1 = array_ops.stack(split1)
         join2 = array_ops.stack(split2)
-        batch1, batch2, join1, join2 = sess.run([batch1, batch2, join1, join2])
+        batch1, batch2, join1, join2 = self.evaluate(
+            [batch1, batch2, join1, join2])
 
       # Verify that processing batch elements together is the same as separate
       self.assertAllClose(batch1, join1, rtol=1e-4, atol=1e-4)
@@ -488,11 +491,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.flip_left_right(inputs)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          sess.run(run_op)
+          self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -518,11 +521,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.random_flip_left_right(inputs)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          sess.run(run_op)
+          self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -548,11 +551,11 @@ class FlipImageBenchmark(test.Benchmark):
             trainable=False,
             dtype=dtypes.float32)
         run_op = image_ops.random_flip_left_right(inputs)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in xrange(warmup_rounds + benchmark_rounds):
           if i == warmup_rounds:
             start = time.time()
-          sess.run(run_op)
+          self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -610,11 +613,11 @@ class AdjustHueBenchmark(test.Benchmark):
       delta = constant_op.constant(0.1, dtype=dtypes.float32)
       outputs = image_ops.adjust_hue(inputs, delta)
       run_op = control_flow_ops.group(outputs)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for i in xrange(warmup_rounds + benchmark_rounds):
         if i == warmup_rounds:
           start = time.time()
-        sess.run(run_op)
+        self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -653,12 +656,12 @@ class AdjustSaturationBenchmark(test.Benchmark):
       delta = constant_op.constant(0.1, dtype=dtypes.float32)
       outputs = image_ops.adjust_saturation(inputs, delta)
       run_op = control_flow_ops.group(outputs)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       for _ in xrange(warmup_rounds):
-        sess.run(run_op)
+        self.evaluate(run_op)
       start = time.time()
       for _ in xrange(benchmark_rounds):
-        sess.run(run_op)
+        self.evaluate(run_op)
     end = time.time()
     step_time = (end - start) / benchmark_rounds
     tag = device + "_%s" % (cpu_count if cpu_count is not None else "_all")
@@ -698,7 +701,7 @@ class ResizeBilinearBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -746,7 +749,7 @@ class ResizeBicubicBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -803,7 +806,7 @@ class ResizeAreaBenchmark(test.Benchmark):
       benchmark_op = control_flow_ops.group(*deps)
 
     with self.benchmark_session() as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       results = self.run_op_benchmark(
           sess,
           benchmark_op,
@@ -2265,7 +2268,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
           image = constant_op.constant(img_np, shape=img_shape)
           y = image_ops.resize_images(image, [target_height, target_width], opt)
           yshape = array_ops.shape(y)
-          resized, newshape = sess.run([y, yshape])
+          resized, newshape = self.evaluate([y, yshape])
           self.assertAllEqual(img_shape, newshape)
           self.assertAllClose(resized, img_np, atol=1e-5)
 
@@ -2379,7 +2382,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         image = constant_op.constant(img_np, shape=img_shape)
         y = image_ops.resize_images(image, [height, width], opt)
         yshape = array_ops.shape(y)
-        resized, newshape = sess.run([y, yshape])
+        resized, newshape = self.evaluate([y, yshape])
         self.assertAllEqual(img_shape, newshape)
         self.assertAllClose(resized, img_np, atol=1e-5)
 
@@ -3066,7 +3069,7 @@ class JpegTest(test_util.TensorFlowTestCase):
       jpeg0 = io_ops.read_file(path)
       image0 = image_ops.decode_jpeg(jpeg0)
       image1 = image_ops.decode_jpeg(image_ops.encode_jpeg(image0))
-      jpeg0, image0, image1 = sess.run([jpeg0, image0, image1])
+      jpeg0, image0, image1 = self.evaluate([jpeg0, image0, image1])
       self.assertEqual(len(jpeg0), 3771)
       self.assertEqual(image0.shape, (256, 128, 3))
       self.assertLess(self.averageError(image0, image1), 1.4)
@@ -3083,7 +3086,7 @@ class JpegTest(test_util.TensorFlowTestCase):
             io_ops.read_file(rgb_path), channels=channels)
         cmyk = image_ops.decode_jpeg(
             io_ops.read_file(cmyk_path), channels=channels)
-        rgb, cmyk = sess.run([rgb, cmyk])
+        rgb, cmyk = self.evaluate([rgb, cmyk])
         self.assertEqual(rgb.shape, shape)
         self.assertEqual(cmyk.shape, shape)
         error = self.averageError(rgb, cmyk)
@@ -3112,7 +3115,7 @@ class JpegTest(test_util.TensorFlowTestCase):
                             image2.get_shape().as_list())
 
         # CropAndDecode should be equal to DecodeJpeg+Crop.
-        image1_crop, image2 = sess.run([image1_crop, image2])
+        image1_crop, image2 = self.evaluate([image1_crop, image2])
         self.assertAllEqual(image1_crop, image2)
 
   def testCropAndDecodeJpegWithInvalidCropWindow(self):
@@ -3131,7 +3134,7 @@ class JpegTest(test_util.TensorFlowTestCase):
         with self.assertRaisesWithPredicateMatch(
             errors.InvalidArgumentError,
             lambda e: "Invalid JPEG data or crop window" in str(e)):
-          sess.run(result)
+          self.evaluate(result)
 
   def testSynthetic(self):
     with self.test_session(use_gpu=True) as sess:
@@ -3141,7 +3144,8 @@ class JpegTest(test_util.TensorFlowTestCase):
       image1 = image_ops.decode_jpeg(jpeg0, dct_method="INTEGER_ACCURATE")
       image2 = image_ops.decode_jpeg(
           image_ops.encode_jpeg(image1), dct_method="INTEGER_ACCURATE")
-      jpeg0, image0, image1, image2 = sess.run([jpeg0, image0, image1, image2])
+      jpeg0, image0, image1, image2 = self.evaluate(
+          [jpeg0, image0, image1, image2])
 
       # The decoded-encoded image should be similar to the input
       self.assertLess(self.averageError(image0, image1), 0.6)
@@ -3161,7 +3165,8 @@ class JpegTest(test_util.TensorFlowTestCase):
       image1 = image_ops.decode_jpeg(jpeg0, dct_method="INTEGER_FAST")
       image2 = image_ops.decode_jpeg(
           image_ops.encode_jpeg(image1), dct_method="INTEGER_FAST")
-      jpeg0, image0, image1, image2 = sess.run([jpeg0, image0, image1, image2])
+      jpeg0, image0, image1, image2 = self.evaluate(
+          [jpeg0, image0, image1, image2])
 
       # The decoded-encoded image should be similar to the input, but
       # note this is worse than the slower algorithm because it is
@@ -3184,7 +3189,7 @@ class JpegTest(test_util.TensorFlowTestCase):
       jpeg0 = image_ops.encode_jpeg(image0)
       image1 = image_ops.decode_jpeg(jpeg0, dct_method="INTEGER_FAST")
       image2 = image_ops.decode_jpeg(jpeg0)
-      image1, image2 = sess.run([image1, image2])
+      image1, image2 = self.evaluate([image1, image2])
 
       # The images should be the same.
       self.assertAllClose(image1, image2)
@@ -3230,7 +3235,7 @@ class PngTest(test_util.TensorFlowTestCase):
         with self.test_session(use_gpu=True) as sess:
           png0 = io_ops.read_file(prefix + filename)
           image0 = image_ops.decode_png(png0, channels=channels)
-          png0, image0 = sess.run([png0, image0])
+          png0, image0 = self.evaluate([png0, image0])
           self.assertEqual(image0.shape, (26, 51, channels or channels_in))
           if channels == channels_in:
             image1 = image_ops.decode_png(image_ops.encode_png(image0))
@@ -3242,7 +3247,7 @@ class PngTest(test_util.TensorFlowTestCase):
       image0 = constant_op.constant(_SimpleColorRamp())
       png0 = image_ops.encode_png(image0, compression=7)
       image1 = image_ops.decode_png(png0)
-      png0, image0, image1 = sess.run([png0, image0, image1])
+      png0, image0, image1 = self.evaluate([png0, image0, image1])
 
       # PNG is lossless
       self.assertAllEqual(image0, image1)
@@ -3257,7 +3262,7 @@ class PngTest(test_util.TensorFlowTestCase):
       image0 = constant_op.constant(_SimpleColorRamp(), dtype=dtypes.uint16)
       png0 = image_ops.encode_png(image0, compression=7)
       image1 = image_ops.decode_png(png0, dtype=dtypes.uint16)
-      png0, image0, image1 = sess.run([png0, image0, image1])
+      png0, image0, image1 = self.evaluate([png0, image0, image1])
 
       # PNG is lossless
       self.assertAllEqual(image0, image1)
@@ -3273,7 +3278,7 @@ class PngTest(test_util.TensorFlowTestCase):
       image0 = constant_op.constant(gray_alpha)
       png0 = image_ops.encode_png(image0, compression=7)
       image1 = image_ops.decode_png(png0)
-      png0, image0, image1 = sess.run([png0, image0, image1])
+      png0, image0, image1 = self.evaluate([png0, image0, image1])
       self.assertEqual(2, image0.shape[-1])
       self.assertAllEqual(image0, image1)
 
@@ -3284,7 +3289,7 @@ class PngTest(test_util.TensorFlowTestCase):
       image0 = constant_op.constant(gray_alpha, dtype=dtypes.uint16)
       png0 = image_ops.encode_png(image0, compression=7)
       image1 = image_ops.decode_png(png0, dtype=dtypes.uint16)
-      png0, image0, image1 = sess.run([png0, image0, image1])
+      png0, image0, image1 = self.evaluate([png0, image0, image1])
       self.assertEqual(2, image0.shape[-1])
       self.assertAllEqual(image0, image1)
 
@@ -3310,7 +3315,7 @@ class GifTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True) as sess:
       gif0 = io_ops.read_file(prefix + filename)
       image0 = image_ops.decode_gif(gif0)
-      gif0, image0 = sess.run([gif0, image0])
+      gif0, image0 = self.evaluate([gif0, image0])
 
       self.assertEqual(image0.shape, shape)
 
@@ -3829,7 +3834,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
         "tensorflow/core/lib/psnr/testdata", filename))
     im = image_ops.decode_jpeg(content, dct_method="INTEGER_ACCURATE")
     im = image_ops.convert_image_dtype(im, dtypes.float32)
-    im, = sess.run([im])
+    im, = self.evaluate([im])
     return np.expand_dims(im, axis=0)
 
   def _LoadTestImages(self):
@@ -3936,7 +3941,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
         "tensorflow/core/lib/ssim/testdata", filename))
     im = image_ops.decode_png(content)
     im = image_ops.convert_image_dtype(im, dtypes.float32)
-    im, = sess.run([im])
+    im, = self.evaluate([im])
     return np.expand_dims(im, axis=0)
 
   def _LoadTestImages(self):
@@ -4028,7 +4033,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
         "tensorflow/core/lib/ssim/testdata", filename))
     im = image_ops.decode_png(content)
     im = image_ops.convert_image_dtype(im, dtypes.float32)
-    im, = sess.run([im])
+    im, = self.evaluate([im])
     return np.expand_dims(im, axis=0)
 
   def _LoadTestImages(self):
@@ -4110,7 +4115,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
       images = [ops.convert_to_tensor(x, dtype=dtypes.float32) for x in images]
       msssim_ops = [image_ops.ssim_multiscale(x, y, 1.0)
                     for x, y in itertools.combinations(images, 2)]
-      msssim = sess.run(msssim_ops)
+      msssim = self.evaluate(msssim_ops)
       msssim = np.squeeze(msssim)
 
     self.assertTrue(np.all(msssim >= 0.0))
@@ -4223,7 +4228,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(jpeg0, dtype=dtypes.uint16)
       image1 = image_ops.convert_image_dtype(image_ops.decode_jpeg(jpeg0),
                                              dtypes.uint16)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testPngUint16(self):
@@ -4233,7 +4238,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(png0, dtype=dtypes.uint16)
       image1 = image_ops.convert_image_dtype(
           image_ops.decode_png(png0, dtype=dtypes.uint16), dtypes.uint16)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testGifUint16(self):
@@ -4243,7 +4248,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(gif0, dtype=dtypes.uint16)
       image1 = image_ops.convert_image_dtype(image_ops.decode_gif(gif0),
                                              dtypes.uint16)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testBmpUint16(self):
@@ -4253,7 +4258,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(bmp0, dtype=dtypes.uint16)
       image1 = image_ops.convert_image_dtype(image_ops.decode_bmp(bmp0),
                                              dtypes.uint16)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testJpegFloat32(self):
@@ -4263,7 +4268,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(jpeg0, dtype=dtypes.float32)
       image1 = image_ops.convert_image_dtype(image_ops.decode_jpeg(jpeg0),
                                              dtypes.float32)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testPngFloat32(self):
@@ -4273,7 +4278,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(png0, dtype=dtypes.float32)
       image1 = image_ops.convert_image_dtype(
           image_ops.decode_png(png0, dtype=dtypes.uint16), dtypes.float32)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testGifFloat32(self):
@@ -4283,7 +4288,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(gif0, dtype=dtypes.float32)
       image1 = image_ops.convert_image_dtype(image_ops.decode_gif(gif0),
                                              dtypes.float32)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
   def testBmpFloat32(self):
@@ -4293,7 +4298,7 @@ class DecodeImageTest(test_util.TensorFlowTestCase):
       image0 = image_ops.decode_image(bmp0, dtype=dtypes.float32)
       image1 = image_ops.convert_image_dtype(image_ops.decode_bmp(bmp0),
                                              dtypes.float32)
-      image0, image1 = sess.run([image0, image1])
+      image0, image1 = self.evaluate([image0, image1])
       self.assertAllEqual(image0, image1)
 
 
diff --git a/tensorflow/python/ops/init_ops_test.py b/tensorflow/python/ops/init_ops_test.py
index 5693c3caaf..1f22248004 100644
--- a/tensorflow/python/ops/init_ops_test.py
+++ b/tensorflow/python/ops/init_ops_test.py
@@ -45,8 +45,8 @@ class InitializersTest(test.TestCase):
       output = variable.numpy()
     else:
       sess = ops.get_default_session()
-      sess.run(variable.initializer)
-      output = sess.run(variable)
+      self.evaluate(variable.initializer)
+      output = self.evaluate(variable)
     lim = 3e-2
     if target_std is not None:
       self.assertGreater(lim, abs(output.std() - target_std))
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index adcaa7abff..cd45b6f136 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -373,7 +373,7 @@ class AddNTest(test_util.TensorFlowTestCase):
             for i in range(0, num_inputs)
         ]
         addn = math_ops.add_n(input_vars)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         add_n_grad = gradients.gradients(addn, input_vars)
         self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
                             [g.eval() for g in add_n_grad])
@@ -461,7 +461,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       a = variables.Variable(2.)
       b = variables.Variable(4.)
       with self.cached_session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         c_grad = gradients.gradients(math_ops.divide(a, b), [a, b])
         self.assertAllEqual([x.eval() for x in c_grad], [.25, -.125])
         c_grad = gradients.gradients(math_ops.div(a, b), [a, b])
diff --git a/tensorflow/python/ops/nccl_ops_test.py b/tensorflow/python/ops/nccl_ops_test.py
index 1b496fec47..3b2e2b0175 100644
--- a/tensorflow/python/ops/nccl_ops_test.py
+++ b/tensorflow/python/ops/nccl_ops_test.py
@@ -102,7 +102,7 @@ class NcclTestCase(test.TestCase):
             continue
 
           # Test execution and results.
-          for t in sess.run(result_tensors):
+          for t in self.evaluate(result_tensors):
             self.assertAllClose(t, np_ans)
 
   def _TestGradient(self, nccl_reduce, numpy_fn):
diff --git a/tensorflow/python/ops/nn_batchnorm_test.py b/tensorflow/python/ops/nn_batchnorm_test.py
index b50bccfde2..31b2790f2b 100644
--- a/tensorflow/python/ops/nn_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_batchnorm_test.py
@@ -235,10 +235,11 @@ class BatchNormalizationTest(test.TestCase):
           odx, odm, odv, odb, odg = gradients_impl.gradients(
               [on], [x, m, v, beta, gamma], [backprop])
           if scale_after_normalization:
-            all_grads = sess.run([dx, dm, dv, db, dg, odx, odm, odv, odb, odg])
+            all_grads = self.evaluate(
+                [dx, dm, dv, db, dg, odx, odm, odv, odb, odg])
             to_check = ["dx", "dm", "dv", "db", "dg"]
           else:
-            all_grads = sess.run([dx, dm, dv, db, odx, odm, odv, odb])
+            all_grads = self.evaluate([dx, dm, dv, db, odx, odm, odv, odb])
             to_check = ["dx", "dm", "dv", "db"]
           for i, _ in enumerate(to_check):
             self.assertAllClose(
@@ -318,7 +319,7 @@ class BatchNormalizationTest(test.TestCase):
                                               gamma_val, epsilon,
                                               scale_after_normalization,
                                               shift_after_normalization)
-            [tf_batch_norm] = sess.run([bn])
+            [tf_batch_norm] = self.evaluate([bn])
             self.assertEquals(x_shape, np_batch_norm.shape)
             self.assertEquals(x_shape, tf_batch_norm.shape)
             self.assertAllClose(np_batch_norm, tf_batch_norm, atol=atol)
@@ -371,9 +372,9 @@ class SufficientStatisticsTest(test.TestCase):
           x.set_shape(x_shape)
           op_c, op_m, op_v, op_s = self._opSuffStats(x, axes, shift, keep_dims)
           if shift:
-            tf_c, tf_m, tf_v, tf_s = sess.run([op_c, op_m, op_v, op_s])
+            tf_c, tf_m, tf_v, tf_s = self.evaluate([op_c, op_m, op_v, op_s])
           else:
-            tf_c, tf_m, tf_v = sess.run([op_c, op_m, op_v])
+            tf_c, tf_m, tf_v = self.evaluate([op_c, op_m, op_v])
         else:
           x = array_ops.placeholder(
               dtype=dtypes.float32, shape=[None] * len(x_shape), name="x")
@@ -432,7 +433,7 @@ class NormalizeMomentsTest(test.TestCase):
           tf_shift_v = None
         opm, opv = self._opNormalizeMoments(tf_counts, tf_mean_ss,
                                             tf_variance_ss, tf_shift_v)
-        tfm, tfv = sess.run([opm, opv])
+        tfm, tfv = self.evaluate([opm, opv])
         self.assertAllClose(npm, tfm, atol=0.000001)
         self.assertAllClose(npv, tfv, atol=0.000001)
 
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index a6c582fcac..4bc33ff8bd 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -82,7 +82,7 @@ class BatchNormalizationTest(test.TestCase):
           epsilon=epsilon,
           data_format=data_format,
           is_training=False)
-      y_val = sess.run(y)
+      y_val = self.evaluate(y)
       y_ref = self._inference_ref(x, scale, offset, mean, var, epsilon,
                                   data_format)
     # An atol value of 1e-3 is too small for float16's, because some adjacent
@@ -127,7 +127,7 @@ class BatchNormalizationTest(test.TestCase):
           epsilon=epsilon,
           data_format=data_format,
           is_training=True)
-      y_val, mean_val, var_val = sess.run([y, mean, var])
+      y_val, mean_val, var_val = self.evaluate([y, mean, var])
       y_ref, mean_ref, var_ref = self._training_ref(x, scale, offset, epsilon,
                                                     data_format)
     y_atol = 2e-3 if x_dtype == np.float16 else 1e-3
@@ -277,10 +277,10 @@ class BatchNormalizationTest(test.TestCase):
       if is_training:
         epsilon = y.op.get_attr('epsilon')
         data_format = y.op.get_attr('data_format')
-        grad_vals = sess.run([grad_x, grad_scale, grad_offset])
+        grad_vals = self.evaluate([grad_x, grad_scale, grad_offset])
         grad_internal = nn_grad._BatchNormGrad(grad_y, x, scale, pop_mean,
                                                pop_var, epsilon, data_format)
-        grad_internal_vals = sess.run(list(grad_internal))
+        grad_internal_vals = self.evaluate(list(grad_internal))
         for grad_val, grad_internal_val in zip(grad_vals, grad_internal_vals):
           self.assertAllClose(grad_val, grad_internal_val, atol=err_tolerance)
 
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 72db0952b4..017bc9dae5 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -1090,7 +1090,7 @@ class TensorArrayTest(PForTest):
     # y = x * x. Hence dy/dx = 2 * x.
     actual_grad = 2.0 * x
     with session.Session() as sess:
-      actual_grad, computed_grad = sess.run([t1, actual_grad])
+      actual_grad, computed_grad = self.evaluate([t1, actual_grad])
       self.assertAllClose(actual_grad, computed_grad)
 
 
@@ -1244,7 +1244,7 @@ class ControlFlowTest(PForTest):
     expected_output = array_ops.transpose(expected_output, [1, 0])
 
     with session.Session() as sess:
-      out, expected = sess.run([out, expected_output])
+      out, expected = self.evaluate([out, expected_output])
       self.assertAllClose(expected, out)
 
   def test_tensor_array_as_loop_variable(self):
@@ -1432,7 +1432,7 @@ class Benchmarks(test.Benchmark):
     sess = session.Session()
     with sess:
       init = variables.global_variables_initializer()
-      sess.run(init)
+      self.evaluate(init)
       run_fn = sess.make_callable(targets)
       run_fn()  # Warm up
       begin = time.time()
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index bbb46539ea..4342833e3e 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -484,8 +484,8 @@ class GradientsTest(test.TestCase):
     pfor_jacobian, while_gradients = create_dynamic_lstm_batch_jacobian(8, 4, 3)
     with session.Session() as sess:
       init = variables.global_variables_initializer()
-      sess.run(init)
-      pfor = sess.run(pfor_jacobian)
+      self.evaluate(init)
+      pfor = self.evaluate(pfor_jacobian)
       for i in range(4):
         while_i = sess.run(while_gradients[i])
         self.assertAllClose(while_i, pfor[:, i, ...])
@@ -560,11 +560,11 @@ class GradientsBenchmarks(test.Benchmark):
     sess = session.Session()
     with sess:
       init = variables.global_variables_initializer()
-      sess.run(init)
-      sess.run(targets)
+      self.evaluate(init)
+      self.evaluate(targets)
       begin = time.time()
       for _ in range(iters):
-        sess.run(targets)
+        self.evaluate(targets)
       end = time.time()
     avg_time_ms = 1000 * (end - begin) / iters
     self.report_benchmark(iters=iters, wall_time=avg_time_ms, name=name)
diff --git a/tensorflow/python/ops/quantized_conv_ops_test.py b/tensorflow/python/ops/quantized_conv_ops_test.py
index f7fa264461..6b469a954f 100644
--- a/tensorflow/python/ops/quantized_conv_ops_test.py
+++ b/tensorflow/python/ops/quantized_conv_ops_test.py
@@ -73,7 +73,7 @@ class Conv2DTest(test.TestCase):
           max_input=x1_max,
           min_filter=x2_min,
           max_filter=x2_max)
-      value = sess.run(conv)
+      value = self.evaluate(conv)
     quantized_output = value[0]
     output_min = value[1]
     output_max = value[2]
diff --git a/tensorflow/python/ops/quantized_ops_test.py b/tensorflow/python/ops/quantized_ops_test.py
index 0f3b04e4ad..b81843d174 100644
--- a/tensorflow/python/ops/quantized_ops_test.py
+++ b/tensorflow/python/ops/quantized_ops_test.py
@@ -41,7 +41,7 @@ class QuantizedOpsTest(test.TestCase):
       x_min = 0.0
       x_max = 255.0
       op = array_ops.quantize(x, x_min, x_max, dtypes.quint8, mode="MIN_FIRST")
-      value = sess.run(op)
+      value = self.evaluate(op)
       self.assertArrayNear(expected_output, value.output, 0.1)
 
   def testDequantizeOp(self):
@@ -52,7 +52,7 @@ class QuantizedOpsTest(test.TestCase):
       x_min = 0.0
       x_max = 255.0
       op = array_ops.dequantize(x, x_min, x_max, mode="MIN_FIRST")
-      value = sess.run(op)
+      value = self.evaluate(op)
       self.assertArrayNear(expected_output, value, 0.1)
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index dcf1feaa69..c52db9e2a1 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -190,7 +190,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
     with self.test_session() as sess:
       if hasattr(expected, 'tolist'):
         expected = expected.tolist()
-      self.assertEqual(sess.run(result).tolist(), expected)
+      self.assertEqual(self.evaluate(result).tolist(), expected)
 
   def testRaggedGatherNdUnknownRankError(self):
     params = ragged.constant([['a', 'b'], ['c', 'd']])
diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
index 373a332f13..228c9bc5e4 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
@@ -119,7 +119,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
 
     segmented = segment_op(rt, segment_ids, num_segments)
     with self.test_session():
-      self.assertListEqual(segmented.eval().tolist(), expected)
+      self.assertListEqual(self.evaluate(segmented).tolist(), expected)
 
   @parameterized.parameters(
       (ragged.segment_sum, sum, [0, 0, 1, 1, 2, 2]),
@@ -173,7 +173,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                  [[411, 412], [321, 322], [331]]    # row 2
                 ]  # pyformat: disable
     with self.test_session():
-      self.assertEqual(segmented1.eval().tolist(), expected1)
+      self.assertEqual(self.evaluate(segmented1).tolist(), expected1)
 
     segment_ids2 = [1, 2, 1, 1]
     segmented2 = ragged.segment_sum(rt, segment_ids2, 3)
@@ -181,7 +181,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                  [[111+411, 112+412, 113, 114], [121+321, 322], [331]],
                  []]  # pyformat: disable
     with self.test_session():
-      self.assertEqual(segmented2.eval().tolist(), expected2)
+      self.assertEqual(self.evaluate(segmented2).tolist(), expected2)
 
   def testRaggedSegmentIds(self):
     rt = ragged.constant([
@@ -196,7 +196,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                 [111+321, 112+322, 113, 114],
                 [121+331+411, 412]]  # pyformat: disable
     with self.test_session():
-      self.assertEqual(segmented.eval().tolist(), expected)
+      self.assertEqual(self.evaluate(segmented).tolist(), expected)
 
   def testShapeMismatchError1(self):
     dt = constant_op.constant([1, 2, 3, 4, 5, 6])
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
index a1c10aff9d..cd382fe0b8 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
@@ -29,7 +29,8 @@ class RaggedTensorBoundingShapeOp(test_util.TensorFlowTestCase):
     # This is the example from ragged.bounding_shape.__doc__.
     rt = ragged.constant([[1, 2, 3, 4], [5], [], [6, 7, 8, 9], [10]])
     with self.test_session():
-      self.assertEqual(ragged.bounding_shape(rt).eval().tolist(), [5, 4])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt)).tolist(), [5, 4])
 
   def test2DRaggedTensorWithOneRaggedDimension(self):
     values = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
@@ -37,9 +38,12 @@ class RaggedTensorBoundingShapeOp(test_util.TensorFlowTestCase):
     rt2 = ragged.from_row_splits(values, [0, 7])
     rt3 = ragged.from_row_splits(values, [0, 0, 7, 7])
     with self.test_session():
-      self.assertEqual(ragged.bounding_shape(rt1).eval().tolist(), [5, 3])
-      self.assertEqual(ragged.bounding_shape(rt2).eval().tolist(), [1, 7])
-      self.assertEqual(ragged.bounding_shape(rt3).eval().tolist(), [3, 7])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7])
 
   def test3DRaggedTensorWithOneRaggedDimension(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
@@ -47,22 +51,26 @@ class RaggedTensorBoundingShapeOp(test_util.TensorFlowTestCase):
     rt2 = ragged.from_row_splits(values, [0, 7])
     rt3 = ragged.from_row_splits(values, [0, 0, 7, 7])
     with self.test_session():
-      self.assertEqual(ragged.bounding_shape(rt1).eval().tolist(), [5, 3, 2])
-      self.assertEqual(ragged.bounding_shape(rt2).eval().tolist(), [1, 7, 2])
-      self.assertEqual(ragged.bounding_shape(rt3).eval().tolist(), [3, 7, 2])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3, 2])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7, 2])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7, 2])
 
   def testNonRaggedTensor(self):
     dt = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
     with self.test_session():
-      self.assertEqual(ragged.bounding_shape(dt).eval().tolist(), [4, 3])
+      self.assertEqual(
+          self.evaluate(ragged.bounding_shape(dt)).tolist(), [4, 3])
 
   def testExplicitAxisOptimizations(self):
     rt = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
     with self.test_session():
-      self.assertEqual(ragged.bounding_shape(rt, 0).eval().tolist(), 5)
-      self.assertEqual(ragged.bounding_shape(rt, 1).eval().tolist(), 3)
+      self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 0)).tolist(), 5)
+      self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 1)).tolist(), 3)
       self.assertEqual(
-          ragged.bounding_shape(rt, [1, 0]).eval().tolist(), [3, 5])
+          self.evaluate(ragged.bounding_shape(rt, [1, 0])).tolist(), [3, 5])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index f66ca102ef..66b15d9bcc 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -264,7 +264,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     with self.test_session():
       self.assertAllEqual(rt_value_rowids, value_rowids)
-      self.assertEqual(rt_nrows.eval(), 5)
+      self.assertEqual(self.evaluate(rt_nrows), 5)
       self.assertEqual(rt.tolist(),
                        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
@@ -287,7 +287,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     with self.test_session():
       self.assertAllEqual(rt_value_rowids, value_rowids)
-      self.assertEqual(rt_nrows.eval(), 5)
+      self.assertEqual(self.evaluate(rt_nrows), 5)
       self.assertEqual(rt.tolist(),
                        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
@@ -345,7 +345,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertEqual(rt.values.shape.as_list(), [0])
     self.assertEqual(ragged.value_rowids(rt).shape.as_list(), [0])
     with self.test_session():
-      self.assertEqual(rt_nrows.eval().tolist(), 0)
+      self.assertEqual(self.evaluate(rt_nrows).tolist(), 0)
       self.assertEqual(rt.tolist(), [])
 
   def testFromRowSplits(self):
@@ -364,7 +364,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_values, values)
     self.assertIs(rt_row_splits, row_splits)
     with self.test_session():
-      self.assertEqual(rt_nrows.eval(), 5)
+      self.assertEqual(self.evaluate(rt_nrows), 5)
       self.assertEqual(rt.tolist(),
                        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
@@ -388,7 +388,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     self.assertIs(rt_values, values)
     with self.test_session():
-      self.assertEqual(rt_nrows.eval(), 5)
+      self.assertEqual(self.evaluate(rt_nrows), 5)
       self.assertAllEqual(rt_row_starts, row_starts)
       self.assertEqual(rt.tolist(),
                        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
@@ -408,7 +408,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     self.assertIs(rt_values, values)
     with self.test_session():
-      self.assertEqual(rt_nrows.eval(), 5)
+      self.assertEqual(self.evaluate(rt_nrows), 5)
       self.assertAllEqual(rt_row_limits, row_limits)
       self.assertEqual(rt.tolist(),
                        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
@@ -429,7 +429,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_values, values)
     self.assertIs(rt_row_lengths, row_lengths)  # cached_nrows
     with self.test_session():
-      self.assertEqual(rt_nrows.eval(), 5)
+      self.assertEqual(self.evaluate(rt_nrows), 5)
       self.assertAllEqual(rt_row_lengths, row_lengths)
       self.assertEqual(rt.tolist(),
                        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
@@ -606,21 +606,28 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       with self.test_session():
         self.assertEqual(rt.tolist(),
                          [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
-        self.assertEqual(rt.values.eval().tolist(),
-                         [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+        self.assertEqual(
+            self.evaluate(rt.values).tolist(),
+            [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
         self.assertEqual(rt.values.shape.dims[0].value, 7)
         self.assertEqual(
-            ragged.value_rowids(rt).eval().tolist(), [0, 0, 2, 2, 2, 3, 4])
-        self.assertEqual(ragged.nrows(rt).eval().tolist(), 5)
-        self.assertEqual(rt.row_splits.eval().tolist(), [0, 2, 2, 5, 6, 7])
-        self.assertEqual(ragged.row_starts(rt).eval().tolist(), [0, 2, 2, 5, 6])
-        self.assertEqual(ragged.row_limits(rt).eval().tolist(), [2, 2, 5, 6, 7])
+            self.evaluate(ragged.value_rowids(rt)).tolist(),
+            [0, 0, 2, 2, 2, 3, 4])
+        self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
+        self.assertEqual(
+            self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
+        self.assertEqual(
+            self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
+        self.assertEqual(
+            self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
+        self.assertEqual(
+            self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
         self.assertEqual(
-            ragged.row_lengths(rt).eval().tolist(), [2, 0, 3, 1, 1])
-        self.assertEqual(rt.inner_values.eval().tolist(),
-                         [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-        self.assertEqual([s.eval().tolist() for s in rt.nested_row_splits],
-                         [[0, 2, 2, 5, 6, 7]])
+            self.evaluate(rt.inner_values).tolist(),
+            [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+        self.assertEqual(
+            [self.evaluate(s).tolist() for s in rt.nested_row_splits],
+            [[0, 2, 2, 5, 6, 7]])
 
   def testRaggedTensorAccessors_3d_with_ragged_rank_1(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
@@ -635,22 +642,27 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                          [[[0, 1], [2, 3]], [], [[4, 5], [6, 7], [8, 9]],
                           [[10, 11]], [[12, 13]]])
         self.assertEqual(
-            rt.values.eval().tolist(),
+            self.evaluate(rt.values).tolist(),
             [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
         self.assertEqual(rt.values.shape.dims[0].value, 7)
         self.assertEqual(
-            ragged.value_rowids(rt).eval().tolist(), [0, 0, 2, 2, 2, 3, 4])
-        self.assertEqual(ragged.nrows(rt).eval().tolist(), 5)
-        self.assertEqual(rt.row_splits.eval().tolist(), [0, 2, 2, 5, 6, 7])
-        self.assertEqual(ragged.row_starts(rt).eval().tolist(), [0, 2, 2, 5, 6])
-        self.assertEqual(ragged.row_limits(rt).eval().tolist(), [2, 2, 5, 6, 7])
+            self.evaluate(ragged.value_rowids(rt)).tolist(),
+            [0, 0, 2, 2, 2, 3, 4])
+        self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
+        self.assertEqual(
+            self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
+        self.assertEqual(
+            self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
+        self.assertEqual(
+            self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
         self.assertEqual(
-            ragged.row_lengths(rt).eval().tolist(), [2, 0, 3, 1, 1])
+            self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
         self.assertEqual(
-            rt.inner_values.eval().tolist(),
+            self.evaluate(rt.inner_values).tolist(),
             [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
-        self.assertEqual([s.eval().tolist() for s in rt.nested_row_splits],
-                         [[0, 2, 2, 5, 6, 7]])
+        self.assertEqual(
+            [self.evaluate(s).tolist() for s in rt.nested_row_splits],
+            [[0, 2, 2, 5, 6, 7]])
 
   def testRaggedTensorAccessors_3d_with_ragged_rank_2(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -670,32 +682,38 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.assertEqual(
             rt.tolist(),
             [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
-        self.assertEqual(rt.values.eval().tolist(),
-                         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+        self.assertEqual(
+            self.evaluate(rt.values).tolist(),
+            [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
         self.assertEqual(rt.values.shape.dims[0].value, 5)
         self.assertEqual(
-            ragged.value_rowids(rt).eval().tolist(), [0, 0, 1, 3, 3])
-        self.assertEqual(ragged.nrows(rt).eval().tolist(), 4)
-        self.assertEqual(rt.row_splits.eval().tolist(), [0, 2, 3, 3, 5])
-        self.assertEqual(ragged.row_starts(rt).eval().tolist(), [0, 2, 3, 3])
-        self.assertEqual(ragged.row_limits(rt).eval().tolist(), [2, 3, 3, 5])
-        self.assertEqual(ragged.row_lengths(rt).eval().tolist(), [2, 1, 0, 2])
-        self.assertEqual(rt.inner_values.eval().tolist(),
-                         [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-        self.assertEqual([s.eval().tolist() for s in rt.nested_row_splits],
-                         [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]])
+            self.evaluate(ragged.value_rowids(rt)).tolist(), [0, 0, 1, 3, 3])
+        self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 4)
+        self.assertEqual(self.evaluate(rt.row_splits).tolist(), [0, 2, 3, 3, 5])
+        self.assertEqual(
+            self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 3, 3])
+        self.assertEqual(
+            self.evaluate(ragged.row_limits(rt)).tolist(), [2, 3, 3, 5])
+        self.assertEqual(
+            self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 1, 0, 2])
+        self.assertEqual(
+            self.evaluate(rt.inner_values).tolist(),
+            [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+        self.assertEqual(
+            [self.evaluate(s).tolist() for s in rt.nested_row_splits],
+            [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]])
 
   def testNRowsWithTensorInput(self):
     dt = constant_op.constant([[1, 2, 3], [4, 5, 6]])
     nrows = ragged.nrows(dt)
     with self.test_session():
-      self.assertEqual(nrows.eval(), 2)
+      self.assertEqual(self.evaluate(nrows), 2)
 
   def testRowLengthsWithTensorInput(self):
     dt = constant_op.constant([[1, 2, 3], [4, 5, 6]])
     row_lengths = ragged.row_lengths(dt)
     with self.test_session():
-      self.assertEqual(row_lengths.eval().tolist(), [3, 3])
+      self.assertEqual(self.evaluate(row_lengths).tolist(), [3, 3])
 
   #=============================================================================
   # RaggedTensor.shape
@@ -751,9 +769,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.test_session():
       tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True)
       tensor_slice_spec2 = _make_tensor_slice_spec(slice_spec, False)
-      value1 = rt.__getitem__(slice_spec).eval()
-      value2 = rt.__getitem__(tensor_slice_spec1).eval()
-      value3 = rt.__getitem__(tensor_slice_spec2).eval()
+      value1 = self.evaluate(rt.__getitem__(slice_spec))
+      value2 = self.evaluate(rt.__getitem__(tensor_slice_spec1))
+      value3 = self.evaluate(rt.__getitem__(tensor_slice_spec2))
       if hasattr(value1, 'tolist'):
         value1 = value1.tolist()
       if hasattr(value2, 'tolist'):
diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py
index 94c685274a..8648f0b514 100644
--- a/tensorflow/python/profiler/model_analyzer_test.py
+++ b/tensorflow/python/profiler/model_analyzer_test.py
@@ -93,10 +93,10 @@ class PrintModelAnalysisTest(test.TestCase):
           config=self._no_rewrite_session_config()) as sess, ops.device(dev):
         x = lib.BuildSmallModel()
 
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         pctx.trace_next_step()
         pctx.dump_next_step()
-        _ = sess.run(x)
+        _ = self.evaluate(x)
 
         pctx.profiler.profile_name_scope(options=opts)
 
@@ -160,7 +160,7 @@ class PrintModelAnalysisTest(test.TestCase):
                         ) as sess, ops.device('/device:CPU:0'):
       x = lib.BuildSmallModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -186,7 +186,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -220,9 +220,9 @@ class PrintModelAnalysisTest(test.TestCase):
       with session.Session(config=self._no_rewrite_session_config()) as sess:
         x = lib.BuildFullModel()
 
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         pctx.trace_next_step()
-        _ = sess.run(x)
+        _ = self.evaluate(x)
         tfprof_node = pctx.profiler.profile_python(options=opts)
 
         # pylint: disable=line-too-long
@@ -281,7 +281,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -309,7 +309,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -345,7 +345,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -391,7 +391,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -424,7 +424,7 @@ class PrintModelAnalysisTest(test.TestCase):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildFullModel()
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(
           x,
@@ -490,7 +490,7 @@ class PrintModelAnalysisTest(test.TestCase):
 
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -555,7 +555,7 @@ class PrintModelAnalysisTest(test.TestCase):
 
     with session.Session(config=self._no_rewrite_session_config()) as sess:
       x = lib.BuildSmallModel()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       run_meta = config_pb2.RunMetadata()
       _ = sess.run(x,
                    options=config_pb2.RunOptions(
@@ -587,10 +587,10 @@ class PrintModelAnalysisTest(test.TestCase):
   def _trainLoop(self, train_op, train_steps, time_dir, time_step,
                  memory_dir, memory_step, profile_dir, dump_step):
     with session.Session(config=self._no_rewrite_session_config()) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       # start from 1 because variable_initializer took one step.
       for i in range(1, train_steps + 1):
-        _ = sess.run(train_op)
+        _ = self.evaluate(train_op)
         if i in time_step:
           ret = gfile.ListDirectory(time_dir)
           self.assertEqual(len(ret), 1)
diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py
index 107ad443c3..680cd71d1f 100644
--- a/tensorflow/python/profiler/profile_context_test.py
+++ b/tensorflow/python/profiler/profile_context_test.py
@@ -48,10 +48,10 @@ class ProfilerContextTest(test.TestCase):
     with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
       pctx.add_auto_profiling("op", options=opts, profile_steps=[15, 50, 100])
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         total_steps = 101
         for i in range(total_steps):
-          sess.run(x)
+          self.evaluate(x)
           if i == 14 or i == 49:
             self.assertTrue(gfile.Exists(outfile))
             gfile.Remove(outfile)
@@ -75,18 +75,18 @@ class ProfilerContextTest(test.TestCase):
 
     with profile_context.ProfileContext(test.get_temp_dir(), debug=True):
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
-          sess.run(x)
+          self.evaluate(x)
           for f in gfile.ListDirectory(test.get_temp_dir()):
             # Warm up, no tracing.
             self.assertFalse("run_meta" in f)
-        sess.run(x)
+        self.evaluate(x)
         self.assertTrue(
             gfile.Exists(os.path.join(test.get_temp_dir(), "run_meta_11")))
         gfile.Remove(os.path.join(test.get_temp_dir(), "run_meta_11"))
         # fetched already.
-        sess.run(x)
+        self.evaluate(x)
         for f in gfile.ListDirectory(test.get_temp_dir()):
           self.assertFalse("run_meta" in f)
 
@@ -96,18 +96,18 @@ class ProfilerContextTest(test.TestCase):
     with profile_context.ProfileContext(test.get_temp_dir(),
                                         enabled=False) as pctx:
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
-          sess.run(x)
+          self.evaluate(x)
       self.assertTrue(pctx.profiler is None)
       self.assertTrue(
           getattr(session.BaseSession, "profile_context", None) is None)
 
     with profile_context.ProfileContext(test.get_temp_dir()) as pctx:
       with session.Session() as sess:
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for _ in range(10):
-          sess.run(x)
+          self.evaluate(x)
       self.assertFalse(pctx.profiler is None)
       self.assertFalse(
           getattr(session.BaseSession, "profile_context", None) is None)
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 648c1c5928..3678e505bd 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -50,7 +50,7 @@ class SavedModelLoaderTest(test.TestCase):
       x = variables.VariableV1(5, name="x")
       y = variables.VariableV1(11, name="y")
       z = x + y
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       foo_sig_def = signature_def_utils.build_signature_def(
           {"foo_input": utils.build_tensor_info(x)},
@@ -104,9 +104,9 @@ class SavedModelLoaderTest(test.TestCase):
     with self.session(graph=graph) as sess:
       # Check that x and y are not initialized
       with self.assertRaises(errors.FailedPreconditionError):
-        sess.run(x)
+        self.evaluate(x)
       with self.assertRaises(errors.FailedPreconditionError):
-        sess.run(y)
+        self.evaluate(y)
 
   def test_load_with_import_scope(self):
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
@@ -138,7 +138,7 @@ class SavedModelLoaderTest(test.TestCase):
       y = variables.VariableV1(0, name="y")
       z = x * y
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       # There are variables to restore, so a saver must be created.
       with self.assertRaises(ValueError):
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index a40ea7687f..e722b6ceae 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -61,7 +61,7 @@ class SavedModelTestBase(test.TestCase):
 
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.VariableV1(variable_value, name=variable_name)
-    sess.run(variables.global_variables_initializer())
+    self.evaluate(variables.global_variables_initializer())
     self.assertEqual(variable_value, self.evaluate(v))
 
   def _build_asset_collection(self, asset_file_name, asset_file_contents,
@@ -389,7 +389,7 @@ class SavedModelTest(SavedModelTestBase):
       a = ops.get_default_graph().get_tensor_by_name(constant_5_name)
       b = constant_op.constant(6.0)
       c = a * b
-      self.assertEqual(30.0, sess.run(c))
+      self.assertEqual(30.0, self.evaluate(c))
 
     # Restore the graph with tag "bar".
     with self.session(graph=ops.Graph()) as sess:
@@ -398,7 +398,7 @@ class SavedModelTest(SavedModelTestBase):
       a = ops.get_default_graph().get_tensor_by_name(constant_6_name)
       b = constant_op.constant(5.0)
       c = a * b
-      self.assertEqual(30.0, sess.run(c))
+      self.assertEqual(30.0, self.evaluate(c))
 
   def testNoOverwrite(self):
     export_dir = self._get_export_dir("test_no_overwrite")
@@ -464,7 +464,7 @@ class SavedModelTest(SavedModelTestBase):
     with self.session(graph=ops.Graph()) as sess:
       v = variables.VariableV1(42, name="v")
       ops.add_to_collection("foo_vars", v)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertEqual(42, self.evaluate(v))
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
@@ -474,7 +474,7 @@ class SavedModelTest(SavedModelTestBase):
     with self.session(graph=ops.Graph()) as sess:
       v = variables.VariableV1(43, name="v")
       ops.add_to_collection("bar_vars", v)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertEqual(43, self.evaluate(v))
       builder.add_meta_graph(["bar"])
 
@@ -802,7 +802,7 @@ class SavedModelTest(SavedModelTestBase):
         add_v1_v2 = math_ops.add(v1._ref(), v2._ref())
         custom_main_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2))
 
-      sess.run(custom_main_op)
+      self.evaluate(custom_main_op)
       builder.add_meta_graph_and_variables(
           sess, ["foo"], main_op=custom_main_op)
 
@@ -836,7 +836,7 @@ class SavedModelTest(SavedModelTestBase):
       assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2))
       legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op")
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(
           sess, ["foo"], legacy_init_op=legacy_init_op)
 
@@ -879,7 +879,7 @@ class SavedModelTest(SavedModelTestBase):
       assign_v2 = state_ops.assign(v2, v1)
       init_op = control_flow_ops.group(assign_v2, name="init_op")
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       ops.add_to_collection(key, control_flow_ops.no_op())
       # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
@@ -902,10 +902,10 @@ class SavedModelTest(SavedModelTestBase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       train_op = state_ops.assign_add(v1, v2)
 
-      sess.run(train_op)
+      self.evaluate(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph_and_variables(sess, ["foo"])
@@ -931,10 +931,10 @@ class SavedModelTest(SavedModelTestBase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       train_op = control_flow_ops.group()
 
-      sess.run(train_op)
+      self.evaluate(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph_and_variables(sess, ["foo"])
@@ -960,11 +960,11 @@ class SavedModelTest(SavedModelTestBase):
       v2 = variables.VariableV1(2, name="v2")
       ops.add_to_collection("v", v2)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["pre_foo"])
 
       train_op = state_ops.assign_add(v1, v2)
-      sess.run(train_op)
+      self.evaluate(train_op)
       # TODO(karmel): remove explicit call when in the public method.
       builder._add_train_op(train_op)
       builder.add_meta_graph(["foo"])
@@ -1090,7 +1090,7 @@ class SavedModelTest(SavedModelTestBase):
       ops.add_to_collection("v", v3)
       ops.add_to_collection("init_op", init_op)
 
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       self.assertEqual(1, ops.get_collection("v")[0].eval())
       self.assertEqual(2, ops.get_collection("v")[1].eval())
 
@@ -1145,7 +1145,7 @@ class SavedModelTest(SavedModelTestBase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       custom_saver = training.Saver(name="my_saver")
       builder.add_meta_graph_and_variables(sess, ["tag"], saver=custom_saver)
 
@@ -1167,7 +1167,7 @@ class SavedModelTest(SavedModelTestBase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       training.Saver(name="my_saver")
       builder.add_meta_graph_and_variables(sess, ["tag"])
 
@@ -1189,7 +1189,7 @@ class SavedModelTest(SavedModelTestBase):
 
     with self.session(graph=ops.Graph()) as sess:
       variables.VariableV1(1, name="v1")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["tag_0"])
 
       saver_1 = training.Saver()
@@ -1298,7 +1298,7 @@ class SavedModelTest(SavedModelTestBase):
       real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
       imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
       math_ops.complex(real_num, imag_num, name="complex")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(
           sess, ["foo"], strip_default_attrs=True)
 
@@ -1308,7 +1308,7 @@ class SavedModelTest(SavedModelTestBase):
       real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
       imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
       math_ops.complex(real_num, imag_num, name="complex")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph(["bar"], strip_default_attrs=False)
 
     # Save the SavedModel to disk in text format.
@@ -1370,7 +1370,7 @@ class SavedModelTest(SavedModelTestBase):
     with session.Session(graph=ops.Graph()) as sess:
       variables.VariableV1(1.0, dtype=dtypes.float64, name="var")
       test_ops.test_attr(T=dtypes.float32, name="test_attr")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       builder.add_meta_graph_and_variables(sess, ["foo"])
 
     # Save the SavedModel to disk in text format.
diff --git a/tensorflow/python/saved_model/simple_save_test.py b/tensorflow/python/saved_model/simple_save_test.py
index 2d404dcea4..0d0665072a 100644
--- a/tensorflow/python/saved_model/simple_save_test.py
+++ b/tensorflow/python/saved_model/simple_save_test.py
@@ -33,7 +33,7 @@ class SimpleSaveTest(test.TestCase):
 
   def _init_and_validate_variable(self, sess, variable_name, variable_value):
     v = variables.Variable(variable_value, name=variable_name)
-    sess.run(variables.global_variables_initializer())
+    self.evaluate(variables.global_variables_initializer())
     self.assertEqual(variable_value, self.evaluate(v))
     return v
 
diff --git a/tensorflow/python/tools/strip_unused_test.py b/tensorflow/python/tools/strip_unused_test.py
index 7cf0c3e3ed..e906ff94ba 100644
--- a/tensorflow/python/tools/strip_unused_test.py
+++ b/tensorflow/python/tools/strip_unused_test.py
@@ -50,7 +50,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase):
           wanted_input_node, 2.0, name="output_node")
       math_ops.add(output_node, 2.0, name="later_node")
       sess = session.Session()
-      output = sess.run(output_node)
+      output = self.evaluate(output_node)
       self.assertNear(-4.0, output, 0.00001)
       graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
 
@@ -113,7 +113,7 @@ class StripUnusedTest(test_util.TensorFlowTestCase):
           input_node1, input_node2, name="output_node")
       math_ops.add(output_node, 2.0, name="later_node")
       sess = session.Session()
-      output = sess.run(output_node)
+      output = self.evaluate(output_node)
       self.assertNear(6.0, output, 0.00001)
       graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
 
diff --git a/tensorflow/python/training/adagrad_da_test.py b/tensorflow/python/training/adagrad_da_test.py
index 761f703cb5..c7c47206a9 100644
--- a/tensorflow/python/training/adagrad_da_test.py
+++ b/tensorflow/python/training/adagrad_da_test.py
@@ -54,14 +54,14 @@ class AdagradDAOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllClose([0.0, 0.0], v0_val)
         self.assertAllClose([0.0, 0.0], v1_val)
 
         # Run a step of AdagradDA
         update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         # Let g to be gradient accumulator, gg to be gradient squared
         # accumulator, T be the global step, lr is the learning rate, and k the
         # initial gradient squared accumulator value.
@@ -119,14 +119,14 @@ class AdagradDAOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run a step of AdagradDA
         update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.904534, -1.603567]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -151,14 +151,14 @@ class AdagradDAOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run a step of AdagradDA
         update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.895489, -1.59555]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -183,14 +183,14 @@ class AdagradDAOptimizerTest(test.TestCase):
             zip([grads0, grads1], [var0, var1]), global_step=global_step)
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run a step of AdagradDA
         update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.046907, -0.093659]), v0_val)
         self.assertAllCloseAccordingToType(
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 3fabb3e086..03810b57e3 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -249,7 +249,7 @@ class LoggingTensorHookTest(test.TestCase):
           tensors=[t.name], at_end=True)
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       self.logged_message = ''
       for _ in range(3):
         mon_sess.run(train_op)
@@ -267,7 +267,7 @@ class LoggingTensorHookTest(test.TestCase):
         tensors=[t.name], every_n_iter=10, at_end=at_end)
     hook.begin()
     mon_sess = monitored_session._HookedSession(sess, [hook])
-    sess.run(variables_lib.global_variables_initializer())
+    self.evaluate(variables_lib.global_variables_initializer())
     mon_sess.run(train_op)
     self.assertRegexpMatches(str(self.logged_message), t.name)
     for _ in range(3):
@@ -314,7 +314,7 @@ class LoggingTensorHookTest(test.TestCase):
           tensors={'foo': t}, every_n_iter=1)
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess.run(train_op)
       self.assertRegexpMatches(str(self.logged_message), 'foo')
       # in first run, elapsed time is None.
@@ -328,7 +328,7 @@ class LoggingTensorHookTest(test.TestCase):
         tensors=[t.name], every_n_secs=1.0, at_end=at_end)
     hook.begin()
     mon_sess = monitored_session._HookedSession(sess, [hook])
-    sess.run(variables_lib.global_variables_initializer())
+    self.evaluate(variables_lib.global_variables_initializer())
 
     mon_sess.run(train_op)
     self.assertRegexpMatches(str(self.logged_message), t.name)
@@ -376,7 +376,7 @@ class LoggingTensorHookTest(test.TestCase):
           formatter=lambda items: 'qqq=%s' % items[t.name])
       hook.begin()
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess.run(train_op)
       self.assertEqual(self.logged_message[0], 'qqq=42.0')
 
@@ -927,7 +927,7 @@ class StepCounterHookTest(test.TestCase):
       hook = basic_session_run_hooks.StepCounterHook(
           summary_writer=summary_writer, every_n_steps=10)
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       with test.mock.patch.object(tf_logging, 'warning') as mock_log:
         for _ in range(30):
@@ -958,7 +958,7 @@ class StepCounterHookTest(test.TestCase):
           summary_writer=summary_writer, every_n_steps=None, every_n_secs=0.1)
 
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
       mock_time.return_value += 0.2
@@ -995,7 +995,7 @@ class StepCounterHookTest(test.TestCase):
           summary_writer=summary_writer, every_n_steps=1, every_n_secs=None)
 
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       mon_sess.run(train_op)
       mon_sess.run(train_op)
@@ -1015,7 +1015,7 @@ class StepCounterHookTest(test.TestCase):
     with ops.Graph().as_default(), session_lib.Session() as sess:
       variables.get_or_create_global_step()
       train_op = training_util._increment_global_step(0)  # keep same.
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       hook = basic_session_run_hooks.StepCounterHook(
           every_n_steps=1, every_n_secs=None)
       hook.begin()
@@ -1042,7 +1042,7 @@ class StepCounterHookTest(test.TestCase):
         summary_writer=self.summary_writer, every_n_steps=every_n_steps)
     self.hook._set_steps_per_run(steps_per_run)
     self.hook.begin()
-    sess.run(variables_lib.global_variables_initializer())
+    self.evaluate(variables_lib.global_variables_initializer())
     self.mon_sess = monitored_session._HookedSession(sess, [self.hook])
 
   @test.mock.patch.object(time, 'time')
@@ -1161,7 +1161,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(30):
         mon_sess.run(self.train_op)
@@ -1193,7 +1193,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(10):
         mon_sess.run(self.train_op)
@@ -1223,7 +1223,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(4):
         mon_sess.run(self.train_op)
@@ -1258,7 +1258,7 @@ class SummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(8):
         mon_sess.run(self.train_op)
@@ -1327,7 +1327,7 @@ class GlobalStepWaiterHookTest(test.TestCase):
         mock_sleep.side_effect = mock_sleep_side_effect
 
         # Run the mocked-out interaction with the hook.
-        sess.run(variables_lib.global_variables_initializer())
+        self.evaluate(variables_lib.global_variables_initializer())
         run_context = session_run_hook.SessionRunContext(
             original_args=None, session=sess)
         hook.before_run(run_context)
@@ -1422,7 +1422,7 @@ class ResourceSummarySaverHookTest(test.TestCase):
 
     with self.cached_session() as sess:
       hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
+      self.evaluate(variables_lib.global_variables_initializer())
       mon_sess = monitored_session._HookedSession(sess, [hook])
       for _ in range(30):
         mon_sess.run(self.train_op)
diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py
index 38d4acf85f..21ad3df1c8 100644
--- a/tensorflow/python/training/checkpoint_ops_test.py
+++ b/tensorflow/python/training/checkpoint_ops_test.py
@@ -47,7 +47,7 @@ class LoadAndRemapWrappersTest(test.TestCase):
       with variable_scope.variable_scope('some_scope'):
         variable_scope.get_variable(name='embeddings', shape=[5, 16],
                                     initializer=initializer)
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver = saver_lib.Saver()
       saver.save(sess, checkpoint_prefix, global_step=5)
     self.checkpoint_file = '{}-5'.format(checkpoint_prefix)
diff --git a/tensorflow/python/training/ftrl_test.py b/tensorflow/python/training/ftrl_test.py
index a61132a966..70b5db31f8 100644
--- a/tensorflow/python/training/ftrl_test.py
+++ b/tensorflow/python/training/ftrl_test.py
@@ -54,7 +54,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllClose([0.0, 0.0], v0_val)
         self.assertAllClose([0.0, 0.0], v1_val)
 
@@ -62,7 +62,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(3):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-2.60260963, -4.29698515]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -90,14 +90,14 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run 3 steps FTRL
         for _ in range(3):
           update.run()
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-2.55607247, -3.98729396]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -137,14 +137,14 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
         # Run 10 steps FTRL
         for _ in range(10):
           update.run()
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-7.66718769, -10.91273689]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -166,7 +166,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
@@ -174,7 +174,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(10):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.24059935, -0.46829352]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -203,7 +203,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([4.0, 3.0], v1_val)
 
@@ -211,7 +211,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(10):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType(
             np.array([-0.22578995, -0.44345796]), v0_val)
         self.assertAllCloseAccordingToType(
@@ -239,7 +239,7 @@ class FtrlOptimizerTest(test.TestCase):
         update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([[1.0], [2.0]], v0_val)
         self.assertAllCloseAccordingToType([[4.0], [3.0]], v1_val)
 
@@ -247,7 +247,7 @@ class FtrlOptimizerTest(test.TestCase):
         for _ in range(10):
           update.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val)
         self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val)
 
@@ -275,7 +275,7 @@ class FtrlOptimizerTest(test.TestCase):
         update1 = opt1.apply_gradients([(grads1, var1)])
         variables.global_variables_initializer().run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         self.assertAllCloseAccordingToType([1.0, 2.0], v0_val)
         self.assertAllCloseAccordingToType([1.0, 2.0], v1_val)
 
@@ -284,12 +284,12 @@ class FtrlOptimizerTest(test.TestCase):
           update0.run()
           update1.run()
 
-        v0_val, v1_val = sess.run([var0, var1])
+        v0_val, v1_val = self.evaluate([var0, var1])
         # var0 is experiencing L2 shrinkage so it should be smaller than var1
         # in magnitude.
         self.assertTrue((v0_val**2 < v1_val**2).all())
-        accum0 = list(sess.run(opt0._slots)["accum"].values())[0]
-        accum1 = list(sess.run(opt1._slots)["accum"].values())[0]
+        accum0 = list(self.evaluate(opt0._slots)["accum"].values())[0]
+        accum1 = list(self.evaluate(opt1._slots)["accum"].values())[0]
         # L2 shrinkage should not change how we update grad accumulator.
         self.assertAllCloseAccordingToType(accum0, accum1)
 
@@ -313,7 +313,7 @@ class FtrlOptimizerTest(test.TestCase):
     variables.global_variables_initializer().run()
 
     sess = ops.get_default_session()
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     if is_sparse:
       self.assertAllCloseAccordingToType([[0.0], [0.0]], v0_val)
       self.assertAllCloseAccordingToType([[0.0], [0.0]], v1_val)
@@ -325,7 +325,7 @@ class FtrlOptimizerTest(test.TestCase):
     for _ in range(steps):
       update.run()
 
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     return v0_val, v1_val
 
   # When variables are initialized with Zero, FTRL-Proximal has two properties:
diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py
index e5aac5da18..327f087138 100644
--- a/tensorflow/python/training/input_test.py
+++ b/tensorflow/python/training/input_test.py
@@ -256,7 +256,7 @@ class StringInputProducerTest(test_lib.TestCase):
           # writing of the `tf.Graph` object. However, many users
           # write code this way, so we include this test to ensure
           # that we can support it.
-          self.assertEquals(string, sess.run(queue.dequeue()))
+          self.assertEquals(string, self.evaluate(queue.dequeue()))
       coord.request_stop()
       coord.join(threads)
 
@@ -348,14 +348,14 @@ class SliceInputProducerTest(test_lib.TestCase):
 
       # No randomness, so just see repeated copies of the input.
       num_items = len(source_strings) * num_epochs
-      output = [sess.run(slices) for _ in range(num_items)]
+      output = [self.evaluate(slices) for _ in range(num_items)]
       out_strings, out_ints = zip(*output)
       self.assertAllEqual(source_strings * num_epochs, out_strings)
       self.assertAllEqual(source_ints * num_epochs, out_ints)
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(slices)
+        self.evaluate(slices)
       for thread in threads:
         thread.join()
 
@@ -383,7 +383,7 @@ class SliceInputProducerTest(test_lib.TestCase):
       for e in expected:
         frequency[e] = 0
       for _ in range(num_epochs):
-        output = [sess.run(slices) for _ in range(len(source_strings))]
+        output = [self.evaluate(slices) for _ in range(len(source_strings))]
         key = b",".join([s + compat.as_bytes(str(i)) for s, i in output])
         self.assertIn(key, expected)
         frequency[key] += 1
@@ -399,7 +399,7 @@ class SliceInputProducerTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(slices)
+        self.evaluate(slices)
       for thread in threads:
         thread.join()
 
@@ -474,7 +474,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -491,7 +491,7 @@ class BatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched_fetch)
+        self.evaluate(batched_fetch)
       for thread in threads:
         thread.join()
 
@@ -507,7 +507,7 @@ class BatchTest(test_lib.TestCase):
     with self.cached_session() as sess:
       coord = coordinator.Coordinator()
       threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord)
-      sess.run(batched)
+      self.evaluate(batched)
       coord.request_stop()
       for thread in threads:
         thread.join()
@@ -518,7 +518,7 @@ class BatchTest(test_lib.TestCase):
     with self.cached_session() as sess:
       coord = coordinator.Coordinator()
       threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord)
-      sess.run(batched)
+      self.evaluate(batched)
       coord.request_stop()
       for thread in threads:
         thread.join()
@@ -539,7 +539,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         expected_results = np.arange(i * batch_size, (i + 1) * batch_size)
         max_len = expected_results[-1]
         self.assertAllEqual(results[0], expected_results)
@@ -549,7 +549,7 @@ class BatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -571,7 +571,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -584,7 +584,7 @@ class BatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -610,7 +610,7 @@ class BatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertAllEqual(results[0], results[1].values)
@@ -624,7 +624,7 @@ class BatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -651,7 +651,7 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual(results[0],
                             np.arange(i * batch_size, (i + 1) * batch_size))
         self.assertAllEqual(
@@ -667,7 +667,7 @@ class BatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertAllEqual(results[0],
                           np.arange(num_batches * batch_size,
                                     num_batches * batch_size + extra_elements))
@@ -681,7 +681,7 @@ class BatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -709,7 +709,7 @@ class BatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertAllEqual(results[0], results[1].values)
@@ -721,7 +721,7 @@ class BatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), extra_elements)
       self.assertAllEqual(results[0], results[1].values)
@@ -736,7 +736,7 @@ class BatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -827,14 +827,14 @@ class BatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1020,7 +1020,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(3, len(results))
         self.assertEqual(batch_size, len(results[0]))
         self.assertEqual(batch_size, len(results[2]))
@@ -1051,7 +1051,7 @@ class BatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched_fetch)
+        self.evaluate(batched_fetch)
       for thread in threads:
         thread.join()
 
@@ -1116,7 +1116,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertEqual(2, len(results))
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[1]), batch_size)
@@ -1148,7 +1148,7 @@ class BatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1201,7 +1201,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -1221,7 +1221,7 @@ class BatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached the final batch with 2 * extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertEqual(len(results[2]), 2 * extra_elements)
@@ -1249,7 +1249,7 @@ class BatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1296,7 +1296,7 @@ class BatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[1]), batch_size)
@@ -1316,7 +1316,7 @@ class BatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached the final batch with 2 * extra_elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       tf_logging.info("Last Batch: %s", results[0])
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertEqual(len(results[1]), 2 * extra_elements)
@@ -1347,7 +1347,7 @@ class BatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1410,7 +1410,7 @@ class BatchJoinTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual(
             [0] * batch_size,
             np.mod(results[0], 2),)
@@ -1421,7 +1421,7 @@ class BatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1579,7 +1579,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
         self.assertAllEqual(
@@ -1597,7 +1597,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched_fetch)
+        self.evaluate(batched_fetch)
       for thread in threads:
         thread.join()
 
@@ -1634,7 +1634,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for _ in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
         self.assertAllEqual(
@@ -1645,7 +1645,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertAllEqual(results[1].dense_shape, [extra_elements, 1])
       self.assertAllEqual(results[2], [b"string"] * extra_elements)
       all_counts.extend(results[0])
@@ -1659,7 +1659,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched_fetch)
+        self.evaluate(batched_fetch)
       for thread in threads:
         thread.join()
 
@@ -1687,7 +1687,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
@@ -1706,7 +1706,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1737,7 +1737,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       all_counts = []
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         all_counts.extend(results[0])
@@ -1749,7 +1749,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         self.assertAllEqual(results[2], [b"string"] * batch_size)
 
       # Reached the final batch with extra elements.
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertAllEqual(results[0].shape, [extra_elements])
       self.assertAllEqual(results[1].dense_shape, [extra_elements, 1])
       self.assertAllEqual(results[2], [b"string"] * extra_elements)
@@ -1764,7 +1764,7 @@ class ShuffleBatchTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1817,14 +1817,14 @@ class ShuffleBatchTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -1990,7 +1990,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched_fetch)
+        results = self.evaluate(batched_fetch)
         self.assertEqual(3, len(results))
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -2020,7 +2020,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched_fetch)
+        self.evaluate(batched_fetch)
       for thread in threads:
         thread.join()
 
@@ -2082,7 +2082,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       saw_both = 0
       num_batches = (num_a + num_b) // batch_size
       for i in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         tf_logging.info("Batch %d: %s", i, results[0])
         self.assertEqual(len(results[0]), batch_size)
         self.assertEqual(len(results[2]), batch_size)
@@ -2102,7 +2102,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
                             [results[0][i] for i in which_b])
 
       # Reached end with 2 * extra_elements left
-      results = sess.run(batched)
+      results = self.evaluate(batched)
       self.assertEqual(len(results[0]), 2 * extra_elements)
       self.assertAllEqual(results[1].dense_shape, [2 * extra_elements, 1])
       self.assertEqual(len(results[2]), 2 * extra_elements)
@@ -2129,7 +2129,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
@@ -2203,14 +2203,14 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       threads = queue_runner_impl.start_queue_runners()
 
       for _ in range(num_batches):
-        results = sess.run(batched)
+        results = self.evaluate(batched)
         self.assertAllEqual([0] * batch_size, np.mod(results[0], 2))
         self.assertAllEqual([0] * batch_size, np.mod(results[1].values, 2))
         self.assertAllEqual([b"string"] * batch_size, results[2])
 
       # Reached the limit.
       with self.assertRaises(errors_impl.OutOfRangeError):
-        sess.run(batched)
+        self.evaluate(batched)
       for thread in threads:
         thread.join()
 
diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py
index 03a32f6ca0..9c31c0924f 100644
--- a/tensorflow/python/training/learning_rate_decay_test.py
+++ b/tensorflow/python/training/learning_rate_decay_test.py
@@ -62,23 +62,22 @@ class LRDecayTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
 
   def testVariables(self):
-    with self.cached_session():
-      step = variables.VariableV1(1)
-      assign_1 = step.assign(1)
-      assign_2 = step.assign(2)
-      assign_100 = step.assign(100)
-      decayed_lr = learning_rate_decay.exponential_decay(.1, step, 3, 0.96,
-                                                         staircase=True)
-      variables.global_variables_initializer().run()
-      # No change to learning rate
-      assign_1.op.run()
-      self.assertAllClose(decayed_lr.eval(), .1, 1e-6)
-      assign_2.op.run()
-      self.assertAllClose(decayed_lr.eval(), .1, 1e-6)
-      # Decayed learning rate
-      assign_100.op.run()
-      expected = .1 * 0.96 ** (100 // 3)
-      self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
+    step = variables.VariableV1(1)
+    assign_1 = step.assign(1)
+    assign_2 = step.assign(2)
+    assign_100 = step.assign(100)
+    decayed_lr = learning_rate_decay.exponential_decay(
+        .1, step, 3, 0.96, staircase=True)
+    self.evaluate(variables.global_variables_initializer())
+    # No change to learning rate
+    self.evaluate(assign_1.op)
+    self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6)
+    self.evaluate(assign_2.op)
+    self.assertAllClose(self.evaluate(decayed_lr), .1, 1e-6)
+    # Decayed learning rate
+    self.evaluate(assign_100.op)
+    expected = .1 * 0.96**(100 // 3)
+    self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
 
   @test_util.run_in_graph_and_eager_modes
   def testPiecewiseConstant(self):
diff --git a/tensorflow/python/training/learning_rate_decay_v2_test.py b/tensorflow/python/training/learning_rate_decay_v2_test.py
index b2ac93f06f..354ddb25be 100644
--- a/tensorflow/python/training/learning_rate_decay_v2_test.py
+++ b/tensorflow/python/training/learning_rate_decay_v2_test.py
@@ -62,23 +62,22 @@ class LRDecayTestV2(test_util.TensorFlowTestCase):
       self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
 
   def testVariables(self):
-    with self.cached_session():
-      step = variables.Variable(1)
-      assign_1 = step.assign(1)
-      assign_2 = step.assign(2)
-      assign_100 = step.assign(100)
-      decayed_lr = learning_rate_decay_v2.exponential_decay(.1, step, 3, 0.96,
-                                                            staircase=True)
-      variables.global_variables_initializer().run()
-      # No change to learning rate
-      assign_1.op.run()
-      self.assertAllClose(decayed_lr().eval(), .1, 1e-6)
-      assign_2.op.run()
-      self.assertAllClose(decayed_lr().eval(), .1, 1e-6)
-      # Decayed learning rate
-      assign_100.op.run()
-      expected = .1 * 0.96 ** (100 // 3)
-      self.assertAllClose(decayed_lr().eval(), expected, 1e-6)
+    step = variables.Variable(1)
+    assign_1 = step.assign(1)
+    assign_2 = step.assign(2)
+    assign_100 = step.assign(100)
+    decayed_lr = learning_rate_decay_v2.exponential_decay(
+        .1, step, 3, 0.96, staircase=True)
+    self.evaluate(variables.global_variables_initializer())
+    # No change to learning rate
+    self.evaluate(assign_1.op)
+    self.assertAllClose(self.evaluate(decayed_lr()), .1, 1e-6)
+    self.evaluate(assign_2.op)
+    self.assertAllClose(self.evaluate(decayed_lr()), .1, 1e-6)
+    # Decayed learning rate
+    self.evaluate(assign_100.op)
+    expected = .1 * 0.96**(100 // 3)
+    self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
 
   @test_util.run_in_graph_and_eager_modes
   def testPiecewiseConstant(self):
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index b828be4499..2ceb387ec3 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -1170,7 +1170,7 @@ class HookedSessionTest(test.TestCase):
       mock_run = FakeSession(sess)
       mon_sess = monitored_session._HookedSession(sess=mock_run, hooks=[])
       a_tensor = constant_op.constant([0], name='a_tensor')
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       output = mon_sess.run(fetches=a_tensor,
                             feed_dict='a_feed',
                             options='an_option',
@@ -1189,7 +1189,7 @@ class HookedSessionTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(
           sess=sess, hooks=[mock_hook, mock_hook2])
       a_tensor = constant_op.constant([0], name='a_tensor')
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       mon_sess.run(a_tensor)
 
       for hook in [mock_hook, mock_hook2]:
@@ -1214,7 +1214,7 @@ class HookedSessionTest(test.TestCase):
       mon_sess = monitored_session._HookedSession(
           sess=sess, hooks=[mock_hook, mock_hook2])
       constant_op.constant([0], name='a_tensor')
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       mon_sess.run(fetches='a_tensor')
       self.assertFalse(mon_sess.should_stop())
@@ -1234,7 +1234,7 @@ class HookedSessionTest(test.TestCase):
       third_tensor = constant_op.constant([10], name='third_tensor')
       mock_hook.request = session_run_hook.SessionRunArgs([another_tensor])
       mock_hook2.request = session_run_hook.SessionRunArgs([third_tensor])
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       output = mon_sess.run(fetches=a_tensor)
       self.assertEqual(output, [0])
@@ -1254,7 +1254,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       self.assertEqual(mon_sess.run(fetches=add_tensor), [15])
 
@@ -1272,7 +1272,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       feed_dict = {c_tensor: [20]}
       self.assertEqual(
@@ -1293,7 +1293,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={a_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'):
         mon_sess.run(fetches=add_tensor)
@@ -1311,7 +1311,7 @@ class HookedSessionTest(test.TestCase):
           None, feed_dict={a_tensor: [5]})
       mock_hook2.request = session_run_hook.SessionRunArgs(
           None, feed_dict={b_tensor: [10]})
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
 
       with self.assertRaisesRegexp(RuntimeError, 'Same tensor is fed'):
         mon_sess.run(fetches=add_tensor, feed_dict={b_tensor: [10]})
diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py
index 8009e3c24e..6ce5de6663 100644
--- a/tensorflow/python/training/moving_averages_test.py
+++ b/tensorflow/python/training/moving_averages_test.py
@@ -274,14 +274,14 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual([], v1_avg.value().op.control_inputs)
       self.assertEqual([], v1_avg.value().op.control_inputs)
       # We should be able to initialize v1_avg before v0.
-      sess.run(v1_avg.initializer)
-      sess.run(v0.initializer)
-      self.assertEqual([10.0], sess.run(v1_avg))
+      self.evaluate(v1_avg.initializer)
+      self.evaluate(v0.initializer)
+      self.assertEqual([10.0], self.evaluate(v1_avg))
       # running ema_op should add to v0 (in addition to updating v1_avg)
-      sess.run(assign_to_v1)
-      sess.run(ema_op)
-      self.assertEqual(1, sess.run(v0))
-      self.assertEqual([17.5], sess.run(v1_avg))
+      self.evaluate(assign_to_v1)
+      self.evaluate(ema_op)
+      self.assertEqual(1, self.evaluate(v0))
+      self.assertEqual([17.5], self.evaluate(v1_avg))
 
   @test_util.run_in_graph_and_eager_modes
   def testBasicEager(self):
diff --git a/tensorflow/python/training/proximal_adagrad_test.py b/tensorflow/python/training/proximal_adagrad_test.py
index 272f9019e7..9d46a6682d 100644
--- a/tensorflow/python/training/proximal_adagrad_test.py
+++ b/tensorflow/python/training/proximal_adagrad_test.py
@@ -48,7 +48,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([0.0, 0.0], v0_val)
       self.assertAllClose([0.0, 0.0], v1_val)
 
@@ -56,7 +56,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       for _ in range(3):
         update.run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([-2.60260963, -4.29698515]), v0_val)
       self.assertAllClose(np.array([-0.28432083, -0.56694895]), v1_val)
       opt_vars = opt.variables()
@@ -85,14 +85,14 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([1.0, 2.0], v0_val)
       self.assertAllClose([4.0, 3.0], v1_val)
 
       # Run 3 steps Proximal Adagrad.
       for _ in range(3):
         update.run()
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([-1.60261, -2.296985]), v0_val)
       self.assertAllClose(np.array([3.715679, 2.433051]), v1_val)
 
@@ -129,14 +129,14 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([1.0, 2.0], v0_val)
       self.assertAllClose([4.0, 3.0], v1_val)
 
       # Run 10 steps Proximal Adagrad
       for _ in range(10):
         update.run()
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([-6.663634, -9.190331]), v0_val)
       self.assertAllClose(np.array([2.959304, 1.029232]), v1_val)
 
@@ -155,7 +155,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([1.0, 2.0], v0_val)
       self.assertAllClose([4.0, 3.0], v1_val)
 
@@ -163,7 +163,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       for _ in range(10):
         update.run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([-0.0495, -0.0995]), v0_val)
       self.assertAllClose(np.array([-0.0045, -0.0095]), v1_val)
 
@@ -191,7 +191,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
     variables.global_variables_initializer().run()
 
     sess = ops.get_default_session()
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     if is_sparse:
       self.assertAllClose([[1.0], [2.0]], v0_val)
       self.assertAllClose([[3.0], [4.0]], v1_val)
@@ -203,7 +203,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
     for _ in range(steps):
       update.run()
 
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     return v0_val, v1_val
 
   def testEquivAdagradwithoutRegularization(self):
diff --git a/tensorflow/python/training/proximal_gradient_descent_test.py b/tensorflow/python/training/proximal_gradient_descent_test.py
index a9355f4824..8797b308eb 100644
--- a/tensorflow/python/training/proximal_gradient_descent_test.py
+++ b/tensorflow/python/training/proximal_gradient_descent_test.py
@@ -50,7 +50,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([0.0, 0.0], v0_val)
       self.assertAllClose([0.0, 0.0], v1_val)
 
@@ -58,7 +58,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       for _ in range(3):
         update.run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([-0.9, -1.8]), v0_val)
       self.assertAllClose(np.array([-0.09, -0.18]), v1_val)
 
@@ -80,7 +80,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([1.0, 2.0], v0_val)
       self.assertAllClose([4.0, 3.0], v1_val)
 
@@ -88,7 +88,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       for _ in range(3):
         update.run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([0.1, 0.2]), v0_val)
       self.assertAllClose(np.array([3.91, 2.82]), v1_val)
 
@@ -123,7 +123,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
       variables.global_variables_initializer().run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose([1.0, 2.0], v0_val)
       self.assertAllClose([4.0, 3.0], v1_val)
 
@@ -131,7 +131,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       for _ in range(10):
         update.run()
 
-      v0_val, v1_val = sess.run([var0, var1])
+      v0_val, v1_val = self.evaluate([var0, var1])
       self.assertAllClose(np.array([-0.0495, -0.0995]), v0_val)
       self.assertAllClose(np.array([-0.0045, -0.0095]), v1_val)
 
@@ -159,7 +159,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
     variables.global_variables_initializer().run()
 
     sess = ops.get_default_session()
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     if is_sparse:
       self.assertAllClose([[1.0], [2.0]], v0_val)
       self.assertAllClose([[3.0], [4.0]], v1_val)
@@ -171,7 +171,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
     for _ in range(steps):
       update.run()
 
-    v0_val, v1_val = sess.run([var0, var1])
+    v0_val, v1_val = self.evaluate([var0, var1])
     return v0_val, v1_val
 
   def testEquivSparseGradientDescentwithoutRegularization(self):
diff --git a/tensorflow/python/training/quantize_training_test.py b/tensorflow/python/training/quantize_training_test.py
index 6edbf7665f..07fd488563 100644
--- a/tensorflow/python/training/quantize_training_test.py
+++ b/tensorflow/python/training/quantize_training_test.py
@@ -73,11 +73,11 @@ class PywrapQuantizeTrainingTest(test.TestCase):
       _ = importer.import_graph_def(result, name='')
 
       # Initialize the variable.
-      sess.run(g.get_operation_by_name(init_op.name))
+      self.evaluate(g.get_operation_by_name(init_op.name))
 
       # Run the graph for one step to assign values to the quantization min/max
       # variables.
-      sess.run(g.get_tensor_by_name(c.name))
+      self.evaluate(g.get_tensor_by_name(c.name))
 
       saver.save(sess, save_path)
 
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index be49e6e715..6b2177b0bb 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -227,7 +227,7 @@ class SaverTest(test.TestCase):
         w1 = resource_variable_ops.ResourceVariable(1.0, name="w1")
         w2 = resource_variable_ops.ResourceVariable(2.0, name="w2")
         graph_saver = saver_module.Saver([w1, w2])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         graph_saver.save(sess, graph_ckpt_prefix)
 
     with context.eager_mode():
@@ -260,7 +260,7 @@ class SaverTest(test.TestCase):
         w3 = resource_variable_ops.ResourceVariable(0.0, name="w3")
         w4 = resource_variable_ops.ResourceVariable(0.0, name="w4")
         graph_saver = saver_module.Saver([w3, w4])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         graph_saver.restore(sess, eager_ckpt_prefix)
         self.assertAllEqual(w3.eval(), 3.0)
         self.assertAllEqual(w4.eval(), 4.0)
@@ -326,7 +326,7 @@ class SaverTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initialize all variables
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
 
       # Check that the parameter nodes have been initialized.
       self.assertEqual(10.0, v0.eval())
@@ -376,7 +376,7 @@ class SaverTest(test.TestCase):
     with self.cached_session() as sess:
       tensor = sess.graph.get_tensor_by_name(
           save.saver_def.filename_tensor_name)
-      self.assertEqual(sess.run(tensor), filename)
+      self.assertEqual(self.evaluate(tensor), filename)
 
   def testInvalidPath(self):
     v0 = variables.VariableV1(0, name="v0")
@@ -407,7 +407,7 @@ class SaverTest(test.TestCase):
 
       with self.assertRaisesWithPredicateMatch(
           errors_impl.OpError, lambda e: "uninitialized value v" in e.message):
-        sess.run(v)
+        self.evaluate(v)
 
       # Restore the saved values in the parameter nodes.
       save.restore(sess, save_path)
@@ -497,10 +497,10 @@ class SaverTest(test.TestCase):
 
       with self.assertRaisesWithPredicateMatch(
           errors_impl.OpError, lambda e: "uninitialized value v0" in e.message):
-        sess.run(v0)
+        self.evaluate(v0)
       with self.assertRaisesWithPredicateMatch(
           errors_impl.OpError, lambda e: "uninitialized value v1" in e.message):
-        sess.run(v1)
+        self.evaluate(v1)
       self.assertEqual(0, len(v2.keys().eval()))
       self.assertEqual(0, len(v2.values().eval()))
 
@@ -742,7 +742,7 @@ class SaverTest(test.TestCase):
       try:
         with self.cached_session() as sess:
           # Initialize all variables
-          sess.run(init_all_op)
+          self.evaluate(init_all_op)
 
           # Check that the parameter nodes have been initialized.
           self.assertEqual(10.0, v0.eval())
@@ -777,7 +777,7 @@ class SaverTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initialize all variables
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
 
       # Check that the parameter nodes have been initialized.
       self.assertEqual(10.0, v0.eval())
@@ -824,11 +824,11 @@ class SaverTest(test.TestCase):
     save_graph = ops_lib.Graph()
     with save_graph.as_default(), self.session(graph=save_graph) as sess:
       orig_vars = _model()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       save = saver_module.Saver(max_to_keep=1)
       variables.global_variables_initializer().run()
       save.save(sess, save_dir)
-      orig_vals = sess.run(orig_vars)
+      orig_vals = self.evaluate(orig_vars)
 
     restore_graph = ops_lib.Graph()
     with restore_graph.as_default(), self.session(
@@ -836,7 +836,7 @@ class SaverTest(test.TestCase):
       restored_vars = _model()
       save = saver_module.Saver(max_to_keep=1)
       save.restore(sess, save_dir)
-      restored_vals = sess.run(restored_vars)
+      restored_vals = self.evaluate(restored_vars)
 
     for orig, restored in zip(orig_vals, restored_vals):
       self.assertAllEqual(orig, restored)
@@ -1747,7 +1747,7 @@ class MetaGraphTest(test.TestCase):
       self.assertEqual([], v1.get_shape())
       with self.assertRaisesWithPredicateMatch(
           errors_impl.OpError, lambda e: "uninitialized value v1" in e.message):
-        sess.run(v1)
+        self.evaluate(v1)
       # Retrieves saver1. Verifies that new_saver1 can restore v1.
       new_saver1 = savers[1]
       new_saver1.restore(sess, saver1_ckpt)
@@ -1927,9 +1927,9 @@ class MetaGraphTest(test.TestCase):
 
     with self.cached_session() as sess:
       # Initializes all the variables.
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
       # Runs to logit.
-      sess.run(logits)
+      self.evaluate(logits)
       # Creates a saver.
       saver0 = saver_module.Saver()
       saver0.save(sess, saver0_ckpt)
@@ -1969,7 +1969,7 @@ class MetaGraphTest(test.TestCase):
       ops_lib.add_to_collection("train_op", train_op)
 
       # Runs train_op.
-      sess.run(train_op)
+      self.evaluate(train_op)
 
       # Generates MetaGraphDef.
       saver_module.export_meta_graph(train_filename)
@@ -1983,7 +1983,7 @@ class MetaGraphTest(test.TestCase):
       # Restores from checkpoint.
       new_saver.restore(sess, saver0_ckpt)
       train_op = ops_lib.get_collection("train_op")[0]
-      sess.run(train_op)
+      self.evaluate(train_op)
 
   def testGraphExtension(self):
     test_dir = self._get_test_dir("graph_extension")
@@ -2015,8 +2015,8 @@ class MetaGraphTest(test.TestCase):
 
       # Generate a MetaGraphDef containing the while loop.
       with session.Session() as sess:
-        sess.run(init_op)
-        sess.run(output)
+        self.evaluate(init_op)
+        self.evaluate(output)
         saver = saver_module.Saver()
         saver.save(sess, saver_ckpt)
         saver.export_meta_graph(filename)
@@ -2031,8 +2031,8 @@ class MetaGraphTest(test.TestCase):
       no_constfold_config.graph_options.rewrite_options.constant_folding = (
           rewriter_config_pb2.RewriterConfig.OFF)
       with session.Session(config=no_constfold_config) as sess:
-        sess.run(init_op)
-        expected_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        expected_grad_value = self.evaluate(grad)
 
     # Restore the MetaGraphDef into a new Graph.
     with ops_lib.Graph().as_default():
@@ -2048,8 +2048,8 @@ class MetaGraphTest(test.TestCase):
       init_op = variables.global_variables_initializer()
 
       with session.Session(config=no_constfold_config) as sess:
-        sess.run(init_op)
-        actual_grad_value = sess.run(grad)
+        self.evaluate(init_op)
+        actual_grad_value = self.evaluate(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
   def _testWhileLoopAndGradientSerDes(self, outer_body_fn):
@@ -2187,7 +2187,7 @@ class MetaGraphTest(test.TestCase):
                                                       logits=logit, name="cost")
       adam.AdamOptimizer().minimize(cost, name="optimize")
       saver = saver_module.Saver()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver.save(sess, filename)
 
     graph = ops_lib.Graph()
@@ -2224,7 +2224,7 @@ class MetaGraphTest(test.TestCase):
 
       # Create a variable in graph_2 under scope "my_scope".
       variables.VariableV1(array_ops.zeros([10]), name="my_scope/my_var")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       # Restore the checkpoint into a different scope "subgraph_2".
       new_saver_2 = saver_module.import_meta_graph(
           filename + ".meta", graph=graph_2, import_scope="subgraph_2")
@@ -2257,7 +2257,7 @@ class MetaGraphTest(test.TestCase):
                                                       logits=logit, name="cost")
       adam.AdamOptimizer().minimize(cost, name="optimize")
       saver = saver_module.Saver()
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver.save(sess, filename)
 
     graph = ops_lib.Graph()
@@ -2294,12 +2294,12 @@ class MetaGraphTest(test.TestCase):
           meta_graph_def, clear_devices=False, import_scope="new_model")
       # Device refers to GPU, which is not available here.
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
 
     with session.Session(graph=ops_lib.Graph()) as sess:
       saver_module.import_meta_graph(
           meta_graph_def, clear_devices=True, import_scope="new_model")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(["new_model/optimize"], {
           "new_model/image:0": np.random.random([1, 784]),
           "new_model/label:0": np.random.randint(
@@ -2326,7 +2326,7 @@ class MetaGraphTest(test.TestCase):
 
     with session.Session(graph=ops_lib.Graph()) as sess:
       saver_module.import_meta_graph(meta_graph_def, import_scope="new_model")
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       sess.run(["new_model/optimize"], {
           "new_model/image:0": np.random.random([1, 784]),
           "new_model/label:0": np.random.randint(
@@ -2352,7 +2352,7 @@ class MetaGraphTest(test.TestCase):
                            meta_graph_def_from_graph_def]:
       with session.Session(graph=ops_lib.Graph()) as sess:
         saver_module.import_meta_graph(meta_graph_def, import_scope="new_model")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         for i in range(10):
           self.assertEqual(i * i, sess.run("new_model/output:0"))
         with self.assertRaises(errors.OutOfRangeError):
@@ -2378,7 +2378,7 @@ class CheckpointReaderTest(test.TestCase):
     save_path = os.path.join(self.get_temp_dir(),
                              "ckpt_for_debug_string" + str(self._WRITE_VERSION))
     with self.cached_session() as sess:
-      sess.run(init_all_op)
+      self.evaluate(init_all_op)
       # Saves a checkpoint.
       save.save(sess, save_path)
 
@@ -2524,7 +2524,7 @@ class ScopedGraphTest(test.TestCase):
       self.assertEqual(["biases:0", "weights:0"], sorted(var_list.keys()))
 
     with self.session(graph=graph) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       saver = saver_module.Saver(var_list=var_list, max_to_keep=1)
       saver.save(sess, os.path.join(test_dir, ckpt_filename), write_state=False)
 
@@ -2587,10 +2587,10 @@ class ScopedGraphTest(test.TestCase):
       saver = saver_module.Saver(var_list=var_list, max_to_keep=1)
       saver.restore(sess, os.path.join(test_dir, ckpt_filename))
       # Verify that we have restored weights1 and biases1.
-      sess.run([weights1, biases1])
+      self.evaluate([weights1, biases1])
       # Initialize the rest of the variables and run logits.
-      sess.run(init_rest_op)
-      sess.run(logits)
+      self.evaluate(init_rest_op)
+      self.evaluate(logits)
 
   # Verifies that we can save the subgraph under "hidden1" and restore it
   # into "new_hidden1" in the new graph.
@@ -2618,7 +2618,7 @@ class ScopedGraphTest(test.TestCase):
 
     # Run the graph and save scoped checkpoint.
     with self.session(graph=graph1) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       _, var_list_1 = meta_graph.export_scoped_meta_graph(
           export_scope="hidden1")
       saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1)
@@ -2674,7 +2674,7 @@ class ScopedGraphTest(test.TestCase):
 
     # Run the graph and save scoped checkpoint.
     with self.session(graph=graph1) as sess:
-      sess.run(variables.global_variables_initializer())
+      self.evaluate(variables.global_variables_initializer())
       _, var_list_1 = meta_graph.export_scoped_meta_graph(
           graph_def=graph1.as_graph_def(), export_scope="hidden1")
       saver = saver_module.Saver(var_list=var_list_1, max_to_keep=1)
@@ -2942,7 +2942,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
     a_saver = saver_module.Saver([a])
     b_saver = saver_module.Saver([b])
     with self.cached_session() as sess:
-      sess.run(a.initializer)
+      self.evaluate(a.initializer)
       save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix)
       with self.assertRaisesRegexp(
           errors.NotFoundError, "Key b not found in checkpoint"):
@@ -2964,7 +2964,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
       a_saver = saver_module.Saver([a])
 
       with self.session(graph=g) as sess:
-        sess.run(a.initializer)
+        self.evaluate(a.initializer)
         save_path = a_saver.save(sess=sess, save_path=checkpoint_prefix)
 
     with ops_lib.Graph().as_default() as g:
diff --git a/tensorflow/python/training/server_lib_same_variables_clear_container_test.py b/tensorflow/python/training/server_lib_same_variables_clear_container_test.py
index 11e6f28ab0..3a5eb712c6 100644
--- a/tensorflow/python/training/server_lib_same_variables_clear_container_test.py
+++ b/tensorflow/python/training/server_lib_same_variables_clear_container_test.py
@@ -60,9 +60,9 @@ class SameVariablesClearContainerTest(test.TestCase):
     session.Session.reset(server0.target, ["local0"])
     sess = session.Session(server0.target)
     with self.assertRaises(errors_impl.FailedPreconditionError):
-      sess.run(v0)
+      self.evaluate(v0)
     # Reinitializes v0 for the following test.
-    sess.run(v0.initializer)
+    self.evaluate(v0.initializer)
 
     # Verifies that v1 is still valid.
     self.assertAllEqual(2.0, sess_1.run(v1))
@@ -71,10 +71,10 @@ class SameVariablesClearContainerTest(test.TestCase):
     session.Session.reset(server1.target, ["local1"])
     sess = session.Session(server1.target)
     with self.assertRaises(errors_impl.FailedPreconditionError):
-      sess.run(v1)
+      self.evaluate(v1)
     # Verifies that v0 is still valid.
     sess = session.Session(server0.target)
-    self.assertAllEqual(1.0, sess.run(v0))
+    self.assertAllEqual(1.0, self.evaluate(v0))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/server_lib_sparse_job_test.py b/tensorflow/python/training/server_lib_sparse_job_test.py
index 1a6b44b90e..8c2745b51a 100644
--- a/tensorflow/python/training/server_lib_sparse_job_test.py
+++ b/tensorflow/python/training/server_lib_sparse_job_test.py
@@ -36,7 +36,7 @@ class SparseJobTest(test.TestCase):
       a = constant_op.constant(1.0)
 
     with session.Session(server.target) as sess:
-      self.assertEqual(1.0, sess.run(a))
+      self.assertEqual(1.0, self.evaluate(a))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py
index b734e9653e..9dc88d78cc 100644
--- a/tensorflow/python/training/supervisor_test.py
+++ b/tensorflow/python/training/supervisor_test.py
@@ -100,7 +100,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir=logdir)
       sess = sv.prepare_or_wait_for_session("")
       for _ in xrange(10):
-        sess.run(my_op)
+        self.evaluate(my_op)
       sess.close()
       sv.stop()
 
@@ -111,7 +111,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir=logdir)
       with sv.managed_session("") as sess:
         for _ in xrange(10):
-          sess.run(my_op)
+          self.evaluate(my_op)
       # Supervisor has been stopped.
       self.assertTrue(sv.should_stop())
 
@@ -128,7 +128,7 @@ class SupervisorTest(test.TestCase):
             if step == 1:
               raise RuntimeError("failing here")
             else:
-              sess.run(my_op)
+              self.evaluate(my_op)
       # Supervisor has been stopped.
       self.assertTrue(sv.should_stop())
       self.assertEqual(1, last_step)
@@ -146,7 +146,7 @@ class SupervisorTest(test.TestCase):
             raise errors_impl.OutOfRangeError(my_op.op.node_def, my_op.op,
                                               "all done")
           else:
-            sess.run(my_op)
+            self.evaluate(my_op)
       # Supervisor has been stopped.  OutOfRangeError was not thrown.
       self.assertTrue(sv.should_stop())
       self.assertEqual(3, last_step)
@@ -335,7 +335,7 @@ class SupervisorTest(test.TestCase):
       sess = sv.prepare_or_wait_for_session(
           "", config=config_pb2.ConfigProto(device_count={"CPU": 2}))
       for _ in xrange(10):
-        sess.run(my_op)
+        self.evaluate(my_op)
       sess.close()
       sv.stop()
 
diff --git a/tensorflow/python/training/warm_starting_util_test.py b/tensorflow/python/training/warm_starting_util_test.py
index b575b8d364..fa1f370f41 100644
--- a/tensorflow/python/training/warm_starting_util_test.py
+++ b/tensorflow/python/training/warm_starting_util_test.py
@@ -49,7 +49,7 @@ class WarmStartingUtilTest(test.TestCase):
     return vocab_file
 
   def _write_checkpoint(self, sess):
-    sess.run(variables.global_variables_initializer())
+    self.evaluate(variables.global_variables_initializer())
     saver = saver_lib.Saver()
     ckpt_prefix = os.path.join(self.get_temp_dir(), "model")
     saver.save(sess, ckpt_prefix, global_step=0)
@@ -70,7 +70,7 @@ class WarmStartingUtilTest(test.TestCase):
         if partitioner:
           self.assertTrue(isinstance(var, variables.PartitionedVariable))
           var = var._get_variable_list()
-        return var, sess.run(var)
+        return var, self.evaluate(var)
 
   def _create_prev_run_vars(self,
                             var_names,
@@ -86,7 +86,7 @@ class WarmStartingUtilTest(test.TestCase):
               shape=shape,
               initializer=initializer))
         self._write_checkpoint(sess)
-        return [sess.run(var) for var in all_vars]
+        return [self.evaluate(var) for var in all_vars]
 
   def _create_dummy_inputs(self):
     return {
@@ -125,7 +125,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose(prev_val, fruit_weights.eval(sess))
 
   def testWarmStartVarPrevVarPartitioned(self):
@@ -143,7 +143,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose(prev_val, fruit_weights.eval(sess))
 
   def testWarmStartVarCurrentVarPartitioned(self):
@@ -162,7 +162,7 @@ class WarmStartingUtilTest(test.TestCase):
         prev_tensor_name, var = ws_util._get_var_info(fruit_weights)
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         fruit_weights = fruit_weights._get_variable_list()
         new_val = np.concatenate(
             [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
@@ -189,7 +189,7 @@ class WarmStartingUtilTest(test.TestCase):
             fruit_weights, prev_tensor_name="old_scope/fruit_weights")
         checkpoint_utils.init_from_checkpoint(self.get_temp_dir(),
                                               {prev_tensor_name: var})
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         fruit_weights = fruit_weights._get_variable_list()
         new_val = np.concatenate(
             [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
@@ -211,7 +211,7 @@ class WarmStartingUtilTest(test.TestCase):
             "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5,
                                            self.get_temp_dir(), prev_vocab_path)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
 
@@ -236,7 +236,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.],
                              [2.3, 2., 0.]], fruit_output_layer.eval(sess))
 
@@ -261,7 +261,7 @@ class WarmStartingUtilTest(test.TestCase):
             self.get_temp_dir(),
             prev_vocab_path,
             previous_vocab_size=2)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Old vocabulary limited to ['apple', 'banana'].
         self.assertAllClose([[0.], [0.], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
@@ -285,7 +285,7 @@ class WarmStartingUtilTest(test.TestCase):
             "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 5,
                                            self.get_temp_dir(), prev_vocab_path)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[2.], [1.5], [1.], [0.5], [0.]],
                             fruit_weights.eval(sess))
 
@@ -312,7 +312,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertAllClose([[0.3, 0.5, 0.], [0.8, 1.0, 0.], [1.2, 1.5, 0.],
                              [2.3, 2., 0.]], fruit_output_layer.eval(sess))
 
@@ -340,7 +340,7 @@ class WarmStartingUtilTest(test.TestCase):
             self.get_temp_dir(),
             prev_vocab_path,
             current_oov_buckets=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_weights, variables.PartitionedVariable))
         fruit_weights_vars = fruit_weights._get_variable_list()
@@ -372,7 +372,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_output_layer, variables.PartitionedVariable))
         fruit_output_layer_vars = fruit_output_layer._get_variable_list()
@@ -404,7 +404,7 @@ class WarmStartingUtilTest(test.TestCase):
             partitioner=lambda shape, dtype: [2, 1])
         ws_util._warm_start_var_with_vocab(fruit_weights, new_vocab_path, 6,
                                            self.get_temp_dir(), prev_vocab_path)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_weights, variables.PartitionedVariable))
         fruit_weights_vars = fruit_weights._get_variable_list()
@@ -438,7 +438,7 @@ class WarmStartingUtilTest(test.TestCase):
                                            prev_ckpt=self.get_temp_dir(),
                                            prev_vocab_path=prev_vocab_path,
                                            axis=1)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         self.assertTrue(
             isinstance(fruit_output_layer, variables.PartitionedVariable))
         fruit_output_layer_vars = fruit_output_layer._get_variable_list()
@@ -463,7 +463,7 @@ class WarmStartingUtilTest(test.TestCase):
             shape=[10, 1],
             initializer=zeros())
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=[var])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started (init overridden to ones).
         self.assertAllEqual(var.eval(), prev_int_val)
 
@@ -483,7 +483,7 @@ class WarmStartingUtilTest(test.TestCase):
             shape=[10, 1],
             initializer=zeros())
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=["v1"])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started (init overridden to ones).
         self.assertAllEqual(var.eval(), prev_int_val)
 
@@ -519,7 +519,7 @@ class WarmStartingUtilTest(test.TestCase):
                            # This warm-starts both v1 and v1/Momentum, but only
                            # v2 (and not v2/Momentum).
                            vars_to_warm_start=["v1", "v2[^/]"])
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify the selection of weights were correctly warm-started (init
         # overridden to ones).
         self.assertAllEqual(v1.eval(), prev_v1_val)
@@ -542,7 +542,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_int], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_int: [np.zeros([10, 1])]},
@@ -553,7 +553,7 @@ class WarmStartingUtilTest(test.TestCase):
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_int], partitioner)
         ws_util.warm_start(self.get_temp_dir(), vars_to_warm_start=".*sc_int.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_int: [prev_int_val]}, sess)
 
@@ -571,7 +571,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_hash], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_hash: [np.zeros([15, 1])]},
@@ -583,7 +583,7 @@ class WarmStartingUtilTest(test.TestCase):
         cols_to_vars = self._create_linear_model([sc_hash], partitioner)
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*sc_hash.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_hash: [prev_hash_val]},
                                   sess)
@@ -605,7 +605,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
@@ -619,7 +619,7 @@ class WarmStartingUtilTest(test.TestCase):
         # vocab is assumed to be same as new vocab.
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*sc_vocab.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
                                   sess)
@@ -641,7 +641,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
@@ -657,7 +657,7 @@ class WarmStartingUtilTest(test.TestCase):
             # Explicitly provide the file prefix instead of just the dir.
             os.path.join(self.get_temp_dir(), "model-0"),
             vars_to_warm_start=".*sc_vocab.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
                                   sess)
@@ -686,7 +686,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([2, 1])]},
@@ -708,7 +708,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab/weights": vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  'banana' isn't in the
         # first two entries of the old vocabulary, so it's newly initialized.
         self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [[[1], [0]]]}, sess)
@@ -729,7 +729,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model([real_bucket], partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, the weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars,
@@ -741,7 +741,7 @@ class WarmStartingUtilTest(test.TestCase):
         cols_to_vars = self._create_linear_model([real_bucket], partitioner)
         ws_util.warm_start(
             self.get_temp_dir(), vars_to_warm_start=".*real_bucketized.*")
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars,
                                   {real_bucket: [prev_bucket_val]}, sess)
@@ -800,7 +800,7 @@ class WarmStartingUtilTest(test.TestCase):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
         cols_to_vars = self._create_linear_model(all_linear_cols, partitioner)
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Without warm-starting, all weights should be initialized using default
         # initializer (which is init_ops.zeros_initializer).
         self._assert_cols_to_vars(cols_to_vars, {
@@ -826,7 +826,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab/weights": vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.
         self._assert_cols_to_vars(cols_to_vars, {
             sc_int: [prev_int_val],
@@ -865,7 +865,7 @@ class WarmStartingUtilTest(test.TestCase):
             "linear_model/sc_vocab/weights",
             initializer=[[0.5], [1.], [2.], [3.]])
         self._write_checkpoint(sess)
-        prev_keys_val = sess.run(sc_keys_weights)
+        prev_keys_val = self.evaluate(sc_keys_weights)
 
     def _partitioner(shape, dtype):  # pylint:disable=unused-argument
       # Partition each var into 2 equal slices.
@@ -892,7 +892,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_hash should not be warm-started.  Var corresponding to sc_vocab
         # should be correctly warm-started after vocab remapping.
@@ -933,7 +933,7 @@ class WarmStartingUtilTest(test.TestCase):
             "linear_model/sc_vocab/weights",
             initializer=[[0.5], [1.], [2.], [3.]])
         self._write_checkpoint(sess)
-        prev_keys_val = sess.run(sc_keys_weights)
+        prev_keys_val = self.evaluate(sc_keys_weights)
 
     # New graph, new session with warm-starting.
     with ops.Graph().as_default() as g:
@@ -955,7 +955,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_hash should not be warm-started.  Var corresponding to sc_vocab
         # should be correctly warm-started after vocab remapping.
@@ -1024,7 +1024,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[sc_keys]):
                     "some_other_name"
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started.  Var corresponding to
         # sc_vocab should be correctly warm-started after vocab remapping,
         # and neither of the other two should be warm-started..
@@ -1091,7 +1091,7 @@ class WarmStartingUtilTest(test.TestCase):
                 ws_util._infer_var_name(cols_to_vars[emb_vocab_column]):
                     vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started. Var corresponding to
         # emb_vocab_column should be correctly warm-started after vocab
         # remapping. Missing values are filled in with the EmbeddingColumn's
@@ -1163,7 +1163,7 @@ class WarmStartingUtilTest(test.TestCase):
             var_name_to_vocab_info={
                 "linear_model/sc_vocab_embedding/embedding_weights": vocab_info
             })
-        sess.run(variables.global_variables_initializer())
+        self.evaluate(variables.global_variables_initializer())
         # Verify weights were correctly warm-started. Var corresponding to
         # emb_vocab should be correctly warm-started after vocab remapping.
         # Missing values are filled in with the EmbeddingColumn's initializer.
-- 
GitLab


From e12ae7b521b47c9c33bcbf5e76dc99d4b309b9df Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Tue, 27 Nov 2018 10:06:35 -0800
Subject: [PATCH 0868/1554] [XLA] Remove interior padding from padded
 dimensions with one-sized operand dimnesions.

PiperOrigin-RevId: 223010061
---
 .../xla/service/algebraic_simplifier.cc       | 21 ++++++++++++
 .../xla/service/algebraic_simplifier_test.cc  | 34 +++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 0b0502b4ff..a9099d9e31 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1671,6 +1671,27 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) {
         pad, HloInstruction::CreateBroadcast(pad->shape(),
                                              pad->mutable_operand(1), {}));
   }
+
+  // Interior padding on one sized dimensions have no effect. As a result it
+  // makes other simplifications possible if there is no interior padding.
+  if (HasInteriorPadding(pad->padding_config())) {
+    PaddingConfig padding_config = pad->padding_config();
+    bool cleared_interior_padding = false;
+    for (int64 i = 0; i < ShapeUtil::Rank(pad->shape()); ++i) {
+      if (padding_config.dimensions(i).interior_padding() > 0 &&
+          pad->operand(0)->shape().dimensions(i) == 1) {
+        cleared_interior_padding = true;
+        padding_config.mutable_dimensions(i)->set_interior_padding(0);
+      }
+    }
+    if (cleared_interior_padding) {
+      TF_ASSIGN_OR_RETURN(HloInstruction * pad_without_interior_pad,
+                          MakePadHlo(pad->mutable_operand(0),
+                                     pad->mutable_operand(1), padding_config));
+      return ReplaceInstruction(pad, pad_without_interior_pad);
+    }
+  }
+
   // Eliminate nop pads (padding all zero), and replace a pad with negative
   // padding with a pad with non-negative padding followed by a slice.
   bool all_zero = true;
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 24c35464ad..48f689c96a 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2189,6 +2189,40 @@ TEST_F(AlgebraicSimplifierTest, NegativePadding) {
       has_negative_padding(computation->root_instruction()->operand(0)));
 }
 
+TEST_F(AlgebraicSimplifierTest, TrivialInteriorPadding) {
+  // Verify that a pad instruction with interior padding on one-sized
+  // dimensions, removes the interior padding.
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShape(F32, {2, 1}), "param"));
+  HloInstruction* zero = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0f)));
+  PaddingConfig padding;
+  for (int i = 0; i < 2; ++i) {
+    auto dimension = padding.add_dimensions();
+    dimension->set_edge_padding_low(3);
+    dimension->set_edge_padding_high(3);
+    dimension->set_interior_padding(i * 3);
+  }
+  HloInstruction* pad = builder.AddInstruction(HloInstruction::CreatePad(
+      ShapeUtil::MakeShape(F32, {8, 7}), param, zero, padding));
+
+  auto module = CreateNewVerifiedModule();
+  HloComputation* computation = module->AddEntryComputation(builder.Build());
+
+  AlgebraicSimplifier simplifier(default_options_);
+
+  ASSERT_THAT(computation->root_instruction(), op::Pad(param, zero));
+  ASSERT_TRUE(HasInteriorPadding(pad->padding_config()));
+
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+
+  EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero));
+  EXPECT_FALSE(
+      HasInteriorPadding(computation->root_instruction()->padding_config()));
+}
+
 TEST_F(AlgebraicSimplifierTest, RemoveNoopReshape) {
   HloComputation::Builder builder(TestName());
   HloInstruction* param =
-- 
GitLab


From 1dab0d879218f36c44e88b2d6bf71e819da0ef05 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Tue, 27 Nov 2018 10:14:36 -0800
Subject: [PATCH 0869/1554] Automated rollback of commit
 50e5b015be1ead47f170013a640bf3028bb0560f

PiperOrigin-RevId: 223011511
---
 tensorflow/c/eager/BUILD                      |  1 +
 tensorflow/c/eager/c_api.cc                   | 11 +++---
 .../compiler/jit/build_xla_ops_pass_test.cc   |  8 +---
 .../compiler/jit/create_xla_launch_op_test.cc |  6 +--
 .../mark_for_compilation_pass_test_helper.cc  |  8 +---
 .../jit/partially_decluster_pass_test.cc      |  6 +--
 tensorflow/compiler/jit/xla_cpu_device.cc     | 11 +++---
 tensorflow/compiler/jit/xla_gpu_device.cc     | 10 ++---
 .../compiler/jit/xla_interpreter_device.cc    |  7 ++--
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  2 +-
 tensorflow/core/BUILD                         |  4 ++
 .../collective_executor_mgr_test.cc           |  6 +--
 .../collective_param_resolver_local_test.cc   |  6 +--
 .../collective_rma_local_test.cc              |  6 +--
 .../core/common_runtime/device_factory.cc     | 17 +++++----
 .../core/common_runtime/device_factory.h      | 13 ++++---
 tensorflow/core/common_runtime/device_mgr.cc  | 37 +++++++++++--------
 tensorflow/core/common_runtime/device_mgr.h   | 15 ++++----
 .../device_resolver_local_test.cc             |  6 +--
 .../core/common_runtime/device_set_test.cc    |  2 +-
 .../core/common_runtime/direct_session.cc     |  4 +-
 tensorflow/core/common_runtime/eager/BUILD    |  1 +
 .../eager/kernel_and_device_test.cc           | 14 ++++---
 .../core/common_runtime/executor_test.cc      |  8 ++--
 .../core/common_runtime/function_test.cc      |  7 ++--
 .../function_threadpool_test.cc               |  6 +--
 .../core/common_runtime/gpu/gpu_device.cc     | 20 +++++-----
 .../core/common_runtime/gpu/gpu_device.h      | 18 ++++-----
 .../common_runtime/gpu/gpu_device_factory.cc  | 21 +++++------
 .../gpu/gpu_device_on_non_gpu_machine_test.cc |  2 +-
 .../common_runtime/gpu/gpu_device_test.cc     | 32 +++++++---------
 .../hierarchical_tree_broadcaster_test.cc     | 11 +++---
 .../kernel_benchmark_testlib.cc               |  8 ++--
 .../common_runtime/kernel_benchmark_testlib.h |  2 +-
 tensorflow/core/common_runtime/placer_test.cc |  2 +-
 .../process_function_library_runtime_test.cc  | 18 +++++----
 .../core/common_runtime/renamed_device.cc     | 14 +++----
 .../core/common_runtime/renamed_device.h      |  7 ++--
 .../core/common_runtime/ring_reducer_test.cc  | 11 +++---
 .../threadpool_device_factory.cc              | 17 +++++----
 tensorflow/core/distributed_runtime/BUILD     |  1 +
 ...lective_param_resolver_distributed_test.cc | 11 +++---
 .../collective_rma_distributed_test.cc        | 11 +++---
 .../device_resolver_distributed_test.cc       | 13 ++++---
 .../eager/eager_service_impl.cc               |  6 +--
 .../eager/eager_service_impl_test.cc          |  9 ++---
 .../rpc/grpc_server_lib.cc                    | 11 ++++--
 .../rpc_collective_executor_mgr_test.cc       |  6 +--
 .../core/distributed_runtime/session_mgr.cc   |  4 +-
 .../distributed_runtime/session_mgr_test.cc   |  8 ++--
 .../core/grappler/grappler_item_builder.cc    |  9 +++--
 tensorflow/core/grappler/optimizers/BUILD     |  2 +-
 .../grappler/optimizers/function_optimizer.cc |  9 +++--
 tensorflow/core/kernels/data/BUILD            |  1 +
 tensorflow/core/kernels/data/iterator_ops.cc  |  6 +--
 .../data/single_threaded_executor_test.cc     |  8 ++--
 tensorflow/lite/delegates/flex/BUILD          |  1 +
 .../lite/delegates/flex/delegate_data.cc      |  7 ++--
 tensorflow/lite/toco/import_tensorflow.cc     |  4 +-
 tensorflow/python/client/device_lib.i         |  7 +---
 tensorflow/python/grappler/tf_optimizer.i     |  5 +--
 61 files changed, 273 insertions(+), 271 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index ba3d8533db..5a0988ed31 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -50,6 +50,7 @@ tf_cuda_library(
         ],
         "//conditions:default": [],
     }) + [
+        "@com_google_absl//absl/memory",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/distributed_runtime/eager:eager_client",
         "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 192044915f..c9e730ef41 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/c_api_internal.h"
@@ -80,7 +81,7 @@ tensorflow::Status GetAllRemoteDevices(
     const std::vector<string>& remote_workers,
     tensorflow::WorkerCacheInterface* worker_cache,
     std::unique_ptr<tensorflow::DeviceMgr>* device_mgr) {
-  std::vector<tensorflow::Device*> remote_devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> remote_devices;
   tensorflow::Status status;
   // TODO(nareshmodi) do this in parallel instead of serially.
   for (const string& remote_worker : remote_workers) {
@@ -93,7 +94,7 @@ tensorflow::Status GetAllRemoteDevices(
           status = s;
           if (s.ok()) {
             for (tensorflow::Device* d : *devices) {
-              remote_devices.push_back(d);
+              remote_devices.emplace_back(d);
             }
           }
           n.Notify();
@@ -101,7 +102,7 @@ tensorflow::Status GetAllRemoteDevices(
     n.WaitForNotification();
   }
   std::unique_ptr<tensorflow::DeviceMgr> remote_device_mgr(
-      new tensorflow::DeviceMgr(remote_devices));
+      new tensorflow::DeviceMgr(std::move(remote_devices)));
 
   TF_RETURN_IF_ERROR(status);
 
@@ -262,13 +263,13 @@ TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx,
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
 
 TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   status->status = tensorflow::DeviceFactory::AddDevices(
       opts->session_options.options, "/job:localhost/replica:0/task:0",
       &devices);
   if (!status->status.ok()) return nullptr;
   std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(devices));
+      new tensorflow::DeviceMgr(std::move(devices)));
 
   tensorflow::Rendezvous* r =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
index 11df946cc1..48a23a4c17 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
@@ -42,14 +42,8 @@ class BuildXlaOpsTest : public ::testing::Test {
               .ok());
   }
 
-  void TearDown() override {
-    for (Device* device : devices_) {
-      delete device;
-    }
-  }
-
  private:
-  std::vector<Device*> devices_;
+  std::vector<std::unique_ptr<Device>> devices_;
 };
 
 using ::tensorflow::testing::FindNodeByName;
diff --git a/tensorflow/compiler/jit/create_xla_launch_op_test.cc b/tensorflow/compiler/jit/create_xla_launch_op_test.cc
index 7386660762..0f872a480f 100644
--- a/tensorflow/compiler/jit/create_xla_launch_op_test.cc
+++ b/tensorflow/compiler/jit/create_xla_launch_op_test.cc
@@ -59,8 +59,9 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 1});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices_));
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) {
@@ -69,7 +70,7 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
     lib_def_ = absl::make_unique<FunctionLibraryDefinition>(
         OpRegistry::Global(), proto);
     OptimizerOptions opts;
-    device_mgr_ = absl::make_unique<DeviceMgr>(devices_);
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
     pflr_ = absl::make_unique<ProcessFunctionLibraryRuntime>(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
@@ -77,7 +78,6 @@ class CreateXlaLaunchOpTest : public ::testing::Test {
   }
 
   FunctionLibraryRuntime* flr_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
index d56d0f8ccf..64a3301745 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test_helper.cc
@@ -34,15 +34,9 @@ namespace tensorflow {
   //
   // It may be worth refactoring out XlaOpRegistry::RegisterCompilationDevice to
   // make this more direct, but probably not worth it solely for this test.
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(*session_options, "", &devices));
 
-  auto delete_devices = gtl::MakeCleanup([&] {
-    for (Device* d : devices) {
-      delete d;
-    }
-  });
-
   GraphOptimizationPassOptions opt_options;
   opt_options.graph = graph;
   opt_options.session_options = session_options;
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index 1fc5da5071..38a54cc5ef 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -386,7 +386,7 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   TF_ASSERT_OK(s.ToGraph(graph.get()));
 
   // This is needed to register the XLA_GPU device.
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_ASSERT_OK(DeviceFactory::AddDevices(
       SessionOptions(), "/job:localhost/replica:0/task:0", &devices));
 
@@ -400,10 +400,6 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) {
   TF_ASSERT_OK(PartiallyDecluster(&graph));
 
   EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0");
-
-  for (Device* d : devices) {
-    delete d;
-  }
 }
 
 TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) {
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 9006dd514b..7df898ad12 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -31,12 +31,12 @@ namespace tensorflow {
 class XlaCpuDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 };
 
-Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
-                                          const string& name_prefix,
-                                          std::vector<Device*>* devices) {
+Status XlaCpuDeviceFactory::CreateDevices(
+    const SessionOptions& session_options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   XlaDeviceFlags* flags = GetXlaDeviceFlags();
   bool compile_on_demand = flags->tf_xla_compile_on_demand;
 
@@ -63,8 +63,7 @@ Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_CPU_XLA_JIT;
   options.use_multiple_streams = false;
-  auto device = absl::make_unique<XlaDevice>(session_options, options);
-  devices->push_back(device.release());
+  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 4419701695..944f732b99 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -29,12 +29,12 @@ namespace tensorflow {
 class XlaGpuDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 };
 
-Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
-                                          const string& name_prefix,
-                                          std::vector<Device*>* devices) {
+Status XlaGpuDeviceFactory::CreateDevices(
+    const SessionOptions& session_options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   XlaOpRegistry::DeviceRegistration registration;
   registration.compilation_device_name = DEVICE_GPU_XLA_JIT;
   registration.autoclustering_policy =
@@ -70,7 +70,7 @@ Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& session_options,
       return status;
     }
 
-    devices->push_back(device.release());
+    devices->push_back(std::move(device));
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index e828bae865..4007309ed1 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -33,12 +33,12 @@ constexpr std::array<DataType, 9> kExecAllTypes = {
 class XlaInterpreterDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 };
 
 Status XlaInterpreterDeviceFactory::CreateDevices(
     const SessionOptions& session_options, const string& name_prefix,
-    std::vector<Device*>* devices) {
+    std::vector<std::unique_ptr<Device>>* devices) {
   static XlaDeviceOpRegistrations* registrations = RegisterXlaDeviceKernels(
       DEVICE_XLA_INTERPRETER, DEVICE_INTERPRETER_XLA_JIT);
   (void)registrations;
@@ -61,8 +61,7 @@ Status XlaInterpreterDeviceFactory::CreateDevices(
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT;
   options.use_multiple_streams = false;
-  auto device = absl::make_unique<XlaDevice>(session_options, options);
-  devices->push_back(device.release());
+  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
 
   return Status::OK();
 }
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 7fdd60145f..ee461a3c07 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -380,7 +380,7 @@ XlaCompiler::XlaCompiler(XlaCompiler::Options options)
       initialization_status_(Status::OK()),
       next_step_id_(1),
       device_(new XlaCompilationDevice(SessionOptions(), options_.device_type)),
-      device_mgr_({device_}) {
+      device_mgr_(absl::WrapUnique(device_)) {
   CHECK(!options_.device_type.type_string().empty());
   if (options_.populate_resource_manager) {
     initialization_status_ =
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6821ac7eb7..20b1916e7d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2980,6 +2980,7 @@ tf_cuda_library(
         ":lib_internal",
         ":proto_text",
         ":protos_all_cc",
+        "@com_google_absl//absl/memory",
         "//third_party/eigen3",
         "//tensorflow/core/grappler:grappler_item",
     ] + mkl_deps(),
@@ -3858,6 +3859,7 @@ tf_cc_tests_gpu(
         ":test",
         ":test_main",
         ":testlib",
+        "@com_google_absl//absl/memory",
     ],
 )
 
@@ -3886,6 +3888,7 @@ tf_cc_tests_gpu(
         ":test",
         ":test_main",
         ":testlib",
+        "@com_google_absl//absl/memory",
     ],
 )
 
@@ -4453,6 +4456,7 @@ tf_cc_test(
         "//tensorflow/core/kernels:random_ops",
         "//tensorflow/core/kernels:shape_ops",
         "//third_party/eigen3",
+        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
index 91994c5731..f3d86aa633 100644
--- a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
+++ b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
@@ -38,8 +38,9 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
     auto* device_count = options.config.mutable_device_count();
     string task_name = "/job:localhost/replica:0/task:0";
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     std::unique_ptr<DeviceResolverInterface> drl(
         new DeviceResolverLocal(device_mgr_.get()));
     std::unique_ptr<ParamResolverInterface> prl(
@@ -50,7 +51,6 @@ class CollectiveExecutorMgrTest : public ::testing::Test {
   }
 
   std::unique_ptr<CollectiveExecutorMgr> cme_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
 };
 
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
index 9a501b3298..94d889c40d 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
@@ -37,8 +37,9 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     string task_name = "/job:localhost/replica:0/task:0";
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
     prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
                                                 task_name));
@@ -73,7 +74,6 @@ class CollectiveParamResolverLocalTest : public ::testing::Test {
     }
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
   std::unique_ptr<CollectiveParamResolverLocal> prl_;
diff --git a/tensorflow/core/common_runtime/collective_rma_local_test.cc b/tensorflow/core/common_runtime/collective_rma_local_test.cc
index a931fe64bd..4263f3a4ad 100644
--- a/tensorflow/core/common_runtime/collective_rma_local_test.cc
+++ b/tensorflow/core/common_runtime/collective_rma_local_test.cc
@@ -42,8 +42,9 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
     prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
                                                 kTaskName));
@@ -51,7 +52,6 @@ class CollectiveRemoteAccessLocalTest : public ::testing::Test {
                                                kStepId));
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
   std::unique_ptr<CollectiveParamResolverLocal> prl_;
diff --git a/tensorflow/core/common_runtime/device_factory.cc b/tensorflow/core/common_runtime/device_factory.cc
index b94900114c..0fad13fe1e 100644
--- a/tensorflow/core/common_runtime/device_factory.cc
+++ b/tensorflow/core/common_runtime/device_factory.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
@@ -89,9 +90,9 @@ DeviceFactory* DeviceFactory::GetFactory(const string& device_type) {
   return it->second.factory.get();
 }
 
-Status DeviceFactory::AddDevices(const SessionOptions& options,
-                                 const string& name_prefix,
-                                 std::vector<Device*>* devices) {
+Status DeviceFactory::AddDevices(
+    const SessionOptions& options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   // CPU first. A CPU device is required.
   auto cpu_factory = GetFactory("CPU");
   if (!cpu_factory) {
@@ -116,16 +117,16 @@ Status DeviceFactory::AddDevices(const SessionOptions& options,
   return Status::OK();
 }
 
-Device* DeviceFactory::NewDevice(const string& type,
-                                 const SessionOptions& options,
-                                 const string& name_prefix) {
+std::unique_ptr<Device> DeviceFactory::NewDevice(const string& type,
+                                                 const SessionOptions& options,
+                                                 const string& name_prefix) {
   auto device_factory = GetFactory(type);
   if (!device_factory) {
     return nullptr;
   }
   SessionOptions opt = options;
   (*opt.config.mutable_device_count())[type] = 1;
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(device_factory->CreateDevices(opt, name_prefix, &devices));
   int expected_num_devices = 1;
   auto iter = options.config.device_count().find(type);
@@ -133,7 +134,7 @@ Device* DeviceFactory::NewDevice(const string& type,
     expected_num_devices = iter->second;
   }
   DCHECK_EQ(devices.size(), static_cast<size_t>(expected_num_devices));
-  return devices[0];
+  return std::move(devices[0]);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/device_factory.h b/tensorflow/core/common_runtime/device_factory.h
index db50226fe8..b3cd7adca9 100644
--- a/tensorflow/core/common_runtime/device_factory.h
+++ b/tensorflow/core/common_runtime/device_factory.h
@@ -40,18 +40,19 @@ class DeviceFactory {
   // CPU devices are added first.
   static Status AddDevices(const SessionOptions& options,
                            const string& name_prefix,
-                           std::vector<Device*>* devices);
+                           std::vector<std::unique_ptr<Device>>* devices);
 
   // Helper for tests.  Create a single device of type "type".  The
   // returned device is always numbered zero, so if creating multiple
   // devices of the same type, supply distinct name_prefix arguments.
-  static Device* NewDevice(const string& type, const SessionOptions& options,
-                           const string& name_prefix);
+  static std::unique_ptr<Device> NewDevice(const string& type,
+                                           const SessionOptions& options,
+                                           const string& name_prefix);
 
   // Most clients should call AddDevices() instead.
-  virtual Status CreateDevices(const SessionOptions& options,
-                               const string& name_prefix,
-                               std::vector<Device*>* devices) = 0;
+  virtual Status CreateDevices(
+      const SessionOptions& options, const string& name_prefix,
+      std::vector<std::unique_ptr<Device>>* devices) = 0;
 
   // Return the device priority number for a "device_type" string.
   //
diff --git a/tensorflow/core/common_runtime/device_mgr.cc b/tensorflow/core/common_runtime/device_mgr.cc
index 470abc1431..1f7d7c4699 100644
--- a/tensorflow/core/common_runtime/device_mgr.cc
+++ b/tensorflow/core/common_runtime/device_mgr.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device_mgr.h"
 
+#include <memory>
 #include <vector>
 #include "tensorflow/core/common_runtime/local_device.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -24,32 +25,32 @@ limitations under the License.
 
 namespace tensorflow {
 
-DeviceMgr::DeviceMgr(const std::vector<Device*>& devices)
-    : name_backing_store_(128) {
-  for (Device* d : devices) {
+DeviceMgr::DeviceMgr(std::vector<std::unique_ptr<Device>> devices)
+    : devices_(std::move(devices)), name_backing_store_(128) {
+  for (auto& d : devices_) {
     CHECK(d->device_mgr_ == nullptr);
     d->device_mgr_ = this;
 
-    devices_.push_back(d);
-
     // Register under the (1) full name and (2) canonical name.
     for (const string& name :
          DeviceNameUtils::GetNamesForDeviceMappings(d->parsed_name())) {
-      device_map_[CopyToBackingStore(name)] = d;
+      device_map_[CopyToBackingStore(name)] = d.get();
     }
     // Register under the (3) local name and (4) legacy local name.
     for (const string& name :
          DeviceNameUtils::GetLocalNamesForDeviceMappings(d->parsed_name())) {
-      device_map_[CopyToBackingStore(name)] = d;
+      device_map_[CopyToBackingStore(name)] = d.get();
     }
     device_type_counts_[d->device_type()]++;
   }
 }
 
-DeviceMgr::~DeviceMgr() {
-  // TODO(b/37437134): Remove destructor after converting to std::unique_ptr.
-  for (Device* p : devices_) delete p;
-}
+DeviceMgr::DeviceMgr(std::unique_ptr<Device> device)
+    : DeviceMgr([&device] {
+        std::vector<std::unique_ptr<Device>> vector;
+        vector.push_back(std::move(device));
+        return vector;
+      }()) {}
 
 StringPiece DeviceMgr::CopyToBackingStore(StringPiece s) {
   size_t n = s.size();
@@ -61,18 +62,22 @@ StringPiece DeviceMgr::CopyToBackingStore(StringPiece s) {
 void DeviceMgr::ListDeviceAttributes(
     std::vector<DeviceAttributes>* devices) const {
   devices->reserve(devices_.size());
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     devices->emplace_back(dev->attributes());
   }
 }
 
 std::vector<Device*> DeviceMgr::ListDevices() const {
-  return std::vector<Device*>(devices_.begin(), devices_.end());
+  std::vector<Device*> devices(devices_.size());
+  for (size_t i = 0; i < devices_.size(); ++i) {
+    devices[i] = devices_[i].get();
+  }
+  return devices;
 }
 
 string DeviceMgr::DebugString() const {
   string out;
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     strings::StrAppend(&out, dev->name(), "\n");
   }
   return out;
@@ -80,7 +85,7 @@ string DeviceMgr::DebugString() const {
 
 string DeviceMgr::DeviceMappingString() const {
   string out;
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     if (!dev->attributes().physical_device_desc().empty()) {
       strings::StrAppend(&out, dev->name(), " -> ",
                          dev->attributes().physical_device_desc(), "\n");
@@ -107,7 +112,7 @@ Status DeviceMgr::LookupDevice(StringPiece name, Device** device) const {
 
 void DeviceMgr::ClearContainers(gtl::ArraySlice<string> containers) const {
   Status s;
-  for (Device* dev : devices_) {
+  for (const auto& dev : devices_) {
     if (containers.empty()) {
       s.Update(dev->resource_manager()->Cleanup(
           dev->resource_manager()->default_container()));
diff --git a/tensorflow/core/common_runtime/device_mgr.h b/tensorflow/core/common_runtime/device_mgr.h
index c1ff10d9b5..bf8694655a 100644
--- a/tensorflow/core/common_runtime/device_mgr.h
+++ b/tensorflow/core/common_runtime/device_mgr.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
 #define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
 
+#include <memory>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -34,15 +35,17 @@ class DeviceAttributes;
 
 class DeviceMgr {
  public:
-  // Takes ownership of each device in 'devices'.
+  // Constructs a DeviceMgr from a list of devices.
   // TODO(zhifengc): Other initialization information.
-  // TODO(b/37437134): Use std::unique_ptr's to track ownership.
-  explicit DeviceMgr(const std::vector<Device*>& devices);
-  ~DeviceMgr();
+  explicit DeviceMgr(std::vector<std::unique_ptr<Device>> devices);
+
+  // Constructs a DeviceMgr managing a single device.
+  explicit DeviceMgr(std::unique_ptr<Device> device);
 
   // Returns attributes of all devices.
   void ListDeviceAttributes(std::vector<DeviceAttributes>* devices) const;
 
+  // Returns raw pointers to the underlying devices.
   std::vector<Device*> ListDevices() const;
 
   // Returns a string listing all devices.
@@ -62,9 +65,7 @@ class DeviceMgr {
   int NumDeviceType(const string& type) const;
 
  private:
-  // TODO(b/37437134): Use std::unique_ptr's to track ownership.
-  typedef gtl::InlinedVector<Device*, 8> DeviceVec;
-  DeviceVec devices_;
+  const std::vector<std::unique_ptr<Device>> devices_;
 
   StringPiece CopyToBackingStore(StringPiece s);
 
diff --git a/tensorflow/core/common_runtime/device_resolver_local_test.cc b/tensorflow/core/common_runtime/device_resolver_local_test.cc
index f5a6471ff7..54f1119e13 100644
--- a/tensorflow/core/common_runtime/device_resolver_local_test.cc
+++ b/tensorflow/core/common_runtime/device_resolver_local_test.cc
@@ -36,12 +36,12 @@ class DeviceResolverLocalTest : public ::testing::Test {
     string task_name = "/job:localhost/replica:0/task:0";
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<DeviceResolverLocal> drl_;
 };
diff --git a/tensorflow/core/common_runtime/device_set_test.cc b/tensorflow/core/common_runtime/device_set_test.cc
index fd9c4222a7..6a8c3d14e5 100644
--- a/tensorflow/core/common_runtime/device_set_test.cc
+++ b/tensorflow/core/common_runtime/device_set_test.cc
@@ -57,7 +57,7 @@ class DeviceSetTest : public ::testing::Test {
 class DummyFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     return Status::OK();
   }
 };
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 178469db51..0434ca47b6 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -156,12 +156,12 @@ class DirectSessionFactory : public SessionFactory {
     if (options.config.graph_options().build_cost_model() > 0) {
       EnableCPUAllocatorFullStats(true);
     }
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
         options, "/job:localhost/replica:0/task:0", &devices));
 
     DirectSession* session =
-        new DirectSession(options, new DeviceMgr(devices), this);
+        new DirectSession(options, new DeviceMgr(std::move(devices)), this);
     {
       mutex_lock l(sessions_lock_);
       sessions_.push_back(session);
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index a7b618c18b..86890ba07d 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -181,6 +181,7 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
index 948bdbcaf5..3ffed3ce32 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/cc/client/client_session.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
@@ -37,12 +38,13 @@ namespace {
 class TestEnv {
  public:
   TestEnv() : flib_def_(OpRegistry::Global(), {}) {
-    Device* device =
-        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0");
-    device_mgr_.reset(new DeviceMgr({device}));
-    flib_runtime_ = NewFunctionLibraryRuntime(device_mgr_.get(), Env::Default(),
-                                              device, TF_GRAPH_DEF_VERSION,
-                                              &flib_def_, nullptr, {}, nullptr);
+    std::vector<std::unique_ptr<Device>> devices;
+    devices.push_back(
+        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0"));
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+    flib_runtime_ = NewFunctionLibraryRuntime(
+        device_mgr_.get(), Env::Default(), device_mgr_->ListDevices()[0],
+        TF_GRAPH_DEF_VERSION, &flib_def_, nullptr, {}, nullptr);
   }
 
   FunctionLibraryRuntime* function_library_runtime() const {
diff --git a/tensorflow/core/common_runtime/executor_test.cc b/tensorflow/core/common_runtime/executor_test.cc
index 7697103faf..c311b2533e 100644
--- a/tensorflow/core/common_runtime/executor_test.cc
+++ b/tensorflow/core/common_runtime/executor_test.cc
@@ -53,17 +53,17 @@ class ExecutorTest : public ::testing::Test {
     // when the test completes.
     CHECK(rendez_->Unref());
     delete exec_;
-    delete device_;
   }
 
   // Resets executor_ with a new executor based on a graph 'gdef'.
   void Create(std::unique_ptr<const Graph> graph) {
     const int version = graph->versions().producer();
     LocalExecutorParams params;
-    params.device = device_;
+    params.device = device_.get();
     params.create_kernel = [this, version](const NodeDef& ndef,
                                            OpKernel** kernel) {
-      return CreateNonCachedKernel(device_, nullptr, ndef, version, kernel);
+      return CreateNonCachedKernel(device_.get(), nullptr, ndef, version,
+                                   kernel);
     };
     params.delete_kernel = [](OpKernel* kernel) {
       DeleteNonCachedKernel(kernel);
@@ -83,7 +83,7 @@ class ExecutorTest : public ::testing::Test {
   }
 
   thread::ThreadPool* thread_pool_ = nullptr;
-  Device* device_ = nullptr;
+  std::unique_ptr<Device> device_;
   Executor* exec_ = nullptr;
   StepStatsCollector step_stats_collector_;
   StepStats step_stats_;
diff --git a/tensorflow/core/common_runtime/function_test.cc b/tensorflow/core/common_runtime/function_test.cc
index 13c189fb87..3b4c976685 100644
--- a/tensorflow/core/common_runtime/function_test.cc
+++ b/tensorflow/core/common_runtime/function_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <atomic>
 #include <utility>
 
+#include "absl/memory/memory.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
 #include "tensorflow/cc/ops/array_ops_internal.h"
@@ -147,14 +148,15 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 3});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices_));
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
     OptimizerOptions opts;
-    device_mgr_.reset(new DeviceMgr(devices_));
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
     pflr_.reset(new ProcessFunctionLibraryRuntime(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
@@ -358,7 +360,6 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   FunctionLibraryRuntime* flr0_;
   FunctionLibraryRuntime* flr1_;
   FunctionLibraryRuntime* flr2_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/core/common_runtime/function_threadpool_test.cc b/tensorflow/core/common_runtime/function_threadpool_test.cc
index 655a68cfc9..bdbe24a70d 100644
--- a/tensorflow/core/common_runtime/function_threadpool_test.cc
+++ b/tensorflow/core/common_runtime/function_threadpool_test.cc
@@ -54,14 +54,15 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 3});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices_));
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
     OptimizerOptions opts;
-    device_mgr_.reset(new DeviceMgr(devices_));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     pflr_.reset(new ProcessFunctionLibraryRuntime(
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
@@ -194,7 +195,6 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   FunctionLibraryRuntime* flr0_;
   FunctionLibraryRuntime* flr1_;
   FunctionLibraryRuntime* flr2_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 81fea311e1..5152d97fde 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -907,9 +907,9 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
-Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options,
-                                           const string& name_prefix,
-                                           std::vector<Device*>* devices) {
+Status BaseGPUDeviceFactory::CreateDevices(
+    const SessionOptions& options, const string& name_prefix,
+    std::vector<std::unique_ptr<Device>>* devices) {
   TF_RETURN_IF_ERROR(ValidateGPUMachineManager());
   se::Platform* gpu_manager = GPUMachineManager();
   if (gpu_manager == nullptr) {
@@ -1073,12 +1073,10 @@ static string GetShortDeviceDescription(PlatformGpuId platform_gpu_id,
   // LINT.ThenChange(//tensorflow/python/platform/test.py)
 }
 
-Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
-                                             const string& name_prefix,
-                                             TfGpuId tf_gpu_id,
-                                             int64 memory_limit,
-                                             const DeviceLocality& dev_locality,
-                                             std::vector<Device*>* devices) {
+Status BaseGPUDeviceFactory::CreateGPUDevice(
+    const SessionOptions& options, const string& name_prefix, TfGpuId tf_gpu_id,
+    int64 memory_limit, const DeviceLocality& dev_locality,
+    std::vector<std::unique_ptr<Device>>* devices) {
   CHECK_GE(tf_gpu_id.value(), 0);
   const string device_name =
       strings::StrCat(name_prefix, "/device:GPU:", tf_gpu_id.value());
@@ -1108,7 +1106,7 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
   // different (which should be an error).
   //
   // TODO(laigd): report error if memory_limit doesn't match stats.bytes_limit.
-  BaseGPUDevice* gpu_device = CreateGPUDevice(
+  std::unique_ptr<BaseGPUDevice> gpu_device = CreateGPUDevice(
       options, device_name, static_cast<Bytes>(stats.bytes_limit), dev_locality,
       tf_gpu_id, GetShortDeviceDescription(platform_gpu_id, desc),
       gpu_allocator, ProcessState::singleton()->GetCPUAllocator(numa_node));
@@ -1116,7 +1114,7 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
             << (stats.bytes_limit >> 20) << " MB memory) -> physical GPU ("
             << GetShortDeviceDescription(platform_gpu_id, desc) << ")";
   TF_RETURN_IF_ERROR(gpu_device->Init(options));
-  devices->push_back(gpu_device);
+  devices->push_back(std::move(gpu_device));
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index 674e8384d5..d002d02c51 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -166,7 +166,7 @@ class BaseGPUDevice : public LocalDevice {
 class BaseGPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override;
+                       std::vector<std::unique_ptr<Device>>* devices) override;
 
   struct InterconnectMap {
     // Name of interconnect technology, if known.
@@ -207,15 +207,13 @@ class BaseGPUDeviceFactory : public DeviceFactory {
   Status CreateGPUDevice(const SessionOptions& options,
                          const string& name_prefix, TfGpuId tf_gpu_id,
                          int64 memory_limit, const DeviceLocality& dev_locality,
-                         std::vector<Device*>* devices);
-
-  virtual BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
-                                         const string& name, Bytes memory_limit,
-                                         const DeviceLocality& dev_locality,
-                                         TfGpuId tf_gpu_id,
-                                         const string& physical_device_desc,
-                                         Allocator* gpu_allocator,
-                                         Allocator* cpu_allocator) = 0;
+                         std::vector<std::unique_ptr<Device>>* devices);
+
+  virtual std::unique_ptr<BaseGPUDevice> CreateGPUDevice(
+      const SessionOptions& options, const string& name, Bytes memory_limit,
+      const DeviceLocality& dev_locality, TfGpuId tf_gpu_id,
+      const string& physical_device_desc, Allocator* gpu_allocator,
+      Allocator* cpu_allocator) = 0;
 
   // Returns into 'ids' the list of valid platform GPU ids, in the order that
   // they should map to TF GPU ids "/device:GPU:0", "/device:GPU:1", etc,
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc b/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
index e1aaf95df6..8dc7197329 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_factory.cc
@@ -59,15 +59,14 @@ class GPUDevice : public BaseGPUDevice {
 
 class GPUDeviceFactory : public BaseGPUDeviceFactory {
  private:
-  BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
-                                 const string& name, Bytes memory_limit,
-                                 const DeviceLocality& locality,
-                                 TfGpuId tf_gpu_id,
-                                 const string& physical_device_desc,
-                                 Allocator* gpu_allocator,
-                                 Allocator* cpu_allocator) override {
-    return new GPUDevice(options, name, memory_limit, locality, tf_gpu_id,
-                         physical_device_desc, gpu_allocator, cpu_allocator);
+  std::unique_ptr<BaseGPUDevice> CreateGPUDevice(
+      const SessionOptions& options, const string& name, Bytes memory_limit,
+      const DeviceLocality& locality, TfGpuId tf_gpu_id,
+      const string& physical_device_desc, Allocator* gpu_allocator,
+      Allocator* cpu_allocator) override {
+    return absl::make_unique<GPUDevice>(options, name, memory_limit, locality,
+                                        tf_gpu_id, physical_device_desc,
+                                        gpu_allocator, cpu_allocator);
   }
 };
 
@@ -108,7 +107,7 @@ class GPUCompatibleCPUDevice : public ThreadPoolDevice {
 class GPUCompatibleCPUDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
     if (iter != options.config.device_count().end()) {
@@ -116,7 +115,7 @@ class GPUCompatibleCPUDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      devices->push_back(new GPUCompatibleCPUDevice(
+      devices->push_back(absl::make_unique<GPUCompatibleCPUDevice>(
           options, name, Bytes(256 << 20), DeviceLocality(), cpu_allocator()));
     }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
index 75be6d60b8..58656ec757 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_on_non_gpu_machine_test.cc
@@ -33,7 +33,7 @@ namespace {
 
 TEST(GPUDeviceOnNonGPUMachineTest, CreateGPUDevicesOnNonGPUMachine) {
   SessionOptions opts;
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, "/job:localhost/replica:0/task:0", &devices));
   EXPECT_TRUE(devices.empty());
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
index 36294094e9..ae623b2adb 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_test.cc
@@ -88,7 +88,7 @@ class GPUDeviceTest : public ::testing::Test {
 
 TEST_F(GPUDeviceTest, FailedToParseVisibleDeviceList) {
   SessionOptions opts = MakeSessionOptions("0,abc");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -97,7 +97,7 @@ TEST_F(GPUDeviceTest, FailedToParseVisibleDeviceList) {
 
 TEST_F(GPUDeviceTest, InvalidGpuId) {
   SessionOptions opts = MakeSessionOptions("100");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -107,7 +107,7 @@ TEST_F(GPUDeviceTest, InvalidGpuId) {
 
 TEST_F(GPUDeviceTest, DuplicateEntryInVisibleDeviceList) {
   SessionOptions opts = MakeSessionOptions("0,0");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -117,7 +117,7 @@ TEST_F(GPUDeviceTest, DuplicateEntryInVisibleDeviceList) {
 
 TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithMemoryFractionSettings) {
   SessionOptions opts = MakeSessionOptions("0", 0.1, 1, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -129,7 +129,7 @@ TEST_F(GPUDeviceTest, GpuDeviceCountTooSmall) {
   // device_count is 0, but with one entry in visible_device_list and one
   // (empty) VirtualDevices messages.
   SessionOptions opts = MakeSessionOptions("0", 0, 0, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::UNKNOWN);
@@ -141,7 +141,7 @@ TEST_F(GPUDeviceTest, NotEnoughGpuInVisibleDeviceList) {
   // Single entry in visible_device_list with two (empty) VirtualDevices
   // messages.
   SessionOptions opts = MakeSessionOptions("0", 0, 8, {{}, {}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::UNKNOWN);
@@ -155,7 +155,7 @@ TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithVisibleDeviceList) {
   // Three entries in visible_device_list with two (empty) VirtualDevices
   // messages.
   SessionOptions opts = MakeSessionOptions("0,1", 0, 8, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
@@ -169,39 +169,36 @@ TEST_F(GPUDeviceTest, VirtualDeviceConfigConflictsWithVisibleDeviceList) {
 TEST_F(GPUDeviceTest, EmptyVirtualDeviceConfig) {
   // It'll create single virtual device when the virtual device config is empty.
   SessionOptions opts = MakeSessionOptions("0");
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_GE(devices[0]->attributes().memory_limit(), 0);
-  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, SingleVirtualDeviceWithNoMemoryLimit) {
   // It'll create single virtual device for the gpu in question when
   // memory_limit_mb is unset.
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_GE(devices[0]->attributes().memory_limit(), 0);
-  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, SingleVirtualDeviceWithMemoryLimit) {
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{123}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(1, devices.size());
   EXPECT_EQ(123 << 20, devices[0]->attributes().memory_limit());
-  gtl::STLDeleteElements(&devices);
 }
 
 TEST_F(GPUDeviceTest, MultipleVirtualDevices) {
   SessionOptions opts = MakeSessionOptions("0", 0, 1, {{123, 456}});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_CHECK_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   EXPECT_EQ(2, devices.size());
@@ -219,7 +216,6 @@ TEST_F(GPUDeviceTest, MultipleVirtualDevices) {
             devices[1]->attributes().locality().links().link(0).type());
   EXPECT_EQ(BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength,
             devices[1]->attributes().locality().links().link(0).strength());
-  gtl::STLDeleteElements(&devices);
 }
 
 // Enabling unified memory on pre-Pascal GPUs results in an initialization
@@ -236,7 +232,7 @@ TEST_F(GPUDeviceTest, UnifiedMemoryUnavailableOnPrePascalGpus) {
   opts.config.mutable_gpu_options()
       ->mutable_experimental()
       ->set_use_unified_memory(true);
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices);
   EXPECT_EQ(status.code(), error::INTERNAL);
@@ -259,7 +255,7 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
   }
 
   SessionOptions opts = MakeSessionOptions("0", kGpuMemoryFraction);
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_ASSERT_OK(DeviceFactory::GetFactory("GPU")->CreateDevices(
       opts, kDeviceNamePrefix, &devices));
   ASSERT_EQ(1, devices.size());
@@ -278,8 +274,6 @@ TEST_F(GPUDeviceTest, UnifiedMemoryAllocation) {
                                      (memory_limit >> 20) << 20);
   EXPECT_NE(ptr, nullptr);
   allocator->DeallocateRaw(ptr);
-
-  gtl::STLDeleteElements(&devices);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
index 2144eea84f..f0656ff533 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h"
 
 #include <algorithm>
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/base_collective_executor.h"
 #include "tensorflow/core/common_runtime/collective_rma_local.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -217,7 +218,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
             << " num_devices_per_worker=" << num_devices_per_worker;
     int total_num_devices = num_workers * num_devices_per_worker;
     device_type_ = device_type;
-    std::vector<Device*> local_devices;
+    std::vector<std::unique_ptr<Device>> local_devices;
     SessionOptions sess_opts;
     sess_opts.env = Env::Default();
     Bytes mem_limit(4 << 20);
@@ -227,7 +228,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
         if (device_type == DEVICE_CPU) {
           string dev_name = strings::StrCat("/job:worker/replica:0/task:", wi,
                                             "/device:CPU:", di);
-          local_devices.push_back(new ThreadPoolDevice(
+          local_devices.push_back(absl::make_unique<ThreadPoolDevice>(
               sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator()));
         } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) {
           int dev_idx = (wi * num_devices_per_worker) + di;
@@ -235,7 +236,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
             LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more "
                          "than one ring node.";
           } else {
-            local_devices.push_back(gpu_devices_[dev_idx]);
+            local_devices.push_back(std::move(gpu_devices_[dev_idx]));
           }
         } else {
           LOG(FATAL) << "Unsupported device_type " << device_type;
@@ -243,7 +244,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
       }
     }
     if (!dev_mgr_ || device_type == DEVICE_CPU) {
-      dev_mgr_.reset(new DeviceMgr(local_devices));
+      dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
     }
     if (!gpu_ring_order_) gpu_ring_order_.reset(new string());
     dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get()));
@@ -714,7 +715,7 @@ class HierarchicalTreeBroadcasterTest : public ::testing::Test {
   std::unique_ptr<DeviceResolverLocal> dev_resolver_;
   std::vector<DeviceInstance*> instances_;
   CollectiveParams col_params_;
-  std::vector<tensorflow::Device*> gpu_devices_;
+  std::vector<std::unique_ptr<tensorflow::Device>> gpu_devices_;
   std::unique_ptr<tensorflow::DeviceMgr> dev_mgr_;
   std::unique_ptr<string> gpu_ring_order_;
   mutex mu_;
diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
index 1f585a8c24..bdd6c0e87d 100644
--- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
+++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.cc
@@ -75,12 +75,12 @@ Benchmark::Benchmark(const string& device, Graph* g,
   const int graph_def_version = g->versions().producer();
 
   LocalExecutorParams params;
-  params.device = device_;
+  params.device = device_.get();
   params.function_library = nullptr;
   params.create_kernel = [this, graph_def_version](const NodeDef& ndef,
                                                    OpKernel** kernel) {
-    return CreateNonCachedKernel(device_, nullptr, ndef, graph_def_version,
-                                 kernel);
+    return CreateNonCachedKernel(device_.get(), nullptr, ndef,
+                                 graph_def_version, kernel);
   };
   params.delete_kernel = [](OpKernel* kernel) {
     DeleteNonCachedKernel(kernel);
@@ -107,7 +107,7 @@ Benchmark::~Benchmark() {
     // run kernel destructors that may attempt to access state borrowed from
     // `device_`, such as the resource manager.
     exec_.reset();
-    delete device_;
+    device_.reset();
     delete pool_;
   }
 }
diff --git a/tensorflow/core/common_runtime/kernel_benchmark_testlib.h b/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
index 555b43f655..b1557c50b0 100644
--- a/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
+++ b/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
@@ -55,7 +55,7 @@ class Benchmark {
 
  private:
   thread::ThreadPool* pool_ = nullptr;
-  Device* device_ = nullptr;
+  std::unique_ptr<Device> device_ = nullptr;
   Rendezvous* rendez_ = nullptr;
   std::unique_ptr<Executor> exec_;
 
diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc
index 009f905f10..04e77e55f6 100644
--- a/tensorflow/core/common_runtime/placer_test.cc
+++ b/tensorflow/core/common_runtime/placer_test.cc
@@ -92,7 +92,7 @@ class FakeDevice : public Device {
 class DummyFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     return Status::OK();
   }
 };
diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
index cce2308011..21cb62118a 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
@@ -62,9 +62,12 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
     SessionOptions options;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", 2});
+    std::vector<std::unique_ptr<Device>> devices;
     TF_CHECK_OK(DeviceFactory::AddDevices(options, "/job:a/replica:0/task:0",
-                                          &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+                                          &devices));
+    device0_ = devices[0].get();
+    device1_ = devices[1].get();
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     FunctionDefLibrary proto;
     for (const auto& fdef : flib) *(proto.add_function()) = fdef;
     lib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), proto));
@@ -138,8 +141,9 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
     return Status::OK();
   }
 
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
+  Device* device0_ = nullptr;  // Not owned. (Owned by device_mgr_.)
+  Device* device1_ = nullptr;  // Not owned. (Owned by device_mgr_.)
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<TestClusterFLR> cluster_flr_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> proc_flr_;
@@ -165,16 +169,16 @@ TEST_F(ProcessFunctionLibraryRuntimeTest, Basic) {
   FunctionLibraryRuntime* flr =
       proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[0]);
+  EXPECT_EQ(flr->device(), device0_);
   flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/device:CPU:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[0]);
+  EXPECT_EQ(flr->device(), device0_);
   flr = proc_flr_->GetFLR("/device:CPU:0");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[0]);
+  EXPECT_EQ(flr->device(), device0_);
   flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:1");
   EXPECT_NE(flr, nullptr);
-  EXPECT_EQ(flr->device(), devices_[1]);
+  EXPECT_EQ(flr->device(), device1_);
   flr = proc_flr_->GetFLR("abc");
   EXPECT_EQ(flr, nullptr);
   rendezvous_->Unref();
diff --git a/tensorflow/core/common_runtime/renamed_device.cc b/tensorflow/core/common_runtime/renamed_device.cc
index 56766a8df4..45541c35fe 100644
--- a/tensorflow/core/common_runtime/renamed_device.cc
+++ b/tensorflow/core/common_runtime/renamed_device.cc
@@ -14,15 +14,14 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/common_runtime/renamed_device.h"
+#include "absl/memory/memory.h"
 
 namespace tensorflow {
 
-// TODO(saeta): Convert to returning a std::unique_ptr?
 /* static */
-Device* RenamedDevice::NewRenamedDevice(const string& new_base,
-                                        Device* underlying,
-                                        bool owns_underlying,
-                                        bool isolate_session_state) {
+std::unique_ptr<Device> RenamedDevice::NewRenamedDevice(
+    const string& new_base, Device* underlying, bool owns_underlying,
+    bool isolate_session_state) {
   DeviceNameUtils::ParsedName parsed_name;
   CHECK(DeviceNameUtils::ParseFullName(new_base, &parsed_name));
   DeviceNameUtils::ParsedName underlying_parsed_name =
@@ -36,8 +35,9 @@ Device* RenamedDevice::NewRenamedDevice(const string& new_base,
                                           parsed_name.id);
   DeviceAttributes attributes(underlying->attributes());
   attributes.set_name(name);
-  return new RenamedDevice(underlying, attributes, owns_underlying,
-                           isolate_session_state);
+  // Call absl::WrapUnique to access private constructor.
+  return absl::WrapUnique(new RenamedDevice(
+      underlying, attributes, owns_underlying, isolate_session_state));
 }
 
 RenamedDevice::RenamedDevice(Device* underlying,
diff --git a/tensorflow/core/common_runtime/renamed_device.h b/tensorflow/core/common_runtime/renamed_device.h
index c00789a556..6d24f496ff 100644
--- a/tensorflow/core/common_runtime/renamed_device.h
+++ b/tensorflow/core/common_runtime/renamed_device.h
@@ -28,9 +28,10 @@ namespace tensorflow {
 // session.
 class RenamedDevice : public Device {
  public:
-  static Device* NewRenamedDevice(const string& new_base, Device* underlying,
-                                  bool owns_underlying,
-                                  bool isolate_session_state);
+  static std::unique_ptr<Device> NewRenamedDevice(const string& new_base,
+                                                  Device* underlying,
+                                                  bool owns_underlying,
+                                                  bool isolate_session_state);
 
   ~RenamedDevice() override;
 
diff --git a/tensorflow/core/common_runtime/ring_reducer_test.cc b/tensorflow/core/common_runtime/ring_reducer_test.cc
index a271bf7b74..7feb29a6db 100644
--- a/tensorflow/core/common_runtime/ring_reducer_test.cc
+++ b/tensorflow/core/common_runtime/ring_reducer_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/ring_reducer.h"
 
 #include <algorithm>
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/base_collective_executor.h"
 #include "tensorflow/core/common_runtime/collective_rma_local.h"
 #include "tensorflow/core/common_runtime/device.h"
@@ -157,7 +158,7 @@ class RingReducerTest : public ::testing::Test {
     InitGPUDevices();
 #endif
     device_type_ = device_type;
-    std::vector<Device*> local_devices;
+    std::vector<std::unique_ptr<Device>> local_devices;
     SessionOptions sess_opts;
     sess_opts.env = Env::Default();
     Bytes mem_limit(4 << 20);
@@ -167,7 +168,7 @@ class RingReducerTest : public ::testing::Test {
         if (device_type == DEVICE_CPU) {
           string dev_name =
               strings::StrCat("/job:worker/replica:0/task:", wi, "/cpu:", di);
-          local_devices.push_back(new ThreadPoolDevice(
+          local_devices.push_back(absl::make_unique<ThreadPoolDevice>(
               sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator()));
         } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) {
           int dev_idx = (wi * num_devices) + di;
@@ -175,7 +176,7 @@ class RingReducerTest : public ::testing::Test {
             LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more "
                          "than one ring node.";
           } else {
-            local_devices.push_back(gpu_devices_[dev_idx]);
+            local_devices.push_back(std::move(gpu_devices_[dev_idx]));
           }
         } else {
           LOG(FATAL) << "Unsupported device_type " << device_type;
@@ -185,7 +186,7 @@ class RingReducerTest : public ::testing::Test {
     if (!dev_mgr_ || device_type == DEVICE_CPU) {
       LOG(ERROR) << "resetting dev_mgr for " << local_devices.size()
                  << " devices: ";
-      dev_mgr_.reset(new DeviceMgr(local_devices));
+      dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
     }
     if (!gpu_ring_order_) gpu_ring_order_.reset(new string());
     dev_resolver_.reset(new DeviceResolverLocal(dev_mgr_.get()));
@@ -544,7 +545,7 @@ class RingReducerTest : public ::testing::Test {
   std::unique_ptr<DeviceResolverLocal> dev_resolver_;
   std::vector<DeviceInstance*> instances_;
   CollectiveParams col_params_;
-  std::vector<tensorflow::Device*> gpu_devices_;
+  std::vector<std::unique_ptr<tensorflow::Device>> gpu_devices_;
   std::unique_ptr<tensorflow::DeviceMgr> dev_mgr_;
   std::unique_ptr<string> gpu_ring_order_;
   mutex mu_;
diff --git a/tensorflow/core/common_runtime/threadpool_device_factory.cc b/tensorflow/core/common_runtime/threadpool_device_factory.cc
index c06a4035a7..f9cbb81749 100644
--- a/tensorflow/core/common_runtime/threadpool_device_factory.cc
+++ b/tensorflow/core/common_runtime/threadpool_device_factory.cc
@@ -13,12 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// Register a factory that provides CPU devices.
-#include "tensorflow/core/common_runtime/threadpool_device.h"
-
 #include <vector>
+
+// Register a factory that provides CPU devices.
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/process_state.h"
+#include "tensorflow/core/common_runtime/threadpool_device.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/public/session_options.h"
@@ -29,7 +30,7 @@ namespace tensorflow {
 class ThreadPoolDeviceFactory : public DeviceFactory {
  public:
   Status CreateDevices(const SessionOptions& options, const string& name_prefix,
-                       std::vector<Device*>* devices) override {
+                       std::vector<std::unique_ptr<Device>>* devices) override {
     int num_numa_nodes = port::NUMANumNodes();
     int n = 1;
     auto iter = options.config.device_count().find("CPU");
@@ -38,7 +39,7 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
     }
     for (int i = 0; i < n; i++) {
       string name = strings::StrCat(name_prefix, "/device:CPU:", i);
-      ThreadPoolDevice* tpd = nullptr;
+      std::unique_ptr<ThreadPoolDevice> tpd;
       if (options.config.experimental().use_numa_affinity()) {
         int numa_node = i % num_numa_nodes;
         if (numa_node != i) {
@@ -49,15 +50,15 @@ class ThreadPoolDeviceFactory : public DeviceFactory {
         }
         DeviceLocality dev_locality;
         dev_locality.set_numa_node(numa_node);
-        tpd = new ThreadPoolDevice(
+        tpd = absl::make_unique<ThreadPoolDevice>(
             options, name, Bytes(256 << 20), dev_locality,
             ProcessState::singleton()->GetCPUAllocator(numa_node));
       } else {
-        tpd = new ThreadPoolDevice(
+        tpd = absl::make_unique<ThreadPoolDevice>(
             options, name, Bytes(256 << 20), DeviceLocality(),
             ProcessState::singleton()->GetCPUAllocator(port::kNUMANoAffinity));
       }
-      devices->push_back(tpd);
+      devices->push_back(std::move(tpd));
     }
 
     return Status::OK();
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index cd9e58ac39..e388d3e6f0 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -625,6 +625,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 4eed856759..40b18d321a 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -29,7 +29,8 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-static Device* NewDevice(const string& type, const string& name) {
+static std::unique_ptr<Device> NewDevice(const string& type,
+                                         const string& name) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -40,7 +41,7 @@ static Device* NewDevice(const string& type, const string& name) {
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(3);  // a non-default value
-  return new FakeDevice(attr);
+  return absl::make_unique<FakeDevice>(attr);
 }
 
 class FakeWorker : public TestWorkerInterface {
@@ -156,16 +157,16 @@ class DeviceResDistTest : public ::testing::Test {
 
   void DefineWorker(const ConfigProto& config, const string& worker_name,
                     const string& device_type, int num_devices) {
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i)));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(devices);
+    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : devices) {
+    for (auto* d : dev_mgr->ListDevices()) {
       dv->push_back(d->name());
     }
     DeviceResolverDistributed* dev_res =
diff --git a/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
index 33e1c8f2c3..26f722a6bd 100644
--- a/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_rma_distributed_test.cc
@@ -41,7 +41,8 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-static Device* NewDevice(const string& type, const string& name) {
+static std::unique_ptr<Device> NewDevice(const string& type,
+                                         const string& name) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -52,7 +53,7 @@ static Device* NewDevice(const string& type, const string& name) {
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(3);  // a non-default value
-  return new FakeDevice(attr);
+  return absl::make_unique<FakeDevice>(attr);
 }
 
 static int64 kStepId = 123;
@@ -211,16 +212,16 @@ class CollRMADistTest : public ::testing::Test {
 
   void DefineWorker(const ConfigProto& config, const string& worker_name,
                     const string& device_type, int num_devices) {
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i)));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(devices);
+    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : devices) {
+    for (auto d : dev_mgr->ListDevices()) {
       dv->push_back(d->name());
     }
     DeviceResolverDistributed* dev_res =
diff --git a/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
index ae44b98bd5..842a2b3b05 100644
--- a/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/device_resolver_distributed_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/device_resolver_distributed.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/distributed_runtime/test_utils.h"
 #include "tensorflow/core/lib/core/notification.h"
@@ -41,8 +42,8 @@ class TestableDeviceResolverDistributed : public DeviceResolverDistributed {
 
 // Create a fake 'Device' whose only interesting attribute is a non-default
 // DeviceLocality.
-static Device* NewDevice(const string& type, const string& name,
-                         int numa_node) {
+static std::unique_ptr<Device> NewDevice(const string& type, const string& name,
+                                         int numa_node) {
   class FakeDevice : public Device {
    public:
     explicit FakeDevice(const DeviceAttributes& attr) : Device(nullptr, attr) {}
@@ -53,7 +54,7 @@ static Device* NewDevice(const string& type, const string& name,
   attr.set_name(name);
   attr.set_device_type(type);
   attr.mutable_locality()->set_numa_node(numa_node);
-  return new FakeDevice(attr);
+  return absl::make_unique<FakeDevice>(attr);
 }
 
 // Create a fake WorkerInterface that responds to requests without RPCs,
@@ -151,19 +152,19 @@ class DeviceResDistTest : public ::testing::Test {
 
   void DefineWorker(const string& worker_name, const string& device_type,
                     int num_devices) {
-    std::vector<Device*> devices;
+    std::vector<std::unique_ptr<Device>> devices;
     for (int i = 0; i < num_devices; ++i) {
       devices.push_back(NewDevice(
           device_type,
           strings::StrCat(worker_name, "/device:", device_type, ":", i), i));
     }
-    DeviceMgr* dev_mgr = new DeviceMgr(devices);
+    DeviceMgr* dev_mgr = new DeviceMgr(std::move(devices));
     TestableDeviceResolverDistributed* dev_res =
         new TestableDeviceResolverDistributed(dev_mgr, &wc_, worker_name);
     resolvers_[worker_name] = dev_res;
     device_mgrs_.push_back(dev_mgr);
     std::vector<string>* dv = &dev_by_task_[worker_name];
-    for (auto d : devices) {
+    for (auto* d : dev_mgr->ListDevices()) {
       dv->push_back(d->name());
     }
     FakeWorker* fw = new FakeWorker(worker_name, dev_mgr, dev_res);
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
index 5b0a420fad..c66466c0a6 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
@@ -87,7 +87,7 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
     return tensorflow::errors::Internal(
         "invalid eager env_ or env_->rendezvous_mgr.");
   }
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
       // TODO(nareshmodi): Correctly set the SessionOptions.
@@ -97,12 +97,12 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
                       request->server_def().task_index()),
       &devices));
   response->mutable_device_attributes()->Reserve(devices.size());
-  for (auto& d : devices) {
+  for (const auto& d : devices) {
     *response->add_device_attributes() = d->attributes();
   }
 
   std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(devices));
+      new tensorflow::DeviceMgr(std::move(devices)));
 
   auto* r = env_->rendezvous_mgr->Find(request->rendezvous_id());
   auto session_name = strings::StrCat("eager_", request->rendezvous_id());
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
index 5ba522c2a2..7a1463e8f0 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
@@ -68,12 +68,9 @@ class EagerServiceImplTest : public ::testing::Test {
     worker_env_.rendezvous_mgr = &rendezvous_mgr_;
     worker_env_.session_mgr = session_mgr_.get();
 
-    Device* device = DeviceFactory::NewDevice(
-        "CPU", {}, "/job:localhost/replica:0/task:0/device:CPU:0");
-
-    worker_env_.local_devices = {device};
-
-    device_mgr_.reset(new DeviceMgr(worker_env_.local_devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(DeviceFactory::NewDevice(
+        "CPU", {}, "/job:localhost/replica:0/task:0/device:CPU:0"));
+    worker_env_.local_devices = device_mgr_->ListDevices();
     worker_env_.device_mgr = device_mgr_.get();
   }
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
index ae722fdfe9..cbd5cd927e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <vector>
 
 #include "grpc/support/alloc.h"
 #include "grpcpp/grpcpp.h"
@@ -156,10 +157,12 @@ Status GrpcServer::Init(
   string name_prefix =
       strings::StrCat("/job:", server_def_.job_name(), "/replica:0",
                       "/task:", server_def_.task_index());
-  TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(sess_opts, name_prefix,
-                                               &master_env_.local_devices));
-  worker_env_.local_devices = master_env_.local_devices;
-  worker_env_.device_mgr = new DeviceMgr(worker_env_.local_devices);
+  std::vector<std::unique_ptr<Device>> devices;
+  TF_RETURN_IF_ERROR(
+      DeviceFactory::AddDevices(sess_opts, name_prefix, &devices));
+  worker_env_.device_mgr = new DeviceMgr(std::move(devices));
+  master_env_.local_devices = worker_env_.device_mgr->ListDevices();
+  worker_env_.local_devices = worker_env_.device_mgr->ListDevices();
   worker_env_.rendezvous_mgr = rendezvous_mgr_func == nullptr
                                    ? new RpcRendezvousMgr(&worker_env_)
                                    : rendezvous_mgr_func(&worker_env_);
diff --git a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
index 0323300fdd..1c87fe9d92 100644
--- a/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/rpc_collective_executor_mgr_test.cc
@@ -42,8 +42,9 @@ class RpcCollectiveExecutorMgrTest : public ::testing::Test {
     WorkerCacheInterface* worker_cache = nullptr;
     auto* device_count = options.config.mutable_device_count();
     device_count->insert({"CPU", NUM_DEVS});
-    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
-    device_mgr_.reset(new DeviceMgr(devices_));
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices));
+    device_mgr_.reset(new DeviceMgr(std::move(devices)));
     std::unique_ptr<DeviceResolverDistributed> dr(new DeviceResolverDistributed(
         device_mgr_.get(), worker_cache, task_name));
     std::unique_ptr<CollectiveParamResolverDistributed> cpr(
@@ -57,7 +58,6 @@ class RpcCollectiveExecutorMgrTest : public ::testing::Test {
   }
 
   std::unique_ptr<RpcCollectiveExecutorMgr> cme_;
-  std::vector<Device*> devices_;
   std::unique_ptr<DeviceMgr> device_mgr_;
 };
 
diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc
index 38833bd202..29fe767e42 100644
--- a/tensorflow/core/distributed_runtime/session_mgr.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr.cc
@@ -78,13 +78,13 @@ Status SessionMgr::CreateSession(const string& session,
 
   if (isolate_session_state) {
     // Create a private copy of the DeviceMgr for the WorkerSession.
-    std::vector<Device*> renamed_devices;
+    std::vector<std::unique_ptr<Device>> renamed_devices;
     for (Device* d : worker_env_->local_devices) {
       renamed_devices.push_back(RenamedDevice::NewRenamedDevice(
           worker_name, d, false, isolate_session_state));
     }
 
-    auto device_mgr = MakeUnique<DeviceMgr>(renamed_devices);
+    auto device_mgr = MakeUnique<DeviceMgr>(std::move(renamed_devices));
     auto graph_mgr = MakeUnique<GraphMgr>(worker_env_, device_mgr.get());
     worker_session.reset(
         new WorkerSession(session, worker_name,
diff --git a/tensorflow/core/distributed_runtime/session_mgr_test.cc b/tensorflow/core/distributed_runtime/session_mgr_test.cc
index 99192119a6..1ab0d20f0b 100644
--- a/tensorflow/core/distributed_runtime/session_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/session_mgr_test.cc
@@ -46,11 +46,9 @@ class SessionMgrTest : public ::testing::Test {
   SessionMgrTest()
       : mgr_(&env_, "/job:mnist/replica:0/task:0",
              std::unique_ptr<WorkerCacheInterface>(), factory_) {
-    Device* device =
-        FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0")
-            .release();
-    env_.local_devices = {device};
-    device_mgr_.reset(new DeviceMgr(env_.local_devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(
+        FakeDevice::MakeCPU("/job:mnist/replica:0/task:0/device:fakecpu:0"));
+    env_.local_devices = device_mgr_->ListDevices();
     env_.device_mgr = device_mgr_.get();
   }
 
diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index cf99f4908b..e69dfa79d1 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -102,10 +102,11 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   }
 
   // Instantiate all variables for function library runtime creation.
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
-  std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(devices));
+  Device* cpu_device = devices[0].get();
+  std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(std::move(devices)));
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              graph_def.library());
   Env* env = Env::Default();
@@ -124,7 +125,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
       new ProcessFunctionLibraryRuntime(dvc_mgr.get(), env,
                                         graph_def.versions().producer(),
                                         &function_library, *optimizer_opts));
-  FunctionLibraryRuntime* flr = pflr->GetFLR(devices[0]->name());
+  FunctionLibraryRuntime* flr = pflr->GetFLR(cpu_device->name());
 
   // Create the GraphOptimizer to optimize the graph def.
   GraphConstructorOptions graph_ctor_opts;
@@ -137,7 +138,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
 
   // Optimize the graph.
   ::tensorflow::GraphOptimizer optimizer(*optimizer_opts);
-  optimizer.Optimize(flr, env, devices[0], &graphptr, /*shape_map=*/nullptr);
+  optimizer.Optimize(flr, env, cpu_device, &graphptr, /*shape_map=*/nullptr);
   graphptr->ToGraphDef(output_graph_def);
 
   // The default values of attributes might have been stripped by the optimizer.
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index b6f989f2c9..8e6629565a 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -142,7 +142,6 @@ cc_library(
         ":graph_optimizer",
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -150,6 +149,7 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
+        "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index f99826ddca..f8ddbeb659 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -16,7 +16,9 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 
 #include <unordered_map>
+#include <vector>
 
+#include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -343,14 +345,15 @@ class FunctionOptimizerContext {
       DeviceAttributes attr;
       attr.set_name("/device:CPU:0");
       attr.set_device_type("CPU");
-      Device* device = new FakeCPUDevice(env, attr);
-      device_mgr_.reset(new DeviceMgr({device}));
+      std::vector<std::unique_ptr<Device>> devices;
+      devices.push_back(absl::make_unique<FakeCPUDevice>(env, attr));
+      device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
       OptimizerOptions optimizer_opts;
       optimizer_opts.set_do_function_inlining(true);
       process_flr_.reset(new ProcessFunctionLibraryRuntime(
           device_mgr_.get(), env, graph_version_, &function_library_,
           optimizer_opts));
-      flr_ = process_flr_->GetFLR(device->name());
+      flr_ = process_flr_->GetFLR(device_mgr_->ListDevices()[0]->name());
     }
   }
 
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 7192684e2d..dcb6975669 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -600,6 +600,7 @@ tf_kernel_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:session_options",
         "//tensorflow/core/kernels:ops_util",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 93999dc095..98b67454d5 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/kernels/data/iterator_ops.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/common_runtime/renamed_device.h"
 #include "tensorflow/core/common_runtime/threadpool_device.h"
@@ -545,10 +546,9 @@ FunctionLibraryRuntime* IteratorHandleOp::CreatePrivateFLR(
   // in its resource manager. The existing device will outlive the
   // IteratorResource, because we are storing the IteratorResource
   // in that device's resource manager.
-  Device* wrapped_device = RenamedDevice::NewRenamedDevice(
+  *device_mgr = absl::make_unique<DeviceMgr>(RenamedDevice::NewRenamedDevice(
       ctx->device()->name(), down_cast<Device*>(ctx->device()),
-      false /* owns_underlying */, false /* isolate_session_state */);
-  device_mgr->reset(new DeviceMgr({wrapped_device}));
+      false /* owns_underlying */, false /* isolate_session_state */));
   flib_def->reset(new FunctionLibraryDefinition(
       *ctx->function_library()->GetFunctionLibraryDefinition()));
   pflr->reset(new ProcessFunctionLibraryRuntime(
diff --git a/tensorflow/core/kernels/data/single_threaded_executor_test.cc b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
index 6244e287bb..7bb51fb8b5 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor_test.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
@@ -51,17 +51,17 @@ class ExecutorTest : public ::testing::Test {
     // when the test completes.
     CHECK(rendez_->Unref());
     delete exec_;
-    delete device_;
   }
 
   // Resets executor_ with a new executor based on a graph 'gdef'.
   void Create(std::unique_ptr<const Graph> graph) {
     const int version = graph->versions().producer();
     LocalExecutorParams params;
-    params.device = device_;
+    params.device = device_.get();
     params.create_kernel = [this, version](const NodeDef& ndef,
                                            OpKernel** kernel) {
-      return CreateNonCachedKernel(device_, nullptr, ndef, version, kernel);
+      return CreateNonCachedKernel(device_.get(), nullptr, ndef, version,
+                                   kernel);
     };
     params.delete_kernel = [](OpKernel* kernel) {
       DeleteNonCachedKernel(kernel);
@@ -86,7 +86,7 @@ class ExecutorTest : public ::testing::Test {
     return exec_->Run(args);
   }
 
-  Device* device_ = nullptr;
+  std::unique_ptr<Device> device_;
   Executor* exec_ = nullptr;
   Executor::Args::Runner runner_;
   Rendezvous* rendez_ = nullptr;
diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD
index 222a043a88..63e86899da 100644
--- a/tensorflow/lite/delegates/flex/BUILD
+++ b/tensorflow/lite/delegates/flex/BUILD
@@ -116,6 +116,7 @@ cc_library(
     hdrs = ["delegate_data.h"],
     deps = [
         ":buffer_map",
+        "@com_google_absl//absl/memory",
         "//tensorflow/core/common_runtime/eager:context",
     ] + select({
         "//tensorflow:android": [
diff --git a/tensorflow/lite/delegates/flex/delegate_data.cc b/tensorflow/lite/delegates/flex/delegate_data.cc
index b62479a448..1483a53038 100644
--- a/tensorflow/lite/delegates/flex/delegate_data.cc
+++ b/tensorflow/lite/delegates/flex/delegate_data.cc
@@ -14,20 +14,21 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/delegates/flex/delegate_data.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tflite {
 namespace flex {
 tensorflow::Status DelegateData::Create(std::unique_ptr<DelegateData>* data) {
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
       tensorflow::SessionOptions(), "/job:localhost/replica:0/task:0",
       &devices));
 
-  std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(devices));
+  std::unique_ptr<tensorflow::DeviceMgr> device_mgr =
+      absl::make_unique<tensorflow::DeviceMgr>(std::move(devices));
   // Note that Rendezvous is ref-counted so it will be automatically deleted.
   tensorflow::Rendezvous* rendezvous =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index c8d1e1bdc5..b51f80c1a7 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2012,13 +2012,13 @@ bool InlineAllFunctions(GraphDef* graphdef) {
   tensorflow::SessionOptions options;
   auto* device_count = options.config.mutable_device_count();
   device_count->insert({"CPU", 1});
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   TF_CHECK_OK(tensorflow::DeviceFactory::AddDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
 
   tensorflow::FunctionLibraryDefinition fld(tensorflow::OpRegistry::Global(),
                                             graphdef_copy.library());
-  tensorflow::DeviceMgr device_mgr(devices);
+  tensorflow::DeviceMgr device_mgr(std::move(devices));
   tensorflow::OptimizerOptions o_opts;
   tensorflow::ProcessFunctionLibraryRuntime pflr(
       &device_mgr, tensorflow::Env::Default(), TF_GRAPH_DEF_VERSION, &fld,
diff --git a/tensorflow/python/client/device_lib.i b/tensorflow/python/client/device_lib.i
index 944e855cee..3e579152d5 100644
--- a/tensorflow/python/client/device_lib.i
+++ b/tensorflow/python/client/device_lib.i
@@ -48,17 +48,14 @@ static std::vector<string> ListDevicesWithSessionConfig(
   std::vector<string> output;
   SessionOptions options;
   options.config = config;
-  std::vector<Device*> devices;
+  std::vector<std::unique_ptr<Device>> devices;
   Status status = DeviceFactory::AddDevices(
       options, "" /* name_prefix */, &devices);
   if (!status.ok()) {
     Set_TF_Status_from_Status(out_status, status);
   }
 
-  std::vector<std::unique_ptr<Device>> device_holder(devices.begin(),
-                                                     devices.end());
-
-  for (const Device* device : devices) {
+  for (const std::unique_ptr<Device>& device : devices) {
     const DeviceAttributes& attr = device->attributes();
     string attr_serialized;
     if (!attr.SerializeToString(&attr_serialized)) {
diff --git a/tensorflow/python/grappler/tf_optimizer.i b/tensorflow/python/grappler/tf_optimizer.i
index daa5bc9444..b746c3ec26 100644
--- a/tensorflow/python/grappler/tf_optimizer.i
+++ b/tensorflow/python/grappler/tf_optimizer.i
@@ -74,13 +74,13 @@ limitations under the License.
 
 void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* device_map) {
   tensorflow::SessionOptions options;
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   tensorflow::Status status = tensorflow::DeviceFactory::AddDevices(options, "", &devices);
   if (!status.ok()) {
     return;
   }
 
-  for (const tensorflow::Device* device : devices) {
+  for (const std::unique_ptr<tensorflow::Device>& device : devices) {
     tensorflow::DeviceProperties& prop = (*device_map)[device->name()];
     prop = tensorflow::grappler::GetDeviceInfo(device->parsed_name());
 
@@ -88,7 +88,6 @@ void DetectDevices(std::unordered_map<string, tensorflow::DeviceProperties>* dev
     // available device memory.
     const tensorflow::DeviceAttributes& attr = device->attributes();
     prop.set_memory_size(attr.memory_limit());
-    delete device;
   }
 }
 
-- 
GitLab


From dd9642c39d1468e0d8d0db66e3664414f1b510be Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 27 Nov 2018 10:27:27 -0800
Subject: [PATCH 0870/1554] Ensure all fuzzers fuzz the op they have to fuzz.

PiperOrigin-RevId: 223013893
---
 tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc | 2 +-
 tensorflow/core/kernels/fuzzing/fuzz_session.h        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc b/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc
index a8f07f4bad..b8d779fb13 100644
--- a/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc
+++ b/tensorflow/core/kernels/fuzzing/encode_base64_fuzz.cc
@@ -19,7 +19,7 @@ limitations under the License.
 namespace tensorflow {
 namespace fuzzing {
 
-class FuzzEncodeBase64 : public FuzzSession {
+class FuzzEncodeBase64 : public FuzzStringInputOp {
   SINGLE_INPUT_OP_BUILDER(DT_STRING, EncodeBase64);
 };
 
diff --git a/tensorflow/core/kernels/fuzzing/fuzz_session.h b/tensorflow/core/kernels/fuzzing/fuzz_session.h
index 9777be1ae8..57d562ddf4 100644
--- a/tensorflow/core/kernels/fuzzing/fuzz_session.h
+++ b/tensorflow/core/kernels/fuzzing/fuzz_session.h
@@ -72,11 +72,11 @@ class FuzzSession {
   // By convention, the graph should have inputs named "input1", ...
   // "inputN", and one output node, named "output".
   // Users of FuzzSession should override this method to create their graph.
-  virtual void BuildGraph(const Scope& scope) {}
+  virtual void BuildGraph(const Scope& scope) = 0;
 
   // Implements the logic that converts an opaque byte buffer
   // from the fuzzer to Tensor inputs to the graph.  Users must override.
-  virtual void FuzzImpl(const uint8_t* data, size_t size) {}
+  virtual void FuzzImpl(const uint8_t* data, size_t size) = 0;
 
   // Initializes the FuzzSession.  Not safe for multithreading.
   // Separate init function because the call to virtual BuildGraphDef
-- 
GitLab


From 411cc508e1a63d648855014f0912663e780a0ff9 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 27 Nov 2018 10:48:57 -0800
Subject: [PATCH 0871/1554] Optimize accesses to the "begin" and "end" tensors
 in `ValidateStridedSliceOp()`.

1. Hoist the call to `Tensor::flat<T>()` on the begin and end tensors out of the loop over dimensions. This avoids revalidating the tensor type, shape, and alignment once per dimension.
2. Use `Tensor::vec<T>()` instead of `Tensor::flat<T>()` because we have already checked that the corresponding input tensors are vectors, and the vector codepath is more efficient.

PiperOrigin-RevId: 223017951
---
 tensorflow/core/util/strided_slice_op.cc | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc
index ad8a44a518..55688e5808 100644
--- a/tensorflow/core/util/strided_slice_op.cc
+++ b/tensorflow/core/util/strided_slice_op.cc
@@ -83,10 +83,17 @@ static Status TF_MUST_USE_RESULT BuildDenseSpec(
   {
     int full_index = 0;
 
-    const auto& strides_flat = sparse.strides_tensor.flat<T>();
+    const T* const strides_flat = sparse.strides_tensor.vec<T>().data();
     dense->begin_valid = sparse.begin_tensor != nullptr;
     dense->end_valid = sparse.end_tensor != nullptr;
 
+    const T* const begin_flat = sparse.begin_tensor != nullptr
+                                    ? sparse.begin_tensor->vec<T>().data()
+                                    : nullptr;
+    const T* const end_flat = sparse.end_tensor != nullptr
+                                  ? sparse.end_tensor->vec<T>().data()
+                                  : nullptr;
+
     for (int i = 0; i < sparse.dims; i++) {
       if ((1 << i) & sparse.ellipsis_mask) {
         // Expand the ellipsis into the appropriate indices
@@ -112,16 +119,14 @@ static Status TF_MUST_USE_RESULT BuildDenseSpec(
         }
 
         // Gather slicing spec into appropriate index
-        if (sparse.begin_tensor != nullptr) {
-          const auto& begin_flat = sparse.begin_tensor->flat<T>();
-          dense->begin[full_index] = internal::SubtleMustCopy<T>(begin_flat(i));
+        if (begin_flat != nullptr) {
+          dense->begin[full_index] = internal::SubtleMustCopy<T>(begin_flat[i]);
         }
-        if (sparse.end_tensor != nullptr) {
-          const auto& end_flat = sparse.end_tensor->flat<T>();
-          dense->end[full_index] = internal::SubtleMustCopy<T>(end_flat(i));
+        if (end_flat != nullptr) {
+          dense->end[full_index] = internal::SubtleMustCopy<T>(end_flat[i]);
         }
         dense->strides[full_index] =
-            internal::SubtleMustCopy<T>(strides_flat(i));
+            internal::SubtleMustCopy<T>(strides_flat[i]);
         if (sparse.begin_mask & (1 << i)) {
           dense->begin_mask |= (1 << full_index);
         }
-- 
GitLab


From 06f3fdb6f14732ef9d4a5c50d2cb10ab605373ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 11:09:45 -0800
Subject: [PATCH 0872/1554] Add min-duration flags to benchmark_model:
 --min_secs, --warmup_min_secs. The existing flags --num_runs, --warmup_runs
 become only minima. Thus the old behavior is restored by passing 0.0 for both
 --min_secs, --warmup_min_secs. The default values --min_secs=1.0,
 --warmup_min_secs=0.5 reflect what it typically takes to get stable benchmark
 results on an Android device. The operating system won't ramp up to max clock
 speed in less than (order of magnitude) 100 ms, so warming up for less than a
 few hundreds ms results in inconsistent timings.

PiperOrigin-RevId: 223021882
---
 .../lite/tools/benchmark/benchmark_model.cc   | 49 ++++++++++++++-----
 .../lite/tools/benchmark/benchmark_model.h    |  3 +-
 .../lite/tools/benchmark/benchmark_test.cc    |  2 +
 3 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_model.cc b/tensorflow/lite/tools/benchmark/benchmark_model.cc
index 05148aea65..e9b485efca 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_model.cc
@@ -51,11 +51,13 @@ using tensorflow::Stat;
 BenchmarkParams BenchmarkModel::DefaultParams() {
   BenchmarkParams params;
   params.AddParam("num_runs", BenchmarkParam::Create<int32_t>(50));
+  params.AddParam("min_secs", BenchmarkParam::Create<float>(1.0f));
   params.AddParam("run_delay", BenchmarkParam::Create<float>(-1.0f));
   params.AddParam("num_threads", BenchmarkParam::Create<int32_t>(1));
   params.AddParam("benchmark_name", BenchmarkParam::Create<std::string>(""));
   params.AddParam("output_prefix", BenchmarkParam::Create<std::string>(""));
   params.AddParam("warmup_runs", BenchmarkParam::Create<int32_t>(1));
+  params.AddParam("warmup_min_secs", BenchmarkParam::Create<float>(0.5f));
   return params;
 }
 
@@ -73,19 +75,34 @@ void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults &results) {
 
 std::vector<Flag> BenchmarkModel::GetFlags() {
   return {
-      CreateFlag<int32_t>("num_runs", &params_, "number of runs"),
+      CreateFlag<int32_t>("num_runs", &params_,
+                          "minimum number of runs, see also min_secs"),
+      CreateFlag<float>(
+          "min_secs", &params_,
+          "minimum number of seconds to rerun for, potentially making the "
+          "actual number of runs to be greater than num_runs"),
       CreateFlag<float>("run_delay", &params_, "delay between runs in seconds"),
       CreateFlag<int32_t>("num_threads", &params_, "number of threads"),
       CreateFlag<std::string>("benchmark_name", &params_, "benchmark name"),
       CreateFlag<std::string>("output_prefix", &params_,
                               "benchmark output prefix"),
-      CreateFlag<int32_t>("warmup_runs", &params_,
-                          "how many runs to initialize model"),
+      CreateFlag<int32_t>(
+          "warmup_runs", &params_,
+          "minimum number of runs performed on initialization, to "
+          "allow performance characteristics to settle, see also "
+          "warmup_min_secs"),
+      CreateFlag<float>(
+          "warmup_min_secs", &params_,
+          "minimum number of seconds to rerun for, potentially making the "
+          "actual number of warm-up runs to be greater than warmup_runs"),
   };
 }
 
 void BenchmarkModel::LogParams() {
-  TFLITE_LOG(INFO) << "Num runs: [" << params_.Get<int32_t>("num_runs") << "]";
+  TFLITE_LOG(INFO) << "Min num runs: [" << params_.Get<int32_t>("num_runs")
+                   << "]";
+  TFLITE_LOG(INFO) << "Min runs duration (seconds): ["
+                   << params_.Get<float>("min_secs") << "]";
   TFLITE_LOG(INFO) << "Inter-run delay (seconds): ["
                    << params_.Get<float>("run_delay") << "]";
   TFLITE_LOG(INFO) << "Num threads: [" << params_.Get<int32_t>("num_threads")
@@ -94,16 +111,24 @@ void BenchmarkModel::LogParams() {
                    << params_.Get<std::string>("benchmark_name") << "]";
   TFLITE_LOG(INFO) << "Output prefix: ["
                    << params_.Get<std::string>("output_prefix") << "]";
-  TFLITE_LOG(INFO) << "Warmup runs: [" << params_.Get<int32_t>("warmup_runs")
-                   << "]";
+  TFLITE_LOG(INFO) << "Min warmup runs: ["
+                   << params_.Get<int32_t>("warmup_runs") << "]";
+  TFLITE_LOG(INFO) << "Min warmup runs duration (seconds): ["
+                   << params_.Get<float>("warmup_min_secs") << "]";
 }
 
 void BenchmarkModel::PrepareInputsAndOutputs() {}
 
-Stat<int64_t> BenchmarkModel::Run(int num_times, RunType run_type) {
+Stat<int64_t> BenchmarkModel::Run(int min_num_times, float min_secs,
+                                  RunType run_type) {
   Stat<int64_t> run_stats;
-  TFLITE_LOG(INFO) << "Running benchmark for " << num_times << " iterations ";
-  for (int run = 0; run < num_times; run++) {
+  TFLITE_LOG(INFO) << "Running benchmark for at least " << min_num_times
+                   << " iterations and at least " << min_secs << " seconds";
+  int64_t min_finish_us =
+      profiling::time::NowMicros() + static_cast<int64_t>(min_secs * 1.e6f);
+  for (int run = 0;
+       run < min_num_times || profiling::time::NowMicros() < min_finish_us;
+       run++) {
     PrepareInputsAndOutputs();
     listeners_.OnSingleRunStart(run_type);
     int64_t start_us = profiling::time::NowMicros();
@@ -145,9 +170,11 @@ void BenchmarkModel::Run() {
 
   uint64_t input_bytes = ComputeInputBytes();
   Stat<int64_t> warmup_time_us =
-      Run(params_.Get<int32_t>("warmup_runs"), WARMUP);
+      Run(params_.Get<int32_t>("warmup_runs"),
+          params_.Get<float>("warmup_min_secs"), WARMUP);
   Stat<int64_t> inference_time_us =
-      Run(params_.Get<int32_t>("num_runs"), REGULAR);
+      Run(params_.Get<int32_t>("num_runs"), params_.Get<float>("min_secs"),
+          REGULAR);
   listeners_.OnBenchmarkEnd(
       {startup_latency_us, input_bytes, warmup_time_us, inference_time_us});
 }
diff --git a/tensorflow/lite/tools/benchmark/benchmark_model.h b/tensorflow/lite/tools/benchmark/benchmark_model.h
index d8a9b05010..31ee5c92aa 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_model.h
+++ b/tensorflow/lite/tools/benchmark/benchmark_model.h
@@ -150,7 +150,8 @@ class BenchmarkModel {
   bool ParseFlags(int argc, char** argv);
   virtual std::vector<Flag> GetFlags();
   virtual uint64_t ComputeInputBytes() = 0;
-  virtual tensorflow::Stat<int64_t> Run(int num_times, RunType run_type);
+  virtual tensorflow::Stat<int64_t> Run(int min_num_times, float min_secs,
+                                        RunType run_type);
   virtual void PrepareInputsAndOutputs();
   virtual void RunImpl() = 0;
   BenchmarkParams params_;
diff --git a/tensorflow/lite/tools/benchmark/benchmark_test.cc b/tensorflow/lite/tools/benchmark/benchmark_test.cc
index 59d23d9008..8191fbcd73 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_test.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_test.cc
@@ -33,6 +33,7 @@ namespace {
 BenchmarkParams CreateParams() {
   BenchmarkParams params;
   params.AddParam("num_runs", BenchmarkParam::Create<int32_t>(2));
+  params.AddParam("min_secs", BenchmarkParam::Create<float>(1.0f));
   params.AddParam("run_delay", BenchmarkParam::Create<float>(-1.0f));
   params.AddParam("num_threads", BenchmarkParam::Create<int32_t>(1));
   params.AddParam("benchmark_name", BenchmarkParam::Create<std::string>(""));
@@ -42,6 +43,7 @@ BenchmarkParams CreateParams() {
   params.AddParam("input_layer", BenchmarkParam::Create<std::string>(""));
   params.AddParam("input_layer_shape", BenchmarkParam::Create<std::string>(""));
   params.AddParam("use_nnapi", BenchmarkParam::Create<bool>(false));
+  params.AddParam("warmup_min_secs", BenchmarkParam::Create<float>(0.5f));
   return params;
 }
 
-- 
GitLab


From 7ac49ea7f13fd02b1e8be036ff5b42641ff5c627 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 27 Nov 2018 11:15:06 -0800
Subject: [PATCH 0873/1554] [TF:XLA] Make DynamicStitch work for index tensors
 with zero elements

PiperOrigin-RevId: 223022861
---
 .../compiler/tests/dynamic_stitch_test.py     |  9 +++++++
 .../tf2xla/kernels/dynamic_stitch_op.cc       | 25 +++++++++++++------
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/tests/dynamic_stitch_test.py b/tensorflow/compiler/tests/dynamic_stitch_test.py
index 50b04daa6b..e89cf975f5 100644
--- a/tensorflow/compiler/tests/dynamic_stitch_test.py
+++ b/tensorflow/compiler/tests/dynamic_stitch_test.py
@@ -58,6 +58,15 @@ class DynamicStitchTest(xla_test.XLATestCase):
         [idx1, idx2], [val1, val2],
         expected=np.array([[], [], [], []], np.int32))
 
+  def testEmptyIndex(self):
+    idx1 = np.array([], dtype=np.int32)
+    idx2 = np.array([[], []], dtype=np.int32)
+    val1 = np.ndarray(shape=(0, 9), dtype=np.int32)
+    val2 = np.ndarray(shape=(2, 0, 9), dtype=np.int32)
+    self._AssertDynamicStitchResultIs([idx1, idx2], [val1, val2],
+                                      expected=np.ndarray(
+                                          shape=(0, 9), dtype=np.int32))
+
   def testSimple1D(self):
     val1 = np.array([0, 4, 7], dtype=np.int32)
     val2 = np.array([1, 6, 2, 3, 5], dtype=np.int32)
diff --git a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
index b2f6ef43fa..6e6ba21daf 100644
--- a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
@@ -113,8 +113,20 @@ class DynamicStitchOp : public XlaOpKernel {
       }
     }
     int number_of_indices = max_index + 1;
-    OP_REQUIRES(ctx, number_of_indices > 0,
-                errors::InvalidArgument("no indices supplied"));
+    int64 result_rank = 1 + data0_shape.dims() - indices0_shape.dims();
+    if (number_of_indices == 0) {
+      std::vector<int64> result_shape(result_rank);
+      for (int d = indices0_shape.dims(); d < data0_shape.dims(); d++) {
+        result_shape[d - indices0_shape.dims() + 1] = data0_shape.dim_size(d);
+      }
+      xla::PrimitiveType element_type =
+          ctx->input_xla_type(ctx->num_inputs() - 1);
+      xla::Literal empty_literal = xla::Literal::CreateFromShape(
+          xla::ShapeUtil::MakeShape(element_type, result_shape));
+      ctx->SetOutput(0, xla::ConstantLiteral(ctx->builder(), empty_literal));
+      return;
+    }
+
     // Construct the reverse mapping, for each index, of which slice of which
     // input it comes from.
     std::vector<int32> src_input_vector(number_of_indices);
@@ -157,12 +169,9 @@ class DynamicStitchOp : public XlaOpKernel {
 
     // Set up the vectors for slicing: the first dimension will vary
     // slice by slice, and the rest take the full common extra shape.
-    std::vector<int64> slice_start(1 + data0_shape.dims() -
-                                   indices0_shape.dims());
-    std::vector<int64> slice_limit(1 + data0_shape.dims() -
-                                   indices0_shape.dims());
-    std::vector<int64> stride(1 + data0_shape.dims() - indices0_shape.dims(),
-                              1);
+    std::vector<int64> slice_start(result_rank);
+    std::vector<int64> slice_limit(result_rank);
+    std::vector<int64> stride(result_rank, 1);
     for (int d = indices0_shape.dims(); d < data0_shape.dims(); d++) {
       slice_limit[1 + d - indices0_shape.dims()] = data0_shape.dim_size(d);
     }
-- 
GitLab


From 5bb914349a484f25845a2e77fb1b989760a5f64f Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 27 Nov 2018 11:46:28 -0800
Subject: [PATCH 0874/1554] Ensure default args of some functions are similar
 in TF 1.0 and TF 2.0.

PiperOrigin-RevId: 223029699
---
 tensorflow/python/ops/math_ops.py | 12 ++++++------
 tensorflow/python/ops/nn_ops.py   |  4 ++++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index d6b2423c1b..c3feb1870d 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -82,8 +82,6 @@ def argmax(input,
            output_type=dtypes.int64):
   axis = deprecation.deprecated_argument_lookup(
       "axis", axis, "dimension", dimension)
-  if axis is None:
-    axis = 0
   return argmax_v2(input, axis, output_type, name)
 
 
@@ -111,6 +109,8 @@ def argmax_v2(input,
   Returns:
     A `Tensor` of type `output_type`.
   """
+  if axis is None:
+    axis = 0
   return gen_math_ops.arg_max(input, axis, name=name, output_type=output_type)
 
 
@@ -127,8 +127,6 @@ def argmin(input,
            output_type=dtypes.int64):
   axis = deprecation.deprecated_argument_lookup(
       "axis", axis, "dimension", dimension)
-  if axis is None:
-    axis = 0
   return argmin_v2(input, axis, output_type, name)
 
 
@@ -156,6 +154,8 @@ def argmin_v2(input,
   Returns:
     A `Tensor` of type `output_type`.
   """
+  if axis is None:
+    axis = 0
   return gen_math_ops.arg_min(input, axis, name=name, output_type=output_type)
 
 
@@ -1521,8 +1521,6 @@ def count_nonzero(input_tensor,
       "axis", axis,
       "reduction_indices", reduction_indices
       )
-  if keepdims is None:
-    keepdims = False
 
   return count_nonzero_v2(input_tensor, axis, keepdims, dtype, name)
 
@@ -1580,6 +1578,8 @@ def count_nonzero_v2(input,  # pylint: disable=redefined-builtin
   Returns:
     The reduced tensor (number of nonzero values).
   """
+  if keepdims is None:
+    keepdims = False
   with ops.name_scope(name, "count_nonzero", [input]):
     input = ops.convert_to_tensor(input, name="input")
     # A scalar of 'zero' is enough as `not_equal` will broadcast.
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 2596f8e9bc..2ffe381216 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2237,6 +2237,8 @@ def softmax_v2(logits, axis=None, name=None):
     InvalidArgumentError: if `logits` is empty or `axis` is beyond the last
       dimension of `logits`.
   """
+  if axis is None:
+    axis = -1
   return _softmax(logits, gen_nn_ops.softmax, axis, name)
 
 
@@ -2292,6 +2294,8 @@ def log_softmax_v2(logits, axis=None, name=None):
     InvalidArgumentError: if `logits` is empty or `axis` is beyond the last
       dimension of `logits`.
   """
+  if axis is None:
+    axis = -1
   return _softmax(logits, gen_nn_ops.log_softmax, axis, name)
 
 
-- 
GitLab


From b186f4dc0da5c437a767e128e788b5ffc595583f Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 27 Nov 2018 11:49:18 -0800
Subject: [PATCH 0875/1554] [XLA] Make no_duplicate literal generation better
 supported for half/bfloat16

PiperOrigin-RevId: 223030267
---
 tensorflow/compiler/xla/tests/BUILD           |   1 +
 tensorflow/compiler/xla/tests/test_utils.cc   | 146 ++++++++++++------
 .../compiler/xla/tests/test_utils_test.cc     |  23 +++
 3 files changed, 119 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index d92255dc48..2c18e2fd10 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -79,6 +79,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_verifier",
         "//tensorflow/compiler/xla/service:transfer_manager",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/base",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/types:span",
     ],
diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc
index 2f18036ff4..8b4be6607f 100644
--- a/tensorflow/compiler/xla/tests/test_utils.cc
+++ b/tensorflow/compiler/xla/tests/test_utils.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <cmath>
 
+#include "absl/base/casts.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -28,65 +29,112 @@ namespace xla {
 namespace {
 
 template <typename FloatT, typename GeneratorT>
-void PopulateWithRandomFloatingPointDataImpl(Literal* literal,
-                                             std::minstd_rand0* engine,
-                                             bool no_duplicates) {
+void PopulateWithRandomFloatingPointData(Literal* literal,
+                                         std::minstd_rand0* engine) {
+  std::uniform_real_distribution<GeneratorT> generator(-0.1f, 0.2f);
+  for (FloatT& value : literal->data<FloatT>()) {
+    value = static_cast<FloatT>(generator(*engine));
+  }
+}
+
+template <typename FloatT>
+void PopulateWithIntNext(Literal* literal);
+
+template <>
+void PopulateWithIntNext<half>(Literal* literal) {
+  // Duplicates may be generated if we don't have enough bits.
+  uint16 next_value = 0;
+  for (half& value : literal->data<half>()) {
+    // Zero-out the MSB of the exponent to avoid Infs and NaNs, and put it into
+    // the sign bit. We could be less wasteful, but this is best-effort anyway.
+    uint16 exponent_msb = next_value & 0x4000;
+    value.x = (next_value & 0xBFFF) | (exponent_msb << 1);
+    next_value++;
+  }
+}
+
+template <>
+void PopulateWithIntNext<bfloat16>(Literal* literal) {
+  // Duplicates may be generated if we don't have enough bits.
+  uint16 next_value = 0;
+  for (bfloat16& value : literal->data<bfloat16>()) {
+    // Zero-out the MSB of the exponent to avoid Infs and NaNs, and put it into
+    // the sign bit. We could be less wasteful, but this is best-effort anyway.
+    uint16 exponent_msb = next_value & 0x4000;
+    value.value = (next_value & 0xBFFF) | (exponent_msb << 1);
+    next_value++;
+  }
+}
+
+template <typename FloatT>
+void PopulateWithNextAfter(Literal* literal) {
+  // Duplicates may be generated if the number of elements in the literal
+  // exceeds the number of positive values supported by the type.
+  float next_value = std::numeric_limits<float>::min();
+  for (float& value : literal->data<float>()) {
+    value = next_value;
+    next_value = std::nextafter(next_value, std::numeric_limits<float>::max());
+  }
+}
+
+template <typename FloatT,
+          typename std::enable_if<std::is_same<bfloat16, FloatT>::value ||
+                                      std::is_same<half, FloatT>::value,
+                                  int>::type = 0>
+void PopulateWithNoDuplicateData(Literal* literal, std::minstd_rand0* engine) {
+  PopulateWithIntNext<FloatT>(literal);
+  std::shuffle(literal->data<FloatT>().begin(), literal->data<FloatT>().end(),
+               *engine);
+}
+
+template <typename FloatT,
+          typename std::enable_if<!std::is_same<bfloat16, FloatT>::value &&
+                                      !std::is_same<half, FloatT>::value,
+                                  int>::type = 0>
+void PopulateWithNoDuplicateData(Literal* literal, std::minstd_rand0* engine) {
+  PopulateWithNextAfter<FloatT>(literal);
+  std::shuffle(literal->data<FloatT>().begin(), literal->data<FloatT>().end(),
+               *engine);
+}
+
+template <typename FloatT>
+void PopulateWithFloatingPointData(Literal* literal, std::minstd_rand0* engine,
+                                   bool no_duplicates) {
   CHECK(engine != nullptr);
   CHECK_EQ(literal->shape().element_type(),
            primitive_util::NativeToPrimitiveType<FloatT>());
   if (no_duplicates) {
-    // Duplicates may be generated if the number of elements in the literal
-    // exceeds the number of positive values supported by the type.
-    FloatT next_value = std::numeric_limits<FloatT>::min();
-    for (FloatT& value : literal->data<FloatT>()) {
-      value = next_value;
-      next_value =
-          std::nextafter(next_value, std::numeric_limits<FloatT>::max());
-    }
-    std::shuffle(literal->data<FloatT>().begin(), literal->data<FloatT>().end(),
-                 *engine);
+    PopulateWithNoDuplicateData<FloatT>(literal, engine);
   } else {
-    std::uniform_real_distribution<GeneratorT> generator(-0.1f, 0.2f);
-    for (FloatT& value : literal->data<FloatT>()) {
-      value = static_cast<FloatT>(generator(*engine));
-    }
+    PopulateWithRandomFloatingPointData<FloatT, FloatT>(literal, engine);
   }
 }
 
-template <typename FloatT>
-void PopulateWithRandomFloatingPointData(Literal* literal,
+template <>
+void PopulateWithFloatingPointData<half>(Literal* literal,
                                          std::minstd_rand0* engine,
                                          bool no_duplicates) {
   CHECK(engine != nullptr);
-  PopulateWithRandomFloatingPointDataImpl<FloatT, FloatT>(literal, engine,
-                                                          no_duplicates);
-}
-
-template <>
-void PopulateWithRandomFloatingPointData<half>(Literal* literal,
-                                               std::minstd_rand0* engine,
-                                               bool no_duplicates) {
-  // no_duplicates is ignored for half types. Unique values can only be
-  // generated for arrays with fewer than ~2**16 elements and no_duplicates is
-  // best-effort anyway.
-  CHECK(engine != nullptr);
-  std::uniform_real_distribution<float> generator(-0.1f, 0.2f);
-  for (half& value : literal->data<half>()) {
-    value = static_cast<half>(generator(*engine));
+  CHECK_EQ(literal->shape().element_type(),
+           primitive_util::NativeToPrimitiveType<half>());
+  if (no_duplicates) {
+    PopulateWithNoDuplicateData<half>(literal, engine);
+  } else {
+    PopulateWithRandomFloatingPointData<half, float>(literal, engine);
   }
 }
 
 template <>
-void PopulateWithRandomFloatingPointData<bfloat16>(Literal* literal,
-                                                   std::minstd_rand0* engine,
-                                                   bool no_duplicates) {
-  // no_duplicates is ignored for bfloat types. Unique values can only be
-  // generated for arrays with fewer than ~2**16 elements and no_duplicates is
-  // best-effort anyway.
+void PopulateWithFloatingPointData<bfloat16>(Literal* literal,
+                                             std::minstd_rand0* engine,
+                                             bool no_duplicates) {
   CHECK(engine != nullptr);
-  std::uniform_real_distribution<float> generator(-0.1f, 0.2f);
-  for (bfloat16& value : literal->data<bfloat16>()) {
-    value = static_cast<bfloat16>(generator(*engine));
+  CHECK_EQ(literal->shape().element_type(),
+           primitive_util::NativeToPrimitiveType<bfloat16>());
+  if (no_duplicates) {
+    PopulateWithNoDuplicateData<bfloat16>(literal, engine);
+  } else {
+    PopulateWithRandomFloatingPointData<bfloat16, float>(literal, engine);
   }
 }
 
@@ -135,20 +183,16 @@ StatusOr<Literal> MakeFakeLiteralInternal(const Shape& shape,
   Literal literal(shape);
   switch (shape.element_type()) {
     case BF16:
-      PopulateWithRandomFloatingPointData<bfloat16>(&literal, engine,
-                                                    no_duplicates);
+      PopulateWithFloatingPointData<bfloat16>(&literal, engine, no_duplicates);
       break;
     case F16:
-      PopulateWithRandomFloatingPointData<half>(&literal, engine,
-                                                no_duplicates);
+      PopulateWithFloatingPointData<half>(&literal, engine, no_duplicates);
       break;
     case F32:
-      PopulateWithRandomFloatingPointData<float>(&literal, engine,
-                                                 no_duplicates);
+      PopulateWithFloatingPointData<float>(&literal, engine, no_duplicates);
       break;
     case F64:
-      PopulateWithRandomFloatingPointData<double>(&literal, engine,
-                                                  no_duplicates);
+      PopulateWithFloatingPointData<double>(&literal, engine, no_duplicates);
       break;
     case S8:
       PopulateWithRandomIntegralData<int8>(&literal, engine, no_duplicates);
diff --git a/tensorflow/compiler/xla/tests/test_utils_test.cc b/tensorflow/compiler/xla/tests/test_utils_test.cc
index e066b3f4f2..e8f5d7a9a7 100644
--- a/tensorflow/compiler/xla/tests/test_utils_test.cc
+++ b/tensorflow/compiler/xla/tests/test_utils_test.cc
@@ -175,5 +175,28 @@ ENTRY %sort.148.1589 (parameter.0: s32[1048576], parameter.1: s32[1048576]) -> (
   }
 }
 
+XLA_TEST_F(TestUtilsTest, NoDuplicatesBfloat16) {
+  // Inputs which are sort keys in key/value sorts should have no duplicates.
+  auto module = ParseHloString(R"(
+HloModule sort, is_scheduled=true
+
+ENTRY %sort. (parameter.0: bf16[2,1452], parameter.1: s32[2,1452]) -> (bf16[2,1452], s32[2,1452]) {
+  %parameter.0 = bf16[2,1452]{1,0} parameter(0)
+  %parameter.1 = s32[2,1452]{1,0} parameter(1)
+  ROOT %sort = (bf16[2,1452]{1,0}, s32[2,1452]{1,0}) sort(bf16[2,1452]{1,0} %parameter.0, s32[2,1452]{1,0} %parameter.1), dimensions={1}
+}
+)")
+                    .ValueOrDie();
+  TF_ASSERT_OK_AND_ASSIGN(std::vector<Literal> args,
+                          MakeFakeArguments(module.get()));
+  ASSERT_EQ(args.size(), 2);
+  const Literal& key_arg = args[0];
+
+  absl::flat_hash_set<uint16> key_set;
+  for (const bfloat16& value : key_arg.data<bfloat16>()) {
+    EXPECT_TRUE(key_set.insert(absl::bit_cast<uint16>(value)).second);
+  }
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 6cef228475b10d655eabecfcdb365eaeb6383cf0 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 27 Nov 2018 12:00:28 -0800
Subject: [PATCH 0876/1554] [XLA:CPU] Fix symbol resolution problem on Mac OS
 X.

Mac OS X prefixes symbols with a leading underscore (e.g. "_expf") but the names used in the XLA call registry are unprefixed (e.g., "expf"). The SimpleOrcJit symbol resolver class seems to receive the prefixed version from LLVM. To work around the problem, in the event of a failed lookup retry lookups with the leading underscore stripped.

PiperOrigin-RevId: 223032111
---
 .../compiler/xla/service/cpu/simple_orc_jit.cc       | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index f77641eb7d..efccadedf2 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -128,8 +128,18 @@ SimpleOrcJIT::SimpleOrcJIT(const llvm::TargetOptions& target_options,
 }
 
 llvm::JITSymbol SimpleOrcJIT::ResolveRuntimeSymbol(const std::string& name) {
-  void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name);
+  void* func_addr = nullptr;
+  if (name.size() > 1 && name.front() == data_layout_.getGlobalPrefix()) {
+    // On Mac OS X, 'name' may have a leading underscore prefix, even though the
+    // registered name may not.
+    std::string stripped_name(name.begin() + 1, name.end());
+    func_addr = CustomCallTargetRegistry::Global()->Lookup(stripped_name);
+  } else {
+    func_addr = CustomCallTargetRegistry::Global()->Lookup(name);
+  }
+
   if (func_addr == nullptr) {
+    VLOG(2) << "Unable to resolve runtime symbol: " << name;
     return nullptr;
   }
   llvm::JITEvaluatedSymbol symbol_info(reinterpret_cast<uint64_t>(func_addr),
-- 
GitLab


From e165762a36738570b224f4bd84e593093264cf56 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 27 Nov 2018 12:15:23 -0800
Subject: [PATCH 0877/1554] Updated `thresholds` argument in confusion matrix
 metrics to accept scalar input and set the default value to be a scalar
 instead of a list. This way the metric result will be a scalar if input
 thresholds is a scalar and a list otherwise.

PiperOrigin-RevId: 223035043
---
 tensorflow/python/keras/metrics.py      | 116 +++++++++++++-----------
 tensorflow/python/keras/metrics_test.py |  44 +++------
 2 files changed, 74 insertions(+), 86 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 60632e5a25..90babf38fd 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -48,6 +48,7 @@ from tensorflow.python.keras.losses import sparse_categorical_crossentropy
 from tensorflow.python.keras.losses import squared_hinge
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
@@ -215,7 +216,8 @@ def _update_confusion_matrix_variables(variables_to_update,
     y_true: A `Tensor` whose shape matches `y_pred`. Will be cast to `bool`.
     y_pred: A floating point `Tensor` of arbitrary shape and whose values are in
       the range `[0, 1]`.
-    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
+    thresholds: A float value or a python list or tuple of float thresholds in
+      `[0, 1]`.
     sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
       `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must
       be either `1`, or the same as the corresponding `y_true` dimension).
@@ -264,6 +266,7 @@ def _update_confusion_matrix_variables(variables_to_update,
         math_ops.cast(y_pred, dtype=dtypes.float32),
         math_ops.cast(y_true, dtype=dtypes.bool), sample_weight)
 
+  thresholds = to_list(thresholds)
   num_thresholds = len(thresholds)
   num_predictions = array_ops.size(y_pred)
 
@@ -868,21 +871,22 @@ class _ConfusionMatrixConditionCount(Metric):
 
     Args:
       confusion_matrix_cond: One of `_ConfusionMatrix` conditions.
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(_ConfusionMatrixConditionCount, self).__init__(name=name, dtype=dtype)
     self._confusion_matrix_cond = confusion_matrix_cond
-    self.thresholds = [0.5] if thresholds is None else thresholds
-    _assert_thresholds_range(self.thresholds)
+    self.thresholds = 0.5 if thresholds is None else thresholds
+    thresholds = to_list(thresholds)
+    _assert_thresholds_range(thresholds)
     self.accumulator = self.add_weight(
         'accumulator',
-        shape=(len(self.thresholds),),
+        shape=(len(thresholds),),
         initializer=init_ops.zeros_initializer)
 
   def update_state(self, y_true, y_pred, sample_weight=None):
@@ -903,7 +907,11 @@ class _ConfusionMatrixConditionCount(Metric):
     }, y_true, y_pred, self.thresholds, sample_weight)
 
   def result(self):
-    return ops.convert_to_tensor(self.accumulator)
+    if isinstance(self.thresholds, (list, tuple)):
+      result = self.accumulator
+    else:
+      result = self.accumulator[0]
+    return ops.convert_to_tensor(result)
 
 
 @tf_export('metrics.FalsePositives', 'keras.metrics.FalsePositives')
@@ -941,11 +949,11 @@ class FalsePositives(_ConfusionMatrixConditionCount):
     """Creates a `FalsePositives` instance.
 
     Args:
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
@@ -991,11 +999,11 @@ class FalseNegatives(_ConfusionMatrixConditionCount):
     """Creates a `FalseNegatives` instance.
 
     Args:
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
@@ -1041,11 +1049,11 @@ class TrueNegatives(_ConfusionMatrixConditionCount):
     """Creates a `TrueNegatives` instance.
 
     Args:
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
@@ -1091,11 +1099,11 @@ class TruePositives(_ConfusionMatrixConditionCount):
     """Creates a `TruePositives` instance.
 
     Args:
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
@@ -1142,23 +1150,25 @@ class Precision(Metric):
     """Creates a `Precision` instance.
 
     Args:
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(Precision, self).__init__(name=name, dtype=dtype)
-    self.thresholds = [0.5] if thresholds is None else thresholds
+    self.thresholds = 0.5 if thresholds is None else thresholds
+    thresholds = to_list(thresholds)
+    _assert_thresholds_range(thresholds)
     self.tp = self.add_weight(
         'true_positives',
-        shape=(len(self.thresholds),),
+        shape=(len(thresholds),),
         initializer=init_ops.zeros_initializer)
     self.fp = self.add_weight(
         'false_positives',
-        shape=(len(self.thresholds),),
+        shape=(len(thresholds),),
         initializer=init_ops.zeros_initializer)
 
   def update_state(self, y_true, y_pred, sample_weight=None):
@@ -1180,10 +1190,8 @@ class Precision(Metric):
     }, y_true, y_pred, self.thresholds, sample_weight)
 
   def result(self):
-    return array_ops.where(
-        math_ops.greater(self.tp + self.fp, 0),
-        math_ops.div(self.tp, self.tp + self.fp),
-        array_ops.zeros_like(self.thresholds))
+    result = math_ops.div_no_nan(self.tp, self.tp + self.fp)
+    return result if isinstance(self.thresholds, (list, tuple)) else result[0]
 
 
 @tf_export('metrics.Recall', 'keras.metrics.Recall')
@@ -1222,23 +1230,25 @@ class Recall(Metric):
     """Creates a `Recall` instance.
 
     Args:
-      thresholds: (Optional) Defaults to [0.5]. A python list/tuple of float
-        threshold values in [0, 1]. A threshold is compared with prediction
-        values to determine the truth value of predictions (i.e., above the
-        threshold is `true`, below is `false`). One metric value is generated
-        for each threshold value.
+      thresholds: (Optional) Defaults to 0.5. A float value or a python
+        list/tuple of float threshold values in [0, 1]. A threshold is compared
+        with prediction values to determine the truth value of predictions
+        (i.e., above the threshold is `true`, below is `false`). One metric
+        value is generated for each threshold value.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(Recall, self).__init__(name=name, dtype=dtype)
-    self.thresholds = [0.5] if thresholds is None else thresholds
+    self.thresholds = 0.5 if thresholds is None else thresholds
+    thresholds = to_list(thresholds)
+    _assert_thresholds_range(thresholds)
     self.tp = self.add_weight(
         'true_positives',
-        shape=(len(self.thresholds),),
+        shape=(len(thresholds),),
         initializer=init_ops.zeros_initializer)
     self.fn = self.add_weight(
         'false_negatives',
-        shape=(len(self.thresholds),),
+        shape=(len(thresholds),),
         initializer=init_ops.zeros_initializer)
 
   def update_state(self, y_true, y_pred, sample_weight=None):
@@ -1260,10 +1270,8 @@ class Recall(Metric):
     }, y_true, y_pred, self.thresholds, sample_weight)
 
   def result(self):
-    return array_ops.where(
-        math_ops.greater(self.tp + self.fn, 0),
-        math_ops.div(self.tp, self.tp + self.fn),
-        array_ops.zeros_like(self.thresholds))
+    result = math_ops.div_no_nan(self.tp, self.tp + self.fn)
+    return result if isinstance(self.thresholds, (list, tuple)) else result[0]
 
 
 @six.add_metaclass(abc.ABCMeta)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index f049b10721..eda434391e 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -548,7 +548,7 @@ class FalsePositivesTest(test.TestCase):
     update_op = fp_obj.update_state(y_true, y_pred)
     self.evaluate(update_op)
     result = fp_obj.result()
-    self.assertAllClose([7.], result)
+    self.assertAllClose(7., result)
 
   def test_weighted(self):
     fp_obj = metrics.FalsePositives()
@@ -559,7 +559,7 @@ class FalsePositivesTest(test.TestCase):
                                    (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
     sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
     result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([14.], self.evaluate(result))
+    self.assertAllClose(14., self.evaluate(result))
 
   def test_unweighted_with_thresholds(self):
     fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
@@ -617,7 +617,7 @@ class FalseNegativesTest(test.TestCase):
     update_op = fn_obj.update_state(y_true, y_pred)
     self.evaluate(update_op)
     result = fn_obj.result()
-    self.assertAllClose([3.], result)
+    self.assertAllClose(3., result)
 
   def test_weighted(self):
     fn_obj = metrics.FalseNegatives()
@@ -628,7 +628,7 @@ class FalseNegativesTest(test.TestCase):
                                    (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
     sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
     result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([5.], self.evaluate(result))
+    self.assertAllClose(5., self.evaluate(result))
 
   def test_unweighted_with_thresholds(self):
     fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
@@ -679,7 +679,7 @@ class TrueNegativesTest(test.TestCase):
     update_op = tn_obj.update_state(y_true, y_pred)
     self.evaluate(update_op)
     result = tn_obj.result()
-    self.assertAllClose([3.], result)
+    self.assertAllClose(3., result)
 
   def test_weighted(self):
     tn_obj = metrics.TrueNegatives()
@@ -690,7 +690,7 @@ class TrueNegativesTest(test.TestCase):
                                    (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
     sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
     result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([4.], self.evaluate(result))
+    self.assertAllClose(4., self.evaluate(result))
 
   def test_unweighted_with_thresholds(self):
     tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
@@ -741,7 +741,7 @@ class TruePositivesTest(test.TestCase):
     update_op = tp_obj.update_state(y_true, y_pred)
     self.evaluate(update_op)
     result = tp_obj.result()
-    self.assertAllClose([7.], result)
+    self.assertAllClose(7., result)
 
   def test_weighted(self):
     tp_obj = metrics.TruePositives()
@@ -752,7 +752,7 @@ class TruePositivesTest(test.TestCase):
                                    (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
     sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
     result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([12.], self.evaluate(result))
+    self.assertAllClose(12., self.evaluate(result))
 
   def test_unweighted_with_thresholds(self):
     tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
@@ -857,7 +857,7 @@ class PrecisionTest(test.TestCase):
     self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
 
   def test_weighted_with_threshold(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 1.1])
+    p_obj = metrics.Precision(thresholds=[0.5, 1.])
     y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
     y_pred = constant_op.constant([[1, 0], [0.6, 0]],
                                   shape=(2, 2),
@@ -872,18 +872,8 @@ class PrecisionTest(test.TestCase):
     expected_precision = weighted_tp / weighted_positives
     self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)
 
-  def test_extreme_thresholds(self):
-    p_obj = metrics.Precision(thresholds=[-1.0, 2.0])  # beyond values range
-    y_pred = math_ops.cast(
-        constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32)
-    y_true = math_ops.cast(
-        constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertArrayNear([0.75, 0.], self.evaluate(result), 0)
-
   def test_multiple_updates(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 1.1])
+    p_obj = metrics.Precision(thresholds=[0.5, 1.])
     y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
     y_pred = constant_op.constant([[1, 0], [0.6, 0]],
                                   shape=(2, 2),
@@ -978,7 +968,7 @@ class RecallTest(test.TestCase):
     self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
 
   def test_weighted_with_threshold(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 1.1])
+    r_obj = metrics.Recall(thresholds=[0.5, 1.])
     y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
     y_pred = constant_op.constant([[1, 0], [0.6, 0]],
                                   shape=(2, 2),
@@ -993,18 +983,8 @@ class RecallTest(test.TestCase):
     expected_recall = weighted_tp / weighted_positives
     self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
 
-  def test_extreme_thresholds(self):
-    r_obj = metrics.Recall(thresholds=[-1.0, 2.0])  # beyond values range
-    y_pred = math_ops.cast(
-        constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32)
-    y_true = math_ops.cast(
-        constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertArrayNear([1.0, 0.], self.evaluate(result), 0)
-
   def test_multiple_updates(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 1.1])
+    r_obj = metrics.Recall(thresholds=[0.5, 1.])
     y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
     y_pred = constant_op.constant([[1, 0], [0.6, 0]],
                                   shape=(2, 2),
-- 
GitLab


From c1e50881f3f687fe8ffb7a9cb6d2d905bb70ec71 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 27 Nov 2018 12:19:07 -0800
Subject: [PATCH 0878/1554] [XLA] Avoid undefined behavior in absl::bit_cast
 with Eigen::half. [XLA] [TF] Fix more compile warnings in Mac OS OSS build.

PiperOrigin-RevId: 223035708
---
 tensorflow/compiler/xla/literal.cc            | 28 +++++++++++++++++--
 tensorflow/compiler/xla/protobuf_util.cc      | 10 -------
 .../common_runtime/accumulate_n_optimizer.cc  |  5 ++--
 .../core/common_runtime/ring_reducer.cc       |  2 +-
 tensorflow/core/graph/graph.h                 |  2 +-
 .../core/kernels/eigen_spatial_convolutions.h | 12 ++++----
 .../stream_executor/device_description.cc     | 12 +-------
 7 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 36ad7c6486..f2fcb93717 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -63,6 +63,14 @@ void ConvertEndianShort(char* bytes, int64 size) {
   }
 }
 
+// Since Eigen::half doesn't satisfy the absl::bit_cast contract, we need to be
+// able to transparently access the raw 16-bit value contained within.
+template <typename T>
+T GetRawValue(T val) {
+  return val;
+}
+uint16 GetRawValue(Eigen::half val) { return val.x; }
+
 }  // namespace
 
 LiteralBase::~LiteralBase() {}
@@ -1206,16 +1214,32 @@ Literal ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
 }
 
 template <typename NativeSrcT, typename NativeDestT>
-typename std::enable_if<(sizeof(NativeSrcT) == sizeof(NativeDestT)),
+typename std::enable_if<(sizeof(NativeSrcT) == sizeof(NativeDestT) &&
+                         !std::is_same<NativeDestT, Eigen::half>::value),
                         Literal>::type
 BitcastBetweenNativeTypes(const LiteralBase& src_literal) {
   auto converter = [](NativeSrcT src) {
-    return absl::bit_cast<NativeDestT>(src);
+    return absl::bit_cast<NativeDestT>(GetRawValue(src));
   };
   return ConvertBetweenNativeTypesWithConverter<NativeSrcT, NativeDestT>(
       src_literal, converter);
 }
 
+template <typename NativeSrcT, typename NativeDestT>
+typename std::enable_if<(sizeof(NativeSrcT) == sizeof(Eigen::half) &&
+                         std::is_same<NativeDestT, Eigen::half>::value),
+                        Literal>::type
+BitcastBetweenNativeTypes(const LiteralBase& src_literal) {
+  // Eigen::half doesn't satisfy the absl::bit_cast contract, so explicitly
+  // cast to unsigned short and then use raw_uint16_to_half.
+  auto converter = [](NativeSrcT src) {
+    return Eigen::half_impl::raw_uint16_to_half(
+        absl::bit_cast<uint16>(GetRawValue(src)));
+  };
+  return ConvertBetweenNativeTypesWithConverter<NativeSrcT, Eigen::half>(
+      src_literal, converter);
+}
+
 // This template specialization is here to make the compiler happy. bit_cast has
 // a static check that the types are the same size. This specialization should
 // never be used because the source and destination types are checked for
diff --git a/tensorflow/compiler/xla/protobuf_util.cc b/tensorflow/compiler/xla/protobuf_util.cc
index b507a2ef79..ac342bf40f 100644
--- a/tensorflow/compiler/xla/protobuf_util.cc
+++ b/tensorflow/compiler/xla/protobuf_util.cc
@@ -40,16 +40,6 @@ bool ProtobufEquals(const tensorflow::protobuf::Message& m1,
 
 namespace {
 
-string SanitizeFilename(const string& file_name) {
-  string safe_file_name = file_name;
-  for (char& c : safe_file_name) {
-    if (c == '/' || c == '\\') {
-      c = '_';
-    }
-  }
-  return safe_file_name;
-}
-
 std::pair<tensorflow::mutex*, std::vector<std::function<string(string)>>*>
 GetDirectoryExpanders() {
   static auto* mutex = new tensorflow::mutex;
diff --git a/tensorflow/core/common_runtime/accumulate_n_optimizer.cc b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
index 822d0065b6..c4bc1a684c 100644
--- a/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
+++ b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
@@ -74,8 +74,7 @@ class AccumulateNV2RemovePass : public GraphOptimizationPass {
 
   Status rewriteNode(Node* n, Graph* g) {
     AttrSlice n_attrs = n->attrs();
-    auto base_make_node = [n, g, &n_attrs](const string& op,
-                                           const string& name) {
+    auto base_make_node = [n, &n_attrs](const string& op, const string& name) {
       NodeBuilder node_builder(name, op);
 
       // The pieces of AccumulateNV2 should all be on the same node.
@@ -86,7 +85,7 @@ class AccumulateNV2RemovePass : public GraphOptimizationPass {
       }
       return node_builder;
     };
-    auto make_node = [n, g, &n_attrs, &base_make_node](string op) {
+    auto make_node = [n, g, &base_make_node](string op) {
       return base_make_node(
           op, g->NewName(strings::StrCat(n->name(), "/Internal")));
     };
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index b1fe928ba7..092f15e49e 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -290,7 +290,7 @@ void RingReducer::Run(StatusCallback done) {
         col_ctx_->device, col_ctx_->op_ctx->input_alloc_attr(0),
         col_ctx_->op_ctx->output_alloc_attr(0), col_ctx_->input,
         col_ctx_->output, 0 /*dev_to_dev_stream_index*/,
-        [this, &note, &status](const Status& s) {
+        [&note, &status](const Status& s) {
           status.Update(s);
           note.Notify();
         });
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 6c6d98b5aa..af0b123706 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -65,7 +65,7 @@ class WhileContext;
 
 class NeighborIter;    // Declared below
 class NodeIter;        // Declared below
-class NodeProperties;  // Defined in .cc
+struct NodeProperties;  // Defined in .cc
 
 class Node {
  public:
diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions.h b/tensorflow/core/kernels/eigen_spatial_convolutions.h
index e8dea4763e..25c735d080 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions.h
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions.h
@@ -552,17 +552,17 @@ class TensorContractionSubMapper<
 
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionSubMapper(
       const ParentMapper& base_mapper, Index vert_offset, Index horiz_offset)
-      : m_base_mapper(base_mapper),
-        m_depth_offset(vert_offset),
-        m_col_offset(horiz_offset) {
+      : m_depth_offset(vert_offset),
+        m_col_offset(horiz_offset),
+        m_base_mapper(base_mapper) {
     m_base_mapper.computeBaseIndices(m_col_offset, m_rowIndex, m_colIndex,
                                      m_otherIndex);
   }
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorContractionSubMapper(
       const Self& base_mapper, Index vert_offset, Index horiz_offset)
-      : m_base_mapper(base_mapper.m_base_mapper),
-        m_depth_offset(vert_offset + base_mapper.m_depth_offset),
-        m_col_offset(horiz_offset + base_mapper.m_col_offset) {
+      : m_depth_offset(vert_offset + base_mapper.m_depth_offset),
+        m_col_offset(horiz_offset + base_mapper.m_col_offset),
+        m_base_mapper(base_mapper.m_base_mapper) {
     m_base_mapper.computeBaseIndices(m_col_offset, m_rowIndex, m_colIndex,
                                      m_otherIndex);
   }
diff --git a/tensorflow/stream_executor/device_description.cc b/tensorflow/stream_executor/device_description.cc
index 4120e230db..0b991b7ba8 100644
--- a/tensorflow/stream_executor/device_description.cc
+++ b/tensorflow/stream_executor/device_description.cc
@@ -140,21 +140,11 @@ void CalculateDimensionality(const DeviceDescription &device_description,
                              uint64 element_count, uint64 *threads_per_block,
                              uint64 *block_count) {
   *threads_per_block = device_description.threads_per_block_limit();
-  *block_count = DivideCeil(element_count, *threads_per_block);
+  *block_count = port::MathUtil::CeilOfRatio(element_count, *threads_per_block);
   if (*block_count == 1) {
     CHECK_LE(element_count, *threads_per_block);
     *threads_per_block = element_count;
   }
 }
 
-// Round value up to a multiple of n.
-static uint64 RoundUp(uint64 value, uint64 n) {
-  return port::MathUtil::CeilOfRatio(value, n) * n;
-}
-
-// Round value down to a multiple of n.
-static uint64 RoundDown(uint64 value, uint64 n) {
-  return port::MathUtil::FloorOfRatio(value, n) * n;
-}
-
 }  // namespace stream_executor
-- 
GitLab


From 0b6ed4887ab1a65d2a0500ec8ebb03e221d8681b Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Tue, 27 Nov 2018 12:27:16 -0800
Subject: [PATCH 0879/1554] Add EagerTensor.backing_device property

For reasons of compatibility with graph execution, EagerTensor.device
returns the device of that ran the operation that produced this tensor.
This is not always the device that holds the tensor's memory.
The latter is useful information for troubleshooting now. It will be
even more so once we have native multi-device functions.

PiperOrigin-RevId: 223036943
---
 tensorflow/c/eager/BUILD                 |  1 +
 tensorflow/c/eager/c_api.cc              | 12 +++++
 tensorflow/c/eager/c_api.h               | 23 +++++++++
 tensorflow/c/eager/c_api_test.cc         | 65 ++++++++++++++++++++++++
 tensorflow/c/eager/c_api_test_util.cc    | 13 +++++
 tensorflow/c/eager/c_api_test_util.h     |  3 ++
 tensorflow/python/eager/pywrap_tensor.cc | 18 +++++++
 tensorflow/python/framework/ops.py       | 12 +++++
 tensorflow/python/framework/ops_test.py  |  7 +++
 9 files changed, 154 insertions(+)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 5a0988ed31..c34a84fcfe 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -144,6 +144,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index c9e730ef41..027d752f42 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -411,6 +411,18 @@ const char* TFE_TensorHandleDeviceName(TFE_TensorHandle* h, TF_Status* status) {
                         : d->name().c_str();
 }
 
+const char* TFE_TensorHandleBackingDeviceName(TFE_TensorHandle* h,
+                                              TF_Status* status) {
+  if (h == nullptr || h->handle == nullptr) {
+    status->status = tensorflow::errors::InvalidArgument(
+        "The passed in handle is a nullptr");
+    return nullptr;
+  }
+  tensorflow::Device* d = h->handle->device();
+  return (d == nullptr) ? "/job:localhost/replica:0/task:0/device:CPU:0"
+                        : d->name().c_str();
+}
+
 TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_TensorHandleCopySharingTensor(
     TFE_TensorHandle* h, TF_Status* status) {
   if (h == nullptr || h->handle == nullptr) {
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index b2454d8722..8d6c8d958d 100755
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -169,10 +169,33 @@ TF_CAPI_EXPORT extern int64_t TFE_TensorHandleNumElements(TFE_TensorHandle* h,
 TF_CAPI_EXPORT extern int64_t TFE_TensorHandleDim(TFE_TensorHandle* h,
                                                   int dim_index,
                                                   TF_Status* status);
+
+// Returns the device of the operation that produced `h`.
+// If `h` was produced by a copy, returns the destination device of
+// the copy. Note that returned device name is not always the device
+// holding the tensor handle's memory. If you want the latter, use
+// TFE_TensorHandleBackingDeviceName.
+// This function will block till the operation that produces `h` has completed.
+//
+// Device on which the kernel of the operation that produced `h` ran.
+//
+// If `h` was produced by a copy, returns the destination device of
+// the copy.
+//
+// Note that returned device name is not always the device that owns the memory
+// that backs the tensor handle. For the latter see
+// TFE_TensorHandleBackingDeviceName.
+//
 // This function will block till the operation that produces `h` has completed.
 TF_CAPI_EXPORT extern const char* TFE_TensorHandleDeviceName(
     TFE_TensorHandle* h, TF_Status* status);
 
+// Returns the name of the device in whose memory `h` resides.
+//
+// This function will block till the operation that produces `h` has completed.
+TF_CAPI_EXPORT extern const char* TFE_TensorHandleBackingDeviceName(
+    TFE_TensorHandle* h, TF_Status* status);
+
 // Return a pointer to a new TFE_TensorHandle that shares the underlying tensor
 // with `h`. On success, `status` is set to OK. On failure, `status` reflects
 // the error and a nullptr is returned.
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 0045bb5622..6b39b79ee8 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/eager/c_api.h"
 
 #include <string.h>
+#include "absl/strings/match.h"
 #include "tensorflow/c/eager/c_api_test_util.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -794,6 +795,14 @@ TEST(CAPI, TensorHandleNullptr) {
 
   TF_SetStatus(status.get(), TF_OK, "");
 
+  device_name = TFE_TensorHandleBackingDeviceName(h, status.get());
+  ASSERT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(status.get()));
+  ASSERT_EQ(device_name, nullptr);
+  ASSERT_EQ("The passed in handle is a nullptr",
+            string(TF_Message(status.get())));
+
+  TF_SetStatus(status.get(), TF_OK, "");
+
   int num_dims = TFE_TensorHandleNumDims(h, status.get());
   ASSERT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(status.get()));
   ASSERT_EQ(num_dims, -1);
@@ -809,6 +818,62 @@ TEST(CAPI, TensorHandleNullptr) {
             string(TF_Message(status.get())));
 }
 
+TEST(CAPI, TensorHandleDevices) {
+  std::unique_ptr<TF_Status, decltype(&TF_DeleteStatus)> status(
+      TF_NewStatus(), TF_DeleteStatus);
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  TFE_Context* ctx = TFE_NewContext(opts, status.get());
+  TFE_DeleteContextOptions(opts);
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+
+  TFE_TensorHandle* hcpu = TestMatrixTensorHandle();
+  const char* device_name = TFE_TensorHandleDeviceName(hcpu, status.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+  ASSERT_TRUE(absl::StrContains(device_name, "CPU:0")) << device_name;
+  const char* backing_device_name =
+      TFE_TensorHandleBackingDeviceName(hcpu, status.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+  ASSERT_TRUE(absl::StrContains(backing_device_name, "CPU:0"))
+      << backing_device_name;
+
+  // Disable the test if no GPU is present.
+  string gpu_device_name;
+  if (GetDeviceName(ctx, &gpu_device_name, "GPU")) {
+    TFE_TensorHandle* hgpu = TFE_TensorHandleCopyToDevice(
+        hcpu, ctx, gpu_device_name.c_str(), status.get());
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+
+    TFE_Op* shape_op = ShapeOp(ctx, hgpu);
+    TFE_OpSetDevice(shape_op, gpu_device_name.c_str(), status.get());
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+    TFE_TensorHandle* retvals[1];
+    int num_retvals = 1;
+    TFE_Execute(shape_op, &retvals[0], &num_retvals, status.get());
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+
+    // .device of shape is GPU since the op is executed on GPU
+    device_name = TFE_TensorHandleDeviceName(retvals[0], status.get());
+    ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+    ASSERT_TRUE(absl::StrContains(device_name, "GPU:0")) << device_name;
+
+    // .backing_device of shape is CPU since the tensor is backed by CPU
+    backing_device_name =
+        TFE_TensorHandleBackingDeviceName(retvals[0], status.get());
+    ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+    ASSERT_TRUE(absl::StrContains(backing_device_name, "CPU:0"))
+        << backing_device_name;
+
+    TFE_DeleteOp(shape_op);
+    TFE_DeleteTensorHandle(retvals[0]);
+    TFE_DeleteTensorHandle(hgpu);
+  }
+
+  TFE_DeleteTensorHandle(hcpu);
+  TFE_ContextAsyncWait(ctx, status.get());
+  EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+  TFE_DeleteContext(ctx);
+}
+
 void Execute_MatMul_CPU(bool async) {
   TF_Status* status = TF_NewStatus();
   TFE_ContextOptions* opts = TFE_NewContextOptions();
diff --git a/tensorflow/c/eager/c_api_test_util.cc b/tensorflow/c/eager/c_api_test_util.cc
index 008f088c2d..bd38127d50 100644
--- a/tensorflow/c/eager/c_api_test_util.cc
+++ b/tensorflow/c/eager/c_api_test_util.cc
@@ -104,6 +104,19 @@ TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b) {
   return op;
 }
 
+TFE_Op* ShapeOp(TFE_Context* ctx, TFE_TensorHandle* a) {
+  TF_Status* status = TF_NewStatus();
+
+  TFE_Op* op = TFE_NewOp(ctx, "Shape", status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_OpAddInput(op, a, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TF_DeleteStatus(status);
+  TFE_OpSetAttrType(op, "T", TFE_TensorHandleDataType(a));
+
+  return op;
+}
+
 TFE_TensorHandle* TestAxisTensorHandle() {
   int64_t dims[] = {1};
   int data[] = {1};
diff --git a/tensorflow/c/eager/c_api_test_util.h b/tensorflow/c/eager/c_api_test_util.h
index 474cae67c8..75ef9459e9 100644
--- a/tensorflow/c/eager/c_api_test_util.h
+++ b/tensorflow/c/eager/c_api_test_util.h
@@ -37,6 +37,9 @@ TFE_TensorHandle* TestMatrixTensorHandle3X2();
 // Return a matmul op multiplying `a` by `b`.
 TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b);
 
+// Return a shape op fetching the shape of `a`.
+TFE_Op* ShapeOp(TFE_Context* ctx, TFE_TensorHandle* a);
+
 // Return an 1-D INT32 tensor containing a single value 1.
 TFE_TensorHandle* TestAxisTensorHandle();
 
diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index ed19047f09..0d0f70d543 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -672,11 +672,29 @@ static PyObject* EagerTensor_device(EagerTensor* self) {
 #endif
 }
 
+// Getter `backing_device`.
+static PyObject* EagerTensor_backing_device(EagerTensor* self) {
+  const char* device =
+      TFE_TensorHandleBackingDeviceName(self->handle, self->status);
+  if (MaybeRaiseExceptionFromTFStatus(self->status, PyExc_ValueError)) {
+    // Cleanup self->status before returning.
+    TF_SetStatus(self->status, TF_OK, "");
+    return nullptr;
+  }
+#if PY_MAJOR_VERSION >= 3
+  return PyUnicode_FromString(device);
+#else
+  return PyBytes_FromString(device);
+#endif
+}
+
 static PyGetSetDef EagerTensor_getseters[] = {
     {const_cast<char*>("_id"), (getter)EagerTensor_getid, nullptr,
      const_cast<char*>("_id"), nullptr},
     {const_cast<char*>("device"), (getter)EagerTensor_device, nullptr,
      const_cast<char*>("device"), nullptr},
+    {const_cast<char*>("backing_device"), (getter)EagerTensor_backing_device,
+     nullptr, const_cast<char*>("backing_device"), nullptr},
     {const_cast<char*>("_handle_data"), (getter)EagerTensor_tensor_handle,
      (setter)EagerTensor_settensor_handle, const_cast<char*>("_tensor_handle"),
      nullptr},
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index ec00778347..31fa111b3f 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -748,6 +748,18 @@ class _EagerTensorBase(Tensor):
   def _numpy(self):
     raise NotImplementedError()
 
+  @property
+  def backing_device(self):
+    """Returns the name of the device holding this tensor's memory.
+
+    `.backing_device` is usually the same as `.device`, which returns
+    the device on which the kernel of the operation that produced this tensor
+    ran. However, some operations can produce tensors on a different device
+    (e.g., an operation that executes on the GPU but produces output tensors
+    in host memory).
+    """
+    raise NotImplementedError()
+
   def __copy__(self):
     # Eager Tensors are immutable so it's safe to return themselves as a copy.
     return self
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 32a24521ad..9c9ef799f7 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -1075,6 +1075,13 @@ class DeviceTest(test_util.TensorFlowTestCase):
       node { name: "FloatOutput" op: "FloatOutput" }
     """, gd)
 
+  def testEagerBackingDevice(self):
+    with context.eager_mode():
+      with ops.device("/device:CPU:0"):
+        t = constant_op.constant(1.0)
+        self.assertRegexpMatches(t.device, "/device:CPU:0")
+        self.assertRegexpMatches(t.backing_device, "/device:CPU:0")
+
   def testDevicePartialString(self):
     g = ops.Graph()
     with g.device("/job:worker/replica:2"):
-- 
GitLab


From 55fed8c55147aa5e2c35b3b1f3bac34224bac916 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 12:39:19 -0800
Subject: [PATCH 0880/1554] Automated rollback of commit
 f56925c38e4fe67fba3e4da7ee8bbb72b1e69d38

PiperOrigin-RevId: 223038945
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0725bd0e8a..d9d40874a4 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "fc0f871496cdaec892245afc9890e8267f73b0fcec5a7f75be0dc914e2972023",
-        strip_prefix = "eigen-eigen-efda481cbd7a",
+        sha256 = "8fa7ba1af23f0320be05f4658061138d6eb8dd1f320669cbf305b3a034f9d1c2",
+        strip_prefix = "eigen-eigen-ea671884cc96",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/efda481cbd7a.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/efda481cbd7a.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
         ],
     )
 
-- 
GitLab


From 6fb83aa0e3a194dcbcf8054cdd576a8d4f1d484b Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Tue, 27 Nov 2018 12:47:30 -0800
Subject: [PATCH 0881/1554] Minor update to tflite_convert documentation.

PiperOrigin-RevId: 223040385
---
 tensorflow/lite/g3doc/devguide.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md
index d42729c7ea..fdd02638f9 100644
--- a/tensorflow/lite/g3doc/devguide.md
+++ b/tensorflow/lite/g3doc/devguide.md
@@ -77,8 +77,8 @@ formats:
     SavedModels using [Python](convert/python_api.md#basic_savedmodel) or using
     the [command line](convert/cmdline_examples.md#savedmodel).
 *   `tf.keras` - A HDF5 file containing a model with weights and input and
-    output arguments generated by `tf.Keras`. See [here] for converting HDF5
-    models. See the documentation for converting HDF5 models using
+    output arguments generated by `tf.Keras`. See the documentation for
+    converting HDF5 models using
     [Python](convert/python_api.md#basic_keras_file) or using the
     [command line](convert/cmdline_examples.md#keras).
 *   `frozen tf.GraphDef` — A subclass of `tf.GraphDef` that does not contain
-- 
GitLab


From 6efff588f689ee0e3a05784ca6bf249092a7a380 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 12:56:54 -0800
Subject: [PATCH 0882/1554] Switch callers to the new location of
 distribute_lib.

PiperOrigin-RevId: 223042010
---
 tensorflow/contrib/optimizer_v2/BUILD                        | 2 +-
 tensorflow/contrib/optimizer_v2/optimizer_v2.py              | 2 +-
 tensorflow/python/keras/BUILD                                | 2 +-
 tensorflow/python/keras/engine/distributed_training_utils.py | 2 +-
 tensorflow/python/keras/engine/training_distributed.py       | 2 +-
 tensorflow/python/training/optimizer.py                      | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD
index 835fb4aec4..6e40140630 100644
--- a/tensorflow/contrib/optimizer_v2/BUILD
+++ b/tensorflow/contrib/optimizer_v2/BUILD
@@ -48,7 +48,6 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:framework",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:resource_variable_ops",
@@ -56,6 +55,7 @@ py_library(
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:reduce_util",
     ],
 )
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 747f5b9b8b..73a556f0b2 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -24,6 +24,7 @@ import abc
 
 import six
 
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -35,7 +36,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import distribution_strategy_context as distribute_ctx
 from tensorflow.python.training import optimizer as optimizer_v1
 from tensorflow.python.training import slot_creator
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index a7f01817fb..69e18ea55f 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -185,7 +185,6 @@ py_library(
         ":engine",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:cudnn_rnn_ops_gen",
-        "//tensorflow/python:distribute",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:framework_ops",
@@ -199,6 +198,7 @@ py_library(
         "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:util",
+        "//tensorflow/python/distribute:distribute_lib",
     ],
 )
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index bb8174e26a..8b0076163e 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -23,6 +23,7 @@ from tensorflow.python.client import session as session_module
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -32,7 +33,6 @@ from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 53261fdd26..d16832318c 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -36,7 +36,7 @@ from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribute as distribute_lib
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.util import nest
 
 
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 8cd5311b31..2556e6274a 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -24,6 +24,7 @@ import abc
 
 import six
 
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -37,7 +38,6 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import distribution_strategy_context as distribute_ctx
 from tensorflow.python.training import slot_creator
 from tensorflow.python.training.checkpointable import base as checkpointable
-- 
GitLab


From 4b3a92f1bf905a58daa878999a9e5891d9b9e931 Mon Sep 17 00:00:00 2001
From: Serge Panev <spanev@nvidia.com>
Date: Tue, 27 Nov 2018 22:18:58 +0100
Subject: [PATCH 0883/1554] Simpler loop

---
 tensorflow/core/framework/dataset.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index e0041492ee..927cf22c68 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -162,7 +162,7 @@ class GraphDefBuilderWrapper {
                     const std::vector<std::pair<StringPiece, AttrValue>>& attrs,
                     Node** output) {
     std::vector<std::pair<size_t, Node*>> enumerated_inputs(inputs.size());
-    for (int i = 0; static_cast<size_t>(i) < inputs.size(); i++) {
+    for (size_t i = 0; i < inputs.size(); i++) {
       enumerated_inputs[i] = std::make_pair(i, inputs[i]);
     }
     return AddDataset(dataset, enumerated_inputs, {}, attrs, output);
-- 
GitLab


From f1e25462e9c96e7f4dda213afc91751f65342711 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Tue, 27 Nov 2018 13:29:47 -0800
Subject: [PATCH 0884/1554] Add build configs for TF integration builder.

This builder will check out TF from Piper and Estimator from GitHub.
It will then build Estimator pip from a pinned revision, and install
the pip package. It will then run TF tests that depend on Estimator
code.

PiperOrigin-RevId: 223047585
---
 tensorflow/contrib/distribute/python/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index b068ef029f..38ce0b2b8e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -416,6 +416,7 @@ cuda_py_test(
         "multi_and_single_gpu",
         "no_oss",  # http://b/119349471
         "no_pip",
+        "tf_integration_test",
     ],
 )
 
@@ -429,6 +430,7 @@ cuda_py_test(
         "multi_and_single_gpu",
         "no_oss",  # http://b/119349471
         "no_pip",
+        "tf_integration_test",
     ],
 )
 
-- 
GitLab


From 891e56199d35d067efb50303e6714253df6d1c83 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Tue, 27 Nov 2018 13:31:04 -0800
Subject: [PATCH 0885/1554] Add warning for the renamed
 tf.estimator.Estimator.export_savedmodel function.

PiperOrigin-RevId: 223047837
---
 tensorflow/tools/compatibility/ast_edits.py   | 28 +++++++++++++++++++
 .../tools/compatibility/ast_edits_test.py     | 24 ++++++++++++++++
 .../tools/compatibility/tf_upgrade_v2.py      | 16 +++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py |  7 +++++
 4 files changed, 75 insertions(+)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 5edb4d4759..90bfab3507 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -40,6 +40,10 @@ class APIChangeSpec(object):
   * `function_reorders`: maps functions whose argument order has changed to the
     list of arguments in the new order
   * `function_handle`: maps function names to custom handlers for the function
+  * `function_warnings`: maps full names of functions to warnings that will be
+    printed out if the function is used. (e.g. tf.nn.convolution())
+  * `unrestricted_function_warnings`: maps names of functions to warnings that
+    will be printed out when the function is used (e.g. foo.convolution()).
 
   For an example, see `TFAPIChangeSpec`.
   """
@@ -195,6 +199,29 @@ class _ASTCallVisitor(ast.NodeVisitor):
     except KeyError:
       pass
 
+  def _print_warning_for_function_unrestricted(self, node):
+    """Print a warning when specific functions are called.
+
+    The function _print_warning_for_function matches the full name of the called
+    function, e.g., tf.foo.bar(). This function matches the function name that
+    is called, as long as the function is an attribute. For example,
+    `tf.foo.bar()` and `foo.bar()` are matched, but not `bar()`.
+
+    Args:
+      node: ast.Call object
+    """
+    function_warnings = getattr(
+        self._api_change_spec, "unrestricted_function_warnings", {})
+    if isinstance(node.func, ast.Attribute):
+      function_name = node.func.attr
+      try:
+        warning_message = function_warnings[function_name]
+        self._file_edit.add(warning_message,
+                            node.lineno, node.col_offset, "", "",
+                            error="%s requires manual check." % function_name)
+      except KeyError:
+        pass
+
   def _get_attribute_full_path(self, node):
     """Traverse an attribute to generate a full name e.g. tf.foo.bar.
 
@@ -276,6 +303,7 @@ class _ASTCallVisitor(ast.NodeVisitor):
     Args:
       node: Current Node
     """
+    self._print_warning_for_function_unrestricted(node)
 
     # Find a simple attribute name path e.g. "tf.foo.bar"
     full_name, name = self._get_attribute_full_path(node.func)
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 08f4ae3fcc..99f20a026f 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -52,6 +52,10 @@ class NoUpdateSpec(ast_edits.APIChangeSpec):
     self.function_handle = {}
     self.function_reorders = {}
     self.function_keyword_renames = {}
+    self.symbol_renames = {}
+    self.function_warnings = {}
+    self.unrestricted_function_warnings = {}
+    self.change_to_function = {}
 
 
 class RenameKeywordSpec(NoUpdateSpec):
@@ -391,6 +395,26 @@ class TestAstEdits(test_util.TensorFlowTestCase):
     _, new_text = self._upgrade(RemoveMultipleKeywordArguments(), text)
     self.assertIn(new_text, acceptable_outputs)
 
+  def testUnrestrictedFunctionWarnings(self):
+    class FooWarningSpec(NoUpdateSpec):
+      """Usages of function attribute foo() prints out a warning."""
+
+      def __init__(self):
+        NoUpdateSpec.__init__(self)
+        self.unrestricted_function_warnings = {"foo": "not good"}
+    texts = ["object.foo()", "get_object().foo()",
+             "get_object().foo()", "object.foo().bar()"]
+    for text in texts:
+      (_, report, _), _ = self._upgrade(FooWarningSpec(), text)
+      self.assertIn("not good", report)
+
+    # Note that foo() won't result in a warning, because in this case foo is
+    # not an attribute, but a name.
+    false_alarms = ["foo", "foo()", "foo.bar()", "obj.run_foo()", "obj.foo"]
+    for text in false_alarms:
+      (_, report, _), _ = self._upgrade(FooWarningSpec(), text)
+      self.assertNotIn("not good", report)
+
 
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 7433e0ecbf..f9cc20c9cd 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -677,6 +677,22 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         if name not in self.function_warnings and name not in excluded_renames
     }
 
+    export_saved_model_renamed = (
+        "(Manual edit required) Please rename the function export_savedmodel() "
+        "to export_saved_model(). Two things to note:\n\t(1) The argument "
+        "strip_default_attributes has been removed. The function will always "
+        "strip the default attributes from ops. If this breaks your code, "
+        "please switch to tf.compat.v1.estimator.Estimator.\n\t(2) This change "
+        "only effects core estimator. If you are using "
+        "tf.contrib.learn.Estimator, please switch to using core estimator.")
+
+    # Specify warnings for functions that aren't restricted to the tf.x.y.z
+    # format. This should only be used for methods with unique names, e.g.
+    # export_savedmodel, which is only defined in Estimator objects.
+    self.unrestricted_function_warnings = {
+        "export_savedmodel": export_saved_model_renamed,
+    }
+
   @staticmethod
   def _dropout_handler(file_edit_recorder, node):
     if len(node.args) < 2:
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index bda748f274..553010d4fc 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -212,6 +212,13 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                      ["test.py:1: Optimizer.compute_gradients "
                       "requires manual check."])
 
+  def testExportSavedModelRename(self):
+    text = "self.est.export_savedmodel(path)"
+    _, report, unused_errors, unused_new_text = self._upgrade(text)
+    self.assertIn(
+        "rename the function export_savedmodel() to export_saved_model()",
+        report)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 654efdeacca7b84bd4245cffe95ae27eb1bd1c75 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 27 Nov 2018 13:40:54 -0800
Subject: [PATCH 0886/1554] Add gradient function for XlaClusterOutput

PiperOrigin-RevId: 223049594
---
 tensorflow/compiler/jit/BUILD                 |  5 +++-
 .../jit/encapsulate_xla_computations_pass.cc  |  7 +++--
 tensorflow/compiler/jit/ops/BUILD             |  6 ++++
 tensorflow/compiler/jit/ops/xla_ops_grad.py   | 29 +++++++++++++++++++
 tensorflow/compiler/tf2xla/BUILD              |  1 +
 tensorflow/contrib/compiler/BUILD             |  1 +
 tensorflow/contrib/compiler/xla.py            |  1 +
 7 files changed, 47 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/compiler/jit/ops/xla_ops_grad.py

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 1f6dadf1ac..a3c7dd22de 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -736,7 +736,10 @@ tf_custom_op_py_library(
     visibility = [
         ":friends",
     ],
-    deps = ["//tensorflow/compiler/jit/ops:xla_ops_wrapper_py"],
+    deps = [
+        "//tensorflow/compiler/jit/ops:xla_ops_grad",
+        "//tensorflow/compiler/jit/ops:xla_ops_wrapper_py",
+    ],
 )
 
 # This target can be used by XLA device plugins to prevent circular dependencies, and provides access to all of the required headers for building a device library.
diff --git a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
index 2ce6fa73fc..d334100aa4 100644
--- a/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_xla_computations_pass.cc
@@ -195,8 +195,11 @@ Status RewriteSubgraph(const std::vector<OutputTensor>& arg_source_tensors,
         e->dst()->attrs().Find(kXlaClusterAttr) == nullptr &&
         e->dst()->type_string() != kXlaClusterOutput) {
       return errors::InvalidArgument(
-          "Undeclared output of XLA computation. A common cause of this error "
-          "is variable initializers that depend on the XLA computation. Edge: ",
+          "Undeclared output of XLA computation. Some common causes of this "
+          "error are: 1) variable initializers that depend on the XLA "
+          "computation; 2) gradient computations that depend on the XLA "
+          "computation, which can be mitigated by moving gradient computations "
+          "inside XLA computation. Offending edge: ",
           e->src()->name(), ":", e->src_output(), " -> ", e->dst()->name(), ":",
           e->dst_input());
     }
diff --git a/tensorflow/compiler/jit/ops/BUILD b/tensorflow/compiler/jit/ops/BUILD
index f72224545b..64409d9334 100644
--- a/tensorflow/compiler/jit/ops/BUILD
+++ b/tensorflow/compiler/jit/ops/BUILD
@@ -18,3 +18,9 @@ tf_gen_op_wrapper_py(
     out = "xla_ops.py",
     deps = ["//tensorflow/compiler/jit/ops:xla_ops"],
 )
+
+py_library(
+    name = "xla_ops_grad",
+    srcs = ["xla_ops_grad.py"],
+    deps = ["//tensorflow/python:framework_ops"],
+)
diff --git a/tensorflow/compiler/jit/ops/xla_ops_grad.py b/tensorflow/compiler/jit/ops/xla_ops_grad.py
new file mode 100644
index 0000000000..2d31d8dc71
--- /dev/null
+++ b/tensorflow/compiler/jit/ops/xla_ops_grad.py
@@ -0,0 +1,29 @@
+"""Gradients for XLA ops."""
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+
+
+@ops.RegisterGradient("XlaClusterOutput")
+def _XlaClusterOutputGrad(_, grad):
+  del grad  # unused
+  raise RuntimeError("Gradient computation of graph in xla.compile() is "
+                     "prohibited because it can cause performance degradation."
+                     "Please move gradient computation inside xla.compile().")
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 3458c7f1c4..25a84fb1b6 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -9,6 +9,7 @@ package_group(
         "//tensorflow/compiler/jit/...",
         "//tensorflow/compiler/tests/...",
         "//tensorflow/compiler/tf2xla/...",
+        "//tensorflow/contrib/compiler/...",
     ],
 )
 
diff --git a/tensorflow/contrib/compiler/BUILD b/tensorflow/contrib/compiler/BUILD
index 1630f010ab..e4566437c6 100644
--- a/tensorflow/contrib/compiler/BUILD
+++ b/tensorflow/contrib/compiler/BUILD
@@ -58,6 +58,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/compiler/jit:xla_ops_py",
+        "//tensorflow/compiler/jit/ops:xla_ops_grad",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/compiler/xla.py b/tensorflow/contrib/compiler/xla.py
index 335ac79464..f867cd15b6 100644
--- a/tensorflow/contrib/compiler/xla.py
+++ b/tensorflow/contrib/compiler/xla.py
@@ -23,6 +23,7 @@ import contextlib
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.compiler.jit.ops import xla_ops
+from tensorflow.compiler.jit.ops import xla_ops_grad  # pylint: disable=unused-import
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.framework import ops
-- 
GitLab


From 3409380f7ca5adc4d381e581a8838fd8aa6d8572 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 27 Nov 2018 14:27:52 -0800
Subject: [PATCH 0887/1554] Fix unnecessarily opaque test.

PiperOrigin-RevId: 223058199
---
 tensorflow/python/ops/variable_scope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index cf33c61b82..87ef046100 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -907,7 +907,7 @@ class _VariableStore(object):
         variable_dtype = None
       else:
         # Instantiate initializer if provided initializer is a type object.
-        if isinstance(initializer, type(init_ops.Initializer)):
+        if tf_inspect.isclass(initializer):
           initializer = initializer(dtype=dtype)
         spec = tf_inspect.getargspec(initializer)
         if shape is not None and shape.is_fully_defined():
-- 
GitLab


From a69210ede18ca9401e72916a3d0f96e6aead80c4 Mon Sep 17 00:00:00 2001
From: Mingxing Tan <tanmingxing@google.com>
Date: Tue, 27 Nov 2018 14:41:24 -0800
Subject: [PATCH 0888/1554] Internal changes.

PiperOrigin-RevId: 223060752
---
 tensorflow/python/keras/layers/normalization.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index aa8598d731..2b360d2114 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -491,6 +491,9 @@ class BatchNormalization(Layer):
 
     return (r, d, new_mean, new_variance)
 
+  def _moments(self, inputs, reduction_axes, keep_dims):
+    return nn.moments(inputs, reduction_axes, keep_dims=keep_dims)
+
   def call(self, inputs, training=None):
     if training is None:
       training = K.learning_phase()
@@ -562,7 +565,8 @@ class BatchNormalization(Layer):
       # Some of the computations here are not necessary when training==False
       # but not a constant. However, this makes the code simpler.
       keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1
-      mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims)
+      mean, variance = self._moments(
+          inputs, reduction_axes, keep_dims=keep_dims)
 
       moving_mean = self.moving_mean
       moving_variance = self.moving_variance
-- 
GitLab


From 891bf2d9fa5773c9588d363faa5efd63f2134a75 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Tue, 27 Nov 2018 14:47:43 -0800
Subject: [PATCH 0889/1554] Add a few tests for upgrade script, fix
 tf.argmin/tf.argmax to rename dimension to axis.

PiperOrigin-RevId: 223061849
---
 tensorflow/tools/compatibility/BUILD          | 10 +++++
 .../compatibility/testdata/test_file_v1_10.py | 41 ++++++++++++++++++-
 .../tools/compatibility/tf_upgrade_v2.py      | 10 ++++-
 .../tools/compatibility/tf_upgrade_v2_test.py | 22 ++++++++++
 4 files changed, 79 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index f46e36bf32..a6574dac53 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -124,6 +124,16 @@ genrule(
     tools = [":tf_upgrade_v2"],
 )
 
+py_test(
+    name = "test_file_v1_10",
+    size = "small",
+    srcs = ["testdata/test_file_v1_10.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
 py_test(
     name = "test_file_v2_0",
     size = "small",
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_10.py b/tensorflow/tools/compatibility/testdata/test_file_v1_10.py
index e5ca8d3e2e..fd688781b0 100644
--- a/tensorflow/tools/compatibility/testdata/test_file_v1_10.py
+++ b/tensorflow/tools/compatibility/testdata/test_file_v1_10.py
@@ -25,10 +25,47 @@ from tensorflow.python.platform import test as test_lib
 class TestUpgrade(test_util.TensorFlowTestCase):
   """Test various APIs that have been changed in 2.0."""
 
+  def setUp(self):
+    tf.enable_eager_execution()
+
   def testRenames(self):
     with self.cached_session():
-      self.assertAllClose(1.04719755, tf.acos(0.5).eval())
-      self.assertAllClose(0.5, tf.rsqrt(4.0).eval())
+      self.assertAllClose(1.04719755, tf.acos(0.5))
+      self.assertAllClose(0.5, tf.rsqrt(4.0))
+
+  def testSerializeSparseTensor(self):
+    sp_input = tf.SparseTensor(
+        indices=tf.constant([[1]], dtype=tf.int64),
+        values=tf.constant([2], dtype=tf.int64),
+        dense_shape=[2])
+
+    with self.cached_session():
+      serialized_sp = tf.serialize_sparse(sp_input, 'serialize_name', tf.string)
+      self.assertEqual((3,), serialized_sp.shape)
+      self.assertTrue(serialized_sp[0].numpy())  # check non-empty
+
+  def testSerializeManySparse(self):
+    sp_input = tf.SparseTensor(
+        indices=tf.constant([[0, 1]], dtype=tf.int64),
+        values=tf.constant([2], dtype=tf.int64),
+        dense_shape=[1, 2])
+
+    with self.cached_session():
+      serialized_sp = tf.serialize_many_sparse(
+          sp_input, 'serialize_name', tf.string)
+      self.assertEqual((1, 3), serialized_sp.shape)
+
+  def testArgMaxMin(self):
+    self.assertAllClose(
+        [1],
+        tf.argmax([[1, 3, 2]], name='abc', dimension=1))
+    self.assertAllClose(
+        [0, 0, 0],
+        tf.argmax([[1, 3, 2]], dimension=0))
+    self.assertAllClose(
+        [0],
+        tf.argmin([[1, 3, 2]], name='abc', dimension=1))
+
 
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index f9cc20c9cd..059d2ef4de 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -31,6 +31,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Maps from a function name to a dictionary that describes how to
     # map from an old argument keyword to the new argument keyword.
     self.function_keyword_renames = {
+        "tf.argmin": {
+            "dimension": "axis",
+        },
+        "tf.argmax": {
+            "dimension": "axis",
+        },
         "tf.image.crop_and_resize": {
             "box_ind": "box_indices",
         },
@@ -408,8 +414,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.function_reorders = {
         "tf.io.serialize_sparse": ["sp_input", "name", "out_type"],
         "tf.io.serialize_many_sparse": ["sp_input", "name", "out_type"],
-        "tf.argmax": ["input", "axis", "name", "dimension", "output_type"],
-        "tf.argmin": ["input", "axis", "name", "dimension", "output_type"],
+        "tf.argmax": ["input", "axis", "name", "axis", "output_type"],
+        "tf.argmin": ["input", "axis", "name", "axis", "output_type"],
         "tf.batch_to_space": ["input", "crops", "block_size", "name"],
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
         "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 553010d4fc..0414becc70 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -219,6 +219,28 @@ class TestUpgrade(test_util.TensorFlowTestCase):
         "rename the function export_savedmodel() to export_saved_model()",
         report)
 
+  def testArgmin(self):
+    text = "tf.argmin(input, name=n, dimension=1, output_type=type)"
+    expected_text = "tf.argmin(input=input, name=n, axis=1, output_type=type)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = "tf.argmin(input, 0)"
+    expected_text = "tf.argmin(input=input, axis=0)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testArgmax(self):
+    text = "tf.argmax(input, name=n, dimension=1, output_type=type)"
+    expected_text = "tf.argmax(input=input, name=n, axis=1, output_type=type)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = "tf.argmax(input, 0)"
+    expected_text = "tf.argmax(input=input, axis=0)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From f670244e49b6c644bb27d448b116ae1f2cf8164c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 14:58:29 -0800
Subject: [PATCH 0890/1554] Make Stream prefer host callbacks with return
 status

PiperOrigin-RevId: 223063676
---
 .../xla/service/interpreter/executor.cc       | 11 ++++++++---
 .../xla/service/interpreter/executor.h        |  3 ++-
 .../stream_executor/cuda/cuda_gpu_executor.cc |  9 +++++++--
 .../stream_executor/cuda/cuda_gpu_executor.h  |  3 ++-
 .../stream_executor/host/host_gpu_executor.cc |  9 +++++++--
 .../stream_executor/host/host_gpu_executor.h  |  3 ++-
 tensorflow/stream_executor/stream.h           | 16 +++++++++++-----
 .../stream_executor_internal.cc               | 19 +++++++++----------
 .../stream_executor_internal.h                |  4 ++--
 9 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/tensorflow/compiler/xla/service/interpreter/executor.cc b/tensorflow/compiler/xla/service/interpreter/executor.cc
index 4fb67bd0b7..e3e5fa7154 100644
--- a/tensorflow/compiler/xla/service/interpreter/executor.cc
+++ b/tensorflow/compiler/xla/service/interpreter/executor.cc
@@ -78,9 +78,14 @@ port::Status XlaInterpreterExecutor::SynchronousMemcpy(
   return port::Status::OK();
 }
 
-bool XlaInterpreterExecutor::HostCallback(Stream *stream,
-                                          std::function<void()> callback) {
-  AsExecutorStream(stream)->EnqueueTask(callback);
+bool XlaInterpreterExecutor::HostCallback(
+    Stream *stream, std::function<port::Status()> callback) {
+  AsExecutorStream(stream)->EnqueueTask([callback]() {
+    port::Status s = callback();
+    if (!s.ok()) {
+      LOG(WARNING) << "Host callback failed: " << s;
+    }
+  });
   return true;
 }
 
diff --git a/tensorflow/compiler/xla/service/interpreter/executor.h b/tensorflow/compiler/xla/service/interpreter/executor.h
index fbb9945784..400c305154 100644
--- a/tensorflow/compiler/xla/service/interpreter/executor.h
+++ b/tensorflow/compiler/xla/service/interpreter/executor.h
@@ -125,7 +125,8 @@ class XlaInterpreterExecutor : public internal::StreamExecutorInterface {
     return port::Status{port::error::UNIMPLEMENTED, ""};
   }
 
-  bool HostCallback(Stream *stream, std::function<void()> callback) override;
+  bool HostCallback(Stream *stream,
+                    std::function<port::Status()> callback) override;
 
   port::Status AllocateEvent(Event *event) override {
     return port::Status{port::error::UNIMPLEMENTED, ""};
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index ad9154226c..4874d096ad 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -662,8 +662,13 @@ bool CUDAExecutor::MemcpyDeviceToDevice(Stream *stream,
 }
 
 bool CUDAExecutor::HostCallback(Stream *stream,
-                                std::function<void()> callback) {
-  auto callback_ptr = new std::function<void()>(callback);
+                                std::function<port::Status()> callback) {
+  auto callback_ptr = new std::function<void()>([callback]() {
+    port::Status s = callback();
+    if (!s.ok()) {
+      LOG(WARNING) << "Host callback failed: " << s;
+    }
+  });
   return CUDADriver::AddStreamCallback(context_, AsCUDAStreamValue(stream),
                                        InternalHostCallback, callback_ptr);
 }
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
index 90bf1c0242..ae8e4abf92 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
@@ -148,7 +148,8 @@ class CUDAExecutor : public internal::StreamExecutorInterface {
                             const DeviceMemoryBase &gpu_src,
                             uint64 size) override;
 
-  bool HostCallback(Stream *stream, std::function<void()> callback) override;
+  bool HostCallback(Stream *stream,
+                    std::function<port::Status()> callback) override;
 
   bool AllocateStream(Stream *stream) override;
 
diff --git a/tensorflow/stream_executor/host/host_gpu_executor.cc b/tensorflow/stream_executor/host/host_gpu_executor.cc
index 8adf739b17..1396a83dfb 100644
--- a/tensorflow/stream_executor/host/host_gpu_executor.cc
+++ b/tensorflow/stream_executor/host/host_gpu_executor.cc
@@ -148,8 +148,13 @@ port::Status HostExecutor::SynchronousMemcpyDeviceToDevice(
 }
 
 bool HostExecutor::HostCallback(Stream *stream,
-                                std::function<void()> callback) {
-  AsHostStream(stream)->EnqueueTask(callback);
+                                std::function<port::Status()> callback) {
+  AsHostStream(stream)->EnqueueTask([callback]() {
+    port::Status s = callback();
+    if (!s.ok()) {
+      LOG(WARNING) << "Host callback failed: " << s;
+    }
+  });
   return true;
 }
 
diff --git a/tensorflow/stream_executor/host/host_gpu_executor.h b/tensorflow/stream_executor/host/host_gpu_executor.h
index 7ba1f18101..56e3c2aa6a 100644
--- a/tensorflow/stream_executor/host/host_gpu_executor.h
+++ b/tensorflow/stream_executor/host/host_gpu_executor.h
@@ -103,7 +103,8 @@ class HostExecutor : public internal::StreamExecutorInterface {
                                                const DeviceMemoryBase &gpu_src,
                                                uint64 size) override;
 
-  bool HostCallback(Stream *stream, std::function<void()> callback) override;
+  bool HostCallback(Stream *stream,
+                    std::function<port::Status()> callback) override;
 
   port::Status AllocateEvent(Event *event) override {
     return port::Status(port::error::UNIMPLEMENTED, "");
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index e1629b5b30..0fc90cf83d 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -2033,9 +2033,20 @@ class Stream {
   // transferred to the caller.
   internal::StreamInterface *implementation() { return implementation_.get(); }
 
+  // Entrains onto the stream a callback to the host (from the device).
+  // Behaves as ThenDoHostCallbackWithStatus below, but the callback should
+  // never fail or its failure is inconsequential.
+  //
+  // This is kept for backward compatibility. Future code should use
+  // ThenDoHostCallbackWithStatus and explicitly return a success status.
+  // TODO(b/112125301): Eventually remove this method.
+  Stream &ThenDoHostCallback(std::function<void()> callback);
+
   // Entrains onto the stream a callback to the host (from the device).
   // Host callbacks block/occupy the stream just as device functions
   // (execute one at a time, block later stream operations).
+  // Whether the callback return status affects the result of BlockHostUntilDone
+  // is platform-dependent.
   //
   // Behavior is undefined when synchronizing using OpenCL user events.
   // Behavior is undefined if host callbacks call device routines or insert
@@ -2043,11 +2054,6 @@ class Stream {
   //
   // On certain platforms, ThenDoHostCallback is expected to have significant
   // negative effects on performance.
-  Stream &ThenDoHostCallback(std::function<void()> callback);
-
-  // Entrains onto the stream a callback to the host (from the device).
-  // Behaves as ThenDoHostCallback above, but returns a Status instead of void.
-  // This overload should be preferred if the callback could fail.
   Stream &ThenDoHostCallbackWithStatus(std::function<port::Status()> callback);
 
   // Returns the StreamExecutor (parent object) associated with this stream.
diff --git a/tensorflow/stream_executor/stream_executor_internal.cc b/tensorflow/stream_executor/stream_executor_internal.cc
index 7df6a361c6..341c6edccd 100644
--- a/tensorflow/stream_executor/stream_executor_internal.cc
+++ b/tensorflow/stream_executor/stream_executor_internal.cc
@@ -36,16 +36,15 @@ StreamExecutorFactory* MakeOpenCLExecutorImplementation() {
 
 StreamExecutorFactory MakeHostExecutorImplementation;
 
-// TODO(b/112125301): Consolodate this down to one implementation of
-// HostCallback, taking a callback that returns a Status.
-bool StreamExecutorInterface::HostCallback(
-    Stream* stream, std::function<port::Status()> callback) {
-  return HostCallback(stream, [callback]() {
-    port::Status s = callback();
-    if (!s.ok()) {
-      LOG(WARNING) << "HostCallback failed: " << s;
-    }
-  });
+// The default implementation just calls the other HostCallback method.
+// It should make all existing code that uses a void() callback still work.
+bool StreamExecutorInterface::HostCallback(Stream* stream,
+                                           std::function<void()> callback) {
+  return HostCallback(
+      stream, std::function<port::Status()>([callback]() -> port::Status {
+        callback();
+        return port::Status::OK();
+      }));
 }
 
 }  // namespace internal
diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h
index 32f75fd1bc..0c2c33cfca 100644
--- a/tensorflow/stream_executor/stream_executor_internal.h
+++ b/tensorflow/stream_executor/stream_executor_internal.h
@@ -237,9 +237,9 @@ class StreamExecutorInterface {
   virtual bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
                                     const DeviceMemoryBase &gpu_src,
                                     uint64 size) = 0;
-  virtual bool HostCallback(Stream *stream, std::function<void()> callback) = 0;
+  virtual bool HostCallback(Stream *stream, std::function<void()> callback);
   virtual bool HostCallback(Stream *stream,
-                            std::function<port::Status()> callback);
+                            std::function<port::Status()> callback) = 0;
   virtual port::Status AllocateEvent(Event *event) = 0;
   virtual port::Status DeallocateEvent(Event *event) = 0;
   virtual port::Status RecordEvent(Stream *stream, Event *event) = 0;
-- 
GitLab


From 71ec2bbdb2f762692c44159057c8a0c63dbbb5f0 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Tue, 27 Nov 2018 15:08:17 -0800
Subject: [PATCH 0891/1554] [TF:XLA] Unbreak test build after DeviceMgr API
 change

PiperOrigin-RevId: 223065543
---
 tensorflow/compiler/tests/randomized_tests.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index a6b5802012..d23fd12516 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -3382,10 +3382,10 @@ int main(int argc, char** argv) {
   }
   // XLA devices register kernels at construction time; create all known devices
   // to make sure the kernels are registered.
-  std::vector<tensorflow::Device*> devices;
+  std::vector<std::unique_ptr<tensorflow::Device>> devices;
   TF_CHECK_OK(tensorflow::DeviceFactory::AddDevices(
       tensorflow::SessionOptions(), "", &devices));
-  tensorflow::DeviceMgr device_mgr(devices);
+  tensorflow::DeviceMgr device_mgr(std::move(devices));
 
   tensorflow::Device* ignored;
   TF_QCHECK_OK(
-- 
GitLab


From d9c76a8e85af52ae39ad36345827c34676068df1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 15:27:25 -0800
Subject: [PATCH 0892/1554] Emit layout info in error message otherwise it's
 pretty hard to debug.

PiperOrigin-RevId: 223068849
---
 tensorflow/compiler/xla/service/hlo_verifier.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 017549ce1b..77db7b098a 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -1379,9 +1379,8 @@ class InstructionVerifier : public DfsHloVisitorWithDefault {
           const Layout& operand_layout = operand_shape.layout();
           TF_RET_CHECK(LayoutUtil::Equal(result_layout, operand_layout))
               << "Instruction shouldn't change layouts "
-              << instruction->ToString() << " From "
-              << ShapeUtil::HumanString(result_shape) << " To "
-              << ShapeUtil::HumanString(operand_shape);
+              << instruction->ToString() << " From " << result_shape << " To "
+              << operand_shape;
         }
       }
     }
-- 
GitLab


From b326178f3d478fab34c247b7f072204284085f85 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 27 Nov 2018 15:31:44 -0800
Subject: [PATCH 0893/1554] Slightly change behavior of fused parameter to
 tf.keras.layers.BatchNormalization in 2.0.

The original behavior is that if True or None is passed to the fused parameter, the layer sets self.fused to True if fused is supported, and False otherwise. No distinction is made between passing fused=True and fused=None to the constructor.

The new behavior is that if True is passed to the fused parameter and fused is not supported, an error is thrown. If None is passed, self.fused is set to True if it is supported, and False otherwise. We do most the checks in the constructor, but we must wait until build() to do the final check, as we do not have the input size information in the constructor.

PiperOrigin-RevId: 223069621
---
 .../python/keras/layers/normalization.py      | 117 +++++++++++++++---
 .../python/keras/layers/normalization_test.py | 113 ++++++++++++++---
 ...ow.keras.layers.-batch-normalization.pbtxt |   3 +-
 ...nsorflow.layers.-batch-normalization.pbtxt |   3 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |   2 +-
 5 files changed, 196 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 2b360d2114..d958497655 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python import tf2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -40,8 +41,8 @@ from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('keras.layers.BatchNormalization')
-class BatchNormalization(Layer):
+@tf_export('keras.layers.BatchNormalization', v1=[])
+class BatchNormalizationV2(Layer):
   """Batch normalization layer (Ioffe and Szegedy, 2014).
 
   Normalize the activations of the previous layer at each batch,
@@ -84,8 +85,10 @@ class BatchNormalization(Layer):
       and should be neither too small (which would add noise) nor too large
       (which would give stale estimates). Note that `momentum` is still applied
       to get the means and variances for inference.
-    fused: if `None` or `True`, use a faster, fused implementation if possible.
-      If `False`, use the system recommended implementation.
+    fused: if `True`, use a faster, fused implementation, or raise a ValueError
+      if the fused implementation cannot be used. If `None`, use the faster
+      implementation if possible. If False, do not used the fused
+      implementation.
     trainable: Boolean, if `True` also add variables to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
     virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`,
@@ -120,6 +123,9 @@ class BatchNormalization(Layer):
         Internal Covariate Shift](https://arxiv.org/abs/1502.03167)
   """
 
+  # The BatchNormalizationV1 subclass sets this to False to use the V1 behavior.
+  _USE_V2_BEHAVIOR = True
+
   def __init__(self,
                axis=-1,
                momentum=0.99,
@@ -143,12 +149,15 @@ class BatchNormalization(Layer):
                adjustment=None,
                name=None,
                **kwargs):
-    super(BatchNormalization, self).__init__(
+    super(BatchNormalizationV2, self).__init__(
         name=name, trainable=trainable, **kwargs)
     if isinstance(axis, list):
       self.axis = axis[:]
-    else:
+    elif isinstance(axis, int):
       self.axis = axis
+    else:
+      raise TypeError('axis must be int or list, type given: %s'
+                      % type(self.axis))
     self.momentum = momentum
     self.epsilon = epsilon
     self.center = center
@@ -165,7 +174,14 @@ class BatchNormalization(Layer):
     self.renorm = renorm
     self.virtual_batch_size = virtual_batch_size
     self.adjustment = adjustment
-    if fused is None:
+    if self._USE_V2_BEHAVIOR:
+      if fused:
+        self._raise_if_fused_cannot_be_used()
+      # We leave fused as None if self._fused_can_be_used()==True, since we
+      # still may set it to False in self.build() if the input rank is not 4.
+      elif fused is None and not self._fused_can_be_used():
+        fused = False
+    elif fused is None:
       fused = True
     self.supports_masking = True
 
@@ -181,6 +197,38 @@ class BatchNormalization(Layer):
       self.renorm_clipping = renorm_clipping
       self.renorm_momentum = renorm_momentum
 
+  def _raise_if_fused_cannot_be_used(self):
+    """Raises a ValueError if fused implementation cannot be used.
+
+    In addition to the checks done in this function, the input tensors rank must
+    be 4. The input rank check can only be done once the input shape is known.
+    """
+    # Currently fused batch norm doesn't support renorm. It also only supports a
+    # channel dimension on axis 1 or 3, when no virtual batch size or adjustment
+    # is used.
+    if self.renorm:
+      raise ValueError('Passing both fused=True and renorm=True is '
+                       'unsupported')
+    axis = [self.axis] if isinstance(self.axis, int) else self.axis
+    # Axis -3 is equivalent to 1, and axis -1 is equivalent to 3, because the
+    # input rank is required to be 4 (which is checked later).
+    if len(axis) > 1 or axis[0] not in (-3, -1, 1, 3):
+      raise ValueError('Passing fused=True is only supported when axis is 1 '
+                       'or 3')
+    if self.virtual_batch_size is not None:
+      raise ValueError('Passing fused=True is unsupported when '
+                       'virtual_batch_size is specified.')
+    if self.adjustment is not None:
+      raise ValueError('Passing fused=True is unsupported when '
+                       'adjustment is specified.')
+
+  def _fused_can_be_used(self):
+    try:
+      self._raise_if_fused_cannot_be_used()
+      return True
+    except ValueError:
+      return False
+
   def build(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape)
     if not input_shape.ndims:
@@ -191,10 +239,6 @@ class BatchNormalization(Layer):
     if isinstance(self.axis, int):
       self.axis = [self.axis]
 
-    if not isinstance(self.axis, list):
-      raise TypeError('axis must be int or list, type given: %s'
-                      % type(self.axis))
-
     for idx, x in enumerate(self.axis):
       if x < 0:
         self.axis[idx] = ndims + x
@@ -219,16 +263,18 @@ class BatchNormalization(Layer):
         raise ValueError('When using virtual_batch_size, adjustment cannot '
                          'be specified')
 
-    if self.fused:
-      # Currently fused batch norm doesn't support renorm. It also only supports
-      # an input tensor of rank 4 and a channel dimension on axis 1 or 3.
+    if self.fused in (None, True):
       # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the
       # output back to its original shape accordingly.
-      self.fused = (not self.renorm and
-                    ndims == 4 and
-                    self.axis in [[1], [3]] and
-                    self.virtual_batch_size is None and
-                    self.adjustment is None)
+      if self._USE_V2_BEHAVIOR:
+        if self.fused is None:
+          self.fused = (ndims == 4)
+        elif self.fused and ndims != 4:
+          raise ValueError('Batch normalization layers with fused=True only '
+                           'support 4D input tensors.')
+      else:
+        assert self.fused is not None
+        self.fused = (ndims == 4 and self._fused_can_be_used())
       # TODO(chrisying): fused batch norm is currently not supported for
       # multi-axis batch norm and by extension virtual batches. In some cases,
       # it might be possible to use fused batch norm but would require reshaping
@@ -672,5 +718,36 @@ class BatchNormalization(Layer):
                       'layer cannot be serialized and has been omitted from '
                       'the layer config. It will not be included when '
                       're-creating the layer from the saved config.')
-    base_config = super(BatchNormalization, self).get_config()
+    base_config = super(BatchNormalizationV2, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
+
+
+def _replace_in_v2_docstring(old, new):
+  string = BatchNormalizationV2.__doc__
+  if old not in string:
+    raise ValueError('Could not find following string in BatchNormalizationV2 '
+                     'docstring: "{}"'.format(old))
+  return string.replace(old, new)
+
+
+@tf_export(v1=['keras.layers.BatchNormalization'])  # pylint: disable=missing-docstring
+class BatchNormalizationV1(BatchNormalizationV2):
+
+  __doc__ = _replace_in_v2_docstring(
+      '''
+    fused: if `True`, use a faster, fused implementation, or raise a ValueError
+      if the fused implementation cannot be used. If `None`, use the faster
+      implementation if possible. If False, do not used the fused
+      implementation.''',
+
+      '''
+    fused: if `None` or `True`, use a faster, fused implementation if possible.
+      If `False`, use the system recommended implementation.''')
+
+  _USE_V2_BEHAVIOR = False
+
+
+if tf2.enabled():
+  BatchNormalization = BatchNormalizationV2
+else:
+  BatchNormalization = BatchNormalizationV1
diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index 92e4128707..2f7f042b7f 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.layers import normalization
 from tensorflow.python.platform import test
 from tensorflow.python.training import gradient_descent
 
@@ -54,6 +55,14 @@ class NormalizationLayersTest(test.TestCase):
         kwargs={'scale': False,
                 'center': False},
         input_shape=(3, 3))
+    testing_utils.layer_test(
+        normalization.BatchNormalizationV2,
+        kwargs={'fused': True},
+        input_shape=(3, 3, 3, 3))
+    testing_utils.layer_test(
+        normalization.BatchNormalizationV2,
+        kwargs={'fused': None},
+        input_shape=(3, 3, 3))
 
   def test_batchnorm_weights(self):
     layer = keras.layers.BatchNormalization(scale=False, center=False)
@@ -78,15 +87,18 @@ class NormalizationLayersTest(test.TestCase):
     self.assertEqual(layer.gamma.constraint, max_norm)
     self.assertEqual(layer.beta.constraint, max_norm)
 
-  def test_batchnorm_correctness(self):
+  def _test_batchnorm_correctness(self, dtype, use_v2=True, fused=False):
     model = keras.models.Sequential()
-    norm = keras.layers.BatchNormalization(input_shape=(10,), momentum=0.8)
+    layer_ctor = (normalization.BatchNormalizationV2 if use_v2
+                  else normalization.BatchNormalizationV1)
+    norm = layer_ctor(input_shape=(2, 2, 2), momentum=0.8, fused=fused)
     model.add(norm)
     model.compile(loss='mse',
                   optimizer=gradient_descent.GradientDescentOptimizer(0.01))
 
     # centered on 5.0, variance 10.0
-    x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
+    x = (np.random.normal(loc=5.0, scale=10.0, size=(1000, 2, 2, 2))
+         .astype(dtype))
     model.fit(x, x, epochs=4, verbose=0)
     out = model.predict(x)
     out -= keras.backend.eval(norm.beta)
@@ -95,23 +107,15 @@ class NormalizationLayersTest(test.TestCase):
     np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
     np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
 
-  def test_batchnorm_mixed_precision(self):
-    model = keras.models.Sequential()
-    norm = keras.layers.BatchNormalization(input_shape=(10,), momentum=0.8)
-    model.add(norm)
-    model.compile(loss='mse',
-                  optimizer=gradient_descent.GradientDescentOptimizer(0.01))
-
-    # centered on 5.0, variance 10.0
-    x = np.random.normal(
-        loc=5.0, scale=10.0, size=(1000, 10)).astype(np.float16)
-    model.fit(x, x, epochs=4, verbose=0)
-    out = model.predict(x)
-    out -= keras.backend.eval(norm.beta)
-    out /= keras.backend.eval(norm.gamma)
+  def test_batchnorm_correctness(self):
+    self._test_batchnorm_correctness(np.float32)
+    self._test_batchnorm_correctness(np.float32, fused=True)
+    self._test_batchnorm_correctness(np.float32, use_v2=False)
 
-    np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
-    np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
+  def test_batchnorm_mixed_precision(self):
+    self._test_batchnorm_correctness(np.float16)
+    self._test_batchnorm_correctness(np.float16, fused=True)
+    self._test_batchnorm_correctness(np.float16, use_v2=False)
 
   def test_batchnorm_convnet(self):
     if test.is_gpu_available(cuda_only=True):
@@ -151,6 +155,77 @@ class NormalizationLayersTest(test.TestCase):
     np.testing.assert_allclose(np.mean(out, axis=(0, 1, 2)), 0.0, atol=1e-1)
     np.testing.assert_allclose(np.std(out, axis=(0, 1, 2)), 1.0, atol=1e-1)
 
+  def test_v1_fused_attribute(self):
+    norm = normalization.BatchNormalizationV1()
+    inp = keras.layers.Input((4, 4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, True)
+
+    norm = normalization.BatchNormalizationV1(fused=False)
+    self.assertEqual(norm.fused, False)
+    inp = keras.layers.Input(shape=(4, 4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, False)
+
+    norm = normalization.BatchNormalizationV1(virtual_batch_size=2)
+    self.assertEqual(norm.fused, True)
+    inp = keras.layers.Input(shape=(2, 2, 2))
+    norm(inp)
+    self.assertEqual(norm.fused, False)
+
+  def test_v2_fused_attribute(self):
+    norm = normalization.BatchNormalizationV2()
+    self.assertEqual(norm.fused, None)
+    inp = keras.layers.Input(shape=(4, 4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, True)
+
+    norm = normalization.BatchNormalizationV2()
+    self.assertEqual(norm.fused, None)
+    inp = keras.layers.Input(shape=(4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, False)
+
+    norm = normalization.BatchNormalizationV2(virtual_batch_size=2)
+    self.assertEqual(norm.fused, False)
+    inp = keras.layers.Input(shape=(4, 4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, False)
+
+    norm = normalization.BatchNormalizationV2(fused=False)
+    self.assertEqual(norm.fused, False)
+    inp = keras.layers.Input(shape=(4, 4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, False)
+
+    norm = normalization.BatchNormalizationV2(fused=True, axis=[3])
+    self.assertEqual(norm.fused, True)
+    inp = keras.layers.Input(shape=(4, 4, 4))
+    norm(inp)
+    self.assertEqual(norm.fused, True)
+
+    with self.assertRaisesRegexp(ValueError, 'fused.*renorm'):
+      normalization.BatchNormalizationV2(fused=True, renorm=True)
+
+    with self.assertRaisesRegexp(ValueError, 'fused.*when axis is 1 or 3'):
+      normalization.BatchNormalizationV2(fused=True, axis=2)
+
+    with self.assertRaisesRegexp(ValueError, 'fused.*when axis is 1 or 3'):
+      normalization.BatchNormalizationV2(fused=True, axis=[1, 3])
+
+    with self.assertRaisesRegexp(ValueError, 'fused.*virtual_batch_size'):
+      normalization.BatchNormalizationV2(fused=True, virtual_batch_size=2)
+
+    with self.assertRaisesRegexp(ValueError, 'fused.*adjustment'):
+      normalization.BatchNormalizationV2(fused=True,
+                                         adjustment=lambda _: (1, 0))
+
+    norm = normalization.BatchNormalizationV2(fused=True)
+    self.assertEqual(norm.fused, True)
+    inp = keras.layers.Input(shape=(4, 4))
+    with self.assertRaisesRegexp(ValueError, '4D input tensors'):
+      norm(inp)
+
 
 class NormalizationLayersGraphModeOnlyTest(test.TestCase):
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
index 820034564f..b3d3c84f92 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.layers.BatchNormalization"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV1\'>"
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
index b132bd43c4..16d9ecce10 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
@@ -1,7 +1,8 @@
 path: "tensorflow.layers.BatchNormalization"
 tf_class {
   is_instance: "<class \'tensorflow.python.layers.normalization.BatchNormalization\'>"
-  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV1\'>"
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
index 820034564f..5da7926812 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.keras.layers.BatchNormalization"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
-- 
GitLab


From f36a16d6f111ee5e7875ef28e1b3026e1ed939ab Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 27 Nov 2018 15:49:17 -0800
Subject: [PATCH 0894/1554] Update BUILD file to include
 matrix_square_root_op_test

PR #20611 missed updating the BUILD file to include the python tests.
Also fixed python test to no longer use test_session.

PiperOrigin-RevId: 223072629
---
 tensorflow/python/kernel_tests/BUILD               | 12 ++++++++++++
 .../kernel_tests/matrix_square_root_op_test.py     | 14 ++++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 19facca5a6..3839928434 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -661,6 +661,18 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "matrix_square_root_op_test",
+    size = "medium",
+    srcs = ["matrix_square_root_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:linalg_ops",
+    ],
+)
+
 cuda_py_test(
     name = "matrix_solve_op_test",
     size = "medium",
diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 1f2144bdee..0bf822a692 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -31,7 +32,7 @@ class SquareRootOpTest(test.TestCase):
 
   def _verifySquareRoot(self, matrix, np_type):
     matrix = matrix.astype(np_type)
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       # Verify that matmul(sqrtm(A), sqrtm(A)) = A
       sqrt = gen_linalg_ops.matrix_square_root(matrix)
       square = math_ops.matmul(sqrt, sqrt)
@@ -96,13 +97,14 @@ class SquareRootOpTest(test.TestCase):
       gen_linalg_ops.matrix_square_root(tensor)
 
   def testNotSquare(self):
-    with self.test_session():
-      with self.assertRaises(ValueError):
-        tensor = constant_op.constant([[1., 0., -1.], [-1., 1., 0.]])
-        gen_linalg_ops.matrix_square_root(tensor).eval()
+    with self.assertRaises(ValueError):
+      tensor = constant_op.constant([[1., 0., -1.], [-1., 1., 0.]])
+      self.evaluate(gen_linalg_ops.matrix_square_root(tensor))
 
   def testConcurrentExecutesWithoutError(self):
-    with self.test_session(use_gpu=True) as sess:
+    self.skipTest("Triggers assert in matrix_sqrt_quasi_triangular_diagonal")
+
+    with test_util.use_gpu():
       matrix1 = random_ops.random_normal([5, 5], seed=42)
       matrix2 = random_ops.random_normal([5, 5], seed=42)
       sqrt1 = gen_linalg_ops.matrix_square_root(matrix1)
-- 
GitLab


From a31118cf96c515238c2bbc2b27e887e40d308f6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 16:07:41 -0800
Subject: [PATCH 0895/1554] Allow XRTCompile to take an XLA DebugOptions to
 configure compilation and aid debugging.

PiperOrigin-RevId: 223075680
---
 tensorflow/compiler/jit/BUILD                 |  1 +
 tensorflow/compiler/xla/client/BUILD          |  3 +-
 .../xla/client/executable_build_options.cc    | 68 +++--------------
 .../xla/client/executable_build_options.h     | 49 ++----------
 .../xla/python/local_computation_builder.i    | 10 +--
 .../compiler/xla/service/local_service.cc     | 30 +-------
 .../xla/tests/xla_hlo_profile_test.cc         |  6 +-
 tensorflow/compiler/xrt/BUILD                 |  6 ++
 .../compiler/xrt/kernels/xrt_compile_ops.cc   |  5 ++
 tensorflow/compiler/xrt/xrt.proto             |  3 +
 tensorflow/compiler/xrt/xrt_util.cc           | 76 +++++++++++++++++++
 tensorflow/compiler/xrt/xrt_util.h            | 34 +++++++++
 12 files changed, 155 insertions(+), 136 deletions(-)
 create mode 100644 tensorflow/compiler/xrt/xrt_util.cc
 create mode 100644 tensorflow/compiler/xrt/xrt_util.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index a3c7dd22de..be91ed4f43 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -268,6 +268,7 @@ cc_library(
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:dump_graph",
         "//tensorflow/compiler/tf2xla:xla_compiler",
+        "//tensorflow/compiler/xla:debug_options_flags",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index e1f3674c4f..ad2e525efd 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -90,11 +90,12 @@ cc_library(
     srcs = ["executable_build_options.cc"],
     hdrs = ["executable_build_options.h"],
     deps = [
+        "//tensorflow/compiler/xla:debug_options_flags",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla:xla_proto",
         "//tensorflow/compiler/xla/service:device_memory_allocator",
-        "//tensorflow/core:lib",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:optional",
diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc
index 0f1745366b..1f594e551a 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.cc
+++ b/tensorflow/compiler/xla/client/executable_build_options.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/executable_build_options.h"
 
 #include "absl/strings/str_format.h"
+#include "tensorflow/compiler/xla/debug_options_flags.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 
 namespace xla {
@@ -39,6 +40,13 @@ ExecutableBuildOptions& ExecutableBuildOptions::set_device_ordinal(
 
 int ExecutableBuildOptions::device_ordinal() const { return device_ordinal_; }
 
+DebugOptions* ExecutableBuildOptions::mutable_debug_options() {
+  if (!has_debug_options()) {
+    debug_options_ = GetDebugOptionsFromFlags();
+  }
+  return &debug_options_.value();
+}
+
 ExecutableBuildOptions& ExecutableBuildOptions::set_result_layout(
     const Shape& shape_with_layout) {
   result_layout_set_ = true;
@@ -55,68 +63,10 @@ string ExecutableBuildOptions::ToString() const {
   if (result_layout_set_) {
     result_layout = ShapeUtil::HumanStringWithLayout(result_layout_);
   }
-  string generate_hlo_graph = "nullopt";
-  if (generate_hlo_graph_.has_value()) {
-    generate_hlo_graph = generate_hlo_graph_.value();
-  }
   return absl::StrFormat(
       "ExecutableBuildOptions{device_ordinal=%d, result_layout=%s, "
       "generate_hlo_graph=%s}",
-      device_ordinal_, result_layout, generate_hlo_graph);
-}
-
-ExecutableBuildOptions& ExecutableBuildOptions::set_generate_hlo_graph(
-    string regex) {
-  generate_hlo_graph_ = std::move(regex);
-  return *this;
-}
-
-const absl::optional<string>& ExecutableBuildOptions::generate_hlo_graph()
-    const {
-  return generate_hlo_graph_;
-}
-
-ExecutableBuildOptions& ExecutableBuildOptions::set_dump_optimized_hlo_proto_to(
-    absl::string_view dirpath) {
-  dump_optimized_hlo_proto_to_ = string(dirpath);
-  return *this;
-}
-
-const absl::optional<string>&
-ExecutableBuildOptions::dump_optimized_hlo_proto_to() const {
-  return dump_optimized_hlo_proto_to_;
-}
-
-ExecutableBuildOptions&
-ExecutableBuildOptions::set_dump_unoptimized_hlo_proto_to(
-    absl::string_view dirpath) {
-  dump_unoptimized_hlo_proto_to_ = string(dirpath);
-  return *this;
-}
-
-const absl::optional<string>&
-ExecutableBuildOptions::dump_unoptimized_hlo_proto_to() const {
-  return dump_unoptimized_hlo_proto_to_;
-}
-
-ExecutableBuildOptions& ExecutableBuildOptions::set_dump_per_pass_hlo_proto_to(
-    absl::string_view dirpath) {
-  dump_per_pass_hlo_proto_to_ = string(dirpath);
-  return *this;
-}
-
-const absl::optional<string>&
-ExecutableBuildOptions::dump_per_pass_hlo_proto_to() const {
-  return dump_per_pass_hlo_proto_to_;
-}
-
-ExecutableBuildOptions& ExecutableBuildOptions::set_hlo_profile(bool enabled) {
-  hlo_profile_ = enabled;
-  return *this;
-}
-
-absl::optional<bool> ExecutableBuildOptions::hlo_profile() const {
-  return hlo_profile_;
+      device_ordinal_, result_layout, debug_options().xla_generate_hlo_graph());
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h
index 93334db88b..dd8cb5598a 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.h
+++ b/tensorflow/compiler/xla/client/executable_build_options.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla.pb.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
@@ -44,6 +45,12 @@ class ExecutableBuildOptions {
   ExecutableBuildOptions& set_result_layout(const Shape& shape_with_layout);
   const Shape* result_layout() const;
 
+  // Expose access to the XLA debug options which will be passed to the
+  // compilation process.
+  bool has_debug_options() const { return debug_options_.has_value(); }
+  const DebugOptions& debug_options() const { return *debug_options_; }
+  DebugOptions* mutable_debug_options();
+
   // If set, this specifies an allocator that can be used to allocate temporary
   // space on the device during compilation.  For example, the compiler might
   // want to run various algorithms on the device and pick the fastest one -- it
@@ -55,56 +62,16 @@ class ExecutableBuildOptions {
       DeviceMemoryAllocator* allocator);
   DeviceMemoryAllocator* device_allocator() const;
 
-  // If set, specifies a regexp of HLO graphs to dump (as in DebugOptions).
-  ExecutableBuildOptions& set_generate_hlo_graph(string regex);
-  const absl::optional<string>& generate_hlo_graph() const;
-
-  // If set, specifies a dirpath to dump the end-of-optimization-pipeline HLO
-  // protobuf to (as in DebugOptions).
-  ExecutableBuildOptions& set_dump_optimized_hlo_proto_to(
-      absl::string_view dirpath);
-  const absl::optional<string>& dump_optimized_hlo_proto_to() const;
-
-  // If set, specifies a dirpath to dump the start-of-optimization-pipeline HLO
-  // protobuf to (as in DebugOptions).
-  ExecutableBuildOptions& set_dump_unoptimized_hlo_proto_to(
-      absl::string_view dirpath);
-  const absl::optional<string>& dump_unoptimized_hlo_proto_to() const;
-
-  // If set, specifies a dirpath to dump the per-pass-in-pipeline HLO protobufs
-  // to (as in DebugOptions).
-  ExecutableBuildOptions& set_dump_per_pass_hlo_proto_to(
-      absl::string_view dirpath);
-  const absl::optional<string>& dump_per_pass_hlo_proto_to() const;
-
-  // If true, specifies that we should record an HLO profile during execution
-  // and log it after execution (as in DebugOptions). If nullopt the default is
-  // used.
-  ExecutableBuildOptions& set_hlo_profile(bool enabled);
-  absl::optional<bool> hlo_profile() const;
-
-  void add_disabled_hlo_pass(absl::string_view pass_name) {
-    disabled_hlo_passes_.push_back(std::string(pass_name));
-  }
-  const absl::Span<const std::string> disabled_hlo_passes() const {
-    return disabled_hlo_passes_;
-  }
-
   // Returns a string representation of the build options, suitable for
   // debugging.
   string ToString() const;
 
  private:
-  absl::optional<bool> hlo_profile_;
   int device_ordinal_ = -1;
   Shape result_layout_;
   bool result_layout_set_ = false;
-  absl::optional<string> generate_hlo_graph_;
-  absl::optional<string> dump_optimized_hlo_proto_to_;
-  absl::optional<string> dump_unoptimized_hlo_proto_to_;
-  absl::optional<string> dump_per_pass_hlo_proto_to_;
+  absl::optional<DebugOptions> debug_options_;
   DeviceMemoryAllocator* device_allocator_ = nullptr;
-  std::vector<std::string> disabled_hlo_passes_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 03698d9ce8..5c2538dcc3 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -921,22 +921,22 @@ tensorflow::ImportNumpy();
     $1 = NULL;
   } else {
     if (!HandleStringAttribute($input, "generate_hlo_graph", [&](string s) {
-      build_options.set_generate_hlo_graph(std::move(s));
+      build_options.mutable_debug_options()->set_xla_generate_hlo_graph(std::move(s));
     })) {
       return nullptr;
     }
     if (!HandleStringAttribute($input, "dump_optimized_hlo_proto_to", [&](string s) {
-      build_options.set_dump_optimized_hlo_proto_to(std::move(s));
+      build_options.mutable_debug_options()->set_xla_dump_optimized_hlo_proto_to(std::move(s));
     })) {
       return nullptr;
     }
     if (!HandleStringAttribute($input, "dump_unoptimized_hlo_proto_to", [&](string s) {
-      build_options.set_dump_unoptimized_hlo_proto_to(std::move(s));
+      build_options.mutable_debug_options()->set_xla_dump_unoptimized_hlo_proto_to(std::move(s));
     })) {
       return nullptr;
     }
     if (!HandleStringAttribute($input, "dump_per_pass_hlo_proto_to", [&](string s) {
-      build_options.set_dump_per_pass_hlo_proto_to(std::move(s));
+      build_options.mutable_debug_options()->set_xla_dump_per_pass_hlo_proto_to(std::move(s));
     })) {
       return nullptr;
     }
@@ -950,7 +950,7 @@ tensorflow::ImportNumpy();
         PyErr_SetString(PyExc_TypeError, "ExecutableBuildOptions.hlo_profile must be a bool or None.");
         SWIG_fail;
       }
-      build_options.set_hlo_profile(o == Py_True);
+      build_options.mutable_debug_options()->set_xla_hlo_profile(o == Py_True);
     }
     Py_DECREF(o);
 
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 5c105908f3..ddc8691e77 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -96,28 +96,8 @@ ExecutionOptions CreateExecutionOptions(
     const ExecutableBuildOptions& build_options,
     const ProgramShape* program_shape) {
   ExecutionOptions execution_options = CreateDefaultExecutionOptions();
-  if (build_options.hlo_profile().has_value()) {
-    execution_options.mutable_debug_options()->set_xla_hlo_profile(
-        *build_options.hlo_profile());
-  }
-  if (build_options.generate_hlo_graph().has_value()) {
-    execution_options.mutable_debug_options()->set_xla_generate_hlo_graph(
-        build_options.generate_hlo_graph().value());
-  }
-  if (build_options.dump_optimized_hlo_proto_to().has_value()) {
-    execution_options.mutable_debug_options()
-        ->set_xla_dump_optimized_hlo_proto_to(
-            build_options.dump_optimized_hlo_proto_to().value());
-  }
-  if (build_options.dump_unoptimized_hlo_proto_to().has_value()) {
-    execution_options.mutable_debug_options()
-        ->set_xla_dump_unoptimized_hlo_proto_to(
-            build_options.dump_unoptimized_hlo_proto_to().value());
-  }
-  if (build_options.dump_per_pass_hlo_proto_to().has_value()) {
-    execution_options.mutable_debug_options()
-        ->set_xla_dump_per_pass_hlo_proto_to(
-            build_options.dump_per_pass_hlo_proto_to().value());
+  if (build_options.has_debug_options()) {
+    *execution_options.mutable_debug_options() = build_options.debug_options();
   }
   if (build_options.result_layout() != nullptr) {
     *execution_options.mutable_shape_with_output_layout() =
@@ -128,12 +108,6 @@ ExecutionOptions CreateExecutionOptions(
     LayoutUtil::SetToDefaultLayout(
         execution_options.mutable_shape_with_output_layout());
   }
-
-  for (const std::string& disabled_pass : build_options.disabled_hlo_passes()) {
-    execution_options.mutable_debug_options()->add_xla_disable_hlo_passes(
-        disabled_pass);
-  }
-
   return execution_options;
 }
 
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index ca036f1ae0..e57d072a06 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -157,10 +157,12 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client,
   TF_ASSERT_OK(transfer_manager->TransferLiteralToDevice(
       stream_ptr.get(), Literal::CreateFromShape(rhs_arg_shape), rhs_arg));
 
+  ExecutableBuildOptions build_options;
+  build_options.mutable_debug_options()->set_xla_hlo_profile(true);
   TF_ASSERT_OK_AND_ASSIGN(
       std::unique_ptr<LocalExecutable> local_executable,
       client->Compile(computation, {&lhs_arg_shape, &rhs_arg_shape},
-                      ExecutableBuildOptions().set_hlo_profile(true)));
+                      build_options));
 
   Executable* executable = local_executable->executable();
   HloExecutionProfile hlo_execution_profile(
@@ -208,7 +210,7 @@ XLA_TEST_F(HloProfileTest, ProfileSingleComputation) {
   string profile_output;
   ExecuteAndFetchProfile(&profile_output, client, computation, lhs_shape,
                          rhs_shape);
-
+  VLOG(4) << "Profile Output:\n" << profile_output;
   std::vector<string> profile_output_lines =
       absl::StrSplit(profile_output, '\n');
 
diff --git a/tensorflow/compiler/xrt/BUILD b/tensorflow/compiler/xrt/BUILD
index 2ff97914f8..2dae746d03 100644
--- a/tensorflow/compiler/xrt/BUILD
+++ b/tensorflow/compiler/xrt/BUILD
@@ -22,6 +22,7 @@ xla_proto_library(
     deps = [
         "//tensorflow/compiler/tf2xla:host_compute_metadata_proto",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla:xla_proto",
         "//tensorflow/compiler/xla/service:hlo_proto",
     ],
 )
@@ -32,20 +33,25 @@ cc_library(
         "xrt_compilation_cache.cc",
         "xrt_device.cc",
         "xrt_state.cc",
+        "xrt_util.cc",
     ],
     hdrs = [
         "xrt_compilation_cache.h",
         "xrt_device.h",
         "xrt_state.h",
+        "xrt_util.h",
     ],
     deps = [
         "//tensorflow/compiler/jit:xla_device",
         "//tensorflow/compiler/tf2xla:xla_compiler",
+        "//tensorflow/compiler/xla:debug_options_flags",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla:xla_proto",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/service:backend",
         "//tensorflow/compiler/xla/service:device_memory_allocator",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
index db43aeaafe..1603b45ff6 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/compiler/xrt/xrt.pb.h"
 #include "tensorflow/compiler/xrt/xrt_compilation_cache.h"
 #include "tensorflow/compiler/xrt/xrt_device.h"
+#include "tensorflow/compiler/xrt/xrt_util.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -117,6 +118,10 @@ Status XRTCompileOp::Compile(OpKernelContext* ctx,
   build_options.set_device_ordinal(client->default_device_ordinal());
   build_options.set_result_layout(config.program_shape().result());
   build_options.set_device_allocator(device_ref.backend()->memory_allocator());
+  if (config.has_debug_options()) {
+    *build_options.mutable_debug_options() =
+        BuildXlaDebugOptions(config.debug_options());
+  }
 
   VLOG(1) << "Building executable";
   auto compile_result =
diff --git a/tensorflow/compiler/xrt/xrt.proto b/tensorflow/compiler/xrt/xrt.proto
index ae44f71740..e149f2f435 100644
--- a/tensorflow/compiler/xrt/xrt.proto
+++ b/tensorflow/compiler/xrt/xrt.proto
@@ -3,6 +3,7 @@ syntax = "proto3";
 package xrt;
 
 import "tensorflow/compiler/tf2xla/host_compute_metadata.proto";
+import "tensorflow/compiler/xla/xla.proto";
 import "tensorflow/compiler/xla/xla_data.proto";
 import "tensorflow/compiler/xla/service/hlo.proto";
 
@@ -46,6 +47,8 @@ message XLAComputationConfig {
   // sent and executed to the set of replica device numbers described in the
   // DeviceAssignment proto.
   DeviceAssignment device_assignment = 6;
+  // The debugging options to be passed to the XLA compilation process.
+  xla.DebugOptions debug_options = 7;
 }
 
 // Options and XLA computation for a compilation.
diff --git a/tensorflow/compiler/xrt/xrt_util.cc b/tensorflow/compiler/xrt/xrt_util.cc
new file mode 100644
index 0000000000..3ef8bedc73
--- /dev/null
+++ b/tensorflow/compiler/xrt/xrt_util.cc
@@ -0,0 +1,76 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xrt/xrt_util.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "tensorflow/compiler/xla/debug_options_flags.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace tensorflow {
+namespace {
+
+bool DebugOptionsPassThroughEnabled() {
+  const char* env = getenv("TF_XLA_DEBUG_OPTIONS_PASSTHROUGH");
+  bool enabled =
+      env != nullptr && (strcmp(env, "1") == 0 || strcmp(env, "true") == 0);
+  if (enabled) {
+    LOG(WARNING) << "Passing through XLA debug options!";
+  } else {
+    LOG(WARNING) << "TF_XLA_DEBUG_OPTIONS_PASSTHROUGH not set, not all options "
+                    "will be retained";
+  }
+  return enabled;
+}
+
+string SafeDebugPath(const string& path) {
+  if (path.empty() || path.compare(0, 5, "gs://") == 0 ||
+      path.compare(0, 11, "bigstore://") == 0) {
+    return path;
+  }
+  LOG(WARNING) << "Invalid config path (will be dropped): " << path;
+  return string();
+}
+
+}  // namespace
+
+xla::DebugOptions BuildXlaDebugOptions(const xla::DebugOptions& ref_options) {
+  static const bool options_passthrough = DebugOptionsPassThroughEnabled();
+  if (options_passthrough) {
+    return ref_options;
+  }
+  xla::DebugOptions options = xla::GetDebugOptionsFromFlags();
+  options.set_xla_generate_hlo_text_to(
+      SafeDebugPath(ref_options.xla_generate_hlo_text_to()));
+  options.set_xla_dump_optimized_hlo_proto_to(
+      SafeDebugPath(ref_options.xla_dump_optimized_hlo_proto_to()));
+  options.set_xla_dump_computations_to(
+      SafeDebugPath(ref_options.xla_dump_computations_to()));
+  options.set_xla_dump_executions_to(
+      SafeDebugPath(ref_options.xla_dump_executions_to()));
+  for (auto& pass : ref_options.xla_disable_hlo_passes()) {
+    options.add_xla_disable_hlo_passes(pass);
+  }
+  options.set_xla_dump_unoptimized_hlo_proto_to(
+      SafeDebugPath(ref_options.xla_dump_unoptimized_hlo_proto_to()));
+  options.set_xla_dump_per_pass_hlo_proto_to(
+      SafeDebugPath(ref_options.xla_dump_per_pass_hlo_proto_to()));
+  return options;
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/xrt/xrt_util.h b/tensorflow/compiler/xrt/xrt_util.h
new file mode 100644
index 0000000000..d9c05a7f34
--- /dev/null
+++ b/tensorflow/compiler/xrt/xrt_util.h
@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Utility functions in support of the XRT API.
+
+#ifndef TENSORFLOW_COMPILER_XRT_XRT_UTIL_H_
+#define TENSORFLOW_COMPILER_XRT_XRT_UTIL_H_
+
+#include "tensorflow/compiler/xla/xla.pb.h"
+
+namespace tensorflow {
+
+// Filters the debug options provided as argument according to the value of the
+// TF_XLA_DEBUG_OPTIONS_PASSTHROUGH environment variable. If such variable is
+// set to "1" or "true", the debug options will be returned as is. Otherwise
+// only a subset of them will be set in the returned ones, and all the paths
+// contained in it, will be limited to gs:// and bigstore:// ones.
+xla::DebugOptions BuildXlaDebugOptions(const xla::DebugOptions& ref_options);
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_XRT_XRT_UTIL_H_
-- 
GitLab


From 06d4711e8cf98165e16d125db88edac461764f9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 16:14:42 -0800
Subject: [PATCH 0896/1554] Add batch_jacobian to GradientTape.

PiperOrigin-RevId: 223076782
---
 tensorflow/python/eager/backprop.py           | 128 +++++++++++++++++-
 tensorflow/python/eager/backprop_test.py      |  94 +++++++++++++
 .../golden/v1/tensorflow.-gradient-tape.pbtxt |   4 +
 .../golden/v2/tensorflow.-gradient-tape.pbtxt |   4 +
 4 files changed, 223 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 99da422781..beed1d5de3 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import functools
 import operator
+import sys
 
 import six
 
@@ -33,6 +34,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import math_ops
@@ -1004,9 +1006,8 @@ class GradientTape(object):
       self._push_tape()
       y = array_ops.gather(target, i)
       self._pop_tape()
-      grad = self.gradient(y, flat_sources,
+      return self.gradient(y, flat_sources,
                            unconnected_gradients=unconnected_gradients)
-      return grad
 
     try:
       target_size = int(target.shape[0])
@@ -1021,11 +1022,13 @@ class GradientTape(object):
       try:
         output = f()
       except ValueError as err:
-        # TODO(agarwal): Fold this error message into err.
-        logging.error("Encountered an exception while vectorizing the jacobian "
-                      "computation. Vectorization can be disabled by setting "
-                      "experimental_use_pfor to False.")
-        raise err
+        six.reraise(
+            ValueError,
+            ValueError(
+                str(err) + "\nEncountered an exception while vectorizing the "
+                "jacobian computation. Vectorization can be disabled by setting"
+                " experimental_use_pfor to False."),
+            sys.exc_info()[2])
     else:
       if context.executing_eagerly():
         if not self._persistent:
@@ -1046,3 +1049,114 @@ class GradientTape(object):
       output[i] = out
 
     return nest.pack_sequence_as(sources, output)
+
+  def batch_jacobian(self,
+                     target,
+                     source,
+                     unconnected_gradients=UnconnectedGradients.NONE,
+                     experimental_use_pfor=True):
+    """Computes and stacks per-example jacobians.
+
+    See http://en.wikipedia.org/wiki/jacobian_matrix_and_determinant for the
+    definition of a Jacobian.  This function is essentially an efficient
+    implementation of the following:
+    `tf.stack([self.jacobian(y[i], x[i]) for i in range(x.shape[0])])`.
+
+    Note that compared to `GradientTape.jacobian` which computes gradient of
+    each output value w.r.t each input value, this function is useful when
+    `target[i,...] is independent of `source[j,...]` for `j != i`. This
+    independence assumption allows more efficient computation as compared to
+    `GradientTape.jacobian`. The output, as well as intermediate activations,
+    are lower dimensional and avoid a bunch of redundant zeros which would
+    result in the jacobian computation given the independence assumption.
+
+    Example usage:
+    with tf.GradientTape() as g:
+      x = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
+      g.watch(x)
+      y = x * x
+    batch_jacobian = g.batch_jacobian(y, x)
+    # batch_jacobian is [[[2,  0], [0,  4]], [[6,  0], [0,  8]]]
+
+    Args:
+      target: A tensor with rank 2 or higher and with shape [b, y1, ..., y_n].
+        `target[i,...]` should only depend on `source[i,...]`.
+      source: A tensor with rank 2 or higher and with shape [b, x1, ..., x_m].
+      unconnected_gradients: a value which can either hold 'none' or 'zero' and
+        alters the value which will be returned if the target and sources are
+        unconnected. The possible values and effects are detailed in
+        'UnconnectedGradients' and it defaults to 'none'.
+      experimental_use_pfor: If true, uses pfor for computing the Jacobian. Else
+        uses a tf.while_loop.
+
+    Returns:
+      A tensor `t` with shape [b, y_1, ..., y_n, x1, ..., x_m] where `t[i, ...]`
+      is the jacobian of `target[i, ...]` w.r.t. `source[i, ...]`, i.e. stacked
+      per-example jacobians.
+
+    Raises:
+      RuntimeError: If called on a non-persistent tape with eager execution
+        enabled and without enabling experimental_use_pfor.
+      ValueError: If vectorization of jacobian computation fails or if first
+        dimension of `target` and `source` do not match.
+    """
+    target_shape = target.shape
+    if not target_shape.with_rank_at_least(2)[0].is_compatible_with(
+        source.shape.with_rank_at_least(2)[0]):
+      raise ValueError(
+          "Need first dimension of target shape (%s) and "
+          "source shape (%s) to match." % (target.shape, source.shape))
+    if target_shape.is_fully_defined():
+      batch_size = int(target_shape[0])
+      target_row_size = target_shape.num_elements() // batch_size
+    else:
+      target_shape = array_ops.shape(target)
+      batch_size = target_shape[0]
+      target_row_size = array_ops.size(target) // batch_size
+    source_shape = array_ops.shape(source)
+    # Flatten target to 2-D.
+    # Note that we push and pop the tape here and below. This is needed since we
+    # need gradients through the enclosed operations.
+    self._push_tape()
+    with ops.control_dependencies(
+        [check_ops.assert_equal(batch_size, source_shape[0])]):
+      target = array_ops.reshape(target, [batch_size, target_row_size])
+    self._pop_tape()
+
+    def loop_fn(i):
+      self._push_tape()
+      y = array_ops.gather(target, i, axis=1)
+      self._pop_tape()
+      return self.gradient(y, source,
+                           unconnected_gradients=unconnected_gradients)
+
+    if experimental_use_pfor:
+      def f():
+        return pfor_ops.pfor(loop_fn, target_row_size)
+      if context.executing_eagerly():
+        f = function.defun(f)
+      try:
+        output = f()
+      except ValueError as err:
+        six.reraise(
+            ValueError,
+            ValueError(
+                str(err) + "\nEncountered an exception while vectorizing the "
+                "batch_jacobian computation. Vectorization can be disabled by "
+                "setting experimental_use_pfor to False."),
+            sys.exc_info()[2])
+    else:
+      if context.executing_eagerly():
+        if not self._persistent:
+          raise RuntimeError(
+              "GradientTape must be created with persistent=True"
+              " to compute the batch_jacobian with eager execution enabled and "
+              " with experimental_use_pfor set to False.")
+      output = pfor_ops.for_loop(loop_fn, target.dtype, target_row_size)
+    if output is None:
+      return None
+    output = array_ops.reshape(output,
+                               [target_row_size, batch_size, -1])
+    output = array_ops.transpose(output, [1, 0, 2])
+    new_shape = array_ops.concat([target_shape, source_shape[1:]], axis=0)
+    return array_ops.reshape(output, new_shape)
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 8b85548e5c..08553b9f27 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -1303,5 +1303,99 @@ class JacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'No converter'):
       g.jacobian(y, x, experimental_use_pfor=True)
 
+
+@test_util.run_all_in_graph_and_eager_modes
+class BatchJacobianTest(test.TestCase):
+
+  def _batch_jacobian(self, experimental_use_pfor):
+    persistent = context.executing_eagerly and not experimental_use_pfor
+    with backprop.GradientTape(persistent=persistent) as g:
+      x = constant_op.constant([[1., 2.], [3., 4.]])
+      y = constant_op.constant([[3., 4.], [5., 6.]])
+      g.watch(x)
+      z = x * x * y
+    batch_jacobian = g.batch_jacobian(
+        z, x, experimental_use_pfor=experimental_use_pfor)
+    answer = array_ops.stack([array_ops.diag(2 * x[0] * y[0]),
+                              array_ops.diag(2 * x[1] * y[1])])
+    return batch_jacobian, answer
+
+  def testPfor(self):
+    batch_jacobian, answer = self._batch_jacobian(experimental_use_pfor=True)
+    self.assertAllEqual(answer, batch_jacobian)
+
+  def testWhileLoop(self):
+    batch_jacobian, answer = self._batch_jacobian(experimental_use_pfor=False)
+    self.assertAllEqual(answer, batch_jacobian)
+
+  def testPforDefun(self):
+
+    @function.defun
+    def _f():
+      return self._batch_jacobian(experimental_use_pfor=True)
+
+    batch_jacobian, answer = _f()
+    self.assertAllEqual(answer, batch_jacobian)
+
+  def testWhileLoopDefun(self):
+
+    @function.defun
+    def _f():
+      return self._batch_jacobian(experimental_use_pfor=False)
+
+    batch_jacobian, answer = _f()
+    self.assertAllEqual(answer, batch_jacobian)
+
+  def testPersistentTape(self):
+    if not context.executing_eagerly():
+      return
+    with backprop.GradientTape() as g:
+      x = constant_op.constant([[1.0, 2.0]])
+      g.watch(x)
+      y = x * x
+    with self.assertRaisesRegexp(RuntimeError, 'persistent'):
+      g.batch_jacobian(y, x, experimental_use_pfor=False)
+
+  def testBadShape(self):
+    x = random_ops.random_uniform([2, 3])
+    with backprop.GradientTape() as g:
+      y = array_ops.concat([x, x], axis=0)
+    with self.assertRaisesRegexp(ValueError, 'Need first dimension'):
+      g.batch_jacobian(y, x)
+
+  def testBadInputRank(self):
+    x = random_ops.random_uniform([2])
+    with backprop.GradientTape() as g:
+      y = random_ops.random_uniform([2, 2])
+    with self.assertRaisesRegexp(ValueError, 'must have rank at least 2'):
+      g.batch_jacobian(y, x)
+
+  def testBadOutputRank(self):
+    x = random_ops.random_uniform([2, 2])
+    with backprop.GradientTape() as g:
+      y = random_ops.random_uniform([2])
+    with self.assertRaisesRegexp(ValueError, 'must have rank at least 2'):
+      g.batch_jacobian(y, x)
+
+  def testPforException(self):
+    var = variables.Variable([1.])
+
+    @custom_gradient.custom_gradient
+    def op(x):
+      def grad(_):
+        # Note that we perform a stateful operation here that will not be
+        # compatible with parallel for construct.
+        with ops.control_dependencies(
+            [var.assign(random_ops.random_uniform([1]))]):
+          return constant_op.constant(1.)
+      return x, grad
+
+    with backprop.GradientTape() as g:
+      x = constant_op.constant([[1.], [2.]])
+      g.watch(x)
+      y = op(x)
+    with self.assertRaisesRegexp(ValueError, 'No converter'):
+      g.batch_jacobian(y, x, experimental_use_pfor=True)
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
index 50af42f4fc..e37d29995a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
@@ -6,6 +6,10 @@ tf_class {
     name: "__init__"
     argspec: "args=[\'self\', \'persistent\', \'watch_accessed_variables\'], varargs=None, keywords=None, defaults=[\'False\', \'True\'], "
   }
+  member_method {
+    name: "batch_jacobian"
+    argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+  }
   member_method {
     name: "gradient"
     argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
index 50af42f4fc..e37d29995a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
@@ -6,6 +6,10 @@ tf_class {
     name: "__init__"
     argspec: "args=[\'self\', \'persistent\', \'watch_accessed_variables\'], varargs=None, keywords=None, defaults=[\'False\', \'True\'], "
   }
+  member_method {
+    name: "batch_jacobian"
+    argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+  }
   member_method {
     name: "gradient"
     argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'UnconnectedGradients.NONE\'], "
-- 
GitLab


From 0ae47a7f3a3cbfb47a432741d525bafc50c6b68a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 16:22:30 -0800
Subject: [PATCH 0897/1554] Fix unit test for matrix square root: Don't try to
 take the matrix square root of a matrix for which the square root may not
 exist.

PiperOrigin-RevId: 223078107
---
 .../python/kernel_tests/matrix_square_root_op_test.py     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 0bf822a692..1e2109b8c4 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -102,13 +102,13 @@ class SquareRootOpTest(test.TestCase):
       self.evaluate(gen_linalg_ops.matrix_square_root(tensor))
 
   def testConcurrentExecutesWithoutError(self):
-    self.skipTest("Triggers assert in matrix_sqrt_quasi_triangular_diagonal")
-
     with test_util.use_gpu():
       matrix1 = random_ops.random_normal([5, 5], seed=42)
       matrix2 = random_ops.random_normal([5, 5], seed=42)
-      sqrt1 = gen_linalg_ops.matrix_square_root(matrix1)
-      sqrt2 = gen_linalg_ops.matrix_square_root(matrix2)
+      square1 = math_ops.matmul(matrix1, matrix1)
+      square2 = math_ops.matmul(matrix2, matrix2)
+      sqrt1 = gen_linalg_ops.matrix_square_root(square1)
+      sqrt2 = gen_linalg_ops.matrix_square_root(square2)
       all_ops = [sqrt1, sqrt2]
       sqrt = self.evaluate(all_ops)
       self.assertAllEqual(sqrt[0], sqrt[1])
-- 
GitLab


From cdea9d0c8a559f7e387b5d2510b6986516e5495d Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Tue, 27 Nov 2018 16:22:39 -0800
Subject: [PATCH 0898/1554] Small addition for test method name consistency.

PiperOrigin-RevId: 223078123
---
 tensorflow/python/framework/test_util.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index a034032e39..fc97a2275c 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -892,8 +892,8 @@ def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
   for name, value in cls.__dict__.copy().items():
-    if callable(value) and name.startswith(
-        "test") and not name.startswith("testSkipEager"):
+    if callable(value) and name.startswith("test") and not (
+        name.startswith("testSkipEager") or name.startswith("test_skip_eager")):
       setattr(cls, name, base_decorator(value))
   return cls
 
-- 
GitLab


From f4f859072f7168190dea6b3f6a1a50bfde728eb1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 16:27:43 -0800
Subject: [PATCH 0899/1554] Tests for placeholder_with_default

PiperOrigin-RevId: 223078802
---
 tensorflow/lite/build_def.bzl                |  1 +
 tensorflow/lite/testing/generate_examples.py | 27 ++++++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index fcd7255910..5d615f0c5a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -269,6 +269,7 @@ def generated_test_models():
         "pack",
         "pad",
         "padv2",
+        "placeholder_with_default",
         "prelu",
         "pow",
         "range",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 5218844299..b143f45b32 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3528,6 +3528,33 @@ def make_unroll_batch_matmul_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_placeholder_with_default_tests(zip_path):
+  """Make a set of tests to test placeholder_with_default."""
+
+  test_parameters = [{
+      "dtype": [tf.float32, tf.int32, tf.int64],
+  }]
+
+  def build_graph(parameters):
+    """Build the placeholder_with_default testing graph."""
+    const_node = tf.constant(
+        [1, 2, 2, 0], shape=[2, 2], dtype=parameters["dtype"])
+    input_tensor = tf.placeholder_with_default(
+        const_node, shape=[2, 2], name="input")
+    out = tf.equal(input_tensor, const_node, name="output")
+
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    numpy_type = _TF_TYPE_INFO[parameters["dtype"]][0]
+    input_value = np.array([[1, 0], [2, 1]], numpy_type)
+    return [input_value], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_value])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
+                    expected_tf_success=3)
+
+
 # Toco binary path provided by the generate rule.
 bin_path = None
 
-- 
GitLab


From f1d0c84f699624382c8d66e2ea10205ac0207868 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 16:29:56 -0800
Subject: [PATCH 0900/1554] Skip overflow testing when running with address
 sanitizer.

PiperOrigin-RevId: 223079147
---
 .../experimental/micro/examples/micro_speech/timer_test.cc   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
index 83a2dfcc65..0487a12b25 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
@@ -26,10 +26,13 @@ TF_LITE_MICRO_TESTS_BEGIN
 TF_LITE_MICRO_TEST(TestTimer) {
   // Make sure that the technically-undefined overflow behavior we rely on below
   // works on this platform. It's still not guaranteed, but at least this is a
-  // sanity check.
+  // sanity check.  Turn off when running with ASan, as it will complain about
+  // the following undefined behavior.
+#ifndef ADDRESS_SANITIZER
   int32_t overflow_value = std::numeric_limits<int32_t>::max();
   overflow_value += 1;
   TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
+#endif
 
   const int32_t first_time = TimeInMilliseconds();
   const int32_t second_time = TimeInMilliseconds();
-- 
GitLab


From 6f09a093d1e5e6947204db1b5fc1d85032e6a78d Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 27 Nov 2018 16:29:56 -0800
Subject: [PATCH 0901/1554] [Go]: Fix #23257

By temporarily disabling the features introduced in
PR #20412

Will re-enable after the C libraries for TensorFlow 1.13 or later
have been released.

PiperOrigin-RevId: 223079148
---
 tensorflow/go/graph.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go
index 702ce85228..6ff41ca916 100644
--- a/tensorflow/go/graph.go
+++ b/tensorflow/go/graph.go
@@ -112,9 +112,17 @@ func (g *Graph) ImportWithOptions(def []byte, options GraphImportOptions) error
 	C.TF_ImportGraphDefOptionsSetPrefix(opts, cprefix)
 
 	if len(options.Device) != 0 {
-		cdev := C.CString(options.Device)
-		defer C.free(unsafe.Pointer(cdev))
-		C.TF_ImportGraphDefOptionsSetDefaultDevice(opts, cdev)
+		// TODO(ashankar): Remove this error and uncomment below
+		// when a release of the C library which includes
+		// https://github.com/tensorflow/tensorflow/commit/e0af5ac53e5a8ad9b07cdd5738c0a8e12f938c4e
+		// has been made.
+		// See https://github.com/tensorflow/tensorflow/issues/23257
+		return fmt.Errorf("GraphImportOptions.Device is only supported with the TensorFlow C library versions after 1.12 (or built from master). See https://github.com/tensorflow/tensorflow/issues/23257")
+		/*
+			cdev := C.CString(options.Device)
+			defer C.free(unsafe.Pointer(cdev))
+			C.TF_ImportGraphDefOptionsSetDefaultDevice(opts, cdev)
+		*/
 	}
 
 	buf := C.TF_NewBuffer()
-- 
GitLab


From 8bf4052fc3123cb141bccd67b007fb084d319d12 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 27 Nov 2018 16:39:09 -0800
Subject: [PATCH 0902/1554] Fix binary accuracy test. Remove the old stateful
 metrics test.

PiperOrigin-RevId: 223080473
---
 tensorflow/python/keras/metrics_test.py | 118 ------------------------
 1 file changed, 118 deletions(-)

diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index eda434391e..536f717517 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -28,13 +28,10 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import layers
 from tensorflow.python.keras import metrics
-from tensorflow.python.keras.engine.training import Model
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
@@ -131,116 +128,6 @@ class KerasMetricsTest(test.TestCase):
       result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=1))
       self.assertEqual(result, 0.)
 
-  def test_stateful_metrics(self):
-    with self.cached_session():
-      np.random.seed(1334)
-
-      class BinaryTruePositives(layers.Layer):
-        """Stateful Metric to count the total true positives over all batches.
-
-        Assumes predictions and targets of shape `(samples, 1)`.
-
-        Arguments:
-            threshold: Float, lower limit on prediction value that counts as a
-                positive class prediction.
-            name: String, name for the metric.
-        """
-
-        def __init__(self, name='true_positives', **kwargs):
-          super(BinaryTruePositives, self).__init__(name=name, **kwargs)
-          self.true_positives = K.variable(value=0, dtype='int32')
-          self.stateful = True
-
-        def reset_states(self):
-          K.set_value(self.true_positives, 0)
-
-        def __call__(self, y_true, y_pred):
-          """Computes the number of true positives in a batch.
-
-          Args:
-              y_true: Tensor, batch_wise labels
-              y_pred: Tensor, batch_wise predictions
-
-          Returns:
-              The total number of true positives seen this epoch at the
-                  completion of the batch.
-          """
-          y_true = math_ops.cast(y_true, 'int32')
-          y_pred = math_ops.cast(math_ops.round(y_pred), 'int32')
-          correct_preds = math_ops.cast(math_ops.equal(y_pred, y_true), 'int32')
-          true_pos = math_ops.cast(
-              math_ops.reduce_sum(correct_preds * y_true), 'int32')
-          current_true_pos = self.true_positives * 1
-          self.add_update(
-              state_ops.assign_add(self.true_positives, true_pos),
-              inputs=[y_true, y_pred])
-          return current_true_pos + true_pos
-
-      metric_fn = BinaryTruePositives()
-      config = metrics.serialize(metric_fn)
-      metric_fn = metrics.deserialize(
-          config, custom_objects={'BinaryTruePositives': BinaryTruePositives})
-
-      # Test on simple model
-      inputs = layers.Input(shape=(2,))
-      outputs = layers.Dense(1, activation='sigmoid')(inputs)
-      model = Model(inputs, outputs)
-      model.compile(optimizer='sgd',
-                    loss='binary_crossentropy',
-                    metrics=['acc', metric_fn])
-
-      # Test fit, evaluate
-      samples = 100
-      x = np.random.random((samples, 2))
-      y = np.random.randint(2, size=(samples, 1))
-      val_samples = 10
-      val_x = np.random.random((val_samples, 2))
-      val_y = np.random.randint(2, size=(val_samples, 1))
-
-      history = model.fit(x, y,
-                          epochs=1,
-                          batch_size=10,
-                          validation_data=(val_x, val_y))
-      outs = model.evaluate(x, y, batch_size=10)
-      preds = model.predict(x)
-
-      def ref_true_pos(y_true, y_pred):
-        return np.sum(np.logical_and(y_pred > 0.5, y_true == 1))
-
-      # Test correctness (e.g. updates should have been run)
-      self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
-
-      # Test correctness of the validation metric computation
-      val_preds = model.predict(val_x)
-      val_outs = model.evaluate(val_x, val_y, batch_size=10)
-      self.assertAllClose(
-          val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5)
-      self.assertAllClose(
-          val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
-
-      # Test with generators
-      gen = [(np.array([x0]), np.array([y0])) for x0, y0 in zip(x, y)]
-      val_gen = [(np.array([x0]), np.array([y0]))
-                 for x0, y0 in zip(val_x, val_y)]
-      history = model.fit_generator(iter(gen),
-                                    epochs=1,
-                                    steps_per_epoch=samples,
-                                    validation_data=iter(val_gen),
-                                    validation_steps=val_samples)
-      outs = model.evaluate_generator(iter(gen), steps=samples)
-      preds = model.predict_generator(iter(gen), steps=samples)
-
-      # Test correctness of the metric results
-      self.assertAllClose(outs[2], ref_true_pos(y, preds), atol=1e-5)
-
-      # Test correctness of the validation metric computation
-      val_preds = model.predict_generator(iter(val_gen), steps=val_samples)
-      val_outs = model.evaluate_generator(iter(val_gen), steps=val_samples)
-      self.assertAllClose(
-          val_outs[2], ref_true_pos(val_y, val_preds), atol=1e-5)
-      self.assertAllClose(
-          val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
-
   @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
   def test_mean(self):
     m = metrics.Mean(name='my_mean')
@@ -423,11 +310,6 @@ class KerasMetricsTest(test.TestCase):
     result = self.evaluate(result_t)
     self.assertAlmostEqual(result, 0.67, 2)  # 4.5/6.7
 
-    # check incompatible shapes
-    with self.assertRaisesRegexp(ValueError,
-                                 r'Shapes \(1,\) and \(2,\) are incompatible'):
-      acc_obj.update_state([1, 1], [1])
-
   @test_util.run_in_graph_and_eager_modes
   def test_binary_accuracy_threshold(self):
     acc_obj = metrics.BinaryAccuracy(threshold=0.7)
-- 
GitLab


From a461299229410d886aaf31e9998ffc7b53f70708 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 27 Nov 2018 16:44:18 -0800
Subject: [PATCH 0903/1554] [tf.data] Remove `tf.data.Iterator` from the V2
 API.

In TensorFlow 2.0, iterating over a `tf.data.Dataset` is possible using a
standard Python for loop:

```python
dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(bs)
for feature_batch, label_batch in dataset:
  # Train on one minibatch.
```

Since eager execution allows iterators to be created and bound to computation
dynamically, there is no need for the concepts of "one-shot", "initializable",
"feedable", and "reinitializable" iterators. This change is a step towards
simplifying the `tf.data` API.

PiperOrigin-RevId: 223081239
---
 tensorflow/python/data/ops/iterator_ops.py    |  2 +-
 .../golden/v2/tensorflow.data.-iterator.pbtxt | 46 -------------------
 .../tools/api/golden/v2/tensorflow.data.pbtxt |  4 --
 3 files changed, 1 insertion(+), 51 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt

diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index 68b03ba93b..e2ca64c802 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -68,7 +68,7 @@ def _device_stack_is_empty():
   return not bool(device_stack)
 
 
-@tf_export("data.Iterator")
+@tf_export(v1=["data.Iterator"])
 class Iterator(checkpointable.CheckpointableBase):
   """Represents the state of iterating through a `Dataset`."""
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt
deleted file mode 100644
index 4f0147a523..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-iterator.pbtxt
+++ /dev/null
@@ -1,46 +0,0 @@
-path: "tensorflow.data.Iterator"
-tf_class {
-  is_instance: "<class \'tensorflow.python.data.ops.iterator_ops.Iterator\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "initializer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_classes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_types"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'iterator_resource\', \'initializer\', \'output_types\', \'output_shapes\', \'output_classes\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_string_handle"
-    argspec: "args=[\'string_handle\', \'output_types\', \'output_shapes\', \'output_classes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "from_structure"
-    argspec: "args=[\'output_types\', \'output_shapes\', \'shared_name\', \'output_classes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "get_next"
-    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "make_initializer"
-    argspec: "args=[\'self\', \'dataset\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "string_handle"
-    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt
index 509bbae833..4c3d6ddd85 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.pbtxt
@@ -8,10 +8,6 @@ tf_module {
     name: "FixedLengthRecordDataset"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Iterator"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Options"
     mtype: "<type \'type\'>"
-- 
GitLab


From a50880247b9faf245b3c5ea77729e3a4edcb0be1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 17:15:16 -0800
Subject: [PATCH 0904/1554] Make pfor compatible with Eager execution.

PiperOrigin-RevId: 223085811
---
 tensorflow/python/eager/backprop.py           |  34 +--
 .../ops/parallel_for/control_flow_ops.py      |  11 +
 .../ops/parallel_for/control_flow_ops_test.py | 227 +++++++++++-------
 3 files changed, 158 insertions(+), 114 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index beed1d5de3..69d444a90e 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -1015,12 +1015,8 @@ class GradientTape(object):
       target_size = array_ops.shape(target)[0]
 
     if experimental_use_pfor:
-      def f():
-        return pfor_ops.pfor(loop_fn, target_size)
-      if context.executing_eagerly():
-        f = function.defun(f)
       try:
-        output = f()
+        output = pfor_ops.pfor(loop_fn, target_size)
       except ValueError as err:
         six.reraise(
             ValueError,
@@ -1030,12 +1026,11 @@ class GradientTape(object):
                 " experimental_use_pfor to False."),
             sys.exc_info()[2])
     else:
-      if context.executing_eagerly():
-        if not self._persistent:
-          raise RuntimeError(
-              "GradientTape must be created with persistent=True"
-              " to compute the jacobian with eager execution enabled and with "
-              " experimental_use_pfor set to False.")
+      if context.executing_eagerly() and not self._persistent:
+        raise RuntimeError(
+            "GradientTape must be created with persistent=True"
+            " to compute the jacobian with eager execution enabled and with "
+            " experimental_use_pfor set to False.")
       output = pfor_ops.for_loop(
           loop_fn, [target.dtype] * len(flat_sources), target_size)
 
@@ -1131,12 +1126,8 @@ class GradientTape(object):
                            unconnected_gradients=unconnected_gradients)
 
     if experimental_use_pfor:
-      def f():
-        return pfor_ops.pfor(loop_fn, target_row_size)
-      if context.executing_eagerly():
-        f = function.defun(f)
       try:
-        output = f()
+        output = pfor_ops.pfor(loop_fn, target_row_size)
       except ValueError as err:
         six.reraise(
             ValueError,
@@ -1146,12 +1137,11 @@ class GradientTape(object):
                 "setting experimental_use_pfor to False."),
             sys.exc_info()[2])
     else:
-      if context.executing_eagerly():
-        if not self._persistent:
-          raise RuntimeError(
-              "GradientTape must be created with persistent=True"
-              " to compute the batch_jacobian with eager execution enabled and "
-              " with experimental_use_pfor set to False.")
+      if context.executing_eagerly() and not self._persistent:
+        raise RuntimeError(
+            "GradientTape must be created with persistent=True"
+            " to compute the batch_jacobian with eager execution enabled and "
+            " with experimental_use_pfor set to False.")
       output = pfor_ops.for_loop(loop_fn, target.dtype, target_row_size)
     if output is None:
       return None
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops.py b/tensorflow/python/ops/parallel_for/control_flow_ops.py
index 3c818f3d6c..8f652e9c50 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
+from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -142,6 +144,15 @@ def pfor(loop_fn, iters, parallel_iterations=None):
   Raises:
     ValueError: If parallel_iterations is not None and not an integer > 1.
   """
+  def f():
+    return _pfor_impl(loop_fn, iters, parallel_iterations=parallel_iterations)
+  if context.executing_eagerly():
+    f = function.defun(f)
+  return f()
+
+
+def _pfor_impl(loop_fn, iters, parallel_iterations=None):
+  """Implementation of pfor."""
   existing_ops = set(ops.get_default_graph().get_operations())
   with ops.name_scope("loop_body"):
     loop_var = array_ops.placeholder(dtypes.int32, shape=[])
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 017bc9dae5..c2484766aa 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -26,10 +26,12 @@ import numpy as np
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import backprop
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import bitwise_ops
 from tensorflow.python.ops import clip_ops
@@ -52,6 +54,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.util import nest
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class PForTest(test.TestCase):
 
   def _run_targets(self, targets1, targets2=None, run_init=True):
@@ -125,6 +128,7 @@ class PForTest(test.TestCase):
       pfor_control_flow_ops.pfor(lambda i: 1, 8, parallel_iterations=1)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class ArrayTest(PForTest):
 
   def test_gather(self):
@@ -316,14 +320,17 @@ class ArrayTest(PForTest):
 
   def test_unary_cwise_ops(self):
     for op in [array_ops.identity, array_ops.stop_gradient]:
-      x = random_ops.random_uniform([3, 5])
+      with backprop.GradientTape(persistent=True) as g:
+        x = random_ops.random_uniform([3, 5])
+        g.watch(x)
 
       # pylint: disable=cell-var-from-loop
       def loop_fn(i):
-        x1 = array_ops.gather(x, i)
-        y = op(x1) + x1
-        loss = nn.l2_loss(y)
-        return op(x), y, gradient_ops.gradients(loss, x1)
+        with g:
+          x1 = array_ops.gather(x, i)
+          y = op(x1) + x1
+          loss = nn.l2_loss(y)
+        return op(x), y, g.gradient(loss, x1)
 
       # pylint: enable=cell-var-from-loop
 
@@ -346,17 +353,21 @@ class ArrayTest(PForTest):
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
 
   def test_strided_slice(self):
-    x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2])
+    with backprop.GradientTape(persistent=True) as g:
+      x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2])
+      g.watch(x)
 
     def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1]
-      loss = nn.l2_loss(y)
-      return y, gradient_ops.gradients(loss, x_i)
+      with g:
+        x_i = array_ops.gather(x, i)
+        y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1]
+        loss = nn.l2_loss(y)
+      return y, g.gradient(loss, x_i)
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class BitwiseTest(PForTest):
 
   def test_unary_cwise(self):
@@ -396,6 +407,7 @@ class BitwiseTest(PForTest):
       self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MathTest(PForTest):
 
   def test_unary_cwise_ops(self):
@@ -452,22 +464,29 @@ class MathTest(PForTest):
         nn.softsign,
     ]
     for op in complex_ops + real_ops:
-      x = random_ops.random_uniform([3, 5])
-      if op in complex_ops:
-        y = random_ops.random_uniform([3, 5])
-        x = math_ops.complex(x, y)
+      with backprop.GradientTape(persistent=True) as g:
+        x = random_ops.random_uniform([3, 5])
+        g.watch(x)
+        if op in complex_ops:
+          y = random_ops.random_uniform([3, 5])
+          g.watch(y)
+          x = math_ops.complex(x, y)
 
       # pylint: disable=cell-var-from-loop
       output_dtypes = []
       def loop_fn(i):
-        x1 = array_ops.gather(x, i)
-        y1 = op(x1)
-        outputs = [op(x), y1]
-        if y1.dtype == dtypes.float32:
-          loss = math_ops.reduce_sum(y1 * y1)
-          grad = gradient_ops.gradients(loss, x1)
-          if grad and grad[0] is not None:
-            outputs.extend(grad)
+        with g:
+          x1 = array_ops.gather(x, i)
+          y1 = op(x1)
+          outputs = [op(x), y1]
+          if y1.dtype == dtypes.float32:
+            loss = math_ops.reduce_sum(y1 * y1)
+          else:
+            loss = None
+        if loss is not None:
+          grad = g.gradient(loss, x1)
+          if grad is not None:
+            outputs.append(grad)
         del output_dtypes[:]
         output_dtypes.extend([t.dtype for t in outputs])
         return outputs
@@ -684,17 +703,19 @@ class MathTest(PForTest):
     x_shape = [2, 3, 4, 5, 6]
     x = random_ops.random_uniform(x_shape)
     for data_format in ("NCHW", "NHWC"):
-      bias_dim = 2 if data_format == "NCHW" else -1
-      bias_shape = x_shape[bias_dim]
-      bias = random_ops.random_uniform([bias_shape])
+      with backprop.GradientTape(persistent=True) as g:
+        bias_dim = 2 if data_format == "NCHW" else -1
+        bias_shape = x_shape[bias_dim]
+        bias = random_ops.random_uniform([bias_shape])
+        g.watch(bias)
 
       # pylint: disable=cell-var-from-loop
       def loop_fn(i):
-        a = array_ops.gather(x, i)
-        y = nn.bias_add(a, bias, data_format=data_format)
-        loss = math_ops.reduce_sum(y * y)
-        return y, gradient_ops.gradients(loss, bias)
-
+        with g:
+          a = array_ops.gather(x, i)
+          y = nn.bias_add(a, bias, data_format=data_format)
+          loss = math_ops.reduce_sum(y * y)
+        return y, g.gradient(loss, bias)
       # pylint: enable=cell-var-from-loop
 
       self._test_loop_fn(
@@ -755,6 +776,7 @@ class MathTest(PForTest):
       self._test_loop_fn(loop_fn, 2)
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class NNTest(PForTest):
 
   def test_conv2d(self):
@@ -807,48 +829,60 @@ class NNTest(PForTest):
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
 
   def test_avg_pool(self):
-    x = random_ops.random_uniform([3, 2, 12, 12, 3])
-    ksize = [1, 3, 3, 1]
+    with backprop.GradientTape(persistent=True) as g:
+      x = random_ops.random_uniform([3, 2, 12, 12, 3])
+      g.watch(x)
+      ksize = [1, 3, 3, 1]
 
     def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      output = nn.avg_pool(
-          x1, ksize, strides=[1, 2, 2, 1], padding="VALID", data_format="NHWC")
-      loss = nn.l2_loss(output)
-      return output, gradient_ops.gradients(loss, x1)
+      with g:
+        x1 = array_ops.gather(x, i)
+        output = nn.avg_pool(
+            x1, ksize, strides=[1, 2, 2, 1], padding="VALID",
+            data_format="NHWC")
+        loss = nn.l2_loss(output)
+      return output, g.gradient(loss, x1)
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
 
   def test_max_pool(self):
-    x = random_ops.random_uniform([3, 2, 12, 12, 3])
-    ksize = [1, 3, 3, 1]
-    strides = [1, 2, 2, 1]
+    with backprop.GradientTape(persistent=True) as g:
+      x = random_ops.random_uniform([3, 2, 12, 12, 3])
+      g.watch(x)
+      ksize = [1, 3, 3, 1]
+      strides = [1, 2, 2, 1]
 
     def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      output = nn.max_pool(
-          x1, ksize, strides=strides, padding="VALID", data_format="NHWC")
-      loss = nn.l2_loss(output)
-      ones = array_ops.ones_like(output)
-      grad = gradient_ops.gradients(loss, x1, grad_ys=ones)
-      grad_grad = gradient_ops.gradients(grad, ones)
+      with g:
+        x1 = array_ops.gather(x, i)
+        output = nn.max_pool(
+            x1, ksize, strides=strides, padding="VALID", data_format="NHWC")
+        loss = nn.l2_loss(output)
+        ones = array_ops.ones_like(output)
+        g.watch(ones)
+        grad = g.gradient(loss, x1, output_gradients=ones)
+      grad_grad = g.gradient(grad, ones)
       return output, grad, grad_grad
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
 
   def test_max_pool3d(self):
-    x = random_ops.random_uniform([3, 3, 2, 12, 12, 3])
-    ksize = [1, 1, 3, 3, 1]
-    strides = [1, 1, 2, 2, 1]
+    with backprop.GradientTape(persistent=True) as g:
+      x = random_ops.random_uniform([3, 3, 2, 12, 12, 3])
+      g.watch(x)
+      ksize = [1, 1, 3, 3, 1]
+      strides = [1, 1, 2, 2, 1]
 
     def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      output = nn.max_pool3d(
-          x1, ksize, strides=strides, padding="VALID", data_format="NDHWC")
-      loss = nn.l2_loss(output)
-      ones = array_ops.ones_like(output)
-      grad = gradient_ops.gradients(loss, x1, grad_ys=ones)
-      grad_grad = gradient_ops.gradients(grad, ones)
+      with g:
+        x1 = array_ops.gather(x, i)
+        output = nn.max_pool3d(
+            x1, ksize, strides=strides, padding="VALID", data_format="NDHWC")
+        loss = nn.l2_loss(output)
+        ones = array_ops.ones_like(output)
+        g.watch(ones)
+        grad = g.gradient(loss, x1, output_gradients=ones)
+      grad_grad = g.gradient(grad, ones)
       return output, grad, grad_grad
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
@@ -859,36 +893,41 @@ class NNTest(PForTest):
       data_formats.append("NCHW")
     for is_training in (True, False):
       for data_format in data_formats:
-        if data_format == "NCHW":
-          x = random_ops.random_uniform([3, 1, 2, 5, 5])
-        else:
-          x = random_ops.random_uniform([3, 1, 5, 5, 2])
-        scale = random_ops.random_uniform([2])
-        offset = random_ops.random_uniform([2])
-        mean = None if is_training else random_ops.random_uniform([2])
-        variance = None if is_training else random_ops.random_uniform([2])
+        with backprop.GradientTape(persistent=True) as g:
+          if data_format == "NCHW":
+            x = random_ops.random_uniform([3, 1, 2, 5, 5])
+          else:
+            x = random_ops.random_uniform([3, 1, 5, 5, 2])
+          g.watch(x)
+          scale = random_ops.random_uniform([2])
+          g.watch(scale)
+          offset = random_ops.random_uniform([2])
+          g.watch(offset)
+          mean = None if is_training else random_ops.random_uniform([2])
+          variance = None if is_training else random_ops.random_uniform([2])
 
         # pylint: disable=cell-var-from-loop
         def loop_fn(i):
-          x1 = array_ops.gather(x, i)
-          outputs = nn.fused_batch_norm(
-              x1,
-              scale,
-              offset,
-              mean=mean,
-              variance=variance,
-              epsilon=0.01,
-              data_format=data_format,
-              is_training=is_training)
-          outputs = list(outputs)
-          # We only test the first value of outputs when is_training is False.
-          # It looks like CPU and GPU have different outputs for batch_mean and
-          # batch_variance for this case.
-          if not is_training:
-            outputs[1] = constant_op.constant(0.)
-            outputs[2] = constant_op.constant(0.)
-          loss = nn.l2_loss(outputs[0])
-          gradients = gradient_ops.gradients(loss, [x1, scale, offset])
+          with g:
+            x1 = array_ops.gather(x, i)
+            outputs = nn.fused_batch_norm(
+                x1,
+                scale,
+                offset,
+                mean=mean,
+                variance=variance,
+                epsilon=0.01,
+                data_format=data_format,
+                is_training=is_training)
+            outputs = list(outputs)
+            # We only test the first value of outputs when is_training is False.
+            # It looks like CPU and GPU have different outputs for batch_mean
+            # and batch_variance for this case.
+            if not is_training:
+              outputs[1] = constant_op.constant(0.)
+              outputs[2] = constant_op.constant(0.)
+            loss = nn.l2_loss(outputs[0])
+          gradients = g.gradient(loss, [x1, scale, offset])
           return outputs + gradients
 
         # pylint: enable=cell-var-from-loop
@@ -896,16 +935,20 @@ class NNTest(PForTest):
         self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 6)
 
   def test_softmax_cross_entropy_with_logits(self):
-    logits = random_ops.random_uniform([3, 2, 4])
-    labels = random_ops.random_uniform([3, 2, 4])
-    labels /= math_ops.reduce_sum(labels, axis=[2], keepdims=True)
+    with backprop.GradientTape(persistent=True) as g:
+      logits = random_ops.random_uniform([3, 2, 4])
+      g.watch(logits)
+      labels = random_ops.random_uniform([3, 2, 4])
+      labels /= math_ops.reduce_sum(labels, axis=[2], keepdims=True)
 
     def loop_fn(i):
-      logits_i = array_ops.gather(logits, i)
-      labels_i = array_ops.gather(labels, i)
-      loss = nn.softmax_cross_entropy_with_logits(
-          labels=labels_i, logits=logits_i)
-      return loss, gradient_ops.gradients(math_ops.reduce_sum(loss), logits_i)
+      with g:
+        logits_i = array_ops.gather(logits, i)
+        labels_i = array_ops.gather(labels, i)
+        loss = nn.softmax_cross_entropy_with_logits(
+            labels=labels_i, logits=logits_i)
+        total_loss = math_ops.reduce_sum(loss)
+      return loss, g.gradient(total_loss, logits_i)
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
 
-- 
GitLab


From 34b397c662adda53da5186b7a95267bed60b1b17 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 27 Nov 2018 17:26:55 -0800
Subject: [PATCH 0905/1554] Remove duplicated code from function optimizer and
 prepare for indirect function calls inlining (PartitionedCallOp).

PiperOrigin-RevId: 223087215
---
 tensorflow/core/grappler/op_types.cc          |   9 +-
 tensorflow/core/grappler/op_types.h           |   5 +-
 .../grappler/optimizers/function_optimizer.cc | 199 +++++++++---------
 3 files changed, 115 insertions(+), 98 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 06248393ba..38fc1fff32 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -551,14 +551,15 @@ bool MaybeHasRefInput(const NodeDef& node) {
   return false;
 }
 
-bool IsFreeOfSideEffect(const NodeDef& node) {
+bool IsFreeOfSideEffect(const NodeDef& node,
+                        const OpRegistryInterface* op_registry) {
   // Placeholders must be preserved to keep the graph feedable.
   if (IsPlaceholder(node)) {
     return false;
   }
   const OpDef* op_def = nullptr;
   const string& op_name = node.op();
-  Status status = OpRegistry::Global()->LookUpOpDef(op_name, &op_def);
+  Status status = op_registry->LookUpOpDef(op_name, &op_def);
   if (!status.ok()) {
     return false;
   }
@@ -582,6 +583,10 @@ bool IsFreeOfSideEffect(const NodeDef& node) {
   return !ModifiesInputsInPlace(node);
 }
 
+bool IsFreeOfSideEffect(const NodeDef& node) {
+  return IsFreeOfSideEffect(node, OpRegistry::Global());
+}
+
 bool ModifiesInputsInPlace(const NodeDef& node) {
   // Some nodes do in-place updates on regular tensor inputs.
   string op_name = node.op();
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index bd286f2c72..67897e8512 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OP_TYPES_H_
 
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
@@ -180,7 +181,9 @@ bool IsCommutative(const NodeDef& node);
 // value.
 bool IsPersistent(const NodeDef& node);
 
-bool IsFreeOfSideEffect(const NodeDef& node);
+bool IsFreeOfSideEffect(const NodeDef& node,
+                        const OpRegistryInterface* op_registry);
+bool IsFreeOfSideEffect(const NodeDef& node);  // use OpRegistry::Global()
 
 // Returns true if the takes a tensor reference as input, or if looking up its
 // OpDef failed.
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index f8ddbeb659..22013ea2db 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -220,13 +220,15 @@ class FunctionOptimizerContext {
                                     const GrapplerItem& item)
       : grappler_item_id_(item.id),
         graph_version_(item.graph.versions().producer()),
+        opt_level_(opt_level),
         function_library_(OpRegistry::Global(), item.graph.library()),
         graph_view_(&item.graph) {
     InitializeTrulyConstNodes(item);
-    InitializeInlinedFunctions(opt_level, item);
     InitializeFetchNodes(item);
   }
 
+  const RewriterConfig::Toggle opt_level() const { return opt_level_; }
+
   const FunctionLibraryDefinition& function_library() const {
     return function_library_;
   }
@@ -255,10 +257,6 @@ class FunctionOptimizerContext {
     return fetch_nodes_.find(node_name) != fetch_nodes_.end();
   }
 
-  bool IsInlinedFunction(const string& name) const {
-    return inlined_functions_.count(name) > 0;
-  }
-
   bool IsTrulyConst(const string& name) const {
     return TrulyConstNode(name) != nullptr;
   }
@@ -267,11 +265,6 @@ class FunctionOptimizerContext {
     return gtl::FindWithDefault(truly_const_nodes_, name, nullptr);
   }
 
-  // Find inlining candidate by name. Return nullptr if not found.
-  const FunctionDef* FindInlinedFunction(const string& name) const {
-    return gtl::FindWithDefault(inlined_functions_, name, nullptr);
-  }
-
   const FunctionSpecialization* FindFunctionSpecialization(
       const FunctionSpecializationSignature& sig) const {
     return gtl::FindOrNull(specialized_functions_, sig);
@@ -312,26 +305,6 @@ class FunctionOptimizerContext {
     }
   }
 
-  void InitializeInlinedFunctions(RewriterConfig::Toggle opt_level,
-                                  const GrapplerItem& item) {
-    bool aggressive = opt_level == RewriterConfig::AGGRESSIVE;
-
-    for (const FunctionDef& func : item.graph.library().function()) {
-      // Can't create IdentityN nodes with no input or output: skip these
-      // functions for now.
-      if (func.signature().input_arg_size() == 0 ||
-          func.signature().output_arg_size() == 0) {
-        continue;
-      }
-      bool marked_noinline = MarkedNoInline(func);
-      bool marked_specialized = MarkedSpecialized(func);
-
-      if (!marked_specialized && (!marked_noinline || aggressive)) {
-        inlined_functions_[func.signature().name()] = &func;
-      }
-    }
-  }
-
   void InitializeFetchNodes(const GrapplerItem& item) {
     for (const string& fetch : item.fetch) {
       fetch_tensors_.insert(fetch);
@@ -359,6 +332,7 @@ class FunctionOptimizerContext {
 
   const string grappler_item_id_;
   const int graph_version_;
+  const RewriterConfig::Toggle opt_level_;
   FunctionLibraryDefinition function_library_;
 
   // These fields initialized lazily only if needed.
@@ -366,8 +340,6 @@ class FunctionOptimizerContext {
   std::unique_ptr<ProcessFunctionLibraryRuntime> process_flr_;
   FunctionLibraryRuntime* flr_ = nullptr;
 
-  // Functions that can be inlined into optimized graph.
-  std::unordered_map<string, const FunctionDef*> inlined_functions_;
   // Nodes that are Const and not in feed.
   std::unordered_map<string, const NodeDef*> truly_const_nodes_;
   // Specialized functions.
@@ -390,6 +362,65 @@ class FunctionOptimizerContext {
   TF_DISALLOW_COPY_AND_ASSIGN(FunctionOptimizerContext);
 };
 
+// Returns a pointer to the called function definition iff the given node is
+// indeed a function call. Otherwise returns nullptr.
+const FunctionDef* FindFunctionCall(const FunctionOptimizerContext& ctx,
+                                    const NodeDef& node) {
+  // Check if a node does indirect function call via PartitionedCallOp.
+  if (IsPartitionedCall(node) || IsStatefulPartitionedCall(node)) {
+    const AttrValue* func_attr = AttrSlice(node).Find("f");
+    return (func_attr != nullptr && func_attr->has_func())
+               ? ctx.function_library().Find(func_attr->func().name())
+               : nullptr;
+  }
+
+  // Check if the function op itself is a function name.
+  return ctx.function_library().Find(node.op());
+}
+
+// Returns true iff `node` is a direct function call of `func`, and we know how
+// to inline it into the main graph.
+bool IsInlinableDirectFunctionCall(const FunctionOptimizerContext& ctx,
+                                   const FunctionDef& func,
+                                   const NodeDef& node) {
+  // Indirect function calls (PartitionedCallOp) have automatic control
+  // dependencies and inlined separately from direct function calls.
+  bool is_direct_function_call = IsDirectFunctionCall(func, node);
+
+  // For direct function  calls we insert IdentityN nodes before/after inlined
+  // function body to preserve function call semantics (all inputs evaluated
+  // before function evaluation starts, and all function body nodes finished
+  // before output consumed by other nodes).
+  bool has_inputs = func.signature().input_arg_size() > 0;
+  // TODO(ezhulenev): Relax constraint on output args?
+  bool has_outputs = func.signature().output_arg_size() > 0;
+
+  // Function must execute all the nodes in a function body that might have side
+  // effects. After inlining these nodes into the main graph, we can no longer
+  // guarantee that. For now we disable inlining functions with side effects.
+  //
+  // Attaching control dependency to the output IdentityN node is not safe,
+  // because it might be split or pruned in a later optimization pass.
+  //
+  // Indirect function calls (via PartitionedCallOp) have automatic dependency
+  // tracking, and allow us to safely inline functions with side effects.
+  bool free_of_side_effects =
+      std::all_of(func.node_def().begin(), func.node_def().end(),
+                  [&ctx](const NodeDef& node) {
+                    return IsFreeOfSideEffect(node, &ctx.function_library());
+                  });
+
+  bool marked_noinline = MarkedNoInline(func);
+  bool marked_specialized = MarkedSpecialized(func);
+
+  // We ignore `_noinline` marker in aggressive mode.
+  bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
+
+  return is_direct_function_call && has_inputs && has_outputs &&
+         free_of_side_effects && !marked_specialized &&
+         (!marked_noinline || aggressive);
+}
+
 gtl::FlatSet<int> GetActiveOutputs(const NodeDef& node,
                                    const FunctionOptimizerContext& ctx,
                                    int size_hint = 0) {
@@ -608,6 +639,9 @@ Status UpdateSpecializedFunctionNode(
   // 2. Remove inputs corresponding to the pushed down consts.
   RemovePushedDownConstInputs(specialization, specialized_func_node);
 
+  // NOTE: PartitionedCallOp has `Tin` and `Tout` attributes for input/output
+  // types, that must be in sync with updated function signature.
+
   // 3. Update input types for the indirect function calls.
   if (is_indirect_call) {
     RemovePushedDownConstInputTypes(specialization, func_node,
@@ -805,13 +839,16 @@ NodeDef InlinedFunctionOutputsNode(const NodeDef& func_node,
   return outputs;
 }
 
-Status InlineFunction(const NodeDef& func_node, const FunctionDef& func,
-                      const FunctionOptimizerContext& ctx,
-                      const int graph_def_version, GraphDef* optimized_graph) {
-  VLOG(2) << "Inline function instantiation: " << SummarizeNodeDef(func_node);
+Status InlineDirectFunctionCall(const NodeDef& func_node,
+                                const FunctionDef& func,
+                                const FunctionOptimizerContext& ctx,
+                                const int graph_def_version,
+                                GraphDef* optimized_graph) {
+  VLOG(2) << "Inline direct function call: " << SummarizeNodeDef(func_node);
 
-  // Specialized function call kernels might have behavior that is not
-  // representable in a graph (e.g. runtime ops device placing).
+  // Indirect function calls (via PartitionedCallOp) have automatic control
+  // dependencies, and doesn't need IdentityN nodes before/after inlined
+  // function body, and we inline them separately.
   if (!IsDirectFunctionCall(func, func_node)) {
     return errors::InvalidArgument("Can't inline indirect function call");
   }
@@ -877,14 +914,16 @@ Status InlineFunction(const NodeDef& func_node, const FunctionDef& func,
     // Make sure the node is placed.
     func_body_node.set_device(func_node.device());
 
-    // Check if a body node is itself a function.
+    // Check if a body node is itself a function call and can be inlined.
     const FunctionDef* func_body_node_func =
-        ctx.FindInlinedFunction(func_body_node.op());
-    if (func_body_node_func != nullptr) {
+        FindFunctionCall(ctx, func_body_node);
+    if (func_body_node_func != nullptr &&
+        IsInlinableDirectFunctionCall(ctx, *func_body_node_func,
+                                      func_body_node)) {
       // Recursively inline function calls.
-      TF_RETURN_IF_ERROR(InlineFunction(func_body_node, *func_body_node_func,
-                                        ctx, graph_def_version,
-                                        optimized_graph));
+      TF_RETURN_IF_ERROR(
+          InlineDirectFunctionCall(func_body_node, *func_body_node_func, ctx,
+                                   graph_def_version, optimized_graph));
     } else {
       // Annotate the node with the function attributes.
       for (const auto& attr : func.attr()) {
@@ -1015,8 +1054,6 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   bool specialize_func = options_.enable_function_specialization;
 
   for (const NodeDef& node : item.graph.node()) {
-    const string op_name = node.op();
-
     // Each node optimization can modify optimized graph only by adding new
     // nodes, we can check node size to make sure that graph was not modified.
     const int num_nodes_before = optimized_graph->node_size();
@@ -1045,11 +1082,13 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // 1. Inline symbolic gradients into the optimized graph.                 //
     // ---------------------------------------------------------------------- //
 
-    if (op_name == "SymbolicGradient" && inline_gradients) {
-      // Inline symbolic gradients only if the corresponding function is inlined
+    if (IsSymbolicGradient(node) && inline_gradients) {
+      // Inline symbolic gradients only if the corresponding function is not
+      // marked as `_noinline`.
       const auto* f_attr = gtl::FindOrNull(node.attr(), "f");
-      string f_name = f_attr != nullptr ? f_attr->func().name() : "";
-      if (ctx.IsInlinedFunction(f_name)) {
+      const string f_name = f_attr != nullptr ? f_attr->func().name() : "";
+      const FunctionDef* func = ctx.function_library().Find(f_name);
+      if (func && !MarkedNoInline(*func)) {
         TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(
             InlineSymbolicGradient(node, &ctx, optimized_graph));
         continue;
@@ -1057,28 +1096,33 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     }
 
     // ---------------------------------------------------------------------- //
-    // 2. Inline or specialize direct function calls.                         //
+    // 2. Inline or specialize function calls.                                //
     // ---------------------------------------------------------------------- //
 
-    const FunctionDef* func = ctx.function_library().Find(op_name);
+    // Find if a node is a function call (direct or indirect).
+    const FunctionDef* func = FindFunctionCall(ctx, node);
+
     if (func != nullptr) {
-      // 2a. Inline it if it's allowed to do so.
-      if (inline_func && ctx.IsInlinedFunction(op_name)) {
+      const string& func_name = func->signature().name();
+
+      // 2a. Inline direct function call if it's inlinable.
+      if (inline_func && IsInlinableDirectFunctionCall(ctx, *func, node)) {
         // Inline function body into the optimized graph}
-        TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(
-            InlineFunction(node, *func, ctx, item.graph.versions().producer(),
-                           optimized_graph));
+        TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(InlineDirectFunctionCall(
+            node, *func, ctx, item.graph.versions().producer(),
+            optimized_graph));
         continue;
       }
 
-      // Do not specialize if function has custom gradient.
-      const string grad_func = ctx.function_library().FindGradient(op_name);
-
       // 2b. Specialize it to it's instantiation context if can't be inlined,
       // and it has something worth specializing.
       bool specialization_worthy = IsParametrized(*func) ||
                                    HasTrulyConstInputs(node, ctx) ||
                                    HasUnusedOutputs(node, *func, ctx);
+
+      // Do not specialize if function has custom gradient.
+      const string grad_func = ctx.function_library().FindGradient(func_name);
+
       if (specialize_func && grad_func.empty() && specialization_worthy) {
         // TODO(ezhulenev): Specialize function call if input has a known shape.
         // Specialize function body for its instantiation attributes and inputs.
@@ -1089,41 +1133,6 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       }
     }
 
-    // ---------------------------------------------------------------------- //
-    // 3. Specialize indirect function calls through the PartitionedCallOp.   //
-    // ---------------------------------------------------------------------- //
-
-    bool is_partitioned_call =
-        IsPartitionedCall(node) || IsStatefulPartitionedCall(node);
-
-    // We can only specialize PartitionedCall ops. Inlining is not supported.
-    if (is_partitioned_call && specialize_func) {
-      const AttrValue* func_attr = AttrSlice(node).Find("f");
-      string indirect_func_name =
-          (func_attr != nullptr && func_attr->has_func())
-              ? func_attr->func().name()
-              : "";
-      const FunctionDef* indirect_func =
-          ctx.function_library().Find(indirect_func_name);
-
-      if (indirect_func != nullptr) {
-        // Do not specialize if function has custom gradient.
-        const string grad_func =
-            ctx.function_library().FindGradient(indirect_func_name);
-
-        // Specialize it to it's instantiation context.
-        bool specialization_worthy =
-            IsParametrized(*indirect_func) || HasTrulyConstInputs(node, ctx) ||
-            HasUnusedOutputs(node, *indirect_func, ctx);
-        if (grad_func.empty() && specialization_worthy) {
-          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(SpecializeFunction(
-              node, *indirect_func, item.graph.versions().producer(), &ctx,
-              optimized_graph));
-          continue;
-        }
-      }
-    }
-
     // ---------------------------------------------------------------------- //
     // If we reached this point, node was not handled by any of the stages
     // (inline, specialize), simply add a copy to the graph.
-- 
GitLab


From caad5b172e4c9900ea302b6680cbf0d9caba48a7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 17:58:30 -0800
Subject: [PATCH 0906/1554] Add a warning against calling _tf_sess() in user
 code.

PiperOrigin-RevId: 223090908
---
 tensorflow/python/training/monitored_session.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 162fef971d..ccd9014648 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -840,6 +840,14 @@ class _MonitoredSession(object):
     return self._coordinated_creator.tf_sess is None
 
   def _tf_sess(self):
+    """Return underlying tf.Session object.
+
+    Warning: accessing the returned object in user code is likely to cause races
+    or "flaky tests".
+
+    Returns:
+      A tf.Session object.
+    """
     return self._coordinated_creator.tf_sess
 
 
-- 
GitLab


From fbd5472f477c277aa1e4044d4f7e29b4ae958542 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 27 Nov 2018 18:02:07 -0800
Subject: [PATCH 0907/1554] Remove metric value and update op return type
 check. Fixes tensorflow#23731

PiperOrigin-RevId: 223091282
---
 tensorflow/python/keras/metrics.py      | 17 ++--------
 tensorflow/python/keras/metrics_test.py | 42 -------------------------
 2 files changed, 2 insertions(+), 57 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 90babf38fd..b74b6ccf5e 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -65,13 +65,6 @@ from tensorflow.python.util.tf_export import tf_export
 from tensorflow.tools.docs import doc_controls
 
 
-def check_is_tensor_or_operation(x, name):
-  """Raises type error if the given input is not a tensor or operation."""
-  if not (isinstance(x, ops.Tensor) or isinstance(x, ops.Operation)):
-    raise TypeError('{0} must be a Tensor or Operation, given: {1}'.format(
-        name, x))
-
-
 def clone_metric(metric):
   """Returns a clone of the metric if stateful, otherwise returns it as is."""
   if isinstance(metric, Metric):
@@ -104,8 +97,6 @@ def update_state_wrapper(update_state_fn):
     update_op = update_state_fn(*args, **kwargs)
     if update_op is not None:  # update_op will be None in eager execution.
       metric_obj.add_update(update_op, inputs=True)
-      check_is_tensor_or_operation(
-          update_op, 'Metric {0}\'s update'.format(metric_obj.name))
     return update_op
 
   return tf_decorator.make_decorator(update_state_fn, decorated)
@@ -130,7 +121,7 @@ def result_wrapper(result_fn):
     `merge_call()`.
   """
 
-  def decorated(metric_obj, *args):
+  def decorated(_, *args):
     """Decorated function with merge_call."""
     replica_context = distribution_strategy_context.get_replica_context()
     if replica_context is None:  # if in cross replica context already
@@ -151,8 +142,6 @@ def result_wrapper(result_fn):
       # replica mode and compute a value in cross replica mode.
       result_t = replica_context.merge_call(
           merge_fn_wrapper, args=(result_fn,) + args)
-    check_is_tensor_or_operation(result_t,
-                                 'Metric {0}\'s result'.format(metric_obj.name))
     return result_t
 
   return tf_decorator.make_decorator(result_fn, decorated)
@@ -607,9 +596,7 @@ class Mean(Metric):
     # updated.
     update_total_op = state_ops.assign_add(self.total, values)
     with ops.control_dependencies([update_total_op]):
-      update_count_op = state_ops.assign_add(self.count, num_values)
-      with ops.control_dependencies([update_count_op]):
-        return control_flow_ops.no_op()
+      return state_ops.assign_add(self.count, num_values)
 
   def result(self):
     return math_ops.div_no_nan(self.total, self.count)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 536f717517..40611a5d75 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -366,48 +366,6 @@ class KerasMetricsTest(test.TestCase):
     result = self.evaluate(result_t)
     self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7
 
-  @test_util.run_in_graph_and_eager_modes
-  def test_invalid_result(self):
-
-    class InvalidResult(metrics.Metric):
-
-      def __init__(self, name='invalid-result', dtype=dtypes.float64):
-        super(InvalidResult, self).__init__(name=name, dtype=dtype)
-
-      def update_state(self, *args, **kwargs):
-        pass
-
-      def result(self):
-        return 1
-
-    invalid_result_obj = InvalidResult()
-    with self.assertRaisesRegexp(
-        TypeError,
-        'Metric invalid-result\'s result must be a Tensor or Operation, given:'
-    ):
-      invalid_result_obj.result()
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_invalid_update(self):
-
-    class InvalidUpdate(metrics.Metric):
-
-      def __init__(self, name='invalid-update', dtype=dtypes.float64):
-        super(InvalidUpdate, self).__init__(name=name, dtype=dtype)
-
-      def update_state(self, *args, **kwargs):
-        return [1]
-
-      def result(self):
-        pass
-
-    invalid_update_obj = InvalidUpdate()
-    with self.assertRaisesRegexp(
-        TypeError,
-        'Metric invalid-update\'s update must be a Tensor or Operation, given:'
-    ):
-      invalid_update_obj.update_state()
-
 
 @test_util.run_all_in_graph_and_eager_modes
 class FalsePositivesTest(test.TestCase):
-- 
GitLab


From 3831f175ce88b90a1eb7caf3a1f72b9c8bab7c76 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 18:47:35 -0800
Subject: [PATCH 0908/1554] Bug fix for tf.ragged.tile RELNOTES: N/A

PiperOrigin-RevId: 223096139
---
 tensorflow/python/ops/ragged/ragged_array_ops.py    | 13 ++++++++++++-
 tensorflow/python/ops/ragged/ragged_tile_op_test.py |  9 +++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index 425f3957c3..815f48a4b1 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -1172,6 +1172,17 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None):
   ragged_rank = rt_input.ragged_rank
   nested_splits = rt_input.nested_row_splits
 
+  # projected_splits[src_axis, dst_axis] contains the split points that divide
+  # the rows from src_axis in the list of dst_axis values.  E.g.,
+  # projected_splits[i, i] = nested_splits[i], and
+  # projected_splits[i, i+1] = gather(nested_splits[i+1], nested_splits[i]).
+  projected_splits = [{i: nested_splits[i]} for i in range(ragged_rank)]
+  for src_axis in range(ragged_rank):
+    for dst_axis in range(src_axis + 1, ragged_rank - 1):
+      projected_splits[src_axis][dst_axis] = array_ops.gather(
+          nested_splits[dst_axis],
+          projected_splits[src_axis][dst_axis - 1])
+
   # For each ragged dimension: nested_splits[axis] -> result_splits[axis].
   result_splits = []
   for axis in range(ragged_rank):
@@ -1188,7 +1199,7 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None):
     repeats = 1
     for d in range(axis - 1, -1, -1):
       if const_multiples is None or const_multiples[d + 1] != 1:
-        splits = nested_splits[d] * repeats
+        splits = projected_splits[d][axis - 1] * repeats
         output_lengths = _repeat_ranges(output_lengths, splits,
                                         multiples[d + 1])
       repeats *= multiples[d + 1]
diff --git a/tensorflow/python/ops/ragged/ragged_tile_op_test.py b/tensorflow/python/ops/ragged/ragged_tile_op_test.py
index bf62d96e7a..672d212114 100644
--- a/tensorflow/python/ops/ragged/ragged_tile_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tile_op_test.py
@@ -170,6 +170,15 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           rt_input=[[[[1], [2]], [[3]]], [[]], [[[4, 5]]]],
           multiples=[1, 1, 1, 0],
           expected=[[[[], []], [[]]], [[]], [[[]]]]),
+      #=========================================================================
+      # multiple=1
+      #=========================================================================
+      dict(
+          descr='rank=4, multiples=1 (no repeats)',
+          rt_input=[[[[1], [2]], [[3], [4]]], [[[5], [6]]]],
+          multiples=[1, 1, 1, 1],
+          expected=[[[[1], [2]], [[3], [4]]],
+                    [[[5], [6]]]]),
 
   ])  # pyformat: disable
   def testRaggedTile(self,
-- 
GitLab


From 7d5ef9170bfe69c31cd57b18c6491fdd16ee93d3 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 27 Nov 2018 19:43:49 -0800
Subject: [PATCH 0909/1554] For cuDNN >= 7.4, undo the work-around for NHWC
 with >2 strides for conv backward data.

In cuDNN 7.4 release notes, they fixed the performance issue that we
work-arounded, and it improves resnet50 a lot.

PiperOrigin-RevId: 223100593
---
 .../xla/service/gpu/gpu_layout_assignment.cc  | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc
index 1c0a23fa3e..f59da2caa1 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.cc
@@ -65,8 +65,8 @@ HeuristicLayoutAssignment(const HloInstruction* instr,
 
   VLOG(2) << "Using heuristic to figure out layouts for " << instr->ToString();
 
-  // Empirically we've found with Volta and cudnn 7 that backward-input convs
-  // with stride are significantly faster with NCHW layouts.
+  // Empirically we've found with Volta and cudnn <= 7.3 that backward-input
+  // convs with stride are significantly faster with NCHW layouts.
   //
   // We could have used a mixed layout combination, e.g. (NHWC, NCHW, NCHW),
   // which on paper gives good performance. However, there are two observations:
@@ -75,11 +75,17 @@ HeuristicLayoutAssignment(const HloInstruction* instr,
   // * we've also observed that for mixed layouts, cuDNN transposes data back
   //   and forth from a different layout combination. If we end up with
   //   transposes anyway, we prefer to have them in XLA, as they can be fused.
-  // TODO(timshen): Figure out the exact condition. This may be achieved by
-  // auto-tuning layouts offline.
-  if (instr->custom_call_target() == kCudnnConvBackwardInputCallTarget &&
-      window_util::HasStride(instr->window())) {
-    return kAllNCHW;
+  if (auto* dnn = stream_executor->AsDnn()) {
+    auto version_status = dnn->GetVersion();
+    if (version_status.ok()) {
+      auto version = version_status.ConsumeValueOrDie();
+      if (std::make_tuple(version.major_version(), version.minor_version()) <=
+              std::make_tuple(7, 3) &&
+          instr->custom_call_target() == kCudnnConvBackwardInputCallTarget &&
+          window_util::HasStride(instr->window())) {
+        return kAllNCHW;
+      }
+    }
   }
 
   // For other Volta f16 convolutions, use NHWC.
-- 
GitLab


From c68b4003f88450455ab2fdfc62bde9e97929b085 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Tue, 27 Nov 2018 20:47:45 -0800
Subject: [PATCH 0910/1554] [Function Library]: Remove unused test variables

This is a cleanup following up the DeviceMgr refactor.

PiperOrigin-RevId: 223105410
---
 tensorflow/core/common_runtime/function_threadpool_test.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/core/common_runtime/function_threadpool_test.cc b/tensorflow/core/common_runtime/function_threadpool_test.cc
index bdbe24a70d..1b803736fb 100644
--- a/tensorflow/core/common_runtime/function_threadpool_test.cc
+++ b/tensorflow/core/common_runtime/function_threadpool_test.cc
@@ -67,9 +67,6 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
         device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def_.get(),
         opts, default_thread_pool, nullptr /* cluster_flr */));
     flr0_ = pflr_->GetFLR("/job:localhost/replica:0/task:0/cpu:0");
-    flr1_ = pflr_->GetFLR("/job:localhost/replica:0/task:0/cpu:1");
-    flr2_ = pflr_->GetFLR("/job:localhost/replica:0/task:0/cpu:2");
-    fdef_lib_ = lib_def_->ToProto();
   }
 
   Status Run(FunctionLibraryRuntime* flr, FunctionLibraryRuntime::Handle handle,
@@ -193,12 +190,9 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
   }
 
   FunctionLibraryRuntime* flr0_;
-  FunctionLibraryRuntime* flr1_;
-  FunctionLibraryRuntime* flr2_;
   std::unique_ptr<DeviceMgr> device_mgr_;
   std::unique_ptr<FunctionLibraryDefinition> lib_def_;
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
-  FunctionDefLibrary fdef_lib_;
 };
 
 TEST_F(FunctionLibraryRuntimeTest, DefaultThreadpool) {
-- 
GitLab


From 5f111d5a6c78f7afcd3ccbabf6e272800caceb9d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 27 Nov 2018 20:49:05 -0800
Subject: [PATCH 0911/1554] Remove calls to test_session() or use test_util
 helpers

PiperOrigin-RevId: 223105516
---
 .../keras/layers/cudnn_recurrent_test.py      | 686 +++++++++---------
 .../kernel_tests/cwise_ops_binary_test.py     |  28 +-
 .../python/kernel_tests/cwise_ops_test.py     |  66 +-
 .../kernel_tests/cwise_ops_unary_test.py      |  18 +-
 .../random/multinomial_op_test.py             |  35 +-
 .../random/stateless_random_ops_test.py       |   3 +-
 .../resource_variable_ops_test.py             |   2 +-
 .../python/kernel_tests/variable_ops_test.py  |  17 +-
 .../ops/ragged/ragged_segment_op_test.py      |  19 +-
 .../ragged_tensor_bounding_shape_op_test.py   |  40 +-
 .../python/ops/ragged/ragged_tensor_test.py   | 449 ++++++------
 11 files changed, 674 insertions(+), 689 deletions(-)

diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index cc93364aae..1f195f3119 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -31,64 +31,76 @@ from tensorflow.python.platform import test
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class CuDNNTest(test.TestCase, parameterized.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes
   def test_cudnn_rnn_basics(self):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        timesteps = 6
-        units = 2
-        num_samples = 32
-        for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
-          for return_sequences in [True, False]:
-            with keras.utils.CustomObjectScope(
-                {'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
-                 'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM}):
-              testing_utils.layer_test(
-                  layer_class,
-                  kwargs={'units': units,
-                          'return_sequences': return_sequences},
-                  input_shape=(num_samples, timesteps, input_size))
-          for go_backwards in [True, False]:
-            with keras.utils.CustomObjectScope(
-                {'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
-                 'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM}):
-              testing_utils.layer_test(
-                  layer_class,
-                  kwargs={'units': units,
-                          'go_backwards': go_backwards},
-                  input_shape=(num_samples, timesteps, input_size))
-
-  @test_util.run_in_graph_and_eager_modes
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      timesteps = 6
+      units = 2
+      num_samples = 32
+      for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
+        for return_sequences in [True, False]:
+          with keras.utils.CustomObjectScope({
+              'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
+              'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM
+          }):
+            testing_utils.layer_test(
+                layer_class,
+                kwargs={
+                    'units': units,
+                    'return_sequences': return_sequences
+                },
+                input_shape=(num_samples, timesteps, input_size))
+        for go_backwards in [True, False]:
+          with keras.utils.CustomObjectScope({
+              'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
+              'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM
+          }):
+            testing_utils.layer_test(
+                layer_class,
+                kwargs={
+                    'units': units,
+                    'go_backwards': go_backwards
+                },
+                input_shape=(num_samples, timesteps, input_size))
+
   def test_trainability(self):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        units = 2
-        for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
-          layer = layer_class(units)
-          layer.build((None, None, input_size))
-          self.assertEqual(len(layer.weights), 3)
-          self.assertEqual(len(layer.trainable_weights), 3)
-          self.assertEqual(len(layer.non_trainable_weights), 0)
-          layer.trainable = False
-          self.assertEqual(len(layer.weights), 3)
-          self.assertEqual(len(layer.non_trainable_weights), 3)
-          self.assertEqual(len(layer.trainable_weights), 0)
-          layer.trainable = True
-          self.assertEqual(len(layer.weights), 3)
-          self.assertEqual(len(layer.trainable_weights), 3)
-          self.assertEqual(len(layer.non_trainable_weights), 0)
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      units = 2
+      for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
+        layer = layer_class(units)
+        layer.build((None, None, input_size))
+        self.assertEqual(len(layer.weights), 3)
+        self.assertEqual(len(layer.trainable_weights), 3)
+        self.assertEqual(len(layer.non_trainable_weights), 0)
+        layer.trainable = False
+        self.assertEqual(len(layer.weights), 3)
+        self.assertEqual(len(layer.non_trainable_weights), 3)
+        self.assertEqual(len(layer.trainable_weights), 0)
+        layer.trainable = True
+        self.assertEqual(len(layer.weights), 3)
+        self.assertEqual(len(layer.trainable_weights), 3)
+        self.assertEqual(len(layer.non_trainable_weights), 0)
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_regularizer(self, layer_class):
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
     if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
+      with test_util.use_gpu():
         input_size = 10
         timesteps = 6
         units = 2
@@ -119,132 +131,140 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_return_state(self, layer_class):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        timesteps = 6
-        units = 2
-        num_samples = 32
-        num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
-
-        inputs = keras.Input(batch_shape=(num_samples, timesteps, input_size))
-        layer = layer_class(units, return_state=True, stateful=True)
-        outputs = layer(inputs)
-        _, state = outputs[0], outputs[1:]
-        self.assertEqual(len(state), num_states)
-        model = keras.models.Model(inputs, state[0])
-
-        inputs = np.random.random((num_samples, timesteps, input_size))
-        state = model.predict(inputs)
-        np.testing.assert_allclose(
-            keras.backend.eval(layer.states[0]), state, atol=1e-4)
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      timesteps = 6
+      units = 2
+      num_samples = 32
+      num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
+
+      inputs = keras.Input(batch_shape=(num_samples, timesteps, input_size))
+      layer = layer_class(units, return_state=True, stateful=True)
+      outputs = layer(inputs)
+      _, state = outputs[0], outputs[1:]
+      self.assertEqual(len(state), num_states)
+      model = keras.models.Model(inputs, state[0])
+
+      inputs = np.random.random((num_samples, timesteps, input_size))
+      state = model.predict(inputs)
+      np.testing.assert_allclose(
+          keras.backend.eval(layer.states[0]), state, atol=1e-4)
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_time_major_input(self, layer_class):
-    if test.is_gpu_available(cuda_only=True):
-      with self.test_session(use_gpu=True):
-        input_size = 10
-        timesteps = 6
-        units = 2
-        num_samples = 32
-
-        model = keras.models.Sequential()
-        model.add(
-            keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
-        layer = layer_class(units, time_major=True, return_sequences=True)
-        model.add(layer)
-        model.add(
-            keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
-        model.compile(loss='categorical_crossentropy', optimizer='adam')
-        model.fit(
-            np.ones((num_samples, timesteps, input_size)),
-            np.ones((num_samples, timesteps, units)))
-        out = model.predict(np.ones((num_samples, timesteps, input_size)))
-        self.assertEqual(out.shape, (num_samples, timesteps, units))
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      timesteps = 6
+      units = 2
+      num_samples = 32
+
+      model = keras.models.Sequential()
+      model.add(
+          keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
+      layer = layer_class(units, time_major=True, return_sequences=True)
+      model.add(layer)
+      model.add(
+          keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
+      model.compile(loss='categorical_crossentropy', optimizer='adam')
+      model.fit(
+          np.ones((num_samples, timesteps, input_size)),
+          np.ones((num_samples, timesteps, units)))
+      out = model.predict(np.ones((num_samples, timesteps, input_size)))
+      self.assertEqual(out.shape, (num_samples, timesteps, units))
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_specify_initial_state_keras_tensor(self, layer_class):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        timesteps = 6
-        units = 2
-        num_samples = 32
-        num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
-
-        inputs = keras.Input((timesteps, input_size))
-        initial_state = [keras.Input((units,)) for _ in range(num_states)]
-        layer = layer_class(units)
-        if len(initial_state) == 1:
-          output = layer(inputs, initial_state=initial_state[0])
-        else:
-          output = layer(inputs, initial_state=initial_state)
-        self.assertIn(initial_state[0], layer._inbound_nodes[0].input_tensors)
-
-        model = keras.models.Model([inputs] + initial_state, output)
-        model.compile(loss='categorical_crossentropy', optimizer='adam')
-
-        inputs = np.random.random((num_samples, timesteps, input_size))
-        initial_state = [
-            np.random.random((num_samples, units)) for _ in range(num_states)
-        ]
-        targets = np.random.random((num_samples, units))
-        model.fit([inputs] + initial_state, targets)
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      timesteps = 6
+      units = 2
+      num_samples = 32
+      num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
+
+      inputs = keras.Input((timesteps, input_size))
+      initial_state = [keras.Input((units,)) for _ in range(num_states)]
+      layer = layer_class(units)
+      if len(initial_state) == 1:
+        output = layer(inputs, initial_state=initial_state[0])
+      else:
+        output = layer(inputs, initial_state=initial_state)
+      self.assertIn(initial_state[0], layer._inbound_nodes[0].input_tensors)
+
+      model = keras.models.Model([inputs] + initial_state, output)
+      model.compile(loss='categorical_crossentropy', optimizer='adam')
+
+      inputs = np.random.random((num_samples, timesteps, input_size))
+      initial_state = [
+          np.random.random((num_samples, units)) for _ in range(num_states)
+      ]
+      targets = np.random.random((num_samples, units))
+      model.fit([inputs] + initial_state, targets)
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_statefulness(self, layer_class):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        timesteps = 6
-        units = 2
-        num_samples = 32
-
-        model = keras.models.Sequential()
-        model.add(
-            keras.layers.Embedding(
-                10,
-                input_size,
-                input_length=timesteps,
-                batch_input_shape=(num_samples, timesteps)))
-        layer = layer_class(
-            units, return_sequences=False, stateful=True, weights=None)
-        model.add(layer)
-        model.compile(optimizer='sgd', loss='mse')
-        out1 = model.predict(np.ones((num_samples, timesteps)))
-        self.assertEqual(out1.shape, (num_samples, units))
-
-        # train once so that the states change
-        model.train_on_batch(
-            np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
-        out2 = model.predict(np.ones((num_samples, timesteps)))
-
-        # if the state is not reset, output should be different
-        self.assertNotEqual(out1.max(), out2.max())
-
-        # check that output changes after states are reset
-        # (even though the model itself didn't change)
-        layer.reset_states()
-        out3 = model.predict(np.ones((num_samples, timesteps)))
-        self.assertNotEqual(out2.max(), out3.max())
-
-        # check that container-level reset_states() works
-        model.reset_states()
-        out4 = model.predict(np.ones((num_samples, timesteps)))
-        self.assertAllClose(out3, out4, atol=1e-5)
-
-        # check that the call to `predict` updated the states
-        out5 = model.predict(np.ones((num_samples, timesteps)))
-        self.assertNotEqual(out4.max(), out5.max())
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      timesteps = 6
+      units = 2
+      num_samples = 32
+
+      model = keras.models.Sequential()
+      model.add(
+          keras.layers.Embedding(
+              10,
+              input_size,
+              input_length=timesteps,
+              batch_input_shape=(num_samples, timesteps)))
+      layer = layer_class(
+          units, return_sequences=False, stateful=True, weights=None)
+      model.add(layer)
+      model.compile(optimizer='sgd', loss='mse')
+      out1 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertEqual(out1.shape, (num_samples, units))
+
+      # train once so that the states change
+      model.train_on_batch(
+          np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
+      out2 = model.predict(np.ones((num_samples, timesteps)))
+
+      # if the state is not reset, output should be different
+      self.assertNotEqual(out1.max(), out2.max())
+
+      # check that output changes after states are reset
+      # (even though the model itself didn't change)
+      layer.reset_states()
+      out3 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertNotEqual(out2.max(), out3.max())
+
+      # check that container-level reset_states() works
+      model.reset_states()
+      out4 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertAllClose(out3, out4, atol=1e-5)
+
+      # check that the call to `predict` updated the states
+      out5 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertNotEqual(out4.max(), out5.max())
 
   @parameterized.named_parameters(
       *test_util.generate_combinations_with_testcase_name(
@@ -254,49 +274,51 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
   def test_load_weights_between_noncudnn_rnn(self, rnn_type, to_cudnn,
                                              bidirectional, implementation,
                                              model_nest_level, model_type):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        timesteps = 6
-        input_shape = (timesteps, input_size)
-        units = 2
-        num_samples = 32
-        inputs = np.random.random((num_samples, timesteps, input_size))
-
-        rnn_layer_kwargs = {
-            'recurrent_activation': 'sigmoid',
-            # ensure biases are non-zero and properly converted
-            'bias_initializer': 'random_uniform',
-            'implementation': implementation
-        }
-        if rnn_type == 'LSTM':
-          rnn_layer_class = keras.layers.LSTM
-          cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
-        else:
-          rnn_layer_class = keras.layers.GRU
-          cudnn_rnn_layer_class = keras.layers.CuDNNGRU
-          rnn_layer_kwargs['reset_after'] = True
-
-        layer = rnn_layer_class(units, **rnn_layer_kwargs)
-        if bidirectional:
-          layer = keras.layers.Bidirectional(layer)
-
-        cudnn_layer = cudnn_rnn_layer_class(units)
-        if bidirectional:
-          cudnn_layer = keras.layers.Bidirectional(cudnn_layer)
-
-        model = self._make_nested_model(input_shape, layer, model_nest_level,
-                                        model_type)
-        cudnn_model = self._make_nested_model(input_shape, cudnn_layer,
-                                              model_nest_level, model_type)
-
-        if to_cudnn:
-          self._convert_model_weights(model, cudnn_model)
-        else:
-          self._convert_model_weights(cudnn_model, model)
-
-        self.assertAllClose(model.predict(inputs), cudnn_model.predict(inputs),
-                            atol=1e-4)
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      timesteps = 6
+      input_shape = (timesteps, input_size)
+      units = 2
+      num_samples = 32
+      inputs = np.random.random((num_samples, timesteps, input_size))
+
+      rnn_layer_kwargs = {
+          'recurrent_activation': 'sigmoid',
+          # ensure biases are non-zero and properly converted
+          'bias_initializer': 'random_uniform',
+          'implementation': implementation
+      }
+      if rnn_type == 'LSTM':
+        rnn_layer_class = keras.layers.LSTM
+        cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
+      else:
+        rnn_layer_class = keras.layers.GRU
+        cudnn_rnn_layer_class = keras.layers.CuDNNGRU
+        rnn_layer_kwargs['reset_after'] = True
+
+      layer = rnn_layer_class(units, **rnn_layer_kwargs)
+      if bidirectional:
+        layer = keras.layers.Bidirectional(layer)
+
+      cudnn_layer = cudnn_rnn_layer_class(units)
+      if bidirectional:
+        cudnn_layer = keras.layers.Bidirectional(cudnn_layer)
+
+      model = self._make_nested_model(input_shape, layer, model_nest_level,
+                                      model_type)
+      cudnn_model = self._make_nested_model(input_shape, cudnn_layer,
+                                            model_nest_level, model_type)
+
+      if to_cudnn:
+        self._convert_model_weights(model, cudnn_model)
+      else:
+        self._convert_model_weights(cudnn_model, model)
+
+      self.assertAllClose(
+          model.predict(inputs), cudnn_model.predict(inputs), atol=1e-4)
 
   def _make_nested_model(self, input_shape, layer, level=1, model_type='func'):
     # example: make_nested_seq_model((1,), Dense(10), level=2).summary()
@@ -334,149 +356,145 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
                                                               to_cudnn):
     # Similar test as test_load_weights_between_noncudnn_rnn() but has different
     # rank of input due to usage of TimeDistributed. Issue: #10356.
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_size = 10
-        steps = 6
-        timesteps = 6
-        input_shape = (timesteps, steps, input_size)
-        units = 2
-        num_samples = 32
-        inputs = np.random.random((num_samples, timesteps, steps, input_size))
-
-        rnn_layer_kwargs = {
-            'recurrent_activation': 'sigmoid',
-            # ensure biases are non-zero and properly converted
-            'bias_initializer': 'random_uniform',
-        }
-        if rnn_type == 'LSTM':
-          rnn_layer_class = keras.layers.LSTM
-          cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
-        else:
-          rnn_layer_class = keras.layers.GRU
-          cudnn_rnn_layer_class = keras.layers.CuDNNGRU
-          rnn_layer_kwargs['reset_after'] = True
-
-        layer = rnn_layer_class(units, **rnn_layer_kwargs)
-        layer = keras.layers.TimeDistributed(layer)
-
-        cudnn_layer = cudnn_rnn_layer_class(units)
-        cudnn_layer = keras.layers.TimeDistributed(cudnn_layer)
-
-        model = self._make_nested_model(input_shape, layer)
-        cudnn_model = self._make_nested_model(input_shape, cudnn_layer)
-
-        if to_cudnn:
-          self._convert_model_weights(model, cudnn_model)
-        else:
-          self._convert_model_weights(cudnn_model, model)
-
-        self.assertAllClose(model.predict(inputs), cudnn_model.predict(inputs),
-                            atol=1e-4)
-
-  @test_util.run_in_graph_and_eager_modes
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_size = 10
+      steps = 6
+      timesteps = 6
+      input_shape = (timesteps, steps, input_size)
+      units = 2
+      num_samples = 32
+      inputs = np.random.random((num_samples, timesteps, steps, input_size))
+
+      rnn_layer_kwargs = {
+          'recurrent_activation': 'sigmoid',
+          # ensure biases are non-zero and properly converted
+          'bias_initializer': 'random_uniform',
+      }
+      if rnn_type == 'LSTM':
+        rnn_layer_class = keras.layers.LSTM
+        cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
+      else:
+        rnn_layer_class = keras.layers.GRU
+        cudnn_rnn_layer_class = keras.layers.CuDNNGRU
+        rnn_layer_kwargs['reset_after'] = True
+
+      layer = rnn_layer_class(units, **rnn_layer_kwargs)
+      layer = keras.layers.TimeDistributed(layer)
+
+      cudnn_layer = cudnn_rnn_layer_class(units)
+      cudnn_layer = keras.layers.TimeDistributed(cudnn_layer)
+
+      model = self._make_nested_model(input_shape, layer)
+      cudnn_model = self._make_nested_model(input_shape, cudnn_layer)
+
+      if to_cudnn:
+        self._convert_model_weights(model, cudnn_model)
+      else:
+        self._convert_model_weights(cudnn_model, model)
+
+      self.assertAllClose(
+          model.predict(inputs), cudnn_model.predict(inputs), atol=1e-4)
+
   def test_cudnnrnn_bidirectional(self):
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        rnn = keras.layers.CuDNNGRU
-        samples = 2
-        dim = 2
-        timesteps = 2
-        output_dim = 2
-        mode = 'concat'
-
-        x = np.random.random((samples, timesteps, dim))
-        target_dim = 2 * output_dim if mode == 'concat' else output_dim
-        y = np.random.random((samples, target_dim))
-
-        # test with Sequential model
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Bidirectional(
-                rnn(output_dim), merge_mode=mode, input_shape=(None, dim)))
-        model.compile(
-            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-        model.fit(x, y, epochs=1, batch_size=1)
-
-        # test config
-        model.get_config()
-        model = keras.models.model_from_json(model.to_json())
-        model.summary()
-
-        # test stacked bidirectional layers
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Bidirectional(
-                rnn(output_dim, return_sequences=True),
-                merge_mode=mode,
-                input_shape=(None, dim)))
-        model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode))
-        model.compile(
-            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-        model.fit(x, y, epochs=1, batch_size=1)
-
-        # test with functional API
-        inputs = keras.Input((timesteps, dim))
-        outputs = keras.layers.Bidirectional(
-            rnn(output_dim), merge_mode=mode)(
-                inputs)
-        model = keras.Model(inputs, outputs)
-        model.compile(
-            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-        model.fit(x, y, epochs=1, batch_size=1)
-
-        # Bidirectional and stateful
-        inputs = keras.Input(batch_shape=(1, timesteps, dim))
-        outputs = keras.layers.Bidirectional(
-            rnn(output_dim, stateful=True), merge_mode=mode)(
-                inputs)
-        model = keras.Model(inputs, outputs)
-        model.compile(
-            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-        model.fit(x, y, epochs=1, batch_size=1)
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      rnn = keras.layers.CuDNNGRU
+      samples = 2
+      dim = 2
+      timesteps = 2
+      output_dim = 2
+      mode = 'concat'
+
+      x = np.random.random((samples, timesteps, dim))
+      target_dim = 2 * output_dim if mode == 'concat' else output_dim
+      y = np.random.random((samples, target_dim))
+
+      # test with Sequential model
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Bidirectional(
+              rnn(output_dim), merge_mode=mode, input_shape=(None, dim)))
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.fit(x, y, epochs=1, batch_size=1)
+
+      # test config
+      model.get_config()
+      model = keras.models.model_from_json(model.to_json())
+      model.summary()
+
+      # test stacked bidirectional layers
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Bidirectional(
+              rnn(output_dim, return_sequences=True),
+              merge_mode=mode,
+              input_shape=(None, dim)))
+      model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode))
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.fit(x, y, epochs=1, batch_size=1)
+
+      # test with functional API
+      inputs = keras.Input((timesteps, dim))
+      outputs = keras.layers.Bidirectional(
+          rnn(output_dim), merge_mode=mode)(
+              inputs)
+      model = keras.Model(inputs, outputs)
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.fit(x, y, epochs=1, batch_size=1)
+
+      # Bidirectional and stateful
+      inputs = keras.Input(batch_shape=(1, timesteps, dim))
+      outputs = keras.layers.Bidirectional(
+          rnn(output_dim, stateful=True), merge_mode=mode)(
+              inputs)
+      model = keras.Model(inputs, outputs)
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.fit(x, y, epochs=1, batch_size=1)
 
   def test_preprocess_weights_for_loading_gru_incompatible(self):
     """Test loading weights between incompatible layers.
 
     Should fail fast with an exception.
     """
-    if test.is_gpu_available(cuda_only=True):
-      with self.session(use_gpu=True):
-        input_shape = (3, 5)
-
-        def gru(cudnn=False, **kwargs):
-          layer_class = keras.layers.CuDNNGRU if cudnn else keras.layers.GRU
-          return layer_class(2, input_shape=input_shape, **kwargs)
-
-        def get_layer_weights(layer):
-          layer.build(input_shape=input_shape)
-          return layer.get_weights()
-
-        def assert_not_compatible(src, dest, message):
-          with self.assertRaises(ValueError) as ex:
-            keras.engine.saving.preprocess_weights_for_loading(
-                dest,
-                get_layer_weights(src))
-          self.assertIn(message, str(ex.exception))
-
-        assert_not_compatible(
-            gru(),
-            gru(cudnn=True),
-            'GRU(reset_after=False) is not compatible with CuDNNGRU')
-        assert_not_compatible(
-            gru(cudnn=True),
-            gru(),
-            'CuDNNGRU is not compatible with GRU(reset_after=False)')
-        assert_not_compatible(
-            gru(),
-            gru(reset_after=True),
-            'GRU(reset_after=False) is not compatible with '
-            'GRU(reset_after=True)')
-        assert_not_compatible(
-            gru(reset_after=True),
-            gru(),
-            'GRU(reset_after=True) is not compatible with '
-            'GRU(reset_after=False)')
+    if not test.is_gpu_available(cuda_only=True):
+      self.skipTest('No CUDA GPU available')
+
+    with test_util.use_gpu():
+      input_shape = (3, 5)
+
+      def gru(cudnn=False, **kwargs):
+        layer_class = keras.layers.CuDNNGRU if cudnn else keras.layers.GRU
+        return layer_class(2, input_shape=input_shape, **kwargs)
+
+      def get_layer_weights(layer):
+        layer.build(input_shape=input_shape)
+        return layer.get_weights()
+
+      def assert_not_compatible(src, dest, message):
+        with self.assertRaises(ValueError) as ex:
+          keras.engine.saving.preprocess_weights_for_loading(
+              dest, get_layer_weights(src))
+        self.assertIn(message, str(ex.exception))
+
+      assert_not_compatible(
+          gru(), gru(cudnn=True),
+          'GRU(reset_after=False) is not compatible with CuDNNGRU')
+      assert_not_compatible(
+          gru(cudnn=True), gru(),
+          'CuDNNGRU is not compatible with GRU(reset_after=False)')
+      assert_not_compatible(
+          gru(), gru(reset_after=True),
+          'GRU(reset_after=False) is not compatible with '
+          'GRU(reset_after=True)')
+      assert_not_compatible(
+          gru(reset_after=True), gru(),
+          'GRU(reset_after=True) is not compatible with '
+          'GRU(reset_after=False)')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
index fc7d4572e2..272c2b1dac 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
@@ -77,7 +77,7 @@ class BinaryOpTest(test.TestCase):
 
   def _compareCpu(self, x, y, np_func, tf_func, also_compare_variables=False):
     np_ans = np_func(x, y)
-    with self.test_session(use_gpu=False):
+    with test_util.force_cpu():
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
@@ -174,7 +174,7 @@ class BinaryOpTest(test.TestCase):
 
   def _compareGpu(self, x, y, np_func, tf_func):
     np_ans = np_func(x, y)
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
@@ -252,10 +252,12 @@ class BinaryOpTest(test.TestCase):
     y = np.array([1, 2]).reshape(2, 1).astype(np.int32)
     var_x = variables.Variable(x)
     var_y = variables.Variable(y)
+
     with self.cached_session() as sess:
       self.evaluate([var_x.initializer, var_y.initializer])
-      left_result = (var_x * y).eval()
-      right_result = (x * var_y).eval()
+      left_result = self.evaluate(var_x * y)
+      right_result = self.evaluate(x * var_y)
+
     np_result = x * y
     self.assertAllEqual(np_result, left_result)
     self.assertAllEqual(np_result, right_result)
@@ -382,7 +384,7 @@ class BinaryOpTest(test.TestCase):
   def testStringComparison(self):
     x = np.array([["abc", "bh"], ["c", ""]])
     y = np.array([["abc", "bh"], ["def", "hi"]])
-    with self.test_session(use_gpu=False) as sess:
+    with test_util.force_cpu():
       cmp_eq = math_ops.equal(x, y)
       cmp_not_eq = math_ops.not_equal(x, y)
       values = self.evaluate([cmp_eq, cmp_not_eq])
@@ -716,35 +718,35 @@ class BinaryOpTest(test.TestCase):
 
   def testPowNegativeExponent(self):
     for dtype in [np.int32, np.int64]:
-      with self.test_session(use_gpu=False) as sess:
+      with test_util.force_cpu():
         with self.assertRaisesRegexp(
             errors_impl.InvalidArgumentError,
             "Integers to negative integer powers are not allowed"):
           x = np.array([5, 2]).astype(dtype)
           y = np.array([-2, 3]).astype(dtype)
-          sess.run(math_ops.pow(x, y))
+          self.evaluate(math_ops.pow(x, y))
 
-      with self.test_session(use_gpu=False) as sess:
+      with test_util.force_cpu():
         with self.assertRaisesRegexp(
             errors_impl.InvalidArgumentError,
             "Integers to negative integer powers are not allowed"):
           x = np.array([5, 2]).astype(dtype)
           y = np.array([2, -3]).astype(dtype)
-          sess.run(math_ops.pow(x, y))
+          self.evaluate(math_ops.pow(x, y))
 
-      with self.test_session(use_gpu=False) as sess:
+      with test_util.force_cpu():
         with self.assertRaisesRegexp(
             errors_impl.InvalidArgumentError,
             "Integers to negative integer powers are not allowed"):
           x = np.array([5, 2]).astype(dtype)
           y = -3
-          sess.run(math_ops.pow(x, y))
+          self.evaluate(math_ops.pow(x, y))
 
 
 class ComparisonOpTest(test.TestCase):
 
   def _compareScalar(self, func, x, y, dtype):
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       out = func(
           ops.convert_to_tensor(np.array([x]).astype(dtype)),
           ops.convert_to_tensor(np.array([y]).astype(dtype)))
@@ -777,7 +779,7 @@ class ComparisonOpTest(test.TestCase):
 
   def _compare(self, x, y, np_func, tf_func):
     np_ans = np_func(x, y)
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       out = tf_func(ops.convert_to_tensor(x), ops.convert_to_tensor(y))
       tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index ab116c400a..7e14f95be4 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -84,7 +84,7 @@ def _default_tolerance(dtype):
 class ComparisonOpTest(test.TestCase):
 
   def _compareScalar(self, func, x, y, dtype):
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       out = func(
           ops.convert_to_tensor(np.array([x]).astype(dtype)),
           ops.convert_to_tensor(np.array([y]).astype(dtype)))
@@ -117,7 +117,7 @@ class ComparisonOpTest(test.TestCase):
 
   def _compare(self, x, y, np_func, tf_func):
     np_ans = np_func(x, y)
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       out = tf_func(ops.convert_to_tensor(x), ops.convert_to_tensor(y))
       tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
@@ -218,8 +218,7 @@ class LogicalOpTest(test.TestCase):
 
   def _compareBinary(self, x, y, np_func, tf_func, use_gpu=False):
     np_ans = np_func(x, y)
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+    with test_util.device(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       out = tf_func(inx, iny)
@@ -230,8 +229,7 @@ class LogicalOpTest(test.TestCase):
 
   def _not(self, x, use_gpu=False):
     np_ans = np.logical_not(x)
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+    with test_util.device(use_gpu=use_gpu):
       out = math_ops.logical_not(ops.convert_to_tensor(x))
       tf_val = self.evaluate(out)
     self.assertEqual(out.dtype, dtypes_lib.bool)
@@ -316,8 +314,7 @@ class SelectOpTest(test.TestCase):
 
   def _compare(self, c, x, y, use_gpu):
     np_ans = np.where(c, x, y)
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+    with test_util.device(use_gpu=use_gpu):
       out = array_ops.where(c, x, y)
       tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
@@ -460,8 +457,7 @@ class BatchSelectOpTest(test.TestCase):
     np_ans = np.dstack(
         [x_i if c_i else y_i for c_i, x_i, y_i in zip(c, x, y)]).transpose(
             [2, 0, 1])
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+    with test_util.device(use_gpu=use_gpu):
       out = array_ops.where(c, x, y)
       tf_ans = self.evaluate(out)
     self.assertAllEqual(np_ans, tf_ans)
@@ -566,9 +562,7 @@ class MinMaxOpTest(test.TestCase):
 
   def _compare(self, x, y, use_gpu):
     np_min, np_max = np.minimum(x, y), np.maximum(x, y)
-    with self.test_session(
-        use_gpu=use_gpu,
-        force_gpu=use_gpu and test_util.is_gpu_available()) as sess:
+    with test_util.device(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(x)
       iny = ops.convert_to_tensor(y)
       omin, omax = math_ops.minimum(inx, iny), math_ops.maximum(inx, iny)
@@ -641,13 +635,13 @@ class MinMaxOpTest(test.TestCase):
 class MathOpsOverloadTest(test.TestCase):
 
   def _computeTensorAndLiteral(self, x, y, dtype, func):
-    with self.test_session(use_gpu=False):
+    with test_util.force_cpu():
       inx = ops.convert_to_tensor(x, dtype=dtype)
       z = func(inx, y)  # Should use __add__, __sub__, etc.
       return self.evaluate(z)
 
   def _computeLiteralAndTensor(self, x, y, dtype, func):
-    with self.test_session(use_gpu=False):
+    with test_util.force_cpu():
       iny = ops.convert_to_tensor(y, dtype=dtype)
       z = func(x, iny)  # Should use __radd__, __rsub__, etc.
       return self.evaluate(z)
@@ -661,7 +655,7 @@ class MathOpsOverloadTest(test.TestCase):
 
   def _compareUnary(self, x, dtype, np_func, tf_func):
     np_ans = np_func(x).astype(dtype.as_numpy_dtype)
-    with self.test_session(use_gpu=False):
+    with test_util.force_cpu():
       self.assertAllClose(
           np_ans, self.evaluate(tf_func(ops.convert_to_tensor(x, dtype=dtype))))
 
@@ -730,9 +724,7 @@ class IsFiniteInfNanTest(test.TestCase):
 
   def _compare(self, x, use_gpu):
     np_finite, np_inf, np_nan = np.isfinite(x), np.isinf(x), np.isnan(x)
-    with self.test_session(
-        use_gpu=use_gpu,
-        force_gpu=use_gpu and test_util.is_gpu_available()) as sess:
+    with test_util.device(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(x)
       ofinite, oinf, onan = math_ops.is_finite(inx), math_ops.is_inf(
           inx), math_ops.is_nan(inx)
@@ -773,7 +765,7 @@ class IsFiniteInfNanTest(test.TestCase):
           x = np.full((size,), value, dtype=dtype)
           np_y = np.sqrt(x)
           np_nan = np.isnan(np_y)
-          with self.test_session(force_gpu=test_util.is_gpu_available()):
+          with test_util.use_gpu():
             tf_y = math_ops.sqrt(x)
             tf_nan = math_ops.is_nan(tf_y)
             if value < 0:
@@ -786,18 +778,20 @@ class RoundingTest(test.TestCase):
 
   def _compare_values(self, x, y=None):
     y = np.rint(x) if y is None else np.asarray(y)
-    with self.cached_session() as sess:
-      tf_rint = math_ops.rint(x)
-      np_rint = self.evaluate(tf_rint)
+
+    tf_rint = math_ops.rint(x)
+    np_rint = self.evaluate(tf_rint)
+
     self.assertAllEqual(y, np_rint)
     self.assertShapeEqual(y, tf_rint)
 
   def _compare(self, x):
     np_floor, np_ceil = np.floor(x), np.ceil(x)
-    with self.cached_session() as sess:
-      inx = ops.convert_to_tensor(x)
-      ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx)
-      tf_floor, tf_ceil = self.evaluate([ofloor, oceil])
+
+    inx = ops.convert_to_tensor(x)
+    ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx)
+    tf_floor, tf_ceil = self.evaluate([ofloor, oceil])
+
     self.assertAllEqual(np_floor, tf_floor)
     self.assertAllEqual(np_ceil, tf_ceil)
     self.assertShapeEqual(np_floor, ofloor)
@@ -828,12 +822,13 @@ class ComplexMakeRealImagTest(test.TestCase):
 
   def _compareMake(self, real, imag, use_gpu):
     np_ans = real + (1j) * imag
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+
+    with test_util.device(use_gpu=use_gpu):
       real = ops.convert_to_tensor(real)
       imag = ops.convert_to_tensor(imag)
       tf_ans = math_ops.complex(real, imag)
       out = self.evaluate(tf_ans)
+
     self.assertAllEqual(np_ans, out)
     self.assertShapeEqual(np_ans, tf_ans)
 
@@ -848,8 +843,8 @@ class ComplexMakeRealImagTest(test.TestCase):
   def _compareRealImag(self, cplx, use_gpu):
     np_real, np_imag = np.real(cplx), np.imag(cplx)
     np_zeros = np_real * 0
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+
+    with test_util.device(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(cplx)
       tf_real = math_ops.real(inx)
       tf_imag = math_ops.imag(inx)
@@ -876,12 +871,12 @@ class ComplexMakeRealImagTest(test.TestCase):
 
   def _compareAngle(self, cplx, use_gpu):
     np_angle = np.angle(cplx)
-    with self.test_session(
-        use_gpu=use_gpu,
-        force_gpu=use_gpu and test_util.is_gpu_available()) as sess:
+
+    with test_util.device(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(cplx)
       tf_angle = math_ops.angle(inx)
       tf_angle_val = self.evaluate(tf_angle)
+
     self.assertAllEqual(np_angle, tf_angle_val)
     self.assertShapeEqual(np_angle, tf_angle)
 
@@ -912,8 +907,7 @@ class ComplexMakeRealImagTest(test.TestCase):
 
   def _compareConj(self, cplx, use_gpu):
     np_ans = np.conj(cplx)
-    with self.test_session(use_gpu=use_gpu,
-                           force_gpu=use_gpu and test_util.is_gpu_available()):
+    with test_util.device(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(cplx)
       tf_conj = math_ops.conj(inx)
       tf_ans = self.evaluate(tf_conj)
diff --git a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
index 7096083a1f..3e8294f34b 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
@@ -76,7 +76,7 @@ class UnaryOpTest(test.TestCase):
     if grad_atol is None:
       grad_atol = _default_tolerance(x.dtype)
     np_ans = np_func(x)
-    with self.test_session(use_gpu=False):
+    with self.cached_session(use_gpu=False):
       inx = ops.convert_to_tensor(x)
       if x.dtype in (np.float32, np.float64,
                      dtypes_lib.bfloat16.as_numpy_dtype):
@@ -121,24 +121,22 @@ class UnaryOpTest(test.TestCase):
   def _check(self, result_tensor, result_np, input_sp_t, tol):
     self.assertTrue(isinstance(result_tensor, sparse_tensor.SparseTensor))
     self.assertTrue(isinstance(input_sp_t, sparse_tensor.SparseTensor))
-    self.assertAllEqual(input_sp_t.indices.eval(), result_tensor.indices.eval())
-    self.assertAllEqual(input_sp_t.dense_shape.eval(),
-                        result_tensor.dense_shape.eval())
+    self.assertAllEqual(input_sp_t.indices, result_tensor.indices)
+    self.assertAllEqual(input_sp_t.dense_shape, result_tensor.dense_shape)
     if tol is None:
-      self.assertAllClose(result_np, result_tensor.values.eval())
+      self.assertAllClose(result_np, result_tensor.values)
     else:
-      self.assertAllClose(
-          result_np, result_tensor.values.eval(), rtol=tol, atol=tol)
+      self.assertAllClose(result_np, result_tensor.values, rtol=tol, atol=tol)
 
   def _compareSparseCpu(self, x, np_func, tf_func, tol):
     x_sp, x_sp_vals = _sparsify(x)
     res_np = np_func(x_sp_vals)
-    with self.test_session(use_gpu=False):
+    with test_util.force_cpu():
       self._check(tf_func(x_sp), res_np, x_sp, tol)
 
   def _compareGpu(self, x, np_func, tf_func):
     np_ans = np_func(x)
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       result = tf_func(ops.convert_to_tensor(x))
       tf_gpu = self.evaluate(result)
     if x.dtype == np.float16:
@@ -150,7 +148,7 @@ class UnaryOpTest(test.TestCase):
   def _compareSparseGpu(self, x, np_func, tf_func, tol):
     x_sp, x_sp_vals = _sparsify(x)
     res_np = np_func(x_sp_vals)
-    with self.test_session(force_gpu=test_util.is_gpu_available()):
+    with test_util.use_gpu():
       self._check(tf_func(x_sp), res_np, x_sp, tol)
 
   def _compareBoth(self, x, np_func, tf_func):
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
index cfec4d08fb..031a1c281c 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
@@ -67,11 +67,11 @@ class MultinomialTest(test.TestCase):
         self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples)
 
   def testOneOpMultipleStepsIndependent(self):
-    with self.test_session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       sample_op1, _ = self._make_ops(10)
       # Consecutive runs shouldn't yield identical output.
-      sample1a = sess.run(sample_op1)
-      sample1b = sess.run(sample_op1)
+      sample1a = self.evaluate(sample_op1)
+      sample1b = self.evaluate(sample_op1)
       self.assertFalse(np.equal(sample1a, sample1b).all())
 
   def testEagerOneOpMultipleStepsIndependent(self):
@@ -81,26 +81,26 @@ class MultinomialTest(test.TestCase):
       self.assertFalse(np.equal(sample1.numpy(), sample2.numpy()).all())
 
   def testTwoOpsIndependent(self):
-    with self.test_session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       sample_op1, sample_op2 = self._make_ops(32)
-      sample1, sample2 = sess.run([sample_op1, sample_op2])
+      sample1, sample2 = self.evaluate([sample_op1, sample_op2])
       # We expect sample1 and sample2 to be independent.
       # 1 in 2^32 chance of this assertion failing.
       self.assertFalse(np.equal(sample1, sample2).all())
 
   def testTwoOpsSameSeedDrawSameSequences(self):
-    with self.test_session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       sample_op1, sample_op2 = self._make_ops(1000, seed=1)
-      sample1, sample2 = sess.run([sample_op1, sample_op2])
+      sample1, sample2 = self.evaluate([sample_op1, sample_op2])
       self.assertAllEqual(sample1, sample2)
 
   def testLargeLogits(self):
     for neg in [True, False]:
-      with self.test_session(use_gpu=True):
+      with test_util.use_gpu():
         logits = np.array([[1000.] * 5])
         if neg:
           logits *= -1
-        samples = random_ops.multinomial(logits, 10).eval()
+        samples = self.evaluate(random_ops.multinomial(logits, 10))
       # Sampled classes should be in-range.
       self.assertTrue((samples >= 0).all())
       self.assertTrue((samples < 5).all())
@@ -157,10 +157,10 @@ class MultinomialTest(test.TestCase):
     Returns:
       Frequencies from sampled classes; shape [batch_size, num_classes].
     """
-    with self.test_session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       random_seed.set_random_seed(1618)
       op = sampler(constant_op.constant(logits), num_samples)
-      d = sess.run(op)
+      d = self.evaluate(op)
 
     batch_size, num_classes = logits.shape
     freqs_mat = []
@@ -186,25 +186,26 @@ class MultinomialTest(test.TestCase):
 
   def testEmpty(self):
     classes = 5
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       for batch in 0, 3:
         for samples in 0, 7:
-          x = random_ops.multinomial(
-              array_ops.zeros([batch, classes]), samples).eval()
+          x = self.evaluate(
+              random_ops.multinomial(
+                  array_ops.zeros([batch, classes]), samples))
           self.assertEqual(x.shape, (batch, samples))
 
   def testEmptyClasses(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       x = random_ops.multinomial(array_ops.zeros([5, 0]), 7)
       with self.assertRaisesOpError("num_classes should be positive"):
         self.evaluate(x)
 
   def testNegativeMinLogits(self):
     random_seed.set_random_seed(78844)
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       logits = constant_op.constant([[np.finfo(np.float32).min] * 1023 + [0]])
       num_samples = 1000
-      samples = random_ops.multinomial(logits, num_samples).eval()
+      samples = self.evaluate(random_ops.multinomial(logits, num_samples))
       self.assertAllEqual([[1023] * num_samples], samples)
 
 
diff --git a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
index d80bea955e..071d6c2998 100644
--- a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import stateless_random_ops as stateless
@@ -58,7 +59,7 @@ class StatelessOpsTest(test.TestCase):
       preseed = invert_philox(key, (seed[0], 0, seed[1], 0)).astype(np.uint64)
       preseed = preseed[::2] | preseed[1::2] << 32
       random_seed.set_random_seed(seed[0])
-      with self.test_session(use_gpu=True):
+      with test_util.use_gpu():
         for stateless_op, stateful_op in cases:
           stateful = stateful_op(seed=seed[1])
           pure = stateless_op(seed=preseed)
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 13b39926ec..30563092c8 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -456,7 +456,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
 
   # TODO(alive): get this to work in Eager mode.
   def testGPU(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       abc = variable_scope.get_variable(
           "abc",
           shape=[1],
diff --git a/tensorflow/python/kernel_tests/variable_ops_test.py b/tensorflow/python/kernel_tests/variable_ops_test.py
index cdfd805a93..c63d7f89c7 100644
--- a/tensorflow/python/kernel_tests/variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/variable_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import math_ops
@@ -164,7 +165,7 @@ class VariableOpTest(test.TestCase):
     self.assertEqual(tensor_shape.unknown_shape(), subbed.get_shape())
 
   def testTemporaryVariable(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       var = gen_state_ops.temporary_variable(
           [1, 2], dtypes.float32, var_name="foo")
       var = state_ops.assign(var, [[4.0, 5.0]])
@@ -173,14 +174,14 @@ class VariableOpTest(test.TestCase):
       self.assertAllClose([[10.0, 12.0]], self.evaluate(final))
 
   def testDestroyNonexistentTemporaryVariable(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       var = gen_state_ops.temporary_variable([1, 2], dtypes.float32)
       final = gen_state_ops.destroy_temporary_variable(var, var_name="bad")
       with self.assertRaises(errors.NotFoundError):
         self.evaluate(final)
 
   def testDuplicateTemporaryVariable(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       var1 = gen_state_ops.temporary_variable(
           [1, 2], dtypes.float32, var_name="dup")
       var1 = state_ops.assign(var1, [[1.0, 2.0]])
@@ -192,7 +193,7 @@ class VariableOpTest(test.TestCase):
         self.evaluate(final)
 
   def testDestroyTemporaryVariableTwice(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       var = gen_state_ops.temporary_variable([1, 2], dtypes.float32)
       val1 = gen_state_ops.destroy_temporary_variable(var, var_name="dup")
       val2 = gen_state_ops.destroy_temporary_variable(var, var_name="dup")
@@ -201,14 +202,14 @@ class VariableOpTest(test.TestCase):
         self.evaluate(final)
 
   def testTemporaryVariableNoLeak(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       var = gen_state_ops.temporary_variable(
           [1, 2], dtypes.float32, var_name="bar")
       final = array_ops.identity(var)
       self.evaluate(final)
 
   def testTwoTemporaryVariablesNoLeaks(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       var1 = gen_state_ops.temporary_variable(
           [1, 2], dtypes.float32, var_name="var1")
       var2 = gen_state_ops.temporary_variable(
@@ -217,13 +218,13 @@ class VariableOpTest(test.TestCase):
       self.evaluate(final)
 
   def testAssignDependencyAcrossDevices(self):
-    with self.test_session(use_gpu=True):
+    with test_util.use_gpu():
       # The variable and an op to increment it are on the GPU.
       var = state_ops.variable_op([1], dtypes.float32)
       self.evaluate(state_ops.assign(var, [1.0]))
       increment = state_ops.assign_add(var, [1.0])
       with ops.control_dependencies([increment]):
-        with ops.device("/cpu:0"):
+        with test_util.force_cpu():
           # This mul op is pinned to the CPU, but reads the variable from the
           # GPU. The test ensures that the dependency on 'increment' is still
           # honored, i.e., the Send and Recv from GPU to CPU should take place
diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
index 228c9bc5e4..40a101b4da 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
@@ -118,8 +118,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                                    combiner)
 
     segmented = segment_op(rt, segment_ids, num_segments)
-    with self.test_session():
-      self.assertListEqual(self.evaluate(segmented).tolist(), expected)
+    self.assertListEqual(self.evaluate(segmented).tolist(), expected)
 
   @parameterized.parameters(
       (ragged.segment_sum, sum, [0, 0, 1, 1, 2, 2]),
@@ -155,9 +154,8 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                                    combiner)
 
     segmented = segment_op(rt, segment_ids, num_segments)
-    with self.test_session():
-      self.assertNestedListAmostEqual(
-          self.evaluate(segmented).tolist(), expected, places=5)
+    self.assertNestedListAmostEqual(
+        self.evaluate(segmented).tolist(), expected, places=5)
 
   def testRaggedRankTwo(self):
     rt = ragged.constant([
@@ -172,16 +170,14 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                  [],                                # row 1
                  [[411, 412], [321, 322], [331]]    # row 2
                 ]  # pyformat: disable
-    with self.test_session():
-      self.assertEqual(self.evaluate(segmented1).tolist(), expected1)
+    self.assertEqual(self.evaluate(segmented1).tolist(), expected1)
 
     segment_ids2 = [1, 2, 1, 1]
     segmented2 = ragged.segment_sum(rt, segment_ids2, 3)
     expected2 = [[],
                  [[111+411, 112+412, 113, 114], [121+321, 322], [331]],
                  []]  # pyformat: disable
-    with self.test_session():
-      self.assertEqual(self.evaluate(segmented2).tolist(), expected2)
+    self.assertEqual(self.evaluate(segmented2).tolist(), expected2)
 
   def testRaggedSegmentIds(self):
     rt = ragged.constant([
@@ -195,8 +191,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
     expected = [[],
                 [111+321, 112+322, 113, 114],
                 [121+331+411, 412]]  # pyformat: disable
-    with self.test_session():
-      self.assertEqual(self.evaluate(segmented).tolist(), expected)
+    self.assertEqual(self.evaluate(segmented).tolist(), expected)
 
   def testShapeMismatchError1(self):
     dt = constant_op.constant([1, 2, 3, 4, 5, 6])
@@ -226,7 +221,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
         array_ops.placeholder_with_default(segment_ids.values, None),
         array_ops.placeholder_with_default(segment_ids.row_splits, None))
     segmented2 = ragged.segment_sum(rt, segment_ids2, 3)
-    with self.test_session():
+    with self.cached_session():
       self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           'segment_ids.shape must be a prefix of data.shape.*', segmented2.eval)
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
index cd382fe0b8..befe30f0e1 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
@@ -28,49 +28,39 @@ class RaggedTensorBoundingShapeOp(test_util.TensorFlowTestCase):
   def testDocStringExample(self):
     # This is the example from ragged.bounding_shape.__doc__.
     rt = ragged.constant([[1, 2, 3, 4], [5], [], [6, 7, 8, 9], [10]])
-    with self.test_session():
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt)).tolist(), [5, 4])
+    self.assertEqual(self.evaluate(ragged.bounding_shape(rt)).tolist(), [5, 4])
 
   def test2DRaggedTensorWithOneRaggedDimension(self):
     values = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
     rt1 = ragged.from_row_splits(values, [0, 2, 5, 6, 6, 7])
     rt2 = ragged.from_row_splits(values, [0, 7])
     rt3 = ragged.from_row_splits(values, [0, 0, 7, 7])
-    with self.test_session():
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3])
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7])
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7])
+    self.assertEqual(self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3])
+    self.assertEqual(self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7])
+    self.assertEqual(self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7])
 
   def test3DRaggedTensorWithOneRaggedDimension(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
     rt1 = ragged.from_row_splits(values, [0, 2, 5, 6, 6, 7])
     rt2 = ragged.from_row_splits(values, [0, 7])
     rt3 = ragged.from_row_splits(values, [0, 0, 7, 7])
-    with self.test_session():
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3, 2])
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7, 2])
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7, 2])
+    self.assertEqual(
+        self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3, 2])
+    self.assertEqual(
+        self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7, 2])
+    self.assertEqual(
+        self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7, 2])
 
   def testNonRaggedTensor(self):
     dt = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
-    with self.test_session():
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(dt)).tolist(), [4, 3])
+    self.assertEqual(self.evaluate(ragged.bounding_shape(dt)).tolist(), [4, 3])
 
   def testExplicitAxisOptimizations(self):
     rt = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
-    with self.test_session():
-      self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 0)).tolist(), 5)
-      self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 1)).tolist(), 3)
-      self.assertEqual(
-          self.evaluate(ragged.bounding_shape(rt, [1, 0])).tolist(), [3, 5])
+    self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 0)).tolist(), 5)
+    self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 1)).tolist(), 3)
+    self.assertEqual(
+        self.evaluate(ragged.bounding_shape(rt, [1, 0])).tolist(), [3, 5])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index 66b15d9bcc..fa681c07bb 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -118,9 +118,8 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     # From section: "Component Tensors"
     rt = ragged.from_row_splits(
         values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8])
-    with self.test_session():
-      self.assertEqual(rt.tolist(),
-                       [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+    self.assertEqual(
+        self.evaluate(rt).tolist(), [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
     del rt
 
     # From section: "Alternative Row-Partitioning Schemes"
@@ -132,9 +131,8 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt4 = ragged.from_row_starts(values, row_starts=[0, 4, 4, 7, 8])
     rt5 = ragged.from_row_limits(values, row_limits=[4, 4, 7, 8, 8])
     for rt in (rt1, rt2, rt3, rt4, rt5):
-      with self.test_session():
-        self.assertEqual(rt.tolist(),
-                         [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+      self.assertEqual(
+          self.evaluate(rt).tolist(), [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
     del rt1, rt2, rt3, rt4, rt5
 
     # From section: "Multiple Ragged Dimensions"
@@ -142,28 +140,27 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8])
     outer_rt = ragged.from_row_splits(values=inner_rt, row_splits=[0, 3, 3, 5])
     self.assertEqual(outer_rt.ragged_rank, 2)
-    with self.test_session():
-      self.assertEqual(outer_rt.tolist(),
-                       [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
+    self.assertEqual(
+        self.evaluate(outer_rt).tolist(),
+        [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
     del inner_rt, outer_rt
 
     # From section: "Multiple Ragged Dimensions"
     rt = ragged.from_nested_row_splits(
         inner_values=[3, 1, 4, 1, 5, 9, 2, 6],
         nested_row_splits=([0, 3, 3, 5], [0, 4, 4, 7, 8, 8]))
-    with self.test_session():
-      self.assertEqual(rt.tolist(),
-                       [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
     del rt
 
     # From section: "Uniform Inner Dimensions"
     rt = ragged.from_row_splits(
         values=array_ops.ones([5, 3]), row_splits=[0, 2, 5])
-    with self.test_session():
-      self.assertEqual(
-          rt.tolist(),
-          [[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1], [1, 1, 1]]])
-      self.assertEqual(rt.shape.as_list(), [2, None, 3])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1], [1, 1, 1]]])
+    self.assertEqual(rt.shape.as_list(), [2, None, 3])
     del rt
 
   #=============================================================================
@@ -208,9 +205,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.RaggedTensor(
         values=values, row_splits=row_splits, internal=True)
 
-    with self.test_session():
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testRaggedTensorConstructionErrors(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -262,11 +259,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
-    with self.test_session():
-      self.assertAllEqual(rt_value_rowids, value_rowids)
-      self.assertEqual(self.evaluate(rt_nrows), 5)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertAllEqual(rt_value_rowids, value_rowids)
+    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromValueRowIdsWithDerivedNRowsDynamic(self):
     # nrows is not known at graph creation time.
@@ -285,11 +282,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
-    with self.test_session():
-      self.assertAllEqual(rt_value_rowids, value_rowids)
-      self.assertEqual(self.evaluate(rt_nrows), 5)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertAllEqual(rt_value_rowids, value_rowids)
+    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromValueRowIdsWithExplicitNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -308,10 +305,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     self.assertIs(rt_nrows, nrows)  # cached_nrows
-    with self.test_session():
-      self.assertEqual(
-          rt.tolist(),
-          [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g'], [], []])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g'], [], []])
 
   def testFromValueRowIdsWithExplicitNRowsEqualToDefault(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -330,11 +326,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     self.assertIs(rt_nrows, nrows)  # cached_nrows
-    with self.test_session():
-      self.assertAllEqual(rt_value_rowids, value_rowids)
-      self.assertAllEqual(rt_nrows, nrows)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertAllEqual(rt_value_rowids, value_rowids)
+    self.assertAllEqual(rt_nrows, nrows)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromValueRowIdsWithEmptyValues(self):
     rt = ragged.from_value_rowids([], [])
@@ -344,9 +340,8 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertEqual(rt.ragged_rank, 1)
     self.assertEqual(rt.values.shape.as_list(), [0])
     self.assertEqual(ragged.value_rowids(rt).shape.as_list(), [0])
-    with self.test_session():
-      self.assertEqual(self.evaluate(rt_nrows).tolist(), 0)
-      self.assertEqual(rt.tolist(), [])
+    self.assertEqual(self.evaluate(rt_nrows).tolist(), 0)
+    self.assertEqual(self.evaluate(rt).tolist(), [])
 
   def testFromRowSplits(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -363,10 +358,10 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_row_splits, row_splits)
-    with self.test_session():
-      self.assertEqual(self.evaluate(rt_nrows), 5)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromRowSplitsWithEmptySplits(self):
     err_msg = 'row_splits tensor may not be empty'
@@ -387,11 +382,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt_nrows = ragged.nrows(rt)
 
     self.assertIs(rt_values, values)
-    with self.test_session():
-      self.assertEqual(self.evaluate(rt_nrows), 5)
-      self.assertAllEqual(rt_row_starts, row_starts)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertAllEqual(rt_row_starts, row_starts)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromRowLimits(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -407,11 +402,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt_nrows = ragged.nrows(rt)
 
     self.assertIs(rt_values, values)
-    with self.test_session():
-      self.assertEqual(self.evaluate(rt_nrows), 5)
-      self.assertAllEqual(rt_row_limits, row_limits)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertAllEqual(rt_row_limits, row_limits)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromRowLengths(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -428,11 +423,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_row_lengths, row_lengths)  # cached_nrows
-    with self.test_session():
-      self.assertEqual(self.evaluate(rt_nrows), 5)
-      self.assertAllEqual(rt_row_lengths, row_lengths)
-      self.assertEqual(rt.tolist(),
-                       [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertAllEqual(rt_row_lengths, row_lengths)
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromNestedValueRowIdsWithDerivedNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -452,12 +447,11 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt_values_value_rowids = ragged.value_rowids(rt_values)
 
     self.assertIs(rt_values_values, values)
-    with self.test_session():
-      self.assertAllEqual(rt_value_rowids, nested_value_rowids[0])
-      self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1])
-      self.assertEqual(
-          rt.tolist(),
-          [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
+    self.assertAllEqual(rt_value_rowids, nested_value_rowids[0])
+    self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
 
   def testFromNestedValueRowIdsWithExplicitNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -483,14 +477,14 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt_values_nrows = ragged.nrows(rt_values)
 
     self.assertIs(rt_values_values, values)
-    with self.test_session():
-      self.assertAllEqual(rt_value_rowids, nested_value_rowids[0])
-      self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1])
-      self.assertAllEqual(rt_nrows, nrows[0])
-      self.assertAllEqual(rt_values_nrows, nrows[1])
-      self.assertEqual(rt.tolist(),
-                       [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [],
-                        [[b'f'], [b'g'], []], [], []])
+    self.assertAllEqual(rt_value_rowids, nested_value_rowids[0])
+    self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1])
+    self.assertAllEqual(rt_nrows, nrows[0])
+    self.assertAllEqual(rt_values_nrows, nrows[1])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g'], []], [],
+         []])
 
   def testFromNestedValueRowIdsWithExplicitNRowsMismatch(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -535,10 +529,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertIs(rt_values_values, inner_values)
     self.assertIs(rt_row_splits, nested_row_splits[0])
     self.assertIs(rt_values_row_splits, nested_row_splits[1])
-    with self.test_session():
-      self.assertEqual(
-          rt.tolist(),
-          [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
 
   def testFromNestedRowSplitsWithNonListInput(self):
     with self.assertRaisesRegexp(TypeError,
@@ -603,31 +596,31 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt2 = ragged.from_value_rowids(values, value_rowids)
 
     for rt in [rt1, rt2]:
-      with self.test_session():
-        self.assertEqual(rt.tolist(),
-                         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
-        self.assertEqual(
-            self.evaluate(rt.values).tolist(),
-            [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-        self.assertEqual(rt.values.shape.dims[0].value, 7)
-        self.assertEqual(
-            self.evaluate(ragged.value_rowids(rt)).tolist(),
-            [0, 0, 2, 2, 2, 3, 4])
-        self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
-        self.assertEqual(
-            self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
-        self.assertEqual(
-            self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
-        self.assertEqual(
-            self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
-        self.assertEqual(
-            self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
-        self.assertEqual(
-            self.evaluate(rt.inner_values).tolist(),
-            [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-        self.assertEqual(
-            [self.evaluate(s).tolist() for s in rt.nested_row_splits],
-            [[0, 2, 2, 5, 6, 7]])
+      self.assertEqual(
+          self.evaluate(rt).tolist(),
+          [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+      self.assertEqual(
+          self.evaluate(rt.values).tolist(),
+          [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+      self.assertEqual(rt.values.shape.dims[0].value, 7)
+      self.assertEqual(
+          self.evaluate(ragged.value_rowids(rt)).tolist(),
+          [0, 0, 2, 2, 2, 3, 4])
+      self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
+      self.assertEqual(
+          self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
+      self.assertEqual(
+          self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
+      self.assertEqual(
+          self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
+      self.assertEqual(
+          self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
+      self.assertEqual(
+          self.evaluate(rt.inner_values).tolist(),
+          [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+      self.assertEqual(
+          [self.evaluate(s).tolist() for s in rt.nested_row_splits],
+          [[0, 2, 2, 5, 6, 7]])
 
   def testRaggedTensorAccessors_3d_with_ragged_rank_1(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
@@ -637,32 +630,32 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt2 = ragged.from_value_rowids(values, value_rowids)
 
     for rt in [rt1, rt2]:
-      with self.test_session():
-        self.assertEqual(rt.tolist(),
-                         [[[0, 1], [2, 3]], [], [[4, 5], [6, 7], [8, 9]],
-                          [[10, 11]], [[12, 13]]])
-        self.assertEqual(
-            self.evaluate(rt.values).tolist(),
-            [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
-        self.assertEqual(rt.values.shape.dims[0].value, 7)
-        self.assertEqual(
-            self.evaluate(ragged.value_rowids(rt)).tolist(),
-            [0, 0, 2, 2, 2, 3, 4])
-        self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
-        self.assertEqual(
-            self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
-        self.assertEqual(
-            self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
-        self.assertEqual(
-            self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
-        self.assertEqual(
-            self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
-        self.assertEqual(
-            self.evaluate(rt.inner_values).tolist(),
-            [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
-        self.assertEqual(
-            [self.evaluate(s).tolist() for s in rt.nested_row_splits],
-            [[0, 2, 2, 5, 6, 7]])
+      self.assertEqual(
+          self.evaluate(rt).tolist(),
+          [[[0, 1], [2, 3]], [], [[4, 5], [6, 7], [8, 9]], [[10, 11]],
+           [[12, 13]]])
+      self.assertEqual(
+          self.evaluate(rt.values).tolist(),
+          [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
+      self.assertEqual(rt.values.shape.dims[0].value, 7)
+      self.assertEqual(
+          self.evaluate(ragged.value_rowids(rt)).tolist(),
+          [0, 0, 2, 2, 2, 3, 4])
+      self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
+      self.assertEqual(
+          self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
+      self.assertEqual(
+          self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
+      self.assertEqual(
+          self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
+      self.assertEqual(
+          self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
+      self.assertEqual(
+          self.evaluate(rt.inner_values).tolist(),
+          [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
+      self.assertEqual(
+          [self.evaluate(s).tolist() for s in rt.nested_row_splits],
+          [[0, 2, 2, 5, 6, 7]])
 
   def testRaggedTensorAccessors_3d_with_ragged_rank_2(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -678,42 +671,39 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt2 = ragged.from_nested_value_rowids(values, nested_value_rowids)
 
     for rt in [rt1, rt2]:
-      with self.test_session():
-        self.assertEqual(
-            rt.tolist(),
-            [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
-        self.assertEqual(
-            self.evaluate(rt.values).tolist(),
-            [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
-        self.assertEqual(rt.values.shape.dims[0].value, 5)
-        self.assertEqual(
-            self.evaluate(ragged.value_rowids(rt)).tolist(), [0, 0, 1, 3, 3])
-        self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 4)
-        self.assertEqual(self.evaluate(rt.row_splits).tolist(), [0, 2, 3, 3, 5])
-        self.assertEqual(
-            self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 3, 3])
-        self.assertEqual(
-            self.evaluate(ragged.row_limits(rt)).tolist(), [2, 3, 3, 5])
-        self.assertEqual(
-            self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 1, 0, 2])
-        self.assertEqual(
-            self.evaluate(rt.inner_values).tolist(),
-            [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-        self.assertEqual(
-            [self.evaluate(s).tolist() for s in rt.nested_row_splits],
-            [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]])
+      self.assertEqual(
+          self.evaluate(rt).tolist(),
+          [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
+      self.assertEqual(
+          self.evaluate(rt.values).tolist(),
+          [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+      self.assertEqual(rt.values.shape.dims[0].value, 5)
+      self.assertEqual(
+          self.evaluate(ragged.value_rowids(rt)).tolist(), [0, 0, 1, 3, 3])
+      self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 4)
+      self.assertEqual(self.evaluate(rt.row_splits).tolist(), [0, 2, 3, 3, 5])
+      self.assertEqual(
+          self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 3, 3])
+      self.assertEqual(
+          self.evaluate(ragged.row_limits(rt)).tolist(), [2, 3, 3, 5])
+      self.assertEqual(
+          self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 1, 0, 2])
+      self.assertEqual(
+          self.evaluate(rt.inner_values).tolist(),
+          [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+      self.assertEqual(
+          [self.evaluate(s).tolist() for s in rt.nested_row_splits],
+          [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]])
 
   def testNRowsWithTensorInput(self):
     dt = constant_op.constant([[1, 2, 3], [4, 5, 6]])
     nrows = ragged.nrows(dt)
-    with self.test_session():
-      self.assertEqual(self.evaluate(nrows), 2)
+    self.assertEqual(self.evaluate(nrows), 2)
 
   def testRowLengthsWithTensorInput(self):
     dt = constant_op.constant([[1, 2, 3], [4, 5, 6]])
     row_lengths = ragged.row_lengths(dt)
-    with self.test_session():
-      self.assertEqual(self.evaluate(row_lengths).tolist(), [3, 3])
+    self.assertEqual(self.evaluate(row_lengths).tolist(), [3, 3])
 
   #=============================================================================
   # RaggedTensor.shape
@@ -766,29 +756,27 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       expected: The expected value of rt.__getitem__(slice_spec), as a python
         list; or an exception class.
     """
-    with self.test_session():
-      tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True)
-      tensor_slice_spec2 = _make_tensor_slice_spec(slice_spec, False)
-      value1 = self.evaluate(rt.__getitem__(slice_spec))
-      value2 = self.evaluate(rt.__getitem__(tensor_slice_spec1))
-      value3 = self.evaluate(rt.__getitem__(tensor_slice_spec2))
-      if hasattr(value1, 'tolist'):
-        value1 = value1.tolist()
-      if hasattr(value2, 'tolist'):
-        value2 = value2.tolist()
-      if hasattr(value3, 'tolist'):
-        value3 = value3.tolist()
-      self.assertEqual(value1, expected, 'slice_spec=%s' % (slice_spec,))
-      self.assertEqual(value2, expected, 'slice_spec=%s' % (slice_spec,))
-      self.assertEqual(value3, expected, 'slice_spec=%s' % (slice_spec,))
+    tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True)
+    tensor_slice_spec2 = _make_tensor_slice_spec(slice_spec, False)
+    value1 = self.evaluate(rt.__getitem__(slice_spec))
+    value2 = self.evaluate(rt.__getitem__(tensor_slice_spec1))
+    value3 = self.evaluate(rt.__getitem__(tensor_slice_spec2))
+    if hasattr(value1, 'tolist'):
+      value1 = value1.tolist()
+    if hasattr(value2, 'tolist'):
+      value2 = value2.tolist()
+    if hasattr(value3, 'tolist'):
+      value3 = value3.tolist()
+    self.assertEqual(value1, expected, 'slice_spec=%s' % (slice_spec,))
+    self.assertEqual(value2, expected, 'slice_spec=%s' % (slice_spec,))
+    self.assertEqual(value3, expected, 'slice_spec=%s' % (slice_spec,))
 
   def _TestGetItemException(self, rt, slice_spec, expected, message):
     """Helper function for testing RaggedTensor.__getitem__ exceptions."""
-    with self.test_session():
-      tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True)
-      self.assertRaisesRegexp(expected, message, rt.__getitem__, slice_spec)
-      self.assertRaisesRegexp(expected, message, rt.__getitem__,
-                              tensor_slice_spec1)
+    tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True)
+    self.assertRaisesRegexp(expected, message, rt.__getitem__, slice_spec)
+    self.assertRaisesRegexp(expected, message, rt.__getitem__,
+                            tensor_slice_spec1)
 
   @parameterized.parameters(
       # Tests for rt[i]
@@ -860,8 +848,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES,
                                 EXAMPLE_RAGGED_TENSOR_2D_SPLITS)
 
-    with self.test_session():
-      self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_2D)
+    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D)
     self._TestGetItem(rt, slice_spec, expected)
 
   # pylint: disable=invalid-slice-index
@@ -905,8 +892,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     # if sys.version_info[0] == 3:
     #   message = 'must be str, not int'
 
-    with self.test_session():
-      self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_2D)
+    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D)
     self._TestGetItemException(rt, slice_spec, expected, message)
 
   @parameterized.parameters(
@@ -980,8 +966,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.from_nested_row_splits(
         EXAMPLE_RAGGED_TENSOR_4D_VALUES,
         [EXAMPLE_RAGGED_TENSOR_4D_SPLITS1, EXAMPLE_RAGGED_TENSOR_4D_SPLITS2])
-    with self.test_session():
-      self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_4D)
+    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_4D)
     self._TestGetItem(rt, slice_spec, expected)
 
   @parameterized.parameters(
@@ -1003,8 +988,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.from_nested_row_splits(
         EXAMPLE_RAGGED_TENSOR_4D_VALUES,
         [EXAMPLE_RAGGED_TENSOR_4D_SPLITS1, EXAMPLE_RAGGED_TENSOR_4D_SPLITS2])
-    with self.test_session():
-      self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_4D)
+    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_4D)
     self._TestGetItemException(rt, slice_spec, expected, message)
 
   @parameterized.parameters(
@@ -1044,8 +1028,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         EXAMPLE_RAGGED_TENSOR_2D_SPLITS, dtype=dtypes.int64)
     splits = array_ops.placeholder_with_default(splits, None)
     rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES, splits)
-    with self.test_session():
-      self.assertEqual(rt.tolist(), EXAMPLE_RAGGED_TENSOR_2D)
+    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D)
     self._TestGetItem(rt, slice_spec, expected)
 
   @parameterized.parameters(
@@ -1065,43 +1048,43 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     splits2 = [0, 2, 2, 3]
     values = constant_op.constant([['a', 'b'], ['c', 'd'], ['e', 'f']])
     rt = ragged.from_nested_row_splits(values, [splits1, splits2])
-    with self.test_session():
-      rt_newaxis0 = rt[array_ops.newaxis]
-      rt_newaxis1 = rt[:, array_ops.newaxis]
-      rt_newaxis2 = rt[:, :, array_ops.newaxis]
-      rt_newaxis3 = rt[:, :, :, array_ops.newaxis]
-      rt_newaxis4 = rt[:, :, :, :, array_ops.newaxis]
-
-      self.assertEqual(rt.tolist(),
-                       [[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []])
-      self.assertEqual(
-          rt_newaxis0.tolist(),
-          [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]])
-      self.assertEqual(
-          rt_newaxis1.tolist(),
-          [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]]], [[]]])
-      self.assertEqual(
-          rt_newaxis2.tolist(),
-          [[[[[b'a', b'b'], [b'c', b'd']]], [[]], [[[b'e', b'f']]]], []])
-      self.assertEqual(
-          rt_newaxis3.tolist(),
-          [[[[[b'a', b'b']], [[b'c', b'd']]], [], [[[b'e', b'f']]]], []])
-      self.assertEqual(
-          rt_newaxis4.tolist(),
-          [[[[[b'a'], [b'b']], [[b'c'], [b'd']]], [], [[[b'e'], [b'f']]]], []])
-
-      self.assertEqual(rt.ragged_rank, 2)
-      self.assertEqual(rt_newaxis0.ragged_rank, 3)
-      self.assertEqual(rt_newaxis1.ragged_rank, 3)
-      self.assertEqual(rt_newaxis2.ragged_rank, 3)
-      self.assertEqual(rt_newaxis3.ragged_rank, 2)
-      self.assertEqual(rt_newaxis4.ragged_rank, 2)
-
-      self.assertEqual(rt_newaxis0.shape.as_list(), [1, None, None, None, 2])
-      self.assertEqual(rt_newaxis1.shape.as_list(), [2, None, None, None, 2])
-      self.assertEqual(rt_newaxis2.shape.as_list(), [2, None, None, None, 2])
-      self.assertEqual(rt_newaxis3.shape.as_list(), [2, None, None, 1, 2])
-      self.assertEqual(rt_newaxis4.shape.as_list(), [2, None, None, 2, 1])
+    rt_newaxis0 = rt[array_ops.newaxis]
+    rt_newaxis1 = rt[:, array_ops.newaxis]
+    rt_newaxis2 = rt[:, :, array_ops.newaxis]
+    rt_newaxis3 = rt[:, :, :, array_ops.newaxis]
+    rt_newaxis4 = rt[:, :, :, :, array_ops.newaxis]
+
+    self.assertEqual(
+        self.evaluate(rt).tolist(),
+        [[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []])
+    self.assertEqual(
+        self.evaluate(rt_newaxis0).tolist(),
+        [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]])
+    self.assertEqual(
+        self.evaluate(rt_newaxis1).tolist(),
+        [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]]], [[]]])
+    self.assertEqual(
+        self.evaluate(rt_newaxis2).tolist(),
+        [[[[[b'a', b'b'], [b'c', b'd']]], [[]], [[[b'e', b'f']]]], []])
+    self.assertEqual(
+        self.evaluate(rt_newaxis3).tolist(),
+        [[[[[b'a', b'b']], [[b'c', b'd']]], [], [[[b'e', b'f']]]], []])
+    self.assertEqual(
+        self.evaluate(rt_newaxis4).tolist(),
+        [[[[[b'a'], [b'b']], [[b'c'], [b'd']]], [], [[[b'e'], [b'f']]]], []])
+
+    self.assertEqual(rt.ragged_rank, 2)
+    self.assertEqual(rt_newaxis0.ragged_rank, 3)
+    self.assertEqual(rt_newaxis1.ragged_rank, 3)
+    self.assertEqual(rt_newaxis2.ragged_rank, 3)
+    self.assertEqual(rt_newaxis3.ragged_rank, 2)
+    self.assertEqual(rt_newaxis4.ragged_rank, 2)
+
+    self.assertEqual(rt_newaxis0.shape.as_list(), [1, None, None, None, 2])
+    self.assertEqual(rt_newaxis1.shape.as_list(), [2, None, None, None, 2])
+    self.assertEqual(rt_newaxis2.shape.as_list(), [2, None, None, None, 2])
+    self.assertEqual(rt_newaxis3.shape.as_list(), [2, None, None, 1, 2])
+    self.assertEqual(rt_newaxis4.shape.as_list(), [2, None, None, 2, 1])
 
   #=============================================================================
   # RaggedTensor.__str__
@@ -1151,13 +1134,15 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt2_times_10 = rt2.with_inner_values(rt2.inner_values * 10)
     rt1_expanded = rt1.with_values(array_ops.expand_dims(rt1.values, axis=1))
 
-    with self.test_session():
-      self.assertEqual(rt1_plus_10.tolist(),
-                       [[11, 12], [13, 14, 15], [16], [], [17]])
-      self.assertEqual(rt2_times_10.tolist(),
-                       [[[10, 20], [30, 40, 50]], [[60]], [], [[], [70]]])
-      self.assertEqual(rt1_expanded.tolist(),
-                       [[[1], [2]], [[3], [4], [5]], [[6]], [], [[7]]])
+    self.assertEqual(
+        self.evaluate(rt1_plus_10).tolist(),
+        [[11, 12], [13, 14, 15], [16], [], [17]])
+    self.assertEqual(
+        self.evaluate(rt2_times_10).tolist(),
+        [[[10, 20], [30, 40, 50]], [[60]], [], [[], [70]]])
+    self.assertEqual(
+        self.evaluate(rt1_expanded).tolist(),
+        [[[1], [2]], [[3], [4], [5]], [[6]], [], [[7]]])
 
   #=============================================================================
   # Session.run
-- 
GitLab


From 59ebe545b3385c4c36d3b1602671d109e44ea38c Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 16 Nov 2018 16:20:05 -0800
Subject: [PATCH 0912/1554] Add CancellationManager to LoopCondOp

---
 tensorflow/core/kernels/control_flow_ops.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/core/kernels/control_flow_ops.cc b/tensorflow/core/kernels/control_flow_ops.cc
index 1587eb5114..21aabd9295 100644
--- a/tensorflow/core/kernels/control_flow_ops.cc
+++ b/tensorflow/core/kernels/control_flow_ops.cc
@@ -600,6 +600,15 @@ LoopCondOp::LoopCondOp(OpKernelConstruction* context) : OpKernel(context) {}
 LoopCondOp::~LoopCondOp() = default;
 
 void LoopCondOp::Compute(OpKernelContext* context) {
+  CancellationManager* cm = context->cancellation_manager();
+  bool already_cancelled = cm->IsCancelled();
+
+  if (already_cancelled) {
+    Tensor continue_running(false);
+    context->set_output(0, continue_running);
+    return;
+  }
+
   context->set_output(0, context->input(0));
 }
 
-- 
GitLab


From c0b128c45396560f26769a293525c60f76850a3f Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 16 Nov 2018 16:20:52 -0800
Subject: [PATCH 0913/1554] Add a python test for while_loop timeout

---
 .../python/kernel_tests/control_flow_ops_py_test.py    | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 37654abd18..595d4ff37b 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1988,6 +1988,16 @@ class ControlFlowTest(test.TestCase):
       for i in xrange(10):
         self.assertEqual([i], q.dequeue().eval())
 
+  def testWhileTimeOut(self):
+    run_options = config_pb2.RunOptions(timeout_in_ms=1)
+    with self.cached_session() as sess:
+      n = constant_op.constant(0)
+      c = lambda x: True
+      b = lambda x: math_ops.add(x, 1)
+      r = control_flow_ops.while_loop(c, b, [n])
+      with self.assertRaises(errors_impl.DeadlineExceededError):
+        sess.run(r, options=run_options)
+
   @test_util.disable_control_flow_v2("b/117119329 (stack)")
   def testWhileStack_1(self):
     with self.cached_session():
-- 
GitLab


From fc5392dedec126f988788a597edb55021fb07b60 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Tue, 20 Nov 2018 22:25:51 -0500
Subject: [PATCH 0914/1554] Raise an error when loop execution is cancelled

---
 tensorflow/core/kernels/control_flow_ops.cc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/control_flow_ops.cc b/tensorflow/core/kernels/control_flow_ops.cc
index 21aabd9295..61547adb73 100644
--- a/tensorflow/core/kernels/control_flow_ops.cc
+++ b/tensorflow/core/kernels/control_flow_ops.cc
@@ -602,12 +602,8 @@ LoopCondOp::~LoopCondOp() = default;
 void LoopCondOp::Compute(OpKernelContext* context) {
   CancellationManager* cm = context->cancellation_manager();
   bool already_cancelled = cm->IsCancelled();
-
-  if (already_cancelled) {
-    Tensor continue_running(false);
-    context->set_output(0, continue_running);
-    return;
-  }
+  OP_REQUIRES(context, !already_cancelled,
+              errors::Cancelled("Loop execution was cancelled."));
 
   context->set_output(0, context->input(0));
 }
-- 
GitLab


From 5bb0553a1b3bd580bd1502ab6b339ca3a1f0b5df Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Tue, 27 Nov 2018 22:05:43 -0800
Subject: [PATCH 0915/1554] Handle the case that CancellationManager is null in
 eager mode

---
 tensorflow/core/kernels/control_flow_ops.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/control_flow_ops.cc b/tensorflow/core/kernels/control_flow_ops.cc
index 61547adb73..b3bdff2575 100644
--- a/tensorflow/core/kernels/control_flow_ops.cc
+++ b/tensorflow/core/kernels/control_flow_ops.cc
@@ -601,9 +601,11 @@ LoopCondOp::~LoopCondOp() = default;
 
 void LoopCondOp::Compute(OpKernelContext* context) {
   CancellationManager* cm = context->cancellation_manager();
-  bool already_cancelled = cm->IsCancelled();
-  OP_REQUIRES(context, !already_cancelled,
-              errors::Cancelled("Loop execution was cancelled."));
+  if (cm != nullptr) {
+    bool already_cancelled = cm->IsCancelled();
+    OP_REQUIRES(context, !already_cancelled,
+                errors::Cancelled("Loop execution was cancelled."));
+  }
 
   context->set_output(0, context->input(0));
 }
-- 
GitLab


From 567c0692de57f0a6b2680b510aa731281a7081e0 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 27 Nov 2018 22:25:15 -0800
Subject: [PATCH 0916/1554] Implement some OptionalVariants functionality.

* ZerosLike variant registry function
* BinaryAdd variant registry function
* Shape variant registry function
* Enables copying nested variants between host/devices
* Refactors some common code from the corresponding TensorList
  functions. This should also enable TensorLists containing
  OptionalVariants (previously some of this code assumed any nested
  variant was also a TensorList).

PiperOrigin-RevId: 223112633
---
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/kernels/data/BUILD            |   5 +
 tensorflow/core/kernels/data/optional_ops.cc  | 104 ++++----------
 .../core/kernels/data/optional_ops.cu.cc      |  37 +++++
 tensorflow/core/kernels/data/optional_ops.h   | 119 ++++++++++++++++
 tensorflow/core/kernels/list_kernels.h        |  87 +-----------
 tensorflow/core/util/tensor_ops_util.h        | 128 ++++++++++++++++++
 .../python/data/kernel_tests/optional_test.py | 120 ++++++++++++++++
 8 files changed, 440 insertions(+), 161 deletions(-)
 create mode 100644 tensorflow/core/kernels/data/optional_ops.cu.cc
 create mode 100644 tensorflow/core/util/tensor_ops_util.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 20b1916e7d..9cd0edabdd 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -928,6 +928,7 @@ tf_cuda_library(
         "util/stream_executor_util.h",
         "util/strided_slice_op.h",
         "util/tensor_format.h",
+        "util/tensor_ops_util.h",
         "util/tensor_slice_reader.h",
         "util/tensor_slice_reader_cache.h",
         "util/tensor_slice_writer.h",
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index dcb6975669..f1eeda20d3 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -621,6 +621,10 @@ tf_kernel_library(
     name = "optional_ops",
     srcs = ["optional_ops.cc"],
     hdrs = ["optional_ops.h"],
+    gpu_srcs = [
+        "optional_ops.cu.cc",
+        "optional_ops.h",
+    ],
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
@@ -628,6 +632,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//third_party/eigen3",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/optional_ops.cc b/tensorflow/core/kernels/data/optional_ops.cc
index 2ab5c83082..bee857f53c 100644
--- a/tensorflow/core/kernels/data/optional_ops.cc
+++ b/tensorflow/core/kernels/data/optional_ops.cc
@@ -22,75 +22,6 @@ limitations under the License.
 namespace tensorflow {
 namespace data {
 namespace {
-const char kOptionalVariantTypeName[] = "tensorflow::data::Optional";
-
-// An `OptionalVariant` can represent either an "actual value" (a tuple of
-// tensors) or "none", and may be stored in a DT_VARIANT tensor.
-class OptionalVariant {
- public:
-  // Create an `OptionalVariant` with no actual value.
-  OptionalVariant() : values_(nullptr) {}
-
-  // Create an `OptionalVariant` with the actual value given by the tuple of
-  // tensors in `values`.
-  explicit OptionalVariant(std::vector<Tensor> values)
-      : values_(new std::vector<Tensor>(std::move(values))) {}
-
-  OptionalVariant(const OptionalVariant& other) : values_(other.values_) {}
-
-  // Returns true if `this` represents an actual value.
-  bool has_value() const { return values_ != nullptr; }
-
-  // REQUIRES: `this->has_value()` must be true.
-  const std::vector<Tensor>& get_values() const {
-    CHECK(values_) << "Tried to get values from an empty OptionalVariant";
-    return *values_;
-  }
-
-  // Implementations of the necessary methods for using `OptionalVariant`
-  // objects in DT_VARIANT tensors.
-  string TypeName() const { return kOptionalVariantTypeName; }
-  void Encode(VariantTensorData* data) const {
-    data->set_metadata(values_ != nullptr);
-    if (values_ != nullptr) {
-      for (const auto& t : *values_) {
-        *(data->add_tensors()) = t;
-      }
-    }
-  }
-
-  bool Decode(const VariantTensorData& data) {
-    if (data.type_name() != TypeName()) {
-      return false;
-    }
-    bool has_value = false;
-    if (!data.get_metadata(&has_value)) {
-      return false;
-    }
-    if (has_value) {
-      values_.reset(new std::vector<Tensor>(data.tensors()));
-    } else {
-      values_.reset();
-    }
-    return true;
-  }
-
-  string DebugString() const {
-    if (values_) {
-      return strings::StrCat("OptionalVariant<", "values: (",
-                             str_util::Join(*values_, ", ",
-                                            [](string* s, const Tensor& elem) {
-                                              *s = elem.DebugString();
-                                            }),
-                             ")>");
-    } else {
-      return strings::StrCat("OptionalVariant<None>");
-    }
-  }
-
- private:
-  std::shared_ptr<const std::vector<Tensor>> values_;
-};
 
 class OptionalNoneOp : public OpKernel {
  public:
@@ -143,6 +74,12 @@ class OptionalGetValueOp : public OpKernel {
   explicit OptionalGetValueOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES(
+        ctx, output_shapes_.size() == output_types_.size(),
+        errors::InvalidArgument(
+            "output_types and output_shapes must be same length, got:\n",
+            "output_types: ", output_types_.size(), "\n",
+            "output_shapes: ", output_shapes_.size()));
   }
 
   void Compute(OpKernelContext* ctx) override {
@@ -162,6 +99,10 @@ class OptionalGetValueOp : public OpKernel {
         ctx, optional->has_value(),
         errors::InvalidArgument("The given optional does not have a value."));
     const auto& components = optional->get_values();
+    OP_REQUIRES(ctx, components.size() == output_types_.size(),
+                errors::InvalidArgument(
+                    "The given optional has ", components.size(),
+                    " components, expected ", output_types_.size()));
     for (int i = 0; i < components.size(); ++i) {
       OP_REQUIRES(
           ctx, components[i].dtype() == output_types_[i],
@@ -213,15 +154,7 @@ static Status OptionalDeviceCopy(
     std::vector<Tensor> to_values;
     to_values.reserve(from_values.size());
     for (const Tensor& t : from_values) {
-      if (t.dtype() == DT_VARIANT) {
-        // TODO(b/116349787): Implement support for nested variants.
-        return errors::Unimplemented(
-            "Support for copying nested variants to device has not yet been "
-            "implemented.");
-      }
-    }
-    for (const Tensor& t : from_values) {
-      if (DMAHelper::CanUseDMA(&t)) {
+      if (DMAHelper::CanUseDMA(&t) || t.dtype() == DT_VARIANT) {
         Tensor tmp(t.dtype());
         TF_RETURN_IF_ERROR(copy(t, &tmp));
         to_values.push_back(std::move(tmp));
@@ -272,5 +205,20 @@ Status WriteOptionalNoneToOutput(OpKernelContext* ctx, int output_index) {
   return Status::OK();
 }
 
+REGISTER_UNARY_VARIANT_UNARY_OP_FUNCTION(ZEROS_LIKE_VARIANT_UNARY_OP,
+                                         DEVICE_CPU, OptionalVariant,
+                                         OptionalZerosLike<CPUDevice>);
+
+REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU,
+                                          OptionalVariant,
+                                          OptionalBinaryAdd<CPUDevice>);
+
+Status OptionalShape(const OptionalVariant& x, TensorShape* s) {
+  *s = TensorShape({});
+  return Status::OK();
+}
+
+REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(OptionalVariant, OptionalShape);
+
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/optional_ops.cu.cc b/tensorflow/core/kernels/data/optional_ops.cu.cc
new file mode 100644
index 0000000000..eb4a95a6f2
--- /dev/null
+++ b/tensorflow/core/kernels/data/optional_ops.cu.cc
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#define EIGEN_USE_THREADS
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/data/optional_ops.h"
+
+#include "tensorflow/core/framework/variant_op_registry.h"
+
+namespace tensorflow {
+namespace data {
+
+REGISTER_UNARY_VARIANT_UNARY_OP_FUNCTION(ZEROS_LIKE_VARIANT_UNARY_OP,
+                                         DEVICE_GPU, OptionalVariant,
+                                         OptionalZerosLike<GPUDevice>);
+
+REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_GPU,
+                                          OptionalVariant,
+                                          OptionalBinaryAdd<GPUDevice>);
+
+}  // namespace data
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/data/optional_ops.h b/tensorflow/core/kernels/data/optional_ops.h
index 2cbf2933f5..ef14e84311 100644
--- a/tensorflow/core/kernels/data/optional_ops.h
+++ b/tensorflow/core/kernels/data/optional_ops.h
@@ -19,10 +19,13 @@ limitations under the License.
 
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant_tensor_data.h"
+#include "tensorflow/core/util/tensor_ops_util.h"
 
 namespace tensorflow {
 namespace data {
 
+const char kOptionalVariantTypeName[] = "tensorflow::data::Optional";
+
 // Stores a DT_VARIANT value representing an Optional with the given value
 // in the `output_index`^th output of the given kernel execution context.
 Status WriteOptionalWithValueToOutput(OpKernelContext* ctx, int output_index,
@@ -32,6 +35,122 @@ Status WriteOptionalWithValueToOutput(OpKernelContext* ctx, int output_index,
 // in the `output_index`^th output of the given kernel execution context.
 Status WriteOptionalNoneToOutput(OpKernelContext* ctx, int output_index);
 
+// An `OptionalVariant` can represent either an "actual value" (a tuple of
+// tensors) or "none", and may be stored in a DT_VARIANT tensor.
+class OptionalVariant {
+ public:
+  // Create an `OptionalVariant` with no actual value.
+  OptionalVariant() : values_(nullptr) {}
+
+  // Create an `OptionalVariant` with the actual value given by the tuple of
+  // tensors in `values`.
+  explicit OptionalVariant(std::vector<Tensor> values)
+      : values_(new std::vector<Tensor>(std::move(values))) {}
+
+  OptionalVariant(const OptionalVariant& other) : values_(other.values_) {}
+
+  // Returns true if `this` represents an actual value.
+  bool has_value() const { return values_ != nullptr; }
+
+  // REQUIRES: `this->has_value()` must be true.
+  const std::vector<Tensor>& get_values() const {
+    DCHECK(values_) << "Tried to get values from an empty OptionalVariant";
+    return *values_;
+  }
+
+  // Implementations of the necessary methods for using `OptionalVariant`
+  // objects in DT_VARIANT tensors.
+  string TypeName() const { return kOptionalVariantTypeName; }
+  void Encode(VariantTensorData* data) const {
+    data->set_metadata(values_ != nullptr);
+    if (values_ != nullptr) {
+      for (const auto& t : *values_) {
+        *(data->add_tensors()) = t;
+      }
+    }
+  }
+
+  bool Decode(const VariantTensorData& data) {
+    if (data.type_name() != TypeName()) {
+      return false;
+    }
+    bool has_value = false;
+    if (!data.get_metadata(&has_value)) {
+      return false;
+    }
+    if (has_value) {
+      values_.reset(new std::vector<Tensor>(data.tensors()));
+    } else {
+      values_.reset();
+    }
+    return true;
+  }
+
+  string DebugString() const {
+    if (values_) {
+      return strings::StrCat("OptionalVariant<", "values: (",
+                             str_util::Join(*values_, ", ",
+                                            [](string* s, const Tensor& elem) {
+                                              *s = elem.DebugString();
+                                            }),
+                             ")>");
+    } else {
+      return strings::StrCat("OptionalVariant<None>");
+    }
+  }
+
+ private:
+  std::shared_ptr<const std::vector<Tensor>> values_;
+};
+
+template <typename Device>
+Status OptionalZerosLike(OpKernelContext* ctx, const OptionalVariant& x,
+                         OptionalVariant* y) {
+  if (!x.has_value()) {
+    *y = x;
+    return Status::OK();
+  }
+  std::vector<Tensor> zero_tensors;
+  for (const Tensor& tensor : x.get_values()) {
+    Tensor zero_t;
+    TF_RETURN_IF_ERROR(ZerosLikeTensor<Device>(ctx, tensor, &zero_t));
+    zero_tensors.push_back(std::move(zero_t));
+  }
+  *y = OptionalVariant(zero_tensors);
+  return Status::OK();
+}
+
+template <typename Device>
+Status OptionalBinaryAdd(OpKernelContext* ctx, const OptionalVariant& a,
+                         const OptionalVariant& b, OptionalVariant* out) {
+  // TODO(skyewm): should adding a value to a non-value be a no-op instead?
+  if (a.has_value() != b.has_value()) {
+    return errors::InvalidArgument(
+        "Cannot add optionals because one has a value and the other doesn't.");
+  }
+  if (!a.has_value()) {
+    *out = a;
+    return Status::OK();
+  }
+  if (a.get_values().size() != b.get_values().size()) {
+    return errors::InvalidArgument(
+        "Cannot add optionals because they have different numbers of "
+        "components (",
+        a.get_values().size(), " vs. ", b.get_values().size(), ").");
+  }
+  std::vector<Tensor> out_tensors;
+  for (int i = 0; i < a.get_values().size(); ++i) {
+    const Tensor& a_tensor = a.get_values()[i];
+    const Tensor& b_tensor = b.get_values()[i];
+    Tensor out_tensor;
+    TF_RETURN_IF_ERROR(
+        BinaryAddTensors<Device>(ctx, a_tensor, b_tensor, &out_tensor));
+    out_tensors.push_back(std::move(out_tensor));
+  }
+  *out = OptionalVariant(out_tensors);
+  return Status::OK();
+}
+
 }  // namespace data
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 75d91aff49..d4adc068a2 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/coding.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/util/tensor_ops_util.h"
 #include "tensorflow/core/util/util.h"
 
 namespace tensorflow {
@@ -357,51 +358,10 @@ Status TensorListBinaryAdd(OpKernelContext* c, const TensorList& a,
   for (int i = 0; i < a.tensors.size(); ++i) {
     const Tensor& a_tensor = a.tensors[i];
     const Tensor& b_tensor = b.tensors[i];
-    if (a_tensor.dtype() == DT_INVALID) {
-      out->tensors.push_back(b_tensor);
-      continue;
-    }
-    if (b_tensor.dtype() == DT_INVALID) {
-      out->tensors.push_back(a_tensor);
-      continue;
-    }
-    if (a_tensor.shape() != b_tensor.shape()) {
-      // TODO(apassos) support broadcasting additions here?
-      return errors::InvalidArgument(
-          "Trying to add two tensors with incompatible element shapes. "
-          "One is ",
-          a_tensor.shape().DebugString(), " and the other is ",
-          b_tensor.shape().DebugString(), " in position ", i);
-    }
     Tensor out_tensor;
-    AllocatorAttributes attr;
-    if (a_tensor.dtype() == DT_VARIANT) {
-      attr.set_on_host(true);
-    }
-    TF_RETURN_IF_ERROR(c->allocate_temp(a_tensor.dtype(), a_tensor.shape(),
-                                        &out_tensor, attr));
+    TF_RETURN_IF_ERROR(
+        BinaryAddTensors<Device>(c, a_tensor, b_tensor, &out_tensor));
     out->tensors.push_back(out_tensor);
-    switch (out_tensor.dtype()) {
-#define DTYPE_CASE(dtype)                                        \
-  case DataTypeToEnum<dtype>::value:                             \
-    out_tensor.flat<dtype>().device(c->eigen_device<Device>()) = \
-        a_tensor.flat<dtype>() + b_tensor.flat<dtype>();         \
-    break;
-
-      TF_CALL_NUMBER_TYPES(DTYPE_CASE)
-
-#undef DTYPE_CASE
-      case DataTypeToEnum<Variant>::value: {
-        Variant* v_out = &(out_tensor.scalar<Variant>()());
-        TF_RETURN_IF_ERROR(BinaryOpVariants<Device>(
-            c, ADD_VARIANT_BINARY_OP, a_tensor.scalar<Variant>()(),
-            b_tensor.scalar<Variant>()(), v_out));
-        break;
-      }
-      default:
-        return errors::InvalidArgument("Trying to add unsupported dtype ",
-                                       out_tensor.dtype());
-    }
   }
   return Status::OK();
 }
@@ -414,46 +374,7 @@ Status TensorListZerosLike(OpKernelContext* c, const TensorList& x,
   y->tensors.reserve(x.tensors.size());
   for (const Tensor& t : x.tensors) {
     Tensor out_tensor;
-    AllocatorAttributes attr;
-    if (t.dtype() == DT_VARIANT) {
-      attr.set_on_host(true);
-    }
-    TF_RETURN_IF_ERROR(
-        c->allocate_temp(t.dtype(), t.shape(), &out_tensor, attr));
-    switch (out_tensor.dtype()) {
-#define DTYPE_CASE(dtype)                                        \
-  case DataTypeToEnum<dtype>::value:                             \
-    out_tensor.flat<dtype>().device(c->eigen_device<Device>()) = \
-        out_tensor.flat<dtype>().constant(dtype(0));             \
-    break;
-
-      TF_CALL_POD_TYPES(DTYPE_CASE)
-
-#undef DTYPE_CASE
-
-      case DT_INVALID: {
-        // Uninitialized tensor in the TensorList.
-        out_tensor = Tensor(DT_INVALID);
-        break;
-      }
-      case DataTypeToEnum<Variant>::value: {
-        const TensorList* inner_x = t.scalar<Variant>()().get<TensorList>();
-        if (inner_x == nullptr) {
-          return errors::InvalidArgument("Input handle is not a list. Saw: '",
-                                         t.scalar<Variant>()().DebugString(),
-                                         "'");
-        }
-        TensorList inner_y;
-        TF_RETURN_IF_ERROR(TensorListZerosLike<Device>(c, *inner_x, &inner_y));
-        out_tensor.scalar<Variant>()() = std::move(inner_y);
-        break;
-      }
-
-      default:
-        return errors::InvalidArgument(
-            "Trying to compute zeros_like for unsupported dtype ",
-            DataTypeString(out_tensor.dtype()));
-    }
+    TF_RETURN_IF_ERROR(ZerosLikeTensor<Device>(c, t, &out_tensor));
     y->tensors.emplace_back(out_tensor);
   }
   return Status::OK();
diff --git a/tensorflow/core/util/tensor_ops_util.h b/tensorflow/core/util/tensor_ops_util.h
new file mode 100644
index 0000000000..615f088a9b
--- /dev/null
+++ b/tensorflow/core/util/tensor_ops_util.h
@@ -0,0 +1,128 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_UTIL_TENSOR_OPS_UTIL_H_
+#define TENSORFLOW_CORE_UTIL_TENSOR_OPS_UTIL_H_
+
+#define EIGEN_USE_THREADS
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+template <typename Device>
+Status ZerosLikeTensor(OpKernelContext* ctx, const Tensor& x, Tensor* out) {
+  AllocatorAttributes attr;
+  if (x.dtype() == DT_VARIANT) {
+    attr.set_on_host(true);
+  }
+  TF_RETURN_IF_ERROR(ctx->allocate_temp(x.dtype(), x.shape(), out, attr));
+
+  switch (out->dtype()) {
+#define DTYPE_CASE(dtype)                                       \
+  case DataTypeToEnum<dtype>::value:                            \
+    /* TODO(skyewm): use SetZeroFunctor like in ZerosLikeOp? */ \
+    out->flat<dtype>().device(ctx->eigen_device<Device>()) =    \
+        out->flat<dtype>().constant(dtype(0));                  \
+    break;
+
+    TF_CALL_POD_TYPES(DTYPE_CASE)
+#undef DTYPE_CASE
+
+    case DT_INVALID: {
+      *out = Tensor(DT_INVALID);
+      break;
+    }
+    case DataTypeToEnum<Variant>::value: {
+      Variant* out_variant = out->scalar<Variant>().data();
+      TF_RETURN_IF_ERROR(
+          UnaryOpVariant<Device>(ctx, ZEROS_LIKE_VARIANT_UNARY_OP,
+                                 x.scalar<Variant>()(), out_variant));
+      break;
+    }
+    default:
+      return errors::InvalidArgument(
+          "Trying to compute zeros_like for unsupported dtype ",
+          DataTypeString(out->dtype()));
+  }
+  return Status::OK();
+}
+
+template <typename Device>
+Status BinaryAddTensors(OpKernelContext* ctx, const Tensor& a, const Tensor& b,
+                        Tensor* out) {
+  if (a.dtype() == DT_INVALID) {
+    *out = b;
+    return Status::OK();
+  }
+  if (b.dtype() == DT_INVALID) {
+    *out = a;
+    return Status::OK();
+  }
+  if (a.dtype() != b.dtype()) {
+    return errors::InvalidArgument(
+        "Trying to add two tensors with incompatible element types. ",
+        "One is ", DataTypeString(a.dtype()), " and the other is ",
+        DataTypeString(b.dtype()));
+  }
+  if (a.shape() != b.shape()) {
+    // TODO(apassos) support broadcasting additions here?
+    return errors::InvalidArgument(
+        "Trying to add two tensors with incompatible element shapes. ",
+        "One is ", a.shape().DebugString(), " and the other is ",
+        b.shape().DebugString());
+  }
+
+  AllocatorAttributes attr;
+  if (a.dtype() == DT_VARIANT) {
+    attr.set_on_host(true);
+  }
+  TF_RETURN_IF_ERROR(ctx->allocate_temp(a.dtype(), a.shape(), out, attr));
+
+  switch (out->dtype()) {
+#define DTYPE_CASE(dtype)                                    \
+  case DataTypeToEnum<dtype>::value:                         \
+    out->flat<dtype>().device(ctx->eigen_device<Device>()) = \
+        a.flat<dtype>() + b.flat<dtype>();                   \
+    break;
+
+    TF_CALL_NUMBER_TYPES(DTYPE_CASE)
+#undef DTYPE_CASE
+
+    case DataTypeToEnum<Variant>::value: {
+      Variant* out_variant = out->scalar<Variant>().data();
+      TF_RETURN_IF_ERROR(BinaryOpVariants<Device>(
+          ctx, ADD_VARIANT_BINARY_OP, a.scalar<Variant>()(),
+          b.scalar<Variant>()(), out_variant));
+      break;
+    }
+    default:
+      return errors::InvalidArgument("Trying to add unsupported dtype ",
+                                     out->dtype());
+  }
+  return Status::OK();
+}
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_UTIL_TENSOR_OPS_UTIL_H_
diff --git a/tensorflow/python/data/kernel_tests/optional_test.py b/tensorflow/python/data/kernel_tests/optional_test.py
index 856985e9fd..864013171d 100644
--- a/tensorflow/python/data/kernel_tests/optional_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
@@ -87,6 +88,90 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(opt.get_value())
 
+  def testAddN(self):
+    devices = ["/cpu:0"]
+    if test_util.is_gpu_available():
+      devices.append("/gpu:0")
+    for device in devices:
+      with ops.device(device):
+        # With value
+        opt1 = optional_ops.Optional.from_value((1.0, 2.0))
+        opt2 = optional_ops.Optional.from_value((3.0, 4.0))
+
+        add_tensor = math_ops.add_n([opt1._variant_tensor,
+                                     opt2._variant_tensor])
+        add_opt = optional_ops._OptionalImpl(add_tensor, opt1.value_structure)
+        self.assertAllEqual(self.evaluate(add_opt.get_value()), (4.0, 6.0))
+
+        # Without value
+        opt_none1 = optional_ops.Optional.none_from_structure(
+            opt1.value_structure)
+        opt_none2 = optional_ops.Optional.none_from_structure(
+            opt2.value_structure)
+        add_tensor = math_ops.add_n([opt_none1._variant_tensor,
+                                     opt_none2._variant_tensor])
+        add_opt = optional_ops._OptionalImpl(add_tensor,
+                                             opt_none1.value_structure)
+        self.assertFalse(self.evaluate(add_opt.has_value()))
+
+  def testNestedAddN(self):
+    devices = ["/cpu:0"]
+    if test_util.is_gpu_available():
+      devices.append("/gpu:0")
+    for device in devices:
+      with ops.device(device):
+        opt1 = optional_ops.Optional.from_value([1, 2.0])
+        opt2 = optional_ops.Optional.from_value([3, 4.0])
+        opt3 = optional_ops.Optional.from_value((5.0, opt1._variant_tensor))
+        opt4 = optional_ops.Optional.from_value((6.0, opt2._variant_tensor))
+
+        add_tensor = math_ops.add_n([opt3._variant_tensor,
+                                     opt4._variant_tensor])
+        add_opt = optional_ops._OptionalImpl(add_tensor, opt3.value_structure)
+        self.assertEqual(self.evaluate(add_opt.get_value()[0]), 11.0)
+
+        inner_add_opt = optional_ops._OptionalImpl(add_opt.get_value()[1],
+                                                   opt1.value_structure)
+        self.assertAllEqual(inner_add_opt.get_value(), [4, 6.0])
+
+  def testZerosLike(self):
+    devices = ["/cpu:0"]
+    if test_util.is_gpu_available():
+      devices.append("/gpu:0")
+    for device in devices:
+      with ops.device(device):
+        # With value
+        opt = optional_ops.Optional.from_value((1.0, 2.0))
+        zeros_tensor = array_ops.zeros_like(opt._variant_tensor)
+        zeros_opt = optional_ops._OptionalImpl(zeros_tensor,
+                                               opt.value_structure)
+        self.assertAllEqual(self.evaluate(zeros_opt.get_value()),
+                            (0.0, 0.0))
+
+        # Without value
+        opt_none = optional_ops.Optional.none_from_structure(
+            opt.value_structure)
+        zeros_tensor = array_ops.zeros_like(opt_none._variant_tensor)
+        zeros_opt = optional_ops._OptionalImpl(zeros_tensor,
+                                               opt_none.value_structure)
+        self.assertFalse(self.evaluate(zeros_opt.has_value()))
+
+  def testNestedZerosLike(self):
+    devices = ["/cpu:0"]
+    if test_util.is_gpu_available():
+      devices.append("/gpu:0")
+    for device in devices:
+      with ops.device(device):
+        opt1 = optional_ops.Optional.from_value(1.0)
+        opt2 = optional_ops.Optional.from_value(opt1._variant_tensor)
+
+        zeros_tensor = array_ops.zeros_like(opt2._variant_tensor)
+        zeros_opt = optional_ops._OptionalImpl(zeros_tensor,
+                                               opt2.value_structure)
+        inner_zeros_opt = optional_ops._OptionalImpl(zeros_opt.get_value(),
+                                                     opt1.value_structure)
+        self.assertEqual(self.evaluate(inner_zeros_opt.get_value()), 0.0)
+
   def testCopyToGPU(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -116,6 +201,41 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
                      self.evaluate(gpu_optional_with_value_values))
     self.assertFalse(self.evaluate(gpu_optional_none_has_value))
 
+  def testNestedCopyToGPU(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    with ops.device("/cpu:0"):
+      optional_with_value = optional_ops.Optional.from_value(
+          (constant_op.constant(37.0), constant_op.constant("Foo"),
+           constant_op.constant(42)))
+      optional_none = optional_ops.Optional.none_from_structure(
+          structure.TensorStructure(dtypes.float32, []))
+      nested_optional = optional_ops.Optional.from_value(
+          (optional_with_value._variant_tensor, optional_none._variant_tensor,
+           1.0))
+
+    with ops.device("/gpu:0"):
+      gpu_nested_optional = optional_ops._OptionalImpl(
+          array_ops.identity(nested_optional._variant_tensor),
+          nested_optional.value_structure)
+
+      gpu_nested_optional_has_value = gpu_nested_optional.has_value()
+      gpu_nested_optional_values = gpu_nested_optional.get_value()
+
+    self.assertTrue(self.evaluate(gpu_nested_optional_has_value))
+
+    inner_with_value = optional_ops._OptionalImpl(
+        gpu_nested_optional_values[0], optional_with_value.value_structure)
+
+    inner_none = optional_ops._OptionalImpl(
+        gpu_nested_optional_values[1], optional_none.value_structure)
+
+    self.assertEqual((37.0, b"Foo", 42),
+                     self.evaluate(inner_with_value.get_value()))
+    self.assertFalse(self.evaluate(inner_none.has_value()))
+    self.assertEqual(1.0, self.evaluate(gpu_nested_optional_values[2]))
+
   def _assertElementValueEqual(self, expected, actual):
     if isinstance(expected, dict):
       self.assertItemsEqual(list(expected.keys()), list(actual.keys()))
-- 
GitLab


From 95e808ba44075dfe0b7db57bb49d2e64a1977a95 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 23:53:33 -0800
Subject: [PATCH 0917/1554] Replace `tf.estimator.inputs` with
 `tf.compat.v1.estimator.inputs`

PiperOrigin-RevId: 223118522
---
 tensorflow/examples/learn/iris_custom_decay_dnn.py | 4 ++--
 tensorflow/examples/learn/iris_custom_model.py     | 4 ++--
 tensorflow/examples/tutorials/layers/cnn_mnist.py  | 9 +++------
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/tensorflow/examples/learn/iris_custom_decay_dnn.py b/tensorflow/examples/learn/iris_custom_decay_dnn.py
index 4a219694d1..73bf20fada 100644
--- a/tensorflow/examples/learn/iris_custom_decay_dnn.py
+++ b/tensorflow/examples/learn/iris_custom_decay_dnn.py
@@ -76,12 +76,12 @@ def main(unused_argv):
   classifier = tf.estimator.Estimator(model_fn=my_model)
 
   # Train.
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+  train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True)
   classifier.train(input_fn=train_input_fn, steps=1000)
 
   # Predict.
-  test_input_fn = tf.estimator.inputs.numpy_input_fn(
+  test_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False)
   predictions = classifier.predict(input_fn=test_input_fn)
   y_predicted = np.array(list(p['class'] for p in predictions))
diff --git a/tensorflow/examples/learn/iris_custom_model.py b/tensorflow/examples/learn/iris_custom_model.py
index c6bdb86ba5..bf34d72ba0 100644
--- a/tensorflow/examples/learn/iris_custom_model.py
+++ b/tensorflow/examples/learn/iris_custom_model.py
@@ -73,12 +73,12 @@ def main(unused_argv):
   classifier = tf.estimator.Estimator(model_fn=my_model)
 
   # Train.
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+  train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True)
   classifier.train(input_fn=train_input_fn, steps=1000)
 
   # Predict.
-  test_input_fn = tf.estimator.inputs.numpy_input_fn(
+  test_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False)
   predictions = classifier.predict(input_fn=test_input_fn)
   y_predicted = np.array(list(p['class'] for p in predictions))
diff --git a/tensorflow/examples/tutorials/layers/cnn_mnist.py b/tensorflow/examples/tutorials/layers/cnn_mnist.py
index 1e8d7d05e1..670e929236 100644
--- a/tensorflow/examples/tutorials/layers/cnn_mnist.py
+++ b/tensorflow/examples/tutorials/layers/cnn_mnist.py
@@ -134,7 +134,7 @@ def main(unused_argv):
       tensors=tensors_to_log, every_n_iter=50)
 
   # Train the model
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+  train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={"x": train_data},
       y=train_labels,
       batch_size=100,
@@ -146,11 +146,8 @@ def main(unused_argv):
       hooks=[logging_hook])
 
   # Evaluate the model and print results
-  eval_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={"x": eval_data},
-      y=eval_labels,
-      num_epochs=1,
-      shuffle=False)
+  eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
+      x={"x": eval_data}, y=eval_labels, num_epochs=1, shuffle=False)
   eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
   print(eval_results)
 
-- 
GitLab


From a99ef3499273abc701441b31819235d85a8bb5e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Nov 2018 23:57:17 -0800
Subject: [PATCH 0918/1554] Replace `tf.estimator.inputs` with
 `tf.compat.v1.estimator.inputs`

PiperOrigin-RevId: 223118818
---
 tensorflow/contrib/boosted_trees/examples/boston.py          | 4 ++--
 tensorflow/contrib/boosted_trees/examples/boston_combined.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py
index 54c4ff059e..09b240a700 100644
--- a/tensorflow/contrib/boosted_trees/examples/boston.py
+++ b/tensorflow/contrib/boosted_trees/examples/boston.py
@@ -90,13 +90,13 @@ def _make_experiment_fn(output_dir):
   (x_train, y_train), (x_test,
                        y_test) = tf.keras.datasets.boston_housing.load_data()
 
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+  train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={"x": x_train},
       y=y_train,
       batch_size=FLAGS.batch_size,
       num_epochs=None,
       shuffle=True)
-  eval_input_fn = tf.estimator.inputs.numpy_input_fn(
+  eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False)
 
   feature_columns = [
diff --git a/tensorflow/contrib/boosted_trees/examples/boston_combined.py b/tensorflow/contrib/boosted_trees/examples/boston_combined.py
index e04b56afbf..d640af354f 100644
--- a/tensorflow/contrib/boosted_trees/examples/boston_combined.py
+++ b/tensorflow/contrib/boosted_trees/examples/boston_combined.py
@@ -80,13 +80,13 @@ def _make_experiment_fn(output_dir):
   (x_train, y_train), (x_test,
                        y_test) = tf.keras.datasets.boston_housing.load_data()
 
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+  train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={"x": x_train},
       y=y_train,
       batch_size=FLAGS.batch_size,
       num_epochs=None,
       shuffle=True)
-  eval_input_fn = tf.estimator.inputs.numpy_input_fn(
+  eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
       x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False)
 
   feature_columns = [
-- 
GitLab


From e5130baf339e9b820178693ff371085d90edcd41 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Wed, 28 Nov 2018 00:03:24 -0800
Subject: [PATCH 0919/1554] Remove summary recording context managers from 2.0
 API

PiperOrigin-RevId: 223119687
---
 tensorflow/contrib/summary/summary.py                  |  1 -
 tensorflow/python/ops/summary_ops_v2.py                | 10 ++++------
 .../tools/api/golden/v2/tensorflow.summary.pbtxt       |  8 --------
 3 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py
index 605625c305..42898e797c 100644
--- a/tensorflow/contrib/summary/summary.py
+++ b/tensorflow/contrib/summary/summary.py
@@ -79,7 +79,6 @@ from tensorflow.python.ops.summary_ops_v2 import image
 from tensorflow.python.ops.summary_ops_v2 import import_event
 from tensorflow.python.ops.summary_ops_v2 import initialize
 from tensorflow.python.ops.summary_ops_v2 import never_record_summaries
-from tensorflow.python.ops.summary_ops_v2 import record_summaries
 from tensorflow.python.ops.summary_ops_v2 import record_summaries_every_n_global_steps
 from tensorflow.python.ops.summary_ops_v2 import scalar
 from tensorflow.python.ops.summary_ops_v2 import should_record_summaries
diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py
index a0ad43b444..3f99b9f877 100644
--- a/tensorflow/python/ops/summary_ops_v2.py
+++ b/tensorflow/python/ops/summary_ops_v2.py
@@ -58,7 +58,6 @@ _RUN_NAME_PATTERNS = re.compile(r"^[^\x00-\x1F<>]{0,512}$")
 _USER_NAME_PATTERNS = re.compile(r"^[a-z]([-a-z0-9]{0,29}[a-z0-9])?$", re.I)
 
 
-@tf_export("summary.should_record_summaries", v1=[])
 def should_record_summaries():
   """Returns boolean Tensor which is true if summaries should be recorded."""
   global _SHOULD_RECORD_SUMMARIES
@@ -67,9 +66,8 @@ def should_record_summaries():
   return should() if callable(should) else should
 
 
-@tf_export("summary.record_summaries", v1=[])
 @tf_contextlib.contextmanager
-def record_summaries(boolean=True):
+def _record_summaries(boolean=True):
   """Sets summary recording on or off per the provided boolean value.
 
   The provided value can be a python boolean, a scalar boolean Tensor, or
@@ -105,17 +103,17 @@ def record_summaries_every_n_global_steps(n, global_step=None):
     should = lambda: math_ops.equal(global_step % n, 0)
     if not context.executing_eagerly():
       should = should()
-  return record_summaries(should)
+  return _record_summaries(should)
 
 
 def always_record_summaries():
   """Sets the should_record_summaries Tensor to always true."""
-  return record_summaries(True)
+  return _record_summaries(True)
 
 
 def never_record_summaries():
   """Sets the should_record_summaries Tensor to always false."""
-  return record_summaries(False)
+  return _record_summaries(False)
 
 
 @tf_export("summary.SummaryWriter", v1=[])
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
index 26c979c0c6..42a74a65fb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
@@ -44,12 +44,4 @@ tf_module {
     name: "import_event"
     argspec: "args=[\'tensor\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "record_summaries"
-    argspec: "args=[\'boolean\'], varargs=None, keywords=None, defaults=[\'True\'], "
-  }
-  member_method {
-    name: "should_record_summaries"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
 }
-- 
GitLab


From daff4a0cb32c28b4e2bfdf72fe961e5b8235146e Mon Sep 17 00:00:00 2001
From: Benjamin Tan Wei Hao <benjamintanweihao@gmail.com>
Date: Wed, 28 Nov 2018 16:45:43 +0800
Subject: [PATCH 0920/1554] Fix typo: s/dimemension/dimension/

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index cb2a1ca87a..c72b5c1c41 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -181,7 +181,7 @@ Status ValidateTensorProperties(const string& producer_node_type,
     if (shape.dim_size(d) < 0) {
       return errors::InvalidArgument(
           "Input tensor with shape ", shape.DebugString(),
-          " has an unknown non-batch dimemension at dim ", d);
+          " has an unknown non-batch dimension at dim ", d);
     }
   }
   return Status::OK();
-- 
GitLab


From 5e4982f018b6704d727ae30e4458a7d281572039 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 01:02:11 -0800
Subject: [PATCH 0921/1554] compat: Update forward compatibility horizon to
 2018-11-28

PiperOrigin-RevId: 223125260
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index cb359cd084..1f43793962 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 27)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 28)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From b093a6d05ba501a6cfa7c9cfd8d85863477640fc Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 28 Nov 2018 02:45:18 -0800
Subject: [PATCH 0922/1554] Add more test coverage over get_variable

PiperOrigin-RevId: 223136044
---
 .../kernel_tests/variable_scope_test.py       | 22 +++++++++++++++++++
 tensorflow/python/ops/variable_scope.py       |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 37012af299..3720f736ac 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -1312,6 +1312,28 @@ class VariableScopeTest(test.TestCase):
     # Ensure it is possible to do get_variable with a _ref dtype passed in.
     _ = variable_scope.get_variable("w", shape=[5, 6], dtype=v.dtype)
 
+  @test_util.run_in_graph_and_eager_modes
+  @run_inside_wrap_function_in_eager_mode
+  def testGetVariableWithInitializerWhichTakesNoArgs(self):
+    v = variable_scope.get_variable("foo", initializer=lambda: [2])
+    self.assertEqual(v.name, "foo:0")
+
+  @test_util.run_in_graph_and_eager_modes
+  @run_inside_wrap_function_in_eager_mode
+  def testGetVariableWithInitializerWhichTakesOptionalArgs(self):
+    v = variable_scope.get_variable("foo", initializer=lambda x=True: [2])
+    self.assertEqual(v.name, "foo:0")
+
+  @test_util.run_in_graph_and_eager_modes
+  @run_inside_wrap_function_in_eager_mode
+  def testGetVariableWithInitializerWhichTakesUnprovidedArgsAndNoShape(self):
+    with self.assertRaisesRegexp(
+        ValueError,
+        "The initializer passed is not valid. It should be a callable with no "
+        "arguments and the shape should not be provided or an instance of "
+        "`tf.keras.initializers.*' and `shape` should be fully defined."):
+      variable_scope.get_variable("foo", initializer=lambda x: [2])
+
   @test_util.run_in_graph_and_eager_modes
   @run_inside_wrap_function_in_eager_mode
   def testTwoGraphs(self):
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 87ef046100..ad81862a58 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -909,12 +909,12 @@ class _VariableStore(object):
         # Instantiate initializer if provided initializer is a type object.
         if tf_inspect.isclass(initializer):
           initializer = initializer(dtype=dtype)
-        spec = tf_inspect.getargspec(initializer)
         if shape is not None and shape.is_fully_defined():
           init_val = lambda: initializer(  # pylint: disable=g-long-lambda
               shape.as_list(), dtype=dtype, partition_info=partition_info)
           variable_dtype = dtype.base_dtype
-        elif len(spec.args) == len(spec.defaults or []):
+        elif len(tf_inspect.getargspec(initializer).args) == len(
+            tf_inspect.getargspec(initializer).defaults or []):
           init_val = initializer
           variable_dtype = None
         else:
-- 
GitLab


From 3f64581ca9022df350e4ae51b34e1a8352858771 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Wed, 28 Nov 2018 03:06:47 -0800
Subject: [PATCH 0923/1554] [TF:XLA] Zero pad or truncate the fft axes for rfft

This matches the tf docs. Previously this was an error.

PiperOrigin-RevId: 223138348
---
 tensorflow/compiler/tests/fft_test.py         | 41 +++++++++++++++++++
 tensorflow/compiler/tf2xla/kernels/fft_ops.cc | 28 ++++++++++++-
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/fft_test.py b/tensorflow/compiler/tests/fft_test.py
index 61abf9c9c0..0edd0c35aa 100644
--- a/tensorflow/compiler/tests/fft_test.py
+++ b/tensorflow/compiler/tests/fft_test.py
@@ -158,6 +158,23 @@ class FFTTest(xla_test.XLATestCase):
 
     self._VerifyFftMethod(INNER_DIMS_3D, np.real, _to_expected, _tf_fn)
 
+  def testRFFT3DMismatchedSize(self):
+
+    def _to_expected(x):
+      return np.fft.rfftn(
+          x,
+          axes=(-3, -2, -1),
+          s=[x.shape[-3] // 2, x.shape[-2], x.shape[-1] * 2])
+
+    def _tf_fn(x):
+      return signal.rfft3d(
+          x,
+          fft_length=[
+              x.shape[-3].value // 2, x.shape[-2].value, x.shape[-1].value * 2
+          ])
+
+    self._VerifyFftMethod(INNER_DIMS_3D, np.real, _to_expected, _tf_fn)
+
   def testIRFFT(self):
 
     def _tf_fn(x):
@@ -202,6 +219,30 @@ class FFTTest(xla_test.XLATestCase):
 
     self._VerifyFftMethod(INNER_DIMS_3D, _to_input, _to_expected, _tf_fn)
 
+  def testIRFFT3DMismatchedSize(self):
+
+    def _to_input(x):
+      return np.fft.rfftn(
+          np.real(x),
+          axes=(-3, -2, -1),
+          s=[x.shape[-3] // 2, x.shape[-2], x.shape[-1] * 2])
+
+    def _to_expected(x):
+      return np.fft.irfftn(
+          x,
+          axes=(-3, -2, -1),
+          s=[x.shape[-3] // 2, x.shape[-2], x.shape[-1] * 2])
+
+    def _tf_fn(x):
+      return signal.irfft3d(
+          x,
+          fft_length=[
+              x.shape[-3].value // 2, x.shape[-2].value, x.shape[-1].value * 2
+          ])
+
+    self._VerifyFftMethod(INNER_DIMS_3D, _to_input, _to_expected, _tf_fn)
+
+
 
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
index 9b06357d9b..6df8b5367d 100644
--- a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -50,11 +51,36 @@ class GenericFftOp : public XlaOpKernel {
         errors::InvalidArgument("input must be at least 1 dimensional"));
 
     std::vector<int64> fft_length;
+    xla::XlaOp input = ctx->Input(0);
     if (fft_type_ == FftType::RFFT || fft_type_ == FftType::IRFFT) {
       OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &fft_length));
       OP_REQUIRES(ctx, fft_length.size() == fft_rank_,
                   errors::InvalidArgument("fft_length must be length ",
                                           fft_rank_, " vector"));
+
+      // Zero pad or truncate the axes we're doing FFT on.
+      absl::InlinedVector<int64, 4> slice_sizes = input_shape.dim_sizes();
+      std::vector<std::pair<int64, int64>> padding_sizes(slice_sizes.size());
+      std::vector<int64> expected_sizes = fft_length;
+      // IRFFT wants the innermost axis to be n / 2 + 1.
+      if (fft_type_ == FftType::IRFFT) {
+        expected_sizes[fft_rank_ - 1] = fft_length[fft_rank_ - 1] / 2 + 1;
+      }
+      for (int i = 0; i < fft_rank_; i++) {
+        int index = input_shape.dims() - fft_rank_ + i;
+        if (input_shape.dim_size(index) > expected_sizes[i]) {
+          slice_sizes[index] = expected_sizes[i];
+        } else {
+          padding_sizes[index].second =
+              expected_sizes[i] - input_shape.dim_size(index);
+        }
+      }
+
+      std::vector<int64> start_indices(input_shape.dims(), 0);
+      std::vector<int64> strides(input_shape.dims(), 1);
+      input = xla::Pad(xla::Slice(input, start_indices, slice_sizes, strides),
+                       XlaHelpers::Zero(ctx->builder(), ctx->input_type(0)),
+                       xla::MakeEdgePaddingConfig(padding_sizes));
     } else {
       // Innermost axis provides the FFT length.
       for (int i = 0; i < fft_rank_; i++) {
@@ -63,7 +89,7 @@ class GenericFftOp : public XlaOpKernel {
       }
     }
 
-    xla::XlaOp fft = xla::Fft(ctx->Input(0), fft_type_, fft_length);
+    xla::XlaOp fft = xla::Fft(input, fft_type_, fft_length);
     ctx->SetOutput(0, fft);
   }
 
-- 
GitLab


From d64dfd97e0ba3b46f69bfeb54cd868fb2ac340df Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Wed, 28 Nov 2018 03:55:55 -0800
Subject: [PATCH 0924/1554] Use internal stack trace library in registry

This change is functionally a no-op. Instead of using the native traceback library, we use tf_stack.extract_stack() to get the frame we care about and tf_stack.convert_stack() to convert this to the same format.

This reduces the number of file posix.stat calls. In my example application, we reduce the time spent in register() by a factor of 15.

We also add a test to verify that we're extracting the same information.

PiperOrigin-RevId: 223142289
---
 tensorflow/python/framework/registry.py      | 11 ++++++-----
 tensorflow/python/framework/registry_test.py |  4 +++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/framework/registry.py b/tensorflow/python/framework/registry.py
index 2e45acb499..4357c76bd6 100644
--- a/tensorflow/python/framework/registry.py
+++ b/tensorflow/python/framework/registry.py
@@ -23,10 +23,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import traceback
-
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
+from tensorflow.python.util import tf_stack
 
 
 # Registry mechanism below is based on mapreduce.python.mrpython.Register.
@@ -57,15 +56,17 @@ class Registry(object):
     if name in self._registry:
       (filename, line_number, function_name, _) = (
           self._registry[name][_LOCATION_TAG])
-      raise KeyError("Registering two %s with name '%s' !"
+      raise KeyError("Registering two %s with name '%s'! "
                      "(Previous registration was in %s %s:%d)" %
                      (self._name, name, function_name, filename, line_number))
 
     logging.vlog(1, "Registering %s (%s) in %s.", name, candidate, self._name)
     # stack trace is [this_function, Register(), user_function,...]
     # so the user function is #2.
-    stack = traceback.extract_stack()
-    self._registry[name] = {_TYPE_TAG: candidate, _LOCATION_TAG: stack[2]}
+    stack = tf_stack.extract_stack()
+    user_function = stack[2]
+    location_tag = tf_stack.convert_stack([user_function])[0]
+    self._registry[name] = {_TYPE_TAG: candidate, _LOCATION_TAG: location_tag}
 
   def list(self):
     """Lists registered items.
diff --git a/tensorflow/python/framework/registry_test.py b/tensorflow/python/framework/registry_test.py
index a821e16f26..1a0d3f200d 100644
--- a/tensorflow/python/framework/registry_test.py
+++ b/tensorflow/python/framework/registry_test.py
@@ -45,7 +45,9 @@ class RegistryTest(test.TestCase):
   def testDuplicate(self):
     myreg = registry.Registry('testbar')
     myreg.register(bar, 'Bar')
-    with self.assertRaises(KeyError):
+    with self.assertRaisesRegexp(
+        KeyError, r'Registering two testbar with name \'Bar\'! '
+        r'\(Previous registration was in [^ ]+ .*.py:[0-9]+\)'):
       myreg.register(bar, 'Bar')
 
 
-- 
GitLab


From 192d588eaf6e37c98fd7583cad19c2d1e55fbb67 Mon Sep 17 00:00:00 2001
From: Peter Buchlovsky <petebu@google.com>
Date: Wed, 28 Nov 2018 07:30:45 -0800
Subject: [PATCH 0925/1554] Fix incompatible device colocation warning.

PiperOrigin-RevId: 223163333
---
 .../contrib/distribute/python/mirrored_strategy.py       | 4 +++-
 .../contrib/distribute/python/one_device_strategy.py     | 9 +++++++--
 .../distribute/python/parameter_server_strategy.py       | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 7719715875..4a594f056e 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import functools
 
+from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import mirrored_strategy
 from tensorflow.python.distribute import values
@@ -133,7 +134,8 @@ class MirroredExtended(CoreMirroredExtended):
     if self._cluster_spec:
       worker_device_pairs = self._worker_devices
     else:
-      worker_device_pairs = [("/job:localhost", self._devices)]
+      worker = device_util.canonicalize("/device:CPU:0")
+      worker_device_pairs = [(worker, self._devices)]
     return values.DatasetIterator(dataset, worker_device_pairs)
 
   def _distribute_dataset(self, dataset_fn):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 7a2e225ca0..e322b6acb8 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import six
 
+from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import values
 from tensorflow.python.framework import constant_op
@@ -68,7 +69,9 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
 
   def _make_dataset_iterator(self, dataset):
     """Make iterator from dataset without splitting the batch."""
-    return values.DatasetIterator(dataset, [("/job:localhost", [self._device])])
+    worker = device_util.canonicalize("/device:CPU:0")
+    worker_device_pairs = [(worker, [self._device])]
+    return values.DatasetIterator(dataset, worker_device_pairs)
 
   def _distribute_dataset(self, dataset_fn):
     return values.PerReplicaDataset(
@@ -78,8 +81,10 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
       self,
       input_fn,
       replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    worker = device_util.canonicalize("/device:CPU:0")
+    worker_device_pairs = [(worker, [self._device])]
     return values.InputFunctionIterator(
-        input_fn, [("/job:localhost", [self._device])],
+        input_fn, worker_device_pairs,
         [distribute_lib.InputContext()])
 
   def _broadcast_to(self, tensor, destinations):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 8c33d3a143..75ee41c4cf 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -199,7 +199,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
 
   def _initialize_local(self, num_gpus_per_worker):
     """Initialize internal devices for local training."""
-    self._worker_device = "/job:localhost"
+    self._worker_device = device_util.canonicalize("/device:CPU:0")
     # Define compute devices which is a list of device strings and one for each
     # replica. When there are GPUs, replicate operations on these GPUs.
     # Otherwise, place operations on CPU.
-- 
GitLab


From 36a445c3531e90948d6de34af2af59e34f361b86 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 28 Nov 2018 08:29:03 -0800
Subject: [PATCH 0926/1554] Deprecate tf.train._ classes in TF 2.0 API in favor
 of tf.keras.optimizers._ ones.

PiperOrigin-RevId: 223171873
---
 tensorflow/python/training/adadelta.py        |  2 +-
 tensorflow/python/training/adagrad.py         |  2 +-
 tensorflow/python/training/adagrad_da.py      |  2 +-
 tensorflow/python/training/adam.py            |  2 +-
 tensorflow/python/training/ftrl.py            |  2 +-
 .../python/training/gradient_descent.py       |  2 +-
 tensorflow/python/training/momentum.py        |  2 +-
 tensorflow/python/training/optimizer.py       |  2 +-
 .../python/training/proximal_adagrad.py       |  2 +-
 tensorflow/python/training/rmsprop.py         |  2 +-
 ...tensorflow.train.-adadelta-optimizer.pbtxt | 51 -------------------
 ...sorflow.train.-adagrad-d-a-optimizer.pbtxt | 51 -------------------
 .../tensorflow.train.-adagrad-optimizer.pbtxt | 51 -------------------
 .../v2/tensorflow.train.-adam-optimizer.pbtxt | 51 -------------------
 .../v2/tensorflow.train.-ftrl-optimizer.pbtxt | 51 -------------------
 ...ow.train.-gradient-descent-optimizer.pbtxt | 51 -------------------
 ...tensorflow.train.-momentum-optimizer.pbtxt | 51 -------------------
 .../v2/tensorflow.train.-optimizer.pbtxt      | 50 ------------------
 ...ow.train.-proximal-adagrad-optimizer.pbtxt | 51 -------------------
 ...nsorflow.train.-r-m-s-prop-optimizer.pbtxt | 51 -------------------
 .../api/golden/v2/tensorflow.train.pbtxt      | 40 ---------------
 tensorflow/tools/compatibility/renames_v2.py  | 10 ++++
 22 files changed, 20 insertions(+), 559 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt

diff --git a/tensorflow/python/training/adadelta.py b/tensorflow/python/training/adadelta.py
index 95eca76496..dd21016000 100644
--- a/tensorflow/python/training/adadelta.py
+++ b/tensorflow/python/training/adadelta.py
@@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.AdadeltaOptimizer")
+@tf_export(v1=["train.AdadeltaOptimizer"])
 class AdadeltaOptimizer(optimizer.Optimizer):
   """Optimizer that implements the Adadelta algorithm.
 
diff --git a/tensorflow/python/training/adagrad.py b/tensorflow/python/training/adagrad.py
index cc0da26b27..10c043bae1 100644
--- a/tensorflow/python/training/adagrad.py
+++ b/tensorflow/python/training/adagrad.py
@@ -28,7 +28,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.AdagradOptimizer")
+@tf_export(v1=["train.AdagradOptimizer"])
 class AdagradOptimizer(optimizer.Optimizer):
   """Optimizer that implements the Adagrad algorithm.
 
diff --git a/tensorflow/python/training/adagrad_da.py b/tensorflow/python/training/adagrad_da.py
index 5ba403554f..e23b7134b3 100644
--- a/tensorflow/python/training/adagrad_da.py
+++ b/tensorflow/python/training/adagrad_da.py
@@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.AdagradDAOptimizer")
+@tf_export(v1=["train.AdagradDAOptimizer"])
 class AdagradDAOptimizer(optimizer.Optimizer):
   """Adagrad Dual Averaging algorithm for sparse linear models.
 
diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py
index 704ad6d3fe..0c701f4712 100644
--- a/tensorflow/python/training/adam.py
+++ b/tensorflow/python/training/adam.py
@@ -29,7 +29,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.AdamOptimizer")
+@tf_export(v1=["train.AdamOptimizer"])
 class AdamOptimizer(optimizer.Optimizer):
   """Optimizer that implements the Adam algorithm.
 
diff --git a/tensorflow/python/training/ftrl.py b/tensorflow/python/training/ftrl.py
index 2fafc9a2d8..a2ef3c76b4 100644
--- a/tensorflow/python/training/ftrl.py
+++ b/tensorflow/python/training/ftrl.py
@@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.FtrlOptimizer")
+@tf_export(v1=["train.FtrlOptimizer"])
 class FtrlOptimizer(optimizer.Optimizer):
   """Optimizer that implements the FTRL algorithm.
 
diff --git a/tensorflow/python/training/gradient_descent.py b/tensorflow/python/training/gradient_descent.py
index ef50f6315d..1a527345ef 100644
--- a/tensorflow/python/training/gradient_descent.py
+++ b/tensorflow/python/training/gradient_descent.py
@@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.GradientDescentOptimizer")
+@tf_export(v1=["train.GradientDescentOptimizer"])
 class GradientDescentOptimizer(optimizer.Optimizer):
   """Optimizer that implements the gradient descent algorithm.
   """
diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py
index 4a280e7c51..f3bc83bbfa 100644
--- a/tensorflow/python/training/momentum.py
+++ b/tensorflow/python/training/momentum.py
@@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.MomentumOptimizer")
+@tf_export(v1=["train.MomentumOptimizer"])
 class MomentumOptimizer(optimizer.Optimizer):
   """Optimizer that implements the Momentum algorithm.
 
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 2556e6274a..900afeed6c 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -214,7 +214,7 @@ def _get_processor(v):
   raise NotImplementedError("Trying to optimize unsupported type ", v)
 
 
-@tf_export("train.Optimizer")
+@tf_export(v1=["train.Optimizer"])
 class Optimizer(
     # Optimizers inherit from CheckpointableBase rather than Checkpointable
     # since they do most of their dependency management themselves (slot
diff --git a/tensorflow/python/training/proximal_adagrad.py b/tensorflow/python/training/proximal_adagrad.py
index 9bd677b8ef..2ea628a56b 100644
--- a/tensorflow/python/training/proximal_adagrad.py
+++ b/tensorflow/python/training/proximal_adagrad.py
@@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.ProximalAdagradOptimizer")
+@tf_export(v1=["train.ProximalAdagradOptimizer"])
 class ProximalAdagradOptimizer(optimizer.Optimizer):
   # pylint: disable=line-too-long
   """Optimizer that implements the Proximal Adagrad algorithm.
diff --git a/tensorflow/python/training/rmsprop.py b/tensorflow/python/training/rmsprop.py
index f38c9861d6..fb53b5883f 100644
--- a/tensorflow/python/training/rmsprop.py
+++ b/tensorflow/python/training/rmsprop.py
@@ -50,7 +50,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.RMSPropOptimizer")
+@tf_export(v1=["train.RMSPropOptimizer"])
 class RMSPropOptimizer(optimizer.Optimizer):
   """Optimizer that implements the RMSProp algorithm.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt
deleted file mode 100644
index 1f1d8b6f9e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adadelta-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.AdadeltaOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.adadelta.AdadeltaOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'epsilon\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.001\', \'0.95\', \'1e-08\', \'False\', \'Adadelta\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
deleted file mode 100644
index a7c05d4849..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.AdagradDAOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.adagrad_da.AdagradDAOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'global_step\', \'initial_gradient_squared_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'0.0\', \'0.0\', \'False\', \'AdagradDA\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt
deleted file mode 100644
index bc8b92389c..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adagrad-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.AdagradOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.adagrad.AdagradOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'False\', \'Adagrad\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt
deleted file mode 100644
index 5d17be9378..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-adam-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.AdamOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.adam.AdamOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'beta1\', \'beta2\', \'epsilon\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-08\', \'False\', \'Adam\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt
deleted file mode 100644
index d265fdeb01..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-ftrl-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.FtrlOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.ftrl.FtrlOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'learning_rate_power\', \'initial_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\', \'accum_name\', \'linear_name\', \'l2_shrinkage_regularization_strength\'], varargs=None, keywords=None, defaults=[\'-0.5\', \'0.1\', \'0.0\', \'0.0\', \'False\', \'Ftrl\', \'None\', \'None\', \'0.0\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt
deleted file mode 100644
index c673e29cd4..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-gradient-descent-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.GradientDescentOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.gradient_descent.GradientDescentOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'GradientDescent\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt
deleted file mode 100644
index 8199f63b9b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-momentum-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.MomentumOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.momentum.MomentumOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'momentum\', \'use_locking\', \'name\', \'use_nesterov\'], varargs=None, keywords=None, defaults=[\'False\', \'Momentum\', \'False\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt
deleted file mode 100644
index 876bb35e39..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-optimizer.pbtxt
+++ /dev/null
@@ -1,50 +0,0 @@
-path: "tensorflow.train.Optimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
deleted file mode 100644
index 14349a74ef..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.ProximalAdagradOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.proximal_adagrad.ProximalAdagradOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'0.0\', \'0.0\', \'False\', \'ProximalAdagrad\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
deleted file mode 100644
index 906384a287..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.RMSPropOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.rmsprop.RMSPropOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'decay\', \'momentum\', \'epsilon\', \'use_locking\', \'centered\', \'name\'], varargs=None, keywords=None, defaults=[\'0.9\', \'0.0\', \'1e-10\', \'False\', \'False\', \'RMSProp\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index d47bc09e66..4a4f387131 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -1,21 +1,5 @@
 path: "tensorflow.train"
 tf_module {
-  member {
-    name: "AdadeltaOptimizer"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "AdagradDAOptimizer"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "AdagradOptimizer"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "AdamOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "BytesList"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -88,18 +72,10 @@ tf_module {
     name: "FloatList"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "FtrlOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "GlobalStepWaiterHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "GradientDescentOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Int64List"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -112,10 +88,6 @@ tf_module {
     name: "LoggingTensorHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "MomentumOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "MonitoredSession"
     mtype: "<type \'type\'>"
@@ -128,22 +100,10 @@ tf_module {
     name: "NanTensorHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Optimizer"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "ProximalAdagradOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ProximalGradientDescentOptimizer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "RMSPropOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Scaffold"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index c5efc04d04..49c152bb86 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -581,10 +581,20 @@ renames = {
     'tf.to_int64': 'tf.compat.v1.to_int64',
     'tf.trace': 'tf.linalg.trace',
     'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread',
+    'tf.train.AdadeltaOptimizer': 'tf.compat.v1.train.AdadeltaOptimizer',
+    'tf.train.AdagradDAOptimizer': 'tf.compat.v1.train.AdagradDAOptimizer',
+    'tf.train.AdagradOptimizer': 'tf.compat.v1.train.AdagradOptimizer',
+    'tf.train.AdamOptimizer': 'tf.compat.v1.train.AdamOptimizer',
+    'tf.train.FtrlOptimizer': 'tf.compat.v1.train.FtrlOptimizer',
+    'tf.train.GradientDescentOptimizer': 'tf.compat.v1.train.GradientDescentOptimizer',
+    'tf.train.MomentumOptimizer': 'tf.compat.v1.train.MomentumOptimizer',
     'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession',
     'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader',
+    'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer',
     'tf.train.ProfilerHook': 'tf.compat.v1.train.ProfilerHook',
+    'tf.train.ProximalAdagradOptimizer': 'tf.compat.v1.train.ProximalAdagradOptimizer',
     'tf.train.QueueRunner': 'tf.compat.v1.train.QueueRunner',
+    'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
     'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
-- 
GitLab


From ae85f10ed9f6aff20d984d188b384ea56fc646af Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Wed, 28 Nov 2018 08:41:39 -0800
Subject: [PATCH 0927/1554] Addressed review comments

---
 tensorflow/core/kernels/BUILD                 |  22 ++
 tensorflow/core/kernels/mkl_avgpooling_op.cc  | 172 +++++++++-----
 tensorflow/core/kernels/mkl_maxpooling_op.cc  | 215 +++++++++++-------
 .../core/kernels/mkl_pooling_ops_common.cc    |  49 ++--
 .../core/kernels/mkl_pooling_ops_common.h     |  44 ++--
 .../kernels/mkl_quantized_pooling_ops_test.cc | 201 ++++++++++++++++
 6 files changed, 526 insertions(+), 177 deletions(-)
 create mode 100644 tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b523e3e718..eb209fc718 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6269,6 +6269,28 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_quantized_pooling_ops_test",
+    size = "small",
+    srcs = ["mkl_quantized_pooling_ops_test.cc"],
+    deps = [
+        ":mkl_input_conversion_op",
+        ":mkl_pooling_ops",
+        ":ops_testutil",
+        ":ops_util",
+        ":quantization_utils",
+        ":quantized_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test(
     name = "quantized_reshape_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/mkl_avgpooling_op.cc b/tensorflow/core/kernels/mkl_avgpooling_op.cc
index 2409f7e9dc..a5dc317969 100644
--- a/tensorflow/core/kernels/mkl_avgpooling_op.cc
+++ b/tensorflow/core/kernels/mkl_avgpooling_op.cc
@@ -53,16 +53,19 @@ class MklAvgPoolingOp : public OpKernel {
 
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument("Sliding window ksize field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window ksize field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument("Sliding window stride field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window stride field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
-                errors::Unimplemented("Pooling is not yet supported on the "
-                                      "batch dimension."));
+                errors::Unimplemented(
+                    "Pooling is not yet supported on the "
+                    "batch dimension."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -228,16 +231,19 @@ class MklAvgPoolingGradOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument("Sliding window ksize field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window ksize field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window strides field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
-                errors::Unimplemented("Pooling is not yet supported on the "
-                                      "batch dimension."));
+                errors::Unimplemented(
+                    "Pooling is not yet supported on the "
+                    "batch dimension."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -357,22 +363,22 @@ class MklAvgPoolingGradOp : public OpKernel {
       if (!outbackprop_in_mkl_format) {
         // For avgpooling, tensor_in_shape should have 1 dimension, and 4
         // elements.
-        OP_REQUIRES(
-            context,
-            tensor_in_shape.dims() == 1 && tensor_in_shape.NumElements() == 4,
-            errors::InvalidArgument("original input shape must be "
-                                    "1-dimensional and 4 elements"));
+        OP_REQUIRES(context, tensor_in_shape.dims() == 1 &&
+                                 tensor_in_shape.NumElements() == 4,
+                    errors::InvalidArgument(
+                        "original input shape must be "
+                        "1-dimensional and 4 elements"));
 
         // For avgpooling, out_backprop should have 4 dimensions.
-        OP_REQUIRES(context, out_backprop.dims() == 4,
-                    errors::InvalidArgument("out_backprop must be "
-                                            "4-dimensional"));
+        OP_REQUIRES(
+            context, out_backprop.dims() == 4,
+            errors::InvalidArgument("out_backprop must be 4-dimensional"));
       } else {
         // Input in MKL format.
         // For avgpooling, out_backprop should have 4 dimensions.
-        OP_REQUIRES(context, out_backprop_shape.GetDimension() == 4,
-                    errors::InvalidArgument("out_backprop must be "
-                                            "4-dimensional"));
+        OP_REQUIRES(
+            context, out_backprop_shape.GetDimension() == 4,
+            errors::InvalidArgument("out_backprop must be 4-dimensional"));
       }
 
       // TODO(inteltf): Get outbackprop layout.
@@ -484,9 +490,9 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase<T> {
           dnn_shape_input.IsMklTensor()
               ? dnn_shape_input.GetSizesAsMklDnnDims()
               : is_pool2d ? TFShapeToMklDnnDimsInNCHW(input_tensor.shape(),
-                                                     this->data_format_tf_)
-                         : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(),
-                                                      this->data_format_tf_);
+                                                      this->data_format_tf_)
+                          : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(),
+                                                       this->data_format_tf_);
       memory::desc input_md = dnn_shape_input.IsMklTensor()
                                   ? dnn_shape_input.GetMklLayout()
                                   : memory::desc(src_dims, MklDnnType<T>(),
@@ -494,9 +500,17 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase<T> {
 
       // Get an average pooling primitive from the op pool
       MklPoolingFwdPrimitive<T>* pooling_fwd = nullptr;
+      prop_kind pooling_prop_kind;
+      bool int8_forward_inference =
+          std::is_same<T, qint8>::value || std::is_same<T, quint8>::value;
+      if (int8_forward_inference)
+        pooling_prop_kind = prop_kind::forward_inference;
+      else
+        pooling_prop_kind = prop_kind::forward_training;
       MklPoolingParams fwdParams(src_dims, output_dims_mkl_order, filter_dims,
                                  strides, padding_left, padding_right,
-                                 algorithm::pooling_avg_exclude_padding);
+                                 algorithm::pooling_avg_exclude_padding,
+                                 pooling_prop_kind);
       pooling_fwd = MklPoolingFwdPrimitiveFactory<T>::Get(fwdParams);
 
       // allocate output tensor
@@ -523,10 +537,31 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase<T> {
 
       // execute pooling
       pooling_fwd->Execute(src_data, dst_data);
-    } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+
+      // Pass min, max from input to output
+      if (int8_forward_inference) {
+        const Tensor& min_input_t = MklGetInput(context, 1);
+        const Tensor& max_input_t = MklGetInput(context, 2);
+        const float min_input = min_input_t.flat<float>()(0);
+        const float max_input = max_input_t.flat<float>()(0);
+
+        Tensor* output_min = nullptr;
+        Tensor* output_max = nullptr;
+        MklDnnShape output_min_mkl_shape, output_max_mkl_shape;
+        output_min_mkl_shape.SetMklTensor(false);
+        output_max_mkl_shape.SetMklTensor(false);
+        AllocateOutputSetMklShape(context, 1, &output_min, {},
+                                  output_min_mkl_shape);
+        AllocateOutputSetMklShape(context, 2, &output_max, {},
+                                  output_max_mkl_shape);
+        output_min->flat<float>()(0) = min_input;
+        output_max->flat<float>()(0) = max_input;
+      }
+    }
+    catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -576,24 +611,26 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase<T> {
           orig_input_mkl_shape.IsMklTensor()
               ? orig_input_mkl_shape.GetSizesAsMklDnnDims()
               : is_pool2d ? TFShapeToMklDnnDimsInNCHW(orig_input_shape,
-                                                     this->data_format_tf_)
-                         : TFShapeToMklDnnDimsInNCDHW(orig_input_shape,
-                                                      this->data_format_tf_);
+                                                      this->data_format_tf_)
+                          : TFShapeToMklDnnDimsInNCDHW(orig_input_shape,
+                                                       this->data_format_tf_);
 
       memory::dims diff_dst_dims =
           grad_mkl_shape.IsMklTensor()
               ? grad_mkl_shape.GetSizesAsMklDnnDims()
               : is_pool2d ? TFShapeToMklDnnDimsInNCHW(grad_tensor.shape(),
-                                                     this->data_format_tf_)
-                         : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(),
-                                                      this->data_format_tf_);
+                                                      this->data_format_tf_)
+                          : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(),
+                                                       this->data_format_tf_);
       memory::dims output_dims_mkl_order;
       this->GetOutputDims(pool_params, &output_dims_mkl_order);
 
-      MklPoolingParams bwdParams(orig_input_dims_mkl_order,
-                                 output_dims_mkl_order, filter_dims, strides,
-                                 padding_left, padding_right,
-                                 algorithm::pooling_avg_exclude_padding);
+      // Pass prop_kind::forward_training to create a forward primitive
+      // that is used in the backward pass
+      MklPoolingParams bwdParams(
+          orig_input_dims_mkl_order, output_dims_mkl_order, filter_dims,
+          strides, padding_left, padding_right,
+          algorithm::pooling_avg_exclude_padding, prop_kind::forward_training);
       MklPoolingBwdPrimitive<T>* pooling_bwd =
           MklPoolingBwdPrimitiveFactory<T>::Get(bwdParams);
 
@@ -623,10 +660,11 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase<T> {
 
       // execute pooling op
       pooling_bwd->Execute(diff_dst_data, diff_src_data);
-    } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+    }
+    catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(context, errors::Aborted("Compute received an exception:",
                                               error_msg));
     }
@@ -645,28 +683,28 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase<T> {
                          const MklDnnShape& original_input_mkl_shape,
                          const MklDnnShape& input_gradient_mkl_shape) {
     if (!original_input_mkl_shape.IsMklTensor()) {
-      OP_REQUIRES(
-          context,
-          tensor_in_shape.dims() == 1 && tensor_in_shape.NumElements() == 4,
-          errors::InvalidArgument("original input shape must be "
-                                  "1-dimensional and 4 elements"));
+      OP_REQUIRES(context, tensor_in_shape.dims() == 1 &&
+                               tensor_in_shape.NumElements() == 4,
+                  errors::InvalidArgument(
+                      "original input shape must be "
+                      "1-dimensional and 4 elements"));
     } else {
-      OP_REQUIRES(context,
-                  original_input_mkl_shape.GetDimension() == 1 &&
-                      original_input_mkl_shape.DimSize(0) == 4,
-                  errors::InvalidArgument("original input shape must be "
-                                          "1-dimensional and 4 elements"));
+      OP_REQUIRES(context, original_input_mkl_shape.GetDimension() == 1 &&
+                               original_input_mkl_shape.DimSize(0) == 4,
+                  errors::InvalidArgument(
+                      "original input shape must be "
+                      "1-dimensional and 4 elements"));
     }
 
     if (!input_gradient_mkl_shape.IsMklTensor()) {
       // For avgpooling, input_gradient_diff_dst should have 4 dimensions.
-      OP_REQUIRES(context, input_gradient_tensor.dims() == 4,
-                  errors::InvalidArgument("Gradient shape must be "
-                                          "4-dimensional"));
+      OP_REQUIRES(
+          context, input_gradient_tensor.dims() == 4,
+          errors::InvalidArgument("Gradient shape must be 4-dimensional"));
     } else {
-      OP_REQUIRES(context, input_gradient_mkl_shape.GetDimension() == 4,
-                  errors::InvalidArgument("Gradient shape must be "
-                                          "4-dimensional"));
+      OP_REQUIRES(
+          context, input_gradient_mkl_shape.GetDimension() == 4,
+          errors::InvalidArgument("Gradient shape must be 4-dimensional"));
     }
   }
 };  // MklAvgPoolingGradOp
@@ -691,6 +729,18 @@ REGISTER_KERNEL_BUILDER(Name("_MklAvgPool")
                             .Label(mkl_op_registry::kMklOpLabel),
                         MklAvgPoolingOp<CPUDevice, float>);
 
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedAvgPool")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<quint8>("T")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklAvgPoolingOp<CPUDevice, quint8>);
+
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedAvgPool")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint8>("T")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklAvgPoolingOp<CPUDevice, qint8>);
+
 REGISTER_KERNEL_BUILDER(Name("_MklAvgPoolGrad")
                             .Device(DEVICE_CPU)
                             .TypeConstraint<float>("T")
diff --git a/tensorflow/core/kernels/mkl_maxpooling_op.cc b/tensorflow/core/kernels/mkl_maxpooling_op.cc
index 256d48f4d5..cd72530bce 100644
--- a/tensorflow/core/kernels/mkl_maxpooling_op.cc
+++ b/tensorflow/core/kernels/mkl_maxpooling_op.cc
@@ -54,16 +54,19 @@ class MklMaxPoolingOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument("Sliding window ksize field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window ksize field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument("Sliding window stride field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window stride field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
-                errors::Unimplemented("Pooling is not yet supported on the "
-                                      "batch dimension."));
+                errors::Unimplemented(
+                    "Pooling is not yet supported on the "
+                    "batch dimension."));
 
     workspace_enabled_ = false;
     // We may not get this attribute for this node if it does not go through
@@ -213,12 +216,14 @@ class MklMaxPoolingGradOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument("Sliding window ksize field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window ksize field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window strides field must "
+                    "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
                 errors::Unimplemented(
@@ -399,19 +404,18 @@ class MklMaxPoolingGradOp : public OpKernel {
       if (workspace_enabled == false) {
         if (convert_input != nullptr) {
           if (input_in_mkl_format == false) {
-            CHECK_EQ(dnnConversionExecute_F32(
-                         convert_input,
-                         const_cast<void*>(static_cast<const void*>(
-                             tensor_in.flat<T>().data())),
-                         input_buf),
-                     E_SUCCESS);
+            CHECK_EQ(
+                dnnConversionExecute_F32(
+                    convert_input, const_cast<void*>(static_cast<const void*>(
+                                       tensor_in.flat<T>().data())),
+                    input_buf),
+                E_SUCCESS);
             CHECK_EQ(dnnDelete_F32(convert_input), E_SUCCESS);
             convert_input = nullptr;
           } else {
             input_shape.GetConvertedFlatData(
-                lt_input_prim,
-                const_cast<void*>(
-                    static_cast<const void*>(tensor_in.flat<T>().data())),
+                lt_input_prim, const_cast<void*>(static_cast<const void*>(
+                                   tensor_in.flat<T>().data())),
                 input_buf);
           }
           pooling_resfwd[dnnResourceSrc] = input_buf;
@@ -456,9 +460,8 @@ class MklMaxPoolingGradOp : public OpKernel {
           CHECK_EQ(dnnDelete_F32(convert_outbackprop), E_SUCCESS);
         } else {
           output_backprop_shape.GetConvertedFlatData(
-              lt_outbackprop_prim,
-              const_cast<void*>(
-                  static_cast<const void*>(out_backprop.flat<T>().data())),
+              lt_outbackprop_prim, const_cast<void*>(static_cast<const void*>(
+                                       out_backprop.flat<T>().data())),
               outbackprop_buf);
         }
         pooling_res[dnnResourceDiffDst] = outbackprop_buf;
@@ -520,7 +523,6 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase<T> {
 
       MklDnnData<T> dnn_data_input(&cpu_engine);
       MklDnnData<T> dnn_data_output(&cpu_engine);
-      MklDnnData<uint8> dnn_data_wksp(&cpu_engine);
 
       // initialize variables for the pooling op
       MklPoolParameters pool_params;
@@ -550,13 +552,13 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase<T> {
           dnn_shape_input.IsMklTensor()
               ? dnn_shape_input.GetMklLayout()
               : is_pool2d ? memory::desc(
-                               TFShapeToMklDnnDimsInNCHW(input_tensor_shape,
-                                                         this->data_format_tf_),
-                               MklDnnType<T>(), this->data_format_mkldnn_)
-                         : memory::desc(
-                               TFShapeToMklDnnDimsInNCDHW(
-                                   input_tensor_shape, this->data_format_tf_),
-                               MklDnnType<T>(), this->data_format_mkldnn_);
+                                TFShapeToMklDnnDimsInNCHW(
+                                    input_tensor_shape, this->data_format_tf_),
+                                MklDnnType<T>(), this->data_format_mkldnn_)
+                          : memory::desc(
+                                TFShapeToMklDnnDimsInNCDHW(
+                                    input_tensor_shape, this->data_format_tf_),
+                                MklDnnType<T>(), this->data_format_mkldnn_);
 
       // Get src/filter/stride/padding information
       memory::dims src_dims =
@@ -564,17 +566,24 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase<T> {
               ? dnn_shape_input.GetSizesAsMklDnnDims()
               : is_pool2d ? TFShapeToMklDnnDimsInNCHW(input_tensor.shape(),
                                                       this->data_format_tf_)
-                         : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(),
-                                                      this->data_format_tf_);
+                          : TFShapeToMklDnnDimsInNCDHW(input_tensor.shape(),
+                                                       this->data_format_tf_);
       memory::dims filter_dims, strides, padding_left, padding_right;
       this->PoolParamsToDims(&pool_params, &filter_dims, &strides,
                              &padding_left, &padding_right, is_pool2d);
 
       // Get a pooling op from the cached pool
       MklPoolingFwdPrimitive<T>* pooling_fwd = nullptr;
+      prop_kind pooling_prop_kind;
+      bool int8_forward_inference =
+          std::is_same<T, qint8>::value || std::is_same<T, quint8>::value;
+      if (int8_forward_inference)
+        pooling_prop_kind = prop_kind::forward_inference;
+      else
+        pooling_prop_kind = prop_kind::forward_training;
       MklPoolingParams fwdParams(src_dims, output_dims_mkl_order, filter_dims,
                                  strides, padding_left, padding_right,
-                                 algorithm::pooling_max);
+                                 algorithm::pooling_max, pooling_prop_kind);
       pooling_fwd = MklPoolingFwdPrimitiveFactory<T>::Get(fwdParams);
 
       // allocate output tensor
@@ -586,10 +595,6 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase<T> {
                                 pooling_fwd->GetDstMemoryFormat(),
                                 output_tensor);
 
-      AllocateWorkspaceTensor(context, *(pooling_fwd->GetPoolingFwdPd()),
-                              &dnn_data_wksp);
-      OP_REQUIRES_OK(context, context->status());
-
       // check wehther we need to reorder src
       const T* src_data = input_tensor.flat<T>().data();
       if (input_md.data.format != pooling_fwd->GetSrcMemoryFormat()) {
@@ -603,14 +608,44 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase<T> {
       }
 
       T* dst_data = output_tensor->flat<T>().data();
-      void* ws_data = dnn_data_wksp.GetOpMem().get_data_handle();
-
-      // execute pooling op
-      pooling_fwd->Execute(src_data, dst_data, ws_data);
-    } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+
+      if (int8_forward_inference) {
+        // Execute pooling op
+        pooling_fwd->Execute(src_data, dst_data);
+
+        // pass min, max from input to output
+        const Tensor& min_input_t = MklGetInput(context, 1);
+        const Tensor& max_input_t = MklGetInput(context, 2);
+        const float min_input = min_input_t.flat<float>()(0);
+        const float max_input = max_input_t.flat<float>()(0);
+
+        Tensor* output_min = nullptr;
+        Tensor* output_max = nullptr;
+        MklDnnShape output_min_mkl_shape, output_max_mkl_shape;
+        output_min_mkl_shape.SetMklTensor(false);
+        output_max_mkl_shape.SetMklTensor(false);
+        AllocateOutputSetMklShape(context, 1, &output_min, {},
+                                  output_min_mkl_shape);
+        AllocateOutputSetMklShape(context, 2, &output_max, {},
+                                  output_max_mkl_shape);
+        output_min->flat<float>()(0) = min_input;
+        output_max->flat<float>()(0) = max_input;
+      } else {
+        MklDnnData<uint8> dnn_data_wksp(&cpu_engine);
+        AllocateWorkspaceTensor(context, *(pooling_fwd->GetPoolingFwdPd()),
+                                &dnn_data_wksp);
+        OP_REQUIRES_OK(context, context->status());
+        T* ws_data =
+            static_cast<T*>(dnn_data_wksp.GetOpMem().get_data_handle());
+
+        // execute pooling op
+        pooling_fwd->Execute(src_data, dst_data, ws_data);
+      }
+    }
+    catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(context, errors::Aborted("Compute received an exception:",
                                               error_msg));
     }
@@ -684,24 +719,25 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
           orig_input_mkl_shape.IsMklTensor()
               ? orig_input_mkl_shape.GetSizesAsMklDnnDims()
               : is_pool2d ? TFShapeToMklDnnDimsInNCHW(orig_input_shape,
-                                                     this->data_format_tf_)
-                         : TFShapeToMklDnnDimsInNCDHW(orig_input_shape,
-                                                      this->data_format_tf_);
+                                                      this->data_format_tf_)
+                          : TFShapeToMklDnnDimsInNCDHW(orig_input_shape,
+                                                       this->data_format_tf_);
 
       memory::dims diff_dst_dims =
           grad_mkl_shape.IsMklTensor()
               ? grad_mkl_shape.GetSizesAsMklDnnDims()
               : is_pool2d ? TFShapeToMklDnnDimsInNCHW(grad_tensor.shape(),
-                                                     this->data_format_tf_)
-                         : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(),
-                                                      this->data_format_tf_);
+                                                      this->data_format_tf_)
+                          : TFShapeToMklDnnDimsInNCDHW(grad_tensor.shape(),
+                                                       this->data_format_tf_);
 
       memory::dims output_dims_mkl_order;
       this->GetOutputDims(pool_params, &output_dims_mkl_order);
 
       MklPoolingParams bwdParams(
           orig_input_dims_mkl_order, output_dims_mkl_order, filter_dims,
-          strides, padding_left, padding_right, algorithm::pooling_max);
+          strides, padding_left, padding_right, algorithm::pooling_max,
+          prop_kind::forward_training);
       MklPoolingBwdPrimitive<T>* pooling_bwd =
           MklPoolingBwdPrimitiveFactory<T>::Get(bwdParams);
 
@@ -736,11 +772,10 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
       if (ws_md.data.format != pooling_bwd->GetWorkspaceFormat()) {
         memory::dims ws_dims;
         ws_dims.assign(ws_md.data.dims, ws_md.data.dims + ws_md.data.ndims);
-        auto target_ws =
-            memory::primitive_desc({{ws_dims},
-                                    pooling_bwd->GetWorkspaceDataType(),
-                                    pooling_bwd->GetWorkspaceFormat()},
-                                   cpu_engine);
+        auto target_ws = memory::primitive_desc(
+            {{ws_dims}, pooling_bwd->GetWorkspaceDataType(),
+             pooling_bwd->GetWorkspaceFormat()},
+            cpu_engine);
         workspace_dnn_data.SetUsrMem(ws_md, &workspace_tensor);
         workspace_dnn_data.CheckReorderToOpMem(target_ws);
         ws_data = workspace_dnn_data.GetOpMem().get_data_handle();
@@ -750,10 +785,11 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
 
       // execute pooling
       pooling_bwd->Execute(diff_dst_data, diff_src_data, ws_data);
-    } catch (mkldnn::error& e) {
-      string error_msg = "Status:" + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ". in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+    }
+    catch (mkldnn::error& e) {
+      string error_msg = "Status:" + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ". in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(context, errors::Aborted("Compute received an exception:",
                                               error_msg));
     }
@@ -788,46 +824,45 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
                          const MklDnnShape& workspace_mkl_shape) {
     if (!orig_input_mkl_shape.IsMklTensor()) {
       OP_REQUIRES(context, orig_input_tensor.dims() == 4,
-                  errors::InvalidArgument("Original input shape must be "
-                                          "4-dimensional"));
+                  errors::InvalidArgument(
+                      "Original input shape must be 4-dimensional"));
     } else {
       OP_REQUIRES(context, orig_input_mkl_shape.GetDimension() == 4,
-                  errors::InvalidArgument("Original input shape must be "
-                                          "4-dimensional"));
+                  errors::InvalidArgument(
+                      "Original input shape must be 4-dimensional"));
     }
     if (!orig_output_mkl_shape.IsMklTensor()) {
-      OP_REQUIRES(context, orig_output_tensor.dims() == 4,
-                  errors::InvalidArgument("Original output must be "
-                                          "4-dimensional"));
+      OP_REQUIRES(
+          context, orig_output_tensor.dims() == 4,
+          errors::InvalidArgument("Original output must be 4-dimensional"));
     } else {
-      OP_REQUIRES(context, orig_output_mkl_shape.GetDimension() == 4,
-                  errors::InvalidArgument("Original output must be "
-                                          "4-dimensional"));
+      OP_REQUIRES(
+          context, orig_output_mkl_shape.GetDimension() == 4,
+          errors::InvalidArgument("Original output must be 4-dimensional"));
     }
     if (!grad_mkl_shape.IsMklTensor()) {
       OP_REQUIRES(context, grad_tensor.dims() == 4,
                   errors::InvalidArgument("Gradient must be 4-dimensional"));
     } else {
       OP_REQUIRES(context, grad_mkl_shape.GetDimension() == 4,
-                  errors::InvalidArgument("Gradient must be "
-                                          "4-dimensional"));
+                  errors::InvalidArgument("Gradient must be 4-dimensional"));
     }
     if (this->workspace_enabled_) {
       // The workspace should not be an MKL tensor
       OP_REQUIRES(context, workspace_mkl_shape.IsMklTensor() == false,
-                  errors::InvalidArgument("Workspace tensor should not"
-                                          " be an MKL Tensor."));
+                  errors::InvalidArgument(
+                      "Workspace tensor should not be an MKL Tensor."));
       // It should only have one dimension
-      OP_REQUIRES(context, workspace_tensor.dims() == 1,
-                  errors::InvalidArgument("Workspace tensor must be "
-                                          "1-dimensional"));
-    } else {
       OP_REQUIRES(
-          context, this->workspace_enabled_,
-          errors::Unimplemented("MKL-DNN Max Pooling does not "
-                                "yet support the use case "
-                                "where MaxPoolGrad is called without first"
-                                " calling MaxPool."));
+          context, workspace_tensor.dims() == 1,
+          errors::InvalidArgument("Workspace tensor must be 1-dimensional"));
+    } else {
+      OP_REQUIRES(context, this->workspace_enabled_,
+                  errors::Unimplemented(
+                      "MKL-DNN Max Pooling does not "
+                      "yet support the use case "
+                      "where MaxPoolGrad is called without first"
+                      " calling MaxPool."));
     }
   }
 };  // MklMaxPoolingGradOp
@@ -852,6 +887,18 @@ REGISTER_KERNEL_BUILDER(Name("_MklMaxPool")
                             .Label(mkl_op_registry::kMklOpLabel),
                         MklMaxPoolingOp<CPUDevice, float>);
 
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedMaxPool")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<quint8>("T")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklMaxPoolingOp<CPUDevice, quint8>);
+
+REGISTER_KERNEL_BUILDER(Name("_MklQuantizedMaxPool")
+                            .Device(DEVICE_CPU)
+                            .TypeConstraint<qint8>("T")
+                            .Label(mkl_op_registry::kMklQuantizedOpLabel),
+                        MklMaxPoolingOp<CPUDevice, qint8>);
+
 REGISTER_KERNEL_BUILDER(Name("_MklMaxPoolGrad")
                             .Device(DEVICE_CPU)
                             .TypeConstraint<float>("T")
diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
index 5398e6113f..cf3f0f1469 100644
--- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc
+++ b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
@@ -41,28 +41,33 @@ void MklPoolingFwdPrimitive<T>::Setup(const MklPoolingParams& fwdParams) {
       << "Pooling algorithm kind is not supported";
 
   context_.alg_kind = fwdParams.alg_kind;
+  context_.prop_kind = fwdParams.prop_kind;
+
   // create memory desc
   // FIXME: Pooling doesn't expose to get the src_primitive_desc,
   //        so src format is currently hard-coded.
   //        A utility function is used to do this,
   //        which may be broken with future CPU architectures
   bool is_2d = (fwdParams.src_dims.size() == 4);
-  context_.src_md.reset(
-      new memory::desc({fwdParams.src_dims}, MklDnnType<T>(),
-                       get_desired_format(fwdParams.src_dims[1], is_2d)));
+  if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value)
+    context_.src_fmt = is_2d ? memory::format::nhwc : memory::format::ndhwc;
+  else
+    context_.src_fmt = get_desired_format(fwdParams.src_dims[1], is_2d);
+
+  context_.src_md.reset(new memory::desc({fwdParams.src_dims}, MklDnnType<T>(),
+                                         context_.src_fmt));
   context_.dst_md.reset(new memory::desc({fwdParams.dst_dims}, MklDnnType<T>(),
                                          memory::format::any));
 
   // create a pooling descriptor
   context_.fwd_desc.reset(new pooling_forward::desc(
-      prop_kind::forward_training, fwdParams.alg_kind, *context_.src_md,
+      fwdParams.prop_kind, fwdParams.alg_kind, *context_.src_md,
       *context_.dst_md, fwdParams.strides, fwdParams.filter_dims,
       fwdParams.padding_left, fwdParams.padding_right, padding_kind::zero));
   context_.fwd_pd.reset(
       new pooling_forward::primitive_desc(*context_.fwd_desc, cpu_engine_));
 
   // store expected primitive format
-  context_.src_fmt = get_desired_format(fwdParams.src_dims[1], is_2d);
   context_.dst_fmt = static_cast<mkldnn::memory::format>(
       context_.fwd_pd.get()->dst_primitive_desc().desc().data.format);
 
@@ -74,7 +79,8 @@ void MklPoolingFwdPrimitive<T>::Setup(const MklPoolingParams& fwdParams) {
       new memory(context_.fwd_pd.get()->dst_primitive_desc(), DummyData));
 
   // for max pooling, need to return workspace(ws) for backward computing
-  if (fwdParams.alg_kind == pooling_max) {
+  if (fwdParams.alg_kind == pooling_max &&
+      fwdParams.prop_kind == prop_kind::forward_training) {
     auto ws_pd = context_.fwd_pd.get()->workspace_primitive_desc().desc().data;
     // store workspace's dims and format to create workspace tensor
     context_.ws_fmt = static_cast<mkldnn::memory::format>(ws_pd.format);
@@ -101,7 +107,9 @@ void MklPoolingFwdPrimitive<T>::Execute(const T* src_data, T* dst_data,
   context_.src_mem->set_data_handle(
       static_cast<void*>(const_cast<T*>(src_data)));
   context_.dst_mem->set_data_handle(static_cast<void*>(dst_data));
-  if (context_.alg_kind == pooling_max) {  // max pooling must have ws
+  if (context_.alg_kind == pooling_max &&
+      context_.prop_kind ==
+          prop_kind::forward_training) {  // max pooling must have ws
     DCHECK(ws_data != nullptr);
     context_.ws_mem->set_data_handle(ws_data);
   }
@@ -110,13 +118,17 @@ void MklPoolingFwdPrimitive<T>::Execute(const T* src_data, T* dst_data,
   // set back data handle
   context_.src_mem->set_data_handle(DummyData);
   context_.dst_mem->set_data_handle(DummyData);
-  if (context_.alg_kind == pooling_max) {  // max pooling must have ws
+  if (context_.alg_kind == pooling_max &&
+      context_.prop_kind ==
+          prop_kind::forward_training) {  // max pooling must have ws
     DCHECK(ws_data != nullptr);
     context_.ws_mem->set_data_handle(DummyData);
   }
 }
 
 template class MklPoolingFwdPrimitive<float>;
+template class MklPoolingFwdPrimitive<quint8>;
+template class MklPoolingFwdPrimitive<qint8>;
 
 template <typename T>
 void MklPoolingBwdPrimitive<T>::Setup(const MklPoolingParams& bwdParams) {
@@ -143,7 +155,7 @@ void MklPoolingBwdPrimitive<T>::Setup(const MklPoolingParams& bwdParams) {
   // create a forward primitive,
   // which will be used as a hint for creating backward primitive
   context_.fwd_desc.reset(new pooling_forward::desc(
-      prop_kind::forward_training, bwdParams.alg_kind, *context_.diff_src_md,
+      bwdParams.prop_kind, bwdParams.alg_kind, *context_.diff_src_md,
       *context_.diff_dst_md, bwdParams.strides, bwdParams.filter_dims,
       bwdParams.padding_left, bwdParams.padding_right, padding_kind::zero));
   context_.fwd_pd.reset(
@@ -368,21 +380,24 @@ void MklPoolParameters::Init(OpKernelContext* context,
     // any padding, and expects the depth_window to equal the depth
     // stride (no overlapping).
     OP_REQUIRES(context, depth % depth_window == 0,
-                errors::Unimplemented("Depthwise max pooling requires the"
-                                      " depth window to evenly divide the"
-                                      " input depth"));
+                errors::Unimplemented(
+                    "Depthwise max pooling requires the"
+                    " depth window to evenly divide the"
+                    " input depth"));
     OP_REQUIRES(context, depth_stride == depth_window,
-                errors::Unimplemented("Depthwise max pooling requires the"
-                                      " depth window to equal the depth"
-                                      " stride"));
+                errors::Unimplemented(
+                    "Depthwise max pooling requires the"
+                    " depth window to equal the depth"
+                    " stride"));
 
     // The current version of depthwise max is only implemented on CPU.
     OP_REQUIRES(context,
                 (DeviceType(static_cast<Device*>(context->device())
                                 ->attributes()
                                 .device_type()) == DeviceType(DEVICE_CPU)),
-                errors::Unimplemented("Depthwise max pooling is currently "
-                                      "only implemented for CPU devices."));
+                errors::Unimplemented(
+                    "Depthwise max pooling is currently "
+                    "only implemented for CPU devices."));
 
     out_depth = depth / depth_window;
   }
diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.h b/tensorflow/core/kernels/mkl_pooling_ops_common.h
index 49f799d7ba..d214c39484 100644
--- a/tensorflow/core/kernels/mkl_pooling_ops_common.h
+++ b/tensorflow/core/kernels/mkl_pooling_ops_common.h
@@ -18,8 +18,8 @@ limitations under the License.
 
 #ifdef INTEL_MKL
 #include <memory>
-#include <vector>
 #include <string>
+#include <vector>
 #include "tensorflow/core/util/mkl_util.h"
 #include "tensorflow/core/util/padding.h"
 
@@ -50,18 +50,20 @@ struct MklPoolingParams {
   memory::dims padding_left;
   memory::dims padding_right;
   mkldnn::algorithm alg_kind;
+  mkldnn::prop_kind prop_kind;
 
   MklPoolingParams(memory::dims src_dims, memory::dims dst_dims,
                    memory::dims filter_dims, memory::dims strides,
                    memory::dims padding_left, memory::dims padding_right,
-                   mkldnn::algorithm alg_kind)
+                   mkldnn::algorithm alg_kind, mkldnn::prop_kind prop_kind)
       : src_dims(src_dims),
         dst_dims(dst_dims),
         filter_dims(filter_dims),
         strides(strides),
         padding_left(padding_left),
         padding_right(padding_right),
-        alg_kind(alg_kind) {}
+        alg_kind(alg_kind),
+        prop_kind(prop_kind) {}
 };
 
 template <typename T>
@@ -97,6 +99,9 @@ class MklPoolingFwdPrimitive : public MklPrimitive {
     // algorithm
     mkldnn::algorithm alg_kind;
 
+    // Kind of propagation, forward or backward
+    mkldnn::prop_kind prop_kind;
+
     // expected memory format
     memory::format src_fmt;
     memory::format dst_fmt;
@@ -187,6 +192,7 @@ class MklPoolingFwdPrimitiveFactory : public MklPrimitiveFactory<T> {
     key_creator.AddAsKey(fwdParams.padding_left);
     key_creator.AddAsKey(fwdParams.padding_right);
     key_creator.AddAsKey<int>(static_cast<int>(fwdParams.alg_kind));
+    key_creator.AddAsKey<int>(static_cast<int>(fwdParams.prop_kind));
     return key_creator.GetKey();
   }
 
@@ -443,25 +449,33 @@ class MklPoolingOpBase : public OpKernel {
   explicit MklPoolingOpBase(OpKernelConstruction* context)
       : OpKernel(context), workspace_enabled_(false) {
     string data_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
+    if (std::is_same<T, qint8>::value || std::is_same<T, quint8>::value) {
+      // current quantized convolution doesn't have data_format attribute.
+      data_format = "NHWC";
+    } else {
+      OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
+    }
     OP_REQUIRES(context, FormatFromString(data_format, &this->data_format_tf_),
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &this->ksize_));
     OP_REQUIRES(context, this->ksize_.size() == 4 || this->ksize_.size() == 5,
-                errors::InvalidArgument("Sliding window ksize field must "
-                                        "specify 4 or 5 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window ksize field must "
+                    "specify 4 or 5 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &this->stride_));
     OP_REQUIRES(context, this->stride_.size() == 4 || this->stride_.size() == 5,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 or 5 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window strides field must "
+                    "specify 4 or 5 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &this->padding_));
     OP_REQUIRES(context, this->ksize_[0] == 1 && this->stride_[0] == 1,
-                errors::Unimplemented("Pooling is not yet supported on the "
-                                      "batch dimension."));
+                errors::Unimplemented(
+                    "Pooling is not yet supported on the "
+                    "batch dimension."));
     bool is_pool2d = (this->ksize_.size() == 4);
     this->data_format_mkldnn_ =
         is_pool2d ? TFDataFormatToMklDnnDataFormat(this->data_format_tf_)
-                 : TFDataFormatToMklDnn3DDataFormat(this->data_format_tf_);
+                  : TFDataFormatToMklDnn3DDataFormat(this->data_format_tf_);
 
     // We may not get this attribute for this node if it does not go through
     // graph rewrite pass. So we do not check for error while retrieving this
@@ -655,10 +669,10 @@ class MklPoolingForwardOpBase : public MklPoolingOpBase<T> {
       OP_REQUIRES(context, input_tensor.dims() == 4 || input_tensor.dims() == 5,
                   errors::InvalidArgument("Input must be 4 or 5-dimensional"));
     } else {
-      OP_REQUIRES(context, input_mkl_shape.GetDimension() == 4 ||
-                               input_mkl_shape.GetDimension() == 5,
-                  errors::InvalidArgument("Input shape must be "
-                                          "4 or 5-dimensional"));
+      OP_REQUIRES(
+          context, input_mkl_shape.GetDimension() == 4 ||
+                       input_mkl_shape.GetDimension() == 5,
+          errors::InvalidArgument("Input shape must be 4 or 5-dimensional"));
     }
   }
   // .Input("value: T")
diff --git a/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc b/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc
new file mode 100644
index 0000000000..92fc1bfd07
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc
@@ -0,0 +1,201 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifdef INTEL_MKL
+#define EIGEN_USE_THREADS
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/quantization_utils.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+// Helper class for converting MKL tensors to TF tensors and comparing to
+// expected values
+
+static const uint8 dummy_tensor[] = {0, 0, 0, 0, 0, 0, 0, 0};
+static const TensorShape dummy_shape({8});
+
+class ConvMklToTF : public OpsTestBase {
+ public:
+  template <typename T>
+  void ConvertMKL2TF(DataType dtype, const Tensor& first, const Tensor& second,
+                     Tensor& output) {
+    // Create an MKL to TF conversion node and execute it
+    TF_EXPECT_OK(NodeDefBuilder("mkl_to_tf_op", "_MklToTf")
+                     .Input(FakeInput(dtype))     // Input
+                     .Input(FakeInput(DT_UINT8))  // Mkl second tensor
+                     .Attr("T", dtype)
+                     .Attr("_kernel", "MklOp")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+    AddInputFromArray<T>(first.shape(), first.flat<T>());
+    AddInputFromArray<uint8>(second.shape(), second.flat<uint8>());
+    TF_ASSERT_OK(RunOpKernel());
+
+    output = *GetOutput(0);
+  }
+  void TestBody() {};
+};
+
+class QuantizedPoolingTest : public OpsTestBase {};
+
+TEST_F(QuantizedPoolingTest, SmallAveragePooling) {
+  const int ksize = 2;
+  const int stride = 2;
+  TF_ASSERT_OK(NodeDefBuilder("quantized_avg_pool_op", "_MklQuantizedAvgPool")
+                   .Input(FakeInput(DT_QUINT8))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                   .Attr("T", DataTypeToEnum<quint8>::v())
+                   .Attr("ksize", {1, ksize, ksize, 1})
+                   .Attr("strides", {1, stride, stride, 1})
+                   .Attr("padding", "SAME")
+                   .Attr("_kernel", "QuantizedMklOp")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  const float input_min = 0.0f;
+  const float input_max = 255.0f;
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+  Tensor input_float(DT_FLOAT, {1, input_height, input_width, input_channels});
+  test::FillValues<float>(
+      &input_float,
+      {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16,
+       17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32});
+  Tensor input_quantized =
+      FloatTensorToQuantized<quint8>(input_float, input_min, input_max);
+
+  const int expected_width = input_width / stride;
+  const int expected_height = input_height / stride;
+
+  // The input pools we are averaging. (NHWC input, quantized.)
+  //   0th channel       1st channel
+  //   1  3 |  5  7      2  4 |  6  8
+  //   9 11 | 13 15     10 12 | 14 16
+  //   -------------     -------------
+  //   17 19 | 21 23     18 20 | 22 24
+  //   25 27 | 29 31     26 28 | 30 32
+  Tensor expected_float(DT_FLOAT,
+                        {1, expected_height, expected_width, input_channels});
+  test::FillValues<float>(&expected_float, {6, 7, 10, 11, 22, 23, 26, 27});
+
+  AddInputFromArray<quint8>(input_quantized.shape(),
+                            input_quantized.flat<quint8>());
+  AddInputFromArray<float>(TensorShape({1}), {input_min});
+  AddInputFromArray<float>(TensorShape({1}), {input_max});
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+
+  TF_ASSERT_OK(RunOpKernel());
+
+  const Tensor& output = *GetOutput(0);
+  const Tensor& mkl_shape_tensor = *GetOutput(3);
+  ConvMklToTF conv_comp;
+  Tensor output_quantized;
+  conv_comp.ConvertMKL2TF<quint8>(DT_QUINT8, output, mkl_shape_tensor,
+                                  output_quantized);
+
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  Tensor output_float =
+      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+
+  test::ExpectTensorNear<float>(expected_float, output_float, 0.2);
+}
+
+TEST_F(QuantizedPoolingTest, SmallMaxPooling) {
+  const int ksize = 2;
+  const int stride = 2;
+  TF_ASSERT_OK(NodeDefBuilder("quantized_max_pool_op", "_MklQuantizedMaxPool")
+                   .Input(FakeInput(DT_QUINT8))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                   .Input(FakeInput(DT_UINT8))  // MKl second tensor
+                   .Attr("T", DataTypeToEnum<quint8>::v())
+                   .Attr("ksize", {1, ksize, ksize, 1})
+                   .Attr("strides", {1, stride, stride, 1})
+                   .Attr("padding", "SAME")
+                   .Attr("_kernel", "QuantizedMklOp")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  const float input_min = 0.0f;
+  const float input_max = 255.0f;
+  const int input_height = 4;
+  const int input_width = 4;
+  const int input_channels = 2;
+  Tensor input_float(DT_FLOAT, {1, input_height, input_width, input_channels});
+  test::FillValues<float>(
+      &input_float,
+      {1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16,
+       17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32});
+  Tensor input_quantized =
+      FloatTensorToQuantized<quint8>(input_float, input_min, input_max);
+  const int expected_width = input_width / stride;
+  const int expected_height = input_height / stride;
+
+  // The max is computed from these input pools. (NHWC input, quantized.)
+  //   0th channel       1st channel
+  //   1  3 |  5  7      2  4 |  6  8
+  //   9 11 | 13 15     10 12 | 14 16
+  //   -------------     -------------
+  //   17 19 | 21 23     18 20 | 22 24
+  //   25 27 | 29 31     26 28 | 30 32
+
+  Tensor expected_float(DT_FLOAT,
+                        {1, expected_height, expected_width, input_channels});
+  test::FillValues<float>(&expected_float, {11, 12, 15, 16, 27, 28, 31, 32});
+  AddInputFromArray<quint8>(input_quantized.shape(),
+                            input_quantized.flat<quint8>());
+  AddInputFromArray<float>(TensorShape({1}), {input_min});
+  AddInputFromArray<float>(TensorShape({1}), {input_max});
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+  AddInputFromArray<uint8>(dummy_shape, dummy_tensor);
+
+  TF_ASSERT_OK(RunOpKernel());
+
+  const Tensor& output = *GetOutput(0);
+  const Tensor& mkl_shape_tensor = *GetOutput(3);
+  ConvMklToTF conv_comp;
+  Tensor output_quantized;
+  conv_comp.ConvertMKL2TF<quint8>(DT_QUINT8, output, mkl_shape_tensor,
+                                  output_quantized);
+
+  const float output_min = GetOutput(1)->flat<float>()(0);
+  const float output_max = GetOutput(2)->flat<float>()(0);
+  Tensor output_float =
+      QuantizedTensorToFloat<quint8>(output_quantized, output_min, output_max);
+
+  test::ExpectTensorNear<float>(expected_float, output_float, 0.2);
+}
+}  // namespace tensorflow
+#endif
-- 
GitLab


From 3bace0cdcd9e773462ae95e26d1e75c4d0c64908 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Wed, 28 Nov 2018 09:00:27 -0800
Subject: [PATCH 0928/1554] Fixed a bug in quantized convolution

---
 tensorflow/core/kernels/mkl_conv_ops.cc | 162 ++++++++++++------------
 1 file changed, 82 insertions(+), 80 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index dc6f783623..8e3b669a59 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -323,7 +323,7 @@ class MklConvFwdPrimitiveFactory : public MklPrimitiveFactory<T> {
       const MklConvFwdParams& convFwdDims, bool do_not_cache) {
     MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>* conv_fwd = nullptr;
 
-    if (do_not_cache) { /* Always create new primitive */
+    if (do_not_cache) {/* Always create new primitive */
       conv_fwd = new MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>(
           convFwdDims);
     } else {
@@ -423,15 +423,16 @@ class MklConvOp : public OpKernel {
     OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES(context, strides_.size() == 4,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window strides field must "
+                    "specify 4 dimensions"));
 
     const int64 stride_n = GetTensorDim(strides_, data_format_, 'N');
     const int64 stride_c = GetTensorDim(strides_, data_format_, 'C');
-    OP_REQUIRES(
-        context, stride_n == 1 && stride_c == 1,
-        errors::InvalidArgument("Current implementation does not yet support "
-                                "strides in the batch and depth dimensions."));
+    OP_REQUIRES(context, stride_n == 1 && stride_c == 1,
+                errors::InvalidArgument(
+                    "Current implementation does not yet support "
+                    "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
   }
 
@@ -465,19 +466,18 @@ class MklConvOp : public OpKernel {
                                         filter.shape().DebugString()));
 
     for (int i = 0; i < 3; i++) {
-      OP_REQUIRES(
-          context,
-          FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()),
-          errors::InvalidArgument("filter too large"));
+      OP_REQUIRES(context, FastBoundsCheck(filter.dim_size(i),
+                                           std::numeric_limits<int>::max()),
+                  errors::InvalidArgument("filter too large"));
     }
 
     const int64 input_depth =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'C')
                             : GetTensorDim(input, data_format_, 'C');
-    OP_REQUIRES(context, input_depth == filter.dim_size(2),
-                errors::InvalidArgument(
-                    "input and filter must have the same depth: ", input_depth,
-                    " vs ", filter.dim_size(2)));
+    OP_REQUIRES(
+        context, input_depth == filter.dim_size(2),
+        errors::InvalidArgument("input and filter must have the same depth: ",
+                                input_depth, " vs ", filter.dim_size(2)));
     // The last dimension for filter is out_depth.
     const int out_depth = static_cast<int>(filter.dim_size(3));
 
@@ -486,10 +486,9 @@ class MklConvOp : public OpKernel {
     const int64 input_rows_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'H')
                             : GetTensorDim(input, data_format_, 'H');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_rows_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input rows too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_rows_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input rows too large"));
     const int input_rows = static_cast<int>(input_rows_raw);
     const int filter_rows = static_cast<int>(filter.dim_size(0));
 
@@ -498,10 +497,9 @@ class MklConvOp : public OpKernel {
     const int64 input_cols_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'W')
                             : GetTensorDim(input, data_format_, 'W');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_cols_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input cols too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_cols_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("Input cols too large"));
     const int input_cols = static_cast<int>(input_cols_raw);
     const int filter_cols = static_cast<int>(filter.dim_size(1));
 
@@ -509,10 +507,9 @@ class MklConvOp : public OpKernel {
     const int64 input_batch_raw =
         input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'N')
                             : GetTensorDim(input, data_format_, 'N');
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(input_batch_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("batch is too large"));
+    OP_REQUIRES(context, FastBoundsCheck(input_batch_raw,
+                                         std::numeric_limits<int>::max()),
+                errors::InvalidArgument("batch is too large"));
     const int batch = static_cast<int>(input_batch_raw);
 
     // For now we take the stride from the second and third dimensions only (we
@@ -730,7 +727,7 @@ class MklConvOp : public OpKernel {
           mkl_prim_convert_input;
       dnnLayout_t mkl_lt_internal_filter, mkl_lt_internal_bias,
           mkl_lt_internal_input;
-      void *mkl_buf_convert_input, *mkl_buf_convert_filter,
+      void* mkl_buf_convert_input, *mkl_buf_convert_filter,
           *mkl_buf_convert_bias;
       mkl_prim_convert_filter = nullptr;
       mkl_prim_convert_bias = nullptr;
@@ -863,21 +860,23 @@ class MklConvOp : public OpKernel {
     OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES(context, (strides_.size() == 4 || strides_.size() == 5),
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 or 5 dimensions"));
+                errors::InvalidArgument(
+                    "Sliding window strides field must "
+                    "specify 4 or 5 dimensions"));
 
     const int64 stride_n = GetTensorDim(strides_, data_format_, 'N');
     const int64 stride_c = GetTensorDim(strides_, data_format_, 'C');
-    OP_REQUIRES(
-        context, stride_n == 1 && stride_c == 1,
-        errors::InvalidArgument("Current implementation does not yet support "
-                                "strides in the batch and depth dimensions."));
+    OP_REQUIRES(context, stride_n == 1 && stride_c == 1,
+                errors::InvalidArgument(
+                    "Current implementation does not yet support "
+                    "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
 
     if (strides_.size() == 4) {
       OP_REQUIRES(context, dilations_.size() == 4,
-                  errors::InvalidArgument("Sliding window dilations field must "
-                                          "specify 4 dimensions"));
+                  errors::InvalidArgument(
+                      "Sliding window dilations field must "
+                      "specify 4 dimensions"));
       const int64 dilation_n = GetTensorDim(dilations_, data_format_, 'N');
       const int64 dilation_c = GetTensorDim(dilations_, data_format_, 'C');
       const int64 dilation_h = GetTensorDim(dilations_, data_format_, 'H');
@@ -891,19 +890,18 @@ class MklConvOp : public OpKernel {
           errors::InvalidArgument("Dilated rates should be larger than 0."));
     } else if (strides_.size() == 5) {
       OP_REQUIRES(context, dilations_.size() == 5,
-                  errors::InvalidArgument("Dilation rates field must "
-                                          "specify 5 dimensions"));
-      OP_REQUIRES(context,
-                  (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
-                   GetTensorDim(dilations_, data_format_, 'C') == 1),
+                  errors::InvalidArgument(
+                      "Dilation rates field must "
+                      "specify 5 dimensions"));
+      OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
+                            GetTensorDim(dilations_, data_format_, 'C') == 1),
                   errors::InvalidArgument(
                       "Current implementation does not yet support "
                       "dilations rates in the batch and depth dimensions."));
       OP_REQUIRES(
-          context,
-          (GetTensorDim(dilations_, data_format_, '0') > 0 &&
-           GetTensorDim(dilations_, data_format_, '1') > 0 &&
-           GetTensorDim(dilations_, data_format_, '2') > 0),
+          context, (GetTensorDim(dilations_, data_format_, '0') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '1') > 0 &&
+                    GetTensorDim(dilations_, data_format_, '2') > 0),
           errors::InvalidArgument("Dilated rates should be larger than 0."));
     }
   }
@@ -918,8 +916,9 @@ class MklConvOp : public OpKernel {
       GetMklShape(context, kInputIndex_Src, &src_mkl_shape);
       GetMklShape(context, kInputIndex_Filter, &filter_mkl_shape);
       OP_REQUIRES(context, filter_mkl_shape.IsMklTensor() == false,
-                  errors::InvalidArgument("Filter should not be in "
-                                          "Mkl Layout"));
+                  errors::InvalidArgument(
+                      "Filter should not be in "
+                      "Mkl Layout"));
 
       MklDnnData<Tinput> src(&cpu_engine_);
       MklDnnData<Tfilter> filter(&cpu_engine_);
@@ -955,9 +954,8 @@ class MklConvOp : public OpKernel {
         filter_mkl_shape.SetMklTensor(false);
         Tensor* output_filter_tensor = nullptr;
         // MklConv2D also outputs converted filter as 2nd output.
-        if (typeid(Tinput) == typeid(float) &&
-            typeid(Tfilter) == typeid(float) &&
-            typeid(Toutput) == typeid(float)) {
+        if (typeid(Tinput) == typeid(float)&&typeid(Tfilter) ==
+            typeid(float)&&typeid(Toutput) == typeid(float)) {
           filter_mkl_shape.SetMklTensor(false);
           AllocateOutputSetMklShape(context, kOutputIndex_Filter,
                                     &output_filter_tensor, filter_tf_shape,
@@ -1044,8 +1042,8 @@ class MklConvOp : public OpKernel {
       AllocateOutputTensor(context, *conv_fwd_pd, dst_dims_mkl_order, tf_fmt,
                            &dst_tensor);
       Tensor* filter_out_tensor = nullptr;
-      if (typeid(Tinput) == typeid(float) && typeid(Tfilter) == typeid(float) &&
-          typeid(Toutput) == typeid(float)) {
+      if (typeid(Tinput) == typeid(float)&&typeid(Tfilter) ==
+          typeid(float)&&typeid(Toutput) == typeid(float)) {
         AllocateFilterOutputTensor(context, *conv_fwd_pd,
                                    TFShapeToMklDnnDims(filter_tf_shape),
                                    &filter_out_tensor);
@@ -1067,8 +1065,14 @@ class MklConvOp : public OpKernel {
       Tfilter* filter_data = nullptr;
       if (filter_md.data.format != conv_fwd->GetFilterMemoryFormat()) {
         filter.SetUsrMem(filter_md, &filter_tensor);
-        filter.CheckReorderToOpMem(conv_fwd_pd.get()->weights_primitive_desc(),
-                                   filter.GetTensorBuffer(filter_out_tensor));
+        if (filter_out_tensor == nullptr) {
+          filter.CheckReorderToOpMem(
+              conv_fwd_pd.get()->weights_primitive_desc());
+        } else {
+          filter.CheckReorderToOpMem(
+              conv_fwd_pd.get()->weights_primitive_desc(),
+              filter.GetTensorBuffer(filter_out_tensor));
+        }
         filter_data =
             static_cast<Tfilter*>(filter.GetOpMem().get_data_handle());
       } else {
@@ -1088,7 +1092,8 @@ class MklConvOp : public OpKernel {
 
       // delete primitive since it is not cached.
       if (do_not_cache) delete conv_fwd;
-    } catch (mkldnn::error& e) {
+    }
+    catch (mkldnn::error& e) {
       string error_msg = tensorflow::strings::StrCat(
           "Status: ", e.status, ", message: ", string(e.message), ", in file ",
           __FILE__, ":", __LINE__);
@@ -1533,8 +1538,8 @@ class MklQuantizedConv2DSumReluOp
     const float max_filter =
         context->input(5 + bias_index_offset).flat<float>()(0);
 
-    reorder_sum_scale = 255.0 * 127.0 /
-                        (std::max(std::abs(max_input), std::abs(min_input)) *
+    reorder_sum_scale =
+        255.0 * 127.0 / (std::max(std::abs(max_input), std::abs(min_input)) *
                          std::max(std::abs(max_filter), std::abs(min_filter)));
     std::vector<float> scales;
     scales.push_back(reorder_sum_scale);
@@ -1784,34 +1789,31 @@ REGISTER_KERNEL_BUILDER(
 #endif  // INTEL_MKL_ML
 
 // Register 2D operations
-#define REGISTER_MKL_CPU_2D(T)                                         \
-  REGISTER_KERNEL_BUILDER(                                             \
-      Name("_MklConv2D")                                               \
-          .Device(DEVICE_CPU)                                          \
-          .TypeConstraint<float>("T")                                  \
-          .Label(mkl_op_registry::kMklOpLabel),                        \
-      MklConvOp<CPUDevice, float, float, float, float, float, false>); \
-  REGISTER_KERNEL_BUILDER(                                             \
-      Name("_MklConv2DWithBias")                                       \
-          .Device(DEVICE_CPU)                                          \
-          .TypeConstraint<float>("T")                                  \
-          .Label(mkl_op_registry::kMklOpLabel),                        \
-      MklConvOp<CPUDevice, float, float, float, float, float, true>);  \
-  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")             \
-                              .Device(DEVICE_CPU)                      \
-                              .TypeConstraint<T>("T")                  \
-                              .Label(mkl_op_registry::kMklOpLabel),    \
+#define REGISTER_MKL_CPU_2D(T)                                                \
+  REGISTER_KERNEL_BUILDER(                                                    \
+      Name("_MklConv2D").Device(DEVICE_CPU).TypeConstraint<float>("T").Label( \
+          mkl_op_registry::kMklOpLabel),                                      \
+      MklConvOp<CPUDevice, float, float, float, float, float, false>);        \
+  REGISTER_KERNEL_BUILDER(                                                    \
+      Name("_MklConv2DWithBias")                                              \
+          .Device(DEVICE_CPU)                                                 \
+          .TypeConstraint<float>("T")                                         \
+          .Label(mkl_op_registry::kMklOpLabel),                               \
+      MklConvOp<CPUDevice, float, float, float, float, float, true>);         \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")                    \
+                              .Device(DEVICE_CPU)                             \
+                              .TypeConstraint<T>("T")                         \
+                              .Label(mkl_op_registry::kMklOpLabel),           \
                           MklDummyOp<CPUDevice, T>);
 
 TF_CALL_float(REGISTER_MKL_CPU_2D);
 
 // Register 3D operations
-#define REGISTER_MKL_CPU_3D(T)                                      \
-  REGISTER_KERNEL_BUILDER(Name("_MklConv3D")                        \
-                              .Device(DEVICE_CPU)                   \
-                              .TypeConstraint<T>("T")               \
-                              .Label(mkl_op_registry::kMklOpLabel), \
-                          MklConvOp<CPUDevice, T, T, T, T, T, false>);
+#define REGISTER_MKL_CPU_3D(T)                                            \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("_MklConv3D").Device(DEVICE_CPU).TypeConstraint<T>("T").Label( \
+          mkl_op_registry::kMklOpLabel),                                  \
+      MklConvOp<CPUDevice, T, T, T, T, T, false>);
 TF_CALL_float(REGISTER_MKL_CPU_3D);
 
 }  // namespace tensorflow
-- 
GitLab


From 0ae35dd8c952350208f443b4c47c538ef3edb728 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 28 Nov 2018 09:03:01 -0800
Subject: [PATCH 0929/1554] Fix path to update script.

PiperOrigin-RevId: 223177077
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 001aa22227..34c600abaf 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -107,7 +107,7 @@ CONSOLE_SCRIPTS = [
     # TensorBoard command, pip will inappropriately remove it during install,
     # even though the command is not removed, just moved to a different wheel.
     'tensorboard = tensorboard.main:run_main',
-    'tf_upgrade_v2 = tensorflow.python.tools.compatibility.tf_upgrade_v2:main',
+    'tf_upgrade_v2 = tensorflow.tools.compatibility.tf_upgrade_v2:main',
 ]
 # pylint: enable=line-too-long
 
-- 
GitLab


From 68f506dd57f9dc84208f518c08a10af031bd582f Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 28 Nov 2018 09:06:42 -0800
Subject: [PATCH 0930/1554] Remove deprecated session-based tf.train._ classes
 from TF 2.0 API.

PiperOrigin-RevId: 223177761
---
 .../python/training/monitored_session.py      |  10 +-
 tensorflow/python/training/session_manager.py |   2 +-
 tensorflow/python/training/supervisor.py      |   2 +-
 ...sorflow.train.-chief-session-creator.pbtxt |  14 --
 ...ain.-monitored-session.-step-context.pbtxt |  21 ---
 .../tensorflow.train.-monitored-session.pbtxt |  34 ----
 .../tensorflow.train.-session-creator.pbtxt   |  12 --
 .../tensorflow.train.-session-manager.pbtxt   |  21 ---
 ...ular-monitored-session.-step-context.pbtxt |  21 ---
 ...ow.train.-singular-monitored-session.pbtxt |  38 -----
 .../v2/tensorflow.train.-supervisor.pbtxt     | 153 ------------------
 ...orflow.train.-worker-session-creator.pbtxt |  14 --
 .../api/golden/v2/tensorflow.train.pbtxt      |  28 ----
 tensorflow/tools/compatibility/renames_v2.py  |   9 +-
 14 files changed, 15 insertions(+), 364 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt

diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index ccd9014648..c40bd2b859 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -508,7 +508,7 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
       stop_grace_period_secs=stop_grace_period_secs)
 
 
-@tf_export('train.SessionCreator')
+@tf_export(v1=['train.SessionCreator'])
 @six.add_metaclass(abc.ABCMeta)
 class SessionCreator(object):
   """A factory for tf.Session."""
@@ -519,7 +519,7 @@ class SessionCreator(object):
         'create_session is not implemented for {}.'.format(self))
 
 
-@tf_export('train.ChiefSessionCreator')
+@tf_export(v1=['train.ChiefSessionCreator'])
 class ChiefSessionCreator(SessionCreator):
   """Creates a tf.Session for a chief."""
 
@@ -571,7 +571,7 @@ class ChiefSessionCreator(SessionCreator):
         init_fn=self._scaffold.init_fn)
 
 
-@tf_export('train.WorkerSessionCreator')
+@tf_export(v1=['train.WorkerSessionCreator'])
 class WorkerSessionCreator(SessionCreator):
   """Creates a tf.Session for a worker."""
 
@@ -851,7 +851,7 @@ class _MonitoredSession(object):
     return self._coordinated_creator.tf_sess
 
 
-@tf_export('train.MonitoredSession')
+@tf_export(v1=['train.MonitoredSession'])
 class MonitoredSession(_MonitoredSession):
   """Session-like object that handles initialization, recovery and hooks.
 
@@ -934,7 +934,7 @@ class MonitoredSession(_MonitoredSession):
         stop_grace_period_secs=stop_grace_period_secs)
 
 
-@tf_export('train.SingularMonitoredSession')
+@tf_export(v1=['train.SingularMonitoredSession'])
 class SingularMonitoredSession(_MonitoredSession):
   """Session-like object that handles initialization, restoring, and hooks.
 
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index cd313c2ce0..14658630c5 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -46,7 +46,7 @@ def _maybe_name(obj):
     return "<no name for %s>" % type(obj)
 
 
-@tf_export("train.SessionManager")
+@tf_export(v1=["train.SessionManager"])
 class SessionManager(object):
   """Training helper that restores from checkpoint and creates session.
 
diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py
index a5e626d320..de60dd456f 100644
--- a/tensorflow/python/training/supervisor.py
+++ b/tensorflow/python/training/supervisor.py
@@ -40,7 +40,7 @@ from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.Supervisor")
+@tf_export(v1=["train.Supervisor"])
 class Supervisor(object):
   """A training helper that checkpoints models and computes summaries.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt
deleted file mode 100644
index abbe273be3..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-chief-session-creator.pbtxt
+++ /dev/null
@@ -1,14 +0,0 @@
-path: "tensorflow.train.ChiefSessionCreator"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.ChiefSessionCreator\'>"
-  is_instance: "<class \'tensorflow.python.training.monitored_session.SessionCreator\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'scaffold\', \'master\', \'config\', \'checkpoint_dir\', \'checkpoint_filename_with_path\'], varargs=None, keywords=None, defaults=[\'None\', \'\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "create_session"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt
deleted file mode 100644
index 03efe6639e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.-step-context.pbtxt
+++ /dev/null
@@ -1,21 +0,0 @@
-path: "tensorflow.train.MonitoredSession.StepContext"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.StepContext\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "session"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "request_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "run_with_hooks"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt
deleted file mode 100644
index 09b7b3fb53..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-monitored-session.pbtxt
+++ /dev/null
@@ -1,34 +0,0 @@
-path: "tensorflow.train.MonitoredSession"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.MonitoredSession\'>"
-  is_instance: "<class \'tensorflow.python.training.monitored_session._MonitoredSession\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "StepContext"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "graph"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'session_creator\', \'hooks\', \'stop_grace_period_secs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'120\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "run"
-    argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "run_step_fn"
-    argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "should_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt
deleted file mode 100644
index beb232715f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-creator.pbtxt
+++ /dev/null
@@ -1,12 +0,0 @@
-path: "tensorflow.train.SessionCreator"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.SessionCreator\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "create_session"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt
deleted file mode 100644
index 448764fe08..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-manager.pbtxt
+++ /dev/null
@@ -1,21 +0,0 @@
-path: "tensorflow.train.SessionManager"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.session_manager.SessionManager\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'local_init_op\', \'ready_op\', \'ready_for_local_init_op\', \'graph\', \'recovery_wait_secs\', \'local_init_run_options\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'30\', \'None\'], "
-  }
-  member_method {
-    name: "prepare_session"
-    argspec: "args=[\'self\', \'master\', \'init_op\', \'saver\', \'checkpoint_dir\', \'checkpoint_filename_with_path\', \'wait_for_checkpoint\', \'max_wait_secs\', \'config\', \'init_feed_dict\', \'init_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'7200\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "recover_session"
-    argspec: "args=[\'self\', \'master\', \'saver\', \'checkpoint_dir\', \'checkpoint_filename_with_path\', \'wait_for_checkpoint\', \'max_wait_secs\', \'config\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'7200\', \'None\'], "
-  }
-  member_method {
-    name: "wait_for_session"
-    argspec: "args=[\'self\', \'master\', \'config\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'None\', \'inf\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
deleted file mode 100644
index 36d8ce7ff8..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
+++ /dev/null
@@ -1,21 +0,0 @@
-path: "tensorflow.train.SingularMonitoredSession.StepContext"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.StepContext\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "session"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "request_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "run_with_hooks"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt
deleted file mode 100644
index de0f2c1c1a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-singular-monitored-session.pbtxt
+++ /dev/null
@@ -1,38 +0,0 @@
-path: "tensorflow.train.SingularMonitoredSession"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.SingularMonitoredSession\'>"
-  is_instance: "<class \'tensorflow.python.training.monitored_session._MonitoredSession\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "StepContext"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "graph"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'hooks\', \'scaffold\', \'master\', \'config\', \'checkpoint_dir\', \'stop_grace_period_secs\', \'checkpoint_filename_with_path\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'None\', \'None\', \'120\', \'None\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "raw_session"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "run"
-    argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "run_step_fn"
-    argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "should_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt
deleted file mode 100644
index 9677e5a98e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-supervisor.pbtxt
+++ /dev/null
@@ -1,153 +0,0 @@
-path: "tensorflow.train.Supervisor"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.supervisor.Supervisor\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "USE_DEFAULT"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "coord"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "global_step"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "init_feed_dict"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "init_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "is_chief"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "ready_for_local_init_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "ready_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "save_model_secs"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "save_path"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "save_summaries_secs"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "saver"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "session_manager"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "summary_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "summary_writer"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "Loop"
-    argspec: "args=[\'self\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "PrepareSession"
-    argspec: "args=[\'self\', \'master\', \'config\', \'wait_for_checkpoint\', \'max_wait_secs\', \'start_standard_services\'], varargs=None, keywords=None, defaults=[\'\', \'None\', \'False\', \'7200\', \'True\'], "
-  }
-  member_method {
-    name: "RequestStop"
-    argspec: "args=[\'self\', \'ex\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "ShouldStop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "StartQueueRunners"
-    argspec: "args=[\'self\', \'sess\', \'queue_runners\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "StartStandardServices"
-    argspec: "args=[\'self\', \'sess\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Stop"
-    argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], "
-  }
-  member_method {
-    name: "StopOnException"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SummaryComputed"
-    argspec: "args=[\'self\', \'sess\', \'summary\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "WaitForStop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'graph\', \'ready_op\', \'ready_for_local_init_op\', \'is_chief\', \'init_op\', \'init_feed_dict\', \'local_init_op\', \'logdir\', \'summary_op\', \'saver\', \'global_step\', \'save_summaries_secs\', \'save_model_secs\', \'recovery_wait_secs\', \'stop_grace_secs\', \'checkpoint_basename\', \'session_manager\', \'summary_writer\', \'init_fn\', \'local_init_run_options\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'0\', \'True\', \'0\', \'None\', \'0\', \'None\', \'0\', \'0\', \'0\', \'120\', \'600\', \'30\', \'120\', \'model.ckpt\', \'None\', \'0\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "loop"
-    argspec: "args=[\'self\', \'timer_interval_secs\', \'target\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "managed_session"
-    argspec: "args=[], varargs=args, keywords=kwds, defaults=None"
-  }
-  member_method {
-    name: "prepare_or_wait_for_session"
-    argspec: "args=[\'self\', \'master\', \'config\', \'wait_for_checkpoint\', \'max_wait_secs\', \'start_standard_services\'], varargs=None, keywords=None, defaults=[\'\', \'None\', \'False\', \'7200\', \'True\'], "
-  }
-  member_method {
-    name: "request_stop"
-    argspec: "args=[\'self\', \'ex\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "should_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "start_queue_runners"
-    argspec: "args=[\'self\', \'sess\', \'queue_runners\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "start_standard_services"
-    argspec: "args=[\'self\', \'sess\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "stop"
-    argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], "
-  }
-  member_method {
-    name: "stop_on_exception"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "summary_computed"
-    argspec: "args=[\'self\', \'sess\', \'summary\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "wait_for_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt
deleted file mode 100644
index ac26358068..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-worker-session-creator.pbtxt
+++ /dev/null
@@ -1,14 +0,0 @@
-path: "tensorflow.train.WorkerSessionCreator"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.WorkerSessionCreator\'>"
-  is_instance: "<class \'tensorflow.python.training.monitored_session.SessionCreator\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'scaffold\', \'master\', \'config\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'None\', \'\', \'None\', \'1800\'], "
-  }
-  member_method {
-    name: "create_session"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 4a4f387131..a30f67350a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -20,10 +20,6 @@ tf_module {
     name: "CheckpointSaverListener"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "ChiefSessionCreator"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ClusterDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -88,10 +84,6 @@ tf_module {
     name: "LoggingTensorHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "MonitoredSession"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "NanLossDuringTrainingError"
     mtype: "<type \'type\'>"
@@ -124,14 +116,6 @@ tf_module {
     name: "ServerDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "SessionCreator"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "SessionManager"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "SessionRunArgs"
     mtype: "<type \'type\'>"
@@ -148,10 +132,6 @@ tf_module {
     name: "SessionRunValues"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "SingularMonitoredSession"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "StepCounterHook"
     mtype: "<type \'type\'>"
@@ -164,14 +144,6 @@ tf_module {
     name: "SummarySaverHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Supervisor"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "WorkerSessionCreator"
-    mtype: "<type \'type\'>"
-  }
   member_method {
     name: "cosine_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 49c152bb86..042ca8adaa 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -117,7 +117,7 @@ renames = {
     'tf.debugging.is_strictly_increasing': 'tf.math.is_strictly_increasing',
     'tf.decode_base64': 'tf.io.decode_base64',
     'tf.decode_compressed': 'tf.io.decode_compressed',
-    'tf.decode_csv': 'tf.io.decode_csv',
+    'tf.decode_csv': 'tf.compat.v1.decode_csv',
     'tf.decode_json_example': 'tf.io.decode_json_example',
     'tf.decode_raw': 'tf.io.decode_raw',
     'tf.delete_session_tensor': 'tf.compat.v1.delete_session_tensor',
@@ -580,6 +580,8 @@ renames = {
     'tf.to_int32': 'tf.compat.v1.to_int32',
     'tf.to_int64': 'tf.compat.v1.to_int64',
     'tf.trace': 'tf.linalg.trace',
+    'tf.train.ChiefSessionCreator': 'tf.compat.v1.train.ChiefSessionCreator',
+    'tf.train.MonitoredSession': 'tf.compat.v1.train.MonitoredSession',
     'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread',
     'tf.train.AdadeltaOptimizer': 'tf.compat.v1.train.AdadeltaOptimizer',
     'tf.train.AdagradDAOptimizer': 'tf.compat.v1.train.AdagradDAOptimizer',
@@ -597,7 +599,12 @@ renames = {
     'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
     'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef',
+    'tf.train.SessionCreator': 'tf.compat.v1.train.SessionCreator',
+    'tf.train.SessionManager': 'tf.compat.v1.train.SessionManager',
+    'tf.train.SingularMonitoredSession': 'tf.compat.v1.train.SingularMonitoredSession',
+    'tf.train.Supervisor': 'tf.compat.v1.train.Supervisor',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
+    'tf.train.WorkerSessionCreator': 'tf.compat.v1.train.WorkerSessionCreator',
     'tf.train.VocabInfo': 'tf.compat.v1.train.VocabInfo',
     'tf.train.add_queue_runner': 'tf.compat.v1.train.add_queue_runner',
     'tf.train.assert_global_step': 'tf.compat.v1.train.assert_global_step',
-- 
GitLab


From 938d66d0f2c1507879174c509382de53f19ea2c9 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Wed, 28 Nov 2018 09:09:55 -0800
Subject: [PATCH 0931/1554] [TF:XLA] Remove unused builder arg from
 XlaHelpers::ConvertElementType

PiperOrigin-RevId: 223178269
---
 .../compiler/tf2xla/kernels/batch_norm_op.cc    | 15 +++++++--------
 tensorflow/compiler/tf2xla/kernels/bias_ops.cc  |  4 ++--
 .../tf2xla/kernels/fake_quantize_ops.cc         |  8 ++++----
 tensorflow/compiler/tf2xla/kernels/image_ops.cc |  5 ++---
 tensorflow/compiler/tf2xla/kernels/l2loss_op.cc |  5 ++---
 tensorflow/compiler/tf2xla/kernels/lrn_ops.cc   | 17 ++++++-----------
 .../tf2xla/kernels/reduction_ops_common.cc      |  2 +-
 tensorflow/compiler/tf2xla/kernels/scan_ops.cc  |  7 +++----
 .../compiler/tf2xla/kernels/softmax_op.cc       | 15 +++++++--------
 .../compiler/tf2xla/kernels/training_ops.cc     |  3 +--
 tensorflow/compiler/tf2xla/xla_helpers.cc       |  3 +--
 tensorflow/compiler/tf2xla/xla_helpers.h        |  3 +--
 12 files changed, 37 insertions(+), 50 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
index a267c0c72f..0e2f335f33 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
@@ -115,9 +115,9 @@ class FusedBatchNormGradOp : public XlaOpKernel {
     // operators. For now, cast everything to the statistics type (which
     // may be more precise than the input type).
     auto grad_backprop =
-        XlaHelpers::ConvertElementType(b, ctx->Input(0), scale_dtype);
+        XlaHelpers::ConvertElementType(ctx->Input(0), scale_dtype);
     auto activations =
-        XlaHelpers::ConvertElementType(b, ctx->Input(1), scale_dtype);
+        XlaHelpers::ConvertElementType(ctx->Input(1), scale_dtype);
     auto scale = ctx->Input(2);
     auto mean = ctx->Input(3);
     auto var = ctx->Input(4);
@@ -151,11 +151,11 @@ class FusedBatchNormGradOp : public XlaOpKernel {
       const DataType accumulation_type =
           XlaHelpers::SumAccumulationType(scale_dtype);
       auto converted =
-          XlaHelpers::ConvertElementType(b, grad_backprop, accumulation_type);
+          XlaHelpers::ConvertElementType(grad_backprop, accumulation_type);
       auto reduce =
           xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                       *ctx->GetOrCreateAdd(accumulation_type), reduction_dims);
-      offset_backprop = XlaHelpers::ConvertElementType(b, reduce, scale_dtype);
+      offset_backprop = XlaHelpers::ConvertElementType(reduce, scale_dtype);
 
       // scratch1 = rsqrt(pop_var + epsilon)
       auto neg_half = XlaHelpers::FloatLiteral(b, scale_dtype, -0.5);
@@ -165,19 +165,18 @@ class FusedBatchNormGradOp : public XlaOpKernel {
       // scratch2 = sum(y_backprop * (x - mean))
       auto mul =
           xla::Mul(grad_backprop, xla::Sub(activations, mean, {feature_index}));
-      converted = XlaHelpers::ConvertElementType(b, mul, accumulation_type);
+      converted = XlaHelpers::ConvertElementType(mul, accumulation_type);
       reduce =
           xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                       *ctx->GetOrCreateAdd(accumulation_type), reduction_dims);
-      auto scratch2 = XlaHelpers::ConvertElementType(b, reduce, scale_dtype);
+      auto scratch2 = XlaHelpers::ConvertElementType(reduce, scale_dtype);
 
       x_backprop =
           xla::Mul(grad_backprop, xla::Mul(scratch1, scale), {feature_index});
       scale_backprop = xla::Mul(scratch1, scratch2);
     }
 
-    ctx->SetOutput(0,
-                   XlaHelpers::ConvertElementType(b, x_backprop, input_dtype));
+    ctx->SetOutput(0, XlaHelpers::ConvertElementType(x_backprop, input_dtype));
     ctx->SetOutput(1, scale_backprop);
     ctx->SetOutput(2, offset_backprop);
     ctx->SetConstantOutput(3, Tensor());
diff --git a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
index 41f540506b..e7f369b761 100644
--- a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
@@ -107,11 +107,11 @@ class BiasAddGradOp : public XlaOpKernel {
     const DataType accumulation_type =
         XlaHelpers::SumAccumulationType(input_type(0));
     auto converted =
-        XlaHelpers::ConvertElementType(b, ctx->Input(0), accumulation_type);
+        XlaHelpers::ConvertElementType(ctx->Input(0), accumulation_type);
     auto reduce =
         xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                     *ctx->GetOrCreateAdd(accumulation_type), reduce_dims);
-    ctx->SetOutput(0, XlaHelpers::ConvertElementType(b, reduce, input_type(0)));
+    ctx->SetOutput(0, XlaHelpers::ConvertElementType(reduce, input_type(0)));
   }
 
  private:
diff --git a/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc b/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc
index cdba6680de..142be030f7 100644
--- a/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc
@@ -260,19 +260,19 @@ class FakeQuantWithMinMaxVarsGradOp : public XlaOpKernel {
     xla::XlaOp below_min = xla::Lt(input, nudged_input_min);
     xla::XlaOp select1 = xla::Select(below_min, gradient, zeroes);
     xla::XlaOp reduce1 = xla::ReduceAll(
-        XlaHelpers::ConvertElementType(b, select1, accumulation_type),
+        XlaHelpers::ConvertElementType(select1, accumulation_type),
         XlaHelpers::Zero(b, accumulation_type),
         *ctx->GetOrCreateAdd(accumulation_type));
-    xla::XlaOp output1 = XlaHelpers::ConvertElementType(b, reduce1, data_type);
+    xla::XlaOp output1 = XlaHelpers::ConvertElementType(reduce1, data_type);
     ctx->SetOutput(1, output1);
 
     xla::XlaOp above_max = xla::Gt(input, nudged_input_max);
     xla::XlaOp select2 = xla::Select(above_max, gradient, zeroes);
     xla::XlaOp reduce2 = xla::ReduceAll(
-        XlaHelpers::ConvertElementType(b, select2, accumulation_type),
+        XlaHelpers::ConvertElementType(select2, accumulation_type),
         XlaHelpers::Zero(b, accumulation_type),
         *ctx->GetOrCreateAdd(accumulation_type));
-    xla::XlaOp output2 = XlaHelpers::ConvertElementType(b, reduce2, data_type);
+    xla::XlaOp output2 = XlaHelpers::ConvertElementType(reduce2, data_type);
     ctx->SetOutput(2, output2);
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
index b49b2516d8..e9bb0a77e9 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
@@ -191,12 +191,11 @@ class AdjustContrastOpV2 : public XlaOpKernel {
     DataType type = context->input_type(0);
 
     const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
-    auto converted =
-        XlaHelpers::ConvertElementType(b, input, accumulation_type);
+    auto converted = XlaHelpers::ConvertElementType(input, accumulation_type);
     auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                               *context->GetOrCreateAdd(accumulation_type),
                               {height_dim, width_dim});
-    auto output = XlaHelpers::ConvertElementType(b, reduce, type);
+    auto output = XlaHelpers::ConvertElementType(reduce, type);
     output =
         xla::Div(output, XlaHelpers::FloatLiteral(b, type, height * width));
 
diff --git a/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc b/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc
index f028e361bc..93f029731c 100644
--- a/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc
@@ -37,12 +37,11 @@ class L2LossOp : public XlaOpKernel {
 
     //  output = sum(t ** 2) / 2
     const DataType accumulation_type = XlaHelpers::SumAccumulationType(dtype);
-    auto t =
-        XlaHelpers::ConvertElementType(b, ctx->Input(0), accumulation_type);
+    auto t = XlaHelpers::ConvertElementType(ctx->Input(0), accumulation_type);
     auto square = xla::Mul(t, t);
     auto reduce = xla::Reduce(square, XlaHelpers::Zero(b, accumulation_type),
                               *ctx->GetOrCreateAdd(accumulation_type), dims);
-    auto deconverted = XlaHelpers::ConvertElementType(b, reduce, dtype);
+    auto deconverted = XlaHelpers::ConvertElementType(reduce, dtype);
     auto two = XlaHelpers::IntegerLiteral(b, dtype, 2);
     ctx->SetOutput(0, xla::Div(deconverted, two));
   }
diff --git a/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc b/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc
index 87ee2d3aed..987901d82b 100644
--- a/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc
@@ -49,16 +49,14 @@ class LRNOp : public XlaOpKernel {
     // We use a window of depth_radius_ * 2 + 1, to account for the current
     // element and a depth_radius_ on either side.
     auto accumulation_type = XlaHelpers::SumAccumulationType(input_type(0));
-    auto converted =
-        XlaHelpers::ConvertElementType(builder, input, accumulation_type);
+    auto converted = XlaHelpers::ConvertElementType(input, accumulation_type);
     auto squared = xla::Mul(converted, converted);
     auto reduce = xla::ReduceWindow(
         squared, XlaHelpers::Zero(builder, accumulation_type),
         *ctx->GetOrCreateAdd(accumulation_type),
         /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1},
         /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame);
-    auto sqr_sum =
-        XlaHelpers::ConvertElementType(builder, reduce, input_type(0));
+    auto sqr_sum = XlaHelpers::ConvertElementType(reduce, input_type(0));
 
     auto scale = xla::Pow(
         xla::Add(xla::ConstantR0<float>(builder, bias_),
@@ -138,15 +136,14 @@ class LRNGradOp : public XlaOpKernel {
 
     auto accumulation_type = XlaHelpers::SumAccumulationType(input_type(0));
     auto converted =
-        XlaHelpers::ConvertElementType(builder, in_image, accumulation_type);
+        XlaHelpers::ConvertElementType(in_image, accumulation_type);
     auto squared = xla::Mul(converted, converted);
     auto reduce = xla::ReduceWindow(
         squared, XlaHelpers::Zero(builder, accumulation_type),
         *ctx->GetOrCreateAdd(accumulation_type),
         /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1},
         /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame);
-    auto sqr_sum =
-        XlaHelpers::ConvertElementType(builder, reduce, input_type(0));
+    auto sqr_sum = XlaHelpers::ConvertElementType(reduce, input_type(0));
 
     auto norm =
         xla::Add(xla::ConstantR0<float>(builder, bias_),
@@ -157,15 +154,13 @@ class LRNGradOp : public XlaOpKernel {
                  xla::Div(out_image, norm)),
         in_grads);
 
-    auto converted_dy =
-        XlaHelpers::ConvertElementType(builder, dy, accumulation_type);
+    auto converted_dy = XlaHelpers::ConvertElementType(dy, accumulation_type);
     auto dy_reduce = xla::ReduceWindow(
         converted_dy, XlaHelpers::Zero(builder, accumulation_type),
         *ctx->GetOrCreateAdd(accumulation_type),
         /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1},
         /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame);
-    auto dy_reduced =
-        XlaHelpers::ConvertElementType(builder, dy_reduce, input_type(0));
+    auto dy_reduced = XlaHelpers::ConvertElementType(dy_reduce, input_type(0));
 
     xla::XlaOp gradients = xla::Add(
         xla::Mul(in_image, dy_reduced),
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
index e96cabbb85..b9bb5edba3 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
@@ -117,7 +117,7 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
   xla::XlaComputation reduction_computation = r.Build().ConsumeValueOrDie();
 
   auto reduce = xla::Reduce(data, initial, reduction_computation, xla_axes);
-  auto deconverted = XlaHelpers::ConvertElementType(b, reduce, input_type(0));
+  auto deconverted = XlaHelpers::ConvertElementType(reduce, input_type(0));
   auto finalized = BuildFinalizer(b, data, deconverted, xla_axes);
   auto result = keep_dims_ ? xla::Reshape(finalized, final_shape) : finalized;
   ctx->SetOutput(0, result);
diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
index 7f4fef146f..4b9e1a578b 100644
--- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
@@ -103,11 +103,10 @@ class ScanOp : public XlaOpKernel {
       reducer = ctx->GetOrCreateMul(dtype);
     }
     auto output = xla::ReduceWindowWithGeneralPadding(
-        XlaHelpers::ConvertElementType(builder, ctx->Input(0), dtype), init,
-        *reducer, window_dims, window_strides,
+        XlaHelpers::ConvertElementType(ctx->Input(0), dtype), init, *reducer,
+        window_dims, window_strides,
         /*base_dilations=*/{}, /*window_dilations=*/{}, padding);
-    output =
-        XlaHelpers::ConvertElementType(builder, output, ctx->input_type(0));
+    output = XlaHelpers::ConvertElementType(output, ctx->input_type(0));
 
     // In exclusive mode, we have computed an extra element containing the sum
     // of all the input elements. Slice off this extra "last" element.
diff --git a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
index d6bd927135..20da803353 100644
--- a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
@@ -71,7 +71,7 @@ class SoftmaxOp : public XlaOpKernel {
     auto reduce =
         xla::Reduce(converted, xla::Zero(b, xla_accumulation_type),
                     *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
-    auto sum = XlaHelpers::ConvertElementType(b, reduce, type);
+    auto sum = XlaHelpers::ConvertElementType(reduce, type);
     auto softmax =
         log_
             // softmax = shifted_logits - log(sum(exp(shifted_logits)))
@@ -111,11 +111,11 @@ std::pair<xla::XlaOp, xla::XlaOp> CrossEntropyWithLogits(
   // sum_{class} (exp(logits - max_logits))
   const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
   auto converted =
-      XlaHelpers::ConvertElementType(b, exp_shifted_logits, accumulation_type);
+      XlaHelpers::ConvertElementType(exp_shifted_logits, accumulation_type);
   auto reduce =
       xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                   *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
-  auto sum_exp = XlaHelpers::ConvertElementType(b, reduce, type);
+  auto sum_exp = XlaHelpers::ConvertElementType(reduce, type);
 
   // log(sum(exp(logits - max_logits)))
   auto log_sum_exp = xla::Log(sum_exp);
@@ -126,11 +126,10 @@ std::pair<xla::XlaOp, xla::XlaOp> CrossEntropyWithLogits(
   // (The subtraction broadcasts along the batch dimension.)
   auto sub = xla::Sub(shifted_logits, log_sum_exp, {kBatchDim});
   auto mul = xla::Mul(xla::Neg(labels), sub);
-  auto sum =
-      xla::Reduce(XlaHelpers::ConvertElementType(b, mul, accumulation_type),
-                  XlaHelpers::Zero(b, accumulation_type),
-                  *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
-  auto loss = XlaHelpers::ConvertElementType(b, sum, type);
+  auto sum = xla::Reduce(XlaHelpers::ConvertElementType(mul, accumulation_type),
+                         XlaHelpers::Zero(b, accumulation_type),
+                         *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
+  auto loss = XlaHelpers::ConvertElementType(sum, type);
 
   // backprop: prob - labels, where
   //   prob = exp(logits - max_logits) / sum(exp(logits - max_logits))
diff --git a/tensorflow/compiler/tf2xla/kernels/training_ops.cc b/tensorflow/compiler/tf2xla/kernels/training_ops.cc
index 7077c2e3a5..960c1462ce 100644
--- a/tensorflow/compiler/tf2xla/kernels/training_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/training_ops.cc
@@ -320,9 +320,8 @@ class ResourceApplyAdagradDA : public XlaOpKernel {
     xla::XlaOp lr = ctx->Input(4);
     xla::XlaOp l1 = ctx->Input(5);
     xla::XlaOp l2 = ctx->Input(6);
-    xla::XlaBuilder* const b = ctx->builder();
     xla::XlaOp global_step =
-        XlaHelpers::ConvertElementType(b, ctx->Input(7), dtype_);
+        XlaHelpers::ConvertElementType(ctx->Input(7), dtype_);
 
     accum = accum + grad;
     squared_accum = squared_accum + xla::Square(grad);
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc
index af378bc95c..c2c0751211 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.cc
+++ b/tensorflow/compiler/tf2xla/xla_helpers.cc
@@ -215,8 +215,7 @@ DataType XlaHelpers::SumAccumulationType(const DataType& dtype) {
   return dtype;
 }
 
-xla::XlaOp XlaHelpers::ConvertElementType(xla::XlaBuilder* const builder,
-                                          const xla::XlaOp& operand,
+xla::XlaOp XlaHelpers::ConvertElementType(const xla::XlaOp& operand,
                                           const DataType new_element_type) {
   xla::PrimitiveType convert_to;
   TF_CHECK_OK(DataTypeToPrimitiveType(new_element_type, &convert_to));
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.h b/tensorflow/compiler/tf2xla/xla_helpers.h
index 39578144ca..4858dfee55 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.h
+++ b/tensorflow/compiler/tf2xla/xla_helpers.h
@@ -80,8 +80,7 @@ class XlaHelpers {
 
   // A helper for creating a ConvertElementType xla op given a DataType rather
   // than the xla::PrimitiveType.
-  static xla::XlaOp ConvertElementType(xla::XlaBuilder* const builder,
-                                       const xla::XlaOp& operand,
+  static xla::XlaOp ConvertElementType(const xla::XlaOp& operand,
                                        const DataType new_element_type);
 };
 
-- 
GitLab


From ebbf59ebc154a0dea65cda4a8ef56416189b43fb Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 28 Nov 2018 09:11:50 -0800
Subject: [PATCH 0932/1554] Remove methods from tf.Variable which in
 RefVariable are to be deprecated and aren't supported in Resource Variable

PiperOrigin-RevId: 223178585
---
 tensorflow/python/ops/variables.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index f72b19bcdd..4824c92a5a 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -1055,27 +1055,6 @@ class Variable(six.with_metaclass(VariableMetaclass,
       else:
         return None
 
-  def __iadd__(self, other):
-    raise NotImplementedError
-
-  def __isub__(self, other):
-    raise NotImplementedError
-
-  def __imul__(self, other):
-    raise NotImplementedError
-
-  def __idiv__(self, other):
-    raise NotImplementedError
-
-  def __itruediv__(self, other):
-    raise NotImplementedError
-
-  def __irealdiv__(self, other):
-    raise NotImplementedError
-
-  def __ipow__(self, other):
-    raise NotImplementedError
-
 
 @tf_export(v1=["Variable"])
 class VariableV1(Variable):
-- 
GitLab


From 01605a8038a70f6622867a0b7bffec61d6e49201 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Wed, 28 Nov 2018 09:32:10 -0800
Subject: [PATCH 0933/1554] Make tf.constant not take verify_shape in v2 and
 limit allowed forms.

tf.constant([], shape=[3]) no longer allowed
tf.constant(0, shape=[3]) still allowed
tf.constant([1,2], shape=[1,1,2]) is still ok.

This makes eager and non-eager constants more consistent.

PiperOrigin-RevId: 223182163
---
 .../batch/categorical_split_handler.py        |  2 +-
 .../batch/categorical_split_handler_test.py   |  8 +-
 .../learner/batch/ordinal_split_handler.py    | 12 +--
 .../batch/ordinal_split_handler_test.py       | 13 +--
 .../python/training/functions/gbdt_batch.py   |  4 +-
 tensorflow/python/framework/constant_op.py    | 81 ++++++++++++++++++-
 tensorflow/python/framework/tensor_util.py    | 28 +++++--
 .../python/kernel_tests/constant_op_test.py   | 22 +++--
 tensorflow/python/ops/init_ops.py             |  2 +-
 .../python/ops/ragged/ragged_const_op_test.py |  4 +-
 .../tools/api/golden/v1/tensorflow.pbtxt      |  2 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  3 +
 13 files changed, 144 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py
index 4da25298cb..d26af58419 100644
--- a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py
+++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler.py
@@ -119,7 +119,7 @@ class EqualitySplitHandler(base_split_handler.BaseSplitHandler):
 
     def not_active_inputs():
       return (constant_op.constant([], dtype=dtypes.int32),
-              constant_op.constant([], dtype=dtypes.int64, shape=[1, 2]),
+              constant_op.constant_v1([], dtype=dtypes.int64, shape=[1, 2]),
               empty_gradients, empty_hessians)
 
     def active_inputs():
diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py
index a2f708081a..386dc19fc7 100644
--- a/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py
+++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/categorical_split_handler_test.py
@@ -36,9 +36,9 @@ def get_empty_tensors(gradient_shape, hessian_shape):
   empty_hess_shape = [1] + hessian_shape.as_list()
   empty_grad_shape = [1] + gradient_shape.as_list()
 
-  empty_gradients = constant_op.constant(
+  empty_gradients = constant_op.constant_v1(
       [], dtype=dtypes.float32, shape=empty_grad_shape)
-  empty_hessians = constant_op.constant(
+  empty_hessians = constant_op.constant_v1(
       [], dtype=dtypes.float32, shape=empty_hess_shape)
 
   return empty_gradients, empty_hessians
@@ -486,8 +486,8 @@ class EqualitySplitHandlerTest(test_util.TensorFlowTestCase):
       gradients = array_ops.constant([0.2, -0.5, 1.2, 4.0])
       hessians = array_ops.constant([0.12, 0.07, 0.2, 0.13])
       partition_ids = [0, 0, 0, 1]
-      indices = array_ops.constant([], dtype=dtypes.int64, shape=[0, 2])
-      values = array_ops.constant([], dtype=dtypes.int64)
+      indices = constant_op.constant_v1([], dtype=dtypes.int64, shape=[0, 2])
+      values = constant_op.constant_v1([], dtype=dtypes.int64)
 
       gradient_shape = tensor_shape.scalar()
       hessian_shape = tensor_shape.scalar()
diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py
index 1fffbb5f66..0476bed2cd 100644
--- a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py
+++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler.py
@@ -605,7 +605,7 @@ def dense_make_stats_update(is_active, are_buckets_ready, float_column,
                             quantile_buckets, example_partition_ids, gradients,
                             hessians, weights, empty_gradients, empty_hessians):
   """Updates the state for dense split handler."""
-  empty_float = constant_op.constant([], dtype=dtypes.float32)
+  empty_float = constant_op.constant_v1([], dtype=dtypes.float32)
 
   quantile_values, quantile_weights = control_flow_ops.cond(
       is_active[1],  # For the next layer, this handler is inactive.
@@ -621,8 +621,8 @@ def dense_make_stats_update(is_active, are_buckets_ready, float_column,
     return (example_partition_ids, quantized_feature, gradients, hessians)
 
   def not_ready_inputs_fn():
-    return (constant_op.constant([], dtype=dtypes.int32),
-            constant_op.constant([[]], dtype=dtypes.int64, shape=[1, 2]),
+    return (constant_op.constant_v1([], dtype=dtypes.int32),
+            constant_op.constant_v1([[]], dtype=dtypes.int64, shape=[1, 2]),
             empty_gradients, empty_hessians)
 
   example_partition_ids, feature_ids, gradients, hessians = (
@@ -708,11 +708,11 @@ def sparse_make_stats_update(
 
   def quantiles_not_ready():
     """The subgraph for when the quantiles are not ready."""
-    return (constant_op.constant([], dtype=dtypes.int32),
-            constant_op.constant([], dtype=dtypes.int64, shape=[1, 2]),
+    return (constant_op.constant_v1([], dtype=dtypes.int32),
+            constant_op.constant_v1([], dtype=dtypes.int64, shape=[1, 2]),
             empty_gradients, empty_hessians)
 
-  empty_float = constant_op.constant([], dtype=dtypes.float32)
+  empty_float = constant_op.constant_v1([], dtype=dtypes.float32)
   handler_not_active = (constant_op.constant(
       [], dtype=dtypes.int64, shape=[0, 2]), empty_float,
                         constant_op.constant([0, 1], dtype=dtypes.int64),
diff --git a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py
index 74b0ea6989..4a1b528646 100644
--- a/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py
+++ b/tensorflow/contrib/boosted_trees/lib/learner/batch/ordinal_split_handler_test.py
@@ -39,9 +39,9 @@ def get_empty_tensors(gradient_shape, hessian_shape):
   empty_hess_shape = [1] + hessian_shape.as_list()
   empty_grad_shape = [1] + gradient_shape.as_list()
 
-  empty_gradients = constant_op.constant(
+  empty_gradients = constant_op.constant_v1(
       [], dtype=dtypes.float32, shape=empty_grad_shape)
-  empty_hessians = constant_op.constant(
+  empty_hessians = constant_op.constant_v1(
       [], dtype=dtypes.float32, shape=empty_hess_shape)
 
   return empty_gradients, empty_hessians
@@ -1476,9 +1476,9 @@ class SparseSplitHandlerTest(test_util.TensorFlowTestCase):
 
   def testEmpty(self):
     with self.cached_session() as sess:
-      indices = array_ops.constant([], dtype=dtypes.int64, shape=[0, 2])
+      indices = constant_op.constant_v1([], dtype=dtypes.int64, shape=[0, 2])
       # No values in this feature column in this mini-batch.
-      values = array_ops.constant([], dtype=dtypes.float32)
+      values = constant_op.constant_v1([], dtype=dtypes.float32)
       sparse_column = sparse_tensor.SparseTensor(indices, values, [4, 1])
 
       gradient_shape = tensor_shape.scalar()
@@ -1549,8 +1549,9 @@ class SparseSplitHandlerTest(test_util.TensorFlowTestCase):
       sparse_column = array_ops.sparse_placeholder(dtypes.float32)
 
       # We have two batches - at first, a sparse feature is empty.
-      empty_indices = array_ops.constant([], dtype=dtypes.int64, shape=[0, 2])
-      empty_values = array_ops.constant([], dtype=dtypes.float32)
+      empty_indices = constant_op.constant_v1([], dtype=dtypes.int64,
+                                              shape=[0, 2])
+      empty_values = constant_op.constant_v1([], dtype=dtypes.float32)
       empty_sparse_column = sparse_tensor.SparseTensor(empty_indices,
                                                        empty_values, [4, 2])
       empty_sparse_column = empty_sparse_column.eval(session=sess)
diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
index 85020c5df2..9fdc2fc0c2 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
@@ -897,9 +897,9 @@ class GradientBoostedDecisionTreeModel(object):
     empty_hess_shape = [1] + self._hessian_shape.as_list()
     empty_grad_shape = [1] + self._gradient_shape.as_list()
 
-    empty_gradients = constant_op.constant(
+    empty_gradients = constant_op.constant_v1(
         [], dtype=dtypes.float32, shape=empty_grad_shape)
-    empty_hessians = constant_op.constant(
+    empty_hessians = constant_op.constant_v1(
         [], dtype=dtypes.float32, shape=empty_hess_shape)
 
     active_handlers = array_ops.unstack(active_handlers, axis=0)
diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index 53d84b2dc7..ade0797dcd 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -114,8 +114,9 @@ def convert_to_eager_tensor(value, ctx, dtype=None):
     return ops.EagerTensor(value, handle, device, dtype)
 
 
-@tf_export("constant")
-def constant(value, dtype=None, shape=None, name="Const", verify_shape=False):
+@tf_export(v1=["constant"])
+def constant_v1(
+    value, dtype=None, shape=None, name="Const", verify_shape=False):
   """Creates a constant tensor.
 
   The resulting tensor is populated with values of type `dtype`, as
@@ -174,6 +175,79 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False):
   Raises:
     TypeError: if shape is incorrectly specified or unsupported.
   """
+  return _constant_impl(value, dtype, shape, name, verify_shape=verify_shape,
+                        allow_broadcast=False)
+
+
+@tf_export("constant", v1=[])
+def constant(value, dtype=None, shape=None, name="Const"):
+  """Creates a constant tensor.
+
+  The resulting tensor is populated with values of type `dtype`, as
+  specified by arguments `value` and (optionally) `shape` (see examples
+  below).
+
+  The argument `value` can be a constant value, or a list of values of type
+  `dtype`. If `value` is a list, then the length of the list must be less
+  than or equal to the number of elements implied by the `shape` argument (if
+  specified). In the case where the list length is less than the number of
+  elements specified by `shape`, the last element in the list will be used
+  to fill the remaining entries.
+
+  The argument `shape` is optional. If present, it specifies the dimensions of
+  the resulting tensor. If not present, the shape of `value` is used.
+
+  If the argument `dtype` is not specified, then the type is inferred from
+  the type of `value`.
+
+  For example:
+
+  ```python
+  # Constant 1-D Tensor populated with value list.
+  tensor = tf.constant([1, 2, 3, 4, 5, 6]) => [1 2 3 4 5 6]
+
+  # Constant 1-D Tensor populated with value list.
+  tensor = tf.constant([1, 2, 3, 4, 5, 6], shape=(2,3))
+       => [[1 2 3], [4 5 6]]
+
+  # Constant 2-D tensor populated with scalar value -1.
+  tensor = tf.constant(-1.0, shape=[2, 3]) => [[-1. -1. -1.]
+                                               [-1. -1. -1.]]
+  ```
+
+  `tf.constant` differs from `tf.fill` in a few ways:
+
+  *   `tf.constant` supports arbitrary constants, not just uniform scalar
+      Tensors like `tf.fill`.
+  *   `tf.constant` creates a `Const` node in the computation graph with the
+      exact value at graph construction time. On the other hand, `tf.fill`
+      creates an Op in the graph that is expanded at runtime.
+  *   Because `tf.constant` only embeds constant values in the graph, it does
+      not support dynamic shapes based on other runtime Tensors, whereas
+      `tf.fill` does.
+
+  Args:
+    value:          A constant value (or list) of output type `dtype`.
+
+    dtype:          The type of the elements of the resulting tensor.
+
+    shape:          Optional dimensions of resulting tensor.
+
+    name:           Optional name for the tensor.
+
+  Returns:
+    A Constant Tensor.
+
+  Raises:
+    TypeError: if shape is incorrectly specified or unsupported.
+  """
+  return _constant_impl(value, dtype, shape, name, verify_shape=False,
+                        allow_broadcast=True)
+
+
+def _constant_impl(
+    value, dtype, shape, name, verify_shape, allow_broadcast):
+  """Implementation of constant."""
   ctx = context.context()
   if ctx.executing_eagerly():
     t = convert_to_eager_tensor(value, ctx, dtype)
@@ -205,7 +279,8 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False):
   tensor_value = attr_value_pb2.AttrValue()
   tensor_value.tensor.CopyFrom(
       tensor_util.make_tensor_proto(
-          value, dtype=dtype, shape=shape, verify_shape=verify_shape))
+          value, dtype=dtype, shape=shape, verify_shape=verify_shape,
+          allow_broadcast=allow_broadcast))
   dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)
   const_tensor = g.create_op(
       "Const", [], [dtype_value.type],
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 9db94f5288..f98f301b38 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -371,8 +371,10 @@ def _AssertCompatible(values, dtype):
                       (dtype.name, repr(mismatch), type(mismatch).__name__))
 
 
+# pylint: disable=invalid-name
 @tf_export(v1=["make_tensor_proto"])
-def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False):
+def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False,
+                      allow_broadcast=False):
   """Create a TensorProto.
 
   Args:
@@ -380,6 +382,8 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False):
     dtype:          Optional tensor_pb2 DataType value.
     shape:          List of integers representing the dimensions of tensor.
     verify_shape:   Boolean that enables verification of a shape of values.
+    allow_broadcast:Boolean that enables allowing scalars and 1 length vector
+        broadcasting. Cannot be true when verify_shape is true.
 
   Returns:
     A `TensorProto`. Depending on the type, it may contain data in the
@@ -416,6 +420,8 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False):
   can not have more elements than what "shape" specifies.
 
   """
+  if allow_broadcast and verify_shape:
+    raise ValueError("allow_broadcast and verify_shape are not both allowed.")
   if isinstance(values, tensor_pb2.TensorProto):
     return values
 
@@ -504,15 +510,22 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False):
     shape_size = np.prod(shape, dtype=np.int64)
     is_same_size = shape_size == nparray.size
 
-    if verify_shape:
-      if not nparray.shape == tuple(shape):
+    if allow_broadcast:
+      if nparray.shape == (1,) or nparray.shape == tuple():
+        pass
+      elif nparray.size != shape_size:
         raise TypeError("Expected Tensor's shape: %s, got %s." %
                         (tuple(shape), nparray.shape))
 
-    if nparray.size > shape_size:
-      raise ValueError(
-          "Too many elements provided. Needed at most %d, but received %d" %
-          (shape_size, nparray.size))
+    else:
+      if verify_shape and nparray.shape != tuple(shape):
+        raise TypeError("Expected Tensor's shape: %s, got %s." %
+                        (tuple(shape), nparray.shape))
+
+      if nparray.size > shape_size:
+        raise ValueError(
+            "Too many elements provided. Needed at most %d, but received %d" %
+            (shape_size, nparray.size))
 
   tensor_proto = tensor_pb2.TensorProto(
       dtype=numpy_dtype.as_datatype_enum,
@@ -560,6 +573,7 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False):
   append_fn(tensor_proto, proto_values)
 
   return tensor_proto
+# pylint: enable=invalid-name
 
 
 @tf_export("make_ndarray")
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 112e201c88..9c3c96bd31 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -219,16 +219,28 @@ class ConstantTest(test.TestCase):
 
   def testShapeInconsistent(self):
     with ops.Graph().as_default():
-      c = constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[10])
+      c = constant_op.constant_v1([1, 2, 3, 4, 5, 6, 7], shape=[10])
+    self.assertEqual(c.get_shape(), [10])
+
+    with ops.Graph().as_default():
+      with self.assertRaisesRegexp(
+          TypeError, "Expected Tensor's shape"):
+        c = constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[10])
+
+  def testPromotionShapes(self):
+    with ops.Graph().as_default():
+      c = constant_op.constant([7], shape=[10])
+    self.assertEqual(c.get_shape(), [10])
+    with ops.Graph().as_default():
+      c = constant_op.constant(3, shape=[10])
     self.assertEqual(c.get_shape(), [10])
 
   # pylint: disable=g-long-lambda
   def testShapeWrong(self):
     with ops.Graph().as_default():
-      with self.assertRaisesWithPredicateMatch(
-          ValueError,
-          lambda e: ("Too many elements provided. Needed at most 5, "
-                     "but received 7" == str(e))):
+      with self.assertRaisesRegexp(ValueError, "Too many elements provided."):
+        constant_op.constant_v1([1, 2, 3, 4, 5, 6, 7], shape=[5])
+      with self.assertRaisesRegexp(TypeError, "Expected Tensor's shape"):
         constant_op.constant([1, 2, 3, 4, 5, 6, 7], shape=[5])
 
   # pylint: enable=g-long-lambda
diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py
index fe54134af2..03d2201a9a 100644
--- a/tensorflow/python/ops/init_ops.py
+++ b/tensorflow/python/ops/init_ops.py
@@ -216,7 +216,7 @@ class Constant(Initializer):
       dtype = self.dtype
     if verify_shape is None:
       verify_shape = self._verify_shape
-    return constant_op.constant(
+    return constant_op.constant_v1(
         self.value, dtype=dtype, shape=shape, verify_shape=verify_shape)
 
   def get_config(self):
diff --git a/tensorflow/python/ops/ragged/ragged_const_op_test.py b/tensorflow/python/ops/ragged/ragged_const_op_test.py
index 66c39475fa..9c3b2ac88a 100644
--- a/tensorflow/python/ops/ragged/ragged_const_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_const_op_test.py
@@ -238,8 +238,8 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       dict(
           pylist=[1, 2, 3],
           inner_shape=(1, 1),
-          exception=ValueError,
-          message='Too many elements provided.'),
+          exception=TypeError,
+          message='Expected Tensor\'s shape'),
       dict(
           pylist=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]],
           inner_shape=(2, 2),
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 656a52945c..a294e3e027 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1482,7 +1482,7 @@ tf_module {
   }
   member_method {
     name: "make_tensor_proto"
-    argspec: "args=[\'values\', \'dtype\', \'shape\', \'verify_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], "
+    argspec: "args=[\'values\', \'dtype\', \'shape\', \'verify_shape\', \'allow_broadcast\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
   }
   member_method {
     name: "map_fn"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index a48095eb91..0659900ffa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -558,7 +558,7 @@ tf_module {
   }
   member_method {
     name: "constant"
-    argspec: "args=[\'value\', \'dtype\', \'shape\', \'name\', \'verify_shape\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Const\', \'False\'], "
+    argspec: "args=[\'value\', \'dtype\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'Const\'], "
   }
   member_method {
     name: "control_dependencies"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 059d2ef4de..3cb78afb32 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -52,6 +52,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.batch_to_space_nd": {
             "block_size": "block_shape",
         },
+        "tf.constant": {
+            "verify_shapes": "verify_shapes_is_now_always_true",
+        },
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
-- 
GitLab


From 2948200bbb991cec64653725fd8f990fa015f603 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 28 Nov 2018 09:40:31 -0800
Subject: [PATCH 0934/1554] Alter documentation for convert_to_tensor, adding
 additional error thrown and removing on the internal method.

PiperOrigin-RevId: 223183729
---
 tensorflow/python/framework/ops.py | 49 +++++-------------------------
 1 file changed, 7 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 31fa111b3f..5a8a2a48ab 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -1036,12 +1036,12 @@ def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None):
       `preferred_dtype` is not possible, this argument has no effect.
 
   Returns:
-    An `Output` based on `value`.
+    An `Tensor` based on `value`.
 
   Raises:
-    TypeError: If no conversion function is registered for `value`.
+    TypeError: If no conversion function is registered for `value` to `dtype`.
     RuntimeError: If a registered conversion function returns an invalid value.
-
+    ValueError: If the `value` is a tensor not of given `dtype` in graph mode.
   """
   return convert_to_tensor_v2(value, dtype, preferred_dtype, name)
 
@@ -1089,12 +1089,12 @@ def convert_to_tensor_v2(value, dtype=None, dtype_hint=None, name=None):
     name: Optional name to use if a new `Tensor` is created.
 
   Returns:
-    An `Output` based on `value`.
+    An `Tensor` based on `value`.
 
   Raises:
-    TypeError: If no conversion function is registered for `value`.
+    TypeError: If no conversion function is registered for `value` to `dtype`.
     RuntimeError: If a registered conversion function returns an invalid value.
-
+    ValueError: If the `value` is a tensor not of given `dtype` in graph mode.
   """
   return internal_convert_to_tensor(
       value=value,
@@ -1115,42 +1115,7 @@ def internal_convert_to_tensor(value,
                                preferred_dtype=None,
                                ctx=None,
                                accept_symbolic_tensors=True):
-  """Converts the given `value` to an `Tensor`.
-
-  This function converts Python objects of various types to `Tensor`
-  objects. It accepts `Tensor` objects, numpy arrays, Python lists,
-  and Python scalars. For example:
-
-  This function can be useful when composing a new operation in Python
-  All standard Python op constructors apply this function to each of their
-  Tensor-valued inputs, which allows those ops to accept numpy arrays, Python
-  lists, and scalars in addition to `Tensor` objects.
-
-  Args:
-    value: An object whose type has a registered `Tensor` conversion function.
-    dtype: Optional element type for the returned tensor. If missing, the
-      type is inferred from the type of `value`.
-    name: Optional name to use if a new `Tensor` is created.
-    as_ref: True if we want the mutable view of Variables, if applicable.
-    preferred_dtype: Optional element type for the returned tensor,
-      used when dtype is None. In some cases, a caller may not have a
-      dtype in mind when converting to a tensor, so preferred_dtype
-      can be used as a soft preference.  If the conversion to
-      `preferred_dtype` is not possible, this argument has no effect.
-    ctx: Optional: The value of context.context().
-    accept_symbolic_tensors: Whether Keras graph tensors should be accepted as
-      a valid tensor type during eager execution.
-      If False, this function will raise an exception if it is passed such
-      a tensor during eager eager execution.
-
-  Returns:
-    A `Tensor` based on `value`.
-
-  Raises:
-    TypeError: If no conversion function is registered for `value`.
-    RuntimeError: If a registered conversion function returns an invalid value.
-
-  """
+  """Implementation of the public convert_to_tensor."""
   if ctx is None: ctx = context.context()
   if isinstance(value, EagerTensor):
     if ctx.executing_eagerly():
-- 
GitLab


From b6f9a085569d30788443b036ff846dcfc79a4522 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Wed, 28 Nov 2018 09:42:33 -0800
Subject: [PATCH 0935/1554] [Eager]: Cleanup: Simplify code style.

This is a by-product of the DeviceMgr refactor.

PiperOrigin-RevId: 223184130
---
 tensorflow/core/distributed_runtime/eager/BUILD              | 1 +
 .../core/distributed_runtime/eager/eager_service_impl.cc     | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/eager/BUILD b/tensorflow/core/distributed_runtime/eager/BUILD
index 055e5dfced..55b2657e74 100644
--- a/tensorflow/core/distributed_runtime/eager/BUILD
+++ b/tensorflow/core/distributed_runtime/eager/BUILD
@@ -69,6 +69,7 @@ cc_library(
         "//tensorflow/core/distributed_runtime:worker_env",
         "//tensorflow/core/distributed_runtime/eager:remote_tensor_handle",
         "//tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
index c66466c0a6..13c959d850 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/distributed_runtime/eager/eager_service_impl.h"
 
+#include "absl/memory/memory.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/tf_status_helper.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -101,8 +102,8 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
     *response->add_device_attributes() = d->attributes();
   }
 
-  std::unique_ptr<tensorflow::DeviceMgr> device_mgr(
-      new tensorflow::DeviceMgr(std::move(devices)));
+  std::unique_ptr<tensorflow::DeviceMgr> device_mgr =
+      absl::make_unique<DeviceMgr>(std::move(devices));
 
   auto* r = env_->rendezvous_mgr->Find(request->rendezvous_id());
   auto session_name = strings::StrCat("eager_", request->rendezvous_id());
-- 
GitLab


From 14979f572d0b310c606607b15a22146f2b7bd685 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 28 Nov 2018 09:45:51 -0800
Subject: [PATCH 0936/1554] Fixes for relative paths when saving weights

Fixes a minor incompatibility in POSIX file handling where CreateDir on a blank directory would throw NotFound rather than AlreadyExists.

When passed a directory path rather than a prefix, saving creates the directory rather than throwing NotFound.

PiperOrigin-RevId: 223184690
---
 .../core/platform/posix/posix_file_system.cc  | 11 ++--
 tensorflow/python/keras/engine/saving_test.py | 53 +++++++++++++++++++
 tensorflow/python/kernel_tests/BUILD          |  1 +
 .../kernel_tests/save_restore_ops_test.py     | 16 ++++++
 .../python/training/checkpointable/util.py    |  2 +
 5 files changed, 79 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/platform/posix/posix_file_system.cc b/tensorflow/core/platform/posix/posix_file_system.cc
index c7afab9583..fc48cab564 100644
--- a/tensorflow/core/platform/posix/posix_file_system.cc
+++ b/tensorflow/core/platform/posix/posix_file_system.cc
@@ -240,11 +240,14 @@ Status PosixFileSystem::DeleteFile(const string& fname) {
 }
 
 Status PosixFileSystem::CreateDir(const string& name) {
-  Status result;
-  if (mkdir(TranslateName(name).c_str(), 0755) != 0) {
-    result = IOError(name, errno);
+  string translated = TranslateName(name);
+  if (translated.empty()) {
+    return errors::AlreadyExists(name);
   }
-  return result;
+  if (mkdir(translated.c_str(), 0755) != 0) {
+    return IOError(name, errno);
+  }
+  return Status::OK();
 }
 
 Status PosixFileSystem::DeleteDir(const string& name) {
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py
index f376f081cf..375d1010d1 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/engine/saving_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import training
+from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
@@ -992,5 +993,57 @@ class TestWeightSavingAndLoadingTFFormat(test.TestCase):
         AssertionError, 'Nothing except the root object matched'):
       m.load_weights(save_path)
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_directory_passed(self):
+    m = keras.Model()
+    v = m.add_weight(name='v', shape=[])
+    self.evaluate(v.assign(42.))
+    prefix = os.path.join(self.get_temp_dir(), '{}'.format(ops.uid()), 'ckpt/')
+    m.save_weights(prefix)
+    self.evaluate(v.assign(2.))
+    m.load_weights(prefix)
+    self.assertEqual(42., self.evaluate(v))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_relative_path(self):
+    m = keras.Model()
+    v = m.add_weight(name='v', shape=[])
+    os.chdir(self.get_temp_dir())
+
+    prefix = 'ackpt'
+    self.evaluate(v.assign(42.))
+    m.save_weights(prefix)
+    self.assertTrue(file_io.file_exists('ackpt.index'))
+    self.evaluate(v.assign(1.))
+    m.load_weights(prefix)
+    self.assertEqual(42., self.evaluate(v))
+
+    prefix = 'subdir/ackpt'
+    self.evaluate(v.assign(43.))
+    m.save_weights(prefix)
+    self.assertTrue(file_io.file_exists('subdir/ackpt.index'))
+    self.evaluate(v.assign(2.))
+    m.load_weights(prefix)
+    self.assertEqual(43., self.evaluate(v))
+
+    prefix = 'ackpt/'
+    self.evaluate(v.assign(44.))
+    m.save_weights(prefix)
+    self.assertTrue(file_io.file_exists('ackpt/.index'))
+    self.evaluate(v.assign(3.))
+    m.load_weights(prefix)
+    self.assertEqual(44., self.evaluate(v))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_nonexistant_prefix_directory(self):
+    m = keras.Model()
+    v = m.add_weight(name='v', shape=[])
+    self.evaluate(v.assign(42.))
+    prefix = os.path.join(self.get_temp_dir(), '{}'.format(ops.uid()), 'bckpt')
+    m.save_weights(prefix)
+    self.evaluate(v.assign(2.))
+    m.load_weights(prefix)
+    self.assertEqual(42., self.evaluate(v))
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 3839928434..de06ec622a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -831,6 +831,7 @@ tf_py_test(
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:io_ops",
         "//tensorflow/python:io_ops_gen",
     ],
diff --git a/tensorflow/python/kernel_tests/save_restore_ops_test.py b/tensorflow/python/kernel_tests/save_restore_ops_test.py
index cb9aa1e34d..be117c4350 100644
--- a/tensorflow/python/kernel_tests/save_restore_ops_test.py
+++ b/tensorflow/python/kernel_tests/save_restore_ops_test.py
@@ -17,14 +17,30 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_io_ops
 from tensorflow.python.ops import io_ops
 from tensorflow.python.platform import test
 
 
+class SaveTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testRelativePath(self):
+    os.chdir(self.get_temp_dir())
+    self.evaluate(io_ops.save_v2(
+        "ckpt", ["x"], [""], [constant_op.constant(100.)]))
+    self.assertAllEqual([100.],
+                        self.evaluate(io_ops.restore_v2(
+                            "ckpt", ["x"], [""], [dtypes.float32])))
+
+
 class ShardedFileOpsTest(test.TestCase):
 
   def testShardedFileName(self):
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 85844393f3..394cc33ad0 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_io_ops as io_ops
 from tensorflow.python.ops import init_ops
@@ -1458,6 +1459,7 @@ class CheckpointableSaver(object):
     elif session is None:
       session = ops.get_default_session()
 
+    file_io.recursive_create_dir(os.path.dirname(file_prefix))
     with ops.device("/cpu:0"):
       save_path = saver.save(
           sess=_SessionWithFeedDictAdditions(
-- 
GitLab


From d6c36d6576fb46f684a33c7ee54b879768811ae6 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Wed, 28 Nov 2018 09:52:03 -0800
Subject: [PATCH 0937/1554] [XLA] Set the layout of the pad instruction if
 interior padding was removed.

PiperOrigin-RevId: 223185645
---
 tensorflow/compiler/xla/service/algebraic_simplifier.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index a9099d9e31..a348bcf0a2 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1685,10 +1685,10 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) {
       }
     }
     if (cleared_interior_padding) {
-      TF_ASSIGN_OR_RETURN(HloInstruction * pad_without_interior_pad,
-                          MakePadHlo(pad->mutable_operand(0),
-                                     pad->mutable_operand(1), padding_config));
-      return ReplaceInstruction(pad, pad_without_interior_pad);
+      return ReplaceWithNewInstruction(
+          pad,
+          HloInstruction::CreatePad(pad->shape(), pad->mutable_operand(0),
+                                    pad->mutable_operand(1), padding_config));
     }
   }
 
-- 
GitLab


From 77296a9c84cde06a3299a8a18cc4ac79051f7db0 Mon Sep 17 00:00:00 2001
From: Karmel Allison <karmel@google.com>
Date: Wed, 28 Nov 2018 10:06:55 -0800
Subject: [PATCH 0938/1554] Exposing export_all_saved_models from the
 experimental Estimator API in advance of removing it from contrib.

PiperOrigin-RevId: 223188420
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py          | 3 ---
 tensorflow/python/saved_model/constants.py                  | 2 +-
 tensorflow/python/saved_model/signature_constants.py        | 2 +-
 .../v1/tensorflow.estimator.-baseline-classifier.pbtxt      | 6 +++++-
 .../v1/tensorflow.estimator.-baseline-estimator.pbtxt       | 6 +++++-
 .../v1/tensorflow.estimator.-baseline-regressor.pbtxt       | 6 +++++-
 .../v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt | 6 +++++-
 .../v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt  | 6 +++++-
 .../golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt  | 6 +++++-
 .../golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt   | 6 +++++-
 ...orflow.estimator.-d-n-n-linear-combined-classifier.pbtxt | 6 +++++-
 ...sorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt | 6 +++++-
 ...sorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt | 6 +++++-
 .../golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt   | 6 +++++-
 .../api/golden/v1/tensorflow.estimator.-estimator.pbtxt     | 6 +++++-
 .../golden/v1/tensorflow.estimator.-linear-classifier.pbtxt | 6 +++++-
 .../golden/v1/tensorflow.estimator.-linear-estimator.pbtxt  | 6 +++++-
 .../golden/v1/tensorflow.estimator.-linear-regressor.pbtxt  | 6 +++++-
 .../api/golden/v1/tensorflow.estimator.experimental.pbtxt   | 4 ++++
 .../v2/tensorflow.estimator.-baseline-classifier.pbtxt      | 6 +++++-
 .../v2/tensorflow.estimator.-baseline-estimator.pbtxt       | 6 +++++-
 .../v2/tensorflow.estimator.-baseline-regressor.pbtxt       | 6 +++++-
 .../v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt | 6 +++++-
 .../v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt  | 6 +++++-
 .../golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt  | 6 +++++-
 .../golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt   | 6 +++++-
 ...orflow.estimator.-d-n-n-linear-combined-classifier.pbtxt | 6 +++++-
 ...sorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt | 6 +++++-
 ...sorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt | 6 +++++-
 .../golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt   | 6 +++++-
 .../api/golden/v2/tensorflow.estimator.-estimator.pbtxt     | 6 +++++-
 .../golden/v2/tensorflow.estimator.-linear-classifier.pbtxt | 6 +++++-
 .../golden/v2/tensorflow.estimator.-linear-estimator.pbtxt  | 6 +++++-
 .../golden/v2/tensorflow.estimator.-linear-regressor.pbtxt  | 6 +++++-
 .../api/golden/v2/tensorflow.estimator.experimental.pbtxt   | 4 ++++
 35 files changed, 160 insertions(+), 35 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 77fa73e613..7171587ff7 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2169,7 +2169,6 @@ class TPUEstimator(estimator_lib.Estimator):
                                builder,
                                input_receiver_fn_map,
                                checkpoint_path,
-                               strip_default_attrs,
                                save_variables=True,
                                mode=model_fn_lib.ModeKeys.PREDICT,
                                export_tags=None,
@@ -2184,7 +2183,6 @@ class TPUEstimator(estimator_lib.Estimator):
         builder,
         input_receiver_fn_map,
         checkpoint_path,
-        strip_default_attrs,
         save_variables,
         mode=mode,
         export_tags=export_tags,
@@ -2201,7 +2199,6 @@ class TPUEstimator(estimator_lib.Estimator):
           builder,
           input_receiver_fn_map,
           checkpoint_path,
-          strip_default_attrs,
           save_variables=False,
           mode=mode,
           export_tags=export_tags,
diff --git a/tensorflow/python/saved_model/constants.py b/tensorflow/python/saved_model/constants.py
index f696d4815b..1edc0c8d93 100644
--- a/tensorflow/python/saved_model/constants.py
+++ b/tensorflow/python/saved_model/constants.py
@@ -54,7 +54,7 @@ tf_export(
             __name__, "MAIN_OP_KEY")
 
 # CollectionDef key for the SavedModel train op.
-# Not exported while export_all_saved_models is in contrib.
+# Not exported while export_all_saved_models is experimental.
 TRAIN_OP_KEY = "saved_model_train_op"
 
 # Schema version for SavedModel.
diff --git a/tensorflow/python/saved_model/signature_constants.py b/tensorflow/python/saved_model/signature_constants.py
index 96460717ec..0efe176343 100644
--- a/tensorflow/python/saved_model/signature_constants.py
+++ b/tensorflow/python/saved_model/signature_constants.py
@@ -135,7 +135,7 @@ tf_export(
 
 ################################################################################
 # Train/Eval API constants.
-# Not exported while export_all_saved_models is in contrib.
+# Not exported while export_all_saved_models is experimental.
 
 SUPERVISED_TRAIN_METHOD_NAME = "tensorflow/supervised/training"
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
index af1659528b..225742539d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt
index d218773dfc..38b27f735f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
index e5794252e4..5c51767d56 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt
index 970abd8622..e138ce936e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-classifier.pbtxt
@@ -33,6 +33,10 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "experimental_feature_importances"
     argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], "
@@ -43,7 +47,7 @@ tf_class {
   }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt
index b5bbad965e..eae0a292a9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-boosted-trees-regressor.pbtxt
@@ -33,6 +33,10 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "experimental_feature_importances"
     argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], "
@@ -43,7 +47,7 @@ tf_class {
   }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
index 77e60d426e..a142ca3290 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt
index 85ff5a4fb1..09e0d38192 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index 07aefed63d..85a20828a0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
index ac13dad2d4..e311f96d3d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index 852e8d2f54..e05c7ce0a2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 2779cbe90e..fc3b1d9813 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt
index eee57462fb..bff6c86cd7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-estimator.pbtxt
@@ -31,9 +31,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
index 6569e92c6a..d213551c0b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt
index 023edec819..2148374fde 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
index d74bf4f197..004dfccb3b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
@@ -33,9 +33,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
index 4b287e1f80..2a9a0346d7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
@@ -8,6 +8,10 @@ tf_module {
     name: "LinearSDCA"
     mtype: "<type \'type\'>"
   }
+  member_method {
+    name: "build_raw_supervised_input_receiver_fn"
+    argspec: "args=[\'features\', \'labels\', \'default_batch_size\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "call_logit_fn"
     argspec: "args=[\'logit_fn\', \'features\', \'mode\', \'params\', \'config\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
index 07483df83e..22cbcf08f1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
index d218773dfc..38b27f735f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
index 292b5f32d8..a965042b41 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt
index 970abd8622..e138ce936e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-classifier.pbtxt
@@ -33,6 +33,10 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "experimental_feature_importances"
     argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], "
@@ -43,7 +47,7 @@ tf_class {
   }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt
index b5bbad965e..eae0a292a9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-boosted-trees-regressor.pbtxt
@@ -33,6 +33,10 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "experimental_feature_importances"
     argspec: "args=[\'self\', \'normalize\'], varargs=None, keywords=None, defaults=[\'False\'], "
@@ -43,7 +47,7 @@ tf_class {
   }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
index c542edf64d..f6bd4d2121 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
index 85ff5a4fb1..09e0d38192 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index 623cbc3648..60627cc197 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
index ac13dad2d4..e311f96d3d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index f45e76537a..dc6aca25dd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 8db2196512..7338abc1d9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt
index 71531fd217..a1f0e76c8b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-estimator.pbtxt
@@ -30,9 +30,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "get_variable_names"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
index 72c226b25d..6559c581fb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
index 023edec819..2148374fde 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
index c4bb19612a..e6ea074ff8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
@@ -32,9 +32,13 @@ tf_class {
     name: "evaluate"
     argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "experimental_export_all_saved_models"
+    argspec: "args=[\'self\', \'export_dir_base\', \'input_receiver_fn_map\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+  }
   member_method {
     name: "export_saved_model"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
   member_method {
     name: "export_savedmodel"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
index 4b287e1f80..2a9a0346d7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
@@ -8,6 +8,10 @@ tf_module {
     name: "LinearSDCA"
     mtype: "<type \'type\'>"
   }
+  member_method {
+    name: "build_raw_supervised_input_receiver_fn"
+    argspec: "args=[\'features\', \'labels\', \'default_batch_size\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "call_logit_fn"
     argspec: "args=[\'logit_fn\', \'features\', \'mode\', \'params\', \'config\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 8514eb2958f5872dbc7e4d8f7ccdfb81cd5619fe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 10:15:52 -0800
Subject: [PATCH 0939/1554] Fix uneven spatial partition.

PiperOrigin-RevId: 223189870
---
 tensorflow/contrib/tpu/python/tpu/tpu_feed.py | 34 +++++++++++--------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
index cf36103277..d5957b7e8e 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
@@ -833,24 +833,30 @@ class _PartitionedInfeedQueue(InfeedQueue):
     dims = np.array(dims)
     self._check_input_partition_dims(tensor, dims)
     output = [tensor]
-    divds, remainders = np.divmod(np.array(tensor.shape.as_list()), dims)
-    for axis, (divd, remainder, dim) in enumerate(
-        np.dstack((divds, remainders, dims))[0]):
+    shape_list = np.array(tensor.shape.as_list())
+    quotients, remainders = np.divmod(shape_list, dims)
+    for axis, (quotient, remainder, dim, original_size) in enumerate(
+        zip(quotients, remainders, dims, shape_list)):
       if dim <= 1:
         continue
       if remainder > 0:
         # For each dimension, when it cannot be evenly partitioned, XLA assumes
-        # the size of last parts are smaller by 1. E.g. 2D tensor with shape
-        # (5, 14) and dims are (2, 4). Since 5 % 2 = 1 and 14 % 4 = 2, [5, 14]
-        # => [[(3, 3), (3, 3), (2, 3), (2, 3)],
-        # [(2, 3), (2, 3), (2, 2), (2, 2)]]
-        output = [
-            array_ops.split(
-                x,
-                num_or_size_splits=[divd + 1] * remainder +
-                [divd] * (dim - remainder),
-                axis=axis) for x in output
-        ]
+        # tensors are partitioned in a greedy manner by using
+        # ceil_ratio(size/dim) first. E.g. 2D tensor with shape (5, 14) and dims
+        # are (2, 4). Since 5 % 2 = 1 and 14 % 4 = 2, [5, 14] =>
+        # [[(3, 4), (3, 4), (2, 4), (2, 2)],
+        # [(2, 4), (2, 4), (2, 4), (2, 2)]]
+        ceil_ratio = quotient + 1
+        num_full_slots, left_over = np.divmod(original_size, ceil_ratio)
+        num_or_size_splits = [ceil_ratio] * num_full_slots + [left_over]
+        if len(num_or_size_splits) < dim:
+          num_or_size_splits += [0] * (dim - len(num_or_size_splits))
+        new_output = []
+        for x in output:
+          new_output.append(
+              array_ops.split(
+                  x, num_or_size_splits=num_or_size_splits, axis=axis))
+        output = new_output
       else:
         output = [array_ops.split(x, dim, axis=axis) for x in output]
       output = nest.flatten(output)
-- 
GitLab


From bb75a3f0f6be966ba01a242f5d06170dddef0235 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 28 Nov 2018 18:37:28 +0000
Subject: [PATCH 0940/1554] Fix broken path in lite docs

This fix fixes broken link in lite docs

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/lite/tools/optimize/g3doc/quantize_weights.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md b/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md
index 2517882c84..cea164c38f 100644
--- a/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md
+++ b/tensorflow/lite/tools/optimize/g3doc/quantize_weights.md
@@ -3,7 +3,7 @@
 ## Recommended usage
 
 The Quantize Weights transformation is integrated with
-[tflite_convert](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/toco/g3doc/cmdline_reference.md#transformation-flags).
+[tflite_convert](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/convert/cmdline_reference.md#transformation-flags).
 
 The recommended way of invoking this tool is by simply adding the
 `--post_training_quantize` flag to your original tflite_convert invocation. For
-- 
GitLab


From 3553486256ae68ac72ed1ed2688a055381d818a5 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Wed, 28 Nov 2018 10:55:57 -0800
Subject: [PATCH 0941/1554] [TF:XLA] Do the final division of reduce_mean in
 accumulation type, not input type

This reduces the probability of overflow in fp16, matching TF without XLA.

PiperOrigin-RevId: 223197560
---
 tensorflow/compiler/tests/reduce_ops_test.py             | 6 ++++++
 tensorflow/compiler/tf2xla/kernels/reduction_ops.cc      | 7 +++----
 tensorflow/compiler/tf2xla/kernels/reduction_ops.h       | 4 ++--
 .../compiler/tf2xla/kernels/reduction_ops_common.cc      | 9 ++++-----
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/tests/reduce_ops_test.py b/tensorflow/compiler/tests/reduce_ops_test.py
index 132c59c32c..e8fc81bbb5 100644
--- a/tensorflow/compiler/tests/reduce_ops_test.py
+++ b/tensorflow/compiler/tests/reduce_ops_test.py
@@ -91,6 +91,7 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase):
       np.array([], dtype=np.bool).reshape(0, 3),
       np.array([[False, True, False], [True, True, False]]),
   ]
+  ONES = [np.ones([34000, 2])]
 
   def testReduceSumF32(self, index_dtype):
     self._testReduction(math_ops.reduce_sum, np.sum, np.float32, self.REAL_DATA,
@@ -149,6 +150,11 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase):
     self._testReduction(math_ops.reduce_mean, np.mean, np.float32,
                         self.NONEMPTY_REAL_DATA, index_dtype)
 
+  def testReduceMeanF16(self, index_dtype):
+    if np.float16 in self.all_types:
+      self._testReduction(math_ops.reduce_mean, np.mean, np.float16, self.ONES,
+                          index_dtype)
+
   def testReduceMeanC64(self, index_dtype):
     self._testReduction(math_ops.reduce_mean, np.mean, np.complex64,
                         self.NONEMPTY_COMPLEX_DATA, index_dtype)
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
index 132160de70..65e158d64f 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
@@ -125,10 +125,9 @@ class MeanOp : public XlaReductionOp {
       auto size = xla::GetDimensionSize(input, dimensions_to_reduce[i]);
       divisor = xla::Mul(divisor, size);
     }
-    xla::PrimitiveType type;
-    TF_CHECK_OK(DataTypeToPrimitiveType(input_type(0), &type));
-    divisor = xla::ConvertElementType(divisor, type);
-    return reduce_output / divisor;
+    divisor = xla::ConvertElementType(divisor, xla_reduction_type_);
+    return XlaHelpers::ConvertElementType(reduce_output / divisor,
+                                          input_type(0));
   }
 };
 
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
index 8f1667df5b..af716eab79 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
@@ -50,8 +50,8 @@ class XlaReductionOp : public XlaOpKernel {
   // Applies a transformation to the output of the reduction. The desired
   // computation should be added to 'builder'. Argument 'input' is the original
   // input of the reduction; 'reduce_output' is the output of the reduction.
-  // Returns the transformed reduction output, Defaults to returning
-  // 'reduce_output' unchanged.
+  // Returns the transformed reduction output. Defaults to returning
+  // 'reduce_output' converted to the input type.
   virtual xla::XlaOp BuildFinalizer(
       xla::XlaBuilder* builder, const xla::XlaOp& input,
       const xla::XlaOp& reduce_output,
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
index b9bb5edba3..2ca2a85244 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
@@ -35,13 +35,13 @@ XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx,
       ctx, DataTypeToPrimitiveType(reduction_type_, &xla_reduction_type_));
 }
 
-// Unless BuildFinalizer is overridden the reduction has no
-// finalizer.
+// The default finalizer converts the results back into the input type. This can
+// be overridden.
 xla::XlaOp XlaReductionOp::BuildFinalizer(
     xla::XlaBuilder* /*builder*/, const xla::XlaOp& /*input*/,
     const xla::XlaOp& reduce_output,
     const std::vector<int64>& /*dimensions_to_reduce*/) {
-  return reduce_output;
+  return XlaHelpers::ConvertElementType(reduce_output, input_type(0));
 }
 
 void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
@@ -117,8 +117,7 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
   xla::XlaComputation reduction_computation = r.Build().ConsumeValueOrDie();
 
   auto reduce = xla::Reduce(data, initial, reduction_computation, xla_axes);
-  auto deconverted = XlaHelpers::ConvertElementType(reduce, input_type(0));
-  auto finalized = BuildFinalizer(b, data, deconverted, xla_axes);
+  auto finalized = BuildFinalizer(b, data, reduce, xla_axes);
   auto result = keep_dims_ ? xla::Reshape(finalized, final_shape) : finalized;
   ctx->SetOutput(0, result);
 }
-- 
GitLab


From 652cc2b1ce266c70a7f9ce30570d477d7f74f12a Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 28 Nov 2018 11:33:29 -0800
Subject: [PATCH 0942/1554] Update TF 2.0 Endpoints

PiperOrigin-RevId: 223205389
---
 tensorflow/python/ops/nn_impl.py              | 40 ++++++++++++++++++-
 tensorflow/python/ops/nn_test.py              | 32 +++++++--------
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  6 +--
 .../tools/compatibility/tf_upgrade_v2.py      | 40 ++++++++++++-------
 4 files changed, 80 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 6591da5be8..26bd13eaee 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -861,7 +861,7 @@ def normalize_moments(counts, mean_ss, variance_ss, shift, name=None):
   return (mean, variance)
 
 
-@tf_export("nn.moments")
+@tf_export(v1=["nn.moments"])
 def moments(
     x,
     axes,
@@ -920,6 +920,42 @@ def moments(
       return (mean, variance)
 
 
+@tf_export("nn.moments", v1=[])
+def moments_v2(
+    x,
+    axes,
+    shift=None,
+    keepdims=False,
+    name=None):
+  """Calculates the mean and variance of `x`.
+
+  The mean and variance are calculated by aggregating the contents of `x`
+  across `axes`.  If `x` is 1-D and `axes = [0]` this is just the mean
+  and variance of a vector.
+
+  Note: shift is currently not used; the true mean is computed and used.
+
+  When using these moments for batch normalization (see
+  `tf.nn.batch_normalization`):
+
+   * for so-called "global normalization", used with convolutional filters with
+     shape `[batch, height, width, depth]`, pass `axes=[0, 1, 2]`.
+   * for simple batch normalization pass `axes=[0]` (batch only).
+
+  Args:
+    x: A `Tensor`.
+    axes: Array of ints.  Axes along which to compute mean and
+      variance.
+    shift: Not used in the current implementation.
+    keepdims: produce moments with the same dimensionality as the input.
+    name: Name used to scope the operations that compute the moments.
+
+  Returns:
+    Two `Tensor` objects: `mean` and `variance`.
+  """
+  return moments(x=x, axes=axes, shift=shift, name=name, keep_dims=keepdims)
+
+
 @tf_export(v1=["nn.weighted_moments"])
 def weighted_moments(x, axes, frequency_weights, name=None, keep_dims=False):
   """Returns the frequency-weighted mean and variance of `x`.
@@ -1076,7 +1112,7 @@ def batch_normalization(x,
         offset - mean * inv if offset is not None else -mean * inv, x.dtype)
 
 
-@tf_export("nn.fused_batch_norm")
+@tf_export(v1=["nn.fused_batch_norm"])
 def fused_batch_norm(
     x,
     scale,
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 96b9d6fc0d..f7c8a7a70a 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -101,8 +101,8 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase):
     x_np = np.random.randn(*x_shape).astype(np.float32)
     y_np = self._softmax(x_np)
     x_tf = constant_op.constant(x_np)
-    y_tf = nn_ops.softmax(x_tf)
-    y_tf_last_dim = nn_ops.softmax(x_tf, 1)
+    y_tf = nn_ops.softmax_v2(x_tf)
+    y_tf_last_dim = nn_ops.softmax_v2(x_tf, 1)
     y_tf_np = self.evaluate(y_tf)
     y_tf_last_dim_np = self.evaluate(y_tf_last_dim)
     eps = 1e-3
@@ -111,9 +111,9 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase):
 
   def testSoftmaxAxes(self):
     arr = np.linspace(0., 1, 12).reshape(3, 4)
-    x_neg_axis = nn_ops.softmax(arr, axis=-2)
-    y_pos_axis = nn_ops.softmax(arr, axis=0)
-    z_gt_axis = nn_ops.softmax(arr, axis=0)
+    x_neg_axis = nn_ops.softmax_v2(arr, axis=-2)
+    y_pos_axis = nn_ops.softmax_v2(arr, axis=0)
+    z_gt_axis = nn_ops.softmax_v2(arr, axis=0)
     x_neg_axis_tf = self.evaluate(x_neg_axis)
     y_pos_axis_tf = self.evaluate(y_pos_axis)
     z_gt_axis_tf = self.evaluate(z_gt_axis)
@@ -126,7 +126,7 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase):
     x_np = np.random.randn(*x_shape).astype(np.float64)
     with self.cached_session():
       x_tf = constant_op.constant(x_np)
-      y_tf = nn_ops.softmax(x_tf)
+      y_tf = nn_ops.softmax_v2(x_tf)
       err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf,
                                                     x_shape)
     eps = 2e-8
@@ -189,16 +189,16 @@ class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase):
     x_np = np.random.randn(*x_shape).astype(np.float32)
     y_np = self._log_softmax(x_np)
     x_tf = constant_op.constant(x_np)
-    y_tf = nn_ops.log_softmax(x_tf)
+    y_tf = nn_ops.log_softmax_v2(x_tf)
     y_tf_np = self.evaluate(y_tf)
     eps = 1e-3
     self.assertAllClose(y_tf_np, y_np, eps)
 
   def testLogSoftmaxAxes(self):
     arr = np.linspace(0., 1, 12).reshape(3, 4)
-    x_neg_axis = nn_ops.log_softmax(arr, axis=-2)
-    y_pos_axis = nn_ops.log_softmax(arr, axis=0)
-    z_gt_axis = nn_ops.log_softmax(arr, axis=0)
+    x_neg_axis = nn_ops.log_softmax_v2(arr, axis=-2)
+    y_pos_axis = nn_ops.log_softmax_v2(arr, axis=0)
+    z_gt_axis = nn_ops.log_softmax_v2(arr, axis=0)
     x_neg_axis_tf = self.evaluate(x_neg_axis)
     y_pos_axis_tf = self.evaluate(y_pos_axis)
     z_gt_axis_tf = self.evaluate(z_gt_axis)
@@ -211,7 +211,7 @@ class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase):
     x_np = np.random.randn(*x_shape).astype(np.float64)
     with self.cached_session():
       x_tf = constant_op.constant(x_np)
-      y_tf = nn_ops.log_softmax(x_tf)
+      y_tf = nn_ops.log_softmax_v2(x_tf)
       err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf,
                                                     x_shape)
     eps = 1e-7
@@ -262,7 +262,7 @@ class L2NormalizeTest(test_lib.TestCase):
     for dim in range(len(x_shape)):
       y_np = self._l2Normalize(x_np, dim)
       x_tf = constant_op.constant(x_np, name="x")
-      y_tf = nn_impl.l2_normalize(x_tf, dim)
+      y_tf = nn_impl.l2_normalize_v2(x_tf, dim)
       self.assertAllClose(y_np, self.evaluate(y_tf))
 
   @test_util.run_in_graph_and_eager_modes
@@ -273,7 +273,7 @@ class L2NormalizeTest(test_lib.TestCase):
     dim = [1, 2]
     y_np = self._l2Normalize(x_np, dim)
     x_tf = constant_op.constant(x_np, name="x")
-    y_tf = nn_impl.l2_normalize(x_tf, dim)
+    y_tf = nn_impl.l2_normalize_v2(x_tf, dim)
     self.assertAllClose(y_np, self.evaluate(y_tf))
 
   def testL2NormalizeGradient(self):
@@ -283,7 +283,7 @@ class L2NormalizeTest(test_lib.TestCase):
     for dim in range(len(x_shape)):
       with self.cached_session():
         x_tf = constant_op.constant(x_np, name="x")
-        y_tf = nn_impl.l2_normalize(x_tf, dim)
+        y_tf = nn_impl.l2_normalize_v2(x_tf, dim)
         err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf,
                                                       x_shape)
       print("L2Normalize gradient err = %g " % err)
@@ -1034,8 +1034,8 @@ class MomentsTest(test_lib.TestCase):
             with self.session(graph=g) as sess:
               inputs = constant_op.constant(
                   input_values, shape=input_shape, dtype=dtypes.float32)
-              mean, variance = nn_impl.moments(
-                  inputs, moments_axes, keep_dims=keep_dims)
+              mean, variance = nn_impl.moments_v2(
+                  inputs, moments_axes, keepdims=keep_dims)
 
               if check_gradients:
                 err = gradient_checker.compute_gradient_error(
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 04b189784a..b4d0f95bc3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -156,10 +156,6 @@ tf_module {
     name: "fractional_max_pool"
     argspec: "args=[\'value\', \'pooling_ratio\', \'pseudo_random\', \'overlapping\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'0\', \'None\'], "
   }
-  member_method {
-    name: "fused_batch_norm"
-    argspec: "args=[\'x\', \'scale\', \'offset\', \'mean\', \'variance\', \'epsilon\', \'data_format\', \'is_training\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'0.001\', \'NHWC\', \'True\', \'None\'], "
-  }
   member_method {
     name: "in_top_k"
     argspec: "args=[\'predictions\', \'targets\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -210,7 +206,7 @@ tf_module {
   }
   member_method {
     name: "moments"
-    argspec: "args=[\'x\', \'axes\', \'shift\', \'name\', \'keep_dims\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], "
+    argspec: "args=[\'x\', \'axes\', \'shift\', \'keepdims\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
   member_method {
     name: "nce_loss"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3cb78afb32..6e8d6e2d0b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -58,26 +58,44 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
+        "tf.linalg.l2_normalize": {
+            "dim": "axis",
+        },
         "tf.math.count_nonzero": {
             "input_tensor": "input",
             "keep_dims": "keepdims",
             "reduction_indices": "axis",
         },
+        "tf.math.l2_normalize": {
+            "dim": "axis",
+        },
+        "tf.math.log_softmax": {
+            "dim": "axis",
+        },
+        "tf.math.softmax": {
+            "dim": "axis"
+        },
+        "tf.nn.l2_normalize": {
+            "dim": "axis",
+        },
+        "tf.nn.log_softmax": {
+            "dim": "axis",
+        },
+        "tf.nn.moments": {
+            "keep_dims": "keepdims",
+        },
         "tf.nn.pool": {
             "dilation_rate": "dilations"
         },
         "tf.nn.separable_conv2d": {
             "rate": "dilations"
         },
+        "tf.nn.softmax": {
+            "dim": "axis"
+        },
         "tf.nn.sufficient_statistics": {
             "keep_dims": "keepdims"
         },
-        "tf.nn.log_softmax": {
-            "dim": "axis",
-        },
-        "tf.nn.softmax": {
-            "dim": "axis",
-        },
         "tf.debugging.assert_all_finite": {
             "t": "x",
             "msg": "message",
@@ -165,15 +183,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
-        "tf.linalg.l2_normalize": {
-            "dim": "axis",
-        },
-        "tf.math.l2_normalize": {
-            "dim": "axis",
-        },
-        "tf.nn.l2_normalize": {
-            "dim": "axis",
-        },
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
@@ -422,6 +431,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.batch_to_space": ["input", "crops", "block_size", "name"],
         "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
         "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
+        "tf.nn.moments": ["x", "axes", "shift", "keepdims", "name"],
         "tf.nn.convolution": [
             "input", "filter", "padding", "strides", "dilation_rate", "name",
             "data_format"
-- 
GitLab


From e55a58c9bf6adf07dce074813b54fc5b16c5be1d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 11:42:51 -0800
Subject: [PATCH 0943/1554] Replace `tf.estimator.inputs` with
 `tf.compat.v1.estimator.inputs`

PiperOrigin-RevId: 223207243
---
 tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py  | 2 +-
 tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py b/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py
index 29552d24f1..59a67636ae 100644
--- a/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py
+++ b/tensorflow/contrib/learn/python/learn/learn_io/numpy_io.py
@@ -27,7 +27,7 @@ from tensorflow.python.estimator.inputs.numpy_io import numpy_input_fn as core_n
 from tensorflow.python.util.deprecation import deprecated
 
 
-@deprecated(None, 'Use tf.estimator.inputs.numpy_input_fn.')
+@deprecated(None, 'Use tf.compat.v1.estimator.inputs.numpy_input_fn.')
 def numpy_input_fn(x,
                    y=None,
                    batch_size=128,
diff --git a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py b/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py
index b4ef055f5a..e9df7258a3 100644
--- a/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py
+++ b/tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py
@@ -53,7 +53,7 @@ PANDAS_DTYPES = {
 }
 
 
-@deprecated(None, 'Please use tf.estimator.inputs.pandas_input_fn')
+@deprecated(None, 'Please use tf.compat.v1.estimator.inputs.pandas_input_fn')
 def pandas_input_fn(x,
                     y=None,
                     batch_size=128,
-- 
GitLab


From e1d54f683f7ca1e7a9377937b34f2f37bcc9a6a8 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 28 Nov 2018 11:50:44 -0800
Subject: [PATCH 0944/1554] Fix clang-format.

---
 .../contrib/tensorrt/convert/convert_nodes.cc     |  8 ++++----
 .../tensorrt/convert/convert_nodes_test.cc        | 15 ++++++---------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 6de3aba4d7..3b741a60a7 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2500,8 +2500,8 @@ tensorflow::Status ConvertSquare(OpConverterParams* params) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
   if (inputs.size() != 1) {
-    return tensorflow::errors::InvalidArgument(
-        "Square expects one input, at ", node_def.name());
+    return tensorflow::errors::InvalidArgument("Square expects one input, at ",
+                                               node_def.name());
   }
   if (inputs.at(0).is_weights()) {
     return tensorflow::errors::Unimplemented(
@@ -2516,8 +2516,8 @@ tensorflow::Status ConvertSquare(OpConverterParams* params) {
   }
   TRT_ShapedWeights weights = params->weight_store->GetTempWeights(
       tensorflow::DataType::DT_FLOAT, dims);
-  auto weights_ptr = static_cast<float*>(const_cast<void*>(
-      weights.GetValues()));
+  auto weights_ptr =
+      static_cast<float*>(const_cast<void*>(weights.GetValues()));
   weights_ptr[0] = 2.f;
   nvinfer1::IConstantLayer* const2_layer =
       params->converter->network()->addConstant(dims, weights.GetTrtWeights());
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index c6cd765887..4894e30b46 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -152,8 +152,7 @@ void ExpectTrtDimsEqualsArray(const std::vector<int>& lhs,
 }
 
 template <typename T>
-void ExpectArrayNear(const std::vector<T>& lhs,
-                     const std::vector<T>& rhs) {
+void ExpectArrayNear(const std::vector<T>& lhs, const std::vector<T>& rhs) {
   ASSERT_EQ(lhs.size(), rhs.size());
   for (int i = 0; i < lhs.size(); i++) {
     EXPECT_FLOAT_EQ(lhs[i], rhs[i]);
@@ -2011,9 +2010,7 @@ void TestConvertSquare(OpConverterTest* test) {
     expected_output_data[i] = value * value;
   }
   std::vector<CType> output_data(num_inputs);
-  test->BuildAndRun<CType>(
-          {{"input", input_data}}, "my_square",
-          &output_data);
+  test->BuildAndRun<CType>({{"input", input_data}}, "my_square", &output_data);
   ExpectArrayNear(expected_output_data, output_data);
 }
 
@@ -2021,9 +2018,8 @@ TEST_F(OpConverterTest, ConvertSquare) {
   {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_square", "Square", {});
-    RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "Square expects one input, at my_square");
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+                               "Square expects one input, at my_square");
   }
   {
     // Input is weights, should fail.
@@ -2033,7 +2029,8 @@ TEST_F(OpConverterTest, ConvertSquare) {
     auto square = ops::Square(s.WithOpName("my_square"), input);
     NodeDef node_def = square.operation.node()->def();
     AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, -5, 6});
-    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
         "Square is only implemented for tensors, at my_square");
   }
 
-- 
GitLab


From 9433922095a43ca2a8a03373f58c01ab00c8cafe Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Wed, 28 Nov 2018 11:54:39 -0800
Subject: [PATCH 0945/1554] Make tflite_convert error if input_shapes are
 invalid.

PiperOrigin-RevId: 223209299
---
 tensorflow/lite/python/convert_saved_model.py      | 12 +++++++++---
 tensorflow/lite/python/convert_saved_model_test.py |  8 ++++++--
 tensorflow/lite/python/lite_test.py                | 13 +++++++------
 3 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/tensorflow/lite/python/convert_saved_model.py b/tensorflow/lite/python/convert_saved_model.py
index 3f54d2559c..ad7c87109c 100644
--- a/tensorflow/lite/python/convert_saved_model.py
+++ b/tensorflow/lite/python/convert_saved_model.py
@@ -197,12 +197,18 @@ def set_tensor_shapes(tensors, shapes):
     tensors: TensorFlow ops.Tensor.
     shapes: Dict of strings representing input tensor names to list of
       integers representing input shapes (e.g., {"foo": : [1, 16, 16, 3]}).
+
+  Raises:
+    ValueError: `shapes` contains an invalid tensor.
   """
   if shapes:
-    for tensor in tensors:
-      shape = shapes.get(tensor_name(tensor))
+    tensor_names_to_tensor = {tensor_name(tensor): tensor for tensor in tensors}
+    for name, shape in shapes.items():
+      if name not in tensor_names_to_tensor:
+        raise ValueError("Invalid tensor \'{}\' found in tensor shapes "
+                         "map.".format(name))
       if shape is not None:
-        tensor.set_shape(shape)
+        tensor_names_to_tensor[name].set_shape(shape)
 
 
 def freeze_saved_model(saved_model_dir, input_arrays, input_shapes,
diff --git a/tensorflow/lite/python/convert_saved_model_test.py b/tensorflow/lite/python/convert_saved_model_test.py
index dff582f1a1..0d32c34391 100644
--- a/tensorflow/lite/python/convert_saved_model_test.py
+++ b/tensorflow/lite/python/convert_saved_model_test.py
@@ -79,8 +79,12 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
 
-    convert_saved_model.set_tensor_shapes([tensor],
-                                          {"invalid-input": [5, 3, 5]})
+    with self.assertRaises(ValueError) as error:
+      convert_saved_model.set_tensor_shapes([tensor],
+                                            {"invalid-input": [5, 3, 5]})
+    self.assertEqual(
+        "Invalid tensor 'invalid-input' found in tensor shapes map.",
+        str(error.exception))
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
 
   def testSetTensorShapeEmpty(self):
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 5a5697db92..8832247f3c 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -931,12 +931,13 @@ class FromKerasFile(test_util.TensorFlowTestCase):
     """Test a Sequential tf.keras model testing input shapes argument."""
     keras_file = self._getSequentialModel()
 
-    # Passing in shape of invalid input array has no impact as long as all input
-    # arrays have a shape.
-    converter = lite.TFLiteConverter.from_keras_model_file(
-        keras_file, input_shapes={'invalid-input': [2, 3]})
-    tflite_model = converter.convert()
-    self.assertTrue(tflite_model)
+    # Passing in shape of invalid input array raises error.
+    with self.assertRaises(ValueError) as error:
+      converter = lite.TFLiteConverter.from_keras_model_file(
+          keras_file, input_shapes={'invalid-input': [2, 3]})
+    self.assertEqual(
+        "Invalid tensor 'invalid-input' found in tensor shapes map.",
+        str(error.exception))
 
     # Passing in shape of valid input array.
     converter = lite.TFLiteConverter.from_keras_model_file(
-- 
GitLab


From f3239b05dbf57c231030ec04162c76f0fc120c5d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Wed, 28 Nov 2018 11:58:50 -0800
Subject: [PATCH 0946/1554] Use the minimal verbosity level by default for
 autograph in tf.function.

PiperOrigin-RevId: 223209980
---
 tensorflow/python/framework/func_graph.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index f1e508c365..9960e23e6f 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -397,7 +397,7 @@ def func_graph_from_py_func(name,
           return autograph.converted_call(
               original_func, None,
               autograph.ConversionOptions(
-                  verbose=True,
+                  verbose=autograph.Verbosity.BRIEF,
                   recursive=True,
                   strip_decorators=(function.defun, def_function.function),
                   optional_features=(),
-- 
GitLab


From 0d40f081369150bdab2ca49fe173d2a19717c0f7 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Wed, 28 Nov 2018 12:16:50 -0800
Subject: [PATCH 0947/1554] Simplify semantics of `build`:

`build(input_shape)` is a method that implementers of subclasses of Layer or Model can override if they need a state-creation step in-between layer instantiation and layer call. This is typically used to create the weights of Layer subclasses.

If you do not override it, it is never called by the framework (i.e. default implementations of `build` are *never called* unless you explicitly call `model.build()` yourself, therefore they never introduce surprising issues).

In addition, you can call `model.build(input_shape)` on a Model instance in a standalone way, as a substitute for calling the model on some data to create its weights. This may raise an exception if the request is impossible (then you have to call the model on some data yourself in order to build it).

The change also fixes two related issues:
- enable `build` to be called on Model subclasses with a signature containing keyword arguments with default values (e.g. `call(self, inputs, training=False, mask=None)`.
- enable a Model subclass to be nested in a Sequential model without having to implement `compute_output_shape` on the Model subclass.

PiperOrigin-RevId: 223213320
---
 tensorflow/python/keras/engine/base_layer.py  | 52 ++++++-----
 tensorflow/python/keras/engine/network.py     | 88 +++++++++++--------
 tensorflow/python/keras/engine/saving_test.py |  1 -
 tensorflow/python/keras/engine/sequential.py  | 19 ++--
 .../python/keras/engine/sequential_test.py    |  1 -
 tensorflow/python/keras/engine/training.py    | 10 +--
 tensorflow/python/keras/layers/local_test.py  | 16 ++--
 .../python/keras/model_subclassing_test.py    | 58 +++++++++++-
 8 files changed, 160 insertions(+), 85 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 8b795935c1..8e35300342 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -200,7 +200,19 @@ class Layer(checkpointable.CheckpointableBase):
       self._initial_weights = None
 
   def build(self, input_shape):
-    """Creates the variables of the layer."""
+    """Creates the variables of the layer (optional, for subclass implementers).
+
+    This is a method that implementers of subclasses of `Layer` or `Model`
+    can override if they need a state-creation step in-between
+    layer instantiation and layer call.
+
+    This is typically used to create the weights of `Layer` subclasses.
+
+    Arguments:
+      input_shape: Instance of `TensorShape`, or list of instances of
+        `TensorShape` if the layer expects a list of inputs
+        (one instance per input).
+    """
     self.built = True
 
   @doc_controls.for_subclass_implementers
@@ -517,25 +529,8 @@ class Layer(checkpointable.CheckpointableBase):
 
     with ops.name_scope(self._name_scope()):
       if not self.built:
-        # Check input assumptions set before layer building, e.g. input rank.
-        input_spec.assert_input_compatibility(
-            self.input_spec, inputs, self.name)
-        if input_list and self._dtype is None:
-          try:
-            self._dtype = input_list[0].dtype.base_dtype.name
-          except AttributeError:
-            pass
-
-        if all(hasattr(x, 'shape') for x in input_list):
-          input_shapes = nest.map_structure(lambda x: x.shape, inputs)
-
-        if (not hasattr(self, '_is_graph_network') or
-            self.__class__.__name__ == 'Sequential' or
-            not hasattr(self.build, '_is_default')):
-          # Only if self is a layer, an instance of a sequential model, or
-          # the user has manually overwritten the build method do we need to
-          # build it.
-          self.build(input_shapes)
+        # Build layer if applicable (if the `build` method has been overridden).
+        self._maybe_build(inputs)
         # We must set self.built since user defined build functions are not
         # constrained to set self.built.
         self.built = True
@@ -1580,6 +1575,23 @@ class Layer(checkpointable.CheckpointableBase):
     """
     return self._call_is_graph_friendly
 
+  def _maybe_build(self, inputs):
+    # Check input assumptions set before layer building, e.g. input rank.
+    input_spec.assert_input_compatibility(
+        self.input_spec, inputs, self.name)
+    input_list = nest.flatten(inputs)
+    if input_list and self._dtype is None:
+      try:
+        self._dtype = input_list[0].dtype.base_dtype.name
+      except AttributeError:
+        pass
+    input_shapes = None
+    if all(hasattr(x, 'shape') for x in input_list):
+      input_shapes = nest.map_structure(lambda x: x.shape, inputs)
+    # Only call `build` if the user has manually overridden the build method.
+    if not hasattr(self.build, '_is_default'):
+      self.build(input_shapes)
+
 
 class Node(object):
   """A `Node` describes the connectivity between two layers.
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 1040fd8ea3..93ae667c8b 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -769,6 +769,11 @@ class Network(base_layer.Layer):
     This is to be used for subclassed models, which do not know at instantiation
     time what their inputs look like.
 
+    This method only exists for users who want to call `model.build()` in a
+    standalone way (as a substitute for calling the model on real data to
+    build it). It will never be called by the framework (and thus it will
+    never throw unexpected errors in an unrelated workflow).
+
     Args:
      input_shape: Single tuple, TensorShape, or list of shapes, where shapes
          are tuples, integers, or TensorShapes.
@@ -805,48 +810,53 @@ class Network(base_layer.Layer):
       # in a Graph. Since tf.Variable is compatible with both eager execution
       # and graph building, the variables created after building the model in
       # a Graph are still valid when executing eagerly.
-      with context.graph_mode():
-        graph = func_graph.FuncGraph('graph')
-        with graph.as_default():
-          if isinstance(input_shape, list):
-            x = [base_layer_utils.generate_placeholders_from_shape(shape)
-                 for shape in input_shape]
+      if context.executing_eagerly():
+        graph = func_graph.FuncGraph('build_graph')
+      else:
+        graph = backend.get_graph()
+      with graph.as_default():
+        if isinstance(input_shape, list):
+          x = [base_layer_utils.generate_placeholders_from_shape(shape)
+               for shape in input_shape]
+        else:
+          x = base_layer_utils.generate_placeholders_from_shape(input_shape)
+
+        kwargs = {}
+        call_signature = tf_inspect.getfullargspec(self.call)
+        call_args = call_signature.args
+        # Exclude `self`, `inputs`, and any argument with a default value.
+        if len(call_args) > 2:
+          if call_signature.defaults:
+            call_args = call_args[2:-len(call_signature.defaults)]
           else:
-            x = base_layer_utils.generate_placeholders_from_shape(input_shape)
-
-          kwargs = {}
-          num_call_args = len(tf_inspect.getfullargspec(self.call).args)
-          if self._expects_training_arg and num_call_args == 3:
-            # Has call signature of call(self, input, training)
-            kwargs['training'] = False
-          elif num_call_args > 2:
-            # Has invalid call signature of call(self, input, *args, **kwargs)
-            raise ValueError('Currently, you cannot build your model if it has '
-                             'positional or keyword arguments that are not '
-                             'inputs to the model, but are required for its '
-                             '`call` method. Instead, in order to instantiate '
-                             'and build your model, `call` your model on real '
-                             'tensor data with all expected call arguments.')
-
-          try:
-            self.call(x, **kwargs)
-          except (errors.InvalidArgumentError, TypeError):
-            raise ValueError('You cannot build your model by calling `build` '
-                             'if your layers do not support float type inputs. '
-                             'Instead, in order to instantiate and build your '
-                             'model, `call` your model on real tensor data (of '
-                             'the correct dtype).')
-
+            call_args = call_args[2:]
+          for arg in call_args:
+            if arg == 'training':
+              # Case where `training` is a positional arg with no default.
+              kwargs['training'] = False
+            else:
+              # Has invalid call signature with unknown positional arguments.
+              raise ValueError(
+                  'Currently, you cannot build your model if it has '
+                  'positional or keyword arguments that are not '
+                  'inputs to the model, but are required for its '
+                  '`call` method. Instead, in order to instantiate '
+                  'and build your model, `call` your model on real '
+                  'tensor data with all expected call arguments.')
+        elif len(call_args) < 2:
+          # Signature without `inputs`.
+          raise ValueError('You can only call `build` on a model if its `call` '
+                           'method accepts an `inputs` argument.')
+        try:
+          self.call(x, **kwargs)
+        except (errors.InvalidArgumentError, TypeError):
+          raise ValueError('You cannot build your model by calling `build` '
+                           'if your layers do not support float type inputs. '
+                           'Instead, in order to instantiate and build your '
+                           'model, `call` your model on real tensor data (of '
+                           'the correct dtype).')
     if self._layers:
       self._track_layers(self._layers)
-    if self.layers:
-      for layer in self.layers:
-        if not layer.built:
-          raise ValueError('Layer: {} was not built in your model. Calling '
-                           '`build` manually on a subclassed model is only '
-                           'allowed for models with a static topology. '
-                           'In this case, you can build your model by '
-                           'calling it on real tensor data.'.format(layer))
     self.built = True
 
   def call(self, inputs, training=None, mask=None):
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py
index 375d1010d1..b92f06449e 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/engine/saving_test.py
@@ -716,7 +716,6 @@ class TestWholeModelSaving(test.TestCase):
       os.close(fd)
       os.remove(fname)
 
-
   def test_saving_constant_initializer_with_numpy(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 26866d4714..6e2f76cd80 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -212,20 +212,16 @@ class Sequential(Model):
       self._init_graph_network(self.inputs, self.outputs, name=self.name)
       self.built = True
 
+  @base_layer.default
   def build(self, input_shape=None):
     if self._is_graph_network:
       self._init_graph_network(self.inputs, self.outputs, name=self.name)
     else:
       if input_shape is None:
         raise ValueError('You must provide an `input_shape` argument.')
+      input_shape = tuple(input_shape)
       self._build_input_shape = input_shape
-      shape = input_shape
-      for layer in self.layers:
-        if not layer.built:
-          with ops.name_scope(layer._name_scope()):
-            layer.build(shape)
-          layer.built = True
-        shape = layer.compute_output_shape(shape)
+      super(Sequential, self).build(input_shape)
     self.built = True
 
   def call(self, inputs, training=None, mask=None):
@@ -237,8 +233,8 @@ class Sequential(Model):
     return outputs
 
   def _call_and_compute_mask(self, inputs, training=None, mask=None):
-    if not self.built:
-      self.build(inputs.shape)
+    if not self.built and self._is_graph_network:
+      self._init_graph_network(self.inputs, self.outputs, name=self.name)
 
     x = inputs
     for layer in self.layers:
@@ -251,6 +247,11 @@ class Sequential(Model):
       if isinstance(layer, Network) and layer._compute_output_and_mask_jointly:
         x, mask = layer._call_and_compute_mask(x, **kwargs)
       else:
+        if not layer.built:
+          # Build layer if applicable.
+          with ops.name_scope(layer._name_scope()):
+            layer._maybe_build(x)
+          layer.built = True
         x = layer.call(x, **kwargs)
         if layer.supports_masking:
           mask = layer.compute_mask(x, mask)
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 54db0f3566..2e2927bf47 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -294,7 +294,6 @@ class TestSequential(test.TestCase, parameterized.TestCase):
 
     model.build((None, 10))
     self.assertTrue(model.built)
-    self.assertTrue(model.layers[-1].built)
     self.assertEqual(len(model.weights), 8)
 
   @tf_test_util.run_in_graph_and_eager_modes
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 4d3fffb25a..1094e549d7 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1455,8 +1455,7 @@ class Model(Network):
 
     if self.__class__.__name__ == 'Sequential' and not self.built:
       if tensor_util.is_tensor(inputs):
-        input_shape = (None,) + tuple(inputs.get_shape().as_list()[1:])
-        self.build(input_shape=input_shape)
+        input_shape = (None,) + tuple(inputs.shape.as_list()[1:])
       elif isinstance(inputs, dict):
         # We assert that the first layer is a FeatureLayer.
         if not training_utils.is_feature_layer(self.layers[0]):
@@ -1464,10 +1463,9 @@ class Model(Network):
                            'which doesn\'t have FeatureLayer as the first layer'
                            ' is an error.')
         input_shape = (None,)
-        self.build(input_shape=input_shape)
       else:
-        input_shape = (None,) + inputs.shape[1:]
-        self.build(input_shape=input_shape)
+        input_shape = (None,) + tuple(inputs.shape[1:])
+      self._build_input_shape = input_shape
 
     # On-the-fly setting of symbolic model inputs (either by using the tensor
     # provided, or by creating a placeholder if Numpy data was provided).
@@ -1486,6 +1484,8 @@ class Model(Network):
         self._feed_input_names.append(k)
         self._feed_input_shapes.append(K.int_shape(v))
 
+    # TODO(fchollet): consider calling `_maybe_build` before calling the model.
+
     if outputs is None:
       # Obtain symbolic outputs by calling the model.
       with K.get_graph().as_default():
diff --git a/tensorflow/python/keras/layers/local_test.py b/tensorflow/python/keras/layers/local_test.py
index 2397a607da..1c20d43706 100644
--- a/tensorflow/python/keras/layers/local_test.py
+++ b/tensorflow/python/keras/layers/local_test.py
@@ -259,13 +259,16 @@ class LocallyConnectedImplementationModeTest(test.TestCase):
                             'kernel_size': kernel_x + kernel_y,
                             'strides': stride_x + stride_y,
                             'data_format': data_format,
-                            'num_classes': num_classes,
-                            'input_shape': inputs.shape
+                            'num_classes': num_classes
                         }
-
                         model_1 = get_model(implementation=1, **kwargs)
                         model_2 = get_model(implementation=2, **kwargs)
 
+                        # Build models.
+                        model_1.train_on_batch(inputs, targets)
+                        model_2.train_on_batch(inputs, targets)
+
+                        # Copy weights.
                         copy_model_weights(model_2, model_1)
 
                         # Compare outputs at initialization.
@@ -279,7 +282,6 @@ class LocallyConnectedImplementationModeTest(test.TestCase):
                                     y=targets,
                                     epochs=num_epochs,
                                     batch_size=num_samples)
-
                         model_2.fit(x=inputs,
                                     y=targets,
                                     epochs=num_epochs,
@@ -289,7 +291,7 @@ class LocallyConnectedImplementationModeTest(test.TestCase):
                         out_1 = model_1.call(inputs)
                         out_2 = model_2.call(inputs)
                         self.assertAllCloseAccordingToType(out_1, out_2,
-                                                           rtol=1e-5, atol=1e-5)
+                                                           atol=1e-4)
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_make_2d(self):
@@ -366,8 +368,7 @@ def get_model(implementation,
               strides,
               layers,
               num_classes,
-              data_format,
-              input_shape):
+              data_format):
   model = keras.Sequential()
 
   if len(kernel_size) == 1:
@@ -396,7 +397,6 @@ def get_model(implementation,
       metrics=[keras.metrics.categorical_accuracy],
       loss=xent
   )
-  model.build(input_shape)
   return model
 
 
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index 87802d8df0..ccfff57a3b 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -726,10 +726,41 @@ class ModelSubclassingTest(test.TestCase):
     _ = model.evaluate(x, y, verbose=0)
 
     self.assertEqual(len(model.weights), 16)
-    self.assertEqual(
-        len(model.non_trainable_weights), 4)
+    self.assertEqual(len(model.non_trainable_weights), 4)
     self.assertEqual(len(model.trainable_weights), 12)
 
+  def test_subclass_nested_in_sequential(self):
+    num_classes = 2
+    num_samples = 100
+    input_dim = 50
+
+    class Inner(keras.Model):
+
+      def __init__(self):
+        super(Inner, self).__init__()
+        self.dense1 = keras.layers.Dense(32, activation='relu')
+        self.dense2 = keras.layers.Dense(num_classes, activation='relu')
+        self.bn = keras.layers.BatchNormalization()
+
+      def call(self, inputs):
+        x = self.dense1(inputs)
+        x = self.dense2(x)
+        return self.bn(x)
+
+    model = keras.Sequential([Inner()])
+    model.compile(loss='mse',
+                  optimizer=RMSPropOptimizer(learning_rate=0.001),
+                  metrics=['acc'])
+
+    x = np.ones((num_samples, input_dim))
+    y = np.zeros((num_samples, num_classes))
+    model.fit(x, y, epochs=2, batch_size=32, verbose=0)
+    _ = model.evaluate(x, y, verbose=0)
+
+    self.assertEqual(len(model.weights), 8)
+    self.assertEqual(len(model.non_trainable_weights), 2)
+    self.assertEqual(len(model.trainable_weights), 6)
+
   def test_support_for_manual_training_arg(self):
     # In most cases, the `training` argument is left unspecified, in which
     # case it defaults to value corresponding to the Model method being used
@@ -1038,6 +1069,16 @@ class TrainingNoDefaultModel(keras.Model):
     return self.dense1(x)
 
 
+class TrainingMaskingModel(keras.Model):
+
+  def __init__(self):
+    super(TrainingMaskingModel, self).__init__()
+    self.dense1 = keras.layers.Dense(1)
+
+  def call(self, x, training=False, mask=None):
+    return self.dense1(x)
+
+
 class CustomCallSignatureTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
@@ -1067,6 +1108,19 @@ class CustomCallSignatureTests(test.TestCase):
                                     'has been properly built.'))
     self.assertTrue(model.built, 'Model should be built after calling `build`.')
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_training_and_mask_args_call_build(self):
+    input_dim = 2
+
+    model = TrainingMaskingModel()
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build((None, input_dim))
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+
   @test_util.run_in_graph_and_eager_modes
   def test_custom_call_kwargs_and_build(self):
     first_input_shape = (2, 3)
-- 
GitLab


From b0c5cbd7bcec9f44df14bd0738d3bb1a434d9fa8 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 28 Nov 2018 12:18:18 -0800
Subject: [PATCH 0948/1554] Add doc generator script for TF2.

This uses the stand-alone generator, in "tensroflow/docs", which can be pip-installed. This will make it possible to build the python api-docs docs from the tensorflow-pip installation without building tensorflow from scratch.

PiperOrigin-RevId: 223213515
---
 tensorflow/tools/docs/BUILD             | 25 +++++--
 tensorflow/tools/docs/generate2.py      | 77 +++++++++++++++++++++
 tensorflow/tools/docs/generate2_test.py | 39 +++++++++++
 tensorflow/tools/docs/generate_1_0.py   | 92 -------------------------
 4 files changed, 135 insertions(+), 98 deletions(-)
 create mode 100644 tensorflow/tools/docs/generate2.py
 create mode 100644 tensorflow/tools/docs/generate2_test.py
 delete mode 100644 tensorflow/tools/docs/generate_1_0.py

diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD
index 1a53f24177..b072853a4e 100644
--- a/tensorflow/tools/docs/BUILD
+++ b/tensorflow/tools/docs/BUILD
@@ -142,17 +142,30 @@ py_test(
     ],
 )
 
-py_binary(
-    name = "generate_1_0",
-    srcs = ["generate_1_0.py"],
+py_test(
+    name = "generate2_test",
+    srcs = ["generate2_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",
+        # No reason to run sanitizers or fastbuild for this test.
+        "noasan",
+        "nomsan",
+        "notsan",
+        "optonly",
+    ],
     deps = [
-        ":generate_lib",
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/python/debug:debug_py",
+        ":generate2",
     ],
 )
 
+py_binary(
+    name = "generate2",
+    srcs = ["generate2.py"],
+    srcs_version = "PY2AND3",
+    deps = ["//tensorflow:tensorflow_py"],
+)
+
 py_library(
     name = "py_guide_parser",
     srcs = ["py_guide_parser.py"],
diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py
new file mode 100644
index 0000000000..3dfd0dca33
--- /dev/null
+++ b/tensorflow/tools/docs/generate2.py
@@ -0,0 +1,77 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""A tool to generate api_docs for TensorFlow2.
+
+```
+python generate2.py --output_dir=/tmp/out
+```
+
+Requires a local installation of:
+  https://github.com/tensorflow/docs/tree/master/tools
+  tf-nightly-2.0-preview
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from os import path
+
+from absl import app
+from absl import flags
+
+import tensorflow as tf
+
+from tensorflow_docs.api_generator import generate_lib
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+    "code_url_prefix",
+    "/code/stable/tensorflow/",
+    "A url to prepend to code paths when creating links to defining code")
+
+flags.DEFINE_string(
+    "output_dir", "/tmp/out",
+    "A directory, where the docs will be output to.")
+
+
+def build_docs(output_dir, code_url_prefix):
+  """Build api docs for tensorflow v2.
+
+  Args:
+    output_dir: A string path, where to put the files.
+    code_url_prefix: prefix for "Defined in" links.
+  """
+  base_dir = path.dirname(tf.__file__)
+  doc_generator = generate_lib.DocGenerator(
+      root_title="TensorFlow 2.0 Preview",
+      py_modules=[("tf", tf)],
+      base_dir=base_dir,
+      search_hints=True,
+      code_url_prefix=code_url_prefix,
+      site_path="api_docs/")
+
+  doc_generator.build(output_dir)
+
+
+def main(argv):
+  del argv
+  build_docs(output_dir=FLAGS.output_dir,
+             code_url_prefix=FLAGS.code_url_prefix)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tensorflow/tools/docs/generate2_test.py b/tensorflow/tools/docs/generate2_test.py
new file mode 100644
index 0000000000..774d45c536
--- /dev/null
+++ b/tensorflow/tools/docs/generate2_test.py
@@ -0,0 +1,39 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.tools.docs.generate2."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import shutil
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.docs import generate2
+
+
+class Generate2Test(googletest.TestCase):
+
+  def test_end_to_end(self):
+    output_dir = os.path.join(googletest.GetTempDir(), 'output')
+    if os.path.exists(output_dir):
+      shutil.rmtree(output_dir)
+    os.makedirs(output_dir)
+    generate2.build_docs(output_dir=output_dir, code_url_prefix='')
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/tools/docs/generate_1_0.py b/tensorflow/tools/docs/generate_1_0.py
deleted file mode 100644
index f4384e0ced..0000000000
--- a/tensorflow/tools/docs/generate_1_0.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Generate docs for the TensorFlow Python API."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import sys
-
-import tensorflow as tf
-
-from tensorflow.python import debug as tf_debug
-from tensorflow.python.util import tf_inspect
-from tensorflow.tools.docs import generate_lib
-
-if __name__ == '__main__':
-  doc_generator = generate_lib.DocGenerator()
-  doc_generator.add_output_dir_argument()
-  doc_generator.add_src_dir_argument()
-
-  # This doc generator works on the TensorFlow codebase. Since this script lives
-  # at tensorflow/tools/docs, and all code is defined somewhere inside
-  # tensorflow/, we can compute the base directory (two levels up), which is
-  # valid unless we're trying to apply this to a different code base, or are
-  # moving the script around.
-  script_dir = os.path.dirname(tf_inspect.getfile(tf_inspect.currentframe()))
-  default_base_dir = os.path.join(script_dir, '..', '..')
-  doc_generator.add_base_dir_argument(default_base_dir)
-
-  flags = doc_generator.parse_known_args()
-
-  # tf_debug is not imported with tf, it's a separate module altogether
-  doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)])
-
-  doc_generator.set_do_not_descend_map({
-      'tf': ['cli', 'lib', 'wrappers'],
-      'tf.contrib': [
-          'compiler',
-          'factorization',
-          'grid_rnn',
-          'labeled_tensor',
-          'quantization',
-          'session_bundle',
-          'slim',
-          'solvers',
-          'specs',
-          'tensor_forest',
-          'tensorboard',
-          'testing',
-          'training',
-          'tfprof',
-      ],
-      'tf.contrib.bayesflow': [
-          'entropy', 'monte_carlo', 'special_math',
-          'stochastic_gradient_estimators', 'stochastic_graph',
-          'stochastic_tensor', 'stochastic_variables', 'variational_inference'
-      ],
-      'tf.contrib.distributions': ['bijector'],
-      'tf.contrib.ffmpeg': ['ffmpeg_ops'],
-      'tf.contrib.graph_editor': [
-          'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util'
-      ],
-      'tf.contrib.layers': ['feature_column', 'summaries'],
-      'tf.contrib.learn': [
-          'datasets',
-          'head',
-          'graph_actions',
-          'io',
-          'models',
-          'monitors',
-          'ops',
-          'preprocessing',
-          'utils',
-      ],
-      'tf.contrib.util': ['loader'],
-  })
-
-  sys.exit(doc_generator.build(flags))
-- 
GitLab


From 8bfd68f96fdafbe378f36cab827d950ec198d2b4 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 28 Nov 2018 12:22:45 -0800
Subject: [PATCH 0949/1554] Change API for string_to_hash_* API for TF 2.0.

PiperOrigin-RevId: 223214232
---
 .../api_def_StringToHashBucket.pbtxt          |  8 +--
 .../python_api/api_def_StringToNumber.pbtxt   |  8 +--
 .../ops/ragged/ragged_elementwise_ops.py      |  2 +-
 tensorflow/python/ops/string_ops.py           | 51 +++++++++++++++++++
 .../api/golden/v2/tensorflow.strings.pbtxt    |  4 +-
 tensorflow/tools/compatibility/renames_v2.py  |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 18 +++++++
 7 files changed, 76 insertions(+), 19 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_StringToHashBucket.pbtxt b/tensorflow/core/api_def/python_api/api_def_StringToHashBucket.pbtxt
index cf0b8831ef..dc4493c841 100644
--- a/tensorflow/core/api_def/python_api/api_def_StringToHashBucket.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_StringToHashBucket.pbtxt
@@ -1,10 +1,4 @@
 op {
   graph_op_name: "StringToHashBucket"
-  endpoint {
-    name: "strings.to_hash_bucket"
-  }
-  endpoint {
-    name: "string_to_hash_bucket"
-    deprecation_version: 2
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_StringToNumber.pbtxt b/tensorflow/core/api_def/python_api/api_def_StringToNumber.pbtxt
index 155dd26750..9c89d02fb7 100644
--- a/tensorflow/core/api_def/python_api/api_def_StringToNumber.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_StringToNumber.pbtxt
@@ -1,10 +1,4 @@
 op {
   graph_op_name: "StringToNumber"
-  endpoint {
-    name: "strings.to_number"
-  }
-  endpoint {
-    name: "string_to_number"
-    deprecation_version: 2
-  }
+  visibility: HIDDEN
 }
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
index 23d0e8b5fc..edf721b5a7 100644
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
@@ -348,7 +348,7 @@ _TF_ELEMENTWISE_OPS = [
     (string_ops.regex_replace, 'input'),
     (string_ops.string_join, '[inputs]'),
     (string_ops.string_strip, 'input'),
-    (string_ops.string_to_hash_bucket, 'string_tensor'),
+    (string_ops.string_to_hash_bucket, 'input'),
     (string_ops.string_to_hash_bucket_fast, 'input'),
     (string_ops.string_to_hash_bucket_strong, 'input'),
     (string_ops.substr, 'input'),
diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py
index b2090da631..b6b329c486 100644
--- a/tensorflow/python/ops/string_ops.py
+++ b/tensorflow/python/ops/string_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_parsing_ops
 from tensorflow.python.ops import gen_string_ops
 from tensorflow.python.ops import math_ops
 
@@ -391,3 +392,53 @@ ops.NotDifferentiable("StringSplit")
 ops.NotDifferentiable("AsString")
 ops.NotDifferentiable("EncodeBase64")
 ops.NotDifferentiable("DecodeBase64")
+
+
+@tf_export("strings.to_number", v1=[])
+def string_to_number(input, out_type=dtypes.float32, name=None):
+  r"""Converts each string in the input Tensor to the specified numeric type.
+
+  (Note that int32 overflow results in an error while float overflow
+  results in a rounded value.)
+
+  Args:
+    input: A `Tensor` of type `string`.
+    out_type: An optional `tf.DType` from: `tf.float32, tf.float64, tf.int32,
+      tf.int64`. Defaults to `tf.float32`.
+      The numeric type to interpret each string in `string_tensor` as.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `out_type`.
+  """
+  return gen_parsing_ops.string_to_number(input, out_type, name)
+tf_export(v1=["strings.to_number", "string_to_number"])(
+    gen_parsing_ops.string_to_number
+    )
+
+
+@tf_export("strings.to_hash_bucket", v1=[])
+def string_to_hash_bucket(input, num_buckets, name=None):
+  # pylint: disable=line-too-long
+  r"""Converts each string in the input Tensor to its hash mod by a number of buckets.
+
+  The hash function is deterministic on the content of the string within the
+  process.
+
+  Note that the hash function may change from time to time.
+  This functionality will be deprecated and it's recommended to use
+  `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
+
+  Args:
+    input: A `Tensor` of type `string`.
+    num_buckets: An `int` that is `>= 1`. The number of buckets.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `int64`.
+  """
+  # pylint: enable=line-too-long
+  return gen_string_ops.string_to_hash_bucket(input, num_buckets, name)
+tf_export(v1=["strings.to_hash_bucket", "string_to_hash_bucket"])(
+    gen_string_ops.string_to_hash_bucket
+    )
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
index 2ccb3a4594..f2f4879fe8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
@@ -38,7 +38,7 @@ tf_module {
   }
   member_method {
     name: "to_hash_bucket"
-    argspec: "args=[\'string_tensor\', \'num_buckets\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'num_buckets\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "to_hash_bucket_fast"
@@ -50,7 +50,7 @@ tf_module {
   }
   member_method {
     name: "to_number"
-    argspec: "args=[\'string_tensor\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\'], "
+    argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\'], "
   }
   member_method {
     name: "unicode_script"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 042ca8adaa..319b0ba7c9 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -552,10 +552,10 @@ renames = {
     'tf.squared_difference': 'tf.math.squared_difference',
     'tf.string_join': 'tf.strings.join',
     'tf.string_strip': 'tf.strings.strip',
-    'tf.string_to_hash_bucket': 'tf.strings.to_hash_bucket',
+    'tf.string_to_hash_bucket': 'tf.compat.v1.string_to_hash_bucket',
     'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
     'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
-    'tf.string_to_number': 'tf.strings.to_number',
+    'tf.string_to_number': 'tf.compat.v1.string_to_number',
     'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 6e8d6e2d0b..af57211cad 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -183,6 +183,18 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.stateless_multinomial": {
             "output_dtype": "dtype",
         },
+        "tf.string_to_number": {
+            "string_tensor": "input",
+        },
+        "tf.strings.to_number": {
+            "string_tensor": "input",
+        },
+        "tf.string_to_hash_bucket": {
+            "string_tensor": "input",
+        },
+        "tf.strings.to_hash_bucket": {
+            "string_tensor": "input",
+        },
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
@@ -393,6 +405,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.sparse.concat",
         "tf.sparse_split":
             "tf.sparse.split",
+        "tf.string_to_hash_bucket":
+            "tf.strings.to_hash_bucket",
+        "tf.string_to_number":
+            "tf.strings.to_number",
         "tf.multinomial":
             "tf.random.categorical",
         "tf.random.multinomial":
@@ -407,6 +423,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.math.confusion_matrix",
         "tf.train.confusion_matrix":
             "tf.math.confusion_matrix",
+        "tf.decode_csv":
+            "tf.io.decode_csv",
     })
     # pylint: enable=line-too-long
 
-- 
GitLab


From 95f218770fe2dfd577625d803756fb9c9459bd6c Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Wed, 28 Nov 2018 12:27:46 -0800
Subject: [PATCH 0950/1554] Fix tflite makefile for new C glob path.

Fixes #23926
Thanks @pglushkov for looking into this fix.

PiperOrigin-RevId: 223214968
---
 tensorflow/lite/tools/make/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 28374f13b7..06a1f36105 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -85,6 +85,7 @@ CORE_CC_ALL_SRCS := \
 $(wildcard tensorflow/lite/*.cc) \
 $(wildcard tensorflow/lite/*.c) \
 $(wildcard tensorflow/lite/c/*.c) \
+$(wildcard tensorflow/lite/core/*.cc) \
 $(wildcard tensorflow/lite/core/api/*.cc)
 ifneq ($(BUILD_TYPE),micro)
 CORE_CC_ALL_SRCS += \
-- 
GitLab


From 9b9fb0c6cf857ec8924373150f36f5f8f0ad5806 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Wed, 28 Nov 2018 12:44:34 -0800
Subject: [PATCH 0951/1554] Add a simple guard for the case when the requested
 output port number is out of range of the num_outputs of the input node.

PiperOrigin-RevId: 223217630
---
 tensorflow/core/grappler/costs/graph_properties.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 270b75269c..1df26d94d1 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -679,6 +679,11 @@ class SymbolicShapeRefiner {
               "' was not previously added to SymbolicShapeRefiner.");
         }
 
+        if (src_output >= c->inference_context->num_outputs())
+          return errors::OutOfRange("src_output = ", src_output,
+                                    ", but num_outputs is only ",
+                                    c->inference_context->num_outputs());
+
         // Propagate input node's NodeContext info to the current node's
         // NodeContext:
         // output_tensor_protos to input_tensor_protos and input_tensors, and
-- 
GitLab


From 065c8648d694faa549c5b7bdf9787f4077fe0328 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 28 Nov 2018 12:50:08 -0800
Subject: [PATCH 0952/1554] Switch sigmoid to sin for tests which used an
 unsupported op (previously sigmoid) to break graph into multiple segments

---
 .../test/binary_tensor_weight_broadcast_test.py      |  8 ++++++--
 .../test/multi_connection_neighbor_engine_test.py    | 12 +++++++++---
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index b53cb3c091..ed56226729 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -56,10 +56,14 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
       ]:
         a = self._ConstOp(weights_shape)
         f = x + a
-        x = math_ops.sigmoid(f)
+        # sin is used to break the graph into separate segments since it is an
+        # unsupported op.
+        x = math_ops.sin(f)
         a = self._ConstOp(weights_shape)
         f = a + x
-        x = math_ops.sigmoid(f)
+        # sin is used to break the graph into separate segments since it is an
+        # unsupported op.
+        x = math_ops.sin(f)
       gen_array_ops.reshape(x, [5, -1], name=output_name)
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 11be4feaf7..1bb4a02a49 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -60,14 +60,20 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
       b = constant_op.constant(
           np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       q = conv - b
-      edge = math_ops.sigmoid(q)
+      # sin is used to break the graph into separate segments since it is an
+      # unsupported op.
+      edge = math_ops.sin(q)
 
       b = constant_op.constant(
           np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       d = b + conv
-      edge3 = math_ops.sigmoid(d)
+      # sin is used to break the graph into separate segments since it is an
+      # unsupported op.
+      edge3 = math_ops.sin(d)
 
-      edge1 = gen_math_ops.tan(conv)
+      # sin is used to break the graph into separate segments since it is an
+      # unsupported op.
+      edge1 = math_ops.sin(conv)
       t = t - edge1
       q = q + edge
       t = t + q
-- 
GitLab


From 49feaad5524dd259cb86637ec44228420440f8be Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 28 Nov 2018 13:03:18 -0800
Subject: [PATCH 0953/1554] Switch to self.trt_incompatible_op

---
 .../test/binary_tensor_weight_broadcast_test.py      |  8 ++------
 .../test/multi_connection_neighbor_engine_test.py    | 12 +++---------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index ed56226729..5ab94cd931 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -56,14 +56,10 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
       ]:
         a = self._ConstOp(weights_shape)
         f = x + a
-        # sin is used to break the graph into separate segments since it is an
-        # unsupported op.
-        x = math_ops.sin(f)
+        x = self.trt_incompatible_op(f)
         a = self._ConstOp(weights_shape)
         f = a + x
-        # sin is used to break the graph into separate segments since it is an
-        # unsupported op.
-        x = math_ops.sin(f)
+        x = self.trt_incompatible_op(f)
       gen_array_ops.reshape(x, [5, -1], name=output_name)
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 1bb4a02a49..468b2c96e1 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -60,20 +60,14 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
       b = constant_op.constant(
           np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       q = conv - b
-      # sin is used to break the graph into separate segments since it is an
-      # unsupported op.
-      edge = math_ops.sin(q)
+      edge = self.trt_incompatible_op(q)
 
       b = constant_op.constant(
           np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       d = b + conv
-      # sin is used to break the graph into separate segments since it is an
-      # unsupported op.
-      edge3 = math_ops.sin(d)
+      edge3 = self.trt_incompatible_op(d)
 
-      # sin is used to break the graph into separate segments since it is an
-      # unsupported op.
-      edge1 = math_ops.sin(conv)
+      edge1 = self.trt_incompatible_op(conv)
       t = t - edge1
       q = q + edge
       t = t + q
-- 
GitLab


From f1f921f4c74e5d18d77476c873c600836443f137 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 28 Nov 2018 13:36:14 -0800
Subject: [PATCH 0954/1554] Function optimizer documentation and tensor
 mapping.

PiperOrigin-RevId: 223226824
---
 .../grappler/optimizers/function_optimizer.cc | 216 ++++++++++--------
 1 file changed, 127 insertions(+), 89 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 22013ea2db..9faab6614c 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_def.pb.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
@@ -109,6 +110,28 @@ AttrSlice FunctionInstantiationAttributes(const FunctionDef& func,
   }
 }
 
+class FakeCPUDevice : public Device {
+ public:
+  FakeCPUDevice(Env* env, const DeviceAttributes& attr) : Device(env, attr) {}
+  Status Sync() override { return Status::OK(); }
+};
+
+// -------------------------------------------------------------------------- //
+// Function specialization.
+//
+// FunctionDef is somewhat similar to function template in C++, given all the
+// type parameters (and attribute values) it generates a statically defined
+// graph from the type parametrized "graph template" (function body).
+//
+// Function specialization instantiates a parametrized FunctionDef into a
+// statically defined graph, and then converts it back to the fully defined
+// FunctionDef (it doesn't have any unknown type parameters or attribute
+// values, known as placeholders).
+//
+// Given the fully specified graph we can apply all the Grappler optimizers to
+// it (see details in MetaOptimizer). Also we can push known constant inputs
+// into the function body, and remove unused outputs/inputs.
+
 // Specialized function instantiation type parameters, body parameters, and
 // const inputs.
 struct FunctionSpecializationSignature {
@@ -208,12 +231,6 @@ struct FunctionSpecialization {
   std::vector<std::pair<int, int>> output_mapping;
 };
 
-class FakeCPUDevice : public Device {
- public:
-  FakeCPUDevice(Env* env, const DeviceAttributes& attr) : Device(env, attr) {}
-  Status Sync() override { return Status::OK(); }
-};
-
 class FunctionOptimizerContext {
  public:
   explicit FunctionOptimizerContext(RewriterConfig::Toggle opt_level,
@@ -242,9 +259,9 @@ class FunctionOptimizerContext {
     return flr_;
   }
 
-  const gtl::FlatMap<string, std::vector<std::pair<int, int>>>&
-  output_mappings() const {
-    return output_mappings_;
+  const gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>&
+  tensor_mapping() const {
+    return tensor_mapping_;
   }
 
   const GraphView& graph_view() const { return graph_view_; }
@@ -275,20 +292,18 @@ class FunctionOptimizerContext {
     specialized_functions_.emplace(sig, specialized_func);
   }
 
-  void AddOutputMapping(const string& func_node,
+  void AddTensorMapping(const string& func_node,
                         const FunctionSpecialization& specialized_func) {
-    output_mappings_.emplace(func_node, specialized_func.output_mapping);
-  }
-
-  // Return true if we had any specialized function that changed it's output
-  // mapping, and it's required to update output consumers to new ports ids.
-  bool RequiresOutputMapping() const {
-    for (const auto& m1 : output_mappings_) {
-      for (const std::pair<int, int>& m2 : m1.second) {
-        if (m2.first != m2.second) return true;
+    for (const auto& pair : specialized_func.output_mapping) {
+      int from_idx = pair.first;
+      int to_idx = pair.second;
+      if (from_idx != to_idx) {
+        SafeTensorId from_tensor(func_node, from_idx);
+        SafeTensorId to_tensor(func_node, to_idx);
+        auto inserted = tensor_mapping_.insert({from_tensor, to_tensor});
+        DCHECK(inserted.second);
       }
     }
-    return false;
   }
 
  private:
@@ -352,9 +367,15 @@ class FunctionOptimizerContext {
   gtl::FlatSet<string> fetch_tensors_;  // format: node_name:port
   gtl::FlatSet<string> fetch_nodes_;    // format: node_name
 
-  // Output mappings that have to be applied to the graph after all functions
-  // are specialized (node name -> output mappings).
-  gtl::FlatMap<string, std::vector<std::pair<int, int>>> output_mappings_;
+  // After function inlining and specialization, the optimized graph might be in
+  // invalid state, nodes can read from non-existing function call nodes that
+  // were inlined, or they can read from output index that is no longer valid
+  // after unused outputs pruning.
+  //
+  // Tensor mapping that has to be applied to the graph after all functions
+  // optimizations (invalidated tensor id -> optimized graph tensor id).
+  gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>
+      tensor_mapping_;
 
   // Use graph view to find active outputs of the function caller nodes.
   GraphView graph_view_;
@@ -378,49 +399,6 @@ const FunctionDef* FindFunctionCall(const FunctionOptimizerContext& ctx,
   return ctx.function_library().Find(node.op());
 }
 
-// Returns true iff `node` is a direct function call of `func`, and we know how
-// to inline it into the main graph.
-bool IsInlinableDirectFunctionCall(const FunctionOptimizerContext& ctx,
-                                   const FunctionDef& func,
-                                   const NodeDef& node) {
-  // Indirect function calls (PartitionedCallOp) have automatic control
-  // dependencies and inlined separately from direct function calls.
-  bool is_direct_function_call = IsDirectFunctionCall(func, node);
-
-  // For direct function  calls we insert IdentityN nodes before/after inlined
-  // function body to preserve function call semantics (all inputs evaluated
-  // before function evaluation starts, and all function body nodes finished
-  // before output consumed by other nodes).
-  bool has_inputs = func.signature().input_arg_size() > 0;
-  // TODO(ezhulenev): Relax constraint on output args?
-  bool has_outputs = func.signature().output_arg_size() > 0;
-
-  // Function must execute all the nodes in a function body that might have side
-  // effects. After inlining these nodes into the main graph, we can no longer
-  // guarantee that. For now we disable inlining functions with side effects.
-  //
-  // Attaching control dependency to the output IdentityN node is not safe,
-  // because it might be split or pruned in a later optimization pass.
-  //
-  // Indirect function calls (via PartitionedCallOp) have automatic dependency
-  // tracking, and allow us to safely inline functions with side effects.
-  bool free_of_side_effects =
-      std::all_of(func.node_def().begin(), func.node_def().end(),
-                  [&ctx](const NodeDef& node) {
-                    return IsFreeOfSideEffect(node, &ctx.function_library());
-                  });
-
-  bool marked_noinline = MarkedNoInline(func);
-  bool marked_specialized = MarkedSpecialized(func);
-
-  // We ignore `_noinline` marker in aggressive mode.
-  bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
-
-  return is_direct_function_call && has_inputs && has_outputs &&
-         free_of_side_effects && !marked_specialized &&
-         (!marked_noinline || aggressive);
-}
-
 gtl::FlatSet<int> GetActiveOutputs(const NodeDef& node,
                                    const FunctionOptimizerContext& ctx,
                                    int size_hint = 0) {
@@ -730,7 +708,7 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
     TF_RETURN_IF_ERROR(UpdateSpecializedFunctionNode(
         func, func_node, *already_specialized, specialized_func_node));
 
-    ctx->AddOutputMapping(specialized_func_node->name(), *already_specialized);
+    ctx->AddTensorMapping(specialized_func_node->name(), *already_specialized);
 
     return Status::OK();
   }
@@ -792,11 +770,78 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
       func, func_node, func_specialization, specialized_func_node));
 
   ctx->AddSpecializedFunction(signature, func_specialization);
-  ctx->AddOutputMapping(specialized_func_node->name(), func_specialization);
+  ctx->AddTensorMapping(specialized_func_node->name(), func_specialization);
 
   return Status::OK();
 }
 
+// -------------------------------------------------------------------------- //
+// Inline direct functions calls.
+//
+// When we inline direct function calls, we instantiate the function body from
+// its FunctionDef and caller node attributes, and embed the instantiated graph
+// into the "main graph". When we do that, we must preserve the function call
+// semantics:
+//
+// 1) All input nodes must be executed before any of function body nodes will
+//    start executing.
+// 2) All function body nodes must be executed before any of the nodes, reading
+//    outputs of the function will start executing.
+// 3) All nodes with side effects inside a function must be executed, this is
+//    different from the nodes with side effects in the main graph, that can be
+//    pruned if they are not in transitive dependency set of any of the fetch
+//    nodes.
+// 4) All nodes of the function body must be execute on the device specified by
+//    the function caller node.
+//
+// To guarantee that function call semantics are preserved after inlining, we
+// insert an IdentityN node before the inlined function body, and hook all
+// inputs into that, and we insert another IdentityN node to hook all function
+// outputs to it.
+
+// Returns true iff `node` is a direct function call of `func`, and we know how
+// to inline it into the main graph.
+bool IsInlinableDirectFunctionCall(const FunctionOptimizerContext& ctx,
+                                   const FunctionDef& func,
+                                   const NodeDef& node) {
+  // Indirect function calls (PartitionedCallOp) have automatic control
+  // dependencies and inlined separately from direct function calls.
+  bool is_direct_function_call = IsDirectFunctionCall(func, node);
+
+  // For direct function  calls we insert IdentityN nodes before/after inlined
+  // function body to preserve function call semantics (all inputs evaluated
+  // before function evaluation starts, and all function body nodes finished
+  // before output consumed by other nodes).
+  bool has_inputs = func.signature().input_arg_size() > 0;
+  // TODO(ezhulenev): Relax constraint on output args?
+  bool has_outputs = func.signature().output_arg_size() > 0;
+
+  // Function must execute all the nodes in a function body that might have side
+  // effects. After inlining these nodes into the main graph, we can no longer
+  // guarantee that. For now we disable inlining functions with side effects.
+  //
+  // Attaching control dependency to the output IdentityN node is not safe,
+  // because it might be split or pruned in a later optimization pass.
+  //
+  // Indirect function calls (via PartitionedCallOp) have automatic dependency
+  // tracking, and allow us to safely inline functions with side effects.
+  bool free_of_side_effects =
+      std::all_of(func.node_def().begin(), func.node_def().end(),
+                  [&ctx](const NodeDef& node) {
+                    return IsFreeOfSideEffect(node, &ctx.function_library());
+                  });
+
+  bool marked_noinline = MarkedNoInline(func);
+  bool marked_specialized = MarkedSpecialized(func);
+
+  // We ignore `_noinline` marker in aggressive mode.
+  bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
+
+  return is_direct_function_call && has_inputs && has_outputs &&
+         free_of_side_effects && !marked_specialized &&
+         (!marked_noinline || aggressive);
+}
+
 // Create an IdentityN node to hook the function inputs to: this ensures that
 // they're all evaluated before the evaluation of the function body starts.
 NodeDef InlinedFunctionInputsNode(const NodeDef& func_node,
@@ -1141,27 +1186,20 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 #undef TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED
   }
 
-  // Function specialization might change the number of function outputs, so we
-  // have to process the final optimized graph and update all the node mapping.
-  if (ctx.RequiresOutputMapping()) {
-    MutableGraphView optimized_graph_view(optimized_graph);
-    for (const auto& output_mapping : ctx.output_mappings()) {
-      const auto& node_name = output_mapping.first;
-      const auto& mappings = output_mapping.second;
-
-      for (const std::pair<int, int>& mapping : mappings) {
-        int from = mapping.first;
-        int to = mapping.second;
-
-        // Get the output port corresponding to the old output position.
-        MutableGraphView::OutputPort from_port =
-            optimized_graph_view.GetOutputPort(node_name, from);
-
-        // Update all input ports that read from old output port.
-        for (MutableGraphView::InputPort to_port :
-             optimized_graph_view.GetFanout(from_port)) {
-          *to_port.node->mutable_input(to_port.port_id) =
-              strings::StrCat(node_name, ":", to);
+  // After function specialization and inlining graph might be in invalid
+  // state, and some nodes can read tensors that do not exists anymore in the
+  // optimized graph: function call node was fully inlined into the graph, or
+  // output index was invalidated by the output pruning.
+
+  if (!ctx.tensor_mapping().empty()) {
+    for (NodeDef& node : *optimized_graph->mutable_node()) {
+      for (int idx = 0; idx < node.input_size(); ++idx) {
+        TensorId input_tensor = ParseTensorName(node.input(idx));
+        if (input_tensor.index() == Graph::kControlSlot) break;
+
+        auto mapping = ctx.tensor_mapping().find(input_tensor);
+        if (mapping != ctx.tensor_mapping().end()) {
+          node.set_input(idx, mapping->second.ToString());
         }
       }
     }
-- 
GitLab


From 3788915eab102422065e5c3ea1eabaafcd1a1f86 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 28 Nov 2018 13:40:02 -0800
Subject: [PATCH 0955/1554] [TF:XLA] Bump open source llvm revision to r347752

PiperOrigin-RevId: 223227528
---
 tensorflow/workspace.bzl                  | 8 ++++----
 third_party/llvm/llvm.autogenerated.BUILD | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index d9d40874a4..30694b43e3 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "7b4f705c532ee2aafb6e8b9013ad22ec8bb1823a153cd2d6ddb6b7faef818874",
-        strip_prefix = "llvm-9ad322c7dfd4385be9a515d734f70700f192ebae",
+        sha256 = "6c9b98f745b1cf2a9ef34f6220d0f94620ee5c828151924299913f1822cd40d1",
+        strip_prefix = "llvm-9d5ac66dc1ff01dee56354344c9da0879f1bdc36",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9ad322c7dfd4385be9a515d734f70700f192ebae.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/9ad322c7dfd4385be9a515d734f70700f192ebae.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9d5ac66dc1ff01dee56354344c9da0879f1bdc36.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/9d5ac66dc1ff01dee56354344c9da0879f1bdc36.tar.gz",
         ],
     )
 
diff --git a/third_party/llvm/llvm.autogenerated.BUILD b/third_party/llvm/llvm.autogenerated.BUILD
index 776935739a..eb468aa65f 100644
--- a/third_party/llvm/llvm.autogenerated.BUILD
+++ b/third_party/llvm/llvm.autogenerated.BUILD
@@ -823,6 +823,7 @@ cc_library(
     ]),
     copts = llvm_copts + ["-Iexternal/llvm/lib/Target/ARM"],
     deps = [
+        ":arm_asm_printer",
         ":arm_desc",
         ":arm_info",
         ":arm_utils",
@@ -2141,6 +2142,7 @@ cc_library(
         ":core",
         ":global_i_sel",
         ":mc",
+        ":profile_data",
         ":selection_dag",
         ":support",
         ":target",
-- 
GitLab


From 9316e38ef9cf53c88bc144f3ef5c9e4635427d66 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 28 Nov 2018 13:51:24 -0800
Subject: [PATCH 0956/1554] Fixing issues with the v2 script and the gfile API.

PiperOrigin-RevId: 223229634
---
 tensorflow/python/lib/io/file_io.py           |  2 +-
 .../api/golden/v1/tensorflow.io.gfile.pbtxt   |  2 +-
 .../api/golden/v2/tensorflow.io.gfile.pbtxt   |  2 +-
 tensorflow/tools/compatibility/renames_v2.py  | 15 ++++++++++--
 .../tools/compatibility/tf_upgrade_v2.py      | 24 +++++++++----------
 5 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py
index 645cf8e95c..4caa5750bf 100644
--- a/tensorflow/python/lib/io/file_io.py
+++ b/tensorflow/python/lib/io/file_io.py
@@ -509,7 +509,7 @@ def rename(oldname, newname, overwrite=False):
 
 
 @tf_export("io.gfile.rename")
-def rename_v2(src, dst, overwrite):
+def rename_v2(src, dst, overwrite=False):
   """Rename or move a file / directory.
 
   Args:
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
index e5aba7eff9..93d9b0fd75 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
@@ -34,7 +34,7 @@ tf_module {
   }
   member_method {
     name: "rename"
-    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "rmtree"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
index e5aba7eff9..93d9b0fd75 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
@@ -34,7 +34,7 @@ tf_module {
   }
   member_method {
     name: "rename"
-    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'src\', \'dst\', \'overwrite\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "rmtree"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 319b0ba7c9..70757556c1 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -94,13 +94,13 @@ renames = {
     'tf.assign_sub': 'tf.compat.v1.assign_sub',
     'tf.batch_to_space_nd': 'tf.compat.v1.batch_to_space_nd',
     'tf.betainc': 'tf.math.betainc',
-    'tf.bincount': 'tf.math.bincount',
+    'tf.bincount': 'tf.compat.v1.bincount',
     'tf.ceil': 'tf.math.ceil',
     'tf.check_numerics': 'tf.debugging.check_numerics',
     'tf.cholesky': 'tf.linalg.cholesky',
     'tf.cholesky_solve': 'tf.linalg.cholesky_solve',
     'tf.colocate_with': 'tf.compat.v1.colocate_with',
-    'tf.confusion_matrix': 'tf.math.confusion_matrix',
+    'tf.confusion_matrix': 'tf.compat.v1.confusion_matrix',
     'tf.conj': 'tf.math.conj',
     'tf.container': 'tf.compat.v1.container',
     'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices',
@@ -299,6 +299,17 @@ renames = {
     'tf.logging.warn': 'tf.compat.v1.logging.warn',
     'tf.logging.warning': 'tf.compat.v1.logging.warning',
     'tf.logical_xor': 'tf.math.logical_xor',
+    'tf.losses.absolute_difference': 'tf.compat.v1.losses.absolute_difference',
+    'tf.losses.compute_weighted_loss': 'tf.compat.v1.losses.compute_weighted_loss',
+    'tf.losses.cosine_distance': 'tf.compat.v1.losses.cosine_distance',
+    'tf.losses.hinge_loss': 'tf.compat.v1.losses.hinge_loss',
+    'tf.losses.huber_loss': 'tf.compat.v1.losses.huber_loss',
+    'tf.losses.log_loss': 'tf.compat.v1.losses.log_loss',
+    'tf.losses.mean_pairwise_squared_error': 'tf.compat.v1.losses.mean_pairwise_squared_error',
+    'tf.losses.mean_squared_error': 'tf.compat.v1.losses.mean_squared_error',
+    'tf.losses.sigmoid_cross_entropy': 'tf.compat.v1.losses.sigmoid_cross_entropy',
+    'tf.losses.softmax_cross_entropy': 'tf.compat.v1.losses.softmax_cross_entropy',
+    'tf.losses.sparse_softmax_cross_entropy': 'tf.compat.v1.losses.sparse_softmax_cross_entropy',
     'tf.make_template': 'tf.compat.v1.make_template',
     'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto',
     'tf.manip.batch_to_space_nd': 'tf.compat.v1.manip.batch_to_space_nd',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index af57211cad..b63e0b0a1b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -286,29 +286,29 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.batch_to_space_nd":
             "tf.batch_to_space",
         "tf.gfile.Copy":
-            "tf.io.gfile.Copy",
+            "tf.io.gfile.copy",
         "tf.gfile.DeleteRecursively":
-            "tf.io.gfile.DeleteRecursively",
+            "tf.io.gfile.rmtree",
         "tf.gfile.Exists":
-            "tf.io.gfile.Exists",
+            "tf.io.gfile.exists",
         "tf.gfile.Glob":
-            "tf.io.gfile.Glob",
+            "tf.io.gfile.glob",
         "tf.gfile.IsDirectory":
-            "tf.io.gfile.IsDirectory",
+            "tf.io.gfile.isdir",
         "tf.gfile.ListDirectory":
-            "tf.io.gfile.ListDirectory",
+            "tf.io.gfile.listdir",
         "tf.gfile.MakeDirs":
-            "tf.io.gfile.MakeDirs",
+            "tf.io.gfile.makedirs",
         "tf.gfile.MkDir":
-            "tf.io.gfile.MkDir",
+            "tf.io.gfile.mkdir",
         "tf.gfile.Remove":
-            "tf.io.gfile.Remove",
+            "tf.io.gfile.remove",
         "tf.gfile.Rename":
-            "tf.io.gfile.Rename",
+            "tf.io.gfile.rename",
         "tf.gfile.Stat":
-            "tf.io.gfile.Stat",
+            "tf.io.gfile.stat",
         "tf.gfile.Walk":
-            "tf.io.gfile.Walk",
+            "tf.io.gfile.walk",
         "tf.contrib.data.AUTOTUNE":
             "tf.data.experimental.AUTOTUNE",
         "tf.contrib.data.Counter":
-- 
GitLab


From 15b43008238cc3c9dc81d31ae3997b582ea190b1 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 28 Nov 2018 14:02:54 -0800
Subject: [PATCH 0957/1554] Add `reset_states` function to confusion matrix
 metrics.

PiperOrigin-RevId: 223231590
---
 tensorflow/python/keras/metrics.py      |  21 +++++
 tensorflow/python/keras/metrics_test.py | 116 ++++++++++++++++++++++++
 2 files changed, 137 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index b74b6ccf5e..0519493a0a 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -25,6 +25,7 @@ import sys
 import types
 import weakref
 from enum import Enum
+import numpy as np
 import six
 
 from tensorflow.python.eager import context
@@ -900,6 +901,11 @@ class _ConfusionMatrixConditionCount(Metric):
       result = self.accumulator[0]
     return ops.convert_to_tensor(result)
 
+  def reset_states(self):
+    num_thresholds = len(to_list(self.thresholds))
+    for v in self.variables:
+      K.set_value(v, np.zeros((num_thresholds,)))
+
 
 @tf_export('metrics.FalsePositives', 'keras.metrics.FalsePositives')
 class FalsePositives(_ConfusionMatrixConditionCount):
@@ -1180,6 +1186,11 @@ class Precision(Metric):
     result = math_ops.div_no_nan(self.tp, self.tp + self.fp)
     return result if isinstance(self.thresholds, (list, tuple)) else result[0]
 
+  def reset_states(self):
+    num_thresholds = len(to_list(self.thresholds))
+    for v in self.variables:
+      K.set_value(v, np.zeros((num_thresholds,)))
+
 
 @tf_export('metrics.Recall', 'keras.metrics.Recall')
 class Recall(Metric):
@@ -1260,6 +1271,11 @@ class Recall(Metric):
     result = math_ops.div_no_nan(self.tp, self.tp + self.fn)
     return result if isinstance(self.thresholds, (list, tuple)) else result[0]
 
+  def reset_states(self):
+    num_thresholds = len(to_list(self.thresholds))
+    for v in self.variables:
+      K.set_value(v, np.zeros((num_thresholds,)))
+
 
 @six.add_metaclass(abc.ABCMeta)
 class SensitivitySpecificityBase(Metric):
@@ -1319,6 +1335,11 @@ class SensitivitySpecificityBase(Metric):
         _ConfusionMatrix.FALSE_NEGATIVES: self.fn,
     }, y_true, y_pred, self.thresholds, sample_weight)
 
+  def reset_states(self):
+    num_thresholds = len(self.thresholds)
+    for v in self.variables:
+      K.set_value(v, np.zeros((num_thresholds,)))
+
 
 class SensitivityAtSpecificity(SensitivitySpecificityBase):
   """Computes the sensitivity at a given specificity.
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 40611a5d75..9a88391dc1 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -28,13 +28,16 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import layers
 from tensorflow.python.keras import metrics
+from tensorflow.python.keras.models import Sequential
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
+from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 class KerasMetricsTest(test.TestCase):
@@ -367,6 +370,19 @@ class KerasMetricsTest(test.TestCase):
     self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7
 
 
+def _get_simple_sequential_model(compile_metrics):
+  model = Sequential()
+  model.add(
+      layers.Dense(
+          3, activation='relu', input_dim=4, kernel_initializer='ones'))
+  model.add(layers.Dense(1, activation='sigmoid', kernel_initializer='ones'))
+  model.compile(
+      loss='mae',
+      metrics=compile_metrics,
+      optimizer=RMSPropOptimizer(learning_rate=0.001))
+  return model
+
+
 @test_util.run_all_in_graph_and_eager_modes
 class FalsePositivesTest(test.TestCase):
 
@@ -435,6 +451,16 @@ class FalsePositivesTest(test.TestCase):
         r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):
       metrics.FalsePositives(thresholds=[-1, 0.5, 2])
 
+  def test_reset_states(self):
+    fp_obj = metrics.FalsePositives()
+    model = _get_simple_sequential_model([fp_obj])
+    x = np.ones((100, 4))
+    y = np.zeros((100, 1))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(fp_obj.accumulator), 100.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(fp_obj.accumulator), 100.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class FalseNegativesTest(test.TestCase):
@@ -497,6 +523,16 @@ class FalseNegativesTest(test.TestCase):
     result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
     self.assertAllClose([4., 16., 23.], self.evaluate(result))
 
+  def test_reset_states(self):
+    fn_obj = metrics.FalseNegatives()
+    model = _get_simple_sequential_model([fn_obj])
+    x = np.zeros((100, 4))
+    y = np.ones((100, 1))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(fn_obj.accumulator), 100.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(fn_obj.accumulator), 100.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class TrueNegativesTest(test.TestCase):
@@ -559,6 +595,16 @@ class TrueNegativesTest(test.TestCase):
     result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
     self.assertAllClose([5., 15., 23.], self.evaluate(result))
 
+  def test_reset_states(self):
+    tn_obj = metrics.TrueNegatives()
+    model = _get_simple_sequential_model([tn_obj])
+    x = np.zeros((100, 4))
+    y = np.zeros((100, 1))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(tn_obj.accumulator), 100.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(tn_obj.accumulator), 100.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class TruePositivesTest(test.TestCase):
@@ -620,6 +666,16 @@ class TruePositivesTest(test.TestCase):
     result = tp_obj(y_true, y_pred, sample_weight=37.)
     self.assertAllClose([222., 111., 37.], self.evaluate(result))
 
+  def test_reset_states(self):
+    tp_obj = metrics.TruePositives()
+    model = _get_simple_sequential_model([tp_obj])
+    x = np.ones((100, 4))
+    y = np.ones((100, 1))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(tp_obj.accumulator), 100.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(tp_obj.accumulator), 100.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class PrecisionTest(test.TestCase):
@@ -732,6 +788,18 @@ class PrecisionTest(test.TestCase):
     self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
                          1e-3)
 
+  def test_reset_states(self):
+    p_obj = metrics.Precision()
+    model = _get_simple_sequential_model([p_obj])
+    x = np.concatenate((np.ones((50, 4)), np.ones((50, 4))))
+    y = np.concatenate((np.ones((50, 1)), np.zeros((50, 1))))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(p_obj.tp), 50.)
+    self.assertEqual(self.evaluate(p_obj.fp), 50.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(p_obj.tp), 50.)
+    self.assertEqual(self.evaluate(p_obj.fp), 50.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class RecallTest(test.TestCase):
@@ -843,6 +911,18 @@ class RecallTest(test.TestCase):
     self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
                          1e-3)
 
+  def test_reset_states(self):
+    r_obj = metrics.Recall()
+    model = _get_simple_sequential_model([r_obj])
+    x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4))))
+    y = np.concatenate((np.ones((50, 1)), np.ones((50, 1))))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(r_obj.tp), 50.)
+    self.assertEqual(self.evaluate(r_obj.fn), 50.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(r_obj.tp), 50.)
+    self.assertEqual(self.evaluate(r_obj.fn), 50.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
@@ -932,6 +1012,24 @@ class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
       metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)
 
+  def test_reset_states(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.5, num_thresholds=1)
+    model = _get_simple_sequential_model([s_obj])
+    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),
+                        np.ones((25, 4))))
+    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
+                        np.zeros((25, 1))))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(s_obj.tp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fn), 25.)
+    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(s_obj.tp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fn), 25.)
+    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
@@ -1021,6 +1119,24 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
       metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
 
+  def test_reset_states(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.5, num_thresholds=1)
+    model = _get_simple_sequential_model([s_obj])
+    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),
+                        np.ones((25, 4))))
+    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
+                        np.zeros((25, 1))))
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(s_obj.tp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fn), 25.)
+    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+    model.evaluate(x, y)
+    self.assertEqual(self.evaluate(s_obj.tp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fp), 25.)
+    self.assertEqual(self.evaluate(s_obj.fn), 25.)
+    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From a04691962177976c60aba998f477f5a8165451ed Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Wed, 28 Nov 2018 14:35:10 -0800
Subject: [PATCH 0958/1554] [XLA] Do not generate denormals when constructing
 no_duplicate bf16 literals

PiperOrigin-RevId: 223237542
---
 tensorflow/compiler/xla/tests/test_utils.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc
index 8b4be6607f..eafa48ed7b 100644
--- a/tensorflow/compiler/xla/tests/test_utils.cc
+++ b/tensorflow/compiler/xla/tests/test_utils.cc
@@ -56,7 +56,8 @@ void PopulateWithIntNext<half>(Literal* literal) {
 template <>
 void PopulateWithIntNext<bfloat16>(Literal* literal) {
   // Duplicates may be generated if we don't have enough bits.
-  uint16 next_value = 0;
+  // Start at 0x80 rather than 0 to avoid denormals.
+  uint16 next_value = 0x80;
   for (bfloat16& value : literal->data<bfloat16>()) {
     // Zero-out the MSB of the exponent to avoid Infs and NaNs, and put it into
     // the sign bit. We could be less wasteful, but this is best-effort anyway.
-- 
GitLab


From 2185cf1ba8cc0ce72624d43f90c21feb96d6d1a0 Mon Sep 17 00:00:00 2001
From: Sachin Joglekar <srjoglekar@google.com>
Date: Wed, 28 Nov 2018 14:37:50 -0800
Subject: [PATCH 0959/1554] Adds string support in Java API

Fixes #21213.

PiperOrigin-RevId: 223238001
---
 .../java/org/tensorflow/lite/DataType.java    |   9 +-
 .../main/java/org/tensorflow/lite/Tensor.java |   2 +
 tensorflow/lite/java/src/main/native/BUILD    |   1 +
 .../native/nativeinterpreterwrapper_jni.cc    |   2 +
 .../lite/java/src/main/native/tensor_jni.cc   | 115 +++++++++++++++---
 .../org/tensorflow/lite/DataTypeTest.java     |   1 +
 .../lite/NativeInterpreterWrapperTest.java    |  47 +++++++
 tensorflow/lite/java/src/testdata/string.bin  | Bin 0 -> 584 bytes
 8 files changed, 160 insertions(+), 17 deletions(-)
 create mode 100644 tensorflow/lite/java/src/testdata/string.bin

diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java
index 41093e8ffe..bd47574f71 100644
--- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java
+++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/DataType.java
@@ -27,7 +27,10 @@ public enum DataType {
   UINT8(3),
 
   /** 64-bit signed integer. */
-  INT64(4);
+  INT64(4),
+
+  /** Strings. */
+  STRING(5);
 
   private final int value;
 
@@ -46,6 +49,8 @@ public enum DataType {
         return 1;
       case INT64:
         return 8;
+      case STRING:
+        return -1;
     }
     throw new IllegalArgumentException(
         "DataType error: DataType " + this + " is not supported yet");
@@ -82,6 +87,8 @@ public enum DataType {
         return "byte";
       case INT64:
         return "long";
+      case STRING:
+        return "string";
     }
     throw new IllegalArgumentException(
         "DataType error: DataType " + this + " is not supported yet");
diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java
index 6ca47aa3ed..7aa24b4198 100644
--- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java
+++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Tensor.java
@@ -162,6 +162,8 @@ public final class Tensor {
         return DataType.UINT8;
       } else if (long.class.equals(c)) {
         return DataType.INT64;
+      } else if (String.class.equals(c)) {
+        return DataType.STRING;
       }
     }
     throw new IllegalArgumentException(
diff --git a/tensorflow/lite/java/src/main/native/BUILD b/tensorflow/lite/java/src/main/native/BUILD
index 2abba24345..8f95f14518 100644
--- a/tensorflow/lite/java/src/main/native/BUILD
+++ b/tensorflow/lite/java/src/main/native/BUILD
@@ -43,6 +43,7 @@ cc_library(
         "//tensorflow/lite:context",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:schema_fbs_version",
+        "//tensorflow/lite:string_util",
     ],
     alwayslink = 1,
 )
diff --git a/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc b/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
index c7389c5811..1e98f94250 100644
--- a/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
+++ b/tensorflow/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
@@ -78,6 +78,8 @@ int getDataType(TfLiteType data_type) {
       return 3;
     case kTfLiteInt64:
       return 4;
+    case kTfLiteString:
+      return 5;
     default:
       return -1;
   }
diff --git a/tensorflow/lite/java/src/main/native/tensor_jni.cc b/tensorflow/lite/java/src/main/native/tensor_jni.cc
index 1d813d50da..035bec6a44 100644
--- a/tensorflow/lite/java/src/main/native/tensor_jni.cc
+++ b/tensorflow/lite/java/src/main/native/tensor_jni.cc
@@ -16,8 +16,10 @@ limitations under the License.
 #include "tensorflow/lite/java/src/main/native/tensor_jni.h"
 #include <cstring>
 #include <memory>
+#include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/java/src/main/native/exception_jni.h"
+#include "tensorflow/lite/string_util.h"
 
 namespace {
 
@@ -48,7 +50,7 @@ TfLiteTensor* GetTensorFromHandle(JNIEnv* env, jlong handle) {
   return reinterpret_cast<TensorHandle*>(handle)->tensor();
 }
 
-size_t elementByteSize(TfLiteType data_type) {
+size_t ElementByteSize(TfLiteType data_type) {
   // The code in this file makes the assumption that the
   // TensorFlow TF_DataTypes and the Java primitive types
   // have the same byte sizes. Validate that:
@@ -77,11 +79,11 @@ size_t elementByteSize(TfLiteType data_type) {
   }
 }
 
-size_t writeOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type,
+size_t WriteOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type,
                                 void* dst, size_t dst_size) {
   jarray array = static_cast<jarray>(object);
   const int num_elements = env->GetArrayLength(array);
-  size_t to_copy = num_elements * elementByteSize(type);
+  size_t to_copy = num_elements * ElementByteSize(type);
   if (to_copy > dst_size) {
     throwException(env, kIllegalStateException,
                    "Internal error: cannot write Java array of %d bytes to "
@@ -126,10 +128,10 @@ size_t writeOneDimensionalArray(JNIEnv* env, jobject object, TfLiteType type,
   }
 }
 
-size_t readOneDimensionalArray(JNIEnv* env, TfLiteType data_type,
+size_t ReadOneDimensionalArray(JNIEnv* env, TfLiteType data_type,
                                const void* src, size_t src_size, jarray dst) {
   const int len = env->GetArrayLength(dst);
-  const size_t size = len * elementByteSize(data_type);
+  const size_t size = len * ElementByteSize(data_type);
   if (size > src_size) {
     throwException(
         env, kIllegalStateException,
@@ -170,17 +172,17 @@ size_t readOneDimensionalArray(JNIEnv* env, TfLiteType data_type,
   return 0;
 }
 
-size_t readMultiDimensionalArray(JNIEnv* env, TfLiteType data_type, char* src,
+size_t ReadMultiDimensionalArray(JNIEnv* env, TfLiteType data_type, char* src,
                                  size_t src_size, int dims_left, jarray dst) {
   if (dims_left == 1) {
-    return readOneDimensionalArray(env, data_type, src, src_size, dst);
+    return ReadOneDimensionalArray(env, data_type, src, src_size, dst);
   } else {
     jobjectArray ndarray = static_cast<jobjectArray>(dst);
     int len = env->GetArrayLength(ndarray);
     size_t size = 0;
     for (int i = 0; i < len; ++i) {
       jarray row = static_cast<jarray>(env->GetObjectArrayElement(ndarray, i));
-      size += readMultiDimensionalArray(env, data_type, src + size,
+      size += ReadMultiDimensionalArray(env, data_type, src + size,
                                         src_size - size, dims_left - 1, row);
       env->DeleteLocalRef(row);
       if (env->ExceptionCheck()) return size;
@@ -189,10 +191,43 @@ size_t readMultiDimensionalArray(JNIEnv* env, TfLiteType data_type, char* src,
   }
 }
 
-size_t writeMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type,
+// Returns the total number of strings read.
+int ReadMultiDimensionalStringArray(JNIEnv* env, TfLiteTensor* tensor,
+                                    int dims_left, int start_str_index,
+                                    jarray dst) {
+  jobjectArray object_array = static_cast<jobjectArray>(dst);
+  int len = env->GetArrayLength(object_array);
+  int num_strings_read = 0;
+
+  // If dst is a 1-dimensional array, copy the strings into it. Else
+  // recursively call ReadMultiDimensionalStringArray over sub-dimensions.
+  if (dims_left == 1) {
+    for (int i = 0; i < len; ++i) {
+      const tflite::StringRef strref =
+          tflite::GetString(tensor, start_str_index + num_strings_read);
+      jstring string_dest = env->NewStringUTF(strref.str);
+      env->SetObjectArrayElement(object_array, i, string_dest);
+      env->DeleteLocalRef(string_dest);
+      ++num_strings_read;
+    }
+  } else {
+    for (int i = 0; i < len; ++i) {
+      jarray row =
+          static_cast<jarray>(env->GetObjectArrayElement(object_array, i));
+      num_strings_read += ReadMultiDimensionalStringArray(
+          env, tensor, dims_left - 1, start_str_index + num_strings_read, row);
+      env->DeleteLocalRef(row);
+      if (env->ExceptionCheck()) return num_strings_read;
+    }
+  }
+
+  return num_strings_read;
+}
+
+size_t WriteMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type,
                                   int dims_left, char** dst, int dst_size) {
   if (dims_left <= 1) {
-    return writeOneDimensionalArray(env, src, type, *dst, dst_size);
+    return WriteOneDimensionalArray(env, src, type, *dst, dst_size);
   } else {
     jobjectArray ndarray = static_cast<jobjectArray>(src);
     int len = env->GetArrayLength(ndarray);
@@ -200,7 +235,7 @@ size_t writeMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type,
     for (int i = 0; i < len; ++i) {
       jobject row = env->GetObjectArrayElement(ndarray, i);
       char* next_dst = *dst + sz;
-      sz += writeMultiDimensionalArray(env, row, type, dims_left - 1, &next_dst,
+      sz += WriteMultiDimensionalArray(env, row, type, dims_left - 1, &next_dst,
                                        dst_size - sz);
       env->DeleteLocalRef(row);
       if (env->ExceptionCheck()) return sz;
@@ -209,6 +244,44 @@ size_t writeMultiDimensionalArray(JNIEnv* env, jobject src, TfLiteType type,
   }
 }
 
+void PopulateStringDynamicBuffer(JNIEnv* env, jobject src,
+                                 tflite::DynamicBuffer* dst_buffer,
+                                 int dims_left) {
+  jobjectArray object_array = static_cast<jobjectArray>(src);
+  const int num_elements = env->GetArrayLength(object_array);
+
+  // If src is a 1-dimensional array, add the strings into dst_buffer. Else
+  // recursively call populateStringDynamicBuffer over sub-dimensions.
+  if (dims_left <= 1) {
+    for (int i = 0; i < num_elements; ++i) {
+      jstring string_obj =
+          static_cast<jstring>(env->GetObjectArrayElement(object_array, i));
+      const char* chars = env->GetStringUTFChars(string_obj, nullptr);
+      // + 1 for terminating character.
+      const int byte_len = env->GetStringUTFLength(string_obj) + 1;
+      dst_buffer->AddString(chars, byte_len);
+      env->ReleaseStringUTFChars(string_obj, chars);
+      env->DeleteLocalRef(string_obj);
+    }
+  } else {
+    for (int i = 0; i < num_elements; ++i) {
+      jobject row = env->GetObjectArrayElement(object_array, i);
+      PopulateStringDynamicBuffer(env, row, dst_buffer, dims_left - 1);
+      env->DeleteLocalRef(row);
+      if (env->ExceptionCheck()) return;
+    }
+  }
+}
+
+void WriteMultiDimensionalStringArray(JNIEnv* env, jobject src,
+                                      TfLiteTensor* tensor) {
+  tflite::DynamicBuffer dst_buffer;
+  PopulateStringDynamicBuffer(env, src, &dst_buffer, tensor->dims->size);
+  if (!env->ExceptionCheck()) {
+    dst_buffer.WriteToTensor(tensor);
+  }
+}
+
 }  // namespace
 
 JNIEXPORT jlong JNICALL Java_org_tensorflow_lite_Tensor_create(
@@ -266,8 +339,14 @@ Java_org_tensorflow_lite_Tensor_readMultiDimensionalArray(JNIEnv* env,
                    "Internal error: Cannot copy empty/scalar Tensors.");
     return;
   }
-  readMultiDimensionalArray(env, tensor->type, tensor->data.raw, tensor->bytes,
-                            num_dims, static_cast<jarray>(value));
+  if (tensor->type == kTfLiteString) {
+    ReadMultiDimensionalStringArray(env, tensor, num_dims, 0,
+                                    static_cast<jarray>(value));
+  } else {
+    ReadMultiDimensionalArray(env, tensor->type, tensor->data.raw,
+                              tensor->bytes, num_dims,
+                              static_cast<jarray>(value));
+  }
 }
 
 JNIEXPORT void JNICALL
@@ -277,7 +356,7 @@ Java_org_tensorflow_lite_Tensor_writeMultiDimensionalArray(JNIEnv* env,
                                                            jobject src) {
   TfLiteTensor* tensor = GetTensorFromHandle(env, handle);
   if (tensor == nullptr) return;
-  if (tensor->data.raw == nullptr) {
+  if (tensor->type != kTfLiteString && tensor->data.raw == nullptr) {
     throwException(env, kIllegalArgumentException,
                    "Internal error: Target Tensor hasn't been allocated.");
     return;
@@ -287,8 +366,12 @@ Java_org_tensorflow_lite_Tensor_writeMultiDimensionalArray(JNIEnv* env,
                    "Internal error: Cannot copy empty/scalar Tensors.");
     return;
   }
-  writeMultiDimensionalArray(env, src, tensor->type, tensor->dims->size,
-                             &tensor->data.raw, tensor->bytes);
+  if (tensor->type == kTfLiteString) {
+    WriteMultiDimensionalStringArray(env, src, tensor);
+  } else {
+    WriteMultiDimensionalArray(env, src, tensor->type, tensor->dims->size,
+                               &tensor->data.raw, tensor->bytes);
+  }
 }
 
 JNIEXPORT jint JNICALL Java_org_tensorflow_lite_Tensor_dtype(JNIEnv* env,
diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java
index 6d6417f895..8412ec0e9d 100644
--- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java
+++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/DataTypeTest.java
@@ -30,6 +30,7 @@ public final class DataTypeTest {
     assertThat(DataType.INT32.byteSize()).isEqualTo(4);
     assertThat(DataType.UINT8.byteSize()).isEqualTo(1);
     assertThat(DataType.INT64.byteSize()).isEqualTo(8);
+    assertThat(DataType.STRING.byteSize()).isEqualTo(-1);
   }
 
   @Test
diff --git a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java
index 07d334c33b..b00efa77cb 100644
--- a/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java
+++ b/tensorflow/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java
@@ -43,6 +43,9 @@ public final class NativeInterpreterWrapperTest {
   private static final String BYTE_MODEL_PATH =
       "tensorflow/lite/java/src/testdata/uint8.bin";
 
+  private static final String STRING_MODEL_PATH =
+      "tensorflow/lite/java/src/testdata/string.bin";
+
   private static final String QUANTIZED_MODEL_PATH =
       "tensorflow/lite/java/src/testdata/quantized.bin";
 
@@ -224,6 +227,50 @@ public final class NativeInterpreterWrapperTest {
     wrapper.close();
   }
 
+  @Test
+  public void testRunWithString() {
+    NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(STRING_MODEL_PATH);
+    String[] oneD = {"s1", "s22", "s333"};
+    String[][] twoD = {oneD, oneD, oneD, oneD, oneD, oneD, oneD, oneD};
+    String[][][] threeD = {twoD, twoD, twoD, twoD, twoD, twoD, twoD, twoD};
+    String[][][][] fourD = {threeD, threeD};
+    Object[] inputs = {fourD};
+    String[][][][] parsedOutputs = new String[2][4][4][12];
+    Map<Integer, Object> outputs = new HashMap<>();
+    outputs.put(0, parsedOutputs);
+    wrapper.run(inputs, outputs);
+    String[] outputOneD = parsedOutputs[0][0][0];
+    String[] expected = {
+      "s1", "s22", "s333", "s1", "s22", "s333", "s1", "s22", "s333", "s1", "s22", "s333"
+    };
+    assertThat(outputOneD).isEqualTo(expected);
+    wrapper.close();
+  }
+
+  @Test
+  public void testRunWithString_wrongShapeError() {
+    NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(STRING_MODEL_PATH);
+    String[] oneD = {"s1", "s22", "s333"};
+    String[][] twoD = {oneD, oneD, oneD, oneD, oneD, oneD, oneD, oneD};
+    String[][][] threeD = {twoD, twoD, twoD, twoD, twoD, twoD, twoD, twoD};
+    String[][][][] fourD = {threeD, threeD};
+    Object[] inputs = {fourD};
+    String[][][][] parsedOutputs = new String[2][4][4][10];
+    Map<Integer, Object> outputs = new HashMap<>();
+    outputs.put(0, parsedOutputs);
+    try {
+      wrapper.run(inputs, outputs);
+      fail();
+    } catch (IllegalArgumentException e) {
+      assertThat(e)
+          .hasMessageThat()
+          .contains(
+              "Cannot copy between a TensorFlowLite tensor with shape [2, 4, 4, 12] and "
+                  + "a Java object with shape [2, 4, 4, 10]");
+    }
+    wrapper.close();
+  }
+
   @Test
   public void testRunWithByteBufferHavingBytes() {
     NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(BYTE_MODEL_PATH);
diff --git a/tensorflow/lite/java/src/testdata/string.bin b/tensorflow/lite/java/src/testdata/string.bin
new file mode 100644
index 0000000000000000000000000000000000000000..36a2509acdfa17841d0c128674e7b4e382ad00fc
GIT binary patch
literal 584
zcmb1OU|<Mw^D$;%;A4<rU}4~3;9(G85MkhBU|?WoU|<knVqo9_(F_a>3JeSkj0_A6
z2@DJj{2+b)&i)F{`FUljMJ1^zdJHTK3=Ad=3=BF95IrFNihuwAhyDBiAH)`5U|?Wk
zU;wKF@j>RE`1k++nScNP^Dv0OZ3l^i>=I#MVBlb2V31*i*bA~7gh6J2+*ZKAz_0^q
zUflox{}~t<SV4Ax<Yb_30oemm4>AXYL2@ATi%W_!^U~up^9o8!7{K=Mfb9pV0hs~P
z2eONSq23unGcX*2*ueyn2bl??RiI%4QqRl4zz~#LoRL_Nst+c?Zes)c7ZSc44D3kZ
z3v(~Xe2{w$!0Iu>2xM-4X-Pq8Nqk9aUU7aASRD)4T##N+9Evc2NKovtGl1O1z#zf^
z3O@!020;)XB&)%|09FNZI|zgP4bl(7AT|R78v_Re3j-V2d`LWj)Pej4!k};isRhvx
Mwd`QE3=9ln00k8^3;+NC

literal 0
HcmV?d00001

-- 
GitLab


From bbb81ea42831b244289a059de9e4be0203dc4c35 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Wed, 28 Nov 2018 14:51:37 -0800
Subject: [PATCH 0960/1554] Clarify the display of the time taken per "step" in
 the progbar ("samples" or (gradient descent) "steps").

Also fix the issue where the progbar would sometime display "0.0001e-6us/step"

PiperOrigin-RevId: 223240441
---
 tensorflow/python/keras/callbacks.py                 |  3 ++-
 tensorflow/python/keras/utils/generic_utils.py       | 12 +++++++-----
 .../golden/v1/tensorflow.keras.utils.-progbar.pbtxt  |  2 +-
 .../golden/v2/tensorflow.keras.utils.-progbar.pbtxt  |  2 +-
 4 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 8223e795bc..3e3c087e3f 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -494,7 +494,8 @@ class ProgbarLogger(Callback):
       self.progbar = Progbar(
           target=self.target,
           verbose=self.verbose,
-          stateful_metrics=self.stateful_metrics)
+          stateful_metrics=self.stateful_metrics,
+          unit_name='step' if self.use_steps else 'sample')
 
   def on_batch_begin(self, batch, logs=None):
     if self.seen < self.target:
diff --git a/tensorflow/python/keras/utils/generic_utils.py b/tensorflow/python/keras/utils/generic_utils.py
index 375bd9d196..c331ce430b 100644
--- a/tensorflow/python/keras/utils/generic_utils.py
+++ b/tensorflow/python/keras/utils/generic_utils.py
@@ -319,14 +319,16 @@ class Progbar(object):
           will be displayed as-is. All others will be averaged
           by the progbar before display.
       interval: Minimum visual progress update interval (in seconds).
+      unit_name: Display name for step counts (usually "step" or "sample").
   """
 
   def __init__(self, target, width=30, verbose=1, interval=0.05,
-               stateful_metrics=None):
+               stateful_metrics=None, unit_name='step'):
     self.target = target
     self.width = width
     self.verbose = verbose
     self.interval = interval
+    self.unit_name = unit_name
     if stateful_metrics:
       self.stateful_metrics = set(stateful_metrics)
     else:
@@ -425,12 +427,12 @@ class Progbar(object):
 
         info = ' - ETA: %s' % eta_format
       else:
-        if time_per_unit >= 1:
-          info += ' %.0fs/step' % time_per_unit
+        if time_per_unit >= 1 or time_per_unit == 0:
+          info += ' %.0fs/%s' % (time_per_unit, self.unit_name)
         elif time_per_unit >= 1e-3:
-          info += ' %.0fms/step' % (time_per_unit * 1e3)
+          info += ' %.0fms/%s' % (time_per_unit * 1e3, self.unit_name)
         else:
-          info += ' %.0fus/step' % (time_per_unit * 1e6)
+          info += ' %.0fus/%s' % (time_per_unit * 1e6, self.unit_name)
 
       for k in self._values_order:
         info += ' - %s:' % k
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt
index be4496e753..8177cc71ed 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.utils.-progbar.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\'], "
+    argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\', \'unit_name\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\', \'step\'], "
   }
   member_method {
     name: "add"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt
index be4496e753..8177cc71ed 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.utils.-progbar.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\'], "
+    argspec: "args=[\'self\', \'target\', \'width\', \'verbose\', \'interval\', \'stateful_metrics\', \'unit_name\'], varargs=None, keywords=None, defaults=[\'30\', \'1\', \'0.05\', \'None\', \'step\'], "
   }
   member_method {
     name: "add"
-- 
GitLab


From a7b3f17a16ea09195839ac0b88335dd783cbef76 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 14:56:02 -0800
Subject: [PATCH 0961/1554] Add parallel_iterations option to batch_jacobian
 and jacobian functions.

PiperOrigin-RevId: 223241273
---
 tensorflow/python/eager/backprop.py            | 18 ++++++++++++++----
 tensorflow/python/eager/backprop_test.py       | 17 +++++++++++++++++
 .../golden/v1/tensorflow.-gradient-tape.pbtxt  |  4 ++--
 .../golden/v2/tensorflow.-gradient-tape.pbtxt  |  4 ++--
 4 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 69d444a90e..29f9b2cda3 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -955,6 +955,7 @@ class GradientTape(object):
                target,
                sources,
                unconnected_gradients=UnconnectedGradients.NONE,
+               parallel_iterations=None,
                experimental_use_pfor=True):
     """Computes the jacobian using operations recorded in context of this tape.
 
@@ -978,6 +979,8 @@ class GradientTape(object):
         alters the value which will be returned if the target and sources are
         unconnected. The possible values and effects are detailed in
         'UnconnectedGradients' and it defaults to 'none'.
+      parallel_iterations: A knob to control how many iterations are dispatched
+        in parallel. This knob can be used to control the total memory usage.
       experimental_use_pfor: If true, vectorizes the jacobian computation. Else
         falls back to a sequential while_loop. Vectorization can sometimes fail
         or lead to excessive memory usage. This option can be used to disable
@@ -1016,7 +1019,8 @@ class GradientTape(object):
 
     if experimental_use_pfor:
       try:
-        output = pfor_ops.pfor(loop_fn, target_size)
+        output = pfor_ops.pfor(loop_fn, target_size,
+                               parallel_iterations=parallel_iterations)
       except ValueError as err:
         six.reraise(
             ValueError,
@@ -1032,7 +1036,8 @@ class GradientTape(object):
             " to compute the jacobian with eager execution enabled and with "
             " experimental_use_pfor set to False.")
       output = pfor_ops.for_loop(
-          loop_fn, [target.dtype] * len(flat_sources), target_size)
+          loop_fn, [target.dtype] * len(flat_sources), target_size,
+          parallel_iterations=parallel_iterations)
 
     for i, out in enumerate(output):
       if out is not None:
@@ -1049,6 +1054,7 @@ class GradientTape(object):
                      target,
                      source,
                      unconnected_gradients=UnconnectedGradients.NONE,
+                     parallel_iterations=None,
                      experimental_use_pfor=True):
     """Computes and stacks per-example jacobians.
 
@@ -1081,6 +1087,8 @@ class GradientTape(object):
         alters the value which will be returned if the target and sources are
         unconnected. The possible values and effects are detailed in
         'UnconnectedGradients' and it defaults to 'none'.
+      parallel_iterations: A knob to control how many iterations are dispatched
+        in parallel. This knob can be used to control the total memory usage.
       experimental_use_pfor: If true, uses pfor for computing the Jacobian. Else
         uses a tf.while_loop.
 
@@ -1127,7 +1135,8 @@ class GradientTape(object):
 
     if experimental_use_pfor:
       try:
-        output = pfor_ops.pfor(loop_fn, target_row_size)
+        output = pfor_ops.pfor(loop_fn, target_row_size,
+                               parallel_iterations=parallel_iterations)
       except ValueError as err:
         six.reraise(
             ValueError,
@@ -1142,7 +1151,8 @@ class GradientTape(object):
             "GradientTape must be created with persistent=True"
             " to compute the batch_jacobian with eager execution enabled and "
             " with experimental_use_pfor set to False.")
-      output = pfor_ops.for_loop(loop_fn, target.dtype, target_row_size)
+      output = pfor_ops.for_loop(loop_fn, target.dtype, target_row_size,
+                                 parallel_iterations=parallel_iterations)
     if output is None:
       return None
     output = array_ops.reshape(output,
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 08553b9f27..3cec40a48f 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -1303,6 +1303,14 @@ class JacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'No converter'):
       g.jacobian(y, x, experimental_use_pfor=True)
 
+  def test_parallel_iterations(self):
+    with backprop.GradientTape(persistent=True) as g:
+      x = constant_op.constant([[1., 2], [3, 4]])
+      g.watch(x)
+      y = math_ops.matmul(x, x)
+    self.assertAllClose(g.jacobian(y, x, parallel_iterations=2),
+                        g.jacobian(y, x, parallel_iterations=3))
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class BatchJacobianTest(test.TestCase):
@@ -1397,5 +1405,14 @@ class BatchJacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'No converter'):
       g.batch_jacobian(y, x, experimental_use_pfor=True)
 
+  def test_parallel_iterations(self):
+    with backprop.GradientTape(persistent=True) as g:
+      x = constant_op.constant([[1., 2], [3, 4]])
+      g.watch(x)
+      w = constant_op.constant([[1., 2, 3, 4], [5, 6, 7, 8]])
+      y = math_ops.matmul(x, w)
+    self.assertAllClose(g.batch_jacobian(y, x, parallel_iterations=2),
+                        g.batch_jacobian(y, x, parallel_iterations=3))
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
index e37d29995a..2299a009d3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-gradient-tape.pbtxt
@@ -8,7 +8,7 @@ tf_class {
   }
   member_method {
     name: "batch_jacobian"
-    argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+    argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], "
   }
   member_method {
     name: "gradient"
@@ -16,7 +16,7 @@ tf_class {
   }
   member_method {
     name: "jacobian"
-    argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+    argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], "
   }
   member_method {
     name: "reset"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
index e37d29995a..2299a009d3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-gradient-tape.pbtxt
@@ -8,7 +8,7 @@ tf_class {
   }
   member_method {
     name: "batch_jacobian"
-    argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+    argspec: "args=[\'self\', \'target\', \'source\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], "
   }
   member_method {
     name: "gradient"
@@ -16,7 +16,7 @@ tf_class {
   }
   member_method {
     name: "jacobian"
-    argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'True\'], "
+    argspec: "args=[\'self\', \'target\', \'sources\', \'unconnected_gradients\', \'parallel_iterations\', \'experimental_use_pfor\'], varargs=None, keywords=None, defaults=[\'UnconnectedGradients.NONE\', \'None\', \'True\'], "
   }
   member_method {
     name: "reset"
-- 
GitLab


From 36325c5234ba3a8449448d9fdabc82a73bb3a9a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 15:09:38 -0800
Subject: [PATCH 0962/1554] Don't create an automatic rename rule for any
 symbol with a manually-specified rename.

PiperOrigin-RevId: 223243781
---
 tensorflow/tools/compatibility/BUILD          |  15 ++-
 tensorflow/tools/compatibility/renames_v2.py  |  34 ++----
 .../tools/compatibility/tf_upgrade_v2.py      |  93 ++--------------
 .../tools/compatibility/tf_upgrade_v2_main.py | 100 ++++++++++++++++++
 tensorflow/tools/compatibility/update/BUILD   |   1 +
 .../update/generate_v2_renames_map.py         |  10 +-
 6 files changed, 133 insertions(+), 120 deletions(-)
 create mode 100644 tensorflow/tools/compatibility/tf_upgrade_v2_main.py

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index a6574dac53..197fa75015 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -51,8 +51,8 @@ py_library(
     srcs_version = "PY2AND3",
 )
 
-py_binary(
-    name = "tf_upgrade_v2",
+py_library(
+    name = "tf_upgrade_v2_lib",
     srcs = [
         "renames_v2.py",
         "tf_upgrade_v2.py",
@@ -61,6 +61,17 @@ py_binary(
     deps = [":ast_edits"],
 )
 
+py_binary(
+    name = "tf_upgrade_v2",
+    srcs = ["tf_upgrade_v2_main.py"],
+    main = "tf_upgrade_v2_main.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ast_edits",
+        ":tf_upgrade_v2_lib",
+    ],
+)
+
 py_test(
     name = "tf_upgrade_v2_test",
     srcs = ["tf_upgrade_v2_test.py"],
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 70757556c1..8dbeb078b6 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -92,15 +92,12 @@ renames = {
     'tf.assign': 'tf.compat.v1.assign',
     'tf.assign_add': 'tf.compat.v1.assign_add',
     'tf.assign_sub': 'tf.compat.v1.assign_sub',
-    'tf.batch_to_space_nd': 'tf.compat.v1.batch_to_space_nd',
     'tf.betainc': 'tf.math.betainc',
-    'tf.bincount': 'tf.compat.v1.bincount',
     'tf.ceil': 'tf.math.ceil',
     'tf.check_numerics': 'tf.debugging.check_numerics',
     'tf.cholesky': 'tf.linalg.cholesky',
     'tf.cholesky_solve': 'tf.linalg.cholesky_solve',
     'tf.colocate_with': 'tf.compat.v1.colocate_with',
-    'tf.confusion_matrix': 'tf.compat.v1.confusion_matrix',
     'tf.conj': 'tf.math.conj',
     'tf.container': 'tf.compat.v1.container',
     'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices',
@@ -110,6 +107,7 @@ renames = {
     'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
+    'tf.data.Iterator': 'tf.compat.v1.data.Iterator',
     'tf.debugging.is_finite': 'tf.math.is_finite',
     'tf.debugging.is_inf': 'tf.math.is_inf',
     'tf.debugging.is_nan': 'tf.math.is_nan',
@@ -117,7 +115,6 @@ renames = {
     'tf.debugging.is_strictly_increasing': 'tf.math.is_strictly_increasing',
     'tf.decode_base64': 'tf.io.decode_base64',
     'tf.decode_compressed': 'tf.io.decode_compressed',
-    'tf.decode_csv': 'tf.compat.v1.decode_csv',
     'tf.decode_json_example': 'tf.io.decode_json_example',
     'tf.decode_raw': 'tf.io.decode_raw',
     'tf.delete_session_tensor': 'tf.compat.v1.delete_session_tensor',
@@ -180,21 +177,9 @@ renames = {
     'tf.get_session_tensor': 'tf.compat.v1.get_session_tensor',
     'tf.get_variable': 'tf.compat.v1.get_variable',
     'tf.get_variable_scope': 'tf.compat.v1.get_variable_scope',
-    'tf.gfile.Copy': 'tf.compat.v1.gfile.Copy',
-    'tf.gfile.DeleteRecursively': 'tf.compat.v1.gfile.DeleteRecursively',
-    'tf.gfile.Exists': 'tf.compat.v1.gfile.Exists',
     'tf.gfile.FastGFile': 'tf.compat.v1.gfile.FastGFile',
     'tf.gfile.GFile': 'tf.compat.v1.gfile.GFile',
-    'tf.gfile.Glob': 'tf.compat.v1.gfile.Glob',
-    'tf.gfile.IsDirectory': 'tf.compat.v1.gfile.IsDirectory',
-    'tf.gfile.ListDirectory': 'tf.compat.v1.gfile.ListDirectory',
-    'tf.gfile.MakeDirs': 'tf.compat.v1.gfile.MakeDirs',
-    'tf.gfile.MkDir': 'tf.compat.v1.gfile.MkDir',
     'tf.gfile.Open': 'tf.compat.v1.gfile.Open',
-    'tf.gfile.Remove': 'tf.compat.v1.gfile.Remove',
-    'tf.gfile.Rename': 'tf.compat.v1.gfile.Rename',
-    'tf.gfile.Stat': 'tf.compat.v1.gfile.Stat',
-    'tf.gfile.Walk': 'tf.compat.v1.gfile.Walk',
     'tf.global_norm': 'tf.linalg.global_norm',
     'tf.global_variables': 'tf.compat.v1.global_variables',
     'tf.global_variables_initializer': 'tf.compat.v1.global_variables_initializer',
@@ -274,7 +259,6 @@ renames = {
     'tf.layers.separable_conv2d': 'tf.compat.v1.layers.separable_conv2d',
     'tf.lbeta': 'tf.math.lbeta',
     'tf.lgamma': 'tf.math.lgamma',
-    'tf.load_file_system_library': 'tf.compat.v1.load_file_system_library',
     'tf.local_variables': 'tf.compat.v1.local_variables',
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log_sigmoid': 'tf.math.log_sigmoid',
@@ -312,7 +296,6 @@ renames = {
     'tf.losses.sparse_softmax_cross_entropy': 'tf.compat.v1.losses.sparse_softmax_cross_entropy',
     'tf.make_template': 'tf.compat.v1.make_template',
     'tf.make_tensor_proto': 'tf.compat.v1.make_tensor_proto',
-    'tf.manip.batch_to_space_nd': 'tf.compat.v1.manip.batch_to_space_nd',
     'tf.manip.gather_nd': 'tf.gather_nd',
     'tf.manip.reshape': 'tf.reshape',
     'tf.manip.reverse': 'tf.reverse',
@@ -367,7 +350,6 @@ renames = {
     'tf.min_max_variable_partitioner': 'tf.compat.v1.min_max_variable_partitioner',
     'tf.model_variables': 'tf.compat.v1.model_variables',
     'tf.moving_average_variables': 'tf.compat.v1.moving_average_variables',
-    'tf.multinomial': 'tf.compat.v1.multinomial',
     'tf.nn.bidirectional_dynamic_rnn': 'tf.compat.v1.nn.bidirectional_dynamic_rnn',
     'tf.nn.conv3d_backprop_filter_v2': 'tf.nn.conv3d_backprop_filter',
     'tf.nn.ctc_beam_search_decoder_v2': 'tf.nn.ctc_beam_search_decoder',
@@ -376,6 +358,7 @@ renames = {
     'tf.nn.depthwise_conv2d_native_backprop_filter': 'tf.nn.depthwise_conv2d_backprop_filter',
     'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
+    'tf.nn.fused_batch_norm': 'tf.compat.v1.nn.fused_batch_norm',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
     'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool',
     'tf.nn.quantized_conv2d': 'tf.compat.v1.nn.quantized_conv2d',
@@ -417,10 +400,8 @@ renames = {
     'tf.python_io.tf_record_iterator': 'tf.compat.v1.python_io.tf_record_iterator',
     'tf.qr': 'tf.linalg.qr',
     'tf.quantize': 'tf.quantization.quantize',
-    'tf.quantize_v2': 'tf.compat.v1.quantize_v2',
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
     'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
-    'tf.random.multinomial': 'tf.compat.v1.random.multinomial',
     'tf.random.set_random_seed': 'tf.compat.v1.random.set_random_seed',
     'tf.random.stateless_multinomial': 'tf.compat.v1.random.stateless_multinomial',
     'tf.random_crop': 'tf.image.random_crop',
@@ -540,7 +521,6 @@ renames = {
     'tf.sparse_segment_sum': 'tf.compat.v1.sparse_segment_sum',
     'tf.sparse_slice': 'tf.sparse.slice',
     'tf.sparse_softmax': 'tf.sparse.softmax',
-    'tf.sparse_split': 'tf.compat.v1.sparse_split',
     'tf.sparse_tensor_dense_matmul': 'tf.sparse.sparse_dense_matmul',
     'tf.sparse_tensor_to_dense': 'tf.sparse.to_dense',
     'tf.sparse_to_dense': 'tf.compat.v1.sparse_to_dense',
@@ -563,10 +543,8 @@ renames = {
     'tf.squared_difference': 'tf.math.squared_difference',
     'tf.string_join': 'tf.strings.join',
     'tf.string_strip': 'tf.strings.strip',
-    'tf.string_to_hash_bucket': 'tf.compat.v1.string_to_hash_bucket',
     'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
     'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
-    'tf.string_to_number': 'tf.compat.v1.string_to_number',
     'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
@@ -591,16 +569,16 @@ renames = {
     'tf.to_int32': 'tf.compat.v1.to_int32',
     'tf.to_int64': 'tf.compat.v1.to_int64',
     'tf.trace': 'tf.linalg.trace',
-    'tf.train.ChiefSessionCreator': 'tf.compat.v1.train.ChiefSessionCreator',
-    'tf.train.MonitoredSession': 'tf.compat.v1.train.MonitoredSession',
-    'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread',
     'tf.train.AdadeltaOptimizer': 'tf.compat.v1.train.AdadeltaOptimizer',
     'tf.train.AdagradDAOptimizer': 'tf.compat.v1.train.AdagradDAOptimizer',
     'tf.train.AdagradOptimizer': 'tf.compat.v1.train.AdagradOptimizer',
     'tf.train.AdamOptimizer': 'tf.compat.v1.train.AdamOptimizer',
+    'tf.train.ChiefSessionCreator': 'tf.compat.v1.train.ChiefSessionCreator',
     'tf.train.FtrlOptimizer': 'tf.compat.v1.train.FtrlOptimizer',
     'tf.train.GradientDescentOptimizer': 'tf.compat.v1.train.GradientDescentOptimizer',
+    'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread',
     'tf.train.MomentumOptimizer': 'tf.compat.v1.train.MomentumOptimizer',
+    'tf.train.MonitoredSession': 'tf.compat.v1.train.MonitoredSession',
     'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession',
     'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader',
     'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer',
@@ -615,8 +593,8 @@ renames = {
     'tf.train.SingularMonitoredSession': 'tf.compat.v1.train.SingularMonitoredSession',
     'tf.train.Supervisor': 'tf.compat.v1.train.Supervisor',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
-    'tf.train.WorkerSessionCreator': 'tf.compat.v1.train.WorkerSessionCreator',
     'tf.train.VocabInfo': 'tf.compat.v1.train.VocabInfo',
+    'tf.train.WorkerSessionCreator': 'tf.compat.v1.train.WorkerSessionCreator',
     'tf.train.add_queue_runner': 'tf.compat.v1.train.add_queue_runner',
     'tf.train.assert_global_step': 'tf.compat.v1.train.assert_global_step',
     'tf.train.basic_train_loop': 'tf.compat.v1.train.basic_train_loop',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index b63e0b0a1b..9e010cd7b9 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import argparse
-
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
 
@@ -275,14 +273,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         },
     }
 
-    # Mapping from function to the new name of the function
-    self.symbol_renames = renames_v2.renames
     # pylint: disable=line-too-long
     # Add additional renames not in renames_v2.py here.
     # IMPORTANT: For the renames in here, if you also need to add to
     # function_reorders or function_keyword_renames, use the OLD function name.
     # These renames happen after the arguments have been processed.
-    self.symbol_renames.update({
+    self.manual_symbol_renames = {
         "tf.batch_to_space_nd":
             "tf.batch_to_space",
         "tf.gfile.Copy":
@@ -425,13 +421,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.math.confusion_matrix",
         "tf.decode_csv":
             "tf.io.decode_csv",
-    })
+    }
     # pylint: enable=line-too-long
 
-    # For custom behavior and if auto-generate rename in renames_v2.py
-    # is incorrect, add the op name here to exclude it from renames_v2.py.
-    excluded_renames = [
-    ]
+    # Mapping from function to the new name of the function
+    self.symbol_renames = renames_v2.renames
+    self.symbol_renames.update(self.manual_symbol_renames)
 
     # Variables that should be changed to functions.
     self.change_to_function = {}
@@ -711,7 +706,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.symbol_renames = {
         name: new_name
         for name, new_name in self.symbol_renames.items()
-        if name not in self.function_warnings and name not in excluded_renames
+        if name not in self.function_warnings
     }
 
     export_saved_model_renamed = (
@@ -772,79 +767,3 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
               "",
               error="{} requires manual check.".format(name))
     return _helper
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(
-      formatter_class=argparse.RawDescriptionHelpFormatter,
-      description="""Convert a TensorFlow Python file to 2.0
-
-Simple usage:
-  tf_upgrade_v2.py --infile foo.py --outfile bar.py
-  tf_upgrade_v2.py --intree ~/code/old --outtree ~/code/new
-""")
-  parser.add_argument(
-      "--infile",
-      dest="input_file",
-      help="If converting a single file, the name of the file "
-      "to convert")
-  parser.add_argument(
-      "--outfile",
-      dest="output_file",
-      help="If converting a single file, the output filename.")
-  parser.add_argument(
-      "--intree",
-      dest="input_tree",
-      help="If converting a whole tree of files, the directory "
-      "to read from (relative or absolute).")
-  parser.add_argument(
-      "--outtree",
-      dest="output_tree",
-      help="If converting a whole tree of files, the output "
-      "directory (relative or absolute).")
-  parser.add_argument(
-      "--copyotherfiles",
-      dest="copy_other_files",
-      help=("If converting a whole tree of files, whether to "
-            "copy the other files."),
-      type=bool,
-      default=False)
-  parser.add_argument(
-      "--reportfile",
-      dest="report_filename",
-      help=("The name of the file where the report log is "
-            "stored."
-            "(default: %(default)s)"),
-      default="report.txt")
-  args = parser.parse_args()
-
-  upgrade = ast_edits.ASTCodeUpgrader(TFAPIChangeSpec())
-  report_text = None
-  report_filename = args.report_filename
-  files_processed = 0
-  if args.input_file:
-    if not args.output_file:
-      raise ValueError(
-          "--outfile=<output file> argument is required when converting a "
-          "single file.")
-    files_processed, report_text, errors = upgrade.process_file(
-        args.input_file, args.output_file)
-    files_processed = 1
-  elif args.input_tree:
-    if not args.output_tree:
-      raise ValueError(
-          "--outtree=<output directory> argument is required when converting a "
-          "file tree.")
-    files_processed, report_text, errors = upgrade.process_tree(
-        args.input_tree, args.output_tree, args.copy_other_files)
-  else:
-    parser.print_help()
-  if report_text:
-    open(report_filename, "w").write(report_text)
-    print("TensorFlow 2.0 Upgrade Script")
-    print("-----------------------------")
-    print("Converted %d files\n" % files_processed)
-    print("Detected %d errors that require attention" % len(errors))
-    print("-" * 80)
-    print("\n".join(errors))
-    print("\nMake sure to read the detailed log %r\n" % report_filename)
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
new file mode 100644
index 0000000000..498258dfa8
--- /dev/null
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -0,0 +1,100 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Upgrader for Python scripts from 1.* TensorFlow to 2.0 TensorFlow."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+
+from tensorflow.tools.compatibility import ast_edits
+from tensorflow.tools.compatibility import tf_upgrade_v2
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.RawDescriptionHelpFormatter,
+      description="""Convert a TensorFlow Python file to 2.0
+
+Simple usage:
+  tf_upgrade_v2.py --infile foo.py --outfile bar.py
+  tf_upgrade_v2.py --intree ~/code/old --outtree ~/code/new
+""")
+  parser.add_argument(
+      "--infile",
+      dest="input_file",
+      help="If converting a single file, the name of the file "
+      "to convert")
+  parser.add_argument(
+      "--outfile",
+      dest="output_file",
+      help="If converting a single file, the output filename.")
+  parser.add_argument(
+      "--intree",
+      dest="input_tree",
+      help="If converting a whole tree of files, the directory "
+      "to read from (relative or absolute).")
+  parser.add_argument(
+      "--outtree",
+      dest="output_tree",
+      help="If converting a whole tree of files, the output "
+      "directory (relative or absolute).")
+  parser.add_argument(
+      "--copyotherfiles",
+      dest="copy_other_files",
+      help=("If converting a whole tree of files, whether to "
+            "copy the other files."),
+      type=bool,
+      default=False)
+  parser.add_argument(
+      "--reportfile",
+      dest="report_filename",
+      help=("The name of the file where the report log is "
+            "stored."
+            "(default: %(default)s)"),
+      default="report.txt")
+  args = parser.parse_args()
+
+  upgrade = ast_edits.ASTCodeUpgrader(tf_upgrade_v2.TFAPIChangeSpec())
+  report_text = None
+  report_filename = args.report_filename
+  files_processed = 0
+  if args.input_file:
+    if not args.output_file:
+      raise ValueError(
+          "--outfile=<output file> argument is required when converting a "
+          "single file.")
+    files_processed, report_text, errors = upgrade.process_file(
+        args.input_file, args.output_file)
+    files_processed = 1
+  elif args.input_tree:
+    if not args.output_tree:
+      raise ValueError(
+          "--outtree=<output directory> argument is required when converting a "
+          "file tree.")
+    files_processed, report_text, errors = upgrade.process_tree(
+        args.input_tree, args.output_tree, args.copy_other_files)
+  else:
+    parser.print_help()
+  if report_text:
+    open(report_filename, "w").write(report_text)
+    print("TensorFlow 2.0 Upgrade Script")
+    print("-----------------------------")
+    print("Converted %d files\n" % files_processed)
+    print("Detected %d errors that require attention" % len(errors))
+    print("-" * 80)
+    print("\n".join(errors))
+    print("\nMake sure to read the detailed log %r\n" % report_filename)
diff --git a/tensorflow/tools/compatibility/update/BUILD b/tensorflow/tools/compatibility/update/BUILD
index 0ee4550815..b9725a74ee 100644
--- a/tensorflow/tools/compatibility/update/BUILD
+++ b/tensorflow/tools/compatibility/update/BUILD
@@ -12,5 +12,6 @@ py_binary(
         "//tensorflow/python:no_contrib",
         "//tensorflow/tools/common:public_api",
         "//tensorflow/tools/common:traverse",
+        "//tensorflow/tools/compatibility:tf_upgrade_v2_lib",
     ],
 )
diff --git a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
index 949946c827..554b42c171 100644
--- a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
+++ b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
@@ -32,6 +32,7 @@ from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_export
 from tensorflow.tools.common import public_api
 from tensorflow.tools.common import traverse
+from tensorflow.tools.compatibility import tf_upgrade_v2
 
 
 _OUTPUT_FILE_PATH = 'third_party/tensorflow/tools/compatibility/renames_v2.py'
@@ -102,7 +103,7 @@ def collect_constant_renames():
   """Looks for constants that need to be renamed in TF 2.0.
 
   Returns:
-    List of tuples of the form (current name, new name).
+    Set of tuples of the form (current name, new name).
   """
   renames = set()
   for module in sys.modules.values():
@@ -135,7 +136,7 @@ def collect_function_renames():
   """Looks for functions/classes that need to be renamed in TF 2.0.
 
   Returns:
-    List of tuples of the form (current name, new name).
+    Set of tuples of the form (current name, new name).
   """
   # Set of rename lines to write to output file in the form:
   #   'tf.deprecated_name': 'tf.canonical_name'
@@ -181,12 +182,15 @@ def update_renames_v2(output_file_path):
   function_renames = collect_function_renames()
   constant_renames = collect_constant_renames()
   all_renames = function_renames.union(constant_renames)
+  manual_renames = set(
+      tf_upgrade_v2.TFAPIChangeSpec().manual_symbol_renames.keys())
 
   # List of rename lines to write to output file in the form:
   #   'tf.deprecated_name': 'tf.canonical_name'
   rename_lines = [
       get_rename_line(name, canonical_name)
-      for name, canonical_name in all_renames]
+      for name, canonical_name in all_renames
+      if 'tf.' + name not in manual_renames]
   renames_file_text = '%srenames = {\n%s\n}\n' % (
       _FILE_HEADER, ',\n'.join(sorted(rename_lines)))
   file_io.write_string_to_file(output_file_path, renames_file_text)
-- 
GitLab


From 672840616bd8b65ed9089ae687a714abfd87f63e Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 28 Nov 2018 15:17:45 -0800
Subject: [PATCH 0963/1554] Remove unused imports

---
 .../tensorrt/test/binary_tensor_weight_broadcast_test.py        | 1 -
 .../tensorrt/test/multi_connection_neighbor_engine_test.py      | 2 --
 2 files changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 5ab94cd931..6840ed20d1 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 468b2c96e1..8587fea6db 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -25,8 +25,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_math_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
 
-- 
GitLab


From 02a69ef199e2ed05a3498d3ee738bd493fe48b77 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 15:18:56 -0800
Subject: [PATCH 0964/1554] Test to show how constant 1D gather crashes toco.

PiperOrigin-RevId: 223245378
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/testing/generate_examples.py  | 32 +++++++++++++++++++
 .../resolve_constant_gather.cc                |  4 +++
 3 files changed, 37 insertions(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 5d615f0c5a..4a49c14cc9 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -242,6 +242,7 @@ def generated_test_models():
         "fully_connected",
         "fused_batch_norm",
         "gather",
+        "gather_buggy",
         "global_batch_norm",
         "greater",
         "greater_equal",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index b143f45b32..71382edaba 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -105,6 +105,8 @@ KNOWN_BUGS = {
     r"div.*int32": "72051395",
     # No support for SplitV
     r"split.*num_or_size_splits=\[2,2\]": "73377559",
+    # Constant 1D gather crashes toco.
+    r"gather_buggy.*input_shape=\[3\].*": "120029508",
 }
 
 
@@ -1233,6 +1235,36 @@ def make_gather_tests(zip_path):
       expected_tf_success=60)
 
 
+def make_gather_buggy_tests(zip_path):
+  """Make a set of tests to show gather crashes toco."""
+
+  test_parameters = [{
+      "input_shape": [[3]],
+      "reference_shape": [[2]],
+  }, {
+      "input_shape": [[2, 3]],
+      "reference_shape": [[2, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build a graph where the inputs to Gather are constants."""
+    reference = tf.placeholder(
+        dtype=tf.int32, shape=parameters["reference_shape"])
+    gather_input = tf.constant(
+        create_tensor_data(tf.int32, parameters["input_shape"]))
+    gather_indices = tf.constant([0, 1], tf.int32)
+    out = tf.equal(reference, tf.gather(gather_input, gather_indices))
+    return [reference], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    reference_values = np.zeros(parameters["reference_shape"], dtype=np.int32)
+    return [reference_values], sess.run(
+        outputs, feed_dict={inputs[0]: reference_values})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
+                    expected_tf_success=2)
+
+
 def make_global_batch_norm_tests(zip_path):
   """Make a set of tests to do batch_norm_with_global_normalization."""
 
diff --git a/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc b/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc
index 1149930131..c72135923e 100644
--- a/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc
+++ b/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc
@@ -47,6 +47,10 @@ inline void Gather(const Array& input_array, int input_rank,
     stride *= input_shape.dims(i);
   }
 
+  // Let's make sure we have enough space for all element in the memcpy()
+  // below, which writes 'stride' elements startng at 'i * stride'.
+  CHECK_EQ(stride * coords_shape.dims(0), output_data.size());
+
   for (int i = 0; i < coords_shape.dims(0); ++i) {
     DCHECK_GE(coords_data[i], 0);
     DCHECK_LT(coords_data[i], input_shape.dims(rev_input_rank));
-- 
GitLab


From 029389e0c69919392bd9a0d070d0d58498b0fb1d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 28 Nov 2018 15:20:44 -0800
Subject: [PATCH 0965/1554] Update Eigen to commit 135398e50bed.

PiperOrigin-RevId: 223245685
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 30694b43e3..0ea5d68ea8 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "8fa7ba1af23f0320be05f4658061138d6eb8dd1f320669cbf305b3a034f9d1c2",
-        strip_prefix = "eigen-eigen-ea671884cc96",
+        sha256 = "6e505fa8bf8d234d0338679b390cb89f850d870214c751b01a5db7f647e4d438",
+        strip_prefix = "eigen-eigen-135398e50bed",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/ea671884cc96.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/135398e50bed.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/135398e50bed.tar.gz",
         ],
     )
 
-- 
GitLab


From b8759f4a5fb718d925f82a9aaf601d786a7419ca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 15:23:27 -0800
Subject: [PATCH 0966/1554] Switch to new tf.distribute.Strategy reduce API. No
 longer allow a list as a reduce or broadcast destination, since it isn't used
 outside of tests and will make future support for model parallelism more
 tricky.

PiperOrigin-RevId: 223246154
---
 .../collective_all_reduce_strategy_test.py    |  7 +--
 .../python/cross_device_ops_test.py           | 10 ++---
 .../distribute/python/minimize_loss_test.py   | 11 +++--
 .../python/mirrored_strategy_multigpu_test.py | 32 +-------------
 .../python/parameter_server_strategy_test.py  |  2 +-
 .../distribute/python/strategy_test_lib.py    |  6 ++-
 .../python/distribute/distribute_lib.py       | 44 +++++++++++++------
 tensorflow/python/distribute/values.py        |  8 ++--
 .../engine/distributed_training_utils.py      |  7 +--
 tensorflow/python/training/moving_averages.py |  3 +-
 .../v1/tensorflow.distribute.-strategy.pbtxt  |  2 +-
 .../v2/tensorflow.distribute.-strategy.pbtxt  |  2 +-
 12 files changed, 56 insertions(+), 78 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 09239ffc72..eba3585a55 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -131,7 +131,7 @@ class CollectiveAllReduceStrategyTestBase(
           before_list.append(fetched)
           with ops.control_dependencies([fetched]):
             # TODO(yuefengz): support non-Mirrored variable as destinations.
-            g = d.reduce(
+            g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
                 d.update(v, update, g, grouped=False)):
@@ -225,10 +225,7 @@ class CollectiveAllReduceStrategyTestBase(
         return array_ops.identity(x)
 
       x = distribution.call_for_each_replica(model_fn)
-      reduced_x = distribution.unwrap(
-          distribution.reduce(
-              reduce_util.ReduceOp.MEAN, x,
-              destinations='/cpu:0'))[0]
+      reduced_x = distribution.reduce(reduce_util.ReduceOp.MEAN, x)
       x = distribution.unwrap(x)[0]
 
       sess.run(variables.global_variables_initializer())
diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 5d8690beb5..3602cc9209 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -132,11 +132,9 @@ class CrossDeviceOpsTestBase(test.TestCase, parameterized.TestCase):
     destination_mirrored = _fake_mirrored(1., devices)
     destination_different = _fake_mirrored(1., _cpu_device)
     destination_str = _cpu_device
-    destination_list = devices
 
     all_destinations = [
         destination_mirrored, destination_different, destination_str,
-        destination_list
     ]
 
     # test reduce()
@@ -320,10 +318,10 @@ class SingleWorkerCrossDeviceOpsTest(CrossDeviceOpsTestBase):
 
     if batch_reduce:
       result = cross_device_ops_instance.batch_reduce(
-          reduce_op, [(per_replica, devices)])
+          reduce_op, [(per_replica, per_replica)])
     else:
       result = cross_device_ops_instance.reduce(
-          reduce_op, per_replica, devices)
+          reduce_op, per_replica, per_replica)
 
     total_indices_with_dups = [1, 1, 3]
     total_indices_without_dups = [1, 3]
@@ -510,11 +508,9 @@ class MultiWorkerCollectiveAllReduceTest(
       destination_mirrored = _fake_mirrored(1., devices)
       destination_different = _fake_mirrored(1., _cpu_device)
       destination_str = _cpu_device
-      destination_list = devices
 
       all_destinations = [
-          destination_different, destination_mirrored, destination_str,
-          destination_list
+          destination_different, destination_mirrored, destination_str
       ]
 
       # test reduce()
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 129b394bb6..dcc9df4cda 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -486,12 +486,11 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
     if not reduced:
       self.assertLen(distribution.unwrap(loss_output),
                      distribution.num_replicas_in_sync)
-      loss_output = distribution.reduce(
-          reduce_util.ReduceOp.MEAN, loss_output, destinations="/device:CPU:0")
-
-    unwrapped_output = distribution.unwrap(loss_output)
-    self.assertLen(unwrapped_output, 1)
-    loss_tensor = unwrapped_output[0]
+      loss_tensor = distribution.reduce(reduce_util.ReduceOp.MEAN, loss_output)
+    else:
+      unwrapped_output = distribution.unwrap(loss_output)
+      self.assertLen(unwrapped_output, 1)
+      loss_tensor = unwrapped_output[0]
     self.assertEqual(initial_loss.dtype, loss_tensor.dtype)
     self.assertEqual(initial_loss.shape, loss_tensor.shape)
 
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index b304f63501..fee37daa42 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -96,14 +96,9 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
   def testReduceToCpu(self, distribution):
     with distribution.scope():
       result = distribution.extended.call_for_each_replica(_replica_id)
-      reduced = distribution.reduce(
-          reduce_util.ReduceOp.SUM,
-          result,
-          destinations="/device:CPU:0")
-      unwrapped = distribution.unwrap(reduced)
-      self.assertEqual(1, len(unwrapped))
+      reduced = distribution.reduce(reduce_util.ReduceOp.SUM, result)
       expected = sum(range(distribution.num_replicas_in_sync))
-      self.assertEqual(expected, self.evaluate(unwrapped[0]))
+      self.assertEqual(expected, self.evaluate(reduced))
 
   def testMakeInputFnIterator(self, distribution):
     dataset_fn = lambda: dataset_ops.Dataset.range(10)
@@ -136,29 +131,6 @@ class MirroredOneDeviceDistributionTest(
     strategy_test_lib.DistributionTestBase,
     parameterized.TestCase):
 
-  @combinations.generate(combinations.combine(
-      distribution=[
-          combinations.NamedDistribution(
-              "Mirrored1CPU",
-              lambda: mirrored_strategy.MirroredStrategy(["/device:CPU:0"]),
-              required_gpus=1),
-          combinations.mirrored_strategy_with_one_gpu,
-          combinations.NamedDistribution(
-              "CoreMirrored1CPU",
-              lambda: mirrored_strategy.CoreMirroredStrategy(["/device:CPU:0"]),
-              required_gpus=1),
-          combinations.core_mirrored_strategy_with_one_gpu],
-      mode=["graph", "eager"]))
-  def testReduceToMultipleDestinations(self, distribution):
-    with distribution.scope():
-      reduced = distribution.extended.reduce_to(
-          reduce_util.ReduceOp.SUM,
-          1.0,
-          destinations=["/device:CPU:0", "/device:GPU:0"])
-      unwrapped = distribution.unwrap(reduced)
-      self.assertLen(unwrapped, 2)
-      self.assertEqual(1.0, self.evaluate(unwrapped[0]))
-
   @combinations.generate(one_device_combinations())
   def testMinimizeLoss(self, distribution):
     if context.executing_eagerly():
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 4debe72ca6..83d7473666 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -481,7 +481,7 @@ class ParameterServerStrategyTestBase(
           before_list.append(fetched)
           with ops.control_dependencies([fetched]):
             # TODO(yuefengz): support non-Mirrored variable as destinations.
-            g = d.reduce(
+            g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
                 d.update(v, update, g, grouped=False)):
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 756e5bdc1e..d50b142c5e 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -116,7 +116,8 @@ class DistributionTestBase(test.TestCase):
           before_list.append(fetched)
           # control_dependencies irrelevant but harmless in eager execution
           with ops.control_dependencies([fetched]):
-            g = d.reduce(reduce_util.ReduceOp.SUM, g, destinations=v)
+            g = d.extended.reduce_to(
+                reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(d.update(
                 v, update, g, grouped=False)):
               after_list.append(d.read_var(v))
@@ -170,7 +171,8 @@ class DistributionTestBase(test.TestCase):
           fetched = d.read_var(v)
           before_list.append(fetched)
           with ops.control_dependencies([fetched]):
-            g = d.reduce(reduce_util.ReduceOp.SUM, g, destinations=v)
+            g = d.extended.reduce_to(
+                reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(d.update(
                 v, update, g, grouped=False)):
               after_list.append(d.read_var(v))
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index a1f03eab61..2213499be0 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -489,10 +489,19 @@ class DistributionStrategy(object):
     kwargs.pop("run_concurrently", None)  # Ignore old option.
     return self._extended.call_for_each_replica(fn, args, kwargs)
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def reduce(self, aggregation, value, destinations):
-    """DEPRECATED: use extended.reduce_to() instead."""
-    return self._extended.reduce_to(aggregation, value, destinations)
+  def reduce(self, reduce_op, value):
+    """Reduce `value` across replicas.
+
+    Args:
+      reduce_op: A `tf.distribute.ReduceOp` value specifying how values should
+        be combined.
+      value: A "per replica" value to be combined into a single tensor.
+
+    Returns:
+      A `Tensor`.
+    """
+    _require_cross_replica_context_extended(self._extended)
+    return self._extended._reduce(reduce_op, value)  # pylint: disable=protected-access
 
   @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
   def batch_reduce(self, aggregation, value_destination_pairs):
@@ -1039,14 +1048,15 @@ class DistributionStrategyExtended(object):
 
     Args:
       tensor: A Tensor value to broadcast.
-      destinations: A mirrored variable, device string, or list of device
-        strings, specifying the destination devices to copy `tensor` to.
+      destinations: A mirrored variable or device string specifying the
+        destination devices to copy `tensor` to.
 
     Returns:
       A value mirrored to `destinations` devices.
     """
     # TODO(josh11b): More docstring
     _require_cross_replica_context_extended(self)
+    assert not isinstance(destinations, (list, tuple))
     return self._broadcast_to(tensor, destinations)
 
   def _broadcast_to(self, tensor, destinations):
@@ -1154,6 +1164,11 @@ class DistributionStrategyExtended(object):
   def _call_for_each_replica(self, fn, args, kwargs):
     raise NotImplementedError("must be implemented in descendants")
 
+  def _reduce(self, reduce_op, value):
+    # Default implementation until we have an implementation for each strategy.
+    return self._unwrap(self._reduce_to(
+        reduce_op, value, device_util.current() or "/device:CPU:0"))[0]
+
   def reduce_to(self, reduce_op, value, destinations):
     """Combine (via e.g. sum or mean) values across replicas.
 
@@ -1163,26 +1178,27 @@ class DistributionStrategyExtended(object):
         `tf.VariableAggregation.SUM`,
         `tf.VariableAggregation.MEAN`,
       value: A per-replica value with one value per replica.
-      destinations: A mirrored variable, a per-replica tensor, a device string,
-        or list of device strings. The return value will be copied to all
-        destination devices (or all the devices where the `destinations` value
-        resides). To perform an all-reduction, pass `value` to `destinations`.
+      destinations: A mirrored variable, a per-replica tensor, or a device
+        string. The return value will be copied to all destination devices (or
+        all the devices where the `destinations` value resides). To perform an
+        all-reduction, pass `value` to `destinations`.
 
     Returns:
       A value mirrored to `destinations`.
     """
     # TODO(josh11b): More docstring
-    # TODO(josh11b): Return an unwrapped value if colocate_with is a
-    # single device.
     _require_cross_replica_context_extended(self)
+    assert not isinstance(destinations, (list, tuple))
 
     # TODO(priyag): Remove this when all callers have been updated.
     if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in [
+      assert reduce_op in (
           variable_scope.VariableAggregation.SUM,
           variable_scope.VariableAggregation.MEAN,
-      ]
+      )
       reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    assert (reduce_op == reduce_util.ReduceOp.SUM or
+            reduce_op == reduce_util.ReduceOp.MEAN)
     return self._reduce_to(reduce_op, value, destinations)
 
   def _reduce_to(self, reduce_op, value, destinations):
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 7dd1062e38..f3f7676f04 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -324,7 +324,7 @@ def _apply_aggregation(strategy, value, aggregation, destinations):
     return strategy.broadcast(strategy.unwrap(value)[0],
                               destinations=destinations)
   reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
-  return strategy.reduce(reduce_op, value=value, destinations=destinations)
+  return strategy.extended.reduce_to(reduce_op, value, destinations)
 
 
 class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable):
@@ -1672,13 +1672,11 @@ class MultiStepContext(object):
         self._last_step_outputs[name] = output
       else:
         distribution = distribution_strategy_context.get_distribution_strategy()
-        self._last_step_outputs[name] = distribution.reduce(
-            reduce_op, output, destinations="/device:CPU:0")
+        self._last_step_outputs[name] = distribution.reduce(reduce_op, output)
     else:
       assert reduce_op is not None
       def merge_fn(distribution, value):
-        self._last_step_outputs[name] = distribution.reduce(
-            reduce_op, value, destinations="/device:CPU:0")
+        self._last_step_outputs[name] = distribution.reduce(reduce_op, value)
         # Setting this inside the `merge_fn` because all replicas share the same
         # context object, so it's more robust to set it only once (even if all
         # the replicas are trying to set the same value).
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 8b0076163e..cd8e0342cd 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -94,11 +94,8 @@ def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs,
                                         grouped_inputs)
   if with_loss_tensor:
     # reduce loss tensor before adding it to the list of fetches
-    loss = distribution_strategy.unwrap(
-        distribution_strategy.reduce(distribute_lib.get_loss_reduction(),
-                                     grouped_outputs[0],
-                                     destinations='/device:CPU:0'))[0]
-
+    loss = distribution_strategy.reduce(distribute_lib.get_loss_reduction(),
+                                        grouped_outputs[0])
     all_outputs = flatten_perdevice_values(distribution_strategy,
                                            grouped_outputs[1:])
     all_outputs = [loss] + all_outputs
diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 9b5449498b..8785f9a8e7 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -96,7 +96,8 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None):
       # In a replica context, we update variable using the mean of value across
       # replicas.
       def merge_fn(strategy, v, value):
-        value = strategy.reduce(ds_reduce_util.ReduceOp.MEAN, value, v)
+        value = strategy.extended.reduce_to(
+            ds_reduce_util.ReduceOp.MEAN, value, v)
         return strategy.update(v, update_fn, value)
 
       return replica_context.merge_call(merge_fn, args=(variable, value))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 0fd9a3b42d..9eb73d2c0d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "run_steps_on_dataset"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 0fd9a3b42d..9eb73d2c0d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value\', \'destinations\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "run_steps_on_dataset"
-- 
GitLab


From dde79c2ef338414c3ebdb62c0cd74821ea2ad089 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 28 Nov 2018 15:43:24 -0800
Subject: [PATCH 0967/1554] Remove more symbols from TF 2.0 API.

- train.CheckpointSaverListener
- train.NanLossDuringTrainingError
- train.Scaffold
- train.SecondOrStepTimer
- train.SessionRunArgs
- train.SessionRunContext
- train.SessionRunValues

PiperOrigin-RevId: 223249504
---
 .../training/basic_session_run_hooks.py       |  6 +--
 .../python/training/monitored_session.py      |  2 +-
 .../python/training/session_run_hook.py       |  6 +--
 ...low.train.-checkpoint-saver-listener.pbtxt | 24 ---------
 ...rain.-nan-loss-during-training-error.pbtxt | 12 -----
 .../v2/tensorflow.train.-scaffold.pbtxt       | 53 -------------------
 ...nsorflow.train.-second-or-step-timer.pbtxt | 26 ---------
 .../tensorflow.train.-session-run-args.pbtxt  | 27 ----------
 ...ensorflow.train.-session-run-context.pbtxt | 25 ---------
 ...tensorflow.train.-session-run-values.pbtxt | 27 ----------
 .../api/golden/v2/tensorflow.train.pbtxt      | 28 ----------
 tensorflow/tools/compatibility/renames_v2.py  |  7 +++
 12 files changed, 14 insertions(+), 229 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt

diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py
index 60db654e9c..b64c7ada62 100644
--- a/tensorflow/python/training/basic_session_run_hooks.py
+++ b/tensorflow/python/training/basic_session_run_hooks.py
@@ -83,7 +83,7 @@ class _HookTimer(object):
     raise NotImplementedError
 
 
-@tf_export("train.SecondOrStepTimer")
+@tf_export(v1=["train.SecondOrStepTimer"])
 class SecondOrStepTimer(_HookTimer):
   """Timer that triggers at most once every N seconds or once every N steps.
   """
@@ -429,7 +429,7 @@ class StopAtStepHook(session_run_hook.SessionRunHook):
         run_context.request_stop()
 
 
-@tf_export("train.CheckpointSaverListener")
+@tf_export(v1=["train.CheckpointSaverListener"])
 class CheckpointSaverListener(object):
   """Interface for listeners that take action before or after checkpoint save.
 
@@ -718,7 +718,7 @@ class StepCounterHook(session_run_hook.SessionRunHook):
     self._last_global_step = stale_global_step
 
 
-@tf_export("train.NanLossDuringTrainingError")
+@tf_export(v1=["train.NanLossDuringTrainingError"])
 class NanLossDuringTrainingError(RuntimeError):
 
   def __str__(self):
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index c40bd2b859..6a7d27df5c 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -54,7 +54,7 @@ _PREEMPTION_ERRORS = (errors.AbortedError, errors.UnavailableError)
 USE_DEFAULT = object()
 
 
-@tf_export('train.Scaffold')
+@tf_export(v1=['train.Scaffold'])
 class Scaffold(object):
   """Structure to create or gather pieces commonly needed to train a model.
 
diff --git a/tensorflow/python/training/session_run_hook.py b/tensorflow/python/training/session_run_hook.py
index 5daea93128..e9a61def74 100644
--- a/tensorflow/python/training/session_run_hook.py
+++ b/tensorflow/python/training/session_run_hook.py
@@ -186,7 +186,7 @@ class SessionRunHook(object):
     pass
 
 
-@tf_export("train.SessionRunArgs")
+@tf_export(v1=["train.SessionRunArgs"])
 class SessionRunArgs(
     collections.namedtuple("SessionRunArgs",
                            ["fetches", "feed_dict", "options"])):
@@ -211,7 +211,7 @@ class SessionRunArgs(
     return super(SessionRunArgs, cls).__new__(cls, fetches, feed_dict, options)
 
 
-@tf_export("train.SessionRunContext")
+@tf_export(v1=["train.SessionRunContext"])
 class SessionRunContext(object):
   """Provides information about the `session.run()` call being made.
 
@@ -263,7 +263,7 @@ class SessionRunContext(object):
     self._stop_requested = True
 
 
-@tf_export("train.SessionRunValues")
+@tf_export(v1=["train.SessionRunValues"])
 class SessionRunValues(
     collections.namedtuple("SessionRunValues",
                            ["results", "options", "run_metadata"])):
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt
deleted file mode 100644
index 9d3688e565..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-listener.pbtxt
+++ /dev/null
@@ -1,24 +0,0 @@
-path: "tensorflow.train.CheckpointSaverListener"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.CheckpointSaverListener\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "after_save"
-    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "before_save"
-    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "begin"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "end"
-    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt
deleted file mode 100644
index e415819b3d..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-loss-during-training-error.pbtxt
+++ /dev/null
@@ -1,12 +0,0 @@
-path: "tensorflow.train.NanLossDuringTrainingError"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.NanLossDuringTrainingError\'>"
-  is_instance: "<type \'exceptions.RuntimeError\'>"
-  member {
-    name: "args"
-    mtype: "<type \'getset_descriptor\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt
deleted file mode 100644
index 38cc98b48e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-scaffold.pbtxt
+++ /dev/null
@@ -1,53 +0,0 @@
-path: "tensorflow.train.Scaffold"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.monitored_session.Scaffold\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "init_feed_dict"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "init_fn"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "init_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "local_init_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "ready_for_local_init_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "ready_op"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "saver"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "summary_op"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'init_op\', \'init_feed_dict\', \'init_fn\', \'ready_op\', \'ready_for_local_init_op\', \'local_init_op\', \'summary_op\', \'saver\', \'copy_from_scaffold\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "default_local_init_op"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_or_default"
-    argspec: "args=[\'arg_name\', \'collection_key\', \'default_constructor\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt
deleted file mode 100644
index 3c5a6ac13c..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-second-or-step-timer.pbtxt
+++ /dev/null
@@ -1,26 +0,0 @@
-path: "tensorflow.train.SecondOrStepTimer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.SecondOrStepTimer\'>"
-  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks._HookTimer\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'every_secs\', \'every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "last_triggered_step"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "should_trigger_for_step"
-    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_last_triggered_step"
-    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt
deleted file mode 100644
index 442990893e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-args.pbtxt
+++ /dev/null
@@ -1,27 +0,0 @@
-path: "tensorflow.train.SessionRunArgs"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunArgs\'>"
-  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunArgs\'>"
-  is_instance: "<type \'tuple\'>"
-  member {
-    name: "feed_dict"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "fetches"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "options"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "count"
-  }
-  member_method {
-    name: "index"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt
deleted file mode 100644
index d5adb15c95..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-context.pbtxt
+++ /dev/null
@@ -1,25 +0,0 @@
-path: "tensorflow.train.SessionRunContext"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunContext\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "original_args"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "session"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "stop_requested"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'original_args\', \'session\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "request_stop"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt
deleted file mode 100644
index 0b401d59c4..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-session-run-values.pbtxt
+++ /dev/null
@@ -1,27 +0,0 @@
-path: "tensorflow.train.SessionRunValues"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunValues\'>"
-  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunValues\'>"
-  is_instance: "<type \'tuple\'>"
-  member {
-    name: "options"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "results"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "run_metadata"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "count"
-  }
-  member_method {
-    name: "index"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index a30f67350a..2cc3b7ee1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -16,10 +16,6 @@ tf_module {
     name: "CheckpointSaverHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "CheckpointSaverListener"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ClusterDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -84,10 +80,6 @@ tf_module {
     name: "LoggingTensorHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "NanLossDuringTrainingError"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "NanTensorHook"
     mtype: "<type \'type\'>"
@@ -96,14 +88,6 @@ tf_module {
     name: "ProximalGradientDescentOptimizer"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Scaffold"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "SecondOrStepTimer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "SequenceExample"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -116,22 +100,10 @@ tf_module {
     name: "ServerDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "SessionRunArgs"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "SessionRunContext"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "SessionRunHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "SessionRunValues"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "StepCounterHook"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 8dbeb078b6..ae57478473 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -573,6 +573,7 @@ renames = {
     'tf.train.AdagradDAOptimizer': 'tf.compat.v1.train.AdagradDAOptimizer',
     'tf.train.AdagradOptimizer': 'tf.compat.v1.train.AdagradOptimizer',
     'tf.train.AdamOptimizer': 'tf.compat.v1.train.AdamOptimizer',
+    'tf.train.CheckpointSaverListener': 'tf.compat.v1.train.CheckpointSaverListener',
     'tf.train.ChiefSessionCreator': 'tf.compat.v1.train.ChiefSessionCreator',
     'tf.train.FtrlOptimizer': 'tf.compat.v1.train.FtrlOptimizer',
     'tf.train.GradientDescentOptimizer': 'tf.compat.v1.train.GradientDescentOptimizer',
@@ -580,6 +581,7 @@ renames = {
     'tf.train.MomentumOptimizer': 'tf.compat.v1.train.MomentumOptimizer',
     'tf.train.MonitoredSession': 'tf.compat.v1.train.MonitoredSession',
     'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession',
+    'tf.train.NanLossDuringTrainingError': 'tf.compat.v1.train.NanLossDuringTrainingError',
     'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader',
     'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer',
     'tf.train.ProfilerHook': 'tf.compat.v1.train.ProfilerHook',
@@ -588,8 +590,13 @@ renames = {
     'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
     'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef',
+    'tf.train.Scaffold': 'tf.compat.v1.train.Scaffold',
+    'tf.train.SecondOrStepTimer': 'tf.compat.v1.train.SecondOrStepTimer',
     'tf.train.SessionCreator': 'tf.compat.v1.train.SessionCreator',
     'tf.train.SessionManager': 'tf.compat.v1.train.SessionManager',
+    'tf.train.SessionRunArgs': 'tf.compat.v1.train.SessionRunArgs',
+    'tf.train.SessionRunContext': 'tf.compat.v1.train.SessionRunContext',
+    'tf.train.SessionRunValues': 'tf.compat.v1.train.SessionRunValues',
     'tf.train.SingularMonitoredSession': 'tf.compat.v1.train.SingularMonitoredSession',
     'tf.train.Supervisor': 'tf.compat.v1.train.Supervisor',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
-- 
GitLab


From 9831eb8bbfd5226db87dad087e509eae0af07e9a Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 28 Nov 2018 15:52:59 -0800
Subject: [PATCH 0968/1554] Factor out some Operation mutations from Function
 into Operation.

These mutations are used to rewrite a forward function call to output its
intermediates when creating a gradient call. This change is in preparation
for making cond_v2 and while_v2 do the same thing, instead of outputing
all intermediates from the forward call when its created.

PiperOrigin-RevId: 223250964
---
 tensorflow/python/eager/function.py | 20 ++++++----------
 tensorflow/python/framework/ops.py  | 36 +++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 68cdb1a871..d9c16aa7b8 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -467,20 +467,14 @@ class Function(object):
 
     # Rewrite an inference call op to be a forward call op
     if op.get_attr("f").name.encode() == self._inference_function.name:
-      func = attr_value_pb2.AttrValue(
-          func=attr_value_pb2.NameAttrList(
-              name=self._forward_function.name))
-      op._set_attr("f", func)
-      types = attr_value_pb2.AttrValue.ListValue(
-          type=self._forward_function._output_types)
-      op._set_attr("Tout", attr_value_pb2.AttrValue(list=types))
-      for i in range(
-          num_inference_outputs, len(self._forward_function._output_types)):
-        t = ops.Tensor(op, i, self._forward_function._output_types[i])
-        t.set_shape(self._forward_function._output_shapes[i])
+      op._set_func_attr("f", self._forward_function.name)
+      op._set_type_list_attr("Tout", self._forward_function._output_types)
+      op._add_outputs(
+          self._forward_function._output_types[num_inference_outputs:],
+          self._forward_function._output_shapes[num_inference_outputs:])
+      for i in range(num_inference_outputs, len(op.outputs)):
         func_graph_output = self._forward_function._func_graph_outputs[i]
-        custom_gradient.copy_handle_data(func_graph_output, t)
-        op._outputs.append(t)
+        custom_gradient.copy_handle_data(func_graph_output, op.outputs[i])
     # pylint: enable=protected-access
     # Compute the gradients using the side outputs
     side_outputs = op.outputs[num_inference_outputs:]
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 5a8a2a48ab..fc692a2510 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2126,6 +2126,23 @@ class Operation(object):
     """Removes any control inputs to this operation."""
     c_api.RemoveAllControlInputs(self._graph._c_graph, self._c_op)  # pylint: disable=protected-access
 
+  def _add_outputs(self, types, shapes):
+    """Adds new Tensors to self.outputs.
+
+    Note: this is generally unsafe to use. This is used in certain situations in
+    conjunction with _set_type_list_attr.
+
+    Arguments:
+      types: list of DTypes
+      shapes: list of TensorShapes
+    """
+    assert len(types) == len(shapes)
+    orig_num_outputs = len(self.outputs)
+    for i in range(len(types)):
+      t = Tensor(self, orig_num_outputs + i, types[i])
+      self._outputs.append(t)
+      t.set_shape(shapes[i])
+
   def __str__(self):
     return str(self.node_def)
 
@@ -2338,6 +2355,25 @@ class Operation(object):
     finally:
       c_api.TF_DeleteBuffer(buf)
 
+  def _set_func_attr(self, attr_name, func_name):
+    """Private method used to set a function attribute in the node_def."""
+    func = attr_value_pb2.NameAttrList(name=func_name)
+    self._set_attr(attr_name, attr_value_pb2.AttrValue(func=func))
+
+  def _set_type_list_attr(self, attr_name, types):
+    """Private method used to set a function attribute in the node_def."""
+    if not types: return
+    if isinstance(types[0], dtypes.DType):
+      types = [dt.as_datatype_enum for dt in types]
+    types_list = attr_value_pb2.AttrValue.ListValue(type=types)
+    self._set_attr(attr_name, attr_value_pb2.AttrValue(list=types_list))
+
+  def _set_shape_list_attr(self, attr_name, shapes):
+    """Private method used to set a function attribute in the node_def."""
+    shapes = [s.as_proto() for s in shapes]
+    shapes_list = attr_value_pb2.AttrValue.ListValue(shape=shapes)
+    self._set_attr(attr_name, attr_value_pb2.AttrValue(list=shapes_list))
+
   def get_attr(self, name):
     """Returns the value of the attr of this op with the given `name`.
 
-- 
GitLab


From 0f98c067fa1aa361267c21a068a18820b35aed70 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 28 Nov 2018 16:00:07 -0800
Subject: [PATCH 0969/1554] Add publicly available corpus for string_to_number
 fuzz.

PiperOrigin-RevId: 223252037
---
 tensorflow/core/kernels/fuzzing/BUILD         |  3 +++
 .../2db83ea58639b6d7d585fa12e3947a82          |  1 +
 .../36b4a931886b941dc41180050d12ca94          |  1 +
 .../50a2fabfdd276f573ff97ace8b11c5f4          |  1 +
 .../62edb2a1eee34b001652cd86584becf2          |  1 +
 .../90013d1ec28c46a5c00574e60c70b6fc          |  1 +
 .../94f3e3cee6957ce5815326d6788c85f4          |  1 +
 .../96f547bc04bb913da0bc08915238ebd8          |  1 +
 .../d3a903d18fc11e1f35c572ad4da690ed          |  1 +
 .../e3b629c92af44260c189deb32d6f06f3          |  1 +
 .../f03eecf3bcfe4967a1888156a3115c8d          |  1 +
 .../fa54ca9186f77122ae2a82684a062e16          |  1 +
 .../fuzzing/tf_ops_fuzz_target_lib.bzl        | 26 ++++++++++++-------
 13 files changed, 30 insertions(+), 10 deletions(-)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16

diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index 6f3a49805c..193041241c 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -18,11 +18,14 @@ cc_library(
 )
 
 load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "tf_ops_fuzz_target_lib")
+load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "tf_oss_fuzz_corpus")
 
 tf_ops_fuzz_target_lib("identity")
 
 tf_ops_fuzz_target_lib("string_to_number")
 
+tf_oss_fuzz_corpus("string_to_number")
+
 tf_ops_fuzz_target_lib("string_split")
 
 tf_ops_fuzz_target_lib("string_split_v2")
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82
new file mode 100644
index 0000000000..3de80927d5
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/2db83ea58639b6d7d585fa12e3947a82
@@ -0,0 +1 @@
+6.023e+23
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94
new file mode 100644
index 0000000000..d531129b28
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/36b4a931886b941dc41180050d12ca94
@@ -0,0 +1 @@
+6.023e-23
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4
new file mode 100644
index 0000000000..d81cc0710e
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/50a2fabfdd276f573ff97ace8b11c5f4
@@ -0,0 +1 @@
+42
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2
new file mode 100644
index 0000000000..72f88139d0
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/62edb2a1eee34b001652cd86584becf2
@@ -0,0 +1 @@
+0xabcdef
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc
new file mode 100644
index 0000000000..c1113b83e8
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/90013d1ec28c46a5c00574e60c70b6fc
@@ -0,0 +1 @@
+3.14159265359
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4
new file mode 100644
index 0000000000..320aa3f00e
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/94f3e3cee6957ce5815326d6788c85f4
@@ -0,0 +1 @@
+0.69314718056
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8
new file mode 100644
index 0000000000..51b7b732f6
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/96f547bc04bb913da0bc08915238ebd8
@@ -0,0 +1 @@
+6.023e23
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed
new file mode 100644
index 0000000000..9a0be0764b
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/d3a903d18fc11e1f35c572ad4da690ed
@@ -0,0 +1 @@
+1.61803
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3
new file mode 100644
index 0000000000..6a0e60d48b
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/e3b629c92af44260c189deb32d6f06f3
@@ -0,0 +1 @@
+-42
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d
new file mode 100644
index 0000000000..ea9cd255bc
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/f03eecf3bcfe4967a1888156a3115c8d
@@ -0,0 +1 @@
+6.023E+23
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16 b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16
new file mode 100644
index 0000000000..00f1e2ed8f
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_to_number/fa54ca9186f77122ae2a82684a062e16
@@ -0,0 +1 @@
+2.71828182846
diff --git a/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl b/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl
index f752b59568..ed54abb742 100644
--- a/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl
+++ b/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl
@@ -1,13 +1,19 @@
 """Fuzzing template for TensorFlow ops."""
 
 def tf_ops_fuzz_target_lib(name):
-  native.cc_library(
-      name = name + "_fuzz_lib",
-      srcs = [name + "_fuzz.cc"],
-      deps = [
-          "//tensorflow/core/kernels/fuzzing:fuzz_session",
-          "//tensorflow/cc:cc_ops",
-      ],
-      tags = ["no_windows"],
-      alwayslink = 1,
-  )
+    native.cc_library(
+        name = name + "_fuzz_lib",
+        srcs = [name + "_fuzz.cc"],
+        deps = [
+            "//tensorflow/core/kernels/fuzzing:fuzz_session",
+            "//tensorflow/cc:cc_ops",
+        ],
+        tags = ["no_windows"],
+        alwayslink = 1,
+    )
+
+def tf_oss_fuzz_corpus(name):
+    native.filegroup(
+        name = name + "_corpus",
+        srcs = native.glob(["corpus/" + name + "/*"]),
+    )
-- 
GitLab


From bd737c846ca453ac06c0a2dadece7c57a1f1efec Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 28 Nov 2018 16:04:22 -0800
Subject: [PATCH 0970/1554] Replace Shape with a C++ class in XLA. No
 functional change. Rename the proto message Shape to ShapeProto and define an
 in-place replacement C++ class named Shape with an interface which mirrors
 the protobuf generated code interface. Having Shape as a C++ class enables
 greater flexibility in the interface, enables enforcement of invariants, and
 potential performance improvements.

PiperOrigin-RevId: 223252977
---
 tensorflow/compiler/aot/codegen.cc            |   9 +-
 tensorflow/compiler/aot/compile.cc            |  16 +-
 .../compiler/aot/tests/tfcompile_test.cc      |   8 +-
 tensorflow/compiler/tf2xla/shape_util.h       |   1 +
 .../xla_jit_compiled_cpu_function_test.cc     |  12 +-
 tensorflow/compiler/xla/client/BUILD          |   1 +
 tensorflow/compiler/xla/client/client.cc      |  18 +-
 .../xla/client/executable_build_options.h     |   1 +
 tensorflow/compiler/xla/client/lib/testing.cc |   6 +-
 .../compiler/xla/client/sharding_builder.cc   |   4 +-
 tensorflow/compiler/xla/client/xla_builder.cc | 294 ++++++++++--------
 tensorflow/compiler/xla/index_util.h          |   1 +
 tensorflow/compiler/xla/literal.cc            |  16 +-
 tensorflow/compiler/xla/literal_test.cc       |  53 ++--
 .../compiler/xla/python_api/xla_shape.py      |   7 +-
 tensorflow/compiler/xla/rpc/BUILD             |   1 -
 tensorflow/compiler/xla/rpc/xla_service.proto |   1 -
 .../xla/service/compile_only_service.cc       |   2 +-
 .../compiler/xla/service/cpu/xfeed_manager.h  |   1 +
 tensorflow/compiler/xla/service/hlo.proto     |   6 +-
 .../compiler/xla/service/hlo_instruction.cc   | 142 ++++-----
 .../compiler/xla/service/hlo_instructions.cc  |   4 +-
 tensorflow/compiler/xla/service/hlo_lexer.h   |   1 +
 tensorflow/compiler/xla/service/hlo_module.cc |   4 +-
 .../compiler/xla/service/hlo_proto_util.cc    |   9 +-
 .../compiler/xla/service/hlo_proto_util.h     |   5 +-
 .../compiler/xla/service/llvm_ir/ir_array.h   |   1 +
 .../compiler/xla/service/llvm_ir/llvm_util.cc |   7 +-
 .../compiler/xla/service/local_service.cc     |   8 +-
 tensorflow/compiler/xla/service/service.cc    |  36 ++-
 .../compiler/xla/service/shape_inference.cc   |   2 +-
 tensorflow/compiler/xla/shape.cc              |  55 +++-
 tensorflow/compiler/xla/shape.h               |  98 +++++-
 tensorflow/compiler/xla/shape_test.cc         |  63 +++-
 tensorflow/compiler/xla/shape_util.cc         |  34 +-
 tensorflow/compiler/xla/shape_util.h          |   6 +-
 tensorflow/compiler/xla/shape_util_test.cc    |  31 --
 .../xla/tests/client_library_test_base.cc     |   4 +-
 tensorflow/compiler/xla/tests/client_test.cc  |   6 +-
 tensorflow/compiler/xla/tests/reshape_test.cc |   6 +-
 .../compiler/xla/tools/replay_computation.cc  |  21 +-
 tensorflow/compiler/xla/util.h                |   7 +
 tensorflow/compiler/xla/xla.proto             |  14 +-
 tensorflow/compiler/xla/xla_data.proto        |  12 +-
 .../compiler/xrt/kernels/xrt_compile_ops.cc   |  11 +-
 tensorflow/compiler/xrt/tests/raw_api_test.cc |  67 ++--
 46 files changed, 670 insertions(+), 442 deletions(-)

diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc
index 697599f3bb..e0ac7130a6 100644
--- a/tensorflow/compiler/aot/codegen.cc
+++ b/tensorflow/compiler/aot/codegen.cc
@@ -175,7 +175,8 @@ Status GenArgMethods(const tf2xla::Config& config,
   }
   for (int i = 0; i < num_args; ++i) {
     std::vector<std::pair<string, string>> rewrites;
-    TF_RETURN_IF_ERROR(AddRewritesForShape(i, ps.parameters(i), &rewrites));
+    TF_RETURN_IF_ERROR(
+        AddRewritesForShape(i, xla::Shape(ps.parameters(i)), &rewrites));
     const string code = R"(
   void set_arg{{NAME}}_data(void* data) {
     set_arg_data({{I}}, data);
@@ -218,8 +219,8 @@ Status GenResultMethods(const tf2xla::Config& config,
   }
   for (int i = 0; i < ps.result().tuple_shapes_size(); ++i) {
     std::vector<std::pair<string, string>> rewrites;
-    TF_RETURN_IF_ERROR(
-        AddRewritesForShape(i, ps.result().tuple_shapes(i), &rewrites));
+    TF_RETURN_IF_ERROR(AddRewritesForShape(
+        i, xla::Shape(ps.result().tuple_shapes(i)), &rewrites));
     string code = R"(
   {{TYPE}}* result{{NAME}}_data() {
     return static_cast<{{TYPE}}*>(result_data({{I}}));
@@ -588,7 +589,7 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction {
       {"{{METHODS_RESULT}}\n", methods_result},
       {"{{NS_END}}\n", ns_end},
       {"{{NS_START}}\n", ns_start},
-      {"{{PROGRAM_SHAPE}}", xla::ShapeUtil::HumanString(ps)},
+      {"{{PROGRAM_SHAPE}}", xla::ShapeUtil::HumanString(xla::ProgramShape(ps))},
       {"{{PROGRAM_SHAPE_SHIM_EXPRESSION}}",
        metadata_result.program_shape_access_shim},
       {"{{RESULT_INDEX}}", absl::StrCat(result_index)},
diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index 3bc99ef7e6..9fc223bdc7 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -58,15 +58,21 @@ Status CompileXla(xla::CompileOnlyClient* client,
   }
   compile_result->program_shape = pshape_or.ValueOrDie()->ToProto();
   xla::ProgramShapeProto* pshape = &compile_result->program_shape;
-  std::vector<const xla::Shape*> arg_layouts;
-  arg_layouts.reserve(pshape->parameters_size());
+
+  // AotXlaComputationInstance::argument_layouts is a vector of Shape
+  // pointers. Accumulate the Shape objects themselves in a separate vector
+  // while building the vector of pointers.
+  std::vector<const xla::Shape*> arg_layout_ptrs(pshape->parameters_size());
+  std::vector<xla::Shape> arg_layouts(pshape->parameters_size());
   for (int i = 0; i < pshape->parameters_size(); ++i) {
-    arg_layouts.push_back(pshape->mutable_parameters(i));
+    arg_layouts[i] = xla::Shape(*pshape->mutable_parameters(i));
+    arg_layout_ptrs[i] = &arg_layouts[i];
   }
   xla::CompileOnlyClient::AotXlaComputationInstance instance;
   instance.computation = &computation;
-  instance.argument_layouts = std::move(arg_layouts);
-  instance.result_layout = &pshape->result();
+  instance.argument_layouts = std::move(arg_layout_ptrs);
+  xla::Shape result_shape(pshape->result());
+  instance.result_layout = &result_shape;
   xla::StatusOr<std::vector<std::unique_ptr<xla::AotCompilationResult>>>
       aot_or = client->CompileAheadOfTime({instance}, aot_opts);
   if (!aot_or.ok()) {
diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc
index 711feed8f3..4dd79e5882 100644
--- a/tensorflow/compiler/aot/tests/tfcompile_test.cc
+++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc
@@ -529,10 +529,12 @@ TEST(TFCompileTest, ProgramShape) {
   const xla::ProgramShapeProto* muladd_shape = muladd.ProgramShape();
   ASSERT_TRUE(muladd_shape != nullptr);
   ASSERT_EQ(muladd_shape->parameters_size(), 2);
-  EXPECT_TRUE(ShapeUtil::Compatible(muladd_shape->parameters(0), f32_2x2));
-  EXPECT_TRUE(ShapeUtil::Compatible(muladd_shape->parameters(1), f32_2x2));
+  EXPECT_TRUE(
+      ShapeUtil::Compatible(xla::Shape(muladd_shape->parameters(0)), f32_2x2));
+  EXPECT_TRUE(
+      ShapeUtil::Compatible(xla::Shape(muladd_shape->parameters(1)), f32_2x2));
 
-  const xla::Shape& muladd_result = muladd_shape->result();
+  const xla::Shape muladd_result(muladd_shape->result());
   ASSERT_EQ(muladd_result.element_type(), xla::TUPLE);
   ASSERT_EQ(ShapeUtil::TupleElementCount(muladd_result), 2);
   const xla::Shape& muladd_result0 =
diff --git a/tensorflow/compiler/tf2xla/shape_util.h b/tensorflow/compiler/tf2xla/shape_util.h
index f7e34a5b40..0b231ea8e7 100644
--- a/tensorflow/compiler/tf2xla/shape_util.h
+++ b/tensorflow/compiler/tf2xla/shape_util.h
@@ -18,6 +18,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_TF2XLA_SHAPE_UTIL_H_
 #define TENSORFLOW_COMPILER_TF2XLA_SHAPE_UTIL_H_
 
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.pb.h"
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
index 4496255d00..8846088678 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
@@ -116,13 +116,13 @@ TEST(XlaJitCompiledCpuFunction, Sum) {
   // Check program shape.
   using xla::ShapeUtil;
   const xla::Shape s32 = ShapeUtil::MakeShape(xla::S32, {});
-  const xla::ProgramShapeProto* program_shape = function.ProgramShape();
-  ASSERT_TRUE(program_shape != nullptr);
-  ASSERT_EQ(program_shape->parameters_size(), 2);
-  EXPECT_TRUE(ShapeUtil::Compatible(program_shape->parameters(0), s32));
-  EXPECT_TRUE(ShapeUtil::Compatible(program_shape->parameters(1), s32));
+  ASSERT_TRUE(function.ProgramShape() != nullptr);
+  const xla::ProgramShape program_shape(*function.ProgramShape());
+  ASSERT_EQ(program_shape.parameters_size(), 2);
+  EXPECT_TRUE(ShapeUtil::Compatible(program_shape.parameters(0), s32));
+  EXPECT_TRUE(ShapeUtil::Compatible(program_shape.parameters(1), s32));
 
-  const xla::Shape& result = program_shape->result();
+  const xla::Shape& result = program_shape.result();
   ASSERT_EQ(result.element_type(), xla::TUPLE);
   ASSERT_EQ(ShapeUtil::TupleElementCount(result), 1);
   const xla::Shape& result0 = ShapeUtil::GetTupleElementShape(result, 0);
diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index ad2e525efd..fe99564d3c 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -81,6 +81,7 @@ cc_library(
         "//tensorflow/core:lib",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index eef2844e0d..74b76f9299 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -20,6 +20,7 @@ limitations under the License.
 
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
+#include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/debug_options_flags.h"
 #include "tensorflow/compiler/xla/execution_options_util.h"
@@ -42,7 +43,7 @@ StatusOr<Literal> Client::Transfer(const GlobalData& data,
   TransferToClientRequest request;
   *request.mutable_data() = data.handle();
   if (shape_with_layout != nullptr) {
-    *request.mutable_shape_with_layout() = *shape_with_layout;
+    *request.mutable_shape_with_layout() = shape_with_layout->ToProto();
   }
   TransferToClientResponse response;
 
@@ -123,7 +124,7 @@ StatusOr<Literal> Client::TransferFromOutfeed(
   }
   request.set_replica_id(replica_id);
   if (shape_with_layout != nullptr) {
-    *request.mutable_shape_with_layout() = *shape_with_layout;
+    *request.mutable_shape_with_layout() = shape_with_layout->ToProto();
   }
   TransferFromOutfeedResponse response;
 
@@ -170,11 +171,14 @@ StatusOr<Literal> Client::ExecuteAndTransfer(
       std::unique_ptr<GlobalData> data,
       Execute(computation, arguments, execution_options, execution_profile));
 
-  const Shape* shape_with_output_layout = nullptr;
+  absl::optional<Shape> shape_with_output_layout;
   if (execution_options && execution_options->has_shape_with_output_layout()) {
-    shape_with_output_layout = &execution_options->shape_with_output_layout();
+    shape_with_output_layout =
+        Shape(execution_options->shape_with_output_layout());
   }
-  return Transfer(*data, shape_with_output_layout);
+  return Transfer(*data, shape_with_output_layout.has_value()
+                             ? &(*shape_with_output_layout)
+                             : nullptr);
 }
 
 StatusOr<Literal> Client::ComputeConstant(const XlaComputation& computation,
@@ -229,7 +233,7 @@ StatusOr<ExecutionHandle> Client::Compile(
 
   // The argument shapes affect how the computation is compiled.
   for (const auto& arg_shape : argument_shapes) {
-    *request.add_input_shape_with_layout() = arg_shape;
+    *request.add_input_shape_with_layout() = arg_shape.ToProto();
   }
 
   CompileResponse response;
@@ -458,7 +462,7 @@ StatusOr<Shape> Client::GetShape(const GlobalData& data) {
     return s;
   }
 
-  return response.shape();
+  return Shape(response.shape());
 }
 
 StatusOr<string> Client::ExecutionStatsAsString(
diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h
index dd8cb5598a..a58090253b 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.h
+++ b/tensorflow/compiler/xla/client/executable_build_options.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/xla.pb.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
diff --git a/tensorflow/compiler/xla/client/lib/testing.cc b/tensorflow/compiler/xla/client/lib/testing.cc
index a44681f586..a95bbf2c8c 100644
--- a/tensorflow/compiler/xla/client/lib/testing.cc
+++ b/tensorflow/compiler/xla/client/lib/testing.cc
@@ -66,7 +66,7 @@ std::unique_ptr<GlobalData> MakeFakeDataViaDeviceOrDie(const Shape& shape,
   XlaComputation computation = b.Build().ConsumeValueOrDie();
 
   auto execution_options = CreateDefaultExecutionOptions();
-  *execution_options.mutable_shape_with_output_layout() = shape;
+  *execution_options.mutable_shape_with_output_layout() = shape.ToProto();
   return client->Execute(computation, /*arguments=*/{}, &execution_options)
       .ConsumeValueOrDie();
 }
@@ -98,8 +98,8 @@ std::vector<std::unique_ptr<GlobalData>> MakeFakeArgumentsOrDie(
   auto program_shape = computation.proto().host_program_shape();
 
   std::vector<std::unique_ptr<GlobalData>> results;
-  for (const Shape& shape : program_shape.parameters()) {
-    results.push_back(MakeFakeDataOrDie(shape, client));
+  for (const ShapeProto& shape : program_shape.parameters()) {
+    results.push_back(MakeFakeDataOrDie(Shape(shape), client));
   }
   return results;
 }
diff --git a/tensorflow/compiler/xla/client/sharding_builder.cc b/tensorflow/compiler/xla/client/sharding_builder.cc
index 176802b33e..fb9ea6ec3f 100644
--- a/tensorflow/compiler/xla/client/sharding_builder.cc
+++ b/tensorflow/compiler/xla/client/sharding_builder.cc
@@ -36,7 +36,7 @@ OpSharding Tile(const Shape& tile_shape,
                 const TileAssignment& tile_assignment) {
   OpSharding result;
   result.set_type(OpSharding::Type::OpSharding_Type_OTHER);
-  *result.mutable_tile_shape() = tile_shape;
+  *result.mutable_tile_shape() = tile_shape.ToProto();
   for (int64 dim : tile_assignment.dimensions()) {
     result.add_tile_assignment_dimensions(dim);
   }
@@ -52,7 +52,7 @@ OpSharding Tile1D(const Shape& tile_shape, int64 num_tiles) {
 
   CHECK_EQ(ShapeUtil::Rank(tile_shape), 1);
   std::vector<int64> dimensions(1, num_tiles);
-  *result.mutable_tile_shape() = tile_shape;
+  *result.mutable_tile_shape() = tile_shape.ToProto();
   auto& tile_dimension =
       (*result.mutable_tile_shape()->mutable_dimensions())[0];
   tile_dimension = CeilOfRatio(static_cast<int64>(tile_dimension), num_tiles);
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index f17bc456a6..0630f3cf66 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -102,7 +102,7 @@ StatusOr<Shape> XlaBuilder::GetShape(const XlaOp& op) const {
   TF_RETURN_IF_ERROR(first_error_);
 
   TF_ASSIGN_OR_RETURN(auto instr, LookUpInstruction(op));
-  return instr->shape();
+  return Shape(instr->shape());
 }
 
 StatusOr<std::vector<Shape>> XlaBuilder::GetOperandShapes(
@@ -155,7 +155,7 @@ StatusOr<ProgramShape> XlaBuilder::GetProgramShape(int64 root_id) const {
 
   ProgramShape program_shape;
 
-  *program_shape.mutable_result() = root_proto->shape();
+  *program_shape.mutable_result() = Shape(root_proto->shape());
 
   // Check that the parameter numbers are continuous from 0, and add parameter
   // shapes and names to the program shape.
@@ -172,7 +172,7 @@ StatusOr<ProgramShape> XlaBuilder::GetProgramShape(int64 root_id) const {
       const int64 index = instr.parameter_number();
       TF_RET_CHECK(index >= 0 && index < param_count)
           << "invalid parameter number: " << index;
-      *program_shape.mutable_parameters(index) = instr.shape();
+      *program_shape.mutable_parameters(index) = Shape(instr.shape());
       *program_shape.mutable_parameter_names(index) = instr.name();
     }
   }
@@ -329,7 +329,7 @@ StatusOr<XlaOp> XlaBuilder::InDimBroadcast(
   TF_RETURN_IF_ERROR(first_error_);
 
   HloInstructionProto instr;
-  *instr.mutable_shape() = shape;
+  *instr.mutable_shape() = shape.ToProto();
   for (int64 dim : broadcast_dimensions) {
     instr.add_dimensions(dim);
   }
@@ -380,8 +380,9 @@ XlaOp XlaBuilder::UnaryOp(HloOpcode unop, const XlaOp& operand) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferUnaryOpShape(unop, operand_shape));
+    *instr.mutable_shape() = shape.ToProto();
     return AddInstruction(std::move(instr), unop, {operand});
   });
 }
@@ -392,9 +393,10 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs));
     TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferBinaryOpShape(
                             binop, lhs_shape, rhs_shape, broadcast_dimensions));
+    *instr.mutable_shape() = shape.ToProto();
 
     const int64 lhs_rank = ShapeUtil::Rank(lhs_shape);
     const int64 rhs_rank = ShapeUtil::Rank(rhs_shape);
@@ -408,7 +410,7 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
       const Shape& from_shape = should_broadcast_lhs ? lhs_shape : rhs_shape;
 
       std::vector<int64> to_size;
-      for (int64 size : instr.shape().dimensions()) {
+      for (int64 size : shape.dimensions()) {
         to_size.push_back(size);
       }
       for (int64 from_dim = 0; from_dim < ShapeUtil::Rank(from_shape);
@@ -428,14 +430,14 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
     }
 
     TF_ASSIGN_OR_RETURN(Shape updated_lhs_shape, GetShape(updated_lhs));
-    if (!ShapeUtil::SameDimensions(instr.shape(), updated_lhs_shape)) {
+    if (!ShapeUtil::SameDimensions(shape, updated_lhs_shape)) {
       TF_ASSIGN_OR_RETURN(updated_lhs,
-                          AddBroadcastSequence(instr.shape(), updated_lhs));
+                          AddBroadcastSequence(shape, updated_lhs));
     }
     TF_ASSIGN_OR_RETURN(Shape updated_rhs_shape, GetShape(updated_rhs));
-    if (!ShapeUtil::SameDimensions(instr.shape(), updated_rhs_shape)) {
+    if (!ShapeUtil::SameDimensions(shape, updated_rhs_shape)) {
       TF_ASSIGN_OR_RETURN(updated_rhs,
-                          AddBroadcastSequence(instr.shape(), updated_rhs));
+                          AddBroadcastSequence(shape, updated_rhs));
     }
 
     return AddInstruction(std::move(instr), binop, {updated_lhs, updated_rhs});
@@ -449,30 +451,28 @@ XlaOp XlaBuilder::TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
     TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs));
     TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs));
     TF_ASSIGN_OR_RETURN(const Shape& ehs_shape, GetShape(ehs));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
-                        ShapeInference::InferTernaryOpShape(
-                            triop, lhs_shape, rhs_shape, ehs_shape));
+    TF_ASSIGN_OR_RETURN(
+        Shape shape, ShapeInference::InferTernaryOpShape(triop, lhs_shape,
+                                                         rhs_shape, ehs_shape));
+    *instr.mutable_shape() = shape.ToProto();
     XlaOp updated_lhs = lhs;
     XlaOp updated_rhs = rhs;
     XlaOp updated_ehs = ehs;
-    if (!ShapeUtil::IsTuple(instr.shape())) {
+    if (!ShapeUtil::IsTuple(shape)) {
       if (!ShapeUtil::IsTuple(lhs_shape) &&
-          !ShapeUtil::SameDimensions(instr.shape(), lhs_shape)) {
+          !ShapeUtil::SameDimensions(shape, lhs_shape)) {
         // lhs is being implicitly broadcasted. Change to explicit.
-        TF_ASSIGN_OR_RETURN(updated_lhs,
-                            AddBroadcastSequence(instr.shape(), lhs));
+        TF_ASSIGN_OR_RETURN(updated_lhs, AddBroadcastSequence(shape, lhs));
       }
       if (!ShapeUtil::IsTuple(rhs_shape) &&
-          !ShapeUtil::SameDimensions(instr.shape(), rhs_shape)) {
+          !ShapeUtil::SameDimensions(shape, rhs_shape)) {
         // rhs is being implicitly broadcasted. Change to explicit.
-        TF_ASSIGN_OR_RETURN(updated_rhs,
-                            AddBroadcastSequence(instr.shape(), rhs));
+        TF_ASSIGN_OR_RETURN(updated_rhs, AddBroadcastSequence(shape, rhs));
       }
       if (!ShapeUtil::IsTuple(ehs_shape) &&
-          !ShapeUtil::SameDimensions(instr.shape(), ehs_shape)) {
+          !ShapeUtil::SameDimensions(shape, ehs_shape)) {
         // ehs is being implicitly broadcasted. Change to explicit.
-        TF_ASSIGN_OR_RETURN(updated_ehs,
-                            AddBroadcastSequence(instr.shape(), ehs));
+        TF_ASSIGN_OR_RETURN(updated_ehs, AddBroadcastSequence(shape, ehs));
       }
     }
     return AddInstruction(std::move(instr), triop,
@@ -493,7 +493,7 @@ XlaOp XlaBuilder::Mul(const XlaOp& lhs, const XlaOp& rhs,
 XlaOp XlaBuilder::ConstantLiteral(const LiteralSlice& literal) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    *instr.mutable_shape() = literal.shape();
+    *instr.mutable_shape() = literal.shape().ToProto();
     *instr.mutable_literal() = literal.ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kConstant);
   });
@@ -502,7 +502,7 @@ XlaOp XlaBuilder::ConstantLiteral(const LiteralSlice& literal) {
 XlaOp XlaBuilder::Iota(const Shape& shape, int64 iota_dimension) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    *instr.mutable_shape() = shape;
+    *instr.mutable_shape() = shape.ToProto();
     instr.add_dimensions(iota_dimension);
     return AddInstruction(std::move(instr), HloOpcode::kIota);
   });
@@ -522,10 +522,10 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
                       [](const Shape& shape) { return &shape; });
     TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape,
                         computation.GetProgramShape());
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferCallShape(operand_shape_ptrs,
-                                       /*to_apply=*/called_program_shape));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferCallShape(
+                                         operand_shape_ptrs,
+                                         /*to_apply=*/called_program_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     AddCalledComputation(computation, &instr);
 
@@ -543,7 +543,7 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
     }
     instr.set_parameter_number(parameter_number);
     instr.set_name(name);
-    *instr.mutable_shape() = shape;
+    *instr.mutable_shape() = shape.ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kParameter);
   });
 }
@@ -601,7 +601,7 @@ StatusOr<XlaOp> XlaBuilder::Reshape(const Shape& shape, const XlaOp& operand) {
   TF_RETURN_IF_ERROR(first_error_);
 
   HloInstructionProto instr;
-  *instr.mutable_shape() = shape;
+  *instr.mutable_shape() = shape.ToProto();
   return AddInstruction(std::move(instr), HloOpcode::kReshape, {operand});
 }
 
@@ -613,9 +613,9 @@ XlaOp XlaBuilder::Slice(const XlaOp& operand,
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferSliceShape(operand_shape, start_indices,
-                                        limit_indices, strides));
+        Shape shape, ShapeInference::InferSliceShape(
+                         operand_shape, start_indices, limit_indices, strides));
+    *instr.mutable_shape() = shape.ToProto();
     for (int i = 0; i < start_indices.size(); i++) {
       auto* slice_config = instr.add_slice_dimensions();
       slice_config->set_start(start_indices[i]);
@@ -650,9 +650,10 @@ XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
     TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape,
                         GetShape(start_indices));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferDynamicSliceShape(
                             operand_shape, start_indices_shape, slice_sizes));
+    *instr.mutable_shape() = shape.ToProto();
 
     for (int64 size : slice_sizes) {
       instr.add_dynamic_slice_sizes(size);
@@ -672,9 +673,10 @@ XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
     TF_ASSIGN_OR_RETURN(const Shape& update_shape, GetShape(update));
     TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape,
                         GetShape(start_indices));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferDynamicUpdateSliceShape(
                             operand_shape, update_shape, start_indices_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     return AddInstruction(std::move(instr), HloOpcode::kDynamicUpdateSlice,
                           {operand, update, start_indices});
@@ -690,9 +692,9 @@ XlaOp XlaBuilder::ConcatInDim(absl::Span<const XlaOp> operands,
     TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(operands));
     absl::c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
                       [](const Shape& shape) { return &shape; });
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferConcatOpShape(operand_shape_ptrs, dimension));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConcatOpShape(
+                                         operand_shape_ptrs, dimension));
+    *instr.mutable_shape() = shape.ToProto();
 
     instr.add_dimensions(dimension);
 
@@ -709,10 +711,9 @@ XlaOp XlaBuilder::Pad(const XlaOp& operand, const XlaOp& padding_value,
     TF_ASSIGN_OR_RETURN(const Shape& padding_value_shape,
                         GetShape(padding_value));
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferPadShape(operand_shape, padding_value_shape,
-                                      padding_config));
-
+        Shape shape, ShapeInference::InferPadShape(
+                         operand_shape, padding_value_shape, padding_config));
+    *instr.mutable_shape() = shape.ToProto();
     *instr.mutable_padding_config() = padding_config;
 
     return AddInstruction(std::move(instr), HloOpcode::kPad,
@@ -725,7 +726,7 @@ XlaOp XlaBuilder::Reshape(const XlaOp& operand,
                           absl::Span<const int64> new_sizes) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(const Shape& shape,
+    TF_ASSIGN_OR_RETURN(const Shape shape,
                         ShapeInference::InferReshapeShape(
                             operand_shape, dimensions, new_sizes));
     XlaOp transposed = IsIdentityPermutation(dimensions)
@@ -738,7 +739,7 @@ XlaOp XlaBuilder::Reshape(const XlaOp& operand,
 XlaOp XlaBuilder::Reshape(const XlaOp& operand,
                           absl::Span<const int64> new_sizes) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    TF_ASSIGN_OR_RETURN(auto shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(Shape shape, GetShape(operand));
     std::vector<int64> dimensions(shape.dimensions_size());
     std::iota(dimensions.begin(), dimensions.end(), 0);
     return Reshape(operand, dimensions, new_sizes);
@@ -788,7 +789,7 @@ XlaOp XlaBuilder::Collapse(const XlaOp& operand,
 void XlaBuilder::Trace(const string& tag, const XlaOp& operand) {
   ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    *instr.mutable_shape() = ShapeUtil::MakeNil();
+    *instr.mutable_shape() = ShapeUtil::MakeNil().ToProto();
     *instr.mutable_literal() = LiteralUtil::CreateR1U8(tag).ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kTrace, {operand});
   });
@@ -814,9 +815,10 @@ XlaOp XlaBuilder::Tuple(absl::Span<const XlaOp> elements) {
     TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(elements));
     absl::c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
                       [](const Shape& shape) { return &shape; });
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(const Shape shape,
                         ShapeInference::InferVariadicOpShape(
                             HloOpcode::kTuple, operand_shape_ptrs));
+    *instr.mutable_shape() = shape.ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kTuple, elements);
   });
 }
@@ -831,7 +833,7 @@ XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
           ShapeUtil::HumanString(tuple_shape));
     }
     *instr.mutable_shape() =
-        ShapeUtil::GetTupleElementShape(tuple_shape, index);
+        ShapeUtil::GetTupleElementShape(tuple_shape, index).ToProto();
 
     instr.set_tuple_index(index);
 
@@ -890,9 +892,10 @@ XlaOp XlaBuilder::DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs));
     TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferDotOpShape(lhs_shape, rhs_shape,
                                                         dimension_numbers));
+    *instr.mutable_shape() = shape.ToProto();
     *instr.mutable_dot_dimension_numbers() = dimension_numbers;
     if (precision_config != nullptr) {
       *instr.mutable_precision_config() = *precision_config;
@@ -1034,10 +1037,11 @@ XlaOp XlaBuilder::ConvGeneralDilated(
                         MakeWindow(window_dimensions, window_strides, padding,
                                    lhs_dilation, rhs_dilation));
 
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferConvolveShape(
                             lhs_shape, rhs_shape, feature_group_count,
                             instr.window(), dimension_numbers));
+    *instr.mutable_shape() = shape.ToProto();
 
     *instr.mutable_convolution_dimension_numbers() = dimension_numbers;
     instr.set_feature_group_count(feature_group_count);
@@ -1110,10 +1114,9 @@ XlaOp XlaBuilder::Fft(const XlaOp& operand, const FftType fft_type,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferFftShape(operand_shape, fft_type, fft_length));
-
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferFftShape(
+                                         operand_shape, fft_type, fft_length));
+    *instr.mutable_shape() = shape.ToProto();
     instr.set_fft_type(fft_type);
     for (int64 i : fft_length) {
       instr.add_fft_length(i);
@@ -1131,7 +1134,7 @@ XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) {
     }
     const Shape infeed_instruction_shape =
         ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()});
-    *instr.mutable_shape() = infeed_instruction_shape;
+    *instr.mutable_shape() = infeed_instruction_shape.ToProto();
     instr.set_infeed_config(config);
 
     if (ShapeUtil::IsArray(shape) && sharding() &&
@@ -1152,7 +1155,7 @@ XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) {
     XlaOp token;
     auto make_token = [&]() {
       HloInstructionProto token_instr;
-      *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+      *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
       return AddInstruction(std::move(token_instr), HloOpcode::kAfterAll, {});
     };
     if (sharding()) {
@@ -1191,7 +1194,7 @@ XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) {
     // TODO(b/80000000): Remove this when clients have been updated to handle
     // tokens.
     HloInstructionProto infeed_data;
-    *infeed_data.mutable_shape() = shape;
+    *infeed_data.mutable_shape() = shape.ToProto();
     infeed_data.set_tuple_index(0);
     return AddInstruction(std::move(infeed_data), HloOpcode::kGetTupleElement,
                           {infeed});
@@ -1207,7 +1210,7 @@ XlaOp XlaBuilder::InfeedWithToken(const XlaOp& token, const Shape& shape,
     }
     const Shape infeed_instruction_shape =
         ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()});
-    *instr.mutable_shape() = infeed_instruction_shape;
+    *instr.mutable_shape() = infeed_instruction_shape.ToProto();
     instr.set_infeed_config(config);
 
     if (ShapeUtil::IsArray(shape) && sharding() &&
@@ -1232,7 +1235,7 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
   ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
 
-    *instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
 
     // Check and set outfeed shape.
     if (!LayoutUtil::HasLayout(shape_with_layout)) {
@@ -1245,14 +1248,14 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
           ShapeUtil::HumanStringWithLayout(shape_with_layout),
           ShapeUtil::HumanStringWithLayout(operand_shape));
     }
-    *instr.mutable_outfeed_shape() = shape_with_layout;
+    *instr.mutable_outfeed_shape() = shape_with_layout.ToProto();
 
     instr.set_outfeed_config(outfeed_config);
 
     // Outfeed takes a token as its second operand. Generate the token to pass
     // to the outfeed.
     HloInstructionProto token_instr;
-    *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr),
                                                     HloOpcode::kAfterAll, {}));
 
@@ -1266,7 +1269,7 @@ void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
     // TODO(b/80000000): Remove this when clients have been updated to handle
     // tokens.
     HloInstructionProto tuple_instr;
-    *tuple_instr.mutable_shape() = ShapeUtil::MakeNil();
+    *tuple_instr.mutable_shape() = ShapeUtil::MakeNil().ToProto();
 
     // The dummy tuple should have no sharding.
     {
@@ -1285,7 +1288,7 @@ XlaOp XlaBuilder::OutfeedWithToken(const XlaOp& operand, const XlaOp& token,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
 
-    *instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
 
     // Check and set outfeed shape.
     if (!LayoutUtil::HasLayout(shape_with_layout)) {
@@ -1298,7 +1301,7 @@ XlaOp XlaBuilder::OutfeedWithToken(const XlaOp& operand, const XlaOp& token,
           ShapeUtil::HumanStringWithLayout(shape_with_layout),
           ShapeUtil::HumanStringWithLayout(operand_shape));
     }
-    *instr.mutable_outfeed_shape() = shape_with_layout;
+    *instr.mutable_outfeed_shape() = shape_with_layout.ToProto();
 
     instr.set_outfeed_config(outfeed_config);
 
@@ -1310,7 +1313,7 @@ XlaOp XlaBuilder::OutfeedWithToken(const XlaOp& operand, const XlaOp& token,
 XlaOp XlaBuilder::CreateToken() {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    *instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kAfterAll);
   });
 }
@@ -1330,7 +1333,7 @@ XlaOp XlaBuilder::AfterAll(absl::Span<const XlaOp> tokens) {
       }
     }
     HloInstructionProto instr;
-    *instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kAfterAll, tokens);
   });
 }
@@ -1347,7 +1350,7 @@ XlaOp XlaBuilder::CustomCall(
           "are reserved for internal use.",
           call_target_name);
     }
-    *instr.mutable_shape() = shape;
+    *instr.mutable_shape() = shape.ToProto();
     instr.set_custom_call_target(call_target_name);
     instr.set_custom_call_opaque(opaque);
     if (operand_shapes_with_layout.has_value()) {
@@ -1371,7 +1374,7 @@ XlaOp XlaBuilder::CustomCall(
               "constrained layout.",
               operand_num);
         }
-        *instr.add_operand_shapes_with_layout() = operand_shape;
+        *instr.add_operand_shapes_with_layout() = operand_shape.ToProto();
         ++operand_num;
       }
     }
@@ -1525,9 +1528,9 @@ XlaOp XlaBuilder::Transpose(const XlaOp& operand,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferTransposeShape(operand_shape, permutation));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferTransposeShape(
+                                         operand_shape, permutation));
+    *instr.mutable_shape() = shape.ToProto();
     for (int64 dim : permutation) {
       instr.add_dimensions(dim);
     }
@@ -1540,9 +1543,9 @@ XlaOp XlaBuilder::Rev(const XlaOp& operand,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferReverseShape(operand_shape, dimensions));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferReverseShape(
+                                         operand_shape, dimensions));
+    *instr.mutable_shape() = shape.ToProto();
     for (int64 dim : dimensions) {
       instr.add_dimensions(dim);
     }
@@ -1561,9 +1564,9 @@ XlaOp XlaBuilder::Sort(const XlaOp& keys, absl::Span<const XlaOp> values,
                         GetOperandShapes(values));
     absl::c_transform(values_shapes, std::back_inserter(operand_shape_ptrs),
                       [](const Shape& shape) { return &shape; });
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
-                        ShapeInference::InferVariadicOpShape(
-                            HloOpcode::kSort, operand_shape_ptrs));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferVariadicOpShape(
+                                         HloOpcode::kSort, operand_shape_ptrs));
+    *instr.mutable_shape() = shape.ToProto();
     if (dimension == -1) {
       TF_ASSIGN_OR_RETURN(const Shape& keys_shape, GetShape(keys));
       dimension = ShapeUtil::Rank(keys_shape) - 1;
@@ -1585,9 +1588,9 @@ XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferConvertShape(operand_shape, new_element_type));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConvertShape(
+                                         operand_shape, new_element_type));
+    *instr.mutable_shape() = shape.ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kConvert, {operand});
   });
 }
@@ -1597,9 +1600,9 @@ XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferConvertShape(operand_shape, new_element_type));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferConvertShape(
+                                         operand_shape, new_element_type));
+    *instr.mutable_shape() = shape.ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kBitcastConvert,
                           {operand});
   });
@@ -1631,11 +1634,11 @@ XlaOp XlaBuilder::Map(absl::Span<const XlaOp> operands,
     TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape,
                         computation.GetProgramShape());
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferMapShape(operand_shape_ptrs, called_program_shape,
-                                      dimensions));
+        Shape shape, ShapeInference::InferMapShape(
+                         operand_shape_ptrs, called_program_shape, dimensions));
+    *instr.mutable_shape() = shape.ToProto();
 
-    const Shape& output_shape = instr.shape();
+    Shape output_shape(instr.shape());
     const int64 output_rank = ShapeUtil::Rank(output_shape);
     AddCalledComputation(computation, &instr);
     std::vector<XlaOp> new_operands(operands.begin(), operands.end());
@@ -1678,7 +1681,7 @@ XlaOp XlaBuilder::RngOp(RandomDistribution distribution,
     }
 
     TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape));
-    *instr.mutable_shape() = shape;
+    *instr.mutable_shape() = shape.ToProto();
 
     instr.set_distribution(distribution);
 
@@ -1706,10 +1709,10 @@ XlaOp XlaBuilder::While(const XlaComputation& condition,
     TF_ASSIGN_OR_RETURN(const auto& condition_program_shape,
                         condition.GetProgramShape());
     TF_ASSIGN_OR_RETURN(const Shape& init_shape, GetShape(init));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferWhileShape(condition_program_shape,
-                                        body_program_shape, init_shape));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferWhileShape(
+                                         condition_program_shape,
+                                         body_program_shape, init_shape));
+    *instr.mutable_shape() = shape.ToProto();
     // Body comes before condition computation in the vector.
     AddCalledComputation(body, &instr);
     AddCalledComputation(condition, &instr);
@@ -1726,10 +1729,10 @@ XlaOp XlaBuilder::Gather(const XlaOp& input, const XlaOp& start_indices,
     TF_ASSIGN_OR_RETURN(const Shape& input_shape, GetShape(input));
     TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape,
                         GetShape(start_indices));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferGatherShape(input_shape, start_indices_shape,
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferGatherShape(
+                                         input_shape, start_indices_shape,
                                          dimension_numbers, slice_sizes));
+    *instr.mutable_shape() = shape.ToProto();
 
     *instr.mutable_gather_dimension_numbers() = dimension_numbers;
     for (int64 bound : slice_sizes) {
@@ -1754,10 +1757,11 @@ XlaOp XlaBuilder::Scatter(const XlaOp& input, const XlaOp& scatter_indices,
     TF_ASSIGN_OR_RETURN(const Shape& updates_shape, GetShape(updates));
     TF_ASSIGN_OR_RETURN(const ProgramShape& to_apply_shape,
                         update_computation.GetProgramShape());
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferScatterShape(
                             input_shape, scatter_indices_shape, updates_shape,
                             to_apply_shape, dimension_numbers));
+    *instr.mutable_shape() = shape.ToProto();
 
     *instr.mutable_scatter_dimension_numbers() = dimension_numbers;
 
@@ -1784,10 +1788,11 @@ XlaOp XlaBuilder::Conditional(const XlaOp& predicate, const XlaOp& true_operand,
     TF_ASSIGN_OR_RETURN(const ProgramShape& false_computation_shape,
                         false_computation.GetProgramShape());
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
+        Shape shape,
         ShapeInference::InferConditionalShape(
             predicate_shape, true_operand_shape, false_operand_shape,
             true_computation_shape, false_computation_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     // The index of true_computation must be 0 and that of false computation
     // must be 1.
@@ -1829,9 +1834,10 @@ XlaOp XlaBuilder::Reduce(absl::Span<const XlaOp> operands,
                       [](const Shape& shape) { return &shape; });
 
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
+        Shape shape,
         ShapeInference::InferReduceShape(
             operand_shape_ptrs, dimensions_to_reduce, called_program_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     for (int64 dim : dimensions_to_reduce) {
       instr.add_dimensions(dim);
@@ -1894,10 +1900,10 @@ XlaOp XlaBuilder::ReduceWindowWithGeneralPadding(
                         MakeWindow(window_dimensions, window_strides, padding,
                                    /*lhs_dilation=*/base_dilations,
                                    /*rhs_dilation=*/window_dilations));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferReduceWindowShape(operand_shape, init_shape,
-                                               instr.window(), to_apply_shape));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferReduceWindowShape(
+                                         operand_shape, init_shape,
+                                         instr.window(), to_apply_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     AddCalledComputation(computation, &instr);
     return AddInstruction(std::move(instr), HloOpcode::kReduceWindow,
@@ -1915,9 +1921,10 @@ XlaOp XlaBuilder::BatchNormTraining(const XlaOp& operand, const XlaOp& scale,
     TF_ASSIGN_OR_RETURN(const Shape& scale_shape, GetShape(scale));
     TF_ASSIGN_OR_RETURN(const Shape& offset_shape, GetShape(offset));
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
+        Shape shape,
         ShapeInference::InferBatchNormTrainingShape(
             operand_shape, scale_shape, offset_shape, feature_index));
+    *instr.mutable_shape() = shape.ToProto();
 
     instr.set_epsilon(epsilon);
     instr.set_feature_index(feature_index);
@@ -1939,10 +1946,11 @@ XlaOp XlaBuilder::BatchNormInference(const XlaOp& operand, const XlaOp& scale,
     TF_ASSIGN_OR_RETURN(const Shape& offset_shape, GetShape(offset));
     TF_ASSIGN_OR_RETURN(const Shape& mean_shape, GetShape(mean));
     TF_ASSIGN_OR_RETURN(const Shape& variance_shape, GetShape(variance));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
-                        ShapeInference::InferBatchNormInferenceShape(
-                            operand_shape, scale_shape, offset_shape,
-                            mean_shape, variance_shape, feature_index));
+    TF_ASSIGN_OR_RETURN(
+        Shape shape, ShapeInference::InferBatchNormInferenceShape(
+                         operand_shape, scale_shape, offset_shape, mean_shape,
+                         variance_shape, feature_index));
+    *instr.mutable_shape() = shape.ToProto();
 
     instr.set_epsilon(epsilon);
     instr.set_feature_index(feature_index);
@@ -1964,10 +1972,11 @@ XlaOp XlaBuilder::BatchNormGrad(const XlaOp& operand, const XlaOp& scale,
     TF_ASSIGN_OR_RETURN(const Shape& batch_mean_shape, GetShape(batch_mean));
     TF_ASSIGN_OR_RETURN(const Shape& batch_var_shape, GetShape(batch_var));
     TF_ASSIGN_OR_RETURN(const Shape& grad_output_shape, GetShape(grad_output));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferBatchNormGradShape(
                             operand_shape, scale_shape, batch_mean_shape,
                             batch_var_shape, grad_output_shape, feature_index));
+    *instr.mutable_shape() = shape.ToProto();
 
     instr.set_epsilon(epsilon);
     instr.set_feature_index(feature_index);
@@ -1998,9 +2007,9 @@ XlaOp XlaBuilder::CrossReplicaSum(
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferCrossReplicaSumShape({&operand_shape}));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferCrossReplicaSumShape(
+                                         {&operand_shape}));
+    *instr.mutable_shape() = shape.ToProto();
 
     for (const ReplicaGroup& group : replica_groups) {
       *instr.add_replica_groups() = group;
@@ -2053,8 +2062,8 @@ XlaOp XlaBuilder::AllToAll(const XlaOp& operand, int64 split_dimension,
     absl::c_transform(slice_shapes, std::back_inserter(slice_shape_ptrs),
                       [](const Shape& shape) { return &shape; });
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferAllToAllTupleShape(slice_shape_ptrs));
+        Shape shape, ShapeInference::InferAllToAllTupleShape(slice_shape_ptrs));
+    *instr.mutable_shape() = shape.ToProto();
     for (const ReplicaGroup& group : replica_groups) {
       *instr.add_replica_groups() = group;
     }
@@ -2079,8 +2088,9 @@ XlaOp XlaBuilder::CollectivePermute(
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
+        Shape shape,
         ShapeInference::InferCollectivePermuteShape(operand_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     for (const auto& pair : source_target_pairs) {
       auto* proto_pair = instr.add_source_target_pairs();
@@ -2129,10 +2139,11 @@ XlaOp XlaBuilder::SelectAndScatterWithGeneralPadding(
     TF_ASSIGN_OR_RETURN(*instr.mutable_window(),
                         MakeWindow(window_dimensions, window_strides, padding,
                                    /*lhs_dilation=*/{}, /*rhs_dilation=*/{}));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferSelectAndScatterShape(
                             operand_shape, select_shape, instr.window(),
                             source_shape, init_shape, scatter_shape));
+    *instr.mutable_shape() = shape.ToProto();
 
     AddCalledComputation(select, &instr);
     AddCalledComputation(scatter, &instr);
@@ -2147,9 +2158,10 @@ XlaOp XlaBuilder::ReducePrecision(const XlaOp& operand, const int exponent_bits,
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+    TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferReducePrecisionShape(
                             operand_shape, exponent_bits, mantissa_bits));
+    *instr.mutable_shape() = shape.ToProto();
     instr.set_exponent_bits(exponent_bits);
     instr.set_mantissa_bits(mantissa_bits);
     return AddInstruction(std::move(instr), HloOpcode::kReducePrecision,
@@ -2164,7 +2176,7 @@ void XlaBuilder::Send(const XlaOp& operand, const ChannelHandle& handle) {
     // TODO(b/80000000): Remove this when clients have been updated to handle
     // tokens.
     HloInstructionProto token_instr;
-    *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr),
                                                     HloOpcode::kAfterAll, {}));
 
@@ -2183,15 +2195,17 @@ XlaOp XlaBuilder::SendWithToken(const XlaOp& operand, const XlaOp& token,
     // token}.
     HloInstructionProto send_instr;
     TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(operand));
-    *send_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
-        {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()});
+    *send_instr.mutable_shape() =
+        ShapeUtil::MakeTupleShape(
+            {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()})
+            .ToProto();
     send_instr.set_channel_id(handle.handle());
     TF_ASSIGN_OR_RETURN(XlaOp send,
                         AddInstruction(std::move(send_instr), HloOpcode::kSend,
                                        {operand, token}));
 
     HloInstructionProto send_done_instr;
-    *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     send_done_instr.set_channel_id(handle.handle());
     return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone,
                           {send});
@@ -2205,7 +2219,7 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
     // TODO(b/80000000): Remove this when clients have been updated to handle
     // tokens.
     HloInstructionProto token_instr;
-    *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *token_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr),
                                                     HloOpcode::kAfterAll, {}));
 
@@ -2216,7 +2230,7 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
     // TODO(b/80000000): Remove this when clients have been updated to handle
     // tokens.
     HloInstructionProto recv_data;
-    *recv_data.mutable_shape() = shape;
+    *recv_data.mutable_shape() = shape.ToProto();
     recv_data.set_tuple_index(0);
     return AddInstruction(std::move(recv_data), HloOpcode::kGetTupleElement,
                           {recv});
@@ -2233,15 +2247,18 @@ XlaOp XlaBuilder::RecvWithToken(const XlaOp& token, const Shape& shape,
     // Recv instruction produces a tuple of {receive buffer, U32 context,
     // token}.
     HloInstructionProto recv_instr;
-    *recv_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
-        {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()});
+    *recv_instr.mutable_shape() =
+        ShapeUtil::MakeTupleShape(
+            {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()})
+            .ToProto();
     recv_instr.set_channel_id(handle.handle());
     TF_ASSIGN_OR_RETURN(XlaOp recv, AddInstruction(std::move(recv_instr),
                                                    HloOpcode::kRecv, {token}));
 
     HloInstructionProto recv_done_instr;
     *recv_done_instr.mutable_shape() =
-        ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()});
+        ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()})
+            .ToProto();
     recv_done_instr.set_channel_id(handle.handle());
     return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone,
                           {recv});
@@ -2275,9 +2292,11 @@ XlaOp XlaBuilder::SendToHost(const XlaOp& operand, const XlaOp& token,
     // Send instruction produces a tuple of {aliased operand, U32 context,
     // token}.
     HloInstructionProto send_instr;
-    *send_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
-        {shape_with_layout, ShapeUtil::MakeShape(U32, {}),
-         ShapeUtil::MakeTokenShape()});
+    *send_instr.mutable_shape() =
+        ShapeUtil::MakeTupleShape({shape_with_layout,
+                                   ShapeUtil::MakeShape(U32, {}),
+                                   ShapeUtil::MakeTokenShape()})
+            .ToProto();
     send_instr.set_channel_id(handle.handle());
     send_instr.set_is_host_transfer(true);
     TF_ASSIGN_OR_RETURN(XlaOp send,
@@ -2285,7 +2304,7 @@ XlaOp XlaBuilder::SendToHost(const XlaOp& operand, const XlaOp& token,
                                        {operand, token}));
 
     HloInstructionProto send_done_instr;
-    *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape().ToProto();
     send_done_instr.set_channel_id(handle.handle());
     send_done_instr.set_is_host_transfer(true);
     return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone,
@@ -2314,8 +2333,10 @@ XlaOp XlaBuilder::RecvFromHost(const XlaOp& token, const Shape& shape,
     // Recv instruction produces a tuple of {receive buffer, U32 context,
     // token}.
     HloInstructionProto recv_instr;
-    *recv_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
-        {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()});
+    *recv_instr.mutable_shape() =
+        ShapeUtil::MakeTupleShape(
+            {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()})
+            .ToProto();
     recv_instr.set_channel_id(handle.handle());
     recv_instr.set_is_host_transfer(true);
     TF_ASSIGN_OR_RETURN(XlaOp recv, AddInstruction(std::move(recv_instr),
@@ -2323,7 +2344,8 @@ XlaOp XlaBuilder::RecvFromHost(const XlaOp& token, const Shape& shape,
 
     HloInstructionProto recv_done_instr;
     *recv_done_instr.mutable_shape() =
-        ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()});
+        ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()})
+            .ToProto();
     recv_done_instr.set_channel_id(handle.handle());
     recv_done_instr.set_is_host_transfer(true);
     return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone,
@@ -2335,9 +2357,9 @@ XlaOp XlaBuilder::GetDimensionSize(const XlaOp& operand, int64 dimension) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const auto& operand_shape, GetShape(operand));
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferGetDimensionSizeShape(operand_shape, dimension));
+    TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferGetDimensionSizeShape(
+                                         operand_shape, dimension));
+    *instr.mutable_shape() = shape.ToProto();
     instr.add_dimensions(dimension);
     return AddInstruction(std::move(instr), HloOpcode::kGetDimensionSize,
                           {operand});
diff --git a/tensorflow/compiler/xla/index_util.h b/tensorflow/compiler/xla/index_util.h
index 458bdaf2f8..d76f61eb62 100644
--- a/tensorflow/compiler/xla/index_util.h
+++ b/tensorflow/compiler/xla/index_util.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include <vector>
 
 #include "absl/types/span.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/macros.h"
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index f2fcb93717..8f480c1f10 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -292,16 +292,17 @@ Status MutableLiteralBase::CopyElementFrom(const LiteralSlice& src_literal,
   if (!proto.has_shape()) {
     return InvalidArgument("LiteralProto has no shape");
   }
-  if (ShapeUtil::HasPrimitiveType(proto.shape(), OPAQUE)) {
+  Shape shape(proto.shape());
+  if (ShapeUtil::HasPrimitiveType(shape, OPAQUE)) {
     return InvalidArgument("Literal shape cannot include OPAQUE sub-shape");
   }
-  if (!LayoutUtil::HasLayout(proto.shape())) {
+  if (!LayoutUtil::HasLayout(shape)) {
     return InvalidArgument("LiteralProto has no layout");
   }
 
-  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(proto.shape()));
+  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape));
 
-  Literal literal(proto.shape());
+  Literal literal(shape);
 
   TF_RETURN_IF_ERROR(literal.root_piece_->ForEachMutableSubpieceWithStatus(
       [&](const ShapeIndex& index, Piece* piece) {
@@ -1794,7 +1795,7 @@ void CopyToRepeatedField(RepeatedFieldT* dest,
 }  // namespace
 
 void LiteralBase::Piece::WriteToProto(LiteralProto* proto) const {
-  *proto->mutable_shape() = subshape();
+  *proto->mutable_shape() = subshape().ToProto();
   switch (subshape().element_type()) {
     case PRED:
       CopyToRepeatedField(proto->mutable_preds(), data<bool>());
@@ -1900,8 +1901,9 @@ Status LiteralBase::Piece::CopyFromProto(const LiteralProto& proto) {
   // These conditions should have been checked in
   // MutableLiteralBase::CreateFromProto.
   TF_RET_CHECK(proto.has_shape());
-  TF_RET_CHECK(LayoutUtil::HasLayout(proto.shape()));
-  TF_RET_CHECK(ShapeUtil::Equal(proto.shape(), subshape()));
+  Shape shape(proto.shape());
+  TF_RET_CHECK(LayoutUtil::HasLayout(shape));
+  TF_RET_CHECK(ShapeUtil::Equal(shape, subshape()));
 
   if (LayoutUtil::IsSparseArray(subshape())) {
     // Compute the number of elements (indices) in the sparse shape and reserve
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index bd93517728..49363ad802 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -1377,13 +1377,26 @@ TEST_F(LiteralUtilTest, BitcastConvertBetweenInvalidTypes) {
       absl::StrContains(status.error_message(), "bit widths are different"));
 }
 
+// Sets the layout of the given ShapeProto to the default.
+void SetDefaultLayoutOnProto(ShapeProto* shape_proto) {
+  CHECK(ShapeUtil::IsArrayPrimitiveType(shape_proto->element_type()));
+  shape_proto->mutable_layout()->set_format(DENSE);
+  auto* minor_to_major =
+      shape_proto->mutable_layout()->mutable_minor_to_major();
+  minor_to_major->Resize(shape_proto->dimensions_size(), 0);
+  const int64 size = minor_to_major->size();
+  for (int64 i = 0; i < size; ++i) {
+    minor_to_major->Set(i, size - 1 - i);
+  }
+}
+
 TEST_F(LiteralUtilTest, CopyFromProto_Bool) {
   LiteralProto p;
   p.mutable_shape()->set_element_type(PRED);
   for (int len = 0; len < 25; ++len) {
     p.mutable_shape()->clear_dimensions();
     p.mutable_shape()->add_dimensions(len);
-    LayoutUtil::SetToDefaultLayout(p.mutable_shape());
+    SetDefaultLayoutOnProto(p.mutable_shape());
     p.clear_preds();
     for (int i = 0; i < len; ++i) {
       p.add_preds((i % 2) == (len % 2));
@@ -1409,7 +1422,7 @@ TEST_F(LiteralUtilTest, ToProto_f16) {
   EXPECT_EQ(4, m.data<half>().size());
 
   LiteralProto p = m.ToProto();
-  EXPECT_EQ(4, ShapeUtil::ElementsIn(p.shape()));
+  EXPECT_EQ(4, ShapeUtil::ElementsIn(Shape(p.shape())));
   EXPECT_EQ(8, p.f16s().size());
   const char* d = p.f16s().data();
   EXPECT_EQ(d[0], 0);
@@ -1432,7 +1445,7 @@ TEST_F(LiteralUtilTest, CopyFromProto_f16) {
   p.mutable_shape()->set_element_type(F16);
   p.mutable_shape()->clear_dimensions();
   p.mutable_shape()->add_dimensions(4);
-  LayoutUtil::SetToDefaultLayout(p.mutable_shape());
+  SetDefaultLayoutOnProto(p.mutable_shape());
   p.clear_f16s();
   p.set_f16s(half_vals, 8);
   TF_ASSERT_OK_AND_ASSIGN(Literal literal, Literal::CreateFromProto(p));
@@ -1454,7 +1467,7 @@ TEST_F(LiteralUtilTest, CopyFromProto_u16) {
   p.mutable_shape()->set_element_type(U16);
   p.mutable_shape()->clear_dimensions();
   p.mutable_shape()->add_dimensions(4);
-  LayoutUtil::SetToDefaultLayout(p.mutable_shape());
+  SetDefaultLayoutOnProto(p.mutable_shape());
   p.clear_u16s();
   p.set_u16s(uint16_vals, 8);
   TF_ASSERT_OK_AND_ASSIGN(Literal literal, Literal::CreateFromProto(p));
@@ -1756,7 +1769,7 @@ TEST_F(LiteralUtilTest, ProtoRoundTrip) {
 TEST_F(LiteralUtilTest, InvalidProtoNoValues) {
   // Proto contains a shape, but no values.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3});
+  *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3}).ToProto();
   Status status = Literal::CreateFromProto(proto).status();
   ASSERT_FALSE(status.ok());
   EXPECT_THAT(status.error_message(),
@@ -1777,7 +1790,7 @@ TEST_F(LiteralUtilTest, InvalidProtoNoShape) {
 TEST_F(LiteralUtilTest, InvalidProtoWrongContainer) {
   // Proto contains values in wrong container.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3});
+  *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {3}).ToProto();
   proto.add_preds(false);
   proto.add_preds(true);
   proto.add_preds(false);
@@ -1790,7 +1803,7 @@ TEST_F(LiteralUtilTest, InvalidProtoWrongContainer) {
 TEST_F(LiteralUtilTest, InvalidProtoTooFewValues) {
   // Proto contains too few values.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {42, 2});
+  *proto.mutable_shape() = ShapeUtil::MakeShape(F32, {42, 2}).ToProto();
   proto.add_f32s(1.0);
   proto.add_f32s(2.0);
   proto.add_f32s(3.0);
@@ -1803,7 +1816,7 @@ TEST_F(LiteralUtilTest, InvalidProtoTooFewValues) {
 TEST_F(LiteralUtilTest, InvalidProtoTooManyValues) {
   // Proto contains too many values.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeShape(S32, {2});
+  *proto.mutable_shape() = ShapeUtil::MakeShape(S32, {2}).ToProto();
   proto.add_s32s(42);
   proto.add_s32s(-10);
   proto.add_s32s(100);
@@ -1816,8 +1829,8 @@ TEST_F(LiteralUtilTest, InvalidProtoTooManyValues) {
 TEST_F(LiteralUtilTest, InvalidProtoMissingLayout) {
   // Proto shape missing layout.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeShape(PRED, {2, 2});
-  LayoutUtil::ClearLayout(proto.mutable_shape());
+  *proto.mutable_shape() = ShapeUtil::MakeShape(PRED, {2, 2}).ToProto();
+  proto.mutable_shape()->clear_layout();
   proto.add_preds(true);
   proto.add_preds(false);
   proto.add_preds(true);
@@ -1830,11 +1843,13 @@ TEST_F(LiteralUtilTest, InvalidProtoMissingLayout) {
 TEST_F(LiteralUtilTest, InvalidProtoTooFewTupleElements) {
   // Proto has the too few tuple elements.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeTupleShape(
-      {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})});
+  *proto.mutable_shape() =
+      ShapeUtil::MakeTupleShape(
+          {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})})
+          .ToProto();
   LiteralProto* element0 = proto.add_tuple_literals();
   *element0->mutable_shape() =
-      ShapeUtil::GetTupleElementShape(proto.shape(), 0);
+      ShapeUtil::GetTupleElementShape(Shape(proto.shape()), 0).ToProto();
   element0->add_preds(false);
   element0->add_preds(true);
 
@@ -1846,19 +1861,21 @@ TEST_F(LiteralUtilTest, InvalidProtoTooFewTupleElements) {
 TEST_F(LiteralUtilTest, InvalidProtoTooManyTupleElements) {
   // Proto has the too many tuple elements.
   LiteralProto proto;
-  *proto.mutable_shape() = ShapeUtil::MakeTupleShape(
-      {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})});
+  *proto.mutable_shape() =
+      ShapeUtil::MakeTupleShape(
+          {ShapeUtil::MakeShape(PRED, {2}), ShapeUtil::MakeShape(F32, {})})
+          .ToProto();
   LiteralProto* element0 = proto.add_tuple_literals();
   *element0->mutable_shape() =
-      ShapeUtil::GetTupleElementShape(proto.shape(), 0);
+      ShapeUtil::GetTupleElementShape(Shape(proto.shape()), 0).ToProto();
   element0->add_preds(false);
   element0->add_preds(true);
   LiteralProto* element1 = proto.add_tuple_literals();
   *element1->mutable_shape() =
-      ShapeUtil::GetTupleElementShape(proto.shape(), 1);
+      ShapeUtil::GetTupleElementShape(Shape(proto.shape()), 1).ToProto();
   element1->add_f32s(42.0);
   LiteralProto* element2 = proto.add_tuple_literals();
-  *element2->mutable_shape() = ShapeUtil::MakeShape(F32, {});
+  *element2->mutable_shape() = ShapeUtil::MakeShape(F32, {}).ToProto();
   element2->add_f32s(123.0);
 
   Status status = Literal::CreateFromProto(proto).status();
diff --git a/tensorflow/compiler/xla/python_api/xla_shape.py b/tensorflow/compiler/xla/python_api/xla_shape.py
index f158f6b241..95b2bf300e 100644
--- a/tensorflow/compiler/xla/python_api/xla_shape.py
+++ b/tensorflow/compiler/xla/python_api/xla_shape.py
@@ -25,9 +25,10 @@ from tensorflow.compiler.xla.python_api import types
 
 
 class Shape(object):
-  """Wraps a xla_data_pb2.Shape message with a convenient Python type.
+  """Wraps a xla_data_pb2.ShapeProto message with a convenient Python type.
 
-  Provides direct access to the underlying xla_data_pb2.Shape message in the
+  Provides direct access to the underlying xla_data_pb2.ShapeProto message in
+  the
   message attribute, along with accessor wrappers to the message's fields.
   Avoid direct access to .message unless interacting directly with protobuf APIs
   like CopyFrom. In other words, prefer hauling the shape around in a Shape, and
@@ -48,7 +49,7 @@ class Shape(object):
     Raises:
       ValueError: if element_type is TUPLE but dimensions are not Shape objects.
     """
-    self.message = xla_data_pb2.Shape()
+    self.message = xla_data_pb2.ShapeProto()
     self.message.element_type = element_type
     if element_type == xla_data_pb2.TUPLE:
       if not all(isinstance(subshape, Shape) for subshape in dimensions):
diff --git a/tensorflow/compiler/xla/rpc/BUILD b/tensorflow/compiler/xla/rpc/BUILD
index 3abb3855a4..26affbcceb 100644
--- a/tensorflow/compiler/xla/rpc/BUILD
+++ b/tensorflow/compiler/xla/rpc/BUILD
@@ -16,7 +16,6 @@ xla_proto_library(
     use_grpc_plugin = True,
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla:xla_proto",
     ],
 )
diff --git a/tensorflow/compiler/xla/rpc/xla_service.proto b/tensorflow/compiler/xla/rpc/xla_service.proto
index e4f332cda2..0ff8adc2ac 100644
--- a/tensorflow/compiler/xla/rpc/xla_service.proto
+++ b/tensorflow/compiler/xla/rpc/xla_service.proto
@@ -43,7 +43,6 @@ limitations under the License.
 syntax = "proto3";
 
 import "tensorflow/compiler/xla/xla.proto";
-import "tensorflow/compiler/xla/xla_data.proto";
 
 package xla;
 
diff --git a/tensorflow/compiler/xla/service/compile_only_service.cc b/tensorflow/compiler/xla/service/compile_only_service.cc
index 0237f16673..1965925fa7 100644
--- a/tensorflow/compiler/xla/service/compile_only_service.cc
+++ b/tensorflow/compiler/xla/service/compile_only_service.cc
@@ -89,7 +89,7 @@ CompileOnlyService::CompileAheadOfTime(
     ExecutionOptions execution_options;
     *execution_options.mutable_debug_options() = debug_options;
     *execution_options.mutable_shape_with_output_layout() =
-        *instance.result_layout;
+        instance.result_layout->ToProto();
     TF_ASSIGN_OR_RETURN(
         std::unique_ptr<HloModuleConfig> module_config,
         CreateModuleConfig(
diff --git a/tensorflow/compiler/xla/service/cpu/xfeed_manager.h b/tensorflow/compiler/xla/service/cpu/xfeed_manager.h
index 990ff94ba2..70008947f3 100644
--- a/tensorflow/compiler/xla/service/cpu/xfeed_manager.h
+++ b/tensorflow/compiler/xla/service/cpu/xfeed_manager.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include <deque>
 
 #include "absl/types/span.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index c62c935af7..414c632712 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -51,7 +51,7 @@ message HloInstructionProto {
 
   string name = 1;
   string opcode = 2;
-  xla.Shape shape = 3;
+  xla.ShapeProto shape = 3;
 
   xla.OpMetadata metadata = 7;
 
@@ -132,7 +132,7 @@ message HloInstructionProto {
   string custom_call_opaque = 53;
 
   // Shape of outfeed request.
-  xla.Shape outfeed_shape = 29;
+  xla.ShapeProto outfeed_shape = 29;
 
   // Describes the dimension numbers used for a dot operation
   xla.DotDimensionNumbers dot_dimension_numbers = 30;
@@ -190,7 +190,7 @@ message HloInstructionProto {
   // 'operand_shapes_with_layout' must contain a shape with layout for each
   // operand.
   bool constrain_layout = 56;
-  repeated Shape operand_shapes_with_layout = 57;
+  repeated xla.ShapeProto operand_shapes_with_layout = 57;
 }
 
 // Serialization of HloComputation.
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 1e3881c34f..2bdb617731 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -93,7 +93,8 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       [&computation_map](int64 id) { return computation_map.contains(id); }))
       << proto.name() << " instruction references invalid computation id(s)";
 
-  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(proto.shape()));
+  Shape shape(proto.shape());
+  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape));
 
   switch (opcode) {
     // Ops migrated to subclasses.
@@ -101,23 +102,23 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.operand_ids_size() == 3)
           << "BatchNormTraining instruction should have 3 operands but sees "
           << proto.operand_ids_size();
-      instruction = CreateBatchNormTraining(
-          proto.shape(), operands(0), operands(1), operands(2), proto.epsilon(),
-          proto.feature_index());
+      instruction =
+          CreateBatchNormTraining(shape, operands(0), operands(1), operands(2),
+                                  proto.epsilon(), proto.feature_index());
       break;
     case HloOpcode::kBatchNormInference:
       TF_RET_CHECK(proto.operand_ids_size() == 5)
           << "BatchNormInference instruction should have 5 operands but sees "
           << proto.operand_ids_size();
       instruction = CreateBatchNormInference(
-          proto.shape(), operands(0), operands(1), operands(2), operands(3),
+          shape, operands(0), operands(1), operands(2), operands(3),
           operands(4), proto.epsilon(), proto.feature_index());
       break;
     case HloOpcode::kBatchNormGrad:
       TF_RET_CHECK(proto.operand_ids_size() == 5)
           << "BatchNormGrad instruction should have 5 operands but sees "
           << proto.operand_ids_size();
-      instruction = CreateBatchNormGrad(proto.shape(), operands(0), operands(1),
+      instruction = CreateBatchNormGrad(shape, operands(0), operands(1),
                                         operands(2), operands(3), operands(4),
                                         proto.epsilon(), proto.feature_index());
       break;
@@ -127,7 +128,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << proto.operand_ids_size();
       std::vector<int64> fft_length(proto.fft_length().begin(),
                                     proto.fft_length().end());
-      instruction = CreateFft(proto.shape(), operands(0), proto.fft_type(),
+      instruction = CreateFft(shape, operands(0), proto.fft_type(),
                               absl::Span<const int64>(fft_length));
       break;
     }
@@ -148,7 +149,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "Recv instruction should have 1 operand but sees "
           << proto.operand_ids_size();
-      instruction = CreateRecv(proto.shape().tuple_shapes(0), operands(0),
+      instruction = CreateRecv(shape.tuple_shapes(0), operands(0),
                                proto.channel_id(), proto.is_host_transfer());
       break;
     case HloOpcode::kRecvDone:
@@ -161,7 +162,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "Reverse instruction should have 1 operand but sees "
           << proto.operand_ids_size();
-      instruction = CreateReverse(proto.shape(), operands(0),
+      instruction = CreateReverse(shape, operands(0),
                                   std::vector<int64>(proto.dimensions().begin(),
                                                      proto.dimensions().end()));
       break;
@@ -170,7 +171,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "Concatenate instruction should have 1 dimension but sees "
           << proto.dimensions_size();
       instruction =
-          CreateConcatenate(proto.shape(), all_operands(), proto.dimensions(0));
+          CreateConcatenate(shape, all_operands(), proto.dimensions(0));
       break;
     case HloOpcode::kReduce:
       TF_RET_CHECK(proto.operand_ids_size() % 2 == 0)
@@ -188,7 +189,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
             absl::MakeSpan(reduce_operands)
                 .subspan(reduce_operands.size() / 2, reduce_operands.size());
         instruction =
-            CreateReduce(proto.shape(), inputs, init_values,
+            CreateReduce(shape, inputs, init_values,
                          std::vector<int64>(proto.dimensions().begin(),
                                             proto.dimensions().end()),
                          computations(0));
@@ -203,7 +204,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       auto sort_operands = all_operands();
       HloInstruction* keys = sort_operands[0];
       instruction = CreateSort(
-          proto.shape(), proto.dimensions(0), keys,
+          shape, proto.dimensions(0), keys,
           absl::Span<HloInstruction* const>(sort_operands).subspan(1));
       break;
     }
@@ -212,7 +213,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "Transpose instruction should have 1 operand but sees "
           << proto.operand_ids_size();
       instruction =
-          CreateTranspose(proto.shape(), operands(0),
+          CreateTranspose(shape, operands(0),
                           std::vector<int64>(proto.dimensions().begin(),
                                              proto.dimensions().end()));
       break;
@@ -221,7 +222,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "Broadcast instruction should have 1 operand but sees "
           << proto.operand_ids_size();
       instruction =
-          CreateBroadcast(proto.shape(), operands(0),
+          CreateBroadcast(shape, operands(0),
                           std::vector<int64>(proto.dimensions().begin(),
                                              proto.dimensions().end()));
       break;
@@ -229,7 +230,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.called_computation_ids_size() == 1)
           << "Map instruction should have 1 called computation but sees "
           << proto.called_computation_ids_size();
-      instruction = CreateMap(proto.shape(), all_operands(), computations(0));
+      instruction = CreateMap(shape, all_operands(), computations(0));
       break;
     case HloOpcode::kSlice: {
       TF_RET_CHECK(proto.operand_ids_size() == 1)
@@ -242,8 +243,8 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
         slice_limits.push_back(slice_dimensions.limit());
         slice_strides.push_back(slice_dimensions.stride());
       }
-      instruction = CreateSlice(proto.shape(), operands(0), slice_starts,
-                                slice_limits, slice_strides);
+      instruction = CreateSlice(shape, operands(0), slice_starts, slice_limits,
+                                slice_strides);
       break;
     }
     case HloOpcode::kConstant: {
@@ -253,7 +254,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
                             Literal::CreateFromProto(proto.literal()));
         instruction = CreateConstant(std::move(literal));
       } else {
-        instruction = absl::make_unique<HloConstantInstruction>(proto.shape());
+        instruction = absl::make_unique<HloConstantInstruction>(shape);
       }
       break;
     }
@@ -284,55 +285,54 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           tensorflow::gtl::FindPtrOrNull(computation_map, fusion_id);
       TF_RET_CHECK(fused_computation != nullptr)
           << "No fusion computation with id " << fusion_id;
-      instruction = CreateFusion(proto.shape(), fusion_kind, all_operands(),
-                                 fused_computation);
+      instruction =
+          CreateFusion(shape, fusion_kind, all_operands(), fused_computation);
       break;
     }
     case HloOpcode::kRng:
-      instruction =
-          CreateRng(proto.shape(), proto.distribution(), all_operands());
+      instruction = CreateRng(shape, proto.distribution(), all_operands());
       break;
     case HloOpcode::kParameter:
-      instruction = CreateParameter(proto.parameter_number(), proto.shape(),
-                                    proto.name());
+      instruction =
+          CreateParameter(proto.parameter_number(), shape, proto.name());
       break;
     case HloOpcode::kGetTupleElement:
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "GetTupleElement instruction should have 1 operand but sees "
           << proto.operand_ids_size();
-      instruction = CreateGetTupleElement(proto.shape(), operands(0),
-                                          proto.tuple_index());
+      instruction =
+          CreateGetTupleElement(shape, operands(0), proto.tuple_index());
       break;
     case HloOpcode::kReducePrecision:
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "ReducePrecision instruction should have 1 operand but sees "
           << proto.operand_ids_size();
-      instruction =
-          CreateReducePrecision(proto.shape(), operands(0),
-                                proto.exponent_bits(), proto.mantissa_bits());
+      instruction = CreateReducePrecision(
+          shape, operands(0), proto.exponent_bits(), proto.mantissa_bits());
       break;
     case HloOpcode::kInfeed: {
-      TF_RET_CHECK(ShapeUtil::IsTuple(proto.shape()) &&
-                   (ShapeUtil::TupleElementCount(proto.shape()) == 2))
+      TF_RET_CHECK(ShapeUtil::IsTuple(shape) &&
+                   (ShapeUtil::TupleElementCount(shape) == 2))
           << "Infeed should have a tuple shape with 2 operands, but has: "
-          << proto.shape();
-      const Shape& data_shape =
-          ShapeUtil::GetTupleElementShape(proto.shape(), 0);
+          << shape;
+      const Shape& data_shape = ShapeUtil::GetTupleElementShape(shape, 0);
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "Infeed instruction should have 1 operand but sees "
           << proto.operand_ids_size();
       instruction =
           CreateInfeed(data_shape, operands(0), proto.infeed_config());
     } break;
-    case HloOpcode::kOutfeed:
+    case HloOpcode::kOutfeed: {
       TF_RET_CHECK(proto.operand_ids_size() == 2)
           << "Outfeed instruction should have 2 operands but sees "
           << proto.operand_ids_size();
+      Shape outfeed_shape(proto.outfeed_shape());
       TF_RETURN_IF_ERROR(
-          ShapeUtil::ValidateShapeWithOptionalLayout(proto.outfeed_shape()));
-      instruction = CreateOutfeed(proto.outfeed_shape(), operands(0),
-                                  operands(1), proto.outfeed_config());
+          ShapeUtil::ValidateShapeWithOptionalLayout(outfeed_shape));
+      instruction = CreateOutfeed(outfeed_shape, operands(0), operands(1),
+                                  proto.outfeed_config());
       break;
+    }
     case HloOpcode::kCrossReplicaSum: {
       TF_RET_CHECK(proto.called_computation_ids_size() == 1)
           << "CrossReplicaSum should have 1 called computation but sees "
@@ -342,7 +342,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
         all_reduce_id = proto.all_reduce_id();
       }
       instruction = CreateCrossReplicaSum(
-          proto.shape(), all_operands(), computations(0),
+          shape, all_operands(), computations(0),
           /*replica_groups=*/
           std::vector<ReplicaGroup>(proto.replica_groups().begin(),
                                     proto.replica_groups().end()),
@@ -352,7 +352,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     }
     case HloOpcode::kAllToAll: {
       instruction = CreateAllToAll(
-          proto.shape(), all_operands(),
+          shape, all_operands(),
           /*replica_groups=*/
           std::vector<ReplicaGroup>(proto.replica_groups().begin(),
                                     proto.replica_groups().end()));
@@ -368,8 +368,8 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
         source_target_pairs[i].first = proto.source_target_pairs(i).source();
         source_target_pairs[i].second = proto.source_target_pairs(i).target();
       }
-      instruction = CreateCollectivePermute(proto.shape(), operands(0),
-                                            source_target_pairs);
+      instruction =
+          CreateCollectivePermute(shape, operands(0), source_target_pairs);
       break;
     }
     case HloOpcode::kConvolution: {
@@ -382,7 +382,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       precision_config.mutable_operand_precision()->Resize(
           proto.operand_ids_size(), PrecisionConfig::DEFAULT);
       instruction = CreateConvolve(
-          proto.shape(), operands(0), operands(1),
+          shape, operands(0), operands(1),
           std::max<int64>(proto.feature_group_count(), 1), proto.window(),
           proto.convolution_dimension_numbers(), precision_config);
       break;
@@ -394,7 +394,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.called_computation_ids_size() == 1)
           << "ReduceWindow should have 1 called computation but sees "
           << proto.called_computation_ids_size();
-      instruction = CreateReduceWindow(proto.shape(), operands(0), operands(1),
+      instruction = CreateReduceWindow(shape, operands(0), operands(1),
                                        proto.window(), computations(0));
       break;
     case HloOpcode::kSelectAndScatter:
@@ -404,9 +404,9 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.called_computation_ids_size() == 2)
           << "SelectAndScatter should have 2 called computations but sees "
           << proto.called_computation_ids_size();
-      instruction = CreateSelectAndScatter(
-          proto.shape(), operands(0), computations(0), proto.window(),
-          operands(1), operands(2), computations(1));
+      instruction = CreateSelectAndScatter(shape, operands(0), computations(0),
+                                           proto.window(), operands(1),
+                                           operands(2), computations(1));
       break;
     case HloOpcode::kCustomCall:
       if (proto.constrain_layout()) {
@@ -414,16 +414,17 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
         // vector of pointers essentially) so create a vector of shapes to pass
         // in.
         std::vector<Shape> operand_shapes;
-        for (const Shape& shape : proto.operand_shapes_with_layout()) {
-          operand_shapes.push_back(shape);
+        for (const ShapeProto& shape_proto :
+             proto.operand_shapes_with_layout()) {
+          operand_shapes.emplace_back(shape_proto);
         }
-        instruction = CreateCustomCall(
-            proto.shape(), all_operands(), proto.custom_call_target(),
-            operand_shapes, proto.custom_call_opaque());
+        instruction =
+            CreateCustomCall(shape, all_operands(), proto.custom_call_target(),
+                             operand_shapes, proto.custom_call_opaque());
       } else {
-        instruction = CreateCustomCall(proto.shape(), all_operands(),
-                                       proto.custom_call_target(),
-                                       proto.custom_call_opaque());
+        instruction =
+            CreateCustomCall(shape, all_operands(), proto.custom_call_target(),
+                             proto.custom_call_opaque());
       }
       if (proto.has_window()) {
         static_cast<HloCustomCallInstruction*>(instruction.get())
@@ -443,8 +444,8 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "Pad instruction should have 2 operands but sees "
           << proto.operand_ids_size();
       TF_RET_CHECK(proto.has_padding_config());
-      instruction = CreatePad(proto.shape(), operands(0), operands(1),
-                              proto.padding_config());
+      instruction =
+          CreatePad(shape, operands(0), operands(1), proto.padding_config());
       break;
     case HloOpcode::kDynamicSlice: {
       TF_RET_CHECK(proto.operand_ids_size() == 2)
@@ -452,8 +453,8 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << proto.operand_ids_size();
       std::vector<int64> slice_sizes(proto.dynamic_slice_sizes_size());
       absl::c_copy(proto.dynamic_slice_sizes(), slice_sizes.begin());
-      instruction = CreateDynamicSlice(proto.shape(), operands(0), operands(1),
-                                       slice_sizes);
+      instruction =
+          CreateDynamicSlice(shape, operands(0), operands(1), slice_sizes);
       break;
     }
     case HloOpcode::kGather: {
@@ -469,7 +470,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       for (int64 bound : proto.gather_slice_sizes()) {
         gather_slice_sizes.push_back(bound);
       }
-      instruction = CreateGather(proto.shape(), operands(0), operands(1),
+      instruction = CreateGather(shape, operands(0), operands(1),
                                  *gather_dimension_numbers, gather_slice_sizes);
       break;
     }
@@ -485,16 +486,15 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       auto scatter_dimension_numbers =
           absl::make_unique<ScatterDimensionNumbers>(
               proto.scatter_dimension_numbers());
-      instruction =
-          CreateScatter(proto.shape(), operands(0), operands(1), operands(2),
-                        computations(0), *scatter_dimension_numbers);
+      instruction = CreateScatter(shape, operands(0), operands(1), operands(2),
+                                  computations(0), *scatter_dimension_numbers);
       break;
     }
     case HloOpcode::kIota:
       TF_RET_CHECK(proto.dimensions_size() == 1)
           << "Iota instruction should have 1 dimension but sees "
           << proto.dimensions_size();
-      instruction = CreateIota(proto.shape(), proto.dimensions(0));
+      instruction = CreateIota(shape, proto.dimensions(0));
       break;
     case HloOpcode::kDot: {
       TF_RET_CHECK(proto.has_dot_dimension_numbers())
@@ -506,8 +506,8 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       precision_config.mutable_operand_precision()->Resize(
           proto.operand_ids_size(), PrecisionConfig::DEFAULT);
       instruction = absl::make_unique<HloDotInstruction>(
-          proto.shape(), operands(0), operands(1),
-          proto.dot_dimension_numbers(), precision_config);
+          shape, operands(0), operands(1), proto.dot_dimension_numbers(),
+          precision_config);
       break;
     }
     case HloOpcode::kDomain: {
@@ -529,7 +529,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
         exit_hlo_sharding = std::make_shared<const HloSharding>(sharding);
       }
       instruction = absl::make_unique<HloDomainInstruction>(
-          proto.shape(), operands(0),
+          shape, operands(0),
           absl::make_unique<ShardingMetadata>(entry_hlo_sharding),
           absl::make_unique<ShardingMetadata>(exit_hlo_sharding));
       break;
@@ -537,11 +537,11 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     case HloOpcode::kGetDimensionSize:
       TF_RET_CHECK(proto.operand_ids_size() == 1);
       TF_RET_CHECK(proto.dimensions_size() == 1);
-      instruction = CreateGetDimensionSize(proto.shape(), operands(0),
-                                           proto.dimensions(0));
+      instruction =
+          CreateGetDimensionSize(shape, operands(0), proto.dimensions(0));
       break;
     default: {
-      instruction = absl::WrapUnique(new HloInstruction(opcode, proto.shape()));
+      instruction = absl::WrapUnique(new HloInstruction(opcode, shape));
       for (const int64 operand_id : proto.operand_ids()) {
         instruction->AppendOperand(instruction_map.at(operand_id));
       }
@@ -2234,7 +2234,7 @@ HloInstructionProto HloInstruction::ToProto() const {
   proto.set_id(unique_id_);
   proto.set_name(name_);
   proto.set_opcode(HloOpcodeString(opcode_));
-  *proto.mutable_shape() = shape_;
+  *proto.mutable_shape() = shape_.ToProto();
   for (const HloInstruction* operand : operands_) {
     proto.add_operand_ids(operand->unique_id());
   }
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index ed3b2f1564..6a57b75488 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -1615,7 +1615,7 @@ HloOutfeedInstruction::HloOutfeedInstruction(const Shape& outfeed_shape,
 HloInstructionProto HloOutfeedInstruction::ToProto() const {
   HloInstructionProto proto = HloInstruction::ToProto();
   proto.set_outfeed_config(outfeed_config());
-  *proto.mutable_outfeed_shape() = outfeed_shape();
+  *proto.mutable_outfeed_shape() = outfeed_shape().ToProto();
   return proto;
 }
 
@@ -1867,7 +1867,7 @@ HloInstructionProto HloCustomCallInstruction::ToProto() const {
   if (layout_constrained()) {
     proto.set_constrain_layout(true);
     for (const Shape& shape : operand_shapes_with_layout_) {
-      *proto.add_operand_shapes_with_layout() = shape;
+      *proto.add_operand_shapes_with_layout() = shape.ToProto();
     }
   }
   return proto;
diff --git a/tensorflow/compiler/xla/service/hlo_lexer.h b/tensorflow/compiler/xla/service/hlo_lexer.h
index 3e2f8bcd52..d6a2b292a3 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.h
+++ b/tensorflow/compiler/xla/service/hlo_lexer.h
@@ -20,6 +20,7 @@ limitations under the License.
 
 #include "absl/strings/string_view.h"
 #include "tensorflow/compiler/xla/service/hlo_token.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index a01853fe1f..fe8371384c 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -257,7 +257,7 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   // the entry parameters and root.
   TF_RET_CHECK(proto.has_host_program_shape())
       << "No program shape found in the proto";
-  const auto& expected_program_shape = proto.host_program_shape();
+  ProgramShape expected_program_shape(proto.host_program_shape());
   TF_RET_CHECK(expected_program_shape.parameters_size() ==
                module_config.entry_computation_layout().parameter_count());
   for (int i = 0; i < expected_program_shape.parameters_size(); ++i) {
@@ -369,7 +369,7 @@ StatusOr<HloModuleConfig> HloModule::CreateModuleConfigFromProto(
     const HloModuleProto& module, const DebugOptions& debug_options) {
   TF_RET_CHECK(module.has_host_program_shape())
       << "No program shape found in the proto";
-  const auto& program_shape = module.host_program_shape();
+  ProgramShape program_shape(module.host_program_shape());
 
   HloModuleConfig module_config(ProgramShape{program_shape});
   module_config.set_debug_options(debug_options);
diff --git a/tensorflow/compiler/xla/service/hlo_proto_util.cc b/tensorflow/compiler/xla/service/hlo_proto_util.cc
index cf33668f5b..981d06ce10 100644
--- a/tensorflow/compiler/xla/service/hlo_proto_util.cc
+++ b/tensorflow/compiler/xla/service/hlo_proto_util.cc
@@ -48,7 +48,7 @@ StatusOr<std::unique_ptr<HloModule>> CreateModuleFromProto(
   return std::move(module);
 }
 
-StatusOr<std::vector<const Shape*>> EntryComputationParameterShapes(
+StatusOr<std::vector<const ShapeProto*>> EntryComputationParameterShapes(
     const HloProto& hlo_proto) {
   if (!hlo_proto.has_hlo_module()) {
     return NotFound("HloProto missing HloModuleProto.");
@@ -57,15 +57,16 @@ StatusOr<std::vector<const Shape*>> EntryComputationParameterShapes(
     return NotFound("HloProto missing program shape.");
   }
 
-  std::vector<const Shape*> parameter_shapes;
+  std::vector<const ShapeProto*> parameter_shapes;
   const auto& program_shape = hlo_proto.hlo_module().host_program_shape();
-  for (const Shape& shape : program_shape.parameters()) {
+  for (const ShapeProto& shape : program_shape.parameters()) {
     parameter_shapes.push_back(&shape);
   }
   return parameter_shapes;
 }
 
-StatusOr<const Shape*> EntryComputationOutputShape(const HloProto& hlo_proto) {
+StatusOr<const ShapeProto*> EntryComputationOutputShape(
+    const HloProto& hlo_proto) {
   if (!hlo_proto.has_hlo_module()) {
     return NotFound("HloProto missing HloModuleProto.");
   }
diff --git a/tensorflow/compiler/xla/service/hlo_proto_util.h b/tensorflow/compiler/xla/service/hlo_proto_util.h
index 1db82dd6fc..31ea2aaffd 100644
--- a/tensorflow/compiler/xla/service/hlo_proto_util.h
+++ b/tensorflow/compiler/xla/service/hlo_proto_util.h
@@ -43,12 +43,13 @@ StatusOr<std::unique_ptr<HloModule>> CreateModuleFromProto(
 
 // Returns the shapes of the parameters of the entry computation. Shape pointers
 // refer to shapes inside of the given HloProto.
-StatusOr<std::vector<const Shape*>> EntryComputationParameterShapes(
+StatusOr<std::vector<const ShapeProto*>> EntryComputationParameterShapes(
     const HloProto& hlo_proto);
 
 // Returns the shape of the output of the entry computation. The shape pointer
 // refers to the output shape inside of the given HloProto.
-StatusOr<const Shape*> EntryComputationOutputShape(const HloProto& hlo_proto);
+StatusOr<const ShapeProto*> EntryComputationOutputShape(
+    const HloProto& hlo_proto);
 
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index 1540a40ef8..d6d84994ee 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Value.h"
 #include "tensorflow/compiler/xla/map_util.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index df78726166..ceea24685a 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -244,10 +244,11 @@ StatusOr<llvm::Value*> EncodeSelfDescribingShapeConstant(const Shape& shape,
 
 StatusOr<Shape> DecodeSelfDescribingShapeConstant(const void* shape_ptr,
                                                   int32 size_bytes) {
-  Shape shape;
-  TF_RET_CHECK(shape.ParseFromArray(shape_ptr, size_bytes));
+  ShapeProto shape_proto;
+  TF_RET_CHECK(shape_proto.ParseFromArray(shape_ptr, size_bytes));
+  Shape shape(shape_proto);
   TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape));
-  return shape;
+  return std::move(shape);
 }
 
 llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal,
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index ddc8691e77..6c89700983 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -101,12 +101,12 @@ ExecutionOptions CreateExecutionOptions(
   }
   if (build_options.result_layout() != nullptr) {
     *execution_options.mutable_shape_with_output_layout() =
-        *build_options.result_layout();
+        build_options.result_layout()->ToProto();
   } else {
+    Shape result_shape(program_shape->result());
+    LayoutUtil::SetToDefaultLayout(&result_shape);
     *execution_options.mutable_shape_with_output_layout() =
-        program_shape->result();
-    LayoutUtil::SetToDefaultLayout(
-        execution_options.mutable_shape_with_output_layout());
+        result_shape.ToProto();
   }
   return execution_options;
 }
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index c4b0a5c080..3b336d5c9d 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -41,6 +41,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/source_map_util.h"
 #include "tensorflow/compiler/xla/service/stream_pool.h"
 #include "tensorflow/compiler/xla/service/transfer_manager.h"
+#include "tensorflow/compiler/xla/shape.h"
 #include "tensorflow/compiler/xla/shape_layout.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
@@ -275,8 +276,8 @@ StatusOr<std::unique_ptr<HloModuleConfig>> Service::CreateModuleConfig(
   }
   if (execution_options != nullptr &&
       execution_options->has_shape_with_output_layout()) {
-    const auto& shape_with_output_layout =
-        execution_options->shape_with_output_layout();
+    const Shape shape_with_output_layout(
+        execution_options->shape_with_output_layout());
     TF_RETURN_IF_ERROR(
         ValidateResultShape(shape_with_output_layout, program_shape.result()));
     TF_RETURN_IF_ERROR(
@@ -818,14 +819,17 @@ Status Service::Compile(const CompileRequest* arg, CompileResponse* result) {
         "The compile request does not support multiple device handles.");
   }
 
-  std::vector<const Shape*> argument_shapes;
-  absl::c_transform(arg->input_shape_with_layout(),
-                    std::back_inserter(argument_shapes),
-                    [](const Shape& shape) { return &shape; });
+  std::vector<Shape> argument_shapes;
+  argument_shapes.reserve(arg->input_shape_with_layout_size());
+  std::vector<const Shape*> argument_shape_ptrs;
+  for (const ShapeProto& shape_proto : arg->input_shape_with_layout()) {
+    argument_shapes.push_back(Shape(shape_proto));
+    argument_shape_ptrs.push_back(&argument_shapes.back());
+  }
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr<HloModuleConfig> module_config,
       CreateModuleConfig(ProgramShape{arg->computation().host_program_shape()},
-                         argument_shapes, &arg->execution_options()));
+                         argument_shape_ptrs, &arg->execution_options()));
   VLOG(3) << "Compile created HloModuleConfig computation layout: "
           << module_config->entry_computation_layout().ToString();
 
@@ -930,14 +934,14 @@ Status Service::TransferToClient(const TransferToClientRequest* arg,
   TF_ASSIGN_OR_RETURN(const ShapedBuffer* shaped_buffer,
                       allocation_tracker_.ResolveForReplica(arg->data(), 0));
 
-  const Shape* return_shape;
+  Shape return_shape;
   if (arg->has_shape_with_layout()) {
-    if (!LayoutUtil::HasLayout(arg->shape_with_layout())) {
+    return_shape = Shape(arg->shape_with_layout());
+    if (!LayoutUtil::HasLayout(return_shape)) {
       return InvalidArgument("shape_with_layout must have layout if present.");
     }
-    return_shape = &arg->shape_with_layout();
   } else {
-    return_shape = &shaped_buffer->on_host_shape();
+    return_shape = Shape(shaped_buffer->on_host_shape());
   }
 
   TF_ASSIGN_OR_RETURN(auto stream, execute_backend_->BorrowStream(
@@ -948,11 +952,11 @@ Status Service::TransferToClient(const TransferToClientRequest* arg,
       execute_backend_->transfer_manager()->TransferLiteralFromDevice(
           stream.get(), *shaped_buffer));
 
-  if (LayoutUtil::LayoutsInShapesEqual(*return_shape, result_literal.shape())) {
+  if (LayoutUtil::LayoutsInShapesEqual(return_shape, result_literal.shape())) {
     *result->mutable_literal() = result_literal.ToProto();
   } else {
     *result->mutable_literal() =
-        result_literal.Relayout(*return_shape).ToProto();
+        result_literal.Relayout(return_shape).ToProto();
   }
   return Status::OK();
 }
@@ -1045,11 +1049,11 @@ Status Service::TransferFromOutfeed(const TransferFromOutfeedRequest* arg,
     executor = replicas[arg->replica_id()];
   }
 
-  auto literal = Literal::CreateFromShape(arg->shape_with_layout());
+  auto literal = Literal::CreateFromShape(Shape(arg->shape_with_layout()));
 
   TF_RETURN_IF_ERROR(
       execute_backend_->transfer_manager()->TransferLiteralFromOutfeed(
-          executor, arg->shape_with_layout(), literal));
+          executor, Shape(arg->shape_with_layout()), literal));
   *result->mutable_literal() = literal.ToProto();
   return Status::OK();
 }
@@ -1103,7 +1107,7 @@ Status Service::ComputeConstantGraph(const ComputeConstantGraphRequest* arg,
 Status Service::GetShape(const GetShapeRequest* arg, GetShapeResponse* result) {
   TF_ASSIGN_OR_RETURN(const ShapedBuffer* buffer,
                       allocation_tracker_.ResolveForReplica(arg->data(), 0));
-  *result->mutable_shape() = buffer->on_host_shape();
+  *result->mutable_shape() = buffer->on_host_shape().ToProto();
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 528d5c0ecc..7e7282a737 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1018,7 +1018,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   switch (opcode) {
     case HloOpcode::kTuple: {
       Shape result = ShapeUtil::MakeTupleShape({});
-      result.mutable_tuple_shapes()->Reserve(operand_shapes.size());
+      result.mutable_tuple_shapes()->reserve(operand_shapes.size());
       for (const Shape* shape : operand_shapes) {
         ShapeUtil::AppendShapeToTuple(*shape, &result);
       }
diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index d209389c74..746ab9e997 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -21,11 +21,56 @@ limitations under the License.
 
 namespace xla {
 
+Shape::Shape(const ShapeProto& shape_proto) {
+  set_element_type(shape_proto.element_type());
+  dimensions_.reserve(shape_proto.dimensions_size());
+  for (const int64 dimension : shape_proto.dimensions()) {
+    add_dimensions(dimension);
+  }
+  tuple_shapes_.reserve(shape_proto.tuple_shapes_size());
+  for (const ShapeProto& element_shape : shape_proto.tuple_shapes()) {
+    *add_tuple_shapes() = Shape(element_shape);
+  }
+  if (shape_proto.has_layout()) {
+    *mutable_layout() = shape_proto.layout();
+  }
+}
+
+ShapeProto Shape::ToProto() const {
+  ShapeProto proto;
+  proto.set_element_type(element_type_);
+  proto.mutable_dimensions()->Reserve(dimensions_size());
+  for (const int64 dimension : dimensions()) {
+    proto.add_dimensions(dimension);
+  }
+  proto.mutable_tuple_shapes()->Reserve(tuple_shapes_size());
+  for (const Shape& shape : tuple_shapes()) {
+    *proto.add_tuple_shapes() = shape.ToProto();
+  }
+  if (has_layout()) {
+    *proto.mutable_layout() = layout();
+  }
+  return proto;
+}
+
+string Shape::ToString(bool print_layout) const {
+  if (print_layout) {
+    return ShapeUtil::HumanStringWithLayout(*this);
+  } else {
+    return ShapeUtil::HumanString(*this);
+  }
+}
+
+std::ostream& operator<<(std::ostream& out, const Shape& shape) {
+  out << shape.ToString(/*print_layout=*/true);
+  return out;
+}
+
 ProgramShape::ProgramShape(const ProgramShapeProto& program_shape_proto) {
-  for (const Shape& shape : program_shape_proto.parameters()) {
-    *add_parameters() = shape;
+  for (const ShapeProto& shape_proto : program_shape_proto.parameters()) {
+    *add_parameters() = Shape(shape_proto);
   }
-  *mutable_result() = program_shape_proto.result();
+  *mutable_result() = Shape(program_shape_proto.result());
   for (const string& name : program_shape_proto.parameter_names()) {
     add_parameter_names(name);
   }
@@ -34,9 +79,9 @@ ProgramShape::ProgramShape(const ProgramShapeProto& program_shape_proto) {
 ProgramShapeProto ProgramShape::ToProto() const {
   ProgramShapeProto proto;
   for (const Shape& shape : parameters()) {
-    *proto.add_parameters() = shape;
+    *proto.add_parameters() = shape.ToProto();
   }
-  *proto.mutable_result() = result();
+  *proto.mutable_result() = result().ToProto();
   for (const string& name : parameter_names()) {
     proto.add_parameter_names(name);
   }
diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
index c3aecb1736..7f6b14ab42 100644
--- a/tensorflow/compiler/xla/shape.h
+++ b/tensorflow/compiler/xla/shape.h
@@ -26,6 +26,102 @@ limitations under the License.
 
 namespace xla {
 
+// A shape describes the number of dimensions in a array, the bounds of each
+// dimension, and the primitive component type. For tuples, shape describes the
+// structure (number of elements and nesting).
+class Shape {
+ public:
+  Shape() = default;
+
+  // Construct a shape from a ShapeProto.
+  explicit Shape(const ShapeProto& shape_proto);
+
+  // Returns a ShapeProto representation of the Shape.
+  ShapeProto ToProto() const;
+
+  // Returns a human-readable string that represents the given shape, with or
+  // without layout. e.g. "F32[42,12] {0, 1}" or "F32[64]".
+  string ToString(bool print_layout = false) const;
+
+  // The following methods mirror the protobuf generated code interface for the
+  // message ShapeProto. This enabled easy migration of this data structure
+  // from a proto to a proper C++ class.
+  // TODO(b/29771030): Replace or augment these methods with a more ergonomic
+  // interface.
+
+  // Methods for accessing the primitive type.
+  PrimitiveType element_type() const { return element_type_; }
+  void set_element_type(PrimitiveType value) { element_type_ = value; }
+
+  // Methods for accessing the dimensions array.
+  int dimensions_size() const { return dimensions_.size(); }
+  int64 dimensions(int index) const { return dimensions_.at(index); }
+  void set_dimensions(int index, int64 value) { dimensions_.at(index) = value; }
+  void add_dimensions(int64 value) { dimensions_.push_back(value); }
+  void clear_dimensions() { dimensions_.clear(); }
+  const std::vector<int64>& dimensions() const { return dimensions_; }
+  std::vector<int64>* mutable_dimensions() { return &dimensions_; }
+
+  // Methods for accessing the tuple subshapes. This field only non-empty for
+  // tuple shapes.
+  int tuple_shapes_size() const { return tuple_shapes_.size(); }
+  const Shape& tuple_shapes(int index) const { return tuple_shapes_.at(index); }
+  Shape* mutable_tuple_shapes(int index) { return &tuple_shapes_.at(index); }
+  Shape* add_tuple_shapes() {
+    tuple_shapes_.push_back(Shape());
+    return &tuple_shapes_.back();
+  }
+  void clear_tuple_shapes() { tuple_shapes_.clear(); }
+  const std::vector<Shape>& tuple_shapes() const { return tuple_shapes_; }
+  std::vector<Shape>* mutable_tuple_shapes() { return &tuple_shapes_; }
+
+  // Methods for accessing the layout field.
+  bool has_layout() const { return layout_.has_value(); }
+  const Layout& layout() const {
+    if (layout_.has_value()) {
+      return *layout_;
+    } else {
+      return Layout::default_instance();
+    }
+  }
+  Layout* mutable_layout() {
+    if (!layout_.has_value()) {
+      layout_ = Layout();
+    }
+    return &layout_.value();
+  }
+  void clear_layout() { layout_.reset(); }
+
+  void Swap(Shape* other) {
+    using std::swap;
+    swap(*this, *other);
+  }
+
+  void Clear() {
+    element_type_ = PRIMITIVE_TYPE_INVALID;
+    dimensions_.clear();
+    tuple_shapes_.clear();
+    layout_.reset();
+  }
+
+  string SerializeAsString() const { return ToProto().SerializeAsString(); }
+  string ShortDebugString() const { return ToProto().ShortDebugString(); }
+  string DebugString() const { return ToProto().DebugString(); }
+
+ public:
+  // The element type of this shape (tuple, array, etc).
+  PrimitiveType element_type_ = PRIMITIVE_TYPE_INVALID;
+
+  // The array bounds of the dimensions. This is nonempty only for array shapes.
+  std::vector<int64> dimensions_;
+
+  // The tuple element subshapes. This is nonempty only for tuple shapes.
+  std::vector<Shape> tuple_shapes_;
+
+  // The array layout of the shape. This is present only for array shapes.
+  absl::optional<Layout> layout_;
+};
+
 // Shape of the parameters and output of an XLA computation. This is analogous
 // to a traditional function signature.
 class ProgramShape {
@@ -61,7 +157,6 @@ class ProgramShape {
   // Methods for accessing and manipulating the Shape of the result.
   const Shape& result() const { return result_; }
   Shape* mutable_result() { return &result_; }
-  void clear_result() { result_.Clear(); }
 
   // Methods for accessing and manipulating the names of the parameters.
   int parameter_names_size() const { return parameter_names_.size(); }
@@ -101,6 +196,7 @@ class ProgramShape {
   Shape result_;
 };
 
+std::ostream& operator<<(std::ostream& out, const Shape& shape);
 std::ostream& operator<<(std::ostream& out, const ProgramShape& program_shape);
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/shape_test.cc b/tensorflow/compiler/xla/shape_test.cc
index cc3a5eb1d6..e396897eee 100644
--- a/tensorflow/compiler/xla/shape_test.cc
+++ b/tensorflow/compiler/xla/shape_test.cc
@@ -30,7 +30,51 @@ limitations under the License.
 namespace xla {
 namespace {
 
-TEST(ShapeTest, ProgramShapeToFromProto) {
+class ShapeTest : public ::testing::Test {
+ protected:
+  const Shape opaque_ = ShapeUtil::MakeOpaqueShape();
+  const Shape token_ = ShapeUtil::MakeTokenShape();
+  const Shape scalar_ = ShapeUtil::MakeShape(F32, {});
+  const Shape matrix_ = ShapeUtil::MakeShape(U32, {1, 2});
+  const Shape matrix2_ = ShapeUtil::MakeShapeWithLayout(S32, {3, 4}, {0, 1});
+  const Shape tuple_ =
+      ShapeUtil::MakeTupleShape({opaque_, scalar_, matrix_, matrix2_});
+  const Shape nested_tuple_ =
+      ShapeUtil::MakeTupleShape({tuple_, matrix_, token_});
+};
+
+TEST_F(ShapeTest, ShapeToFromProto) {
+  for (const Shape& shape :
+       {opaque_, token_, scalar_, matrix_, matrix2_, tuple_, nested_tuple_}) {
+    Shape shape_copy(shape.ToProto());
+    EXPECT_TRUE(ShapeUtil::Equal(shape, shape_copy))
+        << shape << " != " << shape_copy;
+  }
+}
+
+TEST_F(ShapeTest, ShapeToString) {
+  EXPECT_EQ("opaque[]", opaque_.ToString());
+  EXPECT_EQ("token[]", token_.ToString());
+  EXPECT_EQ("f32[]", scalar_.ToString());
+  EXPECT_EQ("u32[1,2]", matrix_.ToString());
+  EXPECT_EQ("s32[3,4]", matrix2_.ToString());
+  EXPECT_EQ("(opaque[], f32[], u32[1,2], s32[3,4])", tuple_.ToString());
+  EXPECT_EQ("((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
+            nested_tuple_.ToString());
+
+  EXPECT_EQ("opaque[]", opaque_.ToString(/*print_layout=*/true));
+  EXPECT_EQ("f32[]", scalar_.ToString(/*print_layout=*/true));
+  EXPECT_EQ("u32[1,2]{1,0}", matrix_.ToString(/*print_layout=*/true));
+  EXPECT_EQ("s32[3,4]{0,1}", matrix2_.ToString(/*print_layout=*/true));
+  EXPECT_EQ("(opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1})",
+            tuple_.ToString(/*print_layout=*/true));
+  EXPECT_EQ(
+      "((opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1}), u32[1,2]{1,0}, "
+      "token[])",
+      nested_tuple_.ToString(/*print_layout=*/true));
+}
+
+TEST_F(ShapeTest, ProgramShapeToFromProto) {
   ProgramShape program_shape;
   *program_shape.add_parameters() = ShapeUtil::MakeShape(F32, {1, 2, 3});
   *program_shape.add_parameters() = ShapeUtil::MakeTokenShape();
@@ -67,17 +111,10 @@ TEST(ShapeTest, ProgramShapeToFromProto) {
   }
 }
 
-TEST(ShapeTest, ProgramShapeToString) {
-  Shape opaque = ShapeUtil::MakeOpaqueShape();
-  Shape token = ShapeUtil::MakeTokenShape();
-  Shape scalar = ShapeUtil::MakeShape(F32, {});
-  Shape matrix = ShapeUtil::MakeShape(U32, {1, 2});
-  Shape matrix2 = ShapeUtil::MakeShapeWithLayout(S32, {3, 4}, {0, 1});
-  Shape tuple = ShapeUtil::MakeTupleShape({opaque, scalar, matrix, matrix2});
-  Shape nested_tuple = ShapeUtil::MakeTupleShape({tuple, matrix, token});
-
+TEST_F(ShapeTest, ProgramShapeToString) {
   ProgramShape prog = ShapeUtil::MakeProgramShape(
-      {opaque, scalar, matrix, matrix2, tuple, nested_tuple}, nested_tuple);
+      {opaque_, scalar_, matrix_, matrix2_, tuple_, nested_tuple_},
+      nested_tuple_);
   EXPECT_EQ(
       "((unknown): opaque[], "
       "(unknown): f32[], "
@@ -87,7 +124,7 @@ TEST(ShapeTest, ProgramShapeToString) {
       "(unknown): ((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])) "
       "-> "
       "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
-      ShapeUtil::HumanString(prog));
+      prog.ToString());
 
   prog.add_parameter_names("arg0");
   prog.add_parameter_names("scalar");
@@ -105,7 +142,7 @@ TEST(ShapeTest, ProgramShapeToString) {
       "token[])) "
       "-> "
       "((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
-      ShapeUtil::HumanString(prog));
+      prog.ToString());
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index b05ec209cc..82662b1dbb 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -79,14 +79,14 @@ bool ShapeIndexView::StartsWith(ShapeIndexView prefix) const {
          indices_.subspan(0, prefix.size()) == prefix.indices_;
 }
 
-namespace {
-
-// Returns whether the given primitive type corresponds to an array shape.
-bool IsArrayPrimitiveType(PrimitiveType primitive_type) {
+/* static */ bool ShapeUtil::IsArrayPrimitiveType(
+    PrimitiveType primitive_type) {
   return primitive_type != PRIMITIVE_TYPE_INVALID && primitive_type != TUPLE &&
          primitive_type != OPAQUE && primitive_type != TOKEN;
 }
 
+namespace {
+
 // Recursive helper for comparing the equality of two shapes. Returns true if
 // the shapes are the same. If compare_layouts is true, then layouts must also
 // match.
@@ -203,7 +203,7 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
 /* static */ ProgramShape ShapeUtil::MakeProgramShape(
     std::initializer_list<Shape> parameters, Shape result) {
   ProgramShape program_shape;
-  for (const auto& shape : parameters) {
+  for (const Shape& shape : parameters) {
     *program_shape.add_parameters() = shape;
   }
   *program_shape.mutable_result() = std::move(result);
@@ -272,7 +272,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ Shape ShapeUtil::MakeTupleShape(absl::Span<const Shape> shapes) {
   Shape result;
   result.set_element_type(TUPLE);
-  result.mutable_tuple_shapes()->Reserve(shapes.size());
+  result.mutable_tuple_shapes()->reserve(shapes.size());
   for (const auto& shape : shapes) {
     AppendShapeToTuple(shape, &result);
   }
@@ -563,20 +563,6 @@ StatusOr<PrimitiveType> StringToPrimitiveType(const string& name) {
                 HumanString(program_shape.result()));
 }
 
-/* static */ string ShapeUtil::HumanString(
-    const ProgramShapeProto& program_shape_proto) {
-  std::vector<string> parameters;
-  for (auto& shape : program_shape_proto.parameters()) {
-    const int i = parameters.size();
-    parameters.push_back(StrCat(i < program_shape_proto.parameter_names_size()
-                                    ? program_shape_proto.parameter_names(i)
-                                    : "(unknown)",
-                                ": ", HumanString(shape)));
-  }
-  return StrCat("(", absl::StrJoin(parameters, ", "), ") -> ",
-                HumanString(program_shape_proto.result()));
-}
-
 namespace {
 // Parses shapes with simple recursive descent structure -- consumes from the
 // front of s and passes that view recursively as required.
@@ -1610,7 +1596,8 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 /* static */ Shape ShapeUtil::DeleteDimension(int64 dim_to_delete,
                                               Shape shape) {
   CHECK(IsArray(shape));
-  shape.mutable_dimensions()->erase(shape.dimensions().begin() + dim_to_delete);
+  shape.mutable_dimensions()->erase(shape.mutable_dimensions()->begin() +
+                                    dim_to_delete);
   if (LayoutUtil::HasLayout(shape)) {
     Layout* layout = shape.mutable_layout();
     layout->set_format(DENSE);
@@ -1644,11 +1631,6 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
   return shape;
 }
 
-std::ostream& operator<<(std::ostream& out, const Shape& shape) {
-  out << ShapeUtil::HumanStringWithLayout(shape);
-  return out;
-}
-
 /*static*/ size_t ShapeUtil::Hash(const Shape& shape) {
   using tensorflow::hash;
   using tensorflow::Hash64Combine;
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index 3796c5be5d..84a27f662a 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -240,7 +240,6 @@ class ShapeUtil {
   //
   // (param_name: f32[42x12], ...) -> f32[24x42]
   static string HumanString(const ProgramShape& program_shape);
-  static string HumanString(const ProgramShapeProto& program_shape_proto);
 
   // Parses a ShapeUtil::HumanString-format shape string back into a shape
   // object.
@@ -469,6 +468,9 @@ class ShapeUtil {
   // arrays.
   static bool IsArray(const Shape& shape);
 
+  // Returns whether the given primitive type corresponds to an array shape.
+  static bool IsArrayPrimitiveType(PrimitiveType primitive_type);
+
   // Returns whether the shape is a tuple with at least one element which is
   // also a tuple.
   static bool IsNestedTuple(const Shape& shape);
@@ -796,8 +798,6 @@ class ShapeUtil {
   TF_DISALLOW_COPY_AND_ASSIGN(ShapeUtil);
 };
 
-std::ostream& operator<<(std::ostream& out, const Shape& shape);
-
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_SHAPE_UTIL_H_
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index ce6330a0dc..60bdbe3020 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -546,37 +546,6 @@ TEST(ShapeUtilTest, IsLeafIndex) {
   EXPECT_TRUE(ShapeUtil::IsLeafIndex(nested_tuple_shape, {1, 1}));
 }
 
-TEST(ShapeUtilTest, HumanString) {
-  Shape opaque = ShapeUtil::MakeOpaqueShape();
-  Shape token = ShapeUtil::MakeTokenShape();
-  Shape scalar = ShapeUtil::MakeShape(F32, {});
-  Shape matrix = ShapeUtil::MakeShape(U32, {1, 2});
-  Shape matrix2 = ShapeUtil::MakeShapeWithLayout(S32, {3, 4}, {0, 1});
-  Shape tuple = ShapeUtil::MakeTupleShape({opaque, scalar, matrix, matrix2});
-  Shape nested_tuple = ShapeUtil::MakeTupleShape({tuple, matrix, token});
-
-  EXPECT_EQ("opaque[]", ShapeUtil::HumanString(opaque));
-  EXPECT_EQ("token[]", ShapeUtil::HumanString(token));
-  EXPECT_EQ("f32[]", ShapeUtil::HumanString(scalar));
-  EXPECT_EQ("u32[1,2]", ShapeUtil::HumanString(matrix));
-  EXPECT_EQ("s32[3,4]", ShapeUtil::HumanString(matrix2));
-  EXPECT_EQ("(opaque[], f32[], u32[1,2], s32[3,4])",
-            ShapeUtil::HumanString(tuple));
-  EXPECT_EQ("((opaque[], f32[], u32[1,2], s32[3,4]), u32[1,2], token[])",
-            ShapeUtil::HumanString(nested_tuple));
-
-  EXPECT_EQ("opaque[]", ShapeUtil::HumanStringWithLayout(opaque));
-  EXPECT_EQ("f32[]", ShapeUtil::HumanStringWithLayout(scalar));
-  EXPECT_EQ("u32[1,2]{1,0}", ShapeUtil::HumanStringWithLayout(matrix));
-  EXPECT_EQ("s32[3,4]{0,1}", ShapeUtil::HumanStringWithLayout(matrix2));
-  EXPECT_EQ("(opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1})",
-            ShapeUtil::HumanStringWithLayout(tuple));
-  EXPECT_EQ(
-      "((opaque[], f32[], u32[1,2]{1,0}, s32[3,4]{0,1}), u32[1,2]{1,0}, "
-      "token[])",
-      ShapeUtil::HumanStringWithLayout(nested_tuple));
-}
-
 TEST(ShapeUtilTest, ForEachSubshapeArray) {
   const Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
   int calls = 0;
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index b98572e24c..12c0299833 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -107,7 +107,7 @@ StatusOr<Literal> ClientLibraryTestBase::ExecuteAndTransfer(
   ExecutionOptions execution_options = execution_options_;
   if (shape_with_output_layout != nullptr) {
     *execution_options.mutable_shape_with_output_layout() =
-        *shape_with_output_layout;
+        shape_with_output_layout->ToProto();
   }
   return client_->ExecuteAndTransfer(computation, arguments,
                                      &execution_options);
@@ -127,7 +127,7 @@ StatusOr<Literal> ClientLibraryTestBase::ExecuteAndTransferReference(
   ExecutionOptions execution_options = execution_options_;
   if (shape_with_output_layout != nullptr) {
     *execution_options.mutable_shape_with_output_layout() =
-        *shape_with_output_layout;
+        shape_with_output_layout->ToProto();
   }
   execution_options.clear_device_handles();
   return ref_client_->ExecuteAndTransfer(computation, arguments,
diff --git a/tensorflow/compiler/xla/tests/client_test.cc b/tensorflow/compiler/xla/tests/client_test.cc
index 6f2ca84bb6..363dee74b2 100644
--- a/tensorflow/compiler/xla/tests/client_test.cc
+++ b/tensorflow/compiler/xla/tests/client_test.cc
@@ -50,7 +50,8 @@ XLA_TEST_F(ClientTest, ExecuteWithLayout) {
       ExecutionOptions execution_options = execution_options_;
       *execution_options.mutable_shape_with_output_layout() =
           ShapeUtil::MakeShapeWithLayout(S32, /*dimensions=*/{2, 2},
-                                         execute_layout);
+                                         execute_layout)
+              .ToProto();
       TF_ASSERT_OK_AND_ASSIGN(
           std::unique_ptr<GlobalData> data,
           client_->Execute(computation, {}, &execution_options));
@@ -84,7 +85,8 @@ XLA_TEST_F(ClientTest, ExecuteWithTupleLayout) {
           {ShapeUtil::MakeShapeWithLayout(S32, /*dimensions=*/{2, 2},
                                           /*minor_to_major=*/{0, 1}),
            ShapeUtil::MakeShapeWithLayout(S32, /*dimensions=*/{2, 2},
-                                          /*minor_to_major=*/{1, 0})});
+                                          /*minor_to_major=*/{1, 0})})
+          .ToProto();
 
   TF_ASSERT_OK_AND_ASSIGN(
       auto result,
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index dedc95b5ae..298136002e 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -618,7 +618,8 @@ XLA_TEST_P(ReshapeTest, R4Dim0MinorLayoutToR2Dim0MajorLayout) {
   ExecutionOptions execution_options = execution_options_;
   *execution_options.mutable_shape_with_output_layout() =
       ShapeUtil::MakeShapeWithLayout(use_bfloat16() ? BF16 : F32, {2, 8},
-                                     {1, 0});
+                                     {1, 0})
+          .ToProto();
   Literal actual =
       client_
           ->ExecuteAndTransfer(computation, {input.get()}, &execution_options)
@@ -767,7 +768,8 @@ XLA_TEST_P(ReshapeTest, NoopReshape) {
   ExecutionOptions execution_options = execution_options_;
   *execution_options.mutable_shape_with_output_layout() =
       ShapeUtil::MakeShapeWithLayout(use_bfloat16() ? BF16 : F32, {7, 2, 3, 5},
-                                     {2, 3, 0, 1});
+                                     {2, 3, 0, 1})
+          .ToProto();
   Literal output_literal =
       client_
           ->ExecuteAndTransfer(computation, {input_data.get()},
diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc
index 47be9f5adf..ff2c339992 100644
--- a/tensorflow/compiler/xla/tools/replay_computation.cc
+++ b/tensorflow/compiler/xla/tools/replay_computation.cc
@@ -82,13 +82,17 @@ struct Options {
 std::unique_ptr<LocalExecutable> CompileExecutable(const HloSnapshot& module,
                                                    LocalClient* client) {
   XlaComputation computation(module.hlo().hlo_module());
-  std::vector<const Shape*> argument_layouts;
-  for (const auto& param :
+  std::vector<Shape> argument_layouts;
+  argument_layouts.reserve(
+      computation.proto().host_program_shape().parameters_size());
+  std::vector<const Shape*> argument_layout_ptrs;
+  for (const ShapeProto& param :
        computation.proto().host_program_shape().parameters()) {
-    argument_layouts.push_back(&param);
+    argument_layouts.push_back(Shape(param));
+    argument_layout_ptrs.push_back(&argument_layouts.back());
   }
   return client
-      ->Compile(computation, argument_layouts, ExecutableBuildOptions())
+      ->Compile(computation, argument_layout_ptrs, ExecutableBuildOptions())
       .ValueOrDie();
 }
 
@@ -149,7 +153,7 @@ StatusOr<Literal> ReplayComputation(const HloSnapshot& module,
               << "--generate_fake_infeed only works if the model has 0 or 1 "
                  "infeed ops, but this one has >= 2.";
           provide_infeed = true;
-          infeed_shape = instruction.shape();
+          infeed_shape = Shape(instruction.shape());
           LOG(INFO) << "Generating fake infeed shape for inferred shape: "
                     << ShapeUtil::HumanString(infeed_shape);
         }
@@ -315,9 +319,10 @@ int RealMain(absl::Span<char* const> args, const Options& opts) {
       if (snapshot.has_result()) {
         Literal literal =
             Literal::CreateFromProto(snapshot.result()).ConsumeValueOrDie();
-        fprintf(stdout, "was %s:%s\n",
-                ShapeUtil::HumanString(snapshot.result().shape()).c_str(),
-                literal.ToString().c_str());
+        fprintf(
+            stdout, "was %s:%s\n",
+            ShapeUtil::HumanString(Shape(snapshot.result().shape())).c_str(),
+            literal.ToString().c_str());
       }
     }
   }
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index b015f4328a..6722641e9d 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -152,6 +152,13 @@ static inline absl::Span<const int64> AsInt64Slice(
                                  slice.size());
 }
 
+// TODO(b/29771030): This nop overload was added to simplify the migration of
+// Shape from a proto to a C++ class. Remove after class has been migrated.
+static inline absl::Span<const int64> AsInt64Slice(
+    absl::Span<const int64> slice) {
+  return slice;
+}
+
 // As above, but for uint64 types.
 static inline absl::Span<const uint64> AsUInt64Slice(
     const tensorflow::protobuf::RepeatedField<tensorflow::protobuf_uint64>& v) {
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index 28df3b03f3..f745fb8506 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -224,7 +224,7 @@ message ExecutionOptions {
   // may be faster when using this layout.
   //
   // We use a Shape here to accommodate computations that return a tuple.
-  Shape shape_with_output_layout = 2;
+  ShapeProto shape_with_output_layout = 2;
 
   // Used to seed random-number generators used in this computation.  If this is
   // 0, we generate a seed ourselves.
@@ -253,7 +253,7 @@ message TransferToClientRequest {
 
   // This optional field directs the service to return the literal in this
   // layout. A shape is used to hold the layout to accommodate tuples.
-  Shape shape_with_layout = 2;
+  ShapeProto shape_with_layout = 2;
 }
 
 message TransferToClientResponse {
@@ -281,7 +281,7 @@ message TransferToInfeedResponse {
 message TransferFromOutfeedRequest {
   // This optional field directs the service to return the literal in this
   // layout. A shape is used to hold the layout to accommodate tuples.
-  Shape shape_with_layout = 1;
+  ShapeProto shape_with_layout = 1;
 
   int64 replica_id = 2;
   DeviceHandle device_handle = 3;
@@ -332,7 +332,7 @@ message CompileRequest {
   // The layouts of the input arguments. If not set, the default layout will be
   // used. Although the real arguments are not needed in compilation, the
   // layouts of the arguments can affect the compilation.
-  repeated Shape input_shape_with_layout = 3;
+  repeated ShapeProto input_shape_with_layout = 3;
 }
 
 message CompileResponse {
@@ -406,7 +406,7 @@ message LoadDataRequest {
   string columnio_field = 2;
 
   // Individual element shape, excluding rows.
-  Shape element_shape = 3;
+  ShapeProto element_shape = 3;
 
   // Warning: ColumnIO does not support random-access, so use offset with
   // caution in performance-critical scenarios.
@@ -422,7 +422,7 @@ message LoadDataRequest {
 
 message LoadDataResponse {
   GlobalDataHandle data = 1;
-  Shape data_shape = 2;
+  ShapeProto data_shape = 2;
   int64 available_rows = 3;
   int64 rows_loaded = 4;
   int64 nanoseconds = 5;
@@ -433,7 +433,7 @@ message GetShapeRequest {
 }
 
 message GetShapeResponse {
-  Shape shape = 1;
+  ShapeProto shape = 1;
 }
 
 message UnpackRequest {
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 27ef86ab2e..013673dd9e 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -154,7 +154,7 @@ message Layout {
 // See the XLA documentation for more information on shapes and layouts.
 //
 // LINT.IfChange
-message Shape {
+message ShapeProto {
   reserved 1;
   reserved "rank";
 
@@ -169,7 +169,7 @@ message Shape {
   repeated int64 dimensions = 3;
 
   // For tuples only, the shapes of constitutent shapes in the tuple sequence.
-  repeated Shape tuple_shapes = 4;
+  repeated ShapeProto tuple_shapes = 4;
 
   // The layout used to back this shape.
   Layout layout = 5;
@@ -184,8 +184,8 @@ message Shape {
 // Shape of the parameters and output of a computation (like a traditional
 // function signature).
 message ProgramShapeProto {
-  repeated Shape parameters = 1;
-  Shape result = 2;
+  repeated ShapeProto parameters = 1;
+  ShapeProto result = 2;
   repeated string parameter_names = 3;
 }
 
@@ -320,7 +320,7 @@ message DeviceAssignmentProto {
 // Transfers to/from the client are encoded in literal form, and the structure
 // of the repeated fields is implied by the shape.
 message LiteralProto {
-  Shape shape = 1;
+  ShapeProto shape = 1;
   repeated bool preds = 2;
   bytes s8s = 15;
   bytes u8s = 3;
@@ -521,7 +521,7 @@ message OpSharding {
   }
   Type type = 1;
   // The shape of the sharded tile.
-  Shape tile_shape = 2;
+  ShapeProto tile_shape = 2;
   // The shape of the tile assignment tensor - this must be the same rank as
   // tile_shape and the product of its dimensions must equal
   // tile_assignment_devices.size().
diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
index 1603b45ff6..2ccdf0f02d 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
@@ -109,14 +109,17 @@ Status XRTCompileOp::Compile(OpKernelContext* ctx,
   TF_ASSIGN_OR_RETURN(xla::XlaComputation computation,
                       client->LoadSnapshot(computation_proto.hlo_snapshot()));
 
-  std::vector<const xla::Shape*> argument_layouts(
+  std::vector<xla::Shape> argument_layouts(
+      config.program_shape().parameters_size());
+  std::vector<const xla::Shape*> argument_layout_ptrs(
       config.program_shape().parameters_size());
   for (int i = 0; i < config.program_shape().parameters_size(); ++i) {
-    argument_layouts[i] = &config.program_shape().parameters(i);
+    argument_layouts[i] = xla::Shape(config.program_shape().parameters(i));
+    argument_layout_ptrs[i] = &argument_layouts[i];
   }
   xla::ExecutableBuildOptions build_options;
   build_options.set_device_ordinal(client->default_device_ordinal());
-  build_options.set_result_layout(config.program_shape().result());
+  build_options.set_result_layout(xla::Shape(config.program_shape().result()));
   build_options.set_device_allocator(device_ref.backend()->memory_allocator());
   if (config.has_debug_options()) {
     *build_options.mutable_debug_options() =
@@ -125,7 +128,7 @@ Status XRTCompileOp::Compile(OpKernelContext* ctx,
 
   VLOG(1) << "Building executable";
   auto compile_result =
-      client->Compile(computation, argument_layouts, build_options);
+      client->Compile(computation, argument_layout_ptrs, build_options);
   if (!compile_result.ok()) {
     return compile_result.status();
   }
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index 7e73db98f7..b9262c1843 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -375,9 +375,12 @@ TEST(RawApiTest, CompileAndExecute) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {2});
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->mutable_result() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
   StoreComputationSnapshot(AddAndScale(), c.mutable_hlo_snapshot());
 
   xrt::XRTExecutionConfig e;
@@ -427,9 +430,12 @@ TEST(RawApiTest, CompileAndExecuteWithArgumentVector) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {2});
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->mutable_result() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
   StoreComputationSnapshot(AddAndScale(), c.mutable_hlo_snapshot());
 
   xrt::XRTExecutionConfig e;
@@ -494,8 +500,8 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = param_shape;
-  *shapes->mutable_result() = result_shape;
+  *shapes->add_parameters() = param_shape.ToProto();
+  *shapes->mutable_result() = result_shape.ToProto();
   StoreComputationSnapshot(xla_computation, c.mutable_hlo_snapshot());
 
   Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
@@ -510,8 +516,9 @@ TEST(RawApiTest, CompileWithXlaReturnShapes) {
   TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(),
                            {c_handle.program_shape}, {release}, &outputs));
 
-  xla::ProgramShapeProto program_shape;
-  EXPECT_TRUE(program_shape.ParseFromString(outputs[0].vec<string>()(0)));
+  xla::ProgramShapeProto program_shape_proto;
+  EXPECT_TRUE(program_shape_proto.ParseFromString(outputs[0].vec<string>()(0)));
+  xla::ProgramShape program_shape(program_shape_proto);
   EXPECT_EQ(program_shape.parameters_size(), 1);
 
   VLOG(2) << "Param: "
@@ -547,11 +554,11 @@ TEST(RawApiTest, DotGeneralWithLayoutTest) {
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
   *shapes->add_parameters() =
-      xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 2}, {0, 1});
+      xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 2}, {0, 1}).ToProto();
   *shapes->add_parameters() =
-      xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1});
+      xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1}).ToProto();
   *shapes->mutable_result() =
-      xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1});
+      xla::ShapeUtil::MakeShapeWithLayout(xla::F32, {2, 1}, {0, 1}).ToProto();
   StoreComputationSnapshot(Dot(), c.mutable_hlo_snapshot());
 
   xrt::XRTExecutionConfig e;
@@ -592,7 +599,7 @@ TEST(RawApiTest, CompileAndExecuteZeroArg) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {});
+  *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::F32, {}).ToProto();
 
   xrt::XRTExecutionConfig e;
   e.set_release_input_handles(true);
@@ -632,10 +639,13 @@ TEST(RawApiTest, CompileAndExecuteReturnTuple) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->mutable_result() = xla::ShapeUtil::MakeTupleShape(
-      {xla::ShapeUtil::MakeShape(xla::F32, {2})});
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->mutable_result() =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {2})})
+          .ToProto();
   StoreComputationSnapshot(AddAndTuple(), c.mutable_hlo_snapshot());
 
   xrt::XRTExecutionConfig e;
@@ -675,10 +685,13 @@ TEST(RawApiTest, LeakCompilationReference) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  *shapes->mutable_result() = xla::ShapeUtil::MakeTupleShape(
-      {xla::ShapeUtil::MakeShape(xla::F32, {2})});
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes->mutable_result() =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {2})})
+          .ToProto();
   StoreComputationSnapshot(AddAndTuple(), c.mutable_hlo_snapshot());
 
   Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
@@ -703,9 +716,9 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
   auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {});
-  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {});
-  *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::S64, {});
+  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {}).ToProto();
+  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::S64, {}).ToProto();
+  *shapes->mutable_result() = xla::ShapeUtil::MakeShape(xla::S64, {}).ToProto();
   StoreComputationSnapshot(AddS64(), c.mutable_hlo_snapshot());
 
   xrt::XRTExecutionConfig e;
@@ -742,8 +755,8 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xla::ProgramShapeProto program_shape;
   EXPECT_TRUE(program_shape.ParseFromString(outputs[1].vec<string>()(0)));
   EXPECT_EQ(program_shape.parameters_size(), 2);
-  EXPECT_TRUE(
-      xla::ShapeUtil::HasPrimitiveType(program_shape.result(), xla::S64));
+  EXPECT_TRUE(xla::ShapeUtil::HasPrimitiveType(
+      xla::Shape(program_shape.result()), xla::S64));
 }
 
 }  // namespace
-- 
GitLab


From 048d6a34bb30f0bf8f57650aa6369587b90bfefc Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 28 Nov 2018 16:08:10 -0800
Subject: [PATCH 0971/1554] Add device info to function debugging utilities

Also, remove the assumption that RetVal nodes have a single input.
Various optimization passes can add control inputs to RetVal nodes.

PiperOrigin-RevId: 223253576
---
 tensorflow/core/BUILD                 |  1 +
 tensorflow/core/framework/function.cc | 44 ++++++++++++++++++++++++---
 2 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 9cd0edabdd..4313d4f1a8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2679,6 +2679,7 @@ tf_cuda_library(
         ":version_lib",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
         "//tensorflow/core/platform/default/build_config:platformlib",
         "//tensorflow/core/kernels:bounds_check",
         "//third_party/eigen3",
diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index 6809c27197..b69a40f312 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <vector>
 
 #include "absl/container/flat_hash_set.h"
+#include "absl/strings/str_join.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/function.pb_text.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -32,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/util/device_name_utils.h"
 #include "tensorflow/core/util/equal_graph_def.h"
 
 namespace tensorflow {
@@ -507,6 +509,16 @@ string Print(const NodeDef& n) {
       entries.push_back(strings::StrCat(a.first, "=", Print(a.second)));
     }
     std::sort(entries.begin(), entries.end());
+    // Add a short device string at the end of all attributes.
+    if (!n.device().empty()) {
+      DeviceNameUtils::ParsedName parsed;
+      if (DeviceNameUtils::ParseFullName(n.device(), &parsed)) {
+        entries.push_back(
+            strings::StrCat("device=", parsed.type, ":", parsed.id));
+      } else {
+        entries.push_back("device=<FAILED_TO_PARSE>");
+      }
+    }
     strings::StrAppend(&out, "[", str_util::Join(entries, ", "), "]");
   }
   strings::StrAppend(&out, "(");
@@ -590,26 +602,50 @@ string Print(gtl::ArraySlice<const NodeDef*> nodes) {
   std::sort(ret.begin(), ret.end(), comp);
   string out;
   strings::StrAppend(&out, "\n(");
-  auto get_type = [](const NodeDef& n) {
+  auto get_type_and_device = [](const NodeDef& n) {
     DataType dt;
     if (!GetNodeAttr(n, "T", &dt).ok()) {
       dt = DT_INVALID;
     }
+    if (!n.device().empty()) {
+      DeviceNameUtils::ParsedName parsed;
+      if (DeviceNameUtils::ParseFullName(n.device(), &parsed)) {
+        return strings::StrCat(DataTypeString(dt), "@", parsed.type, ":",
+                               parsed.id);
+      } else {
+        return strings::StrCat(DataTypeString(dt), "@",
+                               "<FAILED_TO_PARSE_DEVICE>");
+      }
+    }
     return DataTypeString(dt);
   };
   for (size_t i = 0; i < arg.size(); ++i) {
     const NodeDef* n = arg[i];
     if (i > 0) strings::StrAppend(&out, ", ");
     CHECK_GE(n->attr_size(), 2);
-    strings::StrAppend(&out, n->name(), ":", get_type(*n));
+    strings::StrAppend(&out, n->name(), ":", get_type_and_device(*n));
   }
   strings::StrAppend(&out, ") -> (");
   for (size_t i = 0; i < ret.size(); ++i) {
     const NodeDef* n = ret[i];
     if (i > 0) strings::StrAppend(&out, ", ");
     CHECK_LE(2, n->attr_size());
-    CHECK_EQ(1, n->input_size());
-    strings::StrAppend(&out, n->input(0), ":", get_type(*n));
+
+    // The _RetVal op should have a unique non-control input. We assert that
+    // here and add it to the output.
+    bool found_non_control_input = false;
+    for (const string& input : n->input()) {
+      if (!input.empty() && input[0] != '^') {
+        DCHECK_EQ(found_non_control_input, false)
+            << "RetVal node has more than one non-control input: "
+            << absl::StrJoin(n->input(), ", ");
+        strings::StrAppend(&out, n->input(0), ":", get_type_and_device(*n));
+        found_non_control_input = true;
+      }
+    }
+    DCHECK_EQ(found_non_control_input, true)
+        << "RetVal did not have any non-control inputs: "
+        << absl::StrJoin(n->input(), ", ");
   }
   strings::StrAppend(&out, ") {\n");
   for (size_t i = 0; i < body.size(); ++i) {
-- 
GitLab


From b86dd9ee886487cbb9ce62ad315256bc4ea0982f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 16:09:52 -0800
Subject: [PATCH 0972/1554] Improve performance of grouped convolutions.

PiperOrigin-RevId: 223253872
---
 .../convolution_feature_group_converter.cc    | 255 ++++++++++++-----
 .../compiler/xla/tests/convolution_test.cc    | 257 ++++++++++++++++++
 2 files changed, 450 insertions(+), 62 deletions(-)

diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
index 10c53f15f5..09c3f32860 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
@@ -270,78 +270,209 @@ Status ConvolutionVisitor::HandleConvolution(HloInstruction* convolution) {
     TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
         convolution, std::move(new_convolution)));
   } else {
-    // The filter expansion mechanism adds zeroes in the kernel.
-    // For an OF = 12, IF = 6, and kernel IF = 2, the expanded filter mask
-    // would look like (IF on the Y-axis, OF on the X-axis)
-    // 1 1 1 1 0 0 0 0 0 0 0 0
-    // 1 1 1 1 0 0 0 0 0 0 0 0
-    // 0 0 0 0 1 1 1 1 0 0 0 0
-    // 0 0 0 0 1 1 1 1 0 0 0 0
-    // 0 0 0 0 0 0 0 0 1 1 1 1
-    // 0 0 0 0 0 0 0 0 1 1 1 1
-    //
-    // Instead of convolving the above with the input, we instead slice the
-    // kernel into three kernels, each containing islands of 1s from the filter
-    // above. We also slice the activations in the IF dimension with each slice
-    // of size = group_size. For each slice, we perform convolutions, and
-    // concatenate the generated outputs in the output OF dimension.
-
-    std::vector<HloInstruction*> sliced_convolutions;
+    int64 activation_input_feature_dim = dim_numbers.input_feature_dimension();
     auto activation = convolution->mutable_operand(0);
-    std::vector<int64> slice_strides(filter->shape().dimensions_size(), 1);
-    std::vector<int64> filter_slice_starts(filter->shape().dimensions_size(),
-                                           0);
-    std::vector<int64> filter_slice_limits(filter->shape().dimensions().begin(),
-                                           filter->shape().dimensions().end());
-    std::vector<int64> activation_slice_starts(
-        activation->shape().dimensions_size(), 0);
-    std::vector<int64> activation_slice_limits(
-        activation->shape().dimensions().begin(),
-        activation->shape().dimensions().end());
 
     int64 output_feature =
         filter->shape().dimensions(kernel_output_feature_dim);
-    auto output_feature_dim = dim_numbers.output_feature_dimension();
-    int64 filter_slice_width = output_feature / group_count;
 
-    int64 activation_input_feature_dim = dim_numbers.input_feature_dimension();
+    int64 input_feature =
+        activation->shape().dimensions(activation_input_feature_dim);
+
+    // If group_count == output_feature, then we map those grouped convolutions
+    // onto depthwise convolution + reduce. E.g., we would turn
+    // [2, 12]{B, IF} conv [3, 4]{IF, OF} into
+    // [2, 12]{B, IF} depth conv [1, 12]{IF, OF}, and then use a reduce window
+    // of {1, 3} on the generated [2, 12] output to produce the final result of
+    // [2, 4].
+    if (group_count == output_feature && !filter_expansion_) {
+      Shape reshaped_filter_shape = filter->shape();
+
+      if (kernel_input_feature_dim < kernel_output_feature_dim) {
+        // Transpose IF and OF on the kernel.
+        std::vector<int64> filter_dims;
+        for (int64 i = 0; i < dim_numbers.kernel_spatial_dimensions().size();
+             ++i) {
+          filter_dims.push_back(dim_numbers.kernel_spatial_dimensions(i));
+        }
+        filter_dims.push_back(kernel_output_feature_dim);
+        filter_dims.push_back(kernel_input_feature_dim);
+
+        Shape transposed_filter = filter->shape();
+        auto& dimensions = *transposed_filter.mutable_dimensions();
+        std::swap(dimensions[kernel_input_feature_dim],
+                  dimensions[kernel_output_feature_dim]);
+
+        filter = add(HloInstruction::CreateTranspose(transposed_filter, filter,
+                                                     filter_dims));
+      } else {
+        // For depthwise convolutions, we want the kernel input feature
+        // dimension to be smaller than the output feature dimension. If that's
+        // not the case, we swap the dimensions.
+
+        auto& dimensions = *reshaped_filter_shape.mutable_dimensions();
+        std::swap(dimensions[kernel_input_feature_dim],
+                  dimensions[kernel_output_feature_dim]);
+
+        dim_numbers.set_kernel_input_feature_dimension(
+            kernel_output_feature_dim);
+
+        dim_numbers.set_kernel_output_feature_dimension(
+            kernel_input_feature_dim);
+        std::swap(kernel_output_feature_dim, kernel_input_feature_dim);
+      }
 
-    for (int64 i = 0; i < group_count; i++) {
-      filter_slice_starts[kernel_output_feature_dim] = i * filter_slice_width;
-      filter_slice_limits[kernel_output_feature_dim] =
-          (i + 1) * filter_slice_width;
-      auto filter_sliced_shape = filter->shape();
-      filter_sliced_shape.set_dimensions(kernel_output_feature_dim,
-                                         filter_slice_width);
-      auto filter_slice = add(HloInstruction::CreateSlice(
-          filter_sliced_shape, filter, filter_slice_starts, filter_slice_limits,
-          slice_strides));
-
-      activation_slice_starts[activation_input_feature_dim] = i * group_size;
-      activation_slice_limits[activation_input_feature_dim] =
-          (i + 1) * group_size;
-      auto activation_sliced_shape = activation->shape();
-      activation_sliced_shape.set_dimensions(activation_input_feature_dim,
-                                             group_size);
-      auto activation_slice = add(HloInstruction::CreateSlice(
-          activation_sliced_shape, activation, activation_slice_starts,
-          activation_slice_limits, slice_strides));
-
-      auto conv_slice_shape = convolution->shape();
-      conv_slice_shape.set_dimensions(output_feature_dim, filter_slice_width);
+      reshaped_filter_shape.set_dimensions(kernel_input_feature_dim, 1);
+      reshaped_filter_shape.set_dimensions(kernel_output_feature_dim,
+                                           group_count * group_size);
+      auto reshaped_filter =
+          add(HloInstruction::CreateReshape(reshaped_filter_shape, filter));
 
+      Shape reshaped_convolution_shape = convolution->shape();
+      reshaped_convolution_shape.set_dimensions(
+          dim_numbers.output_feature_dimension(), group_count * group_size);
       auto new_convolution = add(HloInstruction::CreateConvolve(
-          conv_slice_shape, activation_slice, filter_slice,
-          /*feature_group_count=*/1, convolution->window(), dim_numbers,
-          convolution->precision_config()));
+          reshaped_convolution_shape, convolution->mutable_operand(0),
+          reshaped_filter, /*feature_group_count=*/input_feature,
+          convolution->window(), dim_numbers, convolution->precision_config()));
+
+      // Create the reduce window.
+      Window window;
+      for (int64 i = 0; i < new_convolution->shape().dimensions_size(); ++i) {
+        auto* dim = window.add_dimensions();
+        dim->set_padding_low(0);
+        dim->set_padding_high(0);
+        dim->set_window_dilation(1);
+        dim->set_base_dilation(1);
+        if (i == dim_numbers.output_feature_dimension()) {
+          dim->set_stride(group_size);
+          dim->set_size(group_size);
+        } else {
+          dim->set_stride(1);
+          dim->set_size(1);
+        }
+      }
 
-      sliced_convolutions.push_back(new_convolution);
-    }
+      auto reduce_window_shape = new_convolution->shape();
+      reduce_window_shape.set_dimensions(dim_numbers.output_feature_dimension(),
+                                         group_count);
+
+      auto zero_literal = LiteralUtil::CreateR0(0.0f);
+      TF_ASSIGN_OR_RETURN(zero_literal, zero_literal.Convert(F32));
+      auto zero = add(HloInstruction::CreateConstant(std::move(zero_literal)));
+
+      auto reduce_function = [&]() -> HloComputation* {
+        HloComputation::Builder b("add_computation");
+        Shape shape = ShapeUtil::MakeShape(F32, {});
+        auto lhs =
+            b.AddInstruction(HloInstruction::CreateParameter(0, shape, "lhs"));
+        auto rhs =
+            b.AddInstruction(HloInstruction::CreateParameter(1, shape, "rhs"));
+        auto scalar_op = b.AddInstruction(
+            HloInstruction::CreateBinary(shape, HloOpcode::kAdd, lhs, rhs));
+        return computation_->parent()->AddEmbeddedComputation(
+            b.Build(scalar_op));
+      };
+
+      // Ensure that data input to reduce window is of type F32.
+      if (primitive_util::BitWidth(new_convolution->shape().element_type()) <
+          primitive_util::BitWidth(F32)) {
+        Shape convert_shape = new_convolution->shape();
+        convert_shape.set_element_type(F32);
+        new_convolution = add(HloInstruction::CreateBitcastConvert(
+            convert_shape, new_convolution));
+      }
 
-    auto new_conv = HloInstruction::CreateConcatenate(
-        convolution->shape(), sliced_convolutions, output_feature_dim);
-    TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
-        convolution, std::move(new_conv)));
+      auto reduce_window = add(HloInstruction::CreateReduceWindow(
+          reduce_window_shape, new_convolution, zero, window,
+          reduce_function()));
+
+      Shape convert_back_shape = reduce_window->shape();
+      convert_back_shape.set_element_type(activation->shape().element_type());
+
+      // Convert reduced data back to the original data type.
+      auto reduce_window_converted = HloInstruction::CreateBitcastConvert(
+          convert_back_shape, reduce_window);
+      TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
+          convolution, std::move(reduce_window_converted)));
+
+    } else {
+      // The filter expansion mechanism adds zeroes in the kernel.
+      // For an OF = 12, IF = 6, and kernel IF = 2, the expanded filter mask
+      // would look like (IF on the Y-axis, OF on the X-axis)
+      // 1 1 1 1 0 0 0 0 0 0 0 0
+      // 1 1 1 1 0 0 0 0 0 0 0 0
+      // 0 0 0 0 1 1 1 1 0 0 0 0
+      // 0 0 0 0 1 1 1 1 0 0 0 0
+      // 0 0 0 0 0 0 0 0 1 1 1 1
+      // 0 0 0 0 0 0 0 0 1 1 1 1
+      //
+      // Instead of convolving the above with the input, we instead slice the
+      // kernel into three kernels, each containing islands of 1s from the
+      // filter above. We also slice the activations in the IF dimension with
+      // each slice of size = group_size. For each slice, we perform
+      // convolutions, and concatenate the generated outputs in the output OF
+      // dimension.
+
+      std::vector<HloInstruction*> sliced_convolutions;
+      auto activation = convolution->mutable_operand(0);
+      std::vector<int64> slice_strides(filter->shape().dimensions_size(), 1);
+      std::vector<int64> filter_slice_starts(filter->shape().dimensions_size(),
+                                             0);
+      std::vector<int64> filter_slice_limits(
+          filter->shape().dimensions().begin(),
+          filter->shape().dimensions().end());
+      std::vector<int64> activation_slice_starts(
+          activation->shape().dimensions_size(), 0);
+      std::vector<int64> activation_slice_limits(
+          activation->shape().dimensions().begin(),
+          activation->shape().dimensions().end());
+
+      int64 output_feature =
+          filter->shape().dimensions(kernel_output_feature_dim);
+      auto output_feature_dim = dim_numbers.output_feature_dimension();
+      int64 filter_slice_width = output_feature / group_count;
+
+      int64 activation_input_feature_dim =
+          dim_numbers.input_feature_dimension();
+
+      for (int64 i = 0; i < group_count; i++) {
+        filter_slice_starts[kernel_output_feature_dim] = i * filter_slice_width;
+        filter_slice_limits[kernel_output_feature_dim] =
+            (i + 1) * filter_slice_width;
+        auto filter_sliced_shape = filter->shape();
+        filter_sliced_shape.set_dimensions(kernel_output_feature_dim,
+                                           filter_slice_width);
+        auto filter_slice = add(HloInstruction::CreateSlice(
+            filter_sliced_shape, filter, filter_slice_starts,
+            filter_slice_limits, slice_strides));
+
+        activation_slice_starts[activation_input_feature_dim] = i * group_size;
+        activation_slice_limits[activation_input_feature_dim] =
+            (i + 1) * group_size;
+        auto activation_sliced_shape = activation->shape();
+        activation_sliced_shape.set_dimensions(activation_input_feature_dim,
+                                               group_size);
+        auto activation_slice = add(HloInstruction::CreateSlice(
+            activation_sliced_shape, activation, activation_slice_starts,
+            activation_slice_limits, slice_strides));
+
+        auto conv_slice_shape = convolution->shape();
+        conv_slice_shape.set_dimensions(output_feature_dim, filter_slice_width);
+
+        auto new_convolution = add(HloInstruction::CreateConvolve(
+            conv_slice_shape, activation_slice, filter_slice,
+            /*feature_group_count=*/1, convolution->window(), dim_numbers,
+            convolution->precision_config()));
+
+        sliced_convolutions.push_back(new_convolution);
+      }
+
+      auto new_conv = HloInstruction::CreateConcatenate(
+          convolution->shape(), sliced_convolutions, output_feature_dim);
+      TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
+          convolution, std::move(new_conv)));
+    }
   }
 
   return Status::OK();
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 7e81905260..4a58a1ed66 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -1410,6 +1410,263 @@ TYPED_TEST(Convolve2D_1x2x2x1024_2x2x128x512_Grouped_Valid, Types) {
   this->RunTest();
 }
 
+template <typename T>
+class Convolve2D_1x2x2x1024_2x2x128x8_Grouped_Valid : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 2, 2, 1024};
+    std::vector<int64> filter_dims = {2, 2, 128, 8};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/8);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape),
+                               static_cast<T>(1));
+
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape),
+                                static_cast<T>(2));
+
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    std::vector<T> output_elems(8, static_cast<T>(1024));
+    auto expected_r1 = LiteralUtil::CreateR1<T>(output_elems);
+    auto expected_r4 = expected_r1.Reshape({1, 1, 1, 8}).ConsumeValueOrDie();
+
+    auto input_literal =
+        client_->TransferToServer(input_r4).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_);
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x2x2x1024_2x2x128x8_Grouped_Valid, TestTypes);
+TYPED_TEST(Convolve2D_1x2x2x1024_2x2x128x8_Grouped_Valid, Types) {
+  this->RunTest();
+}
+
+template <typename T>
+class Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 2, 2, 12};
+    std::vector<int64> filter_dims = {2, 2, 3, 4};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/4);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape));
+    iota_int_init_value(input_elems, 1);
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape));
+    iota_int_init_value(filter_elems, 1);
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    auto expected_r1 =
+        LiteralUtil::CreateR1<T>({static_cast<T>(7712), static_cast<T>(8816),
+                                  static_cast<T>(9992), static_cast<T>(11240)});
+    auto expected_r4 = expected_r1.Reshape({1, 1, 1, 4}).ConsumeValueOrDie();
+
+    auto input_literal =
+        client_->TransferToServer(input_r4).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_);
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid, TestTypes);
+TYPED_TEST(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid, Types) {
+  this->RunTest();
+}
+
+template <typename T>
+class Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid_Filter_OF_In_Sublanes
+    : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 2, 2, 12};
+    std::vector<int64> filter_dims = {2, 2, 4, 3};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(3);
+      dnums.set_kernel_output_feature_dimension(2);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/4);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape));
+    iota_int_init_value(input_elems, 1);
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape));
+    iota_int_init_value(filter_elems, 1);
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+    auto filter_r4_relaid =
+        filter_r4.Relayout(LayoutUtil::MakeLayout({3, 2, 1, 0}));
+    auto expected_r1 = LiteralUtil::CreateR1<T>(
+        {static_cast<T>(6968), static_cast<T>(8516), static_cast<T>(10280),
+         static_cast<T>(12260)});
+    auto expected_r4 = expected_r1.Reshape({1, 1, 1, 4}).ConsumeValueOrDie();
+
+    auto input_literal =
+        client_->TransferToServer(input_r4).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4_relaid).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_);
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid_Filter_OF_In_Sublanes,
+                TestTypes);
+TYPED_TEST(Convolve2D_1x2x2x12_2x2x3x4_Grouped_Valid_Filter_OF_In_Sublanes,
+           Types) {
+  this->RunTest();
+}
+
+template <typename T>
+class Convolve2D_1x1x1x12_1x1x3x4_Grouped_Valid : public ConvolutionTest {
+ public:
+  void RunTest() {
+    XlaBuilder builder(TestName());
+    std::vector<int64> input_dims = {1, 1, 1, 12};
+    std::vector<int64> filter_dims = {1, 1, 3, 4};
+    Shape input_shape = ShapeUtil::MakeShapeWithType<T>(input_dims);
+    Shape filter_shape = ShapeUtil::MakeShapeWithType<T>(filter_dims);
+    {
+      auto input = Parameter(&builder, 0, input_shape, "input");
+      auto filter = Parameter(&builder, 1, filter_shape, "filter");
+
+      // Tensorflow dimension numbers for 2D convolution.
+      ConvolutionDimensionNumbers dnums;
+      dnums.set_input_batch_dimension(0);
+      dnums.set_output_batch_dimension(0);
+      dnums.add_input_spatial_dimensions(1);
+      dnums.add_output_spatial_dimensions(1);
+      dnums.add_input_spatial_dimensions(2);
+      dnums.add_output_spatial_dimensions(2);
+      dnums.set_input_feature_dimension(3);
+      dnums.set_output_feature_dimension(3);
+      dnums.add_kernel_spatial_dimensions(0);
+      dnums.add_kernel_spatial_dimensions(1);
+      dnums.set_kernel_input_feature_dimension(2);
+      dnums.set_kernel_output_feature_dimension(3);
+
+      ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                                /*feature_group_count=*/4);
+    }
+
+    std::vector<T> input_elems(ShapeUtil::ElementsIn(input_shape));
+    iota_int_init_value(input_elems, 1);
+    auto input_r1 = LiteralUtil::CreateR1<T>(input_elems);
+    auto input_r4 = input_r1.Reshape(input_dims).ConsumeValueOrDie();
+
+    std::vector<T> filter_elems(ShapeUtil::ElementsIn(filter_shape));
+    iota_int_init_value(filter_elems, 1);
+    auto filter_r1 = LiteralUtil::CreateR1<T>(filter_elems);
+    auto filter_r4 = filter_r1.Reshape(filter_dims).ConsumeValueOrDie();
+
+    auto expected_r1 =
+        LiteralUtil::CreateR1<T>({static_cast<T>(38), static_cast<T>(98),
+                                  static_cast<T>(176), static_cast<T>(272)});
+    auto expected_r4 = expected_r1.Reshape({1, 1, 1, 4}).ConsumeValueOrDie();
+
+    auto input_literal =
+        client_->TransferToServer(input_r4).ConsumeValueOrDie();
+    auto filter_literal =
+        client_->TransferToServer(filter_r4).ConsumeValueOrDie();
+
+    ComputeAndCompareLiteral(&builder, expected_r4,
+                             {input_literal.get(), filter_literal.get()},
+                             error_spec_);
+  }
+};
+
+TYPED_TEST_CASE(Convolve2D_1x1x1x12_1x1x3x4_Grouped_Valid, TestTypes);
+TYPED_TEST(Convolve2D_1x1x1x12_1x1x3x4_Grouped_Valid, Types) {
+  this->RunTest();
+}
+
 // Test fixture to run convolution tests with and without convolution
 // canonicalization enabled.
 class ConvolveWithAndWithoutCanonicalization
-- 
GitLab


From ea680ae952681ed88738b91deb5463d178b42c43 Mon Sep 17 00:00:00 2001
From: Shining Sun <shiningsun@google.com>
Date: Wed, 28 Nov 2018 16:12:34 -0800
Subject: [PATCH 0973/1554] Remove "return_same_structure" argument from
 while_loop_v2.

PiperOrigin-RevId: 223254267
---
 .../python/kernel_tests/while_v2_test.py      | 89 ++++++++++++++-----
 tensorflow/python/ops/control_flow_ops.py     | 16 ++--
 tensorflow/python/ops/while_v2.py             |  6 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |  2 +-
 4 files changed, 78 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 48b32f06aa..e08699922a 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -45,12 +45,25 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
 
   def testSingleLoopVar(self):
     x = constant_op.constant(2.)
-    ret = while_loop_v2(lambda v: v < 8., lambda v: v * v, [x])
+    ret = while_loop_v2(
+        lambda v: v < 8., lambda v: v * v, [x], return_same_structure=False)
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
       self.assertEqual(self.evaluate(ret), 16.)
       self.assertSequenceEqual(self.evaluate(grad), [32.])
 
+  def testReturnSameStructureTrue(self):
+    x = constant_op.constant(2.)
+    ret = while_loop_v2(
+        lambda v: v < 8., lambda v: v * v, [x], return_same_structure=True)
+    grad = gradients_impl.gradients(ret, [x])
+    with self.cached_session() as sess:
+      eval_result = sess.run(ret)
+      self.assertIsInstance(eval_result, list)
+      self.assertLen(eval_result, 1)
+      self.assertEqual(16., eval_result[0])
+      self.assertSequenceEqual(sess.run(grad), [32.])
+
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
     y = constant_op.constant(3.)
@@ -59,7 +72,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # y = 3.
     # while x < 45.:
     #   x = x * y
-    ret = while_loop_v2(lambda v, _: v < 45., lambda v, w: (v * w, w), [x, y])
+    ret = while_loop_v2(
+        lambda v, _: v < 45.,
+        lambda v, w: (v * w, w), [x, y],
+        return_same_structure=False)
     # ret = [x*y^2, y]
 
     # Note: This is simply d_ret[0]/d_x since d_ret[1]/d_x is 0.
@@ -77,8 +93,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # while x < 45.:
     #   x = x * y
     #   y = x + y
-    ret = while_loop_v2(lambda v, _: v < 45., lambda v, w: (v * w, v + w),
-                        [x, y])
+    ret = while_loop_v2(
+        lambda v, _: v < 45.,
+        lambda v, w: (v * w, v + w), [x, y],
+        return_same_structure=False)
     # ret = [y*x**2 + x*y**2, x*y + x + y]
 
     gradx_0 = gradients_impl.gradients(ret[0], [x])  # [2*x*y + y**2]
@@ -98,8 +116,12 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
 
   def testMultipleWhileLoops(self):
     x = constant_op.constant(2.)
-    ret1 = while_loop_v2(lambda v: v < 4., lambda v: v * v, [x])  # x**2
-    ret2 = while_loop_v2(lambda v: v < 16., lambda v: v * v, [ret1])  # x**4
+    ret1 = while_loop_v2(
+        lambda v: v < 4., lambda v: v * v, [x],
+        return_same_structure=False)  # x**2
+    ret2 = while_loop_v2(
+        lambda v: v < 16., lambda v: v * v, [ret1],
+        return_same_structure=False)  # x**4
     grad = gradients_impl.gradients(ret2, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
@@ -108,7 +130,9 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
 
   def testDoubleDerivative(self):
     x = constant_op.constant(2.)
-    ret = while_loop_v2(lambda v: v < 8., lambda v: v**2, [x])  # x**4
+    ret = while_loop_v2(
+        lambda v: v < 8., lambda v: v**2, [x],
+        return_same_structure=False)  # x**4
     grad = gradients_impl.gradients(ret, [x])  # 4x**3
     grad_grad = gradients_impl.gradients(grad, [x])  # 12x**2
     with self.cached_session() as sess:
@@ -154,7 +178,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
   def testCaptureExternalTensorInCond(self):
     x = constant_op.constant(2.)
     y = constant_op.constant(1.)
-    ret = while_loop_v2(lambda v: v + y < 9., lambda v: v * 3., [x])
+    ret = while_loop_v2(
+        lambda v: v + y < 9.,
+        lambda v: v * 3., [x],
+        return_same_structure=False)
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
       self.assertEqual(self.evaluate(ret), 18.)
@@ -163,7 +190,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
   def testCaptureExternalTensorInBody(self):
     x = constant_op.constant(2.)
     y = constant_op.constant(3.)
-    ret = while_loop_v2(lambda v: v < 8., lambda v: v * y, [x])
+    ret = while_loop_v2(
+        lambda v: v < 8., lambda v: v * y, [x], return_same_structure=False)
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
       self.assertEqual(self.evaluate(ret), 18.)
@@ -184,7 +212,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       tl = list_ops.tensor_list_push_back(tl, constant_op.constant(100.))
       return x**2., tl
 
-    ret = while_loop_v2(Cond, Body, [x, tensor_list])
+    ret = while_loop_v2(
+        Cond, Body, [x, tensor_list], return_same_structure=False)
     grad = gradients_impl.gradients(ret[0], x)
     with self.cached_session() as sess:
       self.assertEqual(sess.run(ret[0]), 16.)
@@ -206,7 +235,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       tl = list_ops.tensor_list_push_back(tl, x)
       return x**2., tl
 
-    ret = while_loop_v2(Cond, Body, [x, tensor_list])
+    ret = while_loop_v2(
+        Cond, Body, [x, tensor_list], return_same_structure=False)
 
     for op in ops.get_default_graph().get_operations():
       if op.type == "While":
@@ -253,7 +283,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     y = array_ops.placeholder(dtype=dtypes.float32, shape=shape)
 
     # Forward pass.
-    ret = while_loop_v2(lambda v, u: v < 8., lambda v, u: (v * v, u), [x, y])
+    ret = while_loop_v2(
+        lambda v, u: v < 8.,
+        lambda v, u: (v * v, u), [x, y],
+        return_same_structure=False)
     while_op = ret[0].op.inputs[0].op
     # Get the TensorList output of While op containing the accumulated values
     # of y.
@@ -277,8 +310,10 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
 
   def _createWhile(self, name):
     """Helper function testDefaultName."""
-    output = while_v2.while_loop(lambda i: i < 3, lambda i: i + 1,
-                                 [constant_op.constant(0)])
+    output = while_v2.while_loop(
+        lambda i: i < 3,
+        lambda i: i + 1, [constant_op.constant(0)],
+        return_same_structure=False)
     while_op = output.op.inputs[0].op
     self.assertEqual(while_op.type, "While")
     return while_op
@@ -329,9 +364,14 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     def Body(i, previous_sum):
       prod = constant_op.constant(1.)
       return i - 1., previous_sum + while_loop_v2(
-          lambda c, _: c > 0, lambda c, v: (c - 1., v * n), [i, prod])[1]
-
-    result = while_loop_v2(lambda i, _: i >= 0, Body, [m, sum_of_powers])[1]
+          lambda c, _: c > 0,
+          lambda c, v: (c - 1., v * n), [i, prod],
+          return_same_structure=False)[1]
+
+    result = while_loop_v2(
+        lambda i, _: i >= 0,
+        Body, [m, sum_of_powers],
+        return_same_structure=False)[1]
     grad = gradients_impl.gradients(result, [n])
     with self.cached_session() as sess:
       self.assertEqual(self.evaluate(result), 364.)
@@ -345,7 +385,8 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       return v * v
 
     x = constant_op.constant(2.)
-    ret = while_loop_v2(lambda v: v < 8., Body, [x])
+    ret = while_loop_v2(
+        lambda v: v < 8., Body, [x], return_same_structure=False)
     grad = gradients_impl.gradients(ret, [x])
     with self.cached_session() as sess:
       self.assertEqual(self.evaluate(ret), 16.)
@@ -363,13 +404,17 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
         return row, col + 1., ta, n
 
       # TODO(b/118457764): Remove n from loop_vars from both loops once fixed.
-      ta = while_loop_v2(lambda _, col, _1, n: col <= n, InnerBody,
-                         [row, constant_op.constant(1.), ta, n])[2]
+      ta = while_loop_v2(
+          lambda _, col, _1, n: col <= n,
+          InnerBody, [row, constant_op.constant(1.), ta, n],
+          return_same_structure=False)[2]
       return row + 1., ta, n
 
     ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=9)
-    ta = while_loop_v2(lambda row, _, _1: row <= n, Body,
-                       [constant_op.constant(1.), ta, n])[1]
+    ta = while_loop_v2(
+        lambda row, _, _1: row <= n,
+        Body, [constant_op.constant(1.), ta, n],
+        return_same_structure=False)[1]
 
     output = array_ops.reshape(ta.stack(), [3, 3])
     self.assertAllEqual(
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index a36a24ebb0..b7e50c1dae 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -3145,7 +3145,6 @@ def while_loop_v2(cond,
                   back_prop=True,
                   swap_memory=False,
                   maximum_iterations=None,
-                  return_same_structure=False,
                   name=None):
   """Repeat `body` while the condition `cond` is true.
 
@@ -3221,17 +3220,11 @@ def while_loop_v2(cond,
       to run.  If provided, the `cond` output is AND-ed with an additional
       condition ensuring the number of iterations executed is no greater than
       `maximum_iterations`.
-    return_same_structure: If True, output has same structure as `loop_vars`. If
-      eager execution is enabled, this is ignored (and always treated as True).
     name: Optional name prefix for the returned tensors.
 
   Returns:
-    The output tensors for the loop variables after the loop.
-     If `return_same_structure` is True, the return value has the same
-     structure as `loop_vars`.
-     If `return_same_structure` is False, the return value is a Tensor,
-     TensorArray or IndexedSlice if the length of `loop_vars` is 1, or a list
-     otherwise.
+    The output tensors for the loop variables after the loop. The return value
+      has the same structure as `loop_vars`.
 
   Raises:
     TypeError: if `cond` or `body` is not callable.
@@ -3318,7 +3311,7 @@ def while_loop_v2(cond,
       swap_memory=swap_memory,
       name=name,
       maximum_iterations=maximum_iterations,
-      return_same_structure=return_same_structure)
+      return_same_structure=True)
 
 
 # pylint: disable=redefined-outer-name
@@ -3501,7 +3494,8 @@ def while_loop(cond,
         loop_vars,
         shape_invariants=shape_invariants,
         maximum_iterations=maximum_iterations,
-        name=name)
+        name=name,
+        return_same_structure=return_same_structure)
 
   with ops.name_scope(name, "while", loop_vars):
     if not loop_vars:
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 6821b63d0e..59ca29e3ba 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -64,7 +64,8 @@ def while_loop(cond,
                loop_vars,
                shape_invariants=None,
                maximum_iterations=None,
-               name=None):
+               name=None,
+               return_same_structure=True):
   """Like tf.while_loop, except emits a single While op."""
   maximum_iterations = _validate_and_convert_to_tensor(maximum_iterations)
   # Keep the original loop_vars around to know which args were TensorArrays.
@@ -258,6 +259,9 @@ def while_loop(cond,
   outputs = _pack_sequence_as(orig_loop_vars,
                               outputs[1:1 + num_flattened_outputs])
 
+  if return_same_structure:
+    return outputs
+
   flattened_outputs = nest.flatten(outputs)
   if len(flattened_outputs) == 1:
     return flattened_outputs[0]
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 0659900ffa..40693e7582 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1130,7 +1130,7 @@ tf_module {
   }
   member_method {
     name: "while_loop"
-    argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'maximum_iterations\', \'return_same_structure\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'False\', \'None\'], "
+    argspec: "args=[\'cond\', \'body\', \'loop_vars\', \'shape_invariants\', \'parallel_iterations\', \'back_prop\', \'swap_memory\', \'maximum_iterations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'False\', \'None\', \'None\'], "
   }
   member_method {
     name: "zeros"
-- 
GitLab


From 3655dc13a2576567a7719bc9307c5faac48fae6b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 16:15:21 -0800
Subject: [PATCH 0974/1554] Fix XLA documentation for ReduceWindow

PiperOrigin-RevId: 223254699
---
 tensorflow/compiler/xla/g3doc/operation_semantics.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index bc87a60c6e..df74a9b3af 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -1796,8 +1796,9 @@ XlaBuilder builder(client_, "reduce_window_2x3");
 auto shape = ShapeUtil::MakeShape(F32, {4, 6});
 auto input = builder.Parameter(0, shape, "input");
 builder.ReduceWindow(
-    input, *max,
+    input,
     /*init_val=*/builder.ConstantLiteral(LiteralUtil::MinValue(F32)),
+    *max,
     /*window_dimensions=*/{2, 3},
     /*window_stride_dimensions=*/{2, 3},
     Padding::kValid);
-- 
GitLab


From d7418be7bff57803de971336dda25cd3d844ac79 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Wed, 28 Nov 2018 16:27:18 -0800
Subject: [PATCH 0975/1554] Ran a newer version of clang-format

---
 tensorflow/core/kernels/mkl_conv_ops.cc | 100 ++++++++++++------------
 1 file changed, 49 insertions(+), 51 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 8e3b669a59..75f08956b4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -323,7 +323,7 @@ class MklConvFwdPrimitiveFactory : public MklPrimitiveFactory<T> {
       const MklConvFwdParams& convFwdDims, bool do_not_cache) {
     MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>* conv_fwd = nullptr;
 
-    if (do_not_cache) {/* Always create new primitive */
+    if (do_not_cache) { /* Always create new primitive */
       conv_fwd = new MklConvFwdPrimitive<T, Tinput, Tfilter, Tbias, Toutput>(
           convFwdDims);
     } else {
@@ -423,16 +423,15 @@ class MklConvOp : public OpKernel {
     OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES(context, strides_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window strides field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 dimensions"));
 
     const int64 stride_n = GetTensorDim(strides_, data_format_, 'N');
     const int64 stride_c = GetTensorDim(strides_, data_format_, 'C');
-    OP_REQUIRES(context, stride_n == 1 && stride_c == 1,
-                errors::InvalidArgument(
-                    "Current implementation does not yet support "
-                    "strides in the batch and depth dimensions."));
+    OP_REQUIRES(
+        context, stride_n == 1 && stride_c == 1,
+        errors::InvalidArgument("Current implementation does not yet support "
+                                "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
   }
 
@@ -727,7 +726,7 @@ class MklConvOp : public OpKernel {
           mkl_prim_convert_input;
       dnnLayout_t mkl_lt_internal_filter, mkl_lt_internal_bias,
           mkl_lt_internal_input;
-      void* mkl_buf_convert_input, *mkl_buf_convert_filter,
+      void *mkl_buf_convert_input, *mkl_buf_convert_filter,
           *mkl_buf_convert_bias;
       mkl_prim_convert_filter = nullptr;
       mkl_prim_convert_bias = nullptr;
@@ -860,23 +859,21 @@ class MklConvOp : public OpKernel {
     OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES(context, (strides_.size() == 4 || strides_.size() == 5),
-                errors::InvalidArgument(
-                    "Sliding window strides field must "
-                    "specify 4 or 5 dimensions"));
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 or 5 dimensions"));
 
     const int64 stride_n = GetTensorDim(strides_, data_format_, 'N');
     const int64 stride_c = GetTensorDim(strides_, data_format_, 'C');
-    OP_REQUIRES(context, stride_n == 1 && stride_c == 1,
-                errors::InvalidArgument(
-                    "Current implementation does not yet support "
-                    "strides in the batch and depth dimensions."));
+    OP_REQUIRES(
+        context, stride_n == 1 && stride_c == 1,
+        errors::InvalidArgument("Current implementation does not yet support "
+                                "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
 
     if (strides_.size() == 4) {
       OP_REQUIRES(context, dilations_.size() == 4,
-                  errors::InvalidArgument(
-                      "Sliding window dilations field must "
-                      "specify 4 dimensions"));
+                  errors::InvalidArgument("Sliding window dilations field must "
+                                          "specify 4 dimensions"));
       const int64 dilation_n = GetTensorDim(dilations_, data_format_, 'N');
       const int64 dilation_c = GetTensorDim(dilations_, data_format_, 'C');
       const int64 dilation_h = GetTensorDim(dilations_, data_format_, 'H');
@@ -890,9 +887,8 @@ class MklConvOp : public OpKernel {
           errors::InvalidArgument("Dilated rates should be larger than 0."));
     } else if (strides_.size() == 5) {
       OP_REQUIRES(context, dilations_.size() == 5,
-                  errors::InvalidArgument(
-                      "Dilation rates field must "
-                      "specify 5 dimensions"));
+                  errors::InvalidArgument("Dilation rates field must "
+                                          "specify 5 dimensions"));
       OP_REQUIRES(context, (GetTensorDim(dilations_, data_format_, 'N') == 1 &&
                             GetTensorDim(dilations_, data_format_, 'C') == 1),
                   errors::InvalidArgument(
@@ -916,9 +912,8 @@ class MklConvOp : public OpKernel {
       GetMklShape(context, kInputIndex_Src, &src_mkl_shape);
       GetMklShape(context, kInputIndex_Filter, &filter_mkl_shape);
       OP_REQUIRES(context, filter_mkl_shape.IsMklTensor() == false,
-                  errors::InvalidArgument(
-                      "Filter should not be in "
-                      "Mkl Layout"));
+                  errors::InvalidArgument("Filter should not be in "
+                                          "Mkl Layout"));
 
       MklDnnData<Tinput> src(&cpu_engine_);
       MklDnnData<Tfilter> filter(&cpu_engine_);
@@ -954,8 +949,9 @@ class MklConvOp : public OpKernel {
         filter_mkl_shape.SetMklTensor(false);
         Tensor* output_filter_tensor = nullptr;
         // MklConv2D also outputs converted filter as 2nd output.
-        if (typeid(Tinput) == typeid(float)&&typeid(Tfilter) ==
-            typeid(float)&&typeid(Toutput) == typeid(float)) {
+        if (typeid(Tinput) == typeid(float) &&
+            typeid(Tfilter) == typeid(float) &&
+            typeid(Toutput) == typeid(float)) {
           filter_mkl_shape.SetMklTensor(false);
           AllocateOutputSetMklShape(context, kOutputIndex_Filter,
                                     &output_filter_tensor, filter_tf_shape,
@@ -1042,8 +1038,8 @@ class MklConvOp : public OpKernel {
       AllocateOutputTensor(context, *conv_fwd_pd, dst_dims_mkl_order, tf_fmt,
                            &dst_tensor);
       Tensor* filter_out_tensor = nullptr;
-      if (typeid(Tinput) == typeid(float)&&typeid(Tfilter) ==
-          typeid(float)&&typeid(Toutput) == typeid(float)) {
+      if (typeid(Tinput) == typeid(float) && typeid(Tfilter) == typeid(float) &&
+          typeid(Toutput) == typeid(float)) {
         AllocateFilterOutputTensor(context, *conv_fwd_pd,
                                    TFShapeToMklDnnDims(filter_tf_shape),
                                    &filter_out_tensor);
@@ -1092,8 +1088,7 @@ class MklConvOp : public OpKernel {
 
       // delete primitive since it is not cached.
       if (do_not_cache) delete conv_fwd;
-    }
-    catch (mkldnn::error& e) {
+    } catch (mkldnn::error& e) {
       string error_msg = tensorflow::strings::StrCat(
           "Status: ", e.status, ", message: ", string(e.message), ", in file ",
           __FILE__, ":", __LINE__);
@@ -1789,31 +1784,34 @@ REGISTER_KERNEL_BUILDER(
 #endif  // INTEL_MKL_ML
 
 // Register 2D operations
-#define REGISTER_MKL_CPU_2D(T)                                                \
-  REGISTER_KERNEL_BUILDER(                                                    \
-      Name("_MklConv2D").Device(DEVICE_CPU).TypeConstraint<float>("T").Label( \
-          mkl_op_registry::kMklOpLabel),                                      \
-      MklConvOp<CPUDevice, float, float, float, float, float, false>);        \
-  REGISTER_KERNEL_BUILDER(                                                    \
-      Name("_MklConv2DWithBias")                                              \
-          .Device(DEVICE_CPU)                                                 \
-          .TypeConstraint<float>("T")                                         \
-          .Label(mkl_op_registry::kMklOpLabel),                               \
-      MklConvOp<CPUDevice, float, float, float, float, float, true>);         \
-  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")                    \
-                              .Device(DEVICE_CPU)                             \
-                              .TypeConstraint<T>("T")                         \
-                              .Label(mkl_op_registry::kMklOpLabel),           \
+#define REGISTER_MKL_CPU_2D(T)                                         \
+  REGISTER_KERNEL_BUILDER(                                             \
+      Name("_MklConv2D")                                               \
+          .Device(DEVICE_CPU)                                          \
+          .TypeConstraint<float>("T")                                  \
+          .Label(mkl_op_registry::kMklOpLabel),                        \
+      MklConvOp<CPUDevice, float, float, float, float, float, false>); \
+  REGISTER_KERNEL_BUILDER(                                             \
+      Name("_MklConv2DWithBias")                                       \
+          .Device(DEVICE_CPU)                                          \
+          .TypeConstraint<float>("T")                                  \
+          .Label(mkl_op_registry::kMklOpLabel),                        \
+      MklConvOp<CPUDevice, float, float, float, float, float, true>);  \
+  REGISTER_KERNEL_BUILDER(Name("__MklDummyConv2DWithBias")             \
+                              .Device(DEVICE_CPU)                      \
+                              .TypeConstraint<T>("T")                  \
+                              .Label(mkl_op_registry::kMklOpLabel),    \
                           MklDummyOp<CPUDevice, T>);
 
 TF_CALL_float(REGISTER_MKL_CPU_2D);
 
 // Register 3D operations
-#define REGISTER_MKL_CPU_3D(T)                                            \
-  REGISTER_KERNEL_BUILDER(                                                \
-      Name("_MklConv3D").Device(DEVICE_CPU).TypeConstraint<T>("T").Label( \
-          mkl_op_registry::kMklOpLabel),                                  \
-      MklConvOp<CPUDevice, T, T, T, T, T, false>);
+#define REGISTER_MKL_CPU_3D(T)                                      \
+  REGISTER_KERNEL_BUILDER(Name("_MklConv3D")                        \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<T>("T")               \
+                              .Label(mkl_op_registry::kMklOpLabel), \
+                          MklConvOp<CPUDevice, T, T, T, T, T, false>);
 TF_CALL_float(REGISTER_MKL_CPU_3D);
 
 }  // namespace tensorflow
-- 
GitLab


From e6b3d3b22a26555ccb00403c6db87813752c675b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 28 Nov 2018 16:15:45 -0800
Subject: [PATCH 0976/1554] Avoid unused variable warnings from XLA_MAKE_BINARY

PiperOrigin-RevId: 223254783
---
 tensorflow/compiler/tf2xla/kernels/binary_ops.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index 47e517a657..74c1dd2406 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -43,6 +43,9 @@ namespace {
         const std::vector<int64>& extend_dimensions) override {          \
       xla::XlaBuilder* b = ctx->builder();                               \
       (void)b;                                                           \
+      (void)lhs_shape;                                                   \
+      (void)rhs_shape;                                                   \
+      (void)extend_dimensions;                                           \
       return HLO;                                                        \
     }                                                                    \
   };                                                                     \
-- 
GitLab


From 22ff3ec66ef0a2453e353d8b1ed21a1866884381 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 16:24:30 -0800
Subject: [PATCH 0977/1554] Adds tf.abs to tf.lite schema

PiperOrigin-RevId: 223256197
---
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   1 +
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   6 +
 tensorflow/lite/schema/schema_generated.h     | 124 +++++++++++++++++-
 5 files changed, 127 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index 2300ff4ed2..63b5cd1960 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -126,6 +126,7 @@ typedef enum {
   kTfLiteBuiltinLeakyRelu = 98,
   kTfLiteBuiltinSquaredDifference = 99,
   kTfLiteBuiltinMirrorPad = 100,
+  kTfLiteBuiltinAbs = 101,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index aa9b372398..d3383bccf0 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -644,6 +644,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     }
 
     // Below are the ops with no builtin_data strcture.
+    case BuiltinOperator_ABS:
     case BuiltinOperator_BATCH_TO_SPACE_ND:
     // TODO(aselle): Implement call in BuiltinOptions, but nullptrs are
     // ok for now, since there is no call implementation either.
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 58288a8dd4..312cedda6c 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -684,6 +684,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_LEAKY_RELU:
       case tflite::BuiltinOperator_SQUARED_DIFFERENCE:
       case tflite::BuiltinOperator_MIRROR_PAD:
+      case tflite::BuiltinOperator_ABS:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 652871d013..094db1a1d5 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -203,6 +203,7 @@ enum BuiltinOperator : byte {
   LEAKY_RELU = 98,
   SQUARED_DIFFERENCE = 99,
   MIRROR_PAD = 100,
+  ABS = 101,
 }
 
 // Options for the builtin operators.
@@ -284,6 +285,7 @@ union BuiltinOptions {
   LeakyReluOptions,
   SquaredDifferenceOptions,
   MirrorPadOptions,
+  AbsOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -635,6 +637,10 @@ table OneHotOptions {
   axis:int;
 }
 
+table AbsOptions {
+}
+
+
 table LogicalAndOptions {
 }
 
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 1464c75613..1d560b3298 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -226,6 +226,9 @@ struct LogicalOrOptionsT;
 struct OneHotOptions;
 struct OneHotOptionsT;
 
+struct AbsOptions;
+struct AbsOptionsT;
+
 struct LogicalAndOptions;
 struct LogicalAndOptionsT;
 
@@ -512,11 +515,12 @@ enum BuiltinOperator {
   BuiltinOperator_LEAKY_RELU = 98,
   BuiltinOperator_SQUARED_DIFFERENCE = 99,
   BuiltinOperator_MIRROR_PAD = 100,
+  BuiltinOperator_ABS = 101,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_MIRROR_PAD
+  BuiltinOperator_MAX = BuiltinOperator_ABS
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[100] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[101] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -617,7 +621,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[100] {
     BuiltinOperator_RESIZE_NEAREST_NEIGHBOR,
     BuiltinOperator_LEAKY_RELU,
     BuiltinOperator_SQUARED_DIFFERENCE,
-    BuiltinOperator_MIRROR_PAD
+    BuiltinOperator_MIRROR_PAD,
+    BuiltinOperator_ABS
   };
   return values;
 }
@@ -725,6 +730,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "LEAKY_RELU",
     "SQUARED_DIFFERENCE",
     "MIRROR_PAD",
+    "ABS",
     nullptr
   };
   return names;
@@ -814,11 +820,12 @@ enum BuiltinOptions {
   BuiltinOptions_LeakyReluOptions = 75,
   BuiltinOptions_SquaredDifferenceOptions = 76,
   BuiltinOptions_MirrorPadOptions = 77,
+  BuiltinOptions_AbsOptions = 78,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_MirrorPadOptions
+  BuiltinOptions_MAX = BuiltinOptions_AbsOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[78] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[79] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -897,7 +904,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[78] {
     BuiltinOptions_ResizeNearestNeighborOptions,
     BuiltinOptions_LeakyReluOptions,
     BuiltinOptions_SquaredDifferenceOptions,
-    BuiltinOptions_MirrorPadOptions
+    BuiltinOptions_MirrorPadOptions,
+    BuiltinOptions_AbsOptions
   };
   return values;
 }
@@ -982,6 +990,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "LeakyReluOptions",
     "SquaredDifferenceOptions",
     "MirrorPadOptions",
+    "AbsOptions",
     nullptr
   };
   return names;
@@ -1304,6 +1313,10 @@ template<> struct BuiltinOptionsTraits<MirrorPadOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_MirrorPadOptions;
 };
 
+template<> struct BuiltinOptionsTraits<AbsOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_AbsOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1951,6 +1964,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_MirrorPadOptions ?
       reinterpret_cast<const MirrorPadOptionsT *>(value) : nullptr;
   }
+  AbsOptionsT *AsAbsOptions() {
+    return type == BuiltinOptions_AbsOptions ?
+      reinterpret_cast<AbsOptionsT *>(value) : nullptr;
+  }
+  const AbsOptionsT *AsAbsOptions() const {
+    return type == BuiltinOptions_AbsOptions ?
+      reinterpret_cast<const AbsOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -6339,6 +6360,46 @@ inline flatbuffers::Offset<OneHotOptions> CreateOneHotOptions(
 
 flatbuffers::Offset<OneHotOptions> CreateOneHotOptions(flatbuffers::FlatBufferBuilder &_fbb, const OneHotOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct AbsOptionsT : public flatbuffers::NativeTable {
+  typedef AbsOptions TableType;
+  AbsOptionsT() {
+  }
+};
+
+struct AbsOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef AbsOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  AbsOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(AbsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<AbsOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct AbsOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit AbsOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  AbsOptionsBuilder &operator=(const AbsOptionsBuilder &);
+  flatbuffers::Offset<AbsOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<AbsOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<AbsOptions> CreateAbsOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  AbsOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<AbsOptions> CreateAbsOptions(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct LogicalAndOptionsT : public flatbuffers::NativeTable {
   typedef LogicalAndOptions TableType;
   LogicalAndOptionsT() {
@@ -7237,6 +7298,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const MirrorPadOptions *builtin_options_as_MirrorPadOptions() const {
     return builtin_options_type() == BuiltinOptions_MirrorPadOptions ? static_cast<const MirrorPadOptions *>(builtin_options()) : nullptr;
   }
+  const AbsOptions *builtin_options_as_AbsOptions() const {
+    return builtin_options_type() == BuiltinOptions_AbsOptions ? static_cast<const AbsOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7576,6 +7640,10 @@ template<> inline const MirrorPadOptions *Operator::builtin_options_as<MirrorPad
   return builtin_options_as_MirrorPadOptions();
 }
 
+template<> inline const AbsOptions *Operator::builtin_options_as<AbsOptions>() const {
+  return builtin_options_as_AbsOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -9854,6 +9922,29 @@ inline flatbuffers::Offset<OneHotOptions> CreateOneHotOptions(flatbuffers::FlatB
       _axis);
 }
 
+inline AbsOptionsT *AbsOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new AbsOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void AbsOptions::UnPackTo(AbsOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<AbsOptions> AbsOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateAbsOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<AbsOptions> CreateAbsOptions(flatbuffers::FlatBufferBuilder &_fbb, const AbsOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const AbsOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateAbsOptions(
+      _fbb);
+}
+
 inline LogicalAndOptionsT *LogicalAndOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new LogicalAndOptionsT();
   UnPackTo(_o, _resolver);
@@ -10708,6 +10799,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const MirrorPadOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_AbsOptions: {
+      auto ptr = reinterpret_cast<const AbsOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -11034,6 +11129,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const MirrorPadOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_AbsOptions: {
+      auto ptr = reinterpret_cast<const AbsOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11348,6 +11447,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const MirrorPadOptionsT *>(value);
       return CreateMirrorPadOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_AbsOptions: {
+      auto ptr = reinterpret_cast<const AbsOptionsT *>(value);
+      return CreateAbsOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -11662,6 +11765,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new MirrorPadOptionsT(*reinterpret_cast<MirrorPadOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_AbsOptions: {
+      value = new AbsOptionsT(*reinterpret_cast<AbsOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -12054,6 +12161,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_AbsOptions: {
+      auto ptr = reinterpret_cast<AbsOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
GitLab


From f6ee54c9b160931623fcc5f52e704e9d6d29f74d Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 28 Nov 2018 16:49:14 -0800
Subject: [PATCH 0978/1554] cond_v2: use optional tensors instead of
 FakeParams.

The purpose of this change is to not waste memory allocating large
FakeParams, which is especially important on GPU.

This also adds a few other fixes needed to get optional variants
working with cond_v2, including on GPU.

PiperOrigin-RevId: 223260005
---
 tensorflow/core/common_runtime/placer.cc      |   2 +-
 tensorflow/core/kernels/control_flow_ops.cc   |   1 +
 tensorflow/python/BUILD                       |  11 ++
 .../python/kernel_tests/cond_v2_test.py       |  18 +--
 .../kernel_tests/control_flow_ops_py_test.py  |   6 +-
 tensorflow/python/ops/cond_v2.py              | 140 +++++++++++++++---
 tensorflow/python/ops/control_flow_util.py    |   5 +
 tensorflow/python/ops/gradients_impl.py       |   1 +
 tensorflow/python/ops/optional_grad.py        |  33 +++++
 9 files changed, 180 insertions(+), 37 deletions(-)
 create mode 100644 tensorflow/python/ops/optional_grad.py

diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index 01e4072f60..515c1971d9 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -1002,7 +1002,7 @@ Status Placer::Run() {
     int assigned_device = -1;
 
     // Heuristic A application.
-    if (IsGeneratorNode(node)) {
+    if (IsGeneratorNode(node) && !node->out_edges().empty()) {
       const Node* output = (*node->out_edges().begin())->dst();
       int output_device_name = output->assigned_device_name_index();
 
diff --git a/tensorflow/core/kernels/control_flow_ops.cc b/tensorflow/core/kernels/control_flow_ops.cc
index 1587eb5114..081ef72c15 100644
--- a/tensorflow/core/kernels/control_flow_ops.cc
+++ b/tensorflow/core/kernels/control_flow_ops.cc
@@ -78,6 +78,7 @@ TF_CALL_QUANTIZED_TYPES(REGISTER_GPU_SWITCH);
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_REF_SWITCH);
 TF_CALL_QUANTIZED_TYPES(REGISTER_GPU_REF_SWITCH);
 REGISTER_GPU_SWITCH(uint64);
+TF_CALL_variant(REGISTER_GPU_SWITCH);
 
 #undef REGISTER_CPU_SWITCH
 #undef REGISTER_CPU_REF_SWITCH
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 19d2af4515..2dbca6ed6e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1972,6 +1972,15 @@ py_library(
     ],
 )
 
+py_library(
+    name = "optional_grad",
+    srcs = ["ops/optional_grad.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_ops",
+    ],
+)
+
 py_library(
     name = "sets",
     srcs = [
@@ -2151,6 +2160,7 @@ py_library(
         ":graph_to_function_def",
         ":pywrap_tensorflow",
         ":util",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/eager:function",
     ],
 )
@@ -2295,6 +2305,7 @@ py_library(
         ":manip_ops",
         ":math_grad",
         ":math_ops",
+        ":optional_grad",
         ":platform",
         ":random_grad",
         ":resource_variable_ops",
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index ace18dbc44..09e41ec8c6 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -126,7 +126,7 @@ class CondV2Test(test.TestCase):
       self.assertEqual(sess.run(out, {pred: False}), (2.0,))
 
   def _createCond(self, name):
-    """Helper function for testDefaultName."""
+    """Creates a cond_v2 call and returns the output tensor and the cond op."""
     pred = constant_op.constant(True, name="pred")
     x = constant_op.constant(1.0, name="x")
 
@@ -139,11 +139,11 @@ class CondV2Test(test.TestCase):
     output = cond_v2.cond_v2(pred, true_fn, false_fn, name=name)
     cond_op = output.op.inputs[0].op
     self.assertEqual(cond_op.type, "If")
-    return cond_op
+    return output, cond_op
 
   def testDefaultName(self):
     with ops.Graph().as_default():
-      cond_op = self._createCond(None)
+      _, cond_op = self._createCond(None)
       self.assertEqual(cond_op.name, "cond")
       self.assertRegexpMatches(
           cond_op.get_attr("then_branch").name, r"cond_true_\d*")
@@ -152,14 +152,14 @@ class CondV2Test(test.TestCase):
 
     with ops.Graph().as_default():
       with ops.name_scope("foo"):
-        cond1_op = self._createCond("")
+        _, cond1_op = self._createCond("")
         self.assertEqual(cond1_op.name, "foo/cond")
         self.assertRegexpMatches(
             cond1_op.get_attr("then_branch").name, r"foo_cond_true_\d*")
         self.assertRegexpMatches(
             cond1_op.get_attr("else_branch").name, r"foo_cond_false_\d*")
 
-        cond2_op = self._createCond(None)
+        _, cond2_op = self._createCond(None)
         self.assertEqual(cond2_op.name, "foo/cond_1")
         self.assertRegexpMatches(
             cond2_op.get_attr("then_branch").name, r"foo_cond_1_true_\d*")
@@ -612,11 +612,11 @@ class CondV2Test(test.TestCase):
   def testLowering(self):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
-        out_cond = self._createCond("cond")
+        cond_output, _ = self._createCond("cond")
 
         run_options = config_pb2.RunOptions(output_partition_graphs=True)
         run_metadata = config_pb2.RunMetadata()
-        sess.run(out_cond, options=run_options, run_metadata=run_metadata)
+        sess.run(cond_output, options=run_options, run_metadata=run_metadata)
 
         # If lowering was enabled, there should be a `Switch` node
         switch_found = any(
@@ -641,12 +641,12 @@ class CondV2Test(test.TestCase):
       # Build the cond_v2 in an XLA context
       xla_context = control_flow_ops.XLAControlFlowContext()
       xla_context.Enter()
-      out_cond = self._createCond("cond")
+      cond_output, _ = self._createCond("cond")
       xla_context.Exit()
 
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       run_metadata = config_pb2.RunMetadata()
-      sess.run(out_cond, options=run_options, run_metadata=run_metadata)
+      sess.run(cond_output, options=run_options, run_metadata=run_metadata)
 
       # Lowering disabled in XLA, there should be no `Switch` node
       switch_found = any(
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 37654abd18..685d0438af 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -446,8 +446,7 @@ class ControlFlowTest(test.TestCase):
         g = gradients_impl.gradients(y, x)[0]
 
       self.assertAllEqual(sess.run(g, {pred: True}), [2.0, 2.0, 2.0])
-      # TODO(b/119791601): Enable this.
-      # self.assertAllEqual(sess.run(g, {pred: False}), [0.0, 0.0, 0.0])
+      self.assertAllEqual(sess.run(g, {pred: False}), [0.0, 0.0, 0.0])
 
   @test_util.disable_control_flow_v2("b/113293074")
   def testCondIndexedSlicesDifferentTypes(self):
@@ -2168,11 +2167,8 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(512.0, self.evaluate(r))
 
   def testNestedWhileCondWhileGrad(self):
-    if control_flow_ops.ENABLE_WHILE_V2 and test_util.is_gpu_available():
-      self.skipTest("b/118459209")
     self._testNestedWhileCondWhileGrad(use_gpu=False)
 
-  @test_util.disable_control_flow_v2("b/118459209")
   def testNestedWhileCondWhileGradGpu(self):
     self._testNestedWhileCondWhileGrad(use_gpu=True)
 
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 927c64919d..f7a95bd909 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -30,7 +30,9 @@ from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import function_def_to_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import control_flow_util_v2 as util
+from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gen_functional_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gradients_impl
@@ -149,7 +151,9 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
                                    true_inputs, false_inputs)
 
   # Add all intermediate tensors as function outputs so they're available for
-  # the gradient computation.
+  # the gradient computation. Since the outputs of the two functions must match,
+  # we wrap all the intermediates in optionals. Each intermediate output will
+  # have a value iff its corresponding branch is taken.
 
   true_intermediates = _get_intermediates(true_graph)
   false_intermediates = _get_intermediates(false_graph)
@@ -157,12 +161,28 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
   # Save the original number of outputs to return to the caller.
   num_cond_outputs = len(true_graph.outputs)
 
-  # Make the number/type of new intermediate outputs match.
-  extra_true_outputs, extra_false_outputs = _pad_params(
-      true_graph, false_graph, true_intermediates, false_intermediates)
+  if control_flow_util.InXlaContext(ops.get_default_graph()):
+    # XLA does not yet support optionals, so output intermediates directly and
+    # make them match via FakeParams, which can be converted to zeros in XLA.
+    # TODO(skyewm,jpienaar): can XLA support optionals?
+    extra_true_outputs, extra_false_outputs = _make_intermediates_match_xla(
+        true_graph, false_graph, true_intermediates, false_intermediates)
+  else:
+    # Wrap intermediates in optionals.
+    wrapped_true_intermediates = _wrap_intermediates(true_graph,
+                                                     true_intermediates)
+    wrapped_false_intermediates = _wrap_intermediates(false_graph,
+                                                      false_intermediates)
+
+    # Make outputs match by adding none optionals.
+    extra_true_outputs, extra_false_outputs = _make_intermediates_match(
+        true_graph, false_graph,
+        wrapped_true_intermediates, wrapped_false_intermediates)
 
   true_graph.outputs.extend(extra_true_outputs)
   false_graph.outputs.extend(extra_false_outputs)
+  # TODO(skyewm): somehow indicate it's a bug if this fails.
+  _check_same_outputs(true_graph, false_graph)
 
   # Create the If op.
   tensors = gen_functional_ops._if(  # pylint: disable=protected-access
@@ -175,7 +195,8 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
       name=name)
 
   # TODO(b/110167197) this approach requires cond_v2 to have at least 1 output
-  util.maybe_set_lowering_attr(tensors[0].op)
+  if_op = tensors[0].op
+  util.maybe_set_lowering_attr(if_op)
 
   # Return identities for each output of the If op, rather than the output of
   # the If op directly. This makes pruning work if the output of cond() is
@@ -187,6 +208,9 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
   # correct output structure
   tensors = [array_ops.identity(t) for t in tensors]
 
+  # Prevent fetching since the variant outputs can't be fetched directly.
+  if_op.graph.prevent_fetching(if_op)
+
   return tensors[:num_cond_outputs]
 
 
@@ -278,7 +302,7 @@ def _create_grad_func(func_graph, grads, name):
   return func_graph_module.func_graph_from_py_func(
       name,
       lambda: _grad_fn(func_graph, grads), [], {},
-      func_graph=util.CondBranchFuncGraph(name, read_only_collections=False))
+      func_graph=_CondGradFuncGraph(name, func_graph))
 
 
 def _resolve_grad_inputs(cond_graph, grad_graph):
@@ -360,28 +384,39 @@ def _separate_unique_inputs(true_inputs, false_inputs):
   return list(shared_inputs), list(true_only_inputs), list(false_only_inputs)
 
 
-def _pad_params(true_graph, false_graph, true_params, false_params):
-  """Returns new param lists that have matching signatures.
+def _make_intermediates_match(true_graph, false_graph,
+                              true_optionals, false_optionals):
+  """Returns new optionals lists that have matching signatures.
 
-  This is done by mirroring each param list in the other using dummy params.
-  There is no merging of params.
+  This is done by mirroring each list in the other using none optionals.
+  There is no merging of like optionals.
 
   Args:
     true_graph: FuncGraph
     false_graph: FuncGraph
-    true_params: a list of Tensors from true_graph
-    false_params: a list of Tensors from false_graph
+    true_optionals: a list of optional Tensors from true_graph
+    false_optionals: a list of optional Tensors from false_graph
 
   Returns:
     A new list of Tensors in true_graph and a new list of Tensors in
-    false_graph. The two lists have the same number of Tensors, with matching
-    types and shapes across the lists.
+    false_graph. The two lists have the same number of Tensors, all of which
+    will be optionals of the same shape/type.
   """
-  new_true_params = (true_params +
-                     _create_dummy_params(true_graph, false_params))
-  new_false_inputs = (_create_dummy_params(false_graph, true_params)
-                      + false_params)
-  return new_true_params, new_false_inputs
+  new_true_optionals = (true_optionals +
+                        _create_none_optionals(true_graph, false_optionals))
+  new_false_optionals = (_create_none_optionals(false_graph, true_optionals)
+                         + false_optionals)
+  return new_true_optionals, new_false_optionals
+
+
+def _make_intermediates_match_xla(true_graph, false_graph, true_intermediates,
+                                  false_intermediates):
+  """Like _make_intermediates_match but for the XLA case."""
+  new_true_intermediates = (true_intermediates +
+                            _create_fakeparams(true_graph, false_intermediates))
+  new_false_intermediates = (_create_fakeparams(false_graph, true_intermediates)
+                             + false_intermediates)
+  return new_true_intermediates, new_false_intermediates
 
 
 def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
@@ -416,11 +451,11 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
   true_graph.inputs = (
       [true_input_to_param[t] for t in shared_inputs] +
       [true_input_to_param[t] for t in true_only_inputs] +
-      _create_dummy_params(true_graph, false_only_inputs))
+      _create_dummy_inputs(true_graph, false_only_inputs))
 
   false_graph.inputs = (
       [false_input_to_param[t] for t in shared_inputs] +
-      _create_dummy_params(false_graph, true_only_inputs) +
+      _create_dummy_inputs(false_graph, true_only_inputs) +
       [false_input_to_param[t] for t in false_only_inputs])
 
   # Rewrite the FuncGraphs' state to reflect the new inputs.
@@ -432,7 +467,12 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
   return new_inputs
 
 
-def _create_dummy_params(func_graph, template_tensors):
+def _wrap_intermediates(func_graph, intermediates):
+  with func_graph.as_default():
+    return [gen_dataset_ops.optional_from_value([t]) for t in intermediates]
+
+
+def _create_dummy_inputs(func_graph, template_tensors):
   """Creates tensors in func_graph to represent template_tensors.
 
   Args:
@@ -442,6 +482,27 @@ def _create_dummy_params(func_graph, template_tensors):
   Returns:
     A list of tensors in func_graph.
   """
+  with func_graph.as_default():
+    return [array_ops.placeholder(t.dtype, shape=t.shape)
+            for t in template_tensors]
+
+
+def _create_none_optionals(func_graph, template_tensors):
+  """Creates none optionals in func_graph to represent template_tensors.
+
+  Args:
+    func_graph: FuncGraph.
+    template_tensors: a list of tensors in func_graph.
+
+  Returns:
+    A list of tensors in func_graph.
+  """
+  with func_graph.as_default():
+    return [gen_dataset_ops.optional_none() for _ in template_tensors]
+
+
+def _create_fakeparams(func_graph, template_tensors):
+  """Create FakeParams for the XLA case."""
   with func_graph.as_default():
     return [gen_functional_ops.fake_param(dtype=t.dtype, shape=t.shape)
             for t in template_tensors]
@@ -474,3 +535,38 @@ def _get_output_shapes(true_graph_outputs, false_graph_outputs):
       for t_out, f_out in zip(true_graph_outputs, false_graph_outputs)
   ]
   return output_shapes
+
+
+class _CondGradFuncGraph(util.CondBranchFuncGraph):
+  """FuncGraph for the gradient function of the branch of an If op.
+
+  Handles unwrapping optional intermediate values that are captured by the
+  gradient computation.
+  """
+
+  def __init__(self, name, forward_graph):
+    super(_CondGradFuncGraph, self).__init__(name, read_only_collections=False)
+    self._forward_graph = forward_graph
+
+  def _capture_helper(self, tensor, name):
+    if (tensor.graph is not self._forward_graph or
+        tensor in self._forward_graph.inputs or
+        tensor in self._forward_graph.outputs):
+      return super(_CondGradFuncGraph, self)._capture_helper(tensor, name)
+
+    # 'tensor' is an intermediate in the forward graph. We find the corresonding
+    # optional tensor, which is output from the If op, and capture it as
+    # normal. We then unwrap the captured optional value to get the raw
+    # intermediate value.
+    for consumer in tensor.consumers():
+      if (consumer.type == "OptionalFromValue"
+          and consumer.outputs[0] in self._forward_graph.outputs):
+        optional = consumer.outputs[0]
+        captured_optional = super(_CondGradFuncGraph, self)._capture_helper(
+            optional, name)
+        return gen_dataset_ops.optional_get_value(
+            captured_optional, [tensor.dtype], [tensor.shape])[0]
+    raise ValueError(
+        "Couldn't find OptionalFromValue consumer for tensor '%s'.\n"
+        "This is an internal bug, please report at "
+        "https://github.com/tensorflow/tensorflow/issues." % tensor.name)
diff --git a/tensorflow/python/ops/control_flow_util.py b/tensorflow/python/ops/control_flow_util.py
index 72c074ed1a..cb628f4aa6 100644
--- a/tensorflow/python/ops/control_flow_util.py
+++ b/tensorflow/python/ops/control_flow_util.py
@@ -38,6 +38,11 @@ def IsInXLAContext(op):
   return GetContainingXLAContext(ctxt) is not None
 
 
+def InXlaContext(graph):
+  ctxt = graph._get_control_flow_context()  # pylint: disable=protected-access
+  return GetContainingXLAContext(ctxt) is not None
+
+
 def IsInWhileLoop(op):
   ctxt = op._get_control_flow_context()  # pylint: disable=protected-access
   return GetContainingWhileContext(ctxt) is not None
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index c8f5cb8349..8cc4d926c7 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -49,6 +49,7 @@ from tensorflow.python.ops import logging_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import manip_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import math_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import optional_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import random_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import tensor_array_ops
diff --git a/tensorflow/python/ops/optional_grad.py b/tensorflow/python/ops/optional_grad.py
new file mode 100644
index 0000000000..0d1eae3cda
--- /dev/null
+++ b/tensorflow/python/ops/optional_grad.py
@@ -0,0 +1,33 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Gradient functions for optional ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_dataset_ops
+
+
+@ops.RegisterGradient("OptionalFromValue")
+def _OptionalFromValueGrad(op, grad):
+  return gen_dataset_ops.optional_get_value(
+      grad, [t.dtype for t in op.inputs], [t.shape for t in op.inputs])
+
+
+@ops.RegisterGradient("OptionalGetValue")
+def _OptionalGetValueGrad(unused_op, *grads):
+  return gen_dataset_ops.optional_from_value(grads)
-- 
GitLab


From 5d14dbd1c35783d072756643c905fef5d7ac28fb Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 28 Nov 2018 17:10:02 -0800
Subject: [PATCH 0979/1554] Fix method name

PiperOrigin-RevId: 223263254
---
 tensorflow/contrib/distribute/python/keras_test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 07027bde8a..6a006cc85f 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1263,7 +1263,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       model.add(keras.layers.Dense(1))
       initial_weights = model.get_weights()
 
-      def fit_and_predict(with_distribution=None):
+      def fit_eval_and_predict(with_distribution=None):
         # We have initialized the model to the same weight for the distribution
         # and non-distribution run.
         model.set_weights(initial_weights)
@@ -1283,10 +1283,10 @@ class TestDistributionStrategyCorrectness(test.TestCase,
 
         return weights, eval_result, predict_result
 
-      wts_with_ds, eval_with_ds, predict_with_ds = fit_and_predict(
+      wts_with_ds, eval_with_ds, predict_with_ds = fit_eval_and_predict(
           with_distribution=distribution)
-      wts_without_ds, eval_without_ds, predict_without_ds = fit_and_predict(
-          with_distribution=None)
+      wts_without_ds, eval_without_ds, predict_without_ds = (
+          fit_eval_and_predict(with_distribution=None))
 
       # Verify that the weights, eval results, predict outputs  are the same
       # within some limits of tolerance.
-- 
GitLab


From 0be39e80a210919c1dd3223b4b0ac23380372940 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Thu, 29 Nov 2018 09:15:50 +0800
Subject: [PATCH 0980/1554] Update tensorflow/core/graph/mkl_layout_pass.cc

Use "DCHECK()" instead of "CHECK_EQ".

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index bac434886f..e6c24ab8d4 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -949,7 +949,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
             e->dst_input() == kPermTensorIndex) {
           // we find the "perm" node, now try to retrieve its value.
           const TensorProto* proto = nullptr;
-          CHECK_EQ(GetNodeAttr(perm_node->def(), "value", &proto).ok(), true);
+          DCHECK(GetNodeAttr(perm_node->def(), "value", &proto).ok());
 
           DataType type;
           GetNodeAttr(perm_node->def(), "dtype", &type);
-- 
GitLab


From 41d41c9730e87e2b98ae24a447f233466d7995af Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Thu, 29 Nov 2018 09:16:49 +0800
Subject: [PATCH 0981/1554] Update tensorflow/core/graph/mkl_layout_pass.cc

Use "DCHECK()" instead of "CHECK_EQ".

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index e6c24ab8d4..55c337cbea 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2694,7 +2694,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   // Create node.
   Node* new_node;
   TF_CHECK_OK(nb.Finalize(&**g, &new_node));
-  CHECK_NOTNULL(new_node);
+  DCHECK(new_node);
 
   // Fill outputs.
   for (const Edge* e : transpose_to_nchw->out_edges()) {
-- 
GitLab


From d95a04c30511992befcd66dc12f3227fa65891e4 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com>
Date: Thu, 29 Nov 2018 09:17:09 +0800
Subject: [PATCH 0982/1554] Update tensorflow/core/graph/mkl_layout_pass.cc

Use "DCHECK()" instead of "CHECK_EQ".

Co-Authored-By: wenxizhu <wenxi.zhu@intel.com>
---
 tensorflow/core/graph/mkl_layout_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 55c337cbea..edea296c3b 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2700,7 +2700,7 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   for (const Edge* e : transpose_to_nchw->out_edges()) {
     if (!e->IsControlEdge()) {
       const int kTransposeWithMklOpOutputSlot = 0;
-      CHECK_NOTNULL((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot,
+      DCHECK((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot,
                                   e->dst(), e->dst_input()));
     }
   }
-- 
GitLab


From 7dbc68791c6d5384e5807fefdeee79ccccc4008b Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Wed, 28 Nov 2018 17:17:33 -0800
Subject: [PATCH 0983/1554] Ran newer version of clang-format

---
 tensorflow/core/kernels/mkl_avgpooling_op.cc  | 51 +++++++-----------
 tensorflow/core/kernels/mkl_maxpooling_op.cc  | 52 ++++++++-----------
 .../core/kernels/mkl_pooling_ops_common.cc    | 19 +++----
 .../core/kernels/mkl_pooling_ops_common.h     | 15 +++---
 .../kernels/mkl_quantized_pooling_ops_test.cc | 14 ++---
 5 files changed, 64 insertions(+), 87 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_avgpooling_op.cc b/tensorflow/core/kernels/mkl_avgpooling_op.cc
index a5dc317969..939cbd6f96 100644
--- a/tensorflow/core/kernels/mkl_avgpooling_op.cc
+++ b/tensorflow/core/kernels/mkl_avgpooling_op.cc
@@ -53,19 +53,16 @@ class MklAvgPoolingOp : public OpKernel {
 
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window ksize field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window ksize field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window stride field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window stride field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
-                errors::Unimplemented(
-                    "Pooling is not yet supported on the "
-                    "batch dimension."));
+                errors::Unimplemented("Pooling is not yet supported on the "
+                                      "batch dimension."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -231,19 +228,16 @@ class MklAvgPoolingGradOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window ksize field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window ksize field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window strides field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
-                errors::Unimplemented(
-                    "Pooling is not yet supported on the "
-                    "batch dimension."));
+                errors::Unimplemented("Pooling is not yet supported on the "
+                                      "batch dimension."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -365,9 +359,8 @@ class MklAvgPoolingGradOp : public OpKernel {
         // elements.
         OP_REQUIRES(context, tensor_in_shape.dims() == 1 &&
                                  tensor_in_shape.NumElements() == 4,
-                    errors::InvalidArgument(
-                        "original input shape must be "
-                        "1-dimensional and 4 elements"));
+                    errors::InvalidArgument("original input shape must be "
+                                            "1-dimensional and 4 elements"));
 
         // For avgpooling, out_backprop should have 4 dimensions.
         OP_REQUIRES(
@@ -557,8 +550,7 @@ class MklAvgPoolingOp : public MklPoolingForwardOpBase<T> {
         output_min->flat<float>()(0) = min_input;
         output_max->flat<float>()(0) = max_input;
       }
-    }
-    catch (mkldnn::error& e) {
+    } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
                          string(e.message) + ", in file " + string(__FILE__) +
                          ":" + std::to_string(__LINE__);
@@ -660,8 +652,7 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase<T> {
 
       // execute pooling op
       pooling_bwd->Execute(diff_dst_data, diff_src_data);
-    }
-    catch (mkldnn::error& e) {
+    } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
                          string(e.message) + ", in file " + string(__FILE__) +
                          ":" + std::to_string(__LINE__);
@@ -685,15 +676,13 @@ class MklAvgPoolingGradOp : public MklPoolingBackwardOpBase<T> {
     if (!original_input_mkl_shape.IsMklTensor()) {
       OP_REQUIRES(context, tensor_in_shape.dims() == 1 &&
                                tensor_in_shape.NumElements() == 4,
-                  errors::InvalidArgument(
-                      "original input shape must be "
-                      "1-dimensional and 4 elements"));
+                  errors::InvalidArgument("original input shape must be "
+                                          "1-dimensional and 4 elements"));
     } else {
       OP_REQUIRES(context, original_input_mkl_shape.GetDimension() == 1 &&
                                original_input_mkl_shape.DimSize(0) == 4,
-                  errors::InvalidArgument(
-                      "original input shape must be "
-                      "1-dimensional and 4 elements"));
+                  errors::InvalidArgument("original input shape must be "
+                                          "1-dimensional and 4 elements"));
     }
 
     if (!input_gradient_mkl_shape.IsMklTensor()) {
diff --git a/tensorflow/core/kernels/mkl_maxpooling_op.cc b/tensorflow/core/kernels/mkl_maxpooling_op.cc
index cd72530bce..0697251c7d 100644
--- a/tensorflow/core/kernels/mkl_maxpooling_op.cc
+++ b/tensorflow/core/kernels/mkl_maxpooling_op.cc
@@ -54,19 +54,16 @@ class MklMaxPoolingOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window ksize field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window ksize field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window stride field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window stride field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
-                errors::Unimplemented(
-                    "Pooling is not yet supported on the "
-                    "batch dimension."));
+                errors::Unimplemented("Pooling is not yet supported on the "
+                                      "batch dimension."));
 
     workspace_enabled_ = false;
     // We may not get this attribute for this node if it does not go through
@@ -216,14 +213,12 @@ class MklMaxPoolingGradOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
     OP_REQUIRES(context, ksize_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window ksize field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window ksize field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
     OP_REQUIRES(context, stride_.size() == 4,
-                errors::InvalidArgument(
-                    "Sliding window strides field must "
-                    "specify 4 dimensions"));
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
     OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
                 errors::Unimplemented(
@@ -641,8 +636,7 @@ class MklMaxPoolingOp : public MklPoolingForwardOpBase<T> {
         // execute pooling op
         pooling_fwd->Execute(src_data, dst_data, ws_data);
       }
-    }
-    catch (mkldnn::error& e) {
+    } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
                          string(e.message) + ", in file " + string(__FILE__) +
                          ":" + std::to_string(__LINE__);
@@ -772,10 +766,11 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
       if (ws_md.data.format != pooling_bwd->GetWorkspaceFormat()) {
         memory::dims ws_dims;
         ws_dims.assign(ws_md.data.dims, ws_md.data.dims + ws_md.data.ndims);
-        auto target_ws = memory::primitive_desc(
-            {{ws_dims}, pooling_bwd->GetWorkspaceDataType(),
-             pooling_bwd->GetWorkspaceFormat()},
-            cpu_engine);
+        auto target_ws =
+            memory::primitive_desc({{ws_dims},
+                                    pooling_bwd->GetWorkspaceDataType(),
+                                    pooling_bwd->GetWorkspaceFormat()},
+                                   cpu_engine);
         workspace_dnn_data.SetUsrMem(ws_md, &workspace_tensor);
         workspace_dnn_data.CheckReorderToOpMem(target_ws);
         ws_data = workspace_dnn_data.GetOpMem().get_data_handle();
@@ -785,8 +780,7 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
 
       // execute pooling
       pooling_bwd->Execute(diff_dst_data, diff_src_data, ws_data);
-    }
-    catch (mkldnn::error& e) {
+    } catch (mkldnn::error& e) {
       string error_msg = "Status:" + std::to_string(e.status) + ", message: " +
                          string(e.message) + ". in file " + string(__FILE__) +
                          ":" + std::to_string(__LINE__);
@@ -857,12 +851,12 @@ class MklMaxPoolingGradOp : public MklPoolingBackwardOpBase<T> {
           context, workspace_tensor.dims() == 1,
           errors::InvalidArgument("Workspace tensor must be 1-dimensional"));
     } else {
-      OP_REQUIRES(context, this->workspace_enabled_,
-                  errors::Unimplemented(
-                      "MKL-DNN Max Pooling does not "
-                      "yet support the use case "
-                      "where MaxPoolGrad is called without first"
-                      " calling MaxPool."));
+      OP_REQUIRES(
+          context, this->workspace_enabled_,
+          errors::Unimplemented("MKL-DNN Max Pooling does not "
+                                "yet support the use case "
+                                "where MaxPoolGrad is called without first"
+                                " calling MaxPool."));
     }
   }
 };  // MklMaxPoolingGradOp
diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
index cf3f0f1469..dc84d3941e 100644
--- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc
+++ b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
@@ -380,24 +380,21 @@ void MklPoolParameters::Init(OpKernelContext* context,
     // any padding, and expects the depth_window to equal the depth
     // stride (no overlapping).
     OP_REQUIRES(context, depth % depth_window == 0,
-                errors::Unimplemented(
-                    "Depthwise max pooling requires the"
-                    " depth window to evenly divide the"
-                    " input depth"));
+                errors::Unimplemented("Depthwise max pooling requires the"
+                                      " depth window to evenly divide the"
+                                      " input depth"));
     OP_REQUIRES(context, depth_stride == depth_window,
-                errors::Unimplemented(
-                    "Depthwise max pooling requires the"
-                    " depth window to equal the depth"
-                    " stride"));
+                errors::Unimplemented("Depthwise max pooling requires the"
+                                      " depth window to equal the depth"
+                                      " stride"));
 
     // The current version of depthwise max is only implemented on CPU.
     OP_REQUIRES(context,
                 (DeviceType(static_cast<Device*>(context->device())
                                 ->attributes()
                                 .device_type()) == DeviceType(DEVICE_CPU)),
-                errors::Unimplemented(
-                    "Depthwise max pooling is currently "
-                    "only implemented for CPU devices."));
+                errors::Unimplemented("Depthwise max pooling is currently "
+                                      "only implemented for CPU devices."));
 
     out_depth = depth / depth_window;
   }
diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.h b/tensorflow/core/kernels/mkl_pooling_ops_common.h
index d214c39484..8a60c3be91 100644
--- a/tensorflow/core/kernels/mkl_pooling_ops_common.h
+++ b/tensorflow/core/kernels/mkl_pooling_ops_common.h
@@ -459,19 +459,16 @@ class MklPoolingOpBase : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
     OP_REQUIRES_OK(context, context->GetAttr("ksize", &this->ksize_));
     OP_REQUIRES(context, this->ksize_.size() == 4 || this->ksize_.size() == 5,
-                errors::InvalidArgument(
-                    "Sliding window ksize field must "
-                    "specify 4 or 5 dimensions"));
+                errors::InvalidArgument("Sliding window ksize field must "
+                                        "specify 4 or 5 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &this->stride_));
     OP_REQUIRES(context, this->stride_.size() == 4 || this->stride_.size() == 5,
-                errors::InvalidArgument(
-                    "Sliding window strides field must "
-                    "specify 4 or 5 dimensions"));
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 or 5 dimensions"));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &this->padding_));
     OP_REQUIRES(context, this->ksize_[0] == 1 && this->stride_[0] == 1,
-                errors::Unimplemented(
-                    "Pooling is not yet supported on the "
-                    "batch dimension."));
+                errors::Unimplemented("Pooling is not yet supported on the "
+                                      "batch dimension."));
     bool is_pool2d = (this->ksize_.size() == 4);
     this->data_format_mkldnn_ =
         is_pool2d ? TFDataFormatToMklDnnDataFormat(this->data_format_tf_)
diff --git a/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc b/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc
index 92fc1bfd07..7c1e32d6e3 100644
--- a/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc
+++ b/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc
@@ -56,7 +56,7 @@ class ConvMklToTF : public OpsTestBase {
 
     output = *GetOutput(0);
   }
-  void TestBody() {};
+  void TestBody(){};
 };
 
 class QuantizedPoolingTest : public OpsTestBase {};
@@ -95,9 +95,9 @@ TEST_F(QuantizedPoolingTest, SmallAveragePooling) {
   const int expected_height = input_height / stride;
 
   // The input pools we are averaging. (NHWC input, quantized.)
-  //   0th channel       1st channel
-  //   1  3 |  5  7      2  4 |  6  8
-  //   9 11 | 13 15     10 12 | 14 16
+  //    0th channel       1st channel
+  //    1  3 |  5  7      2  4 |  6  8
+  //    9 11 | 13 15     10 12 | 14 16
   //   -------------     -------------
   //   17 19 | 21 23     18 20 | 22 24
   //   25 27 | 29 31     26 28 | 30 32
@@ -163,9 +163,9 @@ TEST_F(QuantizedPoolingTest, SmallMaxPooling) {
   const int expected_height = input_height / stride;
 
   // The max is computed from these input pools. (NHWC input, quantized.)
-  //   0th channel       1st channel
-  //   1  3 |  5  7      2  4 |  6  8
-  //   9 11 | 13 15     10 12 | 14 16
+  //    0th channel       1st channel
+  //    1  3 |  5  7      2  4 |  6  8
+  //    9 11 | 13 15     10 12 | 14 16
   //   -------------     -------------
   //   17 19 | 21 23     18 20 | 22 24
   //   25 27 | 29 31     26 28 | 30 32
-- 
GitLab


From 4a68eff3dd3141c75c7cd5e152a1555ef13ae739 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Wed, 28 Nov 2018 17:10:07 -0800
Subject: [PATCH 0984/1554] [XLA] Add methods to mutate the precision
 configuration of an HLO

PiperOrigin-RevId: 223263273
---
 tensorflow/compiler/xla/service/hlo_instruction.cc | 10 ++++++++++
 tensorflow/compiler/xla/service/hlo_instruction.h  |  1 +
 tensorflow/compiler/xla/service/hlo_instructions.h |  2 ++
 3 files changed, 13 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 2bdb617731..36a7a5029b 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -3014,6 +3014,16 @@ const PrecisionConfig& HloInstruction::precision_config() const {
   LOG(FATAL) << "Unimplemented method.";
 }
 
+PrecisionConfig* HloInstruction::mutable_precision_config() {
+  if (auto* convolution = DynCast<HloConvolutionInstruction>(this)) {
+    return convolution->mutable_precision_config();
+  }
+  if (auto* dot = DynCast<HloDotInstruction>(this)) {
+    return dot->mutable_precision_config();
+  }
+  LOG(FATAL) << "Unimplemented method.";
+}
+
 HloModule* HloInstruction::GetModule() const {
   if (parent_) {
     return parent_->parent();
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 87748a771a..8569e86b83 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -1266,6 +1266,7 @@ class HloInstruction {
   // superior.
   // Precondition: opcode must be kConvolution or kDot.
   const PrecisionConfig& precision_config() const;
+  PrecisionConfig* mutable_precision_config();
 
   // Sets the debug metadata for this instruction.
   void set_metadata(const OpMetadata& metadata) { metadata_ = metadata; }
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index 0b07341cb9..e35c6e9150 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -955,6 +955,7 @@ class HloConvolutionInstruction : public HloInstruction {
   // information but it is presumed that the alternate lowering is strictly
   // superior.
   const PrecisionConfig& precision_config() const { return precision_config_; }
+  PrecisionConfig* mutable_precision_config() { return &precision_config_; }
 
   string ToCategory() const override;
   // Returns a serialized representation of this instruction.
@@ -1326,6 +1327,7 @@ class HloDotInstruction : public HloInstruction {
   // information but it is presumed that the alternate lowering is strictly
   // superior.
   const PrecisionConfig& precision_config() const { return precision_config_; }
+  PrecisionConfig* mutable_precision_config() { return &precision_config_; }
 
   // Returns a serialized representation of this instruction.
   HloInstructionProto ToProto() const override;
-- 
GitLab


From 4ed87f979bb217155092b086450cf50d5363f4e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 17:14:53 -0800
Subject: [PATCH 0985/1554] Automated rollback of commit
 0c1eb8861624d6d17c797b70d25330711df5eb2f

PiperOrigin-RevId: 223263979
---
 .../core/common_runtime/function_test.cc      |  4 +-
 tensorflow/core/graph/edgeset.cc              |  2 +-
 tensorflow/core/graph/edgeset.h               | 20 ++++------
 tensorflow/core/graph/graph_test.cc           | 39 -------------------
 tensorflow/core/graph/optimizer_cse_test.cc   | 10 ++---
 5 files changed, 13 insertions(+), 62 deletions(-)

diff --git a/tensorflow/core/common_runtime/function_test.cc b/tensorflow/core/common_runtime/function_test.cc
index 3b4c976685..cab95cb596 100644
--- a/tensorflow/core/common_runtime/function_test.cc
+++ b/tensorflow/core/common_runtime/function_test.cc
@@ -1433,9 +1433,7 @@ TEST_F(FunctionLibraryRuntimeTest, Gradient_AddSum) {
 
     GraphDef actual;
     g->ToGraphDef(&actual);
-    // The optimizer is non-deterministic, so we only check that the number of
-    // nodes is not greater than expected.
-    EXPECT_LE(actual.node_size(), expected.node_size());
+    TF_EXPECT_GRAPH_EQ(expected, actual);
   }
 }
 
diff --git a/tensorflow/core/graph/edgeset.cc b/tensorflow/core/graph/edgeset.cc
index 02315a3e27..2e0c671461 100644
--- a/tensorflow/core/graph/edgeset.cc
+++ b/tensorflow/core/graph/edgeset.cc
@@ -37,7 +37,7 @@ std::pair<EdgeSet::const_iterator, bool> EdgeSet::insert(value_type value) {
       }
     }
     // array is full. convert to set.
-    s = new gtl::FlatSet<const Edge*>;
+    s = new std::set<const Edge*>;
     for (int i = 0; i < kInline; i++) {
       s->insert(static_cast<const Edge*>(ptrs_[i]));
     }
diff --git a/tensorflow/core/graph/edgeset.h b/tensorflow/core/graph/edgeset.h
index 2776c8491c..0a1ee5a666 100644
--- a/tensorflow/core/graph/edgeset.h
+++ b/tensorflow/core/graph/edgeset.h
@@ -17,18 +17,17 @@ limitations under the License.
 #define TENSORFLOW_GRAPH_EDGESET_H_
 
 #include <stddef.h>
-
-#include "tensorflow/core/lib/gtl/flatset.h"
-#include "tensorflow/core/platform/logging.h"
+#include <set>
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
+
+#include "tensorflow/core/platform/logging.h"
 namespace tensorflow {
 
 class Edge;
 
 // An unordered set of edges.  Uses very little memory for small sets.
-// Unlike gtl::FlatSet, EdgeSet does NOT allow mutations during
-// iteration.
+// Unlike std::set, EdgeSet does NOT allow mutations during iteration.
 class EdgeSet {
  public:
   EdgeSet();
@@ -55,15 +54,12 @@ class EdgeSet {
  private:
   // Up to kInline elements are stored directly in ptrs_ (nullptr means none).
   // If ptrs_[0] == this then ptrs_[1] points to a set<const Edge*>.
-  // kInline must be >= 2, and is chosen such that ptrs_ fills a 64 byte
-  // cacheline.
-  static constexpr int kInline = 64 / sizeof(const void*);
+  static const int kInline = 4;  // Must be >= 2.
   const void* ptrs_[kInline];
 
-  gtl::FlatSet<const Edge*>* get_set() const {
+  std::set<const Edge*>* get_set() const {
     if (ptrs_[0] == this) {
-      return static_cast<gtl::FlatSet<const Edge*>*>(
-          const_cast<void*>(ptrs_[1]));
+      return static_cast<std::set<const Edge*>*>(const_cast<void*>(ptrs_[1]));
     } else {
       return nullptr;
     }
@@ -103,7 +99,7 @@ class EdgeSet::const_iterator {
   friend class EdgeSet;
 
   void const* const* array_iter_ = nullptr;
-  typename gtl::FlatSet<const Edge*>::const_iterator tree_iter_;
+  typename std::set<const Edge*>::const_iterator tree_iter_;
 
 #ifdef NDEBUG
   inline void Init(const EdgeSet* e) {}
diff --git a/tensorflow/core/graph/graph_test.cc b/tensorflow/core/graph/graph_test.cc
index e7762fd414..333c32567f 100644
--- a/tensorflow/core/graph/graph_test.cc
+++ b/tensorflow/core/graph/graph_test.cc
@@ -799,44 +799,5 @@ BENCHMARK(BM_GraphCreation)->ArgPair(1 << 9, 16);
 BENCHMARK(BM_GraphCreation)->ArgPair(1 << 12, 16);
 BENCHMARK(BM_GraphCreation)->ArgPair(1 << 15, 16);
 
-static void BM_ToGraphDef(int iters, int num_nodes, int num_edges_per_node) {
-  testing::StopTiming();
-  const GraphDef graph_def = CreateGraphDef(num_nodes, num_edges_per_node);
-  const auto registry = OpRegistry::Global();
-  GraphConstructorOptions opts;
-  // Warmup step.
-  Graph graph(registry);
-  TF_CHECK_OK(ConvertGraphDefToGraph(opts, graph_def, &graph));
-  int64 sum = 0;
-  testing::StartTiming();
-  for (int i = 0; i < iters; ++i) {
-    GraphDef graph_def;
-    graph.ToGraphDef(&graph_def);
-    sum += graph_def.node_size();
-  }
-  VLOG(1) << sum;
-  testing::StopTiming();
-}
-BENCHMARK(BM_ToGraphDef)->ArgPair(10, 2);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 6, 2);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 9, 2);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 12, 2);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 15, 2);
-BENCHMARK(BM_ToGraphDef)->ArgPair(10, 4);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 6, 4);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 9, 4);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 12, 4);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 15, 4);
-BENCHMARK(BM_ToGraphDef)->ArgPair(10, 8);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 6, 8);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 9, 8);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 12, 8);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 15, 8);
-BENCHMARK(BM_ToGraphDef)->ArgPair(10, 16);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 6, 16);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 9, 16);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 12, 16);
-BENCHMARK(BM_ToGraphDef)->ArgPair(1 << 15, 16);
-
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/optimizer_cse_test.cc b/tensorflow/core/graph/optimizer_cse_test.cc
index 642298fa95..c1f93ce05a 100644
--- a/tensorflow/core/graph/optimizer_cse_test.cc
+++ b/tensorflow/core/graph/optimizer_cse_test.cc
@@ -337,13 +337,9 @@ TEST_F(OptimizerCSETest, Constant_Dedup) {
   EXPECT_EQ(OriginalGraph(),
             "n/_0(Const);n/_1(Const);n/_2(Const);n/_3(Const);"
             "n/_4(Const);n/_5(Const);n/_6(Const);n/_7(Const)|");
-  std::vector<string> nodes = str_util::Split(DoCSE(), ";|");
-  std::set<string> node_set(nodes.begin(), nodes.end());
-  // Expect exactly one of each type of node to be retained after CSE.
-  EXPECT_EQ(node_set.count("n/_0(Const)") + node_set.count("n/_7(Const)"), 1);
-  EXPECT_EQ(node_set.count("n/_1(Const)") + node_set.count("n/_6(Const)"), 1);
-  EXPECT_EQ(node_set.count("n/_2(Const)") + node_set.count("n/_5(Const)"), 1);
-  EXPECT_EQ(node_set.count("n/_3(Const)") + node_set.count("n/_4(Const)"), 1);
+  // In theory, there are 2^4 possible correct output of CSE.  In this
+  // test, it happens to eliminate the last 4 nodes.
+  EXPECT_EQ(DoCSE(), "n/_0(Const);n/_1(Const);n/_2(Const);n/_3(Const)|");
 }
 
 static void BM_CSE(int iters, int op_nodes) {
-- 
GitLab


From d9540902ee49425262fc96146b981b005c77ee6c Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 28 Nov 2018 17:29:49 -0800
Subject: [PATCH 0986/1554] Filter gpu tags from Windows CPU.

PiperOrigin-RevId: 223265780
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 177ef390db..872072a056 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -126,8 +126,8 @@ N_JOBS="${NUMBER_OF_PROCESSORS}"
 # which will result testing system installed tensorflow
 bazel test --announce_rc --config=opt -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
-  --test_tag_filters=-no_pip,-no_windows,-no_oss \
-  --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
+  --test_tag_filters=-no_pip,-no_windows,-no_oss,-gpu \
+  --build_tag_filters=-no_pip,-no_windows,-no_oss,-gpu --build_tests_only \
   --test_size_filters=small,medium \
   --jobs="${N_JOBS}" --test_timeout="300,450,1200,3600" \
   --flaky_test_attempts=3 \
-- 
GitLab


From b97727bc3c7a9216670f361b639a60ed516917e0 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Wed, 28 Nov 2018 17:32:18 -0800
Subject: [PATCH 0987/1554] Use tf_stack instead of tf_inspect to get file and
 lineno in deprecation.py

This is a no-op change, that prevents deprecation warnings triggering large numbers of posix.stat calls.

In a test application, this prevents 64150 calls to lstat and 1499 calls to stat.

PiperOrigin-RevId: 223266120
---
 tensorflow/python/util/deprecation.py | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/util/deprecation.py b/tensorflow/python/util/deprecation.py
index 4c68d1aaae..9aaf0c2de9 100644
--- a/tensorflow/python/util/deprecation.py
+++ b/tensorflow/python/util/deprecation.py
@@ -28,6 +28,7 @@ from tensorflow.python.util import is_in_graph_mode
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
+from tensorflow.python.util import tf_stack
 
 
 # Allow deprecation warnings to be silenced temporarily with a context manager.
@@ -98,21 +99,9 @@ def _validate_deprecation_args(date, instructions):
 
 def _call_location(outer=False):
   """Returns call location given level up from current call."""
-  frame = tf_inspect.currentframe()
-  if frame:
-    # CPython internals are available, use them for performance.
-    # walk back two frames to get to deprecated function caller.
-    frame = frame.f_back
-    if frame.f_back:
-      frame = frame.f_back
-    if outer and frame.f_back:
-      frame = frame.f_back
-    return '%s:%d' % (frame.f_code.co_filename, frame.f_lineno)
-  else:
-    # Slow fallback path
-    stack = tf_inspect.stack(0)  # 0 avoids generating unused context
-    entry = stack[3 if outer else 2]
-    return '%s:%d' % (entry[1], entry[2])
+  stack = tf_stack.extract_stack()
+  frame = stack[-4 if outer else -3]
+  return '{filename}:{lineno}'.format(filename=frame[0], lineno=frame[1])
 
 
 def _wrap_decorator(wrapped_function):
-- 
GitLab


From da2c2faca13ca71ebf5c3528b7a0e78321e5c544 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Wed, 28 Nov 2018 17:37:41 -0800
Subject: [PATCH 0988/1554] Enable testCondAndTensorArray and
 testCondAndTensorArrayInDefun.

PiperOrigin-RevId: 223266910
---
 tensorflow/python/kernel_tests/cond_v2_test.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 09e41ec8c6..4baa8b0212 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -719,10 +719,6 @@ class CondV2Test(test.TestCase):
 
   @test_util.enable_control_flow_v2
   def testCondAndTensorArray(self):
-    if test_util.is_gpu_available():
-      old_enable_tensor_array_v2 = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2
-      # TODO(b/119689663): Enable this.
-      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = False
     x = math_ops.range(-5, 5)
     output = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=x.shape[0])
 
@@ -744,15 +740,9 @@ class CondV2Test(test.TestCase):
     output_t = output.stack()
     self.assertAllEqual(
         self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
-    if test_util.is_gpu_available():
-      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = old_enable_tensor_array_v2
 
   @test_util.enable_control_flow_v2
   def testCondAndTensorArrayInDefun(self):
-    if test_util.is_gpu_available():
-      old_enable_tensor_array_v2 = tensor_array_ops.ENABLE_TENSOR_ARRAY_V2
-      # TODO(b/119689663): Enable this.
-      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = False
 
     @function.defun
     def f():
@@ -780,9 +770,6 @@ class CondV2Test(test.TestCase):
     self.assertAllEqual(
         self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
 
-    if test_util.is_gpu_available():
-      tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 = old_enable_tensor_array_v2
-
 
 class CondV2CollectionTest(test.TestCase):
 
-- 
GitLab


From 99c20bf32e29a90dbb31f480360eb8881f0bd411 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 17:48:11 -0800
Subject: [PATCH 0989/1554] Register DataFormatVecPermute with label "host" on
 CPU to work around crash due to missing kernel.

PiperOrigin-RevId: 223268202
---
 tensorflow/core/kernels/data_format_ops.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/core/kernels/data_format_ops.cc b/tensorflow/core/kernels/data_format_ops.cc
index 23319e6d0c..27020cdabd 100644
--- a/tensorflow/core/kernels/data_format_ops.cc
+++ b/tensorflow/core/kernels/data_format_ops.cc
@@ -156,6 +156,16 @@ TF_CALL_int32(REGISTER_KERNEL);
 TF_CALL_int64(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
 
+#define REGISTER_KERNEL(T)                             \
+  REGISTER_KERNEL_BUILDER(Name("DataFormatVecPermute") \
+                              .Device(DEVICE_CPU)      \
+                              .Label("host")           \
+                              .TypeConstraint<T>("T"), \
+                          DataFormatVecPermuteOp<CPUDevice, T>);
+TF_CALL_int32(REGISTER_KERNEL);
+TF_CALL_int64(REGISTER_KERNEL);
+#undef REGISTER_KERNEL
+
 #if GOOGLE_CUDA
 // Forward declarations of the functor specializations for GPU.
 namespace functor {
-- 
GitLab


From 693c1ad4a94aec4dc2783096bfc204774927709e Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 28 Nov 2018 17:55:17 -0800
Subject: [PATCH 0990/1554] Save main_op and train_op in SignatureDefs instead
 of collections.

PiperOrigin-RevId: 223268983
---
 tensorflow/cc/saved_model/BUILD               |   1 +
 tensorflow/cc/saved_model/constants.h         |   9 +-
 tensorflow/cc/saved_model/loader.cc           |  75 ++--
 tensorflow/cc/saved_model/loader_test.cc      |  14 +
 .../half_plus_two_v2/00000123/assets/foo.txt  |   1 +
 .../half_plus_two_v2/00000123/saved_model.pb  | Bin 0 -> 10774 bytes
 .../variables/variables.data-00000-of-00001   | Bin 0 -> 12 bytes
 .../00000123/variables/variables.index        | Bin 0 -> 151 bytes
 .../python/saved_model/keras_saved_model.py   |   8 +-
 .../saved_model/keras_saved_model_test.py     |  21 +-
 tensorflow/python/saved_model/BUILD           |   2 +
 tensorflow/python/saved_model/builder_impl.py | 206 ++++++-----
 tensorflow/python/saved_model/constants.py    |   6 +-
 tensorflow/python/saved_model/loader_impl.py  |  54 ++-
 tensorflow/python/saved_model/loader_test.py  | 109 ++++--
 .../python/saved_model/saved_model_test.py    | 340 +++++++++---------
 .../python/saved_model/signature_def_utils.py |   2 +
 .../saved_model/signature_def_utils_impl.py   |  51 ++-
 .../saved_model/signature_def_utils_test.py   |  18 +
 tensorflow/python/saved_model/utils_impl.py   |  21 ++
 .../golden/v2/tensorflow.saved_model.pbtxt    |   4 -
 21 files changed, 580 insertions(+), 362 deletions(-)
 create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt
 create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/saved_model.pb
 create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.data-00000-of-00001
 create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.index

diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 3d3895c8fa..52345a376c 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -133,5 +133,6 @@ filegroup(
         "testdata/half_plus_two_pbtxt/**",
         "testdata/half_plus_two_main_op/**",
         "testdata/half_plus_two/**",
+        "testdata/half_plus_two_v2/**",
     ]),
 )
diff --git a/tensorflow/cc/saved_model/constants.h b/tensorflow/cc/saved_model/constants.h
index 645a3f101d..6f00dc324b 100644
--- a/tensorflow/cc/saved_model/constants.h
+++ b/tensorflow/cc/saved_model/constants.h
@@ -33,10 +33,10 @@ constexpr char kSavedModelFilenamePb[] = "saved_model.pb";
 /// SavedModel text format proto filename.
 constexpr char kSavedModelFilenamePbTxt[] = "saved_model.pbtxt";
 
-/// SavedModel legacy init op key.
+/// SavedModel legacy init op collection key. Used in v1 SavedModels.
 constexpr char kSavedModelLegacyInitOpKey[] = "legacy_init_op";
 
-/// SavedModel main op key.
+/// SavedModel main op collection key. Used in v1 SavedModels.
 constexpr char kSavedModelMainOpKey[] = "saved_model_main_op";
 
 /// Directory in which to save the SavedModel variables.
@@ -45,6 +45,11 @@ constexpr char kSavedModelVariablesDirectory[] = "variables";
 /// SavedModel variables filename.
 constexpr char kSavedModelVariablesFilename[] = "variables";
 
+/// SavedModel SignatureDef keys for the initialization and train ops. Used in
+/// V2 SavedModels.
+constexpr char kSavedModelInitOpSignatureKey[] = "__saved_model_init_op";
+constexpr char kSavedModelTrainOpSignatureKey[] = "__saved_model_train_op";
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CC_SAVED_MODEL_CONSTANTS_H_
diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index ec116f68cf..85d3dd01fa 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -122,34 +122,54 @@ Status RunOnce(const RunOptions& run_options,
   return run_status;
 }
 
-bool HasMainOp(const MetaGraphDef& meta_graph_def) {
+// RunInitOp will return OK if the initialization op was run successfully.
+// An empty init_op_name indicates that there are no init ops to run.
+Status RunInitOp(const RunOptions& run_options, const string& export_dir,
+                 const MetaGraphDef& meta_graph_def,
+                 const std::vector<AssetFileDef>& asset_file_defs,
+                 Session* session, const string& init_op_name) {
+  if (!init_op_name.empty()) {
+    LOG(INFO) << "Running initialization op on SavedModel bundle.";
+    std::vector<std::pair<string, Tensor>> inputs;
+    AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);
+    RunMetadata run_metadata;
+    return RunOnce(run_options, inputs, {}, {init_op_name},
+                   nullptr /* outputs */, &run_metadata, session);
+  }
+  return Status::OK();
+}
+
+// A SavedModel may store the name of the initialization op to run in the
+// in the SignatureDef (v2) or a collection (v1). If an init_op collection
+// exists, then the collection must contain exactly one op.
+Status GetInitOp(const string& export_dir, const MetaGraphDef& meta_graph_def,
+                 string* init_op_name) {
+  const auto& sig_def_map = meta_graph_def.signature_def();
+  const auto& init_op_sig_it =
+      meta_graph_def.signature_def().find(kSavedModelInitOpSignatureKey);
+  if (init_op_sig_it != sig_def_map.end()) {
+    *init_op_name = init_op_sig_it->second.outputs()
+                        .find(kSavedModelInitOpSignatureKey)
+                        ->second.name();
+    return Status::OK();
+  }
+
   const auto& collection_def_map = meta_graph_def.collection_def();
+  string init_op_collection_key;
   if (collection_def_map.find(kSavedModelMainOpKey) !=
       collection_def_map.end()) {
-    return true;
+    init_op_collection_key = kSavedModelMainOpKey;
+  } else {
+    init_op_collection_key = kSavedModelLegacyInitOpKey;
   }
-  return false;
-}
 
-Status RunMainOp(const RunOptions& run_options, const string& export_dir,
-                 const MetaGraphDef& meta_graph_def,
-                 const std::vector<AssetFileDef>& asset_file_defs,
-                 Session* session, const string& main_op_key) {
-  LOG(INFO) << "Running MainOp with key " << main_op_key
-            << " on SavedModel bundle.";
-  const auto& collection_def_map = meta_graph_def.collection_def();
-  const auto main_op_it = collection_def_map.find(main_op_key);
-  if (main_op_it != collection_def_map.end()) {
-    if (main_op_it->second.node_list().value_size() != 1) {
+  const auto init_op_it = collection_def_map.find(init_op_collection_key);
+  if (init_op_it != collection_def_map.end()) {
+    if (init_op_it->second.node_list().value_size() != 1) {
       return errors::FailedPrecondition(
           strings::StrCat("Expected exactly one main op in : ", export_dir));
     }
-    std::vector<std::pair<string, Tensor>> inputs;
-    AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);
-    RunMetadata run_metadata;
-    const StringPiece main_op_name = main_op_it->second.node_list().value(0);
-    return RunOnce(run_options, inputs, {}, {string(main_op_name)},
-                   nullptr /* outputs */, &run_metadata, session);
+    *init_op_name = init_op_it->second.node_list().value(0);
   }
   return Status::OK();
 }
@@ -236,15 +256,12 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
                  bundle->meta_graph_def.saver_def().restore_op_name(),
                  bundle->meta_graph_def.saver_def().filename_tensor_name(),
                  asset_file_defs, bundle->session.get()));
-  if (HasMainOp(bundle->meta_graph_def)) {
-    TF_RETURN_IF_ERROR(RunMainOp(run_options, export_dir,
-                                 bundle->meta_graph_def, asset_file_defs,
-                                 bundle->session.get(), kSavedModelMainOpKey));
-  } else {
-    TF_RETURN_IF_ERROR(RunMainOp(
-        run_options, export_dir, bundle->meta_graph_def, asset_file_defs,
-        bundle->session.get(), kSavedModelLegacyInitOpKey));
-  }
+  string init_op_name;
+  TF_RETURN_IF_ERROR(
+      GetInitOp(export_dir, bundle->meta_graph_def, &init_op_name));
+  TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, bundle->meta_graph_def,
+                               asset_file_defs, bundle->session.get(),
+                               init_op_name));
   return Status::OK();
 }
 
diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc
index 72b8bc1871..597e42bb65 100644
--- a/tensorflow/cc/saved_model/loader_test.cc
+++ b/tensorflow/cc/saved_model/loader_test.cc
@@ -36,6 +36,8 @@ constexpr char kTestDataMainOp[] =
     "cc/saved_model/testdata/half_plus_two_main_op/00000123";
 constexpr char kTestDataSharded[] =
     "cc/saved_model/testdata/half_plus_two/00000123";
+constexpr char kTestDataInitOpV2[] =
+    "cc/saved_model/testdata/half_plus_two_v2/00000123";
 
 class LoaderTest : public ::testing::Test {
  protected:
@@ -227,5 +229,17 @@ TEST_F(LoaderTest, MaybeSavedModelDirectory) {
   EXPECT_FALSE(MaybeSavedModelDirectory(invalid_export_dir));
 }
 
+TEST_F(LoaderTest, SavedModelInitOpV2Format) {
+  SavedModelBundle bundle;
+  SessionOptions session_options;
+  RunOptions run_options;
+
+  const string export_dir =
+      io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataInitOpV2);
+  TF_ASSERT_OK(LoadSavedModel(session_options, run_options, export_dir,
+                              {kSavedModelTagServe}, &bundle));
+  CheckSavedModelBundle(export_dir, bundle);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt
new file mode 100644
index 0000000000..f9ff036688
--- /dev/null
+++ b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/assets/foo.txt
@@ -0,0 +1 @@
+asset-file-contents
\ No newline at end of file
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/saved_model.pb b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/saved_model.pb
new file mode 100644
index 0000000000000000000000000000000000000000..a10bbf8fb6bca0fcee6414b2927d2f706de85ebc
GIT binary patch
literal 10774
zcmd;J6q*>qHG@~^Cl8ku7qeqZiVzzYV}%l92#Bs!VhoXD<6^8*VhmA|=3)#HVkxOC
zNVO8+;xppm6J}&$VPj?H<Pc)#s^DUCEH2JW&lBS2VlGNeQ(_EhU}O~H;$kgJ%qdL;
znIpu-m0wy?P+AfXmsjG1o2ewl#aEV?lbMoOk{Vx}kywx_#FCVspCiSj!Kft8#a&vQ
z8lRJ&oSm7Mj*yta$jD{E#p;}&S6m{+&&38YUx_sZ<Vq!esO3UzC8>GE`9(^+To4(E
zFSzu$I6PBQ^GY&HDnSm-%qu7@0XdW#ZV+P#k_)*~xrBUEi_%lWjGQx4ld}u*GxJJ{
zg~Yjpli}R>f}+&4%!<@v33eedE}@jv;*!j~#FEVXJgAHWyOJ~)e@bdjYDsE*eojh!
zN@h_p!b2U5j9j){%)X^Lpx`GV8U(@dFoBVgi;auLFW<j_%a@BKATc>xh>wd690kQn
zj3HW#eo`pD<KbfT6JpNHE74#yU_|n(5<eG9Vnt>#SWb#bf`RKj6BkcFVo`CbYeiyi
zK~Aa=9~W0~YEfolPG(hViUhk5Cl_m8VlLR{5?tKH1t7KY*{PMq66{*+eh|Kp7#CMc
zYF=?FOoGi1%oUR0;sZ0{Q&Q6sOLIz!jo3oK0#edkd=S0ynRzLh$*IK>oN(i$WVv{u
zk`ON%@j$tt_%D`{f=h!F6pB_cE*`K2@lc&m`;-K@;EsiaixNK<%#~09RW1~3gt&4t
zi%T>>R%&W6TCsDn8Zk3+vMb4RLCwUaKo;f%i0g#7!RCNDni`B;&RpC9If==s8TmOW
zsYRH{8#Og6iEy!kwF*HfDHbax2}Z66E-@rS!ZS-UTp%GV#K*-7N%4?8LC}0YRP(u_
zx!8kJAvSVz!EzX5h!8(iof2CJSQa(Gp=5Zd3P^U8VlrU1V&h^lVq)gx%ID$?N-ZwQ
zFG>wF65`@wgJdfSb|C>S9!O@6hb1-<E&;GT@rijU@x?jdB*HEw$i)s(S8T)vE)9y6
zB)Fg~)c7!91ZBD$F1FyrGBRxwBGonzF8<(*#G;hc6t~QrRFF@Ec(^!{5{toXNM<Sq
zDUe_W<+HrfTu_l%lu|6gEXBjckp|Ty!OrE!#T8sql$n?Am7kd>#LLA7j#E&cWAu~a
zf)!*E>`JIPU7U-vIJF?LD6u5JNQkW%Y=RV%5Cc~+7gtzfQD$ONPHLEu6hEXahoq1O
zMn)yHM1WSJDv5J(Cg<msBxdHNq8O_r$;A!wPihJ{+K_GRU}RKcElw>eOV#2s)H5{F
zGtezcHZWo@&CAZqFV7R|wBt(V;!o7i%*!mvOw55~S0PA|uf(gLl3JFToNDD95Nc(h
z#SJUpnG6`U#JKq4A@u<$><Utgg;=<ltQfRJVYLE}6Bh@g5Q7#A0|SFSR|ywmq7ahb
z$ue6I)tgL03|joiK>`xshUO0;CMyQ6sf=74iTco*6Iz2xF(yi(dl=m{f~XZDlLl`8
zvV*EvP=JVFRIngzB3x|o$vKI|#X?+M971e4`N>ueiCocKY>E0ssfj5<@G4S@F%jJr
zxPlnTN^FKFbMYt9I6fUf@tK4#KJms2aVZHiw!!h4gb|-fQs^E=4@u(VCP|BoxCg~&
z5=ML`p}PWayd~kvQK0;pOjLa0N-x;5q=N%WB!J>G8DD%7mG?1YjEMZ1j1iy7Qv7Hh
zMh{8i<1?Au_)Nx#&t!C0;Em5@obegQ#Z{6Pp9*c~AvZ$MEs(${W7)XC?G7zoSSbhL
zBE=3nR|=O9a!XPl+<eDTqzl4IYYrzi4n`qp0Sd0sg+PTG4i^Y9aOHCequ7$2S{Ywq
zgu@yn_c3ZoVfc!Ji_MBfh>3$yOA_WD0Vh5VMj?nGs0IO5ol+<kLR&}r;MNIYdztat
zD+aTd*NK~h8Dg&$BO})&MlKO-whF--XHwWgK?;vc(c_qh3)W0yGQjUfSaRbk<dR2m
z6AA~^+9@U+9PC;+qgD$gQLuv&1y?zj8j39_97sbS)E9|2z+tlxEPOeg*g4pR7@Qa@
zkjo!-oSCJ9NQ(_|SS$>)n9GR+Y%x=X5pg#EW8#uT@g_1y2&J<og_NPC2*r&Qks(Q>
zIzuTexm^ltqD4=7a%deSJ}w?eHiVE`yokOUlLUhn54@wsB*CO*g^`OjxYVo!g}6EX
z!vG^FOJfnz5<=_YF>x_y3BbE^;7$r7BUdIDV+B(C2zO+l`%4|eX`)=h*xUinQDKlV
z0nosJ6tra~#aMxEE=;)=u@-PeK!$uk<33Ul!y(NCbd!lSPz&Y+u3#?4N~FL6*??{+
zOf4zSW%Vu1i8mC2xE0bwrAk1=aIre3q<~CzOi7V~*doOSsbkUIO?qI2aWPdIA%zCS
zF3Qc#;bN*lHdh+C2+&97SQ%1oTO=1aT#QgdpQ*x#a-*TaZzKdtCy?Mb!WI0aCmT@k
z8>0t56<nPO9eRWo?YQa)EfZK7ukEA>F5{K;OL7bJ^NTXmGxHL2^iwMe@{3CJ6N`&e
zON#Z=^7HjdDoRi)0j^XoSy-<>9x||jwZ*L^0kewVi5F}YJ2vBRxs<DziyzGz<Tf{o
zlW>^<YR@9~^+8QGyqztseT-aUXtts^xzSxEg+DZKx(%9k*tM`X(3RkwVF@lVArWwA
z7|mrM?;(#xGifk#m2runsX}j?qq`qgPO@teV<nD$6ITcq4`k>-A3TT&ONJDtfw_$E
zv3%Hw1iJI3Q0zr`hKixefGZBUY`9oJgF!;zVQwkTIN11!6lWZ)>n+6@2kW(=l}%iB
zpdqr%l6XTh_47gX8$tAQiF5ITToa#|lM`P88a6A2N*HnRrx)dy7R0Bd78L7)hU6g<
zQtWYH4MtJ|5C%dgmm!xRnjW}2p}NqEaV`rkE`%1S4FYipA+XI-Lg*s#hFn@)JP2h-
z_CS)96gQF<u1GH7;>5C4eONNr2M;sC5*$kB1k`wD&CO3q%^|9|;L79TMKM|k$AGmI
zHWz@bfb_3GLwiWQINS~PR4y*CeUM%!F3V8eM20mjj9e06dl193kU%Srhjj(CY+!}8
zv6CUVu-1+*NzE;YPcg7eHB3u1GEFtKG)XZtu&_)tNHk6|wlGREOtVNe(l1CXLaiTg
zW|mSeel%AIAq^NyAq9mLPUnKgND!mNpz<6?fdncD{Dhb!n79I=)*?o(VQB@Wqs1)5
zV8zG?3SC%_o*Ac4vbm%nJ^_u3BL>bv-Rq(hsOe~#T?^AP2I4I1VdN6QW}6WDxV;p5
z@JiuvB~&pYP|)3rBV4)4VQC3GMWGLA0Dy<marLBSU`b2ZNsxmbJYK-;#F*&BnB>Hm
zj2s5+c!nLAxhlA1Ai)5!7@XCRCR%U=JIWvl%yI@N1}Buz$8U2VBNsQ4)k4te87ZvM
z0!zCPS0V?G6khj(hT0jEq!=+q&II9eMUd${Mn<j$j9lVi2Z3fdit=+n<$7vfN@`v*
zaz6*V^LXOGIw3PmL`FI&Gld8-8L(@q!&@b?T+%|4;8qF5Jk+efwT6*P8!Ul2O|6e*
zj#>yjL5wRzAx_5hEV}dIO)7TKTr!gc{_!&|=)5v$O1TFXqL|JV!ZOn?MT*~$OEz>D
zi*jL1z=K+Gos7_$2)-x)G53K|PC`Nk)6r5A5E;T@h9jTDqnxXXOBx)J@JUcQ=J{$a
zc{ID}nC(|Ga`B<sE`*o~Mvhb|d;ugy$Oq_20yDpBDPRoJfQFx#Bp5;S>LAQD9acm^
zT1-N))-q$F6q+B<t%K$nMjYJ`taEZ$IvzOd%Gt1_3~^b!A;N7*QUa*qi#akthU-X~
zmEuBooe{!y$r!G~=%y3MJEVjuwz(Isa4sQm02M*jAAly0i&CHsYiKzR2~{aRNOp(t
z;tkRG=nm(y;ev!b%;3bF9H_Cv5QAV6px$!|T9Q#>;*gfI;o<>14cv6KG7y5aJz&L^
zl>s#HVCE#|<S4LeFq$$r$hcWBDXnJY5-ur9%*+GzQsT>?v;4(ElNp63FmmZ}F(+CX
z2*GEgr8pp?YgPtEf@rhvRt6Se^+_Q0@GchzWaJQ}9&L6Pral>@9@a0D;z$N}l0oXx
zCeuLbmE^ei!Q&zEpe{#2Vo64^5H}Ym7dzDNN(&jeI0v~Ov`V@7ic-^yQj3e@E8<J?
z<12-dx#U3o$AZ$5Vj&SO9>g@8m4N`e6q5v_kSZ5DWYt2kkQ5hlrImpIqZH<#wUiX3
zk&~8_U#<^YJ&~E0t`D_KtDZ{$%8ait0^4gWl+C3Iw^*8srNRh@!9r>X`=z;9Dvfcd
z!|Q-TE`F$+VC%Vr;<)5NqvcXu%oR9H6p{yx7?bMff}+%v%;XZSGA@4bDu&FoO5(#>
zg^R5?Ilm|se^5(f32K;KT5epT@$sNcnGz4~^Tvbw_W1=ux<cBxq=ZmA-k@mLD(B*d
e#~C=vjD%7MrWLff!WF0Z;>t(_DN_l-1`7eQb}<zI

literal 0
HcmV?d00001

diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.data-00000-of-00001 b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.data-00000-of-00001
new file mode 100644
index 0000000000000000000000000000000000000000..15b75d6ef6bffc336d138d923badb3928b8c4c13
GIT binary patch
literal 12
TcmZQzV6bOkU~phyaBu(s1VaG;

literal 0
HcmV?d00001

diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.index b/tensorflow/cc/saved_model/testdata/half_plus_two_v2/00000123/variables/variables.index
new file mode 100644
index 0000000000000000000000000000000000000000..7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326
GIT binary patch
literal 151
zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5<V^x`6<9P_Oa=Z{_%kr_CW8eyG+0dE
zUu!zdz`(%32qK<{ZrcN*!JGr17H(i*WJ+Ohf(u1#dimX*BZLnmKnREbZs=Aib-xV&
DMWhyT

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
index 4c8db94d6f..ffba514bb9 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
@@ -125,7 +125,7 @@ def save_keras_model(
   export_dir = export_helpers.get_timestamped_export_dir(saved_model_path)
   temp_export_dir = export_helpers.get_temp_export_dir(export_dir)
 
-  builder = saved_model_builder.SavedModelBuilder(temp_export_dir)
+  builder = saved_model_builder._SavedModelBuilder(temp_export_dir)
 
   # Manually save variables to export them in an object-based checkpoint. This
   # skips the `builder.add_meta_graph_and_variables()` step, which saves a
@@ -227,9 +227,10 @@ def _export_mode(
       g.add_to_collection(ops.GraphKeys.GLOBAL_STEP, clone.optimizer.iterations)
 
     # Extract update and train ops from train/test/predict functions.
+    train_op = None
     if mode == model_fn_lib.ModeKeys.TRAIN:
       clone._make_train_function()
-      builder._add_train_op(clone.train_function.updates_op)
+      train_op = clone.train_function.updates_op
     elif mode == model_fn_lib.ModeKeys.EVAL:
       clone._make_test_function()
     else:
@@ -264,7 +265,8 @@ def _export_mode(
         model_fn_lib.EXPORT_TAG_MAP[mode],
         signature_def_map=_create_signature_def_map(clone, mode),
         saver=saver_lib.Saver(clone_var_list),
-        main_op=variables.local_variables_initializer())
+        init_op=variables.local_variables_initializer(),
+        train_op=train_op)
     return None
 
 
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
index d8637effe2..93d73e1b48 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
@@ -35,7 +35,6 @@ from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
-from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.training import training as training_module
@@ -254,7 +253,7 @@ def load_model(sess, path, mode):
   outputs = {
       k: sess.graph.get_tensor_by_name(v.name)
       for k, v in meta_graph_def.signature_def[sig_def_key].outputs.items()}
-  return inputs, outputs
+  return inputs, outputs, meta_graph_def
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -331,8 +330,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
 
     # Load predict graph, and test predictions
     with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs = load_model(sess, output_path,
-                                   model_fn_lib.ModeKeys.PREDICT)
+      inputs, outputs, _ = load_model(sess, output_path,
+                                      model_fn_lib.ModeKeys.PREDICT)
 
       predictions = sess.run(outputs[output_name],
                              {inputs[input_name]: input_arr})
@@ -341,8 +340,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
     if optimizer:
       # Load eval graph, and test predictions, loss and metric values
       with session.Session(graph=ops.Graph()) as sess:
-        inputs, outputs = load_model(sess, output_path,
-                                     model_fn_lib.ModeKeys.EVAL)
+        inputs, outputs, _ = load_model(sess, output_path,
+                                        model_fn_lib.ModeKeys.EVAL)
 
         # First obtain the loss and predictions, and run the metric update op by
         # feeding in the inputs and targets.
@@ -365,8 +364,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
 
       # Load train graph, and check for the train op, and prediction values
       with session.Session(graph=ops.Graph()) as sess:
-        inputs, outputs = load_model(sess, output_path,
-                                     model_fn_lib.ModeKeys.TRAIN)
+        inputs, outputs, meta_graph_def = load_model(
+            sess, output_path, model_fn_lib.ModeKeys.TRAIN)
         self.assertEqual(int(train_before_export),
                          sess.run(training_module.get_global_step()))
         self.assertIn('loss', outputs)
@@ -375,7 +374,7 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
         self.assertIn('predictions/' + output_name, outputs)
 
         # Train for a step
-        train_op = ops.get_collection(constants.TRAIN_OP_KEY)
+        train_op = loader_impl.get_train_op(meta_graph_def)
         train_outputs, _ = sess.run(
             [outputs, train_op], {inputs[input_name]: input_arr,
                                   inputs[target_name]: target_arr})
@@ -402,8 +401,8 @@ class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
       output_path = keras_saved_model.save_keras_model(
           model, saved_model_path, custom_objects={'relu6': relu6})
     with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs = load_model(sess, output_path,
-                                   model_fn_lib.ModeKeys.PREDICT)
+      inputs, outputs, _ = load_model(sess, output_path,
+                                      model_fn_lib.ModeKeys.PREDICT)
       input_name = model.input_names[0]
       output_name = model.output_names[0]
       predictions = sess.run(
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index e7a3b8afd5..31b5a59da2 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -83,6 +83,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":constants",
+        ":signature_def_utils",
         ":utils",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_for_generated_wrappers",
@@ -114,6 +115,7 @@ py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index ce7641cd98..fc84979b6e 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -33,6 +33,7 @@ from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.saved_model import constants
+from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training import saver as tf_saver
 from tensorflow.python.util import compat
@@ -40,6 +41,8 @@ from tensorflow.python.util.deprecation import deprecated_args
 from tensorflow.python.util.tf_export import tf_export
 
 
+# Base class for the SavedModelBuilder that is only used by Tensorflow
+# internally. Please use tf.compat.v1.saved_model.SavedModelBuilder instead.
 class _SavedModelBuilder(object):
   """Builds the `SavedModel` protocol buffer and saves variables and assets.
 
@@ -144,52 +147,6 @@ class _SavedModelBuilder(object):
     # Copy assets from source path to destination path.
     self._copy_assets_to_destination_dir(asset_filename_map)
 
-  def _maybe_add_main_op(self, main_op):
-    """Adds main op to the SavedModel.
-
-    Args:
-      main_op: Main op to run as part of graph initialization. If None, no
-        main op will be added to the graph.
-
-    Raises:
-      TypeError: if main op is provided but is not of type `Operation`.
-      ValueError: if the Graph already contains an init op.
-    """
-    if main_op is None:
-      return
-
-    if not isinstance(main_op, ops.Operation):
-      raise TypeError("main_op needs to be an Operation: %r" % main_op)
-
-    # Validate that no other init ops have been added to this graph already.
-    # We check main_op and legacy_init_op for thoroughness and explicitness.
-    for init_op_key in (constants.MAIN_OP_KEY, constants.LEGACY_INIT_OP_KEY):
-      if ops.get_collection(init_op_key):
-        raise ValueError(
-            "Graph already contains one or more main ops under the "
-            "collection {}.".format(init_op_key))
-
-    ops.add_to_collection(constants.MAIN_OP_KEY, main_op)
-
-  def _add_train_op(self, train_op):
-    """Add train op to the SavedModel.
-
-    Note that this functionality is in development, and liable to be
-    moved elsewhere.
-
-    Args:
-      train_op: Op or group of ops that are used for training. These are
-        stored as a collection with key TRAIN_OP_KEY, but not executed.
-
-    Raises:
-      TypeError if Train op is not of type `Operation`.
-    """
-    if train_op is not None:
-      if (not isinstance(train_op, ops.Tensor) and
-          not isinstance(train_op, ops.Operation)):
-        raise TypeError("train_op needs to be a Tensor or Op: %r" % train_op)
-      ops.add_to_collection(constants.TRAIN_OP_KEY, train_op)
-
   def _tag_and_add_meta_graph(self, meta_graph_def, tags, signature_def_map):
     """Tags the meta graph def and adds it to the SavedModel.
 
@@ -245,26 +202,32 @@ class _SavedModelBuilder(object):
 
     Validation of entries in the signature def map includes ensuring that the
     `name` and `dtype` fields of the TensorInfo protos of the `inputs` and
-    `outputs` of each `SignatureDef` are populated.
+    `outputs` of each `SignatureDef` are populated. Also ensures that reserved
+    SigantureDef keys for the initialization and train ops are not used.
 
     Args:
       signature_def_map: The map of signature defs to be validated.
-    """
-    if signature_def_map is not None:
-      for signature_def_key in signature_def_map:
-        signature_def = signature_def_map[signature_def_key]
-        inputs = signature_def.inputs
-        outputs = signature_def.outputs
-        for inputs_key in inputs:
-          self._validate_tensor_info(inputs[inputs_key])
-        for outputs_key in outputs:
-          self._validate_tensor_info(outputs[outputs_key])
-
-  def _add_collections(self, main_op, train_op):
-    """Add asset and op collections to be saved."""
-    self._maybe_add_main_op(main_op)
 
-    self._add_train_op(train_op)
+    Raises:
+      AssertionError: If a TensorInfo is not valid.
+      KeyError: If a reserved signature key is used in the map.
+    """
+    for signature_def_key in signature_def_map:
+      signature_def = signature_def_map[signature_def_key]
+      inputs = signature_def.inputs
+      outputs = signature_def.outputs
+      for inputs_key in inputs:
+        self._validate_tensor_info(inputs[inputs_key])
+      for outputs_key in outputs:
+        self._validate_tensor_info(outputs[outputs_key])
+    if constants.INIT_OP_SIGNATURE_KEY in signature_def_map:
+      raise KeyError(
+          "SignatureDef map key \"{}\" is reserved for initialization. Please "
+          "use a different key.".format(constants.INIT_OP_SIGNATURE_KEY))
+    if constants.TRAIN_OP_SIGNATURE_KEY in signature_def_map:
+      raise KeyError(
+          "SignatureDef map key \"{}\" is reserved for the train op. Please "
+          "use a different key.".format(constants.TRAIN_OP_SIGNATURE_KEY))
 
   def _maybe_create_saver(self, saver=None):
     """Creates a sharded saver if one does not already exist."""
@@ -278,19 +241,14 @@ class _SavedModelBuilder(object):
           allow_empty=True)
     return saver
 
-  @deprecated_args(None,
-                   "Pass your op to the equivalent parameter main_op instead.",
-                   "legacy_init_op")
   def add_meta_graph(self,
                      tags,
                      signature_def_map=None,
                      assets_list=None,
-                     legacy_init_op=None,
                      clear_devices=False,
-                     main_op=None,
-                     strip_default_attrs=False,
+                     init_op=None,
+                     train_op=None,
                      saver=None):
-    # pylint: disable=line-too-long
     """Adds the current meta graph to the SavedModel.
 
     Creates a Saver in the current scope and uses the Saver to export the meta
@@ -304,16 +262,14 @@ class _SavedModelBuilder(object):
       assets_list: Assets to be saved with SavedModel. Note
           that this list should be a subset of the assets saved as part of
           the first meta graph in the SavedModel.
-      legacy_init_op: Legacy support for op or group of ops to execute after the
-          restore op upon a load. Deprecated; please use main_op instead.
       clear_devices: Set to true if the device info on the default graph should
           be cleared.
-      main_op: Op or group of ops to execute when the graph is loaded. Note
-          that when the main_op is specified it is run after the restore op at
+      init_op: Op or group of ops to execute when the graph is loaded. Note
+          that when the init_op is specified it is run after the restore op at
           load-time.
-      strip_default_attrs: Boolean. If `True`, default-valued attributes will be
-        removed from the NodeDefs. For a detailed guide, see
-        [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
+      train_op: Op or group of opts that trains the model when run. This will
+        not be run automatically when the graph is loaded, instead saved in
+        a SignatureDef accessible through the exported MetaGraph.
       saver: An instance of tf.train.Saver that will be used to export the
         metagraph. If None, a sharded Saver that restores all variables will
         be used.
@@ -322,7 +278,6 @@ class _SavedModelBuilder(object):
       AssertionError: If the variables for the SavedModel have not been saved
           yet, or if the graph already contains one or more legacy init ops.
     """
-    # pylint: enable=line-too-long
     if not self._has_saved_variables:
       raise AssertionError(
           "Graph state including variables and assets has not been saved yet. "
@@ -330,14 +285,15 @@ class _SavedModelBuilder(object):
 
     # Validate the signature def map to ensure all included TensorInfos are
     # properly populated.
+    signature_def_map = signature_def_map or {}
     self._validate_signature_def_map(signature_def_map)
 
-    # legacy_init_op is deprecated, and going away in TF 2.0.
-    # Re-mapping to main_op, as treatment is identical regardless.
-    main_op = main_op or legacy_init_op
-
-    # Add ops to collection.
-    self._add_collections(main_op=main_op, train_op=None)
+    # Create a SignatureDef pointing to the graph initialization op, which will
+    # be added to the MetaGraphDef.
+    _add_op_to_signature_def_map(signature_def_map, init_op,
+                                 constants.INIT_OP_SIGNATURE_KEY)
+    _add_op_to_signature_def_map(signature_def_map, train_op,
+                                 constants.TRAIN_OP_SIGNATURE_KEY)
 
     saver = self._maybe_create_saver(saver)
 
@@ -349,7 +305,7 @@ class _SavedModelBuilder(object):
     # resolved, we just leave the option set to False for now.
     # TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
     meta_graph_def = saver.export_meta_graph(
-        clear_devices=clear_devices, strip_default_attrs=strip_default_attrs)
+        clear_devices=clear_devices, strip_default_attrs=True)
 
     # Save asset files and write them to disk, if any.
     self._save_and_write_assets(meta_graph_def, assets_list)
@@ -357,17 +313,14 @@ class _SavedModelBuilder(object):
     # Tag the meta graph def and add it to the SavedModel.
     self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
 
-  @deprecated_args(None,
-                   "Pass your op to the equivalent parameter main_op instead.",
-                   "legacy_init_op")
   def add_meta_graph_and_variables(self,
                                    sess,
                                    tags,
                                    signature_def_map=None,
                                    assets_list=None,
-                                   legacy_init_op=None,
                                    clear_devices=False,
-                                   main_op=None,
+                                   init_op=None,
+                                   train_op=None,
                                    strip_default_attrs=False,
                                    saver=None):
     # pylint: disable=line-too-long
@@ -386,13 +339,14 @@ class _SavedModelBuilder(object):
       signature_def_map: The map of signature def map to add to the meta graph
         def.
       assets_list: Assets to be saved with SavedModel.
-      legacy_init_op: Legacy support for op or group of ops to execute after the
-          restore op upon a load. Deprecated; please use main_op instead.
       clear_devices: Set to true if the device info on the default graph should
           be cleared.
-      main_op: Op or group of ops to execute when the graph is loaded. Note
-          that when the main_op is specified it is run after the restore op at
+      init_op: Op or group of ops to execute when the graph is loaded. Note
+          that when the init_op is specified it is run after the restore op at
           load-time.
+      train_op: Op or group of ops that trains the model when run. This will
+        not be run automatically when the graph is loaded, instead saved in
+        a SignatureDef accessible through the exported MetaGraph.
       strip_default_attrs: Boolean. If `True`, default-valued attributes will be
         removed from the NodeDefs. For a detailed guide, see
         [Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
@@ -409,14 +363,15 @@ class _SavedModelBuilder(object):
 
     # Validate the signature def map to ensure all included TensorInfos are
     # properly populated.
+    signature_def_map = signature_def_map or {}
     self._validate_signature_def_map(signature_def_map)
 
-    # legacy_init_op is deprecated, and going away in TF 2.0.
-    # Re-mapping to main_op, as treatment is identical regardless.
-    main_op = main_op or legacy_init_op
-
-    # Add ops to collection.
-    self._add_collections(main_op=main_op, train_op=None)
+    # Create a SignatureDef pointing to the graph initialization op, which will
+    # be added to the MetaGraphDef.
+    _add_op_to_signature_def_map(signature_def_map, init_op,
+                                 constants.INIT_OP_SIGNATURE_KEY)
+    _add_op_to_signature_def_map(signature_def_map, train_op,
+                                 constants.TRAIN_OP_SIGNATURE_KEY)
 
     saved_model_utils.get_or_create_variables_dir(self._export_dir)
     variables_path = saved_model_utils.get_variables_path(self._export_dir)
@@ -517,6 +472,52 @@ class SavedModelBuilder(_SavedModelBuilder):
     # Copy assets from source path to destination path.
     self._copy_assets_to_destination_dir(asset_filename_map)
 
+  def _maybe_add_main_op(self, main_op):
+    """Adds main op to the SavedModel.
+
+    Args:
+      main_op: Main op to run as part of graph initialization. If None, no main
+        op will be added to the graph.
+
+    Raises:
+      TypeError: if main op is provided but is not of type `Operation`.
+      ValueError: if the Graph already contains an init op.
+    """
+    if main_op is None:
+      return
+
+    if not isinstance(main_op, ops.Operation):
+      raise TypeError("main_op needs to be an Operation: %r" % main_op)
+
+    # Validate that no other init ops have been added to this graph already.
+    # We check main_op and legacy_init_op for thoroughness and explicitness.
+    for init_op_key in (constants.MAIN_OP_KEY, constants.LEGACY_INIT_OP_KEY):
+      if ops.get_collection(init_op_key):
+        raise ValueError(
+            "Graph already contains one or more main ops under the "
+            "collection {}.".format(init_op_key))
+
+    ops.add_to_collection(constants.MAIN_OP_KEY, main_op)
+
+  def _add_train_op(self, train_op):
+    """Add train op to the SavedModel.
+
+    Note that this functionality is in development, and liable to be
+    moved elsewhere.
+
+    Args:
+      train_op: Op or group of ops that are used for training. These are stored
+        as a collection with key TRAIN_OP_KEY, but not executed.
+
+    Raises:
+      TypeError if Train op is not of type `Operation`.
+    """
+    if train_op is not None:
+      if (not isinstance(train_op, ops.Tensor) and
+          not isinstance(train_op, ops.Operation)):
+        raise TypeError("train_op needs to be a Tensor or Op: %r" % train_op)
+      ops.add_to_collection(constants.TRAIN_OP_KEY, train_op)
+
   @deprecated_args(None,
                    "Pass your op to the equivalent parameter main_op instead.",
                    "legacy_init_op")
@@ -536,6 +537,7 @@ class SavedModelBuilder(_SavedModelBuilder):
 
     # Validate the signature def map to ensure all included TensorInfos are
     # properly populated.
+    signature_def_map = signature_def_map or {}
     self._validate_signature_def_map(signature_def_map)
 
     # legacy_init_op is deprecated, and going away in TF 2.0.
@@ -580,6 +582,7 @@ class SavedModelBuilder(_SavedModelBuilder):
 
     # Validate the signature def map to ensure all included TensorInfos are
     # properly populated.
+    signature_def_map = signature_def_map or {}
     self._validate_signature_def_map(signature_def_map)
 
     # legacy_init_op is deprecated, and going away in TF 2.0.
@@ -774,3 +777,8 @@ def _add_asset_to_collection(asset_filename, asset_tensor):
   asset_any_proto = Any()
   asset_any_proto.Pack(asset_proto)
   ops.add_to_collection(constants.ASSETS_KEY, asset_any_proto)
+
+
+def _add_op_to_signature_def_map(signature_def_map, op, key):
+  if op is not None:
+    signature_def_map[key] = signature_def_utils.op_signature_def(op, key)
diff --git a/tensorflow/python/saved_model/constants.py b/tensorflow/python/saved_model/constants.py
index 1edc0c8d93..0cd5588a8f 100644
--- a/tensorflow/python/saved_model/constants.py
+++ b/tensorflow/python/saved_model/constants.py
@@ -48,7 +48,6 @@ tf_export(
 # CollectionDef key for the SavedModel main op.
 MAIN_OP_KEY = "saved_model_main_op"
 tf_export(
-    "saved_model.MAIN_OP_KEY",
     v1=["saved_model.MAIN_OP_KEY",
         "saved_model.constants.MAIN_OP_KEY"]).export_constant(
             __name__, "MAIN_OP_KEY")
@@ -105,3 +104,8 @@ tf_export(
         "saved_model.VARIABLES_FILENAME",
         "saved_model.constants.VARIABLES_FILENAME"
     ]).export_constant(__name__, "VARIABLES_FILENAME")
+
+# The initialization and train ops for a MetaGraph are stored in the
+# signature def map. The ops are added to the map with the following keys.
+INIT_OP_SIGNATURE_KEY = "__saved_model_init_op"
+TRAIN_OP_SIGNATURE_KEY = "__saved_model_train_op"
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index f50a07fee4..6bf39a2c67 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -31,6 +31,7 @@ from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.saved_model import constants
+from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training import saver as tf_saver
 from tensorflow.python.util import compat
@@ -141,15 +142,46 @@ def _get_main_op_tensor(
     RuntimeError: If the collection def corresponding to the main op key has
         other than exactly one tensor.
   """
+  # TODO(kathywu): Rename this method to _get_op_from_collection when
+  # dependency from SavedModelEstimator is removed.
   collection_def = meta_graph_def_to_load.collection_def
-  main_op_tensor = None
+  init_op = None
   if init_op_key in collection_def:
-    main_ops = collection_def[init_op_key].node_list.value
-    if len(main_ops) != 1:
-      raise RuntimeError("Expected exactly one SavedModel main op. "
-                         "Found: {}".format(main_ops))
-    main_op_tensor = ops.get_collection(init_op_key)[0]
-  return main_op_tensor
+    init_op_list = collection_def[init_op_key].node_list.value
+    if len(init_op_list) != 1:
+      raise RuntimeError("Expected exactly one SavedModel init op. "
+                         "Found: {}".format(init_op_list))
+    init_op = ops.get_collection(init_op_key)[0]
+  return init_op
+
+
+def _get_op_from_collection(meta_graph_def, op_key):
+  return _get_main_op_tensor(meta_graph_def, op_key)
+
+
+def _get_op_from_signature_def(meta_graph_def, op_signature_key, import_scope):
+  """Retrieve op stored in the imported meta graph's signature def."""
+  if op_signature_key in meta_graph_def.signature_def:
+    return signature_def_utils.load_op_from_signature_def(
+        meta_graph_def.signature_def[op_signature_key], op_signature_key,
+        import_scope)
+  else:
+    return None
+
+
+def get_init_op(meta_graph_def, import_scope=None):
+  return (_get_op_from_signature_def(
+      meta_graph_def, constants.INIT_OP_SIGNATURE_KEY, import_scope) or
+          _get_op_from_collection(meta_graph_def, constants.MAIN_OP_KEY) or
+          _get_op_from_collection(meta_graph_def, constants.LEGACY_INIT_OP_KEY))
+
+
+def get_train_op(meta_graph_def, import_scope=None):
+  train_op = _get_op_from_signature_def(
+      meta_graph_def, constants.TRAIN_OP_SIGNATURE_KEY, import_scope)
+  if train_op is None:
+    train_op = _get_op_from_collection(meta_graph_def, constants.TRAIN_OP_KEY)
+  return train_op
 
 
 @tf_export(v1=[
@@ -359,11 +391,9 @@ class SavedModelLoader(object):
       asset_tensors_dictionary = _get_asset_tensors(
           self._export_dir, meta_graph_def, import_scope=import_scope)
 
-      main_op_tensor = (
-          _get_main_op_tensor(meta_graph_def, constants.MAIN_OP_KEY) or
-          _get_main_op_tensor(meta_graph_def, constants.LEGACY_INIT_OP_KEY))
-      if main_op_tensor is not None:
-        sess.run(fetches=[main_op_tensor], feed_dict=asset_tensors_dictionary)
+      init_op = get_init_op(meta_graph_def, import_scope)
+      if init_op is not None:
+        sess.run(fetches=[init_op], feed_dict=asset_tensors_dictionary)
 
   def load(self, sess, tags, import_scope=None, **saver_kwargs):
     """Load the MetaGraphDef graph and restore variable values into the session.
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 3678e505bd..3378bf08cd 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -19,11 +19,13 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import shutil
+
+from absl.testing import parameterized
 
 from tensorflow.python.client import session
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
-from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
@@ -42,55 +44,72 @@ SIMPLE_ADD_SAVED_MODEL = _get_export_dir("simple_add_saved_model")
 SAVED_MODEL_WITH_MAIN_OP = _get_export_dir("saved_model_with_main_op")
 
 
-class SavedModelLoaderTest(test.TestCase):
+def build_graph_helper():
+  g = ops.Graph()
+  with g.as_default():
+    x = variables.VariableV1(5, name="x")
+    y = variables.VariableV1(11, name="y")
+    z = x + y
 
-  def setUp(self):
-    """Write test SavedModels to a temp directory."""
-    with session.Session(graph=ops.Graph()) as sess:
-      x = variables.VariableV1(5, name="x")
-      y = variables.VariableV1(11, name="y")
-      z = x + y
-      self.evaluate(variables.global_variables_initializer())
+    foo_sig_def = signature_def_utils.build_signature_def({
+        "foo_input": utils.build_tensor_info(x)
+    }, {"foo_output": utils.build_tensor_info(z)})
+    bar_sig_def = signature_def_utils.build_signature_def({
+        "bar_x": utils.build_tensor_info(x),
+        "bar_y": utils.build_tensor_info(y)
+    }, {"bar_z": utils.build_tensor_info(z)})
+  return g, {"foo": foo_sig_def, "bar": bar_sig_def}, y
 
-      foo_sig_def = signature_def_utils.build_signature_def(
-          {"foo_input": utils.build_tensor_info(x)},
-          {"foo_output": utils.build_tensor_info(z)})
-      bar_sig_def = signature_def_utils.build_signature_def(
-          {"bar_x": utils.build_tensor_info(x),
-           "bar_y": utils.build_tensor_info(y)},
-          {"bar_z": utils.build_tensor_info(z)})
 
-      builder = saved_model_builder.SavedModelBuilder(SIMPLE_ADD_SAVED_MODEL)
-      builder.add_meta_graph_and_variables(
-          sess, ["foo_graph"], {"foo": foo_sig_def, "bar": bar_sig_def})
+@parameterized.parameters((saved_model_builder.SavedModelBuilder,),
+                          (saved_model_builder._SavedModelBuilder,))
+class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
+
+  def export_simple_graph(self, builder_cls):
+    g, sig_def_map, _ = build_graph_helper()
+    with session.Session(graph=g) as sess:
+      self.evaluate(variables.global_variables_initializer())
+      builder = builder_cls(SIMPLE_ADD_SAVED_MODEL)
+      builder.add_meta_graph_and_variables(sess, ["foo_graph"], sig_def_map)
       builder.save()
 
-      # Write SavedModel with a main_op
+  def export_graph_with_main_op(self, builder_cls):
+    g, sig_def_map, y = build_graph_helper()
+    with session.Session(graph=g) as sess:
+      self.evaluate(variables.global_variables_initializer())
       assign_op = control_flow_ops.group(state_ops.assign(y, 7))
 
-      builder = saved_model_builder.SavedModelBuilder(SAVED_MODEL_WITH_MAIN_OP)
-      builder.add_meta_graph_and_variables(
-          sess, ["foo_graph"], {"foo": foo_sig_def, "bar": bar_sig_def},
-          main_op=assign_op)
+      builder = builder_cls(SAVED_MODEL_WITH_MAIN_OP)
+
+      if builder_cls == saved_model_builder._SavedModelBuilder:
+        builder.add_meta_graph_and_variables(
+            sess, ["foo_graph"], sig_def_map, init_op=assign_op)
+      else:
+        builder.add_meta_graph_and_variables(
+            sess, ["foo_graph"], sig_def_map, main_op=assign_op)
       builder.save()
 
   def tearDown(self):
-    file_io.delete_recursively(test.get_temp_dir())
+    super(SavedModelLoaderTest, self).tearDown()
+    shutil.rmtree(test.get_temp_dir(), ignore_errors=True)
 
-  def test_load_function(self):
+  def test_load_function(self, builder_cls):
+    self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
     with self.session(graph=ops.Graph()) as sess:
       loader.load(sess, ["foo_graph"])
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval())
 
+    self.export_graph_with_main_op(builder_cls)
     loader2 = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     with self.session(graph=ops.Graph()) as sess:
       loader2.load(sess, ["foo_graph"])
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("y:0").eval())
 
-  def test_load_graph(self):
+  def test_load_graph(self, builder_cls):
+    self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
     graph = ops.Graph()
     loader.load_graph(graph, ["foo_graph"])
@@ -101,14 +120,15 @@ class SavedModelLoaderTest(test.TestCase):
     with self.assertRaises(KeyError):
       graph.get_tensor_by_name("z:0")
 
-    with self.session(graph=graph) as sess:
+    with self.session(graph=graph):
       # Check that x and y are not initialized
       with self.assertRaises(errors.FailedPreconditionError):
         self.evaluate(x)
       with self.assertRaises(errors.FailedPreconditionError):
         self.evaluate(y)
 
-  def test_load_with_import_scope(self):
+  def test_load_with_import_scope(self, builder_cls):
+    self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     with self.session(graph=ops.Graph()) as sess:
       saver, _ = loader.load_graph(
@@ -119,7 +139,13 @@ class SavedModelLoaderTest(test.TestCase):
         loader.restore_variables(sess, tf_saver.Saver())
 
       loader.restore_variables(sess, saver)
-      loader.run_init_ops(sess, ["foo_graph"])
+
+      if builder_cls == saved_model_builder._SavedModelBuilder:
+        with self.assertRaises(errors.NotFoundError):
+          loader.run_init_ops(sess, ["foo_graph"])
+        loader.run_init_ops(sess, ["foo_graph"], import_scope="baz")
+      else:
+        loader.run_init_ops(sess, ["foo_graph"])
 
       self.assertEqual(5, sess.graph.get_tensor_by_name("baz/x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("baz/y:0").eval())
@@ -131,7 +157,8 @@ class SavedModelLoaderTest(test.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("baa/x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("baa/y:0").eval())
 
-  def test_restore_variables(self):
+  def test_restore_variables(self, builder_cls):
+    self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     with self.session(graph=ops.Graph()) as sess:
       x = variables.VariableV1(0, name="x")
@@ -147,7 +174,8 @@ class SavedModelLoaderTest(test.TestCase):
       loader.restore_variables(sess, tf_saver.Saver())
       self.assertEqual(55, self.evaluate(z))
 
-  def test_run_init_op(self):
+  def test_run_init_op(self, builder_cls):
+    self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     graph = ops.Graph()
     saver, _ = loader.load_graph(graph, ["foo_graph"])
@@ -160,14 +188,16 @@ class SavedModelLoaderTest(test.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("y:0").eval())
 
-  def test_parse_saved_model(self):
+  def test_parse_saved_model(self, builder_cls):
+    self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
     meta_graph = loader.get_meta_graph_def_from_tags(["foo_graph"])
     self.assertIsNotNone(meta_graph)
     self.assertIn("foo", meta_graph.signature_def)
     self.assertIn("bar", meta_graph.signature_def)
 
-  def test_load_invalid_meta_graph(self):
+  def test_load_invalid_meta_graph(self, builder_cls):
+    self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
     with self.assertRaises(RuntimeError):
       loader.get_meta_graph_def_from_tags([])
@@ -176,13 +206,16 @@ class SavedModelLoaderTest(test.TestCase):
     with self.assertRaises(RuntimeError):
       loader.get_meta_graph_def_from_tags(["not_a_graph"])
 
-  def test_load_saved_model_with_no_variables(self):
+  def test_load_saved_model_with_no_variables(self, builder_cls):
     """Test that SavedModel runs saver when there appear to be no variables.
 
     When no variables are detected, this may mean that the variables were saved
     to different collections, or the collections weren't saved to the
     SavedModel. If the SavedModel MetaGraphDef contains a saver, it should still
     run in either of these cases.
+
+    Args:
+      builder_cls: SavedModelBuilder or _SavedModelBuilder class
     """
     path = _get_export_dir("no_variable_saved_model")
     with session.Session(graph=ops.Graph()) as sess:
@@ -192,7 +225,7 @@ class SavedModelLoaderTest(test.TestCase):
           11, name="y", collections=["not_global_variable"])
       self.assertFalse(variables._all_saveable_objects())
       z = x + y
-      sess.run(variables.variables_initializer([x, y]))
+      self.evaluate(variables.variables_initializer([x, y]))
 
       foo_sig_def = signature_def_utils.build_signature_def(
           {"foo_input": utils.build_tensor_info(x)},
@@ -215,8 +248,9 @@ class SavedModelLoaderTest(test.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval())
 
-  def test_load_saved_model_graph_with_return_elements(self):
+  def test_load_saved_model_graph_with_return_elements(self, builder_cls):
     """Ensure that the correct elements are returned."""
+    self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
     graph = ops.Graph()
     _, ret = loader.load_graph(graph, ["foo_graph"],
@@ -228,5 +262,6 @@ class SavedModelLoaderTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "not found in graph"):
       loader.load_graph(graph, ["foo_graph"], return_elements=["z:0"])
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index e722b6ceae..efcd21cce3 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -146,6 +146,18 @@ class SavedModelTest(SavedModelTestBase):
           sess, ["foo"],
           signature_def_map={"foo_key": foo_signature})
 
+  def _validate_sig_def_keys(self, builder, valid_tensor_info, invalid_key):
+    with self.session(graph=ops.Graph()) as sess:
+      self._init_and_validate_variable(sess, "v", 42)
+
+      foo_signature = signature_def_utils.build_signature_def(
+          dict(), {"foo_key": valid_tensor_info}, "foo")
+      self.assertRaises(
+          KeyError,
+          builder.add_meta_graph_and_variables,
+          sess, ["foo"],
+          signature_def_map={invalid_key: foo_signature})
+
   def testMaybeSavedModelDir(self):
     base_path = test.test_src_dir_path("/python/saved_model")
     self.assertFalse(loader.maybe_saved_model_directory(base_path))
@@ -583,6 +595,15 @@ class SavedModelTest(SavedModelTestBase):
     self._validate_inputs_tensor_info_fail(builder, tensor_empty)
     self._validate_outputs_tensor_info_fail(builder, tensor_empty)
 
+    valid_tensor_info = meta_graph_pb2.TensorInfo()
+    valid_tensor_info.name = "foo"
+    valid_tensor_info.dtype = types_pb2.DT_FLOAT
+
+    self._validate_sig_def_keys(builder, valid_tensor_info,
+                                constants.INIT_OP_SIGNATURE_KEY)
+    self._validate_sig_def_keys(builder, valid_tensor_info,
+                                constants.TRAIN_OP_SIGNATURE_KEY)
+
   def testSignatureDefValidationSucceedsWithName(self):
     tensor_with_name = meta_graph_pb2.TensorInfo()
     tensor_with_name.name = "foo"
@@ -782,7 +803,7 @@ class SavedModelTest(SavedModelTestBase):
       self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
                             "foo bar baz 0", "asset_file_tensor_0:0")
 
-  def testCustomMainOp(self):
+  def testCustomInitOp(self):
     export_dir = self._get_export_dir("test_main_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
 
@@ -800,11 +821,11 @@ class SavedModelTest(SavedModelTestBase):
       # Set up an assignment op to be run as part of the main_op.
       with ops.control_dependencies([main_op.main_op()]):
         add_v1_v2 = math_ops.add(v1._ref(), v2._ref())
-        custom_main_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2))
+        custom_init_op = control_flow_ops.group(state_ops.assign(v3, add_v1_v2))
 
-      self.evaluate(custom_main_op)
+      self.evaluate(custom_init_op)
       builder.add_meta_graph_and_variables(
-          sess, ["foo"], main_op=custom_main_op)
+          sess, ["foo"], init_op=custom_init_op)
 
     # Save the SavedModel to disk.
     builder.save()
@@ -817,80 +838,6 @@ class SavedModelTest(SavedModelTestBase):
       # the main_op, following a restore.
       self.assertEqual(3, ops.get_collection("v")[2].eval())
 
-  def testLegacyInitOp(self):
-    export_dir = self._get_export_dir("test_legacy_init_op")
-    builder = saved_model_builder._SavedModelBuilder(export_dir)
-
-    with self.session(graph=ops.Graph()) as sess:
-      # Add `v1` and `v2` variables to the graph.
-      v1 = variables.VariableV1(1, name="v1")
-      ops.add_to_collection("v", v1)
-      v2 = variables.VariableV1(2, name="v2")
-      ops.add_to_collection("v", v2)
-
-      # Initialize another variable `v3` to 42.
-      v3 = variables.VariableV1(42, name="v3", trainable=False, collections=[])
-      ops.add_to_collection("v", v3)
-
-      # Set up an assignment op to be run as part of the legacy_init_op.
-      assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2))
-      legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op")
-
-      self.evaluate(variables.global_variables_initializer())
-      builder.add_meta_graph_and_variables(
-          sess, ["foo"], legacy_init_op=legacy_init_op)
-
-    # Save the SavedModel to disk.
-    builder.save()
-
-    with self.session(graph=ops.Graph()) as sess:
-      loader.load(sess, ["foo"], export_dir)
-      self.assertEqual(1, ops.get_collection("v")[0].eval())
-      self.assertEqual(2, ops.get_collection("v")[1].eval())
-      # Evaluates to the sum of the first two variables and assigned as part of
-      # the legacy_init_op, following a restore.
-      self.assertEqual(3, ops.get_collection("v")[2].eval())
-
-  def testLegacyInitOpWithNonEmptyCollection(self):
-    export_dir = self._get_export_dir(
-        "test_legacy_init_op_with_non_empty_collection")
-    self._testInitOpsWithNonEmptyCollection(
-        export_dir, constants.LEGACY_INIT_OP_KEY)
-
-  def testMainOpWithNonEmptyCollection(self):
-    export_dir = self._get_export_dir(
-        "test_main_op_with_non_empty_collection")
-    self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY)
-
-  def _testInitOpsWithNonEmptyCollection(self, export_dir, key):
-    builder = saved_model_builder._SavedModelBuilder(export_dir)
-
-    g = ops.Graph()
-    with self.session(graph=g) as sess:
-      # Initialize variable `v1` to 1.
-      v1 = variables.VariableV1(1, name="v1")
-      ops.add_to_collection("v", v1)
-
-      # Initialize another variable `v2` to 42.
-      v2 = variables.VariableV1(42, name="v2", trainable=False, collections=[])
-      ops.add_to_collection("v", v2)
-
-      # Set up an assignment op to be run as part of the init op.
-      assign_v2 = state_ops.assign(v2, v1)
-      init_op = control_flow_ops.group(assign_v2, name="init_op")
-
-      self.evaluate(variables.global_variables_initializer())
-
-      ops.add_to_collection(key, control_flow_ops.no_op())
-      # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
-      # is not empty and we don't support multiple init ops.
-      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
-        builder.add_meta_graph_and_variables(
-            sess, ["foo"], legacy_init_op=init_op)
-      # We shouldn't be able to add as MAIN_OP, either.
-      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
-        builder.add_meta_graph_and_variables(sess, ["foo"], main_op=init_op)
-
   def testTrainOp(self):
     export_dir = self._get_export_dir("test_train_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -906,19 +853,17 @@ class SavedModelTest(SavedModelTestBase):
       train_op = state_ops.assign_add(v1, v2)
 
       self.evaluate(train_op)
-      # TODO(karmel): remove explicit call when in the public method.
-      builder._add_train_op(train_op)
-      builder.add_meta_graph_and_variables(sess, ["foo"])
+      builder.add_meta_graph_and_variables(sess, ["foo"], train_op=train_op)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
-      loader.load(sess, ["foo"], export_dir)
+      meta_graph_def = loader.load(sess, ["foo"], export_dir)
       self.assertEqual(3, ops.get_collection("v")[0].eval())
       self.assertEqual(2, ops.get_collection("v")[1].eval())
       self.assertIsInstance(
-          ops.get_collection(constants.TRAIN_OP_KEY)[0], ops.Tensor)
+          loader_impl.get_train_op(meta_graph_def), ops.Tensor)
 
   def testTrainOpGroup(self):
     export_dir = self._get_export_dir("test_train_op_group")
@@ -935,19 +880,17 @@ class SavedModelTest(SavedModelTestBase):
       train_op = control_flow_ops.group()
 
       self.evaluate(train_op)
-      # TODO(karmel): remove explicit call when in the public method.
-      builder._add_train_op(train_op)
-      builder.add_meta_graph_and_variables(sess, ["foo"])
+      builder.add_meta_graph_and_variables(sess, ["foo"], train_op=train_op)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
-      loader.load(sess, ["foo"], export_dir)
+      meta_graph_def = loader.load(sess, ["foo"], export_dir)
       self.assertEqual(1, ops.get_collection("v")[0].eval())
       self.assertEqual(2, ops.get_collection("v")[1].eval())
       self.assertIsInstance(
-          ops.get_collection(constants.TRAIN_OP_KEY)[0], ops.Operation)
+          loader_impl.get_train_op(meta_graph_def), ops.Operation)
 
   def testTrainOpAfterVariables(self):
     export_dir = self._get_export_dir("test_train_op_after_variables")
@@ -965,17 +908,15 @@ class SavedModelTest(SavedModelTestBase):
 
       train_op = state_ops.assign_add(v1, v2)
       self.evaluate(train_op)
-      # TODO(karmel): remove explicit call when in the public method.
-      builder._add_train_op(train_op)
-      builder.add_meta_graph(["foo"])
+      builder.add_meta_graph(["foo"], train_op=train_op)
 
     # Save the SavedModel to disk.
     builder.save()
 
     with self.session(graph=ops.Graph()) as sess:
-      loader.load(sess, ["foo"], export_dir)
+      meta_graph_def = loader.load(sess, ["foo"], export_dir)
       self.assertIsInstance(
-          ops.get_collection(constants.TRAIN_OP_KEY)[0], ops.Tensor)
+          loader_impl.get_train_op(meta_graph_def), ops.Tensor)
 
     with self.session(graph=ops.Graph()) as sess:
       loader.load(sess, ["pre_foo"], export_dir)
@@ -1288,76 +1229,6 @@ class SavedModelTest(SavedModelTestBase):
       self.assertEqual(
           42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval())
 
-  def testStripDefaultAttrs(self):
-    export_dir = self._get_export_dir("test_strip_default_attrs")
-    builder = saved_model_builder._SavedModelBuilder(export_dir)
-
-    # Add a graph with two float32 variables and a Complex Op composing them
-    # with strip_default_attrs enabled.
-    with session.Session(graph=ops.Graph()) as sess:
-      real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
-      imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
-      math_ops.complex(real_num, imag_num, name="complex")
-      self.evaluate(variables.global_variables_initializer())
-      builder.add_meta_graph_and_variables(
-          sess, ["foo"], strip_default_attrs=True)
-
-    # Add a graph with the same float32 variables and a Complex Op composing
-    # them with strip_default_attrs disabled.
-    with session.Session(graph=ops.Graph()) as sess:
-      real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
-      imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
-      math_ops.complex(real_num, imag_num, name="complex")
-      self.evaluate(variables.global_variables_initializer())
-      builder.add_meta_graph(["bar"], strip_default_attrs=False)
-
-    # Save the SavedModel to disk in text format.
-    builder.save(as_text=True)
-
-    # Loading graph "foo" via the loader must restore the defaults for the
-    # "Complex" node based on the "Complex" OpDef in the Op registry.
-    sess = session.Session(graph=ops.Graph())
-    meta_graph_def = loader.load(sess, ["foo"], export_dir)
-    complex_node = test_util.get_node_def_from_graph("complex",
-                                                     meta_graph_def.graph_def)
-    self.assertIn("T", complex_node.attr)
-    self.assertIn("Tout", complex_node.attr)
-
-    # Load graph "foo" from disk as-is to verify default attrs are stripped.
-    # pylint: disable=protected-access
-    saved_model_pb = loader_impl._parse_saved_model(export_dir)
-    self.assertIsNotNone(saved_model_pb)
-    # pylint: enable=protected-access
-
-    meta_graph_foo_def = None
-    meta_graph_bar_def = None
-    for meta_graph_def in saved_model_pb.meta_graphs:
-      if set(meta_graph_def.meta_info_def.tags) == set(["foo"]):
-        meta_graph_foo_def = meta_graph_def
-      elif set(meta_graph_def.meta_info_def.tags) == set(["bar"]):
-        meta_graph_bar_def = meta_graph_def
-
-    self.assertIsNotNone(meta_graph_foo_def)
-    self.assertIsNotNone(meta_graph_bar_def)
-
-    # "Complex" Op has 2 attributes with defaults:
-    #   o "T"    : float32.   (input type)
-    #   o "Tout" : complex64. (output type)
-
-    # "Complex" Op in graph "foo" shouldn't have attributes "T" and "Tout".
-    # Graph "foo" was saved with strip_default_attrs set to True.
-    node_def = test_util.get_node_def_from_graph("complex",
-                                                 meta_graph_foo_def.graph_def)
-    self.assertNotIn("T", node_def.attr)
-    self.assertNotIn("Tout", node_def.attr)
-
-    # "Complex" Op in graph "bar" must have attributes "T" and "Tout".
-    # Graph "bar" was saved with strip_default_attrs set to False.
-    node_def = test_util.get_node_def_from_graph("complex",
-                                                 meta_graph_bar_def.graph_def)
-    self.assertIn("T", node_def.attr)
-    self.assertIn("Tout", node_def.attr)
-
   # Tests the behavior of loading SavedModels that having missing attrs or attrs
   # with incorrect types.
   def testInconsistentConsumerDefaultAttrs(self):
@@ -1484,6 +1355,149 @@ class SavedModelV1Test(SavedModelTestBase):
           compat.as_bytes("ignored.txt"))
       self.assertFalse(file_io.file_exists(ignored_asset_path))
 
+  def testLegacyInitOpWithNonEmptyCollection(self):
+    export_dir = self._get_export_dir(
+        "test_legacy_init_op_with_non_empty_collection")
+    self._testInitOpsWithNonEmptyCollection(export_dir,
+                                            constants.LEGACY_INIT_OP_KEY)
+
+  def testMainOpWithNonEmptyCollection(self):
+    export_dir = self._get_export_dir("test_main_op_with_non_empty_collection")
+    self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY)
+
+  def _testInitOpsWithNonEmptyCollection(self, export_dir, key):
+    builder = saved_model_builder.SavedModelBuilder(export_dir)
+
+    g = ops.Graph()
+    with self.session(graph=g) as sess:
+      # Initialize variable `v1` to 1.
+      v1 = variables.VariableV1(1, name="v1")
+      ops.add_to_collection("v", v1)
+
+      # Initialize another variable `v2` to 42.
+      v2 = variables.VariableV1(42, name="v2", trainable=False, collections=[])
+      ops.add_to_collection("v", v2)
+
+      # Set up an assignment op to be run as part of the init op.
+      assign_v2 = state_ops.assign(v2, v1)
+      init_op = control_flow_ops.group(assign_v2, name="init_op")
+
+      self.evaluate(variables.global_variables_initializer())
+
+      ops.add_to_collection(key, control_flow_ops.no_op())
+      # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
+      # is not empty and we don't support multiple init ops.
+      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
+        builder.add_meta_graph_and_variables(
+            sess, ["foo"], legacy_init_op=init_op)
+      # We shouldn't be able to add as MAIN_OP, either.
+      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
+        builder.add_meta_graph_and_variables(sess, ["foo"], main_op=init_op)
+
+  def testStripDefaultAttrs(self):
+    export_dir = self._get_export_dir("test_strip_default_attrs")
+    builder = saved_model_builder.SavedModelBuilder(export_dir)
+
+    # Add a graph with two float32 variables and a Complex Op composing them
+    # with strip_default_attrs enabled.
+    with session.Session(graph=ops.Graph()) as sess:
+      real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
+      imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
+      math_ops.complex(real_num, imag_num, name="complex")
+      self.evaluate(variables.global_variables_initializer())
+      builder.add_meta_graph_and_variables(
+          sess, ["foo"], strip_default_attrs=True)
+
+    # Add a graph with the same float32 variables and a Complex Op composing
+    # them with strip_default_attrs disabled.
+    with session.Session(graph=ops.Graph()) as sess:
+      real_num = variables.VariableV1(1.0, dtype=dtypes.float32, name="real")
+      imag_num = variables.VariableV1(2.0, dtype=dtypes.float32, name="imag")
+      math_ops.complex(real_num, imag_num, name="complex")
+      self.evaluate(variables.global_variables_initializer())
+      builder.add_meta_graph(["bar"], strip_default_attrs=False)
+
+    # Save the SavedModel to disk in text format.
+    builder.save(as_text=True)
+
+    # Loading graph "foo" via the loader must restore the defaults for the
+    # "Complex" node based on the "Complex" OpDef in the Op registry.
+    sess = session.Session(graph=ops.Graph())
+    meta_graph_def = loader.load(sess, ["foo"], export_dir)
+    complex_node = test_util.get_node_def_from_graph("complex",
+                                                     meta_graph_def.graph_def)
+    self.assertIn("T", complex_node.attr)
+    self.assertIn("Tout", complex_node.attr)
+
+    # Load graph "foo" from disk as-is to verify default attrs are stripped.
+    # pylint: disable=protected-access
+    saved_model_pb = loader_impl._parse_saved_model(export_dir)
+    self.assertIsNotNone(saved_model_pb)
+    # pylint: enable=protected-access
+
+    meta_graph_foo_def = None
+    meta_graph_bar_def = None
+    for meta_graph_def in saved_model_pb.meta_graphs:
+      if set(meta_graph_def.meta_info_def.tags) == set(["foo"]):
+        meta_graph_foo_def = meta_graph_def
+      elif set(meta_graph_def.meta_info_def.tags) == set(["bar"]):
+        meta_graph_bar_def = meta_graph_def
+
+    self.assertIsNotNone(meta_graph_foo_def)
+    self.assertIsNotNone(meta_graph_bar_def)
+
+    # "Complex" Op has 2 attributes with defaults:
+    #   o "T"    : float32.   (input type)
+    #   o "Tout" : complex64. (output type)
+
+    # "Complex" Op in graph "foo" shouldn't have attributes "T" and "Tout".
+    # Graph "foo" was saved with strip_default_attrs set to True.
+    node_def = test_util.get_node_def_from_graph("complex",
+                                                 meta_graph_foo_def.graph_def)
+    self.assertNotIn("T", node_def.attr)
+    self.assertNotIn("Tout", node_def.attr)
+
+    # "Complex" Op in graph "bar" must have attributes "T" and "Tout".
+    # Graph "bar" was saved with strip_default_attrs set to False.
+    node_def = test_util.get_node_def_from_graph("complex",
+                                                 meta_graph_bar_def.graph_def)
+    self.assertIn("T", node_def.attr)
+    self.assertIn("Tout", node_def.attr)
+
+  def testLegacyInitOp(self):
+    export_dir = self._get_export_dir("test_legacy_init_op")
+    builder = saved_model_builder.SavedModelBuilder(export_dir)
+
+    with self.session(graph=ops.Graph()) as sess:
+      # Add `v1` and `v2` variables to the graph.
+      v1 = variables.VariableV1(1, name="v1")
+      ops.add_to_collection("v", v1)
+      v2 = variables.VariableV1(2, name="v2")
+      ops.add_to_collection("v", v2)
+
+      # Initialize another variable `v3` to 42.
+      v3 = variables.VariableV1(42, name="v3", trainable=False, collections=[])
+      ops.add_to_collection("v", v3)
+
+      # Set up an assignment op to be run as part of the init_op.
+      assign_v3 = state_ops.assign(v3, math_ops.add(v1, v2))
+      legacy_init_op = control_flow_ops.group(assign_v3, name="legacy_init_op")
+
+      self.evaluate(variables.global_variables_initializer())
+      builder.add_meta_graph_and_variables(
+          sess, ["foo"], legacy_init_op=legacy_init_op)
+
+    # Save the SavedModel to disk.
+    builder.save()
+
+    with self.session(graph=ops.Graph()) as sess:
+      loader.load(sess, ["foo"], export_dir)
+      self.assertEqual(1, ops.get_collection("v")[0].eval())
+      self.assertEqual(2, ops.get_collection("v")[1].eval())
+      # Evaluates to the sum of the first two variables and assigned as part of
+      # the legacy_init_op, following a restore.
+      self.assertEqual(3, ops.get_collection("v")[2].eval())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/signature_def_utils.py b/tensorflow/python/saved_model/signature_def_utils.py
index 27d6b70e9d..6a3c0aaf38 100644
--- a/tensorflow/python/saved_model/signature_def_utils.py
+++ b/tensorflow/python/saved_model/signature_def_utils.py
@@ -24,6 +24,8 @@ from __future__ import print_function
 from tensorflow.python.saved_model.signature_def_utils_impl import build_signature_def
 from tensorflow.python.saved_model.signature_def_utils_impl import classification_signature_def
 from tensorflow.python.saved_model.signature_def_utils_impl import is_valid_signature
+from tensorflow.python.saved_model.signature_def_utils_impl import load_op_from_signature_def
+from tensorflow.python.saved_model.signature_def_utils_impl import op_signature_def
 from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def
 from tensorflow.python.saved_model.signature_def_utils_impl import regression_signature_def
 from tensorflow.python.saved_model.signature_def_utils_impl import supervised_eval_signature_def
diff --git a/tensorflow/python/saved_model/signature_def_utils_impl.py b/tensorflow/python/saved_model/signature_def_utils_impl.py
index 6e5e3bc682..f6e6e1d13e 100644
--- a/tensorflow/python/saved_model/signature_def_utils_impl.py
+++ b/tensorflow/python/saved_model/signature_def_utils_impl.py
@@ -21,9 +21,10 @@ from __future__ import print_function
 
 from tensorflow.core.framework import types_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.saved_model import utils
+from tensorflow.python.saved_model import utils_impl as utils
 from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
@@ -349,3 +350,51 @@ def _is_valid_classification_signature(signature_def):
     return False
 
   return True
+
+
+def op_signature_def(op, key):
+  """Creates a signature def with the output pointing to an op.
+
+  Note that op isn't strictly enforced to be an Op object, and may be a Tensor.
+  It is recommended to use the build_signature_def() function for Tensors.
+
+  Args:
+    op: An Op (or possibly Tensor).
+    key: Key to graph element in the SignatureDef outputs.
+
+  Returns:
+    A SignatureDef with a single output pointing to the op.
+  """
+  # Use build_tensor_info_from_op, which creates a TensorInfo from the element's
+  # name.
+  return build_signature_def(outputs={key: utils.build_tensor_info_from_op(op)})
+
+
+def load_op_from_signature_def(signature_def, key, import_scope=None):
+  """Load an Op from a SignatureDef created by op_signature_def().
+
+  Args:
+    signature_def: a SignatureDef proto
+    key: string key to op in the SignatureDef outputs.
+    import_scope: Scope used to import the op
+
+  Returns:
+    Op (or possibly Tensor) in the graph with the same name as saved in the
+      SignatureDef.
+
+  Raises:
+    NotFoundError: If the op could not be found in the graph.
+  """
+  tensor_info = signature_def.outputs[key]
+  try:
+    # The init and train ops are not strictly enforced to be operations, so
+    # retrieve any graph element (can be either op or tensor).
+    return utils.get_element_from_tensor_info(
+        tensor_info, import_scope=import_scope)
+  except KeyError:
+    raise errors.NotFoundError(
+        None, None,
+        'The {0} could not be found in the graph. Please make sure the '
+        'SavedModel was created by the internal _SavedModelBuilder. If you '
+        'are using the public API, please make sure the SignatureDef in the '
+        'SavedModel does not contain the key "{0}".'.format(key))
diff --git a/tensorflow/python/saved_model/signature_def_utils_test.py b/tensorflow/python/saved_model/signature_def_utils_test.py
index 18c55d8d33..cc0fd8ce05 100644
--- a/tensorflow/python/saved_model/signature_def_utils_test.py
+++ b/tensorflow/python/saved_model/signature_def_utils_test.py
@@ -23,6 +23,7 @@ from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils_impl
@@ -413,5 +414,22 @@ class SignatureDefUtilsTest(test.TestCase):
         {},
         signature_constants.PREDICT_METHOD_NAME)
 
+  def testOpSignatureDef(self):
+    key = "adding_1_and_2_key"
+    add_op = math_ops.add(1, 2, name="adding_1_and_2")
+    signature_def = signature_def_utils_impl.op_signature_def(add_op, key)
+    self.assertIn(key, signature_def.outputs)
+    self.assertEqual(add_op.name, signature_def.outputs[key].name)
+
+  def testLoadOpFromSignatureDef(self):
+    key = "adding_1_and_2_key"
+    add_op = math_ops.add(1, 2, name="adding_1_and_2")
+    signature_def = signature_def_utils_impl.op_signature_def(add_op, key)
+
+    self.assertEqual(
+        add_op,
+        signature_def_utils_impl.load_op_from_signature_def(signature_def, key))
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py
index 1066741976..5caabe59fe 100644
--- a/tensorflow/python/saved_model/utils_impl.py
+++ b/tensorflow/python/saved_model/utils_impl.py
@@ -141,6 +141,27 @@ def get_tensor_from_tensor_info(tensor_info, graph=None, import_scope=None):
     raise ValueError("Invalid TensorInfo.encoding: %s" % encoding)
 
 
+def get_element_from_tensor_info(tensor_info, graph=None, import_scope=None):
+  """Returns the element in the graph described by a TensorInfo proto.
+
+  Args:
+    tensor_info: A TensorInfo proto describing an Op or Tensor by name.
+    graph: The tf.Graph in which tensors are looked up. If None, the current
+      default graph is used.
+    import_scope: If not None, names in `tensor_info` are prefixed with this
+      string before lookup.
+
+  Returns:
+    Op or tensor in `graph` described by `tensor_info`.
+
+  Raises:
+    KeyError: If `tensor_info` does not correspond to an op or tensor in `graph`
+  """
+  graph = graph or ops.get_default_graph()
+  return graph.as_graph_element(
+      ops.prepend_name_scope(tensor_info.name, import_scope=import_scope))
+
+
 # Path helpers.
 
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
index d946c666c2..63bebb20bc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.saved_model.pbtxt
@@ -32,10 +32,6 @@ tf_module {
     name: "GPU"
     mtype: "<type \'str\'>"
   }
-  member {
-    name: "MAIN_OP_KEY"
-    mtype: "<type \'str\'>"
-  }
   member {
     name: "PREDICT_INPUTS"
     mtype: "<type \'str\'>"
-- 
GitLab


From 1a7e7e3009ae0f66c3338d9099c263c58ce77139 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 17:57:39 -0800
Subject: [PATCH 0991/1554] nn.erosion2d changes for TF 2.0 API

* `kernel` is renamed to `filters`
* `rates` is renamed to dilations
* `data_format` is introduced, must be set to NCHW or an error is raised

PiperOrigin-RevId: 223269242
---
 tensorflow/python/ops/nn_ops.py               | 71 ++++++++++++++++++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  6 ++
 3 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 2ffe381216..74e88b7653 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3597,7 +3597,7 @@ def _calc_dilation2d_flops(graph, node):
   return ops.OpStats("flops", (output_count * filter_height * filter_width * 2))
 
 
-@tf_export("nn.erosion2d")
+@tf_export(v1=["nn.erosion2d"])
 def erosion2d(value, kernel, strides, rates, padding, name=None):
   """Computes the grayscale erosion of 4-D `value` and 3-D `kernel` tensors.
 
@@ -3656,6 +3656,75 @@ def erosion2d(value, kernel, strides, rates, padding, name=None):
             name=name))
 
 
+@tf_export("nn.erosion2d", v1=[])
+def erosion2d_v2(value,
+                 filters,
+                 strides,
+                 padding,
+                 data_format,
+                 dilations,
+                 name=None):
+  """Computes the grayscale erosion of 4-D `value` and 3-D `filters` tensors.
+
+  The `value` tensor has shape `[batch, in_height, in_width, depth]` and the
+  `filters` tensor has shape `[filters_height, filters_width, depth]`, i.e.,
+  each input channel is processed independently of the others with its own
+  structuring function. The `output` tensor has shape
+  `[batch, out_height, out_width, depth]`. The spatial dimensions of the
+  output tensor depend on the `padding` algorithm. We currently only support the
+  default "NHWC" `data_format`.
+
+  In detail, the grayscale morphological 2-D erosion is given by:
+
+      output[b, y, x, c] =
+         min_{dy, dx} value[b,
+                            strides[1] * y - dilations[1] * dy,
+                            strides[2] * x - dilations[2] * dx,
+                            c] -
+                      filters[dy, dx, c]
+
+  Duality: The erosion of `value` by the `filters` is equal to the negation of
+  the dilation of `-value` by the reflected `filters`.
+
+  Args:
+    value: A `Tensor`. 4-D with shape `[batch, in_height, in_width, depth]`.
+    filters: A `Tensor`. Must have the same type as `value`.
+      3-D with shape `[filters_height, filters_width, depth]`.
+    strides: A list of `ints` that has length `>= 4`.
+      1-D of length 4. The stride of the sliding window for each dimension of
+      the input tensor. Must be: `[1, stride_height, stride_width, 1]`.
+    padding: A `string` from: `"SAME", "VALID"`.
+      The type of padding algorithm to use.
+    data_format: A `string`, only `"NHWC"` is currently supported.
+    dilations: A list of `ints` that has length `>= 4`.
+      1-D of length 4. The input stride for atrous morphological dilation.
+      Must be: `[1, rate_height, rate_width, 1]`.
+    name: A name for the operation (optional). If not specified "erosion2d"
+      is used.
+
+  Returns:
+    A `Tensor`. Has the same type as `value`.
+    4-D with shape `[batch, out_height, out_width, depth]`.
+
+  Raises:
+    ValueError: If the `value` depth does not match `filters`' shape, or if
+      padding is other than `'VALID'` or `'SAME'`.
+  """
+  if data_format != "NHWC":
+    raise ValueError("Data formats other than NHWC are not yet supported")
+
+  with ops.name_scope(name, "erosion2d", [value, filters]) as name:
+    # Reduce erosion to dilation by duality.
+    return math_ops.negative(
+        gen_nn_ops.dilation2d(
+            input=math_ops.negative(value),
+            filter=array_ops.reverse_v2(filters, [0, 1]),
+            strides=strides,
+            rates=dilations,
+            padding=padding,
+            name=name))
+
+
 @tf_export("math.in_top_k", "nn.in_top_k")
 def in_top_k(predictions, targets, k, name=None):
   r"""Says whether the targets are in the top `K` predictions.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index b4d0f95bc3..4779448760 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -142,7 +142,7 @@ tf_module {
   }
   member_method {
     name: "erosion2d"
-    argspec: "args=[\'value\', \'kernel\', \'strides\', \'rates\', \'padding\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'value\', \'filters\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "fixed_unigram_candidate_sampler"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9e010cd7b9..d75e52bd0d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -64,6 +64,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "keep_dims": "keepdims",
             "reduction_indices": "axis",
         },
+        "tf.nn.erosion2d": {
+            "kernel": "filters",
+            "rates": "dilations",
+        },
         "tf.math.l2_normalize": {
             "dim": "axis",
         },
@@ -699,6 +703,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.conv2d_backprop_input":
         "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" "
         "was renamed to \"filters\"",
+        "tf.nn.erosion2d":
+        "WARNING: <function name> now requires a data_format argument",
         "tf.zeros_like": tf_01s_like_no_optimize_comment,
         "tf.ones_like": tf_01s_like_no_optimize_comment,
     }
-- 
GitLab


From f7b3f57f11c4a4661a456e484310b3e131ddb7f1 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 28 Nov 2018 18:02:09 -0800
Subject: [PATCH 0992/1554] Internal change.

PiperOrigin-RevId: 223269764
---
 .../ci_build/windows/cpu/pip/build_tf_windows.sh     | 12 +++++++++---
 .../ci_build/windows/gpu/pip/build_tf_windows.sh     | 11 +++++++++--
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 872072a056..32438260b4 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -58,6 +58,7 @@ PY_TEST_DIR="py_test_dir"
 SKIP_TEST=0
 RELEASE_BUILD=0
 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
+EXTRA_BUILD_FLAGS=${ADDITIONAL_BUILD_ARGS:-}
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
@@ -88,7 +89,11 @@ fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   python tensorflow/tools/ci_build/update_version.py --nightly
-  EXTRA_PIP_FLAG="--nightly_flag"
+  if [ -z ${EXTRA_PIP_FLAGS} ]; then
+    EXTRA_PIP_FLAGS="--nightly_flag"
+  else
+    EXTRA_PIP_FLAGS="${EXTRA_PIP_FLAGS} --nightly_flag"
+  fi
 fi
 
 # Enable short object file path to avoid long path issue on Windows.
@@ -100,7 +105,8 @@ fi
 
 run_configure_for_cpu_build
 
-bazel build --announce_rc --config=opt tensorflow/tools/pip_package:build_pip_package || exit $?
+bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS} \
+  tensorflow/tools/pip_package:build_pip_package || exit $?
 
 if [[ "$SKIP_TEST" == 1 ]]; then
   exit 0
@@ -109,7 +115,7 @@ fi
 # Create a python test directory to avoid package name conflict
 create_python_test_dir "${PY_TEST_DIR}"
 
-./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" "${EXTRA_PIP_FLAG}"
+./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" "${EXTRA_PIP_FLAGS}"
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   exit 0
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 6178d7794d..6a701fc5a1 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -58,6 +58,7 @@ PY_TEST_DIR="py_test_dir"
 SKIP_TEST=0
 RELEASE_BUILD=0
 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
+EXTRA_BUILD_FLAGS=${ADDITIONAL_BUILD_ARGS:-}
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
@@ -88,7 +89,11 @@ fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   python tensorflow/tools/ci_build/update_version.py --nightly
-  EXTRA_PIP_FLAG="--nightly_flag"
+  if [ -z ${EXTRA_PIP_FLAGS} ]; then
+    EXTRA_PIP_FLAGS="--nightly_flag"
+  else
+    EXTRA_PIP_FLAGS="${EXTRA_PIP_FLAGS} --nightly_flag"
+  fi
 fi
 
 # Enable short object file path to avoid long path issue on Windows.
@@ -104,6 +109,7 @@ fi
 run_configure_for_gpu_build
 
 bazel build --announce_rc --config=opt --define=no_tensorflow_py_deps=true \
+  ${EXTRA_BUILD_FLAGS} \
   tensorflow/tools/pip_package:build_pip_package || exit $?
 
 if [[ "$SKIP_TEST" == 1 ]]; then
@@ -113,7 +119,8 @@ fi
 # Create a python test directory to avoid package name conflict
 create_python_test_dir "${PY_TEST_DIR}"
 
-./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" --gpu "${EXTRA_PIP_FLAG}"
+./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" \
+  --gpu "${EXTRA_PIP_FLAGS}"
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   exit 0
-- 
GitLab


From e94b2367284b35a20fd5ab5d3d78858a629c74da Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Wed, 28 Nov 2018 18:07:18 -0800
Subject: [PATCH 0993/1554] Make some feature_column_v2 tests work without
 sessions

This allows the tests to run in v2 mode although quite a few
use graph mode when not necessary.

This reduces the number of failing tests from 90 to 43.

PiperOrigin-RevId: 223270643
---
 .../feature_column/feature_column_v2_test.py  | 2343 +++++++++--------
 1 file changed, 1314 insertions(+), 1029 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 23131e22ed..3147754bee 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -228,11 +228,11 @@ class LazyColumnTest(test.TestCase):
                     dense_shape=[0],
                     values=np.array([]))
         })
-    with self.cached_session():
-      spv = transformation_cache.get('a', None).eval()
-      self.assertAllEqual(np.array([0, 1], dtype=np.int64), spv.dense_shape)
-      self.assertAllEqual(
-          np.reshape(np.array([], dtype=np.int64), (0, 2)), spv.indices)
+
+    spv = self.evaluate(transformation_cache.get('a', None))
+    self.assertAllEqual(np.array([0, 1], dtype=np.int64), spv.dense_shape)
+    self.assertAllEqual(
+        np.reshape(np.array([], dtype=np.int64), (0, 2)), spv.indices)
 
 
 class NumericColumnTest(test.TestCase):
@@ -317,40 +317,44 @@ class NumericColumnTest(test.TestCase):
 
   def test_parse_example_no_default_value(self):
     price = fc.numeric_column('price', shape=[2])
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'price':
-                feature_pb2.Feature(float_list=feature_pb2.FloatList(
-                    value=[20., 110.]))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'price':
+                    feature_pb2.Feature(
+                        float_list=feature_pb2.FloatList(value=[20., 110.]))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([price]))
     self.assertIn('price', features)
-    with self.cached_session():
-      self.assertAllEqual([[20., 110.]], features['price'].eval())
+
+    self.assertAllEqual([[20., 110.]], self.evaluate(features['price']))
 
   def test_parse_example_with_default_value(self):
     price = fc.numeric_column('price', shape=[2], default_value=11.)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'price':
-                feature_pb2.Feature(float_list=feature_pb2.FloatList(
-                    value=[20., 110.]))
-        }))
-    no_data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'something_else':
-                feature_pb2.Feature(float_list=feature_pb2.FloatList(
-                    value=[20., 110.]))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'price':
+                    feature_pb2.Feature(
+                        float_list=feature_pb2.FloatList(value=[20., 110.]))
+            }))
+    no_data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'something_else':
+                    feature_pb2.Feature(
+                        float_list=feature_pb2.FloatList(value=[20., 110.]))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString(),
                     no_data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([price]))
     self.assertIn('price', features)
-    with self.cached_session():
-      self.assertAllEqual([[20., 110.], [11., 11.]], features['price'].eval())
+
+    self.assertAllEqual([[20., 110.], [11., 11.]],
+                        self.evaluate(features['price']))
 
   def test_normalizer_fn_must_be_callable(self):
     with self.assertRaisesRegexp(TypeError, 'must be a callable'):
@@ -365,8 +369,8 @@ class NumericColumnTest(test.TestCase):
     output = fc._transform_features_v2({
         'price': [[1., 2.], [5., 6.]]
     }, [price], None)
-    with self.cached_session():
-      self.assertAllEqual([[3., 4.], [7., 8.]], output[price].eval())
+
+    self.assertAllEqual([[3., 4.], [7., 8.]], self.evaluate(output[price]))
 
   def test_get_dense_tensor(self):
 
@@ -471,17 +475,17 @@ class BucketizedColumnTest(test.TestCase):
 
   def test_invalid_boundaries(self):
     a = fc.numeric_column('aaa')
-    with self.assertRaisesRegexp(
-        ValueError, 'boundaries must be a sorted list'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'boundaries must be a sorted list'):
       fc.bucketized_column(a, boundaries=None)
-    with self.assertRaisesRegexp(
-        ValueError, 'boundaries must be a sorted list'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'boundaries must be a sorted list'):
       fc.bucketized_column(a, boundaries=1.)
-    with self.assertRaisesRegexp(
-        ValueError, 'boundaries must be a sorted list'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'boundaries must be a sorted list'):
       fc.bucketized_column(a, boundaries=[1, 0])
-    with self.assertRaisesRegexp(
-        ValueError, 'boundaries must be a sorted list'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'boundaries must be a sorted list'):
       fc.bucketized_column(a, boundaries=[1, 1])
 
   def test_name(self):
@@ -518,18 +522,19 @@ class BucketizedColumnTest(test.TestCase):
   def test_parse_example(self):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'price':
-                feature_pb2.Feature(float_list=feature_pb2.FloatList(
-                    value=[20., 110.]))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'price':
+                    feature_pb2.Feature(
+                        float_list=feature_pb2.FloatList(value=[20., 110.]))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([bucketized_price]))
     self.assertIn('price', features)
-    with self.cached_session():
-      self.assertAllEqual([[20., 110.]], features['price'].eval())
+
+    self.assertAllEqual([[20., 110.]], self.evaluate(features['price']))
 
   def test_transform_feature(self):
     price = fc.numeric_column('price', shape=[2])
@@ -538,9 +543,12 @@ class BucketizedColumnTest(test.TestCase):
       transformed_tensor = fc._transform_features_v2({
           'price': [[-1., 1.], [5., 6.]]
       }, [bucketized_price], None)
-      with _initialized_session():
-        self.assertAllEqual([[0, 1], [3, 4]],
-                            transformed_tensor[bucketized_price].eval())
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllEqual([[0, 1], [3, 4]],
+                          self.evaluate(transformed_tensor[bucketized_price]))
 
   def test_get_dense_tensor_one_input_value(self):
     """Tests _get_dense_tensor() for input with shape=[1]."""
@@ -550,14 +558,17 @@ class BucketizedColumnTest(test.TestCase):
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1.], [1.], [5.], [6.]]
       })
-      with _initialized_session():
-        bucketized_price_tensor = bucketized_price.get_dense_tensor(
-            transformation_cache, None)
-        self.assertAllClose(
-            # One-hot tensor.
-            [[[1., 0., 0., 0., 0.]], [[0., 1., 0., 0., 0.]],
-             [[0., 0., 0., 1., 0.]], [[0., 0., 0., 0., 1.]]],
-            self.evaluate(bucketized_price_tensor))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      bucketized_price_tensor = bucketized_price.get_dense_tensor(
+          transformation_cache, None)
+      self.assertAllClose(
+          # One-hot tensor.
+          [[[1., 0., 0., 0., 0.]], [[0., 1., 0., 0., 0.]],
+           [[0., 0., 0., 1., 0.]], [[0., 0., 0., 0., 1.]]],
+          self.evaluate(bucketized_price_tensor))
 
   def test_get_dense_tensor_two_input_values(self):
     """Tests _get_dense_tensor() for input with shape=[2]."""
@@ -567,14 +578,17 @@ class BucketizedColumnTest(test.TestCase):
       transformation_cache = fc.FeatureTransformationCache({
           'price': [[-1., 1.], [5., 6.]]
       })
-      with _initialized_session():
-        bucketized_price_tensor = bucketized_price.get_dense_tensor(
-            transformation_cache, None)
-        self.assertAllClose(
-            # One-hot tensor.
-            [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]],
-             [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]],
-            self.evaluate(bucketized_price_tensor))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      bucketized_price_tensor = bucketized_price.get_dense_tensor(
+          transformation_cache, None)
+      self.assertAllClose(
+          # One-hot tensor.
+          [[[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.]],
+           [[0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]]],
+          self.evaluate(bucketized_price_tensor))
 
   def test_get_sparse_tensors_one_input_value(self):
     """Tests _get_sparse_tensors() for input with shape=[1]."""
@@ -589,8 +603,8 @@ class BucketizedColumnTest(test.TestCase):
             transformation_cache, None)
         self.assertIsNone(id_weight_pair.weight_tensor)
         id_tensor_value = sess.run(id_weight_pair.id_tensor)
-        self.assertAllEqual(
-            [[0, 0], [1, 0], [2, 0], [3, 0]], id_tensor_value.indices)
+        self.assertAllEqual([[0, 0], [1, 0], [2, 0], [3, 0]],
+                            id_tensor_value.indices)
         self.assertAllEqual([0, 1, 3, 4], id_tensor_value.values)
         self.assertAllEqual([4, 1], id_tensor_value.dense_shape)
 
@@ -607,8 +621,8 @@ class BucketizedColumnTest(test.TestCase):
             transformation_cache, None)
         self.assertIsNone(id_weight_pair.weight_tensor)
         id_tensor_value = sess.run(id_weight_pair.id_tensor)
-        self.assertAllEqual(
-            [[0, 0], [0, 1], [1, 0], [1, 1]], id_tensor_value.indices)
+        self.assertAllEqual([[0, 0], [0, 1], [1, 0], [1, 1]],
+                            id_tensor_value.indices)
         # Values 0-4 correspond to the first column of the input price.
         # Values 5-9 correspond to the second column of the input price.
         self.assertAllEqual([0, 6, 3, 9], id_tensor_value.values)
@@ -649,8 +663,8 @@ class BucketizedColumnTest(test.TestCase):
                             self.evaluate(bucketized_price_var))
         self.assertAllClose([[0.], [0.], [0.], [0.]],
                             self.evaluate(predictions))
-        sess.run(bucketized_price_var.assign(
-            [[10.], [20.], [30.], [40.], [50.]]))
+        sess.run(
+            bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]]))
         # price -1. is in the 0th bucket, whose weight is 10.
         # price 1. is in the 1st bucket, whose weight is 20.
         # price 5. is in the 3rd bucket, whose weight is 40.
@@ -677,9 +691,9 @@ class BucketizedColumnTest(test.TestCase):
             [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]],
             self.evaluate(bucketized_price_var))
         self.assertAllClose([[0.], [0.]], self.evaluate(predictions))
-        sess.run(bucketized_price_var.assign(
-            [[10.], [20.], [30.], [40.], [50.],
-             [60.], [70.], [80.], [90.], [100.]]))
+        sess.run(
+            bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.],
+                                         [60.], [70.], [80.], [90.], [100.]]))
         # 1st example:
         #   price -1. is in the 0th bucket, whose weight is 10.
         #   price 1. is in the 6th bucket, whose weight is 70.
@@ -856,24 +870,25 @@ class HashedCategoricalColumnTest(test.TestCase):
 
   def test_parse_example(self):
     a = fc.categorical_column_with_hash_bucket('aaa', 10)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer']))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer']))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
   def test_strings_should_be_hashed(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
@@ -887,12 +902,14 @@ class HashedCategoricalColumnTest(test.TestCase):
     output = outputs[hashed_sparse]
     # Check exact hashed output. If hashing changes this test will break.
     expected_values = [6, 4, 1]
-    with self.cached_session():
-      self.assertEqual(dtypes.int64, output.values.dtype)
-      self.assertAllEqual(expected_values, output.values.eval())
-      self.assertAllEqual(wire_tensor.indices.eval(), output.indices.eval())
-      self.assertAllEqual(wire_tensor.dense_shape.eval(),
-                          output.dense_shape.eval())
+
+    self.assertEqual(dtypes.int64, output.values.dtype)
+    self.assertAllEqual(expected_values, self.evaluate(output.values))
+    self.assertAllEqual(
+        self.evaluate(wire_tensor.indices), self.evaluate(output.indices))
+    self.assertAllEqual(
+        self.evaluate(wire_tensor.dense_shape),
+        self.evaluate(output.dense_shape))
 
   def test_tensor_dtype_should_be_string_or_integer(self):
     string_fc = fc.categorical_column_with_hash_bucket(
@@ -902,17 +919,11 @@ class HashedCategoricalColumnTest(test.TestCase):
     float_fc = fc.categorical_column_with_hash_bucket(
         'a_float', 10, dtype=dtypes.string)
     int_tensor = sparse_tensor.SparseTensor(
-        values=[101],
-        indices=[[0, 0]],
-        dense_shape=[1, 1])
+        values=[101], indices=[[0, 0]], dense_shape=[1, 1])
     string_tensor = sparse_tensor.SparseTensor(
-        values=['101'],
-        indices=[[0, 0]],
-        dense_shape=[1, 1])
+        values=['101'], indices=[[0, 0]], dense_shape=[1, 1])
     float_tensor = sparse_tensor.SparseTensor(
-        values=[101.],
-        indices=[[0, 0]],
-        dense_shape=[1, 1])
+        values=[101.], indices=[[0, 0]], dense_shape=[1, 1])
     transformation_cache = fc.FeatureTransformationCache({
         'a_int': int_tensor,
         'a_string': string_tensor,
@@ -943,8 +954,8 @@ class HashedCategoricalColumnTest(test.TestCase):
     output = transformation_cache.get(hashed_sparse, None)
     # Check exact hashed output. If hashing changes this test will break.
     expected_values = [3, 7, 5]
-    with self.cached_session():
-      self.assertAllEqual(expected_values, output.values.eval())
+
+    self.assertAllEqual(expected_values, self.evaluate(output.values))
 
   def test_int32_64_is_compatible(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket(
@@ -957,8 +968,8 @@ class HashedCategoricalColumnTest(test.TestCase):
     output = transformation_cache.get(hashed_sparse, None)
     # Check exact hashed output. If hashing changes this test will break.
     expected_values = [3, 7, 5]
-    with self.cached_session():
-      self.assertAllEqual(expected_values, output.values.eval())
+
+    self.assertAllEqual(expected_values, self.evaluate(output.values))
 
   def test_get_sparse_tensors(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
@@ -999,15 +1010,17 @@ class HashedCategoricalColumnTest(test.TestCase):
                   dense_shape=(2, 2))
       })
       wire_var, bias = model.variables
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
-                            self.evaluate(wire_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
-        # 'marlo' -> 3: wire_var[3] = 4
-        # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
-        self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,))))
+      # 'marlo' -> 3: wire_var[3] = 4
+      # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
+      self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
@@ -1022,15 +1035,17 @@ class HashedCategoricalColumnTest(test.TestCase):
       }, (wire_column,))
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
-                            self.evaluate(wire_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
-        # 'marlo' -> 3: wire_var[3] = 4
-        # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
-        self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,))))
+      # 'marlo' -> 3: wire_var[3] = 4
+      # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
+      self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
@@ -1050,13 +1065,13 @@ class HashedCategoricalColumnTest(test.TestCase):
 class CrossedColumnTest(test.TestCase):
 
   def test_keys_empty(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'keys must be a list with length > 1'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'keys must be a list with length > 1'):
       fc.crossed_column([], 10)
 
   def test_keys_length_one(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'keys must be a list with length > 1'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'keys must be a list with length > 1'):
       fc.crossed_column(['a'], 10)
 
   def test_key_type_unsupported(self):
@@ -1069,18 +1084,15 @@ class CrossedColumnTest(test.TestCase):
           ['a', fc.categorical_column_with_hash_bucket('c', 10)], 10)
 
   def test_hash_bucket_size_negative(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'hash_bucket_size must be > 1'):
+    with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be > 1'):
       fc.crossed_column(['a', 'c'], -1)
 
   def test_hash_bucket_size_zero(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'hash_bucket_size must be > 1'):
+    with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be > 1'):
       fc.crossed_column(['a', 'c'], 0)
 
   def test_hash_bucket_size_none(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'hash_bucket_size must be > 1'):
+    with self.assertRaisesRegexp(ValueError, 'hash_bucket_size must be > 1'):
       fc.crossed_column(['a', 'c'], None)
 
   def test_name(self):
@@ -1142,7 +1154,10 @@ class CrossedColumnTest(test.TestCase):
     crossed1 = fc.crossed_column(['d1', 'd2'], 10)
     crossed2 = fc.crossed_column([b, 'c', crossed1], 15, hash_key=5)
     crossed2_copy = copy.deepcopy(crossed2)
-    self.assertEqual('a_bucketized_X_c_X_d1_X_d2', crossed2_copy.name,)
+    self.assertEqual(
+        'a_bucketized_X_c_X_d1_X_d2',
+        crossed2_copy.name,
+    )
     self.assertEqual(15, crossed2_copy.hash_bucket_size)
     self.assertEqual(5, crossed2_copy.hash_key)
 
@@ -1150,27 +1165,30 @@ class CrossedColumnTest(test.TestCase):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
     price_cross_wire = fc.crossed_column([bucketized_price, 'wire'], 10)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'price':
-                feature_pb2.Feature(float_list=feature_pb2.FloatList(
-                    value=[20., 110.])),
-            'wire':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer'])),
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'price':
+                    feature_pb2.Feature(
+                        float_list=feature_pb2.FloatList(value=[20., 110.])),
+                'wire':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer'])),
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([price_cross_wire]))
     self.assertIn('price', features)
     self.assertIn('wire', features)
-    with self.cached_session():
-      self.assertAllEqual([[20., 110.]], features['price'].eval())
-      wire_sparse = features['wire']
-      self.assertAllEqual([[0, 0], [0, 1]], wire_sparse.indices.eval())
-      # Use byte constants to pass the open-source test.
-      self.assertAllEqual([b'omar', b'stringer'], wire_sparse.values.eval())
-      self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval())
+
+    self.assertAllEqual([[20., 110.]], self.evaluate(features['price']))
+    wire_sparse = features['wire']
+    self.assertAllEqual([[0, 0], [0, 1]], self.evaluate(wire_sparse.indices))
+    # Use byte constants to pass the open-source test.
+    self.assertAllEqual([b'omar', b'stringer'],
+                        self.evaluate(wire_sparse.values))
+    self.assertAllEqual([1, 2], self.evaluate(wire_sparse.dense_shape))
 
   def test_transform_feature(self):
     price = fc.numeric_column('price', shape=[2])
@@ -1179,21 +1197,22 @@ class CrossedColumnTest(test.TestCase):
     price_cross_wire = fc.crossed_column([bucketized_price, 'wire'],
                                          hash_bucket_size)
     features = {
-        'price': constant_op.constant([[1., 2.], [5., 6.]]),
-        'wire': sparse_tensor.SparseTensor(
-            values=['omar', 'stringer', 'marlo'],
-            indices=[[0, 0], [1, 0], [1, 1]],
-            dense_shape=[2, 2]),
+        'price':
+            constant_op.constant([[1., 2.], [5., 6.]]),
+        'wire':
+            sparse_tensor.SparseTensor(
+                values=['omar', 'stringer', 'marlo'],
+                indices=[[0, 0], [1, 0], [1, 1]],
+                dense_shape=[2, 2]),
     }
     outputs = fc._transform_features_v2(features, [price_cross_wire], None)
     output = outputs[price_cross_wire]
-    with self.cached_session() as sess:
-      output_val = self.evaluate(output)
-      self.assertAllEqual(
-          [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices)
-      for val in output_val.values:
-        self.assertIn(val, list(range(hash_bucket_size)))
-      self.assertAllEqual([2, 4], output_val.dense_shape)
+    output_val = self.evaluate(output)
+    self.assertAllEqual([[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]],
+                        output_val.indices)
+    for val in output_val.values:
+      self.assertIn(val, list(range(hash_bucket_size)))
+    self.assertAllEqual([2, 4], output_val.dense_shape)
 
   def test_get_sparse_tensors(self):
     a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
@@ -1221,19 +1240,21 @@ class CrossedColumnTest(test.TestCase):
                   dense_shape=(2, 2)),
       })
       id_weight_pair = crossed2.get_sparse_tensors(transformation_cache, None)
-      with _initialized_session():
-        id_tensor_eval = id_weight_pair.id_tensor.eval()
-        self.assertAllEqual(
-            ((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5),
-             (1, 6), (1, 7), (1, 8), (1, 9), (1, 10), (1, 11), (1, 12), (1, 13),
-             (1, 14), (1, 15)),
-            id_tensor_eval.indices)
-        # Check exact hashed output. If hashing changes this test will break.
-        # All values are within [0, hash_bucket_size).
-        expected_values = (
-            6, 14, 0, 13, 8, 8, 10, 12, 2, 0, 1, 9, 8, 12, 2, 0, 10, 11)
-        self.assertAllEqual(expected_values, id_tensor_eval.values)
-        self.assertAllEqual((2, 16), id_tensor_eval.dense_shape)
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      id_tensor_eval = self.evaluate(id_weight_pair.id_tensor)
+      self.assertAllEqual(
+          ((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5),
+           (1, 6), (1, 7), (1, 8), (1, 9), (1, 10), (1, 11), (1, 12), (1, 13),
+           (1, 14), (1, 15)), id_tensor_eval.indices)
+      # Check exact hashed output. If hashing changes this test will break.
+      # All values are within [0, hash_bucket_size).
+      expected_values = (6, 14, 0, 13, 8, 8, 10, 12, 2, 0, 1, 9, 8, 12, 2, 0,
+                         10, 11)
+      self.assertAllEqual(expected_values, id_tensor_eval.values)
+      self.assertAllEqual((2, 16), id_tensor_eval.dense_shape)
 
   def test_get_sparse_tensors_simple(self):
     """Same as test_get_sparse_tensors, but with simpler values."""
@@ -1251,16 +1272,18 @@ class CrossedColumnTest(test.TestCase):
                   dense_shape=(2, 2)),
       })
       id_weight_pair = crossed.get_sparse_tensors(transformation_cache, None)
-      with _initialized_session():
-        id_tensor_eval = id_weight_pair.id_tensor.eval()
-        self.assertAllEqual(
-            ((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3)),
-            id_tensor_eval.indices)
-        # Check exact hashed output. If hashing changes this test will break.
-        # All values are within [0, hash_bucket_size).
-        expected_values = (1, 0, 1, 3, 4, 2)
-        self.assertAllEqual(expected_values, id_tensor_eval.values)
-        self.assertAllEqual((2, 4), id_tensor_eval.dense_shape)
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      id_tensor_eval = self.evaluate(id_weight_pair.id_tensor)
+      self.assertAllEqual(((0, 0), (0, 1), (1, 0), (1, 1), (1, 2), (1, 3)),
+                          id_tensor_eval.indices)
+      # Check exact hashed output. If hashing changes this test will break.
+      # All values are within [0, hash_bucket_size).
+      expected_values = (1, 0, 1, 3, 4, 2)
+      self.assertAllEqual(expected_values, id_tensor_eval.values)
+      self.assertAllEqual((2, 4), id_tensor_eval.dense_shape)
 
   def test_linear_model(self):
     """Tests linear_model.
@@ -1310,10 +1333,11 @@ class CrossedColumnTest(test.TestCase):
       @property
       def parse_example_spec(self):
         return {
-            self.name: parsing_ops.VarLenFeature(dtypes.int32),
-            '{}_weights'.format(self.name): parsing_ops.VarLenFeature(
-                dtypes.float32),
-            }
+            self.name:
+                parsing_ops.VarLenFeature(dtypes.int32),
+            '{}_weights'.format(self.name):
+                parsing_ops.VarLenFeature(dtypes.float32),
+        }
 
       @property
       def num_buckets(self):
@@ -1537,7 +1561,6 @@ class CrossedColumnTest(test.TestCase):
     self.assertIs(b, new_crossed.keys[0])
 
 
-
 class LinearModelTest(test.TestCase):
 
   def test_raises_if_empty_feature_columns(self):
@@ -1692,8 +1715,9 @@ class LinearModelTest(test.TestCase):
       predictions = model(features)
       dense_and_sparse_column_var, bias = model.variables
       with _initialized_session() as sess:
-        sess.run(dense_and_sparse_column_var.assign(
-            [[10.], [100.], [1000.], [10000.]]))
+        sess.run(
+            dense_and_sparse_column_var.assign([[10.], [100.], [1000.],
+                                                [10000.]]))
         sess.run(bias.assign([5.]))
         self.assertAllClose([[1005.], [10015.]], self.evaluate(predictions))
 
@@ -1727,9 +1751,9 @@ class LinearModelTest(test.TestCase):
         self.assertAllClose(np.zeros((3,)), self.evaluate(bias))
         self.assertAllClose(np.zeros((4, 3)), self.evaluate(wire_cast_var))
         sess.run(
-            wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [
-                1000., 1100., 1200.
-            ], [10000., 11000., 12000.]]))
+            wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.],
+                                  [1000., 1100., 1200.],
+                                  [10000., 11000., 12000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
                             self.evaluate(predictions))
@@ -1848,10 +1872,7 @@ class LinearModelTest(test.TestCase):
     price1 = fc.numeric_column('price1', shape=2)
     price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
-      features = {
-          'price1': [[1., 2.], [5., 6.]],
-          'price2': [[3.], [4.]]
-      }
+      features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       model = fc.LinearModel([price1, price2])
       predictions = model(features)
       price1_var, price2_var, bias = model.variables
@@ -2107,11 +2128,16 @@ class LinearModelTest(test.TestCase):
 
     # Provides 1-dim tensor and dense tensor.
     features = {
-        'price': constant_op.constant([-1., 12.,]),
-        'body-style': sparse_tensor.SparseTensor(
-            indices=((0,), (1,)),
-            values=('sedan', 'hardtop'),
-            dense_shape=(2,)),
+        'price':
+            constant_op.constant([
+                -1.,
+                12.,
+            ]),
+        'body-style':
+            sparse_tensor.SparseTensor(
+                indices=((0,), (1,)),
+                values=('sedan', 'hardtop'),
+                dense_shape=(2,)),
     }
     self.assertEqual(1, features['price'].shape.ndims)
     self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0])
@@ -2152,9 +2178,7 @@ class LinearModelTest(test.TestCase):
 
     price_data = np.array([-1., 12.])
     body_style_data = sparse_tensor.SparseTensorValue(
-        indices=((0,), (1,)),
-        values=('sedan', 'hardtop'),
-        dense_shape=(2,))
+        indices=((0,), (1,)), values=('sedan', 'hardtop'), dense_shape=(2,))
     country_data = np.array(['US', 'CA'])
 
     model = fc.LinearModel([price_buckets, body_style, country])
@@ -2602,15 +2626,18 @@ class OldLinearModelTest(test.TestCase):
           partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)):
         fc_old.linear_model(
             features, [price1, price2], cols_to_vars=cols_to_vars)
-      with _initialized_session():
-        self.assertEqual([0.], cols_to_vars['bias'][0].eval())
-        # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables.
-        self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval())
-        self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval())
-        # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and
-        # a [1, 1] Variable.
-        self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval())
-        self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval())
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertEqual([0.], self.evaluate(cols_to_vars['bias'][0]))
+      # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables.
+      self.assertAllEqual([[0.]], self.evaluate(cols_to_vars[price1][0]))
+      self.assertAllEqual([[0.]], self.evaluate(cols_to_vars[price1][1]))
+      # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and
+      # a [1, 1] Variable.
+      self.assertAllEqual([[0.], [0.]], self.evaluate(cols_to_vars[price2][0]))
+      self.assertAllEqual([[0.]], self.evaluate(cols_to_vars[price2][1]))
 
   def test_fills_cols_to_output_tensors(self):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
@@ -2968,8 +2995,11 @@ class OldLinearModelTest(test.TestCase):
       }
       fc_old.linear_model(features, all_cols)
       bias = get_linear_model_bias()
-      with _initialized_session():
-        self.assertAllClose([0.], self.evaluate(bias))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v1_shared_embedding_with_v2_cat_all_other_v2(self):
     price = fc.numeric_column('price')  # v2
@@ -3004,8 +3034,11 @@ class OldLinearModelTest(test.TestCase):
       }
       fc_old.linear_model(features, all_cols)
       bias = get_linear_model_bias()
-      with _initialized_session():
-        self.assertAllClose([0.], self.evaluate(bias))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v1_v2_mix(self):
     price = fc.numeric_column('price')  # v2
@@ -3040,8 +3073,11 @@ class OldLinearModelTest(test.TestCase):
       }
       fc_old.linear_model(features, all_cols)
       bias = get_linear_model_bias()
-      with _initialized_session():
-        self.assertAllClose([0.], self.evaluate(bias))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([0.], self.evaluate(bias))
 
   def test_linear_model_v2_shared_embedding_all_other_v1(self):
     price = fc.numeric_column('price')  # v1
@@ -3099,6 +3135,7 @@ class DenseFeaturesTest(test.TestCase):
       categorical_column = fc.categorical_column_with_identity(
           key='a', num_buckets=3)
       embedding_dimension = 2
+
       def _embedding_column_initializer(shape, dtype, partition_info):
         del shape  # unused
         del dtype  # unused
@@ -3204,16 +3241,22 @@ class DenseFeaturesTest(test.TestCase):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
       net = fc.DenseFeatures(fc.numeric_column('a'))(features)
-      with _initialized_session():
-        self.assertAllClose([[0.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
       columns = (fc.numeric_column(key) for key in features)
       net = fc.DenseFeatures(columns)(features)
-      with _initialized_session():
-        self.assertAllClose([[0., 1.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0., 1.]], self.evaluate(net))
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
@@ -3230,16 +3273,22 @@ class DenseFeaturesTest(test.TestCase):
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       net = fc.DenseFeatures([price])(features)
-      with _initialized_session():
-        self.assertAllClose([[1.], [5.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
     price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc.DenseFeatures([price])(features)
-      with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_compute_output_shape(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -3252,10 +3301,12 @@ class DenseFeaturesTest(test.TestCase):
       dense_features = fc.DenseFeatures([price1, price2])
       self.assertEqual((None, 6), dense_features.compute_output_shape((None,)))
       net = dense_features(features)
-      with _initialized_session():
-        self.assertAllClose(
-            [[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]],
-            self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2., 3., 4., 5., 6.], [5., 6., 7., 8., 9., 10.]],
+                          self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column('price', shape=2)
@@ -3271,20 +3322,23 @@ class DenseFeaturesTest(test.TestCase):
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc.DenseFeatures([price])(features)
-      with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
     price1 = fc.numeric_column('price1', shape=2)
     price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
-      features = {
-          'price1': [[1., 2.], [5., 6.]],
-          'price2': [[3.], [4.]]
-      }
+      features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       net = fc.DenseFeatures([price1, price2])(features)
-      with _initialized_session():
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_cols_to_output_tensors(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -3294,10 +3348,14 @@ class DenseFeaturesTest(test.TestCase):
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       dense_features = fc.DenseFeatures([price1, price2])
       net = dense_features(features, cols_dict)
-      with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], cols_dict[price1].eval())
-        self.assertAllClose([[3.], [4.]], cols_dict[price2].eval())
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2.], [5., 6.]],
+                          self.evaluate(cols_dict[price1]))
+      self.assertAllClose([[3.], [4.]], self.evaluate(cols_dict[price2]))
+      self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_column_order(self):
     price_a = fc.numeric_column('price_a')
@@ -3309,9 +3367,12 @@ class DenseFeaturesTest(test.TestCase):
       }
       net1 = fc.DenseFeatures([price_a, price_b])(features)
       net2 = fc.DenseFeatures([price_b, price_a])(features)
-      with _initialized_session():
-        self.assertAllClose([[1., 3.]], self.evaluate(net1))
-        self.assertAllClose([[1., 3.]], self.evaluate(net2))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 3.]], self.evaluate(net1))
+      self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
     animal = fc.categorical_column_with_identity('animal', num_buckets=4)
@@ -3497,6 +3558,7 @@ class DenseFeaturesTest(test.TestCase):
         (6., 7., 8., 9., 10.),  # id 1
         (11., 12., 13., 14., 15.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       del shape, dtype, partition_info
       return embedding_values
@@ -3528,10 +3590,8 @@ class DenseFeaturesTest(test.TestCase):
 
       # Each row is formed by concatenating `embedded_body_style`,
       # `one_hot_body_style`, and `price` in order.
-      self.assertAllEqual(
-          [[11., 12., 13., 14., 15., 0., 0., 1., 11.],
-           [1., 2., 3., 4., 5., 1., 0., 0., 12]],
-          sess.run(net))
+      self.assertAllEqual([[11., 12., 13., 14., 15., 0., 0., 1., 11.],
+                           [1., 2., 3., 4., 5., 1., 0., 0., 12]], sess.run(net))
 
       coord.request_stop()
       coord.join(threads)
@@ -3542,6 +3602,7 @@ class DenseFeaturesTest(test.TestCase):
         (6., 7., 8., 9., 10.),  # id 1
         (11., 12., 13., 14., 15.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       del shape, dtype, partition_info
       return embedding_values
@@ -3562,13 +3623,19 @@ class DenseFeaturesTest(test.TestCase):
 
     # Provides 1-dim tensor and dense tensor.
     features = {
-        'price': constant_op.constant([11., 12.,]),
-        'body-style': sparse_tensor.SparseTensor(
-            indices=((0,), (1,)),
-            values=('sedan', 'hardtop'),
-            dense_shape=(2,)),
+        'price':
+            constant_op.constant([
+                11.,
+                12.,
+            ]),
+        'body-style':
+            sparse_tensor.SparseTensor(
+                indices=((0,), (1,)),
+                values=('sedan', 'hardtop'),
+                dense_shape=(2,)),
         # This is dense tensor for the categorical_column.
-        'country': constant_op.constant(['CA', 'US']),
+        'country':
+            constant_op.constant(['CA', 'US']),
     }
     self.assertEqual(1, features['price'].shape.ndims)
     self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0])
@@ -3581,10 +3648,9 @@ class DenseFeaturesTest(test.TestCase):
 
       # Each row is formed by concatenating `embedded_body_style`,
       # `one_hot_body_style`, and `price` in order.
-      self.assertAllEqual(
-          [[0., 0., 1., 11., 12., 13., 14., 15., 11.],
-           [1., 0., 0., 1., 2., 3., 4., 5., 12.]],
-          sess.run(net))
+      self.assertAllEqual([[0., 0., 1., 11., 12., 13., 14., 15., 11.],
+                           [1., 0., 0., 1., 2., 3., 4., 5., 12.]],
+                          sess.run(net))
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     embedding_values = (
@@ -3592,6 +3658,7 @@ class DenseFeaturesTest(test.TestCase):
         (6., 7.),  # id 1
         (11., 12.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       del shape, dtype, partition_info
       return embedding_values
@@ -3623,9 +3690,7 @@ class DenseFeaturesTest(test.TestCase):
 
     price_data = np.array([11., 12.])
     body_style_data = sparse_tensor.SparseTensorValue(
-        indices=((0,), (1,)),
-        values=('sedan', 'hardtop'),
-        dense_shape=(2,))
+        indices=((0,), (1,)), values=('sedan', 'hardtop'), dense_shape=(2,))
     country_data = np.array([['US'], ['CA']])
 
     net = fc.DenseFeatures([price, one_hot_body_style, embedded_country])(
@@ -3794,16 +3859,22 @@ class FunctionalInputLayerTest(test.TestCase):
     with ops.Graph().as_default():
       features = features = {'a': [0.]}
       net = fc_old.input_layer(features, fc.numeric_column('a'))
-      with _initialized_session():
-        self.assertAllClose([[0.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0.]], self.evaluate(net))
 
   def test_column_generator(self):
     with ops.Graph().as_default():
       features = features = {'a': [0.], 'b': [1.]}
       columns = (fc.numeric_column(key) for key in features)
       net = fc_old.input_layer(features, columns)
-      with _initialized_session():
-        self.assertAllClose([[0., 1.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0., 1.]], self.evaluate(net))
 
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
@@ -3818,16 +3889,22 @@ class FunctionalInputLayerTest(test.TestCase):
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
       net = fc_old.input_layer(features, [price])
-      with _initialized_session():
-        self.assertAllClose([[1.], [5.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1.], [5.]], self.evaluate(net))
 
   def test_multi_dimension(self):
     price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
       net = fc_old.input_layer(features, [price])
-      with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_raises_if_shape_mismatch(self):
     price = fc.numeric_column('price', shape=2)
@@ -3843,8 +3920,11 @@ class FunctionalInputLayerTest(test.TestCase):
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
       net = fc_old.input_layer(features, [price])
-      with _initialized_session():
-        self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2.], [5., 6.]], self.evaluate(net))
 
   def test_multi_column(self):
     price1 = fc.numeric_column('price1', shape=2)
@@ -3852,8 +3932,11 @@ class FunctionalInputLayerTest(test.TestCase):
     with ops.Graph().as_default():
       features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
       net = fc_old.input_layer(features, [price1, price2])
-      with _initialized_session():
-        self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], self.evaluate(net))
 
   def test_fills_cols_to_vars(self):
     # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
@@ -3983,9 +4066,12 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       net1 = fc_old.input_layer(features, [price_a, price_b])
       net2 = fc_old.input_layer(features, [price_b, price_a])
-      with _initialized_session():
-        self.assertAllClose([[1., 3.]], self.evaluate(net1))
-        self.assertAllClose([[1., 3.]], self.evaluate(net2))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[1., 3.]], self.evaluate(net1))
+      self.assertAllClose([[1., 3.]], self.evaluate(net2))
 
   def test_fails_for_categorical_column(self):
     animal = fc.categorical_column_with_identity('animal', num_buckets=4)
@@ -4323,8 +4409,11 @@ class MakeParseExampleSpecTest(test.TestCase):
         key2: parse_spec2,
         key3: parse_spec3
     })))
-    self.assertDictEqual(
-        {key1: parse_spec1, key2: parse_spec2, key3: parse_spec3}, actual)
+    self.assertDictEqual({
+        key1: parse_spec1,
+        key2: parse_spec2,
+        key3: parse_spec3
+    }, actual)
 
 
 def _assert_sparse_tensor_value(test_case, expected, actual):
@@ -4332,7 +4421,8 @@ def _assert_sparse_tensor_value(test_case, expected, actual):
   test_case.assertAllEqual(expected.indices, actual.indices)
 
   test_case.assertEqual(
-      np.array(expected.values).dtype, np.array(actual.values).dtype)
+      np.array(expected.values).dtype,
+      np.array(actual.values).dtype)
   test_case.assertAllEqual(expected.values, actual.values)
 
   test_case.assertEqual(np.int64, np.array(actual.dense_shape).dtype)
@@ -4418,8 +4508,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     with self.assertRaisesRegexp(errors.OpError, 'file_does_not_exist'):
-      with self.cached_session():
-        lookup_ops.tables_initializer().run()
+      self.evaluate(lookup_ops.tables_initializer())
 
   def test_invalid_vocabulary_size(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid vocabulary_size'):
@@ -4447,8 +4536,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     with self.assertRaisesRegexp(errors.OpError, 'Invalid vocab_size'):
-      with self.cached_session():
-        lookup_ops.tables_initializer().run()
+      self.evaluate(lookup_ops.tables_initializer())
 
   def test_invalid_num_oov_buckets(self):
     with self.assertRaisesRegexp(ValueError, 'Invalid num_oov_buckets'):
@@ -4467,8 +4555,8 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           dtype=dtypes.float64)
 
   def test_invalid_buckets_and_default_value(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'both num_oov_buckets and default_value'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'both num_oov_buckets and default_value'):
       fc.categorical_column_with_vocabulary_file(
           key='aaa',
           vocabulary_file=self._wire_vocabulary_file_name,
@@ -4511,24 +4599,25 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer']))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer']))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
   def test_get_sparse_tensors(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4544,14 +4633,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_none_vocabulary_size(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4565,14 +4657,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(self,
-                                  sparse_tensor.SparseTensorValue(
-                                      indices=inputs.indices,
-                                      values=np.array(
-                                          (2, -1, 0), dtype=np.int64),
-                                      dense_shape=inputs.dense_shape),
-                                  id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_transform_feature(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4586,13 +4681,16 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     id_tensor = fc._transform_features_v2({
         'aaa': inputs
     }, [column], None)[column]
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4604,14 +4702,16 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': (('marlo', ''), ('skywalker', 'omar'))
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1)),
-              values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=(2, 2)),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1)),
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4628,14 +4728,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, 2, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, 2, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4652,14 +4755,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, 33, 0, 62), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, 33, 0, 62), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_small_vocabulary_size(self):
     # 'marlo' is the last entry in our vocabulary file, so be setting
@@ -4678,14 +4784,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((-1, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((-1, -1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_int32(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4702,14 +4811,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, -1, 0, 4), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0, 4), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
@@ -4724,14 +4836,16 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': ((11, -1, -1), (100, 30, -1), (-1, -1, 22))
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1), (2, 2)),
-              values=np.array((2, default_value, 0, 4), dtype=np.int64),
-              dense_shape=(3, 3)),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1), (2, 2)),
+            values=np.array((2, default_value, 0, 4), dtype=np.int64),
+            dense_shape=(3, 3)), self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_file(
@@ -4749,14 +4863,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, 60, 0, 4), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, 60, 0, 4), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_file(
@@ -4775,15 +4892,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
                   dense_shape=(2, 2))
       })
       wire_var, bias = model.variables
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
-                            self.evaluate(wire_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
-        # 'marlo' -> 2: wire_var[2] = 3
-        # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,))))
+      # 'marlo' -> 2: wire_var[2] = 3
+      # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
+      self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_file(
@@ -4802,15 +4921,17 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       }, (wire_column,))
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
-                            self.evaluate(wire_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
-        # 'marlo' -> 2: wire_var[2] = 3
-        # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,))))
+      # 'marlo' -> 2: wire_var[2] = 3
+      # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
+      self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     wire_column = fc.categorical_column_with_vocabulary_file(
@@ -4892,34 +5013,34 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           dtype=dtypes.float32)
 
   def test_invalid_mapping_dtype(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'vocabulary dtype must be string or integer'):
+    with self.assertRaisesRegexp(ValueError,
+                                 r'vocabulary dtype must be string or integer'):
       fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12., 24., 36.))
 
   def test_mismatched_int_dtype(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'dtype.*and vocabulary dtype.*do not match'):
+    with self.assertRaisesRegexp(ValueError,
+                                 r'dtype.*and vocabulary dtype.*do not match'):
       fc.categorical_column_with_vocabulary_list(
           key='aaa',
           vocabulary_list=('omar', 'stringer', 'marlo'),
           dtype=dtypes.int32)
 
   def test_mismatched_string_dtype(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'dtype.*and vocabulary dtype.*do not match'):
+    with self.assertRaisesRegexp(ValueError,
+                                 r'dtype.*and vocabulary dtype.*do not match'):
       fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.string)
 
   def test_none_mapping(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'vocabulary_list.*must be non-empty'):
+    with self.assertRaisesRegexp(ValueError,
+                                 r'vocabulary_list.*must be non-empty'):
       fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=None)
 
   def test_empty_mapping(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'vocabulary_list.*must be non-empty'):
+    with self.assertRaisesRegexp(ValueError,
+                                 r'vocabulary_list.*must be non-empty'):
       fc.categorical_column_with_vocabulary_list(
           key='aaa', vocabulary_list=tuple([]))
 
@@ -4934,8 +5055,8 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
           key='aaa', vocabulary_list=(12, 24, 36), num_oov_buckets=-1)
 
   def test_invalid_buckets_and_default_value(self):
-    with self.assertRaisesRegexp(
-        ValueError, 'both num_oov_buckets and default_value'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'both num_oov_buckets and default_value'):
       fc.categorical_column_with_vocabulary_list(
           key='aaa',
           vocabulary_list=(12, 24, 36),
@@ -4971,46 +5092,46 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
   def test_parse_example_string(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer']))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer']))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
   def test_parse_example_int(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(11, 21, 31))
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(int64_list=feature_pb2.Int64List(
-                    value=[11, 21]))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        int64_list=feature_pb2.Int64List(value=[11, 21]))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=[11, 21],
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]], values=[11, 21], dense_shape=[1, 2]),
+        self.evaluate(features['aaa']))
 
   def test_get_sparse_tensors(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5024,14 +5145,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_transform_feature(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5043,13 +5167,16 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     id_tensor = fc._transform_features_v2({
         'aaa': inputs
     }, [column], None)[column]
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5059,14 +5186,16 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             'aaa': (('marlo', ''), ('skywalker', 'omar'))
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1)),
-              values=np.array((2, -1, 0), dtype=np.int64),
-              dense_shape=(2, 2)),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1)),
+            values=np.array((2, -1, 0), dtype=np.int64),
+            dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5082,14 +5211,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, 2, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, 2, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5105,14 +5237,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, 33, 0, 62), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, 33, 0, 62), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_int32(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5128,14 +5263,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, -1, 0, 4), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, -1, 0, 4), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
@@ -5151,14 +5289,16 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
                          dtype=np.int32)
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1), (2, 2)),
-              values=np.array((2, default_value, 0, 4), dtype=np.int64),
-              dense_shape=(3, 3)),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1), (2, 2)),
+            values=np.array((2, default_value, 0, 4), dtype=np.int64),
+            dense_shape=(3, 3)), self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_list(
@@ -5175,14 +5315,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((2, 60, 0, 4), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((2, 60, 0, 4), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_list(
@@ -5200,15 +5343,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
                   dense_shape=(2, 2))
       })
       wire_var, bias = model.variables
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
-                            self.evaluate(wire_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
-        # 'marlo' -> 2: wire_var[2] = 3
-        # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,))))
+      # 'marlo' -> 2: wire_var[2] = 3
+      # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
+      self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_list(
@@ -5226,15 +5371,17 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       }, (wire_column,))
       bias = get_linear_model_bias()
       wire_var = get_linear_model_column_var(wire_column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)),
-                            self.evaluate(wire_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
-        # 'marlo' -> 2: wire_var[2] = 3
-        # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
-        self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,))))
+      # 'marlo' -> 2: wire_var[2] = 3
+      # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
+      self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     wire_column = fc.categorical_column_with_vocabulary_list(
@@ -5257,7 +5404,6 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
                      fc.VocabularyListCategoricalColumn._from_config(config))
 
 
-
 class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_constructor(self):
@@ -5315,61 +5461,63 @@ class IdentityCategoricalColumnTest(test.TestCase):
 
   def test_parse_example(self):
     a = fc.categorical_column_with_identity(key='aaa', num_buckets=30)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(int64_list=feature_pb2.Int64List(
-                    value=[11, 21]))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        int64_list=feature_pb2.Int64List(value=[11, 21]))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([11, 21], dtype=np.int64),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([11, 21], dtype=np.int64),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
   def test_get_sparse_tensors(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(0, 1, 0),
-        dense_shape=(2, 2))
+        indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2))
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((0, 1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_transform_feature(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(0, 1, 0),
-        dense_shape=(2, 2))
+        indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2))
     id_tensor = fc._transform_features_v2({
         'aaa': inputs
     }, [column], None)[column]
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((0, 1, 0), dtype=np.int64),
+            dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
@@ -5378,46 +5526,49 @@ class IdentityCategoricalColumnTest(test.TestCase):
             'aaa': ((0, -1), (1, 0))
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1)),
-              values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=(2, 2)),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1)),
+            values=np.array((0, 1, 0), dtype=np.int64),
+            dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_with_inputs_too_small(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(1, -1, 0),
-        dense_shape=(2, 2))
+        indices=((0, 0), (1, 0), (1, 1)), values=(1, -1, 0), dense_shape=(2, 2))
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      with self.assertRaisesRegexp(
-          errors.OpError, 'assert_greater_or_equal_0'):
-        id_weight_pair.id_tensor.eval()
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    with self.assertRaisesRegexp(errors.OpError, 'assert_greater_or_equal_0'):
+      self.evaluate(id_weight_pair.id_tensor)
 
   def test_get_sparse_tensors_with_inputs_too_big(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(1, 99, 0),
-        dense_shape=(2, 2))
+        indices=((0, 0), (1, 0), (1, 1)), values=(1, 99, 0), dense_shape=(2, 2))
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      with self.assertRaisesRegexp(
-          errors.OpError, 'assert_less_than_num_buckets'):
-        id_weight_pair.id_tensor.eval()
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    with self.assertRaisesRegexp(errors.OpError,
+                                 'assert_less_than_num_buckets'):
+      self.evaluate(id_weight_pair.id_tensor)
 
   def test_get_sparse_tensors_with_default_value(self):
     column = fc.categorical_column_with_identity(
@@ -5431,14 +5582,17 @@ class IdentityCategoricalColumnTest(test.TestCase):
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array((1, 3, 3), dtype=np.int64),
-              dense_shape=inputs.dense_shape),
-          id_weight_pair.id_tensor.eval())
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array((1, 3, 3), dtype=np.int64),
+            dense_shape=inputs.dense_shape),
+        self.evaluate(id_weight_pair.id_tensor))
 
   def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self):
     column = fc.categorical_column_with_identity(
@@ -5447,14 +5601,15 @@ class IdentityCategoricalColumnTest(test.TestCase):
     input_values = array_ops.placeholder(dtype=dtypes.int32)
     input_shape = array_ops.placeholder(dtype=dtypes.int64)
     inputs = sparse_tensor.SparseTensorValue(
-        indices=input_indices,
-        values=input_values,
-        dense_shape=input_shape)
+        indices=input_indices, values=input_values, dense_shape=input_shape)
     id_weight_pair = column.get_sparse_tensors(
         fc.FeatureTransformationCache({
             'aaa': inputs
         }), None)
     self.assertIsNone(id_weight_pair.weight_tensor)
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
     with _initialized_session():
       _assert_sparse_tensor_value(
           self,
@@ -5462,11 +5617,12 @@ class IdentityCategoricalColumnTest(test.TestCase):
               indices=np.array(((0, 0), (1, 0), (1, 1)), dtype=np.int64),
               values=np.array((1, 3, 3), dtype=np.int64),
               dense_shape=np.array((2, 2), dtype=np.int64)),
-          id_weight_pair.id_tensor.eval(feed_dict={
-              input_indices: ((0, 0), (1, 0), (1, 1)),
-              input_values: (1, -1, 99),
-              input_shape: (2, 2),
-          }))
+          id_weight_pair.id_tensor.eval(
+              feed_dict={
+                  input_indices: ((0, 0), (1, 0), (1, 1)),
+                  input_values: (1, -1, 99),
+                  input_shape: (2, 2),
+              }))
 
   def test_linear_model(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
@@ -5481,14 +5637,17 @@ class IdentityCategoricalColumnTest(test.TestCase):
                   dense_shape=(2, 2))
       })
       weight_var, bias = model.variables
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] = 1
-        # weight_var[2] + weight_var[1] = 3+2 = 5
-        self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] = 1
+      # weight_var[2] + weight_var[1] = 3+2 = 5
+      self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
@@ -5503,14 +5662,17 @@ class IdentityCategoricalColumnTest(test.TestCase):
       }, (column,))
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] = 1
-        # weight_var[2] + weight_var[1] = 3+2 = 5
-        self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] = 1
+      # weight_var[2] + weight_var[1] = 3+2 = 5
+      self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
   def test_serialization(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
@@ -5546,11 +5708,16 @@ class TransformFeaturesTest(test.TestCase):
       }
       transformed = fc._transform_features_v2(
           features, [bucketized_price, hashed_sparse], None)
-      with _initialized_session():
-        self.assertIn(bucketized_price.name, transformed[bucketized_price].name)
-        self.assertAllEqual([[0], [3]], transformed[bucketized_price].eval())
-        self.assertIn(hashed_sparse.name, transformed[hashed_sparse].name)
-        self.assertAllEqual([6, 4, 1], transformed[hashed_sparse].values.eval())
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertIn(bucketized_price.name, transformed[bucketized_price].name)
+      self.assertAllEqual([[0], [3]],
+                          self.evaluate(transformed[bucketized_price]))
+      self.assertIn(hashed_sparse.name, transformed[hashed_sparse].name)
+      self.assertAllEqual([6, 4, 1],
+                          self.evaluate(transformed[hashed_sparse].values))
 
   def test_column_order(self):
     """When the column is both dense and sparse, uses sparse tensors."""
@@ -5615,9 +5782,9 @@ class IndicatorColumnTest(test.TestCase):
         'animal': ['fox', 'fox']
     })
     output = transformation_cache.get(animal, None)
-    with self.cached_session():
-      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
-                          self.evaluate(output))
+
+    self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
+                        self.evaluate(output))
 
   def test_2D_shape_succeeds(self):
     # TODO(ispir/cassandrax): Swith to categorical_column_with_keys when ready.
@@ -5631,9 +5798,9 @@ class IndicatorColumnTest(test.TestCase):
                 dense_shape=[2, 1])
     })
     output = transformation_cache.get(animal, None)
-    with self.cached_session():
-      self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
-                          self.evaluate(output))
+
+    self.assertAllEqual([[0., 0., 1., 0.], [0., 0., 1., 0.]],
+                        self.evaluate(output))
 
   def test_multi_hot(self):
     animal = fc.indicator_column(
@@ -5645,8 +5812,8 @@ class IndicatorColumnTest(test.TestCase):
                 indices=[[0, 0], [0, 1]], values=[1, 1], dense_shape=[1, 2])
     })
     output = transformation_cache.get(animal, None)
-    with self.cached_session():
-      self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output))
+
+    self.assertAllEqual([[0., 2., 0., 0.]], self.evaluate(output))
 
   def test_multi_hot2(self):
     animal = fc.indicator_column(
@@ -5657,8 +5824,8 @@ class IndicatorColumnTest(test.TestCase):
                 indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
     })
     output = transformation_cache.get(animal, None)
-    with self.cached_session():
-      self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
+
+    self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
   def test_deep_copy(self):
     a = fc.categorical_column_with_hash_bucket('a', 4)
@@ -5672,40 +5839,44 @@ class IndicatorColumnTest(test.TestCase):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_indicator = fc.indicator_column(a)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer']))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer']))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a_indicator]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
   def test_transform(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_indicator = fc.indicator_column(a)
     features = {
-        'aaa': sparse_tensor.SparseTensorValue(
-            indices=((0, 0), (1, 0), (1, 1)),
-            values=('marlo', 'skywalker', 'omar'),
-            dense_shape=(2, 2))
+        'aaa':
+            sparse_tensor.SparseTensorValue(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=('marlo', 'skywalker', 'omar'),
+                dense_shape=(2, 2))
     }
     indicator_tensor = fc._transform_features_v2(features, [a_indicator],
                                                  None)[a_indicator]
-    with _initialized_session():
-      self.assertAllEqual([[0, 0, 1], [1, 0, 0]],
-                          self.evaluate(indicator_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual([[0, 0, 1], [1, 0, 0]], self.evaluate(indicator_tensor))
 
   def test_transform_with_weighted_column(self):
     # Github issue 12557
@@ -5719,8 +5890,11 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = fc._transform_features_v2(features, [indicator],
                                                  None)[indicator]
-    with _initialized_session():
-      self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
@@ -5734,8 +5908,11 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = fc._transform_features_v2(features, [indicator],
                                                  None)[indicator]
-    with _initialized_session():
-      self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
 
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
@@ -5747,8 +5924,11 @@ class IndicatorColumnTest(test.TestCase):
     }
     indicator_tensor = fc._transform_features_v2(features, [indicator],
                                                  None)[indicator]
-    with _initialized_session():
-      self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
   def test_linear_model(self):
     animal = fc.indicator_column(
@@ -5763,12 +5943,15 @@ class IndicatorColumnTest(test.TestCase):
       model = fc.LinearModel([animal])
       predictions = model(features)
       weight_var, _ = model.variables
-      with _initialized_session():
-        # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
-        self.assertAllClose([[0.]], self.evaluate(predictions))
-        weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # All should be zero-initialized.
+      self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+      self.assertAllClose([[0.]], self.evaluate(predictions))
+      self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]]))
+      self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_old_linear_model(self):
     animal = fc.indicator_column(
@@ -5782,12 +5965,15 @@ class IndicatorColumnTest(test.TestCase):
 
       predictions = fc_old.linear_model(features, [animal])
       weight_var = get_linear_model_column_var(animal)
-      with _initialized_session():
-        # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
-        self.assertAllClose([[0.]], self.evaluate(predictions))
-        weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # All should be zero-initialized.
+      self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+      self.assertAllClose([[0.]], self.evaluate(predictions))
+      self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]]))
+      self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
     animal = fc.indicator_column(
@@ -5801,12 +5987,15 @@ class IndicatorColumnTest(test.TestCase):
 
       predictions = fc_old.linear_model(features, [animal])
       weight_var = get_linear_model_column_var(animal)
-      with _initialized_session():
-        # All should be zero-initialized.
-        self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
-        self.assertAllClose([[0.]], self.evaluate(predictions))
-        weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
-        self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # All should be zero-initialized.
+      self.assertAllClose([[0.], [0.], [0.], [0.]], self.evaluate(weight_var))
+      self.assertAllClose([[0.]], self.evaluate(predictions))
+      self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]]))
+      self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
   def test_dense_features(self):
     animal = fc.indicator_column(
@@ -5818,8 +6007,11 @@ class IndicatorColumnTest(test.TestCase):
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
       net = fc.DenseFeatures([animal])(features)
-      with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_input_layer(self):
     animal = fc.indicator_column(
@@ -5831,8 +6023,11 @@ class IndicatorColumnTest(test.TestCase):
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
       net = fc_old.input_layer(features, [animal])
-      with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_input_layer_old_categorical(self):
     animal = fc.indicator_column(
@@ -5844,8 +6039,11 @@ class IndicatorColumnTest(test.TestCase):
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
       net = fc_old.input_layer(features, [animal])
-      with _initialized_session():
-        self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
   def test_serialization(self):
     parent = fc.categorical_column_with_identity('animal', num_buckets=4)
@@ -5875,7 +6073,6 @@ class IndicatorColumnTest(test.TestCase):
     self.assertIs(parent, new_animal.categorical_column)
 
 
-
 class _TestStateManager(fc.StateManager):
 
   def __init__(self, trainable=True):
@@ -6013,40 +6210,45 @@ class EmbeddingColumnTest(test.TestCase):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_embedded = fc.embedding_column(a, dimension=2)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer']))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer']))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a_embedded]))
     self.assertIn('aaa', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
   def test_transform_feature(self):
     a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     a_embedded = fc.embedding_column(a, dimension=2)
     features = {
-        'aaa': sparse_tensor.SparseTensor(
-            indices=((0, 0), (1, 0), (1, 1)),
-            values=(0, 1, 0),
-            dense_shape=(2, 2))
+        'aaa':
+            sparse_tensor.SparseTensor(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=(0, 1, 0),
+                dense_shape=(2, 2))
     }
     outputs = fc._transform_features_v2(features, [a, a_embedded], None)
     output_a = outputs[a]
     output_embedded = outputs[a_embedded]
-    with _initialized_session():
-      _assert_sparse_tensor_value(self, self.evaluate(output_a),
-                                  self.evaluate(output_embedded))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(self, self.evaluate(output_a),
+                                self.evaluate(output_embedded))
 
   def test_get_dense_tensor(self):
     # Inputs.
@@ -6067,6 +6269,7 @@ class EmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -6105,9 +6308,12 @@ class EmbeddingColumnTest(test.TestCase):
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertItemsEqual(('embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_old_categorical(self):
     # Inputs.
@@ -6165,9 +6371,12 @@ class EmbeddingColumnTest(test.TestCase):
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertItemsEqual(('embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_3d(self):
     # Inputs.
@@ -6184,11 +6393,12 @@ class EmbeddingColumnTest(test.TestCase):
     # Embedding variable.
     embedding_dimension = 3
     embedding_values = (
-        (1., 2., 4.),   # id 0
-        (3., 5., 1.),   # id 1
+        (1., 2., 4.),  # id 0
+        (3., 5., 1.),  # id 1
         (7., 11., 2.),  # id 2
-        (2., 7., 12.)   # id 3
+        (2., 7., 12.)  # id 3
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -6228,9 +6438,12 @@ class EmbeddingColumnTest(test.TestCase):
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertItemsEqual(('embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
@@ -6251,6 +6464,7 @@ class EmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -6294,16 +6508,21 @@ class EmbeddingColumnTest(test.TestCase):
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
-        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
+    self.assertItemsEqual(('embedding_weights:0',),
+                          tuple([v.name for v in global_vars]))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
     with _initialized_session():
-      self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval(
-          feed_dict={
-              input_indices: sparse_input.indices,
-              input_values: sparse_input.values,
-              input_shape: sparse_input.dense_shape,
-          }))
+      self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
+      self.assertAllEqual(
+          expected_lookups,
+          embedding_lookup.eval(
+              feed_dict={
+                  input_indices: sparse_input.indices,
+                  input_values: sparse_input.values,
+                  input_shape: sparse_input.dense_shape,
+              }))
 
   def test_get_dense_tensor_restore_from_ckpt(self):
     # Inputs.
@@ -6359,11 +6578,14 @@ class EmbeddingColumnTest(test.TestCase):
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
-        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
+    self.assertItemsEqual(('embedding_weights:0',),
+                          tuple([v.name for v in global_vars]))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
   def test_linear_model(self):
     # Inputs.
@@ -6382,6 +6604,7 @@ class EmbeddingColumnTest(test.TestCase):
     embedding_dimension = 2
     embedding_shape = (vocabulary_size, embedding_dimension)
     zeros_embedding_values = np.zeros(embedding_shape)
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual(embedding_shape, shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -6408,39 +6631,42 @@ class EmbeddingColumnTest(test.TestCase):
           expected_var_names,
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
       trainable_vars = {
-          v.name: v for v in ops.get_collection(
-              ops.GraphKeys.TRAINABLE_VARIABLES)
+          v.name: v
+          for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
       }
       self.assertItemsEqual(expected_var_names, trainable_vars.keys())
       bias = trainable_vars['linear_model/bias_weights:0']
       embedding_weights = trainable_vars[
           'linear_model/aaa_embedding/embedding_weights:0']
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
-      with _initialized_session():
-        # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
-        self.assertAllClose(zeros_embedding_values,
-                            self.evaluate(embedding_weights))
-        self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
-        self.assertAllClose(
-            np.zeros((batch_size, 1)), self.evaluate(predictions))
-
-        # Predictions with all non-zero weights.
-        embedding_weights.assign((
-            (1., 2.),  # id 0
-            (3., 5.),  # id 1
-            (7., 11.)  # id 2
-        )).eval()
-        linear_weights.assign(((4.,), (6.,))).eval()
-        # example 0, ids [2], embedding[0] = [7, 11]
-        # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
-        # example 2, ids [], embedding[2] = [0, 0]
-        # example 3, ids [1], embedding[3] = [3, 5]
-        # sum(embeddings * linear_weights)
-        # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
-                            self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # Predictions with all zero weights.
+      self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+      self.assertAllClose(zeros_embedding_values,
+                          self.evaluate(embedding_weights))
+      self.assertAllClose(
+          np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
+      self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions))
+
+      # Predictions with all non-zero weights.
+      self.evaluate(
+          embedding_weights.assign((
+              (1., 2.),  # id 0
+              (3., 5.),  # id 1
+              (7., 11.)  # id 2
+          )))
+      self.evaluate(linear_weights.assign(((4.,), (6.,))))
+      # example 0, ids [2], embedding[0] = [7, 11]
+      # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
+      # example 2, ids [], embedding[2] = [0, 0]
+      # example 3, ids [1], embedding[3] = [3, 5]
+      # sum(embeddings * linear_weights)
+      # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
+      self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                          self.evaluate(predictions))
 
   def test_dense_features(self):
     # Inputs.
@@ -6461,6 +6687,7 @@ class EmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -6500,9 +6727,12 @@ class EmbeddingColumnTest(test.TestCase):
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in trainable_vars]))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(trainable_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_dense_features_not_trainable(self):
     # Inputs.
@@ -6523,6 +6753,7 @@ class EmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -6559,11 +6790,14 @@ class EmbeddingColumnTest(test.TestCase):
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
-    self.assertItemsEqual(
-        [], ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, global_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
+    self.assertItemsEqual([],
+                          ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_input_layer(self):
     # Inputs.
@@ -6623,9 +6857,12 @@ class EmbeddingColumnTest(test.TestCase):
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     self.assertItemsEqual(('input_layer/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in trainable_vars]))
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, trainable_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(feature_layer))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(trainable_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(feature_layer))
 
   def test_old_linear_model(self):
     # Inputs.
@@ -6680,31 +6917,34 @@ class EmbeddingColumnTest(test.TestCase):
       embedding_weights = trainable_vars[
           'linear_model/aaa_embedding/embedding_weights:0']
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
-      with _initialized_session():
-        # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
-        self.assertAllClose(zeros_embedding_values,
-                            self.evaluate(embedding_weights))
-        self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
-        self.assertAllClose(
-            np.zeros((batch_size, 1)), self.evaluate(predictions))
-
-        # Predictions with all non-zero weights.
-        embedding_weights.assign((
-            (1., 2.),  # id 0
-            (3., 5.),  # id 1
-            (7., 11.)  # id 2
-        )).eval()
-        linear_weights.assign(((4.,), (6.,))).eval()
-        # example 0, ids [2], embedding[0] = [7, 11]
-        # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
-        # example 2, ids [], embedding[2] = [0, 0]
-        # example 3, ids [1], embedding[3] = [3, 5]
-        # sum(embeddings * linear_weights)
-        # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
-                            self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # Predictions with all zero weights.
+      self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+      self.assertAllClose(zeros_embedding_values,
+                          self.evaluate(embedding_weights))
+      self.assertAllClose(
+          np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
+      self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions))
+
+      # Predictions with all non-zero weights.
+      self.evaluate(
+          embedding_weights.assign((
+              (1., 2.),  # id 0
+              (3., 5.),  # id 1
+              (7., 11.)  # id 2
+          )))
+      self.evaluate(linear_weights.assign(((4.,), (6.,))))
+      # example 0, ids [2], embedding[0] = [7, 11]
+      # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
+      # example 2, ids [], embedding[2] = [0, 0]
+      # example 3, ids [1], embedding[3] = [3, 5]
+      # sum(embeddings * linear_weights)
+      # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
+      self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                          self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
     # Inputs.
@@ -6759,31 +6999,34 @@ class EmbeddingColumnTest(test.TestCase):
       embedding_weights = trainable_vars[
           'linear_model/aaa_embedding/embedding_weights:0']
       linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
-      with _initialized_session():
-        # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
-        self.assertAllClose(zeros_embedding_values,
-                            self.evaluate(embedding_weights))
-        self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
-        self.assertAllClose(
-            np.zeros((batch_size, 1)), self.evaluate(predictions))
-
-        # Predictions with all non-zero weights.
-        embedding_weights.assign((
-            (1., 2.),  # id 0
-            (3., 5.),  # id 1
-            (7., 11.)  # id 2
-        )).eval()
-        linear_weights.assign(((4.,), (6.,))).eval()
-        # example 0, ids [2], embedding[0] = [7, 11]
-        # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
-        # example 2, ids [], embedding[2] = [0, 0]
-        # example 3, ids [1], embedding[3] = [3, 5]
-        # sum(embeddings * linear_weights)
-        # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
-        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
-                            self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # Predictions with all zero weights.
+      self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+      self.assertAllClose(zeros_embedding_values,
+                          self.evaluate(embedding_weights))
+      self.assertAllClose(
+          np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights))
+      self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions))
+
+      # Predictions with all non-zero weights.
+      self.evaluate(
+          embedding_weights.assign((
+              (1., 2.),  # id 0
+              (3., 5.),  # id 1
+              (7., 11.)  # id 2
+          )))
+      self.evaluate(linear_weights.assign(((4.,), (6.,))))
+      # example 0, ids [2], embedding[0] = [7, 11]
+      # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
+      # example 2, ids [], embedding[2] = [0, 0]
+      # example 3, ids [1], embedding[3] = [3, 5]
+      # sum(embeddings * linear_weights)
+      # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
+      self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
+                          self.evaluate(predictions))
 
   def test_serialization(self):
 
@@ -6972,49 +7215,52 @@ class SharedEmbeddingColumnTest(test.TestCase):
     b = fc.categorical_column_with_vocabulary_list(
         key='bbb', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2)
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer'])),
-            'bbb':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'stringer', b'marlo'])),
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer'])),
+                'bbb':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'stringer', b'marlo'])),
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a_embedded, b_embedded]))
     self.assertIn('aaa', features)
     self.assertIn('bbb', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'stringer', b'marlo'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['bbb'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'stringer', b'marlo'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['bbb']))
 
   def test_transform_feature(self):
     a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     b = fc.categorical_column_with_identity(key='bbb', num_buckets=3)
     a_embedded, b_embedded = fc.shared_embedding_columns_v2([a, b], dimension=2)
     features = {
-        'aaa': sparse_tensor.SparseTensor(
-            indices=((0, 0), (1, 0), (1, 1)),
-            values=(0, 1, 0),
-            dense_shape=(2, 2)),
-        'bbb': sparse_tensor.SparseTensor(
-            indices=((0, 0), (1, 0), (1, 1)),
-            values=(1, 2, 1),
-            dense_shape=(2, 2)),
+        'aaa':
+            sparse_tensor.SparseTensor(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=(0, 1, 0),
+                dense_shape=(2, 2)),
+        'bbb':
+            sparse_tensor.SparseTensor(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=(1, 2, 1),
+                dense_shape=(2, 2)),
     }
     outputs = fc._transform_features_v2(features,
                                         [a, a_embedded, b, b_embedded], None)
@@ -7022,26 +7268,28 @@ class SharedEmbeddingColumnTest(test.TestCase):
     output_a_embedded = outputs[a_embedded]
     output_b = outputs[b]
     output_b_embedded = outputs[b_embedded]
-    with _initialized_session():
-      _assert_sparse_tensor_value(self, self.evaluate(output_a),
-                                  self.evaluate(output_a_embedded))
-      _assert_sparse_tensor_value(self, self.evaluate(output_b),
-                                  self.evaluate(output_b_embedded))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(self, self.evaluate(output_a),
+                                self.evaluate(output_a_embedded))
+    _assert_sparse_tensor_value(self, self.evaluate(output_b),
+                                self.evaluate(output_b_embedded))
 
   def test_get_dense_tensor(self):
     # Inputs.
     vocabulary_size = 3
     # -1 values are ignored.
-    input_a = np.array(
-        [[2, -1, -1],  # example 0, ids [2]
-         [0, 1, -1]])  # example 1, ids [0, 1]
-    input_b = np.array(
-        [[0, -1, -1],  # example 0, ids [0]
-         [-1, -1, -1]])  # example 1, ids []
-    input_features = {
-        'aaa': input_a,
-        'bbb': input_b
-    }
+    input_a = np.array([
+        [2, -1, -1],  # example 0, ids [2]
+        [0, 1, -1]
+    ])  # example 1, ids [0, 1]
+    input_b = np.array([
+        [0, -1, -1],  # example 0, ids [0]
+        [-1, -1, -1]
+    ])  # example 1, ids []
+    input_features = {'aaa': input_a, 'bbb': input_b}
 
     # Embedding variable.
     embedding_dimension = 2
@@ -7050,6 +7298,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -7091,21 +7340,26 @@ class SharedEmbeddingColumnTest(test.TestCase):
     self.assertItemsEqual(('aaa_bbb_shared_embedding:0',),
                           tuple([v.name for v in global_vars]))
     embedding_var = global_vars[0]
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, self.evaluate(embedding_var))
-      self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a))
-      self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values, self.evaluate(embedding_var))
+    self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a))
+    self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b))
 
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
     vocabulary_size = 3
     # -1 values are ignored.
-    input_a = np.array(
-        [[2, -1, -1],  # example 0, ids [2]
-         [0, 1, -1]])  # example 1, ids [0, 1]
-    input_b = np.array(
-        [[0, -1, -1],  # example 0, ids [0]
-         [-1, -1, -1]])  # example 1, ids []
+    input_a = np.array([
+        [2, -1, -1],  # example 0, ids [2]
+        [0, 1, -1]
+    ])  # example 1, ids [0, 1]
+    input_b = np.array([
+        [0, -1, -1],  # example 0, ids [0]
+        [-1, -1, -1]
+    ])  # example 1, ids []
     # Specify shape, because dense input must have rank specified.
     input_a_placeholder = array_ops.placeholder(
         dtype=dtypes.int64, shape=[None, 3])
@@ -7127,6 +7381,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -7157,17 +7412,20 @@ class SharedEmbeddingColumnTest(test.TestCase):
     batch_size = 2
     vocabulary_size = 3
     # -1 values are ignored.
-    input_a = np.array(
-        [[2, -1, -1],  # example 0, ids [2]
-         [0, 1, -1]])  # example 1, ids [0, 1]
-    input_b = np.array(
-        [[0, -1, -1],  # example 0, ids [0]
-         [-1, -1, -1]])  # example 1, ids []
+    input_a = np.array([
+        [2, -1, -1],  # example 0, ids [2]
+        [0, 1, -1]
+    ])  # example 1, ids [0, 1]
+    input_b = np.array([
+        [0, -1, -1],  # example 0, ids [0]
+        [-1, -1, -1]
+    ])  # example 1, ids []
 
     # Embedding variable.
     embedding_dimension = 2
     embedding_shape = (vocabulary_size, embedding_dimension)
     zeros_embedding_values = np.zeros(embedding_shape)
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual(embedding_shape, shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -7203,8 +7461,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
           expected_var_names,
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
       trainable_vars = {
-          v.name: v for v in ops.get_collection(
-              ops.GraphKeys.TRAINABLE_VARIABLES)
+          v.name: v
+          for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
       }
       self.assertItemsEqual(expected_var_names, trainable_vars.keys())
       bias = trainable_vars['linear_model/bias_weights:0']
@@ -7213,35 +7471,38 @@ class SharedEmbeddingColumnTest(test.TestCase):
           'linear_model/aaa_shared_embedding/weights:0']
       linear_weights_b = trainable_vars[
           'linear_model/bbb_shared_embedding/weights:0']
-      with _initialized_session():
-        # Predictions with all zero weights.
-        self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
-        self.assertAllClose(zeros_embedding_values,
-                            self.evaluate(embedding_weights))
-        self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a))
-        self.assertAllClose(
-            np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b))
-        self.assertAllClose(
-            np.zeros((batch_size, 1)), self.evaluate(predictions))
-
-        # Predictions with all non-zero weights.
-        embedding_weights.assign((
-            (1., 2.),  # id 0
-            (3., 5.),  # id 1
-            (7., 11.)  # id 2
-        )).eval()
-        linear_weights_a.assign(((4.,), (6.,))).eval()
-        # example 0, ids [2], embedding[0] = [7, 11]
-        # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
-        # sum(embeddings * linear_weights)
-        # = [4*7 + 6*11, 4*2 + 6*3.5] = [94, 29]
-        linear_weights_b.assign(((3.,), (5.,))).eval()
-        # example 0, ids [0], embedding[0] = [1, 2]
-        # example 1, ids [], embedding[1] = 0, 0]
-        # sum(embeddings * linear_weights)
-        # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
-        self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      # Predictions with all zero weights.
+      self.assertAllClose(np.zeros((1,)), self.evaluate(bias))
+      self.assertAllClose(zeros_embedding_values,
+                          self.evaluate(embedding_weights))
+      self.assertAllClose(
+          np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_a))
+      self.assertAllClose(
+          np.zeros((embedding_dimension, 1)), self.evaluate(linear_weights_b))
+      self.assertAllClose(np.zeros((batch_size, 1)), self.evaluate(predictions))
+
+      # Predictions with all non-zero weights.
+      self.evaluate(
+          embedding_weights.assign((
+              (1., 2.),  # id 0
+              (3., 5.),  # id 1
+              (7., 11.)  # id 2
+          )))
+      self.evaluate(linear_weights_a.assign(((4.,), (6.,))))
+      # example 0, ids [2], embedding[0] = [7, 11]
+      # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
+      # sum(embeddings * linear_weights)
+      # = [4*7 + 6*11, 4*2 + 6*3.5] = [94, 29]
+      self.evaluate(linear_weights_b.assign(((3.,), (5.,))))
+      # example 0, ids [0], embedding[0] = [1, 2]
+      # example 1, ids [], embedding[1] = 0, 0]
+      # sum(embeddings * linear_weights)
+      # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
+      self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions))
 
   def _test_dense_features(self, trainable=True):
     # Inputs.
@@ -7278,6 +7539,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -7349,9 +7611,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
     else:
       self.assertItemsEqual([], tuple([v.name for v in trainable_vars]))
     shared_embedding_vars = global_vars
-    with _initialized_session():
-      self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval())
-      self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    self.assertAllEqual(embedding_values,
+                        self.evaluate(shared_embedding_vars[0]))
+    self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
   def test_dense_features(self):
     self._test_dense_features()
@@ -7379,7 +7645,6 @@ class SharedEmbeddingColumnTest(test.TestCase):
     # TODO(rohanj): Add tests for (from|get)_config once implemented
 
 
-
 class WeightedCategoricalColumnTest(test.TestCase):
 
   def test_defaults(self):
@@ -7463,43 +7728,43 @@ class WeightedCategoricalColumnTest(test.TestCase):
         indices=((0, 0), (1, 0), (1, 1)),
         values=('omar', 'stringer', 'marlo'),
         dense_shape=(2, 2))
-    with self.assertRaisesRegexp(
-        ValueError, 'values is not in features dictionary'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'values is not in features dictionary'):
       fc._transform_features_v2({'ids': inputs}, (column,), None)
 
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
     a_weighted = fc.weighted_categorical_column(a, weight_feature_key='weights')
-    data = example_pb2.Example(features=feature_pb2.Features(
-        feature={
-            'aaa':
-                feature_pb2.Feature(bytes_list=feature_pb2.BytesList(
-                    value=[b'omar', b'stringer'])),
-            'weights':
-                feature_pb2.Feature(float_list=feature_pb2.FloatList(
-                    value=[1., 10.]))
-        }))
+    data = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'aaa':
+                    feature_pb2.Feature(
+                        bytes_list=feature_pb2.BytesList(
+                            value=[b'omar', b'stringer'])),
+                'weights':
+                    feature_pb2.Feature(
+                        float_list=feature_pb2.FloatList(value=[1., 10.]))
+            }))
     features = parsing_ops.parse_example(
         serialized=[data.SerializeToString()],
         features=fc.make_parse_example_spec_v2([a_weighted]))
     self.assertIn('aaa', features)
     self.assertIn('weights', features)
-    with self.cached_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([b'omar', b'stringer'], dtype=np.object_),
-              dense_shape=[1, 2]),
-          features['aaa'].eval())
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=[[0, 0], [0, 1]],
-              values=np.array([1., 10.], dtype=np.float32),
-              dense_shape=[1, 2]),
-          features['weights'].eval())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([b'omar', b'stringer'], dtype=np.object_),
+            dense_shape=[1, 2]), self.evaluate(features['aaa']))
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [0, 1]],
+            values=np.array([1., 10.], dtype=np.float32),
+            dense_shape=[1, 2]), self.evaluate(features['weights']))
 
   def test_transform_features(self):
     column = fc.weighted_categorical_column(
@@ -7507,9 +7772,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(0, 1, 0),
-        dense_shape=(2, 2))
+        indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2))
     weights = sparse_tensor.SparseTensorValue(
         indices=((0, 0), (1, 0), (1, 1)),
         values=(0.5, 1.0, 0.1),
@@ -7518,19 +7781,22 @@ class WeightedCategoricalColumnTest(test.TestCase):
         'ids': inputs,
         'values': weights,
     }, (column,), None)[column]
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array(inputs.values, dtype=np.int64),
-              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=weights.indices,
-              values=np.array(weights.values, dtype=np.float32),
-              dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array(inputs.values, dtype=np.int64),
+            dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=weights.indices,
+            values=np.array(weights.values, dtype=np.float32),
+            dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_input(self):
     column = fc.weighted_categorical_column(
@@ -7545,19 +7811,22 @@ class WeightedCategoricalColumnTest(test.TestCase):
         'ids': ((0, -1), (1, 0)),
         'values': weights,
     }, (column,), None)[column]
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1)),
-              values=np.array((0, 1, 0), dtype=np.int64),
-              dense_shape=(2, 2)), self.evaluate(id_tensor))
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=weights.indices,
-              values=np.array(weights.values, dtype=np.float32),
-              dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1)),
+            values=np.array((0, 1, 0), dtype=np.int64),
+            dense_shape=(2, 2)), self.evaluate(id_tensor))
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=weights.indices,
+            values=np.array(weights.values, dtype=np.float32),
+            dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
   def test_transform_features_dense_weights(self):
     column = fc.weighted_categorical_column(
@@ -7565,26 +7834,27 @@ class WeightedCategoricalColumnTest(test.TestCase):
             key='ids', num_buckets=3),
         weight_feature_key='values')
     inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(2, 1, 0),
-        dense_shape=(2, 2))
+        indices=((0, 0), (1, 0), (1, 1)), values=(2, 1, 0), dense_shape=(2, 2))
     id_tensor, weight_tensor = fc._transform_features_v2({
         'ids': inputs,
         'values': ((.5, 0.), (1., .1)),
     }, (column,), None)[column]
-    with _initialized_session():
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=inputs.indices,
-              values=np.array(inputs.values, dtype=np.int64),
-              dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
-      _assert_sparse_tensor_value(
-          self,
-          sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1)),
-              values=np.array((.5, 1., .1), dtype=np.float32),
-              dense_shape=(2, 2)), self.evaluate(weight_tensor))
+
+    self.evaluate(variables_lib.global_variables_initializer())
+    self.evaluate(lookup_ops.tables_initializer())
+
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=inputs.indices,
+            values=np.array(inputs.values, dtype=np.int64),
+            dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
+    _assert_sparse_tensor_value(
+        self,
+        sparse_tensor.SparseTensorValue(
+            indices=((0, 0), (1, 0), (1, 1)),
+            values=np.array((.5, 1., .1), dtype=np.float32),
+            dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
   def test_linear_model(self):
     column = fc.weighted_categorical_column(
@@ -7606,15 +7876,18 @@ class WeightedCategoricalColumnTest(test.TestCase):
                   dense_shape=(2, 2))
       })
       weight_var, bias = model.variables
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
-        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
-        # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+      # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+      # = 3*1 + 2*.1 = 3+.2 = 3.2
+      self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_linear_model_mismatched_shape(self):
     column = fc.weighted_categorical_column(
@@ -7678,15 +7951,18 @@ class WeightedCategoricalColumnTest(test.TestCase):
           'values': ((.5,), (1.,), (.1,))
       })
       weight_var, bias = model.variables
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
-        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
-        # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+      # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+      # = 3*1 + 2*.1 = 3+.2 = 3.2
+      self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model(self):
     column = fc.weighted_categorical_column(
@@ -7708,15 +7984,18 @@ class WeightedCategoricalColumnTest(test.TestCase):
       }, (column,))
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
-        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
-        # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+      # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+      # = 3*1 + 2*.1 = 3+.2 = 3.2
+      self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model_mismatched_shape(self):
     column = fc.weighted_categorical_column(
@@ -7779,15 +8058,18 @@ class WeightedCategoricalColumnTest(test.TestCase):
       }, (column,))
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
-        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
-        # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+      # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+      # = 3*1 + 2*.1 = 3+.2 = 3.2
+      self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   def test_old_linear_model_old_categorical(self):
     column = fc.weighted_categorical_column(
@@ -7809,15 +8091,18 @@ class WeightedCategoricalColumnTest(test.TestCase):
       }, (column,))
       bias = get_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
-      with _initialized_session():
-        self.assertAllClose((0.,), self.evaluate(bias))
-        self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
-        self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
-        weight_var.assign(((1.,), (2.,), (3.,))).eval()
-        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
-        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
-        # = 3*1 + 2*.1 = 3+.2 = 3.2
-        self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
+
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose((0.,), self.evaluate(bias))
+      self.assertAllClose(((0.,), (0.,), (0.,)), self.evaluate(weight_var))
+      self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions))
+      self.evaluate(weight_var.assign(((1.,), (2.,), (3.,))))
+      # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+      # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+      # = 3*1 + 2*.1 = 3+.2 = 3.2
+      self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
   # TODO(ptucker): Add test with embedding of weighted categorical.
 
-- 
GitLab


From 5916a9f0e4b5b2c4f80767ff83a001a6f86b4395 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 18:39:40 -0800
Subject: [PATCH 0994/1554]  tf.lite.abs support

PiperOrigin-RevId: 223273635
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 11 +++++++++
 tensorflow/lite/kernels/elementwise.cc        | 13 ++++++++++
 tensorflow/lite/kernels/elementwise_test.cc   | 13 ++++++++++
 tensorflow/lite/kernels/register.cc           |  2 ++
 tensorflow/lite/testing/generate_examples.py  | 24 +++++++++++++++++++
 .../propagate_fixed_sizes.cc                  |  1 +
 tensorflow/lite/toco/import_tensorflow.cc     |  1 +
 tensorflow/lite/toco/model.h                  | 14 ++++++++++-
 tensorflow/lite/toco/tflite/export.cc         |  1 -
 tensorflow/lite/toco/tflite/operator.cc       |  4 +++-
 tensorflow/lite/toco/tooling_util.cc          |  1 +
 12 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 4a49c14cc9..3a31144593 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -221,6 +221,7 @@ def json_to_tflite(name, src, out):
 # generated_test_models_failing().
 def generated_test_models():
     return [
+        "abs",
         "add",
         "arg_min_max",
         "avg_pool",
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index 2864c6aaf4..a96f64cefe 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -139,6 +139,17 @@ following common ops are not supported at the moment:
 The following TensorFlow Lite operations are fully supported and used in place
 of the TensorFlow operations listed above:
 
+**ABS**
+
+```
+Inputs {
+  0: a tensor
+}
+Outputs {
+  0: elementwise abs of the input
+}
+```
+
 **ADD**
 
 ```
diff --git a/tensorflow/lite/kernels/elementwise.cc b/tensorflow/lite/kernels/elementwise.cc
index 416a69eb0e..a79388b900 100644
--- a/tensorflow/lite/kernels/elementwise.cc
+++ b/tensorflow/lite/kernels/elementwise.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <cmath>
 #include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
 #include "tensorflow/lite/kernels/internal/tensor.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
 
@@ -74,6 +75,10 @@ inline TfLiteStatus EvalLogical(TfLiteContext* context, TfLiteNode* node,
   return EvalImpl<bool>(context, node, bool_func, kTfLiteBool);
 }
 
+TfLiteStatus AbsEval(TfLiteContext* context, TfLiteNode* node) {
+  return EvalNumeric(context, node, std::abs);
+}
+
 TfLiteStatus SinEval(TfLiteContext* context, TfLiteNode* node) {
   return EvalNumeric(context, node, std::sin);
 }
@@ -101,6 +106,14 @@ TfLiteStatus LogicalNotEval(TfLiteContext* context, TfLiteNode* node) {
 }  // namespace
 }  // namespace elementwise
 
+TfLiteRegistration* Register_ABS() {
+  static TfLiteRegistration r = {
+      /*init=*/nullptr, /*free=*/nullptr,
+      elementwise::GenericPrepare<elementwise::IsNumericSupportedType>,
+      elementwise::AbsEval};
+  return &r;
+}
+
 TfLiteRegistration* Register_SIN() {
   static TfLiteRegistration r = {
       /*init=*/nullptr, /*free=*/nullptr,
diff --git a/tensorflow/lite/kernels/elementwise_test.cc b/tensorflow/lite/kernels/elementwise_test.cc
index 52df8dc3cc..7d24320081 100644
--- a/tensorflow/lite/kernels/elementwise_test.cc
+++ b/tensorflow/lite/kernels/elementwise_test.cc
@@ -74,6 +74,19 @@ TEST(ElementWise, Log) {
   EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({1, 1, 4, 1}));
 }
 
+TEST(FloatActivationsOpTest, Abs) {
+  ElementWiseOpFloatModel m(BuiltinOperator_ABS, {1, 2, 4, 1});
+  m.PopulateTensor<float>(m.input(), {
+                                         0.f, -6.2f, 2.f, 4.f,  //
+                                         3.f, -2.f, 10.f, 1.f,  //
+                                     });
+  m.Invoke();
+  EXPECT_THAT(m.ExtractVector<float>(m.output()), ElementsAreArray({
+                                                      0.f, 6.2f, 2.f, 4.f,  //
+                                                      3.f, 2.f, 10.f, 1.f,  //
+                                                  }));
+}
+
 TEST(ElementWise, Sqrt) {
   ElementWiseOpFloatModel m(BuiltinOperator_SQRT, {1, 1, 4, 1});
   m.PopulateTensor<float>(m.input(), {0, 1, 2, 4});
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index f4aa5cc438..5f48aacf59 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -31,6 +31,7 @@ TfLiteRegistration* Register_RELU_1();
 
 namespace builtin {
 
+TfLiteRegistration* Register_ABS();
 TfLiteRegistration* Register_RELU();
 TfLiteRegistration* Register_RELU_N1_TO_1();
 TfLiteRegistration* Register_RELU6();
@@ -154,6 +155,7 @@ const TfLiteRegistration* BuiltinOpResolver::FindOp(const char* op,
 }
 
 BuiltinOpResolver::BuiltinOpResolver() {
+  AddBuiltin(BuiltinOperator_ABS, Register_ABS());
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
   AddBuiltin(BuiltinOperator_RELU_N1_TO_1, Register_RELU_N1_TO_1());
   AddBuiltin(BuiltinOperator_RELU6, Register_RELU6());
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 71382edaba..566723cdc2 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -626,6 +626,30 @@ def make_max_pool_tests(zip_path):
   make_pool_tests(tf.nn.max_pool)(zip_path)
 
 
+def make_abs_tests(zip_path):
+  """Make a set of tests to do relu."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3],
+                      [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.abs(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-10, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_relu_tests(zip_path):
   """Make a set of tests to do relu."""
 
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 664424860e..b520f7f559 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1707,6 +1707,7 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) {
   }
 
   switch (op->type) {
+    case OperatorType::kAbs:
     case OperatorType::kBatchNormalization:
     case OperatorType::kL2Normalization:
     case OperatorType::kDequantize:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index b51f80c1a7..e42c2e2f20 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2255,6 +2255,7 @@ ConverterMapType GetTensorFlowNodeConverterMapForFlex() {
 
 ConverterMapType GetTensorFlowNodeConverterMap() {
   return std::unordered_map<std::string, ConverterType>({
+      {"Abs", ConvertSimpleOperator<AbsOperator>},
       {"Add", ConvertSimpleOperator<AddOperator, 2>},
       {"AddN", ConvertSimpleOperatorFlexOk<AddNOperator>},
       {"All", ConvertSimpleOperator<TensorFlowAllOperator>},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 92be42f47c..a4883e4694 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -154,7 +154,8 @@ enum class OperatorType : uint8 {
   kUnpack,
   kZerosLike,
   kResizeNearestNeighbor,
-  kLeakyRelu
+  kLeakyRelu,
+  kAbs
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -655,6 +656,17 @@ struct MulOperator : Operator {
   MulOperator() : Operator(OperatorType::kMul) {}
 };
 
+// Element-wise Abs operator:
+//   x -> abs(x)
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Relu
+struct AbsOperator : Operator {
+  AbsOperator() : Operator(OperatorType::kAbs) {}
+};
+
 // Element-wise Relu operator:
 //   x -> max(0, x)
 //
diff --git a/tensorflow/lite/toco/tflite/export.cc b/tensorflow/lite/toco/tflite/export.cc
index f17ce900eb..8b9448486d 100644
--- a/tensorflow/lite/toco/tflite/export.cc
+++ b/tensorflow/lite/toco/tflite/export.cc
@@ -126,7 +126,6 @@ OperatorKey::OperatorKey(
     type_ = builtin_ops.at(name);
     return;
   }
-
   // The logic below is all for custom ops or Flex ops.
   is_custom_op_ = true;
   type_ = BuiltinOperator_CUSTOM;
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 83325f1f79..b28781c32e 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1484,6 +1484,7 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
                                     OperatorType::kMaxPool));
   ops.push_back(
       MakeUnique<Mul>(::tflite::BuiltinOperator_MUL, OperatorType::kMul));
+
   ops.push_back(
       MakeUnique<Pad>(::tflite::BuiltinOperator_PAD, OperatorType::kPad));
   ops.push_back(
@@ -1642,7 +1643,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       "SQUARE", OperatorType::kSquare));
   ops.push_back(MakeUnique<SimpleOperator<TensorFlowZerosLikeOperator>>(
       "ZEROS_LIKE", OperatorType::kZerosLike));
-
+  ops.push_back(
+      MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
   return ops;
 }
 }  // namespace
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 611add9daf..e3f3fe36f7 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -308,6 +308,7 @@ const char* OperatorTypeName(OperatorType type) {
 #define HANDLE_OPERATORTYPENAME_CASE(c) \
   case OperatorType::k##c:              \
     return #c;
+    HANDLE_OPERATORTYPENAME_CASE(Abs)
     HANDLE_OPERATORTYPENAME_CASE(Add)
     HANDLE_OPERATORTYPENAME_CASE(AddN)
     HANDLE_OPERATORTYPENAME_CASE(AveragePool)
-- 
GitLab


From c7d52fa78a67cd02bfa96f310308e1edeb8bb356 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 28 Nov 2018 18:40:58 -0800
Subject: [PATCH 0995/1554] Add publicly available corpus for string_split
 fuzz.

PiperOrigin-RevId: 223273755
---
 .../string_split/4c01a1504da9de2216894743ecc44424  |   1 +
 .../string_split/5bf16424630b5afbcffe711fb9834440  |   1 +
 .../string_split/a7185605aef0a8fd682fcb4656e4a736  |   1 +
 .../string_split/d5606def44fdbb9385dd764612069db0  | Bin 0 -> 42 bytes
 .../string_split/dbac766f3160de65894bf5153f478146  |   1 +
 .../string_split/e85ff62f6d457666f54a37a19a115a24  |   1 +
 .../core/kernels/fuzzing/string_split_fuzz.cc      |   3 +--
 7 files changed, 6 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/d5606def44fdbb9385dd764612069db0
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24

diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424 b/tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424
new file mode 100644
index 0000000000..eb84b9e610
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/4c01a1504da9de2216894743ecc44424
@@ -0,0 +1 @@
+./,abcd.efgh/abcd,efgh.abcd/efgh,abcd.efgh/a
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440 b/tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440
new file mode 100644
index 0000000000..4cd522da7b
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/5bf16424630b5afbcffe711fb9834440
@@ -0,0 +1 @@
+
.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736 b/tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736
new file mode 100644
index 0000000000..03cfb6256f
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/a7185605aef0a8fd682fcb4656e4a736
@@ -0,0 +1 @@
+./, abcde.fghab/cdefg,habcd efgha.bcdef/ghabc
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/d5606def44fdbb9385dd764612069db0 b/tensorflow/core/kernels/fuzzing/corpus/string_split/d5606def44fdbb9385dd764612069db0
new file mode 100644
index 0000000000000000000000000000000000000000..304b0d66fe08fd1a29827488727702dd9b9bce3e
GIT binary patch
literal 42
ScmZQbOiE5kO-s)pNCN;-W)Kzt

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146 b/tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146
new file mode 100644
index 0000000000..a8740444aa
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/dbac766f3160de65894bf5153f478146
@@ -0,0 +1 @@
+./, ?abcdef.ghabcd/efghab,cdefgh abcdef?ghabcd
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24 b/tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24
new file mode 100644
index 0000000000..47d551466a
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split/e85ff62f6d457666f54a37a19a115a24
@@ -0,0 +1 @@
+./abc.def/gha.bcd/efg.hab/cde.fgh/abc.def/g
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc b/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc
index 87a548a999..2564f8ed03 100644
--- a/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc
+++ b/tensorflow/core/kernels/fuzzing/string_split_fuzz.cc
@@ -37,8 +37,7 @@ class FuzzStringSplit : public FuzzSession {
       // The spec for split is that the delimeter should be 0 or 1 characters.
       // Naturally, fuzz it with something larger.  (This omits the possibility
       // of handing it a > int32_max size string, which should be tested for in
-      // an
-      // explicit test).
+      // an explicit test).
       size_t delim_len = static_cast<size_t>(data[0]);
       if (delim_len > size) {
         delim_len = size - 1;
-- 
GitLab


From 16c0e72b92497587b6f7e1614869066ed122c9e4 Mon Sep 17 00:00:00 2001
From: Sherry Yang <sherryy@google.com>
Date: Wed, 28 Nov 2018 18:46:28 -0800
Subject: [PATCH 0996/1554] Dump graph after graph optimization passes.

PiperOrigin-RevId: 223274324
---
 tensorflow/core/BUILD                         |   2 +
 .../common_runtime/optimization_registry.cc   |  14 ++
 .../grappler/optimizers/meta_optimizer.cc     |   7 +
 tensorflow/core/util/dump_graph.cc            | 121 ++++++++++++++++++
 tensorflow/core/util/dump_graph.h             |  48 +++++++
 tensorflow/core/util/dump_graph_test.cc       |  62 +++++++++
 6 files changed, 254 insertions(+)
 create mode 100644 tensorflow/core/util/dump_graph.cc
 create mode 100644 tensorflow/core/util/dump_graph.h
 create mode 100644 tensorflow/core/util/dump_graph_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 4313d4f1a8..962780836e 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -911,6 +911,7 @@ tf_cuda_library(
         "util/bcast.h",
         "util/cuda_kernel_helper.h",
         "util/device_name_utils.h",
+        "util/dump_graph.h",
         "util/events_writer.h",
         "util/example_proto_fast_parsing.h",
         "util/example_proto_helper.h",
@@ -3731,6 +3732,7 @@ tf_cc_tests(
         "util/bcast_test.cc",
         "util/command_line_flags_test.cc",
         "util/device_name_utils_test.cc",
+        "util/dump_graph_test.cc",
         "util/equal_graph_def_test.cc",
         "util/events_writer_test.cc",
         "util/example_proto_fast_parsing_test.cc",
diff --git a/tensorflow/core/common_runtime/optimization_registry.cc b/tensorflow/core/common_runtime/optimization_registry.cc
index 6ac047295d..9be540b019 100644
--- a/tensorflow/core/common_runtime/optimization_registry.cc
+++ b/tensorflow/core/common_runtime/optimization_registry.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/util/dump_graph.h"
 
 namespace tensorflow {
 
@@ -39,6 +40,19 @@ Status OptimizationPassRegistry::RunGrouping(
         VLOG(1) << "Running optimization pass: " << pass->name();
         Status s = pass->Run(options);
         if (!s.ok()) return s;
+        if (VLOG_IS_ON(1)) {
+          DumpGraphToFile(
+              strings::StrCat("after_phase_", phase.first, "_", pass->name()),
+              **options.graph);
+          if (options.partition_graphs) {
+            for (auto& part : *options.partition_graphs) {
+              DumpGraphToFile(
+                  strings::StrCat("after_phase_", phase.first, "_",
+                                  pass->name(), "_partition_", part.first),
+                  *part.second);
+            }
+          }
+        }
       }
     }
   }
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 6975fa715b..304ddc7710 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/util/dump_graph.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
@@ -462,6 +463,9 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   // optimize TPU functions with Grappler, this check preserves that.
   if (IsTPUGraphDef(*optimized_graph)) {
     VLOG(2) << "Skipping optimizing funcs for TPU graphs";
+    if (VLOG_IS_ON(1)) {
+      DumpGraphDefToFile("after_MetaOptimizer", *optimized_graph);
+    }
     return Status::OK();
   }
 
@@ -555,6 +559,9 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   VLOG(1) << "Optimized " << optimized_funcs.size()
           << " functions: " << str_util::Join(optimized_funcs, ", ");
 
+  if (VLOG_IS_ON(1)) {
+    DumpGraphDefToFile("after_MetaOptimizer", *optimized_graph);
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/core/util/dump_graph.cc b/tensorflow/core/util/dump_graph.cc
new file mode 100644
index 0000000000..5df5cb51cb
--- /dev/null
+++ b/tensorflow/core/util/dump_graph.cc
@@ -0,0 +1,121 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Helper functions for dumping Graphs, GraphDefs, and FunctionDefs to files for
+// debugging.
+
+#include "tensorflow/core/util/dump_graph.h"
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/core/lib/strings/proto_serialization.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+
+namespace tensorflow {
+
+namespace {
+
+struct NameCounts {
+  mutex counts_mutex;
+  std::unordered_map<string, int> counts;
+};
+
+string MakeUniqueFilename(string name) {
+  static NameCounts& instance = *new NameCounts;
+
+  // Remove illegal characters from `name`.
+  for (int i = 0; i < name.size(); ++i) {
+    char ch = name[i];
+    if (ch == '/' || ch == '[' || ch == ']' || ch == '*' || ch == '?') {
+      name[i] = '_';
+    }
+  }
+
+  int count;
+  {
+    mutex_lock lock(instance.counts_mutex);
+    count = instance.counts[name]++;
+  }
+
+  string filename = name;
+  if (count > 0) {
+    absl::StrAppend(&filename, "_", count);
+  }
+  absl::StrAppend(&filename, ".pbtxt");
+  return filename;
+}
+
+#if defined(TENSORFLOW_LITE_PROTOS)
+Status WriteToFile(const string& filepath,
+                   const ::tensorflow::protobuf::MessageLite& proto) {
+  string s;
+  if (!SerializeToStringDeterministic(proto, &s)) {
+    return errors::Internal("Failed to serialize proto to string.");
+  }
+  return WriteStringToFile(Env::Default(), filepath, s);
+}
+#else
+Status WriteToFile(const string& filepath,
+                   const ::tensorflow::protobuf::Message& proto) {
+  return WriteTextProto(Env::Default(), filepath, proto);
+}
+#endif
+
+template <class T>
+string WriteTextProtoToUniqueFile(Env* env, const string& name,
+                                  const char* proto_type, T& proto) {
+  const char* dirname = getenv("TF_DUMP_GRAPH_PREFIX");
+  if (!dirname) {
+    return "(TF_DUMP_GRAPH_PREFIX not specified)";
+  }
+  Status status = env->RecursivelyCreateDir(dirname);
+  if (!status.ok()) {
+    LOG(WARNING) << "Failed to create " << dirname << " for dumping "
+                 << proto_type << ": " << status;
+    return "(unavailable)";
+  }
+  string filepath = absl::StrCat(dirname, "/", MakeUniqueFilename(name));
+  status = WriteToFile(filepath, proto);
+  if (!status.ok()) {
+    LOG(WARNING) << "Failed to dump " << proto_type << " to file: " << filepath
+                 << " : " << status;
+    return "(unavailable)";
+  }
+  LOG(INFO) << "Dumped " << proto_type << " to " << filepath;
+  return filepath;
+}
+
+}  // anonymous namespace
+
+string DumpGraphDefToFile(const string& name, GraphDef const& graph_def) {
+  return WriteTextProtoToUniqueFile(Env::Default(), name, "GraphDef",
+                                    graph_def);
+}
+
+string DumpGraphToFile(const string& name, Graph const& graph,
+                       const FunctionLibraryDefinition* flib_def) {
+  GraphDef graph_def;
+  graph.ToGraphDef(&graph_def);
+  if (flib_def) {
+    *graph_def.mutable_library() = flib_def->ToProto();
+  }
+  return DumpGraphDefToFile(name, graph_def);
+}
+
+string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef) {
+  return WriteTextProtoToUniqueFile(Env::Default(), name, "FunctionDef", fdef);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/util/dump_graph.h b/tensorflow/core/util/dump_graph.h
new file mode 100644
index 0000000000..05e0b79f55
--- /dev/null
+++ b/tensorflow/core/util/dump_graph.h
@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Helper functions for dumping Graphs, GraphDefs, and FunctionDefs to files for
+// debugging.
+
+#ifndef TENSORFLOW_CORE_UTIL_DUMP_GRAPH_H_
+#define TENSORFLOW_CORE_UTIL_DUMP_GRAPH_H_
+
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/graph.h"
+
+namespace tensorflow {
+
+// Dumps 'graph_def' to a file, as a GraphDef text proto. Returns the file name
+// chosen.
+//
+// Automatically picks a file name. Prefixes 'name' with the value of the
+// TF_DUMP_GRAPH_PREFIX environment variable and suffixes it with ".pbtxt" to
+// form a name. If a graph has already been dumped by this process with the same
+// name, suffixes with "_n.pbtxt", where 'n' is a sequence number.
+string DumpGraphDefToFile(const string& name, GraphDef const& graph_def);
+
+// Similar to DumpGraphDefToFile, but builds the GraphDef to dump from a 'graph'
+// and an optional function library 'flib_def'. Returns the file name chosen.
+string DumpGraphToFile(const string& name, Graph const& graph,
+                       const FunctionLibraryDefinition* flib_def = nullptr);
+
+// Similar to DumpGraphDefToFile, but dumps a function as a FunctionDef text
+// proto. Returns the file name chosen.
+string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef);
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_UTIL_DUMP_GRAPH_H_
diff --git a/tensorflow/core/util/dump_graph_test.cc b/tensorflow/core/util/dump_graph_test.cc
new file mode 100644
index 0000000000..d01c1c5a02
--- /dev/null
+++ b/tensorflow/core/util/dump_graph_test.cc
@@ -0,0 +1,62 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/util/dump_graph.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/proto_serialization.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+TEST(DumpGraph, DumpGraphToFileSuccess) {
+  Graph graph(OpRegistry::Global());
+  Node* node;
+  TF_CHECK_OK(NodeBuilder("A", "NoOp").Finalize(&graph, &node));
+
+  setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1);
+  string ret = DumpGraphToFile("graph", graph);
+  EXPECT_EQ(ret, io::JoinPath(testing::TmpDir(), "graph.pbtxt"));
+  ret = DumpGraphToFile("graph", graph);
+  EXPECT_EQ(ret, io::JoinPath(testing::TmpDir(), "graph_1.pbtxt"));
+
+  GraphDef gdef;
+  TF_CHECK_OK(ReadTextProto(
+      Env::Default(), io::JoinPath(testing::TmpDir(), "graph.pbtxt"), &gdef));
+  string read, written;
+  gdef.AppendToString(&read);
+  graph.ToGraphDefDebug().AppendToString(&written);
+  EXPECT_EQ(read, written);
+}
+
+TEST(DumpGraph, DumpGraphToFileNoEnvPrefix) {
+  Graph graph(OpRegistry::Global());
+  unsetenv("TF_DUMP_GRAPH_PREFIX");
+  string ret = DumpGraphToFile("graph", graph);
+  EXPECT_EQ(ret, "(TF_DUMP_GRAPH_PREFIX not specified)");
+}
+
+TEST(DumpGraph, DumpFunctionDefToFileSuccess) {
+  FunctionDef fdef;
+  setenv("TF_DUMP_GRAPH_PREFIX", testing::TmpDir().c_str(), 1);
+  string ret = DumpFunctionDefToFile("function", fdef);
+  EXPECT_EQ(ret, io::JoinPath(testing::TmpDir(), "function.pbtxt"));
+}
+
+}  // namespace
+}  // namespace tensorflow
-- 
GitLab


From 5cf68508e9223adc911d347a1dd89d547310617a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 19:15:32 -0800
Subject: [PATCH 0997/1554] TensorFlow 2.0 fixups for tf.sparse.{add, concat}.
 Fix their bogus rewrite rules.

PiperOrigin-RevId: 223276761
---
 tensorflow/python/ops/sparse_ops.py           | 107 ++++++++++--------
 .../tools/api/golden/v1/tensorflow.pbtxt      |   2 +-
 .../api/golden/v1/tensorflow.sparse.pbtxt     |   2 +-
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 .../api/golden/v2/tensorflow.sparse.pbtxt     |   2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  20 +++-
 6 files changed, 77 insertions(+), 60 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 91baa6f7b8..245080cb26 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -185,12 +185,6 @@ def sparse_eye(num_rows,
         dense_shape=[num_rows, num_columns])
 
 
-@tf_export("sparse.concat", "sparse_concat", v1=[])
-def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim=False,
-                     concat_dim=None, name=None):
-  return sparse_concat(axis, sp_inputs, name, expand_nonconcat_dim, concat_dim)
-
-
 # pylint: disable=protected-access
 @tf_export(v1=["sparse.concat", "sparse_concat"])
 @deprecation.deprecated_endpoints("sparse_concat")
@@ -298,6 +292,11 @@ def sparse_concat(axis,
   """
   axis = deprecation.deprecated_argument_lookup("axis", axis, "concat_dim",
                                                 concat_dim)
+  return sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim, name)
+
+
+@tf_export("sparse.concat", v1=[])
+def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim=False, name=None):  # pylint: disable=missing-docstring
   sp_inputs = _convert_to_sparse_tensors(sp_inputs)
 
   if len(sp_inputs) == 1:  # Degenerate case of one tensor.
@@ -325,9 +324,15 @@ def sparse_concat(axis,
   return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
 
 
+sparse_concat_v2.__doc__ = sparse_concat.__doc__.replace(
+    "    concat_dim: The old (deprecated) name for axis.\n", "")
+
+
 @tf_export(v1=["sparse.add", "sparse_add"])
 @deprecation.deprecated_endpoints("sparse_add")
-def sparse_add(a, b, thresh=0):
+@deprecation.deprecated_args(
+    None, "thresh is deprecated, use threshold instead", "thresh")
+def sparse_add(a, b, threshold=None, thresh=None):
   """Adds two tensors, at least one of each is a `SparseTensor`.
 
   If one `SparseTensor` and one `Tensor` are passed in, returns a `Tensor`.  If
@@ -365,12 +370,14 @@ def sparse_add(a, b, thresh=0):
 
   Args:
     a: The first operand; `SparseTensor` or `Tensor`.
-    b: The second operand; `SparseTensor` or `Tensor`.  At least one operand
+    b: The second operand; `SparseTensor` or `Tensor`. At least one operand
       must be sparse.
-    thresh: A 0-D `Tensor`.  The magnitude threshold that determines if an
-    output value/index pair takes space.  Its dtype should match that of the
-    values if they are real; if the latter are complex64/complex128, then the
-    dtype should be float32/float64, correspondingly.
+    threshold: An optional 0-D `Tensor` (defaults to `0`). The magnitude
+      threshold that determines if an output value/index pair takes space. Its
+      dtype should match that of the values if they are real; if the latter are
+      complex64/complex128, then the dtype should be float32/float64,
+      correspondingly.
+    thresh: Deprecated alias for `threshold`.
 
   Returns:
     A `SparseTensor` or a `Tensor`, representing the sum.
@@ -378,34 +385,11 @@ def sparse_add(a, b, thresh=0):
   Raises:
     TypeError: If both `a` and `b` are `Tensor`s.  Use `tf.add()` instead.
   """
-  sparse_classes = (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)
-  if not any(isinstance(inp, sparse_classes) for inp in [a, b]):
-    raise TypeError("At least one input should be SparseTensor; do you mean to"
-                    " use tf.add()?")
-
-  if all(isinstance(inp, sparse_classes) for inp in [a, b]):
-    a = _convert_to_sparse_tensor(a)
-    b = _convert_to_sparse_tensor(b)
-    thresh = ops.convert_to_tensor(
-        thresh, dtype=a.values.dtype.real_dtype.base_dtype, name="thresh")
-    output_ind, output_val, output_shape = (
-        gen_sparse_ops.sparse_add(a.indices, a.values, a.dense_shape,
-                                  b.indices, b.values, b.dense_shape, thresh))
-
-    # Attempt to get output_shape statically.
-    a.get_shape().assert_is_compatible_with(b.get_shape())
-    static_shape = array_ops.broadcast_static_shape(a.get_shape(),
-                                                    b.get_shape())
-    if static_shape.is_fully_defined():
-      output_shape = static_shape.as_list()
-
-    return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
-  else:
-    # swap to make `a` the SparseTensor.
-    if isinstance(b, sparse_classes):
-      a, b = b, a
-    return gen_sparse_ops.sparse_tensor_dense_add(a.indices, a.values,
-                                                  a.dense_shape, b)
+  threshold = deprecation.deprecated_argument_lookup("threshold", threshold,
+                                                     "thresh", thresh)
+  if threshold is None:
+    threshold = 0
+  return sparse_add_v2(a, b, threshold)
 
 
 @tf_export("sparse.add", v1=[])
@@ -448,12 +432,12 @@ def sparse_add_v2(a, b, threshold=0):
 
   Args:
     a: The first operand; `SparseTensor` or `Tensor`.
-    b: The second operand; `SparseTensor` or `Tensor`.  At least one operand
+    b: The second operand; `SparseTensor` or `Tensor`. At least one operand
       must be sparse.
-    threshold: A 0-D `Tensor`.  The magnitude threshold that determines if an
-    output value/index pair takes space.  Its dtype should match that of the
-    values if they are real; if the latter are complex64/complex128, then the
-    dtype should be float32/float64, correspondingly.
+    threshold: A 0-D `Tensor`. The magnitude threshold that determines if an
+      output value/index pair takes space. Its dtype should match that of the
+      values if they are real; if the latter are complex64/complex128, then the
+      dtype should be float32/float64, correspondingly.
 
   Returns:
     A `SparseTensor` or a `Tensor`, representing the sum.
@@ -461,10 +445,35 @@ def sparse_add_v2(a, b, threshold=0):
   Raises:
     TypeError: If both `a` and `b` are `Tensor`s.  Use `tf.add()` instead.
   """
-  return sparse_add(
-      a=a,
-      b=b,
-      thresh=threshold)
+  sparse_classes = (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)
+  if not any(isinstance(inp, sparse_classes) for inp in [a, b]):
+    raise TypeError("At least one input should be SparseTensor; do you mean to"
+                    " use tf.add()?")
+
+  if all(isinstance(inp, sparse_classes) for inp in [a, b]):
+    a = _convert_to_sparse_tensor(a)
+    b = _convert_to_sparse_tensor(b)
+    threshold = ops.convert_to_tensor(
+        threshold, dtype=a.values.dtype.real_dtype.base_dtype, name="threshold")
+    output_ind, output_val, output_shape = (
+        gen_sparse_ops.sparse_add(a.indices, a.values, a.dense_shape,
+                                  b.indices, b.values, b.dense_shape,
+                                  threshold))
+
+    # Attempt to get output_shape statically.
+    a.get_shape().assert_is_compatible_with(b.get_shape())
+    static_shape = array_ops.broadcast_static_shape(a.get_shape(),
+                                                    b.get_shape())
+    if static_shape.is_fully_defined():
+      output_shape = static_shape.as_list()
+
+    return sparse_tensor.SparseTensor(output_ind, output_val, output_shape)
+  else:
+    # swap to make `a` the SparseTensor.
+    if isinstance(b, sparse_classes):
+      a, b = b, a
+    return gen_sparse_ops.sparse_tensor_dense_add(a.indices, a.values,
+                                                  a.dense_shape, b)
 
 
 @tf_export("sparse.cross")
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index a294e3e027..9c836c5bf8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1982,7 +1982,7 @@ tf_module {
   }
   member_method {
     name: "sparse_add"
-    argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], "
+    argspec: "args=[\'a\', \'b\', \'threshold\', \'thresh\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "sparse_concat"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
index ee4f31774e..33e342bc75 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.pbtxt
@@ -10,7 +10,7 @@ tf_module {
   }
   member_method {
     name: "add"
-    argspec: "args=[\'a\', \'b\', \'thresh\'], varargs=None, keywords=None, defaults=[\'0\'], "
+    argspec: "args=[\'a\', \'b\', \'threshold\', \'thresh\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "concat"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 40693e7582..f2ae039d9f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1020,10 +1020,6 @@ tf_module {
     name: "space_to_batch_nd"
     argspec: "args=[\'input\', \'block_shape\', \'paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "sparse_concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'concat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\'], "
-  }
   member_method {
     name: "split"
     argspec: "args=[\'value\', \'num_or_size_splits\', \'axis\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \'split\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index 4ad94568b2..9808200d72 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -14,7 +14,7 @@ tf_module {
   }
   member_method {
     name: "concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'concat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\'], "
+    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
   member_method {
     name: "cross"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index d75e52bd0d..649d1c17a3 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -102,7 +102,18 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "t": "x",
             "msg": "message",
         },
-        "tf.sparse.add": ["a", "b", "thresh"],
+        "tf.sparse.add": {
+            "thresh": "threshold",
+        },
+        "tf.sparse_add": {
+            "thresh": "threshold",
+        },
+        "tf.sparse.concat": {
+            "concat_dim": "axis",
+        },
+        "tf.sparse_concat": {
+            "concat_dim": "axis",
+        },
         "tf.sparse.split": {
             "split_dim": "axis",
         },
@@ -197,9 +208,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.strings.to_hash_bucket": {
             "string_tensor": "input",
         },
-        "tf.sparse.concat": [
-            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
-        ],
         "tf.reduce_all": {
             "reduction_indices": "axis",
             "keep_dims": "keepdims",
@@ -481,9 +489,13 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.size": ["input", "name", "out_type"],
         "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
         "tf.sparse.add": ["a", "b", "thresh"],
+        "tf.sparse_add": ["a", "b", "thresh"],
         "tf.sparse.concat": [
             "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
         ],
+        "tf.sparse_concat": [
+            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
+        ],
         "tf.sparse.segment_mean": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
-- 
GitLab


From 158bf75a3078e14839c670c109fee7d4c23ce183 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 28 Nov 2018 21:00:52 -0800
Subject: [PATCH 0998/1554] Internal-only change.

PiperOrigin-RevId: 223284903
---
 tensorflow/core/BUILD     | 3 ++-
 tensorflow/tensorflow.bzl | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 962780836e..7b46e8c9b4 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -96,6 +96,7 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_tests_gpu")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule")
 load("//tensorflow:tensorflow.bzl", "if_nccl")
+load("//tensorflow:tensorflow.bzl", "tensorflow_opensource_extra_deps")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_only_cc_test")
 
 # For platform specific build config
@@ -1469,7 +1470,7 @@ tf_cuda_library(
         ":gpu_runtime",
         ":lib",
         ":ops",
-    ],
+    ] + tensorflow_opensource_extra_deps(),
 )
 
 cc_library(
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 4bc68445ac..ed1de5a31c 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -2032,3 +2032,6 @@ register_extension_info(
     extension_name = "cc_library_with_android_deps",
     label_regex_for_dep = "{extension_name}",
 )
+
+def tensorflow_opensource_extra_deps():
+    return []
-- 
GitLab


From 454fd39652993f387bafb83329e93cce7c0db9db Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Nov 2018 21:37:25 -0800
Subject: [PATCH 0999/1554] Implement SplitV.

PiperOrigin-RevId: 223287758
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/builtin_ops.h                 |   1 +
 tensorflow/lite/c/builtin_op_data.h           |   4 +
 tensorflow/lite/c/builtin_op_data_test.cc     |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   8 +
 .../writer/option_writer_generator.cc         |   1 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md |  16 ++
 tensorflow/lite/kernels/BUILD                 |  17 ++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/kernels/split_v.cc            | 207 ++++++++++++++++++
 tensorflow/lite/kernels/split_v_test.cc       | 175 +++++++++++++++
 tensorflow/lite/nnapi_delegate.cc             |   1 +
 tensorflow/lite/schema/schema.fbs             |   6 +
 tensorflow/lite/schema/schema_generated.h     | 141 +++++++++++-
 tensorflow/lite/testing/generate_examples.py  |  27 ++-
 tensorflow/lite/toco/export_tensorflow.cc     |  27 +++
 .../propagate_array_data_types.cc             |   7 +
 .../propagate_fixed_sizes.cc                  |  96 ++++++++
 tensorflow/lite/toco/import_tensorflow.cc     |  20 ++
 tensorflow/lite/toco/model.h                  |   7 +
 tensorflow/lite/toco/tflite/operator.cc       |  22 ++
 tensorflow/lite/toco/tflite/operator_test.cc  |   8 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 23 files changed, 787 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/lite/kernels/split_v.cc
 create mode 100644 tensorflow/lite/kernels/split_v_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 3a31144593..33cee1ab20 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -294,6 +294,7 @@ def generated_test_models():
         "space_to_depth",
         "sparse_to_dense",
         "split",
+        "splitv",
         "sqrt",
         "square",
         "squared_difference",
diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index 63b5cd1960..f97d3ac4bf 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -127,6 +127,7 @@ typedef enum {
   kTfLiteBuiltinSquaredDifference = 99,
   kTfLiteBuiltinMirrorPad = 100,
   kTfLiteBuiltinAbs = 101,
+  kTfLiteBuiltinSplitV = 102,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h
index 33aaac3c80..6a5a027a9d 100644
--- a/tensorflow/lite/c/builtin_op_data.h
+++ b/tensorflow/lite/c/builtin_op_data.h
@@ -277,6 +277,10 @@ typedef struct {
   int num_splits;
 } TfLiteSplitParams;
 
+typedef struct {
+  int num_splits;
+} TfLiteSplitVParams;
+
 typedef struct {
   // TODO(ahentz): We can't have dynamic data in this struct, at least not yet.
   // For now we will fix the maximum possible number of dimensions.
diff --git a/tensorflow/lite/c/builtin_op_data_test.cc b/tensorflow/lite/c/builtin_op_data_test.cc
index 0e33dcd8c8..4ce7c481e1 100644
--- a/tensorflow/lite/c/builtin_op_data_test.cc
+++ b/tensorflow/lite/c/builtin_op_data_test.cc
@@ -63,6 +63,7 @@ TEST(IntArray, CanCompileStructs) {
   TfLiteTransposeParams transpose_params;
   TfLiteReducerParams reducer_params;
   TfLiteSplitParams split_params;
+  TfLiteSplitVParams split_v_params;
   TfLiteSqueezeParams squeeze_params;
   TfLiteStridedSliceParams strided_slice_params;
   TfLiteArgMaxParams arg_max_params;
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index d3383bccf0..c00a0a3a54 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -506,6 +506,14 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       *builtin_data = reinterpret_cast<void*>(params);
       break;
     }
+    case BuiltinOperator_SPLIT_V: {
+      auto* params = allocator->AllocatePOD<TfLiteSplitParams>();
+      if (auto* schema_params = op->builtin_options_as_SplitVOptions()) {
+        params->num_splits = schema_params->num_splits();
+      }
+      *builtin_data = reinterpret_cast<void*>(params);
+      break;
+    }
     case BuiltinOperator_SQUEEZE: {
       auto* params = allocator->AllocatePOD<TfLiteSqueezeParams>();
       if (auto* schema_params = op->builtin_options_as_SqueezeOptions()) {
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index b44750e8b2..fa360a2f47 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -56,6 +56,7 @@ static const char* param_structs[] = {"TfLiteConvParams",
                                       "TfLiteTransposeParams",
                                       "TfLiteReducerParams",
                                       "TfLiteSplitParams",
+                                      "TfLiteSplitVParams",
                                       "TfLiteSqueezeParams",
                                       "TfLiteStridedSliceParams",
                                       "TfLiteArgMaxParams",
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index a96f64cefe..6976f058d1 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -866,6 +866,22 @@ Options {
 }
 ```
 
+**SPLIT_V**
+
+```
+Inputs {
+  0: tensor (input)
+  1: 1-D tensor (size_splits)
+  2: 0-D tensor (axis)
+}
+Outputs {
+  0-N: subtensors built from the input tensors
+}
+Options {
+  num_splits: Specifies number of outputs
+}
+```
+
 **SQRT**
 
 ```
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 0bf4f01ac3..c1b005b580 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -219,6 +219,7 @@ cc_library(
         "sparse_output_fully_connected.cc",
         "sparse_to_dense.cc",
         "split.cc",
+        "split_v.cc",
         "squared_difference.cc",
         "squeeze.cc",
         "strided_slice.cc",
@@ -1065,6 +1066,22 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "split_v_test",
+    size = "small",
+    srcs = ["split_v_test.cc"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "squeeze_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 5f48aacf59..7f0b9239ee 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -75,6 +75,7 @@ TfLiteRegistration* Register_GATHER();
 TfLiteRegistration* Register_TRANSPOSE();
 TfLiteRegistration* Register_MEAN();
 TfLiteRegistration* Register_SPLIT();
+TfLiteRegistration* Register_SPLIT_V();
 TfLiteRegistration* Register_SQUEEZE();
 TfLiteRegistration* Register_STRIDED_SLICE();
 TfLiteRegistration* Register_EXP();
@@ -211,6 +212,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_DIV, Register_DIV());
   AddBuiltin(BuiltinOperator_SUB, Register_SUB());
   AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT());
+  AddBuiltin(BuiltinOperator_SPLIT_V, Register_SPLIT_V());
   AddBuiltin(BuiltinOperator_SQUEEZE, Register_SQUEEZE());
   AddBuiltin(BuiltinOperator_STRIDED_SLICE, Register_STRIDED_SLICE());
   AddBuiltin(BuiltinOperator_EXP, Register_EXP());
diff --git a/tensorflow/lite/kernels/split_v.cc b/tensorflow/lite/kernels/split_v.cc
new file mode 100644
index 0000000000..060e3c5f79
--- /dev/null
+++ b/tensorflow/lite/kernels/split_v.cc
@@ -0,0 +1,207 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <vector>
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace split_v {
+
+struct OpContext {
+  OpContext(TfLiteContext* context, TfLiteNode* node) {
+    params = reinterpret_cast<TfLiteSplitVParams*>(node->builtin_data);
+    input = GetInput(context, node, 0);
+    size_splits = GetInput(context, node, 1);
+    axis = GetInput(context, node, 2);
+  }
+  TfLiteSplitVParams* params;
+  const TfLiteTensor* input;
+  const TfLiteTensor* size_splits;
+  const TfLiteTensor* axis;
+};
+
+TfLiteStatus UseDynamicOutputTensors(TfLiteContext* context, TfLiteNode* node) {
+  for (int i = 0; i < NumOutputs(node); ++i) {
+    SetTensorToDynamic(GetOutput(context, node, i));
+  }
+  return kTfLiteOk;
+}
+
+template <typename T>
+void GetSizeSplitsVector(const TfLiteTensor* size_splits,
+                         std::vector<int64_t>* size_splits_vector) {
+  const auto num_elements = NumElements(size_splits);
+  for (int i = 0; i < num_elements; ++i) {
+    size_splits_vector->push_back(GetTensorData<T>(size_splits)[i]);
+  }
+}
+
+TfLiteStatus ResizeOutputTensors(TfLiteContext* context, TfLiteNode* node,
+                                 const TfLiteTensor* input,
+                                 const TfLiteTensor* size_splits,
+                                 const TfLiteTensor* axis) {
+  int axis_value = GetTensorData<int>(axis)[0];
+  if (axis_value < 0) {
+    axis_value += NumDimensions(input);
+  }
+
+  std::vector<int64_t> size_splits_vector;
+  if (size_splits->type == kTfLiteInt32) {
+    GetSizeSplitsVector<int32_t>(size_splits, &size_splits_vector);
+  } else if (size_splits->type == kTfLiteInt64) {
+    GetSizeSplitsVector<int64_t>(size_splits, &size_splits_vector);
+  } else {
+    context->ReportError(context, "size_splits only support type int32|int64.");
+    return kTfLiteError;
+  }
+
+  int minus_one_index = -1;
+  int64_t size_splits_sum = 0;
+
+  for (int i = 0; i < size_splits_vector.size(); ++i) {
+    if (size_splits_vector.at(i) == -1) {
+      if (minus_one_index == -1) {
+        minus_one_index = i;
+      } else {
+        context->ReportError(context,
+                             "The size_splits contains more than one -1.");
+      }
+    } else {
+      size_splits_sum += size_splits_vector.at(i);
+    }
+  }
+
+  const int input_size = SizeOfDimension(input, axis_value);
+
+  if (minus_one_index != -1) {
+    if (size_splits_sum > input_size) {
+      context->ReportError(
+          context,
+          "The sum of size_splits must be less than the dimension of value.");
+    } else {
+      size_splits_vector[minus_one_index] = input_size - size_splits_sum;
+    }
+  } else if (size_splits_sum != input_size) {
+    context->ReportError(
+        context,
+        "The size_splits must sum to the dimension of value along axis.");
+  }
+
+  for (int i = 0; i < NumOutputs(node); ++i) {
+    TfLiteIntArray* output_dims = TfLiteIntArrayCopy(input->dims);
+    output_dims->data[axis_value] = size_splits_vector.at(i);
+    TfLiteTensor* output = GetOutput(context, node, i);
+    TF_LITE_ENSURE_STATUS(context->ResizeTensor(context, output, output_dims));
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 3);
+
+  OpContext op_context(context, node);
+
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), op_context.params->num_splits);
+
+  auto input_type = op_context.input->type;
+  TF_LITE_ENSURE(context, input_type == kTfLiteFloat32 ||
+                              input_type == kTfLiteUInt8 ||
+                              input_type == kTfLiteInt16);
+  for (int i = 0; i < NumOutputs(node); ++i) {
+    GetOutput(context, node, i)->type = input_type;
+  }
+
+  auto size_splits = op_context.size_splits;
+  TF_LITE_ENSURE_EQ(context, NumDimensions(size_splits), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), NumElements(size_splits));
+
+  // If we know the contents of the 'size_splits' tensor and the 'axis' tensor,
+  // resize all outputs. Otherwise, wait until Eval().
+  if (IsConstantTensor(op_context.size_splits) &&
+      IsConstantTensor(op_context.axis)) {
+    return ResizeOutputTensors(context, node, op_context.input,
+                               op_context.size_splits, op_context.axis);
+  } else {
+    return UseDynamicOutputTensors(context, node);
+  }
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  OpContext op_context(context, node);
+
+  // When the 'size_splits' and the 'axis' tensor is non-const we can't resize
+  // output tensors in Prepare(), and we have to do it now.
+  if (!IsConstantTensor(op_context.axis) ||
+      !IsConstantTensor(op_context.size_splits)) {
+    TF_LITE_ENSURE_OK(
+        context, ResizeOutputTensors(context, node, op_context.input,
+                                     op_context.size_splits, op_context.axis));
+  }
+
+  int axis_value = GetTensorData<int>(op_context.axis)[0];
+
+  // Use split function to build the outputs since they share the same logic.
+#define TF_LITE_SPLIT_V(scalar)                                     \
+  VectorOfTensors<scalar> all_outputs(*context, *node->outputs);    \
+  tflite::SplitParams op_params;                                    \
+  op_params.num_split = NumOutputs(node);                           \
+  op_params.axis = axis_value;                                      \
+  reference_ops::Split(op_params, GetTensorShape(op_context.input), \
+                       GetTensorData<scalar>(op_context.input),     \
+                       all_outputs.shapes(), all_outputs.data());
+  switch (op_context.input->type) {
+    case kTfLiteFloat32: {
+      TF_LITE_SPLIT_V(float);
+      break;
+    }
+    case kTfLiteUInt8: {
+      TF_LITE_SPLIT_V(uint8_t);
+      break;
+    }
+    case kTfLiteInt16: {
+      TF_LITE_SPLIT_V(int16_t);
+      break;
+    }
+    default:
+      context->ReportError(
+          context,
+          "Only float32, uint8 and int16 are currently supported, got %d.",
+          op_context.input->type);
+      return kTfLiteError;
+  }
+#undef TF_LITE_SPLIT_V
+
+  return kTfLiteOk;
+}
+
+}  // namespace split_v
+
+TfLiteRegistration* Register_SPLIT_V() {
+  static TfLiteRegistration r = {nullptr, nullptr, split_v::Prepare,
+                                 split_v::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/split_v_test.cc b/tensorflow/lite/kernels/split_v_test.cc
new file mode 100644
index 0000000000..2d1d36d685
--- /dev/null
+++ b/tensorflow/lite/kernels/split_v_test.cc
@@ -0,0 +1,175 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <initializer_list>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+constexpr int kAxisIsATensor = -1000;
+
+class SplitVOpModel : public SingleOpModel {
+ public:
+  SplitVOpModel(const TensorData& input, const TensorData& size_splits,
+                int num_splits, int axis) {
+    input_ = AddInput(input);
+    size_splits_ = AddInput(size_splits);
+    if (axis == kAxisIsATensor) {
+      axis_ = AddInput({TensorType_INT32, {1}});
+    } else {
+      axis_ = AddConstInput(TensorType_INT32, {axis}, {1});
+    }
+    for (int i = 0; i < num_splits; ++i) {
+      outputs_.push_back(AddOutput(input.type));
+    }
+    SetBuiltinOp(BuiltinOperator_SPLIT_V, BuiltinOptions_SplitVOptions,
+                 CreateSplitVOptions(builder_, num_splits).Union());
+    if (axis == kAxisIsATensor) {
+      BuildInterpreter(
+          {GetShape(input_), GetShape(size_splits_), GetShape(axis_)});
+    } else {
+      BuildInterpreter({GetShape(input_), GetShape(size_splits_), {}});
+    }
+  }
+
+  void SetInput(std::initializer_list<float> data) {
+    PopulateTensor(input_, data);
+  }
+  void SetSizeSplits(std::initializer_list<int> data) {
+    PopulateTensor(size_splits_, data);
+  }
+  void SetAxis(int axis) { PopulateTensor(axis_, {axis}); }
+
+  std::vector<float> GetOutput(int i) {
+    return ExtractVector<float>(outputs_[i]);
+  }
+  std::vector<int> GetOutputShape(int i) { return GetTensorShape(outputs_[i]); }
+
+ private:
+  int input_;
+  int size_splits_;
+  int axis_;
+  std::vector<int> outputs_;
+};
+
+// TODO(ruic): Add tests to test quantized values. b/119638735
+using TensorValues = std::initializer_list<float>;
+
+void Check(int axis, std::initializer_list<int> input_shape,
+           std::initializer_list<int> size_splits_shape,
+           std::vector<std::initializer_list<int>> output_shapes,
+           const TensorValues& input_data,
+           const std::initializer_list<int>& size_splits_data,
+           const std::vector<TensorValues>& output_data) {
+  int num_splits = size_splits_data.size();
+  SplitVOpModel m({TensorType_FLOAT32, input_shape},
+                  {TensorType_INT32, size_splits_shape}, num_splits,
+                  kAxisIsATensor);
+  m.SetInput(input_data);
+  m.SetSizeSplits(size_splits_data);
+  m.SetAxis(axis);
+  m.Invoke();
+  for (int i = 0; i < num_splits; ++i) {
+    EXPECT_THAT(m.GetOutput(i), ElementsAreArray(output_data[i]));
+    EXPECT_THAT(m.GetOutputShape(i), ElementsAreArray(output_shapes[i]));
+  }
+
+  SplitVOpModel const_m({TensorType_FLOAT32, input_shape},
+                        {TensorType_INT32, size_splits_shape}, num_splits,
+                        axis);
+  const_m.SetInput(input_data);
+  const_m.SetSizeSplits(size_splits_data);
+  const_m.Invoke();
+  for (int i = 0; i < num_splits; ++i) {
+    EXPECT_THAT(const_m.GetOutput(i), ElementsAreArray(output_data[i]));
+    EXPECT_THAT(const_m.GetOutputShape(i), ElementsAreArray(output_shapes[i]));
+  }
+}
+
+TEST(SplitVOpTest, TwoDimensional) {
+  // Input shape: {4, 3}
+  // size_splits: {1, 1, 3}
+  // axis: 0
+  // We should have 3 outpus with shapes respectively:
+  //  output 0 : {1, 3}
+  //  output 1 : {1, 3}
+  //  output 1 : {2, 3}
+  Check(/*axis=*/0, {4, 3}, {3}, {{1, 3}, {1, 3}, {2, 3}},
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, {1, 1, 2},
+        {{1, 2, 3}, {4, 5, 6}, {7, 8, 9, 10, 11, 12}});
+}
+
+TEST(SplitVOpTest, FourDimensional) {
+  Check(/*axis=*/0, {2, 2, 2, 2}, {2}, {{1, 2, 2, 2}, {1, 2, 2, 2}},
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1},
+        {
+            {1, 2, 3, 4, 5, 6, 7, 8},
+            {9, 10, 11, 12, 13, 14, 15, 16},
+        });
+  Check(/*axis=*/1, {2, 2, 2, 2}, {2}, {{2, 1, 2, 2}, {2, 1, 2, 2}},
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, -1},
+        {
+            {1, 2, 3, 4, 9, 10, 11, 12},
+            {5, 6, 7, 8, 13, 14, 15, 16},
+        });
+  Check(/*axis=*/2, {2, 2, 2, 2}, {2}, {{2, 2, 1, 2}, {2, 2, 1, 2}},
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1},
+        {
+            {1, 2, 5, 6, 9, 10, 13, 14},
+            {3, 4, 7, 8, 11, 12, 15, 16},
+        });
+  Check(/*axis=*/3, {2, 2, 2, 2}, {2}, {{2, 2, 2, 1}, {2, 2, 2, 1}},
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1},
+        {
+            {1, 3, 5, 7, 9, 11, 13, 15},
+            {2, 4, 6, 8, 10, 12, 14, 16},
+        });
+}
+
+TEST(SplitVOpTest, OneDimensional) {
+  Check(/*axis=*/0, {8}, {8}, {{1}, {1}, {1}, {1}, {1}, {1}, {1}, {1}},
+        {1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1, 1, 1, 1, 1},
+        {{1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}});
+}
+
+TEST(SplitVOpTest, OneDimensional2) {
+  Check(/*axis=*/0, {8}, {8}, {{1}, {1}, {1}, {1}, {1}, {1}, {2}, {0}},
+        {1, 2, 3, 4, 5, 6, 7, 8}, {1, 1, 1, 1, 1, 1, 2, -1},
+        {{1}, {2}, {3}, {4}, {5}, {6}, {7, 8}, {}});
+}
+
+TEST(SplitVOpTest, NegativeAxis) {
+  Check(/*axis=*/-4, {2, 2, 2, 2}, {2}, {{1, 2, 2, 2}, {1, 2, 2, 2}},
+        {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {1, 1},
+        {
+            {1, 2, 3, 4, 5, 6, 7, 8},
+            {9, 10, 11, 12, 13, 14, 15, 16},
+        });
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 312cedda6c..26d75696a1 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -685,6 +685,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_SQUARED_DIFFERENCE:
       case tflite::BuiltinOperator_MIRROR_PAD:
       case tflite::BuiltinOperator_ABS:
+      case tflite::BuiltinOperator_SPLIT_V:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 094db1a1d5..6436167303 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -204,6 +204,7 @@ enum BuiltinOperator : byte {
   SQUARED_DIFFERENCE = 99,
   MIRROR_PAD = 100,
   ABS = 101,
+  SPLIT_V = 102,
 }
 
 // Options for the builtin operators.
@@ -286,6 +287,7 @@ union BuiltinOptions {
   SquaredDifferenceOptions,
   MirrorPadOptions,
   AbsOptions,
+  SplitVOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -534,6 +536,10 @@ table SplitOptions {
   num_splits: int;
 }
 
+table SplitVOptions {
+  num_splits: int;
+}
+
 table StridedSliceOptions {
   begin_mask: int;
   end_mask: int;
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 1d560b3298..af8b143364 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -148,6 +148,9 @@ struct SqueezeOptionsT;
 struct SplitOptions;
 struct SplitOptionsT;
 
+struct SplitVOptions;
+struct SplitVOptionsT;
+
 struct StridedSliceOptions;
 struct StridedSliceOptionsT;
 
@@ -516,11 +519,12 @@ enum BuiltinOperator {
   BuiltinOperator_SQUARED_DIFFERENCE = 99,
   BuiltinOperator_MIRROR_PAD = 100,
   BuiltinOperator_ABS = 101,
+  BuiltinOperator_SPLIT_V = 102,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_ABS
+  BuiltinOperator_MAX = BuiltinOperator_SPLIT_V
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[101] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[102] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -622,7 +626,8 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[101] {
     BuiltinOperator_LEAKY_RELU,
     BuiltinOperator_SQUARED_DIFFERENCE,
     BuiltinOperator_MIRROR_PAD,
-    BuiltinOperator_ABS
+    BuiltinOperator_ABS,
+    BuiltinOperator_SPLIT_V
   };
   return values;
 }
@@ -731,6 +736,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "SQUARED_DIFFERENCE",
     "MIRROR_PAD",
     "ABS",
+    "SPLIT_V",
     nullptr
   };
   return names;
@@ -821,11 +827,12 @@ enum BuiltinOptions {
   BuiltinOptions_SquaredDifferenceOptions = 76,
   BuiltinOptions_MirrorPadOptions = 77,
   BuiltinOptions_AbsOptions = 78,
+  BuiltinOptions_SplitVOptions = 79,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_AbsOptions
+  BuiltinOptions_MAX = BuiltinOptions_SplitVOptions
 };
 
-inline const BuiltinOptions (&EnumValuesBuiltinOptions())[79] {
+inline const BuiltinOptions (&EnumValuesBuiltinOptions())[80] {
   static const BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -905,7 +912,8 @@ inline const BuiltinOptions (&EnumValuesBuiltinOptions())[79] {
     BuiltinOptions_LeakyReluOptions,
     BuiltinOptions_SquaredDifferenceOptions,
     BuiltinOptions_MirrorPadOptions,
-    BuiltinOptions_AbsOptions
+    BuiltinOptions_AbsOptions,
+    BuiltinOptions_SplitVOptions
   };
   return values;
 }
@@ -991,6 +999,7 @@ inline const char * const *EnumNamesBuiltinOptions() {
     "SquaredDifferenceOptions",
     "MirrorPadOptions",
     "AbsOptions",
+    "SplitVOptions",
     nullptr
   };
   return names;
@@ -1317,6 +1326,10 @@ template<> struct BuiltinOptionsTraits<AbsOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_AbsOptions;
 };
 
+template<> struct BuiltinOptionsTraits<SplitVOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_SplitVOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1972,6 +1985,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_AbsOptions ?
       reinterpret_cast<const AbsOptionsT *>(value) : nullptr;
   }
+  SplitVOptionsT *AsSplitVOptions() {
+    return type == BuiltinOptions_SplitVOptions ?
+      reinterpret_cast<SplitVOptionsT *>(value) : nullptr;
+  }
+  const SplitVOptionsT *AsSplitVOptions() const {
+    return type == BuiltinOptions_SplitVOptions ?
+      reinterpret_cast<const SplitVOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -5048,6 +5069,60 @@ inline flatbuffers::Offset<SplitOptions> CreateSplitOptions(
 
 flatbuffers::Offset<SplitOptions> CreateSplitOptions(flatbuffers::FlatBufferBuilder &_fbb, const SplitOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct SplitVOptionsT : public flatbuffers::NativeTable {
+  typedef SplitVOptions TableType;
+  int32_t num_splits;
+  SplitVOptionsT()
+      : num_splits(0) {
+  }
+};
+
+struct SplitVOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef SplitVOptionsT NativeTableType;
+  enum {
+    VT_NUM_SPLITS = 4
+  };
+  int32_t num_splits() const {
+    return GetField<int32_t>(VT_NUM_SPLITS, 0);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyField<int32_t>(verifier, VT_NUM_SPLITS) &&
+           verifier.EndTable();
+  }
+  SplitVOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(SplitVOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<SplitVOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct SplitVOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_num_splits(int32_t num_splits) {
+    fbb_.AddElement<int32_t>(SplitVOptions::VT_NUM_SPLITS, num_splits, 0);
+  }
+  explicit SplitVOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  SplitVOptionsBuilder &operator=(const SplitVOptionsBuilder &);
+  flatbuffers::Offset<SplitVOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<SplitVOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<SplitVOptions> CreateSplitVOptions(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    int32_t num_splits = 0) {
+  SplitVOptionsBuilder builder_(_fbb);
+  builder_.add_num_splits(num_splits);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<SplitVOptions> CreateSplitVOptions(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct StridedSliceOptionsT : public flatbuffers::NativeTable {
   typedef StridedSliceOptions TableType;
   int32_t begin_mask;
@@ -7301,6 +7376,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const AbsOptions *builtin_options_as_AbsOptions() const {
     return builtin_options_type() == BuiltinOptions_AbsOptions ? static_cast<const AbsOptions *>(builtin_options()) : nullptr;
   }
+  const SplitVOptions *builtin_options_as_SplitVOptions() const {
+    return builtin_options_type() == BuiltinOptions_SplitVOptions ? static_cast<const SplitVOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -7644,6 +7722,10 @@ template<> inline const AbsOptions *Operator::builtin_options_as<AbsOptions>() c
   return builtin_options_as_AbsOptions();
 }
 
+template<> inline const SplitVOptions *Operator::builtin_options_as<SplitVOptions>() const {
+  return builtin_options_as_SplitVOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -9261,6 +9343,32 @@ inline flatbuffers::Offset<SplitOptions> CreateSplitOptions(flatbuffers::FlatBuf
       _num_splits);
 }
 
+inline SplitVOptionsT *SplitVOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new SplitVOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void SplitVOptions::UnPackTo(SplitVOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+  { auto _e = num_splits(); _o->num_splits = _e; };
+}
+
+inline flatbuffers::Offset<SplitVOptions> SplitVOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateSplitVOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<SplitVOptions> CreateSplitVOptions(flatbuffers::FlatBufferBuilder &_fbb, const SplitVOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const SplitVOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  auto _num_splits = _o->num_splits;
+  return tflite::CreateSplitVOptions(
+      _fbb,
+      _num_splits);
+}
+
 inline StridedSliceOptionsT *StridedSliceOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new StridedSliceOptionsT();
   UnPackTo(_o, _resolver);
@@ -10803,6 +10911,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const AbsOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_SplitVOptions: {
+      auto ptr = reinterpret_cast<const SplitVOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -11133,6 +11245,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const AbsOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_SplitVOptions: {
+      auto ptr = reinterpret_cast<const SplitVOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -11451,6 +11567,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const AbsOptionsT *>(value);
       return CreateAbsOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_SplitVOptions: {
+      auto ptr = reinterpret_cast<const SplitVOptionsT *>(value);
+      return CreateSplitVOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -11769,6 +11889,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new AbsOptionsT(*reinterpret_cast<AbsOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_SplitVOptions: {
+      value = new SplitVOptionsT(*reinterpret_cast<SplitVOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -12166,6 +12290,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_SplitVOptions: {
+      auto ptr = reinterpret_cast<SplitVOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 566723cdc2..55796846b3 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -103,8 +103,6 @@ KNOWN_BUGS = {
     r"batch_to_space_nd.*input_shape=\[8,2,2,2,1,1\]": "70594733",
     # Div will use floordiv.
     r"div.*int32": "72051395",
-    # No support for SplitV
-    r"split.*num_or_size_splits=\[2,2\]": "73377559",
     # Constant 1D gather crashes toco.
     r"gather_buggy.*input_shape=\[3\].*": "120029508",
 }
@@ -1638,7 +1636,7 @@ def make_split_tests(zip_path):
 
   test_parameters = [{
       "input_shape": [[1, 3, 4, 6], [2, 4, 1], [6, 4], [8]],
-      "num_or_size_splits": [1, 2, 3, 4, 5, [2, 2]],
+      "num_or_size_splits": [1, 2, 3, 4, 5],
       "axis": [0, 1, 2, 3, -4, -3, -2, -1],
   }]
 
@@ -1656,6 +1654,29 @@ def make_split_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_splitv_tests(zip_path):
+  """Make a set of tests to do tf.split_v."""
+
+  test_parameters = [{
+      "input_shape": [[1, 3, 4, 6], [2, 4, 1], [6, 4], [8]],
+      "size_splits": [[2, 2], [1, 3], [4, 2], [5, 3],
+                      [-1, 1], [-1, 2], [-1, 4]],
+      "axis": [0, 1, 2, 3, -4, -3, -2, -1],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    out = tf.split(input_tensor, parameters["size_splits"], parameters["axis"])
+    return [input_tensor], [out[0]]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [create_tensor_data(np.float32, parameters["input_shape"])]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_concat_tests(zip_path):
   """Make a set of tests to do concatenation."""
 
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index bdc3a5b0fb..9fff001552 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -1158,6 +1158,29 @@ void ConvertSplitOperator(const Model& model,
                                   tensorflow_graph);
 }
 
+void ConvertSplitVOperator(const Model& model,
+                           const TensorFlowSplitVOperator& src_op,
+                           GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* split_v_op = tensorflow_graph->add_node();
+  split_v_op->set_op("SplitV");
+  split_v_op->set_name(src_op.outputs[0]);
+  for (const auto& input : src_op.inputs) {
+    *split_v_op->add_input() = input;
+  }
+  (*split_v_op->mutable_attr())["T"].set_type(
+      GetTensorFlowDataType(model, src_op.inputs[0]));
+  (*split_v_op->mutable_attr())["num_split"].set_i(src_op.num_split);
+  const auto& split_dim_array = model.GetArray(src_op.inputs[1]);
+  CHECK(split_dim_array.buffer);
+  CHECK(split_dim_array.data_type == ArrayDataType::kInt32);
+  const auto& split_dim_data =
+      split_dim_array.GetBuffer<ArrayDataType::kInt32>().data;
+  CHECK_EQ(split_dim_data.size(), 1);
+  const int split_dim = split_dim_data[0];
+  CreateDummyConcatDimTensorConst(src_op.inputs[0], split_dim,
+                                  tensorflow_graph);
+}
+
 void ConvertCastOperator(const Model& model, const CastOperator& src_op,
                          GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* cast_op = tensorflow_graph->add_node();
@@ -2133,6 +2156,10 @@ void ConvertOperator(const Model& model, const Operator& src_op,
     ConvertSplitOperator(model,
                          static_cast<const TensorFlowSplitOperator&>(src_op),
                          tensorflow_graph);
+  } else if (src_op.type == OperatorType::kSplitV) {
+    ConvertSplitVOperator(model,
+                          static_cast<const TensorFlowSplitVOperator&>(src_op),
+                          tensorflow_graph);
   } else if (src_op.type == OperatorType::kFakeQuant) {
     ConvertFakeQuantOperator(static_cast<const FakeQuantOperator&>(src_op),
                              tensorflow_graph);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index 9a458dccb9..cbae6610d7 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -86,6 +86,13 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kSplitV: {
+      // These operators produce output with the same type as its 1st input
+      CHECK_GE(op->inputs.size(), 3);
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     case OperatorType::kTransposeConv: {
       // These operators produce an output with the same type as their 3rd input
       CHECK_GE(op->inputs.size(), 3);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index b520f7f559..fc2ed07aa0 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include <algorithm>
 #include <iterator>
 #include <memory>
+#include <numeric>
 #include <string>
 #include <unordered_map>
 #include <vector>
@@ -786,6 +787,97 @@ void ProcessTensorFlowSplitOperator(Model* model, TensorFlowSplitOperator* op) {
   }
 }
 
+void ProcessTensorFlowSplitVOperator(Model* model,
+                                     TensorFlowSplitVOperator* op) {
+  CHECK_EQ(op->inputs.size(), 3);
+
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+  const Shape& input_shape = input_array.shape();
+
+  // Yield until size_splits is constant.
+  if (!IsConstantParameterArray(*model, op->inputs[1])) {
+    return;
+  }
+  const auto& size_array = model->GetArray(op->inputs[1]);
+  // Yield until size_splits dims have been resolved.
+  if (!size_array.has_shape()) {
+    return;
+  }
+  const Shape& size_shape = size_array.shape();
+
+  CHECK(size_array.data_type == ArrayDataType::kInt32 ||
+        size_array.data_type == ArrayDataType::kInt64)
+      << "size_splits must be int32, int64";
+  CHECK_EQ(size_shape.dimensions_count(), 1) << "size_splits must be 1-D";
+
+  std::vector<int64> size_splits_vector;
+  if (size_array.data_type == ArrayDataType::kInt32) {
+    for (const auto each_size :
+         size_array.GetBuffer<ArrayDataType::kInt32>().data) {
+      size_splits_vector.push_back(each_size);
+    }
+  } else {
+    size_splits_vector = size_array.GetBuffer<ArrayDataType::kInt64>().data;
+  }
+
+  // Yield until axis is constant.
+  if (!IsConstantParameterArray(*model, op->inputs[2])) {
+    return;
+  }
+  const auto& axis_array = model->GetArray(op->inputs[2]);
+  // Yield until axis dims have been resolved.
+  if (!axis_array.has_shape()) {
+    return;
+  }
+
+  CHECK(axis_array.data_type == ArrayDataType::kInt32)
+      << "Axis array must be int32.";
+  CHECK_EQ(RequiredBufferSizeForShape(axis_array.shape()), 1)
+      << "Axis array must be scalar.";
+
+  int axis = axis_array.GetBuffer<ArrayDataType::kInt32>().data[0];
+  if (axis < 0) {
+    axis += input_shape.dimensions_count();
+  }
+
+  CHECK_EQ(op->num_split, size_splits_vector.size());
+
+  int64_t minus_one_count = 0, size_splits_sum = 0;
+  for (auto size : size_splits_vector) {
+    if (size == -1) {
+      ++minus_one_count;
+    } else {
+      size_splits_sum += size;
+    }
+  }
+
+  const int input_size = input_shape.dims(axis);
+
+  CHECK_LE(minus_one_count, 1) << "size_splits can contain at most one -1.";
+
+  if (minus_one_count == 1) {
+    CHECK_LE(size_splits_sum, input_size);
+    auto iter =
+        std::find(size_splits_vector.begin(), size_splits_vector.end(), -1);
+    *iter = input_size - size_splits_sum;
+  } else {
+    CHECK_EQ(size_splits_sum, input_size);
+  }
+
+  CHECK_EQ(op->outputs.size(), op->num_split);
+
+  for (int i = 0; i < op->outputs.size(); ++i) {
+    const auto& output = op->outputs[i];
+    Shape output_shape = input_shape;
+    (*output_shape.mutable_dims())[axis] = size_splits_vector.at(i);
+    model->GetArray(output).copy_shape(output_shape);
+  }
+}
+
 void ProcessAveragePoolOperator(Model* model, AveragePoolOperator* op) {
   const string& input_name = op->inputs[0];
   const auto& input_array = model->GetArray(input_name);
@@ -1837,6 +1929,10 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) {
       ProcessTensorFlowSplitOperator(model,
                                      static_cast<TensorFlowSplitOperator*>(op));
       break;
+    case OperatorType::kSplitV:
+      ProcessTensorFlowSplitVOperator(
+          model, static_cast<TensorFlowSplitVOperator*>(op));
+      break;
     case OperatorType::kSqueeze:
       ProcessSqueezeOperator(model, static_cast<SqueezeOperator*>(op));
       break;
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index e42c2e2f20..aa6b4a3bc5 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -935,6 +935,25 @@ tensorflow::Status ConvertSplitOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertSplitVOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "SplitV");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 3));
+  auto* op = new TensorFlowSplitVOperator;
+  op->inputs.push_back(node.input(0));
+  op->inputs.push_back(node.input(1));
+  op->inputs.push_back(node.input(2));
+  const int num_split = GetIntAttr(node, "num_split");
+  op->outputs.push_back(node.name());
+  for (int i = 1; i < num_split; i++) {
+    op->outputs.push_back(absl::StrCat(node.name(), ":", i));
+  }
+  op->num_split = num_split;
+  model->operators.emplace_back(op);
+  return tensorflow::Status::OK();
+}
+
 tensorflow::Status ConvertSwitchOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -2350,6 +2369,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"SpaceToDepth", ConvertSpaceToDepthOperator},
       {"SparseToDense", ConvertSparseToDenseOperator},
       {"Split", ConvertSplitOperator},
+      {"SplitV", ConvertSplitVOperator},
       {"Sqrt", ConvertSimpleOperator<TensorFlowSqrtOperator, 1>},
       {"Square", ConvertSimpleOperator<TensorFlowSquareOperator, 1>},
       {"SquaredDifference",
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index a4883e4694..d4fe62ac75 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -121,6 +121,7 @@ enum class OperatorType : uint8 {
   kRsqrt,
   kShape,
   kSplit,
+  kSplitV,
   kSqrt,
   kSquare,
   kSquaredDifference,
@@ -1401,6 +1402,12 @@ struct TensorFlowSplitOperator : Operator {
   int num_split = 0;
 };
 
+// TensorFlow SplitV equivalent. Refer to TensorFlow documentation for details.
+struct TensorFlowSplitVOperator : Operator {
+  TensorFlowSplitVOperator() : Operator(OperatorType::kSplitV) {}
+  int num_split = 0;
+};
+
 // TensorFlow Concat equivalent. Refer to TensorFlow documentation for details.
 // Not fully supported, just a placeholder to handle TensorFlow graphs and
 // support graph transformations to other operator types by matching sub-graphs.
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index b28781c32e..1c3bee3c5a 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -978,6 +978,26 @@ class Split
   int GetVersion(const Operator& op) const override { return 1; }
 };
 
+class SplitV
+    : public BuiltinOperator<TensorFlowSplitVOperator, ::tflite::SplitVOptions,
+                             ::tflite::BuiltinOptions_SplitVOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateSplitVOptions(*builder, op.num_split);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->num_split = options.num_splits();
+  }
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
 class StridedSlice
     : public BuiltinOperator<StridedSliceOperator,
                              ::tflite::StridedSliceOptions,
@@ -1521,6 +1541,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
                                     OperatorType::kSqueeze));
   ops.push_back(
       MakeUnique<Split>(::tflite::BuiltinOperator_SPLIT, OperatorType::kSplit));
+  ops.push_back(MakeUnique<SplitV>(::tflite::BuiltinOperator_SPLIT_V,
+                                   OperatorType::kSplitV));
   ops.push_back(MakeUnique<StridedSlice>(
       ::tflite::BuiltinOperator_STRIDED_SLICE, OperatorType::kStridedSlice));
   ops.push_back(MakeUnique<TopK_V2>(::tflite::BuiltinOperator_TOPK_V2,
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 16514760de..09e4435a18 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -310,6 +310,14 @@ TEST_F(OperatorTest, CustomSplit) {
   EXPECT_EQ(op.num_split, output_toco_op->num_split);
 }
 
+TEST_F(OperatorTest, CustomSplitV) {
+  TensorFlowSplitVOperator op;
+  op.num_split = 123;
+  auto output_toco_op = SerializeAndDeserialize(
+      GetOperator("SPLIT_V", OperatorType::kSplitV), op);
+  EXPECT_EQ(op.num_split, output_toco_op->num_split);
+}
+
 TEST_F(OperatorTest, BuiltinAveragePool) {
   AveragePoolOperator op;
   op.fused_activation_function = FusedActivationFunctionType::kRelu6;
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index e3f3fe36f7..44fa658f9c 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -372,6 +372,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Shape)
     HANDLE_OPERATORTYPENAME_CASE(Slice)
     HANDLE_OPERATORTYPENAME_CASE(Split)
+    HANDLE_OPERATORTYPENAME_CASE(SplitV)
     HANDLE_OPERATORTYPENAME_CASE(Sqrt)
     HANDLE_OPERATORTYPENAME_CASE(Square)
     HANDLE_OPERATORTYPENAME_CASE(Switch)
-- 
GitLab


From 71f40f044450736cd6acd29e92ffbfc0e571ee14 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Wed, 28 Nov 2018 21:47:10 -0800
Subject: [PATCH 1000/1554] Remove tf.clip_by_average_norm for TensorFlow 2.0.
 Also add deprecation warning.

PiperOrigin-RevId: 223288438
---
 tensorflow/python/kernel_tests/clip_ops_test.py | 16 ++++++++++++++++
 tensorflow/python/ops/clip_ops.py               |  7 ++++++-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt |  4 ----
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/kernel_tests/clip_ops_test.py b/tensorflow/python/kernel_tests/clip_ops_test.py
index d0cd7eb302..5f1b6b6917 100644
--- a/tensorflow/python/kernel_tests/clip_ops_test.py
+++ b/tensorflow/python/kernel_tests/clip_ops_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
@@ -440,6 +441,21 @@ class ClipTest(test.TestCase):
 
     self.assertAllClose(np_ans, tf_ans)
 
+  def testClipByAverageNormReplacedWithClipByNorm(self):
+    # Check clip_by_average_norm(t) is the same as
+    # clip_by_norm(t, clip_norm * tf.to_float(tf.size(t)))
+    with self.session(use_gpu=True):
+      x = constant_op.constant([-3.0, 0.0, 0.0, 4.0, 0.0, 0.0], shape=[2, 3])
+      # Average norm of x = sqrt(3^2 + 4^2) / 6 = 0.83333333
+      # expected answer [[-2.88, 0.0, 0.0], [3.84, 0.0, 0.0]]
+      clip_norm = constant_op.constant(0.8)
+      with_norm = clip_ops.clip_by_average_norm(x, clip_norm)
+      without_norm = clip_ops.clip_by_norm(
+          x, clip_norm * math_ops.to_float(array_ops.size(x)))
+      clip_by_average_norm_ans = self.evaluate(with_norm)
+      clip_by_norm_ans = self.evaluate(without_norm)
+      self.assertAllClose(clip_by_average_norm_ans, clip_by_norm_ans)
+
   def testClipByValueEmptyTensor(self):
     # Test case for GitHub issue 19337
     zero = array_ops.placeholder(dtype=dtypes.float32, shape=None)
diff --git a/tensorflow/python/ops/clip_ops.py b/tensorflow/python/ops/clip_ops.py
index 5cd626b92d..82803ac351 100644
--- a/tensorflow/python/ops/clip_ops.py
+++ b/tensorflow/python/ops/clip_ops.py
@@ -300,7 +300,12 @@ def clip_by_global_norm(t_list, clip_norm, use_norm=None, name=None):
   return list_clipped, use_norm
 
 
-@tf_export("clip_by_average_norm")
+@deprecation.deprecated(
+    date=None,
+    instructions=
+    "clip_by_average_norm is deprecated in TensorFlow 2.0. Please use "
+    "clip_by_norm(t, clip_norm * tf.to_float(tf.size(t), name)) instead.")
+@tf_export(v1=["clip_by_average_norm"])
 def clip_by_average_norm(t, clip_norm, name=None):
   """Clips tensor values to a maximum average L2-norm.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index f2ae039d9f..98ea3b6839 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -528,10 +528,6 @@ tf_module {
     name: "cast"
     argspec: "args=[\'x\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "clip_by_average_norm"
-    argspec: "args=[\'t\', \'clip_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "clip_by_global_norm"
     argspec: "args=[\'t_list\', \'clip_norm\', \'use_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-- 
GitLab


From f3db4e300b782afbff93223777ed1b5cd3f1f7cd Mon Sep 17 00:00:00 2001
From: Yuxin Wu <ppwwyyxxc@gmail.com>
Date: Wed, 28 Nov 2018 21:56:27 -0800
Subject: [PATCH 1001/1554] Fix deprecated use of `sparse_to_dense`.

Calling `sparse_to_dense` gives a deprecation warning that asks users to use `sparse.to_dense`:
https://github.com/tensorflow/tensorflow/blob/71f40f044450736cd6acd29e92ffbfc0e571ee14/tensorflow/python/ops/sparse_ops.py#L952-L955

However, `sparse.to_dense` calls `sparse_to_dense`, which again produces the deprecation warning.
---
 tensorflow/python/ops/sparse_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 245080cb26..914616ff6d 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -1422,7 +1422,7 @@ def sparse_tensor_to_dense(sp_input,
   """
   sp_input = _convert_to_sparse_tensor(sp_input)
 
-  return sparse_to_dense(
+  return gen_sparse_ops.sparse_to_dense(
       sp_input.indices,
       sp_input.dense_shape,
       sp_input.values,
-- 
GitLab


From f7ac6ef4e2454938c5fa6a3367892fdc112edab4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 01:05:54 -0800
Subject: [PATCH 1002/1554] compat: Update forward compatibility horizon to
 2018-11-29

PiperOrigin-RevId: 223304764
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 1f43793962..d9613021a9 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 28)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 29)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 8e16ae7463698f891f7290e62c759c9b17733a57 Mon Sep 17 00:00:00 2001
From: Thomas Joerg <tjoerg@google.com>
Date: Thu, 29 Nov 2018 01:48:48 -0800
Subject: [PATCH 1003/1554] [XLA:GPU] Prefactoring: Extract
 ShapesCompatibleForFusion from GpuMultiOutputFusion.

PiperOrigin-RevId: 223310314
---
 .../compiler/xla/service/gpu/gpu_fusible.cc   |  51 ++++
 .../compiler/xla/service/gpu/gpu_fusible.h    |  10 +
 .../xla/service/gpu/gpu_fusible_test.cc       | 264 ++++++++++++++++++
 .../xla/service/gpu/multi_output_fusion.cc    |  47 +---
 4 files changed, 327 insertions(+), 45 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
index 392b149abd..5eafac7232 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
@@ -81,5 +81,56 @@ bool IsInputFusibleReduction(const HloInstruction& instr) {
   return IsReduceInputFusion(instr) || IsReductionToVector(instr);
 }
 
+bool ShapesCompatibleForMultiOutputFusion(const HloInstruction& instr1,
+                                          const HloInstruction& instr2) {
+  // Returns the instructions that determines the emitter used for lowering,
+  // sometimes referred to as "the real hero".
+  auto get_real_hero =
+      [&](const HloInstruction* instr) -> const HloInstruction* {
+    if (instr->opcode() == HloOpcode::kFusion) {
+      auto fused_expression_root = instr->fused_expression_root();
+      if (instr->IsMultiOutputFusion()) {
+        // If possible, we want to pick a reduction-to-vector operand of the
+        // fusion root, because it has the most constraints.
+        for (const auto* inst : fused_expression_root->operands()) {
+          if (IsReductionToVector(*inst)) {
+            return inst;
+          }
+        }
+        return fused_expression_root->operands()[0];
+      }
+      return fused_expression_root;
+    }
+    return instr;
+  };
+
+  // Multi-output fusion kernels share a common parallel loop. The loop
+  // dimenstions are determined by instruction shapes.
+  auto get_loop_shape = [&](const HloInstruction* element_instr) {
+    // Special-case reduction-to-vector ops: The loop dimensions are determined
+    // by the shape of the first operand.
+    if (IsReductionToVector(*element_instr)) {
+      return element_instr->operand(0)->shape();
+    }
+    return element_instr->shape();
+  };
+
+  // All shapes of the root tuple of multi-output fusions should agree, i.e. all
+  // root ops should have equal output shapes. An exception are
+  // reduction-to-vector ops. Here the input shapes of the reduction (first
+  // operand shape) need to be considered.
+  auto* instr_1 = get_real_hero(&instr1);
+  auto* instr_2 = get_real_hero(&instr2);
+  // TODO(tjoerg): Relax the shape constraint. The datatype does not matter.
+  if (IsReductionToVector(*instr_1) && IsReductionToVector(*instr_2) &&
+      !ShapeUtil::Equal(instr_1->shape(), instr_2->shape())) {
+    return false;
+  }
+  // The elementwise output shapes must be the same (including layout).
+  // TODO(tjoerg): Further relax the constraint. The datatype does not matter.
+  return ShapeUtil::EqualIgnoringFpPrecision(get_loop_shape(instr_1),
+                                             get_loop_shape(instr_2));
+}
+
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.h b/tensorflow/compiler/xla/service/gpu/gpu_fusible.h
index c0be354730..e9d7ba1c4c 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.h
@@ -46,6 +46,16 @@ bool IsReduceInputFusion(const HloInstruction& instr);
 // is either an unfused reduction-to-vector op or a reduce input fusion.
 bool IsInputFusibleReduction(const HloInstruction& instr);
 
+// Whether instruction shapes are compatible for multi-output fusion, i.e.
+// whether the emitters support lowering the resulting fusion.
+// This function works for both, sibling and producer-conumser multi-output
+// fusion.
+// So far, multi-output fusion is supported for loop fusions and reduce
+// input fusions only. It is up to the caller to ensure the instructions
+// themselves are fusible!
+bool ShapesCompatibleForMultiOutputFusion(const HloInstruction& instr1,
+                                          const HloInstruction& instr2);
+
 }  // namespace gpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
index 12222500ea..ca07851f8b 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
@@ -336,5 +336,269 @@ TEST_F(GpuFusibleTest,
   EXPECT_FALSE(IsInputFusibleReduction(*reduce));
 }
 
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_LoopFusions) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_computation_1 {
+      p0.1 = f32[6400]{0} parameter(0)
+      ROOT mul = f32[6400]{0} multiply(p0.1, p0.1)
+    }
+
+    fused_computation_2 {
+      p0.2 = f32[6400]{0} parameter(0)
+      const.2 = f32[] constant(1)
+      ROOT div = f32[6400]{0} divide(p0.2, const.2)
+    }
+
+    ENTRY entry {
+      p0 = f32[6400]{0} parameter(0)
+      fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1
+      fusion.2 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_2
+      ROOT root = (f32[6400]{0}, f32[6400]{0}) tuple(fusion.1, fusion.2)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_IgnoreFpPrecision) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_computation_1 {
+      p0.1 = f32[6400]{0} parameter(0)
+      ROOT mul = f32[6400]{0} multiply(p0.1, p0.1)
+    }
+
+    fused_computation_2 {
+      p0.2 = f32[6400]{0} parameter(0)
+      ROOT convert = f16[6400]{0} convert(p0.2)
+    }
+
+    ENTRY entry {
+      p0 = f32[6400]{0} parameter(0)
+      fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1
+      fusion.2 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_2
+      ROOT root = (f32[6400]{0}, f32[6400]{0}) tuple(fusion.1, fusion.2)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_Reduce) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_computation_1 {
+      p0.1 = f32[6400]{0} parameter(0)
+      ROOT mul = f32[6400]{0} multiply(p0.1, p0.1)
+    }
+
+    ENTRY entry {
+      p0 = f32[6400]{0} parameter(0)
+      fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1
+      const.2 = f32[] constant(0)
+      reduce = f32[] reduce(p0, const.2), dimensions={0}, to_apply=scalar_add
+      ROOT root = (f32[6400]{0}, f32[]) tuple(fusion.1, reduce)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* reduce =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion, *reduce));
+}
+
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_Elementwise) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_computation_1 {
+      p0.1 = f32[6400]{0} parameter(0)
+      ROOT mul = f32[6400]{0} multiply(p0.1, p0.1)
+    }
+
+    ENTRY entry {
+      p0 = f32[6400]{0} parameter(0)
+      fusion.1 = f32[6400]{0} fusion(p0), kind=kLoop, calls=fused_computation_1
+      const.2 = f32[] constant(1)
+      div = f32[6400]{0} divide(p0, const.2)
+      ROOT root = (f32[6400]{0}, f32[6400]{0}) tuple(fusion.1, div)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* div =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion, *div));
+}
+
+TEST_F(GpuFusibleTest,
+       ShapesCompatibleForMultiOutputFusion_MultiOutputLoopFusion) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_computation_1 {
+      p0.1 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} parameter(0)
+      mul = f32[8,1,5,16,1,1]{5,4,3,2,1,0} multiply(p0.1, p0.1)
+      exp = f32[8,1,5,16,1,1]{5,4,3,2,1,0} exponential(p0.1)
+      ROOT tuple = (f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}) tuple(mul, exp)
+    }
+
+    fused_computation_2 {
+      p0.2 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} parameter(0)
+      const.2 = f32[] constant(0)
+      ROOT add = f32[8,1,5,16,1,1]{5,4,3,2,1,0} add(p0.2, const.2)
+    }
+
+    ENTRY entry {
+      p0 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} parameter(0)
+      fusion.1 = (f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}) fusion(p0), kind=kLoop, calls=fused_computation_1
+      fusion.2 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} fusion(p0), kind=kLoop, calls=fused_computation_2
+      gte0 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} get-tuple-element(fusion.1), index=0
+      gte1 = f32[8,1,5,16,1,1]{5,4,3,2,1,0} get-tuple-element(fusion.1), index=1
+      ROOT root = (f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}, f32[8,1,5,16,1,1]{5,4,3,2,1,0}) tuple(gte0, gte1, fusion.2)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0)->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1)->operand(0);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_UnfusedOps) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    ENTRY reduce {
+      p0 = f32[2,2,2]{2,1,0} parameter(0)
+      c0 = f32[] constant(0)
+      exp = f32[2,2,2]{2,1,0} exponential(p0)
+      reduce = f32[2,2]{1,0} reduce(exp, c0), dimensions={2}, to_apply=scalar_add
+      ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(reduce, exp)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* reduce =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* exp =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*reduce, *exp));
+}
+
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_DifferentLayouts) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    ENTRY reduce {
+      p0 = f32[2,2,2]{2,1,0} parameter(0)
+      p1 = f32[2,2,2]{0,1,2} parameter(1)
+      c0 = f32[] constant(0)
+      exp = f32[2,2,2]{2,1,0} exponential(p0)
+      reduce = f32[2,2]{0,1} reduce(p1, c0), dimensions={2}, to_apply=scalar_add
+      ROOT root = (f32[2,2]{0,1}, f32[2,2,2]{2,1,0}) tuple(reduce, exp)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* reduce =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* exp =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_FALSE(ShapesCompatibleForMultiOutputFusion(*reduce, *exp));
+}
+
+TEST_F(GpuFusibleTest,
+       ShapesCompatibleForMultiOutputFusion_MultiOutputReduceFusion) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_select {
+      p1.1 = f32[2,2,2]{2,1,0} parameter(1)
+      c0 = f32[] constant(0)
+      broadcast = f32[2,2,2]{2,1,0} broadcast(f32[] c0), dimensions={}
+      greater-than = pred[2,2,2]{2,1,0} greater-than(f32[2,2,2]{2,1,0} p1.1, f32[2,2,2]{2,1,0} broadcast)
+      p0.1 = f32[2,2,2]{2,1,0} parameter(0)
+      ROOT select = f32[2,2,2]{2,1,0} select(pred[2,2,2]{2,1,0} greater-than, f32[2,2,2]{2,1,0} p0.1, f32[2,2,2]{2,1,0} broadcast)
+    }
+
+    fused_reduce {
+      p0.2 = f32[2,2,2]{2,1,0} parameter(0)
+      c1 = f32[] constant(0)
+      r1 = f32[2,2]{1,0} reduce(p0.2, c1), dimensions={2}, to_apply=scalar_add
+      mul = f32[2,2,2]{2,1,0} multiply(p0.2, p0.2)
+      r2 = f32[2,2]{1,0} reduce(mul, c1), dimensions={2}, to_apply=scalar_add
+      ROOT tuple = (f32[2,2]{1,0}, f32[2,2]{1,0}) tuple(r1, r2)
+    }
+
+    ENTRY reduce {
+      p0 = f32[2,2,2]{2,1,0} parameter(0)
+      p1 = f32[2,2,2]{2,1,0} parameter(1)
+      select = f32[2,2,2]{2,1,0} fusion(p0, p1), kind=kLoop, calls=fused_select
+      fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(select), kind=kInput, calls=fused_reduce
+      gte0 = f32[2,2]{1,0} get-tuple-element(fusion), index=0
+      gte1 = f32[2,2]{1,0} get-tuple-element(fusion), index=1
+      ROOT root = (f32[2,2]{1,0}, f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(gte1, gte1, select)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0)->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1)->operand(0);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
+TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_ReduceFusions) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_reduce_1 {
+      p0.1 = f32[2,2,2]{2,1,0} parameter(0)
+      c0 = f32[] constant(0)
+      ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} p0.1, f32[] c0), dimensions={0}, to_apply=scalar_add
+    }
+
+    fused_reduce_2 {
+      p0.2 = f32[2,2,2]{2,1,0} parameter(0)
+      mul = f32[2,2,2]{2,1,0} multiply(f32[2,2,2]{2,1,0} p0.2, f32[2,2,2]{2,1,0} p0.2)
+      c1 = f32[] constant(0)
+      ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} mul, f32[] c1), dimensions={0}, to_apply=scalar_add
+    }
+
+    ENTRY reduce {
+      p0 = f32[2,2,2]{2,1,0} parameter(0)
+      p1 = f32[2,2,2]{2,1,0} parameter(1)
+      reduce_1 = f32[2,2]{1,0} fusion(p0), kind=kLoop, calls=fused_reduce_1
+      reduce_2 = f32[2,2]{1,0} fusion(p1), kind=kLoop, calls=fused_reduce_2
+      ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(reduce_1, reduce_2)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
+TEST_F(GpuFusibleTest,
+       ShapesCompatibleForMultiOutputFusion_NoReductionToVector) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_element_wise {
+      p0.1 = f32[2,2,2]{2,1,0} parameter(0)
+      p1.1 = f32[2,2,2]{2,1,0} parameter(1)
+      ROOT add = f32[2,2,2]{2,1,0} add(p0.1, p1.1)
+    }
+
+    fused_reduce {
+      p0.2 = f32[2,2,2]{2,1,0} parameter(0)
+      mul = f32[2,2,2]{2,1,0} multiply(f32[2,2,2]{2,1,0} p0.2, f32[2,2,2]{2,1,0} p0.2)
+      c1 = f32[] constant(0)
+      // Note that reduce is not a reduction-to-vector.
+      ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} mul, f32[] c1), dimensions={1}, to_apply=scalar_add
+    }
+
+    ENTRY reduce {
+      p0 = f32[2,2,2]{2,1,0} parameter(0)
+      p1 = f32[2,2,2]{2,1,0} parameter(1)
+      element_wise = f32[2,2,2]{2,1,0} fusion(p0, p1), kind=kLoop, calls=fused_element_wise
+      fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(element_wise), kind=kLoop, calls=fused_reduce
+      ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(fusion, element_wise)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_FALSE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
index d9b06828e2..01fddcede6 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
@@ -41,50 +41,7 @@ GpuMultiOutputFusion::GpuMultiOutputFusion() : MultiOutputFusion(INT64_MAX) {}
 
 bool GpuMultiOutputFusion::ShapesCompatibleForFusion(HloInstruction* instr1,
                                                      HloInstruction* instr2) {
-  auto get_element_instr =
-      [&](const HloInstruction* instr) -> const HloInstruction* {
-    const HloInstruction* element_instr = instr;
-    if (instr->opcode() == HloOpcode::kFusion) {
-      auto fused_expression_root = instr->fused_expression_root();
-      if (instr->IsMultiOutputFusion()) {
-        // If possible, we want to pick a reduce operand of the fusion root,
-        // because it has the most constraints.
-        for (const auto* inst : fused_expression_root->operands()) {
-          if (IsReductionToVector(*inst)) {
-            return inst;
-          }
-        }
-        return fused_expression_root->operands()[0];
-      } else {
-        element_instr = fused_expression_root;
-      }
-    }
-    return element_instr;
-  };
-
-  auto get_element_shape = [&](const HloInstruction* element_instr) {
-    // Special handling of kReduce instructions -- the fusion
-    // applies to the first operand.
-    if (IsReductionToVector(*element_instr)) {
-      return element_instr->operand(0)->shape();
-    }
-    return element_instr->shape();
-  };
-
-  // The shapes in all tuple operands should agree, unless it is a reduce.
-  // In that case, the operand of the reduce needs to have the same shape
-  // as the other tuple operands, but also we need to compare the output
-  // shapes of the reduces.
-  auto* element_instr_1 = get_element_instr(instr1);
-  auto* element_instr_2 = get_element_instr(instr2);
-  if (element_instr_1->opcode() == HloOpcode::kReduce &&
-      element_instr_2->opcode() == HloOpcode::kReduce &&
-      !ShapeUtil::Equal(element_instr_1->shape(), element_instr_2->shape())) {
-    return false;
-  }
-  // The elementwise output shapes must be the same (including layout).
-  return ShapeUtil::EqualIgnoringFpPrecision(
-      get_element_shape(element_instr_1), get_element_shape(element_instr_2));
+  return ShapesCompatibleForMultiOutputFusion(*instr1, *instr2);
 }
 
 bool GpuMultiOutputFusion::IsFusible(HloInstruction* instr) {
@@ -205,7 +162,7 @@ bool GpuMultiOutputFusion::DoProducerConsumerMultiOutputFusion() {
         VLOG(3) << producer->name() << " is not a loop fusion.";
         continue;
       }
-      if (!ShapesCompatibleForFusion(producer, consumer)) {
+      if (!ShapesCompatibleForMultiOutputFusion(*producer, *consumer)) {
         VLOG(3) << producer->name() << " has an incompatible shape.";
         continue;
       }
-- 
GitLab


From 85cc252af0b927ce9ebe582678986f264362076f Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Thu, 29 Nov 2018 02:38:40 -0800
Subject: [PATCH 1004/1554] [TF:XLA] Fix implementation of xdivy/xlogy

Select only implicitly broadcasts the predicate operand (it's weird). Get the
zeros in the right type. As far as I can see this never worked.

PiperOrigin-RevId: 223315554
---
 tensorflow/compiler/tests/binary_ops_test.py     | 15 +++++++++++++++
 tensorflow/compiler/tf2xla/kernels/binary_ops.cc | 16 ++++++++--------
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index 332381c59e..9a5423c1b2 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -218,6 +218,21 @@ class BinaryOpsTest(xla_test.XLATestCase):
             ],
             equality_test=self.ListsAreClose)
 
+      # TF doesn't define these for bf16.
+      if dtype != dtypes.bfloat16.as_numpy_dtype:
+        self._testBinary(
+            gen_math_ops.xdivy,
+            np.array([0, 4, 3, 2, 1, 0], dtype=dtype),
+            np.array([0, 5, 6, 7, 8, float("NaN")], dtype=dtype),
+            expected=np.array([0, 0.8, 0.5, 0.285714, 0.125, 0], dtype=dtype))
+
+        self._testBinary(
+            gen_math_ops.xlogy,
+            np.array([0, 4, 3, 2, 1, 0], dtype=dtype),
+            np.array([0, 5, 6, 7, 8, float("NaN")], dtype=dtype),
+            expected=np.array([0, 6.437752, 5.375278, 3.89182, 2.079442, 0],
+                              dtype=dtype))
+
   def testIntOps(self):
     for dtype in self.signed_int_types:
       self._testBinary(
diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index 74c1dd2406..5e9280c1fe 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -106,23 +106,23 @@ static xla::XlaOp FloorDivImpl(xla::XlaBuilder* b, DataType dtype, xla::XlaOp x,
 XLA_MAKE_BINARY(FloorDiv,
                 FloorDivImpl(b, input_type(0), lhs, rhs, broadcast_helper));
 
-static xla::XlaOp XlogyImpl(xla::XlaBuilder* b, DataType dtype, xla::XlaOp x,
-                            xla::XlaOp y, const BCast& broadcast_helper) {
+xla::XlaOp XlogyImpl(xla::XlaOp x, xla::XlaOp y,
+                     const BCast& broadcast_helper) {
   std::tie(x, y) = XlaBinaryOp::Broadcast(x, y, broadcast_helper);
-  auto zero = XlaHelpers::Zero(b, dtype);
+  auto zero = xla::ZerosLike(x);
   auto is_zero = xla::Eq(x, zero);
   return xla::Select(is_zero, zero, xla::Mul(x, xla::Log(y)));
 }
-XLA_MAKE_BINARY(Xlogy, XlogyImpl(b, input_type(0), lhs, rhs, broadcast_helper));
+XLA_MAKE_BINARY(Xlogy, XlogyImpl(lhs, rhs, broadcast_helper));
 
-static xla::XlaOp XdivyImpl(xla::XlaBuilder* b, DataType dtype, xla::XlaOp x,
-                            xla::XlaOp y, const BCast& broadcast_helper) {
+xla::XlaOp XdivyImpl(xla::XlaOp x, xla::XlaOp y,
+                     const BCast& broadcast_helper) {
   std::tie(x, y) = XlaBinaryOp::Broadcast(x, y, broadcast_helper);
-  auto zero = XlaHelpers::Zero(b, dtype);
+  auto zero = xla::ZerosLike(x);
   auto is_zero = xla::Eq(x, zero);
   return xla::Select(is_zero, zero, xla::Div(x, y));
 }
-XLA_MAKE_BINARY(Xdivy, XdivyImpl(b, input_type(0), lhs, rhs, broadcast_helper));
+XLA_MAKE_BINARY(Xdivy, XdivyImpl(lhs, rhs, broadcast_helper));
 
 // Implementation of FloorMod. Pseudo-code:
 // T trunc_mod = std::fmod(x, y);
-- 
GitLab


From 3db3e972a33760ebe6662341d5f4320a906582f7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 03:01:51 -0800
Subject: [PATCH 1005/1554] Enable formatting and format
 depthwiseconv_uint8_3x3_filter.h

This was disabled due to a bug in clang-format, which is fixed now.

PiperOrigin-RevId: 223317889
---
 .../depthwiseconv_uint8_3x3_filter.h          | 168 ++++++++++--------
 1 file changed, 91 insertions(+), 77 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
index 3f2ed0b1f0..5859bcaed4 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
@@ -23,11 +23,6 @@ limitations under the License.
 namespace tflite {
 namespace optimized_ops {
 
-// clang-format gets confused with this file and ends up formatting lines to
-// be larger than 80 characters. Turn off here and back on at the end of the
-// file.
-// clang-format off
-
 // See CategorizeDotProductKernel for definitive taxonomy.
 enum class DotProduct3x3KernelType {
   kNone = 0,  // Parameter combination is not supported for dot product kernels.
@@ -120,42 +115,58 @@ struct DepthwiseConvParams {
 #define OFFSET_OUTPUT_WIDTH 84
 #define OFFSET_OUTPUT_HEIGHT 88
 
-static_assert(offsetof(DepthwiseConvParams, input_depth) ==
-                  OFFSET_INPUT_DEPTH, "");
+static_assert(offsetof(DepthwiseConvParams, input_depth) == OFFSET_INPUT_DEPTH,
+              "");
 static_assert(offsetof(DepthwiseConvParams, input_row_size) ==
-                  OFFSET_INPUT_ROW_SIZE, "");
+                  OFFSET_INPUT_ROW_SIZE,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_depth) ==
-                  OFFSET_OUTPUT_DEPTH, "");
+                  OFFSET_OUTPUT_DEPTH,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_row_size) ==
-                  OFFSET_OUTPUT_ROW_SIZE, "");
+                  OFFSET_OUTPUT_ROW_SIZE,
+              "");
 static_assert(offsetof(DepthwiseConvParams, filter_row_size) ==
-                  OFFSET_FILTER_ROW_SIZE, "");
+                  OFFSET_FILTER_ROW_SIZE,
+              "");
 static_assert(offsetof(DepthwiseConvParams, input_offset) ==
-                  OFFSET_INPUT_OFFSET, "");
+                  OFFSET_INPUT_OFFSET,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_offset) ==
-                  OFFSET_OUTPUT_OFFSET, "");
+                  OFFSET_OUTPUT_OFFSET,
+              "");
 static_assert(offsetof(DepthwiseConvParams, filter_offset) ==
-                  OFFSET_FILTER_OFFSET, "");
+                  OFFSET_FILTER_OFFSET,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_multiplier) ==
-                  OFFSET_OUTPUT_MULTIPLIER, "");
+                  OFFSET_OUTPUT_MULTIPLIER,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_activation_min) ==
-                  OFFSET_OUTPUT_ACTIVATION_MIN, "");
+                  OFFSET_OUTPUT_ACTIVATION_MIN,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_activation_max) ==
-                  OFFSET_OUTPUT_ACTIVATION_MAX, "");
+                  OFFSET_OUTPUT_ACTIVATION_MAX,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_right_shift) ==
-                  OFFSET_OUTPUT_RIGHT_SHIFT, "");
-static_assert(offsetof(DepthwiseConvParams, input_width) ==
-                  OFFSET_INPUT_WIDTH, "");
+                  OFFSET_OUTPUT_RIGHT_SHIFT,
+              "");
+static_assert(offsetof(DepthwiseConvParams, input_width) == OFFSET_INPUT_WIDTH,
+              "");
 static_assert(offsetof(DepthwiseConvParams, input_height) ==
-                  OFFSET_INPUT_HEIGHT, "");
+                  OFFSET_INPUT_HEIGHT,
+              "");
 static_assert(offsetof(DepthwiseConvParams, stride_width) ==
-                  OFFSET_STRIDE_WIDTH, "");
+                  OFFSET_STRIDE_WIDTH,
+              "");
 static_assert(offsetof(DepthwiseConvParams, stride_height) ==
-                  OFFSET_STRIDE_HEIGHT, "");
+                  OFFSET_STRIDE_HEIGHT,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_width) ==
-                  OFFSET_OUTPUT_WIDTH, "");
+                  OFFSET_OUTPUT_WIDTH,
+              "");
 static_assert(offsetof(DepthwiseConvParams, output_height) ==
-                  OFFSET_OUTPUT_HEIGHT, "");
+                  OFFSET_OUTPUT_HEIGHT,
+              "");
 
 template <int32 kDepth, int32 kStrideWidth, int32 kStrideHeight>
 struct DepthwiseConvWindow {};
@@ -164,10 +175,10 @@ template <>
 struct DepthwiseConvWindow<8, 1, 1> {
  public:
   static inline void Run(const uint8* input_ptr, const uint8* filter_ptr,
-                  const int32* bias_ptr, uint8* output_ptr, int64_t input_depth,
-                  int64_t input_row_size, int32 output_window_height,
-                  int32 output_window_width,
-                  const DepthwiseConvParams* params_ptr) {
+                         const int32* bias_ptr, uint8* output_ptr,
+                         int64_t input_depth, int64_t input_row_size,
+                         int32 output_window_height, int32 output_window_width,
+                         const DepthwiseConvParams* params_ptr) {
     const int64_t input_width_increment = 2 * input_depth;
     const int64_t input_height_increment = 2 * input_row_size;
     const int64_t output_height_increment = 2 * params_ptr->output_row_size;
@@ -1147,10 +1158,10 @@ struct DepthwiseConvWindow<8, 1, 1> {
 template <>
 struct DepthwiseConvWindow<8, 2, 2> {
   static inline void Run(const uint8* input_ptr, const uint8* filter_ptr,
-                  const int32* bias_ptr, uint8* output_ptr, int64_t input_depth,
-                  int64_t input_row_size, int32 output_window_height,
-                  int32 output_window_width,
-                  const DepthwiseConvParams* params_ptr) {
+                         const int32* bias_ptr, uint8* output_ptr,
+                         int64_t input_depth, int64_t input_row_size,
+                         int32 output_window_height, int32 output_window_width,
+                         const DepthwiseConvParams* params_ptr) {
     const int64_t input_width_increment = 4 * input_depth;
     const int64_t input_height_increment = 4 * input_row_size;
     const int64_t output_height_increment = 2 * params_ptr->output_row_size;
@@ -2990,11 +3001,10 @@ struct ShuffleParams {
   ShuffleParams() = default;
   ShuffleParams(int32 output_width, int32 output_height, int32 stride_width,
                 int32 stride_height)
-  : output_width(output_width)
-  , output_height(output_height)
-  , input_width(get_shuffle_input_size(stride_width, output_width))
-  , input_height(get_shuffle_input_size(stride_height, output_height)) {
-  }
+      : output_width(output_width),
+        output_height(output_height),
+        input_width(get_shuffle_input_size(stride_width, output_width)),
+        input_height(get_shuffle_input_size(stride_height, output_height)) {}
 };
 
 template <int32 kStrideWidth, int32 kStrideHeight>
@@ -3003,10 +3013,10 @@ struct DepthwiseConvThroughDepth {
   // |start_depth| to |end_depth|. Keep this not inlined to maintain a small
   // binary size. We use a DepthwiseConvParams struct for read only params
   // to minimize call overhead.
-  static __attribute__((noinline)) void Run(const uint8* input_ptr,
-      const uint8* filter_ptr, const int32* bias_ptr, uint8* output_ptr,
-      int64_t start_depth, int64_t end_depth, int64_t input_depth,
-      int64_t input_row_size, int32 output_window_height,
+  static __attribute__((noinline)) void Run(
+      const uint8* input_ptr, const uint8* filter_ptr, const int32* bias_ptr,
+      uint8* output_ptr, int64_t start_depth, int64_t end_depth,
+      int64_t input_depth, int64_t input_row_size, int32 output_window_height,
       int32 output_window_width, const DepthwiseConvParams& params) {
     for (; start_depth <= end_depth - 8; start_depth += 8) {
       DepthwiseConvWindow<8, kStrideWidth, kStrideHeight>::Run(
@@ -3029,12 +3039,15 @@ struct DepthwiseConvMultiRow {
                          uint8* output_data, const DepthwiseConvParams& params,
                          const ShuffleParams& shuffle_params,
                          uint8* shuffle_workspace) {
-    TFLITE_DCHECK(shuffle_params.input_height ==
+    TFLITE_DCHECK(
+        shuffle_params.input_height ==
         get_shuffle_input_size(kStrideHeight, shuffle_params.output_height));
-    TFLITE_DCHECK(shuffle_params.input_width ==
+    TFLITE_DCHECK(
+        shuffle_params.input_width ==
         get_shuffle_input_size(kStrideWidth, shuffle_params.output_width));
-    TFLITE_DCHECK(64 * shuffle_params.input_width * shuffle_params.input_height
-                  <= DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE);
+    TFLITE_DCHECK(64 * shuffle_params.input_width *
+                      shuffle_params.input_height <=
+                  DEPTHWISECONV_SHUFFLE_WORKSPACE_SIZE);
 
     int32 out_x = start_x;
 
@@ -3045,7 +3058,7 @@ struct DepthwiseConvMultiRow {
     if (params.output_depth > 64 ||
         (params.output_depth <= 64 && params.input_width > 150)) {
       for (; out_x <= (end_x - shuffle_params.output_width);
-             out_x += shuffle_params.output_width) {
+           out_x += shuffle_params.output_width) {
         const uint8* input_ptr = input_data;
         const int32* bias_ptr = bias_data;
         const uint8* filter_ptr = filter_data;
@@ -3091,8 +3104,8 @@ struct DepthwiseConvMultiRow {
         }
 
         // Handle leftover depth.
-        ConvKernel::Run(input_ptr, filter_ptr, bias_ptr, output_ptr,
-                        depth, params.output_depth, params.input_depth,
+        ConvKernel::Run(input_ptr, filter_ptr, bias_ptr, output_ptr, depth,
+                        params.output_depth, params.input_depth,
                         params.input_row_size, shuffle_params.output_height,
                         shuffle_params.output_width, params);
 
@@ -3119,13 +3132,15 @@ struct DepthwiseConvMultiRow {
 //   * Horizontal edges.
 //   * Vertical edges.
 inline void DepthwiseConvHandlePadding(const uint8* input_data,
-    const uint8* filter_data, const int32* bias_data, uint8* output_data,
-    const DepthwiseConvParams& params) {
+                                       const uint8* filter_data,
+                                       const int32* bias_data,
+                                       uint8* output_data,
+                                       const DepthwiseConvParams& params) {
   if (params.input_width == 1 && params.input_height == 1) {
-    const uint8* filter_ptr = filter_data + params.filter_row_size
-        + params.output_depth;
-    DepthwiseConvPartial<EdgeType::kCenter, 1, 1>::Run(input_data, filter_ptr,
-        bias_data, output_data, &params);
+    const uint8* filter_ptr =
+        filter_data + params.filter_row_size + params.output_depth;
+    DepthwiseConvPartial<EdgeType::kCenter, 1, 1>::Run(
+        input_data, filter_ptr, bias_data, output_data, &params);
     return;
   }
 
@@ -3136,27 +3151,27 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data,
 
   // Handle top row.
   const uint8* input_ptr = input_data;
-  const uint8* filter_ptr = filter_data + params.filter_row_size
-      + params.output_depth;
+  const uint8* filter_ptr =
+      filter_data + params.filter_row_size + params.output_depth;
   uint8* output_ptr = output_data;
 
-  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(input_ptr, filter_ptr,
-      bias_data, output_ptr, &params);
+  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(
+      input_ptr, filter_ptr, bias_data, output_ptr, &params);
 
   input_ptr += (params.stride_width - 1) * params.input_depth;
   filter_ptr = filter_data + params.filter_row_size;
   output_ptr += params.output_depth;
 
   for (int32 out_x = out_x_start_corner + 1; out_x < out_x_end_corner;
-           out_x++) {
+       out_x++) {
     DepthwiseConvPartial<EdgeType::kHorizontal, 1, 1>::Run(
         input_ptr, filter_ptr, bias_data, output_ptr, &params);
     input_ptr += params.stride_width * params.input_depth;
     output_ptr += params.output_depth;
   }
 
-  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(input_ptr, filter_ptr,
-      bias_data, output_ptr, &params);
+  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(
+      input_ptr, filter_ptr, bias_data, output_ptr, &params);
 
   // Handle left side.
   input_ptr = input_data + (params.stride_width - 1) * params.input_row_size;
@@ -3164,7 +3179,7 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data,
   output_ptr = output_data + params.output_row_size;
 
   for (int32 out_y = out_y_start_corner + 1; out_y < out_y_end_corner;
-           out_y++) {
+       out_y++) {
     DepthwiseConvPartial<EdgeType::kVertical, 1, 1>::Run(
         input_ptr, filter_ptr, bias_data, output_ptr, &params);
     input_ptr += params.stride_width * params.input_row_size;
@@ -3172,14 +3187,14 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data,
   }
 
   // Handle right side.
-  input_ptr = input_data + (params.input_width - 2) * params.input_depth
-      + (params.stride_width - 1) * params.input_row_size;
+  input_ptr = input_data + (params.input_width - 2) * params.input_depth +
+              (params.stride_width - 1) * params.input_row_size;
   filter_ptr = filter_data;
   output_ptr = output_data + params.output_row_size +
-      (params.output_width - 1) * params.output_depth;
+               (params.output_width - 1) * params.output_depth;
 
   for (int32 out_y = out_y_start_corner + 1; out_y < out_y_end_corner;
-         out_y++) {
+       out_y++) {
     DepthwiseConvPartial<EdgeType::kVertical, 1, 1>::Run(
         input_ptr, filter_ptr, bias_data, output_ptr, &params);
     input_ptr += params.stride_width * params.input_row_size;
@@ -3189,26 +3204,26 @@ inline void DepthwiseConvHandlePadding(const uint8* input_data,
   // Handle bottom row.
   input_ptr = input_data + (params.input_height - 2) * params.input_row_size;
   filter_ptr = filter_data + params.output_depth;
-  output_ptr = output_data +
-      (params.output_height - 1) * params.output_row_size;
+  output_ptr =
+      output_data + (params.output_height - 1) * params.output_row_size;
 
-  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(input_ptr, filter_ptr,
-      bias_data, output_ptr, &params);
+  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(
+      input_ptr, filter_ptr, bias_data, output_ptr, &params);
 
   input_ptr += (params.stride_width == 1) ? 0 : params.input_depth;
   filter_ptr = filter_data;
   output_ptr += params.output_depth;
 
   for (int32 out_x = out_x_start_corner + 1; out_x < out_x_end_corner;
-           out_x++) {
+       out_x++) {
     DepthwiseConvPartial<EdgeType::kHorizontal, 1, 1>::Run(
         input_ptr, filter_ptr, bias_data, output_ptr, &params);
     input_ptr += params.stride_width * params.input_depth;
     output_ptr += params.output_depth;
   }
 
-  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(input_ptr, filter_ptr,
-      bias_data, output_ptr, &params);
+  DepthwiseConvPartial<EdgeType::kCorner, 1, 1>::Run(
+      input_ptr, filter_ptr, bias_data, output_ptr, &params);
 }
 
 inline bool Fast3x3FilterKernelSupported(
@@ -3383,8 +3398,8 @@ inline void DepthwiseConv3x3Filter(
       const int in_x = (out_x * stride_width) - pad_width;
       const int in_y = (out_y * stride_height) - pad_height;
       input_ptr += in_y * params.input_row_size + in_x * params.input_depth;
-      output_ptr += out_y * params.output_row_size
-          + out_x * params.output_depth;
+      output_ptr +=
+          out_y * params.output_row_size + out_x * params.output_depth;
     }
 
     // Shuffling shapes that maximize width over the shuffle workspace size
@@ -3439,7 +3454,6 @@ inline void DepthwiseConv3x3Filter(
     }
   }
 }
-// clang-format on
 
 #endif  // __aarch64__
 
-- 
GitLab


From 0c7a31a168aae02e323c97bc6b81f2d3f19cbb2a Mon Sep 17 00:00:00 2001
From: Nutti <nutti.metro@gmail.com>
Date: Thu, 29 Nov 2018 22:32:54 +0900
Subject: [PATCH 1006/1554] Fix: clang-format error

---
 tensorflow/core/kernels/partitioned_function_ops.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index 72310f33ae..8aac78f56f 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -179,10 +179,9 @@ class PartitionedCallOp : public AsyncOpKernel {
                              done);
 
         OP_REQUIRES_OK_ASYNC(
-            ctx,
-            OptimizationPassRegistry::Global()->RunGrouping(
-                OptimizationPassRegistry::POST_REWRITE_FOR_EXEC,
-                optimization_options),
+            ctx, OptimizationPassRegistry::Global()->RunGrouping(
+                     OptimizationPassRegistry::POST_REWRITE_FOR_EXEC,
+                     optimization_options),
             done);
 
         std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
-- 
GitLab


From d00013e72cd3a1a4805395eb8e66748dcf387295 Mon Sep 17 00:00:00 2001
From: Wen yun <shiqing.fsq@alibaba-inc.com>
Date: Wed, 31 Oct 2018 15:22:14 +0800
Subject: [PATCH 1007/1554] fix the case when input value are MirroredVariable
 for assign_moving_average

---
 .../distribute/python/moving_averages_test.py | 20 +++++++++++++++++++
 .../python/distribute/mirrored_strategy.py    |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/moving_averages_test.py b/tensorflow/contrib/distribute/python/moving_averages_test.py
index c492d8bafc..da3353b2d5 100644
--- a/tensorflow/contrib/distribute/python/moving_averages_test.py
+++ b/tensorflow/contrib/distribute/python/moving_averages_test.py
@@ -139,6 +139,26 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
            (2.0 * 0.25 + 0.0) / (1.0 * 0.25 + 1.0)],
           var.eval())
 
+  @combinations.generate(all_combinations)
+  def testAssignVariable(self, distribution):
+    def replica_fn():
+      var = variables.Variable([10.0, 11.0])
+      # Here we expect to check the case when input value are variable.
+      val = variables.Variable([1., 2.])
+      decay = 0.25
+      assign = moving_averages.assign_moving_average(
+          var, val, decay, zero_debias=False)
+      return var, assign
+
+    with distribution.scope(), self.cached_session() as sess:
+      var, assign = distribution.call_for_each_replica(replica_fn)
+      variables.global_variables_initializer().run()
+      self.assertAllClose([10.0, 11.0], var.eval())
+      sess.run(distribution.unwrap(assign))
+      self.assertAllClose(
+          [10 * 0.25 + 1. * (1 - 0.25),
+           11 * 0.25 + 2. * (1 - 0.25)],
+          var.eval())
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index d6d40df5ce..3cd5cf09c0 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -598,6 +598,9 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     return self._cross_device_ops
 
   def _reduce_to(self, reduce_op, value, destinations):
+    if (isinstance(value, values.Mirrored) and
+        reduce_op == reduce_util.ReduceOp.MEAN):
+      return value
     assert not isinstance(value, values.Mirrored)
     if not isinstance(value, values.DistributedValues):
       # This function handles reducing values that are not PerReplica or
-- 
GitLab


From 567b2af37ef10fb0a6ae9727410b202c9b33adb3 Mon Sep 17 00:00:00 2001
From: Thomas Joerg <tjoerg@google.com>
Date: Thu, 29 Nov 2018 07:15:16 -0800
Subject: [PATCH 1008/1554] [XLA:GPU] Make sure reduction dimensions match for
 multi-output fusion.

PiperOrigin-RevId: 223340884
---
 .../compiler/xla/service/gpu/gpu_fusible.cc   |  5 +--
 .../xla/service/gpu/gpu_fusible_test.cc       | 31 +++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
index 5eafac7232..452e763a8e 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible.cc
@@ -118,12 +118,13 @@ bool ShapesCompatibleForMultiOutputFusion(const HloInstruction& instr1,
   // All shapes of the root tuple of multi-output fusions should agree, i.e. all
   // root ops should have equal output shapes. An exception are
   // reduction-to-vector ops. Here the input shapes of the reduction (first
-  // operand shape) need to be considered.
+  // operand shape) and the reduction dimensions need to match.
   auto* instr_1 = get_real_hero(&instr1);
   auto* instr_2 = get_real_hero(&instr2);
   // TODO(tjoerg): Relax the shape constraint. The datatype does not matter.
   if (IsReductionToVector(*instr_1) && IsReductionToVector(*instr_2) &&
-      !ShapeUtil::Equal(instr_1->shape(), instr_2->shape())) {
+      (!ShapeUtil::Equal(instr_1->shape(), instr_2->shape()) ||
+       instr_1->dimensions() != instr_2->dimensions())) {
     return false;
   }
   // The elementwise output shapes must be the same (including layout).
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
index ca07851f8b..15d4ee206c 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_fusible_test.cc
@@ -568,6 +568,37 @@ TEST_F(GpuFusibleTest, ShapesCompatibleForMultiOutputFusion_ReduceFusions) {
   EXPECT_TRUE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
 }
 
+TEST_F(GpuFusibleTest,
+       ShapesCompatibleForMultiOutputFusion_DifferentReduceDimensions) {
+  auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
+    fused_reduce_1 {
+      p0.1 = f32[2,2,2]{2,1,0} parameter(0)
+      c0 = f32[] constant(0)
+      ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} p0.1, f32[] c0), dimensions={0}, to_apply=scalar_add
+    }
+
+    fused_reduce_2 {
+      p0.2 = f32[2,2,2]{2,1,0} parameter(0)
+      mul = f32[2,2,2]{2,1,0} multiply(f32[2,2,2]{2,1,0} p0.2, f32[2,2,2]{2,1,0} p0.2)
+      c1 = f32[] constant(0)
+      ROOT reduce = f32[2,2]{1,0} reduce(f32[2,2,2]{2,1,0} mul, f32[] c1), dimensions={2}, to_apply=scalar_add
+    }
+
+    ENTRY reduce {
+      p0 = f32[2,2,2]{2,1,0} parameter(0)
+      p1 = f32[2,2,2]{2,1,0} parameter(1)
+      reduce_1 = f32[2,2]{1,0} fusion(p0), kind=kLoop, calls=fused_reduce_1
+      reduce_2 = f32[2,2]{1,0} fusion(p1), kind=kLoop, calls=fused_reduce_2
+      ROOT root = (f32[2,2]{1,0}, f32[2,2,2]{2,1,0}) tuple(reduce_1, reduce_2)
+    })"))
+                    .ValueOrDie();
+  const HloInstruction* fusion_1 =
+      module->entry_computation()->root_instruction()->operand(0);
+  const HloInstruction* fusion_2 =
+      module->entry_computation()->root_instruction()->operand(1);
+  EXPECT_FALSE(ShapesCompatibleForMultiOutputFusion(*fusion_1, *fusion_2));
+}
+
 TEST_F(GpuFusibleTest,
        ShapesCompatibleForMultiOutputFusion_NoReductionToVector) {
   auto module = ParseHloString(absl::StrCat(kModulePrefix, R"(
-- 
GitLab


From 8497600aded3df12d781758428e12c4f8178f008 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 29 Nov 2018 07:40:41 -0800
Subject: [PATCH 1009/1554] Add cond_v2 and while_v2 as a dependency for tests

PiperOrigin-RevId: 223343922
---
 tensorflow/python/BUILD       | 2 ++
 tensorflow/python/debug/BUILD | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 2dbca6ed6e..6e4945e6fd 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1393,6 +1393,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = ["no_pip"],  # test_ops_2 is not available in pip.
     deps = [
+        ":cond_v2",
         ":control_flow_ops",
         ":errors",
         ":framework",
@@ -1407,6 +1408,7 @@ py_test(
         ":util",
         ":variable_scope",
         ":variables",
+        ":while_v2",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:function",
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index 7995123209..c6abd476d9 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -557,6 +557,7 @@ py_test(
         ":source_utils",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client",
+        "//tensorflow/python:cond_v2",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_ops",
@@ -566,6 +567,7 @@ py_test(
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
+        "//tensorflow/python:while_v2",
         "//third_party/py/numpy",
     ],
 )
-- 
GitLab


From b692e8a23d91cda733e45041fb36fe045a43872f Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Thu, 29 Nov 2018 07:45:13 -0800
Subject: [PATCH 1010/1554] Improve performance of Operation.get_attr somewhat

This function is the fourth most expensive in a simple example I have been benchmarking. It had a somewhat odd loop over possible fields when the code has already checked which field is present using WhichOneof. I've fixed this (though there is still a loop when the oneof field was a list, which I believe is unavoidable.)

PiperOrigin-RevId: 223344446
---
 tensorflow/python/framework/ops.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index fc692a2510..bd798f9ffa 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2386,7 +2386,7 @@ class Operation(object):
     Raises:
       ValueError: If this op does not have an attr with the given `name`.
     """
-    fields = ["s", "i", "f", "b", "type", "shape", "tensor", "func"]
+    fields = ("s", "i", "f", "b", "type", "shape", "tensor", "func")
     try:
       with c_api_util.tf_buffer() as buf:
         c_api.TF_OperationGetAttrValueProto(self._c_op, name, buf)
@@ -2397,25 +2397,21 @@ class Operation(object):
     x = attr_value_pb2.AttrValue()
     x.ParseFromString(data)
 
-    # Treat an empty oneof value as an empty list.
-    if not x.WhichOneof("value"):
+    oneof_value = x.WhichOneof("value")
+    if oneof_value is None:
       return []
-    if x.HasField("list"):
+    if oneof_value == "list":
       for f in fields:
         if getattr(x.list, f):
           if f == "type":
-            return [dtypes.as_dtype(x) for x in list(getattr(x.list, f))]
+            return [dtypes.as_dtype(t) for t in x.list.type]
           else:
             return list(getattr(x.list, f))
       return []
-    else:
-      for f in fields:
-        if x.HasField(f):
-          if f == "type":
-            return dtypes.as_dtype(getattr(x, f))
-          else:
-            return getattr(x, f)
-      assert False, "Unsupported field type in " + str(x)
+    if oneof_value == "type":
+      return dtypes.as_dtype(x.type)
+    assert oneof_value in fields, "Unsupported field type in " + str(x)
+    return getattr(x, oneof_value)
 
   def run(self, feed_dict=None, session=None):
     """Runs this operation in a `Session`.
-- 
GitLab


From c04f93a0f90ca48e2a3959dbd6554bc3f3f9b320 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 29 Nov 2018 07:46:00 -0800
Subject: [PATCH 1011/1554] `defun` is not currently being exposed in the V2
 API, so we cannot pass it via generated code. For this reason we handle it
 separately.

PiperOrigin-RevId: 223344539
---
 tensorflow/python/autograph/core/converter.py | 13 ++++++++++---
 tensorflow/python/framework/func_graph.py     |  2 +-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/autograph/core/converter.py b/tensorflow/python/autograph/core/converter.py
index 49e24895a2..e88c4674ee 100644
--- a/tensorflow/python/autograph/core/converter.py
+++ b/tensorflow/python/autograph/core/converter.py
@@ -82,6 +82,7 @@ from tensorflow.python.autograph.pyct.static_analysis import live_values
 from tensorflow.python.autograph.pyct.static_analysis import liveness
 from tensorflow.python.autograph.pyct.static_analysis import reaching_definitions
 from tensorflow.python.autograph.pyct.static_analysis import type_info
+from tensorflow.python.eager import function
 
 # TODO(mdan): These contexts can be refactored into first class objects.
 # For example, we could define Program and Entity abstractions that hold on
@@ -96,7 +97,7 @@ class Verbosity(IntEnum):
   Attributes:
    * BRIEF: No logging, minimal error messages.
    * VERBOSE: Detailed logging of generated code, detailed error messages.
- """
+  """
   BRIEF = 0
   VERBOSE = 1
 
@@ -151,7 +152,7 @@ class ConversionOptions(object):
                optional_features=Feature.ALL):
     self.recursive = recursive
     self.verbose = verbose
-    self.strip_decorators = strip_decorators or ()
+    self._strip_decorators = strip_decorators or ()
     self.force_conversion = force_conversion
     # TODO(mdan): Rename to conversion_recursion_depth?
     self.internal_convert_user_code = internal_convert_user_code
@@ -161,6 +162,12 @@ class ConversionOptions(object):
     optional_features = frozenset(optional_features)
     self.optional_features = optional_features
 
+  @property
+  def strip_decorators(self):
+    # A few decorators are included by default.
+    # TODO(mdan): Revert if function.defun becomes a public symbol.
+    return self._strip_decorators + (function.defun,)
+
   def uses(self, feature):
     return (Feature.ALL in self.optional_features or
             feature in self.optional_features)
@@ -216,7 +223,7 @@ class ConversionOptions(object):
             as_qualified_name(ConversionOptions)),
         recursive_val=parser.parse_expression(str(self.recursive)),
         verbose_val=parser.parse_expression(str(int(self.verbose))),
-        strip_decorators_val=list_of_names(self.strip_decorators),
+        strip_decorators_val=list_of_names(self._strip_decorators),
         force_conversion_val=parser.parse_expression(
             str(self.force_conversion)),
         internal_convert_user_code_val=parser.parse_expression(
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 9960e23e6f..a8a7948b99 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -399,7 +399,7 @@ def func_graph_from_py_func(name,
               autograph.ConversionOptions(
                   verbose=autograph.Verbosity.BRIEF,
                   recursive=True,
-                  strip_decorators=(function.defun, def_function.function),
+                  strip_decorators=(def_function.function,),
                   optional_features=(),
               ), *args, **kwargs)
 
-- 
GitLab


From c25a0170cd29e0bbc3fd662104494872de38f392 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 29 Nov 2018 08:19:42 -0800
Subject: [PATCH 1012/1554] [tf.data] Mitigating the effect map transformation
 stragglers have on the performance of the MapAndBatchDataset kernel.

This CL automatically increases the size of the internal MapAndBatchDataset::GetNext() buffer to prevent map transformation stragglers from reducing effective parallelism of the kernel.

PiperOrigin-RevId: 223348858
---
 .../kernels/data/map_and_batch_dataset_op.cc  | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
index f389ff12c5..014f37686f 100644
--- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
@@ -38,6 +38,9 @@ namespace tensorflow {
 namespace data {
 namespace {
 
+// Maximum number of batch results to buffer.
+const int64 kMaxBatchResults = 16;
+
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
@@ -243,7 +246,11 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
             cond_var_(std::make_shared<condition_variable>()),
             num_parallel_calls_(std::make_shared<model::SharedState>(
                 params.dataset->num_parallel_calls_, mu_, cond_var_)),
-            map_func_(std::move(map_func)) {
+            map_func_(std::move(map_func)),
+            max_batch_results_(std::min(kMaxBatchResults,
+                                        (params.dataset->num_parallel_calls_ +
+                                         params.dataset->batch_size_ - 1) /
+                                            params.dataset->batch_size_)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
         prefix_end_ = components.back();
@@ -280,9 +287,11 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
           EnsureRunnerThreadStarted(ctx);
           while (batch_results_.empty() ||
                  batch_results_.front()->num_calls > 0) {
+            ++waiting_;
             RecordStop(ctx);
             cond_var_->wait(l);
             RecordStart(ctx);
+            --waiting_;
           }
           std::swap(result, batch_results_.front());
           batch_results_.pop_front();
@@ -572,18 +581,24 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         }
         auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
           int64 num_parallel_calls = num_parallel_calls_->value;
-          int64 max_batch_results =
-              (num_parallel_calls + dataset()->batch_size_ - 1) /
-              dataset()->batch_size_;
           return num_calls_ >= num_parallel_calls ||
-                 (batch_results_.size() > max_batch_results ||
-                  (batch_results_.size() == max_batch_results &&
+                 (batch_results_.size() > max_batch_results_ ||
+                  (batch_results_.size() == max_batch_results_ &&
                    call_counter_ % dataset()->batch_size_ == 0));
         };
         while (true) {
           {
             mutex_lock l(*mu_);
             while (!cancelled_ && busy()) {
+              if (waiting_ > 0 && num_calls_ < num_parallel_calls_->value &&
+                  max_batch_results_ < kMaxBatchResults) {
+                // If there is a caller waiting for a batch and the number of
+                // outstanding calls is not maxed out, it means we are out of
+                // `batch_results_` slots. Instead of waiting for a slot to open
+                // up, we create a new one to utilize CPU efficiently.
+                max_batch_results_++;
+                continue;
+              }
               RecordStop(ctx.get());
               cond_var_->wait(l);
               RecordStart(ctx.get());
@@ -761,8 +776,14 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       std::unique_ptr<IteratorBase> input_impl_;
       // Buffer for storing the (intermediate) batch results.
       std::deque<std::shared_ptr<BatchResult>> batch_results_ GUARDED_BY(*mu_);
+      // Background thread used for coordinating input processing.
       std::unique_ptr<Thread> runner_thread_ GUARDED_BY(*mu_);
+      // Determines whether the transformation has been cancelled.
       bool cancelled_ GUARDED_BY(*mu_) = false;
+      // Identifies the number of callers currently waiting for a batch result.
+      int64 waiting_ GUARDED_BY(*mu_) = 0;
+      // Identifies the maximum number of batch results to store.
+      int64 max_batch_results_ GUARDED_BY(*mu_);
       string prefix_end_;
     };
 
-- 
GitLab


From 2685945640fb1d62135e43b1d0d05c21649adb3f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 08:22:27 -0800
Subject: [PATCH 1013/1554] internal change only

PiperOrigin-RevId: 223349157
---
 tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 10 +++++-----
 tensorflow/compiler/xla/service/hlo_graph_dumper.h  | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 7e9e94ca5f..804feff290 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -1268,12 +1268,12 @@ const HloInstruction* HloDotDumper::GetNodeForEdge(
 
 class GraphRendererRegistry {
  public:
-  void AddRenderer(GraphRendererInterface* graph_renderer) {
+  void SetRenderer(std::shared_ptr<GraphRendererInterface> graph_renderer) {
     tensorflow::mutex_lock lock(mu_);
     graph_renderer_ = graph_renderer;
   }
 
-  GraphRendererInterface* GetDefaultRenderer() {
+  std::shared_ptr<GraphRendererInterface> GetDefaultRenderer() {
     tensorflow::mutex_lock lock(mu_);
     return graph_renderer_;
   }
@@ -1285,13 +1285,13 @@ class GraphRendererRegistry {
 
  private:
   tensorflow::mutex mu_;
-  GraphRendererInterface* graph_renderer_ = nullptr;
+  std::shared_ptr<GraphRendererInterface> graph_renderer_ GUARDED_BY(mu_);
 };
 
 }  // namespace
 
-Registrar::Registrar(GraphRendererInterface* dumper) {
-  GraphRendererRegistry::Default()->AddRenderer(dumper);
+Registrar::Registrar(std::shared_ptr<GraphRendererInterface> dumper) {
+  GraphRendererRegistry::Default()->SetRenderer(dumper);
 }
 
 namespace {
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index 0b11f34abb..8d5945aba8 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -87,13 +87,13 @@ void DumpText(const HloModule& module, const string& label,
 // Class that registers a graph renderer.
 class Registrar {
  public:
-  Registrar(GraphRendererInterface* dumper);
+  Registrar(std::shared_ptr<GraphRendererInterface> dumper);
 };
 
-#define XLA_INTERNAL_REGISTER_GRAPH_RENDERER(factory, ctr, ...)   \
-  static ::xla::hlo_graph_dumper::Registrar                       \
-      XLA_INTERNAL_REGISTER_GRAPH_RENDERER_NAME(ctr)(new factory, \
-                                                     ##__VA_ARGS__)
+#define XLA_INTERNAL_REGISTER_GRAPH_RENDERER(factory, ctr, ...) \
+  static ::xla::hlo_graph_dumper::Registrar                     \
+      XLA_INTERNAL_REGISTER_GRAPH_RENDERER_NAME(ctr)(           \
+          std::make_shared<factory>(), ##__VA_ARGS__)
 
 // __COUNTER__ must go through another macro to be properly expanded
 #define XLA_INTERNAL_REGISTER_GRAPH_RENDERER_NAME(ctr) ___##ctr##__object_
-- 
GitLab


From 0b3da9fad5a41276753113b4ceb57ab821c5397f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 08:34:36 -0800
Subject: [PATCH 1014/1554] Update protobuf to 3.6.1.1

protobuf 3.6.1.1 contains a fix for protobuf bzl file. This will fix TF Windows build with Bazel@HEAD, allowing us to test TensorFlow in downstream.

PiperOrigin-RevId: 223350741
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0ea5d68ea8..016572a543 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -347,11 +347,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     )
 
     PROTOBUF_URLS = [
-        "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.tar.gz",
-        "https://github.com/google/protobuf/archive/v3.6.1.tar.gz",
+        "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.1.tar.gz",
+        "https://github.com/google/protobuf/archive/v3.6.1.1.tar.gz",
     ]
-    PROTOBUF_SHA256 = "3d4e589d81b2006ca603c1ab712c9715a76227293032d05b26fca603f90b3f5b"
-    PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1"
+    PROTOBUF_SHA256 = "1ade182f91f0fa6c6116195def5d22270e01b9d03fe91319e4c6215022d0d24b"
+    PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1.1"
 
     tf_http_archive(
         name = "protobuf_archive",
-- 
GitLab


From aafc5f86303ffea242ba99ae487f5206375b94f7 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Thu, 29 Nov 2018 09:07:24 -0800
Subject: [PATCH 1015/1554] Add a note about tf.flags to the converter, and
 default the copy_other_files option to true. Fix broken tf_upgrade_v2
 installation in pip, and optimize RE compilation in AST editor.

PiperOrigin-RevId: 223355625
---
 tensorflow/tools/compatibility/ast_edits.py   | 21 ++++++++++++-------
 .../tools/compatibility/tf_upgrade_v2.py      | 20 ++++++++++++------
 .../tools/compatibility/tf_upgrade_v2_main.py |  8 +++++--
 .../tools/compatibility/tf_upgrade_v2_test.py |  5 ++---
 tensorflow/tools/pip_package/setup.py         |  2 +-
 5 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 90bfab3507..eac2150502 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -21,11 +21,16 @@ from __future__ import print_function
 import ast
 import collections
 import os
+import re
 import shutil
 import sys
 import tempfile
 import traceback
 
+# Some regular expressions we will need for parsing
+FIND_OPEN = re.compile(r"^\s*(\[).*$")
+FIND_STRING_CHARS = re.compile(r"['\"]")
+
 
 class APIChangeSpec(object):
   """This class defines the transformations that need to happen.
@@ -57,7 +62,7 @@ class _FileEditTuple(
   Fields:
     comment: A description of the edit and why it was made.
     line: The line number in the file where the edit occurs (1-indexed).
-    start: The line number in the file where the edit occurs (0-indexed).
+    start: The column number in the file where the edit occurs (0-indexed).
     old: text string to remove (this must match what was in file).
     new: text string to add in place of `old`.
   """
@@ -248,13 +253,12 @@ class _ASTCallVisitor(ast.NodeVisitor):
     This is necessary mainly because ListComp's location reporting reports
     the next token after the list comprehension list opening.
 
+    Returns:
+      lineno, offset for the given node
+
     Args:
       node: Node for which we wish to know the lineno and col_offset
     """
-    import re
-    find_open = re.compile("^\s*(\\[).*$")
-    find_string_chars = re.compile("['\"]")
-
     if isinstance(node, ast.ListComp):
       # Strangely, ast.ListComp returns the col_offset of the first token
       # after the '[' token which appears to be a bug. Workaround by
@@ -268,7 +272,7 @@ class _ASTCallVisitor(ast.NodeVisitor):
         reversed_preceding_text = text[:col][::-1]
         # First find if a [ can be found with only whitespace between it and
         # col.
-        m = find_open.match(reversed_preceding_text)
+        m = FIND_OPEN.match(reversed_preceding_text)
         if m:
           new_col_offset = col - m.start(1) - 1
           return line, new_col_offset
@@ -287,7 +291,7 @@ class _ASTCallVisitor(ast.NodeVisitor):
             comment_start = prev_line.find("#")
             if comment_start == -1:
               col = len(prev_line) - 1
-            elif find_string_chars.search(prev_line[comment_start:]) is None:
+            elif FIND_STRING_CHARS.search(prev_line[comment_start:]) is None:
               col = comment_start
             else:
               return None, None
@@ -391,8 +395,9 @@ class _ASTCallVisitor(ast.NodeVisitor):
     """
     full_name, _ = self._get_attribute_full_path(node)
     if full_name:
-      self._rename_functions(node, full_name)
+      # Make sure the warning comes first, otherwise the name may have changed
       self._print_warning_for_function(node, full_name)
+      self._rename_functions(node, full_name)
     if full_name in self._api_change_spec.change_to_function:
       if not hasattr(node, "is_function_for_call"):
         new_text = full_name + "()"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 649d1c17a3..242bbbc3a7 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -619,11 +619,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     }
 
     decay_function_comment = (
-        "ERROR: <function name> has been changed to return a callable instead "
-        "of a tensor when graph building, but its functionality remains "
+        "WARNING: <function name> has been changed to return a callable instead"
+        " of a tensor when graph building, but its functionality remains "
         "unchanged during eager execution (returns a callable like "
         "before). The converter cannot detect and fix this reliably, so "
-        "you need to inspect this usage manually.\n"
+        "this usage has been converted to compat.v1 (even though it may already"
+        " be correct).\n"
     )
 
     # TODO(b/118888586): add default value change to update script.
@@ -635,17 +636,23 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     assert_return_type_comment = (
         "WARNING: assert_* functions have been changed to return None, the "
         "data argument has been removed, and arguments have been reordered."
+        "\nThe calls have been converted to compat.v1 for safety (even though "
+        " they may already have been correct)."
     )
 
     assert_rank_comment = (
         "WARNING: assert_rank_* functions have been changed to return None, and"
         " the data and summarize arguments have been removed."
+        "\nThe calls have been converted to compat.v1 for safety (even though "
+        " they may already have been correct)."
     )
 
     tf_01s_like_no_optimize_comment = (
         "WARNING: tf.zeros_like and tf.ones_like no longer have the optimize "
         "argument in TF 2.0 or after (also, `tensor' argument is renamed to "
         "`input')."
+        "\nThe calls have been converted to compat.v1 for safety (even though "
+        " they may already have been correct)."
     )
 
     # Function warnings. <function name> placeholder inside warnings will be
@@ -670,6 +677,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.debugging.assert_rank": assert_rank_comment,
         "tf.debugging.assert_rank_at_least": assert_rank_comment,
         "tf.debugging.assert_rank_in": assert_rank_comment,
+        "tf.flags": "tf.flags has been removed, please use the argparse or absl"
+                    " module if you need command line parsing.",
         "tf.train.exponential_decay":
             decay_function_comment,
         "tf.train.piecewise_constant":
@@ -720,15 +729,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.zeros_like": tf_01s_like_no_optimize_comment,
         "tf.ones_like": tf_01s_like_no_optimize_comment,
     }
-    # Right now we can't have both a rename and a warning.
+
     self.symbol_renames = {
         name: new_name
         for name, new_name in self.symbol_renames.items()
-        if name not in self.function_warnings
     }
 
     export_saved_model_renamed = (
-        "(Manual edit required) Please rename the function export_savedmodel() "
+        "(Manual edit required) Please rename the method export_savedmodel() "
         "to export_saved_model(). Two things to note:\n\t(1) The argument "
         "strip_default_attributes has been removed. The function will always "
         "strip the default attributes from ops. If this breaks your code, "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index 498258dfa8..543d078642 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -24,7 +24,7 @@ from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import tf_upgrade_v2
 
 
-if __name__ == "__main__":
+def main():
   parser = argparse.ArgumentParser(
       formatter_class=argparse.RawDescriptionHelpFormatter,
       description="""Convert a TensorFlow Python file to 2.0
@@ -58,7 +58,7 @@ Simple usage:
       help=("If converting a whole tree of files, whether to "
             "copy the other files."),
       type=bool,
-      default=False)
+      default=True)
   parser.add_argument(
       "--reportfile",
       dest="report_filename",
@@ -98,3 +98,7 @@ Simple usage:
     print("-" * 80)
     print("\n".join(errors))
     print("\nMake sure to read the detailed log %r\n" % report_filename)
+
+
+if __name__ == "__main__":
+  main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0414becc70..c490ebd480 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -97,8 +97,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                   "tf.train.noisy_linear_cosine_decay"]:
 
       text = "%s(a, b)\n" % decay
-      _, report, errors, new_text = self._upgrade(text)
-      self.assertEqual(text, new_text)
+      _, report, errors, _ = self._upgrade(text)
       self.assertEqual(errors, ["test.py:1: %s requires manual check." % decay])
       self.assertIn("%s has been changed" % decay, report)
 
@@ -216,7 +215,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     text = "self.est.export_savedmodel(path)"
     _, report, unused_errors, unused_new_text = self._upgrade(text)
     self.assertIn(
-        "rename the function export_savedmodel() to export_saved_model()",
+        "rename the method export_savedmodel() to export_saved_model()",
         report)
 
   def testArgmin(self):
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 34c600abaf..85c913f158 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -107,7 +107,7 @@ CONSOLE_SCRIPTS = [
     # TensorBoard command, pip will inappropriately remove it during install,
     # even though the command is not removed, just moved to a different wheel.
     'tensorboard = tensorboard.main:run_main',
-    'tf_upgrade_v2 = tensorflow.tools.compatibility.tf_upgrade_v2:main',
+    'tf_upgrade_v2 = tensorflow.tools.compatibility.tf_upgrade_v2_main:main',
 ]
 # pylint: enable=line-too-long
 
-- 
GitLab


From 8b9064ab985706fa62031d1dd0c961e9ceb82061 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 09:12:18 -0800
Subject: [PATCH 1016/1554] Don't skip MetaOptimizer if the feed nodes are
 Const or the shape is undefined. Fix bug in arithmetic optimizer: Skip Pow
 nodes with unknown input shapes.

PiperOrigin-RevId: 223356331
---
 .../common_runtime/graph_execution_state.cc   | 82 ++++++++++++++-----
 .../optimizers/arithmetic_optimizer.cc        | 18 ++--
 2 files changed, 69 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 0d36930324..9a56c67162 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -25,9 +25,11 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/graph.pb_text.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph.h"
@@ -37,6 +39,7 @@ limitations under the License.
 #include "tensorflow/core/graph/validate.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/flatset.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
@@ -393,6 +396,42 @@ Status ValidateFeedAndFetchDevices(
   }
   return Status::OK();
 }
+
+Status GetFeedShapeAndTypeFromAttribute(const NodeDef& node,
+                                        PartialTensorShape* shape,
+                                        DataType* type) {
+  static const gtl::FlatSet<string>* const kHasExplicitShapeAttribute =
+      CHECK_NOTNULL((new gtl::FlatSet<string>{
+          "Placeholder", "PlaceholderV2", "PlaceholderWithDefault",
+          "ParallelConcat", "ImmutableConst", "_ParallelConcatStart",
+          "InfeedDequeue", "OutfeedDequeue", "CollectiveBcastSend",
+          "CollectiveBcastRecv", "AccumulateNV2", "VariableV2", "Variable",
+          "TemporaryVariable", "NcclBroadcast", "_ScopedAllocator",
+          "_ScopedAllocatorConcat"}));
+
+  // All the node types handled here have their output datatype set in
+  // either attribute 'dtype' or 'T'.
+  if (!GetNodeAttr(node, "dtype", type).ok() &&
+      !GetNodeAttr(node, "T", type).ok()) {
+    return errors::InvalidArgument(
+        "Could not determine output type for feed node: ", node.name(),
+        " of type ", node.op());
+  }
+
+  // First handle the case of feeding a const node.
+  if (node.op() == "Const" && HasNodeAttr(node, "value")) {
+    *shape =
+        PartialTensorShape(node.attr().at("value").tensor().tensor_shape());
+  } else if (kHasExplicitShapeAttribute->find(node.op()) !=
+             kHasExplicitShapeAttribute->end()) {
+    TF_RETURN_IF_ERROR(GetNodeAttr(node, "shape", shape));
+  } else {
+    return errors::InvalidArgument("Could not determine shape for feed node: ",
+                                   node.name(), " of type ", node.op());
+  }
+  return Status::OK();
+}
+
 }  // namespace
 
 Status GraphExecutionState::PruneGraph(
@@ -553,9 +592,6 @@ Status GraphExecutionState::OptimizeGraph(
   }
 
   if (grappler::MetaOptimizerEnabled(session_options_->config)) {
-    // Adding this functionality in steps. The first step is to make sure
-    // we don't break dependencies. The second step will be to turn the
-    // functionality on by default.
     grappler::GrapplerItem item;
     item.id = "tf_graph";
     graph_->ToGraphDef(&item.graph);
@@ -599,26 +635,30 @@ Status GraphExecutionState::OptimizeGraph(
         if (feeds.find(node.name()) == feeds.end()) {
           continue;
         }
-        if (node.attr().count("dtype") == 0 ||
-            node.attr().count("shape") == 0) {
-          return errors::InvalidArgument("Missing node shape or type");
-        }
-        TensorShapeProto shape_proto(node.attr().at("shape").shape());
-        // If the shape of the placeholder value is only partially known,
-        // we're free to use any dimension we want to feed the placeholder. We
-        // choose 1 to minimize the memory impact. Note that this only matters
-        // if an optimizer choose to run the graph to build its cost model,
-        // which doesn't happen (yet)
-        if (shape_proto.unknown_rank()) {
-          shape_proto.set_unknown_rank(false);
-        }
-        for (auto& dim : *shape_proto.mutable_dim()) {
-          if (dim.size() < 0) {
-            dim.set_size(1);
+        // Get the type and shape of the feed node.
+        PartialTensorShape partial_shape;
+        DataType type;
+        TF_RETURN_IF_ERROR(
+            GetFeedShapeAndTypeFromAttribute(node, &partial_shape, &type));
+        // If the shape of the placeholder is only partially known, we are free
+        // to set unknown dimensions of its shape to any value we desire. We
+        // choose 0 to minimize the memory impact. Note that this only matters
+        // if an optimizer chooses to run the graph.
+        TensorShape shape;
+        if (partial_shape.unknown_rank()) {
+          shape = TensorShape({0});
+        } else {
+          for (int i = 0; i < partial_shape.dims(); ++i) {
+            if (partial_shape.dim_size(i) < 0) {
+              partial_shape.set_dim(i, 0);
+            }
+          }
+          if (!partial_shape.AsTensorShape(&shape)) {
+            return errors::InvalidArgument(
+                "Could not derive shape for feed node: ", node.DebugString());
           }
         }
-        TensorShape shape(shape_proto);
-        DataType type = node.attr().at("dtype").type();
+
         Tensor fake_input(type, shape);
         item.feed.emplace_back(node.name(), fake_input);
       }
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index e676323bf4..e3ac89b50d 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -2407,11 +2407,10 @@ class ConvertPowStage : public ArithmeticOptimizerStage {
   Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
     const auto& pow_props =
         ctx().graph_properties->GetInputProperties(node->name())[1];
-    for (int i = 0; i < pow_props.shape().dim_size(); ++i) {
-      if (pow_props.shape().dim(i).size() < 0) {
-        // skip if p is not fully defined.
-        return Status::OK();
-      }
+    PartialTensorShape shape(pow_props.shape());
+    if (!shape.IsFullyDefined()) {
+      // skip if p is not fully defined.
+      return Status::OK();
     }
     if (TensorShape::IsValid(pow_props.shape()) && pow_props.has_value()) {
       Tensor pow(pow_props.dtype(), pow_props.shape());
@@ -2459,11 +2458,10 @@ class ConvertPowStage : public ArithmeticOptimizerStage {
         AddToOptimizationQueue(y);
       } else if (curr == complex128(0, 0) &&
                  ShapesSymbolicallyEqual(value_props.shape(), output_shape)) {
-        for (int i = 0; i < value_props.shape().dim_size(); ++i) {
-          if (value_props.shape().dim(i).size() < 0) {
-            // skip if b is not fully defined.
-            return Status::OK();
-          }
+        PartialTensorShape shape(value_props.shape());
+        if (!shape.IsFullyDefined()) {
+          // skip if b is not fully defined.
+          return Status::OK();
         }
         if (TensorShape::IsValid(value_props.shape()) &&
             value_props.has_value()) {
-- 
GitLab


From 30f5cf80481edad74dc3eae4e8add8e9b15874ea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 09:15:09 -0800
Subject: [PATCH 1017/1554] Nit: Added a missing space in an error message.
 PiperOrigin-RevId: 223356708

---
 tensorflow/python/client/session.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 22303e95c1..87a200ed33 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -1097,7 +1097,7 @@ class BaseSession(SessionInterface):
           if isinstance(subfeed_val, ops.Tensor):
             raise TypeError('The value of a feed cannot be a tf.Tensor object. '
                             'Acceptable feed values include Python scalars, '
-                            'strings, lists, numpy ndarrays, or TensorHandles.'
+                            'strings, lists, numpy ndarrays, or TensorHandles. '
                             'For reference, the tensor object was ' +
                             str(feed_val) + ' which was passed to the '
                             'feed with key ' + str(feed) + '.')
-- 
GitLab


From 6498c22816a9af179739631d86ef0fd19560d435 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 29 Nov 2018 09:19:44 -0800
Subject: [PATCH 1018/1554] Replace GraphRewriter with GraphView and helper
 functions in ModelPruner.

PiperOrigin-RevId: 223357322
---
 tensorflow/core/grappler/optimizers/BUILD     |  19 +-
 .../grappler/optimizers/graph_rewriter.cc     | 214 ------------------
 .../core/grappler/optimizers/graph_rewriter.h | 102 ---------
 .../grappler/optimizers/memory_optimizer.cc   |   1 -
 .../core/grappler/optimizers/model_pruner.cc  | 164 ++++++++++++--
 5 files changed, 152 insertions(+), 348 deletions(-)
 delete mode 100644 tensorflow/core/grappler/optimizers/graph_rewriter.cc
 delete mode 100644 tensorflow/core/grappler/optimizers/graph_rewriter.h

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 8e6629565a..9149ee275d 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -178,22 +178,6 @@ tf_cuda_cc_test(
     ],
 )
 
-cc_library(
-    name = "graph_rewriter",
-    srcs = ["graph_rewriter.cc"],
-    hdrs = [
-        "graph_rewriter.h",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
-    ],
-)
-
 cc_library(
     name = "graph_optimizer",
     hdrs = [
@@ -352,10 +336,10 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":graph_optimizer",
-        ":graph_rewriter",
         "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "@com_google_absl//absl/container:flat_hash_map",
@@ -419,7 +403,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":graph_optimizer",
-        ":graph_rewriter",
         ":static_schedule",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/grappler/optimizers/graph_rewriter.cc b/tensorflow/core/grappler/optimizers/graph_rewriter.cc
deleted file mode 100644
index b45ceb12a7..0000000000
--- a/tensorflow/core/grappler/optimizers/graph_rewriter.cc
+++ /dev/null
@@ -1,214 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/grappler/optimizers/graph_rewriter.h"
-#include <unordered_map>
-#include <unordered_set>
-#include "tensorflow/core/framework/function.pb.h"
-#include "tensorflow/core/framework/node_def.pb.h"
-#include "tensorflow/core/framework/node_def_util.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/op_def.pb.h"
-#include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/op_types.h"
-#include "tensorflow/core/grappler/utils.h"
-
-namespace tensorflow {
-namespace grappler {
-
-GraphRewriter::GraphRewriter(const GrapplerItem& item) {
-  OpRegistryInterface* op_registry = OpRegistry::Global();
-  for (auto& node : item.graph.node()) {
-    NodeInfo* info = new NodeInfo();
-    info->def = &node;
-
-    const OpRegistrationData* op_reg_data = nullptr;
-    Status s = op_registry->LookUp(node.op(), &op_reg_data);
-    // TODO(bsteiner): make this not a best-effort lookup and evaluation?
-    if (s.ok()) {
-      DataTypeVector inputs;
-      s = InOutTypesForNode(node, op_reg_data->op_def, &inputs, &info->outputs);
-      if (!s.ok()) {
-        info->outputs.clear();
-      }
-    }
-
-    nodes_[node.name()].reset(info);
-  }
-
-  std::unordered_set<string> function_names;
-  for (const auto& function : item.graph.library().function()) {
-    function_names.insert(function.signature().name());
-  }
-
-  for (auto& node : item.graph.node()) {
-    RecordConnectivity(node, function_names);
-  }
-}
-
-void GraphRewriter::ForwardInputs(
-    const NodeDef& original_node,
-    const std::unordered_set<const NodeDef*>& nodes_to_delete,
-    NodeDef* new_node) {
-  ForwardInputsInternal(original_node, nodes_to_delete, false, new_node);
-  if (!new_node->name().empty()) {
-    optimized_nodes_[new_node->name()] = new_node;
-  }
-  // Reorder inputs such that control inputs come after regular inputs.
-  int pos = 0;
-  for (int i = 0; i < new_node->input_size(); ++i) {
-    if (!IsControlInput(new_node->input(i))) {
-      new_node->mutable_input()->SwapElements(pos, i);
-      ++pos;
-    }
-  }
-  DedupControlInputs(new_node);
-}
-
-bool GraphRewriter::DrivesControlDependency(const NodeDef& node) const {
-  return control_dependency_drivers_.find(&node) !=
-         control_dependency_drivers_.end();
-}
-
-bool GraphRewriter::FeedsMerge(const NodeDef& node) const {
-  return merge_feeders_.find(&node) != merge_feeders_.end();
-}
-
-bool GraphRewriter::IsDrivenByControlDependency(const NodeDef& node) const {
-  for (const auto& input : node.input()) {
-    CHECK(!input.empty());
-    if (input[0] == '^') {
-      return true;
-    }
-  }
-  return false;
-}
-
-bool GraphRewriter::IsConnectedToFunction(const NodeDef& node) const {
-  return function_neighbors_.find(&node) != function_neighbors_.end();
-}
-
-bool GraphRewriter::IsDrivenByAnotherDevice(const NodeDef& node) const {
-  return cross_device_receivers_.find(&node) != cross_device_receivers_.end();
-}
-
-bool GraphRewriter::ReceivesRefValue(const NodeDef& node) const {
-  return ref_receivers_.find(&node) != ref_receivers_.end();
-}
-
-bool GraphRewriter::IsDrivenBySwitch(const NodeDef& node) const {
-  return switch_receivers_.find(&node) != switch_receivers_.end();
-}
-
-bool GraphRewriter::RemovalIncreasesEdgeCount(const NodeDef& node) const {
-  const int in_degree = node.input_size();
-  auto itr = nodes_.find(node.name());
-  if (itr == nodes_.end()) {
-    return true;
-  }
-  const int out_degree = itr->second->out_degree;
-  return in_degree * out_degree > in_degree + out_degree;
-}
-
-void GraphRewriter::RecordConnectivity(
-    const NodeDef& node, const std::unordered_set<string>& function_names) {
-  const bool is_function =
-      function_names.find(node.op()) != function_names.end();
-
-  bool ref_receiver = false;
-  bool switch_receiver = false;
-  for (const auto& input : node.input()) {
-    int position = 0;
-    string input_node_name = ParseNodeName(input, &position);
-    auto itr = nodes_.find(input_node_name);
-    if (itr == nodes_.end()) {
-      continue;
-    }
-
-    NodeInfo* fanin_info = itr->second.get();
-    const NodeDef* fanin = fanin_info->def;
-    if (IsMerge(node)) {
-      merge_feeders_.insert(fanin);
-    }
-    // Update out_degree of fanin.
-    ++fanin_info->out_degree;
-    if (position < 0) {
-      // This is a control edge
-      control_dependency_drivers_.insert(fanin);
-    } else {
-      // This is a regular edge
-      if (function_names.find(fanin->op()) != function_names.end()) {
-        function_neighbors_.insert(&node);
-      }
-      if (is_function) {
-        function_neighbors_.insert(fanin);
-      }
-      if (IsSwitch(*fanin)) {
-        switch_receiver = true;
-      }
-      if (position < fanin_info->outputs.size() &&
-          IsRefType(fanin_info->outputs[position])) {
-        ref_receiver = true;
-      }
-    }
-    if (fanin->device() != node.device()) {
-      cross_device_receivers_.insert(&node);
-    }
-  }
-
-  if (ref_receiver) {
-    ref_receivers_.insert(&node);
-  }
-  if (switch_receiver) {
-    switch_receivers_.insert(&node);
-  }
-}
-
-void GraphRewriter::ForwardInputsInternal(
-    const NodeDef& node,
-    const std::unordered_set<const NodeDef*>& nodes_to_delete,
-    bool add_as_control, NodeDef* new_node) {
-  // To speed things up, use the optimized version of the node if
-  // available.
-  auto itr = optimized_nodes_.find(node.name());
-  if (itr != optimized_nodes_.end()) {
-    for (const string& input : itr->second->input()) {
-      *new_node->add_input() =
-          add_as_control ? AsControlDependency(NodeName(input)) : input;
-    }
-    return;
-  }
-  for (const auto& input : node.input()) {
-    const string input_node_name = NodeName(input);
-    auto itr = nodes_.find(input_node_name);
-    if (itr == nodes_.end()) {
-      // Invalid input, preserve it as is.
-      *new_node->add_input() =
-          add_as_control ? AsControlDependency(NodeName(input)) : input;
-      continue;
-    }
-    const NodeDef* input_node = itr->second->def;
-    if (nodes_to_delete.find(input_node) != nodes_to_delete.end()) {
-      ForwardInputsInternal(*input_node, nodes_to_delete,
-                            add_as_control || IsControlInput(input), new_node);
-    } else {
-      *new_node->add_input() =
-          add_as_control ? AsControlDependency(NodeName(input)) : input;
-    }
-  }
-}
-
-}  // end namespace grappler
-}  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/graph_rewriter.h b/tensorflow/core/grappler/optimizers/graph_rewriter.h
deleted file mode 100644
index 4a5a150dc9..0000000000
--- a/tensorflow/core/grappler/optimizers/graph_rewriter.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_GRAPH_REWRITER_H_
-#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_GRAPH_REWRITER_H_
-
-#include <unordered_map>
-#include <unordered_set>
-#include "tensorflow/core/grappler/grappler_item.h"
-
-namespace tensorflow {
-namespace grappler {
-
-// Tools and utilities to simplify common graph rewrites.
-class GraphRewriter {
- public:
-  GraphRewriter(const GrapplerItem& item);
-
-  // Forward the inputs of original_node as needed to skip over the nodes that
-  // are to be deleted. In other words, if I is an input of 'original_node', and
-  // I doesn't belong to one of the nodes in 'nodes_to_delete', I will be an
-  // input to 'new_node'. On the other hand, if I belong to a node that will be
-  // deleted, I will be replaced with the inputs J of the deleted node (unless J
-  // belong to nodes that will be deleted, in which case we'll look for
-  // preserved inputs further down the graph).
-  void ForwardInputs(const NodeDef& original_node,
-                     const std::unordered_set<const NodeDef*>& nodes_to_delete,
-                     NodeDef* new_node);
-
-  // Returns true if at least one of the edges in the direct fanout of 'node' is
-  // a control dependency edge.
-  bool DrivesControlDependency(const NodeDef& node) const;
-
-  // Returns true if at least one of the incident edges is a control dependency
-  // edge.
-  bool IsDrivenByControlDependency(const NodeDef& node) const;
-
-  // Returns true if at least one of the nodes in the direct fanin or the direct
-  // fanout (excluding control dependencies) of 'node' is a function.
-  bool IsConnectedToFunction(const NodeDef& node) const;
-
-  // Returns true if the node is driven by at least one node placed on another
-  // device.
-  bool IsDrivenByAnotherDevice(const NodeDef& node) const;
-
-  // Returns true if the node has input from a stateful op.
-  bool ReceivesRefValue(const NodeDef& node) const;
-
-  // Returns true if the node is driven by a Switch node.
-  bool IsDrivenBySwitch(const NodeDef& node) const;
-
-  // Returns true if the node feeds a Merge node.
-  bool FeedsMerge(const NodeDef& node) const;
-
-  // Returns true if removal of this degree would increase edge count, i.e. if
-  // in-degree * out-degree > in-degree + out-degree or if the condition could
-  // not be verified.
-  bool RemovalIncreasesEdgeCount(const NodeDef& node) const;
-
- private:
-  void RecordConnectivity(const NodeDef& node,
-                          const std::unordered_set<string>& function_names);
-  void ForwardInputsInternal(
-      const NodeDef& original_node,
-      const std::unordered_set<const NodeDef*>& nodes_to_delete,
-      bool add_as_control, NodeDef* new_node);
-
-  struct NodeInfo {
-    int out_degree = 0;
-    const NodeDef* def;
-
-    // These are filled in when the NodeInfo is built, but not that they
-    // may be empty - if the op could not be loaded from the registry.
-    DataTypeVector outputs;
-  };
-
-  std::unordered_map<string, std::unique_ptr<NodeInfo>> nodes_;
-  std::unordered_map<string, const NodeDef*> optimized_nodes_;
-  std::unordered_set<const NodeDef*> control_dependency_drivers_;
-  std::unordered_set<const NodeDef*> function_neighbors_;
-  std::unordered_set<const NodeDef*> cross_device_receivers_;
-  std::unordered_set<const NodeDef*> ref_receivers_;
-  std::unordered_set<const NodeDef*> switch_receivers_;
-  std::unordered_set<const NodeDef*> merge_feeders_;
-};
-
-}  // end namespace grappler
-}  // end namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_GRAPH_REWRITER_H_
diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer.cc b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
index e0a913565f..453db5d91e 100644
--- a/tensorflow/core/grappler/optimizers/memory_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
-#include "tensorflow/core/grappler/optimizers/graph_rewriter.h"
 #include "tensorflow/core/grappler/optimizers/static_schedule.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
diff --git a/tensorflow/core/grappler/optimizers/model_pruner.cc b/tensorflow/core/grappler/optimizers/model_pruner.cc
index 1be87a9d0d..c548c570e0 100644
--- a/tensorflow/core/grappler/optimizers/model_pruner.cc
+++ b/tensorflow/core/grappler/optimizers/model_pruner.cc
@@ -23,30 +23,164 @@ limitations under the License.
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
-#include "tensorflow/core/grappler/optimizers/graph_rewriter.h"
 #include "tensorflow/core/grappler/utils.h"
 
 namespace tensorflow {
 namespace grappler {
 
-bool IsTrivialOp(const NodeDef& node, const GraphRewriter& rewriter) {
+bool IsTrivialIdentity(const NodeDef& node,
+                       const MutableGraphView& graph_view) {
+  for (const auto input :
+       graph_view.GetFanins(node, /*include_controlling_nodes=*/true)) {
+    if (input.port_id == Graph::kControlSlot) {
+      // Node is driven by control dependency.
+      return false;
+    } else if (IsSwitch(*input.node)) {  // Node is driven by switch.
+      return false;
+    }
+  }
+  for (const auto output :
+       graph_view.GetFanouts(node, /*include_controlled_nodes=*/true)) {
+    if (output.port_id == Graph::kControlSlot) {
+      // Node drives control dependency.
+      return false;
+    } else if (IsMerge(*output.node)) {  // Node feeds merge.
+      return false;
+    }
+  }
+  return true;
+}
+
+bool IsTrivialOp(const NodeDef& node, const MutableGraphView& graph_view) {
   // Remove the stop gradient nodes since they serve no purpose once the graph
   // is built. Also remove Identity ops.
   if (IsStopGradient(node)) {
     return true;
   }
   if (IsIdentity(node) || IsIdentityNSingleInput(node)) {
-    return !(rewriter.FeedsMerge(node) || rewriter.IsDrivenBySwitch(node) ||
-             rewriter.IsDrivenByControlDependency(node) ||
-             rewriter.DrivesControlDependency(node));
+    return IsTrivialIdentity(node, graph_view);
   }
 
   return IsAddN(node) && NumNonControlInputs(node) <= 1;
 }
 
+bool RemovalIncreasesEdgeCount(const NodeDef& node,
+                               const MutableGraphView& graph_view) {
+  int in_degree =
+      graph_view.NumFanins(node, /*include_controlling_nodes=*/true);
+  int out_degree =
+      graph_view.NumFanouts(node, /*include_controlling_nodes=*/true);
+  return in_degree * out_degree > in_degree + out_degree;
+}
+
+bool IsOutputPortRefValue(const NodeDef& node, int port_id,
+                          const OpRegistryInterface& op_registry) {
+  const OpRegistrationData* op_reg_data = nullptr;
+  Status s = op_registry.LookUp(node.op(), &op_reg_data);
+  if (s.ok()) {
+    DataType output_type;
+    s = OutputTypeForNode(node, op_reg_data->op_def, port_id, &output_type);
+    if (s.ok() && IsRefType(output_type)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool CanRemoveNode(const NodeDef& node, const MutableGraphView& graph_view,
+                   const absl::flat_hash_set<string>& function_names,
+                   const OpRegistryInterface& op_registry) {
+  if (RemovalIncreasesEdgeCount(node, graph_view)) {
+    return false;
+  }
+  for (const auto input :
+       graph_view.GetFanins(node, /*include_controlling_nodes=*/true)) {
+    if (node.device() != input.node->device()) {
+      // Node is driven by a different device.
+      return false;
+    } else if (input.port_id == Graph::kControlSlot) {
+      // Node is driven by control dependency.
+      continue;
+    } else if (function_names.find(input.node->op()) != function_names.end()) {
+      // Node input is a function call.
+      return false;
+    } else if (IsOutputPortRefValue(*input.node, input.port_id, op_registry)) {
+      return false;
+    }
+  }
+  for (const auto output :
+       graph_view.GetFanouts(node, /*include_controlled_nodes=*/false)) {
+    if (function_names.find(output.node->op()) != function_names.end()) {
+      // Node output is a function call.
+      return false;
+    }
+  }
+  return true;
+}
+
+void ForwardInputsInternal(
+    const NodeDef& node,
+    const absl::flat_hash_set<const NodeDef*>& nodes_to_delete,
+    bool add_as_control, NodeDef* new_node,
+    const absl::flat_hash_map<string, const NodeDef*>& optimized_nodes,
+    const MutableGraphView& graph_view) {
+  // To speed things up, use the optimized version of the node if
+  // available.
+  auto itr = optimized_nodes.find(node.name());
+  if (itr != optimized_nodes.end()) {
+    for (const string& input : itr->second->input()) {
+      *new_node->add_input() =
+          add_as_control ? AsControlDependency(NodeName(input)) : input;
+    }
+    return;
+  }
+  for (const auto& input : node.input()) {
+    const NodeDef* input_node = graph_view.GetNode(NodeName(input));
+    if (input_node == nullptr) {
+      // Invalid input, preserve it as is.
+      *new_node->add_input() =
+          add_as_control ? AsControlDependency(NodeName(input)) : input;
+      continue;
+    }
+    if (nodes_to_delete.find(input_node) != nodes_to_delete.end()) {
+      ForwardInputsInternal(*input_node, nodes_to_delete,
+                            add_as_control || IsControlInput(input), new_node,
+                            optimized_nodes, graph_view);
+    } else {
+      *new_node->add_input() =
+          add_as_control ? AsControlDependency(NodeName(input)) : input;
+    }
+  }
+}
+
+void ForwardInputs(const NodeDef& original_node,
+                   const absl::flat_hash_set<const NodeDef*>& nodes_to_delete,
+                   NodeDef* new_node,
+                   absl::flat_hash_map<string, const NodeDef*>* optimized_nodes,
+                   const MutableGraphView& graph_view) {
+  // Forwards inputs of nodes to be deleted to their respective outputs.
+  ForwardInputsInternal(original_node, nodes_to_delete,
+                        /*add_as_control=*/false, new_node, *optimized_nodes,
+                        graph_view);
+  if (!new_node->name().empty()) {
+    (*optimized_nodes)[new_node->name()] = new_node;
+  }
+  // Reorder inputs such that control inputs come after regular inputs.
+  int pos = 0;
+  for (int i = 0; i < new_node->input_size(); ++i) {
+    if (!IsControlInput(new_node->input(i))) {
+      new_node->mutable_input()->SwapElements(pos, i);
+      ++pos;
+    }
+  }
+  DedupControlInputs(new_node);
+}
+
 absl::flat_hash_map<string, absl::flat_hash_set<int>> IdentityNTerminalPorts(
     const NodeMap& node_map, const std::vector<string>& terminal_nodes,
     int graph_size) {
@@ -313,12 +447,17 @@ Status ModelPruner::Optimize(Cluster* cluster, const GrapplerItem& item,
     runnable_item = item;
   }
 
-  GraphRewriter rewriter(runnable_item);
+  MutableGraphView graph_view(&runnable_item.graph);
+  absl::flat_hash_set<string> function_names;
+  for (const auto& function : item.graph.library().function()) {
+    function_names.insert(function.signature().name());
+  }
+  OpRegistryInterface* op_registry = OpRegistry::Global();
 
   // Check if we can further prune the graph, by removing the trivial ops.
-  std::unordered_set<const NodeDef*> nodes_to_delete;
+  absl::flat_hash_set<const NodeDef*> nodes_to_delete;
   for (auto& node : runnable_item.graph.node()) {
-    if (!IsTrivialOp(node, rewriter)) {
+    if (!IsTrivialOp(node, graph_view)) {
       continue;
     }
 
@@ -341,10 +480,7 @@ Status ModelPruner::Optimize(Cluster* cluster, const GrapplerItem& item,
     //   converting references to non-references. It is important to preserve
     //   these non-references since the partitioner will avoid sending
     //   non-references across partitions more than once.
-    if (!rewriter.RemovalIncreasesEdgeCount(node) &&
-        !rewriter.IsConnectedToFunction(node) &&
-        !rewriter.IsDrivenByAnotherDevice(node) &&
-        !rewriter.ReceivesRefValue(node)) {
+    if (CanRemoveNode(node, graph_view, function_names, *op_registry)) {
       nodes_to_delete.insert(&node);
     }
   }
@@ -360,13 +496,15 @@ Status ModelPruner::Optimize(Cluster* cluster, const GrapplerItem& item,
 
   const bool fetches_are_known = !item.fetch.empty();
   pruned_graph->mutable_node()->Reserve(runnable_item.graph.node_size());
+  absl::flat_hash_map<string, const NodeDef*> optimized_nodes;
   for (auto& node : runnable_item.graph.node()) {
     if (!fetches_are_known ||
         nodes_to_delete.find(&node) == nodes_to_delete.end()) {
       NodeDef* new_node = pruned_graph->add_node();
       *new_node = node;
       new_node->clear_input();
-      rewriter.ForwardInputs(node, nodes_to_delete, new_node);
+      ForwardInputs(node, nodes_to_delete, new_node, &optimized_nodes,
+                    graph_view);
     }
   }
   VLOG(1) << "Pruned " << nodes_to_delete.size()
-- 
GitLab


From 6afcfdfb75857bbce7d680f272bc54231c351e97 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Nov 2018 09:36:29 -0800
Subject: [PATCH 1019/1554] Splitting up CapturedFunction into a
 CapturedFunction which handles the static state of functions like
 captured_inputs etc. and InstantiatedCapturedFunction that holds the handle
 and other state. CapturedFunctions are meant to be owned by the Dataset
 objects and InstantiatedCapturedFunctions by iterators. This is to facilitate
 splitting the Dataset and Iterator objects more naturally and create Dataset
 objects as we go.

This CL also creates a FunctionHandleCache that owns all the function handles
corresponding to an Iterator. Having this enables us to implement different
semantics for clearing function state when an iterator is reset or when an
iterator is re-started in a RepeatDataset.

PiperOrigin-RevId: 223359901
---
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/framework/dataset.cc          |   3 +
 tensorflow/core/framework/dataset.h           |  10 +
 .../core/framework/function_handle_cache.cc   |  66 ++++++
 .../core/framework/function_handle_cache.h    |  53 +++++
 .../core/kernels/data/captured_function.cc    | 214 ++++++++----------
 .../core/kernels/data/captured_function.h     | 127 ++++++-----
 tensorflow/core/kernels/data/dataset_utils.cc |   8 +-
 tensorflow/core/kernels/data/dataset_utils.h  |   4 +-
 .../numa_map_and_batch_dataset_op.cc          |   8 +-
 .../core/kernels/data/filter_dataset_op.cc    |  20 +-
 .../core/kernels/data/flat_map_dataset_op.cc  |   7 +-
 .../core/kernels/data/generator_dataset_op.cc |  23 +-
 .../data/group_by_reducer_dataset_op.cc       |  31 +--
 .../data/group_by_window_dataset_op.cc        |  25 +-
 .../kernels/data/interleave_dataset_op.cc     |   8 +-
 tensorflow/core/kernels/data/iterator_ops.cc  | 175 ++++++++------
 .../kernels/data/map_and_batch_dataset_op.cc  |  31 +--
 .../core/kernels/data/map_dataset_op.cc       |  25 +-
 .../kernels/data/multi_device_iterator_ops.cc |  50 ++--
 .../core/kernels/data/optimize_dataset_op.cc  |   7 +
 .../data/parallel_interleave_dataset_op.cc    |  17 +-
 .../kernels/data/parallel_map_dataset_op.cc   | 131 ++++++-----
 .../kernels/data/parallel_map_iterator.cc     |  30 +--
 .../core/kernels/data/parallel_map_iterator.h |  43 ++--
 .../kernels/data/parse_example_dataset_op.cc  | 199 ++++++++--------
 .../core/kernels/data/scan_dataset_op.cc      |   8 +-
 tensorflow/core/kernels/data/writer_ops.cc    |  17 +-
 28 files changed, 806 insertions(+), 535 deletions(-)
 create mode 100644 tensorflow/core/framework/function_handle_cache.cc
 create mode 100644 tensorflow/core/framework/function_handle_cache.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7b46e8c9b4..97628a2561 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -869,6 +869,7 @@ tf_cuda_library(
         "framework/dataset_stateful_op_whitelist.h",
         "framework/device_base.h",
         "framework/function.h",
+        "framework/function_handle_cache.h",
         "framework/graph_def_util.h",
         "framework/graph_to_functiondef.h",
         "framework/kernel_def_builder.h",
diff --git a/tensorflow/core/framework/dataset.cc b/tensorflow/core/framework/dataset.cc
index fc6b5dde0c..6af14150b7 100644
--- a/tensorflow/core/framework/dataset.cc
+++ b/tensorflow/core/framework/dataset.cc
@@ -13,10 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/framework/dataset.h"
+#include <unordered_map>
 
 #include "tensorflow/core/framework/device_base.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/platform/mutex.h"
 
 namespace tensorflow {
 namespace data {
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index 9b11449b30..b2689bbdb7 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <deque>
 #include <memory>
+#include <unordered_map>
 
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -259,6 +260,7 @@ class GraphDefBuilderWrapper {
 };
 
 class StatsAggregator;
+class FunctionHandleCache;
 
 // A cut-down version of `OpKernelContext` for running computations in
 // iterators. Note that we cannot simply use `OpKernelContext` here because we
@@ -279,6 +281,7 @@ class IteratorContext {
           env(ctx->env()),
           function_library(ctx->function_library()),
           lib(ctx->lib()),
+          function_handle_cache(ctx->function_handle_cache()),
           model(ctx->model()),
           runner(*(ctx->runner())),
           runner_threadpool_size(ctx->runner_threadpool_size()),
@@ -315,6 +318,9 @@ class IteratorContext {
     // The FunctionLibraryRuntime object to be used to make function calls.
     FunctionLibraryRuntime* lib = nullptr;
 
+    // A FunctionHandleCache that owns all the function handles. Not owned.
+    FunctionHandleCache* function_handle_cache = nullptr;
+
     // If non-null, identifies the object used for performance modeling.
     std::shared_ptr<model::Model> model = nullptr;
 
@@ -350,6 +356,10 @@ class IteratorContext {
 
   FunctionLibraryRuntime* lib() { return params_.lib; }
 
+  FunctionHandleCache* function_handle_cache() {
+    return params_.function_handle_cache;
+  }
+
   const std::shared_ptr<model::Model>& model() { return params_.model; }
 
   std::function<void(std::function<void()>)>* runner() {
diff --git a/tensorflow/core/framework/function_handle_cache.cc b/tensorflow/core/framework/function_handle_cache.cc
new file mode 100644
index 0000000000..2b93b6b2f8
--- /dev/null
+++ b/tensorflow/core/framework/function_handle_cache.cc
@@ -0,0 +1,66 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/function_handle_cache.h"
+
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
+
+namespace tensorflow {
+namespace data {
+
+FunctionHandleCache::FunctionHandleCache(FunctionLibraryRuntime* lib)
+    : lib_(lib), state_handle_(strings::Printf("%lld", random::New64())) {}
+
+FunctionHandleCache::~FunctionHandleCache() {
+  Status s = Clear();
+  if (!s.ok()) {
+    LOG(ERROR) << "Failed to clear function handle cache: " << s.ToString();
+  }
+}
+
+Status FunctionHandleCache::Instantiate(
+    const string& function_name, AttrSlice attrs,
+    FunctionLibraryRuntime::InstantiateOptions options,
+    FunctionLibraryRuntime::Handle* handle) {
+  string key = Canonicalize(function_name, attrs, options);
+  FunctionLibraryRuntime::Handle h;
+  {
+    tf_shared_lock l(mu_);
+    h = gtl::FindWithDefault(handles_, key, kInvalidHandle);
+  }
+  if (h == kInvalidHandle) {
+    options.state_handle = state_handle_;
+    TF_RETURN_IF_ERROR(
+        lib_->Instantiate(function_name, attrs, options, handle));
+    mutex_lock l(mu_);
+    handles_[key] = *handle;
+  } else {
+    *handle = h;
+  }
+  return Status::OK();
+}
+
+Status FunctionHandleCache::Clear() {
+  mutex_lock l(mu_);
+  for (auto entry : handles_) {
+    TF_RETURN_IF_ERROR(lib_->ReleaseHandle(entry.second));
+  }
+  handles_.clear();
+  return Status::OK();
+}
+
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/framework/function_handle_cache.h b/tensorflow/core/framework/function_handle_cache.h
new file mode 100644
index 0000000000..2800a598e0
--- /dev/null
+++ b/tensorflow/core/framework/function_handle_cache.h
@@ -0,0 +1,53 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_FRAMEWORK_FUNCTION_HANDLE_CACHE_H_
+#define TENSORFLOW_CORE_FRAMEWORK_FUNCTION_HANDLE_CACHE_H_
+
+#include <string>
+
+#include "tensorflow/core/framework/function.h"
+
+namespace tensorflow {
+namespace data {
+
+class FunctionHandleCache {
+ public:
+  explicit FunctionHandleCache(FunctionLibraryRuntime* lib);
+
+  ~FunctionHandleCache();
+
+  // Looks up the function to be instantiated in the cache first. If present,
+  // returns handle from there. Otherwise, instantiates a new function
+  // and stores handle in the cache.
+  Status Instantiate(const string& function_name, AttrSlice attrs,
+                     FunctionLibraryRuntime::InstantiateOptions options,
+                     FunctionLibraryRuntime::Handle* handle);
+
+  // Releases all the handles in the cache, clearing out the state for all
+  // functions involved.
+  Status Clear();
+
+ private:
+  mutex mu_;
+  FunctionLibraryRuntime* lib_ = nullptr;  // not owned
+  const string state_handle_;
+  std::unordered_map<string, FunctionLibraryRuntime::Handle> handles_
+      GUARDED_BY(mu_);
+};
+
+}  // namespace data
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_FRAMEWORK_FUNCTION_HANDLE_CACHE_H_
diff --git a/tensorflow/core/kernels/data/captured_function.cc b/tensorflow/core/kernels/data/captured_function.cc
index 64834e507f..973b6b0604 100644
--- a/tensorflow/core/kernels/data/captured_function.cc
+++ b/tensorflow/core/kernels/data/captured_function.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/cancellation.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/lib/random/random.h"
@@ -118,10 +119,34 @@ Status CapturedFunction::Create(
   return Status::OK();
 }
 
-CapturedFunction::~CapturedFunction() {
-  if (lib_ != nullptr && f_handle_ != kInvalidHandle) {
-    lib_->ReleaseHandle(f_handle_).IgnoreError();
+Status CapturedFunction::Instantiate(
+    IteratorContext* ctx, std::unique_ptr<InstantiatedCapturedFunction>*
+                              instantiated_captured_function) {
+  // The context's runtime will be used for all subsequent calls.
+  FunctionLibraryRuntime* lib = ctx->lib();
+  FunctionLibraryRuntime::InstantiateOptions inst_opts;
+  inst_opts.overlay_lib = ctx->function_library().get();
+  inst_opts.create_kernels_eagerly = true;
+  if (!use_inter_op_parallelism_) {
+    inst_opts.executor_type = "SINGLE_THREADED_EXECUTOR";
   }
+
+  FunctionLibraryRuntime::Handle f_handle;
+  TF_RETURN_IF_ERROR(ctx->function_handle_cache()->Instantiate(
+      func_.name(), AttrSlice(&func_.attr()), inst_opts, &f_handle));
+  const FunctionBody* fbody = lib->GetFunctionBody(f_handle);
+  if (fbody == nullptr) {
+    return errors::Internal("Failed to instantiate function body.");
+  }
+
+  DataTypeVector ret_types;
+  for (const auto& ret_type : fbody->ret_types) {
+    ret_types.push_back(ret_type);
+  }
+
+  instantiated_captured_function->reset(new InstantiatedCapturedFunction(
+      lib, f_handle, std::move(ret_types), *ctx->runner(), this));
+  return Status::OK();
 }
 
 namespace {
@@ -244,35 +269,35 @@ class BorrowedArgsCallFrame : public CallFrameBase {
 
 }  // namespace
 
-Status CapturedFunction::GetHandle(IteratorContext* ctx,
-                                   FunctionLibraryRuntime::Handle* out_handle) {
-  tf_shared_lock l(mu_);
-  if (lib_ == nullptr) {
-    return errors::Internal("Captured function \"", func_.name(),
-                            "\" was called before it was instantiated.");
-  }
-  if (ctx->lib() != lib_) {
-    return errors::Internal("Captured function \"", func_.name(),
-                            "\" was called with a different "
-                            "FunctionLibraryRuntime*, which is not permitted.");
-  }
-  *out_handle = f_handle_;
-  return Status::OK();
-}
-
-Status CapturedFunction::Run(IteratorContext* ctx, std::vector<Tensor>&& args,
-                             std::vector<Tensor>* rets) {
-  FunctionLibraryRuntime::Handle handle;
-  TF_RETURN_IF_ERROR(GetHandle(ctx, &handle));
-
+InstantiatedCapturedFunction::InstantiatedCapturedFunction(
+    FunctionLibraryRuntime* lib, FunctionLibraryRuntime::Handle f_handle,
+    DataTypeVector ret_types, std::function<void(std::function<void()>)> runner,
+    CapturedFunction* captured_func)
+    : lib_(lib),
+      f_handle_(f_handle),
+      ret_types_(std::move(ret_types)),
+      captured_runner_(std::move(runner)),
+      captured_func_(captured_func) {}
+
+// NOTE: We don't release f_handle_ here and instead delegate the function
+// handle releasing to the FunctionHandleCache. This is because in some cases
+// (RepeatDatasetOp in particular), we want to keep the function state (e.g.
+// random number generator) even after the Iterator is reset after going through
+// one epoch.
+InstantiatedCapturedFunction::~InstantiatedCapturedFunction() {}
+
+Status InstantiatedCapturedFunction::Run(IteratorContext* ctx,
+                                         std::vector<Tensor>&& args,
+                                         std::vector<Tensor>* rets) const {
   FunctionLibraryRuntime::Options f_opts;
-  f_opts.step_id = CapturedFunction::generate_step_id();
-  ScopedStepContainer step_container(f_opts.step_id, [ctx](const string& name) {
-    ctx->lib()->device()->resource_manager()->Cleanup(name).IgnoreError();
-  });
+  f_opts.step_id = InstantiatedCapturedFunction::generate_step_id();
+  ScopedStepContainer step_container(
+      f_opts.step_id, [this](const string& name) {
+        lib_->device()->resource_manager()->Cleanup(name).IgnoreError();
+      });
   f_opts.step_container = &step_container;
   f_opts.runner = ctx->runner();
-  if (ctx->lib()->device()->device_type() != DEVICE_CPU) {
+  if (lib_->device()->device_type() != DEVICE_CPU) {
     f_opts.create_rendezvous = true;
   }
   // TODO(mrry): Add cancellation manager support to IteratorContext
@@ -284,10 +309,11 @@ Status CapturedFunction::Run(IteratorContext* ctx, std::vector<Tensor>&& args,
   CancellationManager c_mgr;
   f_opts.cancellation_manager = &c_mgr;
 
-  OwnedArgsCallFrame frame(std::move(args), &captured_inputs_, ret_types_);
+  OwnedArgsCallFrame frame(std::move(args), &captured_func_->captured_inputs(),
+                           ret_types_);
   Notification n;
   Status s;
-  ctx->lib()->Run(f_opts, handle, &frame, [&n, &s](Status func_status) {
+  lib_->Run(f_opts, f_handle_, &frame, [&n, &s](Status func_status) {
     s.Update(func_status);
     n.Notify();
   });
@@ -296,20 +322,18 @@ Status CapturedFunction::Run(IteratorContext* ctx, std::vector<Tensor>&& args,
   return frame.ConsumeRetvals(rets);
 }
 
-Status CapturedFunction::RunWithBorrowedArgs(IteratorContext* ctx,
-                                             const std::vector<Tensor>& args,
-                                             std::vector<Tensor>* rets) {
-  FunctionLibraryRuntime::Handle handle;
-  TF_RETURN_IF_ERROR(GetHandle(ctx, &handle));
-
+Status InstantiatedCapturedFunction::RunWithBorrowedArgs(
+    IteratorContext* ctx, const std::vector<Tensor>& args,
+    std::vector<Tensor>* rets) const {
   FunctionLibraryRuntime::Options f_opts;
-  f_opts.step_id = CapturedFunction::generate_step_id();
-  ScopedStepContainer step_container(f_opts.step_id, [ctx](const string& name) {
-    ctx->lib()->device()->resource_manager()->Cleanup(name).IgnoreError();
-  });
+  f_opts.step_id = InstantiatedCapturedFunction::generate_step_id();
+  ScopedStepContainer step_container(
+      f_opts.step_id, [this](const string& name) {
+        lib_->device()->resource_manager()->Cleanup(name).IgnoreError();
+      });
   f_opts.step_container = &step_container;
   f_opts.runner = ctx->runner();
-  if (ctx->lib()->device()->device_type() != DEVICE_CPU) {
+  if (lib_->device()->device_type() != DEVICE_CPU) {
     f_opts.create_rendezvous = true;
   }
   // TODO(mrry): Add cancellation manager support to IteratorContext
@@ -321,11 +345,12 @@ Status CapturedFunction::RunWithBorrowedArgs(IteratorContext* ctx,
   CancellationManager c_mgr;
   f_opts.cancellation_manager = &c_mgr;
 
-  BorrowedArgsCallFrame frame(args, &captured_inputs_, ret_types_);
+  BorrowedArgsCallFrame frame(args, &captured_func_->captured_inputs(),
+                              ret_types_);
   Notification n;
   Status s;
 
-  ctx->lib()->Run(f_opts, handle, &frame, [&n, &s](Status func_status) {
+  lib_->Run(f_opts, f_handle_, &frame, [&n, &s](Status func_status) {
     s.Update(func_status);
     n.Notify();
   });
@@ -334,65 +359,17 @@ Status CapturedFunction::RunWithBorrowedArgs(IteratorContext* ctx,
   return frame.ConsumeRetvals(rets);
 }
 
-Status CapturedFunction::Instantiate(IteratorContext* ctx) {
-  mutex_lock l(mu_);
-  if (lib_ == nullptr) {
-    // The context's runtime will be used for all subsequent calls.
-    lib_ = ctx->lib();
-    DCHECK(f_handle_ == kInvalidHandle);
-    FunctionLibraryRuntime::InstantiateOptions inst_opts;
-    inst_opts.overlay_lib = ctx->function_library().get();
-    inst_opts.state_handle = std::to_string(random::New64());
-    inst_opts.create_kernels_eagerly = true;
-    if (!use_inter_op_parallelism_) {
-      inst_opts.executor_type = "SINGLE_THREADED_EXECUTOR";
-    }
-    Status s = (lib_->Instantiate(func_.name(), AttrSlice(&func_.attr()),
-                                  inst_opts, &f_handle_));
-    TF_RETURN_IF_ERROR(s);
-    const FunctionBody* fbody = lib_->GetFunctionBody(f_handle_);
-    if (fbody == nullptr) {
-      return errors::Internal("Failed to instantiate function body.");
-    }
-    ret_types_ = fbody->ret_types;
-  } else {
-    if (ctx->lib() != lib_) {
-      return errors::Internal(
-          "Captured function was called with a different "
-          "FunctionLibraryRuntime*, which is not permitted.");
-    }
-  }
-  if (captured_runner_ == nullptr) {
-    captured_runner_ = *ctx->runner();
-  }
-  return Status::OK();
-}
-
-Status CapturedFunction::RunInstantiated(const std::vector<Tensor>& args,
-                                         std::vector<Tensor>* rets) {
-  FunctionLibraryRuntime* lib;
-  FunctionLibraryRuntime::Handle handle;
-  std::function<void(std::function<void()>)>* runner;
-  {
-    tf_shared_lock l(mu_);
-    if (lib_ == nullptr) {
-      return errors::FailedPrecondition(
-          "`CapturedFunction::Instantiate()` must be called before a call to "
-          "`CapturedFunction::RunInstantiated()`.");
-    }
-    lib = lib_;
-    handle = f_handle_;
-    runner = &captured_runner_;
-  }
-
+Status InstantiatedCapturedFunction::RunInstantiated(
+    const std::vector<Tensor>& args, std::vector<Tensor>* rets) {
   FunctionLibraryRuntime::Options f_opts;
-  f_opts.step_id = CapturedFunction::generate_step_id();
-  ScopedStepContainer step_container(f_opts.step_id, [lib](const string& name) {
-    lib->device()->resource_manager()->Cleanup(name).IgnoreError();
-  });
+  f_opts.step_id = InstantiatedCapturedFunction::generate_step_id();
+  ScopedStepContainer step_container(
+      f_opts.step_id, [this](const string& name) {
+        lib_->device()->resource_manager()->Cleanup(name).IgnoreError();
+      });
   f_opts.step_container = &step_container;
-  f_opts.runner = runner;
-  if (lib->device()->device_type() != DEVICE_CPU) {
+  f_opts.runner = &captured_runner_;
+  if (lib_->device()->device_type() != DEVICE_CPU) {
     f_opts.create_rendezvous = true;
   }
   // TODO(mrry): Add cancellation manager support to IteratorContext
@@ -404,11 +381,12 @@ Status CapturedFunction::RunInstantiated(const std::vector<Tensor>& args,
   CancellationManager c_mgr;
   f_opts.cancellation_manager = &c_mgr;
 
-  BorrowedArgsCallFrame frame(args, &captured_inputs_, ret_types_);
+  BorrowedArgsCallFrame frame(args, &captured_func_->captured_inputs(),
+                              ret_types_);
   Notification n;
   Status s;
 
-  lib->Run(f_opts, handle, &frame, [&n, &s](Status func_status) {
+  lib_->Run(f_opts, f_handle_, &frame, [&n, &s](Status func_status) {
     s.Update(func_status);
     n.Notify();
   });
@@ -417,33 +395,25 @@ Status CapturedFunction::RunInstantiated(const std::vector<Tensor>& args,
   return frame.ConsumeRetvals(rets);
 }
 
-void CapturedFunction::RunAsync(IteratorContext* ctx,
-                                std::vector<Tensor>&& args,
-                                std::vector<Tensor>* rets,
-                                FunctionLibraryRuntime::DoneCallback done,
-                                const string& prefix) {
+void InstantiatedCapturedFunction::RunAsync(
+    IteratorContext* ctx, std::vector<Tensor>&& args, std::vector<Tensor>* rets,
+    FunctionLibraryRuntime::DoneCallback done, const string& prefix) const {
   // NOTE(mrry): This method does not transfer ownership of `ctx`, and it may
   // be deleted before `done` is called. Take care not to capture `ctx` in any
   // code that may execute asynchronously in this function.
-  FunctionLibraryRuntime::Handle handle;
-  Status s = GetHandle(ctx, &handle);
-  if (!s.ok()) {
-    done(s);
-    return;
-  }
-  OwnedArgsCallFrame* frame =
-      new OwnedArgsCallFrame(std::move(args), &captured_inputs_, ret_types_);
+  OwnedArgsCallFrame* frame = new OwnedArgsCallFrame(
+      std::move(args), &captured_func_->captured_inputs(), ret_types_);
 
   FunctionLibraryRuntime::Options f_opts;
-  f_opts.step_id = CapturedFunction::generate_step_id();
-  ResourceMgr* resource_mgr = ctx->lib()->device()->resource_manager();
+  f_opts.step_id = InstantiatedCapturedFunction::generate_step_id();
+  ResourceMgr* resource_mgr = lib_->device()->resource_manager();
   ScopedStepContainer* step_container = new ScopedStepContainer(
       f_opts.step_id, [resource_mgr](const string& name) {
         resource_mgr->Cleanup(name).IgnoreError();
       });
   f_opts.step_container = step_container;
   f_opts.runner = ctx->runner();
-  if (ctx->lib()->device()->device_type() != DEVICE_CPU) {
+  if (lib_->device()->device_type() != DEVICE_CPU) {
     f_opts.create_rendezvous = true;
   }
   // TODO(mrry): Add cancellation manager support to IteratorContext
@@ -480,7 +450,7 @@ void CapturedFunction::RunAsync(IteratorContext* ctx,
           stats_aggregator->AddToHistogram(
               strings::StrCat(
                   str_util::Split(prefix, "::", str_util::SkipEmpty()).back(),
-                  "::", func_.name(), "::execution_time"),
+                  "::", captured_func_->func().name(), "::execution_time"),
               {static_cast<float>(stats_collector->processing_time())});
         }
         if (model) {
@@ -495,15 +465,13 @@ void CapturedFunction::RunAsync(IteratorContext* ctx,
       std::move(done), ctx->model(), ctx->stats_aggregator(), prefix,
       std::move(stats_collector), std::placeholders::_1);
 
-  ctx->lib()->Run(f_opts, handle, frame, std::move(callback));
+  lib_->Run(f_opts, f_handle_, frame, std::move(callback));
 }
 
 CapturedFunction::CapturedFunction(const NameAttrList& func,
                                    std::vector<Tensor> captured_inputs,
                                    bool use_inter_op_parallelism)
     : func_(func),
-      lib_(nullptr),
-      f_handle_(kInvalidHandle),
       captured_inputs_(std::move(captured_inputs)),
       use_inter_op_parallelism_(use_inter_op_parallelism) {}
 
diff --git a/tensorflow/core/kernels/data/captured_function.h b/tensorflow/core/kernels/data/captured_function.h
index c6a5fe9e1e..cffaf405ec 100644
--- a/tensorflow/core/kernels/data/captured_function.h
+++ b/tensorflow/core/kernels/data/captured_function.h
@@ -34,59 +34,41 @@ class ResourceMgr;
 
 namespace data {
 
-// A `CapturedFunction` encapsulates a TensorFlow function and all of
-// the runtime support required to execute it.
+class CapturedFunction;
+
+// An InstantiatedCapturedFunction encapsulates all the runtime support needed
+// to execute a tensorflow function.
 //
-// The `Dataset`-related classes use `CapturedFunction` to execute
-// TensorFlow functions outside a the normal `OpKernel::Compute()`
-// context.
-class CapturedFunction {
+// While CapturedFunction (below) encapsulates the more permanent attributes
+// of the function i.e. name, captured arguments etc.,
+// InstantiatedCapturedFunction encapsulates the more runtime aspects i.e.
+// FunctionLibraryRuntime, function handle etc.
+//
+// The `Iterator-`related classes use `InstantiatedCapturedFunction` to execute
+// functions outside a the normal `OpKernel::Compute()` context.
+class InstantiatedCapturedFunction {
  public:
-  // Creates a new instance using a list of named attributes, fetching captured
-  // inputs from a context argument.
-  static Status Create(const NameAttrList& func, OpKernelContext* ctx,
-                       const string& argument,
-                       std::unique_ptr<CapturedFunction>* out_function);
-
-  // Creates a new instance using a list of named attributes, fetching captured
-  // inputs from a context argument.
-  //
-  // If `use_inter_op_parallelism` is false, the runtime may use an executor
-  // that is optimized for small functions.
-  static Status Create(const NameAttrList& func, OpKernelContext* ctx,
-                       const string& argument, bool use_inter_op_parallelism,
-                       std::unique_ptr<CapturedFunction>* out_function);
-
-  ~CapturedFunction();
+  ~InstantiatedCapturedFunction();
 
-  // Runs the "Captured function" using the given FLR and caches the lib and
-  // handle generated during instantiation. If Run is called with a different
-  // lib afterwards, generates an error. This method takes ownership of the
-  // tensors in `args`, in order to be able to deallocate them as early as
+  // Runs the "Instantiated Captured function". This method takes ownership of
+  // the tensors in `args`, in order to be able to deallocate them as early as
   // possible. Use `RunWithBorrowedArgs()` if the caller needs to retain
   // ownership of the `args`.
   Status Run(IteratorContext* ctx, std::vector<Tensor>&& args,
-             std::vector<Tensor>* rets);
+             std::vector<Tensor>* rets) const;
 
   // Synchronously runs the captured function on the given `args`, and stores
   // the results in `*rets`. Prefer to use `Run()` or `RunAsync()` when
   // possible.
   Status RunWithBorrowedArgs(IteratorContext* ctx,
                              const std::vector<Tensor>& args,
-                             std::vector<Tensor>* rets);
-
-  // Explicitly instantiate this function for use in the given
-  // context. This method, and the context-less overload
-  // `RunInstantiated()` below can be useful for calling a captured
-  // function in cases where an `IteratorContext*` is not available
-  // (such as a destructor).
-  Status Instantiate(IteratorContext* ctx);
+                             std::vector<Tensor>* rets) const;
 
   // Synchronously runs the captured function on the given `args`, and stores
   // the results in `*rets`. Prefer to use `Run()` or `RunAsync()` when
-  // possible.
-  //
-  // REQUIRES: `this->Instantiate()` must have been called before this method.
+  // possible. This can be useful for calling a captured
+  // function in cases where an `IteratorContext*` is not available
+  // (such as a destructor).
   Status RunInstantiated(const std::vector<Tensor>& args,
                          std::vector<Tensor>* rets);
 
@@ -97,16 +79,9 @@ class CapturedFunction {
   void RunAsync(IteratorContext* ctx, std::vector<Tensor>&& args,
                 std::vector<Tensor>* rets,
                 FunctionLibraryRuntime::DoneCallback done,
-                const string& prefix);
-
-  // Returns the named list of function arguments.
-  const NameAttrList& func() { return func_; }
+                const string& prefix) const;
 
-  // Returns that additional captured inputs that will be passed to the function
-  // when `Run*()` is called.
-  const std::vector<Tensor>& captured_inputs() { return captured_inputs_; }
-
-  // Returns a step ID for use when running a `CapturedFunction`.
+  // Returns a step ID for use when running an `InstantiatedCapturedFunction`.
   static int64 generate_step_id() {
     // Choose a step ID that is guaranteed not to clash with any
     // Session-generated step ID. DirectSession only generates
@@ -116,26 +91,66 @@ class CapturedFunction {
     return -std::abs(static_cast<int64>(random::New64()));
   }
 
+ private:
+  InstantiatedCapturedFunction(
+      FunctionLibraryRuntime* lib, FunctionLibraryRuntime::Handle f_handle,
+      DataTypeVector ret_types,
+      std::function<void(std::function<void()>)> runner,
+      CapturedFunction* captured_func);
+
+  friend class CapturedFunction;
+
+  FunctionLibraryRuntime* const lib_;
+  const FunctionLibraryRuntime::Handle f_handle_;
+  const DataTypeVector ret_types_;
+  std::function<void(std::function<void()>)> captured_runner_;
+  CapturedFunction* const captured_func_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(InstantiatedCapturedFunction);
+};
+
+// A `CapturedFunction` encapsulates a TensorFlow function, plus any "captured"
+// arguments that it closed over in the user program.
+class CapturedFunction {
+ public:
+  // Creates a new instance using a list of named attributes, fetching captured
+  // inputs from a context argument.
+  static Status Create(const NameAttrList& func, OpKernelContext* ctx,
+                       const string& argument,
+                       std::unique_ptr<CapturedFunction>* out_function);
+
+  // Creates a new instance using a list of named attributes, fetching captured
+  // inputs from a context argument.
+  //
+  // If `use_inter_op_parallelism` is false, the runtime may use an executor
+  // that is optimized for small functions.
+  static Status Create(const NameAttrList& func, OpKernelContext* ctx,
+                       const string& argument, bool use_inter_op_parallelism,
+                       std::unique_ptr<CapturedFunction>* out_function);
+
+  // Instantiates this function for use in the given context, providing an
+  // InstantiatedCapturedFunction that can be used to execute functions.
+  Status Instantiate(IteratorContext* ctx,
+                     std::unique_ptr<InstantiatedCapturedFunction>*
+                         instantiated_captured_function);
+
+  // Returns the named list of function arguments.
+  const NameAttrList& func() { return func_; }
+
+  // Returns that additional captured inputs that will be passed to the function
+  const std::vector<Tensor>& captured_inputs() { return captured_inputs_; }
+
  private:
   CapturedFunction(const NameAttrList& func,
                    std::vector<Tensor> captured_inputs,
                    bool use_inter_op_parallelism);
 
-  Status GetHandle(IteratorContext* ctx,
-                   FunctionLibraryRuntime::Handle* out_handle);
-
-  mutex mu_;
   const NameAttrList func_;
-  FunctionLibraryRuntime* lib_ GUARDED_BY(mu_);
-  FunctionLibraryRuntime::Handle f_handle_ GUARDED_BY(mu_);
   const std::vector<Tensor> captured_inputs_;
-  DataTypeSlice ret_types_;
-  std::function<void(std::function<void()>)> captured_runner_ = nullptr;
   const bool use_inter_op_parallelism_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(CapturedFunction);
 };
-
 }  // namespace data
 
 // TODO(b/114112161): Remove these aliases when all users have moved over to the
diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 1be38c4cb2..4d92d314d3 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -81,12 +81,12 @@ std::vector<bool> ComputeMoveVector(const std::vector<int>& indices) {
 
 Status MakeIteratorFromInputElement(
     IteratorContext* ctx, const std::vector<Tensor>& input_element,
-    int64 thread_index, CapturedFunction* captured_func, StringPiece prefix,
-    std::unique_ptr<IteratorBase>* out_iterator) {
+    int64 thread_index, const InstantiatedCapturedFunction& inst_captured_func,
+    StringPiece prefix, std::unique_ptr<IteratorBase>* out_iterator) {
   std::vector<Tensor> return_values;
 
-  TF_RETURN_IF_ERROR(
-      captured_func->RunWithBorrowedArgs(ctx, input_element, &return_values));
+  TF_RETURN_IF_ERROR(inst_captured_func.RunWithBorrowedArgs(ctx, input_element,
+                                                            &return_values));
 
   if (!(return_values.size() == 1 && return_values[0].dtype() == DT_VARIANT &&
         TensorShapeUtils::IsScalar(return_values[0].shape()))) {
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index 0b2816119d..23a3d93ed1 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -44,8 +44,8 @@ std::vector<bool> ComputeMoveVector(const std::vector<int>& indices);
 
 Status MakeIteratorFromInputElement(
     IteratorContext* ctx, const std::vector<Tensor>& input_element,
-    int64 thread_index, CapturedFunction* captured_func, StringPiece prefix,
-    std::unique_ptr<IteratorBase>* out_iterator);
+    int64 thread_index, const InstantiatedCapturedFunction& inst_captured_func,
+    StringPiece prefix, std::unique_ptr<IteratorBase>* out_iterator);
 
 // Returns Status::OK() if `expected` and `received` types match,
 // errors::InvalidArgument otherwise.
diff --git a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
index 068f854023..75eacb4b5b 100644
--- a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
@@ -206,7 +206,8 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         }
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        TF_RETURN_IF_ERROR(dataset()->captured_func_->Instantiate(ctx));
+        TF_RETURN_IF_ERROR(dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_));
         return Status::OK();
       }
 
@@ -1052,8 +1053,8 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
           {
             tracing::ScopedActivity trace(
                 "NumaMapAndBatch::Iterator::Worker::FunctionExecution");
-            s = dataset()->captured_func_->Run(ctx.get(), std::move(input),
-                                               &return_values);
+            s = instantiated_captured_func_->Run(ctx.get(), std::move(input),
+                                                 &return_values);
           }
           WORKER_VLOG(4) << "ran function for index: " << index
                          << ", sequence_number: " << sequence_number;
@@ -1099,6 +1100,7 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       const std::shared_ptr<condition_variable> autotune_cond_var_;
       // The maximum number of parallel calls (can be auto-tuned).
       const std::shared_ptr<model::SharedState> num_parallel_calls_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
 
       // Caches the last-seen value of num_parallel_calls_->value to
       // short-circuit starting workers.
diff --git a/tensorflow/core/kernels/data/filter_dataset_op.cc b/tensorflow/core/kernels/data/filter_dataset_op.cc
index 40cbb12425..b8b657d343 100644
--- a/tensorflow/core/kernels/data/filter_dataset_op.cc
+++ b/tensorflow/core/kernels/data/filter_dataset_op.cc
@@ -34,7 +34,8 @@ namespace {
 class FilterDatasetOp : public UnaryDatasetOpKernel {
  public:
   using FilterIteratorPredicate =
-      std::function<Status(IteratorContext*, std::vector<Tensor>, bool*)>;
+      std::function<Status(IteratorContext*, InstantiatedCapturedFunction*,
+                           std::vector<Tensor>, bool*)>;
 
   explicit FilterDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
@@ -55,13 +56,12 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
 
     FilterIteratorPredicate filter_pred;
     if (indices.empty()) {
-      CapturedFunction* raw_captured_func = captured_func.get();
-      filter_pred = [raw_captured_func](IteratorContext* ctx,
-                                        const std::vector<Tensor>& args,
-                                        bool* out_matched) {
+      filter_pred = [](IteratorContext* ctx,
+                       InstantiatedCapturedFunction* inst_captured_func,
+                       const std::vector<Tensor>& args, bool* out_matched) {
         std::vector<Tensor> result;
         TF_RETURN_IF_ERROR(
-            raw_captured_func->RunWithBorrowedArgs(ctx, args, &result));
+            inst_captured_func->RunWithBorrowedArgs(ctx, args, &result));
 
         if (result.size() != 1 || result[0].dtype() != DT_BOOL ||
             result[0].NumElements() != 1) {
@@ -73,6 +73,7 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
       };
     } else {
       filter_pred = [indices](IteratorContext* ctx,
+                              InstantiatedCapturedFunction* inst_captured_func,
                               const std::vector<Tensor>& args,
                               bool* out_matched) {
         const Tensor& predicate = args[indices[0]];
@@ -169,7 +170,8 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -197,7 +199,8 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
             return Status::OK();
           }
 
-          TF_RETURN_IF_ERROR(filter_pred_(ctx, *out_tensors, &matched));
+          TF_RETURN_IF_ERROR(filter_pred_(
+              ctx, instantiated_captured_func_.get(), *out_tensors, &matched));
           if (!matched) {
             // Clear the output tensor list since it didn't match.
             out_tensors->clear();
@@ -274,6 +277,7 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
       int64 dropped_elements_ GUARDED_BY(mu_);
       const FilterIteratorPredicate filter_pred_;
       string prefix_end_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/flat_map_dataset_op.cc b/tensorflow/core/kernels/data/flat_map_dataset_op.cc
index 9b42981ed7..3846334622 100644
--- a/tensorflow/core/kernels/data/flat_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/flat_map_dataset_op.cc
@@ -122,7 +122,8 @@ class FlatMapDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -243,8 +244,7 @@ class FlatMapDatasetOp : public UnaryDatasetOpKernel {
           EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         return MakeIteratorFromInputElement(
             ctx, captured_func_inputs_, element_index_++,
-            dataset()->captured_func_.get(), prefix(),
-            &current_element_iterator_);
+            *instantiated_captured_func_, prefix(), &current_element_iterator_);
       }
 
       mutex mu_;
@@ -252,6 +252,7 @@ class FlatMapDatasetOp : public UnaryDatasetOpKernel {
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> current_element_iterator_ GUARDED_BY(mu_);
       std::vector<Tensor> captured_func_inputs_ GUARDED_BY(mu_);
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/generator_dataset_op.cc b/tensorflow/core/kernels/data/generator_dataset_op.cc
index ed18d6ed9d..c089bfc45f 100644
--- a/tensorflow/core/kernels/data/generator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/generator_dataset_op.cc
@@ -73,7 +73,8 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
     ~Iterator() override {
       if (!finalized_) {
         std::vector<Tensor> ignored;
-        Status s = dataset()->finalize_func_->RunInstantiated(state_, &ignored);
+        Status s =
+            instantiated_finalize_func_->RunInstantiated(state_, &ignored);
         if (!s.ok()) {
           LOG(WARNING)
               << "Error occurred when finalizing GeneratorDataset iterator: "
@@ -83,9 +84,12 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
     }
 
     Status Initialize(IteratorContext* ctx) override {
-      TF_RETURN_IF_ERROR(dataset()->init_func_->Instantiate(ctx));
-      TF_RETURN_IF_ERROR(dataset()->next_func_->Instantiate(ctx));
-      TF_RETURN_IF_ERROR(dataset()->finalize_func_->Instantiate(ctx));
+      TF_RETURN_IF_ERROR(
+          dataset()->init_func_->Instantiate(ctx, &instantiated_init_func_));
+      TF_RETURN_IF_ERROR(
+          dataset()->next_func_->Instantiate(ctx, &instantiated_next_func_));
+      TF_RETURN_IF_ERROR(dataset()->finalize_func_->Instantiate(
+          ctx, &instantiated_finalize_func_));
       return Status::OK();
     }
 
@@ -96,7 +100,7 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
 
       if (!initialized_) {
         TF_RETURN_IF_ERROR(
-            dataset()->init_func_->RunWithBorrowedArgs(ctx, {}, &state_));
+            instantiated_init_func_->RunWithBorrowedArgs(ctx, {}, &state_));
         initialized_ = true;
       }
 
@@ -105,8 +109,8 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
         return Status::OK();
       }
 
-      Status s =
-          dataset()->next_func_->RunWithBorrowedArgs(ctx, state_, out_tensors);
+      Status s = instantiated_next_func_->RunWithBorrowedArgs(ctx, state_,
+                                                              out_tensors);
       if (s.ok()) {
         *end_of_sequence = false;
       } else if (errors::IsOutOfRange(s)) {
@@ -119,7 +123,7 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
         // finalize function.
         std::vector<Tensor> ignored;
         TF_RETURN_IF_ERROR(
-            dataset()->finalize_func_->RunInstantiated(state_, &ignored));
+            instantiated_finalize_func_->RunInstantiated(state_, &ignored));
         finalized_ = true;
       }
       return s;
@@ -136,6 +140,9 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
     bool initialized_ GUARDED_BY(mu_) = false;
     bool finalized_ GUARDED_BY(mu_) = false;
     std::vector<Tensor> state_ GUARDED_BY(mu_);
+    std::unique_ptr<InstantiatedCapturedFunction> instantiated_init_func_;
+    std::unique_ptr<InstantiatedCapturedFunction> instantiated_next_func_;
+    std::unique_ptr<InstantiatedCapturedFunction> instantiated_finalize_func_;
   };
 
   const std::unique_ptr<CapturedFunction> init_func_;
diff --git a/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc b/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc
index dc1925a21f..a01085491e 100644
--- a/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc
+++ b/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc
@@ -191,11 +191,14 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        TF_RETURN_IF_ERROR(dataset()->captured_key_func_->Instantiate(ctx));
-        TF_RETURN_IF_ERROR(dataset()->captured_init_func_->Instantiate(ctx));
-        TF_RETURN_IF_ERROR(dataset()->captured_reduce_func_->Instantiate(ctx));
-        TF_RETURN_IF_ERROR(
-            dataset()->captured_finalize_func_->Instantiate(ctx));
+        TF_RETURN_IF_ERROR(dataset()->captured_key_func_->Instantiate(
+            ctx, &instantiated_key_func_));
+        TF_RETURN_IF_ERROR(dataset()->captured_init_func_->Instantiate(
+            ctx, &instantiated_init_func_));
+        TF_RETURN_IF_ERROR(dataset()->captured_reduce_func_->Instantiate(
+            ctx, &instantiated_reduce_func_));
+        TF_RETURN_IF_ERROR(dataset()->captured_finalize_func_->Instantiate(
+            ctx, &instantiated_finalize_func_));
         return Status::OK();
       }
 
@@ -213,9 +216,8 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
           if (!end_of_input_) {
             // Run the key function on the input element.
             std::vector<Tensor> key_func_output;
-            TF_RETURN_IF_ERROR(
-                dataset()->captured_key_func_->RunWithBorrowedArgs(
-                    ctx, next_input_element, &key_func_output));
+            TF_RETURN_IF_ERROR(instantiated_key_func_->RunWithBorrowedArgs(
+                ctx, next_input_element, &key_func_output));
 
             if (key_func_output.size() != 1 ||
                 key_func_output[0].dtype() != DT_INT64 ||
@@ -229,7 +231,7 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
             if (states_.find(key) == states_.end()) {
               // Run the init function to create the initial state.
               std::vector<Tensor> init_func_output;
-              TF_RETURN_IF_ERROR(dataset()->captured_init_func_->Run(
+              TF_RETURN_IF_ERROR(instantiated_init_func_->Run(
                   ctx, std::move(key_func_output), &init_func_output));
               states_[key] = init_func_output;
             }
@@ -243,7 +245,7 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
                       std::back_inserter(args));
 
             std::vector<Tensor> reduce_func_output;
-            TF_RETURN_IF_ERROR(dataset()->captured_reduce_func_->Run(
+            TF_RETURN_IF_ERROR(instantiated_reduce_func_->Run(
                 ctx, std::move(args), &reduce_func_output));
             states_[key] = reduce_func_output;
           } else {
@@ -259,9 +261,8 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
           *end_of_sequence = true;
           return Status::OK();
         }
-        TF_RETURN_IF_ERROR(
-            dataset()->captured_finalize_func_->RunWithBorrowedArgs(
-                ctx, states_[keys_[keys_index_++]], out_tensors));
+        TF_RETURN_IF_ERROR(instantiated_finalize_func_->RunWithBorrowedArgs(
+            ctx, states_[keys_[keys_index_++]], out_tensors));
         *end_of_sequence = false;
         return Status::OK();
       }
@@ -384,6 +385,10 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
       std::map<int64, std::vector<Tensor>> states_ GUARDED_BY(mu_);
       std::vector<int64> keys_ GUARDED_BY(mu_);
       int64 keys_index_ GUARDED_BY(mu_) = 0;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_key_func_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_init_func_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_reduce_func_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_finalize_func_;
     };
 
     const NameAttrList& key_func() const { return captured_key_func_->func(); }
diff --git a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc b/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
index 64db5df31e..d9f504a170 100644
--- a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
@@ -175,10 +175,12 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        TF_RETURN_IF_ERROR(dataset()->captured_key_func_->Instantiate(ctx));
-        TF_RETURN_IF_ERROR(dataset()->captured_reduce_func_->Instantiate(ctx));
-        TF_RETURN_IF_ERROR(
-            dataset()->captured_window_size_func_->Instantiate(ctx));
+        TF_RETURN_IF_ERROR(dataset()->captured_key_func_->Instantiate(
+            ctx, &instantiated_key_func_));
+        TF_RETURN_IF_ERROR(dataset()->captured_reduce_func_->Instantiate(
+            ctx, &instantiated_reduce_func_));
+        TF_RETURN_IF_ERROR(dataset()->captured_window_size_func_->Instantiate(
+            ctx, &instantiated_window_size_func_));
         return Status::OK();
       }
 
@@ -215,9 +217,8 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
               // Run the key function on the input element to identify its
               // group.
               std::vector<Tensor> key_func_output;
-              TF_RETURN_IF_ERROR(
-                  dataset()->captured_key_func_->RunWithBorrowedArgs(
-                      ctx, next_input_element, &key_func_output));
+              TF_RETURN_IF_ERROR(instantiated_key_func_->RunWithBorrowedArgs(
+                  ctx, next_input_element, &key_func_output));
 
               if (key_func_output.size() != 1 ||
                   key_func_output[0].dtype() != DT_INT64 ||
@@ -232,7 +233,7 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
                 // Run the window size function on the key to identify its
                 // window size.
                 std::vector<Tensor> window_size_func_output;
-                TF_RETURN_IF_ERROR(dataset()->captured_window_size_func_->Run(
+                TF_RETURN_IF_ERROR(instantiated_window_size_func_->Run(
                     ctx, std::move(key_func_output), &window_size_func_output));
 
                 if (window_size_func_output.size() != 1 ||
@@ -452,8 +453,8 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
         std::vector<Tensor> args(
             {std::move(key_arg), std::move(group_dataset_arg)});
         std::vector<Tensor> return_values;
-        TF_RETURN_IF_ERROR(dataset()->captured_reduce_func_->Run(
-            ctx, std::move(args), &return_values));
+        TF_RETURN_IF_ERROR(instantiated_reduce_func_->Run(ctx, std::move(args),
+                                                          &return_values));
 
         if (!(return_values.size() == 1 &&
               return_values[0].dtype() == DT_VARIANT &&
@@ -482,6 +483,10 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
       std::map<int64, std::vector<std::vector<Tensor>>> groups_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> current_group_iterator_ GUARDED_BY(mu_);
       std::map<int64, int64> window_sizes_ GUARDED_BY(mu_);
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_key_func_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_reduce_func_;
+      std::unique_ptr<InstantiatedCapturedFunction>
+          instantiated_window_size_func_;
     };
 
     Status OtherArgumentsNodeAndType(
diff --git a/tensorflow/core/kernels/data/interleave_dataset_op.cc b/tensorflow/core/kernels/data/interleave_dataset_op.cc
index 9574e400a2..54e3645612 100644
--- a/tensorflow/core/kernels/data/interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/interleave_dataset_op.cc
@@ -149,7 +149,8 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       void AdvanceToNextInCycle() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
@@ -195,7 +196,7 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel {
             if (!end_of_input_) {
               TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
                   ctx, args_list_[cycle_index_], cycle_index_,
-                  dataset()->captured_func_.get(), prefix(),
+                  *instantiated_captured_func_, prefix(),
                   &current_elements_[cycle_index_]));
               ++num_open_;
             }
@@ -286,7 +287,7 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel {
                   &args_list_[idx][i]));
             }
             TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
-                ctx, args_list_[idx], idx, dataset()->captured_func_.get(),
+                ctx, args_list_[idx], idx, *instantiated_captured_func_,
                 prefix(), &current_elements_[idx]));
             TF_RETURN_IF_ERROR(
                 RestoreInput(ctx, reader, current_elements_[idx]));
@@ -306,6 +307,7 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel {
       int64 block_index_ GUARDED_BY(mu_) = 0;
       bool end_of_input_ GUARDED_BY(mu_) = false;
       size_t num_open_ GUARDED_BY(mu_) = 0;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 98b67454d5..c50af846f9 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/renamed_device.h"
 #include "tensorflow/core/common_runtime/threadpool_device.h"
 #include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
@@ -59,25 +60,25 @@ class IteratorResource : public ResourceBase {
                    std::unique_ptr<ProcessFunctionLibraryRuntime> pflr,
                    FunctionLibraryRuntime* lib)
       : device_mgr_(std::move(device_mgr)),
-        flib_def_(std::move(flib_def)),
-        pflr_(std::move(pflr)),
-        lib_(lib),
-        iterator_(nullptr),
+        iterator_state_(
+            new State(std::move(flib_def), std::move(pflr), lib, nullptr)),
         output_dtypes_(output_dtypes),
         output_shapes_(output_shapes) {}
 
   Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
                  bool* end_of_sequence) {
-    IteratorContext::Params params(ctx);
-    std::shared_ptr<IteratorBase> captured_iterator;
+    std::shared_ptr<State> captured_state;
     {
       tf_shared_lock l(mu_);
-      captured_iterator = iterator_;
-      params.lib = lib_;
+      captured_state = iterator_state_;
     }
-    if (captured_iterator) {
-      return captured_iterator->GetNext(IteratorContext(std::move(params)),
-                                        out_tensors, end_of_sequence);
+    if (captured_state->iterator) {
+      IteratorContext::Params params(ctx);
+      params.lib = captured_state->lib;
+      params.function_handle_cache =
+          captured_state->function_handle_cache.get();
+      return captured_state->iterator->GetNext(
+          IteratorContext(std::move(params)), out_tensors, end_of_sequence);
     } else {
       return errors::FailedPrecondition(
           "GetNext() failed because the iterator has not been initialized. "
@@ -92,13 +93,13 @@ class IteratorResource : public ResourceBase {
   }
 
   Status Save(SerializationContext* ctx, IteratorStateWriter* writer) {
-    std::shared_ptr<IteratorBase> captured_iterator;
+    std::shared_ptr<State> captured_state;
     {
       tf_shared_lock l(mu_);
-      captured_iterator = iterator_;
+      captured_state = iterator_state_;
     }
-    if (captured_iterator) {
-      return captured_iterator->Save(ctx, writer);
+    if (captured_state) {
+      return captured_state->iterator->Save(ctx, writer);
     } else {
       return errors::FailedPrecondition(
           "Save() failed because the iterator has not been initialized. "
@@ -130,64 +131,62 @@ class IteratorResource : public ResourceBase {
     // because some of the OpKernels in the graph might call functions that are
     // only defined in the loaded GraphDef.
     FunctionLibraryRuntime* lib;
-    std::unique_ptr<DeviceMgr> device_mgr(nullptr);
     std::unique_ptr<FunctionLibraryDefinition> flib_def(nullptr);
     std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
     TF_RETURN_IF_ERROR(ctx->function_library()->Clone(&flib_def, &pflr, &lib));
     TF_RETURN_IF_ERROR(flib_def->AddLibrary(graph_def.library()));
+    std::unique_ptr<State> new_state(
+        new State(std::move(flib_def), std::move(pflr), lib, nullptr));
 
     TF_RETURN_IF_ERROR(
-        graph_runner.Run(&graph, lib, {}, {output_node}, &outputs));
+        graph_runner.Run(&graph, new_state->lib, {}, {output_node}, &outputs));
     TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset));
 
-    std::unique_ptr<IteratorBase> iterator;
-    {
-      IteratorContext::Params params(ctx);
-      params.lib = lib;
-      TF_RETURN_IF_ERROR(dataset->MakeIterator(
-          IteratorContext(std::move(params)), "Iterator", &iterator));
-    }
-    TF_RETURN_IF_ERROR(
-        VerifyTypesMatch(output_dtypes_, iterator->output_dtypes()));
+    IteratorContext::Params params(ctx);
+    params.lib = new_state->lib;
+    params.function_handle_cache = new_state->function_handle_cache.get();
+    TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
+                                             "Iterator", &new_state->iterator));
     TF_RETURN_IF_ERROR(
-        VerifyShapesCompatible(output_shapes_, iterator->output_shapes()));
+        VerifyTypesMatch(output_dtypes_, new_state->iterator->output_dtypes()));
+    TF_RETURN_IF_ERROR(VerifyShapesCompatible(
+        output_shapes_, new_state->iterator->output_shapes()));
 
     {
       IteratorContext::Params params(ctx);
-      params.lib = lib;
-      DeviceBase* device = lib->device();
+      params.lib = new_state->lib;
+      params.function_handle_cache = new_state->function_handle_cache.get();
+      DeviceBase* device = new_state->lib->device();
       params.allocator_getter = [device](AllocatorAttributes attrs) {
         return device->GetAllocator(attrs);
       };
       IteratorContext iter_ctx(std::move(params));
-      TF_RETURN_IF_ERROR(iterator->Restore(&iter_ctx, reader));
-    }
-
-    std::shared_ptr<IteratorBase> old_iterator;
-    {
-      mutex_lock l(mu_);
-      std::swap(device_mgr_, device_mgr);
-      std::swap(flib_def_, flib_def);
-      std::swap(pflr_, pflr);
-      lib_ = lib;
-      old_iterator = iterator_;
-      iterator_ = std::move(iterator);
+      TF_RETURN_IF_ERROR(new_state->iterator->Restore(&iter_ctx, reader));
     }
 
+    mutex_lock l(mu_);
+    iterator_state_ = std::move(new_state);
     return Status::OK();
   }
 
   Status AddLibrary(const FunctionLibraryDefinition& flib_def) {
     mutex_lock l(mu_);
-    return flib_def_->AddLibrary(flib_def);
+    return iterator_state_->flib_def->AddLibrary(flib_def);
   }
 
   Status SetIteratorFromDataset(OpKernelContext* ctx, DatasetBase* dataset) {
-    mutex_lock l(mu_);
+    std::shared_ptr<State> new_state;
+    {
+      tf_shared_lock l(mu_);
+      new_state.reset(new State(iterator_state_->flib_def,
+                                iterator_state_->pflr, iterator_state_->lib,
+                                nullptr, nullptr));
+    }
+
     // Ensure that the iterator has access to all functions in the current
     // subgraph, because some functions may have been defined after the resource
     // was initially created.
-    Status s = flib_def_->AddLibrary(
+    Status s = new_state->flib_def->AddLibrary(
         *ctx->function_library()->GetFunctionLibraryDefinition());
 
     if (!s.ok()) {
@@ -196,29 +195,33 @@ class IteratorResource : public ResourceBase {
       // executing graph. In that case, we create a new function runtime for
       // this iterator, based on the current `OpKernelContext`, which will have
       // the functions we need.
-      iterator_.reset();
       FunctionLibraryRuntime* lib;
-      std::unique_ptr<DeviceMgr> device_mgr(nullptr);
       std::unique_ptr<FunctionLibraryDefinition> flib_def(nullptr);
       std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
       TF_RETURN_IF_ERROR(
           ctx->function_library()->Clone(&flib_def, &pflr, &lib));
-      std::swap(device_mgr_, device_mgr);
-      std::swap(flib_def_, flib_def);
-      std::swap(pflr_, pflr);
-      lib_ = lib;
+      new_state->flib_def = std::move(flib_def);
+      new_state->pflr = std::move(pflr);
+      new_state->lib = lib;
     }
 
+    new_state->function_handle_cache.reset(
+        new FunctionHandleCache(new_state->lib));
+    // Create new iterator.
     std::unique_ptr<IteratorBase> iterator;
     IteratorContext::Params params(ctx);
-    params.lib = lib_;
+    params.lib = new_state->lib;
+    params.function_handle_cache = new_state->function_handle_cache.get();
     TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
                                              "Iterator", &iterator));
     TF_RETURN_IF_ERROR(
         VerifyTypesMatch(output_dtypes_, iterator->output_dtypes()));
     TF_RETURN_IF_ERROR(
         VerifyShapesCompatible(output_shapes_, iterator->output_shapes()));
-    iterator_.reset(iterator.release());
+    std::swap(new_state->iterator, iterator);
+
+    mutex_lock l(mu_);
+    std::swap(iterator_state_, new_state);
     return Status::OK();
   }
 
@@ -231,12 +234,37 @@ class IteratorResource : public ResourceBase {
   }
 
  private:
+  struct State {
+    State(std::shared_ptr<FunctionLibraryDefinition> flib_def,
+          std::shared_ptr<ProcessFunctionLibraryRuntime> pflr,
+          FunctionLibraryRuntime* lib, std::unique_ptr<IteratorBase> iterator)
+        : flib_def(flib_def),
+          pflr(pflr),
+          lib(lib),
+          function_handle_cache(absl::make_unique<FunctionHandleCache>(lib)),
+          iterator(std::move(iterator)) {}
+
+    State(std::shared_ptr<FunctionLibraryDefinition> flib_def,
+          std::shared_ptr<ProcessFunctionLibraryRuntime> pflr,
+          FunctionLibraryRuntime* lib,
+          std::unique_ptr<FunctionHandleCache> function_handle_cache,
+          std::unique_ptr<IteratorBase> iterator)
+        : flib_def(flib_def),
+          pflr(pflr),
+          lib(lib),
+          function_handle_cache(std::move(function_handle_cache)),
+          iterator(std::move(iterator)) {}
+
+    std::shared_ptr<FunctionLibraryDefinition> flib_def;
+    std::shared_ptr<ProcessFunctionLibraryRuntime> pflr;
+    FunctionLibraryRuntime* lib = nullptr;  // not owned.
+    std::unique_ptr<FunctionHandleCache> function_handle_cache;
+    std::unique_ptr<IteratorBase> iterator;
+  };
+
   mutex mu_;
-  std::unique_ptr<DeviceMgr> device_mgr_ GUARDED_BY(mu_);
-  std::unique_ptr<FunctionLibraryDefinition> flib_def_ GUARDED_BY(mu_);
-  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_ GUARDED_BY(mu_);
-  FunctionLibraryRuntime* lib_ GUARDED_BY(mu_) = nullptr;  // not owned.
-  std::shared_ptr<IteratorBase> iterator_ GUARDED_BY(mu_);
+  const std::unique_ptr<DeviceMgr> device_mgr_ GUARDED_BY(mu_);
+  std::shared_ptr<State> iterator_state_ GUARDED_BY(mu_);
   const DataTypeVector output_dtypes_;
   const std::vector<PartialTensorShape> output_shapes_;
 };
@@ -641,10 +669,15 @@ class ToSingleElementOp : public AsyncOpKernel {
       OP_REQUIRES_OK_ASYNC(
           ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done);
       std::unique_ptr<IteratorBase> iterator;
+      IteratorContext::Params params(ctx);
+      std::unique_ptr<FunctionHandleCache> function_handle_cache(
+          new FunctionHandleCache(params.lib));
+      params.function_handle_cache = function_handle_cache.get();
+      IteratorContext iter_ctx(std::move(params));
+
       OP_REQUIRES_OK_ASYNC(
           ctx,
-          dataset->MakeIterator(IteratorContext(ctx), "SingleElementIterator",
-                                &iterator),
+          dataset->MakeIterator(&iter_ctx, "SingleElementIterator", &iterator),
           done);
 
       // NOTE(jsimsa): We must destroy the iterator before calling `done()`, to
@@ -658,8 +691,8 @@ class ToSingleElementOp : public AsyncOpKernel {
       components.reserve(dataset->output_dtypes().size());
       bool end_of_sequence = false;
 
-      Status s = raw_iterator->GetNext(IteratorContext(ctx), &components,
-                                       &end_of_sequence);
+      Status s =
+          raw_iterator->GetNext(&iter_ctx, &components, &end_of_sequence);
       if (!s.ok()) {
         ctx->SetStatus(s);
         return;
@@ -674,8 +707,8 @@ class ToSingleElementOp : public AsyncOpKernel {
       }
 
       components.clear();
-      Status s2 = raw_iterator->GetNext(IteratorContext(ctx), &components,
-                                        &end_of_sequence);
+      Status s2 =
+          raw_iterator->GetNext(&iter_ctx, &components, &end_of_sequence);
       if (!s2.ok()) {
         ctx->SetStatus(s2);
         return;
@@ -724,8 +757,16 @@ class ReduceDatasetOp : public AsyncOpKernel {
                                    use_inter_op_parallelism_, &captured_func),
           done);
 
-      IteratorContext iter_ctx(ctx);
-      OP_REQUIRES_OK_ASYNC(ctx, captured_func->Instantiate(&iter_ctx), done);
+      IteratorContext::Params params(ctx);
+      std::unique_ptr<FunctionHandleCache> function_handle_cache(
+          new FunctionHandleCache(params.lib));
+      params.function_handle_cache = function_handle_cache.get();
+      IteratorContext iter_ctx(std::move(params));
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func;
+      OP_REQUIRES_OK_ASYNC(
+          ctx,
+          captured_func->Instantiate(&iter_ctx, &instantiated_captured_func),
+          done);
 
       std::unique_ptr<IteratorBase> iterator;
       OP_REQUIRES_OK_ASYNC(
@@ -759,8 +800,8 @@ class ReduceDatasetOp : public AsyncOpKernel {
                   std::back_inserter(args));
 
         std::vector<Tensor> reduce_func_output;
-        status =
-            captured_func->Run(&iter_ctx, std::move(args), &reduce_func_output);
+        status = instantiated_captured_func->Run(&iter_ctx, std::move(args),
+                                                 &reduce_func_output);
         if (!status.ok()) {
           break;
         }
diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
index 014f37686f..ac001c22b9 100644
--- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
@@ -46,7 +46,8 @@ const int64 kMaxBatchResults = 16;
 class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
  public:
   using MapAndBatchIteratorFunction =
-      std::function<void(IteratorContext*, const string&, std::vector<Tensor>,
+      std::function<void(IteratorContext*, InstantiatedCapturedFunction*,
+                         const string&, std::vector<Tensor>,
                          std::shared_ptr<std::vector<Tensor>>, StatusCallback)>;
 
   explicit MapAndBatchDatasetOp(OpKernelConstruction* ctx)
@@ -105,19 +106,20 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     MapAndBatchIteratorFunction map_func;
     CapturedFunction* raw_captured_func = captured_func.get();
     if (indices.empty()) {
-      map_func = [raw_captured_func](
-                     IteratorContext* ctx, const string& prefix,
-                     std::vector<Tensor> args,
-                     std::shared_ptr<std::vector<Tensor>> out_tensors,
-                     StatusCallback done) {
-        raw_captured_func->RunAsync(ctx, std::move(args), out_tensors.get(),
-                                    std::move(done), prefix);
+      map_func = [](IteratorContext* ctx,
+                    InstantiatedCapturedFunction* instantiated_captured_func,
+                    const string& prefix, std::vector<Tensor> args,
+                    std::shared_ptr<std::vector<Tensor>> out_tensors,
+                    StatusCallback done) {
+        instantiated_captured_func->RunAsync(
+            ctx, std::move(args), out_tensors.get(), std::move(done), prefix);
       };
     } else {
       std::vector<bool> can_move = ComputeMoveVector(indices);
       map_func = [raw_captured_func, indices, can_move](
-                     IteratorContext* ctx, const string& prefix,
-                     std::vector<Tensor> args,
+                     IteratorContext* ctx,
+                     InstantiatedCapturedFunction* instantiated_captured_func,
+                     const string& prefix, std::vector<Tensor> args,
                      std::shared_ptr<std::vector<Tensor>> out_tensors,
                      StatusCallback done) {
         const std::vector<Tensor>& captured_inputs =
@@ -275,7 +277,8 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         }
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -464,8 +467,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
 
         // Apply the map function on `input_element`, storing the result in
         // `return_values`, and invoking `done` when finished.
-        map_func_(ctx.get(), prefix(), std::move(input_element),
-                  std::move(return_values), std::move(done));
+        map_func_(ctx.get(), instantiated_captured_func_.get(), prefix(),
+                  std::move(input_element), std::move(return_values),
+                  std::move(done));
       }
 
       Status CopyPartialBatch(Tensor* output, const Tensor& value,
@@ -785,6 +789,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       // Identifies the maximum number of batch results to store.
       int64 max_batch_results_ GUARDED_BY(*mu_);
       string prefix_end_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index ab20b83298..a4e61e02e2 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -30,8 +30,9 @@ namespace {
 
 class MapDatasetOp : public UnaryDatasetOpKernel {
  public:
-  using MapIteratorFunction = std::function<Status(
-      IteratorContext*, std::vector<Tensor>, std::vector<Tensor>*)>;
+  using MapIteratorFunction =
+      std::function<Status(IteratorContext*, InstantiatedCapturedFunction*,
+                           std::vector<Tensor>, std::vector<Tensor>*)>;
 
   explicit MapDatasetOp(OpKernelConstruction* ctx) : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
@@ -54,15 +55,18 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
     MapIteratorFunction map_func;
     CapturedFunction* raw_captured_func = captured_func.get();
     if (indices.empty()) {
-      map_func = [raw_captured_func](IteratorContext* ctx,
-                                     std::vector<Tensor> args,
-                                     std::vector<Tensor>* out_tensors) {
-        return raw_captured_func->Run(ctx, std::move(args), out_tensors);
+      map_func = [](IteratorContext* ctx,
+                    InstantiatedCapturedFunction* inst_captured_func,
+                    std::vector<Tensor> args,
+                    std::vector<Tensor>* out_tensors) {
+        return inst_captured_func->Run(ctx, std::move(args), out_tensors);
       };
     } else {
       std::vector<bool> can_move = ComputeMoveVector(indices);
       map_func = [raw_captured_func, indices, can_move](
-                     IteratorContext* ctx, std::vector<Tensor> args,
+                     IteratorContext* ctx,
+                     InstantiatedCapturedFunction* inst_captured_func,
+                     std::vector<Tensor> args,
                      std::vector<Tensor>* out_tensors) {
         const std::vector<Tensor>& captured_inputs =
             raw_captured_func->captured_inputs();
@@ -177,7 +181,8 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -194,7 +199,8 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
           return Status::OK();
         }
 
-        Status s = map_func_(ctx, args, out_tensors);
+        Status s = map_func_(ctx, instantiated_captured_func_.get(), args,
+                             out_tensors);
         if (errors::IsOutOfRange(s)) {
           // `f` may deliberately raise `errors::OutOfRange` to indicate
           // that we should terminate the iteration early.
@@ -226,6 +232,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
      private:
       std::unique_ptr<IteratorBase> input_impl_;
       const MapIteratorFunction map_func_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index 5268007e3d..a070456414 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
@@ -40,18 +41,21 @@ using MultiDeviceIteratorCallback =
 
 class MultiDeviceIterator : public ResourceBase {
  public:
-  MultiDeviceIterator(const DataTypeVector& output_types,
-                      const std::vector<PartialTensorShape>& output_shapes,
-                      const std::vector<string>& devices,
-                      std::unique_ptr<FunctionLibraryDefinition> flib_def,
-                      std::unique_ptr<ProcessFunctionLibraryRuntime> pflr,
-                      FunctionLibraryRuntime* lib)
+  MultiDeviceIterator(
+      const DataTypeVector& output_types,
+      const std::vector<PartialTensorShape>& output_shapes,
+      const std::vector<string>& devices,
+      std::unique_ptr<FunctionLibraryDefinition> flib_def,
+      std::unique_ptr<ProcessFunctionLibraryRuntime> pflr,
+      FunctionLibraryRuntime* lib,
+      std::unique_ptr<FunctionHandleCache> function_handle_cache)
       : output_types_(output_types),
         output_shapes_(output_shapes),
         devices_(devices),
         flib_def_(std::move(flib_def)),
         pflr_(std::move(pflr)),
-        lib_(lib) {
+        lib_(lib),
+        function_handle_cache_(std::move(function_handle_cache)) {
     DCHECK(lib_ != nullptr);
   }
 
@@ -93,6 +97,7 @@ class MultiDeviceIterator : public ResourceBase {
     } else {
       IteratorContext::Params params(ctx);
       params.lib = lib_;
+      params.function_handle_cache = function_handle_cache_.get();
       IteratorContext iter_ctx(std::move(params));
       tf_shared_lock l(mu_);
       multi_device_buffer_->GetNextFromShard(
@@ -116,6 +121,10 @@ class MultiDeviceIterator : public ResourceBase {
     return lib_;
   }
 
+  FunctionHandleCache* function_handle_cache() {
+    return function_handle_cache_.get();
+  }
+
  private:
   // A private class that uses a background thread to keep a per device buffer
   // full.
@@ -340,6 +349,7 @@ class MultiDeviceIterator : public ResourceBase {
   const std::unique_ptr<FunctionLibraryDefinition> flib_def_;
   const std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
   FunctionLibraryRuntime* const lib_ = nullptr;  // not owned.
+  const std::unique_ptr<FunctionHandleCache> function_handle_cache_;
   std::shared_ptr<const FunctionLibraryDefinition> lib_def_ GUARDED_BY(mu_);
 
   int64 incarnation_id_ GUARDED_BY(mu_) = 0;
@@ -383,21 +393,24 @@ class MultiDeviceIteratorHandleOp : public OpKernel {
         std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
         OP_REQUIRES_OK(context, context->function_library()->Clone(
                                     &flib_def, &pflr, &lib));
+        std::unique_ptr<FunctionHandleCache> function_handle_cache(
+            new FunctionHandleCache(lib));
         ResourceMgr* mgr = context->resource_manager();
         OP_REQUIRES_OK(context, cinfo_.Init(mgr, def()));
 
         MultiDeviceIterator* resource;
-        OP_REQUIRES_OK(
-            context,
-            mgr->LookupOrCreate<MultiDeviceIterator>(
-                cinfo_.container(), cinfo_.name(), &resource,
-                [this, lib, &flib_def, &pflr](MultiDeviceIterator** ret)
-                    EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-                      *ret = new MultiDeviceIterator(
-                          output_types_, output_shapes_, devices_,
-                          std::move(flib_def), std::move(pflr), lib);
-                      return Status::OK();
-                    }));
+        OP_REQUIRES_OK(context,
+                       mgr->LookupOrCreate<MultiDeviceIterator>(
+                           cinfo_.container(), cinfo_.name(), &resource,
+                           [this, lib, &flib_def, &pflr,
+                            &function_handle_cache](MultiDeviceIterator** ret)
+                               EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+                                 *ret = new MultiDeviceIterator(
+                                     output_types_, output_shapes_, devices_,
+                                     std::move(flib_def), std::move(pflr), lib,
+                                     std::move(function_handle_cache));
+                                 return Status::OK();
+                               }));
 
         Status s = VerifyResource(resource);
         if (TF_PREDICT_FALSE(!s.ok())) {
@@ -463,6 +476,7 @@ class MultiDeviceIteratorInitOp : public OpKernel {
     std::unique_ptr<IteratorBase> iterator;
     IteratorContext::Params params(ctx);
     params.lib = resource->lib();
+    params.function_handle_cache = resource->function_handle_cache();
     IteratorContext iter_ctx(std::move(params));
     OP_REQUIRES_OK(
         ctx, dataset->MakeIterator(std::move(iter_ctx), "Iterator", &iterator));
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index f5bb35d360..cac6c43565 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/device_base.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/graph/graph_constructor.h"
@@ -124,6 +125,9 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       TF_RETURN_IF_ERROR(
           ctx->function_library()->Clone(&flib_def_, &pflr_, &lib_));
 
+      // Create a FunctionHandleCache.
+      function_handle_cache_.reset(new FunctionHandleCache(lib_));
+
       // Some functions may have been modified without having their names
       // changed (for example, nested dataset graphs from FlatMap or
       // Interleave). To avoid name conflicts, we remove these functions from
@@ -176,6 +180,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         IteratorContext::Params params(ctx);
         params.lib = dataset()->lib_;
+        params.function_handle_cache = dataset()->function_handle_cache_.get();
         return dataset()->optimized_input_->MakeIterator(
             IteratorContext(std::move(params)), prefix(), &input_impl_);
       }
@@ -185,6 +190,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
                              bool* end_of_sequence) override {
         IteratorContext::Params params(ctx);
         params.lib = dataset()->lib_;
+        params.function_handle_cache = dataset()->function_handle_cache_.get();
         return input_impl_->GetNext(IteratorContext(std::move(params)),
                                     out_tensors, end_of_sequence);
       }
@@ -284,6 +290,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
     FunctionLibraryRuntime* lib_ = nullptr;
     std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_ = nullptr;
     std::unique_ptr<FunctionLibraryDefinition> flib_def_ = nullptr;
+    std::unique_ptr<FunctionHandleCache> function_handle_cache_ = nullptr;
     const DatasetBase* input_;
     const std::vector<string> optimizations_;
     const DataTypeVector output_types_;
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 23e6adc57a..b5a1679082 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -249,7 +249,8 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       // It is implemented so that it matches the deterministic interleave
@@ -693,7 +694,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
               worker_thread_states_[thread_index].iterator_creation_status =
                   MakeIteratorFromInputElement(
                       ctx.get(), worker_thread_states_[thread_index].input,
-                      thread_index, dataset()->captured_func_.get(), prefix(),
+                      thread_index, *instantiated_captured_func_, prefix(),
                       &worker_thread_states_[thread_index].iterator);
               iterator_creation_status =
                   worker_thread_states_[thread_index].iterator_creation_status;
@@ -927,7 +928,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
           std::unique_ptr<IteratorBase> iterator;
           Status s = MakeIteratorFromInputElement(
               ctx, worker_thread_states_[index].input, index,
-              dataset()->captured_func_.get(), prefix(), &iterator);
+              *instantiated_captured_func_, prefix(), &iterator);
           TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, iterator));
           worker_thread_states_[index].iterator.swap(iterator);
         }
@@ -1032,6 +1033,8 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       // input_impl_ is reset when we have exhausted its input.
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
 
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+
       // The WorkerState structs the worker threads operate on.
       // workers_ elements are in at most one of interleave_ and staging_.
       std::vector<WorkerState> workers_ GUARDED_BY(mu_);
@@ -1268,7 +1271,8 @@ class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
         }
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -1511,7 +1515,7 @@ class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
               if (!end_of_input_) {
                 Status status = MakeIteratorFromInputElement(
                     ctx.get(), args_list_[cycle_index_], cycle_index_,
-                    dataset()->captured_func_.get(), prefix(),
+                    *instantiated_captured_func_, prefix(),
                     &current_elements_[cycle_index_]);
                 if (!status.ok()) {
                   invocation_results_.emplace_back(new InvocationResult());
@@ -1658,7 +1662,7 @@ class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
                   &args_list_[idx][i]));
             }
             TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
-                ctx, args_list_[idx], idx, dataset()->captured_func_.get(),
+                ctx, args_list_[idx], idx, *instantiated_captured_func_.get(),
                 prefix(), &current_elements_[idx]));
             TF_RETURN_IF_ERROR(
                 RestoreInput(ctx, reader, current_elements_[idx]));
@@ -1722,6 +1726,7 @@ class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
       // Identifies whether background activity should be cancelled.
       bool cancelled_ GUARDED_BY(*mu_) = false;
       string prefix_end_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index 6e4005ff6d..6fe582c9ae 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -61,52 +61,9 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     std::vector<int> indices;
     OP_REQUIRES_OK(ctx, ComputeShortCircuitIndices(ctx, func_, &indices));
 
-    ParallelMapIteratorFunction map_func;
-    CapturedFunction* raw_captured_func = captured_func.get();
-    if (indices.empty()) {
-      map_func = [raw_captured_func](IteratorContext* ctx, const string& prefix,
-                                     std::vector<Tensor> args,
-                                     std::vector<Tensor>* out_tensors,
-                                     StatusCallback done) {
-        raw_captured_func->RunAsync(ctx, std::move(args), out_tensors,
-                                    std::move(done), prefix);
-      };
-      if (!use_inter_op_parallelism_) {
-        map_func = [map_func](IteratorContext* ctx, const string& prefix,
-                              std::vector<Tensor> args,
-                              std::vector<Tensor>* out_tensors,
-                              StatusCallback done) {
-          (*ctx->runner())(std::bind(map_func, ctx, prefix, std::move(args),
-                                     out_tensors, std::move(done)));
-        };
-      }
-    } else {
-      std::vector<bool> can_move = ComputeMoveVector(indices);
-      map_func = [raw_captured_func, indices, can_move](
-                     IteratorContext* ctx, const string& prefix,
-                     std::vector<Tensor> args, std::vector<Tensor>* out_tensors,
-                     StatusCallback done) {
-        const std::vector<Tensor>& captured_inputs =
-            raw_captured_func->captured_inputs();
-        size_t num_args = args.size();
-        for (size_t i = 0; i < indices.size(); ++i) {
-          if (indices[i] < num_args) {
-            if (can_move[i]) {
-              out_tensors->push_back(std::move(args[indices[i]]));
-            } else {
-              out_tensors->push_back(args[indices[i]]);
-            }
-          } else {
-            out_tensors->push_back(captured_inputs[indices[i] - num_args]);
-          }
-        }
-        done(Status::OK());
-      };
-    }
-
     *output = new Dataset(ctx, input, func_, num_parallel_calls, output_types_,
                           output_shapes_, use_inter_op_parallelism_, sloppy_,
-                          std::move(captured_func), std::move(map_func));
+                          std::move(captured_func), indices);
   }
 
  private:
@@ -118,7 +75,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
             const std::vector<PartialTensorShape>& output_shapes,
             bool use_inter_op_parallelism, bool sloppy,
             std::unique_ptr<CapturedFunction> captured_func,
-            ParallelMapIteratorFunction map_func)
+            const std::vector<int> indices)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
@@ -128,7 +85,9 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
           use_inter_op_parallelism_(use_inter_op_parallelism),
           sloppy_(sloppy),
           captured_func_(std::move(captured_func)),
-          map_func_(std::move(map_func)) {
+          indices_(indices),
+          can_move_(indices.empty() ? std::vector<bool>()
+                                    : ComputeMoveVector(indices)) {
       input_->Ref();
     }
 
@@ -136,13 +95,15 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      auto init_func = [this](IteratorContext* ctx) {
-        return captured_func_->Instantiate(ctx);
-      };
-
+      std::unique_ptr<ParallelMapFunctor> parallel_map_functor(nullptr);
+      if (indices_.empty()) {
+        parallel_map_functor.reset(new ParallelMapDatasetFunctor(this));
+      } else {
+        parallel_map_functor.reset(new ShortCircuitFunctor(this));
+      }
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParallelMap")}, input_,
-          std::move(init_func), map_func_, num_parallel_calls_, sloppy_);
+          std::move(parallel_map_functor), num_parallel_calls_, sloppy_);
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -215,6 +176,71 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     }
 
    private:
+    class ShortCircuitFunctor : public ParallelMapFunctor {
+     public:
+      explicit ShortCircuitFunctor(const Dataset* dataset)
+          : dataset_(dataset) {}
+
+      void MapFunc(IteratorContext* ctx, const string& prefix,
+                   std::vector<Tensor> input_element,
+                   std::vector<Tensor>* result, StatusCallback done) override {
+        const std::vector<Tensor>& captured_inputs =
+            dataset_->captured_func_->captured_inputs();
+        size_t num_args = input_element.size();
+        for (size_t i = 0; i < dataset_->indices_.size(); ++i) {
+          if (dataset_->indices_[i] < num_args) {
+            if (dataset_->can_move_[i]) {
+              result->push_back(
+                  std::move(input_element[dataset_->indices_[i]]));
+            } else {
+              result->push_back(input_element[dataset_->indices_[i]]);
+            }
+          } else {
+            result->push_back(
+                captured_inputs[dataset_->indices_[i] - num_args]);
+          }
+        }
+        done(Status::OK());
+      }
+
+      const Dataset* const dataset_;
+    };
+
+    class ParallelMapDatasetFunctor : public ParallelMapFunctor {
+     public:
+      explicit ParallelMapDatasetFunctor(const Dataset* dataset)
+          : dataset_(dataset) {}
+
+      Status InitFunc(IteratorContext* ctx) override {
+        return dataset_->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
+      }
+
+      void MapFunc(IteratorContext* ctx, const string& prefix,
+                   std::vector<Tensor> input_element,
+                   std::vector<Tensor>* result, StatusCallback done) override {
+        auto map_func = [this](IteratorContext* ctx, const string& prefix,
+                               std::vector<Tensor> input_element,
+                               std::vector<Tensor>* result,
+                               StatusCallback done) {
+          instantiated_captured_func_->RunAsync(
+              ctx, std::move(input_element), result, std::move(done), prefix);
+        };
+        if (!dataset_->use_inter_op_parallelism_) {
+          (*ctx->runner())(std::bind(map_func, ctx, prefix,
+                                     std::move(input_element), result,
+                                     std::move(done)));
+        } else {
+          map_func(ctx, prefix, std::move(input_element), result,
+                   std::move(done));
+        }
+      }
+
+     private:
+      const Dataset* const dataset_;
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+    };
+
     const DatasetBase* const input_;
     const NameAttrList func_;
     const int32 num_parallel_calls_;
@@ -223,7 +249,8 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     const bool use_inter_op_parallelism_;
     const bool sloppy_;
     const std::unique_ptr<CapturedFunction> captured_func_;
-    const ParallelMapIteratorFunction map_func_;
+    const std::vector<int> indices_;
+    const std::vector<bool> can_move_;
   };
 
   DataTypeVector output_types_;
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index 5d6c12ede9..02ccf6b004 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -34,13 +34,11 @@ class ParallelMapIterator : public DatasetBaseIterator {
  public:
   ParallelMapIterator(const typename DatasetBaseIterator::BaseParams& params,
                       const DatasetBase* input_dataset,
-                      std::function<Status(IteratorContext*)> init_func,
-                      ParallelMapIteratorFunction map_func,
+                      std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
                       int32 num_parallel_calls, bool sloppy)
       : DatasetBaseIterator(params),
         input_dataset_(input_dataset),
-        init_func_(std::move(init_func)),
-        map_func_(std::move(map_func)),
+        parallel_map_functor_(std::move(parallel_map_functor)),
         mu_(std::make_shared<mutex>()),
         cond_var_(std::make_shared<condition_variable>()),
         num_parallel_calls_(std::make_shared<model::SharedState>(
@@ -70,10 +68,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
     }
     TF_RETURN_IF_ERROR(
         input_dataset_->MakeIterator(ctx, prefix(), &input_impl_));
-    if (init_func_) {
-      TF_RETURN_IF_ERROR(init_func_(ctx));
-    }
-    return Status::OK();
+    return parallel_map_functor_->InitFunc(ctx);
   }
 
   Status GetNextInternal(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
@@ -225,8 +220,9 @@ class ParallelMapIterator : public DatasetBaseIterator {
 
     // Apply the map function on `input_element`, storing the result in
     // `result->return_values`, and invoking `done` when finished.
-    map_func_(ctx.get(), prefix(), std::move(input_element),
-              &result->return_values, std::move(done));
+    parallel_map_functor_->MapFunc(ctx.get(), prefix(),
+                                   std::move(input_element),
+                                   &result->return_values, std::move(done));
   }
 
   Status ProcessResult(const std::shared_ptr<InvocationResult>& result,
@@ -360,8 +356,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
   }
 
   const DatasetBase* const input_dataset_;  // Not owned.
-  const std::function<Status(IteratorContext*)> init_func_;
-  const ParallelMapIteratorFunction map_func_;
+  std::unique_ptr<ParallelMapFunctor> parallel_map_functor_;
   // Used for coordination between the main thread and the runner thread.
   const std::shared_ptr<mutex> mu_;
   // Used for coordination between the main thread and the runner thread. In
@@ -390,12 +385,11 @@ class ParallelMapIterator : public DatasetBaseIterator {
 std::unique_ptr<IteratorBase> NewParallelMapIterator(
     const DatasetBaseIterator::BaseParams& params,
     const DatasetBase* input_dataset,
-    std::function<Status(IteratorContext*)> init_func,
-    ParallelMapIteratorFunction map_func, int32 num_parallel_calls,
-    bool sloppy) {
-  return MakeUnique<ParallelMapIterator>(
-      params, input_dataset, std::move(init_func), std::move(map_func),
-      num_parallel_calls, sloppy);
+    std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
+    int32 num_parallel_calls, bool sloppy) {
+  return MakeUnique<ParallelMapIterator>(params, input_dataset,
+                                         std::move(parallel_map_functor),
+                                         num_parallel_calls, sloppy);
 }
 
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.h b/tensorflow/core/kernels/data/parallel_map_iterator.h
index d715b9a497..08c16a6c11 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.h
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.h
@@ -22,28 +22,33 @@ limitations under the License.
 namespace tensorflow {
 namespace data {
 
-// A function that transforms elements of one dataset into another
-// asynchronously. The arguments are:
-// 1. An `IteratorContext*` for the context in which the function should
-// execute.
-// 2. A `std::vector<Tensor>` containing the input element.
-// 3. A `std::vector<Tensor>*` to which the function will write the result.
-// 4. A `StatusCallback` that should be invoked when the function is complete.
-using ParallelMapIteratorFunction =
-    std::function<void(IteratorContext*, const string&, std::vector<Tensor>,
-                       std::vector<Tensor>*, StatusCallback)>;
-
-// Returns a new iterator that applies `map_func` to the elements of
-// `input_dataset` using the given degree of parallelism. `init_func` (if
-// specified) will be executed when the iterator is initialized (see
-// `IteratorBase::Initialize()`) and enables the user to specify error checking
-// logic that can fail early.
+class ParallelMapFunctor {
+ public:
+  virtual ~ParallelMapFunctor() {}
+
+  // A function that runs when the Iterator is initialized. It enables the user
+  // to specify error checking logic that can fail early.
+  virtual Status InitFunc(IteratorContext* ctx) { return Status::OK(); }
+
+  // A function that transforms elements of one dataset into another
+  // asynchronously. The arguments are:
+  // 1. An `IteratorContext*` for the context in which the function should
+  // execute.
+  // 2. A `std::vector<Tensor>` containing the input element.
+  // 3. A `std::vector<Tensor>*` to which the function will write the result.
+  // 4. A `StatusCallback` that should be invoked when the function is complete.
+  virtual void MapFunc(IteratorContext* ctx, const string& prefix,
+                       std::vector<Tensor> input, std::vector<Tensor>* output,
+                       StatusCallback callback) = 0;
+};
+
+// Returns a new iterator that uses `parallel_map_functor` to apply `MapFunc`
+// to the elements of `input_dataset` using the given degree of parallelism.
 std::unique_ptr<IteratorBase> NewParallelMapIterator(
     const DatasetBaseIterator::BaseParams& params,
     const DatasetBase* input_dataset,
-    std::function<Status(IteratorContext*)> init_func,
-    ParallelMapIteratorFunction map_func, int32 num_parallel_calls,
-    bool sloppy);
+    std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
+    int32 num_parallel_calls, bool sloppy);
 
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/parse_example_dataset_op.cc
index 608b39d5f5..c4ab4b28df 100644
--- a/tensorflow/core/kernels/data/parse_example_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parse_example_dataset_op.cc
@@ -183,99 +183,11 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      auto map_fn = [this](IteratorContext* ctx, const string& prefix,
-                           std::vector<Tensor> input_element,
-                           std::vector<Tensor>* result, StatusCallback done) {
-        (*ctx->runner())([this, ctx, input_element, result, done]() {
-          thread::ThreadPool* device_threadpool =
-              ctx->lib()->device()->tensorflow_cpu_worker_threads()->workers;
-          std::vector<string> slice_vec;
-          for (const Tensor& t : input_element) {
-            auto serialized_t = t.flat<string>();
-            gtl::ArraySlice<string> slice(serialized_t.data(),
-                                          serialized_t.size());
-            for (auto it = slice.begin(); it != slice.end(); it++)
-              slice_vec.push_back(*it);
-          }
-          example::FastParseExampleConfig config = config_;
-          // local copy of config_ for modification.
-          auto stats_aggregator = ctx->stats_aggregator();
-          if (stats_aggregator) {
-            config.collect_feature_stats = true;
-          }
-          example::Result example_result;
-          Status s = FastParseExample(config, slice_vec, {}, device_threadpool,
-                                      &example_result);
-          if (s.ok()) {
-            (*result).resize(key_to_output_index_.size());
-            for (int d = 0; d < dense_keys_.size(); ++d) {
-              int output_index = key_to_output_index_.at(dense_keys_[d]);
-              CHECK(example_result.dense_values[d].dtype() ==
-                    output_dtypes()[output_index])
-                  << "Got wrong type for FastParseExample return value " << d
-                  << " (expected "
-                  << DataTypeString(output_dtypes()[output_index]) << ", got "
-                  << DataTypeString(example_result.dense_values[d].dtype())
-                  << ").";
-              CHECK(output_shapes()[output_index].IsCompatibleWith(
-                  example_result.dense_values[d].shape()))
-                  << "Got wrong shape for FastParseExample return value " << d
-                  << " (expected "
-                  << output_shapes()[output_index].DebugString() << ", got "
-                  << example_result.dense_values[d].shape().DebugString()
-                  << ").";
-              (*result)[output_index] = example_result.dense_values[d];
-            }
-            for (int d = 0; d < sparse_keys_.size(); ++d) {
-              int output_index = key_to_output_index_.at(sparse_keys_[d]);
-              (*result)[output_index] =
-                  Tensor(ctx->allocator({}), DT_VARIANT, {3});
-              Tensor& serialized_sparse = (*result)[output_index];
-              auto serialized_sparse_t = serialized_sparse.vec<Variant>();
-              serialized_sparse_t(0) = example_result.sparse_indices[d];
-              serialized_sparse_t(1) = example_result.sparse_values[d];
-              serialized_sparse_t(2) = example_result.sparse_shapes[d];
-              CHECK(serialized_sparse.dtype() == output_dtypes()[output_index])
-                  << "Got wrong type for FastParseExample return value " << d
-                  << " (expected "
-                  << DataTypeString(output_dtypes()[output_index]) << ", got "
-                  << DataTypeString(serialized_sparse.dtype()) << ").";
-              CHECK(output_shapes()[output_index].IsCompatibleWith(
-                  serialized_sparse.shape()))
-                  << "Got wrong shape for FastParseExample return value " << d
-                  << " (expected "
-                  << output_shapes()[output_index].DebugString() << ", got "
-                  << serialized_sparse.shape().DebugString() << ").";
-            }
-            // TODO(b/111553342): User provided tags instead of fixed tag.
-            if (stats_aggregator) {
-              stats_aggregator->IncrementCounter(
-                  "examples_count", "trainer",
-                  example_result.feature_stats.size());
-              for (example::PerExampleFeatureStats feature_stats :
-                   example_result.feature_stats) {
-                stats_aggregator->AddToHistogram(
-                    "features",
-                    {static_cast<double>(feature_stats.features_count)});
-                stats_aggregator->IncrementCounter(
-                    "features_count", "trainer", feature_stats.features_count);
-                stats_aggregator->IncrementCounter(
-                    "feature_values_count", "trainer",
-                    feature_stats.feature_values_count);
-                stats_aggregator->AddToHistogram(
-                    "feature-values",
-                    {static_cast<double>(feature_stats.feature_values_count)});
-              }
-            }
-          }
-          done(s);
-        });
-      };
-
+      std::unique_ptr<ParallelMapFunctor> parse_example_functor(
+          new ParseExampleFunctor(this));
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParseExample")}, input_,
-          /*init_func=*/nullptr, std::move(map_fn), num_parallel_calls_,
-          sloppy_);
+          std::move(parse_example_functor), num_parallel_calls_, sloppy_);
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -341,6 +253,111 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
     }
 
    private:
+    class ParseExampleFunctor : public ParallelMapFunctor {
+     public:
+      explicit ParseExampleFunctor(const Dataset* dataset)
+          : dataset_(dataset) {}
+
+      void MapFunc(IteratorContext* ctx, const string& prefix,
+                   std::vector<Tensor> input, std::vector<Tensor>* output,
+                   StatusCallback callback) override {
+        (*ctx->runner())([this, ctx, input, output, callback]() {
+          thread::ThreadPool* device_threadpool =
+              ctx->lib()->device()->tensorflow_cpu_worker_threads()->workers;
+          std::vector<string> slice_vec;
+          for (const Tensor& t : input) {
+            auto serialized_t = t.flat<string>();
+            gtl::ArraySlice<string> slice(serialized_t.data(),
+                                          serialized_t.size());
+            for (auto it = slice.begin(); it != slice.end(); it++)
+              slice_vec.push_back(*it);
+          }
+          example::FastParseExampleConfig config = dataset_->config_;
+          // local copy of config_ for modification.
+          auto stats_aggregator = ctx->stats_aggregator();
+          if (stats_aggregator) {
+            config.collect_feature_stats = true;
+          }
+          example::Result example_result;
+          Status s = FastParseExample(config, slice_vec, {}, device_threadpool,
+                                      &example_result);
+          if (s.ok()) {
+            (*output).resize(dataset_->key_to_output_index_.size());
+            for (int d = 0; d < dataset_->dense_keys_.size(); ++d) {
+              int output_index =
+                  dataset_->key_to_output_index_.at(dataset_->dense_keys_[d]);
+              DCHECK(example_result.dense_values[d].dtype() ==
+                     dataset_->output_dtypes()[output_index])
+                  << "Got wrong type for FastParseExample return value " << d
+                  << " (expected "
+                  << DataTypeString(dataset_->output_dtypes()[output_index])
+                  << ", got "
+                  << DataTypeString(example_result.dense_values[d].dtype())
+                  << ").";
+              DCHECK(dataset_->output_shapes()[output_index].IsCompatibleWith(
+                  example_result.dense_values[d].shape()))
+                  << "Got wrong shape for FastParseExample return value " << d
+                  << " (expected "
+                  << dataset_->output_shapes()[output_index].DebugString()
+                  << ", got "
+                  << example_result.dense_values[d].shape().DebugString()
+                  << ").";
+              (*output)[output_index] = example_result.dense_values[d];
+            }
+            for (int d = 0; d < dataset_->sparse_keys_.size(); ++d) {
+              int output_index =
+                  dataset_->key_to_output_index_.at(dataset_->sparse_keys_[d]);
+              (*output)[output_index] =
+                  Tensor(ctx->allocator({}), DT_VARIANT, {3});
+              Tensor& serialized_sparse = (*output)[output_index];
+              auto serialized_sparse_t = serialized_sparse.vec<Variant>();
+              serialized_sparse_t(0) = example_result.sparse_indices[d];
+              serialized_sparse_t(1) = example_result.sparse_values[d];
+              serialized_sparse_t(2) = example_result.sparse_shapes[d];
+              DCHECK(serialized_sparse.dtype() ==
+                     dataset_->output_dtypes()[output_index])
+                  << "Got wrong type for FastParseExample return value " << d
+                  << " (expected "
+                  << DataTypeString(dataset_->output_dtypes()[output_index])
+                  << ", got " << DataTypeString(serialized_sparse.dtype())
+                  << ").";
+              DCHECK(dataset_->output_shapes()[output_index].IsCompatibleWith(
+                  serialized_sparse.shape()))
+                  << "Got wrong shape for FastParseExample return value " << d
+                  << " (expected "
+                  << dataset_->output_shapes()[output_index].DebugString()
+                  << ", got " << serialized_sparse.shape().DebugString()
+                  << ").";
+            }
+            // TODO(b/111553342): User provided tags instead of fixed tag.
+            if (stats_aggregator) {
+              stats_aggregator->IncrementCounter(
+                  "examples_count", "trainer",
+                  example_result.feature_stats.size());
+              for (example::PerExampleFeatureStats feature_stats :
+                   example_result.feature_stats) {
+                stats_aggregator->AddToHistogram(
+                    "features",
+                    {static_cast<double>(feature_stats.features_count)});
+                stats_aggregator->IncrementCounter(
+                    "features_count", "trainer", feature_stats.features_count);
+                stats_aggregator->IncrementCounter(
+                    "feature_values_count", "trainer",
+                    feature_stats.feature_values_count);
+                stats_aggregator->AddToHistogram(
+                    "feature-values",
+                    {static_cast<double>(feature_stats.feature_values_count)});
+              }
+            }
+          }
+          callback(s);
+        });
+      }
+
+     private:
+      const Dataset* dataset_;
+    };
+
     const DatasetBase* const input_;
     const std::vector<Tensor> dense_defaults_;
     const std::vector<string> sparse_keys_;
diff --git a/tensorflow/core/kernels/data/scan_dataset_op.cc b/tensorflow/core/kernels/data/scan_dataset_op.cc
index d9182d15be..12d07f1919 100644
--- a/tensorflow/core/kernels/data/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/scan_dataset_op.cc
@@ -143,7 +143,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
             dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(ctx);
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -168,8 +169,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
         state_and_output.reserve(dataset()->state_types_.size() +
                                  output_dtypes().size());
 
-        Status s = dataset()->captured_func_->Run(ctx, std::move(args),
-                                                  &state_and_output);
+        Status s = instantiated_captured_func_->Run(ctx, std::move(args),
+                                                    &state_and_output);
         if (s.ok()) {
           state_.clear();
           size_t i = 0;
@@ -252,6 +253,7 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
       mutex mu_;
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
       std::vector<Tensor> state_ GUARDED_BY(mu_);
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
     const DatasetBase* const input_;
diff --git a/tensorflow/core/kernels/data/writer_ops.cc b/tensorflow/core/kernels/data/writer_ops.cc
index 66e759a135..c00089d06f 100644
--- a/tensorflow/core/kernels/data/writer_ops.cc
+++ b/tensorflow/core/kernels/data/writer_ops.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/kernels/ops_util.h"
@@ -67,20 +68,24 @@ class ToTFRecordOp : public AsyncOpKernel {
       OP_REQUIRES_OK_ASYNC(
           ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done);
       std::unique_ptr<IteratorBase> iterator;
+      IteratorContext::Params params(ctx);
+      std::unique_ptr<FunctionHandleCache> function_handle_cache(
+          new FunctionHandleCache(params.lib));
+      params.function_handle_cache = function_handle_cache.get();
+      IteratorContext iter_ctx(std::move(params));
+
       OP_REQUIRES_OK_ASYNC(
           ctx,
-          dataset->MakeIterator(IteratorContext(ctx), "ToTFRecordOpIterator",
-                                &iterator),
+          dataset->MakeIterator(&iter_ctx, "ToTFRecordOpIterator", &iterator),
           done);
 
       std::vector<Tensor> components;
       components.reserve(dataset->output_dtypes().size());
       bool end_of_sequence;
       do {
-        OP_REQUIRES_OK_ASYNC(ctx,
-                             iterator->GetNext(IteratorContext(ctx),
-                                               &components, &end_of_sequence),
-                             done);
+        OP_REQUIRES_OK_ASYNC(
+            ctx, iterator->GetNext(&iter_ctx, &components, &end_of_sequence),
+            done);
 
         if (!end_of_sequence) {
           OP_REQUIRES_OK_ASYNC(
-- 
GitLab


From f6e8f7a1fb7c530a98ba93d40dba1f6c859a873b Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Thu, 29 Nov 2018 09:53:22 -0800
Subject: [PATCH 1020/1554] [tf.data]: Deprecate dataset.shard.

In preparation for a distributed-aware tf.data system, and TF 2.0, this CL
deprecates the legacy `.shard` method on tf.data.Dataset objects. The old
behavior is preserved under an experimental `filter_for_shard` dataset
transformation, as well as in the V1 API (albeit now marked with a deprecation
warning).

PiperOrigin-RevId: 223362539
---
 .../python/data/experimental/__init__.py      |   2 +
 tensorflow/python/data/experimental/ops/BUILD |  13 ++
 .../experimental/ops/filter_for_shard_ops.py  | 106 +++++++++++++++
 tensorflow/python/data/ops/BUILD              |   1 +
 tensorflow/python/data/ops/dataset_ops.py     | 128 ++++++++----------
 tensorflow/python/distribute/BUILD            |   1 +
 tensorflow/python/distribute/input_ops.py     |  13 +-
 .../v1/tensorflow.data.experimental.pbtxt     |   4 +
 .../golden/v2/tensorflow.data.-dataset.pbtxt  |   4 -
 ...ow.data.-fixed-length-record-dataset.pbtxt |   4 -
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   4 -
 .../tensorflow.data.-text-line-dataset.pbtxt  |   4 -
 ...rflow.data.experimental.-csv-dataset.pbtxt |   4 -
 ...ow.data.experimental.-random-dataset.pbtxt |   4 -
 ...rflow.data.experimental.-sql-dataset.pbtxt |   4 -
 .../v2/tensorflow.data.experimental.pbtxt     |   4 +
 16 files changed, 194 insertions(+), 106 deletions(-)
 create mode 100644 tensorflow/python/data/experimental/ops/filter_for_shard_ops.py

diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 12aa0a09e1..8cec75b599 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -39,6 +39,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@copy_to_device
 @@dense_to_sparse_batch
 @@enumerate_dataset
+@@filter_for_shard
 @@get_next_as_optional
 @@get_single_element
 @@group_by_reducer
@@ -74,6 +75,7 @@ from tensorflow.python.data.experimental.ops.batching import unbatch
 from tensorflow.python.data.experimental.ops.counter import Counter
 from tensorflow.python.data.experimental.ops.enumerate_ops import enumerate_dataset
 from tensorflow.python.data.experimental.ops.error_ops import ignore_errors
+from tensorflow.python.data.experimental.ops.filter_for_shard_ops import filter_for_shard
 from tensorflow.python.data.experimental.ops.get_single_element import get_single_element
 from tensorflow.python.data.experimental.ops.grouping import bucket_by_sequence_length
 from tensorflow.python.data.experimental.ops.grouping import group_by_reducer
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index f9544857a1..f85e774887 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -139,6 +139,18 @@ py_library(
     ],
 )
 
+py_library(
+    name = "filter_for_shard_ops",
+    srcs = ["filter_for_shard_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:ops",
+        "//tensorflow/python:tensor_util",
+    ],
+)
+
 py_library(
     name = "error_ops",
     srcs = ["error_ops.py"],
@@ -403,6 +415,7 @@ py_library(
         ":counter",
         ":enumerate_ops",
         ":error_ops",
+        ":filter_for_shard_ops",
         ":get_single_element",
         ":grouping",
         ":indexed_dataset_ops",
diff --git a/tensorflow/python/data/experimental/ops/filter_for_shard_ops.py b/tensorflow/python/data/experimental/ops/filter_for_shard_ops.py
new file mode 100644
index 0000000000..91d3dca3e9
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/filter_for_shard_ops.py
@@ -0,0 +1,106 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Naive shard dataset transformation."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import math_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export("data.experimental.filter_for_shard")
+def filter_for_shard(num_shards, shard_index):
+  """Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
+
+  This dataset operator is very useful when running distributed training, as
+  it allows each worker to read a unique subset.
+
+  When reading a single input file, you can skip elements as follows:
+
+  ```python
+  d = tf.data.TFRecordDataset(FLAGS.input_file)
+  d = d.apply(tf.data.experimental.naive_shard(FLAGS.num_workers,
+                                               FLAGS.worker_index))
+  d = d.repeat(FLAGS.num_epochs)
+  d = d.shuffle(FLAGS.shuffle_buffer_size)
+  d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
+  ```
+
+  Important caveats:
+
+  - Be sure to shard before you use any randomizing operator (such as
+    shuffle).
+  - Generally it is best if the shard operator is used early in the dataset
+    pipeline. For example, when reading from a set of TFRecord files, shard
+    before converting the dataset to input samples. This avoids reading every
+    file on every worker. The following is an example of an efficient
+    sharding strategy within a complete pipeline:
+
+  ```python
+  d = Dataset.list_files(FLAGS.pattern)
+  d = d.apply(tf.data.experimental.naive_shard(FLAGS.num_workers,
+                                               FLAGS.worker_index))
+  d = d.repeat(FLAGS.num_epochs)
+  d = d.shuffle(FLAGS.shuffle_buffer_size)
+  d = d.interleave(tf.data.TFRecordDataset,
+                   cycle_length=FLAGS.num_readers, block_length=1)
+  d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
+  ```
+
+  Args:
+    num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
+      shards operating in parallel.
+    shard_index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    `tf.data.Dataset.apply`.
+
+  Raises:
+    ValueError: if `num_shards` or `shard_index` are illegal values. Note: error
+      checking is done on a best-effort basis, and errors aren't guaranteed to
+      be caught upon dataset creation. (e.g. providing in a placeholder tensor
+      bypasses the early checking, and will instead result in an error during
+      a session.run call.)
+  """
+  num_shards = ops.convert_to_tensor(
+      num_shards, name="num_shards", dtype=dtypes.int64)
+  num_shards_static = tensor_util.constant_value(num_shards)
+  shard_index = ops.convert_to_tensor(shard_index, name="shard_index",
+                                      dtype=dtypes.int64)
+  shard_index_static = tensor_util.constant_value(shard_index)
+
+  if num_shards_static is not None and num_shards_static < 1:
+    raise ValueError("num_shards must be >= 1; got: %s" % num_shards_static)
+  if shard_index_static is not None and shard_index_static < 0:
+    raise ValueError("shard_index must be >= 0; got: %s" % shard_index_static)
+  if (shard_index_static is not None and num_shards_static is not None and
+      shard_index_static >= num_shards_static):
+    raise ValueError("shard_index must be < num_shards; %s is not < %s" %
+                     (shard_index_static, num_shards_static))
+
+  def filter_fn(elem_index, _):
+    mod_result = math_ops.mod(elem_index, num_shards)
+    return math_ops.equal(mod_result, shard_index)
+
+  def _apply_fn(dataset):
+    # pylint: disable=protected-access
+    return dataset._enumerate().filter(filter_fn).map(lambda _, elem: elem)
+
+  return _apply_fn
diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD
index dcbb0f1868..27c9175ccb 100644
--- a/tensorflow/python/data/ops/BUILD
+++ b/tensorflow/python/data/ops/BUILD
@@ -26,6 +26,7 @@ py_library(
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python/data/experimental/ops:filter_for_shard_ops",
         "//tensorflow/python/data/experimental/ops:stats_options",
         "//tensorflow/python/data/experimental/ops:threading_options",
         "//tensorflow/python/data/util:nest",
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 71175fc6a2..51123aaf44 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -26,6 +26,7 @@ import numpy as np
 import six
 
 from tensorflow.python.compat import compat
+from tensorflow.python.data.experimental.ops import filter_for_shard_ops
 from tensorflow.python.data.experimental.ops import stats_options
 from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.ops import iterator_ops
@@ -821,78 +822,6 @@ class DatasetV2(object):
     """
     return SkipDataset(self, count)
 
-  def shard(self, num_shards, index):
-    """Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
-
-    This dataset operator is very useful when running distributed training, as
-    it allows each worker to read a unique subset.
-
-    When reading a single input file, you can skip elements as follows:
-
-    ```python
-    d = tf.data.TFRecordDataset(FLAGS.input_file)
-    d = d.shard(FLAGS.num_workers, FLAGS.worker_index)
-    d = d.repeat(FLAGS.num_epochs)
-    d = d.shuffle(FLAGS.shuffle_buffer_size)
-    d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
-    ```
-
-    Important caveats:
-
-    - Be sure to shard before you use any randomizing operator (such as
-      shuffle).
-    - Generally it is best if the shard operator is used early in the dataset
-      pipeline. For example, when reading from a set of TFRecord files, shard
-      before converting the dataset to input samples. This avoids reading every
-      file on every worker. The following is an example of an efficient
-      sharding strategy within a complete pipeline:
-
-    ```python
-    d = Dataset.list_files(FLAGS.pattern)
-    d = d.shard(FLAGS.num_workers, FLAGS.worker_index)
-    d = d.repeat(FLAGS.num_epochs)
-    d = d.shuffle(FLAGS.shuffle_buffer_size)
-    d = d.interleave(tf.data.TFRecordDataset,
-                     cycle_length=FLAGS.num_readers, block_length=1)
-    d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
-    ```
-
-    Args:
-      num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
-        shards operating in parallel.
-      index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
-
-    Returns:
-      Dataset: A `Dataset`.
-
-    Raises:
-      ValueError: if `num_shards` or `index` are illegal values. Note: error
-        checking is done on a best-effort basis, and errors aren't guaranteed
-        to be caught upon dataset creation. (e.g. providing in a placeholder
-        tensor bypasses the early checking, and will instead result in an error
-        during a session.run call.)
-    """
-    num_shards = ops.convert_to_tensor(
-        num_shards, name="num_shards", dtype=dtypes.int64)
-    num_shards_static = tensor_util.constant_value(num_shards)
-    index = ops.convert_to_tensor(index, name="index", dtype=dtypes.int64)
-    index_static = tensor_util.constant_value(index)
-
-    if num_shards_static is not None and num_shards_static < 1:
-      raise ValueError("num_shards must be >= 1; got: %s" % num_shards_static)
-    if index_static is not None and index_static < 0:
-      raise ValueError("index must be >= 0; got: %s" % index_static)
-    if (index_static is not None and num_shards_static is not None and
-        index_static >= num_shards_static):
-      raise ValueError("index must be <= num_shards; %s is not < %s" %
-                       (index_static, num_shards_static))
-
-    def filter_fn(elem_index, _):
-      mod_result = math_ops.mod(elem_index, num_shards)
-      return math_ops.equal(mod_result, index)
-
-    return self._enumerate().filter(filter_fn).map(lambda _, elem: elem)
-
   def batch(self, batch_size, drop_remainder=False):
     """Combines consecutive elements of this dataset into batches.
 
@@ -1486,9 +1415,60 @@ class DatasetV1(DatasetV2):
   def skip(self, count):
     return DatasetV1Adapter(super(DatasetV1, self).skip(count))
 
-  @functools.wraps(DatasetV2.shard)
+  @deprecation.deprecated(
+      None, "Use `dataset.apply(tf.data.experimental.filter_for_shard(...))`.")
   def shard(self, num_shards, index):
-    return DatasetV1Adapter(super(DatasetV1, self).shard(num_shards, index))
+    """Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
+
+    This dataset operator is very useful when running distributed training, as
+    it allows each worker to read a unique subset.
+
+    When reading a single input file, you can skip elements as follows:
+
+    ```python
+    d = tf.data.TFRecordDataset(FLAGS.input_file)
+    d = d.shard(FLAGS.num_workers, FLAGS.worker_index)
+    d = d.repeat(FLAGS.num_epochs)
+    d = d.shuffle(FLAGS.shuffle_buffer_size)
+    d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
+    ```
+
+    Important caveats:
+
+    - Be sure to shard before you use any randomizing operator (such as
+      shuffle).
+    - Generally it is best if the shard operator is used early in the dataset
+      pipeline. For example, when reading from a set of TFRecord files, shard
+      before converting the dataset to input samples. This avoids reading every
+      file on every worker. The following is an example of an efficient
+      sharding strategy within a complete pipeline:
+
+    ```python
+    d = Dataset.list_files(FLAGS.pattern)
+    d = d.shard(FLAGS.num_workers, FLAGS.worker_index)
+    d = d.repeat(FLAGS.num_epochs)
+    d = d.shuffle(FLAGS.shuffle_buffer_size)
+    d = d.interleave(tf.data.TFRecordDataset,
+                     cycle_length=FLAGS.num_readers, block_length=1)
+    d = d.map(parser_fn, num_parallel_calls=FLAGS.num_map_threads)
+    ```
+
+    Args:
+      num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
+        shards operating in parallel.
+      index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
+
+    Returns:
+      Dataset: A `Dataset`.
+
+    Raises:
+      ValueError: if `num_shards` or `index` are illegal values. Note: error
+        checking is done on a best-effort basis, and errors aren't guaranteed
+        to be caught upon dataset creation. (e.g. providing in a placeholder
+        tensor bypasses the early checking, and will instead result in an error
+        during a session.run call.)
+    """
+    return self.apply(filter_for_shard_ops.filter_for_shard(num_shards, index))
 
   @functools.wraps(DatasetV2.batch)
   def batch(self, batch_size, drop_remainder=False):
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 5afbcec3a9..ec438d00f0 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -245,6 +245,7 @@ py_library(
     srcs = ["input_ops.py"],
     deps = [
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python/data/experimental/ops:filter_for_shard_ops",
         "//tensorflow/python/data/util:nest",
     ],
 )
diff --git a/tensorflow/python/distribute/input_ops.py b/tensorflow/python/distribute/input_ops.py
index c40b2bf27a..d7974942a1 100644
--- a/tensorflow/python/distribute/input_ops.py
+++ b/tensorflow/python/distribute/input_ops.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.data.experimental.ops import filter_for_shard_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.data.util import nest
@@ -41,7 +42,8 @@ def auto_shard_dataset(dataset, num_shards, index):
     dataset: A `tf.data.Dataset` instance, typically the result of a bunch of
       dataset transformations.
     num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
-        shards operating in parallel. Same usage as in `Dataset.shard`.
+        shards operating in parallel. Same usage as in
+        `tf.data.experimental.filter_for_shard`.
     index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
       Same usage as in `Dataset.shard`.
 
@@ -74,9 +76,11 @@ def auto_shard_dataset(dataset, num_shards, index):
         # constructor. Eventually we will change all cases to clone datasets
         # instead of updating in-place.
         return dataset._clone(
-            filenames=dataset._filenames.shard(num_shards, index))
+            filenames=dataset._filenames.apply(
+                filter_for_shard_ops.filter_for_shard(num_shards, index)))
       elif isinstance(dataset, dataset_ops.RangeDataset):
-        return dataset.shard(num_shards, index)
+        return dataset.apply(
+            filter_for_shard_ops.filter_for_shard(num_shards, index))
       elif hasattr(dataset, "_map_func"):
         # TODO(priyag): Make this check more robust by enforcing some common
         # property on all map/flatmap/interleave datasets.
@@ -142,6 +146,7 @@ def auto_shard_dataset(dataset, num_shards, index):
     # TODO(priyag): This will shard the filenames before any shuffling of the
     # filename dataset. It might be desirable to shard after shuffling
     # filenames? If so, how do we achieve that?
-    return dataset.shard(num_shards, index)
+    return dataset.apply(
+        filter_for_shard_ops.filter_for_shard(num_shards, index))
 
   return _auto_shard_impl(dataset=dataset, found_reader_op=False)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index 7bc3faaedc..a3cb799fc3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -68,6 +68,10 @@ tf_module {
     name: "enumerate_dataset"
     argspec: "args=[\'start\'], varargs=None, keywords=None, defaults=[\'0\'], "
   }
+  member_method {
+    name: "filter_for_shard"
+    argspec: "args=[\'num_shards\', \'shard_index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_next_as_optional"
     argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
index 9394f4b767..39a6e1ee71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
@@ -97,10 +97,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
index 8c32c773b7..ef367238d0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -100,10 +100,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
index 9f32bce109..a8fc6fbec1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -99,10 +99,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
index 0eedfdbfe1..697f371344 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
@@ -100,10 +100,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
index 08214ec3cf..17ac098910 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -100,10 +100,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
index 608253298e..f005d36e1a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -100,10 +100,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
index 3335eb1dc7..b0c0b73ad6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -100,10 +100,6 @@ tf_class {
     name: "repeat"
     argspec: "args=[\'self\', \'count\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "shard"
-    argspec: "args=[\'self\', \'num_shards\', \'index\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "shuffle"
     argspec: "args=[\'self\', \'buffer_size\', \'seed\', \'reshuffle_each_iteration\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index 7bc3faaedc..a3cb799fc3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -68,6 +68,10 @@ tf_module {
     name: "enumerate_dataset"
     argspec: "args=[\'start\'], varargs=None, keywords=None, defaults=[\'0\'], "
   }
+  member_method {
+    name: "filter_for_shard"
+    argspec: "args=[\'num_shards\', \'shard_index\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_next_as_optional"
     argspec: "args=[\'iterator\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From bfec75923018512a782fc8405de1d61857edcc94 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 29 Nov 2018 10:02:50 -0800
Subject: [PATCH 1021/1554] Test that all outputs of the converter are in the
 v2 API.

Fixes issue where piecewise_constant was a warning only despite not
being a part of the v2 API.

PiperOrigin-RevId: 223364226
---
 tensorflow/tools/compatibility/BUILD          |  3 +
 tensorflow/tools/compatibility/renames_v2.py  |  6 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  6 +-
 .../tools/compatibility/tf_upgrade_v2_test.py | 65 ++++++++++++++++++-
 4 files changed, 75 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 197fa75015..152a79f542 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -78,8 +78,11 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":tf_upgrade_v2",
+        "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/tools/common:public_api",
+        "//tensorflow/tools/common:traverse",
         "@six_archive//:six",
     ],
 )
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index ae57478473..ba93cbbfa9 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -97,6 +97,7 @@ renames = {
     'tf.check_numerics': 'tf.debugging.check_numerics',
     'tf.cholesky': 'tf.linalg.cholesky',
     'tf.cholesky_solve': 'tf.linalg.cholesky_solve',
+    'tf.clip_by_average_norm': 'tf.compat.v1.clip_by_average_norm',
     'tf.colocate_with': 'tf.compat.v1.colocate_with',
     'tf.conj': 'tf.math.conj',
     'tf.container': 'tf.compat.v1.container',
@@ -107,7 +108,6 @@ renames = {
     'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
-    'tf.data.Iterator': 'tf.compat.v1.data.Iterator',
     'tf.debugging.is_finite': 'tf.math.is_finite',
     'tf.debugging.is_inf': 'tf.math.is_inf',
     'tf.debugging.is_nan': 'tf.math.is_nan',
@@ -358,7 +358,6 @@ renames = {
     'tf.nn.depthwise_conv2d_native_backprop_filter': 'tf.nn.depthwise_conv2d_backprop_filter',
     'tf.nn.depthwise_conv2d_native_backprop_input': 'tf.nn.depthwise_conv2d_backprop_input',
     'tf.nn.dynamic_rnn': 'tf.compat.v1.nn.dynamic_rnn',
-    'tf.nn.fused_batch_norm': 'tf.compat.v1.nn.fused_batch_norm',
     'tf.nn.log_uniform_candidate_sampler': 'tf.random.log_uniform_candidate_sampler',
     'tf.nn.quantized_avg_pool': 'tf.compat.v1.nn.quantized_avg_pool',
     'tf.nn.quantized_conv2d': 'tf.compat.v1.nn.quantized_conv2d',
@@ -427,13 +426,14 @@ renames = {
     'tf.rsqrt': 'tf.math.rsqrt',
     'tf.saved_model.Builder': 'tf.compat.v1.saved_model.Builder',
     'tf.saved_model.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.LEGACY_INIT_OP_KEY',
+    'tf.saved_model.MAIN_OP_KEY': 'tf.compat.v1.saved_model.MAIN_OP_KEY',
     'tf.saved_model.TRAINING': 'tf.saved_model.TRANING',
     'tf.saved_model.build_tensor_info': 'tf.compat.v1.saved_model.build_tensor_info',
     'tf.saved_model.builder.SavedModelBuilder': 'tf.compat.v1.saved_model.builder.SavedModelBuilder',
     'tf.saved_model.constants.ASSETS_DIRECTORY': 'tf.saved_model.ASSETS_DIRECTORY',
     'tf.saved_model.constants.ASSETS_KEY': 'tf.saved_model.ASSETS_KEY',
     'tf.saved_model.constants.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.constants.LEGACY_INIT_OP_KEY',
-    'tf.saved_model.constants.MAIN_OP_KEY': 'tf.saved_model.MAIN_OP_KEY',
+    'tf.saved_model.constants.MAIN_OP_KEY': 'tf.compat.v1.saved_model.constants.MAIN_OP_KEY',
     'tf.saved_model.constants.SAVED_MODEL_FILENAME_PB': 'tf.saved_model.SAVED_MODEL_FILENAME_PB',
     'tf.saved_model.constants.SAVED_MODEL_FILENAME_PBTXT': 'tf.saved_model.SAVED_MODEL_FILENAME_PBTXT',
     'tf.saved_model.constants.SAVED_MODEL_SCHEMA_VERSION': 'tf.saved_model.SAVED_MODEL_SCHEMA_VERSION',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 242bbbc3a7..f8c1f4111f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -433,6 +433,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.math.confusion_matrix",
         "tf.decode_csv":
             "tf.io.decode_csv",
+        "tf.data.Iterator":
+            "tf.compat.v1.data.Iterator",
+        "tf.nn.fused_batch_norm":
+            "tf.compat.v1.nn.fused_batch_norm",
     }
     # pylint: enable=line-too-long
 
@@ -681,7 +685,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
                     " module if you need command line parsing.",
         "tf.train.exponential_decay":
             decay_function_comment,
-        "tf.train.piecewise_constant":
+        "tf.train.piecewise_constant_decay":
             decay_function_comment,
         "tf.train.polynomial_decay":
             decay_function_comment,
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index c490ebd480..9c334451bf 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -17,11 +17,21 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+
 import os
 import tempfile
+
 import six
+import tensorflow as tf
+# OSS TF V2 import placeholder.
+
+
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_export
+from tensorflow.tools.common import public_api
+from tensorflow.tools.common import traverse
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import tf_upgrade_v2
 
@@ -64,6 +74,51 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, "tf.math.rsqrt(tf.math.log_sigmoid(3.8))\n")
 
+  def testAllAPI(self):
+    if not hasattr(tf.compat, "v2"):
+      return
+
+    v2_symbols = set([])
+    attr_v2 = tf_export.API_ATTRS[
+        tf_export.TENSORFLOW_API_NAME].names
+
+    def symbol_collector(unused_path, unused_parent, children):
+      for child in children:
+        _, attr = tf_decorator.unwrap(child[1])
+        if not hasattr(attr, "__dict__"):
+          continue
+        api_names_v2 = attr.__dict__.get(attr_v2, [])
+        for name in api_names_v2:
+          v2_symbols.add("tf." + name)
+
+    visitor = public_api.PublicAPIVisitor(symbol_collector)
+    traverse.traverse(tf.compat.v2, visitor)
+
+    attr_v1 = (
+        tf_export.API_ATTRS_V1[tf_export.TENSORFLOW_API_NAME].names)
+
+    # Converts all symbols in the v1 namespace to the v2 namespace, raising
+    # an error if the target of the conversion is not in the v2 namespace.
+    def conversion_visitor(unused_path, unused_parent, children):
+      for child in children:
+        _, attr = tf_decorator.unwrap(child[1])
+        if not hasattr(attr, "__dict__"):
+          continue
+        api_names = attr.__dict__.get(attr_v1, [])
+        for name in api_names:
+          _, _, _, text = self._upgrade("tf." + name)
+          if (text and
+              not text.startswith("tf.compat.v1") and
+              text not in v2_symbols):
+            self.assertFalse(
+                True, "Symbol %s generated from %s not in v2 API" % (
+                    text, name))
+
+    visitor = public_api.PublicAPIVisitor(conversion_visitor)
+    visitor.do_not_descend_map["tf"].append("contrib")
+    visitor.private_map["tf.compat"] = ["v1", "v2"]
+    traverse.traverse(tf.compat.v1, visitor)
+
   def testRenameConstant(self):
     text = "tf.MONOLITHIC_BUILD\n"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
@@ -89,7 +144,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                      "tf.boolean_mask(tensor=a, mask=b, name=c, axis=d)\n")
 
   def testLearningRateDecay(self):
-    for decay in ["tf.train.exponential_decay", "tf.train.piecewise_constant",
+    for decay in ["tf.train.exponential_decay",
                   "tf.train.polynomial_decay", "tf.train.natural_exp_decay",
                   "tf.train.inverse_time_decay", "tf.train.cosine_decay",
                   "tf.train.cosine_decay_restarts",
@@ -101,6 +156,14 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       self.assertEqual(errors, ["test.py:1: %s requires manual check." % decay])
       self.assertIn("%s has been changed" % decay, report)
 
+  def testPiecewiseDecay(self):
+    text = "tf.train.piecewise_constant_decay(a, b)\n"
+    _, report, errors, _ = self._upgrade(text)
+    self.assertEqual(
+        errors,
+        ["test.py:1: tf.train.piecewise_constant_decay requires manual check."])
+    self.assertIn("tf.train.piecewise_constant_decay has been changed", report)
+
   def testEstimatorLossReductionChange(self):
     classes = [
         "LinearClassifier", "LinearRegressor", "DNNLinearCombinedClassifier",
-- 
GitLab


From 51d694e20b5593d98514878af859a34bb39454b2 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 29 Nov 2018 10:16:10 -0800
Subject: [PATCH 1022/1554] Add publicly available corpus for string_split_v2
 fuzz.

PiperOrigin-RevId: 223366651
---
 tensorflow/core/kernels/fuzzing/BUILD              |   4 ++++
 .../00fd47bf73afcb72e7ed51bffd5f5fec               |   1 +
 .../14908973e6720513a5f37676cb9fcc29               |   1 +
 .../2779ba7c4d23eee9f79efa3660084c5d               |   1 +
 .../5bf16424630b5afbcffe711fb9834440               |   1 +
 .../89734a96b93275e495a9498b806fafe1               |   1 +
 .../d5606def44fdbb9385dd764612069db0               | Bin 0 -> 42 bytes
 7 files changed, 9 insertions(+)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/string_split_v2/d5606def44fdbb9385dd764612069db0

diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index 193041241c..7d59beb09b 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -28,8 +28,12 @@ tf_oss_fuzz_corpus("string_to_number")
 
 tf_ops_fuzz_target_lib("string_split")
 
+tf_oss_fuzz_corpus("string_split")
+
 tf_ops_fuzz_target_lib("string_split_v2")
 
+tf_oss_fuzz_corpus("string_split_v2")
+
 tf_ops_fuzz_target_lib("encode_base64")
 
 tf_ops_fuzz_target_lib("decode_base64")
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec
new file mode 100644
index 0000000000..f1410e184b
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/00fd47bf73afcb72e7ed51bffd5f5fec
@@ -0,0 +1 @@
+./abc./de./fg./ha./bc./de./fg./ha./bc./de./
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29
new file mode 100644
index 0000000000..e118d2d351
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/14908973e6720513a5f37676cb9fcc29
@@ -0,0 +1 @@
+./, abcde./, fg./, ha./, bc./, de./, fg./, ha
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d
new file mode 100644
index 0000000000..9a6c809197
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/2779ba7c4d23eee9f79efa3660084c5d
@@ -0,0 +1 @@
+./,abcd./,ef./,gh./,ab./,cd./,ef./,gh./,ab./
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440
new file mode 100644
index 0000000000..4cd522da7b
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/5bf16424630b5afbcffe711fb9834440
@@ -0,0 +1 @@
+
.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd.ef.gh.ab.cd
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1
new file mode 100644
index 0000000000..5301a91d8e
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/89734a96b93275e495a9498b806fafe1
@@ -0,0 +1 @@
+./, ?abcdef./, ?gh./, ?ab./, ?cd./, ?ef./, ?gh
\ No newline at end of file
diff --git a/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/d5606def44fdbb9385dd764612069db0 b/tensorflow/core/kernels/fuzzing/corpus/string_split_v2/d5606def44fdbb9385dd764612069db0
new file mode 100644
index 0000000000000000000000000000000000000000..304b0d66fe08fd1a29827488727702dd9b9bce3e
GIT binary patch
literal 42
ScmZQbOiE5kO-s)pNCN;-W)Kzt

literal 0
HcmV?d00001

-- 
GitLab


From d2253ab51816bd3c536a7574fff5e6b705a801a8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 10:16:39 -0800
Subject: [PATCH 1023/1554] Adding more checks for categorical splits.

PiperOrigin-RevId: 223366746
---
 .../boosted_trees/kernels/split_handler_ops.cc        | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc b/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc
index 8edb5d6c64..6d78e27e8f 100644
--- a/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/split_handler_ops.cc
@@ -834,8 +834,13 @@ class BuildCategoricalEqualitySplitsOp : public OpKernel {
       root_gradient_stats *= normalizer_ratio;
       NodeStats root_stats = state->ComputeNodeStats(root_gradient_stats);
       int32 best_feature_idx = 0;
+      bool best_feature_updated = false;
       NodeStats best_right_node_stats(0);
       NodeStats best_left_node_stats(0);
+      CHECK(end_index - start_index >= 2)
+          << "Partition should have a non bias feature. Start index "
+          << start_index << " and end index " << end_index;
+
       for (int64 feature_idx = start_index + 1; feature_idx < end_index;
            ++feature_idx) {
         GradientStats left_gradient_stats(*gradients_t, *hessians_t,
@@ -845,11 +850,13 @@ class BuildCategoricalEqualitySplitsOp : public OpKernel {
             root_gradient_stats - left_gradient_stats;
         NodeStats left_stats = state->ComputeNodeStats(left_gradient_stats);
         NodeStats right_stats = state->ComputeNodeStats(right_gradient_stats);
-        if (left_stats.gain + right_stats.gain > best_gain) {
+        if (!best_feature_updated ||
+            left_stats.gain + right_stats.gain > best_gain) {
           best_gain = left_stats.gain + right_stats.gain;
           best_left_node_stats = left_stats;
           best_right_node_stats = right_stats;
           best_feature_idx = feature_idx;
+          best_feature_updated = true;
         }
       }
       SplitInfo split_info;
@@ -864,7 +871,7 @@ class BuildCategoricalEqualitySplitsOp : public OpKernel {
           << feature_ids(best_feature_idx, 0) << ", "
           << feature_ids(best_feature_idx, 1)
           << "\nPartition IDS: " << partition_ids(start_index) << "  "
-          << partition_ids(best_feature_idx);
+          << partition_ids(best_feature_idx) << " and best gain " << best_gain;
       equality_split->set_feature_id(feature_ids(best_feature_idx, 0));
       auto* left_child = split_info.mutable_left_child();
       auto* right_child = split_info.mutable_right_child();
-- 
GitLab


From 2b559a9a086f7e8e79557c642c6d4f5115f855c5 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 29 Nov 2018 10:21:35 -0800
Subject: [PATCH 1024/1554] Improves constant shape inference for resource
 variables.

PiperOrigin-RevId: 223367586
---
 .../kernel_tests/resource_variable_ops_test.py     |  9 +++++++++
 tensorflow/python/ops/resource_variable_ops.py     | 14 +++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 30563092c8..d30ab6a9c2 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -137,6 +138,14 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.evaluate(v[0].assign(2.0))
     self.assertAllEqual(self.evaluate(v), [2.0, 2.0])
 
+  @test_util.run_in_graph_and_eager_modes
+  def testVariableShape(self):
+    v = resource_variable_ops.ResourceVariable([1., 1.])
+    self.assertAllEqual(
+        tensor_util.constant_value(
+            resource_variable_ops.variable_shape(v.handle)),
+        [2])
+
   def testDifferentAssignGraph(self):
     with ops.Graph().as_default():
       v = resource_variable_ops.ResourceVariable(1.0)
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 5c74dffb05..1066b357b4 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -26,6 +26,7 @@ from tensorflow.core.framework import variable_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import cpp_shape_inference_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -64,6 +65,7 @@ def eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
                                                    name=name,
                                                    container=container)
   if graph_mode:
+    handle._handle_data = get_resource_handle_data(handle)  # pylint: disable=protected-access
     return handle
 
   # We do not want two distinct ResourceVariable objects for the same
@@ -1410,13 +1412,23 @@ def _ReadGrad(_, grad):
   return grad
 
 
+def variable_shape(handle, out_type=dtypes.int32):
+  if getattr(
+      handle, "_handle_data", None) is None or not handle._handle_data.is_set:
+    return gen_resource_variable_ops.variable_shape(handle, out_type=out_type)
+  shape_proto = handle._handle_data.shape_and_type[0].shape
+  if shape_proto.unknown_rank or any(x.size == -1 for x in shape_proto.dim):
+    return gen_resource_variable_ops.variable_shape(handle, out_type=out_type)
+  return constant_op.constant([x.size for x in shape_proto.dim], dtype=out_type)
+
+
 @ops.RegisterGradient("ResourceGather")
 def _GatherGrad(op, grad):
   """Gradient for gather op."""
   # Build appropriately shaped IndexedSlices
   handle = op.inputs[0]
   indices = op.inputs[1]
-  params_shape = gen_resource_variable_ops.variable_shape(handle)
+  params_shape = variable_shape(handle)
   size = array_ops.expand_dims(array_ops.size(indices), 0)
   values_shape = array_ops.concat([size, params_shape[1:]], 0)
   values = array_ops.reshape(grad, values_shape)
-- 
GitLab


From f1263f34f5b0ce7800e6cc9fdc8c3e572e1966f9 Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Thu, 29 Nov 2018 10:51:56 -0800
Subject: [PATCH 1025/1554] Update the error message for missing output files
 in create_python_api.py.

PiperOrigin-RevId: 223373079
---
 tensorflow/python/tools/api/generator/create_python_api.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py
index be988c572c..51c2bfba7c 100644
--- a/tensorflow/python/tools/api/generator/create_python_api.py
+++ b/tensorflow/python/tools/api/generator/create_python_api.py
@@ -463,8 +463,9 @@ def create_api_files(output_files, packages, root_init_template, output_dir,
     raise ValueError(
         """Missing outputs for genrule:\n%s. Be sure to add these targets to
 tensorflow/python/tools/api/generator/api_init_files_v1.bzl and
-tensorflow/python/tools/api/generator/api_init_files.bzl""" % ',\n'.join(
-    sorted(missing_output_files)))
+tensorflow/python/tools/api/generator/api_init_files.bzl (tensorflow repo), or
+tensorflow_estimator/python/estimator/api/api_gen.bzl (estimator repo)"""
+        % ',\n'.join(sorted(missing_output_files)))
 
 
 def main():
-- 
GitLab


From a1f1abbe53e937cb7fe59ab4992dc9d107f8f0c2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 10:52:26 -0800
Subject: [PATCH 1026/1554] Add broadcasting support for RaggedTensors

RELNOTES: Broadcasting support for Ragged Tensors.
PiperOrigin-RevId: 223373179
---
 tensorflow/python/ops/ragged/BUILD            |  34 ++
 tensorflow/python/ops/ragged/__init__.py      |   9 +
 .../python/ops/ragged/ragged_array_ops.py     |  87 +--
 .../ops/ragged/ragged_elementwise_ops.py      |  55 +-
 .../ops/ragged/ragged_elementwise_ops_test.py |  55 +-
 .../python/ops/ragged/ragged_factory_ops.py   |  30 +
 tensorflow/python/ops/ragged/ragged_tensor.py |   1 +
 .../python/ops/ragged/ragged_tensor_shape.py  | 570 ++++++++++++++++++
 .../ops/ragged/ragged_tensor_shape_test.py    | 487 +++++++++++++++
 .../ops/ragged/ragged_to_sparse_op_test.py    |   3 +-
 tensorflow/python/ops/ragged/ragged_util.py   |  49 ++
 11 files changed, 1273 insertions(+), 107 deletions(-)
 create mode 100644 tensorflow/python/ops/ragged/ragged_tensor_shape.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_tensor_shape_test.py

diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 9ff5f26804..8608bda647 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -33,6 +33,7 @@ py_library(
         ":ragged_math_ops",
         ":ragged_operators",
         ":ragged_tensor",
+        ":ragged_tensor_shape",
         ":ragged_tensor_value",
         ":ragged_util",
         ":segment_id_ops",
@@ -155,6 +156,7 @@ py_library(
     deps = [
         ":ragged_factory_ops",
         ":ragged_tensor",
+        ":ragged_tensor_shape",
         ":ragged_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:clip_ops",
@@ -190,6 +192,25 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_tensor_shape",
+    srcs = ["ragged_tensor_shape.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_conversion_ops",
+        ":ragged_factory_ops",
+        ":ragged_tensor",
+        ":ragged_util",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:tensor_util",
+    ],
+)
+
 py_library(
     name = "ragged_tensor_value",
     srcs = ["ragged_tensor_value.py"],
@@ -207,6 +228,7 @@ py_library(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:ragged_math_ops_gen",
     ],
 )
 
@@ -690,3 +712,15 @@ py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
+
+py_test(
+    name = "ragged_tensor_shape_test",
+    srcs = ["ragged_tensor_shape_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index 3a28848545..3e802485a3 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -143,6 +143,11 @@ The following operations are specific to ragged tensors:
 <!-- Elementwise Ops -->
 @@make_elementwise_op
 
+<!-- Shape & broadcasting -->
+@@RaggedTensorDynamicShape
+@@broadcast_to
+@@broadcast_dynamic_shape
+
 <!-- Symbols from  ragged_elementwise_ops._symbols_to_export are whitelisted -->
 """
 
@@ -214,6 +219,10 @@ from tensorflow.python.ops.ragged.ragged_tensor import is_ragged
 from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
 from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensorType
 
+from tensorflow.python.ops.ragged.ragged_tensor_shape import broadcast_dynamic_shape
+from tensorflow.python.ops.ragged.ragged_tensor_shape import broadcast_to
+from tensorflow.python.ops.ragged.ragged_tensor_shape import RaggedTensorDynamicShape
+
 from tensorflow.python.ops.ragged.ragged_tensor_value import RaggedTensorValue
 
 from tensorflow.python.ops.ragged.segment_id_ops import row_splits_to_segment_ids
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index 815f48a4b1..603e39d1dc 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -225,6 +225,28 @@ def row_lengths(rt_input, axis=1, name=None):
       return array_ops.ones(shape[:axis], dtypes.int64) * shape[axis]
 
 
+def nested_row_lengths(rt_input, name=None):
+  """Returns a tuple containing the row_lengths for all ragged dimensions.
+
+  `nested_row_lengths(rt)` is a tuple containing the `row_lengths` tensors for
+  all ragged dimensions in `rt`, ordered from outermost to innermost.
+
+  Args:
+    rt_input: A potentially ragged tensor.
+    name: A name prefix for the returned tensors (optional).
+
+  Returns:
+    A `tuple` of 1-D `int64` `Tensors`.  The length of the tuple is equal to
+    `rt_input.ragged_rank`.
+  """
+  with ops.name_scope(name, 'RaggedNestedRowLengths', [rt_input]):
+    rt_nested_row_lengths = []
+    while isinstance(rt_input, ragged_tensor.RaggedTensor):
+      rt_nested_row_lengths.append(row_lengths(rt_input))
+      rt_input = rt_input.values
+    return tuple(rt_nested_row_lengths)
+
+
 #===============================================================================
 # Bounding Shape
 #===============================================================================
@@ -451,8 +473,7 @@ def batch_gather(params, indices, name=None):
         adjusted_indices = math_ops.to_int64(indices) + adjustments
         return gather(params.values, adjusted_indices)
       else:
-        raise ValueError(
-            'batch shape from indices does not match params shape')
+        raise ValueError('batch shape from indices does not match params shape')
 
 
 #===============================================================================
@@ -719,7 +740,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
             int_mask = ragged_functional_ops.map_inner_values(
                 math_ops.cast, mask, dtype=dtypes.int64)
             masked_row_lengths = ragged_math_ops.reduce_sum(int_mask, axis=1)
-            splits.append(_lengths_to_splits(masked_row_lengths))
+            splits.append(ragged_util.lengths_to_splits(masked_row_lengths))
           mask = mask.values
           data = data.values
 
@@ -741,7 +762,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
       # masks back to a splits tensor.
       lengths = row_lengths(data)
       masked_lengths = array_ops.boolean_mask(lengths, mask)
-      masked_splits = _lengths_to_splits(masked_lengths)
+      masked_splits = ragged_util.lengths_to_splits(masked_lengths)
 
       # Get the masked values: first get row ids corresponding to each
       # value, then use tf.gather to build a boolean mask that's false for
@@ -977,7 +998,7 @@ def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
   # If we are performing a stack operation, then add another splits.
   if stack_values:
     stack_lengths = array_ops.stack([nrows(rt) for rt in rt_inputs])
-    stack_splits = _lengths_to_splits(stack_lengths)
+    stack_splits = ragged_util.lengths_to_splits(stack_lengths)
     concatenated_nested_splits.insert(0, stack_splits)
 
   return ragged_factory_ops.from_nested_row_splits(concatenated_inner_values,
@@ -1131,7 +1152,8 @@ def _tile_ragged_values(rt_input, multiples, const_multiples=None):
 
     # Repeat each element in this ragged dimension `multiples[axis]` times.
     if const_multiples is None or const_multiples[axis] != 1:
-      inner_value_ids = _repeat_ranges(inner_value_ids, splits, multiples[axis])
+      inner_value_ids = ragged_util.repeat_ranges(inner_value_ids, splits,
+                                                  multiples[axis])
 
     prev_splits = splits
 
@@ -1200,15 +1222,15 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None):
     for d in range(axis - 1, -1, -1):
       if const_multiples is None or const_multiples[d + 1] != 1:
         splits = projected_splits[d][axis - 1] * repeats
-        output_lengths = _repeat_ranges(output_lengths, splits,
-                                        multiples[d + 1])
+        output_lengths = ragged_util.repeat_ranges(output_lengths, splits,
+                                                   multiples[d + 1])
       repeats *= multiples[d + 1]
 
     # Tile splits for the outermost (uniform) dimension.
     output_lengths = array_ops.tile(output_lengths, multiples[:1])
 
     # Convert to splits.
-    result_splits.append(_lengths_to_splits(output_lengths))
+    result_splits.append(ragged_util.lengths_to_splits(output_lengths))
 
   return result_splits
 
@@ -1436,11 +1458,6 @@ def _coordinate_where(condition):
 #===============================================================================
 
 
-def _lengths_to_splits(lengths):
-  """Returns splits corresponding to the given lengths."""
-  return array_ops.concat([[0], math_ops.cumsum(lengths)], axis=0)
-
-
 def _increase_ragged_rank_to(rt_input, ragged_rank):
   """Adds ragged dimensions to `rt_input` so it has the desired ragged rank."""
   if ragged_rank > 0:
@@ -1460,45 +1477,3 @@ def _concat_ragged_splits(splits_list):
     pieces.append(splits[1:] + splits_offset)
     splits_offset += splits[-1]
   return array_ops.concat(pieces, axis=0)
-
-
-def _repeat_ranges(params, splits, multiple):
-  """Repeats each range of `params` (as specified by `splits`) `multiple` times.
-
-  Let the `i`th range of `params` be defined as
-  `params[splits[i]:splits[i + 1]]`.  Then this function returns a tensor
-  containing range 0 repeated `multiple` times, followed by range 1 repeated
-  `multiple`, ..., followed by the last range repeated `multiple` times.
-
-  Args:
-    params: The `Tensor` whose values should be repeated.
-    splits: A splits tensor indicating the ranges of `params` that should be
-      repeated.
-    multiple: The number of times each range should be repeated.
-
-  Returns:
-    A `Tensor` with the same rank and type as `params`.
-
-  #### Example:
-    ```python
-    >>> _repeat_ranges(['a', 'b', 'c'], [0, 2, 3], 3)
-    ['a', 'b', 'a', 'b', 'a', 'b', 'c', 'c', 'c']
-    ```
-  """
-  # Repeat each split value `multiple` times.  E.g., if `splits=[0 3 4]` and
-  # `multiples=3`, then `repeated_splits=[0 0 0 3 3 3 4 4 4]`.
-  repeated_splits = array_ops.tile(
-      array_ops.expand_dims(splits, axis=1), array_ops.stack([1, multiple]))
-  repeated_splits = array_ops.reshape(repeated_splits, [-1])
-
-  # Divide the splits into repeated starts & repeated limits.  E.g., if
-  # `repeated_splits=[0 0 0 3 3 3 4 4 4]` then `repeated_starts=[0 0 0 3 3 3]`
-  # and `repeated_limits=[3 3 3 4 4 4]`.
-  n_splits = array_ops.shape(repeated_splits, out_type=dtypes.int64)[0]
-  repeated_starts = repeated_splits[:n_splits - multiple]
-  repeated_limits = repeated_splits[multiple:]
-
-  # Get indices for each range from starts to limits, and use those to gather
-  # the values in the desired repetition pattern.
-  offsets = ragged_math_ops.range(repeated_starts, repeated_limits).values
-  return array_ops.gather(params, offsets)
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
index edf721b5a7..a497500a6a 100644
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
@@ -28,7 +28,7 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
-from tensorflow.python.ops.ragged import ragged_util
+from tensorflow.python.ops.ragged import ragged_tensor_shape
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_export
 from tensorflow.python.util import tf_inspect
@@ -209,28 +209,45 @@ def _broadcast_elementwise_args(elementwise_args):
     if not any(is_ragged):
       return elementwise_args, (), ()
 
-    # Support limited broadcasting (namely, scalar + ragged).  Full
-    # broadcasting support will be added later.
-    if all((ragged_tensor.is_ragged(t) or t.shape.ndims == 0)
-           for t in elementwise_args.values()):
+    # If we have a single ragged tensor plus a set of scalars, then we can
+    # rely on the underlying elementwise op to do broadcasting.
+    if (sum(is_ragged) == 1 and
+        all((ragged_tensor.is_ragged(t) or t.shape.ndims == 0)
+            for t in elementwise_args.values())):
       nested_splits_lists = [
           t.nested_row_splits
           for t in elementwise_args.values()
-          if ragged_tensor.is_ragged(t)
-      ]
-      if len(nested_splits_lists) == 1:
-        checks = ()
-      else:
-        if any(t.shape.ndims is None for t in elementwise_args.values()):
-          raise ValueError('Ragged elementwise ops require that rank (number '
-                           'of dimensions) be statically known.')
-        if len(set(t.shape.ndims for t in elementwise_args.values())) != 1:
-          raise ValueError('Ragged elementwise ops do not support '
-                           'broadcasting yet')
-        checks = ragged_util.assert_splits_match(nested_splits_lists)
-      return (elementwise_args, nested_splits_lists[0], checks)
+          if ragged_tensor.is_ragged(t)][0]
+      return elementwise_args, nested_splits_lists, ()
+
     else:
-      raise ValueError('Ragged elementwise ops do not support broadcasting yet')
+      # Get the shapes of all the elementwise arguments.
+      shapes = [ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(t)
+                for t in elementwise_args.values()]
+
+      # Broadcast the shapes to all have the same rank (the max rank).
+      ranks = [t.shape.ndims for t in elementwise_args.values()]
+      if any(rank is None for rank in ranks):
+        raise ValueError('Unable to broadcast: unknown rank')
+      broadcast_rank = max(ranks)
+      shapes = [shape.broadcast_to_rank(broadcast_rank) for shape in shapes]
+
+      # For each dimension, broadcast the shapes to be compatible.
+      for axis in range(broadcast_rank):
+        # For each i, broadcast shape[i+1] to be compatible with shape[i]; and
+        # then finally broadcast shape[0] to be compatible with shape[-1].
+        for i in range(len(shapes)):
+          j = (i + 1) % len(shapes)
+          dim_size = shapes[i].dimension_size(axis)
+          shapes[j] = shapes[j].broadcast_dimension(axis, dim_size)
+      broadcast_shape = shapes[0]
+
+      # Broadcast every elementwise arg to the shape that we calculated.
+      elementwise_args = dict([
+          (key, ragged_tensor_shape.broadcast_to(t, broadcast_shape, False))
+          for (key, t) in elementwise_args.items()])
+      nested_splits_lists = list(elementwise_args.values())[0].nested_row_splits
+      return elementwise_args, nested_splits_lists, ()
 
 
 # A list of symbols that should be exported in the "ragged" package.
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
index 5dfa5cff45..26e6b8b6d4 100644
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
+++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
@@ -399,44 +399,37 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
     y = ragged.from_row_splits(
         array_ops.placeholder_with_default([1, 2, 3], shape=None), x.row_splits)
     with self.assertRaisesRegexp(
-        ValueError, r'Ragged elementwise ops require that rank \(number '
-        r'of dimensions\) be statically known.'):
+        ValueError, r'Unable to broadcast: unknown rank'):
       ragged.add(x, y)
 
-  def testBroadcastError1(self):
-    x = ragged.constant([[1, 2], [3]])
-    y = [[12]]
-    with self.assertRaisesRegexp(
-        ValueError, 'Ragged elementwise ops do not support broadcasting yet'):
-      ragged.add(x, y)
-
-  def testBroadcastError2(self):
-    x = ragged.constant([[[1, 2], [3, 4]], [[5]]], ragged_rank=2)
-    y = ragged.constant([[[8], [3]], [[2]]], ragged_rank=1)
-    with self.assertRaisesRegexp(ValueError,
-                                 'Inputs must have identical ragged splits'):
-      ragged.add(x, y)
-
-  def testBroadcastError3(self):
-    x = ragged.constant([[[1, 2], [3]], [[4, 5], [6]]], ragged_rank=2)
-    y = ragged.constant([[7, 8], [9]], ragged_rank=1)
-    with self.assertRaisesRegexp(
-        ValueError, 'Ragged elementwise ops do not support broadcasting yet'):
-      ragged.add(x, y)
-
-  def testBroadcastError4(self):
-    x = ragged.constant([[[1]]])
-    y = ragged.constant([[1]])
-    with self.assertRaisesRegexp(
-        ValueError, 'Ragged elementwise ops do not support broadcasting yet'):
-      ragged.add(x, y)
+  @parameterized.parameters([
+      dict(
+          x=ragged.constant_value([[1, 2], [3]]),
+          y=[[10]],
+          expected=[[11, 12], [13]]),
+      dict(
+          x=ragged.constant_value([[[1, 2], [3, 4]], [[5]]], ragged_rank=2),
+          y=ragged.constant_value([[[10], [20]], [[30]]], ragged_rank=1),
+          expected=[[[11, 12], [23, 24]], [[35]]]),
+      dict(
+          x=ragged.constant_value([[[1]]]),
+          y=ragged.constant_value([[1]]),
+          expected=[[[2]]]),
+  ])
+  def testBroadcastAdd(self, x, y, expected):
+    x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32)
+    y = ragged.convert_to_tensor_or_ragged_tensor(y, dtype=dtypes.int32)
+    result = x + y
+    with self.cached_session():
+      self.assertEqual(result.eval().tolist(), expected)
 
   def testShapeMismatch(self):
     x = ragged.constant([[1, 2, 3], [4, 5]])
     y = ragged.constant([[1, 2, 3], [4, 5, 6]])
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 'Inputs must have identical ragged splits'):
-      ragged.add(x, y)
+                                 'Incompatible shapes'):
+      with self.cached_session():
+        ragged.add(x, y).eval()
 
   def testDocstring(self):
     self.assertRegexpMatches(
diff --git a/tensorflow/python/ops/ragged/ragged_factory_ops.py b/tensorflow/python/ops/ragged/ragged_factory_ops.py
index de3a2d5b10..d1f301bc58 100644
--- a/tensorflow/python/ops/ragged/ragged_factory_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_factory_ops.py
@@ -676,3 +676,33 @@ def from_nested_row_splits(inner_values, nested_row_splits, name=None):
     for splits in reversed(nested_row_splits):
       result = from_row_splits(result, splits)
     return result
+
+
+def from_nested_row_lengths(inner_values, nested_row_lengths, name=None):
+  """Creates a `RaggedTensor` from a nested list of `row_lengths` tensors.
+
+  Equivalent to:
+
+  ```python
+  result = inner_values
+  for row_lengths in reversed(nested_row_lengths):
+    result = from_row_lengths(result, row_lengths)
+  ```
+
+  Args:
+    inner_values: A potentially ragged tensor.
+    nested_row_lengths: A list of 1-D int64 tensors.  The `i`th tensor is used
+      as the `row_lengths` for the `i`th ragged dimension.
+    name: A name prefix for the RaggedTensor (optional).
+
+  Returns:
+    A `RaggedTensor` (or `inner_values` if `nested_row_lengths` is empty).
+  """
+  if isinstance(nested_row_lengths, ops.Tensor):
+    raise TypeError('nested_row_lengths must be a list of Tensors')
+  with ops.name_scope(name, 'RaggedFromNestedRowlengths',
+                      [inner_values] + list(nested_row_lengths)):
+    result = inner_values
+    for lengths in reversed(nested_row_lengths):
+      result = from_row_lengths(result, lengths)
+    return result
diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py
index abb27fc3c0..90f0dafd99 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor.py
@@ -257,6 +257,7 @@ class RaggedTensor(object):
       raise TypeError("Row-partitioning argument must be a Tensor.")
     values.shape.with_rank_at_least(1)
     row_splits.shape.assert_has_rank(1)
+    row_splits.set_shape([None])
 
     self._values = values
     self._row_splits = row_splits
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape.py b/tensorflow/python/ops/ragged/ragged_tensor_shape.py
new file mode 100644
index 0000000000..9129b4b10b
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_tensor_shape.py
@@ -0,0 +1,570 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Shapes & broadcasting for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_util
+
+
+class RaggedTensorDynamicShape(object):
+  """A collection of tensors encoding the shape of a potentially ragged tensor.
+
+  Each `RaggedTensorDynamicShape` consists of an ordered list of dimension
+  sizes.  There are two dimension types:
+
+    * "Uniform dimensions" are dimenisons where all slices have the same
+      length.  `RaggedTensorDynamicShape` records the size of each uniform
+      dimension using a single scalar integer.
+
+    * "Ragged dimensions" are dimensions whose slices may have different
+      lengths.  `RaggedTensorDynamicShape` records the size of each ragged
+      dimension using an integer vector containing the slice lengths for all
+      the slices across that dimension.
+
+  Furthermore, there are two ways a dimension might be encoded:
+
+    * "Partitioned dimensions" are dimensions that are encoded using a
+      `RaggedTensor`'s `nested_row_splits`.  The outermostmost partitioned
+      dimension must be uniform, and the innermost partitioned dimension must
+      be ragged.
+
+    * "Inner dimensions" are dimensions that are encoded using a
+      `RaggedTensor`'s `inner_values`.  Inner dimensions are always uniform.
+
+  The sizes of partitioned dimensions are recorded using `partitioned_dim_sizes`
+  and `inner_dim_sizes`:
+
+    * `paritioned_dim_sizes` is a list of tensors (one for each partitioned
+      dimension).
+
+      * For uniform dimensions, the tensor is an integer scalar specifying the
+        size of all slices across that dimension.
+      * For ragged dimensions, the tensor is an integer vector specifying the
+        size of each slice across that dimension.
+
+    * `inner_dim_sizes` is a single integer vector, where each element
+      specifies the size of a single inner dimension.
+
+  Examples:
+
+  Tensor                         | Ragged | Partitioned Dim Sizes  | Inner Dim
+                                 : Rank   :                        : Sizes
+  ------------------------------ | ------ | ---------------------- | ----------
+  `[[1, 2, 3], [4, 5, 6]]`       |      0 |                        | `2, 3`
+  `[[1, 2], [], [3, 4, 5]]`      |      1 | `3, (2, 0, 3)`         |
+  `[[[1, 2], [3, 4]], [[5, 6]]]` |      1 | `2, (2, 1)`            | 2
+  `[[[1, 2], [3]], [[4, 5]]]`    |      2 | `2, (2, 1), (2, 1, 2)` |
+  """
+
+  def __init__(self, partitioned_dim_sizes, inner_dim_sizes):
+    """Creates a RaggedTensorDynamicShape.
+
+    Args:
+      partitioned_dim_sizes: A `list` of 0-D or 1-D integer `Tensor`, one for
+        each partitioned dimension.  If dimension `d` is uniform, then
+        `partitioned_dim_sizes[d]` must be an integer scalar, specifying the
+        size of all slices across dimension `d`.  If dimension `d` is ragged,
+        then `partitioned_dim_sizes[d]` must be an integer vector, specifying
+        the size of each slice across dimension `d`.
+      inner_dim_sizes: A 1-D integer `Tensor`, whose length is equal to the
+        number of inner dimensions.  `inner_dim_sizes[n]` is the size of all
+        slices across the `n`th inner dimension (which is the
+        `(len(partitioned_dim_sizes)+n)`th dimension in the overall tensor.
+    """
+    assert isinstance(partitioned_dim_sizes, (list, tuple))
+    with ops.name_scope(None, 'RaggedTensorDynamicShape',
+                        (partitioned_dim_sizes, inner_dim_sizes)):
+      partitioned_dim_sizes = tuple(
+          ragged_util.convert_to_int_tensor(
+              size, dtype=dtypes.int64, name='partitioned_dimension_size')
+          for size in partitioned_dim_sizes)
+      inner_dim_sizes = ragged_util.convert_to_int_tensor(
+          inner_dim_sizes, dtype=dtypes.int64, name='inner_dim_sizes')
+
+      # Validate shapes.
+      if partitioned_dim_sizes:
+        for axis, dimension_size in enumerate(partitioned_dim_sizes):
+          if dimension_size.shape.ndims is None:
+            raise ValueError(
+                'rank of partitioned_dim_sizes[%d] is unknown' % axis)
+          dimension_size.shape.with_rank_at_most(1)
+        if partitioned_dim_sizes[0].shape.ndims == 1:
+          raise ValueError('outermost partitioned dimension must be uniform')
+        if partitioned_dim_sizes[-1].shape.ndims == 0:
+          raise ValueError('innermost partitioned dimension must be ragged')
+      inner_dim_sizes.shape.assert_has_rank(1)
+
+      self._partitioned_dim_sizes = partitioned_dim_sizes
+      self._inner_dim_sizes = inner_dim_sizes
+
+  def __repr__(self):
+    return ('RaggedTensorDynamicShape'
+            '(partitioned_dim_sizes=%r, inner_dim_sizes=%r)' %
+            (self._partitioned_dim_sizes, self._inner_dim_sizes))
+
+  @staticmethod
+  def from_dim_sizes(dim_sizes):
+    """Constructs a ragged shape from a list of dimension sizes.
+
+    This list contains a single tensor for each dimension, where the tensor
+    is a scalar if the dimension is uniform, or a vector if the dimension is
+    ragged.
+
+    Args:
+      dim_sizes: List of int64 scalars or vectors.
+
+    Returns:
+      A RaggedTensorDynamicShape.
+    """
+    with ops.name_scope(None, 'RaggedTensorDynamicShapeFromDimensionSizes',
+                        [dim_sizes]):
+      dim_sizes = tuple(
+          ragged_util.convert_to_int_tensor(
+              size, dtype=dtypes.int64, name='dim_sizes') for size in dim_sizes)
+      # Split the dimensions into partitioned & inner dimensions.
+      inner_split = 0
+      for dim, dim_size in enumerate(dim_sizes):
+        if dim_size.shape.ndims == 1:
+          inner_split = dim + 1
+        elif dim_size.shape.ndims != 0:
+          raise ValueError('Each dim_size must be a scalar or a vector')
+      return RaggedTensorDynamicShape(dim_sizes[:inner_split],
+                                      dim_sizes[inner_split:])
+
+  @classmethod
+  def from_tensor(cls, rt_input):
+    """Constructs a ragged shape for a potentially ragged tensor."""
+    with ops.name_scope(None, 'RaggedTensorDynamicShapeFromTensor', [rt_input]):
+      rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(rt_input)
+      if not ragged_tensor.is_ragged(rt_input):
+        return cls([], array_ops.shape(rt_input))
+      else:
+        partitioned_dim_sizes = ((ragged_array_ops.nrows(rt_input),) +
+                                 ragged_array_ops.nested_row_lengths(rt_input))
+        return RaggedTensorDynamicShape(
+            partitioned_dim_sizes,
+            array_ops.shape(rt_input.inner_values)[1:])
+
+  def dimension_size(self, axis):
+    """Returns the size of slices across the specified dimension."""
+    if not isinstance(axis, int):
+      raise TypeError('axis must be an integer')
+    partitioned_ndims = len(self._partitioned_dim_sizes)
+    if axis < partitioned_ndims:
+      return self._partitioned_dim_sizes[axis]
+    else:
+      return self._inner_dim_sizes[axis - partitioned_ndims]
+
+  def is_ragged(self, axis):
+    """Returns true if the indicated dimension is ragged."""
+    if not isinstance(axis, int):
+      raise TypeError('axis must be an integer')
+    rank = self.rank
+    if axis < 0:
+      raise ValueError('Negative axis values are not supported')
+    elif rank is not None and axis >= rank:
+      raise ValueError('Expected axis=%s < rank=%s' % (axis, rank))
+    else:
+      return (axis > 0 and axis < len(self._partitioned_dim_sizes) and
+              self._partitioned_dim_sizes[axis].shape.ndims == 1)
+
+  @property
+  def rank(self):
+    """The number of dimensions in this shape, or None if unknown."""
+    inner_ndims = self._inner_dim_sizes.shape[0].value
+    if inner_ndims is None:
+      return None
+    else:
+      return len(self._partitioned_dim_sizes) + inner_ndims
+
+  @property
+  def partitioned_dim_sizes(self):
+    """The partitioned dimension sizes for this shape.
+
+    Returns:
+      A `list` of 0-D or 1-D integer `Tensor`.
+    """
+    return self._partitioned_dim_sizes
+
+  @property
+  def inner_dim_sizes(self):
+    """The inner dimension sizes for this shape.
+
+    Returns:
+      A 1-D integer `Tensor`.
+    """
+    return self._inner_dim_sizes
+
+  @property
+  def num_partitioned_dimensions(self):
+    """The number of partitioned dimensions in this shape."""
+    return len(self._partitioned_dim_sizes)
+
+  @property
+  def num_inner_dimensions(self):
+    """The number of inner dimensions, or `None` if not statically known."""
+    return self._inner_dim_sizes.shape[0].value
+
+  def broadcast_to_rank(self, rank):
+    """Adds leading size-1 dimensions to broadcast `self` to the given rank.
+
+    E.g., if `shape1` is `[3, (D2), 4]`, then `shape1.broadcast_to_rank(5)`
+    is `[1, 1, 3, (D2), 4]`.
+
+    Args:
+      rank: The rank for the returned shape.
+
+    Returns:
+      A RaggedTensorDynamicShape with `rank` dimensions, whose inner dimensions
+      have the same size as `self` and whose outer dimensions have size `1`.
+
+    Raises:
+      ValueError: If `self.rank` is unknown or greater than `rank`.
+    """
+    if self.rank is None:
+      raise ValueError('Unable to broadcast: self.rank is unknown')
+    dims_to_add = rank - self.rank
+    if dims_to_add < 0:
+      raise ValueError('Unable to broadcast: rank=%d must be greater than '
+                       'self.rank=%d.' % (rank, self.rank))
+    elif dims_to_add == 0:
+      return self
+    elif self._partitioned_dim_sizes:
+      partitioned_dims = (1,) * dims_to_add + self._partitioned_dim_sizes
+      return RaggedTensorDynamicShape(partitioned_dims, self._inner_dim_sizes)
+    else:
+      inner_dims = array_ops.concat(
+          [array_ops.ones([dims_to_add], dtypes.int64), self.inner_dim_sizes],
+          axis=0)
+      return RaggedTensorDynamicShape([], inner_dims)
+
+  def broadcast_dimension(self, axis, lengths):
+    """Returns a shape that is broadcast-compatible with self & lengths.
+
+    * If dimension[axis] is uniform and lengths is a scalar, the check
+      that either lengths==1 or axis==1 or lengths==axis, and tile
+      dimension[axis] with tf.where(lengths==axis, 1, axis) repeats.
+
+    * If dimension[axis] is uniform and lengths is a vector, then check
+      that dimension[axis]==1, and raggedly tile dimension[axis] with
+      lengths repeats.  (we can skip tiling if we statically know that
+      slice_lengths == 1??)
+
+    * If dimension[axis] is ragged and lengths is a scalar, then check
+      that lengths==1.
+
+    * If dimension[axis] is ragged and lengths is a vector, then check
+      that self.dimension_size(axis) == lengths.
+
+    Args:
+      axis: `int`.  The dimension to broadcast.
+      lengths: 0-D or 1-D integer `Tensor`.
+
+    Returns:
+      A `RaggedTensorDynamicShape`.
+    """
+    lengths = ragged_util.convert_to_int_tensor(
+        lengths, name='lengths', dtype=dtypes.int64)
+    # Check whether lengths is a scalar (for uniform dimensions) or
+    # vector (for ragged dimensions).
+    if lengths.shape.ndims is None:
+      raise ValueError('lengths must have a known rank.')
+    elif lengths.shape.ndims > 1:
+      raise ValueError('lengths must be a scalar or vector')
+    else:
+      lengths_is_scalar = (lengths.shape.ndims == 0)
+
+    # Verify that the shapes are compatible.
+    if self.is_ragged(axis):
+      if lengths_is_scalar:
+        condition = math_ops.equal(lengths, 1)
+      else:
+        condition = math_ops.reduce_all(
+            math_ops.equal(lengths, self.dimension_size(axis)))
+    else:
+      axis_dim_size = self.dimension_size(axis)
+      if lengths_is_scalar:
+        condition = (
+            math_ops.equal(lengths, 1) | math_ops.equal(axis_dim_size, 1)
+            | math_ops.equal(axis_dim_size, lengths))
+      else:
+        condition = math_ops.equal(axis_dim_size, 1)
+    broadcast_err = [
+        'Unable to broadcast: dimension size mismatch in dimension', axis,
+        'lengths=', lengths, 'dim_size=',
+        self.dimension_size(axis)
+    ]
+    broadcast_check = control_flow_ops.Assert(
+        condition, data=broadcast_err, summarize=10)
+
+    with ops.control_dependencies([broadcast_check]):
+      # Partitioned dimensions:
+      if axis < self.num_partitioned_dimensions:
+        if self.is_ragged(axis):
+          # Use an identity op to make sure the check actually gets run.
+          return RaggedTensorDynamicShape(
+              self._partitioned_dim_sizes,
+              array_ops.identity(self.inner_dim_sizes))
+        else:
+          return self._broadcast_uniform_partitioned_dimension(axis, lengths)
+
+      # Inner dimensions:
+      else:
+        if lengths_is_scalar:
+          return self._broadcast_inner_dimension_to_uniform(axis, lengths)
+        else:
+          if axis == 0:
+            raise ValueError('Unable to broadcast: '
+                             'outermost dimension must be uniform.')
+          return self._broadcast_inner_dimension_to_ragged(axis, lengths)
+
+  def num_slices_in_dimension(self, axis):
+    """Returns the total number of slices across the indicated dimension."""
+    if axis < 0:
+      return constant_op.constant(1, dtype=dtypes.int64)
+    elif self.is_ragged(axis):
+      return math_ops.reduce_sum(self._partitioned_dim_sizes[axis])
+    else:
+      return self.dimension_size(axis) * self.num_slices_in_dimension(axis - 1)
+
+  def _broadcast_uniform_partitioned_dimension(self, axis, lengths):
+    """Broadcasts the partitioned dimension `axis` to match `lengths`."""
+    axis_dim_size = self.dimension_size(axis)
+    partitioned_sizes = list(self._partitioned_dim_sizes[:axis])
+
+    if lengths.shape.ndims == 0:
+      lengths = array_ops.where(
+          math_ops.equal(axis_dim_size, 1), lengths, axis_dim_size)
+      repeats = array_ops.where(math_ops.equal(axis_dim_size, 1), lengths, 1)
+      splits = array_ops.stack([0, self.num_slices_in_dimension(axis)])
+    else:
+      splits = math_ops.range(
+          array_ops.size(lengths, out_type=dtypes.int64) + 1)
+      repeats = lengths
+
+    partitioned_sizes.append(lengths)
+
+    for dim_size in self._partitioned_dim_sizes[axis + 1:]:
+      if dim_size.shape.ndims == 0:
+        partitioned_sizes.append(dim_size)
+        splits *= dim_size
+      else:
+        partitioned_sizes.append(
+            ragged_util.repeat_ranges(dim_size, splits, repeats))
+        splits = array_ops.gather(
+            ragged_util.lengths_to_splits(dim_size), splits)
+    inner_sizes = self._inner_dim_sizes
+    return RaggedTensorDynamicShape(partitioned_sizes, inner_sizes)
+
+  def _broadcast_inner_dimension_to_uniform(self, axis, length):
+    """Broadcasts the inner dimension `axis` to match `lengths`."""
+    dim_size = self.dimension_size(axis)
+    axis_in_inner_dims = axis - self.num_partitioned_dimensions
+    partitioned_sizes = self._partitioned_dim_sizes
+    inner_sizes = array_ops.concat([
+        self._inner_dim_sizes[:axis_in_inner_dims],
+        [array_ops.where(math_ops.equal(dim_size, 1), length, dim_size)],
+        self._inner_dim_sizes[axis_in_inner_dims + 1:]
+    ],
+                                   axis=0)
+    return RaggedTensorDynamicShape(partitioned_sizes, inner_sizes)
+
+  def _broadcast_inner_dimension_to_ragged(self, axis, lengths):
+    axis_in_inner_dims = axis - self.num_partitioned_dimensions
+    partitioned_sizes = (
+        self._partitioned_dim_sizes + tuple([
+            self._inner_dim_sizes[i] for i in range(axis_in_inner_dims)
+        ]) + (lengths,))
+    inner_sizes = self._inner_dim_sizes[axis_in_inner_dims + 1:]
+    return RaggedTensorDynamicShape(partitioned_sizes, inner_sizes)
+
+
+def broadcast_dynamic_shape(shape_x, shape_y):
+  """Returns the shape formed by broadcasting two shapes to be compatible.
+
+  Args:
+    shape_x: A `RaggedTensorDynamicShape`
+    shape_y: A `RaggedTensorDynamicShape`
+
+  Returns:
+    A `RaggedTensorDynamicShape`.
+  Raises:
+    ValueError: If `shape_x` and `shape_y` are not broadcast-compatible.
+  """
+  if not isinstance(shape_x, RaggedTensorDynamicShape):
+    raise TypeError('shape_x must be a RaggedTensorDynamicShape')
+  if not isinstance(shape_y, RaggedTensorDynamicShape):
+    raise TypeError('shape_y must be a RaggedTensorDynamicShape')
+
+  # Broadcast both shapes to have the same rank.
+  if shape_x.rank is None or shape_y.rank is None:
+    raise ValueError('Unable to broadcast: unknown rank')
+  broadcast_rank = max(shape_x.rank, shape_y.rank)
+  shape_x = shape_x.broadcast_to_rank(broadcast_rank)
+  shape_y = shape_y.broadcast_to_rank(broadcast_rank)
+
+  # Broadcast dimensions one at a time, starting from the outermost dimension.
+  for axis in range(broadcast_rank):
+    shape_x = shape_x.broadcast_dimension(axis, shape_y.dimension_size(axis))
+    shape_y = shape_y.broadcast_dimension(axis, shape_x.dimension_size(axis))
+
+  return shape_x
+
+
+def broadcast_to(rt_input, shape, broadcast_inner_dimensions=True):
+  """Broadcasts a potentially ragged tensor to a ragged shape.
+
+  Tiles `rt_input` as necessary to match the given shape.
+
+  Behavior is undefined if `rt_input` is not broadcast-compatible with `shape`.
+
+  Args:
+    rt_input: The potentially ragged tensor to broadcast.
+    shape: A `RaggedTensorDynamicShape`
+    broadcast_inner_dimensions: If false, then inner dimensions will not be
+      tiled.
+
+  Returns:
+    A potentially ragged tensor whose values are taken from
+    `rt_input`, and whose shape matches `shape`.
+  """
+  if not isinstance(shape, RaggedTensorDynamicShape):
+    raise TypeError('shape must be a RaggedTensorDynamicShape')
+  rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(rt_input)
+
+  # Broadcasting to a uniform shape.
+  if shape.num_partitioned_dimensions == 0:
+    return _broadcast_to_uniform_shape(rt_input, shape,
+                                       broadcast_inner_dimensions)
+  else:
+    return _broadcast_to_ragged_shape(rt_input, shape,
+                                      broadcast_inner_dimensions)
+
+
+def _broadcast_to_uniform_shape(rt_input, shape, broadcast_inner_dimensions):
+  """Broadcasts rt_input to the uniform shape `shape`."""
+  if isinstance(rt_input, ragged_tensor.RaggedTensor):
+    raise ValueError('Incompatible with shape: ragged rank mismatch')
+  if broadcast_inner_dimensions:
+    return array_ops.broadcast_to(rt_input, shape.inner_dim_sizes)
+  else:
+    return rt_input
+
+
+def _broadcast_to_ragged_shape(rt_input, dst_shape, broadcast_inner_dimensions):
+  """Broadcasts rt_input to the ragged shape `dst_shape`."""
+  # dst_shape's rank and ragged_rank must be greater than or equal to rt_input's
+  if rt_input.shape.ndims is None or dst_shape.rank is None:
+    raise ValueError('Unable to broadcast: unknown rank')
+  if rt_input.shape.ndims > dst_shape.rank:
+    raise ValueError('Incompatible with shape: rank mismatch')
+  if (isinstance(rt_input, ragged_tensor.RaggedTensor) and
+      rt_input.ragged_rank >= dst_shape.num_partitioned_dimensions):
+    raise ValueError('Incompatible with shape: ragged rank mismatch')
+
+  src_shape = RaggedTensorDynamicShape.from_tensor(rt_input)
+  src_shape = src_shape.broadcast_to_rank(dst_shape.rank)
+
+  # Add dimensions to rt_input so its rank and ragged_rank matches dst_shape.
+  if dst_shape.rank > rt_input.shape.ndims:
+    if rt_input.shape.ndims < dst_shape.num_inner_dimensions + 1:
+      rt_input = array_ops.reshape(
+          rt_input, array_ops.concat([[-1], dst_shape.inner_dim_sizes], axis=0))
+    for _ in range(dst_shape.rank - rt_input.shape.ndims):
+      rt_input = ragged_factory_ops.from_row_lengths(
+          rt_input, [ragged_array_ops.nrows(rt_input)])
+
+  # Add ragged dimensions to match dst_shape.
+  if ragged_tensor.is_ragged(rt_input):
+    inner_rank_diff = (
+        rt_input.inner_values.shape.ndims - 1 - dst_shape.num_inner_dimensions)
+    if inner_rank_diff > 0:
+      rt_input = rt_input.with_inner_values(
+          ragged_conversion_ops.from_tensor(
+              rt_input.inner_values, ragged_rank=inner_rank_diff))
+  else:
+    rt_input = ragged_conversion_ops.from_tensor(
+        rt_input, ragged_rank=dst_shape.num_partitioned_dimensions - 1)
+
+  # Do broadcasting for any dimensions that will remain uniform.  We can do
+  # these all at once, since they're independent of one another.
+  multiples = [1] * dst_shape.rank
+  for axis in range(dst_shape.num_partitioned_dimensions):
+    if not src_shape.is_ragged(axis) and not dst_shape.is_ragged(axis):
+      src_size = src_shape.dimension_size(axis)
+      dst_size = dst_shape.dimension_size(axis)
+      if ((tensor_util.constant_value(src_size) in (1, None)) and
+          (tensor_util.constant_value(dst_size) != 1)):
+        multiples[axis] = array_ops.where(
+            math_ops.equal(src_size, 1), dst_size, 1)
+  if not all(isinstance(v, int) and v == 1 for v in multiples):
+    multiples = array_ops.stack(multiples, axis=0)
+    rt_input = ragged_array_ops.tile(rt_input, multiples)
+
+  if broadcast_inner_dimensions:
+    rt_input = rt_input.with_inner_values(
+        array_ops.reshape(
+            rt_input.inner_values,
+            array_ops.concat([[-1], dst_shape.inner_dim_sizes], axis=0)))
+
+  # Do broadcasting for dimensions that become ragged.  We must do these from
+  # outermost to innermost.
+  for axis in range(dst_shape.num_partitioned_dimensions):
+    if not src_shape.is_ragged(axis) and dst_shape.is_ragged(axis):
+      dst_size = dst_shape.dimension_size(axis)
+      rt_input = _ragged_tile_axis(rt_input, axis, dst_size)
+
+  return rt_input
+
+
+def _ragged_tile_axis(rt_input, axis, repeats):
+  """Tile a dimension of a RaggedTensor to match a ragged shape."""
+  assert axis > 0  # Outermost dimension may not be ragged.
+
+  if not ragged_tensor.is_ragged(rt_input):
+    rt_input = ragged_conversion_ops.from_tensor(rt_input, ragged_rank=1)
+
+  if axis > 1:
+    return rt_input.with_values(
+        _ragged_tile_axis(rt_input.values, axis - 1, repeats))
+  else:
+    src_row_splits = rt_input.nested_row_splits
+    src_row_lengths = ragged_array_ops.nested_row_lengths(rt_input)
+    splits = src_row_splits[0]
+
+    dst_row_lengths = [repeats]
+    for i in range(1, len(src_row_lengths)):
+      dst_row_lengths.append(
+          ragged_util.repeat_ranges(src_row_lengths[i], splits, repeats))
+      splits = array_ops.gather(src_row_splits[i], splits)
+    dst_values = ragged_util.repeat_ranges(rt_input.inner_values, splits,
+                                           repeats)
+    return ragged_factory_ops.from_nested_row_lengths(dst_values,
+                                                      dst_row_lengths)
+
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
new file mode 100644
index 0000000000..9c2dd26050
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
@@ -0,0 +1,487 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tf.ragged.ragged_tensor_shape."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import ragged
+from tensorflow.python.platform import googletest
+
+
+class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
+                            parameterized.TestCase):
+
+  def assertShapeEq(self, x, y):
+    assert isinstance(x, ragged.RaggedTensorDynamicShape)
+    assert isinstance(y, ragged.RaggedTensorDynamicShape)
+    x_partitioned_dim_sizes = [
+        splits.eval().tolist()  #
+        for splits in x.partitioned_dim_sizes
+    ]
+    y_partitioned_dim_sizes = [
+        splits.eval().tolist()  #
+        for splits in y.partitioned_dim_sizes
+    ]
+    self.assertEqual(x_partitioned_dim_sizes, y_partitioned_dim_sizes)
+    self.assertEqual(x.inner_dim_sizes.eval().tolist(),
+                     y.inner_dim_sizes.eval().tolist())
+
+  @parameterized.parameters([
+      dict(value='x', expected_dim_sizes=[]),
+      dict(value=['a', 'b', 'c'], expected_dim_sizes=[3]),
+      dict(value=[['a', 'b', 'c'], ['d', 'e', 'f']], expected_dim_sizes=[2, 3]),
+      dict(
+          value=[[['a', 'b', 'c'], ['d', 'e', 'f']]],
+          expected_dim_sizes=[1, 2, 3]),
+      dict(
+          value=ragged.constant_value([['a', 'b', 'c'], ['d', 'e']]),
+          expected_dim_sizes=[2, [3, 2]]),
+      dict(
+          value=ragged.constant_value([[['a', 'b', 'c'], ['d', 'e']]]),
+          expected_dim_sizes=[1, [2], [3, 2]]),
+      dict(
+          value=ragged.constant_value([[['a', 'b', 'c'], ['d', 'e', 'f']]],
+                                      ragged_rank=1),
+          expected_dim_sizes=[1, [2], 3]),
+      dict(
+          value=ragged.constant_value([[[[1], [2]], [[3], [4]]],
+                                       [[[5], [6]]]], ragged_rank=1),
+          expected_dim_sizes=[2, [2, 1], 2, 1]),
+      dict(
+          value=ragged.constant_value([[10, 20], [30]]),
+          expected_dim_sizes=[2, [2, 1]]),
+      # Docstring examples:
+      dict(value=[[1, 2, 3], [4, 5, 6]], expected_dim_sizes=[2, 3]),
+      dict(
+          value=ragged.constant_value([[1, 2], [], [3, 4, 5]]),
+          expected_dim_sizes=[3, [2, 0, 3]]),
+      dict(
+          value=ragged.constant_value([[[1, 2], [3, 4]], [[5, 6]]],
+                                      ragged_rank=1),
+          expected_dim_sizes=[2, [2, 1], 2]),
+      dict(
+          value=ragged.constant_value([[[1, 2], [3]], [[4, 5]]]),
+          expected_dim_sizes=[2, [2, 1], [2, 1, 2]]),
+  ])
+  def testFromTensor(self, value, expected_dim_sizes):
+    shape = ragged.RaggedTensorDynamicShape.from_tensor(value)
+    expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(
+        expected_dim_sizes)
+    with self.cached_session():
+      self.assertShapeEq(shape, expected)
+
+  @parameterized.parameters([
+      dict(dim_sizes=[], rank=0, expected_dim_sizes=[]),
+      dict(dim_sizes=[], rank=3, expected_dim_sizes=[1, 1, 1]),
+      dict(dim_sizes=[3], rank=1, expected_dim_sizes=[3]),
+      dict(dim_sizes=[3], rank=3, expected_dim_sizes=[1, 1, 3]),
+      dict(dim_sizes=[2, 3], rank=3, expected_dim_sizes=[1, 2, 3]),
+      dict(dim_sizes=[3, [3, 2, 4]], rank=2, expected_dim_sizes=[3, [3, 2, 4]]),
+      dict(
+          dim_sizes=[3, [3, 2, 4]],
+          rank=4,
+          expected_dim_sizes=[1, 1, 3, [3, 2, 4]]),
+      dict(
+          dim_sizes=[3, [3, 2, 4], 2, 3],
+          rank=5,
+          expected_dim_sizes=[1, 3, [3, 2, 4], 2, 3]),
+  ])
+  def testBroadcastToRank(self, dim_sizes, rank, expected_dim_sizes):
+    shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes)
+    expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(
+        expected_dim_sizes)
+    broadcasted_shape = shape.broadcast_to_rank(rank)
+    with self.cached_session():
+      self.assertShapeEq(broadcasted_shape, expected)
+      self.assertEqual(broadcasted_shape.rank, rank)
+
+  @parameterized.parameters([
+      #=========================================================================
+      # dimension[axis] is uniform inner; and row_lengths is a scalar
+      #=========================================================================
+      # shape: [BROADCAST(UNIFORM), UNIFORM, UNIFORM]
+      dict(axis=0,
+           row_length=3,
+           original_dim_sizes=[1, 4, 5],
+           broadcast_dim_sizes=[3, 4, 5]),
+
+      # shape: [UNIFORM, UNIFORM, BROADCAST(UNIFORM)]
+      dict(axis=2,
+           row_length=5,
+           original_dim_sizes=[3, 4, 1],
+           broadcast_dim_sizes=[3, 4, 5]),
+
+      # shape: [UNIFORM, RAGGED, BROADCAST(UNIFORM)]
+      dict(axis=2,
+           row_length=5,
+           original_dim_sizes=[3, [3, 2, 8], 1],
+           broadcast_dim_sizes=[3, [3, 2, 8], 5]),
+
+      # shape: [UNIFORM, RAGGED, RAGGED, UNIFORM, UNIFORM, BROADCAST(UNIFORM)]
+      dict(axis=5,
+           row_length=5,
+           original_dim_sizes=[2, [2, 1], [3, 2, 8], 3, 4, 1],
+           broadcast_dim_sizes=[2, [2, 1], [3, 2, 8], 3, 4, 5]),
+
+      #=========================================================================
+      # dimension[axis] is uniform inner; and row_lengths is a vector
+      #=========================================================================
+      # shape: [UNIFORM, BROADCAST(UNIFORM)]
+      dict(axis=1,
+           row_length=[2, 0, 1],
+           original_dim_sizes=[3, 1],
+           broadcast_dim_sizes=[3, [2, 0, 1]]),
+      # shape: [UNIFORM, BROADCAST(UNIFORM), UNIFORM]
+      dict(axis=1,
+           row_length=[2, 0, 1],
+           original_dim_sizes=[3, 1, 5],
+           broadcast_dim_sizes=[3, [2, 0, 1], 5]),
+
+      # shape: [UNIFORM, UNIFORM, BROADCAST(UNIFORM)]
+      dict(axis=2,
+           row_length=[2, 0, 1, 3, 8, 2, 3, 4, 1, 8, 7, 0],
+           original_dim_sizes=[4, 3, 1],
+           broadcast_dim_sizes=[4, 3, [2, 0, 1, 3, 8, 2, 3, 4, 1, 8, 7, 0]]),
+
+      # shape: [UNIFORM, RAGGED, BROADCAST(UNIFORM)]
+      dict(axis=2,
+           row_length=[2, 5, 3],
+           original_dim_sizes=[2, [2, 1], 1],
+           broadcast_dim_sizes=[2, [2, 1], [2, 5, 3]]),
+
+      # shape: [UNIFORM, RAGGED, UNIFORM, UNIFORM, BROADCAST(UNIFORM), UNIFORM]
+      dict(axis=4,
+           row_length=list(range(18)),
+           original_dim_sizes=[2, [2, 1], 3, 2, 1, 8],
+           broadcast_dim_sizes=[2, [2, 1], 3, 2, list(range(18)), 8]),
+
+      #=========================================================================
+      # dimension[axis] is uniform partitioned; and row_lengths is a scalar
+      #=========================================================================
+      # shape: [BROADCAST(UNIFORM), RAGGED]
+      dict(axis=0,
+           row_length=3,
+           original_dim_sizes=[1, [5]],
+           broadcast_dim_sizes=[3, [5, 5, 5]]),
+
+      # shape: [BROADCAST(UNIFORM), UNIFORM, RAGGED]
+      dict(axis=0,
+           row_length=2,
+           original_dim_sizes=[1, 3, [3, 0, 2]],
+           broadcast_dim_sizes=[2, 3, [3, 0, 2, 3, 0, 2]]),
+
+      # shape: [BROADCAST(UNIFORM), RAGGED, RAGGED, UNIFORM, UNIFORM]
+      dict(axis=0,
+           row_length=3,
+           original_dim_sizes=[1, [3], [3, 5, 2], 9, 4, 5],
+           broadcast_dim_sizes=[3, [3, 3, 3], [3, 5, 2, 3, 5, 2, 3, 5, 2],
+                                9, 4, 5]),
+
+      # shape: [BROADCAST(UNIFORM), UNIFORM, RAGGED, UNIFORM]
+      dict(axis=0,
+           row_length=2,
+           original_dim_sizes=[1, 2, [2, 1], [3, 5, 2], 2],
+           broadcast_dim_sizes=[2, 2, [2, 1, 2, 1], [3, 5, 2, 3, 5, 2], 2]),
+
+      # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED, UNIFORM]
+      dict(axis=1,
+           row_length=2,
+           original_dim_sizes=[3, 1, [4, 0, 2], 5],
+           broadcast_dim_sizes=[3, 2, [4, 0, 2, 4, 0, 2], 5]),
+
+      # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED]
+      dict(axis=1,
+           row_length=1,
+           original_dim_sizes=[2, 3, (1, 2, 3, 4, 5, 6)],
+           broadcast_dim_sizes=[2, 3, (1, 2, 3, 4, 5, 6)]),
+
+      #=========================================================================
+      # dimension[axis] is uniform partitioned; and row_lengths is a vector
+      #=========================================================================
+      # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED, UNIFORM]
+      dict(axis=1,
+           row_length=[4, 1, 2],
+           original_dim_sizes=[
+               3,                          # axis=0
+               1,                          # axis=1 (broadcast)
+               [3, 1, 2],                  # axis=2
+               5],                         # axis=3
+           broadcast_dim_sizes=[
+               3,                          # axis=0
+               [4, 1, 2],                  # axis=1 (broadcast)
+               [3, 3, 3, 3, 1, 2, 2],      # axis=2
+               5]),                        # axis=3
+
+      # shape: [UNIFORM, BROADCAST(UNIFORM), RAGGED, RAGGED]
+      dict(axis=1,
+           row_length=[2, 0, 3],
+           original_dim_sizes=[
+               3,                                         # axis=0
+               1,                                         # axis=1 (broadcast)
+               [3, 1, 2],                                 # axis=2
+               [3, 1, 4, 1, 5, 9]],                       # axis=3
+           broadcast_dim_sizes=[
+               3,                                         # axis=0
+               [2, 0, 3],                                 # axis=1 (broadcast)
+               [3, 3, 2, 2, 2],                           # axis=2
+               [3, 1, 4, 3, 1, 4, 5, 9, 5, 9, 5, 9]]),    # axis=3
+
+      # shape: [UNIFORM, RAGGED, BROADCAST(UNIFORM), RAGGED, RAGGED, UNIFORM]
+      dict(axis=2,
+           row_length=[4, 1, 2],
+           original_dim_sizes=[
+               3,                                         # axis=0
+               [2, 0, 1],                                 # axis=1
+               1,                                         # axis=2 (broadcast)
+               [3, 2, 1],                                 # axis=3
+               [1, 0, 1, 0, 2, 3],                        # axis=4
+               5],                                        # axis=5
+           broadcast_dim_sizes=[
+               3,                                         # axis=0
+               [2, 0, 1],                                 # axis=2
+               [4, 1, 2],                                 # axis=2 (broadcast)
+               [3, 3, 3, 3, 2, 1, 1],                     # axis=3
+               [1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0,    # axis=4
+                2, 3, 3],
+               5]),                                       # axis=5
+
+      dict(axis=0,
+           row_length=2,
+           original_dim_sizes=[1, 1, 2, (2, 1)],
+           broadcast_dim_sizes=[2, 1, 2, (2, 1, 2, 1)]),
+      dict(axis=1,
+           row_length=(2, 1),
+           original_dim_sizes=[2, 1, 2, (2, 1, 2, 1)],
+           broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
+      dict(axis=2,
+           row_length=2,
+           original_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)],
+           broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
+      dict(axis=3,
+           row_length=(2, 1, 2, 1, 2, 1),
+           original_dim_sizes=[2, (2, 1), 2, 1],
+           broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
+  ])  # pyformat: disable
+  def testBroadcastDimension(self, axis, row_length, original_dim_sizes,
+                             broadcast_dim_sizes):
+    """Tests for the broadcast_dimension method.
+
+    Verifies that:
+
+    * `original.broadcast_dimension(axis, row_length) == broadcast`
+    * `broadcast.broadcast_dimension(axis, row_length) == broadcast`
+    * `broadcast.broadcast_dimension(axis, 1) == broadcast`
+
+    Args:
+      axis: The axis to broadcast
+      row_length: The slice lengths to broadcast to.
+      original_dim_sizes: The dimension sizes before broadcasting.
+        original_dim_sizes[axis] should be equal to `1` or `row_length`.
+      broadcast_dim_sizes: THe dimension sizes after broadcasting.
+    """
+    original_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(
+        original_dim_sizes)
+    broadcast_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(
+        broadcast_dim_sizes)
+    self.assertEqual(original_shape.rank, broadcast_shape.rank)
+    with self.cached_session():
+      # shape[axis].value == 1 and row_length > 1:
+      bcast1 = original_shape.broadcast_dimension(axis, row_length)
+      # shape[axis].value > 1 and row_length == shape[axis].value:
+      bcast2 = broadcast_shape.broadcast_dimension(axis, row_length)
+      # shape[axis].value > 1 and row_length == 1:
+      bcast3 = broadcast_shape.broadcast_dimension(axis, 1)
+
+      self.assertShapeEq(bcast1, broadcast_shape)
+      self.assertShapeEq(bcast2, broadcast_shape)
+      self.assertShapeEq(bcast3, broadcast_shape)
+
+  @parameterized.parameters(
+      [
+          # Broadcast scalar
+          dict(x_dims=[], y_dims=[], expected_dims=[]),
+          dict(x_dims=[], y_dims=[2], expected_dims=[2]),
+          dict(x_dims=[], y_dims=[2, 3], expected_dims=[2, 3]),
+          dict(
+              x_dims=[],
+              y_dims=[2, (2, 3), (5, 7, 2, 0, 9)],
+              expected_dims=[2, (2, 3), (5, 7, 2, 0, 9)]),
+          # Broadcast vector
+          dict(x_dims=[3], y_dims=[4, 2, 3], expected_dims=[4, 2, 3]),
+          dict(x_dims=[1], y_dims=[4, 2, 3], expected_dims=[4, 2, 3]),
+          dict(x_dims=[3], y_dims=[4, 2, 1], expected_dims=[4, 2, 3]),
+          dict(
+              x_dims=[3],
+              y_dims=[3, (2, 3, 1), 1],
+              expected_dims=[3, (2, 3, 1), 3]),
+          dict(x_dims=[1], y_dims=[3, (2, 1, 3)], expected_dims=[3, (2, 1, 3)]),
+          dict(
+              x_dims=[1],
+              y_dims=[3, (2, 1, 3), 8],
+              expected_dims=[3, (2, 1, 3), 8]),
+          dict(
+              x_dims=[1],
+              y_dims=[2, (2, 3), (5, 7, 2, 0, 9)],
+              expected_dims=[2, (2, 3), (5, 7, 2, 0, 9)]),
+          # Mixed broadcasting
+          dict(
+              x_dims=[
+                  1,  # axis=0
+                  3,  # axis=1
+                  (3, 0, 2),  # axis=2
+                  1,  # axis=3
+                  2,  # axis=4
+              ],
+              y_dims=[
+                  2,  # axis=0
+                  1,  # axis=1
+                  1,  # axis=2
+                  (7, 2),  # axis=3
+                  1,  # axis=4
+              ],
+              expected_dims=[
+                  2,  # axis=0
+                  3,  # axis=1
+                  (3, 0, 2, 3, 0, 2),  # axis=2
+                  (7, 7, 7, 7, 7, 2, 2, 2, 2, 2),  # axis=3
+                  2,  # axis=4
+              ]),
+          dict(
+              x_dims=[2, (2, 1), 2, 1],
+              y_dims=[1, 1, 2, (2, 1)],
+              expected_dims=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
+      ])
+  def testBroadcastDynamicShape(self, x_dims, y_dims, expected_dims):
+    x_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(x_dims)
+    y_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(y_dims)
+    expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(expected_dims)
+    result1 = ragged.broadcast_dynamic_shape(x_shape, y_shape)
+    result2 = ragged.broadcast_dynamic_shape(y_shape, x_shape)
+    with self.cached_session():
+      self.assertShapeEq(expected, result1)
+      self.assertShapeEq(expected, result2)
+
+  def testRepr(self):
+    shape = ragged.RaggedTensorDynamicShape.from_dim_sizes([2, (2, 1), 2, 1])
+    self.assertRegexpMatches(
+        repr(shape),
+        r'RaggedTensorDynamicShape\('
+        r'partitioned_dim_sizes=\(<[^>]+>, <[^>]+>\), '
+        r'inner_dim_sizes=<[^>]+>\)')
+
+  @parameterized.parameters([
+      dict(
+          x=[[10], [20], [30]],  # shape=[3, 1]
+          dim_sizes=[3, 2],
+          expected=[[10, 10], [20, 20], [30, 30]]),
+      dict(
+          x=[[10], [20], [30]],  # shape=[3, 1]
+          dim_sizes=[3, [3, 0, 2]],
+          expected=ragged.constant_value([[10, 10, 10], [], [30, 30]],
+                                         dtype=np.int32)),
+      dict(
+          x=[[[1, 2, 3]], [[4, 5, 6]]],  # shape = [2, 1, 3]
+          dim_sizes=[2, [2, 3], 3],
+          expected=ragged.constant_value(
+              [[[1, 2, 3], [1, 2, 3]], [[4, 5, 6], [4, 5, 6], [4, 5, 6]]],
+              dtype=np.int32,
+              ragged_rank=1)),
+      dict(
+          x=[[[1]], [[2]]],  # shape = [2, 1, 1]
+          dim_sizes=[2, [2, 3], [0, 2, 1, 2, 0]],
+          expected=ragged.constant_value([[[], [1, 1]], [[2], [2, 2], []]],
+                                         dtype=np.int32,
+                                         ragged_rank=2)),
+      dict(
+          x=10,
+          dim_sizes=[3, [3, 0, 2]],
+          expected=ragged.constant_value([[10, 10, 10], [], [10, 10]])),
+  ])
+  def testRaggedBroadcastTo(self, x, dim_sizes, expected):
+    shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes)
+    result = ragged.broadcast_to(x, shape)
+    with self.cached_session():
+      self.assertEqual(
+          getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank',
+                                                     0))
+      if hasattr(expected, 'tolist'):
+        expected = expected.tolist()
+      self.assertEqual(result.eval().tolist(), expected)
+
+  @parameterized.parameters([
+      dict(
+          doc='x.shape=[3, (D1)]; y.shape=[3, 1]; bcast.shape=[3, (D1)]',
+          x=ragged.constant_value([[1, 2, 3], [], [4, 5]], dtype=np.int32),
+          y=[[10], [20], [30]],
+          expected=ragged.constant_value([[11, 12, 13], [], [34, 35]])),
+      dict(
+          doc='x.shape=[3, (D1)]; y.shape=[]; bcast.shape=[3, (D1)]',
+          x=ragged.constant_value([[1, 2, 3], [], [4, 5]], dtype=np.int32),
+          y=10,
+          expected=ragged.constant_value([[11, 12, 13], [], [14, 15]])),
+      dict(
+          doc='x.shape=[1, (D1)]; y.shape=[3, 1]; bcast.shape=[3, (D1)]',
+          x=ragged.constant_value([[1, 2, 3]], dtype=np.int32),
+          y=[[10], [20], [30]],
+          expected=ragged.constant_value(
+              [[11, 12, 13], [21, 22, 23], [31, 32, 33]], dtype=np.int32)),
+      dict(
+          doc=('x.shape=[2, (D1), 1]; y.shape=[1, (D2)]; '
+               'bcast.shape=[2, (D1), (D2)]'),
+          x=ragged.constant_value([[[1], [2], [3]], [[4]]], ragged_rank=1),
+          y=ragged.constant_value([[10, 20, 30]]),
+          expected=ragged.constant_value([[[11, 21, 31], [12, 22, 32],
+                                           [13, 23, 33]], [[14, 24, 34]]])),
+      dict(
+          doc=('x.shape=[2, (D1), 1]; y.shape=[1, 1, 4]; '
+               'bcast.shape=[2, (D1), 4]'),
+          x=ragged.constant_value([[[10], [20]], [[30]]], ragged_rank=1),
+          y=[[[1, 2, 3, 4]]],
+          expected=ragged.constant_value(
+              [[[11, 12, 13, 14], [21, 22, 23, 24]], [[31, 32, 33, 34]]],
+              ragged_rank=1)),
+      dict(
+          doc=('x.shape=[2, (D1), 2, 1]; y.shape=[2, (D2)]; '
+               'bcast.shape=[2, (D1), (2), (D2)'),
+          x=ragged.constant_value([[[[1], [2]], [[3], [4]]],
+                                   [[[5], [6]]]],
+                                  ragged_rank=1),
+          y=ragged.constant_value([[10, 20], [30]]),
+          expected=ragged.constant_value(
+              [[[[11, 21], [32]], [[13, 23], [34]]],
+               [[[15, 25], [36]]]])),
+  ])
+  def testRaggedAddWithBroadcasting(self, x, y, expected, doc):
+    expected_rrank = getattr(expected, 'ragged_rank', 0)
+    x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32)
+    y = ragged.convert_to_tensor_or_ragged_tensor(y, dtype=dtypes.int32)
+    result = x + y
+    result_rrank = getattr(result, 'ragged_rank', 0)
+    self.assertEqual(expected_rrank, result_rrank)
+    if hasattr(expected, 'tolist'):
+      expected = expected.tolist()
+    with self.cached_session():
+      self.assertEqual(result.eval().tolist(), expected)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
index 2fd31837c6..cb8e82c00d 100644
--- a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import ragged
 from tensorflow.python.platform import googletest
 
@@ -178,7 +179,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
                           ragged_rank=2)
     rt2 = ragged.constant([[[[9.0, 8.0], [7.0, 6.0]], [[5.0, 4.0]]]],
                           ragged_rank=2)
-    rt = rt1 + rt2 * 2.0
+    rt = ragged.map_inner_values(math_ops.add, rt1, rt2 * 2.0)
     st = ragged.to_sparse(rt)
 
     g1, g2 = gradients_impl.gradients(st.values, [rt1.inner_values,
diff --git a/tensorflow/python/ops/ragged/ragged_util.py b/tensorflow/python/ops/ragged/ragged_util.py
index 03f050de51..a832f937d1 100644
--- a/tensorflow/python/ops/ragged/ragged_util.py
+++ b/tensorflow/python/ops/ragged/ragged_util.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import gen_ragged_math_ops
 from tensorflow.python.ops import math_ops
 
 
@@ -229,3 +230,51 @@ def _with_nonzero_rank(data):
     return array_ops.reshape(
         data,
         array_ops.concat([[1], data_shape], axis=0)[-data_ndims:])
+
+
+def lengths_to_splits(lengths):
+  """Returns splits corresponding to the given lengths."""
+  return array_ops.concat([[0], math_ops.cumsum(lengths)], axis=-1)
+
+
+def repeat_ranges(params, splits, repeats):
+  """Repeats each range of `params` (as specified by `splits`) `repeats` times.
+
+  Let the `i`th range of `params` be defined as
+  `params[splits[i]:splits[i + 1]]`.  Then this function returns a tensor
+  containing range 0 repeated `repeats[0]` times, followed by range 1 repeated
+  `repeats[1]`, ..., followed by the last range repeated `repeats[-1]` times.
+
+  Args:
+    params: The `Tensor` whose values should be repeated.
+    splits: A splits tensor indicating the ranges of `params` that should be
+      repeated.
+    repeats: The number of times each range should be repeated.  Supports
+      broadcasting from a scalar value.
+
+  Returns:
+    A `Tensor` with the same rank and type as `params`.
+
+  #### Example:
+    ```python
+    >>> repeat_ranges(['a', 'b', 'c'], [0, 2, 3], 3)
+    ['a', 'b', 'a', 'b', 'a', 'b', 'c', 'c', 'c']
+    ```
+  """
+  # Divide `splits` into starts and limits, and repeat them `repeats` times.
+  if repeats.shape.ndims != 0:
+    repeated_starts = repeat(splits[:-1], repeats, axis=0)
+    repeated_limits = repeat(splits[1:], repeats, axis=0)
+  else:
+    # Optimization: we can just call repeat once, and then slice the result.
+    repeated_splits = repeat(splits, repeats, axis=0)
+    n_splits = array_ops.shape(repeated_splits, out_type=dtypes.int64)[0]
+    repeated_starts = repeated_splits[:n_splits - repeats]
+    repeated_limits = repeated_splits[repeats:]
+
+  # Get indices for each range from starts to limits, and use those to gather
+  # the values in the desired repetition pattern.
+  one = array_ops.ones((), repeated_starts.dtype)
+  offsets = gen_ragged_math_ops.ragged_range(
+      repeated_starts, repeated_limits, one)
+  return array_ops.gather(params, offsets.rt_dense_values)
-- 
GitLab


From 6d14dcba7225d205f2e7834551f42385802aa2cf Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 29 Nov 2018 11:07:17 -0800
Subject: [PATCH 1027/1554] Update the windows RBE toolchain image

PiperOrigin-RevId: 223376167
---
 third_party/toolchains/preconfig/win_1803/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/toolchains/preconfig/win_1803/BUILD b/third_party/toolchains/preconfig/win_1803/BUILD
index 45209d260d..ac599bc2f3 100644
--- a/third_party/toolchains/preconfig/win_1803/BUILD
+++ b/third_party/toolchains/preconfig/win_1803/BUILD
@@ -17,7 +17,7 @@ platform(
     remote_execution_properties = """
         properties:{
           name:"container-image"
-          value:"docker://gcr.io/tensorflow-testing/tf-rbe-win@sha256:bd22c6bfff6afc1fa4304ec4411df2410d93645494117585332a4e2258358422"
+          value:"docker://gcr.io/tensorflow-testing/tf-rbe-win@sha256:fbc5713566011cc27fc3651183a6e7c2fd56fc6f006618c53f8fc71e742feebd"
         }
         properties:{
           name: "OSFamily" value: "Windows"
-- 
GitLab


From 2e87f3502f28d8c112d4cd13fa3f23615b5fad5f Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 29 Nov 2018 11:11:30 -0800
Subject: [PATCH 1028/1554] More tf v2 API work.

PiperOrigin-RevId: 223376950
---
 .../core/api_def/python_api/api_def_LinSpace.pbtxt   |  1 +
 tensorflow/core/api_def/python_api/api_def_Log.pbtxt |  1 +
 .../core/api_def/python_api/api_def_Log1p.pbtxt      |  1 +
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt      | 12 ------------
 tensorflow/tools/compatibility/renames_v2.py         |  3 +++
 5 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_LinSpace.pbtxt b/tensorflow/core/api_def/python_api/api_def_LinSpace.pbtxt
index b1de2cb207..3835661be5 100644
--- a/tensorflow/core/api_def/python_api/api_def_LinSpace.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_LinSpace.pbtxt
@@ -2,6 +2,7 @@ op {
   graph_op_name: "LinSpace"
   endpoint {
     name: "lin_space"
+    deprecation_version: 2
   }
   endpoint {
     name: "linspace"
diff --git a/tensorflow/core/api_def/python_api/api_def_Log.pbtxt b/tensorflow/core/api_def/python_api/api_def_Log.pbtxt
index ac4a4454c7..b6d2da6d32 100644
--- a/tensorflow/core/api_def/python_api/api_def_Log.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Log.pbtxt
@@ -5,5 +5,6 @@ op {
   }
   endpoint {
     name: "log"
+    deprecation_version: 2
   }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Log1p.pbtxt b/tensorflow/core/api_def/python_api/api_def_Log1p.pbtxt
index 5a2d77a417..e3da451de3 100644
--- a/tensorflow/core/api_def/python_api/api_def_Log1p.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Log1p.pbtxt
@@ -5,5 +5,6 @@ op {
   }
   endpoint {
     name: "log1p"
+    deprecation_version: 2
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 98ea3b6839..2a30688b46 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -736,10 +736,6 @@ tf_module {
     name: "less_equal"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "lin_space"
-    argspec: "args=[\'start\', \'stop\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "linspace"
     argspec: "args=[\'start\', \'stop\', \'num\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -752,14 +748,6 @@ tf_module {
     name: "load_op_library"
     argspec: "args=[\'library_filename\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "log"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "log1p"
-    argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "logical_and"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index ba93cbbfa9..dda7943821 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -259,8 +259,11 @@ renames = {
     'tf.layers.separable_conv2d': 'tf.compat.v1.layers.separable_conv2d',
     'tf.lbeta': 'tf.math.lbeta',
     'tf.lgamma': 'tf.math.lgamma',
+    'tf.lin_space': 'tf.linspace',
     'tf.local_variables': 'tf.compat.v1.local_variables',
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
+    'tf.log': 'tf.math.log',
+    'tf.log1p': 'tf.math.log1p',
     'tf.log_sigmoid': 'tf.math.log_sigmoid',
     'tf.logging.DEBUG': 'tf.compat.v1.logging.DEBUG',
     'tf.logging.ERROR': 'tf.compat.v1.logging.ERROR',
-- 
GitLab


From 5839858d0fff35ae9eb58d2697366a97a4944e86 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 29 Nov 2018 11:12:26 -0800
Subject: [PATCH 1029/1554] Make TF unittests use TEST_TMPDIR if it is set.

PiperOrigin-RevId: 223377097
---
 tensorflow/python/platform/googletest.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index 8141cf92c5..4d34c508da 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -104,10 +104,13 @@ def GetTempDir():
   """Return a temporary directory for tests to use."""
   global _googletest_temp_dir
   if not _googletest_temp_dir:
-    first_frame = tf_inspect.stack()[-1][0]
-    temp_dir = os.path.join(tempfile.gettempdir(),
-                            os.path.basename(tf_inspect.getfile(first_frame)))
-    temp_dir = tempfile.mkdtemp(prefix=temp_dir.rstrip('.py'))
+    if os.environ.get('TEST_TMPDIR'):
+      temp_dir = tempfile.mkdtemp(prefix=os.environ['TEST_TMPDIR'])
+    else:
+      first_frame = tf_inspect.stack()[-1][0]
+      temp_dir = os.path.join(tempfile.gettempdir(),
+                              os.path.basename(tf_inspect.getfile(first_frame)))
+      temp_dir = tempfile.mkdtemp(prefix=temp_dir.rstrip('.py'))
 
     def delete_temp_dir(dirname=temp_dir):
       try:
-- 
GitLab


From b5e3e7c66523c3dbb6324b2fb093fb8fc253102c Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 29 Nov 2018 11:19:56 -0800
Subject: [PATCH 1030/1554] Choose the TensorRT version automatically based on
 cuda version.

PiperOrigin-RevId: 223378409
---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc            | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc            | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes.h             | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc       | 2 +-
 .../contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h     | 2 +-
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.h             | 2 +-
 tensorflow/contrib/tensorrt/log/trt_logger.h                    | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin.h                 | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h         | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc   | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h           | 2 +-
 tensorflow/contrib/tensorrt/resources/trt_allocator.h           | 2 +-
 tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h     | 2 +-
 tensorflow/contrib/tensorrt/resources/trt_resources.h           | 2 +-
 tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc                | 2 +-
 tensorflow/contrib/tensorrt/tensorrt_test.cc                    | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 3b32f72bc1..ecbd4ea802 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -59,7 +59,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index fee095668e..938cadc7c4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -48,7 +48,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 // Check if the types are equal. Cast to int first so that failure log message
 // would work!
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 54e19b7395..daa3111192 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -34,7 +34,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 443033379f..4790622e83 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -44,7 +44,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda.h"
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
index 189e9c939b..f658e45569 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index b545f497f3..b801480a30 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -31,7 +31,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/contrib/tensorrt/log/trt_logger.h
index 96ccacb791..58d9d05d01 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.h
@@ -20,7 +20,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h b/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
index 754920b60c..167e8197a7 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
index bbae9fb65c..51393d2092 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
@@ -27,7 +27,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
index 129bdcdbc2..2346cb9ba0 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h b/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
index 274ce42fec..5ded702c41 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index f857a9de05..b03fe7b8b5 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -22,7 +22,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
index 65466c9741..e8f08ad9f4 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
@@ -26,7 +26,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index aac9e5c7bd..0be5d44f7a 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -31,7 +31,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
index f30dba59ad..ad9703325f 100644
--- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
+++ b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
@@ -23,7 +23,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace shape_inference {
diff --git a/tensorflow/contrib/tensorrt/tensorrt_test.cc b/tensorflow/contrib/tensorrt/tensorrt_test.cc
index 769982c645..102a1d3891 100644
--- a/tensorflow/contrib/tensorrt/tensorrt_test.cc
+++ b/tensorflow/contrib/tensorrt/tensorrt_test.cc
@@ -22,7 +22,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda.h"
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/include/NvInfer.h"
+#include "tensorrt/NvInfer.h"
 
 namespace tensorflow {
 namespace {
-- 
GitLab


From fd6ecc5e9d813a6c2e5c74603f891aac19ed2618 Mon Sep 17 00:00:00 2001
From: Leon Graser <leongrch@gmail.com>
Date: Thu, 29 Nov 2018 11:27:15 -0800
Subject: [PATCH 1031/1554] fixed typo from of to or

---
 tensorflow/go/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/go/README.md b/tensorflow/go/README.md
index 3989f9b25a..f53f6fc989 100644
--- a/tensorflow/go/README.md
+++ b/tensorflow/go/README.md
@@ -23,7 +23,7 @@ from source.
 
 -   [bazel](https://www.bazel.build/versions/master/docs/install.html)
 -   Environment to build TensorFlow from source code
-    ([Linux of macOS](https://www.tensorflow.org/install/source)).
+    ([Linux or macOS](https://www.tensorflow.org/install/source)).
     If you don't need GPU support, then try the following:
 
     ```sh
-- 
GitLab


From 7beeb3d3fb3a1471a20bb82c1749196f65ba9a73 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 11:20:49 -0800
Subject: [PATCH 1032/1554] Add tiling-based layout to protobuf.

PiperOrigin-RevId: 223378550
---
 .../g3doc/images/xla_array_layout_figure1.png | Bin 0 -> 20398 bytes
 .../g3doc/images/xla_array_layout_figure2.png | Bin 0 -> 7913 bytes
 .../compiler/xla/g3doc/layout_with_tiling.md  | 159 ++++++++++++++++++
 tensorflow/compiler/xla/layout_util.cc        |   7 +
 tensorflow/compiler/xla/shape_util.cc         |  17 ++
 tensorflow/compiler/xla/xla_data.proto        |  24 +++
 6 files changed, 207 insertions(+)
 create mode 100644 tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure1.png
 create mode 100644 tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure2.png
 create mode 100644 tensorflow/compiler/xla/g3doc/layout_with_tiling.md

diff --git a/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure1.png b/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure1.png
new file mode 100644
index 0000000000000000000000000000000000000000..00cefe4c7806c1c09dd51499375e720bfb0baac6
GIT binary patch
literal 20398
zcmeAS@N?(olHy`uVBq!ia0y~yU@m81V07bPV_;x7*_CF_z~Eo%>EaktaqI0}-Wrjs
zd;fp@8~sjqEn{HtY_5!u6(I{3En2|FtHLNEqNSmg;ULGv!ofdr!Hq6f*52@dEXRv&
zVxl6t8X93ai<frAZOL4!yFF=kR!*|C>DyjYFNOX8|EQ%GsZYLXGx4V9rupArm&V<E
zlUG)jW`45f`<(lh6FC@$D=1;{UK94<@+V6ZkPk1^O^Niai85>l1w6y8b~&{Zx(@?D
zG7i5@FSYERC<BTkhK&8wW)w`%*lHK%y){axofV|u*7UVutGC>G=W(}cw{DpCgAX7<
zhu;MUT6S&?^ES?%ymQ@z!^|MTw})9)_kQ1bkF(;ir3*;lLLIO9!*^zUU~{&_uWYdf
ziDt<Ea?er=j(k`gJ&m*cv84}4@WQszZBZ+)o>D5EZhztQ(P!Jv$*6(EGk%wR{G=uP
zBy8rg#Px@=G-c$#qV~U{au;9y7P<CB)@x-`?pa{_K{90FSKb?<7~2^^dT(vt7bYzH
z3B+W0d!<15-LC=-5NE;PQ)UnIRN9$9oLlSfhTTrx+6xV|`Bf2X&t!RqX>X58=i>w^
z*kb=PEIL^)zyAu)MHaAtvX%F;Z?{xuz5ng<Z@oA_NbCY%N!OJtw|qaw>_3w=+rkDc
zX4iCA?D8I~*%!8V9R};nwZ5^|h;d!;l;0(rZ`G~VeNq5&r^D^V#>wxDbQeziopm=h
za7kK6nmL~=$b=UE>KDhN)?Ue~ocO}FEY8X$xcX<;+G)oUL8dhnWLTWua*J`bXjE<h
zw|siVl4aulw_l{SID@o07-v54GGp|+zUsE*%E&1(`<HFIWM*>Xty%x-$s1qz%w|4Q
z2C`e>S=Ee$poFb^C(cT2f4ejD#ww%f-y?E!m!wIg1#B&f>Nh`F{6{5dvs+xXRi0n}
zvTLHxW-b6FuLZIaxqi<~TXK4?ysh+l@ZpFk@4mfCrh8{SUOP>G<xIsbXIdV|$tW=}
zFf4d`@ub6+IG^g7Tim;68M-XqyKDQ*3E={XCo~xt7%r$~yzlvT;pK<j4`ujFKJVRp
zp=HlU89oLE1_hg1Wz*8oY$empZcMrt?`Qp1GTmG8<p(I%45taLjJ(_{C%%Pk?eRNF
zQVa|X4HBNcSI)@IJhae3`_jGmShgfla0GMA<2$OOd_z&;R`YUa6Zc|`3)gt0&DP{c
zMcr~+JvG92O<3aAyJ4q&e0|@Fu>Zb){BLbl)i15=`{t(K3yqA7ens1dXZ)?vD7|$p
zd+L(g*RSiIc-yjf{ztX2_1O&PcW=HrsgjkkW#MMakLR8QC^-B&=;+z9)c8VMg!+qS
zpBbyS+!7Pc;aU@x(-hY(7nJihe%jSlHrHQvmK1XLrLM9}Tlh!+qHyNje-{=y=X3wC
z6@FEBEOAHo$=A0|o!>eAr}Uv}76*+I!B=Z#*p4*#t=rUdxWP;$?_1Q`JGZNs+Lm4W
z8hzID&kw#m|D{rnZhr3>_S$hr_b%Dp?#Hrf0$g`IW@cc>?fZ7?Wr2^_JU5*ezt^;Y
z^?zZ>sdiSM8W#P&yeWU~fu%1z=hS)K4-hE0tADXFW_4+C{_&k>KX0}*$#AVa<M?aF
zy?|Go^1rrDE{=aMU}$_NxBK1V-<B@i<x**LYA>$8S8(}`aMay-m20kO{Jwd%QqD>w
z==)(?pKH}Q(p-%5a(~Lcc1e$ZR>9k4rgB~S-Iuy|@6Q%ry}m48xZnA0-L84d<?f$b
z_xsDG*K3c<T{!SqW#_kz(ayhD#GkEw{bsq{^JDXNe~$Tck9pm9|4Eham%d1zS?g2w
zJ~;Srk@EW0o8_x=e*KoYo|YS2KR@R9(qiK}^~o{E%^d9ojP`SLOU~j{xb|iKhcZ6q
zq!k|LS6cF9C_h|xCTr=%s#Smdo~-%pfAOAO@uz27-rl>AFjt}P%JH(h%P;N9Tl3xJ
z>YcC$n`LFMg}$gfvvJ<>D`79{=B=ABH8s}m!Or}~*B`n6=6&WnePwdV>!0fkoWt*a
zzy5x?iTRzG0o%gbn|H~1>x=L3-}Y7F<>w9i^orjtKd*dh<JE;*x%VEw#?oVYFZbo+
z9nQVRBG=z6-@Mjl|MT~E{lkBp*u3xEN14U3y<b0A{?2SWv8%suBWF-{t=whVB2%9=
z-`^xJHRsv+?W5WG+U_rZ-@jTZw*UK*p9Trj&sQfbOt)@bcJ<!%J#1Ay(|)}9FSPgU
z`{u5Uhv!1upV#T_n#+9dafWY7@b;DQzWV!qe>}DKed_-Ep6}oH{OkE%`Ek0KgHERH
zNftilBoBX<YXuS)+z!R|y;dn*Jkv#WXX9`Gf<trW3Ct3H&3R5m=iJ{jCl}32cyIpw
zb&B6Ft!SxtMn*=L&UpUJdwbAOF>_0s%<GTeENyw$eMoGVxV!J(eIKD<>oX=%=jWQ=
ziJZRqZ0F}FUg=xPi_RZqIj*&Q`ihkg&&i}4GJ0>neOUCtwkoH7x$8x|epOC+-=bg7
z)cSH`mfSR#<z*M1-&kjUUAMSu%ADMn;r<_1nlu(J%+rp#(INKk%=!FF&okowGrcd9
z;wV42?fC5Dr_K6TUVm$7n^60wjmvudLc8?K*2`@xe%@=yn%6J4Scundmu&cqtNfPR
zW<S4oy>k7FvbhfHj^ExR7hGcftnT()IW7MSD~;as|7}Q|`qNhP;_(lb^Q!jS^43U9
zc=LHzmcZ%@{MK{iJg*zxezz-r@okg$^0w!eo_oBrKK{$DXD!xM?-*Kq|4SIE2yZko
za>_Tn6?Q;6-!JCw{B7d1Cts4gcVMY{+dakVW#;~KBt_=l6U%onQ2(UMRlY2I#hG-A
zCmEk5f^8lyKG!q-SxU_O$6Dt(_q$jJn)Yu$_JyzP6Zf%QdzZCY3!gcXZ|0}oVd>02
zH+d=J!Oi<upZWXCyd)>~_9yFYvrPgl4_>k4^If^VN9^66_uRZLYCLbVoOTG8>#q>@
z@%iBQsU{;}cYoS5zuoC)(&fvu49cALZ|;8=X?M1rPfADo&U@|U*S(o1|Ni^7R`<+#
z@ycz%FY07tfA2i=d)G|P-dDDH&JX?Kjd|nhY>U*sh?=L*K3BWN{Kn?HET0da{P(P7
z_p_P{j0}zmI~bCD|DO~ql(=wh#)rwtRqk4I9@bSEz2N0uc6{YFOZDPg$w__^-(Q}r
zo04l{eP^a{RQiwfxw+Qrf6gWE@YC5`>vv?Yv#~U9xt;oL&u35SgNqN(eVMqrTq(E7
zYu0Y@6P&j98gdS~6eb_PChM{1_!*vsYii}r6@FNKJUeSg_NxoGG9I2ww|~8t@7{vP
zJg!&oCC{(jbN0<+h4Xf5Cf{V{)|TWR|9Lad?tFb`y7jL;rx&qSe{bi0<~+e`|MA{!
z=k2za8|`O)e0HCRdHU>UKc~4@uD!mC+3X>oL-mGf&J7Zm>sMIHux&|Qc3t)MoRD{)
z=WH;&!X;(JS;7>?eJHj_cw6j^8=a}%hvrtEvs4S&nSID>mh`(h2OhpH{cPP?{_eHx
zwQC19zJ2rkbMKZj8OIf`f!r$1Tkd=3I4AGhWZ}cHiCZ2&zA__|zbILrZ^mt}vnnQY
z&lX(p?fs+o>*^iV*$+bBtlX{hJ?}Qp`kr0B`3hgEb7KxpEnsJ0xF9xj@}<4JXL(*+
z5t$ju6FldhWxjsrYSmqSOy7UaS)nGAxFh=hfo+qUILg;4c3Yj%kFLJq<DZnqeJFO`
z@ANx+Z1a0ttoJA1KKuEzU;m{WY<-bV?i}UMOfxUqnHk>6%~!aSx-<T=+o9NHpKInm
zJkqPn_TE_M^2&*;4V>H5XGERxtGoRzr+7{I>|YOknIGo0R_0a-zGGluxK*}8`u)7d
zjg1B2smuJ{Kb!mIa4FNO$&+<TUiF;0aL#M~uN_k~9_B56{?TgYybh=H->d&P^|q;{
zf9k%*Q~t~)>d&iVb3CnQ3BNm|GrelQ%>3tONAF&~;_%{TUSIC>w!XEi#SD)>ugtg`
zx-B^PX58Vq*=ff#1Lvi0%l1g@K0H^r=+CsIf-Kd!&+iB9FqY2?nKA!jXQ|gBUCWCH
z?9$CY{L}kpnf5L>?^E7o$vBzRv^%lyzDI4je%a@O(+*|^299pYMWIP)Q_p-}qkZjU
zsnE`&rJI&7T<cZMXrtaJZ7aO0=bK+iqFnKG{~g+<hmYMk&G2Mrd(<xfMSCaLSqS%7
zs82e}S-x)8q9Ym)^-k*FnHjZglbFwnOE=%-Z~UEdD`)u)pEDQNF1+yAYTHj<+34%r
zyzU4;eo<JKR#)<M%eK=6>2?Jg`A;h33eV4-xx+R6)}5DpZfXJHyVARI*Q8yx=?^mI
ze$}P+XP(<?<1;;<9y(+eZ+kh*CT;Ki-)D?liq}`(SE;mD`P8!8@m{10BLjngLGh)i
zN3%}Vt)Dq{)19qs5tE*~Y(6sQMV_gxRP|rKcXPMy{c<F{xkK__=tJJ}bqe7*eoo(K
z2w%MWZgL*4(z;07@>!Ap<~-%yW}$xf?3<mZ3w>MnAASGij_KXMWx_7kW<TDneM)EU
z%-)v851&~~$=qSWxcSN&W{X`LFC@>qtoD7Q>DLC;d&${VT-s+p$L2R}`25TF<D5U+
zcNaY8QVd=9c;_*h^IG3O+Ds~VUbj9&f2!p(>9ZSk3)^(fRPXG}4qvQme$iuruhDz?
z>WF2Qb=GII_gt4beeUedWv`hhDSfuyd$s1p=JUc&mhG6f;zmXtyVjZa9lQNZFYZyB
zs#c-?`MMYbg94k+`*vgRXL7<K+h%>;WA*dy{VT<9jnp?q&t2GJr+!ZOOlSPUc>>mZ
z8SS`z(q{=@aoRj@&b$kLAN7{xR?Yo1=l87R>pt7eHN7?`an`)IB0HRWBNyDUvn)#!
zF>$u}Y<oNAQPi9$dF!h@<*RG=v~7D`%u{XYtuv!G?kOL~x(h)9xoqC&4Wpml`uC!3
z+tcqdXFaPOgYW6z`)i{9i8Zjz?U{Qt&)$uHzwG0Cms|Ng@U^I~$GW@j@0XZf{C4rZ
zrJMCMHI>{VF-`OA*QMESDu3@OOXXdgd;Nyp?rVZuzF&_#e!*rx(|)U4$F^n0wd+1J
z{8~HpF>mKOex7Z!f9IMUuiH{S|7^R&uZrl|LP?Aa3>VJ*_}ann@yON^_b~T+pbi^H
zxkLMtsL6%PcZ+RUoGZ-0z|dm7z`6Kp#;<elZU%1SWMF8pSoqa*S)A_7#f=q<-dFz(
zSgdI}^KIp9P!EJS>@eFfKk?n}o5jzjRNgLE_x)9o?R9E-{hN!AAK33Iwf^Gi<fQQS
zaZAhLr`N74-1`6J&CjFTrgYo={l_b8Khs)$_nF&ykyC!(YHIzw=I?pE&)aq%ix2&G
zsDbhSzvTXR?Dns2nNFzHe}C&~LBR2^{Oa?+9KGDEziP+VV`uuGpP3o??D8tHXHP?X
z=UHz*bGJy{|KAVJ*QNjWmcFsu68F7-cM0dJ(AtYjPw!viJ>4kx|GP_1H;4WHHzoMq
zy~)$Qwa+cTAFH4=^VX-&pXcA-Q~6oy`R}Kvr{_Q0F3-Tgu<_dAEX%OlnOke06|Ih}
z2%aahH$L$8y?=MvgUkxMWd7W`{&aD+@l${4d6WLO=AX>}ULWziZtc~yGbbiF*VnEt
zz5Q?c(+k@ErytJ0qki>zub}0+*&*k5#PRLjy!+hWPetM(PxYcS>i-<yckA2q_`Iq0
z@ircO-)sWy=2^SiRV2;*^RxH)6Nit|rSGmw<vPCm^Qu17MtAMaf8U$udmN4aemcGX
zo(j7<&%{@ccb|IReeC4_x*rjHf9u8A_$!57x_E8vo|v$+v(-~OJ2%z63X$BI-2Ua}
z{&*oRt*rN#XT9C!+Rf*gz4@Etn&&#Vw{=GRR`B#x>K5;g`2FDP8RPRZ5B#|q7#Nb4
z&c1prYVDn@*K5r#W!0{oCJ?gYvcU3%euw9_?NK-s<8x^4cG0~N!k?;CPcOW_-JQ35
zY4)_!3m^AQ<qyBB_x{gY^R;r%k_6wqYTa8dKe737(3206>Yhj6U;ou+&0Bc%<jJxX
zmnVq3%=o;Gv$C>XHDl>gC1Ld_>l^M%cHF4WR;*8AU|<m7)(q3$dP__A6Vu}sYwH`e
zi+78xuQpLzsJHClxh;pw?<#%jU4MF^`TX_b?@TV#Ui>YhS@-qk0q0nq!*efdy+1wa
z@ZJ9P!XHfTe6=cY@T&SUL6hHJW^zFI?c(Xv!&NtK%A4D<wLd5bTsJuQ^lpus$yw2O
zx#0V4=G0F&XWpOo{e7LnvV(fp9-iZ?*nVc>tIxkrFZ}**x7nS&+dmtwv{Jk%S0}kT
zLHu<=xb%l}7Awm-p80k^J9UqdS>-gB{YIWLL;mutr(P4Mh)i6i)t1V{z@VV^Yj(nQ
z^DyZv1>siz#WeTl_?&5um3ZhE{dEHO(+{$hTOY3bbSMAxeYsQpnH#v3?_^$9oE6o)
ze0q6Y&B7d6W3^`o^)}4feXK_!@N&01s7iA<uw~jhi=fNJ;>MMq#l5Z;9zQK#_w)7+
z&hqy8_cbQn3C@c%SpVwa$MBQSYkwcHxzQ^6ZVQ{gdgRah_t#IZFZ0~cp4uk(E-9$^
z`PTG)kM(hW?`01gfhw07x7UOTyzb50Rg#jiw=A*MdiVEtbLuwzTUFQ|_qu-CpXAKE
zJ1y3`zvmqiu&g+%`ssJv@)n^tPLk7JN|>DJDs2T1FtNB*zb&yyTOX0XW5xOHPxe+{
zKN`E^@~z#|&)dKMaKGRFT--l}d57leO%AAi^zqY?rIH`T-);Gp_c5t0l`SXi+Z(0*
zUyYNmK07ln_TlzzscUY%iM%O#(>Q%<>(yr|=Qn+;cZ-b3J$L7(Xx;1fW2?^2ogFKf
zT6_M+mYc;|n>Mfgp5&3cZR`Kdo6@JXUd=mma^;703#Cq%oxZfRzHoi^tJvozFW!sm
zocNdW?#{{7#I32P58eCE`u%gfK!=N3L~gc(w4U|1i0Wl$c5s*b+p4hpS*_f)Dp<X8
z`IECdxXO3&&HIzCzBBt4@59#J23hak{M+R;@k_~}*aIdBtzXSuZ_l}T^J?Os*uq(&
z8&=i-|8ZV*jamA)o~)m*n}xzQABruNs1KeyZL+cUl>KY{W*evHeU-oeZ+FYpwC5?|
z_WSEMw_MGOFxYRk=D+SOx%x+)%|eS(Pk-WT-Tqy$^4(6^(8PE9MZaD90CKBYRIZnV
zv|Y`Wz)!R7ryq~qarx^zyI0rN{@ScxZ@EQy!iGb0^-A<6cGyOzMf|UGTDdcL*S)#d
z?i%V$E^yzWd`Iw|RmzUt`DbSZ96sC3fA6R3jE8xDe|>FNiCOtaoJDCOb0W{l5UIRF
zHk{nLciz@V+P%EB_1EO&UB_Z~Tvl`cyZ@H!na5cXX_Y%Yii{ri!Hmq<QJwq#-l+<{
z^7s3GpA&DfR;Za8_FCpxo4wt$89#W+!@6G=Tz~2PQS#js(RUKlJi!j(C~w)_XQ5;K
zrr`Jnp9OA}5z=aR1m8Vc^$cb#NBNaIfyM4Sn3=-vOgns6zy5n;&z?_GKhJHQS@gec
z`KvqI{&hZodwYxMzi0Z5ExY-@|BX@W*?T7My#2T4s_p;3J@@yz_r|lab$57uxQ6on
z;*~Lmc~flm{XJfqw>$aj+oG#SHX8l;l=$h9^m$gHclRp43rbq+xvKd{DCzInvFBWF
zw%iQWnKNIO{l4AazqM}1<<bvoep5~?@f5nbd;7n)s;Z{jw@#mBEsW$Lkb94B@KMk^
ze7EQ4_7nZ{>Mri9Nz8Jad;808^Zzk=v&;8sO-|S4P`RUfa;Db$y!#6C_kX`{WK>%)
z_x8NMCu@yrPP9&{{Q9)9b+_hbwUd|i^QPJVd22qY?&Srm?U@@cueYw-8M^N59;*oj
z=GxU$-c%+}t%>=!WJ&s**yr2c*S+6$Yr^(_59eItFPD1b<*gbNs5r6k@vHmm4!)a}
zee3Q9pF^_HI5X6lJj?o!tibe#@48m2o$$`zn;q}>U8lnCLDRv~rZ|Cze#~>(toPQO
zWPN^Z=iB-3-%Klyzb}`uWA}$gj$5an-~Ynq{%@VgdFkIDu?oHmVw)dXeN+0~7TbN6
z{^#tToBaG7&lP2Vr&r=*{q*4A*Y~znZFMPopBH#u*0j9E8tRC}&-!LrHx!5$?U1fJ
zy2DKL-Ii9_cb}#`yS2C4@#-B@v3DgF%f8*YyCoy$&7r&dYK3N6g(@9AYHC}RntJ6<
zBQ)i$*tz&wUjh4;<%u(ygWIm`eEVpN;I|EDba&sIAib%GiL*S)|NF7e(%H>%YYxw4
zGnXw}dDy6(XZMnGxtE{&ZLEIVG$+8dJ@sAf>$>WQXB~pfS?jOApX=~_b@=;<(#v`H
z^CHqxm%6z1{@VY#yGvzLNv7}X`|(P*w*?lnBm1H)l_}Tu&Px}~b4xT=%iYU~zyA7*
z@tgHl2fO0d+g9oC`DhPHo?UTk<_0`nZvSh2mEwE-d^>&a^ceYw9o0|G^{@J^`1Bx|
z|Hk#q6Su1$m|AZAu345pZ`+;s=lE6by#4(*O49Zj({9_{wJU4q3V+J`|IbCT{_N3j
zE!wX8YZY#AhJJgqa_2eOPg|Zo3xc~WIYJE-+6!!3toLuLxLM!2OXLv$`(3J6mI~yZ
z{~x!)TXD^v*T+6gbBfsBSNhHYmL;vT-bqa3ESXYp+^T#{@o}qig*$@pILceBv)--P
z*)8_&(xpp9mo8s+{}e9>j>>h-?bccEILgmi&9q9K%=fxXEw?Lm$K_Jv&FkdT`F~#f
zYO;6x{l9yEelyZ>Y}?(t{jPfO`W@TueVyzct9zYSTDf@rKEn$I+pAyuZu<3Yb#4CX
zxrRR9a?W$>+*up5`A(dZ+w>`wyPurB^}4(3{sisylDab^uiw^PQLz2jKAoBKN>}dn
zTXD_6`uhI=2mRkHd{;HwFHAILM|A0T=c506`>*VL_ci|AbZxiZ$=U1AO{px-<1@K)
zx8!?HWYPUUR@ZH}RnO(wo3tUO%-3!2?I_bbYhyGg-rTHeq<`wa(%P4GuU7KK7H=;~
zb@}$@<Nn-lZ(e{iy!H*=?M!hSuWpN;&i?T}G!zzp0~b}rYbW2l#5uFi<jzZ}nZH7r
z=d4@)a-VMct<p(m6$}3SS=r9SU7jsg7vILaC*oL)&7Qai#ZGsBd^w%^eARpwp?79$
z=BYe(o2Tq89vAn1+s(=CCl(a@O+B<+j7{tv8^7<y{p+eU_MZzf>bKc=F0ZQc(Uk0`
zN6qJDe=S(Pna6tT>Fn!gS8n!u!L6^ieRiCm?5DQ}4&8lw|LpXG;=EI*o_@bx{D<8C
zkDI?eov3{5#OnPw#65QGR!raX<z4kEoe0Ct=f0l4c3;Q-#;!~I{=EHso<r@<OP|wL
zk2l4|?Tb-rZ7t2e{+^v@!ihFs*=^7A?(OMVvfO>ih7JF6Z*Oy5;xGUI_O0l%u~}JJ
z>6e?!5B%D7JuCQ&)xY2-XhxNocJtt&yA$=67A<*rE;8!rW&8Il%^$yW{~bR~e(gTn
z3kBa_%fFu@{=Rxm+>YJFZu9Pb`IH;{?6mUsC+tG+PJLfL>CyG~UwF%>*8gj)t)1}D
zulo6y)otec<*eQwyu38yOLlhqZrl7dlXhAzetTMff6?|kHK%`AZO?u`_ngC;z2%Or
zySIPer!OskGfMxCrLXhd)@HqF_9A~xClqXdx?g@{^IY>gnYaJks*<$JTM=XZ`BcK8
zxozgL+M)XIZv>hjnL7Qt&!eT+w;nFIekF0{)2D(rH!Q4c(AMpZc<v?`xob;p+wKh0
z@~{Q3mpGej6+fT#G)()%i4z`|U;kXs2`MMe%067z+-m*aNIS9JI@ht?IybS+I``tC
zx!Vj5-(9_5d&<M<{F9Gw|8FrNab|C;wd}H=&-aVm*jxSk|MP_h+EU%P%h$irn)Lbn
z?tn78&#N8UK_a*7HeckcNS}AxeO_I*bI#$rJ(5B<_eLvyzY~0U)her;vvcEj%(gI8
zs{VfM%bVPLcMPxZ+{_d^N%Wmq)Y=nSs$nlqZ*PjL|5pu(qQK(g8_tx;yz48Odq$bJ
z-0Drx{#RDB?j_DlY_tBp!DsDm?yQKk8xPM#nt%Bxe>XZk;@Ru}pS?Zzm#lbG^Y!Kd
zwUCR6GZ(wf&5ysPIMd4H^3L4Hi_K3KpRd2aZ_VCvDVAi(cXb~R%`V%J{jT85HLmhs
ze{ZFKetTMc$=vko4_K{qg0tV>Ic#M5C3ElH8HT35)t{^~K<>XRXPvuprQl4Hm5<jK
zo9Eovu+O=jFLv2sv#l>vU+=Q}qs|Jh(2^t69<H0)x|`iBZ|&TGrK+z}<T62J&|TqG
z8E@}Sd4254y?;-f%h~M~p3<+|aC4rGtG@lezow;5bK~aQni*f<4E4Gf6KTD$%WvK$
zoBWcuxmC3jgjZ!OxXc;4h26hBF6+eX;>$Xbn{52|UES{gR%+4hx{azEI78D?`zLV<
zx1G3u+U)j1Kj(ShZe)JA+!>Y^WukY1Gjv|Xm!G#dL%*zMvz_a*^!?1gZyT;pns9wi
z!E)hc7L%8~2<I(h)n7Z&TW|M;c_FT^KQYY?aAlseZt|+ht6qG+<^BBK!OJH@q6`*#
zRYyI4w{rQv{7X}+->sDX61G0?W{cL_%DL9sca}!vPW|>QbNxKgFVmJRTlVeIel1X8
z4N7&1Gv6xQ*}8S9fXM^fePym0-imuZ9oGG(qMfMQy1+62c7IyyHjyU2^EDqfvO7g{
zCBBdTe%)`0(6Jy6>$+cEvfmCxaVh@4xU}g0H0#6BT#nW6zrF9jUHY!<O@~LJ;p;o5
z0$)C@S#Z7i{{~16aRppYfU68Bah*@x@H%IB$s^IzRja2)=59Z?&+`H^IKmb`>r0#o
zu0Q$;t{*%cTfJ&0_oA)aXYE$zs1gmkojNVd<B7dK#3!b(Drh&K*t=W7XJZA{+Frk7
zYW$+q&);7@w{^F#+Kb$+w<@kd&E`&zNHaZb#0hI^T)%iI_Qqi&Yp=7h#?5g}qHok@
z%(IDg&fhEgC+KH>&kvsR*xje@%=vjWaqCRuv=e_Si&r1ccpA3j^563t_1Z5Mi9bEH
z=jY9<iCYf4%c^$g%~QXeH&57WcERVQ%152*c3b}Ozq$79MsojO<FxzrkEN&b+kUwa
z|8ui5v_=KB39tV<JXih2*Mjv6%}ZWfxV_@Mb4=@Q|1|}|&pbQ6-Dxdl`s8_C@9O*#
zTeU6q2e_X-Z~Ml-<Ck~*q@U)g?cdI)|NA37BcQba(td}uip3>jC2VbNYkOyQq%JS5
zTyyR1$9O4lfUcW*`_9hEGtt#|-`jI8?MB+dt6rrGOJ46%`4`;`HgIdu>C?OWI71;}
z4T_wzvZm#Rra3n@*sZ&_r}x(G<Mp`$pccUHGP!q%wmZmM;`I;DU7daX?3HJt+5K|D
z8y1+Vf>c6M0@yKq1@2(u_$%kGDVUzwx|`YRwwi5a2JcCbv}rldX%%p?f$*-}+1fc_
zd*>`0nNE?_Z$Z5Qgncj-t<s-LezF8lo_?~vWQ)tRt6UI=A2zb?irl>EzhU{kKV2K5
zzO9LD7dtaMS}<(3Y4Nj#bKY2rCT@KbX}sp|{7}KL)92=uYNW^cxkl((S3g_$^K*K5
z%hgTksq&|0TCZ1HV|Myhz4n99maBPljGtfl$}JYsay9Hu@p=3CHJ?tJui5{N_12dn
zo&E89D*ip^7K@1&4BP$x-_`$I#s8=4#a5-e|FB;syguS#UjNIZ$v;_4POGhM{dST8
z60{5q4u9ipwD06@yjAwPh{>JBcVYe%qt#zQd1%GX<OnqehK$LK>l+p~E<4e8<;$Jc
zwgUGZi=Xi^Feqdlc%9g~TT6D;$%`5BLTjhxZ{52r(=nr{;r45>l`~b1uV{vY(tH+Z
z+<wvJU)I+SWyhC9q^!KEwPxG37Mth47aY1Sws~#<s5#fW_0pw|9_i=x(Vm_zYQBp=
z7L@o%N=k-p^Iq-Y>+Aa@V~hOjwP8;=i)?*%+?G7>EzC7*n*EDAafj}%uiq1Vi7h7N
z{V(3__q3Ds<n*S;8o0#$d^qjEq1dC=-@Si5$p3jS?8~FIn<~C_I2?|>__}_YwtIZ=
z`(M`6)=dI6#6S&@ZtFOHojbOFud$u{aZdH?y7_xcR>hS0w$;wQw|wj2yN%4I2lEav
z{J(8_Bes~=w48xq!DC6YoQQbs8&e~6+ZSKlm(z3Q%aY2MoC%fMdG5@Q!-U`5I<m&v
zm$!WS99xfF^|~wXebr2#tDc`<6F4vK;{qSK($a*7d1pHQUDE#9QguE5_eCAassD0k
zmV9k(Y~3Bd=fkok{Ch+6WJ(J6-d2mVKKgIh<K^pbDC&0<gnzgswNv4(jpgsAy06!T
zPf6z2pN*WSYJS&t_tnF(si|kLU%B#Y``S3!iFbELUn_P$E6c!;ad_c6|Ej8AQJe4G
zW$E)czF=>b>cm;Ub}KL3TH35+YFcAu{46&t+xT^=$@1s#_86aARlfJP6>pVe>+V(G
zuX(*<z5nIyMeVx3=Nt~t{dVi`!WZlJ*_QjAo?E_pO-x<A!b896C)@tNGrT{~I`{r>
z$8}#US8uo3RvnvQT7Iu!dw%?ps<%o(F)^1nhlja=owNVU5lzdm+qqkVV$aN3I77Di
z+q&MZx8zobM&<qxw3+seuOrhn?Df@aW!o%G%R!Bkx`%$z7apF=ym%<~-l3(FEDNh1
zn3i8WG`G$E?atk0&!!r3m%F-xT0hIazL%evdwa!BW(J10?UF^7j@2*LC~S{so_qCL
z^!@%VPO<5I0mlQMaHUv<eD)C9{q0t!?EQ+>x&2qZ+)3TGbgu2Cg5`%zOCz3b+bzde
z5|I|oTYh=@nW?Ikr<08qcA4MF%)fU-+WqGA`VFN^W2?nOb`)Q~+A9_MecPQFP+E7m
zy~MUCEJ1qR-lA*QO_!ywJG86jz?#JAlP6BRIA8f{Vdnd&O`M5V3r{f$Uy5qpS6DJ*
z`MTWeYHev|>UUl)(LDF`o~{4J{dQX_md4DRlz0DIw|owt$(>l^Uwdwyj!J(XqgPqH
z^vO91v5FnDjWWaDK2@Baa^hb2{fU$Ptz6I9zjxaG^1>N~!*hk7Y?(X1s_$><r$evb
zTqr-8Dt_v)eYEW4r2^O1CeO77t(|F#Zn|3a(ug(8<Ft%bNyfaDQ*UQhJ=XH=k6FlB
zcGK$j(yzvMEvi0hY6o3zo_@Ehf91lw?#zYkd#35N7Q9}vbD>`AhIi{Y7wWYx*qqX~
zQ>5wKq?1<VJf~S07BI7JRavbX{V=<s&2)9IX=85Z)v$=^+$LLQ&fA{!wzl}|0nw=3
z#XIA-@vXh$87L7axxeI@*M64!r(1S6h25LiX7$!GLgHIr{jaI(k36?3e}Ct^Rk>oD
zH7Iy9wkxh_Z!-`+@@uxjv{zr|%`|gWSGkxLJ3qf^=C-$L9ip=umaRRNl~i@(%kRC<
z%4Cfq6Kszey`6aZ_|=C~!@Lb|FaORPnAvdqRdMaCow<<=w@bwQ-iv<fJD$Fy`mdk$
zoy;A*+Y7#L-frgcHPF3P>YdZ-ovT+QM5r+=h-FdtfBlp%aGu#M(N(Wnwabm2#kgZ~
z7i;I-Jj;`=Q?|88c;B3`N4wUZ`26|Y<8|fRGGpUj*yz6vES|SL`@Yr3Q;WC0?Yg3q
zWOtx+?zd-eO|O6F%-CHS7A^g(@AZZ=%&Y5!>L0%A{P}75yt}LUqZOaeU9|sL=#J{2
z>8o9xYh(CM>*$HSyxLvBzF;qV!x9a~1y^T(n`9!O6?N;_ua=2v=5Nn_<TmIv&HlaG
zhH?4EZI`7R7HzSe7x3izDjP>nDGtA_XHCCcSfBf`^4#mc-|p_RJ6$ZJUJ);EJm=zH
zy9Mn$Z~MN@%haExSG$0F*6Ud%s%ke<*>}Xw?c8irlww>YH?em4Ts76=ebV(0-%Zng
z8vcAv>5etGHwOJVv;L9Roxk6I&3^h}`v2f}Gpr2%9lGnS-??O&k&f_1zZt@_D-PXW
zJyU(t;^>M`yEdq*ubVgdhUr<G&fF~Rp2XX?TZErQm}V-kH97WM_Q>m9e?9KE|N57A
z@_lRM=`|<!cc*D21lL_>H+}e?eQ$_EPsV{L!IdjpFNZyNd6K(}k6l*&nxDO?iSH?W
zIl20W@0#*YDJQSD*z*0%zE2O!KR?#G^EdNW#?Skg{U;sQ_wC+4Yxh!b*Sx(`3&ed4
zRvAw@5#9c9<<WxeGV>>|PvxC+A-BJ5Q-<A5yGg>IH!Yj<VwR%!S<BA5w+;zk-|8;)
zZjbY#sExh+3(SvifBA%K<Ey1vOTIoZmu0vxV~$6Mv+@4frrvzDrjGx99M=6P{4Q=!
z=o9JVr_OS3TFvEF?`~&Zop7gR_uT!}K6bDFnf`qI=i{f}mnS`5|7=};{KUJvA7@1@
zo2hn7GS2jbdPb_bLCDG2=8fk?3%q6-Oj&V4{mBp38KHKKDW1&p7xedtF>P^QyKr*x
zzq<?!4F}HeRWh}0U3g-yb-dlak~KAx>MH-V>{d+gx>L9H$<uS|qD<_=_gO66K4ne4
zmfoSew@+_=vY7kVN$&kOHRR<lyDrh_YfH_$t!9)l;dAV6u5zoM<9DT(nF{%76pJ3u
z|K4J-oMGP~(H-9^nx|DqOfFwBX*t_seOn38LSFA}QH3)ezN?q3bK|Si^}M(HWnZ8C
zyLUUno~)VddM`S=efOQ#?kCNqv!6cbw0-Ko|Hk!&hwo~izpEWDbKAAdes+HS|Hr4>
zJHKo_6y_?^vinldL^<0zal6e-b%L)yxcFd+ZP2+lX=)X3jSm0U^6@DC-j-Y9G&5M{
z{g;ePh9wMg3=9e~cMtKEOKtl1=dWqd)wdR!``+&BlYS?6YEygisnw_7yq{aJ-0}6|
zndUbS8z)PCNRoV4(sst*-!`yrcT(Eh(r-C4gV$~_)%wP$zs$5;>dltpc|SkhT78J4
zTx#2z2UC~rO<uM=ce>=Mh}5dvWmcPKXGUE9J?oy4lN1AkgSAlAj?2pqz6+S1{N(+0
zzsdGFH5(td>@I&NKGSfG+Ks(M)o;JPD|pUg^Y&o(Re`e+t-BXh7eD!YB=6e<pYLDG
zcC7BY0+Q2y)nHoQ+G=WBW;L<v`LqS~+f8Q7Ozew3<05~HdHd}%aVr?}cU<fE_?(}C
zp`qaVtA~EG*HwlWUaS0c^t%6)uFdSH_WQrrp8wrrTYE0sMY~s*T$eomxb}YA7txbm
z;xmo5s9oum&1jwe$7<nu>33UR3iz%)_<PUy?N2Tr%nYi$#$Z_yv`O|?)b83lUnQ!G
zCSS|@zhlS#J+r^cT28PH`*&46^3?fPy6bb}!t}pfjeMTB`KsTRTQ9?`FJ~=}(zV?2
z&GY)=)o+g;iQYZ8X48qC&u^WUJt3JM@w+JP>?d}+jrtw;)r0xNANU04RlQKlxbXPA
z3<CqhL%n4W^Zxgjop4{gZrb6w`<3Q@w{cz7XB)T5`ft08*Na<rhho1!b`i4buK8*2
zva8PZ$qW7NnAt&>tDnET7`??{_busnUeoXE&!1nnOy*7Dq?&un*M~%&-w5iTIN$Bd
z{r&KrbZ1cMm+&Va>PsfA(u<#Z_?qF5*H1M+ZQgG7<J*g;Iy02HpWP4m8!zk@y8pKF
zNv;^qP4+)sEH4;){+szBw0hxZ|NWm*UstMasrvn7M$0w+vg0z!NqfUC?cIIC@_B^g
z*OG}#R3=ux6I{6}`qB3<m;J-HTzfykuWZre*)wKY#P(~q%ULooFdT|KbN1{!<>EB`
zEw^*ceJ1nw+xo@*zczDI#j7>9UN`3$r>C4e%bm`8?cE$h&#rx6eSaE!wAfjF>uYIV
z^dxilXXnmNob%JN=ycD^PrHNtCw|}muQamwYQ*nd3ud#OX8pX0IeWYBY@Pjfd$&)K
ztDmE@|6^O#_Vee@Ze+R>v^tVeW7EN`P($tAr%#@-{4;5_yRSa`r`o`YCprW_ozJiJ
zF3Z0aQvG{zzum9Q>_7izESzq2@?_ZD6D{_s3=9k$x_9ncWKWDKd){vJbJ^Zcp3{Y&
z91J}D=6&+jv=raz=jWZ}+ao(?s@By`&Y+(kc(-2HJ#})`EW7e9oj7wRI~zmCI<sBd
z&(yu)nz^;I#OKx*%ivZ2zwFK7+%>h=XX>hToS*L9k$x2ue`WP_(Jeu1!lXW575Jp~
z<Y?L4-BY!*v)+4oMde1MWo2belB}C~biM%t1H+cig7TlA_*3f77bnZTNs@e*#T?h4
zytL@@m0Jbo>Cbq}Hr$?cQ$KAXXs(pAB7RQ691;C_dG-HKK4ARxu2cQkmhJ2OcI?*E
z|MvP$=7GzR#@THda)mpjL5-eu(#PlPgRGcYuzcaWUOP!c{<4jH<+oP3eVgpOa{2YP
z-Sg*O_P-}_$M_j&D%LOh=*+!u1HNyUKIRd+U2l^~agoMBy)6aH-FgHf_wEXv=Ct9^
zU9Q|eKbaU99A<NrM@DkLc{szkIIgd|;QP&B<0ZdZ^gbQEbUO9vlh=HIU$2gz==xf}
zx4Yo`bk@mFFN9m47WeG@^e|NU$+9&w7q;$py{>B|;<e+l)Sf3PH~(Fpzpr}9e(Q4^
zeEyyShf9mxJ2@|T(Wv#Y8V}E9Nxk8lKTAWO&*u9F@4x4HRByBjzS~k;cFVJ9MRrrI
z<(UHU&$Fz}WoPZI{G1>C`1^4ki+!iLDmwp2GBYqZta$h?y!O|rSK9A;R~IaQ`{CsA
zlj^a%L)P#4_2~y!`Ch+C&7H^F<lh~Vnzphev*qRG``K&nD{9~Iz36(%*vN?O!*U)5
z28M;LyZ6f%uko4xfAtRTa`yZ^9;^11OgVFUm)*KUu~#?7K7CMl{7Cr@ZJjCo=PVxI
zU;DE<MND7+;=Daqu3X9c<H^jxpm68torjD2Hl?5L*<SFy@}Gre&6WScv0RGt4&8nD
z{}2C4t7YCV9_nqGWgQx$Gk<Qup@Y}{#a;?XZ{1VFTbZW#`P`pOh$(0G@Bj1kLR0LF
zhknzYFC<8pKV0{D`rDaCk9R#=w<qR_^!*h)ugXtxm7WaSp>3e3-#vkyfx%%%cjqR(
z<400=Y<~Ujk-?qJ4cWpE_0A;D)H+b`{KSb9FY;S=`-W*>mb2xp{j(Bm52*9nvO6rK
z^3O;1<geDtyj6K*tx9I<%}&_5Htd4*x%r^Q{D)#S9_p1olzVxve$JelgS|!1M73Yt
zDZBT?_tNJ*oZsGNn(z1Bp?qiS-&wmQ+O5M+yk29R9hJL&W=-nM(pdd}e?SxB?~=sc
znOw*`aQXCcHU2flx3BNB4AG1I*SO?NrKg{)jrN3s@Gpm7KUsBfZ|bY1wNKlN+pg^X
z`}OK$)$%)*jjg-0-}_F^=X<A_&SM+dmhyFd@$9<$`9Cx-{XX>W+utUR^4rVeR2QFn
zqx<;0o%1L6<yUv+{yY&f)8>PY<mLAzno%ZOuO7ZT?}Xe;k(nyN%E778Dbagxdn>7a
zd;50PGd;J%M$$&0!O)j4U#>1WvFvJD_KVF;aZL4Gj9bb-tDXIQ_s+@U(}%LW^CN#v
zKRere{l<Tn(z8yVKVNS*?Y@4*h6i=`SQ%~|-Ec<H{Owd>8}51X^{-mN(|&sOQC%`W
z0+-ePd{lS$^__1|CY^rr$#MCqubH(>+~wkaHo<u@^($+4y6<0qb*FFhTw#XuXFT{+
z?&x~$USPI^_itL!+}mP2k{R`}g+B#$ez%rfEBCHQd1sL1$}RfK-GW}<1x-bsFuihT
z>jcpkR*{PNTllVTiE906wQ#ThAtPPw-5F8W=4`rf>rAC+ak`&A_v)v*bN+8P@Ymb_
zeee6dvd_MypPTpb<d>P3PaVqIoSqVzlJ2X|z4~s|=d%mjx>HY2YyI_0e@EiMgB#ph
zGpZkbROf5^`Tl92^}7o5Gmq<^&n@5QyMvXZym#9{Oa0r9=d8aNi9I}bw|Tkax!Sw-
z^XBONmfxp%q2T$ugVwnZI`?glEx0Q=$?WqShxXKWa|+kqb$-70-@%X5Pu{fMr?*~z
zhRlr(XYNkC`?cytn0wgoIRCS;-OX`}B)9E6o%J;A_KJh+ZP{#jBNDHD@|$b8-fGRm
zd$-<2e~n)_FUIcU5n=hZoquO;-pto#dMEbwoHZ{E<v+d4>t7Qu|LXPsJH_W)fBkx%
z_AIPb^4+yN+tsf8*I&Q)TU6nU+y8%_um4v#<M#GL_oVz@H=4=4D><>0ZTmy-{~_<q
z-k#*HideQS{%-YD9^2K+MgCV5?Y*5O`0m)<%1)K}lT{{P>;1r6mL6LcwZOF8|E`6f
z(XTmL`Oo}Tmfx1oo0<6iPAb2gfX9w%%}<khx_TycdAPnU{jKyn#Vw^x#JwvuCS}eP
z*HbHYX8&C<J-00<Y|Hiq;_n~k-Ta=<|Mkfyb^D(u*%R~Zt<%#|ZQ6JL+5N8T_4WVD
zJk3MfB;S48w{5%8nZNa4F1r7hJAYeT_675T-1gm*+0wkW&wGArZ;79O?DX~DSI!Ms
zdff6c^V|T}=DFW)Wa`ho|JJd6_r}wD+p=FR%aw?HzpdEq`?M)%>Kgr4EIOF?CbBl=
z{-c+r>*}9hn`!j6Y4h9Dx0m-%e|_%l?N<-;+*8_Z-)zzf3Vr@-k;g=xUY)Jc|2^vO
zSJ`B}i_(4Z^@8igSlg;|7tWeRF9!MJ@?!gSJU^dgnLpIqQ}BKB^LgE`uiM`{eAivR
z_KV|*_4R@e`4sM$?*6*_)4dtGtqTsm{d0ML-u2s-rsdP;O^>PGeeLnt-{)>Nwp#O+
z#lNpzQ(~`tF|TUww@1E}PmC<pUcR{XdMmqrN5OL2GOIU52}R|*cV@reXKizIUdGc)
z#(tCI&&<2=>S3NsO3M<Lpf^?4%eVPef80}AdF9U0nitRfguiT?eKJCbv%F>ZE#uvb
z7TFu>7}p)X`}_TV`B$&yzil|PA^Y7SzVgKU`H3?dPDZ@dz0<n)+}!(XW^Rn(I%u7p
zwr0B1g-D~rQCx|!DYCEcZ0%hzeSvL7+QKhf#x2E((GhWtv%pil-2A5HpaI;5522B3
zg&+2nzKeL?S8)B%g6omu?%NODt=b$>HF4eI=@Ta!Hh>0q4;!uAnH-V!=gOYIV)iX_
z6K8@(c4PN|=8KzBS@())-&wn1pWFr3%WKzXuURmCnYX^cyA?Z?!Ly~WcN|t~on^hi
z_eRoI&KB!Kv3Ee)lG1K$NUUA6VEYlS%IY(VUN)ZJ<GBMg%`YwonuAV`P!rgv8Ft$P
zJiab5P4dGB-}Gl^?>eqq3La~|a;LQ&<Q0x`haV-&b{DNy1=SfKFC~e5`6K2le^1KU
zd?jezz^vUrYX5>+m#Y>W)4B{ANzQsFF-_9#!e!UX-(R)pM6F-Gw+7@U(4f2EyIa>c
zcr1Lzml3?Td}-A@*B#1tK+9^xr}gyo`~wxmKDBX<-q$WbJO>F#LmlA^>q~3Ggib#M
z&v%%XcU+o1{mS0z@8*4Wzh2Z{+in^5TK?&}xeyyx?99&FF<W+~?#_uZpt4_dVprAn
z`%*hY=XKe*o{cpIg|V&WGWPZSkm2NqdMUpgFITKyyY<#T58vDmVc|C}$L-}>eK&9G
zxwE|G_vOm>f3eALDtT}3r+4aK<lNg;yfgKe$C$)|s;~_O-w#_lryM&`S9JBky-&|(
zWxu|DE;UX;|9Su7XL<_~XEGnJ_dd7ljfrLMgU+JAvwCj5UT#w~^UJ;syT8a^_5Q_c
zuezaNd%_6^P`$PDOV8yeUs)$-uh)+<33Z!$GkRHU!8RTFnO<jO84eqLcmH;J*9oiN
zo9gb$+>Ckt?U2Vpo-UqAyc2n^+}XPMt+9^q2lHzzd**l?=$<Y74ip#<IS$QZv)y>h
zQTWf7{IKWC+IC;oo0(ki8~=&@W#PFWw_dY9-=m`*U$D4#XL(fK-UrvC9_D=$O}`o6
zT~{&JN2<(i?(Jo9x5C@zT@8-gk<4RS&e*a$Q`+s6<>Knw)7S6#>SX=Ga(nNtpG?24
z_O5yO?pF2XDBC-k4X1ZLsrg-Wu;p}W;4Rjz+igo9TkL-MwA=5k=h;|>hvzawCu&Iw
zODeM~v1hH<?w@k{%83x`8(WWqvcf~Yj%CxYmbqn#S;Qx8-52H!8q^2PU}QVjB_Eo*
zE$j2vFUK-B|GPS8Kl9&pd&1A{NpgMrJw5*Og4(&4cQ4mHkvQ|`<)AllHs4GCR}}5C
zG0|Te+qU~#Sn+b(66McqKiLk*{oSV)DXq_667lTE=JOMd=Su|Z|A{Tni{JThPu-O}
ztR_0a?MZ$;K5NXs1hr?{NmYi3xr^OiYZT;pHrD!<-C?7~W4v8g&Mew_pQk?R(k%Xe
zfz#Jry_}|NVPRXHxO7{l@pJjEtKTCx7xSN=C3yXz-=ngKXDzN*6aRhwS)TML@!u91
zL)l4{C$ip0zl~h2+jHd#Z+W)g+1UGr<=3UFD$kYjAJ@6FcE?AhncwGbuFcQR+_CuC
zx|a|As^140{l2=p?$R#7qEnUkCu}^e$CnfF%zFE|E$yjw<rQ7;bAOBZPw%dttFr#w
z=_SwBO+7sK+l}6io(Z7pH7{B=QrBDA`>mX9?ybr0XJao17PBAOsqbfFc!h7uojH-4
zDjv?6vHi}RnOCLWJv(!Lnk9GA{j_suSIx7`pQUqZrtsx&p4Zovot`;SwL9(Hl-bMg
z%6RSm+7`V!zUt-DTi+Yw`ksEDucj9E`F8!05*0`D{nE3f-kp=!>f<M0m%aCOu=u-`
zveQ;>Ef+smqc@>od&M7y-PiSQz18I{zjDWP?Xz{~Rn<>!n8WyW-TuPAOUpLQ-e6^W
zfiv{j7wcHL9a%1WU%!{PUb9WzzxsXIxgFDH?GBTh_S5MJ>r3gE+AAl9?0)(EyUV+k
z#yr8*U!sf_a)An8#=^Jv)#oUD%sAdBJN44a;Edb<zg+hJFL|nM^5nN-x4s>|SKpR;
z%q95n-SB>U*FO8NPP<<|(RBXY^<+uo%&l9?q&|H4wLmSza_{ktMTPs{l@+V`7hb<-
zyRX8fTP<YavvnT}mY0?spEpC{?s<<b(YMYNr~LTy`N^Kb+Bfs|9C?zQmA|_xHEVUd
zHE+&xRXJ_u$-8gs)|bqZ{qp7Y?47}HI73T3mIY6DKO6hH&3gZ`y%q0nSzp_kZJ710
zp(1Vmo8!wL862Lw_piX=*o@o!)&DumbG7bNMj!kBPvh_UI)yDu1-#_uCb3&x+i7j2
z8_VTb{pwHF`=qTRhq^bfo2;%Dd(x`h-Abvw^xcY`%mVu#p0f?nn|J8$xw+PdC57`3
zLCoZx=9DoTR5eAhzrM5aT&xXf3DSz4%#WgOn3gBD?v}MK3u%#j_vUT8#5<?kbvJ(d
z?@+#TbU|S8^)0h@_es55u`~PThBMc)Ky#SvM@+Nc9dZQCQnu{wlLjqvdSBpvHns+2
zCP%qL4F72!LDTX?`FhZtdV%<vnX%SucPii6$_J{9xQl=DoMw4sb0@G^+z(Vqfo8Qq
z%b~8^DLn!z6HLn;VirI1J9N9Q<6$3Y$=$=gS-TemfaZuy%Rla}O^i?z(1Sa`x-FHh
z15$Y@E(9%XJMwkI8G*oJ@oyj1!SkdC=iCe|c0U{20bcm_zsW%c<jg~y<yXK}k^5O$
zkokQQ??Cc=9objzcvjrzHw4$VplV561mx<)&-CiH>+_suIplga)&iuoui*HGGYlZ3
zWgqIPfGW#|9dgqom9pL)vik>`dVQ!j!x+*&xVUHK=Y0_cf1A2C+_Kev_w)H=f4iS;
zUqiUIynGr_m*2Yk`y<O3BOT$6&!Awm0QtHtl}+kh+L>R+ugv_cbtvm{+BtiZ`fp#l
zHr)EYf4j<+$)~H|@BQANm{*_nt>^smbpN#+tIwXARi}}D45UW(^gd9d_y9A1%dcPe
zORvX%Kl=Ip{-&m;^usp)`0ai?uujaYe|c$X`KNdR4nfm$p?lw)dBANL(2nJU0mb6r
zsA6VdV2Bb0<*&eEcF<~ba2WN?vIgyX)?M&S?_Ab7A>oxfSFcdG!w8yf{JF<_`VN`N
zKMzgQo%(61{?Yiek<ZU5sn3_WZ(9EQ`)k7==KKFLFfhCba9U^0eSUrY-*C%U8oR~S
z*JYhslc26XSLD8Fx&9snqk;$Of4(s@Ff2%J-5s|#tIFun+Gp=xF1Meuxc}PKy2q=Z
z<s9Ldc-g=I{jA-0Vtzi}Q7HfMHfZ#b@9E>ydcu`I9`4{Sf4_XW{FH^xu6cV+_hp?s
z*0;6#-J@;K)~!0V&hpPw+wXhM=i4(dFj!5w{`|bSq^Z`vwv=g8J+EGL5AS9>EobW_
zIZM*(>`XRk*=aX7FAsTl<6~ybwL7^t6ei7DHJ$6{TToZLJ!$@j*B8zlI`o6He6m_+
zO8WoJ--`0`YOc)5<j<|zbz)ntmC5IBf!Dl;Z~ng&G;xVax`y6E(9UmJojWVf&UXFv
zg>xyp_OruA;p^Akyk7tJwe528^L7vGU#;wmEnZeSbr;W}eYM}V*UMUcDc<>P){m7t
zFY_A)1-VSq+q%E@+01ml4^U69-F|PC<<GgcyO)KlA7j*C_Rz0+F5AvWk9L{HOVyNA
zJlj<&yZPCh1C6sBRyzyzKA&H?>=2)|)CVVyuZDN)c|TZ*Toz5^EC)qQ!MviXHCxi1
zndh}N^qIK?SFhf(uHb@#^^KVYfsy(QU*GE+rTLtF+80#3Z0{zi4BN@iZrr|pJ*Iu>
zx3|)tuiK>kUjP36la&3P#S+s%yJz~2n)77v&TCWTWx2I_sp10Li_ce|kDkB(U3;7K
z+pVj2%OBlmzx?pGE#m&p^S+g^ZvTJq%AC-!Yb`gc->u|aEtT>8O=iXKsOIJCcYb^F
zvhY&vx-~iTP0Md_ul&8grt<7RehC5H9YUbV`wIsT&lN5*dEz@KqWIE!?z!1}r+sVK
zQndQYgR8UpZ?26#`}ykgwR?m5?WA;9h?q-EFuB3H<Ns9kIR!he_Ah@|{rsHi-u1?5
z9VP}(w#J>Ez3-XSr)7R?Z>HDIX^8vm@ZDZ<VcP$#;=3CErh`@+Fj&n|pI`HdH}~4l
zyvA)3_4NmS+Fd)Jcz0K6Jy+p=Zn?@Q6U%SZRovcga~hn+h}$ScM^y_BOInp=#6?9#
zB|7Wc+ScwU{(t4l6_?k)YhM3SdY%3~I^LX5+HB4G`^nMUQ=@X1U+wz(@7npbo9gzp
zKewA5b35|sEkFL(zqej9`<8k6^tIBxvo`r(`zy5OxOT^-Wk24Q3mgxV58wWH-#M$W
z+BMIwpRNAWpJILQ@q628RXum_A8*R@-+nvn{`YI2y-nYHZ;V>{+w|kk_a3k9qSvlx
z|FSnDHg5g;WbbX+xyv_Ko-LpK_Wj0v@r&L^{*z*4V0hrc?9pg>@b2Y^9hVc*nPQig
z-j#a&iOcrk)_v<wojBp~+2)<w=N-G({?@%Lob=an_0*{T$!~VepZ7p4`uk4b!wYWu
zZMt_p>*=|&HK$d>R&UsLc|WK6JI%D)b!!jYN#AmFS#6X2g^g?P&ih%nM^5|vKXI9p
z-{uy)*?iaOyrEvt$KPJ}t&isZsLkI$|I6N`4_7tv+fSSMI5Oql`|K~ZEA6vm<*wXG
zNOziRcQ5SozuSxMXJ@E902Pd(hvr%rm<Y_;a@^0xw(^#D*ednqb`Mt_`22b0vtPZu
zto*_}tMAwbT`QaPr6sDGY4!&Ho4e*_*ml_|s!S+YemsPEqqfGKXSUbY3cPzIYrpyK
z5`)8Y^X}jFzJ2W8_tJaqf!`Lk@8e&)?ewlYCwInGzx6qA`}UdKwApXdD%Z_VVv3Wi
zp9m_wR~IaIe!K0r%`vxfhr%lBKZmBy6WCo{BE?ZIy_jX=Cbc({?|R=j@;Be$PNs!%
z;>zIQL*3RFD)rQ7K3@LN;wWeQyMpw#MSWKj8+>0LHME^`i+StuoYm3Ahvw!vH}5LF
z-Li}&G5^EGd;V{Gzx<B;{_$dR`n=#R)1LhO+0)a*_-oG>X2!Q4v+_1vdA{uUCF?2S
z{Sz(oq&98a(Leiqb#hbXUbkO&wjKSttF-%@#j{^>&(gQeniu`PdgbIwJL{j_EDQ_{
zGg@|Qs9&0UCg7~l{K>cc>~62VeERy8IMvI}5z8ExasCcjq8Q}d$fhYQAQUb1Zq7Zi
zTAuS?pFZ26T6t*Nye0Z3-&elfTe9tOR_nXR`AmgVUzf>#Gv7Ax?;7JLA=PZL*Tv0*
z7A&{;{UEFQPt&yNl4{XUf8N|X&mq!U=W=PxqODHlD~#W(e}B8Qj`8&Sy%}x3Rr7EA
zJ>O}$cX{1$-KdVQ+|$$kzu_0~@t^liOMJKU&d>KQUcV+<^w&<nqvVhpI0g!iJUsVB
z?dR83KfWov<XqdvUpy&#sjaH-jj&}m!|H+$&%JhI=G!;9UUMhEWL;^pwc*`|pr><E
z%+>GATv%DvwC~VPzxmCt3y)X(Kbw0`seDKFn&;;@4XVy;t-EORy=Z^K8_vmJtKLa*
zmp`j`RlChv@y?eAQ)k^;zv8yZ<n9h*&JN3aGt~k_T{z37HU-=29=$R@bZ3^;WS<88
z(ku(#f?IE8mq|a}xz_M_`P!G3SNK^rohiPZd?<8s^*gB>=M?fKifcJ0yx)`W+xzle
zTi^1^JvE;zgnxh|Mwzdicgq<K`|X!Y=g0Pb{WEiN<?`~y(ZPr3zHBYtwt4Hud8)-h
zx@k_+G^Muqti3O%KE*`+&dzzxxwTX6+A3>!Q?l<KKXO?+a{0Rr3ujrsZ!tN?S)aVw
z$nE9SY=b*9=WR^N@1HXB!&1M5uSMSurnjAwNmmQX-_iZ4F5>B0`{K6GRm^+##h<B8
z4`y~WKiRTl?!|eVe=~FJpLJoGL1J+HZ?5;%afdf?u3q`fHuzq|-G$m2;$~t@j0_A7
zYOU6`H?K`j$nQ3@UMjh6TEeS!UVAquxN(*(y|>{~#E#;sFrUAZ?c6+_ZFQgRdt95f
zR_M$&EnDa3QoQAUFACbqX3f7Csb!uSP-I)Tba(j%%ZKNxlivGmdML|RwdCQs7KQ2+
zi~RF%Wh{Tbdg6q4Ure55ua9OvEY`i)=DEys+vhv)y`1~iZhgpR7RN|or=YMY4r=qB
zD$bI9cjusAfAjfj{fl30!m19ZOnE#ZvuetBcWE=w42FsIovmA*otqQU|BY|^7wg8a
zuUoUu`SkfcxL9!E(DRw^-%4hA?J%BPedF*mpN`c==T%Ry++Ez)cyGJvL%mal*<TBu
zO9n};&CPYv&@8;W{}NB(o2wB!jE}E8#l!jb$!2|CyX|iCCcpn!<Qoxm;ZnhKDXr*N
z;%{Qs7tY&U;uLIWU~t$h5)^n08+pr@#c7^=U$s9VLG50FU&7>BQhsO3gle~4UahJA
z^w#7j=10!v--r*M?qra-?7&LP%a5Oa;h34Q@t*ZcpJ<+Pt2vi!-pzV;CQedShrdYo
zgzmEMW{Ocej1LA?$9Ml>>b>Tfm7S4xR;^|CMZ5Rya?gIQ&G+14?0D~G;_Isie(PO~
zv_Eh2Z;FOvarb5sW@kgt420JXWA(LP(^i*MZ8I^bXrA)&;>#;n3WRhuW=;M-v97eZ
zuJMw-$V{t)Y5b>6ZpUR;<%a##c^Z(A7k&BK;kl6>yNym=;_&<>BRy-2TRijr$N7Ez
z7JH-a6+Dlcv2RP`g7-S26J8WY`)&E<|GoNx?Z@8NJA1!<`(b8!$8vr1&xS?O%HQuM
zWxWgX>0KTldr#~;&nyN828W1;>kjOGH=UE)`mSwK8pmv7*4pJ_Et6x<d2Ukw^43)S
z|K1q&cgc6v|E^1HFBO|tb~R<~_ng(956^A&zvmgkHAhPO{Su9fhvrrrO$y^Euij&u
z((Ak8(JYtrnkm=IOPC+(*`L~%SGd35W^-zp@#GyLE~_7YY1!cW?)o{?i{6gP#}Car
za>Vw}^*#ArJ}dmQ{B{`I9-VrzN^0?*xNjFyF0YQZ_Flfzzii9H(;&zC9G?5~=FNP+
zsO_`n>wbGFdh@Es+mPn&8M7z93R*txv`$uB);8;d>M~vN42Gu_MRufpTT?#i-j#xM
zr<~UFpV)=Zl&0F5rrfjAf4=YOxix)j1>TuiEevk=FjKvgd49&s1ukc@Lm$kVT9T}M
zT=C-9s2oH4-M4RNKjSM;D+@Gq7VG{y>-E;?WV!75m2cjzQ=I5KQ}gQg+Q+fR8#EuD
zd)b?_hV8-^udo@<Z`_oAHg}%i=a^Yr?twNVU&y%?y-0nbZ|rOD!xL9-zLn|sfP0Uy
zPsWAMvnwxW&QpkxN$p5jBzbG)!^HvqTmDwn&Y3A*`z?FP%!`{Yu(Z5kODmnS?9J{r
z<Ki_!w&y>a?KeGjM&S6xb-&Au_L{Sv6YzWf<2S45=Cx0b9h%0unbnVfS*6{~dC5~=
z3WsezyK!Us|D2qQCG)SREq|Tv{=4O}%;x@;ynE&M{LYWt-TQ1~X=Uf0v_1BdZQjqW
z*>7d}K3n+d*K;|iKASm*zCNq7di{1b>&<Ej+mFeuUYNKX6bZAh&$(%|c+MI1Z2r4~
zIr~q7`wR>h3U=JSJSS4h)_(tHRa?1Rzf;;Szf}bFyBHi^x9k?XF0-|y*WmgRB`46X
zSB3`VC7YwOe5Y+|OAE@$wVJ(f^W9@=x}YPm=-AMtf>jr${jCo&oY(n2&1VP5I8RqU
Jmvv4FO#qulvt$4O

literal 0
HcmV?d00001

diff --git a/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure2.png b/tensorflow/compiler/xla/g3doc/images/xla_array_layout_figure2.png
new file mode 100644
index 0000000000000000000000000000000000000000..6439c6e40272ae6b2954e9d7f3de2df470a2b36d
GIT binary patch
literal 7913
zcmeAS@N?(olHy`uVBq!ia0y~yV6<jnVA#yT#=yYv&13g51_rqeo-U3d6}R5b<(=XC
z^w#n8^Yh;8Hyax7>k<$;=|0(|{fY9zC5tRduFQPYyM2OlA;-i;H7&}PaeVHw=XT68
zGF5k3xKhn|PRrxR8l@9u3cZfK`5pUT{rtXbZ<f72b8qwY%~sF;?_c+P*4}G9Ki7Yq
z@q7FG-*wk+X)84?(&(Z#dUYu@C+E-cxcn;@E!Ka3@#4gZ;McbE-uJn=h*%qLtFn16
z*VSXgsbAq1RrjN7qU6iJNt4t+#Y@}Hd++<RdT(6#-^Gg-`OTd5yIFAFBT>2YH&-j|
zjek|#*VktyD*Dv)QT+|!g@0yOD(`Pmck?OPVp#wC&hNj;cfR)bb{3wsw={hC)r)<1
zRH@$Nx%2B(({GhdxV(RtE1zEE+HK{3KKn!*S9fiHdg6_1(Y9RcpX(yIZ*I=*kJ#|>
zVKw`|IokFS#|x7@w#h_HOg>|;XLm64{}ip2cEZK|jP@0C)=yNNpQjb_`_ih=udz42
zulsRpt62PU=_M&Ibq;=O)#o~FfA4&#+@!czy_~q!-%^)_n?L@4l1;NRanp}=*_!*(
zdarIwsDC@_<+@+jS4Vr_`LsURc=3*)L+^4{Z!4;KQQ+RRX3fW=JC=*Y2<hICtty<!
zb>Q#%HRkW;XB|A}`~S-nExXIb^%duYEaNwCd!uKwAmMrT)tDOod#`yOoRU!eysYcW
znF-w1pG4pE{O+$m@}B?3&X@c*c0S7gX?DhEea(jI6;ZQJnRM~ZGZHJ1xc}v?{{H@+
zFH=?2PVFm|?ybuHuNio8-LLf>oga4^es6fY-Mnnln<XyutCrh;YTTG`neqAS399_d
zRd&9zw=q2$`d{$s>MrSj`>gY)ss~T<){=c=cI+tUCh5xu`3}hRcT8CCW|Ma(`uF?~
z6)|7!bARqO{4Sxfyu5O6(58Z27acyc&j}5x6%u{A>i@fp^1t7Yxa@uuKWVp5nXCJ6
zE!j8cA{n>uPFVjj!I}SM6YK81mC>=B-{PD0f4%<f)Scu7^&OwD|5bj>`2A4Iuia1c
zE9JgcY0JOyb9dV%l#_8Jx^j#5y)XX@53>BrW7)Mo)33gK&-cC2mWki}S5GO>e$W-m
zQ}*lsi}z6*q8DHKU47@X`Wp`ELwy^R-*8wTYJ*UH8x%pjz75Jb9M&+ob#oqXeOmnF
zwg3H(`H#Pz>Fhss`LX=O^W{&h-|ertcO~eywfUY^dkpzn3OGGCY>4hU9a{HnN7GmJ
zH%$`7)7MU19kpfBuCMDRShMj=TdR2IvSikV`jQV*Y`(mx$h`4GYybb-t?V(4XWuV7
zGbwNHZ^>5KbM;4dEWh&edUnCmJN9o3x1JYy!!WJ-S4r@8>(s4T>lB`ty!p4QWZIFV
z$EI*)Rh*w|7RFjVx$WzdHwWML{bXJxxNScB3#n_XSNCxX9D2Rp?&brIH|J!3I=I<Y
zyzl>hiLEC8@T{Y+c6^w+p_rkp{%3Tfii(=ngQq!KQ>5!kyT9yvZI^uT--C|hp$pd6
z)NbChEp>Ux&aY*xahEq7wmEX-nCI*=*`7mu&-qWRJ$HK*TLb@(nN6{;v)4^JSpI#|
z-stNmS9hmtaVY+n+jKWLx!rTSpSj0oHQqlPZqNC=>CPR+)BlYv<Yc;q|2e<k8~gX4
zPff?cqqQHR{@h{t|81`AZjBF?jMj60-}o>2N%)51=G^*N@2~$IZ&<SIu48UrmE)Da
zKXOC(o@?)KZu(uUe6TL}cG;&pi|l`HXSCU~d#~h^yBwm8vPLV9osH7CFsmi@;6LG4
zy=hzDp6j0;7pZ^wnT{cw^_j2ly649%GF~%Zd*dNS>DWd0uB_WCU-m&db9ZCiqZK)(
zbH3<nZ@zS4Ww5dGi=a(=HvG2y|NGsZP)EMcPB$!BtPij~on0HBwe^+Oi#snO|2{mw
zQ|mC-wkhj6{;%Kc?)6{s=BF!fT-_4ebHs{!&Bgx6D=cPy<NNK+y3A`4^CW&Y?4RT9
z<uhUJ&eJmQXR;nT$G83FWOr*eYui&zclUqWyLyxMl*@kK-@N&!ao63}<b=I`%UtIT
zhmSicp7?*+PI!a!dcM><&5Cy}$gi?r&}Sr6V9LAu`Sdkm8=S*9Z!C)ExUgU3kb6ig
z*Y2jT+A5~1k*O6D%Fcf}7IhuqJ2=mFMf}P$eB7*RiS0&mOG`B(b6peLf4bifnSNcP
z^zBWv#PZIA?|O5qMM?s^+HNi2|8`$t@u8i1`+sZ<SeDpcB--G(Kji&kCe}^4!W*1D
zTOFc5oT<2K;qKO+P`s%}m%B{L<lDZmf7;x=2j6{ge|>e4%Z7Omn`DhFZ6{<ZZ{E5-
zM7WG^`bO{T&!n4W6N2};wQ-e9o?f>g=gf-8Xx$SD33>%G%fvD!9CT&*WBrD?!S?{4
z<WyOsT`ey6&#AmIXuDtE|K-gk$FJ))I6T<0U)JnI+T~SCS0!Ic{(I`@u>^4oeVt_4
zCfQ%1-&ncUoR}4}z~MfZ?=Qp2^9<|eN{g{gIQz1((xFT;CoFp68KL6dsR?`e?=4*C
z#?IXl%DsGbkJ^V#JYUw%Y2)2=s+du}q;sC)sVw1ClTDU?rKZnh*z@h<dIkF++lU)C
z|1v*Uy>U8^V|Njs$q`2BDIpwIDdrIiO|KeXzCQh7l=W`|*67~`vBzFrEjVMjpnnbD
z%%|_u_hqn@RXr7)$y?>J;~1ahxm5yoEtARvKJaz@@9ORnOp(ohw@&J*=*|_UBKfgS
zY<<42?Y)QjmR4(A-_iPb(W7T;Z)shyGpy~qcr_x$P(!w`R=g_pVBCGT8-itfVtfAW
ziR}RwU%!OQT4WR5Z(N)Y<{s+%2j&VwL<P%QcoW@kfO*ak%Ke4_RL%^na_05Ke)puL
zq*%!x$;GE<|Fi#f{MfM_W-s2)`+xC-M_SpLH8XA+Zca{fzaa?GfX&Q$Xt{~gHDA;r
zWh_QfD}Csk;|-m<yr!zS><4u=OLnjM`|w+R)|(AMw`Ui}O?%_`_etkO{+4I*EN?#S
zxwHJZ{in&F=UHgow&$>BPZWPO`NiRn8jd#BqUYBgePOQ>&Gl`*!L$D7mmlqUyEyuN
zo!g)3>u>)M&bReBn3w&|=w#N_m^JqDnb(#yR+X)}yaAM&JG(xuI&N``k%8f%#hc%U
z-}1{d=pOp^x%KDjC7y??UNPSFj-RncPWz?En-3q}T344%tSZf(YPIalmsh8v`462_
zym2;mn_J^C-Wz|ds+YuET{N}IPv=1M^@_5!n*Z(pScV&AT?^ZAJ-PH=nDRf1_}%(_
zOY`<>JJoqqYVe=*^Ysjy;CMjs#>WXcEc=5uNPfMbAX)YPEtf{wiibD1{*aTq^;YH0
zyO)=J5AxYn$;%1sOFWxoUtyr`cktb^vbCXN>Fu)5cBfD0(@CD&cK3s!@YUcK)!*K}
z=X&#@A;V@)RKESUCnjGvWCZ_9R<l)}yX)?^ZjI!^cf8W^qMw{kHS<mi<z7CW%ZgWd
zO0nO;XKxO^{kLGdGp{87zDE^vTIPP^x5?*WSnvNmUQ@z~b@!$}6Hg}Y{#&Xm+Vb2_
z+WUqjqjieN34?jF_MQ;Ear1a~blIvie2)32<R`Gd3D}i(_lD;|{g4|WPww2mvCTax
z>9JSi{mdJRCjt)L`o1upMZof|F?Y*cIlsf_-U;vEueEMqQ_bB_yE{Q#<ITShvHOem
zRsH?hR}*?k>9)!nnO8S=^6tJ@es_73?D2*J=1=eMh~#Fy@m%b$=UK*0cLGkI=$d11
zqdh6Hjmi4U>kHPmcRmcQzt*@fHd0~7l*IN+q3P3hEs3+sUuAfMl`my_eXLdYhTS2F
z%f+?dNn6Eyz0ef9gyZV}vrT(9eZ9Qi>+6R-*KCw-o0k6j;C}wgQBlid;mLv9jvYOw
z@k3#;;{w*S>giT*E;wIaYJH&n^ZEMJg@!kH_wAluUn+JXH)4HWK(WJ(zta!!HEeF?
zUB5A?ZK{{K-d~3>&i@IUPj~$|Ur{*ol*Zj%Usz_!{wmj4C{V%>m)a!D(ERwI5~G+O
zsK|SK{pQWTh5Jsd*jQ2IyZP9go3X1lI8X4Oz4^%gxtmvB(8zg^`hK$FZZE%;NjjU4
z9C&J`@!0n;sQ4AVG2!fs+Kfr;(mB&lxgMR-AFysC_jT8!AGS}w+;Ufy&05Eeq2ff|
zy!cH~uS|CsEjhq{^?x5@^HlXcrEOo2e6e&p{o~o%TdP_HFK=Le60mDg%V95-Q<^!i
zMV;?C9<<ib^Str!)}}6&7}wJlCeN+!-HB~H_9RPVPn~({v`_DgUcY@c%lY#2{!0F*
zxAqlG*?2<D&8}koojZ!!Pj<6|T2|Y;FV5fI4K9n^u@)FwmT&yr+;$0NSlj>g(%T{N
z=7avi{nZEey^jSq{ldL&$bPO$Eq}=P_~5quyKX=4{W^ay`k?61t{?M@Z_3YXl`hWz
zzJG)A{izMwCu5(j`+q62=-%UP`;Uv(pZVLkV#O);-FtpiSQ=GN{Fw0j)5VWLDStQ4
zmzea{Ys2CC<x{WjdRc$tQ+e{ct^Wm2?^ii--jO@{K<}w(^-=j-@_zo_RWjlHqD3)Z
z?~A|fvwv3S>Qv*owE3>C<jkUdZfzHN9$(%ss&Rf<nAYFgX6>il0(Ki!Wd5Cf{3!P8
zep|zf=~eIE{wr5i|EaQf*T*-DQVy?Xy8kaU=*(~VpAI$r_q6wwtzB9jWg7lB|MVKg
zWq-fk+3xGFy{=@#VOfFs{#9;&KPsOtF#7N<=l0|8R$m?&eT>?0J9a}j!yMzcRwoXs
zuI~MH{HXelSM!5Zucl|cJN?Ojb=cK#qslG69*0Y|U3UClx9-PZF~8%-g=T(s@6Y+@
z^U*YWjYIR>Tc(#|zaHPa?${sZXZ!rl_MBqhJ>`nlz8^m-;@8iRxG&i6Zt82;b=c;E
z_5RJr*u!34(b4m(bc;-7*{2qr92PR={fwvYjbF$MOFv#!A5$}D#*FH>^|z#6uf4SK
zr0f5d9~B$h)^AVzzigU)jj*s#&yUCDi|l*0xvswv@w9j1oEcA6)q4ePGOYi*z5MIZ
zo$Ke$EZleL;JR&J-xcM2+#ED#o?3jrq?-Guuv7n>SwG$U=zqwH@6BDM_;a(h?x${@
zR2*(TvAH#W<(a(y|8k3#xu;vk|2O*i>SkwA#_Q~3e*{0|{aU_c*$<yHwrqdSmF)|x
zd?>SVebv>heV2`&dw53iTK@jPVYxM_cg~G{-hJ<F1ilp-y_fyDu-5%)Y0=&2toW)Z
zUgdxB`Tqj;%{2->7k}HWOWHj2c++1~W24xw>!Vlq?RXlmt-SeDYT5PcX95@tD!luY
z9mB3!7(NWN%y%?9+$7ud|H=i%P46Qn7XG)owtLR(e{UxV*Dibh?=-9Nk{w}fO!u~i
zN%lJYPJ3JB&Jcdh;_nw7&66iW4rJ#Zytd{4&56l>{#}pUP<na)+^<)8|8t9_%)aIU
z>etyS#`XEjX6xMe@+0krz|F*^*Y4lDczDP5Z~B`qRhj?gkNlT+=C8PS`svoJ(5JmG
zBrHGwn0$=sEdQR!&r4Odu3FRITYA-AU+-M#fAhzWV?X}?{(wEDvc~CYbg%8}k1K?Z
zmmD}KEH1jZq?p-kKTG`=_PaBxS#Q5_{M7t(y|>oo$GaQ9pXR@IU+(YO--kB`#f8+|
zI;D4q!|cnc3l^r4cK?L_-u-aw$MJW*r|$PUgx4<Nz5Vk2r2n1Y?bfaNeyl2%dq<LZ
z^W1;-zjlkhk@)g`$1`7ijS6@}1=3#mdnsZ2>lf}u1uq(ceotE)s}x&RGA(cKu2Z-E
zT<v4~eeLe-louNm|GnKWuRZCau-sH>tGM-j+3pjzM@_vhzyJG0!^2N@eaoH7-{<`M
z|N9;8SL$m1R!^9|amsc1x(}`A^WAyA&VPRF(Do<m8p~IG=D)h%rg7uPt1&wl|IWR%
zv(|m4VcsSGefuk($;>oQU6#DzWc<|NW%Y^otS{|9KR3(1@1W-Yt=-eT`_FCnxZXE6
z_2rtM|Nd$}-@oMB!y5+feh2xUw-q1RDRyh!>C3;_@0YHBb8o}k4cy{y1PVB`?s%{M
zzV`MH@tunTYeh>KWcP{t9AIVMy=U!uO?{R<_soCZ?Bv=c|L*>>roH9+Z_70DIxJQ+
ze*5#}rA?*WTjd#8YBts%c(D6vr^5R-jrI1=Y9IA{tb6h5bIz3|3w0vw!uN6>ls_(!
zcSH7X0{^?kQ}&2vr@Vj0A6j5<o#>_cAjEsER<fMjLyjE2-1|%4DPH|2yFOo^d)_?N
z<97diW|`~>ii`Z=zEytxN8^Xz?r$pFdu9KdqSAR1x7gPg#{A%tcx>9Id1u4v%F3l%
za<0Y9ResQQ{YHQCh2MtqZ*QjvmOX5WZ9iwe>cFPMc?I7TD)SEV{Qvekx=5z^?p58{
zn)=pn+H9xkCad>;Yu;WOrtO}<yn<<k)$Gr4bBqs!r&l%<US2)h;t~G@b|?9U&4)!L
z%@#=H-4K1cKU^!jy6#em<c;rnU!@N6{^<E}*17K>pTWl8%A1?z7217{|0`~9+S9qu
z=HH1M43c~2+J2}jwQ2CZwl(+9=Lz$|oEz2eY|uV&_rZCokH?z*yDlDY{LlTjn4!qv
zUrVfIn`{|_!X~*Cs~i1q4xcm<lUyQO#=y(a?86>;u4~THzVBxkBvK|<$O?XVAG}J%
z`dvTIxwuoW8^fG5KFf!gC9q0w;*|X(?A90f!SRDN>+S<O{4+&2$^K9}opX25Qm&E)
z{U2wvEb6CpohV{ds93n=s6!YlgZ0hL&L_KeUQe|-C;NH9{{G|*`NCz1ZL$Y0Z?BjC
zDmBwTZ7t8um@nH5Zzw)?oUpjXt@8TH*S>v+*d*m1{Q7rKH}#O7nc20#`jks2?!CEb
z9Je^1ox%F*=F1Bl>pj>_H}U2?-pYC7p#01$SGvv59p;~K`DV=e=v~uKl^lB8&m+Fz
zs;zZOnbe-s@ewM$ugp1i<%{vWF}SZ0UU=>UXNz>=bAvYm3C7QSby@c(Y+BuQ=-sDZ
z(>?1BO8?7MPwcLfFpB)G<Z(4C^zZExMuBlwjMffop3B;bTzTWrZ2mwzXT4s;4aFVn
z^6qR2S<g39q)FDX&B6V=)(zLLPrsBWH%<ThUHQY?RGmKEhzHZI%O^0WYzRoyPk;N_
zFy{la;++5WZL(g$)-D?*nu^`E5AKU-T9g0!=y^_+H!=;fMzVKx+#fuwwt8*IXni34
z-KCp#HyFxZq*`7yc4m{DVfpaRR(mgxDKq>I@ZB`tv}==?Udj1Onl~1fu)cWHxU@OI
z{H=Pg)bghK|JCx=n;aNtEOHmWahm(go2J-_y7}h?6z&Usm|3i$ZkiK#iRoA4aS6T~
ziFq4uWzFZ7;XWt&VDouaag9Sf*1we)n&cV#jxv8a^QK$W?*Lz9wEykr=NCry*D3aK
zK2;S9nDgQIPR1wS7WT8>t9o&?tKc_#)nV>#$Hw2)Y(H<-JmoB7F=)DLH{s}wdNzeQ
zQiq;zntEJ*|L^C20=F?8Igw=|aOiFSzLW*_eVh%<Ia}&8Qui#6*}#6{A@hZo^JM=p
zMpq=)zuR=RMOI&HnZ3T&GuNZwt{tSQ1usU;V1gK3LaePZP?HUNXL35vn>`aJO_}nd
zIQc>US>60!@n8IXKJ7FG^%<WB2A+8{BNEhOJbngwObuh$4Q4j1Yq;1R+Ee|@2^ofa
z;ScGtqIO)%Pae)YzL4qVrNx`;tG~3kKi%~$*Soh=`v0Sd4f#4X-oL)R4}MkgPO4&g
zm52Ghef!J1l{Xwby#C|wj|zEn?B&ko{<o7m_xH$^jeQ68_CMf#d2RLTPsf`#d_BCQ
zP-gC~drRa*`u=k-Pnf=6%iU`G`xo^RrgC?B{#Iz!{x*yJ*T|k_%{Y7N=lG!g`%Am0
zeY%m~zM9?T@acEGhWV#_-2Z&6-n=e%@t?Q+%*V6eyuP(@^DO1%3GM8q+4FnXKb!XJ
z-aFpgR}XLE+--1(uaR}dgOi`X)oK?_TV+^wDMPe}VS@06YSsLupV$Jv+A(Cj*q~VV
zfPusB09(UVgXnE#oiP_z?E7DRdER^-&Cp3tCmJ4>X^wrpChSxDRQ+Fnzew}-F@OH_
zJ=yir!q2@%<uBiCILw!@GJ17iPUP;mTz=_44WL5u)z<_2%1<u|e5~^P>C`NR4~KYX
z@11QQ^3V0>`<2NrOTEi0>n@c%U0Ta{IBR9--{TT~|M!SyFY!FAo%+98Ui(D+-5o1G
z#r!*@#9II6_{N9nUuHX4B(@j7uBfdpea8^H_!qC&Ud}}61?q1OaacFFFspy4-rZNX
zgh_wK8oh`ccQ1Ubs{A?Cb!kF*<=@jG*A{O#&Q4vC%(>m(;n%}|x&MlLdwah!OB`V3
znA;-DkY3-Md1cAMs@(bS8JpkheJy_2B+FEB@bYFaR(T_yJ#G*3W4}-1`j^1Gch3gK
zD-2@QW=j)Svv2ZEvG@BS{v>b0oz&HTZ#%Q|1kIG&XJ?(*Rd2sZ`*&d*cUePi;yvqE
zrU$rIynfGE^a0$xoXTp=vU~5Il1)8Z5A#X#8|lVoRjn;Pc;4csvGliyZA^1&@Ap&&
z%eKy~UC^_2`kGx+T*G8}UO2zJ$(V5>ZQ8k6>y3CF<R9!_{5x0cH`9+fZ)P&qeAVB*
zCq$E7LUH%Oj}~_~AO3mu)QOPejXh@75B~mpzwiw2z8<~$^L!JI%iLz{W)u)AV@qtW
zZLHN1-r$_pnzOEa|Lu?i%DYR1${3{nFngB@Z*X4OWiB#DJMQ-8th=YLsO}Zs!2Cks
zLb9CP$K(w)8<vM0Gfuvv^78uT{^Sje2?1J6TN)!}_sor3;Lyo);Z?EQvDU{gFa8v}
zdcH)e%%Z;H^`v<liW$oo_@thwg()*9E*ICl_H2C^1H+FwACA9wa}(u|3f^$o^3k4&
zp|=#bYkmL!=l1&ilOLyVf7mvcjkhYbO5RJWx@~LO$D*szMNDQVRxr%kB<67Ga+byi
z3r_1P?;0hSzDr+}-?o$CMRWXuk5gXXDV6MIl(!Dyv~J$u%qgAYCa8B~p}*<Ro1JwJ
z7(D!b@XJ^^=yDy1=bya6`I?nQ^@RJ?#TRdLm1Vxn{MRG(ukmT=4lxY|t0}o0Iy?vX
z<4Zl4SBHI@da6J#-~fBA^V{_g+l0hwoz?Yi9o^dPBsXcj?bi|CYos&nhIeMl?uTu%
zr^2>yewpo;u)ASHu|Qd)!%U0hJy8}0jMiVC3VivNyT*yB!9s7jp6x}sZ;Aiy&dDDr
ze)WfW@7Mo~M%G_+4L9h|nEK(<A-;oh6`k$xLLXidjuC2X484``JMHO5{ntXmdnFou
zxTi0&;IuxVc0vAlOYu=9)^oDIH)Op&vQ+)U!J5~Vd^(RA9jf0?*znl(=$GfNNB69l
z0UfggQ5fYT#<T>aJdFKOYGib2s>AL26%MdIa`wTpq~ChKc0b{N@5Q-0Id1)*$B&*b
zo+$(BK7GGd|LviIy{77%f6rsSxu-omcIWor{d}L2XC5;3TO09s)&G~`Vkay6mk0ey
ze#h_P8ftgr>CxjSrz^(%jlcNu>x@KjH(AQ+z|@5H|9aZ1OE3T5@bvw$JK6u0r~TjV
zcyEu~y25V{1*h%5Zu(!|s%#plm;9~2?5KS4ejB@mOD-K;mz~C|>E~}eD^cd&q{TL~
zqgH<5?|pUpqvzsYwq4)y1;wAQ`v10*)9!L{{`dFOr@YbjT%BmM`$yW_s>b(sHABz*
zuKzQ$#{5H_<$X2pU7_pWt*SpSC7E<9{(SB1EmB9P@|Eq_`2WY1jj#IUBikl7mT&l=
zyZ`#eqvyM>H{G@Gx+li){Qle9r^HV`m-^b@-(P;!Ue)v{xR<;>ZtJY~XCC{nvad^#
z@MFJwa!<L-@453V+2=<josq9yb$RnM@50B+xJ+)Wn>q32D*L=w=1=7<zHc=$(w2WS
zDR`29xTmk>RYS{{w|xTxpPzSi`QcY(vv2XliK-7}man~EdE(bDJ*!{s*+G-0q^G2$
z814((e(c4M9rgL$pTrjTC!eynvij9D``^aGjp9CM{+|DN{P@vW58stVua>E7{--;6
zlK)|IMZ1F?|6g|R|C~R~{eDY}@rNTGJ{~1k?JW&Y&ZvKICwhUsh-09LD~-|9zJKz|
YV|3EW+_cv-FfcH9y85}Sb4q9e0BqX<ZU6uP

literal 0
HcmV?d00001

diff --git a/tensorflow/compiler/xla/g3doc/layout_with_tiling.md b/tensorflow/compiler/xla/g3doc/layout_with_tiling.md
new file mode 100644
index 0000000000..5e990851af
--- /dev/null
+++ b/tensorflow/compiler/xla/g3doc/layout_with_tiling.md
@@ -0,0 +1,159 @@
+# Tiled layout
+
+*Note: This doc describes how tiled layout is intended to work. Tiling is being
+implemented, but this is an early effort and it is currently not even guaranteed
+to get an Unimplemented error if one tries to use tiling - it may be just
+silently ignored.*
+
+<center> ![](images/xla_array_layout_figure1.png)
+
+Figure 1 </center>
+
+Figure 1 shows how an array F32[3,5] is laid out in memory with 2x2 tiling. A
+shape with this layout is written as F32[3,5]{1,0:(2,2)}, where 1,0 relates to
+the physical order of dimensions (minor_to_major field in Layout) while (2,2)
+after the colon indicates tiling of the physical dimensions by a 2x2 tile.
+
+Intuitively tiles are laid out to cover the shape and then within each tile,
+elements are then laid out without tiling, as in the example above, where the
+right part of the example shows the layout in memory, including the white
+padding elements that are added in order to have complete 2x2 tiles even though
+the original array bounds are not even.
+
+The extra elements in the padding are not required to contain any particular
+value.
+
+## Linear index formulas for tiling given a shape and a tile
+
+Without tiling, an element e=(e<sub>n</sub>, e<sub>n-1</sub>, ... ,
+e<sub>1</sub>) in an array with array bounds d=(d<sub>n</sub>, d<sub>n-1</sub>,
+... , d<sub>1</sub>) (d1 is the most minor dimension) is laid out by major to
+minor order at position:
+
+&nbsp;&nbsp; linear_index(e, d) \
+= linear_index((e<sub>n</sub>, e<sub>n-1</sub>, ... , e<sub>1</sub>),
+(d<sub>n</sub>, d<sub>n-1</sub>, ... , d<sub>1</sub>)) \
+= e<sub>n</sub>d<sub>n-1</sub>...d<sub>1</sub> +
+e<sub>n-1</sub>d<sub>n-2</sub>...d<sub>1</sub> + ... + e<sub>1</sub>
+
+For simplicity of notation in this document we assume a tile has the same number
+of dimensions as the array. In XLA's implementation of tiling, this is
+generalized to tilings with fewer dimensions by leaving the initial most-major
+dimensions unchanged and applying the tiling only to the most minor dimensions,
+so that the tiling that is specified mentions a suffix of the physical
+dimensions of the shape being tiled.
+
+When tiling of size (t<sub>n</sub>, t<sub>n-1</sub>, ... , t<sub>1</sub>) is
+used, an element in the array with indices (e<sub>n</sub>, e<sub>n-1</sub>, ...
+, e<sub>1</sub>) is mapped to this position in the final layout:
+
+&nbsp;&nbsp; linear_index_with_tile(e, d, t) \
+= linear_index((⌊e/t⌋, e mod t), (⌈d/t⌉, t)) &nbsp; &nbsp; (arithmetic is
+elementwise, (a,b) is concatenation) \
+= linear_index((⌊e<sub>n</sub>/t<sub>n</sub>⌋, ... ,
+⌊e<sub>1</sub>/t<sub>1</sub>⌋, e<sub>n</sub> mod t<sub>n</sub>, ... ,
+e<sub>1</sub> mod t<sub>1</sub>), (⌈d<sub>n</sub>/t<sub>n</sub>⌉, ... ,
+⌈d<sub>1</sub>/t<sub>1</sub>⌉, t<sub>n</sub>, t<sub>n-1</sub>, ... ,
+t<sub>1</sub>)) \
+= linear_index((⌊e<sub>n</sub>/t<sub>n</sub>⌋, ... ,
+⌊e<sub>1</sub>/t<sub>1</sub>⌋), (⌈d<sub>n</sub>/t<sub>n</sub>⌉, ... ,
+⌈d<sub>1</sub>/t<sub>1</sub>⌉))∙t<sub>n</sub>t<sub>n-1</sub>...t<sub>1</sub> +
+linear_index((e<sub>n</sub> mod t<sub>n</sub>, ... , e<sub>1</sub> mod
+t<sub>1</sub>), (t<sub>n</sub>, t<sub>n-1</sub>, ... , t<sub>1</sub>))
+
+The layout can be thought of as having two parts:
+(⌊e<sub>n</sub>/t<sub>n</sub>⌋, ... , ⌊e<sub>1</sub>/t<sub>1</sub>⌋), which
+corresponds to a tile index in an array of tiles of size
+(⌈d<sub>n</sub>/t<sub>n</sub>⌉, ... , ⌈d<sub>1</sub>/t<sub>1</sub>⌉), and
+(e<sub>n</sub> mod t<sub>n</sub>, ... , e<sub>1</sub> mod t<sub>1</sub>), which
+corresponds to a within-tile index. The ceil function appears in
+⌈d<sub>i</sub>/t<sub>i</sub>⌉ because if tiles overrun the bounds of the larger
+array, padding is inserted as in Figure 1. Both the tiles and elements within
+tiles are laid out recursively without tiling.
+
+For the example in Figure 1, element (2,3) has tile index (1,1), and within-tile
+index (0,1), for a combined coordinate vector of (1, 1, 0, 1). The tile indices
+have bounds (2, 3) and the tile itself is (2, 2) for a combined vector of (2, 3,
+2, 2). The linear index with tile for the element with index (2, 3) in the
+logical shape is then
+
+&nbsp;&nbsp; linear_index_with_tile((2,3), (3,5), (2,2)) \
+= linear_index((1,1,0,1), (2,3,2,2)) \
+= linear_index((1,1), (2,3)) ∙ 2 ∙ 2 + linear_index((0,1), (2,2)) \
+= (1 ∙ 3 + 1) ∙ 2 ∙ 2 + (0 ∙ 2 + 1) \
+= 17.
+
+# Tiling as pad-reshape-transpose
+
+Tiling-based layout operates as follows: \
+Consider an array of dimensions (d<sub>n</sub>, d<sub>n-1</sub>, ... , d1) (d1
+is the most minor dimension). When it’s laid out with tiling of size
+(t<sub>n</sub>, t<sub>n-1</sub>, ... , t<sub>1</sub>) (t<sub>1</sub> is the most
+minor dimension), that tiling can be described in terms of pad-reshape-transpose
+in the following way.
+
+1.  The array is padded to (⌈d<sub>n</sub>/t<sub>n</sub>⌉∙t<sub>n</sub>, ... ,
+    ⌈d<sub>1</sub>/t<sub>1</sub>⌉∙t<sub>1</sub>).
+2.  Each dimension i is broken into (⌈d<sub>i</sub>/t</sub>i</sub>⌉,
+    t<sub>i</sub>), i.e. the array is reshaped to \
+    &nbsp; &nbsp; (⌈d<sub>n</sub>/t<sub>n</sub>⌉, t<sub>n</sub>, ... ,
+    ⌈d<sub>1</sub>/t<sub>1</sub>⌉, t<sub>1</sub>). \
+    There is no physical layout change in this reshape by itself, so this
+    reshape is a bitcast. If one is not explicitly thinking of a tiling, this
+    reshape could express any shape with the same number of elements as the
+    padded shape - the example here is of how to express a tile in this way.
+3.  A transpose happens by moving t<sub>n</sub>, ... , t<sub>1</sub> to the most
+    minor dimensions while keeping their relative order, so that the order of
+    dimensions from most major to most minor becomes \
+    &nbsp; &nbsp; (⌈d<sub>n</sub>/t<sub>n</sub>⌉, ... ,
+    ⌈d<sub>1</sub>/t<sub>1</sub>⌉, t<sub>n</sub>, ... , t<sub>1</sub>).
+
+The final shape has the prefix \
+&nbsp; &nbsp; (⌈d<sub>n</sub>/t<sub>n</sub>⌉, ... ,
+⌈d<sub>1</sub>/t<sub>1</sub>⌉), which describes the number of tiles in each
+dimension. An element in the array (e<sub>n</sub>, ... , e<sub>1</sub>) is
+mapped to this element in the final shape: \
+&nbsp; &nbsp; (⌊e<sub>n</sub>/t<sub>n</sub>⌋, ... ,
+⌊e<sub>0</sub>/t<sub>0</sub>⌋, e<sub>n</sub> mod t<sub>n</sub>, ... ,
+e<sub>1</sub> mod t<sub>1</sub>). It is easy to see that the linear index of the
+element follows the formula above as expected.
+
+# Repeated tiling
+
+XLA's tiling becomes even more flexible by applying it repeatedly.
+
+<center> ![](images/xla_array_layout_figure2.png)
+
+Figure 2 </center>
+
+Figure 2 shows how an array of size 4x8 is tiled by two levels of tiling (first
+2x4 then 2x1). We represent this repeated tiling as (2,4)(2,1). Each color
+indicates a 2x4 tile and each red border box is a 2x1 tile. The numbers
+indicates the linear index in memory of that element in the tiled format. This
+format matches the format used for BF16 on TPU, except that the initial tile is
+bigger, namely the tiling is (8,128)(2,1), where the purpose of the second
+tiling by 2x1 is to collect together two 16 bit values to form one 32 bit value
+in a way that aligns with the architecture of a TPU.
+
+Note that a second or later tile can refer to both the minor within-tile
+dimensions, which just rearranges data within the tile, as in this example with
+(8,128)(2,1), but can also refer to the major cross-tile dimensions from the
+prior tiling.
+
+# Combining dimensions using tiles
+
+XLA's tiling also supports combining dimensions. For example, it can combine
+dimensions in F32[2,7,8,11,10]{4,3,2,1,0} into F32[112,110]{1,0} first before
+tiling it with (2,3). The tile used is (&lowast;,&lowast;,2,&lowast;,3). Here an
+asterisk in a tile implies taking that dimension and combining it with the next
+more minor dimension. Multiple adjacent dimensions can be subsumed together into
+one dimension. A subsumed dimension is represented by a tile value of -1 in that
+dimension of the tile, which is not otherwise valid in a tile as a dimension
+size.
+
+More precisely, if dimension i of the shape is eliminated via an asterisk in the
+tile, then before the prior definition of tiling is applied, that dimension is
+removed from both the shape being tiled and the tile vector, and what was
+dimension i-1 of the shape has its array bound increased from d<sub>i-1</sub> to
+d<sub>i</sub>d<sub>i-1</sub>. This step is repeated for each asterisk in the
+tile vector.
diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc
index 2398470dd4..dbb81381ac 100644
--- a/tensorflow/compiler/xla/layout_util.cc
+++ b/tensorflow/compiler/xla/layout_util.cc
@@ -460,6 +460,13 @@ std::ostream& operator<<(std::ostream& out, const Layout& layout) {
   }
   hash_value = Hash64Combine(hash_value, layout.max_sparse_elements());
 
+  for (Tile tile : layout.tiles()) {
+    for (int64 tile_dim : tile.dimensions()) {
+      hash_value = Hash64Combine(hash_value, hash<int64>()(tile_dim));
+    }
+  }
+  hash_value = Hash64Combine(hash_value, layout.element_size_in_bits());
+
   return hash_value;
 }
 
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 82662b1dbb..b95fabf488 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -121,6 +121,23 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
         VLOG(3) << "CompareShapes: lhs layout != rhs layout";
         return false;
       }
+
+      const auto& lhs_tiles = lhs.layout().tiles();
+      const auto& rhs_tiles = rhs.layout().tiles();
+      if (lhs_tiles.size() != rhs_tiles.size()) {
+        return false;
+      }
+      for (int64 i = 0; i < lhs_tiles.size(); i++) {
+        if (!absl::c_equal(lhs_tiles[i].dimensions(),
+                           rhs_tiles[i].dimensions())) {
+          return false;
+        }
+      }
+
+      if (lhs.layout().element_size_in_bits() !=
+          rhs.layout().element_size_in_bits()) {
+        return false;
+      }
     }
   }
 
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 013673dd9e..85ec83437a 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -108,6 +108,16 @@ enum Format {
   SPARSE = 2;
 }
 
+// Describes a tile used in tiling-based layout. Refer to
+// g3doc/layout_with_tiling.md for details about tiling-based layout.
+message Tile {
+  // Number of elements in each dimension of the tile. It's ordered from the
+  // most major dimension of the tile to the most minor dimension of the tile.
+  // The dimensions correspond to a suffix of the dimensions of the shape being
+  // tiled.
+  repeated int64 dimensions = 1;
+}
+
 // A layout describes how the array is placed in (1D) memory space.  This
 // includes the minor-to-major ordering of dimensions within a shape.
 //
@@ -138,6 +148,20 @@ message Layout {
   // memory.  This field must be unset unless the format is SPARSE.
   int64 max_sparse_elements = 5;
 
+  // A sequence of tiles, starting from the tile that's applied first to the
+  // Shape.
+  //
+  // TODO(b/119839262): implement tiling in each backend or add Unimplemented
+  // error.
+  repeated Tile tiles = 6;
+
+  // Bit size of each element. If the size is bigger than what the element
+  // type requires, the value is stored in the least significant
+  // bits and the additional most significant bits are filled with 0's.
+  //
+  // TODO(b/119839262): implement in each backend or add Unimplemented error.
+  int64 element_size_in_bits = 7;
+
   // Important: if any field is added, be sure to modify ShapeUtil::Equal() and
   // LayoutUtil::Hash appropriately to account for the new field.
 }
-- 
GitLab


From d3125a94f3b160ae4bb1ca48ef4c67cc65071c8c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 11:23:59 -0800
Subject: [PATCH 1033/1554] Constructing a single scalar tensor requires
 multiple steps currently. - Construct a tensor - Fetch tf data - Copy into
 tf-data - Convert to a TFE_Tensor.

This is a fast-path version that collapses all those steps into a single call.
This call path requires no TF_Status checking.

PiperOrigin-RevId: 223379214
---
 tensorflow/c/c_api_experimental.cc | 10 ++++++++++
 tensorflow/c/c_api_experimental.h  |  5 +++++
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 69de4cb711..0c593185db 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -8870,3 +8870,13 @@ void TF_InitMain(const char* usage, int* argc, char*** argv) {
 int TF_PickUnusedPortOrDie() {
   return tensorflow::internal::PickUnusedPortOrDie();
 }
+
+TFE_TensorHandle* TFE_NewTensorHandleFromScalar(TF_DataType dtype_arg,
+                                                void* data, size_t len) {
+  auto dtype = static_cast<tensorflow::DataType>(dtype_arg);
+  DCHECK(tensorflow::DataTypeCanUseMemcpy(dtype));
+
+  tensorflow::Tensor tensor(dtype, tensorflow::TensorShape({}));
+  std::memcpy(tensorflow::TensorCApi::Buffer(tensor)->data(), data, len);
+  return new TFE_TensorHandle(tensor, nullptr, nullptr);
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index c04cd441bf..80c8bfe594 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -241,6 +241,11 @@ TF_CAPI_EXPORT void TF_InitMain(const char* usage, int* argc, char*** argv);
 // in tests only.)
 TF_CAPI_EXPORT int TF_PickUnusedPortOrDie();
 
+// Fast path method that makes constructing a single scalar tensor require less
+// overhead and copies.
+TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_NewTensorHandleFromScalar(
+    TF_DataType dtype, void* scalar, size_t len);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
-- 
GitLab


From 143307d680bf17154bdca86d7ad9b3f364f273a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 11:25:55 -0800
Subject: [PATCH 1034/1554] internal change

PiperOrigin-RevId: 223379551
---
 .../TFLiteBenchmark.xcodeproj/project.pbxproj | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj
index 958936a660..a5f5bfbbda 100644
--- a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj
+++ b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj
@@ -20,7 +20,7 @@
 
 /* Begin PBXFileReference section */
 		6FE7579920D59CE500F01636 /* benchmark_params.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = benchmark_params.json; sourceTree = "<group>"; };
-		6FE7579C20D5A5E000F01636 /* benchmark-lib.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "benchmark-lib.a"; path = "$SRCROOT/../../../../../../../tensorflow/lite/tools/make/gen/lib/benchmark-lib.a"; sourceTree = "<group>"; };
+		6FE7579C20D5A5E000F01636 /* benchmark-lib.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "benchmark-lib.a"; path = "$SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib/benchmark-lib.a"; sourceTree = "<group>"; };
 		6FE7579E20D5A6A700F01636 /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; };
 		6FE757A020D5AB8000F01636 /* mobilenet_v1_1.0_224.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_v1_1.0_224.tflite; sourceTree = "<group>"; };
 		6FE93FF820D592D8008C9FE4 /* TFLiteBenchmark.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TFLiteBenchmark.app; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -309,19 +309,19 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_STYLE = Automatic;
 				"HEADER_SEARCH_PATHS[arch=*]" = (
-					$SRCROOT/../../../../../../../,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/eigen,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include,
+					$SRCROOT/../../../../../../,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/eigen,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include,
 				);
 				INFOPLIST_FILE = TFLiteBenchmark/Info.plist;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				"LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../../tensorflow/lite/tools/make/gen/lib;
+				"LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib;
 				PRODUCT_BUNDLE_IDENTIFIER = example.TFLiteBenchmark;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				TARGETED_DEVICE_FAMILY = "1,2";
@@ -335,19 +335,19 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CODE_SIGN_STYLE = Automatic;
 				"HEADER_SEARCH_PATHS[arch=*]" = (
-					$SRCROOT/../../../../../../../,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/eigen,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src,
-					$SRCROOT/../../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include,
+					$SRCROOT/../../../../../../,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/eigen,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/gemmlowp,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/neon_2_sse,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/farmhash/src,
+					$SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/flatbuffers/include,
 				);
 				INFOPLIST_FILE = TFLiteBenchmark/Info.plist;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				"LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../../tensorflow/lite/tools/make/gen/lib;
+				"LIBRARY_SEARCH_PATHS[arch=*]" = $SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib;
 				PRODUCT_BUNDLE_IDENTIFIER = example.TFLiteBenchmark;
 				PRODUCT_NAME = "$(TARGET_NAME)";
 				TARGETED_DEVICE_FAMILY = "1,2";
-- 
GitLab


From 4aef7dde40a3ffb1871d9a58aafec20711e7131f Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Thu, 29 Nov 2018 11:35:53 -0800
Subject: [PATCH 1035/1554] Enable user specifiable Keras symbolic tensors.
 This allows users to plumb through Keras objects which are convertible to
 tf.Tensors. (This functionality is similar to
 `tf.register_tensor_conversion_function`.)

PiperOrigin-RevId: 223381634
---
 tensorflow/python/keras/BUILD                 |  11 ++
 tensorflow/python/keras/utils/tf_utils.py     |  44 +++++-
 .../python/keras/utils/tf_utils_test.py       | 134 ++++++++++++++++++
 3 files changed, 184 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/python/keras/utils/tf_utils_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 69e18ea55f..48cdbf1e66 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -596,6 +596,17 @@ py_test(
     ],
 )
 
+py_test(
+    name = "tf_utils_test",
+    size = "small",
+    srcs = ["utils/tf_utils_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":keras",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_test(
     name = "io_utils_test",
     size = "small",
diff --git a/tensorflow/python/keras/utils/tf_utils.py b/tensorflow/python/keras/utils/tf_utils.py
index 6b7c6c34a2..7b4c9e7239 100644
--- a/tensorflow/python/keras/utils/tf_utils.py
+++ b/tensorflow/python/keras/utils/tf_utils.py
@@ -161,6 +161,9 @@ def are_all_symbolic_tensors(tensors):
   return all(is_symbolic_tensor(tensor) for tensor in tensors)
 
 
+_user_convertible_tensor_types = set()
+
+
 def is_symbolic_tensor(tensor):
   """Returns whether a tensor is symbolic (from a TF graph) or an eager tensor.
 
@@ -176,9 +179,40 @@ def is_symbolic_tensor(tensor):
   if isinstance(tensor, variables.Variable):
     return not context.executing_eagerly()
   if isinstance(tensor, (ops.Tensor, sparse_tensor.SparseTensor)):
-    try:
-      _ = tensor.graph
-      return True
-    except AttributeError:
-      return False
+    return hasattr(tensor, 'graph')
+  if isinstance(tensor, tuple(_user_convertible_tensor_types)):
+    return hasattr(ops.convert_to_tensor(tensor), 'graph')
   return False
+
+
+def register_symbolic_tensor_type(cls):
+  """Allows users to specify types regarded as symbolic `Tensor`s.
+
+  Used in conjunction with `tf.register_tensor_conversion_function`, calling
+  `tf.keras.utils.register_symbolic_tensor_type(cls)` allows non-`Tensor`
+  objects to be plumbed through Keras layers.
+
+  Example:
+
+  ```python
+  # One-time setup.
+  class Foo(object):
+    def __init__(self, input_):
+      self._input = input_
+    def value(self):
+      return tf.constant(42.)
+
+  tf.register_tensor_conversion_function(
+      Foo, lambda x, *args, **kwargs: x.value())
+
+  tf.keras.utils.register_symbolic_tensor_type(Foo)
+
+  # User-land.
+  layer = tf.keras.layers.Lambda(lambda input_: Foo(input_))
+  ```
+
+  Arguments:
+    cls: A `class` type which shall be regarded as a symbolic `Tensor`.
+  """
+  global _user_convertible_tensor_types
+  _user_convertible_tensor_types.add(cls)
diff --git a/tensorflow/python/keras/utils/tf_utils_test.py b/tensorflow/python/keras/utils/tf_utils_test.py
new file mode 100644
index 0000000000..9833a49299
--- /dev/null
+++ b/tensorflow/python/keras/utils/tf_utils_test.py
@@ -0,0 +1,134 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Keras TF utils."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python import keras
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras.utils import tf_utils
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TestIsSymbolicTensor(test.TestCase):
+
+  def test_default_behavior(self):
+    if context.executing_eagerly():
+      self.assertFalse(tf_utils.is_symbolic_tensor(
+          variables.Variable(name='blah', initial_value=0.)))
+      self.assertFalse(tf_utils.is_symbolic_tensor(
+          ops.convert_to_tensor(0.)))
+      self.assertFalse(tf_utils.is_symbolic_tensor(
+          sparse_tensor.SparseTensor(
+              indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])))
+    else:
+      self.assertTrue(tf_utils.is_symbolic_tensor(
+          variables.Variable(name='blah', initial_value=0.)))
+      self.assertTrue(tf_utils.is_symbolic_tensor(
+          ops.convert_to_tensor(0.)))
+      self.assertTrue(tf_utils.is_symbolic_tensor(
+          sparse_tensor.SparseTensor(
+              indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])))
+
+  def test_works_with_registered(self):
+
+    class CustomClass(object):
+
+      def value(self):
+        return ops.convert_to_tensor(42.)
+
+    ops.register_tensor_conversion_function(
+        CustomClass, lambda value, **_: value.value())
+
+    tf_utils.register_symbolic_tensor_type(CustomClass)
+
+    if context.executing_eagerly():
+      self.assertFalse(tf_utils.is_symbolic_tensor(
+          variables.Variable(name='blah', initial_value=0.)))
+      self.assertFalse(tf_utils.is_symbolic_tensor(
+          ops.convert_to_tensor(0.)))
+      self.assertFalse(tf_utils.is_symbolic_tensor(
+          sparse_tensor.SparseTensor(
+              indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])))
+      self.assertFalse(tf_utils.is_symbolic_tensor(CustomClass()))
+    else:
+      self.assertTrue(tf_utils.is_symbolic_tensor(
+          variables.Variable(name='blah', initial_value=0.)))
+      self.assertTrue(tf_utils.is_symbolic_tensor(
+          ops.convert_to_tensor(0.)))
+      self.assertTrue(tf_utils.is_symbolic_tensor(
+          sparse_tensor.SparseTensor(
+              indices=[[0, 0], [1, 2]], values=[1, 2], dense_shape=[3, 4])))
+      self.assertTrue(tf_utils.is_symbolic_tensor(CustomClass()))
+
+  def test_enables_nontensor_plumbing(self):
+    # Setup.
+
+    class Foo(object):
+
+      def __init__(self, input_):
+        self._input = input_
+        self.value = ops.convert_to_tensor(42.)
+
+    ops.register_tensor_conversion_function(
+        Foo, lambda x, *args, **kwargs: x.value)
+    tf_utils.register_symbolic_tensor_type(Foo)
+
+    class PlumbingLayer(keras.layers.Lambda):
+
+      def __init__(self, fn, **kwargs):
+        def _fn(*fargs, **fkwargs):
+          d = fn(*fargs, **fkwargs)
+          x = ops.convert_to_tensor(d)
+          d.shape = x.shape
+          d.get_shape = x.get_shape
+          return d, x
+        super(PlumbingLayer, self).__init__(_fn, **kwargs)
+        self._enter_dunder_call = False
+
+      def __call__(self, inputs, *args, **kwargs):
+        self._enter_dunder_call = True
+        d, _ = super(PlumbingLayer, self).__call__(inputs, *args, **kwargs)
+        self._enter_dunder_call = False
+        return d
+
+      def call(self, inputs, *args, **kwargs):
+        d, v = super(PlumbingLayer, self).call(inputs, *args, **kwargs)
+        if self._enter_dunder_call:
+          return d, v
+        return d
+
+    # User-land.
+    model = keras.Sequential([
+        keras.layers.InputLayer([]),
+        PlumbingLayer(Foo),  # Makes a `Foo` object.
+    ])
+    # Let's ensure Keras graph history is preserved by composing the models.
+    model = keras.Model(model.inputs, model(model.outputs))
+    # Now we instantiate the model and verify we have a `Foo` object, not a
+    # `Tensor`.
+    y = model(ops.convert_to_tensor(7.))
+    self.assertIsInstance(y, Foo)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From e19faa08c54a564dd8663cf18722f29977b81a17 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 29 Nov 2018 11:38:43 -0800
Subject: [PATCH 1036/1554] Fix tf.function on functools.partial objects

Needed some unwrapping in autograph.

PiperOrigin-RevId: 223382148
---
 tensorflow/python/autograph/impl/api.py       | 11 ++++++++++
 tensorflow/python/autograph/impl/api_test.py  | 21 +++++++++++++++++++
 .../python/autograph/impl/conversion.py       |  8 ++++++-
 tensorflow/python/eager/def_function_test.py  |  7 +++++++
 4 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 69674b2be3..19a472064a 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -195,6 +195,17 @@ def converted_call(f, owner, options, *args, **kwargs):
   if not options.internal_convert_user_code:
     return f(*args, **kwargs)
 
+  # Unwrap functools.partial objects
+  # TODO(allenl, mdan): Consider sharing unwrapping logic with tf_inspect.
+  while isinstance(f, functools.partial):
+    args = f.args + args
+    new_kwargs = {}
+    if f.keywords is not None:
+      new_kwargs.update(f.keywords)
+    new_kwargs.update(kwargs)
+    kwargs = new_kwargs
+    f = f.func
+
   if tf_inspect.isfunction(f) or tf_inspect.ismethod(f):
     # Regular functions
     target_entity = f
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 44cb99d657..a0fa501fb8 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 import gc
 
 import numpy as np
@@ -210,6 +211,26 @@ class ApiTest(test.TestCase):
                              constant_op.constant(-1))
       self.assertEqual(1, self.evaluate(x))
 
+  def test_converted_call_functools_partial(self):
+
+    def test_fn(x, y, z):
+      if x < 0:
+        return -x, -y, -z
+      return x, y, z
+
+    x = api.converted_call(
+        functools.partial(test_fn, constant_op.constant(-1), z=-3),
+        None, converter.ConversionOptions(),
+        constant_op.constant(-2))
+    self.assertEqual((1, 2, 3), self.evaluate(x))
+
+    x = api.converted_call(
+        functools.partial(
+            functools.partial(test_fn, constant_op.constant(-1)), z=-3),
+        None, converter.ConversionOptions(),
+        constant_op.constant(-2))
+    self.assertEqual((1, 2, 3), self.evaluate(x))
+
   def test_converted_call_method_explicit_owner(self):
     # TODO(mdan): Implement.
     pass
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 3dfc12eb76..055769c73a 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 import imp
 
 import gast
@@ -72,7 +73,12 @@ def is_whitelisted_for_graph(o):
   Returns:
     Boolean
   """
-  m = tf_inspect.getmodule(o)
+  if isinstance(o, functools.partial):
+    # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since
+    # functools.partial objects do not have a __module__ attribute.
+    m = functools
+  else:
+    m = tf_inspect.getmodule(o)
   for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
     if m.__name__.startswith(prefix):
       return True
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 54991344b7..da85735c47 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
@@ -207,6 +208,12 @@ class DefFunctionTest(test.TestCase):
     m1 = MyModel()
     self.assertAllEqual(m1.apply(3.0), 6.0)
 
+  def test_functools_partial(self):
+    self.assertAllClose(
+        3.,
+        def_function.function(functools.partial(lambda x, y: x + y, 1.))(
+            constant_op.constant(2.)))
+
   def test_optimizer(self):
     x = constant_op.constant([[3., 4.]])
     y = constant_op.constant([2.])
-- 
GitLab


From ba4761a4499a89fbfad3610d996ec3bae582d2c6 Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Thu, 29 Nov 2018 11:47:31 -0800
Subject: [PATCH 1037/1554] [XLA] Change BroadcastInDim interface to take a
 span of output dimension sizes instead of a shape.

PiperOrigin-RevId: 223383859
---
 .../compiler/tf2xla/kernels/permute_op.cc     |  3 +-
 .../compiler/tf2xla/kernels/resampler_ops.cc  | 32 +++-----
 .../tf2xla/kernels/xla_broadcast_helper_op.cc |  7 +-
 tensorflow/compiler/tf2xla/lib/broadcast.cc   |  6 +-
 tensorflow/compiler/xla/client/xla_builder.cc | 29 ++++---
 tensorflow/compiler/xla/client/xla_builder.h  |  8 +-
 .../compiler/xla/client/xla_builder_test.cc   |  4 +-
 .../xla/python/local_computation_builder.cc   | 75 +++++++++----------
 .../xla/python/local_computation_builder.h    |  3 +-
 tensorflow/compiler/xla/python/xla_client.py  |  3 +-
 .../xla/tests/broadcast_simple_test.cc        | 17 ++---
 11 files changed, 89 insertions(+), 98 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/permute_op.cc b/tensorflow/compiler/tf2xla/kernels/permute_op.cc
index 94b51e1a58..71920bf5c1 100644
--- a/tensorflow/compiler/tf2xla/kernels/permute_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/permute_op.cc
@@ -75,8 +75,7 @@ class DataFormatVecPermuteOp : public XlaOpKernel {
     }
     auto keys = xla::ConstantR1(builder, absl::Span<const int32>(dst_indices));
     if (input_rank == 2) {
-      keys = xla::BroadcastInDim(
-          keys, xla::ShapeUtil::MakeShape(xla::S32, {4, 2}), {0});
+      keys = xla::BroadcastInDim(keys, {4, 2}, {0});
     }
     auto sorted = xla::Sort(keys, {ctx->Input(0)}, 0);
     auto output = xla::GetTupleElement(sorted, 1);
diff --git a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
index 8a8f33c8f3..769e0cd140 100644
--- a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
@@ -67,11 +67,8 @@ XlaOp BilinearWeights(XlaOpKernelContext* ctx, XlaOp ratio,
   std::vector<int64> last_two_dims_indices = {(broadcast_dims_size - 2),
                                               (broadcast_dims_size - 1)};
 
-  xla::Shape broadcast_shape =
-      xla::ShapeUtil::MakeShape(xla_type, broadcast_dims);
-
   auto broadcast_first_term =
-      xla::BroadcastInDim(first_term, broadcast_shape, last_two_dims_indices);
+      xla::BroadcastInDim(first_term, broadcast_dims, last_two_dims_indices);
 
   // Ratio is of the same dimension as warp, which is [batch, dim_0,... dim_n,
   // 2], we broadcast ratio tensor to 'broadcast_dim' by keeping the
@@ -82,7 +79,7 @@ XlaOp BilinearWeights(XlaOpKernelContext* ctx, XlaOp ratio,
   ratio_broadcast_indices.erase(ratio_broadcast_indices.end() - 2);
 
   auto broadcast_ratio =
-      xla::BroadcastInDim(ratio, broadcast_shape, ratio_broadcast_indices);
+      xla::BroadcastInDim(ratio, broadcast_dims, ratio_broadcast_indices);
 
   auto first_term_subtract_weights = broadcast_first_term - broadcast_ratio;
 
@@ -93,7 +90,7 @@ XlaOp BilinearWeights(XlaOpKernelContext* ctx, XlaOp ratio,
   sign_change = xla::ConvertElementType(sign_change, xla_type);
 
   auto broadcast_sign_change =
-      xla::BroadcastInDim(sign_change, broadcast_shape, last_two_dims_indices);
+      xla::BroadcastInDim(sign_change, broadcast_dims, last_two_dims_indices);
 
   auto flipped = first_term_subtract_weights * broadcast_sign_change;
 
@@ -229,21 +226,19 @@ XlaOp CalculateGradData(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio,
 
   std::vector<int64> weights_with_channels_dims = reshaped_weights_dims;
   weights_with_channels_dims.push_back(data_channels);
-  auto weights_with_channels_shape =
-      xla::ShapeUtil::MakeShape(warp_type, weights_with_channels_dims);
   std::vector<int64> reshaped_weights_indices(reshaped_weights_dims.size());
   std::iota(reshaped_weights_indices.begin(), reshaped_weights_indices.end(),
             0);
 
   // The dimension is [batch, dim_0, ..., dim_n, 2, 2, data_channel].
   auto broadcast_reshaped_weights = xla::BroadcastInDim(
-      reshaped_weights, weights_with_channels_shape, reshaped_weights_indices);
+      reshaped_weights, weights_with_channels_dims, reshaped_weights_indices);
 
   std::vector<int64> grad_output_indices(warp_dims_without_last_dims.size());
   std::iota(grad_output_indices.begin(), grad_output_indices.end(), 0);
   grad_output_indices.push_back(weights_with_channels_dims.size() - 1);
   XlaOp broadcast_grad_output = xla::BroadcastInDim(
-      grad_output, weights_with_channels_shape, grad_output_indices);
+      grad_output, weights_with_channels_dims, grad_output_indices);
 
   auto grad_output_multiply_weights =
       broadcast_grad_output * broadcast_reshaped_weights;
@@ -291,13 +286,10 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio,
   std::vector<int64> warp_dims_without_last_dims(warp_dims.begin(),
                                                  warp_dims.end() - 1);
 
+  // With dimension [batch, dim_0, ...dim_n, 4]
   std::vector<int64> neighbor_broadcast_dims = warp_dims_without_last_dims;
   neighbor_broadcast_dims.push_back(4);
 
-  // With dimension [batch, dim_0, ...dim_n, 4]
-  auto neighbor_broadcast_shape =
-      xla::ShapeUtil::MakeShape(data_type, neighbor_broadcast_dims);
-
   // The dimension is [batch, dim_0, ... dim_n, 4, data_channels]
   auto neighbors_data = Gather2by2Neighbors(
       ctx->builder(), data, gather_indices, data_channels, warp_shape.dims());
@@ -323,7 +315,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio,
       xla::BroadcastInDim(
           xla::ConvertElementType(
               xla::ConstantR1<float>(ctx->builder(), {0, 0, -1, 1}), data_type),
-          neighbor_broadcast_shape, {last_warp_dim}),
+          neighbor_broadcast_dims, {last_warp_dim}),
       neighbors_data, dot_dims, /*precision_config=*/nullptr);
 
   // img_cxfy - img_fxfy
@@ -331,7 +323,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio,
       xla::BroadcastInDim(
           xla::ConvertElementType(
               xla::ConstantR1<float>(ctx->builder(), {-1, 1, 0, 0}), data_type),
-          neighbor_broadcast_shape, {last_warp_dim}),
+          neighbor_broadcast_dims, {last_warp_dim}),
       neighbors_data, dot_dims, /*precision_config=*/nullptr);
 
   // img_cxcy - img_cxfy
@@ -339,7 +331,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio,
       xla::BroadcastInDim(
           xla::ConvertElementType(
               xla::ConstantR1<float>(ctx->builder(), {0, -1, 0, 1}), data_type),
-          neighbor_broadcast_shape, {last_warp_dim}),
+          neighbor_broadcast_dims, {last_warp_dim}),
       neighbors_data, dot_dims, /*precision_config=*/nullptr);
 
   // img_fxcy - img_fxfy
@@ -347,7 +339,7 @@ XlaOp CalculateGradWarp(XlaOpKernelContext* ctx, XlaOp grad_output, XlaOp ratio,
       xla::BroadcastInDim(
           xla::ConvertElementType(
               xla::ConstantR1<float>(ctx->builder(), {-1, 0, 1, 0}), data_type),
-          neighbor_broadcast_shape, {last_warp_dim}),
+          neighbor_broadcast_dims, {last_warp_dim}),
       neighbors_data, dot_dims, /*precision_config=*/nullptr);
 
   // Slice out x and y.
@@ -491,13 +483,11 @@ class ResamplerOp : public XlaOpKernel {
     auto warp_dims = warp_shape.dim_sizes();
     std::vector<int64> result_dims(warp_dims.begin(), warp_dims.end() - 1);
     result_dims.push_back(data_channels);
-    xla::Shape broadcasted_shape =
-        xla::ShapeUtil::MakeShape(xla::PrimitiveType::PRED, result_dims);
 
     std::vector<int64> broadcasted_dims(warp_dims.size() - 1);
     std::iota(broadcasted_dims.begin(), broadcasted_dims.end(), 0);
     auto broadcasted_is_in_bound =
-        xla::BroadcastInDim(is_in_bound, broadcasted_shape, broadcasted_dims);
+        xla::BroadcastInDim(is_in_bound, result_dims, broadcasted_dims);
 
     // Set out of bound samples to zero.
     auto zeros =
diff --git a/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc b/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc
index a9f88a6df2..ad8e707e11 100644
--- a/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/xla_broadcast_helper_op.cc
@@ -89,13 +89,10 @@ class XlaBroadcastHelperOp : public XlaOpKernel {
               lhs_shape.DebugString(), " and ", rhs_shape.DebugString()));
       broadcast_shape[dim] = min_rank_shape->dim_size(i);
     }
-    xla::PrimitiveType type = context->input_xla_type(0);
-    xla::Shape broadcast_xla_shape =
-        xla::ShapeUtil::MakeShape(type, broadcast_shape);
     if (broadcast_lhs) {
-      lhs = xla::BroadcastInDim(lhs, broadcast_xla_shape, broadcast_dims);
+      lhs = xla::BroadcastInDim(lhs, broadcast_shape, broadcast_dims);
     } else {
-      rhs = xla::BroadcastInDim(rhs, broadcast_xla_shape, broadcast_dims);
+      rhs = xla::BroadcastInDim(rhs, broadcast_shape, broadcast_dims);
     }
     context->SetOutput(0, lhs);
     context->SetOutput(1, rhs);
diff --git a/tensorflow/compiler/tf2xla/lib/broadcast.cc b/tensorflow/compiler/tf2xla/lib/broadcast.cc
index 3e402ef855..be31f11668 100644
--- a/tensorflow/compiler/tf2xla/lib/broadcast.cc
+++ b/tensorflow/compiler/tf2xla/lib/broadcast.cc
@@ -80,10 +80,8 @@ xla::StatusOr<xla::XlaOp> BroadcastTo(xla::XlaOp input,
     broadcast_dim = broadcast_shape_size - broadcast_dim - 1;
   }
   absl::c_reverse(broadcast_shape);
-  xla::XlaOp output = xla::BroadcastInDim(
-      input,
-      xla::ShapeUtil::MakeShape(input_shape.element_type(), broadcast_shape),
-      broadcast_dims);
+  xla::XlaOp output =
+      xla::BroadcastInDim(input, broadcast_shape, broadcast_dims);
   if (broadcast_shape != output_dims) {
     output = xla::Reshape(output, output_dims);
   }
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 0630f3cf66..60df2ec395 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -573,27 +573,35 @@ XlaOp XlaBuilder::Broadcast(const XlaOp& operand,
 }
 
 XlaOp XlaBuilder::BroadcastInDim(
-    const XlaOp& operand, const Shape& shape,
+    const XlaOp& operand, const absl::Span<const int64> out_dim_size,
     const absl::Span<const int64> broadcast_dimensions) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
-    TF_RETURN_IF_ERROR(ShapeInference::InferBroadcastShape(operand_shape, shape,
-                                                           broadcast_dimensions)
+    // Output shape, in the case of degenerate broadcast, the out_dim_size is
+    // not necessarily the same as the dimension sizes of the output shape.
+    const auto& output_shape =
+        ShapeUtil::MakeShape(operand_shape.element_type(), out_dim_size);
+
+    TF_RETURN_IF_ERROR(ShapeInference::InferBroadcastShape(
+                           operand_shape, output_shape, broadcast_dimensions)
                            .status());
-    std::vector<int64> in_dim_size(ShapeUtil::Rank(shape));
-    absl::c_copy(shape.dimensions(), in_dim_size.begin());
+    std::vector<int64> in_dim_size(out_dim_size.begin(), out_dim_size.end());
     for (int i = 0; i < broadcast_dimensions.size(); i++) {
       in_dim_size[broadcast_dimensions[i]] = operand_shape.dimensions(i);
     }
     const auto& in_dim_shape =
-        ShapeUtil::MakeShape(shape.element_type(), in_dim_size);
+        ShapeUtil::MakeShape(operand_shape.element_type(), in_dim_size);
     TF_ASSIGN_OR_RETURN(
         XlaOp in_dim_broadcast,
         InDimBroadcast(in_dim_shape, operand, broadcast_dimensions));
-    if (ShapeUtil::Equal(in_dim_shape, shape)) {
+
+    // If broadcast is not degenerate, return broadcasted result.
+    if (ShapeUtil::Equal(in_dim_shape, output_shape)) {
       return in_dim_broadcast;
     }
-    return AddBroadcastSequence(shape, in_dim_broadcast);
+
+    // Otherwise handle degenerate broadcast case.
+    return AddBroadcastSequence(output_shape, in_dim_broadcast);
   });
 }
 
@@ -2665,9 +2673,10 @@ XlaOp Broadcast(const XlaOp& operand, absl::Span<const int64> broadcast_sizes) {
   return operand.builder()->Broadcast(operand, broadcast_sizes);
 }
 
-XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape,
+XlaOp BroadcastInDim(const XlaOp& operand,
+                     const absl::Span<const int64> out_dim_size,
                      const absl::Span<const int64> broadcast_dimensions) {
-  return operand.builder()->BroadcastInDim(operand, shape,
+  return operand.builder()->BroadcastInDim(operand, out_dim_size,
                                            broadcast_dimensions);
 }
 
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 78c90dbccc..098efb60f9 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -323,7 +323,8 @@ class XlaBuilder {
   XlaOp Broadcast(const XlaOp& operand,
                   absl::Span<const int64> broadcast_sizes);
 
-  XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape,
+  XlaOp BroadcastInDim(const XlaOp& operand,
+                       const absl::Span<const int64> out_dim_size,
                        const absl::Span<const int64> broadcast_dimensions);
 
   XlaOp Pad(const XlaOp& operand, const XlaOp& padding_value,
@@ -824,7 +825,7 @@ class XlaBuilder {
                          absl::Span<const int64> broadcast_sizes);
 
   friend XlaOp BroadcastInDim(
-      const XlaOp& operand, const Shape& shape,
+      const XlaOp& operand, const absl::Span<const int64> out_dim_size,
       const absl::Span<const int64> broadcast_dimensions);
 
   friend XlaOp Pad(const XlaOp& operand, const XlaOp& padding_value,
@@ -1217,7 +1218,8 @@ XlaOp Broadcast(const XlaOp& operand, absl::Span<const int64> broadcast_sizes);
 //   will generate output
 //   {{1 , 1},
 //    {2 , 2}}
-XlaOp BroadcastInDim(const XlaOp& operand, const Shape& shape,
+XlaOp BroadcastInDim(const XlaOp& operand,
+                     const absl::Span<const int64> out_dim_size,
                      const absl::Span<const int64> broadcast_dimensions);
 
 // Enqueues a pad operation onto the computation that pads the given value on
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index e534fb67fd..b3f5be300d 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -267,7 +267,7 @@ TEST_F(XlaBuilderTest, BinopHasInDimAndDegenerateBroadcast) {
 TEST_F(XlaBuilderTest, BroadcastInDim) {
   XlaBuilder b(TestName());
   auto x = Parameter(&b, 0, ShapeUtil::MakeShape(F32, {2, 3}), "x");
-  BroadcastInDim(x, ShapeUtil::MakeShape(F32, {2, 4, 3}),
+  BroadcastInDim(x, {2, 4, 3},
                  /*broadcast_dimensions=*/{0, 2});
   TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
   auto root = module->entry_computation()->root_instruction();
@@ -277,7 +277,7 @@ TEST_F(XlaBuilderTest, BroadcastInDim) {
 TEST_F(XlaBuilderTest, BroadcastInDimWithDegeneratedDim) {
   XlaBuilder b(TestName());
   auto x = Parameter(&b, 0, ShapeUtil::MakeShape(F32, {2, 1, 4}), "x");
-  BroadcastInDim(x, ShapeUtil::MakeShape(F32, {2, 3, 4}),
+  BroadcastInDim(x, {2, 3, 4},
                  /*broadcast_dimensions=*/{0, 1, 2});
   TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
   EXPECT_THAT(module->entry_computation()->root_instruction(),
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 2768ed618d..c0b57e7d26 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -323,42 +323,41 @@ StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
                                         GetReplicaCount());
 
     for (int replica = 0; replica < GetReplicaCount(); ++replica) {
-      pool.Schedule(
-          [this, client, replica, &argument_handles, &results] {
-            StatusOr<int> device_ordinal_status =
-                client->ReplicaNumberToDeviceOrdinal(replica);
-            if (!device_ordinal_status.ok()) {
-              results[replica] = device_ordinal_status.status();
-              return;
-            }
-            const int device_ordinal = device_ordinal_status.ValueOrDie();
-            VLOG(3) << "Replica " << replica
-                    << " mapped to device ordinal for execution: "
-                    << device_ordinal;
-
-            std::vector<const ShapedBuffer*> argument_buffers;
-            argument_buffers.reserve(argument_handles.size());
-            for (auto& handle : argument_handles) {
-              argument_buffers.push_back(handle->shaped_buffer());
-            }
-
-            DeviceAssignment device_assignment =
-                client->backend()
-                    .computation_placer()
-                    ->AssignDevices(GetReplicaCount(), /*computation_count=*/1)
-                    .ConsumeValueOrDie();
-
-            ExecutableRunOptions options;
-            options.set_device_ordinal(device_ordinal);
-            options.set_allocator(client->backend().memory_allocator());
-            options.set_intra_op_thread_pool(
-                client->backend().eigen_intra_op_thread_pool_device());
-            options.set_device_assignment(&device_assignment);
-            StatusOr<ScopedShapedBuffer> result_buffer_status =
-                executable_->Run(argument_buffers, options);
-
-            results[replica] = std::move(result_buffer_status);
-          });
+      pool.Schedule([this, client, replica, &argument_handles, &results] {
+        StatusOr<int> device_ordinal_status =
+            client->ReplicaNumberToDeviceOrdinal(replica);
+        if (!device_ordinal_status.ok()) {
+          results[replica] = device_ordinal_status.status();
+          return;
+        }
+        const int device_ordinal = device_ordinal_status.ValueOrDie();
+        VLOG(3) << "Replica " << replica
+                << " mapped to device ordinal for execution: "
+                << device_ordinal;
+
+        std::vector<const ShapedBuffer*> argument_buffers;
+        argument_buffers.reserve(argument_handles.size());
+        for (auto& handle : argument_handles) {
+          argument_buffers.push_back(handle->shaped_buffer());
+        }
+
+        DeviceAssignment device_assignment =
+            client->backend()
+                .computation_placer()
+                ->AssignDevices(GetReplicaCount(), /*computation_count=*/1)
+                .ConsumeValueOrDie();
+
+        ExecutableRunOptions options;
+        options.set_device_ordinal(device_ordinal);
+        options.set_allocator(client->backend().memory_allocator());
+        options.set_intra_op_thread_pool(
+            client->backend().eigen_intra_op_thread_pool_device());
+        options.set_device_assignment(&device_assignment);
+        StatusOr<ScopedShapedBuffer> result_buffer_status =
+            executable_->Run(argument_buffers, options);
+
+        results[replica] = std::move(result_buffer_status);
+      });
     }
   }
 
@@ -585,9 +584,9 @@ LocalOp LocalComputationBuilder::Broadcast(
 }
 
 LocalOp LocalComputationBuilder::BroadcastInDim(
-    const LocalOp& operand, const Shape& shape,
+    const LocalOp& operand, absl::Span<const int64> out_dim_sizes,
     absl::Span<const int64> broadcast_dimensions) {
-  return xla::BroadcastInDim(operand.op(), shape, broadcast_dimensions);
+  return xla::BroadcastInDim(operand.op(), out_dim_sizes, broadcast_dimensions);
 }
 
 LocalOp LocalComputationBuilder::Pad(const LocalOp& operand,
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 9e617c48bd..c9b7ae824a 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -282,7 +282,8 @@ class LocalComputationBuilder {
   LocalOp Broadcast(const LocalOp& operand,
                     absl::Span<const int64> broadcast_sizes);
 
-  LocalOp BroadcastInDim(const LocalOp& operand, const Shape& shape,
+  LocalOp BroadcastInDim(const LocalOp& operand,
+                         absl::Span<const int64> out_dim_sizes,
                          absl::Span<const int64> broadcast_dimensions);
 
   LocalOp Pad(const LocalOp& operand, const LocalOp& padding_value,
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 5994e55387..e5fba0d7ac 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -778,8 +778,7 @@ class ComputationBuilder(object):
     Returns:
       A LocalOp representing the added broadcast-in-dimensions op.
     """
-    xla_shape = Shape.array_shape(self.GetShape(operand).element_type(), shape)
-    return self._client.BroadcastInDim(operand, xla_shape, broadcast_dimensions)
+    return self._client.BroadcastInDim(operand, shape, broadcast_dimensions)
 
   def Concatenate(self, operands, dimension):
     """Enqueues a concatenate operation onto the computation.
diff --git a/tensorflow/compiler/xla/tests/broadcast_simple_test.cc b/tensorflow/compiler/xla/tests/broadcast_simple_test.cc
index dde19fb65d..702fb32adf 100644
--- a/tensorflow/compiler/xla/tests/broadcast_simple_test.cc
+++ b/tensorflow/compiler/xla/tests/broadcast_simple_test.cc
@@ -161,8 +161,7 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D) {
 
 XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsUsual) {
   XlaBuilder b(TestName());
-  BroadcastInDim(ConstantR1<float>(&b, {1, 2}),
-                 ShapeUtil::MakeShape(F32, {2, 2}), {1});
+  BroadcastInDim(ConstantR1<float>(&b, {1, 2}), {2, 2}, {1});
 
   Array2D<float> expected(2, 2);
   expected(0, 0) = 1;
@@ -175,8 +174,7 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsUsual) {
 
 XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsTranspose) {
   XlaBuilder b(TestName());
-  BroadcastInDim(ConstantR1<float>(&b, {1, 2}),
-                 ShapeUtil::MakeShape(F32, {2, 2}), {0});
+  BroadcastInDim(ConstantR1<float>(&b, {1, 2}), {2, 2}, {0});
 
   Array2D<float> expected(2, 2);
   expected(0, 0) = 1;
@@ -189,8 +187,8 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsTranspose) {
 
 XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDims) {
   XlaBuilder b(TestName());
-  BroadcastInDim(ConstantR2<float>(&b, {{1.0, 5.0}, {2.0, 6.0}}),
-                 ShapeUtil::MakeShape(F32, {2, 2, 2}), {0, 1});
+  BroadcastInDim(ConstantR2<float>(&b, {{1.0, 5.0}, {2.0, 6.0}}), {2, 2, 2},
+                 {0, 1});
 
   Array3D<float> expected(2, 2, 2);
   expected(0, 0, 0) = 1.0;
@@ -207,8 +205,8 @@ XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDims) {
 
 XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDimsNotPossibleWithBroadCast) {
   XlaBuilder b(TestName());
-  BroadcastInDim(ConstantR2<float>(&b, {{1.0, 5.0}, {2.0, 6.0}}),
-                 ShapeUtil::MakeShape(F32, {2, 2, 2}), {0, 2});
+  BroadcastInDim(ConstantR2<float>(&b, {{1.0, 5.0}, {2.0, 6.0}}), {2, 2, 2},
+                 {0, 2});
 
   Array3D<float> expected(2, 2, 2);
   expected(0, 0, 0) = 1.0;
@@ -225,8 +223,7 @@ XLA_TEST_F(BroadcastSimpleTest, 2DTo3D_WithDimsNotPossibleWithBroadCast) {
 
 XLA_TEST_F(BroadcastSimpleTest, 1DTo2D_WithDimsNotPossibleWithBroadCast) {
   XlaBuilder b(TestName());
-  BroadcastInDim(ConstantR1<float>(&b, {1, 2}),
-                 ShapeUtil::MakeShape(F32, {3, 2}), {1});
+  BroadcastInDim(ConstantR1<float>(&b, {1, 2}), {3, 2}, {1});
 
   Array2D<float> expected(3, 2);
   expected(0, 0) = 1;
-- 
GitLab


From f1668a718e287b1178f3a3903602d52a2820607e Mon Sep 17 00:00:00 2001
From: Robert Neale <rneale@google.com>
Date: Thu, 29 Nov 2018 11:53:20 -0800
Subject: [PATCH 1038/1554] Add new unicode_encode op, which encodes integer
 codepoints into the desired unicode formatted string.

RELNOTES: Add new unicode_encode op.
PiperOrigin-RevId: 223384812
---
 tensorflow/core/BUILD                         |   6 +-
 .../base_api/api_def_UnicodeEncode.pbtxt      |  73 +++++
 tensorflow/core/kernels/BUILD                 |   4 +-
 tensorflow/core/kernels/unicode_ops.cc        | 154 +++++++--
 tensorflow/core/ops/string_ops.cc             |  36 +++
 tensorflow/python/__init__.py                 |   1 +
 tensorflow/python/kernel_tests/BUILD          |  15 +
 .../kernel_tests/unicode_encode_op_test.py    | 301 ++++++++++++++++++
 tensorflow/python/ops/ragged/BUILD            |  16 +
 tensorflow/python/ops/ragged/__init__.py      |   1 +
 .../python/ops/ragged/ragged_string_ops.py    | 119 +++++++
 .../api/golden/v1/tensorflow.strings.pbtxt    |   4 +
 .../api/golden/v2/tensorflow.strings.pbtxt    |   4 +
 13 files changed, 702 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_UnicodeEncode.pbtxt
 create mode 100644 tensorflow/python/kernel_tests/unicode_encode_op_test.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_string_ops.py

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 97628a2561..c268605711 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1117,7 +1117,11 @@ tf_gen_op_libs(
     op_lib_names = [
         "string_ops",
     ],
-    deps = ["@com_google_absl//absl/strings"],
+    deps = [
+        ":lib_internal",
+        ":lib_proto_parsing",
+        "@com_google_absl//absl/strings",
+    ],
 )
 
 tf_gen_op_libs(
diff --git a/tensorflow/core/api_def/base_api/api_def_UnicodeEncode.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnicodeEncode.pbtxt
new file mode 100644
index 0000000000..26f7865860
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_UnicodeEncode.pbtxt
@@ -0,0 +1,73 @@
+op {
+  graph_op_name: "UnicodeEncode"
+  visibility: HIDDEN
+  endpoint {
+    name: "UnicodeEncode"
+  }
+  in_arg {
+    name: "input_values"
+    description: <<END
+A 1D tensor containing the unicode codepoints that should be encoded.
+END
+  }
+  in_arg {
+    name: "input_splits"
+    description: <<END
+A 1D tensor specifying how the unicode codepoints should be split into strings.
+In particular, `output[i]` is constructed by encoding the codepoints in the
+slice `input_values[input_splits[i]:input_splits[i+1]]`.
+END
+  }
+  attr {
+    name: "output_encoding"
+    description: <<END
+Unicode encoding of the output strings. Valid encodings are: `"UTF-8",
+"UTF-16-BE", and "UTF-32-BE"`.
+END
+  }
+  attr {
+    name: "errors"
+    description: <<END
+Error handling policy when there is invalid formatting found in the input.
+The value of 'strict' will cause the operation to produce a InvalidArgument
+error on any invalid input formatting. A value of 'replace' (the default) will
+cause the operation to replace any invalid formatting in the input with the
+`replacement_char` codepoint. A value of 'ignore' will cause the operation to
+skip any invalid formatting in the input and produce no corresponding output
+character.
+END
+  }
+  attr {
+    name: "replacement_char"
+    description: <<END
+The replacement character codepoint to be used in place of any invalid
+formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+be used. The default value is the default unicode replacement character is
+0xFFFD (U+65533).
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+The 1-D Tensor of strings encoded from the provided unicode codepoints.
+END
+  }
+  summary: "Encode a tensor of ints into unicode strings."
+  description: <<END
+Returns a vector of strings, where `output[i]` is constructed by encoding the
+Unicode codepoints in `input_values[input_splits[i]:input_splits[i+1]]`
+using `output_encoding`.
+
+---
+
+Example:
+
+```
+input_values = [72, 101, 108, 108, 111, 87, 111, 114, 108, 100]
+input_splits = [0, 5, 10]
+output_encoding = 'UTF-8'
+
+output = ['Hello', 'World']
+```
+END
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1efce939a3..986b21045d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4826,11 +4826,13 @@ tf_kernel_library(
     name = "unicode_ops",
     prefix = "unicode_ops",
     deps = [
+        ":bounds_check",
         ":string_util",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:string_ops_op_lib",
+        "//third_party/eigen3",
         "//third_party/icu/data:conversion_data",
         "@icu//:common",
     ],
@@ -5731,8 +5733,8 @@ filegroup(
             "batch_kernels.*",
             "regex_full_match_op.cc",
             "regex_replace_op.cc",
-            "unicode_script_op.cc",
             "unicode_ops.cc",
+            "unicode_script_op.cc",
             # Ops that are inherently incompatible with Android (e.g. tied to x86 platform).
             "mkl_*",
             "xsmm_*",
diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc
index eb0c1d1228..6c4ed1eaaf 100644
--- a/tensorflow/core/kernels/unicode_ops.cc
+++ b/tensorflow/core/kernels/unicode_ops.cc
@@ -13,9 +13,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <stdint.h>
+#include <cstddef>
+#include <functional>
 #include <memory>
 #include <string>
+#include <vector>
 
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "unicode/appendable.h"  // TF:icu
+#include "unicode/schriter.h"  // TF:icu
+#include "unicode/uchar.h"  // TF:icu
 #include "unicode/ucnv.h"  // TF:icu
 #include "unicode/ucnv_err.h"  // TF:icu
 #include "unicode/umachine.h"  // TF:icu
@@ -23,18 +31,58 @@ limitations under the License.
 #include "unicode/unistr.h"  // TF:icu
 #include "unicode/uset.h"  // TF:icu
 #include "unicode/utypes.h"  // TF:icu
+#include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/string_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/bcast.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
-
 namespace {
 
+void Encode(const UnicodeEncoding encoding, const icu::UnicodeString& in,
+            string* out) {
+  if (encoding == UnicodeEncoding::UTF8) {
+    out->clear();
+    in.toUTF8String(*out);
+  } else if (encoding == UnicodeEncoding::UTF16BE) {
+    // TODO(gbillock): consider using the
+    // extract(char *dest, int32_t destCapacity, UConverter *cnv)
+    // for UTF16/32
+    out->clear();  // subtle: must come before reserve()
+    out->reserve(2 * in.length() + 1);
+    const char16_t* buf = in.getBuffer();
+    for (int i = 0; i < in.length(); ++i) {
+      // Emit big-endian encoding for UTF-16 always.
+      out->push_back((buf[i] & 0xFF00) >> 8);
+      out->push_back(buf[i] & 0x00FF);
+    }
+  } else if (encoding == UnicodeEncoding::UTF32BE) {
+    out->clear();  // subtle: must come before reserve()
+    out->reserve(4 * in.countChar32() + 1);
+    icu::StringCharacterIterator it(in);
+    UChar32 ch;
+    while (it.hasNext()) {
+      ch = it.next32PostInc();
+      out->push_back((ch & 0xFF000000) >> 24);
+      out->push_back((ch & 0x00FF0000) >> 16);
+      out->push_back((ch & 0x0000FF00) >> 8);
+      out->push_back((ch & 0x000000FF));
+    }
+  }
+}
+
 // This error callback is only useful for finding illegal encoding errors when
 // we want to be strict -- otherwise illegal encodings are replaced on read
 // with 0xFFFD and signaled to the callback.
@@ -183,8 +231,10 @@ Status GetErrorOptions(OpKernelConstruction* ctx, ErrorOptions* out) {
         "replacement_char out of unicode codepoint range");
   }
 
-  TF_RETURN_IF_ERROR(ctx->GetAttr("replace_control_characters",
-                                  &(out->replace_control_chars)));
+  if (ctx->HasAttr("replace_control_characters")) {
+    TF_RETURN_IF_ERROR(ctx->GetAttr("replace_control_characters",
+                                    &(out->replace_control_chars)));
+  }
 
   return Status::OK();
 }
@@ -289,33 +339,7 @@ class UnicodeTranscodeOp : public OpKernel {
                   found_any_format_error, std::placeholders::_1,
                   std::placeholders::_2, std::placeholders::_3));
 
-    if (output_encoding_ == UnicodeEncoding::UTF8) {
-      s->clear();
-      source.toUTF8String(*s);
-    } else if (output_encoding_ == UnicodeEncoding::UTF16BE) {
-      // TODO(gbillock): consider using the
-      // extract(char *dest, int32_t destCapacity, UConverter *cnv)
-      // for UTF16/32
-      s->clear();  // subtle: must come before reserve()
-      s->reserve(2 * source.length() + 1);
-      const char16_t* buf = source.getBuffer();
-      for (int i = 0; i < source.length(); ++i) {
-        // Emit big-endian encoding for UTF-16 always.
-        s->push_back((buf[i] & 0xFF00) >> 8);
-        s->push_back(buf[i] & 0x00FF);
-      }
-    } else if (output_encoding_ == UnicodeEncoding::UTF32BE) {
-      s->clear();  // subtle: must come before reserve()
-      s->reserve(4 * source.countChar32() + 1);
-      for (int i = 0; i < source.countChar32(); ++i) {
-        // Emit big-endian encoding for UTF-32 always.
-        UChar32 ch = source.char32At(i);
-        s->push_back((ch & 0xFF000000) >> 24);
-        s->push_back((ch & 0x00FF0000) >> 16);
-        s->push_back((ch & 0x0000FF00) >> 8);
-        s->push_back((ch & 0x000000FF));
-      }
-    }
+    Encode(output_encoding_, source, s);
   }
 
   string input_encoding_;
@@ -443,4 +467,74 @@ class UnicodeDecodeWithOffsetsOp : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("UnicodeDecodeWithOffsets").Device(DEVICE_CPU),
                         UnicodeDecodeWithOffsetsOp);
 
+class UnicodeEncodeOp : public OpKernel {
+ public:
+  explicit UnicodeEncodeOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    string encoding_tmp;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_encoding", &encoding_tmp));
+    OP_REQUIRES_OK(ctx, ParseUnicodeEncoding(encoding_tmp, &encoding_));
+    OP_REQUIRES_OK(ctx, GetErrorOptions(ctx, &error_options_));
+  }
+
+  /**
+   * Encodes Unicode codepoints into the desired string representation.
+   *
+   * We lose a dimension while encoding, since a series of integer codepoints is
+   * encoded into a single string.
+   *
+   * This accepts two input tensors: a rank 1 tensor of code point values and
+   * a single rank 1 tensor of splits which determine where each string begins
+   * and ends from the provided code points.
+   */
+  void Compute(OpKernelContext* context) override {
+    // Get inputs
+    const Tensor& input_tensor = context->input(0);
+    const auto input_tensor_flat = input_tensor.flat<int32>();
+    const Tensor& input_splits = context->input(1);
+    const auto input_splits_flat = input_splits.flat<int64>();
+
+    // Since we limit to a 2-D input (inner_values of rank 1 and a single splits
+    // tensor), our output dimension will be 1 with it's size equal to the
+    // number of splits (outer dimension or ragged tensor).
+    TensorShape output_shape({input_splits.dim_size(0) - 1});
+    Tensor* output_tensor;
+    OP_REQUIRES_OK(context, context->allocate_output("output", output_shape,
+                                                     &output_tensor));
+    auto output_tensor_flat = output_tensor->flat<string>();
+
+    // Use a single index over the flattened input values tensor.
+    int idx = 0;
+    // Loop through our split dimension to create a new string at each split.
+    for (int i = 1; i < input_splits_flat.size(); ++i) {
+      icu::UnicodeString unicode_string;
+      icu::UnicodeStringAppendable appendable_unicode_string(unicode_string);
+      for (; idx < input_splits_flat(i); ++idx) {
+        int32 code_point = input_tensor_flat(idx);
+        // Check for invalid code point
+        if (code_point > UCHAR_MAX_VALUE || code_point < UCHAR_MIN_VALUE) {
+          if (error_options_.error_on_malformatting) {
+            context->CtxFailure(errors::InvalidArgument(
+                "Code point value out of valid Unicode range."));
+            return;
+          } else if (!error_options_.elide_replacement) {
+            code_point = error_options_.subst;
+          }
+        }
+        appendable_unicode_string.appendCodePoint(code_point);
+      }
+      // Encode our string and save in the output.
+      string result;
+      Encode(encoding_, unicode_string, &result);
+      output_tensor_flat(i - 1) = result;
+    }
+  }
+
+ private:
+  UnicodeEncoding encoding_;
+  ErrorOptions error_options_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("UnicodeEncode").Device(DEVICE_CPU),
+                        UnicodeEncodeOp);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/string_ops.cc b/tensorflow/core/ops/string_ops.cc
index fbecff11df..8ea74f1d43 100644
--- a/tensorflow/core/ops/string_ops.cc
+++ b/tensorflow/core/ops/string_ops.cc
@@ -13,13 +13,24 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <string>
+#include <vector>
+
 #include "absl/strings/str_split.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
 
+namespace shape_inference {
+class InferenceContext;
+}  // namespace shape_inference
+
 using shape_inference::DimensionHandle;
 using shape_inference::InferenceContext;
 using shape_inference::ShapeHandle;
@@ -250,6 +261,31 @@ REGISTER_OP("UnicodeScript")
     .Output("output: int32")
     .SetShapeFn(shape_inference::UnchangedShape);
 
+REGISTER_OP("UnicodeEncode")
+    .Input("input_values: int32")
+    .Input("input_splits: int64")
+    .Attr("errors: {'ignore', 'replace', 'strict'} = 'replace'")
+    .Attr("output_encoding: {'UTF-8', 'UTF-16-BE', 'UTF-32-BE'}")
+    .Attr("replacement_char: int = 65533")  // 0xFFFD unicode replacement char
+    .Output("output: string")
+    .SetShapeFn([](InferenceContext* c) {
+      // Check rank of inner values
+      ShapeHandle input_inner_values_shape = c->input(0);
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(input_inner_values_shape, 1, &unused));
+
+      // Check rank of input_splits
+      ShapeHandle splits_shape = c->input(1);
+      TF_RETURN_IF_ERROR(c->WithRank(splits_shape, 1, &unused));
+
+      // Output shape is a 1-D tensor with size equal to number of splits.
+      std::vector<DimensionHandle> dims(1);
+      TF_RETURN_IF_ERROR(c->Subtract(c->Dim(splits_shape, 0), 1, &dims[0]));
+      c->set_output(0, c->MakeShape(dims));
+
+      return Status::OK();
+    });
+
 REGISTER_OP("UnicodeTranscode")
     .Input("input: string")
     .Output("output: string")
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 3b462c7de8..547043030b 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -86,6 +86,7 @@ from tensorflow.python.ops import image_ops as image
 from tensorflow.python.ops import manip_ops as manip
 from tensorflow.python.ops import metrics
 from tensorflow.python.ops import nn
+from tensorflow.python.ops import ragged
 from tensorflow.python.ops import sets
 from tensorflow.python.ops.distributions import distributions
 from tensorflow.python.ops.linalg import linalg
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index de06ec622a..97ac21b8ad 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1157,6 +1157,21 @@ cuda_py_test(
     ],
 )
 
+tf_py_test(
+    name = "unicode_encode_op_test",
+    size = "small",
+    srcs = ["unicode_encode_op_test.py"],
+    additional_deps = [
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:errors",
+        "//tensorflow/python/ops/ragged:ragged_factory_ops",
+        "//tensorflow/python/ops/ragged:ragged_string_ops",
+    ],
+)
+
 tf_py_test(
     name = "unicode_transcode_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/unicode_encode_op_test.py b/tensorflow/python/kernel_tests/unicode_encode_op_test.py
new file mode 100644
index 0000000000..a5a5c2017c
--- /dev/null
+++ b/tensorflow/python/kernel_tests/unicode_encode_op_test.py
@@ -0,0 +1,301 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for UnicodeEncode op from ragged_string_ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors_impl as errors
+from tensorflow.python.framework import ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_string_ops
+from tensorflow.python.platform import test
+
+
+class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
+
+  def testScalar(self):
+    with self.cached_session():
+      with self.assertRaises(ValueError):
+        ragged_string_ops.unicode_encode(72, "UTF-8")
+    with self.cached_session():
+      with self.assertRaises(ValueError):
+        ragged_string_ops.unicode_encode(constant_op.constant(72), "UTF-8")
+
+  def testRequireParams(self):
+    with self.cached_session():
+      with self.assertRaises(TypeError):
+        ragged_string_ops.unicode_encode()
+    with self.cached_session():
+      with self.assertRaises(TypeError):
+        ragged_string_ops.unicode_encode(72)
+    with self.cached_session():
+      with self.assertRaises(TypeError):
+        ragged_string_ops.unicode_encode(encoding="UTF-8")
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testStrictErrors(self, encoding):
+    test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
+    with self.cached_session():
+      with self.assertRaises(errors.InvalidArgumentError):
+        ragged_string_ops.unicode_encode(test_value, encoding, "strict").eval()
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testIgnoreErrors(self, encoding):
+    test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
+    expected_value = u"Heo".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
+                                                         "ignore")
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testReplaceErrors(self, encoding):
+    test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
+    expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
+                                                         "replace")
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+    # Test custom replacement character
+    test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
+    expected_value = u"Heooo".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
+                                                         "replace", 111)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+    # Verify "replace" is default
+    test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
+    expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+    # Replacement_char must be within range
+    test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
+                                                         "replace", 1114112)
+    with self.cached_session():
+      with self.assertRaises(errors.InvalidArgumentError):
+        unicode_encode_op.eval()
+
+  # -- regular Tensor tests -- #
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testVector(self, encoding):
+    test_value = np.array([72, 101, 108, 108, 111], np.int32)
+    expected_value = u"Hello".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+    test_value = np.array([72, 101, 195, 195, 128516], np.int32)
+    expected_value = u"He\xc3\xc3\U0001f604".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+    # Single character string
+    test_value = np.array([72], np.int32)
+    expected_value = u"H".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+    test_value = np.array([128516], np.int32)
+    expected_value = u"\U0001f604".encode(encoding)
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(result, bytes)
+      self.assertAllEqual(result, expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testMatrix(self, encoding):
+    test_value = np.array(
+        [[72, 128516, 108, 108, 111], [87, 128516, 114, 108, 100]], np.int32)
+    expected_value = [
+        u"H\U0001f604llo".encode(encoding), u"W\U0001f604rld".encode(encoding)
+    ]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(unicode_encode_op, ops.Tensor)
+      self.assertAllEqual(result, expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def test3DimMatrix(self, encoding):
+    test_value = constant_op.constant(
+        [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]],
+         [[102, 105, 120, 101, 100], [119, 111, 114, 100, 115]],
+         [[72, 121, 112, 101, 114], [99, 117, 98, 101, 46]]], np.int32)
+    expected_value = [[u"Hello".encode(encoding), u"World".encode(encoding)],
+                      [u"fixed".encode(encoding), u"words".encode(encoding)],
+                      [u"Hyper".encode(encoding), u"cube.".encode(encoding)]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(unicode_encode_op, ops.Tensor)
+      self.assertAllEqual(result, expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def test4DimMatrix(self, encoding):
+    test_value = constant_op.constant(
+        [[[[72, 101, 108, 108, 111]], [[87, 111, 114, 108, 100]]],
+         [[[102, 105, 120, 101, 100]], [[119, 111, 114, 100, 115]]],
+         [[[72, 121, 112, 101, 114]], [[99, 117, 98, 101, 46]]]], np.int32)
+    expected_value = [[[u"Hello".encode(encoding)],
+                       [u"World".encode(encoding)]],
+                      [[u"fixed".encode(encoding)],
+                       [u"words".encode(encoding)]],
+                      [[u"Hyper".encode(encoding)],
+                       [u"cube.".encode(encoding)]]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(unicode_encode_op, ops.Tensor)
+      self.assertAllEqual(result, expected_value)
+
+  # -- Ragged Tensor tests -- #
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testRaggedMatrix(self, encoding):
+    test_value = ragged_factory_ops.constant(
+        [[72, 195, 108, 108, 111], [87, 128516, 114, 108, 100, 46]], np.int32)
+    expected_value = [
+        u"H\xc3llo".encode(encoding), u"W\U0001f604rld.".encode(encoding)
+    ]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertIsInstance(unicode_encode_op, ops.Tensor)
+      self.assertAllEqual(result, expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def test3DimMatrixWithRagged2ndDim(self, encoding):
+    test_value = ragged_factory_ops.constant(
+        [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]],
+         [[102, 105, 120, 101, 100]],
+         [[72, 121, 112, 101, 114], [119, 111, 114, 100, 115],
+          [99, 117, 98, 101, 46]]], np.int32)
+    expected_value = [[u"Hello".encode(encoding), u"World".encode(encoding)],
+                      [u"fixed".encode(encoding)],
+                      [
+                          u"Hyper".encode(encoding), u"words".encode(encoding),
+                          u"cube.".encode(encoding)
+                      ]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertEqual(unicode_encode_op.ragged_rank, 1)
+      self.assertAllEqual(result.tolist(), expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def test3DimMatrixWithRagged3rdDim(self, encoding):
+    test_value = ragged_factory_ops.constant(
+        [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]],
+         [[68, 111, 110, 39, 116], [119, 195, 114, 114, 121, 44, 32, 98, 101]],
+         [[128516], []]], np.int32)
+    expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)],
+                      [
+                          u"Don't".encode(encoding),
+                          u"w\xc3rry, be".encode(encoding)
+                      ], [u"\U0001f604".encode(encoding), u"".encode(encoding)]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertEqual(unicode_encode_op.ragged_rank, 1)
+      self.assertAllEqual(result.tolist(), expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def test3DimMatrixWithRagged2ndAnd3rdDim(self, encoding):
+    test_value = ragged_factory_ops.constant(
+        [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]], [],
+         [[128516]]], np.int32)
+    expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)],
+                      [], [u"\U0001f604".encode(encoding)]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertEqual(unicode_encode_op.ragged_rank, 1)
+      self.assertAllEqual(result.tolist(), expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def test4DimRaggedMatrix(self, encoding):
+    test_value = ragged_factory_ops.constant(
+        [[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]]],
+         [[[]], [[72, 121, 112, 101]]]], np.int32)
+    expected_value = [[[u"Hello".encode(encoding), u"World".encode(encoding)]],
+                      [[u"".encode(encoding)], [u"Hype".encode(encoding)]]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertEqual(unicode_encode_op.ragged_rank, 2)
+      self.assertAllEqual(result.tolist(), expected_value)
+
+  @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  def testRaggedMatrixWithMultiDimensionInnerValues(self, encoding):
+    test_inner_values = constant_op.constant([[[72, 101, 108, 108, 111],
+                                               [87, 111, 114, 108, 100]],
+                                              [[102, 105, 120, 101, 100],
+                                               [119, 111, 114, 100, 115]],
+                                              [[72, 121, 112, 101, 114],
+                                               [99, 117, 98, 101, 46]]])
+    test_row_splits = [
+        constant_op.constant([0, 2, 3], dtype=np.int64),
+        constant_op.constant([0, 1, 1, 3], dtype=np.int64)
+    ]
+    test_value = ragged_factory_ops.from_nested_row_splits(test_inner_values,
+                                                           test_row_splits)
+    expected_value = [[[[u"Hello".encode(encoding), u"World".encode(encoding)]],
+                       []],
+                      [[[u"fixed".encode(encoding), u"words".encode(encoding)],
+                        [u"Hyper".encode(encoding),
+                         u"cube.".encode(encoding)]]]]
+    unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
+    with self.cached_session():
+      result = unicode_encode_op.eval()
+      self.assertEqual(unicode_encode_op.ragged_rank, 2)
+      self.assertAllEqual(result.tolist(), expected_value)
+      # These next two assertions don't necessarily need to be here as they test
+      # internal representations and we already verified the value is correct.
+      self.assertAllEqual(len(result.nested_row_splits), len(test_row_splits))
+      self.assertEqual(unicode_encode_op.inner_values.shape.ndims,
+                       test_inner_values.shape.ndims - 1)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 8608bda647..e335c5cb6f 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -32,6 +32,7 @@ py_library(
         ":ragged_map_ops",
         ":ragged_math_ops",
         ":ragged_operators",
+        ":ragged_string_ops",
         ":ragged_tensor",
         ":ragged_tensor_shape",
         ":ragged_tensor_value",
@@ -180,6 +181,21 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_string_ops",
+    srcs = ["ragged_string_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_conversion_ops",
+        ":ragged_factory_ops",
+        ":ragged_tensor",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:util",
+    ],
+)
+
 py_library(
     name = "ragged_tensor",
     srcs = ["ragged_tensor.py"],
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index 3e802485a3..1b2a7be95f 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -156,6 +156,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.ops.ragged import ragged_operators
+from tensorflow.python.ops.ragged import ragged_string_ops
 
 from tensorflow.python.ops.ragged.ragged_array_ops import batch_gather
 from tensorflow.python.ops.ragged.ragged_array_ops import boolean_mask
diff --git a/tensorflow/python/ops/ragged/ragged_string_ops.py b/tensorflow/python/ops/ragged/ragged_string_ops.py
new file mode 100644
index 0000000000..cdcdbdff07
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_string_ops.py
@@ -0,0 +1,119 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ragged operations for working with string Tensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_string_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.util.tf_export import tf_export
+
+
+# pylint: disable=redefined-builtin
+@tf_export("strings.unicode_encode")
+def unicode_encode(input, output_encoding, errors="replace",
+                   replacement_char=65533, name=None):
+  r"""Encodes each sequence of Unicode code points in `input` into a string.
+
+  `result[i1...iN]` is the string formed by concatenating the Unicode
+  codepoints `input[1...iN, :]`, encoded using `output_encoding`.
+
+  Args:
+    input: An `N+1` dimensional potentially ragged integer tensor with
+        shape `[D1...DN, num_chars]`.
+    output_encoding: Unicode encoding that should be used to encode each
+      codepoint sequence.  Can be `"UTF-8"`, `"UTF-16-BE"`, or `"UTF-32-BE"`.
+    errors: Specifies the response when an invalid codepoint is encountered
+      (optional). One of:
+            * `'replace'`: Replace invalid codepoint with the
+              `replacement_char`. (default)
+            * `'ignore'`: Skip invalid codepoints.
+            * `'strict'`: Raise an exception for any invalid codepoint.
+    replacement_char: The replacement character codepoint to be used in place of
+      any invalid input when `errors='replace'`. Any valid unicode codepoint may
+      be used. The default value is the default unicode replacement character
+      which is 0xFFFD (U+65533).
+    name: A name for the operation (optional).
+
+  Returns:
+    A `N` dimensional `string` tensor with shape `[D1...DN]`.
+
+  #### Example:
+    ```python
+      >>> input = [[71, 246, 246, 100, 110, 105, 103, 104, 116], [128522]]
+      >>> unicode_encode(input, 'UTF8')
+      ['G\xc3\xb6\xc3\xb6dnight', '\xf0\x9f\x98\x8a']
+    ```
+  """
+  with ops.name_scope(name, "UnicodeEncode", [input]):
+    input_tensor = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(input)
+    if input_tensor.shape.ndims is None:
+      raise ValueError("Rank of input_tensor must be statically known.")
+    if ragged_tensor.is_ragged(input_tensor):
+      if input_tensor.inner_values.shape.ndims > 1:
+        # If the inner_values of our ragged tensor is multi-dimensional, we can
+        # process it separately and our output will have the same nested splits
+        # as our input.
+        return input_tensor.with_inner_values(
+            unicode_encode(input_tensor.inner_values, output_encoding, errors,
+                           replacement_char))
+      elif input_tensor.ragged_rank > 1:
+        # Recursively process the values of the ragged tensor.
+        return input_tensor.with_values(
+            unicode_encode(input_tensor.values, output_encoding, errors,
+                           replacement_char))
+      else:
+        # Our ragged tensor is of the correct shape (rank 1 inner_values tensor
+        # with ragged_rank of 1) so we can process it as normal.
+        return gen_string_ops.unicode_encode(
+            input_values=input_tensor.values,
+            input_splits=input_tensor.row_splits,
+            output_encoding=output_encoding,
+            errors=errors,
+            replacement_char=replacement_char)
+    else:
+      if input_tensor.shape.ndims == 2:
+        # The input tensor is of the correct 2-D shape, it's just not ragged.
+        return unicode_encode(ragged_conversion_ops.from_tensor(input_tensor),
+                              output_encoding, errors, replacement_char)
+      elif input_tensor.shape.ndims > 2:
+        # We need to initially flatten the input tensor to 2-D, and then can
+        # reshape the output of our processed flattened tensor.
+        flat_input_tensor = array_ops.reshape(
+            input_tensor,
+            array_ops.stack([-1, array_ops.shape(input_tensor)[-1]]))
+        flat_output_tensor = unicode_encode(flat_input_tensor, output_encoding,
+                                            errors, replacement_char)
+        return array_ops.reshape(flat_output_tensor, input_tensor.shape[:-1])
+      elif input_tensor.shape.ndims == 0:
+        raise ValueError("input_tensor's rank must be at least 1.")
+      else:
+        # Our input tensor is rank 1, so we create a ragged tensor with an added
+        # dimension to create the correct input shape & type, and then remove
+        # the additional dimension from the output and return the string scalar.
+        ragged_input_tensor = ragged_factory_ops.from_row_splits(
+            input_tensor,
+            array_ops.stack([0, array_ops.shape(input_tensor,
+                                                out_type=dtypes.int64)[0]]))
+        output_tensor = unicode_encode(ragged_input_tensor, output_encoding,
+                                       errors, replacement_char)
+        return array_ops.reshape(output_tensor, [])
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
index 03144cbe70..a1cd581a86 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
@@ -52,6 +52,10 @@ tf_module {
     name: "to_number"
     argspec: "args=[\'string_tensor\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\'], "
   }
+  member_method {
+    name: "unicode_encode"
+    argspec: "args=[\'input\', \'output_encoding\', \'errors\', \'replacement_char\', \'name\'], varargs=None, keywords=None, defaults=[\'replace\', \'65533\', \'None\'], "
+  }
   member_method {
     name: "unicode_script"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
index f2f4879fe8..f6e32ed08c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
@@ -52,6 +52,10 @@ tf_module {
     name: "to_number"
     argspec: "args=[\'input\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\'], "
   }
+  member_method {
+    name: "unicode_encode"
+    argspec: "args=[\'input\', \'output_encoding\', \'errors\', \'replacement_char\', \'name\'], varargs=None, keywords=None, defaults=[\'replace\', \'65533\', \'None\'], "
+  }
   member_method {
     name: "unicode_script"
     argspec: "args=[\'input\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 4dac3ae470e40153a4253dee9e83bb28aabba55c Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 29 Nov 2018 11:54:56 -0800
Subject: [PATCH 1039/1554] Add publicly available corpus for decode_bmp_fuzz.

PiperOrigin-RevId: 223385117
---
 tensorflow/core/kernels/fuzzing/BUILD            |   2 ++
 .../012e3ad384a4a1165f8498b5c94ba0d32a73e187     | Bin 0 -> 420 bytes
 .../055d77f7810048caa28323f6eb552a53d156040b     | Bin 0 -> 594 bytes
 .../131e251bfb82c681cb075d32b99f18fceaca115d     | Bin 0 -> 287 bytes
 .../1399ab0bd9f2c91d270cb43251bdc5729bef3526     | Bin 0 -> 1684 bytes
 .../16a6ce88f66d2e9686c8354cad8ba915cf0c11de     | Bin 0 -> 207 bytes
 .../185097ed0588195164619ea930ddd8274a5f32ad     | Bin 0 -> 3648 bytes
 .../27711a87e06a50c81571c27c3aa403a6ad5dc55c     | Bin 0 -> 2218 bytes
 .../decode_bmp/298c3787ad1722b22569cbc405c464d2  | Bin 0 -> 4234 bytes
 .../decode_bmp/2b95ba6d8141ce0d29ff279770903922  | Bin 0 -> 4234 bytes
 .../321fb3d758b86e37fc340ae2b09b8ed9fa73a4cb     | Bin 0 -> 4234 bytes
 .../331a98b4e4c87840efea69223766ebd0e1736542     | Bin 0 -> 420 bytes
 .../352d73f841223ecb630b5836585d2ba7b0f9d883     | Bin 0 -> 2249 bytes
 .../decode_bmp/3a84f409d4c117edfdebc508cd23e8fc  | Bin 0 -> 3194 bytes
 .../3ef5cc982c0b45f69a26fd0f7d376415fdebabd1     | Bin 0 -> 220 bytes
 .../401c7de8e122018a0e17f57c93db7ee49ab0e906     | Bin 0 -> 865 bytes
 .../52fee71bb8c9c79068e1fe580677ad739a2d0415     | Bin 0 -> 361 bytes
 .../57b11507813d5727b7789354d888eda83d5f3d86     | Bin 0 -> 2846 bytes
 .../57dff0fa53ee0ef24a43cca6ab0523bfdc1f720d     | Bin 0 -> 864 bytes
 .../5c42d3df0dc400a7a4175b8d4eec6cc8ee2437b2     | Bin 0 -> 2246 bytes
 .../5cca20637ae75fddad9370ee930837baef8aeb43     | Bin 0 -> 4234 bytes
 .../5d34bc9cef0c844b9c5ebe948145c4ca11b5ca09     | Bin 0 -> 2639 bytes
 .../5e162fe883bd12fb1c4131d4e0c979a12bd15eac     | Bin 0 -> 76 bytes
 .../5e83f8faab9c1a51a33d5e29edbb9dcec23c6092     | Bin 0 -> 1412 bytes
 .../decode_bmp/61b29dc2fcef7b6fbe3e0cc88769a7ef  | Bin 0 -> 3194 bytes
 .../6361eca190157ece389665ee523ccc3aefcd957f     | Bin 0 -> 3708 bytes
 .../65150515ab3b11d657519b22bb887d74e94b2d7f     | Bin 0 -> 4234 bytes
 .../656f38ef6dcd58c6a909d61db11f777def69c394     | Bin 0 -> 1406 bytes
 .../66e0d2cafd592bf9d61ad900fade8ee530d5f3d7     | Bin 0 -> 203 bytes
 .../6b5b42cb105a2c4c5fd6034e9885cbe457f1b50c     | Bin 0 -> 361 bytes
 .../722ed0197cb92ecbf9745edb38275e7a9aaf322f     | Bin 0 -> 454 bytes
 .../77bdd2efdf328366cbbf3c5688768dc0a88d02b1     | Bin 0 -> 16 bytes
 .../7841bfa002c05c61d5a5d9241f214cc17a336166     | Bin 0 -> 864 bytes
 .../decode_bmp/7899e22fc83f6be28e9130c4a1c91a48  | Bin 0 -> 4234 bytes
 .../7dddccaebd16ae0c26daeffc42df50f529891119     | Bin 0 -> 2249 bytes
 .../8157442eee4bbfdd9716e264b11085d61a9955b7     | Bin 0 -> 48 bytes
 .../81ff28ed63d5435ddc4c8771dd5d40aa658cbbe0     | Bin 0 -> 2052 bytes
 .../820c8c0d33c18f6c4d9edd314e91289186931ad0     | Bin 0 -> 1318 bytes
 .../849e9d7cee1c52105242327086997296e452b981     | Bin 0 -> 1237 bytes
 .../84ddb92c63e0fad7018f6069daf8779ce11501e2     | Bin 0 -> 1167 bytes
 .../86bc3d5dbb9313137502080e58551edd2e649c70     | Bin 0 -> 929 bytes
 .../87d94d88fe29d277c76e1a52042b02c092d5ae14     | Bin 0 -> 420 bytes
 .../8c4646f3357945c4e19a59ff79fffe3c874dbf16     | Bin 0 -> 189 bytes
 .../90632bc6dee4eb836f3d7db1d16446a9c8510080     | Bin 0 -> 3895 bytes
 .../94d06016aa949e8e7203217e4cc6625ded7f4244     | Bin 0 -> 3102 bytes
 .../9875819b9e5783e7489c29a81cc9d4279209956a     | Bin 0 -> 22 bytes
 .../9c1cc734114b29aac6c51782d5c17e9dbe1faca2     | Bin 0 -> 3194 bytes
 .../decode_bmp/9d2961871eeb201ef8a6f5503d8a8b62  | Bin 0 -> 3194 bytes
 .../9f39e11cdd88344a4894b678e5a04a810880064d     | Bin 0 -> 972 bytes
 .../a350588a6dabe4376a066aed44ef8786d8e752e7     | Bin 0 -> 861 bytes
 .../a6101a79919d444e1fc50aefab5837c39e3f4a19     | Bin 0 -> 1237 bytes
 .../a9c8793f8fb063bec839ee1280406fe5396545e5     | Bin 0 -> 420 bytes
 .../ad4e9d2234e8599bdf12607c6b8cab4edae82c4e     | Bin 0 -> 420 bytes
 .../decode_bmp/b90b6830917919e94186d312f06481bd  | Bin 0 -> 3194 bytes
 .../b98fd4cb1d7031240414301c19b03097c0035c6b     | Bin 0 -> 857 bytes
 .../ba976fcdb4daf092ef17ce43bf2b78d9d8bc2aeb     | Bin 0 -> 2262 bytes
 .../bc112b571eafee0f5a031f3c9cce6244216d128d     | Bin 0 -> 4234 bytes
 .../decode_bmp/c42b981c28a1715c375050f6fcf53f1d  | Bin 0 -> 3194 bytes
 .../c6049874b33eadb016fccf0c5fa66e556ae069b9     | Bin 0 -> 3194 bytes
 .../c8697bf2369f6ab85f501376c4d93bb8a56974a3     | Bin 0 -> 420 bytes
 .../c8daf283e0aef2fd7b630c0430e05dc28f24ecf6     | Bin 0 -> 220 bytes
 .../cacff56e1af4b8fde912822da06b10fb8c545a19     | Bin 0 -> 1033 bytes
 .../decode_bmp/ce4dcc22b1d595c49a25121c0b580104  | Bin 0 -> 4234 bytes
 .../d0cd71dbf039fd64cf42eff30da92a71a919226a     | Bin 0 -> 587 bytes
 .../d5ce626ac3264bed6af5580e341a89406857cbb9     | Bin 0 -> 1760 bytes
 .../d77ada02e9bc8c24b2711eca6a8f52ae356bfc21     | Bin 0 -> 361 bytes
 .../d7eb9c5a0f9803df4c00390793b8ab57bd7c9484     | Bin 0 -> 3792 bytes
 .../dc1efccdeec17e151a1ec8228c09ab61c3040b33     | Bin 0 -> 35 bytes
 .../decode_bmp/dcea22c66c60088165a2f1772036473f  | Bin 0 -> 4234 bytes
 .../de539ae7442fa05dafcfe1a021f0186ef74a2b0e     | Bin 0 -> 103 bytes
 .../e2306b1d6b88d0ccc4e2c3a9edb07462a5a32215     | Bin 0 -> 110 bytes
 .../e2778da0240fdd15ef5844905d81c4e05f34a8bd     | Bin 0 -> 3194 bytes
 .../e6642e9266875f9d908942e534bf898103a2c794     | Bin 0 -> 2186 bytes
 .../ec6cdb929c08d8daf2bd7fc185fbf4d787b45120     | Bin 0 -> 1942 bytes
 .../ed8636357f79439b6a03eb14469b686cc401a1c9     | Bin 0 -> 3895 bytes
 .../ee313e9acecb5c688ce8c9bb10e70e136fbb9c6d     | Bin 0 -> 3616 bytes
 .../ef689af320e7d9e22231109faae2e8149cb86e1c     | Bin 0 -> 834 bytes
 .../fda6b9a9f6ffdf4765c00465619c7ceb3f7db2e4     | Bin 0 -> 1136 bytes
 .../ffe829bb0adac20d9c0756f68a22d1255e4fdb54     | Bin 0 -> 3194 bytes
 79 files changed, 2 insertions(+)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/012e3ad384a4a1165f8498b5c94ba0d32a73e187
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/055d77f7810048caa28323f6eb552a53d156040b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/131e251bfb82c681cb075d32b99f18fceaca115d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/1399ab0bd9f2c91d270cb43251bdc5729bef3526
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/16a6ce88f66d2e9686c8354cad8ba915cf0c11de
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/185097ed0588195164619ea930ddd8274a5f32ad
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/27711a87e06a50c81571c27c3aa403a6ad5dc55c
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/298c3787ad1722b22569cbc405c464d2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/2b95ba6d8141ce0d29ff279770903922
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/321fb3d758b86e37fc340ae2b09b8ed9fa73a4cb
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/331a98b4e4c87840efea69223766ebd0e1736542
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/352d73f841223ecb630b5836585d2ba7b0f9d883
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3a84f409d4c117edfdebc508cd23e8fc
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3ef5cc982c0b45f69a26fd0f7d376415fdebabd1
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/401c7de8e122018a0e17f57c93db7ee49ab0e906
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/52fee71bb8c9c79068e1fe580677ad739a2d0415
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57b11507813d5727b7789354d888eda83d5f3d86
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57dff0fa53ee0ef24a43cca6ab0523bfdc1f720d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5c42d3df0dc400a7a4175b8d4eec6cc8ee2437b2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5cca20637ae75fddad9370ee930837baef8aeb43
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5d34bc9cef0c844b9c5ebe948145c4ca11b5ca09
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e162fe883bd12fb1c4131d4e0c979a12bd15eac
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e83f8faab9c1a51a33d5e29edbb9dcec23c6092
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/61b29dc2fcef7b6fbe3e0cc88769a7ef
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6361eca190157ece389665ee523ccc3aefcd957f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/65150515ab3b11d657519b22bb887d74e94b2d7f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/656f38ef6dcd58c6a909d61db11f777def69c394
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/66e0d2cafd592bf9d61ad900fade8ee530d5f3d7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6b5b42cb105a2c4c5fd6034e9885cbe457f1b50c
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/722ed0197cb92ecbf9745edb38275e7a9aaf322f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/77bdd2efdf328366cbbf3c5688768dc0a88d02b1
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7841bfa002c05c61d5a5d9241f214cc17a336166
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7899e22fc83f6be28e9130c4a1c91a48
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7dddccaebd16ae0c26daeffc42df50f529891119
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8157442eee4bbfdd9716e264b11085d61a9955b7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/81ff28ed63d5435ddc4c8771dd5d40aa658cbbe0
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/820c8c0d33c18f6c4d9edd314e91289186931ad0
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/849e9d7cee1c52105242327086997296e452b981
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/84ddb92c63e0fad7018f6069daf8779ce11501e2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/86bc3d5dbb9313137502080e58551edd2e649c70
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/87d94d88fe29d277c76e1a52042b02c092d5ae14
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8c4646f3357945c4e19a59ff79fffe3c874dbf16
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/90632bc6dee4eb836f3d7db1d16446a9c8510080
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/94d06016aa949e8e7203217e4cc6625ded7f4244
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9875819b9e5783e7489c29a81cc9d4279209956a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9c1cc734114b29aac6c51782d5c17e9dbe1faca2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9d2961871eeb201ef8a6f5503d8a8b62
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9f39e11cdd88344a4894b678e5a04a810880064d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a350588a6dabe4376a066aed44ef8786d8e752e7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a6101a79919d444e1fc50aefab5837c39e3f4a19
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a9c8793f8fb063bec839ee1280406fe5396545e5
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ad4e9d2234e8599bdf12607c6b8cab4edae82c4e
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b90b6830917919e94186d312f06481bd
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b98fd4cb1d7031240414301c19b03097c0035c6b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ba976fcdb4daf092ef17ce43bf2b78d9d8bc2aeb
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/bc112b571eafee0f5a031f3c9cce6244216d128d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c42b981c28a1715c375050f6fcf53f1d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c6049874b33eadb016fccf0c5fa66e556ae069b9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8697bf2369f6ab85f501376c4d93bb8a56974a3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8daf283e0aef2fd7b630c0430e05dc28f24ecf6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/cacff56e1af4b8fde912822da06b10fb8c545a19
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ce4dcc22b1d595c49a25121c0b580104
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d0cd71dbf039fd64cf42eff30da92a71a919226a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d5ce626ac3264bed6af5580e341a89406857cbb9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d77ada02e9bc8c24b2711eca6a8f52ae356bfc21
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d7eb9c5a0f9803df4c00390793b8ab57bd7c9484
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dc1efccdeec17e151a1ec8228c09ab61c3040b33
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dcea22c66c60088165a2f1772036473f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/de539ae7442fa05dafcfe1a021f0186ef74a2b0e
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2306b1d6b88d0ccc4e2c3a9edb07462a5a32215
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2778da0240fdd15ef5844905d81c4e05f34a8bd
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e6642e9266875f9d908942e534bf898103a2c794
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ec6cdb929c08d8daf2bd7fc185fbf4d787b45120
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ed8636357f79439b6a03eb14469b686cc401a1c9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ee313e9acecb5c688ce8c9bb10e70e136fbb9c6d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ef689af320e7d9e22231109faae2e8149cb86e1c
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/fda6b9a9f6ffdf4765c00465619c7ceb3f7db2e4
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ffe829bb0adac20d9c0756f68a22d1255e4fdb54

diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index 7d59beb09b..3ea9dd0638 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -42,6 +42,8 @@ tf_ops_fuzz_target_lib("encode_jpeg")
 
 tf_ops_fuzz_target_lib("decode_bmp")
 
+tf_oss_fuzz_corpus("decode_bmp")
+
 tf_ops_fuzz_target_lib("decode_png")
 
 tf_ops_fuzz_target_lib("decode_wav")
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/012e3ad384a4a1165f8498b5c94ba0d32a73e187 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/012e3ad384a4a1165f8498b5c94ba0d32a73e187
new file mode 100644
index 0000000000000000000000000000000000000000..7a1b8966c21c74f1e0a3f3af4240551bc10ef36c
GIT binary patch
literal 420
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~uS=>LC)|Ns9pP-+as
kP$&1GVh9VF{Ey%`Aeb-~6N~|J1}+Tp3&b-p)kBE}09IruQUCw|

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/055d77f7810048caa28323f6eb552a53d156040b b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/055d77f7810048caa28323f6eb552a53d156040b
new file mode 100644
index 0000000000000000000000000000000000000000..24f658497f182188a6cbf431e1ff810ca268e709
GIT binary patch
literal 594
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
h#sB~RXBbdSwXmSCM~HIvf6C(r8ejjRG@L>c0084c$3*}D

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/131e251bfb82c681cb075d32b99f18fceaca115d b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/131e251bfb82c681cb075d32b99f18fceaca115d
new file mode 100644
index 0000000000000000000000000000000000000000..a2d8f84cab77d805ae2582fb551fe2f074b2cfbb
GIT binary patch
literal 287
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)FoPjT0F3`LFhCFk0|OfaBg21&
p|Nqf3h7c=;5+p4csxb|OtA>~ZR{+uq=Ro8@Lbxzdo(7r6001>fj5z=R

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/1399ab0bd9f2c91d270cb43251bdc5729bef3526 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/1399ab0bd9f2c91d270cb43251bdc5729bef3526
new file mode 100644
index 0000000000000000000000000000000000000000..6206dab82b1e4a3d96ee4ae276006183349aa8e6
GIT binary patch
literal 1684
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=QG0cn@L{
zjhWC@{{R1<0fM0vjQd{!A`IuF%cJrc85l`Y54V;`^YN)ibqPL|RIm)M@wm+=!Hu{L
z8L$v0)6?X*7dfCv)z3zTJGsF*iVTgIj{D9)6=R5v|NY28h0LBkn_<F);L@ZC6BuUC
zp3Sgk%^HRS2M#b?xpIZ!$&)AOB><Ux4!AxA1qHCV*RNk^ICbh2!@hm{7<TN~0mdLU
tNDL$gvKyopu93=&n>TM#*+d2gkeLr2JOI<zu3cjQVPr9?TR30_0suS_9Nqu`

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/16a6ce88f66d2e9686c8354cad8ba915cf0c11de b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/16a6ce88f66d2e9686c8354cad8ba915cf0c11de
new file mode 100644
index 0000000000000000000000000000000000000000..bcc7f481ae90d0075912bf80469a18a0fd27682e
GIT binary patch
literal 207
zcmZ?r?Gj)BgDwUJh8hM21_cOCsmx2v%}kCj$<NQxD=tY)&W<n6tV)f~1uI4)n9%tD
z8Jv8({{R2a@E@+NG^d1tfssLhff=M$fPq1n8-hVxDE`mD06`$WlRJnc1qT%~004bM
B9CiQz

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/185097ed0588195164619ea930ddd8274a5f32ad b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/185097ed0588195164619ea930ddd8274a5f32ad
new file mode 100644
index 0000000000000000000000000000000000000000..92bddb6dca98bcc26ab9257bf568a462b5db8a36
GIT binary patch
literal 3648
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)H3=9lx
zP?{jcgs$TM|Nji57*`m4A6#Jra?KErFa46l;%qBi;XNuobn@a2P-+|$*q-!2`!gsU
zPP)6PmO&KBwY!9j0_Z*&ddPL(kkiLNwK#>R>AL^_88{gj7>8U~;7KP8EYuID;S@fj
iei=<41k(WXPz$4P@G^~{-|&<jgHSKY5drcZEHD5o={mCj

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/27711a87e06a50c81571c27c3aa403a6ad5dc55c b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/27711a87e06a50c81571c27c3aa403a6ad5dc55c
new file mode 100644
index 0000000000000000000000000000000000000000..082b1e5752a95bed0941c60f68e9efb83dcea73e
GIT binary patch
literal 2218
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lje
z6c1Ll{~>^Tu(}WCE@&LXxOnOR4FCU+V(NtfuE2K0B|&>}c=UoIntIVqJ(EE0qCaND
ZAL;Lj@*jz@$H+*chEe6CAutj{003P%43hu=

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/298c3787ad1722b22569cbc405c464d2 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/298c3787ad1722b22569cbc405c464d2
new file mode 100644
index 0000000000000000000000000000000000000000..af1091428d59645a8158218a953fbd60e0c463d8
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lje
u6pyMM4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVCGApih??guRZ

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/2b95ba6d8141ce0d29ff279770903922 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/2b95ba6d8141ce0d29ff279770903922
new file mode 100644
index 0000000000000000000000000000000000000000..fd711cb0e51bb10d84902b03d3a27c086ec8dfbc
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXBfq!VKABoM$^D(8W>FjqiJ9?4UDFN(KIlc21e7sXc`zz1EXnRSf>F18>suz

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/321fb3d758b86e37fc340ae2b09b8ed9fa73a4cb b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/321fb3d758b86e37fc340ae2b09b8ed9fa73a4cb
new file mode 100644
index 0000000000000000000000000000000000000000..6748826bd88a8ff6616536e0badb9b5d5184d9cb
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXBa`4sE`MR2YNV-@<&5pGz5lt2#l<<ZX}iKgWj)0Jm;W0KR(z$G{_lUbd*0D
h0;3@?8UlkL1cD2T5UyZ^FhwBLC^;GeqagqZ0RVbHosj?l

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/331a98b4e4c87840efea69223766ebd0e1736542 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/331a98b4e4c87840efea69223766ebd0e1736542
new file mode 100644
index 0000000000000000000000000000000000000000..9cf1b9d3af82244e2f1f5f4e9d696fa006a9416f
GIT binary patch
literal 420
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)FoPjT0F3`LFhCH9=j0w#45CTF
eOlXS#GyMNgf6R){PEuS1F^2>%5Eb$u^B4e#p1@B4

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/352d73f841223ecb630b5836585d2ba7b0f9d883 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/352d73f841223ecb630b5836585d2ba7b0f9d883
new file mode 100644
index 0000000000000000000000000000000000000000..932e78b3547e36ee7892f97135b7ac16d03bdff8
GIT binary patch
literal 2249
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)H3=9lx
zP#Tr`|Ns9VR1pRyG(N-s|NloZ&M^2sxWWeH8k}y0$0M8pQjUgipz-mkrLCMj=_&6I
u>5kt^TThVghf(*8hS_Ku_|H7l!r&X}WzT<bN+(AjJ@t?r5ugeP=0E^`5i=bC

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3a84f409d4c117edfdebc508cd23e8fc b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3a84f409d4c117edfdebc508cd23e8fc
new file mode 100644
index 0000000000000000000000000000000000000000..89a090d74ee4af9f5352f3e954e41db64727bfcc
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#LR0vE6pV(z
eXb6mkz-S1JhQMeDjE2By2#kinXb6m$5C8zfulSMx

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3ef5cc982c0b45f69a26fd0f7d376415fdebabd1 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/3ef5cc982c0b45f69a26fd0f7d376415fdebabd1
new file mode 100644
index 0000000000000000000000000000000000000000..286949bc56a4fd135b6d645c2f6f6543fb54cefe
GIT binary patch
literal 220
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lje
V6vNg2|Nox>jNu{-KM+h*762#p4;}yj

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/401c7de8e122018a0e17f57c93db7ee49ab0e906 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/401c7de8e122018a0e17f57c93db7ee49ab0e906
new file mode 100644
index 0000000000000000000000000000000000000000..8d5c7d136e51b0b71a574de271c1d076ad118ec7
GIT binary patch
literal 865
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo?^*kifve@SoxTf5I56R*+_f
z|6meio|Ah}F^Fa$$0=Zq5VQV+*$i;wAY7tICUjNA_-#0<K@SHw{~MeIW8et0-$bPg
ZykS>QI9=h@Pei(;l74iX(ESdQ2LPU}wmtv=

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/52fee71bb8c9c79068e1fe580677ad739a2d0415 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/52fee71bb8c9c79068e1fe580677ad739a2d0415
new file mode 100644
index 0000000000000000000000000000000000000000..f77ffec08653f23af6ec36d2bc3851d37a15a7ef
GIT binary patch
literal 361
qcmZ?rt>R$-gDM6Fh8!@9HUtx1V;S+vVoLI33Jq2fC*Ljs1_l7r3<Ea+

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57b11507813d5727b7789354d888eda83d5f3d86 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57b11507813d5727b7789354d888eda83d5f3d86
new file mode 100644
index 0000000000000000000000000000000000000000..f9af0697d53b40b1617eff3dba43955053de8bff
GIT binary patch
literal 2846
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~uS=>LC)|Ns9pP-+as
zP$&1GVh9Td>3}KD1QP-|gBZ*J*8$b}p9TPGG;JwP+8RF~rV|r`!$r;TNr?lJRECqK
s!zD?R<&WVMkyP}HH5JVw(kQZ`584D6x_XHW914^T>!_q4Tn4*V04VKEwg3PC

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57dff0fa53ee0ef24a43cca6ab0523bfdc1f720d b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/57dff0fa53ee0ef24a43cca6ab0523bfdc1f720d
new file mode 100644
index 0000000000000000000000000000000000000000..109ab7948ff737251655b35d801b7a147107da71
GIT binary patch
literal 864
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
j#sB~RXCMWm>&ML}#nd6Gb_mDfkP2^7(&v8$P;v$UzN)0q

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5c42d3df0dc400a7a4175b8d4eec6cc8ee2437b2 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5c42d3df0dc400a7a4175b8d4eec6cc8ee2437b2
new file mode 100644
index 0000000000000000000000000000000000000000..bf9772902653f1b4bf258c4268937e18f9abbcf8
GIT binary patch
literal 2246
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXBfq!VKABoM$-W4X#iAojD{QO;YQyuBfh%%Kga<GG49hKMiD7$(Cf@mJ~#vb
DBB6uo

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5cca20637ae75fddad9370ee930837baef8aeb43 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5cca20637ae75fddad9370ee930837baef8aeb43
new file mode 100644
index 0000000000000000000000000000000000000000..cf7a78e9488a8fd55813862e78a360064dff19b7
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXBfq!VKB(jz_mf<p3yuqT273nf&c#*N=DPcXgU~81EXnRGz|>;H1K^ijs|@k
SQMa9Ye}wsWwBJTy`T+ozM*H;u

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5d34bc9cef0c844b9c5ebe948145c4ca11b5ca09 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5d34bc9cef0c844b9c5ebe948145c4ca11b5ca09
new file mode 100644
index 0000000000000000000000000000000000000000..e5621aa3d1ba240abb65a521507762d520d1d4bf
GIT binary patch
literal 2639
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)H3=9lx
zAo~Bn0g&~CFcZ3h|Ns9pjAC42@O?CFNX`N1DGVDQkqn_MIwnC}aZPt|x?6~rnE#;h
z2iiSUx1Z7y9$GYF7^48LDHs?S7#TE(s41{z02+u9$U=$)<Z&j35@;}xqW?Y_dPp&U
zu&BizjsLL<6LvGsC?Kqr>UoEtTd?{O)G)##F__%X$-uyf;vSfL@g!LW2GrmHYk{c;
JNizIr002{=H%$Nl

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e162fe883bd12fb1c4131d4e0c979a12bd15eac b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e162fe883bd12fb1c4131d4e0c979a12bd15eac
new file mode 100644
index 0000000000000000000000000000000000000000..eea39d6b2f856ab8f1debf18d4d99543076a1b9a
GIT binary patch
literal 76
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVEgVBle3U=U{LVn9{{=YZ5GFbF6x000~|
B2EhOT

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e83f8faab9c1a51a33d5e29edbb9dcec23c6092 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/5e83f8faab9c1a51a33d5e29edbb9dcec23c6092
new file mode 100644
index 0000000000000000000000000000000000000000..fabcbdbe3d4b0e2b8f1de3bf0c7a388369d11547
GIT binary patch
literal 1412
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
v#sB~RXBa`4sE`MR2YNUS5`Od`aR*^%fKoMKRfAT=|3RDx25q1XC5Hh3<uJtT

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/61b29dc2fcef7b6fbe3e0cc88769a7ef b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/61b29dc2fcef7b6fbe3e0cc88769a7ef
new file mode 100644
index 0000000000000000000000000000000000000000..8dfc17e8e05f948210d4ed34113b3517445c418e
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#LR0vE6pV%d
zT|(d=5}=}m|Ni~^`}gnf-@kwU{P_b6e*XOV`}gm^fB*jb_m4u0{{H>@>({TZU%!6*
z`0?%Ax39tA?c28>KYslB_3N)+zyALHOSTn%|Ni~)<HyI3A78$F`QX8Wn>TM>1B074
zZ$5bN;N{DgA3uKl@#6=1RzM7Y|Ni}xCr_?lzkce}seSwQ?Er&)`}UnWb?W-{>rb9M
zdH?=B8CLxJ_wU!QUmrhyeDdVUl`B^c95}FM&6?S>XHNiw*|TS_S+nNAfdf~rTzT^3
z$;Xc$fBpJJQe6D~`}gbDuP<M|yng-qfddDYE?wHu(NS1fm<9%gg@qj*9ZQ!kJ#gT_
z_3PJPzI^%h>sOLO>G$v7A3uJ4@ZiCzQ>WIfS<}(ck(`|D?d@$32HxJ@$;rtb9UW`d
ztT}b+)Pn~PK7Rc8`}c1mBl_RJe?NczeEat8&6_v(?b|nd_UyvKLT_(x0|NsEFfcGM
z@b>mDEG(Qod-lG4`)=O6`S$JGpFe*R8ASj7{rmCb$LrUxuU)&gW5<pO6DFjkrP<rt
zD<~-Nfq{a8g1x<cT3XtK2@`hg*m3RJwb!p-|M>A^@R(22@cIA$KTYHJ|NsBAPM@^S
zUl1?+`0;~A<pU(AK#Cw5m2Yq>XjeW%te|QA0<i)@B2|x+R$CAQN6FC;AQ%Dw7?UPG

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6361eca190157ece389665ee523ccc3aefcd957f b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6361eca190157ece389665ee523ccc3aefcd957f
new file mode 100644
index 0000000000000000000000000000000000000000..141e331ad0148af9b17e2824d8b64a442ac18b38
GIT binary patch
literal 3708
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXZZjBHv@zQu}AS}SPX?UpfY6BEF%LW+h`giD~<gg4L7pFjcR$N1YBPXtQ$tt
rz-ai7rUCRcph~qoPF&d5)7d_B|Bmue;;9NVJiZU*vU)Ui(I)l)DzI$>

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/65150515ab3b11d657519b22bb887d74e94b2d7f b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/65150515ab3b11d657519b22bb887d74e94b2d7f
new file mode 100644
index 0000000000000000000000000000000000000000..567c645c00ee0e47ca2a840c7a115f68ae889e3d
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lje
z6jP{{gF^kI+MyvZ4@!?xqaiR@LLfdqo?-6XxnK-p4;FU~ggZv_=Rkz|sGayi0F);O
zB7fezd6VJ6g9i*C`r5T?3?K|*!^H5pnG!h#1qFsDPo6MbzkZ$J)TvVp`}XZ)*s)^=
z7=zd#F_0Wc4M;5|c45;CGWW`rD+~t?9AH?pW(~va*|QlYOqc-1AT~%0BnMIhQj5(D
za>YR53-TMt{H05mGIVrwFccORGNh%YfiZ{;5(CMB)PU52^pI;Gx;{|cg2ET%N02?q
z$;k}f-rfxM_V!>5VuQp$av(JzwIDs{=8(e&rE`#9KyeHTXOLe(_L&))GJxV4gbfS~
z!0rLbfz*K1g7kp&l4B>b9#9?wrEySN1I0He%t8JJ*~`br#{j}0Hb@L42T}u43(^D9
zi)<D-Y>*isvp{Bo%m&#3vI}G<$Zik@u|Z-WIglEVT96))UUKY2)&mMdP*{S(6co0g
zFb0J`D9l0jf-r~;5(CMB)PU52^nmpGBO6908<dtnX$q9KKxqt=)<E`w{0zb%Hb@L4
z2T}u43(`ZTbr{+}egWkvP~HONF_1kVzk<RWgh6bO7)TDJ2Ba3G2g4w&0*nw*5eS8g
z1eGbEvIS&5$d90K2E{c9gV-Q3kQ_)2NG&dN$Q1|0Gss+!-^ev%*y&>#^gaRr>cBJ;

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/656f38ef6dcd58c6a909d61db11f777def69c394 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/656f38ef6dcd58c6a909d61db11f777def69c394
new file mode 100644
index 0000000000000000000000000000000000000000..e1cdb4e5bf9d55e87b4495ab8db931f269e22d61
GIT binary patch
literal 1406
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hgs~F?
z4|VNhBWx!`1rvlqkpKVxXTZRJ{xg6@!E*aC<-y`$74(rqIF~s0W4e!#fq|VqZY3pH
zKrSLhJwCPHL55L?LE7=zG05d`=1`0Tj+sC}UMCteLJ}lDwaum6Je;Y3XdnN9g#zV{
IAu_oD0K3URi2wiq

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/66e0d2cafd592bf9d61ad900fade8ee530d5f3d7 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/66e0d2cafd592bf9d61ad900fade8ee530d5f3d7
new file mode 100644
index 0000000000000000000000000000000000000000..73e53b460ec81a8accdf2b6601dfcd8679465b6a
GIT binary patch
literal 203
zcmZ?r?Gj)B11AOsh8hM21_dZ)WKdvW2FVC8FbH#l82GW1uR|3NL`f9`149nf7?5#{
U3=;nt{{JTdGmxMbVy;s>07X?zX8-^I

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6b5b42cb105a2c4c5fd6034e9885cbe457f1b50c b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/6b5b42cb105a2c4c5fd6034e9885cbe457f1b50c
new file mode 100644
index 0000000000000000000000000000000000000000..f29b9b217184c4cc8496f1f711caec0c7c632ca4
GIT binary patch
literal 361
vcmZ?rt>R$-gDM6Fh8!@9HUtx1V;S+vVoLI33gHl8!68mJVJF`%0R{#D)hGiy

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/722ed0197cb92ecbf9745edb38275e7a9aaf322f b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/722ed0197cb92ecbf9745edb38275e7a9aaf322f
new file mode 100644
index 0000000000000000000000000000000000000000..3b0c338ce203a0fd72936ae40120f140aae712b4
GIT binary patch
literal 454
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRMU5OYkr-2y=s33=9m+u?!3hT?`BiHBe<B<%|pp
z49pA+47vgkr3mF<ZT}e<APB^Bat|s7(fF_vgKyV=hX4OD@E<H<*whg#W<^jTK4%Ud
FIRJ$yZwLSY

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/77bdd2efdf328366cbbf3c5688768dc0a88d02b1 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/77bdd2efdf328366cbbf3c5688768dc0a88d02b1
new file mode 100644
index 0000000000000000000000000000000000000000..61dd2583cd6e9c7679c2c30ff6d35df09264fd91
GIT binary patch
literal 16
XcmZ?r?Gj-4&+z~Ee+C8yh8hL{G{gm-

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7841bfa002c05c61d5a5d9241f214cc17a336166 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7841bfa002c05c61d5a5d9241f214cc17a336166
new file mode 100644
index 0000000000000000000000000000000000000000..907ec3b5a3ba35bf86b8988ad6d25592e9015c17
GIT binary patch
literal 864
zcmZ?r?Gj)BgDwUJh8hM21_cOKU|?imU<SzuFfa&nLokR7#s3)?APB^Fat|s7(WGD|
zbjAPw|7RctqwB}bC&g5lTGt|6=0P0}Q%ED4w6K6X0%v?OFfu66CO-dv2Zt1Wf`z0M
W0uL3;l=UBv9C>!2>-x_Cib4Qhx|fdt

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7899e22fc83f6be28e9130c4a1c91a48 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7899e22fc83f6be28e9130c4a1c91a48
new file mode 100644
index 0000000000000000000000000000000000000000..7e3b1990ad7c09a19fcef334aee9545a31d05380
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXBfq!VKABoNJ|6%{{3S>#`Fk3kbQsu{$=?6`!~bSpFbIX{P+RJAT~%0BnMJM
z3%7vG{`Kn@!`H8089sje$nf^<TZY%KUxP7-4H5&%fz*K1Qqv6}^FV$D+41t_ONIvz
z9x&Xzd6VJVwQFDuVuQp$av(JzwIDsza0AF3ka_Rlzh`*z<O##|>(?1hojS#^Z{I$K
z9Xob_F^CNk1IdBZfYgHYfb>%022fmr!V}~tkhxc`TwyqH-~hv#HES4V&z{XNVZsD3
z2C+e6AUTj4kXn!)kY11(<hlXmeo&l(!WHB<koilOE@kNG=wK);EM!PaO9NvN8zcsj
z1E~S21?d6l1(`vv{h%}r@&_nxLE#JXBgmfQ<YWeKZ*K;BdwVbju|Z-WIglEVT96))
zUXU5&*bfRfP`(1CZBY6G#W5(HL4F0<XJBB!prD`t#vnFG3?v6q15yjp1JVmJ17sGd
zZUE^4r6Eur1Eq6NS_8#5D9l0r2HDNW$HxG|AT~%0BnMIhQVY@p(hD*JWY$o$9~5?=
zFa(7qC`>_N3kqXUScAeG<ZloLu|Z-WIglEVT96))UXU5&_#YHrptJ)@L!h(-N>iY;
z1xjO}v<9*d6y_idVuQp$av(JzwIDqpy&y9{W|4zIc?Xn-KzRw2r$BiNl*d5!fcy%I
zZx9BtL1G{|kQ$I$kRFg;kQwCI4blS&H&7V@@&~9)0hKKv^Fe+Dg)=CvfiQ>-5(CMB
z)PU52^nmn&%mA51E(W<Dl!ifND#%=r-$3CDiepeZ2VoE!BnFZLsR5}4=>h2_*Is0O
zAag+Gf&2stS5Vx7(ibR?fiQ>-5(CMB)PU52^dOr-DH~)SC_F)N3bF%~wn6zDgh6bO
z7)TDJ2Bel!`>^Q;#U;pWP?`qiE08}y7{msNf#g7Hu<50?7|0%EOl|W=%_FTnJQ`j@
GGQ0p83n_j8

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7dddccaebd16ae0c26daeffc42df50f529891119 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/7dddccaebd16ae0c26daeffc42df50f529891119
new file mode 100644
index 0000000000000000000000000000000000000000..0329a2826a89961de414d0e3c51a7c81ea02d8ef
GIT binary patch
literal 2249
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)H3=9lx
zP#Tr`|Ns9VR1pRyG(N-s|NloZ&M^2sxWWeH8k}y0$0M8pQjUgipz-mkrLCMj=_&8e
qpm99uZl`Tvj=E<we1}vTU><7m^^Kge2UJ#*qmQ0?NR9|(PXPeno-<zn

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8157442eee4bbfdd9716e264b11085d61a9955b7 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8157442eee4bbfdd9716e264b11085d61a9955b7
new file mode 100644
index 0000000000000000000000000000000000000000..7e9ef4b3dd47fbbdf0d4087a4d4fdf5323c3bf23
GIT binary patch
literal 48
mcmZ?rt>R$-gDM6Fh8zY41_dZ)WH68bOEB;-FbH!)FarQ#>jKRH

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/81ff28ed63d5435ddc4c8771dd5d40aa658cbbe0 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/81ff28ed63d5435ddc4c8771dd5d40aa658cbbe0
new file mode 100644
index 0000000000000000000000000000000000000000..6390e6b2b30b9e4512b191b72f0bed8d4005c0cd
GIT binary patch
literal 2052
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvm@&!u~BACz>{Qv)-flMq$rap4*BS#-E1FHYf
z1Ca_|aPpmkCtjFA0WJV`I+|0!VpOo7TIN900kw=9H3v22jfTf)c#NijA(#e01^8gW
E0O*=DI{*Lx

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/820c8c0d33c18f6c4d9edd314e91289186931ad0 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/820c8c0d33c18f6c4d9edd314e91289186931ad0
new file mode 100644
index 0000000000000000000000000000000000000000..0084212a656bd4f97a2ba22f4d7ced7f8746946f
GIT binary patch
literal 1318
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)H3=9lx
zP#Tr`|Ns9VR1pRyG(N-s|NlwBBq=7vU>d2tK_hdC3J21Hb|76kyzFKsDs~2upT7+v
Qw@@|gNHz?VWnh5;07lz<w*UYD

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/849e9d7cee1c52105242327086997296e452b981 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/849e9d7cee1c52105242327086997296e452b981
new file mode 100644
index 0000000000000000000000000000000000000000..a36c88daf011cf7809049396476863fd66d5dc2d
GIT binary patch
literal 1237
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo?^*kN~FsGcZ69i09-UR1BhV
z<8K&JOz0y2|Nm$B|Nl1wga)yxhlhnHPy$AqP~d>V5gei*#z10w6nBH2^dE~54V|b-
zg4xQ@aIJ#U5ch%&_>RPecmOO6VPYzWir>ZqPUDdyQECH3Jw6wGgSrTxnxP|Chcm<h
DO=tdb

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/84ddb92c63e0fad7018f6069daf8779ce11501e2 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/84ddb92c63e0fad7018f6069daf8779ce11501e2
new file mode 100644
index 0000000000000000000000000000000000000000..b5d34609b88399ffe7198acfb06dac91fd50d4cd
GIT binary patch
literal 1167
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXCMJXosDh^Ha-ca;&A~;3hYD>V{qZ28;it+<8qQx261MBqmpXoQq7=&aQtAf
J^FIS92muIShqC|x

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/86bc3d5dbb9313137502080e58551edd2e649c70 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/86bc3d5dbb9313137502080e58551edd2e649c70
new file mode 100644
index 0000000000000000000000000000000000000000..a9ef2b5a50b36814ecaada8813a5f4b056f17ce8
GIT binary patch
literal 929
zcmZ?r?Gj*MU|?ck=we`Cs9|7WP=I1a1_cIY1_lNO0R{$PZU%N?ZV(5K|1&T^5J<$y
zJ*XH&GeC`FAZQ%ONU#nl0Ws@8ga<bc!X=7iLRa<w|9^)6|9>+;Xb>BW|3ml;=;Cm`
z6G#OF!^J>cLIFgCk04fqoq=pLP1raLMGqxpz7vCQ7dX(U;KF~%X5dd1$SRq^sS4Rq
Nl(QKrH(~%y004$l0dN2S

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/87d94d88fe29d277c76e1a52042b02c092d5ae14 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/87d94d88fe29d277c76e1a52042b02c092d5ae14
new file mode 100644
index 0000000000000000000000000000000000000000..83de83f4eb59e2cfea819e1fdee97b3f6aec525c
GIT binary patch
literal 420
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH$I<>coVF)$c{1i<(|0|Nwscuwv?
j#UPp#%!H=+Kg0k3^vA6D>?FlS5OYZI0#P9kGLHcOQzODN

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8c4646f3357945c4e19a59ff79fffe3c874dbf16 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/8c4646f3357945c4e19a59ff79fffe3c874dbf16
new file mode 100644
index 0000000000000000000000000000000000000000..fa47e75a6323a989daa16e2a482a44f9ab2f2705
GIT binary patch
literal 189
zcmZ?r?Gj)BgDwUJh8hM21_cOCsmx2v%}kCj$<NQxD=tY)&W<n6tV)f~EzK#(EXc`B
zEn;9~P+(wYU|{&q;N;u&|Nno6|Hz6M;PMOt3=G2D5Dc~oLi`8mWcUx^IJrY;BvJ>7
KjfELh%m4tnA|Rds

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/90632bc6dee4eb836f3d7db1d16446a9c8510080 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/90632bc6dee4eb836f3d7db1d16446a9c8510080
new file mode 100644
index 0000000000000000000000000000000000000000..e739e858b860e774e3cdf9142de13206f508d2ea
GIT binary patch
literal 3895
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXMkWR1>+7OdT_a%lYtQuphGAG;fY}2-Ar+0GBeQI6MqS~`91-8db^i2`>5!6
zDjG$|5Buq87a_Nxm$@)Lp2C=rk@S~`IhrbTe0)5^+_`hX7{sQEp&&z8K?DN>1B`}p
zVbsl=HyIv0c)$RnuU)&w0Ky<PObn)u+O&d#0>hIhPZ+LWzs_*#)G3C2`}Q&H*s%kQ
zL2Qs1NDib1q?X$DgUr2h<qE@r0|yw^tXab_d-iOG2@@uOF^CNk1IdBZfYcIae>_pj
zK;aAW8_4{nOP4ZqbaXHj78Ww3rKN!}hz$}0$$`{>)PnR7WjGOvKyeESUyvU`_9Q1K
zGkAM@GuYePgE5E=5(CMB)PU52^blb*K_#HH4e|>pjzQrJ@+-(bGeZ*wP&|XMfq?<o
zJs>%d8jxC$9*|yw77$PX%44834oYjF_y&bJ$loA)`S|!4Kp4aZiGk!mYCvj1dO&&!
z7*9|dWCqAAkeMK}L3V)b0@(?&8-zh@kQhh~qz0rGqz9yzpaldJfWi<ImY^^Ng)JzI
zLE#SybCA6t3}S=CKyn~8AhjSpAiV^PCrTQWmOyC=l(s-=43ySD_JRBi!XP$C3?v6q
z15yjpLzLk}C<6Hfl&3&>3zWw|_JI5f3U3ewu|Z-WIglEVT96*7(VS44A_`QdfXWt-
z`5-@n!Wk6TAPi!I#6WT&H6XPVS%yV3D4s#)g8T*wUr-!_(isSY*dQ^G97qjFEfyoF
ZA_NLoP~3uOP#One5E~{&6-$P#Apju*$>0D0

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/94d06016aa949e8e7203217e4cc6625ded7f4244 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/94d06016aa949e8e7203217e4cc6625ded7f4244
new file mode 100644
index 0000000000000000000000000000000000000000..c989a76df155ff09d69d2a765d6e097928b8344f
GIT binary patch
literal 3102
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXP_4zq28uf@Zz%vBaTPo0wYN9h2IF3qhsMfj}X59|3AZT>iCJM`~&h6dVxF2
t9}R)g5E#TEK-2aR3#dpKJQy_^{zJfDRMFwilfwZh^fx3di6YT#0suAS7!d#f

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9875819b9e5783e7489c29a81cc9d4279209956a b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9875819b9e5783e7489c29a81cc9d4279209956a
new file mode 100644
index 0000000000000000000000000000000000000000..6ff64a7d2bab262cee2e8b6bebf65af9aafda630
GIT binary patch
literal 22
XcmZ?r?Gj)BgDwUJh8hrylYs#MB7Oon

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9c1cc734114b29aac6c51782d5c17e9dbe1faca2 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9c1cc734114b29aac6c51782d5c17e9dbe1faca2
new file mode 100644
index 0000000000000000000000000000000000000000..2d1d8576a29b050160e63a142b48927db9540b8d
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B32=N)iKi7
nx>4IkLttcvz?q@oZJnXu$Wb?rhQMeDjE2By2#kin&<z0qRAvO`

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9d2961871eeb201ef8a6f5503d8a8b62 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9d2961871eeb201ef8a6f5503d8a8b62
new file mode 100644
index 0000000000000000000000000000000000000000..a5865c60a614829812b37c8439f6bfc00f7198c0
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#LR0vkmVkkQ
zftJSqA2lC;@X|Vb@Y_BrI~oE6Lx9#vbkuzOQGiw*jB-aqU^E0qLtr!nMo0(%056#1
AQvd(}

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9f39e11cdd88344a4894b678e5a04a810880064d b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/9f39e11cdd88344a4894b678e5a04a810880064d
new file mode 100644
index 0000000000000000000000000000000000000000..8f712aea9ff2d6e930f0269002f9e9d1a0caa016
GIT binary patch
literal 972
zcmZ?r?Gj*MU|?ck=we`Cs9|7WP=I1a1_cIY1_lNO0R{$PZU|;zU|<9Zfi*xF{}~t{
z2*h=A4=M)Hq+lj=#sB~RXCMMQ;ZcM~3S>6AjWp!r_6yjH47lY9_yOdP!G!&(>t5=b
UMo}o!Fm<3C4NeneI2K|K0OV@Q4gdfE

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a350588a6dabe4376a066aed44ef8786d8e752e7 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a350588a6dabe4376a066aed44ef8786d8e752e7
new file mode 100644
index 0000000000000000000000000000000000000000..b7ad89078b5fc1b96e811aa117cc11c7f4f0467b
GIT binary patch
literal 861
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)HL|~Bd
zNJjtv{~xAF491`i{r^7$$Z%3I>6SYX*1||w$*2nQLx6}x3`)UB7C@3NH0dFwUN$5(
NXiT*9geHrX3jm2YMWFxy

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a6101a79919d444e1fc50aefab5837c39e3f4a19 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a6101a79919d444e1fc50aefab5837c39e3f4a19
new file mode 100644
index 0000000000000000000000000000000000000000..173c941952bf798f796a9ef4c751d1e6e6e3a09c
GIT binary patch
literal 1237
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B32=N)iKi7
Tx>4IkLtxN`zyM}Zs#*;Imdpe4

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a9c8793f8fb063bec839ee1280406fe5396545e5 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/a9c8793f8fb063bec839ee1280406fe5396545e5
new file mode 100644
index 0000000000000000000000000000000000000000..644560ced96b56c960032d7df08b3730053bc3ea
GIT binary patch
literal 420
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+ljA
j#s3-P|Nm$B55bJTAvBCXRA{L4iE%DbA^-n>Jp%&({h7bD

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ad4e9d2234e8599bdf12607c6b8cab4edae82c4e b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ad4e9d2234e8599bdf12607c6b8cab4edae82c4e
new file mode 100644
index 0000000000000000000000000000000000000000..f1826b06e88c9ca21599abaee7481ae2f07ee840
GIT binary patch
literal 420
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+ljA
U#s3-p{~z+01TPR3@*wjV0QXD57ytkO

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b90b6830917919e94186d312f06481bd b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b90b6830917919e94186d312f06481bd
new file mode 100644
index 0000000000000000000000000000000000000000..2da6be376f2af7b9cefc00d5b0d096c7e68fea69
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B39w3y3r6A
d4S~@R7!85Z5Eu=C(GVC7fzc2c4S``70sxg!1ML6+

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b98fd4cb1d7031240414301c19b03097c0035c6b b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/b98fd4cb1d7031240414301c19b03097c0035c6b
new file mode 100644
index 0000000000000000000000000000000000000000..b84b57bb53177764b84b10a75624dfe949ddbe81
GIT binary patch
literal 857
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~uS=>LC)|Ns9p;5Nn<
zX3qcj|Np=F^8df^H>e2^3S<b_C=l1lJ*XH&lY*H@QA~Zc5dVY1U@%|?^w9WEJYZ#D
e0L3Ll1BgW`W*Q73K%^T$(e|Ab4}vt1q8<P-t*A)=

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ba976fcdb4daf092ef17ce43bf2b78d9d8bc2aeb b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ba976fcdb4daf092ef17ce43bf2b78d9d8bc2aeb
new file mode 100644
index 0000000000000000000000000000000000000000..4ee9cbdfc3defea2195af8ccd749fc0a326a7d00
GIT binary patch
literal 2262
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXP_4zq24Ajcxf0Rs1X24d4o0G{UtHb=$3}bbr*&MQR9NjWdQkUh$v@)0s*}s
z8RZXf2oO__`=k~ZyJ93oP;?T&ECl4qF_Qv4s6qW70{)_k4tJg$4nU#5Az?`riDnZ3
DcS8F@

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/bc112b571eafee0f5a031f3c9cce6244216d128d b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/bc112b571eafee0f5a031f3c9cce6244216d128d
new file mode 100644
index 0000000000000000000000000000000000000000..af1091428d59645a8158218a953fbd60e0c463d8
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lje
u6pyMM4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVCGApih??guRZ

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c42b981c28a1715c375050f6fcf53f1d b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c42b981c28a1715c375050f6fcf53f1d
new file mode 100644
index 0000000000000000000000000000000000000000..996e8c826cbd548e4567bfc600496ae31084c288
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B39w3y3r6A
z4FRMOh>wq-3kLD=@kqi{VBWlW^TC4$H*em&1_p30<wh$gC_H)c<ofmNr%s*Pw{PDL
zFxa<m->Fllu3x|Y<jE5S1yDglwna~#Jh^h^%7Ft1)~s1Gd-m)JU@&|3>@{oF95`^`
z%9Sfmo;)Gj@S8VpUcY|*z<~oxmoDw-=qM~KOap_$!orS@j-^YN9yoB|`t|EKZ{8%&
ziU$uKoH})C&6+hG9UaNZ$==@H_F&-c?VX&Q+|kjoX3d&Yr%pY1@PG{S<KyFR-n_YQ
z-@e(iXBQS0dV6~t7#Jvkfq{X6x3_m;Vd3oAv-j=Wck|{=GE&ssxpS{wyS8J;jtLVc
zq@|_V+uJKBDDZ)Sf`Wp*y?t6*+Jp%ccI?=3?b@}ubLW!k1zMX=>+oS<V4!LIGB7aE
vI(;%QFwiuAfm#?eE*~JN4ql*vi??fVE<~1EB<;#)YPoRKoY4>%Y#{&ub--4f

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c6049874b33eadb016fccf0c5fa66e556ae069b9 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c6049874b33eadb016fccf0c5fa66e556ae069b9
new file mode 100644
index 0000000000000000000000000000000000000000..4863878ca02ca42e550f521cb1ba5b8bd7046ecc
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B32=N)iKi7
zy5V9Q$8hlo?ffxF6FluaJZkc22#kina18-ay8!Gr4KO)MjD`T2A;3qbzEN$%DFgsk
Cq6AI=

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8697bf2369f6ab85f501376c4d93bb8a56974a3 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8697bf2369f6ab85f501376c4d93bb8a56974a3
new file mode 100644
index 0000000000000000000000000000000000000000..30aacc2f98820e139b1495c9c1527f9119dec97b
GIT binary patch
literal 420
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=QG0VEF%^
e;XmaVWbA*^&B13inMP8@JGk6MRLFyZfdK%8dhO`|

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8daf283e0aef2fd7b630c0430e05dc28f24ecf6 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/c8daf283e0aef2fd7b630c0430e05dc28f24ecf6
new file mode 100644
index 0000000000000000000000000000000000000000..b831633f02b50870db441110db29e03331decdaf
GIT binary patch
literal 220
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvWW?&$wfPq1nn}LBr078S=VDrGle+C8!0`Z*O
dgNhj#-h-G(7$W~4A_7qYX0w4wA_yj2Y5?yJ4;}yj

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/cacff56e1af4b8fde912822da06b10fb8c545a19 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/cacff56e1af4b8fde912822da06b10fb8c545a19
new file mode 100644
index 0000000000000000000000000000000000000000..ff492d29d76ce94f0fb8db9a0b6c481c407b205b
GIT binary patch
literal 1033
zcmZ?r?Gj*MU|?ck=we`Cs9|7WP=I1a1_cIY1_lNO0R{$PZU}brt>OU*F)&myFfimm
zRf2RdGDtuqcpxf4Iv@fJ3{LJr#V|IpG!wB3QPllsV1OWyC?RYAGyMNg7A8v*5tcFj
zXJC{jLLIH_r>D862mv)53LuVzCR+yLk}W7TKrtkF{D<%er4WcJR1za$l9PyN5Xq>n
MB*=q?0Nf!A01)5EJOBUy

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ce4dcc22b1d595c49a25121c0b580104 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ce4dcc22b1d595c49a25121c0b580104
new file mode 100644
index 0000000000000000000000000000000000000000..ea776fb0a94b79362822f76bfa1f0d9364a09c4a
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lje
z6pyMM4S~@R7!3hzArK!Q&oFoHTrdW)v8kbj*v*?a86G@%zyPAJUAx8r!XP$Gj23o+
z%v4ZNV0iN63B&d4*BMToI>oSW-#&&NJ9dCEhz$}0$$`{>)PnR-6NAjXa^(ucfddB^
z)~s2>FnjiFh6xiUfH8;-5(CMB)PU4d({51sg8T+Df9cYt3>_UE426Y-3~6a;U<_h|
z#6WT&H6XPhJ=AgoC~iUF3-Tk#p5)|Y25)a~277yZFb1(fVjww?8jxC$9%|SRO4}g6
zfZ`Yw&LF>n>@zShU;xE42!q%lF_0Wc4M;6W4@fViZUE&mP#OoNHBfwm!W`spkiC3-
zd<-BAVuQp$av(JzwIDqpy+hr8P#A*35)`JOumy!ND6BzY4)Qk$gV-Q3kQ_)2NG(VY
zNH3-S2k8f;B~Y3Ir7chz1En>PeV{N0VGtW629g7*0jUM)0qLa%2Kfb)r$BiNl*d5!
zfcy%IZx9BtL1G{|kQ$I$kREE-4Ko8&rhv*8koh1#g2EY;)<77<28n^>Kx#m0Vfv^=
zgW?%vF34}7@CC&&D4l~ahz$}0$$`{>)Kbe{baO!A3W{404a#F63}VB?(ACq54@%ph
W{0+h&Hm&R#HFq=wMnhoOh5!H`xLMo)

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d0cd71dbf039fd64cf42eff30da92a71a919226a b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d0cd71dbf039fd64cf42eff30da92a71a919226a
new file mode 100644
index 0000000000000000000000000000000000000000..b9dc5d0f4ee4edb8d45ae3e5ed182f57e5d4e5db
GIT binary patch
literal 587
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Am|Nk@m$ATHyAku_MCb$wP
z3!8D&7E{LJWHdV%A?}J`KnodEWzaAJg%$pAV))MhMlg%9h8n8b*h7}OYQ7V4$3I;5
Np#=*#ApV0r2>^j|C42w?

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d5ce626ac3264bed6af5580e341a89406857cbb9 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d5ce626ac3264bed6af5580e341a89406857cbb9
new file mode 100644
index 0000000000000000000000000000000000000000..b17294ec90a31bc77f439f7baae0d897bf860ec4
GIT binary patch
literal 1760
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B32=N)iKi7
vx>4I;AuveO1I&4&^k@iR3jsE4f}>(&g#c#dgUy&9{~-XI983&XrNsaMI1~<L

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d77ada02e9bc8c24b2711eca6a8f52ae356bfc21 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d77ada02e9bc8c24b2711eca6a8f52ae356bfc21
new file mode 100644
index 0000000000000000000000000000000000000000..9cc65607fecd34abe27d5e0956e9fa462f0c48f1
GIT binary patch
literal 361
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#!mE%Guk4VP
Lbn@*IU|;|MCshR(

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d7eb9c5a0f9803df4c00390793b8ab57bd7c9484 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/d7eb9c5a0f9803df4c00390793b8ab57bd7c9484
new file mode 100644
index 0000000000000000000000000000000000000000..531427a99c0f1bbef21f35fa90027a7d662b39ec
GIT binary patch
literal 3792
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)H+#rHT
z%!ID)|Ns9C2>kCmf{iRbY}iDFJSYs%!)XZdL1~GUbVPyIU`Y_45yK|!AmtPK`sq7L
z9>rldz9=A0PKP*ULrfJ>IiBdUc8GZ%&AHG@2TgRib6JPG2S@$KI~oKu41p)40~)%9
S=0*?CdXVP9Hfq6$4FLd&N(T-A

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dc1efccdeec17e151a1ec8228c09ab61c3040b33 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dc1efccdeec17e151a1ec8228c09ab61c3040b33
new file mode 100644
index 0000000000000000000000000000000000000000..5a5c1c30eb0d70bea1562137328ac2d74871e43d
GIT binary patch
literal 35
jcmZ?rt>R$-gDM6Fh8zY41_K5L1_e0=21W)6us8z%KuiLI

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dcea22c66c60088165a2f1772036473f b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/dcea22c66c60088165a2f1772036473f
new file mode 100644
index 0000000000000000000000000000000000000000..44d2ebfb3d0f37c6355fe4e44c27f7457ebee63c
GIT binary patch
literal 4234
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+lj=
z#sB~RXBa39a?3#34YGICeue>y`@s<YAUBNSfy@7+VL$L;KUxNk>@pA(|D$*`1V%$(
RGz3ONU^E0qLtsRO003l9=92&b

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/de539ae7442fa05dafcfe1a021f0186ef74a2b0e b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/de539ae7442fa05dafcfe1a021f0186ef74a2b0e
new file mode 100644
index 0000000000000000000000000000000000000000..f9a8f33443d84d5494b23e5b3317015f10b0496a
GIT binary patch
literal 103
zcmZ?r?Gj)BgDwUJwHgKn1_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~weGcf#z@SNO(ia{(C
J(?9|+3;@+b2*dyY

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2306b1d6b88d0ccc4e2c3a9edb07462a5a32215 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2306b1d6b88d0ccc4e2c3a9edb07462a5a32215
new file mode 100644
index 0000000000000000000000000000000000000000..71bf61cebe4e69dc71714916d804d979824a7d0b
GIT binary patch
literal 110
zcmZ?r?c!wsgDwUJh8hM21_dZ)WKdvWW?*2@6<}Zx=7wOfLJ0AnfdPU*TqpOSVi1iF
JJ2Cin0RZon2yOrX

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2778da0240fdd15ef5844905d81c4e05f34a8bd b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e2778da0240fdd15ef5844905d81c4e05f34a8bd
new file mode 100644
index 0000000000000000000000000000000000000000..1bad15905ffd4776122e34df11873b390abee72a
GIT binary patch
literal 3194
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B32=N)iKi7
zx>4IkLtyxaz*j6@JcC7S&<W`bIu{`v6pdhxvPMH-Gz3ONU^D~<QwTT=CdUj4w>aW)
F1pwWl2DbnJ

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e6642e9266875f9d908942e534bf898103a2c794 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/e6642e9266875f9d908942e534bf898103a2c794
new file mode 100644
index 0000000000000000000000000000000000000000..f9d9de9c9c21f18a6eb88afb89f4450d4fd6771f
GIT binary patch
literal 2186
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)HxbZg(
zDJFE0|Ns9p{Qv)(0YZb=Lmh*>0`?S$F^a)KK~|iSrHO(72WbVn7Q`6E;Gm#)98hN8
heiGfxz%W`u4Yq_n=qrLM%ryEPBi#+QP#vu90RU(y=d1t#

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ec6cdb929c08d8daf2bd7fc185fbf4d787b45120 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ec6cdb929c08d8daf2bd7fc185fbf4d787b45120
new file mode 100644
index 0000000000000000000000000000000000000000..782a0925210623c0d6e33ba7c885435bb19654d3
GIT binary patch
literal 1942
zcmZ?r?Gj*MU|?ck=we`Cs9|7WP=I1a1_cIY1_lNO0R{$PZU}brt>OU*F)&myFfimm
zRf2RdGDtuqcpxf4Iv@fJ3{LJr#V|Gl1H=FS+W-ImXBc>V`SNA*oPp$-0cG+IsO@yI
z{Qox^S@Q%ppplKF89VUVgS@<f;cjRN!$4dKgA_>r85lqaT2xSK48&lxq6i`gvIfdx
jaDXvJ>A@2M<)k_Vnr1N4B8B251oY!}CKI|8ey0Hd2er#3

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ed8636357f79439b6a03eb14469b686cc401a1c9 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ed8636357f79439b6a03eb14469b686cc401a1c9
new file mode 100644
index 0000000000000000000000000000000000000000..efd9312d94dde30707099125b0280be21f6b2101
GIT binary patch
literal 3895
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+An=KKu*8K4NncXAIZ2GOKo
zCUnLB|Nm!zU?>IS4k3Ezx*Q((LnyRq>}HB1lbM0uf%%t!o9`2lC)vHWkQ__0KC;|F
zmL}{Dq@q!T{Jx)#b`f$5dYKF58(<X1j0}u}$l3oH@YPehVD*$RC)7U-hA;-SlKKzf
zqtsJQAThG4KL%Lc2R9aE6cQ%AHbl})91~VugWLjj25}lkRbhqz_LdHz%rx@k_tUt>
x#a>nq@o=ufbpQ9!a2!;%3&rW~H)b3YYbM13F#=kdm?bvxB8(8F>=24dBmi&wrSbp(

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ee313e9acecb5c688ce8c9bb10e70e136fbb9c6d b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ee313e9acecb5c688ce8c9bb10e70e136fbb9c6d
new file mode 100644
index 0000000000000000000000000000000000000000..03e09e28193a513347433ee41f005f0926329d20
GIT binary patch
literal 3616
zcmZ?r?Gj)RU|?Wi=we`Cs9|7WPyl1FI7k>xoPmLnL4ko8BqPAUAj}QHATAXDXJCLJ
z5Z}o?s2D_(f|<}2|NsA=0f9@&1;~a_gFWCOK@Cq3VFpsz<5$hVz`%hXccXktLI9jE
zDKP-5mkmnOmqO%OXu2WO`UW!f;nLQKOM)cvFp^Z0qj=C3!O-YLDVB$x7&%cwgdV&_
z;9$>)g#!VI2vjO6!It|mN-FG08@nhm^&|N@i7|at&1m?JrUAO80Z>p6z3L9Vj6*9@
O!O~!{|6meCfLH*NS(IV`

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ef689af320e7d9e22231109faae2e8149cb86e1c b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ef689af320e7d9e22231109faae2e8149cb86e1c
new file mode 100644
index 0000000000000000000000000000000000000000..f8688710452dafe40fb247674471ea9f642b3778
GIT binary patch
literal 834
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo+Aq{}~t{2*h`C4=M)Hq+ljw
z#UP9R|NqZG3P#qCpG}IXAhq~SLzV@p9-26~G>Mq-9wg}l6z0?oX9_%q<dFaW836y^
BM?L@m

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/fda6b9a9f6ffdf4765c00465619c7ceb3f7db2e4 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/fda6b9a9f6ffdf4765c00465619c7ceb3f7db2e4
new file mode 100644
index 0000000000000000000000000000000000000000..20efec0d1e19ea1f2b317f82c08f0ec5d41a66ca
GIT binary patch
literal 1136
zcmZ?rt>R$-gDM6Fh8zY41_dZ)WRL)hGw?7l2y;U)M1X<8$vvnT#wM0#B32=N)iKi7
lx}j{F3auRfgH{GN(8^%?o2y2ul`Pmb$Y2+yrl=zx!vHe$1`z-N

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ffe829bb0adac20d9c0756f68a22d1255e4fdb54 b/tensorflow/core/kernels/fuzzing/corpus/decode_bmp/ffe829bb0adac20d9c0756f68a22d1255e4fdb54
new file mode 100644
index 0000000000000000000000000000000000000000..e24c09dacce1b52dc3737b1d921920846b627b2f
GIT binary patch
literal 3194
zcmZ?r?Gj)BgDwUJh8hM21_dZ)WKdvW2FVC8FbH!)Fo=t(10)L71Z55`3hH7e1_qEH
z(cA{|G6NHuAjALv|4G8=`f>9~H5s3J+_sF$&?E$qvH=4_0s{jBaj66?2SK!gG&B4M
zlOVS`xd#=4Xa;C%Wq_nsQ1T>{Bf;9h24OJ{DoG4QaW24@R<X$u<ET+JqaiSeLSVE6
qAgNx&mg|Rt7|s;;4MPS;srj4oaH=P+cA-glVt8{@U^E1%8Ug?r`AQ)G

literal 0
HcmV?d00001

-- 
GitLab


From e44afc5bed4f683a6e912fa5a172f11b8bf5bc48 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 29 Nov 2018 11:59:37 -0800
Subject: [PATCH 1040/1554] Ensure run_deprecated_v1 only works when tf2 is
 enabled

PiperOrigin-RevId: 223385893
---
 tensorflow/python/framework/test_util.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index fc97a2275c..155e45d16c 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -50,6 +50,7 @@ from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import pywrap_tensorflow
+from tensorflow.python import tf2
 from tensorflow.python.client import device_lib
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
@@ -1026,7 +1027,10 @@ def run_deprecated_v1(func=None):
       raise ValueError("`run_deprecated_v1` only supports test methods.")
 
     def decorated(self, *args, **kwargs):
-      with context.graph_mode():
+      if tf2.enabled():
+        with context.graph_mode():
+          f(self, *args, **kwargs)
+      else:
         f(self, *args, **kwargs)
 
     return decorated
-- 
GitLab


From ed1683498b553b69b2246ac2ed614253be10869b Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 29 Nov 2018 12:02:06 -0800
Subject: [PATCH 1041/1554] Fix the broken TFLite makefile

PiperOrigin-RevId: 223386342
---
 tensorflow/lite/tools/make/Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 06a1f36105..363a069d5e 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -114,6 +114,10 @@ ifeq ($(BUILD_TYPE),micro)
 CORE_CC_EXCLUDE_SRCS += \
 tensorflow/lite/mmap_allocation.cc \
 tensorflow/lite/nnapi_delegate.cc
+else
+CORE_CC_EXCLUDE_SRCS += \
+tensorflow/contrib/lite/mmap_allocation_disabled.cc \
+tensorflow/contrib/lite/nnapi_delegate_disabled.cc
 endif
 # Filter out all the excluded files.
 TF_LITE_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS))
-- 
GitLab


From 20179793c2c96bbcbbaaf400b247e35aa81d9deb Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Nov 2018 12:11:14 -0800
Subject: [PATCH 1042/1554] Applying kernel priorities to datasets that have
 both CPU and GPU implementations. This makes sure that placer would prefer
 the CPU kernel over the GPU kernel if no with tf.device(..) is specified.

This is necessary for creating dataset kernels as we go (i.e. in the __init__ methods of the Dataset class).

PiperOrigin-RevId: 223388100
---
 .../core/kernels/data/generator_dataset_op.cc | 10 ++--
 tensorflow/core/kernels/data/iterator_ops.cc  | 58 +++++++++++--------
 tensorflow/core/kernels/data/optional_ops.cc  | 28 +++++----
 .../core/kernels/data/prefetch_dataset_op.cc  |  5 +-
 4 files changed, 59 insertions(+), 42 deletions(-)

diff --git a/tensorflow/core/kernels/data/generator_dataset_op.cc b/tensorflow/core/kernels/data/generator_dataset_op.cc
index c089bfc45f..48697ec6c8 100644
--- a/tensorflow/core/kernels/data/generator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/generator_dataset_op.cc
@@ -182,11 +182,13 @@ void GeneratorDatasetOp::MakeDataset(OpKernelContext* ctx,
 }
 
 namespace {
-REGISTER_KERNEL_BUILDER(Name("GeneratorDataset").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("GeneratorDataset").Device(DEVICE_CPU).Priority(2),
+                        GeneratorDatasetOp);
+REGISTER_KERNEL_BUILDER(Name("GeneratorDataset")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("handle")
+                            .Priority(1),
                         GeneratorDatasetOp);
-REGISTER_KERNEL_BUILDER(
-    Name("GeneratorDataset").Device(DEVICE_GPU).HostMemory("handle"),
-    GeneratorDatasetOp);
 }  // namespace
 
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index c50af846f9..cb7477f9e2 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -1272,50 +1272,60 @@ class DeserializeIteratorOp : public OpKernel {
 
 
 REGISTER_KERNEL_BUILDER(Name("Iterator").Device(DEVICE_CPU), IteratorHandleOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorV2").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("IteratorV2").Device(DEVICE_CPU).Priority(2),
                         IteratorHandleOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorV2").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("IteratorV2").Device(DEVICE_GPU).Priority(1),
                         IteratorHandleOp);
-REGISTER_KERNEL_BUILDER(Name("MakeIterator").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("MakeIterator").Device(DEVICE_CPU).Priority(2),
                         MakeIteratorOp);
 REGISTER_KERNEL_BUILDER(
-    Name("MakeIterator").Device(DEVICE_GPU).HostMemory("dataset"),
+    Name("MakeIterator").Device(DEVICE_GPU).Priority(1).HostMemory("dataset"),
     MakeIteratorOp);
-REGISTER_KERNEL_BUILDER(Name("AnonymousIterator").Device(DEVICE_CPU),
-                        AnonymousIteratorHandleOp);
-REGISTER_KERNEL_BUILDER(Name("AnonymousIterator").Device(DEVICE_GPU),
-                        AnonymousIteratorHandleOp);
+REGISTER_KERNEL_BUILDER(
+    Name("AnonymousIterator").Device(DEVICE_CPU).Priority(2),
+    AnonymousIteratorHandleOp);
+REGISTER_KERNEL_BUILDER(
+    Name("AnonymousIterator").Device(DEVICE_GPU).Priority(1),
+    AnonymousIteratorHandleOp);
 REGISTER_KERNEL_BUILDER(Name("DatasetToSingleElement").Device(DEVICE_CPU),
                         ToSingleElementOp);
 REGISTER_KERNEL_BUILDER(Name("ReduceDataset").Device(DEVICE_CPU),
                         ReduceDatasetOp);
 REGISTER_KERNEL_BUILDER(Name("OneShotIterator").Device(DEVICE_CPU),
                         OneShotIteratorOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_CPU).Priority(2),
                         IteratorGetNextOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_GPU).Priority(1),
                         IteratorGetNextOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorGetNextSync").Device(DEVICE_CPU),
-                        IteratorGetNextSyncOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorGetNextSync").Device(DEVICE_GPU),
-                        IteratorGetNextSyncOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorGetNextAsOptional").Device(DEVICE_CPU),
-                        IteratorGetNextAsOptionalOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorGetNextAsOptional").Device(DEVICE_GPU),
-                        IteratorGetNextAsOptionalOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorToStringHandle").Device(DEVICE_CPU),
-                        IteratorToStringHandleOp);
+REGISTER_KERNEL_BUILDER(
+    Name("IteratorGetNextSync").Device(DEVICE_CPU).Priority(2),
+    IteratorGetNextSyncOp);
+REGISTER_KERNEL_BUILDER(
+    Name("IteratorGetNextSync").Device(DEVICE_GPU).Priority(1),
+    IteratorGetNextSyncOp);
+REGISTER_KERNEL_BUILDER(
+    Name("IteratorGetNextAsOptional").Device(DEVICE_CPU).Priority(2),
+    IteratorGetNextAsOptionalOp);
+REGISTER_KERNEL_BUILDER(
+    Name("IteratorGetNextAsOptional").Device(DEVICE_GPU).Priority(1),
+    IteratorGetNextAsOptionalOp);
+REGISTER_KERNEL_BUILDER(
+    Name("IteratorToStringHandle").Device(DEVICE_CPU).Priority(2),
+    IteratorToStringHandleOp);
 REGISTER_KERNEL_BUILDER(Name("IteratorToStringHandle")
                             .Device(DEVICE_GPU)
-                            .HostMemory("string_handle"),
+                            .HostMemory("string_handle")
+                            .Priority(1),
                         IteratorToStringHandleOp);
 REGISTER_KERNEL_BUILDER(Name("IteratorFromStringHandle").Device(DEVICE_CPU),
                         IteratorFromStringHandleOp);
-REGISTER_KERNEL_BUILDER(Name("IteratorFromStringHandleV2").Device(DEVICE_CPU),
-                        IteratorFromStringHandleOp);
+REGISTER_KERNEL_BUILDER(
+    Name("IteratorFromStringHandleV2").Device(DEVICE_CPU).Priority(2),
+    IteratorFromStringHandleOp);
 REGISTER_KERNEL_BUILDER(Name("IteratorFromStringHandleV2")
                             .Device(DEVICE_GPU)
-                            .HostMemory("string_handle"),
+                            .HostMemory("string_handle")
+                            .Priority(1),
                         IteratorFromStringHandleOp);
 REGISTER_KERNEL_BUILDER(Name("SerializeIterator").Device(DEVICE_CPU),
                         SerializeIteratorOp);
diff --git a/tensorflow/core/kernels/data/optional_ops.cc b/tensorflow/core/kernels/data/optional_ops.cc
index bee857f53c..d8a7f21c5f 100644
--- a/tensorflow/core/kernels/data/optional_ops.cc
+++ b/tensorflow/core/kernels/data/optional_ops.cc
@@ -127,23 +127,27 @@ class OptionalGetValueOp : public OpKernel {
   std::vector<PartialTensorShape> output_shapes_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("OptionalNone").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("OptionalNone").Device(DEVICE_CPU).Priority(2),
                         OptionalNoneOp);
-REGISTER_KERNEL_BUILDER(Name("OptionalNone").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("OptionalNone").Device(DEVICE_GPU).Priority(1),
                         OptionalNoneOp);
-REGISTER_KERNEL_BUILDER(Name("OptionalFromValue").Device(DEVICE_CPU),
-                        OptionalFromValueOp);
-REGISTER_KERNEL_BUILDER(Name("OptionalFromValue").Device(DEVICE_GPU),
-                        OptionalFromValueOp);
+REGISTER_KERNEL_BUILDER(
+    Name("OptionalFromValue").Device(DEVICE_CPU).Priority(2),
+    OptionalFromValueOp);
+REGISTER_KERNEL_BUILDER(
+    Name("OptionalFromValue").Device(DEVICE_GPU).Priority(1),
+    OptionalFromValueOp);
 
-REGISTER_KERNEL_BUILDER(Name("OptionalHasValue").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("OptionalHasValue").Device(DEVICE_CPU).Priority(2),
                         OptionalHasValueOp);
-REGISTER_KERNEL_BUILDER(
-    Name("OptionalHasValue").Device(DEVICE_GPU).HostMemory("has_value"),
-    OptionalHasValueOp);
-REGISTER_KERNEL_BUILDER(Name("OptionalGetValue").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("OptionalHasValue")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("has_value")
+                            .Priority(1),
+                        OptionalHasValueOp);
+REGISTER_KERNEL_BUILDER(Name("OptionalGetValue").Device(DEVICE_CPU).Priority(2),
                         OptionalGetValueOp);
-REGISTER_KERNEL_BUILDER(Name("OptionalGetValue").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("OptionalGetValue").Device(DEVICE_GPU).Priority(1),
                         OptionalGetValueOp);
 
 static Status OptionalDeviceCopy(
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 960373b74f..9e518131eb 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -391,13 +391,14 @@ void PrefetchDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input,
 }
 
 namespace {
-REGISTER_KERNEL_BUILDER(Name("PrefetchDataset").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("PrefetchDataset").Device(DEVICE_CPU).Priority(2),
                         PrefetchDatasetOp);
 REGISTER_KERNEL_BUILDER(Name("PrefetchDataset")
                             .Device(DEVICE_GPU)
                             .HostMemory("buffer_size")
                             .HostMemory("input_dataset")
-                            .HostMemory("handle"),
+                            .HostMemory("handle")
+                            .Priority(1),
                         PrefetchDatasetOp);
 }  // namespace
 
-- 
GitLab


From 74230d7295e25b0ff9d4f78aa2dc02281a487fad Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 29 Nov 2018 12:20:59 -0800
Subject: [PATCH 1043/1554] Update distribution strategy library
 `get_loss_reduction()` function to check for V2 Loss Reduction Enum.

PiperOrigin-RevId: 223389700
---
 .../python/distribute/distribute_lib.py       |  3 +-
 tensorflow/python/keras/losses.py             |  5 ++-
 tensorflow/python/keras/losses_test.py        | 25 ++++++------
 tensorflow/python/keras/utils/losses_utils.py | 38 ++++---------------
 tensorflow/python/ops/losses/losses_impl.py   | 24 ++++++++++++
 .../tensorflow.keras.losses.-reduction.pbtxt  |  4 +-
 .../v2/tensorflow.losses.-reduction.pbtxt     |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  6 +++
 8 files changed, 59 insertions(+), 50 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 2213499be0..eddd6ff8b1 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -81,7 +81,8 @@ class UpdateContext(object):
 def get_loss_reduction():
   """`tf.distribute.ReduceOp` corresponding to the last loss reduction."""
   loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
-  if loss_reduction == losses_impl.Reduction.SUM:
+  if (loss_reduction == losses_impl.Reduction.SUM or
+      loss_reduction == losses_impl.ReductionV2.SUM):
     return reduce_util.ReduceOp.SUM
   return reduce_util.ReduceOp.MEAN
 
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 1bd9f729c5..83318d6c57 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -28,7 +28,6 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss
-from tensorflow.python.keras.utils.losses_utils import ReductionV2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops.losses import losses_impl
@@ -56,7 +55,9 @@ class Loss(object):
     name: Optional name for the op.
   """
 
-  def __init__(self, reduction=ReductionV2.SUM_OVER_BATCH_SIZE, name=None):
+  def __init__(self,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
     self.reduction = reduction
     self.name = name
 
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index d80b272b12..b5e9a24c99 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -27,6 +27,7 @@ from tensorflow.python import keras
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 
 try:
@@ -146,9 +147,9 @@ class MeanSquaredErrorTest(test.TestCase):
 
   def test_config(self):
     mse_obj = keras.losses.MeanSquaredError(
-        reduction=keras.losses.ReductionV2.SUM, name='mse_1')
+        reduction=losses_impl.ReductionV2.SUM, name='mse_1')
     self.assertEqual(mse_obj.name, 'mse_1')
-    self.assertEqual(mse_obj.reduction, keras.losses.ReductionV2.SUM)
+    self.assertEqual(mse_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_all_correct_unweighted(self):
     mse_obj = keras.losses.MeanSquaredError()
@@ -214,7 +215,7 @@ class MeanSquaredErrorTest(test.TestCase):
 
   def test_no_reduction(self):
     mse_obj = keras.losses.MeanSquaredError(
-        reduction=keras.losses.ReductionV2.NONE)
+        reduction=losses_impl.ReductionV2.NONE)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -225,7 +226,7 @@ class MeanSquaredErrorTest(test.TestCase):
 
   def test_sum_reduction(self):
     mse_obj = keras.losses.MeanSquaredError(
-        reduction=keras.losses.ReductionV2.SUM)
+        reduction=losses_impl.ReductionV2.SUM)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -239,9 +240,9 @@ class MeanAbsoluteErrorTest(test.TestCase):
 
   def test_config(self):
     mae_obj = keras.losses.MeanAbsoluteError(
-        reduction=keras.losses.ReductionV2.SUM, name='mae_1')
+        reduction=losses_impl.ReductionV2.SUM, name='mae_1')
     self.assertEqual(mae_obj.name, 'mae_1')
-    self.assertEqual(mae_obj.reduction, keras.losses.ReductionV2.SUM)
+    self.assertEqual(mae_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_all_correct_unweighted(self):
     mae_obj = keras.losses.MeanAbsoluteError()
@@ -307,7 +308,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
 
   def test_no_reduction(self):
     mae_obj = keras.losses.MeanAbsoluteError(
-        reduction=keras.losses.ReductionV2.NONE)
+        reduction=losses_impl.ReductionV2.NONE)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -318,7 +319,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
 
   def test_sum_reduction(self):
     mae_obj = keras.losses.MeanAbsoluteError(
-        reduction=keras.losses.ReductionV2.SUM)
+        reduction=losses_impl.ReductionV2.SUM)
     y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
     y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
                                   shape=(2, 3),
@@ -332,9 +333,9 @@ class MeanAbsolutePercentageErrorTest(test.TestCase):
 
   def test_config(self):
     mape_obj = keras.losses.MeanAbsolutePercentageError(
-        reduction=keras.losses.ReductionV2.SUM, name='mape_1')
+        reduction=losses_impl.ReductionV2.SUM, name='mape_1')
     self.assertEqual(mape_obj.name, 'mape_1')
-    self.assertEqual(mape_obj.reduction, keras.losses.ReductionV2.SUM)
+    self.assertEqual(mape_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_unweighted(self):
     mape_obj = keras.losses.MeanAbsolutePercentageError()
@@ -389,9 +390,9 @@ class MeanSquaredLogarithmicErrorTest(test.TestCase):
 
   def test_config(self):
     msle_obj = keras.losses.MeanSquaredLogarithmicError(
-        reduction=keras.losses.ReductionV2.SUM, name='mape_1')
+        reduction=losses_impl.ReductionV2.SUM, name='mape_1')
     self.assertEqual(msle_obj.name, 'mape_1')
-    self.assertEqual(msle_obj.reduction, keras.losses.ReductionV2.SUM)
+    self.assertEqual(msle_obj.reduction, losses_impl.ReductionV2.SUM)
 
   def test_unweighted(self):
     msle_obj = keras.losses.MeanSquaredLogarithmicError()
diff --git a/tensorflow/python/keras/utils/losses_utils.py b/tensorflow/python/keras/utils/losses_utils.py
index d11d785356..fc4b4ac7df 100644
--- a/tensorflow/python/keras/utils/losses_utils.py
+++ b/tensorflow/python/keras/utils/losses_utils.py
@@ -25,31 +25,7 @@ from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
-from tensorflow.python.util.tf_export import tf_export
-
-
-@tf_export('losses.Reduction', 'keras.losses.Reduction', v1=[])
-class ReductionV2(object):
-  """Types of loss reduction.
-
-  Contains the following values:
-  `NONE`: Un-reduced weighted losses with the same shape as input.
-  `SUM`: Scalar sum of weighted losses.
-  `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
-  """
-
-  NONE = None
-  SUM = 'sum'
-  SUM_OVER_BATCH_SIZE = 'sum_over_batch_size'
-
-  @classmethod
-  def all(cls):
-    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
-
-  @classmethod
-  def validate(cls, key):
-    if key not in cls.all():
-      raise ValueError('Invalid Reduction Key %s.' % key)
+from tensorflow.python.ops.losses import losses_impl
 
 
 def squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
@@ -144,21 +120,21 @@ def _num_elements(losses):
     return math_ops.cast(array_ops.size(losses, name=scope), dtype=losses.dtype)
 
 
-def _reduce_weighted_loss(weighted_losses,
-                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE):
+def _reduce_weighted_loss(
+    weighted_losses, reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE):
   """Reduces the individual weighted loss measurements."""
-  if reduction == ReductionV2.NONE:
+  if reduction == losses_impl.ReductionV2.NONE:
     loss = weighted_losses
   else:
     loss = math_ops.reduce_sum(weighted_losses)
-    if reduction == ReductionV2.SUM_OVER_BATCH_SIZE:
+    if reduction == losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE:
       loss = _safe_mean(loss, _num_elements(weighted_losses))
   return loss
 
 
 def compute_weighted_loss(losses,
                           sample_weight=None,
-                          reduction=ReductionV2.SUM_OVER_BATCH_SIZE,
+                          reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
                           name=None):
   """Computes the weighted loss.
 
@@ -177,7 +153,7 @@ def compute_weighted_loss(losses,
     Weighted loss `Tensor` of the same type as `losses`. If `reduction` is
     `NONE`, this has the same shape as `losses`; otherwise, it is scalar.
   """
-  ReductionV2.validate(reduction)
+  losses_impl.ReductionV2.validate(reduction)
   if sample_weight is None:
     sample_weight = 1.0
   with ops.name_scope(name, 'weighted_loss', (losses, sample_weight)):
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 1b470937d4..9e9de62e6c 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -33,6 +33,30 @@ from tensorflow.python.util.deprecation import deprecated_argument_lookup
 from tensorflow.python.util.tf_export import tf_export
 
 
+@tf_export("losses.Reduction", "keras.losses.Reduction", v1=[])
+class ReductionV2(object):
+  """Types of loss reduction.
+
+  Contains the following values:
+  `NONE`: Un-reduced weighted losses with the same shape as input.
+  `SUM`: Scalar sum of weighted losses.
+  `SUM_OVER_BATCH_SIZE`: Scalar `SUM` divided by number of elements in losses.
+  """
+
+  NONE = "none"
+  SUM = "sum"
+  SUM_OVER_BATCH_SIZE = "sum_over_batch_size"
+
+  @classmethod
+  def all(cls):
+    return (cls.NONE, cls.SUM, cls.SUM_OVER_BATCH_SIZE)
+
+  @classmethod
+  def validate(cls, key):
+    if key not in cls.all():
+      raise ValueError("Invalid Reduction Key %s." % key)
+
+
 @tf_export(v1=["losses.Reduction"])
 class Reduction(object):
   """Types of loss reduction.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
index 031d9b171f..f20ed26e2e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-reduction.pbtxt
@@ -1,10 +1,10 @@
 path: "tensorflow.keras.losses.Reduction"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.utils.losses_utils.ReductionV2\'>"
+  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "NONE"
-    mtype: "<type \'NoneType\'>"
+    mtype: "<type \'str\'>"
   }
   member {
     name: "SUM"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
index ad72e3194a..6a44e4ce66 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.-reduction.pbtxt
@@ -1,10 +1,10 @@
 path: "tensorflow.losses.Reduction"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.utils.losses_utils.ReductionV2\'>"
+  is_instance: "<class \'tensorflow.python.ops.losses.losses_impl.ReductionV2\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "NONE"
-    mtype: "<type \'NoneType\'>"
+    mtype: "<type \'str\'>"
   }
   member {
     name: "SUM"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index f8c1f4111f..83ef73b8fa 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -437,6 +437,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.data.Iterator",
         "tf.nn.fused_batch_norm":
             "tf.compat.v1.nn.fused_batch_norm",
+        "tf.losses.Reduction.MEAN":
+            "tf.compat.v1.losses.Reduction.MEAN",
+        "tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS":
+            "tf.compat.v1.losses.Reduction.SUM_BY_NONZERO_WEIGHTS",
+        "tf.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS":
+            "tf.compat.v1.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS",
     }
     # pylint: enable=line-too-long
 
-- 
GitLab


From 6b61ad6ab2d0ce8c9e0c218f8d3571ce498e3fc8 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 29 Nov 2018 12:25:41 -0800
Subject: [PATCH 1044/1554] Add test cases for constant folding
 cast(const(foo)).

PiperOrigin-RevId: 223390529
---
 .../optimizers/constant_folding_test.cc       | 326 +++++++++++-------
 1 file changed, 208 insertions(+), 118 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index f6fdb32e98..192f48272f 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -14,10 +14,12 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
+#include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/array_ops_internal.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/grappler_test.h"
@@ -72,9 +74,9 @@ class ConstantFoldingTest : public GrapplerTest {
       GrapplerItem item;
       TF_CHECK_OK(s.ToGraphDef(&item.graph));
       item.fetch = {"mul1", "mul2", "add1", "add2"};
-      ConstantFolding optimizer(nullptr /* cpu_device */);
+      ConstantFolding optimizer(/*cpu_device=*/nullptr);
       GraphDef output;
-      Status status = optimizer.Optimize(nullptr, item, &output);
+      Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
       TF_EXPECT_OK(status);
 
       EXPECT_EQ(7, output.node_size());
@@ -132,9 +134,9 @@ TEST_F(ConstantFoldingTest, SimpleFolding) {
   item.fetch.push_back("d");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(1, output.node_size());
@@ -178,9 +180,9 @@ TEST_F(ConstantFoldingTest, AddTree) {
   item.fetch = {"add_parent", "mul_parent", "addmul_parent"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // We expect the following rewrite(s) to occur:
@@ -276,13 +278,11 @@ TEST_F(ConstantFoldingTest, ConvPushDownTest) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  std::cout << output.DebugString() << std::endl;
-
   EXPECT_EQ(5, output.node_size());
   int found = 0;
   for (const auto& node : output.node()) {
@@ -366,9 +366,9 @@ TEST_F(ConstantFoldingTest, NeutralElement) {
     TF_CHECK_OK(s.ToGraphDef(&item.graph));
     item.fetch = {"stack", "matmul3", "matmul4"};
 
-    ConstantFolding optimizer(nullptr /* cpu_device */);
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
     GraphDef output;
-    Status status = optimizer.Optimize(nullptr, item, &output);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
     TF_EXPECT_OK(status);
 
     const string suffix =
@@ -521,9 +521,9 @@ TEST_F(ConstantFoldingTest, StrengthReduce_Reciprocal) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   item.fetch = {"div_f", "div_i", "realdiv"};
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(8, output.node_size());
@@ -611,9 +611,9 @@ TEST_F(ConstantFoldingTest, NeutralElement_PartialShape_UnknownOutputShape) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(15, output.node_size());
@@ -683,9 +683,9 @@ TEST_F(ConstantFoldingTest, NeutralElement_PartialShape_KnownOutputShape) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(10, output.node_size());
@@ -741,9 +741,9 @@ TEST_F(ConstantFoldingTest, CreateConstNodes) {
 
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(24, output.node_size());
@@ -790,9 +790,9 @@ TEST_F(ConstantFoldingTest, FoldingNodeWithTwoOutputs) {
   item.fetch.push_back("f");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(2, output.node_size());
@@ -831,9 +831,9 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
   item.fetch.push_back("e");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> expected_nodes = {"dflt", "p1", "p2", "e"};
@@ -874,9 +874,9 @@ TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> expected_nodes = {"dflt", "p1", "p2", "c",
@@ -932,9 +932,9 @@ TEST_F(ConstantFoldingTest, ControlDependenciesDeduplicate) {
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
   auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
   EXPECT_EQ(1, tensors_expected.size());
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> expected_nodes = {"dflt", "p1", "p2", "i2"};
@@ -1009,9 +1009,9 @@ TEST_F(ConstantFoldingTest, VariableNumberOfOutputs) {
   }
 
   item.fetch = outputs;
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int constant_folded = 0;
@@ -1047,9 +1047,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterialization) {
   item.fetch.push_back("p2");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1097,9 +1097,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationEmptyFetch) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1163,9 +1163,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationShapeN) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   int found = 0;
   for (const auto& node : output.node()) {
@@ -1235,9 +1235,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationShapeN_MultipleOutputs) {
   item.fetch.push_back("ia");
   item.fetch.push_back("ib");
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1307,9 +1307,9 @@ TEST_F(ConstantFoldingTest, SwitchNodesEmptyFetch) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::set<string> present_nodes = {"v_in",     "v_ctrl",
@@ -1409,9 +1409,9 @@ TEST_F(ConstantFoldingTest, SwitchNodes) {
 
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   std::set<string> present_nodes = {"v_in",     "v_ctrl",
                                     "switch",   "i",
@@ -1505,9 +1505,9 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   item.fetch = {"out1", "idx1", "out2", "idx2", "out3", "idx3", "out4", "idx4"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(19, output.node_size());
@@ -1590,9 +1590,9 @@ TEST_F(ConstantFoldingTest, SplitRemoval) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -1636,9 +1636,9 @@ TEST_F(ConstantFoldingTest, SplitVRemoval) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -1686,9 +1686,9 @@ TEST_F(ConstantFoldingTest, TransposeOnSize1DimsRemoval) {
   item.fetch = {"out1"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -1723,9 +1723,9 @@ TEST_F(ConstantFoldingTest, RandomShuffleOnScalarRemoval) {
   item.fetch = {"out1", "out2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -1769,9 +1769,9 @@ TEST_F(ConstantFoldingTest, ReverseOnSize1DimsRemoval) {
   item.fetch = {"out1"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -1805,9 +1805,9 @@ TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) {
     item.fetch = {"out"};
     TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-    ConstantFolding optimizer(nullptr /* cpu_device */);
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
     GraphDef got;
-    Status status = optimizer.Optimize(nullptr, item, &got);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
     TF_EXPECT_OK(status);
 
     GraphDef want;
@@ -1852,9 +1852,9 @@ TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) {
     item.fetch = {"out"};
     TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-    ConstantFolding optimizer(nullptr /* cpu_device */);
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
     GraphDef got;
-    Status status = optimizer.Optimize(nullptr, item, &got);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
     TF_EXPECT_OK(status);
 
     GraphDef want;
@@ -1901,9 +1901,9 @@ TEST_F(ConstantFoldingTest, StridedSliceWithSameDimensionRemoval) {
     item.fetch = {"out"};
     TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-    ConstantFolding optimizer(nullptr /* cpu_device */);
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
     GraphDef got;
-    Status status = optimizer.Optimize(nullptr, item, &got);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
     TF_EXPECT_OK(status);
 
     GraphDef want;
@@ -1959,9 +1959,9 @@ TEST_F(ConstantFoldingTest, StridedSliceWithSameDimensionRemoval) {
     item.fetch = {"out"};
     TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-    ConstantFolding optimizer(nullptr /* cpu_device */);
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
     GraphDef got;
-    Status status = optimizer.Optimize(nullptr, item, &got);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
     TF_EXPECT_OK(status);
 
     GraphDef want;
@@ -2012,9 +2012,9 @@ TEST_F(ConstantFoldingTest, TileWithMultipliesBeingOne) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2045,9 +2045,9 @@ TEST_F(ConstantFoldingTest, MergeConcat) {
   item.fetch = {"c2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2075,9 +2075,9 @@ TEST_F(ConstantFoldingTest, MergeConcat_SameInput) {
   item.fetch = {"c2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2106,9 +2106,9 @@ TEST_F(ConstantFoldingTest, MergeConcat_ConcatWithConst) {
   item.fetch = {"c2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2137,9 +2137,9 @@ TEST_F(ConstantFoldingTest, MergeConcat_AxisMismatch) {
   item.fetch = {"c2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2175,9 +2175,9 @@ TEST_F(ConstantFoldingTest, PaddingWithZeroSize) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2221,9 +2221,9 @@ TEST_F(ConstantFoldingTest, SqueezeWithAllDimesionsGreaterThanOne) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef got;
-  Status status = optimizer.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
@@ -2269,9 +2269,9 @@ TEST_F(ConstantFoldingTest, NoOpReduction) {
   item.fetch = {"s", "p2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -2338,9 +2338,9 @@ TEST_F(ConstantFoldingTest, SingleElementEmptyAxisReduction) {
   item.fetch = {"mean_1", "mean_2", "mean_3", "mean_4", "mean_5"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Ensure Mean node is optimized to Reshape.
@@ -2433,9 +2433,9 @@ TEST_F(ConstantFoldingTest, NoOpReshape) {
   item.fetch = {"s1", "s2", "s3", "s4"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -2495,9 +2495,9 @@ TEST_F(ConstantFoldingTest, Packing) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   const std::vector<string> fetch_nodes = {"i1", "i2"};
@@ -2538,9 +2538,9 @@ TEST_F(ConstantFoldingTest, MaterializeBroadcastGradientArgs) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> fetch_nodes = {"o1", "o2", "p1", "p2"};
@@ -2552,7 +2552,7 @@ TEST_F(ConstantFoldingTest, MaterializeBroadcastGradientArgs) {
 
   // Run a second time to make sure the optimization is idempotent.
   item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -2619,14 +2619,14 @@ TEST_F(ConstantFoldingTest, MaterializeBroadcastGradientArgs_InfiniteLoop) {
   auto tensors_expected = EvaluateNodes(item.graph, fetch_nodes, {{"a", a_t}});
   EXPECT_EQ(fetch_nodes.size(), tensors_expected.size());
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Run a second time to make sure the optimization is idempotent.
   item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(11, output.node_size());
@@ -2711,14 +2711,14 @@ TEST_F(ConstantFoldingTest, MaterializeReductionIndices) {
     // Use aggressive mode to force the shape inference to propagate placeholder
     // shapes.
     ConstantFolding optimizer(RewriterConfig::AGGRESSIVE,
-                              nullptr /* cpu_device */);
+                              /*cpu_device=*/nullptr);
     GraphDef output;
-    Status status = optimizer.Optimize(nullptr, item, &output);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
     TF_EXPECT_OK(status);
 
     // Run a second time to make sure the optimization is idempotent.
     item.graph.Swap(&output);
-    status = optimizer.Optimize(nullptr, item, &output);
+    status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
     TF_EXPECT_OK(status);
 
     int found = 0;
@@ -2767,9 +2767,9 @@ TEST_F(ConstantFoldingTest, MaterializeReductionIndices_NotFullReduction) {
     // Use aggressive mode to force the shape inference to propagate placeholder
     // shapes.
     ConstantFolding optimizer(RewriterConfig::AGGRESSIVE,
-                              nullptr /* cpu_device */);
+                              /*cpu_device=*/nullptr);
     GraphDef output;
-    Status status = optimizer.Optimize(nullptr, item, &output);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
     TF_EXPECT_OK(status);
 
     CompareGraphs(item.graph, output);
@@ -2788,9 +2788,9 @@ TEST_F(ConstantFoldingTest, LargeConstant) {
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
   item.fetch.push_back("out");
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Make sure the diag node hasn't been folded, since it would use too much
@@ -2833,9 +2833,9 @@ TEST_F(ConstantFoldingTest, SwitchIdenticalInputs) {
   item.fetch.push_back("id_true");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(6, output.node_size());
@@ -2925,9 +2925,9 @@ TEST_F(ConstantFoldingTest, PartialFolding_AssociativeAndCommutative) {
     TF_CHECK_OK(s.ToGraphDef(&item.graph));
     item.fetch = {"stack"};
 
-    ConstantFolding optimizer(nullptr /* cpu_device */);
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
     GraphDef output;
-    Status status = optimizer.Optimize(nullptr, item, &output);
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
     TF_EXPECT_OK(status);
 
     EXPECT_EQ(16, output.node_size());
@@ -3017,13 +3017,13 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
 
   auto tensors_expected = EvaluateNodes(item.graph, {"concat0"});
   EXPECT_EQ(1, tensors_expected.size());
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   // Run the optimizer twice to make sure the rewrite is idempotent.
   item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(21, output.node_size());
@@ -3090,9 +3090,9 @@ TEST_F(ConstantFoldingTest, PartialFolding_IdentityN) {
   item.fetch.push_back("add0");
   item.fetch.push_back("add1");
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   EXPECT_EQ(8, output.node_size());
   for (const auto& node : output.node()) {
@@ -3152,9 +3152,9 @@ TEST_F(ConstantFoldingTest, TrivialPack) {
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
   item.fetch = {"stack", "stack_no_axis"};
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   EXPECT_EQ(7, output.node_size());
   int found = 0;
@@ -3234,13 +3234,13 @@ TEST_F(ConstantFoldingTest, Enter) {
   item.fetch.push_back("id3");
   item.fetch.push_back("id4");
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   // Run the optimizer twice to make sure the rewrite is idempotent.
   item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(9, output.node_size());
@@ -3289,13 +3289,13 @@ TEST_F(ConstantFoldingTest, TensorArraySize) {
   auto tensors_expected =
       EvaluateNodes(item.graph, {"dynamic_sz", "static_sz"});
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
   // Run the optimizer twice to make sure the rewrite is idempotent.
   item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(8, output.node_size());
@@ -3327,9 +3327,9 @@ TEST_F(ConstantFoldingTest, FoldingPreservesDenormalFlushing) {
   item.fetch.push_back("c");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(1, output.node_size());
@@ -3363,9 +3363,9 @@ TEST_F(ConstantFoldingTest, EvaluatingLargeConstantNoFoldingMergingLoop) {
   item.fetch.push_back("result");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(/*cpu_device=*/nullptr);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> fetch = {"result"};
@@ -3376,6 +3376,96 @@ TEST_F(ConstantFoldingTest, EvaluatingLargeConstantNoFoldingMergingLoop) {
   EXPECT_EQ(tensors_expected[0].shape(), tensors[0].shape());
 }
 
+class ConstantFoldingCastConstTest : public GrapplerTest {
+ protected:
+  void ConstantFoldingCastConst(bool fetch_const, bool fetch_cast,
+                                bool fetch_const_child, bool fetch_cast_child) {
+    if (!fetch_const && !fetch_cast && !fetch_const_child &&
+        !fetch_cast_child) {
+      return;
+    }
+
+    tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+    CreateCastConstGraph(s);
+    GrapplerItem item;
+    int expected_output_size = SetFetch(&item, fetch_const, fetch_cast,
+                                        fetch_const_child, fetch_cast_child);
+    TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+    GraphDef output = ConstantFoldingOptimize(item);
+    EXPECT_EQ(expected_output_size, output.node_size());
+
+    EvaluateAndCompareUnoptimized(item.graph, output, item.fetch);
+  }
+
+ private:
+  void CreateCastConstGraph(const tensorflow::Scope& s) {
+    Output const1 = ops::Const(s.WithOpName("const1"), 2, {5, 5});
+    Output cast = ops::Cast(s.WithOpName("cast"), const1, DT_FLOAT);
+    Output const1_child = ops::Identity(s.WithOpName("const1_child"), const1);
+    Output cast_child = ops::Identity(s.WithOpName("cast_child"), cast);
+  }
+
+  int SetFetch(GrapplerItem* item, bool fetch_const, bool fetch_cast,
+               bool fetch_const_child, bool fetch_cast_child) {
+    int expected_output_size = 0;
+    if (fetch_const) {
+      item->fetch.push_back("const1");
+      expected_output_size++;
+    }
+    if (fetch_cast) {
+      item->fetch.push_back("cast");
+      expected_output_size++;
+    }
+    if (fetch_const_child) {
+      item->fetch.push_back("const1_child");
+      expected_output_size++;
+    }
+    if (fetch_cast_child) {
+      item->fetch.push_back("cast_child");
+      expected_output_size++;
+    }
+    return expected_output_size;
+  }
+
+  GraphDef ConstantFoldingOptimize(const GrapplerItem& item) {
+    ConstantFolding optimizer(/*cpu_device=*/nullptr);
+    GraphDef output;
+    Status status = optimizer.Optimize(/*cluster=*/nullptr, item, &output);
+    TF_EXPECT_OK(status);
+    return output;
+  }
+
+  void EvaluateAndCompareUnoptimized(const GraphDef& unoptimized_graph,
+                                     const GraphDef& optimized_graph,
+                                     const std::vector<string>& fetch_nodes) {
+    auto tensors_expected = EvaluateNodes(unoptimized_graph, fetch_nodes);
+    auto tensors = EvaluateNodes(optimized_graph, fetch_nodes);
+    ASSERT_EQ(fetch_nodes.size(), tensors_expected.size());
+    ASSERT_EQ(fetch_nodes.size(), tensors.size());
+    for (int i = 0; i < fetch_nodes.size(); i++) {
+      if (fetch_nodes[i] == "const1" || fetch_nodes[i] == "const1_child") {
+        test::ExpectTensorEqual<int>(tensors_expected[i], tensors[i]);
+      } else {
+        test::ExpectTensorEqual<float>(tensors_expected[i], tensors[i]);
+      }
+    }
+  }
+};
+
+TEST_F(ConstantFoldingCastConstTest, CastConstFolding) {
+  for (bool fetch_const : {false, true}) {
+    for (bool fetch_cast : {false, true}) {
+      for (bool fetch_const_child : {false, true}) {
+        for (bool fetch_cast_child : {false, true}) {
+          ConstantFoldingCastConst(fetch_const, fetch_cast, fetch_const_child,
+                                   fetch_cast_child);
+        }
+      }
+    }
+  }
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From d1198909057c635de2bae3e1c4fb8505466ee325 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Thu, 29 Nov 2018 12:39:54 -0800
Subject: [PATCH 1045/1554] Naming for valid scope name in eager mode.

PiperOrigin-RevId: 223392695
---
 tensorflow/python/ops/data_flow_ops.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py
index bb08dbaea1..2030332e4e 100644
--- a/tensorflow/python/ops/data_flow_ops.py
+++ b/tensorflow/python/ops/data_flow_ops.py
@@ -171,7 +171,10 @@ class QueueBase(object):
       self._names = None
     self._queue_ref = queue_ref
     if context.executing_eagerly():
-      self._name = context.context().scope_name
+      if context.context().scope_name:
+        self._name = context.context().scope_name
+      else:
+        self._name = "Empty"
       self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
           queue_ref, None)
     else:
-- 
GitLab


From c53aa864f9e2f2be89892feeab19877f6107bb9e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 12:56:25 -0800
Subject: [PATCH 1046/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 223395339
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 47 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 47 +++++++++++++++++++
 2 files changed, 94 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index ba0bf553d0..414964ad3f 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77282,6 +77282,53 @@ op {
     }
   }
 }
+op {
+  name: "UnicodeEncode"
+  input_arg {
+    name: "input_values"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_splits"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output"
+    type: DT_STRING
+  }
+  attr {
+    name: "errors"
+    type: "string"
+    default_value {
+      s: "replace"
+    }
+    allowed_values {
+      list {
+        s: "ignore"
+        s: "replace"
+        s: "strict"
+      }
+    }
+  }
+  attr {
+    name: "output_encoding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "UTF-8"
+        s: "UTF-16-BE"
+        s: "UTF-32-BE"
+      }
+    }
+  }
+  attr {
+    name: "replacement_char"
+    type: "int"
+    default_value {
+      i: 65533
+    }
+  }
+}
 op {
   name: "UnicodeScript"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index bae50a7139..eae87da6b5 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -37163,6 +37163,53 @@ op {
     }
   }
 }
+op {
+  name: "UnicodeEncode"
+  input_arg {
+    name: "input_values"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_splits"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output"
+    type: DT_STRING
+  }
+  attr {
+    name: "errors"
+    type: "string"
+    default_value {
+      s: "replace"
+    }
+    allowed_values {
+      list {
+        s: "ignore"
+        s: "replace"
+        s: "strict"
+      }
+    }
+  }
+  attr {
+    name: "output_encoding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "UTF-8"
+        s: "UTF-16-BE"
+        s: "UTF-32-BE"
+      }
+    }
+  }
+  attr {
+    name: "replacement_char"
+    type: "int"
+    default_value {
+      i: 65533
+    }
+  }
+}
 op {
   name: "UnicodeScript"
   input_arg {
-- 
GitLab


From 06025ed0f91cc1c75b391f31099b6975904382a1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 13:00:01 -0800
Subject: [PATCH 1047/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 223395920

---
 tensorflow/go/op/wrappers.go | 208 +++++++++++++++++------------------
 1 file changed, 104 insertions(+), 104 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 02a1335149..406382977b 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5716,6 +5716,77 @@ func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.Data
 	return values
 }
 
+// MapPeekAttr is an optional argument to MapPeek.
+type MapPeekAttr func(optionalAttr)
+
+// MapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekCapacity(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekMemoryLimit(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapPeekContainer(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapPeekSharedName(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op peeks at the values at the specified key.  If the
+//
+// underlying container does not contain this key
+// this op will block until it does.
+func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapPeek",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapPeek", err)
+		return
+	}
+	return values
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -6539,21 +6610,6 @@ func Sin(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Computes the complementary error function of `x` element-wise.
-func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Erfc",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes Psi, the derivative of Lgamma (the log of the absolute value of
 //
 // `Gamma(x)`), element-wise.
@@ -13842,6 +13898,39 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 	return op.Output(0)
 }
 
+// Computes the complementary error function of `x` element-wise.
+func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Erfc",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the number of tensors in the input tensor list.
+//
+// input_handle: the input list
+// length: the number of tensors in the list
+func TensorListLength(scope *Scope, input_handle tf.Output) (length tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListLength",
+		Input: []tf.Input{
+			input_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Determine the script codes of a given tensor of Unicode integer code points.
 //
 // This operation converts Unicode code points to script codes corresponding to
@@ -25573,77 +25662,6 @@ func Roll(scope *Scope, input tf.Output, shift tf.Output, axis tf.Output) (outpu
 	return op.Output(0)
 }
 
-// MapPeekAttr is an optional argument to MapPeek.
-type MapPeekAttr func(optionalAttr)
-
-// MapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekCapacity(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekMemoryLimit(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapPeekContainer(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapPeekSharedName(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op peeks at the values at the specified key.  If the
-//
-// underlying container does not contain this key
-// this op will block until it does.
-func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapPeek",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapPeek", err)
-		return
-	}
-	return values
-}
-
 // Looks up keys in a table, outputs the corresponding values.
 //
 // The tensor `keys` must of the same type as the keys of the table.
@@ -26838,24 +26856,6 @@ func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) {
 	return op.Output(0)
 }
 
-// Returns the number of tensors in the input tensor list.
-//
-// input_handle: the input list
-// length: the number of tensors in the list
-func TensorListLength(scope *Scope, input_handle tf.Output) (length tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorListLength",
-		Input: []tf.Input{
-			input_handle,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // The shape of the elements of the given list, as a tensor.
 //
 //   input_handle: the list
-- 
GitLab


From 886fab08090cb9048e5cb6169b1e8ab260f650a1 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 29 Nov 2018 13:11:43 -0800
Subject: [PATCH 1048/1554] [XLA] Update doc for collective permute.

PiperOrigin-RevId: 223398050
---
 .../compiler/xla/g3doc/operation_semantics.md | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index df74a9b3af..e0807518bc 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -418,6 +418,33 @@ then v12 == f32[8x3] {{10, 11, 12},
 
 ```
 
+## CollectivePermute
+
+See also
+[`XlaBuilder::CollectivePermute`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/xla_builder.h).
+
+CollectivePermute is a collective operation that sends and receives data cross
+replicas.
+
+<b> `CollectivePermute(operand, source_target_pairs)` </b>
+
+| Arguments             | Type                    | Semantics                  |
+| --------------------- | ----------------------- | -------------------------- |
+| `operand`             | `XlaOp`                 | n dimensional input array  |
+| `source_target_pairs` | `<int64, int64>` vector | A list of                  |
+:                       :                         : (source_replica_id,        :
+:                       :                         : target_replica_id) pairs.  :
+:                       :                         : For each pair, the operand :
+:                       :                         : is sent from source        :
+:                       :                         : replica to target replica. :
+
+Note that there are the following restrictions on the `source_target_pair`:
+
+-   Any two pairs should not have the same target replica id, and they should
+    not have the same source replica id.
+-   If a replica id is not a target in any pair, then the output on that replica
+    is a tensor consists of 0(s) with the same shape as the input.
+
 ## Concatenate
 
 See also
-- 
GitLab


From 23ddb5c69a6c98e8654b6114b1aa33606460638a Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 29 Nov 2018 13:16:22 -0800
Subject: [PATCH 1049/1554] Move to do into comment.

PiperOrigin-RevId: 223398784
---
 tensorflow/lite/python/op_hint.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 718b23075d..8d7f9316bf 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -104,9 +104,9 @@ class OpHint(object):
   that make up the pseudo op. A similar process is done to any output that
   is to be exported from the current op.
 
-  TODO(aselle): When TensorFlow functions functionality works for arbitrary
-  constructs, this mechanism can be retired and changed to use python defun's.
   """
+  # TODO(aselle): When TensorFlow functions functionality works for arbitrary
+  # constructs, this mechanism can be retired and changed to use python defun's.
 
   # Attr constants that are used for representation in the GraphDef. These
   # will be used on every Identity op that is involved in a total OpHint.
-- 
GitLab


From a1f87073027572737b275401202726f736350206 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Thu, 29 Nov 2018 13:19:39 -0800
Subject: [PATCH 1050/1554] Extracted ~logically independent bits from
 `_VarStore._get_partitioned_variables`

PiperOrigin-RevId: 223399274
---
 tensorflow/python/ops/variable_scope.py | 142 ++++++++++++------------
 1 file changed, 74 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index ad81862a58..ccce9e2f93 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -648,8 +648,6 @@ class _VariableStore(object):
     """
     initializing_from_value = initializer is not None and isinstance(
         initializer, ops.Tensor)
-    reuse_without_partition = reuse and not partitioner
-
     if name in self._vars:
       raise ValueError(
           "A partitioner was provided, but an unpartitioned version of the "
@@ -660,30 +658,9 @@ class _VariableStore(object):
     if initializing_from_value:
       shape = shape.merge_with(initializer.get_shape())
 
-    if not reuse_without_partition:
-      if not shape.is_fully_defined():
-        raise ValueError("Shape of a new partitioned variable (%s) must be "
-                         "fully defined, but instead was %s." % (name, shape))
-
-      if shape.ndims < 1:
-        raise ValueError("A partitioned Variable must have rank at least 1, "
-                         "shape: %s" % shape)
-
-      partitions = partitioner(shape=shape, dtype=dtype)
-
-      if not isinstance(partitions, collections_lib.Sequence):
-        raise ValueError("Partitioner must return a sequence, but saw: %s"
-                         % partitions)
-
-      if len(partitions) != shape.ndims:
-        raise ValueError(
-            "Partitioner returned a partition list that does not match the "
-            "Variable's rank: %s vs. %s" % (partitions, shape))
-
-      if any(p < 1 for p in partitions):
-        raise ValueError(
-            "Partitioner returned zero partitions for some axes: %s" %
-            partitions)
+    partitions = None
+    if not reuse or partitioner:
+      partitions = _call_partitioner(partitioner, shape, dtype)
 
     if name in self._partitioned_vars:
       if reuse is False:
@@ -705,7 +682,7 @@ class _VariableStore(object):
             % (name, dtype.name, existing_var.dtype.name))
 
       # pylint: disable=protected-access
-      if (not reuse_without_partition and
+      if (partitions is not None and
           existing_var._get_partitions() != partitions):
         raise ValueError(
             "Trying to reuse partitioned variable %s, but specified partitions "
@@ -720,14 +697,7 @@ class _VariableStore(object):
                        "created with tf.get_variable(). Did you mean to set "
                        "reuse=False or reuse=tf.AUTO_REUSE in VarScope?" % name)
 
-    slice_dim, slice_shape = _compute_slice_dim_and_shape(
-        shape.as_list(), partitions)
-
-    vs = []
-    num_slices = partitions[slice_dim]
-    num_slices_with_excess = shape.dims[slice_dim].value % num_slices
-
-    slice_offset = [0] * shape.ndims
+    slice_dim, num_slices = _get_slice_dim_and_num_slices(partitions)
 
     if "%s/part_0" % name in self._vars:
       if "%s/part_%d" % (name, num_slices - 1) not in self._vars:
@@ -743,15 +713,14 @@ class _VariableStore(object):
             "%s/part_0 was found, but so was the extra shard %s/part_%d."
             % (num_slices, name, name, num_slices))
 
-    for i in xrange(num_slices):
-      var_shape = slice_shape[:]
-      var_offset = slice_offset[:]
+    vs = []
+    for i, (var_offset, var_shape) in enumerate(_iter_slices(
+        shape.as_list(),
+        num_slices,
+        slice_dim
+    )):
       partition_info = _PartitionInfo(
           full_shape=shape.as_list(), var_offset=var_offset)
-      if i < num_slices_with_excess:
-        var_shape[slice_dim] += 1
-      slice_offset[slice_dim] += var_shape[slice_dim]
-
       var_full_name = "%s/part_%d" % (name, i)
       with ops.name_scope(var_full_name + "/PartitionedInitializer"):
         # Create the tensor to initialize the variable with default value.
@@ -2400,34 +2369,71 @@ def variable_op_scope(values,
     yield scope
 
 
-def _compute_slice_dim_and_shape(full_shape, slicing):
-  """Computes which dimension is being sliced and the typical slice shape."""
-
-  slice_shape = [0] * len(full_shape)
-  slice_dim = None
-  for dim, num_slices in enumerate(slicing):
-    dim_size = full_shape[dim]
-    if num_slices <= 0 or dim_size < num_slices:
-      raise ValueError("Cannot create %d slices for size %d. shape: %s, "
-                       "slicing: %s" %
-                       (num_slices, full_shape[dim], full_shape, slicing))
-    if num_slices == 1:
-      # Not slicing in this dimension.
-      slice_shape[dim] = dim_size
-    elif slice_dim is not None:
-      # We only support slicing along one of the dimensions.
-      raise ValueError("Can only slice a variable along one dimension: "
-                       "shape: %s, slicing: %s" % (full_shape, slicing))
-    else:
-      # Note: We will add any extras onto the last slice, later.
-      slice_dim = dim
-      slice_shape[dim] = dim_size // num_slices
+def _call_partitioner(partitioner, shape, dtype):
+  """Call partitioner validating its inputs/output.
 
-  # Degenerate case: If "slicing" was all ones, pretend we are slicing along
-  # the first dimension.
-  if slice_dim is None:
+  Args:
+    partitioner: a function mapping `Tensor` shape and dtype to a
+        list of partitions.
+    shape: shape of the `Tensor` to partition, must have at least two
+        dimensions.
+    dtype: dtype of the elements in the `Tensor`.
+
+  Returns:
+    A list with elements >=1 and exactly one >1. The index of that
+    element corresponds to the partitioning axis.
+  """
+  if not shape.is_fully_defined():
+    raise ValueError("Shape of a new partitioned variable must be "
+                     "fully defined, but instead was %s." % (shape,))
+  if shape.ndims < 1:
+    raise ValueError("A partitioned Variable must have rank at least 1, "
+                     "shape: %s" % shape)
+
+  slicing = partitioner(shape=shape, dtype=dtype)
+  if not isinstance(slicing, collections_lib.Sequence):
+    raise ValueError("Partitioner must return a sequence, but saw: %s"
+                     % slicing)
+  if len(slicing) != shape.ndims:
+    raise ValueError(
+        "Partitioner returned a partition list that does not match the "
+        "Variable's rank: %s vs. %s" % (slicing, shape))
+  if any(p < 1 for p in slicing):
+    raise ValueError(
+        "Partitioner returned zero partitions for some axes: %s" %
+        slicing)
+  if sum(p > 1 for p in slicing) > 1:
+    raise ValueError(
+        "Can only slice a variable along one dimension: "
+        "shape: %s, partitioning: %s" % (shape, slicing))
+  return slicing
+
+
+# TODO(slebedev): could be inlined, but
+# `_VariableStore._get_partitioned_variable` is too complex even
+# without this logic.
+def _get_slice_dim_and_num_slices(slicing):
+  """Get slicing dimension and number of slices from the partitioner output."""
+  for slice_dim, num_slices in enumerate(slicing):
+    if num_slices > 1:
+      break
+  else:
+    # Degenerate case: no partitioning applied.
     slice_dim = 0
-  return slice_dim, slice_shape
+    num_slices = 1
+  return slice_dim, num_slices
+
+
+def _iter_slices(full_shape, num_slices, slice_dim):
+  """Slices a given a shape along the specified dimension."""
+  num_slices_with_excess = full_shape[slice_dim] % num_slices
+  offset = [0] * len(full_shape)
+  min_slice_len = full_shape[slice_dim] // num_slices
+  for i in xrange(num_slices):
+    shape = full_shape[:]
+    shape[slice_dim] = min_slice_len + bool(i < num_slices_with_excess)
+    yield offset[:], shape
+    offset[slice_dim] += shape[slice_dim]
 
 
 def _get_trainable_value(synchronization, trainable):
-- 
GitLab


From 825621b3ca15c1b6b96195544e3259d2fa72e5de Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Thu, 29 Nov 2018 13:20:43 -0800
Subject: [PATCH 1051/1554] Make sure Keras sequential clone doesn't create
 unnecessary placeholders as well.

PiperOrigin-RevId: 223399456
---
 tensorflow/python/keras/models.py      | 9 ++++++---
 tensorflow/python/keras/models_test.py | 3 ++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 4813b8061e..2637191bb7 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -211,14 +211,17 @@ def _clone_sequential_model(model, input_tensors=None):
   # Use model._layers to ensure that all layers are cloned. The model's layers
   # property will exclude the initial InputLayer (if it exists) in the model,
   # resulting in a different Sequential model structure.
-  layers = [clone(layer) for layer in model._layers]
   if input_tensors is None:
+    layers = [clone(layer) for layer in model._layers]
     return Sequential(layers=layers, name=model.name)
   else:
     # If input tensors are provided, the original model's InputLayer is
     # overwritten with a different InputLayer.
-    if isinstance(layers[0], InputLayer):
-      layers = layers[1:]
+    layers = [
+        clone(layer)
+        for layer in model._layers
+        if not isinstance(layer, InputLayer)
+    ]
     if len(generic_utils.to_list(input_tensors)) != 1:
       raise ValueError('To clone a `Sequential` model, we expect '
                        ' at most one tensor '
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 23321a2d16..c68c80f0cc 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -235,7 +235,8 @@ class TestModelCloning(test.TestCase):
 
   def test_sequential_cloning_does_not_create_unnecessary_placeholders(self):
     with ops.Graph().as_default():
-      model = keras.models.Sequential([keras.layers.Dense(4)])
+      model = keras.models.Sequential()
+      model.add(keras.layers.Dense(4, input_shape=(4,)))
     graph = ops.Graph()
     with graph.as_default():
       x = array_ops.ones((10, 4))
-- 
GitLab


From fd7b50ee620ab2e363c1d473c166225d103dddfb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 13:29:05 -0800
Subject: [PATCH 1052/1554] Automated rollback of commit
 5f111d5a6c78f7afcd3ccbabf6e272800caceb9d

PiperOrigin-RevId: 223400918
---
 .../keras/layers/cudnn_recurrent_test.py      | 686 +++++++++---------
 1 file changed, 334 insertions(+), 352 deletions(-)

diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index 1f195f3119..cc93364aae 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -31,76 +31,64 @@ from tensorflow.python.platform import test
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
-@test_util.run_all_in_graph_and_eager_modes
 class CuDNNTest(test.TestCase, parameterized.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes
   def test_cudnn_rnn_basics(self):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      timesteps = 6
-      units = 2
-      num_samples = 32
-      for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
-        for return_sequences in [True, False]:
-          with keras.utils.CustomObjectScope({
-              'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
-              'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM
-          }):
-            testing_utils.layer_test(
-                layer_class,
-                kwargs={
-                    'units': units,
-                    'return_sequences': return_sequences
-                },
-                input_shape=(num_samples, timesteps, input_size))
-        for go_backwards in [True, False]:
-          with keras.utils.CustomObjectScope({
-              'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
-              'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM
-          }):
-            testing_utils.layer_test(
-                layer_class,
-                kwargs={
-                    'units': units,
-                    'go_backwards': go_backwards
-                },
-                input_shape=(num_samples, timesteps, input_size))
-
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        timesteps = 6
+        units = 2
+        num_samples = 32
+        for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
+          for return_sequences in [True, False]:
+            with keras.utils.CustomObjectScope(
+                {'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
+                 'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM}):
+              testing_utils.layer_test(
+                  layer_class,
+                  kwargs={'units': units,
+                          'return_sequences': return_sequences},
+                  input_shape=(num_samples, timesteps, input_size))
+          for go_backwards in [True, False]:
+            with keras.utils.CustomObjectScope(
+                {'keras.layers.CuDNNGRU': keras.layers.CuDNNGRU,
+                 'keras.layers.CuDNNLSTM': keras.layers.CuDNNLSTM}):
+              testing_utils.layer_test(
+                  layer_class,
+                  kwargs={'units': units,
+                          'go_backwards': go_backwards},
+                  input_shape=(num_samples, timesteps, input_size))
+
+  @test_util.run_in_graph_and_eager_modes
   def test_trainability(self):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      units = 2
-      for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
-        layer = layer_class(units)
-        layer.build((None, None, input_size))
-        self.assertEqual(len(layer.weights), 3)
-        self.assertEqual(len(layer.trainable_weights), 3)
-        self.assertEqual(len(layer.non_trainable_weights), 0)
-        layer.trainable = False
-        self.assertEqual(len(layer.weights), 3)
-        self.assertEqual(len(layer.non_trainable_weights), 3)
-        self.assertEqual(len(layer.trainable_weights), 0)
-        layer.trainable = True
-        self.assertEqual(len(layer.weights), 3)
-        self.assertEqual(len(layer.trainable_weights), 3)
-        self.assertEqual(len(layer.non_trainable_weights), 0)
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        units = 2
+        for layer_class in [keras.layers.CuDNNGRU, keras.layers.CuDNNLSTM]:
+          layer = layer_class(units)
+          layer.build((None, None, input_size))
+          self.assertEqual(len(layer.weights), 3)
+          self.assertEqual(len(layer.trainable_weights), 3)
+          self.assertEqual(len(layer.non_trainable_weights), 0)
+          layer.trainable = False
+          self.assertEqual(len(layer.weights), 3)
+          self.assertEqual(len(layer.non_trainable_weights), 3)
+          self.assertEqual(len(layer.trainable_weights), 0)
+          layer.trainable = True
+          self.assertEqual(len(layer.weights), 3)
+          self.assertEqual(len(layer.trainable_weights), 3)
+          self.assertEqual(len(layer.non_trainable_weights), 0)
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_regularizer(self, layer_class):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
     if test.is_gpu_available(cuda_only=True):
-      with test_util.use_gpu():
+      with self.session(use_gpu=True):
         input_size = 10
         timesteps = 6
         units = 2
@@ -131,140 +119,132 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_return_state(self, layer_class):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      timesteps = 6
-      units = 2
-      num_samples = 32
-      num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
-
-      inputs = keras.Input(batch_shape=(num_samples, timesteps, input_size))
-      layer = layer_class(units, return_state=True, stateful=True)
-      outputs = layer(inputs)
-      _, state = outputs[0], outputs[1:]
-      self.assertEqual(len(state), num_states)
-      model = keras.models.Model(inputs, state[0])
-
-      inputs = np.random.random((num_samples, timesteps, input_size))
-      state = model.predict(inputs)
-      np.testing.assert_allclose(
-          keras.backend.eval(layer.states[0]), state, atol=1e-4)
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        timesteps = 6
+        units = 2
+        num_samples = 32
+        num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
+
+        inputs = keras.Input(batch_shape=(num_samples, timesteps, input_size))
+        layer = layer_class(units, return_state=True, stateful=True)
+        outputs = layer(inputs)
+        _, state = outputs[0], outputs[1:]
+        self.assertEqual(len(state), num_states)
+        model = keras.models.Model(inputs, state[0])
+
+        inputs = np.random.random((num_samples, timesteps, input_size))
+        state = model.predict(inputs)
+        np.testing.assert_allclose(
+            keras.backend.eval(layer.states[0]), state, atol=1e-4)
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_time_major_input(self, layer_class):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      timesteps = 6
-      units = 2
-      num_samples = 32
-
-      model = keras.models.Sequential()
-      model.add(
-          keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
-      layer = layer_class(units, time_major=True, return_sequences=True)
-      model.add(layer)
-      model.add(
-          keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
-      model.compile(loss='categorical_crossentropy', optimizer='adam')
-      model.fit(
-          np.ones((num_samples, timesteps, input_size)),
-          np.ones((num_samples, timesteps, units)))
-      out = model.predict(np.ones((num_samples, timesteps, input_size)))
-      self.assertEqual(out.shape, (num_samples, timesteps, units))
+    if test.is_gpu_available(cuda_only=True):
+      with self.test_session(use_gpu=True):
+        input_size = 10
+        timesteps = 6
+        units = 2
+        num_samples = 32
+
+        model = keras.models.Sequential()
+        model.add(
+            keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
+        layer = layer_class(units, time_major=True, return_sequences=True)
+        model.add(layer)
+        model.add(
+            keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2])))
+        model.compile(loss='categorical_crossentropy', optimizer='adam')
+        model.fit(
+            np.ones((num_samples, timesteps, input_size)),
+            np.ones((num_samples, timesteps, units)))
+        out = model.predict(np.ones((num_samples, timesteps, input_size)))
+        self.assertEqual(out.shape, (num_samples, timesteps, units))
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_specify_initial_state_keras_tensor(self, layer_class):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      timesteps = 6
-      units = 2
-      num_samples = 32
-      num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
-
-      inputs = keras.Input((timesteps, input_size))
-      initial_state = [keras.Input((units,)) for _ in range(num_states)]
-      layer = layer_class(units)
-      if len(initial_state) == 1:
-        output = layer(inputs, initial_state=initial_state[0])
-      else:
-        output = layer(inputs, initial_state=initial_state)
-      self.assertIn(initial_state[0], layer._inbound_nodes[0].input_tensors)
-
-      model = keras.models.Model([inputs] + initial_state, output)
-      model.compile(loss='categorical_crossentropy', optimizer='adam')
-
-      inputs = np.random.random((num_samples, timesteps, input_size))
-      initial_state = [
-          np.random.random((num_samples, units)) for _ in range(num_states)
-      ]
-      targets = np.random.random((num_samples, units))
-      model.fit([inputs] + initial_state, targets)
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        timesteps = 6
+        units = 2
+        num_samples = 32
+        num_states = 2 if layer_class is keras.layers.CuDNNLSTM else 1
+
+        inputs = keras.Input((timesteps, input_size))
+        initial_state = [keras.Input((units,)) for _ in range(num_states)]
+        layer = layer_class(units)
+        if len(initial_state) == 1:
+          output = layer(inputs, initial_state=initial_state[0])
+        else:
+          output = layer(inputs, initial_state=initial_state)
+        self.assertIn(initial_state[0], layer._inbound_nodes[0].input_tensors)
+
+        model = keras.models.Model([inputs] + initial_state, output)
+        model.compile(loss='categorical_crossentropy', optimizer='adam')
+
+        inputs = np.random.random((num_samples, timesteps, input_size))
+        initial_state = [
+            np.random.random((num_samples, units)) for _ in range(num_states)
+        ]
+        targets = np.random.random((num_samples, units))
+        model.fit([inputs] + initial_state, targets)
 
   @parameterized.named_parameters(
       ('cudnngru', keras.layers.CuDNNGRU),
       ('cudnnlstm', keras.layers.CuDNNLSTM),
   )
   def test_statefulness(self, layer_class):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      timesteps = 6
-      units = 2
-      num_samples = 32
-
-      model = keras.models.Sequential()
-      model.add(
-          keras.layers.Embedding(
-              10,
-              input_size,
-              input_length=timesteps,
-              batch_input_shape=(num_samples, timesteps)))
-      layer = layer_class(
-          units, return_sequences=False, stateful=True, weights=None)
-      model.add(layer)
-      model.compile(optimizer='sgd', loss='mse')
-      out1 = model.predict(np.ones((num_samples, timesteps)))
-      self.assertEqual(out1.shape, (num_samples, units))
-
-      # train once so that the states change
-      model.train_on_batch(
-          np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
-      out2 = model.predict(np.ones((num_samples, timesteps)))
-
-      # if the state is not reset, output should be different
-      self.assertNotEqual(out1.max(), out2.max())
-
-      # check that output changes after states are reset
-      # (even though the model itself didn't change)
-      layer.reset_states()
-      out3 = model.predict(np.ones((num_samples, timesteps)))
-      self.assertNotEqual(out2.max(), out3.max())
-
-      # check that container-level reset_states() works
-      model.reset_states()
-      out4 = model.predict(np.ones((num_samples, timesteps)))
-      self.assertAllClose(out3, out4, atol=1e-5)
-
-      # check that the call to `predict` updated the states
-      out5 = model.predict(np.ones((num_samples, timesteps)))
-      self.assertNotEqual(out4.max(), out5.max())
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        timesteps = 6
+        units = 2
+        num_samples = 32
+
+        model = keras.models.Sequential()
+        model.add(
+            keras.layers.Embedding(
+                10,
+                input_size,
+                input_length=timesteps,
+                batch_input_shape=(num_samples, timesteps)))
+        layer = layer_class(
+            units, return_sequences=False, stateful=True, weights=None)
+        model.add(layer)
+        model.compile(optimizer='sgd', loss='mse')
+        out1 = model.predict(np.ones((num_samples, timesteps)))
+        self.assertEqual(out1.shape, (num_samples, units))
+
+        # train once so that the states change
+        model.train_on_batch(
+            np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
+        out2 = model.predict(np.ones((num_samples, timesteps)))
+
+        # if the state is not reset, output should be different
+        self.assertNotEqual(out1.max(), out2.max())
+
+        # check that output changes after states are reset
+        # (even though the model itself didn't change)
+        layer.reset_states()
+        out3 = model.predict(np.ones((num_samples, timesteps)))
+        self.assertNotEqual(out2.max(), out3.max())
+
+        # check that container-level reset_states() works
+        model.reset_states()
+        out4 = model.predict(np.ones((num_samples, timesteps)))
+        self.assertAllClose(out3, out4, atol=1e-5)
+
+        # check that the call to `predict` updated the states
+        out5 = model.predict(np.ones((num_samples, timesteps)))
+        self.assertNotEqual(out4.max(), out5.max())
 
   @parameterized.named_parameters(
       *test_util.generate_combinations_with_testcase_name(
@@ -274,51 +254,49 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
   def test_load_weights_between_noncudnn_rnn(self, rnn_type, to_cudnn,
                                              bidirectional, implementation,
                                              model_nest_level, model_type):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      timesteps = 6
-      input_shape = (timesteps, input_size)
-      units = 2
-      num_samples = 32
-      inputs = np.random.random((num_samples, timesteps, input_size))
-
-      rnn_layer_kwargs = {
-          'recurrent_activation': 'sigmoid',
-          # ensure biases are non-zero and properly converted
-          'bias_initializer': 'random_uniform',
-          'implementation': implementation
-      }
-      if rnn_type == 'LSTM':
-        rnn_layer_class = keras.layers.LSTM
-        cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
-      else:
-        rnn_layer_class = keras.layers.GRU
-        cudnn_rnn_layer_class = keras.layers.CuDNNGRU
-        rnn_layer_kwargs['reset_after'] = True
-
-      layer = rnn_layer_class(units, **rnn_layer_kwargs)
-      if bidirectional:
-        layer = keras.layers.Bidirectional(layer)
-
-      cudnn_layer = cudnn_rnn_layer_class(units)
-      if bidirectional:
-        cudnn_layer = keras.layers.Bidirectional(cudnn_layer)
-
-      model = self._make_nested_model(input_shape, layer, model_nest_level,
-                                      model_type)
-      cudnn_model = self._make_nested_model(input_shape, cudnn_layer,
-                                            model_nest_level, model_type)
-
-      if to_cudnn:
-        self._convert_model_weights(model, cudnn_model)
-      else:
-        self._convert_model_weights(cudnn_model, model)
-
-      self.assertAllClose(
-          model.predict(inputs), cudnn_model.predict(inputs), atol=1e-4)
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        timesteps = 6
+        input_shape = (timesteps, input_size)
+        units = 2
+        num_samples = 32
+        inputs = np.random.random((num_samples, timesteps, input_size))
+
+        rnn_layer_kwargs = {
+            'recurrent_activation': 'sigmoid',
+            # ensure biases are non-zero and properly converted
+            'bias_initializer': 'random_uniform',
+            'implementation': implementation
+        }
+        if rnn_type == 'LSTM':
+          rnn_layer_class = keras.layers.LSTM
+          cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
+        else:
+          rnn_layer_class = keras.layers.GRU
+          cudnn_rnn_layer_class = keras.layers.CuDNNGRU
+          rnn_layer_kwargs['reset_after'] = True
+
+        layer = rnn_layer_class(units, **rnn_layer_kwargs)
+        if bidirectional:
+          layer = keras.layers.Bidirectional(layer)
+
+        cudnn_layer = cudnn_rnn_layer_class(units)
+        if bidirectional:
+          cudnn_layer = keras.layers.Bidirectional(cudnn_layer)
+
+        model = self._make_nested_model(input_shape, layer, model_nest_level,
+                                        model_type)
+        cudnn_model = self._make_nested_model(input_shape, cudnn_layer,
+                                              model_nest_level, model_type)
+
+        if to_cudnn:
+          self._convert_model_weights(model, cudnn_model)
+        else:
+          self._convert_model_weights(cudnn_model, model)
+
+        self.assertAllClose(model.predict(inputs), cudnn_model.predict(inputs),
+                            atol=1e-4)
 
   def _make_nested_model(self, input_shape, layer, level=1, model_type='func'):
     # example: make_nested_seq_model((1,), Dense(10), level=2).summary()
@@ -356,145 +334,149 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
                                                               to_cudnn):
     # Similar test as test_load_weights_between_noncudnn_rnn() but has different
     # rank of input due to usage of TimeDistributed. Issue: #10356.
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_size = 10
-      steps = 6
-      timesteps = 6
-      input_shape = (timesteps, steps, input_size)
-      units = 2
-      num_samples = 32
-      inputs = np.random.random((num_samples, timesteps, steps, input_size))
-
-      rnn_layer_kwargs = {
-          'recurrent_activation': 'sigmoid',
-          # ensure biases are non-zero and properly converted
-          'bias_initializer': 'random_uniform',
-      }
-      if rnn_type == 'LSTM':
-        rnn_layer_class = keras.layers.LSTM
-        cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
-      else:
-        rnn_layer_class = keras.layers.GRU
-        cudnn_rnn_layer_class = keras.layers.CuDNNGRU
-        rnn_layer_kwargs['reset_after'] = True
-
-      layer = rnn_layer_class(units, **rnn_layer_kwargs)
-      layer = keras.layers.TimeDistributed(layer)
-
-      cudnn_layer = cudnn_rnn_layer_class(units)
-      cudnn_layer = keras.layers.TimeDistributed(cudnn_layer)
-
-      model = self._make_nested_model(input_shape, layer)
-      cudnn_model = self._make_nested_model(input_shape, cudnn_layer)
-
-      if to_cudnn:
-        self._convert_model_weights(model, cudnn_model)
-      else:
-        self._convert_model_weights(cudnn_model, model)
-
-      self.assertAllClose(
-          model.predict(inputs), cudnn_model.predict(inputs), atol=1e-4)
-
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_size = 10
+        steps = 6
+        timesteps = 6
+        input_shape = (timesteps, steps, input_size)
+        units = 2
+        num_samples = 32
+        inputs = np.random.random((num_samples, timesteps, steps, input_size))
+
+        rnn_layer_kwargs = {
+            'recurrent_activation': 'sigmoid',
+            # ensure biases are non-zero and properly converted
+            'bias_initializer': 'random_uniform',
+        }
+        if rnn_type == 'LSTM':
+          rnn_layer_class = keras.layers.LSTM
+          cudnn_rnn_layer_class = keras.layers.CuDNNLSTM
+        else:
+          rnn_layer_class = keras.layers.GRU
+          cudnn_rnn_layer_class = keras.layers.CuDNNGRU
+          rnn_layer_kwargs['reset_after'] = True
+
+        layer = rnn_layer_class(units, **rnn_layer_kwargs)
+        layer = keras.layers.TimeDistributed(layer)
+
+        cudnn_layer = cudnn_rnn_layer_class(units)
+        cudnn_layer = keras.layers.TimeDistributed(cudnn_layer)
+
+        model = self._make_nested_model(input_shape, layer)
+        cudnn_model = self._make_nested_model(input_shape, cudnn_layer)
+
+        if to_cudnn:
+          self._convert_model_weights(model, cudnn_model)
+        else:
+          self._convert_model_weights(cudnn_model, model)
+
+        self.assertAllClose(model.predict(inputs), cudnn_model.predict(inputs),
+                            atol=1e-4)
+
+  @test_util.run_in_graph_and_eager_modes
   def test_cudnnrnn_bidirectional(self):
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      rnn = keras.layers.CuDNNGRU
-      samples = 2
-      dim = 2
-      timesteps = 2
-      output_dim = 2
-      mode = 'concat'
-
-      x = np.random.random((samples, timesteps, dim))
-      target_dim = 2 * output_dim if mode == 'concat' else output_dim
-      y = np.random.random((samples, target_dim))
-
-      # test with Sequential model
-      model = keras.Sequential()
-      model.add(
-          keras.layers.Bidirectional(
-              rnn(output_dim), merge_mode=mode, input_shape=(None, dim)))
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-      model.fit(x, y, epochs=1, batch_size=1)
-
-      # test config
-      model.get_config()
-      model = keras.models.model_from_json(model.to_json())
-      model.summary()
-
-      # test stacked bidirectional layers
-      model = keras.Sequential()
-      model.add(
-          keras.layers.Bidirectional(
-              rnn(output_dim, return_sequences=True),
-              merge_mode=mode,
-              input_shape=(None, dim)))
-      model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode))
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-      model.fit(x, y, epochs=1, batch_size=1)
-
-      # test with functional API
-      inputs = keras.Input((timesteps, dim))
-      outputs = keras.layers.Bidirectional(
-          rnn(output_dim), merge_mode=mode)(
-              inputs)
-      model = keras.Model(inputs, outputs)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-      model.fit(x, y, epochs=1, batch_size=1)
-
-      # Bidirectional and stateful
-      inputs = keras.Input(batch_shape=(1, timesteps, dim))
-      outputs = keras.layers.Bidirectional(
-          rnn(output_dim, stateful=True), merge_mode=mode)(
-              inputs)
-      model = keras.Model(inputs, outputs)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
-      model.fit(x, y, epochs=1, batch_size=1)
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        rnn = keras.layers.CuDNNGRU
+        samples = 2
+        dim = 2
+        timesteps = 2
+        output_dim = 2
+        mode = 'concat'
+
+        x = np.random.random((samples, timesteps, dim))
+        target_dim = 2 * output_dim if mode == 'concat' else output_dim
+        y = np.random.random((samples, target_dim))
+
+        # test with Sequential model
+        model = keras.Sequential()
+        model.add(
+            keras.layers.Bidirectional(
+                rnn(output_dim), merge_mode=mode, input_shape=(None, dim)))
+        model.compile(
+            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+        model.fit(x, y, epochs=1, batch_size=1)
+
+        # test config
+        model.get_config()
+        model = keras.models.model_from_json(model.to_json())
+        model.summary()
+
+        # test stacked bidirectional layers
+        model = keras.Sequential()
+        model.add(
+            keras.layers.Bidirectional(
+                rnn(output_dim, return_sequences=True),
+                merge_mode=mode,
+                input_shape=(None, dim)))
+        model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode))
+        model.compile(
+            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+        model.fit(x, y, epochs=1, batch_size=1)
+
+        # test with functional API
+        inputs = keras.Input((timesteps, dim))
+        outputs = keras.layers.Bidirectional(
+            rnn(output_dim), merge_mode=mode)(
+                inputs)
+        model = keras.Model(inputs, outputs)
+        model.compile(
+            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+        model.fit(x, y, epochs=1, batch_size=1)
+
+        # Bidirectional and stateful
+        inputs = keras.Input(batch_shape=(1, timesteps, dim))
+        outputs = keras.layers.Bidirectional(
+            rnn(output_dim, stateful=True), merge_mode=mode)(
+                inputs)
+        model = keras.Model(inputs, outputs)
+        model.compile(
+            loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+        model.fit(x, y, epochs=1, batch_size=1)
 
   def test_preprocess_weights_for_loading_gru_incompatible(self):
     """Test loading weights between incompatible layers.
 
     Should fail fast with an exception.
     """
-    if not test.is_gpu_available(cuda_only=True):
-      self.skipTest('No CUDA GPU available')
-
-    with test_util.use_gpu():
-      input_shape = (3, 5)
-
-      def gru(cudnn=False, **kwargs):
-        layer_class = keras.layers.CuDNNGRU if cudnn else keras.layers.GRU
-        return layer_class(2, input_shape=input_shape, **kwargs)
-
-      def get_layer_weights(layer):
-        layer.build(input_shape=input_shape)
-        return layer.get_weights()
-
-      def assert_not_compatible(src, dest, message):
-        with self.assertRaises(ValueError) as ex:
-          keras.engine.saving.preprocess_weights_for_loading(
-              dest, get_layer_weights(src))
-        self.assertIn(message, str(ex.exception))
-
-      assert_not_compatible(
-          gru(), gru(cudnn=True),
-          'GRU(reset_after=False) is not compatible with CuDNNGRU')
-      assert_not_compatible(
-          gru(cudnn=True), gru(),
-          'CuDNNGRU is not compatible with GRU(reset_after=False)')
-      assert_not_compatible(
-          gru(), gru(reset_after=True),
-          'GRU(reset_after=False) is not compatible with '
-          'GRU(reset_after=True)')
-      assert_not_compatible(
-          gru(reset_after=True), gru(),
-          'GRU(reset_after=True) is not compatible with '
-          'GRU(reset_after=False)')
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        input_shape = (3, 5)
+
+        def gru(cudnn=False, **kwargs):
+          layer_class = keras.layers.CuDNNGRU if cudnn else keras.layers.GRU
+          return layer_class(2, input_shape=input_shape, **kwargs)
+
+        def get_layer_weights(layer):
+          layer.build(input_shape=input_shape)
+          return layer.get_weights()
+
+        def assert_not_compatible(src, dest, message):
+          with self.assertRaises(ValueError) as ex:
+            keras.engine.saving.preprocess_weights_for_loading(
+                dest,
+                get_layer_weights(src))
+          self.assertIn(message, str(ex.exception))
+
+        assert_not_compatible(
+            gru(),
+            gru(cudnn=True),
+            'GRU(reset_after=False) is not compatible with CuDNNGRU')
+        assert_not_compatible(
+            gru(cudnn=True),
+            gru(),
+            'CuDNNGRU is not compatible with GRU(reset_after=False)')
+        assert_not_compatible(
+            gru(),
+            gru(reset_after=True),
+            'GRU(reset_after=False) is not compatible with '
+            'GRU(reset_after=True)')
+        assert_not_compatible(
+            gru(reset_after=True),
+            gru(),
+            'GRU(reset_after=True) is not compatible with '
+            'GRU(reset_after=False)')
 
 
 if __name__ == '__main__':
-- 
GitLab


From a26f3b05986eaec30fa3d9547decbaa9607291dd Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 29 Nov 2018 13:30:32 -0800
Subject: [PATCH 1053/1554] Moves ClusterResolvers into
 tensorflow.python.distribute in preparation for TensorFlow 2.0

PiperOrigin-RevId: 223401165
---
 tensorflow/contrib/cluster_resolver/BUILD     | 162 +------
 .../contrib/cluster_resolver/__init__.py      |  16 +-
 .../cluster_resolver_initialization_test.py   |  53 +++
 .../python/training/__init__.py               |  41 +-
 .../python/training/cluster_resolver.py       | 368 +--------------
 .../python/training/gce_cluster_resolver.py   | 199 +-------
 .../training/kubernetes_cluster_resolver.py   | 161 +------
 .../python/training/slurm_cluster_resolver.py | 215 +--------
 .../training/tfconfig_cluster_resolver.py     | 159 +------
 .../python/training/tpu_cluster_resolver.py   | 414 +----------------
 tensorflow/contrib/tpu/BUILD                  |   4 +-
 tensorflow/python/BUILD                       |  13 +
 tensorflow/python/distribute/BUILD            |   1 +
 .../python/distribute/cluster_resolver/BUILD  | 180 ++++++++
 .../distribute}/cluster_resolver/README.md    |   0
 .../distribute/cluster_resolver}/README.slurm |   0
 .../distribute/cluster_resolver/__init__.py   |  57 +++
 .../cluster_resolver/cluster_resolver.py      | 374 ++++++++++++++++
 .../cluster_resolver_test.py                  |   4 +-
 .../cluster_resolver/gce_cluster_resolver.py  | 206 +++++++++
 .../gce_cluster_resolver_test.py              |   4 +-
 .../kubernetes_cluster_resolver.py            | 173 +++++++
 .../kubernetes_cluster_resolver_test.py       |   2 +-
 .../slurm_cluster_resolver.py                 | 226 ++++++++++
 .../slurm_cluster_resolver_test.py            |   2 +-
 .../tfconfig_cluster_resolver.py              | 171 +++++++
 .../tfconfig_cluster_resolver_test.py         |   2 +-
 .../cluster_resolver/tpu_cluster_resolver.py  | 423 ++++++++++++++++++
 .../tpu_cluster_resolver_test.py              |   2 +-
 29 files changed, 2014 insertions(+), 1618 deletions(-)
 create mode 100644 tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py
 create mode 100644 tensorflow/python/distribute/cluster_resolver/BUILD
 rename tensorflow/{contrib => python/distribute}/cluster_resolver/README.md (100%)
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/README.slurm (100%)
 create mode 100644 tensorflow/python/distribute/cluster_resolver/__init__.py
 create mode 100644 tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/cluster_resolver_test.py (98%)
 create mode 100644 tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/gce_cluster_resolver_test.py (98%)
 create mode 100644 tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/kubernetes_cluster_resolver_test.py (98%)
 create mode 100644 tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/slurm_cluster_resolver_test.py (98%)
 create mode 100644 tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/tfconfig_cluster_resolver_test.py (98%)
 create mode 100644 tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
 rename tensorflow/{contrib/cluster_resolver/python/training => python/distribute/cluster_resolver}/tpu_cluster_resolver_test.py (99%)

diff --git a/tensorflow/contrib/cluster_resolver/BUILD b/tensorflow/contrib/cluster_resolver/BUILD
index 9e1867ea9d..f944b7f884 100644
--- a/tensorflow/contrib/cluster_resolver/BUILD
+++ b/tensorflow/contrib/cluster_resolver/BUILD
@@ -21,173 +21,25 @@ py_library(
 
 py_library(
     name = "cluster_resolver_py",
-    srcs = [
+    srcs = glob([
         "__init__.py",
-        "python/training/__init__.py",
-    ],
+        "python/training/*.py",
+    ]),
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
-    deps = [
-        ":base_cluster_resolver_py",
-        ":gce_cluster_resolver_py",
-        ":kubernetes_cluster_resolver_py",
-        ":slurm_cluster_resolver_py",
-        ":tfconfig_cluster_resolver_py",
-        ":tpu_cluster_resolver_py",
-        "//tensorflow/python:util",
-    ],
-)
-
-py_library(
-    name = "base_cluster_resolver_py",
-    srcs = ["python/training/cluster_resolver.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/python:training",
-    ],
-)
-
-py_library(
-    name = "gce_cluster_resolver_py",
-    srcs = ["python/training/gce_cluster_resolver.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":base_cluster_resolver_py",
-        "//tensorflow/python:training",
-    ],
-)
-
-py_library(
-    name = "tfconfig_cluster_resolver_py",
-    srcs = ["python/training/tfconfig_cluster_resolver.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":base_cluster_resolver_py",
-        "//tensorflow/python:training",
-    ],
-)
-
-py_library(
-    name = "tpu_cluster_resolver_py",
-    srcs = ["python/training/tpu_cluster_resolver.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":base_cluster_resolver_py",
-        "//tensorflow/python:training",
-    ],
-)
-
-py_library(
-    name = "slurm_cluster_resolver_py",
-    srcs = ["python/training/slurm_cluster_resolver.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":base_cluster_resolver_py",
-        "//tensorflow/python:training",
-    ],
-)
-
-py_library(
-    name = "kubernetes_cluster_resolver_py",
-    srcs = ["python/training/kubernetes_cluster_resolver.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":base_cluster_resolver_py",
-        "//tensorflow/python:training",
-    ],
-)
-
-tf_py_test(
-    name = "base_cluster_resolver_py_test",
-    srcs = ["python/training/cluster_resolver_test.py"],
-    additional_deps = [
-        ":cluster_resolver_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:training",
-    ],
-    main = "python/training/cluster_resolver_test.py",
-)
-
-tf_py_test(
-    name = "gce_cluster_resolver_py_test",
-    size = "small",
-    srcs = ["python/training/gce_cluster_resolver_test.py"],
-    additional_deps = [
-        ":cluster_resolver_py",
-        ":gce_cluster_resolver_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:training",
-    ],
-    main = "python/training/gce_cluster_resolver_test.py",
-)
-
-tf_py_test(
-    name = "tfconfig_cluster_resolver_py_test",
-    size = "small",
-    srcs = ["python/training/tfconfig_cluster_resolver_test.py"],
-    additional_deps = [
-        ":tfconfig_cluster_resolver_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:training",
-    ],
-    grpc_enabled = True,
-    main = "python/training/tfconfig_cluster_resolver_test.py",
-)
-
-tf_py_test(
-    name = "tpu_cluster_resolver_py_test",
-    size = "small",
-    srcs = ["python/training/tpu_cluster_resolver_test.py"],
-    additional_deps = [
-        ":tpu_cluster_resolver_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:training",
-    ],
-    grpc_enabled = True,
-    main = "python/training/tpu_cluster_resolver_test.py",
-)
-
-tf_py_test(
-    name = "slurm_cluster_resolver_py_test",
-    size = "small",
-    srcs = ["python/training/slurm_cluster_resolver_test.py"],
-    additional_deps = [
-        ":cluster_resolver_py",
-        ":slurm_cluster_resolver_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:training",
-    ],
-    main = "python/training/slurm_cluster_resolver_test.py",
-    tags = [],
+    deps = ["//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib"],
 )
 
 tf_py_test(
-    name = "kubernetes_cluster_resolver_py_test",
-    size = "small",
-    srcs = ["python/training/kubernetes_cluster_resolver_test.py"],
+    name = "cluster_resolver_initialization_test",
+    srcs = ["cluster_resolver_initialization_test.py"],
     additional_deps = [
         ":cluster_resolver_py",
-        ":kubernetes_cluster_resolver_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:training",
     ],
-    main = "python/training/kubernetes_cluster_resolver_test.py",
+    main = "cluster_resolver_initialization_test.py",
 )
diff --git a/tensorflow/contrib/cluster_resolver/__init__.py b/tensorflow/contrib/cluster_resolver/__init__.py
index ab0746ab83..390b3e7550 100644
--- a/tensorflow/contrib/cluster_resolver/__init__.py
+++ b/tensorflow/contrib/cluster_resolver/__init__.py
@@ -20,14 +20,14 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=wildcard-import,unused-import
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.kubernetes_cluster_resolver import KubernetesClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.tfconfig_cluster_resolver import TFConfigClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
+from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
+from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver
 # pylint: enable=wildcard-import,unused-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py b/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py
new file mode 100644
index 0000000000..01ff1478c6
--- /dev/null
+++ b/tensorflow/contrib/cluster_resolver/cluster_resolver_initialization_test.py
@@ -0,0 +1,53 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests to ensure ClusterResolvers are usable via the old contrib path."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.cluster_resolver import SimpleClusterResolver
+from tensorflow.contrib.cluster_resolver.python.training import cluster_resolver
+from tensorflow.contrib.cluster_resolver.python.training import UnionClusterResolver
+from tensorflow.python.platform import test
+from tensorflow.python.training import server_lib
+
+
+class ClusterResolverInitializationTest(test.TestCase):
+
+  def testCreateSimpleClusterResolverFromLib(self):
+    base_cluster_spec = server_lib.ClusterSpec({
+        "ps": ["ps0:2222", "ps1:2222"],
+        "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
+    })
+    cluster_resolver.SimpleClusterResolver(base_cluster_spec)
+
+  def testCreateSimpleClusterResolver(self):
+    base_cluster_spec = server_lib.ClusterSpec({
+        "ps": ["ps0:2222", "ps1:2222"],
+        "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
+    })
+    SimpleClusterResolver(base_cluster_spec)
+
+  def testCreateUnionClusterResolver(self):
+    base_cluster_spec = server_lib.ClusterSpec({
+        "ps": ["ps0:2222", "ps1:2222"],
+        "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
+    })
+    simple_cr = SimpleClusterResolver(base_cluster_spec)
+    UnionClusterResolver(simple_cr)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/cluster_resolver/python/training/__init__.py b/tensorflow/contrib/cluster_resolver/python/training/__init__.py
index 6d9120a3b9..10d93549eb 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/__init__.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/__init__.py
@@ -18,11 +18,36 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.kubernetes_cluster_resolver import KubernetesClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.tfconfig_cluster_resolver import TFConfigClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
+
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
+from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
+from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver
+
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = [
+    'cluster_resolver',
+    'gce_cluster_resolver',
+    'kubernetes_cluster_resolver',
+    'slurm_cluster_resolver',
+    'tfconfig_cluster_resolver',
+    'tpu_cluster_resolver',
+    'ClusterResolver',
+    'SimpleClusterResolver',
+    'UnionClusterResolver',
+    'GceClusterResolver',
+    'KubernetesClusterResolver',
+    'TFConfigClusterResolver',
+    'TPUClusterResolver',
+    'SlurmClusterResolver',
+]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
index 7774ac0e12..99840fb516 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,363 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Cluster Resolvers are used for dynamic cluster IP/hostname resolution."""
+"""Stub file for ClusterResolver to maintain backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import abc
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
 
-import six
+# pylint: disable=unused-import
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
+# pylint: enable=unused-import
 
-from tensorflow.python.training.server_lib import ClusterSpec
+from tensorflow.python.util.all_util import remove_undocumented
 
+_allowed_symbols = [
+    'ClusterResolver',
+    'SimpleClusterResolver',
+    'UnionClusterResolver',
+]
 
-def format_master_url(master, rpc_layer=None):
-  if rpc_layer:
-    return '%s://%s' % (rpc_layer, master)
-  else:
-    return master
+remove_undocumented(__name__, _allowed_symbols)
 
-
-@six.add_metaclass(abc.ABCMeta)
-class ClusterResolver(object):
-  """Abstract class for all implementations of ClusterResolvers.
-
-  This defines the skeleton for all implementations of ClusterResolvers.
-  ClusterResolvers are a way for TensorFlow to communicate with various cluster
-  management systems (e.g. GCE, AWS, etc...).
-
-  By letting TensorFlow communicate with these systems, we will be able to
-  automatically discover and resolve IP addresses for various TensorFlow
-  workers. This will eventually allow us to automatically recover from
-  underlying machine failures and scale TensorFlow worker clusters up and down.
-
-  Note to Implementors: In addition to these abstract methods, you must also
-  implement the task_type, task_index, and rpc_layer attributes. You may choose
-  to implement them either as properties with getters or setters or directly
-  set the attributes.
-
-  - task_type is the name of the server's current named job (e.g. 'worker',
-     'ps' in a distributed parameterized training job).
-  - task_index is the ordinal index of the server within the task type.
-  - rpc_layer is the protocol used by TensorFlow to communicate with other
-      TensorFlow servers in a distributed environment.
-  """
-
-  @abc.abstractmethod
-  def cluster_spec(self):
-    """Retrieve the current state of the cluster and returns a ClusterSpec.
-
-    Returns:
-      A ClusterSpec representing the state of the cluster at the moment this
-      function is called.
-
-    Implementors of this function must take care in ensuring that the
-    ClusterSpec returned is up-to-date at the time of calling this function.
-    This usually means retrieving the information from the underlying cluster
-    management system every time this function is invoked and reconstructing
-    a cluster_spec, rather than attempting to cache anything.
-    """
-    raise NotImplementedError()
-
-  @abc.abstractmethod
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Retrieves the name or URL of the session master.
-
-    Args:
-      task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
-      rpc_layer: (Optional) The RPC protocol for the given cluster.
-
-    Returns:
-      The name or URL of the session master.
-
-    Implementors of this function must take care in ensuring that the master
-    returned is up-to-date at the time to calling this function. This usually
-    means retrieving the master every time this function is invoked.
-    """
-    raise NotImplementedError()
-
-  @abc.abstractmethod
-  def num_accelerators_per_worker(self, session_config=None):
-    """Returns the number of accelerator cores per worker.
-
-    This returns the number of accelerator cores (such as GPUs and TPUs)
-    available per worker. If workers only has CPU cores available, then this
-    should return 0. This method will query the master for this information
-    if it is not otherwise known.
-
-    Args:
-      session_config: (Optional) Configuration for starting a new session to
-        query how many accelerator cores it has.
-    """
-    raise NotImplementedError()
-
-  @abc.abstractproperty
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in."""
-    raise NotImplementedError()
-
-
-class SimpleClusterResolver(ClusterResolver):
-  """Simple implementation of ClusterResolver that accepts a ClusterSpec."""
-
-  def __init__(self, cluster_spec, master='', task_type=None, task_index=None,
-               environment='', num_accelerators_per_worker=0,
-               rpc_layer=None):
-    """Creates a SimpleClusterResolver from a ClusterSpec."""
-    super(SimpleClusterResolver, self).__init__()
-
-    self._task_type = task_type
-    self._task_index = task_index
-    self._environment = environment
-    self._num_accelerators_per_worker = num_accelerators_per_worker
-    self._rpc_layer = rpc_layer
-
-    if not isinstance(cluster_spec, ClusterSpec):
-      raise TypeError('cluster_spec must be a ClusterSpec.')
-    self._cluster_spec = cluster_spec
-
-    if not isinstance(master, str):
-      raise TypeError('master must be a string.')
-    self._master = master
-
-  def cluster_spec(self):
-    """Returns the ClusterSpec passed into the constructor."""
-    return self._cluster_spec
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Returns the master address to use when creating a session.
-
-    Args:
-      task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
-      rpc_layer: (Optional) The RPC used by distributed TensorFlow.
-
-    Returns:
-      The name or URL of the session master.
-
-    If a task_type and task_index is given, this will override the `master`
-    string passed into the initialization function.
-    """
-    if task_type is not None and task_index is not None:
-      master = self.cluster_spec().task_address(task_type, task_index)
-    else:
-      master = self._master
-
-    return format_master_url(master, rpc_layer=rpc_layer or self._rpc_layer)
-
-  @property
-  def task_type(self):
-    return self._task_type
-
-  @property
-  def task_index(self):
-    return self._task_index
-
-  @task_type.setter
-  def task_type(self, task_type):
-    self._task_type = task_type
-
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
-
-  @property
-  def environment(self):
-    return self._environment
-
-  def num_accelerators_per_worker(self, session_config=None):
-    """Returns the number of accelerator cores per worker.
-
-    Args:
-      session_config: Unused. The SimpleClusterResolver does not do automatic
-        detection of accelerators, so a TensorFlow session will never be
-        created, and thus a `session_config` is never necessary here, and will
-        be ignored.
-    """
-    del session_config
-    return self._num_accelerators_per_worker
-
-  @property
-  def rpc_layer(self):
-    return self._rpc_layer
-
-  @rpc_layer.setter
-  def rpc_layer(self, rpc_layer):
-    self._rpc_layer = rpc_layer
-
-
-class UnionClusterResolver(ClusterResolver):
-  """Performs a union on underlying ClusterResolvers.
-
-  This class performs a union given two or more existing ClusterResolvers. It
-  merges the underlying ClusterResolvers, and returns one unified ClusterSpec
-  when cluster_spec is called. The details of the merge function is
-  documented in the cluster_spec function.
-
-  For additional Cluster Resolver properties such as task type, task index,
-  rpc layer, environment, etc..., we will return the value from the first
-  ClusterResolver in the union.
-  """
-
-  def __init__(self, *args, **kwargs):
-    """Initializes a UnionClusterResolver with other ClusterResolvers.
-
-    Args:
-      *args: `ClusterResolver` objects to be unionized.
-      **kwargs:
-        rpc_layer - (Optional) Override value for the RPC layer used by
-          TensorFlow.
-        task_type - (Optional) Override value for the current task type.
-        task_index - (Optional) Override value for the current task index.
-
-    Raises:
-      TypeError: If any argument is not a subclass of `ClusterResolvers`.
-      ValueError: If there are no arguments passed.
-    """
-    super(UnionClusterResolver, self).__init__()
-
-    self._rpc_layer = kwargs.pop('rpc_layer', None)
-    self._task_type = kwargs.pop('task_type', None)
-    self._task_index = kwargs.pop('task_index', None)
-
-    if kwargs:
-      raise ValueError('Unexpected kwargs provided {!r}'.format(kwargs))
-
-    if not args:
-      raise ValueError('At least one ClusterResolver is required.')
-
-    for cluster_resolver in args:
-      if not isinstance(cluster_resolver, ClusterResolver):
-        raise TypeError('All arguments must be a sub-class of '
-                        '`ClusterResolver.`')
-    self._cluster_resolvers = args
-
-  def cluster_spec(self):
-    """Returns a union of all the ClusterSpecs from the ClusterResolvers.
-
-    Returns:
-      A ClusterSpec containing host information merged from all the underlying
-      ClusterResolvers.
-
-    Raises:
-      KeyError: If there are conflicting keys detected when merging two or
-      more dictionaries, this exception is raised.
-
-    Note: If there are multiple ClusterResolvers exposing ClusterSpecs with the
-    same job name, we will merge the list/dict of workers.
-
-    If *all* underlying ClusterSpecs expose the set of workers as lists, we will
-    concatenate the lists of workers, starting with the list of workers from
-    the first ClusterResolver passed into the constructor.
-
-    If *any* of the ClusterSpecs expose the set of workers as a dict, we will
-    treat all the sets of workers as dicts (even if they are returned as lists)
-    and will only merge them into a dict if there is no conflicting keys. If
-    there is a conflicting key, we will raise a `KeyError`.
-    """
-
-    merged_cluster = {}
-
-    # We figure out whether it is all lists for a particular job, or whether
-    # there are dicts inside.
-    for cluster_resolver in self._cluster_resolvers:
-      cluster_spec = cluster_resolver.cluster_spec()
-      cluster_dict = cluster_spec.as_dict()
-
-      for job_name, tasks in cluster_dict.items():
-        if job_name in merged_cluster:
-          # If we see a dict, then we write a dict out regardless.
-          if isinstance(tasks, dict):
-            merged_cluster[job_name] = {}
-        else:
-          # We take whichever type is present.
-          if isinstance(tasks, list):
-            merged_cluster[job_name] = []
-          else:
-            merged_cluster[job_name] = {}
-
-    # We then do the merge as appropriate in merged_cluster[job].
-    for cluster_resolver in self._cluster_resolvers:
-      cluster_spec = cluster_resolver.cluster_spec()
-      cluster_dict = cluster_spec.as_dict()
-
-      for job_name, tasks in cluster_dict.items():
-        if isinstance(merged_cluster[job_name], list):
-          # We all have lists, we can just concatenate and be done.
-          merged_cluster[job_name].extend(tasks)
-        else:
-          if isinstance(tasks, list):
-            # We convert to a dictionary if the type is a list.
-            task_dict = dict(zip(range(0, len(tasks)), tasks))
-          else:
-            # We can simply make a copy (for update) and be done.
-            task_dict = tasks.copy()
-
-          # We detect if there are duplicates, and raise an error if so.
-          task_keys = set(task_dict)
-          merged_keys = set(merged_cluster[job_name].keys())
-          intersected_keys = task_keys.intersection(merged_keys)
-          if intersected_keys:
-            raise KeyError('Duplicate keys detected when merging two '
-                           'ClusterSpecs: %s' % repr(intersected_keys))
-
-          # We do the merge after all the processing.
-          merged_cluster[job_name].update(task_dict)
-
-    return ClusterSpec(merged_cluster)
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Returns the master address to use when creating a session.
-
-    This usually returns the master from the first ClusterResolver passed in,
-    but you can override this by specifying the task_type and task_index.
-
-    Args:
-      task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
-      rpc_layer: (Optional) The RPC protocol for the given cluster.
-
-    Returns:
-      The name or URL of the session master.
-    """
-    if task_type is not None and task_index is not None:
-      master = self.cluster_spec().task_address(task_type, task_index)
-      return format_master_url(master, rpc_layer or self._rpc_layer)
-
-    return self._cluster_resolvers[0].master(rpc_layer=rpc_layer)
-
-  @property
-  def task_type(self):
-    return self._task_type or self._cluster_resolvers[0].task_type
-
-  @property
-  def task_index(self):
-    return self._task_index or self._cluster_resolvers[0].task_index
-
-  @task_type.setter
-  def task_type(self, task_type):
-    self._task_type = task_type
-
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
-
-  @property
-  def environment(self):
-    return self._cluster_resolvers[0].environment
-
-  def num_accelerators_per_worker(self, session_config=None):
-    return self._cluster_resolvers[0].num_accelerators_per_worker(
-        session_config)
-
-  @property
-  def rpc_layer(self):
-    return self._rpc_layer or self._cluster_resolvers[0].rpc_layer
-
-  @rpc_layer.setter
-  def rpc_layer(self, rpc_layer):
-    self._rpc_layer = rpc_layer
diff --git a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py
index 195b68959b..55e61155c6 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,197 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Implementation of Cluster Resolvers for GCE Instance Groups."""
+"""Stub file for GceClusterResolver to maintain backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.python.training.server_lib import ClusterSpec
+# pylint: disable=unused-import
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+# pylint: enable=unused-import
 
-_GOOGLE_API_CLIENT_INSTALLED = True
-try:
-  from googleapiclient import discovery  # pylint: disable=g-import-not-at-top
-  from oauth2client.client import GoogleCredentials  # pylint: disable=g-import-not-at-top
-except ImportError:
-  _GOOGLE_API_CLIENT_INSTALLED = False
+from tensorflow.python.util.all_util import remove_undocumented
 
+_allowed_symbols = [
+    'GceClusterResolver',
+]
 
-def _format_master_url(master, rpc_layer=None):
-  return '%s://%s' % (rpc_layer, master) if rpc_layer else master
-
-
-class GceClusterResolver(ClusterResolver):
-  """Cluster Resolver for Google Compute Engine.
-
-  This is an implementation of cluster resolvers for the Google Compute Engine
-  instance group platform. By specifying a project, zone, and instance group,
-  this will retrieve the IP address of all the instances within the instance
-  group and return a Cluster Resolver object suitable for use for distributed
-  TensorFlow.
-  """
-
-  def __init__(self,
-               project,
-               zone,
-               instance_group,
-               port,
-               task_type='worker',
-               task_index=0,
-               rpc_layer='grpc',
-               num_accelerators_per_worker=0,
-               credentials='default',
-               service=None):
-    """Creates a new GceClusterResolver object.
-
-    This takes in a few parameters and creates a GceClusterResolver project. It
-    will then use these parameters to query the GCE API for the IP addresses of
-    each instance in the instance group.
-
-    Args:
-      project: Name of the GCE project.
-      zone: Zone of the GCE instance group.
-      instance_group: Name of the GCE instance group.
-      port: Port of the listening TensorFlow server (default: 8470)
-      task_type: Name of the TensorFlow job this GCE instance group of VM
-        instances belong to.
-      task_index: The task index for this particular VM, within the GCE
-        instance group. In particular, every single instance should be assigned
-        a unique ordinal index within an instance group manually so that they
-        can be distinguished from each other.
-      rpc_layer: The RPC layer TensorFlow should use to communicate across
-        instances.
-      num_accelerators_per_worker: Number of accelerators (GPUs) present per
-        instance.
-      credentials: GCE Credentials. If nothing is specified, this defaults to
-        GoogleCredentials.get_application_default().
-      service: The GCE API object returned by the googleapiclient.discovery
-        function. (Default: discovery.build('compute', 'v1')). If you specify a
-        custom service object, then the credentials parameter will be ignored.
-
-    Raises:
-      ImportError: If the googleapiclient is not installed.
-    """
-    self._project = project
-    self._zone = zone
-    self._instance_group = instance_group
-    self._task_type = task_type
-    self._task_index = task_index
-    self._rpc_layer = rpc_layer
-    self._port = port
-    self._credentials = credentials
-
-    if credentials == 'default':
-      if _GOOGLE_API_CLIENT_INSTALLED:
-        self._credentials = GoogleCredentials.get_application_default()
-
-    if service is None:
-      if not _GOOGLE_API_CLIENT_INSTALLED:
-        raise ImportError('googleapiclient must be installed before using the '
-                          'GCE cluster resolver')
-      self._service = discovery.build(
-          'compute', 'v1',
-          credentials=self._credentials)
-    else:
-      self._service = service
-
-  def cluster_spec(self):
-    """Returns a ClusterSpec object based on the latest instance group info.
-
-    This returns a ClusterSpec object for use based on information from the
-    specified instance group. We will retrieve the information from the GCE APIs
-    every time this method is called.
-
-    Returns:
-      A ClusterSpec containing host information retrieved from GCE.
-    """
-    request_body = {'instanceState': 'RUNNING'}
-    request = self._service.instanceGroups().listInstances(
-        project=self._project,
-        zone=self._zone,
-        instanceGroups=self._instance_group,
-        body=request_body,
-        orderBy='name')
-
-    worker_list = []
-
-    while request is not None:
-      response = request.execute()
-
-      items = response['items']
-      for instance in items:
-        instance_name = instance['instance'].split('/')[-1]
-
-        instance_request = self._service.instances().get(
-            project=self._project,
-            zone=self._zone,
-            instance=instance_name)
-
-        if instance_request is not None:
-          instance_details = instance_request.execute()
-          ip_address = instance_details['networkInterfaces'][0]['networkIP']
-          instance_url = '%s:%s' % (ip_address, self._port)
-          worker_list.append(instance_url)
-
-      request = self._service.instanceGroups().listInstances_next(
-          previous_request=request,
-          previous_response=response)
-
-    worker_list.sort()
-    return ClusterSpec({self._task_type: worker_list})
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    task_type = task_type if task_type is not None else self._task_type
-    task_index = task_index if task_index is not None else self._task_index
-
-    if task_type is not None and task_index is not None:
-      master = self.cluster_spec().task_address(task_type, task_index)
-      if rpc_layer or self._rpc_layer:
-        return '%s://%s' % (rpc_layer or self._rpc_layer, master)
-      else:
-        return master
-
-    return ''
-
-  @property
-  def task_type(self):
-    return self._task_type
-
-  @property
-  def task_index(self):
-    return self._task_index
-
-  @task_type.setter
-  def task_type(self, task_type):
-    raise RuntimeError(
-        'You cannot reset the task_type of the GceClusterResolver after it has '
-        'been created.')
-
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
-
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in.
-
-    For users in the GCE environment, the environment property is always an
-    empty string, and Google users will not use this ClusterResolver for running
-    on internal systems.
-    """
-    return ''
-
-  @property
-  def rpc_layer(self):
-    return self._rpc_layer
-
-  @rpc_layer.setter
-  def rpc_layer(self, rpc_layer):
-    self._rpc_layer = rpc_layer
-
-  def num_accelerators_per_worker(self, session_config=None):
-    del session_config  # Unused, since this is set manually in __init__.
-    return self._num_accelerators_per_worker
-
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
index eab1359a5b..a8eaf33629 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver.py
@@ -12,162 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Implementation of Cluster Resolvers for Kubernetes."""
+"""Stub file for KubernetesClusterResolver for backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import format_master_url
-from tensorflow.python.client import device_lib
-from tensorflow.python.training import server_lib
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
 
-_KUBERNETES_API_CLIENT_INSTALLED = True
-try:
-  from kubernetes import client as k8sclient  # pylint: disable=g-import-not-at-top
-  from kubernetes import config as k8sconfig  # pylint: disable=g-import-not-at-top
-except ImportError:
-  _KUBERNETES_API_CLIENT_INSTALLED = False
+# pylint: disable=unused-import
+from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
+# pylint: enable=unused-import
 
+from tensorflow.python.util.all_util import remove_undocumented
 
-class KubernetesClusterResolver(ClusterResolver):
-  """Cluster Resolver for Kubernetes.
+_allowed_symbols = [
+    'KubernetesClusterResolver',
+]
 
-  This is an implementation of cluster resolvers for Kubernetes. When given the
-  the Kubernetes namespace and label selector for pods, we will retrieve the
-  pod IP addresses of all running pods matching the selector, and return a
-  ClusterSpec based on that information.
-  """
+remove_undocumented(__name__, _allowed_symbols)
 
-  def __init__(self,
-               job_to_label_mapping=None,
-               tf_server_port=8470,
-               rpc_layer='grpc',
-               override_client=None):
-    """Initializes a new KubernetesClusterResolver.
-
-    This initializes a new Kubernetes Cluster Resolver. The Cluster Resolver
-    will attempt to talk to the Kubernetes master to retrieve all the instances
-    of pods matching a label selector.
-
-    Args:
-      job_to_label_mapping: A mapping of TensorFlow jobs to label selectors.
-        This allows users to specify many TensorFlow jobs in one Cluster
-        Resolver, and each job can have pods belong with different label
-        selectors. For example, a sample mapping might be
-        ```
-        {'worker': ['job-name=worker-cluster-a', 'job-name=worker-cluster-b'],
-         'ps': ['job-name=ps-1', 'job-name=ps-2']}
-        ```
-      tf_server_port: The port the TensorFlow server is listening on.
-      rpc_layer: (Optional) The RPC layer TensorFlow should use to communicate
-        between tasks in Kubernetes. Defaults to 'grpc'.
-      override_client: The Kubernetes client (usually automatically retrieved
-        using `from kubernetes import client as k8sclient`). If you pass this
-        in, you are responsible for setting Kubernetes credentials manually.
-
-    Raises:
-      ImportError: If the Kubernetes Python client is not installed and no
-        `override_client` is passed in.
-      RuntimeError: If autoresolve_task is not a boolean or a callable.
-    """
-    if _KUBERNETES_API_CLIENT_INSTALLED:
-      k8sconfig.load_kube_config()
-
-    if not job_to_label_mapping:
-      job_to_label_mapping = {'worker': ['job-name=tensorflow']}
-
-    if not override_client and not _KUBERNETES_API_CLIENT_INSTALLED:
-      raise ImportError('The Kubernetes Python client must be installed before'
-                        'using the Kubernetes Cluster Resolver. To install the'
-                        'Kubernetes Python client, run `pip install '
-                        'kubernetes` on your command line.')
-
-    self._job_to_label_mapping = job_to_label_mapping
-    self._tf_server_port = tf_server_port
-    self._override_client = override_client
-
-    self.task_type = None
-    self.task_index = None
-    self.rpc_layer = rpc_layer
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Returns the master address to use when creating a session.
-
-    You must have set the task_type and task_index object properties before
-    calling this function, or pass in the `task_type` and `task_index`
-    parameters when using this function. If you do both, the function parameters
-    will override the object properties.
-
-    Args:
-      task_type: (Optional) The type of the TensorFlow task of the master.
-      task_index: (Optional) The index of the TensorFlow task of the master.
-      rpc_layer: (Optional) The RPC protocol for the given cluster.
-
-    Returns:
-      The name or URL of the session master.
-    """
-    if task_type is not None and task_index is not None:
-      return format_master_url(
-          self.cluster_spec().task_address(task_type, task_index),
-          rpc_layer or self.rpc_layer)
-
-    if self.task_type is not None and self.task_index is not None:
-      return format_master_url(
-          self.cluster_spec().task_address(self.task_type, self.task_index),
-          rpc_layer or self.rpc_layer)
-
-    return ''
-
-  def cluster_spec(self):
-    """Returns a ClusterSpec object based on the latest info from Kubernetes.
-
-    We retrieve the information from the Kubernetes master every time this
-    method is called.
-
-    Returns:
-      A ClusterSpec containing host information returned from Kubernetes.
-
-    Raises:
-      RuntimeError: If any of the pods returned by the master is not in the
-        `Running` phase.
-    """
-    if not self._override_client:
-      k8sconfig.load_kube_config()
-
-    client = self._override_client or k8sclient.CoreV1Api()
-    cluster_map = {}
-
-    for tf_job in self._job_to_label_mapping:
-      all_pods = []
-      for selector in self._job_to_label_mapping[tf_job]:
-        ret = client.list_pod_for_all_namespaces(label_selector=selector)
-        selected_pods = []
-
-        # Sort the list by the name to make sure it doesn't change call to call.
-        for pod in sorted(ret.items, key=lambda x: x.metadata.name):
-          if pod.status.phase == 'Running':
-            selected_pods.append(
-                '%s:%s' % (pod.status.host_ip, self._tf_server_port))
-          else:
-            raise RuntimeError('Pod "%s" is not running; phase: "%s"' %
-                               (pod.metadata.name, pod.status.phase))
-        all_pods.extend(selected_pods)
-      cluster_map[tf_job] = all_pods
-
-    return server_lib.ClusterSpec(cluster_map)
-
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in.
-
-    For users in the Cloud environment, the environment property is always an
-    empty string, and Google users will not use this ClusterResolver for running
-    on internal systems.
-    """
-    return ''
-
-  def num_accelerators_per_worker(self, session_config=None):
-    local_devices = device_lib.list_local_devices(session_config)
-    return len([d for d in local_devices if d.device_type == 'GPU'])
diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py
index f590ecead9..fcd2a846ee 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver.py
@@ -12,215 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Implementation of Cluster Resolvers for Slurm workload manager."""
+"""Stub file for SlurmClusterResolver to maintain backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-import os
-import subprocess
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.python.training.server_lib import ClusterSpec
+# pylint: disable=unused-import
+from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
+# pylint: enable=unused-import
 
+from tensorflow.python.util.all_util import remove_undocumented
 
-class SlurmClusterResolver(ClusterResolver):
-  """Cluster Resolver for system with Slurm workload manager.
+_allowed_symbols = [
+    'SlurmClusterResolver',
+]
 
-  This is an implementation of cluster resolvers for Slurm clusters. This allows
-  the specification of jobs and task counts, number of tasks per node, number of
-  GPUs on each node and number of GPUs for each task, It retrieves system
-  attributes by Slurm environment variables, resolves allocated computing node
-  names, construct a cluster and return a Cluster Resolver object which an be
-  use for distributed TensorFlow.
-  """
-
-  def _resolve_hostnames(self):
-    """Resolve host names of nodes allocated in current jobs.
-
-    Returns:
-      A list of node names as strings.
-    """
-    hostlist = (subprocess.check_output(['scontrol', 'show', 'hostname']).
-                decode('utf-8').strip().split('\n'))
-    return hostlist
-
-  def __init__(self,
-               jobs,
-               port_base=8888,
-               gpus_per_node=1,
-               gpus_per_task=1,
-               tasks_per_node=None,
-               auto_set_gpu=True,
-               rpc_layer='grpc'):
-    """Creates a new SlurmClusterResolver object.
-
-    This takes in parameters and creates a SlurmClusterResolver object. It uses
-    those parameters to check which nodes will processes reside and resolves
-    their hostnames. With the number of the GPUs on each node and number of GPUs
-    for each task it offsets the port number for each processes and allocate
-    GPUs to tasks by setting environment variables. The resolver currently
-    supports homogeneous tasks and default Slurm process allocation.
-
-    Args:
-      jobs: Dictionary with job names as key and number of tasks in the job as
-        value
-      port_base: The first port number to start with for processes on a node.
-      gpus_per_node: Number of GPUs available on each node.
-      gpus_per_task: Number of GPUs to be used for each task.
-      tasks_per_node: Number of tasks to run on each node, if not set defaults
-        to Slurm's output environment variable SLURM_NTASKS_PER_NODE.
-      auto_set_gpu: Set the visible CUDA devices automatically while resolving
-        the cluster by setting CUDA_VISIBLE_DEVICES environment variable.
-        Defaults to True.
-      rpc_layer: (Optional) The protocol TensorFlow uses to communicate between
-        nodes. Defaults to 'grpc'.
-
-    Returns:
-      A ClusterResolver object which can be used with distributed TensorFlow.
-
-    Raises:
-      RuntimeError: If requested more GPUs per node then available or requested
-      more tasks then assigned tasks.
-    """
-
-    # check if launched by mpirun
-    if 'OMPI_COMM_WORLD_RANK' in os.environ:
-      self._rank = int(os.environ['OMPI_COMM_WORLD_RANK'])
-      num_tasks = int(os.environ['OMPI_COMM_WORLD_SIZE'])
-    else:
-      self._rank = int(os.environ['SLURM_PROCID'])
-      num_tasks = int(os.environ['SLURM_NTASKS'])
-
-    self._jobs = collections.OrderedDict(sorted(jobs.items()))
-    self._port_base = port_base
-
-    # user specification overrides SLURM specification
-    if tasks_per_node is not None:
-      self._tasks_per_node = tasks_per_node
-    elif tasks_per_node is None and 'SLURM_NTASKS_PER_NODE' in os.environ:
-      self._tasks_per_node = int(os.environ['SLURM_NTASKS_PER_NODE'])
-    else:
-      raise RuntimeError('Neither `tasks_per_node` or '
-                         'SLURM_NTASKS_PER_NODE is set.')
-
-    self._gpus_per_node = gpus_per_node
-    self._gpus_per_task = gpus_per_task
-
-    self._auto_set_gpu = auto_set_gpu
-    self.task_type = None
-    self.task_index = None
-    self.rpc_layer = rpc_layer
-
-    self._gpu_allocation = []
-    self._cluster_allocation = {}
-
-    if self._tasks_per_node * self._gpus_per_task > self._gpus_per_node:
-      raise RuntimeError('Requested more GPUs per node then available.')
-
-    if sum(self._jobs.values()) != num_tasks:
-      raise RuntimeError('Requested more tasks then assigned tasks.')
-
-  def cluster_spec(self):
-    """Returns a ClusterSpec object based on the latest instance group info.
-
-    This returns a ClusterSpec object for use based on information from the
-    specified initialization parameters and Slurm environment variables. The
-    cluster specification is resolved each time this function is called. The
-    resolver extract hostnames of nodes by scontrol and pack tasks in that
-    order until a node a has number of tasks that is equal to specification.
-    GPUs on nodes are allocated to tasks by specification through setting
-    CUDA_VISIBLE_DEVICES environment variable.
-
-    Returns:
-      A ClusterSpec containing host information retrieved from Slurm's
-        environment variables.
-    """
-    hostlist = self._resolve_hostnames()
-
-    task_list = []
-    self._gpu_allocation = []
-    self._cluster_allocation = {}
-
-    for host in hostlist:
-      for port_offset, gpu_offset in zip(
-          range(self._tasks_per_node),
-          range(0, self._gpus_per_node, self._gpus_per_task)):
-
-        host_addr = '%s:%d' % (host, self._port_base + port_offset)
-        task_list.append(host_addr)
-        gpu_id_list = []
-
-        for gpu_id in range(gpu_offset, gpu_offset + self._gpus_per_task):
-          gpu_id_list.append(str(gpu_id))
-
-        self._gpu_allocation.append(','.join(gpu_id_list))
-
-    cluster_rank_offset_start = 0
-    cluster_rank_offset_end = 0
-
-    for task_type, num_tasks in self._jobs.items():
-      cluster_rank_offset_end = cluster_rank_offset_start + num_tasks
-
-      self._cluster_allocation[task_type] = (
-          task_list[cluster_rank_offset_start:cluster_rank_offset_end])
-
-      if cluster_rank_offset_start <= self._rank < cluster_rank_offset_end:
-        self.task_type = task_type
-        self.task_index = self._rank - cluster_rank_offset_start
-
-      cluster_rank_offset_start = cluster_rank_offset_end
-
-    if self._auto_set_gpu is True:
-      os.environ['CUDA_VISIBLE_DEVICES'] = self._gpu_allocation[self._rank]
-
-    return ClusterSpec(self._cluster_allocation)
-
-  def get_task_info(self):
-    """Returns job name and task_index for the process which calls this.
-
-    This returns the job name and task index for the process which calls this
-    function according to its rank and cluster specification. The job name and
-    task index are set after a cluster is constructed by cluster_spec otherwise
-    defaults to None.
-
-    Returns:
-      A string specifying job name the process belongs to and an integner
-        specifying the task index the process belongs to in that job.
-    """
-    return self.task_type, self.task_index
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Returns the master string for connecting to a TensorFlow master.
-
-    Args:
-      task_type: (Optional) Overrides the default auto-selected task type.
-      task_index: (Optional) Overrides the default auto-slected task index.
-      rpc_layer: (Optional) Overrides the default RPC protocol TensorFlow uses
-        to communicate across nodes.
-
-    Returns:
-      A connection string for connecting to a TensorFlow master.
-    """
-    task_type = task_type if task_type is not None else self.task_type
-    task_index = task_index if task_index is not None else self.task_index
-    rpc_layer = rpc_layer or self.rpc_layer
-    master = self.cluster_spec().task_address(task_type, task_index)
-
-    return '%s://%s' % (rpc_layer, master) if rpc_layer else master
-
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in.
-
-    For users in the Slurm environment, the environment property is always an
-    empty string, and Google users will not use this ClusterResolver for running
-    on internal systems.
-    """
-    return ''
-
-  def num_accelerators_per_worker(self, session_config=None):
-    del session_config  # Unused, since this is set in __init__ manually.
-    return self._gpus_per_node
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py
index 95aad0de13..9db7f47dcb 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver.py
@@ -12,160 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Implementation of Cluster Resolvers for TF_CONFIG Environment Variables."""
-
+"""Stub file for TFConfigClusterResolver to maintain backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import json
-import os
-
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.python.training.server_lib import ClusterSpec
-
-_TF_CONFIG_ENV = 'TF_CONFIG'
-_SESSION_MASTER_KEY = 'session_master'
-_RPC_LAYER_KEY = 'rpc_layer'
-_TASK_KEY = 'task'
-
-
-def format_master_url(master, rpc_layer=None):
-  if rpc_layer:
-    return '%s://%s' % (rpc_layer, master)
-  else:
-    return master
-
-
-def _load_tf_config():
-  return json.loads(os.environ.get(_TF_CONFIG_ENV, '{}'))
-
-
-def _get_value_in_tfconfig(key, default=None):
-  tf_config = _load_tf_config()
-  return tf_config[key] if key in tf_config else default
-
-
-class TFConfigClusterResolver(ClusterResolver):
-  """Implementation of a ClusterResolver which reads the TF_CONFIG EnvVar."""
-
-  def __init__(self,
-               task_type=None,
-               task_index=None,
-               rpc_layer=None,
-               environment=None,
-               num_accelerators_per_worker=0):
-    """Creates a new TFConfigClusterResolver.
-
-    Args:
-      task_type: (String, optional) Overrides the task type specified in the
-        TF_CONFIG environment variable.
-      task_index: (Integer, optional) Overrides the task index specified in the
-        TF_CONFIG environment variable.
-      rpc_layer: (String, optional) Overrides the rpc layer TensorFlow uses.
-      environment: (String, optional) Overrides the environment TensorFlow
-        operates in.
-      num_accelerators_per_worker: (Integer, optional) Specifies the number of
-        accelerators (e.g. GPUs, TPUs, others) that each node has.
-    """
-
-    self._task_type = task_type
-    self._task_index = task_index
-    self._rpc_layer = rpc_layer
-    self._environment = environment
-    self._num_accelerators_per_worker = num_accelerators_per_worker
-
-  @property
-  def task_type(self):
-    if self._task_type is None:
-      task_info = _get_value_in_tfconfig(_TASK_KEY, {})
-      return task_info['type'] if 'type' in task_info else None
-    else:
-      return self._task_type
-
-  @property
-  def task_index(self):
-    if self._task_type is None:
-      task_info = _get_value_in_tfconfig(_TASK_KEY, {})
-      return task_info['index'] if 'index' in task_info else None
-    else:
-      return self._task_index
-
-  @task_type.setter
-  def task_type(self, task_type):
-    self._task_type = task_type
-
-  @task_index.setter
-  def task_index(self, task_index):
-    self._task_index = task_index
-
-  @property
-  def environment(self):
-    return self._environment
-
-  @property
-  def rpc_layer(self):
-    if self._rpc_layer is None:
-      return _get_value_in_tfconfig(_RPC_LAYER_KEY)
-    else:
-      return self._rpc_layer
-
-  @rpc_layer.setter
-  def rpc_layer(self, rpc_layer):
-    self._rpc_layer = rpc_layer
-
-  def num_accelerators_per_worker(self, session_config=None):
-    # TODO(frankchn): Connect to server (w/ session_config) in the future.
-    del session_config  # Unused, we do not connect to another server here.
-    return self._num_accelerators_per_worker
-
-  def cluster_spec(self):
-    """Returns a ClusterSpec based on the TF_CONFIG environment variable.
-
-    Returns:
-      A ClusterSpec with information from the TF_CONFIG environment variable.
-    """
-    tf_config = _load_tf_config()
-    if 'cluster' not in tf_config:
-      return ClusterSpec({})
-    return ClusterSpec(tf_config['cluster'])
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Returns the master address to use when creating a TensorFlow session.
-
-    Args:
-      task_type: (String, optional) Overrides and sets the task_type of the
-        master.
-      task_index: (Integer, optional) Overrides and sets the task id of the
-        master.
-      rpc_layer: (String, optional) Overrides and sets the protocol over which
-        TensorFlow nodes communicate with each other.
-
-    Returns:
-      The address of the master.
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
 
-    Raises:
-      RuntimeError: If the task_type or task_id is not specified and the
-        `TF_CONFIG` environment variable does not contain a task section.
-    """
+# pylint: disable=unused-import
+from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
+# pylint: enable=unused-import
 
-    # If `session_master` is set, just use that.
-    session_master = _get_value_in_tfconfig(_SESSION_MASTER_KEY)
-    if session_master is not None:
-      return session_master
+from tensorflow.python.util.all_util import remove_undocumented
 
-    # Return an empty string if we are the only job in the ClusterSpec.
-    cluster_spec = self.cluster_spec()
-    if (not cluster_spec.jobs or
-        (len(cluster_spec.jobs) == 1 and
-         len(cluster_spec.job_tasks(cluster_spec.jobs[0])) == 1)):
-      return ''
+_allowed_symbols = [
+    'TFConfigClusterResolver',
+]
 
-    # We try to auto-detect the task type and id, but uses the user-supplied one
-    # where available
-    task_type = task_type if task_type is not None else self.task_type
-    task_index = task_index if task_index is not None else self.task_index
+remove_undocumented(__name__, _allowed_symbols)
 
-    return format_master_url(cluster_spec.task_address(task_type, task_index),
-                             self.rpc_layer)
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
index d5537a4100..3a1eaccd06 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,412 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Implementation of Cluster Resolvers for Cloud TPUs."""
+"""Stub file for TPUClusterResolver to maintain backwards compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
+# This file (and all files in this directory in general) is a backwards
+# compatibility shim that exists to re-export ClusterResolvers such that
+# existing OSS code will not be broken.
 
-from six.moves.urllib.request import Request
-from six.moves.urllib.request import urlopen
+# pylint: disable=unused-import
+from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver
+# pylint: enable=unused-import
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import ClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import format_master_url
-from tensorflow.python.training import server_lib
-from tensorflow.python.util import compat
+from tensorflow.python.util.all_util import remove_undocumented
 
-_GOOGLE_API_CLIENT_INSTALLED = True
-try:
-  from googleapiclient import discovery  # pylint: disable=g-import-not-at-top
-  from oauth2client.client import GoogleCredentials  # pylint: disable=g-import-not-at-top
-except ImportError:
-  _GOOGLE_API_CLIENT_INSTALLED = False
+_allowed_symbols = [
+    'TPUClusterResolver',
+]
 
-
-_GKE_ENV_VARIABLE = 'KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS'
-_ENDPOINTS_SEPARATOR = ','
-_DEFAULT_ENV_VARIABLE = 'TPU_NAME'
-_DISCOVERY_SERVICE_URL_ENV_VARIABLE = 'TPU_API_DISCOVERY_URL'
-
-
-class TPUClusterResolver(ClusterResolver):
-  """Cluster Resolver for Google Cloud TPUs.
-
-  This is an implementation of cluster resolvers for the Google Cloud TPU
-  service. As Cloud TPUs are in alpha, you will need to specify a API definition
-  file for this to consume, in addition to a list of Cloud TPUs in your Google
-  Cloud Platform project.
-  """
-
-  def _tpuService(self):
-    """Creates a new Cloud TPU API object.
-
-    This works around an issue where the underlying HTTP connection sometimes
-    times out when the script has been running for too long. Other methods in
-    this object calls this method to get a new API object whenever they need
-    to communicate with the Cloud API.
-
-    Returns:
-      A Google Cloud TPU API object.
-    """
-    if self._service:
-      return self._service
-
-    credentials = self._credentials
-    if credentials is None or credentials == 'default':
-      credentials = GoogleCredentials.get_application_default()
-
-    if self._discovery_url:
-      return discovery.build(
-          'tpu', 'v1alpha1',
-          credentials=credentials,
-          discoveryServiceUrl=self._discovery_url)
-    else:
-      return discovery.build(
-          'tpu', 'v1alpha1',
-          credentials=credentials)
-
-  def _requestComputeMetadata(self, path):
-    req = Request('http://metadata/computeMetadata/v1/%s' % path,
-                  headers={'Metadata-Flavor': 'Google'})
-    resp = urlopen(req)
-    return compat.as_bytes(resp.read())
-
-  def _shouldResolve(self):
-    if isinstance(self._should_resolve_override, bool):
-      return self._should_resolve_override
-    if (self._tpu == compat.as_bytes('') or
-        self._tpu == compat.as_bytes('local') or
-        self._tpu.startswith(compat.as_bytes('/bns')) or
-        self._tpu.startswith(compat.as_bytes('localhost:')) or
-        self._tpu.startswith(compat.as_bytes('grpc://'))):
-      return False
-    return True
-
-  @staticmethod
-  def _inGke():
-    """When running in GKE, the environment variable will be set."""
-    return _GKE_ENV_VARIABLE in os.environ
-
-  @staticmethod
-  def _gkeEndpoints():
-    return os.environ[_GKE_ENV_VARIABLE]
-
-  @staticmethod
-  def _envVarFallback():
-    if _DEFAULT_ENV_VARIABLE in os.environ:
-      return os.environ[_DEFAULT_ENV_VARIABLE]
-    return None
-
-  @staticmethod
-  def _environmentDiscoveryUrl():
-    return os.environ.get(_DISCOVERY_SERVICE_URL_ENV_VARIABLE)
-
-  def __init__(self,
-               tpu=None,
-               zone=None,
-               project=None,
-               job_name='worker',
-               coordinator_name=None,
-               coordinator_address=None,
-               credentials='default',
-               service=None,
-               discovery_url=None):
-    """Creates a new TPUClusterResolver object.
-
-    The ClusterResolver will then use the parameters to query the Cloud TPU APIs
-    for the IP addresses and ports of each Cloud TPU listed.
-
-    Args:
-      tpu: Either a string, or a list of strings corresponding to the TPUs to
-        use. If the single string is the empty string, the string 'local', or a
-        string that begins with 'grpc://' or '/bns', then it is assumed to not
-        correspond with a Cloud TPU and will instead be passed as the session
-        master and no ClusterSpec propagation will be done.
-      zone: Zone where the TPUs are located. If omitted or empty, we will assume
-        that the zone of the TPU is the same as the zone of the GCE VM, which we
-        will try to discover from the GCE metadata service.
-      project: Name of the GCP project containing Cloud TPUs. If omitted or
-        empty, we will try to discover the project name of the GCE VM from the
-        GCE metadata service.
-      job_name: Name of the TensorFlow job the TPUs belong to.
-      coordinator_name: The name to use for the coordinator. Set to None if the
-        coordinator should not be included in the computed ClusterSpec.
-      coordinator_address: The address of the coordinator (typically an ip:port
-        pair). If set to None, a TF server will be started. If coordinator_name
-        is None, a TF server will not be started even if coordinator_address is
-        None.
-      credentials: GCE Credentials. If None, then we use default credentials
-        from the oauth2client
-      service: The GCE API object returned by the googleapiclient.discovery
-        function. If you specify a custom service object, then the credentials
-        parameter will be ignored.
-      discovery_url: A URL template that points to the location of
-        the discovery service. It should have two parameters {api} and
-        {apiVersion} that when filled in produce an absolute URL to the
-        discovery document for that service. The environment variable
-        'TPU_API_DISCOVERY_URL' will override this.
-
-    Raises:
-      ImportError: If the googleapiclient is not installed.
-      ValueError: If no TPUs are specified.
-    """
-    if isinstance(tpu, list):
-      if not tpu:
-        raise ValueError('At least one TPU must be specified.')
-      if len(tpu) != 1:
-        raise NotImplementedError(
-            'Using multiple TPUs in a single session is not yet implemented')
-      tpu = tpu[0]
-
-    in_gke = self._inGke()
-    # When using GKE with Cloud TPUs, the env variable will be set.
-    if tpu is None:
-      if in_gke:
-        tpu = self._gkeEndpoints()
-      else:
-        tpu = self._envVarFallback()
-
-    if tpu is None:
-      raise ValueError('Please provide a TPU Name to connect to.')
-
-    self._tpu = compat.as_bytes(tpu)  # self._tpu is always bytes
-
-    # By default the task_type is 'worker` and the task_index is 0 (which is the
-    # first worker in the task).
-    self.task_type = job_name
-    self.task_index = 0
-
-    if tpu.startswith('grpc://'):
-      # Cloud environment, where we are using GRPC to communicate to TPUs.
-      self._environment = ''
-    elif tpu == 'local' or not tpu:
-      # Google environment, where the TPU is attached to the host.
-      self._environment = 'google'
-    elif tpu.startswith('/bns'):
-      # Google environment, where we reach the TPU through BNS.
-      self._environment = 'google'
-
-    # If TPU is in the Google environment or exists locally, we don't use any
-    # RPC layer.
-    if tpu.startswith('/bns') or tpu == 'local' or not tpu:
-      self.rpc_layer = None
-    else:
-      self.rpc_layer = 'grpc'
-
-    # Setting this overrides the return value of self._shouldResolve()
-    self._should_resolve_override = None
-
-    # We strip out the protocol if it is included, and override the
-    # shouldResolve function to never resolve. We are adding the protocol back
-    # in later in self.master().
-    if self.rpc_layer is not None and tpu.startswith(self.rpc_layer + '://'):
-      tpu = tpu[len(self.rpc_layer + '://'):]
-      self._tpu = tpu
-      self._should_resolve_override = False
-
-    # Whether we should actually attempt to contact Cloud APIs
-    should_resolve = self._shouldResolve()
-
-    # We error out if we are in a non-Cloud environment which cannot talk to the
-    # Cloud APIs using the standard class and a special object is not passed in.
-    self._service = service
-    if (self._service is None and should_resolve and
-        not _GOOGLE_API_CLIENT_INSTALLED):
-      raise ImportError('googleapiclient and oauth2client must be installed '
-                        'before using the TPU cluster resolver. Execute: '
-                        '`pip install --upgrade google-api-python-client` '
-                        'and `pip install --upgrade oauth2client` to '
-                        'install with pip.')
-
-    # We save user-passed credentials, unless the user didn't pass in anything.
-    self._credentials = credentials
-    if (credentials == 'default' and should_resolve and
-        _GOOGLE_API_CLIENT_INSTALLED):
-      self._credentials = None
-
-    # Automatically detect project and zone if unspecified.
-    if not project and should_resolve:
-      project = compat.as_str(
-          self._requestComputeMetadata('project/project-id'))
-    if not zone and should_resolve:
-      zone_path = compat.as_str(self._requestComputeMetadata('instance/zone'))
-      zone = zone_path.split('/')[-1]
-    self._project = project
-    self._zone = zone
-
-    self._discovery_url = self._environmentDiscoveryUrl() or discovery_url
-
-    self._coordinator_name = coordinator_name
-    if (coordinator_name and not coordinator_address and
-        (should_resolve or in_gke)):
-      self._start_local_server()
-    else:
-      self._coordinator_address = coordinator_address
-
-  def master(self, task_type=None, task_index=None, rpc_layer=None):
-    """Get the Master string to be used for the session.
-
-    In the normal case, this returns the grpc path (grpc://1.2.3.4:8470) of
-    first instance in the ClusterSpec returned by the cluster_spec function.
-
-    If a non-TPU name is used when constructing a TPUClusterResolver, that will
-    be returned instead (e.g. If the tpus argument's value when constructing
-    this TPUClusterResolver was 'grpc://10.240.1.2:8470',
-    'grpc://10.240.1.2:8470' will be returned).
-
-    Args:
-      task_type: (Optional, string) The type of the TensorFlow task of the
-        master.
-      task_index: (Optional, integer) The index of the TensorFlow task of the
-        master.
-      rpc_layer: (Optional, string) The RPC protocol TensorFlow should use to
-        communicate with TPUs.
-
-    Returns:
-      string, the connection string to use when creating a session.
-
-    Raises:
-      ValueError: If none of the TPUs specified exists.
-    """
-    if self._shouldResolve():
-      # We are going to communicate with the Cloud TPU APIs to get a Cluster.
-      cluster_spec = self.cluster_spec()
-      if task_type is not None and task_index is not None:
-        # task_type and task_index is from the function parameter
-        master = cluster_spec.task_address(task_type, task_index)
-      elif self.task_type is not None and self.task_index is not None:
-        # task_type and task_index is from the object
-        master = cluster_spec.task_address(self.task_type, self.task_index)
-      else:
-        # by default we take the first item in the cluster with the right name
-        job_tasks = cluster_spec.job_tasks(self.task_type)
-        if not job_tasks:
-          raise ValueError('No TPUs with the specified names exist.')
-        master = job_tasks[0]
-    else:
-      if isinstance(self._tpu, (bytes, bytearray)):
-        master = self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0]
-      else:
-        master = self._tpu.split(_ENDPOINTS_SEPARATOR)[0]
-    return format_master_url(master, rpc_layer or self.rpc_layer)
-
-  def get_master(self):
-    return self.master()
-
-  def get_job_name(self):
-    if self._shouldResolve():
-      return self.task_type
-
-  def cluster_spec(self):
-    """Returns a ClusterSpec object based on the latest TPU information.
-
-    We retrieve the information from the GCE APIs every time this method is
-    called.
-
-    Returns:
-      A ClusterSpec containing host information returned from Cloud TPUs.
-
-    Raises:
-      RuntimeError: If the provided TPU is not healthy.
-    """
-    ############################################################################
-    # There are 5 potential cases this code must handle:
-    #  1. [Normal case.] We should resolve the TPU name to a set of tasks, and
-    #      a. Create a ClusterSpec that includes the coordinator job
-    #      b. Create a ClusterSpec without the coordinator job.
-    #  2. [GKE / No API Access.] We should not resolve the TPU name to a set of
-    #     tasks and
-    #      a. Create a ClusterSpec with the coordinator
-    #      b. Create a ClusterSpec without the coordinator
-    #  3. [Other (legacy non-gRPC).] We should return an empty ClusterSpec.
-    ############################################################################
-
-    if self._shouldResolve():
-      # Case 1.
-      full_name = 'projects/%s/locations/%s/nodes/%s' % (
-          self._project, self._zone, compat.as_text(self._tpu))
-      service = self._tpuService()
-      request = service.projects().locations().nodes().get(name=full_name)
-      response = request.execute()
-
-      if 'state' in response and response['state'] != 'READY':
-        raise RuntimeError('TPU "%s" is not yet ready; state: "%s"' %
-                           (compat.as_text(self._tpu), response['state']))
-
-      if 'health' in response and response['health'] != 'HEALTHY':
-        raise RuntimeError('TPU "%s" is unhealthy: "%s"' %
-                           (compat.as_text(self._tpu), response['health']))
-
-      if 'networkEndpoints' in response:
-        worker_list = [
-            '%s:%s' % (endpoint['ipAddress'], endpoint['port'])
-            for endpoint in response['networkEndpoints']
-        ]
-      else:
-        # Fall back to the deprecated response format
-        instance_url = '%s:%s' % (response['ipAddress'], response['port'])
-        worker_list = [instance_url]
-
-      cluster_spec = {self.task_type: worker_list}
-    else:
-      if self.rpc_layer is None:
-        # Case 3.
-        return None
-      # Case 2.
-      tpus = []
-      for tpu in self._tpu.split(_ENDPOINTS_SEPARATOR):
-        # We are working around the fact that GKE environment variable that is
-        # supplied to us has the protocol string embedded in it, but we want
-        # to strip it out for the ClusterSpec.
-        if (self.rpc_layer is not None and
-            tpu.startswith(self.rpc_layer + '://')):
-          tpus.append(tpu[len(self.rpc_layer + '://'):])
-        else:
-          tpus.append(tpu)
-      cluster_spec = {self.task_type: tpus}
-
-    if self._coordinator_address:
-      # {1, 2}.a
-      cluster_spec[self._coordinator_name] = [self._coordinator_address]
-
-    return server_lib.ClusterSpec(cluster_spec)
-
-  def num_accelerators_per_worker(self, session_config=None):
-    """Returns the number of TPU cores per worker.
-
-    This defaults to 8 for all current TPU configurations, and we do not need
-    to query any remote systems for this.
-
-    Args:
-      session_config: Unused. Not currently necessary to query anything as this
-        number is 8 for all TPU configurations.
-    """
-    del session_config  # Unused. Not necessary to query anything.
-    return 8
-
-  @property
-  def environment(self):
-    """Returns the current environment which TensorFlow is running in."""
-    return self._environment
-
-  def _start_local_server(self):
-    address = self._requestComputeMetadata('instance/network-interfaces/0/ip')
-    self._server = server_lib.Server(
-        {
-            'local': ['0.0.0.0:0']
-        }, protocol='grpc', config=None, start=True)
-    # self._server.target is of the form: grpc://ipaddress:port
-    target = compat.as_bytes(self._server.target)
-    splits = target.split(compat.as_bytes(':'))
-    assert len(splits) == 3, self._server.target
-    assert splits[0] == compat.as_bytes('grpc'), self._server.target
-    self._coordinator_port = compat.as_text(splits[2])
-    self._coordinator_address = '%s:%s' % (
-        address, compat.as_text(self._coordinator_port))
-
-  def __deepcopy__(self, memo):
-    # TODO(b/73668574): Remove this once RunConfig avoids performing deepcopy.
-    return self
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 999274018b..05d2ebd2e8 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -216,7 +216,7 @@ py_library(
     ],
     deps = [
         ":tpu_lib",
-        "//tensorflow/contrib/cluster_resolver:tpu_cluster_resolver_py",
+        "//tensorflow/contrib/cluster_resolver:cluster_resolver_py",
         "//tensorflow/contrib/distribute",
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/tpu/proto:compilation_result_proto_py",
@@ -264,7 +264,7 @@ py_library(
         ":tpu_py",
         "//tensorflow/compiler/xla/experimental/xla_sharding",
         "//tensorflow/compiler/xla/python_api:xla_shape",
-        "//tensorflow/contrib/cluster_resolver:tpu_cluster_resolver_py",
+        "//tensorflow/contrib/cluster_resolver:cluster_resolver_py",
         "//tensorflow/contrib/compiler:xla",
         "//tensorflow/contrib/tpu/proto:compilation_result_proto_py",
         "//tensorflow/contrib/tpu/proto:optimization_parameters_proto_py",
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 6e4945e6fd..92aac5fc0a 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3534,6 +3534,19 @@ py_library(
     ],
 )
 
+# Dependency added and used by ClusterResolvers to avoid circular dependency between keras, distribute, and training.
+py_library(
+    name = "training_server_lib",
+    srcs = ["training/server_lib.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework",
+        ":pywrap_tensorflow",
+        ":util",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
 py_library(
     name = "saveable_object",
     srcs = ["training/saveable_object.py"],
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index ec438d00f0..2d9a1764db 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -124,6 +124,7 @@ py_library(
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
         "//tensorflow/python/ops/losses",
         "//tensorflow/tools/docs:doc_controls",
     ],
diff --git a/tensorflow/python/distribute/cluster_resolver/BUILD b/tensorflow/python/distribute/cluster_resolver/BUILD
new file mode 100644
index 0000000000..360a2993cd
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/BUILD
@@ -0,0 +1,180 @@
+# Description: Operations defined for Cluster Resolvers
+
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+
+package(
+    default_visibility = [
+        "//tensorflow:__subpackages__",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+py_library(
+    name = "cluster_resolver_lib",
+    srcs = [
+        "__init__.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":base_cluster_resolver_py",
+        ":gce_cluster_resolver_py",
+        ":kubernetes_cluster_resolver_py",
+        ":slurm_cluster_resolver_py",
+        ":tfconfig_cluster_resolver_py",
+        ":tpu_cluster_resolver_py",
+        "//tensorflow/python:util",
+    ],
+)
+
+py_library(
+    name = "base_cluster_resolver_py",
+    srcs = ["cluster_resolver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:training_server_lib",
+    ],
+)
+
+py_library(
+    name = "gce_cluster_resolver_py",
+    srcs = ["gce_cluster_resolver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":base_cluster_resolver_py",
+        "//tensorflow/python:training_server_lib",
+    ],
+)
+
+py_library(
+    name = "tfconfig_cluster_resolver_py",
+    srcs = ["tfconfig_cluster_resolver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":base_cluster_resolver_py",
+        "//tensorflow/python:training_server_lib",
+    ],
+)
+
+py_library(
+    name = "tpu_cluster_resolver_py",
+    srcs = ["tpu_cluster_resolver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":base_cluster_resolver_py",
+        "//tensorflow/python:training_server_lib",
+    ],
+)
+
+py_library(
+    name = "slurm_cluster_resolver_py",
+    srcs = ["slurm_cluster_resolver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":base_cluster_resolver_py",
+        "//tensorflow/python:training_server_lib",
+    ],
+)
+
+py_library(
+    name = "kubernetes_cluster_resolver_py",
+    srcs = ["kubernetes_cluster_resolver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":base_cluster_resolver_py",
+        "//tensorflow/python:training_server_lib",
+    ],
+)
+
+tf_py_test(
+    name = "base_cluster_resolver_py_test",
+    srcs = ["cluster_resolver_test.py"],
+    additional_deps = [
+        ":base_cluster_resolver_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:training_server_lib",
+    ],
+    main = "cluster_resolver_test.py",
+)
+
+tf_py_test(
+    name = "gce_cluster_resolver_py_test",
+    size = "small",
+    srcs = ["gce_cluster_resolver_test.py"],
+    additional_deps = [
+        ":gce_cluster_resolver_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:training_server_lib",
+    ],
+    main = "gce_cluster_resolver_test.py",
+)
+
+tf_py_test(
+    name = "tfconfig_cluster_resolver_py_test",
+    size = "small",
+    srcs = ["tfconfig_cluster_resolver_test.py"],
+    additional_deps = [
+        ":tfconfig_cluster_resolver_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:training_server_lib",
+    ],
+    grpc_enabled = True,
+    main = "tfconfig_cluster_resolver_test.py",
+)
+
+tf_py_test(
+    name = "tpu_cluster_resolver_py_test",
+    size = "small",
+    srcs = ["tpu_cluster_resolver_test.py"],
+    additional_deps = [
+        ":tpu_cluster_resolver_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:training_server_lib",
+    ],
+    grpc_enabled = True,
+    main = "tpu_cluster_resolver_test.py",
+)
+
+tf_py_test(
+    name = "slurm_cluster_resolver_py_test",
+    size = "small",
+    srcs = ["slurm_cluster_resolver_test.py"],
+    additional_deps = [
+        ":slurm_cluster_resolver_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:training_server_lib",
+    ],
+    main = "slurm_cluster_resolver_test.py",
+    tags = [],
+)
+
+tf_py_test(
+    name = "kubernetes_cluster_resolver_py_test",
+    size = "small",
+    srcs = ["kubernetes_cluster_resolver_test.py"],
+    additional_deps = [
+        ":kubernetes_cluster_resolver_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:training_server_lib",
+    ],
+    main = "kubernetes_cluster_resolver_test.py",
+)
diff --git a/tensorflow/contrib/cluster_resolver/README.md b/tensorflow/python/distribute/cluster_resolver/README.md
similarity index 100%
rename from tensorflow/contrib/cluster_resolver/README.md
rename to tensorflow/python/distribute/cluster_resolver/README.md
diff --git a/tensorflow/contrib/cluster_resolver/python/training/README.slurm b/tensorflow/python/distribute/cluster_resolver/README.slurm
similarity index 100%
rename from tensorflow/contrib/cluster_resolver/python/training/README.slurm
rename to tensorflow/python/distribute/cluster_resolver/README.slurm
diff --git a/tensorflow/python/distribute/cluster_resolver/__init__.py b/tensorflow/python/distribute/cluster_resolver/__init__.py
new file mode 100644
index 0000000000..ef87f59b7f
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/__init__.py
@@ -0,0 +1,57 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library Imports for Cluster Resolvers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.distribute.cluster_resolver import cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import gce_cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import kubernetes_cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import slurm_cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import tfconfig_cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver
+
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import UnionClusterResolver
+from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver.kubernetes_cluster_resolver import KubernetesClusterResolver
+from tensorflow.python.distribute.cluster_resolver.slurm_cluster_resolver import SlurmClusterResolver
+from tensorflow.python.distribute.cluster_resolver.tfconfig_cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver import TPUClusterResolver
+
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = [
+    'cluster_resolver',
+    'gce_cluster_resolver',
+    'kubernetes_cluster_resolver',
+    'slurm_cluster_resolver',
+    'tfconfig_cluster_resolver',
+    'tpu_cluster_resolver',
+    'ClusterResolver',
+    'SimpleClusterResolver',
+    'UnionClusterResolver',
+    'GceClusterResolver',
+    'KubernetesClusterResolver',
+    'TFConfigClusterResolver',
+    'TPUClusterResolver',
+    'SlurmClusterResolver',
+]
+
+remove_undocumented(__name__, _allowed_symbols)
+
diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
new file mode 100644
index 0000000000..7774ac0e12
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
@@ -0,0 +1,374 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Cluster Resolvers are used for dynamic cluster IP/hostname resolution."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import abc
+
+import six
+
+from tensorflow.python.training.server_lib import ClusterSpec
+
+
+def format_master_url(master, rpc_layer=None):
+  if rpc_layer:
+    return '%s://%s' % (rpc_layer, master)
+  else:
+    return master
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ClusterResolver(object):
+  """Abstract class for all implementations of ClusterResolvers.
+
+  This defines the skeleton for all implementations of ClusterResolvers.
+  ClusterResolvers are a way for TensorFlow to communicate with various cluster
+  management systems (e.g. GCE, AWS, etc...).
+
+  By letting TensorFlow communicate with these systems, we will be able to
+  automatically discover and resolve IP addresses for various TensorFlow
+  workers. This will eventually allow us to automatically recover from
+  underlying machine failures and scale TensorFlow worker clusters up and down.
+
+  Note to Implementors: In addition to these abstract methods, you must also
+  implement the task_type, task_index, and rpc_layer attributes. You may choose
+  to implement them either as properties with getters or setters or directly
+  set the attributes.
+
+  - task_type is the name of the server's current named job (e.g. 'worker',
+     'ps' in a distributed parameterized training job).
+  - task_index is the ordinal index of the server within the task type.
+  - rpc_layer is the protocol used by TensorFlow to communicate with other
+      TensorFlow servers in a distributed environment.
+  """
+
+  @abc.abstractmethod
+  def cluster_spec(self):
+    """Retrieve the current state of the cluster and returns a ClusterSpec.
+
+    Returns:
+      A ClusterSpec representing the state of the cluster at the moment this
+      function is called.
+
+    Implementors of this function must take care in ensuring that the
+    ClusterSpec returned is up-to-date at the time of calling this function.
+    This usually means retrieving the information from the underlying cluster
+    management system every time this function is invoked and reconstructing
+    a cluster_spec, rather than attempting to cache anything.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Retrieves the name or URL of the session master.
+
+    Args:
+      task_type: (Optional) The type of the TensorFlow task of the master.
+      task_index: (Optional) The index of the TensorFlow task of the master.
+      rpc_layer: (Optional) The RPC protocol for the given cluster.
+
+    Returns:
+      The name or URL of the session master.
+
+    Implementors of this function must take care in ensuring that the master
+    returned is up-to-date at the time to calling this function. This usually
+    means retrieving the master every time this function is invoked.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def num_accelerators_per_worker(self, session_config=None):
+    """Returns the number of accelerator cores per worker.
+
+    This returns the number of accelerator cores (such as GPUs and TPUs)
+    available per worker. If workers only has CPU cores available, then this
+    should return 0. This method will query the master for this information
+    if it is not otherwise known.
+
+    Args:
+      session_config: (Optional) Configuration for starting a new session to
+        query how many accelerator cores it has.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractproperty
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in."""
+    raise NotImplementedError()
+
+
+class SimpleClusterResolver(ClusterResolver):
+  """Simple implementation of ClusterResolver that accepts a ClusterSpec."""
+
+  def __init__(self, cluster_spec, master='', task_type=None, task_index=None,
+               environment='', num_accelerators_per_worker=0,
+               rpc_layer=None):
+    """Creates a SimpleClusterResolver from a ClusterSpec."""
+    super(SimpleClusterResolver, self).__init__()
+
+    self._task_type = task_type
+    self._task_index = task_index
+    self._environment = environment
+    self._num_accelerators_per_worker = num_accelerators_per_worker
+    self._rpc_layer = rpc_layer
+
+    if not isinstance(cluster_spec, ClusterSpec):
+      raise TypeError('cluster_spec must be a ClusterSpec.')
+    self._cluster_spec = cluster_spec
+
+    if not isinstance(master, str):
+      raise TypeError('master must be a string.')
+    self._master = master
+
+  def cluster_spec(self):
+    """Returns the ClusterSpec passed into the constructor."""
+    return self._cluster_spec
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master address to use when creating a session.
+
+    Args:
+      task_type: (Optional) The type of the TensorFlow task of the master.
+      task_index: (Optional) The index of the TensorFlow task of the master.
+      rpc_layer: (Optional) The RPC used by distributed TensorFlow.
+
+    Returns:
+      The name or URL of the session master.
+
+    If a task_type and task_index is given, this will override the `master`
+    string passed into the initialization function.
+    """
+    if task_type is not None and task_index is not None:
+      master = self.cluster_spec().task_address(task_type, task_index)
+    else:
+      master = self._master
+
+    return format_master_url(master, rpc_layer=rpc_layer or self._rpc_layer)
+
+  @property
+  def task_type(self):
+    return self._task_type
+
+  @property
+  def task_index(self):
+    return self._task_index
+
+  @task_type.setter
+  def task_type(self, task_type):
+    self._task_type = task_type
+
+  @task_index.setter
+  def task_index(self, task_index):
+    self._task_index = task_index
+
+  @property
+  def environment(self):
+    return self._environment
+
+  def num_accelerators_per_worker(self, session_config=None):
+    """Returns the number of accelerator cores per worker.
+
+    Args:
+      session_config: Unused. The SimpleClusterResolver does not do automatic
+        detection of accelerators, so a TensorFlow session will never be
+        created, and thus a `session_config` is never necessary here, and will
+        be ignored.
+    """
+    del session_config
+    return self._num_accelerators_per_worker
+
+  @property
+  def rpc_layer(self):
+    return self._rpc_layer
+
+  @rpc_layer.setter
+  def rpc_layer(self, rpc_layer):
+    self._rpc_layer = rpc_layer
+
+
+class UnionClusterResolver(ClusterResolver):
+  """Performs a union on underlying ClusterResolvers.
+
+  This class performs a union given two or more existing ClusterResolvers. It
+  merges the underlying ClusterResolvers, and returns one unified ClusterSpec
+  when cluster_spec is called. The details of the merge function is
+  documented in the cluster_spec function.
+
+  For additional Cluster Resolver properties such as task type, task index,
+  rpc layer, environment, etc..., we will return the value from the first
+  ClusterResolver in the union.
+  """
+
+  def __init__(self, *args, **kwargs):
+    """Initializes a UnionClusterResolver with other ClusterResolvers.
+
+    Args:
+      *args: `ClusterResolver` objects to be unionized.
+      **kwargs:
+        rpc_layer - (Optional) Override value for the RPC layer used by
+          TensorFlow.
+        task_type - (Optional) Override value for the current task type.
+        task_index - (Optional) Override value for the current task index.
+
+    Raises:
+      TypeError: If any argument is not a subclass of `ClusterResolvers`.
+      ValueError: If there are no arguments passed.
+    """
+    super(UnionClusterResolver, self).__init__()
+
+    self._rpc_layer = kwargs.pop('rpc_layer', None)
+    self._task_type = kwargs.pop('task_type', None)
+    self._task_index = kwargs.pop('task_index', None)
+
+    if kwargs:
+      raise ValueError('Unexpected kwargs provided {!r}'.format(kwargs))
+
+    if not args:
+      raise ValueError('At least one ClusterResolver is required.')
+
+    for cluster_resolver in args:
+      if not isinstance(cluster_resolver, ClusterResolver):
+        raise TypeError('All arguments must be a sub-class of '
+                        '`ClusterResolver.`')
+    self._cluster_resolvers = args
+
+  def cluster_spec(self):
+    """Returns a union of all the ClusterSpecs from the ClusterResolvers.
+
+    Returns:
+      A ClusterSpec containing host information merged from all the underlying
+      ClusterResolvers.
+
+    Raises:
+      KeyError: If there are conflicting keys detected when merging two or
+      more dictionaries, this exception is raised.
+
+    Note: If there are multiple ClusterResolvers exposing ClusterSpecs with the
+    same job name, we will merge the list/dict of workers.
+
+    If *all* underlying ClusterSpecs expose the set of workers as lists, we will
+    concatenate the lists of workers, starting with the list of workers from
+    the first ClusterResolver passed into the constructor.
+
+    If *any* of the ClusterSpecs expose the set of workers as a dict, we will
+    treat all the sets of workers as dicts (even if they are returned as lists)
+    and will only merge them into a dict if there is no conflicting keys. If
+    there is a conflicting key, we will raise a `KeyError`.
+    """
+
+    merged_cluster = {}
+
+    # We figure out whether it is all lists for a particular job, or whether
+    # there are dicts inside.
+    for cluster_resolver in self._cluster_resolvers:
+      cluster_spec = cluster_resolver.cluster_spec()
+      cluster_dict = cluster_spec.as_dict()
+
+      for job_name, tasks in cluster_dict.items():
+        if job_name in merged_cluster:
+          # If we see a dict, then we write a dict out regardless.
+          if isinstance(tasks, dict):
+            merged_cluster[job_name] = {}
+        else:
+          # We take whichever type is present.
+          if isinstance(tasks, list):
+            merged_cluster[job_name] = []
+          else:
+            merged_cluster[job_name] = {}
+
+    # We then do the merge as appropriate in merged_cluster[job].
+    for cluster_resolver in self._cluster_resolvers:
+      cluster_spec = cluster_resolver.cluster_spec()
+      cluster_dict = cluster_spec.as_dict()
+
+      for job_name, tasks in cluster_dict.items():
+        if isinstance(merged_cluster[job_name], list):
+          # We all have lists, we can just concatenate and be done.
+          merged_cluster[job_name].extend(tasks)
+        else:
+          if isinstance(tasks, list):
+            # We convert to a dictionary if the type is a list.
+            task_dict = dict(zip(range(0, len(tasks)), tasks))
+          else:
+            # We can simply make a copy (for update) and be done.
+            task_dict = tasks.copy()
+
+          # We detect if there are duplicates, and raise an error if so.
+          task_keys = set(task_dict)
+          merged_keys = set(merged_cluster[job_name].keys())
+          intersected_keys = task_keys.intersection(merged_keys)
+          if intersected_keys:
+            raise KeyError('Duplicate keys detected when merging two '
+                           'ClusterSpecs: %s' % repr(intersected_keys))
+
+          # We do the merge after all the processing.
+          merged_cluster[job_name].update(task_dict)
+
+    return ClusterSpec(merged_cluster)
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master address to use when creating a session.
+
+    This usually returns the master from the first ClusterResolver passed in,
+    but you can override this by specifying the task_type and task_index.
+
+    Args:
+      task_type: (Optional) The type of the TensorFlow task of the master.
+      task_index: (Optional) The index of the TensorFlow task of the master.
+      rpc_layer: (Optional) The RPC protocol for the given cluster.
+
+    Returns:
+      The name or URL of the session master.
+    """
+    if task_type is not None and task_index is not None:
+      master = self.cluster_spec().task_address(task_type, task_index)
+      return format_master_url(master, rpc_layer or self._rpc_layer)
+
+    return self._cluster_resolvers[0].master(rpc_layer=rpc_layer)
+
+  @property
+  def task_type(self):
+    return self._task_type or self._cluster_resolvers[0].task_type
+
+  @property
+  def task_index(self):
+    return self._task_index or self._cluster_resolvers[0].task_index
+
+  @task_type.setter
+  def task_type(self, task_type):
+    self._task_type = task_type
+
+  @task_index.setter
+  def task_index(self, task_index):
+    self._task_index = task_index
+
+  @property
+  def environment(self):
+    return self._cluster_resolvers[0].environment
+
+  def num_accelerators_per_worker(self, session_config=None):
+    return self._cluster_resolvers[0].num_accelerators_per_worker(
+        session_config)
+
+  @property
+  def rpc_layer(self):
+    return self._rpc_layer or self._cluster_resolvers[0].rpc_layer
+
+  @rpc_layer.setter
+  def rpc_layer(self, rpc_layer):
+    self._rpc_layer = rpc_layer
diff --git a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
similarity index 98%
rename from tensorflow/contrib/cluster_resolver/python/training/cluster_resolver_test.py
rename to tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
index b94c9612b5..b5448faec6 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import SimpleClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import UnionClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
new file mode 100644
index 0000000000..b167bc8fc8
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
@@ -0,0 +1,206 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implementation of Cluster Resolvers for GCE Instance Groups."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.training.server_lib import ClusterSpec
+
+_GOOGLE_API_CLIENT_INSTALLED = True
+try:
+  from googleapiclient import discovery  # pylint: disable=g-import-not-at-top
+  from oauth2client.client import GoogleCredentials  # pylint: disable=g-import-not-at-top
+except ImportError:
+  _GOOGLE_API_CLIENT_INSTALLED = False
+
+
+def _format_master_url(master, rpc_layer=None):
+  return '%s://%s' % (rpc_layer, master) if rpc_layer else master
+
+
+class GceClusterResolver(ClusterResolver):
+  """Cluster Resolver for Google Compute Engine.
+
+  This is an implementation of cluster resolvers for the Google Compute Engine
+  instance group platform. By specifying a project, zone, and instance group,
+  this will retrieve the IP address of all the instances within the instance
+  group and return a Cluster Resolver object suitable for use for distributed
+  TensorFlow.
+  """
+
+  def __init__(self,
+               project,
+               zone,
+               instance_group,
+               port,
+               task_type='worker',
+               task_index=0,
+               rpc_layer='grpc',
+               num_accelerators_per_worker=0,
+               credentials='default',
+               service=None):
+    """Creates a new GceClusterResolver object.
+
+    This takes in a few parameters and creates a GceClusterResolver project. It
+    will then use these parameters to query the GCE API for the IP addresses of
+    each instance in the instance group.
+
+    Args:
+      project: Name of the GCE project.
+      zone: Zone of the GCE instance group.
+      instance_group: Name of the GCE instance group.
+      port: Port of the listening TensorFlow server (default: 8470)
+      task_type: Name of the TensorFlow job this GCE instance group of VM
+        instances belong to.
+      task_index: The task index for this particular VM, within the GCE
+        instance group. In particular, every single instance should be assigned
+        a unique ordinal index within an instance group manually so that they
+        can be distinguished from each other.
+      rpc_layer: The RPC layer TensorFlow should use to communicate across
+        instances.
+      num_accelerators_per_worker: Number of accelerators (GPUs) present per
+        instance.
+      credentials: GCE Credentials. If nothing is specified, this defaults to
+        GoogleCredentials.get_application_default().
+      service: The GCE API object returned by the googleapiclient.discovery
+        function. (Default: discovery.build('compute', 'v1')). If you specify a
+        custom service object, then the credentials parameter will be ignored.
+
+    Raises:
+      ImportError: If the googleapiclient is not installed.
+    """
+    self._project = project
+    self._zone = zone
+    self._instance_group = instance_group
+    self._task_type = task_type
+    self._task_index = task_index
+    self._rpc_layer = rpc_layer
+    self._port = port
+    self._credentials = credentials
+
+    if credentials == 'default':
+      if _GOOGLE_API_CLIENT_INSTALLED:
+        self._credentials = GoogleCredentials.get_application_default()
+
+    if service is None:
+      if not _GOOGLE_API_CLIENT_INSTALLED:
+        raise ImportError('googleapiclient must be installed before using the '
+                          'GCE cluster resolver')
+      self._service = discovery.build(
+          'compute', 'v1',
+          credentials=self._credentials)
+    else:
+      self._service = service
+
+  def cluster_spec(self):
+    """Returns a ClusterSpec object based on the latest instance group info.
+
+    This returns a ClusterSpec object for use based on information from the
+    specified instance group. We will retrieve the information from the GCE APIs
+    every time this method is called.
+
+    Returns:
+      A ClusterSpec containing host information retrieved from GCE.
+    """
+    request_body = {'instanceState': 'RUNNING'}
+    request = self._service.instanceGroups().listInstances(
+        project=self._project,
+        zone=self._zone,
+        instanceGroups=self._instance_group,
+        body=request_body,
+        orderBy='name')
+
+    worker_list = []
+
+    while request is not None:
+      response = request.execute()
+
+      items = response['items']
+      for instance in items:
+        instance_name = instance['instance'].split('/')[-1]
+
+        instance_request = self._service.instances().get(
+            project=self._project,
+            zone=self._zone,
+            instance=instance_name)
+
+        if instance_request is not None:
+          instance_details = instance_request.execute()
+          ip_address = instance_details['networkInterfaces'][0]['networkIP']
+          instance_url = '%s:%s' % (ip_address, self._port)
+          worker_list.append(instance_url)
+
+      request = self._service.instanceGroups().listInstances_next(
+          previous_request=request,
+          previous_response=response)
+
+    worker_list.sort()
+    return ClusterSpec({self._task_type: worker_list})
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    task_type = task_type if task_type is not None else self._task_type
+    task_index = task_index if task_index is not None else self._task_index
+
+    if task_type is not None and task_index is not None:
+      master = self.cluster_spec().task_address(task_type, task_index)
+      if rpc_layer or self._rpc_layer:
+        return '%s://%s' % (rpc_layer or self._rpc_layer, master)
+      else:
+        return master
+
+    return ''
+
+  @property
+  def task_type(self):
+    return self._task_type
+
+  @property
+  def task_index(self):
+    return self._task_index
+
+  @task_type.setter
+  def task_type(self, task_type):
+    raise RuntimeError(
+        'You cannot reset the task_type of the GceClusterResolver after it has '
+        'been created.')
+
+  @task_index.setter
+  def task_index(self, task_index):
+    self._task_index = task_index
+
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in.
+
+    For users in the GCE environment, the environment property is always an
+    empty string, and Google users will not use this ClusterResolver for running
+    on internal systems.
+    """
+    return ''
+
+  @property
+  def rpc_layer(self):
+    return self._rpc_layer
+
+  @rpc_layer.setter
+  def rpc_layer(self, rpc_layer):
+    self._rpc_layer = rpc_layer
+
+  def num_accelerators_per_worker(self, session_config=None):
+    del session_config  # Unused, since this is set manually in __init__.
+    return self._num_accelerators_per_worker
diff --git a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
similarity index 98%
rename from tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver_test.py
rename to tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
index c691552e86..d4f0660c92 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/gce_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cluster_resolver.python.training.cluster_resolver import UnionClusterResolver
-from tensorflow.contrib.cluster_resolver.python.training.gce_cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver import GceClusterResolver
+from tensorflow.python.distribute.cluster_resolver import UnionClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
diff --git a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
new file mode 100644
index 0000000000..041c081540
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
@@ -0,0 +1,173 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implementation of Cluster Resolvers for Kubernetes."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.client import device_lib
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url
+from tensorflow.python.training import server_lib
+
+_KUBERNETES_API_CLIENT_INSTALLED = True
+try:
+  from kubernetes import client as k8sclient  # pylint: disable=g-import-not-at-top
+  from kubernetes import config as k8sconfig  # pylint: disable=g-import-not-at-top
+except ImportError:
+  _KUBERNETES_API_CLIENT_INSTALLED = False
+
+
+class KubernetesClusterResolver(ClusterResolver):
+  """Cluster Resolver for Kubernetes.
+
+  This is an implementation of cluster resolvers for Kubernetes. When given the
+  the Kubernetes namespace and label selector for pods, we will retrieve the
+  pod IP addresses of all running pods matching the selector, and return a
+  ClusterSpec based on that information.
+  """
+
+  def __init__(self,
+               job_to_label_mapping=None,
+               tf_server_port=8470,
+               rpc_layer='grpc',
+               override_client=None):
+    """Initializes a new KubernetesClusterResolver.
+
+    This initializes a new Kubernetes Cluster Resolver. The Cluster Resolver
+    will attempt to talk to the Kubernetes master to retrieve all the instances
+    of pods matching a label selector.
+
+    Args:
+      job_to_label_mapping: A mapping of TensorFlow jobs to label selectors.
+        This allows users to specify many TensorFlow jobs in one Cluster
+        Resolver, and each job can have pods belong with different label
+        selectors. For example, a sample mapping might be
+        ```
+        {'worker': ['job-name=worker-cluster-a', 'job-name=worker-cluster-b'],
+         'ps': ['job-name=ps-1', 'job-name=ps-2']}
+        ```
+      tf_server_port: The port the TensorFlow server is listening on.
+      rpc_layer: (Optional) The RPC layer TensorFlow should use to communicate
+        between tasks in Kubernetes. Defaults to 'grpc'.
+      override_client: The Kubernetes client (usually automatically retrieved
+        using `from kubernetes import client as k8sclient`). If you pass this
+        in, you are responsible for setting Kubernetes credentials manually.
+
+    Raises:
+      ImportError: If the Kubernetes Python client is not installed and no
+        `override_client` is passed in.
+      RuntimeError: If autoresolve_task is not a boolean or a callable.
+    """
+    if _KUBERNETES_API_CLIENT_INSTALLED:
+      k8sconfig.load_kube_config()
+
+    if not job_to_label_mapping:
+      job_to_label_mapping = {'worker': ['job-name=tensorflow']}
+
+    if not override_client and not _KUBERNETES_API_CLIENT_INSTALLED:
+      raise ImportError('The Kubernetes Python client must be installed before'
+                        'using the Kubernetes Cluster Resolver. To install the'
+                        'Kubernetes Python client, run `pip install '
+                        'kubernetes` on your command line.')
+
+    self._job_to_label_mapping = job_to_label_mapping
+    self._tf_server_port = tf_server_port
+    self._override_client = override_client
+
+    self.task_type = None
+    self.task_index = None
+    self.rpc_layer = rpc_layer
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master address to use when creating a session.
+
+    You must have set the task_type and task_index object properties before
+    calling this function, or pass in the `task_type` and `task_index`
+    parameters when using this function. If you do both, the function parameters
+    will override the object properties.
+
+    Args:
+      task_type: (Optional) The type of the TensorFlow task of the master.
+      task_index: (Optional) The index of the TensorFlow task of the master.
+      rpc_layer: (Optional) The RPC protocol for the given cluster.
+
+    Returns:
+      The name or URL of the session master.
+    """
+    if task_type is not None and task_index is not None:
+      return format_master_url(
+          self.cluster_spec().task_address(task_type, task_index),
+          rpc_layer or self.rpc_layer)
+
+    if self.task_type is not None and self.task_index is not None:
+      return format_master_url(
+          self.cluster_spec().task_address(self.task_type, self.task_index),
+          rpc_layer or self.rpc_layer)
+
+    return ''
+
+  def cluster_spec(self):
+    """Returns a ClusterSpec object based on the latest info from Kubernetes.
+
+    We retrieve the information from the Kubernetes master every time this
+    method is called.
+
+    Returns:
+      A ClusterSpec containing host information returned from Kubernetes.
+
+    Raises:
+      RuntimeError: If any of the pods returned by the master is not in the
+        `Running` phase.
+    """
+    if not self._override_client:
+      k8sconfig.load_kube_config()
+
+    client = self._override_client or k8sclient.CoreV1Api()
+    cluster_map = {}
+
+    for tf_job in self._job_to_label_mapping:
+      all_pods = []
+      for selector in self._job_to_label_mapping[tf_job]:
+        ret = client.list_pod_for_all_namespaces(label_selector=selector)
+        selected_pods = []
+
+        # Sort the list by the name to make sure it doesn't change call to call.
+        for pod in sorted(ret.items, key=lambda x: x.metadata.name):
+          if pod.status.phase == 'Running':
+            selected_pods.append(
+                '%s:%s' % (pod.status.host_ip, self._tf_server_port))
+          else:
+            raise RuntimeError('Pod "%s" is not running; phase: "%s"' %
+                               (pod.metadata.name, pod.status.phase))
+        all_pods.extend(selected_pods)
+      cluster_map[tf_job] = all_pods
+
+    return server_lib.ClusterSpec(cluster_map)
+
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in.
+
+    For users in the Cloud environment, the environment property is always an
+    empty string, and Google users will not use this ClusterResolver for running
+    on internal systems.
+    """
+    return ''
+
+  def num_accelerators_per_worker(self, session_config=None):
+    local_devices = device_lib.list_local_devices(session_config)
+    return len([d for d in local_devices if d.device_type == 'GPU'])
diff --git a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py
similarity index 98%
rename from tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
rename to tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py
index c63a98af6c..a9750fa60b 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/kubernetes_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cluster_resolver.python.training import KubernetesClusterResolver
+from tensorflow.python.distribute.cluster_resolver import KubernetesClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
diff --git a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
new file mode 100644
index 0000000000..fd3c6d6a18
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
@@ -0,0 +1,226 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implementation of Cluster Resolvers for Slurm workload manager."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import os
+import subprocess
+
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.training.server_lib import ClusterSpec
+
+
+class SlurmClusterResolver(ClusterResolver):
+  """Cluster Resolver for system with Slurm workload manager.
+
+  This is an implementation of cluster resolvers for Slurm clusters. This allows
+  the specification of jobs and task counts, number of tasks per node, number of
+  GPUs on each node and number of GPUs for each task, It retrieves system
+  attributes by Slurm environment variables, resolves allocated computing node
+  names, construct a cluster and return a Cluster Resolver object which an be
+  use for distributed TensorFlow.
+  """
+
+  def _resolve_hostnames(self):
+    """Resolve host names of nodes allocated in current jobs.
+
+    Returns:
+      A list of node names as strings.
+    """
+    hostlist = (subprocess.check_output(['scontrol', 'show', 'hostname']).
+                decode('utf-8').strip().split('\n'))
+    return hostlist
+
+  def __init__(self,
+               jobs,
+               port_base=8888,
+               gpus_per_node=1,
+               gpus_per_task=1,
+               tasks_per_node=None,
+               auto_set_gpu=True,
+               rpc_layer='grpc'):
+    """Creates a new SlurmClusterResolver object.
+
+    This takes in parameters and creates a SlurmClusterResolver object. It uses
+    those parameters to check which nodes will processes reside and resolves
+    their hostnames. With the number of the GPUs on each node and number of GPUs
+    for each task it offsets the port number for each processes and allocate
+    GPUs to tasks by setting environment variables. The resolver currently
+    supports homogeneous tasks and default Slurm process allocation.
+
+    Args:
+      jobs: Dictionary with job names as key and number of tasks in the job as
+        value
+      port_base: The first port number to start with for processes on a node.
+      gpus_per_node: Number of GPUs available on each node.
+      gpus_per_task: Number of GPUs to be used for each task.
+      tasks_per_node: Number of tasks to run on each node, if not set defaults
+        to Slurm's output environment variable SLURM_NTASKS_PER_NODE.
+      auto_set_gpu: Set the visible CUDA devices automatically while resolving
+        the cluster by setting CUDA_VISIBLE_DEVICES environment variable.
+        Defaults to True.
+      rpc_layer: (Optional) The protocol TensorFlow uses to communicate between
+        nodes. Defaults to 'grpc'.
+
+    Returns:
+      A ClusterResolver object which can be used with distributed TensorFlow.
+
+    Raises:
+      RuntimeError: If requested more GPUs per node then available or requested
+      more tasks then assigned tasks.
+    """
+
+    # check if launched by mpirun
+    if 'OMPI_COMM_WORLD_RANK' in os.environ:
+      self._rank = int(os.environ['OMPI_COMM_WORLD_RANK'])
+      num_tasks = int(os.environ['OMPI_COMM_WORLD_SIZE'])
+    else:
+      self._rank = int(os.environ['SLURM_PROCID'])
+      num_tasks = int(os.environ['SLURM_NTASKS'])
+
+    self._jobs = collections.OrderedDict(sorted(jobs.items()))
+    self._port_base = port_base
+
+    # user specification overrides SLURM specification
+    if tasks_per_node is not None:
+      self._tasks_per_node = tasks_per_node
+    elif tasks_per_node is None and 'SLURM_NTASKS_PER_NODE' in os.environ:
+      self._tasks_per_node = int(os.environ['SLURM_NTASKS_PER_NODE'])
+    else:
+      raise RuntimeError('Neither `tasks_per_node` or '
+                         'SLURM_NTASKS_PER_NODE is set.')
+
+    self._gpus_per_node = gpus_per_node
+    self._gpus_per_task = gpus_per_task
+
+    self._auto_set_gpu = auto_set_gpu
+    self.task_type = None
+    self.task_index = None
+    self.rpc_layer = rpc_layer
+
+    self._gpu_allocation = []
+    self._cluster_allocation = {}
+
+    if self._tasks_per_node * self._gpus_per_task > self._gpus_per_node:
+      raise RuntimeError('Requested more GPUs per node then available.')
+
+    if sum(self._jobs.values()) != num_tasks:
+      raise RuntimeError('Requested more tasks then assigned tasks.')
+
+  def cluster_spec(self):
+    """Returns a ClusterSpec object based on the latest instance group info.
+
+    This returns a ClusterSpec object for use based on information from the
+    specified initialization parameters and Slurm environment variables. The
+    cluster specification is resolved each time this function is called. The
+    resolver extract hostnames of nodes by scontrol and pack tasks in that
+    order until a node a has number of tasks that is equal to specification.
+    GPUs on nodes are allocated to tasks by specification through setting
+    CUDA_VISIBLE_DEVICES environment variable.
+
+    Returns:
+      A ClusterSpec containing host information retrieved from Slurm's
+        environment variables.
+    """
+    hostlist = self._resolve_hostnames()
+
+    task_list = []
+    self._gpu_allocation = []
+    self._cluster_allocation = {}
+
+    for host in hostlist:
+      for port_offset, gpu_offset in zip(
+          range(self._tasks_per_node),
+          range(0, self._gpus_per_node, self._gpus_per_task)):
+
+        host_addr = '%s:%d' % (host, self._port_base + port_offset)
+        task_list.append(host_addr)
+        gpu_id_list = []
+
+        for gpu_id in range(gpu_offset, gpu_offset + self._gpus_per_task):
+          gpu_id_list.append(str(gpu_id))
+
+        self._gpu_allocation.append(','.join(gpu_id_list))
+
+    cluster_rank_offset_start = 0
+    cluster_rank_offset_end = 0
+
+    for task_type, num_tasks in self._jobs.items():
+      cluster_rank_offset_end = cluster_rank_offset_start + num_tasks
+
+      self._cluster_allocation[task_type] = (
+          task_list[cluster_rank_offset_start:cluster_rank_offset_end])
+
+      if cluster_rank_offset_start <= self._rank < cluster_rank_offset_end:
+        self.task_type = task_type
+        self.task_index = self._rank - cluster_rank_offset_start
+
+      cluster_rank_offset_start = cluster_rank_offset_end
+
+    if self._auto_set_gpu is True:
+      os.environ['CUDA_VISIBLE_DEVICES'] = self._gpu_allocation[self._rank]
+
+    return ClusterSpec(self._cluster_allocation)
+
+  def get_task_info(self):
+    """Returns job name and task_index for the process which calls this.
+
+    This returns the job name and task index for the process which calls this
+    function according to its rank and cluster specification. The job name and
+    task index are set after a cluster is constructed by cluster_spec otherwise
+    defaults to None.
+
+    Returns:
+      A string specifying job name the process belongs to and an integner
+        specifying the task index the process belongs to in that job.
+    """
+    return self.task_type, self.task_index
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master string for connecting to a TensorFlow master.
+
+    Args:
+      task_type: (Optional) Overrides the default auto-selected task type.
+      task_index: (Optional) Overrides the default auto-slected task index.
+      rpc_layer: (Optional) Overrides the default RPC protocol TensorFlow uses
+        to communicate across nodes.
+
+    Returns:
+      A connection string for connecting to a TensorFlow master.
+    """
+    task_type = task_type if task_type is not None else self.task_type
+    task_index = task_index if task_index is not None else self.task_index
+    rpc_layer = rpc_layer or self.rpc_layer
+    master = self.cluster_spec().task_address(task_type, task_index)
+
+    return '%s://%s' % (rpc_layer, master) if rpc_layer else master
+
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in.
+
+    For users in the Slurm environment, the environment property is always an
+    empty string, and Google users will not use this ClusterResolver for running
+    on internal systems.
+    """
+    return ''
+
+  def num_accelerators_per_worker(self, session_config=None):
+    del session_config  # Unused, since this is set in __init__ manually.
+    return self._gpus_per_node
diff --git a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py
similarity index 98%
rename from tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py
rename to tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py
index 7c76e133fe..076539d16f 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/slurm_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.contrib.cluster_resolver.python.training.slurm_cluster_resolver import SlurmClusterResolver
+from tensorflow.python.distribute.cluster_resolver import SlurmClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
new file mode 100644
index 0000000000..a3246e77f4
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
@@ -0,0 +1,171 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implementation of Cluster Resolvers for TF_CONFIG Environment Variables."""
+
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+import os
+
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.training.server_lib import ClusterSpec
+
+_TF_CONFIG_ENV = 'TF_CONFIG'
+_SESSION_MASTER_KEY = 'session_master'
+_RPC_LAYER_KEY = 'rpc_layer'
+_TASK_KEY = 'task'
+
+
+def format_master_url(master, rpc_layer=None):
+  if rpc_layer:
+    return '%s://%s' % (rpc_layer, master)
+  else:
+    return master
+
+
+def _load_tf_config():
+  return json.loads(os.environ.get(_TF_CONFIG_ENV, '{}'))
+
+
+def _get_value_in_tfconfig(key, default=None):
+  tf_config = _load_tf_config()
+  return tf_config[key] if key in tf_config else default
+
+
+class TFConfigClusterResolver(ClusterResolver):
+  """Implementation of a ClusterResolver which reads the TF_CONFIG EnvVar."""
+
+  def __init__(self,
+               task_type=None,
+               task_index=None,
+               rpc_layer=None,
+               environment=None,
+               num_accelerators_per_worker=0):
+    """Creates a new TFConfigClusterResolver.
+
+    Args:
+      task_type: (String, optional) Overrides the task type specified in the
+        TF_CONFIG environment variable.
+      task_index: (Integer, optional) Overrides the task index specified in the
+        TF_CONFIG environment variable.
+      rpc_layer: (String, optional) Overrides the rpc layer TensorFlow uses.
+      environment: (String, optional) Overrides the environment TensorFlow
+        operates in.
+      num_accelerators_per_worker: (Integer, optional) Specifies the number of
+        accelerators (e.g. GPUs, TPUs, others) that each node has.
+    """
+
+    self._task_type = task_type
+    self._task_index = task_index
+    self._rpc_layer = rpc_layer
+    self._environment = environment
+    self._num_accelerators_per_worker = num_accelerators_per_worker
+
+  @property
+  def task_type(self):
+    if self._task_type is None:
+      task_info = _get_value_in_tfconfig(_TASK_KEY, {})
+      return task_info['type'] if 'type' in task_info else None
+    else:
+      return self._task_type
+
+  @property
+  def task_index(self):
+    if self._task_type is None:
+      task_info = _get_value_in_tfconfig(_TASK_KEY, {})
+      return task_info['index'] if 'index' in task_info else None
+    else:
+      return self._task_index
+
+  @task_type.setter
+  def task_type(self, task_type):
+    self._task_type = task_type
+
+  @task_index.setter
+  def task_index(self, task_index):
+    self._task_index = task_index
+
+  @property
+  def environment(self):
+    return self._environment
+
+  @property
+  def rpc_layer(self):
+    if self._rpc_layer is None:
+      return _get_value_in_tfconfig(_RPC_LAYER_KEY)
+    else:
+      return self._rpc_layer
+
+  @rpc_layer.setter
+  def rpc_layer(self, rpc_layer):
+    self._rpc_layer = rpc_layer
+
+  def num_accelerators_per_worker(self, session_config=None):
+    # TODO(frankchn): Connect to server (w/ session_config) in the future.
+    del session_config  # Unused, we do not connect to another server here.
+    return self._num_accelerators_per_worker
+
+  def cluster_spec(self):
+    """Returns a ClusterSpec based on the TF_CONFIG environment variable.
+
+    Returns:
+      A ClusterSpec with information from the TF_CONFIG environment variable.
+    """
+    tf_config = _load_tf_config()
+    if 'cluster' not in tf_config:
+      return ClusterSpec({})
+    return ClusterSpec(tf_config['cluster'])
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Returns the master address to use when creating a TensorFlow session.
+
+    Args:
+      task_type: (String, optional) Overrides and sets the task_type of the
+        master.
+      task_index: (Integer, optional) Overrides and sets the task id of the
+        master.
+      rpc_layer: (String, optional) Overrides and sets the protocol over which
+        TensorFlow nodes communicate with each other.
+
+    Returns:
+      The address of the master.
+
+    Raises:
+      RuntimeError: If the task_type or task_id is not specified and the
+        `TF_CONFIG` environment variable does not contain a task section.
+    """
+
+    # If `session_master` is set, just use that.
+    session_master = _get_value_in_tfconfig(_SESSION_MASTER_KEY)
+    if session_master is not None:
+      return session_master
+
+    # Return an empty string if we are the only job in the ClusterSpec.
+    cluster_spec = self.cluster_spec()
+    if (not cluster_spec.jobs or
+        (len(cluster_spec.jobs) == 1 and
+         len(cluster_spec.job_tasks(cluster_spec.jobs[0])) == 1)):
+      return ''
+
+    # We try to auto-detect the task type and id, but uses the user-supplied one
+    # where available
+    task_type = task_type if task_type is not None else self.task_type
+    task_index = task_index if task_index is not None else self.task_index
+
+    return format_master_url(cluster_spec.task_address(task_type, task_index),
+                             self.rpc_layer)
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
similarity index 98%
rename from tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py
rename to tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
index 3db6d5447f..c20e51bc0b 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tfconfig_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.contrib.cluster_resolver.python.training.tfconfig_cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
new file mode 100644
index 0000000000..1956bd75a8
--- /dev/null
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -0,0 +1,423 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implementation of Cluster Resolvers for Cloud TPUs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from six.moves.urllib.request import Request
+from six.moves.urllib.request import urlopen
+
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
+from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url
+from tensorflow.python.training import server_lib
+from tensorflow.python.util import compat
+
+_GOOGLE_API_CLIENT_INSTALLED = True
+try:
+  from googleapiclient import discovery  # pylint: disable=g-import-not-at-top
+  from oauth2client.client import GoogleCredentials  # pylint: disable=g-import-not-at-top
+except ImportError:
+  _GOOGLE_API_CLIENT_INSTALLED = False
+
+
+_GKE_ENV_VARIABLE = 'KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS'
+_ENDPOINTS_SEPARATOR = ','
+_DEFAULT_ENV_VARIABLE = 'TPU_NAME'
+_DISCOVERY_SERVICE_URL_ENV_VARIABLE = 'TPU_API_DISCOVERY_URL'
+
+
+class TPUClusterResolver(ClusterResolver):
+  """Cluster Resolver for Google Cloud TPUs.
+
+  This is an implementation of cluster resolvers for the Google Cloud TPU
+  service. As Cloud TPUs are in alpha, you will need to specify a API definition
+  file for this to consume, in addition to a list of Cloud TPUs in your Google
+  Cloud Platform project.
+  """
+
+  def _tpuService(self):
+    """Creates a new Cloud TPU API object.
+
+    This works around an issue where the underlying HTTP connection sometimes
+    times out when the script has been running for too long. Other methods in
+    this object calls this method to get a new API object whenever they need
+    to communicate with the Cloud API.
+
+    Returns:
+      A Google Cloud TPU API object.
+    """
+    if self._service:
+      return self._service
+
+    credentials = self._credentials
+    if credentials is None or credentials == 'default':
+      credentials = GoogleCredentials.get_application_default()
+
+    if self._discovery_url:
+      return discovery.build(
+          'tpu', 'v1alpha1',
+          credentials=credentials,
+          discoveryServiceUrl=self._discovery_url)
+    else:
+      return discovery.build(
+          'tpu', 'v1alpha1',
+          credentials=credentials)
+
+  def _requestComputeMetadata(self, path):
+    req = Request('http://metadata/computeMetadata/v1/%s' % path,
+                  headers={'Metadata-Flavor': 'Google'})
+    resp = urlopen(req)
+    return compat.as_bytes(resp.read())
+
+  def _shouldResolve(self):
+    if isinstance(self._should_resolve_override, bool):
+      return self._should_resolve_override
+    if (self._tpu == compat.as_bytes('') or
+        self._tpu == compat.as_bytes('local') or
+        self._tpu.startswith(compat.as_bytes('/bns')) or
+        self._tpu.startswith(compat.as_bytes('localhost:')) or
+        self._tpu.startswith(compat.as_bytes('grpc://'))):
+      return False
+    return True
+
+  @staticmethod
+  def _inGke():
+    """When running in GKE, the environment variable will be set."""
+    return _GKE_ENV_VARIABLE in os.environ
+
+  @staticmethod
+  def _gkeEndpoints():
+    return os.environ[_GKE_ENV_VARIABLE]
+
+  @staticmethod
+  def _envVarFallback():
+    if _DEFAULT_ENV_VARIABLE in os.environ:
+      return os.environ[_DEFAULT_ENV_VARIABLE]
+    return None
+
+  @staticmethod
+  def _environmentDiscoveryUrl():
+    return os.environ.get(_DISCOVERY_SERVICE_URL_ENV_VARIABLE)
+
+  def __init__(self,
+               tpu=None,
+               zone=None,
+               project=None,
+               job_name='worker',
+               coordinator_name=None,
+               coordinator_address=None,
+               credentials='default',
+               service=None,
+               discovery_url=None):
+    """Creates a new TPUClusterResolver object.
+
+    The ClusterResolver will then use the parameters to query the Cloud TPU APIs
+    for the IP addresses and ports of each Cloud TPU listed.
+
+    Args:
+      tpu: Either a string, or a list of strings corresponding to the TPUs to
+        use. If the single string is the empty string, the string 'local', or a
+        string that begins with 'grpc://' or '/bns', then it is assumed to not
+        correspond with a Cloud TPU and will instead be passed as the session
+        master and no ClusterSpec propagation will be done.
+      zone: Zone where the TPUs are located. If omitted or empty, we will assume
+        that the zone of the TPU is the same as the zone of the GCE VM, which we
+        will try to discover from the GCE metadata service.
+      project: Name of the GCP project containing Cloud TPUs. If omitted or
+        empty, we will try to discover the project name of the GCE VM from the
+        GCE metadata service.
+      job_name: Name of the TensorFlow job the TPUs belong to.
+      coordinator_name: The name to use for the coordinator. Set to None if the
+        coordinator should not be included in the computed ClusterSpec.
+      coordinator_address: The address of the coordinator (typically an ip:port
+        pair). If set to None, a TF server will be started. If coordinator_name
+        is None, a TF server will not be started even if coordinator_address is
+        None.
+      credentials: GCE Credentials. If None, then we use default credentials
+        from the oauth2client
+      service: The GCE API object returned by the googleapiclient.discovery
+        function. If you specify a custom service object, then the credentials
+        parameter will be ignored.
+      discovery_url: A URL template that points to the location of
+        the discovery service. It should have two parameters {api} and
+        {apiVersion} that when filled in produce an absolute URL to the
+        discovery document for that service. The environment variable
+        'TPU_API_DISCOVERY_URL' will override this.
+
+    Raises:
+      ImportError: If the googleapiclient is not installed.
+      ValueError: If no TPUs are specified.
+    """
+    if isinstance(tpu, list):
+      if not tpu:
+        raise ValueError('At least one TPU must be specified.')
+      if len(tpu) != 1:
+        raise NotImplementedError(
+            'Using multiple TPUs in a single session is not yet implemented')
+      tpu = tpu[0]
+
+    in_gke = self._inGke()
+    # When using GKE with Cloud TPUs, the env variable will be set.
+    if tpu is None:
+      if in_gke:
+        tpu = self._gkeEndpoints()
+      else:
+        tpu = self._envVarFallback()
+
+    if tpu is None:
+      raise ValueError('Please provide a TPU Name to connect to.')
+
+    self._tpu = compat.as_bytes(tpu)  # self._tpu is always bytes
+
+    # By default the task_type is 'worker` and the task_index is 0 (which is the
+    # first worker in the task).
+    self.task_type = job_name
+    self.task_index = 0
+
+    if tpu.startswith('grpc://'):
+      # Cloud environment, where we are using GRPC to communicate to TPUs.
+      self._environment = ''
+    elif tpu == 'local' or not tpu:
+      # Google environment, where the TPU is attached to the host.
+      self._environment = 'google'
+    elif tpu.startswith('/bns'):
+      # Google environment, where we reach the TPU through BNS.
+      self._environment = 'google'
+
+    # If TPU is in the Google environment or exists locally, we don't use any
+    # RPC layer.
+    if tpu.startswith('/bns') or tpu == 'local' or not tpu:
+      self.rpc_layer = None
+    else:
+      self.rpc_layer = 'grpc'
+
+    # Setting this overrides the return value of self._shouldResolve()
+    self._should_resolve_override = None
+
+    # We strip out the protocol if it is included, and override the
+    # shouldResolve function to never resolve. We are adding the protocol back
+    # in later in self.master().
+    if self.rpc_layer is not None and tpu.startswith(self.rpc_layer + '://'):
+      tpu = tpu[len(self.rpc_layer + '://'):]
+      self._tpu = tpu
+      self._should_resolve_override = False
+
+    # Whether we should actually attempt to contact Cloud APIs
+    should_resolve = self._shouldResolve()
+
+    # We error out if we are in a non-Cloud environment which cannot talk to the
+    # Cloud APIs using the standard class and a special object is not passed in.
+    self._service = service
+    if (self._service is None and should_resolve and
+        not _GOOGLE_API_CLIENT_INSTALLED):
+      raise ImportError('googleapiclient and oauth2client must be installed '
+                        'before using the TPU cluster resolver. Execute: '
+                        '`pip install --upgrade google-api-python-client` '
+                        'and `pip install --upgrade oauth2client` to '
+                        'install with pip.')
+
+    # We save user-passed credentials, unless the user didn't pass in anything.
+    self._credentials = credentials
+    if (credentials == 'default' and should_resolve and
+        _GOOGLE_API_CLIENT_INSTALLED):
+      self._credentials = None
+
+    # Automatically detect project and zone if unspecified.
+    if not project and should_resolve:
+      project = compat.as_str(
+          self._requestComputeMetadata('project/project-id'))
+    if not zone and should_resolve:
+      zone_path = compat.as_str(self._requestComputeMetadata('instance/zone'))
+      zone = zone_path.split('/')[-1]
+    self._project = project
+    self._zone = zone
+
+    self._discovery_url = self._environmentDiscoveryUrl() or discovery_url
+
+    self._coordinator_name = coordinator_name
+    if (coordinator_name and not coordinator_address and
+        (should_resolve or in_gke)):
+      self._start_local_server()
+    else:
+      self._coordinator_address = coordinator_address
+
+  def master(self, task_type=None, task_index=None, rpc_layer=None):
+    """Get the Master string to be used for the session.
+
+    In the normal case, this returns the grpc path (grpc://1.2.3.4:8470) of
+    first instance in the ClusterSpec returned by the cluster_spec function.
+
+    If a non-TPU name is used when constructing a TPUClusterResolver, that will
+    be returned instead (e.g. If the tpus argument's value when constructing
+    this TPUClusterResolver was 'grpc://10.240.1.2:8470',
+    'grpc://10.240.1.2:8470' will be returned).
+
+    Args:
+      task_type: (Optional, string) The type of the TensorFlow task of the
+        master.
+      task_index: (Optional, integer) The index of the TensorFlow task of the
+        master.
+      rpc_layer: (Optional, string) The RPC protocol TensorFlow should use to
+        communicate with TPUs.
+
+    Returns:
+      string, the connection string to use when creating a session.
+
+    Raises:
+      ValueError: If none of the TPUs specified exists.
+    """
+    if self._shouldResolve():
+      # We are going to communicate with the Cloud TPU APIs to get a Cluster.
+      cluster_spec = self.cluster_spec()
+      if task_type is not None and task_index is not None:
+        # task_type and task_index is from the function parameter
+        master = cluster_spec.task_address(task_type, task_index)
+      elif self.task_type is not None and self.task_index is not None:
+        # task_type and task_index is from the object
+        master = cluster_spec.task_address(self.task_type, self.task_index)
+      else:
+        # by default we take the first item in the cluster with the right name
+        job_tasks = cluster_spec.job_tasks(self.task_type)
+        if not job_tasks:
+          raise ValueError('No TPUs with the specified names exist.')
+        master = job_tasks[0]
+    else:
+      if isinstance(self._tpu, (bytes, bytearray)):
+        master = self._tpu.split(compat.as_bytes(_ENDPOINTS_SEPARATOR))[0]
+      else:
+        master = self._tpu.split(_ENDPOINTS_SEPARATOR)[0]
+    return format_master_url(master, rpc_layer or self.rpc_layer)
+
+  def get_master(self):
+    return self.master()
+
+  def get_job_name(self):
+    if self._shouldResolve():
+      return self.task_type
+
+  def cluster_spec(self):
+    """Returns a ClusterSpec object based on the latest TPU information.
+
+    We retrieve the information from the GCE APIs every time this method is
+    called.
+
+    Returns:
+      A ClusterSpec containing host information returned from Cloud TPUs.
+
+    Raises:
+      RuntimeError: If the provided TPU is not healthy.
+    """
+    ############################################################################
+    # There are 5 potential cases this code must handle:
+    #  1. [Normal case.] We should resolve the TPU name to a set of tasks, and
+    #      a. Create a ClusterSpec that includes the coordinator job
+    #      b. Create a ClusterSpec without the coordinator job.
+    #  2. [GKE / No API Access.] We should not resolve the TPU name to a set of
+    #     tasks and
+    #      a. Create a ClusterSpec with the coordinator
+    #      b. Create a ClusterSpec without the coordinator
+    #  3. [Other (legacy non-gRPC).] We should return an empty ClusterSpec.
+    ############################################################################
+
+    if self._shouldResolve():
+      # Case 1.
+      full_name = 'projects/%s/locations/%s/nodes/%s' % (
+          self._project, self._zone, compat.as_text(self._tpu))
+      service = self._tpuService()
+      request = service.projects().locations().nodes().get(name=full_name)
+      response = request.execute()
+
+      if 'state' in response and response['state'] != 'READY':
+        raise RuntimeError('TPU "%s" is not yet ready; state: "%s"' %
+                           (compat.as_text(self._tpu), response['state']))
+
+      if 'health' in response and response['health'] != 'HEALTHY':
+        raise RuntimeError('TPU "%s" is unhealthy: "%s"' %
+                           (compat.as_text(self._tpu), response['health']))
+
+      if 'networkEndpoints' in response:
+        worker_list = [
+            '%s:%s' % (endpoint['ipAddress'], endpoint['port'])
+            for endpoint in response['networkEndpoints']
+        ]
+      else:
+        # Fall back to the deprecated response format
+        instance_url = '%s:%s' % (response['ipAddress'], response['port'])
+        worker_list = [instance_url]
+
+      cluster_spec = {self.task_type: worker_list}
+    else:
+      if self.rpc_layer is None:
+        # Case 3.
+        return None
+      # Case 2.
+      tpus = []
+      for tpu in self._tpu.split(_ENDPOINTS_SEPARATOR):
+        # We are working around the fact that GKE environment variable that is
+        # supplied to us has the protocol string embedded in it, but we want
+        # to strip it out for the ClusterSpec.
+        if (self.rpc_layer is not None and
+            tpu.startswith(self.rpc_layer + '://')):
+          tpus.append(tpu[len(self.rpc_layer + '://'):])
+        else:
+          tpus.append(tpu)
+      cluster_spec = {self.task_type: tpus}
+
+    if self._coordinator_address:
+      # {1, 2}.a
+      cluster_spec[self._coordinator_name] = [self._coordinator_address]
+
+    return server_lib.ClusterSpec(cluster_spec)
+
+  def num_accelerators_per_worker(self, session_config=None):
+    """Returns the number of TPU cores per worker.
+
+    This defaults to 8 for all current TPU configurations, and we do not need
+    to query any remote systems for this.
+
+    Args:
+      session_config: Unused. Not currently necessary to query anything as this
+        number is 8 for all TPU configurations.
+    """
+    del session_config  # Unused. Not necessary to query anything.
+    return 8
+
+  @property
+  def environment(self):
+    """Returns the current environment which TensorFlow is running in."""
+    return self._environment
+
+  def _start_local_server(self):
+    address = self._requestComputeMetadata('instance/network-interfaces/0/ip')
+    self._server = server_lib.Server(
+        {
+            'local': ['0.0.0.0:0']
+        }, protocol='grpc', config=None, start=True)
+    # self._server.target is of the form: grpc://ipaddress:port
+    target = compat.as_bytes(self._server.target)
+    splits = target.split(compat.as_bytes(':'))
+    assert len(splits) == 3, self._server.target
+    assert splits[0] == compat.as_bytes('grpc'), self._server.target
+    self._coordinator_port = compat.as_text(splits[2])
+    self._coordinator_address = '%s:%s' % (
+        address, compat.as_text(self._coordinator_port))
+
+  def __deepcopy__(self, memo):
+    # TODO(b/73668574): Remove this once RunConfig avoids performing deepcopy.
+    return self
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
similarity index 99%
rename from tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
rename to tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
index 365bd52ee2..0f22ede3d9 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.contrib.cluster_resolver.python.training.tpu_cluster_resolver import TPUClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 from tensorflow.python.util import compat
-- 
GitLab


From 94008f8c576c8e56e401ba9aee56137e43b6fa0e Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 29 Nov 2018 13:30:39 -0800
Subject: [PATCH 1054/1554] Make input ops test windows compatible by using
 path separator from os module.

PiperOrigin-RevId: 223401186
---
 tensorflow/python/distribute/input_ops_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py
index 2689dbbec8..d5f41b7093 100644
--- a/tensorflow/python/distribute/input_ops_test.py
+++ b/tensorflow/python/distribute/input_ops_test.py
@@ -126,7 +126,7 @@ class AutoShardDatasetTest(test.TestCase):
 
   def testListfiles(self):
     filenames = self._createTFRecordFiles()
-    file_pattern = filenames[0].rsplit("/", 1)[0] + "/tf_record.*.txt"
+    file_pattern = filenames[0].rsplit(os.sep, 1)[0] + "/tf_record.*.txt"
     dataset = dataset_ops.Dataset.list_files(file_pattern, shuffle=False)
     dataset = dataset.flat_map(readers.TFRecordDataset)
     dataset = input_ops.auto_shard_dataset(
-- 
GitLab


From 7e70dc43172e1164672ad06be1fbf0361600a52c Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 29 Nov 2018 13:54:40 -0800
Subject: [PATCH 1055/1554] Fix relative path to current directory being empty
 case for windows file system CreateDir function.

PiperOrigin-RevId: 223405610
---
 tensorflow/core/platform/windows/windows_file_system.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc
index 6cf79634d7..993b9906b1 100644
--- a/tensorflow/core/platform/windows/windows_file_system.cc
+++ b/tensorflow/core/platform/windows/windows_file_system.cc
@@ -439,6 +439,9 @@ Status WindowsFileSystem::DeleteFile(const string& fname) {
 Status WindowsFileSystem::CreateDir(const string& name) {
   Status result;
   std::wstring ws_name = Utf8ToWideChar(name);
+  if (ws_name.empty()) {
+    return errors::AlreadyExists(name);
+  }
   if (_wmkdir(ws_name.c_str()) != 0) {
     result = IOError("Failed to create a directory: " + name, errno);
   }
-- 
GitLab


From 1f2eee4e233b050d9584e59f4702d5fabf76472b Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Thu, 29 Nov 2018 14:06:19 -0800
Subject: [PATCH 1056/1554] Add CosineProximity to metrics and losses

PiperOrigin-RevId: 223407786
---
 tensorflow/python/keras/losses.py       | 34 +++++++++++++++
 tensorflow/python/keras/losses_test.py  | 57 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 37 ++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 33 ++++++++++++++
 4 files changed, 161 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 83318d6c57..6b8d560102 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -395,6 +395,40 @@ def cosine_proximity(y_true, y_pred):
   return -math_ops.reduce_sum(y_true * y_pred, axis=-1)
 
 
+class CosineProximity(Loss):
+  """Computes the cosine distance between `y_true` and `y_pred`.
+
+  Usage:
+
+  ```python
+  cosine_loss = tf.losses.CosineProximity()
+  loss = cosine_loss([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: -0.5
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.CosineProximity())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    """Calculates the cosine proximity loss.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Cosine distance loss.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return cosine_proximity(y_true, y_pred)
+
+
 # Aliases.
 
 mse = MSE = mean_squared_error
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index b5e9a24c99..cbf3c3524c 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -442,5 +442,62 @@ class MeanSquaredLogarithmicErrorTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class CosineProximityTest(test.TestCase):
+
+  def test_config(self):
+    cosine_obj = keras.losses.CosineProximity(
+        reduction=losses_impl.ReductionV2.SUM, name='cosine_loss')
+    self.assertEqual(cosine_obj.name, 'cosine_loss')
+    self.assertEqual(cosine_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    cosine_obj = keras.losses.CosineProximity()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = cosine_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), -0.18722, 3)
+
+  def test_scalar_weighted(self):
+    cosine_obj = keras.losses.CosineProximity()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = cosine_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), -0.43060, 3)
+
+  def test_sample_weighted(self):
+    cosine_obj = keras.losses.CosineProximity()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0.15599, 3)
+
+  def test_timestep_weighted(self):
+    cosine_obj = keras.losses.CosineProximity()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3, 1),
+                                  dtype=dtypes.float32)
+    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
+    loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), -2.0000, 3)
+
+  def test_zero_weighted(self):
+    cosine_obj = keras.losses.CosineProximity()
+    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
+    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = cosine_obj(y_true, y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 0519493a0a..3c2682e4c6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1477,6 +1477,43 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
                                self.tn[min_index] + self.fp[min_index])
 
 
+class CosineProximity(MeanMetricWrapper):
+  """Computes the cosine distance between the labels and predictions.
+
+  For example, if `y_true` is [0, 1, 1], and `y_pred` is [1, 0, 1], the cosine
+  proximity is -0.5.
+
+  This metric keeps the average cosine distance between `predictions` and
+  `labels` over a stream of data.
+
+  Usage:
+  ```python
+  m = tf.metrics.CosineProximity()
+  m.update_state([0, 1, 1], [1, 0, 1])
+  print('Final result: ', m.result().numpy())  # Final result: -0.5
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+      'sgd',
+      loss='mse',
+      metrics=[tf.metrics.CosineProximity()])
+  ```
+  """
+
+  def __init__(self, name='cosine_proximity', dtype=None):
+    super(CosineProximity, self).__init__(cosine, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(CosineProximity, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 9a88391dc1..92398acd8e 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1138,5 +1138,38 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(self.evaluate(s_obj.tn), 25.)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class CosineProximityTest(test.TestCase):
+
+  def test_config(self):
+    cosine_obj = metrics.CosineProximity(name='my_cos', dtype=dtypes.int32)
+    self.assertEqual(cosine_obj.name, 'my_cos')
+    self.assertEqual(cosine_obj._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    cosine_obj = metrics.CosineProximity()
+    self.evaluate(variables.variables_initializer(cosine_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = cosine_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = cosine_obj.result()
+    self.assertAllClose(-0.60723, result, atol=1e-5)
+
+  def test_weighted(self):
+    cosine_obj = metrics.CosineProximity()
+    self.evaluate(variables.variables_initializer(cosine_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(-0.59916, self.evaluate(result), atol=1e-5)
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 29574450f8701385b821f8b6701e19df1d41dbe5 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 29 Nov 2018 14:17:23 -0800
Subject: [PATCH 1057/1554] Delegate EagerTensor __bool__ behavior directly to
 numpy.

Numpy also casts non-bools to bool, so after this change EagerTensors that are not dtype bool can be converted to python bools. This also allows any shape EagerTensor with a single element to be converted to a python bool.

PiperOrigin-RevId: 223409869
---
 tensorflow/python/eager/tensor_test.py | 10 +++++++---
 tensorflow/python/framework/ops.py     |  8 +-------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index 8c9d5dabe7..25442ff048 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -175,9 +175,13 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     self.assertEqual(dtypes.float64, t.dtype)
 
   def testBool(self):
-    t = _create_tensor(False)
-    if t:
-      self.assertFalse(True)
+    self.assertFalse(bool(_create_tensor(False)))
+    self.assertFalse(bool(_create_tensor([False])))
+    self.assertFalse(bool(_create_tensor([[False]])))
+    self.assertFalse(bool(_create_tensor([0])))
+    self.assertFalse(bool(_create_tensor([0.])))
+    self.assertTrue(bool(_create_tensor([1])))
+    self.assertTrue(bool(_create_tensor([1.])))
 
   def testIntDowncast(self):
     t = _create_tensor(3)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index bd798f9ffa..b5175d3c93 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -910,13 +910,7 @@ class _EagerTensorBase(Tensor):
     return self._copy(context.context(), "GPU:" + str(gpu_index))
 
   def __bool__(self):
-    if self._shape_tuple() != ():  # pylint: disable=g-explicit-bool-comparison
-      raise ValueError(
-          "Non-scalar tensor %s cannot be converted to boolean." % repr(self))
-    if self.dtype != dtypes.bool:
-      raise ValueError(
-          "Non-boolean tensor %s cannot be converted to boolean." % repr(self))
-    return bool(self.cpu().numpy())
+    return bool(self.numpy())
 
   def __nonzero__(self):
     return self.__bool__()
-- 
GitLab


From 675f415603ad0bb0217459672fffbbe798e01d84 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Fri, 24 Aug 2018 17:01:55 -0700
Subject: [PATCH 1058/1554] Upgrade Dockerfile assembler system

This is a big upgrade to the Dockerfile assembler I wrote a couple of
months ago. The spec has changed, the script has been rewritten, and
there are new features throughout:

- The assembler can build and upload images to Docker Hub.
- The assembler can also run tests (!), although the testing system is
  extremely rudimentary. It could be expanded with parallelism later, if
  execution time becomes a problem.
- spec.yml is totally different, and now defines both dockerfiles and
  images. It handles the combinatorial explosion of multiple optional features
  without excessive duplication, unlike the previous spec format.
- Partials are the same, but I dumped the extensive dockerfile
  documentation support because I don't think anyone would have used it.
- Dockerfiles are handled under the same kind of system as images, which
  is neat. The new Dockerfiles aren't so duplicated.
- I've upgraded the images with new tensorflow tutorial files (jupyter
  only) and fixed some others that didn't actually work.
- I've improved the development documentation by suggesting aliases.
- Added "static-dockerfiles" directory to track independent Dockerfiles.

These changes should better support changes like #23194.
---
 tensorflow/tools/dockerfiles/.gitignore       |   1 +
 tensorflow/tools/dockerfiles/README.md        |  49 +-
 tensorflow/tools/dockerfiles/assembler.py     | 904 ++++++++++--------
 .../dockerfiles/cpu-devel-jupyter.Dockerfile  |  63 +-
 .../dockerfiles/cpu-devel.Dockerfile          |  46 +-
 .../dockerfiles/cpu-jupyter.Dockerfile        |  52 +-
 .../dockerfiles/dockerfiles/cpu.Dockerfile    |  35 +-
 ...ockerfile => gpu-devel-jupyter.Dockerfile} |  75 +-
 ...-devel.Dockerfile => gpu-devel.Dockerfile} |  58 +-
 ...yter.Dockerfile => gpu-jupyter.Dockerfile} |  61 +-
 .../{nvidia.Dockerfile => gpu.Dockerfile}     |  44 +-
 .../partials/jupyter.partial.Dockerfile       |  15 +-
 .../partials/tensorflow.partial.Dockerfile    |   7 +-
 .../partials/test-import.partial.Dockerfile   |   0
 .../partials/ubuntu.partial.Dockerfile        |   2 -
 .../{ => ubuntu}/bazel.partial.Dockerfile     |  14 +
 .../cpu-devel.partial.Dockerfile}             |   7 +-
 .../partials/ubuntu/cpu.partial.Dockerfile    |   1 +
 .../nvidia-devel.partial.Dockerfile           |  18 +-
 .../{ => ubuntu}/nvidia.partial.Dockerfile    |   8 +-
 .../{ => ubuntu}/python.partial.Dockerfile    |   5 +-
 .../ubuntu/test-devel.partial.Dockerfile      |   0
 .../ubuntu/version.partial.Dockerfile         |   1 +
 tensorflow/tools/dockerfiles/spec.yml         | 320 +++----
 .../tools/dockerfiles/tests/build-cpu.sh      |  22 +
 .../tools/dockerfiles/tests/build-gpu.sh      |  20 +
 .../tools/dockerfiles/tests/import-gpu.sh     |   2 +
 tensorflow/tools/dockerfiles/tests/import.sh  |   3 +
 ...{assembler.Dockerfile => tools.Dockerfile} |   5 +-
 29 files changed, 1018 insertions(+), 820 deletions(-)
 create mode 100644 tensorflow/tools/dockerfiles/.gitignore
 rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia-devel-jupyter.Dockerfile => gpu-devel-jupyter.Dockerfile} (67%)
 rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia-devel.Dockerfile => gpu-devel.Dockerfile} (76%)
 rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia-jupyter.Dockerfile => gpu-jupyter.Dockerfile} (63%)
 rename tensorflow/tools/dockerfiles/dockerfiles/{nvidia.Dockerfile => gpu.Dockerfile} (69%)
 create mode 100644 tensorflow/tools/dockerfiles/partials/test-import.partial.Dockerfile
 delete mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile
 rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/bazel.partial.Dockerfile (58%)
 rename tensorflow/tools/dockerfiles/partials/{ubuntu-devel.partial.Dockerfile => ubuntu/cpu-devel.partial.Dockerfile} (86%)
 create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile
 rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/nvidia-devel.partial.Dockerfile (78%)
 rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/nvidia.partial.Dockerfile (78%)
 rename tensorflow/tools/dockerfiles/partials/{ => ubuntu}/python.partial.Dockerfile (66%)
 create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/test-devel.partial.Dockerfile
 create mode 100644 tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile
 create mode 100755 tensorflow/tools/dockerfiles/tests/build-cpu.sh
 create mode 100755 tensorflow/tools/dockerfiles/tests/build-gpu.sh
 create mode 100755 tensorflow/tools/dockerfiles/tests/import-gpu.sh
 create mode 100755 tensorflow/tools/dockerfiles/tests/import.sh
 rename tensorflow/tools/dockerfiles/{assembler.Dockerfile => tools.Dockerfile} (95%)

diff --git a/tensorflow/tools/dockerfiles/.gitignore b/tensorflow/tools/dockerfiles/.gitignore
new file mode 100644
index 0000000000..d7efa472a9
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/.gitignore
@@ -0,0 +1 @@
+dockerfiles/*.temp.Dockerfile
diff --git a/tensorflow/tools/dockerfiles/README.md b/tensorflow/tools/dockerfiles/README.md
index 7c8ca1d1c7..2ac68666d0 100644
--- a/tensorflow/tools/dockerfiles/README.md
+++ b/tensorflow/tools/dockerfiles/README.md
@@ -1,8 +1,12 @@
 # TensorFlow Dockerfiles
 
-This directory houses TensorFlow's Dockerfiles. **DO NOT EDIT THE DOCKERFILES
-MANUALLY!** They are maintained by `assembler.py`, which builds Dockerfiles from
-the files in `partials/` and the rules in `spec.yml`. See [the Contributing
+This directory houses TensorFlow's Dockerfiles and the infrastructure used to
+create and deploy them to [Docker
+Hub](https://hub.docker.com/r/tensorflow/tensorflow).
+
+**DO NOT EDIT THE DOCKERFILES/ DIRECTORY MANUALLY!** The files within are
+maintained by `assembler.py`, which builds Dockerfiles from the files in
+`partials/` and the rules in `spec.yml`. See [the Contributing
 section](#contributing) for more information.
 
 These Dockerfiles are planned to replace the Dockerfiles used to generate
@@ -20,10 +24,10 @@ $ docker build -f ./dockerfiles/cpu.Dockerfile -t tf .
 Each Dockerfile has its own set of available `--build-arg`s which are documented
 in the Dockerfile itself.
 
-## Running
+## Running Locally Built Images
 
 After building the image with the tag `tf` (for example), use `docker run` to
-run the images. Examples are below.
+run the images.
 
 Note for new Docker users: the `-v` and `-u` flags share directories between
 the Docker container and your machine, and very important. Without
@@ -42,8 +46,10 @@ $ docker run -u $(id -u):$(id -g) -v $(pwd):/my-devel -it tf
 # GPU-based images (set up nvidia-docker2 first)
 $ docker run --runtime=nvidia -u $(id -u):$(id -g) -v $(pwd):/my-devel -it tf
 
-# Images with Jupyter run on port 8888, and needs a volume for notebooks
-$ docker run --user $(id -u):$(id -g) -p 8888:8888 -v $(pwd):/notebooks -it tf
+# Images with Jupyter run on port 8888 and need a volume for your notebooks
+# You can change $(PWD) to the full path to a directory if your notebooks
+# live outside the current directory.
+$ docker run --user $(id -u):$(id -g) -p 8888:8888 -v $(PWD):/tf/notebooks -it tf
 ```
 
 These images do not come with the TensorFlow source code -- but the development
@@ -60,11 +66,32 @@ You can use the `Dockerfile` in this directory to build an editing environment
 that has all of the Python dependencies you'll need:
 
 ```bash
-$ docker build -t tf-assembler -f assembler.Dockerfile .
+# Build the tools-helper image so you can run the assembler
+$ docker build -t tf-tools -f tools.Dockerfile .
 
 # Set --user to set correct permissions on generated files
-$ docker run --user $(id -u):$(id -g) -it -v $(pwd):/tf tf-assembler bash 
+$ docker run --user $(id -u):$(id -g) -it -v $(pwd):/tf tf-tools bash 
+
+# Next you can make a handy alias depending on what you're doing. When building
+# Docker images, you need to run as root with docker.sock mounted so that the
+# container can run Docker commands. When assembling Dockerfiles, though, you'll
+# want to run as your user so that new files have the right permissions.
+
+# If you're BUILDING OR DEPLOYING DOCKER IMAGES, run as root with docker.sock:
+$ alias asm_images="docker run --rm -v $(pwd):/tf -v /var/run/docker.sock:/var/run/docker.sock tf-tools python3 assembler.py "
+
+# If you're REBUILDING OR ADDING DOCKERFILES, remove docker.sock and add -u:
+$ alias asm_dockerfiles="docker run --rm -u $(id -u):$(id -g) -v $(pwd):/tf tf-tools python3 assembler.py "
+
+# Check flags
+$ asm_dockerfiles --help
+
+# Assemble all of the Dockerfiles
+$ asm_dockerfiles --release ubuntu-dockerfiles --construct_dockerfiles
+
+# Build all of the "nightly" images on your local machine:
+$ asm_images --release nightly --build_images
 
-# In the container...
-/tf $ python3 ./assembler.py -o dockerfiles -s spec.yml
+# Build version release for version 99.0, except "gpu" tags:
+$ asm_images --release versioned --arg _TAG_PREFIX=99.0 --build_images --exclude_tags_matching '*.gpu.*'
 ```
diff --git a/tensorflow/tools/dockerfiles/assembler.py b/tensorflow/tools/dockerfiles/assembler.py
index 9cdd9bb0cb..8d97e1d7dc 100644
--- a/tensorflow/tools/dockerfiles/assembler.py
+++ b/tensorflow/tools/dockerfiles/assembler.py
@@ -1,73 +1,140 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Assemble common TF Dockerfiles from many parts.
-
-This script constructs TF's Dockerfiles by aggregating partial
-Dockerfiles. See README.md for usage examples.
+"""Multipurpose TensorFlow Docker Helper.
+
+- Assembles Dockerfiles
+- Builds images (and optionally runs image tests)
+- Pushes images to Docker Hub (provided with credentials)
+
+Read README.md (in this directory) for instructions!
 """
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import copy
 import errno
+import itertools
+import multiprocessing
 import os
-import os.path
 import re
 import shutil
-import textwrap
+import sys
 
 from absl import app
 from absl import flags
 import cerberus
+import docker
 import yaml
 
 FLAGS = flags.FLAGS
 
+flags.DEFINE_string('hub_username', None,
+                    'Dockerhub username, only used with --upload_to_hub')
+
+flags.DEFINE_string(
+    'hub_password', None,
+    ('Dockerhub password, only used with --upload_to_hub. Use from an env param'
+     'so your password isn\'t in your history.'))
+
+flags.DEFINE_integer('hub_timeout', 3600,
+                     'Abort Hub upload if it takes longer than this.')
+
+flags.DEFINE_string(
+    'repository', 'tensorflow',
+    'Tag local images as {repository}:tag (in addition to the '
+    'hub_repository, if uploading to hub)')
+
+flags.DEFINE_string(
+    'hub_repository', None,
+    'Push tags to this Docker Hub repository, e.g. tensorflow/tensorflow')
+
+flags.DEFINE_boolean(
+    'upload_to_hub',
+    False,
+    ('Push built images to Docker Hub (you must also provide --hub_username, '
+     '--hub_password, and --hub_repository)'),
+    short_name='u',
+)
+
+flags.DEFINE_boolean(
+    'construct_dockerfiles', False, 'Do not build images', short_name='d')
+
 flags.DEFINE_boolean(
-    'dry_run', False, 'Do not actually generate Dockerfiles', short_name='n')
+    'keep_temp_dockerfiles',
+    False,
+    'Retain .temp.Dockerfiles created while building images.',
+    short_name='k')
+
+flags.DEFINE_boolean(
+    'build_images', False, 'Do not build images', short_name='b')
 
 flags.DEFINE_string(
-    'spec_file',
-    './spec.yml',
-    'Path to a YAML specification file',
-    short_name='s')
+    'run_tests_path', None,
+    ('Execute test scripts on generated Dockerfiles before pushing them. '
+     'Flag value must be a full path to the "tests" directory, which is usually'
+     ' $(realpath ./tests). A failed tests counts the same as a failed build.'))
+
+flags.DEFINE_boolean(
+    'stop_on_failure', False,
+    ('Stop processing tags if any one build fails. If False or not specified, '
+     'failures are reported but do not affect the other images.'))
+
+flags.DEFINE_boolean(
+    'dry_run',
+    False,
+    'Do not build or deploy anything at all.',
+    short_name='n',
+)
+
+flags.DEFINE_string(
+    'exclude_tags_matching',
+    None,
+    ('Regular expression that skips processing on any tag it matches. Must '
+     'match entire string, e.g. ".*gpu.*" ignores all GPU tags.'),
+    short_name='x')
+
+flags.DEFINE_string(
+    'only_tags_matching',
+    None,
+    ('Regular expression that skips processing on any tag it does not match. '
+     'Must match entire string, e.g. ".*gpu.*" includes only GPU tags.'),
+    short_name='i')
 
 flags.DEFINE_string(
-    'output_dir',
-    './dockerfiles', ('Path to an output directory for Dockerfiles. '
-                      'Will be created if it doesn\'t exist.'),
+    'dockerfile_dir',
+    './dockerfiles', 'Path to an output directory for Dockerfiles.'
+    ' Will be created if it doesn\'t exist.'
+    ' Existing files in this directory will be deleted when new Dockerfiles'
+    ' are made.',
     short_name='o')
 
 flags.DEFINE_string(
     'partial_dir',
     './partials',
-    'Path to a directory containing foo.partial.Dockerfile partial files.',
+    'Path to a directory containing foo.partial.Dockerfile partial files.'
+    ' can have subdirectories, e.g. "bar/baz.partial.Dockerfile".',
     short_name='p')
 
-flags.DEFINE_boolean(
-    'quiet_dry_run',
-    True,
-    'Do not print contents of dry run Dockerfiles.',
-    short_name='q')
+flags.DEFINE_multi_string(
+    'release', [],
+    'Set of releases to build and tag. Defaults to every release type.',
+    short_name='r')
 
-flags.DEFINE_boolean(
-    'validate', True, 'Validate generated Dockerfiles', short_name='c')
+flags.DEFINE_multi_string(
+    'arg', [],
+    ('Extra build arguments. These are used for expanding tag names if needed '
+     '(e.g. --arg _TAG_PREFIX=foo) and for using as build arguments (unused '
+     'args will print a warning).'),
+    short_name='a')
+
+flags.DEFINE_string(
+    'spec_file',
+    './spec.yml',
+    'Path to the YAML specification file',
+    short_name='s')
 
-# Schema to verify the contents of spec.yml with Cerberus.
+# Schema to verify the contents of tag-spec.yml with Cerberus.
 # Must be converted to a dict from yaml to work.
 # Note: can add python references with e.g.
 # !!python/name:builtins.str
@@ -76,478 +143,513 @@ SCHEMA_TEXT = """
 header:
   type: string
 
-partials:
+slice_sets:
   type: dict
   keyschema:
     type: string
   valueschema:
-    type: dict
-    schema:
-      desc:
-        type: string
-      args:
+     type: list
+     schema:
         type: dict
-        keyschema:
-          type: string
-        valueschema:
-          anyof:
-            - type: [ boolean, number, string ]
-            - type: dict
-              schema:
-                 default:
-                    type: [ boolean, number, string ]
-                 desc:
-                    type: string
-                 options:
-                    type: list
-                    schema:
-                       type: string
-
-images:
+        schema:
+           add_to_name:
+             type: string
+           dockerfile_exclusive_name:
+             type: string
+           partials:
+             type: list
+             schema:
+               type: string
+               ispartial: true
+           test_runtime:
+             type: string
+             required: false
+           tests:
+             type: list
+             default: []
+             schema:
+               type: string
+           args:
+             type: list
+             default: []
+             schema:
+               type: string
+               isfullarg: true
+
+releases:
+  type: dict
   keyschema:
     type: string
   valueschema:
     type: dict
     schema:
-      desc:
-        type: string
-      arg-defaults:
-        type: list
-        schema:
-          anyof:
-            - type: dict
-              keyschema:
-                type: string
-                arg_in_use: true
-              valueschema:
-                type: string
-            - type: string
-              isimage: true
-      create-dockerfile:
+      is_dockerfiles:
         type: boolean
-      partials:
+        required: false
+        default: false
+      upload_images:
+        type: boolean
+        required: false
+        default: true
+      tag_specs:
         type: list
+        required: true
         schema:
-          anyof:
-            - type: dict
-              keyschema:
-                type: string
-                regex: image
-              valueschema:
-                type: string
-                isimage: true
-            - type: string
-              ispartial: true
+          type: string
 """
 
 
-class TfDockerValidator(cerberus.Validator):
-  """Custom Cerberus validator for TF dockerfile spec.
+class TfDockerTagValidator(cerberus.Validator):
+  """Custom Cerberus validator for TF tag spec.
 
   Note: Each _validate_foo function's docstring must end with a segment
   describing its own validation schema, e.g. "The rule's arguments are...". If
   you add a new validator, you can copy/paste that section.
   """
 
-  def _validate_ispartial(self, ispartial, field, value):
-    """Validate that a partial references an existing partial spec.
+  def __init__(self, *args, **kwargs):
+    # See http://docs.python-cerberus.org/en/stable/customize.html
+    if 'partials' in kwargs:
+      self.partials = kwargs['partials']
+    super(cerberus.Validator, self).__init__(*args, **kwargs)
+
+  def _validate_isvariant(self, isvariant, field, value):
+    """Validate that a variant references an existing variant spec.
 
     Args:
-      ispartial: Value of the rule, a bool
+      isvariant: Value of the rule, a bool
       field: The field being validated
       value: The field's value
-
     The rule's arguments are validated against this schema:
     {'type': 'boolean'}
     """
-    if ispartial and value not in self.root_document.get('partials', dict()):
-      self._error(field, '{} is not an existing partial.'.format(value))
+    if isvariant and value not in self.root_document.get('variants', dict()):
+      self._error(field, '{} is not an existing variant.'.format(value))
 
-  def _validate_isimage(self, isimage, field, value):
-    """Validate that an image references an existing partial spec.
+  def _validate_ispartial(self, ispartial, field, value):
+    """Validate that a partial references an existing partial spec.
 
     Args:
-      isimage: Value of the rule, a bool
+      ispartial: Value of the rule, a bool
       field: The field being validated
       value: The field's value
-
     The rule's arguments are validated against this schema:
     {'type': 'boolean'}
     """
-    if isimage and value not in self.root_document.get('images', dict()):
-      self._error(field, '{} is not an existing image.'.format(value))
+    if ispartial and value not in self.partials:
+      self._error(field,
+                  '{} is not present in the partials directory.'.format(value))
 
-  def _validate_arg_in_use(self, arg_in_use, field, value):
-    """Validate that an arg references an existing partial spec's args.
+  def _validate_isfullarg(self, isfullarg, field, value):
+    """Validate that a string is either a FULL=arg or NOT.
 
     Args:
-      arg_in_use: Value of the rule, a bool
+      isfullarg: Value of the rule, a bool
       field: The field being validated
       value: The field's value
-
     The rule's arguments are validated against this schema:
     {'type': 'boolean'}
     """
-    if arg_in_use:
-      for partial in self.root_document.get('partials', dict()).values():
-        if value in partial.get('args', tuple()):
-          return
+    if isfullarg and '=' not in value:
+      self._error(field, '{} should be of the form ARG=VALUE.'.format(value))
+    if not isfullarg and '=' in value:
+      self._error(field, '{} should be of the form ARG (no =).'.format(value))
 
-      self._error(field, '{} is not an arg used in any partial.'.format(value))
 
+def eprint(*args, **kwargs):
+  print(*args, file=sys.stderr, flush=True, **kwargs)
 
-def build_partial_description(partial_spec):
-  """Create the documentation lines for a specific partial.
 
-  Generates something like this:
+def aggregate_all_slice_combinations(spec, slice_set_names):
+  """Figure out all of the variant groupings for a spec."""
+  slice_sets = copy.deepcopy(spec['slice_sets'])
 
-    # This is the partial's description, from spec.yml.
-    # --build-arg ARG_NAME=argdefault
-    #    this is one of the args.
-    # --build-arg ANOTHER_ARG=(some|choices)
-    #    another arg.
+  for name in slice_set_names:
+    for slice_set in slice_sets[name]:
+      slice_set['set_name'] = name
 
-  Args:
-    partial_spec: A dict representing one of the partials from spec.yml. Doesn't
-      include the name of the partial; is a dict like { desc: ..., args: ... }.
+  slices_grouped_but_not_keyed = [slice_sets[name] for name in slice_set_names]
+  all_slice_combos = list(itertools.product(*slices_grouped_but_not_keyed))
+  return all_slice_combos
 
-  Returns:
-    A commented string describing this partial.
-  """
 
-  # Start from linewrapped desc field
-  lines = []
-  wrapper = textwrap.TextWrapper(
-      initial_indent='# ', subsequent_indent='# ', width=80)
-  description = wrapper.fill(partial_spec.get('desc', '( no comments )'))
-  lines.extend(['#', description])
-
-  # Document each arg
-  for arg, arg_data in partial_spec.get('args', dict()).items():
-    # Wrap arg description with comment lines
-    desc = arg_data.get('desc', '( no description )')
-    desc = textwrap.fill(
-        desc,
-        initial_indent='#    ',
-        subsequent_indent='#    ',
-        width=80,
-        drop_whitespace=False)
-
-    # Document (each|option|like|this)
-    if 'options' in arg_data:
-      arg_options = ' ({})'.format('|'.join(arg_data['options']))
-    else:
-      arg_options = ''
+def build_name_from_slices(format_string, slices, args, is_dockerfile=False):
+  """Build the tag name (cpu-devel...) from a list of slices."""
+  name_formatter = copy.deepcopy(args)
+  name_formatter.update({s['set_name']: s['add_to_name'] for s in slices})
+  name_formatter.update({
+      s['set_name']: s['dockerfile_exclusive_name']
+      for s in slices
+      if is_dockerfile and 'dockerfile_exclusive_name' in s
+  })
+  name = format_string.format(**name_formatter)
+  return name
 
-    # Add usage sample
-    arg_use = '# --build-arg {}={}{}'.format(arg,
-                                             arg_data.get('default', '(unset)'),
-                                             arg_options)
-    lines.extend([arg_use, desc])
 
-  return '\n'.join(lines)
+def update_args_dict(args_dict, updater):
+  """Update a dict of arg values with more values from a list or dict."""
+  if isinstance(updater, list):
+    for arg in updater:
+      key, sep, value = arg.partition('=')
+      if sep == '=':
+        args_dict[key] = value
+  if isinstance(updater, dict):
+    for key, value in updater.items():
+      args_dict[key] = value
+  return args_dict
 
 
-def construct_contents(partial_specs, image_spec):
-  """Assemble the dockerfile contents for an image spec.
+def get_slice_sets_and_required_args(slice_sets, tag_spec):
+  """Extract used-slice-sets and required CLI arguments from a spec string.
 
-  It assembles a concrete list of partial references into a single, large
-  string.
-  Also expands argument defaults, so that the resulting Dockerfile doesn't have
-  to be configured with --build-arg=... every time. That is, any ARG directive
-  will be updated with a new default value.
+  For example, {FOO}{bar}{bat} finds FOO, bar, and bat. Assuming bar and bat
+  are both named slice sets, FOO must be specified on the command line.
 
   Args:
-    partial_specs: The dict from spec.yml["partials"].
-    image_spec: One of the dict values from spec.yml["images"].
+     slice_sets: Dict of named slice sets
+     tag_spec: The tag spec string, e.g. {_FOO}{blep}
 
   Returns:
-    A string containing a valid Dockerfile based on the partials listed in
-    image_spec.
+     (used_slice_sets, required_args), a tuple of lists
   """
-  processed_partial_strings = []
-  for partial_name in image_spec['partials']:
-    # Apply image arg-defaults to existing arg defaults
-    partial_spec = copy.deepcopy(partial_specs[partial_name])
-    args = partial_spec.get('args', dict())
-    for k_v in image_spec.get('arg-defaults', []):
-      arg, value = list(k_v.items())[0]
-      if arg in args:
-        args[arg]['default'] = value
-
-    # Read partial file contents
-    filename = partial_spec.get('file', partial_name)
-    partial_path = os.path.join(FLAGS.partial_dir,
-                                '{}.partial.Dockerfile'.format(filename))
-    with open(partial_path, 'r') as f_partial:
-      partial_contents = f_partial.read()
-
-    # Replace ARG FOO=BAR with ARG FOO=[new-default]
-    for arg, arg_data in args.items():
-      if 'default' in arg_data and arg_data['default']:
-        default = '={}'.format(arg_data['default'])
-      else:
-        default = ''
-      partial_contents = re.sub(r'ARG {}.*'.format(arg), 'ARG {}{}'.format(
-          arg, default), partial_contents)
-
-    # Store updated partial contents
-    processed_partial_strings.append(partial_contents)
-
-  # Join everything together
-  return '\n'.join(processed_partial_strings)
-
-
-def mkdir_p(path):
-  """Create a directory and its parents, even if it already exists."""
-  try:
-    os.makedirs(path)
-  except OSError as e:
-    if e.errno != errno.EEXIST:
-      raise
-
-
-def construct_documentation(header, partial_specs, image_spec):
-  """Assemble all of the documentation for a single dockerfile.
-
-  Builds explanations of included partials and available build args.
-
-  Args:
-    header: The string from spec.yml["header"]; will be commented and wrapped.
-    partial_specs: The dict from spec.yml["partials"].
-    image_spec: The spec for the dockerfile being built.
-
-  Returns:
-    A string containing a commented header that documents the contents of the
-    dockerfile.
-
-  """
-  # Comment and wrap header and image description
-  commented_header = '\n'.join(
-      [('# ' + l).rstrip() for l in header.splitlines()])
-  commented_desc = '\n'.join(
-      ['# ' + l for l in image_spec.get('desc', '').splitlines()])
-  partial_descriptions = []
-
-  # Build documentation for each partial in the image
-  for partial in image_spec['partials']:
-    # Copy partial data for default args unique to this image
-    partial_spec = copy.deepcopy(partial_specs[partial])
-    args = partial_spec.get('args', dict())
-
-    # Overwrite any existing arg defaults
-    for k_v in image_spec.get('arg-defaults', []):
-      arg, value = list(k_v.items())[0]
-      if arg in args:
-        args[arg]['default'] = value
-
-    # Build the description from new args
-    partial_description = build_partial_description(partial_spec)
-    partial_descriptions.append(partial_description)
-
-  contents = [commented_header, '#', commented_desc] + partial_descriptions
-  return '\n'.join(contents) + '\n'
-
-
-def normalize_partial_args(partial_specs):
-  """Normalize the shorthand form of a partial's args specification.
-
-  Turns this:
-
-    partial:
-      args:
-        SOME_ARG: arg_value
+  required_args = []
+  used_slice_sets = []
+
+  extract_bracketed_words = re.compile(r'\{([^}]+)\}')
+  possible_args_or_slice_set_names = extract_bracketed_words.findall(tag_spec)
+  for name in possible_args_or_slice_set_names:
+    if name in slice_sets:
+      used_slice_sets.append(name)
+    else:
+      required_args.append(name)
 
-  Into this:
+  return (used_slice_sets, required_args)
 
-    partial:
-       args:
-         SOME_ARG:
-            default: arg_value
 
-  Args:
-    partial_specs: The dict from spec.yml["partials"]. This dict is modified in
-      place.
+def gather_tag_args(slices, cli_input_args, required_args):
+  """Build a dictionary of all the CLI and slice-specified args for a tag."""
+  args = dict()
 
-  Returns:
-    The modified contents of partial_specs.
-
-  """
-  for _, partial in partial_specs.items():
-    args = partial.get('args', dict())
-    for arg, value in args.items():
-      if not isinstance(value, dict):
-        new_value = {'default': value}
-        args[arg] = new_value
-
-  return partial_specs
+  for s in slices:
+    args = update_args_dict(args, s['args'])
 
+  args = update_args_dict(args, cli_input_args)
+  for arg in required_args:
+    if arg not in args:
+      eprint(('> Error: {} is not a valid variant, and also isn\'t an arg '
+              'provided on the command line. If it is an arg, please specify '
+              'it with --arg. If not, check the variants list.'.format(arg)))
+      exit(1)
 
-def flatten_args_references(image_specs):
-  """Resolve all default-args in each image spec to a concrete dict.
+  return args
 
-  Turns this:
 
-    example-image:
-      arg-defaults:
-        - MY_ARG: ARG_VALUE
+def gather_slice_list_items(slices, key):
+  """For a list of slices, get the flattened list of all of a certain key."""
+  return list(itertools.chain(*[s[key] for s in slices if key in s]))
 
-    another-example:
-      arg-defaults:
-        - ANOTHER_ARG: ANOTHER_VALUE
-        - example_image
 
-  Into this:
+def find_first_slice_value(slices, key):
+  """For a list of slices, get the first value for a certain key."""
+  for s in slices:
+    if key in s:
+      return s[key]
 
-    example-image:
-      arg-defaults:
-        - MY_ARG: ARG_VALUE
 
-    another-example:
-      arg-defaults:
-        - ANOTHER_ARG: ANOTHER_VALUE
-        - MY_ARG: ARG_VALUE
+def assemble_tags(spec, cli_args, enabled_releases, all_partials):
+  """Gather all the tags based on our spec.
 
   Args:
-    image_specs: A dict of image_spec dicts; should be the contents of the
-      "images" key in the global spec.yaml. This dict is modified in place and
-      then returned.
+    spec: Nested dict containing full Tag spec
+    cli_args: List of ARG=foo arguments to pass along to Docker build
+    enabled_releases: List of releases to parse. Empty list = all
+    all_partials: Dict of every partial, for reference
 
   Returns:
-    The modified contents of image_specs.
+    Dict of tags and how to build them
   """
-  for _, image_spec in image_specs.items():
-    too_deep = 0
-    while str in map(type, image_spec.get('arg-defaults', [])) and too_deep < 5:
-      new_args = []
-      for arg in image_spec['arg-defaults']:
-        if isinstance(arg, str):
-          new_args.extend(image_specs[arg]['arg-defaults'])
-        else:
-          new_args.append(arg)
-
-      image_spec['arg-defaults'] = new_args
-      too_deep += 1
+  tag_data = collections.defaultdict(list)
 
-  return image_specs
+  for name, release in spec['releases'].items():
+    for tag_spec in release['tag_specs']:
+      if enabled_releases and name not in enabled_releases:
+        eprint('> Skipping release {}'.format(name))
+        continue
 
+      used_slice_sets, required_cli_args = get_slice_sets_and_required_args(
+          spec['slice_sets'], tag_spec)
 
-def flatten_partial_references(image_specs):
-  """Resolve all partial references in each image spec to a concrete list.
+      slice_combos = aggregate_all_slice_combinations(spec, used_slice_sets)
+      for slices in slice_combos:
 
-  Turns this:
+        tag_args = gather_tag_args(slices, cli_args, required_cli_args)
+        tag_name = build_name_from_slices(tag_spec, slices, tag_args,
+                                          release['is_dockerfiles'])
+        used_partials = gather_slice_list_items(slices, 'partials')
+        used_tests = gather_slice_list_items(slices, 'tests')
+        test_runtime = find_first_slice_value(slices, 'test_runtime')
+        dockerfile_contents = merge_partials(spec['header'], used_partials,
+                                             all_partials)
 
-    example-image:
-      partials:
-        - foo
+        tag_data[tag_name].append({
+            'release': name,
+            'tag_spec': tag_spec,
+            'is_dockerfiles': release['is_dockerfiles'],
+            'upload_images': release['upload_images'],
+            'cli_args': tag_args,
+            'partials': used_partials,
+            'tests': used_tests,
+            'test_runtime': test_runtime,
+            'dockerfile_contents': dockerfile_contents,
+        })
 
-    another-example:
-      partials:
-        - bar
-        - image: example-image
-        - bat
+  return tag_data
 
-  Into this:
 
-    example-image:
-      partials:
-        - foo
+def merge_partials(header, used_partials, all_partials):
+  """Merge all partial contents with their header."""
+  used_partials = list(used_partials)
+  return '\n'.join([header] + [all_partials[u] for u in used_partials])
 
-    another-example:
-      partials:
-        - bar
-        - foo
-        - bat
-  Args:
-    image_specs: A dict of image_spec dicts; should be the contents of the
-      "images" key in the global spec.yaml. This dict is modified in place and
-      then returned.
 
-  Returns:
-    The modified contents of image_specs.
-  """
-  for _, image_spec in image_specs.items():
-    too_deep = 0
-    while dict in map(type, image_spec['partials']) and too_deep < 5:
-      new_partials = []
-      for partial in image_spec['partials']:
-        if isinstance(partial, str):
-          new_partials.append(partial)
-        else:
-          new_partials.extend(image_specs[partial['image']]['partials'])
+def upload_in_background(hub_repository, dock, image, tag):
+  """Upload a docker image (to be used by multiprocessing)."""
+  image.tag(hub_repository, tag=tag)
+  for line in list(dock.images.push(hub_repository, tag=tag, stream=True)):
+    print(line)
 
-      image_spec['partials'] = new_partials
-      too_deep += 1
 
-  return image_specs
+def mkdir_p(path):
+  """Create a directory and its parents, even if it already exists."""
+  try:
+    os.makedirs(path)
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      raise
 
 
-def construct_dockerfiles(tf_spec):
-  """Generate a mapping of {"cpu": <cpu dockerfile contents>, ...}.
+def gather_existing_partials(partial_path):
+  """Find and read all available partials.
 
   Args:
-    tf_spec: The full spec.yml loaded as a python object.
+    partial_path (string): read partials from this directory.
 
   Returns:
-    A string:string dict of short names ("cpu-devel") to Dockerfile contents.
+    Dict[string, string] of partial short names (like "ubuntu/python" or
+      "bazel") to the full contents of that partial.
   """
-  names_to_contents = dict()
-  image_specs = tf_spec['images']
-  image_specs = flatten_partial_references(image_specs)
-  image_specs = flatten_args_references(image_specs)
-  partial_specs = tf_spec['partials']
-  partial_specs = normalize_partial_args(partial_specs)
-
-  for name, image_spec in image_specs.items():
-    if not image_spec.get('create-dockerfile', True):
-      continue
-    documentation = construct_documentation(tf_spec['header'], partial_specs,
-                                            image_spec)
-    contents = construct_contents(partial_specs, image_spec)
-    names_to_contents[name] = '\n'.join([documentation, contents])
-
-  return names_to_contents
+  partials = dict()
+  for path, _, files in os.walk(partial_path):
+    for name in files:
+      fullpath = os.path.join(path, name)
+      if '.partial.Dockerfile' not in fullpath:
+        eprint(('> Probably not a problem: skipping {}, which is not a '
+                'partial.').format(fullpath))
+        continue
+      # partial_dir/foo/bar.partial.Dockerfile -> foo/bar
+      simple_name = fullpath[len(partial_path) + 1:-len('.partial.dockerfile')]
+      with open(fullpath, 'r') as f:
+        partial_contents = f.read()
+      partials[simple_name] = partial_contents
+  return partials
 
 
 def main(argv):
   if len(argv) > 1:
-    raise app.UsageError('Unexpected command line args found: {}'.format(argv))
+    raise app.UsageError('Too many command-line arguments.')
 
+  # Read the full spec file, used for everything
   with open(FLAGS.spec_file, 'r') as spec_file:
-    tf_spec = yaml.load(spec_file)
+    tag_spec = yaml.load(spec_file)
+
+  # Get existing partial contents
+  partials = gather_existing_partials(FLAGS.partial_dir)
 
   # Abort if spec.yaml is invalid
-  if FLAGS.validate:
-    schema = yaml.load(SCHEMA_TEXT)
-    v = TfDockerValidator(schema)
-    if not v.validate(tf_spec):
-      print('>> ERROR: {} is an invalid spec! The errors are:'.format(
-          FLAGS.spec_file))
-      print(yaml.dump(v.errors, indent=2))
+  schema = yaml.load(SCHEMA_TEXT)
+  v = TfDockerTagValidator(schema, partials=partials)
+  if not v.validate(tag_spec):
+    eprint('> Error: {} is an invalid spec! The errors are:'.format(
+        FLAGS.spec_file))
+    eprint(yaml.dump(v.errors, indent=2))
+    exit(1)
+  tag_spec = v.normalized(tag_spec)
+
+  # Assemble tags and images used to build them
+  all_tags = assemble_tags(tag_spec, FLAGS.arg, FLAGS.release, partials)
+
+  # Empty Dockerfile directory if building new Dockerfiles
+  if FLAGS.construct_dockerfiles:
+    eprint('> Emptying Dockerfile dir "{}"'.format(FLAGS.dockerfile_dir))
+    shutil.rmtree(FLAGS.dockerfile_dir, ignore_errors=True)
+    mkdir_p(FLAGS.dockerfile_dir)
+
+  # Set up Docker helper
+  dock = docker.from_env()
+
+  # Login to Docker if uploading images
+  if FLAGS.upload_to_hub:
+    if not FLAGS.hub_username:
+      eprint('> Error: please set --hub_username when uploading to Dockerhub.')
       exit(1)
-  else:
-    print('>> WARNING: Not validating {}'.format(FLAGS.spec_file))
-
-  # Generate mapping of { "cpu-devel": "<cpu-devel dockerfile contents>", ... }
-  names_to_contents = construct_dockerfiles(tf_spec)
-
-  # Write each completed Dockerfile
-  if not FLAGS.dry_run:
-    print('>> Emptying destination dir "{}"'.format(FLAGS.output_dir))
-    shutil.rmtree(FLAGS.output_dir, ignore_errors=True)
-    mkdir_p(FLAGS.output_dir)
-  else:
-    print('>> Skipping creation of {} (dry run)'.format(FLAGS.output_dir))
-  for name, contents in names_to_contents.items():
-    path = os.path.join(FLAGS.output_dir, name + '.Dockerfile')
-    if FLAGS.dry_run:
-      print('>> Skipping writing contents of {} (dry run)'.format(path))
-      print(contents)
-    else:
-      mkdir_p(FLAGS.output_dir)
-      print('>> Writing {}'.format(path))
-      with open(path, 'w') as f:
-        f.write(contents)
+    if not FLAGS.hub_repository:
+      eprint(
+          '> Error: please set --hub_repository when uploading to Dockerhub.')
+      exit(1)
+    if not FLAGS.hub_password:
+      eprint('> Error: please set --hub_password when uploading to Dockerhub.')
+      exit(1)
+    dock.login(
+        username=FLAGS.hub_username,
+        password=FLAGS.hub_password,
+    )
+
+  # Each tag has a name ('tag') and a definition consisting of the contents
+  # of its Dockerfile, its build arg list, etc.
+  failed_tags = []
+  for tag, tag_defs in all_tags.items():
+    for tag_def in tag_defs:
+      eprint('> Working on {}'.format(tag))
+
+      if FLAGS.exclude_tags_matching and re.match(FLAGS.exclude_tags_matching,
+                                                  tag):
+        eprint('>> Excluded due to match against "{}".'.format(
+            FLAGS.exclude_tags_matching))
+        continue
+
+      if FLAGS.only_tags_matching and not re.match(FLAGS.only_tags_matching,
+                                                   tag):
+        eprint('>> Excluded due to failure to match against "{}".'.format(
+            FLAGS.only_tags_matching))
+        continue
+
+      # Write releases marked "is_dockerfiles" into the Dockerfile directory
+      if FLAGS.construct_dockerfiles:
+        path = os.path.join(FLAGS.dockerfile_dir, tag + '.Dockerfile')
+        if tag_def['is_dockerfiles']:
+          eprint('>> Writing {}...'.format(path))
+          if not FLAGS.dry_run:
+            with open(path, 'w') as f:
+              f.write(tag_def['dockerfile_contents'])
+
+      # Don't build any images for dockerfile-only releases
+      if not FLAGS.build_images:
+        continue
+
+      # Generate a temporary Dockerfile to use to build, since docker-py
+      # needs a filepath relative to the build context (i.e. the current
+      # directory)
+      dockerfile = os.path.join(FLAGS.dockerfile_dir, tag + '.temp.Dockerfile')
+      if not FLAGS.dry_run:
+        with open(dockerfile, 'w') as f:
+          f.write(tag_def['dockerfile_contents'])
+      eprint('>> (Temporary) writing {}...'.format(dockerfile))
+
+      repo_tag = '{}:{}'.format(FLAGS.repository, tag)
+      eprint('>> Building {} using build args:'.format(repo_tag))
+      for arg, value in tag_def['cli_args'].items():
+        eprint('>>> {}={}'.format(arg, value))
+
+      # Note that we are NOT using cache_from, which appears to limit
+      # available cache layers to those from explicitly specified layers. Many
+      # of our layers are similar between local builds, so we want to use the
+      # implied local build cache.
+      tag_failed = False
+      image, logs = None, []
+      if not FLAGS.dry_run:
+        try:
+          image, logs = dock.images.build(
+              timeout=FLAGS.hub_timeout,
+              path='.',
+              dockerfile=dockerfile,
+              buildargs=tag_def['cli_args'],
+              tag=repo_tag)
+
+          # Print logs after finishing
+          log_lines = [l.get('stream', '') for l in logs]
+          eprint(''.join(log_lines))
+
+          # Run tests if requested, and dump output
+          # Could be improved by backgrounding, but would need better
+          # multiprocessing support to track failures properly.
+          if FLAGS.run_tests_path:
+            if not tag_def['tests']:
+              eprint('>>> No tests to run.')
+            for test in tag_def['tests']:
+              eprint('>> Testing {}...'.format(test))
+              container, = dock.containers.run(
+                  image,
+                  '/tests/' + test,
+                  working_dir='/',
+                  log_config={'type': 'journald'},
+                  detach=True,
+                  stderr=True,
+                  stdout=True,
+                  volumes={FLAGS.run_tests_path:
+                           {'bind': '/tests', 'mode': 'ro'}},
+                  runtime=tag_def['test_runtime']),
+              ret = container.wait()
+              code = ret['StatusCode']
+              out = container.logs(stdout=True, stderr=False)
+              err = container.logs(stdout=False, stderr=True)
+              container.remove()
+              if out:
+                eprint('>>> Output stdout:')
+                eprint(out.decode('utf-8'))
+              else:
+                eprint('>>> No test standard out.')
+              if err:
+                eprint('>>> Output stderr:')
+                eprint(out.decode('utf-8'))
+              else:
+                eprint('>>> No test standard err.')
+              if code != 0:
+                eprint('>> {} failed tests with status: "{}"'.format(
+                    repo_tag, code))
+                failed_tags.append(tag)
+                tag_failed = True
+                if FLAGS.stop_on_failure:
+                  eprint('>> ABORTING due to --stop_on_failure!')
+                  exit(1)
+              else:
+                eprint('>> Tests look good!')
+
+        except docker.errors.BuildError as e:
+          eprint('>> {} failed to build with message: "{}"'.format(
+              repo_tag, e.msg))
+          eprint('>> Build logs follow:')
+          log_lines = [l.get('stream', '') for l in e.build_log]
+          eprint(''.join(log_lines))
+          failed_tags.append(tag)
+          tag_failed = True
+          if FLAGS.stop_on_failure:
+            eprint('>> ABORTING due to --stop_on_failure!')
+            exit(1)
+
+        # Clean temporary dockerfiles if they were created earlier
+        if not FLAGS.keep_temp_dockerfiles:
+          os.remove(dockerfile)
+
+      # Upload new images to DockerHub as long as they built + passed tests
+      if FLAGS.upload_to_hub:
+        if not tag_def['upload_images']:
+          continue
+        if tag_failed:
+          continue
+
+        eprint('>> Uploading to {}:{}'.format(FLAGS.hub_repository, tag))
+        if not FLAGS.dry_run:
+          p = multiprocessing.Process(
+              target=upload_in_background,
+              args=(FLAGS.hub_repository, dock, image, tag))
+          p.start()
+
+  if failed_tags:
+    eprint(
+        '> Some tags failed to build or failed testing, check scrollback for '
+        'errors: {}'.format(
+            ','.join(failed_tags)))
+    exit(1)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
index dab7178db3..ecc8fabcce 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
@@ -16,27 +16,12 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, CPU-only environment for developing changes for TensorFlow, with Jupyter included.
-#
-# Start from Ubuntu, with TF development packages (no GPU support)
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the latest version of Bazel and Python development tools.
-#
-# Configure TensorFlow's shell prompt and login tools.
-#
-# Launch Jupyter on execution instead of a bash prompt.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
 
 ARG UBUNTU_VERSION=16.04
-FROM ubuntu:${UBUNTU_VERSION}
+
+FROM ubuntu:${UBUNTU_VERSION} AS base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -48,7 +33,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
-        python-dev \
         rsync \
         software-properties-common \
         unzip \
@@ -59,8 +43,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
+ 
+ENV CI_BUILD_PYTHON python
+
 
-ARG USE_PYTHON_3_NOT_2=True
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -72,10 +59,13 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
@@ -84,6 +74,20 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON}-dev \
     swig
 
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
 # Install bazel
 RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
     curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
@@ -93,11 +97,18 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
-RUN ${PIP} install jupyter
+RUN ${PIP} install jupyter matplotlib
 
-RUN mkdir /notebooks && chmod a+rwx /notebooks
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-WORKDIR /notebooks
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
 EXPOSE 8888
 
-CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"]
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
index 68566ccc8a..2f4a3d6beb 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
@@ -16,25 +16,12 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, CPU-only environment for developing changes for TensorFlow.
-#
-# Start from Ubuntu, with TF development packages (no GPU support)
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the latest version of Bazel and Python development tools.
-#
-# Configure TensorFlow's shell prompt and login tools.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
 
 ARG UBUNTU_VERSION=16.04
-FROM ubuntu:${UBUNTU_VERSION}
+
+FROM ubuntu:${UBUNTU_VERSION} AS base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -46,7 +33,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
-        python-dev \
         rsync \
         software-properties-common \
         unzip \
@@ -57,8 +43,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
+ 
+ENV CI_BUILD_PYTHON python
+
 
-ARG USE_PYTHON_3_NOT_2=True
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -70,10 +59,13 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
@@ -82,6 +74,20 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON}-dev \
     swig
 
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
 # Install bazel
 RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
     curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
index f889ed6f91..166e255289 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
@@ -16,31 +16,14 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, CPU-only environment for using TensorFlow, with Jupyter included.
-#
-# Start from Ubuntu (no GPU support)
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the TensorFlow Python package.
-# --build-arg TF_PACKAGE=tensorflow (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu)
-#    The specific TensorFlow Python package to install
-#
-# Configure TensorFlow's shell prompt and login tools.
-#
-# Launch Jupyter on execution instead of a bash prompt.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
 
 ARG UBUNTU_VERSION=16.04
-FROM ubuntu:${UBUNTU_VERSION}
 
-ARG USE_PYTHON_3_NOT_2=True
+FROM ubuntu:${UBUNTU_VERSION} as base
+
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -52,21 +35,36 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
 ARG TF_PACKAGE=tensorflow
 RUN ${PIP} install ${TF_PACKAGE}
 
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
-RUN ${PIP} install jupyter
+RUN ${PIP} install jupyter matplotlib
 
-RUN mkdir /notebooks && chmod a+rwx /notebooks
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-WORKDIR /notebooks
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
 EXPOSE 8888
 
-CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"]
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
index 182a534bed..323f89155b 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
@@ -16,29 +16,14 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, CPU-only environment for using TensorFlow
-#
-# Start from Ubuntu (no GPU support)
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the TensorFlow Python package.
-# --build-arg TF_PACKAGE=tensorflow (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu)
-#    The specific TensorFlow Python package to install
-#
-# Configure TensorFlow's shell prompt and login tools.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
 
 ARG UBUNTU_VERSION=16.04
-FROM ubuntu:${UBUNTU_VERSION}
 
-ARG USE_PYTHON_3_NOT_2=True
+FROM ubuntu:${UBUNTU_VERSION} as base
+
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -50,10 +35,18 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
 ARG TF_PACKAGE=tensorflow
 RUN ${PIP} install ${TF_PACKAGE}
 
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
similarity index 67%
rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel-jupyter.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
index 17faa84a68..b77ba52f25 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
@@ -16,28 +16,12 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, Nvidia-GPU-enabled environment for developing changes for TensorFlow, with Jupyter included.
-#
-# Start from Nvidia's Ubuntu base image with CUDA and CuDNN, with TF development
-# packages.
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the latest version of Bazel and Python development tools.
-#
-# Configure TensorFlow's shell prompt and login tools.
-#
-# Launch Jupyter on execution instead of a bash prompt.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
 
 ARG UBUNTU_VERSION=16.04
-FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION}
+
+FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -60,6 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
+        python-dev \
         rsync \
         software-properties-common \
         unzip \
@@ -82,11 +67,19 @@ RUN mkdir /usr/local/cuda-9.0/lib &&  \
     ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
     ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
 
-# TODO(tobyboyd): Remove after license is excluded from BUILD file.
-RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \
-    cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/
+# Configure the build for our CUDA configuration.
+ENV CI_BUILD_PYTHON python
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV TF_NEED_CUDA 1
+ENV TF_NEED_TENSORRT 1
+ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
+ENV TF_CUDA_VERSION=9.0
+ENV TF_CUDNN_VERSION=7
+
+# NCCL 2.x
+ENV TF_NCCL_VERSION=2
 
-ARG USE_PYTHON_3_NOT_2=True
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -98,10 +91,13 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
@@ -110,6 +106,20 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON}-dev \
     swig
 
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
 # Install bazel
 RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
     curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
@@ -119,11 +129,18 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
-RUN ${PIP} install jupyter
+RUN ${PIP} install jupyter matplotlib
 
-RUN mkdir /notebooks && chmod a+rwx /notebooks
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-WORKDIR /notebooks
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
 EXPOSE 8888
 
-CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"]
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
similarity index 76%
rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
index a3ba02a684..bcac1f7015 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
@@ -16,26 +16,12 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, Nvidia-GPU-enabled environment for developing changes for TensorFlow.
-#
-# Start from Nvidia's Ubuntu base image with CUDA and CuDNN, with TF development
-# packages.
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the latest version of Bazel and Python development tools.
-#
-# Configure TensorFlow's shell prompt and login tools.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
 
 ARG UBUNTU_VERSION=16.04
-FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION}
+
+FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -58,6 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
+        python-dev \
         rsync \
         software-properties-common \
         unzip \
@@ -80,11 +67,19 @@ RUN mkdir /usr/local/cuda-9.0/lib &&  \
     ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
     ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
 
-# TODO(tobyboyd): Remove after license is excluded from BUILD file.
-RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \
-    cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/
+# Configure the build for our CUDA configuration.
+ENV CI_BUILD_PYTHON python
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV TF_NEED_CUDA 1
+ENV TF_NEED_TENSORRT 1
+ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
+ENV TF_CUDA_VERSION=9.0
+ENV TF_CUDNN_VERSION=7
+
+# NCCL 2.x
+ENV TF_NCCL_VERSION=2
 
-ARG USE_PYTHON_3_NOT_2=True
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -96,10 +91,13 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
@@ -108,6 +106,20 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON}-dev \
     swig
 
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
 # Install bazel
 RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
     curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
similarity index 63%
rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia-jupyter.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index fbdea4628a..9d7340abf3 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -16,30 +16,13 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow, with Jupyter included.
-#
-# NVIDIA with CUDA and CuDNN, no dev stuff
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the TensorFlow Python package.
-# --build-arg TF_PACKAGE=tensorflow-gpu (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu)
-#    The specific TensorFlow Python package to install
-#
-# Configure TensorFlow's shell prompt and login tools.
-#
-# Launch Jupyter on execution instead of a bash prompt.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
 
-FROM nvidia/cuda:9.0-base-ubuntu16.04
+FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
 
-# Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         cuda-command-line-tools-9-0 \
@@ -48,6 +31,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         cuda-curand-9-0 \
         cuda-cusolver-9-0 \
         cuda-cusparse-9-0 \
+        curl \
         libcudnn7=7.2.1.38-1+cuda9.0 \
         libnccl2=2.2.13-1+cuda9.0 \
         libfreetype6-dev \
@@ -55,6 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
+        rsync \
         software-properties-common \
         unzip \
         && \
@@ -66,7 +51,10 @@ RUN apt-get update && \
         apt-get update && \
         apt-get install libnvinfer4=4.1.2-1+cuda9.0
 
-ARG USE_PYTHON_3_NOT_2=True
+# For CUDA profiling, TensorFlow requires CUPTI.
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -78,21 +66,36 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
-ARG TF_PACKAGE=tensorflow-gpu
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
 RUN ${PIP} install ${TF_PACKAGE}
 
 COPY bashrc /etc/bash.bashrc
 RUN chmod a+rwx /etc/bash.bashrc
 
-RUN ${PIP} install jupyter
+RUN ${PIP} install jupyter matplotlib
 
-RUN mkdir /notebooks && chmod a+rwx /notebooks
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-WORKDIR /notebooks
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
 EXPOSE 8888
 
-CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"]
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/nvidia.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
similarity index 69%
rename from tensorflow/tools/dockerfiles/dockerfiles/nvidia.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
index e0312dbc29..e8e6ceafe2 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/nvidia.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
@@ -16,28 +16,13 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# below. Please refer to the the TensorFlow dockerfiles documentation for
-# more information. Build args are documented as their default value.
-#
-# Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow.
-#
-# NVIDIA with CUDA and CuDNN, no dev stuff
-# --build-arg UBUNTU_VERSION=16.04
-#    ( no description )
-#
-# Python is required for TensorFlow and other libraries.
-# --build-arg USE_PYTHON_3_NOT_2=True
-#    Install python 3 over Python 2
-#
-# Install the TensorFlow Python package.
-# --build-arg TF_PACKAGE=tensorflow-gpu (tensorflow|tensorflow-gpu|tf-nightly|tf-nightly-gpu)
-#    The specific TensorFlow Python package to install
-#
-# Configure TensorFlow's shell prompt and login tools.
+# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# for more information.
+
+ARG UBUNTU_VERSION=16.04
 
-FROM nvidia/cuda:9.0-base-ubuntu16.04
+FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
 
-# Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         cuda-command-line-tools-9-0 \
@@ -46,6 +31,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         cuda-curand-9-0 \
         cuda-cusolver-9-0 \
         cuda-cusparse-9-0 \
+        curl \
         libcudnn7=7.2.1.38-1+cuda9.0 \
         libnccl2=2.2.13-1+cuda9.0 \
         libfreetype6-dev \
@@ -53,6 +39,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
+        rsync \
         software-properties-common \
         unzip \
         && \
@@ -64,7 +51,10 @@ RUN apt-get update && \
         apt-get update && \
         apt-get install libnvinfer4=4.1.2-1+cuda9.0
 
-ARG USE_PYTHON_3_NOT_2=True
+# For CUDA profiling, TensorFlow requires CUPTI.
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+
+ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
 ARG PIP=pip${_PY_SUFFIX}
@@ -76,11 +66,19 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
 
-ARG TF_PACKAGE=tensorflow-gpu
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
 RUN ${PIP} install ${TF_PACKAGE}
 
 COPY bashrc /etc/bash.bashrc
diff --git a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
index 2c9b9f3f9a..51b22d1fdf 100644
--- a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
@@ -1,8 +1,15 @@
-RUN ${PIP} install jupyter
+RUN ${PIP} install jupyter matplotlib
 
-RUN mkdir /notebooks && chmod a+rwx /notebooks
+RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
 RUN mkdir /.local && chmod a+rwx /.local
-WORKDIR /notebooks
+RUN apt-get install -y --no-install-recommends wget
+WORKDIR /tf/tensorflow-tutorials
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
+RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+RUN apt-get autoremove -y && apt-get remove -y wget
+WORKDIR /tf
 EXPOSE 8888
 
-CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/notebooks --ip 0.0.0.0 --no-browser --allow-root"]
+RUN ${PYTHON} -m ipykernel.kernelspec
+
+CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
diff --git a/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile
index 96e79547f0..76758bd147 100644
--- a/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/tensorflow.partial.Dockerfile
@@ -1,2 +1,7 @@
-ARG TF_PACKAGE
+# Options:
+#   tensorflow
+#   tensorflow-gpu
+#   tf-nightly
+#   tf-nightly-gpu
+ARG TF_PACKAGE=tensorflow
 RUN ${PIP} install ${TF_PACKAGE}
diff --git a/tensorflow/tools/dockerfiles/partials/test-import.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/test-import.partial.Dockerfile
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile
deleted file mode 100644
index 0a50735bf8..0000000000
--- a/tensorflow/tools/dockerfiles/partials/ubuntu.partial.Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-ARG UBUNTU_VERSION=16.04
-FROM ubuntu:${UBUNTU_VERSION}
diff --git a/tensorflow/tools/dockerfiles/partials/bazel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/bazel.partial.Dockerfile
similarity index 58%
rename from tensorflow/tools/dockerfiles/partials/bazel.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/bazel.partial.Dockerfile
index b08d8bdd14..156bb01991 100644
--- a/tensorflow/tools/dockerfiles/partials/bazel.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/bazel.partial.Dockerfile
@@ -6,6 +6,20 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON}-dev \
     swig
 
+RUN ${PIP} --no-cache-dir install \
+    Pillow \
+    h5py \
+    keras_applications \
+    keras_preprocessing \
+    matplotlib \
+    mock \
+    numpy \
+    scipy \
+    sklearn \
+    pandas \
+    && test "${USE_PYTHON_3_NOT_2}" -eq 1 && true || ${PIP} --no-cache-dir install \
+    enum34
+
 # Install bazel
 RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
     curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile
similarity index 86%
rename from tensorflow/tools/dockerfiles/partials/ubuntu-devel.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile
index bc79272276..901652cc28 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu-devel.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile
@@ -1,5 +1,4 @@
-ARG UBUNTU_VERSION=16.04
-FROM ubuntu:${UBUNTU_VERSION}
+FROM ubuntu:${UBUNTU_VERSION} AS base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -11,7 +10,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
-        python-dev \
         rsync \
         software-properties-common \
         unzip \
@@ -22,3 +20,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
+ 
+ENV CI_BUILD_PYTHON python
+
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile
new file mode 100644
index 0000000000..d01b26e27f
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/cpu.partial.Dockerfile
@@ -0,0 +1 @@
+FROM ubuntu:${UBUNTU_VERSION} as base
diff --git a/tensorflow/tools/dockerfiles/partials/nvidia-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile
similarity index 78%
rename from tensorflow/tools/dockerfiles/partials/nvidia-devel.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile
index 45159f711f..48d457e40c 100644
--- a/tensorflow/tools/dockerfiles/partials/nvidia-devel.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile
@@ -1,5 +1,4 @@
-ARG UBUNTU_VERSION=16.04
-FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION}
+FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -22,6 +21,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
+        python-dev \
         rsync \
         software-properties-common \
         unzip \
@@ -44,6 +44,14 @@ RUN mkdir /usr/local/cuda-9.0/lib &&  \
     ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
     ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
 
-# TODO(tobyboyd): Remove after license is excluded from BUILD file.
-RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \
-    cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/
+# Configure the build for our CUDA configuration.
+ENV CI_BUILD_PYTHON python
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV TF_NEED_CUDA 1
+ENV TF_NEED_TENSORRT 1
+ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
+ENV TF_CUDA_VERSION=9.0
+ENV TF_CUDNN_VERSION=7
+
+# NCCL 2.x
+ENV TF_NCCL_VERSION=2
diff --git a/tensorflow/tools/dockerfiles/partials/nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
similarity index 78%
rename from tensorflow/tools/dockerfiles/partials/nvidia.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
index 1064390af3..1dc8e43aad 100644
--- a/tensorflow/tools/dockerfiles/partials/nvidia.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile
@@ -1,6 +1,5 @@
-FROM nvidia/cuda:9.0-base-ubuntu16.04
+FROM nvidia/cuda:9.0-base-ubuntu${UBUNTU_VERSION} as base
 
-# Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         cuda-command-line-tools-9-0 \
@@ -9,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         cuda-curand-9-0 \
         cuda-cusolver-9-0 \
         cuda-cusparse-9-0 \
+        curl \
         libcudnn7=7.2.1.38-1+cuda9.0 \
         libnccl2=2.2.13-1+cuda9.0 \
         libfreetype6-dev \
@@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libpng12-dev \
         libzmq3-dev \
         pkg-config \
+        rsync \
         software-properties-common \
         unzip \
         && \
@@ -26,3 +27,6 @@ RUN apt-get update && \
         apt-get install nvinfer-runtime-trt-repo-ubuntu1604-4.0.1-ga-cuda9.0 && \
         apt-get update && \
         apt-get install libnvinfer4=4.1.2-1+cuda9.0
+
+# For CUDA profiling, TensorFlow requires CUPTI.
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
diff --git a/tensorflow/tools/dockerfiles/partials/python.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile
similarity index 66%
rename from tensorflow/tools/dockerfiles/partials/python.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile
index ee08af73a8..6af4731953 100644
--- a/tensorflow/tools/dockerfiles/partials/python.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/python.partial.Dockerfile
@@ -10,6 +10,9 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON}-pip
 
-RUN ${PIP} install --upgrade \
+RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
+
+# Some TF tools expect a "python" binary
+RUN ln -s $(which ${PYTHON}) /usr/local/bin/python 
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/test-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/test-devel.partial.Dockerfile
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile
new file mode 100644
index 0000000000..6ecd2b8b1a
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/version.partial.Dockerfile
@@ -0,0 +1 @@
+ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/spec.yml b/tensorflow/tools/dockerfiles/spec.yml
index 28bf9a55da..5049e8dcfb 100644
--- a/tensorflow/tools/dockerfiles/spec.yml
+++ b/tensorflow/tools/dockerfiles/spec.yml
@@ -1,195 +1,135 @@
-# ======
-# HEADER
-# ======
-#
-# This is commented-out and prepended to each generated Dockerfile.
 header: |
-    Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-    ============================================================================
-
-    THIS IS A GENERATED DOCKERFILE.
-
-    This file was assembled from multiple pieces, whose use is documented
-    below. Please refer to the the TensorFlow dockerfiles documentation for
-    more information. Build args are documented as their default value.
-
-# ========
-# PARTIALS
-# ========
+    # Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+    #
+    # Licensed under the Apache License, Version 2.0 (the "License");
+    # you may not use this file except in compliance with the License.
+    # You may obtain a copy of the License at
+    #
+    #     http://www.apache.org/licenses/LICENSE-2.0
+    #
+    # Unless required by applicable law or agreed to in writing, software
+    # distributed under the License is distributed on an "AS IS" BASIS,
+    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    # See the License for the specific language governing permissions and
+    # limitations under the License.
+    # ============================================================================
+    #
+    # THIS IS A GENERATED DOCKERFILE.
+    #
+    # This file was assembled from multiple pieces, whose use is documented
+    # throughout. Please refer to the the TensorFlow dockerfiles documentation
+    # for more information.
+
+# A combinatorial explosion of Docker images and Dockerfiles.
+# Each "release" defines all of the ways to combine related but separate chunks
+# of functionality ("slices") by listing all of the "slice sets" to use when
+# building.
 #
-# Represent and document pieces of a Dockerfile. Spec:
-# 
-# name: the name of the partial, is referenced from the images section
-#   desc: A description, inserted later into the Dockerfile
-#   file: Alternative file prefix, e.g. file.partial.Dockerfile. The default is
-#         the name of the partial.
-#   args: A dict of ARGs in the Dockerfile; each entry has the format
-#      ARG_NAME: VALUE where VALUE is one of:
-#         - a dict:
-#             desc: Documentation for the arg
-#             default: Default value for the arg; is written to the Dockerfile
-#             options: List of strings, part of documentation
-#         - a concrete value: the same as a dictionary with default: [value].
-
-partials:
-    ubuntu:
-        desc: Start from Ubuntu (no GPU support)
-        args:
-            UBUNTU_VERSION: 16.04
-
-    ubuntu-devel:
-        desc: Start from Ubuntu, with TF development packages (no GPU support)
-        args:
-            UBUNTU_VERSION: 16.04
-
-    bazel:
-        desc: Install the latest version of Bazel and Python development tools.
-
-    nvidia:
-        desc: NVIDIA with CUDA and CuDNN, no dev stuff
-        args:
-            UBUNTU_VERSION: 16.04
-
-    nvidia-devel:
-        desc: >
-            Start from Nvidia's Ubuntu base image with CUDA and CuDNN, with TF
-            development packages.
-        args:
-            UBUNTU_VERSION: 16.04
+# For example, a release that uses {nightly}{py} would create 4 Dockerfiles
+# (which could become images or concrete Dockerfiles), because the "nightly"
+# and "py" slice sets both have two entries:
+#
+#   - nightly (no -py2 because the Python 2 slice set has add_to_name: ""
+#   - nightly-py3
+#   - nightly-gpu (similar)
+#   - nightly-gpu-py3
+
+releases:
+    nightly:
+        tag_specs:
+            - "{nightly}{py}{jupyter}"
+
+    versioned:
+        tag_specs:
+            - "{_TAG_PREFIX}{ubuntu}{py}{jupyter}"
+
+    ubuntu-dockerfiles:
+        is_dockerfiles: true
+        upload_images: false
+        tag_specs:
+            - "{ubuntu}{jupyter}"
+
+slice_sets:
+
+    py:
+        - add_to_name: ""
+          args:
+              - USE_PYTHON_3_NOT_2=
+        - add_to_name: "-py3"
+          args:
+              - USE_PYTHON_3_NOT_2=1
 
-    python:
-        desc: Python is required for TensorFlow and other libraries.
-        args:
-            USE_PYTHON_3_NOT_2:
-                default: true
-                desc: Install python 3 over Python 2
-                
-    tensorflow:
-        desc: Install the TensorFlow Python package.
-        args:
-            TF_PACKAGE:
-                default: tensorflow
-                options:
-                    - tensorflow
-                    - tensorflow-gpu
-                    - tf-nightly
-                    - tf-nightly-gpu
-                desc: The specific TensorFlow Python package to install
-    shell:
-        desc: Configure TensorFlow's shell prompt and login tools.
     jupyter:
-        desc: Launch Jupyter on execution instead of a bash prompt.
-
-# ======
-# IMAGES
-# ======
-# 
-# Represent Dockerfiles. Spec:
-# 
-# name: the name of the image, possibly referenced by other images
-#   desc: A description, inserted later into the Dockerfile
-#   create-dockerfile: Create a dockerfile based on this. Useful for creating
-#      extensible base images that don't need a file. Default is true.
-#   partials: List of VALUEs, where a VALUE is either:
-#      - the name of a partial, which inserts that partial into this image
-#      - image: [name of another image], which inserts the partials from that
-#        image into this image
-#   arg-defaults: List of VALUEs, where a VALUE is either:
-#      - ARG_NAME: VALUE, which sets the ARG_NAME to VALUE wherever it appears
-#        in this image's partials
-#      - [name of another image], which loads the default args from that image
-images:
-
-    nodev:
-        create-dockerfile: false
-        partials:
-            - python
-            - tensorflow
-            - shell
-
-    dev:
-        create-dockerfile: false
-        partials:
-            - python
-            - bazel
-            - shell
-
-    cpu:
-      desc: Ubuntu-based, CPU-only environment for using TensorFlow
-      partials:
-        - ubuntu
-        - image: nodev
-
-    cpu-devel:
-      desc: >
-          Ubuntu-based, CPU-only environment for developing changes for
-          TensorFlow.
-      partials:
-        - ubuntu-devel
-        - image: dev
+        - add_to_name: ""
+        - add_to_name: "-jupyter"
+          partials:
+              - jupyter
 
-    nvidia:
-      desc: Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow.
-      arg-defaults: 
-        - TF_PACKAGE: tensorflow-gpu
-      partials:
-        - nvidia
-        - image: nodev
-
-    nvidia-devel:
-      desc: >
-          Ubuntu-based, Nvidia-GPU-enabled environment for developing changes
-          for TensorFlow.
-      arg-defaults: 
-        - TF_PACKAGE: tensorflow-gpu
-      partials:
-        - nvidia-devel
-        - image: dev
-
-    cpu-jupyter:
-      desc: >
-          Ubuntu-based, CPU-only environment for using TensorFlow, with Jupyter
-          included.
-      partials:
-        - image: cpu
-        - jupyter
-
-    cpu-devel-jupyter:
-      desc: >
-         Ubuntu-based, CPU-only environment for developing changes for
-         TensorFlow, with Jupyter included.
-      partials:
-        - image: cpu-devel
-        - jupyter
-
-    nvidia-jupyter:
-      desc: >
-        Ubuntu-based, Nvidia-GPU-enabled environment for using TensorFlow, with
-        Jupyter included.
-      arg-defaults: 
-        - nvidia
-      partials:
-        - image: nvidia
-        - jupyter
-
-    nvidia-devel-jupyter:
-      desc: >
-        Ubuntu-based, Nvidia-GPU-enabled environment for developing changes for
-        TensorFlow, with Jupyter included.
-      arg-defaults: 
-        - nvidia-devel
-      partials:
-        - image: nvidia-devel
-        - jupyter
+    ubuntu:
+        - add_to_name: ""
+          dockerfile_exclusive_name: "cpu"
+          partials:
+              - ubuntu/version
+              - ubuntu/cpu
+              - ubuntu/python
+              - tensorflow
+              - shell
+        - add_to_name: "-gpu"
+          dockerfile_exclusive_name: "gpu"
+          args:
+              - TF_PACKAGE=tensorflow-gpu
+          partials:
+              - ubuntu/version
+              - ubuntu/nvidia
+              - ubuntu/python
+              - tensorflow
+              - shell
+          tests:
+              - import-gpu.sh
+          test_runtime: nvidia
+        - add_to_name: "-devel"
+          dockerfile_exclusive_name: "cpu-devel"
+          partials:
+              - ubuntu/version
+              - ubuntu/cpu-devel
+              - ubuntu/python
+              - ubuntu/bazel
+              - shell
+          tests:
+              - build-cpu.sh
+        - add_to_name: "-gpu-devel"
+          dockerfile_exclusive_name: "gpu-devel"
+          partials:
+              - ubuntu/version
+              - ubuntu/nvidia-devel
+              - ubuntu/python
+              - ubuntu/bazel
+              - shell
+          tests:
+              - build-gpu.sh
+          test_runtime: nvidia
+
+    nightly:
+        - add_to_name: "nightly"
+          partials:
+              - ubuntu/version
+              - ubuntu/cpu
+              - ubuntu/python
+              - tensorflow
+              - shell
+          args:
+              - TF_PACKAGE=tf-nightly
+          tests:
+              - import.sh
+        - add_to_name: "nightly-gpu"
+          partials:
+              - ubuntu/version
+              - ubuntu/nvidia
+              - ubuntu/python
+              - tensorflow
+              - shell
+          test_runtime: nvidia
+          tests:
+              - import-gpu.sh
+          args:
+              - TF_PACKAGE=tf-nightly-gpu
diff --git a/tensorflow/tools/dockerfiles/tests/build-cpu.sh b/tensorflow/tools/dockerfiles/tests/build-cpu.sh
new file mode 100755
index 0000000000..337239dc38
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/tests/build-cpu.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Download and build TensorFlow.
+set -euxo pipefail
+git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow
+cd /tensorflow
+
+ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+# For optimized builds appropriate for the hardware platform of your choosing, uncomment below...
+# For ivy-bridge or sandy-bridge
+# --copt=-march="ivybridge" \
+# for haswell, broadwell, or skylake
+# --copt=-march="haswell" \
+tensorflow/tools/ci_build/builds/configured CPU \
+  bazel build -c opt --copt=-mavx --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+      tensorflow/tools/pip_package:build_pip_package && \
+  bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
+  pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
+  rm -rf /tmp/pip && \
+  rm -rf /root/.cache
+
diff --git a/tensorflow/tools/dockerfiles/tests/build-gpu.sh b/tensorflow/tools/dockerfiles/tests/build-gpu.sh
new file mode 100755
index 0000000000..7a1e7ad340
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/tests/build-gpu.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Download and build TensorFlow.
+set -euxo pipefail
+git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow
+cd /tensorflow
+
+ln -s $(which ${PYTHON}) /usr/local/bin/python 
+
+ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
+
+LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
+tensorflow/tools/ci_build/builds/configured GPU \
+bazel build -c opt --copt=-mavx --config=cuda \
+    --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+    tensorflow/tools/pip_package:build_pip_package && \
+rm /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
+pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
+rm -rf /tmp/pip && \
+rm -rf /root/.cache
diff --git a/tensorflow/tools/dockerfiles/tests/import-gpu.sh b/tensorflow/tools/dockerfiles/tests/import-gpu.sh
new file mode 100755
index 0000000000..214e5ccf2c
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/tests/import-gpu.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+python -c 'import tensorflow as tf; tf.test.is_gpu_available() or exit(1)'
diff --git a/tensorflow/tools/dockerfiles/tests/import.sh b/tensorflow/tools/dockerfiles/tests/import.sh
new file mode 100755
index 0000000000..79998aad77
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/tests/import.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+python -c 'import tensorflow as tf'
diff --git a/tensorflow/tools/dockerfiles/assembler.Dockerfile b/tensorflow/tools/dockerfiles/tools.Dockerfile
similarity index 95%
rename from tensorflow/tools/dockerfiles/assembler.Dockerfile
rename to tensorflow/tools/dockerfiles/tools.Dockerfile
index 7a8e07fced..e8929295a5 100644
--- a/tensorflow/tools/dockerfiles/assembler.Dockerfile
+++ b/tensorflow/tools/dockerfiles/tools.Dockerfile
@@ -20,8 +20,9 @@
 FROM debian:stretch
 LABEL maintainer="Austin Anderson <angerson@google.com>"
 
-RUN apt-get update && apt-get install -y python3 python3-pip bash
-RUN pip3 install --upgrade pip setuptools pyyaml absl-py cerberus
+RUN apt-get update && apt-get install -y python3 python3-pip bash curl
+RUN curl -sSL https://get.docker.com/ | sh
+RUN pip3 install --upgrade pip setuptools pyyaml absl-py cerberus docker
 
 WORKDIR /tf
 VOLUME ["/tf"]
-- 
GitLab


From a9f443cd1f053949391ace983f271b3aa2a54631 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 29 Nov 2018 14:20:55 -0800
Subject: [PATCH 1059/1554] Add publicly available corpus and dictionary for
 decode_png_fuzz.

PiperOrigin-RevId: 223410548
---
 tensorflow/core/kernels/fuzzing/BUILD         |   5 ++
 .../010dc3d4b05288fcc40de2721052b3dc699f1cb3  | Bin 0 -> 233 bytes
 .../0555cd5e9d99629819cc985285f80da0f00be1e9  | Bin 0 -> 474 bytes
 .../0a0352aa168803ff65455792d9f6ee555c3e7c3f  | Bin 0 -> 232 bytes
 .../0ed54162df93ef8d00f993ce6b59ba422903d381  | Bin 0 -> 221 bytes
 .../1547b448171c700613c3946d730de496c9b9863f  | Bin 0 -> 260 bytes
 .../17859046cbe4ac598a645173d679ce2a52c6afba  | Bin 0 -> 525 bytes
 .../1df76c07817fbc3653a26f34d97658e9973627c2  | Bin 0 -> 666 bytes
 .../1f0717f8856d7782e3ab7992d3a72d783a018443  | Bin 0 -> 122 bytes
 .../23b911e4ce936def88bc9a46b8b433c0e83fba2a  | Bin 0 -> 375 bytes
 .../25592201c3edff0578dbdac6b0e4f2be109ce151  | Bin 0 -> 329 bytes
 .../266fd8495e0b8eb64387c1a62264185e061fee73  | Bin 0 -> 360 bytes
 .../27f178cf415b4ff8671131ddf1d042dafac2fb3e  | Bin 0 -> 677 bytes
 .../2a0bdc4d9cc5ea5bb21dd256d6ac96075376a94f  | Bin 0 -> 246 bytes
 .../2e5d25add6adc68e0457b358c7a34abf3d41c938  | Bin 0 -> 165 bytes
 .../2e6c5b6a766dd5e9bd41eacfd0a36572bd2f7544  | Bin 0 -> 320 bytes
 .../2e9c935cf82f6ca640e9a9abc3c30a578ad46176  | Bin 0 -> 666 bytes
 .../2fcf1ed4477f7eaee028f5b3f9edeb5f1a737826  | Bin 0 -> 507 bytes
 .../3480713774f590908ca5dba16d121cdfb8fba62b  | Bin 0 -> 665 bytes
 .../39289afcec60d98802b333e0fbb1da4d7aed4ce5  | Bin 0 -> 247 bytes
 .../3adc488e21d4aca7bed9422f0241a42d0f93e7d9  | Bin 0 -> 366 bytes
 .../3cbf274da522483dc991fad9df43a22ac4fb3173  | Bin 0 -> 474 bytes
 .../3d840cdff7f5ad16fe8bcb985ed4946c03459432  | Bin 0 -> 666 bytes
 .../3f1e6753c1fca958e859189857449746592158ea  | Bin 0 -> 94 bytes
 .../3fa4075993cb0f9bfa8eea785174a2038a69aa1b  | Bin 0 -> 343 bytes
 .../4023a373e977be58413e55350380310c5dd1fd6a  | Bin 0 -> 241 bytes
 .../40caba69dce1cfc48e0e43184d2bfbc6daa4399a  | Bin 0 -> 470 bytes
 .../41841e9561d8135945c1c1e55ab9e9a1e933653b  | Bin 0 -> 247 bytes
 .../41d40f2d66fa43e34537385594ee9911e65deadf  | Bin 0 -> 677 bytes
 .../421bd39810b50309a71adb2dadc3b19f01a52312  | Bin 0 -> 221 bytes
 .../446c305b2c0665736f94fb2b62dbdef445eff0cf  | Bin 0 -> 174 bytes
 .../449cee952bb645f6f4241a6665d3c6028c073c7a  | Bin 0 -> 374 bytes
 .../45520b07609978c5aa3516d803527438b93fbadb  | Bin 0 -> 158 bytes
 .../4da74a34bcede234b0415f77fbd87d70bf9a777e  | Bin 0 -> 94 bytes
 .../51db5d31d2c5300d34831d9f23bcdd0aff9a998b  | Bin 0 -> 677 bytes
 .../5cde2a9167798cb77f10abbfb2640a5c357f99fc  | Bin 0 -> 507 bytes
 .../5e352fc10ac476cfbe1d755f092e069820223249  | Bin 0 -> 169 bytes
 .../63661677dd1306cec4b5a565190e65adf2446e52  | Bin 0 -> 374 bytes
 .../65887ed3db382aab1d9485c500f4401318d303b9  | Bin 0 -> 224 bytes
 .../67b5181f8f0644597e9bde539e8f083b5cacd0e7  | Bin 0 -> 165 bytes
 .../74c9dcf7afee2a6cb1ab3a2c0de744d1b03c1466  | Bin 0 -> 307 bytes
 .../792181ca19e6ded261434e588bb7fc2a4816d4ce  | Bin 0 -> 544 bytes
 .../79f0e2a475487f8fa69e68c1cc947c5851bda741  | Bin 0 -> 355 bytes
 .../7e5fcdfeb557ce379ed96925c68505eaac0112db  | Bin 0 -> 315 bytes
 .../7eec7530acf34b3a96fa9189783453999f7b6838  | Bin 0 -> 671 bytes
 .../80114bf9781bffc9db411413d83541d8deaaf7c1  | Bin 0 -> 343 bytes
 .../80425fb92bb86627e854892f23823fa804e5fdc3  | Bin 0 -> 441 bytes
 .../821cdd6eeb919a8dd7f35289abbd583828dd4945  | Bin 0 -> 677 bytes
 .../83e1a31785285338b0ddb3334b0ed098e63dedde  | Bin 0 -> 285 bytes
 .../8a4c8100dedd0fb5f2a8b468c678f7ad8269deeb  | Bin 0 -> 666 bytes
 .../8ae8268c24dc866c1edb3826b93a1c75dbf74ff4  | Bin 0 -> 677 bytes
 .../90f72038cc627f34f074ea72eadbba87a5e3e288  | Bin 0 -> 677 bytes
 .../92b67faee4a49df2cdbed785e27b4a1cddcfffa3  | Bin 0 -> 507 bytes
 .../9463810467aacdc9923b2b20a2236116b760d75b  | Bin 0 -> 258 bytes
 .../94d7c96aea32ad41ce643d35b951a6d8990b81d6  | Bin 0 -> 546 bytes
 .../98cc7e9fe87df914d89a0aef008930f27b3c26f5  | Bin 0 -> 311 bytes
 .../99172dfdb4f59aaced29c7681ac6e6ce8356e814  | Bin 0 -> 104 bytes
 .../9ae3b647d895af97fe872c0b1442df7b5b767160  | Bin 0 -> 332 bytes
 .../9d2b1d2121b0508a4fa8d1508adb9d05633fdac3  | Bin 0 -> 222 bytes
 .../a335af37917ccf0c8b11bb884a3a74f3f1d2a7c6  | Bin 0 -> 677 bytes
 .../a738609112d3a6772c50a71e2c3504ebc515b709  | Bin 0 -> 337 bytes
 .../a8cecab5d917da5a4729632a7a18c564d7e1607d  | Bin 0 -> 258 bytes
 .../ade919ab2b4a458e806575c941dfe50ae3fd3621  | Bin 0 -> 677 bytes
 .../b1251621a5eb5e7fda9cac9baead1c993a285c36  | Bin 0 -> 277 bytes
 .../b1516b78c3dfe77eeb554985fd7344c0478fbbcb  | Bin 0 -> 258 bytes
 .../b41241740f5f8ad2c1d408f7bb6a313bd863c158  | Bin 0 -> 133 bytes
 .../b799c8596523a7ebeb8e11ada08818c10f7eabfc  | Bin 0 -> 582 bytes
 .../ba48d0521a111222dc95a3a997c7c92dea5f4443  | Bin 0 -> 249 bytes
 .../c01457c6889fb1b597d308363a36412c0b7f90e7  | Bin 0 -> 497 bytes
 .../c82ebc0d6688d104af04fd20d6d3da591dc391f7  | Bin 0 -> 198 bytes
 .../c9a03eb758dd84e954e3d70916e2311e8fd21f3c  | Bin 0 -> 249 bytes
 .../cf892756b33578a54ab20044514e573328d2f1d7  | Bin 0 -> 249 bytes
 .../d3bc3f158a63f1d50b474addd3f7b3d17f23e8e9  | Bin 0 -> 507 bytes
 .../d4906950aa9d60ad09dc0f5413c3d88080c3bc37  | Bin 0 -> 262 bytes
 .../da31578a8068bad65e1c7a3d06e8f543a2a0bc65  | Bin 0 -> 332 bytes
 .../dd4a9b5d0740679c249fc884efc499433b29436b  | Bin 0 -> 666 bytes
 .../deea4ecc6f0b2a6d89fd25ff76762299f21602fb  | Bin 0 -> 677 bytes
 .../e1040c7ffcb39915e0f539018c81f9798924cba6  | Bin 0 -> 94 bytes
 .../e381dc85682cc33ad99f622b89d145b47f7d6392  | Bin 0 -> 485 bytes
 .../ea24498fc7a144fccc6f1665ebf7020df803dd1a  | Bin 0 -> 677 bytes
 .../eaa5d677e797c07bac98c3c7051abad91852e7c6  | Bin 0 -> 507 bytes
 .../ed7871269315725535d8bffec7836c45a3fc5c26  | Bin 0 -> 527 bytes
 .../ee8460f4077064c5a2137075b48eba7d3db5c570  | Bin 0 -> 94 bytes
 .../ef09f26e0ee61329f84a9f589629a865ae9ee0a6  | Bin 0 -> 276 bytes
 .../f477da4d7d8ff2066041e1dd5ee4e833b7111a1a  | Bin 0 -> 527 bytes
 .../f8a379b2498a4eb452a85791a49adf065dab59ae  | Bin 0 -> 666 bytes
 .../fe67bccb06f2174523943cc684518fcf1f7f8046  | Bin 0 -> 80 bytes
 .../ff1e67d17c1c27ef0d97900d0ea276b563a64628  | Bin 0 -> 271 bytes
 .../fuzzing/dictionaries/decode_png.dict      |  50 ++++++++++++++++++
 .../fuzzing/tf_ops_fuzz_target_lib.bzl        |   6 +++
 90 files changed, 61 insertions(+)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/010dc3d4b05288fcc40de2721052b3dc699f1cb3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/0555cd5e9d99629819cc985285f80da0f00be1e9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/0a0352aa168803ff65455792d9f6ee555c3e7c3f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/0ed54162df93ef8d00f993ce6b59ba422903d381
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/1547b448171c700613c3946d730de496c9b9863f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/17859046cbe4ac598a645173d679ce2a52c6afba
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/1df76c07817fbc3653a26f34d97658e9973627c2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/1f0717f8856d7782e3ab7992d3a72d783a018443
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/23b911e4ce936def88bc9a46b8b433c0e83fba2a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/25592201c3edff0578dbdac6b0e4f2be109ce151
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/266fd8495e0b8eb64387c1a62264185e061fee73
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/27f178cf415b4ff8671131ddf1d042dafac2fb3e
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2a0bdc4d9cc5ea5bb21dd256d6ac96075376a94f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2e5d25add6adc68e0457b358c7a34abf3d41c938
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2e6c5b6a766dd5e9bd41eacfd0a36572bd2f7544
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2e9c935cf82f6ca640e9a9abc3c30a578ad46176
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/2fcf1ed4477f7eaee028f5b3f9edeb5f1a737826
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3480713774f590908ca5dba16d121cdfb8fba62b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/39289afcec60d98802b333e0fbb1da4d7aed4ce5
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3adc488e21d4aca7bed9422f0241a42d0f93e7d9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3cbf274da522483dc991fad9df43a22ac4fb3173
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3d840cdff7f5ad16fe8bcb985ed4946c03459432
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3f1e6753c1fca958e859189857449746592158ea
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/3fa4075993cb0f9bfa8eea785174a2038a69aa1b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/4023a373e977be58413e55350380310c5dd1fd6a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/40caba69dce1cfc48e0e43184d2bfbc6daa4399a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/41841e9561d8135945c1c1e55ab9e9a1e933653b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/41d40f2d66fa43e34537385594ee9911e65deadf
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/421bd39810b50309a71adb2dadc3b19f01a52312
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/446c305b2c0665736f94fb2b62dbdef445eff0cf
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/449cee952bb645f6f4241a6665d3c6028c073c7a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/45520b07609978c5aa3516d803527438b93fbadb
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/4da74a34bcede234b0415f77fbd87d70bf9a777e
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/51db5d31d2c5300d34831d9f23bcdd0aff9a998b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/5cde2a9167798cb77f10abbfb2640a5c357f99fc
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/5e352fc10ac476cfbe1d755f092e069820223249
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/63661677dd1306cec4b5a565190e65adf2446e52
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/65887ed3db382aab1d9485c500f4401318d303b9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/67b5181f8f0644597e9bde539e8f083b5cacd0e7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/74c9dcf7afee2a6cb1ab3a2c0de744d1b03c1466
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/792181ca19e6ded261434e588bb7fc2a4816d4ce
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/79f0e2a475487f8fa69e68c1cc947c5851bda741
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/7e5fcdfeb557ce379ed96925c68505eaac0112db
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/7eec7530acf34b3a96fa9189783453999f7b6838
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/80114bf9781bffc9db411413d83541d8deaaf7c1
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/80425fb92bb86627e854892f23823fa804e5fdc3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/821cdd6eeb919a8dd7f35289abbd583828dd4945
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/83e1a31785285338b0ddb3334b0ed098e63dedde
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/8a4c8100dedd0fb5f2a8b468c678f7ad8269deeb
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/8ae8268c24dc866c1edb3826b93a1c75dbf74ff4
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/90f72038cc627f34f074ea72eadbba87a5e3e288
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/92b67faee4a49df2cdbed785e27b4a1cddcfffa3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/9463810467aacdc9923b2b20a2236116b760d75b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/94d7c96aea32ad41ce643d35b951a6d8990b81d6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/98cc7e9fe87df914d89a0aef008930f27b3c26f5
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/99172dfdb4f59aaced29c7681ac6e6ce8356e814
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/9ae3b647d895af97fe872c0b1442df7b5b767160
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/9d2b1d2121b0508a4fa8d1508adb9d05633fdac3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/a335af37917ccf0c8b11bb884a3a74f3f1d2a7c6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/a738609112d3a6772c50a71e2c3504ebc515b709
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/a8cecab5d917da5a4729632a7a18c564d7e1607d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ade919ab2b4a458e806575c941dfe50ae3fd3621
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b1251621a5eb5e7fda9cac9baead1c993a285c36
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b1516b78c3dfe77eeb554985fd7344c0478fbbcb
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b41241740f5f8ad2c1d408f7bb6a313bd863c158
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/b799c8596523a7ebeb8e11ada08818c10f7eabfc
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ba48d0521a111222dc95a3a997c7c92dea5f4443
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/c01457c6889fb1b597d308363a36412c0b7f90e7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/c82ebc0d6688d104af04fd20d6d3da591dc391f7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/c9a03eb758dd84e954e3d70916e2311e8fd21f3c
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/cf892756b33578a54ab20044514e573328d2f1d7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/d3bc3f158a63f1d50b474addd3f7b3d17f23e8e9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/d4906950aa9d60ad09dc0f5413c3d88080c3bc37
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/da31578a8068bad65e1c7a3d06e8f543a2a0bc65
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/dd4a9b5d0740679c249fc884efc499433b29436b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/deea4ecc6f0b2a6d89fd25ff76762299f21602fb
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/e1040c7ffcb39915e0f539018c81f9798924cba6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/e381dc85682cc33ad99f622b89d145b47f7d6392
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ea24498fc7a144fccc6f1665ebf7020df803dd1a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/eaa5d677e797c07bac98c3c7051abad91852e7c6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ed7871269315725535d8bffec7836c45a3fc5c26
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ee8460f4077064c5a2137075b48eba7d3db5c570
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ef09f26e0ee61329f84a9f589629a865ae9ee0a6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/f477da4d7d8ff2066041e1dd5ee4e833b7111a1a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/f8a379b2498a4eb452a85791a49adf065dab59ae
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/fe67bccb06f2174523943cc684518fcf1f7f8046
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_png/ff1e67d17c1c27ef0d97900d0ea276b563a64628
 create mode 100644 tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict

diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index 3ea9dd0638..bb92057bb6 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -19,6 +19,7 @@ cc_library(
 
 load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "tf_ops_fuzz_target_lib")
 load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "tf_oss_fuzz_corpus")
+load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "tf_oss_fuzz_dict")
 
 tf_ops_fuzz_target_lib("identity")
 
@@ -46,6 +47,10 @@ tf_oss_fuzz_corpus("decode_bmp")
 
 tf_ops_fuzz_target_lib("decode_png")
 
+tf_oss_fuzz_corpus("decode_png")
+
+tf_oss_fuzz_dict("decode_png")
+
 tf_ops_fuzz_target_lib("decode_wav")
 
 tf_ops_fuzz_target_lib("example_proto_fast_parsing")
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/010dc3d4b05288fcc40de2721052b3dc699f1cb3 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/010dc3d4b05288fcc40de2721052b3dc699f1cb3
new file mode 100644
index 0000000000000000000000000000000000000000..9dbc560e1e4b50f98060fdad36ae0f65f0c0c92b
GIT binary patch
literal 233
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;IMg|53hG<6jNCpN5rX+877Y4?=UBXKk7#KJU
zJR*x37`TN&n2}-D90>*n2KEw9Usv`=+_KCvj2F+=zF=TrknnVI4B@z*oUp)X#sB~R
z?X5n}*LWmv#>9~R!FJCl^Rw$27#LJbTq8<S5=&C8l8aIkOHvt(3=A!F4GndTEJF+p
zt&GjA42^XSEUXL+oQxLtplHa=PsvQHMAKkoVP#+l(Xd_i=@*b6JYD@<);T3K0RUSW
BJvaaW

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/0555cd5e9d99629819cc985285f80da0f00be1e9 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/0555cd5e9d99629819cc985285f80da0f00be1e9
new file mode 100644
index 0000000000000000000000000000000000000000..fab6d15ebe37176ddc7f3868f576b15d27518d00
GIT binary patch
literal 474
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4rT@h2A3sW#~2tGk^_7~Tp1V`7+4q-6cnC3
zd2;>wbw&mT1`&oUSFRj5aA3`vHM3{Wo-kp;&6_uuE?wHu(NS1fn3k5tz~KMj!Glw$
zP9-NNdwYA^+uO&-$M4&>&%nT7?%cW8u3g))V+S7}pQ)i41H-)k|Nq~)#NNihz`&T~
z?e3zmIoq~@fq{Xuz$3Dlfq`2Xgc%uT&5>YWU|=ut^mS!_#4XD#!+7y*?F$A5hS{Dj
zjv*GO?@qie)TAKLX1?Hw%7hb+2WtLr*IB;Ksrb_DW$%;H)1TirKH%q9Eu1c(=Kat(
zCQdF-<FQkLoAjO~Q(CrpNiPod?qu1qkK5{W$ooCzdy}@U=4O%HS;p+teA(ty0B4Zi
zp$e0}6Vic2zv4dR6h`N7v`pEWcJ|uBXTi54ytc3`o|=Ec@7W&%c{#Blt`CQOj$is;
zvzPe}V`mK4gINu2(F_a>swJ)wB`Jv|saDBFsfi`23`Pcq7P^Lpx<-~EhK5$gW>$vA
vx&{_j1_n+>i+fNs<mRVjrd6V8FtV^RFobB>F8lNgD3Ux~{an^LB{Ts5Cf~1>

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/0a0352aa168803ff65455792d9f6ee555c3e7c3f b/tensorflow/core/kernels/fuzzing/corpus/decode_png/0a0352aa168803ff65455792d9f6ee555c3e7c3f
new file mode 100644
index 0000000000000000000000000000000000000000..7918406ac4bc04196bf07a3e8804b0dbc946eaf3
GIT binary patch
literal 232
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8K^*%(-a*g5{+V&DMD*fUIxIA%CyTes`S{|r5Q>UL&q
z3G$n*s;Kh3+4R-d$Nw25EB^Ma58rj%zl!DCv@0$(HEzHCttDETl;(U(G}+jbZ2$iz
E00B%iYybcN

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/0ed54162df93ef8d00f993ce6b59ba422903d381 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/0ed54162df93ef8d00f993ce6b59ba422903d381
new file mode 100644
index 0000000000000000000000000000000000000000..c294b3180f7d1c315466efacee0167a1685ba68f
GIT binary patch
literal 221
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;IMg|53hG<6jNCpN5rX+877Y4?=UBXKk7#KJU
zJR*x37`TN&n2}-D90>*n2KEw9Usv`=+_KCvj2F+=zF=Tr;PG^E4B@z*oFH+Bfq`KG
z17o6Utvdq)gKCLuL`h0wNvc(HQEFmIDua=Mp@puYp{|i-h@qjCv6+>jv95uIm4Shi
n(c&Hy4Y~O#nQ4`18jLKg3=AO}w#z>K0&=&ftDnm{r-UW|mDe*h

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/1547b448171c700613c3946d730de496c9b9863f b/tensorflow/core/kernels/fuzzing/corpus/decode_png/1547b448171c700613c3946d730de496c9b9863f
new file mode 100644
index 0000000000000000000000000000000000000000..0eb3eff90d7fa0c03fc827b65da2e21050f27ebf
GIT binary patch
literal 260
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I7G?$phQ^Te;|vT8`~f~8t_&cc!@zKcfr0V=
z|Nkp5Y=~xHU|>x0c6VXv&$t=Lz`(#+;1OBOz`!jG!i)^F=14FwFtC?+`ns||;+AEW
zVZ3;@_5}k2gQTa6V~EE2<OB)U5Qd&3n!yqo<w@*kb}VI(>|#*;m~HW%fq_A_#5JNM
zC9x#cD!C{%u_Tqj$iUD-*U(Vc$TGyx(8}1%%FtNXz{1MFz{zNF4~mA|{FKbJN;C~d
Z7FGs^5DnX9pMC-P#?#f$Wt~$(697lsLR<g<

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/17859046cbe4ac598a645173d679ce2a52c6afba b/tensorflow/core/kernels/fuzzing/corpus/decode_png/17859046cbe4ac598a645173d679ce2a52c6afba
new file mode 100644
index 0000000000000000000000000000000000000000..deb7b0a784aa876e880755bca4c2f96c1167fa27
GIT binary patch
literal 525
zcmZ?wbhEHbRA5kG_^Qmn$H%9jpkQENU~g~l?d_eMoSc@HR#;fr(a|ws!i3qgXD?m4
zblsXYJ9g~Yw{PEp0|!o>I(6mBm220oUB7<)=FOWA9z1yR<jKpIFJHfY{r2tK_wV0-
z{P^+f*RMZ*{P_9v=dWMCe*gac_wV0-|Ni~||DS;dK=CIF*i^;;+@86K>8ZYn>6yvd
z?CFWQxrw$03_2i5kXISl{vDWD;GrYce_~0|$sDWs7gm&TiAK6!S){@8sl-YyfWw=&
z)?zjf+vJ-Cb_+9CY~C57A;h_~=w`{IfMO{zVF5l~UOoX~F{yfgC3$HvL4F=CPEIZ!
zenBy5`4k^T8F68LZVq-fHg*o~;L;?S1|J1!Q2`zfRu*Pv7FG@(0a0lMUnMz75k4+<
z7A8hUCKh%sJ`qW|MV?Bs5<<M3Y|M-lgz<Ev+*e;TuLwJ~@F)fyIT5(Fv*gW#i-(V&
oKRr49(wfZA&(g$1_2Le6Omu(Bby3G-vR3S=Eh~eS92pp_0YE#>1poj5

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/1df76c07817fbc3653a26f34d97658e9973627c2 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/1df76c07817fbc3653a26f34d97658e9973627c2
new file mode 100644
index 0000000000000000000000000000000000000000..2b9721d742ad143de0881894c52407af1865b79b
GIT binary patch
literal 666
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0^lFtRc-AOj|57B)_f|3?^f1sIqZ8CjWm;qr`3
z%q$G7Y=T1U9Gt=;5|WCBj$z@`mtDCk`u_-nI3oxko5aA#%=G^VgOVU412Y37GYd0J
zl97Q)keNluP*KUT(J64^|62?^%wRD=27886tChrDissGpTFFzm#i6B=b5dwe%BqNO
z8yV&_o+#gVg29vF=!xE2M|3)NXg*(AENUX;^)T|(qG?v4-i9YCSXm}NU^Lshx@4_y
zM8}qnNa3t1?}FS<&ra=KKL7KYj*@8)Za+Ocwc6kQd8EKg-Gkdt&(``jum0R76)PR#
zgZ;T*%iAoPwx?Y$ndG<Sv!k8D+Czoy9ut^Fna_GHG<kXUafi0qCePZk6VKM}%ss@{
z@S|9iXIt6&trx<UnK50xJ#V%}((G2hKgMo3`T0eQmQ`7m&XQPqWO`=Sb)#c#8e7hV
zScGSCPFR%Lc_8Vj;1}5+bNXie@~91${#GUVDSQ6gTc`O}_x`r!`DuFo+ndh#qjPM^
zFC9$lh?m~~h^w#e-NbsesO-z0u2(AkGIjP$IKAWK62mu=^G>IqJ{>3ZJ-V*7zF{+8
a-lDmd+vQi?y07@7|NZef*AG1Ze-i*fJ=3-T

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/1f0717f8856d7782e3ab7992d3a72d783a018443 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/1f0717f8856d7782e3ab7992d3a72d783a018443
new file mode 100644
index 0000000000000000000000000000000000000000..e0c330f7f4ee396723fedbb530197cd890045730
GIT binary patch
literal 122
zcmZ?wbhEHbRA5kG_{0DKj7$s+91IK$d<+Z>3JeSy3=9m4KUu(XivPJia}(23eG}6&
zle5{=6LWJDZ4DT7K$0LM7??FUcHQ}BaLRM_UW;sxZT}NA`cy(R+*Yp3dA+Yd=X{Cg
Ql^>>0YJ|V^voKf#0K+&TPXGV_

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/23b911e4ce936def88bc9a46b8b433c0e83fba2a b/tensorflow/core/kernels/fuzzing/corpus/decode_png/23b911e4ce936def88bc9a46b8b433c0e83fba2a
new file mode 100644
index 0000000000000000000000000000000000000000..41fc2fe9516ecaf369c9093c5d815a709963c2b1
GIT binary patch
literal 375
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo*B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)wL2e>vEG3_`mf5wh2KoC
g{uB}ZaXV}J*68T#owxi~XU6J;UwFmK*YN))03-Q9g8%>k

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/25592201c3edff0578dbdac6b0e4f2be109ce151 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/25592201c3edff0578dbdac6b0e4f2be109ce151
new file mode 100644
index 0000000000000000000000000000000000000000..8b5755c4bc513368a2605e05c8b3117feadab086
GIT binary patch
literal 329
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4h9AWhA=@@4F(1VrX+877l!}s{b%+wFfecy
zctjR6FmMZlFeAgPIT8#E4D2PIzOL+#xMi7T7%!f!eZj!M@WRu@F~s8Z+ljt;hYUno
z*n1vnZCh>Pa)3#Id)1vrr=48skF^BDUkF`v{UN$wO53qGi%=CY2Z19wyV531`111Q
zUdacV3@h&R9cFm=!q8x$75l<l=FEEwXPx2^6HgMD*^nx9+@9;r+0S#z9$s7b`*OwB
z8(JHdvCaIE+VJkt(Gul@^^R}d5+^_U*Z;oMBs%x)m(Op)^&KXpd!0<#@4qA{Qtp9V
z#S$it&Z83LG1njVM5^fQmwfvy{xYUp7di!{CO_5^6i0Qd?9(rxF!FTub6Mw<&;$Ux
C%z{$@

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/266fd8495e0b8eb64387c1a62264185e061fee73 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/266fd8495e0b8eb64387c1a62264185e061fee73
new file mode 100644
index 0000000000000000000000000000000000000000..c4d2d8d7f1adc091d4340f3cb45a304df4617a17
GIT binary patch
literal 360
zcmex=<NpH&hX1!2oEeyzKmbHCGcz-@u(5yu2OAqJ8z%=B7bgcNCl?Pt4;MEdHzy~r
zATJ-kfPkO?7mtvzkbp41fPldN0}R|83<?YijEsT||Bo<;GB7eSpaRbSM;K%U7#LZY
zk)@aznOWHwScKR){@-HY0Lj=hOpQ2ZFlAe}>&O2LJ$veQW^4&synHWHwRc#YlT*Co
z^0_^w$9AKd%|KnlWn&ZVCbneC1Sqq<JmjPlSHTdsf{m4pi;I(!lY@hpplE4Qn)5Bu
LWMfaV{r{T)r|U~s

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/27f178cf415b4ff8671131ddf1d042dafac2fb3e b/tensorflow/core/kernels/fuzzing/corpus/decode_png/27f178cf415b4ff8671131ddf1d042dafac2fb3e
new file mode 100644
index 0000000000000000000000000000000000000000..1608e5b08373fdbfb62f83c5a708e76e3dcd3db0
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3<TEn3<SC6caNu3mXdvaImqlvT<^7
zadC2Ra&qzT^KfzVadUF=3i9&t3kV1baPbHU3keAG3kV2+3}IwuW@ce#;b39m5a8zI
z79bh?KfoZ!!JxpPz|1Jfz$D1XEXer(2!jLz12ZEd12SOY<Y4`Ogh5?^fr*KciJ66w
z6)MZf#LUVf$i~1f#38I`D54aYSlIOc2!j|SBbo^eOdzA>1R0o^85x+Eq1qW37#NuZ
z6&;xkm9-cCzs0}<k`!bXV6bPH>XPyL+pDGGoEH0P|E*yt{kZt=SA`p5b)6OJB5&+P
zpZ)p1^P0$>;}Z|_?wRrN!I~cxJA_@Ii|uy#cW3_uW@YQ?T+2@x^e$9no*w<8X#KRZ
z@+q6y8_H+u<ybsT`PLn%%k22G@X)>|D<&=ZB&fb^{R7+0PacL$<%wXN@{8dG*F@1&
z@pbEteEhvmwOVFR-=y6_`zMEGxZBBPxd_fVA~BI?P2Wcg6{9z@R#qA;Tc^bJRtuld
z+gbXiu*iDXn_d5mBNwXtoVi}mwNNl|$MhLHvQ9FFJ3qZE*f8xc!_`pVwQpNQO{UHa
zlf0?>kk3p<S7+0<?Mq!*HeRrIULdVJ<wKnFhig;Br>?Wxd&g%@Cf9$4>24SQ+5OtD
zA|mVN`RHqHc+|A@{?j&Y^?h{FJ32f1Hs{-QZ$7Ot-V}W<XU@u5k*T@aId43Z7ucN-
t$XM^d9{pEk;=*sHSAU8K|G1qseQR{|_0C)Vt21MD!Y{ny<!ktV69Aa@*F*pS

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/2a0bdc4d9cc5ea5bb21dd256d6ac96075376a94f b/tensorflow/core/kernels/fuzzing/corpus/decode_png/2a0bdc4d9cc5ea5bb21dd256d6ac96075376a94f
new file mode 100644
index 0000000000000000000000000000000000000000..12e4140981d115312f32b710bb2882d5b1ede161
GIT binary patch
literal 246
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+I;nTd&+iJ6(1g^dLSIM~=&**H1S
zm4H;BDdFPc<lyAw;^F7v;^yP#<m46P<>MC+5ES6z5fT;>5at&U5cq$9ft!OtfkA<h
zQIO&P5e88PMn(oy!1@0OgRB4pBMURK6cZyeD+2?I5Ie{JTMQf^8GDAQ5yuRtZ0mOY
r_@AL?Pu<RpEkS;>RTWj9H=Dlt`uM+O#oxa5;S9(9t608GyW#=>4KFNM

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/2e5d25add6adc68e0457b358c7a34abf3d41c938 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/2e5d25add6adc68e0457b358c7a34abf3d41c938
new file mode 100644
index 0000000000000000000000000000000000000000..ecf597f7365bf06492af4d53bca4166f5ecf744a
GIT binary patch
literal 165
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2L~r77Y{!V7dIa_Cnv8UFCV{vfS>>ukC3pCfH1#+fWZF)4BQ+H3JeO2jDigRk1&Wb
gFfuZr0M7qM7(@i%f-F$U|F;-8;CdJs?El{c0OisZZU6uP

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/2e6c5b6a766dd5e9bd41eacfd0a36572bd2f7544 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/2e6c5b6a766dd5e9bd41eacfd0a36572bd2f7544
new file mode 100644
index 0000000000000000000000000000000000000000..e5a18917e3744daf0f8d4eb5cbbb702daff21188
GIT binary patch
literal 320
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8K^*%(-a*g5{+V&DMD*fUIxIA$<qTes`S{|r5Q>UL&q
z30k~-FH^O5Se%nnyyNn@J*CHX%f=?!O>D`O2~cL`VPIg0t6+#*fz!3f4#DXRmzo;4
l-~QGTuX_ukqg7;;IWAeWDP&%5X;PZ=Ezx9SPqO|0n*eW1JZb;{

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/2e9c935cf82f6ca640e9a9abc3c30a578ad46176 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/2e9c935cf82f6ca640e9a9abc3c30a578ad46176
new file mode 100644
index 0000000000000000000000000000000000000000..50be7f686b791d7aa08652159592311c728ec79f
GIT binary patch
literal 666
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0^lFtRc-AOj|57B)_f|3?^f1sIqZ8CjWm;qr`3
z%q$G7Y=T1U9Gt=;5|WBWj$z@`mtDCk`u_-nI3oxko5aA#%=G^VgOVU412Y37GYd0J
zl97Q)keNluP*KUT(J64^|62?^%wRD=27886tChrDissGpTFFzm#i6B=b5dwe%BqNO
z8yV&_o+#gVg29vF=!xE2M|3)NXg*(AENUX;^)T|(qG?v4`wUN1u(C{kz-YF0b;(-a
zh>k5Ck-}M3-UYdzo}JpeeE#P(9VOEq+<tm?YPG-p^GJc0x(BzPo~`w5Uj4aEDpoqe
z2m5osmbY0nZBM&iGRbetXGc4QwTBAZJti=VGN1KYX!7#x;|^`JO`f%7C!Vd{nR|$@
z;YYD3&$hDlTQ7tyGh@1Xd){n|q}i>0e~jI7^7D%pEvvFBoh7mK$n?yt>qf`gG`5@z
zu?WxPoUkaf^FY#5!7s8u=Jd__<xv|h{jEy!Q}+C~w@&k|?)`1c^V9VFw>O>fN9Wj-
zUpko95ih;}5m#T`yNUH`QQ4P0U9VL7W$NsiaC*ndC5CS#=bcVJeL7C+dvslEeZywH
ayhU>@x67}(bzkvE|NG-}t{-^*|0V#zAJi%U

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/2fcf1ed4477f7eaee028f5b3f9edeb5f1a737826 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/2fcf1ed4477f7eaee028f5b3f9edeb5f1a737826
new file mode 100644
index 0000000000000000000000000000000000000000..00eba4c39a92d546e7b4f6f77b18e9d3ddaec399
GIT binary patch
literal 507
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0cwE%
z2N(o77!(*3m>C5bm;@P_1sVSzVUS{AU}R-vKn6_AENq+{|Bo=}3NSD+GO{xB!sQv6
zm{}ND*#w2yIXHzyBqS9L9mB$>FS~M8^#2hCaYhh8Hi?0end$!#1|>m824)6EW)^0c
zBqIZpATx`QqM?#wqf_9-|F;-;n89L#4ECWtDXSvBZDg3!c%ppc2?kGwqbGWA9ntC7
zq4|7ev8aiV*TcwDi>6tHdK;doU}c&7fYEI0>XNm-5gl7PB89W6ybE$aJv+5``TY5B
zZ=L2_-TT{?=cnoUZ*Mx|kIu0vzjQFIBVKy{Bd)%>cN6Q?qOvc0x?ZXD%hcI3;q;D^
uOAHw_=bcVJeL7C+dvslEeZywHyhU>@x67}(bzkvE|NG-}t{-^*|0V#7czSC9

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/3480713774f590908ca5dba16d121cdfb8fba62b b/tensorflow/core/kernels/fuzzing/corpus/decode_png/3480713774f590908ca5dba16d121cdfb8fba62b
new file mode 100644
index 0000000000000000000000000000000000000000..af3afc499d820cdd966ec9a67d3ec6fb39b3e240
GIT binary patch
literal 665
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0W5$W
zix@)a{{aR;4h97V1!hJ;1|~s9W<kdPM;Ig+7?>Fu8IS=BCkN~QBMj;S3`|T+jLa;I
ztWa4-CT3O^K{f_<Ar4_hLlLFG#KNYO%Dlwf%;flz{QMlfvdp5A(!`wj+|=CsqRROE
zvecrCR3>Jqb_NCpMkYZ;M`lCi#)bcHG4L=mGB61;3o_U<Om)e4{q5CKaZZbUwg1*I
zlzv?N_p8DUvAWI*b&)srqR;+(-+4`B&+&<edH2lt_+ZVCiXFnP&&8}={@vL>fmzvl
zI@j`32E7XvnWsm;C|W<QZ1a@h%=Fy+%oK~KDc`ySb(tN179QI7WW}T<p9Iynt$$#<
z`N_kOsXP&kQ+_eL;F>6!D!y*rk&nOEsaDJE>6^4$X#eD}40k)ZEEmB!M<gcltm*q`
zp<?t#*2+qQW$To<)@tDsdOJ(s6c$<Udb8`FVdO%UpEK7BlBwMos@=6vFmXpxy8G0V
bjN#5t?+P|dqtH1UxB5Q1=*`R5@c$+N?Sa(W

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/39289afcec60d98802b333e0fbb1da4d7aed4ce5 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/39289afcec60d98802b333e0fbb1da4d7aed4ce5
new file mode 100644
index 0000000000000000000000000000000000000000..02c187a49225947f8f20ee87b970783f93eaed76
GIT binary patch
literal 247
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2N6oR
zxHvgDIk|ZFdAPXwxH&m_1$p`S1q1{IxOjwwg#?881q1~CA7J3-U{GLCU}O|z_<w{!
zl!1|v0TpomKf)j@z`)4Dj4Z{($jr*dz#_!X@&6VB2S~=AVQR!N!ztUkT|fS3=-E@Z
mGh<7T-)vPymFLZ-uf9J1&mdXxw{Ly;uH*hyEZ?SGaRC52DlHfQ

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/3adc488e21d4aca7bed9422f0241a42d0f93e7d9 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/3adc488e21d4aca7bed9422f0241a42d0f93e7d9
new file mode 100644
index 0000000000000000000000000000000000000000..1cf24048f8ba10e68082ea717656c2c889ebf7f5
GIT binary patch
literal 366
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4+jqyHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0X*Qx
z`}gnhNF${FA7BvVU{GLCU}h9#U=n0x7G(T?gh7IVftit!0U5Axa<KkC!k{j|z{JGF
z$jrjX3YBGKVrFF#WMg0#;t*Ce6j3_DAjZhZh)`*$+_>=nEe0NDMg}H9W<dsfhN&(Y
zufM%oD$Z%KulC;>hEgzrZ15X<(Pw|Y@4P0m=lI0KynAMRe6Z$+og5!Gr#mOFV3v#E
NoFftwdDist0swI5K6d~B

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/3cbf274da522483dc991fad9df43a22ac4fb3173 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/3cbf274da522483dc991fad9df43a22ac4fb3173
new file mode 100644
index 0000000000000000000000000000000000000000..7f9c0c93ec9bc8f135e7f78010b275079e052b48
GIT binary patch
literal 474
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0X*Qx
z`}glb(vBbkAO3%UL6Cz%fkA<pQILU2kdaxC@&6G92?hpcMn(o?z{1JF`u_-nx&Q+c
z6B8pd3nMF3mXV2>l|_(^fnA70SkXX4DKN3H>HiT1F-Ar-6Bw94M#~8@FflVSFfl{5
zGcYhPG6^a=G8-y4F8qIsfd?ch$SlZU&oI>`<Mp>!OT{@Y_SODd!%+Hh@!zisH^k~X
zE7V2a*o!{<^L^(vkv+#J9_HOM<Ku%hKPq+zyFPz)WH+O-L;g=?>*-v}13FI`^e$9n
zo*w<8X#KRZ%~Q(R8_H+u<ybsT`PLn%%k22G@X)>|D<&=ZB&fa()gl%a7Ettfx$`RA
J;F1#K0sve<X}ACY

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/3d840cdff7f5ad16fe8bcb985ed4946c03459432 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/3d840cdff7f5ad16fe8bcb985ed4946c03459432
new file mode 100644
index 0000000000000000000000000000000000000000..f48cb4cd19a65487d555b2fdce297d638736edb7
GIT binary patch
literal 666
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnm_agp0$v~rivk8lPd6q;H)jTBuqGuz
zhEaes1Qsyjlp<Y1P=Jd^NLWZfm|s9Z;Qs*zZVm<o1_ee&L5BZF7(^Ku(e2{=e}q9+
zfPs;P8Ci;nk(rf&fklX&<NqxN4v>sJ!_<gl22-|myMFx7(6gs*XU3KwzuBsaD$kou
ZUwwW2pFy(XZ{PZGhU5NKq&u1n9RODKGV1^U

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/3f1e6753c1fca958e859189857449746592158ea b/tensorflow/core/kernels/fuzzing/corpus/decode_png/3f1e6753c1fca958e859189857449746592158ea
new file mode 100644
index 0000000000000000000000000000000000000000..df07889441d48fd938b74aac94e01b39dfa8f63e
GIT binary patch
literal 94
zcmZ?wbhEHbRA5kG_`tyM|Nnmm1_s5SEDRu^_@CP|H!(fcH!(dkIh#E_F*i5S)__3=
tBnQ&Kz$Dkwzw-23{>5{)-0I$ZZ_jW3wnrY*o^>vJb!yu??%Avi)&O2EA{+n!

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/3fa4075993cb0f9bfa8eea785174a2038a69aa1b b/tensorflow/core/kernels/fuzzing/corpus/decode_png/3fa4075993cb0f9bfa8eea785174a2038a69aa1b
new file mode 100644
index 0000000000000000000000000000000000000000..5f9cec9ab5cb27973b8d8e71b47cf61601c0ee3d
GIT binary patch
literal 343
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8K^*%(-a*g5{+V&DMD*fUIxIA%CyTes`S{|r5Q>UL&q
z30k~-FH^O5Se%nnyyNn@J*CHX%f=?!O>D`O2~cKzdB{mAu7V+M1y<LhJ4Z$N>GxZy
zH_fB78*g1xek$)CxYdZkZ?>wU%JXK^S6?6hXOOJ;+qXV=*Kz+UmT%LpxYX3R{r0z(
ic->nN9jzj(%yG%0O(FAgOOw)^Z;2)wdy?({-vj_CR#v3|

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/4023a373e977be58413e55350380310c5dd1fd6a b/tensorflow/core/kernels/fuzzing/corpus/decode_png/4023a373e977be58413e55350380310c5dd1fd6a
new file mode 100644
index 0000000000000000000000000000000000000000..385b8b0c35936d3b9c99facefccb33835569470b
GIT binary patch
literal 241
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4h9AWhA=@@4F(1VrX+877l!}s{b%+wFfecy
zctjR6FmMZlFeAgPIT8#E4D2PIzOL+#xMi7T7%!f!eZj!M@WRu@F~s8Z+ljt;hYUno
z*n1vnZCh>Pa)3#Id)1wVD<}8O&<W~1Hc`infq_A_#5JNMC9x#cD!C{%u_Tqj$iUD-
z*U(Vc$TGyx(8}1%%FtNXz{1MFz{zNF4^%@4SVL}pN@iLmng$~aD+5D_hV8OXzkvMW
M>FVdQ&MBb@0EH7l6aWAK

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/40caba69dce1cfc48e0e43184d2bfbc6daa4399a b/tensorflow/core/kernels/fuzzing/corpus/decode_png/40caba69dce1cfc48e0e43184d2bfbc6daa4399a
new file mode 100644
index 0000000000000000000000000000000000000000..22f1649adc45eae591f46c4b1a1e0db6bf0cc82e
GIT binary patch
literal 470
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo*B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27AV-E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%o>I=<
zuw$lPj>XfIZ{2~q%#J?`5AA!BGiT+j$kg2IoHw4y3+&DZWUO~!kN&GNap5=9t3O4A
df85TRzBM}fdgm?w)tRw6;TK-<@-_Uw2>^SUawPx&

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/41841e9561d8135945c1c1e55ab9e9a1e933653b b/tensorflow/core/kernels/fuzzing/corpus/decode_png/41841e9561d8135945c1c1e55ab9e9a1e933653b
new file mode 100644
index 0000000000000000000000000000000000000000..16c0c33b93dadec4fd326f2eff73a81d6c82f508
GIT binary patch
literal 247
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2N6oR
zIJtQEdAPXwxH&m_1$p`S1q1{IxOjwwg#?881q1~CA7J3-U{GLCU}O|z_<w{!l!1|v
z0TXckKf)j@z`)4Dj4aK>%E-pdz#_!X@&6VB2S~=AVQR!NgDKm(T|fS3=-E@ZGh<7T
i-)vPymFLZ-uf9J1&mdXxw{Ly;uH*hyEZ?SGaRC52$Sm^!

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/41d40f2d66fa43e34537385594ee9911e65deadf b/tensorflow/core/kernels/fuzzing/corpus/decode_png/41d40f2d66fa43e34537385594ee9911e65deadf
new file mode 100644
index 0000000000000000000000000000000000000000..6e44f2adc7fd1343044613a087c1c3efdaef5081
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Z@3
zOsp(|Yz*u|9Kwo5B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%ZYZC^
zUOrPV$Kq*<?$5w)%-xQKhxR>LF=@#sLG^9xAJ}ew@-Sp7PXyzXUkop}CW@wtuUmKI
z<L`B<)iQhfChZp5KRGPJ-A*pcMR3j$iHSUG`aW8y7`>6TveIDLIwh{PTKI(C&eAu9
zMb^9C?D}UMxlrZj%=Lo!wF?o_?pi3AxFadued<ZZaObCY1skUQWw;vZyY_91sL9ls
zVUjm>AM%;$=<00RwtcB9%f<`#&I_cKr+kQW{%~!I_|$cFd++$H$>jRaFx~CqKf7Q1
fRYYXnJRg0n4Ud|(-hbN0t-g;gdh_x%{J#kRtRT^B

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/421bd39810b50309a71adb2dadc3b19f01a52312 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/421bd39810b50309a71adb2dadc3b19f01a52312
new file mode 100644
index 0000000000000000000000000000000000000000..131004b8943e997ec7ba3212f3ba0d555c4ba4d2
GIT binary patch
literal 221
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;IMg|53hG<6jNCpN5rX+877Y4?=UBXKk7#KJU
zJR*x37`TN&n2}-D90>*n2KEw9Usv`=+_KCvj2F+=zF=Tr;PG^E4B@z*oFH+Bfq`KG
z17o6Utvdq)gKCLuL`h0wNvc(HQEFmIDua=Mp@puYp{|i-h@qjCv6+>jv95uIm4Shi
l(c&H?4InpyT!p5=$im9N5Taqb?9(qG6`rnsF6*2Ung9jfFc<&;

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/446c305b2c0665736f94fb2b62dbdef445eff0cf b/tensorflow/core/kernels/fuzzing/corpus/decode_png/446c305b2c0665736f94fb2b62dbdef445eff0cf
new file mode 100644
index 0000000000000000000000000000000000000000..1f2f90b3bc41f8daa93d2df651bf55b2a3ddc78b
GIT binary patch
literal 174
zcmZ?wbhEHbRA5kG_{_@y0*nj{%nSk`mI#9a149i11A}v@e{fM@FhduEK$q|T{|x{C
z{|Cu4Fev_H0W%c;b9?3{rl<NQre`K+v!^HK<|f)2FzA3JK}IvMxCor^T)o%g_1WG3
z6Eyl%rg|_sOv!n@uR!PgmkycI8*ggf|F`2V@X(R!Ke42!v&XP%=ZX@qR8cO*g9{jw
PrNkLp56<pkV6X-NCk{2d

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/449cee952bb645f6f4241a6665d3c6028c073c7a b/tensorflow/core/kernels/fuzzing/corpus/decode_png/449cee952bb645f6f4241a6665d3c6028c073c7a
new file mode 100644
index 0000000000000000000000000000000000000000..c671f7e4c0d7c8816a5fa4915b1fe0cbe9be98e1
GIT binary patch
literal 374
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8Mgu`#d+v2*;t#lQiQv1gbXam--Kwr<yt|Al(?)a}gJ
z60~^vUZ!gAusA2Dc*o^)drFV(mW@rco7j>m6Od9;S&*s}SHTdsf*P(>QGWXUR_aZY
z=<LQ@*OZ^iy9aJHV(^=-s;Kh3+4R-d$Nw25EB^Ma58id$zl!Bs&lQ)N8n@s6))KFK
f3!<Y{WR*ECS+pr+UT$epn)5BuWMfaV{r{T)*_K-G

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/45520b07609978c5aa3516d803527438b93fbadb b/tensorflow/core/kernels/fuzzing/corpus/decode_png/45520b07609978c5aa3516d803527438b93fbadb
new file mode 100644
index 0000000000000000000000000000000000000000..c6f2f7052d2c169c84429800d57e79d8876e2a03
GIT binary patch
literal 158
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2L~r77Y{!V7dIa_Cnv8UFCV{vfS>>ukC3pCfH1#+fWZF)4BQ+H3JeO2jDigRk1&Wb
cFfuZr0M7qM7(@i%f-F$U|F;-8AbM^B09&>c8vp<R

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/4da74a34bcede234b0415f77fbd87d70bf9a777e b/tensorflow/core/kernels/fuzzing/corpus/decode_png/4da74a34bcede234b0415f77fbd87d70bf9a777e
new file mode 100644
index 0000000000000000000000000000000000000000..605ad2d2014ba7bcc2a0b94a9a0b9bacf055d7c5
GIT binary patch
literal 94
zcmZ?wbhEHbRA5kG_`m=Kia%MvEXDuap1FzXslJKnnaSDg>4~|yiM9p|Iv`1q1_ma%
nmj0Ee-|{b>v*lLz=6id7^S3?nnD(r5*{f6A-f_=nWv~VSZx0_I

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/51db5d31d2c5300d34831d9f23bcdd0aff9a998b b/tensorflow/core/kernels/fuzzing/corpus/decode_png/51db5d31d2c5300d34831d9f23bcdd0aff9a998b
new file mode 100644
index 0000000000000000000000000000000000000000..bcacbe623f8ca1ac88de8d3dfdce160de04f2f29
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo*B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%p0f6B
zi>S%enPHMQb-M#~nH_%?9@_V0#iS*l1l6~#e_*@$$-|JTJQ0jjelfh@nkbqozHZ%-
zkH6QcR?F<^o3vYK|KzX?cRRT(7r{A4Bqs8#>HBD*V)RDV%1VP}>y)_GYT*-lJ4@db
z7Fq9lv+JL6<U*C7GuI2c778ZrNJ@8~dXh2R`RQH3hG~BpuKxcI@jTci5Bbb=baggu
z+rHG5W#a{V=LOQrQ$EBwf4DY9eCj&8y?1=pWODsynC^D*pWUzhDk8FOo{zrPhDS|X
z?>}whR^LY#y`!_EZ*#s~_vX_Y<4w`$a^|d@6`7iwo%6;sd4b*efQ<DH?9qQ!CNBJD
hdiAG>@Q>SB)3-)PU+=u-zdAEkC;Y-IUcQF^Hvt?t)Up5o

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/5cde2a9167798cb77f10abbfb2640a5c357f99fc b/tensorflow/core/kernels/fuzzing/corpus/decode_png/5cde2a9167798cb77f10abbfb2640a5c357f99fc
new file mode 100644
index 0000000000000000000000000000000000000000..2619e1d87638b8f37900a750dba325d075739ad1
GIT binary patch
literal 507
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0^lFtRc-AOj|57B)_f|3?^f1sIqZ8CjWm;qr`3
z%q$G7Y=T1U9Gt=;5|WCBj$z@`mtDCk`u_-nI3oxko5aA#%=G^VgOVU412Y37GYd0J
zl97Q)keNkD(NM{;(J64^|62?^%wRD=27886tChrDissGpTFFzm#i6B=b5dwe%BqNO
z8yV&_o+#gVg29vF=!xE2M|3)NXg*(AENUX;^)T|(qG?v4-i9YCSXm}NU^Lshx@4_y
zM8}qnNa3t1?}FS<&ra=KK7an(Tc`O}_x`r!`DuFo+ndh#qjPM^FC9$lh?m~~h^w#e
z-NbsesO-z0u2(AkGIjP$IKAWK5<>>fd8gA)pN^CI9$nX3->{i4Z_!-K?eeQ`-B<k4
O|Ni)#>j$3yzX<@4UVzpB

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/5e352fc10ac476cfbe1d755f092e069820223249 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/5e352fc10ac476cfbe1d755f092e069820223249
new file mode 100644
index 0000000000000000000000000000000000000000..cb55f03ee184e66592d578050dc44a7bf70fe4ac
GIT binary patch
literal 169
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<u&^+of(BGB8t?xB
z25t@p1qKC1MnQ)EM;JsI7#Y!&aB}hr^78Qu2nY&ref$6aKS-KIh@IpAEd~ydVta<E
P5yuRsZ0mOY_@4m)=ye#v

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/63661677dd1306cec4b5a565190e65adf2446e52 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/63661677dd1306cec4b5a565190e65adf2446e52
new file mode 100644
index 0000000000000000000000000000000000000000..be952039a4ec42a60e188115aa9c27888c692b06
GIT binary patch
literal 374
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8K^*%(-a*g5{+V&DMD*fUIxIA%CyTes`S{|q}b>U#EU
z30k~-FH^O5Se%nnyyNn@J*CHX%f=?!O>D`O2~cKzdB{mAu7V+M#XkFQ`TaNEidKb`
z@Go$9nd^Q0giD6W=}mdtuBWcvq<G4%Z*c}w+Fmm+747p0#om(M8Qa&(|29AVdTO=n
zH7oWD3ME^URFt27zm<B^JUYAa)-~m)^6r6KjTro9t17BIZ#I4P_3?iO$%?;y>w|Y4
w_pf64HtmW_O^w@ce`|@?y#>+HDzeHPmn_;8GB39@Db4woXtJ><+5Z1c0K?*R^#A|>

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/65887ed3db382aab1d9485c500f4401318d303b9 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/65887ed3db382aab1d9485c500f4401318d303b9
new file mode 100644
index 0000000000000000000000000000000000000000..776adbe8d4bfd9e9b2532525a8f3187d90453d93
GIT binary patch
literal 224
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;IMg|53hG<6jNCpN5rX+877Y4?=UBXKk7#KJU
zJR*x37`TN&n2}-D90>*n2KEw9Usv`=+_KCvj2F+=zF=Tr;P-TK4B@z*{DYr`r+GR9
z!>yO~jnhK{rZ6xtsFt`!l%yn<q*^5xr6!i7G8h>cTId=Y>Ka*w7#dm`n^_qe>l#>C
r85lSjE$%_lkei>9nO2FW!N|hOz!0KgyX@01Ag6n}`njxgN@xNAZ`3xt

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/67b5181f8f0644597e9bde539e8f083b5cacd0e7 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/67b5181f8f0644597e9bde539e8f083b5cacd0e7
new file mode 100644
index 0000000000000000000000000000000000000000..5bee1d494a574bf8a675933d3045150b32395478
GIT binary patch
literal 165
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2L~r77Y{!V7dIa_Cnv8UFCV{vfS>>ukC3pCfH1#+fWZF)4BQ+H3JeO2jDigRk1&Wb
fFfuZr0M7qM7(@i%f-F$U|F;-8V0u7S|Gx<U<xUkm

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/74c9dcf7afee2a6cb1ab3a2c0de744d1b03c1466 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/74c9dcf7afee2a6cb1ab3a2c0de744d1b03c1466
new file mode 100644
index 0000000000000000000000000000000000000000..ea3e0d2bd444e8fd484bd57facdf53f318762b86
GIT binary patch
literal 307
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fo-fRGBTh7&i_Xk#03}`(If;N6+!CJg&7!_K<Wh<
s8JJOxVH9-ye~W>KnUR4>kXewyo?)s3<Fp0_hGJ3Eh6!M$jQ?)}0H)d@!T<mO

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/792181ca19e6ded261434e588bb7fc2a4816d4ce b/tensorflow/core/kernels/fuzzing/corpus/decode_png/792181ca19e6ded261434e588bb7fc2a4816d4ce
new file mode 100644
index 0000000000000000000000000000000000000000..521deb8d44ea8afc4d5ef285d41f1a42909e8ad5
GIT binary patch
literal 544
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<|x3$GwAAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo5B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhTG@Cm)0rEdz0tarWH_0KqRp~}yh>ji(WQ>~WS(>H0i(EiC`8SZv+SuTQe
zjz~=8S=0B?LdEEftd*4p%hoAzt<_kyyA}#2?np{^pL&up-1+HU!G>vn8Lo!<u6^4g
zYBF_ZnB+~}hkRx_x;mS-ZC~ojvhjkw^8#t*DIemTKU|w4K6Rbl-a9^PGP(XUOn1BZ
i&+gZL6%kptd^fk^@Th6){iki*>ig)TH!okq|C;~^(U3a;

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/79f0e2a475487f8fa69e68c1cc947c5851bda741 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/79f0e2a475487f8fa69e68c1cc947c5851bda741
new file mode 100644
index 0000000000000000000000000000000000000000..885332337762ff92dd6a1ad00fe0a1694995a5c0
GIT binary patch
literal 355
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fo-cQGBPrt07lOLM;N387(o)uNHUB}f`)|?L3+Sa
zNHPozjLiR!FvtipGBPkRBb&#_B*?627?}9~76T76BLkBlvmk>#!>I*e&HUM>`~I+q
z2c?Jk`g&I0eDpW-Y@1EY%*4K6P2EXnU;R6Kvb1*gq`#BCOnR3qxq^X#`TtD-2xd8f

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/7e5fcdfeb557ce379ed96925c68505eaac0112db b/tensorflow/core/kernels/fuzzing/corpus/decode_png/7e5fcdfeb557ce379ed96925c68505eaac0112db
new file mode 100644
index 0000000000000000000000000000000000000000..cc011aedc9b5234cb26140a73dc93a85c0e1c6c3
GIT binary patch
literal 315
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8K^*%(-a*g5{+V&DMD*fUIxIA%CyTes`S{|q}b>U#EU
z30k~-FH^O5Se%nnyyNn@J*CHX%f=?!O>D`O2~cKzdB{mAu7V+M#XkFQ`TaNEZcS2A
ze)|1Z>P_?L?8aNyl%LAG2W~ZD@SClwsPeqo^wrnL{~07J{`Rd8-gVr+isjq1D=sxP
mZomDlC0_RyL`SR0Dsx=2Xj90%+|r~p=Ubx5#-3#R|2F|$|5?2N

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/7eec7530acf34b3a96fa9189783453999f7b6838 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/7eec7530acf34b3a96fa9189783453999f7b6838
new file mode 100644
index 0000000000000000000000000000000000000000..7e3b6f564f0dc029247355c9a920245354f7cd1f
GIT binary patch
literal 671
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo*B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL`-27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%o>I=<
zP(D*H$Kq+qx9&h)X2+j}hxR>LF=@#sLG^9xAJ}ew@-Sp7PXyzXUkop}CW@wtuUmKI
z<L`B<)iQhfChZp5KRGPJ-A*pcMR3j$iHSUG`aW8y7`>6T8f+14rNOdwN?dET@Cm)0
erEdz0tarWH_0KqRp~}yh>jhm41rw1I$qoQ?0h6@=

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/80114bf9781bffc9db411413d83541d8deaaf7c1 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/80114bf9781bffc9db411413d83541d8deaaf7c1
new file mode 100644
index 0000000000000000000000000000000000000000..4828092a8aa538f619fe0238f01041ec24187f0c
GIT binary patch
literal 343
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!Fj0%j5f(-wUFo-fR
zGBTh7&i_XkWCa))S(uTfm>8K^*%(-a*g5{+V&DMD*fUIxIA%CyTes`S{|r5Q>UL&q
z30k~-FH^O5Se%nnyyNn@J*CHX%f=?!O>D`O2~cKzdB{mAu7V+M1)C!l*P=T|MfvIX
zTd6nAqq7@tT~mH4?;g0-h{12Rs-nvCX46++AOB~NtoYlvK6ux0|0<Sm)2_JG)VTfj
kx0ZO_TM!+sBCE`C$)Zgm^Kwg*(wuLJCL4Q_?f>5d07Y$9;s5{u

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/80425fb92bb86627e854892f23823fa804e5fdc3 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/80425fb92bb86627e854892f23823fa804e5fdc3
new file mode 100644
index 0000000000000000000000000000000000000000..2ed0139989f994cef05524f606058404ebc7614a
GIT binary patch
literal 441
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4h9AWhA=@@4F(1VrX+877l!}s{b%+wFfecy
zctjR6FmMZlFeAgPIT8#E4D2PIzOL+#xMi7T7%!f!eZj!M@WRu@F~s8Z+ljt;hYUno
z*n1vnZCh>Pa)3#Id)1wVPCL2MA8QGUzYx0U`a^WVl(u7W7NII)4gyDVcBPf&e|h<G
zujB(wh81`E4l_J_VQ8?>ihbcNbLPE;vrh4di6;rnY)BP4ZqN1R?B_XU53jBJeYs-m
z4Xq8!*k=AnZFu+SXo>Q{ddIhJiIX4w>wjNr5}nIme^vb8+b^HrgzGy@NcTFKvfqD6
zP^8=gxr!xB9Gyoc%44oS?1@y-*)RF_S^VW*mi)cnclXBq<kq$64QIXfrou_O=)-i&
zjK3mN3|Lza&baZ;^40&Tsq22PoZL4<C#dt-L>)5*1_sp<*NBpo#FA92<f7EXl2isG
z149d4LqlC7%Me3DD`PV&Lt|Y73o8QyC!@tZC>nC}Q!>*k(KHxYSQ!{XG;EiB`UMmx
Mp00i_>zopr06f{Iz5oCK

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/821cdd6eeb919a8dd7f35289abbd583828dd4945 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/821cdd6eeb919a8dd7f35289abbd583828dd4945
new file mode 100644
index 0000000000000000000000000000000000000000..28925e3c80c2210e90461b82d950f63427bd5439
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0cwE%
z2N(o77!(*3m>C5bm;@P_1sVSzVUS>8U}j`wKn5(F9IXG3FsKVKFflPPGP5wULS-45
zm|0l_*%;V`ID{1qMU(;)3!DBQVGv_vL^FYb31qaKAOjOKBLfpNR67F$10$25q9e1R
za^u4Pw-|Upl7h^F4E79DT{2#Od$m-Y(_&xkzcmb{9~b}qs&GTBuCqd2<c+=Pvp?T=
zUK81KeBxo=Ju^N&So5P|hp_AOS4Va;Dm&!=RJNYZwfvMp??Of9>CrEW)=w+jJf)ny
zp?s!Zj>XfIZ{2~q%#J?`5AA!hV$zaNg6iAWKd{~W<YCBEo(RS%zZhO{O%zQPU$^ea
z$KUHztA+RUP1-HAe{xucyPaH?i{P9i5)*mW^nJ8YF?u6wWu?KgbxK@oweShOouzLI
zi>!CO+4avja-qu4nd=2z4FwZ-B&EAgJ;@mE{PeD1!?eE)S3`Z*zHJdTnL0B}@}}-X
zJ~JI%olV=eFLh<vc){Lzfwb~;w~PPme(hHgk#+NY^tCoTYTA1LX&bltKDy`~ogICf
z^X<AfpVk;}iawV!XXUKO)ZFZxH=fB0?9K;dtao6K{;M)^;WyK(KShLp+|HW5H9Gow
U=Pm!$nXx+I7hdu5HT=H`08RqUr~m)}

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/83e1a31785285338b0ddb3334b0ed098e63dedde b/tensorflow/core/kernels/fuzzing/corpus/decode_png/83e1a31785285338b0ddb3334b0ed098e63dedde
new file mode 100644
index 0000000000000000000000000000000000000000..9a5487fbfbe79979ddb06faf9d1b29a445db75eb
GIT binary patch
literal 285
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I7G?$phQ^Te;|vT8(g8jpt_%zejtmS84h#$o
z(F_ba3||=-7|t*-F#iAl|C{aRs|*Ybj7i?^E*x$5rdBa9FmM)lL>4nJa0`PlBg3pY
z5)2Fs>?NMQuI!JvWtn9dFP^P^!N9<v=IP=XqH#Vs!GVF1&4@wRa#kCELR)}NOXw!1
z!b1+$;?@brby*oobQ#({tvfTDfq_A_#5JNMC9x#cD!C{%u_Tqj$iUD-*U(Vc$TGyx
z(8}1%%FtNXz{1MFz{zNF4~mA|{FKbJN;C~d7FGs^5DnX9pMGIrU|{fc^>bP0l+XkK
D`Qu0|

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/8a4c8100dedd0fb5f2a8b468c678f7ad8269deeb b/tensorflow/core/kernels/fuzzing/corpus/decode_png/8a4c8100dedd0fb5f2a8b468c678f7ad8269deeb
new file mode 100644
index 0000000000000000000000000000000000000000..c6c8b7c717310177d1034ec4e1643afbdc6994bf
GIT binary patch
literal 666
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnm_agp0$v~rivk8lPd6q;H)jTBuqGuz
zhEaes1Qsyjlp<Y1P=Jd^NLWZfm|s9Z;Qs*zZVm<o1_ee&L5BZF7(^Ku(e2{=e}q9+
zfPs;P8Ci;nk(rf&fklX&<NqxN4v>sJ!_<gl22-|myMFx7(6gs*XU3KwzuBsaD$kou
ZUwwW2pFy(XZ{PZGhU5NKBs!Wr?Evr*JCgta

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/8ae8268c24dc866c1edb3826b93a1c75dbf74ff4 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/8ae8268c24dc866c1edb3826b93a1c75dbf74ff4
new file mode 100644
index 0000000000000000000000000000000000000000..ce211f7cfd7e6c6fc33c30d8fdcaf63e401a9365
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKe!i@iqFi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Z@3
zOsp(|Yz*u|9Kwo5B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%ZYZC^
zUOqF$;;CMa?$5w)%-xQKhxR>LF=@#sLG^9xAJ}ew@-Sp7PXyzXUkop}CW@wtuUmKI
z<L`B<)iQhfChZp5KRGPJ-A*pcMR3j$iHSUG`aW8y7`>6TveIDLIwh{PTKI(C&eAu9
zMb^9C?D}UMxlrZj%=LmO+869N_z*?A(3}7N8N%^scP$i5+>w;-KJ_GHxbxGyf(_ID
zGF%PyUHi60)MV<+Fv**`5Bbb=baggu+rHG5W#a{V=LOQrQ$EBwf4DY9eCj&8y?1=p
rWODsynC^D*pWUzhDk8FOo{zrPhDS|X?>}whR^LY#y?OZ>{@(-uuXWV<

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/90f72038cc627f34f074ea72eadbba87a5e3e288 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/90f72038cc627f34f074ea72eadbba87a5e3e288
new file mode 100644
index 0000000000000000000000000000000000000000..3f12cb5f6595439978754328620f1f8f0fb43291
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo5B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%o>I=<
zP(D*H$Kq+qx9&h)X2+j}hxR>LF=@#sLG^9xAJ}ew@-Sp7PXyzXUkop}CW@wtuUmKI
z<L`B<)iQhfChZp5KRGPJ-A*pcMR3j$iHSUG`aW8y7`>6TveIDLIwh{PTKI(C&eAu9
zMb^9C?D}UMxlrZj%=Ln<g@TDYlG5F$o@5MnetK81VcK7YtD(MY-?oUFOr04fc~kcx
zpP7!X&ZceKm%6fSykPIVKw5dqhdAdC*QSV1U1zuVj?bD*uKx_v-7fyK`?X(1MApso
z(bwAWsA=o{r)}Ko`{<%~bawP@&bRB{d|G3?Df(Q_oRza8Q**O(-gqW2usa`+vEG3_
o`mf5wh2KoC{uB}ZaXV}J*68T#owxi~XU6J;UwFmK*YN))01qhEXaE2J

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/92b67faee4a49df2cdbed785e27b4a1cddcfffa3 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/92b67faee4a49df2cdbed785e27b4a1cddcfffa3
new file mode 100644
index 0000000000000000000000000000000000000000..18ff654a110c137350e93dec975a12e17cc0761c
GIT binary patch
literal 507
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0aAbl
zNb&yz41ydC3JeO&jDieIf{e_9jQ@`?NHH)lvNAFt114q`HcpQJM;LSk7?>CtS($m^
z@{CN(EDWq{f<o*ZoWdd!l8T0oVd2x4UAZdy{|JLPBM2ay#K6eR^#2Hhk{}}kGXo<t
z3o}fTk%38&nMKG@QOU8<DRAQdTMRtRU@<`kdxlf1mBd_%=FI~I5(_z@^nlT9>*|uV
zz7ZW;IwFO$s=NzwKRr9OclrGJZ*QIETiyHHmglGO`EPGJ<B!g<DZg|uts`D~|0Axx
zx_1-n)uOU5d%9k!^vl%QGvV}(lS>TWNX|Q*e)@Er)c5GR*7}Cce0huJT5gwLb?d(3
PkN)?^=UhMV{Qpe=OU-Nd

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/9463810467aacdc9923b2b20a2236116b760d75b b/tensorflow/core/kernels/fuzzing/corpus/decode_png/9463810467aacdc9923b2b20a2236116b760d75b
new file mode 100644
index 0000000000000000000000000000000000000000..eec341bf2bd623f50bbb5e393f4aaa245bf15fec
GIT binary patch
literal 258
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;ICT0c(20oEV=?n}EJOMr-uK)l4zyJUL|Nr0r
z|1UYw^=~=@0|R4{x4R2N2dk_H0|NtRfk$L90|U1(2s1Lwnj^u$z`$PO>Fdh=h+B?X
zhVkOr+7}EA3{svhjv*44lM@t#41^YUbVgi0!OEVI%V@>G5K+x=WcTYmYZw?9R7+eV
zN>dU`QmvA$QWHy38H@}JEp!bHb&V`T3=OS}&8!TKbqy@63=Eu%7WY6klz=tl=BH$)
fRibGyvam8RglO0<`}7L~0|SGntDnm{r-UW|Mj}Xb

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/94d7c96aea32ad41ce643d35b951a6d8990b81d6 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/94d7c96aea32ad41ce643d35b951a6d8990b81d6
new file mode 100644
index 0000000000000000000000000000000000000000..776f17c6b218bd55751e740b35b3ad6f4097a288
GIT binary patch
literal 546
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbS8u&}VOaImqlvT<^7
zadC2Ra&qzT^KfzVadUF=3i9&t3kV1baPbHU3keAG3kV2+3}FQ6WM<)DVc`(q=HwP2
z8T>!MAjrX>z@Wg)D9FGh$jB_n`2Prl1Oo#zBO?PcVBzFo{eOf(U4Vg!iHVV!g^?91
z%gAIXq7;}|*!2GhgBT+tnl=U|ka==~3{1?73{1>WJq!%Cj7)-xj?9M2jSK(ZV&DNu
z3Ni~a*fUIZ$$0&Zje%W=Ls*d<gBjokha~1@pd0*ZsW_*_zS@6l7)lra{i<+7tgf>{
zUF40u=(9iHcU}|Ob9~}q-aRuuK3MakVu!G+^vff=|J~U?fmzvlI@j`32E7XvnIC<v
z4Ud|(-hbN0t-g;gdPiqR-{yR~?#-t)#+#zg<=k00D>5}VJLipO@&dc_0U7HZ*rWfd
mOkDWQ^y*I$;UBlNrf-dozTSDue|2W8PWXjaynGG+Zvp^V6@Z%n

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/98cc7e9fe87df914d89a0aef008930f27b3c26f5 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/98cc7e9fe87df914d89a0aef008930f27b3c26f5
new file mode 100644
index 0000000000000000000000000000000000000000..d7296ca03c4fa909de8fb18d8dda7f9cc6bafce7
GIT binary patch
literal 311
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I7G?$phQ^Te;|vT8S^+*Gt_%zeMTNl(j0_A6
zBBB1l3=GcyK>)0rnL&VoL7>Z*p^Jf`hT(Ed?_LH52F4_BcNezAv>mk!3=EtF9+AZi
z4BWyX%*Zfnjsyb(1AB?5uPgf_Zdqm-#*1fbUobE*7<#%mhG?8mPFP?rkdU-&_gt1n
z?YxyHDGiDYLdwnztQ!_{U*IXp&^!Ei=N8^{!x%1x{N;K}g18n2FfcHvmbgZgq$HN4
zS|t~yCYGc!7#SE^=o%X88d-)I8d@2fSs5DZ8dz8v7&sX%?m^Lzo1c=IR*9y;$im9N
X5Taqb?9(p{3=9mOu6{1-oD!M<apX|7

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/99172dfdb4f59aaced29c7681ac6e6ce8356e814 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/99172dfdb4f59aaced29c7681ac6e6ce8356e814
new file mode 100644
index 0000000000000000000000000000000000000000..82559facc80344c43d682b2ee9de939b517264bf
GIT binary patch
literal 104
zcmZ?wbhEHbRA5kG_{hxg|9{E-|NsC0|Nj5~f5o3H3?QKRpW8DxF+J5cF+DRmn>{@-
zH#gDNfI$Z&2hzsCq%^00<>|Nli|2f4?b>`Vri-ubk;k-Woy#;9Y<tIj{FBf12u22L
E0I*mqK>z>%

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/9ae3b647d895af97fe872c0b1442df7b5b767160 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/9ae3b647d895af97fe872c0b1442df7b5b767160
new file mode 100644
index 0000000000000000000000000000000000000000..9e2eff2c4ec0c6ac3ba6e5558cd0a881b4dbb042
GIT binary patch
literal 332
zcmZ?wbhEHbRA5kG_`(1Je0+Qg3JL}W2KM&$-rnBv@$t#Y$!Te6g@uJ39UT)UOqe};
z_T0I1mo8npX3d%%J9g~bx9`A#1E)@%x^m^pwQJX|U%!6y=FJBW9z1#S1Y{g4R{Y5V
z7Et`p?U|dHp6Z*Jo|&A@o}QSSn`mpmpaYVWWl&&XVBMsk(3g@qFJslZoY(sbbk3LL
z9(TJe@x3B(y4Tb4jhsf)JXx4JeRL!lb)6^LNOZG4R$Q2pCf<Eyp@)J1XRC9P<}riG
z?T-Q!1i09bCN~|7RpQ}fWoBYxX657&6t~gm6&2v-U}0ipWMbjq77*n%=$|-w>hzhj
z=NhopH#WDncXs#c7ZjD0RaDg&B_^k)XJ+RbhlE8$#l)GodwTo&2b#G!x!5XMT3@+w
Q^VaP<ckkWTRAjIQ06fQb`v3p{

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/9d2b1d2121b0508a4fa8d1508adb9d05633fdac3 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/9d2b1d2121b0508a4fa8d1508adb9d05633fdac3
new file mode 100644
index 0000000000000000000000000000000000000000..6d17e06d470d501c0984f339d75de0f3ba59b4fd
GIT binary patch
literal 222
zcmZ?wbhEHbRA5kG_`(1JjN+UOEDU^nd<-HC3JMB_2Bu~vX7={>3=IC>-rn)?@yW@_
zX=z@Cg@p_Z^Ex^@CQO(xd-m+PbLTEyx^&H&H9L0f*tc)rfddClojP^p%9U%^u3f)=
z{pQV^4<0;t^5lu)PZp3J42u7`J#!P&Q+*TDGn2E~(-U)Z6KzcybQrQ;gXI|*cz$L^
zsJaTudR*aK%<B2Gjj7N?ijRZ!@Jj|EcmHKfi5dbNtjx}?>y(USB*n$V#3g0qS+^<6
W%E?L}z5n3hwVRKgJ~L8eum%7zgHT8S

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/a335af37917ccf0c8b11bb884a3a74f3f1d2a7c6 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/a335af37917ccf0c8b11bb884a3a74f3f1d2a7c6
new file mode 100644
index 0000000000000000000000000000000000000000..ce8245f2da2025b6c37911a22179c45d06b90f5e
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo*B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^rhmxq0`DXPg(o6
zMbu>K%rMEDy4``g%#J?`5AA!hV$zaNg6iAWKd{~W<YCBEo(RS%zZhO{O%zQPU$^ea
z$KUHzt7Z1|P1-HAe{xucyPaH?i{P9i5)*mW^nJ8YF?u6wWu?KgbxK@oweShOouzLI
zi>!CO+4avja-qu4nd=2z3k4H*B&EAgJ;@mE{PeD1!?eE)S3`Y~T=J05Oh;E|)3)tP
zU0F6>uy<Y{tvuyJob!ikQ^cpPv)g;eXH6#8e}?I97ysG)+OHxa>*o3BYi)ScwDtbe
zHg5HObkRFHJNh=~+jVa~tufvdeJ*Ft%2|=Ax!E~yJd+pLoe#)Z@4z1YS7qYDZ>Cp&
fiU|L>oi%-HboBMkTmGvvV|BtWyyE3+_<s`sAEMQa

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/a738609112d3a6772c50a71e2c3504ebc515b709 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/a738609112d3a6772c50a71e2c3504ebc515b709
new file mode 100644
index 0000000000000000000000000000000000000000..a980c777a8531fd113f9f54dee0fa00946f7a420
GIT binary patch
literal 337
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fo-cQGBPrt07lOLM;Igp7#WyRWf%n&9TP!X7%{al
zfwTyMBvAA)3MvXQ{lCS)!_3IQB*-ktV9#*XfYI!a@UBG-3=9mj3pd7gC7s`2blT?W
Z&AIo|%O{nm#ooL*S37}$fr0=3O#qg%F>C+;

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/a8cecab5d917da5a4729632a7a18c564d7e1607d b/tensorflow/core/kernels/fuzzing/corpus/decode_png/a8cecab5d917da5a4729632a7a18c564d7e1607d
new file mode 100644
index 0000000000000000000000000000000000000000..31a0fe82b99e564781cf22c7de167c25fc4d5e37
GIT binary patch
literal 258
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;ICT0c(20oEV=?n}EJOMr-uK)l4zyJUL|Nr0r
z|1UYw^=~=@0|R4{x4R2N2dk_H0|NtRfk$L90|U1(2s1Lwnj^u$z`$PO>Fdh=h+B?X
zhVkOr+7}EA3{svhjv*44lM@t#41^YUbVgi0!OEVI%V@>G5K+x=WcTYmYZw?9R7+eV
zN>UO_Qmw3#Qxi*48H@}JEp!bHb&V`T3=OS}&8!TKbqy@63=Eu%7WbfN$jwj5Osixt
hF)%hmXfU#{GBAW_*e?6@3j+fKgQu&X%Q~loCICg|NOb@J

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ade919ab2b4a458e806575c941dfe50ae3fd3621 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ade919ab2b4a458e806575c941dfe50ae3fd3621
new file mode 100644
index 0000000000000000000000000000000000000000..776f0b88dcca473db2eb6cb7ccfab99183260ce8
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAORKzP7c=pM;O!v7?_xt7@1iZS)sDb
zj7+R7f@}=zLL9=1Mj}dqiG@x7k1&WaGNPKmz`z7DT27FGiJ6gsi5aS$fq{XMNl?*|
z*-*K0;s09<JRnIyW<dsfhN&(YufM%oD$Z%KulC;>hSHCV|9(}tAy(H}p)T^qUi8_Y
z?>n!F>^VO1Fz=ojA0MpwQL#hV^|_d}%fCDOCon5pPv=^G%Aj|lBJ=d<7e(u*m2GY)
zpTb@~Q!mHjX^QU8z;Ddmj)jNzJy|hn$tOYeZR;P{ZhrDGWGYVt<CI?vFSsU(ri!my
zcjV*mb*j}ed-^8r7TP~KEW_PSF3Ux5&Jl@;JZt(sTBsPkk+rhYVA(n)uC-eDgx=25
zH-$ykyWZ^jXB@du<>$=xg7~!y5z_8jD44h-DcybQNyc#Jr*{P#ru}8O8tS|DZHuVM
z)R|$DH+3KKnd#{2Y}&ScsVmFI3--<nq?M<9h;#mMZHoBRb#{C2_^ip~`p+=k?czVX
hU;9-=WZgU;eXR|Tnzr75+QzNEk1l%i@-_Uw2>`4l(QE(!

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/b1251621a5eb5e7fda9cac9baead1c993a285c36 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/b1251621a5eb5e7fda9cac9baead1c993a285c36
new file mode 100644
index 0000000000000000000000000000000000000000..ba6aa256542e6dc9a6d802784ab451ed6b93b141
GIT binary patch
literal 277
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I7G?$phQ^Te;|vT85&=FTt_%ze416FN1_li#
z1_llgpYi|y|8Y4dBN!MM7?Zr+UD(AEJ6<y|FmM)lL>4nJa0`PlBg3pY5)2Fs>?NMQ
zuI!JvWtn9dFP^P^!N9<v=;`7ZqH#VsVS#4ClC>WN798;5k7nc9Fkzi)gUQpJ9ZMN}
zH5lrdt^`eGU|>)!ag8WRNi0dVN-j!GEJ<ZBGBC8zH8j*UvJ5dav@$lcGBnmTu&^>P
pa57rlgQ6ifKP5A*5>11Vg_VIJM8kI3r(Zz+@pScbS?83{1OVIYMZf?6

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/b1516b78c3dfe77eeb554985fd7344c0478fbbcb b/tensorflow/core/kernels/fuzzing/corpus/decode_png/b1516b78c3dfe77eeb554985fd7344c0478fbbcb
new file mode 100644
index 0000000000000000000000000000000000000000..c4ec4ad4b9c4a38459c974e8718532bc8ff4d345
GIT binary patch
literal 258
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;ICT0c(20oEV=?n}EJOMr-uK)l4zyJUL|Nr0r
z|1UYw^=~=@0|R4{x4R2N2dk_H0|NtRfk$L90|U1(0~AOwFfg!}c>21sKjM~SmSMbj
zw)O=B1A~;Oi(`ny<>UkfAp@bs9i0)EPq4CQ<T6?@Fho={9NGPP&l&~>2GtVRh?11V
zl2j|J<kZBHR0bmhLknF)LtP`w5JN*NV>2s5V_gFaD+2>3qs2Wa8glbfGSey<Obm?8
Y5E_gutPBhx8n(+m{Q`5^I;Vst0A)-^RsaA1

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/b41241740f5f8ad2c1d408f7bb6a313bd863c158 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/b41241740f5f8ad2c1d408f7bb6a313bd863c158
new file mode 100644
index 0000000000000000000000000000000000000000..5413efd933685ec8073cdccd087a5009439ffe17
GIT binary patch
literal 133
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;IMg|53hG<6jNCpN5rX+877Y4?=UBXKk7#KJU
zJR*x37`TN&n2}-D90>*n2KEw9Usv`=+_KCvj2F+=zF=Tr;PG^E4B@z*oFH+Bfq`KG
g17o6Utvdq)gKCLuL`h0wNvc(HQEH;gI;Vst04E<D_y7O^

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/b799c8596523a7ebeb8e11ada08818c10f7eabfc b/tensorflow/core/kernels/fuzzing/corpus/decode_png/b799c8596523a7ebeb8e11ada08818c10f7eabfc
new file mode 100644
index 0000000000000000000000000000000000000000..9cd72246b91b803783c6224511d825edc14c299f
GIT binary patch
literal 582
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$Atc-!cIKK>;ovAz>i_VSWJt0gxe#Af3!C94ssx0^FS30wja~
z2N(o77!(*3m>C5bm;@P_1sNC^8JGl_S%eG}l^h$L0w?~z#lXW1787K!XE?Q5NzA2a
z-aN0BJce5wS}Hjwh4!SZiuksXVNT<T@{K1LJQ<Fj=)HABr(=ia^OeP-CPH2hBTp@w
zW)<pfc%p)pW%2_?v#qO3*7`<tZ0U#;&Z_b*$o=%})ZXRuKd<R1nfBoJ)3Z~n{q3Je
z3cS=kxc&5Ot#9+{&uvn%(h)w`pZm4E&7x_0+Vzr2ep@~}+9|9(RM_q@fmxLKtk*)L
zmzz9m%T7F7yEFF?U&D`LQJ!sO>$hGATV}>|_4d5k7D=;P{r(uc<>coVEm~G(RXR&z
z>5=J~S=WuQm~{4Whqf6u<C2~Vev$n#r*GCTkJ@nQZ&i|?vgg0Ob((K=?{8b4E$2ck
z!ZSH1EXwRWfLGy9)AQfnbjBZ@V^e<VU|L7K^!`U&eRc09)~iKjU-ndsS~S;kyZowK
S_Z5Hizdt_b`hn;FZvp^X_}M1_

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ba48d0521a111222dc95a3a997c7c92dea5f4443 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ba48d0521a111222dc95a3a997c7c92dea5f4443
new file mode 100644
index 0000000000000000000000000000000000000000..66aac674cd26032159ef1a195cee7e70cd8511ec
GIT binary patch
literal 249
zcmZ?wbhEHbRA5kG_`(1Jj0_A6EDU^nd<-HC3JMAa1_q{vX7={>3=IC>-rn)?@yW@_
zX=z@Cg@p_Z^Ex^@CQO(xd-m+Pb2|h4+<CdAxOVKY+qdt)fdi*bow{=6%C&3Pu3x`?
z^XAP54<0;u@<j0`3&;)z#sA!%xrynizKQ9X$=U4biMhFnwgwD33|X)D<$Wwvlw>&a
zGc!WfRnU`l@y{zBReWtsg(gy59IS_5G6=c*FJns75a3{C4mw%U*WmhGsZfKbwaM|}
zgcz&t$0^C3CIYO?%&Z)|??3yN=G4?RG_|yKbeRTm=H0&k;Ni8KkDfj=Qe?0O0DvA}
ATL1t6

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/c01457c6889fb1b597d308363a36412c0b7f90e7 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/c01457c6889fb1b597d308363a36412c0b7f90e7
new file mode 100644
index 0000000000000000000000000000000000000000..eff793b204a409edde4d0b601cb5e43e3fe1d088
GIT binary patch
literal 497
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fo-iSGBPqUAOl8b&i_Xk6a^TVm>5}@nBh{4Obmj|
zEJBKofrW;R6G3`GTG8|}F@y99GBPkRF*7i-!Zb572r?-$8#*Qm1r|1b_<xImhnbOq
zNsw8P!Jgr1h>MFuT5y(!iMt$+>UTX$i|yS(lV&q;-F|a(AIGB|>ubN+6q^@Y@9$Kd
zrlh2#q<m*-+^^1U>)CYeLXWpcwuda2vuXXv&Lug)Sn=Mfa!pl!(X6s*QOW1~&5b@C
zJ)U3sIDF~ln{Tbz4$l{>EK06Sdc0-Xo`902DLHp9MVLt%a2;XeVO%ZG{;mAkVL3gY
v4ab!g0=$<$*^v2WL*0W{IXOG6R{fqC{F3YMe}?S%{_>|0Gdn<F{{Nc*v<!Jc

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/c82ebc0d6688d104af04fd20d6d3da591dc391f7 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/c82ebc0d6688d104af04fd20d6d3da591dc391f7
new file mode 100644
index 0000000000000000000000000000000000000000..ba604969f6e0275a1f9db37b2755e9d693d60f5c
GIT binary patch
literal 198
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I7G?$phQ^Te;|vT8S^+*Gt_%zeMTNl(j0_A6
zBBB1l3=GcyK>)0rnL&VoL7>Z*p^Jf`hT(Ed?_LH52F4_BcNezAv>nzA3=EtF9+AZi
z4BWyX%*Zfnjsyb(1AB?5uPgf_Zdqm-#*1fbUobE*7<#%mhG?8GNl7e8wMs5ZHMGz*
oG}JY+3^6pcGP1BTFw`}$ure^%F8lNg0|Nttr>mdKI;Vst041?5UH||9

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/c9a03eb758dd84e954e3d70916e2311e8fd21f3c b/tensorflow/core/kernels/fuzzing/corpus/decode_png/c9a03eb758dd84e954e3d70916e2311e8fd21f3c
new file mode 100644
index 0000000000000000000000000000000000000000..c23fb3da9ce10cfab5eed3382e34f887bb117942
GIT binary patch
literal 249
zcmZ?wbhEHbRA5kG_`(1Jj0_A6EDU^nd<-HC3JMAa1_q{vX7={>3=IC>-rn)?@yW@_
zX=z@Cg@p_Z^Ex^@CQO(xd-m+Pb2|h4+<CdAxOVKY+qdt)fdi*bow{=6%C&3Pu3x`?
z^XAP54<0;u@<j0`3&;)z#sA!%xrynizKQ9X$=U4biMhFnwgwD33|X)D<$Wwvlw>&a
zGc!WfRnU`l&Ce?yReWtsg(gy59IS_5G6=c*FJns75a3{C4mw%U*WmhGsZfKbwaM|}
zgcz&t$0^C3CIYO?%&Z)|??3yN=G4?RG_|yKbeRTm=H0&k;Ni8KkDfj=Qe?0O0E0hY
AWB>pF

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/cf892756b33578a54ab20044514e573328d2f1d7 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/cf892756b33578a54ab20044514e573328d2f1d7
new file mode 100644
index 0000000000000000000000000000000000000000..42f1f9a29827e46a86ee16dd90a8779e7bc927cb
GIT binary patch
literal 249
zcmZ?wbhEHbRA5kG_`(1Jj0_A6EDU^nd<-HC3JMAa1_q{vX7={>3=IC>-rn)?@yW@_
zX=z@Cg@p_Z^Ex^@CQO(xd-m+Pb2|h4+<CdAxOVKY+qdt)fdi*bow{=6%C&3PZd|{9
z^XAP54<0;u@<j0`3&;)z#sA!%xrynizKQ9X$=U4biMhFnwuTHk3|X)D<$Wwvlw>&a
zGc!WfRnU`l&Ce?yReWtsg(gy59IS_5G6=c*FJns75a3{C4mw%U*WmhGsZfKbwaM|}
zgcz&t$0^C3CIYO?%&Z)|??3yN=G4?RG_|yKbeRTm=H0&k;Ni8KkDfj=Qe?0O0E9+g
AW&i*H

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/d3bc3f158a63f1d50b474addd3f7b3d17f23e8e9 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/d3bc3f158a63f1d50b474addd3f7b3d17f23e8e9
new file mode 100644
index 0000000000000000000000000000000000000000..6b1183f4ffaab6f8b429eeb7ce3640bd3df887d6
GIT binary patch
literal 507
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0cwE%
z2N(o77!(*3m>C5bm;@P_1sVSzVUS{AU}R-vKn6_AENq+{|Bo=}3NSD+GO{xB!sQv6
zm{}ND*#w2yIXHzyBqS9L9K*t=FS~M8^#2hCaYhh8Hi?0end$!#1|>m824)6EW)^0c
zBqIZpATx`QqM?#wqf_9-|F;-;n89L#4ECWtDXSvBZDg3!c%ppc2?kGwqbGWA9ntC7
zq4|7ev8aiV*TcwDi>6tHdK;doU}c&7fYEI0>XNm-5gl7PB89W6ybE$aJv+5``TY5B
zZ=L2_-TT{?=cnoUZ*Mx|kIu0vzjQFIBVKy{Bd)%>cN6Q?qOvc0x?ZXD%hcI3;q;D^
uOAHw_=bcVJeL7C+dvslEeZywHyhU>@x67}(bzkvE|NG-}t{-^*|0V#6gL-NJ

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/d4906950aa9d60ad09dc0f5413c3d88080c3bc37 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/d4906950aa9d60ad09dc0f5413c3d88080c3bc37
new file mode 100644
index 0000000000000000000000000000000000000000..2d8fd3f3f0d34782904ba50077e62b6c82ef4f13
GIT binary patch
literal 262
zcmZ?wbhEHbRA5kG_`(1Jj0_A6EDU^nd<-HC3JMAa1_q{vX7={>3=IC>-rn)?@yW@_
zX=(re|L0<0V07r{m@r|&?AiHAbLTEyx^#`+njJfK?Ay2Rz<~p&PMx}vp6Z*Jo|&A@
zo}QSSn;2=tpu>>$dSBkhLRRv?@4rOpE8t|XuuMGiGc!WfRnU`l@y{zB@_cPfg(gyb
d9ITS!Vq)TwGV-k3lx5{)rH@J<4pZf}1^^ccI}!i@

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/da31578a8068bad65e1c7a3d06e8f543a2a0bc65 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/da31578a8068bad65e1c7a3d06e8f543a2a0bc65
new file mode 100644
index 0000000000000000000000000000000000000000..dc37f788a1acf12c1e252c880f26eb0a4f53809e
GIT binary patch
literal 332
zcmZ?wbhEHbRA5kG_`(1Je0+Qg3JL}W2KM&$-rnBv@$t#Y$!Te6g@uJ39UT)UOqe};
z_T0I1mo8npX3d%%J9g~bx9`A#1E)@%x^m^pwQJX|U%!6y=FJBW9z1#S1Y{g4R{Y5V
z7Et`p?U|dHp6Z*Jo|&A@o}QSSn`mpmpaYTwIfa3BlY&BDO6I(bRqJwI?<>$bUy^&=
z?Xtx8ip1$&Ps=xQ8cp+LVe0hJkz~|$o@gV{&H7kzVM>~K_mPDj3Id$1&Pkfb3?{cf
z3Q!Q>Vn3SPbTC$lhm)0=iHVt&lSfe8MxR$yfSZGbiII_sg@ao_l-Hnt;^e8*XU?8$
zz*gVb+}hsR-K$?vR8m$^Rb!NxoSL4QoogHt77-N_XX5VZ?du<C=HldHt7K_?<;Klh
Ox9{A&cVAPH!5RQQ_;&sP

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/dd4a9b5d0740679c249fc884efc499433b29436b b/tensorflow/core/kernels/fuzzing/corpus/decode_png/dd4a9b5d0740679c249fc884efc499433b29436b
new file mode 100644
index 0000000000000000000000000000000000000000..82c5120f464521aa502f234fbccf4ea3e5763b7b
GIT binary patch
literal 666
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0^lFtRc-AOj|57B)_f|3?^f1sIqZ8CjWm;qr`3
z%q$G7Y=T1U9Gt=;5|WCBj$z@`mtDCk`u_-nI3oxko5aA#%=G^VgOVU412Y37GYd0J
zl97Q)keNluP*KUT(J64^|62?^%wRD=27886tChrDissGpTFFzm#i6B=b5dwe%BqNO
z8yV&_o+#gVg29vF=!xE2M|3)NXg*(AENUX;^)T|(qG?v4-i9YCSXm}NU^Lshx@4_y
zM8}qnNa3t1?}FS<&ra=KKL7KYj*@8)Za+Ocwc6kQd8EKg-Gkdt&(``jum0R76)PR#
zgZ;T*%iAoPwx?Y$ndG<Sv!k8D+Czoy9ut^Fna_GHG<kXUafi0qCePZk6VKM}%ss@{
z@S|9iXIt6&trx<UnK50xJ#V%}((G2hKgI{+O?Im-T2^IMI!j{dk?EOP*Nu*~X>2(c
zViBInIbl&|=YgcBf?s5R%;}r;%cC}2`dgLcr|kJ}Z=L2_-TT{?=cnoUZ*Mx|kIu0v
zzjQFIBVKy{Bd)%>cN6Q?qOvc0x?ZXD%hcI3;q;D^OAOyg&O4oc`gEMs_vpIT`i9MX
ad5h*+ZkJzm>%QWT{`be{TtD#q|4jg1iPPx-

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/deea4ecc6f0b2a6d89fd25ff76762299f21602fb b/tensorflow/core/kernels/fuzzing/corpus/decode_png/deea4ecc6f0b2a6d89fd25ff76762299f21602fb
new file mode 100644
index 0000000000000000000000000000000000000000..6daa5452a159945afb1d5e072f3f5daa3a1cc2d6
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3<TEn3<SC6caNu3mXdvaImqlvT<^7
zadC2Ra&qzT^KfzVadUF=3i9&t3kV1baPbHU3keAG3kV2+3}IwuW@ce#;b39m5a8zI
z79bh?KfoZ!!JxpPz|1Jfz$D1XEXer(2!jLz12ZEd12SOY<Y4`Ogh5?^fr*KciJ66w
z6)MZf#LUVf$i~1f#38I`D54aYSlIOc2!j|SBbo^eOdzA>1R0o^85x+Eq1qW37#NuZ
z6&;xkm9-cCzs0}<k`!bXV6bPH>XPyL+pDGGoEH0P|E*yt{kZt=SA`p5b)6OJB5&+P
zpZ)p1^P0$>;}Z|_?wRrN!I~cxJA_@Ii|uy#cW3_uW@YQ?T+2@x^e$9no*w<8X#KRZ
z%~Q(R8_H+u<ybsT`PLn%%k22G@X)>|D<&=ZB&fb^{R7+0PacL$<%wXN@{8dG*F@1&
z@pbEteEhvmwOVFR-=y6_`zMEGxZBBPxd_fVA~BI?P2Wcg6{9z@R#qA;Tc^bJRtuld
z+gbXiu*iDXn_d5mBNwXtoVi}mwNNl|$MhLHvQ9FFJ3qZE*f8xc!_`pVwQpNQO{UHa
zlf0?>kk3p<S7+0<?Mq!*HeRrIULdVJ<wKnFhig;Br>?Wxd&g%@Cf9$4>24SQ+5OtD
zA|mVN`RHqHc+|A@{?j&Y^?h{FJ32f1Hs{-QZ$7Ot-V}W<XU@u5k*T@aId43Z7ucN-
t$XM^d9{pEk;=*sHSAU8K|G1qseQR{|_0C)Vt21MD!Y{ny<!ktV69AcR*F*pS

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/e1040c7ffcb39915e0f539018c81f9798924cba6 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/e1040c7ffcb39915e0f539018c81f9798924cba6
new file mode 100644
index 0000000000000000000000000000000000000000..306bbf464bf804c169e405550ecc2c68d33c7020
GIT binary patch
literal 94
zcmZ?wbhEHbRA5kG_`tyM|Nnmm1_s5SEDQ`r3=E3@xjl0e(^GvD(=(H^+0zqqa}#Y%
n8FUyJ7#Khr7?|W*`d6NQ%fEQemRsGM@9p``-}VTZ?O_c7bS)#i

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/e381dc85682cc33ad99f622b89d145b47f7d6392 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/e381dc85682cc33ad99f622b89d145b47f7d6392
new file mode 100644
index 0000000000000000000000000000000000000000..36487c0002294149201115e745543cc2363515d4
GIT binary patch
literal 485
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4h9AWhA=@@4F(1VrX+877l!}s{b%+wFfecy
zctjR6FmMZlFeAgPIT8#E4D2PIzOL+#xMi7T7%!f!eZj!MDB|hj7-Dhy?&ZMzCIbQX
z5B2MhHt8yP9g6$**ZzuJZ>L&{)WX1)xbGWQFyGvCeQmyP-N~)x|AiUt<qyd-WL5Ao
z9NDhG@{GYz<%mkc{javoQ@T`q8Q$GGz4Pk?tIkpv|AyV0SHFJNHPbivSnM5#4{NQ%
zt{y!2r1kh?+42jqud;UEw4Sl4wP%V9<Gb^vTPtsyZcu+wDP+v+@L*pq-|xHMI#k78
zhh8fdI4tyL{W;U)<qE~`Ui}nnj^7?NVUA?-^aGcqj3=wKIZU`@rtg0uA^y~zEZ4>H
z6TB`hI(q2fl=ZH&RunFF5m_!*^)}(tB9F%&J&jj*EaPjp-O8|JTfiZrdh_yzBg)?{
zKmNzGj9Iy@`?HMYM}4!`%%2$wCzQ9?8Za<0sFt`!l%yn<q*^5xr6!i7G8h>cTId=Y
z>Ka*w7#dm`n^_qe>l#>C85lSjE$%_lkei>9nO2FW!N|hOz!0KgyX@01p!oB2^>bP0
Hl+XkKwtKx6

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ea24498fc7a144fccc6f1665ebf7020df803dd1a b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ea24498fc7a144fccc6f1665ebf7020df803dd1a
new file mode 100644
index 0000000000000000000000000000000000000000..ab99a8374aa80d829891b5670078d95b776a6957
GIT binary patch
literal 677
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0>kFf%eTAOjXo4%Yui7}NzAn3$LtnOPWFp|Xrj
z%&aVeYz*u|9Kwo*B1(aYg-!pDFo-cSqM5+J1TtDqkb#Mrk%5UBs-1y>fssj2(UI9u
zxpCqDTMRrPNkL{o27897E*Y=Cy;>^HX|b>N-x`L}kBk3)Rk$Hm*IA)1^2T2D*`Mz_
zuZiqAKJhT`o*5q>toc!~L)i7Xn6=BlJNqXvD_c+JT7Jr)ccCKl^yn8w>!+1%p0f6B
zi>S%enPHMQb-M#~nH_%?9@_V0#iS*l1l6~#e_*@$$-|JTJQ0jjelfh@nkbqozHZ%-
zkH6QcR?F<^o3vYK|KzX?cRRT(7r{A4Bqs8#>HBD*V)RDVD!8z8>y)_GYT*-lJ4@db
z7Fq9lv+JL6<U*C7GuI2c778ZrNJ@8~dXh2R`RQH3hG~Bpu7>&|x#S_AnU1c`rfu7o
zy0UD%VDG#@T6xNcIOh-7rif2nXSer`&zelG{|wXJF8;IowO>U<*3I+L*V^!?Y3u!`
zZQSbn=%ROYcJyt|x9i?~T4TH^`drSOm9rvKbF*{acqT8fJ0Fm--hn;(ugb)Q-%PLm
f6cPS$J8SyZ=;-U6xBORU#_EJ$c*V=t@c$+Np0(E{

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/eaa5d677e797c07bac98c3c7051abad91852e7c6 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/eaa5d677e797c07bac98c3c7051abad91852e7c6
new file mode 100644
index 0000000000000000000000000000000000000000..63ff2676ae32de1b7293ec4a8c34b550b2c22ad5
GIT binary patch
literal 507
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0^lFtRc-AOj|57B)_f|3?^f1sIqZ8CjWm;qr`3
z%q$G7Y=T1U9Gt=;5|WCBj$z@`mtDCk`u_-nI3oxko5aA#%=G^VgOVU412Y37GYd0J
zl97Q)keNluP*KUT(J64^|62?^%wRD=27886tChrDissGpTFFzm#i6B=b5dwe%BqNO
z8yV&_o+#gVg29vF=!xE2M|3)NXg*(AENUX;^)T|(qG?v4-i9YCSXm}NU^Lshx@4_y
zM8}qnNa3t1?}FS<&ra=KK7an(Tc`O}_x`r!`DuFo+ndh#qjPM^FC9$lh?m~~h^w#e
z-NbsesO-z0u2(AkGIjP$IKAWK62mu=^G>IqJ{>3ZJ-V*7zF{+8-lDmd+vQi?y07@7
O|NZef*AG1Ze-i+(N`bNf

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ed7871269315725535d8bffec7836c45a3fc5c26 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ed7871269315725535d8bffec7836c45a3fc5c26
new file mode 100644
index 0000000000000000000000000000000000000000..d2a4b9aafd7b745a50401a3ac94660ccc787f99a
GIT binary patch
literal 527
zcmZ?wbhEHbRA5kG_^Qmn$H%9jpkQENU~g~l?d_eMoSc@HR#;fr(a|ws!i3qgXD?m4
zbj_MIJ9g~Yw{PEp0|!o>I(6mBm220oUB7<)=FOWA9z1yR<jKpIFJHfY{r2tK_wV0-
z{P^+f*RMZ*{P_9v=dWMCe*gac_wV0-|Ni~||DS;dK=CIF*i^;;+@86K>8ZYn>6yvd
z?CFWQxrw$03_2i5kXISl{vDWD;GrYce_~0|$sDWs7gm&TiAK6!S){@8sl-YyfWw=&
z)?zjf+vJ-Cb_+9CY~C57A;h_~=w`{IfMO{zVF5l~UOoX~F{yfgC3$HvL4F=CPEIZ!
zenBy5`4k^T8F68LZVq-fHg*o~;L;?S1|J1!Q2`zfRu*Pv7FG@(0a0lMUnMz75k4+<
z7A8hUCKh%sJ`qW|MV?Bs5<<M3Y|M-d42;ZdoV-F3vUff1KYaZ3`ODXD-+%Nub>`dx
z&&6rHv(55+^KL!8`Z|0=;?X`!#Z<1TT4C!FPA+<?a%qXzbfer?Uo@`>JGSsB1|2yO
yxVE$8&4Y`FkDot1IsMX_%+Jr##6<Pt4s=X(f68@H$78Zq?5Qm)gOwZ^7_0#(Ah{j@

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ee8460f4077064c5a2137075b48eba7d3db5c570 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ee8460f4077064c5a2137075b48eba7d3db5c570
new file mode 100644
index 0000000000000000000000000000000000000000..c3b2bd442c1f47cc24db060eb85a842c26f35a30
GIT binary patch
literal 94
zcmZ?wbhEHbRA5kG_`m=Kia%MvEXDuap1FzXslJKnnaSDg>4~|yiM9p|It&a9>>v#c
pOmZ##D^I`WUp!~at?teD_Wb5=d*m_gS?97>r?$Q0p3TZ&4FGT_A0_|*

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ef09f26e0ee61329f84a9f589629a865ae9ee0a6 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ef09f26e0ee61329f84a9f589629a865ae9ee0a6
new file mode 100644
index 0000000000000000000000000000000000000000..2422f7cb3fefac0242fc41a5b1a5aec6c0ffc42e
GIT binary patch
literal 276
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+I;nTd&+nTeU1g^dLSIM~=&**H14
zxHvgDIk|ZFdAPXwxH&m_1$p`S1q1{IxOjwwg#?881q1~CA7J3-P*h;J&B!Ro@c#&d
zI0GXiBO?P6fN>a^7??Q(9oZEN|KDQZ0I9KOcr}Mxt7ju~9b1@W5dXwxfmIPN(o(87
z?B=S$5HRprHAzCIbY)cSjo?*@i4l{IJl=b@V2M$iwB%%lQz37XQmQv>`Bih;XH!$e
soUc2hYL{PrxGHgmz=7v`eG3+7wMt3!aINMsV86*4DiOdwasL0C06X_f9smFU

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/f477da4d7d8ff2066041e1dd5ee4e833b7111a1a b/tensorflow/core/kernels/fuzzing/corpus/decode_png/f477da4d7d8ff2066041e1dd5ee4e833b7111a1a
new file mode 100644
index 0000000000000000000000000000000000000000..2ec0b7ae29cea7c8aa406470f5ad812102ba207b
GIT binary patch
literal 527
zcmZ?wbhEHbRA5kG_^Qmn$H%9jpkQENU~g~l?d_eMoSc@HR#;fr(a|ws!i3qgXD?m4
zbj_MIJ9g~Yw{PEp0|!o>I(6mBm220oUB7<)=FOWA9z1yR<jKpIFJHfY{r2tK_wV0-
z{P^+f*RMZ*{P_9v=dWMCe*gac_wV0-|Ni~||DS;dK=CIF*i^;;+@86K>8ZYn>6yvd
z?CFWQxrw$03_2i5kXISl{vDWD;GrYce_~0|$sDWs7gm&TiAK6!S){@8sl-YyfWw=&
z)?zjf+vJ-Cb_+9CY~C57A;h_~=w`{IfMO{zVF5l~UOoX~F{yfgC3$HvL4F=CPEIZ!
zenBy5`4k^T8F68LZVq-fHg*nfeqnK$1|J1!Q2`zfRu*Pv7FG@(0a0lMUnMz75k4+<
z7A8hUCKh%sJ`qW|MV?Bs5<<M3Y|M-d42;ZdoV-F3vUff1KYaZ3`ODXD-+%Nub>`dx
z&&6rHv(55+^Pb+i`Z|0=;?X`!#Z<1TT4C!FPA+<?a%qXzbfer?Uo@`>JGSsB1|2yO
yxVE$8&4Y`FkDot1IsMX_%+Jr##6<Pt4s=X(f68@H$78Zq?5Qm)gOwZ^7_0$=Rk$qx

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/f8a379b2498a4eb452a85791a49adf065dab59ae b/tensorflow/core/kernels/fuzzing/corpus/decode_png/f8a379b2498a4eb452a85791a49adf065dab59ae
new file mode 100644
index 0000000000000000000000000000000000000000..5431f584cd14d3c2bb0f1cbea21b1ea8a8c505c5
GIT binary patch
literal 666
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+0LTzVkWOY64i**;0d7ui0g}Q0
z0}O&33<?Yi%#4BzOoEKef{g!<Fi0^lFtRc-AOj|57B)_f|3?^f1sIqZ8CjWm;qr`3
z%q$G7Y=T1U9Gt=;5|WBWj$z@`mtDCk`u_-nI3oxko5aA#%=G^VgOVU412Y37GYd0J
zl97Q)keNluP*KUT(J64^|62?^%wRD=27886tChrDissGpTFFzm#i6B=b5dwe%BqNO
z8yV&_o+#gVg29vF=!xE2M|3)NXg*(AENUX;^)T|(qG?v4-i9YCSXm}NU^Lshx@4_y
zM8}qnNa3t1?}FS<&ra=KKL7KYj*@8)Za+Ocwc6kQd8EKg-Gkdt&(``jum0R76)PR#
zgZ;T*%iAoPwx?Y$ndG<Sv!k8D+Czoy9ut^Fna_GHG<kXUafi0qCePZk6VKM}%ss@{
z@S|9iXIt6&trx<UnK50xJ#V%}((G2hKgMo3`T0eQmQ`7m&XQPqWO`=Sb)#c#8e7hV
zScGSCPFR%Lc_8Vj;1}5+bNXie@~91${#GUVDSQ6gTc`O}_x`r!`DuFo+ndh#qjPM^
zFC9$lh?m~~h^w#e-NbsesO-z0u2(AkGIjP$IKAWK62mu=^G>IqJ{>3ZJ-V*7zF{+8
a-lDmd+vQi?y07@7|NZef*AG1Ze-i*g<kPnR

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/fe67bccb06f2174523943cc684518fcf1f7f8046 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/fe67bccb06f2174523943cc684518fcf1f7f8046
new file mode 100644
index 0000000000000000000000000000000000000000..cf043445f4b62caa450c89b2c5ee44f84d837242
GIT binary patch
literal 80
zcmZ?wbhEHbRA5kG_`t}(z`*eT|9{1wEFczx;(u<>+{E-$-^BFH<ZSlz#N6COTLT6i
ekR(V01Cx16|H{*E`4`XGA~x0KnLq#ogEat-P!~}E

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_png/ff1e67d17c1c27ef0d97900d0ea276b563a64628 b/tensorflow/core/kernels/fuzzing/corpus/decode_png/ff1e67d17c1c27ef0d97900d0ea276b563a64628
new file mode 100644
index 0000000000000000000000000000000000000000..b1f8d120a3955c09b390dbaa8cab59ecf063b506
GIT binary patch
literal 271
zcmex=<NpH&0WUXCHwH#VCI%S>83u;`w;7xnn3+HTL^3loGqbR<fB**@8!H<p2NxG7
z2PY>N4?hnVHy<}AC$AtcAHRTrpa2(-kg$+|Fu#C+!2bgb+#C!F3<`{lf(-wUFo-iS
zGBPqUAOki|uK!0Elm!?VSlF1Dk>nVem|0la*ad}zMMT96#sA-8-~eg0XE?Qhfq_BR
z?pBrfr{9roxBWGr{PmhY-DCG28QXPhjB`GHe6!Z<@$J~DJ&)DDgf=iRbnX4R=Gv{B
xRlgp4EPN<GGib}6YvpWLqC}R-)JX{*pY7f0HEFTqN)|VlCG(7%^%wuY2>=w!M9BaE

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict b/tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict
new file mode 100644
index 0000000000..d795ae7f71
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/dictionaries/decode_png.dict
@@ -0,0 +1,50 @@
+header_87a="87a"
+header_89a="89a"
+header_gif="GIF"
+header_jfif="JFIF\x00"
+header_jfxx="JFXX\x00"
+header_png="\x89PNG\x0d\x0a\x1a\x0a"
+marker_2c=","
+marker_3b=";"
+section_2101="!\x01\x12"
+section_21f9="!\xf9\x04"
+section_21fe="!\xfe"
+section_21ff="!\xff\x11"
+section_IDAT="IDAT"
+section_IEND="IEND"
+section_IHDR="IHDR"
+section_PLTE="PLTE"
+section_bKGD="bKGD"
+section_cHRM="cHRM"
+section_fRAc="fRAc"
+section_ffc0="\xff\xc0"
+section_ffc2="\xff\xc2"
+section_ffc4="\xff\xc4"
+section_ffd0="\xff\xd0"
+section_ffd8="\xff\xd8"
+section_ffd9="\xff\xd9"
+section_ffda="\xff\xda"
+section_ffdb="\xff\xdb"
+section_ffdd="\xff\xdd"
+section_ffe0="\xff\xe0"
+section_ffe1="\xff\xe1"
+section_fffe="\xff\xfe"
+section_gAMA="gAMA"
+section_gIFg="gIFg"
+section_gIFt="gIFt"
+section_gIFx="gIFx"
+section_hIST="hIST"
+section_iCCP="iCCP"
+section_iTXt="iTXt"
+section_oFFs="oFFs"
+section_pCAL="pCAL"
+section_pHYs="pHYs"
+section_sBIT="sBIT"
+section_sCAL="sCAL"
+section_sPLT="sPLT"
+section_sRGB="sRGB"
+section_sTER="sTER"
+section_tEXt="tEXt"
+section_tIME="tIME"
+section_tRNS="tRNS"
+section_zTXt="zTXt"
diff --git a/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl b/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl
index ed54abb742..e932213359 100644
--- a/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl
+++ b/tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl
@@ -17,3 +17,9 @@ def tf_oss_fuzz_corpus(name):
         name = name + "_corpus",
         srcs = native.glob(["corpus/" + name + "/*"]),
     )
+
+def tf_oss_fuzz_dict(name):
+    native.filegroup(
+        name = name + "_dict",
+        srcs = native.glob(["dictionaries/" + name + ".dict"]),
+    )
-- 
GitLab


From 41cc1de5cf929a7ec18a76df9a2e3916b19b6b6d Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 29 Nov 2018 14:22:06 -0800
Subject: [PATCH 1060/1554] Support estimator names when generating
 renames_v2.py.

PiperOrigin-RevId: 223410761
---
 tensorflow/tools/compatibility/renames_v2.py  |  2 +-
 .../tools/compatibility/tf_upgrade_v2_test.py | 39 ++++++++----
 .../update/generate_v2_renames_map.py         | 60 +++++++++++++++----
 3 files changed, 78 insertions(+), 23 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index dda7943821..5a27eb241c 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -603,7 +603,7 @@ renames = {
     'tf.train.SingularMonitoredSession': 'tf.compat.v1.train.SingularMonitoredSession',
     'tf.train.Supervisor': 'tf.compat.v1.train.Supervisor',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
-    'tf.train.VocabInfo': 'tf.compat.v1.train.VocabInfo',
+    'tf.train.VocabInfo': 'tf.estimator.VocabInfo',
     'tf.train.WorkerSessionCreator': 'tf.compat.v1.train.WorkerSessionCreator',
     'tf.train.add_queue_runner': 'tf.compat.v1.train.add_queue_runner',
     'tf.train.assert_global_step': 'tf.compat.v1.train.assert_global_step',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 9c334451bf..dbddf170d6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -36,6 +36,32 @@ from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import tf_upgrade_v2
 
 
+_TENSORFLOW_API_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.TENSORFLOW_API_NAME].names)
+_TENSORFLOW_API_ATTR = tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].names
+_ESTIMATOR_API_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].names)
+_ESTIMATOR_API_ATTR = tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].names
+
+
+def get_v1_names(symbol):
+  names_v1 = []
+  if hasattr(symbol, _TENSORFLOW_API_ATTR_V1):
+    names_v1.extend(getattr(symbol, _TENSORFLOW_API_ATTR_V1))
+  if hasattr(symbol, _ESTIMATOR_API_ATTR_V1):
+    names_v1.extend(getattr(symbol, _ESTIMATOR_API_ATTR_V1))
+  return names_v1
+
+
+def get_v2_names(symbol):
+  names_v2 = set()
+  if hasattr(symbol, _TENSORFLOW_API_ATTR):
+    names_v2.update(getattr(symbol, _TENSORFLOW_API_ATTR))
+  if hasattr(symbol, _ESTIMATOR_API_ATTR):
+    names_v2.update(getattr(symbol, _ESTIMATOR_API_ATTR))
+  return list(names_v2)
+
+
 class TestUpgrade(test_util.TensorFlowTestCase):
   """Test various APIs that have been changed in 2.0.
 
@@ -79,32 +105,23 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       return
 
     v2_symbols = set([])
-    attr_v2 = tf_export.API_ATTRS[
-        tf_export.TENSORFLOW_API_NAME].names
 
     def symbol_collector(unused_path, unused_parent, children):
       for child in children:
         _, attr = tf_decorator.unwrap(child[1])
-        if not hasattr(attr, "__dict__"):
-          continue
-        api_names_v2 = attr.__dict__.get(attr_v2, [])
+        api_names_v2 = get_v2_names(attr)
         for name in api_names_v2:
           v2_symbols.add("tf." + name)
 
     visitor = public_api.PublicAPIVisitor(symbol_collector)
     traverse.traverse(tf.compat.v2, visitor)
 
-    attr_v1 = (
-        tf_export.API_ATTRS_V1[tf_export.TENSORFLOW_API_NAME].names)
-
     # Converts all symbols in the v1 namespace to the v2 namespace, raising
     # an error if the target of the conversion is not in the v2 namespace.
     def conversion_visitor(unused_path, unused_parent, children):
       for child in children:
         _, attr = tf_decorator.unwrap(child[1])
-        if not hasattr(attr, "__dict__"):
-          continue
-        api_names = attr.__dict__.get(attr_v1, [])
+        api_names = get_v1_names(attr)
         for name in api_names:
           _, _, _, text = self._upgrade("tf." + name)
           if (text and
diff --git a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
index 554b42c171..19ad6c3a2a 100644
--- a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
+++ b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
@@ -72,6 +72,50 @@ _TENSORFLOW_CONSTANTS_ATTR_V1 = (
 _TENSORFLOW_CONSTANTS_ATTR = (
     tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].constants)
 
+_ESTIMATOR_API_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].names)
+_ESTIMATOR_API_ATTR = tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].names
+_ESTIMATOR_CONSTANTS_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].constants)
+_ESTIMATOR_CONSTANTS_ATTR = (
+    tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].constants)
+
+
+def get_v1_names(symbol):
+  names_v1 = []
+  if hasattr(symbol, _TENSORFLOW_API_ATTR_V1):
+    names_v1.extend(getattr(symbol, _TENSORFLOW_API_ATTR_V1))
+  if hasattr(symbol, _ESTIMATOR_API_ATTR_V1):
+    names_v1.extend(getattr(symbol, _ESTIMATOR_API_ATTR_V1))
+  return names_v1
+
+
+def get_v2_names(symbol):
+  names_v2 = []
+  if hasattr(symbol, _TENSORFLOW_API_ATTR):
+    names_v2.extend(getattr(symbol, _TENSORFLOW_API_ATTR))
+  if hasattr(symbol, _ESTIMATOR_API_ATTR):
+    names_v2.extend(getattr(symbol, _ESTIMATOR_API_ATTR))
+  return list(names_v2)
+
+
+def get_v1_constants(module):
+  constants_v1 = []
+  if hasattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1):
+    constants_v1.extend(getattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1))
+  if hasattr(module, _ESTIMATOR_CONSTANTS_ATTR_V1):
+    constants_v1.extend(getattr(module, _ESTIMATOR_CONSTANTS_ATTR_V1))
+  return constants_v1
+
+
+def get_v2_constants(module):
+  constants_v2 = []
+  if hasattr(module, _TENSORFLOW_CONSTANTS_ATTR):
+    constants_v2.extend(getattr(module, _TENSORFLOW_CONSTANTS_ATTR))
+  if hasattr(module, _ESTIMATOR_CONSTANTS_ATTR):
+    constants_v2.extend(getattr(module, _ESTIMATOR_CONSTANTS_ATTR))
+  return constants_v2
+
 
 def get_canonical_name(v2_names, v1_name):
   if v2_names:
@@ -87,9 +131,7 @@ def get_all_v2_names():
     """Visitor that collects TF 2.0 names."""
     for child in children:
       _, attr = tf_decorator.unwrap(child[1])
-      if not hasattr(attr, '__dict__'):
-        continue
-      api_names_v2 = attr.__dict__.get(_TENSORFLOW_API_ATTR, [])
+      api_names_v2 = get_v2_names(attr)
       for name in api_names_v2:
         v2_names.add(name)
 
@@ -107,10 +149,8 @@ def collect_constant_renames():
   """
   renames = set()
   for module in sys.modules.values():
-    if not hasattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1):
-      continue
-    constants_v1_list = getattr(module, _TENSORFLOW_CONSTANTS_ATTR_V1)
-    constants_v2_list = getattr(module, _TENSORFLOW_CONSTANTS_ATTR)
+    constants_v1_list = get_v1_constants(module)
+    constants_v2_list = get_v2_constants(module)
 
     # _tf_api_constants attribute contains a list of tuples:
     # (api_names_list, constant_name)
@@ -146,10 +186,8 @@ def collect_function_renames():
     """Visitor that collects rename strings to add to rename_line_set."""
     for child in children:
       _, attr = tf_decorator.unwrap(child[1])
-      if not hasattr(attr, '__dict__'):
-        continue
-      api_names_v1 = attr.__dict__.get(_TENSORFLOW_API_ATTR_V1, [])
-      api_names_v2 = attr.__dict__.get(_TENSORFLOW_API_ATTR, [])
+      api_names_v1 = get_v1_names(attr)
+      api_names_v2 = get_v2_names(attr)
       deprecated_api_names = set(api_names_v1) - set(api_names_v2)
       for name in deprecated_api_names:
         renames.add((name, get_canonical_name(api_names_v2, name)))
-- 
GitLab


From f299e057dbb01c45a2b699da2fbd8193e63cb059 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 29 Nov 2018 14:26:27 -0800
Subject: [PATCH 1061/1554] Add make_dataset_iterator implementation to
 parameter server strategy and collective all reduce strategy.

PiperOrigin-RevId: 223411477
---
 .../python/collective_all_reduce_strategy.py      | 15 +++++++++++----
 .../python/parameter_server_strategy.py           |  5 +++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 617a95f3c4..6e9f9facd0 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -24,6 +24,7 @@ from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import cross_device_utils
+from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import values
@@ -80,6 +81,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
       ]
     else:
       local_devices = ["/device:CPU:0"]
+    self._worker_device = device_util.canonicalize("/device:CPU:0")
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     super(CollectiveAllReduceExtended, self).__init__(
@@ -116,14 +118,14 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
                                                 task_id)
 
-    worker_device = "/job:%s/task:%d" % (task_type, task_id)
+    self._worker_device = "/job:%s/task:%d" % (task_type, task_id)
     if num_gpus_per_worker:
       local_devices = [
-          "%s/device:GPU:%d" % (worker_device, i)
+          "%s/device:GPU:%d" % (self._worker_device, i)
           for i in range(num_gpus_per_worker)
       ]
     else:
-      local_devices = [worker_device]
+      local_devices = [self._worker_device]
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     super(CollectiveAllReduceExtended, self).__init__(
@@ -222,6 +224,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._devices, True)
 
+  def _make_dataset_iterator(self, dataset):
+    worker_device_pairs = [(self._worker_device, self._devices)]
+    return values.DatasetIterator(dataset, worker_device_pairs,
+                                  self._num_replicas_in_sync)
+
   def _make_input_fn_iterator(
       self,
       input_fn,
@@ -238,7 +245,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
         num_replicas_in_sync=self._num_replicas_in_sync)
 
     return values.InputFunctionIterator(
-        input_fn, [(self._default_device, self._devices)], [input_context])
+        input_fn, [(self._worker_device, self._devices)], [input_context])
 
   def _configure(self,
                  session_config=None,
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 75ee41c4cf..eaeb4d7030 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -237,6 +237,11 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
     return values.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._compute_devices, True)
 
+  def _make_dataset_iterator(self, dataset):
+    worker_device_pairs = [(self._worker_device, self._compute_devices)]
+    return values.DatasetIterator(dataset, worker_device_pairs,
+                                  self._num_replicas_in_sync)
+
   def _make_input_fn_iterator(
       self,
       input_fn,
-- 
GitLab


From cac337447456cf0c0a2d3a362d02f4dbccf13bb3 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Thu, 29 Nov 2018 14:28:01 -0800
Subject: [PATCH 1062/1554] Change underlying definitions of TFLite constants
 to be tf.dtypes.

PiperOrigin-RevId: 223411751
---
 tensorflow/lite/python/BUILD                  |  2 +
 tensorflow/lite/python/convert.py             | 41 +++++++++++++++++--
 tensorflow/lite/python/convert_test.py        | 22 ++++++++++
 tensorflow/lite/python/lite.py                |  6 +--
 tensorflow/lite/python/lite_constants.py      | 25 ++++++-----
 tensorflow/lite/python/tflite_convert.py      | 33 ++++++++++++---
 .../golden/v1/tensorflow.lite.constants.pbtxt | 10 ++---
 .../golden/v2/tensorflow.lite.constants.pbtxt | 20 ---------
 8 files changed, 107 insertions(+), 52 deletions(-)

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 017dd72f78..acf827892b 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -89,6 +89,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/lite/toco:toco_flags_proto_py",
+        "//tensorflow/python:dtypes",
     ],
 )
 
@@ -103,6 +104,7 @@ py_library(
         "//tensorflow/lite/toco:toco_flags_proto_py",
         "//tensorflow/lite/toco/python:tensorflow_wrap_toco",
         "//tensorflow/lite/toco/python:toco_from_protos",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:platform",
     ],
 )
diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 9991fb2a73..756c9daabe 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -28,6 +28,8 @@ import tempfile as _tempfile
 from tensorflow.lite.python import lite_constants
 from tensorflow.lite.toco import model_flags_pb2 as _model_flags_pb2
 from tensorflow.lite.toco import toco_flags_pb2 as _toco_flags_pb2
+from tensorflow.lite.toco import types_pb2 as _types_pb2
+from tensorflow.python.framework import dtypes
 from tensorflow.python.platform import resource_loader as _resource_loader
 from tensorflow.python.util import deprecation
 from tensorflow.python.util.lazy_loader import LazyLoader
@@ -53,6 +55,18 @@ else:
 if _toco_from_proto_bin and not _os.path.exists(_toco_from_proto_bin):
   _toco_from_proto_bin = "toco_from_protos"
 
+
+# Map of tf.dtypes to TFLite types_flag_pb2.
+_MAP_TF_TO_TFLITE_TYPES = {
+    dtypes.float32: _types_pb2.FLOAT,
+    dtypes.int32: _types_pb2.INT32,
+    dtypes.int64: _types_pb2.INT64,
+    dtypes.string: _types_pb2.STRING,
+    dtypes.uint8: _types_pb2.QUANTIZED_UINT8,
+    dtypes.complex64: _types_pb2.COMPLEX64
+}
+
+
 def _try_convert_to_unicode(output):
   if output is None:
     return u""
@@ -65,6 +79,24 @@ def _try_convert_to_unicode(output):
   return output
 
 
+def convert_dtype_to_tflite_type(tf_dtype):
+  """Converts tf.dtype to TFLite proto type.
+
+  Args:
+    tf_dtype: tf.dtype
+
+  Raises:
+    ValueError: Unsupported tf.dtype.
+
+  Returns:
+    types_flag_pb2.
+  """
+  result = _MAP_TF_TO_TFLITE_TYPES.get(tf_dtype)
+  if result is None:
+    raise ValueError("Unsupported tf.dtype {0}".format(tf_dtype))
+  return result
+
+
 class OpsSet(enum.Enum):
   """Enum class defining the sets of ops available to generate TFLite models.
 
@@ -214,10 +246,10 @@ def build_toco_convert_protos(input_tensors,
       `foo.get_shape()` and `foo.dtype`.
     output_tensors: List of output tensors (only .name is used from this).
     inference_type: Target data type of real-number arrays in the output file.
-      Must be `{FLOAT, QUANTIZED_UINT8}`.  (default FLOAT)
+      Must be `{tf.float32, tf.uint8}`.  (default tf.float32)
     inference_input_type: Target data type of real-number input arrays. Allows
       for a different type for input arrays in the case of quantization.
-      Must be `{FLOAT, QUANTIZED_UINT8}`. (default `inference_type`)
+      Must be `{tf.float32, tf.uint8}`. (default `inference_type`)
     input_format: Type of data to read Currently must be
       `{TENSORFLOW_GRAPHDEF}`. (default TENSORFLOW_GRAPHDEF)
     input_shapes: Input array shape. It needs to be a list of the same length
@@ -276,9 +308,10 @@ def build_toco_convert_protos(input_tensors,
   toco = _toco_flags_pb2.TocoFlags()
   toco.input_format = input_format
   toco.output_format = output_format
-  toco.inference_type = inference_type
+  toco.inference_type = convert_dtype_to_tflite_type(inference_type)
   if inference_input_type:
-    toco.inference_input_type = inference_input_type
+    toco.inference_input_type = convert_dtype_to_tflite_type(
+        inference_input_type)
   else:
     toco.inference_input_type = toco.inference_type
   toco.drop_control_dependency = drop_control_dependency
diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index 7a0bce921b..40576e16db 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -23,6 +23,7 @@ from tensorflow.lite.python import convert
 from tensorflow.lite.python import lite_constants
 from tensorflow.lite.python import op_hint
 from tensorflow.lite.python.interpreter import Interpreter
+from tensorflow.lite.toco import types_pb2 as _types_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
@@ -329,6 +330,27 @@ class ConvertTestOpHint(test_util.TensorFlowTestCase):
               output_nodes=[op_hint._tensor_name_base(output.name)]),
           set(["agg", "Const", "Identity"]))
 
+  def testConvertDtype(self):
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(lite_constants.FLOAT),
+        _types_pb2.FLOAT)
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(dtypes.float32), _types_pb2.FLOAT)
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(dtypes.int32), _types_pb2.INT32)
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(dtypes.int64), _types_pb2.INT64)
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(dtypes.string), _types_pb2.STRING)
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(dtypes.uint8),
+        _types_pb2.QUANTIZED_UINT8)
+    self.assertEqual(
+        convert.convert_dtype_to_tflite_type(dtypes.complex64),
+        _types_pb2.COMPLEX64)
+    with self.assertRaises(ValueError):
+      convert.convert_dtype_to_tflite_type(dtypes.bool)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py
index 5810553da2..1fb5618466 100644
--- a/tensorflow/lite/python/lite.py
+++ b/tensorflow/lite/python/lite.py
@@ -25,8 +25,6 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice.
 @@convert_op_hints_to_stubs
 @@build_toco_convert_protos
 
-@@FLOAT
-@@QUANTIZED_UINT8
 @@TFLITE
 @@GRAPHVIZ_DOT
 
@@ -78,10 +76,10 @@ class TFLiteConverter(object):
   Attributes:
 
     inference_type: Target data type of real-number arrays in the output file.
-      Must be `{FLOAT, QUANTIZED_UINT8}`.  (default FLOAT)
+      Must be `{tf.float32, tf.uint8}`. (default tf.float32)
     inference_input_type: Target data type of real-number input arrays. Allows
       for a different type for input arrays in the case of quantization.
-      Must be `{FLOAT, QUANTIZED_UINT8}`. (default `inference_type`)
+      Must be `{tf.float32, tf.uint8}`. (default `inference_type`)
     output_format: Output file format. Currently must be `{TFLITE,
       GRAPHVIZ_DOT}`. (default TFLITE)
     quantized_input_stats: Dict of strings representing input tensor names
diff --git a/tensorflow/lite/python/lite_constants.py b/tensorflow/lite/python/lite_constants.py
index fdefc5e6cf..f5d6d10379 100644
--- a/tensorflow/lite/python/lite_constants.py
+++ b/tensorflow/lite/python/lite_constants.py
@@ -19,26 +19,25 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.lite.toco import toco_flags_pb2 as _toco_flags_pb2
-from tensorflow.lite.toco import types_pb2 as _types_pb2
+from tensorflow.python.framework import dtypes
 from tensorflow.python.util.all_util import remove_undocumented
 from tensorflow.python.util.tf_export import tf_export as _tf_export
 
-# Enum types from the protobuf promoted to the API
-FLOAT = _types_pb2.FLOAT
-INT32 = _types_pb2.INT32
-INT64 = _types_pb2.INT64
-STRING = _types_pb2.STRING
-QUANTIZED_UINT8 = _types_pb2.QUANTIZED_UINT8
-COMPLEX64 = _types_pb2.COMPLEX64
+FLOAT = dtypes.float32
+INT32 = dtypes.int32
+INT64 = dtypes.int64
+STRING = dtypes.string
+QUANTIZED_UINT8 = dtypes.uint8
+COMPLEX64 = dtypes.complex64
 TENSORFLOW_GRAPHDEF = _toco_flags_pb2.TENSORFLOW_GRAPHDEF
 TFLITE = _toco_flags_pb2.TFLITE
 GRAPHVIZ_DOT = _toco_flags_pb2.GRAPHVIZ_DOT
 
-_tf_export("lite.constants.FLOAT").export_constant(__name__, "FLOAT")
-_tf_export("lite.constants.INT32").export_constant(__name__, "INT32")
-_tf_export("lite.constants.INT64").export_constant(__name__, "INT64")
-_tf_export("lite.constants.STRING").export_constant(__name__, "STRING")
-_tf_export("lite.constants.QUANTIZED_UINT8").export_constant(
+_tf_export(v1=["lite.constants.FLOAT"]).export_constant(__name__, "FLOAT")
+_tf_export(v1=["lite.constants.INT32"]).export_constant(__name__, "INT32")
+_tf_export(v1=["lite.constants.INT64"]).export_constant(__name__, "INT64")
+_tf_export(v1=["lite.constants.STRING"]).export_constant(__name__, "STRING")
+_tf_export(v1=["lite.constants.QUANTIZED_UINT8"]).export_constant(
     __name__, "QUANTIZED_UINT8")
 _tf_export("lite.constants.TFLITE").export_constant(__name__, "TFLITE")
 _tf_export("lite.constants.GRAPHVIZ_DOT").export_constant(
diff --git a/tensorflow/lite/python/tflite_convert.py b/tensorflow/lite/python/tflite_convert.py
index 00ea6d722e..341b539bea 100644
--- a/tensorflow/lite/python/tflite_convert.py
+++ b/tensorflow/lite/python/tflite_convert.py
@@ -25,7 +25,6 @@ import sys
 from tensorflow.lite.python import lite
 from tensorflow.lite.python import lite_constants
 from tensorflow.lite.toco import toco_flags_pb2 as _toco_flags_pb2
-from tensorflow.lite.toco import types_pb2 as _types_pb2
 from tensorflow.python.platform import app
 
 
@@ -41,6 +40,27 @@ def _parse_set(values):
   return None
 
 
+def _parse_inference_type(value, flag):
+  """Converts the inference type to the value of the constant.
+
+  Args:
+    value: str representing the inference type.
+    flag: str representing the flag name.
+
+  Returns:
+    tf.dtype.
+
+  Raises:
+    ValueError: Unsupported value.
+  """
+  if value == "FLOAT":
+    return lite_constants.FLOAT
+  if value == "QUANTIZED_UINT8":
+    return lite_constants.QUANTIZED_UINT8
+  raise ValueError("Unsupported value for --{0}. Only FLOAT and "
+                   "QUANTIZED_UINT8 are supported.".format(flag))
+
+
 def _get_toco_converter(flags):
   """Makes a TFLiteConverter object based on the flags provided.
 
@@ -101,10 +121,11 @@ def _convert_model(flags):
   # Create converter.
   converter = _get_toco_converter(flags)
   if flags.inference_type:
-    converter.inference_type = _types_pb2.IODataType.Value(flags.inference_type)
+    converter.inference_type = _parse_inference_type(flags.inference_type,
+                                                     "inference_type")
   if flags.inference_input_type:
-    converter.inference_input_type = _types_pb2.IODataType.Value(
-        flags.inference_input_type)
+    converter.inference_input_type = _parse_inference_type(
+        flags.inference_input_type, "inference_input_type")
   if flags.output_format:
     converter.output_format = _toco_flags_pb2.FileFormat.Value(
         flags.output_format)
@@ -115,7 +136,7 @@ def _convert_model(flags):
 
     # In quantized inference, mean_value has to be integer so that the real
     # value 0.0 is exactly representable.
-    if flags.inference_type == lite_constants.QUANTIZED_UINT8:
+    if converter.inference_type == lite_constants.QUANTIZED_UINT8:
       mean_values = _parse_array(flags.mean_values, type_fn=int)
     else:
       mean_values = _parse_array(flags.mean_values, type_fn=float)
@@ -156,7 +177,7 @@ def _convert_model(flags):
 
   if flags.post_training_quantize:
     converter.post_training_quantize = flags.post_training_quantize
-    if flags.inference_type == lite_constants.QUANTIZED_UINT8:
+    if converter.inference_type == lite_constants.QUANTIZED_UINT8:
       print("--post_training_quantize quantizes a graph of inference_type "
             "FLOAT. Overriding inference type QUANTIZED_UINT8 to FLOAT.")
       converter.inference_type = lite_constants.FLOAT
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt
index 08845553e5..ef6c777665 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.lite.constants.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.lite.constants"
 tf_module {
   member {
     name: "FLOAT"
-    mtype: "<type \'int\'>"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
   member {
     name: "GRAPHVIZ_DOT"
@@ -10,19 +10,19 @@ tf_module {
   }
   member {
     name: "INT32"
-    mtype: "<type \'int\'>"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
   member {
     name: "INT64"
-    mtype: "<type \'int\'>"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
   member {
     name: "QUANTIZED_UINT8"
-    mtype: "<type \'int\'>"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
   member {
     name: "STRING"
-    mtype: "<type \'int\'>"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
   member {
     name: "TFLITE"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt
index 08845553e5..4d5c4893b4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.lite.constants.pbtxt
@@ -1,29 +1,9 @@
 path: "tensorflow.lite.constants"
 tf_module {
-  member {
-    name: "FLOAT"
-    mtype: "<type \'int\'>"
-  }
   member {
     name: "GRAPHVIZ_DOT"
     mtype: "<type \'int\'>"
   }
-  member {
-    name: "INT32"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "INT64"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "QUANTIZED_UINT8"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "STRING"
-    mtype: "<type \'int\'>"
-  }
   member {
     name: "TFLITE"
     mtype: "<type \'int\'>"
-- 
GitLab


From 45573542df95999110be0690175bb82b2ed64ded Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Thu, 29 Nov 2018 14:35:25 -0800
Subject: [PATCH 1063/1554] [XLA] Add support for iota of PRED

PiperOrigin-RevId: 223413074
---
 .../xla/service/elemental_ir_emitter.cc       |  6 ++++--
 tensorflow/compiler/xla/tests/iota_test.cc    | 21 +++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 00bb430206..6f1f95f2e9 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -2245,13 +2245,15 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
                 : iota->shape();
         PrimitiveType component_element_type = component_shape.element_type();
         llvm::Value* iota_result;
-        if (ShapeUtil::ElementIsIntegral(component_shape)) {
+        if (primitive_util::IsIntegralType(component_element_type) ||
+            component_element_type == PRED) {
           iota_result = b_->CreateIntCast(
               elem_index_linear,
               llvm_ir::PrimitiveTypeToIrType(component_element_type, module_),
               /*isSigned=*/false);
         } else {
-          TF_RET_CHECK(ShapeUtil::ElementIsFloating(component_shape))
+          TF_RET_CHECK(
+              primitive_util::IsFloatingPointType(component_element_type))
               << component_element_type;
           llvm::Type* float_ir_type;
           if (component_element_type == BF16) {
diff --git a/tensorflow/compiler/xla/tests/iota_test.cc b/tensorflow/compiler/xla/tests/iota_test.cc
index 310f349592..65205f53dd 100644
--- a/tensorflow/compiler/xla/tests/iota_test.cc
+++ b/tensorflow/compiler/xla/tests/iota_test.cc
@@ -113,5 +113,26 @@ INSTANTIATE_TEST_CASE_P(IotaR3TestInstantiation, IotaR3Test,
                                                             /*step=*/10),
                                            ::testing::Values(0, 1, 2)));
 
+class IotaR3PredTest : public ClientLibraryTestBase,
+                       public ::testing::WithParamInterface<int> {};
+
+TEST_P(IotaR3PredTest, DoIt) {
+  const auto element_type = PRED;
+  const int64 num_elements = 2;
+  const int64 iota_dim = GetParam();
+  XlaBuilder builder(TestName() + "_" + PrimitiveType_Name(element_type));
+  std::vector<int64> dimensions = {42, 19};
+  dimensions.insert(dimensions.begin() + iota_dim, num_elements);
+  Iota(&builder, ShapeUtil::MakeShape(element_type, dimensions), iota_dim);
+  if (primitive_util::IsFloatingPointType(element_type)) {
+    ComputeAndCompare(&builder, {}, ErrorSpec{0.0001});
+  } else {
+    ComputeAndCompare(&builder, {});
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(IotaR3PredTestInstantiation, IotaR3PredTest,
+                        ::testing::Values(0, 1, 2));
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 18944dd7210dc2f1a13a34a6946b8a08084cda76 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 29 Nov 2018 14:45:44 -0800
Subject: [PATCH 1064/1554] [TF:XLA] Bump open source llvm revision to r347843

PiperOrigin-RevId: 223414920
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 016572a543..dde10ef73d 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "6c9b98f745b1cf2a9ef34f6220d0f94620ee5c828151924299913f1822cd40d1",
-        strip_prefix = "llvm-9d5ac66dc1ff01dee56354344c9da0879f1bdc36",
+        sha256 = "f4791ba3e166918bca82df34e2f854e8e188d6055888c64cb28743fd43f2d0d7",
+        strip_prefix = "llvm-b2a42b2112a511a5077fd747fb21e45349cff08d",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9d5ac66dc1ff01dee56354344c9da0879f1bdc36.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/9d5ac66dc1ff01dee56354344c9da0879f1bdc36.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/b2a42b2112a511a5077fd747fb21e45349cff08d.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/b2a42b2112a511a5077fd747fb21e45349cff08d.tar.gz",
         ],
     )
 
-- 
GitLab


From 84f93b879e99b05bf92e610da75e4ed84c2a01ba Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 29 Nov 2018 15:00:18 -0800
Subject: [PATCH 1065/1554] [tf.data] Nesting `tf.data.Options()` optimization
 options under `experimental_optimization`.

PiperOrigin-RevId: 223417385
---
 .../python/data/experimental/__init__.py      |  5 +-
 .../kernel_tests/optimization/BUILD           |  9 ++
 .../optimization/filter_fusion_test.py        |  4 +-
 .../optimization/hoist_random_uniform_test.py |  7 +-
 .../optimization/map_and_batch_fusion_test.py |  4 +-
 .../map_and_filter_fusion_test.py             |  7 +-
 .../optimization/map_fusion_test.py           |  4 +-
 .../optimization/map_parallelization_test.py  |  4 +-
 .../optimization/map_vectorization_test.py    |  4 +-
 .../optimization/noop_elimination_test.py     |  4 +-
 .../shuffle_and_repeat_fusion_test.py         |  4 +-
 tensorflow/python/data/experimental/ops/BUILD | 10 ++
 .../experimental/ops/optimization_options.py  | 83 +++++++++++++++++
 .../data/experimental/ops/stats_options.py    |  2 +-
 .../experimental/ops/threading_options.py     |  2 +-
 tensorflow/python/data/kernel_tests/BUILD     |  1 +
 .../python/data/kernel_tests/dataset_test.py  |  8 +-
 .../multi_device_iterator_test.py             |  4 +-
 tensorflow/python/data/ops/BUILD              |  1 +
 tensorflow/python/data/ops/dataset_ops.py     | 91 ++++++-------------
 .../golden/v1/tensorflow.data.-options.pbtxt  | 34 +------
 ...a.experimental.-optimization-options.pbtxt | 46 ++++++++++
 .../v1/tensorflow.data.experimental.pbtxt     |  4 +
 .../golden/v2/tensorflow.data.-options.pbtxt  | 34 +------
 ...a.experimental.-optimization-options.pbtxt | 46 ++++++++++
 .../v2/tensorflow.data.experimental.pbtxt     |  4 +
 26 files changed, 274 insertions(+), 152 deletions(-)
 create mode 100644 tensorflow/python/data/experimental/ops/optimization_options.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt

diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 8cec75b599..8a1048513a 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -25,6 +25,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@Counter
 @@CheckpointInputPipelineHook
 @@CsvDataset
+@@OptimizationOptions
 @@Optional
 @@RandomDataset
 @@Reducer
@@ -86,10 +87,8 @@ from tensorflow.python.data.experimental.ops.interleave_ops import parallel_inte
 from tensorflow.python.data.experimental.ops.interleave_ops import sample_from_datasets
 from tensorflow.python.data.experimental.ops.iterator_ops import CheckpointInputPipelineHook
 from tensorflow.python.data.experimental.ops.iterator_ops import make_saveable_from_iterator
-
-# Optimization constant that can be used to enable auto-tuning.
 from tensorflow.python.data.experimental.ops.optimization import AUTOTUNE
-
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.experimental.ops.parsing_ops import parse_example_dataset
 from tensorflow.python.data.experimental.ops.prefetching_ops import copy_to_device
 from tensorflow.python.data.experimental.ops.prefetching_ops import prefetch_to_device
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 121798ad3e..e05f382171 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -42,6 +42,7 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "@absl_py//absl/testing:parameterized",
@@ -68,6 +69,7 @@ py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "@absl_py//absl/testing:parameterized",
@@ -127,6 +129,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
@@ -148,6 +151,7 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "@absl_py//absl/testing:parameterized",
@@ -167,6 +171,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "@absl_py//absl/testing:parameterized",
@@ -192,6 +197,7 @@ py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "@absl_py//absl/testing:parameterized",
@@ -227,6 +233,7 @@ py_test(
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
@@ -272,6 +279,7 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
@@ -313,6 +321,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
index fe2c104e94..7371cf31df 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -71,7 +72,8 @@ class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset.cache()
     options = dataset_ops.Options()
-    options.experimental_filter_fusion = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.filter_fusion = True
     dataset = dataset.with_options(options)
     expected_output = []
     for x in range(5):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
index e86b19438e..0aacf8bb07 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
@@ -91,7 +92,8 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
             ["Zip[0]", "Map"] if will_optimize else ["Map"])).map(function)
 
     options = dataset_ops.Options()
-    options.experimental_hoist_random_uniform = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.hoist_random_uniform = True
     dataset = dataset.with_options(options)
     self._testDataset(dataset)
 
@@ -107,7 +109,8 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.range(5).apply(
         optimization.assert_next(["Zip[0]", "Map"])).map(random_with_capture)
     options = dataset_ops.Options()
-    options.experimental_hoist_random_uniform = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.hoist_random_uniform = True
     dataset = dataset.with_options(options)
     self._testDataset(dataset)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
index 67f3ceeabe..801f664f09 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import test_util
@@ -32,7 +33,8 @@ class MapAndBatchFusionTest(test_base.DatasetTestBase):
         optimization.assert_next(
             ["MapAndBatch"])).map(lambda x: x * x).batch(10)
     options = dataset_ops.Options()
-    options.experimental_map_and_batch_fusion = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.map_and_batch_fusion = True
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset, expected_output=[[x * x for x in range(10)]])
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
index a898c38440..db8f214fbf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -83,7 +84,8 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(
             ["Map", "FilterByLastComponent"])).map(function).filter(predicate)
     options = dataset_ops.Options()
-    options.experimental_map_and_filter_fusion = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.map_and_filter_fusion = True
     dataset = dataset.with_options(options)
     self._testMapAndFilter(dataset, function, predicate)
 
@@ -101,7 +103,8 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(["Map",
                                   "Filter"])).map(function).filter(predicate)
     options = dataset_ops.Options()
-    options.experimental_map_and_filter_fusion = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.map_and_filter_fusion = True
     dataset = dataset.with_options(options)
     self._testMapAndFilter(dataset, function, predicate)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
index 47a1b0896c..d8d6390374 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import test_util
@@ -74,7 +75,8 @@ class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset.cache()
     options = dataset_ops.Options()
-    options.experimental_map_fusion = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.map_fusion = True
     dataset = dataset.with_options(options)
     expected_output = []
     for x in range(5):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
index 042b9ce54b..0ff3fff4f8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
@@ -67,7 +68,8 @@ class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.range(5).apply(
         optimization.assert_next(next_nodes)).map(function)
     options = dataset_ops.Options()
-    options.experimental_map_parallelization = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.map_parallelization = True
     dataset = dataset.with_options(options)
     if should_optimize:
       self.assertDatasetProduces(
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 4f05f02669..d979aaa5a0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -353,7 +354,8 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     optimized = _make_dataset(["Batch", map_node_name]
                               if expect_optimized else [map_node_name, "Batch"])
     options = dataset_ops.Options()
-    options.experimental_map_vectorization = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.map_vectorization = True
     optimized = optimized.with_options(options)
     return unoptimized, optimized
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
index d957e8007c..ce86bfa4e0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -42,7 +43,8 @@ class NoopEliminationTest(test_base.DatasetTestBase):
     dataset = dataset.repeat(some_tensor).skip(5).take(-1).skip(0).repeat(
         1).prefetch(0).prefetch(1).cache()
     options = dataset_ops.Options()
-    options.experimental_noop_elimination = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.noop_elimination = True
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=range(5))
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
index f1d00a59c4..5f746ec63a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
@@ -32,7 +33,8 @@ class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(["ShuffleAndRepeat"])).shuffle(10).repeat(2)
     options = dataset_ops.Options()
-    options.experimental_shuffle_and_repeat_fusion = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.shuffle_and_repeat_fusion = True
     dataset = dataset.with_options(options)
     get_next = self.getNext(dataset)
 
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index f85e774887..50f5127833 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -235,6 +235,16 @@ py_library(
     ],
 )
 
+py_library(
+    name = "optimization_options",
+    srcs = ["optimization_options.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/util:options",
+    ],
+)
+
 py_library(
     name = "parsing_ops",
     srcs = ["parsing_ops.py"],
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
new file mode 100644
index 0000000000..dc9d319374
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -0,0 +1,83 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Experimental API for controlling optimizations in `tf.data` pipelines."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.data.util import options
+from tensorflow.python.util.tf_export import tf_export
+
+
+@tf_export("data.experimental.OptimizationOptions")
+class OptimizationOptions(options.OptionsBase):
+  """Represents options for dataset optimizations.
+
+  You can apply `OptimizationOptions` to a `dataset` object, as follows:
+
+  ```python
+  options = tf.data.Options()
+  options.optimization = tf.data.experimental.OptimizationOptions()
+  options.optimization.map_and_batch_fusion = True
+  dataset = dataset.with_options(options)
+  ```
+  """
+
+  filter_fusion = options.create_option(
+      name="filter_fusion",
+      ty=bool,
+      docstring="Whether to fuse filter transformations.")
+
+  hoist_random_uniform = options.create_option(
+      name="hoist_random_uniform",
+      ty=bool,
+      docstring=
+      "Whether to hoist `tf.random_uniform()` ops out of map transformations.")
+
+  map_and_batch_fusion = options.create_option(
+      name="map_and_batch_fusion",
+      ty=bool,
+      docstring="Whether to fuse map and batch transformations.")
+
+  map_and_filter_fusion = options.create_option(
+      name="map_and_filter_fusion",
+      ty=bool,
+      docstring="Whether to fuse map and filter transformations.")
+
+  map_fusion = options.create_option(
+      name="map_and_filter_fusion",
+      ty=bool,
+      docstring="Whether to fuse map transformations.")
+
+  map_parallelization = options.create_option(
+      name="map_parallelization",
+      ty=bool,
+      docstring="Whether to parallelize stateless map transformations.")
+
+  map_vectorization = options.create_option(
+      name="map_vectorization",
+      ty=bool,
+      docstring="Whether to vectorize map transformations.")
+
+  noop_elimination = options.create_option(
+      name="noop_elimination",
+      ty=bool,
+      docstring="Whether to eliminate no-op transformations.")
+
+  shuffle_and_repeat_fusion = options.create_option(
+      name="shuffle_and_repeat_fusion",
+      ty=bool,
+      docstring="Whether to fuse shuffle and repeat transformations.")
diff --git a/tensorflow/python/data/experimental/ops/stats_options.py b/tensorflow/python/data/experimental/ops/stats_options.py
index cd7fdcb723..6df608c608 100644
--- a/tensorflow/python/data/experimental/ops/stats_options.py
+++ b/tensorflow/python/data/experimental/ops/stats_options.py
@@ -34,7 +34,7 @@ class StatsOptions(options.OptionsBase):
   ```python
   aggretator = tf.data.experimental.StatsAggregator()
 
-  options = dataset_ops.Options()
+  options = tf.data.Options()
   options.experimental_stats = tf.data.experimental.StatsOptions()
   options.experimental_stats.aggregator = aggregator
   dataset = dataset.with_options(options)
diff --git a/tensorflow/python/data/experimental/ops/threading_options.py b/tensorflow/python/data/experimental/ops/threading_options.py
index 98df371c25..dbf662186f 100644
--- a/tensorflow/python/data/experimental/ops/threading_options.py
+++ b/tensorflow/python/data/experimental/ops/threading_options.py
@@ -29,7 +29,7 @@ class ThreadingOptions(options.OptionsBase):
   To apply `ThreadingOptions` to a `dataset` object, use the following pattern:
 
   ```python
-  options = dataset_ops.Options()
+  options = tf.data.Options()
   options.experimental_threading = tf.data.experimental.ThreadingOptions()
   options.experimental_threading.private_threadpool_size = 10
   dataset = dataset.with_options(options)
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 0867471d74..9f7ce99cbc 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -408,6 +408,7 @@ cuda_py_test(
         "//tensorflow/python/data/ops:multi_device_iterator_ops",
         "//tensorflow/python/data/ops:iterator_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
diff --git a/tensorflow/python/data/kernel_tests/dataset_test.py b/tensorflow/python/data/kernel_tests/dataset_test.py
index 7dbab60f9c..2952c08be0 100644
--- a/tensorflow/python/data/kernel_tests/dataset_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_test.py
@@ -227,12 +227,12 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options1 = dataset_ops.Options()
     options1.experimental_autotune = True
     options2 = dataset_ops.Options()
-    options2.experimental_filter_fusion = False
+    options2.experimental_deterministic = False
     ds = dataset_ops.Dataset.range(0).with_options(options1).with_options(
         options2)
     self.assertTrue(ds.options().experimental_autotune)
     # Explicitly check that flag is False since assertFalse allows None
-    self.assertIs(ds.options().experimental_filter_fusion, False)
+    self.assertIs(ds.options().experimental_deterministic, False)
 
   def testOptionsTwiceDifferentError(self):
     options1 = dataset_ops.Options()
@@ -247,12 +247,12 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options1 = dataset_ops.Options()
     options1.experimental_autotune = True
     options2 = dataset_ops.Options()
-    options2.experimental_filter_fusion = True
+    options2.experimental_deterministic = True
     ds = dataset_ops.Dataset.zip(
         (dataset_ops.Dataset.range(0).with_options(options1),
          dataset_ops.Dataset.range(0).with_options(options2)))
     self.assertTrue(ds.options().experimental_autotune)
-    self.assertTrue(ds.options().experimental_filter_fusion)
+    self.assertTrue(ds.options().experimental_deterministic)
 
   # TODO(b/119882922): use-after-free bug in eager mode.
   # pylint: disable=g-long-lambda
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 886c9acc03..622ebb55de 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import multi_device_iterator_ops
@@ -265,7 +266,8 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
     dataset = dataset.cache()
 
     options = dataset_ops.Options()
-    options.experimental_noop_elimination = True
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.noop_elimination = True
     dataset = dataset.with_options(options)
 
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD
index 27c9175ccb..0c5acda180 100644
--- a/tensorflow/python/data/ops/BUILD
+++ b/tensorflow/python/data/ops/BUILD
@@ -27,6 +27,7 @@ py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python/data/experimental/ops:filter_for_shard_ops",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/experimental/ops:stats_options",
         "//tensorflow/python/data/experimental/ops:threading_options",
         "//tensorflow/python/data/util:nest",
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 51123aaf44..c57ddab15f 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -27,6 +27,7 @@ import six
 
 from tensorflow.python.compat import compat
 from tensorflow.python.data.experimental.ops import filter_for_shard_ops
+from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.experimental.ops import stats_options
 from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.ops import iterator_ops
@@ -1587,56 +1588,15 @@ class Options(options_lib.OptionsBase):
       "Whether to dynamically adjust the values of tunable parameters (e.g. "
       "degrees of parallelism).")
 
-  experimental_filter_fusion = options_lib.create_option(
-      name="experimental_filter_fusion",
-      ty=bool,
-      docstring="Whether to fuse filter transformations.")
-
-  experimental_hoist_random_uniform = options_lib.create_option(
-      name="experimental_hoist_random_uniform",
-      ty=bool,
-      docstring=
-      "Whether to hoist `tf.random_uniform()` ops out of map transformations.")
-
-  experimental_map_and_batch_fusion = options_lib.create_option(
-      name="experimental_map_and_batch_fusion",
-      ty=bool,
-      docstring="Whether to fuse map and batch transformations.")
-
-  experimental_map_and_filter_fusion = options_lib.create_option(
-      name="experimental_map_and_filter_fusion",
-      ty=bool,
-      docstring="Whether to fuse map and filter transformations.")
-
-  experimental_map_fusion = options_lib.create_option(
-      name="experimental_map_and_filter_fusion",
-      ty=bool,
-      docstring="Whether to fuse map transformations.")
-
-  experimental_map_parallelization = options_lib.create_option(
-      name="experimental_map_parallelization",
-      ty=bool,
-      docstring="Whether to parallelize stateless map transformations.")
-
-  experimental_map_vectorization = options_lib.create_option(
-      name="experimental_map_vectorization",
-      ty=bool,
-      docstring="Whether to vectorize map transformations.")
-
-  experimental_noop_elimination = options_lib.create_option(
-      name="experimental_noop_elimination",
-      ty=bool,
-      docstring="Whether to eliminate no-op transformations.")
-
   experimental_numa_aware = options_lib.create_option(
       name="experimental_numa_aware",
       ty=bool,
       docstring="Whether to use NUMA-aware operations.")
 
-  experimental_shuffle_and_repeat_fusion = options_lib.create_option(
-      name="experimental_shuffle_and_repeat_fusion",
-      ty=bool,
-      docstring="Whether to fuse shuffle and repeat transformations.")
+  experimental_optimization = options_lib.create_option(
+      name="experimental_optimization",
+      ty=optimization_options.OptimizationOptions,
+      docstring="Associates the given optimization options with the dataset.")
 
   experimental_stats = options_lib.create_option(
       name="experimental_stats",
@@ -1650,29 +1610,30 @@ class Options(options_lib.OptionsBase):
 
   def _static_optimizations(self):
     """Produces the list of enabled static optimizations."""
-    experimental_optimizations = [
-        "filter_fusion",
-        "hoist_random_uniform",
-        "map_and_batch_fusion",
-        "map_and_filter_fusion",
-        "map_fusion",
-        "map_parallelization",
-        "map_vectorization",
-        "noop_elimination",
-        "shuffle_and_repeat_fusion",
-    ]
-    result = []
-    for exp_opt in experimental_optimizations:
-      if getattr(self, "experimental_" + exp_opt):
-        result.append(exp_opt)
 
-    if getattr(self, "experimental_numa_aware"):
+    result = []
+    exp_optimization_options = self.experimental_optimization
+    if exp_optimization_options:
+      optimizations = [
+          "filter_fusion",
+          "hoist_random_uniform",
+          "map_and_batch_fusion",
+          "map_and_filter_fusion",
+          "map_fusion",
+          "map_parallelization",
+          "map_vectorization",
+          "noop_elimination",
+          "shuffle_and_repeat_fusion",
+      ]
+      for optimization in optimizations:
+        if getattr(exp_optimization_options, optimization):
+          result.append(optimization)
+    if self.experimental_numa_aware:
       result.append("make_numa_aware")
-    if getattr(self, "experimental_deterministic") is False:
+    if self.experimental_deterministic is False:
       result.append("make_sloppy")
-    experimental_stats_options = getattr(self, "experimental_stats")
-    if experimental_stats_options and getattr(experimental_stats_options,
-                                              "latency_all_edges"):
+    exp_stats_options = self.experimental_stats
+    if exp_stats_options and exp_stats_options.latency_all_edges:
       result.append("latency_all_edges")
     return result
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt
index 024b4514ba..72fc2c3a9e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-options.pbtxt
@@ -11,44 +11,12 @@ tf_class {
     name: "experimental_deterministic"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "experimental_filter_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_hoist_random_uniform"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_and_batch_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_and_filter_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_parallelization"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_vectorization"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_noop_elimination"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "experimental_numa_aware"
     mtype: "<type \'property\'>"
   }
   member {
-    name: "experimental_shuffle_and_repeat_fusion"
+    name: "experimental_optimization"
     mtype: "<type \'property\'>"
   }
   member {
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt
new file mode 100644
index 0000000000..9ca75828e5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt
@@ -0,0 +1,46 @@
+path: "tensorflow.data.experimental.OptimizationOptions"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.optimization_options.OptimizationOptions\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "filter_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "hoist_random_uniform"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_and_batch_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_and_filter_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_parallelization"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_vectorization"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "noop_elimination"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shuffle_and_repeat_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index a3cb799fc3..ad10b82283 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CsvDataset"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "OptimizationOptions"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Optional"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt
index 024b4514ba..72fc2c3a9e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-options.pbtxt
@@ -11,44 +11,12 @@ tf_class {
     name: "experimental_deterministic"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "experimental_filter_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_hoist_random_uniform"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_and_batch_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_and_filter_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_fusion"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_parallelization"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_map_vectorization"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "experimental_noop_elimination"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "experimental_numa_aware"
     mtype: "<type \'property\'>"
   }
   member {
-    name: "experimental_shuffle_and_repeat_fusion"
+    name: "experimental_optimization"
     mtype: "<type \'property\'>"
   }
   member {
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt
new file mode 100644
index 0000000000..9ca75828e5
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt
@@ -0,0 +1,46 @@
+path: "tensorflow.data.experimental.OptimizationOptions"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.experimental.ops.optimization_options.OptimizationOptions\'>"
+  is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "filter_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "hoist_random_uniform"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_and_batch_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_and_filter_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_parallelization"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "map_vectorization"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "noop_elimination"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shuffle_and_repeat_fusion"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index a3cb799fc3..ad10b82283 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CsvDataset"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "OptimizationOptions"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Optional"
     mtype: "<type \'type\'>"
-- 
GitLab


From 1223dd41a235c2f3a7debd1a08f0332fe6b2aed2 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Thu, 29 Nov 2018 15:03:50 -0800
Subject: [PATCH 1066/1554] Visibility changes.

PiperOrigin-RevId: 223418219
---
 tensorflow/contrib/framework/BUILD | 5 +++++
 tensorflow/contrib/layers/BUILD    | 5 +++++
 tensorflow/contrib/learn/BUILD     | 5 +++++
 3 files changed, 15 insertions(+)

diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index 53efae1e10..dad50a3a73 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -47,6 +47,11 @@ tf_custom_op_py_library(
         ":variable_ops_op_lib",
     ],
     srcs_version = "PY2AND3",
+    visibility = [
+        "//learning/brain:__subpackages__",
+        "//tensorflow:__subpackages__",
+        "//video/youtube/personalization:__subpackages__",
+    ],
     deps = [
         ":gen_variable_ops",
         "//tensorflow/contrib/util:util_py",
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index 795591ea62..9ca6f8df5d 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -78,6 +78,11 @@ tf_custom_op_py_library(
         ":sparse_feature_cross_op_op_lib",
     ],
     srcs_version = "PY2AND3",
+    visibility = [
+        "//learning/brain:__subpackages__",
+        "//tensorflow:__subpackages__",
+        "//video/youtube/personalization:__subpackages__",
+    ],
     deps = [
         ":sparse_feature_cross_op",
         "//tensorflow/contrib/framework:framework_py",
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 61185f65a9..238504f6d6 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -24,6 +24,11 @@ py_library(
         exclude = ["python/learn/**/*_test.py"],
     ),
     srcs_version = "PY2AND3",
+    visibility = [
+        "//learning/brain:__subpackages__",
+        "//tensorflow:__subpackages__",
+        "//video/youtube/personalization:__subpackages__",
+    ],
     # This library should not depend on sklearn, even though some of the code
     # refers to it. (The code handles the presence of sklearn conditionally.)
     deps = [
-- 
GitLab


From ee5f4ebbf4b93283372ba708925318a2a8ce83a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 15:23:05 -0800
Subject: [PATCH 1067/1554] Allow delegates to allocate memory while copying
 them from their buffers.

PiperOrigin-RevId: 223421612
---
 tensorflow/lite/c/c_api_internal.c            | 11 +++++++++
 tensorflow/lite/c/c_api_internal.h            | 11 ++++++---
 tensorflow/lite/core/subgraph.h               |  5 ++--
 tensorflow/lite/delegates/flex/delegate.cc    | 23 +++++++++----------
 tensorflow/lite/delegates/flex/kernel_test.cc |  4 ++--
 5 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c
index 7f67b1c272..598d74be84 100644
--- a/tensorflow/lite/c/c_api_internal.c
+++ b/tensorflow/lite/c/c_api_internal.c
@@ -139,3 +139,14 @@ const char* TfLiteTypeGetName(TfLiteType type) {
   return "Unknown type";
 }
 
+TfLiteDelegate TfLiteDelegateCreate() {
+  TfLiteDelegate d = {
+      .data_ = NULL,
+      .Prepare = NULL,
+      .CopyFromBufferHandle = NULL,
+      .CopyToBufferHandle = NULL,
+      .FreeBufferHandle = NULL,
+      .flags = kTfLiteDelegateFlagsNone,
+  };
+  return d;
+}
diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index d7bf06442b..6280bf825d 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -488,12 +488,13 @@ typedef struct _TfLiteDelegate {
   // delegated subgraphs of the original graph.
   TfLiteStatus (*Prepare)(TfLiteContext* context, TfLiteDelegate* delegate);
 
-  // Copy the data from delegate buffer handle to raw memory.
-  // This can be null if the delegate doesn't use its own buffer.
+  // Copy the data from delegate buffer handle into raw memory of the given
+  // 'tensor'. This cannot be null. The delegate is allowed to allocate the raw
+  // bytes as long as it follows the rules for kTfLiteDynamic tensors.
   TfLiteStatus (*CopyFromBufferHandle)(TfLiteContext* context,
                                        TfLiteDelegate* delegate,
                                        TfLiteBufferHandle buffer_handle,
-                                       void* data, size_t size);
+                                       TfLiteTensor* tensor);
 
   // Copy the data from raw memory to delegate buffer handle.
   // This can be null if the delegate doesn't use its own buffer.
@@ -513,6 +514,10 @@ typedef struct _TfLiteDelegate {
   int64_t flags;
 } TfLiteDelegate;
 
+// Build a 'null' delegate, with all the fields properly set to their default
+// values.
+TfLiteDelegate TfLiteDelegateCreate();
+
 // WARNING: This is an experimental interface that is subject to change.
 //
 // Currently, TfLiteDelegateParams has to be allocated in a way that it's
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index e85d6df974..6b44a4731c 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -216,10 +216,9 @@ class Subgraph {
     if (t->data_is_stale) {
       TF_LITE_ENSURE(context_, t->delegate != nullptr);
       TF_LITE_ENSURE(context_, t->buffer_handle != kTfLiteNullBufferHandle);
-      // This can be null if the delegate doesn't use its own buffer.
       TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle != nullptr);
-      t->delegate->CopyFromBufferHandle(context_, t->delegate, t->buffer_handle,
-                                        t->data.raw, t->bytes);
+      TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle(
+                                   context_, t->delegate, t->buffer_handle, t));
       t->data_is_stale = false;
     }
     return kTfLiteOk;
diff --git a/tensorflow/lite/delegates/flex/delegate.cc b/tensorflow/lite/delegates/flex/delegate.cc
index 4fc2d82b49..12dcaa11d3 100644
--- a/tensorflow/lite/delegates/flex/delegate.cc
+++ b/tensorflow/lite/delegates/flex/delegate.cc
@@ -57,8 +57,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteDelegate* delegate) {
 
 TfLiteStatus CopyFromBufferHandle(TfLiteContext* context,
                                   TfLiteDelegate* delegate,
-                                  TfLiteBufferHandle buffer_handle, void* data,
-                                  size_t size) {
+                                  TfLiteBufferHandle buffer_handle,
+                                  TfLiteTensor* output) {
   BufferMap* buffer_map =
       reinterpret_cast<DelegateData*>(delegate->data_)->GetBufferMap(context);
 
@@ -70,13 +70,13 @@ TfLiteStatus CopyFromBufferHandle(TfLiteContext* context,
   tensorflow::Tensor t = buffer_map->GetTensor(buffer_handle);
   tensorflow::StringPiece t_data = t.tensor_data();
 
-  if (size != t_data.size()) {
+  if (output->bytes != t_data.size()) {
     context->ReportError(
         context, "Not enough space to store TensorFlow's aligned buffer.");
     return kTfLiteError;
   }
 
-  memcpy(data, t_data.data(), t_data.size());
+  memcpy(output->data.raw, t_data.data(), t_data.size());
   return kTfLiteOk;
 }
 
@@ -104,14 +104,13 @@ std::unique_ptr<FlexDelegate> FlexDelegate::Create() {
 }
 
 FlexDelegate::FlexDelegate(std::unique_ptr<flex::DelegateData> delegate_data)
-    : TfLiteDelegate{
-          /*data_=*/delegate_data.get(),
-          /*nullptr,*/ &flex::delegate::Prepare,
-          /*CopyFromBufferHandle=*/&flex::delegate::CopyFromBufferHandle,
-          /*CopyToBufferHandle=*/nullptr,
-          /*FreeBufferHandle=*/nullptr,
-          /*flags=*/kTfLiteDelegateFlagsAllowDynamicTensors},
-      delegate_data_(std::move(delegate_data)) {}
+    : TfLiteDelegate(TfLiteDelegateCreate()),
+      delegate_data_(std::move(delegate_data)) {
+  data_ = delegate_data_.get();
+  Prepare = &flex::delegate::Prepare;
+  CopyFromBufferHandle = &flex::delegate::CopyFromBufferHandle;
+  flags = kTfLiteDelegateFlagsAllowDynamicTensors;
+}
 
 FlexDelegate::~FlexDelegate() {}
 
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index f55759594d..efb7300b0b 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -59,12 +59,12 @@ class KernelTest : public testing::FlexModelTest {
     delegate_.CopyFromBufferHandle = [](TfLiteContext* context,
                                         TfLiteDelegate* delegate,
                                         TfLiteBufferHandle buffer_handle,
-                                        void* data, size_t size) {
+                                        TfLiteTensor* output) {
       auto* delegate_data = reinterpret_cast<DelegateData*>(delegate->data_);
       tensorflow::StringPiece values = delegate_data->GetBufferMap(context)
                                            ->GetTensor(buffer_handle)
                                            .tensor_data();
-      memcpy(data, values.data(), values.size());
+      memcpy(output->data.raw, values.data(), values.size());
       return kTfLiteOk;
     };
     CHECK(interpreter_->ModifyGraphWithDelegate(&delegate_) == kTfLiteOk);
-- 
GitLab


From 3a9b2ae05da54b65759a3656db918706115a50dc Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 29 Nov 2018 15:26:27 -0800
Subject: [PATCH 1068/1554] Add V2 APIs for losses: `MeanAbsoluteError`,
 `MeanAbsolutePercentageError`, `MeanSquaredLogarithmicError`

PiperOrigin-RevId: 223422208
---
 tensorflow/python/keras/losses.py             | 21 ++++++++++--------
 ...ow.keras.losses.-mean-absolute-error.pbtxt | 22 +++++++++++++++++++
 ...sses.-mean-absolute-percentage-error.pbtxt | 22 +++++++++++++++++++
 ...ses.-mean-squared-logarithmic-error.pbtxt} |  4 ++--
 .../golden/v1/tensorflow.keras.losses.pbtxt   | 12 ++++++++++
 .../api/golden/v1/tensorflow.losses.pbtxt     |  4 ----
 ...ow.keras.losses.-mean-absolute-error.pbtxt | 22 +++++++++++++++++++
 ...sses.-mean-absolute-percentage-error.pbtxt | 22 +++++++++++++++++++
 ...ses.-mean-squared-logarithmic-error.pbtxt} |  4 ++--
 .../golden/v2/tensorflow.keras.losses.pbtxt   | 12 ++++++++++
 .../api/golden/v2/tensorflow.losses.pbtxt     |  4 ----
 11 files changed, 128 insertions(+), 21 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
 rename tensorflow/tools/api/golden/v1/{tensorflow.losses.-mean-squared-error.pbtxt => tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt} (89%)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
 rename tensorflow/tools/api/golden/v2/{tensorflow.losses.-mean-squared-error.pbtxt => tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt} (89%)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 6b8d560102..f6953f8499 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -116,7 +116,7 @@ class Loss(object):
     NotImplementedError('Must be implemented in subclasses.')
 
 
-@tf_export('losses.MeanSquaredError', 'keras.losses.MeanSquaredError')
+@tf_export('keras.losses.MeanSquaredError')
 class MeanSquaredError(Loss):
   """Computes the mean of squares of errors between labels and predictions.
 
@@ -126,7 +126,7 @@ class MeanSquaredError(Loss):
   Usage:
 
   ```python
-  mse = tf.losses.MeanSquaredError()
+  mse = tf.keras.losses.MeanSquaredError()
   loss = mse([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Loss: ', loss.numpy())  # Loss: 0.75
   ```
@@ -135,7 +135,7 @@ class MeanSquaredError(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.MeanSquaredError())
+  model.compile('sgd', loss=tf.keras.losses.MeanSquaredError())
   ```
   """
 
@@ -154,6 +154,7 @@ class MeanSquaredError(Loss):
     return mean_squared_error(y_true, y_pred)
 
 
+@tf_export('keras.losses.MeanAbsoluteError')
 class MeanAbsoluteError(Loss):
   """Computes the mean of absolute difference between labels and predictions.
 
@@ -163,7 +164,7 @@ class MeanAbsoluteError(Loss):
   Usage:
 
   ```python
-  mae = tf.losses.MeanAbsoluteError()
+  mae = tf.keras.losses.MeanAbsoluteError()
   loss = mae([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Loss: ', loss.numpy())  # Loss: 0.75
   ```
@@ -172,7 +173,7 @@ class MeanAbsoluteError(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.MeanAbsoluteError())
+  model.compile('sgd', loss=tf.keras.losses.MeanAbsoluteError())
   ```
   """
 
@@ -191,6 +192,7 @@ class MeanAbsoluteError(Loss):
     return mean_absolute_error(y_true, y_pred)
 
 
+@tf_export('keras.losses.MeanAbsolutePercentageError')
 class MeanAbsolutePercentageError(Loss):
   """Computes the mean absolute percentage error between `y_true` and `y_pred`.
 
@@ -200,7 +202,7 @@ class MeanAbsolutePercentageError(Loss):
   Usage:
 
   ```python
-  mape = tf.losses.MeanAbsolutePercentageError()
+  mape = tf.keras.losses.MeanAbsolutePercentageError()
   loss = mape([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Loss: ', loss.numpy())  # Loss: 5e+08
   ```
@@ -209,7 +211,7 @@ class MeanAbsolutePercentageError(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.MeanAbsolutePercentageError())
+  model.compile('sgd', loss=tf.keras.losses.MeanAbsolutePercentageError())
   ```
   """
 
@@ -228,6 +230,7 @@ class MeanAbsolutePercentageError(Loss):
     return mean_absolute_percentage_error(y_true, y_pred)
 
 
+@tf_export('keras.losses.MeanSquaredLogarithmicError')
 class MeanSquaredLogarithmicError(Loss):
   """Computes the mean squared logarithmic error between `y_true` and `y_pred`.
 
@@ -237,7 +240,7 @@ class MeanSquaredLogarithmicError(Loss):
   Usage:
 
   ```python
-  msle = tf.losses.MeanSquaredLogarithmicError()
+  msle = tf.keras.losses.MeanSquaredLogarithmicError()
   loss = msle([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Loss: ', loss.numpy())  # Loss: 0.36034
   ```
@@ -246,7 +249,7 @@ class MeanSquaredLogarithmicError(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.losses.MeanSquaredLogarithmicError())
+  model.compile('sgd', loss=tf.keras.losses.MeanSquaredLogarithmicError())
   ```
   """
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt
new file mode 100644
index 0000000000..712bb2ecd3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.MeanAbsoluteError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsoluteError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
new file mode 100644
index 0000000000..7fe362da89
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.MeanAbsolutePercentageError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsolutePercentageError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
similarity index 89%
rename from tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
index a626d9c7e6..200006db35 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.losses.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
@@ -1,6 +1,6 @@
-path: "tensorflow.losses.MeanSquaredError"
+path: "tensorflow.keras.losses.MeanSquaredLogarithmicError"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredLogarithmicError\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index a0af6a29f0..c6638bd5e0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -1,9 +1,21 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "MeanAbsoluteError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanAbsolutePercentageError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanSquaredError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "MeanSquaredLogarithmicError"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "KLD"
     argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt
index a198db1b35..c1d190ae11 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.losses.pbtxt
@@ -1,9 +1,5 @@
 path: "tensorflow.losses"
 tf_module {
-  member {
-    name: "MeanSquaredError"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Reduction"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt
new file mode 100644
index 0000000000..712bb2ecd3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.MeanAbsoluteError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsoluteError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
new file mode 100644
index 0000000000..7fe362da89
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-absolute-percentage-error.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.MeanAbsolutePercentageError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanAbsolutePercentageError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
similarity index 89%
rename from tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
rename to tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
index a626d9c7e6..200006db35 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-mean-squared-logarithmic-error.pbtxt
@@ -1,6 +1,6 @@
-path: "tensorflow.losses.MeanSquaredError"
+path: "tensorflow.keras.losses.MeanSquaredLogarithmicError"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredError\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.MeanSquaredLogarithmicError\'>"
   is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index cb156e2248..5204a29a82 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -1,9 +1,21 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "MeanAbsoluteError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "MeanAbsolutePercentageError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanSquaredError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "MeanSquaredLogarithmicError"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Reduction"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
index 87f5ef3491..233b1a0131 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.losses.pbtxt
@@ -1,9 +1,5 @@
 path: "tensorflow.losses"
 tf_module {
-  member {
-    name: "MeanSquaredError"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Reduction"
     mtype: "<type \'type\'>"
-- 
GitLab


From bbad7c0a070b25b37a5522217982e53e8c60c95c Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 29 Nov 2018 15:28:37 -0800
Subject: [PATCH 1069/1554] Fix keras_multi_worker test failure.

PiperOrigin-RevId: 223422542
---
 tensorflow/python/keras/engine/distributed_training_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index cd8e0342cd..c587287ff9 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -328,7 +328,7 @@ def init_restore_or_wait_for_variables():
   """Initialize or restore variables or wait for variables to be initialized."""
   session = K._get_session()  # pylint: disable=protected-access
   worker_context = dc_context.get_current_worker_context()
-  if not worker_context or worker_context.should_init:
+  if not worker_context or worker_context.experimental_should_init:
     # TODO(yuefengz): if checkpoints exit, restore from checkpoint.
     K._initialize_variables(session)  # pylint: disable=protected-access
   else:
-- 
GitLab


From 24f578cd66bfc3ec35017fc77e136e43c4b74742 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 29 Nov 2018 15:30:12 -0800
Subject: [PATCH 1070/1554] Add @run_deprecated_v1 annotation to tests failing
 in v2

PiperOrigin-RevId: 223422907
---
 tensorflow/examples/adding_an_op/fact_test.py |   2 +
 .../examples/adding_an_op/zero_out_1_test.py  |   2 +
 .../examples/adding_an_op/zero_out_2_test.py  |   5 +
 .../examples/adding_an_op/zero_out_3_test.py  |   5 +
 .../autograph/integration_tests/keras_test.py |   2 +
 .../examples/speech_commands/freeze_test.py   |   4 +
 .../speech_commands/input_data_test.py        |   9 +
 .../examples/speech_commands/models_test.py   |   6 +
 .../speech_commands/wav_to_features_test.py   |   2 +
 .../autograph/converters/asserts_test.py      |   2 +
 .../converters/builtin_functions_test.py      |   4 +
 .../autograph/converters/control_flow_test.py |  13 +
 .../converters/function_scopes_test.py        |   5 +
 .../converters/logical_expressions_test.py    |   4 +
 .../converters/side_effect_guards_test.py     |   3 +
 .../python/autograph/core/errors_test.py      |   4 +
 .../autograph/core/function_wrapping_test.py  |   2 +
 tensorflow/python/autograph/impl/api_test.py  |  10 +
 .../autograph/operators/control_flow_test.py  |   6 +
 .../operators/data_structures_test.py         |   4 +
 .../autograph/operators/exceptions_test.py    |   3 +
 .../autograph/operators/logical_test.py       |   3 +
 .../autograph/operators/py_builtins_test.py   |   4 +
 .../python/autograph/utils/misc_test.py       |   3 +
 .../autograph/utils/tensor_list_test.py       |   3 +
 .../python/autograph/utils/type_check_test.py |   1 +
 .../python/client/session_partial_run_test.py |  24 ++
 tensorflow/python/client/timeline_test.py     |   1 +
 .../kernel_tests/copy_to_device_test.py       |  10 +
 .../experimental/kernel_tests/counter_test.py |   2 +
 .../dense_to_sparse_batch_test.py             |   5 +
 .../directed_interleave_dataset_test.py       |   4 +
 .../kernel_tests/enumerate_dataset_test.py    |   2 +
 .../function_buffering_resource_test.py       |   4 +
 .../kernel_tests/get_single_element_test.py   |   2 +
 .../kernel_tests/group_by_reducer_test.py     |   6 +
 .../kernel_tests/group_by_window_test.py      |  11 +
 .../kernel_tests/ignore_errors_test.py        |   4 +
 .../kernel_tests/indexed_dataset_ops_test.py  |   3 +
 .../make_batched_features_dataset_test.py     |   2 +
 .../kernel_tests/make_csv_dataset_test.py     |   9 +
 .../kernel_tests/map_and_batch_test.py        |  13 +
 .../kernel_tests/matching_files_test.py       |   7 +
 .../kernel_tests/override_threadpool_test.py  |   3 +
 .../parse_example_dataset_test.py             |   4 +
 .../kernel_tests/prefetch_to_device_test.py   |   5 +
 .../kernel_tests/rejection_resample_test.py   |   4 +
 .../kernel_tests/restructured_dataset_test.py |   2 +
 .../experimental/kernel_tests/scan_test.py    |   3 +
 .../checkpoint_input_pipeline_hook_test.py    |   4 +
 .../kernel_tests/shuffle_and_repeat_test.py   |   7 +
 .../experimental/kernel_tests/sleep_test.py   |   2 +
 .../kernel_tests/stats_dataset_ops_test.py    |  17 ++
 .../experimental/kernel_tests/unbatch_test.py |  10 +
 .../experimental/kernel_tests/unique_test.py  |   3 +
 .../python/data/kernel_tests/flat_map_test.py |   1 +
 .../data/kernel_tests/from_generator_test.py  |  17 ++
 .../from_sparse_tensor_slices_test.py         |   1 +
 .../data/kernel_tests/from_tensors_test.py    |   1 +
 .../python/data/kernel_tests/iterator_test.py |  17 ++
 .../data/kernel_tests/list_files_test.py      |   1 +
 .../python/data/kernel_tests/optional_test.py |   3 +
 .../data/kernel_tests/padded_batch_test.py    |   3 +
 .../python/data/kernel_tests/reduce_test.py   |   1 +
 .../python/data/kernel_tests/shuffle_test.py  |   1 +
 tensorflow/python/data/util/convert_test.py   |   2 +
 tensorflow/python/data/util/sparse_test.py    |   3 +
 tensorflow/python/data/util/structure_test.py |   2 +
 .../python/debug/cli/analyzer_cli_test.py     |  33 +++
 .../python/debug/cli/cli_shared_test.py       |  11 +
 .../debug/cli/profile_analyzer_cli_test.py    |   5 +
 tensorflow/python/debug/lib/common_test.py    |   3 +
 .../python/debug/lib/debug_gradients_test.py  |  14 ++
 .../python/debug/lib/debug_utils_test.py      |  10 +
 .../debug/lib/session_debug_file_test.py      |   2 +
 .../python/debug/lib/source_utils_test.py     |   1 +
 .../python/distribute/all_reduce_test.py      |   5 +
 .../python/distribute/device_util_test.py     |   3 +
 .../python/distribute/input_ops_test.py       |  12 +
 .../python/eager/function_gradients_test.py   |   1 +
 tensorflow/python/eager/function_test.py      |   1 +
 .../python/eager/graph_only_ops_test.py       |   2 +
 .../feature_column/feature_column_test.py     | 130 ++++++++++
 .../feature_column/feature_column_v2_test.py  | 141 +++++++++++
 .../python/framework/file_system_test.py      |   2 +
 .../framework/function_def_to_graph_test.py   |   3 +
 tensorflow/python/framework/function_test.py  |  26 ++
 .../python/framework/graph_util_test.py       |   2 +
 .../python/framework/meta_graph_test.py       |  17 ++
 tensorflow/python/framework/ops_test.py       |  57 +++++
 .../python/framework/smart_cond_test.py       |   5 +
 .../python/framework/sparse_tensor_test.py    |   2 +
 tensorflow/python/framework/subscribe_test.py |  10 +
 .../python/framework/tensor_spec_test.py      |   4 +
 .../python/framework/tensor_util_test.py      |  12 +
 tensorflow/python/framework/test_util_test.py |   5 +
 .../python/grappler/cost_analyzer_test.py     |   4 +
 tensorflow/python/grappler/item_test.py       |   2 +
 .../python/grappler/layout_optimizer_test.py  |   3 +
 .../python/grappler/memory_optimizer_test.py  |   3 +
 .../python/grappler/model_analyzer_test.py    |   3 +
 .../python/grappler/tf_optimizer_test.py      |   4 +
 tensorflow/python/keras/activations_test.py   |   4 +
 tensorflow/python/keras/backend_test.py       |   8 +
 tensorflow/python/keras/callbacks_test.py     |  10 +
 tensorflow/python/keras/engine/saving_test.py |   8 +
 .../python/keras/engine/sequential_test.py    |   2 +
 .../python/keras/engine/topology_test.py      |  10 +
 .../keras/engine/training_dataset_test.py     |   1 +
 .../python/keras/engine/training_test.py      |  10 +
 tensorflow/python/keras/initializers_test.py  |  11 +
 tensorflow/python/keras/integration_test.py   |   8 +
 tensorflow/python/keras/layers/local_test.py  |   4 +
 tensorflow/python/keras/layers/lstm_test.py   |   4 +
 tensorflow/python/keras/layers/merge_test.py  |   1 +
 .../python/keras/layers/normalization_test.py |   1 +
 .../python/keras/layers/simplernn_test.py     |   3 +
 .../python/keras/layers/unified_rnn_test.py   |   2 +
 .../python/keras/layers/wrappers_test.py      |   4 +
 .../python/keras/model_subclassing_test.py    |   4 +
 tensorflow/python/keras/models_test.py        |  10 +
 .../keras/optimizer_v2/adadelta_test.py       |   1 +
 .../python/keras/optimizer_v2/adagrad_test.py |   7 +
 .../python/keras/optimizer_v2/adam_test.py    |   6 +
 .../python/keras/optimizer_v2/adamax_test.py  |   5 +
 .../python/keras/optimizer_v2/ftrl_test.py    |  14 ++
 .../optimizer_v2/gradient_descent_test.py     |   9 +
 .../python/keras/optimizer_v2/nadam_test.py   |   5 +
 .../python/keras/optimizer_v2/rmsprop_test.py |   5 +
 tensorflow/python/keras/optimizers_test.py    |   9 +
 tensorflow/python/keras/regularizers_test.py  |   2 +
 .../python/kernel_tests/accumulate_n_test.py  |   4 +
 .../python/kernel_tests/ackermann_test.py     |   2 +
 .../python/kernel_tests/aggregate_ops_test.py |   3 +
 .../python/kernel_tests/argmax_op_test.py     |   3 +
 .../python/kernel_tests/array_ops_test.py     |  52 ++++
 .../python/kernel_tests/as_string_op_test.py  |   7 +
 .../python/kernel_tests/atrous_conv2d_test.py |   6 +
 .../python/kernel_tests/barrier_ops_test.py   |  18 ++
 .../python/kernel_tests/basic_gpu_test.py     |   3 +
 .../kernel_tests/batch_gather_op_test.py      |   4 +
 .../kernel_tests/batch_scatter_ops_test.py    |   4 +
 .../kernel_tests/batchtospace_op_test.py      |  21 ++
 .../python/kernel_tests/bcast_ops_test.py     |   6 +
 .../python/kernel_tests/betainc_op_test.py    |   8 +
 .../python/kernel_tests/bias_op_test.py       |  12 +
 .../python/kernel_tests/bincount_op_test.py   |   7 +
 .../python/kernel_tests/bitcast_op_test.py    |   2 +
 .../boosted_trees/prediction_ops_test.py      |  15 ++
 .../boosted_trees/quantile_ops_test.py        |   4 +
 .../boosted_trees/resource_ops_test.py        |   3 +
 .../boosted_trees/stats_ops_test.py           |   1 +
 .../boosted_trees/training_ops_test.py        |  10 +
 .../kernel_tests/broadcast_to_ops_test.py     |  11 +
 .../python/kernel_tests/bucketize_op_test.py  |   2 +
 .../candidate_sampler_ops_test.py             |   3 +
 .../python/kernel_tests/cast_op_test.py       |   9 +
 .../python/kernel_tests/check_ops_test.py     |  45 ++++
 .../kernel_tests/checkpoint_ops_test.py       |  10 +
 .../python/kernel_tests/cholesky_op_test.py   |   9 +
 .../python/kernel_tests/clip_ops_test.py      |  10 +
 .../python/kernel_tests/concat_op_test.py     |  26 ++
 .../python/kernel_tests/cond_v2_test.py       |   8 +
 .../conditional_accumulator_test.py           |  24 ++
 .../kernel_tests/confusion_matrix_test.py     |  23 ++
 .../python/kernel_tests/constant_op_test.py   |  40 ++++
 .../kernel_tests/control_flow_ops_py_test.py  |  55 +++++
 .../conv2d_backprop_filter_grad_test.py       |   2 +
 .../kernel_tests/conv2d_transpose_test.py     |   2 +
 .../conv3d_backprop_filter_v2_grad_test.py    |   2 +
 .../kernel_tests/conv3d_transpose_test.py     |   3 +
 .../python/kernel_tests/conv_ops_3d_test.py   |  17 ++
 .../python/kernel_tests/cross_grad_test.py    |   2 +
 .../kernel_tests/ctc_decoder_ops_test.py      |   3 +
 .../python/kernel_tests/ctc_loss_op_test.py   |  23 ++
 .../kernel_tests/cwise_ops_binary_test.py     |  73 ++++++
 .../python/kernel_tests/cwise_ops_test.py     |  15 ++
 .../kernel_tests/cwise_ops_unary_test.py      |   8 +
 .../kernel_tests/decode_compressed_op_test.py |   3 +
 .../kernel_tests/decode_image_op_test.py      |   4 +
 .../python/kernel_tests/decode_raw_op_test.py |   7 +
 .../python/kernel_tests/denormal_test.py      |   3 +
 .../dense_update_ops_no_tsan_test.py          |   5 +
 .../kernel_tests/dense_update_ops_test.py     |   5 +
 .../kernel_tests/depthtospace_op_test.py      |  17 ++
 .../python/kernel_tests/diag_op_test.py       |  26 ++
 .../distributions/bernoulli_test.py           |   5 +
 .../distributions/bijector_test.py            |   2 +
 .../distributions/categorical_test.py         |  17 ++
 .../dirichlet_multinomial_test.py             |  13 +
 .../distributions/identity_bijector_test.py   |   2 +
 .../distributions/kullback_leibler_test.py    |   2 +
 .../kernel_tests/distributions/normal_test.py |   1 +
 .../distributions/special_math_test.py        |   9 +
 .../kernel_tests/distributions/util_test.py   |  48 ++++
 .../python/kernel_tests/duplicate_op_test.py  |   2 +
 .../kernel_tests/dynamic_partition_op_test.py |   8 +
 .../kernel_tests/dynamic_stitch_op_test.py    |   8 +
 .../python/kernel_tests/embedding_ops_test.py |  44 ++++
 .../extract_image_patches_grad_test.py        |   3 +
 .../fractional_avg_pool_op_test.py            |   5 +
 .../fractional_max_pool_op_test.py            |   5 +
 .../kernel_tests/functional_ops_test.py       |  33 +++
 .../python/kernel_tests/gather_nd_op_test.py  |  11 +
 .../python/kernel_tests/gather_op_test.py     |   8 +
 .../kernel_tests/gradient_correctness_test.py |  11 +
 .../kernel_tests/identity_n_op_py_test.py     |   4 +
 .../kernel_tests/identity_op_py_test.py       |   5 +
 .../python/kernel_tests/init_ops_test.py      |  59 +++++
 .../python/kernel_tests/inplace_ops_test.py   |   7 +
 tensorflow/python/kernel_tests/io_ops_test.py |   3 +
 .../linalg/linear_operator_addition_test.py   |  13 +
 .../linalg/linear_operator_circulant_test.py  |  20 ++
 .../linear_operator_composition_test.py       |   4 +
 .../linalg/linear_operator_diag_test.py       |   4 +
 .../linear_operator_full_matrix_test.py       |   2 +
 .../linalg/linear_operator_identity_test.py   |  15 ++
 .../linalg/linear_operator_kronecker_test.py  |   2 +
 .../linalg/linear_operator_test.py            |   9 +
 .../linalg/linear_operator_util_test.py       |  25 ++
 .../linalg/linear_operator_zeros_test.py      |   7 +
 .../python/kernel_tests/linalg_grad_test.py   |   2 +
 .../python/kernel_tests/linalg_ops_test.py    |   5 +
 .../python/kernel_tests/list_ops_test.py      |  15 ++
 .../python/kernel_tests/logging_ops_test.py   |   4 +
 .../python/kernel_tests/lookup_ops_test.py    |  64 +++++
 tensorflow/python/kernel_tests/losses_test.py |  62 +++++
 tensorflow/python/kernel_tests/lrn_op_test.py |   4 +
 .../python/kernel_tests/manip_ops_test.py     |  12 +
 .../python/kernel_tests/map_stage_op_test.py  |  13 +
 .../python/kernel_tests/matmul_op_test.py     |   4 +
 .../matrix_exponential_op_test.py             |   4 +
 .../kernel_tests/matrix_solve_ls_op_test.py   |   4 +
 .../kernel_tests/matrix_solve_op_test.py      |   6 +
 .../matrix_triangular_solve_op_test.py        |   7 +
 .../python/kernel_tests/metrics_test.py       | 226 ++++++++++++++++++
 .../kernel_tests/morphological_ops_test.py    |   3 +
 .../neon_depthwise_conv_op_test.py            |   3 +
 .../kernel_tests/nth_element_op_test.py       |   8 +
 .../python/kernel_tests/numerics_test.py      |   5 +
 tensorflow/python/kernel_tests/pad_op_test.py |  12 +
 .../parameterized_truncated_normal_op_test.py |  11 +
 .../parse_single_example_op_test.py           |   7 +
 .../python/kernel_tests/parsing_ops_test.py   |  18 ++
 .../partitioned_variables_test.py             |  13 +
 tensorflow/python/kernel_tests/pool_test.py   |   4 +
 .../kernel_tests/pooling_ops_3d_test.py       |  19 ++
 .../python/kernel_tests/pooling_ops_test.py   |  12 +
 .../python/kernel_tests/py_func_test.py       |   2 +
 tensorflow/python/kernel_tests/qr_op_test.py  |   2 +
 .../random/multinomial_op_big_test.py         |   2 +
 .../random/multinomial_op_test.py             |   3 +
 .../kernel_tests/random/random_crop_test.py   |   3 +
 .../kernel_tests/random/random_gamma_test.py  |   6 +
 .../kernel_tests/random/random_grad_test.py   |  11 +
 .../kernel_tests/random/random_ops_test.py    |  22 ++
 .../random/random_poisson_test.py             |   6 +
 .../random/stateless_random_ops_test.py       |   6 +
 .../python/kernel_tests/reader_ops_test.py    |  25 ++
 .../python/kernel_tests/record_input_test.py  |   4 +
 .../kernel_tests/reduce_join_op_test.py       |   6 +
 .../python/kernel_tests/reduction_ops_test.py |  57 +++++
 .../kernel_tests/regex_full_match_op_test.py  |   7 +
 .../kernel_tests/regex_replace_op_test.py     |   9 +
 .../python/kernel_tests/relu_op_test.py       |  21 ++
 .../python/kernel_tests/reshape_op_test.py    |   4 +
 .../resource_variable_ops_test.py             |  19 ++
 .../kernel_tests/reverse_sequence_op_test.py  |   3 +
 tensorflow/python/kernel_tests/rnn_test.py    |  10 +
 .../kernel_tests/save_restore_ops_test.py     |   3 +
 .../python/kernel_tests/scan_ops_test.py      |  24 ++
 .../kernel_tests/scatter_nd_ops_test.py       |  29 +++
 .../python/kernel_tests/scatter_ops_test.py   |  28 +++
 .../segment_reduction_ops_test.py             |  32 +++
 tensorflow/python/kernel_tests/sets_test.py   |  16 ++
 .../python/kernel_tests/shape_ops_test.py     |  21 ++
 .../kernel_tests/signal/dct_ops_test.py       |   2 +
 .../kernel_tests/signal/fft_ops_test.py       |  16 ++
 .../kernel_tests/signal/mel_ops_test.py       |   4 +
 .../kernel_tests/signal/mfcc_ops_test.py      |   4 +
 .../signal/reconstruction_ops_test.py         |   5 +
 .../kernel_tests/signal/shape_ops_test.py     |  12 +
 .../kernel_tests/signal/window_ops_test.py    |   3 +
 .../python/kernel_tests/slice_op_test.py      |   8 +
 .../python/kernel_tests/softmax_op_test.py    |   3 +
 .../python/kernel_tests/softplus_op_test.py   |   5 +
 .../python/kernel_tests/softsign_op_test.py   |   3 +
 .../kernel_tests/spacetobatch_op_test.py      |  36 +++
 .../kernel_tests/spacetodepth_op_test.py      |  11 +
 .../python/kernel_tests/sparse_add_op_test.py |   3 +
 .../kernel_tests/sparse_concat_op_test.py     |   5 +
 .../sparse_conditional_accumulator_test.py    |  24 ++
 .../kernel_tests/sparse_cross_op_test.py      |  18 ++
 .../kernel_tests/sparse_matmul_op_test.py     |   7 +
 .../python/kernel_tests/sparse_ops_test.py    |  32 +++
 .../kernel_tests/sparse_reorder_op_test.py    |   4 +
 .../kernel_tests/sparse_reshape_op_test.py    |  18 ++
 .../sparse_serialization_ops_test.py          |  25 ++
 .../kernel_tests/sparse_slice_op_test.py      |   8 +
 .../kernel_tests/sparse_split_op_test.py      |   8 +
 .../sparse_tensor_dense_matmul_grad_test.py   |   2 +
 .../sparse_tensor_dense_matmul_op_test.py     |   4 +
 .../sparse_tensors_map_ops_test.py            |   6 +
 .../sparse_to_dense_op_py_test.py             |  18 ++
 .../kernel_tests/sparse_xent_op_test.py       |  10 +-
 .../python/kernel_tests/sparsemask_op_test.py |   2 +
 .../python/kernel_tests/split_op_test.py      |   7 +
 .../python/kernel_tests/stack_op_test.py      |  17 ++
 .../python/kernel_tests/stack_ops_test.py     |  15 ++
 .../python/kernel_tests/stage_op_test.py      |   8 +
 .../kernel_tests/string_join_op_test.py       |   2 +
 .../kernel_tests/string_length_op_test.py     |   3 +
 .../kernel_tests/string_split_op_test.py      |   5 +
 .../string_to_hash_bucket_op_test.py          |   5 +
 .../kernel_tests/string_to_number_op_test.py  |   5 +
 .../python/kernel_tests/substr_op_test.py     |   9 +
 .../kernel_tests/summary_v1_image_op_test.py  |   3 +
 .../kernel_tests/summary_v1_ops_test.py       |   2 +
 .../python/kernel_tests/template_test.py      |   5 +
 .../kernel_tests/tensor_array_ops_test.py     |  42 ++++
 .../python/kernel_tests/topk_op_test.py       |   4 +
 .../python/kernel_tests/trace_op_test.py      |   2 +
 .../kernel_tests/unicode_decode_op_test.py    |   4 +
 .../kernel_tests/unicode_script_op_test.py    |   3 +
 .../kernel_tests/unicode_transcode_op_test.py |   7 +
 .../python/kernel_tests/unstack_op_test.py    |   6 +
 .../python/kernel_tests/variable_ops_test.py  |  22 ++
 .../kernel_tests/variable_scope_test.py       |  14 ++
 .../python/kernel_tests/variables_test.py     |  27 +++
 .../kernel_tests/weights_broadcast_test.py    |  31 +++
 .../python/kernel_tests/where_op_test.py      |  17 ++
 .../python/kernel_tests/while_v2_test.py      |  15 ++
 .../python/kernel_tests/xent_op_test.py       |   7 +
 .../python/kernel_tests/zero_division_test.py |   1 +
 tensorflow/python/layers/base_test.py         |   4 +
 .../python/layers/convolutional_test.py       |  56 +++++
 tensorflow/python/layers/core_test.py         |  19 ++
 .../python/layers/normalization_test.py       |  37 +++
 tensorflow/python/layers/pooling_test.py      |   4 +
 tensorflow/python/ops/bitwise_ops_test.py     |   6 +
 tensorflow/python/ops/clip_ops_test.py        |   2 +
 tensorflow/python/ops/collective_ops_test.py  |   5 +
 .../python/ops/gradient_checker_test.py       |  14 ++
 tensorflow/python/ops/histogram_ops_test.py   |   4 +
 tensorflow/python/ops/image_grad_test.py      |  12 +
 tensorflow/python/ops/image_ops_test.py       |  92 +++++++
 tensorflow/python/ops/losses/util_test.py     |   2 +
 tensorflow/python/ops/math_grad_test.py       |  26 ++
 tensorflow/python/ops/math_ops_test.py        |  24 ++
 tensorflow/python/ops/nn_batchnorm_test.py    |  14 ++
 tensorflow/python/ops/nn_grad_test.py         |   2 +
 tensorflow/python/ops/nn_test.py              |  20 ++
 tensorflow/python/ops/nn_xent_test.py         |   6 +
 ...vert_to_tensor_or_ragged_tensor_op_test.py |   2 +
 .../ops/ragged/ragged_batch_gather_op_test.py |   3 +
 .../ops/ragged/ragged_boolean_mask_op_test.py |   2 +
 .../ops/ragged/ragged_concat_op_test.py       |   5 +
 .../python/ops/ragged/ragged_const_op_test.py |   2 +
 .../ops/ragged/ragged_elementwise_ops_test.py |   1 +
 .../ops/ragged/ragged_expand_dims_op_test.py  |   1 +
 .../ops/ragged/ragged_from_sparse_op_test.py  |   5 +
 .../ops/ragged/ragged_from_tensor_op_test.py  |   5 +
 .../ops/ragged/ragged_gather_nd_op_test.py    |   3 +
 .../ops/ragged/ragged_gather_op_test.py       |  10 +
 .../ops/ragged/ragged_map_fn_op_test.py       |   5 +
 .../ragged/ragged_map_inner_values_op_test.py |  13 +
 .../ops/ragged/ragged_operators_test.py       |   3 +
 .../python/ops/ragged/ragged_range_op_test.py |   9 +
 .../ops/ragged/ragged_reduce_op_test.py       |   4 +
 .../ops/ragged/ragged_row_lengths_op_test.py  |   1 +
 ...agged_row_splits_to_segment_ids_op_test.py |   3 +
 ...agged_segment_ids_to_row_splits_op_test.py |   4 +
 .../ops/ragged/ragged_segment_op_test.py      |   5 +
 .../python/ops/ragged/ragged_stack_op_test.py |   2 +
 .../python/ops/ragged/ragged_tensor_test.py   |  33 +++
 .../python/ops/ragged/ragged_tile_op_test.py  |   2 +
 .../ops/ragged/ragged_to_sparse_op_test.py    |   7 +
 .../ops/ragged/ragged_to_tensor_op_test.py    |   3 +
 .../python/ops/ragged/ragged_where_op_test.py |   1 +
 tensorflow/python/ops/sort_ops_test.py        |  10 +
 .../python/ops/special_math_ops_test.py       |   7 +
 .../profiler/internal/run_metadata_test.py    |   3 +
 .../python/profiler/pprof_profiler_test.py    |   2 +
 .../python/profiler/profile_context_test.py   |   4 +
 tensorflow/python/profiler/profiler_test.py   |   3 +
 tensorflow/python/saved_model/loader_test.py  |   7 +
 tensorflow/python/saved_model/save_test.py    |   1 +
 .../python/saved_model/saved_model_test.py    |  31 +++
 .../saved_model/signature_def_utils_test.py   |   9 +
 .../python/saved_model/simple_save_test.py    |   2 +
 tensorflow/python/summary/summary_test.py     |  13 +
 .../python/summary/writer/writer_test.py      |  15 ++
 tensorflow/python/tools/freeze_graph_test.py  |   2 +
 .../tools/optimize_for_inference_test.py      |   5 +
 tensorflow/python/training/adadelta_test.py   |   1 +
 tensorflow/python/training/adagrad_da_test.py |   7 +
 tensorflow/python/training/adagrad_test.py    |   9 +
 tensorflow/python/training/adam_test.py       |   6 +
 .../python/training/basic_loops_test.py       |   4 +
 .../training/basic_session_run_hooks_test.py  |   9 +
 .../training/checkpoint_management_test.py    |   5 +
 .../python/training/device_setter_test.py     |  11 +
 tensorflow/python/training/ftrl_test.py       |  14 ++
 .../python/training/gradient_descent_test.py  |  10 +
 tensorflow/python/training/input_test.py      | 108 +++++++++
 .../training/learning_rate_decay_test.py      |   2 +
 .../training/learning_rate_decay_v2_test.py   |   1 +
 tensorflow/python/training/momentum_test.py   |   6 +
 .../python/training/monitored_session_test.py |  27 +++
 .../python/training/moving_averages_test.py   |  23 ++
 tensorflow/python/training/optimizer_test.py  |   4 +
 .../python/training/proximal_adagrad_test.py  |   9 +
 .../proximal_gradient_descent_test.py         |   8 +
 .../python/training/quantize_training_test.py |   2 +
 .../python/training/queue_runner_test.py      |  16 ++
 tensorflow/python/training/rmsprop_test.py    |   6 +
 tensorflow/python/training/saver_test.py      |  35 +++
 .../server_lib_multiple_containers_test.py    |   2 +
 ...lib_same_variables_clear_container_test.py |   2 +
 .../server_lib_same_variables_clear_test.py   |   2 +
 ...server_lib_same_variables_no_clear_test.py |   2 +
 .../training/server_lib_sparse_job_test.py    |   2 +
 .../python/training/session_manager_test.py   |  15 ++
 .../python/training/slot_creator_test.py      |   8 +
 tensorflow/python/training/supervisor_test.py |  12 +
 .../python/training/training_ops_test.py      |   8 +
 .../python/training/training_util_test.py     |   3 +
 tensorflow/python/util/deprecation_test.py    |  16 ++
 tensorflow/python/util/nest_test.py           |   1 +
 tensorflow/python/util/tf_should_use_test.py  |   7 +
 .../tools/api/tests/api_compatibility_test.py |   3 +
 431 files changed, 4673 insertions(+), 1 deletion(-)

diff --git a/tensorflow/examples/adding_an_op/fact_test.py b/tensorflow/examples/adding_an_op/fact_test.py
index 11163e7ba5..46beaebe0c 100644
--- a/tensorflow/examples/adding_an_op/fact_test.py
+++ b/tensorflow/examples/adding_an_op/fact_test.py
@@ -19,10 +19,12 @@ from __future__ import division
 from __future__ import print_function
 
 import tensorflow as tf
+from tensorflow.python.framework import test_util
 
 
 class FactTest(tf.test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test(self):
     with self.cached_session():
       print(tf.user_ops.my_fact().eval())
diff --git a/tensorflow/examples/adding_an_op/zero_out_1_test.py b/tensorflow/examples/adding_an_op/zero_out_1_test.py
index 342d3a020c..459ac2dc27 100644
--- a/tensorflow/examples/adding_an_op/zero_out_1_test.py
+++ b/tensorflow/examples/adding_an_op/zero_out_1_test.py
@@ -23,10 +23,12 @@ import os.path
 
 import tensorflow as tf
 from tensorflow.examples.adding_an_op import zero_out_op_1
+from tensorflow.python.framework import test_util
 
 
 class ZeroOut1Test(tf.test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test(self):
     with self.cached_session():
       result = zero_out_op_1.zero_out([5, 4, 3, 2, 1])
diff --git a/tensorflow/examples/adding_an_op/zero_out_2_test.py b/tensorflow/examples/adding_an_op/zero_out_2_test.py
index 4504597817..650fd9546b 100644
--- a/tensorflow/examples/adding_an_op/zero_out_2_test.py
+++ b/tensorflow/examples/adding_an_op/zero_out_2_test.py
@@ -24,20 +24,24 @@ import tensorflow as tf
 
 from tensorflow.examples.adding_an_op import zero_out_grad_2  # pylint: disable=unused-import
 from tensorflow.examples.adding_an_op import zero_out_op_2
+from tensorflow.python.framework import test_util
 
 
 class ZeroOut2Test(tf.test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test(self):
     with self.cached_session():
       result = zero_out_op_2.zero_out([5, 4, 3, 2, 1])
       self.assertAllEqual(result.eval(), [5, 0, 0, 0, 0])
 
+  @test_util.run_deprecated_v1
   def test_2d(self):
     with self.cached_session():
       result = zero_out_op_2.zero_out([[6, 5, 4], [3, 2, 1]])
       self.assertAllEqual(result.eval(), [[6, 0, 0], [0, 0, 0]])
 
+  @test_util.run_deprecated_v1
   def test_grad(self):
     with self.cached_session():
       shape = (5,)
@@ -46,6 +50,7 @@ class ZeroOut2Test(tf.test.TestCase):
       err = tf.test.compute_gradient_error(x, shape, y, shape)
       self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def test_grad_2d(self):
     with self.cached_session():
       shape = (2, 3)
diff --git a/tensorflow/examples/adding_an_op/zero_out_3_test.py b/tensorflow/examples/adding_an_op/zero_out_3_test.py
index 2327e7cd8f..8cbe2b6793 100644
--- a/tensorflow/examples/adding_an_op/zero_out_3_test.py
+++ b/tensorflow/examples/adding_an_op/zero_out_3_test.py
@@ -21,26 +21,31 @@ from __future__ import print_function
 
 import tensorflow as tf
 from tensorflow.examples.adding_an_op import zero_out_op_3
+from tensorflow.python.framework import test_util
 
 
 class ZeroOut3Test(tf.test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test(self):
     with self.cached_session():
       result = zero_out_op_3.zero_out([5, 4, 3, 2, 1])
       self.assertAllEqual(result.eval(), [5, 0, 0, 0, 0])
 
+  @test_util.run_deprecated_v1
   def testAttr(self):
     with self.cached_session():
       result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=3)
       self.assertAllEqual(result.eval(), [0, 0, 0, 2, 0])
 
+  @test_util.run_deprecated_v1
   def testNegative(self):
     with self.cached_session():
       result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=-1)
       with self.assertRaisesOpError("Need preserve_index >= 0, got -1"):
         self.evaluate(result)
 
+  @test_util.run_deprecated_v1
   def testLarge(self):
     with self.cached_session():
       result = zero_out_op_3.zero_out([5, 4, 3, 2, 1], preserve_index=17)
diff --git a/tensorflow/examples/autograph/integration_tests/keras_test.py b/tensorflow/examples/autograph/integration_tests/keras_test.py
index fc0b073696..3fe33df920 100644
--- a/tensorflow/examples/autograph/integration_tests/keras_test.py
+++ b/tensorflow/examples/autograph/integration_tests/keras_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import tensorflow as tf
 
 from tensorflow.python import autograph
+from tensorflow.python.framework import test_util
 
 
 class MinimalKeras(tf.keras.Model):
@@ -84,6 +85,7 @@ class KerasTest(tf.test.TestCase):
     model = ModelWithStaticConditional(True)
     self.assertEqual(model.call(), 25)
 
+  @test_util.run_deprecated_v1
   def test_recursive_true(self):
     with self.assertRaisesRegexp(NotImplementedError,
                                  'Object conversion is not yet supported.'):
diff --git a/tensorflow/examples/speech_commands/freeze_test.py b/tensorflow/examples/speech_commands/freeze_test.py
index 0c7ca9bc01..9ed9050035 100644
--- a/tensorflow/examples/speech_commands/freeze_test.py
+++ b/tensorflow/examples/speech_commands/freeze_test.py
@@ -19,11 +19,13 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.examples.speech_commands import freeze
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class FreezeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCreateInferenceGraphWithMfcc(self):
     with self.cached_session() as sess:
       freeze.create_inference_graph(
@@ -43,6 +45,7 @@ class FreezeTest(test.TestCase):
       ops = [node.op for node in sess.graph_def.node]
       self.assertEqual(1, ops.count('Mfcc'))
 
+  @test_util.run_deprecated_v1
   def testCreateInferenceGraphWithoutMfcc(self):
     with self.cached_session() as sess:
       freeze.create_inference_graph(
@@ -62,6 +65,7 @@ class FreezeTest(test.TestCase):
       ops = [node.op for node in sess.graph_def.node]
       self.assertEqual(0, ops.count('Mfcc'))
 
+  @test_util.run_deprecated_v1
   def testFeatureBinCount(self):
     with self.cached_session() as sess:
       freeze.create_inference_graph(
diff --git a/tensorflow/examples/speech_commands/input_data_test.py b/tensorflow/examples/speech_commands/input_data_test.py
index 33b58b9d09..9269bb6c0b 100644
--- a/tensorflow/examples/speech_commands/input_data_test.py
+++ b/tensorflow/examples/speech_commands/input_data_test.py
@@ -26,6 +26,7 @@ import tensorflow as tf
 from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio
 from tensorflow.examples.speech_commands import input_data
 from tensorflow.examples.speech_commands import models
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -96,6 +97,7 @@ class InputDataTest(test.TestCase):
         input_data.which_set("foo_nohash_0.wav", 10, 10),
         input_data.which_set("foo_nohash_1.wav", 10, 10))
 
+  @test_util.run_deprecated_v1
   def testPrepareDataIndex(self):
     tmp_dir = self.get_temp_dir()
     self._saveWavFolders(tmp_dir, ["a", "b", "c"], 100)
@@ -125,6 +127,7 @@ class InputDataTest(test.TestCase):
                                     10, self._model_settings(), tmp_dir)
     self.assertTrue("Expected to find" in str(e.exception))
 
+  @test_util.run_deprecated_v1
   def testPrepareBackgroundData(self):
     tmp_dir = self.get_temp_dir()
     background_dir = os.path.join(tmp_dir, "_background_noise_")
@@ -156,6 +159,7 @@ class InputDataTest(test.TestCase):
     self.assertIsNotNone(loaded_data)
     self.assertEqual(16000, len(loaded_data))
 
+  @test_util.run_deprecated_v1
   def testPrepareProcessingGraph(self):
     tmp_dir = self.get_temp_dir()
     wav_dir = os.path.join(tmp_dir, "wavs")
@@ -186,15 +190,19 @@ class InputDataTest(test.TestCase):
     self.assertIsNotNone(audio_processor.background_volume_placeholder_)
     self.assertIsNotNone(audio_processor.output_)
 
+  @test_util.run_deprecated_v1
   def testGetDataAverage(self):
     self._runGetDataTest("average", 10)
 
+  @test_util.run_deprecated_v1
   def testGetDataAverageLongWindow(self):
     self._runGetDataTest("average", 30)
 
+  @test_util.run_deprecated_v1
   def testGetDataMfcc(self):
     self._runGetDataTest("mfcc", 30)
 
+  @test_util.run_deprecated_v1
   def testGetUnprocessedData(self):
     tmp_dir = self.get_temp_dir()
     wav_dir = os.path.join(tmp_dir, "wavs")
@@ -216,6 +224,7 @@ class InputDataTest(test.TestCase):
     self.assertEqual(10, len(result_data))
     self.assertEqual(10, len(result_labels))
 
+  @test_util.run_deprecated_v1
   def testGetFeaturesForWav(self):
     tmp_dir = self.get_temp_dir()
     wav_dir = os.path.join(tmp_dir, "wavs")
diff --git a/tensorflow/examples/speech_commands/models_test.py b/tensorflow/examples/speech_commands/models_test.py
index 04478c0962..cb9304eab8 100644
--- a/tensorflow/examples/speech_commands/models_test.py
+++ b/tensorflow/examples/speech_commands/models_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import tensorflow as tf
 
 from tensorflow.examples.speech_commands import models
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -47,6 +48,7 @@ class ModelsTest(test.TestCase):
             feature_bin_count=40,
             preprocess="mfcc"))
 
+  @test_util.run_deprecated_v1
   def testCreateModelConvTraining(self):
     model_settings = self._modelSettings()
     with self.cached_session() as sess:
@@ -58,6 +60,7 @@ class ModelsTest(test.TestCase):
       self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name))
       self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name))
 
+  @test_util.run_deprecated_v1
   def testCreateModelConvInference(self):
     model_settings = self._modelSettings()
     with self.cached_session() as sess:
@@ -67,6 +70,7 @@ class ModelsTest(test.TestCase):
       self.assertIsNotNone(logits)
       self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name))
 
+  @test_util.run_deprecated_v1
   def testCreateModelLowLatencyConvTraining(self):
     model_settings = self._modelSettings()
     with self.cached_session() as sess:
@@ -78,6 +82,7 @@ class ModelsTest(test.TestCase):
       self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name))
       self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name))
 
+  @test_util.run_deprecated_v1
   def testCreateModelFullyConnectedTraining(self):
     model_settings = self._modelSettings()
     with self.cached_session() as sess:
@@ -98,6 +103,7 @@ class ModelsTest(test.TestCase):
                             "bad_architecture", True)
       self.assertTrue("not recognized" in str(e.exception))
 
+  @test_util.run_deprecated_v1
   def testCreateModelTinyConvTraining(self):
     model_settings = self._modelSettings()
     with self.cached_session() as sess:
diff --git a/tensorflow/examples/speech_commands/wav_to_features_test.py b/tensorflow/examples/speech_commands/wav_to_features_test.py
index cb8ea912fa..6234490b26 100644
--- a/tensorflow/examples/speech_commands/wav_to_features_test.py
+++ b/tensorflow/examples/speech_commands/wav_to_features_test.py
@@ -24,6 +24,7 @@ import tensorflow as tf
 
 from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio
 from tensorflow.examples.speech_commands import wav_to_features
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -49,6 +50,7 @@ class WavToFeaturesTest(test.TestCase):
         file_path = os.path.join(dir_name, "some_audio_%d.wav" % i)
         self._saveTestWavFile(file_path, wav_data)
 
+  @test_util.run_deprecated_v1
   def testWavToFeatures(self):
     tmp_dir = self.get_temp_dir()
     wav_dir = os.path.join(tmp_dir, "wavs")
diff --git a/tensorflow/python/autograph/converters/asserts_test.py b/tensorflow/python/autograph/converters/asserts_test.py
index 803b6a06da..9ae448892a 100644
--- a/tensorflow/python/autograph/converters/asserts_test.py
+++ b/tensorflow/python/autograph/converters/asserts_test.py
@@ -23,12 +23,14 @@ from tensorflow.python.autograph.converters import side_effect_guards
 from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_control_flow_ops
 from tensorflow.python.platform import test
 
 
 class AssertsTest(converter_testing.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_basic(self):
 
     def test_fn(a):
diff --git a/tensorflow/python/autograph/converters/builtin_functions_test.py b/tensorflow/python/autograph/converters/builtin_functions_test.py
index 30cfb13233..2683be16ec 100644
--- a/tensorflow/python/autograph/converters/builtin_functions_test.py
+++ b/tensorflow/python/autograph/converters/builtin_functions_test.py
@@ -24,12 +24,14 @@ from tensorflow.python.autograph.converters import builtin_functions
 from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class BuiltinFunctionsTest(converter_testing.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_len(self):
 
     def test_fn(a):
@@ -41,6 +43,7 @@ class BuiltinFunctionsTest(converter_testing.TestCase):
         ops = result.test_fn(p)
         self.assertEqual(sess.run(ops, {p: [0, 0, 0]}), 3)
 
+  @test_util.run_deprecated_v1
   def test_print(self):
 
     if six.PY2:
@@ -54,6 +57,7 @@ class BuiltinFunctionsTest(converter_testing.TestCase):
         with self.assertPrints('a\n'):
           sess.run(result.test_fn('a'))
 
+  @test_util.run_deprecated_v1
   def test_print_multiple_values(self):
 
     if six.PY2:
diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index 03fdfc804e..034fcbe386 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.autograph.pyct import transformer
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -36,6 +37,7 @@ class ControlFlowTest(converter_testing.TestCase):
       with self.cached_session() as sess:
         self.assertEqual(sess.run(result.test_fn(*inputs)), expected)
 
+  @test_util.run_deprecated_v1
   def test_while_basic(self):
 
     def test_fn(n):
@@ -48,6 +50,7 @@ class ControlFlowTest(converter_testing.TestCase):
 
     self.assertTransformedResult(test_fn, constant_op.constant(5), (10, 5, 5))
 
+  @test_util.run_deprecated_v1
   def test_while_nested(self):
 
     def test_fn(n):
@@ -66,6 +69,7 @@ class ControlFlowTest(converter_testing.TestCase):
     self.assertTransformedResult(test_fn, constant_op.constant(5),
                                  (25, 5, 0, 5))
 
+  @test_util.run_deprecated_v1
   def test_while_single_output(self):
 
     def test_fn(n):
@@ -86,6 +90,7 @@ class ControlFlowTest(converter_testing.TestCase):
     with self.assertRaises(NameError):
       control_flow.transform(node, ctx)
 
+  @test_util.run_deprecated_v1
   def test_if_basic(self):
 
     def test_fn(n):
@@ -100,6 +105,7 @@ class ControlFlowTest(converter_testing.TestCase):
     self.assertTransformedResult(test_fn, constant_op.constant(1), (-1, 0))
     self.assertTransformedResult(test_fn, constant_op.constant(-1), (0, -2))
 
+  @test_util.run_deprecated_v1
   def test_if_complex_outputs(self):
 
     class TestClass(object):
@@ -124,6 +130,7 @@ class ControlFlowTest(converter_testing.TestCase):
         res_obj = result.test_fn(constant_op.constant(-1), TestClass(0, 0))
         self.assertEqual(sess.run((res_obj.a, res_obj.b)), (0, -2))
 
+  @test_util.run_deprecated_v1
   def test_if_single_output(self):
 
     def test_fn(n):
@@ -133,6 +140,7 @@ class ControlFlowTest(converter_testing.TestCase):
 
     self.assertTransformedResult(test_fn, constant_op.constant(1), -1)
 
+  @test_util.run_deprecated_v1
   def test_if_semi(self):
 
     def test_fn(n):
@@ -143,6 +151,7 @@ class ControlFlowTest(converter_testing.TestCase):
     self.assertTransformedResult(test_fn, constant_op.constant(2), 3)
     self.assertTransformedResult(test_fn, constant_op.constant(-3), -3)
 
+  @test_util.run_deprecated_v1
   def test_if_local_var(self):
 
     def test_fn(n):
@@ -154,6 +163,7 @@ class ControlFlowTest(converter_testing.TestCase):
     self.assertTransformedResult(test_fn, constant_op.constant(1), 5)
     self.assertTransformedResult(test_fn, constant_op.constant(-1), -1)
 
+  @test_util.run_deprecated_v1
   def test_if_no_outputs(self):
 
     def test_fn(n):
@@ -177,6 +187,7 @@ class ControlFlowTest(converter_testing.TestCase):
     with self.assertRaises(transformer.AutographParseError):
       control_flow.transform(node, ctx)
 
+  @test_util.run_deprecated_v1
   def test_simple_for(self):
 
     def test_fn(l):
@@ -191,6 +202,7 @@ class ControlFlowTest(converter_testing.TestCase):
     empty_vector = constant_op.constant([], shape=(0,), dtype=dtypes.int32)
     self.assertTransformedResult(test_fn, empty_vector, (0, 0))
 
+  @test_util.run_deprecated_v1
   def test_for_single_output(self):
 
     def test_fn(l):
@@ -235,6 +247,7 @@ class ControlFlowTest(converter_testing.TestCase):
     with self.assertRaises(NameError):
       control_flow.transform(node, ctx)
 
+  @test_util.run_deprecated_v1
   def test_for_tuple_unpacking(self):
     def test_fn(x_list):
       z = tf.constant(0)  # pylint:disable=undefined-variable
diff --git a/tensorflow/python/autograph/converters/function_scopes_test.py b/tensorflow/python/autograph/converters/function_scopes_test.py
index e5ce03a109..5a1248c801 100644
--- a/tensorflow/python/autograph/converters/function_scopes_test.py
+++ b/tensorflow/python/autograph/converters/function_scopes_test.py
@@ -22,11 +22,13 @@ from tensorflow.python.autograph.converters import function_scopes
 from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class FunctionBodyTransformerTest(converter_testing.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_basic(self):
 
     def test_fn(l):
@@ -40,6 +42,7 @@ class FunctionBodyTransformerTest(converter_testing.TestCase):
       self.assertIn('test_fn/', result_op.op.name)
       self.assertEqual('Docstring.', result.test_fn.__doc__)
 
+  @test_util.run_deprecated_v1
   def test_multiline_docstring(self):
 
     tf = None
@@ -58,6 +61,7 @@ class FunctionBodyTransformerTest(converter_testing.TestCase):
       self.assertIn('First sentence.', result.test_fn.__doc__)
       self.assertIn('Second sentence.', result.test_fn.__doc__)
 
+  @test_util.run_deprecated_v1
   def test_nested_functions(self):
 
     def test_fn(l):
@@ -74,6 +78,7 @@ class FunctionBodyTransformerTest(converter_testing.TestCase):
       self.assertNotIn('inner_fn', first.op.name)
       self.assertIn('test_fn/inner_fn/', second.op.name)
 
+  @test_util.run_deprecated_v1
   def test_method(self):
 
     class TestClass(object):
diff --git a/tensorflow/python/autograph/converters/logical_expressions_test.py b/tensorflow/python/autograph/converters/logical_expressions_test.py
index 99db04a775..687412750e 100644
--- a/tensorflow/python/autograph/converters/logical_expressions_test.py
+++ b/tensorflow/python/autograph/converters/logical_expressions_test.py
@@ -21,11 +21,13 @@ from __future__ import print_function
 from tensorflow.python.autograph.converters import logical_expressions
 from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class LogicalExpressionTest(converter_testing.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_equals(self):
 
     def test_fn(a, b):
@@ -36,6 +38,7 @@ class LogicalExpressionTest(converter_testing.TestCase):
         self.assertTrue(sess.run(result.test_fn(constant_op.constant(1), 1)))
         self.assertFalse(sess.run(result.test_fn(constant_op.constant(1), 2)))
 
+  @test_util.run_deprecated_v1
   def test_bool_ops(self):
 
     def test_fn(a, b, c):
@@ -48,6 +51,7 @@ class LogicalExpressionTest(converter_testing.TestCase):
         self.assertFalse(
             sess.run(result.test_fn(constant_op.constant(True), False, True)))
 
+  @test_util.run_deprecated_v1
   def test_comparison(self):
 
     def test_fn(a, b, c, d):
diff --git a/tensorflow/python/autograph/converters/side_effect_guards_test.py b/tensorflow/python/autograph/converters/side_effect_guards_test.py
index f6d0f73b5b..645267e560 100644
--- a/tensorflow/python/autograph/converters/side_effect_guards_test.py
+++ b/tensorflow/python/autograph/converters/side_effect_guards_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.autograph.core import converter_testing
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
@@ -34,6 +35,7 @@ tf = None  # Will be replaced by a mock.
 
 class SideEffectGuardsTest(converter_testing.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_side_effect_on_return_only_variable(self):
 
     def test_fn(a):
@@ -75,6 +77,7 @@ class SideEffectGuardsTest(converter_testing.TestCase):
         # Right now it's 3 or 4 based on whether the read is synchronized.
         self.assertEqual(3, self.evaluate(v))
 
+  @test_util.run_deprecated_v1
   def test_side_effect_on_tensor(self):
 
     def test_fn(a):
diff --git a/tensorflow/python/autograph/core/errors_test.py b/tensorflow/python/autograph/core/errors_test.py
index 00c8a726ed..845a28a522 100644
--- a/tensorflow/python/autograph/core/errors_test.py
+++ b/tensorflow/python/autograph/core/errors_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.autograph.core import errors
 from tensorflow.python.autograph.pyct import origin_info
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors as tf_errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import tf_inspect
@@ -47,6 +48,7 @@ class RuntimeErrorsTest(test.TestCase):
                                     'test_comment')
     return loc, origin
 
+  @test_util.run_deprecated_v1
   def test_improved_errors_basic(self):
     loc, origin = self.fake_origin(zero_div, 2)
     zero_div_caller.ag_source_map = {loc: origin}
@@ -62,6 +64,7 @@ class RuntimeErrorsTest(test.TestCase):
       self.assertNotEqual('zero_div', function_name)
     self.assertIn(origin.as_frame(), set(cm.exception.custom_traceback))
 
+  @test_util.run_deprecated_v1
   def test_improved_errors_no_matching_lineno(self):
     loc, origin = self.fake_origin(zero_div, -1)
     zero_div_caller.ag_source_map = {loc: origin}
@@ -79,6 +82,7 @@ class RuntimeErrorsTest(test.TestCase):
       self.assertNotEqual('test_function_name', function_name)
     self.assertIn('zero_div', all_function_names)
 
+  @test_util.run_deprecated_v1
   def test_improved_errors_failures(self):
     loc, _ = self.fake_origin(zero_div, 2)
     zero_div_caller.ag_source_map = {loc: 'bogus object'}
diff --git a/tensorflow/python/autograph/core/function_wrapping_test.py b/tensorflow/python/autograph/core/function_wrapping_test.py
index 5e217055c7..7e21b979db 100644
--- a/tensorflow/python/autograph/core/function_wrapping_test.py
+++ b/tensorflow/python/autograph/core/function_wrapping_test.py
@@ -20,11 +20,13 @@ from __future__ import print_function
 
 from tensorflow.python.autograph.core import function_wrapping
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class FunctionWrappingTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_function_scope_name(self):
     with function_wrapping.function_scope('test_name'):
       t = constant_op.constant(1)
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index a0fa501fb8..66edda5119 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.autograph.impl import api
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.utils import py_func
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import sequential
 from tensorflow.python.keras.layers import core
 from tensorflow.python.ops import variables
@@ -44,6 +45,7 @@ class TestResource(str):
 
 class ApiTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_decorator_recurses(self):
 
     class TestClass(object):
@@ -66,6 +68,7 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
+  @test_util.run_deprecated_v1
   def test_decorator_does_not_recurse(self):
 
     class TestClass(object):
@@ -86,6 +89,7 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
+  @test_util.run_deprecated_v1
   def test_decorator_calls_unconverted_graph(self):
 
     class TestClass(object):
@@ -107,6 +111,7 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
+  @test_util.run_deprecated_v1
   def test_decorator_calls_unconverted_py_func(self):
 
     class TestClass(object):
@@ -133,6 +138,7 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], self.evaluate(x).tolist())
 
+  @test_util.run_deprecated_v1
   def test_decorator_calls_decorated(self):
 
     class TestClass(object):
@@ -172,6 +178,7 @@ class ApiTest(test.TestCase):
         list(tf_inspect.getfullargspec(tc.called_member)),
         list(tf_inspect.getfullargspec(tc.called_member_converted)))
 
+  @test_util.run_deprecated_v1
   def test_convert_call_site_decorator(self):
 
     class TestClass(object):
@@ -326,6 +333,7 @@ class ApiTest(test.TestCase):
                              constant_op.constant(0))
       self.assertTrue(self.evaluate(x))
 
+  @test_util.run_deprecated_v1
   def test_converted_call_no_user_code(self):
 
     def f(x):
@@ -400,6 +408,7 @@ class ApiTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       self.assertAllEqual(True, self.evaluate(x))
 
+  @test_util.run_deprecated_v1
   def test_to_graph_basic(self):
 
     def test_fn(x, s):
@@ -413,6 +422,7 @@ class ApiTest(test.TestCase):
       x = compiled_fn(constant_op.constant([4, 8]), 4)
       self.assertListEqual([1, 2], self.evaluate(x).tolist())
 
+  @test_util.run_deprecated_v1
   def test_to_graph_with_defaults(self):
 
     foo = 4
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 05b5660941..0a7d4b6402 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -22,12 +22,14 @@ from tensorflow.python.autograph.operators import control_flow
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 class ForLoopTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_tensor(self):
     s = control_flow.for_stmt(
         constant_op.constant([1, 2, 3, 4]),
@@ -45,6 +47,7 @@ class ForLoopTest(test.TestCase):
         init_state=(0,))
     self.assertEqual(10, s)
 
+  @test_util.run_deprecated_v1
   def test_dataset(self):
     to_int32 = lambda i: math_ops.cast(i, dtypes.int32)
     s = control_flow.for_stmt(
@@ -58,6 +61,7 @@ class ForLoopTest(test.TestCase):
 
 class WhileLoopTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_tensor(self):
     n = constant_op.constant(5)
     results = control_flow.while_stmt(
@@ -87,6 +91,7 @@ class IfStmtTest(test.TestCase):
     return control_flow.if_stmt(
         cond=cond, body=lambda: (1, 2), orelse=lambda: (-1, -2))
 
+  @test_util.run_deprecated_v1
   def test_tensor(self):
     with self.cached_session() as sess:
       t = self.single_return_if_stmt(constant_op.constant(True))
@@ -98,6 +103,7 @@ class IfStmtTest(test.TestCase):
     self.assertEqual(1, self.single_return_if_stmt(True))
     self.assertEqual(-1, self.single_return_if_stmt(False))
 
+  @test_util.run_deprecated_v1
   def test_tensor_multiple_returns(self):
     with self.cached_session() as sess:
       t = self.multi_return_if_stmt(constant_op.constant(True))
diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py
index 0433e3f130..9397b9acb8 100644
--- a/tensorflow/python/autograph/operators/data_structures_test.py
+++ b/tensorflow/python/autograph/operators/data_structures_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.autograph.operators import data_structures
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.platform import test
@@ -59,6 +60,7 @@ class ListTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertAllEqual(self.evaluate(t), [3, 4, 5])
 
+  @test_util.run_deprecated_v1
   def test_tf_tensor_list_new_illegal_input(self):
     with self.assertRaises(ValueError):
       data_structures.tf_tensor_list_new([3, 4.0])
@@ -104,6 +106,7 @@ class ListTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertAllEqual(self.evaluate(t), [[1, 2, 3]])
 
+  @test_util.run_deprecated_v1
   def test_append_tensorarray(self):
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l1 = data_structures.list_append(l, 1)
@@ -154,6 +157,7 @@ class ListTest(test.TestCase):
       t = data_structures.list_stack(l, opts)
       self.assertAllEqual(self.evaluate(t), self.evaluate(initial_list))
 
+  @test_util.run_deprecated_v1
   def test_stack_tensor_list_empty(self):
     l = list_ops.empty_tensor_list(
         element_shape=None, element_dtype=dtypes.variant)
diff --git a/tensorflow/python/autograph/operators/exceptions_test.py b/tensorflow/python/autograph/operators/exceptions_test.py
index 24d3f1bd35..21ba76bb95 100644
--- a/tensorflow/python/autograph/operators/exceptions_test.py
+++ b/tensorflow/python/autograph/operators/exceptions_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.autograph.operators import exceptions
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -32,6 +33,7 @@ class ExceptionsTest(test.TestCase):
           constant_op.constant(True), lambda: constant_op.constant('ignored'))
       self.evaluate(t)
 
+  @test_util.run_deprecated_v1
   def test_assert_tf_triggered(self):
     with self.cached_session() as sess:
       t = exceptions.assert_stmt(
@@ -42,6 +44,7 @@ class ExceptionsTest(test.TestCase):
                                    'test message'):
         self.evaluate(t)
 
+  @test_util.run_deprecated_v1
   def test_assert_tf_multiple_printed_values(self):
     two_tensors = [
         constant_op.constant('test message'),
diff --git a/tensorflow/python/autograph/operators/logical_test.py b/tensorflow/python/autograph/operators/logical_test.py
index ebf6458f01..e22f39932d 100644
--- a/tensorflow/python/autograph/operators/logical_test.py
+++ b/tensorflow/python/autograph/operators/logical_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.autograph.operators import logical
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -42,6 +43,7 @@ class LogicalOperatorsTest(test.TestCase):
     self.assertFalse(logical.and_(lambda: False, lambda: True))
     self.assertFalse(logical.and_(lambda: False, self.assertNotCalled))
 
+  @test_util.run_deprecated_v1
   def test_and_tf(self):
     with self.cached_session() as sess:
       t = logical.and_(self._tf_true, self._tf_true)
@@ -60,6 +62,7 @@ class LogicalOperatorsTest(test.TestCase):
     self.assertTrue(logical.or_(lambda: False, lambda: True))
     self.assertTrue(logical.or_(lambda: True, self.assertNotCalled))
 
+  @test_util.run_deprecated_v1
   def test_or_tf(self):
     with self.cached_session() as sess:
       t = logical.or_(self._tf_false, self._tf_true)
diff --git a/tensorflow/python/autograph/operators/py_builtins_test.py b/tensorflow/python/autograph/operators/py_builtins_test.py
index 4d9eec77c3..c856e39d14 100644
--- a/tensorflow/python/autograph/operators/py_builtins_test.py
+++ b/tensorflow/python/autograph/operators/py_builtins_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.platform import test
@@ -81,6 +82,7 @@ class PyBuiltinsTest(test.TestCase):
     with self.assertRaises(ValueError):
       py_builtins.len_(constant_op.constant(1))
 
+  @test_util.run_deprecated_v1
   def test_len_dynamic_shape(self):
     with self.cached_session() as sess:
       p = array_ops.placeholder(dtype=dtypes.int32, shape=None)
@@ -91,6 +93,7 @@ class PyBuiltinsTest(test.TestCase):
         t = py_builtins.len_(p)
         sess.run(t, {p: 1})
 
+  @test_util.run_deprecated_v1
   def test_print_tensors(self):
     try:
       out_capturer = six.StringIO()
@@ -101,6 +104,7 @@ class PyBuiltinsTest(test.TestCase):
     finally:
       sys.stdout = sys.__stdout__
 
+  @test_util.run_deprecated_v1
   def test_print_complex(self):
     try:
       out_capturer = six.StringIO()
diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py
index c813e0f5c9..c78df48d62 100644
--- a/tensorflow/python/autograph/utils/misc_test.py
+++ b/tensorflow/python/autograph/utils/misc_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.autograph.utils.misc import alias_tensors
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework.constant_op import constant
 from tensorflow.python.ops.variables import Variable
 from tensorflow.python.platform import test
@@ -26,6 +27,7 @@ from tensorflow.python.platform import test
 
 class MiscTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_alias_single_tensor(self):
     a = constant(1)
 
@@ -34,6 +36,7 @@ class MiscTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertEqual(1, self.evaluate(new_a))
 
+  @test_util.run_deprecated_v1
   def test_alias_tensors(self):
     a = constant(1)
     v = Variable(2)
diff --git a/tensorflow/python/autograph/utils/tensor_list_test.py b/tensorflow/python/autograph/utils/tensor_list_test.py
index c655f773b0..0bd3007249 100644
--- a/tensorflow/python/autograph/utils/tensor_list_test.py
+++ b/tensorflow/python/autograph/utils/tensor_list_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.client.session import Session
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework.constant_op import constant
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import tensor_array_ops
@@ -34,6 +35,7 @@ class TensorListTest(test.TestCase):
   def _shape(self, shape_tuple):
     return constant(shape_tuple, dtypes.int32)
 
+  @test_util.run_deprecated_v1
   def test_dynamic_list_append(self):
     l = []
     l = tl.dynamic_list_append(l, 1)
@@ -80,6 +82,7 @@ class TensorListTest(test.TestCase):
       l[0] = ops.convert_to_tensor(b)
       self.assertEqual(l[0].numpy(), b.numpy())
 
+  @test_util.run_deprecated_v1
   def test_list_append_tf(self):
     a = constant(3.0)
     l = tl.TensorList(a.shape, a.dtype)
diff --git a/tensorflow/python/autograph/utils/type_check_test.py b/tensorflow/python/autograph/utils/type_check_test.py
index b3d1304e16..2521dc9f92 100644
--- a/tensorflow/python/autograph/utils/type_check_test.py
+++ b/tensorflow/python/autograph/utils/type_check_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.platform import test
 
 class TypeCheckTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_checks(self):
     self.assertTrue(type_check.is_tensor(constant_op.constant([1, 2, 3])))
     self.assertTrue(
diff --git a/tensorflow/python/client/session_partial_run_test.py b/tensorflow/python/client/session_partial_run_test.py
index a9bd5ab7e0..a97930635a 100644
--- a/tensorflow/python/client/session_partial_run_test.py
+++ b/tensorflow/python/client/session_partial_run_test.py
@@ -188,6 +188,7 @@ class PartialRunTest(test_util.TensorFlowTestCase):
     r = sess.partial_run(h, [b], {})
     self.assertEqual([6.0], r)
 
+  @test_util.run_deprecated_v1
   def testInvalidPartialRunSetup(self):
     sess = session.Session()
     x = array_ops.placeholder(dtypes.float32, shape=[])
@@ -196,6 +197,7 @@ class PartialRunTest(test_util.TensorFlowTestCase):
         'specify at least one target to fetch or execute.'):
       sess.partial_run_setup(fetches=[], feeds=[x])
 
+  @test_util.run_deprecated_v1
   def testPartialRunSetupNoFeedsPassed(self):
     sess = session.Session()
     r1 = constant_op.constant([6.0])
@@ -204,80 +206,102 @@ class PartialRunTest(test_util.TensorFlowTestCase):
     result1 = sess.partial_run(h, r1)
     self.assertEqual([6.0], result1)
 
+  @test_util.run_deprecated_v1
   def testPartialRunDirect(self):
     self.RunTestPartialRun(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunIncompleteDirect(self):
     self.RunTestPartialRunIncomplete(session.Session())
 
+  @test_util.run_deprecated_v1
   def testConcurrentPartialRunDirect(self):
     self.RunTestConcurrentPartialRun(session.Session())
 
+  @test_util.run_deprecated_v1
   def testManyPartialRunDirect(self):
     self.RunTestManyPartialRun(session.Session())
 
+  @test_util.run_deprecated_v1
   def testRunAndPartialRunDirect(self):
     self.RunTestRunAndPartialRun(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunMissingPlaceholderFeedExceptionDirect(self):
     self.RunTestPartialRunMissingPlaceholderFeedException(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunUnspecifiedFeedDirect(self):
     self.RunTestPartialRunUnspecifiedFeed(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunUnspecifiedFetchDirect(self):
     self.RunTestPartialRunUnspecifiedFetch(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunAlreadyFedDirect(self):
     self.RunTestPartialRunAlreadyFed(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunAlreadyFetchedDirect(self):
     self.RunTestPartialRunAlreadyFetched(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunEmptyFetchesDirect(self):
     self.RunTestPartialRunEmptyFetches(session.Session())
 
+  @test_util.run_deprecated_v1
   def testPartialRunDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRun(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunIncompleteDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunIncomplete(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testConcurrentPartialRunDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestConcurrentPartialRun(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testManyPartialRunDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestManyPartialRun(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testRunAndPartialRunDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestRunAndPartialRun(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunMissingPlaceholderFeedExceptionDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunMissingPlaceholderFeedException(
         session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunUnspecifiedFeedDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunUnspecifiedFeed(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunUnspecifiedFetchDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunUnspecifiedFetch(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunAlreadyFedDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunAlreadyFed(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunAlreadyFetchedDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunAlreadyFetched(session.Session(server.target))
 
+  @test_util.run_deprecated_v1
   def testPartialRunEmptyFetchesDist(self):
     server = server_lib.Server.create_local_server()
     self.RunTestPartialRunEmptyFetches(session.Session(server.target))
diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index f9bd50957a..61c0da01b8 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -57,6 +57,7 @@ class TimelineTest(test.TestCase):
     ctf = tl.generate_chrome_trace_format()
     self._validateTrace(ctf)
 
+  @test_util.run_deprecated_v1
   def testTimelineCpu(self):
     run_options = config_pb2.RunOptions(
         trace_level=config_pb2.RunOptions.FULL_TRACE)
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index 7edaab81f4..b0516573f5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.util import compat as util_compat
 
 class CopyToDeviceTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testCopyToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -61,6 +62,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyToDeviceInt32(self):
     host_dataset = dataset_ops.Dataset.from_tensors([0, 1, 2, 3])
     device_dataset = host_dataset.apply(
@@ -86,6 +88,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyToSameDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -112,6 +115,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyToDeviceWithPrefetch(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -138,6 +142,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyDictToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
     device_dataset = host_dataset.apply(
@@ -164,6 +169,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyDictToDeviceWithPrefetch(self):
     host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
     device_dataset = host_dataset.apply(
@@ -190,6 +196,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopySparseTensorsToDevice(self):
 
     def make_tensor(i):
@@ -224,6 +231,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopySparseTensorsToDeviceWithPrefetch(self):
 
     def make_tensor(i):
@@ -426,6 +434,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         with self.assertRaises(errors.OutOfRangeError):
           self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyToDeviceWithReInit(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -456,6 +465,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testCopyToDeviceWithReInitAndPrefetch(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
diff --git a/tensorflow/python/data/experimental/kernel_tests/counter_test.py b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
index d1dd07a879..b370e0029e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/counter_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
@@ -20,11 +20,13 @@ from __future__ import print_function
 from tensorflow.python.data.experimental.ops import counter
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class CounterTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testCounter(self):
     """Test dataset construction using `count`."""
     iterator = (counter.Counter(start=3, step=4)
diff --git a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
index d9bbfb9c99..4b84446be8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
@@ -24,12 +24,14 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class DenseToSparseBatchTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testDenseToSparseBatchDataset(self):
     components = np.random.randint(12, size=(100,)).astype(np.int32)
     iterator = (
@@ -58,6 +60,7 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next)
 
+  @test_util.run_deprecated_v1
   def testDenseToSparseBatchDatasetWithUnknownShape(self):
     components = np.random.randint(5, size=(40,)).astype(np.int32)
     iterator = (
@@ -91,12 +94,14 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next)
 
+  @test_util.run_deprecated_v1
   def testDenseToSparseBatchDatasetWithInvalidShape(self):
     input_tensor = array_ops.constant([[1]])
     with self.assertRaisesRegexp(ValueError, "Dimension -2 must be >= 0"):
       dataset_ops.Dataset.from_tensors(input_tensor).apply(
           batching.dense_to_sparse_batch(4, [-2])).make_initializable_iterator()
 
+  @test_util.run_deprecated_v1
   def testDenseToSparseBatchDatasetShapeErrors(self):
     input_tensor = array_ops.placeholder(dtypes.int32)
     iterator = (
diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
index 768a8d774b..269c27dde2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
@@ -24,11 +24,13 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     selector_dataset = dataset_ops.Dataset.range(10).repeat(100)
     input_datasets = [
@@ -77,6 +79,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
 
     return freqs
 
+  @test_util.run_deprecated_v1
   def testSampleFromDatasets(self):
     random_seed.set_random_seed(1619)
     num_samples = 5000
@@ -96,6 +99,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
       freqs = self._testSampleFromDatasetsHelper(probs_ds, classes, num_samples)
       self.assertLess(self._chi2(probs, freqs / num_samples), 1e-2)
 
+  @test_util.run_deprecated_v1
   def testSelectFromDatasets(self):
     words = [b"foo", b"bar", b"baz"]
     datasets = [dataset_ops.Dataset.from_tensors(w).repeat() for w in words]
diff --git a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
index f32d1d0a6f..3c2e1bb7f3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
@@ -24,11 +24,13 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class EnumerateDatasetTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testEnumerateDataset(self):
     components = (["a", "b"], [1, 2], [37.0, 38])
     start = constant_op.constant(20, dtype=dtypes.int64)
diff --git a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
index 860442571e..a78fb4d229 100644
--- a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
@@ -107,11 +107,13 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
       self.assertEqual(elem, [5.0])
       self.evaluate(destroy_op)
 
+  @test_util.run_deprecated_v1
   def testSameDeviceCPU(self):
     self._prefetch_fn_helper_one_shot("same_device_cpu",
                                       "/job:localhost/replica:0/task:0/cpu:0",
                                       "/job:localhost/replica:0/task:0/cpu:0")
 
+  @test_util.run_deprecated_v1
   def testDifferentDeviceCPU(self):
     self._prefetch_fn_helper_one_shot("diff_device_cpu",
                                       "/job:localhost/replica:0/task:0/cpu:0",
@@ -125,6 +127,7 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
                                       "/job:localhost/replica:0/task:0/cpu:0",
                                       "/job:localhost/replica:0/task:0/gpu:0")
 
+  @test_util.run_deprecated_v1
   def testReinitialization(self):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
 
@@ -165,6 +168,7 @@ class FunctionBufferingResourceTest(test_base.DatasetTestBase):
       self.assertEqual(elem, [5.0])
       self.evaluate(destroy_op)
 
+  @test_util.run_deprecated_v1
   def testReinitializationOutOfRange(self):
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py b/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py
index 0147988c59..ef576563a1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/get_single_element_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -39,6 +40,7 @@ class GetSingleElementTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("MoreThanOne", 0, 2, errors.InvalidArgumentError,
        "Dataset had more than one element."),
   )
+  @test_util.run_deprecated_v1
   def testGetSingleElement(self, skip, take, error=None, error_msg=None):
     skip_t = array_ops.placeholder(dtypes.int64, shape=[])
     take_t = array_ops.placeholder(dtypes.int64, shape=[])
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
index f9856500c5..c7366f6641 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -44,6 +45,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next)
 
+  @test_util.run_deprecated_v1
   def testSum(self):
     reducer = grouping.Reducer(
         init_func=lambda _: np.int64(0),
@@ -55,6 +57,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       self.checkResults(
           dataset, shapes=tensor_shape.scalar(), values=[(i - 1) * i, i * i])
 
+  @test_util.run_deprecated_v1
   def testAverage(self):
 
     def reduce_fn(x, y):
@@ -72,6 +75,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       self.checkResults(
           dataset, shapes=tensor_shape.scalar(), values=[i - 1, i])
 
+  @test_util.run_deprecated_v1
   def testConcat(self):
     components = np.array(list("abcdefghijklmnopqrst")).view(np.chararray)
     reducer = grouping.Reducer(
@@ -88,6 +92,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
           shapes=tensor_shape.scalar(),
           values=[b"acegikmoqs" [:i], b"bdfhjlnprt" [:i]])
 
+  @test_util.run_deprecated_v1
   def testSparseSum(self):
     def _sparse(i):
       return sparse_tensor.SparseTensorValue(
@@ -105,6 +110,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
       self.checkResults(
           dataset, shapes=tensor_shape.scalar(), values=[(i - 1) * i, i * i])
 
+  @test_util.run_deprecated_v1
   def testChangingStateShape(self):
 
     def reduce_fn(x, _):
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
index d5a36e7cb1..1e54091c7d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import string_ops
@@ -49,6 +50,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
              32, (tensor_shape.TensorShape([]), tensor_shape.TensorShape(
                  [None]), tensor_shape.TensorShape([3])))))
 
+  @test_util.run_deprecated_v1
   def testSingleBucket(self):
 
     def _map_fn(v):
@@ -84,6 +86,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
       self.assertAllEqual(expected_unk_int64, bucketed_values[1])
       self.assertAllEqual(expected_vec3_str, bucketed_values[2])
 
+  @test_util.run_deprecated_v1
   def testEvenOddBuckets(self):
 
     def _map_fn(v):
@@ -141,6 +144,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
       self.assertAllEqual(expected_unk_int64, bucketed_values_odd[1])
       self.assertAllEqual(expected_vec3_str, bucketed_values_odd[2])
 
+  @test_util.run_deprecated_v1
   def testEvenOddBucketsFilterOutAllOdd(self):
 
     def _map_fn(v):
@@ -188,6 +192,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
       self.assertAllEqual(
           np.arange(64, 128, 2, dtype=np.int64), bucketed_values_even1["x"])
 
+  @test_util.run_deprecated_v1
   def testDynamicWindowSize(self):
     components = np.arange(100).astype(np.int64)
 
@@ -221,6 +226,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
 
       self.assertEqual(batches, 15)
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     components = np.random.randint(100, size=(200,)).astype(np.int64)
     iterator = (
@@ -248,6 +254,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
       self.assertGreaterEqual(num_full_batches, 24)
       self.assertTrue(all(c == 4 for c in counts[:num_full_batches]))
 
+  @test_util.run_deprecated_v1
   def testImmediateOutput(self):
     components = np.array(
         [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0], dtype=np.int64)
@@ -270,6 +277,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
         self.assertAllEqual([2, 2, 2, 2], self.evaluate(get_next))
         self.assertAllEqual([0, 0, 0, 0], self.evaluate(get_next))
 
+  @test_util.run_deprecated_v1
   def testSmallGroups(self):
     components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64)
     iterator = (
@@ -288,6 +296,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
       self.assertAllEqual([0, 0, 0], self.evaluate(get_next))
       self.assertAllEqual([1], self.evaluate(get_next))
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     iterator = (
         dataset_ops.Dataset.range(4).apply(
@@ -303,6 +312,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
           "Window size must be greater than zero, but got 0."):
         print(self.evaluate(get_next))
 
+  @test_util.run_deprecated_v1
   def testReduceFuncError(self):
     components = np.random.randint(100, size=(200,)).astype(np.int64)
 
@@ -327,6 +337,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(get_next)
 
+  @test_util.run_deprecated_v1
   def testConsumeWindowDatasetMoreThanOnce(self):
     components = np.random.randint(50, size=(200,)).astype(np.int64)
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
index 522b196060..bd323592e4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.data.experimental.ops import error_ops
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import io_ops
 from tensorflow.python.platform import test
@@ -35,6 +36,7 @@ _NUMPY_RANDOM_SEED = 42
 
 class IgnoreErrorsTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testMapIgnoreError(self):
     components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32)
 
@@ -53,6 +55,7 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next)
 
+  @test_util.run_deprecated_v1
   def testParallelMapIgnoreError(self):
     components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32)
 
@@ -71,6 +74,7 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next)
 
+  @test_util.run_deprecated_v1
   def testReadFileIgnoreError(self):
 
     def write_string_to_file(value, filename):
diff --git a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
index 0a436034a8..c3c4ccd077 100644
--- a/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/indexed_dataset_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.platform import test
@@ -31,6 +32,7 @@ from tensorflow.python.platform import test
 
 class IndexedDatasetOpsTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testLowLevelIndexedDatasetOps(self):
     identity = ged_ops.experimental_identity_indexed_dataset(
         ops.convert_to_tensor(16, dtype=dtypes.uint64))
@@ -49,6 +51,7 @@ class IndexedDatasetOpsTest(test_base.DatasetTestBase):
       self.evaluate(materialize)
       self.assertEqual([3], sess.run(get_op, feed_dict={index: 3}))
 
+  @test_util.run_deprecated_v1
   def testIdentityIndexedDataset(self):
     ds = indexed_dataset_ops.IdentityIndexedDataset(16)
     materialized = ds.materialize()
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
index 109b3696b8..48916471b3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
@@ -98,6 +99,7 @@ class MakeBatchedFeaturesDatasetTest(
             with self.assertRaises(errors.OutOfRangeError):
               self._next_actual_batch(sess)
 
+  @test_util.run_deprecated_v1
   def testReadWithEquivalentDataset(self):
     features = {
         "file": parsing_ops.FixedLenFeature([], dtypes.int64),
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index 1f509384d7..bcbaf1a7c4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -127,6 +128,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         self._verify_output(sess, dataset, batch_size, num_epochs, label_name,
                             expected_output, expected_keys)
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset(self):
     """Tests making a CSV dataset with keys and defaults provided."""
     record_defaults = [
@@ -158,6 +160,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withBatchSizeAndEpochs(self):
     """Tests making a CSV dataset with keys and defaults provided."""
     record_defaults = [
@@ -189,6 +192,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withCompressionType(self):
     """Tests `compression_type` argument."""
     record_defaults = [
@@ -257,6 +261,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           label_name="not_a_real_label",
           column_names=column_names)
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withNoLabel(self):
     """Tests making a CSV dataset with no label provided."""
     record_defaults = [
@@ -286,6 +291,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withNoHeader(self):
     """Tests that datasets can be created from CSV files with no header line.
     """
@@ -347,6 +353,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withNoColNames(self):
     """Tests that datasets can be created when column names are not specified.
 
@@ -451,6 +458,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         header=True,
     )
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withSelectCols(self):
     record_defaults = [
         constant_op.constant([], dtypes.int32),
@@ -557,6 +565,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           label_name=None,
           select_columns=["invalid_col_name"])
 
+  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withShuffle(self):
     record_defaults = [
         constant_op.constant([], dtypes.int32),
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index 8449c0651d..fc97f63358 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -48,6 +49,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("ParallelCallsNUMA", 2, None, True),
       ("ParallelBatchesNUMA", None, 10, True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatch(self, num_parallel_calls, num_parallel_batches,
                       numa_aware):
     """Test a dataset that maps a TF function across its input elements."""
@@ -132,6 +134,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("EvenNUMA", False, True),
       ("UnevenNUMA", True, True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchPartialBatch(self, drop_remainder, numa_aware):
     dataset = (
         dataset_ops.Dataset.range(10).apply(
@@ -163,6 +166,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchYieldsPartialBatch(self, numa_aware):
     dataset = (
         dataset_ops.Dataset.range(10).apply(
@@ -187,6 +191,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchParallelGetNext(self, numa_aware):
     dataset = dataset_ops.Dataset.range(50000).apply(
         batching.map_and_batch(lambda x: x, batch_size=100))
@@ -214,6 +219,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchParallelGetNextDropRemainder(self, numa_aware):
     dataset = dataset_ops.Dataset.range(49999).apply(
         batching.map_and_batch(
@@ -243,6 +249,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchSparse(self, numa_aware):
 
     def _sparse(i):
@@ -277,6 +284,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchFails(self, numa_aware):
     """Test a dataset that maps a TF function across its input elements."""
     dataset = dataset_ops.Dataset.from_tensors(
@@ -299,6 +307,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchShapeMismatch(self, numa_aware):
     """Test a dataset that maps a TF function across its input elements."""
 
@@ -370,6 +379,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("5NUMA", 95, True),
       ("6NUMA", 99, True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchOutOfRangeError(self, threshold, numa_aware):
 
     def raising_py_fn(i):
@@ -452,6 +462,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Swap", (None, None), lambda x, y: (y, x), None),
       ("Project", (None, None), lambda x, y: x, None),
   )
+  @test_util.run_deprecated_v1
   def testShortCircuit(self, structure, map_fn, num_parallel_calls):
     dataset = self.structuredDataset(structure).repeat().apply(
         batching.map_and_batch(map_fn, batch_size=10))
@@ -466,6 +477,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
             sess.run(self.structuredElement(structure, shape=[10])))
       self.assertAllEqual(expected, self.evaluate(get_next))
 
+  @test_util.run_deprecated_v1
   def testShortCircuitCapturedInput(self):
     captured_t = array_ops.placeholder(dtypes.int64, shape=[])
     dataset = self.structuredDataset(None).repeat().apply(
@@ -481,6 +493,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("Normal", False),
       ("NUMA", True),
   )
+  @test_util.run_deprecated_v1
   def testMapAndBatchControlFlow(self, numa_aware):
 
     def map_fn(x):
diff --git a/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
index 938dd4aff4..787a7a91a4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
@@ -24,6 +24,7 @@ import tempfile
 from tensorflow.python.data.experimental.ops import matching_files
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -40,6 +41,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
     for filename in filenames:
       open(os.path.join(self.tmp_dir, filename), 'a').close()
 
+  @test_util.run_deprecated_v1
   def testNonExistingDirectory(self):
     """Test the MatchingFiles dataset with a non-existing directory."""
 
@@ -51,6 +53,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.NotFoundError):
         sess.run(next_element)
 
+  @test_util.run_deprecated_v1
   def testEmptyDirectory(self):
     """Test the MatchingFiles dataset with an empty directory."""
 
@@ -61,6 +64,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.NotFoundError):
         sess.run(next_element)
 
+  @test_util.run_deprecated_v1
   def testSimpleDirectory(self):
     """Test the MatchingFiles dataset with a simple directory."""
 
@@ -83,6 +87,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  @test_util.run_deprecated_v1
   def testFileSuffixes(self):
     """Test the MatchingFiles dataset using the suffixes of filename."""
 
@@ -104,6 +109,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  @test_util.run_deprecated_v1
   def testFileMiddles(self):
     """Test the MatchingFiles dataset using the middles of filename."""
 
@@ -125,6 +131,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  @test_util.run_deprecated_v1
   def testNestedDirectories(self):
     """Test the MatchingFiles dataset with nested directories."""
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index 1dfe854f18..7116e7549f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
@@ -81,6 +82,7 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
       ("8", 4, 1),
       ("9", 4, 4),
   )
+  @test_util.run_deprecated_v1
   def testNumThreadsDeprecated(self, num_threads, max_intra_op_parallelism):
 
     def override_threadpool_fn(dataset):
@@ -107,6 +109,7 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
       ("11", 4, 4),
       ("12", None, None),
   )
+  @test_util.run_deprecated_v1
   def testNumThreads(self, num_threads, max_intra_op_parallelism):
 
     def override_threadpool_fn(dataset):
diff --git a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
index c74f754fef..76e0d4d72a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
@@ -144,6 +144,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
         expected_values=expected_output,
         create_iterator_twice=True)
 
+  @test_util.run_deprecated_v1
   def testEmptySerializedWithoutDefaultsShouldFail(self):
     input_features = {
         "st_a":
@@ -177,6 +178,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
         expected_err=(errors_impl.InvalidArgumentError,
                       "Feature: c \\(data type: float\\) is required"))
 
+  @test_util.run_deprecated_v1
   def testDenseNotMatchingShapeShouldFail(self):
     original = [
         example(features=features({
@@ -669,6 +671,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
     for batch_size in (1, 10, 20, 100, 256):
       self._testSerializedContainingVarLenDenseLargerBatch(batch_size)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSerializedShapeMismatch(self):
     aname = "a"
     bname = "b"
@@ -706,6 +709,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
         expected_err=(ValueError,
                       "Cannot reshape a tensor with 0 elements to shape"))
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingVarLenDense(self):
     aname = "a"
     bname = "b"
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index 8fc18e1ccd..56befc9e7d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.platform import test
 
 class PrefetchToDeviceTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testPrefetchToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -61,6 +62,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testPrefetchToSameDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -91,6 +93,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testPrefetchDictToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
     device_dataset = host_dataset.apply(
@@ -121,6 +124,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testPrefetchSparseTensorsToDevice(self):
     def make_tensor(i):
       return sparse_tensor.SparseTensorValue(
@@ -174,6 +178,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testPrefetchToDeviceWithReInit(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
diff --git a/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py b/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py
index 4c879dbae6..675209d189 100644
--- a/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import string_ops
@@ -63,6 +64,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase):
   @parameterized.named_parameters(
       ("InitialDistributionKnown", True),
       ("InitialDistributionUnknown", False))
+  @test_util.run_deprecated_v1
   def testDistribution(self, initial_known):
     classes = np.random.randint(5, size=(20000,))  # Uniformly sampled
     target_dist = [0.9, 0.05, 0.05, 0.0, 0.0]
@@ -97,6 +99,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase):
   @parameterized.named_parameters(
       ("OnlyInitial", True),
       ("NotInitial", False))
+  @test_util.run_deprecated_v1
   def testEdgeCasesSampleFromInitialDataset(self, only_initial_dist):
     init_dist = [0.5, 0.5]
     target_dist = [0.5, 0.5] if only_initial_dist else [0.0, 1.0]
@@ -122,6 +125,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase):
         while True:
           returned.append(sess.run(get_next))
 
+  @test_util.run_deprecated_v1
   def testRandomClasses(self):
     init_dist = [0.25, 0.25, 0.25, 0.25]
     target_dist = [0.0, 0.0, 0.0, 1.0]
diff --git a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
index 516e489d04..658e6120cf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
@@ -22,12 +22,14 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class RestructuredDatasetTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testRestructureDataset(self):
     components = (array_ops.placeholder(dtypes.int32),
                   (array_ops.placeholder(dtypes.int32, shape=[None]),
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index dc8a7bca27..43a765b9e0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -40,6 +40,7 @@ class ScanTest(test_base.DatasetTestBase):
     return dataset_ops.Dataset.from_tensors(0).repeat().apply(
         scan_ops.scan(start, scan_fn))
 
+  @test_util.run_deprecated_v1
   def testCount(self):
     def make_scan_fn(step):
       return lambda state, _: (state + step, state)
@@ -83,6 +84,7 @@ class ScanTest(test_base.DatasetTestBase):
     self.assertEqual(5, self.evaluate(next_element()))
     self.assertEqual(8, self.evaluate(next_element()))
 
+  @test_util.run_deprecated_v1
   def testSparseCount(self):
     def _sparse(i):
       return sparse_tensor.SparseTensorValue(
@@ -114,6 +116,7 @@ class ScanTest(test_base.DatasetTestBase):
         with self.assertRaises(errors.OutOfRangeError):
           self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testChangingStateShape(self):
     # Test the fixed-point shape invariant calculations: start with
     # initial values with known shapes, and use a scan function that
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
index 94393d6d4b..140ed517ef 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.estimator import model_fn
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -68,6 +69,7 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
   def _build_iterator_saver_hook(self, est):
     return iterator_ops.CheckpointInputPipelineHook(est)
 
+  @test_util.run_deprecated_v1
   def testReturnDatasetFromInputFn(self):
 
     def _input_fn():
@@ -80,6 +82,7 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
     est.train(_input_fn, steps=2, hooks=[self._build_iterator_saver_hook(est)])
     self.assertSequenceEqual(self._read_vars(est.model_dir), (4, 3))
 
+  @test_util.run_deprecated_v1
   def testBuildIteratorInInputFn(self):
 
     def _input_fn():
@@ -94,6 +97,7 @@ class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
     est.train(_input_fn, steps=2, hooks=[self._build_iterator_saver_hook(est)])
     self.assertSequenceEqual(self._read_vars(est.model_dir), (4, 3))
 
+  @test_util.run_deprecated_v1
   def testDoNotRestore(self):
 
     def _input_fn():
diff --git a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
index 2e8b93feaf..ce63da6bf9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -44,6 +45,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
           self.evaluate(get_next)
     return outputs
 
+  @test_util.run_deprecated_v1
   def testCorrectOutput(self):
     output = self._gen_outputs(lambda: self._build_ds(10), 100)
     self.assertSequenceEqual(
@@ -52,6 +54,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
     for i in range(5):
       self.assertSequenceEqual(sorted(output[i * 20:(i + 1) * 20]), range(20))
 
+  @test_util.run_deprecated_v1
   def testReshuffling(self):
     # Check that the output orders of different epochs are indeed different.
     output = self._gen_outputs(lambda: self._build_ds(10), 100)
@@ -60,17 +63,20 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
       epoch2 = output[(i + 1) * 20:(i + 2) * 20]
       self.assertNotEqual(epoch1, epoch2)
 
+  @test_util.run_deprecated_v1
   def testSameOrderForSameSeeds(self):
     output1 = self._gen_outputs(lambda: self._build_ds(10), 100)
     output2 = self._gen_outputs(lambda: self._build_ds(10), 100)
     self.assertEqual(output1, output2)
 
+  @test_util.run_deprecated_v1
   def testDifferentOrderForDifferentSeeds(self):
     output1 = self._gen_outputs(lambda: self._build_ds(10), 100)
     output2 = self._gen_outputs(lambda: self._build_ds(20), 100)
     self.assertNotEqual(output1, output2)
     self.assertEqual(sorted(output1), sorted(output2))
 
+  @test_util.run_deprecated_v1
   def testCountNone(self):
     output1 = self._gen_outputs(
         lambda: self._build_ds(10, count=None), 100, verify_exhausted=False)
@@ -79,6 +85,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
     self.assertNotEqual(output1, output2)
     self.assertEqual(sorted(output1), sorted(output2))
 
+  @test_util.run_deprecated_v1
   def testCountMinusOne(self):
     output1 = self._gen_outputs(
         lambda: self._build_ds(10, count=-1), 100, verify_exhausted=False)
diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
index 1a6d5522ef..8c9e8225d6 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.data.experimental.ops import sleep
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 _NUMPY_RANDOM_SEED = 42
@@ -30,6 +31,7 @@ _NUMPY_RANDOM_SEED = 42
 
 class SleepTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testSleep(self):
     sleep_microseconds = 100
     dataset = dataset_ops.Dataset.range(10).apply(
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index e816006933..b89aa20432 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.data.experimental.ops import stats_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -59,6 +60,7 @@ def function_apply_options(dataset, aggregator, prefix="", counter_prefix=""):
 )
 class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testBytesProduced(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).map(
@@ -85,6 +87,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasCount(summary_str, "bytes_produced", 100.0)
       self._assertSummaryHasSum(summary_str, "bytes_produced", expected_sum)
 
+  @test_util.run_deprecated_v1
   def testLatencyStats(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -105,6 +108,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasCount(
           self.evaluate(summary_t), "record_latency", 100.0)
 
+  @test_util.run_deprecated_v1
   def testPrefetchBufferUtilization(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).map(
@@ -132,6 +136,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasCount(summary_str, "Prefetch::buffer_utilization",
                                   100)
 
+  @test_util.run_deprecated_v1
   def testPrefetchBufferScalars(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(10).map(
@@ -154,6 +159,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testFilteredElementsStats(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(101).filter(
@@ -180,6 +186,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasScalarValue(
           self.evaluate(summary_t), "Filter::filtered_elements", 34.0)
 
+  @test_util.run_deprecated_v1
   def testMapBufferUtilization(self, dataset_transformation):
 
     def dataset_fn():
@@ -194,6 +201,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         dataset_transformation,
         function_processing_time=True)
 
+  @test_util.run_deprecated_v1
   def testMapAutoTuneBufferUtilization(self, dataset_transformation):
 
     def dataset_fn():
@@ -211,6 +219,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         dataset_transformation,
         function_processing_time=True)
 
+  @test_util.run_deprecated_v1
   def testInterleaveAutoTuneBufferUtilization(self, dataset_transformation):
 
     def dataset_fn():
@@ -227,6 +236,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     self._testParallelCallsStats(dataset_fn, "ParallelInterleaveV2", 10,
                                  dataset_transformation)
 
+  @test_util.run_deprecated_v1
   def testMapAndBatchAutoTuneBufferUtilization(self, dataset_transformation):
 
     def dataset_fn():
@@ -248,6 +258,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         check_elements=False,
         function_processing_time=True)
 
+  @test_util.run_deprecated_v1
   def testReinitialize(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -270,6 +281,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         self._assertSummaryHasCount(
             self.evaluate(summary_t), "record_latency", (j + 1) * 100.0)
 
+  @test_util.run_deprecated_v1
   def testNoAggregatorRegistered(self, dataset_transformation):
     dataset = dataset_ops.Dataset.range(100).apply(
         stats_ops.latency_stats("record_latency"))
@@ -283,6 +295,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testMultipleTags(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -308,6 +321,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasCount(
           self.evaluate(summary_t), "record_latency_2", 100.0)
 
+  @test_util.run_deprecated_v1
   def testRepeatedTags(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -329,6 +343,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasCount(
           self.evaluate(summary_t), "record_latency", 200.0)
 
+  @test_util.run_deprecated_v1
   def testMultipleIteratorsSameAggregator(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -350,6 +365,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
       self._assertSummaryHasCount(
           self.evaluate(summary_t), "record_latency", 200.0)
 
+  @test_util.run_deprecated_v1
   def testMultipleDatasetWithPrefixes(self, dataset_transformation):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_ops.Dataset.range(100).apply(
@@ -390,6 +406,7 @@ class FeatureStatsDatasetTest(
     stats_dataset_test_base.StatsDatasetTestBase,
     reader_dataset_ops_test_base.MakeBatchedFeaturesDatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testFeaturesStats(self, dataset_transformation):
     num_epochs = 5
     total_records = num_epochs * self._num_records
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index cb94bb4144..9c6830d993 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import string_ops
@@ -37,6 +38,7 @@ from tensorflow.python.util import compat
 
 class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testUnbatchWithUnknownRankInput(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     dataset = dataset_ops.Dataset.from_tensors(placeholder).apply(
@@ -51,6 +53,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_elem)
 
+  @test_util.run_deprecated_v1
   def testUnbatchScalarDataset(self):
     data = tuple([math_ops.range(10) for _ in range(3)])
     data = dataset_ops.Dataset.from_tensor_slices(data)
@@ -70,6 +73,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(op)
 
+  @test_util.run_deprecated_v1
   def testUnbatchDatasetWithStrings(self):
     data = tuple([math_ops.range(10) for _ in range(3)])
     data = dataset_ops.Dataset.from_tensor_slices(data)
@@ -90,6 +94,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(op)
 
+  @test_util.run_deprecated_v1
   def testUnbatchDatasetWithSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
         indices=[[i, i] for i in range(10)],
@@ -111,6 +116,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testUnbatchDatasetWithDenseAndSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
         indices=[[i, i] for i in range(10)],
@@ -133,6 +139,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testUnbatchSingleElementTupleDataset(self):
     data = tuple([(math_ops.range(10),) for _ in range(3)])
     data = dataset_ops.Dataset.from_tensor_slices(data)
@@ -152,6 +159,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(op)
 
+  @test_util.run_deprecated_v1
   def testUnbatchMultiElementTupleDataset(self):
     data = tuple([(math_ops.range(10 * i, 10 * i + 10),
                    array_ops.fill([10], "hi")) for i in range(3)])
@@ -173,6 +181,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(op)
 
+  @test_util.run_deprecated_v1
   def testUnbatchEmpty(self):
     data = dataset_ops.Dataset.from_tensors(
         (constant_op.constant([]), constant_op.constant([], shape=[0, 4]),
@@ -191,6 +200,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(ValueError):
       data.apply(batching.unbatch())
 
+  @test_util.run_deprecated_v1
   def testUnbatchDynamicShapeMismatch(self):
     ph1 = array_ops.placeholder(dtypes.int32, shape=[None])
     ph2 = array_ops.placeholder(dtypes.int32, shape=None)
diff --git a/tensorflow/python/data/experimental/kernel_tests/unique_test.py b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
index 91f4bc84e9..ddec968858 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unique_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -57,6 +58,7 @@ class UniqueTest(test_base.DatasetTestBase):
         with self.assertRaises(errors.OutOfRangeError):
           self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testSimpleInt(self):
     for dtype in [dtypes.int32, dtypes.int64]:
       self._testSimpleHelper(dtype, [
@@ -69,6 +71,7 @@ class UniqueTest(test_base.DatasetTestBase):
           ([[1, 1], [1, 1], [2, 2], [3, 3], [1, 1]], [[1, 1], [2, 2], [3, 3]]),
       ])
 
+  @test_util.run_deprecated_v1
   def testSimpleString(self):
     self._testSimpleHelper(dtypes.string, [
         ([], []),
diff --git a/tensorflow/python/data/kernel_tests/flat_map_test.py b/tensorflow/python/data/kernel_tests/flat_map_test.py
index 5f11c2e3a7..ff52821b10 100644
--- a/tensorflow/python/data/kernel_tests/flat_map_test.py
+++ b/tensorflow/python/data/kernel_tests/flat_map_test.py
@@ -60,6 +60,7 @@ class FlatMapTest(test_base.DatasetTestBase):
     self.assertDatasetProduces(dataset, expected_output=expected_output)
 
   # Note: no eager mode coverage, session specific test.
+  @test_util.run_deprecated_v1
   def testSkipEagerSharedResourceNestedFlatMapDataset(self):
     repeats = [[1, 2], [3, 4], [5, 0], [1, 7]]
     components = np.array(repeats, dtype=np.int64)
diff --git a/tensorflow/python/data/kernel_tests/from_generator_test.py b/tensorflow/python/data/kernel_tests/from_generator_test.py
index 4d82c2111c..d58e3c2364 100644
--- a/tensorflow/python/data/kernel_tests/from_generator_test.py
+++ b/tensorflow/python/data/kernel_tests/from_generator_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
@@ -69,6 +70,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorUsingFunction(self):
     def generator():
       for i in range(1, 100):
@@ -79,18 +81,21 @@ class FromGeneratorTest(test_base.DatasetTestBase):
     self._testFromGeneratorOneShot(generator, elem_sequence, 1)
     self._testFromGeneratorOneShot(generator, elem_sequence, 5)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorUsingList(self):
     generator = lambda: [[i] * i for i in range(1, 100)]
     elem_sequence = list(generator())
     self._testFromGenerator(generator, elem_sequence, 1)
     self._testFromGenerator(generator, elem_sequence, 5)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorUsingNdarray(self):
     generator = lambda: np.arange(100, dtype=np.int64)
     elem_sequence = list(generator())
     self._testFromGenerator(generator, elem_sequence, 1, output_types=np.int64)
     self._testFromGenerator(generator, elem_sequence, 5, output_types=np.int64)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorUsingGeneratorExpression(self):
     # NOTE(mrry): Generator *expressions* are not repeatable (or in
     # general reusable), because they eagerly evaluate the `for`
@@ -102,6 +107,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
     self._testFromGenerator(generator, elem_sequence, 1)
     self._testFromGenerator(generator, elem_sequence, 5)
 
+  @test_util.run_deprecated_v1
   def testFromMultipleConcurrentGenerators(self):
     num_inner_repeats = 5
     num_outer_repeats = 100
@@ -199,6 +205,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorImplicitConversion(self):
     def generator():
       yield [1]
@@ -223,6 +230,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorString(self):
     def generator():
       yield "foo"
@@ -243,6 +251,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorTypeError(self):
     def generator():
       yield np.array([1, 2, 3], dtype=np.int64)
@@ -266,6 +275,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorShapeError(self):
     def generator():
       yield np.array([1, 2, 3], dtype=np.int64)
@@ -289,6 +299,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorStructureError(self):
     def generator():
       yield 1, 2
@@ -317,6 +328,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorHeterogeneous(self):
     def generator():
       yield 1
@@ -335,6 +347,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorStopShort(self):
 
     def generator():
@@ -353,6 +366,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       self.assertAllEqual(0, sess.run(get_next))
       self.assertAllEqual(1, sess.run(get_next))
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorDestructorCalled(self):
     # Use an `Event` to signal that the generator has been deleted.
     event = threading.Event()
@@ -387,6 +401,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       # iterator terminates (and the generator iterator is deleted).
       self.assertTrue(event.is_set())
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorWithArgs(self):
 
     def flat_map_fn(elem):
@@ -414,6 +429,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testFromGeneratorWithTwoArgs(self):
 
     def flat_map_fn(elem, message):
@@ -446,6 +462,7 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testGeneratorDatasetFinalizeFunctionCalled(self):
     # NOTE(mrry): This test tests the internal `_GeneratorDataset`,
     # which affords more control over what the finalize function can do than
diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
index d23ac0ebe9..80ed26e7fb 100644
--- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.platform import test
 @test_util.run_all_in_graph_and_eager_modes
 class FromSparseTensorSlicesTest(test_base.DatasetTestBase):
 
+  @test_util.run_deprecated_v1
   def testSkipEagerFromSparseTensorSlices(self):
     """Test a dataset based on slices of a `tf.SparseTensor`."""
     st = array_ops.sparse_placeholder(dtypes.float64)
diff --git a/tensorflow/python/data/kernel_tests/from_tensors_test.py b/tensorflow/python/data/kernel_tests/from_tensors_test.py
index 2857817e14..ce70637572 100644
--- a/tensorflow/python/data/kernel_tests/from_tensors_test.py
+++ b/tensorflow/python/data/kernel_tests/from_tensors_test.py
@@ -154,6 +154,7 @@ class FromTensorsTest(test_base.DatasetTestBase):
     self.assertEquals(([], ([], []), []), dataset.output_shapes)
 
   # TODO(b/117581999): more specific shapes in eager mode.
+  @test_util.run_deprecated_v1
   def testSkipEagerNestedStructure(self):
     components = (np.array([1, 2, 3], dtype=np.int64), (np.array([4., 5.]),
                                                         np.array([6., 7.])),
diff --git a/tensorflow/python/data/kernel_tests/iterator_test.py b/tensorflow/python/data/kernel_tests/iterator_test.py
index de95a53e57..b836a6aecf 100644
--- a/tensorflow/python/data/kernel_tests/iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_test.py
@@ -55,6 +55,7 @@ from tensorflow.python.util import compat
 
 class IteratorTest(test.TestCase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoGradients(self):
     component = constant_op.constant([1.])
     side = constant_op.constant(0.)
@@ -65,6 +66,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     self.assertIsNone(gradients_impl.gradients(value, side)[0])
     self.assertIsNone(gradients_impl.gradients(value, [component, side])[0])
 
+  @test_util.run_deprecated_v1
   def testCapturingStateInOneShotRaisesException(self):
     var = variables.Variable(37.0, name="myvar")
     dataset = (
@@ -75,6 +77,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         "datasets that capture stateful objects.+myvar"):
       dataset.make_one_shot_iterator()
 
+  @test_util.run_deprecated_v1
   def testOneShotIterator(self):
     components = (np.arange(7),
                   np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
@@ -100,6 +103,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testOneShotIteratorCaptureByValue(self):
     components = (np.arange(7),
                   np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
@@ -162,6 +166,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testOneShotIteratorNonBlocking(self):
     dataset = dataset_ops.Dataset.from_tensors([1, 2, 3]).map(lambda x: x * x)
     iterator = dataset.make_one_shot_iterator()
@@ -200,6 +205,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
                        len([None for r in results if r is None]))
       self.assertAllEqual([[1, 4, 9]], [r for r in results if r is not None])
 
+  @test_util.run_deprecated_v1
   def testOneShotIteratorInitializerFails(self):
     # Define a dataset whose initialization will always fail.
     dataset = dataset_ops.Dataset.from_tensors(
@@ -280,6 +286,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testNotInitializedError(self):
     components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
     iterator = (
@@ -292,6 +299,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
                                    "iterator has not been initialized"):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testReinitializableIterator(self):
     dataset_3 = dataset_ops.Dataset.from_tensors(
         constant_op.constant([1, 2, 3]))
@@ -331,6 +339,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_deprecated_v1
   def testReinitializableIteratorWithFunctions(self):
 
     def g():
@@ -390,6 +399,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
               (constant_op.constant([1, 2, 3], dtype=dtypes.int64),
                constant_op.constant([4., 5., 6., 7.], dtype=dtypes.float64))))
 
+  @test_util.run_deprecated_v1
   def testIteratorStringHandle(self):
     dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
     dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40])
@@ -445,6 +455,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         sess.run(
             next_element, feed_dict={handle_placeholder: iterator_4_handle})
 
+  @test_util.run_deprecated_v1
   def testIteratorStringHandleFuture(self):
     with forward_compat.forward_compatibility_horizon(2018, 8, 4):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
@@ -508,6 +519,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
           sess.run(
               next_element, feed_dict={handle_placeholder: iterator_4_handle})
 
+  @test_util.run_deprecated_v1
   def testIteratorStringHandleReuseTensorObject(self):
     dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
     one_shot_iterator = dataset.make_one_shot_iterator()
@@ -536,6 +548,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     self.assertEqual("foo_1", handle_with_same_name.op.name)
     self.assertIsNot(handle_with_name, handle_with_same_name)
 
+  @test_util.run_deprecated_v1
   def testIteratorStringHandleError(self):
     dataset_int_scalar = (
         dataset_ops.Dataset.from_tensor_slices([1, 2, 3]).repeat())
@@ -576,6 +589,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
             feedable_int_vector.get_next(),
             feed_dict={handle_placeholder: handle_float_vector}))
 
+  @test_util.run_deprecated_v1
   def testRemoteIteratorUsingRemoteCallOpDirectSession(self):
     worker_config = config_pb2.ConfigProto()
     worker_config.device_count["CPU"] = 3
@@ -632,6 +646,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
                 target_placeholder: "/job:localhost/replica:0/task:0/cpu:1"
             })
 
+  @test_util.run_deprecated_v1
   def testRemoteIteratorUsingRemoteCallOpMultiWorkers(self):
     s1 = server_lib.Server.create_local_server()
     s2 = server_lib.Server.create_local_server()
@@ -739,6 +754,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
                 target_placeholder: "/job:localhost/replica:0/task:0/cpu:0"
             })
 
+  @test_util.run_deprecated_v1
   def testIncorrectIteratorRestore(self):
 
     def _path():
@@ -797,6 +813,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         with self.assertRaises(errors.InvalidArgumentError):
           sess.run(restore_op)
 
+  @test_util.run_deprecated_v1
   def testRepeatedGetNextWarning(self):
     iterator = dataset_ops.Dataset.range(10).make_one_shot_iterator()
     warnings.simplefilter("always")
diff --git a/tensorflow/python/data/kernel_tests/list_files_test.py b/tensorflow/python/data/kernel_tests/list_files_test.py
index 26c536086b..789f1ab6de 100644
--- a/tensorflow/python/data/kernel_tests/list_files_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_test.py
@@ -44,6 +44,7 @@ class ListFilesTest(test_base.DatasetTestBase):
       open(path.join(self.tmp_dir, filename), 'a').close()
 
   # Note: eager mode fails in assertion error same as initializer in graph mode.
+  @test_util.run_deprecated_v1
   def testSkipEagerEmptyDirectory(self):
     dataset = dataset_ops.Dataset.list_files(path.join(self.tmp_dir, '*'))
     self.assertDatasetProduces(dataset, expected_output=[])
diff --git a/tensorflow/python/data/kernel_tests/optional_test.py b/tensorflow/python/data/kernel_tests/optional_test.py
index 864013171d..c2c62e9423 100644
--- a/tensorflow/python/data/kernel_tests/optional_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_test.py
@@ -74,6 +74,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertAllEqual(expected.dense_shape,
                           self.evaluate(actual.dense_shape))
 
+  @test_util.run_deprecated_v1
   def testFromNone(self):
     value_structure = structure.TensorStructure(dtypes.float32, [])
     opt = optional_ops.Optional.none_from_structure(value_structure)
@@ -267,6 +268,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
        optional_ops.OptionalStructure(
            structure.TensorStructure(dtypes.float32, []))),
   )
+  @test_util.run_deprecated_v1
   def testSkipEagerOptionalStructure(self, tf_value_fn,
                                      expected_value_structure):
     tf_value = tf_value_fn()
@@ -322,6 +324,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
                     indices=[[0, 1], [1, 0]], values=[37.0, 42.0],
                     dense_shape=[2, 2])}, False),
   )
+  @test_util.run_deprecated_v1
   def testSkipEagerIteratorGetNextAsOptional(self, np_value, tf_value_fn,
                                              works_on_gpu):
     if not works_on_gpu and test.is_gpu_available():
diff --git a/tensorflow/python/data/kernel_tests/padded_batch_test.py b/tensorflow/python/data/kernel_tests/padded_batch_test.py
index 5f20d7b424..dcfb2f507b 100644
--- a/tensorflow/python/data/kernel_tests/padded_batch_test.py
+++ b/tensorflow/python/data/kernel_tests/padded_batch_test.py
@@ -93,6 +93,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
+  @test_util.run_deprecated_v1
   def testPaddedBatchShortPadding(self):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(
@@ -155,6 +156,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = self.getNext(padded_dataset)
     self.evaluate(next_element())
 
+  @test_util.run_deprecated_v1
   def testSkipEagerPaddedBatchDatasetShapeSpecifications(self):
     int_placeholder = array_ops.placeholder(dtypes.int32)
     float_placeholder = array_ops.placeholder(dtypes.float32)
@@ -226,6 +228,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       _ = dataset_ops.Dataset.range(10).padded_batch(
           5, padded_shapes=shape_as_tensor)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerPaddedBatchShapeError(self):
     with self.assertRaisesRegexp(
         ValueError,
diff --git a/tensorflow/python/data/kernel_tests/reduce_test.py b/tensorflow/python/data/kernel_tests/reduce_test.py
index d7b653961d..14bbc0bf72 100644
--- a/tensorflow/python/data/kernel_tests/reduce_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_test.py
@@ -68,6 +68,7 @@ class ReduceTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual(((i + 1) * i) // 2, s)
       self.assertEqual(i, c)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSquareUsingPlaceholder(self):
     delta = array_ops.placeholder(dtype=dtypes.int64)
 
diff --git a/tensorflow/python/data/kernel_tests/shuffle_test.py b/tensorflow/python/data/kernel_tests/shuffle_test.py
index 49460a1a4e..05d5d814c0 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_test.py
@@ -115,6 +115,7 @@ class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(get_next())
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSeedZero(self):
     """Test for same behavior when the seed is a Python or Tensor zero."""
     iterator = (
diff --git a/tensorflow/python/data/util/convert_test.py b/tensorflow/python/data/util/convert_test.py
index 3058e2b3f6..78ca6e9513 100644
--- a/tensorflow/python/data/util/convert_test.py
+++ b/tensorflow/python/data/util/convert_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.data.util import convert
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -60,6 +61,7 @@ class ConvertTest(test.TestCase):
                             convert.partial_shape_to_tensor(
                                 constant_op.constant([1], dtype=dtypes.int64))))
 
+  @test_util.run_deprecated_v1
   def testPartialShapeToTensorUnknownDimension(self):
     self.assertAllEqual([-1],
                         self.evaluate(
diff --git a/tensorflow/python/data/util/sparse_test.py b/tensorflow/python/data/util/sparse_test.py
index 4ba314f06a..06acf55ab9 100644
--- a/tensorflow/python/data/util/sparse_test.py
+++ b/tensorflow/python/data/util/sparse_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -296,6 +297,7 @@ class SparseTest(test.TestCase):
       self.assertAllEqual(a.eval().values, self.evaluate(b).values)
       self.assertAllEqual(a.eval().dense_shape, self.evaluate(b).dense_shape)
 
+  @test_util.run_deprecated_v1
   def testSerializeDeserialize(self):
     test_cases = (
         (),
@@ -325,6 +327,7 @@ class SparseTest(test.TestCase):
       for a, e in zip(nest.flatten(actual), nest.flatten(expected)):
         self.assertSparseValuesEqual(a, e)
 
+  @test_util.run_deprecated_v1
   def testSerializeManyDeserialize(self):
     test_cases = (
         (),
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index 65a41a50f1..e9e2f5be0a 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -115,6 +116,7 @@ class StructureTest(test.TestCase, parameterized.TestCase):
                   indices=[[0], [1], [2]], values=[4, 5, 6], dense_shape=[3])
       }, (constant_op.constant(15.0), constant_op.constant([4, 5, 6]))]),
   )
+  @test_util.run_deprecated_v1
   def testIsCompatibleWithStructure(
       self, original_value_fn, compatible_values_fn, incompatible_values_fn):
     original_value = original_value_fn()
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index 5aa7d1bb4c..322ecf9466 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -645,6 +645,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual(len("Size (B)") + 1, dump_size_col_width)
     self.assertEqual(len("Op type") + 1, op_type_col_width)
 
+  @test_util.run_deprecated_v1
   def testMeasureTensorListColumnWidthsGivesRightAnswerForData(self):
     dump = self._debug_dump.dumped_tensor_data[0]
     self.assertLess(dump.dump_size_bytes, 1000)
@@ -660,6 +661,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     # column should be determined by the length of "VariableV2".
     self.assertEqual(len("VariableV2") + 1, op_type_col_width)
 
+  @test_util.run_deprecated_v1
   def testListTensors(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", [])
@@ -673,6 +675,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     # Check the main menu.
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInReverseTimeOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "timestamp", "-r"])
@@ -688,6 +691,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=True)
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInDumpSizeOrderWorks(self):
     out = self._registry.dispatch_command("lt", ["-s", "dump_size"])
     assert_listed_tensors(
@@ -701,6 +705,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         sort_by="dump_size")
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInReverseDumpSizeOrderWorks(self):
     out = self._registry.dispatch_command("lt", ["-s", "dump_size", "-r"])
     assert_listed_tensors(
@@ -720,6 +725,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertIn("ValueError: Unsupported key to sort tensors by: foobar",
                   out.lines)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInOpTypeOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "op_type"])
@@ -735,6 +741,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=False)
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInReverseOpTypeOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "op_type", "-r"])
@@ -750,6 +757,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=True)
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInTensorNameOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "tensor_name"])
@@ -765,6 +773,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=False)
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsInReverseTensorNameOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "tensor_name", "-r"])
@@ -780,6 +789,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=True)
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorsFilterByNodeNameRegex(self):
     out = self._registry.dispatch_command("list_tensors",
                                           ["--node_name_filter", ".*read.*"])
@@ -793,6 +803,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     assert_listed_tensors(self, out, [], [], node_name_regex="^read")
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorFilterByOpTypeRegex(self):
     out = self._registry.dispatch_command("list_tensors",
                                           ["--op_type_filter", "Identity"])
@@ -821,6 +832,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         op_type_regex="(Add|MatMul)")
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  @test_util.run_deprecated_v1
   def testListTensorWithFilterAndNodeNameExclusionWorks(self):
     # First, create and register the filter.
     def is_2x1_vector(datum, tensor):
@@ -877,6 +889,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     out = self._registry.dispatch_command("list_tensors", ["--bar"])
     check_syntax_error_output(self, out, "list_tensors")
 
+  @test_util.run_deprecated_v1
   def testNodeInfoByNodeName(self):
     node_name = "simple_mul_add/matmul"
     out = self._registry.dispatch_command("node_info", [node_name])
@@ -901,6 +914,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         [(len(out.lines[0]) - len(node_name), len(out.lines[0]), "bold")],
         out.font_attr_segs[0])
 
+  @test_util.run_deprecated_v1
   def testNodeInfoShowAttributes(self):
     node_name = "simple_mul_add/matmul"
     out = self._registry.dispatch_command("node_info", ["-a", node_name])
@@ -924,6 +938,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         print_tensor_node_name=node_name,
         list_outputs_node_name=node_name)
 
+  @test_util.run_deprecated_v1
   def testNodeInfoShowDumps(self):
     node_name = "simple_mul_add/matmul"
     out = self._registry.dispatch_command("node_info", ["-d", node_name])
@@ -948,6 +963,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
                     len(out.lines[16]) - len(out.lines[16].strip()),
                     len(out.lines[16]), "pt %s:0 -n 0" % node_name)
 
+  @test_util.run_deprecated_v1
   def testNodeInfoShowStackTraceUnavailableIsIndicated(self):
     self._debug_dump.set_python_graph(None)
 
@@ -971,6 +987,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         print_tensor_node_name=node_name,
         list_outputs_node_name=node_name)
 
+  @test_util.run_deprecated_v1
   def testNodeInfoShowStackTraceAvailableWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
 
@@ -994,6 +1011,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         print_tensor_node_name=node_name,
         list_outputs_node_name=node_name)
 
+  @test_util.run_deprecated_v1
   def testNodeInfoByTensorName(self):
     node_name = "simple_mul_add/u/read"
     tensor_name = node_name + ":0"
@@ -1363,6 +1381,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         break
     return index
 
+  @test_util.run_deprecated_v1
   def testPrintSourceForOpNamesWholeFileWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1415,6 +1434,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual("pt simple_mul_add/add",
                      out.font_attr_segs[index + 1][0][2].content)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceForTensorNamesWholeFileWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1435,6 +1455,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual("pt simple_mul_add/u:0",
                      out.font_attr_segs[index + 2][0][2].content)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceForOpNamesStartingAtSpecifiedLineWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1461,6 +1482,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual("pt simple_mul_add/u/read",
                      out.font_attr_segs[index + 3][0][2].content)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceForOpNameSettingMaximumElementCountWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1505,6 +1527,7 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         self.assertTrue(cli_shared.COLOR_GRAY in attr_seg[2] or
                         attr_seg[2] == cli_shared.COLOR_GRAY)
 
+  @test_util.run_deprecated_v1
   def testListSourceWithNodeNameFilterWithMatchesWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command("list_source", ["-n", ".*/read"])
@@ -1719,6 +1742,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
     # Tear down temporary dump directory.
     shutil.rmtree(cls._dump_root)
 
+  @test_util.run_deprecated_v1
   def testNodeInfoWithControlDependencies(self):
     # Call node_info on a node with control inputs.
     out = self._registry.dispatch_command("node_info",
@@ -1759,6 +1783,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[z_line]),
                     "ni -a -d -t control_deps/ctrl_dep_z")
 
+  @test_util.run_deprecated_v1
   def testListInputsNonRecursiveNoControl(self):
     """List inputs non-recursively, without any control inputs."""
 
@@ -1801,6 +1826,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[3]) - len("control_deps/ctrl_dep_y"),
                     len(out.lines[3]), "li -c -r control_deps/ctrl_dep_y")
 
+  @test_util.run_deprecated_v1
   def testListInputsNonRecursiveNoControlUsingTensorName(self):
     """List inputs using the name of an output tensor of the node."""
 
@@ -1829,6 +1855,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[3]) - len("control_deps/ctrl_dep_y"),
                     len(out.lines[3]), "li -c -r control_deps/ctrl_dep_y")
 
+  @test_util.run_deprecated_v1
   def testListInputsNonRecursiveWithControls(self):
     """List inputs non-recursively, with control inputs."""
     node_name = "control_deps/ctrl_dep_z"
@@ -1859,6 +1886,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[5]) - len("control_deps/x"),
                     len(out.lines[5]), "li -c -r control_deps/x")
 
+  @test_util.run_deprecated_v1
   def testListInputsRecursiveWithControls(self):
     """List inputs recursively, with control inputs."""
     node_name = "control_deps/ctrl_dep_z"
@@ -1904,6 +1932,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[18]) - len("control_deps/x"),
                     len(out.lines[18]), "li -c -r control_deps/x")
 
+  @test_util.run_deprecated_v1
   def testListInputsRecursiveWithControlsWithDepthLimit(self):
     """List inputs recursively, with control inputs and a depth limit."""
     node_name = "control_deps/ctrl_dep_z"
@@ -1963,6 +1992,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
         "ERROR: There is no node named \"control_deps/z/foo\" in the "
         "partition graphs"], out.lines)
 
+  @test_util.run_deprecated_v1
   def testListRecipientsRecursiveWithControlsWithDepthLimit(self):
     """List recipients recursively, with control inputs and a depth limit."""
 
@@ -2034,6 +2064,7 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
     # Tear down temporary dump directory.
     shutil.rmtree(cls._dump_root)
 
+  @test_util.run_deprecated_v1
   def testMultipleDumpsPrintTensorNoNumber(self):
     output = self._registry.dispatch_command("pt", ["while/Identity:0"])
 
@@ -2051,6 +2082,7 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
     self.assertEqual("For example:", output.lines[-2])
     self.assertEqual("  print_tensor while/Identity:0 -n 0", output.lines[-1])
 
+  @test_util.run_deprecated_v1
   def testMultipleDumpsPrintTensorWithNumber(self):
     for i in xrange(5):
       output = self._registry.dispatch_command(
@@ -2064,6 +2096,7 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
       self.assertTrue(output.lines[4].startswith("array(%d" % i))
       self.assertTrue(output.lines[4].endswith(")"))
 
+  @test_util.run_deprecated_v1
   def testMultipleDumpsPrintTensorInvalidNumber(self):
     output = self._registry.dispatch_command("pt",
                                              ["while/Identity:0", "-n", "10"])
diff --git a/tensorflow/python/debug/cli/cli_shared_test.py b/tensorflow/python/debug/cli/cli_shared_test.py
index 07b364db9f..d191a234fd 100644
--- a/tensorflow/python/debug/cli/cli_shared_test.py
+++ b/tensorflow/python/debug/cli/cli_shared_test.py
@@ -118,6 +118,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
   def tearDown(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testSingleFetchNoFeeds(self):
     run_start_intro = cli_shared.get_run_start_intro(12, self.const_a, None, {})
 
@@ -181,6 +182,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     run_start_intro = cli_shared.get_run_start_intro(1, self.sparse_d, None, {})
     self.assertEqual(str(self.sparse_d), run_start_intro.lines[4].strip())
 
+  @test_util.run_deprecated_v1
   def testTwoFetchesListNoFeeds(self):
     fetches = [self.const_a, self.const_b]
     run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {})
@@ -197,6 +199,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     description = cli_shared.get_run_short_description(1, fetches, None)
     self.assertEqual("run #1: 2 fetches; 0 feeds", description)
 
+  @test_util.run_deprecated_v1
   def testNestedListAsFetches(self):
     fetches = [self.const_c, [self.const_a, self.const_b]]
     run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {})
@@ -210,6 +213,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     description = cli_shared.get_run_short_description(1, fetches, None)
     self.assertEqual("run #1: 3 fetches; 0 feeds", description)
 
+  @test_util.run_deprecated_v1
   def testNestedDictAsFetches(self):
     fetches = {"c": self.const_c, "ab": {"a": self.const_a, "b": self.const_b}}
     run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {})
@@ -227,6 +231,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     description = cli_shared.get_run_short_description(1, fetches, None)
     self.assertEqual("run #1: 3 fetches; 0 feeds", description)
 
+  @test_util.run_deprecated_v1
   def testTwoFetchesAsTupleNoFeeds(self):
     fetches = (self.const_a, self.const_b)
     run_start_intro = cli_shared.get_run_start_intro(1, fetches, None, {})
@@ -243,6 +248,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     description = cli_shared.get_run_short_description(1, fetches, None)
     self.assertEqual("run #1: 2 fetches; 0 feeds", description)
 
+  @test_util.run_deprecated_v1
   def testTwoFetchesAsNamedTupleNoFeeds(self):
     fetches_namedtuple = namedtuple("fetches", "x y")
     fetches = fetches_namedtuple(self.const_b, self.const_c)
@@ -260,6 +266,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     description = cli_shared.get_run_short_description(1, fetches, None)
     self.assertEqual("run #1: 2 fetches; 0 feeds", description)
 
+  @test_util.run_deprecated_v1
   def testWithFeedDict(self):
     feed_dict = {
         self.const_a: 10.0,
@@ -283,6 +290,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
                                                        feed_dict)
     self.assertEqual("run #1: 1 fetch (c:0); 2 feeds", description)
 
+  @test_util.run_deprecated_v1
   def testTensorFilters(self):
     feed_dict = {self.const_a: 10.0}
     tensor_filters = {
@@ -313,11 +321,13 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     command_set.add(annot[2].content)
     self.assertEqual({"run -f filter_a", "run -f filter_b"}, command_set)
 
+  @test_util.run_deprecated_v1
   def testGetRunShortDescriptionWorksForTensorFeedKey(self):
     short_description = cli_shared.get_run_short_description(
         1, self.const_a, {self.const_a: 42.0})
     self.assertEqual("run #1: 1 fetch (a:0); 1 feed (a:0)", short_description)
 
+  @test_util.run_deprecated_v1
   def testGetRunShortDescriptionWorksForUnicodeFeedKey(self):
     short_description = cli_shared.get_run_short_description(
         1, self.const_a, {u"foo": 42.0})
@@ -332,6 +342,7 @@ class GetErrorIntroTest(test_util.TensorFlowTestCase):
   def tearDown(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testShapeError(self):
     tf_error = errors.OpError(None, self.var_a.initializer, "foo description",
                               None)
diff --git a/tensorflow/python/debug/cli/profile_analyzer_cli_test.py b/tensorflow/python/debug/cli/profile_analyzer_cli_test.py
index 60b6047970..effcd500c7 100644
--- a/tensorflow/python/debug/cli/profile_analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/profile_analyzer_cli_test.py
@@ -348,6 +348,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
     ops.reset_default_graph()
     super(ProfileAnalyzerPrintSourceTest, self).tearDown()
 
+  @test_util.run_deprecated_v1
   def testPrintSourceForWhileLoop(self):
     prof_output = self.prof_analyzer.print_source([__file__])
 
@@ -361,6 +362,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         r"\[(\|)+(\s)*\] .*us .*7\(55\) .*L%d.*(\S)+" % self.loop_lineno,
         prof_output.lines)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceOutputContainsClickableLinks(self):
     prof_output = self.prof_analyzer.print_source([__file__])
     any_match, line_index = _at_least_one_line_matches(
@@ -377,6 +379,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         break
     self.assertTrue(any_menu_item_match)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceWithNonDefaultTimeUnit(self):
     prof_output = self.prof_analyzer.print_source([
         __file__, "--time_unit", "ms"])
@@ -391,6 +394,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         r"\[(\|)+(\s)*\] .*ms .*7\(55\) .*L%d.*(\S)+" % self.loop_lineno,
         prof_output.lines)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceWithNodeNameFilter(self):
     prof_output = self.prof_analyzer.print_source([
         __file__, "--node_name_filter", "x$"])
@@ -423,6 +427,7 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         break
     self.assertTrue(any_menu_item_match)
 
+  @test_util.run_deprecated_v1
   def testPrintSourceWithOpTypeFilter(self):
     prof_output = self.prof_analyzer.print_source([
         __file__, "--op_type_filter", "Less"])
diff --git a/tensorflow/python/debug/lib/common_test.py b/tensorflow/python/debug/lib/common_test.py
index 5af0dafcf9..f6413f6b7b 100644
--- a/tensorflow/python/debug/lib/common_test.py
+++ b/tensorflow/python/debug/lib/common_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.platform import googletest
 
 class CommonTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testOnFeedOneFetch(self):
     a = constant_op.constant(10.0, name="a")
     b = constant_op.constant(20.0, name="b")
@@ -35,6 +36,7 @@ class CommonTest(test_util.TensorFlowTestCase):
     self.assertItemsEqual(["a:0"], loaded[0])
     self.assertItemsEqual(["b:0"], loaded[1])
 
+  @test_util.run_deprecated_v1
   def testGetRunKeyFlat(self):
     a = constant_op.constant(10.0, name="a")
     b = constant_op.constant(20.0, name="b")
@@ -43,6 +45,7 @@ class CommonTest(test_util.TensorFlowTestCase):
     self.assertItemsEqual(["a:0"], loaded[0])
     self.assertItemsEqual(["a:0", "b:0"], loaded[1])
 
+  @test_util.run_deprecated_v1
   def testGetRunKeyNestedFetches(self):
     a = constant_op.constant(10.0, name="a")
     b = constant_op.constant(20.0, name="b")
diff --git a/tensorflow/python/debug/lib/debug_gradients_test.py b/tensorflow/python/debug/lib/debug_gradients_test.py
index 01867fc69d..1c53147863 100644
--- a/tensorflow/python/debug/lib/debug_gradients_test.py
+++ b/tensorflow/python/debug/lib/debug_gradients_test.py
@@ -54,6 +54,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     ops.reset_default_graph()
     debug_gradients.clear_gradient_debuggers()
 
+  @test_util.run_deprecated_v1
   def testIdentifyGradientGivesCorrectTensorObjectWithoutContextManager(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     id_grad_w = grad_debugger.identify_gradient(self.w)
@@ -84,6 +85,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
+  @test_util.run_deprecated_v1
   def testIdentifyGradientGivesCorrectTensorObjectWithTfGradients(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     id_grad_w = grad_debugger.identify_gradient(self.w)
@@ -115,6 +117,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
+  @test_util.run_deprecated_v1
   def testCallingIdentifyGradientTwiceWithTheSameGradientsDebuggerErrors(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     grad_debugger.identify_gradient(self.w)
@@ -122,6 +125,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
                                  "The graph already contains an op named .*"):
       grad_debugger.identify_gradient(self.w)
 
+  @test_util.run_deprecated_v1
   def testIdentifyGradientWorksOnMultipleLosses(self):
     grad_debugger_1 = debug_gradients.GradientsDebugger()
     grad_debugger_2 = debug_gradients.GradientsDebugger()
@@ -150,6 +154,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy))
     self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy))
 
+  @test_util.run_deprecated_v1
   def testIdentifyGradientRaisesLookupErrorForUnknownXTensor(self):
     grad_debugger_1 = debug_gradients.GradientsDebugger()
     grad_debugger_2 = debug_gradients.GradientsDebugger()
@@ -170,6 +175,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
         r"This GradientsDebugger has not received any gradient tensor for "):
       grad_debugger_2.gradient_tensor(self.w)
 
+  @test_util.run_deprecated_v1
   def testIdentifyGradientRaisesTypeErrorForNonTensorOrTensorNameInput(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     with self.assertRaisesRegexp(
@@ -178,6 +184,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
         r"has type .*Operation.*"):
       grad_debugger.gradient_tensor(variables.global_variables_initializer())
 
+  @test_util.run_deprecated_v1
   def testIdentifyGradientTensorWorksWithGradientDescentOptimizer(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     id_grad_w = grad_debugger.identify_gradient(self.w)
@@ -193,6 +200,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
+  @test_util.run_deprecated_v1
   def testWatchGradientsByXTensorNamesWorks(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -219,6 +227,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
+  @test_util.run_deprecated_v1
   def testWatchGradientsByXTensorNamesWorksWithoutContextManager(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -245,6 +254,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
+  @test_util.run_deprecated_v1
   def testWatchGradientsWorksOnRefTensor(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -263,6 +273,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(3.0, self.sess.run(
         grad_debugger.gradient_tensor("u:0")))
 
+  @test_util.run_deprecated_v1
   def testWatchGradientsWorksOnMultipleTensors(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -283,6 +294,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(3.0, self.sess.run(
         grad_debugger.gradient_tensor("u:0")))
 
+  @test_util.run_deprecated_v1
   def testWatchGradientsByXTensorsWorks(self):
     y = math_ops.add(self.w, -1.0, name="foo/y")
     z = math_ops.square(y, name="foo/z")
@@ -305,6 +317,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(10.0, self.sess.run(w_grad))
     self.assertAllClose(30.0, self.sess.run(u_grad))
 
+  @test_util.run_deprecated_v1
   def testWatchGradientsByTensorCanWorkOnMultipleLosses(self):
     y = math_ops.add(self.w, -1.0, name="y")
     z1 = math_ops.square(y, name="z1")
@@ -330,6 +343,7 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy))
     self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy))
 
+  @test_util.run_deprecated_v1
   def testGradientsValuesFromDumpWorks(self):
     y = math_ops.add(self.w, -1.0, name="y")
     z = math_ops.square(y, name="z")
diff --git a/tensorflow/python/debug/lib/debug_utils_test.py b/tensorflow/python/debug/lib/debug_utils_test.py
index 23ab98444c..cf59b30e3d 100644
--- a/tensorflow/python/debug/lib/debug_utils_test.py
+++ b/tensorflow/python/debug/lib/debug_utils_test.py
@@ -185,6 +185,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
     self.assertEqual(["file:///tmp/tfdbg_1", "file:///tmp/tfdbg_2"],
                      watch_0.debug_urls)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_allNodes(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -216,6 +217,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
     self.assertTrue("p1" in node_names)
     self.assertTrue("s" in node_names)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_nodeNameWhitelist(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -230,6 +232,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         sorted(["a1_init", "a1", "a1/Assign", "a1/read", "p1"]),
         sorted(node_names))
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_opTypeWhitelist(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -255,6 +258,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertEqual(["p1"], node_names)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_tensorDTypeWhitelist(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -267,6 +271,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertItemsEqual(["a1", "a1/Assign", "b", "b/Assign"], node_names)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_nodeNameAndTensorDTypeWhitelists(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -280,6 +285,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertItemsEqual(["a1", "a1/Assign"], node_names)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_nodeNameBlacklist(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -294,6 +300,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         sorted(["b_init", "b", "b/Assign", "b/read", "c", "s"]),
         sorted(node_names))
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_opTypeBlacklist(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -306,6 +313,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertEqual(sorted(["p1", "s"]), sorted(node_names))
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_nodeNameAndOpTypeBlacklists(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -319,6 +327,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertEqual(["s"], node_names)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_tensorDTypeBlacklists(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -335,6 +344,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
     self.assertNotIn("b/Assign", node_names)
     self.assertIn("s", node_names)
 
+  @test_util.run_deprecated_v1
   def testWatchGraph_nodeNameAndTensorDTypeBlacklists(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
diff --git a/tensorflow/python/debug/lib/session_debug_file_test.py b/tensorflow/python/debug/lib/session_debug_file_test.py
index 1874160dd6..f5f9ba29ab 100644
--- a/tensorflow/python/debug/lib/session_debug_file_test.py
+++ b/tensorflow/python/debug/lib/session_debug_file_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.debug.lib import debug_utils
 from tensorflow.python.debug.lib import session_debug_testlib
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
@@ -44,6 +45,7 @@ class SessionDebugFileTest(session_debug_testlib.SessionDebugTestBase):
     else:
       return os.path.join(self._dump_root, "run_%d" % run_number)
 
+  @test_util.run_deprecated_v1
   def testAllowsDifferentWatchesOnDifferentRuns(self):
     """Test watching different tensors on different runs of the same graph."""
 
diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py
index a16d68329a..9083297fdb 100644
--- a/tensorflow/python/debug/lib/source_utils_test.py
+++ b/tensorflow/python/debug/lib/source_utils_test.py
@@ -65,6 +65,7 @@ class GuessIsTensorFlowLibraryTest(test_util.TensorFlowTestCase):
     self.assertTrue(
         source_utils.guess_is_tensorflow_py_library(source_utils.__file__))
 
+  @test_util.run_deprecated_v1
   def testFileInPythonKernelsPathReturnsTrue(self):
     x = constant_op.constant(42.0, name="x")
     self.assertTrue(
diff --git a/tensorflow/python/distribute/all_reduce_test.py b/tensorflow/python/distribute/all_reduce_test.py
index 5bf983a1b2..2c6b853124 100644
--- a/tensorflow/python/distribute/all_reduce_test.py
+++ b/tensorflow/python/distribute/all_reduce_test.py
@@ -37,6 +37,7 @@ from tensorflow.python.platform import tf_logging
 
 class AllReduceTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testFlattenTensorsShapesDefined(self):
     x = array_ops.placeholder(types_pb2.DT_FLOAT, [None])
     with self.assertRaisesRegexp(ValueError,
@@ -100,6 +101,7 @@ class AllReduceTest(test_util.TensorFlowTestCase):
           input_tensors.append(array_ops.identity(t8))
     return input_tensors, device_names
 
+  @test_util.run_deprecated_v1
   def testBuildRingGatherPassStructure(self):
     # 1 worker, 1 device
     input_tensors, device_names = self._buildInput(1, 1)
@@ -170,6 +172,7 @@ class AllReduceTest(test_util.TensorFlowTestCase):
                     "subdiv=%d elapsed=%f" %
                     (num_workers, num_gpus, shape, subdiv, elapsed))
 
+  @test_util.run_deprecated_v1
   def testRingAllReduce(self):
     self._testRingAllReduce(1, 2, [], 1)
     self._testRingAllReduce(1, 2, [8], 1)
@@ -199,6 +202,7 @@ class AllReduceTest(test_util.TensorFlowTestCase):
     tf_logging.info("ShuffleAllReduce num_workers=%d num_gpus=%d shape=%s "
                     "elapsed=%f" % (num_workers, num_gpus, shape, elapsed))
 
+  @test_util.run_deprecated_v1
   def testShuffleAllReduce(self):
     self._testShuffleAllReduce(1, 2, [], 1)
     self._testShuffleAllReduce(1, 2, [8], 1)
@@ -225,6 +229,7 @@ class AllReduceTest(test_util.TensorFlowTestCase):
                     "shape=%s elapsed=%f" %
                     (num_workers, num_gpus, shape, elapsed))
 
+  @test_util.run_deprecated_v1
   def testRecursiveHDAllReduce(self):
     self._testRecursiveHDAllReduce(1, 2, [8])
     self._testRecursiveHDAllReduce(1, 2, [4, 4])
diff --git a/tensorflow/python/distribute/device_util_test.py b/tensorflow/python/distribute/device_util_test.py
index baecd43c8e..2f0d7ed3b3 100644
--- a/tensorflow/python/distribute/device_util_test.py
+++ b/tensorflow/python/distribute/device_util_test.py
@@ -21,11 +21,13 @@ from __future__ import print_function
 from tensorflow.python.distribute import device_util
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class DeviceUtilTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCurrentDeviceWithGlobalGraph(self):
     with ops.device("/cpu:0"):
       self.assertEqual(device_util.current(), "/device:CPU:0")
@@ -49,6 +51,7 @@ class DeviceUtilTest(test.TestCase):
         self.assertEqual(device_util.current(),
                          "/job:localhost/replica:0/task:0/device:CPU:0")
 
+  @test_util.run_deprecated_v1
   def testCanonicalizeWithoutDefaultDevice(self):
     self.assertEqual(
         device_util.canonicalize("/cpu:0"),
diff --git a/tensorflow/python/distribute/input_ops_test.py b/tensorflow/python/distribute/input_ops_test.py
index d5f41b7093..dcf946ba47 100644
--- a/tensorflow/python/distribute/input_ops_test.py
+++ b/tensorflow/python/distribute/input_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.distribute import input_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.lib.io import python_io
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
@@ -96,6 +97,7 @@ class AutoShardDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testTFRecordDataset(self):
     dataset = readers.TFRecordDataset(self._createTFRecordFiles())
     dataset = input_ops.auto_shard_dataset(
@@ -103,6 +105,7 @@ class AutoShardDatasetTest(test.TestCase):
 
     self._verifySimpleShardingOutput(dataset, self._record)
 
+  @test_util.run_deprecated_v1
   def testFlatMap(self):
     dataset = dataset_ops.Dataset.from_tensor_slices(
         self._createTFRecordFiles())
@@ -112,6 +115,7 @@ class AutoShardDatasetTest(test.TestCase):
 
     self._verifySimpleShardingOutput(dataset, self._record)
 
+  @test_util.run_deprecated_v1
   def testInterleave(self):
     dataset = dataset_ops.Dataset.from_tensor_slices(
         self._createTFRecordFiles())
@@ -124,6 +128,7 @@ class AutoShardDatasetTest(test.TestCase):
     # contain records in order of files.
     self._verifySimpleShardingOutput(dataset, self._record)
 
+  @test_util.run_deprecated_v1
   def testListfiles(self):
     filenames = self._createTFRecordFiles()
     file_pattern = filenames[0].rsplit(os.sep, 1)[0] + "/tf_record.*.txt"
@@ -144,6 +149,7 @@ class AutoShardDatasetTest(test.TestCase):
         self.evaluate(next_element)
       self.assertAllEqual(expected, actual)
 
+  @test_util.run_deprecated_v1
   def testComplexPipeline(self):
     # Setup a complex input pipeline.
     batch_size = 2
@@ -183,6 +189,7 @@ class AutoShardDatasetTest(test.TestCase):
 
       self.assertAllEqual(sorted(expected), sorted(actual))
 
+  @test_util.run_deprecated_v1
   def testZip(self):
     dataset1 = readers.TFRecordDataset(self._createTFRecordFiles())
     dataset2 = readers.TextLineDataset(self._createTextFiles())
@@ -193,6 +200,7 @@ class AutoShardDatasetTest(test.TestCase):
     record_fn = lambda r, f: (self._record(r, f), self._text_line(r, f))
     self._verifySimpleShardingOutput(dataset, record_fn)
 
+  @test_util.run_deprecated_v1
   def testConcat(self):
     dataset1 = readers.TFRecordDataset(self._createTFRecordFiles())
     dataset2 = readers.TextLineDataset(self._createTextFiles())
@@ -213,6 +221,7 @@ class AutoShardDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
+  @test_util.run_deprecated_v1
   def testTextLineReader(self):
     dataset = readers.TextLineDataset(self._createTextFiles())
     dataset = input_ops.auto_shard_dataset(
@@ -220,6 +229,7 @@ class AutoShardDatasetTest(test.TestCase):
 
     self._verifySimpleShardingOutput(dataset, self._text_line)
 
+  @test_util.run_deprecated_v1
   def testTextLineReaderWithFlatMap(self):
     dataset = dataset_ops.Dataset.from_tensor_slices(self._createTextFiles())
     dataset = dataset.flat_map(readers.TextLineDataset)
@@ -228,6 +238,7 @@ class AutoShardDatasetTest(test.TestCase):
 
     self._verifySimpleShardingOutput(dataset, self._text_line)
 
+  @test_util.run_deprecated_v1
   def testFixedLengthReader(self):
     dataset = readers.FixedLengthRecordDataset(
         self._createFixedLengthRecordFiles(), self._record_bytes)
@@ -236,6 +247,7 @@ class AutoShardDatasetTest(test.TestCase):
 
     self._verifySimpleShardingOutput(dataset, self._fixed_length_record)
 
+  @test_util.run_deprecated_v1
   def testFixedLengthReaderWithFlatMap(self):
     dataset = dataset_ops.Dataset.from_tensor_slices(
         self._createFixedLengthRecordFiles())
diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py
index 1ba596573f..9b83f57089 100644
--- a/tensorflow/python/eager/function_gradients_test.py
+++ b/tensorflow/python/eager/function_gradients_test.py
@@ -187,6 +187,7 @@ class FunctionGradientsTest(test.TestCase, parameterized.TestCase):
 
     self.assertAllEqual(2, g(constant_op.constant(2.)))
 
+  @test_util.run_deprecated_v1
   def testGraphModeEagerGradError(self):
     with context.graph_mode():
       def f():
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index a206b1f791..e47c5083a3 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1283,6 +1283,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       defined.get_concrete_function(
           tensor_spec.TensorSpec(shape=(3,), dtype=dtypes.float32))
 
+  @test_util.run_deprecated_v1
   def testInputSignatureForFunctionWithNonTensorInputsNotAllowed(self):
 
     def foo(a, training=True):
diff --git a/tensorflow/python/eager/graph_only_ops_test.py b/tensorflow/python/eager/graph_only_ops_test.py
index 3aedf5fee1..914b4d9a95 100644
--- a/tensorflow/python/eager/graph_only_ops_test.py
+++ b/tensorflow/python/eager/graph_only_ops_test.py
@@ -29,12 +29,14 @@ from tensorflow.python.platform import test
 
 class GraphOnlyOpsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testGraphZerosLike(self):
     x = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32)
     z_tf = graph_only_ops.graph_zeros_like(x)
     with self.cached_session():
       self.assertAllClose(np.zeros((2, 3)), self.evaluate(z_tf))
 
+  @test_util.run_deprecated_v1
   def testGraphPlaceholder(self):
     x_tf = graph_only_ops.graph_placeholder(dtypes.int32, shape=(1,))
     y_tf = math_ops.square(x_tf)
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 2c70d66810..daa0a3b3a4 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -170,6 +170,7 @@ class LazyColumnTest(test.TestCase):
         TypeError, '"key" must be either a "str" or "_FeatureColumn".'):
       builder.get(NotAFeatureColumn())
 
+  @test_util.run_deprecated_v1
   def test_expand_dim_rank_1_sparse_tensor_empty_batch(self):
     # empty 1-D sparse tensor:
     builder = _LazyBuilder(features={'a': sparse_tensor.SparseTensor(
@@ -185,6 +186,7 @@ class LazyColumnTest(test.TestCase):
 
 class NumericColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     a = fc._numeric_column('aaa')
     self.assertEqual('aaa', a.key)
@@ -263,6 +265,7 @@ class NumericColumnTest(test.TestCase):
         'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32)
     }, a._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_parse_example_no_default_value(self):
     price = fc._numeric_column('price', shape=[2])
     data = example_pb2.Example(features=feature_pb2.Features(
@@ -278,6 +281,7 @@ class NumericColumnTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
+  @test_util.run_deprecated_v1
   def test_parse_example_with_default_value(self):
     price = fc._numeric_column('price', shape=[2], default_value=11.)
     data = example_pb2.Example(features=feature_pb2.Features(
@@ -304,6 +308,7 @@ class NumericColumnTest(test.TestCase):
     with self.assertRaisesRegexp(TypeError, 'must be a callable'):
       fc._numeric_column('price', normalizer_fn='NotACallable')
 
+  @test_util.run_deprecated_v1
   def test_normalizer_fn_transform_feature(self):
 
     def _increment_two(input_tensor):
@@ -314,6 +319,7 @@ class NumericColumnTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual([[3., 4.], [7., 8.]], output[price].eval())
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor(self):
 
     def _increment_two(input_tensor):
@@ -333,6 +339,7 @@ class NumericColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'must be a Tensor'):
       price._transform_feature(builder)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]])
     a_copy = copy.deepcopy(a)
@@ -345,6 +352,7 @@ class NumericColumnTest(test.TestCase):
         'aaa', shape=[1, 2], default_value=np.array([[3., 2.]]))
     self.assertEqual(a.default_value, ((3., 2.),))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     price = fc._numeric_column('price')
     with ops.Graph().as_default():
@@ -359,6 +367,7 @@ class NumericColumnTest(test.TestCase):
         sess.run(price_var.assign([[10.]]))
         self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     price = fc._numeric_column('price')
     with ops.Graph().as_default():
@@ -433,6 +442,7 @@ class BucketizedColumnTest(test.TestCase):
     # Column 'aaa` has shape [2] times three buckets -> num_buckets=6.
     self.assertEqual(6, b._num_buckets)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     price = fc._numeric_column('price', shape=[2])
     bucketized_price = fc._bucketized_column(price, boundaries=[0, 50])
@@ -449,6 +459,7 @@ class BucketizedColumnTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual([[20., 110.]], features['price'].eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     price = fc._numeric_column('price', shape=[2])
     bucketized_price = fc._bucketized_column(price, boundaries=[0, 2, 4, 6])
@@ -531,6 +542,7 @@ class BucketizedColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'must be a Tensor'):
       bucketized_price._transform_feature(builder)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc._numeric_column('aaa', shape=[2])
     a_bucketized = fc._bucketized_column(a, boundaries=[0, 1])
@@ -658,6 +670,7 @@ class BucketizedColumnTest(test.TestCase):
 
 class HashedCategoricalColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     a = fc._categorical_column_with_hash_bucket('aaa', 10)
     self.assertEqual('aaa', a.name)
@@ -685,6 +698,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
       fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc._categorical_column_with_hash_bucket('aaa', 10)
     for column in (original, copy.deepcopy(original)):
@@ -705,6 +719,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, a._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_hash_bucket('aaa', 10)
     data = example_pb2.Example(features=feature_pb2.Features(
@@ -726,6 +741,7 @@ class HashedCategoricalColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_strings_should_be_hashed(self):
     hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     wire_tensor = sparse_tensor.SparseTensor(
@@ -781,6 +797,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'):
       builder.get(hashed_sparse)
 
+  @test_util.run_deprecated_v1
   def test_ints_should_be_hashed(self):
     hashed_sparse = fc._categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
@@ -795,6 +812,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual(expected_values, output.values.eval())
 
+  @test_util.run_deprecated_v1
   def test_int32_64_is_compatible(self):
     hashed_sparse = fc._categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
@@ -809,6 +827,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual(expected_values, output.values.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     builder = _LazyBuilder({
@@ -837,6 +856,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10)
     builder = _LazyBuilder({'wire': (('omar', ''), ('stringer', 'marlo'))})
@@ -844,6 +864,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     self.assertIsNone(id_weight_pair.weight_tensor)
     self.assertEqual(builder.get(hashed_sparse), id_weight_pair.id_tensor)
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     wire_column = fc._categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column._num_buckets)
@@ -866,6 +887,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
         self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     wire_column = fc._categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column._num_buckets)
@@ -975,6 +997,7 @@ class CrossedColumnTest(test.TestCase):
     crossed = fc._crossed_column([b, 'c'], 15)
     self.assertEqual(15, crossed._num_buckets)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc._numeric_column('a', dtype=dtypes.int32)
     b = fc._bucketized_column(a, boundaries=[0, 1])
@@ -985,6 +1008,7 @@ class CrossedColumnTest(test.TestCase):
     self.assertEqual(15, crossed2_copy.hash_bucket_size)
     self.assertEqual(5, crossed2_copy.hash_key)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     price = fc._numeric_column('price', shape=[2])
     bucketized_price = fc._bucketized_column(price, boundaries=[0, 50])
@@ -1011,6 +1035,7 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual([b'omar', b'stringer'], wire_sparse.values.eval())
       self.assertAllEqual([1, 2], wire_sparse.dense_shape.eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     price = fc._numeric_column('price', shape=[2])
     bucketized_price = fc._bucketized_column(price, boundaries=[0, 50])
@@ -1034,6 +1059,7 @@ class CrossedColumnTest(test.TestCase):
         self.assertIn(val, list(range(hash_bucket_size)))
       self.assertAllEqual([2, 4], output_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,))
     b = fc._bucketized_column(a, boundaries=(0, 1))
@@ -1101,6 +1127,7 @@ class CrossedColumnTest(test.TestCase):
         self.assertAllEqual(expected_values, id_tensor_eval.values)
         self.assertAllEqual((2, 4), id_tensor_eval.dense_shape)
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     """Tests linear_model.
 
@@ -1182,6 +1209,7 @@ class CrossedColumnTest(test.TestCase):
                 dense_shape=(2, 2)),
         }, (crossed,))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     """Tests _LinearModel.
 
@@ -1854,6 +1882,7 @@ class LinearModelTest(test.TestCase):
                 features['price2']: [[1.], [5.]],
             })
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     price = fc._numeric_column('price')
     price_buckets = fc._bucketized_column(
@@ -1889,6 +1918,7 @@ class LinearModelTest(test.TestCase):
       self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
                           self.evaluate(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc._numeric_column('price')
     price_buckets = fc._bucketized_column(
@@ -1936,6 +1966,7 @@ class LinearModelTest(test.TestCase):
                                   features['country']: country_data
                               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     price = fc._numeric_column('price')
     features = {
@@ -2488,6 +2519,7 @@ class _LinearModelTest(test.TestCase):
                 features['price2']: [[1.], [5.]],
             })
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     price = fc._numeric_column('price')
     price_buckets = fc._bucketized_column(
@@ -2529,6 +2561,7 @@ class _LinearModelTest(test.TestCase):
       self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
                           self.evaluate(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc._numeric_column('price')
     price_buckets = fc._bucketized_column(
@@ -2575,6 +2608,7 @@ class _LinearModelTest(test.TestCase):
                                   features['country']: country_data
                               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     price = fc._numeric_column('price')
     features = {
@@ -2815,6 +2849,7 @@ class FunctionalInputLayerTest(test.TestCase):
                             variables_lib.Variable)
       self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10])
 
+  @test_util.run_deprecated_v1
   def test_fills_cols_to_vars_shared_embedding(self):
     # Provide 5 DenseColumn's to input_layer: a NumericColumn, a
     # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The
@@ -3012,6 +3047,7 @@ class FunctionalInputLayerTest(test.TestCase):
           expected_var_names,
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_multiple_layers_with_same_shared_embedding_column(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -3045,6 +3081,7 @@ class FunctionalInputLayerTest(test.TestCase):
           ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'],
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -3096,6 +3133,7 @@ class FunctionalInputLayerTest(test.TestCase):
           ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'],
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     embedding_values = (
         (1., 2., 3., 4., 5.),  # id 0
@@ -3146,6 +3184,7 @@ class FunctionalInputLayerTest(test.TestCase):
            [1., 0., 0., 1., 2., 3., 4., 5., 12.]],
           sess.run(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     embedding_values = (
         (1., 2.),  # id 0
@@ -3205,6 +3244,7 @@ class FunctionalInputLayerTest(test.TestCase):
                   features['country']: country_data
               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     # price has 1 dimension in input_layer
     price = fc._numeric_column('price')
@@ -3335,6 +3375,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         'python/feature_column/testdata/wire_vocabulary.txt')
     self._wire_vocabulary_size = 3
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
@@ -3351,6 +3392,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       fc._categorical_column_with_vocabulary_file(
           key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3363,6 +3405,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3387,6 +3430,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       fc._categorical_column_with_vocabulary_file(
           key='aaa', vocabulary_file='', vocabulary_size=3)
 
+  @test_util.run_deprecated_v1
   def test_invalid_vocabulary_file(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10)
@@ -3411,6 +3455,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=0)
 
+  @test_util.run_deprecated_v1
   def test_too_large_vocabulary_size(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3477,6 +3522,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
@@ -3499,6 +3545,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3519,6 +3566,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_none_vocabulary_size(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file=self._wire_vocabulary_file_name)
@@ -3537,6 +3585,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
                                       dense_shape=inputs.dense_shape),
                                   id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3573,6 +3622,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3592,6 +3642,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=(2, 2)),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3613,6 +3664,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_oov_buckets(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3634,6 +3686,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_small_vocabulary_size(self):
     # 'marlo' is the last entry in our vocabulary file, so be setting
     # `vocabulary_size` to 1 less than number of entries in file, we take
@@ -3657,6 +3710,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3678,6 +3732,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
     column = fc._categorical_column_with_vocabulary_file(
@@ -3700,6 +3755,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=(3, 3)),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
     column = fc._categorical_column_with_vocabulary_file(
         key='aaa',
@@ -3722,6 +3778,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     wire_column = fc._categorical_column_with_vocabulary_file(
         key='wire',
@@ -3748,6 +3805,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     wire_column = fc._categorical_column_with_vocabulary_file(
         key='wire',
@@ -3805,6 +3863,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int64)
     }, column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -3816,6 +3875,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32)
@@ -3904,6 +3964,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
+  @test_util.run_deprecated_v1
   def test_parse_example_string(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -3926,6 +3987,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_parse_example_int(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(11, 21, 31))
@@ -3948,6 +4010,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -3966,6 +4029,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -3998,6 +4062,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -4015,6 +4080,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=(2, 2)),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4035,6 +4101,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_oov_buckets(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4055,6 +4122,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4075,6 +4143,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
     column = fc._categorical_column_with_vocabulary_list(
@@ -4098,6 +4167,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=(3, 3)),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
     column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4119,6 +4189,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     wire_column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4144,6 +4215,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
         self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     wire_column = fc._categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4187,6 +4259,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
       fc._categorical_column_with_identity(key=('aaa',), num_buckets=3)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     for column in (original, copy.deepcopy(original)):
@@ -4223,6 +4296,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'Invalid input, not integer'):
       column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_identity(key='aaa', num_buckets=30)
     data = example_pb2.Example(features=feature_pb2.Features(
@@ -4244,6 +4318,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -4261,6 +4336,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -4291,6 +4367,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         [], ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES))
     self.assertItemsEqual([], ops.get_collection('my_weights'))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     id_weight_pair = column._get_sparse_tensors(
@@ -4307,6 +4384,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               dense_shape=(2, 2)),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_inputs_too_small(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -4320,6 +4398,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           errors.OpError, 'assert_greater_or_equal_0'):
         id_weight_pair.id_tensor.eval()
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_inputs_too_big(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -4333,6 +4412,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
           errors.OpError, 'assert_less_than_num_buckets'):
         id_weight_pair.id_tensor.eval()
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_default_value(self):
     column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
@@ -4351,6 +4431,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               dense_shape=inputs.dense_shape),
           id_weight_pair.id_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self):
     column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
@@ -4376,6 +4457,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               input_shape: (2, 2),
           }))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column._num_buckets)
@@ -4397,6 +4479,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
         # weight_var[2] + weight_var[1] = 3+2 = 5
         self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     column = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column._num_buckets)
@@ -4548,6 +4631,7 @@ class IndicatorColumnTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc._categorical_column_with_hash_bucket('a', 4)
     column = fc._indicator_column(a)
@@ -4556,6 +4640,7 @@ class IndicatorColumnTest(test.TestCase):
     self.assertEqual(column.name, 'a_indicator')
     self.assertEqual(column._variable_shape, [1, 4])
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -4579,6 +4664,7 @@ class IndicatorColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_transform(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -4594,6 +4680,7 @@ class IndicatorColumnTest(test.TestCase):
       self.assertAllEqual([[0, 0, 1], [1, 0, 0]],
                           self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_with_weighted_column(self):
     # Github issue 12557
     ids = fc._categorical_column_with_vocabulary_list(
@@ -4608,6 +4695,7 @@ class IndicatorColumnTest(test.TestCase):
     with _initialized_session():
       self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
     ids = fc._categorical_column_with_vocabulary_list(
@@ -4622,6 +4710,7 @@ class IndicatorColumnTest(test.TestCase):
     with _initialized_session():
       self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
     ids = fc._categorical_column_with_vocabulary_list(
@@ -4634,6 +4723,7 @@ class IndicatorColumnTest(test.TestCase):
     with _initialized_session():
       self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     animal = fc._indicator_column(
         fc._categorical_column_with_identity('animal', num_buckets=4))
@@ -4653,6 +4743,7 @@ class IndicatorColumnTest(test.TestCase):
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     animal = fc._indicator_column(
         fc._categorical_column_with_identity('animal', num_buckets=4))
@@ -4672,6 +4763,7 @@ class IndicatorColumnTest(test.TestCase):
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
         self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_input_layer(self):
     animal = fc._indicator_column(
         fc._categorical_column_with_identity('animal', num_buckets=4))
@@ -4688,6 +4780,7 @@ class IndicatorColumnTest(test.TestCase):
 
 class EmbeddingColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -4709,6 +4802,7 @@ class EmbeddingColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -4737,6 +4831,7 @@ class EmbeddingColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -4770,6 +4865,7 @@ class EmbeddingColumnTest(test.TestCase):
           'aaa': parsing_ops.VarLenFeature(dtypes.int64)
       }, embedding_column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_invalid_initializer(self):
     categorical_column = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -4777,6 +4873,7 @@ class EmbeddingColumnTest(test.TestCase):
       fc._embedding_column(
           categorical_column, dimension=2, initializer='not_fn')
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -4800,6 +4897,7 @@ class EmbeddingColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['aaa'].eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     a = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     a_embedded = fc._embedding_column(a, dimension=2)
@@ -4816,6 +4914,7 @@ class EmbeddingColumnTest(test.TestCase):
       _assert_sparse_tensor_value(self, self.evaluate(output_a),
                                   self.evaluate(output_embedded))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor(self):
     # Inputs.
     vocabulary_size = 3
@@ -4875,6 +4974,7 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(embedding_values, global_vars[0].eval())
       self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_3d(self):
     # Inputs.
     vocabulary_size = 4
@@ -4936,6 +5036,7 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(embedding_values, global_vars[0].eval())
       self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_weight_collections(self):
     sparse_input = sparse_tensor.SparseTensorValue(
         # example 0, ids [2]
@@ -4965,6 +5066,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertItemsEqual(
         ('embedding_weights:0',), tuple([v.name for v in my_vars]))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
     vocabulary_size = 3
@@ -5036,6 +5138,7 @@ class EmbeddingColumnTest(test.TestCase):
               input_shape: sparse_input.dense_shape,
           }))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_restore_from_ckpt(self):
     # Inputs.
     vocabulary_size = 3
@@ -5094,6 +5197,7 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(embedding_values, global_vars[0].eval())
       self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     # Inputs.
     batch_size = 4
@@ -5173,6 +5277,7 @@ class EmbeddingColumnTest(test.TestCase):
         self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
                             self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     # Inputs.
     batch_size = 4
@@ -5252,6 +5357,7 @@ class EmbeddingColumnTest(test.TestCase):
         self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
                             self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_input_layer(self):
     # Inputs.
     vocabulary_size = 3
@@ -5313,6 +5419,7 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(embedding_values, trainable_vars[0].eval())
       self.assertAllEqual(expected_lookups, self.evaluate(input_layer))
 
+  @test_util.run_deprecated_v1
   def test_input_layer_not_trainable(self):
     # Inputs.
     vocabulary_size = 3
@@ -5376,6 +5483,7 @@ class EmbeddingColumnTest(test.TestCase):
 
 class SharedEmbeddingColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -5420,6 +5528,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         'bbb': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column_b._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -5471,6 +5580,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         'bbb': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column_b._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -5509,6 +5619,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
           'aaa': parsing_ops.VarLenFeature(dtypes.int64)
       }, embedding_column_a._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_invalid_initializer(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -5520,6 +5631,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
           dimension=2,
           initializer='not_fn')
 
+  @test_util.run_deprecated_v1
   def test_incompatible_column_type(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -5535,6 +5647,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
           [categorical_column_a, categorical_column_b, categorical_column_c],
           dimension=2)
 
+  @test_util.run_deprecated_v1
   def test_weighted_categorical_column_ok(self):
     categorical_column_a = fc._categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -5552,6 +5665,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         [weighted_categorical_column_a, weighted_categorical_column_b],
         dimension=2)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5589,6 +5703,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['bbb'].eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     a = fc._categorical_column_with_identity(key='aaa', num_buckets=3)
     b = fc._categorical_column_with_identity(key='bbb', num_buckets=3)
@@ -5615,6 +5730,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
       _assert_sparse_tensor_value(self, self.evaluate(output_b),
                                   self.evaluate(output_b_embedded))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor(self):
     # Inputs.
     vocabulary_size = 3
@@ -5683,6 +5799,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a))
       self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_weight_collections(self):
     # Inputs.
     vocabulary_size = 3
@@ -5735,6 +5852,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         ('input_layer/aaa_bbb_shared_embedding/embedding_weights:0',),
         tuple(v.name for v in my_vars))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
     vocabulary_size = 3
@@ -5791,6 +5909,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     with _initialized_session() as sess:
       sess.run([embedding_lookup_a, embedding_lookup_b], feed_dict=feed_dict)
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     # Inputs.
     batch_size = 2
@@ -5881,6 +6000,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
         self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     # Inputs.
     batch_size = 2
@@ -6048,15 +6168,18 @@ class SharedEmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(embedding_values, shared_embedding_vars[0].eval())
       self.assertAllEqual(expected_lookups, self.evaluate(input_layer))
 
+  @test_util.run_deprecated_v1
   def test_input_layer(self):
     self._test_input_layer()
 
+  @test_util.run_deprecated_v1
   def test_input_layer_no_trainable(self):
     self._test_input_layer(trainable=False)
 
 
 class WeightedCategoricalColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     column = fc._weighted_categorical_column(
         categorical_column=fc._categorical_column_with_identity(
@@ -6070,6 +6193,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         'values': parsing_ops.VarLenFeature(dtypes.float32)
     }, column._parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     """Tests deepcopy of categorical_column_with_hash_bucket."""
     original = fc._weighted_categorical_column(
@@ -6132,6 +6256,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         ValueError, 'values is not in features dictionary'):
       _transform_features({'ids': inputs}, (column,))
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc._categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -6167,6 +6292,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=[1, 2]),
           features['weights'].eval())
 
+  @test_util.run_deprecated_v1
   def test_transform_features(self):
     column = fc._weighted_categorical_column(
         categorical_column=fc._categorical_column_with_identity(
@@ -6198,6 +6324,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
               values=np.array(weights.values, dtype=np.float32),
               dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_features_dense_input(self):
     column = fc._weighted_categorical_column(
         categorical_column=fc._categorical_column_with_identity(
@@ -6225,6 +6352,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
               values=np.array(weights.values, dtype=np.float32),
               dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_features_dense_weights(self):
     column = fc._weighted_categorical_column(
         categorical_column=fc._categorical_column_with_identity(
@@ -6252,6 +6380,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
               values=np.array((.5, 1., .1), dtype=np.float32),
               dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
+  @test_util.run_deprecated_v1
   def test_keras_linear_model(self):
     column = fc._weighted_categorical_column(
         categorical_column=fc._categorical_column_with_identity(
@@ -6354,6 +6483,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         # = 3*1 + 2*.1 = 3+.2 = 3.2
         self.assertAllClose(((.5,), (3.2,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     column = fc._weighted_categorical_column(
         categorical_column=fc._categorical_column_with_identity(
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 3147754bee..083cd526e4 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -218,6 +218,7 @@ class LazyColumnTest(test.TestCase):
         TypeError, '"key" must be either a "str" or "FeatureColumn".'):
       transformation_cache.get(NotAFeatureColumn(), None)
 
+  @test_util.run_deprecated_v1
   def test_expand_dim_rank_1_sparse_tensor_empty_batch(self):
     # empty 1-D sparse tensor:
     transformation_cache = fc.FeatureTransformationCache(
@@ -237,6 +238,7 @@ class LazyColumnTest(test.TestCase):
 
 class NumericColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     a = fc.numeric_column('aaa')
     self.assertEqual('aaa', a.key)
@@ -315,6 +317,7 @@ class NumericColumnTest(test.TestCase):
         'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32)
     }, a.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_parse_example_no_default_value(self):
     price = fc.numeric_column('price', shape=[2])
     data = example_pb2.Example(
@@ -331,6 +334,7 @@ class NumericColumnTest(test.TestCase):
 
     self.assertAllEqual([[20., 110.]], self.evaluate(features['price']))
 
+  @test_util.run_deprecated_v1
   def test_parse_example_with_default_value(self):
     price = fc.numeric_column('price', shape=[2], default_value=11.)
     data = example_pb2.Example(
@@ -360,6 +364,7 @@ class NumericColumnTest(test.TestCase):
     with self.assertRaisesRegexp(TypeError, 'must be a callable'):
       fc.numeric_column('price', normalizer_fn='NotACallable')
 
+  @test_util.run_deprecated_v1
   def test_normalizer_fn_transform_feature(self):
 
     def _increment_two(input_tensor):
@@ -372,6 +377,7 @@ class NumericColumnTest(test.TestCase):
 
     self.assertAllEqual([[3., 4.], [7., 8.]], self.evaluate(output[price]))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor(self):
 
     def _increment_two(input_tensor):
@@ -395,6 +401,7 @@ class NumericColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'must be a Tensor'):
       price.transform_feature(transformation_cache, None)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc.numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]])
     a_copy = copy.deepcopy(a)
@@ -407,6 +414,7 @@ class NumericColumnTest(test.TestCase):
         'aaa', shape=[1, 2], default_value=np.array([[3., 2.]]))
     self.assertEqual(a.default_value, ((3., 2.),))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     price = fc.numeric_column('price')
     with ops.Graph().as_default():
@@ -435,6 +443,7 @@ class NumericColumnTest(test.TestCase):
         sess.run(price_var.assign([[10.]]))
         self.assertAllClose([[10.], [50.]], self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
 
     def _increment_two(input_tensor):
@@ -519,6 +528,7 @@ class BucketizedColumnTest(test.TestCase):
     # Column 'aaa` has shape [2] times three buckets -> num_buckets=6.
     self.assertEqual(6, b.num_buckets)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
@@ -536,6 +546,7 @@ class BucketizedColumnTest(test.TestCase):
 
     self.assertAllEqual([[20., 110.]], self.evaluate(features['price']))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
@@ -639,6 +650,7 @@ class BucketizedColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'must be a Tensor'):
       bucketized_price.transform_feature(transformation_cache, None)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc.numeric_column('aaa', shape=[2])
     a_bucketized = fc.bucketized_column(a, boundaries=[0, 1])
@@ -789,6 +801,7 @@ class BucketizedColumnTest(test.TestCase):
         self.assertAllClose([[11.], [21.], [41.], [51.]],
                             self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
@@ -821,6 +834,7 @@ class BucketizedColumnTest(test.TestCase):
 
 class HashedCategoricalColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     a = fc.categorical_column_with_hash_bucket('aaa', 10)
     self.assertEqual('aaa', a.name)
@@ -848,6 +862,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'):
       fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc.categorical_column_with_hash_bucket('aaa', 10)
     for column in (original, copy.deepcopy(original)):
@@ -868,6 +883,7 @@ class HashedCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, a.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_hash_bucket('aaa', 10)
     data = example_pb2.Example(
@@ -890,6 +906,7 @@ class HashedCategoricalColumnTest(test.TestCase):
             values=np.array([b'omar', b'stringer'], dtype=np.object_),
             dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_strings_should_be_hashed(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     wire_tensor = sparse_tensor.SparseTensor(
@@ -943,6 +960,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'dtype must be compatible'):
       transformation_cache.get(hashed_sparse, None)
 
+  @test_util.run_deprecated_v1
   def test_ints_should_be_hashed(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
@@ -957,6 +975,7 @@ class HashedCategoricalColumnTest(test.TestCase):
 
     self.assertAllEqual(expected_values, self.evaluate(output.values))
 
+  @test_util.run_deprecated_v1
   def test_int32_64_is_compatible(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket(
         'wire', 10, dtype=dtypes.int64)
@@ -971,6 +990,7 @@ class HashedCategoricalColumnTest(test.TestCase):
 
     self.assertAllEqual(expected_values, self.evaluate(output.values))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     transformation_cache = fc.FeatureTransformationCache({
@@ -986,6 +1006,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     self.assertEqual(
         transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor)
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     hashed_sparse = fc.categorical_column_with_hash_bucket('wire', 10)
     transformation_cache = fc.FeatureTransformationCache({
@@ -997,6 +1018,7 @@ class HashedCategoricalColumnTest(test.TestCase):
     self.assertEqual(
         transformation_cache.get(hashed_sparse, None), id_weight_pair.id_tensor)
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(4, wire_column.num_buckets)
@@ -1047,6 +1069,7 @@ class HashedCategoricalColumnTest(test.TestCase):
       # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
       self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
     self.assertEqual(['wire'], wire_column.parents)
@@ -1148,6 +1171,7 @@ class CrossedColumnTest(test.TestCase):
     crossed = fc.crossed_column([b, 'c'], 15)
     self.assertEqual(15, crossed.num_buckets)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc.numeric_column('a', dtype=dtypes.int32)
     b = fc.bucketized_column(a, boundaries=[0, 1])
@@ -1161,6 +1185,7 @@ class CrossedColumnTest(test.TestCase):
     self.assertEqual(15, crossed2_copy.hash_bucket_size)
     self.assertEqual(5, crossed2_copy.hash_key)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
@@ -1190,6 +1215,7 @@ class CrossedColumnTest(test.TestCase):
                         self.evaluate(wire_sparse.values))
     self.assertAllEqual([1, 2], self.evaluate(wire_sparse.dense_shape))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     price = fc.numeric_column('price', shape=[2])
     bucketized_price = fc.bucketized_column(price, boundaries=[0, 50])
@@ -1214,6 +1240,7 @@ class CrossedColumnTest(test.TestCase):
       self.assertIn(val, list(range(hash_bucket_size)))
     self.assertAllEqual([2, 4], output_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
     b = fc.bucketized_column(a, boundaries=(0, 1))
@@ -1285,6 +1312,7 @@ class CrossedColumnTest(test.TestCase):
       self.assertAllEqual(expected_values, id_tensor_eval.values)
       self.assertAllEqual((2, 4), id_tensor_eval.dense_shape)
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     """Tests linear_model.
 
@@ -1520,6 +1548,7 @@ class CrossedColumnTest(test.TestCase):
         sess.run(bias.assign((.1,)))
         self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
     b = fc.bucketized_column(a, boundaries=(0, 1))
@@ -2077,6 +2106,7 @@ class LinearModelTest(test.TestCase):
                 features['price2']: [[1.], [5.]],
             })
 
+  @test_util.run_deprecated_v1
   def test_with_numpy_input_fn(self):
     price = fc.numeric_column('price')
     price_buckets = fc.bucketized_column(
@@ -2115,6 +2145,7 @@ class LinearModelTest(test.TestCase):
       coord.request_stop()
       coord.join(threads)
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     price = fc.numeric_column('price')
     price_buckets = fc.bucketized_column(
@@ -2154,6 +2185,7 @@ class LinearModelTest(test.TestCase):
       self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
                           self.evaluate(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
     price_buckets = fc.bucketized_column(
@@ -2198,6 +2230,7 @@ class LinearModelTest(test.TestCase):
                                   features['country']: country_data
                               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     price = fc.numeric_column('price')
     features = {
@@ -2835,6 +2868,7 @@ class OldLinearModelTest(test.TestCase):
                 features['price2']: [[1.], [5.]],
             })
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     price = fc.numeric_column('price')
     price_buckets = fc.bucketized_column(
@@ -2875,6 +2909,7 @@ class OldLinearModelTest(test.TestCase):
       self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
                           self.evaluate(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
     price_buckets = fc.bucketized_column(
@@ -2920,6 +2955,7 @@ class OldLinearModelTest(test.TestCase):
                                   features['country']: country_data
                               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     price = fc.numeric_column('price')
     features = {
@@ -2962,6 +2998,7 @@ class OldLinearModelTest(test.TestCase):
         sess.run(bias2.assign([5.]))
         self.assertAllClose([[25.], [105.]], self.evaluate(predictions2))
 
+  @test_util.run_deprecated_v1
   def test_linear_model_v1_shared_embedding_all_other_v2(self):
     price = fc.numeric_column('price')  # v2
     some_sparse_column = fc.categorical_column_with_hash_bucket(
@@ -3001,6 +3038,7 @@ class OldLinearModelTest(test.TestCase):
 
       self.assertAllClose([0.], self.evaluate(bias))
 
+  @test_util.run_deprecated_v1
   def test_linear_model_v1_shared_embedding_with_v2_cat_all_other_v2(self):
     price = fc.numeric_column('price')  # v2
     some_sparse_column = fc.categorical_column_with_hash_bucket(
@@ -3040,6 +3078,7 @@ class OldLinearModelTest(test.TestCase):
 
       self.assertAllClose([0.], self.evaluate(bias))
 
+  @test_util.run_deprecated_v1
   def test_linear_model_v1_v2_mix(self):
     price = fc.numeric_column('price')  # v2
     some_sparse_column = fc.categorical_column_with_hash_bucket(
@@ -3079,6 +3118,7 @@ class OldLinearModelTest(test.TestCase):
 
       self.assertAllClose([0.], self.evaluate(bias))
 
+  @test_util.run_deprecated_v1
   def test_linear_model_v2_shared_embedding_all_other_v1(self):
     price = fc.numeric_column('price')  # v1
     some_sparse_column = fc.categorical_column_with_hash_bucket(
@@ -3468,6 +3508,7 @@ class DenseFeaturesTest(test.TestCase):
           expected_var_names,
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_multiple_layers_with_same_shared_embedding_column(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -3501,6 +3542,7 @@ class DenseFeaturesTest(test.TestCase):
           ['aaa_bbb_shared_embedding:0'],
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_multiple_layers_with_same_shared_embedding_column_diff_graphs(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -3552,6 +3594,7 @@ class DenseFeaturesTest(test.TestCase):
           ['aaa_bbb_shared_embedding:0'],
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_with_numpy_input_fn(self):
     embedding_values = (
         (1., 2., 3., 4., 5.),  # id 0
@@ -3596,6 +3639,7 @@ class DenseFeaturesTest(test.TestCase):
       coord.request_stop()
       coord.join(threads)
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     embedding_values = (
         (1., 2., 3., 4., 5.),  # id 0
@@ -3652,6 +3696,7 @@ class DenseFeaturesTest(test.TestCase):
                            [1., 0., 0., 1., 2., 3., 4., 5., 12.]],
                           sess.run(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     embedding_values = (
         (1., 2.),  # id 0
@@ -3710,6 +3755,7 @@ class DenseFeaturesTest(test.TestCase):
                   features['country']: country_data
               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     # price has 1 dimension in dense_features
     price = fc.numeric_column('price')
@@ -3967,6 +4013,7 @@ class FunctionalInputLayerTest(test.TestCase):
                             variables_lib.Variable)
       self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10])
 
+  @test_util.run_deprecated_v1
   def test_fills_cols_to_vars_shared_embedding(self):
     # Provide 5 DenseColumn's to input_layer: a NumericColumn, a
     # BucketizedColumn, an EmbeddingColumn, two SharedEmbeddingColumns. The
@@ -4167,6 +4214,7 @@ class FunctionalInputLayerTest(test.TestCase):
           expected_var_names,
           [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
 
+  @test_util.run_deprecated_v1
   def test_with_1d_sparse_tensor(self):
     embedding_values = (
         (1., 2., 3., 4., 5.),  # id 0
@@ -4223,6 +4271,7 @@ class FunctionalInputLayerTest(test.TestCase):
                            [1., 0., 0., 1., 2., 3., 4., 5., 12.]],
                           sess.run(net))
 
+  @test_util.run_deprecated_v1
   def test_with_1d_unknown_shape_sparse_tensor(self):
     embedding_values = (
         (1., 2.),  # id 0
@@ -4281,6 +4330,7 @@ class FunctionalInputLayerTest(test.TestCase):
                   features['country']: country_data
               }))
 
+  @test_util.run_deprecated_v1
   def test_with_rank_0_feature(self):
     # price has 1 dimension in input_layer
     price = fc.numeric_column('price')
@@ -4444,6 +4494,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         'python/feature_column/testdata/wire_vocabulary.txt')
     self._wire_vocabulary_size = 3
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
@@ -4460,6 +4511,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       fc.categorical_column_with_vocabulary_file(
           key=('aaa',), vocabulary_file='path_to_file', vocabulary_size=3)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4472,6 +4524,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, column.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4496,6 +4549,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       fc.categorical_column_with_vocabulary_file(
           key='aaa', vocabulary_file='', vocabulary_size=3)
 
+  @test_util.run_deprecated_v1
   def test_invalid_vocabulary_file(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='file_does_not_exist', vocabulary_size=10)
@@ -4522,6 +4576,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
           vocabulary_file=self._wire_vocabulary_file_name,
           vocabulary_size=0)
 
+  @test_util.run_deprecated_v1
   def test_too_large_vocabulary_size(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4596,6 +4651,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
               'aaa': inputs
           }), None)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file='path_to_file', vocabulary_size=3)
@@ -4619,6 +4675,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             values=np.array([b'omar', b'stringer'], dtype=np.object_),
             dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4645,6 +4702,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_none_vocabulary_size(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa', vocabulary_file=self._wire_vocabulary_file_name)
@@ -4669,6 +4727,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4692,6 +4751,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             values=np.array((2, -1, 0), dtype=np.int64),
             dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4713,6 +4773,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             values=np.array((2, -1, 0), dtype=np.int64),
             dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4740,6 +4801,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4767,6 +4829,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_small_vocabulary_size(self):
     # 'marlo' is the last entry in our vocabulary file, so be setting
     # `vocabulary_size` to 1 less than number of entries in file, we take
@@ -4796,6 +4859,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4823,6 +4887,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
     column = fc.categorical_column_with_vocabulary_file(
@@ -4847,6 +4912,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             values=np.array((2, default_value, 0, 4), dtype=np.int64),
             dense_shape=(3, 3)), self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_file(
         key='aaa',
@@ -4875,6 +4941,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_file(
         key='wire',
@@ -4933,6 +5000,7 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
       # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
       self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     wire_column = fc.categorical_column_with_vocabulary_file(
         key='wire',
@@ -4984,6 +5052,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int64)
     }, column.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -4995,6 +5064,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int32)
     }, column.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(12, 24, 36), dtype=dtypes.int32)
@@ -5089,6 +5159,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
               'aaa': inputs
           }), None)
 
+  @test_util.run_deprecated_v1
   def test_parse_example_string(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5112,6 +5183,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             values=np.array([b'omar', b'stringer'], dtype=np.object_),
             dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_parse_example_int(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=(11, 21, 31))
@@ -5133,6 +5205,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             indices=[[0, 0], [0, 1]], values=[11, 21], dense_shape=[1, 2]),
         self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5157,6 +5230,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5178,6 +5252,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             values=np.array((2, -1, 0), dtype=np.int64),
             dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5197,6 +5272,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             values=np.array((2, -1, 0), dtype=np.int64),
             dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_default_value_in_vocabulary(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -5223,6 +5299,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -5249,6 +5326,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -5275,6 +5353,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_dense_input(self):
     default_value = -100
     column = fc.categorical_column_with_vocabulary_list(
@@ -5300,6 +5379,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             values=np.array((2, default_value, 0, 4), dtype=np.int64),
             dense_shape=(3, 3)), self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_int32_with_oov_buckets(self):
     column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -5327,6 +5407,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     wire_column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -5383,6 +5464,7 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
       # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
       self.assertAllClose(((3.,), (5.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     wire_column = fc.categorical_column_with_vocabulary_list(
         key='aaa',
@@ -5420,6 +5502,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'key must be a string.'):
       fc.categorical_column_with_identity(key=('aaa',), num_buckets=3)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     original = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     for column in (original, copy.deepcopy(original)):
@@ -5459,6 +5542,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
               'aaa': inputs
           }), None)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_identity(key='aaa', num_buckets=30)
     data = example_pb2.Example(
@@ -5480,6 +5564,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
             values=np.array([11, 21], dtype=np.int64),
             dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -5501,6 +5586,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -5519,6 +5605,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
             values=np.array((0, 1, 0), dtype=np.int64),
             dense_shape=inputs.dense_shape), self.evaluate(id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_dense_input(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     id_weight_pair = column.get_sparse_tensors(
@@ -5537,6 +5624,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
             values=np.array((0, 1, 0), dtype=np.int64),
             dense_shape=(2, 2)), self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_inputs_too_small(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -5553,6 +5641,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
     with self.assertRaisesRegexp(errors.OpError, 'assert_greater_or_equal_0'):
       self.evaluate(id_weight_pair.id_tensor)
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_inputs_too_big(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     inputs = sparse_tensor.SparseTensorValue(
@@ -5570,6 +5659,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
                                  'assert_less_than_num_buckets'):
       self.evaluate(id_weight_pair.id_tensor)
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_default_value(self):
     column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
@@ -5594,6 +5684,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
             dense_shape=inputs.dense_shape),
         self.evaluate(id_weight_pair.id_tensor))
 
+  @test_util.run_deprecated_v1
   def test_get_sparse_tensors_with_default_value_and_placeholder_inputs(self):
     column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=4, default_value=3)
@@ -5624,6 +5715,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
                   input_shape: (2, 2),
               }))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column.num_buckets)
@@ -5674,6 +5766,7 @@ class IdentityCategoricalColumnTest(test.TestCase):
       # weight_var[2] + weight_var[1] = 3+2 = 5
       self.assertAllClose(((1.,), (5.,)), self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
 
@@ -5827,6 +5920,7 @@ class IndicatorColumnTest(test.TestCase):
 
     self.assertAllEqual([[0., 1., 1., 0.]], self.evaluate(output))
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     a = fc.categorical_column_with_hash_bucket('a', 4)
     column = fc.indicator_column(a)
@@ -5835,6 +5929,7 @@ class IndicatorColumnTest(test.TestCase):
     self.assertEqual(column.name, 'a_indicator')
     self.assertEqual(column.variable_shape, [1, 4])
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5859,6 +5954,7 @@ class IndicatorColumnTest(test.TestCase):
             values=np.array([b'omar', b'stringer'], dtype=np.object_),
             dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_transform(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -5878,6 +5974,7 @@ class IndicatorColumnTest(test.TestCase):
 
     self.assertAllEqual([[0, 0, 1], [1, 0, 0]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_with_weighted_column(self):
     # Github issue 12557
     ids = fc.categorical_column_with_vocabulary_list(
@@ -5896,6 +5993,7 @@ class IndicatorColumnTest(test.TestCase):
 
     self.assertAllEqual([[6., 4., 3.]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_with_missing_value_in_weighted_column(self):
     # Github issue 12583
     ids = fc.categorical_column_with_vocabulary_list(
@@ -5914,6 +6012,7 @@ class IndicatorColumnTest(test.TestCase):
 
     self.assertAllEqual([[0., 4., 2.]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_with_missing_value_in_categorical_column(self):
     # Github issue 12583
     ids = fc.categorical_column_with_vocabulary_list(
@@ -5930,6 +6029,7 @@ class IndicatorColumnTest(test.TestCase):
 
     self.assertAllEqual([[0., 1., 1.]], self.evaluate(indicator_tensor))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     animal = fc.indicator_column(
         fc.categorical_column_with_identity('animal', num_buckets=4))
@@ -5997,6 +6097,7 @@ class IndicatorColumnTest(test.TestCase):
       self.evaluate(weight_var.assign([[1.], [2.], [3.], [4.]]))
       self.assertAllClose([[2. + 3.]], self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_dense_features(self):
     animal = fc.indicator_column(
         fc.categorical_column_with_identity('animal', num_buckets=4))
@@ -6013,6 +6114,7 @@ class IndicatorColumnTest(test.TestCase):
 
       self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
+  @test_util.run_deprecated_v1
   def test_input_layer(self):
     animal = fc.indicator_column(
         fc.categorical_column_with_identity('animal', num_buckets=4))
@@ -6045,6 +6147,7 @@ class IndicatorColumnTest(test.TestCase):
 
       self.assertAllClose([[0., 1., 1., 0.]], self.evaluate(net))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     parent = fc.categorical_column_with_identity('animal', num_buckets=4)
     animal = fc.indicator_column(parent)
@@ -6114,6 +6217,7 @@ class _TestStateManager(fc.StateManager):
 
 class EmbeddingColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -6142,6 +6246,7 @@ class EmbeddingColumnTest(test.TestCase):
         categorical_column, dimension=embedding_dimension)
     self.assertFalse(embedding_column._is_v2_column)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -6168,6 +6273,7 @@ class EmbeddingColumnTest(test.TestCase):
         'aaa': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -6200,12 +6306,14 @@ class EmbeddingColumnTest(test.TestCase):
           'aaa': parsing_ops.VarLenFeature(dtypes.int64)
       }, embedding_column.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_invalid_initializer(self):
     categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
     with self.assertRaisesRegexp(ValueError, 'initializer must be callable'):
       fc.embedding_column(categorical_column, dimension=2, initializer='not_fn')
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -6230,6 +6338,7 @@ class EmbeddingColumnTest(test.TestCase):
             values=np.array([b'omar', b'stringer'], dtype=np.object_),
             dense_shape=[1, 2]), self.evaluate(features['aaa']))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     a_embedded = fc.embedding_column(a, dimension=2)
@@ -6250,6 +6359,7 @@ class EmbeddingColumnTest(test.TestCase):
     _assert_sparse_tensor_value(self, self.evaluate(output_a),
                                 self.evaluate(output_embedded))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor(self):
     # Inputs.
     vocabulary_size = 3
@@ -6315,6 +6425,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_old_categorical(self):
     # Inputs.
     vocabulary_size = 3
@@ -6378,6 +6489,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_3d(self):
     # Inputs.
     vocabulary_size = 4
@@ -6445,6 +6557,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
     vocabulary_size = 3
@@ -6524,6 +6637,7 @@ class EmbeddingColumnTest(test.TestCase):
                   input_shape: sparse_input.dense_shape,
               }))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_restore_from_ckpt(self):
     # Inputs.
     vocabulary_size = 3
@@ -6587,6 +6701,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     # Inputs.
     batch_size = 4
@@ -6668,6 +6783,7 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
                           self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_dense_features(self):
     # Inputs.
     vocabulary_size = 3
@@ -6734,6 +6850,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(embedding_values, self.evaluate(trainable_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
+  @test_util.run_deprecated_v1
   def test_dense_features_not_trainable(self):
     # Inputs.
     vocabulary_size = 3
@@ -6799,6 +6916,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(embedding_values, self.evaluate(global_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
+  @test_util.run_deprecated_v1
   def test_input_layer(self):
     # Inputs.
     vocabulary_size = 3
@@ -7028,6 +7146,7 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllClose(((94.,), (29.,), (0.,), (42.,)),
                           self.evaluate(predictions))
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
 
     def _initializer(shape, dtype, partition_info):
@@ -7081,6 +7200,7 @@ class EmbeddingColumnTest(test.TestCase):
 
 class SharedEmbeddingColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -7105,6 +7225,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         'bbb': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column_b.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_all_constructor_args(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -7136,6 +7257,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         'bbb': parsing_ops.VarLenFeature(dtypes.int64)
     }, embedding_column_b.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -7167,6 +7289,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
           'aaa': parsing_ops.VarLenFeature(dtypes.int64)
       }, embedding_column_a.parse_example_spec)
 
+  @test_util.run_deprecated_v1
   def test_invalid_initializer(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -7178,6 +7301,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
           dimension=2,
           initializer='not_fn')
 
+  @test_util.run_deprecated_v1
   def test_incompatible_column_type(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -7192,6 +7316,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
           [categorical_column_a, categorical_column_b, categorical_column_c],
           dimension=2)
 
+  @test_util.run_deprecated_v1
   def test_weighted_categorical_column_ok(self):
     categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=3)
@@ -7209,6 +7334,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         [weighted_categorical_column_a, weighted_categorical_column_b],
         dimension=2)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -7246,6 +7372,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
             values=np.array([b'stringer', b'marlo'], dtype=np.object_),
             dense_shape=[1, 2]), self.evaluate(features['bbb']))
 
+  @test_util.run_deprecated_v1
   def test_transform_feature(self):
     a = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     b = fc.categorical_column_with_identity(key='bbb', num_buckets=3)
@@ -7277,6 +7404,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     _assert_sparse_tensor_value(self, self.evaluate(output_b),
                                 self.evaluate(output_b_embedded))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor(self):
     # Inputs.
     vocabulary_size = 3
@@ -7348,6 +7476,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a))
     self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b))
 
+  @test_util.run_deprecated_v1
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
     vocabulary_size = 3
@@ -7407,6 +7536,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
     with _initialized_session() as sess:
       sess.run([embedding_lookup_a, embedding_lookup_b], feed_dict=feed_dict)
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     # Inputs.
     batch_size = 2
@@ -7619,12 +7749,15 @@ class SharedEmbeddingColumnTest(test.TestCase):
                         self.evaluate(shared_embedding_vars[0]))
     self.assertAllEqual(expected_lookups, self.evaluate(dense_features))
 
+  @test_util.run_deprecated_v1
   def test_dense_features(self):
     self._test_dense_features()
 
+  @test_util.run_deprecated_v1
   def test_dense_features_no_trainable(self):
     self._test_dense_features(trainable=False)
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
 
     def _initializer(shape, dtype, partition_info):
@@ -7647,6 +7780,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
 
 class WeightedCategoricalColumnTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_defaults(self):
     column = fc.weighted_categorical_column(
         categorical_column=fc.categorical_column_with_identity(
@@ -7667,6 +7801,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
         weight_feature_key='values')
     self.assertFalse(column._is_v2_column)
 
+  @test_util.run_deprecated_v1
   def test_deep_copy(self):
     """Tests deepcopy of categorical_column_with_hash_bucket."""
     original = fc.weighted_categorical_column(
@@ -7732,6 +7867,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
                                  'values is not in features dictionary'):
       fc._transform_features_v2({'ids': inputs}, (column,), None)
 
+  @test_util.run_deprecated_v1
   def test_parse_example(self):
     a = fc.categorical_column_with_vocabulary_list(
         key='aaa', vocabulary_list=('omar', 'stringer', 'marlo'))
@@ -7766,6 +7902,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
             values=np.array([1., 10.], dtype=np.float32),
             dense_shape=[1, 2]), self.evaluate(features['weights']))
 
+  @test_util.run_deprecated_v1
   def test_transform_features(self):
     column = fc.weighted_categorical_column(
         categorical_column=fc.categorical_column_with_identity(
@@ -7798,6 +7935,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
             values=np.array(weights.values, dtype=np.float32),
             dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_features_dense_input(self):
     column = fc.weighted_categorical_column(
         categorical_column=fc.categorical_column_with_identity(
@@ -7828,6 +7966,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
             values=np.array(weights.values, dtype=np.float32),
             dense_shape=weights.dense_shape), self.evaluate(weight_tensor))
 
+  @test_util.run_deprecated_v1
   def test_transform_features_dense_weights(self):
     column = fc.weighted_categorical_column(
         categorical_column=fc.categorical_column_with_identity(
@@ -7856,6 +7995,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
             values=np.array((.5, 1., .1), dtype=np.float32),
             dense_shape=(2, 2)), self.evaluate(weight_tensor))
 
+  @test_util.run_deprecated_v1
   def test_linear_model(self):
     column = fc.weighted_categorical_column(
         categorical_column=fc.categorical_column_with_identity(
@@ -8106,6 +8246,7 @@ class WeightedCategoricalColumnTest(test.TestCase):
 
   # TODO(ptucker): Add test with embedding of weighted categorical.
 
+  @test_util.run_deprecated_v1
   def test_serialization(self):
     categorical_column = fc.categorical_column_with_identity(
         key='ids', num_buckets=3)
diff --git a/tensorflow/python/framework/file_system_test.py b/tensorflow/python/framework/file_system_test.py
index 066d34e781..8687bc5a78 100644
--- a/tensorflow/python/framework/file_system_test.py
+++ b/tensorflow/python/framework/file_system_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import os
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import load_library
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import io_ops
@@ -36,6 +37,7 @@ class FileSystemTest(test.TestCase):
                                        "test_file_system.so")
     load_library.load_file_system_library(file_system_library)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.cached_session() as sess:
       reader = io_ops.WholeFileReader("test_reader")
diff --git a/tensorflow/python/framework/function_def_to_graph_test.py b/tensorflow/python/framework/function_def_to_graph_test.py
index b2ef64f873..ddf1a6e74d 100644
--- a/tensorflow/python/framework/function_def_to_graph_test.py
+++ b/tensorflow/python/framework/function_def_to_graph_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import function_def_to_graph
 from tensorflow.python.framework import graph_to_function_def
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import test_ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -52,6 +53,7 @@ class FunctionDefToGraphTest(test.TestCase):
     fdef.signature.name = "_whats_in_a_name"
     return fdef
 
+  @test_util.run_deprecated_v1
   def testInputsAndOutputs(self):
     fdef = self._build_function_def()
     g = function_def_to_graph.function_def_to_graph(fdef)
@@ -186,6 +188,7 @@ class FunctionDefToGraphDefTest(test.TestCase):
     self.assertEqual(g.node[0].attr["shape"].shape.unknown_rank, False)
     self.assertFalse("shape" in g.node[2].attr)
 
+  @test_util.run_deprecated_v1
   def testFunctionCallsFromFunction(self):
     x = constant_op.constant(5.0)
     y = constant_op.constant(10.0)
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 1a17a48050..d71f06ea52 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import function
 from tensorflow.python.framework import graph_to_function_def
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework.errors import InvalidArgumentError
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -104,6 +105,7 @@ class FunctionTest(test.TestCase):
       with session.Session() as sess:
         self.assertAllEqual([18.0], self.evaluate(call))
 
+  @test_util.run_deprecated_v1
   def testIdentityImplicitDeref(self):
 
     @function.Defun(dtypes.float32, func_name="MyIdentity")
@@ -322,6 +324,7 @@ class FunctionTest(test.TestCase):
       self.assertEqual(x.get_shape(), dx.get_shape())
       self.assertEqual(y.get_shape(), dy.get_shape())
 
+  @test_util.run_deprecated_v1
   def testSymGradAttr(self):
 
     @function.Defun(noinline=True)
@@ -438,6 +441,7 @@ class FunctionTest(test.TestCase):
                                    "assertion failed.*-3"):
         self.assertAllEqual(Foo(constant_op.constant(-3.0)).eval(), 6.0)
 
+  @test_util.run_deprecated_v1
   def testAssertWrapper(self):
 
     @function.Defun(dtypes.float32)
@@ -452,6 +456,7 @@ class FunctionTest(test.TestCase):
                                    "assertion"):
         _ = MyFn(100.0).eval()
 
+  @test_util.run_deprecated_v1
   def testWhileLoopCallsFunc(self):
     with self.session(use_gpu=True) as sess:
 
@@ -471,6 +476,7 @@ class FunctionTest(test.TestCase):
       ans = self.evaluate(loop)
       self.assertAllClose(ans, 131072.)
 
+  @test_util.run_deprecated_v1
   def testControlFlowStrictness(self):
     """Inlined functions must not execute in a untaken control flow branch."""
 
@@ -517,6 +523,7 @@ class FunctionTest(test.TestCase):
                                    "assertion"):
         sess.run(loop, {pred: True, x: 3})
 
+  @test_util.run_deprecated_v1
   def testVar(self):
 
     @function.Defun(dtypes.float32)
@@ -532,6 +539,7 @@ class FunctionTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllEqual(z.eval(), 101.)
 
+  @test_util.run_deprecated_v1
   def testResourceVarAsImplicitInput(self):
     g = ops.Graph()
     with g.as_default(), ops.device("cpu:0"):
@@ -707,6 +715,7 @@ class FunctionTest(test.TestCase):
     gdef = g.as_graph_def()
     self.assertEqual(0, len(gdef.library.function))
 
+  @test_util.run_deprecated_v1
   def testReduction(self):
     g = ops.Graph()
 
@@ -735,6 +744,7 @@ class FunctionTest(test.TestCase):
       self.assertAllClose(vals[0], vals[1])
       self.assertAllClose(vals[2], vals[3])
 
+  @test_util.run_deprecated_v1
   def testCapture(self):
     g = ops.Graph()
     with g.as_default():
@@ -781,6 +791,7 @@ class FunctionTest(test.TestCase):
         # NOTE: We still do not support capturing control deps.
         _ = Foo(x)
 
+  @test_util.run_deprecated_v1
   def testCaptureInWhileLoop(self):
     g = ops.Graph()
     with g.as_default():
@@ -796,6 +807,7 @@ class FunctionTest(test.TestCase):
     with self.session(graph=g) as sess:
       self.assertEqual(self.evaluate(y), 10)
 
+  @test_util.run_deprecated_v1
   def testCaptureInCond(self):
     g = ops.Graph()
     with g.as_default():
@@ -825,6 +837,7 @@ class FunctionTest(test.TestCase):
       self.assertEqual("Foo_aCYSbwBkR5A",
                        Foo.instantiate([dtypes.float32] * 3).name)
 
+  @test_util.run_deprecated_v1
   def testSignatureHash(self):
     # Foo.Inner and Bar.Inner have identical function body but have
     # different signatures. They should be treated as two different functions.
@@ -877,6 +890,7 @@ class FunctionTest(test.TestCase):
       y = Bar(array_ops.zeros([1, 2, 3]))
       self.assertAllEqual(y.get_shape().as_list(), [1, 1, 2, 3])
 
+  @test_util.run_deprecated_v1
   def testVariableReuse(self):
 
     def LinearWithReuse(input_tensor, reuse=None):
@@ -905,6 +919,7 @@ class FunctionTest(test.TestCase):
           output_op, feed_dict={input_op: np.random.rand(32, 100)})
       self.assertEqual(output_val.shape, (32, 100))
 
+  @test_util.run_deprecated_v1
   def testFunctionCallInDifferentVariableScopes(self):
 
     @function.Defun(dtypes.float32)
@@ -968,6 +983,7 @@ class FunctionTest(test.TestCase):
       self.assertAllClose(
           np.array([1.0, 0.0]).astype(np.float32), sess.run(dinp, {inp: x}))
 
+  @test_util.run_deprecated_v1
   def testFunctionMarkedStateful(self):
 
     @function.Defun(dtypes.int32, dtypes.float32)
@@ -995,6 +1011,7 @@ class FunctionTest(test.TestCase):
       self.assertEqual(100, self.evaluate(result_2))
       self.assertEqual((4.0, 100), sess.run((result_1, result_2)))
 
+  @test_util.run_deprecated_v1
   def testStatefulFunction(self):
 
     @function.Defun()
@@ -1037,6 +1054,7 @@ class FunctionTest(test.TestCase):
         self.assertFalse(all(val3 == val1))
         self.assertFalse(all(val4 == val2))
 
+  @test_util.run_deprecated_v1
   def testSameFunctionOnTwoDevices(self):
 
     @function.Defun(dtypes.float32)
@@ -1056,6 +1074,7 @@ class FunctionTest(test.TestCase):
         self.assertEqual(44.0, self.evaluate(f_1))
         self.assertEqual((42.0, 44.0), sess.run((f_0, f_1)))
 
+  @test_util.run_deprecated_v1
   def testGuaranteedConstsAreCaptured(self):
     var = variables.Variable(1.0)
     const = array_ops.guarantee_const(var)
@@ -1079,6 +1098,7 @@ class FunctionTest(test.TestCase):
       self.evaluate(var.initializer)
       _ = sess.run(CapturesGuaranteedConst(), {also_not_const: 1.0})
 
+  @test_util.run_deprecated_v1
   def testSameFunctionDifferentGrads(self):
 
     def PartOne(x):
@@ -1150,6 +1170,7 @@ class FunctionsFromProtos(test.TestCase):
     self.assertEqual(func.declared_input_types, new_func.declared_input_types)
     self.assertEqual(func.captured_inputs, new_func.captured_inputs)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
 
     @function.Defun(dtypes.float32, dtypes.float32)
@@ -1359,6 +1380,7 @@ class FunctionsFromProtos(test.TestCase):
 
 class FunctionOverloadTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
 
     @function.Defun()
@@ -1411,6 +1433,7 @@ class FunctionOverloadTest(test.TestCase):
 
 class FunctionCaptureByValueTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCaptureByValue(self):
     g = ops.Graph()
     with g.as_default():
@@ -1634,6 +1657,7 @@ class FunctionInlineControlTest(test.TestCase):
 
 class ModuleFunctionTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
 
     @function.Defun(*[dtypes.float32] * 3)
@@ -1717,10 +1741,12 @@ class VariableHoistingTest(test.TestCase):
     self.assertAllEqual(db.shape, (64,))
     self.assertAllClose(np.sum(db), 0.509, rtol=1e-2)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     self._testSimpleModel(True)
     self._testSimpleModel(False)
 
+  @test_util.run_deprecated_v1
   def testBasicResource(self):
     self._testSimpleModel(True, use_resource=True)
     self._testSimpleModel(False, use_resource=True)
diff --git a/tensorflow/python/framework/graph_util_test.py b/tensorflow/python/framework/graph_util_test.py
index 10a01c71f2..4e7408ad49 100644
--- a/tensorflow/python/framework/graph_util_test.py
+++ b/tensorflow/python/framework/graph_util_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import graph_util
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import math_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import math_ops as math_ops_lib
@@ -102,6 +103,7 @@ class DeviceFunctionsTest(test.TestCase):
     self.assertDeviceEqual(var_5.device, "/device:GPU:0")
     self.assertDeviceEqual(var_6.device, "/device:CPU:0")
 
+  @test_util.run_deprecated_v1
   def testNestedDeviceFunctions(self):
     with ops.Graph().as_default():
       var_0 = variables.VariableV1(0)
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index cc93f8b1b8..46ce4616a5 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -63,6 +63,7 @@ def _TestDir(test_name):
 
 class SimpleMetaGraphTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoVariables(self):
     test_dir = _TestDir("no_variables")
     filename = os.path.join(test_dir, "metafile")
@@ -116,6 +117,7 @@ class SimpleMetaGraphTest(test.TestCase):
                                   {new_input_tensor: input_feed_value})
       self.assertEqual(new_output_value, output_value)
 
+  @test_util.run_deprecated_v1
   def testStrippedOpListNestedFunctions(self):
     with self.cached_session():
       # Square two levels deep
@@ -158,6 +160,7 @@ class SimpleMetaGraphTest(test.TestCase):
     op_list = meta_graph.stripped_op_list_for_graph(graph)
     self.assertEqual(["Const"], [op.name for op in op_list.op])
 
+  @test_util.run_deprecated_v1
   def testDefaultAttrStripping(self):
     """Verifies that default attributes are stripped from a graph def."""
 
@@ -210,6 +213,7 @@ class SimpleMetaGraphTest(test.TestCase):
       self.assertEqual(node_def.attr["Tout"].type, dtypes.complex128)
       self.assertTrue(meta_graph_def.meta_info_def.stripped_default_attrs)
 
+  @test_util.run_deprecated_v1
   def testDefaultAttrStrippingNestedFunctions(self):
     """Verifies that default attributes are stripped from function node defs."""
     with self.cached_session():
@@ -261,6 +265,7 @@ class SimpleMetaGraphTest(test.TestCase):
       self.assertEqual(node_def.attr["attr_1"].i, 1)
       self.assertTrue(meta_graph_def.meta_info_def.stripped_default_attrs)
 
+  @test_util.run_deprecated_v1
   def testVariableObjectsAreSharedAmongCollections(self):
     with ops.Graph().as_default() as graph1:
       v = variables.Variable(3.0)
@@ -454,6 +459,7 @@ class ScopedMetaGraphTest(test.TestCase):
 
   # Verifies that we can export the subgraph under each layer and import
   # them into new layers in a new graph.
+  @test_util.run_deprecated_v1
   def testScopedExportAndImport(self):
     test_dir = _TestDir("scoped_export_import")
     filenames = [
@@ -522,6 +528,7 @@ class ScopedMetaGraphTest(test.TestCase):
         actual_grad_value = self.evaluate(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
+  @test_util.run_deprecated_v1
   def testImportWhileLoopInWhileLoop(self):
     # Create a simple while loop.
     with ops.Graph().as_default():
@@ -547,6 +554,7 @@ class ScopedMetaGraphTest(test.TestCase):
         self.evaluate(variables.global_variables_initializer())
         self.evaluate(x)
 
+  @test_util.run_deprecated_v1
   def testScopedImportUnderNameScope(self):
     graph = ops.Graph()
     with graph.as_default():
@@ -562,6 +570,7 @@ class ScopedMetaGraphTest(test.TestCase):
         self.assertEqual(list(imported_variables.values())[0].name,
                          "foo/bar/myvar:0")
 
+  @test_util.run_deprecated_v1
   def testScopedImportUnderNameScopeNoVarScope(self):
     graph = ops.Graph()
     with graph.as_default():
@@ -590,6 +599,7 @@ class ScopedMetaGraphTest(test.TestCase):
         self.assertEqual(list(imported_variables.values())[0].name,
                          "s" + suffix + "/v:0")
 
+  @test_util.run_deprecated_v1
   def testScopedImportWithSelectedCollections(self):
     meta_graph_filename = os.path.join(
         _TestDir("selected_collections_import"), "meta_graph.pb")
@@ -687,6 +697,7 @@ class ScopedMetaGraphTest(test.TestCase):
 
   # Verifies that we can export the subgraph containing a FIFOQueue under
   # "queue1" and import it into "new_queue1" in a new graph.
+  @test_util.run_deprecated_v1
   def testScopedWithQueue(self):
     test_dir = _TestDir("scoped_with_queue")
     orig_meta_graph = self._testScopedExportWithQueue(test_dir,
@@ -749,12 +760,15 @@ class ScopedMetaGraphTest(test.TestCase):
     for n, e in zip(nodes, expected):
       self.assertEqual([e], graph2.get_operation_by_name(n).get_attr("_class"))
 
+  @test_util.run_deprecated_v1
   def testExportNestedNames(self):
     self.doTestExportNestedNames(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testExportNestedNamesResource(self):
     self.doTestExportNestedNames(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testPotentialCycle(self):
     graph1 = ops.Graph()
     with graph1.as_default():
@@ -783,6 +797,7 @@ class ScopedMetaGraphTest(test.TestCase):
                   4.0, shape=[2, 2])
           })
 
+  @test_util.run_deprecated_v1
   def testClearDevices(self):
     graph1 = ops.Graph()
     with graph1.as_default():
@@ -842,6 +857,7 @@ class ScopedMetaGraphTest(test.TestCase):
 
 class MetaGraphWithVariableScopeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
 
     def _enqueue_vector(sess, queue, values, shape=None):
@@ -899,6 +915,7 @@ class MetaGraphWithVariableScopeTest(test.TestCase):
 
 class ExportImportAcrossScopesTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testPartionedVariables(self):
 
     def make_graph_with_partitioned_variables(use_resource):
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 9c9ef799f7..04a9ed05fd 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -57,11 +57,13 @@ ops._set_call_cpp_shape_fn(common_shapes.call_cpp_shape_fn)
 
 class ResourceTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBuildGraph(self):
     with self.cached_session():
       pt = test_ops.stub_resource_handle_op(container="a", shared_name="b")
       test_ops.resource_create_op(pt).run()
 
+  @test_util.run_deprecated_v1
   def testInitialize(self):
     with self.cached_session():
       handle = test_ops.stub_resource_handle_op(container="a", shared_name="b")
@@ -106,6 +108,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase):
       c = a + b
       self.assertEqual([2, 3], c.shape)
 
+  @test_util.run_deprecated_v1
   def testUnknownDim(self):
     with self.cached_session():
       a = array_ops.placeholder(dtype=dtypes.float32, shape=[2, None, 3])
@@ -113,6 +116,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase):
       c = a + b
       self.assertEqual([2, None, 3], c.shape.as_list())
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     with self.cached_session():
       a = array_ops.placeholder(dtype=dtypes.float32, shape=None)
@@ -120,6 +124,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase):
       c = a + b
       self.assertEqual(tensor_shape.unknown_shape(), c.shape)
 
+  @test_util.run_deprecated_v1
   def testScalarShape(self):
     with self.cached_session():
       a = array_ops.placeholder(dtype=dtypes.float32, shape=[])
@@ -127,6 +132,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase):
       c = a + b
       self.assertEqual(tensor_shape.scalar(), c.shape)
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionError(self):
     with self.cached_session():
       a = array_ops.ones([1, 2, 3])
@@ -140,6 +146,7 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase):
 
 class IndexedSlicesTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testToTensor(self):
     with self.cached_session():
       values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
@@ -149,6 +156,7 @@ class IndexedSlicesTest(test_util.TensorFlowTestCase):
       tensor = ops.convert_to_tensor(x, name="tensor")
       self.assertAllEqual(tensor.eval(), [[2, 3], [0, 0], [5, 7]])
 
+  @test_util.run_deprecated_v1
   def testNegation(self):
     with self.cached_session():
       values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
@@ -157,6 +165,7 @@ class IndexedSlicesTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(x.values.eval(), [[-2, -3], [-5, -7]])
       self.assertAllEqual(x.indices.eval(), [0, 2])
 
+  @test_util.run_deprecated_v1
   def testScalarMul(self):
     with self.cached_session():
       values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
@@ -190,6 +199,7 @@ def _apply_op(g, *args, **kwargs):
 
 class OperationTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testNoInputs(self):
     op = test_ops.float_output_string_output(name="myop").a.op
     self.assertEqual(2, len(op.values()))
@@ -212,6 +222,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertProtoEquals("op:'FloatOutputStringOutput' name:'myop'",
                            op.node_def)
 
+  @test_util.run_deprecated_v1
   def testNoOutputs(self):
     op1 = test_ops.float_output(name="myop1").op
     float_t, = op1.values()
@@ -227,6 +238,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertProtoEquals("op:'FloatInput' name:'myop2' input:'myop1'",
                            op2.node_def)
 
+  @test_util.run_deprecated_v1
   def testInputsAndOutputs(self):
     op1 = test_ops.float_output(name="myop1").op
     self.assertEqual(1, len(op1.values()))
@@ -308,6 +320,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     with self.assertRaises(ValueError):
       ops.Operation(ops._NodeDef("op", "invalid:0"), g)
 
+  @test_util.run_deprecated_v1
   def testNoShapeFunction(self):
     op = test_ops.a()
     self.assertEqual(tensor_shape.unknown_shape(), op.get_shape())
@@ -333,6 +346,7 @@ class OperationTest(test_util.TensorFlowTestCase):
       converted = ops.convert_to_tensor(1)
       self.assertTrue(isinstance(converted, ops.EagerTensor))
 
+  @test_util.run_deprecated_v1
   def testConvertToTensorNestedTuple(self):
     with self.cached_session():
       values = ((2,), (3,), (5,), (7,))
@@ -384,6 +398,7 @@ class OperationTest(test_util.TensorFlowTestCase):
       values = [1.23]
       _ = ops.convert_to_tensor(values, dtype=dtypes.int64)
 
+  @test_util.run_deprecated_v1
   def testNoConvert(self):
     # Operation cannot be converted to Tensor.
     op = control_flow_ops.no_op()
@@ -401,6 +416,7 @@ class OperationTest(test_util.TensorFlowTestCase):
         ops._NodeDef("None", "op1"), ops.Graph(), [], [dtypes.float32])
     self.assertEqual("<tf.Operation 'op1' type=None>", repr(op))
 
+  @test_util.run_deprecated_v1
   def testGetAttr(self):
     op = test_ops.default_attrs()
     self.assertEqual(op.get_attr("string_val"), b"abc")
@@ -446,6 +462,7 @@ class OperationTest(test_util.TensorFlowTestCase):
 
   # TODO(b/65162920): remove this test when users who are directly mutating the
   # node_def have been updated to proper usage.
+  @test_util.run_deprecated_v1
   def testSetAttr(self):
     op = test_ops.int_attr().op
     op._set_attr("foo", attr_value_pb2.AttrValue(i=2))
@@ -466,6 +483,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(z.control_inputs, [x, y])
     self.assertEqual(x._control_outputs, [z])
 
+  @test_util.run_deprecated_v1
   def testRemoveAllControlInputs(self):
     a = constant_op.constant(1)
     with ops.control_dependencies([a]):
@@ -490,6 +508,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(f.op.control_inputs, [])
     self.assertEqual(list(f.op.inputs), [d, e])
 
+  @test_util.run_deprecated_v1
   def testControlInputCycle(self):
     graph = ops.Graph()
     with graph.as_default():
@@ -582,6 +601,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     ):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
+  @test_util.run_deprecated_v1
   def testOpDef(self):
     x = constant_op.constant(0)
     y = constant_op.constant(1)
@@ -681,6 +701,7 @@ class CreateOpTest(test_util.TensorFlowTestCase):
 # the low-level behavior.
 class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     g = ops.Graph()
     with g.as_default():
@@ -731,6 +752,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(op3.name, "myop_2")
     self.assertEqual(op4.name, "myop_1_1")
 
+  @test_util.run_deprecated_v1
   def testCond(self):
     g = ops.Graph()
     with g.as_default():
@@ -760,6 +782,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
                      "cond/cond_text")
     # pylint: enable=protected-access
 
+  @test_util.run_deprecated_v1
   def testWhileLoop(self):
     g = ops.Graph()
     with g.as_default():
@@ -789,6 +812,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
                      "myloop/while_context")
     # pylint: enable=protected-access
 
+  @test_util.run_deprecated_v1
   def testWhileLoopWithInternalControlDep(self):
     g = ops.Graph()
     with g.as_default():
@@ -812,6 +836,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
     # Internal control dep is preserved
     self.assertEqual(op.control_inputs, [c])
 
+  @test_util.run_deprecated_v1
   def testWhileLoopWithExternalControlDep(self):
     g = ops.Graph()
     with g.as_default():
@@ -945,6 +970,7 @@ class NameStackTest(test_util.TensorFlowTestCase):
     self.assertEqual("bar_2", g.unique_name("bar", mark_as_used=False))
     self.assertEqual("bar_2", g.unique_name("bar"))
 
+  @test_util.run_deprecated_v1
   def testNameAndVariableScope(self):
     with self.cached_session() as sess:
       with sess.graph.name_scope("l0"):
@@ -1671,6 +1697,7 @@ def _CopyOverrideGrad(op, x_grad):  # pylint: disable=invalid-name
 
 class RegistrationTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testRegisterGradients(self):
     x = test_ops.float_output()
     y = test_ops.copy_op(x)
@@ -1710,6 +1737,7 @@ class ComparisonTest(test_util.TensorFlowTestCase):
 
 class ControlDependenciesTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     g = ops.Graph()
     with g.as_default():
@@ -1953,6 +1981,7 @@ class OpScopeTest(test_util.TensorFlowTestCase):
       with ops.name_scope(None, "default2") as scope2:
         self.assertEqual(scope2, "default/default2/")
 
+  @test_util.run_deprecated_v1
   def testNoScopeName(self):
     g0 = ops.Graph()
     values = [
@@ -1966,6 +1995,7 @@ class OpScopeTest(test_util.TensorFlowTestCase):
       with ops.name_scope(None, None, values):
         pass
 
+  @test_util.run_deprecated_v1
   def testEmptyScopeName(self):
     g0 = ops.Graph()
     a = g0.create_op("A", [], [dtypes.float32])
@@ -1977,6 +2007,7 @@ class OpScopeTest(test_util.TensorFlowTestCase):
       self.assertEqual("", scope)
       self.assertEqual(g0, ops.get_default_graph())
 
+  @test_util.run_deprecated_v1
   def testDefaultScopeName(self):
     g0 = ops.Graph()
     a = g0.create_op("A", [], [dtypes.float32])
@@ -2001,12 +2032,14 @@ class OpScopeTest(test_util.TensorFlowTestCase):
       with ops.name_scope(scope_name, values=graph_elements + [a]):
         pass
 
+  @test_util.run_deprecated_v1
   def testTensor(self):
     g0 = ops.Graph()
     a = g0.create_op("A", [], [dtypes.float32])
     b = g0.create_op("B", [], [dtypes.float32])
     self._testGraphElements([a, b])
 
+  @test_util.run_deprecated_v1
   def testSparseTensor(self):
     g0 = ops.Graph()
     a = g0.create_op("A", [], [dtypes.float32])
@@ -2017,6 +2050,7 @@ class OpScopeTest(test_util.TensorFlowTestCase):
         _apply_op(g0, "Int64Output", [], [dtypes.int64]))
     self._testGraphElements([a, sparse, b])
 
+  @test_util.run_deprecated_v1
   def testVariable(self):
     g0 = ops.Graph()
     with g0.as_default():
@@ -2221,6 +2255,7 @@ class InitScopeTest(test_util.TensorFlowTestCase):
       self.assertEqual(4, int(compiled_outer(inner=compiled_inner)))
       self.assertEqual(7, int(compiled_outer(inner=compiled_inner)))
 
+  @test_util.run_deprecated_v1
   def testFallsBackToGlobalGraphWhenAllGraphsAreBuildingFunctions(self):
     with context.graph_mode():
       ops.reset_default_graph()
@@ -2357,6 +2392,7 @@ class GraphTest(test_util.TensorFlowTestCase):
     g.prevent_feeding(a)
     self.assertFalse(g.is_feedable(a))
 
+  @test_util.run_deprecated_v1
   def testPreventFetching(self):
     g = ops.Graph()
     a = constant_op.constant(2.0)
@@ -2440,10 +2476,12 @@ class AttrScopeTest(test_util.TensorFlowTestCase):
       b = None
     return (a, b)
 
+  @test_util.run_deprecated_v1
   def testNoLabel(self):
     with self.cached_session():
       self.assertAllEqual((None, None), self._get_test_attrs())
 
+  @test_util.run_deprecated_v1
   def testLabelMap(self):
     with self.cached_session() as sess:
       a1 = self._get_test_attrs()
@@ -2478,11 +2516,13 @@ ops.RegisterShape("KernelLabel")(common_shapes.scalar_shape)
 
 class KernelLabelTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testNoLabel(self):
     with self.cached_session():
       self.assertAllEqual(b"My label is: default",
                           test_ops.kernel_label().eval())
 
+  @test_util.run_deprecated_v1
   def testLabelMap(self):
     with self.cached_session() as sess:
       default_1 = test_ops.kernel_label()
@@ -2599,6 +2639,7 @@ class StatisticsTest(test_util.TensorFlowTestCase):
 
 class DeviceStackTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBasicDeviceAssignmentMetadata(self):
 
     def device_func(unused_op):
@@ -2630,6 +2671,7 @@ class DeviceStackTest(test_util.TensorFlowTestCase):
     expected_regex = r"device_func<.*ops_test.py, [0-9]+"
     self.assertRegexpMatches(func_description, expected_regex)
 
+  @test_util.run_deprecated_v1
   def testDeviceAssignmentMetadataForGraphDeviceAndTfDeviceFunctions(self):
 
     with ops.device("/cpu"):
@@ -2649,6 +2691,7 @@ class DeviceStackTest(test_util.TensorFlowTestCase):
 
 class ColocationGroupTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     a = constant_op.constant([2.0], name="a")
     with ops.colocate_with(a.op):
@@ -2659,6 +2702,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     with self.assertRaises(ValueError):
       c.op.get_attr("_class")
 
+  @test_util.run_deprecated_v1
   def testBasicColocationMetadata(self):
     const_two = constant_op.constant([2.0], name="two")
     with ops.colocate_with(const_two.op):
@@ -2671,6 +2715,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     # colocation statement.
     self.assertEqual("ops_test.py", os.path.basename(metadata.filename))
 
+  @test_util.run_deprecated_v1
   def testColocationDeviceInteraction(self):
     with ops.device("/cpu:0"):
       with ops.device("/device:GPU:0"):
@@ -2683,6 +2728,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     self.assertEqual([b"loc:@a"], b.op.colocation_groups())
     self.assertEqual(a.op.device, b.op.device)
 
+  @test_util.run_deprecated_v1
   def testColocationCanonicalization(self):
     with ops.device("/device:GPU:0"):
       _ = constant_op.constant(2.0)
@@ -2698,6 +2744,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     # inherits B's device name, after canonicalizing the names.
     self.assertEqual(b.op.device, c.op.device)
 
+  @test_util.run_deprecated_v1
   def testLocationOverrides(self):
     with ops.device("/cpu:0"):
       with ops.device("/device:GPU:0"):
@@ -2719,6 +2766,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     self.assertEqual("/device:GPU:0", c.op.device)
     self.assertEqual("/device:CPU:0", d.op.device)
 
+  @test_util.run_deprecated_v1
   def testNestedColocateWith(self):
     a = constant_op.constant([2.0], name="a")
     with ops.colocate_with(a.op):
@@ -2728,6 +2776,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     self.assertEqual([b"loc:@a"], b.op.colocation_groups())
     self.assertEqual([b"loc:@a"], c.op.colocation_groups())
 
+  @test_util.run_deprecated_v1
   def testMultiColocationGroups(self):
     a = constant_op.constant([2.0], name="a")
     b = constant_op.constant(3.0, name="b")
@@ -2736,6 +2785,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
         c = constant_op.constant(4.0)
     self.assertEqual(set([b"loc:@a", b"loc:@b"]), set(c.op.colocation_groups()))
 
+  @test_util.run_deprecated_v1
   def testColocationIgnoreStack(self):
     a = constant_op.constant([2.0], name="a")
     b = constant_op.constant(3.0, name="b")
@@ -2744,6 +2794,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
         c = constant_op.constant(4.0)
     self.assertEqual(set([b"loc:@b"]), set(c.op.colocation_groups()))
 
+  @test_util.run_deprecated_v1
   def testColocateWithReset(self):
     a = constant_op.constant([2.0], name="a")
     with ops.colocate_with(a.op):
@@ -2753,6 +2804,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     self.assertEqual([b"loc:@a"], b.op.colocation_groups())
     self.assertEqual([b"loc:@c"], c.op.colocation_groups())
 
+  @test_util.run_deprecated_v1
   def testColocateWithInitialNoneThenNested(self):
     a = constant_op.constant([2.0], name="a")
     with ops.colocate_with(a.op):
@@ -2763,12 +2815,14 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     self.assertEqual([b"loc:@b"], b.op.colocation_groups())
     self.assertEqual([b"loc:@b"], c.op.colocation_groups())
 
+  @test_util.run_deprecated_v1
   def testColocateVariables(self):
     a = variables.Variable([2.0], name="a")
     with ops.colocate_with(a.op):
       b = variables.Variable([3.0], name="b")
     self.assertEqual([b"loc:@a"], b.op.colocation_groups())
 
+  @test_util.run_deprecated_v1
   def testInconsistentDeviceWithinColocate(self):
     with ops.device("/device:GPU:0"):
       a = constant_op.constant([2.0], name="a")
@@ -2782,6 +2836,7 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
 
     self.assertEqual("/device:CPU:0", b.device)
 
+  @test_util.run_deprecated_v1
   def testMakeColocationConflictMessage(self):
     """Test that provides an example of a complicated error message."""
     # We could test the message with any ops, but this test will be more
@@ -2926,6 +2981,7 @@ class NameScopeTest(test_util.TensorFlowTestCase):
 
 class TracebackTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testTracebackWithStartLines(self):
     with self.cached_session() as sess:
       a = constant_op.constant(2.0)
@@ -2947,6 +3003,7 @@ class TracebackTest(test_util.TensorFlowTestCase):
 
 class EnableEagerExecutionTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBadArgumentsToEnableEagerExecution(self):
     with self.assertRaisesRegexp(TypeError, "config must be a tf.ConfigProto"):
       ops.enable_eager_execution(context.DEVICE_PLACEMENT_SILENT)
diff --git a/tensorflow/python/framework/smart_cond_test.py b/tensorflow/python/framework/smart_cond_test.py
index 174ada9fe1..f964c87f02 100644
--- a/tensorflow/python/framework/smart_cond_test.py
+++ b/tensorflow/python/framework/smart_cond_test.py
@@ -35,6 +35,7 @@ def raise_exception():
 
 class SmartCondTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testTrue(self):
     with ops.Graph().as_default():
       with session.Session():
@@ -44,6 +45,7 @@ class SmartCondTest(test_util.TensorFlowTestCase):
                                   lambda: math_ops.multiply(y, 5))
         self.assertEqual(z.eval(), 32)
 
+  @test_util.run_deprecated_v1
   def testFalse(self):
     with ops.Graph().as_default():
       with session.Session():
@@ -99,6 +101,7 @@ class SmartCondTest(test_util.TensorFlowTestCase):
 
 class SmartCaseTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testTrue(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     conditions = [(True, lambda: constant_op.constant(1)),
@@ -112,6 +115,7 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
       self.assertEqual(self.evaluate(y), 1)
       self.assertEqual(self.evaluate(z), 1)
 
+  @test_util.run_deprecated_v1
   def testFalse(self):
     conditions = [(False, raise_exception)]
     y = smart_cond.smart_case(conditions,
@@ -124,6 +128,7 @@ class SmartCaseTest(test_util.TensorFlowTestCase):
       self.assertEqual(self.evaluate(y), 1)
       self.assertEqual(self.evaluate(z), 1)
 
+  @test_util.run_deprecated_v1
   def testMix(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     y = constant_op.constant(10)
diff --git a/tensorflow/python/framework/sparse_tensor_test.py b/tensorflow/python/framework/sparse_tensor_test.py
index 9ee1bd75a5..a999c12ca8 100644
--- a/tensorflow/python/framework/sparse_tensor_test.py
+++ b/tensorflow/python/framework/sparse_tensor_test.py
@@ -65,6 +65,7 @@ class SparseTensorTest(test_util.TensorFlowTestCase):
         sparse_tensor.is_sparse(
             sparse_tensor.SparseTensorValue([[0]], [0], [1])))
 
+  @test_util.run_deprecated_v1
   def testConsumers(self):
     sp = sparse_tensor.SparseTensor([[0, 0], [1, 2]], [1.0, 3.0], [3, 4])
     w = ops.convert_to_tensor(np.ones([4, 1], np.float32))
@@ -87,6 +88,7 @@ class ConvertToTensorOrSparseTensorTest(test_util.TensorFlowTestCase):
           value)
       self.assertAllEqual(value, self.evaluate(from_value))
 
+  @test_util.run_deprecated_v1
   def test_convert_sparse(self):
     with self.cached_session():
       indices = [[0, 1], [1, 0]]
diff --git a/tensorflow/python/framework/subscribe_test.py b/tensorflow/python/framework/subscribe_test.py
index 5322204ce6..61c6ea6519 100644
--- a/tensorflow/python/framework/subscribe_test.py
+++ b/tensorflow/python/framework/subscribe_test.py
@@ -43,6 +43,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertTrue(
         all(subscribe._is_subscribed_identity(x) for x in container))
 
+  @test_util.run_deprecated_v1
   def testSideEffect(self):
     a = constant_op.constant(1)
     b = constant_op.constant(1)
@@ -75,6 +76,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertEqual(d_out, [42])
     self.assertEqual(shared, [2, 2, 2])
 
+  @test_util.run_deprecated_v1
   def testSupportedTypes(self):
     """Confirm that supported types are correctly detected and handled."""
 
@@ -120,6 +122,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
       subscribe.subscribe(c.name,
                           lambda t: script_ops.py_func(sub, [t], [t.dtype]))
 
+  @test_util.run_deprecated_v1
   def testCaching(self):
     """Confirm caching of control output is recalculated between calls."""
     a = constant_op.constant(1)
@@ -152,6 +155,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertEqual(d_out, [11])
     self.assertEqual(shared, {2: 1, 1: 1})
 
+  @test_util.run_deprecated_v1
   def testIsSubscribedIdentity(self):
     """Confirm subscribed identity ops are correctly detected."""
     a = constant_op.constant(1)
@@ -165,6 +169,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertFalse(subscribe._is_subscribed_identity(idop))
     self.assertTrue(subscribe._is_subscribed_identity(c_sub))
 
+  @test_util.run_deprecated_v1
   def testSubscribeExtend(self):
     """Confirm side effect are correctly added for different input types."""
     a = constant_op.constant(1)
@@ -210,6 +215,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertIn('graph2', shared)
     self.assertIn('graph3', shared)
 
+  @test_util.run_deprecated_v1
   def testSubscribeVariable(self):
     """Confirm that variables can be subscribed."""
     v1 = variables.VariableV1(0.0)
@@ -248,6 +254,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
       # Make sure the values read from the variable match the expected ones.
       self.assertEqual([0.0, 3.0], shared)
 
+  @test_util.run_deprecated_v1
   def testResourceType(self):
     """Confirm that subscribe correctly handles tensors with 'resource' type."""
     tensor_array = tensor_array_ops.TensorArray(
@@ -276,6 +283,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
       self.evaluate([reader])
     self.assertEqual(0, len(shared))
 
+  @test_util.run_deprecated_v1
   def testMultipleOutputs(self):
     """Handle subscriptions to multiple outputs from the same op."""
     sparse_tensor_1 = sparse_tensor.SparseTensor(
@@ -309,6 +317,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     # All three ops have been processed.
     self.assertEqual(3, len(shared))
 
+  @test_util.run_deprecated_v1
   def test_subscribe_tensors_on_different_devices(self):
     """Side effect ops are added with the same device of the subscribed op."""
     c1 = constant_op.constant(10)
@@ -335,6 +344,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertEqual(add.device, add_sub.device)
     self.assertEqual(mul.device, mul_sub.device)
 
+  @test_util.run_deprecated_v1
   def test_subscribe_tensors_within_control_flow_context(self):
     """Side effect ops are added with the same control flow context."""
     c1 = constant_op.constant(10)
diff --git a/tensorflow/python/framework/tensor_spec_test.py b/tensorflow/python/framework/tensor_spec_test.py
index e3aad7cc23..75c197df09 100644
--- a/tensorflow/python/framework/tensor_spec_test.py
+++ b/tensorflow/python/framework/tensor_spec_test.py
@@ -45,6 +45,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase):
     desc = tensor_spec.TensorSpec(shape=None, dtype=dtypes.float32)
     self.assertEqual(desc.shape, tensor_shape.TensorShape(None))
 
+  @test_util.run_deprecated_v1
   def testShapeCompatibility(self):
     unknown = array_ops.placeholder(dtypes.int64)
     partial = array_ops.placeholder(dtypes.int64, shape=[None, 1])
@@ -75,6 +76,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase):
     self.assertFalse(desc_rank3.is_compatible_with(full))
     self.assertTrue(desc_rank3.is_compatible_with(rank3))
 
+  @test_util.run_deprecated_v1
   def testTypeCompatibility(self):
     floats = array_ops.placeholder(dtypes.float32, shape=[10, 10])
     ints = array_ops.placeholder(dtypes.int32, shape=[10, 10])
@@ -106,6 +108,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase):
     spec_2 = tensor_spec.TensorSpec.from_spec(spec_1)
     self.assertEqual(spec_1, spec_2)
 
+  @test_util.run_deprecated_v1
   def testFromTensor(self):
     zero = constant_op.constant(0)
     spec = tensor_spec.TensorSpec.from_tensor(zero)
@@ -113,6 +116,7 @@ class TensorSpecTest(test_util.TensorFlowTestCase):
     self.assertEqual(spec.shape, [])
     self.assertEqual(spec.name, "Const")
 
+  @test_util.run_deprecated_v1
   def testFromPlaceholder(self):
     unknown = array_ops.placeholder(dtypes.int64, name="unknown")
     partial = array_ops.placeholder(dtypes.float32,
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index 87d65c8c46..0033754618 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -758,6 +758,7 @@ class TensorUtilTest(test.TestCase):
     self.assertFalse(tensor_util.ShapeEquals(t, [1, 4]))
     self.assertFalse(tensor_util.ShapeEquals(t, [4]))
 
+  @test_util.run_deprecated_v1
   def testMockArray(self):
 
     class MockArray(object):
@@ -787,6 +788,7 @@ class ConstantValueTest(test.TestCase):
     tf_val = constant_op.constant(np_val)
     self.assertAllClose(np_val, tensor_util.constant_value(tf_val))
 
+  @test_util.run_deprecated_v1
   def testUnknown(self):
     tf_val = gen_state_ops.variable(
         shape=[3, 4, 7],
@@ -815,12 +817,14 @@ class ConstantValueTest(test.TestCase):
     c_val = tensor_util.constant_value(tf_val)
     self.assertEqual(6, c_val)
 
+  @test_util.run_deprecated_v1
   def testSizeOfScalar(self):
     tf_val = array_ops.size(constant_op.constant(0.0))
     c_val = tensor_util.constant_value(tf_val)
     self.assertEqual(1, c_val)
     self.assertEqual(np.ndarray, type(c_val))
 
+  @test_util.run_deprecated_v1
   def testRank(self):
     tf_val = array_ops.rank(constant_op.constant(0.0, shape=[1, 2, 3]))
     c_val = tensor_util.constant_value(tf_val)
@@ -852,6 +856,7 @@ class ConstantValueTest(test.TestCase):
     c_val = tensor_util.constant_value(tf_val)
     self.assertAllClose(np_val.astype(np.float64), c_val)
 
+  @test_util.run_deprecated_v1
   def testConcat(self):
     np_val = np.random.rand(3, 4, 7).astype(np.float32)
     tf_val = array_ops.concat(
@@ -871,6 +876,7 @@ class ConstantValueTest(test.TestCase):
     c_val = tensor_util.constant_value(tf_val)
     self.assertIs(None, c_val)
 
+  @test_util.run_deprecated_v1
   def testPack_Axis0(self):
     inputs = [np.random.rand(4, 7) for _ in range(3)]
     np_val = np.array(inputs)
@@ -883,6 +889,7 @@ class ConstantValueTest(test.TestCase):
     c_val = tensor_util.constant_value(tf_val)
     self.assertIs(None, c_val)
 
+  @test_util.run_deprecated_v1
   def testPack_Axis1(self):
     inputs = [np.random.rand(4, 7) for _ in range(3)]
     tf_val = array_ops.stack(inputs, axis=1)
@@ -894,6 +901,7 @@ class ConstantValueTest(test.TestCase):
     c_val = tensor_util.constant_value(tf_val)
     self.assertIs(None, c_val)
 
+  @test_util.run_deprecated_v1
   def testPack_Partial_Axis0(self):
     input_ = np.random.rand(4, 7)
     tf_val = array_ops.stack([input_, array_ops.placeholder(dtypes.float32)])
@@ -901,6 +909,7 @@ class ConstantValueTest(test.TestCase):
     self.assertAllClose(input_, c_val[0])
     self.assertIsNone(c_val[1])
 
+  @test_util.run_deprecated_v1
   def testPack_Partial_Axis1(self):
     input_ = np.random.rand(4, 7)
     tf_val = array_ops.stack([input_, array_ops.placeholder(dtypes.float32)],
@@ -966,12 +975,14 @@ class ConstantValueAsShapeTest(test.TestCase):
     c_val = tensor_util.constant_value_as_shape(tf_val)
     self.assertEqual([None, 1, None], c_val.as_list())
 
+  @test_util.run_deprecated_v1
   def testPack(self):
     tf_val = array_ops.stack(
         [constant_op.constant(16), 37, array_ops.placeholder(dtypes.int32)])
     c_val = tensor_util.constant_value_as_shape(tf_val)
     self.assertEqual([16, 37, None], c_val.as_list())
 
+  @test_util.run_deprecated_v1
   def testConcat(self):
     tf_val = array_ops.concat(
         [[16, 37], array_ops.placeholder(
@@ -985,6 +996,7 @@ class ConstantValueAsShapeTest(test.TestCase):
     c_val = tensor_util.constant_value_as_shape(tf_val)
     self.assertEqual([16, 37, None, 48], c_val.as_list())
 
+  @test_util.run_deprecated_v1
   def testSlice(self):
     tf_val = array_ops.placeholder(dtypes.int32, shape=(4,))[0:2]
     c_val = tensor_util.constant_value_as_shape(tf_val)
diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py
index 2a37253db6..dfdced5a98 100644
--- a/tensorflow/python/framework/test_util_test.py
+++ b/tensorflow/python/framework/test_util_test.py
@@ -49,6 +49,7 @@ from tensorflow.python.platform import googletest
 
 class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_assert_ops_in_graph(self):
     with self.test_session():
       constant_op.constant(["hello", "taffy"], name="hello")
@@ -60,6 +61,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertRaises(ValueError, test_util.assert_ops_in_graph,
                       {"hello": "Variable"}, ops.get_default_graph())
 
+  @test_util.run_deprecated_v1
   def test_session_functions(self):
     with self.test_session() as sess:
       sess_ref = weakref.ref(sess)
@@ -551,6 +553,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.assertRaises(AssertionError):
       self.assertAllLessEqual(x, 95.0)
 
+  @test_util.run_deprecated_v1
   def testAssertAllInRangeWithNonNumericValuesFails(self):
     s1 = constant_op.constant("Hello, ", name="s1")
     c = constant_op.constant([1 + 2j, -3 + 5j], name="c")
@@ -614,6 +617,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.assertRaises(AssertionError):
       self.assertAllInSet(x, (42,))
 
+  @test_util.run_deprecated_v1
   def testRandomSeed(self):
     # Call setUp again for WithCApi case (since it makes a new defeault graph
     # after setup).
@@ -706,6 +710,7 @@ class TestUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     test_util.run_in_graph_and_eager_modes(_test)(self)
     self.assertEqual(modes, ["graph"])
 
+  @test_util.run_deprecated_v1
   def test_run_in_graph_and_eager_modes_setup_in_same_mode(self):
     modes = []
     mode_name = lambda: "eager" if context.executing_eagerly() else "graph"
diff --git a/tensorflow/python/grappler/cost_analyzer_test.py b/tensorflow/python/grappler/cost_analyzer_test.py
index de80df1879..ee3e289f65 100644
--- a/tensorflow/python/grappler/cost_analyzer_test.py
+++ b/tensorflow/python/grappler/cost_analyzer_test.py
@@ -38,6 +38,7 @@ from tensorflow.python.training import adam
 
 class CostAnalysisTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasicCost(self):
     """Make sure arguments can be passed correctly."""
     a = constant_op.constant(10, name="a")
@@ -62,6 +63,7 @@ class CostAnalysisTest(test.TestCase):
     # Also print the report to make it easier to debug
     print("{}".format(report))
 
+  @test_util.run_deprecated_v1
   def testVerbose(self):
     """Make sure the full report is generated with verbose=True."""
     a = constant_op.constant(10, name="a")
@@ -81,6 +83,7 @@ class CostAnalysisTest(test.TestCase):
     # Also print the report to make it easier to debug
     print("{}".format(report))
 
+  @test_util.run_deprecated_v1
   def testSmallNetworkCost(self):
     image = array_ops.placeholder(dtypes.float32, shape=[1, 28, 28, 1])
     label = array_ops.placeholder(dtypes.float32, shape=[1, 10])
@@ -129,6 +132,7 @@ class CostAnalysisTest(test.TestCase):
       # self.assertTrue(0 < upper)
       # self.assertTrue(lower <= upper)
 
+  @test_util.run_deprecated_v1
   def testBasicMemory(self):
     """Make sure arguments can be passed correctly."""
     with test_util.device(use_gpu=False):
diff --git a/tensorflow/python/grappler/item_test.py b/tensorflow/python/grappler/item_test.py
index d3d96c646c..78604b259c 100644
--- a/tensorflow/python/grappler/item_test.py
+++ b/tensorflow/python/grappler/item_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import item
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_array_ops
@@ -107,6 +108,7 @@ class ItemTest(test.TestCase):
     newest_tf_item = grappler_item.tf_item
     self.assertEqual(new_tf_item, newest_tf_item)
 
+  @test_util.run_deprecated_v1
   def testColocationContraints(self):
     with ops.Graph().as_default() as g:
       c = constant_op.constant([10])
diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 55ccfbb93c..98f2e6d718 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import cluster as gcluster
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.layers import convolutional as conv_layers
@@ -1441,6 +1442,7 @@ class LayoutOptimizerTest(test.TestCase):
       self._assert_trans_nchw_to_nhwc('Add-0-0', nodes)
       self.assertAllClose(output_val_ref, output_val, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     meta_graph = _simple_metagraph()
     config = config_pb2.ConfigProto()
@@ -1458,6 +1460,7 @@ class LayoutOptimizerTest(test.TestCase):
         self.assertEqual(node.attr['data_format'].s, b'NCHW')
     self.assertEqual(found, 5)
 
+  @test_util.run_deprecated_v1
   def testDepthwise(self):
     meta_graph = _simple_metagraph(depthwise=True)
     config = config_pb2.ConfigProto()
diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py
index d233629cbb..6eb16fbd39 100644
--- a/tensorflow/python/grappler/memory_optimizer_test.py
+++ b/tensorflow/python/grappler/memory_optimizer_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -37,6 +38,7 @@ from tensorflow.python.training import training as train
 class MemoryOptimizerSwapTest(test.TestCase):
   """Tests the Grappler memory optimizer."""
 
+  @test_util.run_deprecated_v1
   def testNoSwapping(self):
     """Make sure the graph is preserved when there is nothing to swap."""
     a = variables.VariableV1(10, name='a')
@@ -60,6 +62,7 @@ class MemoryOptimizerSwapTest(test.TestCase):
     self.assertEqual(len(graph.node), graph_size)
     self.assertItemsEqual([node.name for node in graph.node], nodes)
 
+  @test_util.run_deprecated_v1
   def testSimpleSwap(self):
     """Check that the swap annotations are followed."""
     a = variables.VariableV1(10, name='a')
diff --git a/tensorflow/python/grappler/model_analyzer_test.py b/tensorflow/python/grappler/model_analyzer_test.py
index ec172755f1..d000cfa1ba 100644
--- a/tensorflow/python/grappler/model_analyzer_test.py
+++ b/tensorflow/python/grappler/model_analyzer_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import model_analyzer
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -28,6 +29,7 @@ from tensorflow.python.platform import test
 
 class PyWrapOptimizeGraphTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     """Make sure arguments can be passed correctly."""
     a = constant_op.constant([10, 11], name="a")
@@ -49,6 +51,7 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     # Also print the report to make it easier to debug
     print("{}".format(report))
 
+  @test_util.run_deprecated_v1
   def testDebugMode(self):
     """Make sure arguments can be passed correctly."""
     a = constant_op.constant([10, 11], name="a")
diff --git a/tensorflow/python/grappler/tf_optimizer_test.py b/tensorflow/python/grappler/tf_optimizer_test.py
index 0a4d4cbe2d..06ccaa813f 100644
--- a/tensorflow/python/grappler/tf_optimizer_test.py
+++ b/tensorflow/python/grappler/tf_optimizer_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.grappler import item as gitem
 from tensorflow.python.grappler import tf_optimizer
 from tensorflow.python.ops import array_ops
@@ -34,6 +35,7 @@ from tensorflow.python.platform import test
 
 class PyWrapOptimizeGraphTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     """Make sure arguments can be passed correctly."""
     a = constant_op.constant(10, name='a')
@@ -55,6 +57,7 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     self.assertEqual(len(graph.node), 1)
     self.assertItemsEqual([node.name for node in graph.node], ['d'])
 
+  @test_util.run_deprecated_v1
   def testKeepNodes(self):
     g = ops.Graph()
     with g.as_default():
@@ -83,6 +86,7 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     self.assertEqual(len(optimized_graph_nodes), len(expected_nodes))
     self.assertAllInSet(optimized_graph_nodes, expected_nodes)
 
+  @test_util.run_deprecated_v1
   def testLoops(self):
     g = ops.Graph()
     with g.as_default():
diff --git a/tensorflow/python/keras/activations_test.py b/tensorflow/python/keras/activations_test.py
index ad238cb0a9..6b7bfb698b 100644
--- a/tensorflow/python/keras/activations_test.py
+++ b/tensorflow/python/keras/activations_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -67,6 +68,7 @@ class KerasActivationsTest(test.TestCase):
     expected = _ref_softmax(test_values[0, 0])
     self.assertAllClose(result[0, 0], expected, rtol=1e-05)
 
+  @test_util.run_deprecated_v1
   def test_selu(self):
     x = keras.backend.placeholder(ndim=2)
     f = keras.backend.function([x], [keras.activations.selu(x)])
@@ -124,6 +126,7 @@ class KerasActivationsTest(test.TestCase):
     expected = sigmoid(test_values)
     self.assertAllClose(result, expected, rtol=1e-05)
 
+  @test_util.run_deprecated_v1
   def test_hard_sigmoid(self):
     def ref_hard_sigmoid(x):
       x = (x * 0.2) + 0.5
@@ -147,6 +150,7 @@ class KerasActivationsTest(test.TestCase):
     # No negative values in test values...
     self.assertAllClose(result, test_values, rtol=1e-05)
 
+  @test_util.run_deprecated_v1
   def test_elu(self):
     with self.cached_session():
       x = keras.backend.placeholder(ndim=2)
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 48fdd56e9f..fa32b1ecd7 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1422,6 +1422,7 @@ class TestCTC(test.TestCase):
                 decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
       self.assertAllClose(log_prob_truth, log_prob_pred)
 
+  @test_util.run_deprecated_v1
   def test_ctc_batch_cost(self):
     with self.cached_session():
       label_lens = np.expand_dims(np.asarray([5, 4]), 1)
@@ -1507,6 +1508,7 @@ class TestRandomOps(test.TestCase):
 
 class BackendGraphTests(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_is_placeholder(self):
     x = keras.backend.placeholder(shape=(1,))
     self.assertEqual(keras.backend.is_placeholder(x), True)
@@ -1546,6 +1548,7 @@ class BackendGraphTests(test.TestCase):
     output_values = f([None, None])
     self.assertEqual(output_values, [5., 6.])
 
+  @test_util.run_deprecated_v1
   def test_function_tf_feed_symbols(self):
     # Test Keras backend functions with TF tensor inputs.
     with self.cached_session():
@@ -1579,6 +1582,7 @@ class BackendGraphTests(test.TestCase):
       outs = f([y5, y2, None])
       self.assertEqual(outs, [11., 2.])
 
+  @test_util.run_deprecated_v1
   def test_function_tf_fetches(self):
     # Additional operations can be passed to tf.Session().run() via its
     # `fetches` arguments. In contrast to `updates` argument of
@@ -1601,6 +1605,7 @@ class BackendGraphTests(test.TestCase):
       self.assertEqual(keras.backend.get_session().run(fetches=[x, y]),
                        [11., 5.])
 
+  @test_util.run_deprecated_v1
   def test_function_tf_feed_dict(self):
     # Additional substitutions can be passed to `tf.Session().run()` via its
     # `feed_dict` arguments. Note that the feed_dict is passed once in the
@@ -1633,6 +1638,7 @@ class BackendGraphTests(test.TestCase):
       self.assertEqual(keras.backend.get_session().run(fetches=[x, y]),
                        [30., 40.])
 
+  @test_util.run_deprecated_v1
   def test_function_tf_run_options_with_run_metadata(self):
     with self.cached_session():
       x_placeholder = keras.backend.placeholder(shape=())
@@ -1658,6 +1664,7 @@ class BackendGraphTests(test.TestCase):
       self.assertEqual(output1, [30.])
       self.assertEqual(len(run_metadata.partition_graphs), 0)
 
+  @test_util.run_deprecated_v1
   def test_function_fetch_callbacks(self):
 
     class CallbackStub(object):
@@ -1694,6 +1701,7 @@ class BackendGraphTests(test.TestCase):
     x = keras.backend.placeholder(shape=(3, 4), sparse=True)
     self.assertEqual(x.get_shape().as_list(), [3, 4])
 
+  @test_util.run_deprecated_v1
   def test_batch_normalization(self):
     # No eager CPU kernel.
     g_val = np.random.random((3,))
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index 9d9ede22c0..ed05572c68 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -404,6 +404,7 @@ class KerasCallbacksTest(test.TestCase):
           float(keras.backend.get_value(
               model.optimizer.lr)) - 0.01 / 4) < keras.backend.epsilon()
 
+  @test_util.run_deprecated_v1
   def test_ReduceLROnPlateau(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -675,6 +676,7 @@ class KerasCallbacksTest(test.TestCase):
       self.assertEqual(len(loss), 1)
       self.assertEqual(loss[0], np.inf)
 
+  @test_util.run_deprecated_v1
   def test_TensorBoard(self):
     np.random.seed(1337)
 
@@ -778,6 +780,7 @@ class KerasCallbacksTest(test.TestCase):
           data_generator(True), len(x_train), epochs=2, callbacks=cbks)
       assert os.path.exists(temp_dir)
 
+  @test_util.run_deprecated_v1
   def test_TensorBoard_histogram_freq_must_have_validation_data(self):
     np.random.seed(1337)
     tmpdir = self.get_temp_dir()
@@ -850,6 +853,7 @@ class KerasCallbacksTest(test.TestCase):
       # Make sure file writer cache is clear to avoid failures during cleanup.
       writer_cache.FileWriterCache.clear()
 
+  @test_util.run_deprecated_v1
   def test_TensorBoard_multi_input_output(self):
     np.random.seed(1337)
     tmpdir = self.get_temp_dir()
@@ -921,6 +925,7 @@ class KerasCallbacksTest(test.TestCase):
                           callbacks=callbacks_factory(histogram_freq=1))
       assert os.path.isdir(filepath)
 
+  @test_util.run_deprecated_v1
   def test_Tensorboard_histogram_summaries_in_test_function(self):
 
     class FileWriterStub(object):
@@ -998,6 +1003,7 @@ class KerasCallbacksTest(test.TestCase):
 
       self.assertAllEqual(tsb.writer.steps_seen, [0, 0.5, 1, 1.5, 2, 2.5])
 
+  @test_util.run_deprecated_v1
   def test_Tensorboard_histogram_summaries_with_generator(self):
     np.random.seed(1337)
     tmpdir = self.get_temp_dir()
@@ -1129,6 +1135,7 @@ class KerasCallbacksTest(test.TestCase):
 
       assert os.path.exists(temp_dir)
 
+  @test_util.run_deprecated_v1
   def test_Tensorboard_batch_logging(self):
 
     class FileWriterStub(object):
@@ -1163,6 +1170,7 @@ class KerasCallbacksTest(test.TestCase):
     self.assertEqual(tb_cbk.writer.summary_values, [0., 1., 2., 3., 4.])
     self.assertEqual(tb_cbk.writer.summary_tags, ['batch_acc'] * 5)
 
+  @test_util.run_deprecated_v1
   def test_Tensorboard_epoch_and_batch_logging(self):
 
     class FileWriterStub(object):
@@ -1234,6 +1242,7 @@ class KerasCallbacksTest(test.TestCase):
 
     self.assertTrue(os.path.exists(temp_dir))
 
+  @test_util.run_deprecated_v1
   def test_TensorBoard_update_freq(self):
 
     class FileWriterStub(object):
@@ -1325,6 +1334,7 @@ class KerasCallbacksTest(test.TestCase):
             callbacks=cbks,
             epochs=1)
 
+  @test_util.run_deprecated_v1
   def test_fit_generator_with_callback(self):
 
     class TestCallback(keras.callbacks.Callback):
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py
index b92f06449e..6d9d9a2fca 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/engine/saving_test.py
@@ -289,6 +289,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                                  r'element\(s\)\.'):
       saving.load_weights_from_hdf5_group_by_name(f_model, model.layers)
 
+  @test_util.run_deprecated_v1
   def test_sequential_weight_loading_group_name_with_incorrect_shape(self):
     if h5py is None:
       return
@@ -331,6 +332,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
 
 class TestWholeModelSaving(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_sequential_model_saving(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -383,6 +385,7 @@ class TestWholeModelSaving(test.TestCase):
       out2 = new_model.predict(x)
       self.assertAllClose(out, out2, atol=1e-05)
 
+  @test_util.run_deprecated_v1
   def test_sequential_model_saving_without_input_shape(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -443,6 +446,7 @@ class TestWholeModelSaving(test.TestCase):
       out2 = new_model.predict(x)
       self.assertAllClose(out, out2, atol=1e-05)
 
+  @test_util.run_deprecated_v1
   def test_sequential_model_saving_2(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -479,6 +483,7 @@ class TestWholeModelSaving(test.TestCase):
       out2 = model.predict(x)
       self.assertAllClose(out, out2, atol=1e-05)
 
+  @test_util.run_deprecated_v1
   def test_functional_model_saving(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -630,6 +635,7 @@ class TestWholeModelSaving(test.TestCase):
       os.close(fd)
       os.remove(fname)
 
+  @test_util.run_deprecated_v1
   def test_saving_model_with_long_weights_names(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -675,6 +681,7 @@ class TestWholeModelSaving(test.TestCase):
       os.close(fd)
       os.remove(fname)
 
+  @test_util.run_deprecated_v1
   def test_model_saving_to_pre_created_h5py_file(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -749,6 +756,7 @@ class SubclassedModel(training.Model):
 
 class TestWeightSavingAndLoadingTFFormat(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_keras_optimizer_warning(self):
     graph = ops.Graph()
     with graph.as_default(), self.session(graph):
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 2e2927bf47..fbf893c663 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -132,6 +132,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     self.assertFalse(model._is_graph_network)
 
   @parameterized.parameters((True,), (False,))
+  @tf_test_util.run_deprecated_v1
   def test_training_and_eval_methods_on_symbolic_tensors(self, deferred):
     with self.cached_session():
 
@@ -219,6 +220,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     inner_model.trainable = True
     self.assertEqual(len(model.trainable_weights), 4)
 
+  @tf_test_util.run_deprecated_v1
   def test_sequential_update_disabling(self):
     val_a = np.random.random((10, 4))
     val_out = np.random.random((10, 4))
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index b4a4babf25..03bfd35589 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -42,6 +42,7 @@ except ImportError:
 
 class TopologyConstructionTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_get_updates(self):
 
     class MyLayer(keras.layers.Layer):
@@ -115,6 +116,7 @@ class TopologyConstructionTest(test.TestCase):
     self.assertEqual(len(layer.get_updates_for(x1)), 2)
     self.assertEqual(len(layer.get_updates_for(None)), 0)
 
+  @test_util.run_deprecated_v1
   def test_get_losses(self):
 
     class MyLayer(keras.layers.Layer):
@@ -268,6 +270,7 @@ class TopologyConstructionTest(test.TestCase):
     self.assertEqual(test_layer.input_shape, [(None, 32), (None, 32)])
     self.assertEqual(test_layer.output_shape, (None, 32))
 
+  @test_util.run_deprecated_v1
   def testBasicNetwork(self):
     # minimum viable network
     x = input_layer_lib.Input(shape=(32,))
@@ -341,6 +344,7 @@ class TopologyConstructionTest(test.TestCase):
     self.assertListEqual(model.trainable_weights, [])
     self.assertListEqual(model.non_trainable_weights, weights)
 
+  @test_util.run_deprecated_v1
   def test_layer_call_arguments(self):
     # Test the ability to pass and serialize arguments to `call`.
     inp = keras.layers.Input(shape=(2,))
@@ -491,6 +495,7 @@ class TopologyConstructionTest(test.TestCase):
       fn_outputs = fn([input_a_np, input_b_np])
       self.assertListEqual([x.shape for x in fn_outputs], [(10, 64), (10, 5)])
 
+  @test_util.run_deprecated_v1
   def test_recursion(self):
     with self.cached_session():
       a = keras.layers.Input(shape=(32,), name='input_a')
@@ -675,6 +680,7 @@ class TopologyConstructionTest(test.TestCase):
     with self.assertRaises(Exception):
       keras.models.Model([j, k], [m, n, 0])
 
+  @test_util.run_deprecated_v1
   def test_raw_tf_compatibility(self):
     # test calling layers/models on TF tensors
     a = keras.layers.Input(shape=(32,), name='input_a')
@@ -719,6 +725,7 @@ class TopologyConstructionTest(test.TestCase):
     model = keras.models.Model(a, b)
     self.assertEqual(model.output_mask.get_shape().as_list(), [None, 10])
 
+  @test_util.run_deprecated_v1
   def testMaskingSingleInput(self):
 
     class MaskedLayer(keras.layers.Layer):
@@ -756,6 +763,7 @@ class TopologyConstructionTest(test.TestCase):
       y_2 = network(x_2)
       self.assertEqual(y_2.get_shape().as_list(), [None, 32])
 
+  @test_util.run_deprecated_v1
   def test_activity_regularization_with_model_composition(self):
 
     def reg(x):
@@ -825,6 +833,7 @@ class TopologyConstructionTest(test.TestCase):
       output_val_2 = m2.predict(x_val)
       self.assertAllClose(output_val, output_val_2, atol=1e-6)
 
+  @test_util.run_deprecated_v1
   def test_explicit_training_argument(self):
     with self.cached_session():
       a = keras.layers.Input(shape=(2,))
@@ -1145,6 +1154,7 @@ class DefaultShapeInferenceBehaviorTest(test.TestCase):
 
 class GraphUtilsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testGetReachableFromInputs(self):
 
     with self.cached_session():
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index e79e5842a1..8020326377 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -274,6 +274,7 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
 
       model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
 
+  @tf_test_util.run_deprecated_v1
   def test_dataset_input_shape_validation(self):
     with self.cached_session():
       model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 97dfe6d900..4674bce244 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -449,6 +449,7 @@ class TrainingTest(test.TestCase):
           optimizer=keras.optimizers.Adam(lr=0.0001),
           metrics=['accuracy'])
 
+  @tf_test_util.run_deprecated_v1
   def test_that_trainable_disables_updates(self):
     val_a = np.random.random((10, 4))
     val_out = np.random.random((10, 4))
@@ -1146,6 +1147,7 @@ class LossMaskingTest(test.TestCase):
 
 class TestDynamicTrainability(test.TestCase):
 
+  @tf_test_util.run_deprecated_v1
   def test_trainable_warning(self):
     with self.cached_session():
       x = np.random.random((5, 3))
@@ -1159,6 +1161,7 @@ class TestDynamicTrainability(test.TestCase):
       model.train_on_batch(x, y)
       self.assertRaises(Warning)
 
+  @tf_test_util.run_deprecated_v1
   def test_trainable_argument(self):
     with self.cached_session():
       x = np.random.random((5, 3))
@@ -1289,6 +1292,7 @@ class TestDynamicTrainability(test.TestCase):
 
 class TestTrainingWithDataTensors(test.TestCase):
 
+  @tf_test_util.run_deprecated_v1
   def test_training_and_eval_methods_on_symbolic_tensors_single_io(self):
     with self.cached_session():
       x = keras.layers.Input(shape=(3,), name='input')
@@ -1329,6 +1333,7 @@ class TestTrainingWithDataTensors(test.TestCase):
                 epochs=1, steps_per_epoch=2, verbose=0,
                 validation_data=(inputs, targets), validation_steps=2)
 
+  @tf_test_util.run_deprecated_v1
   def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self):
     with self.cached_session():
       a = keras.layers.Input(shape=(3,), name='input_a')
@@ -1424,6 +1429,7 @@ class TestTrainingWithDataTensors(test.TestCase):
       model.predict([input_a_tf, input_b_tf], steps=2)
       model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
 
+  @tf_test_util.run_deprecated_v1
   def test_model_with_input_feed_tensor(self):
     """We test building a model with a TF variable as input.
 
@@ -1602,6 +1608,7 @@ class TestTrainingWithDataTensors(test.TestCase):
       # evaluate
       _ = model.evaluate(input_a_np, [output_a_np])
 
+  @tf_test_util.run_deprecated_v1
   def test_model_with_external_loss(self):
     with self.cached_session():
       # None loss, only regularization loss.
@@ -1797,6 +1804,7 @@ class TestTrainingWithDataTensors(test.TestCase):
       model.train_on_batch(input_val, None,
                            sample_weight={'dense_a': np.random.random((10,))})
 
+  @tf_test_util.run_deprecated_v1
   def test_model_custom_target_tensors(self):
     with self.cached_session():
       a = keras.Input(shape=(3,), name='input_a')
@@ -2018,6 +2026,7 @@ class TestTrainingWithMetrics(test.TestCase):
       scores = model.train_on_batch(x, y, sample_weight=w)
       self.assertArrayNear(scores, [0.3328, 0.8], 0.001)
 
+  @tf_test_util.run_deprecated_v1
   def test_add_metric_with_tensor_on_model_in_graph_mode(self):
     with self.cached_session():
       x = keras.layers.Input(shape=(1,))
@@ -2180,6 +2189,7 @@ class TestTrainingWithMetrics(test.TestCase):
       self.assertEqual(history.history['metric_1'][-1], 5)
       self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
 
+  @tf_test_util.run_deprecated_v1
   def test_model_metrics_list(self):
     with self.cached_session():
       x = keras.layers.Input(shape=(1,))
diff --git a/tensorflow/python/keras/initializers_test.py b/tensorflow/python/keras/initializers_test.py
index 2b758a98f3..4f91bea1e3 100644
--- a/tensorflow/python/keras/initializers_test.py
+++ b/tensorflow/python/keras/initializers_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.ops import init_ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -38,6 +39,7 @@ class KerasInitializersTest(test.TestCase):
     output_2 = keras.backend.get_value(variable)
     self.assertAllClose(output, output_2, atol=1e-4)
 
+  @test_util.run_deprecated_v1
   def test_uniform(self):
     tensor_shape = (9, 6, 7)
     with self.cached_session():
@@ -47,6 +49,7 @@ class KerasInitializersTest(test.TestCase):
                    tensor_shape,
                    target_mean=0., target_max=1, target_min=-1)
 
+  @test_util.run_deprecated_v1
   def test_normal(self):
     tensor_shape = (8, 12, 99)
     with self.cached_session():
@@ -54,6 +57,7 @@ class KerasInitializersTest(test.TestCase):
                    tensor_shape,
                    target_mean=0., target_std=1)
 
+  @test_util.run_deprecated_v1
   def test_truncated_normal(self):
     tensor_shape = (12, 99, 7)
     with self.cached_session():
@@ -69,6 +73,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.Constant(2), tensor_shape,
                    target_mean=2, target_max=2, target_min=2)
 
+  @test_util.run_deprecated_v1
   def test_lecun_uniform(self):
     tensor_shape = (5, 6, 4, 2)
     with self.cached_session():
@@ -77,6 +82,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.lecun_uniform(seed=123), tensor_shape,
                    target_mean=0., target_std=std)
 
+  @test_util.run_deprecated_v1
   def test_glorot_uniform(self):
     tensor_shape = (5, 6, 4, 2)
     with self.cached_session():
@@ -85,6 +91,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.glorot_uniform(seed=123), tensor_shape,
                    target_mean=0., target_std=std)
 
+  @test_util.run_deprecated_v1
   def test_he_uniform(self):
     tensor_shape = (5, 6, 4, 2)
     with self.cached_session():
@@ -93,6 +100,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.he_uniform(seed=123), tensor_shape,
                    target_mean=0., target_std=std)
 
+  @test_util.run_deprecated_v1
   def test_lecun_normal(self):
     tensor_shape = (5, 6, 4, 2)
     with self.cached_session():
@@ -101,6 +109,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.lecun_normal(seed=123), tensor_shape,
                    target_mean=0., target_std=std)
 
+  @test_util.run_deprecated_v1
   def test_glorot_normal(self):
     tensor_shape = (5, 6, 4, 2)
     with self.cached_session():
@@ -109,6 +118,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.glorot_normal(seed=123), tensor_shape,
                    target_mean=0., target_std=std)
 
+  @test_util.run_deprecated_v1
   def test_he_normal(self):
     tensor_shape = (5, 6, 4, 2)
     with self.cached_session():
@@ -117,6 +127,7 @@ class KerasInitializersTest(test.TestCase):
       self._runner(keras.initializers.he_normal(seed=123), tensor_shape,
                    target_mean=0., target_std=std)
 
+  @test_util.run_deprecated_v1
   def test_orthogonal(self):
     tensor_shape = (20, 20)
     with self.cached_session():
diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index 3c0f73b1c3..f1a0932613 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.layers import core as tf_core_layers
 from tensorflow.python.ops import nn
@@ -34,6 +35,7 @@ class KerasIntegrationTest(test.TestCase):
   def test_version(self):
     self.assertTrue(keras.__version__.endswith('-tf'))
 
+  @test_util.run_deprecated_v1
   def test_vector_classification_sequential(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -59,6 +61,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_deprecated_v1
   def test_vector_classification_functional(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -83,6 +86,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_deprecated_v1
   def test_temporal_classification_sequential(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -105,6 +109,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_deprecated_v1
   def test_temporal_classification_sequential_tf_rnn(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -163,6 +168,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_deprecated_v1
   def test_video_classification_functional(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -191,6 +197,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_deprecated_v1
   def test_vector_classification_shared_sequential(self):
     # Test that Sequential models that feature internal updates
     # and internal losses can be shared.
@@ -225,6 +232,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_deprecated_v1
   def test_vector_classification_shared_model(self):
     # Test that functional models that feature internal updates
     # and internal losses can be shared.
diff --git a/tensorflow/python/keras/layers/local_test.py b/tensorflow/python/keras/layers/local_test.py
index 1c20d43706..a23c5c38fe 100644
--- a/tensorflow/python/keras/layers/local_test.py
+++ b/tensorflow/python/keras/layers/local_test.py
@@ -32,6 +32,7 @@ class LocallyConnected1DLayersTest(test.TestCase):
   # fails inside a graph function in an eager context (fails with error
   # "Incompatible shapes between op input and calculated input gradient").
 
+  @tf_test_util.run_deprecated_v1
   def test_locallyconnected_1d(self):
     with self.cached_session():
       num_samples = 2
@@ -120,6 +121,7 @@ class LocallyConnected2DLayersTest(test.TestCase):
   # fails inside a graph function in an eager context (fails with error
   # "Incompatible shapes between op input and calculated input gradient").
 
+  @tf_test_util.run_deprecated_v1
   def test_locallyconnected_2d(self):
     with self.cached_session():
       num_samples = 8
@@ -155,6 +157,7 @@ class LocallyConnected2DLayersTest(test.TestCase):
                   kwargs=kwargs,
                   input_shape=(num_samples, num_row, num_col, stack_size))
 
+  @tf_test_util.run_deprecated_v1
   def test_locallyconnected_2d_channels_first(self):
     with self.cached_session():
       num_samples = 8
@@ -232,6 +235,7 @@ class LocallyConnected2DLayersTest(test.TestCase):
 
 class LocallyConnectedImplementationModeTest(test.TestCase):
 
+  @tf_test_util.run_deprecated_v1
   def test_locallyconnected_implementation(self):
     with self.cached_session():
       num_samples = 4
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index 9db697871f..732bbcfa18 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -114,6 +114,7 @@ class LSTMLayerTest(test.TestCase):
     self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
     self.assertEqual(layer.cell.bias.constraint, b_constraint)
 
+  @tf_test_util.run_deprecated_v1
   def test_with_masking_layer_LSTM(self):
     layer_class = keras.layers.LSTM
     inputs = np.random.random((2, 3, 4))
@@ -126,6 +127,7 @@ class LSTMLayerTest(test.TestCase):
                   optimizer=RMSPropOptimizer(0.01))
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
+  @tf_test_util.run_deprecated_v1
   def test_masking_with_stacking_LSTM(self):
     inputs = np.random.random((2, 3, 4))
     targets = np.abs(np.random.random((2, 3, 5)))
@@ -311,6 +313,7 @@ class LSTMLayerTest(test.TestCase):
 
 class LSTMLayerGraphOnlyTest(test.TestCase):
 
+  @tf_test_util.run_deprecated_v1
   def test_statefulness_LSTM(self):
     num_samples = 2
     timesteps = 3
@@ -374,6 +377,7 @@ class LSTMLayerGraphOnlyTest(test.TestCase):
 
       self.assertAllClose(out7, out6, atol=1e-5)
 
+  @tf_test_util.run_deprecated_v1
   def test_regularizers_LSTM(self):
     embedding_dim = 4
     layer_class = keras.layers.LSTM
diff --git a/tensorflow/python/keras/layers/merge_test.py b/tensorflow/python/keras/layers/merge_test.py
index 698c5662b6..fcb161ae20 100644
--- a/tensorflow/python/keras/layers/merge_test.py
+++ b/tensorflow/python/keras/layers/merge_test.py
@@ -207,6 +207,7 @@ class MergeLayersGraphOnlyTest(test.TestCase):
       mask = layer.output_mask
       self.assertListEqual(mask.get_shape().as_list(), [None, 4])
 
+  @tf_test_util.run_deprecated_v1
   def test_merge_add_dynamic_shape(self):
     with self.cached_session():
       i1 = array_ops.placeholder(shape=(4, None), dtype='float32')
diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index 2f7f042b7f..9138c0a08a 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -301,6 +301,7 @@ class NormalizationLayersGraphModeOnlyTest(test.TestCase):
       x2 = model.predict(val_a)
       self.assertAllClose(x1, x2, atol=1e-7)
 
+  @tf_test_util.run_deprecated_v1
   def test_batchnorm_trainable(self):
     """Tests that batchnorm layer is trainable when learning phase is enabled.
 
diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py
index 93456b5e3a..b49b159b71 100644
--- a/tensorflow/python/keras/layers/simplernn_test.py
+++ b/tensorflow/python/keras/layers/simplernn_test.py
@@ -98,6 +98,7 @@ class SimpleRNNLayerTest(test.TestCase):
     self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
     self.assertEqual(layer.cell.bias.constraint, b_constraint)
 
+  @tf_test_util.run_deprecated_v1
   def test_with_masking_layer_SimpleRNN(self):
     layer_class = keras.layers.SimpleRNN
     inputs = np.random.random((2, 3, 4))
@@ -120,6 +121,7 @@ class SimpleRNNLayerTest(test.TestCase):
 
 class SimpleRNNLayerGraphOnlyTest(test.TestCase):
 
+  @tf_test_util.run_deprecated_v1
   def test_statefulness_SimpleRNN(self):
     num_samples = 2
     timesteps = 3
@@ -183,6 +185,7 @@ class SimpleRNNLayerGraphOnlyTest(test.TestCase):
 
       np.testing.assert_allclose(out7, out6, atol=1e-5)
 
+  @tf_test_util.run_deprecated_v1
   def test_regularizers_SimpleRNN(self):
     embedding_dim = 4
     layer_class = keras.layers.SimpleRNN
diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index b08ff3cafc..e26e47000d 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -69,6 +69,7 @@ class RNNTest(test.TestCase):
   def tearDown(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def test_unifiedRNN(self):
     input_shape = 10
     rnn_state_size = 8
@@ -113,6 +114,7 @@ class RNNTest(test.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
+  @test_util.run_deprecated_v1
   def test_unifiedRNN_with_cond(self):
     # This test is to demonstrate the graph rewrite of grappler plugin under
     # the condition that the function returns different number of internal
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index bbafa96aab..427314faf1 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -201,6 +201,7 @@ class TimeDistributedTest(test.TestCase):
     assert len(layer.updates) == 2
     assert len(layer.trainable_weights) == 2
 
+  @tf_test_util.run_deprecated_v1
   def test_TimeDistributed_with_masked_embedding_and_unspecified_shape(self):
     with self.cached_session():
       # test with unspecified shape and Embeddings with mask_zero
@@ -233,6 +234,7 @@ class TimeDistributedTest(test.TestCase):
         self.assertAllEqual(mask_outputs_val[i], ref_mask_val[i])
       self.assertIs(mask_outputs[-1], None)  # final layer
 
+  @tf_test_util.run_deprecated_v1
   def test_TimeDistributed_with_masking_layer(self):
     with self.cached_session():
       # test with Masking layer
@@ -375,6 +377,7 @@ class BidirectionalTest(test.TestCase):
       model.compile(loss='mse', optimizer='sgd')
       model.fit(x, y, epochs=1, batch_size=1)
 
+  @tf_test_util.run_deprecated_v1
   def test_Bidirectional_merged_value(self):
     rnn = keras.layers.LSTM
     samples = 2
@@ -505,6 +508,7 @@ class BidirectionalTest(test.TestCase):
       layer.trainable = True
       assert len(layer.trainable_weights) == 6
 
+  @tf_test_util.run_deprecated_v1
   def test_Bidirectional_updates(self):
     with self.cached_session():
       x = keras.layers.Input(shape=(3, 2))
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index ccfff57a3b..ca7dded60d 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -917,6 +917,7 @@ class ModelSubclassingTest(test.TestCase):
 
 class GraphSpecificModelSubclassingTests(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_single_io_workflow_with_tensors(self):
     num_classes = 2
     num_samples = 10
@@ -934,6 +935,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model.fit(x, y, epochs=2, steps_per_epoch=10, verbose=0)
       _ = model.evaluate(steps=10, verbose=0)
 
+  @test_util.run_deprecated_v1
   def test_multi_io_workflow_with_tensors(self):
     num_classes = (2, 3)
     num_samples = 10
@@ -953,6 +955,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model.fit([x1, x2], [y1, y2], epochs=2, steps_per_epoch=10, verbose=0)
       _ = model.evaluate(steps=10, verbose=0)
 
+  @test_util.run_deprecated_v1
   def test_updates_and_losses_for_nested_models_in_subclassed_model(self):
 
     # Case 1: deferred-build sequential nested in subclass.
@@ -1020,6 +1023,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       self.assertEqual(len(model.get_updates_for(x)), 2)
       self.assertEqual(len(model.get_losses_for(x)), 1)
 
+  @test_util.run_deprecated_v1
   def test_multi_io_workflow_with_numpy_arrays_and_custom_placeholders(self):
     num_classes = (2, 3)
     num_samples = 1000
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index c68c80f0cc..8af3cc05f5 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -69,6 +69,7 @@ def sequential_model(add_input_layer, include_input_shape=True):
 
 class TestModelCloning(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_clone_sequential_model(self):
     with self.cached_session():
       val_a = np.random.random((10, 4))
@@ -101,7 +102,10 @@ class TestModelCloning(test.TestCase):
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch(None, val_out)
 
+  @test_util.run_deprecated_v1
   def test_clone_sequential_model_input_layer(self):
+
+    @test_util.run_deprecated_v1
     def test_input_layer(include_inputs):
       with self.cached_session():
         val_a = np.random.random((10, 4))
@@ -138,6 +142,7 @@ class TestModelCloning(test.TestCase):
     test_input_layer(True)
     test_input_layer(False)
 
+  @test_util.run_deprecated_v1
   def test_clone_functional_model(self):
     with self.cached_session():
       val_a = np.random.random((10, 4))
@@ -397,6 +402,7 @@ class TestCloneAndBuildModel(test.TestCase):
       new_model.train_on_batch(inp, out)
       new_model.evaluate(inp, out)
 
+  @test_util.run_deprecated_v1
   def test_clone_and_build_compiled_sequential_model(self):
     with self.cached_session():
       model = keras.models.Sequential()
@@ -409,6 +415,7 @@ class TestCloneAndBuildModel(test.TestCase):
 
     self._clone_and_build_test_helper(model)
 
+  @test_util.run_deprecated_v1
   def test_clone_and_build_functional_model(self):
     with self.cached_session():
       input_a = keras.Input(shape=(4,))
@@ -425,6 +432,7 @@ class TestCloneAndBuildModel(test.TestCase):
 
     self._clone_and_build_test_helper(model)
 
+  @test_util.run_deprecated_v1
   def test_clone_and_build_subclassed_model(self):
     class SubclassedModel(keras.Model):
 
@@ -473,9 +481,11 @@ class TestCloneAndBuildModel(test.TestCase):
   def test_replace_tf_optimizer_iterations_variable(self):
     self.assert_optimizer_iterations_increases(adam.AdamOptimizer(0.01))
 
+  @test_util.run_deprecated_v1
   def test_replace_keras_optimizer_iterations_variable(self):
     self.assert_optimizer_iterations_increases('adam')
 
+  @test_util.run_deprecated_v1
   def test_clone_and_build_sequential_model_without_inputs_defined(self):
     with self.cached_session():
       model = sequential_model(False, False)
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index ef95d27abf..6409f3ead4 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -147,6 +147,7 @@ class AdadeltaOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_resource=True, use_callable_params=True)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index 5ddeb1ad80..8004907bf5 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -160,6 +160,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -179,6 +180,7 @@ class AdagradOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             [[0, 1], [3, 4]], var0.eval(), atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -211,6 +213,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseBasic(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -255,6 +258,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndices(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -287,6 +291,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllClose(aggregated_update_var.eval(),
                               repeated_index_update_var.eval())
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndicesByEmbeddingLookUp(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -311,6 +316,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(
               var_repeated.eval(), var_aggregated.eval())
 
+  @test_util.run_deprecated_v1
   def testSparseStability(self):
     for dtype in [dtypes.half]:
       with self.cached_session():
@@ -343,6 +349,7 @@ class AdagradOptimizerTest(test.TestCase):
                   -0.01029443
               ]]), var0.eval())
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index e2bc6a39f9..b1b17f8f08 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -62,6 +62,7 @@ def get_beta_accumulators(opt, dtype):
 
 class AdamOptimizerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -106,6 +107,7 @@ class AdamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseDevicePlacement(self):
     for index_dtype in [dtypes.int32, dtypes.int64]:
       with self.cached_session(force_gpu=test.is_gpu_available()):
@@ -119,6 +121,7 @@ class AdamOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
         minimize_op.run()
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndices(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -208,6 +211,7 @@ class AdamOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_callable_params=True)
 
+  @test_util.run_deprecated_v1
   def testBasicWithLearningRateDecay(self):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
       with self.session(graph=ops.Graph()):
@@ -254,6 +258,7 @@ class AdamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -292,6 +297,7 @@ class AdamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index aa215b0faf..7521a6196e 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -124,9 +124,11 @@ class AdamaxOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, var0.eval())
           self.assertAllCloseAccordingToType(var1_np, var1.eval())
 
+  @test_util.run_deprecated_v1
   def testResourceSparse(self):
     self.doTestSparse(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testSparseDevicePlacement(self):
     for index_dtype in [dtypes.int32, dtypes.int64]:
       with self.cached_session(force_gpu=test.is_gpu_available()):
@@ -140,6 +142,7 @@ class AdamaxOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
         minimize_op.run()
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndices(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -272,6 +275,7 @@ class AdamaxOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1),
                                              rtol=1e-2)
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -308,6 +312,7 @@ class AdamaxOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, var0.eval())
           self.assertAllCloseAccordingToType(var1_np, var1.eval())
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl_test.py b/tensorflow/python/keras/optimizer_v2/ftrl_test.py
index ca8c33dfa6..bec400e8cb 100644
--- a/tensorflow/python/keras/optimizer_v2/ftrl_test.py
+++ b/tensorflow/python/keras/optimizer_v2/ftrl_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras.optimizer_v2 import ftrl
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
@@ -68,12 +69,15 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.28432083, -0.56694895]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithoutRegularization(self):
     self.doTestFtrlwithoutRegularization(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceFtrlWithoutRegularization(self):
     self.doTestFtrlwithoutRegularization(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testFtrlwithoutRegularization2(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session() as sess:
@@ -103,6 +107,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.28232238, -0.56096673]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -121,6 +126,7 @@ class FtrlOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session() as sess:
@@ -150,6 +156,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.93460727, -1.86147261]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1_L2(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session() as sess:
@@ -180,6 +187,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.02406147, -0.04830509]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1_L2_L2Shrinkage(self):
     """Test the new FTRL op with support for l2 shrinkage.
 
@@ -217,6 +225,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.14378493, -0.13229476]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1_L2_L2ShrinkageSparse(self):
     """Tests the new FTRL op with support for l2 shrinkage on sparse grads."""
     for dtype in [dtypes.half, dtypes.float32]:
@@ -251,6 +260,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val)
         self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self):
     """Verifies that l2 shrinkage in FTRL does not change lr schedule."""
     for dtype in [dtypes.half, dtypes.float32]:
@@ -335,6 +345,7 @@ class FtrlOptimizerTest(test.TestCase):
   # with Adagrad.
   # So, basing on these two properties, we test if our implementation of
   # FTRL-Proximal performs same updates as Adagrad or GradientDescent.
+  @test_util.run_deprecated_v1
   def testEquivAdagradwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
@@ -355,6 +366,7 @@ class FtrlOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(val0, val2)
       self.assertAllCloseAccordingToType(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivSparseAdagradwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
@@ -378,6 +390,7 @@ class FtrlOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(val0, val2)
       self.assertAllCloseAccordingToType(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivSparseGradientDescentwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
@@ -401,6 +414,7 @@ class FtrlOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(val0, val2)
       self.assertAllCloseAccordingToType(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivGradientDescentwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index 348d2728c8..dae2a47735 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -134,6 +134,7 @@ class GradientDescentOptimizerTest(test.TestCase):
                                            self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - 1.0], self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -173,6 +174,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
                                            self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testGradWrtRef(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -206,6 +208,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]],
                                            self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseBasicWithLearningRateDecay(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -331,6 +334,7 @@ class MomentumOptimizerTest(test.TestCase):
                 3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
             ]), self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testNesterovMomentum(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -358,6 +362,7 @@ class MomentumOptimizerTest(test.TestCase):
           self.assertAllClose(var0_np, self.evaluate(var0))
           self.assertAllClose(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseNesterovMomentum(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -399,6 +404,7 @@ class MomentumOptimizerTest(test.TestCase):
           self.assertAllClose(var1_np, self.evaluate(var1))
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       # This test invokes the ResourceSparseApplyMomentum operation, which
@@ -452,6 +458,7 @@ class MomentumOptimizerTest(test.TestCase):
     self.evaluate(sgd_op)
     self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0))
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRateAndMomentum(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -510,6 +517,7 @@ class MomentumOptimizerTest(test.TestCase):
                 3.98 - ((0.9 * 0.01 + 0.01) * 2.0)
             ]), self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -585,6 +593,7 @@ class MomentumOptimizerTest(test.TestCase):
             ]),
             self.evaluate(var1)[2])
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
index b7132bbea7..d991e3117c 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras.optimizer_v2 import nadam
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -111,9 +112,11 @@ class NadamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, var0.eval())
           self.assertAllCloseAccordingToType(var1_np, var1.eval())
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     self.doTestSparse(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceSparse(self):
     self.doTestSparse(use_resource=True)
 
@@ -158,9 +161,11 @@ class NadamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, var0.eval())
           self.assertAllCloseAccordingToType(var1_np, var1.eval())
 
+  @test_util.run_deprecated_v1
   def testResourceBasic(self):
     self.doTestBasic(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testBasicWithLearningRateDecay(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index a320cc0c49..a8658a8550 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -86,6 +86,7 @@ class RMSpropOptimizerTest(test.TestCase):
       var_t[gindex] = var[gindex] - mom_t[gindex]
     return var_t, mg_t, rms_t, mom_t
 
+  @test_util.run_deprecated_v1
   def testDense(self):
     for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
       with test_util.use_gpu():
@@ -158,6 +159,7 @@ class RMSpropOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testDenseWithLearningRateDecay(self):
     var0_np = np.array([1.0, 2.0])
     grads0_np = np.array([0.1, 0.2])
@@ -225,6 +227,7 @@ class RMSpropOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
       self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -249,6 +252,7 @@ class RMSpropOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariableCentered(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -273,6 +277,7 @@ class RMSpropOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     for (dtype, learning_rate, rho, momentum, epsilon, centered) in _TESTPARAMS:
       with test_util.use_gpu():
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index 46bb0274c6..d3cacb702c 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -91,22 +91,26 @@ def _test_optimizer(optimizer, target=0.75):
 
 class KerasOptimizersTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_sgd(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.SGD(lr=0.01,
                                            momentum=0.9,
                                            nesterov=True))
 
+  @test_util.run_deprecated_v1
   def test_rmsprop(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.RMSprop())
       _test_optimizer(keras.optimizers.RMSprop(decay=1e-3))
 
+  @test_util.run_deprecated_v1
   def test_adagrad(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adagrad())
       _test_optimizer(keras.optimizers.Adagrad(decay=1e-3))
 
+  @test_util.run_deprecated_v1
   def test_adadelta(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adadelta(), target=0.6)
@@ -115,27 +119,32 @@ class KerasOptimizersTest(test.TestCase):
       # the accuracy.
       _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
 
+  @test_util.run_deprecated_v1
   def test_adam(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adam())
       _test_optimizer(keras.optimizers.Adam(decay=1e-3))
       _test_optimizer(keras.optimizers.Adam(amsgrad=True))
 
+  @test_util.run_deprecated_v1
   def test_adamax(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adamax())
       _test_optimizer(keras.optimizers.Adamax(decay=1e-3))
 
+  @test_util.run_deprecated_v1
   def test_nadam(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Nadam())
 
+  @test_util.run_deprecated_v1
   def test_clipnorm(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.SGD(lr=0.01,
                                            momentum=0.9,
                                            clipnorm=0.5))
 
+  @test_util.run_deprecated_v1
   def test_clipvalue(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.SGD(lr=0.01,
diff --git a/tensorflow/python/keras/regularizers_test.py b/tensorflow/python/keras/regularizers_test.py
index bba4ebb287..3d6b259d87 100644
--- a/tensorflow/python/keras/regularizers_test.py
+++ b/tensorflow/python/keras/regularizers_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python import keras
 from tensorflow.python.keras import testing_utils
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
@@ -61,6 +62,7 @@ class KerasRegularizersTest(test.TestCase):
         model.fit(x_train, y_train, batch_size=10,
                   epochs=1, verbose=0)
 
+  @test_util.run_deprecated_v1
   def test_activity_regularization(self):
     with self.cached_session():
       (x_train, y_train), _ = get_data()
diff --git a/tensorflow/python/kernel_tests/accumulate_n_test.py b/tensorflow/python/kernel_tests/accumulate_n_test.py
index c7f11f854d..5eece9c941 100644
--- a/tensorflow/python/kernel_tests/accumulate_n_test.py
+++ b/tensorflow/python/kernel_tests/accumulate_n_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.platform import googletest
 class AccumulateNV2Test(test_util.TensorFlowTestCase):
   """Tests of the new, differentiable version of accumulate_n."""
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     np.random.seed(12345)
     x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
@@ -41,6 +42,7 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase):
       self.assertAllClose(x[0] * 5,
                           math_ops.accumulate_n([tf_x[0]] * 5).eval())
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     np.random.seed(54321)
     x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)]
@@ -50,12 +52,14 @@ class AccumulateNV2Test(test_util.TensorFlowTestCase):
       self.assertAllEqual(x[0] * 6,
                           math_ops.accumulate_n([tf_x[0]] * 6).eval())
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     with self.session(use_gpu=True):
       x0 = array_ops.placeholder(dtype=dtypes_lib.int32, shape=[None])
       acc = math_ops.accumulate_n([x0, x0], shape=[None])
       self.assertAllEqual([2, 4], acc.eval(feed_dict={x0: [1, 2]}))
 
+  @test_util.run_deprecated_v1
   def testGrad(self):
     np.random.seed(42)
     for num_inputs in range(1, 10):
diff --git a/tensorflow/python/kernel_tests/ackermann_test.py b/tensorflow/python/kernel_tests/ackermann_test.py
index d267e49752..6c20b19be9 100644
--- a/tensorflow/python/kernel_tests/ackermann_test.py
+++ b/tensorflow/python/kernel_tests/ackermann_test.py
@@ -20,12 +20,14 @@ from __future__ import print_function
 import os
 
 from tensorflow.python.framework import load_library
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import resource_loader
 from tensorflow.python.platform import test
 
 
 class AckermannTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     library_filename = os.path.join(resource_loader.get_data_files_path(),
                                     'ackermann_op.so')
diff --git a/tensorflow/python/kernel_tests/aggregate_ops_test.py b/tensorflow/python/kernel_tests/aggregate_ops_test.py
index 874d616658..d9787cc3bf 100644
--- a/tensorflow/python/kernel_tests/aggregate_ops_test.py
+++ b/tensorflow/python/kernel_tests/aggregate_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.core.framework import tensor_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import logging_ops
 from tensorflow.python.ops import math_ops
@@ -67,6 +68,7 @@ class AddNTest(test.TestCase):
           tol = 5e-3 if dtype == dtypes.float16 else 5e-7
           self.assertAllClose(expected, actual, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testUnknownShapes(self):
     np.random.seed(12345)
     with self.session(use_gpu=True) as sess:
@@ -80,6 +82,7 @@ class AddNTest(test.TestCase):
           tol = 5e-3 if dtype == dtypes.float16 else 5e-7
           self.assertAllClose(expected, actual, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testVariant(self):
 
     def create_constant_variant(value):
diff --git a/tensorflow/python/kernel_tests/argmax_op_test.py b/tensorflow/python/kernel_tests/argmax_op_test.py
index d34a1dc9b2..06ec0948c2 100644
--- a/tensorflow/python/kernel_tests/argmax_op_test.py
+++ b/tensorflow/python/kernel_tests/argmax_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -110,12 +111,14 @@ class ArgMaxTest(test.TestCase):
             r"Reduction axis 0 is empty in shape \[0\]"):
           op([], 0).eval()
 
+  @test_util.run_deprecated_v1
   def testDefaultAxis(self):
     with self.cached_session():
       for op in math_ops.argmin, math_ops.argmax:
         ans = op([1]).eval()
         self.assertAllEqual(ans, 0)
 
+  @test_util.run_deprecated_v1
   def testOutputEmpty(self):
     with self.cached_session():
       for op in math_ops.argmin, math_ops.argmax:
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index afc158f697..f4c442b7b1 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -143,36 +143,43 @@ class BooleanMaskTest(test_util.TensorFlowTestCase):
 
       self.assertAllClose(masked_arr, masked_tensor.eval())
 
+  @test_util.run_deprecated_v1
   def testMaskDim1ArrDim2Axis1(self):
     ndims_mask = 1
     for arr_shape in [(1, 1), (2, 2), (2, 5)]:
       self.CheckVersusNumpy(ndims_mask, arr_shape, axis=1)
 
+  @test_util.run_deprecated_v1
   def testMaskDim2ArrDim2Axis1(self):
     ndims_mask = 2
     for arr_shape in [(1, 1), (2, 2), (2, 5)]:
       self.CheckVersusNumpy(ndims_mask, arr_shape, axis=1)
 
+  @test_util.run_deprecated_v1
   def testMaskDim1ArrDim1(self):
     ndims_mask = 1
     for arr_shape in [(1,), (2,), (3,), (10,)]:
       self.CheckVersusNumpy(ndims_mask, arr_shape)
 
+  @test_util.run_deprecated_v1
   def testMaskDim1ArrDim2(self):
     ndims_mask = 1
     for arr_shape in [(1, 1), (2, 2), (2, 5)]:
       self.CheckVersusNumpy(ndims_mask, arr_shape)
 
+  @test_util.run_deprecated_v1
   def testMaskDim2ArrDim2(self):
     ndims_mask = 2
     for arr_shape in [(1, 1), (2, 2), (2, 5)]:
       self.CheckVersusNumpy(ndims_mask, arr_shape)
 
+  @test_util.run_deprecated_v1
   def testMaskDim2ArrDim3(self):
     ndims_mask = 2
     for arr_shape in [(1, 1, 1), (1, 2, 2), (2, 2, 1)]:
       self.CheckVersusNumpy(ndims_mask, arr_shape)
 
+  @test_util.run_deprecated_v1
   def testEmptyInput2D(self):
     mask = np.array([True, False])
     arr = np.array([[], []]).astype(np.float32)
@@ -191,6 +198,7 @@ class BooleanMaskTest(test_util.TensorFlowTestCase):
     with self.cached_session():
       self.assertAllClose(numpy_result, tf_result.eval())
 
+  @test_util.run_deprecated_v1
   def testEmptyOutput(self):
     make_mask = lambda shape: np.zeros(shape, dtype=bool)
     for ndims_mask in range(1, 4):
@@ -199,6 +207,7 @@ class BooleanMaskTest(test_util.TensorFlowTestCase):
           arr_shape = np.random.randint(1, 5, size=ndims_arr)
           self.CheckVersusNumpy(ndims_mask, arr_shape, make_mask=make_mask)
 
+  @test_util.run_deprecated_v1
   def testWorksWithDimensionsEqualToNoneDuringGraphBuild(self):
     # The rank of the mask tensor must be specified. This is explained
     # in the docstring as well.
@@ -217,6 +226,7 @@ class BooleanMaskTest(test_util.TensorFlowTestCase):
           })
       np.testing.assert_allclose(masked_tensor, arr[mask])
 
+  @test_util.run_deprecated_v1
   def testMaskDimensionsSetToNoneRaises(self):
     # The rank of the mask tensor must be specified. This is explained
     # in the docstring as well.
@@ -283,6 +293,7 @@ class OperatorShapeTest(test_util.TensorFlowTestCase):
 
 class ReverseV2Test(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testReverse0DimAuto(self):
     x_np = 4
     for use_gpu in [False, True]:
@@ -327,6 +338,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
 
   # This test covers the axis validation in the shape function
   # (no eval())
+  @test_util.run_deprecated_v1
   def testInvalidAxis(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
     with self.assertRaisesRegexp(ValueError,
@@ -345,6 +357,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
   #
   # Note: this test passes placeholder as constant axis is validated
   # in shape function (see testInvalidAxis)
+  @test_util.run_deprecated_v1
   def testInvalid(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
     axis = array_ops.placeholder(dtypes.int32)
@@ -359,6 +372,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
                                    "axis 0 specified more than once"):
         array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [0, -2]})
 
+  @test_util.run_deprecated_v1
   def testReverse1DimAuto(self):
     for dtype in [
         np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64, np.bool,
@@ -367,6 +381,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
     ]:
       self._reverse1DimAuto(dtype)
 
+  @test_util.run_deprecated_v1
   def testReverse2DimAuto(self):
     for dtype in [
         np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64, np.bool,
@@ -375,6 +390,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
     ]:
       self._reverse2DimAuto(dtype)
 
+  @test_util.run_deprecated_v1
   def testUnknownDims(self):
     reverse_v2 = array_ops.reverse_v2
     data_t = array_ops.placeholder(dtypes.float32)
@@ -392,6 +408,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
     reverse_2d_t = reverse_v2(data_2d_t, axis_2d_t)
     self.assertEqual(2, reverse_2d_t.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testReverseRowsOf3Channels(self):
     """Tests optimized code for reversing rows with last dim size = 3."""
     with self.session(use_gpu=True):
@@ -405,6 +422,7 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
             np_answer = x_np[:, ::-1, :]
             self.assertAllEqual(x_tf, np_answer)
 
+  @test_util.run_deprecated_v1
   def testReverseRowsOf4Channels(self):
     with self.session(use_gpu=True):
       for reverse_f in [array_ops.reverse_v2, array_ops.reverse]:
@@ -454,6 +472,7 @@ class MeshgridTest(test_util.TensorFlowTestCase):
         for x_np, x_tf in zip(numpy_out, tf_out):
           self.assertAllEqual(x_np, x_tf.eval())
 
+  @test_util.run_deprecated_v1
   def testCompare(self):
     for t in (np.float16, np.float32, np.float64, np.int32, np.int64,
               np.complex64, np.complex128):
@@ -526,6 +545,7 @@ STRIDED_SLICE_TYPES = [
 class StridedSliceTest(test_util.TensorFlowTestCase):
   """Test the strided slice operation with variants of slices."""
 
+  @test_util.run_deprecated_v1
   def test_basic_slice(self):
     for tensor_type in STRIDED_SLICE_TYPES:
       with self.cached_session(use_gpu=True):
@@ -581,6 +601,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       v = variables.Variable([1., 2.])
       v[0]  # pylint: disable=pointless-statement
 
+  @test_util.run_deprecated_v1
   def testDegenerateSlices(self):
     with self.session(use_gpu=True):
       checker = StridedSliceChecker(self, StridedSliceChecker.REF_TENSOR)
@@ -591,6 +612,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       # empty interval in every dimension
       _ = checker[-1:0, 2:2, 2:3:-1]
 
+  @test_util.run_deprecated_v1
   def testEllipsis(self):
     with self.session(use_gpu=True):
       raw = [[[[[1, 2], [3, 4], [5, 6]]], [[[7, 8], [9, 10], [11, 12]]]]]
@@ -611,6 +633,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(ValueError, "Multiple ellipses"):
         _ = checker[..., :, ...].eval()
 
+  @test_util.run_deprecated_v1
   def testShrink(self):
     with self.session(use_gpu=True):
       raw = [[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]],
@@ -621,6 +644,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       _ = checker[:, 0]
       _ = checker[:, :, 0]
 
+  @test_util.run_deprecated_v1
   def testBothNewAxisAndShrink(self):
     with self.session(use_gpu=True):
       ones = array_ops.placeholder(shape=[2, 2], dtype=dtypes.int16)
@@ -629,6 +653,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
               feed_dict={ones: [[1, 1], [1, 1]]}),
           [[1, 1]])
 
+  @test_util.run_deprecated_v1
   def testTensorIndexing(self):
     with self.session(use_gpu=True):
       raw = [[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]],
@@ -654,6 +679,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(TypeError, expected):
         _ = checker[constant_op.constant(0.0)]
 
+  @test_util.run_deprecated_v1
   def testExpand(self):
     with self.session(use_gpu=True):
       raw = [[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]],
@@ -671,6 +697,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       # Ellipsis in middle of two newaxis
       _ = checker[np.newaxis, ..., np.newaxis]
 
+  @test_util.run_deprecated_v1
   def testExpandVariable(self):
     with self.session(use_gpu=True):
       x = variables.Variable(7, dtype=dtypes.int32)
@@ -679,6 +706,7 @@ class StridedSliceTest(test_util.TensorFlowTestCase):
       self.assertEqual(y.shape, (1,))
       self.assertAllEqual(y, (7,))
 
+  @test_util.run_deprecated_v1
   def testOptimizedCases(self):
     with self.session(use_gpu=True):
       checker = StridedSliceChecker(self,
@@ -708,6 +736,7 @@ class StridedSliceShapeChecker(object):
 class StridedSliceShapeTest(test_util.TensorFlowTestCase):
   """Test the shape inference of StridedSliceShapes."""
 
+  @test_util.run_deprecated_v1
   def testUnknown(self):
     with self.session(use_gpu=True):
       uncertain_tensor = array_ops.placeholder(dtypes.float32)
@@ -719,6 +748,7 @@ class StridedSliceShapeTest(test_util.TensorFlowTestCase):
     self.assertTrue(x is not None and y is not None or x is None and y is None)
     self.assertEqual(x.as_list(), y.as_list())
 
+  @test_util.run_deprecated_v1
   def testTensorShapeUncertain(self):
     with self.session(use_gpu=True):
       uncertain_tensor = array_ops.placeholder(
@@ -742,6 +772,7 @@ class StridedSliceShapeTest(test_util.TensorFlowTestCase):
       self.tensorShapeEqual(a[::-1, :, array_ops.newaxis, ::-2],
                             tensor_shape.TensorShape([5, None, 1, 4]))
 
+  @test_util.run_deprecated_v1
   def testTensorValuedIndexShape(self):
     with self.session(use_gpu=True):
       defined_shape_tensor = array_ops.placeholder(
@@ -798,6 +829,7 @@ class GradSliceChecker(object):
 class StridedSliceGradTest(test_util.TensorFlowTestCase):
   """Test that strided slice's custom gradient produces correct gradients."""
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.session(use_gpu=True) as sess:
       var = variables.Variable(
@@ -827,6 +859,7 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase):
       grad = GradSliceChecker(self, sess, var, np.array(8))
       _ = grad[tuple()]
 
+  @test_util.run_deprecated_v1
   def testInt64Indices(self):
     with self.session(use_gpu=True) as sess:
       a = math_ops.range(3, dtype=dtypes.float32)
@@ -839,6 +872,7 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase):
 class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
   """Test varied index types and host located memory."""
 
+  @test_util.run_deprecated_v1
   def testHostVsDevice(self):
     with self.session(use_gpu=True) as sess:
       var2 = variables.Variable(
@@ -853,6 +887,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
       foo = array_ops.strided_slice_grad(varshape, begin, end, strides, var2)
       sess.run(foo)
 
+  @test_util.run_deprecated_v1
   def testInt64Shape(self):
     with self.session(use_gpu=True) as sess:
       original_dy = array_ops.reshape(
@@ -867,6 +902,7 @@ class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
                                         original_dy)
       sess.run(dx)
 
+  @test_util.run_deprecated_v1
   def testMixedIndexTypes(self):
     with self.session(use_gpu=True) as sess:
       original_dy = array_ops.reshape(
@@ -975,6 +1011,7 @@ class StridedSliceAssignChecker(object):
 
 class SliceAssignTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testInvalidSlice(self):
     with self.cached_session() as sess:
       foo = constant_op.constant([1, 2, 3])
@@ -1012,12 +1049,15 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     checker2[...] = 6  # ellipsis
     checker2[None] = [6]  # new axis
 
+  @test_util.run_deprecated_v1
   def testSliceAssign(self):
     self.doTestSliceAssign(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testSliceAssignResource(self):
     self.doTestSliceAssign(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testUninitialized(self):
     with self.assertRaisesRegexp(
         errors.FailedPreconditionError,
@@ -1036,6 +1076,7 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
     with self.assertRaises(TypeError):
       v[:].assign(too_large_val)
 
+  @test_util.run_deprecated_v1
   def testTypeErrorResource(self):
     init_val = constant_op.constant([1, 2], dtype=dtypes.int32)
     too_small_val = constant_op.constant([3, 4], dtype=dtypes.int8)
@@ -1092,6 +1133,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(ValueError, "maxlen must be scalar"):
         array_ops.sequence_mask([10, 20], [10, 20])
 
+  @test_util.run_deprecated_v1
   def testOneDimensionalWithMaxlen(self):
     with self.cached_session():
       res = array_ops.sequence_mask(constant_op.constant([1, 3, 2]), 5)
@@ -1101,6 +1143,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
           [[True, False, False, False, False], [True, True, True, False, False],
            [True, True, False, False, False]])
 
+  @test_util.run_deprecated_v1
   def testOneDimensionalDtypeWithoutMaxlen(self):
     with self.cached_session():
       # test dtype and default maxlen:
@@ -1111,6 +1154,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
           res.eval(),
           [[0.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0]])
 
+  @test_util.run_deprecated_v1
   def testOneDimensionalWithoutMaxlen(self):
     with self.cached_session():
       res = array_ops.sequence_mask(
@@ -1139,11 +1183,13 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
           [[[0.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0]],
            [[1.0, 0.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 1.0, 0.0]]])
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     lengths = array_ops.placeholder(dtype=dtypes.int32)
     res = array_ops.sequence_mask(lengths)
     self.assertEqual(res.shape, None)
 
+  @test_util.run_deprecated_v1
   def testDtypes(self):
 
     def check_dtypes(lengths_dtype, maxlen_dtype):
@@ -1166,6 +1212,7 @@ class SequenceMaskTest(test_util.TensorFlowTestCase):
 class ConcatSliceResourceTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testConcatSlice(self):
     r1 = test_ops.stub_resource_handle_op(container="a", shared_name="b")
     r2 = test_ops.stub_resource_handle_op(container="a", shared_name="c")
@@ -1221,6 +1268,7 @@ class PadTest(test_util.TensorFlowTestCase):
 
 class InvertPermutationTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testInvertPermutation(self):
     for dtype in [dtypes.int32, dtypes.int64]:
       with self.cached_session(use_gpu=True):
@@ -1255,12 +1303,14 @@ class UnravelIndexTest(test_util.TensorFlowTestCase):
 
 class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     with self.cached_session():
       a = array_ops.constant(10)
       guarantee_a = array_ops.guarantee_const(a)
       self.assertEqual(10, guarantee_a.eval())
 
+  @test_util.run_deprecated_v1
   def testVariables(self):
     with self.cached_session() as sess:
       for use_resource in [False, True]:
@@ -1272,6 +1322,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
         self.evaluate(variables.global_variables_initializer())
         self.assertEqual(10.0, guarantee_a.eval())
 
+  @test_util.run_deprecated_v1
   def testResourceRejection(self):
     with self.cached_session() as sess:
       a = variable_scope.get_variable(
@@ -1287,6 +1338,7 @@ class GuaranteeConstOpTest(test_util.TensorFlowTestCase):
 
 class SnapshotOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testInvertPermutation(self):
     for dtype in [dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64]:
       with self.cached_session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/as_string_op_test.py b/tensorflow/python/kernel_tests/as_string_op_test.py
index dd4a90e5f6..287701a73e 100644
--- a/tensorflow/python/kernel_tests/as_string_op_test.py
+++ b/tensorflow/python/kernel_tests/as_string_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
@@ -27,6 +28,7 @@ from tensorflow.python.platform import test
 
 class AsStringOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     float_inputs_ = [
         0, 1, -1, 0.5, 0.25, 0.125, float("INF"), float("NAN"), float("-INF")
@@ -78,6 +80,7 @@ class AsStringOpTest(test.TestCase):
         output = string_ops.as_string(input_, fill="ab")
         output.eval(feed_dict={input_: float_inputs_})
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     # Cannot use values outside -128..127 for test, because we're also
     # testing int8
@@ -112,6 +115,7 @@ class AsStringOpTest(test.TestCase):
         output = string_ops.as_string(input_, precision=0)
         output.eval(feed_dict={input_: int_inputs_})
 
+  @test_util.run_deprecated_v1
   def testLargeInt(self):
     # Cannot use values outside -128..127 for test, because we're also
     # testing int8
@@ -130,6 +134,7 @@ class AsStringOpTest(test.TestCase):
       result = output.eval(feed_dict={input_: int_inputs_})
       self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_])
 
+  @test_util.run_deprecated_v1
   def testHalfInt(self):
     s = lambda strs: [x.decode("ascii") for x in strs]
 
@@ -140,6 +145,7 @@ class AsStringOpTest(test.TestCase):
       result = output.eval(feed_dict={input_: int_inputs_})
       self.assertAllEqual(s(result), ["%d" % x for x in int_inputs_])
 
+  @test_util.run_deprecated_v1
   def testBool(self):
     bool_inputs_ = [False, True]
     s = lambda strs: [x.decode("ascii") for x in strs]
@@ -152,6 +158,7 @@ class AsStringOpTest(test.TestCase):
         result = output.eval(feed_dict={input_: bool_inputs_})
         self.assertAllEqual(s(result), ["false", "true"])
 
+  @test_util.run_deprecated_v1
   def testComplex(self):
     float_inputs_ = [
         0, 1, -1, 0.5, 0.25, 0.125, complex("INF"), complex("NAN"),
diff --git a/tensorflow/python/kernel_tests/atrous_conv2d_test.py b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
index fefb797995..a13e325835 100644
--- a/tensorflow/python/kernel_tests/atrous_conv2d_test.py
+++ b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_impl
@@ -58,6 +59,7 @@ def _upsample_filters(filters, rate):
 
 class AtrousConv2DTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAtrousConv2DForward(self):
     with self.session(use_gpu=True):
       # Input: [batch, height, width, input_depth]
@@ -82,6 +84,7 @@ class AtrousConv2DTest(test.TestCase):
                 self.assertAllClose(
                     y1.eval(), self.evaluate(y2), rtol=1e-3, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testAtrousSequence(self):
     """Tests optimization of sequence of atrous convolutions.
 
@@ -135,6 +138,7 @@ class AtrousConv2DTest(test.TestCase):
               self.assertAllClose(
                   y1.eval(), self.evaluate(y2), rtol=1e-2, atol=1e-2)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.session(use_gpu=True):
       # Input: [batch, height, width, input_depth]
@@ -162,6 +166,7 @@ class AtrousConv2DTest(test.TestCase):
 
 class AtrousConv2DTransposeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAtrousConv2DTransposeForward(self):
     with self.session(use_gpu=True):
       # Input: [batch, height, width, input_depth]
@@ -201,6 +206,7 @@ class AtrousConv2DTransposeTest(test.TestCase):
 
 class AtrousDepthwiseConv2DTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAtrousDepthwiseConv2DForward(self):
     strides = [1, 1, 1, 1]
     with self.session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/barrier_ops_test.py b/tensorflow/python/kernel_tests/barrier_ops_test.py
index 495bbe7b34..60fe6f0eec 100644
--- a/tensorflow/python/kernel_tests/barrier_ops_test.py
+++ b/tensorflow/python/kernel_tests/barrier_ops_test.py
@@ -25,6 +25,7 @@ import numpy as np
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.platform import test
 
@@ -66,6 +67,7 @@ class BarrierTest(test.TestCase):
       attr { key: 'shared_name' value: { s: 'B' } }
       """, b.barrier_ref.op.node_def)
 
+  @test_util.run_deprecated_v1
   def testInsertMany(self):
     with self.cached_session():
       b = data_flow_ops.Barrier(
@@ -90,6 +92,7 @@ class BarrierTest(test.TestCase):
         data_flow_ops.Barrier(
             (dtypes.float32, dtypes.float32), shapes=((1,), (0,)), name="B")
 
+  @test_util.run_deprecated_v1
   def testInsertManyEmptyTensorUnknown(self):
     with self.cached_session():
       b = data_flow_ops.Barrier((dtypes.float32, dtypes.float32), name="B")
@@ -102,6 +105,7 @@ class BarrierTest(test.TestCase):
           ".*Tensors with no elements are not supported.*"):
         insert_0_op.run()
 
+  @test_util.run_deprecated_v1
   def testTakeMany(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(
@@ -127,6 +131,7 @@ class BarrierTest(test.TestCase):
       self.assertEqual(values_0_val[idx], v0)
       self.assertEqual(values_1_val[idx], v1)
 
+  @test_util.run_deprecated_v1
   def testTakeManySmallBatch(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(
@@ -191,6 +196,7 @@ class BarrierTest(test.TestCase):
       with self.assertRaisesOpError("is closed"):
         insert_1_3_op.run()
 
+  @test_util.run_deprecated_v1
   def testUseBarrierWithShape(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(
@@ -220,6 +226,7 @@ class BarrierTest(test.TestCase):
       self.assertAllEqual(values_0_val[idx], v0)
       self.assertAllEqual(values_1_val[idx], v1)
 
+  @test_util.run_deprecated_v1
   def testParallelInsertMany(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(dtypes.float32, shapes=())
@@ -240,6 +247,7 @@ class BarrierTest(test.TestCase):
       idx = keys_val.tolist().index(k)
       self.assertEqual(values_val[idx], v)
 
+  @test_util.run_deprecated_v1
   def testParallelTakeMany(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(dtypes.float32, shapes=())
@@ -274,6 +282,7 @@ class BarrierTest(test.TestCase):
     self.assertItemsEqual(
         zip(keys, values), [(k[0], v[0]) for k, v in zip(key_vals, value_vals)])
 
+  @test_util.run_deprecated_v1
   def testBlockingTakeMany(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(dtypes.float32, shapes=())
@@ -296,6 +305,7 @@ class BarrierTest(test.TestCase):
         insert_op.run()
       t.join()
 
+  @test_util.run_deprecated_v1
   def testParallelInsertManyTakeMany(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(
@@ -375,6 +385,7 @@ class BarrierTest(test.TestCase):
              2 + outer_indices_from_keys + inner_indices_from_keys)).T
         self.assertAllEqual(taken_i["values_1"], expected_values_1)
 
+  @test_util.run_deprecated_v1
   def testClose(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(
@@ -433,6 +444,7 @@ class BarrierTest(test.TestCase):
       with self.assertRaisesOpError("is closed and has insufficient elements"):
         sess.run(take_t[0])
 
+  @test_util.run_deprecated_v1
   def testCancel(self):
     with self.cached_session() as sess:
       b = data_flow_ops.Barrier(
@@ -495,6 +507,7 @@ class BarrierTest(test.TestCase):
       with self.assertRaisesOpError("is closed and has insufficient elements"):
         self.evaluate(take_t)
 
+  @test_util.run_deprecated_v1
   def testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(self):
     self._testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(cancel=False)
     self._testClosedEmptyBarrierTakeManyAllowSmallBatchRaises(cancel=True)
@@ -569,9 +582,11 @@ class BarrierTest(test.TestCase):
           sorted(taken),
           [0] * (num_iterations // 2) + [10] * (num_iterations // 2))
 
+  @test_util.run_deprecated_v1
   def testParallelInsertManyTakeManyCloseHalfwayThrough(self):
     self._testParallelInsertManyTakeManyCloseHalfwayThrough(cancel=False)
 
+  @test_util.run_deprecated_v1
   def testParallelInsertManyTakeManyCancelHalfwayThrough(self):
     self._testParallelInsertManyTakeManyCloseHalfwayThrough(cancel=True)
 
@@ -669,12 +684,15 @@ class BarrierTest(test.TestCase):
       else:
         self.assertEqual(taken, [10] * num_iterations)
 
+  @test_util.run_deprecated_v1
   def testParallelPartialInsertManyTakeManyCloseHalfwayThrough(self):
     self._testParallelPartialInsertManyTakeManyCloseHalfwayThrough(cancel=False)
 
+  @test_util.run_deprecated_v1
   def testParallelPartialInsertManyTakeManyCancelHalfwayThrough(self):
     self._testParallelPartialInsertManyTakeManyCloseHalfwayThrough(cancel=True)
 
+  @test_util.run_deprecated_v1
   def testIncompatibleSharedBarrierErrors(self):
     with self.cached_session():
       # Do component types and shapes.
diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py
index cd33048121..1a8513d022 100644
--- a/tensorflow/python/kernel_tests/basic_gpu_test.py
+++ b/tensorflow/python/kernel_tests/basic_gpu_test.py
@@ -159,6 +159,7 @@ class BroadcastSimpleTest(test.TestCase):
     with self.cached_session(use_gpu=True) as sess:
       return sess.run(broadcast_gradient_args(xs, ys))
 
+  @test_util.run_deprecated_v1
   def testBroadcast(self):
     r0, r1 = self._GetGradientArgs([2, 3, 5], [1])
     self.assertAllEqual(r0, [])
@@ -219,6 +220,7 @@ class BroadcastSimpleTest(test.TestCase):
     self.assertShapeEqual(np_ans, out)
     # TODO(zhifengc/ke): make gradient checker work on GPU.
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     x = (1 + np.linspace(0, 5, np.prod([1, 3, 2]))).astype(np.float32).reshape(
         [1, 3, 2])
@@ -255,6 +257,7 @@ class GpuMultiSessionMemoryTest(test_util.TensorFlowTestCase):
           if len(results) != 1:
             break
 
+  @test_util.run_deprecated_v1
   def testConcurrentSessions(self):
     n_threads = 4
     threads = []
diff --git a/tensorflow/python/kernel_tests/batch_gather_op_test.py b/tensorflow/python/kernel_tests/batch_gather_op_test.py
index ad4e879131..7e0b3e1b5e 100644
--- a/tensorflow/python/kernel_tests/batch_gather_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_gather_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -87,6 +88,7 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         self.assertAllEqual(np_val, gather_val)
         self.assertEqual(np_val.shape, gather_t.get_shape())
 
+  @test_util.run_deprecated_v1
   def testString(self):
     params = np.array([[b"asdf", b"zxcv"], [b"qwer", b"uiop"]])
     with self.cached_session():
@@ -94,6 +96,7 @@ class GatherTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual([[b"qwer", b"uiop"]],
                           array_ops.batch_gather(params, indices_tf).eval())
 
+  @test_util.run_deprecated_v1
   def testUnknownIndices(self):
     params = constant_op.constant([[0, 1, 2]])
     indices = array_ops.placeholder(dtypes.int32, shape=[None, None])
@@ -106,6 +109,7 @@ class GatherTest(test.TestCase, parameterized.TestCase):
       with self.assertRaisesOpError(r"indices\[0\] = 7 is not in \[0, 2\)"):
         array_ops.batch_gather(params, [7]).eval()
 
+  @test_util.run_deprecated_v1
   def testEmptySlices(self):
     with self.session(use_gpu=True):
       for dtype in _TEST_TYPES:
diff --git a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
index a4b461bc87..eefcdc508f 100644
--- a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -73,6 +74,7 @@ class ScatterTest(test.TestCase):
           tf_scatter(ref, indices, updates).eval()
           self.assertAllClose(ref.eval(), new)
 
+  @test_util.run_deprecated_v1
   def testVariableRankUpdate(self):
     vtypes = [np.float32, np.float64]
     for vtype in vtypes:
@@ -80,6 +82,7 @@ class ScatterTest(test.TestCase):
         self._VariableRankTest(
             state_ops.batch_scatter_update, vtype, itype)
 
+  @test_util.run_deprecated_v1
   def testBooleanScatterUpdate(self):
     with self.session(use_gpu=False) as session:
       var = variables.Variable([True, False])
@@ -93,6 +96,7 @@ class ScatterTest(test.TestCase):
 
       self.assertAllEqual([False, True], self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testScatterOutOfRange(self):
     params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32)
     updates = np.array([-3, -4, -5]).astype(np.float32)
diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py
index 03f3f64353..c422df8806 100644
--- a/tensorflow/python/kernel_tests/batchtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py
@@ -27,6 +27,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gradient_checker
@@ -50,6 +51,7 @@ class CppOpImpl(object):
 class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl):
 
   # Verifies that: batch_to_space(x) = transpose(depth_to_space(transpose(x)))
+  @test_util.run_deprecated_v1
   def testDepthToSpaceTranspose(self):
     x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7])
     block_size = 2
@@ -70,6 +72,7 @@ class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl):
 
 class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl):
 
+  @test_util.run_deprecated_v1
   def testInputWrongDimMissingBatch(self):
     # The input is missing the first dimension ("batch")
     x_np = [[[1], [2]], [[3], [4]]]
@@ -78,6 +81,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl):
     with self.assertRaises(ValueError):
       _ = self.batch_to_space(x_np, crops, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSize0(self):
     # The block size is 0.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -87,6 +91,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl):
       out_tf = self.batch_to_space(x_np, crops, block_size)
       out_tf.eval()
 
+  @test_util.run_deprecated_v1
   def testBlockSizeOne(self):
     # The block size is 1. The block size needs to be > 1.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -96,6 +101,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl):
       out_tf = self.batch_to_space(x_np, crops, block_size)
       out_tf.eval()
 
+  @test_util.run_deprecated_v1
   def testBlockSizeLarger(self):
     # The block size is too large for this input.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -105,6 +111,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl):
       out_tf = self.batch_to_space(x_np, crops, block_size)
       out_tf.eval()
 
+  @test_util.run_deprecated_v1
   def testBlockSizeSquaredNotDivisibleBatch(self):
     # The block size squared does not divide the batch.
     x_np = [[[[1], [2], [3]], [[3], [4], [7]]]]
@@ -113,6 +120,7 @@ class BatchToSpaceErrorHandlingTest(test.TestCase, PythonOpImpl):
     with self.assertRaises(ValueError):
       _ = self.batch_to_space(x_np, crops, block_size)
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     t = self.batch_to_space(
         array_ops.placeholder(dtypes.float32),
@@ -160,28 +168,35 @@ class BatchToSpaceNDErrorHandlingTest(test.TestCase):
     self._testStaticShape(input_shape, block_shape, paddings, error)
     self._testDynamicShape(input_shape, block_shape, paddings)
 
+  @test_util.run_deprecated_v1
   def testInputWrongDimMissingBatch(self):
     self._testShape([2, 2], [2, 2], [[0, 0], [0, 0]], ValueError)
     self._testShape([2, 2, 3], [2, 2, 3], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testBlockSize0(self):
     # The block size is 0.
     self._testShape([1, 2, 2, 1], [0, 1], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNegative(self):
     self._testShape([1, 2, 2, 1], [-1, 1], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testNegativePadding(self):
     self._testShape([1, 2, 2], [1, 1], [[0, -1], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testCropTooLarge(self):
     # The amount to crop exceeds the padded size.
     self._testShape([1 * 2 * 2, 2, 3, 1], [2, 2], [[3, 2], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeSquaredNotDivisibleBatch(self):
     # The batch dimension is not divisible by the product of the block_shape.
     self._testShape([3, 1, 1, 1], [2, 3], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     # Verify that input shape and paddings shape can be unknown.
     _ = array_ops.batch_to_space_nd(
@@ -263,18 +278,21 @@ class BatchToSpaceGradientTest(test.TestCase, PythonOpImpl):
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
+  @test_util.run_deprecated_v1
   def testSmall(self):
     block_size = 2
     crop_beg = 0
     crop_end = 0
     self._compare(1, 2, 3, 5, block_size, crop_beg, crop_end)
 
+  @test_util.run_deprecated_v1
   def testSmall2(self):
     block_size = 2
     crop_beg = 0
     crop_end = 0
     self._compare(2, 4, 3, 2, block_size, crop_beg, crop_end)
 
+  @test_util.run_deprecated_v1
   def testSmallCrop1x1(self):
     block_size = 2
     crop_beg = 1
@@ -316,14 +334,17 @@ class BatchToSpaceNDGradientTest(test.TestCase):
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
+  @test_util.run_deprecated_v1
   def testSmall(self):
     for dtype in [dtypes.int64, dtypes.int32]:
       self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype)
 
+  @test_util.run_deprecated_v1
   def testSmall2(self):
     for dtype in [dtypes.int64, dtypes.int32]:
       self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype)
 
+  @test_util.run_deprecated_v1
   def testSmallCrop1x1(self):
     for dtype in [dtypes.int64, dtypes.int32]:
       self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype)
diff --git a/tensorflow/python/kernel_tests/bcast_ops_test.py b/tensorflow/python/kernel_tests/bcast_ops_test.py
index 3ec820aead..ae00955ac2 100644
--- a/tensorflow/python/kernel_tests/bcast_ops_test.py
+++ b/tensorflow/python/kernel_tests/bcast_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops.gen_array_ops import broadcast_args
 from tensorflow.python.ops.gen_array_ops import broadcast_gradient_args
 from tensorflow.python.platform import test
@@ -35,6 +36,7 @@ class BcastOpsTest(test.TestCase):
     with self.cached_session() as sess:
       return sess.run(broadcast_gradient_args(xs, ys))
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     r = self._GetBroadcastShape([2, 3, 5], [1])
     self.assertAllEqual(r, [2, 3, 5])
@@ -66,6 +68,7 @@ class BcastOpsTest(test.TestCase):
     r = self._GetBroadcastShape([3, 1], [2, 1, 5])
     self.assertAllEqual(r, [2, 3, 5])
 
+  @test_util.run_deprecated_v1
   def testBasicGradient(self):
     r0, r1 = self._GetGradientArgs([2, 3, 5], [1])
     self.assertAllEqual(r0, [])
@@ -107,6 +110,7 @@ class BcastOpsTest(test.TestCase):
     self.assertAllEqual(r0, [0, 2])
     self.assertAllEqual(r1, [1])
 
+  @test_util.run_deprecated_v1
   def testZeroDims(self):
     r = self._GetBroadcastShape([2, 0, 3, 0, 5], [3, 0, 5])
     self.assertAllEqual(r, [2, 0, 3, 0, 5])
@@ -120,6 +124,7 @@ class BcastOpsTest(test.TestCase):
     r = self._GetBroadcastShape([3, 1, 5], [2, 0, 3, 0, 5])
     self.assertAllEqual(r, [2, 0, 3, 0, 5])
 
+  @test_util.run_deprecated_v1
   def testZeroDimsGradient(self):
     r0, r1 = self._GetGradientArgs([2, 0, 3, 0, 5], [3, 0, 5])
     self.assertAllEqual(r0, [])
@@ -137,6 +142,7 @@ class BcastOpsTest(test.TestCase):
     self.assertAllEqual(r0, [0, 1, 3])
     self.assertAllEqual(r1, [])
 
+  @test_util.run_deprecated_v1
   def testDataTypes(self):
     for dtype in [dtypes.int32, dtypes.int64]:
       r = self._GetBroadcastShape(
diff --git a/tensorflow/python/kernel_tests/betainc_op_test.py b/tensorflow/python/kernel_tests/betainc_op_test.py
index 5d7446042e..9dc34a6062 100644
--- a/tensorflow/python/kernel_tests/betainc_op_test.py
+++ b/tensorflow/python/kernel_tests/betainc_op_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -109,36 +110,42 @@ class BetaincTest(test.TestCase):
     except ImportError as e:
       tf_logging.warn("Cannot test special functions: %s" % str(e))
 
+  @test_util.run_deprecated_v1
   def testBetaIncFloat(self):
     a_s = np.abs(np.random.randn(10, 10) * 30)  # in (0, infty)
     b_s = np.abs(np.random.randn(10, 10) * 30)  # in (0, infty)
     x_s = np.random.rand(10, 10)  # in (0, 1)
     self._testBetaInc(a_s, b_s, x_s, dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def testBetaIncDouble(self):
     a_s = np.abs(np.random.randn(10, 10) * 30)  # in (0, infty)
     b_s = np.abs(np.random.randn(10, 10) * 30)  # in (0, infty)
     x_s = np.random.rand(10, 10)  # in (0, 1)
     self._testBetaInc(a_s, b_s, x_s, dtypes.float64)
 
+  @test_util.run_deprecated_v1
   def testBetaIncDoubleVeryLargeValues(self):
     a_s = np.abs(np.random.randn(10, 10) * 1e15)  # in (0, infty)
     b_s = np.abs(np.random.randn(10, 10) * 1e15)  # in (0, infty)
     x_s = np.random.rand(10, 10)  # in (0, 1)
     self._testBetaInc(a_s, b_s, x_s, dtypes.float64)
 
+  @test_util.run_deprecated_v1
   def testBetaIncDoubleVerySmallValues(self):
     a_s = np.abs(np.random.randn(10, 10) * 1e-16)  # in (0, infty)
     b_s = np.abs(np.random.randn(10, 10) * 1e-16)  # in (0, infty)
     x_s = np.random.rand(10, 10)  # in (0, 1)
     self._testBetaInc(a_s, b_s, x_s, dtypes.float64)
 
+  @test_util.run_deprecated_v1
   def testBetaIncFloatVerySmallValues(self):
     a_s = np.abs(np.random.randn(10, 10) * 1e-8)  # in (0, infty)
     b_s = np.abs(np.random.randn(10, 10) * 1e-8)  # in (0, infty)
     x_s = np.random.rand(10, 10)  # in (0, 1)
     self._testBetaInc(a_s, b_s, x_s, dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def testBetaIncFpropAndBpropAreNeverNAN(self):
     with self.cached_session() as sess:
       space = np.logspace(-8, 5).tolist()
@@ -159,6 +166,7 @@ class BetaincTest(test.TestCase):
       self.assertAllEqual(np.zeros_like(grads_x).astype(np.bool),
                           np.isnan(grads_x))
 
+  @test_util.run_deprecated_v1
   def testBetaIncGrads(self):
     err_tolerance = 1e-3
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/bias_op_test.py b/tensorflow/python/kernel_tests/bias_op_test.py
index 749d6a791e..66f442dbdd 100644
--- a/tensorflow/python/kernel_tests/bias_op_test.py
+++ b/tensorflow/python/kernel_tests/bias_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -89,10 +90,12 @@ class BiasAddTest(test.TestCase):
       self._testBiasNCHW(np_inputs, np_bias, use_gpu=True)
 
 
+  @test_util.run_deprecated_v1
   def testInputDims(self):
     with self.assertRaises(ValueError):
       nn_ops.bias_add([1, 2], [1])
 
+  @test_util.run_deprecated_v1
   def testBiasVec(self):
     with self.assertRaises(ValueError):
       nn_ops.bias_add(
@@ -101,6 +104,7 @@ class BiasAddTest(test.TestCase):
           array_ops.reshape(
               [1, 2], shape=[1, 2]))
 
+  @test_util.run_deprecated_v1
   def testBiasInputsMatch(self):
     with self.assertRaises(ValueError):
       nn_ops.bias_add(
@@ -109,23 +113,27 @@ class BiasAddTest(test.TestCase):
           array_ops.reshape(
               [1], shape=[1]))
 
+  @test_util.run_deprecated_v1
   def testIntTypes(self):
     for t in [np.int8, np.int16, np.int32, np.int64]:
       self._testAll(
           np.array([[10, 20, 30], [40, 50, 60]]).astype(t),
           np.array([1, 2, 3]).astype(t))
 
+  @test_util.run_deprecated_v1
   def testFloatTypes(self):
     for t in [np.float16, np.float32, np.float64]:
       self._testAll(
           np.random.rand(4, 3, 3).astype(t), np.random.rand(3).astype(t))
 
+  @test_util.run_deprecated_v1
   def test4DFloatTypes(self):
     for t in [np.float16, np.float32, np.float64]:
       self._testAll(
           np.random.rand(4, 3, 2, 3).astype(t),
           np.random.rand(3).astype(t))
 
+  @test_util.run_deprecated_v1
   def test5DFloatTypes(self):
     for t in [np.float16, np.float32, np.float64]:
       self._testAll(
@@ -187,6 +195,7 @@ class BiasAddTest(test.TestCase):
       self.assertAllClose(bias_jacob_t, bias_jacob_n, threshold, threshold)
       self.assertAllClose(grad_jacob_t, grad_jacob_n, threshold, threshold)
 
+  @test_util.run_deprecated_v1
   def testGradientTensor(self):
     # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once
     # all dimensions are supported.
@@ -198,6 +207,7 @@ class BiasAddTest(test.TestCase):
         bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
+  @test_util.run_deprecated_v1
   def testGradientTensor4D(self):
     # BiasAddGrad with NCHW support 4D so all are enabled.
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
@@ -209,11 +219,13 @@ class BiasAddTest(test.TestCase):
         bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     np.random.seed(7)
     for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3):
       self._testAll(np.random.randn(*shape), np.random.randn(shape[-1]))
 
+  @test_util.run_deprecated_v1
   def testEmptyGradient(self):
     # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once
     # all dimensions are supported.
diff --git a/tensorflow/python/kernel_tests/bincount_op_test.py b/tensorflow/python/kernel_tests/bincount_op_test.py
index 49eb835847..d064d736cf 100644
--- a/tensorflow/python/kernel_tests/bincount_op_test.py
+++ b/tensorflow/python/kernel_tests/bincount_op_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest
 
 class BincountTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def test_empty(self):
     with self.session(use_gpu=True):
       self.assertAllEqual(
@@ -43,6 +44,7 @@ class BincountTest(test_util.TensorFlowTestCase):
           math_ops.bincount([], minlength=3, dtype=np.float64).eval().dtype,
           np.float64)
 
+  @test_util.run_deprecated_v1
   def test_values(self):
     with self.session(use_gpu=True):
       self.assertAllEqual(
@@ -58,12 +60,14 @@ class BincountTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(
           math_ops.bincount(np.arange(10000)).eval(), np.ones(10000))
 
+  @test_util.run_deprecated_v1
   def test_maxlength(self):
     with self.session(use_gpu=True):
       self.assertAllEqual(math_ops.bincount([5], maxlength=3).eval(), [0, 0, 0])
       self.assertAllEqual(math_ops.bincount([1], maxlength=3).eval(), [0, 1])
       self.assertAllEqual(math_ops.bincount([], maxlength=3).eval(), [])
 
+  @test_util.run_deprecated_v1
   def test_random_with_weights(self):
     num_samples = 10000
     with self.session(use_gpu=True):
@@ -77,6 +81,7 @@ class BincountTest(test_util.TensorFlowTestCase):
         self.assertAllClose(
             math_ops.bincount(arr, weights).eval(), np.bincount(arr, weights))
 
+  @test_util.run_deprecated_v1
   def test_random_without_weights(self):
     num_samples = 10000
     with self.session(use_gpu=True):
@@ -87,6 +92,7 @@ class BincountTest(test_util.TensorFlowTestCase):
         self.assertAllClose(
             math_ops.bincount(arr, None).eval(), np.bincount(arr, weights))
 
+  @test_util.run_deprecated_v1
   def test_zero_weights(self):
     with self.session(use_gpu=True):
       self.assertAllEqual(
@@ -99,6 +105,7 @@ class BincountTest(test_util.TensorFlowTestCase):
       with self.assertRaises(errors.InvalidArgumentError):
         math_ops.bincount([1, 2, 3, -1, 6, 8]).eval()
 
+  @test_util.run_deprecated_v1
   def test_shape_function(self):
     # size must be scalar.
     with self.assertRaisesRegexp(
diff --git a/tensorflow/python/kernel_tests/bitcast_op_test.py b/tensorflow/python/kernel_tests/bitcast_op_test.py
index 5ceffcfeda..b4f9a21a89 100644
--- a/tensorflow/python/kernel_tests/bitcast_op_test.py
+++ b/tensorflow/python/kernel_tests/bitcast_op_test.py
@@ -60,6 +60,7 @@ class BitcastTest(test.TestCase):
     shape = [3, 4]
     self._testBitcast(x, dtypes.int64, shape)
 
+  @test_util.run_deprecated_v1
   def testErrors(self):
     x = np.zeros([1, 1], np.int8)
     datatype = dtypes.int32
@@ -72,6 +73,7 @@ class BitcastTest(test.TestCase):
     shape = [4]
     self._testBitcast(x, datatype, shape)
 
+  @test_util.run_deprecated_v1
   def testUnknown(self):
     x = array_ops.placeholder(dtypes.float32)
     datatype = dtypes.int8
diff --git a/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py
index 7cdc67f83f..6b04e8abf4 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.platform import googletest
 class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
   """Tests prediction ops for training."""
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionOnEmptyEnsemble(self):
     """Tests that prediction on a dummy ensemble does not fail."""
     with self.cached_session() as session:
@@ -61,6 +62,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(cached_node_ids, new_node_ids)
       self.assertAllClose([[0], [0]], logits_updates)
 
+  @test_util.run_deprecated_v1
   def testNoCachedPredictionButTreeExists(self):
     """Tests that predictions are updated once trees are added."""
     with self.cached_session() as session:
@@ -127,6 +129,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose([2, 1], new_node_ids)
       self.assertAllClose([[0.1 * 8.79], [0.1 * 1.14]], logits_updates)
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionIsCurrent(self):
     """Tests that prediction based on previous node in the tree works."""
     with self.cached_session() as session:
@@ -199,6 +202,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(cached_node_ids, new_node_ids)
       self.assertAllClose([[0], [0]], logits_updates)
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionFromTheSameTree(self):
     """Tests that prediction based on previous node in the tree works."""
     with self.cached_session() as session:
@@ -313,6 +317,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       # 1.65 and -3.875, and then multiply them by 0.1 (lr)
       self.assertAllClose([[0.1 * 1.65], [0.1 * -3.875]], logits_updates)
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionFromPreviousTree(self):
     """Tests the predictions work when we have cache from previous trees."""
     with self.cached_session() as session:
@@ -445,6 +450,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       #            change= 0.1(1.14+7.0-7.0)
       self.assertAllClose([[1], [0.114]], logits_updates)
 
+  @test_util.run_deprecated_v1
   def testCategoricalSplits(self):
     """Tests the training prediction work for categorical splits."""
     with self.cached_session() as session:
@@ -517,6 +523,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose([3, 4, 2], new_node_ids)
       self.assertAllClose([[5.], [6.], [7.]], logits_updates)
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionFromTheSameTreeWithPostPrunedNodes(self):
     """Tests that prediction based on previous node in the tree works."""
     with self.cached_session() as session:
@@ -647,6 +654,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose([[0.01], [0.01], [0.0553], [0.0783], [0.01], [0.01]],
                           logits_updates + cached_values)
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionFromThePreviousTreeWithPostPrunedNodes(self):
     """Tests that prediction based on previous node in the tree works."""
     with self.cached_session() as session:
@@ -792,6 +800,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
                            [root + 0.0783], [root + 0.01], [root + 0.01]],
                           logits_updates + cached_values)
 
+  @test_util.run_deprecated_v1
   def testCachedPredictionTheWholeTreeWasPruned(self):
     """Tests that prediction based on previous node in the tree works."""
     with self.cached_session() as session:
@@ -864,6 +873,7 @@ class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
 class PredictionOpsTest(test_util.TensorFlowTestCase):
   """Tests prediction ops for inference."""
 
+  @test_util.run_deprecated_v1
   def testPredictionOnEmptyEnsemble(self):
     """Tests that prediction on a empty ensemble does not fail."""
     with self.cached_session() as session:
@@ -886,6 +896,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
       logits = session.run(predict_op)
       self.assertAllClose(expected_logits, logits)
 
+  @test_util.run_deprecated_v1
   def testPredictionMultipleTree(self):
     """Tests the predictions work when we have multiple trees."""
     with self.cached_session() as session:
@@ -996,6 +1007,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
       logits = session.run(predict_op)
       self.assertAllClose(expected_logits, logits)
 
+  @test_util.run_deprecated_v1
   def testCategoricalSplits(self):
     """Tests the predictions work for categorical splits."""
     with self.cached_session() as session:
@@ -1062,6 +1074,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 class FeatureContribsOpsTest(test_util.TensorFlowTestCase):
   """Tests feature contribs ops for model understanding."""
 
+  @test_util.run_deprecated_v1
   def testContribsForOnlyABiasNode(self):
     """Tests case when, after training, only left with a bias node.
 
@@ -1122,6 +1135,7 @@ class FeatureContribsOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(feature_ids, expected_feature_ids)
       self.assertAllClose(logits_paths, expected_logits_paths)
 
+  @test_util.run_deprecated_v1
   def testContribsMultipleTreeWhenFirstTreeIsABiasNode(self):
     """Tests case when, after training, first tree contains only a bias node."""
     with self.cached_session() as session:
@@ -1219,6 +1233,7 @@ class FeatureContribsOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(feature_ids, expected_feature_ids)
       self.assertAllClose(logits_paths, expected_logits_paths)
 
+  @test_util.run_deprecated_v1
   def testContribsMultipleTree(self):
     """Tests that the contribs work when we have multiple trees."""
     with self.cached_session() as session:
diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index 1a7b1a7e90..390672febe 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -82,6 +82,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
     self.max_elements = 1 << 16
     self.num_quantiles = constant_op.constant(3, dtype=dtypes.int64)
 
+  @test_util.run_deprecated_v1
   def testBasicQuantileBucketsSingleResource(self):
     with self.cached_session() as sess:
       quantile_accumulator_handle = self.create_resource("floats", self.eps,
@@ -106,6 +107,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval())
       self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval())
 
+  @test_util.run_deprecated_v1
   def testBasicQuantileBucketsMultipleResources(self):
     with self.cached_session() as sess:
       quantile_accumulator_handle_0 = self.create_resource("float_0", self.eps,
@@ -140,6 +142,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval())
       self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval())
 
+  @test_util.run_deprecated_v1
   def testSaveRestoreAfterFlush(self):
     save_dir = os.path.join(self.get_temp_dir(), "save_restore")
     save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
@@ -172,6 +175,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
+  @test_util.run_deprecated_v1
   def testSaveRestoreBeforeFlush(self):
     save_dir = os.path.join(self.get_temp_dir(), "save_restore")
     save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
diff --git a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
index 493cad80f3..0a34277bbd 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest
 class ResourceOpsTest(test_util.TensorFlowTestCase):
   """Tests resource_ops."""
 
+  @test_util.run_deprecated_v1
   def testCreate(self):
     with self.cached_session():
       ensemble = boosted_trees_ops.TreeEnsemble('ensemble')
@@ -43,6 +44,7 @@ class ResourceOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(0, self.evaluate(num_attempted_layers))
       self.assertAllEqual([0, 1], self.evaluate(nodes_range))
 
+  @test_util.run_deprecated_v1
   def testCreateWithProto(self):
     with self.cached_session():
       ensemble_proto = boosted_trees_pb2.TreeEnsemble()
@@ -160,6 +162,7 @@ class ResourceOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(6, self.evaluate(num_attempted_layers))
       self.assertAllEqual([16, 19], self.evaluate(nodes_range))
 
+  @test_util.run_deprecated_v1
   def testSerializeDeserialize(self):
     with self.cached_session():
       # Initialize.
diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
index e1036b0b75..e2e23486b5 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
@@ -327,6 +327,7 @@ class StatsOpsTest(test_util.TensorFlowTestCase):
            max_splits=max_splits)
       self.assertAllEqual([[], []], self.evaluate(node_ids_list))
 
+  @test_util.run_deprecated_v1
   def testMakeStatsSummarySimple(self):
     """Simple test for MakeStatsSummary."""
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py
index ea022820e4..afc0564fc5 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest
 class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
   """Tests for growing tree ensemble from split candidates."""
 
+  @test_util.run_deprecated_v1
   def testGrowWithEmptyEnsemble(self):
     """Test growing an empty ensemble."""
     with self.cached_session() as session:
@@ -139,6 +140,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testBiasCenteringOnEmptyEnsemble(self):
     """Test growing with bias centering on an empty ensemble."""
     with self.cached_session() as session:
@@ -182,6 +184,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testGrowExistingEnsembleTreeNotFinalized(self):
     """Test growing an existing ensemble with the last tree not finalized."""
     with self.cached_session() as session:
@@ -366,6 +369,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testGrowExistingEnsembleTreeFinalized(self):
     """Test growing an existing ensemble with the last tree finalized."""
     with self.cached_session() as session:
@@ -515,6 +519,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testPrePruning(self):
     """Test growing an existing ensemble with pre-pruning."""
     with self.cached_session() as session:
@@ -671,6 +676,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testMetadataWhenCantSplitDueToEmptySplits(self):
     """Test that the metadata is updated even though we can't split."""
     with self.cached_session() as session:
@@ -782,6 +788,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testMetadataWhenCantSplitDuePrePruning(self):
     """Test metadata is updated correctly when no split due to prepruning."""
     with self.cached_session() as session:
@@ -917,6 +924,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 1)
       self.assertProtoEquals(expected_result, tree_ensemble)
 
+  @test_util.run_deprecated_v1
   def testPostPruningOfSomeNodes(self):
     """Test growing an ensemble with post-pruning."""
     with self.cached_session() as session:
@@ -1251,6 +1259,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(new_stamp, 3)
       self.assertProtoEquals(expected_result, res_ensemble)
 
+  @test_util.run_deprecated_v1
   def testPostPruningOfAllNodes(self):
     """Test growing an ensemble with post-pruning, with all nodes are pruned."""
     with self.cached_session() as session:
@@ -1434,6 +1443,7 @@ class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
       }
       """, res_ensemble)
 
+  @test_util.run_deprecated_v1
   def testPostPruningChangesNothing(self):
     """Test growing an ensemble with post-pruning with all gains >0."""
     with self.cached_session() as session:
diff --git a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
index 233c166405..b9eb2391b4 100644
--- a/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
+++ b/tensorflow/python/kernel_tests/broadcast_to_ops_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.platform import test as test_lib
 
 class BroadcastToTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBroadcastToBasic(self):
     for dtype in [np.uint8, np.uint16, np.int8, np.int16, np.int32, np.int64]:
       with self.session(use_gpu=True):
@@ -37,6 +38,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
         v_np = np.broadcast_to(x, [3, 3])
         self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
   def testBroadcastToString(self):
     with self.session(use_gpu=True):
       x = np.array([b"1", b"2", b"3"])
@@ -44,6 +46,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
       v_np = np.broadcast_to(x, [3, 3])
       self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
   def testBroadcastToBool(self):
     with self.session(use_gpu=True):
       x = np.array([True, False, True], dtype=np.bool)
@@ -51,6 +54,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
       v_np = np.broadcast_to(x, [3, 3])
       self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
   def testBroadcastToShape(self):
     for input_dim in range(1, 6):
       for output_dim in range(input_dim, 6):
@@ -62,6 +66,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
           v_np = np.broadcast_to(x, output_shape)
           self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
   def testBroadcastToScalar(self):
     with self.session(use_gpu=True):
       x = np.array(1, dtype=np.int32)
@@ -69,6 +74,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
       v_np = np.broadcast_to(x, [3, 3])
       self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
   def testBroadcastScalarToNonScalar(self):
     with self.session(use_gpu=True):
       x = np.array(1.0, dtype=np.float)
@@ -76,6 +82,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
       v_np = np.broadcast_to(x, [2, 3, 4])
       self.assertAllEqual(v_tf.eval(), v_np)
 
+  @test_util.run_deprecated_v1
   def testBroadcastToShapeTypeAndInference(self):
     for dtype in [dtypes.int32, dtypes.int64]:
       with self.cached_session(use_gpu=True):
@@ -89,6 +96,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
         # check shape inference when shape input is constant
         self.assertAllEqual(shape, v_np.shape)
 
+  @test_util.run_deprecated_v1
   def testGradientForScalar(self):
     x = constant_op.constant(1, dtype=dtypes.float32)
     v = array_ops.broadcast_to(x, [2, 4, 3])
@@ -98,6 +106,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
                                                     out.get_shape())
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientWithSameRank(self):
     x = constant_op.constant(np.reshape(np.arange(6), (2, 1, 3)),
                              dtype=dtypes.float32)
@@ -108,6 +117,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
                                                     out, out.get_shape())
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientWithIncreasingRank(self):
     x = constant_op.constant([[1], [2]],
                              dtype=dtypes.float32)
@@ -118,6 +128,7 @@ class BroadcastToTest(test_util.TensorFlowTestCase):
                                                     out, out.get_shape())
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientWithBroadcastAllDimensions(self):
     x = constant_op.constant([[1, 2, 3], [4, 5, 6]], dtype=dtypes.float32)
     v = array_ops.broadcast_to(x, [5, 4, 6])
diff --git a/tensorflow/python/kernel_tests/bucketize_op_test.py b/tensorflow/python/kernel_tests/bucketize_op_test.py
index f40ca82527..95df694370 100644
--- a/tensorflow/python/kernel_tests/bucketize_op_test.py
+++ b/tensorflow/python/kernel_tests/bucketize_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -50,6 +51,7 @@ class BucketizationOpTest(test.TestCase):
     with self.session(use_gpu=True) as sess:
       self.assertAllEqual(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def testInvalidBoundariesOrder(self):
     op = math_ops._bucketize(
         constant_op.constant([-5, 0]), boundaries=[0, 8, 3, 11])
diff --git a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
index 031accee55..fa6eb5c968 100644
--- a/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
+++ b/tensorflow/python/kernel_tests/candidate_sampler_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import candidate_sampling_ops
 from tensorflow.python.ops import math_ops
@@ -37,6 +38,7 @@ class RangeSamplerOpsTest(test.TestCase):
 
   TRUE_LABELS = [[1, 2], [0, 4], [3, 3]]
 
+  @test_util.run_deprecated_v1
   def testTrueCandidates(self):
     with self.cached_session() as sess:
       indices = constant_op.constant([0, 0, 1, 1, 2, 2])
@@ -106,6 +108,7 @@ class RangeSamplerOpsTest(test.TestCase):
       self.assertTrue(id_ in self.TRUE_LABELS[index])
       self.assertLess(weight, -1.0e37)
 
+  @test_util.run_deprecated_v1
   def testSeed(self):
 
     def draw(seed):
diff --git a/tensorflow/python/kernel_tests/cast_op_test.py b/tensorflow/python/kernel_tests/cast_op_test.py
index 2cfe084d95..b3187e1637 100644
--- a/tensorflow/python/kernel_tests/cast_op_test.py
+++ b/tensorflow/python/kernel_tests/cast_op_test.py
@@ -25,6 +25,7 @@ import platform
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
@@ -90,10 +91,12 @@ class CastOpTest(test.TestCase):
     if x.dtype == np.float32 or x.dtype == np.float64:
       self._testTypes(x, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     self._testAll(np.arange(-10, 10).reshape(2, 10))
     self._testAll(np.linspace(-10, 10, 17))
 
+  @test_util.run_deprecated_v1
   def testSmallValues(self):
     f4 = np.finfo(np.float32)
     f8 = np.finfo(np.float64)
@@ -112,6 +115,7 @@ class CastOpTest(test.TestCase):
       b = math_ops.cast(math_ops.cast(a, dtypes.bfloat16), dtypes.float32)
       self.assertAllClose(a, self.evaluate(b), rtol=1 / 128.)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     self._testAll(np.random.normal(0, 10, 210).reshape([2, 3, 5, 7]))
     self._testAll(np.random.normal(0, 1e6, 210).reshape([2, 3, 5, 7]))
@@ -124,6 +128,7 @@ class CastOpTest(test.TestCase):
         self._cast(
             x, dst_dtype, use_gpu=use_gpu), dst_dtype(expected))
 
+  @test_util.run_deprecated_v1
   def testIntToFloatBoundary(self):
     i4 = np.iinfo(np.int32)
     i8 = np.iinfo(np.int64)
@@ -138,6 +143,7 @@ class CastOpTest(test.TestCase):
     self._compare(i8.max, np.float64, i8.max, False)
     # NOTE: GPU does not support int32/int64 for casting.
 
+  @test_util.run_deprecated_v1
   def testInfNan(self):
     i4 = np.iinfo(np.int32)
     i8 = np.iinfo(np.int64)
@@ -181,6 +187,7 @@ class CastOpTest(test.TestCase):
   def testNotImplemented(self):
     self._OpError(np.arange(0, 10), dtypes.string, "Cast.*int64.*string.*")
 
+  @test_util.run_deprecated_v1
   def testCastToTypeOfVariable(self):
     with self.cached_session() as sess:
       x = variables.Variable(5, dtype=dtypes.float32)
@@ -189,6 +196,7 @@ class CastOpTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(1.0, self.evaluate(cast))
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     t = [dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128]
     for src_t in t:
@@ -203,6 +211,7 @@ class CastOpTest(test.TestCase):
 
 class SparseTensorCastTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCast(self):
     indices = constant_op.constant([[0], [1], [2]], dtypes.int64)
     values = constant_op.constant(np.array([1, 2, 3], np.int64))
diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py
index 15124a19a2..95bac85027 100644
--- a/tensorflow/python/kernel_tests/check_ops_test.py
+++ b/tensorflow/python/kernel_tests/check_ops_test.py
@@ -173,6 +173,7 @@ class AssertEqualTest(test.TestCase):
       assert x is None
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_greater(self):
     # Static check
     static_small = constant_op.constant([1, 2], name="small")
@@ -180,6 +181,7 @@ class AssertEqualTest(test.TestCase):
     with self.assertRaisesRegexp(errors.InvalidArgumentError, "fail"):
       check_ops.assert_equal(static_big, static_small, message="fail")
 
+  @test_util.run_deprecated_v1
   def test_raises_when_greater_dynamic(self):
     with self.cached_session():
       small = array_ops.placeholder(dtypes.int32, name="small")
@@ -251,6 +253,7 @@ First 2 elements of y:
                                summarize=2)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_less(self):
     # Static check
     static_small = constant_op.constant([3, 1], name="small")
@@ -258,6 +261,7 @@ First 2 elements of y:
     with self.assertRaisesRegexp(errors.InvalidArgumentError, "fail"):
       check_ops.assert_equal(static_big, static_small, message="fail")
 
+  @test_util.run_deprecated_v1
   def test_raises_when_less_dynamic(self):
     with self.cached_session():
       small = array_ops.placeholder(dtypes.int32, name="small")
@@ -317,6 +321,7 @@ class AssertNoneEqualTest(test.TestCase):
     self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_equal(self):
     small = constant_op.constant([3, 1], name="small")
     with self.assertRaisesOpError("x != y did not hold"):
@@ -506,6 +511,7 @@ class AssertAllCloseTest(test.TestCase):
 class AssertLessTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_equal(self):
     small = constant_op.constant([1, 2], name="small")
     with self.assertRaisesOpError("failure message.*\n*.* x < y did not hold"):
@@ -516,6 +522,7 @@ class AssertLessTest(test.TestCase):
       self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_greater(self):
     small = constant_op.constant([1, 2], name="small")
     big = constant_op.constant([3, 4], name="big")
@@ -582,6 +589,7 @@ class AssertLessEqualTest(test.TestCase):
     self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_greater(self):
     small = constant_op.constant([1, 2], name="small")
     big = constant_op.constant([3, 4], name="big")
@@ -637,6 +645,7 @@ class AssertLessEqualTest(test.TestCase):
 class AssertGreaterTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_equal(self):
     small = constant_op.constant([1, 2], name="small")
     with self.assertRaisesOpError("fail"):
@@ -647,6 +656,7 @@ class AssertGreaterTest(test.TestCase):
       self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_less(self):
     small = constant_op.constant([1, 2], name="small")
     big = constant_op.constant([3, 4], name="big")
@@ -706,6 +716,7 @@ class AssertGreaterEqualTest(test.TestCase):
     self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_less(self):
     small = constant_op.constant([1, 2], name="small")
     big = constant_op.constant([3, 4], name="big")
@@ -770,6 +781,7 @@ class AssertNegativeTest(test.TestCase):
     self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_positive(self):
     doug = constant_op.constant([1, 2], name="doug")
     with self.assertRaisesOpError("fail"):
@@ -780,6 +792,7 @@ class AssertNegativeTest(test.TestCase):
       self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_zero(self):
     claire = constant_op.constant([0], name="claire")
     with self.assertRaisesOpError("x < 0 did not hold"):
@@ -802,6 +815,7 @@ class AssertNegativeTest(test.TestCase):
 class AssertPositiveTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_negative(self):
     freddie = constant_op.constant([-1, -2], name="freddie")
     with self.assertRaisesOpError("fail"):
@@ -819,6 +833,7 @@ class AssertPositiveTest(test.TestCase):
     self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_zero(self):
     meechum = constant_op.constant([0], name="meechum")
     with self.assertRaisesOpError("x > 0 did not hold"):
@@ -841,26 +856,31 @@ class AssertPositiveTest(test.TestCase):
 class EnsureShapeTest(test.TestCase):
 
   # Static shape inference
+  @test_util.run_deprecated_v1
   def testStaticShape(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     ensure_shape_op = check_ops.ensure_shape(placeholder, (3, 3, 3))
     self.assertEqual(ensure_shape_op.get_shape(), (3, 3, 3))
 
+  @test_util.run_deprecated_v1
   def testStaticShape_MergesShapes(self):
     placeholder = array_ops.placeholder(dtypes.int32, shape=(None, None, 3))
     ensure_shape_op = check_ops.ensure_shape(placeholder, (5, 4, None))
     self.assertEqual(ensure_shape_op.get_shape(), (5, 4, 3))
 
+  @test_util.run_deprecated_v1
   def testStaticShape_RaisesErrorWhenRankIncompatible(self):
     placeholder = array_ops.placeholder(dtypes.int32, shape=(None, None, 3))
     with self.assertRaises(ValueError):
       check_ops.ensure_shape(placeholder, (2, 3))
 
+  @test_util.run_deprecated_v1
   def testStaticShape_RaisesErrorWhenDimIncompatible(self):
     placeholder = array_ops.placeholder(dtypes.int32, shape=(None, None, 3))
     with self.assertRaises(ValueError):
       check_ops.ensure_shape(placeholder, (2, 2, 4))
 
+  @test_util.run_deprecated_v1
   def testStaticShape_CanSetUnknownShape(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
@@ -868,6 +888,7 @@ class EnsureShapeTest(test.TestCase):
     self.assertEqual(ensure_shape_op.get_shape(), None)
 
   # Dynamic shape check
+  @test_util.run_deprecated_v1
   def testEnsuresDynamicShape_RaisesError(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = math_ops.divide(placeholder, 3, name="MyDivide")
@@ -880,6 +901,7 @@ class EnsureShapeTest(test.TestCase):
           r"expected shape \[3,3,3\]."):
         sess.run(derived, feed_dict={placeholder: feed_val})
 
+  @test_util.run_deprecated_v1
   def testEnsuresDynamicShape_RaisesErrorDimUnknown(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
@@ -892,6 +914,7 @@ class EnsureShapeTest(test.TestCase):
           r"expected shape \[\?,\?,3\]."):
         sess.run(derived, feed_dict={placeholder: feed_val})
 
+  @test_util.run_deprecated_v1
   def testEnsuresDynamicShape(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
@@ -900,6 +923,7 @@ class EnsureShapeTest(test.TestCase):
     with self.cached_session() as sess:
       sess.run(derived, feed_dict={placeholder: feed_val})
 
+  @test_util.run_deprecated_v1
   def testEnsuresDynamicShape_WithUnknownDims(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
@@ -908,6 +932,7 @@ class EnsureShapeTest(test.TestCase):
     with self.cached_session() as sess:
       sess.run(derived, feed_dict={placeholder: feed_val})
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     placeholder = array_ops.placeholder(dtypes.float32)
     derived = check_ops.ensure_shape(placeholder, (None, None))
@@ -1003,6 +1028,7 @@ class AssertRankTest(test.TestCase):
               tensor, desired_rank, message="fail")]):
         self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_zero_tensor_raises_if_rank_too_small_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1021,6 +1047,7 @@ class AssertRankTest(test.TestCase):
         [check_ops.assert_rank(tensor, desired_rank)]):
       self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_zero_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1038,6 +1065,7 @@ class AssertRankTest(test.TestCase):
           [check_ops.assert_rank(tensor, desired_rank)]):
         self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_raises_if_rank_too_large_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1055,6 +1083,7 @@ class AssertRankTest(test.TestCase):
         [check_ops.assert_rank(tensor, desired_rank)]):
       self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1072,6 +1101,7 @@ class AssertRankTest(test.TestCase):
           [check_ops.assert_rank(tensor, desired_rank)]):
         self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_raises_if_rank_too_small_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1087,6 +1117,7 @@ class AssertRankTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "Rank must be a scalar"):
       check_ops.assert_rank(tensor, np.array([], dtype=np.int32))
 
+  @test_util.run_deprecated_v1
   def test_raises_if_rank_is_not_scalar_dynamic(self):
     with self.cached_session():
       tensor = constant_op.constant(
@@ -1104,6 +1135,7 @@ class AssertRankTest(test.TestCase):
                                  "must be of type <dtype: 'int32'>"):
       check_ops.assert_rank(tensor, .5)
 
+  @test_util.run_deprecated_v1
   def test_raises_if_rank_is_not_integer_dynamic(self):
     with self.cached_session():
       tensor = constant_op.constant(
@@ -1127,6 +1159,7 @@ class AssertRankInTest(test.TestCase):
           check_ops.assert_rank_in(tensor_rank0, (1, 2), message="fail")]):
         self.evaluate(array_ops.identity(tensor_rank0))
 
+  @test_util.run_deprecated_v1
   def test_rank_zero_tensor_raises_if_rank_mismatch_dynamic_rank(self):
     with self.cached_session():
       tensor_rank0 = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1143,6 +1176,7 @@ class AssertRankInTest(test.TestCase):
           check_ops.assert_rank_in(tensor_rank0, desired_ranks)]):
         self.evaluate(array_ops.identity(tensor_rank0))
 
+  @test_util.run_deprecated_v1
   def test_rank_zero_tensor_doesnt_raise_if_rank_matches_dynamic_rank(self):
     with self.cached_session():
       tensor_rank0 = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1159,6 +1193,7 @@ class AssertRankInTest(test.TestCase):
           check_ops.assert_rank_in(tensor_rank1, desired_ranks)]):
         self.evaluate(array_ops.identity(tensor_rank1))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_doesnt_raise_if_rank_matches_dynamic_rank(self):
     with self.cached_session():
       tensor_rank1 = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1177,6 +1212,7 @@ class AssertRankInTest(test.TestCase):
           check_ops.assert_rank_in(tensor_rank1, (0, 2))]):
         self.evaluate(array_ops.identity(tensor_rank1))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_raises_if_rank_mismatches_dynamic_rank(self):
     with self.cached_session():
       tensor_rank1 = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1196,6 +1232,7 @@ class AssertRankInTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "Rank must be a scalar"):
       check_ops.assert_rank_in(tensor, desired_ranks)
 
+  @test_util.run_deprecated_v1
   def test_raises_if_rank_is_not_scalar_dynamic(self):
     with self.cached_session():
       tensor = constant_op.constant(
@@ -1218,6 +1255,7 @@ class AssertRankInTest(test.TestCase):
                                  "must be of type <dtype: 'int32'>"):
       check_ops.assert_rank_in(tensor, (1, .5,))
 
+  @test_util.run_deprecated_v1
   def test_raises_if_rank_is_not_integer_dynamic(self):
     with self.cached_session():
       tensor = constant_op.constant(
@@ -1241,6 +1279,7 @@ class AssertRankAtLeastTest(test.TestCase):
           [check_ops.assert_rank_at_least(tensor, desired_rank)]):
         self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_zero_tensor_raises_if_rank_too_small_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1258,6 +1297,7 @@ class AssertRankAtLeastTest(test.TestCase):
         [check_ops.assert_rank_at_least(tensor, desired_rank)]):
       self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_zero_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1274,6 +1314,7 @@ class AssertRankAtLeastTest(test.TestCase):
         [check_ops.assert_rank_at_least(tensor, desired_rank)]):
       self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_ten_doesnt_raise_if_rank_too_large_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1290,6 +1331,7 @@ class AssertRankAtLeastTest(test.TestCase):
         [check_ops.assert_rank_at_least(tensor, desired_rank)]):
       self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_doesnt_raise_if_rank_just_right_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1307,6 +1349,7 @@ class AssertRankAtLeastTest(test.TestCase):
           [check_ops.assert_rank_at_least(tensor, desired_rank)]):
         self.evaluate(array_ops.identity(tensor))
 
+  @test_util.run_deprecated_v1
   def test_rank_one_tensor_raises_if_rank_too_small_dynamic_rank(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtypes.float32, name="my_tensor")
@@ -1320,6 +1363,7 @@ class AssertRankAtLeastTest(test.TestCase):
 class AssertNonNegativeTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_negative(self):
     zoe = constant_op.constant([-1, -2], name="zoe")
     with self.assertRaisesOpError("x >= 0 did not hold"):
@@ -1356,6 +1400,7 @@ class AssertNonPositiveTest(test.TestCase):
     self.evaluate(out)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_raises_when_positive(self):
     rachel = constant_op.constant([0, 2], name="rachel")
     with self.assertRaisesOpError("x <= 0 did not hold"):
diff --git a/tensorflow/python/kernel_tests/checkpoint_ops_test.py b/tensorflow/python/kernel_tests/checkpoint_ops_test.py
index 213ac292d3..b8c8c9edb5 100644
--- a/tensorflow/python/kernel_tests/checkpoint_ops_test.py
+++ b/tensorflow/python/kernel_tests/checkpoint_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_checkpoint_ops
 from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
@@ -48,6 +49,7 @@ class GenerateVocabRemappingTest(test.TestCase):
     with open(self.old_vocab_file, 'w') as f:
       f.write('\n'.join(['knitting', 'eminem', 'MISSING']) + '\n')
 
+  @test_util.run_deprecated_v1
   def test_generate_remapping_with_no_vocab_changes(self):
     """Tests where vocab does not change at all."""
     remapping, num_present = gen_checkpoint_ops.generate_vocab_remapping(
@@ -368,6 +370,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
           np.reshape(initializing_values, (new_rows, num_cols)),
           self.evaluate(remapped_matrix))
 
+  @test_util.run_deprecated_v1
   def test_loading_rows_divisible_by_max_rows(self):
     """Tests loading normal var when rows are evenly divisible by max_rows."""
     self._test_loading_variable_with_max_rows(
@@ -376,6 +379,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         # 9 is evenly divisible by 3.
         max_rows_in_memory=3)
 
+  @test_util.run_deprecated_v1
   def test_loading_rows_not_divisible_by_max_rows(self):
     """Tests loading normal var when rows aren't divisible by max_rows."""
     self._test_loading_variable_with_max_rows(
@@ -384,6 +388,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         # 9 is not evenly divisible by 4.
         max_rows_in_memory=4)
 
+  @test_util.run_deprecated_v1
   def test_loading_rows_less_than_max_rows(self):
     """Tests loading normal var as a single slice.
 
@@ -395,6 +400,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         # 10 > 9.
         max_rows_in_memory=10)
 
+  @test_util.run_deprecated_v1
   def test_loading_no_max_rows(self):
     """Tests loading normal var as a single slice with no valid max_rows."""
     self._test_loading_variable_with_max_rows(
@@ -402,6 +408,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         partitioner=None,
         max_rows_in_memory=-1)
 
+  @test_util.run_deprecated_v1
   def test_loading_partitions_equals_max_rows(self):
     """Tests loading partitioned var sliced on partition boundary."""
     self._test_loading_variable_with_max_rows(
@@ -411,6 +418,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         # exactly 3 rows.
         max_rows_in_memory=3)
 
+  @test_util.run_deprecated_v1
   def test_loading_partitions_greater_than_max_rows(self):
     """Tests loading partitioned var with more slices than partitions."""
     self._test_loading_variable_with_max_rows(
@@ -420,6 +428,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         # row at a time.
         max_rows_in_memory=1)
 
+  @test_util.run_deprecated_v1
   def test_loading_partitions_less_than_max_rows(self):
     """Tests loading partitioned var as a single slice.
 
@@ -430,6 +439,7 @@ class LoadAndRemapMatrixWithMaxRowsTest(test.TestCase):
         partitioner=partitioned_variables.fixed_size_partitioner(3),
         max_rows_in_memory=10)
 
+  @test_util.run_deprecated_v1
   def test_loading_partitions_no_max_rows(self):
     """Tests loading partitioned var as single slice with no valid max_rows."""
     self._test_loading_variable_with_max_rows(
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index 1a509a43d1..f3947236b1 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_linalg_ops
@@ -145,6 +146,7 @@ class CholeskyOpTest(test.TestCase):
       matrices[i] = np.dot(matrices[i].T.conj(), matrices[i])
     self._verifyCholesky(matrices)
 
+  @test_util.run_deprecated_v1
   def testNonSquareMatrix(self):
     with self.assertRaises(ValueError):
       linalg_ops.cholesky(np.array([[1., 2., 3.], [3., 4., 5.]]))
@@ -175,6 +177,7 @@ class CholeskyOpTest(test.TestCase):
     self._verifyCholesky(np.empty([0, 2, 2]))
     self._verifyCholesky(np.empty([2, 0, 0]))
 
+  @test_util.run_deprecated_v1
   def testConcurrentExecutesWithoutError(self):
     with self.session(use_gpu=True) as sess:
       matrix1 = random_ops.random_normal([5, 5], seed=42)
@@ -193,18 +196,21 @@ class CholeskyGradTest(test.TestCase):
   def getShapes(self, shapeList):
     return ((elem, int(np.floor(1.2 * elem))) for elem in shapeList)
 
+  @test_util.run_deprecated_v1
   def testSmallMatrices(self):
     np.random.seed(0)
     shapes = self.getShapes([1, 2, 10])
     self.runFiniteDifferences(
         shapes, dtypes=(dtypes_lib.float32, dtypes_lib.float64))
 
+  @test_util.run_deprecated_v1
   def testSmallMatricesComplex(self):
     np.random.seed(0)
     shapes = self.getShapes([1, 2, 10])
     self.runFiniteDifferences(
         shapes, dtypes=(dtypes_lib.complex64, dtypes_lib.complex128))
 
+  @test_util.run_deprecated_v1
   def testOneBlockMatrices(self):
     np.random.seed(0)
     shapes = self.getShapes([self._backprop_block_size + 1])
@@ -213,12 +219,14 @@ class CholeskyGradTest(test.TestCase):
         dtypes=(dtypes_lib.float32, dtypes_lib.float64),
         scalarTest=True)
 
+  @test_util.run_deprecated_v1
   def testTwoBlockMatrixFloat(self):
     np.random.seed(0)
     shapes = self.getShapes([2 * self._backprop_block_size + 1])
     self.runFiniteDifferences(
         shapes, dtypes=(dtypes_lib.float32,), scalarTest=True)
 
+  @test_util.run_deprecated_v1
   def testTwoBlockMatrixDouble(self):
     np.random.seed(0)
     shapes = self.getShapes([2 * self._backprop_block_size + 1])
@@ -231,6 +239,7 @@ class CholeskyGradTest(test.TestCase):
     self.runFiniteDifferences(
         shapes, dtypes=(dtypes_lib.complex64,), scalarTest=True)
 
+  @test_util.run_deprecated_v1
   def testTwoBlockMatrixComplexDouble(self):
     np.random.seed(0)
     shapes = self.getShapes([2 * self._backprop_block_size + 1])
diff --git a/tensorflow/python/kernel_tests/clip_ops_test.py b/tensorflow/python/kernel_tests/clip_ops_test.py
index 5f1b6b6917..45f1e6152a 100644
--- a/tensorflow/python/kernel_tests/clip_ops_test.py
+++ b/tensorflow/python/kernel_tests/clip_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import gradient_checker
@@ -166,6 +167,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans, tf_ans)
     self.assertAllClose(np_ans, tf_ans_tensor)
 
+  @test_util.run_deprecated_v1
   def testClipByNormGradientZeros(self):
     with self.session(use_gpu=True):
       x = array_ops.zeros([3])
@@ -242,6 +244,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans, tf_ans)
 
   # ClipByGlobalNorm tests
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormClipped(self):
     # Norm clipping when clip_norm < 5
     with self.session(use_gpu=True):
@@ -263,6 +266,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans_0, tf_ans_1)
     self.assertAllClose(np_ans_1, tf_ans_2)
 
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormClippedTensor(self):
     # Norm clipping when clip_norm < 5
     with self.session(use_gpu=True):
@@ -284,6 +288,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans_0, tf_ans_1)
     self.assertAllClose(np_ans_1, tf_ans_2)
 
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormSupportsNone(self):
     # Norm clipping when clip_norm < 5
     with self.session(use_gpu=True):
@@ -307,6 +312,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans_0, tf_ans_1)
     self.assertAllClose(np_ans_1, tf_ans_2)
 
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormWithIndexedSlicesClipped(self):
     # Norm clipping when clip_norm < 5
     with self.session(use_gpu=True):
@@ -340,6 +346,7 @@ class ClipTest(test.TestCase):
     self.assertEqual(dense_shape, slices.dense_shape)
     self.assertEqual(dense_shape, modified_slices.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormNotClipped(self):
     # No norm clipping when clip_norm >= 5
     with self.session(use_gpu=True):
@@ -359,6 +366,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans_0, tf_ans_1)
     self.assertAllClose(np_ans_1, tf_ans_2)
 
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormZero(self):
     # No norm clipping when norm = 0
     with self.session(use_gpu=True):
@@ -378,6 +386,7 @@ class ClipTest(test.TestCase):
     self.assertAllClose(np_ans_0, tf_ans_1)
     self.assertAllClose(np_ans_1, tf_ans_2)
 
+  @test_util.run_deprecated_v1
   def testClipByGlobalNormInf(self):
     with self.session(use_gpu=True):
       x0 = constant_op.constant([-2.0, 0.0, np.inf, 4.0, 0.0, 0.0],
@@ -456,6 +465,7 @@ class ClipTest(test.TestCase):
       clip_by_norm_ans = self.evaluate(without_norm)
       self.assertAllClose(clip_by_average_norm_ans, clip_by_norm_ans)
 
+  @test_util.run_deprecated_v1
   def testClipByValueEmptyTensor(self):
     # Test case for GitHub issue 19337
     zero = array_ops.placeholder(dtype=dtypes.float32, shape=None)
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 27137f76bd..474760a93f 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.platform import test
 
 class ConcatOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testHStack(self):
     with self.session(use_gpu=True):
       p1 = array_ops.placeholder(dtypes.float32, shape=[4, 4])
@@ -50,6 +51,7 @@ class ConcatOpTest(test.TestCase):
     self.assertAllEqual(result[:4, :], params[p1])
     self.assertAllEqual(result[4:, :], params[p2])
 
+  @test_util.run_deprecated_v1
   def testVStack(self):
     with self.session(use_gpu=True):
       p1 = array_ops.placeholder(dtypes.float32, shape=[4, 4])
@@ -138,6 +140,7 @@ class ConcatOpTest(test.TestCase):
       else:
         self.assertAllClose(result[ind], params[p[i]], 0.01)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     self._testRandom(dtypes.bool)
     self._testRandom(dtypes.float32)
@@ -148,6 +151,7 @@ class ConcatOpTest(test.TestCase):
     self._testRandom(dtypes.complex64)
     self._testRandom(dtypes.complex128)
 
+  @test_util.run_deprecated_v1
   def testInvalidConcatDimTypeAndShape(self):
     a = variables.Variable(constant_op.constant(1.0, shape=[1]))
     b = variables.Variable(constant_op.constant(2.0, shape=[1]))
@@ -199,10 +203,12 @@ class ConcatOpTest(test.TestCase):
         result = self.evaluate(concated_grad)
     self.assertAllEqual(result, grad_inp)
 
+  @test_util.run_deprecated_v1
   def testGradientsSimple(self):
     self._testGradientsSimple(dtypes.float32)
     self._testGradientsSimple(dtypes.complex64)
 
+  @test_util.run_deprecated_v1
   def testGradientsFirstDim(self):
     with test_util.use_gpu():
       inp = []
@@ -227,6 +233,7 @@ class ConcatOpTest(test.TestCase):
 
     self.assertAllEqual(result, grad_inp)
 
+  @test_util.run_deprecated_v1
   def testGradientsLastDim(self):
     # Test both positive and negative concat axis.
     # -1 and 2 correspond to the same axis for 3-dimensional tensors.
@@ -284,10 +291,12 @@ class ConcatOpTest(test.TestCase):
 
     self.assertAllEqual(result, grad_inp)
 
+  @test_util.run_deprecated_v1
   def testGradientsRandom(self):
     for _ in range(5):
       self._RunAndVerifyGradientsRandom()
 
+  @test_util.run_deprecated_v1
   def testGradientWithUnknownInputDim(self):
     with self.session(use_gpu=True):
       x = array_ops.placeholder(dtypes.float32)
@@ -309,6 +318,7 @@ class ConcatOpTest(test.TestCase):
 
       self.assertAllEqual(result, grad_inp)
 
+  @test_util.run_deprecated_v1
   def testShapeError(self):
     # Rank doesn't match.
     with self.assertRaises(ValueError):
@@ -338,6 +348,7 @@ class ConcatOpTest(test.TestCase):
            constant_op.constant(20.0, shape=[4, 4, 4])
           ], -4)
 
+  @test_util.run_deprecated_v1
   def testShapeWithUnknownConcatDim(self):
     p1 = array_ops.placeholder(dtypes.float32)
     c1 = constant_op.constant(10.0, shape=[4, 4, 4, 4])
@@ -356,6 +367,7 @@ class ConcatOpTest(test.TestCase):
     with self.assertRaises(ValueError):
       array_ops.concat([p1, c1, p2, c3], dim)
 
+  @test_util.run_deprecated_v1
   def testZeroSize(self):
     # Verify that concat doesn't crash and burn for zero size inputs
     np.random.seed(7)
@@ -377,6 +389,7 @@ class ConcatOpTest(test.TestCase):
               dxs = self.evaluate(gradients_impl.gradients(c, xs, dc))
               self.assertAllEqual(dc, np.concatenate(dxs, axis=axis))
 
+  @test_util.run_deprecated_v1
   def testTensorConcatDim0Grad(self):
     x_shapes = [[20, 7, 3], [10, 7, 3], [14, 7, 3]]
     output_shape = [44, 7, 3]
@@ -391,6 +404,7 @@ class ConcatOpTest(test.TestCase):
                                                     output_shape)
     self.assertLess(err, 1e-11)
 
+  @test_util.run_deprecated_v1
   def testTensorConcatDim1Grad(self):
     x_shapes = [[20, 7, 3], [20, 3, 3], [20, 1, 3]]
     output_shape = [20, 11, 3]
@@ -405,6 +419,7 @@ class ConcatOpTest(test.TestCase):
                                                     output_shape)
     self.assertLess(err, 1e-11)
 
+  @test_util.run_deprecated_v1
   def testIndexedSlicesConcatDim0Grad(self):
     x_shapes = [[20, 7, 3], [10, 7, 3], [14, 7, 3]]
     output_shape = [4, 7, 3]
@@ -420,6 +435,7 @@ class ConcatOpTest(test.TestCase):
                                                     output_shape)
     self.assertLess(err, 1e-11)
 
+  @test_util.run_deprecated_v1
   def testIndexedSlicesConcatDim1Grad(self):
     x_shapes = [[20, 7, 3], [20, 3, 3], [20, 1, 3]]
     output_shape = [4, 11, 3]
@@ -435,6 +451,7 @@ class ConcatOpTest(test.TestCase):
                                                     output_shape)
     self.assertLess(err, 1e-11)
 
+  @test_util.run_deprecated_v1
   def testIndexedSlicesConcatDim2Grad(self):
     x_shapes = [[20, 7, 3], [20, 7, 1], [20, 7, 2]]
     output_shape = [4, 7, 6]
@@ -450,6 +467,7 @@ class ConcatOpTest(test.TestCase):
                                                     output_shape)
     self.assertLess(err, 1e-11)
 
+  @test_util.run_deprecated_v1
   def testIndexedSlicesConcatDim1Grad_UnknownInputDim(self):
     x_shapes = [[20, 7, 3], [20, 3, 3], [20, 1, 3]]
     output_shape = [4, 11, 3]
@@ -479,6 +497,7 @@ class ConcatOpTest(test.TestCase):
     self.assertAllEqual(
         self.evaluate(concat_list_t), self.evaluate(concat_tuple_t))
 
+  @test_util.run_deprecated_v1
   def testConcatNoScalars(self):
     scalar = constant_op.constant(7)
     dim = array_ops.placeholder(dtypes.int32)
@@ -488,6 +507,7 @@ class ConcatOpTest(test.TestCase):
 
   # important as gpu implementation could fail if
   # shared memory is not large for all the inputs
+  @test_util.run_deprecated_v1
   def testConcatLargeNumberOfTensors(self):
     with self.session(use_gpu=True):
       for concat_dim in range(2):
@@ -529,6 +549,7 @@ class ConcatOpTest(test.TestCase):
       output = gen_array_ops.concat_v2([t1, t2], 0)
       self.assertFalse(self.evaluate(output))  # Checks that output is empty
 
+  @test_util.run_deprecated_v1
   def testConcatInvalidAxis(self):
     with self.assertRaises(ValueError):
       with test_util.use_gpu():
@@ -578,6 +599,7 @@ class ConcatOpTest(test.TestCase):
       result = concated_grad.eval(feed_dict=feed_dict)
       self.assertAllEqual(result, grad_inp)
 
+  @test_util.run_deprecated_v1
   def testGradientsNegativeAxis(self):
     x1 = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
     x2 = [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]
@@ -630,6 +652,7 @@ class ConcatOffsetTest(test.TestCase):
       ans = self.evaluate(off)
       self.assertAllEqual(ans, [[0, 0, 0], [0, 3, 0], [0, 10, 0]])
 
+  @test_util.run_deprecated_v1
   def testNotVector(self):
     cdim = constant_op.constant(1, dtypes.int32)
     s0 = constant_op.constant([[2, 3, 5]], dtypes.int32)
@@ -639,6 +662,7 @@ class ConcatOffsetTest(test.TestCase):
                                  r"should be a vector"):
       self.evaluate(off)
 
+  @test_util.run_deprecated_v1
   def testConcatDimOutOfRange(self):
     cdim = constant_op.constant(4, dtypes.int32)
     s0 = constant_op.constant([2, 3, 5], dtypes.int32)
@@ -648,6 +672,7 @@ class ConcatOffsetTest(test.TestCase):
                                  r"Concat dim is out of range: 4 vs. 3"):
       self.evaluate(off)
 
+  @test_util.run_deprecated_v1
   def testDimMismatch(self):
     cdim = constant_op.constant(1, dtypes.int32)
     s0 = constant_op.constant([2, 3, 5], dtypes.int32)
@@ -657,6 +682,7 @@ class ConcatOffsetTest(test.TestCase):
                                  r"should contain 3 elem"):
       self.evaluate(off)
 
+  @test_util.run_deprecated_v1
   def testSizeMismatch(self):
     cdim = constant_op.constant(1, dtypes.int32)
     s0 = constant_op.constant([2, 3, 5], dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 4baa8b0212..1f4b37ce2a 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -68,6 +68,7 @@ class CondV2Test(test.TestCase):
       self.assertEqual(expected_val, actual_val)
       self.assertEqual(expected_grad_val, actual_grad_val)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
@@ -82,6 +83,7 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [x, y])
     self._testCond(true_fn, false_fn, [y])
 
+  @test_util.run_deprecated_v1
   def testMultipleOutputs(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(3.0, name="y")
@@ -96,6 +98,7 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [x, y])
     self._testCond(true_fn, false_fn, [y])
 
+  @test_util.run_deprecated_v1
   def testBasic2(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
@@ -110,6 +113,7 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [x, y])
     self._testCond(true_fn, false_fn, [y])
 
+  @test_util.run_deprecated_v1
   def testNoInputs(self):
     with self.cached_session() as sess:
       pred = array_ops.placeholder(dtypes.bool, name="pred")
@@ -540,6 +544,7 @@ class CondV2Test(test.TestCase):
               pred_inner: False
           }), [5., 0.])
 
+  @test_util.run_deprecated_v1
   def testSecondDerivative(self):
     with self.cached_session() as sess:
       pred = array_ops.placeholder(dtypes.bool, name="pred")
@@ -636,6 +641,7 @@ class CondV2Test(test.TestCase):
         self.assertFalse(if_found,
                          "An `If` op was found, but it should be lowered.")
 
+  @test_util.run_deprecated_v1
   def testLoweringDisabledInXLA(self):
     with self.session(graph=ops.Graph()) as sess:
       # Build the cond_v2 in an XLA context
@@ -668,6 +674,7 @@ class CondV2Test(test.TestCase):
           if_found,
           "An `If` op was not found, but the graph should not be lowered.")
 
+  @test_util.run_deprecated_v1
   def testLoweringDisabledWithSingleThreadedExecutorContext(self):
     with self.session(graph=ops.Graph()) as sess:
       @function.defun
@@ -702,6 +709,7 @@ class CondV2Test(test.TestCase):
     self.assertEqual(self.evaluate(output[1]), 9.)
 
   @test_util.enable_control_flow_v2
+  @test_util.run_deprecated_v1
   def testRaisesOutputStructuresMismatch(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(3.0, name="y")
diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
index 7ee1a4bc32..5847e4639b 100644
--- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -79,11 +80,13 @@ class ConditionalAccumulatorTest(test.TestCase):
       attr { key: 'reduction_type' value {s: 'MEAN'} }
       """, q.accumulator_ref.op.node_def)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSizeEmpty(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(dtypes_lib.float32, name="Q")
       self.assertEqual(q.num_accumulated().eval(), 0)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSetGlobalStep(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -91,6 +94,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       set_global_step_op = q.set_global_step(1)
       set_global_step_op.run()
 
+  @test_util.run_deprecated_v1
   def testAccumulatorApplyGradFloat32(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -98,6 +102,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       accum_op = q.apply_grad((10.0,))
       accum_op.run()
 
+  @test_util.run_deprecated_v1
   def testDtypes(self):
     with self.cached_session() as sess:
       dtypes = [dtypes_lib.float16, dtypes_lib.float32, dtypes_lib.float64]
@@ -115,6 +120,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
         self.assertEqual(sum(elems) / len(elems), result)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorMultipleAccumulators(self):
     with self.cached_session():
       q_f32_0 = data_flow_ops.ConditionalAccumulator(
@@ -134,6 +140,7 @@ class ConditionalAccumulatorTest(test.TestCase):
         result = accums[i].take_grad(1).eval()
         self.assertEqual(result, i + 10.0)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorApplyAndTakeGradWithShape(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -155,6 +162,7 @@ class ConditionalAccumulatorTest(test.TestCase):
           is_all_equal &= (val[i][j] == elems_ave[i][j])
       self.assertTrue(is_all_equal)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorApplyGradWithWrongShape(self):
     q = data_flow_ops.ConditionalAccumulator(
         dtypes_lib.float32, name="Q", shape=(3, 2))
@@ -165,6 +173,7 @@ class ConditionalAccumulatorTest(test.TestCase):
     with self.assertRaises(ValueError):
       q.apply_grad([[1.0], [2.0], [3.0]])
 
+  @test_util.run_deprecated_v1
   def testAccumulatorDynamicShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -190,6 +199,7 @@ class ConditionalAccumulatorTest(test.TestCase):
           is_all_equal &= (val[i][j] == elems_ave[i][j])
       self.assertTrue(is_all_equal)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorWrongDynamicShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -208,6 +218,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       with self.assertRaises(errors_impl.InvalidArgumentError):
         sess.run(accum_op, feed_dict={x: [[1.0], [2.0], [3.0]]})
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSizeAfterApplyGrad(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -219,6 +230,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
       self.assertEqual(q.num_accumulated().eval(), 2)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSizeAfterApplyGradAndTakeGrad(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -247,6 +259,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       extract_t.op.run()
       self.assertEqual(q.num_accumulated().eval(), 0)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorTakeGradMean(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -271,6 +284,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       val = self.evaluate(takeg_t)
       self.assertEqual(15.0, val)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorTakeGradSum(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -298,6 +312,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       val = self.evaluate(takeg_t)
       self.assertEqual(30.0, val)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorTakeGradInvalidReductionType(self):
     with self.assertRaises(ValueError):
       data_flow_ops.ConditionalAccumulator(
@@ -306,6 +321,7 @@ class ConditionalAccumulatorTest(test.TestCase):
           shape=tensor_shape.TensorShape([1]),
           reduction_type="Invalid")
 
+  @test_util.run_deprecated_v1
   def testAccumulatorInvalidTakeGrad(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -321,6 +337,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       with self.assertRaises(errors_impl.InvalidArgumentError):
         self.evaluate(takeg_t)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorRepeatedTakeGradMean(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -348,6 +365,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       val = self.evaluate(takeg_t)
       self.assertEqual(elems_ave + 0.0, val)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorRepeatedTakeGradSum(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -378,6 +396,7 @@ class ConditionalAccumulatorTest(test.TestCase):
       val = self.evaluate(takeg_t)
       self.assertEqual(elems_sum, val)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorIncrementGlobalStep(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -394,6 +413,7 @@ class ConditionalAccumulatorTest(test.TestCase):
         set_global_step_op.run()
         self.evaluate(inc_global_step)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSetGlobalStepPreventsAccumulation(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -415,6 +435,7 @@ class ConditionalAccumulatorTest(test.TestCase):
                                    if x >= ls) / sum(1 for x in local_steps
                                                      if x >= ls), val)
 
+  @test_util.run_deprecated_v1
   def testParallelApplyGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -440,6 +461,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
       self.assertEqual(val, sum(elems) / len(elems))
 
+  @test_util.run_deprecated_v1
   def testParallelTakeGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -472,6 +494,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
       self.assertItemsEqual(elems, results)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorApplyAndBlockingTake(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -505,6 +528,7 @@ class ConditionalAccumulatorTest(test.TestCase):
     with self.assertRaisesOpError("was cancelled"):
       self.evaluate(takeg_op)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py
index b001341c03..ae13c8e32e 100644
--- a/tensorflow/python/kernel_tests/confusion_matrix_test.py
+++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py
@@ -71,9 +71,11 @@ class ConfusionMatrixTest(test.TestCase):
 
     self._testConfMatrix(labels=labels, predictions=predictions, truth=truth)
 
+  @test_util.run_deprecated_v1
   def testInt32Basic(self):
     self._testBasic(dtype=np.int32)
 
+  @test_util.run_deprecated_v1
   def testInt64Basic(self):
     self._testBasic(dtype=np.int64)
 
@@ -111,9 +113,11 @@ class ConfusionMatrixTest(test.TestCase):
       self.assertEqual(cm_out.dtype, np_dtype)
       self.assertAllClose(cm_out, truth, atol=1e-10)
 
+  @test_util.run_deprecated_v1
   def testOnTensors_int32(self):
     self._testConfMatrixOnTensors(dtypes.int32, np.int32)
 
+  @test_util.run_deprecated_v1
   def testOnTensors_int64(self):
     self._testConfMatrixOnTensors(dtypes.int64, np.int64)
 
@@ -133,9 +137,11 @@ class ConfusionMatrixTest(test.TestCase):
 
     self._testConfMatrix(labels=labels, predictions=predictions, truth=truth)
 
+  @test_util.run_deprecated_v1
   def testInt32DifferentLabels(self, dtype=np.int32):
     self._testDifferentLabelsInPredictionAndTarget(dtype)
 
+  @test_util.run_deprecated_v1
   def testInt64DifferentLabels(self, dtype=np.int64):
     self._testDifferentLabelsInPredictionAndTarget(dtype)
 
@@ -155,12 +161,15 @@ class ConfusionMatrixTest(test.TestCase):
 
     self._testConfMatrix(labels=labels, predictions=predictions, truth=truth)
 
+  @test_util.run_deprecated_v1
   def testInt32MultipleLabels(self, dtype=np.int32):
     self._testMultipleLabels(dtype)
 
+  @test_util.run_deprecated_v1
   def testInt64MultipleLabels(self, dtype=np.int64):
     self._testMultipleLabels(dtype)
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     labels = np.arange(5, dtype=np.int32)
     predictions = np.arange(5, dtype=np.int32)
@@ -177,6 +186,7 @@ class ConfusionMatrixTest(test.TestCase):
     self._testConfMatrix(
         labels=labels, predictions=predictions, weights=weights, truth=truth)
 
+  @test_util.run_deprecated_v1
   def testLabelsTooLarge(self):
     labels = np.asarray([1, 1, 0, 3, 5], dtype=np.int32)
     predictions = np.asarray([2, 1, 0, 2, 2], dtype=np.int32)
@@ -191,6 +201,7 @@ class ConfusionMatrixTest(test.TestCase):
       self._testConfMatrix(
           labels=labels, predictions=predictions, num_classes=3, truth=None)
 
+  @test_util.run_deprecated_v1
   def testPredictionsTooLarge(self):
     labels = np.asarray([1, 1, 0, 2, 2], dtype=np.int32)
     predictions = np.asarray([2, 1, 0, 3, 5], dtype=np.int32)
@@ -205,6 +216,7 @@ class ConfusionMatrixTest(test.TestCase):
       self._testConfMatrix(
           labels=labels, predictions=predictions, num_classes=3, truth=None)
 
+  @test_util.run_deprecated_v1
   def testInvalidRank_predictionsTooBig(self):
     labels = np.asarray([1, 2, 3])
     predictions = np.asarray([[1, 2, 3]])
@@ -212,6 +224,7 @@ class ConfusionMatrixTest(test.TestCase):
                             confusion_matrix.confusion_matrix, predictions,
                             labels)
 
+  @test_util.run_deprecated_v1
   def testInvalidRank_predictionsTooSmall(self):
     labels = np.asarray([[1, 2, 3]])
     predictions = np.asarray([1, 2, 3])
@@ -219,6 +232,7 @@ class ConfusionMatrixTest(test.TestCase):
                             confusion_matrix.confusion_matrix, predictions,
                             labels)
 
+  @test_util.run_deprecated_v1
   def testInputDifferentSize(self):
     labels = np.asarray([1, 2])
     predictions = np.asarray([1, 2, 3])
@@ -247,6 +261,7 @@ class ConfusionMatrixTest(test.TestCase):
 
 class RemoveSqueezableDimensionsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBothScalarShape(self):
     label_values = 1.0
     prediction_values = 0.0
@@ -272,6 +287,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testSameShape(self):
     label_values = np.ones(shape=(2, 3, 1))
     prediction_values = np.zeros_like(label_values)
@@ -297,6 +313,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testSameShapeExpectedRankDiff0(self):
     label_values = np.ones(shape=(2, 3, 1))
     prediction_values = np.zeros_like(label_values)
@@ -322,6 +339,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testSqueezableLabels(self):
     label_values = np.ones(shape=(2, 3, 1))
     prediction_values = np.zeros(shape=(2, 3))
@@ -348,6 +366,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testSqueezableLabelsExpectedRankDiffPlus1(self):
     label_values = np.ones(shape=(2, 3, 1))
     prediction_values = np.zeros(shape=(2, 3, 5))
@@ -374,6 +393,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testSqueezablePredictions(self):
     label_values = np.ones(shape=(2, 3))
     prediction_values = np.zeros(shape=(2, 3, 1))
@@ -402,6 +422,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
           expected_prediction_values,
           dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testSqueezablePredictionsExpectedRankDiffMinus1(self):
     label_values = np.ones(shape=(2, 3, 5))
     prediction_values = np.zeros(shape=(2, 3, 1))
@@ -430,6 +451,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
           expected_prediction_values,
           dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testUnsqueezableLabels(self):
     label_values = np.ones(shape=(2, 3, 2))
     prediction_values = np.zeros(shape=(2, 3))
@@ -455,6 +477,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testUnsqueezablePredictions(self):
     label_values = np.ones(shape=(2, 3))
     prediction_values = np.zeros(shape=(2, 3, 2))
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 9c3c96bd31..583082c2aa 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -70,6 +70,7 @@ class ConstantTest(test.TestCase):
     with self.assertRaises(TypeError):
       constant_op.constant(dtypes_lib.string, "[,]")
 
+  @test_util.run_deprecated_v1
   def testBFloat16(self):
     bfloat16 = dtypes_lib.bfloat16.as_numpy_dtype
     self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(bfloat16))
@@ -77,36 +78,42 @@ class ConstantTest(test.TestCase):
         np.random.normal(size=30).reshape([2, 3, 5]).astype(bfloat16))
     self._testAll(np.empty((2, 0, 5)).astype(bfloat16))
 
+  @test_util.run_deprecated_v1
   def testHalf(self):
     self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float16))
     self._testAll(
         np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float16))
     self._testAll(np.empty((2, 0, 5)).astype(np.float16))
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32))
     self._testAll(
         np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float32))
     self._testAll(np.empty((2, 0, 5)).astype(np.float32))
 
+  @test_util.run_deprecated_v1
   def testDouble(self):
     self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float64))
     self._testAll(
         np.random.normal(size=30).reshape([2, 3, 5]).astype(np.float64))
     self._testAll(np.empty((2, 0, 5)).astype(np.float64))
 
+  @test_util.run_deprecated_v1
   def testInt32(self):
     self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.int32))
     self._testAll((100 * np.random.normal(size=30)).reshape([2, 3, 5]).astype(
         np.int32))
     self._testAll(np.empty((2, 0, 5)).astype(np.int32))
 
+  @test_util.run_deprecated_v1
   def testInt64(self):
     self._testAll(np.arange(-15, 15).reshape([2, 3, 5]).astype(np.int64))
     self._testAll((100 * np.random.normal(size=30)).reshape([2, 3, 5]).astype(
         np.int64))
     self._testAll(np.empty((2, 0, 5)).astype(np.int64))
 
+  @test_util.run_deprecated_v1
   def testComplex64(self):
     self._testAll(
         np.complex(1, 2) *
@@ -116,6 +123,7 @@ class ConstantTest(test.TestCase):
         np.random.normal(size=30).reshape([2, 3, 5]).astype(np.complex64))
     self._testAll(np.empty((2, 0, 5)).astype(np.complex64))
 
+  @test_util.run_deprecated_v1
   def testComplex128(self):
     self._testAll(
         np.complex(1, 2) *
@@ -125,12 +133,14 @@ class ConstantTest(test.TestCase):
         np.random.normal(size=30).reshape([2, 3, 5]).astype(np.complex128))
     self._testAll(np.empty((2, 0, 5)).astype(np.complex128))
 
+  @test_util.run_deprecated_v1
   def testString(self):
     self._testCpu(
         np.array([compat.as_bytes(str(x)) for x in np.arange(-15, 15)]).reshape(
             [2, 3, 5]))
     self._testCpu(np.empty((2, 0, 5)).astype(np.str_))
 
+  @test_util.run_deprecated_v1
   def testVariant(self):
     # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant
     # copying between CPU and GPU is supported.
@@ -161,6 +171,7 @@ class ConstantTest(test.TestCase):
           message="Variant storing an int, decoded const value:").op
       logging_const_op.run()
 
+  @test_util.run_deprecated_v1
   def testStringWithNulls(self):
     with self.cached_session():
       val = ops.convert_to_tensor(b"\0\0\0\0").eval()
@@ -265,6 +276,7 @@ class ConstantTest(test.TestCase):
                                    "GraphDef cannot be larger than 2GB."):
         g.as_graph_def()
 
+  @test_util.run_deprecated_v1
   def testSparseValuesRaiseErrors(self):
     with self.assertRaisesRegexp(ValueError,
                                  "setting an array element with a sequence"):
@@ -342,6 +354,7 @@ class AsTensorTest(test.TestCase):
       ops.convert_to_tensor(
           tensor_shape.TensorShape([1, 2, 3]), dtype=dtypes_lib.float32)
 
+  @test_util.run_deprecated_v1
   def testAsTensorForDimensionInput(self):
     with self.cached_session():
       x = ops.convert_to_tensor(tensor_shape.TensorShape([1, 2, 3])[1])
@@ -409,6 +422,7 @@ class ZerosTest(test.TestCase):
     self.assertShapeEqual(np_ans, d)
     self.assertShapeEqual(np_ans, z)
 
+  @test_util.run_deprecated_v1
   def testDtype(self):
     with self.cached_session():
       d = array_ops.fill([2, 3], 12., name="fill")
@@ -477,6 +491,7 @@ class ZerosLikeTest(test.TestCase):
       self.assertFalse(np.any(z_value))
       self.assertEqual((2, 3), z_value.shape)
 
+  @test_util.run_deprecated_v1
   def testZerosLikeCPU(self):
     for dtype in [
         dtypes_lib.half, dtypes_lib.float32, dtypes_lib.float64,
@@ -487,6 +502,7 @@ class ZerosLikeTest(test.TestCase):
       self._compareZeros(dtype, fully_defined_shape=False, use_gpu=False)
       self._compareZeros(dtype, fully_defined_shape=True, use_gpu=False)
 
+  @test_util.run_deprecated_v1
   def testZerosLikeGPU(self):
     for dtype in [
         dtypes_lib.half, dtypes_lib.float32, dtypes_lib.float64,
@@ -496,11 +512,13 @@ class ZerosLikeTest(test.TestCase):
       self._compareZeros(dtype, fully_defined_shape=False, use_gpu=True)
       self._compareZeros(dtype, fully_defined_shape=True, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testZerosLikePartialShape(self):
     d = array_ops.placeholder(dtypes_lib.float32, shape=[None, 4, None])
     z = array_ops.zeros_like(d)
     self.assertEqual(d.get_shape().as_list(), z.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testZerosLikeDtype(self):
     # Make sure zeros_like works even for dtypes that cannot be cast between
     with self.cached_session():
@@ -514,6 +532,7 @@ class ZerosLikeTest(test.TestCase):
           self.assertEqual(y.shape, shape)
           self.assertAllEqual(y, np.zeros(shape, dtype=out_type))
 
+  @test_util.run_deprecated_v1
   def testZerosLikeVariant(self):
     # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant
     # copying between CPU and GPU is supported AND we register a
@@ -574,6 +593,7 @@ class OnesTest(test.TestCase):
     self.assertShapeEqual(np_ans, d)
     self.assertShapeEqual(np_ans, z)
 
+  @test_util.run_deprecated_v1
   def testAutoPack(self):
     with self.cached_session():
       h = array_ops.placeholder(dtypes_lib.int32, shape=[])
@@ -582,6 +602,7 @@ class OnesTest(test.TestCase):
       out = z.eval(feed_dict={h: 4, w: 16})
     self.assertAllEqual(out, np.array([[1] * 16] * 4))
 
+  @test_util.run_deprecated_v1
   def testDtype(self):
     with self.cached_session():
       d = array_ops.fill([2, 3], 12., name="fill")
@@ -635,6 +656,7 @@ class OnesLikeTest(test.TestCase):
       self.assertTrue(np.array_equal(z_value, np.array([[1] * 3] * 2)))
       self.assertEqual([2, 3], z_var.get_shape())
 
+  @test_util.run_deprecated_v1
   def testOnesLikePartialShape(self):
     d = array_ops.placeholder(dtypes_lib.float32, shape=[None, 4, None])
     z = array_ops.ones_like(d)
@@ -679,12 +701,14 @@ class FillTest(test.TestCase):
     np_ans = np.array([[0.15 + 0.3j] * 3] * 2).astype(np.complex128)
     self._compareAll([2, 3], np_ans[0][0], np_ans)
 
+  @test_util.run_deprecated_v1
   def testFillString(self):
     np_ans = np.array([[b"yolo"] * 3] * 2)
     with self.session(use_gpu=False):
       tf_ans = array_ops.fill([2, 3], np_ans[0][0], name="fill").eval()
     self.assertAllEqual(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testFillNegative(self):
     with self.cached_session():
       for shape in (-1,), (2, -1), (-1, 2), (-2), (-3):
@@ -698,6 +722,7 @@ class FillTest(test.TestCase):
         with self.assertRaises(errors_impl.InvalidArgumentError):
           fill_t.eval({dims: shape})
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionEdgeCases(self):
     # Non-vector dimensions.
     with self.assertRaises(ValueError):
@@ -716,6 +741,7 @@ class FillTest(test.TestCase):
             dtypes_lib.int32, shape=()), 17], 1.0)
     self.assertEqual([None, 17], f.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session():
       in_v = constant_op.constant(5.0)
@@ -728,6 +754,7 @@ class FillTest(test.TestCase):
 
 class PlaceholderTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testDtype(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.float32, shape=(10, 10), name="p")
@@ -740,6 +767,7 @@ class PlaceholderTest(test.TestCase):
           "must feed a value for placeholder tensor 'p' with dtype float"):
         self.evaluate(p_identity)
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.float32, shape=(10, 10), name="p")
@@ -757,6 +785,7 @@ class PlaceholderTest(test.TestCase):
           ValueError, lambda e: "Cannot feed value of shape" in str(e)):
         p_identity.eval(feed_dict={p: feed_array[:5, :5]})
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.float32, shape=None, name="p")
@@ -769,12 +798,14 @@ class PlaceholderTest(test.TestCase):
       self.assertAllClose(
           p_identity.eval(feed_dict={p: feed_array}), feed_array)
 
+  @test_util.run_deprecated_v1
   def testScalarShape(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.float32, shape=[], name="p")
       p_identity = array_ops.identity(p)
       self.assertAllClose(p_identity.eval(feed_dict={p: 5}), 5)
 
+  @test_util.run_deprecated_v1
   def testPartialShape(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.float32, shape=[None, 3], name="p")
@@ -787,6 +818,7 @@ class PlaceholderTest(test.TestCase):
           ValueError, lambda e: "Cannot feed value of shape" in str(e)):
         p_identity.eval(feed_dict={p: feed_array[:5, :2]})
 
+  @test_util.run_deprecated_v1
   def testPartialShapeWhenNotFed(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.float32, shape=[None, 3], name="p")
@@ -797,6 +829,7 @@ class PlaceholderTest(test.TestCase):
           "must feed a value for placeholder tensor 'p' with dtype float"):
         self.evaluate(p_identity)
 
+  @test_util.run_deprecated_v1
   def testControlDependency(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes_lib.int32, shape=[], name="p")
@@ -806,10 +839,12 @@ class PlaceholderTest(test.TestCase):
       val = np.array(2).astype(np.int)
       self.assertEqual(10, d.eval(feed_dict={p: val}))
 
+  @test_util.run_deprecated_v1
   def testBadShape(self):
     with self.assertRaises(ValueError):
       array_ops.placeholder(dtypes_lib.float32, shape=(-1, 10))
 
+  @test_util.run_deprecated_v1
   def testTensorStr(self):
     a = array_ops.placeholder(dtypes_lib.float32, shape=None, name="a")
     self.assertEqual("<tf.Tensor 'a:0' shape=<unknown> dtype=float32>", repr(a))
@@ -825,6 +860,7 @@ class PlaceholderTest(test.TestCase):
       self.assertEqual(
           "<tf.Tensor 'c:0' shape=(32, ?, 2) dtype=qint32>", repr(c))
 
+  @test_util.run_deprecated_v1
   def testOldGraph(self):
     # Load graph generated from earlier version of TF where
     # placeholder shape was not set.
@@ -904,6 +940,7 @@ versions {
 
 class PlaceholderWithDefaultTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testFullShape(self):
     with self.session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([[2, 2], [2, 2]], shape=[2, 2])
@@ -915,6 +952,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
       with self.assertRaises(ValueError):
         a.eval(feed_dict={p: [[6, 6, 6], [6, 6, 6]]})
 
+  @test_util.run_deprecated_v1
   def testPartialShape(self):
     with self.session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([1, 2, 3], shape=[None])
@@ -925,6 +963,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
       with self.assertRaises(ValueError):
         a.eval(feed_dict={p: [[2, 2], [2, 2]]})
 
+  @test_util.run_deprecated_v1
   def testNoShape(self):
     with self.session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([17], shape=None)
@@ -934,6 +973,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
       self.assertAllEqual(
           [[3, 3], [3, 3]], a.eval(feed_dict={p: [[3, 3], [3, 3]]}))
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.session(force_gpu=test_util.is_gpu_available()):
       x = array_ops.placeholder(dtypes_lib.float32, [5, 7])
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 685d0438af..f1efc5ce59 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -141,6 +141,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(9, self.evaluate(v2))
 
+  @test_util.run_deprecated_v1
   def testRefEnter(self):
     with self.cached_session():
       v = variables.VariableV1(7)
@@ -154,6 +155,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(9, self.evaluate(v3))
 
+  @test_util.run_deprecated_v1
   def testRefSwitch(self):
     with self.cached_session():
       v = variables.VariableV1(7)
@@ -176,6 +178,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(exit_op)
     self.assertAllEqual(np.array([x * 5 for x in [1, 2, 3, 4, 5, 6]]), result)
 
+  @test_util.run_deprecated_v1
   def testEnterShapePropagation(self):
     with self.cached_session():
       v = variables.Variable([0.0, 0.0], dtype=dtypes.float32)
@@ -272,6 +275,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(exit_n)
     self.assertAllEqual(10, result)
 
+  @test_util.run_deprecated_v1
   def testLoop_1(self):
     with self.cached_session():
       zero = constant_op.constant(0)
@@ -333,6 +337,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaisesOpError("has inputs from different frames"):
         res.eval(feed_dict={data: 1.0})
 
+  @test_util.run_deprecated_v1
   def testCondBool(self):
     values = constant_op.constant(10)
     fn1 = lambda: math_ops.add(values, 1)
@@ -340,6 +345,7 @@ class ControlFlowTest(test.TestCase):
     with self.assertRaisesRegexp(TypeError, "must not be a Python bool"):
       _ = control_flow_ops.cond(False, fn1, fn2)
 
+  @test_util.run_deprecated_v1
   def testCondInt(self):
     p = array_ops.placeholder(dtypes.bool, shape=[])
     v = constant_op.constant(10)
@@ -627,6 +633,7 @@ class ControlFlowTest(test.TestCase):
       test_result = self.evaluate(r)
       self.assertDictEqual({"a": 210, "b": 210}, test_result)
 
+  @test_util.run_deprecated_v1
   def testEmbeddedListOutput(self):
     with self.cached_session() as sess:
       x = constant_op.constant(10)
@@ -678,6 +685,7 @@ class ControlFlowTest(test.TestCase):
         r = control_flow_ops.cond(pred, fn1, fn2)
         self.evaluate(r)
 
+  @test_util.run_deprecated_v1
   def testCondRef(self):
 
     with self.cached_session():
@@ -693,6 +701,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual([2.0], self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/79881896 (control deps)")
+  @test_util.run_deprecated_v1
   def testCondWithControl(self):
     with self.cached_session():
       control_holder = array_ops.placeholder(dtypes.float32, shape=())
@@ -773,6 +782,7 @@ class ControlFlowTest(test.TestCase):
       grad = gradients_impl.gradients(r, [x])[0]
       self.assertAllEqual(1.0, self.evaluate(grad))
 
+  @test_util.run_deprecated_v1
   def testCondGrad_2(self):
     with self.cached_session():
       c = array_ops.placeholder(dtypes.int32, shape=[])
@@ -788,6 +798,7 @@ class ControlFlowTest(test.TestCase):
 
   @test_util.disable_control_flow_v2(
       "b/110550782 (gradient w.r.t external variable)")
+  @test_util.run_deprecated_v1
   def testCondGrad_3(self):
     with self.cached_session():
       c = array_ops.placeholder(dtypes.int32, shape=[])
@@ -805,6 +816,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(980.0, r.eval(feed_dict={c: 1}))
       self.assertAllEqual(30.0, r.eval(feed_dict={c: 3}))
 
+  @test_util.run_deprecated_v1
   def testCondGradMultiDevice(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 2},
                                     allow_soft_placement=True)
@@ -849,6 +861,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(1.0, self.evaluate(result))
 
   @test_util.disable_control_flow_v2("b/113327884")
+  @test_util.run_deprecated_v1
   def testCondGrad_Gather(self):
     with self.cached_session() as sess:
       v1 = variables.Variable([1.0, 42.0])
@@ -1030,6 +1043,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(v.eval(), 1.0)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_deprecated_v1
   def testWhileWithRefs_1(self):
     with self.cached_session() as sess:
       x = variables.VariableV1(0)._ref()  # pylint: disable=protected-access
@@ -1269,6 +1283,7 @@ class ControlFlowTest(test.TestCase):
 
   # Have more than 10 parallel iterations and hence exercise k-bound
   # most of the time.
+  @test_util.run_deprecated_v1
   def testWhile_3(self):
     with self.cached_session():
 
@@ -1289,6 +1304,7 @@ class ControlFlowTest(test.TestCase):
       result = r[3].eval()
     self.assertAllEqual(10100, result)
 
+  @test_util.run_deprecated_v1
   def testWhile_4(self):
     with self.cached_session():
 
@@ -1402,6 +1418,7 @@ class ControlFlowTest(test.TestCase):
       r = r[1] * array_ops.ones([8, 8])
       self.assertAllEqual(np.ones((8, 8)), self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testWhileWithNonTensorInput_Scalar(self):
     with self.cached_session():
       n = 0
@@ -1476,6 +1493,7 @@ class ControlFlowTest(test.TestCase):
             [i.get_shape(), tensor_shape.TensorShape([5])])
 
   @test_util.disable_control_flow_v2("b/116282023 (IndexedSlices)")
+  @test_util.run_deprecated_v1
   def testWhileShapeInferenceIndexedSlices(self):
     with self.cached_session():
       values = constant_op.constant([[2.0, 4.0], [3.0, 5.0]], name="values")
@@ -1582,6 +1600,7 @@ class ControlFlowTest(test.TestCase):
           condition, body, [n, r], parallel_iterations=1)
       self.assertAllEqual(12, res[1].eval())
 
+  @test_util.run_deprecated_v1
   def testWhileWithControl_2(self):
     with self.cached_session():
       r = constant_op.constant(0)
@@ -1736,6 +1755,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual([2.0], sess.run(r1, {p: False}))
 
   @test_util.disable_control_flow_v2("b/116743589")
+  @test_util.run_deprecated_v1
   def testCondWhile_3(self):
     self._testCondWhile_3(use_gpu=False)
     self._testCondWhile_3(use_gpu=True)
@@ -1779,6 +1799,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [n])
       self.assertAllEqual(10, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testWhileCondGradMultiDevice(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 2},
                                     allow_soft_placement=True)
@@ -1810,6 +1831,7 @@ class ControlFlowTest(test.TestCase):
 
   # NOTE: It is ok to have parallel_iterations > 1
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_deprecated_v1
   def testWhileUpdateVariable_1(self):
     with self.cached_session():
       select = variables.Variable([3.0, 4.0, 5.0])
@@ -1860,6 +1882,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(np.array([10.0, 10.0, 10.0]), result2)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_deprecated_v1
   def testWhileUpdateVariable_3(self):
     with self.cached_session():
       select = variables.Variable([3.0, 4.0, 5.0])
@@ -1882,6 +1905,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllClose(np.array([10.0, 10.0, 10.0]), result)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_deprecated_v1
   def testWhileUpdateVariable_4(self):
     with self.cached_session():
       var_a = variables.Variable(0, name="a")
@@ -1910,6 +1934,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(10, self.evaluate(var_b))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_deprecated_v1
   def testWhileUpdateVariable_5(self):
     with self.cached_session():
       # Create some variables.
@@ -2089,6 +2114,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual([None], r.get_shape().as_list())
       self.assertAllClose([810.0, 2560.0], r.eval(feed_dict={x: [3.0, 4.0]}))
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_BaseShape(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(dtypes.float32, [None])
@@ -2137,6 +2163,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(81.0, grad_v_val)
 
   @test_util.disable_control_flow_v2("b/116630618 (parallel_iters: times out)")
+  @test_util.run_deprecated_v1
   def testWhileGrad_Mul(self):
     self._testWhileGrad_Mul(use_gpu=False, p_iters=1)
     self._testWhileGrad_Mul(use_gpu=False, p_iters=10)
@@ -2166,9 +2193,11 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(512.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testNestedWhileCondWhileGrad(self):
     self._testNestedWhileCondWhileGrad(use_gpu=False)
 
+  @test_util.run_deprecated_v1
   def testNestedWhileCondWhileGradGpu(self):
     self._testNestedWhileCondWhileGrad(use_gpu=True)
 
@@ -2184,6 +2213,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllClose(216.0, r[0].eval())
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_ResourceVariable(self):
     with self.cached_session():
       a = resource_variable_ops.ResourceVariable(3.0)
@@ -2213,6 +2243,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(9.0, r.eval(feed_dict={x: 1.0}))
 
   @test_util.disable_control_flow_v2("b/116340060")
+  @test_util.run_deprecated_v1
   def testGradInWhileWrtInitialLoopVal(self):
     with self.cached_session():
       x = array_ops.placeholder(dtypes.float32, shape=(), name="x")
@@ -2330,6 +2361,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients([rx], y)
       self.assertAllClose(120.0, r[0].eval())
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_Dependency(self):
     with self.cached_session():
       i = constant_op.constant(0, name="i")
@@ -2381,6 +2413,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllClose(np.ones([2, 3]), sess.run(grad[0]))
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_Const(self):
     with self.cached_session() as sess:
       c0 = constant_op.constant(0.0, name="c0")
@@ -2475,6 +2508,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(8.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testNestedWhileGrad_Simple(self):
     self._testNestedWhileGrad_Simple(use_gpu=False)
     self._testNestedWhileGrad_Simple(use_gpu=True)
@@ -2502,6 +2536,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(256.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testNestedWhileGrad_ParallelInner(self):
     with self.cached_session():
       v = constant_op.constant(1.0)
@@ -2564,11 +2599,13 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(1024.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/117519152")
+  @test_util.run_deprecated_v1
   def testWhileCondGrad_Simple(self):
     self._testWhileCondGrad_Simple(use_gpu=False)
     self._testWhileCondGrad_Simple(use_gpu=True)
 
   @test_util.disable_control_flow_v2("b/117276490")
+  @test_util.run_deprecated_v1
   def testWhileCondGrad_UnknownShape(self):
     with self.cached_session() as sess:
       v = array_ops.placeholder(dtypes.float32)
@@ -2586,6 +2623,7 @@ class ControlFlowTest(test.TestCase):
       r = sess.run(r, feed_dict={v: 2.0})
       self.assertAllClose(1024.0, r)
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_Concat(self):
     with self.cached_session() as sess:
       x = variable_scope.get_variable("x", initializer=[[1., 2.]])
@@ -2661,6 +2699,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)")
+  @test_util.run_deprecated_v1
   def testWhileGrad_SparseTensor(self):
     with self.cached_session():
       values = constant_op.constant([2.0, 4.0], name="values")
@@ -2703,6 +2742,7 @@ class ControlFlowTest(test.TestCase):
           c, b, [i0, constant_op.constant(0.0)])
       self.assertAllClose(600.0, self.evaluate(output_grad)[1])
 
+  @test_util.run_deprecated_v1
   def testWhileAndTensorArray(self):
     with self.cached_session() as sess:
       param = constant_op.constant(2.0)
@@ -2722,6 +2762,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, param)[0]
       self.assertAllClose(107520.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_StopGrad(self):
     with self.cached_session():
       x = constant_op.constant(3.0, name="x")
@@ -2765,6 +2806,7 @@ class ControlFlowTest(test.TestCase):
           math_ops.add(array_ops.stop_gradient(rx), ry), y)[0]
       self.assertEqual(32.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_StopGradInside(self):
     with self.cached_session():
       x = constant_op.constant(3.0, name="x")
@@ -2784,6 +2826,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(rx, x)[0]
       self.assertAllClose(156.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testWhileGrad_StopGradInsideNoShape(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(dtypes.float32)
@@ -2808,6 +2851,7 @@ class ControlFlowTest(test.TestCase):
       self.assertFalse(any(name in op.name for op in all_ops))
 
   @test_util.disable_control_flow_v2("b/117954949")
+  @test_util.run_deprecated_v1
   def testWhileGradGradFail(self):
     theta = variables.Variable(initial_value=1.)
 
@@ -2822,6 +2866,7 @@ class ControlFlowTest(test.TestCase):
     grad_theta_stopped = array_ops.stop_gradient(grad_theta)
     gradients_impl.gradients(grad_theta_stopped, theta)
 
+  @test_util.run_deprecated_v1
   def testStopGradOnWhileGrad(self):
     with self.cached_session():
       x = constant_op.constant(2.0, name="x")
@@ -2839,6 +2884,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(388.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_deprecated_v1
   def testWhileGradientWithNontrainablePath1(self):
     q = variables.Variable([7., 8.])
 
@@ -2940,6 +2986,7 @@ class ControlFlowTest(test.TestCase):
       # False case: c = 0 is not >= 1
       self.assertEqual([2], i.eval(feed_dict={c: 0}))
 
+  @test_util.run_deprecated_v1
   def testExampleCond(self):
 
     with self.cached_session():
@@ -3191,6 +3238,7 @@ class ControlFlowTest(test.TestCase):
     self.assertEqual(op.type, "NoOp")
     self.assertEqual(op.control_inputs, [])
 
+  @test_util.run_deprecated_v1
   def testMergeShapes(self):
     # All inputs unknown.
     p1 = array_ops.placeholder(dtypes.float32)
@@ -3245,6 +3293,7 @@ class ControlFlowTest(test.TestCase):
     self.assertEqual([None, None], m.get_shape().as_list())
     self.assertEqual([], index.get_shape())
 
+  @test_util.run_deprecated_v1
   def testRefSelect(self):
     index = array_ops.placeholder(dtypes.int32)
 
@@ -3278,6 +3327,7 @@ class ControlFlowTest(test.TestCase):
     s = control_flow_ops.ref_select(index, [v1, v2])
     self.assertEqual(None, s.get_shape())
 
+  @test_util.run_deprecated_v1
   def testRunLoopTensor(self):
     with self.cached_session() as sess:
       tensor_list = []
@@ -3364,6 +3414,7 @@ class ControlFlowTest(test.TestCase):
       result = control_flow_ops.merge([v_f, v_t])
       self.evaluate(result)
 
+  @test_util.run_deprecated_v1
   def testQIntArgAndRet(self):
 
     @function.Defun(dtypes.qint8)
@@ -3411,6 +3462,7 @@ class ControlFlowContextCheckTest(test.TestCase):
         "is in a while loop. See info log for more details."):
       math_ops.add(1, while_tensor)
 
+  @test_util.run_deprecated_v1
   def testInvalidContextInCond(self):
     # Accessing a while loop tensor in cond is illegal.
     while_tensor = self._getWhileTensor()
@@ -3479,6 +3531,7 @@ class ControlFlowContextCheckTest(test.TestCase):
 
     control_flow_ops.while_loop(lambda i: i < 5, body, [0])
 
+  @test_util.run_deprecated_v1
   def testInvalidNestedContexts(self):
     # Accessing a tensor from a while context in a different while context, all
     # inside a cond context, is illegal.
@@ -3527,6 +3580,7 @@ class TupleTest(test.TestCase):
           self.assertAllClose([30.0], self.evaluate(t2))
           self.assertAllClose([1.0], self.evaluate(v1))
 
+  @test_util.run_deprecated_v1
   def testIndexedSlices(self):
     for v1_first in [True, False]:
       with self.cached_session():
@@ -3582,6 +3636,7 @@ class TupleTest(test.TestCase):
 
 class AssertTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testGuardedAssertDoesNotCopyWhenTrue(self):
     with self.session(use_gpu=True) as sess:
       with ops.device(test.gpu_device_name()):
diff --git a/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py b/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py
index af6ffc1d19..7b3b560b24 100644
--- a/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py
+++ b/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
@@ -31,6 +32,7 @@ from tensorflow.python.platform import test
 
 class Conv2DBackpropFilterGradTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session():
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/conv2d_transpose_test.py b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
index d9aa4ab967..c603c08630 100644
--- a/tensorflow/python/kernel_tests/conv2d_transpose_test.py
+++ b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
@@ -23,6 +23,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
@@ -154,6 +155,7 @@ class Conv2DTransposeTest(test.TestCase):
 
     self.assertAllClose(cache_values, value)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     x_shape = [2, 6, 4, 3]
     f_shape = [3, 3, 2, 3]
diff --git a/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py b/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py
index 89b64068ac..7e913febed 100644
--- a/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py
+++ b/tensorflow/python/kernel_tests/conv3d_backprop_filter_v2_grad_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
@@ -31,6 +32,7 @@ from tensorflow.python.platform import test
 
 class Conv3DBackpropFilterV2GradTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session():
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/conv3d_transpose_test.py b/tensorflow/python/kernel_tests/conv3d_transpose_test.py
index d4e7ec14da..22ba5b9037 100644
--- a/tensorflow/python/kernel_tests/conv3d_transpose_test.py
+++ b/tensorflow/python/kernel_tests/conv3d_transpose_test.py
@@ -23,6 +23,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -119,6 +120,7 @@ class Conv3DTransposeTest(test.TestCase):
                   target = 3.0
                 self.assertAllClose(target, value[n, d, h, w, k])
 
+  @test_util.run_deprecated_v1
   def testConv3DTransposeShapeMismatch(self):
     # Test case for GitHub issue 18460
     x_shape = [2, 2, 3, 4, 3]
@@ -201,6 +203,7 @@ class Conv3DTransposeTest(test.TestCase):
 
     self.assertAllClose(cache_values, value)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     x_shape = [2, 3, 4, 3, 2]
     f_shape = [3, 3, 3, 2, 2]
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index 3ec5c29df7..4a689b3fdf 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -462,6 +462,7 @@ class Conv3DTest(test.TestCase):
       self._ConstructAndTestGradientForConfig(data_format=data_format,
                                               use_gpu=use_gpu, **kwargs)
 
+  @test_util.run_deprecated_v1
   def testInputGradientValidPaddingStrideOne(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -473,6 +474,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientValidPaddingStrideOne(self):
     self.ConstructAndTestGradient(
         batch=4,
@@ -484,6 +486,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientValidPaddingStrideTwo(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -495,6 +498,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientValidPaddingStrideTwo(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -506,6 +510,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientValidPaddingStrideThree(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -517,6 +522,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientValidPaddingStrideThree(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -528,6 +534,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientSamePaddingStrideOne(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -539,6 +546,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientSamePaddingStrideOne(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -550,6 +558,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientSamePaddingStrideTwo(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -561,6 +570,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientSamePaddingStrideTwo(self):
     self.ConstructAndTestGradient(
         batch=4,
@@ -572,6 +582,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientSamePaddingStrideThree(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -583,6 +594,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientSamePaddingStrideThree(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -594,6 +606,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientSamePaddingDifferentStrides(self):
     self.ConstructAndTestGradient(
         batch=1,
@@ -605,6 +618,7 @@ class Conv3DTest(test.TestCase):
         padding="SAME",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientKernelSizeMatchesInputSize(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -616,6 +630,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=False)
 
+  @test_util.run_deprecated_v1
   def testInputGradientKernelSizeMatchesInputSize(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -640,6 +655,7 @@ class Conv3DTest(test.TestCase):
 
   # Test the fast path in gemm_pack_rhs/mkldnn_gemm_pack, when channel
   # dimension is a multiple of packet size.
+  @test_util.run_deprecated_v1
   def testInputGradientValidPaddingStrideOneFastPath(self):
     self.ConstructAndTestGradient(
         batch=2,
@@ -651,6 +667,7 @@ class Conv3DTest(test.TestCase):
         padding="VALID",
         test_input=True)
 
+  @test_util.run_deprecated_v1
   def testFilterGradientValidPaddingStrideOneFastPath(self):
     self.ConstructAndTestGradient(
         batch=2,
diff --git a/tensorflow/python/kernel_tests/cross_grad_test.py b/tensorflow/python/kernel_tests/cross_grad_test.py
index 0bd4006d6a..b397133fd7 100644
--- a/tensorflow/python/kernel_tests/cross_grad_test.py
+++ b/tensorflow/python/kernel_tests/cross_grad_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
@@ -26,6 +27,7 @@ from tensorflow.python.platform import test
 
 class CrossOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testGradientRandomValues(self):
     with self.cached_session():
       us = [2, 3]
diff --git a/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py b/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py
index d818fbd75c..0d86d13c71 100644
--- a/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py
+++ b/tensorflow/python/kernel_tests/ctc_decoder_ops_test.py
@@ -25,6 +25,7 @@ from six.moves import zip_longest
 
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ctc_ops
 from tensorflow.python.platform import test
@@ -94,6 +95,7 @@ class CTCGreedyDecoderTest(test.TestCase):
         with self.assertRaisesOpError(expected_err_re):
           sess.run(decoded_unwrapped + [log_probability])
 
+  @test_util.run_deprecated_v1
   def testCTCGreedyDecoder(self):
     """Test two batch entries - best path decoder."""
     max_time_steps = 6
@@ -170,6 +172,7 @@ class CTCGreedyDecoderTest(test.TestCase):
     self._testCTCDecoder(ctc_ops.ctc_greedy_decoder, inputs, seq_lens,
                          log_prob_truth, decode_truth)
 
+  @test_util.run_deprecated_v1
   def testCTCDecoderBeamSearch(self):
     """Test one batch, two beams - hibernating beam search."""
     # max_time_steps == 8
diff --git a/tensorflow/python/kernel_tests/ctc_loss_op_test.py b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
index 36cae2846c..e6b5835079 100644
--- a/tensorflow/python/kernel_tests/ctc_loss_op_test.py
+++ b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ctc_ops
 from tensorflow.python.ops import gradients_impl
@@ -105,6 +106,7 @@ class CTCLossTest(test.TestCase):
         with self.assertRaisesOpError(expected_err_re):
           self.evaluate([loss, grad])
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     """Test two batch entries."""
     # Input and ground truth from Alex Graves' implementation.
@@ -240,6 +242,7 @@ class CTCLossTest(test.TestCase):
 
     self._testCTCLoss(inputs, seq_lens, labels, loss_truth, grad_truth)
 
+  @test_util.run_deprecated_v1
   def test_time_major(self):
     """Testing time_major param.
 
@@ -269,6 +272,7 @@ class CTCLossTest(test.TestCase):
       (tf_loss, tf_loss_transposed) = self.evaluate([loss, loss_transposed])
       self.assertAllEqual(tf_loss, tf_loss_transposed)
 
+  @test_util.run_deprecated_v1
   def testInvalidSecondGradient(self):
     inputs = np.random.randn(2, 2, 3).astype(np.float32)
     inputs_t = constant_op.constant(inputs)
@@ -285,6 +289,7 @@ class CTCLossTest(test.TestCase):
                                    "explicitly disabled"):
         _ = gradients_impl._hessian_vector_product(loss, [inputs_t], v)
 
+  @test_util.run_deprecated_v1
   def testEmptyBatch(self):
     inputs = constant_op.constant([], dtype=dtypes.float32, shape=(1, 0, 2))
     sequence_lengths = constant_op.constant([], dtype=dtypes.int32)
@@ -301,6 +306,7 @@ class CTCLossTest(test.TestCase):
 
 class CTCLossTestV2(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCtcLossV2(self):
     random_seed.set_random_seed(5)
 
@@ -345,6 +351,7 @@ class CTCLossTestV2(test.TestCase):
             logit_length=logit_length,
             blank_index=0))
 
+  @test_util.run_deprecated_v1
   def testCtcLossDenseIsSameAsCtcLoss(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       random_seed.set_random_seed(5)
@@ -398,6 +405,7 @@ class CTCLossTestV2(test.TestCase):
               rtol=2e-06,
               atol=2e-06)
 
+  @test_util.run_deprecated_v1
   def testCtcLossDenseUniqueFastPathIsSameAsCtcLoss(self):
     random_seed.set_random_seed(5)
 
@@ -451,6 +459,7 @@ class CTCLossTestV2(test.TestCase):
             rtol=2e-06,
             atol=2e-06)
 
+  @test_util.run_deprecated_v1
   def testCtcLossDenseWithBlankIndexIsSameAsCtcLoss(self):
     random_seed.set_random_seed(5)
 
@@ -507,6 +516,7 @@ class CTCLossTestV2(test.TestCase):
             rtol=2e-06,
             atol=2e-06)
 
+  @test_util.run_deprecated_v1
   def testCtcLossDenseWithNegativeBlankIndexIsSameAsCtcLoss(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       random_seed.set_random_seed(5)
@@ -555,6 +565,7 @@ class CTCLossTestV2(test.TestCase):
               rtol=2e-06,
               atol=2e-06)
 
+  @test_util.run_deprecated_v1
   def testCollapseRepeated(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 3, 3, 3, 0],
@@ -568,6 +579,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 4, 0, 0],
          [4, 2, 9, 4]])
 
+  @test_util.run_deprecated_v1
   def testCollapseRepeatedPreservesDtypes(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=constant_op.constant(
@@ -585,6 +597,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 4, 0, 0],
          [4, 2, 9, 4]])
 
+  @test_util.run_deprecated_v1
   def testCollapseRepeatedExtraPadding(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 3, 3, 3, 0, 0, 0],
@@ -598,6 +611,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 4, 0, 0],
          [4, 2, 9, 4]])
 
+  @test_util.run_deprecated_v1
   def testCollapseRepeatedFrontRepeats(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 1, 1, 2, 2],
@@ -611,6 +625,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 2],
          [1, 0]])
 
+  @test_util.run_deprecated_v1
   def testCollapseRepeatedAllLabelsTheSame(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 1, 1, 1, 1],
@@ -643,6 +658,7 @@ class CTCLossTestV2(test.TestCase):
 
     self.assertAllEqual(padded_dense, new_dense)
 
+  @test_util.run_deprecated_v1
   def testUnique(self):
     labels = [
         [3, 4, 4, 3],
@@ -658,6 +674,7 @@ class CTCLossTestV2(test.TestCase):
         [0, 0, 0, 1],
     ], idx)
 
+  @test_util.run_deprecated_v1
   def testSumStates(self):
     idx = [
         [0, 1, 0, 1],
@@ -677,6 +694,7 @@ class CTCLossTestV2(test.TestCase):
          [1.8, 0.8, 0.0, 0.0]]
     ], sum_of_states)
 
+  @test_util.run_deprecated_v1
   def testStateToOlabel(self):
     labels = [
         [3, 4, 3, 4],
@@ -715,6 +733,7 @@ class CTCLossTestV2(test.TestCase):
          [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
     ])
 
+  @test_util.run_deprecated_v1
   def testStateToOlabelUnique(self):
     labels = [
         [3, 4, 3, 4],
@@ -753,6 +772,7 @@ class CTCLossTestV2(test.TestCase):
          [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
     ])
 
+  @test_util.run_deprecated_v1
   def testScan(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       out = ctc_ops._scan(
@@ -785,6 +805,7 @@ class CTCLossTestV2(test.TestCase):
           constant_op.constant([23.0, 24.0]))
       self.assertAllEqual([[23.0, 25.0], [25.0, 28.0], [29.0, 33.0]], out)
 
+  @test_util.run_deprecated_v1
   def testScanCapturesVariables(self):
     with self.cached_session() as sess:
       x = random_ops.random_uniform([])
@@ -794,6 +815,7 @@ class CTCLossTestV2(test.TestCase):
           [23.0 + x * 0.0, 23.0 + x * 1.0, 23.0 + x * 3.0], out
       ]))
 
+  @test_util.run_deprecated_v1
   def testScanMultipleAccumulators(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       def fn(accum, elem):
@@ -806,6 +828,7 @@ class CTCLossTestV2(test.TestCase):
       self.assertAllEqual([24.0, 26.0, 29.0], a)
       self.assertAllEqual([[1.0, 2.0], [2.0, 4.0], [6.0, 12.0]], b)
 
+  @test_util.run_deprecated_v1
   def testScanMultipleElements(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       def fn(accum, elem):
diff --git a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
index 272c2b1dac..49dbbb125a 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_binary_test.py
@@ -196,6 +196,7 @@ class BinaryOpTest(test.TestCase):
         self._compareGradientY(x, y, np_func, tf_func)
       self._compareGpu(x, y, np_func, tf_func)
 
+  @test_util.run_deprecated_v1
   def testFloatBasic(self):
     x = np.linspace(-5, 20, 15).reshape(1, 3, 5).astype(np.float32)
     y = np.linspace(20, -5, 15).reshape(1, 3, 5).astype(np.float32)
@@ -233,6 +234,7 @@ class BinaryOpTest(test.TestCase):
     except ImportError as e:
       tf_logging.warn("Cannot test special functions: %s" % str(e))
 
+  @test_util.run_deprecated_v1
   def testFloatDifferentShapes(self):
     x = np.array([1, 2, 3, 4]).reshape(2, 2).astype(np.float32)
     y = np.array([1, 2]).reshape(2, 1).astype(np.float32)
@@ -262,6 +264,7 @@ class BinaryOpTest(test.TestCase):
     self.assertAllEqual(np_result, left_result)
     self.assertAllEqual(np_result, right_result)
 
+  @test_util.run_deprecated_v1
   def testDoubleBasic(self):
     x = np.linspace(-5, 20, 15).reshape(1, 3, 5).astype(np.float64)
     y = np.linspace(20, -5, 15).reshape(1, 3, 5).astype(np.float64)
@@ -353,6 +356,7 @@ class BinaryOpTest(test.TestCase):
     self._compareBoth(x, y, np.floor_divide, _FLOORDIV)
     self._compareBoth(x, y, np.mod, _MOD)
 
+  @test_util.run_deprecated_v1
   def testComplex64Basic(self):
     x = np.complex(1, 1) * np.linspace(-10, 10, 6).reshape(1, 3, 2).astype(
         np.complex64)
@@ -367,6 +371,7 @@ class BinaryOpTest(test.TestCase):
     self._compareBoth(x, y, np.multiply, _MUL)
     self._compareBoth(x, y + 0.1, np.true_divide, _TRUEDIV)
 
+  @test_util.run_deprecated_v1
   def testComplex128Basic(self):
     x = np.complex(1, 1) * np.linspace(-10, 10, 6).reshape(1, 3, 2).astype(
         np.complex128)
@@ -480,198 +485,263 @@ class BinaryOpTest(test.TestCase):
     ]
     self._testBCastByFunc(funcs, xs, ys)
 
+  @test_util.run_deprecated_v1
   def testBCast_0A(self):
     self._testBCastA([1, 3, 2], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_0B(self):
     self._testBCastB([1, 3, 2], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_0C(self):
     self._testBCastC([1, 3, 2], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_0D(self):
     self._testBCastD([1, 3, 2], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_1A(self):
     self._testBCastA([1, 3, 2], [2])
 
+  @test_util.run_deprecated_v1
   def testBCast_1B(self):
     self._testBCastB([1, 3, 2], [2])
 
+  @test_util.run_deprecated_v1
   def testBCast_1C(self):
     self._testBCastC([1, 3, 2], [2])
 
+  @test_util.run_deprecated_v1
   def testBCast_1D(self):
     self._testBCastD([1, 3, 2], [2])
 
+  @test_util.run_deprecated_v1
   def testBCast_2A(self):
     self._testBCastA([1, 3, 2], [3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_2B(self):
     self._testBCastB([1, 3, 2], [3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_2C(self):
     self._testBCastC([1, 3, 2], [3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_2D(self):
     self._testBCastD([1, 3, 2], [3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_3A(self):
     self._testBCastA([1, 3, 2], [3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_3B(self):
     self._testBCastB([1, 3, 2], [3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_3C(self):
     self._testBCastC([1, 3, 2], [3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_3D(self):
     self._testBCastD([1, 3, 2], [3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_4A(self):
     self._testBCastA([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_4B(self):
     self._testBCastB([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_4C(self):
     self._testBCastC([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_4D(self):
     self._testBCastD([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_5A(self):
     self._testBCastA([1, 3, 2], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_5B(self):
     self._testBCastB([1, 3, 2], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_5C(self):
     self._testBCastC([1, 3, 2], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_5D(self):
     self._testBCastD([1, 3, 2], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_6A(self):
     self._testBCastA([1, 3, 2], [2, 1, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_6B(self):
     self._testBCastB([1, 3, 2], [2, 1, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_6C(self):
     self._testBCastC([1, 3, 2], [2, 1, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_6D(self):
     self._testBCastD([1, 3, 2], [2, 1, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_7A(self):
     self._testBCastA([1, 3, 2], [1, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_7B(self):
     self._testBCastB([1, 3, 2], [1, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_7C(self):
     self._testBCastC([1, 3, 2], [1, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_7D(self):
     self._testBCastD([1, 3, 2], [1, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_8A(self):
     self._testBCastA([2, 1, 5], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_8B(self):
     self._testBCastB([2, 1, 5], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_8C(self):
     self._testBCastC([2, 1, 5], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_8D(self):
     self._testBCastD([2, 1, 5], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_9A(self):
     self._testBCastA([2, 0, 5], [2, 0, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_9B(self):
     self._testBCastB([2, 0, 5], [2, 0, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_9C(self):
     self._testBCastC([2, 0, 5], [2, 0, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_9D(self):
     self._testBCastD([2, 0, 5], [2, 0, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_10A(self):
     self._testBCastA([2, 3, 0], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_10B(self):
     self._testBCastB([2, 3, 0], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_10C(self):
     self._testBCastC([2, 3, 0], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_10D(self):
     self._testBCastD([2, 3, 0], [2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testBCast_11A(self):
     self._testBCastA([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_11B(self):
     self._testBCastB([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_11C(self):
     self._testBCastC([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_11D(self):
     self._testBCastD([1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_12A(self):
     self._testBCastA([1, 1, 1, 1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_12B(self):
     self._testBCastB([1, 1, 1, 1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_12C(self):
     self._testBCastC([1, 1, 1, 1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_12D(self):
     self._testBCastD([1, 1, 1, 1, 3, 2], [1, 3, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_13A(self):
     self._testBCastA([1, 3, 2, 1, 1], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_13B(self):
     self._testBCastB([1, 3, 2, 1, 1], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_13C(self):
     self._testBCastC([1, 3, 2, 1, 1], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_13D(self):
     self._testBCastD([1, 3, 2, 1, 1], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_14A(self):
     self._testBCastA([2, 3, 1, 1, 5], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_14B(self):
     self._testBCastB([2, 3, 1, 1, 5], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_14C(self):
     self._testBCastC([2, 3, 1, 1, 5], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_14D(self):
     self._testBCastD([2, 3, 1, 1, 5], [1])
 
+  @test_util.run_deprecated_v1
   def testBCast_15A(self):
     self._testBCastA([10, 3, 1, 2], [3, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_15B(self):
     self._testBCastB([10, 3, 1, 2], [3, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_15C(self):
     self._testBCastC([10, 3, 1, 2], [3, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testBCast_15D(self):
     self._testBCastD([10, 3, 1, 2], [3, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testMismatchedDimensions(self):
     for func in [
         math_ops.add, math_ops.subtract, math_ops.multiply, math_ops.div, _ADD,
@@ -683,6 +753,7 @@ class BinaryOpTest(test.TestCase):
             ops.convert_to_tensor([10.0, 20.0, 30.0]),
             ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]]))
 
+  @test_util.run_deprecated_v1
   def testZeroPowGrad(self):
     with self.cached_session():
       for dtype in (np.float16, np.float32, np.float64, np.complex64,
@@ -693,6 +764,7 @@ class BinaryOpTest(test.TestCase):
         error = gradient_checker.compute_gradient_error(y, [], z, [])
         self.assertEqual(error, 0)
 
+  @test_util.run_deprecated_v1
   def testComplexPowGrad(self):
     with self.cached_session():
       for dtype in np.complex64, np.complex128:
@@ -861,6 +933,7 @@ class ComparisonOpTest(test.TestCase):
     self._testBCastByFunc(
         np.not_equal, math_ops.not_equal, include_complex=True)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     dtypes = [np.float16, np.float32, np.float64, np.int32, np.int64]
     funcs = [
diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 7e14f95be4..9bb7d8b8b1 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -199,6 +199,7 @@ class ComparisonOpTest(test.TestCase):
     self._testBCastByFunc(
         np.not_equal, math_ops.not_equal, include_complex=True)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     dtypes = [np.float16, np.float32, np.float64, np.int32, np.int64]
     funcs = [
@@ -280,6 +281,7 @@ class LogicalOpTest(test.TestCase):
         self._compareBinary(x, y, np.logical_or, math_ops.logical_or, use_gpu)
         self._compareBinary(x, y, np.logical_xor, math_ops.logical_xor, use_gpu)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     x = np.random.randint(0, 2, 6).astype(np.bool).reshape(1, 3, 2)
     y = np.random.randint(0, 2, 6).astype(np.bool).reshape(3, 2, 1)
@@ -288,6 +290,7 @@ class LogicalOpTest(test.TestCase):
           ValueError, lambda e: "Dimensions must" in str(e)):
         f(x, y)
 
+  @test_util.run_deprecated_v1
   def testUsingAsPythonValueFails(self):
     # Ensure that we raise an error when the user attempts to treat a
     # `Tensor` as a Python `bool`.
@@ -396,6 +399,7 @@ class SelectOpTest(test.TestCase):
       if t in [np.float16, np.float32, np.float64]:
         self._compare(c, xt, yt, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     c = np.random.randint(0, 2, 6).astype(np.bool).reshape(1, 3, 2)
     x = np.random.rand(1, 3, 2) * 100
@@ -415,6 +419,7 @@ class SelectOpTest(test.TestCase):
         self._compareGradientX(c, xt, yt)
         self._compareGradientY(c, xt, yt)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     c = np.random.randint(0, 2, 6).astype(np.bool).reshape(1, 3, 2)
     x = np.random.rand(1, 3, 2) * 100
@@ -428,6 +433,7 @@ class SelectOpTest(test.TestCase):
       with self.assertRaises(ValueError):
         array_ops.where(c, xt, yt)
 
+  @test_util.run_deprecated_v1
   def testEmptyTensor(self):
     c = np.random.randint(0, 3, 0).astype(np.bool).reshape(1, 3, 0)
     x = np.random.rand(1, 3, 0) * 100
@@ -439,6 +445,7 @@ class SelectOpTest(test.TestCase):
       z = array_ops.where(c, xt, yt).eval()
       self.assertAllEqual(z_expected, z)
 
+  @test_util.run_deprecated_v1
   def testNan(self):
     """Verify that nans don't propagate where they shouldn't."""
     with self.cached_session():
@@ -525,6 +532,7 @@ class BatchSelectOpTest(test.TestCase):
       if t in [np.float16, np.float32, np.float64]:
         self._compare(c, xt, yt, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     c = np.random.randint(0, 2, 16).astype(np.bool)
     x = np.random.rand(16, 2, 8) * 100
@@ -544,6 +552,7 @@ class BatchSelectOpTest(test.TestCase):
         self._compareGradientX(c, xt, yt)
         self._compareGradientY(c, xt, yt)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     c = np.random.randint(0, 2, 8).astype(np.bool)
     x = np.random.rand(16, 3, 2) * 100
@@ -622,6 +631,7 @@ class MinMaxOpTest(test.TestCase):
     elif x.dtype == np.float64:
       self.assertAllClose(jacob_t, jacob_n, rtol=1e-5, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     x = np.random.rand(1, 3, 2) * 100.
     # ensure x != y
@@ -898,6 +908,7 @@ class ComplexMakeRealImagTest(test.TestCase):
     # build failures on GPU (See #10643 for context).
     # self._compareAngle(cplx, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testRealReal(self):
     for dtype in (dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.float32,
                   dtypes_lib.float64):
@@ -928,6 +939,7 @@ class ComplexMakeRealImagTest(test.TestCase):
     self._compareConj(cplx, use_gpu=False)
     self._compareConj(cplx, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testConjReal(self):
     for dtype in (dtypes_lib.int32, dtypes_lib.int64, dtypes_lib.float16,
                   dtypes_lib.float32, dtypes_lib.float64):
@@ -935,6 +947,7 @@ class ComplexMakeRealImagTest(test.TestCase):
       y = math_ops.conj(x)
       self.assertEqual(x, y)
 
+  @test_util.run_deprecated_v1
   def testConjString(self):
     x = array_ops.placeholder(dtypes_lib.string)
     with self.assertRaisesRegexp(TypeError,
@@ -971,6 +984,7 @@ class ComplexMakeRealImagTest(test.TestCase):
             x_, list(x.shape), z, [1], x_init_value=x, delta=epsilon)
         self.assertAllClose(jacob_t, jacob_n, rtol=epsilon, atol=epsilon)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     # complex64
     data = np.arange(1, 2, 0.10).reshape([5, 2]).astype(np.float32)
@@ -1006,6 +1020,7 @@ class ComplexMakeRealImagTest(test.TestCase):
           inp, list(data.shape), loss, [1], x_init_value=data, delta=epsilon)
     self.assertAllClose(jacob_t, jacob_n, rtol=epsilon, atol=epsilon)
 
+  @test_util.run_deprecated_v1
   def testMulGradient(self):
     data = np.arange(1, 2, 0.125).reshape([2, 4]).astype(np.float32)
     self._compareMulGradient(data)
diff --git a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
index 3e8294f34b..709a20f3d0 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py
@@ -184,6 +184,7 @@ class UnaryOpTest(test.TestCase):
 
     return func
 
+  @test_util.run_deprecated_v1
   def testFloatBasic(self):
     x = np.arange(-3, 3).reshape(1, 3, 2).astype(np.float32)
     w = x - x.min() + 1.02  # all greater than 1
@@ -238,12 +239,14 @@ class UnaryOpTest(test.TestCase):
     self._compareBothSparse(y, np.sign, math_ops.sign)
     self._compareBothSparse(x, np.vectorize(math.erf), math_ops.erf)
 
+  @test_util.run_deprecated_v1
   def testFloatTanhEdge(self):
     x = np.arange(40, 40 + 6).reshape(6).astype(np.float32)
     self._compareBoth(x, np.tanh, math_ops.tanh)
     x = np.arange(-40, -40 + 6).reshape(6).astype(np.float32)
     self._compareBoth(x, np.tanh, math_ops.tanh)
 
+  @test_util.run_deprecated_v1
   def testFloatEmpty(self):
     x = np.empty((2, 0, 5), dtype=np.float32)
     self._compareBoth(x, np.abs, math_ops.abs)
@@ -289,6 +292,7 @@ class UnaryOpTest(test.TestCase):
     self._compareBothSparse(x, np.sign, math_ops.sign)
     self._compareBothSparse(x, np.sign, math_ops.erf)
 
+  @test_util.run_deprecated_v1
   def testDoubleBasic(self):
     x = np.arange(-3, 3).reshape(1, 3, 2).astype(np.float64)
     w = x - x.min() + 1.02  # all greater than 1
@@ -342,6 +346,7 @@ class UnaryOpTest(test.TestCase):
     self._compareBothSparse(y, np.sign, math_ops.sign)
     self._compareBothSparse(x, np.vectorize(math.erf), math_ops.erf)
 
+  @test_util.run_deprecated_v1
   def testHalfBasic(self):
     x = np.arange(-3, 3).reshape(1, 3, 2).astype(np.float16)
     y = (x + .5).astype(np.float16)  # no zero
@@ -414,6 +419,7 @@ class UnaryOpTest(test.TestCase):
     self._compareCpu(x, np.square, math_ops.square)
     self._compareBothSparse(x, np.square, math_ops.square)
 
+  @test_util.run_deprecated_v1
   def testComplex64Basic(self):
     x = np.complex(1, 1) * np.arange(-3, 3).reshape(1, 3, 2).astype(
         np.complex64)
@@ -458,6 +464,7 @@ class UnaryOpTest(test.TestCase):
     self._compareBoth(y, complex_sign, math_ops.sign)
     self._compareBothSparse(y, complex_sign, math_ops.sign)
 
+  @test_util.run_deprecated_v1
   def testComplex128Basic(self):
     x = np.complex(1, 1) * np.arange(-3, 3).reshape(1, 3, 2).astype(
         np.complex128)
@@ -497,6 +504,7 @@ class UnaryOpTest(test.TestCase):
     self._compareBoth(y, complex_sign, math_ops.sign)
     self._compareBothSparse(y, complex_sign, math_ops.sign)
 
+  @test_util.run_deprecated_v1
   def testGradGrad(self):
     np.random.seed(7)
     shape = (5,)
diff --git a/tensorflow/python/kernel_tests/decode_compressed_op_test.py b/tensorflow/python/kernel_tests/decode_compressed_op_test.py
index 1cc1c7da30..fd871c0090 100644
--- a/tensorflow/python/kernel_tests/decode_compressed_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_compressed_op_test.py
@@ -24,6 +24,7 @@ import zlib
 from six import BytesIO
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
@@ -42,6 +43,7 @@ class DecodeCompressedOpTest(test.TestCase):
         f.write(bytes_in)
       return out.getvalue()
 
+  @test_util.run_deprecated_v1
   def testDecompress(self):
     for compression_type in ["ZLIB", "GZIP", ""]:
       with self.cached_session():
@@ -55,6 +57,7 @@ class DecodeCompressedOpTest(test.TestCase):
                                   self._compress(b"bBbb", compression_type)]})
         self.assertAllEqual([b"AaAA", b"bBbb"], result)
 
+  @test_util.run_deprecated_v1
   def testDecompressWithRaw(self):
     for compression_type in ["ZLIB", "GZIP", ""]:
       with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/decode_image_op_test.py b/tensorflow/python/kernel_tests/decode_image_op_test.py
index 267afdeb5e..ba5770001a 100644
--- a/tensorflow/python/kernel_tests/decode_image_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_image_op_test.py
@@ -23,6 +23,7 @@ import os.path
 import numpy as np
 
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import image_ops
 from tensorflow.python.ops import io_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -44,6 +45,7 @@ class DecodeImageOpTest(test.TestCase):
       self.assertEqual(len(bmp0), 4194)
       self.assertAllEqual(image0, image1)
 
+  @test_util.run_deprecated_v1
   def testGif(self):
     # Read some real GIFs
     path = os.path.join(prefix_path, "gif", "testdata", "scan.gif")
@@ -78,6 +80,7 @@ class DecodeImageOpTest(test.TestCase):
         with self.assertRaises(errors_impl.InvalidArgumentError):
           self.evaluate(bad_channels)
 
+  @test_util.run_deprecated_v1
   def testJpeg(self):
     # Read a real jpeg and verify shape
     path = os.path.join(prefix_path, "jpeg", "testdata", "jpeg_merge_test1.jpg")
@@ -108,6 +111,7 @@ class DecodeImageOpTest(test.TestCase):
           self.assertEqual(image0.shape, (26, 51, channels or channels_in))
           self.assertAllEqual(image0, image1)
 
+  @test_util.run_deprecated_v1
   def testInvalidBytes(self):
     image_bytes = b"ThisIsNotAnImage!"
     decode = image_ops.decode_image(image_bytes)
diff --git a/tensorflow/python/kernel_tests/decode_raw_op_test.py b/tensorflow/python/kernel_tests/decode_raw_op_test.py
index dcc984811c..008e59ba3e 100644
--- a/tensorflow/python/kernel_tests/decode_raw_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_raw_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
@@ -28,6 +29,7 @@ from tensorflow.python.platform import test
 
 class DecodeRawOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testToUint8(self):
     with self.cached_session():
       in_bytes = array_ops.placeholder(dtypes.string, shape=[2])
@@ -46,6 +48,7 @@ class DecodeRawOpTest(test.TestCase):
           "element 1 has size 5 != 6"):
         decode.eval(feed_dict={in_bytes: ["short", "longer"]})
 
+  @test_util.run_deprecated_v1
   def testToInt16(self):
     with self.cached_session():
       in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
@@ -61,6 +64,7 @@ class DecodeRawOpTest(test.TestCase):
           "size of int16"):
         decode.eval(feed_dict={in_bytes: ["123", "456"]})
 
+  @test_util.run_deprecated_v1
   def testEndianness(self):
     with self.cached_session():
       in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
@@ -73,6 +77,7 @@ class DecodeRawOpTest(test.TestCase):
       result = decode_be.eval(feed_dict={in_bytes: ["\x01\x02\x03\x04"]})
       self.assertAllEqual([[0x01020304]], result)
 
+  @test_util.run_deprecated_v1
   def testToFloat16(self):
     with self.cached_session():
       in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
@@ -84,6 +89,7 @@ class DecodeRawOpTest(test.TestCase):
 
       self.assertAllEqual(expected_result, result)
 
+  @test_util.run_deprecated_v1
   def testEmptyStringInput(self):
     with self.cached_session():
       in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
@@ -93,6 +99,7 @@ class DecodeRawOpTest(test.TestCase):
         result = decode.eval(feed_dict={in_bytes: [""] * num_inputs})
         self.assertEqual((num_inputs, 0), result.shape)
 
+  @test_util.run_deprecated_v1
   def testToUInt16(self):
     with self.cached_session():
       in_bytes = array_ops.placeholder(dtypes.string, shape=[None])
diff --git a/tensorflow/python/kernel_tests/denormal_test.py b/tensorflow/python/kernel_tests/denormal_test.py
index 71a528c4aa..80a3033ecc 100644
--- a/tensorflow/python/kernel_tests/denormal_test.py
+++ b/tensorflow/python/kernel_tests/denormal_test.py
@@ -22,6 +22,7 @@ import numpy as np
 import platform
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -50,10 +51,12 @@ class DenormalTest(test.TestCase):
           # Make sure the flags don't leak out
           self.testPythonHasDenormals()
 
+  @test_util.run_deprecated_v1
   def testFlushDenormalsCPU(self):
     # On CPUs, the processor flags flush for both single and double precision.
     self._flushDenormalsTest(use_gpu=False, dtypes=(np.float32, np.float64))
 
+  @test_util.run_deprecated_v1
   def testFlushDenormalsGPU(self):
     # On GPUs, only single precision can flush to zero.
     self._flushDenormalsTest(use_gpu=True, dtypes=(np.float32,))
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
index 0676664685..4f74e1e741 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
@@ -32,6 +33,7 @@ class AssignOpTest(test.TestCase):
   # NOTE(mrry): We exclude thess tests from the TSAN TAP target, because they
   #   contain benign and deliberate data races when multiple threads update
   #   the same parameters without a lock.
+  @test_util.run_deprecated_v1
   def testParallelUpdateWithoutLocking(self):
     with self.cached_session() as sess:
       ones_t = array_ops.fill([1024, 1024], 1.0)
@@ -59,6 +61,7 @@ class AssignOpTest(test.TestCase):
       self.assertTrue((vals >= ones).all())
       self.assertTrue((vals <= ones * 20).all())
 
+  @test_util.run_deprecated_v1
   def testParallelAssignWithoutLocking(self):
     with self.cached_session() as sess:
       ones_t = array_ops.fill([1024, 1024], float(1))
@@ -91,6 +94,7 @@ class AssignOpTest(test.TestCase):
   # contain non-benign but known data races between the variable assignment and
   # returning the output tensors. This issue will be resolved with the new
   # resource variables.
+  @test_util.run_deprecated_v1
   def testParallelUpdateWithLocking(self):
     with self.cached_session() as sess:
       zeros_t = array_ops.fill([1024, 1024], 0.0)
@@ -118,6 +122,7 @@ class AssignOpTest(test.TestCase):
       ones = np.ones((1024, 1024)).astype(np.float32)
       self.assertAllEqual(vals, ones * 20)
 
+  @test_util.run_deprecated_v1
   def testParallelAssignWithLocking(self):
     with self.cached_session() as sess:
       zeros_t = array_ops.fill([1024, 1024], 0.0)
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_test.py b/tensorflow/python/kernel_tests/dense_update_ops_test.py
index a4766fed72..309da88bef 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
@@ -81,9 +82,11 @@ class AssignOpTest(test.TestCase):
         self.assertAllEqual(x - y, var_value)
         self.assertAllEqual(x - y, op_value)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     self._testTypes(np.arange(0, 20).reshape([4, 5]))
 
+  @test_util.run_deprecated_v1
   def testAssignNonStrictShapeChecking(self):
     with self.cached_session():
       data = array_ops.fill([1024, 1024], 0)
@@ -98,6 +101,7 @@ class AssignOpTest(test.TestCase):
       a2.op.run()
       self.assertAllEqual(p.eval(), self.evaluate(data2))
 
+  @test_util.run_deprecated_v1
   def testInitRequiredAssignAdd(self):
     with self.cached_session():
       p = variables.VariableV1(array_ops.fill([1024, 1024], 1), dtypes.int32)
@@ -105,6 +109,7 @@ class AssignOpTest(test.TestCase):
       with self.assertRaisesOpError("use uninitialized"):
         a.op.run()
 
+  @test_util.run_deprecated_v1
   def testInitRequiredAssignSub(self):
     with self.cached_session():
       p = variables.VariableV1(array_ops.fill([1024, 1024], 1), dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 19f145865f..96c9b5258e 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -53,12 +53,14 @@ class DepthToSpaceTest(test.TestCase):
         output_nhwc = test_util.NCHWToNHWC(output_nchw)
         self.assertAllEqual(output_nhwc.eval(), outputs)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     x_np = [[[[1, 2, 3, 4]]]]
     block_size = 2
     x_out = [[[[1], [2]], [[3], [4]]]]
     self._testOne(x_np, block_size, x_out)
 
+  @test_util.run_deprecated_v1
   def testBasicFloat16(self):
     x_np = [[[[1, 2, 3, 4]]]]
     block_size = 2
@@ -67,6 +69,7 @@ class DepthToSpaceTest(test.TestCase):
 
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
+  @test_util.run_deprecated_v1
   def testBlockSize2(self):
     x_np = [[[[1, 2, 3, 4],
               [5, 6, 7, 8]],
@@ -79,6 +82,7 @@ class DepthToSpaceTest(test.TestCase):
               [[11], [12], [15], [16]]]]
     self._testOne(x_np, block_size, x_out)
 
+  @test_util.run_deprecated_v1
   def testBlockSize2Batch10(self):
     block_size = 2
     def batch_input_elt(i):
@@ -115,6 +119,7 @@ class DepthToSpaceTest(test.TestCase):
         self.evaluate(x_tf)
 
   # Tests for different width and height.
+  @test_util.run_deprecated_v1
   def testNonSquare(self):
     x_np = [[[[1, 10, 2, 20, 3, 30, 4, 40]],
              [[5, 50, 6, 60, 7, 70, 8, 80]],
@@ -130,6 +135,7 @@ class DepthToSpaceTest(test.TestCase):
 
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
+  @test_util.run_deprecated_v1
   def testBlockSize4FlatInput(self):
     x_np = [[[[1, 2, 5, 6, 3, 4, 7, 8, 9, 10, 13, 14, 11, 12, 15, 16]]]]
     block_size = 4
@@ -141,6 +147,7 @@ class DepthToSpaceTest(test.TestCase):
 
   # Tests for larger input depths.
   # To make sure elements are properly interleaved in depth.
+  @test_util.run_deprecated_v1
   def testDepthInterleaved(self):
     x_np = [[[[1, 10, 2, 20, 3, 30, 4, 40]]]]
     block_size = 2
@@ -150,6 +157,7 @@ class DepthToSpaceTest(test.TestCase):
 
   # Tests for larger input depths. Here an odd depth.
   # To make sure elements are properly interleaved in depth.
+  @test_util.run_deprecated_v1
   def testDepthInterleavedDepth3(self):
     x_np = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
     block_size = 2
@@ -159,6 +167,7 @@ class DepthToSpaceTest(test.TestCase):
 
   # Tests for larger input depths.
   # To make sure elements are properly interleaved in depth.
+  @test_util.run_deprecated_v1
   def testDepthInterleavedLarger(self):
     x_np = [[[[1, 10, 2, 20, 3, 30, 4, 40],
               [5, 50, 6, 60, 7, 70, 8, 80]],
@@ -175,6 +184,7 @@ class DepthToSpaceTest(test.TestCase):
 
   # Tests for a block larger for the depth. In this case should raise an
   # exception.
+  @test_util.run_deprecated_v1
   def testBlockSizeTooLarge(self):
     x_np = [[[[1, 2, 3, 4],
               [5, 6, 7, 8]],
@@ -188,6 +198,7 @@ class DepthToSpaceTest(test.TestCase):
       self.evaluate(out_tf)
 
   # Test when the block size is 0.
+  @test_util.run_deprecated_v1
   def testBlockSize0(self):
     x_np = [[[[1], [2]],
              [[3], [4]]]]
@@ -197,6 +208,7 @@ class DepthToSpaceTest(test.TestCase):
       self.evaluate(out_tf)
 
   # Test when the block size is 1. The block size should be > 1.
+  @test_util.run_deprecated_v1
   def testBlockSizeOne(self):
     x_np = [[[[1, 1, 1, 1],
               [2, 2, 2, 2]],
@@ -207,6 +219,7 @@ class DepthToSpaceTest(test.TestCase):
       out_tf = array_ops.depth_to_space(x_np, block_size)
       self.evaluate(out_tf)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeLargerThanInput(self):
     # The block size is too large for this input.
     x_np = [[[[1], [2]],
@@ -216,6 +229,7 @@ class DepthToSpaceTest(test.TestCase):
       out_tf = array_ops.space_to_depth(x_np, block_size)
       self.evaluate(out_tf)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleDepth(self):
     # The depth is not divisible by the square of the block size.
     x_np = [[[[1, 1, 1, 1],
@@ -226,6 +240,7 @@ class DepthToSpaceTest(test.TestCase):
     with self.assertRaises(ValueError):
       _ = array_ops.space_to_depth(x_np, block_size)
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     t = array_ops.depth_to_space(
         array_ops.placeholder(dtypes.float32), block_size=4)
@@ -343,11 +358,13 @@ class DepthToSpaceGradientTest(test.TestCase):
 
   # Don't use very large numbers as dimensions here, as the result is tensor
   # with cartesian product of the dimensions.
+  @test_util.run_deprecated_v1
   def testSmall(self):
     block_size = 2
     self._compare(3, 2, 5, 3, block_size, "NHWC")
     self._compare(3, 2, 5, 3, block_size, "NCHW")
 
+  @test_util.run_deprecated_v1
   def testSmall2(self):
     block_size = 3
     self._compare(1, 2, 3, 2, block_size, "NHWC")
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index f7a9cd8d6e..ed2a9e8e47 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -31,6 +32,7 @@ from tensorflow.python.platform import tf_logging
 
 class MatrixDiagTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testVector(self):
     with self.session(use_gpu=True):
       v = np.array([1.0, 2.0, 3.0])
@@ -49,6 +51,7 @@ class MatrixDiagTest(test.TestCase):
       self.assertEqual((2, 3, 3), v_batch_diag.get_shape())
       self.assertAllEqual(v_batch_diag.eval(), mat_batch)
 
+  @test_util.run_deprecated_v1
   def testBatchVector(self):
     self._testBatchVector(np.float32)
     self._testBatchVector(np.float64)
@@ -56,16 +59,19 @@ class MatrixDiagTest(test.TestCase):
     self._testBatchVector(np.int64)
     self._testBatchVector(np.bool)
 
+  @test_util.run_deprecated_v1
   def testInvalidShape(self):
     with self.assertRaisesRegexp(ValueError, "must be at least rank 1"):
       array_ops.matrix_diag(0)
 
+  @test_util.run_deprecated_v1
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
       with self.assertRaisesOpError("input must be at least 1-dim"):
         array_ops.matrix_diag(v).eval(feed_dict={v: 0.0})
 
+  @test_util.run_deprecated_v1
   def testGrad(self):
     shapes = ((3,), (7, 4))
     with self.session(use_gpu=True):
@@ -81,6 +87,7 @@ class MatrixDiagTest(test.TestCase):
 
 class MatrixSetDiagTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSquare(self):
     with self.session(use_gpu=True):
       v = np.array([1.0, 2.0, 3.0])
@@ -91,6 +98,7 @@ class MatrixSetDiagTest(test.TestCase):
       self.assertEqual((3, 3), output.get_shape())
       self.assertAllEqual(mat_set_diag, self.evaluate(output))
 
+  @test_util.run_deprecated_v1
   def testRectangular(self):
     with self.session(use_gpu=True):
       v = np.array([3.0, 4.0])
@@ -123,6 +131,7 @@ class MatrixSetDiagTest(test.TestCase):
       self.assertEqual((2, 3, 3), output.get_shape())
       self.assertAllEqual(mat_set_diag_batch, self.evaluate(output))
 
+  @test_util.run_deprecated_v1
   def testSquareBatch(self):
     self._testSquareBatch(np.float32)
     self._testSquareBatch(np.float64)
@@ -130,6 +139,7 @@ class MatrixSetDiagTest(test.TestCase):
     self._testSquareBatch(np.int64)
     self._testSquareBatch(np.bool)
 
+  @test_util.run_deprecated_v1
   def testRectangularBatch(self):
     with self.session(use_gpu=True):
       v_batch = np.array([[-1.0, -2.0], [-4.0, -5.0]])
@@ -142,12 +152,14 @@ class MatrixSetDiagTest(test.TestCase):
       self.assertEqual((2, 2, 3), output.get_shape())
       self.assertAllEqual(mat_set_diag_batch, self.evaluate(output))
 
+  @test_util.run_deprecated_v1
   def testInvalidShape(self):
     with self.assertRaisesRegexp(ValueError, "must be at least rank 2"):
       array_ops.matrix_set_diag(0, [0])
     with self.assertRaisesRegexp(ValueError, "must be at least rank 1"):
       array_ops.matrix_set_diag([[0]], 0)
 
+  @test_util.run_deprecated_v1
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -157,6 +169,7 @@ class MatrixSetDiagTest(test.TestCase):
           r"but received input shape: \[1,1\] and diagonal shape: \[\]"):
         array_ops.matrix_set_diag([[v]], v).eval(feed_dict={v: 0.0})
 
+  @test_util.run_deprecated_v1
   def testGrad(self):
     shapes = ((3, 4, 4), (3, 3, 4), (3, 4, 3), (7, 4, 8, 8))
     with self.session(use_gpu=True):
@@ -178,6 +191,7 @@ class MatrixSetDiagTest(test.TestCase):
             y.get_shape().as_list())
         self.assertLess(error_x_diag, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradWithNoShapeInformation(self):
     with self.session(use_gpu=True) as sess:
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -200,6 +214,7 @@ class MatrixSetDiagTest(test.TestCase):
 
 class MatrixDiagPartTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSquare(self):
     with self.session(use_gpu=True):
       v = np.array([1.0, 2.0, 3.0])
@@ -208,6 +223,7 @@ class MatrixDiagPartTest(test.TestCase):
       self.assertEqual((3,), mat_diag.get_shape())
       self.assertAllEqual(mat_diag.eval(), v)
 
+  @test_util.run_deprecated_v1
   def testRectangular(self):
     with self.session(use_gpu=True):
       mat = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
@@ -228,6 +244,7 @@ class MatrixDiagPartTest(test.TestCase):
       self.assertEqual((2, 3), mat_batch_diag.get_shape())
       self.assertAllEqual(mat_batch_diag.eval(), v_batch)
 
+  @test_util.run_deprecated_v1
   def testSquareBatch(self):
     self._testSquareBatch(np.float32)
     self._testSquareBatch(np.float64)
@@ -235,6 +252,7 @@ class MatrixDiagPartTest(test.TestCase):
     self._testSquareBatch(np.int64)
     self._testSquareBatch(np.bool)
 
+  @test_util.run_deprecated_v1
   def testRectangularBatch(self):
     with self.session(use_gpu=True):
       v_batch = np.array([[1.0, 2.0], [4.0, 5.0]])
@@ -245,16 +263,19 @@ class MatrixDiagPartTest(test.TestCase):
       self.assertEqual((2, 2), mat_batch_diag.get_shape())
       self.assertAllEqual(mat_batch_diag.eval(), v_batch)
 
+  @test_util.run_deprecated_v1
   def testInvalidShape(self):
     with self.assertRaisesRegexp(ValueError, "must be at least rank 2"):
       array_ops.matrix_diag_part(0)
 
+  @test_util.run_deprecated_v1
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
       with self.assertRaisesOpError("input must be at least 2-dim"):
         array_ops.matrix_diag_part(v).eval(feed_dict={v: 0.0})
 
+  @test_util.run_deprecated_v1
   def testGrad(self):
     shapes = ((3, 3), (2, 3), (3, 2), (5, 3, 3))
     with self.session(use_gpu=True):
@@ -407,6 +428,7 @@ class DiagTest(test.TestCase):
           dtype=dtype)
       self.diagOp(x, dtype, expected_ans)
 
+  @test_util.run_deprecated_v1
   def testInvalidRank(self):
     with self.assertRaisesRegexp(ValueError, "must be at least rank 1"):
       array_ops.diag(0.0)
@@ -476,6 +498,7 @@ class DiagPartOpTest(test.TestCase):
     self.diagPartOp(x, np.complex64, expected_ans)
     self.diagPartOp(x, np.complex128, expected_ans)
 
+  @test_util.run_deprecated_v1
   def testOddRank(self):
     w = np.random.rand(2)
     x = np.random.rand(2, 2, 2)
@@ -484,6 +507,7 @@ class DiagPartOpTest(test.TestCase):
     with self.assertRaises(ValueError):
       array_ops.diag_part(0.0)
 
+  @test_util.run_deprecated_v1
   def testUnevenDimensions(self):
     w = np.random.rand(2, 5)
     x = np.random.rand(2, 1, 2, 3)
@@ -493,6 +517,7 @@ class DiagPartOpTest(test.TestCase):
 
 class DiagGradOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testDiagGrad(self):
     np.random.seed(0)
     shapes = ((3,), (3, 3), (3, 3, 3))
@@ -513,6 +538,7 @@ class DiagGradOpTest(test.TestCase):
 
 class DiagGradPartOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testDiagPartGrad(self):
     np.random.seed(0)
     shapes = ((3, 3), (3, 3, 3, 3))
diff --git a/tensorflow/python/kernel_tests/distributions/bernoulli_test.py b/tensorflow/python/kernel_tests/distributions/bernoulli_test.py
index 37b35ba51a..e6d560b4bc 100644
--- a/tensorflow/python/kernel_tests/distributions/bernoulli_test.py
+++ b/tensorflow/python/kernel_tests/distributions/bernoulli_test.py
@@ -151,6 +151,7 @@ class BernoulliTest(test.TestCase):
       self.assertAllClose(self.evaluate(dist.prob(x)), expected_pmf)
       self.assertAllClose(self.evaluate(dist.log_prob(x)), np.log(expected_pmf))
 
+  @test_util.run_deprecated_v1
   def testPmfCorrectBroadcastDynamicShape(self):
     with self.cached_session():
       p = array_ops.placeholder(dtype=dtypes.float32)
@@ -167,6 +168,7 @@ class BernoulliTest(test.TestCase):
           }), [[0.2, 0.7, 0.4]])
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testPmfInvalid(self):
     p = [0.1, 0.2, 0.7]
     dist = bernoulli.Bernoulli(probs=p, validate_args=True)
@@ -193,6 +195,7 @@ class BernoulliTest(test.TestCase):
         self.evaluate(
             bernoulli.Bernoulli(probs=p, validate_args=False).log_prob(samps)))
 
+  @test_util.run_deprecated_v1
   def testBroadcasting(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes.float32)
@@ -207,6 +210,7 @@ class BernoulliTest(test.TestCase):
               p: [0.5, 0.5, 0.5]
           }))
 
+  @test_util.run_deprecated_v1
   def testPmfShapes(self):
     with self.cached_session():
       p = array_ops.placeholder(dtypes.float32, shape=[None, 1])
@@ -276,6 +280,7 @@ class BernoulliTest(test.TestCase):
     grad_p = tape.gradient(samples, p)
     self.assertIsNone(grad_p)
 
+  @test_util.run_deprecated_v1
   def testSampleActsLikeSampleN(self):
     with self.cached_session() as sess:
       p = [0.2, 0.6]
diff --git a/tensorflow/python/kernel_tests/distributions/bijector_test.py b/tensorflow/python/kernel_tests/distributions/bijector_test.py
index e20f59f48a..a0e0a36fec 100644
--- a/tensorflow/python/kernel_tests/distributions/bijector_test.py
+++ b/tensorflow/python/kernel_tests/distributions/bijector_test.py
@@ -132,6 +132,7 @@ class BijectorTestEventNdims(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "Expected scalar"):
       bij.inverse_log_det_jacobian(1., event_ndims=(1, 2))
 
+  @test_util.run_deprecated_v1
   def testBijectorDynamicEventNdims(self):
     bij = BrokenBijector(validate_args=True)
     event_ndims = array_ops.placeholder(dtype=np.int32, shape=None)
@@ -301,6 +302,7 @@ class BijectorReduceEventDimsTest(test.TestCase):
         8.,
         self.evaluate(bij.inverse_log_det_jacobian(x, event_ndims=2)))
 
+  @test_util.run_deprecated_v1
   def testHandlesNonStaticEventNdims(self):
     x_ = [[[1., 2.], [3., 4.]]]
     x = array_ops.placeholder_with_default(x_, shape=None)
diff --git a/tensorflow/python/kernel_tests/distributions/categorical_test.py b/tensorflow/python/kernel_tests/distributions/categorical_test.py
index 9c593d2737..ec1d4ed207 100644
--- a/tensorflow/python/kernel_tests/distributions/categorical_test.py
+++ b/tensorflow/python/kernel_tests/distributions/categorical_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.eager import backprop
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
@@ -44,6 +45,7 @@ def make_categorical(batch_shape, num_classes, dtype=dtypes.int32):
 
 class CategoricalTest(test.TestCase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testP(self):
     p = [0.2, 0.8]
     dist = categorical.Categorical(probs=p)
@@ -51,6 +53,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(p, dist.probs.eval())
       self.assertAllEqual([2], dist.logits.get_shape())
 
+  @test_util.run_deprecated_v1
   def testLogits(self):
     p = np.array([0.2, 0.8], dtype=np.float32)
     logits = np.log(p) - 50.
@@ -61,6 +64,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(dist.probs.eval(), p)
       self.assertAllClose(dist.logits.eval(), logits)
 
+  @test_util.run_deprecated_v1
   def testShapes(self):
     with self.cached_session():
       for batch_shape in ([], [1], [2, 3, 4]):
@@ -107,6 +111,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       self.assertEqual(dist.dtype, dtype)
       self.assertEqual(dist.dtype, dist.sample(5).dtype)
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     with self.cached_session():
       logits = array_ops.placeholder(dtype=dtypes.float32)
@@ -121,18 +126,21 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
           feed_dict={logits: [[-1000.0, 1000.0], [1000.0, -1000.0]]})
       self.assertAllEqual([1, 0], sample_value_batch)
 
+  @test_util.run_deprecated_v1
   def testPMFWithBatch(self):
     histograms = [[0.2, 0.8], [0.6, 0.4]]
     dist = categorical.Categorical(math_ops.log(histograms) - 50.)
     with self.cached_session():
       self.assertAllClose(dist.prob([0, 1]).eval(), [0.2, 0.4])
 
+  @test_util.run_deprecated_v1
   def testPMFNoBatch(self):
     histograms = [0.2, 0.8]
     dist = categorical.Categorical(math_ops.log(histograms) - 50.)
     with self.cached_session():
       self.assertAllClose(dist.prob(0).eval(), 0.2)
 
+  @test_util.run_deprecated_v1
   def testCDFWithDynamicEventShapeKnownNdims(self):
     """Test that dynamically-sized events with unknown shape work."""
     batch_size = 2
@@ -184,6 +192,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     actual_cdf = self.evaluate(cdf_op)
     self.assertAllClose(actual_cdf, expected_cdf)
 
+  @test_util.run_deprecated_v1
   def testCDFWithBatch(self):
     histograms = [[0.1, 0.2, 0.3, 0.25, 0.15],
                   [0.0, 0.75, 0.2, 0.05, 0.0]]
@@ -195,6 +204,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     with self.cached_session():
       self.assertAllClose(cdf_op.eval(), expected_cdf)
 
+  @test_util.run_deprecated_v1
   def testCDFNoBatch(self):
     histogram = [0.1, 0.2, 0.3, 0.4]
     event = 2
@@ -205,6 +215,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     with self.cached_session():
       self.assertAlmostEqual(cdf_op.eval(), expected_cdf)
 
+  @test_util.run_deprecated_v1
   def testCDFBroadcasting(self):
     # shape: [batch=2, n_bins=3]
     histograms = [[0.2, 0.1, 0.7],
@@ -298,6 +309,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual(run_result["cat_log_cdf"].shape,
                         run_result["norm_log_cdf"].shape)
 
+  @test_util.run_deprecated_v1
   def testLogPMF(self):
     logits = np.log([[0.2, 0.8], [0.6, 0.4]]) - 50.
     dist = categorical.Categorical(logits)
@@ -305,6 +317,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(dist.log_prob([0, 1]).eval(), np.log([0.2, 0.4]))
       self.assertAllClose(dist.log_prob([0.0, 1.0]).eval(), np.log([0.2, 0.4]))
 
+  @test_util.run_deprecated_v1
   def testEntropyNoBatch(self):
     logits = np.log([0.2, 0.8]) - 50.
     dist = categorical.Categorical(logits)
@@ -312,6 +325,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(dist.entropy().eval(),
                           -(0.2 * np.log(0.2) + 0.8 * np.log(0.8)))
 
+  @test_util.run_deprecated_v1
   def testEntropyWithBatch(self):
     logits = np.log([[0.2, 0.8], [0.6, 0.4]]) - 50.
     dist = categorical.Categorical(logits)
@@ -321,6 +335,7 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
           -(0.6 * np.log(0.6) + 0.4 * np.log(0.4))
       ])
 
+  @test_util.run_deprecated_v1
   def testEntropyGradient(self):
     with self.cached_session() as sess:
       logits = constant_op.constant([[1., 2., 3.], [2., 5., 1.]])
@@ -440,12 +455,14 @@ class CategoricalTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(3, log_prob.get_shape().ndims)
     self.assertAllEqual([2, 2, 2], log_prob.get_shape())
 
+  @test_util.run_deprecated_v1
   def testMode(self):
     with self.cached_session():
       histograms = [[[0.2, 0.8], [0.6, 0.4]]]
       dist = categorical.Categorical(math_ops.log(histograms) - 50.)
       self.assertAllEqual(dist.mode().eval(), [[1, 0]])
 
+  @test_util.run_deprecated_v1
   def testCategoricalCategoricalKL(self):
 
     def np_softmax(logits):
diff --git a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py
index 3662ca1ad1..c530037e1e 100644
--- a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py
+++ b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import backprop
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import dirichlet_multinomial
@@ -36,6 +37,7 @@ class DirichletMultinomialTest(test.TestCase):
   def setUp(self):
     self._rng = np.random.RandomState(42)
 
+  @test_util.run_deprecated_v1
   def testSimpleShapes(self):
     with self.cached_session():
       alpha = np.random.rand(3)
@@ -45,6 +47,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertEqual(tensor_shape.TensorShape([3]), dist.event_shape)
       self.assertEqual(tensor_shape.TensorShape([]), dist.batch_shape)
 
+  @test_util.run_deprecated_v1
   def testComplexShapes(self):
     with self.cached_session():
       alpha = np.random.rand(3, 2, 2)
@@ -55,6 +58,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertEqual(tensor_shape.TensorShape([2]), dist.event_shape)
       self.assertEqual(tensor_shape.TensorShape([3, 2]), dist.batch_shape)
 
+  @test_util.run_deprecated_v1
   def testNproperty(self):
     alpha = [[1., 2, 3]]
     n = [[5.]]
@@ -63,6 +67,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertEqual([1, 1], dist.total_count.get_shape())
       self.assertAllClose(n, dist.total_count.eval())
 
+  @test_util.run_deprecated_v1
   def testAlphaProperty(self):
     alpha = [[1., 2, 3]]
     with self.cached_session():
@@ -70,6 +75,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertEqual([1, 3], dist.concentration.get_shape())
       self.assertAllClose(alpha, dist.concentration.eval())
 
+  @test_util.run_deprecated_v1
   def testPmfNandCountsAgree(self):
     alpha = [[1., 2, 3]]
     n = [[5.]]
@@ -83,6 +89,7 @@ class DirichletMultinomialTest(test.TestCase):
           "last-dimension must sum to `self.total_count`"):
         dist.prob([3., 3, 0]).eval()
 
+  @test_util.run_deprecated_v1
   def testPmfNonIntegerCounts(self):
     alpha = [[1., 2, 3]]
     n = [[5.]]
@@ -178,6 +185,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertAllClose([1 / 3., 2 / 5.], self.evaluate(pmf))
       self.assertAllEqual([2], pmf.get_shape())
 
+  @test_util.run_deprecated_v1
   def testPmfForOneVoteIsTheMeanWithOneRecordInput(self):
     # The probabilities of one vote falling into class k is the mean for class
     # k.
@@ -194,6 +202,7 @@ class DirichletMultinomialTest(test.TestCase):
         self.assertAllEqual([3], mean.shape)
         self.assertAllEqual([], pmf.shape)
 
+  @test_util.run_deprecated_v1
   def testMeanDoubleTwoVotes(self):
     # The probabilities of two votes falling into class k for
     # DirichletMultinomial(2, alpha) is twice as much as the probability of one
@@ -215,6 +224,7 @@ class DirichletMultinomialTest(test.TestCase):
         self.assertAllClose(mean2[class_num], 2 * mean1[class_num])
         self.assertAllEqual([3], mean1.shape)
 
+  @test_util.run_deprecated_v1
   def testCovarianceFromSampling(self):
     # We will test mean, cov, var, stddev on a DirichletMultinomial constructed
     # via broadcast between alpha, n.
@@ -412,6 +422,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertLess(5 * self.evaluate(pmf_different), self.evaluate(pmf_same))
       self.assertEqual((), pmf_same.get_shape())
 
+  @test_util.run_deprecated_v1
   def testNonStrictTurnsOffAllChecks(self):
     # Make totally invalid input.
     with self.cached_session():
@@ -421,6 +432,7 @@ class DirichletMultinomialTest(test.TestCase):
       dist = ds.DirichletMultinomial(n, alpha, validate_args=False)
       dist.prob(counts).eval()  # Should not raise.
 
+  @test_util.run_deprecated_v1
   def testSampleUnbiasedNonScalarBatch(self):
     with self.cached_session() as sess:
       dist = ds.DirichletMultinomial(
@@ -450,6 +462,7 @@ class DirichletMultinomialTest(test.TestCase):
       self.assertAllClose(
           actual_covariance_, sample_covariance_, atol=0., rtol=0.20)
 
+  @test_util.run_deprecated_v1
   def testSampleUnbiasedScalarBatch(self):
     with self.cached_session() as sess:
       dist = ds.DirichletMultinomial(
diff --git a/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py b/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py
index e35a8e1cdd..62b562387d 100644
--- a/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py
+++ b/tensorflow/python/kernel_tests/distributions/identity_bijector_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops.distributions import bijector_test_util
 from tensorflow.python.ops.distributions import identity_bijector
 from tensorflow.python.platform import test
@@ -41,6 +42,7 @@ class IdentityBijectorTest(test.TestCase):
         self.evaluate(
             bijector.forward_log_det_jacobian(x, event_ndims=3)))
 
+  @test_util.run_deprecated_v1
   def testScalarCongruency(self):
     with self.cached_session():
       bijector = identity_bijector.Identity()
diff --git a/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py b/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py
index b8bc2e55cf..1e967de570 100644
--- a/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py
+++ b/tensorflow/python/kernel_tests/distributions/kullback_leibler_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.distributions import kullback_leibler
 from tensorflow.python.ops.distributions import normal
@@ -45,6 +46,7 @@ class KLTest(test.TestCase):
     a = MyDist(loc=0.0, scale=1.0)
     self.assertEqual("OK", kullback_leibler.kl_divergence(a, a, name="OK"))
 
+  @test_util.run_deprecated_v1
   def testDomainErrorExceptions(self):
 
     class MyDistException(normal.Normal):
diff --git a/tensorflow/python/kernel_tests/distributions/normal_test.py b/tensorflow/python/kernel_tests/distributions/normal_test.py
index 6625a88843..f2a193e69b 100644
--- a/tensorflow/python/kernel_tests/distributions/normal_test.py
+++ b/tensorflow/python/kernel_tests/distributions/normal_test.py
@@ -511,6 +511,7 @@ class NormalTest(test.TestCase):
     self.assertAllEqual(self.evaluate(normal.event_shape_tensor()), [])
     self.assertEqual(normal.event_shape, tensor_shape.TensorShape([]))
 
+  @test_util.run_deprecated_v1
   def testNormalShapeWithPlaceholders(self):
     mu = array_ops.placeholder(dtype=dtypes.float32)
     sigma = array_ops.placeholder(dtype=dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/distributions/special_math_test.py b/tensorflow/python/kernel_tests/distributions/special_math_test.py
index 0f800b95fa..d97fcfa655 100644
--- a/tensorflow/python/kernel_tests/distributions/special_math_test.py
+++ b/tensorflow/python/kernel_tests/distributions/special_math_test.py
@@ -104,6 +104,7 @@ class NdtriTest(test.TestCase):
     x = special_math.ndtri(p)
     self.assertAllClose(expected_x, self.evaluate(x), atol=0.)
 
+  @test_util.run_deprecated_v1
   def testNdtriDynamicShape(self):
     """Verifies that ndtri computation is correct."""
     with self.cached_session() as sess:
@@ -213,9 +214,11 @@ class NdtrTest(test.TestCase):
         rtol=error_spec.rtol,
         atol=error_spec.atol)
 
+  @test_util.run_deprecated_v1
   def test_float32(self):
     self._test_grid(np.float32, self._grid32, self._error32)
 
+  @test_util.run_deprecated_v1
   def test_float64(self):
     self._test_grid(np.float64, self._grid64, self._error64)
 
@@ -338,10 +341,12 @@ class NdtrGradientTest(test.TestCase):
           rtol=error_spec.rtol,
           atol=error_spec.atol)
 
+  @test_util.run_deprecated_v1
   def test_float32(self):
     self._test_grad_accuracy(np.float32, self._grid, self._error32)
     self._test_grad_finite(np.float32)
 
+  @test_util.run_deprecated_v1
   def test_float64(self):
     self._test_grad_accuracy(np.float64, self._grid, self._error64)
     self._test_grad_finite(np.float64)
@@ -418,6 +423,7 @@ class LogCDFLaplaceTest(test.TestCase):
           rtol=error_spec.rtol,
           atol=error_spec.atol)
 
+  @test_util.run_deprecated_v1
   def test_float32_lower_and_mid_segment_scipy_float32_ok(self):
     # Choose values mild enough that we can use scipy in float32, which will
     # allow for a high accuracy match to scipy (since we both use float32).
@@ -427,6 +433,7 @@ class LogCDFLaplaceTest(test.TestCase):
         GridSpec(min=-10, max=self.CUTOFF_FLOAT32_UPPER - 5, shape=[100]),
         ErrorSpec(rtol=5e-4, atol=0))
 
+  @test_util.run_deprecated_v1
   def test_float32_all_segments_with_scipy_float64_ok(self):
     # Choose values outside the range where scipy float32 works.
     # Let scipy use float64.  This means we
@@ -437,6 +444,7 @@ class LogCDFLaplaceTest(test.TestCase):
         GridSpec(min=-50, max=self.CUTOFF_FLOAT32_UPPER + 5, shape=[100]),
         ErrorSpec(rtol=0.05, atol=0))
 
+  @test_util.run_deprecated_v1
   def test_float32_extreme_values_result_and_gradient_finite_and_nonzero(self):
     with self.cached_session() as sess:
       # On the lower branch, log_cdf_laplace(x) = x, so we know this will be
@@ -456,6 +464,7 @@ class LogCDFLaplaceTest(test.TestCase):
       self.assertFalse(np.any(actual_ == 0))
       self.assertFalse(np.any(grad_ == 0))
 
+  @test_util.run_deprecated_v1
   def test_float64_extreme_values_result_and_gradient_finite_and_nonzero(self):
     with self.cached_session() as sess:
       # On the lower branch, log_cdf_laplace(x) = x, so we know this will be
diff --git a/tensorflow/python/kernel_tests/distributions/util_test.py b/tensorflow/python/kernel_tests/distributions/util_test.py
index d3fa513f05..030ad601bf 100644
--- a/tensorflow/python/kernel_tests/distributions/util_test.py
+++ b/tensorflow/python/kernel_tests/distributions/util_test.py
@@ -59,6 +59,7 @@ def _logit(x):
 
 class AssertCloseTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAssertIntegerForm(self):
     # This should only be detected as an integer.
     x = array_ops.placeholder(dtypes.float32)
@@ -112,6 +113,7 @@ class MaybeGetStaticTest(test.TestCase):
     self.assertAllClose(
         np.array(2.), du.maybe_get_static_value(x, dtype=np.float64))
 
+  @test_util.run_deprecated_v1
   def testGetStaticPlaceholder(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[1])
     self.assertEqual(None, du.maybe_get_static_value(x))
@@ -235,6 +237,7 @@ class GetLogitsAndProbsTest(test.TestCase):
         probs=p4, multidimensional=True, validate_args=False)
     self.evaluate(prob)
 
+  @test_util.run_deprecated_v1
   def testProbsMultidimShape(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -249,6 +252,7 @@ class GetLogitsAndProbsTest(test.TestCase):
             probs=p, multidimensional=True, validate_args=True)
         prob.eval(feed_dict={p: np.ones([int(2**11+1)])})
 
+  @test_util.run_deprecated_v1
   def testLogitsMultidimShape(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -266,6 +270,7 @@ class GetLogitsAndProbsTest(test.TestCase):
 
 class EmbedCheckCategoricalEventShapeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testTooSmall(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -280,6 +285,7 @@ class EmbedCheckCategoricalEventShapeTest(test.TestCase):
             param)
         checked_param.eval(feed_dict={param: np.ones([1])})
 
+  @test_util.run_deprecated_v1
   def testTooLarge(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -305,6 +311,7 @@ class EmbedCheckCategoricalEventShapeTest(test.TestCase):
 
 class EmbedCheckIntegerCastingClosedTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCorrectlyAssertsNonnegative(self):
     with self.cached_session():
       with self.assertRaisesOpError("Elements must be non-negative"):
@@ -313,6 +320,7 @@ class EmbedCheckIntegerCastingClosedTest(test.TestCase):
             x, target_dtype=dtypes.int16)
         x_checked.eval(feed_dict={x: np.array([1, -1], dtype=np.float16)})
 
+  @test_util.run_deprecated_v1
   def testCorrectlyAssersIntegerForm(self):
     with self.cached_session():
       with self.assertRaisesOpError("Elements must be int16-equivalent."):
@@ -321,6 +329,7 @@ class EmbedCheckIntegerCastingClosedTest(test.TestCase):
             x, target_dtype=dtypes.int16)
         x_checked.eval(feed_dict={x: np.array([1, 1.5], dtype=np.float16)})
 
+  @test_util.run_deprecated_v1
   def testCorrectlyAssertsLargestPossibleInteger(self):
     with self.cached_session():
       with self.assertRaisesOpError("Elements cannot exceed 32767."):
@@ -329,6 +338,7 @@ class EmbedCheckIntegerCastingClosedTest(test.TestCase):
             x, target_dtype=dtypes.int16)
         x_checked.eval(feed_dict={x: np.array([1, 2**15], dtype=np.int32)})
 
+  @test_util.run_deprecated_v1
   def testCorrectlyAssertsSmallestPossibleInteger(self):
     with self.cached_session():
       with self.assertRaisesOpError("Elements cannot be smaller than 0."):
@@ -369,6 +379,7 @@ class LogCombinationsTest(test.TestCase):
 
 class DynamicShapeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSameDynamicShape(self):
     with self.cached_session():
       scalar = constant_op.constant(2.0)
@@ -493,6 +504,7 @@ class RotateTransposeTest(test.TestCase):
             self._np_rotate_transpose(x, shift), self.evaluate(y))
         self.assertAllEqual(np.roll(x.shape, shift), y.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testRollDynamic(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(dtypes.float32)
@@ -511,6 +523,7 @@ class RotateTransposeTest(test.TestCase):
 
 class PickVectorTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCorrectlyPicksVector(self):
     with self.cached_session():
       x = np.arange(10, 12)
@@ -529,36 +542,42 @@ class PickVectorTest(test.TestCase):
 
 class PreferStaticRankTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNonEmptyConstantTensor(self):
     x = array_ops.zeros((2, 3, 4))
     rank = du.prefer_static_rank(x)
     self.assertIsInstance(rank, np.ndarray)
     self.assertEqual(3, rank)
 
+  @test_util.run_deprecated_v1
   def testEmptyConstantTensor(self):
     x = constant_op.constant([])
     rank = du.prefer_static_rank(x)
     self.assertIsInstance(rank, np.ndarray)
     self.assertEqual(1, rank)
 
+  @test_util.run_deprecated_v1
   def testScalarTensor(self):
     x = constant_op.constant(1.)
     rank = du.prefer_static_rank(x)
     self.assertIsInstance(rank, np.ndarray)
     self.assertEqual(0, rank)
 
+  @test_util.run_deprecated_v1
   def testDynamicRankEndsUpBeingNonEmpty(self):
     x = array_ops.placeholder(np.float64, shape=None)
     rank = du.prefer_static_rank(x)
     with self.cached_session():
       self.assertAllEqual(2, rank.eval(feed_dict={x: np.zeros((2, 3))}))
 
+  @test_util.run_deprecated_v1
   def testDynamicRankEndsUpBeingEmpty(self):
     x = array_ops.placeholder(np.int32, shape=None)
     rank = du.prefer_static_rank(x)
     with self.cached_session():
       self.assertAllEqual(1, rank.eval(feed_dict={x: []}))
 
+  @test_util.run_deprecated_v1
   def testDynamicRankEndsUpBeingScalar(self):
     x = array_ops.placeholder(np.int32, shape=None)
     rank = du.prefer_static_rank(x)
@@ -568,36 +587,42 @@ class PreferStaticRankTest(test.TestCase):
 
 class PreferStaticShapeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNonEmptyConstantTensor(self):
     x = array_ops.zeros((2, 3, 4))
     shape = du.prefer_static_shape(x)
     self.assertIsInstance(shape, np.ndarray)
     self.assertAllEqual(np.array([2, 3, 4]), shape)
 
+  @test_util.run_deprecated_v1
   def testEmptyConstantTensor(self):
     x = constant_op.constant([])
     shape = du.prefer_static_shape(x)
     self.assertIsInstance(shape, np.ndarray)
     self.assertAllEqual(np.array([0]), shape)
 
+  @test_util.run_deprecated_v1
   def testScalarTensor(self):
     x = constant_op.constant(1.)
     shape = du.prefer_static_shape(x)
     self.assertIsInstance(shape, np.ndarray)
     self.assertAllEqual(np.array([]), shape)
 
+  @test_util.run_deprecated_v1
   def testDynamicShapeEndsUpBeingNonEmpty(self):
     x = array_ops.placeholder(np.float64, shape=None)
     shape = du.prefer_static_shape(x)
     with self.cached_session():
       self.assertAllEqual((2, 3), shape.eval(feed_dict={x: np.zeros((2, 3))}))
 
+  @test_util.run_deprecated_v1
   def testDynamicShapeEndsUpBeingEmpty(self):
     x = array_ops.placeholder(np.int32, shape=None)
     shape = du.prefer_static_shape(x)
     with self.cached_session():
       self.assertAllEqual(np.array([0]), shape.eval(feed_dict={x: []}))
 
+  @test_util.run_deprecated_v1
   def testDynamicShapeEndsUpBeingScalar(self):
     x = array_ops.placeholder(np.int32, shape=None)
     shape = du.prefer_static_shape(x)
@@ -607,24 +632,28 @@ class PreferStaticShapeTest(test.TestCase):
 
 class PreferStaticValueTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNonEmptyConstantTensor(self):
     x = array_ops.zeros((2, 3, 4))
     value = du.prefer_static_value(x)
     self.assertIsInstance(value, np.ndarray)
     self.assertAllEqual(np.zeros((2, 3, 4)), value)
 
+  @test_util.run_deprecated_v1
   def testEmptyConstantTensor(self):
     x = constant_op.constant([])
     value = du.prefer_static_value(x)
     self.assertIsInstance(value, np.ndarray)
     self.assertAllEqual(np.array([]), value)
 
+  @test_util.run_deprecated_v1
   def testScalarTensor(self):
     x = constant_op.constant(1.)
     value = du.prefer_static_value(x)
     self.assertIsInstance(value, np.ndarray)
     self.assertAllEqual(np.array(1.), value)
 
+  @test_util.run_deprecated_v1
   def testDynamicValueEndsUpBeingNonEmpty(self):
     x = array_ops.placeholder(np.float64, shape=None)
     value = du.prefer_static_value(x)
@@ -632,12 +661,14 @@ class PreferStaticValueTest(test.TestCase):
       self.assertAllEqual(np.zeros((2, 3)),
                           value.eval(feed_dict={x: np.zeros((2, 3))}))
 
+  @test_util.run_deprecated_v1
   def testDynamicValueEndsUpBeingEmpty(self):
     x = array_ops.placeholder(np.int32, shape=None)
     value = du.prefer_static_value(x)
     with self.cached_session():
       self.assertAllEqual(np.array([]), value.eval(feed_dict={x: []}))
 
+  @test_util.run_deprecated_v1
   def testDynamicValueEndsUpBeingScalar(self):
     x = array_ops.placeholder(np.int32, shape=None)
     value = du.prefer_static_value(x)
@@ -698,43 +729,55 @@ class FillTriangularTest(test.TestCase):
     self.assertAllClose(expected, actual_, rtol=1e-8, atol=1e-9)
     self.assertAllClose(x_, grad_actual_, rtol=1e-8, atol=1e-9)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakes1x1TriLower(self):
     self._run_test(self._rng.randn(3, int(1*2/2)))
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesNoBatchTriLower(self):
     self._run_test(self._rng.randn(int(4*5/2)))
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatchTriLower(self):
     self._run_test(self._rng.randn(2, 3, int(3*4/2)))
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatchTriLowerUnknownShape(self):
     self._run_test(self._rng.randn(2, 3, int(3*4/2)), use_deferred_shape=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatch7x7TriLowerUnknownShape(self):
     self._run_test(self._rng.randn(2, 3, int(7*8/2)), use_deferred_shape=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatch7x7TriLower(self):
     self._run_test(self._rng.randn(2, 3, int(7*8/2)))
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakes1x1TriUpper(self):
     self._run_test(self._rng.randn(3, int(1*2/2)), upper=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesNoBatchTriUpper(self):
     self._run_test(self._rng.randn(int(4*5/2)), upper=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatchTriUpper(self):
     self._run_test(self._rng.randn(2, 2, int(3*4/2)), upper=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatchTriUpperUnknownShape(self):
     self._run_test(self._rng.randn(2, 2, int(3*4/2)),
                    use_deferred_shape=True,
                    upper=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatch7x7TriUpperUnknownShape(self):
     self._run_test(self._rng.randn(2, 3, int(7*8/2)),
                    use_deferred_shape=True,
                    upper=True)
 
+  @test_util.run_deprecated_v1
   def testCorrectlyMakesBatch7x7TriUpper(self):
     self._run_test(self._rng.randn(2, 3, int(7*8/2)), upper=True)
 
@@ -773,6 +816,7 @@ class ReduceWeightedLogSumExp(test.TestCase):
       m = np.squeeze(m, axis=axis)
     return m + np.log(sgn * sum_), sgn
 
+  @test_util.run_deprecated_v1
   def testNoWeights(self):
     logx_ = np.array([[0., -1, 1000.],
                       [0, 1, -1000.],
@@ -903,6 +947,7 @@ class SoftplusTest(test.TestCase):
     self.assertAllEqual(np.ones_like(tf_softplus_inverse).astype(np.bool),
                         np.isfinite(tf_softplus_inverse))
 
+  @test_util.run_deprecated_v1
   def testNumbers(self):
     for t in [np.float16, np.float32, np.float64]:
       lower = {np.float16: -15, np.float32: -50, np.float64: -50}.get(t, -100)
@@ -933,6 +978,7 @@ class SoftplusTest(test.TestCase):
           ],
           use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -949,6 +995,7 @@ class SoftplusTest(test.TestCase):
     tf_logging.vlog(2, "softplus (float) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testInverseSoftplusGradientNeverNan(self):
     with self.cached_session():
       # Note that this range contains both zero and inf.
@@ -958,6 +1005,7 @@ class SoftplusTest(test.TestCase):
       # Equivalent to `assertAllFalse` (if it existed).
       self.assertAllEqual(np.zeros_like(grads).astype(np.bool), np.isnan(grads))
 
+  @test_util.run_deprecated_v1
   def testInverseSoftplusGradientFinite(self):
     with self.cached_session():
       # This range of x is all finite, and so is 1 / x.  So the
diff --git a/tensorflow/python/kernel_tests/duplicate_op_test.py b/tensorflow/python/kernel_tests/duplicate_op_test.py
index 654267a582..fef3127d4a 100644
--- a/tensorflow/python/kernel_tests/duplicate_op_test.py
+++ b/tensorflow/python/kernel_tests/duplicate_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import os
 
 from tensorflow.python.framework import load_library
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import resource_loader
 from tensorflow.python.platform import test
@@ -27,6 +28,7 @@ from tensorflow.python.platform import test
 
 class DuplicateOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     library_filename = os.path.join(resource_loader.get_data_files_path(),
                                     'duplicate_op.so')
diff --git a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
index 3622fde3f3..8c44819407 100644
--- a/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_partition_op_test.py
@@ -25,6 +25,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import gradients_impl
@@ -34,6 +35,7 @@ from tensorflow.python.platform import test
 
 class DynamicPartitionTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSimpleOneDimensional(self):
     with self.session(use_gpu=True) as sess:
       data = constant_op.constant([0, 13, 2, 39, 4, 17], dtype=dtypes.float32)
@@ -54,6 +56,7 @@ class DynamicPartitionTest(test.TestCase):
     self.assertEqual([None], partitions[2].get_shape().as_list())
     self.assertEqual([None], partitions[3].get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testSimpleTwoDimensional(self):
     with self.session(use_gpu=True) as sess:
       data = constant_op.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
@@ -151,6 +154,7 @@ class DynamicPartitionTest(test.TestCase):
                                  dtype=np.float64).reshape(-1, 4),
                         partition_vals[3])
 
+  @test_util.run_deprecated_v1
   def testHigherRank(self):
     np.random.seed(7)
     with self.session(use_gpu=True) as sess:
@@ -287,6 +291,7 @@ class DynamicPartitionTest(test.TestCase):
     for i in range(40):
       self.assertAllEqual([], partition_vals[i])
 
+  @test_util.run_deprecated_v1
   def testErrorIndexOutOfRange(self):
     with self.cached_session() as sess:
       data = constant_op.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
@@ -297,6 +302,7 @@ class DynamicPartitionTest(test.TestCase):
       with self.assertRaisesOpError(r"partitions\[2\] = 99 is not in \[0, 4\)"):
         self.evaluate(partitions)
 
+  @test_util.run_deprecated_v1
   def testScalarIndexOutOfRange(self):
     with self.cached_session() as sess:
       bad = 17
@@ -305,6 +311,7 @@ class DynamicPartitionTest(test.TestCase):
       with self.assertRaisesOpError(r"partitions = 17 is not in \[0, 7\)"):
         self.evaluate(partitions)
 
+  @test_util.run_deprecated_v1
   def testHigherRankIndexOutOfRange(self):
     with self.cached_session() as sess:
       shape = (2, 3)
@@ -320,6 +327,7 @@ class DynamicPartitionTest(test.TestCase):
               r"partitions\[%d,%d\] = 17 is not in \[0, 7\)" % (i, j)):
             sess.run(partitions, feed_dict={indices: bad})
 
+  @test_util.run_deprecated_v1
   def testErrorWrongDimsIndices(self):
     data = constant_op.constant([[0], [1], [2]])
     indices = constant_op.constant([[0], [0]])
diff --git a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
index 3d063c4e0e..4f338880aa 100644
--- a/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
+++ b/tensorflow/python/kernel_tests/dynamic_stitch_op_test.py
@@ -47,6 +47,7 @@ class DynamicStitchTestBase(object):
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([2], stitched_t.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testShapeInferenceForScalarWithNonConstantIndices(self):
     with test_util.use_gpu():
       indices = [
@@ -134,6 +135,7 @@ class DynamicStitchTestBase(object):
       # Dimension 0 is max(flatten(indices))+1.
       self.assertEqual([8, 2], stitched_t.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testHigherRank(self):
     with self.session(use_gpu=True) as sess:
       indices = [
@@ -160,6 +162,7 @@ class DynamicStitchTestBase(object):
       for datum, grad in zip(data, sess.run(grads[3:])):
         self.assertAllEqual(7. * self.evaluate(datum), grad)
 
+  @test_util.run_deprecated_v1
   def testErrorIndicesMultiDimensional(self):
     indices = [
         constant_op.constant([0, 4, 7]),
@@ -172,6 +175,7 @@ class DynamicStitchTestBase(object):
     with self.assertRaises(ValueError):
       self.stitch_op(indices, data)
 
+  @test_util.run_deprecated_v1
   def testErrorDataNumDimsMismatch(self):
     indices = [
         constant_op.constant([0, 4, 7]),
@@ -184,6 +188,7 @@ class DynamicStitchTestBase(object):
     with self.assertRaises(ValueError):
       self.stitch_op(indices, data)
 
+  @test_util.run_deprecated_v1
   def testErrorDataDimSizeMismatch(self):
     indices = [
         constant_op.constant([0, 4, 5]),
@@ -196,6 +201,7 @@ class DynamicStitchTestBase(object):
     with self.assertRaises(ValueError):
       self.stitch_op(indices, data)
 
+  @test_util.run_deprecated_v1
   def testErrorDataAndIndicesSizeMismatch(self):
     indices = [
         constant_op.constant([0, 4, 7]),
@@ -233,6 +239,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([2], stitched_t.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testHigherRank(self):
     with self.session(use_gpu=True) as sess:
       indices = [
@@ -271,6 +278,7 @@ class ParallelDynamicStitchTest(DynamicStitchTestBase, test.TestCase):
         # Dimension 0 is max(flatten(indices))+1.
         self.assertEqual([2], stitched_t.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testHigherRankGPU(self):
     with self.cached_session() as sess:
       indices = [
diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py
index 39c0575cd5..6019245d0f 100644
--- a/tensorflow/python/kernel_tests/embedding_ops_test.py
+++ b/tensorflow/python/kernel_tests/embedding_ops_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import embedding_ops
@@ -87,16 +88,19 @@ class ScatterAddSubTest(test.TestCase):
             vals_shape[0], -1)[i, :])
     self.assertTrue(all((p_init == result).ravel()))
 
+  @test_util.run_deprecated_v1
   def testNoRepetitions(self):
     self._TestCase([2, 2], [1])
     self._TestCase([4, 4, 4], [2, 0])
     self._TestCase([43, 20, 10, 10], [42, 5, 6, 1, 3, 5, 7, 9])
 
+  @test_util.run_deprecated_v1
   def testWithRepetitions(self):
     self._TestCase([2, 2], [1, 1])
     self._TestCase([5, 3, 9, 5], [2, 0, 4, 1, 3, 1, 4, 0, 4, 3])
     self._TestCase([32, 4, 4], [31] * 8)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     # Random shapes of rank 4, random indices
     for _ in range(5):
@@ -104,6 +108,7 @@ class ScatterAddSubTest(test.TestCase):
       indices = np.random.randint(shape[0], size=2 * shape[0])
       self._TestCase(_AsLong(list(shape)), list(indices))
 
+  @test_util.run_deprecated_v1
   def testSubRandom(self):
     # Random shapes of rank 4, random indices
     for _ in range(5):
@@ -111,6 +116,7 @@ class ScatterAddSubTest(test.TestCase):
       indices = np.random.randint(shape[0], size=2 * shape[0])
       self._TestCase(_AsLong(list(shape)), list(indices), state_ops.scatter_sub)
 
+  @test_util.run_deprecated_v1
   def testWrongShape(self):
     # Indices and values mismatch.
     var = variables.Variable(
@@ -241,6 +247,7 @@ class EmbeddingLookupTest(test.TestCase):
   # both the ids are in the first shard, one of the resulting lookup
   # vector is going to be empty. The subsequent DivOp fails because of that.
   # TODO(keveman): Disabling the test until the underlying problem is fixed.
+  @test_util.run_deprecated_v1
   def testSimpleSharded(self):
     with self.cached_session():
       num_shards = 2
@@ -257,6 +264,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testMaxNorm(self):
     with self.cached_session():
       embeddings = constant_op.constant([[2.0]])
@@ -267,6 +275,7 @@ class EmbeddingLookupTest(test.TestCase):
 
       self.assertAllEqual(embedding.eval(), [[1.0]])
 
+  @test_util.run_deprecated_v1
   def testMaxNormNontrivial(self):
     with self.cached_session():
       embeddings = constant_op.constant([[2.0, 4.0], [3.0, 1.0]])
@@ -280,6 +289,7 @@ class EmbeddingLookupTest(test.TestCase):
       normalized = embeddings / array_ops.stack([norms, norms], axis=1)
       self.assertAllEqual(embedding.eval(), 2 * self.evaluate(normalized))
 
+  @test_util.run_deprecated_v1
   def testSimpleShardedPartitionedVariable(self):
     with self.cached_session() as sess:
       num_shards = 2
@@ -302,6 +312,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testSimpleShardedPartitionedResourceVariable(self):
     with self.cached_session() as sess:
       num_shards = 2
@@ -325,6 +336,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testShardedModPartitioningInt32Ids(self):
     with self.cached_session():
       num_shards = 5
@@ -347,6 +359,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testShardedModPartitioningInt64Ids(self):
     with self.cached_session():
       num_shards = 5
@@ -369,6 +382,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testShardedDivPartitioningInt32Ids(self):
     with self.cached_session():
       num_shards = 5
@@ -393,6 +407,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testShardedDivPartitioningInt32IdsPartitionedVariable(self):
     with self.cached_session():
       num_shards = 5
@@ -418,6 +433,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testShardedDivPartitioningInt64Ids(self):
     with self.cached_session():
       num_shards = 5
@@ -442,6 +458,7 @@ class EmbeddingLookupTest(test.TestCase):
     self.assertAllEqual(np_result, tf_result)
     self.assertShapeEqual(np_result, embedding)
 
+  @test_util.run_deprecated_v1
   def testShardedDivPartitioningUnknownParamShape(self):
     with self.cached_session():
       num_shards = 5
@@ -468,6 +485,7 @@ class EmbeddingLookupTest(test.TestCase):
         params, id_vals, num_shards, vocab_size, partition_strategy="div")
     self.assertAllEqual(np_result, tf_result)
 
+  @test_util.run_deprecated_v1
   def testGradientsEmbeddingLookup(self):
     vocab_size = 9
     num_ids = 10
@@ -488,6 +506,7 @@ class EmbeddingLookupTest(test.TestCase):
               x, x_shape, y, y_shape, x_init_value=x_init_value)
         self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientsEmbeddingLookupWithComputedParams(self):
     vocab_size = 9
     num_ids = 5
@@ -526,6 +545,7 @@ class EmbeddingLookupTest(test.TestCase):
         ids = constant_op.constant([0, 1, 1, 17], dtype=dtypes.int32)
       embedding_ops.embedding_lookup(p, ids)
 
+  @test_util.run_deprecated_v1
   def testHigherRank(self):
     np.random.seed(8)
     with self.cached_session():
@@ -546,6 +566,7 @@ class EmbeddingLookupTest(test.TestCase):
             sharded = embedding_ops.embedding_lookup(split_params, ids).eval()
             self.assertAllEqual(simple, sharded)
 
+  @test_util.run_deprecated_v1
   def testHigherRankMaxNorm(self):
     np.random.seed(8)
     with self.cached_session():
@@ -574,6 +595,7 @@ class EmbeddingLookupTest(test.TestCase):
                 split_params, ids, max_norm=1.0).eval()
             self.assertAllEqual(simple, sharded)
 
+  @test_util.run_deprecated_v1
   def testTransform(self):
     # This tests all combinations of:
     #   - ids rank 0, 1, >1
@@ -648,6 +670,7 @@ class EmbeddingLookupSparseTest(test.TestCase):
       index += num_val
     return grouped_vals
 
+  @test_util.run_deprecated_v1
   def testEmbeddingLookupSparse(self):
     vocab_size = 13
     batch_size = 10
@@ -706,6 +729,7 @@ class EmbeddingLookupSparseTest(test.TestCase):
         atol = rtol
         self.assertAllClose(np_embedding_sum, tf_embedding_sum, rtol, atol)
 
+  @test_util.run_deprecated_v1
   def testGradientsEmbeddingLookupSparse(self):
     vocab_size = 12
     batch_size = 4
@@ -733,6 +757,7 @@ class EmbeddingLookupSparseTest(test.TestCase):
             x, x_shape, y, y_shape, x_init_value=x_init_value)
       self.assertLess(err, 1e-5 if dtype == dtypes.float64 else 2e-3)
 
+  @test_util.run_deprecated_v1
   def testIncompatibleShapes(self):
     with self.cached_session():
       x, _, _ = _EmbeddingParams(1, 10, dtype=dtypes.float32)
@@ -820,6 +845,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
 
     return sparse_ids, sparse_weights
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_return_zero_vector(self):
     with self.cached_session():
       embedding_weights = self._random_weights()
@@ -833,6 +859,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
           [(1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) /
            3.0, [0] * 4, [0] * 4, embedding_weights[0][2], [0] * 4])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_return_special_vector(self):
     with self.cached_session():
       embedding_weights = self._random_weights()
@@ -847,6 +874,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
            3.0, embedding_weights[0][3], embedding_weights[0][3],
            embedding_weights[0][2], embedding_weights[0][3]])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_no_weights(self):
     with self.cached_session():
       embedding_weights = self._random_weights()
@@ -861,6 +889,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
            [0] * 4, embedding_weights[0][2], (
                embedding_weights[0][0] + embedding_weights[0][1]) / 2.0])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_partitioned(self):
     with self.cached_session():
       embedding_weights = self._random_weights(num_shards=3)
@@ -875,6 +904,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
                            [0] * 4, [0] * 4, embedding_weights[2],
                            (embedding_weights[0] + embedding_weights[1]) / 2.0])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_partitioned_inconsistent_weights(self):
     with self.cached_session():
       embedding_weights = self._random_weights(num_shards=3)
@@ -890,6 +920,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       self.assertRaises(ValueError, embedding_ops.safe_embedding_lookup_sparse,
                         embedding_weights, sparse_ids, sparse_weights)
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_3d_return_zero_vector(self):
     with self.cached_session():
       embedding_weights = self._random_weights()
@@ -903,6 +934,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
           [0] * 4, [0] * 4
       ], [embedding_weights[0][2], [0] * 4, [0] * 4]])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_3d_return_special_vector(self):
     with self.cached_session():
       embedding_weights = self._random_weights()
@@ -919,6 +951,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
                 embedding_weights[0][3]
             ]])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_3d_no_weights(self):
     with self.cached_session():
       embedding_weights = self._random_weights()
@@ -935,6 +968,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
               (embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4
           ]])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_3d_partitioned(self):
     with self.cached_session():
       embedding_weights = self._random_weights(num_shards=3)
@@ -951,6 +985,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
           (embedding_weights[0] + embedding_weights[1]) / 2.0, [0] * 4
       ]])
 
+  @test_util.run_deprecated_v1
   def test_safe_embedding_lookup_sparse_3d_partitioned_inconsistent_weights(
       self):
     with self.cached_session():
@@ -970,6 +1005,7 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
 
 class DynamicStitchOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCint32Cpu(self):
     with self.session(use_gpu=False):
       indices = [
@@ -983,6 +1019,7 @@ class DynamicStitchOpTest(test.TestCase):
       self.assertAllEqual(
           data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testCint32Gpu(self):
     with self.session(use_gpu=True):
       indices = [
@@ -996,6 +1033,7 @@ class DynamicStitchOpTest(test.TestCase):
       self.assertAllEqual(
           data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testInt32Cpu(self):
     with self.session(use_gpu=False):
       indices = [
@@ -1009,6 +1047,7 @@ class DynamicStitchOpTest(test.TestCase):
       self.assertAllEqual(
           data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testInt32Gpu(self):
     with self.session(use_gpu=True):
       indices = [
@@ -1022,6 +1061,7 @@ class DynamicStitchOpTest(test.TestCase):
       self.assertAllEqual(
           data_flow_ops.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testSumGradArgs(self):
     with self.session(use_gpu=False):
       indices = [
@@ -1036,6 +1076,7 @@ class DynamicStitchOpTest(test.TestCase):
           data_flow_ops.dynamic_stitch(indices, values).eval(), [2, 3, 1, 1])
 
   # We expect that the values are merged in order.
+  @test_util.run_deprecated_v1
   def testStitchOrder(self):
     with self.cached_session():
       indices = []
@@ -1051,6 +1092,7 @@ class DynamicStitchOpTest(test.TestCase):
 
 class ParallelDynamicStitchOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCint32Cpu(self):
     with self.session(use_gpu=False):
       indices = [
@@ -1065,6 +1107,7 @@ class ParallelDynamicStitchOpTest(test.TestCase):
           data_flow_ops.parallel_dynamic_stitch(indices, values).eval(),
           [12, 23, 1, 2, 34, 3, 45])
 
+  @test_util.run_deprecated_v1
   def testInt32Cpu(self):
     with self.session(use_gpu=False):
       indices = [
@@ -1079,6 +1122,7 @@ class ParallelDynamicStitchOpTest(test.TestCase):
           data_flow_ops.parallel_dynamic_stitch(indices, values).eval(),
           [12, 23, 1, 2, 3, 34, 45, 56])
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     with self.session(use_gpu=False):
       indices = [ops.convert_to_tensor([0, 1]), ops.convert_to_tensor([2, 3])]
diff --git a/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py b/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py
index 7d9d4e5175..7ba2dc6c20 100644
--- a/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py
+++ b/tensorflow/python/kernel_tests/extract_image_patches_grad_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed as random_seed_lib
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -78,6 +79,7 @@ class ExtractImagePatchesGradTest(test.TestCase):
       },
   ]
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     # Set graph seed for determinism.
     random_seed = 42
@@ -102,6 +104,7 @@ class ExtractImagePatchesGradTest(test.TestCase):
           print('extract_image_patches gradient err: %.4e' % err)
           self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testConstructGradientWithLargeImages(self):
     batch_size = 4
     height = 1024
diff --git a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
index 272adecfb8..0d5928aefa 100644
--- a/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_avg_pool_op_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
@@ -281,6 +282,7 @@ class FractionalAvgTest(test.TestCase):
     self._ValidateFractionalAvgPoolResult(rand_mat, [1, 2, 2, 1], pseudo_random,
                                           overlapping)
 
+  @test_util.run_deprecated_v1
   def testDifferentInputTensorShape(self):
     """Runs the operation in one session with different input tensor shapes."""
     with self.cached_session() as sess:
@@ -427,6 +429,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               self.assertShapeEqual(input_backprop, fap_input_backprop_tensor)
               self.assertAllClose(input_backprop, fap_input_backprop)
 
+  @test_util.run_deprecated_v1
   def testAllInputOptionsThroughGradientError(self):
     input_shape = (1, 7, 13, 1)
     input_data = self._GenerateRandomInputTensor(input_shape)
@@ -455,6 +458,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
               delta=1e-2)
           self.assertLess(gradient_error, error_margin)
 
+  @test_util.run_deprecated_v1
   def testDifferentTensorShapesThroughGradientError(self):
     pseudo_random = True
     overlapping = True
@@ -486,6 +490,7 @@ class FractionalAvgPoolGradTest(test.TestCase):
                   delta=1e-2)
               self.assertLess(gradient_error, error_margin)
 
+  @test_util.run_deprecated_v1
   def testLargePoolingRatioThroughGradientError(self):
     input_shape = (1, 17, 23, 1)
     input_data = self._GenerateRandomInputTensor(input_shape)
diff --git a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
index 9b1e73b318..fa886cc215 100644
--- a/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
+++ b/tensorflow/python/kernel_tests/fractional_max_pool_op_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
@@ -278,6 +279,7 @@ class FractionalMaxPoolTest(test.TestCase):
     self._ValidateFractionalMaxPoolResult(rand_mat, [1, 2, 2, 1], pseudo_random,
                                           overlapping)
 
+  @test_util.run_deprecated_v1
   def testDifferentInputTensorShape(self):
     """Runs the operation in one session with different input tensor shapes."""
     with self.cached_session() as sess:
@@ -430,6 +432,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
               self.assertShapeEqual(input_backprop, fmp_input_backprop_tensor)
               self.assertAllClose(input_backprop, fmp_input_backprop)
 
+  @test_util.run_deprecated_v1
   def testAllInputOptionsThroughGradientError(self):
     input_shape = (1, 7, 13, 1)
     input_data = self._GenerateUniqueRandomInputTensor(input_shape)
@@ -460,6 +463,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
               delta=1e-2)
           self.assertLess(gradient_error, error_margin)
 
+  @test_util.run_deprecated_v1
   def testDifferentTensorShapesThroughGradientError(self):
     pseudo_random = True
     overlapping = True
@@ -493,6 +497,7 @@ class FractionalMaxPoolGradTest(test.TestCase):
                   delta=1e-2)
               self.assertLess(gradient_error, error_margin)
 
+  @test_util.run_deprecated_v1
   def testLargePoolingRatioThroughGradientError(self):
     input_shape = (1, 17, 23, 1)
     input_data = self._GenerateUniqueRandomInputTensor(input_shape)
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 23b3c7e1cc..c489623fe5 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -101,6 +101,7 @@ class FunctionalOpsTest(test.TestCase):
                              (elems, other_elems), initializer)
     self.assertAllEqual([1.0, 2.0, 3.0], self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testFoldl_Scoped(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope("root") as varscope:
@@ -153,6 +154,7 @@ class FunctionalOpsTest(test.TestCase):
                              initializer)
     self.assertAllEqual(1, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testFoldr_Scoped(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope("root") as varscope:
@@ -173,6 +175,7 @@ class FunctionalOpsTest(test.TestCase):
         self.assertAllEqual(1282, self.evaluate(r))
 
   # pylint: disable=unnecessary-lambda
+  @test_util.run_deprecated_v1
   def testFold_Grad(self):
     with self.cached_session():
       elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data")
@@ -214,6 +217,7 @@ class FunctionalOpsTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "not a scalar"):
       functional_ops.map_fn(lambda x: x, 1)
 
+  @test_util.run_deprecated_v1
   def testMap_Scoped(self):
     with self.cached_session() as sess:
 
@@ -245,6 +249,7 @@ class FunctionalOpsTest(test.TestCase):
         self.assertEqual(len(variables.trainable_variables()), 1)
         self.assertAllEqual(doubles, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testMap_Grad(self):
     with self.cached_session():
       param = constant_op.constant(2.0)
@@ -381,6 +386,7 @@ class FunctionalOpsTest(test.TestCase):
         ValueError, "two structures don't have the same nested structure"):
       functional_ops.scan(lambda a, x: (a, -a), elems, initializer)
 
+  @test_util.run_deprecated_v1
   def testScan_Scoped(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope("root") as varscope:
@@ -425,6 +431,7 @@ class FunctionalOpsTest(test.TestCase):
     #   t_1 == 1, b == 4.5,       y == 0.5, returns b * y * x = 9
     self.assertAllClose([1., 1., 2.25, 9.], self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testScan_Control(self):
     with self.cached_session() as sess:
       s = array_ops.placeholder(dtypes.float32, shape=[None])
@@ -436,6 +443,7 @@ class FunctionalOpsTest(test.TestCase):
           np.array([1.0, 3.0, 9.0]), sess.run(c, {s: [1, 3, 3],
                                                   b: True}))
 
+  @test_util.run_deprecated_v1
   def testScan_Grad(self):
     with self.cached_session():
       elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data")
@@ -448,6 +456,7 @@ class FunctionalOpsTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllEqual(873.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testScanGradientWithPartStopGradient(self):
     a = variables.Variable(0.0, name="a")
     b = variables.Variable(0.0, name="b")
@@ -477,6 +486,7 @@ class FunctionalOpsTest(test.TestCase):
     y = functional_ops.map_fn(lambda e: e, x)
     self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
 
+  @test_util.run_deprecated_v1
   def testMapUnknownShape(self):
     x = array_ops.placeholder(dtypes.float32)
     y = functional_ops.map_fn(lambda e: e, x)
@@ -484,6 +494,7 @@ class FunctionalOpsTest(test.TestCase):
 
   @test_util.disable_control_flow_v2("b/119323354")
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testMapEmptyScalar(self):
     map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([]))
     self.assertAllEqual([0], map_return.get_shape().dims)
@@ -492,6 +503,7 @@ class FunctionalOpsTest(test.TestCase):
   # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
   # so the body of the while loop never executes
   @test_util.disable_control_flow_v2("b/119323354")
+  @test_util.run_deprecated_v1
   def testMapEmptyTensor(self):
     with self.cached_session():
       map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]),
@@ -512,6 +524,7 @@ class FunctionalOpsTest(test.TestCase):
 
   # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
   # so the body of the while loop never executes
+  @test_util.run_deprecated_v1
   def testScanEmptyTensor(self):
     with self.cached_session():
       x = functional_ops.scan(
@@ -519,6 +532,7 @@ class FunctionalOpsTest(test.TestCase):
       self.assertAllEqual([0, 2, 4], x.get_shape())
       self.assertAllEqual(x.get_shape(), self.evaluate(x).shape)
 
+  @test_util.run_deprecated_v1
   def testScanUnknownShape(self):
     x = array_ops.placeholder(dtypes.float32)
     initializer = array_ops.placeholder(dtypes.float32)
@@ -529,6 +543,7 @@ class FunctionalOpsTest(test.TestCase):
     y = functional_ops.scan(fn, x, initializer=initializer)
     self.assertIs(None, y.get_shape().dims)
 
+  @test_util.run_deprecated_v1
   def testScanVaryingShape(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 2])
@@ -545,6 +560,7 @@ class FunctionalOpsTest(test.TestCase):
       sess.run([result, result_t, result_grad, result_t_grad],
                feed_dict={x: [[1.0, 2.0]]})
 
+  @test_util.run_deprecated_v1
   def testRemoteFunction(self):
     worker_config = config_pb2.ConfigProto()
     worker_config.device_count["CPU"] = 2
@@ -571,6 +587,7 @@ class FunctionalOpsTest(test.TestCase):
       mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
+  @test_util.run_deprecated_v1
   def testRemoteFunctionDirectSession(self):
     worker_config = config_pb2.ConfigProto()
     worker_config.device_count["CPU"] = 2
@@ -595,6 +612,7 @@ class FunctionalOpsTest(test.TestCase):
       mul = self.evaluate(remote_op)
       self.assertEqual(mul, [6])
 
+  @test_util.run_deprecated_v1
   def testRemoteFunctionSameDeviceDirectSession(self):
 
     @function.Defun(dtypes.int32, dtypes.int32)
@@ -680,6 +698,7 @@ class FunctionalOpsTest(test.TestCase):
       ret = self.evaluate(remote_op)
       self.assertAllEqual(ret, [b"a"])
 
+  @test_util.run_deprecated_v1
   def testRemoteFunctionCrossProcess(self):
     workers, _ = test_util.create_local_cluster(2, 1)
 
@@ -703,6 +722,7 @@ class FunctionalOpsTest(test.TestCase):
       mul = self.evaluate(remote_op)
       self.assertEqual(mul, 9)
 
+  @test_util.run_deprecated_v1
   def testIf(self):
 
     @function.Defun(dtypes.float32)
@@ -742,6 +762,7 @@ class FunctionalOpsTest(test.TestCase):
           self.assertAllEqual(Run(sess, 20.), 210.)
           self.assertAllEqual(Run(sess, 100.), 5050.)
 
+  @test_util.run_deprecated_v1
   def testWhileLowering(self):
 
     def Run(n, fetch_by_name):
@@ -776,6 +797,7 @@ class FunctionalOpsTest(test.TestCase):
     self.assertAllEqual(Run(100., False), 5050.)
     self.assertAllEqual(Run(100., True), 5050.)
 
+  @test_util.run_deprecated_v1
   def testWhileError(self):
     for use_gpu in (True, False):
       with ops.Graph().as_default() as g:
@@ -922,6 +944,7 @@ class FunctionalOpsTest(test.TestCase):
     self.assertTrue("TestBody_Cond" in names)
     self.assertTrue("TestBody_Body" in names)
 
+  @test_util.run_deprecated_v1
   def testForCapturedInputs(self):
     v = variables.Variable(1.0)
 
@@ -996,12 +1019,15 @@ class FunctionalOpsTest(test.TestCase):
     tf_for_ans = self._tfMLP(xval, wsval, bsval, rewrite_with_while)
     self.assertAllClose(np_ans, tf_for_ans)
 
+  @test_util.run_deprecated_v1
   def testForMLP(self):
     self._testForMLP(False)
 
+  @test_util.run_deprecated_v1
   def testForMLPWhile(self):
     self._testForMLP(True)
 
+  @test_util.run_deprecated_v1
   def testForError(self):
 
     @function.Defun(dtypes.int32, dtypes.float32)
@@ -1024,6 +1050,7 @@ class FunctionalOpsTest(test.TestCase):
           "For loop body returned 2 arguments. Expected: 1"):
         functional_ops.For(0, 10, 1, [0.0], ReturnsTooManyArgs)[0].eval()
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
 
     @function.Defun(dtypes.float32)
@@ -1049,6 +1076,7 @@ class FunctionalOpsTest(test.TestCase):
 # below test cases.
 class PartitionedCallTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasicSingleDevice(self):
 
     @function.Defun(*[dtypes.float32] * 2)
@@ -1064,6 +1092,7 @@ class PartitionedCallTest(test.TestCase):
                   constant_op.constant(2.)], f=Body))
     self.assertEqual(output, 6.)
 
+  @test_util.run_deprecated_v1
   def testBasicMultiDevice(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
 
@@ -1107,6 +1136,7 @@ class PartitionedCallTest(test.TestCase):
                   constant_op.constant(2.)], f=Body))
     self.assertEqual(output, 6.)
 
+  @test_util.run_deprecated_v1
   def testBasicNoDeviceAnnotations(self):
 
     @function.Defun(*[dtypes.float32] * 2)
@@ -1121,6 +1151,7 @@ class PartitionedCallTest(test.TestCase):
                   constant_op.constant(2.)], f=Body))
     self.assertEqual(output, 6.)
 
+  @test_util.run_deprecated_v1
   def testShardsRunOnRequestedDevices(self):
     config = config_pb2.ConfigProto(device_count={"CPU": 4})
 
@@ -1150,6 +1181,7 @@ class PartitionedCallTest(test.TestCase):
     self.assertIn(compat.as_bytes("CPU:1"), outputs[1])
     self.assertIn(compat.as_bytes("CPU:2"), outputs[2])
 
+  @test_util.run_deprecated_v1
   def testAssignAddResourceVariable(self):
 
     v = resource_variable_ops.ResourceVariable(1.0)
@@ -1201,6 +1233,7 @@ class PartitionedCallTest(test.TestCase):
       self.assertAllEqual(expected, result)
 
   # Use an invalid executor name to test the plumbing of the executor_type attr.
+  @test_util.run_deprecated_v1
   def testExecutorTypeAttrExecutorNotFound(self):
     @function.Defun(dtypes.int32)
     def AddFive(x):
diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py
index 532d8903ee..320ffc9674 100644
--- a/tensorflow/python/kernel_tests/gather_nd_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import variables
@@ -54,6 +55,7 @@ class GatherNdTest(test.TestCase):
     self._testSimpleDtype(np.complex128)
     self._testSimpleDtype("|S")  # byte strings in python2 + 3
 
+  @test_util.run_deprecated_v1
   def testEmptyIndicesAndParamsOKButJustEmptyParamsFails(self):
     with self.session(use_gpu=True):
       params = np.ones((3, 3), dtype=np.float32)
@@ -190,6 +192,7 @@ class GatherNdTest(test.TestCase):
   def assertIndexedSlices(self, t):
     self.assertIsInstance(t, ops.IndexedSlices)
 
+  @test_util.run_deprecated_v1
   def testUnknownIndices(self):
     params = constant_op.constant([[0, 1, 2]])
     indices = array_ops.placeholder(dtypes.int32)
@@ -198,6 +201,7 @@ class GatherNdTest(test.TestCase):
     self.assertEqual(None, shape.ndims)
     self.assertEqual(None, tensor_shape.dimension_value(shape[0]))
 
+  @test_util.run_deprecated_v1
   def testBadIndicesCPU(self):
     with self.session(use_gpu=False):
       params = [0, 1, 2]
@@ -220,6 +224,7 @@ class GatherNdTest(test.TestCase):
           r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"):
         self.evaluate(gather_nd)
 
+  @test_util.run_deprecated_v1
   def testBadIndicesWithSlicesCPU(self):
     with self.session(use_gpu=False):
       params = [[0, 1, 2]]
@@ -242,6 +247,7 @@ class GatherNdTest(test.TestCase):
           r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"):
         self.evaluate(gather_nd)
 
+  @test_util.run_deprecated_v1
   def testGradientsRank2Elements(self):
     indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32)
     inputs = constant_op.constant([[1, 2], [3, 4]], dtype=dtypes.float64)
@@ -253,6 +259,7 @@ class GatherNdTest(test.TestCase):
     with self.session(use_gpu=True):
       assert np.array_equal(expected_grads, self.evaluate(grads))
 
+  @test_util.run_deprecated_v1
   def testGradientsRank2Slices(self):
     indices = constant_op.constant([[1], [0]], dtype=dtypes.int32)
     inputs = constant_op.constant([[1, 2], [3, 4]], dtype=dtypes.float64)
@@ -265,6 +272,7 @@ class GatherNdTest(test.TestCase):
       self.assertIndexedSlices(grads)
       self.assertAllEqual(expected_grads, ops.convert_to_tensor(grads).eval())
 
+  @test_util.run_deprecated_v1
   def testGradientsRank3Elements(self):
     indices = constant_op.constant(
         [[[0, 1], [1, 0]], [[0, 0], [1, 1]]], dtype=dtypes.int32)
@@ -280,6 +288,7 @@ class GatherNdTest(test.TestCase):
     with self.session(use_gpu=True):
       self.assertAllEqual(expected_grads, self.evaluate(grads))
 
+  @test_util.run_deprecated_v1
   def testGradientsRank7Elements(self):
     # Shape [1,1,2,1,1,2,2]
     indices = constant_op.constant(
@@ -309,6 +318,7 @@ class GatherNdTest(test.TestCase):
     with self.session(use_gpu=True):
       self.assertAllEqual(expected_grads, self.evaluate(grads))
 
+  @test_util.run_deprecated_v1
   def testGradientsInt64Indices(self):
     indices = constant_op.constant(
         [[[0, 1], [1, 0]], [[0, 0], [1, 1]]], dtype=dtypes.int64)
@@ -324,6 +334,7 @@ class GatherNdTest(test.TestCase):
     with self.session(use_gpu=True):
       self.assertAllEqual(expected_grads, self.evaluate(grads))
 
+  @test_util.run_deprecated_v1
   def testGradientsRank2SlicesWithEmptySpace(self):
     indices = constant_op.constant([[2], [0], [5]], dtype=dtypes.int32)
     inputs = constant_op.constant(
diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py
index 326e4aacd2..fc86068c3f 100644
--- a/tensorflow/python/kernel_tests/gather_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.platform import test
@@ -87,6 +88,7 @@ class GatherTest(test.TestCase):
           expected_shape = data.shape[:axis] + (4,) + data.shape[axis + 1:]
           self.assertEqual(expected_shape, gather_t.get_shape())
 
+  @test_util.run_deprecated_v1
   def testHigherRank(self):
     # We check that scalar and empty indices shapes work as well
     shape = (2, 1, 3, 2)
@@ -148,6 +150,7 @@ class GatherTest(test.TestCase):
                 atol=2e-6,
                 rtol=2e-6)
 
+  @test_util.run_deprecated_v1
   def testString(self):
     params = np.array([[b"asdf", b"zxcv"], [b"qwer", b"uiop"]])
     with self.cached_session():
@@ -156,6 +159,7 @@ class GatherTest(test.TestCase):
       self.assertAllEqual([b"asdf", b"qwer"],
                           array_ops.gather(params, 0, axis=1).eval())
 
+  @test_util.run_deprecated_v1
   def testUInt32AndUInt64(self):
     for unsigned_type in (dtypes.uint32, dtypes.uint64):
       params = self._buildParams(
@@ -165,12 +169,14 @@ class GatherTest(test.TestCase):
                             array_ops.gather(params, 1, axis=0).eval())
         self.assertAllEqual([1, 7], array_ops.gather(params, 0, axis=1).eval())
 
+  @test_util.run_deprecated_v1
   def testUnknownIndices(self):
     params = constant_op.constant([[0, 1, 2]])
     indices = array_ops.placeholder(dtypes.int32)
     gather_t = array_ops.gather(params, indices)
     self.assertEqual(None, gather_t.get_shape())
 
+  @test_util.run_deprecated_v1
   def testUnknownAxis(self):
     params = constant_op.constant([[0, 1, 2]])
     indices = constant_op.constant([[0, 0], [0, 0]])
@@ -204,6 +210,7 @@ class GatherTest(test.TestCase):
       with self.assertRaisesOpError(r"indices\[0,0\] = 7 is not in \[0, 3\)"):
         array_ops.gather(params, [[7]], axis=1).eval()
 
+  @test_util.run_deprecated_v1
   def testBadAxis(self):
     with self.session(use_gpu=True):
       params = [0, 1, 2]
@@ -220,6 +227,7 @@ class GatherTest(test.TestCase):
           array_ops.gather(params_ph, indices, axis=bad_axis).eval(
               feed_dict={params_ph: params})
 
+  @test_util.run_deprecated_v1
   def testEmptySlices(self):
     with self.session(use_gpu=True):
       for dtype in _TEST_TYPES:
diff --git a/tensorflow/python/kernel_tests/gradient_correctness_test.py b/tensorflow/python/kernel_tests/gradient_correctness_test.py
index 12b8a4c8e3..0148de5047 100644
--- a/tensorflow/python/kernel_tests/gradient_correctness_test.py
+++ b/tensorflow/python/kernel_tests/gradient_correctness_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -29,6 +30,7 @@ from tensorflow.python.platform import test
 
 class GradientCorrectnessTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testMultipleOutputChainedGradients(self):
     with self.cached_session() as sess:
       x = constant_op.constant(1.0, dtype=dtypes.float32)
@@ -40,18 +42,21 @@ class GradientCorrectnessTest(test.TestCase):
       # [dexp(x)/dx + d(log(exp(x)))/dx] @ x=1 == exp(1) + 1
       self.assertAllClose(grad_vals[0], exp1_plus_one)
 
+  @test_util.run_deprecated_v1
   def testIdentityGradient(self):
     x = constant_op.constant(3.)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
       self.assertAllClose(1., self.evaluate(dx_dx))
 
+  @test_util.run_deprecated_v1
   def testIntegerIdentityGradient(self):
     x = constant_op.constant(3)
     dx_dx, = gradients_impl.gradients(x, x)
     with self.cached_session() as sess:
       self.assertAllClose(1, self.evaluate(dx_dx))
 
+  @test_util.run_deprecated_v1
   def testGradientWithIntegerPath(self):
     x = constant_op.constant([3.9, 4.1])
     k = math_ops.to_float(math_ops.to_int32(x))
@@ -60,6 +65,7 @@ class GradientCorrectnessTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertAllClose([3., 4.], self.evaluate(dy_dx))
 
+  @test_util.run_deprecated_v1
   def testNoIntegerGradient1(self):
     x = constant_op.constant([3.9, 4.1])
     k = math_ops.to_float(math_ops.to_int32(x))
@@ -67,6 +73,7 @@ class GradientCorrectnessTest(test.TestCase):
     dy_dx, = gradients_impl.gradients(y, x)
     self.assertIsNone(dy_dx)
 
+  @test_util.run_deprecated_v1
   def testNoIntegerGradient2(self):
     k = constant_op.constant([3, 4])
     x = math_ops.to_float(k)
@@ -74,18 +81,21 @@ class GradientCorrectnessTest(test.TestCase):
     dy_dk, = gradients_impl.gradients(y, k)
     self.assertIsNone(dy_dk)
 
+  @test_util.run_deprecated_v1
   def testNoIntegerGradient3(self):
     k = constant_op.constant([3, 4])
     m = k * k
     dm_dk, = gradients_impl.gradients(m, k)
     self.assertIsNone(dm_dk)
 
+  @test_util.run_deprecated_v1
   def testNoIntegerGradient4(self):
     k = constant_op.constant([3, 4])
     m = k * k * k
     dm_dk, = gradients_impl.gradients(m, k)
     self.assertIsNone(dm_dk)
 
+  @test_util.run_deprecated_v1
   def testNoIntegerGradient5(self):
     k = constant_op.constant([3, 4])
     m = k * k
@@ -93,6 +103,7 @@ class GradientCorrectnessTest(test.TestCase):
     dn_dk, = gradients_impl.gradients(n, k)
     self.assertIsNone(dn_dk)
 
+  @test_util.run_deprecated_v1
   def testNoIntegerGradient6(self):
     k = constant_op.constant(3)
     x = math_ops.to_float(k)
diff --git a/tensorflow/python/kernel_tests/identity_n_op_py_test.py b/tensorflow/python/kernel_tests/identity_n_op_py_test.py
index 518733cd8e..a1110d640f 100644
--- a/tensorflow/python/kernel_tests/identity_n_op_py_test.py
+++ b/tensorflow/python/kernel_tests/identity_n_op_py_test.py
@@ -21,12 +21,14 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class IdentityNOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInt32String_6(self):
     with self.cached_session() as sess:
       [value0, value1] = sess.run(
@@ -36,6 +38,7 @@ class IdentityNOpTest(test.TestCase):
     self.assertAllEqual(
         np.array([b"a", b"b", b"C", b"d", b"E", b"f", b"g"]), value1)
 
+  @test_util.run_deprecated_v1
   def testInt32_shapes(self):
     with self.cached_session() as sess:
       inp0 = constant_op.constant([10, 20, 30, 40, 50, 60], shape=[2, 3])
@@ -50,6 +53,7 @@ class IdentityNOpTest(test.TestCase):
         np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12], [13, 14, 15]]),
         value2)
 
+  @test_util.run_deprecated_v1
   def testString(self):
     source = [b"A", b"b", b"C", b"d", b"E", b"f"]
     with self.cached_session() as sess:
diff --git a/tensorflow/python/kernel_tests/identity_op_py_test.py b/tensorflow/python/kernel_tests/identity_op_py_test.py
index 88ea10c22a..1a6794e896 100644
--- a/tensorflow/python/kernel_tests/identity_op_py_test.py
+++ b/tensorflow/python/kernel_tests/identity_op_py_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import variables
@@ -30,17 +31,20 @@ from tensorflow.python.platform import test
 
 class IdentityOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInt32_6(self):
     with self.cached_session():
       value = array_ops.identity([1, 2, 3, 4, 5, 6]).eval()
     self.assertAllEqual(np.array([1, 2, 3, 4, 5, 6]), value)
 
+  @test_util.run_deprecated_v1
   def testInt32_2_3(self):
     with self.cached_session():
       inp = constant_op.constant([10, 20, 30, 40, 50, 60], shape=[2, 3])
       value = array_ops.identity(inp).eval()
     self.assertAllEqual(np.array([[10, 20, 30], [40, 50, 60]]), value)
 
+  @test_util.run_deprecated_v1
   def testString(self):
     source = [b"A", b"b", b"C", b"d", b"E", b"f"]
     with self.cached_session():
@@ -58,6 +62,7 @@ class IdentityOpTest(test.TestCase):
       self.assertEquals(shape,
                         array_ops.identity(np.array(array_2x3)).get_shape())
 
+  @test_util.run_deprecated_v1
   def testRefIdentityShape(self):
     with self.cached_session():
       shape = [2, 3]
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 87c7bbef3c..09b9944baa 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.layers import convolutional
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -106,6 +107,7 @@ def _init_sampler(tc, init, num):
 
 class ConstantInitializersTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testZerosInitializer(self):
     with self.session(use_gpu=True):
       shape = [2, 3]
@@ -114,6 +116,7 @@ class ConstantInitializersTest(test.TestCase):
       x.initializer.run()
       self.assertAllEqual(x.eval(), np.zeros(shape))
 
+  @test_util.run_deprecated_v1
   def testOnesInitializer(self):
     with self.session(use_gpu=True):
       shape = [2, 3]
@@ -122,6 +125,7 @@ class ConstantInitializersTest(test.TestCase):
       x.initializer.run()
       self.assertAllEqual(x.eval(), np.ones(shape))
 
+  @test_util.run_deprecated_v1
   def testConstantZeroInitializer(self):
     with self.session(use_gpu=True):
       shape = [2, 3]
@@ -130,6 +134,7 @@ class ConstantInitializersTest(test.TestCase):
       x.initializer.run()
       self.assertAllEqual(x.eval(), np.zeros(shape))
 
+  @test_util.run_deprecated_v1
   def testConstantOneInitializer(self):
     with self.session(use_gpu=True):
       shape = [2, 3]
@@ -138,6 +143,7 @@ class ConstantInitializersTest(test.TestCase):
       x.initializer.run()
       self.assertAllEqual(x.eval(), np.ones(shape))
 
+  @test_util.run_deprecated_v1
   def testConstantIntInitializer(self):
     with self.session(use_gpu=True):
       shape = [2, 3]
@@ -150,6 +156,7 @@ class ConstantInitializersTest(test.TestCase):
       self.assertEqual(x.dtype.base_dtype, dtypes.int32)
       self.assertAllEqual(x.eval(), 7 * np.ones(shape, dtype=np.int32))
 
+  @test_util.run_deprecated_v1
   def testConstantTupleInitializer(self):
     with self.session(use_gpu=True):
       shape = [3]
@@ -173,6 +180,7 @@ class ConstantInitializersTest(test.TestCase):
       for a, e in zip(actual, expected):
         self.assertEqual(a, e)
 
+  @test_util.run_deprecated_v1
   def testNDimConstantInitializer(self):
     value = [0, 1, 2, 3, 4, 5]
     shape = [2, 3]
@@ -199,6 +207,7 @@ class ConstantInitializersTest(test.TestCase):
         e = expected[i] if i < len(expected) else expected[-1]
         self.assertEqual(a, e)
 
+  @test_util.run_deprecated_v1
   def testNDimConstantInitializerLessValues(self):
     value = [0, 1, 2, 3, 4, 5]
     shape = [2, 4]
@@ -222,6 +231,7 @@ class ConstantInitializersTest(test.TestCase):
           shape=shape,
           initializer=init)
 
+  @test_util.run_deprecated_v1
   def testNDimConstantInitializerMoreValues(self):
     value = [0, 1, 2, 3, 4, 5, 6, 7]
     shape = [2, 3]
@@ -243,18 +253,21 @@ class ConstantInitializersTest(test.TestCase):
 
 class RandomNormalInitializationTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
       init2 = init_ops.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
       init2 = init_ops.random_normal_initializer(0.0, 1.0, seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.random_normal_initializer(0.0, 1.0)
     self.assertFalse(duplicated_initializer(self, init, 1))
@@ -270,6 +283,7 @@ class RandomNormalInitializationTest(test.TestCase):
 
 class TruncatedNormalInitializationTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.truncated_normal_initializer(
@@ -278,6 +292,7 @@ class TruncatedNormalInitializationTest(test.TestCase):
           0.0, 1.0, seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.truncated_normal_initializer(
@@ -286,6 +301,7 @@ class TruncatedNormalInitializationTest(test.TestCase):
           0.0, 1.0, seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.truncated_normal_initializer(0.0, 1.0)
     self.assertFalse(duplicated_initializer(self, init, 1))
@@ -301,18 +317,21 @@ class TruncatedNormalInitializationTest(test.TestCase):
 
 class RandomUniformInitializationTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64, dtypes.int64]:
       init1 = init_ops.random_uniform_initializer(0, 7, seed=1, dtype=dtype)
       init2 = init_ops.random_uniform_initializer(0, 7, seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64, dtypes.int32, dtypes.int64]:
       init1 = init_ops.random_uniform_initializer(0, 7, seed=1, dtype=dtype)
       init2 = init_ops.random_uniform_initializer(0, 7, seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.random_uniform_initializer(0.0, 1.0)
     self.assertFalse(duplicated_initializer(self, init, 1))
@@ -320,6 +339,7 @@ class RandomUniformInitializationTest(test.TestCase):
 
 class UniformUnitScalingInitializationTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.uniform_unit_scaling_initializer(seed=1, dtype=dtype)
@@ -331,6 +351,7 @@ class UniformUnitScalingInitializationTest(test.TestCase):
           1.5, seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init3, init4))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.uniform_unit_scaling_initializer(seed=1, dtype=dtype)
@@ -341,6 +362,7 @@ class UniformUnitScalingInitializationTest(test.TestCase):
       self.assertFalse(identicaltest(self, init1, init3))
       self.assertFalse(identicaltest(self, init2, init3))
 
+  @test_util.run_deprecated_v1
   def testZeroSize(self):
     shape = [0, 2]
     with self.cached_session():
@@ -351,6 +373,7 @@ class UniformUnitScalingInitializationTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllEqual(shape, self.evaluate(x).shape)
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.uniform_unit_scaling_initializer()
     self.assertFalse(duplicated_initializer(self, init, 1))
@@ -364,6 +387,7 @@ class UniformUnitScalingInitializationTest(test.TestCase):
 
 class VarianceScalingInitializationTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testTruncatedNormalDistribution(self):
     shape = [100, 100]
     expect_mean = 0.
@@ -381,6 +405,7 @@ class VarianceScalingInitializationTest(test.TestCase):
     self.assertNear(np.mean(x), expect_mean, err=1e-2)
     self.assertNear(np.var(x), expect_var, err=1e-2)
 
+  @test_util.run_deprecated_v1
   def testNormalDistribution(self):
     shape = [100, 100]
     expect_mean = 0.
@@ -397,6 +422,7 @@ class VarianceScalingInitializationTest(test.TestCase):
     self.assertNear(np.mean(x), expect_mean, err=1e-2)
     self.assertNear(np.var(x), expect_var, err=1e-2)
 
+  @test_util.run_deprecated_v1
   def testUntruncatedNormalDistribution(self):
     shape = [100, 100]
     expect_mean = 0.
@@ -414,6 +440,7 @@ class VarianceScalingInitializationTest(test.TestCase):
     self.assertNear(np.mean(x), expect_mean, err=1e-2)
     self.assertNear(np.var(x), expect_var, err=1e-2)
 
+  @test_util.run_deprecated_v1
   def testUniformDistribution(self):
     shape = [100, 100]
     expect_mean = 0.
@@ -449,6 +476,7 @@ class RangeTest(test.TestCase):
             self._Range(100, 500, 100), np.array([100, 200, 300, 400])))
     self.assertEqual(math_ops.range(0, 5, 1).dtype, dtypes.int32)
 
+  @test_util.run_deprecated_v1
   def testLimitOnly(self):
     with self.session(use_gpu=True):
       self.assertAllEqual(np.arange(5), math_ops.range(5).eval())
@@ -583,18 +611,21 @@ class DeviceTest(test.TestCase):
 
 class OrthogonalInitializerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.orthogonal_initializer(seed=1, dtype=dtype)
       init2 = init_ops.orthogonal_initializer(seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2, (10, 10)))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.orthogonal_initializer(seed=1, dtype=dtype)
       init2 = init_ops.orthogonal_initializer(seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2, (10, 10)))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.orthogonal_initializer()
     self.assertFalse(duplicated_initializer(self, init, 1, (10, 10)))
@@ -608,6 +639,7 @@ class OrthogonalInitializerTest(test.TestCase):
     with self.session(graph=ops.Graph(), use_gpu=True):
       self.assertRaises(ValueError, init1, shape=[5])
 
+  @test_util.run_deprecated_v1
   def testGain(self):
     shape = (10, 10)
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -618,6 +650,7 @@ class OrthogonalInitializerTest(test.TestCase):
         t2 = init2(shape).eval()
       self.assertAllClose(t1, t2 / 3.14)
 
+  @test_util.run_deprecated_v1
   def testShapesValues(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       for shape in [(10, 10), (10, 9, 8), (100, 5, 5), (50, 40), (40, 50)]:
@@ -639,18 +672,21 @@ class OrthogonalInitializerTest(test.TestCase):
 
 class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_delta_orthogonal(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_delta_orthogonal(seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2, (3, 3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_delta_orthogonal(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_delta_orthogonal(seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2, (3, 3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.convolutional_delta_orthogonal()
     self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 10, 10)))
@@ -665,6 +701,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
     with self.session(graph=ops.Graph(), use_gpu=True):
       self.assertRaises(ValueError, init1, shape=[3, 3, 6, 5])
 
+  @test_util.run_deprecated_v1
   def testGain(self):
     shape = (3, 3, 10, 10)
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -676,6 +713,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
         t2 = init2(shape).eval()
       self.assertAllClose(t1, t2 / 3.14)
 
+  @test_util.run_deprecated_v1
   def testShapesValues(self):
     gain = 3.14
     for dtype in [dtypes.float32]:
@@ -711,6 +749,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
           # Check isometry of the delta-orthogonal kernel.
           self.assertAllClose(self.evaluate(ratio), gain, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testNonuniformity(self):
     value = 0
     abs_value = 0
@@ -738,18 +777,21 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
 
 class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_orthogonal_1d(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_orthogonal_1d(seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2, (3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_orthogonal_1d(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_orthogonal_1d(seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2, (3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.convolutional_orthogonal_1d()
     self.assertFalse(duplicated_initializer(self, init, 1, (3, 10, 10)))
@@ -764,6 +806,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
     with self.session(graph=ops.Graph(), use_gpu=True):
       self.assertRaises(ValueError, init1, shape=[3, 6, 5])
 
+  @test_util.run_deprecated_v1
   def testGain(self):
     shape = (3, 10, 10)
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -775,6 +818,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
         t2 = init2(shape).eval()
       self.assertAllClose(t1, t2 / 3.14)
 
+  @test_util.run_deprecated_v1
   def testNonuniformity(self):
     value = 0
     abs_value = 0
@@ -799,6 +843,7 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
       # Compute the sum of the absolute values of 'count' determinants
       self.assertAllClose(abs_value, count, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testShapesValues(self):
     def circular_pad(input_, width, kernel_size):
       """Pad input_ for computing (circular) convolution.
@@ -852,18 +897,21 @@ class ConvolutionOrthogonal1dInitializerTest(test.TestCase):
 
 class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2, (3, 3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_orthogonal_2d(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_orthogonal_2d(seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2, (3, 3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.convolutional_orthogonal_2d()
     self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 10, 10)))
@@ -878,6 +926,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
     with self.session(graph=ops.Graph(), use_gpu=True):
       self.assertRaises(ValueError, init1, shape=[3, 3, 6, 5])
 
+  @test_util.run_deprecated_v1
   def testGain(self):
     shape = (3, 3, 10, 10)
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -889,6 +938,7 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
         t2 = init2(shape).eval()
       self.assertAllClose(t1, t2 / 3.14)
 
+  @test_util.run_deprecated_v1
   def testShapesValues(self):
     def circular_pad(input_, width, kernel_size):
       """Pad input_ for computing (circular) convolution.
@@ -947,18 +997,21 @@ class ConvolutionOrthogonal2dInitializerTest(test.TestCase):
 
 class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitializerIdentical(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_orthogonal_3d(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_orthogonal_3d(seed=1, dtype=dtype)
       self.assertTrue(identicaltest(self, init1, init2, (3, 3, 3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testInitializerDifferent(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       init1 = init_ops.convolutional_orthogonal_3d(seed=1, dtype=dtype)
       init2 = init_ops.convolutional_orthogonal_3d(seed=2, dtype=dtype)
       self.assertFalse(identicaltest(self, init1, init2, (3, 3, 3, 10, 10)))
 
+  @test_util.run_deprecated_v1
   def testDuplicatedInitializer(self):
     init = init_ops.convolutional_orthogonal_3d()
     self.assertFalse(duplicated_initializer(self, init, 1, (3, 3, 3, 10, 10)))
@@ -973,6 +1026,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
     with self.session(graph=ops.Graph(), use_gpu=True):
       self.assertRaises(ValueError, init1, shape=[3, 3, 3, 6, 5])
 
+  @test_util.run_deprecated_v1
   def testGain(self):
     shape = (3, 3, 3, 10, 10)
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -984,6 +1038,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
         t2 = init2(shape).eval()
       self.assertAllClose(t1, t2 / 3.14)
 
+  @test_util.run_deprecated_v1
   def testNonuniformity(self):
     value = 0
     abs_value = 0
@@ -1008,6 +1063,7 @@ class ConvolutionOrthogonal3dInitializerTest(test.TestCase):
       # Compute the sum of the absolute values of 'count' determinants
       self.assertAllClose(abs_value, count, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testShapesValues(self):
     def circular_pad(input_, width, kernel_size):
       """Padding input_ for computing circular convolution.
@@ -1083,12 +1139,14 @@ class IdentityInitializerTest(test.TestCase):
       self.assertRaises(ValueError, init, shape=[5])
       self.assertRaises(ValueError, init, shape=[])
 
+  @test_util.run_deprecated_v1
   def testNonSquare(self):
     init = init_ops.identity_initializer()
     shape = (10, 5)
     with self.session(graph=ops.Graph(), use_gpu=True):
       self.assertAllClose(init(shape).eval(), np.eye(*shape))
 
+  @test_util.run_deprecated_v1
   def testGain(self):
     shape = (10, 10)
     for dtype in [dtypes.float32, dtypes.float64]:
@@ -1099,6 +1157,7 @@ class IdentityInitializerTest(test.TestCase):
       with self.session(graph=ops.Graph(), use_gpu=True):
         self.assertAllClose(init_custom(shape).eval(), np.eye(*shape) * 0.9)
 
+  @test_util.run_deprecated_v1
   def testPartitions(self):
     shape = (10, 10)
     init = init_ops.identity_initializer()
diff --git a/tensorflow/python/kernel_tests/inplace_ops_test.py b/tensorflow/python/kernel_tests/inplace_ops_test.py
index e0c36d3d2e..9eaaac7a24 100644
--- a/tensorflow/python/kernel_tests/inplace_ops_test.py
+++ b/tensorflow/python/kernel_tests/inplace_ops_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.platform import test as test_lib
 
 class InplaceOpsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBasicUpdate(self):
     for dtype in [dtypes.float32, dtypes.int32, dtypes.int64]:
       with self.session(use_gpu=True):
@@ -48,6 +49,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
         y[5, :] = 7
         self.assertAllClose(x.eval(), y)
 
+  @test_util.run_deprecated_v1
   def testBasicUpdateBool(self):
     with self.session(use_gpu=True):
       x = array_ops.ones([7, 3], dtypes.bool)
@@ -65,6 +67,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
       y[5, :] = False
       self.assertAllClose(x.eval(), y)
 
+  @test_util.run_deprecated_v1
   def testBasicAdd(self):
     for dtype in [dtypes.float32, dtypes.int32, dtypes.int64]:
       with self.cached_session(use_gpu=True):
@@ -84,6 +87,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
         y[:, :] += 99
         self.assertAllClose(x.eval(), y)
 
+  @test_util.run_deprecated_v1
   def testBasicSub(self):
     for dtype in [dtypes.float32, dtypes.int32, dtypes.int64]:
       with self.cached_session(use_gpu=True):
@@ -103,6 +107,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
         y[:, :] -= 99
         self.assertAllClose(x.eval(), y)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     with self.session(use_gpu=True):
       d0, d1, d2 = 100, 3, 5
@@ -123,6 +128,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
           y[idx, :] -= val
         self.assertAllClose(x.eval(), y)
 
+  @test_util.run_deprecated_v1
   def testRandom1D(self):
     with self.session(use_gpu=True):
       d0 = 100
@@ -164,6 +170,7 @@ class InplaceOpsTest(test_util.TensorFlowTestCase):
                                    "i and x shape doesn't match"):
         _ = inplace_ops.inplace_update([[1.]], [0, 1], [[10]]).eval()
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     for dtype in [
         dtypes.float32, dtypes.float64, dtypes.int32, dtypes.int64, dtypes.bool,
diff --git a/tensorflow/python/kernel_tests/io_ops_test.py b/tensorflow/python/kernel_tests/io_ops_test.py
index a6b477062e..c5df5231bf 100644
--- a/tensorflow/python/kernel_tests/io_ops_test.py
+++ b/tensorflow/python/kernel_tests/io_ops_test.py
@@ -23,6 +23,7 @@ import os
 import shutil
 import tempfile
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import io_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
@@ -30,6 +31,7 @@ from tensorflow.python.util import compat
 
 class IoOpsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testReadFile(self):
     cases = ['', 'Some contents', 'Неки садржаји на српском']
     for contents in cases:
@@ -78,6 +80,7 @@ class IoOpsTest(test.TestCase):
         compat.as_bytes(files[i].name) for i in range(len(files))
         if i in indices)
 
+  @test_util.run_deprecated_v1
   def testMatchingFiles(self):
     cases = [
         'ABcDEF.GH', 'ABzDEF.GH', 'ABasdfjklDEF.GH', 'AB3DEF.GH', 'AB4DEF.GH',
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py
index 628ed998c5..627349c69b 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_addition_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_addition
@@ -69,6 +70,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
     with self.assertRaisesRegexp(TypeError, "contain only LinearOperator"):
       add_operators([1, 2])
 
+  @test_util.run_deprecated_v1
   def test_two_diag_operators(self):
     op_a = linalg.LinearOperatorDiag(
         [1., 1.], is_positive_definite=True, name="A")
@@ -89,6 +91,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
       # Enforce particular name for this simple case
       self.assertEqual("Add/B__A/", op.name)
 
+  @test_util.run_deprecated_v1
   def test_three_diag_operators(self):
     op1 = linalg.LinearOperatorDiag(
         [1., 1.], is_positive_definite=True, name="op1")
@@ -109,6 +112,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
       # Positive definite ==> non-singular
       self.assertTrue(op.is_non_singular)
 
+  @test_util.run_deprecated_v1
   def test_diag_tril_diag(self):
     op1 = linalg.LinearOperatorDiag(
         [1., 1.], is_non_singular=True, name="diag_a")
@@ -134,6 +138,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
       # Since no custom hint was provided, we default to None (unknown).
       self.assertEqual(None, op.is_non_singular)
 
+  @test_util.run_deprecated_v1
   def test_matrix_diag_tril_diag_uses_custom_name(self):
     op0 = linalg.LinearOperatorFullMatrix(
         [[-1., -1.], [-1., -1.]], name="matrix")
@@ -217,6 +222,7 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase):
     self.assertEqual(1, len(op_sum))
     self.assertIsInstance(op_sum[0], linalg.LinearOperatorLowerTriangular)
 
+  @test_util.run_deprecated_v1
   def test_cannot_add_everything_so_return_more_than_one_operator(self):
     diag1 = linalg.LinearOperatorDiag([1.])
     diag2 = linalg.LinearOperatorDiag([2.])
@@ -261,6 +267,7 @@ class AddAndReturnScaledIdentityTest(test.TestCase):
   def setUp(self):
     self._adder = linear_operator_addition._AddAndReturnScaledIdentity()
 
+  @test_util.run_deprecated_v1
   def test_identity_plus_identity(self):
     id1 = linalg.LinearOperatorIdentity(num_rows=2)
     id2 = linalg.LinearOperatorIdentity(num_rows=2, batch_shape=[3])
@@ -279,6 +286,7 @@ class AddAndReturnScaledIdentityTest(test.TestCase):
     self.assertTrue(operator.is_non_singular)
     self.assertEqual("my_operator", operator.name)
 
+  @test_util.run_deprecated_v1
   def test_identity_plus_scaled_identity(self):
     id1 = linalg.LinearOperatorIdentity(num_rows=2, batch_shape=[3])
     id2 = linalg.LinearOperatorScaledIdentity(num_rows=2, multiplier=2.2)
@@ -297,6 +305,7 @@ class AddAndReturnScaledIdentityTest(test.TestCase):
     self.assertTrue(operator.is_non_singular)
     self.assertEqual("my_operator", operator.name)
 
+  @test_util.run_deprecated_v1
   def test_scaled_identity_plus_scaled_identity(self):
     id1 = linalg.LinearOperatorScaledIdentity(
         num_rows=2, multiplier=[2.2, 2.2, 2.2])
@@ -322,6 +331,7 @@ class AddAndReturnDiagTest(test.TestCase):
   def setUp(self):
     self._adder = linear_operator_addition._AddAndReturnDiag()
 
+  @test_util.run_deprecated_v1
   def test_identity_plus_identity_returns_diag(self):
     id1 = linalg.LinearOperatorIdentity(num_rows=2)
     id2 = linalg.LinearOperatorIdentity(num_rows=2, batch_shape=[3])
@@ -340,6 +350,7 @@ class AddAndReturnDiagTest(test.TestCase):
     self.assertTrue(operator.is_non_singular)
     self.assertEqual("my_operator", operator.name)
 
+  @test_util.run_deprecated_v1
   def test_diag_plus_diag(self):
     diag1 = rng.rand(2, 3, 4)
     diag2 = rng.rand(4)
@@ -366,6 +377,7 @@ class AddAndReturnTriLTest(test.TestCase):
   def setUp(self):
     self._adder = linear_operator_addition._AddAndReturnTriL()
 
+  @test_util.run_deprecated_v1
   def test_diag_plus_tril(self):
     diag = linalg.LinearOperatorDiag([1., 2.])
     tril = linalg.LinearOperatorLowerTriangular([[10., 0.], [30., 0.]])
@@ -389,6 +401,7 @@ class AddAndReturnMatrixTest(test.TestCase):
   def setUp(self):
     self._adder = linear_operator_addition._AddAndReturnMatrix()
 
+  @test_util.run_deprecated_v1
   def test_diag_plus_diag(self):
     diag1 = linalg.LinearOperatorDiag([1., 2.])
     diag2 = linalg.LinearOperatorDiag([-1., 3.])
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
index 09867435a7..6366083ac5 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_circulant_test.py
@@ -21,6 +21,7 @@ import contextlib
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import spectral_ops_test_util
@@ -129,6 +130,7 @@ class LinearOperatorCirculantTestSelfAdjointOperator(
 
     return operator, mat
 
+  @test_util.run_deprecated_v1
   def test_simple_hermitian_spectrum_gives_operator_with_zero_imag_part(self):
     with self.cached_session():
       spectrum = math_ops.cast([1., 1j, -1j], dtypes.complex64)
@@ -196,6 +198,7 @@ class LinearOperatorCirculantTestHermitianSpectrum(
 
     return operator, mat
 
+  @test_util.run_deprecated_v1
   def test_simple_hermitian_spectrum_gives_operator_with_zero_imag_part(self):
     with self.cached_session():
       spectrum = math_ops.cast([1., 1j, -1j], dtypes.complex64)
@@ -251,6 +254,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
 
     return operator, mat
 
+  @test_util.run_deprecated_v1
   def test_simple_hermitian_spectrum_gives_operator_with_zero_imag_part(self):
     with self.cached_session():
       spectrum = math_ops.cast([1., 1j, -1j], dtypes.complex64)
@@ -262,6 +266,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       np.testing.assert_allclose(
           0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3)
 
+  @test_util.run_deprecated_v1
   def test_simple_positive_real_spectrum_gives_self_adjoint_pos_def_oper(self):
     with self.cached_session() as sess:
       spectrum = math_ops.cast([6., 4, 2], dtypes.complex64)
@@ -274,6 +279,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       operator.assert_positive_definite().run()  # Should not fail
       operator.assert_self_adjoint().run()  # Should not fail
 
+  @test_util.run_deprecated_v1
   def test_defining_operator_using_real_convolution_kernel(self):
     with self.cached_session():
       convolution_kernel = [1., 2., 1.]
@@ -304,6 +310,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       np.testing.assert_allclose(
           0, self.evaluate(imag_matrix), rtol=0, atol=eps * 3 * 4)
 
+  @test_util.run_deprecated_v1
   def test_convolution_kernel_same_as_first_row_of_to_dense(self):
     spectrum = [[3., 2., 1.], [2., 1.5, 1.]]
     with self.cached_session():
@@ -315,6 +322,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       self.assertAllEqual((2, 3, 3), c.get_shape())
       self.assertAllClose(h.eval(), self.evaluate(c)[:, :, 0])
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_fails_for_singular_operator(self):
     spectrum = math_ops.cast([0, 4, 2j + 2], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant(spectrum)
@@ -322,12 +330,14 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       with self.assertRaisesOpError("Singular operator"):
         operator.assert_non_singular().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_does_not_fail_for_non_singular_operator(self):
     spectrum = math_ops.cast([-3j, 4, 2j + 2], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant(spectrum)
     with self.cached_session():
       operator.assert_non_singular().run()  # Should not fail
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite_fails_for_non_positive_definite(self):
     spectrum = math_ops.cast([6., 4, 2j], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant(spectrum)
@@ -335,6 +345,7 @@ class LinearOperatorCirculantTestNonHermitianSpectrum(
       with self.assertRaisesOpError("Not positive definite"):
         operator.assert_positive_definite().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite_does_not_fail_when_pos_def(self):
     spectrum = math_ops.cast([6., 4, 2j + 2], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant(spectrum)
@@ -529,6 +540,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
 
     return operator, mat
 
+  @test_util.run_deprecated_v1
   def test_real_hermitian_spectrum_gives_real_symmetric_operator(self):
     with self.cached_session() as sess:
       # This is a real and hermitian spectrum.
@@ -546,6 +558,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
       np.testing.assert_allclose(0, imag_matrix, atol=1e-6)
       self.assertAllClose(matrix, matrix_transpose, atol=0)
 
+  @test_util.run_deprecated_v1
   def test_real_spectrum_gives_self_adjoint_operator(self):
     with self.cached_session() as sess:
       # This is a real and hermitian spectrum.
@@ -560,6 +573,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
       matrix, matrix_h = self.evaluate([matrix_tensor, matrix_h])
       self.assertAllClose(matrix, matrix_h, atol=0)
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_fails_for_singular_operator(self):
     spectrum = math_ops.cast([[0, 4], [2j + 2, 3.]], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant2D(spectrum)
@@ -567,12 +581,14 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
       with self.assertRaisesOpError("Singular operator"):
         operator.assert_non_singular().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_does_not_fail_for_non_singular_operator(self):
     spectrum = math_ops.cast([[-3j, 4], [2j + 2, 3.]], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant2D(spectrum)
     with self.cached_session():
       operator.assert_non_singular().run()  # Should not fail
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite_fails_for_non_positive_definite(self):
     spectrum = math_ops.cast([[6., 4], [2j, 3.]], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant2D(spectrum)
@@ -580,6 +596,7 @@ class LinearOperatorCirculant2DTestNonHermitianSpectrum(
       with self.assertRaisesOpError("Not positive definite"):
         operator.assert_positive_definite().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite_does_not_fail_when_pos_def(self):
     spectrum = math_ops.cast([[6., 4], [2j + 2, 3.]], dtypes.complex64)
     operator = linalg.LinearOperatorCirculant2D(spectrum)
@@ -618,6 +635,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       with spectral_ops_test_util.fft_kernel_label_map():
         yield sess
 
+  @test_util.run_deprecated_v1
   def test_real_spectrum_gives_self_adjoint_operator(self):
     with self.cached_session() as sess:
       # This is a real and hermitian spectrum.
@@ -635,6 +653,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       self.assertAllEqual((2, 2 * 3 * 5, 2 * 3 * 5), matrix.shape)
       self.assertAllClose(matrix, matrix_h)
 
+  @test_util.run_deprecated_v1
   def test_defining_operator_using_real_convolution_kernel(self):
     with self.cached_session():
       convolution_kernel = linear_operator_test_util.random_normal(
@@ -653,6 +672,7 @@ class LinearOperatorCirculant3DTest(test.TestCase):
       self.assertAllEqual((2, 2 * 3 * 5, 2 * 3 * 5), matrix.shape)
       np.testing.assert_allclose(0, np.imag(matrix), atol=1e-6)
 
+  @test_util.run_deprecated_v1
   def test_defining_spd_operator_by_taking_real_part(self):
     with self.cached_session() as sess:
       # S is real and positive.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
index 3f19dc4bff..214b73aa2f 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
@@ -21,6 +21,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
@@ -179,6 +180,7 @@ class NonSquareLinearOperatorCompositionTest(
 
     return operator, mat
 
+  @test_util.run_deprecated_v1
   def test_static_shapes(self):
     operators = [
         linalg.LinearOperatorFullMatrix(rng.rand(2, 3, 4)),
@@ -187,6 +189,7 @@ class NonSquareLinearOperatorCompositionTest(
     operator = linalg.LinearOperatorComposition(operators)
     self.assertAllEqual((2, 3, 5), operator.shape)
 
+  @test_util.run_deprecated_v1
   def test_shape_tensors_when_statically_available(self):
     operators = [
         linalg.LinearOperatorFullMatrix(rng.rand(2, 3, 4)),
@@ -196,6 +199,7 @@ class NonSquareLinearOperatorCompositionTest(
     with self.cached_session():
       self.assertAllEqual((2, 3, 5), operator.shape_tensor().eval())
 
+  @test_util.run_deprecated_v1
   def test_shape_tensors_when_only_dynamically_available(self):
     mat_1 = rng.rand(1, 2, 3, 4)
     mat_2 = rng.rand(1, 2, 4, 5)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index 80889a162f..dcbc0dd7c9 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -80,6 +81,7 @@ class LinearOperatorDiagTest(
       with self.assertRaisesOpError("non-positive real.*not positive definite"):
         operator.assert_positive_definite().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite_does_not_raise_if_pd_and_complex(self):
     with self.cached_session():
       x = [1., 2.]
@@ -96,6 +98,7 @@ class LinearOperatorDiagTest(
       with self.assertRaisesOpError("Singular operator"):
         operator.assert_non_singular().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_does_not_raise_for_complex_nonsingular(self):
     with self.cached_session():
       x = [1., 0.]
@@ -113,6 +116,7 @@ class LinearOperatorDiagTest(
       with self.assertRaisesOpError("imaginary.*not self-adjoint"):
         operator.assert_self_adjoint().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_self_adjoint_does_not_raise_for_diag_with_zero_imag(self):
     with self.cached_session():
       x = [1., 0.]
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
index 36575ceec3..aff0b1ae14 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
@@ -69,6 +70,7 @@ class SquareLinearOperatorFullMatrixTest(
     # Auto-detected.
     self.assertTrue(operator.is_square)
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_raises_if_cond_too_big_but_finite(self):
     with self.cached_session():
       tril = linear_operator_test_util.random_tril_matrix(
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index e9fd91c6cf..2da5e712d7 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -60,16 +61,19 @@ class LinearOperatorIdentityTest(
 
     return operator, mat
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorIdentity(num_rows=2)
       operator.assert_positive_definite().run()  # Should not fail
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorIdentity(num_rows=2)
       operator.assert_non_singular().run()  # Should not fail
 
+  @test_util.run_deprecated_v1
   def test_assert_self_adjoint(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorIdentity(num_rows=2)
@@ -109,6 +113,7 @@ class LinearOperatorIdentityTest(
     with self.assertRaisesRegexp(ValueError, "must be non-negative"):
       linalg_lib.LinearOperatorIdentity(num_rows=2, batch_shape=[-2])
 
+  @test_util.run_deprecated_v1
   def test_non_scalar_num_rows_raises_dynamic(self):
     with self.cached_session():
       num_rows = array_ops.placeholder(dtypes.int32)
@@ -117,6 +122,7 @@ class LinearOperatorIdentityTest(
       with self.assertRaisesOpError("must be a 0-D Tensor"):
         operator.to_dense().eval(feed_dict={num_rows: [2]})
 
+  @test_util.run_deprecated_v1
   def test_negative_num_rows_raises_dynamic(self):
     with self.cached_session():
       num_rows = array_ops.placeholder(dtypes.int32)
@@ -125,6 +131,7 @@ class LinearOperatorIdentityTest(
       with self.assertRaisesOpError("must be non-negative"):
         operator.to_dense().eval(feed_dict={num_rows: -2})
 
+  @test_util.run_deprecated_v1
   def test_non_1d_batch_shape_raises_dynamic(self):
     with self.cached_session():
       batch_shape = array_ops.placeholder(dtypes.int32)
@@ -133,6 +140,7 @@ class LinearOperatorIdentityTest(
       with self.assertRaisesOpError("must be a 1-D"):
         operator.to_dense().eval(feed_dict={batch_shape: 2})
 
+  @test_util.run_deprecated_v1
   def test_negative_batch_shape_raises_dynamic(self):
     with self.cached_session():
       batch_shape = array_ops.placeholder(dtypes.int32)
@@ -147,6 +155,7 @@ class LinearOperatorIdentityTest(
     with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"):
       operator.matmul(x)
 
+  @test_util.run_deprecated_v1
   def test_wrong_matrix_dimensions_raises_dynamic(self):
     num_rows = array_ops.placeholder(dtypes.int32)
     x = array_ops.placeholder(dtypes.float32)
@@ -172,6 +181,7 @@ class LinearOperatorIdentityTest(
       self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape())
       self.assertAllClose(*self.evaluate([operator_matmul, expected]))
 
+  @test_util.run_deprecated_v1
   def test_default_batch_shape_broadcasts_with_everything_dynamic(self):
     # These cannot be done in the automated (base test class) tests since they
     # test shapes that tf.batch_matmul cannot handle.
@@ -209,6 +219,7 @@ class LinearOperatorIdentityTest(
       self.assertAllEqual(operator_matmul.get_shape(), expected.get_shape())
       self.assertAllClose(*self.evaluate([operator_matmul, expected]))
 
+  @test_util.run_deprecated_v1
   def test_broadcast_matmul_dynamic_shapes(self):
     # These cannot be done in the automated (base test class) tests since they
     # test shapes that tf.batch_matmul cannot handle.
@@ -309,6 +320,7 @@ class LinearOperatorScaledIdentityTest(
 
     return operator, matrix
 
+  @test_util.run_deprecated_v1
   def test_assert_positive_definite_does_not_raise_when_positive(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorScaledIdentity(
@@ -322,6 +334,7 @@ class LinearOperatorScaledIdentityTest(
       with self.assertRaisesOpError("not positive definite"):
         operator.assert_positive_definite().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_non_singular_does_not_raise_when_non_singular(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorScaledIdentity(
@@ -335,6 +348,7 @@ class LinearOperatorScaledIdentityTest(
       with self.assertRaisesOpError("was singular"):
         operator.assert_non_singular().run()
 
+  @test_util.run_deprecated_v1
   def test_assert_self_adjoint_does_not_raise_when_self_adjoint(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorScaledIdentity(
@@ -372,6 +386,7 @@ class LinearOperatorScaledIdentityTest(
     with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"):
       operator.matmul(x)
 
+  @test_util.run_deprecated_v1
   def test_wrong_matrix_dimensions_raises_dynamic(self):
     num_rows = array_ops.placeholder(dtypes.int32)
     x = array_ops.placeholder(dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
index 2b1ae6e1f5..513b246803 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
@@ -21,6 +21,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_kronecker as kronecker
@@ -53,6 +54,7 @@ def _kronecker_dense(factors):
 
 class KroneckerDenseTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testKroneckerDenseMatrix(self):
     x = ops.convert_to_tensor([[2., 3.], [1., 2.]], dtype=dtypes.float32)
     y = ops.convert_to_tensor([[1., 2.], [5., -1.]], dtype=dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
index 2f67df408c..8f8b15e8ed 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -107,6 +108,7 @@ class LinearOperatorTest(test.TestCase):
     self.assertAllEqual(4, operator.domain_dimension)
     self.assertAllEqual(3, operator.range_dimension)
 
+  @test_util.run_deprecated_v1
   def test_all_shape_methods_defined_by_the_one_method_shape(self):
     with self.cached_session():
       shape = (1, 2, 3, 4)
@@ -136,6 +138,7 @@ class LinearOperatorTest(test.TestCase):
       self.assertAllEqual((2, 3, 4), operator_dense.get_shape())
       self.assertAllClose(matrix, self.evaluate(operator_dense))
 
+  @test_util.run_deprecated_v1
   def test_generic_to_dense_method_non_square_matrix_tensor(self):
     matrix = rng.randn(2, 3, 4)
     matrix_ph = array_ops.placeholder(dtypes.float64)
@@ -175,6 +178,7 @@ class LinearOperatorTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "but.*was square"):
       _ = LinearOperatorShape(shape=(2, 4, 4), is_square=False).is_square
 
+  @test_util.run_deprecated_v1
   def test_is_square_set_inconsistent_with_other_hints_raises(self):
     with self.assertRaisesRegexp(ValueError, "is always square"):
       matrix = array_ops.placeholder(dtypes.float32)
@@ -185,6 +189,7 @@ class LinearOperatorTest(test.TestCase):
       LinearOperatorMatmulSolve(
           matrix, is_positive_definite=True, is_square=False)
 
+  @test_util.run_deprecated_v1
   def test_non_square_operators_raise_on_determinant_and_solve(self):
     operator = LinearOperatorShape((2, 3))
     with self.assertRaisesRegexp(NotImplementedError, "not be square"):
@@ -199,6 +204,7 @@ class LinearOperatorTest(test.TestCase):
       LinearOperatorMatmulSolve(
           matrix, is_positive_definite=True, is_square=False)
 
+  @test_util.run_deprecated_v1
   def test_is_square_manual_set_works(self):
     matrix = array_ops.placeholder(dtypes.float32)
     # Default is None.
@@ -208,6 +214,7 @@ class LinearOperatorTest(test.TestCase):
     operator = LinearOperatorMatmulSolve(matrix, is_square=True)
     self.assertTrue(operator.is_square)
 
+  @test_util.run_deprecated_v1
   def test_linear_operator_matmul_hints_closed(self):
     matrix = array_ops.placeholder(dtypes.float32)
     operator1 = LinearOperatorMatmulSolve(matrix)
@@ -234,6 +241,7 @@ class LinearOperatorTest(test.TestCase):
     self.assertTrue(operator_matmul.is_self_adjoint)
     self.assertEqual(None, operator_matmul.is_positive_definite)
 
+  @test_util.run_deprecated_v1
   def test_linear_operator_matmul_hints_false(self):
     matrix = array_ops.placeholder(dtypes.float32)
     operator1 = LinearOperatorMatmulSolve(
@@ -266,6 +274,7 @@ class LinearOperatorTest(test.TestCase):
     self.assertEqual(None, operator_matmul.is_self_adjoint)
     self.assertEqual(None, operator_matmul.is_positive_definite)
 
+  @test_util.run_deprecated_v1
   def test_linear_operator_matmul_hint_infer_square(self):
     matrix1 = array_ops.placeholder(shape=[2, 3], dtype=dtypes.float32)
     matrix2 = array_ops.placeholder(shape=[3, 2], dtype=dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
index f12714677e..d1e6c37e35 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
@@ -21,6 +21,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -32,12 +33,14 @@ rng = np.random.RandomState(0)
 
 class AssertZeroImagPartTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_real_tensor_doesnt_raise(self):
     x = ops.convert_to_tensor([0., 2, 3])
     with self.cached_session():
       # Should not raise.
       linear_operator_util.assert_zero_imag_part(x, message="ABC123").run()
 
+  @test_util.run_deprecated_v1
   def test_complex_tensor_with_imag_zero_doesnt_raise(self):
     x = ops.convert_to_tensor([1., 0, 3])
     y = ops.convert_to_tensor([0., 0, 0])
@@ -57,6 +60,7 @@ class AssertZeroImagPartTest(test.TestCase):
 
 class AssertNoEntriesWithModulusZeroTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_nonzero_real_tensor_doesnt_raise(self):
     x = ops.convert_to_tensor([1., 2, 3])
     with self.cached_session():
@@ -64,6 +68,7 @@ class AssertNoEntriesWithModulusZeroTest(test.TestCase):
       linear_operator_util.assert_no_entries_with_modulus_zero(
           x, message="ABC123").run()
 
+  @test_util.run_deprecated_v1
   def test_nonzero_complex_tensor_doesnt_raise(self):
     x = ops.convert_to_tensor([1., 0, 3])
     y = ops.convert_to_tensor([1., 2, 0])
@@ -104,6 +109,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
     with self.cached_session():
       self.assertAllClose(arr, self.evaluate(tensor))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast(self):
     # x.batch_shape = [3, 1, 2]
     # y.batch_shape = [4, 1]
@@ -142,6 +148,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
       self.assertAllClose(x_bc_expected, x_bc_)
       self.assertAllClose(y_bc_expected, y_bc_)
 
+  @test_util.run_deprecated_v1
   def test_dynamic_dims_broadcast_32bit(self):
     # x.batch_shape = [3, 1, 2]
     # y.batch_shape = [4, 1]
@@ -162,6 +169,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
       self.assertAllClose(x_bc_expected, x_bc_)
       self.assertAllClose(y_bc_expected, y_bc_)
 
+  @test_util.run_deprecated_v1
   def test_dynamic_dims_broadcast_32bit_second_arg_higher_rank(self):
     # x.batch_shape =    [1, 2]
     # y.batch_shape = [3, 4, 1]
@@ -195,6 +203,7 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
 
 class CholeskySolveWithBroadcastTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast(self):
     # batch_shape = [2]
     chol = rng.rand(3, 3)
@@ -207,6 +216,7 @@ class CholeskySolveWithBroadcastTest(test.TestCase):
       expected = linalg_ops.cholesky_solve(chol_broadcast, rhs)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2, 2]
     chol = rng.rand(2, 3, 3)
@@ -233,6 +243,7 @@ class CholeskySolveWithBroadcastTest(test.TestCase):
 
 class MatmulWithBroadcastTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_x_has_extra_dims(self):
     # batch_shape = [2]
     # for each batch member, we have a 1x3 matrix times a 3x7 matrix ==> 1x7
@@ -246,6 +257,7 @@ class MatmulWithBroadcastTest(test.TestCase):
       expected = math_ops.matmul(x, y_broadcast)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_y_has_extra_dims(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -263,6 +275,7 @@ class MatmulWithBroadcastTest(test.TestCase):
       expected = math_ops.matmul(x_broadcast, y)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_y_has_extra_dims_transpose_a_and_b(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -282,6 +295,7 @@ class MatmulWithBroadcastTest(test.TestCase):
           x_broadcast, y, transpose_a=True, transpose_b=True)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_y_has_extra_dims_transpose_dynamic(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -308,6 +322,7 @@ class MatmulWithBroadcastTest(test.TestCase):
                               y_ph: y
                           }))
 
+  @test_util.run_deprecated_v1
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2]
     # for each batch member, we have a 1x3 matrix times a 3x7 matrix ==> 1x7
@@ -333,6 +348,7 @@ class MatmulWithBroadcastTest(test.TestCase):
 
 class MatrixSolveWithBroadcastTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_matrix_has_extra_dims(self):
     # batch_shape = [2]
     matrix = rng.rand(2, 3, 3)
@@ -346,6 +362,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
       expected = linalg_ops.matrix_solve(matrix, rhs_broadcast)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_rhs_has_extra_dims(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -364,6 +381,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
       expected = linalg_ops.matrix_solve(matrix_broadcast, rhs)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_rhs_has_extra_dims_dynamic(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -391,6 +409,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
               rhs_ph: rhs
           }))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_rhs_has_extra_dims_and_adjoint(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -410,6 +429,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
       expected = linalg_ops.matrix_solve(matrix_broadcast, rhs, adjoint=True)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2, 2]
     matrix = rng.rand(2, 3, 3)
@@ -436,6 +456,7 @@ class MatrixSolveWithBroadcastTest(test.TestCase):
 
 class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_matrix_has_extra_dims(self):
     # batch_shape = [2]
     matrix = rng.rand(2, 3, 3)
@@ -449,6 +470,7 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
       expected = linalg_ops.matrix_triangular_solve(matrix, rhs_broadcast)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_rhs_has_extra_dims(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -468,6 +490,7 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
       expected = linalg_ops.matrix_triangular_solve(matrix_broadcast, rhs)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_static_dims_broadcast_rhs_has_extra_dims_and_adjoint(self):
     # Since the second arg has extra dims, and the domain dim of the first arg
     # is larger than the number of linear equations, code will "flip" the extra
@@ -488,6 +511,7 @@ class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
           matrix_broadcast, rhs, adjoint=True)
       self.assertAllClose(expected.eval(), self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2]
     matrix = rng.rand(2, 3, 3)
@@ -522,6 +546,7 @@ class DomainDimensionStubOperator(object):
 
 class AssertCompatibleMatrixDimensionsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_compatible_dimensions_do_not_raise(self):
     with self.cached_session():
       x = ops.convert_to_tensor(rng.rand(2, 3, 4))
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
index e875579a7a..eb0b8ef127 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.ops.linalg import linear_operator_test_util
@@ -73,6 +74,7 @@ class LinearOperatorZerosTest(
       operator = linalg_lib.LinearOperatorZeros(num_rows=2)
       operator.assert_non_singular()
 
+  @test_util.run_deprecated_v1
   def test_assert_self_adjoint(self):
     with self.cached_session():
       operator = linalg_lib.LinearOperatorZeros(num_rows=2)
@@ -108,6 +110,7 @@ class LinearOperatorZerosTest(
     with self.assertRaisesRegexp(ValueError, "must be non-negative"):
       linalg_lib.LinearOperatorZeros(num_rows=2, batch_shape=[-2])
 
+  @test_util.run_deprecated_v1
   def test_non_scalar_num_rows_raises_dynamic(self):
     with self.cached_session():
       num_rows = array_ops.placeholder(dtypes.int32)
@@ -116,6 +119,7 @@ class LinearOperatorZerosTest(
       with self.assertRaisesOpError("must be a 0-D Tensor"):
         operator.to_dense().eval(feed_dict={num_rows: [2]})
 
+  @test_util.run_deprecated_v1
   def test_negative_num_rows_raises_dynamic(self):
     with self.cached_session():
       n = array_ops.placeholder(dtypes.int32)
@@ -129,6 +133,7 @@ class LinearOperatorZerosTest(
       with self.assertRaisesOpError("must be non-negative"):
         operator.to_dense().eval(feed_dict={n: -2})
 
+  @test_util.run_deprecated_v1
   def test_non_1d_batch_shape_raises_dynamic(self):
     with self.cached_session():
       batch_shape = array_ops.placeholder(dtypes.int32)
@@ -137,6 +142,7 @@ class LinearOperatorZerosTest(
       with self.assertRaisesOpError("must be a 1-D"):
         operator.to_dense().eval(feed_dict={batch_shape: 2})
 
+  @test_util.run_deprecated_v1
   def test_negative_batch_shape_raises_dynamic(self):
     with self.cached_session():
       batch_shape = array_ops.placeholder(dtypes.int32)
@@ -151,6 +157,7 @@ class LinearOperatorZerosTest(
     with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"):
       operator.matmul(x)
 
+  @test_util.run_deprecated_v1
   def test_wrong_matrix_dimensions_raises_dynamic(self):
     num_rows = array_ops.placeholder(dtypes.int32)
     x = array_ops.placeholder(dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/linalg_grad_test.py b/tensorflow/python/kernel_tests/linalg_grad_test.py
index 709ecbfc35..28e1d7e168 100644
--- a/tensorflow/python/kernel_tests/linalg_grad_test.py
+++ b/tensorflow/python/kernel_tests/linalg_grad_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -39,6 +40,7 @@ def _AddTest(test, op_name, testcase_name, fn):
 
 class ShapeTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBatchGradientUnknownSize(self):
     with self.cached_session():
       batch_size = constant_op.constant(3)
diff --git a/tensorflow/python/kernel_tests/linalg_ops_test.py b/tensorflow/python/kernel_tests/linalg_ops_test.py
index b5eeee0998..028167a786 100644
--- a/tensorflow/python/kernel_tests/linalg_ops_test.py
+++ b/tensorflow/python/kernel_tests/linalg_ops_test.py
@@ -25,6 +25,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -52,6 +53,7 @@ class CholeskySolveTest(test.TestCase):
   def setUp(self):
     self.rng = np.random.RandomState(0)
 
+  @test_util.run_deprecated_v1
   def test_works_with_five_different_random_pos_def_matrices(self):
     for n in range(1, 6):
       for np_type, atol in [(np.float32, 0.05), (np.float64, 1e-5)]:
@@ -73,6 +75,7 @@ class LogdetTest(test.TestCase):
   def setUp(self):
     self.rng = np.random.RandomState(42)
 
+  @test_util.run_deprecated_v1
   def test_works_with_five_different_random_pos_def_matrices(self):
     for n in range(1, 6):
       for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5),
@@ -102,6 +105,7 @@ class SlogdetTest(test.TestCase):
   def setUp(self):
     self.rng = np.random.RandomState(42)
 
+  @test_util.run_deprecated_v1
   def test_works_with_five_different_random_pos_def_matrices(self):
     for n in range(1, 6):
       for np_dtype, atol in [(np.float32, 0.05), (np.float64, 1e-5),
@@ -232,6 +236,7 @@ class EyeTest(parameterized.TestCase, test.TestCase):
               dtypes.complex128
           ])
       )
+  @test_util.run_deprecated_v1
   def test_eye_with_placeholder(
       self, num_rows, num_columns, batch_shape, dtype):
     eye_np = np.eye(num_rows, M=num_columns, dtype=dtype.as_numpy_dtype)
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 1d9f4032d1..8df1156438 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -66,6 +66,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with context.device("gpu:0"):
       self._testPushPop(max_num_elements)
 
+  @test_util.run_deprecated_v1
   def testPushInFullListFails(self):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32, element_shape=[], max_num_elements=1)
@@ -77,6 +78,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
+  @test_util.run_deprecated_v1
   def testPopFromEmptyTensorListFails(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -114,6 +116,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 3))
+  @test_util.run_deprecated_v1
   def testStackWithUnknownElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -134,6 +137,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 3))
+  @test_util.run_deprecated_v1
   def testStackWithPartiallyDefinedElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -154,6 +158,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
+  @test_util.run_deprecated_v1
   def testStackEmptyList(self, max_num_elements):
     # Should be able to stack empty lists with fully defined element_shape.
     l = list_ops.empty_tensor_list(
@@ -204,6 +209,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 3))
+  @test_util.run_deprecated_v1
   def testGatherWithUnknownElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -227,6 +233,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 3))
+  @test_util.run_deprecated_v1
   def testGatherWithPartiallyDefinedElementShape(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -250,6 +257,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 3))
+  @test_util.run_deprecated_v1
   def testGatherEmptyList(self, max_num_elements):
     # Should be able to gather from empty lists with fully defined
     # element_shape.
@@ -337,6 +345,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(self.evaluate(e), 10.0)
     self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0)
 
+  @test_util.run_deprecated_v1
   def testSetOnEmptyListWithMaxNumElementsFails(self):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32, element_shape=[], max_num_elements=3)
@@ -572,12 +581,14 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(self.evaluate(grad_c), [0.0, 4.0])
     self.assertAllEqual(self.evaluate(grad_c2), 6.0)
 
+  @test_util.run_deprecated_v1
   def testSetOutOfBounds(self):
     c = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(c, element_shape=[])
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(list_ops.tensor_list_set_item(l, 20, 3.0))
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSetItemWithMismatchedShapeFails(self):
     with self.cached_session() as sess:
       ph = array_ops.placeholder(dtypes.float32)
@@ -612,6 +623,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                 [[1.0, 2.0]] * 4)
     self.assertAllEqual(self.evaluate(updated_v_stacked), expected)
 
+  @test_util.run_deprecated_v1
   def testConcat(self):
     c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32)
     l0 = list_ops.tensor_list_from_tensor(c, element_shape=[])
@@ -670,6 +682,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           list_ops.tensor_list_concat_lists(l_batch_0, l_batch_of_int_tls,
                                             element_dtype=dtypes.float32))
 
+  @test_util.run_deprecated_v1
   def testPushBackBatch(self):
     c = constant_op.constant([1.0, 2.0], dtype=dtypes.float32)
     l0 = list_ops.tensor_list_from_tensor(c, element_shape=[])
@@ -797,6 +810,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(self.evaluate(res_1), [0.])
     self.assertAllEqual(self.evaluate(res_2), [0., 0.])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorListGetItemGradAggregation(self):
     l = list_ops.tensor_list_reserve(
         element_shape=[], num_elements=1, element_dtype=dtypes.float32)
@@ -808,6 +822,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.cached_session() as sess:
       self.assertSequenceEqual(self.evaluate(grad), [2.])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerBuildElementShape(self):
     fn = list_ops._build_element_shape
     # Unknown shape -> -1.
diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py
index e8fa1cfa28..85035e5f7d 100644
--- a/tensorflow/python/kernel_tests/logging_ops_test.py
+++ b/tensorflow/python/kernel_tests/logging_ops_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.platform import test
 
 class LoggingOpsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAssertDivideByZero(self):
     with self.cached_session() as sess:
       epsilon = ops.convert_to_tensor(1e-20)
@@ -305,12 +306,14 @@ class PrintV2Test(test.TestCase):
             tensor, output_stream="unknown")
         self.evaluate(print_op)
 
+  @test_util.run_deprecated_v1
   def testPrintOpName(self):
     with self.cached_session():
       tensor = math_ops.range(10)
       print_op = logging_ops.print_v2(tensor, name="print_name")
       self.assertEqual(print_op.name, "print_name")
 
+  @test_util.run_deprecated_v1
   def testNoDuplicateFormatOpGraphModeAfterExplicitFormat(self):
     with self.cached_session():
       tensor = math_ops.range(10)
@@ -379,6 +382,7 @@ class PrintGradientTest(test.TestCase):
     inp_printed = logging_ops.Print(inp, ["hello"])
     self.assertEqual(inp.get_shape(), inp_printed.get_shape())
 
+  @test_util.run_deprecated_v1
   def testPrintGradient(self):
     with self.cached_session():
       inp = constant_op.constant(2.0, shape=[100, 32], name="in")
diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 79961d8dd1..ad81e0be64 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -37,6 +37,7 @@ from tensorflow.python.training import server_lib
 
 class HashTableOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testHashTable(self):
     with self.cached_session():
       default_val = -1
@@ -61,6 +62,7 @@ class HashTableOpTest(test.TestCase):
                             self.evaluate(exported_keys_tensor))
       self.assertItemsEqual([0, 1, 2], self.evaluate(exported_values_tensor))
 
+  @test_util.run_deprecated_v1
   def testHashTableFindHighRank(self):
     with self.cached_session():
       default_val = -1
@@ -79,6 +81,7 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([[0, 1], [-1, -1]], result)
 
+  @test_util.run_deprecated_v1
   def testHashTableInitWithPythonArrays(self):
     with self.cached_session():
       default_val = -1
@@ -97,6 +100,7 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
+  @test_util.run_deprecated_v1
   def testHashTableInitWithNumPyArrays(self):
     with self.cached_session():
       default_val = -1
@@ -114,6 +118,7 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
+  @test_util.run_deprecated_v1
   def testMultipleHashTables(self):
     with self.cached_session() as sess:
       default_val = -1
@@ -142,6 +147,7 @@ class HashTableOpTest(test.TestCase):
       self.assertAllEqual([0, 1, -1], out2)
       self.assertAllEqual([0, 1, -1], out3)
 
+  @test_util.run_deprecated_v1
   def testHashTableWithTensorDefault(self):
     with self.cached_session():
       default_val = constant_op.constant(-1, dtypes.int64)
@@ -157,6 +163,7 @@ class HashTableOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
+  @test_util.run_deprecated_v1
   def testHashTableWithSparseTensorInput(self):
     with self.cached_session() as sess:
       default_val = constant_op.constant(-1, dtypes.int64)
@@ -180,6 +187,7 @@ class HashTableOpTest(test.TestCase):
       self.assertAllEqual(sp_indices, out_indices)
       self.assertAllEqual(sp_shape, out_shape)
 
+  @test_util.run_deprecated_v1
   def testSignatureMismatch(self):
     with self.cached_session():
       default_val = -1
@@ -210,6 +218,7 @@ class HashTableOpTest(test.TestCase):
             lookup_ops.KeyValueTensorInitializer(["a"], [1], [dtypes.string],
                                                  dtypes.int64), default_val)
 
+  @test_util.run_deprecated_v1
   def testNotInitialized(self):
     with self.cached_session():
       default_val = -1
@@ -223,6 +232,7 @@ class HashTableOpTest(test.TestCase):
       with self.assertRaisesOpError("Table not initialized"):
         self.evaluate(output)
 
+  @test_util.run_deprecated_v1
   def testInitializeTwice(self):
     with self.cached_session():
       default_val = -1
@@ -235,6 +245,7 @@ class HashTableOpTest(test.TestCase):
       with self.assertRaisesOpError("Table already initialized"):
         table.initializer.run()
 
+  @test_util.run_deprecated_v1
   def testInitializationWithInvalidDimensions(self):
     with self.cached_session():
       default_val = -1
@@ -245,6 +256,7 @@ class HashTableOpTest(test.TestCase):
         lookup_ops.HashTable(
             lookup_ops.KeyValueTensorInitializer(keys, values), default_val)
 
+  @test_util.run_deprecated_v1
   def testMultipleSessions(self):
     # Start a server
     server = server_lib.Server(
@@ -274,6 +286,7 @@ class HashTableOpTest(test.TestCase):
       table.initializer.run()
       self.assertAllEqual(3, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testHashTableInt32String(self):
     with self.cached_session():
       default_val = "n/a"
@@ -298,6 +311,7 @@ class IndexTableFromFile(test.TestCase):
       f.write("\n".join(values) + "\n")
     return vocabulary_file
 
+  @test_util.run_deprecated_v1
   def test_string_index_table_from_file(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
     with self.cached_session():
@@ -310,6 +324,7 @@ class IndexTableFromFile(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_string_index_table_from_multicolumn_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain\t300", "salad\t20", "surgery\t1"))
@@ -326,6 +341,7 @@ class IndexTableFromFile(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_string_index_table_from_multicolumn_file_custom_delimiter(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain 300", "salad 20", "surgery 1"))
@@ -343,6 +359,7 @@ class IndexTableFromFile(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_string_index_table_from_file_tensor_filename(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
     with self.cached_session():
@@ -358,6 +375,7 @@ class IndexTableFromFile(test.TestCase):
       self.assertEqual(1,
                        len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
 
+  @test_util.run_deprecated_v1
   def test_string_index_table_from_file_placeholder_filename(self):
     vocabulary_file = self._createVocabFile("f2i_vocab1.txt")
     with self.cached_session():
@@ -375,6 +393,7 @@ class IndexTableFromFile(test.TestCase):
       self.assertEqual(0,
                        len(ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS)))
 
+  @test_util.run_deprecated_v1
   def test_int32_index_table_from_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab2.txt", values=("42", "1", "-1000"))
@@ -391,6 +410,7 @@ class IndexTableFromFile(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_int64_index_table_from_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab3.txt", values=("42", "1", "-1000"))
@@ -407,6 +427,7 @@ class IndexTableFromFile(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_default_value(self):
     default_value = -42
     vocabulary_file = self._createVocabFile("f2i_vocab4.txt")
@@ -420,6 +441,7 @@ class IndexTableFromFile(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, default_value), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_oov_buckets(self):
     vocabulary_file = self._createVocabFile("f2i_vocab5.txt")
     with self.cached_session():
@@ -468,6 +490,7 @@ class IndexTableFromFile(test.TestCase):
         vocabulary_file=vocabulary_file,
         vocab_size=0)
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_vocab_size_too_small(self):
     vocabulary_file = self._createVocabFile("f2i_vocab6.txt")
     with self.cached_session():
@@ -481,6 +504,7 @@ class IndexTableFromFile(test.TestCase):
       self.assertAllEqual((1, -1, -1), self.evaluate(ids))
       self.assertEqual(2, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_vocab_size_too_large(self):
     vocabulary_file = self._createVocabFile("f2i_vocab7.txt")
     with self.cached_session():
@@ -489,6 +513,7 @@ class IndexTableFromFile(test.TestCase):
       self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                               "Invalid vocab_size", table.initializer.run)
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_file_with_vocab_size(self):
     vocabulary_file = self._createVocabFile("f2i_vocab8.txt")
 
@@ -577,6 +602,7 @@ class KeyValueTensorInitializerTest(test.TestCase):
       table = lookup_ops.HashTable(init, default_value=-1)
       table.initializer.run()
 
+  @test_util.run_deprecated_v1
   def test_int32(self):
     with ops.Graph().as_default(), self.cached_session():
       init = lookup_ops.KeyValueTensorInitializer((42, 1, -1000), (0, 1, 2),
@@ -590,6 +616,7 @@ class KeyValueTensorInitializerTest(test.TestCase):
 class IndexTableFromTensor(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def test_index_table_from_tensor_with_tensor_init(self):
     table = lookup_ops.index_table_from_tensor(
         vocabulary_list=("brain", "salad", "surgery"), num_oov_buckets=1)
@@ -606,6 +633,7 @@ class IndexTableFromTensor(test.TestCase):
     ids = table.lookup(constant_op.constant(("salad", "surgery", "tarkus")))
     self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_int32_index_table_from_tensor_with_tensor_init(self):
     with self.cached_session():
       table = lookup_ops.index_table_from_tensor(
@@ -618,6 +646,7 @@ class IndexTableFromTensor(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_int64_index_table_from_tensor_with_tensor_init(self):
     with self.cached_session():
       table = lookup_ops.index_table_from_tensor(
@@ -630,6 +659,7 @@ class IndexTableFromTensor(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((1, 2, 3), self.evaluate(ids))
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_tensor_with_default_value(self):
     default_value = -42
     with self.cached_session():
@@ -650,6 +680,7 @@ class IndexTableFromTensor(test.TestCase):
         lookup_ops.index_table_from_tensor(
             vocabulary_list=None, num_oov_buckets=1)
 
+  @test_util.run_deprecated_v1
   def test_index_table_from_tensor_empty_vocabulary_list(self):
     with self.cached_session():
       table = lookup_ops.index_table_from_tensor(
@@ -686,6 +717,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       f.write("\n".join(values) + "\n")
     return vocabulary_file
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table(self):
     vocabulary_path = self._createVocabFile("i2f_vocab1.txt")
     # vocabulary_file supports string and tensor
@@ -703,6 +735,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
         self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                             self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_from_multicolumn_file(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain\t300", "salad\t20", "surgery\t1"))
@@ -718,6 +751,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_from_multicolumn_file_custom_delimiter(self):
     vocabulary_file = self._createVocabFile(
         "f2i_vocab1.txt", values=("brain 300", "salad 20", "surgery 1"))
@@ -734,6 +768,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_default_value(self):
     default_value = b"NONE"
     vocabulary_file = self._createVocabFile("f2i_vocab2.txt")
@@ -747,6 +782,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       self.assertAllEqual((b"salad", b"surgery", default_value),
                           self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_vocab_size_too_small(self):
     default_value = b"NONE"
     vocabulary_file = self._createVocabFile("f2i_vocab2.txt")
@@ -762,6 +798,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       self.assertAllEqual((b"salad", default_value, default_value),
                           self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_vocab_size_too_large(self):
     vocabulary_file = self._createVocabFile("f2i_vocab6.txt")
     with self.cached_session():
@@ -775,6 +812,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
       self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                               "Invalid vocab_size", init.run)
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_with_vocab_size(self):
     vocabulary_file = self._createVocabFile("f2i_vocab7.txt")
     with self.cached_session():
@@ -791,6 +829,7 @@ class IndexToStringTableFromFileTest(test.TestCase):
 
 class IndexToStringTableFromTensorTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_table_from_tensor(self):
     with self.cached_session():
       vocabulary_list = constant_op.constant(["brain", "salad", "surgery"])
@@ -806,6 +845,7 @@ class IndexToStringTableFromTensorTest(test.TestCase):
       self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"),
                           self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_duplicate_entries(self):
     with self.cached_session():
       vocabulary_list = constant_op.constant(["hello", "hello"])
@@ -816,6 +856,7 @@ class IndexToStringTableFromTensorTest(test.TestCase):
       lookup_ops.tables_initializer().run()
       self.assertAllEqual((b"hello", b"hello", b"UNK"), self.evaluate(features))
 
+  @test_util.run_deprecated_v1
   def test_index_to_string_with_default_value(self):
     default_value = b"NONE"
     with self.cached_session():
@@ -855,6 +896,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
     result = self.evaluate(output)
     self.assertAllEqual([0, 1, -1], result)
 
+  @test_util.run_deprecated_v1
   def testInitializeInt64Table(self):
     vocabulary_file = self._createVocabFile(
         "one_column_int64.txt", values=("42", "1", "-1000"))
@@ -874,6 +916,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
+  @test_util.run_deprecated_v1
   def testInitializeIndexTable(self):
     vocabulary_file = self._createVocabFile("one_column_2.txt")
 
@@ -893,6 +936,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([b"brain", b"salad", b"surgery", b"UNK"], result)
 
+  @test_util.run_deprecated_v1
   def testMultiColumn(self):
     vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt")
     with open(vocabulary_file, "w") as f:
@@ -915,6 +959,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([1, 5, 6], result)
 
+  @test_util.run_deprecated_v1
   def testInvalidDataTypeInMultiColumn(self):
     vocabulary_file = os.path.join(self.get_temp_dir(), "three_columns.txt")
     with open(vocabulary_file, "w") as f:
@@ -945,6 +990,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
                                            key_index, dtypes.string,
                                            value_index), default_value)
 
+  @test_util.run_deprecated_v1
   def testInvalidIndex(self):
     vocabulary_file = self._createVocabFile("one_column_4.txt")
     with self.cached_session():
@@ -959,6 +1005,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       with self.assertRaisesOpError("Invalid number of columns"):
         table.initializer.run()
 
+  @test_util.run_deprecated_v1
   def testInitializeSameTableWithMultipleNodes(self):
     vocabulary_file = self._createVocabFile("one_column_5.txt")
 
@@ -1010,6 +1057,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
                 dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
             default_value)
 
+  @test_util.run_deprecated_v1
   def testInitializeWithVocabSize(self):
     with self.cached_session():
       default_value = -1
@@ -1056,6 +1104,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       table3.initializer.run()
       self.assertEquals(vocab_size, table3.size().eval())
 
+  @test_util.run_deprecated_v1
   def testFeedVocabularyName(self):
     vocabulary_file = self._createVocabFile("feed_vocabulary.txt")
 
@@ -1082,6 +1131,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       result = self.evaluate(output)
       self.assertAllEqual([0, 1, -1], result)
 
+  @test_util.run_deprecated_v1
   def testInvalidFilenames(self):
     vocabulary_file = self._createVocabFile("filename_shape.txt")
 
@@ -1106,6 +1156,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
                 dtypes.int64, lookup_ops.TextFileIndex.LINE_NUMBER),
             default_value)
 
+  @test_util.run_deprecated_v1
   def testIdToStringTable(self):
     vocab_file = self._createVocabFile("feat_to_id_1.txt")
     with self.cached_session():
@@ -1124,6 +1175,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
                           self.evaluate(out))
       self.assertEquals(vocab_size, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testStringToIdTable(self):
     vocab_file = self._createVocabFile("feat_to_id_2.txt")
     with self.cached_session():
@@ -1140,6 +1192,7 @@ class InitializeTableFromFileOpTest(test.TestCase):
       self.assertAllEqual([0, 1, 2, -1], self.evaluate(out))
       self.assertEquals(vocab_size, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testInt64ToIdTable(self):
     vocab_file = self._createVocabFile(
         "feat_to_id_3.txt", values=("42", "1", "-1000"))
@@ -1166,6 +1219,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       f.write("\n".join(values) + "\n")
     return vocabulary_file
 
+  @test_util.run_deprecated_v1
   def testStringIdTableWithHashBuckets(self):
     vocab_file = self._createVocabFile("feat_to_id_1.txt")
     with self.cached_session():
@@ -1186,6 +1240,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertAllEqual([0, 1, 2, 3], self.evaluate(out))
       self.assertEquals(vocab_size + oov_buckets, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testInt32IdTableWithHashBuckets(self):
     vocab_file = self._createVocabFile("feat_to_id_2.txt", ("42", "1", "-1000"))
     with self.cached_session():
@@ -1208,6 +1263,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertAllEqual([0, 1, 2, 3], self.evaluate(out))
       self.assertEquals(vocab_size + oov_buckets, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testInt64IdTableWithHashBuckets(self):
     vocab_file = self._createVocabFile("feat_to_id_3.txt", ("42", "1", "-1000"))
     with self.cached_session():
@@ -1228,6 +1284,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertAllEqual([0, 1, 2, 3], self.evaluate(out))
       self.assertEquals(vocab_size + oov_buckets, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testStringIdTableWithOnlyHashBucket(self):
     with self.cached_session():
       oov_buckets = 5
@@ -1249,6 +1306,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
           self.evaluate(out))
       self.assertEquals(oov_buckets, table.size().eval())
 
+  @test_util.run_deprecated_v1
   def testInt32IdTableWithOnlyHashBucket(self):
     with self.cached_session():
       oov_buckets = 5
@@ -1283,6 +1341,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
         lookup_ops.IdTableWithHashBuckets(
             None, num_oov_buckets=5, key_dtype=dtypes.bool)
 
+  @test_util.run_deprecated_v1
   def testIdTableWithHashBucketsWithMultipleInitializers(self):
     vocab_file = self._createVocabFile("feat_to_id_4.txt")
     with self.cached_session() as sess:
@@ -1323,6 +1382,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
           "table2_Lookup/hash_bucket": "StringToHashBucketStrong",
       }, sess.graph)
 
+  @test_util.run_deprecated_v1
   def testIdTableWithHashBucketsInitializationAcrossSessions(self):
     vocab_file = self._createVocabFile("feat_to_id_5.txt")
     shared_name = "across-sessions"
@@ -1368,6 +1428,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertAllEqual([3, 1, 3], self.evaluate(out2))
       self.assertEquals(vocab_size + oov_buckets, table2.size().eval())
 
+  @test_util.run_deprecated_v1
   def testIdTableWithHashBucketsWithMultipleInitializersDifferentDefault(self):
     vocab_file = self._createVocabFile("feat_to_id_6.txt")
     with self.cached_session() as sess:
@@ -1402,6 +1463,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertEquals(vocab_size + oov_buckets, table1.size().eval())
       self.assertEquals(vocab_size + oov_buckets, table2.size().eval())
 
+  @test_util.run_deprecated_v1
   def testSparseTensor(self):
     vocab_file = self._createVocabFile("feat_to_id_7.txt")
     input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]]
@@ -1430,6 +1492,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val)
       self.assertAllEqual(input_shape, sp_ids_shape)
 
+  @test_util.run_deprecated_v1
   def testInt32SparseTensor(self):
     input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]]
     input_shape = [4, 4]
@@ -1458,6 +1521,7 @@ class IdTableWithHashBucketsTest(test.TestCase):
       self.assertAllEqual([0, 1, 0, 2, 3], sp_ids_val)
       self.assertAllEqual(input_shape, sp_ids_shape)
 
+  @test_util.run_deprecated_v1
   def testInt64SparseTensor(self):
     input_indices = [[0, 0], [0, 1], [2, 0], [2, 2], [3, 0]]
     input_shape = [4, 4]
diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py
index bda63bcaa9..abff61f81b 100644
--- a/tensorflow/python/kernel_tests/losses_test.py
+++ b/tensorflow/python/kernel_tests/losses_test.py
@@ -51,22 +51,26 @@ class AbsoluteDifferenceLossTest(test.TestCase):
         losses.absolute_difference(
             self._predictions, self._predictions, weights=None)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectNoLossWeight(self):
     loss = losses.absolute_difference(self._predictions, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLoss(self):
     loss = losses.absolute_difference(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(5.5, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
       self.assertAlmostEqual(5.5 * weights, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.absolute_difference(self._labels, self._predictions,
@@ -123,6 +127,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         losses.softmax_cross_entropy(labels, logits, weights=None)
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     with self.cached_session():
       logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -132,6 +137,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals('softmax_cross_entropy_loss/value', loss.op.name)
       self.assertAlmostEqual(loss.eval(), 0.0, 3)
 
+  @test_util.run_deprecated_v1
   def testAllWrong(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -142,6 +148,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 10.0, 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPythonScalarWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -151,6 +158,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       loss = losses.softmax_cross_entropy(labels, logits, weights)
       self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -200,6 +208,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         losses.softmax_cross_entropy(labels, logits, weights=weights).eval()
 
+  @test_util.run_deprecated_v1
   def testSoftmaxLabelSmoothing(self):
     with self.cached_session():
       # Softmax Cross Entropy Loss is:
@@ -232,6 +241,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         losses.sparse_softmax_cross_entropy(labels, logits, weights=None)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectInt32Labels(self):
     with self.cached_session():
       logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -248,6 +258,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
     labels = constant_op.constant([[0], [1], [2]], dtype=dtypes.int32)
     losses.sparse_softmax_cross_entropy(labels, logits)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectInt64Labels(self):
     with self.cached_session():
       logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -257,6 +268,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 0.0, 3)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectNonColumnLabels(self):
     with self.cached_session():
       logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
@@ -266,6 +278,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 0.0, 3)
 
+  @test_util.run_deprecated_v1
   def testAllWrongInt32Labels(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -276,6 +289,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 10.0, 3)
 
+  @test_util.run_deprecated_v1
   def testAllWrongInt64Labels(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -286,6 +300,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 10.0, 3)
 
+  @test_util.run_deprecated_v1
   def testAllWrongNonColumnLabels(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -296,6 +311,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 10.0, 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPythonScalarWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -305,6 +321,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
       self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -325,6 +342,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
           labels, logits, constant_op.constant((weights,)))
       self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPlaceholderForWeights(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0],
                                    [0.0, 10.0, 0.0],
@@ -337,6 +355,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
                           feed_dict={weights: ((1.2,), (3.4,), (5.6,))})
       self.assertAlmostEqual((1.2 + 3.4 + 5.6) * 10.0 / 3.0, loss_val, 3)
 
+  @test_util.run_deprecated_v1
   def testUnknownShapePlaceholderForLogitsLabelsButScalarWeights(self):
     logits = array_ops.placeholder(dtypes.float32)
     labels = array_ops.placeholder(dtypes.int32)
@@ -352,6 +371,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
                           })
       self.assertAlmostEqual((1.0 + 1.0 + 1.0) * 10.0 / 3.0, loss_val, 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPlaceholderForLogitsLabelsAndWeights(self):
     logits = array_ops.placeholder(dtypes.float32, shape=(None, 3))
     labels = array_ops.placeholder(dtypes.int32, shape=(None, 1))
@@ -406,6 +426,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
       self.assertAlmostEqual(12.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testMeasurementSpecificWeightsRaisesException(self):
     with self.cached_session():
       logits = constant_op.constant([[100.0, -100.0, -100.0],
@@ -444,6 +465,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
         losses.sparse_softmax_cross_entropy(
             labels, logits, weights=weights).eval()
 
+  @test_util.run_deprecated_v1
   def testInconsistentWeightShapeRaisesException(self):
     """The weight tensor has incorrect shape."""
     with self.cached_session():
@@ -458,6 +480,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
         losses.sparse_softmax_cross_entropy(
             labels, logits, weights=weights).eval()
 
+  @test_util.run_deprecated_v1
   def testInconsistentLabelShapeRaisesException(self):
     """The label tensor has incorrect shape."""
     with self.cached_session():
@@ -475,6 +498,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
 
 class SigmoidCrossEntropyLossTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAllCorrectSigmoid(self):
     with self.cached_session():
       logits = constant_op.constant([[100.0, -100.0, -100.0],
@@ -486,6 +510,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name)
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testLossWithSingleDimPlaceholderForLogitsAndWeights1(self):
     logits = array_ops.placeholder(dtypes.float32, shape=(None, 1))
     labels = array_ops.placeholder(dtypes.float32, shape=(None, 1))
@@ -502,6 +527,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
                       })
       self.assertAlmostEqual(0.313, loss, 3)
 
+  @test_util.run_deprecated_v1
   def testLossWithSingleDimPlaceholderForLogitsAndWeights2(self):
     logits = array_ops.placeholder(dtypes.float32, shape=(None, 2))
     labels = array_ops.placeholder(dtypes.float32, shape=(None, 2))
@@ -518,6 +544,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
                       })
       self.assertAlmostEqual(0.313, loss, 3)
 
+  @test_util.run_deprecated_v1
   def testAllWrongSigmoid(self):
     with self.cached_session():
       logits = constant_op.constant([[100.0, -100.0, -100.0],
@@ -529,6 +556,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name)
       self.assertAlmostEqual(loss.eval(), 600.0 / 9.0, 3)
 
+  @test_util.run_deprecated_v1
   def testAllWrongSigmoidWithMeasurementSpecificWeights(self):
     with self.cached_session():
       logits = constant_op.constant([[100.0, -100.0, -100.0],
@@ -541,6 +569,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       self.assertEquals('sigmoid_cross_entropy_loss/value', loss.op.name)
       self.assertAlmostEqual(1700.0 / 7.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testMultiCorrectSigmoid(self):
     logits = constant_op.constant([[100.0, -100.0, 100.0],
                                    [100.0, 100.0, -100.0],
@@ -582,6 +611,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       self.assertAllClose(((0., 0., 0.), (0., 100., 100.), (100., 0., 100.)),
                           self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testSigmoidLabelSmoothingCorrect(self):
     with self.cached_session():
       logits = constant_op.constant([[100.0, -100.0, -100.0]])
@@ -605,6 +635,7 @@ class SigmoidCrossEntropyLossTest(test.TestCase):
       expected_value = (100.0 + 50.0 * label_smoothing) / 3.0
       self.assertAlmostEqual(loss.eval(), expected_value, 3)
 
+  @test_util.run_deprecated_v1
   def testSigmoidLabelSmoothingEqualsSoftmaxTwoLabel(self):
     with self.cached_session():
       label_smoothing = 0.1
@@ -646,11 +677,13 @@ class LogLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         losses.log_loss(self._labels, self._labels, weights=None)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectNoLossWeight(self):
     loss = losses.log_loss(self._labels, self._labels)
     with self.cached_session():
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectNoLossWeightWithPlaceholder(self):
     tf_predictions = array_ops.placeholder(
         dtypes.float32, shape=self._np_labels.shape)
@@ -659,12 +692,14 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(
           0.0, loss.eval(feed_dict={tf_predictions: self._np_labels}), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLoss(self):
     loss = losses.log_loss(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(-np.sum(self._expected_losses) / 6.0,
                              self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.log_loss(self._labels, self._predictions, weights)
@@ -672,6 +707,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
                              self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.log_loss(self._labels, self._predictions,
@@ -680,6 +716,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
                              self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeightAndPlaceholder(self):
     tf_predictions = array_ops.placeholder(
         dtypes.float32, shape=self._np_predictions.shape)
@@ -691,6 +728,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
                              loss, 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeightAndPlaceholderWithRankOnly(self):
     tf_predictions = array_ops.placeholder(dtypes.float32, shape=[None, None])
     weights = 2.3
@@ -750,6 +788,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(-np.sum(expected_losses) / 5.0,
                              self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithMeasurementSpecificWeightsWithPlaceholder(self):
     weights = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3))
     expected_losses = np.multiply(self._expected_losses, weights)
@@ -777,6 +816,7 @@ class LogLossTest(test.TestCase):
     with self.cached_session():
       self.assertAlmostEqual(-np.sum(expected_losses), self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithSampleSpecificWeightsMostZeroWithPlaceholder(self):
     weights = np.array([0, 0, 0, 0, 0, 2]).reshape((2, 3))
     expected_losses = np.multiply(self._expected_losses, weights)
@@ -805,6 +845,7 @@ class HingeLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         _ = losses.hinge_loss(labels, logits).eval()
 
+  @test_util.run_deprecated_v1
   def testAllOutsideMargin(self):
     with self.cached_session():
       logits = constant_op.constant([1.2, -1.4, -1.0, 2.1])
@@ -812,6 +853,7 @@ class HingeLossTest(test.TestCase):
       loss = losses.hinge_loss(labels, logits)
       self.assertAllClose(loss.eval(), 0.0, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testSomeInsideMargin(self):
     with self.cached_session():
       logits = constant_op.constant([[-0.7], [-1.4], [1.4], [0.6]])
@@ -821,6 +863,7 @@ class HingeLossTest(test.TestCase):
       # the margin so they incur some (small) loss.
       self.assertAllClose(loss.eval(), 0.175, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testSomeMisclassified(self):
     with self.cached_session():
       logits = constant_op.constant([[[1.2], [0.4], [-1.0], [-1.1]]])
@@ -840,6 +883,7 @@ class HuberLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         _ = losses.huber_loss(labels, predictions).eval()
 
+  @test_util.run_deprecated_v1
   def testAllQuadratic(self):
     with self.cached_session():
       predictions = constant_op.constant([1.5, -1.4, -1.0, 0.0])
@@ -848,6 +892,7 @@ class HuberLossTest(test.TestCase):
       self.assertAllClose(loss.eval(),
                           0.5 * (0.25 + 0.16 + 1.0 + 0.25) / 4., atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testAllLinear(self):
     with self.cached_session():
       predictions = constant_op.constant([1.5, -1.4, -1.0, 0.0])
@@ -856,6 +901,7 @@ class HuberLossTest(test.TestCase):
       self.assertAllClose(loss.eval(),
                           (1.5 + 2.4 + 1.0 + 1.5) / 4. - 0.5, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testMixedQuadraticLinear(self):
     with self.cached_session():
       predictions = constant_op.constant([[1.5, -1.4, -1.0, 0.0],
@@ -901,6 +947,7 @@ class MeanSquaredErrorTest(test.TestCase):
         losses.mean_squared_error(
             self._predictions, self._predictions, weights=None)
 
+  @test_util.run_deprecated_v1
   def testScalar(self):
     with self.cached_session():
       self.assertEqual(
@@ -908,22 +955,26 @@ class MeanSquaredErrorTest(test.TestCase):
           losses.mean_squared_error(predictions=constant_op.constant(0),
                                     labels=constant_op.constant(0)).eval())
 
+  @test_util.run_deprecated_v1
   def testAllCorrectNoLossWeight(self):
     loss = losses.mean_squared_error(self._predictions, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLoss(self):
     loss = losses.mean_squared_error(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(49.5, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
       self.assertAlmostEqual(49.5 * weights, self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.mean_squared_error(self._labels, self._predictions,
@@ -1017,10 +1068,12 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
       self.assertAlmostEqual(
           expected_loss, dynamic_inputs_op.eval(feed_dict=feed_dict), places=3)
 
+  @test_util.run_deprecated_v1
   def testAllCorrectNoLossWeight(self):
     self._test_valid_weights(
         self._labels, self._labels, expected_loss=0.0)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLoss(self):
     self._test_valid_weights(
         self._labels, self._predictions,
@@ -1051,6 +1104,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
           np_grad = self.evaluate(grad)
           self.assertFalse(np.isnan(np_grad).any())
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithPythonScalarWeight(self):
     weight = 2.3
     self._test_valid_weights(
@@ -1058,6 +1112,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
         expected_loss=weight * np.sum(self._expected_losses),
         weights=weight)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.mean_pairwise_squared_error(
@@ -1068,10 +1123,12 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
       self.assertAlmostEqual(weights * np.sum(self._expected_losses),
                              self.evaluate(loss), 3)
 
+  @test_util.run_deprecated_v1
   def testNonZeroLossWithScalarZeroWeight(self):
     self._test_valid_weights(
         self._labels, self._predictions, expected_loss=0.0, weights=0.0)
 
+  @test_util.run_deprecated_v1
   def test3d(self):
     labels = np.array([
         [[1, 9, 2], [12, 11, 10], [9, 8, 7]],
@@ -1083,6 +1140,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
     ])
     self._test_valid_weights(labels, predictions, expected_loss=137.5)
 
+  @test_util.run_deprecated_v1
   def test3dWeightedScalar(self):
     labels = np.array([
         [[1, 9, 2], [12, 11, 10], [9, 8, 7]],
@@ -1121,6 +1179,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
             weights_placeholder: weights,
         })
 
+  @test_util.run_deprecated_v1
   def testInvalid3dWeighted2x0(self):
     labels = np.array([
         [[1, 9, 2], [12, 11, 10], [9, 8, 7]],
@@ -1133,6 +1192,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
     self._test_invalid_weights(
         labels, predictions, weights=np.asarray((1.2, 3.4)))
 
+  @test_util.run_deprecated_v1
   def test3dWeighted2x3x3(self):
     labels = np.array([
         [[1, 9, 2], [12, 11, 10], [9, 8, 7]],
@@ -1149,6 +1209,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
         expected_loss=9 * 137.5,
         weights=np.ones((2, 3, 3)))
 
+  @test_util.run_deprecated_v1
   def testLossWithAllZeroBatchSpecificWeights(self):
     self._test_valid_weights(
         self._labels, self._predictions, expected_loss=0.0,
@@ -1268,6 +1329,7 @@ class CosineDistanceLossTest(test.TestCase):
     with self.cached_session():
       self.assertEqual(3.0 / 4.0, self.evaluate(loss))
 
+  @test_util.run_deprecated_v1
   def testMeasurementSpecificWeightsWithPlaceholderWithShape(self):
     tf_predictions = array_ops.placeholder(
         dtypes.float32, shape=self._labels.shape)
diff --git a/tensorflow/python/kernel_tests/lrn_op_test.py b/tensorflow/python/kernel_tests/lrn_op_test.py
index 7ebeb91d90..fbe628c394 100644
--- a/tensorflow/python/kernel_tests/lrn_op_test.py
+++ b/tensorflow/python/kernel_tests/lrn_op_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -92,6 +93,7 @@ class LRNOpTest(test.TestCase):
       self.assertTrue(err < 1e-2)
     self.assertShapeEqual(expected, lrn_t)
 
+  @test_util.run_deprecated_v1
   def testCompute(self):
     for _ in range(2):
       self._RunAndVerify(dtypes.float32)
@@ -99,6 +101,7 @@ class LRNOpTest(test.TestCase):
       if not test.is_gpu_available():
         self._RunAndVerify(dtypes.float16)
 
+  @test_util.run_deprecated_v1
   def testGradientsZeroInput(self):
     with self.session(use_gpu=True):
       shape = [4, 4, 4, 4]
@@ -147,6 +150,7 @@ class LRNOpTest(test.TestCase):
     else:
       self.assertLess(err, 1.0)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     for _ in range(2):
       self._RunAndVerifyGradients(dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/manip_ops_test.py b/tensorflow/python/kernel_tests/manip_ops_test.py
index f71857a3cb..5700db4b95 100644
--- a/tensorflow/python/kernel_tests/manip_ops_test.py
+++ b/tensorflow/python/kernel_tests/manip_ops_test.py
@@ -62,6 +62,7 @@ class RollTest(test_util.TensorFlowTestCase):
     if np_input.dtype == np.float32:
       self._testGradient(np_input, shift, axis)
 
+  @test_util.run_deprecated_v1
   def testIntTypes(self):
     for t in [np.int32, np.int64]:
       self._testAll(np.random.randint(-100, 100, (5)).astype(t), 3, 0)
@@ -73,6 +74,7 @@ class RollTest(test_util.TensorFlowTestCase):
             np.random.randint(-100, 100, (4, 2, 1, 3)).astype(t), [0, 1, -2],
             [1, 2, 3])
 
+  @test_util.run_deprecated_v1
   def testFloatTypes(self):
     for t in [np.float32, np.float64]:
       self._testAll(np.random.rand(5).astype(t), 2, 0)
@@ -80,6 +82,7 @@ class RollTest(test_util.TensorFlowTestCase):
         self._testAll(np.random.rand(3, 4).astype(t), [1, 2], [1, 0])
         self._testAll(np.random.rand(1, 3, 4).astype(t), [1, 0, -3], [0, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testComplexTypes(self):
     for t in [np.complex64, np.complex128]:
       x = np.random.rand(4, 4).astype(t)
@@ -90,6 +93,7 @@ class RollTest(test_util.TensorFlowTestCase):
         x = np.random.rand(3, 2, 1, 1).astype(t)
         self._testAll(x + 1j * x, [2, 1, 1, 0], [0, 3, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testNegativeAxis(self):
     self._testAll(np.random.randint(-100, 100, (5)).astype(np.int32), 3, -1)
     self._testAll(np.random.randint(-100, 100, (4, 4)).astype(np.int32), 3, -2)
@@ -100,12 +104,14 @@ class RollTest(test_util.TensorFlowTestCase):
         manip_ops.roll(np.random.randint(-100, 100, (4, 4)).astype(np.int32),
                        3, -10).eval()
 
+  @test_util.run_deprecated_v1
   def testInvalidInputShape(self):
     # The input should be 1-D or higher, checked in shape function.
     with self.assertRaisesRegexp(
         ValueError, "Shape must be at least rank 1 but is rank 0"):
       manip_ops.roll(7, 1, 0)
 
+  @test_util.run_deprecated_v1
   def testRollInputMustVectorHigherRaises(self):
     # The input should be 1-D or higher, checked in kernel.
     tensor = array_ops.placeholder(dtype=dtypes.int32)
@@ -116,12 +122,14 @@ class RollTest(test_util.TensorFlowTestCase):
                                    "input must be 1-D or higher"):
         manip_ops.roll(tensor, shift, axis).eval(feed_dict={tensor: 7})
 
+  @test_util.run_deprecated_v1
   def testInvalidAxisShape(self):
     # The axis should be a scalar or 1-D, checked in shape function.
     with self.assertRaisesRegexp(
         ValueError, "Shape must be at most rank 1 but is rank 2"):
       manip_ops.roll([[1, 2], [3, 4]], 1, [[0, 1]])
 
+  @test_util.run_deprecated_v1
   def testRollAxisMustBeScalarOrVectorRaises(self):
     # The axis should be a scalar or 1-D, checked in kernel.
     tensor = [[1, 2], [3, 4]]
@@ -132,12 +140,14 @@ class RollTest(test_util.TensorFlowTestCase):
                                    "axis must be a scalar or a 1-D vector"):
         manip_ops.roll(tensor, shift, axis).eval(feed_dict={axis: [[0, 1]]})
 
+  @test_util.run_deprecated_v1
   def testInvalidShiftShape(self):
     # The shift should be a scalar or 1-D, checked in shape function.
     with self.assertRaisesRegexp(
         ValueError, "Shape must be at most rank 1 but is rank 2"):
       manip_ops.roll([[1, 2], [3, 4]], [[0, 1]], 1)
 
+  @test_util.run_deprecated_v1
   def testRollShiftMustBeScalarOrVectorRaises(self):
     # The shift should be a scalar or 1-D, checked in kernel.
     tensor = [[1, 2], [3, 4]]
@@ -148,11 +158,13 @@ class RollTest(test_util.TensorFlowTestCase):
                                    "shift must be a scalar or a 1-D vector"):
         manip_ops.roll(tensor, shift, axis).eval(feed_dict={shift: [[0, 1]]})
 
+  @test_util.run_deprecated_v1
   def testInvalidShiftAndAxisNotEqualShape(self):
     # The shift and axis must be same size, checked in shape function.
     with self.assertRaisesRegexp(ValueError, "both shapes must be equal"):
       manip_ops.roll([[1, 2], [3, 4]], [1], [0, 1])
 
+  @test_util.run_deprecated_v1
   def testRollShiftAndAxisMustBeSameSizeRaises(self):
     # The shift and axis must be same size, checked in kernel.
     tensor = [[1, 2], [3, 4]]
diff --git a/tensorflow/python/kernel_tests/map_stage_op_test.py b/tensorflow/python/kernel_tests/map_stage_op_test.py
index d503f3d7c9..dd16fad690 100644
--- a/tensorflow/python/kernel_tests/map_stage_op_test.py
+++ b/tensorflow/python/kernel_tests/map_stage_op_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -29,6 +30,7 @@ TIMEOUT = 1
 
 class MapStageTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -50,6 +52,7 @@ class MapStageTest(test.TestCase):
         _, yval = sess.run([stage, y], feed_dict={x: i, pi: i + 1, gi: i})
         self.assertAllClose(4 * (i - 1) * (i - 1) * 128, yval, rtol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testMultiple(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -72,6 +75,7 @@ class MapStageTest(test.TestCase):
         self.assertAllClose(
             4 * (i - 1) * (i - 1) * (i - 1) * 128, yval, rtol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testDictionary(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -121,6 +125,7 @@ class MapStageTest(test.TestCase):
 
     G.finalize()
 
+  @test_util.run_deprecated_v1
   def testPeek(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -150,6 +155,7 @@ class MapStageTest(test.TestCase):
 
       self.assertTrue(sess.run(size) == 10)
 
+  @test_util.run_deprecated_v1
   def testSizeAndClear(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -176,6 +182,7 @@ class MapStageTest(test.TestCase):
       sess.run(clear)
       self.assertEqual(sess.run(size), 0)
 
+  @test_util.run_deprecated_v1
   def testCapacity(self):
     capacity = 3
 
@@ -239,6 +246,7 @@ class MapStageTest(test.TestCase):
 
       self.assertTrue(sess.run(size) == 0)
 
+  @test_util.run_deprecated_v1
   def testMemoryLimit(self):
     memory_limit = 512 * 1024  # 512K
     chunk = 200 * 1024  # 256K
@@ -303,6 +311,7 @@ class MapStageTest(test.TestCase):
 
       self.assertTrue(sess.run(size) == 0)
 
+  @test_util.run_deprecated_v1
   def testOrdering(self):
     import six
     import random
@@ -341,6 +350,7 @@ class MapStageTest(test.TestCase):
 
       self.assertTrue(sess.run(size) == 0)
 
+  @test_util.run_deprecated_v1
   def testPartialDictInsert(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -400,6 +410,7 @@ class MapStageTest(test.TestCase):
               'v': 3
           }])
 
+  @test_util.run_deprecated_v1
   def testPartialIndexInsert(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -443,6 +454,7 @@ class MapStageTest(test.TestCase):
       # We can now obtain tuple associated with key 1
       self.assertTrue(sess.run([key, ret], feed_dict={gi: 1}) == [1, [1, 3, 2]])
 
+  @test_util.run_deprecated_v1
   def testPartialDictGetsAndPeeks(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -540,6 +552,7 @@ class MapStageTest(test.TestCase):
       # Nothing is left
       self.assertTrue(sess.run([size, isize]) == [0, 0])
 
+  @test_util.run_deprecated_v1
   def testPartialIndexGets(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py
index 6167e01864..983f463f5e 100644
--- a/tensorflow/python/kernel_tests/matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/matmul_op_test.py
@@ -194,6 +194,7 @@ except AttributeError:
 
 class MatMulInfixOperatorTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testMismatchedShape(self):
     with self.assertRaisesWithPredicateMatch(ValueError,
                                              lambda e: "Shape must" in str(e)):
@@ -201,6 +202,7 @@ class MatMulInfixOperatorTest(test_lib.TestCase):
           ops.convert_to_tensor([10.0, 20.0, 30.0]),
           ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]]))
 
+  @test_util.run_deprecated_v1
   def testMismatchedDimensions(self):
     with self.assertRaisesWithPredicateMatch(
         ValueError, lambda e: "Dimensions must" in str(e)):
@@ -208,12 +210,14 @@ class MatMulInfixOperatorTest(test_lib.TestCase):
           ops.convert_to_tensor([[10.0, 20.0, 30.0]]),
           ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]]))
 
+  @test_util.run_deprecated_v1
   def testInfixMatmulIsTfMatmul(self):
     a = ops.convert_to_tensor([[10.0, 20.0, 30.0]])
     b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]])
     c = infix_matmul(a, b)
     self.assertEqual(c.op.type, "MatMul")
 
+  @test_util.run_deprecated_v1
   def testInfixMatmulDoesDotProduct(self):
     a = ops.convert_to_tensor([[10.0, 20.0, 30.0]])
     b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]])
diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
index 83f4216e4d..372b6dc17f 100644
--- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py
@@ -122,12 +122,14 @@ class ExponentialOpTest(test.TestCase):
     # Complex batch
     self._verifyExponentialComplex(self._makeBatch(matrix1, matrix2))
 
+  @test_util.run_deprecated_v1
   def testNonSquareMatrix(self):
     # When the exponential of a non-square matrix is attempted we should return
     # an error
     with self.assertRaises(ValueError):
       linalg_impl.matrix_exponential(np.array([[1., 2., 3.], [3., 4., 5.]]))
 
+  @test_util.run_deprecated_v1
   def testWrongDimensions(self):
     # The input to the exponential should be at least a 2-dimensional tensor.
     tensor3 = constant_op.constant([1., 2.])
@@ -138,6 +140,7 @@ class ExponentialOpTest(test.TestCase):
     self._verifyExponentialReal(np.empty([0, 2, 2]))
     self._verifyExponentialReal(np.empty([2, 0, 0]))
 
+  @test_util.run_deprecated_v1
   def testDynamic(self):
     with self.session(use_gpu=True) as sess:
       inp = array_ops.placeholder(ops.dtypes.float32)
@@ -145,6 +148,7 @@ class ExponentialOpTest(test.TestCase):
       matrix = np.array([[1., 2.], [3., 4.]])
       sess.run(expm, feed_dict={inp: matrix})
 
+  @test_util.run_deprecated_v1
   def testConcurrentExecutesWithoutError(self):
     with self.session(use_gpu=True) as sess:
       matrix1 = random_ops.random_normal([5, 5], seed=42)
diff --git a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
index 13a7df7f95..a6f5da9d3d 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import linalg_ops
@@ -133,6 +134,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
       self.assertEqual(np_ans.shape, tf_ans_val.shape)
       self.assertAllClose(np_ans, tf_ans_val, atol=2 * tol, rtol=2 * tol)
 
+  @test_util.run_deprecated_v1
   def testWrongDimensions(self):
     # The matrix and right-hand sides should have the same number of rows.
     with self.session(use_gpu=True):
@@ -141,6 +143,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
       with self.assertRaises(ValueError):
         linalg_ops.matrix_solve_ls(matrix, rhs)
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     full = np.array([[1., 2.], [3., 4.], [5., 6.]])
     empty0 = np.empty([3, 0])
@@ -156,6 +159,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
         tf_ans = linalg_ops.matrix_solve_ls(empty1, empty1, fast=fast).eval()
         self.assertEqual(tf_ans.shape, (2, 2))
 
+  @test_util.run_deprecated_v1
   def testBatchResultSize(self):
     # 3x3x3 matrices, 3x3x1 right-hand sides.
     matrix = np.array([1., 2., 3., 4., 5., 6., 7., 8., 9.] * 3).reshape(3, 3, 3)
diff --git a/tensorflow/python/kernel_tests/matrix_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
index 80badee896..db7c4802f6 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_op_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import linalg_ops
@@ -75,6 +76,7 @@ class MatrixSolveOpTest(test.TestCase):
         [m, n]))
     return matrix
 
+  @test_util.run_deprecated_v1
   def testSolve(self):
     for n in 1, 2, 4, 9:
       matrix = self._generateMatrix(n, n)
@@ -82,6 +84,7 @@ class MatrixSolveOpTest(test.TestCase):
         rhs = self._generateMatrix(n, nrhs)
         self._verifySolve(matrix, rhs)
 
+  @test_util.run_deprecated_v1
   def testSolveBatch(self):
     for n in 2, 5:
       matrix = self._generateMatrix(n, n)
@@ -90,6 +93,7 @@ class MatrixSolveOpTest(test.TestCase):
         for batch_dims in [[2], [2, 2], [7, 4]]:
           self._verifySolve(matrix, rhs, batch_dims=batch_dims)
 
+  @test_util.run_deprecated_v1
   def testNonSquareMatrix(self):
     # When the solve of a non-square matrix is attempted we should return
     # an error
@@ -98,6 +102,7 @@ class MatrixSolveOpTest(test.TestCase):
         matrix = constant_op.constant([[1., 2., 3.], [3., 4., 5.]])
         linalg_ops.matrix_solve(matrix, matrix)
 
+  @test_util.run_deprecated_v1
   def testWrongDimensions(self):
     # The matrix and right-hand sides should have the same number of rows.
     with self.session(use_gpu=True):
@@ -115,6 +120,7 @@ class MatrixSolveOpTest(test.TestCase):
                                        [0., -1., 1.]])
         linalg_ops.matrix_solve(matrix, matrix).eval()
 
+  @test_util.run_deprecated_v1
   def testConcurrent(self):
     with self.session(use_gpu=True) as sess:
       all_ops = []
diff --git a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py
index 317b8f8716..dde83f12f3 100644
--- a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.platform import test
@@ -93,6 +94,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
         self.assertEqual(np_ans.shape, tf_val.shape)
         self.assertAllClose(np_ans, tf_val)
 
+  @test_util.run_deprecated_v1
   def testSolve(self):
     # 1x1 matrix, single rhs.
     matrix = np.array([[0.1]])
@@ -106,6 +108,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
     rhs1 = np.array([[1., 0., 1.], [0., 1., 1.]])
     self._verifySolveAllWaysReal(matrix, rhs1)
 
+  @test_util.run_deprecated_v1
   def testSolveComplex(self):
     # 1x1 matrix, single rhs.
     matrix = np.array([[0.1 + 1j * 0.1]])
@@ -122,6 +125,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
     rhs1 += 1j * rhs1
     self._verifySolveAllWaysComplex(matrix, rhs1)
 
+  @test_util.run_deprecated_v1
   def testSolveBatch(self):
     matrix = np.array([[1., 2.], [3., 4.]])
     rhs = np.array([[1., 0., 1.], [0., 1., 1.]])
@@ -130,6 +134,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
     # Batch of 3x2x2x2 matrices, 3x2x2x3 right-hand sides.
     self._verifySolveAllWaysReal(matrix, rhs, batch_dims=[3, 2])
 
+  @test_util.run_deprecated_v1
   def testSolveBatchComplex(self):
     matrix = np.array([[1., 2.], [3., 4.]]).astype(np.complex64)
     matrix += 1j * matrix
@@ -140,6 +145,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
     # Batch of 3x2x2x2 matrices, 3x2x2x3 right-hand sides.
     self._verifySolveAllWaysComplex(matrix, rhs, batch_dims=[3, 2])
 
+  @test_util.run_deprecated_v1
   def testNonSquareMatrix(self):
     # A non-square matrix should cause an error.
     matrix = np.array([[1., 2., 3.], [3., 4., 5.]])
@@ -149,6 +155,7 @@ class MatrixTriangularSolveOpTest(test.TestCase):
       with self.assertRaises(ValueError):
         self._verifySolve(matrix, matrix, batch_dims=[2, 3])
 
+  @test_util.run_deprecated_v1
   def testWrongDimensions(self):
     # The matrix should have the same number of rows as the
     # right-hand sides.
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index eb5f99582c..64dd591455 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -175,22 +176,26 @@ class MeanTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean(array_ops.ones([4, 3]))
     _assert_metric_variables(self, ('mean/count:0', 'mean/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.mean(
         array_ops.ones([4, 3]), metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean(
         array_ops.ones([4, 3]), updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.cached_session() as sess:
       values_queue = data_flow_ops.FIFOQueue(
@@ -208,6 +213,7 @@ class MeanTest(test.TestCase):
         self.evaluate(update_op)
       self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
 
+  @test_util.run_deprecated_v1
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
       values_queue = data_flow_ops.FIFOQueue(
@@ -229,6 +235,7 @@ class MeanTest(test.TestCase):
 
       self.assertAlmostEqual(1.65, self.evaluate(mean), 5)
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     values = _test_values((3, 2, 4, 1))
     mean_results = (
@@ -271,37 +278,44 @@ class MeanTest(test.TestCase):
       self.assertAlmostEqual(expected, update_op.eval(), places=5)
       self.assertAlmostEqual(expected, mean.eval(), places=5)
 
+  @test_util.run_deprecated_v1
   def test1x1x1Weighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
         weights=np.asarray((5,)).reshape((1, 1, 1)))
 
+  @test_util.run_deprecated_v1
   def test1x1xNWeighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
         weights=np.asarray((5, 7, 11, 3)).reshape((1, 1, 4)))
 
+  @test_util.run_deprecated_v1
   def test1xNx1Weighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
         weights=np.asarray((5, 11)).reshape((1, 2, 1)))
 
+  @test_util.run_deprecated_v1
   def test1xNxNWeighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
         weights=np.asarray((5, 7, 11, 3, 2, 13, 7, 5)).reshape((1, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testNx1x1Weighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
         weights=np.asarray((5, 7, 11)).reshape((3, 1, 1)))
 
+  @test_util.run_deprecated_v1
   def testNx1xNWeighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
         weights=np.asarray((
             5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3)).reshape((3, 1, 4)))
 
+  @test_util.run_deprecated_v1
   def testNxNxNWeighted(self):
     self._test_3d_weighted(
         _test_values((3, 2, 4)),
@@ -309,6 +323,7 @@ class MeanTest(test.TestCase):
             5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3,
             2, 17, 11, 3, 5, 7, 11, 3, 2, 12, 7, 5)).reshape((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidWeights(self):
     values_placeholder = array_ops.placeholder(dtype=dtypes_lib.float32)
     values = _test_values((3, 2, 4, 1))
@@ -341,23 +356,27 @@ class MeanTensorTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_tensor(array_ops.ones([4, 3]))
     _assert_metric_variables(self,
                              ('mean/total_tensor:0', 'mean/count_tensor:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.mean_tensor(
         array_ops.ones([4, 3]), metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_tensor(
         array_ops.ones([4, 3]), updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.cached_session() as sess:
       values_queue = data_flow_ops.FIFOQueue(
@@ -375,6 +394,7 @@ class MeanTensorTest(test.TestCase):
         self.evaluate(update_op)
       self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean))
 
+  @test_util.run_deprecated_v1
   def testMultiDimensional(self):
     with self.cached_session() as sess:
       values_queue = data_flow_ops.FIFOQueue(
@@ -397,6 +417,7 @@ class MeanTensorTest(test.TestCase):
       self.assertAllClose([[[1, 2], [1, 2]], [[2, 3], [5, 6]]],
                           self.evaluate(mean))
 
+  @test_util.run_deprecated_v1
   def testUpdateOpsReturnsCurrentValue(self):
     with self.cached_session() as sess:
       values_queue = data_flow_ops.FIFOQueue(
@@ -418,6 +439,7 @@ class MeanTensorTest(test.TestCase):
 
       self.assertAllClose([[-0.9 / 4., 3.525]], self.evaluate(mean), 5)
 
+  @test_util.run_deprecated_v1
   def testBinaryWeighted1d(self):
     with self.cached_session() as sess:
       # Create the queue that populates the values.
@@ -445,6 +467,7 @@ class MeanTensorTest(test.TestCase):
         self.evaluate(update_op)
       self.assertAllClose([[3.25, 0.5]], self.evaluate(mean), 5)
 
+  @test_util.run_deprecated_v1
   def testWeighted1d(self):
     with self.cached_session() as sess:
       # Create the queue that populates the values.
@@ -472,6 +495,7 @@ class MeanTensorTest(test.TestCase):
         self.evaluate(update_op)
       self.assertAllClose([[0.8, 3.52]], self.evaluate(mean), 5)
 
+  @test_util.run_deprecated_v1
   def testWeighted2d_1(self):
     with self.cached_session() as sess:
       # Create the queue that populates the values.
@@ -499,6 +523,7 @@ class MeanTensorTest(test.TestCase):
         self.evaluate(update_op)
       self.assertAllClose([[-2.1, 0.5]], self.evaluate(mean), 5)
 
+  @test_util.run_deprecated_v1
   def testWeighted2d_2(self):
     with self.cached_session() as sess:
       # Create the queue that populates the values.
@@ -532,6 +557,7 @@ class AccuracyTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.accuracy(
         predictions=array_ops.ones((10, 1)),
@@ -540,6 +566,7 @@ class AccuracyTest(test.TestCase):
     _assert_metric_variables(self,
                              ('my_accuracy/count:0', 'my_accuracy/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.accuracy(
@@ -548,6 +575,7 @@ class AccuracyTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.accuracy(
@@ -556,12 +584,14 @@ class AccuracyTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self):
     predictions = array_ops.ones((10, 3))
     labels = array_ops.ones((10, 4))
     with self.assertRaises(ValueError):
       metrics.accuracy(labels, predictions)
 
+  @test_util.run_deprecated_v1
   def testPredictionsAndWeightsOfDifferentSizeRaisesValueError(self):
     predictions = array_ops.ones((10, 3))
     labels = array_ops.ones((10, 3))
@@ -569,6 +599,7 @@ class AccuracyTest(test.TestCase):
     with self.assertRaises(ValueError):
       metrics.accuracy(labels, predictions, weights)
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=3, dtype=dtypes_lib.int64, seed=1)
@@ -588,6 +619,7 @@ class AccuracyTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_accuracy, accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdates(self):
     with self.cached_session() as sess:
       # Create the queue that populates the predictions.
@@ -616,6 +648,7 @@ class AccuracyTest(test.TestCase):
       self.assertEqual(0.5, self.evaluate(update_op))
       self.assertEqual(0.5, accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testEffectivelyEquivalentSizes(self):
     predictions = array_ops.ones((40, 1))
     labels = array_ops.ones((40,))
@@ -626,6 +659,7 @@ class AccuracyTest(test.TestCase):
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testEffectivelyEquivalentSizesWithScalarWeight(self):
     predictions = array_ops.ones((40, 1))
     labels = array_ops.ones((40,))
@@ -636,6 +670,7 @@ class AccuracyTest(test.TestCase):
       self.assertEqual(1.0, update_op.eval())
       self.assertEqual(1.0, accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testEffectivelyEquivalentSizesWithStaticShapedWeight(self):
     predictions = ops.convert_to_tensor([1, 1, 1])  # shape 3,
     labels = array_ops.expand_dims(ops.convert_to_tensor([1, 0, 0]),
@@ -653,6 +688,7 @@ class AccuracyTest(test.TestCase):
       self.assertGreater(update_op.eval(), .95)
       self.assertGreater(accuracy.eval(), .95)
 
+  @test_util.run_deprecated_v1
   def testEffectivelyEquivalentSizesWithDynamicallyShapedWeight(self):
     predictions = ops.convert_to_tensor([1, 1, 1])  # shape 3,
     labels = array_ops.expand_dims(ops.convert_to_tensor([1, 0, 0]),
@@ -674,6 +710,7 @@ class AccuracyTest(test.TestCase):
       self.assertGreater(update_op.eval(feed_dict=feed_dict), .95)
       self.assertGreater(accuracy.eval(feed_dict=feed_dict), .95)
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdatesWithWeightedValues(self):
     with self.cached_session() as sess:
       # Create the queue that populates the predictions.
@@ -718,12 +755,14 @@ class PrecisionTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.precision(
         predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
     _assert_metric_variables(self, ('precision/false_positives/count:0',
                                     'precision/true_positives/count:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.precision(
@@ -732,6 +771,7 @@ class PrecisionTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.precision(
@@ -740,6 +780,7 @@ class PrecisionTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
@@ -759,6 +800,7 @@ class PrecisionTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_precision, precision.eval())
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -771,6 +813,7 @@ class PrecisionTest(test.TestCase):
       self.assertAlmostEqual(1, self.evaluate(update_op))
       self.assertAlmostEqual(1, precision.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrect_multipleInputDtypes(self):
     for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32):
       predictions = math_ops.cast(
@@ -784,6 +827,7 @@ class PrecisionTest(test.TestCase):
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, precision.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted1d(self):
     predictions = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
     labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
@@ -798,6 +842,7 @@ class PrecisionTest(test.TestCase):
       self.assertAlmostEqual(expected_precision, update_op.eval())
       self.assertAlmostEqual(expected_precision, precision.eval())
 
+  @test_util.run_deprecated_v1
   def testWeightedScalar_placeholders(self):
     predictions = array_ops.placeholder(dtype=dtypes_lib.float32)
     labels = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -817,6 +862,7 @@ class PrecisionTest(test.TestCase):
       self.assertAlmostEqual(
           expected_precision, precision.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testWeighted1d_placeholders(self):
     predictions = array_ops.placeholder(dtype=dtypes_lib.float32)
     labels = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -837,6 +883,7 @@ class PrecisionTest(test.TestCase):
       self.assertAlmostEqual(
           expected_precision, precision.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testWeighted2d(self):
     predictions = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
     labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
@@ -853,6 +900,7 @@ class PrecisionTest(test.TestCase):
       self.assertAlmostEqual(expected_precision, update_op.eval())
       self.assertAlmostEqual(expected_precision, precision.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted2d_placeholders(self):
     predictions = array_ops.placeholder(dtype=dtypes_lib.float32)
     labels = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -875,6 +923,7 @@ class PrecisionTest(test.TestCase):
       self.assertAlmostEqual(
           expected_precision, precision.eval(feed_dict=feed_dict))
 
+  @test_util.run_deprecated_v1
   def testAllIncorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -887,6 +936,7 @@ class PrecisionTest(test.TestCase):
       self.evaluate(update_op)
       self.assertAlmostEqual(0, precision.eval())
 
+  @test_util.run_deprecated_v1
   def testZeroTrueAndFalsePositivesGivesZeroPrecision(self):
     predictions = constant_op.constant([0, 0, 0, 0])
     labels = constant_op.constant([0, 0, 0, 0])
@@ -904,6 +954,7 @@ class RecallTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.recall(
         predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
@@ -911,6 +962,7 @@ class RecallTest(test.TestCase):
         self,
         ('recall/false_negatives/count:0', 'recall/true_positives/count:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.recall(
@@ -919,6 +971,7 @@ class RecallTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.recall(
@@ -927,6 +980,7 @@ class RecallTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
@@ -946,6 +1000,7 @@ class RecallTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_recall, recall.eval())
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     np_inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -958,6 +1013,7 @@ class RecallTest(test.TestCase):
       self.evaluate(update_op)
       self.assertEqual(1, recall.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrect_multipleInputDtypes(self):
     for dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32):
       predictions = math_ops.cast(
@@ -971,6 +1027,7 @@ class RecallTest(test.TestCase):
         self.assertAlmostEqual(0.5, update_op.eval())
         self.assertAlmostEqual(0.5, recall.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted1d(self):
     predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
     labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
@@ -985,6 +1042,7 @@ class RecallTest(test.TestCase):
       self.assertAlmostEqual(expected_precision, update_op.eval())
       self.assertAlmostEqual(expected_precision, recall.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted2d(self):
     predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
     labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
@@ -999,6 +1057,7 @@ class RecallTest(test.TestCase):
       self.assertAlmostEqual(expected_precision, update_op.eval())
       self.assertAlmostEqual(expected_precision, recall.eval())
 
+  @test_util.run_deprecated_v1
   def testAllIncorrect(self):
     np_inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -1011,6 +1070,7 @@ class RecallTest(test.TestCase):
       self.evaluate(update_op)
       self.assertEqual(0, recall.eval())
 
+  @test_util.run_deprecated_v1
   def testZeroTruePositivesAndFalseNegativesGivesZeroRecall(self):
     predictions = array_ops.zeros((1, 4))
     labels = array_ops.zeros((1, 4))
@@ -1028,6 +1088,7 @@ class AUCTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.auc(predictions=array_ops.ones((10, 1)),
                 labels=array_ops.ones((10, 1)))
@@ -1035,6 +1096,7 @@ class AUCTest(test.TestCase):
                              ('auc/true_positives:0', 'auc/false_negatives:0',
                               'auc/false_positives:0', 'auc/true_negatives:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.auc(predictions=array_ops.ones((10, 1)),
@@ -1042,6 +1104,7 @@ class AUCTest(test.TestCase):
                           metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.auc(predictions=array_ops.ones((10, 1)),
@@ -1049,6 +1112,7 @@ class AUCTest(test.TestCase):
                                updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
@@ -1068,6 +1132,7 @@ class AUCTest(test.TestCase):
       for _ in range(10):
         self.assertAlmostEqual(initial_auc, auc.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     self.allCorrectAsExpected('ROC')
 
@@ -1084,6 +1149,7 @@ class AUCTest(test.TestCase):
 
       self.assertEqual(1, auc.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrect_multipleLabelDtypes(self):
     with self.cached_session() as sess:
       for label_dtype in (
@@ -1099,6 +1165,7 @@ class AUCTest(test.TestCase):
 
         self.assertAlmostEqual(0.5, auc.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted1d(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1112,6 +1179,7 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(0.5, auc.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testWeighted2d(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1127,6 +1195,7 @@ class AUCTest(test.TestCase):
 
   # Regarding the AUC-PR tests: note that the preferred method when
   # calculating AUC-PR is summation_method='careful_interpolation'.
+  @test_util.run_deprecated_v1
   def testCorrectAUCPRSpecialCase(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1141,6 +1210,7 @@ class AUCTest(test.TestCase):
       self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
+  @test_util.run_deprecated_v1
   def testCorrectAnotherAUCPRSpecialCase(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1157,6 +1227,7 @@ class AUCTest(test.TestCase):
       self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
+  @test_util.run_deprecated_v1
   def testThirdCorrectAUCPRSpecialCase(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1173,6 +1244,7 @@ class AUCTest(test.TestCase):
       self.assertAlmostEqual(expected, self.evaluate(update_op), delta=1e-3)
       self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
 
+  @test_util.run_deprecated_v1
   def testIncorrectAUCPRSpecialCase(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1186,6 +1258,7 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3)
 
+  @test_util.run_deprecated_v1
   def testAnotherIncorrectAUCPRSpecialCase(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1201,6 +1274,7 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3)
 
+  @test_util.run_deprecated_v1
   def testThirdIncorrectAUCPRSpecialCase(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1216,6 +1290,7 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(0.90277, auc.eval(), delta=1e-3)
 
+  @test_util.run_deprecated_v1
   def testAllIncorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -1229,6 +1304,7 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(0, auc.eval())
 
+  @test_util.run_deprecated_v1
   def testZeroTruePositivesAndFalseNegativesGivesOneAUC(self):
     with self.cached_session() as sess:
       predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
@@ -1240,6 +1316,7 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(1, auc.eval(), 6)
 
+  @test_util.run_deprecated_v1
   def testRecallOneAndPrecisionOneGivesOnePRAUC(self):
     with self.cached_session() as sess:
       predictions = array_ops.ones([4], dtype=dtypes_lib.float32)
@@ -1278,6 +1355,7 @@ class AUCTest(test.TestCase):
     tp = np.cumsum(sorted_weights * is_positive) / num_positives
     return np.sum((sorted_weights * tp)[~is_positive]) / num_negatives
 
+  @test_util.run_deprecated_v1
   def testWithMultipleUpdates(self):
     num_samples = 1000
     batch_size = 10
@@ -1334,6 +1412,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.specificity_at_sensitivity(
         predictions=array_ops.ones((10, 1)),
@@ -1345,6 +1424,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
                               'specificity_at_sensitivity/false_positives:0',
                               'specificity_at_sensitivity/true_negatives:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.specificity_at_sensitivity(
@@ -1354,6 +1434,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.specificity_at_sensitivity(
@@ -1363,6 +1444,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
@@ -1383,6 +1465,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
       for _ in range(10):
         self.assertAlmostEqual(initial_specificity, specificity.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -1396,6 +1479,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
       self.assertEqual(1, self.evaluate(update_op))
       self.assertEqual(1, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrectHighSensitivity(self):
     predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.45, 0.5, 0.8, 0.9]
     labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
@@ -1411,6 +1495,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
       self.assertAlmostEqual(1.0, self.evaluate(update_op))
       self.assertAlmostEqual(1.0, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrectLowSensitivity(self):
     predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26]
     labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
@@ -1427,6 +1512,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
       self.assertAlmostEqual(0.6, self.evaluate(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted1d_multipleLabelDtypes(self):
     for label_dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32):
       predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26]
@@ -1446,6 +1532,7 @@ class SpecificityAtSensitivityTest(test.TestCase):
         self.assertAlmostEqual(0.6, self.evaluate(update_op))
         self.assertAlmostEqual(0.6, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted2d(self):
     predictions_values = [0.1, 0.2, 0.4, 0.3, 0.0, 0.1, 0.2, 0.2, 0.26, 0.26]
     labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
@@ -1471,6 +1558,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.sensitivity_at_specificity(
         predictions=array_ops.ones((10, 1)),
@@ -1482,6 +1570,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
                               'sensitivity_at_specificity/false_positives:0',
                               'sensitivity_at_specificity/true_negatives:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.sensitivity_at_specificity(
@@ -1491,6 +1580,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.sensitivity_at_specificity(
@@ -1500,6 +1590,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
@@ -1520,6 +1611,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
       for _ in range(10):
         self.assertAlmostEqual(initial_sensitivity, sensitivity.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -1533,6 +1625,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
       self.assertEqual(1, self.evaluate(update_op))
       self.assertEqual(1, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrectHighSpecificity(self):
     predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
     labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
@@ -1548,6 +1641,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
       self.assertAlmostEqual(0.8, self.evaluate(update_op))
       self.assertAlmostEqual(0.8, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrectLowSpecificity(self):
     predictions_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
     labels_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
@@ -1563,6 +1657,7 @@ class SensitivityAtSpecificityTest(test.TestCase):
       self.assertAlmostEqual(0.6, self.evaluate(update_op))
       self.assertAlmostEqual(0.6, specificity.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted_multipleLabelDtypes(self):
     for label_dtype in (dtypes_lib.bool, dtypes_lib.int32, dtypes_lib.float32):
       predictions_values = [
@@ -1590,6 +1685,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.precision_at_thresholds(
         predictions=array_ops.ones((10, 1)),
@@ -1600,6 +1696,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         'precision_at_thresholds/false_positives:0',
     ))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     prec, _ = metrics.precision_at_thresholds(
@@ -1614,6 +1711,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [prec, rec])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, precision_op = metrics.precision_at_thresholds(
@@ -1629,6 +1727,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
     self.assertListEqual(
         ops.get_collection(my_collection_name), [precision_op, recall_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
@@ -1652,6 +1751,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         self.assertAllClose(initial_rec, rec.eval())
 
   # TODO(nsilberman): fix tests (passing but incorrect).
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -1670,6 +1770,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       self.assertEqual(1, prec.eval())
       self.assertEqual(1, rec.eval())
 
+  @test_util.run_deprecated_v1
   def testSomeCorrect_multipleLabelDtypes(self):
     with self.cached_session() as sess:
       for label_dtype in (
@@ -1690,6 +1791,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
         self.assertAlmostEqual(0.5, prec.eval())
         self.assertAlmostEqual(0.5, rec.eval())
 
+  @test_util.run_deprecated_v1
   def testAllIncorrect(self):
     inputs = np.random.randint(0, 2, size=(100, 1))
 
@@ -1708,6 +1810,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(0, prec.eval())
       self.assertAlmostEqual(0, rec.eval())
 
+  @test_util.run_deprecated_v1
   def testWeights1d(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1738,6 +1841,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(1.0, rec_low.eval(), places=5)
       self.assertAlmostEqual(0.0, rec_high.eval(), places=5)
 
+  @test_util.run_deprecated_v1
   def testWeights2d(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1768,6 +1872,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(1.0, rec_low.eval(), places=5)
       self.assertAlmostEqual(0.0, rec_high.eval(), places=5)
 
+  @test_util.run_deprecated_v1
   def testExtremeThresholds(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -1792,6 +1897,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(1.0, rec_low.eval())
       self.assertAlmostEqual(0.0, rec_high.eval())
 
+  @test_util.run_deprecated_v1
   def testZeroLabelsPredictions(self):
     with self.cached_session() as sess:
       predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
@@ -1808,6 +1914,7 @@ class PrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(0, prec.eval(), 6)
       self.assertAlmostEqual(0, rec.eval(), 6)
 
+  @test_util.run_deprecated_v1
   def testWithMultipleUpdates(self):
     num_samples = 1000
     batch_size = 10
@@ -1990,6 +2097,7 @@ class SingleLabelPrecisionAtKTest(test.TestCase):
     self._test_average_precision_at_k = functools.partial(
         _test_average_precision_at_k, test_case=self)
 
+  @test_util.run_deprecated_v1
   def test_at_k1_nan(self):
     for labels in self._labels:
       # Classes 0,1,2 have 0 predictions, classes -1 and 4 are out of range.
@@ -1999,6 +2107,7 @@ class SingleLabelPrecisionAtKTest(test.TestCase):
         self._test_precision_at_top_k(
             self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_at_k1(self):
     for labels in self._labels:
       # Class 3: 1 label, 2 predictions, 1 correct.
@@ -2026,6 +2135,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
     self._test_average_precision_at_k = functools.partial(
         _test_average_precision_at_k, test_case=self)
 
+  @test_util.run_deprecated_v1
   def test_average_precision(self):
     # Example 1.
     # Matches example here:
@@ -2101,6 +2211,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
           expected=streaming_average_precision[i],
           weights=weights)
 
+  @test_util.run_deprecated_v1
   def test_average_precision_some_labels_out_of_range(self):
     """Tests that labels outside the [0, n_classes) range are ignored."""
     labels_ex1 = (-1, 0, 1, 2, 3, 4, 7)
@@ -2120,6 +2231,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
       self._test_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex1[i])
 
+  @test_util.run_deprecated_v1
   def test_three_labels_at_k5_no_predictions(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
@@ -2136,6 +2248,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
         self._test_precision_at_top_k(
             predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_three_labels_at_k5_no_labels(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
@@ -2152,6 +2265,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
         self._test_precision_at_top_k(
             predictions_idx, labels, k=5, expected=0.0, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_three_labels_at_k5(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
@@ -2185,6 +2299,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
       self._test_precision_at_top_k(
           predictions_idx, labels, k=5, expected=3.0 / 10)
 
+  @test_util.run_deprecated_v1
   def test_three_labels_at_k5_some_out_of_range(self):
     """Tests that labels outside the [0, n_classes) range are ignored."""
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
@@ -2221,6 +2336,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
     self._test_precision_at_top_k(
         predictions_idx, sp_labels, k=5, expected=3.0 / 10)
 
+  @test_util.run_deprecated_v1
   def test_3d_nan(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
@@ -2239,6 +2355,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
       self._test_precision_at_top_k(
           predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_3d_no_labels(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
@@ -2257,6 +2374,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
       self._test_precision_at_top_k(
           predictions_idx, labels, k=5, expected=0.0, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_3d(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
@@ -2292,6 +2410,7 @@ class MultiLabelPrecisionAtKTest(test.TestCase):
     self._test_precision_at_top_k(
         predictions_idx, labels, k=5, expected=7.0 / 20)
 
+  @test_util.run_deprecated_v1
   def test_3d_ignore_some(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
@@ -2433,6 +2552,7 @@ class SingleLabelRecallAtKTest(test.TestCase):
     self._test_recall_at_top_k = functools.partial(
         _test_recall_at_top_k, test_case=self)
 
+  @test_util.run_deprecated_v1
   def test_at_k1_nan(self):
     # Classes 0,1 have 0 labels, 0 predictions, classes -1 and 4 are out of
     # range.
@@ -2443,6 +2563,7 @@ class SingleLabelRecallAtKTest(test.TestCase):
         self._test_recall_at_top_k(
             self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_at_k1_no_predictions(self):
     for labels in self._labels:
       # Class 2: 0 predictions.
@@ -2451,6 +2572,7 @@ class SingleLabelRecallAtKTest(test.TestCase):
       self._test_recall_at_top_k(
           self._predictions_idx, labels, k=1, expected=0.0, class_id=2)
 
+  @test_util.run_deprecated_v1
   def test_one_label_at_k1(self):
     for labels in self._labels:
       # Class 3: 1 label, 2 predictions, 1 correct.
@@ -2464,6 +2586,7 @@ class SingleLabelRecallAtKTest(test.TestCase):
       self._test_recall_at_top_k(
           self._predictions_idx, labels, k=1, expected=1.0 / 2)
 
+  @test_util.run_deprecated_v1
   def test_one_label_at_k1_weighted_class_id3(self):
     predictions = self._predictions
     predictions_idx = self._predictions_idx
@@ -2505,6 +2628,7 @@ class SingleLabelRecallAtKTest(test.TestCase):
           predictions_idx, labels, k=1, expected=2.0 / 2, class_id=3,
           weights=(2.0, 3.0))
 
+  @test_util.run_deprecated_v1
   def test_one_label_at_k1_weighted(self):
     predictions = self._predictions
     predictions_idx = self._predictions_idx
@@ -2554,6 +2678,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase):
     self._test_recall_at_top_k = functools.partial(
         _test_recall_at_top_k, test_case=self)
 
+  @test_util.run_deprecated_v1
   def test_at_k5_nan(self):
     for labels in self._labels:
       # Classes 0,3,4,6,9 have 0 labels, class 10 is out of range.
@@ -2563,6 +2688,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase):
         self._test_recall_at_top_k(
             self._predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_at_k5_no_predictions(self):
     for labels in self._labels:
       # Class 8: 1 label, no predictions.
@@ -2571,6 +2697,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase):
       self._test_recall_at_top_k(
           self._predictions_idx, labels, k=5, expected=0.0 / 1, class_id=8)
 
+  @test_util.run_deprecated_v1
   def test_at_k5(self):
     for labels in self._labels:
       # Class 2: 2 labels, both correct.
@@ -2596,6 +2723,7 @@ class MultiLabel2dRecallAtKTest(test.TestCase):
       self._test_recall_at_top_k(
           self._predictions_idx, labels, k=5, expected=3.0 / 6)
 
+  @test_util.run_deprecated_v1
   def test_at_k5_some_out_of_range(self):
     """Tests that labels outside the [0, n_classes) count in denominator."""
     labels = sparse_tensor.SparseTensorValue(
@@ -2648,6 +2776,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase):
     self._test_recall_at_top_k = functools.partial(
         _test_recall_at_top_k, test_case=self)
 
+  @test_util.run_deprecated_v1
   def test_3d_nan(self):
     # Classes 0,3,4,6,9 have 0 labels, class 10 is out of range.
     for class_id in (0, 3, 4, 6, 9, 10):
@@ -2657,6 +2786,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase):
           self._predictions_idx, self._labels, k=5, expected=NAN,
           class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_3d_no_predictions(self):
     # Classes 1,8 have 0 predictions, >=1 label.
     for class_id in (1, 8):
@@ -2666,6 +2796,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase):
           self._predictions_idx, self._labels, k=5, expected=0.0,
           class_id=class_id)
 
+  @test_util.run_deprecated_v1
   def test_3d(self):
     # Class 2: 4 labels, all correct.
     self._test_recall_at_k(
@@ -2694,6 +2825,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase):
     self._test_recall_at_top_k(
         self._predictions_idx, self._labels, k=5, expected=7.0 / 12)
 
+  @test_util.run_deprecated_v1
   def test_3d_ignore_all(self):
     for class_id in xrange(10):
       self._test_recall_at_k(
@@ -2720,6 +2852,7 @@ class MultiLabel3dRecallAtKTest(test.TestCase):
         self._predictions_idx, self._labels, k=5, expected=NAN,
         weights=[[0, 0], [0, 0]])
 
+  @test_util.run_deprecated_v1
   def test_3d_ignore_some(self):
     # Class 2: 2 labels, both correct.
     self._test_recall_at_k(
@@ -2775,12 +2908,14 @@ class MeanAbsoluteErrorTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_absolute_error(
         predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
     _assert_metric_variables(
         self, ('mean_absolute_error/count:0', 'mean_absolute_error/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.mean_absolute_error(
@@ -2789,6 +2924,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_absolute_error(
@@ -2797,6 +2933,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_normal((10, 3), seed=1)
     labels = random_ops.random_normal((10, 3), seed=2)
@@ -2814,6 +2951,7 @@ class MeanAbsoluteErrorTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_error, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithErrorAndWeights(self):
     predictions = constant_op.constant(
         [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32)
@@ -2834,6 +2972,7 @@ class MeanRelativeErrorTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_relative_error(
         predictions=array_ops.ones((10, 1)),
@@ -2842,6 +2981,7 @@ class MeanRelativeErrorTest(test.TestCase):
     _assert_metric_variables(
         self, ('mean_relative_error/count:0', 'mean_relative_error/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.mean_relative_error(
@@ -2851,6 +2991,7 @@ class MeanRelativeErrorTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_relative_error(
@@ -2860,6 +3001,7 @@ class MeanRelativeErrorTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_normal((10, 3), seed=1)
     labels = random_ops.random_normal((10, 3), seed=2)
@@ -2879,6 +3021,7 @@ class MeanRelativeErrorTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_error, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateNormalizedByLabels(self):
     np_predictions = np.asarray([2, 4, 6, 8], dtype=np.float32)
     np_labels = np.asarray([1, 3, 2, 3], dtype=np.float32)
@@ -2897,6 +3040,7 @@ class MeanRelativeErrorTest(test.TestCase):
       self.assertEqual(expected_error, self.evaluate(update_op))
       self.assertEqual(expected_error, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateNormalizedByZeros(self):
     np_predictions = np.asarray([2, 4, 6, 8], dtype=np.float32)
 
@@ -2919,12 +3063,14 @@ class MeanSquaredErrorTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_squared_error(
         predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
     _assert_metric_variables(
         self, ('mean_squared_error/count:0', 'mean_squared_error/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.mean_squared_error(
@@ -2933,6 +3079,7 @@ class MeanSquaredErrorTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_squared_error(
@@ -2941,6 +3088,7 @@ class MeanSquaredErrorTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_normal((10, 3), seed=1)
     labels = random_ops.random_normal((10, 3), seed=2)
@@ -2958,6 +3106,7 @@ class MeanSquaredErrorTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_error, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateZeroError(self):
     predictions = array_ops.zeros((1, 3), dtype=dtypes_lib.float32)
     labels = array_ops.zeros((1, 3), dtype=dtypes_lib.float32)
@@ -2969,6 +3118,7 @@ class MeanSquaredErrorTest(test.TestCase):
       self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithError(self):
     predictions = constant_op.constant(
         [2, 4, 6], shape=(1, 3), dtype=dtypes_lib.float32)
@@ -2982,6 +3132,7 @@ class MeanSquaredErrorTest(test.TestCase):
       self.assertEqual(6, self.evaluate(update_op))
       self.assertEqual(6, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithErrorAndWeights(self):
     predictions = constant_op.constant(
         [2, 4, 6, 8], shape=(1, 4), dtype=dtypes_lib.float32)
@@ -2996,6 +3147,7 @@ class MeanSquaredErrorTest(test.TestCase):
       self.assertEqual(13, self.evaluate(update_op))
       self.assertEqual(13, error.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleBatchesOfSizeOne(self):
     with self.cached_session() as sess:
       # Create the queue that populates the predictions.
@@ -3020,6 +3172,7 @@ class MeanSquaredErrorTest(test.TestCase):
 
       self.assertAlmostEqual(208.0 / 6, error.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testMetricsComputedConcurrently(self):
     with self.cached_session() as sess:
       # Create the queue that populates one set of predictions.
@@ -3063,6 +3216,7 @@ class MeanSquaredErrorTest(test.TestCase):
       self.assertAlmostEqual(208.0 / 6, mse0, 5)
       self.assertAlmostEqual(79.0 / 6, mse1, 5)
 
+  @test_util.run_deprecated_v1
   def testMultipleMetricsOnMultipleBatchesOfSizeOne(self):
     with self.cached_session() as sess:
       # Create the queue that populates the predictions.
@@ -3095,6 +3249,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.root_mean_squared_error(
         predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
@@ -3102,6 +3257,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
         self,
         ('root_mean_squared_error/count:0', 'root_mean_squared_error/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.root_mean_squared_error(
@@ -3110,6 +3266,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.root_mean_squared_error(
@@ -3118,6 +3275,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_normal((10, 3), seed=1)
     labels = random_ops.random_normal((10, 3), seed=2)
@@ -3135,6 +3293,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_error, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateZeroError(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -3148,6 +3307,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
 
       self.assertEqual(0, rmse.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithError(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -3161,6 +3321,7 @@ class RootMeanSquaredErrorTest(test.TestCase):
       self.assertAlmostEqual(math.sqrt(6), update_op.eval(), 5)
       self.assertAlmostEqual(math.sqrt(6), rmse.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithErrorAndWeights(self):
     with self.cached_session() as sess:
       predictions = constant_op.constant(
@@ -3188,6 +3349,7 @@ class MeanCosineDistanceTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_cosine_distance(
         predictions=array_ops.ones((10, 3)),
@@ -3198,6 +3360,7 @@ class MeanCosineDistanceTest(test.TestCase):
         'mean_cosine_distance/total:0',
     ))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.mean_cosine_distance(
@@ -3207,6 +3370,7 @@ class MeanCosineDistanceTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_cosine_distance(
@@ -3216,6 +3380,7 @@ class MeanCosineDistanceTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     predictions = random_ops.random_normal((10, 3), seed=1)
     labels = random_ops.random_normal((10, 3), seed=2)
@@ -3233,6 +3398,7 @@ class MeanCosineDistanceTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_error, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateZeroError(self):
     np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0'))
 
@@ -3248,6 +3414,7 @@ class MeanCosineDistanceTest(test.TestCase):
       self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithError1(self):
     np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0'))
     np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0'))
@@ -3264,6 +3431,7 @@ class MeanCosineDistanceTest(test.TestCase):
       self.assertAlmostEqual(1, self.evaluate(update_op), 5)
       self.assertAlmostEqual(1, error.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithError2(self):
     np_predictions = np.matrix(
         ('0.819031913261206 0.567041924552012 0.087465312324590;'
@@ -3285,6 +3453,7 @@ class MeanCosineDistanceTest(test.TestCase):
       self.assertAlmostEqual(1.0, self.evaluate(update_op), 5)
       self.assertAlmostEqual(1.0, error.eval(), 5)
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithErrorAndWeights1(self):
     np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0'))
     np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0'))
@@ -3304,6 +3473,7 @@ class MeanCosineDistanceTest(test.TestCase):
       self.assertEqual(0, self.evaluate(update_op))
       self.assertEqual(0, error.eval())
 
+  @test_util.run_deprecated_v1
   def testSingleUpdateWithErrorAndWeights2(self):
     np_predictions = np.matrix(('1 0 0;' '0 0 -1;' '1 0 0'))
     np_labels = np.matrix(('1 0 0;' '0 0 1;' '0 1 0'))
@@ -3329,6 +3499,7 @@ class PcntBelowThreshTest(test.TestCase):
   def setUp(self):
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.percentage_below(values=array_ops.ones((10,)), threshold=2)
     _assert_metric_variables(self, (
@@ -3336,6 +3507,7 @@ class PcntBelowThreshTest(test.TestCase):
         'percentage_below_threshold/total:0',
     ))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollection(self):
     my_collection_name = '__metrics__'
     mean, _ = metrics.percentage_below(
@@ -3344,6 +3516,7 @@ class PcntBelowThreshTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.percentage_below(
@@ -3352,6 +3525,7 @@ class PcntBelowThreshTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testOneUpdate(self):
     with self.cached_session() as sess:
       values = constant_op.constant(
@@ -3369,6 +3543,7 @@ class PcntBelowThreshTest(test.TestCase):
       self.assertAlmostEqual(0.75, pcnt1, 5)
       self.assertAlmostEqual(0.0, pcnt2, 5)
 
+  @test_util.run_deprecated_v1
   def testSomePresentOneUpdate(self):
     with self.cached_session() as sess:
       values = constant_op.constant(
@@ -3399,6 +3574,7 @@ class MeanIOUTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_iou(
         predictions=array_ops.ones([10, 1]),
@@ -3406,6 +3582,7 @@ class MeanIOUTest(test.TestCase):
         num_classes=2)
     _assert_metric_variables(self, ('mean_iou/total_confusion_matrix:0',))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollections(self):
     my_collection_name = '__metrics__'
     mean_iou, _ = metrics.mean_iou(
@@ -3415,6 +3592,7 @@ class MeanIOUTest(test.TestCase):
         metrics_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [mean_iou])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_iou(
@@ -3424,12 +3602,14 @@ class MeanIOUTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self):
     predictions = array_ops.ones([10, 3])
     labels = array_ops.ones([10, 4])
     with self.assertRaises(ValueError):
       metrics.mean_iou(labels, predictions, num_classes=2)
 
+  @test_util.run_deprecated_v1
   def testLabelsAndWeightsOfDifferentSizeRaisesValueError(self):
     predictions = array_ops.ones([10])
     labels = array_ops.ones([10])
@@ -3437,6 +3617,7 @@ class MeanIOUTest(test.TestCase):
     with self.assertRaises(ValueError):
       metrics.mean_iou(labels, predictions, num_classes=2, weights=weights)
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     num_classes = 3
     predictions = random_ops.random_uniform(
@@ -3458,6 +3639,7 @@ class MeanIOUTest(test.TestCase):
       for _ in range(10):
         self.assertEqual(initial_mean_iou, mean_iou.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdates(self):
     num_classes = 3
     with self.cached_session() as sess:
@@ -3489,6 +3671,7 @@ class MeanIOUTest(test.TestCase):
       desired_output = np.mean([1.0 / 2.0, 1.0 / 4.0, 0.])
       self.assertEqual(desired_output, miou.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdatesWithWeights(self):
     num_classes = 2
     with self.cached_session() as sess:
@@ -3534,6 +3717,7 @@ class MeanIOUTest(test.TestCase):
       desired_output = np.mean([2.0 / 3.0, 1.0 / 2.0])
       self.assertAlmostEqual(desired_output, mean_iou.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdatesWithMissingClass(self):
     # Test the case where there are no predicions and labels for
     # one class, and thus there is one row and one column with
@@ -3570,6 +3754,7 @@ class MeanIOUTest(test.TestCase):
       desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
+  @test_util.run_deprecated_v1
   def testUpdateOpEvalIsAccumulatedConfusionMatrix(self):
     predictions = array_ops.concat(
         [
@@ -3594,6 +3779,7 @@ class MeanIOUTest(test.TestCase):
       desired_miou = np.mean([3. / 5., 5. / 7.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     predictions = array_ops.zeros([40])
     labels = array_ops.zeros([40])
@@ -3604,6 +3790,7 @@ class MeanIOUTest(test.TestCase):
       self.assertEqual(40, update_op.eval()[0])
       self.assertEqual(1.0, miou.eval())
 
+  @test_util.run_deprecated_v1
   def testAllWrong(self):
     predictions = array_ops.zeros([40])
     labels = array_ops.ones([40])
@@ -3614,6 +3801,7 @@ class MeanIOUTest(test.TestCase):
       self.assertAllEqual([[0, 0], [40, 0]], update_op.eval())
       self.assertEqual(0., miou.eval())
 
+  @test_util.run_deprecated_v1
   def testResultsWithSomeMissing(self):
     predictions = array_ops.concat(
         [
@@ -3646,6 +3834,7 @@ class MeanIOUTest(test.TestCase):
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  @test_util.run_deprecated_v1
   def testMissingClassInLabels(self):
     labels = constant_op.constant([
         [[0, 0, 1, 1, 0, 0],
@@ -3666,6 +3855,7 @@ class MeanIOUTest(test.TestCase):
           1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
           miou.eval())
 
+  @test_util.run_deprecated_v1
   def testMissingClassOverallSmall(self):
     labels = constant_op.constant([0])
     predictions = constant_op.constant([0])
@@ -3676,6 +3866,7 @@ class MeanIOUTest(test.TestCase):
       self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
       self.assertAlmostEqual(1, miou.eval())
 
+  @test_util.run_deprecated_v1
   def testMissingClassOverallLarge(self):
     labels = constant_op.constant([
         [[0, 0, 1, 1, 0, 0],
@@ -3702,6 +3893,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.mean_per_class_accuracy(
         predictions=array_ops.ones([10, 1]),
@@ -3710,6 +3902,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     _assert_metric_variables(self, ('mean_accuracy/count:0',
                                     'mean_accuracy/total:0'))
 
+  @test_util.run_deprecated_v1
   def testMetricsCollections(self):
     my_collection_name = '__metrics__'
     mean_accuracy, _ = metrics.mean_per_class_accuracy(
@@ -3720,6 +3913,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
     self.assertListEqual(
         ops.get_collection(my_collection_name), [mean_accuracy])
 
+  @test_util.run_deprecated_v1
   def testUpdatesCollection(self):
     my_collection_name = '__updates__'
     _, update_op = metrics.mean_per_class_accuracy(
@@ -3729,12 +3923,14 @@ class MeanPerClassAccuracyTest(test.TestCase):
         updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  @test_util.run_deprecated_v1
   def testPredictionsAndLabelsOfDifferentSizeRaisesValueError(self):
     predictions = array_ops.ones([10, 3])
     labels = array_ops.ones([10, 4])
     with self.assertRaises(ValueError):
       metrics.mean_per_class_accuracy(labels, predictions, num_classes=2)
 
+  @test_util.run_deprecated_v1
   def testLabelsAndWeightsOfDifferentSizeRaisesValueError(self):
     predictions = array_ops.ones([10])
     labels = array_ops.ones([10])
@@ -3743,6 +3939,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
       metrics.mean_per_class_accuracy(
           labels, predictions, num_classes=2, weights=weights)
 
+  @test_util.run_deprecated_v1
   def testValueTensorIsIdempotent(self):
     num_classes = 3
     predictions = random_ops.random_uniform(
@@ -3795,6 +3992,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
       desired_output = np.mean([1.0, 1.0 / 3.0, 0.0])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdatesWithWeights(self):
     num_classes = 2
     with self.cached_session() as sess:
@@ -3840,6 +4038,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
       desired_output = np.mean([2.0 / 2.0, 0.5 / 1.5])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testMultipleUpdatesWithMissingClass(self):
     # Test the case where there are no predicions and labels for
     # one class, and thus there is one row and one column with
@@ -3877,6 +4076,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
       desired_output = np.mean([1.0 / 2.0, 2.0 / 3.0, 0.])
       self.assertAlmostEqual(desired_output, mean_accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testAllCorrect(self):
     predictions = array_ops.zeros([40])
     labels = array_ops.zeros([40])
@@ -3888,6 +4088,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
       self.assertEqual(1.0, update_op.eval()[0])
       self.assertEqual(1.0, mean_accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testAllWrong(self):
     predictions = array_ops.zeros([40])
     labels = array_ops.ones([40])
@@ -3899,6 +4100,7 @@ class MeanPerClassAccuracyTest(test.TestCase):
       self.assertAllEqual([0.0, 0.0], update_op.eval())
       self.assertEqual(0., mean_accuracy.eval())
 
+  @test_util.run_deprecated_v1
   def testResultsWithSomeMissing(self):
     predictions = array_ops.concat([
         constant_op.constant(0, shape=[5]), constant_op.constant(1, shape=[5])
@@ -3927,12 +4129,14 @@ class FalseNegativesTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.false_negatives(
         labels=(0, 1, 0, 1),
         predictions=(0, 0, 1, 1))
     _assert_metric_variables(self, ('false_negatives/count:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -3951,6 +4155,7 @@ class FalseNegativesTest(test.TestCase):
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -3977,6 +4182,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.false_negatives_at_thresholds(
         predictions=array_ops.ones((10, 1)),
@@ -3984,6 +4190,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
     _assert_metric_variables(self, ('false_negatives/false_negatives:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4000,6 +4207,7 @@ class FalseNegativesAtThresholdsTest(test.TestCase):
       self.assertAllEqual((0, 2, 3), fn_update_op.eval())
       self.assertAllEqual((0, 2, 3), fn.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4026,12 +4234,14 @@ class FalsePositivesTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.false_positives(
         labels=(0, 1, 0, 1),
         predictions=(0, 0, 1, 1))
     _assert_metric_variables(self, ('false_positives/count:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -4050,6 +4260,7 @@ class FalsePositivesTest(test.TestCase):
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -4076,6 +4287,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.false_positives_at_thresholds(
         predictions=array_ops.ones((10, 1)),
@@ -4083,6 +4295,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
     _assert_metric_variables(self, ('false_positives/false_positives:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4099,6 +4312,7 @@ class FalsePositivesAtThresholdsTest(test.TestCase):
       self.assertAllEqual((7, 4, 2), fp_update_op.eval())
       self.assertAllEqual((7, 4, 2), fp.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4127,12 +4341,14 @@ class TrueNegativesTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.true_negatives(
         labels=(0, 1, 0, 1),
         predictions=(0, 0, 1, 1))
     _assert_metric_variables(self, ('true_negatives/count:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -4151,6 +4367,7 @@ class TrueNegativesTest(test.TestCase):
       self.assertAllClose(3., tn_update_op.eval())
       self.assertAllClose(3., tn.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -4177,6 +4394,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.true_negatives_at_thresholds(
         predictions=array_ops.ones((10, 1)),
@@ -4184,6 +4402,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
     _assert_metric_variables(self, ('true_negatives/true_negatives:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4200,6 +4419,7 @@ class TrueNegativesAtThresholdsTest(test.TestCase):
       self.assertAllEqual((2, 5, 7), tn_update_op.eval())
       self.assertAllEqual((2, 5, 7), tn.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4226,12 +4446,14 @@ class TruePositivesTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.true_positives(
         labels=(0, 1, 0, 1),
         predictions=(0, 0, 1, 1))
     _assert_metric_variables(self, ('true_positives/count:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -4250,6 +4472,7 @@ class TruePositivesTest(test.TestCase):
       self.assertAllClose(7., tn_update_op.eval())
       self.assertAllClose(7., tn.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     labels = constant_op.constant(((0, 1, 0, 1, 0),
                                    (0, 0, 1, 1, 1),
@@ -4276,6 +4499,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
     np.random.seed(1)
     ops.reset_default_graph()
 
+  @test_util.run_deprecated_v1
   def testVars(self):
     metrics.true_positives_at_thresholds(
         predictions=array_ops.ones((10, 1)),
@@ -4283,6 +4507,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
         thresholds=[0.15, 0.5, 0.85])
     _assert_metric_variables(self, ('true_positives/true_positives:0',))
 
+  @test_util.run_deprecated_v1
   def testUnweighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
@@ -4299,6 +4524,7 @@ class TruePositivesAtThresholdsTest(test.TestCase):
       self.assertAllEqual((3, 1, 0), tp_update_op.eval())
       self.assertAllEqual((3, 1, 0), tp.eval())
 
+  @test_util.run_deprecated_v1
   def testWeighted(self):
     predictions = constant_op.constant(((0.9, 0.2, 0.8, 0.1),
                                         (0.2, 0.9, 0.7, 0.6),
diff --git a/tensorflow/python/kernel_tests/morphological_ops_test.py b/tensorflow/python/kernel_tests/morphological_ops_test.py
index 4ee04209cc..f54aaf30d0 100644
--- a/tensorflow/python/kernel_tests/morphological_ops_test.py
+++ b/tensorflow/python/kernel_tests/morphological_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -291,6 +292,7 @@ class DilationTest(test.TestCase):
         padding="SAME",
         use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testDilationGrad(self):
     for use_gpu in True, False:
       self._testDilationGradValidPadding_1x1x1(use_gpu)
@@ -566,6 +568,7 @@ class ErosionTest(test.TestCase):
         padding="SAME",
         use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testErosionGrad(self):
     for use_gpu in True, False:
       self._testErosionGradValidPadding_1x1x1(use_gpu)
diff --git a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
index 87f1991aa7..380d2860da 100644
--- a/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn_impl
 from tensorflow.python.ops import nn_ops
@@ -153,6 +154,7 @@ class DepthwiseConv2DTest(test.TestCase):
     self.assertShapeEqual(native_result, conv_native)
     self.assertShapeEqual(native_result, conv_interface)
 
+  @test_util.run_deprecated_v1
   def testDepthwiseConv2D(self):
     for index, (input_size, filter_size, _, stride,
                 padding) in enumerate(ConfigsToTest()):
@@ -216,6 +218,7 @@ class DepthwiseConv2DTest(test.TestCase):
     self.assertAllClose(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
 
+  @test_util.run_deprecated_v1
   def testConv2D2x2Filter(self):
     # The inputs look like this (it's a 3 x 2 matrix, each of depth 2):
     #
diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py
index 6cd4974671..4be78b2d5c 100644
--- a/tensorflow/python/kernel_tests/nth_element_op_test.py
+++ b/tensorflow/python/kernel_tests/nth_element_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
@@ -111,17 +112,20 @@ class NthElementTest(test.TestCase):
     self._testEnumerateN([10, 10, 10])
     self._testEnumerateN([10, 10, 10, 10])
 
+  @test_util.run_deprecated_v1
   def testInvalidInput(self):
     with self.assertRaisesRegexp(ValueError,
                                  "at least rank 1 but is rank 0"):
       nn_ops.nth_element(5, 0)
 
+  @test_util.run_deprecated_v1
   def testInvalidInputAtEval(self):
     with self.session(use_gpu=False):
       v = array_ops.placeholder(dtype=dtypes.float32)
       with self.assertRaisesOpError("Input must be >= 1-D"):
         nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0})
 
+  @test_util.run_deprecated_v1
   def testInvalidN(self):
     with self.assertRaisesRegexp(ValueError,
                                  "non-negative but is -1"):
@@ -130,6 +134,7 @@ class NthElementTest(test.TestCase):
                                  "scalar but has rank 1"):
       nn_ops.nth_element([5, 6, 3], [1])
 
+  @test_util.run_deprecated_v1
   def testInvalidNAtEval(self):
     inputs = [[0.1, 0.2], [0.3, 0.4]]
     with self.session(use_gpu=False):
@@ -138,12 +143,14 @@ class NthElementTest(test.TestCase):
       with self.assertRaisesOpError("Need n >= 0, got -7"):
         values.eval(feed_dict={n: -7})
 
+  @test_util.run_deprecated_v1
   def testNTooLarge(self):
     inputs = [[0.1, 0.2], [0.3, 0.4]]
     with self.assertRaisesRegexp(ValueError,
                                  "must have last dimension > n = 2"):
       nn_ops.nth_element(inputs, 2)
 
+  @test_util.run_deprecated_v1
   def testNTooLargeAtEval(self):
     inputs = [[0.1, 0.2], [0.3, 0.4]]
     with self.session(use_gpu=False):
@@ -152,6 +159,7 @@ class NthElementTest(test.TestCase):
       with self.assertRaisesOpError(r"Input must have at least n\+1 columns"):
         values.eval(feed_dict={n: 2})
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     with self.session(use_gpu=False) as sess:
       inputs = array_ops.placeholder(dtypes.float32, shape=[3, 5])
diff --git a/tensorflow/python/kernel_tests/numerics_test.py b/tensorflow/python/kernel_tests/numerics_test.py
index e3210dcddc..5751f3fe76 100644
--- a/tensorflow/python/kernel_tests/numerics_test.py
+++ b/tensorflow/python/kernel_tests/numerics_test.py
@@ -66,6 +66,7 @@ class VerifyTensorAllFiniteTest(test.TestCase):
 
 class NumericsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInf(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant(1.0)
@@ -76,6 +77,7 @@ class NumericsTest(test.TestCase):
       with self.assertRaisesOpError("Inf"):
         self.evaluate(a)
 
+  @test_util.run_deprecated_v1
   def testNaN(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant(0.0)
@@ -86,6 +88,7 @@ class NumericsTest(test.TestCase):
       with self.assertRaisesOpError("NaN"):
         self.evaluate(a)
 
+  @test_util.run_deprecated_v1
   def testBoth(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant([1.0, 0.0])
@@ -104,6 +107,7 @@ class NumericsTest(test.TestCase):
       self.assertAllEqual(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), value)
       self.assertEqual([2, 3], checked.get_shape())
 
+  @test_util.run_deprecated_v1
   def testControlFlowCond(self):
     predicate = array_ops.placeholder(dtypes.bool, shape=[])
     _ = control_flow_ops.cond(predicate,
@@ -116,6 +120,7 @@ class NumericsTest(test.TestCase):
         r"or `tf.while_loop\(\)`\."):
       numerics.add_check_numerics_ops()
 
+  @test_util.run_deprecated_v1
   def testControlFlowWhile(self):
     predicate = array_ops.placeholder(dtypes.bool, shape=[])
     _ = control_flow_ops.while_loop(lambda _: predicate,
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index 6fe98d2559..7b1b054ae0 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.platform import test
@@ -116,6 +117,7 @@ class PadOpTest(test.TestCase):
           self._testGradient(np_inputs, paddings, mode=mode,
                              constant_values=constant_values)
 
+  @test_util.run_deprecated_v1
   def testInputDims(self):
     with self.session(use_gpu=True):
       with self.assertRaises(ValueError):
@@ -124,6 +126,7 @@ class PadOpTest(test.TestCase):
                       array_ops.reshape(
                           [1, 2], shape=[1, 2]))
 
+  @test_util.run_deprecated_v1
   def testPaddingsDim(self):
     with self.session(use_gpu=True):
       with self.assertRaises(ValueError):
@@ -132,6 +135,7 @@ class PadOpTest(test.TestCase):
                       array_ops.reshape(
                           [1, 2], shape=[2]))
 
+  @test_util.run_deprecated_v1
   def testPaddingsDim2(self):
     with self.session(use_gpu=True):
       with self.assertRaises(ValueError):
@@ -140,6 +144,7 @@ class PadOpTest(test.TestCase):
                       array_ops.reshape(
                           [1, 2], shape=[2, 1]))
 
+  @test_util.run_deprecated_v1
   def testPaddingsDim3(self):
     with self.session(use_gpu=True):
       with self.assertRaises(ValueError):
@@ -148,6 +153,7 @@ class PadOpTest(test.TestCase):
                       array_ops.reshape(
                           [1, 2], shape=[1, 2]))
 
+  @test_util.run_deprecated_v1
   def testPaddingsDim4(self):
     with self.session(use_gpu=True):
       with self.assertRaises(ValueError):
@@ -156,6 +162,7 @@ class PadOpTest(test.TestCase):
                       array_ops.reshape(
                           [1, 2, 3, 4, 5, 6], shape=[3, 2]))
 
+  @test_util.run_deprecated_v1
   def testPaddingsNonNegative(self):
     with self.session(use_gpu=True):
       with self.assertRaisesRegexp(ValueError, "must be non-negative"):
@@ -164,6 +171,7 @@ class PadOpTest(test.TestCase):
                       constant_op.constant(
                           [-1, 0], shape=[1, 2]))
 
+  @test_util.run_deprecated_v1
   def testPaddingsNonNegative2(self):
     with self.session(use_gpu=True):
       with self.assertRaisesRegexp(ValueError, "must be non-negative"):
@@ -223,6 +231,7 @@ class PadOpTest(test.TestCase):
           np.random.randint(-100, 100, (4, 2, 1, 3)).astype(t),
           [[0, 0], [0, 0], [0, 0], [0, 0]], -123)
 
+  @test_util.run_deprecated_v1
   def testFloatTypes(self):
     for t in [np.float32, np.float64]:
       self._testAll(np.random.rand(2, 5).astype(t), [[1, 0], [2, 0]], 0.0)
@@ -261,6 +270,7 @@ class PadOpTest(test.TestCase):
           [[b"Hello", b"World", b"World"], [b"Hello", b"World", b"World"],
            [b"Goodnight", b"Moon", b"Moon"]], self.evaluate(symmetric))
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionEdgeCases(self):
     # Unknown paddings shape.
     inp = constant_op.constant(0.0, shape=[4, 4, 4, 4])
@@ -277,6 +287,7 @@ class PadOpTest(test.TestCase):
     padded = array_ops.pad(inp, array_ops.placeholder(dtypes.int32))
     self.assertAllEqual(None, padded.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testPartialShapeInformation(self):
     unknown = array_ops.placeholder(dtypes.int32)
 
@@ -341,6 +352,7 @@ class PadOpTest(test.TestCase):
       self.assertAllEqual(inp, out)
       self.assertShapeEqual(inp, tf_val)
 
+  @test_util.run_deprecated_v1
   def testCollapseAdjacentNonPaddedDimensions(self):
     # pyformat: disable
     paddings_values = [[[0, 0], [0, 0], [0, 0], [0, 1]],
diff --git a/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py b/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py
index c9221f8c20..f87f517053 100644
--- a/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py
+++ b/tensorflow/python/kernel_tests/parameterized_truncated_normal_op_test.py
@@ -29,6 +29,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
@@ -166,30 +167,39 @@ class ParameterizedTruncatedNormalTest(test.TestCase):
     except ImportError as e:
       tf_logging.warn("Cannot test truncated normal op: %s" % str(e))
 
+  @test_util.run_deprecated_v1
   def testDefaults(self):
     self.validateMoments([10**5], 0.0, 1.0, -2.0, 2.0)
 
+  @test_util.run_deprecated_v1
   def testShifted(self):
     self.validateMoments([10**5], -1.0, 1.0, -2.0, 2.0)
 
+  @test_util.run_deprecated_v1
   def testRightTail(self):
     self.validateMoments([10**5], 0.0, 1.0, 4.0, np.infty)
 
+  @test_util.run_deprecated_v1
   def testLeftTail(self):
     self.validateMoments([10**5], 0.0, 1.0, -np.infty, -4.0)
 
+  @test_util.run_deprecated_v1
   def testLeftTailTwoSidedBounds(self):
     self.validateMoments([10**5], 0.0, 1.0, -6.0, -3.0)
 
+  @test_util.run_deprecated_v1
   def testTwoSidedLeftTailShifted(self):
     self.validateKolmogorovSmirnov([10**5], 6.0, 1.0, -1.0, 1.0)
 
+  @test_util.run_deprecated_v1
   def testRightTailShifted(self):
     self.validateMoments([10**5], -5.0, 1.0, 2.0, np.infty)
 
+  @test_util.run_deprecated_v1
   def testSmallStddev(self):
     self.validateKolmogorovSmirnov([10**5], 0.0, 0.1, 0.05, 0.10)
 
+  @test_util.run_deprecated_v1
   def testSamplingWithSmallStdDevFarFromBound(self):
     sample_op = random_ops.parameterized_truncated_normal(
         shape=(int(1e5),), means=0.8, stddevs=0.05, minvals=-1., maxvals=1.)
@@ -202,6 +212,7 @@ class ParameterizedTruncatedNormalTest(test.TestCase):
       no_neg_samples = np.sum(samples < 0.)
       self.assertEqual(no_neg_samples, 0.)
 
+  @test_util.run_deprecated_v1
   def testSamplingAtRandnSwitchover(self):
     # The randn sampler is used as the bounds are moved farther from the mean,
     # and the probability of accepting a sample increases the farther the
diff --git a/tensorflow/python/kernel_tests/parse_single_example_op_test.py b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
index 3f50087282..43c8fa4ab5 100644
--- a/tensorflow/python/kernel_tests/parse_single_example_op_test.py
+++ b/tensorflow/python/kernel_tests/parse_single_example_op_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
@@ -121,6 +122,7 @@ class ParseExampleTest(test.TestCase):
           self.assertEqual(
               tuple(out[k].dense_shape.get_shape().as_list()), (1,))
 
+  @test_util.run_deprecated_v1
   def testEmptySerializedWithAllDefaults(self):
     sparse_name = "st_a"
     a_name = "a"
@@ -229,6 +231,7 @@ class ParseExampleTest(test.TestCase):
         },
         expected_err=(ValueError, "Missing shape for feature a"))
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingSparse(self):
     original = [
         example(features=features({
@@ -552,6 +555,7 @@ class ParseExampleTest(test.TestCase):
           }
       }, expected_output)
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingSparseAndSparseFeatureAndDenseWithNoDefault(self):
     original = [
         example(features=features({
@@ -618,6 +622,7 @@ class ParseExampleTest(test.TestCase):
           },
           expected_output)
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingSparseAndSparseFeatureWithReuse(self):
     original = [
         example(features=features({
@@ -658,6 +663,7 @@ class ParseExampleTest(test.TestCase):
           }
       }, expected_output)
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingVarLenDense(self):
     aname = "a"
     bname = "b"
@@ -869,6 +875,7 @@ class ParseSingleExampleTest(test.TestCase):
           self.assertEqual(
               tuple(out[k].dense_shape.get_shape().as_list()), (1,))
 
+  @test_util.run_deprecated_v1
   def testSingleExampleWithSparseAndSparseFeatureAndDense(self):
     original = example(features=features({
         "c": float_feature([3, 4]),
diff --git a/tensorflow/python/kernel_tests/parsing_ops_test.py b/tensorflow/python/kernel_tests/parsing_ops_test.py
index 1f677103dc..af76e09f39 100644
--- a/tensorflow/python/kernel_tests/parsing_ops_test.py
+++ b/tensorflow/python/kernel_tests/parsing_ops_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import parsing_ops
@@ -121,6 +122,7 @@ class ParseExampleTest(test.TestCase):
           self.assertEqual(
               tuple(out[k].dense_shape.get_shape().as_list()), (2,))
 
+  @test_util.run_deprecated_v1
   def testEmptySerializedWithAllDefaults(self):
     sparse_name = "st_a"
     a_name = "a"
@@ -243,6 +245,7 @@ class ParseExampleTest(test.TestCase):
         },
         expected_err=(ValueError, "Missing shape for feature a"))
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingSparse(self):
     original = [
         example(features=features({
@@ -571,6 +574,7 @@ class ParseExampleTest(test.TestCase):
         }
     }, expected_output)
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingSparseAndSparseFeatureAndDenseWithNoDefault(self):
     expected_st_a = (  # indices, values, shape
         np.empty((0, 2), dtype=np.int64),  # indices
@@ -631,6 +635,7 @@ class ParseExampleTest(test.TestCase):
         },
         expected_output)
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingSparseAndSparseFeatureWithReuse(self):
     expected_idx = (  # indices, values, shape
         np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=np.int64),
@@ -740,6 +745,7 @@ class ParseExampleTest(test.TestCase):
     for batch_size in (1, 10, 20, 100, 256):
       self._testSerializedContainingVarLenDenseLargerBatch(batch_size)
 
+  @test_util.run_deprecated_v1
   def testSerializedContainingVarLenDense(self):
     aname = "a"
     bname = "b"
@@ -962,6 +968,7 @@ class ParseSingleExampleTest(test.TestCase):
           self.assertEqual(
               tuple(out[k].dense_shape.get_shape().as_list()), (1,))
 
+  @test_util.run_deprecated_v1
   def testSingleExampleWithSparseAndSparseFeatureAndDense(self):
     original = example(
         features=features({
@@ -1180,6 +1187,7 @@ class ParseSequenceExampleTest(test.TestCase):
         expected_err=expected_err,
         batch=True)
 
+  @test_util.run_deprecated_v1
   def testSequenceExampleWithSparseAndDenseContext(self):
     original = sequence_example(
         context=features({
@@ -1223,6 +1231,7 @@ class ParseSequenceExampleTest(test.TestCase):
         },
         expected_context_values=expected_context_output)
 
+  @test_util.run_deprecated_v1
   def testSequenceExampleWithMultipleSizeFeatureLists(self):
     original = sequence_example(
         feature_lists=feature_lists({
@@ -1286,6 +1295,7 @@ class ParseSequenceExampleTest(test.TestCase):
         },
         expected_feat_list_values=expected_feature_list_output)
 
+  @test_util.run_deprecated_v1
   def testSequenceExampleWithoutDebugName(self):
     original = sequence_example(
         feature_lists=feature_lists({
@@ -1343,6 +1353,7 @@ class ParseSequenceExampleTest(test.TestCase):
         },
         expected_feat_list_values=expected_feature_list_output)
 
+  @test_util.run_deprecated_v1
   def testSequenceExampleWithSparseAndDenseFeatureLists(self):
     original = sequence_example(
         feature_lists=feature_lists({
@@ -1401,6 +1412,7 @@ class ParseSequenceExampleTest(test.TestCase):
         },
         expected_feat_list_values=expected_feature_list_output)
 
+  @test_util.run_deprecated_v1
   def testSequenceExampleWithEmptyFeatureInFeatureLists(self):
     original = sequence_example(
         feature_lists=feature_lists({
@@ -1541,6 +1553,7 @@ class ParseSequenceExampleTest(test.TestCase):
             " feature_list_dense_missing_assumed_empty or"
             " feature_list_dense_defaults?"))
 
+  @test_util.run_deprecated_v1
   def testSequenceExampleBatch(self):
     first = sequence_example(
         feature_lists=feature_lists({
@@ -1695,6 +1708,7 @@ class DecodeJSONExampleTest(test.TestCase):
             })),
     ])
 
+  @test_util.run_deprecated_v1
   def testInvalidSyntax(self):
     with self.cached_session() as sess:
       json_tensor = constant_op.constant(["{]"])
@@ -1705,6 +1719,7 @@ class DecodeJSONExampleTest(test.TestCase):
 
 class ParseTensorOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testToFloat32(self):
     with self.cached_session():
       expected = np.random.rand(3, 4, 5).astype(np.float32)
@@ -1718,6 +1733,7 @@ class ParseTensorOpTest(test.TestCase):
 
       self.assertAllEqual(expected, result)
 
+  @test_util.run_deprecated_v1
   def testToUint8(self):
     with self.cached_session():
       expected = np.random.rand(3, 4, 5).astype(np.uint8)
@@ -1731,6 +1747,7 @@ class ParseTensorOpTest(test.TestCase):
 
       self.assertAllEqual(expected, result)
 
+  @test_util.run_deprecated_v1
   def testTypeMismatch(self):
     with self.cached_session():
       expected = np.random.rand(3, 4, 5).astype(np.uint8)
@@ -1744,6 +1761,7 @@ class ParseTensorOpTest(test.TestCase):
           r"\(uint16\)"):
         tensor.eval(feed_dict={serialized: tensor_proto.SerializeToString()})
 
+  @test_util.run_deprecated_v1
   def testInvalidInput(self):
     with self.cached_session():
       serialized = array_ops.placeholder(dtypes.string)
diff --git a/tensorflow/python/kernel_tests/partitioned_variables_test.py b/tensorflow/python/kernel_tests/partitioned_variables_test.py
index 0c04656196..48655391fa 100644
--- a/tensorflow/python/kernel_tests/partitioned_variables_test.py
+++ b/tensorflow/python/kernel_tests/partitioned_variables_test.py
@@ -26,6 +26,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import partitioned_variables
@@ -322,6 +323,7 @@ class PartitionedVariablesTestCase(test.TestCase):
     for i in xrange(len(expected_specs)):
       self.assertEquals(expected_specs[i], slices[i]._save_slice_info.spec)
 
+  @test_util.run_deprecated_v1
   def testVecConstantInit(self):
     with self.cached_session():
       rnd_par = constant_op.constant([1, 2, 3, 4])
@@ -333,6 +335,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       self.assertEqual([dtypes.int32] * 4, [v.dtype.base_dtype for v in vs])
       self._TestSaveSpec(vs, ["4 0,1", "4 1,1", "4 2,1", "4 3,1"])
 
+  @test_util.run_deprecated_v1
   def testConstantInit(self):
     with self.cached_session():
       rnd_par = constant_op.constant([[1, 2, 3, 4], [5, 6, 7, 8]])
@@ -401,12 +404,15 @@ class PartitionedVariablesTestCase(test.TestCase):
       self.assertEqual(var2_name + "/part_0:0", vs2[0].name)
       self.assertEqual(var2_name + "/part_1:0", vs2[1].name)
 
+  @test_util.run_deprecated_v1
   def testName(self):
     self._testNameHelper(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceName(self):
     self._testNameHelper(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testRandomInitValue(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([200, 40]))
@@ -424,6 +430,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           "200 40 0,200:36,4"
       ])
 
+  @test_util.run_deprecated_v1
   def testRandomInitUnevenPartitions(self):
     with self.cached_session():
       rnd = variables.Variable(
@@ -462,6 +469,7 @@ class PartitionedVariablesTestCase(test.TestCase):
         if i < len(save_specs):
           self._TestSaveSpec(vs, save_specs[i])
 
+  @test_util.run_deprecated_v1
   def testDegenerate(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([10, 43]))
@@ -473,6 +481,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       self.assertAllClose(rnd, val)
       self._TestSaveSpec(vs, ["10 43 0,10:0,43"])
 
+  @test_util.run_deprecated_v1
   def testSliceSizeOne(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([10, 43]))
@@ -488,6 +497,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           "10 43 6,1:0,43", "10 43 7,1:0,43", "10 43 8,1:0,43", "10 43 9,1:0,43"
       ])
 
+  @test_util.run_deprecated_v1
   def testIotaInitializer(self):
     self.assertAllClose([0., 1., 2., 3.], _IotaInitializer([4]))
     self.assertAllClose([[0., 1.], [0., 10.], [0., 100.], [0., 1000.]],
@@ -503,6 +513,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       self.assertAllClose(slice0 + slice1 + slice2, val)
       self._TestSaveSpec(vs, ["13 5 0,5:0,5", "13 5 5,4:0,5", "13 5 9,4:0,5"])
 
+  @test_util.run_deprecated_v1
   def testRandomInitializer(self):
     # Sanity check that the slices uses a different seed when using a random
     # initializer function.
@@ -546,6 +557,7 @@ class PartitionedVariablesTestCase(test.TestCase):
         partitioned_variables.create_partitioned_variables(
             [10, 43], [1, 50], rnd.initialized_value())
 
+  @test_util.run_deprecated_v1
   def testControlDepsNone(self):
     with self.cached_session() as session:
       c = constant_op.constant(1.0)
@@ -572,6 +584,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       for op in reading_ops:
         self.assertEqual([], op.control_inputs)
 
+  @test_util.run_deprecated_v1
   def testConcat(self):
     with self.cached_session() as session:
       var_x = variable_scope.get_variable(
diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py
index 92016a49a2..78e786f01c 100644
--- a/tensorflow/python/kernel_tests/pool_test.py
+++ b/tensorflow/python/kernel_tests/pool_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -301,6 +302,7 @@ class PoolingTest(test.TestCase):
     err_tolerance = 1e-2
     self.assertLess(err, err_tolerance)
 
+  @test_util.run_deprecated_v1
   def testGradient1D(self):
     with self.session(use_gpu=test.is_gpu_available()):
       for padding in ["SAME", "VALID"]:
@@ -327,6 +329,7 @@ class PoolingTest(test.TestCase):
                     dilation_rate=[1],
                     strides=strides)
 
+  @test_util.run_deprecated_v1
   def testGradient2D(self):
     with self.session(use_gpu=test.is_gpu_available()):
       for padding in ["SAME", "VALID"]:
@@ -353,6 +356,7 @@ class PoolingTest(test.TestCase):
                     dilation_rate=[1, 1],
                     strides=strides)
 
+  @test_util.run_deprecated_v1
   def testGradient3D(self):
     with self.session(use_gpu=test.is_gpu_available()):
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
index a8e962bc3a..24fb51fc47 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
@@ -294,6 +294,7 @@ class PoolingTest(test.TestCase):
                                               use_gpu=use_gpu,
                                               **kwargs)
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradValidPadding1_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -303,6 +304,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradValidPadding2_1_6_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -312,6 +314,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradValidPadding2_1_7_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -321,6 +324,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradValidPadding1_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -330,6 +334,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradValidPadding2_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -339,6 +344,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradSamePadding1_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -348,6 +354,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradSamePadding1_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -357,6 +364,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradSamePadding2_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -366,6 +374,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradSamePadding2_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -375,6 +384,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradSamePadding3_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.max_pool3d,
@@ -384,6 +394,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradValidPadding1_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -393,6 +404,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradValidPadding1_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -402,6 +414,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradValidPadding2_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -411,6 +424,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradValidPadding2_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -420,6 +434,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="VALID")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradSamePadding1_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -429,6 +444,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradSamePadding1_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -438,6 +454,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradSamePadding2_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -447,6 +464,7 @@ class PoolingTest(test.TestCase):
         strides=(1, 1, 1),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradSamePadding2_2_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
@@ -456,6 +474,7 @@ class PoolingTest(test.TestCase):
         strides=(2, 2, 2),
         padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGradSamePadding3_1_3d(self):
     self._ConstructAndTestGradient(
         nn_ops.avg_pool3d,
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 81222719f2..c33b59bb99 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -384,6 +384,7 @@ class PoolingTest(test.TestCase):
         expected=[],
         use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testAvgPooling(self):
     for use_gpu in True, False:
       self._testAvgPoolValidPadding(use_gpu)
@@ -577,6 +578,7 @@ class PoolingTest(test.TestCase):
         expected=[],
         use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testMaxPooling(self):
     for use_gpu in True, False:
       self._testMaxPoolValidPadding(use_gpu)
@@ -588,6 +590,7 @@ class PoolingTest(test.TestCase):
       self._testMaxPoolEmptyInput(use_gpu)
 
   # Tests for DepthwiseMaxPooling on CPU only.
+  @test_util.run_deprecated_v1
   def testDepthwiseMaxPool1x1DepthWindow1(self):
     # input is:
     # [1.0, ..., 10.0] along depth,
@@ -613,6 +616,7 @@ class PoolingTest(test.TestCase):
           use_gpu=False,
           v2=v2)
 
+  @test_util.run_deprecated_v1
   def testDepthwiseMaxPool2x2DepthWindow3(self):
     # input is:
     #
@@ -639,6 +643,7 @@ class PoolingTest(test.TestCase):
           use_gpu=False,
           v2=v2)
 
+  @test_util.run_deprecated_v1
   def testKernelSmallerThanStrideValid(self):
     for use_gpu in [True, False]:
       self._VerifyValues(
@@ -670,6 +675,7 @@ class PoolingTest(test.TestCase):
           expected=[5, 8, 26, 29],
           use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testKernelSmallerThanStrideSame(self):
     for use_gpu in [True, False]:
       for pool_func in [nn_ops.max_pool, nn_ops.avg_pool]:
@@ -1167,6 +1173,7 @@ class PoolingTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1497,6 +1504,7 @@ class PoolingTest(test.TestCase):
     else:
       del os.environ["TF_ENABLE_MAXPOOL_NANPROP"]
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradDirect(self):
     self._testMaxPoolGradDirect1_1()
     self._testMaxPoolGradDirect1_2()
@@ -1616,6 +1624,7 @@ class PoolingTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testMaxPoolGradGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradGradValidPadding1_1(data_format, use_gpu)
@@ -1649,6 +1658,7 @@ class PoolingTest(test.TestCase):
         orig_input, orig_output, grad, [1, window_rows, window_cols, 1],
         [1, row_stride, col_stride, 1], padding)
 
+  @test_util.run_deprecated_v1
   def testAvgPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testAvgPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1778,6 +1788,7 @@ class PoolingTest(test.TestCase):
         data_format=data_format,
         use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionEdgeCases(self):
     # All shapes unknown.
     for pool_func in [nn_ops.max_pool, nn_ops.avg_pool]:
@@ -1806,6 +1817,7 @@ class PoolingTest(test.TestCase):
             strides=[1, 1, 1, 1],
             padding="SAME")
 
+  @test_util.run_deprecated_v1
   def testOpEdgeCases(self):
     with self.session(use_gpu=test.is_gpu_available()) as sess:
       pool_funcs = [nn_ops.max_pool, nn_ops.avg_pool]
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index c9cbe44a7f..1f3f02a9f0 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -514,6 +514,7 @@ class PyFuncTest(test.TestCase):
       self.assertAllEqual(ret, [[3.0], [3.0], [3.0]])
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testEagerExceptionHandling(self):
     with test_util.device(use_gpu=True):
       self._testExceptionHandling(
@@ -533,6 +534,7 @@ class PyFuncTest(test.TestCase):
       self._testExceptionHandling(WeirdError, errors.UnknownError, eager=True)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testEagerReturningVariableRaisesError(self):
     def return_variable():
       return resource_variable_ops.ResourceVariable(0.0)
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 305b5aa364..0f2537b371 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import linalg_ops
@@ -49,6 +50,7 @@ class QrOpTest(test.TestCase):
                                  "Shape must be at least rank 2 but is rank 1"):
       linalg_ops.qr(vector)
 
+  @test_util.run_deprecated_v1
   def testConcurrentExecutesWithoutError(self):
     with self.session(use_gpu=True) as sess:
       all_ops = []
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
index cab841741e..576720528e 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 
@@ -66,6 +67,7 @@ class MultinomialTest(test.TestCase):
             counts_by_indices[index] = count
     self.assertEqual(counts_by_indices[0], 100000000)
 
+  @test_util.run_deprecated_v1
   def testLargeDynamicRange3(self):
     random_seed.set_random_seed(10)
     counts_by_indices = {}
diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
index 031a1c281c..5d123307a8 100644
--- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py
+++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
@@ -66,6 +66,7 @@ class MultinomialTest(test.TestCase):
             logits, num_samples, output_dtype=output_dtype))
         self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples)
 
+  @test_util.run_deprecated_v1
   def testOneOpMultipleStepsIndependent(self):
     with test_util.use_gpu():
       sample_op1, _ = self._make_ops(10)
@@ -88,6 +89,7 @@ class MultinomialTest(test.TestCase):
       # 1 in 2^32 chance of this assertion failing.
       self.assertFalse(np.equal(sample1, sample2).all())
 
+  @test_util.run_deprecated_v1
   def testTwoOpsSameSeedDrawSameSequences(self):
     with test_util.use_gpu():
       sample_op1, sample_op2 = self._make_ops(1000, seed=1)
@@ -194,6 +196,7 @@ class MultinomialTest(test.TestCase):
                   array_ops.zeros([batch, classes]), samples))
           self.assertEqual(x.shape, (batch, samples))
 
+  @test_util.run_deprecated_v1
   def testEmptyClasses(self):
     with test_util.use_gpu():
       x = random_ops.multinomial(array_ops.zeros([5, 0]), 7)
diff --git a/tensorflow/python/kernel_tests/random/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py
index 491d19d6a0..724bee0715 100644
--- a/tensorflow/python/kernel_tests/random/random_crop_test.py
+++ b/tensorflow/python/kernel_tests/random/random_crop_test.py
@@ -20,12 +20,14 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 
 
 class RandomCropTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     # No random cropping is performed since the size is value.shape.
     for shape in (2, 1, 1), (2, 1, 3), (4, 5, 3):
@@ -48,6 +50,7 @@ class RandomCropTest(test.TestCase):
         self.assertAllEqual(y.shape, target)
         self.assertTrue(tuple(y.ravel()) in value_set)
 
+  @test_util.run_deprecated_v1
   def testRandomization(self):
     # Run 1x1 crop num_samples times in an image and ensure that one finds each
     # pixel 1/size of the time.
diff --git a/tensorflow/python/kernel_tests/random/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py
index d18e3feb04..a5952a2196 100644
--- a/tensorflow/python/kernel_tests/random/random_gamma_test.py
+++ b/tensorflow/python/kernel_tests/random/random_gamma_test.py
@@ -26,6 +26,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -53,9 +54,11 @@ class RandomGammaTest(test.TestCase):
 
     return func
 
+  @test_util.run_deprecated_v1
   def testMomentsFloat32(self):
     self._testMoments(dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def testMomentsFloat64(self):
     self._testMoments(dtypes.float64)
 
@@ -208,6 +211,7 @@ class RandomGammaTest(test.TestCase):
         sy = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=use_gpu, seed=345)
         self.assertAllEqual(sx(), sy())
 
+  @test_util.run_deprecated_v1
   def testNoCSE(self):
     """CSE = constant subexpression eliminator.
 
@@ -222,6 +226,7 @@ class RandomGammaTest(test.TestCase):
           diff = rnd2 - rnd1
           self.assertGreater(np.linalg.norm(diff.eval()), 0.1)
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     # Fully known shape.
     rnd = random_ops.random_gamma([150], 2.0)
@@ -253,6 +258,7 @@ class RandomGammaTest(test.TestCase):
     rnd = random_ops.random_gamma([50], array_ops.placeholder(dtypes.float32))
     self.assertIs(None, rnd.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testPositive(self):
     n = int(10e3)
     for dt in [dtypes.float16, dtypes.float32, dtypes.float64]:
diff --git a/tensorflow/python/kernel_tests/random/random_grad_test.py b/tensorflow/python/kernel_tests/random/random_grad_test.py
index d89056c485..aac6eeac06 100644
--- a/tensorflow/python/kernel_tests/random/random_grad_test.py
+++ b/tensorflow/python/kernel_tests/random/random_grad_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
@@ -45,6 +46,7 @@ class AddLeadingUnitDimensionsTest(test.TestCase):
     ret = random_grad.add_leading_unit_dimensions(1.0, 2)
     self.assertAllEqual(ret.shape, [1, 1])
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     x = array_ops.placeholder(dtypes.float32)
     num_dimensions = array_ops.placeholder(dtypes.int32)
@@ -72,6 +74,7 @@ class RandomGammaGradTest(test.TestCase):
   some statistical properties of the derivative.
   """
 
+  @test_util.run_deprecated_v1
   def testGradientsShape(self):
     shape = [2, 3]
     alpha = array_ops.ones([2, 2])
@@ -81,6 +84,7 @@ class RandomGammaGradTest(test.TestCase):
     self.assertAllEqual(grads_alpha.shape, alpha.shape)
     self.assertAllEqual(grads_beta.shape, beta.shape)
 
+  @test_util.run_deprecated_v1
   def testGradientsShapeWithOneSamplePerParameter(self):
     shape = []
     alpha = array_ops.ones([2, 2])
@@ -90,6 +94,7 @@ class RandomGammaGradTest(test.TestCase):
     self.assertAllEqual(grads_alpha.shape, alpha.shape)
     self.assertAllEqual(grads_beta.shape, beta.shape)
 
+  @test_util.run_deprecated_v1
   def testGradientsUnknownShape(self):
     shape = array_ops.placeholder(dtypes.int32)
     alpha = array_ops.placeholder(dtypes.float32)
@@ -138,9 +143,11 @@ class RandomGammaGradTest(test.TestCase):
     except ImportError as e:
       tf_logging.warn("Cannot use special functions in a test: %s" % str(e))
 
+  @test_util.run_deprecated_v1
   def testCompareToExplicitDerivativeFloat(self):
     self._testCompareToExplicitDerivative(dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def testCompareToExplicitDerivativeDouble(self):
     self._testCompareToExplicitDerivative(dtypes.float64)
 
@@ -182,12 +189,15 @@ class RandomGammaGradTest(test.TestCase):
 
     self.assertAllClose(actual_val, expected_val, rtol=1e-3, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testCompareToImplicitDerivativeFloat(self):
     self._testCompareToImplicitDerivative(dtypes.float32)
 
+  @test_util.run_deprecated_v1
   def testCompareToImplicitDerivativeDouble(self):
     self._testCompareToImplicitDerivative(dtypes.float64)
 
+  @test_util.run_deprecated_v1
   def testAverageAlphaGradient(self):
     """Statistical test for the gradient.
 
@@ -207,6 +217,7 @@ class RandomGammaGradTest(test.TestCase):
     dsample_dalpha_val = self.evaluate(dsample_dalpha)
     self.assertAllClose(dsample_dalpha_val, [1.0] * 3, atol=1e-1, rtol=1e-1)
 
+  @test_util.run_deprecated_v1
   def testQuadraticLoss(self):
     """Statistical test for the gradient.
 
diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
index 76618316b2..1384c3f446 100644
--- a/tensorflow/python/kernel_tests/random/random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/random_ops_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -92,6 +93,7 @@ class RandomNormalTest(RandomOpTestCommon):
 
   # Checks that the CPU and GPU implementation returns the same results,
   # given the same random seed
+  @test_util.run_deprecated_v1
   def testCPUGPUMatch(self):
     for dt in dtypes.float16, dtypes.float32, dtypes.float64:
       results = {}
@@ -104,12 +106,14 @@ class RandomNormalTest(RandomOpTestCommon):
       else:
         self.assertAllClose(results[False], results[True], rtol=1e-6, atol=1e-6)
 
+  @test_util.run_deprecated_v1
   def testSeed(self):
     for dt in dtypes.float16, dtypes.float32, dtypes.float64:
       sx = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=True, seed=345)
       sy = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=True, seed=345)
       self.assertAllEqual(sx(), sy())
 
+  @test_util.run_deprecated_v1
   def testNoCSE(self):
     for use_gpu in [False, True]:
       with self.session(use_gpu=use_gpu):
@@ -119,12 +123,14 @@ class RandomNormalTest(RandomOpTestCommon):
         diff = rnd2 - rnd1
         self.assertTrue(np.linalg.norm(diff.eval()) > 0.1)
 
+  @test_util.run_deprecated_v1
   def testSingleSessionNotConstant(self):
     for use_gpu in [False, True]:
       for dt in dtypes.float16, dtypes.float32, dtypes.float64:
         self._testSingleSessionNotConstant(
             random_ops.random_normal, 100, dt, 0.0, 1.0, use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testSingleSessionOpSeedNotConstant(self):
     for use_gpu in [False, True]:
       for dt in dtypes.float16, dtypes.float32, dtypes.float64:
@@ -137,6 +143,7 @@ class RandomNormalTest(RandomOpTestCommon):
             use_gpu=use_gpu,
             op_seed=1345)
 
+  @test_util.run_deprecated_v1
   def testSingleSessionGraphSeedNotConstant(self):
     for use_gpu in [False, True]:
       for dt in dtypes.float16, dtypes.float32, dtypes.float64:
@@ -185,6 +192,7 @@ class TruncatedNormalTest(test.TestCase):
 
   # Checks that the CPU and GPU implementation returns the same results,
   # given the same random seed
+  @test_util.run_deprecated_v1
   def testCPUGPUMatch(self):
     # Skip the test if there is no GPU.
     if not test.is_gpu_available():
@@ -203,6 +211,7 @@ class TruncatedNormalTest(test.TestCase):
       else:
         self.assertAllClose(results[False], results[True], rtol=1e-6, atol=1e-6)
 
+  @test_util.run_deprecated_v1
   def testSeed(self):
     for dt in dtypes.float16, dtypes.float32, dtypes.float64:
       sx = self._Sampler(1000, 0.0, 1.0, dt, use_gpu=True, seed=345)
@@ -219,6 +228,7 @@ class TruncatedNormalTest(test.TestCase):
       print("std(x)", np.std(x), abs(np.std(x) / stddev - 0.85))
       self.assertTrue(abs(np.std(x) / stddev - 0.85) < 0.04)
 
+  @test_util.run_deprecated_v1
   def testLargeShape(self):
     with self.session(use_gpu=True):
       v = variables.Variable(
@@ -226,6 +236,7 @@ class TruncatedNormalTest(test.TestCase):
       n = random_ops.truncated_normal(v.shape)
       self.assertEqual([8589934592, 1], n.shape.as_list())
 
+  @test_util.run_deprecated_v1
   def testNoCSE(self):
     with self.session(use_gpu=True):
       shape = [2, 3, 4]
@@ -287,6 +298,7 @@ class RandomUniformTest(RandomOpTestCommon):
         print("count = ", count)
       self.assertTrue(count < count_limit)
 
+  @test_util.run_deprecated_v1
   def testUniformIntsWithInvalidShape(self):
     for dtype in dtypes.int32, dtypes.int64:
       with self.assertRaisesRegexp(
@@ -299,6 +311,7 @@ class RandomUniformTest(RandomOpTestCommon):
             [1000], minval=1, maxval=[2, 3], dtype=dtype)
 
   # Check that uniform ints actually follow a uniform distribution.
+  @test_util.run_deprecated_v1
   def testUniformInts(self):
     minv = -2
     maxv = 15
@@ -331,6 +344,7 @@ class RandomUniformTest(RandomOpTestCommon):
 
   # Checks that the CPU and GPU implementation returns the same results,
   # given the same random seed
+  @test_util.run_deprecated_v1
   def testCPUGPUMatch(self):
     for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32,
                dtypes.int64):
@@ -342,6 +356,7 @@ class RandomUniformTest(RandomOpTestCommon):
         results[use_gpu] = sampler()
       self.assertAllEqual(results[False], results[True])
 
+  @test_util.run_deprecated_v1
   def testSeed(self):
     for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32,
                dtypes.int64):
@@ -350,6 +365,7 @@ class RandomUniformTest(RandomOpTestCommon):
         sy = self._Sampler(1000, 0, 17, dtype=dt, use_gpu=True, seed=seed)
         self.assertAllEqual(sx(), sy())
 
+  @test_util.run_deprecated_v1
   def testNoCSE(self):
     shape = [2, 3, 4]
     for dtype in dtypes.float16, dtypes.float32, dtypes.int32:
@@ -359,6 +375,7 @@ class RandomUniformTest(RandomOpTestCommon):
         diff = (rnd2 - rnd1).eval()
         self.assertTrue(np.linalg.norm(diff) > 0.1)
 
+  @test_util.run_deprecated_v1
   def testSingleSessionNotConstant(self):
     for use_gpu in [False, True]:
       for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32,
@@ -366,6 +383,7 @@ class RandomUniformTest(RandomOpTestCommon):
         self._testSingleSessionNotConstant(
             random_ops.random_uniform, 100, dt, 0, 17, use_gpu=use_gpu)
 
+  @test_util.run_deprecated_v1
   def testSingleSessionOpSeedNotConstant(self):
     for use_gpu in [False, True]:
       for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32,
@@ -379,6 +397,7 @@ class RandomUniformTest(RandomOpTestCommon):
             use_gpu=use_gpu,
             op_seed=1345)
 
+  @test_util.run_deprecated_v1
   def testSingleSessionGraphSeedNotConstant(self):
     for use_gpu in [False, True]:
       for dt in (dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32,
@@ -395,6 +414,7 @@ class RandomUniformTest(RandomOpTestCommon):
 
 class RandomShapeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testTruncatedNormal(self):
     # Fully known shape.
     rnd1 = random_ops.truncated_normal([1, 2, 3])
@@ -407,6 +427,7 @@ class RandomShapeTest(test.TestCase):
     rnd3 = random_ops.truncated_normal(array_ops.placeholder(dtypes.int32))
     self.assertIs(None, rnd3.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testRandomNormal(self):
     # Fully known shape.
     rnd1 = random_ops.random_normal([1, 2, 3])
@@ -419,6 +440,7 @@ class RandomShapeTest(test.TestCase):
     rnd3 = random_ops.random_normal(array_ops.placeholder(dtypes.int32))
     self.assertIs(None, rnd3.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testRandomUniform(self):
     # Fully known shape.
     rnd1 = random_ops.random_uniform([1, 2, 3])
diff --git a/tensorflow/python/kernel_tests/random/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
index 47c0858db7..0a6b004d68 100644
--- a/tensorflow/python/kernel_tests/random/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random/random_poisson_test.py
@@ -23,6 +23,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
@@ -104,6 +105,7 @@ class RandomPoissonTest(test.TestCase):
 
   # Checks that the CPU and GPU implementation returns the same results,
   # given the same random seed
+  @test_util.run_deprecated_v1
   def testCPUGPUMatch(self):
     for dt in _SUPPORTED_DTYPES:
       results = {}
@@ -115,12 +117,14 @@ class RandomPoissonTest(test.TestCase):
       else:
         self.assertAllClose(results[False], results[True], rtol=1e-6, atol=1e-6)
 
+  @test_util.run_deprecated_v1
   def testSeed(self):
     for dt in dtypes.float16, dtypes.float32, dtypes.float64:
       sx = self._Sampler(1000, 1.0, dt, use_gpu=True, seed=345)
       sy = self._Sampler(1000, 1.0, dt, use_gpu=True, seed=345)
       self.assertAllEqual(sx(), sy())
 
+  @test_util.run_deprecated_v1
   def testNoCSE(self):
     """CSE = constant subexpression eliminator.
 
@@ -142,6 +146,7 @@ class RandomPoissonTest(test.TestCase):
       self.assertEqual([0], rnd.get_shape().as_list())
       self.assertAllClose(np.array([], dtype=np.float32), self.evaluate(rnd))
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     # Fully known shape
     rnd = random_ops.random_poisson(2.0, [150], seed=12345)
@@ -184,6 +189,7 @@ class RandomPoissonTest(test.TestCase):
         seed=12345)
     self.assertIs(None, rnd.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testDTypeCombinationsV2(self):
     """Tests random_poisson_v2() for all supported dtype combinations."""
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
index 071d6c2998..898f38444b 100644
--- a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py
@@ -129,23 +129,29 @@ class StatelessOpsTest(test.TestCase):
           yield (functools.partial(stateless.stateless_multinomial, **kwds),
                  functools.partial(random_ops.multinomial, **kwds))
 
+  @test_util.run_deprecated_v1
   def testMatchFloat(self):
     self._test_match(self._float_cases())
 
+  @test_util.run_deprecated_v1
   def testMatchInt(self):
     self._test_match(self._int_cases())
 
+  @test_util.run_deprecated_v1
   def testMatchMultinomial(self):
     self._test_match(self._multinomial_cases())
 
+  @test_util.run_deprecated_v1
   def testDeterminismFloat(self):
     self._test_determinism(
         self._float_cases(shape_dtypes=(dtypes.int32, dtypes.int64)))
 
+  @test_util.run_deprecated_v1
   def testDeterminismInt(self):
     self._test_determinism(
         self._int_cases(shape_dtypes=(dtypes.int32, dtypes.int64)))
 
+  @test_util.run_deprecated_v1
   def testDeterminismMultinomial(self):
     self._test_determinism(self._multinomial_cases())
 
diff --git a/tensorflow/python/kernel_tests/reader_ops_test.py b/tensorflow/python/kernel_tests/reader_ops_test.py
index a4a18c5219..43d15817e9 100644
--- a/tensorflow/python/kernel_tests/reader_ops_test.py
+++ b/tensorflow/python/kernel_tests/reader_ops_test.py
@@ -28,6 +28,7 @@ import zlib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.lib.io import tf_record
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import io_ops
@@ -145,6 +146,7 @@ class IdentityReaderTest(test.TestCase):
     self.assertAllEqual(expected, k)
     self.assertAllEqual(expected, v)
 
+  @test_util.run_deprecated_v1
   def testOneEpoch(self):
     reader = io_ops.IdentityReader("test_reader")
     work_completed = reader.num_work_units_completed()
@@ -178,6 +180,7 @@ class IdentityReaderTest(test.TestCase):
     self.assertAllEqual(3, self.evaluate(produced))
     self.assertAllEqual(0, self.evaluate(queued_length))
 
+  @test_util.run_deprecated_v1
   def testMultipleEpochs(self):
     reader = io_ops.IdentityReader("test_reader")
     queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
@@ -198,6 +201,7 @@ class IdentityReaderTest(test.TestCase):
                                   "\\(requested 1, current size 0\\)"):
       self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testSerializeRestore(self):
     reader = io_ops.IdentityReader("test_reader")
     produced = reader.num_records_produced()
@@ -252,6 +256,7 @@ class IdentityReaderTest(test.TestCase):
         "Could not parse state for IdentityReader 'test_reader'"):
       self.evaluate(reader.restore_state(b"BOGUS" + state[5:]))
 
+  @test_util.run_deprecated_v1
   def testReset(self):
     reader = io_ops.IdentityReader("test_reader")
     work_completed = reader.num_work_units_completed()
@@ -302,6 +307,7 @@ class WholeFileReaderTest(test.TestCase):
     self.assertAllEqual(compat.as_bytes(self._filenames[index]), k)
     self.assertAllEqual(self._content[index], v)
 
+  @test_util.run_deprecated_v1
   def testOneEpoch(self):
     reader = io_ops.WholeFileReader("test_reader")
     queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
@@ -317,6 +323,7 @@ class WholeFileReaderTest(test.TestCase):
                                   "\\(requested 1, current size 0\\)"):
       self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testInfiniteEpochs(self):
     reader = io_ops.WholeFileReader("test_reader")
     queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
@@ -376,12 +383,15 @@ class TextLineReaderTest(test.TestCase):
                                   "\\(requested 1, current size 0\\)"):
       k, v = self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testOneEpochLF(self):
     self._testOneEpoch(self._CreateFiles(crlf=False))
 
+  @test_util.run_deprecated_v1
   def testOneEpochCRLF(self):
     self._testOneEpoch(self._CreateFiles(crlf=True))
 
+  @test_util.run_deprecated_v1
   def testSkipHeaderLines(self):
     files = self._CreateFiles()
     reader = io_ops.TextLineReader(skip_header_lines=1, name="test_reader")
@@ -562,6 +572,7 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase):
                                   "\\(requested 1, current size 0\\)"):
       k, v = self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testOneEpoch(self):
     for num_records in [0, 7]:
       # gap_bytes=0: hop_bytes=0
@@ -570,6 +581,7 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase):
         files = self._CreateFiles(num_records, gap_bytes)
         self._TestOneEpoch(files, num_records, gap_bytes)
 
+  @test_util.run_deprecated_v1
   def testGzipOneEpoch(self):
     for num_records in [0, 7]:
       # gap_bytes=0: hop_bytes=0
@@ -578,6 +590,7 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase):
         files = self._CreateGzipFiles(num_records, gap_bytes)
         self._TestOneEpoch(files, num_records, gap_bytes, encoding="GZIP")
 
+  @test_util.run_deprecated_v1
   def testZlibOneEpoch(self):
     for num_records in [0, 7]:
       # gap_bytes=0: hop_bytes=0
@@ -586,17 +599,20 @@ class FixedLengthRecordReaderTest(TFCompressionTestCase):
         files = self._CreateZlibFiles(num_records, gap_bytes)
         self._TestOneEpoch(files, num_records, gap_bytes, encoding="ZLIB")
 
+  @test_util.run_deprecated_v1
   def testOneEpochWithHopBytes(self):
     for num_overlapped_records in [0, 2]:
       files = self._CreateOverlappedRecordFiles(num_overlapped_records)
       self._TestOneEpochWithHopBytes(files, num_overlapped_records)
 
+  @test_util.run_deprecated_v1
   def testGzipOneEpochWithHopBytes(self):
     for num_overlapped_records in [0, 2]:
       files = self._CreateGzipOverlappedRecordFiles(num_overlapped_records,)
       self._TestOneEpochWithHopBytes(
           files, num_overlapped_records, encoding="GZIP")
 
+  @test_util.run_deprecated_v1
   def testZlibOneEpochWithHopBytes(self):
     for num_overlapped_records in [0, 2]:
       files = self._CreateZlibOverlappedRecordFiles(num_overlapped_records)
@@ -609,6 +625,7 @@ class TFRecordReaderTest(TFCompressionTestCase):
   def setUp(self):
     super(TFRecordReaderTest, self).setUp()
 
+  @test_util.run_deprecated_v1
   def testOneEpoch(self):
     files = self._CreateFiles()
     reader = io_ops.TFRecordReader(name="test_reader")
@@ -627,6 +644,7 @@ class TFRecordReaderTest(TFCompressionTestCase):
                                   "\\(requested 1, current size 0\\)"):
       k, v = self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testReadUpTo(self):
     files = self._CreateFiles()
     reader = io_ops.TFRecordReader(name="test_reader")
@@ -654,6 +672,7 @@ class TFRecordReaderTest(TFCompressionTestCase):
     self.assertEqual(self._num_files * self._num_records, num_k)
     self.assertEqual(self._num_files * self._num_records, num_v)
 
+  @test_util.run_deprecated_v1
   def testReadZlibFiles(self):
     options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB)
     files = self._CreateFiles(options)
@@ -670,6 +689,7 @@ class TFRecordReaderTest(TFCompressionTestCase):
         self.assertTrue(compat.as_text(k).startswith("%s:" % files[i]))
         self.assertAllEqual(self._Record(i, j), v)
 
+  @test_util.run_deprecated_v1
   def testReadGzipFiles(self):
     options = tf_record.TFRecordOptions(TFRecordCompressionType.GZIP)
     files = self._CreateFiles(options)
@@ -689,6 +709,7 @@ class TFRecordReaderTest(TFCompressionTestCase):
 
 class AsyncReaderTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoDeadlockFromQueue(self):
     """Tests that reading does not block main execution threads."""
     config = config_pb2.ConfigProto(
@@ -737,6 +758,7 @@ class LMDBReaderTest(test.TestCase):
     self.db_path = os.path.join(self.get_temp_dir(), "data.mdb")
     shutil.copy(path, self.db_path)
 
+  @test_util.run_deprecated_v1
   def testReadFromFile(self):
     reader = io_ops.LMDBReader(name="test_read_from_file")
     queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
@@ -754,6 +776,7 @@ class LMDBReaderTest(test.TestCase):
                                   "\\(requested 1, current size 0\\)"):
       k, v = self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testReadFromSameFile(self):
     with self.cached_session() as sess:
       reader1 = io_ops.LMDBReader(name="test_read_from_same_file1")
@@ -773,6 +796,7 @@ class LMDBReaderTest(test.TestCase):
       coord.request_stop()
       coord.join(threads)
 
+  @test_util.run_deprecated_v1
   def testReadFromFolder(self):
     reader = io_ops.LMDBReader(name="test_read_from_folder")
     queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
@@ -790,6 +814,7 @@ class LMDBReaderTest(test.TestCase):
                                   "\\(requested 1, current size 0\\)"):
       k, v = self.evaluate([key, value])
 
+  @test_util.run_deprecated_v1
   def testReadFromFileRepeatedly(self):
     with self.cached_session() as sess:
       reader = io_ops.LMDBReader(name="test_read_from_file_repeated")
diff --git a/tensorflow/python/kernel_tests/record_input_test.py b/tensorflow/python/kernel_tests/record_input_test.py
index 74020667d9..ad8188b372 100644
--- a/tensorflow/python/kernel_tests/record_input_test.py
+++ b/tensorflow/python/kernel_tests/record_input_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import os
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework.errors_impl import NotFoundError
 from tensorflow.python.lib.io import tf_record
 from tensorflow.python.ops import data_flow_ops
@@ -94,6 +95,7 @@ class RecordInputOpTest(test.TestCase):
 
       self.assertEqual(self.evaluate(yield_op), b"0000000000")
 
+  @test_util.run_deprecated_v1
   def testRecordInputEpochs(self):
     files = 100
     records_per_file = 100
@@ -140,6 +142,7 @@ class RecordInputOpTest(test.TestCase):
         for _ in range(50):
           self.evaluate(yield_op)
 
+  @test_util.run_deprecated_v1
   def testEmptyGlob(self):
     with self.cached_session() as sess:
       record_input = data_flow_ops.RecordInput(file_pattern="foo")
@@ -148,6 +151,7 @@ class RecordInputOpTest(test.TestCase):
       with self.assertRaises(NotFoundError):
         self.evaluate(yield_op)
 
+  @test_util.run_deprecated_v1
   def testBufferTooSmall(self):
     files = 10
     records_per_file = 10
diff --git a/tensorflow/python/kernel_tests/reduce_join_op_test.py b/tensorflow/python/kernel_tests/reduce_join_op_test.py
index c26e62738c..49b6620779 100644
--- a/tensorflow/python/kernel_tests/reduce_join_op_test.py
+++ b/tensorflow/python/kernel_tests/reduce_join_op_test.py
@@ -25,6 +25,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
@@ -230,6 +231,7 @@ class ReduceJoinTest(UnicodeTestCase):
         axis=1,
         separator="  ")
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     input_array = [["a"], ["b"]]
     truth = ["ab"]
@@ -241,6 +243,7 @@ class ReduceJoinTest(UnicodeTestCase):
       self.assertAllEqualUnicode(truth, output_array)
       self.assertAllEqual(truth_shape, reduced.get_shape())
 
+  @test_util.run_deprecated_v1
   def testUnknownIndices(self):
     input_array = [["this", "is", "a", "test"],
                    ["please", "do", "not", "panic"]]
@@ -297,6 +300,7 @@ class ReduceJoinTest(UnicodeTestCase):
       for permutation in itertools.permutations(xrange(num_dims), i):
         self._testMultipleReduceJoin(input_array, axis=permutation)
 
+  @test_util.run_deprecated_v1
   def testInvalidReductionIndices(self):
     with self.cached_session():
       with self.assertRaisesRegexp(ValueError, "Invalid reduction dim"):
@@ -325,6 +329,7 @@ class ReduceJoinTest(UnicodeTestCase):
       output_shape = self.evaluate(output).shape
       self.assertAllEqual([0], output_shape)
 
+  @test_util.run_deprecated_v1
   def testInvalidArgsUnknownShape(self):
     with self.cached_session():
       placeholder = array_ops.placeholder(dtypes.string, name="placeholder")
@@ -335,6 +340,7 @@ class ReduceJoinTest(UnicodeTestCase):
       with self.assertRaisesOpError("Duplicate reduction dimension 1"):
         duplicate_index.eval(feed_dict={placeholder.name: [[""]]})
 
+  @test_util.run_deprecated_v1
   def testInvalidArgsUnknownIndices(self):
     with self.cached_session():
       placeholder = array_ops.placeholder(dtypes.int32, name="placeholder")
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index 4eb329796e..67a89461f3 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
@@ -60,6 +61,7 @@ class ReducedShapeTest(test.TestCase):
     output = math_ops.reduced_shape(shape, axes=axes)
     self.assertAllEqual(output.eval(), result)
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     with self.cached_session():
       self._check([3], [], [3])
@@ -69,6 +71,7 @@ class ReducedShapeTest(test.TestCase):
       self._check([5, 3], [1], [5, 1])
       self._check([5, 3], [0, 1], [1, 1])
 
+  @test_util.run_deprecated_v1
   def testZeros(self):
     """Check that reduced_shape does the right thing with zero dimensions."""
     with self.cached_session():
@@ -83,6 +86,7 @@ class ReducedShapeTest(test.TestCase):
       self._check([3, 0], [1], [3, 1])
       self._check([3, 0], [0, 1], [1, 1])
 
+  @test_util.run_deprecated_v1
   def testNegAxes(self):
     with self.cached_session():
       self._check([10, 10, 10], [-1], [10, 10, 1])
@@ -94,6 +98,7 @@ class ReducedShapeTest(test.TestCase):
 
 class ReductionUnknownShape(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.cached_session():
       for dtype, reductions in [(dtypes.float32,
@@ -188,6 +193,7 @@ class SumReductionTest(BaseReductionTest):
         tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -195,11 +201,13 @@ class SumReductionTest(BaseReductionTest):
           np_arr = np.array([special_value_x, special_value_y]).astype(dtype)
           self._compareAll(np_arr, None)
 
+  @test_util.run_deprecated_v1
   def testInt32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.int32)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testFloat16(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float16)
@@ -219,6 +227,7 @@ class SumReductionTest(BaseReductionTest):
       tf_out_mean = self.evaluate(tf_mean)
     self.assertAllClose(tf_out_mean, 1.)
 
+  @test_util.run_deprecated_v1
   def testFloat32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float32)
@@ -256,21 +265,25 @@ class SumReductionTest(BaseReductionTest):
           self.assertAllClose(sum_y, tf_out_sum_y)
           self.assertAllClose(sum_xz, tf_out_sum_xz)
 
+  @test_util.run_deprecated_v1
   def testFloat64(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float64)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testComplex64(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.complex64)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testComplex128(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.complex128)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testInvalidIndex(self):
     np_arr = np.arange(0, 10).reshape([2, 5]).astype(np.float32)
     input_tensor = ops.convert_to_tensor(np_arr)
@@ -284,6 +297,7 @@ class SumReductionTest(BaseReductionTest):
         ValueError, lambda e: "Invalid reduction dimension" in str(e)):
       math_ops.reduce_sum(input_tensor, [0, 2])
 
+  @test_util.run_deprecated_v1
   def testPartialShapes(self):
     np.random.seed(1618)
 
@@ -317,6 +331,7 @@ class SumReductionTest(BaseReductionTest):
         c_unknown_indices, unknown_indices, keepdims=True)
     self.assertEqual(2, s_unknown_indices_keep.get_shape().rank)
 
+  @test_util.run_deprecated_v1
   def testWrongShapeForReductionIndices(self):
     reduction_axes = [[1], [2]]
     c_unknown = array_ops.placeholder(dtypes.float32)
@@ -326,6 +341,7 @@ class SumReductionTest(BaseReductionTest):
 
   # Int64??
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     for dtype in [
         dtypes.float32, dtypes.float64, dtypes.complex64, dtypes.complex128
@@ -333,6 +349,7 @@ class SumReductionTest(BaseReductionTest):
       x = self._makeIncremental([2, 3, 4, 2], dtype)
       self._compareGradientAxes(x)
 
+  @test_util.run_deprecated_v1
   def testHighRank(self):
     # Do a bunch of random high dimensional reductions
     np.random.seed(42)
@@ -350,11 +367,13 @@ class SumReductionTest(BaseReductionTest):
                    np.arange(1, rank, 2)):
         self._compareAll(data, axes)
 
+  @test_util.run_deprecated_v1
   def testExpand(self):
     # Reduce an empty tensor to a nonempty tensor
     x = np.zeros((5, 0))
     self._compareAll(x, [1])
 
+  @test_util.run_deprecated_v1
   def testEmptyGradients(self):
     with self.session(use_gpu=True):
       x = array_ops.zeros([0, 3])
@@ -362,6 +381,7 @@ class SumReductionTest(BaseReductionTest):
       error = gradient_checker.compute_gradient_error(x, [0, 3], y, [0])
       self.assertEqual(error, 0)
 
+  @test_util.run_deprecated_v1
   def testDegenerate(self):
     with self.session(use_gpu=True):
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
@@ -403,6 +423,7 @@ class MeanReductionTest(BaseReductionTest):
         tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -410,37 +431,44 @@ class MeanReductionTest(BaseReductionTest):
           np_arr = np.array([special_value_x, special_value_y]).astype(dtype)
           self._compareAll(np_arr, None)
 
+  @test_util.run_deprecated_v1
   def testInt32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.int32)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testFloat32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float32)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testFloat64(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float64)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testComplex64(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.complex64)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testComplex128(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.complex128)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     s = [2, 3, 4, 2]
     for dtype in [dtypes.float32, dtypes.float64]:
       x = self._makeIncremental(s, dtype)
       self._compareGradientAxes(x, rtol=1e-3, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testEmptyGradients(self):
     with self.session(use_gpu=True):
       x = array_ops.zeros([0, 3])
@@ -448,6 +476,7 @@ class MeanReductionTest(BaseReductionTest):
       error = gradient_checker.compute_gradient_error(x, [0, 3], y, [0])
       self.assertEqual(error, 0)
 
+  @test_util.run_deprecated_v1
   def testDegenerate(self):
     with self.session(use_gpu=True):
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
@@ -476,6 +505,7 @@ class ProdReductionTest(BaseReductionTest):
         tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -483,6 +513,7 @@ class ProdReductionTest(BaseReductionTest):
           np_arr = np.array([special_value_x, special_value_y]).astype(dtype)
           self._compareAll(np_arr, None)
 
+  @test_util.run_deprecated_v1
   def testInt32(self):
     # Numpy automatically upgrades the type of np.prod from int32 to int64, so
     # Numpy does not overflow an int32 np.prod while TensorFlow does. To avoid
@@ -491,26 +522,31 @@ class ProdReductionTest(BaseReductionTest):
       np_arr = self._makeIncremental((2,) * rank, dtypes.int32) / 2
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testFloat32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float32)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testFloat64(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float64)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testComplex64(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.complex64)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testComplex128(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.complex128)
       self._compareAllAxes(np_arr)
 
+  @test_util.run_deprecated_v1
   def testGradientWithZeros(self):
     s = [2, 3, 4, 2]
     x = self._makeIncremental(s, dtypes.float32) / 20.
@@ -533,6 +569,7 @@ class ProdReductionTest(BaseReductionTest):
     x4[:, :, :, :] = 0
     self._compareGradientAxes(x4, rtol=1e-3, atol=1e-3)
 
+  @test_util.run_deprecated_v1
   def testEmptyGradients(self):
     with self.session(use_gpu=True):
       x = array_ops.zeros([0, 3])
@@ -540,6 +577,7 @@ class ProdReductionTest(BaseReductionTest):
       error = gradient_checker.compute_gradient_error(x, [0, 3], y, [0])
       self.assertEqual(error, 0)
 
+  @test_util.run_deprecated_v1
   def testDegenerate(self):
     with self.session(use_gpu=True):
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
@@ -579,6 +617,7 @@ class MinReductionTest(test.TestCase):
         tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -614,6 +653,7 @@ class MinReductionTest(test.TestCase):
     self._compareAll(np_arr, [0, 2])
     self._compareAll(np_arr, [0, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     s = [2, 3, 4, 2]
     x = np.arange(1.0, 49.0).reshape(s).astype(np.float64)
@@ -624,6 +664,7 @@ class MinReductionTest(test.TestCase):
           t, s, su, [2, 2], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient2(self):
     s = [2, 3, 4, 2]
     x = np.arange(1.0, 49.0).reshape(s).astype(np.float64)
@@ -634,6 +675,7 @@ class MinReductionTest(test.TestCase):
           t, s, su, [2, 4, 2], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient3(self):
     s = [2, 3, 4, 2]
     x = np.arange(1.0, 49.0).reshape(s).astype(np.float64)
@@ -644,6 +686,7 @@ class MinReductionTest(test.TestCase):
           t, s, su, [2, 3, 2], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient4(self):
     s = [2, 3, 4, 2]
     x = np.arange(1.0, 49.0).reshape(s).astype(np.float64)
@@ -654,6 +697,7 @@ class MinReductionTest(test.TestCase):
           t, s, su, [1], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testEmptyGradients(self):
     with self.cached_session():
       x = array_ops.zeros([0, 3])
@@ -692,6 +736,7 @@ class MaxReductionTest(test.TestCase):
         tf_v = self.evaluate(v)
       self.assertAllEqual(tf_v, 0)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -741,6 +786,7 @@ class MaxReductionTest(test.TestCase):
     self._compareAll(np_arr, [0, 2])
     self._compareAll(np_arr, [0, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     s = [2, 3, 4, 2]
     x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64)
@@ -751,6 +797,7 @@ class MaxReductionTest(test.TestCase):
           t, s, su, [2, 2], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient2(self):
     s = [2, 3, 4, 2]
     x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64)
@@ -761,6 +808,7 @@ class MaxReductionTest(test.TestCase):
           t, s, su, [2, 4, 2], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient3(self):
     s = [2, 3, 4, 2]
     x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64)
@@ -771,6 +819,7 @@ class MaxReductionTest(test.TestCase):
           t, s, su, [2, 3, 2], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient4(self):
     s = [2, 3, 4, 2]
     x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64)
@@ -781,6 +830,7 @@ class MaxReductionTest(test.TestCase):
           t, s, su, [1], x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testEmptyGradients(self):
     with self.cached_session():
       x = array_ops.zeros([0, 3])
@@ -913,6 +963,7 @@ class CountNonzeroReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True, feed_dict=feed_dict)
     self._compare(x, reduction_axes, True, use_gpu=False, feed_dict=feed_dict)
 
+  @test_util.run_deprecated_v1
   def testBoolReduce1D(self):
     # Create a 1D array of floats
     np_arr = np.asarray([False, False, True, False, False, True])
@@ -920,11 +971,13 @@ class CountNonzeroReductionTest(test.TestCase):
     self._compareAll(np_arr, [])
     self._compareAll(np_arr, [0])
 
+  @test_util.run_deprecated_v1
   def testFloatReduce1D(self):
     # Create a 1D array of floats
     np_arr = np.asarray([0.0, 1.0, -1.0, 0.0, 0.0, 3.0]).astype(np.float32)
     self._compareAll(np_arr, [0])
 
+  @test_util.run_deprecated_v1
   def testFloatReduce4D(self):
     # Create a 4D array of floats and reduce across some
     # dimensions
@@ -944,11 +997,13 @@ class CountNonzeroReductionTest(test.TestCase):
     self._compareAll(np_arr, [1, 2, 3])
     self._compareAll(np_arr, [0, 1, 2, 3])
 
+  @test_util.run_deprecated_v1
   def testExpand(self):
     # Reduce an empty tensor to a nonempty tensor
     x = np.zeros((5, 0))
     self._compareAll(x, [1])
 
+  @test_util.run_deprecated_v1
   def testDegenerate(self):
     for use_gpu in False, True:
       with self.cached_session(use_gpu=use_gpu):
@@ -964,6 +1019,7 @@ class CountNonzeroReductionTest(test.TestCase):
       v = math_ops.count_nonzero(constant_op.constant(["test"]))
       self.assertAllClose(self.evaluate(v), 1)
 
+  @test_util.run_deprecated_v1
   def testStringReduce1D(self):
     # Create a 1D array of strings
     x = np.asarray(["", "", "a", "", "", "b"])
@@ -974,6 +1030,7 @@ class CountNonzeroReductionTest(test.TestCase):
     self._compare(x, [], keepdims=True, zero=np.str(""))
     self._compare(x, [0], keepdims=True, zero=np.str(""))
 
+  @test_util.run_deprecated_v1
   def testStringReduce2D(self):
     # Create a 2D array of strings
     x = np.asarray([["", "", "a", "", "", "b"],
diff --git a/tensorflow/python/kernel_tests/regex_full_match_op_test.py b/tensorflow/python/kernel_tests/regex_full_match_op_test.py
index 4edd3e98d9..488ec85ab2 100644
--- a/tensorflow/python/kernel_tests/regex_full_match_op_test.py
+++ b/tensorflow/python/kernel_tests/regex_full_match_op_test.py
@@ -23,6 +23,7 @@ from absl.testing import parameterized
 from tensorflow.python.compat import compat
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_string_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
@@ -33,6 +34,7 @@ from tensorflow.python.platform import test
     (gen_string_ops.static_regex_full_match))
 class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testRegexFullMatch(self, op):
     values = ["abaaba", "abcdabcde"]
     with self.cached_session():
@@ -40,6 +42,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
       matched = op(input_tensor, "a.*a").eval()
       self.assertAllEqual([True, False], matched)
 
+  @test_util.run_deprecated_v1
   def testRegexFullMatchTwoDims(self, op):
     values = [["abaaba", "abcdabcde"], ["acdcba", "ebcda"]]
     with self.cached_session():
@@ -47,6 +50,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
       matched = op(input_tensor, "a.*a").eval()
       self.assertAllEqual([[True, False], [True, False]], matched)
 
+  @test_util.run_deprecated_v1
   def testEmptyMatch(self, op):
     values = ["abc", "1"]
     with self.cached_session():
@@ -54,6 +58,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
       matched = op(input_tensor, "").eval()
       self.assertAllEqual([False, False], matched)
 
+  @test_util.run_deprecated_v1
   def testInvalidPattern(self, op):
     values = ["abc", "1"]
     with self.cached_session():
@@ -66,6 +71,7 @@ class RegexFullMatchOpVariantsTest(test.TestCase, parameterized.TestCase):
 
 class RegexFullMatchOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testRegexFullMatchDelegation(self):
     with compat.forward_compatibility_horizon(2018, 11, 1):
       with self.cached_session():
@@ -78,6 +84,7 @@ class RegexFullMatchOpTest(test.TestCase):
         op_tensor = string_ops.regex_full_match(input_tensor, pattern_tensor)
         self.assertTrue(op_tensor.name.startswith("RegexFullMatch"), op.name)
 
+  @test_util.run_deprecated_v1
   def testStaticRegexFullMatchDelegation(self):
     with compat.forward_compatibility_horizon(2018, 11, 20):
       with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/regex_replace_op_test.py b/tensorflow/python/kernel_tests/regex_replace_op_test.py
index ce9a1b5279..6c7dfee7b4 100644
--- a/tensorflow/python/kernel_tests/regex_replace_op_test.py
+++ b/tensorflow/python/kernel_tests/regex_replace_op_test.py
@@ -22,6 +22,7 @@ from absl.testing import parameterized
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_string_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
@@ -32,6 +33,7 @@ from tensorflow.python.platform import test
     (gen_string_ops.static_regex_replace))
 class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testForwarding(self, op):
     with self.cached_session():
       # Generate an input that is uniquely consumed by the regex op.
@@ -45,6 +47,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
       stripped = op(inp, "\\p{Ll}", ".").eval()
       self.assertAllEqual([b"A.C.E", b"H.J.L"], stripped)
 
+  @test_util.run_deprecated_v1
   def testRemovePrefix(self, op):
     values = ["a:foo", "a:bar", "a:foo", "b:baz", "b:qux", "ca:b"]
     with self.cached_session():
@@ -53,6 +56,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual([b"foo", b"bar", b"foo", b"baz", b"qux", b"ca:b"],
                           stripped)
 
+  @test_util.run_deprecated_v1
   def testRegexReplace(self, op):
     values = ["aba\naba", "abcdabcde"]
     with self.cached_session():
@@ -60,6 +64,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
       stripped = op(input_vector, "a.*a", "(\\0)").eval()
       self.assertAllEqual([b"(aba)\n(aba)", b"(abcda)bcde"], stripped)
 
+  @test_util.run_deprecated_v1
   def testEmptyMatch(self, op):
     values = ["abc", "1"]
     with self.cached_session():
@@ -67,6 +72,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
       stripped = op(input_vector, "", "x").eval()
       self.assertAllEqual([b"xaxbxcx", b"x1x"], stripped)
 
+  @test_util.run_deprecated_v1
   def testInvalidPattern(self, op):
     values = ["abc", "1"]
     with self.cached_session():
@@ -76,6 +82,7 @@ class RegexReplaceOpVariantsTest(test.TestCase, parameterized.TestCase):
       with self.assertRaisesOpError("Invalid pattern"):
         self.evaluate(replace)
 
+  @test_util.run_deprecated_v1
   def testGlobal(self, op):
     values = ["ababababab", "abcabcabc", ""]
     with self.cached_session():
@@ -98,6 +105,7 @@ class RegexReplaceTest(test.TestCase, parameterized.TestCase):
       (as_string, as_tensor),
       (as_tensor, as_string),
       (as_tensor, as_tensor))
+  @test_util.run_deprecated_v1
   def testRegexReplaceDelegation(self, pattern_fn, rewrite_fn):
     with self.cached_session():
       input_vector = constant_op.constant("foo", dtypes.string)
@@ -106,6 +114,7 @@ class RegexReplaceTest(test.TestCase, parameterized.TestCase):
       op = string_ops.regex_replace(input_vector, pattern, replace)
       self.assertTrue(op.name.startswith("RegexReplace"))
 
+  @test_util.run_deprecated_v1
   def testStaticRegexReplaceDelegation(self):
     with self.cached_session():
       input_vector = constant_op.constant("foo", dtypes.string)
diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index 30cef90885..55e68f4884 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -105,6 +106,7 @@ class ReluTest(test.TestCase):
 
   # The gradient test for ReLU is a bit tricky as the derivative is not well
   # defined at around zero and we want to avoid that in terms of input values.
+  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -124,6 +126,7 @@ class ReluTest(test.TestCase):
   # The gradient for fp16 is inaccurate due to the low-precision.
   # Instead of relying on compute_gradient_error, we compare the fp16 analytical
   # gradient against their fp32 counterpart.
+  @test_util.run_deprecated_v1
   def testGradientFloat16(self):
     with self.session(use_gpu=True) as sess:
       # Randomly construct a 1D shape from [1, 40)
@@ -150,6 +153,7 @@ class ReluTest(test.TestCase):
         dx_f32_v, dx_f16_v = self.evaluate([dx_f32, dx_f16])
         self.assertAllClose(dx_f32_v, dx_f16_v, atol=3e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -167,6 +171,7 @@ class ReluTest(test.TestCase):
     print("relu (float64) gradient err = ", err)
     self.assertLess(err, 1e-10)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -184,6 +189,7 @@ class ReluTest(test.TestCase):
     print("relu (float32) gradient of gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -252,6 +258,7 @@ class Relu6Test(test.TestCase):
   # The gradient test for ReLU6 is a bit tricky as the derivative is
   # not well defined at around zero and six and we want to avoid that
   # in terms of input values.
+  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -268,6 +275,7 @@ class Relu6Test(test.TestCase):
     print("relu6 (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -325,6 +333,7 @@ class LeakyReluTest(test.TestCase):
   # The gradient test for Leaky ReLU is a bit tricky as the derivative is not
   # well defined at around zero and we want to avoid that in terms of input
   # values.
+  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.test_session():
       x = constant_op.constant(
@@ -341,6 +350,7 @@ class LeakyReluTest(test.TestCase):
     print("leaky_relu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.test_session():
       x = constant_op.constant(
@@ -358,6 +368,7 @@ class LeakyReluTest(test.TestCase):
     print("leaky_relu (float64) gradient err = ", err)
     self.assertLess(err, 1e-10)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with compat.forward_compatibility_horizon(2018, 11, 2):
       with self.test_session():
@@ -376,6 +387,7 @@ class LeakyReluTest(test.TestCase):
       print("leaky_relu (float32) gradient of gradient err = ", err)
       self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with compat.forward_compatibility_horizon(2018, 11, 2):
       with self.test_session():
@@ -439,6 +451,7 @@ class EluTest(test.TestCase):
     for t in [np.float16, np.float32, np.float64]:
       self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
+  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
@@ -450,6 +463,7 @@ class EluTest(test.TestCase):
     print("elu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
@@ -461,6 +475,7 @@ class EluTest(test.TestCase):
     print("elu (float64) gradient err = ", err)
     self.assertLess(err, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testGradGrad(self):
     with self.cached_session():
       x = array_ops.placeholder(dtype=dtypes.float32)
@@ -472,6 +487,7 @@ class EluTest(test.TestCase):
         err = np.abs(gg.eval(feed_dict={x: x_val}) - _elu_grad_grad(x_val))
         self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -489,6 +505,7 @@ class EluTest(test.TestCase):
     print("elu (float32) gradient of gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -539,6 +556,7 @@ class SeluTest(test.TestCase):
         self._testSelu(
             np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
+  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
@@ -550,6 +568,7 @@ class SeluTest(test.TestCase):
     print("selu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
@@ -561,6 +580,7 @@ class SeluTest(test.TestCase):
     print("selu (float64) gradient err = ", err)
     self.assertLess(err, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -578,6 +598,7 @@ class SeluTest(test.TestCase):
     print("selu (float32) gradient of gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with self.cached_session():
       x = constant_op.constant(
diff --git a/tensorflow/python/kernel_tests/reshape_op_test.py b/tensorflow/python/kernel_tests/reshape_op_test.py
index 84539c2b02..db3e88a104 100644
--- a/tensorflow/python/kernel_tests/reshape_op_test.py
+++ b/tensorflow/python/kernel_tests/reshape_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.platform import test
@@ -91,6 +92,7 @@ class ReshapeTest(test.TestCase):
   # TODO(vrv): Add tests for failure conditions once python test_util
   # reports errors.
 
+  @test_util.run_deprecated_v1
   def testFloatReshapeGradThreeDimensions(self):
     x = np.arange(1., 25.).reshape([2, 3, 4]).astype(np.float32)
     s = list(np.shape(x))
@@ -111,6 +113,7 @@ class ReshapeTest(test.TestCase):
     self._testBothReshape(x, [0, 0, 0])
     self._testBothReshape(x, [1, -1, 5])
 
+  @test_util.run_deprecated_v1
   def testErrors(self):
     y = constant_op.constant(0.0, shape=[23, 29, 31])
     with self.assertRaisesRegexp(ValueError, "must be evenly divisible by 17"):
@@ -121,6 +124,7 @@ class ReshapeTest(test.TestCase):
                                  "Cannot reshape a tensor with 4096 elements"):
       array_ops.reshape(z, [4095])
 
+  @test_util.run_deprecated_v1
   def testPartialShapes(self):
     x = array_ops.placeholder(dtypes.float32)
 
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index d30ab6a9c2..b57d9d47aa 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -54,6 +54,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     # involving objects with __del__ defined.
     self.assertEqual(0, len(gc.garbage))
 
+  @test_util.run_deprecated_v1
   def testHandleDtypeShapeMatch(self):
     with self.cached_session():
       handle = resource_variable_ops.var_handle_op(dtype=dtypes.int32, shape=[])
@@ -123,6 +124,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       # values.
       self.assertFalse(np.allclose(variable.numpy(), copied_variable.numpy()))
 
+  @test_util.run_deprecated_v1
   def testGraphDeepCopy(self):
     with self.cached_session():
       init_value = np.ones((4, 4, 4))
@@ -146,6 +148,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
             resource_variable_ops.variable_shape(v.handle)),
         [2])
 
+  @test_util.run_deprecated_v1
   def testDifferentAssignGraph(self):
     with ops.Graph().as_default():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -153,12 +156,14 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     v.assign(2.0)  # Note: this fails if we run convert_to_tensor on not the
     # variable graph.
 
+  @test_util.run_deprecated_v1
   def testFetchHandle(self):
     with self.cached_session():
       handle = resource_variable_ops.var_handle_op(
           dtype=dtypes.int32, shape=[1], name="foo")
       self.assertGreater(len(handle.eval()), 0)
 
+  @test_util.run_deprecated_v1
   def testCachedValueReadBeforeWrite(self):
     with self.cached_session() as sess:
       v = resource_variable_ops.ResourceVariable(0.0, caching_device="cpu:0")
@@ -435,6 +440,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
     self.assertEqual(self.evaluate(read), [[6]])
 
+  @test_util.run_deprecated_v1
   def testScatterUpdateString(self):
     handle = resource_variable_ops.var_handle_op(
         dtype=dtypes.string, shape=[1, 1])
@@ -446,6 +452,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.assertEqual(compat.as_bytes(self.evaluate(read)[0][0]),
                      compat.as_bytes("b"))
 
+  @test_util.run_deprecated_v1
   def testScatterUpdateStringScalar(self):
     handle = resource_variable_ops.var_handle_op(
         dtype=dtypes.string, shape=[1, 1])
@@ -500,6 +507,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
           initial_value=lambda: 1, constraint=constraint, name="var1")
 
   # TODO(alive): how should this work in Eager mode?
+  @test_util.run_deprecated_v1
   def testInitFn(self):
     with self.cached_session():
       v = resource_variable_ops.ResourceVariable(
@@ -591,6 +599,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
           variable_def=other_v_def)
       self.assertTrue(other_v_prime._cached_value is not None)
 
+  @test_util.run_deprecated_v1
   def testVariableDefInitializedInstances(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v_def = resource_variable_ops.ResourceVariable(
@@ -646,6 +655,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     value = self.evaluate(v.sparse_read([0, 3, 1, 2]))
     self.assertAllEqual(init_value[[0, 3, 1, 2], ...], value)
 
+  @test_util.run_deprecated_v1
   def testToFromProto(self):
     with self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -694,6 +704,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.assertEqual(0.0, self.evaluate(v.value()))
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testDestroyResource(self):
     v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
@@ -707,6 +718,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.evaluate(resource_variable_ops.destroy_resource_op(
         handle, ignore_lookup_error=True))
 
+  @test_util.run_deprecated_v1
   def testAssignDifferentShapes(self):
     with self.cached_session() as sess, variable_scope.variable_scope(
         "foo", use_resource=True):
@@ -727,6 +739,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
           assign = var.assign(np.zeros(shape=[2, 2]))
           self.evaluate(assign)
 
+  @test_util.run_deprecated_v1
   def testDtypeAfterFromProto(self):
     v = resource_variable_ops.ResourceVariable(2.0)
     w = resource_variable_ops.ResourceVariable.from_proto(v.to_proto())
@@ -734,6 +747,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.assertEqual(v.dtype, w.dtype)
 
   # TODO(alive): get caching to work in eager mode.
+  @test_util.run_deprecated_v1
   def testCachingDevice(self):
     with ops.device("/job:server/task:1"):
       v = resource_variable_ops.ResourceVariable(
@@ -749,6 +763,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       with self.assertRaises(ValueError):
         _ = w.value().op.get_attr("_class")
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       v = resource_variable_ops.ResourceVariable(300.0, name="var4")
@@ -767,6 +782,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       with self.assertRaisesOpError("Resource .*/var5/.* does not exist"):
         resource_variable_ops.read_variable_op(x, v.dtype.base_dtype).eval()
 
+  @test_util.run_deprecated_v1
   def testSharedNameWithNamescope(self):
     with self.cached_session():
       with ops.name_scope("foo"):
@@ -795,6 +811,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
           "<unknown>",
           str(v.sparse_read(array_ops.placeholder(dtypes.int32)).shape))
 
+  @test_util.run_deprecated_v1
   def testSetInitialValue(self):
     with self.cached_session():
       # Initialize variable with a value different from the initial value passed
@@ -803,6 +820,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       v.initializer.run(feed_dict={v.initial_value: 3.0})
       self.assertEqual(3.0, v.value().eval())
 
+  @test_util.run_deprecated_v1
   def testControlFlowInitialization(self):
     """Expects an error if an initializer is in a control-flow scope."""
 
@@ -939,6 +957,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(self.evaluate(v.assign_add(1)), [1, 2, 3, 4])
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testCopyToGraphUninitialized(self):
     v = resource_variable_ops.ResourceVariable([0, 1, 2, 3])
     copy_to_graph = ops.Graph()
diff --git a/tensorflow/python/kernel_tests/reverse_sequence_op_test.py b/tensorflow/python/kernel_tests/reverse_sequence_op_test.py
index 91d054ad9a..05307c9834 100644
--- a/tensorflow/python/kernel_tests/reverse_sequence_op_test.py
+++ b/tensorflow/python/kernel_tests/reverse_sequence_op_test.py
@@ -23,6 +23,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.platform import test
@@ -107,6 +108,7 @@ class ReverseSequenceTest(test.TestCase):
   def testComplex128Basic(self):
     self._testBasic(np.complex128)
 
+  @test_util.run_deprecated_v1
   def testFloatReverseSequenceGrad(self):
     x = np.asarray(
         [[[1, 2, 3, 4], [5, 6, 7, 8]], [[9, 10, 11, 12], [13, 14, 15, 16]],
@@ -133,6 +135,7 @@ class ReverseSequenceTest(test.TestCase):
     print("ReverseSequence gradient error = %g" % err)
     self.assertLess(err, 1e-8)
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionEdgeCases(self):
     t = array_ops.reverse_sequence(
         array_ops.placeholder(
diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py
index 0090b7332f..3bc457f8fb 100644
--- a/tensorflow/python/kernel_tests/rnn_test.py
+++ b/tensorflow/python/kernel_tests/rnn_test.py
@@ -262,6 +262,7 @@ class RNNTest(test.TestCase):
       rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32, sequence_length=[4])
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testTensorArrayStateIsAccepted(self):
     cell = TensorArrayStateRNNCell()
     in_eager_mode = context.executing_eagerly()
@@ -285,6 +286,7 @@ class RNNTest(test.TestCase):
     self.assertAllEqual(4, state[0])
     self.assertAllEqual([[[1]], [[2]], [[3]], [[4]]], state[1])
 
+  @test_util.run_deprecated_v1
   def testCellGetInitialState(self):
     cell = rnn_cell_impl.BasicRNNCell(5)
     with self.assertRaisesRegexp(
@@ -345,6 +347,7 @@ class RNNTest(test.TestCase):
     self._assert_cell_builds(contrib_rnn.IndyLSTMCell, f32, 5, 7, 3)
     self._assert_cell_builds(contrib_rnn.IndyLSTMCell, f64, 5, 7, 3)
 
+  @test_util.run_deprecated_v1
   def testRNNWithKerasSimpleRNNCell(self):
     with self.cached_session() as sess:
       input_shape = 10
@@ -378,6 +381,7 @@ class RNNTest(test.TestCase):
       self.assertEqual(len(outputs), batch)
       self.assertEqual(len(state), batch)
 
+  @test_util.run_deprecated_v1
   def testRNNWithKerasGRUCell(self):
     with self.cached_session() as sess:
       input_shape = 10
@@ -411,6 +415,7 @@ class RNNTest(test.TestCase):
       self.assertEqual(len(outputs), batch)
       self.assertEqual(len(state), batch)
 
+  @test_util.run_deprecated_v1
   def testRNNWithKerasLSTMCell(self):
     with self.cached_session() as sess:
       input_shape = 10
@@ -448,6 +453,7 @@ class RNNTest(test.TestCase):
       self.assertEqual(len(state[0]), batch)
       self.assertEqual(len(state[1]), batch)
 
+  @test_util.run_deprecated_v1
   def testRNNWithStackKerasCell(self):
     with self.cached_session() as sess:
       input_shape = 10
@@ -491,6 +497,7 @@ class RNNTest(test.TestCase):
       for s in state:
         self.assertEqual(len(s), batch)
 
+  @test_util.run_deprecated_v1
   def testStaticRNNWithKerasSimpleRNNCell(self):
     with self.cached_session() as sess:
       input_shape = 10
@@ -529,6 +536,7 @@ class RNNTest(test.TestCase):
       self.assertEqual(len(outputs[0]), batch)
       self.assertEqual(len(state), batch)
 
+  @test_util.run_deprecated_v1
   def testKerasAndTFRNNLayerOutputComparison(self):
     input_shape = 10
     output_shape = 5
@@ -562,6 +570,7 @@ class RNNTest(test.TestCase):
     self.assertAllClose(tf_out, k_out)
     self.assertAllClose(tf_state, k_state)
 
+  @test_util.run_deprecated_v1
   def testSimpleRNNCellAndBasicRNNCellComparison(self):
     input_shape = 10
     output_shape = 5
@@ -601,6 +610,7 @@ class RNNTest(test.TestCase):
     self.assertAllClose(tf_out, k_out, atol=1e-5)
     self.assertAllClose(tf_state, k_state, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testBasicLSTMCellInterchangeWithLSTMCell(self):
     with self.session(graph=ops_lib.Graph()) as sess:
       basic_cell = rnn_cell_impl.BasicLSTMCell(1)
diff --git a/tensorflow/python/kernel_tests/save_restore_ops_test.py b/tensorflow/python/kernel_tests/save_restore_ops_test.py
index be117c4350..fecc9a3800 100644
--- a/tensorflow/python/kernel_tests/save_restore_ops_test.py
+++ b/tensorflow/python/kernel_tests/save_restore_ops_test.py
@@ -55,6 +55,7 @@ class ShardedFileOpsTest(test.TestCase):
 
 class ShapeInferenceTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testRestoreV2WithSliceInput(self):
     op = io_ops.restore_v2("model", ["var1", "var2"], ["", "3 4 0,1:-"],
                            [dtypes.float32, dtypes.float32])
@@ -62,11 +63,13 @@ class ShapeInferenceTest(test.TestCase):
     self.assertFalse(op[0].get_shape().is_fully_defined())
     self.assertEqual([1, 4], op[1].get_shape())
 
+  @test_util.run_deprecated_v1
   def testRestoreV2NumSlicesNotMatch(self):
     with self.assertRaises(ValueError):
       io_ops.restore_v2("model", ["var1", "var2", "var3"], ["", "3 4 0,1:-"],
                         [dtypes.float32, dtypes.float32])
 
+  @test_util.run_deprecated_v1
   def testRestoreSlice(self):
     op = gen_io_ops.restore_slice("model", "var", "3 4 0,1:-", dtypes.float32)
     self.assertEqual([1, 4], op.get_shape())
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index c48e0e2e67..33e491fee1 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -88,12 +89,14 @@ class CumsumTest(test.TestCase):
       for reverse in [True, False]:
         self._compare(x, axis, exclusive, reverse)
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     for dtype in self.valid_dtypes:
       x = np.zeros([0]).astype(dtype)
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def testAxisType(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
@@ -102,30 +105,35 @@ class CumsumTest(test.TestCase):
           axis = constant_op.constant(0, axis_dtype)
           tf_out = math_ops.cumsum(x, axis).eval()
 
+  @test_util.run_deprecated_v1
   def test1D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def test2D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(0, 10).reshape([2, 5]).astype(dtype)
       for axis in (-2, -1, 0, 1):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def test3D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(0, 20).reshape([2, 2, 5]).astype(dtype)
       for axis in (-3, -2, -1, 0, 1, 2):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def test6D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 145).reshape([2, 2, 3, 3, 2, 2]).astype(dtype)
       for axis in range(-6, 6, 3):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def testLarge(self):
     for dtype in self.valid_dtypes:
       x = np.ones([1000000], dtype=dtype) / 1024
@@ -157,22 +165,27 @@ class CumsumTest(test.TestCase):
           t, shape, result, shape, x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     for axis in (-1, 0):
       self._compareGradient([50], axis, False, False)
 
+  @test_util.run_deprecated_v1
   def testGradientReverse(self):
     for axis in (-1, 0):
       self._compareGradient([50], axis, False, True)
 
+  @test_util.run_deprecated_v1
   def testGradientExclusive(self):
     for axis in (-1, 0):
       self._compareGradient([50], axis, True, False)
 
+  @test_util.run_deprecated_v1
   def testGradientExclusiveReverse(self):
     for axis in (-1, 0):
       self._compareGradient([50], axis, True, True)
 
+  @test_util.run_deprecated_v1
   def testGradient2D(self):
     for axis in (-1, 0, 1):
       for exclusive in [True, False]:
@@ -199,12 +212,14 @@ class CumprodTest(test.TestCase):
       for reverse in [True, False]:
         self._compare(x, axis, exclusive, reverse)
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     for dtype in self.valid_dtypes:
       x = np.zeros([0]).astype(dtype)
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def testAxisType(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
@@ -213,24 +228,28 @@ class CumprodTest(test.TestCase):
           axis = constant_op.constant(0, axis_dtype)
           tf_out = math_ops.cumprod(x, axis).eval()
 
+  @test_util.run_deprecated_v1
   def test1D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def test2D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 11).reshape([2, 5]).astype(dtype)
       for axis in (-2, -1, 0, 1):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def test3D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 21).reshape([2, 2, 5]).astype(dtype)
       for axis in (-3, -2, -1, 0, 1, 2):
         self._compareAll(x, axis)
 
+  @test_util.run_deprecated_v1
   def test6D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 145).reshape([2, 2, 3, 3, 2, 2]).astype(dtype)
@@ -263,22 +282,27 @@ class CumprodTest(test.TestCase):
           t, shape, result, shape, x_init_value=x, delta=1)
     self.assertAllClose(jacob_t, jacob_n, rtol=1e-8, atol=1e-8)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     for axis in (-1, 0):
       self._compareGradient([8], axis, False, False)
 
+  @test_util.run_deprecated_v1
   def testGradientReverse(self):
     for axis in (-1, 0):
       self._compareGradient([8], axis, False, True)
 
+  @test_util.run_deprecated_v1
   def testGradientExclusive(self):
     for axis in (-1, 0):
       self._compareGradient([8], axis, True, False)
 
+  @test_util.run_deprecated_v1
   def testGradientExclusiveReverse(self):
     for axis in (-1, 0):
       self._compareGradient([8], axis, True, True)
 
+  @test_util.run_deprecated_v1
   def testGradient2D(self):
     for axis in (-2, -1, 0, 1):
       for exclusive in [True, False]:
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index c388121982..298db1547a 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -165,6 +165,7 @@ class StatefulScatterNdTest(test.TestCase):
       result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
+  @test_util.run_deprecated_v1
   def testSimpleResource(self):
     indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32)
     updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32)
@@ -207,12 +208,15 @@ class StatefulScatterNdTest(test.TestCase):
       result = self.evaluate(scatter)
       self.assertAllClose(result, expected)
 
+  @test_util.run_deprecated_v1
   def testVariableRankUpdate(self):
     self._VariableRankTests(_NumpyUpdate, state_ops.scatter_nd_update)
 
+  @test_util.run_deprecated_v1
   def testVariableRankAdd(self):
     self._VariableRankTests(_NumpyAdd, state_ops.scatter_nd_add)
 
+  @test_util.run_deprecated_v1
   def testVariableRankSub(self):
     self._VariableRankTests(_NumpySub, state_ops.scatter_nd_sub)
 
@@ -230,6 +234,7 @@ class StatefulScatterNdTest(test.TestCase):
         self._VariableRankTest(
             np_scatter, tf_scatter, vtype, itype, repeat_indices=True)
 
+  @test_util.run_deprecated_v1
   def testScatterRepeatIndices(self):
     """This tests scatter_add using indices that repeat."""
     self._ScatterRepeatIndicesTest(_NumpyAdd, state_ops.scatter_nd_add)
@@ -251,6 +256,7 @@ class StatefulScatterNdTest(test.TestCase):
   #     session.run([update0, update1])
   #     self.assertAllEqual([False, True], self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testScatterOutOfRangeCpu(self):
     # TODO(simister): Re-enable once binary size increase due to
     # scatter_nd ops is under control.
@@ -287,6 +293,7 @@ class StatefulScatterNdTest(test.TestCase):
         state_ops.scatter_nd_update(ref, indices,
                                     updates).get_shape().as_list(), shape)
 
+  @test_util.run_deprecated_v1
   def testResVarInvalidOutputShape(self):
     res = variables.Variable(
         initial_value=lambda: array_ops.zeros(shape=[], dtype=dtypes.float32),
@@ -296,6 +303,7 @@ class StatefulScatterNdTest(test.TestCase):
       with self.assertRaisesOpError("Output must be at least 1-D"):
         state_ops.scatter_nd_update(res, [[0]], [0.22]).eval()
 
+  @test_util.run_deprecated_v1
   def testExtraIndicesDimensions(self):
     indices = array_ops.zeros([1, 1, 2], dtypes.int32)
     updates = array_ops.zeros([1, 1], dtypes.int32)
@@ -309,6 +317,7 @@ class StatefulScatterNdTest(test.TestCase):
       ref.initializer.run()
       self.assertAllEqual(expected_result, self.evaluate(scatter_update))
 
+  @test_util.run_deprecated_v1
   def testRank3InvalidShape1(self):
     indices = array_ops.zeros([3, 2, 2], dtypes.int32)
     updates = array_ops.zeros([2, 2, 2], dtypes.int32)
@@ -318,6 +327,7 @@ class StatefulScatterNdTest(test.TestCase):
         ValueError, "The outer \\d+ dimensions of indices\\.shape="):
       state_ops.scatter_nd_update(ref, indices, updates)
 
+  @test_util.run_deprecated_v1
   def testRank3InvalidShape2(self):
     indices = array_ops.zeros([2, 2, 1], dtypes.int32)
     updates = array_ops.zeros([2, 2], dtypes.int32)
@@ -327,6 +337,7 @@ class StatefulScatterNdTest(test.TestCase):
         ValueError, "The inner \\d+ dimensions of input\\.shape="):
       state_ops.scatter_nd_update(ref, indices, updates)
 
+  @test_util.run_deprecated_v1
   def testConcurrentUpdates(self):
     num_updates = 10000
     update_values = np.random.rand(num_updates)
@@ -455,6 +466,7 @@ class ScatterNdTest(test.TestCase):
     self.assertAllEqual(
         self.scatter_nd(indices, updates, shape).get_shape().as_list(), shape)
 
+  @test_util.run_deprecated_v1
   def testExtraIndicesDimensions(self):
     indices = array_ops.zeros([1, 1, 2], dtypes.int32)
     updates = array_ops.zeros([1, 1], dtypes.int32)
@@ -465,24 +477,28 @@ class ScatterNdTest(test.TestCase):
     with self.cached_session():
       self.assertAllEqual(expected_result, self.evaluate(scatter))
 
+  @test_util.run_deprecated_v1
   def testUndefinedIndicesShape(self):
     indices = array_ops.placeholder(dtypes.int32, shape=None)
     updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2])
     shape = constant_op.constant([2, 2, 2], dtypes.int32)
     self.scatter_nd(indices, updates, shape)
 
+  @test_util.run_deprecated_v1
   def testUndefinedUpdatesShape(self):
     indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2])
     updates = array_ops.placeholder(dtypes.int32, shape=None)
     shape = constant_op.constant([2, 2, 2], dtypes.int32)
     self.scatter_nd(indices, updates, shape)
 
+  @test_util.run_deprecated_v1
   def testUndefinedOutputShape(self):
     indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2])
     updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2])
     shape = array_ops.placeholder(dtypes.int32, shape=[None])
     self.scatter_nd(indices, updates, shape)
 
+  @test_util.run_deprecated_v1
   def testEmptyOutputShape1(self):
     indices = array_ops.zeros([2, 2, 2], dtypes.int32)
     updates = array_ops.zeros([2, 2, 2], dtypes.int32)
@@ -492,6 +508,7 @@ class ScatterNdTest(test.TestCase):
         ValueError, "Indices and updates specified for empty output shape"):
       self.scatter_nd(indices, updates, shape)
 
+  @test_util.run_deprecated_v1
   def testEmptyOutputShape2(self):
     indices = array_ops.placeholder(dtypes.int32, shape=None)
     updates = array_ops.placeholder(dtypes.int32, shape=None)
@@ -505,6 +522,7 @@ class ScatterNdTest(test.TestCase):
             updates: np.zeros([2, 2, 2], dtype=np.int32)
         })
 
+  @test_util.run_deprecated_v1
   def testEmptyOutputShape3(self):
     indices = array_ops.zeros([0], dtypes.int32)
     updates = array_ops.zeros([0], dtypes.int32)
@@ -514,6 +532,7 @@ class ScatterNdTest(test.TestCase):
     with self.cached_session():
       self.assertEqual(scatter.eval().size, 0)
 
+  @test_util.run_deprecated_v1
   def testRank3InvalidShape1(self):
     indices = array_ops.zeros([3, 2, 2], dtypes.int32)
     updates = array_ops.zeros([2, 2, 2], dtypes.int32)
@@ -522,6 +541,7 @@ class ScatterNdTest(test.TestCase):
         ValueError, "The outer \\d+ dimensions of indices\\.shape="):
       self.scatter_nd(indices, updates, shape)
 
+  @test_util.run_deprecated_v1
   def testRank3InvalidShape2(self):
     indices = array_ops.zeros([2, 2, 1], dtypes.int32)
     updates = array_ops.zeros([2, 2], dtypes.int32)
@@ -530,6 +550,7 @@ class ScatterNdTest(test.TestCase):
         ValueError, "The inner \\d+ dimensions of (input|output)\\.shape="):
       self.scatter_nd(indices, updates, shape)
 
+  @test_util.run_deprecated_v1
   def testGradientsRank2ElementUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
       indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32)
@@ -549,6 +570,7 @@ class ScatterNdTest(test.TestCase):
         if self.non_aliasing_add_test:
           self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
+  @test_util.run_deprecated_v1
   def testGradientsRank2SliceUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
       indices = constant_op.constant([[1], [0]], dtype=dtypes.int32)
@@ -569,6 +591,7 @@ class ScatterNdTest(test.TestCase):
         if self.non_aliasing_add_test:
           self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
+  @test_util.run_deprecated_v1
   def testGradientsRank3SliceUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
       indices = constant_op.constant([[[0, 1], [1, 0]], [[0, 0], [1, 1]]],
@@ -592,6 +615,7 @@ class ScatterNdTest(test.TestCase):
         if self.non_aliasing_add_test:
           self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
+  @test_util.run_deprecated_v1
   def testGradientsRank7SliceUpdate(self):
     for dtype in GRADIENT_TESTS_DTYPES:
       indices = constant_op.constant(
@@ -619,6 +643,7 @@ class ScatterNdTest(test.TestCase):
         if self.non_aliasing_add_test:
           self.assertAllEqual(expected_input_grad, self.evaluate(input_grad))
 
+  @test_util.run_deprecated_v1
   def testScatterNdRepatedIndicesAdd(self):
     indices = array_ops.zeros([100000, 1], dtypes.int32)
     values = np.random.randn(100000)
@@ -627,6 +652,7 @@ class ScatterNdTest(test.TestCase):
       val = self.scatter_nd(indices, values, shape).eval()
     self.assertAllClose([np.sum(values)], val)
 
+  @test_util.run_deprecated_v1
   def testSmokeScatterNdBatch2DSliceDim2(self):
     with self.cached_session():
       indices = array_ops.zeros([3, 5, 2], dtype=dtypes.int32)
@@ -634,6 +660,7 @@ class ScatterNdTest(test.TestCase):
       shape = [4, 6, 7]
       self.scatter_nd(indices, values, shape).eval()
 
+  @test_util.run_deprecated_v1
   def testSmokeScatterNdBatch1DSliceDim2(self):
     with self.cached_session():
       indices = array_ops.zeros([0, 2], dtype=dtypes.int32)
@@ -641,6 +668,7 @@ class ScatterNdTest(test.TestCase):
       shape = [4, 6, 7]
       self.scatter_nd(indices, values, shape).eval()
 
+  @test_util.run_deprecated_v1
   def testSmokeScatterNdBatch1DSliceDim3ShapeRank7(self):
     with self.cached_session():
       indices = array_ops.zeros([1, 3], dtype=dtypes.int32)
@@ -648,6 +676,7 @@ class ScatterNdTest(test.TestCase):
       shape = [3, 4, 5, 6, 7, 8, 9]
       self.scatter_nd(indices, values, shape).eval()
 
+  @test_util.run_deprecated_v1
   def testSmokeScatterNdBatch2DSliceDim3ShapeRank7(self):
     with self.cached_session():
       indices = array_ops.zeros([1, 2, 3], dtype=dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/scatter_ops_test.py b/tensorflow/python/kernel_tests/scatter_ops_test.py
index a4daad7adc..623c17d373 100644
--- a/tensorflow/python/kernel_tests/scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_ops_test.py
@@ -197,84 +197,111 @@ class ScatterTest(test.TestCase):
         self._VariableRankTest(tf_scatter, vtype, itype, repeat_indices,
                                updates_are_scalar)
 
+  @test_util.run_deprecated_v1
   def testVariableRankUpdate(self):
     self._VariableRankTests(state_ops.scatter_update, False)
 
+  @test_util.run_deprecated_v1
   def testVariableRankAdd(self):
     self._VariableRankTests(state_ops.scatter_add, False)
 
+  @test_util.run_deprecated_v1
   def testVariableRankSub(self):
     self._VariableRankTests(state_ops.scatter_sub, False)
 
+  @test_util.run_deprecated_v1
   def testVariableRankMul(self):
     self._VariableRankTests(state_ops.scatter_mul, False)
 
+  @test_util.run_deprecated_v1
   def testVariableRankDiv(self):
     self._VariableRankTests(state_ops.scatter_div, False)
 
+  @test_util.run_deprecated_v1
   def testVariableRankMin(self):
     self._VariableRankTests(state_ops.scatter_min, False)
 
+  @test_util.run_deprecated_v1
   def testVariableRankMax(self):
     self._VariableRankTests(state_ops.scatter_max, False)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesAdd(self):
     self._VariableRankTests(state_ops.scatter_add, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesSub(self):
     self._VariableRankTests(state_ops.scatter_sub, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesMul(self):
     self._VariableRankTests(state_ops.scatter_mul, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesDiv(self):
     self._VariableRankTests(state_ops.scatter_div, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesMin(self):
     self._VariableRankTests(state_ops.scatter_min, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesMax(self):
     self._VariableRankTests(state_ops.scatter_max, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankUpdateScalar(self):
     self._VariableRankTests(state_ops.scatter_update, False, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankAddScalar(self):
     self._VariableRankTests(state_ops.scatter_add, False, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankSubScalar(self):
     self._VariableRankTests(state_ops.scatter_sub, False, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankMulScalar(self):
     self._VariableRankTests(state_ops.scatter_mul, False, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankDivScalar(self):
     self._VariableRankTests(state_ops.scatter_div, False, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankMinScalar(self):
     self._VariableRankTests(state_ops.scatter_min, False, True)
 
+  @test_util.run_deprecated_v1
   def testVariableRankMaxScalar(self):
     self._VariableRankTests(state_ops.scatter_max, False, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesAddScalar(self):
     self._VariableRankTests(state_ops.scatter_add, True, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesSubScalar(self):
     self._VariableRankTests(state_ops.scatter_sub, True, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesMulScalar(self):
     self._VariableRankTests(state_ops.scatter_mul, True, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesDivScalar(self):
     self._VariableRankTests(state_ops.scatter_div, True, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesMinScalar(self):
     self._VariableRankTests(state_ops.scatter_min, True, True)
 
+  @test_util.run_deprecated_v1
   def testRepeatIndicesMaxScalar(self):
     self._VariableRankTests(state_ops.scatter_max, True, True)
 
+  @test_util.run_deprecated_v1
   def testBooleanScatterUpdate(self):
     if not test.is_gpu_available():
       with self.session(use_gpu=False) as session:
@@ -289,6 +316,7 @@ class ScatterTest(test.TestCase):
 
         self.assertAllEqual([False, True], self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testScatterOutOfRangeCpu(self):
     for op, _ in _TF_OPS_TO_NUMPY.items():
       params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32)
diff --git a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py
index 5ab889895e..8af1b47e83 100644
--- a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
@@ -126,6 +127,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
             # and may therefore vary dynamically.
             self.assertAllEqual(np_ans.shape[1:], tf_ans.shape[1:])
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsShape(self):
     shape = [4, 4]
     tf_x, _ = self._input(shape)
@@ -133,6 +135,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
     with self.assertRaises(ValueError):
       math_ops.segment_sum(data=tf_x, segment_ids=indices)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsSize(self):
     shape = [4, 4]
     for use_gpu in [True, False]:
@@ -143,6 +146,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         with self.assertRaisesOpError("segment_ids should be the same size"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsValid(self):
     # This is a baseline for the following SegmentIdsInvalid* tests.
     shape = [4, 4]
@@ -175,6 +179,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         tf_ans = self.evaluate(s)
         self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsInvalid1(self):
     shape = [4, 4]
     with self.cached_session():
@@ -186,6 +191,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
           "'segment_ids' input is not sorted."):
         self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsInvalid2(self):
     shape = [4, 4]
     with self.cached_session():
@@ -195,6 +201,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
       with self.assertRaisesOpError("segment ids are not increasing"):
         self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsInvalid3(self):
     shape = [4, 4]
     with self.cached_session():
@@ -206,6 +213,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
           "because 'segment_ids' input is not sorted."):
         self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsInvalid4(self):
     shape = [4, 4]
     for use_gpu in [True, False]:
@@ -216,6 +224,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         with self.assertRaisesOpError("segment ids must be >= 0"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentIdsInvalid5(self):
     shape = [4, 4]
     for use_gpu in [True, False]:
@@ -226,6 +235,7 @@ class SegmentReductionOpTest(SegmentReductionHelper):
         with self.assertRaisesOpError("segment ids must be >= 0"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     shape = [4, 4]
     indices = [0, 1, 2, 2]
@@ -324,6 +334,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
         self.assertAllClose(np_ans, tf_ans)
         self.assertShapeEqual(np_ans, s)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     num_cols = 2
     indices_flat = np.array([0, 4, 0, -1, 3, -1, 4, 7, 7, 3])
@@ -346,6 +357,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
                   delta=1)
             self.assertAllClose(jacob_t, jacob_n)
 
+  @test_util.run_deprecated_v1
   def testProdGrad(self):
     # additional test for the prod gradient to ensure correct handling of zeros
     values = np.array([0, 0, 1, 0, 2, 2, 3, 3, 3], dtype=np.float32)
@@ -370,6 +382,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
           self.assertAllClose(jacob_t, jacob_n)
           self.assertAllClose(jacob_t, grad_gt)
 
+  @test_util.run_deprecated_v1
   def testGradientMatchesSegmentSum(self):
     # Strategy: compute the gradient for UnsortedSegmentSum and SegmentSum
     # and compare the outputs, which should be identical.
@@ -403,6 +416,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
       self.assertAllClose(unsorted_jacob_t, sorted_jacob_t)
       self.assertAllClose(unsorted_jacob_n, sorted_jacob_n)
 
+  @test_util.run_deprecated_v1
   def testBadIndices(self):
     # Note: GPU kernel does not return the out-of-range error needed for this
     # test, so this test is marked as cpu-only.
@@ -414,6 +428,7 @@ class UnsortedSegmentTest(SegmentReductionHelper):
             r"segment_ids\[0,0\] = %d is out of range \[0, 2\)" % bad[0][0]):
           self.evaluate(unsorted)
 
+  @test_util.run_deprecated_v1
   def testEmptySecondDimension(self):
     dtypes = [np.float16, np.float32, np.float64, np.int64, np.int32,
               np.complex64, np.complex128]
@@ -590,6 +605,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         s = tf_op(data=tf_x, indices=tf_indices, segment_ids=segment_indices)
         self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testIndicesInvalid1(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -602,6 +618,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             r"indices\[1\] == -1 out of range \[0, 10\)"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testIndicesInvalid2(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -614,6 +631,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             r"indices\[3\] == 10 out of range \[0, 10\)"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentsInvalid2(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -625,6 +643,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError("segment ids are not increasing"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentsInvalid3(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -638,6 +657,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             "'segment_ids' input is not sorted"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentsInvalid4(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -651,6 +671,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             "'segment_ids' input is not sorted"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentsInvalid6(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -662,6 +683,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError("segment ids must be >= 0"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentsInvalid7(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [math_ops.sparse_segment_sum, math_ops.sparse_segment_mean]
@@ -692,6 +714,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             num_segments=num_segments)
         self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentWithNumSegmentsInvalid1(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [
@@ -711,6 +734,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError("segment ids must be < num_segments"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testSegmentWithNumSegmentsInvalid2(self):
     tf_x, _ = self._input([10, 4], dtype=dtypes_lib.float32)
     ops_list = [
@@ -730,6 +754,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
               segment_ids=segment_indices,
               num_segments=num_segments)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     shape = [10, 4]
 
@@ -748,6 +773,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
             delta=1)
       self.assertAllClose(jacob_t, jacob_n)
 
+  @test_util.run_deprecated_v1
   def testGradientWithEmptySegmentsAtEnd(self):
     shape = [10, 4]
 
@@ -787,6 +813,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         s = tf_op(tf_x, tf_indices, segment_indices, 10)
         self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradientIndicesInvalid1(self):
     tf_x, _ = self._input([3, 4], dtype=dtypes_lib.float32)
     ops_list = [
@@ -800,6 +827,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError(r"Index 10 out of range \[0, 10\)"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradientIndicesInvalid2(self):
     tf_x, _ = self._input([3, 4], dtype=dtypes_lib.float32)
     ops_list = [
@@ -813,6 +841,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError(r"Index -1 out of range \[0, 10\)"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradientSegmentsInvalid1(self):
     tf_x, _ = self._input(
         [3, 4], dtype=dtypes_lib.float32)  # expecting 3 segments
@@ -827,6 +856,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError("Invalid number of segments"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradientSegmentsInvalid2(self):
     tf_x, _ = self._input([1, 4], dtype=dtypes_lib.float32)
     ops_list = [
@@ -840,6 +870,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError(r"Segment id 1 out of range \[0, 1\)"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradientSegmentsInvalid3(self):
     tf_x, _ = self._input([2, 4], dtype=dtypes_lib.float32)
     ops_list = [
@@ -853,6 +884,7 @@ class SparseSegmentReductionOpTest(SparseSegmentReductionHelper):
         with self.assertRaisesOpError(r"Segment id -1 out of range \[0, 2\)"):
           self.evaluate(s)
 
+  @test_util.run_deprecated_v1
   def testGradientSegmentsInvalid4(self):
     tf_x, _ = self._input([0, 4], dtype=dtypes_lib.float32)
     ops_list = [
diff --git a/tensorflow/python/kernel_tests/sets_test.py b/tensorflow/python/kernel_tests/sets_test.py
index ba3d32b192..b4f2322934 100644
--- a/tensorflow/python/kernel_tests/sets_test.py
+++ b/tensorflow/python/kernel_tests/sets_test.py
@@ -70,6 +70,7 @@ def _dense_to_sparse(dense, dtype):
 
 class SetOpsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def test_set_size_2d(self):
     for dtype in _DTYPES:
       self._test_set_size_2d(dtype)
@@ -83,6 +84,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(
         [0, 3], self._set_size(_dense_to_sparse([[], [1, 9, 2]], dtype)))
 
+  @test_util.run_deprecated_v1
   def test_set_size_duplicates_2d(self):
     for dtype in _DTYPES:
       self._test_set_size_duplicates_2d(dtype)
@@ -96,6 +98,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
                                 6, 7, 8, 8, 6, 7, 5, 3, 3, 0, 6, 6, 9, 0, 0, 0
                             ], [999, 1, -1000], [], [-1]], dtype)))
 
+  @test_util.run_deprecated_v1
   def test_set_size_3d(self):
     for dtype in _DTYPES:
       self._test_set_size_3d(dtype)
@@ -163,6 +166,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(results[0], results[1])
     return results[0]
 
+  @test_util.run_deprecated_v1
   def test_set_intersection_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_set_intersection_multirow_2d(dtype)
@@ -199,6 +203,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_counts,
                         self._set_intersection_count(sp_a, sp_b))
 
+  @test_util.run_deprecated_v1
   def test_dense_set_intersection_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_dense_set_intersection_multirow_2d(dtype)
@@ -223,6 +228,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
         dtype=dtype)
     self.assertAllEqual(expected_counts, self._set_intersection_count(a, b))
 
+  @test_util.run_deprecated_v1
   def test_set_intersection_duplicates_2d(self):
     for dtype in _DTYPES:
       self._test_set_intersection_duplicates_2d(dtype)
@@ -270,6 +276,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_counts,
                         self._set_intersection_count(sp_a, sp_b))
 
+  @test_util.run_deprecated_v1
   def test_set_intersection_3d(self):
     for dtype in _DTYPES:
       self._test_set_intersection_3d(dtype=dtype)
@@ -536,6 +543,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     with self.cached_session() as sess:
       return self.evaluate(op)
 
+  @test_util.run_deprecated_v1
   def test_set_difference_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_set_difference_multirow_2d(dtype)
@@ -604,6 +612,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_counts,
                         self._set_difference_count(sp_a, sp_b, False))
 
+  @test_util.run_deprecated_v1
   def test_dense_set_difference_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_dense_set_difference_multirow_2d(dtype)
@@ -647,6 +656,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_counts,
                         self._set_difference_count(a, b, False))
 
+  @test_util.run_deprecated_v1
   def test_sparse_set_difference_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_sparse_set_difference_multirow_2d(dtype)
@@ -688,6 +698,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_counts,
                         self._set_difference_count(sp_a, sp_b, False))
 
+  @test_util.run_deprecated_v1
   def test_set_difference_duplicates_2d(self):
     for dtype in _DTYPES:
       self._test_set_difference_duplicates_2d(dtype)
@@ -755,6 +766,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(expected_counts,
                         self._set_difference_count(a, sp_b, False))
 
+  @test_util.run_deprecated_v1
   def test_sparse_set_difference_3d(self):
     for dtype in _DTYPES:
       self._test_sparse_set_difference_3d(dtype)
@@ -974,6 +986,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
     with self.cached_session() as sess:
       return self.evaluate(op)
 
+  @test_util.run_deprecated_v1
   def test_set_union_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_set_union_multirow_2d(dtype)
@@ -1001,6 +1014,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
         expected_indices, expected_values, expected_shape, union, dtype=dtype)
     self.assertAllEqual(expected_counts, self._set_union_count(sp_a, sp_b))
 
+  @test_util.run_deprecated_v1
   def test_dense_set_union_multirow_2d(self):
     for dtype in _DTYPES:
       self._test_dense_set_union_multirow_2d(dtype)
@@ -1021,6 +1035,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
         expected_indices, expected_values, expected_shape, union, dtype=dtype)
     self.assertAllEqual(expected_counts, self._set_union_count(a, b))
 
+  @test_util.run_deprecated_v1
   def test_set_union_duplicates_2d(self):
     for dtype in _DTYPES:
       self._test_set_union_duplicates_2d(dtype)
@@ -1047,6 +1062,7 @@ class SetOpsTest(test_util.TensorFlowTestCase):
         expected_indices, expected_values, expected_shape, union, dtype=dtype)
     self.assertAllEqual([2], self._set_union_count(sp_a, sp_b))
 
+  @test_util.run_deprecated_v1
   def test_sparse_set_union_3d(self):
     for dtype in _DTYPES:
       self._test_sparse_set_union_3d(dtype)
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index a0506fbfc5..c8e7c143ad 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -227,6 +228,7 @@ class ShapeOpsTest(test.TestCase):
     self._compareExpandDimsAll(choice([2, 3, 5]), -3)
     self._compareExpandDimsAll(choice([2, 3, 5]), -4)
 
+  @test_util.run_deprecated_v1
   def testExpandDimsErrors(self):
     with self.cached_session():
       self.assertRaises(ValueError, array_ops.expand_dims,
@@ -238,6 +240,7 @@ class ShapeOpsTest(test.TestCase):
       self.assertRaises(ValueError, array_ops.expand_dims,
                         [False, True, True], 4)
 
+  @test_util.run_deprecated_v1
   def testExpandDimsGradient(self):
     with self.cached_session():
       inp = constant_op.constant(
@@ -248,6 +251,7 @@ class ShapeOpsTest(test.TestCase):
                                                     [4, 1, 2])
     self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testExpandDimsScalar(self):
     with self.cached_session():
       inp = constant_op.constant(7)
@@ -353,6 +357,7 @@ class ShapeOpsTest(test.TestCase):
         tf_ans = self.evaluate(tensor)
         self.assertEqual(np.shape(1), tf_ans.shape)
 
+  @test_util.run_deprecated_v1
   def testSqueezeOnlyOnes(self):
     for use_gpu in [False, True]:
       with self.cached_session(use_gpu=use_gpu):
@@ -362,6 +367,7 @@ class ShapeOpsTest(test.TestCase):
         self._compareSqueezeAll(input_1x1x3, [1])
         self.assertRaises(ValueError, array_ops.squeeze, input_1x1x3, [2])
 
+  @test_util.run_deprecated_v1
   def testSqueezeErrors(self):
     for use_gpu in [False, True]:
       with self.cached_session(use_gpu=use_gpu):
@@ -374,6 +380,7 @@ class ShapeOpsTest(test.TestCase):
         self.assertRaises(ValueError, array_ops.squeeze,
                           np.zeros([1, 2, 1]), [2, 3])
 
+  @test_util.run_deprecated_v1
   def testSqueezeGradient(self):
     with self.cached_session():
       inp = np.random.rand(4, 2).astype("f")
@@ -384,6 +391,7 @@ class ShapeOpsTest(test.TestCase):
                                                     [4, 2])
     self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testSqueezeGradientWithSqueezeDims(self):
     with self.cached_session():
       inp = np.random.rand(4, 2).astype("f")
@@ -394,6 +402,7 @@ class ShapeOpsTest(test.TestCase):
                                                     [4, 2, 1])
     self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testSqueezeWithUnknownShape(self):
     with self.cached_session():
       a = array_ops.placeholder(dtypes.float32, shape=[2, None])
@@ -451,6 +460,7 @@ class TileTest(test.TestCase):
     self.assertEqual(result.shape, (10, 0))
     self.assertEqual([10, 0], tiled.get_shape())
 
+  @test_util.run_deprecated_v1
   def testUnknownInputShape(self):
     """Importing can call _TileShape without shape of <multiples> known."""
     with self.cached_session():
@@ -502,6 +512,7 @@ class TileTest(test.TestCase):
       self.assertEqual([4, 4], tiled.get_shape())
       self.assertAllEqual(result, np.tile(inp, (1, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidDim(self):
     with self.cached_session():
       inp = np.random.rand(4, 1).astype("f")
@@ -545,6 +556,7 @@ class TileTest(test.TestCase):
     for _ in range(5):
       self._RunAndVerifyResult(10, use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testGradientSimpleReduction(self):
     with self.cached_session():
       inp = np.random.rand(4, 1).astype("f")
@@ -560,6 +572,7 @@ class TileTest(test.TestCase):
       result = self.evaluate(grad)
     self.assertAllClose(np.sum(grad_inp, axis=1).reshape(4, 1), result, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradientStridedReduction(self):
     with self.cached_session():
       inp = np.random.rand(4, 2).astype("f")
@@ -579,6 +592,7 @@ class TileTest(test.TestCase):
     expected[:, 1] = grad_inp[:, 1] + grad_inp[:, 3]
     self.assertTrue((np.abs(expected - result) < 1e-3).all())
 
+  @test_util.run_deprecated_v1
   def testGradientSimpleReductionOnGPU(self):
     with self.session(use_gpu=True):
       inp = np.random.rand(4, 1).astype("f")
@@ -593,6 +607,7 @@ class TileTest(test.TestCase):
       result = self.evaluate(grad)
     self.assertAllClose(np.sum(grad_inp, axis=1).reshape(4, 1), result, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradientStridedReductionOnGPU(self):
     with self.session(use_gpu=True):
       inp = np.random.rand(4, 2).astype("f")
@@ -624,15 +639,18 @@ class TileTest(test.TestCase):
       print("tile(float) error = ", err)
       self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradientRandomScalar(self):
     self._RunAndVerifyGradientResult([], [])
 
+  @test_util.run_deprecated_v1
   def testGradientRandom(self):
     self._RunAndVerifyGradientResult([2, 2, 1, 1, 3], [1, 1, 1, 1, 1])
     self._RunAndVerifyGradientResult([2, 2, 1, 1, 3], [1, 2, 1, 3, 1])
     self._RunAndVerifyGradientResult([2, 3, 1, 1, 3], [3, 1, 1, 2, 2])
     self._RunAndVerifyGradientResult([2, 1, 3, 3, 2], [1, 3, 3, 1, 2])
 
+  @test_util.run_deprecated_v1
   def testGradientStridedReductionGC(self):
     with self.cached_session():
       inp = np.random.rand(4, 2).astype("f")
@@ -642,6 +660,7 @@ class TileTest(test.TestCase):
       err = gradient_checker.compute_gradient_error(a, [4, 2], tiled, [4, 4])
     self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradientWithSparseGradWithRank1(self):
     inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0],
                                   dtype=dtypes.float32)
@@ -653,6 +672,7 @@ class TileTest(test.TestCase):
           outputs, outputs.get_shape().as_list())
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientWithSparseGradWithRank3(self):
     inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0],
                                   dtype=dtypes.float32)
@@ -665,6 +685,7 @@ class TileTest(test.TestCase):
           outputs, outputs.get_shape().as_list())
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionEdgeCases(self):
     # Unknown multiples shape.
     inp = constant_op.constant(0.0, shape=[4, 4, 4, 4])
diff --git a/tensorflow/python/kernel_tests/signal/dct_ops_test.py b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
index af4939332f..a3ac15bab8 100644
--- a/tensorflow/python/kernel_tests/signal/dct_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/dct_ops_test.py
@@ -23,6 +23,7 @@ import importlib
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import spectral_ops_test_util
 from tensorflow.python.ops.signal import dct_ops
 from tensorflow.python.platform import test
@@ -132,6 +133,7 @@ class DCTOpsTest(parameterized.TestCase, test.TestCase):
 
   @parameterized.parameters([
       [[2]], [[3]], [[10]], [[2, 20]], [[2, 3, 25]]])
+  @test_util.run_deprecated_v1
   def test_random(self, shape):
     """Test randomly generated batches of data."""
     with spectral_ops_test_util.fft_kernel_label_map():
diff --git a/tensorflow/python/kernel_tests/signal/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
index 3eeecc12a8..5b1053428c 100644
--- a/tensorflow/python/kernel_tests/signal/fft_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
@@ -25,6 +25,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_spectral_ops
 from tensorflow.python.ops import gradient_checker
@@ -157,6 +158,7 @@ class FFTOpsTest(BaseFFTOpsTest):
     else:
       raise ValueError("invalid rank")
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type in (np.complex64, np.complex128):
@@ -166,6 +168,7 @@ class FFTOpsTest(BaseFFTOpsTest):
             self.assertEqual(x.shape, self._tfFFT(x, rank).shape)
             self.assertEqual(x.shape, self._tfIFFT(x, rank).shape)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 1e-8)):
@@ -194,6 +197,7 @@ class FFTOpsTest(BaseFFTOpsTest):
   #           np.mod(np.arange(np.power(128, dims)), 64).reshape(
   #               (128,) * dims).astype(np.complex64), rank)
 
+  @test_util.run_deprecated_v1
   def testBasicPlaceholder(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 1e-8)):
@@ -204,6 +208,7 @@ class FFTOpsTest(BaseFFTOpsTest):
                     (4,) * dims).astype(np_type),
                 rank, use_placeholder=True, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.complex64, 1e-4), (np.complex128, 5e-6)):
@@ -218,6 +223,7 @@ class FFTOpsTest(BaseFFTOpsTest):
             self._compare(gen((4,) * dims).astype(np_type), rank,
                           rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testRandom1D(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type in (np.complex64, np.complex128):
@@ -240,6 +246,7 @@ class FFTOpsTest(BaseFFTOpsTest):
         for dim in (127, 255, 511, 1023):
           self._compare(gen((dim,)).astype(np_type), 1, rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testError(self):
     for rank in VALID_FFT_RANKS:
       for dims in xrange(0, rank):
@@ -251,6 +258,7 @@ class FFTOpsTest(BaseFFTOpsTest):
             ValueError, "Shape must be .*rank {}.*".format(rank)):
           self._tfIFFT(x, rank)
 
+  @test_util.run_deprecated_v1
   def testGrad_Simple(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.float32, 1e-4), (np.float64, 1e-10)):
@@ -263,6 +271,7 @@ class FFTOpsTest(BaseFFTOpsTest):
             self._checkGradComplex(self._tfIFFTForRank(rank), re, im,
                                    rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testGrad_Random(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for np_type, tol in ((np.float32, 1e-2), (np.float64, 1e-10)):
@@ -330,6 +339,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
     else:
       raise ValueError("invalid rank")
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -339,6 +349,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
           x = np.zeros((0,) * dims).astype(np.complex64)
           self.assertEqual(x.shape, self._tfIFFT(x, rank).shape)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -366,6 +377,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                        10).reshape((size,) * (dims - 1) + (inner_dim,))
           self._compareBackward(c2r.astype(np.complex64), rank, (size,) * rank)
 
+  @test_util.run_deprecated_v1
   def testBasicPlaceholder(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -427,6 +439,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                   fft_length,
                   use_placeholder=True)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       def gen_real(shape):
@@ -451,6 +464,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
             self._compareBackward(
                 gen_complex(complex_dims), rank, (size,) * rank)
 
+  @test_util.run_deprecated_v1
   def testError(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -507,6 +521,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
           with self.cached_session():
             irfft_fn(x, fft_length).eval()
 
+  @test_util.run_deprecated_v1
   def testGrad_Simple(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
@@ -521,6 +536,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
             self._checkGradComplex(
                 self._tfIFFTForRank(rank), re, im, result_is_complex=False)
 
+  @test_util.run_deprecated_v1
   def testGrad_Random(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
diff --git a/tensorflow/python/kernel_tests/signal/mel_ops_test.py b/tensorflow/python/kernel_tests/signal/mel_ops_test.py
index 2b3dde30f3..3134503dae 100644
--- a/tensorflow/python/kernel_tests/signal/mel_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/mel_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.kernel_tests.signal import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.signal import mel_ops
@@ -143,12 +144,14 @@ class LinearToMelTest(test.TestCase):
         mel_matrix = mel_ops.linear_to_mel_weight_matrix(*config)
         self.assertAllClose(mel_matrix_np, self.evaluate(mel_matrix), atol=3e-6)
 
+  @tf_test_util.run_deprecated_v1
   def test_dtypes(self):
     # LinSpace is not supported for tf.float16.
     for dtype in (dtypes.bfloat16, dtypes.float32, dtypes.float64):
       self.assertEqual(dtype,
                        mel_ops.linear_to_mel_weight_matrix(dtype=dtype).dtype)
 
+  @tf_test_util.run_deprecated_v1
   def test_error(self):
     with self.assertRaises(ValueError):
       mel_ops.linear_to_mel_weight_matrix(num_mel_bins=0)
@@ -177,6 +180,7 @@ class LinearToMelTest(test.TestCase):
         rewritten_graph = test_util.grappler_optimize(g, [mel_matrix])
         self.assertEqual(1, len(rewritten_graph.node))
 
+  @tf_test_util.run_deprecated_v1
   def test_num_spectrogram_bins_dynamic(self):
     with self.session(use_gpu=True):
       num_spectrogram_bins = array_ops.placeholder(shape=(),
diff --git a/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py b/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py
index 79d23d77d1..935922657c 100644
--- a/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/mfcc_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import spectral_ops_test_util
@@ -32,6 +33,7 @@ from tensorflow.python.platform import test
 # HTK conventions.
 class MFCCTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_error(self):
     # num_mel_bins must be positive.
     with self.assertRaises(ValueError):
@@ -43,6 +45,7 @@ class MFCCTest(test.TestCase):
       signal = array_ops.zeros((2, 3, 5), dtype=dtypes.float64)
       mfcc_ops.mfccs_from_log_mel_spectrograms(signal)
 
+  @test_util.run_deprecated_v1
   def test_basic(self):
     """A basic test that the op runs on random input."""
     with spectral_ops_test_util.fft_kernel_label_map():
@@ -50,6 +53,7 @@ class MFCCTest(test.TestCase):
         signal = random_ops.random_normal((2, 3, 5))
         mfcc_ops.mfccs_from_log_mel_spectrograms(signal).eval()
 
+  @test_util.run_deprecated_v1
   def test_unknown_shape(self):
     """A test that the op runs when shape and rank are unknown."""
     with spectral_ops_test_util.fft_kernel_label_map():
diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index de3351e543..d9b45f67c3 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
@@ -62,6 +63,7 @@ class ReconstructionOpsTest(test.TestCase):
 
       self.assertAllClose(output, expected_output)
 
+  @test_util.run_deprecated_v1
   def test_simple(self):
     def make_input(frame_length, num_frames=3):
       """Generate a tensor of num_frames frames of frame_length."""
@@ -133,6 +135,7 @@ class ReconstructionOpsTest(test.TestCase):
       self.assertEqual(output.shape, (1, 9))
       self.assertEqual(string_output, self.expected_string)
 
+  @test_util.run_deprecated_v1
   def test_gradient(self):
     configurations = [
         ((1, 128), 1),
@@ -154,6 +157,7 @@ class ReconstructionOpsTest(test.TestCase):
         gradient = sess.run(gradients_impl.gradients([loss], [signal])[0])
         self.assertTrue((gradient == 1.0).all())
 
+  @test_util.run_deprecated_v1
   def test_gradient_batch(self):
     with self.session(use_gpu=True) as sess:
       signal = array_ops.zeros((2, 10, 10))
@@ -176,6 +180,7 @@ class ReconstructionOpsTest(test.TestCase):
           np.reshape(np.arange(100).astype(np.float32), (10, 10))])
       self.assertAllEqual(expected_gradient, gradient)
 
+  @test_util.run_deprecated_v1
   def test_gradient_numerical(self):
     with self.session(use_gpu=True):
       shape = (2, 10, 10)
diff --git a/tensorflow/python/kernel_tests/signal/shape_ops_test.py b/tensorflow/python/kernel_tests/signal/shape_ops_test.py
index 21a6b23b30..32ac76e80d 100644
--- a/tensorflow/python/kernel_tests/signal/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/shape_ops_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.kernel_tests.signal import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -32,6 +33,7 @@ from tensorflow.python.platform import test
 
 class FrameTest(test.TestCase):
 
+  @tf_test_util.run_deprecated_v1
   def test_mapping_of_indices_without_padding(self):
     with self.session(use_gpu=True):
       tensor = constant_op.constant(np.arange(9152), dtypes.int32)
@@ -47,6 +49,7 @@ class FrameTest(test.TestCase):
 
       self.assertAllEqual(expected, result)
 
+  @tf_test_util.run_deprecated_v1
   def test_mapping_of_indices_with_padding(self):
     with self.session(use_gpu=True):
       tensor = constant_op.constant(np.arange(10000), dtypes.int32)
@@ -64,6 +67,7 @@ class FrameTest(test.TestCase):
 
       self.assertAllEqual(expected, result)
 
+  @tf_test_util.run_deprecated_v1
   def test_invalid_inputs(self):
     # Rank 0 input signal.
     with self.assertRaises(ValueError):
@@ -84,6 +88,7 @@ class FrameTest(test.TestCase):
     with self.assertRaises(ValueError):
       shape_ops.frame([1], 1, 1, pad_end=True, pad_value=[1])
 
+  @tf_test_util.run_deprecated_v1
   def test_length_zero(self):
     signal = constant_op.constant([], dtype=dtypes.float32)
     frame_length = 2
@@ -98,6 +103,7 @@ class FrameTest(test.TestCase):
                                pad_end=False).eval()
       self.assertEqual((0, 2), result.shape)
 
+  @tf_test_util.run_deprecated_v1
   def test_shape_inference(self):
     signal = array_ops.placeholder(dtypes.int32, shape=[1, 1])
     frame_length = 2
@@ -153,6 +159,7 @@ class FrameTest(test.TestCase):
             result = self.evaluate(op)
           self.assertEqual(op.shape.as_list(), list(result.shape))
 
+  @tf_test_util.run_deprecated_v1
   def test_basic_mono(self):
     signal = np.arange(6)
     frame_length = 3
@@ -178,6 +185,7 @@ class FrameTest(test.TestCase):
                                  pad_end=False).eval()
         self.assertAllEqual(expected, result)
 
+  @tf_test_util.run_deprecated_v1
   def test_basic_stereo(self):
     signal = np.vstack([np.arange(6),
                         np.arange(6) + 10])
@@ -207,6 +215,7 @@ class FrameTest(test.TestCase):
                                  pad_end=False).eval()
         self.assertAllEqual(expected, result)
 
+  @tf_test_util.run_deprecated_v1
   def test_complex_shape(self):
     signal = np.vstack([np.arange(6),
                         np.arange(6) + 10,
@@ -274,6 +283,7 @@ class FrameTest(test.TestCase):
                    [[14, 15], [0, 0], [0, 0]]]]
       self.assertAllEqual(expected, self.evaluate(result))
 
+  @tf_test_util.run_deprecated_v1
   def test_window_larger_than_signal(self):
     signal = constant_op.constant([[1, 2], [11, 12]], dtype=dtypes.float32)
     frame_length = 4
@@ -307,6 +317,7 @@ class FrameTest(test.TestCase):
       result = shape_ops.frame(signal, frame_length, frame_step)
       self.assertEqual(result.dtype, signal.dtype)
 
+  @tf_test_util.run_deprecated_v1
   def test_dynamic_tensor(self):
     # Show that frame works even when the dimensions of its input are
     # not known at graph creation time.
@@ -325,6 +336,7 @@ class FrameTest(test.TestCase):
                            [[10, 11], [12, 13]],
                            [[20, 21], [22, 23]]], result)
 
+  @tf_test_util.run_deprecated_v1
   def test_gradient_numerical(self):
     with self.session(use_gpu=True):
       signal_shape = (2, 128)
diff --git a/tensorflow/python/kernel_tests/signal/window_ops_test.py b/tensorflow/python/kernel_tests/signal/window_ops_test.py
index 2f19134f5a..a72cdb288b 100644
--- a/tensorflow/python/kernel_tests/signal/window_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/window_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.kernel_tests.signal import test_util
 from tensorflow.python.ops.signal import window_ops
 from tensorflow.python.platform import test
@@ -75,6 +76,7 @@ class WindowOpsTest(test.TestCase):
                                   dtype=tf_dtype).eval()
             self.assertAllClose(expected, actual, tol, tol)
 
+  @tf_test_util.run_deprecated_v1
   def test_hann_window(self):
     """Check that hann_window matches scipy.signal.hann behavior."""
     # The Hann window is a raised cosine window with parameters alpha=0.5 and
@@ -84,6 +86,7 @@ class WindowOpsTest(test.TestCase):
         functools.partial(_scipy_raised_cosine, a=0.5, b=0.5),
         window_ops.hann_window)
 
+  @tf_test_util.run_deprecated_v1
   def test_hamming_window(self):
     """Check that hamming_window matches scipy.signal.hamming's behavior."""
     # The Hamming window is a raised cosine window with parameters alpha=0.54
diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py
index ee48c6eb0e..8f7245214a 100644
--- a/tensorflow/python/kernel_tests/slice_op_test.py
+++ b/tensorflow/python/kernel_tests/slice_op_test.py
@@ -24,6 +24,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
@@ -145,6 +146,7 @@ class SliceTest(test.TestCase):
         slice_val = self.evaluate(slice_t)
         self.assertAllEqual(slice_val, inp[lo:hi])
 
+  @test_util.run_deprecated_v1
   def testScalarInput(self):
     input_val = 0
     with self.cached_session() as sess:
@@ -159,6 +161,7 @@ class SliceTest(test.TestCase):
                                                "out of range"):
         sess.run([slice_t], feed_dict={input_t: input_val})
 
+  @test_util.run_deprecated_v1
   def testInvalidIndex(self):
     input_val = [1, 2]
     with self.cached_session() as sess:
@@ -179,6 +182,7 @@ class SliceTest(test.TestCase):
     np_ans = x[begin:begin + size, :]
     self.assertAllEqual(tf_ans, np_ans)
 
+  @test_util.run_deprecated_v1
   def testSliceMatrixDim0(self):
     x = np.random.rand(8, 4).astype("f")
     self._testSliceMatrixDim0(x, 1, 2)
@@ -213,6 +217,7 @@ class SliceTest(test.TestCase):
     self.assertEqual(slice_val.shape, slice_t.get_shape())
     self.assertEqual(slice2_val.shape, slice2_t.get_shape())
 
+  @test_util.run_deprecated_v1
   def testComplex(self):
     with self.session(use_gpu=True):
       inp = np.random.rand(4, 10, 10, 4).astype("f")
@@ -316,6 +321,7 @@ class SliceTest(test.TestCase):
       g1_val, g2_val = self.evaluate([g1, g2])
     self.assertAllEqual(g1_val, g2_val)
 
+  @test_util.run_deprecated_v1
   def testGradientsAll(self):
     # Slice the middle square out of a 4x4 input
     self._testGradientSlice([4, 4], [1, 1], [2, 2])
@@ -335,6 +341,7 @@ class SliceTest(test.TestCase):
     # Use -1 as a slice dimension on a 2D tensor.
     self._testGradientVariableSize2D()
 
+  @test_util.run_deprecated_v1
   def testNotIterable(self):
     # NOTE(mrry): If we register __getitem__ as an overloaded
     # operator, Python will valiantly attempt to iterate over the
@@ -346,6 +353,7 @@ class SliceTest(test.TestCase):
       for _ in c:
         pass
 
+  @test_util.run_deprecated_v1
   def testComputedShape(self):
     # NOTE(mrry): We cannot currently handle partially-known values,
     # because `tf.slice()` uses -1 to specify a wildcard size, and
diff --git a/tensorflow/python/kernel_tests/softmax_op_test.py b/tensorflow/python/kernel_tests/softmax_op_test.py
index 8b1a2e4c4e..707b8a429f 100644
--- a/tensorflow/python/kernel_tests/softmax_op_test.py
+++ b/tensorflow/python/kernel_tests/softmax_op_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.platform import test
@@ -206,6 +207,7 @@ class SoftmaxTest(test.TestCase):
                          [[5., 4., 3., 2.], [1., 2., 3., 4.]]])
     self.assertEqual([3, 2, 4], op.get_shape())
 
+  @test_util.run_deprecated_v1
   def testEmptyInput(self):
     with self.cached_session():
       x = array_ops.placeholder(dtypes.float32, shape=[0, 3])
@@ -229,6 +231,7 @@ class SoftmaxTest(test.TestCase):
       with self.assertRaises(errors_impl.InvalidArgumentError):
         nn_ops.softmax(ones, axis=2).eval()
 
+  @test_util.run_deprecated_v1
   def testLargeDims(self):
     # Make sure that we properly handle large inputs. See
     # https://github.com/tensorflow/tensorflow/issues/4425 for details
diff --git a/tensorflow/python/kernel_tests/softplus_op_test.py b/tensorflow/python/kernel_tests/softplus_op_test.py
index 48445a7380..5273dd7ffc 100644
--- a/tensorflow/python/kernel_tests/softplus_op_test.py
+++ b/tensorflow/python/kernel_tests/softplus_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import nn_ops
@@ -70,6 +71,7 @@ class SoftplusTest(test.TestCase):
           ],
           use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -86,6 +88,7 @@ class SoftplusTest(test.TestCase):
     print("softplus (float) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradGrad(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -103,6 +106,7 @@ class SoftplusTest(test.TestCase):
     print("softplus (float) gradient of gradient err = ", err)
     self.assertLess(err, 5e-5)
 
+  @test_util.run_deprecated_v1
   def testGradGradGrad(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -121,6 +125,7 @@ class SoftplusTest(test.TestCase):
     print("softplus (float) third-order gradient err = ", err)
     self.assertLess(err, 5e-5)
 
+  @test_util.run_deprecated_v1
   def testNoInts(self):
     with self.cached_session():
       with self.assertRaisesRegexp(
diff --git a/tensorflow/python/kernel_tests/softsign_op_test.py b/tensorflow/python/kernel_tests/softsign_op_test.py
index 71aac7e48e..5554240c82 100644
--- a/tensorflow/python/kernel_tests/softsign_op_test.py
+++ b/tensorflow/python/kernel_tests/softsign_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -49,6 +50,7 @@ class SoftsignTest(test.TestCase):
           np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t),
           use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session():
       x = constant_op.constant(
@@ -65,6 +67,7 @@ class SoftsignTest(test.TestCase):
     print("softsign (float) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testNoInts(self):
     with self.cached_session():
       with self.assertRaisesRegexp(
diff --git a/tensorflow/python/kernel_tests/spacetobatch_op_test.py b/tensorflow/python/kernel_tests/spacetobatch_op_test.py
index 21134adf2c..8641156604 100644
--- a/tensorflow/python/kernel_tests/spacetobatch_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetobatch_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gradient_checker
@@ -115,6 +116,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl):
     self._testPad(inputs, paddings, block_size, outputs)
 
   # [1, 2, 2, 1] <-> [4, 1, 1, 1]
+  @test_util.run_deprecated_v1
   def testSmallInput2x2(self):
     x_np = [[[[1], [2]], [[3], [4]]]]
     block_size = 2
@@ -122,6 +124,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl):
     self._testOne(x_np, block_size, x_out)
 
   # [1, 2, 2, 1] <-> [1, 3, 3, 1] (padding) <-> [9, 1, 1, 1]
+  @test_util.run_deprecated_v1
   def testSmallInput2x2Pad1x0(self):
     x_np = [[[[1], [2]], [[3], [4]]]]
     paddings = np.array([[1, 0], [1, 0]], dtype=np.int32)
@@ -132,6 +135,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl):
 
   # Test with depth larger than 1.
   # [1, 2, 2, 3] <-> [4, 1, 1, 3]
+  @test_util.run_deprecated_v1
   def testDepthInput2x2(self):
     x_np = [[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]]
     block_size = 2
@@ -140,6 +144,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl):
 
   # Test for larger input dimensions.
   # [1, 4, 4, 1] <-> [4, 2, 2, 1]
+  @test_util.run_deprecated_v1
   def testLargerInput2x2(self):
     x_np = [[[[1], [2], [3], [4]], [[5], [6], [7], [8]],
              [[9], [10], [11], [12]], [[13], [14], [15], [16]]]]
@@ -150,6 +155,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl):
 
   # Test with batch larger than 1.
   # [2, 2, 4, 1] <-> [8, 1, 2, 1]
+  @test_util.run_deprecated_v1
   def testBatchInput2x2(self):
     x_np = [[[[1], [2], [3], [4]], [[5], [6], [7], [8]]],
             [[[9], [10], [11], [12]], [[13], [14], [15], [16]]]]
@@ -162,6 +168,7 @@ class SpaceToBatchTest(test.TestCase, PythonOpImpl):
   # that elements are correctly laid out spatially and properly interleaved
   # along the batch dimension.
   # [2, 4, 4, 1] <-> [8, 2, 2, 1]
+  @test_util.run_deprecated_v1
   def testLargerInputBatch2x2(self):
     x_np = [[[[1], [2], [3], [4]], [[5], [6], [7], [8]],
              [[9], [10], [11], [12]], [[13], [14], [15], [16]]],
@@ -206,6 +213,7 @@ class SpaceToBatchNDTest(test.TestCase):
     self._testPad(inputs, block_shape, paddings,
                   space_to_batch_direct(inputs, block_shape, paddings))
 
+  @test_util.run_deprecated_v1
   def testZeroBlockDimsZeroRemainingDims(self):
     self._testPad(
         inputs=[1, 2],
@@ -213,6 +221,7 @@ class SpaceToBatchNDTest(test.TestCase):
         paddings=[],
         outputs=[1, 2],)
 
+  @test_util.run_deprecated_v1
   def testZeroBlockDimsOneRemainingDim(self):
     self._testPad(
         inputs=[[1, 2], [3, 4]],
@@ -227,6 +236,7 @@ class SpaceToBatchNDTest(test.TestCase):
         paddings=[[0, 0]],
         outputs=[[1, 2], [3, 4]])
 
+  @test_util.run_deprecated_v1
   def testZeroBlockDimsTwoRemainingDims(self):
     self._testPad(
         inputs=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]],
@@ -248,6 +258,7 @@ class SpaceToBatchNDTest(test.TestCase):
         paddings=[[0, 0], [0, 0]],
         outputs=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
 
+  @test_util.run_deprecated_v1
   def testOneBlockDimZeroRemainingDims(self):
     self._testPad(
         inputs=[[1, 2, 3], [4, 5, 6]],
@@ -255,6 +266,7 @@ class SpaceToBatchNDTest(test.TestCase):
         paddings=[1, 0],
         outputs=[[0, 2], [0, 5], [1, 3], [4, 6]])
 
+  @test_util.run_deprecated_v1
   def testOneBlockDimOneRemainingDim(self):
     self._testPad(
         inputs=[[[1, 11], [2, 21], [3, 31]], [[4, 41], [5, 51], [6, 61]]],
@@ -263,6 +275,7 @@ class SpaceToBatchNDTest(test.TestCase):
         outputs=[[[0, 0], [2, 21]], [[0, 0], [5, 51]], [[1, 11], [3, 31]],
                  [[4, 41], [6, 61]]])
 
+  @test_util.run_deprecated_v1
   def testDirect(self):
     # Test with zero-size remaining dimension.
     self._testDirect(
@@ -300,6 +313,7 @@ class SpaceToBatchNDTest(test.TestCase):
 class SpaceToBatchSpaceToDepth(test.TestCase, PythonOpImpl):
 
   # Verifies that: space_to_batch(x) = transpose(space_to_depth(transpose(x)))
+  @test_util.run_deprecated_v1
   def testSpaceToDepthTranspose(self):
     x = np.arange(5 * 10 * 16 * 7, dtype=np.float32).reshape([5, 10, 16, 7])
     block_size = 2
@@ -319,6 +333,7 @@ class SpaceToBatchSpaceToDepthCpp(SpaceToBatchSpaceToDepth, CppOpImpl):
 
 class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
 
+  @test_util.run_deprecated_v1
   def testInputWrongDimMissingBatch(self):
     # The input is missing the first dimension ("batch")
     x_np = [[[1], [2]], [[3], [4]]]
@@ -327,6 +342,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
     with self.assertRaises(ValueError):
       _ = self.space_to_batch(x_np, paddings, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSize0(self):
     # The block size is 0.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -336,6 +352,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
       out_tf = self.space_to_batch(x_np, paddings, block_size)
       out_tf.eval()
 
+  @test_util.run_deprecated_v1
   def testBlockSizeOne(self):
     # The block size is 1. The block size needs to be > 1.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -345,6 +362,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
       out_tf = self.space_to_batch(x_np, paddings, block_size)
       out_tf.eval()
 
+  @test_util.run_deprecated_v1
   def testBlockSizeLarger(self):
     # The block size is too large for this input.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -354,6 +372,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
       out_tf = self.space_to_batch(x_np, paddings, block_size)
       out_tf.eval()
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleWidth(self):
     # The block size divides width but not height.
     x_np = [[[[1], [2], [3]], [[3], [4], [7]]]]
@@ -362,6 +381,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
     with self.assertRaises(ValueError):
       _ = self.space_to_batch(x_np, paddings, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleHeight(self):
     # The block size divides height but not width.
     x_np = [[[[1], [2]], [[3], [4]], [[5], [6]]]]
@@ -370,6 +390,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
     with self.assertRaises(ValueError):
       _ = self.space_to_batch(x_np, paddings, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleBoth(self):
     # The block size does not divide neither width or height.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -378,6 +399,7 @@ class SpaceToBatchErrorHandlingTest(test.TestCase, PythonOpImpl):
     with self.assertRaises(ValueError):
       _ = self.space_to_batch(x_np, paddings, block_size)
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     t = self.space_to_batch(
         array_ops.placeholder(dtypes.float32),
@@ -424,25 +446,31 @@ class SpaceToBatchNDErrorHandlingTest(test.TestCase):
     self._testStaticShape(input_shape, block_shape, paddings, error)
     self._testDynamicShape(input_shape, block_shape, paddings)
 
+  @test_util.run_deprecated_v1
   def testBlockSize0(self):
     # The block size is 0.
     self._testShape([1, 2, 2], [0, 2], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNegative(self):
     self._testShape([1, 2, 2], [-1, 2], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testNegativePadding(self):
     # The padding is negative.
     self._testShape([1, 2, 2], [1, 1], [[0, -1], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisible(self):
     # The padded size is not divisible by the block size.
     self._testShape([1, 2, 3, 1], [3, 3], [[0, 0], [0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testBlockDimsMismatch(self):
     # Shape of block_shape does not match shape of paddings.
     self._testStaticShape([1, 3, 3, 1], [3, 3], [[0, 0]], ValueError)
 
+  @test_util.run_deprecated_v1
   def testUnknown(self):
     # Verify that input shape and paddings shape can be unknown.
     _ = array_ops.space_to_batch_nd(
@@ -522,18 +550,21 @@ class SpaceToBatchGradientTest(test.TestCase, PythonOpImpl):
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
+  @test_util.run_deprecated_v1
   def testSmall(self):
     block_size = 2
     pad_beg = 0
     pad_end = 0
     self._compare(1, 2, 3, 5, block_size, pad_beg, pad_end)
 
+  @test_util.run_deprecated_v1
   def testSmall2(self):
     block_size = 2
     pad_beg = 0
     pad_end = 0
     self._compare(2, 4, 3, 2, block_size, pad_beg, pad_end)
 
+  @test_util.run_deprecated_v1
   def testSmallPad1x1(self):
     block_size = 2
     pad_beg = 1
@@ -572,15 +603,19 @@ class SpaceToBatchNDGradientTest(test.TestCase):
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
+  @test_util.run_deprecated_v1
   def testSmall(self):
     self._compare([1, 4, 6, 5], [2, 2], [[0, 0], [0, 0]])
 
+  @test_util.run_deprecated_v1
   def testSmall2(self):
     self._compare([2, 8, 6, 2], [2, 2], [[0, 0], [0, 0]])
 
+  @test_util.run_deprecated_v1
   def testSmallPad1(self):
     self._compare([2, 4, 6, 2], [2, 2], [[1, 1], [1, 1]])
 
+  @test_util.run_deprecated_v1
   def testSmallPadThreeBlockDims(self):
     self._compare([2, 2, 4, 3, 2], [2, 2, 2], [[1, 1], [1, 1], [1, 0]])
 
@@ -644,6 +679,7 @@ class RequiredSpaceToBatchPaddingsTest(test.TestCase):
     self.assertAllEqual(paddings_result, paddings_const)
     self.assertAllEqual(crops_result, crops_const)
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     self._test(
         input_shape=np.zeros((0,), np.int32),
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index c9aaa68971..e96bc09f36 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -159,6 +159,7 @@ class SpaceToDepthTest(test.TestCase):
 
   # Error handling:
 
+  @test_util.run_deprecated_v1
   def testInputWrongDimMissingDepth(self):
     # The input is missing the last dimension ("depth")
     x_np = [[[1, 2], [3, 4]]]
@@ -167,6 +168,7 @@ class SpaceToDepthTest(test.TestCase):
       out_tf = array_ops.space_to_depth(x_np, block_size)
       self.evaluate(out_tf)
 
+  @test_util.run_deprecated_v1
   def testInputWrongDimMissingBatch(self):
     # The input is missing the first dimension ("batch")
     x_np = [[[1], [2]], [[3], [4]]]
@@ -174,6 +176,7 @@ class SpaceToDepthTest(test.TestCase):
     with self.assertRaises(ValueError):
       _ = array_ops.space_to_depth(x_np, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSize0(self):
     # The block size is 0.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -182,6 +185,7 @@ class SpaceToDepthTest(test.TestCase):
       out_tf = array_ops.space_to_depth(x_np, block_size)
       self.evaluate(out_tf)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeOne(self):
     # The block size is 1. The block size needs to be > 1.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -190,6 +194,7 @@ class SpaceToDepthTest(test.TestCase):
       out_tf = array_ops.space_to_depth(x_np, block_size)
       self.evaluate(out_tf)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeLarger(self):
     # The block size is too large for this input.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -198,6 +203,7 @@ class SpaceToDepthTest(test.TestCase):
       out_tf = array_ops.space_to_depth(x_np, block_size)
       self.evaluate(out_tf)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleWidth(self):
     # The block size divides width but not height.
     x_np = [[[[1], [2], [3]], [[3], [4], [7]]]]
@@ -205,6 +211,7 @@ class SpaceToDepthTest(test.TestCase):
     with self.assertRaises(ValueError):
       _ = array_ops.space_to_depth(x_np, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleHeight(self):
     # The block size divides height but not width.
     x_np = [[[[1], [2]], [[3], [4]], [[5], [6]]]]
@@ -212,6 +219,7 @@ class SpaceToDepthTest(test.TestCase):
     with self.assertRaises(ValueError):
       _ = array_ops.space_to_depth(x_np, block_size)
 
+  @test_util.run_deprecated_v1
   def testBlockSizeNotDivisibleBoth(self):
     # The block size does not divide neither width or height.
     x_np = [[[[1], [2]], [[3], [4]]]]
@@ -219,6 +227,7 @@ class SpaceToDepthTest(test.TestCase):
     with self.assertRaises(ValueError):
       _ = array_ops.space_to_depth(x_np, block_size)
 
+  @test_util.run_deprecated_v1
   def testUnknownShape(self):
     t = array_ops.space_to_depth(
         array_ops.placeholder(dtypes.float32), block_size=4)
@@ -334,11 +343,13 @@ class SpaceToDepthGradientTest(test.TestCase):
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
+  @test_util.run_deprecated_v1
   def testSmall(self):
     block_size = 2
     self._compare(1, 2, 3, 5, block_size, "NHWC")
     self._compare(1, 2, 3, 5, block_size, "NCHW")
 
+  @test_util.run_deprecated_v1
   def testSmall2(self):
     block_size = 2
     self._compare(2, 4, 3, 2, block_size, "NHWC")
diff --git a/tensorflow/python/kernel_tests/sparse_add_op_test.py b/tensorflow/python/kernel_tests/sparse_add_op_test.py
index c61f863355..00eff54077 100644
--- a/tensorflow/python/kernel_tests/sparse_add_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_add_op_test.py
@@ -140,6 +140,7 @@ class SparseAddTest(test.TestCase):
       self.assertAllClose(sum_out.values, [2, 6, -.2])
       self.assertAllEqual(sum_out.dense_shape, [3, 3])
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     np.random.seed(1618)  # Make it reproducible.
     with self.session(use_gpu=False):
@@ -176,6 +177,7 @@ class SparseAddTest(test.TestCase):
           self.assertAllEqual(dense_np + rand_vals_np, s)
           self.assertTrue(s.dtype == dtype)
 
+  @test_util.run_deprecated_v1
   def testSparseTensorDenseAddGradients(self):
     np.random.seed(1618)  # Make it reproducible.
     n, m = np.random.randint(30, size=2)
@@ -191,6 +193,7 @@ class SparseAddTest(test.TestCase):
                                                     [(nnz,), (n, m)], s, (n, m))
       self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testInvalidSparseTensor(self):
     with test_util.force_cpu():
       shape = [2, 2]
diff --git a/tensorflow/python/kernel_tests/sparse_concat_op_test.py b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
index 368a533e56..04b6b9b8d2 100644
--- a/tensorflow/python/kernel_tests/sparse_concat_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_concat_op_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
@@ -253,6 +254,7 @@ class SparseConcatTest(test.TestCase):
                             [b"a", b"b", b"e", b"c", b"d", b"f", b"g", b"h"])
         self.assertAllEqual(concat_out.dense_shape, [3, 8])
 
+  @test_util.run_deprecated_v1
   def testMismatchedRank(self):
     with self.session(use_gpu=False):
       sp_a = self._SparseTensor_3x3()
@@ -263,6 +265,7 @@ class SparseConcatTest(test.TestCase):
         with self.assertRaises(ValueError):
           sparse_ops.sparse_concat(concat_dim, [sp_a, sp_e])
 
+  @test_util.run_deprecated_v1
   def testMismatchedRankExpandNonconcatDim(self):
     with self.session(use_gpu=False):
       sp_a = self._SparseTensor_3x3()
@@ -275,6 +278,7 @@ class SparseConcatTest(test.TestCase):
           sparse_ops.sparse_concat(
               concat_dim, [sp_a, sp_e], expand_nonconcat_dim=True)
 
+  @test_util.run_deprecated_v1
   def testMismatchedShapes(self):
     with self.session(use_gpu=False) as sess:
       sp_a = self._SparseTensor_3x3()
@@ -321,6 +325,7 @@ class SparseConcatTest(test.TestCase):
                               [1, 1, 2, 1, 1, 1, 2, 3, 4, 2, 1, 0, 2])
           self.assertAllEqual(sp_concat_dim1_out.dense_shape, [3, 13])
 
+  @test_util.run_deprecated_v1
   def testShapeInferenceUnknownShapes(self):
     with self.session(use_gpu=False):
       sp_inputs = [
diff --git a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
index 66589fa315..275c86e534 100644
--- a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.platform import test
@@ -98,12 +99,14 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       attr { key: 'reduction_type' value {s: 'MEAN'} }
       """, q.accumulator_ref.op.node_def)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSizeEmpty(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
           dtypes_lib.float32, name="Q")
       self.assertEqual(q.num_accumulated().eval(), 0)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorSetGlobalStep(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -111,6 +114,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       set_global_step_op = q.set_global_step(1)
       set_global_step_op.run()
 
+  @test_util.run_deprecated_v1
   def testAccumulatorApplyGradFloat32(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -122,6 +126,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       accum_op.run()
       self.assertEqual(q.num_accumulated().eval(), 1)
 
+  @test_util.run_deprecated_v1
   def testDtypes(self):
     with self.cached_session() as sess:
       dtypes = [dtypes_lib.float16, dtypes_lib.float32, dtypes_lib.float64]
@@ -144,6 +149,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
         self._assertEqual_nparray(sum_elems / len(elems), result, sess)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorMultipleAccumulators(self):
     with self.cached_session() as sess:
       q_f32_0 = data_flow_ops.SparseConditionalAccumulator(
@@ -174,6 +180,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
         result = sess.run(accums[i].take_indexed_slices_grad(1))
         self._assertEqual_indexedslices(expected_tensors[i], result)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorTakeGradMean(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -194,6 +201,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       self.assertAllEqual([[0.5, 0.5], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorTakeGradSum(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -214,11 +222,13 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       self.assertAllEqual([[1, 1], [0, 2], [3, 0]], val.values)
       self.assertAllEqual([-1, 2], val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorTakeGradInvalidReductionType(self):
     with self.assertRaises(ValueError):
       data_flow_ops.SparseConditionalAccumulator(
           dtypes_lib.float32, name="Q", shape=(), reduction_type="Invalid")
 
+  @test_util.run_deprecated_v1
   def testAccumulatorRepeatedTakeGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -257,6 +267,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       self.assertAllEqual(val.values, [[5, 5], [0, 20], [30, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
 
+  @test_util.run_deprecated_v1
   def testParallelApplyGradMean(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -288,6 +299,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
           np.array([[expected_val, 0], [0, expected_val]]).astype(np.float32),
           val, sess)
 
+  @test_util.run_deprecated_v1
   def testParallelApplyGradSum(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -322,6 +334,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
           np.array([[expected_val, 0], [0, expected_val]]).astype(np.float32),
           val, sess)
 
+  @test_util.run_deprecated_v1
   def testParallelTakeGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -361,6 +374,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
         self._assertEqual_nparray(
             np.array([[0, 0], [elems[i], 0]]), results[i], sess)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorApplyAndBlockingTake(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -396,6 +410,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
     with self.assertRaisesOpError("was cancelled"):
       self.evaluate(takeg_op)
 
+  @test_util.run_deprecated_v1
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -415,6 +430,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
       takeg_thread.join()
 
+  @test_util.run_deprecated_v1
   def testNonVectorIndices(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -427,6 +443,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
             grad_indices=[[0, 1], [1, 0]],
             grad_values=np.array([1, 2]).astype(np.float32)).run()
 
+  @test_util.run_deprecated_v1
   def testZeroDimensionValues(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -437,6 +454,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
         q.apply_grad(
             grad_indices=[0], grad_values=np.array(1).astype(np.float32)).run()
 
+  @test_util.run_deprecated_v1
   def testWrongNonEmptyInputValues(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -448,6 +466,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
             grad_indices=[0, 1],
             grad_values=np.array([[0, 1, 1]]).astype(np.float32)).run()
 
+  @test_util.run_deprecated_v1
   def testDynamicNonVectorIndices(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -467,6 +486,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
                      x_values: np.array([1, 2]).astype(np.float32)
                  })
 
+  @test_util.run_deprecated_v1
   def testDynamicWrongNonEmptyInputValues(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -485,6 +505,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
                      x_values: np.array([[0, 1, 1]]).astype(np.float32)
                  })
 
+  @test_util.run_deprecated_v1
   def testEmptyShapeApply(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -510,6 +531,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       q.apply_grad(grad_indices=[0], grad_values=[1.0], grad_shape=[]).run()
       q.apply_grad(grad_indices=[0], grad_values=[1.0]).run()
 
+  @test_util.run_deprecated_v1
   def testValidateShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -605,6 +627,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
                 [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]).astype(np.float32),
             local_step=1).run()
 
+  @test_util.run_deprecated_v1
   def testReturnShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -630,6 +653,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       val = self.evaluate(q.take_indexed_slices_grad(1))
       self.assertAllEqual(val.dense_shape, [-1, 2, 2, 3])
 
+  @test_util.run_deprecated_v1
   def testApplyGradtInt32IndicesAndShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
diff --git a/tensorflow/python/kernel_tests/sparse_cross_op_test.py b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
index 8451b96c56..566bbb56f0 100644
--- a/tensorflow/python/kernel_tests/sparse_cross_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_cross_op_test.py
@@ -24,12 +24,14 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
 
 
 class SparseCrossOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_simple(self):
     """Tests a simple scenario."""
     op = sparse_ops.sparse_cross([
@@ -45,6 +47,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_dense(self):
     """Tests only dense inputs."""
     op = sparse_ops.sparse_cross([
@@ -65,6 +68,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_integer_mixed_string_sparse(self):
     """Tests mixed type."""
     op = sparse_ops.sparse_cross([
@@ -79,6 +83,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_integer_mixed_string_dense(self):
     """Tests mixed dense inputs."""
     op = sparse_ops.sparse_cross([
@@ -97,6 +102,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_sparse_cross_dense(self):
     """Tests sparse and dense inputs."""
     op = sparse_ops.sparse_cross([
@@ -114,6 +120,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_integer_sparse_input(self):
     """Tests mixed type sparse and dense inputs."""
     op = sparse_ops.sparse_cross([
@@ -130,6 +137,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_permutation_3x3x3(self):
     """Tests 3x3x3 permutation."""
     op = sparse_ops.sparse_cross([
@@ -172,6 +180,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_permutation_3x1x2(self):
     """Tests 3x1x2 permutation."""
     op = sparse_ops.sparse_cross([
@@ -191,6 +200,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_large_batch(self):
     """Tests with large batch size to force multithreading."""
     batch_size = 5000
@@ -224,6 +234,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_one_column_empty(self):
     """Tests when one column is empty.
 
@@ -237,6 +248,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_empty(self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_some_columns_empty(self):
     """Tests when more than one columns are empty.
 
@@ -256,6 +268,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_all_columns_empty(self):
     """Tests when all columns are empty.
 
@@ -269,6 +282,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_empty(self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_hashed_zero_bucket_no_hash_key(self):
     op = sparse_ops.sparse_cross_hashed([
         self._sparse_tensor([['batch1-FC1-F1']]),
@@ -280,6 +294,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_hashed_zero_bucket(self):
     op = sparse_ops.sparse_cross_hashed(
         [
@@ -294,6 +309,7 @@ class SparseCrossOpTest(test.TestCase):
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
   # TODO(sibyl-Aix6ihai): Add benchmark to compare Hashed vs Non-hashed.
+  @test_util.run_deprecated_v1
   def test_hashed_no_hash_key(self):
     op = sparse_ops.sparse_cross_hashed(
         [
@@ -307,6 +323,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_hashed_output(self):
     op = sparse_ops.sparse_cross_hashed(
         [
@@ -321,6 +338,7 @@ class SparseCrossOpTest(test.TestCase):
     with self.cached_session() as sess:
       self._assert_sparse_tensor_equals(expected_out, self.evaluate(op))
 
+  @test_util.run_deprecated_v1
   def test_hashed__has_no_collision(self):
     """Tests that fingerprint concatenation has no collisions."""
     # Although the last 10 bits of 359 and 1024+359 are identical.
diff --git a/tensorflow/python/kernel_tests/sparse_matmul_op_test.py b/tensorflow/python/kernel_tests/sparse_matmul_op_test.py
index 4de69a26e3..2e17a9c608 100644
--- a/tensorflow/python/kernel_tests/sparse_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_matmul_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -71,6 +72,7 @@ class SparseMatMulTest(test.TestCase):
     self.assertShapeEqual(np_ans, tf_ans)
     self.assertAllCloseAccordingToType(np_ans, out, rtol=1e-4, atol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     x = np.arange(0., 4.).reshape([4, 1]).astype(np.float32)
     y = np.arange(-1., 1.).reshape([1, 2]).astype(np.float32)
@@ -78,6 +80,7 @@ class SparseMatMulTest(test.TestCase):
       for y_dtype in (dtypes.float32, dtypes.bfloat16):
         self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype)
 
+  @test_util.run_deprecated_v1
   def testZeroDim(self):
     x = np.ones((4, 0)).astype(np.float32)
     y = np.ones((0, 3)).astype(np.float32)
@@ -85,6 +88,7 @@ class SparseMatMulTest(test.TestCase):
       for y_dtype in (dtypes.float32, dtypes.bfloat16):
         self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype)
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     x = np.ones((0, 0)).astype(np.float32)
     y = np.ones((0, 0)).astype(np.float32)
@@ -93,6 +97,7 @@ class SparseMatMulTest(test.TestCase):
         self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype)
 
   # Tests setting one dimension to be a high value.
+  @test_util.run_deprecated_v1
   def testLarge(self):
     r1 = np.random.randint(6000, 20000)
     r2 = np.random.randint(1, 10)
@@ -105,6 +110,7 @@ class SparseMatMulTest(test.TestCase):
           self._testCpuMatmul(x, y, x_dtype=x_dtype, y_dtype=y_dtype)
 
   # Tests random sized matrices.
+  @test_util.run_deprecated_v1
   def testRandom(self):
     for tr_a in [True, False]:
       for tr_b in [True, False]:
@@ -159,6 +165,7 @@ class MatMulGradientTest(test.TestCase):
               delta=delta))
     self.assertLessEqual(err, delta / 2.)
 
+  @test_util.run_deprecated_v1
   def testGradientInput(self):
     for tr_a in [True, False]:
       for tr_b in [True, False]:
diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index 605aaf3dd1..75f65e6251 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -71,6 +71,7 @@ class SparseToIndicatorTest(test_util.TensorFlowTestCase):
         constant_op.constant(val, dtype),
         constant_op.constant(shape, dtypes.int64))
 
+  @test_util.run_deprecated_v1
   def testInt32(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_5x6(dtypes.int32)
@@ -83,6 +84,7 @@ class SparseToIndicatorTest(test_util.TensorFlowTestCase):
 
       self.assertAllEqual(output, expected_output)
 
+  @test_util.run_deprecated_v1
   def testInt64(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_5x6(dtypes.int64)
@@ -95,6 +97,7 @@ class SparseToIndicatorTest(test_util.TensorFlowTestCase):
 
       self.assertAllEqual(output, expected_output)
 
+  @test_util.run_deprecated_v1
   def testHigherRank(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_2x3x4(dtypes.int64)
@@ -296,6 +299,7 @@ class SparseRetainTest(test_util.TensorFlowTestCase):
   def _SparseTensor_5x6(self):
     return sparse_tensor.SparseTensor.from_value(self._SparseTensorValue_5x6())
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.session(use_gpu=False) as sess:
       for sp_input in (self._SparseTensorValue_5x6(), self._SparseTensor_5x6()):
@@ -353,12 +357,14 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
     return sparse_tensor.SparseTensorValue(self._IND_2_5_6, self._VAL_2_5_6,
                                            self._SHP_2_5_6)
 
+  @test_util.run_deprecated_v1
   def testStaticShapeInfoPreservedWhenNewShapeIsProvidedAndStatic(self):
     sp_input = self._SparseTensor_2x5x6()
     new_shape = np.array([3, 6, 7], dtype=np.int64)
     sp_output = sparse_ops.sparse_reset_shape(sp_input, new_shape)
     self.assertAllEqual([3, 6, 7], sp_output.get_shape())
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensor_2x5x6()
@@ -372,6 +378,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(output.values, [0, 10, 13, 14, 32, 33])
       self.assertAllEqual(output.dense_shape, [3, 6, 7])
 
+  @test_util.run_deprecated_v1
   def testInputUnavailableInGraphConstructionOk(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorValue_2x5x6()
@@ -385,6 +392,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(output.values, [0, 10, 13, 14, 32, 33])
       self.assertAllEqual(output.dense_shape, [3, 6, 7])
 
+  @test_util.run_deprecated_v1
   def testFeedInputUnavailableInGraphConstructionOk(self):
     with self.session(use_gpu=False) as sess:
       sp_input = array_ops.sparse_placeholder(dtype=dtypes.int32)
@@ -422,6 +430,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(output.values.shape, [0])
       self.assertAllEqual(output.dense_shape, [0, 0, 0])
 
+  @test_util.run_deprecated_v1
   def testInvalidRank(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_2x5x6()
@@ -430,6 +439,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       with self.assertRaises(ValueError):
         sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
+  @test_util.run_deprecated_v1
   def testInvalidRankNewShapeUnavailableInGraphConstruction(self):
     with self.session(use_gpu=False) as sess:
       new_shape = array_ops.placeholder(dtype=dtypes.int64)
@@ -439,6 +449,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       with self.assertRaisesOpError("x == y did not hold element-wise"):
         sess.run(out, feed_dict={new_shape: np.array([3, 7], dtype=np.int64)})
 
+  @test_util.run_deprecated_v1
   def testInvalidDimensionSizeStatic(self):
     sp_input = self._SparseTensor_2x5x6()
     new_shape = np.array([3, 7, 5], dtype=np.int64)
@@ -446,6 +457,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
     with self.assertRaisesRegexp(ValueError, "should have dimension sizes"):
       sparse_ops.sparse_reset_shape(sp_input, new_shape)
 
+  @test_util.run_deprecated_v1
   def testInvalidDimensionSizeDynamic(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensor_2x5x6()
@@ -455,6 +467,7 @@ class SparseResetShapeTest(test_util.TensorFlowTestCase):
       with self.assertRaisesOpError("x <= y did not hold element-wise"):
         sess.run(out, feed_dict={new_shape: [3, 7, 5]})
 
+  @test_util.run_deprecated_v1
   def testInvalidDimensionSizeInputUnavailableInGraphConstruction(self):
     sp_input = array_ops.sparse_placeholder(dtype=dtypes.int32)
     with self.session(use_gpu=False) as sess:
@@ -496,6 +509,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase):
         constant_op.constant(val, dtypes.int32),
         constant_op.constant(shape, dtypes.int64))
 
+  @test_util.run_deprecated_v1
   def testFillNumber(self):
     with self.session(use_gpu=False) as sess:
       for sp_input in (self._SparseTensorValue_5x6(), self._SparseTensor_5x6()):
@@ -513,6 +527,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(empty_row_indicator_out,
                             np.array([0, 0, 1, 0, 1]).astype(np.bool))
 
+  @test_util.run_deprecated_v1
   def testFillFloat(self):
     with self.session(use_gpu=False) as sess:
       values = constant_op.constant(
@@ -547,6 +562,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase):
       self.assertGreater(default_value_grad_err, 0)
       self.assertLess(default_value_grad_err, 1e-8)
 
+  @test_util.run_deprecated_v1
   def testFillString(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensor_String5x6()
@@ -565,6 +581,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(empty_row_indicator_out,
                           np.array([0, 0, 1, 0, 1]).astype(np.bool))
 
+  @test_util.run_deprecated_v1
   def testNoEmptyRows(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensor_2x6()
@@ -582,6 +599,7 @@ class SparseFillEmptyRowsTest(test_util.TensorFlowTestCase):
 
 class SparseAddTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testValuesInVariable(self):
     indices = constant_op.constant([[1]], dtype=dtypes.int64)
     values = variables.Variable([1], trainable=False, dtype=dtypes.float32)
@@ -657,6 +675,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
     self._compare(sp_t, reduction_axes, ndims, True, False)
     self._compare(sp_t, reduction_axes, ndims, True, True)
 
+  @test_util.run_deprecated_v1
   def testSimpleAndRandomInputs(self):
     if np.__version__ == "1.13.0":
       self.skipTest("numpy 1.13.0 bug")
@@ -696,6 +715,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase):
       with self.assertRaisesOpError("Invalid reduction dimension 2"):
         sparse_ops.sparse_reduce_max(sp_t, 2).eval()
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     if np.__version__ == "1.13.0":
       self.skipTest("numpy 1.13.0 bug")
@@ -778,6 +798,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase):
                                                result_tensor.values).eval()
     self.assertAllEqual(result_np, res_densified)
 
+  @test_util.run_deprecated_v1
   def testCwiseDivAndMul(self):
     np.random.seed(1618)
     sp_shapes = [(10, 10, 10), (5, 5), (1618,), (3, 3, 7)]
@@ -801,6 +822,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase):
             res = sp_t / dense_t  # should invoke "__truediv__"
             self.assertEqual(res.values.eval().dtype, np.float64)
 
+  @test_util.run_deprecated_v1
   def testCwiseAdd(self):
     with self.session(use_gpu=False):
       # Identity(2) + AllOnes(2,2).  Should be equal to 2 * Identity(2).
@@ -820,6 +842,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase):
           sparse_ops.sparse_dense_cwise_add(sp_t, dense_t),
           np.identity(2) * 2, sp_t)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     np.random.seed(1618)
     sp_shapes = [(10, 10, 10), (5, 5), (1618,), (3, 3, 7)]
@@ -853,6 +876,7 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase):
 
 class SparseSoftmaxTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testEquivalentToDensified(self):
     np.random.seed(1618)
     n, m = np.random.choice(20, size=2)
@@ -872,6 +896,7 @@ class SparseSoftmaxTest(test_util.TensorFlowTestCase):
 
         self.assertAllClose(dense_result.eval(), sp_result)
 
+  @test_util.run_deprecated_v1
   def testHigherRanks(self):
     # For the first shape:
     # First batch:
@@ -901,6 +926,7 @@ class SparseSoftmaxTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(sp_t.indices.eval(), result.indices)
         self.assertAllEqual(shape, result.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     x_shape = [2, 5, 10]
     with self.cached_session(use_gpu=False):
@@ -920,6 +946,7 @@ class SparseMinimumMaximumTest(test_util.TensorFlowTestCase):
     self.assertAllEqual(a.values, b.values)
     self.assertAllEqual(a.dense_shape, b.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.cached_session(use_gpu=False):
       # 1-D, values at index 0.
@@ -939,6 +966,7 @@ class SparseMinimumMaximumTest(test_util.TensorFlowTestCase):
       self._assertSparseTensorValueEqual(expected.eval(), max_tf)
       self._assertSparseTensorValueEqual(expected.eval(), min_tf)
 
+  @test_util.run_deprecated_v1
   def testRandom(self):
     np.random.seed(1618)
     shapes = [(13,), (6, 8), (1, 7, 1)]
@@ -980,6 +1008,7 @@ class SparseMinimumMaximumTest(test_util.TensorFlowTestCase):
 
 class SparseTransposeTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testTranspose(self):
     if np.__version__ == "1.13.0":
       self.skipTest("numpy 1.13.0 bug")
@@ -1002,16 +1031,19 @@ class SparseTransposeTest(test.TestCase):
 
 class SparsePlaceholderTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testPlaceholder(self):
     foo = array_ops.sparse_placeholder(dtypes.float32, shape=(10, 47))
     self.assertAllEqual([10, 47], foo.get_shape())
     self.assertAllEqual([None, 2], foo.indices.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testPartialShapePlaceholder(self):
     foo = array_ops.sparse_placeholder(dtypes.float32, shape=(None, 47))
     self.assertAllEqual([None, None], foo.get_shape().as_list())
     self.assertAllEqual([None, 2], foo.indices.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testNoShapePlaceholder(self):
     foo = array_ops.sparse_placeholder(dtypes.float32, shape=None)
     self.assertAllEqual(None, foo.get_shape())
diff --git a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
index bbf2f39202..93fcc6a18e 100644
--- a/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reorder_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import sparse_ops
@@ -65,6 +66,7 @@ class SparseReorderTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testFeedAlreadyInOrder(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -89,6 +91,7 @@ class SparseReorderTest(test.TestCase):
         self.assertAllEqual(output_val.dense_shape,
                             expected_output_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testFeedOutOfOrder(self):
     expected_output_val = self._SparseTensorValue_5x6(np.arange(6))
     with self.session(use_gpu=False) as sess:
@@ -103,6 +106,7 @@ class SparseReorderTest(test.TestCase):
         self.assertAllEqual(output_val.dense_shape,
                             expected_output_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     with self.session(use_gpu=False):
       for _ in range(5):  # To test various random permutations
diff --git a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
index 918af27091..9341228d57 100644
--- a/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_reshape_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
@@ -64,12 +65,14 @@ class SparseReshapeTest(test.TestCase):
     sp_output = sparse_ops.sparse_reshape(sp_input, shape=(2, -1))
     self.assertAllEqual((2, 3 * 4), sp_output.get_shape())
 
+  @test_util.run_deprecated_v1
   def testRaisesIfMoreThanOneInferredDim(self):
     sp_input = sparse_tensor.SparseTensor.from_value(
         self._SparseTensorValue_2x3x4())
     with self.assertRaisesRegexp(ValueError, "At most one dimension can"):
       sparse_ops.sparse_reshape(sp_input, shape=(-1, 2, -1))
 
+  @test_util.run_deprecated_v1
   def testRaisesIfInferredShapeNotPossible(self):
     sp_input = sparse_tensor.SparseTensor.from_value(
         self._SparseTensorValue_2x3x4())
@@ -86,6 +89,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testFeedSameShape(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -97,6 +101,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testWorksWellWithTfShape(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -109,6 +114,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testFeedSameShapeWithInferredDim(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -120,6 +126,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, input_val.dense_shape)
 
+  @test_util.run_deprecated_v1
   def testFeedNewShapeSameRank(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -133,6 +140,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, [3, 10])
 
+  @test_util.run_deprecated_v1
   def testFeedNewShapeSameRankWithInferredDim(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -158,6 +166,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, [2, 3, 5])
 
+  @test_util.run_deprecated_v1
   def testFeedUpRank(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -171,6 +180,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, [2, 3, 5])
 
+  @test_util.run_deprecated_v1
   def testFeedUpRankWithInferredDim(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -184,6 +194,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, [2, 3, 5])
 
+  @test_util.run_deprecated_v1
   def testFeedDownRank(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -197,6 +208,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, [6, 4])
 
+  @test_util.run_deprecated_v1
   def testFeedDownRankWithInferredDim(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -210,6 +222,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertAllEqual(output_val.values, input_val.values)
       self.assertAllEqual(output_val.dense_shape, [6, 4])
 
+  @test_util.run_deprecated_v1
   def testFeedMultipleInferredDims(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -218,12 +231,14 @@ class SparseReshapeTest(test.TestCase):
       with self.assertRaisesOpError("only one output dimension may be -1"):
         sess.run(sp_output, {sp_input: input_val})
 
+  @test_util.run_deprecated_v1
   def testProvideStaticallyMismatchedSizes(self):
     input_val = self._SparseTensorValue_5x6()
     sp_input = sparse_tensor.SparseTensor.from_value(input_val)
     with self.assertRaisesRegexp(ValueError, "Cannot reshape"):
       sparse_ops.sparse_reshape(sp_input, [4, 7])
 
+  @test_util.run_deprecated_v1
   def testFeedMismatchedSizes(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -233,6 +248,7 @@ class SparseReshapeTest(test.TestCase):
           "Input to reshape is a tensor with 30 dense values"):
         sess.run(sp_output, {sp_input: input_val})
 
+  @test_util.run_deprecated_v1
   def testFeedMismatchedSizesWithInferredDim(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -241,6 +257,7 @@ class SparseReshapeTest(test.TestCase):
       with self.assertRaisesOpError("requested shape requires a multiple"):
         sess.run(sp_output, {sp_input: input_val})
 
+  @test_util.run_deprecated_v1
   def testFeedPartialShapes(self):
     with self.session(use_gpu=False):
       # Incorporate new rank into shape information if known
@@ -266,6 +283,7 @@ class SparseReshapeTest(test.TestCase):
       self.assertListEqual(sp_output.indices.get_shape().as_list(), [5, None])
       self.assertListEqual(sp_output.dense_shape.get_shape().as_list(), [None])
 
+  @test_util.run_deprecated_v1
   def testFeedDenseReshapeSemantics(self):
     with self.session(use_gpu=False) as sess:
       # Compute a random rank-5 initial shape and new shape, randomly sparsify
diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
index 39a9ab9b49..5a48eb825d 100644
--- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
@@ -110,14 +111,17 @@ class SerializeSparseTest(test.TestCase):
       self.assertAllEqual(combined_values[6:], sp_input[1])
       self.assertAllEqual(combined_shape, [2, 5, 6])
 
+  @test_util.run_deprecated_v1
   def testSerializeDeserializeBatch(self):
     self._testSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse,
                                               sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testSerializeDeserializeManyBatch(self):
     self._testSerializeDeserializeBatchHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse)
 
+  @test_util.run_deprecated_v1
   def testVariantSerializeDeserializeBatch(self):
     self._testSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse,
                                               sparse_ops.deserialize_sparse,
@@ -145,10 +149,12 @@ class SerializeSparseTest(test.TestCase):
       self.assertAllEqual(combined_values[6:], sp_input1[1])
       self.assertAllEqual(combined_shape, [2, 5, 6])
 
+  @test_util.run_deprecated_v1
   def testSerializeDeserializeBatchInconsistentShape(self):
     self._testSerializeDeserializeBatchInconsistentShapeHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testVariantSerializeDeserializeBatchInconsistentShape(self):
     self._testSerializeDeserializeBatchInconsistentShapeHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse,
@@ -188,10 +194,12 @@ class SerializeSparseTest(test.TestCase):
 
       self.assertAllEqual(combined_shape, [2, 2, 5, 6])
 
+  @test_util.run_deprecated_v1
   def testSerializeDeserializeNestedBatch(self):
     self._testSerializeDeserializeNestedBatchHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testVariantSerializeDeserializeNestedBatch(self):
     self._testSerializeDeserializeNestedBatchHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse,
@@ -224,14 +232,17 @@ class SerializeSparseTest(test.TestCase):
       self.assertAllEqual(combined_values[6:], input1_val[1])
       self.assertAllEqual(combined_shape, [2, 5, 6])
 
+  @test_util.run_deprecated_v1
   def testFeedSerializeDeserializeBatch(self):
     self._testFeedSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse,
                                                   sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testFeedSerializeDeserializeManyBatch(self):
     self._testFeedSerializeDeserializeBatchHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse)
 
+  @test_util.run_deprecated_v1
   def testFeedVariantSerializeDeserializeBatch(self):
     self._testFeedSerializeDeserializeBatchHelper(sparse_ops.serialize_sparse,
                                                   sparse_ops.deserialize_sparse,
@@ -256,6 +267,7 @@ class SerializeSparseTest(test.TestCase):
           })
       self.assertEqual(serialized_value.shape, (4, 3))
 
+  @test_util.run_deprecated_v1
   def testSerializeManyShape(self):
     self._testSerializeManyShapeHelper(sparse_ops.serialize_many_sparse)
 
@@ -287,19 +299,23 @@ class SerializeSparseTest(test.TestCase):
       self.assertAllEqual(deserialized_value.values, values_value)
       self.assertAllEqual(deserialized_value.dense_shape, shape_value)
 
+  @test_util.run_deprecated_v1
   def testSerializeManyDeserializeBatch(self):
     self._testSerializeManyDeserializeBatchHelper(
         sparse_ops.serialize_many_sparse, sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testSerializeManyDeserializeManyBatch(self):
     self._testSerializeManyDeserializeBatchHelper(
         sparse_ops.serialize_many_sparse, sparse_ops.deserialize_many_sparse)
 
+  @test_util.run_deprecated_v1
   def testVariantSerializeManyDeserializeBatch(self):
     self._testSerializeManyDeserializeBatchHelper(
         sparse_ops.serialize_many_sparse, sparse_ops.deserialize_sparse,
         dtypes.variant)
 
+  @test_util.run_deprecated_v1
   def testVariantSerializeDeserializeScalar(self):
     with self.session(use_gpu=False) as sess:
       indices_value = np.array([[]], dtype=np.int64)
@@ -321,6 +337,7 @@ class SerializeSparseTest(test.TestCase):
       self.assertAllEqual(deserialized_value.values, values_value)
       self.assertAllEqual(deserialized_value.dense_shape, shape_value)
 
+  @test_util.run_deprecated_v1
   def testVariantSerializeDeserializeScalarBatch(self):
     with self.session(use_gpu=False) as sess:
       indices_value = np.array([[]], dtype=np.int64)
@@ -367,14 +384,17 @@ class SerializeSparseTest(test.TestCase):
                  {sp_input0: input0_val,
                   sp_input1: input1_val})
 
+  @test_util.run_deprecated_v1
   def testDeserializeFailsWrongType(self):
     self._testDeserializeFailsWrongTypeHelper(sparse_ops.serialize_sparse,
                                               sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testDeserializeManyFailsWrongType(self):
     self._testDeserializeFailsWrongTypeHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse)
 
+  @test_util.run_deprecated_v1
   def testVariantDeserializeFailsWrongType(self):
     self._testDeserializeFailsWrongTypeHelper(sparse_ops.serialize_sparse,
                                               sparse_ops.deserialize_sparse,
@@ -402,14 +422,17 @@ class SerializeSparseTest(test.TestCase):
                  {sp_input0: input0_val,
                   sp_input1: input1_val})
 
+  @test_util.run_deprecated_v1
   def testDeserializeFailsInconsistentRank(self):
     self._testDeserializeFailsInconsistentRankHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testDeserializeManyFailsInconsistentRank(self):
     self._testDeserializeFailsInconsistentRankHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse)
 
+  @test_util.run_deprecated_v1
   def testVariantDeserializeFailsInconsistentRank(self):
     self._testDeserializeFailsInconsistentRankHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_sparse,
@@ -431,10 +454,12 @@ class SerializeSparseTest(test.TestCase):
       with self.assertRaisesOpError(r"Could not parse serialized proto"):
         sess.run(sp_deserialized, {sp_input0: input0_val})
 
+  @test_util.run_deprecated_v1
   def testDeserializeFailsInvalidProto(self):
     self._testDeserializeFailsInvalidProtoHelper(sparse_ops.serialize_sparse,
                                                  sparse_ops.deserialize_sparse)
 
+  @test_util.run_deprecated_v1
   def testDeserializeManyFailsInvalidProto(self):
     self._testDeserializeFailsInvalidProtoHelper(
         sparse_ops.serialize_sparse, sparse_ops.deserialize_many_sparse)
diff --git a/tensorflow/python/kernel_tests/sparse_slice_op_test.py b/tensorflow/python/kernel_tests/sparse_slice_op_test.py
index 098353741f..7f8c91bde6 100644
--- a/tensorflow/python/kernel_tests/sparse_slice_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_slice_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import sparse_ops
 import tensorflow.python.ops.sparse_grad  # pylint: disable=unused-import
@@ -79,6 +80,7 @@ class SparseSliceOpTest(test.TestCase):
     return sparse_tensor.SparseTensor.from_value(
         self._SparseTensorValue_3x4x2())
 
+  @test_util.run_deprecated_v1
   def testSliceMatrixRows(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_4x6()
@@ -96,6 +98,7 @@ class SparseSliceOpTest(test.TestCase):
                           [20, 23, 25, 30, 32, 33, 35])
       self.assertAllEqual(sp_tensor1.dense_shape.eval(), [2, 6])
 
+  @test_util.run_deprecated_v1
   def testSliceMatrixUnevenCols(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_5x7()
@@ -137,6 +140,7 @@ class SparseSliceOpTest(test.TestCase):
       self.assertAllEqual(sp_tensor3.values.eval(), [16, 46])
       self.assertAllEqual(sp_tensor3.dense_shape.eval(), [5, 1])
 
+  @test_util.run_deprecated_v1
   def testSliceMatrixUnevenRows(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_5x7()
@@ -173,6 +177,7 @@ class SparseSliceOpTest(test.TestCase):
       self.assertAllEqual(sp_tensor2.dense_shape.eval(), [1, 7])
     return
 
+  @test_util.run_deprecated_v1
   def testSliceAllRows(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_4x6()
@@ -195,6 +200,7 @@ class SparseSliceOpTest(test.TestCase):
       self.assertAllEqual(sp_tensor3.values.eval(), [30, 32, 33, 35])
       self.assertAllEqual(sp_tensor3.dense_shape.eval(), [1, 6])
 
+  @test_util.run_deprecated_v1
   def testSliceColumns(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_4x6()
@@ -215,6 +221,7 @@ class SparseSliceOpTest(test.TestCase):
       self.assertAllEqual(sparse_tensor2.values.eval(), [4, 5, 14, 25, 35])
       self.assertAllEqual(sparse_tensor2.dense_shape.eval(), [4, 2])
 
+  @test_util.run_deprecated_v1
   def testSliceAllColumns(self):
     with self.session(use_gpu=False):
       sp_input = self._SparseTensor_4x6()
@@ -246,6 +253,7 @@ class SparseSliceOpTest(test.TestCase):
       self.assertAllEqual(sparse_tensor5.values.eval(), [5, 25, 35])
       self.assertAllEqual(sparse_tensor5.dense_shape.eval(), [4, 1])
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     sp_input = self._SparseTensor_4x6(val_dtype=np.float32)
     start_and_size = [([0, 0], [4, 2]),
diff --git a/tensorflow/python/kernel_tests/sparse_split_op_test.py b/tensorflow/python/kernel_tests/sparse_split_op_test.py
index 95661ded4b..f4bb7498b0 100644
--- a/tensorflow/python/kernel_tests/sparse_split_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_split_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
 
@@ -75,6 +76,7 @@ class SparseSplitOpTest(test.TestCase):
     return sparse_tensor.SparseTensor.from_value(self._SparseTensorValue_3x4x2(
     ))
 
+  @test_util.run_deprecated_v1
   def testSplitMatrixRows(self):
     with self.session(use_gpu=False):
       sp_tensors = sparse_ops.sparse_split(
@@ -92,6 +94,7 @@ class SparseSplitOpTest(test.TestCase):
                           [20, 23, 25, 30, 32, 33, 35])
       self.assertAllEqual(sp_tensors[1].dense_shape.eval(), [2, 6])
 
+  @test_util.run_deprecated_v1
   def testSplitMatrixUnevenCols(self):
     with self.session(use_gpu=False):
       sp_tensors_3 = sparse_ops.sparse_split(
@@ -131,6 +134,7 @@ class SparseSplitOpTest(test.TestCase):
       self.assertAllEqual(sp_tensors_4[3].values.eval(), [16, 46])
       self.assertAllEqual(sp_tensors_4[3].dense_shape.eval(), [5, 1])
 
+  @test_util.run_deprecated_v1
   def testSplitMatrixUnevenRows(self):
     with self.session(use_gpu=False):
       sp_tensors_2 = sparse_ops.sparse_split(
@@ -167,6 +171,7 @@ class SparseSplitOpTest(test.TestCase):
       self.assertAllEqual(sp_tensors_3[2].dense_shape.eval(), [1, 7])
     return
 
+  @test_util.run_deprecated_v1
   def testSplitAllRows(self):
     with self.session(use_gpu=False):
       sp_tensors = sparse_ops.sparse_split(
@@ -189,6 +194,7 @@ class SparseSplitOpTest(test.TestCase):
       self.assertAllEqual(sp_tensors[3].values.eval(), [30, 32, 33, 35])
       self.assertAllEqual(sp_tensors[3].dense_shape.eval(), [1, 6])
 
+  @test_util.run_deprecated_v1
   def testSplitColumns(self):
     with self.session(use_gpu=False):
       sparse_tensors = sparse_ops.sparse_split(
@@ -207,6 +213,7 @@ class SparseSplitOpTest(test.TestCase):
       self.assertAllEqual(sparse_tensors[2].values.eval(), [4, 5, 14, 25, 35])
       self.assertAllEqual(sparse_tensors[2].dense_shape.eval(), [4, 2])
 
+  @test_util.run_deprecated_v1
   def testSplitAllColumns(self):
     with self.session(use_gpu=False):
       sparse_tensors = sparse_ops.sparse_split(
@@ -234,6 +241,7 @@ class SparseSplitOpTest(test.TestCase):
       self.assertAllEqual(sparse_tensors[5].values.eval(), [5, 25, 35])
       self.assertAllEqual(sparse_tensors[5].dense_shape.eval(), [4, 1])
 
+  @test_util.run_deprecated_v1
   def testSliceConcat(self):
     for sp_input in (self._SparseTensorValue_3x4x2(),
                      self._SparseTensor_3x4x2()):
diff --git a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py
index b8f33d6a81..fa2bab1fca 100644
--- a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_grad_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import sparse_ops
 import tensorflow.python.ops.sparse_grad  # pylint: disable=unused-import
@@ -89,6 +90,7 @@ class SparseTensorDenseMatMulGradientTest(test.TestCase):
         self._testGradients(adjoint_a, adjoint_b, name, values_dtype,
                             indices_dtype)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     np.random.seed(5)  # Fix seed to avoid flakiness
     self._testGradientsType(np.float32, np.int64)
diff --git a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py
index e605cb1c35..637cfaec99 100644
--- a/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensor_dense_matmul_op_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -96,6 +97,7 @@ class SparseTensorDenseMatMulTest(test.TestCase):
 
     self._testMatmul(x, y, indices_dtype=indices_dtype)
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     np.random.seed(127)  # Repeatable results
     self._testBasic(np.int32)
@@ -106,6 +108,7 @@ class SparseTensorDenseMatMulTest(test.TestCase):
     self._testBasic(np.int32, indices_dtype=np.int32)
     self._testBasic(np.float32, indices_dtype=np.int32)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     x = np.random.rand(10, 10)
     x[np.abs(x) < 0.5] = 0  # Make it sparse
@@ -229,6 +232,7 @@ class SparseTensorDenseMatMulTest(test.TestCase):
     self._testLarge(np.complex128)
 
   # Tests random sized matrices.
+  @test_util.run_deprecated_v1
   def testFloatRandom(self):
     np.random.seed(127)  # Repeatable results
     for _ in range(8):
diff --git a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
index 538e7c69b5..6039ff1afa 100644
--- a/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_tensors_map_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import variables
@@ -76,6 +77,7 @@ class SparseTensorsMapTest(test.TestCase):
     shape = np.array([3, 4, 5]).astype(np.int64)
     return sparse_tensor_lib.SparseTensorValue(ind, val, shape)
 
+  @test_util.run_deprecated_v1
   def testAddTakeMany(self):
     with self.session(graph=ops.Graph(), use_gpu=False) as sess:
       sp_input0 = self._SparseTensorValue_5x6(np.arange(6))
@@ -98,6 +100,7 @@ class SparseTensorsMapTest(test.TestCase):
       self.assertAllEqual(combined_values[6:], sp_input1[1])
       self.assertAllEqual(combined_shape, [2, 5, 6])
 
+  @test_util.run_deprecated_v1
   def testFeedAddTakeMany(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -125,6 +128,7 @@ class SparseTensorsMapTest(test.TestCase):
       self.assertAllEqual(combined_values[6:], input1_val[1])
       self.assertAllEqual(combined_shape, [2, 5, 6])
 
+  @test_util.run_deprecated_v1
   def testAddManyTakeManyRoundTrip(self):
     with self.session(use_gpu=False) as sess:
       # N == 4 because shape_value == [4, 5]
@@ -147,6 +151,7 @@ class SparseTensorsMapTest(test.TestCase):
       self.assertAllEqual(roundtrip_value.values, values_value)
       self.assertAllEqual(roundtrip_value.dense_shape, shape_value)
 
+  @test_util.run_deprecated_v1
   def testDeserializeFailsInconsistentRank(self):
     with self.session(use_gpu=False) as sess:
       sp_input = self._SparseTensorPlaceholder()
@@ -168,6 +173,7 @@ class SparseTensorsMapTest(test.TestCase):
           r"SparseTensor\[1\] was: 3 but rank of SparseTensor\[1\] is: 4"):
         self.evaluate(sp_roundtrip)
 
+  @test_util.run_deprecated_v1
   def testTakeManyFailsWrongInputOp(self):
     with self.session(use_gpu=False) as sess:
       input_val = self._SparseTensorValue_5x6(np.arange(6))
diff --git a/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py b/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py
index fa6cb13432..c6c45db4f9 100644
--- a/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py
+++ b/tensorflow/python/kernel_tests/sparse_to_dense_op_py_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
@@ -41,36 +42,42 @@ def _SparseToDense(sparse_indices,
 
 class SparseToDenseTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     with self.session(use_gpu=False):
       tf_ans = _SparseToDense([1, 3], [5], 1, 0).eval()
     np_ans = np.array([0, 1, 0, 1, 0]).astype(np.int32)
     self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     with self.session(use_gpu=False):
       tf_ans = _SparseToDense([1, 3], [5], 1.0, 0.0).eval()
     np_ans = np.array([0, 1, 0, 1, 0]).astype(np.float32)
     self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testString(self):
     with self.session(use_gpu=False):
       tf_ans = _SparseToDense([1, 3], [5], "a", "b").eval()
     np_ans = np.array(["b", "a", "b", "a", "b"]).astype(np.string_)
     self.assertAllEqual(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testSetValue(self):
     with self.session(use_gpu=False):
       tf_ans = _SparseToDense([1, 3], [5], [1, 2], -1).eval()
     np_ans = np.array([-1, 1, -1, 2, -1]).astype(np.int32)
     self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testSetSingleValue(self):
     with self.session(use_gpu=False):
       tf_ans = _SparseToDense([1, 3], [5], 1, -1).eval()
     np_ans = np.array([-1, 1, -1, 1, -1]).astype(np.int32)
     self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def test2d(self):
     # pylint: disable=bad-whitespace
     with self.session(use_gpu=False):
@@ -80,11 +87,13 @@ class SparseToDenseTest(test.TestCase):
                        [ 1, -1, -1, -1]]).astype(np.int32)
     self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testZeroDefault(self):
     with self.cached_session():
       x = sparse_ops.sparse_to_dense(2, [4], 7).eval()
       self.assertAllEqual(x, [0, 0, 7, 0])
 
+  @test_util.run_deprecated_v1
   def test3d(self):
     with self.session(use_gpu=False):
       tf_ans = _SparseToDense([[1, 3, 0], [2, 0, 1]], [3, 4, 2], 1, -1).eval()
@@ -93,11 +102,13 @@ class SparseToDenseTest(test.TestCase):
     np_ans[2, 0, 1] = 1
     self.assertAllClose(np_ans, tf_ans)
 
+  @test_util.run_deprecated_v1
   def testBadShape(self):
     with self.cached_session():
       with self.assertRaisesWithPredicateMatch(ValueError, "must be rank 1"):
         _SparseToDense([1, 3], [[5], [3]], 1, -1)
 
+  @test_util.run_deprecated_v1
   def testBadValue(self):
     with self.cached_session():
       dense = _SparseToDense([1, 3], [5], [[5], [3]], -1)
@@ -106,6 +117,7 @@ class SparseToDenseTest(test.TestCase):
           r"should be \[\] or \[2\]"):
         self.evaluate(dense)
 
+  @test_util.run_deprecated_v1
   def testBadNumValues(self):
     with self.cached_session():
       dense = _SparseToDense([1, 3], [5], [1, 2, 3], -1)
@@ -113,12 +125,14 @@ class SparseToDenseTest(test.TestCase):
           r"sparse_values has incorrect shape \[3\], should be \[\] or \[2\]"):
         self.evaluate(dense)
 
+  @test_util.run_deprecated_v1
   def testBadDefault(self):
     with self.cached_session():
       dense = _SparseToDense([1, 3], [5], [1, 2], [0])
       with self.assertRaisesOpError("default_value should be a scalar"):
         self.evaluate(dense)
 
+  @test_util.run_deprecated_v1
   def testOutOfBoundsIndicesWithWithoutValidation(self):
     with self.cached_session():
       dense = _SparseToDense(
@@ -139,6 +153,7 @@ class SparseToDenseTest(test.TestCase):
             validate_indices=False)
         self.evaluate(dense_without_validation)
 
+  @test_util.run_deprecated_v1
   def testRepeatingIndicesWithWithoutValidation(self):
     with self.cached_session():
       dense = _SparseToDense(
@@ -157,6 +172,7 @@ class SparseToDenseTest(test.TestCase):
           validate_indices=False)
       self.evaluate(dense_without_validation)
 
+  @test_util.run_deprecated_v1
   def testUnsortedIndicesWithWithoutValidation(self):
     with self.cached_session():
       dense = _SparseToDense(
@@ -175,6 +191,7 @@ class SparseToDenseTest(test.TestCase):
           validate_indices=False)
       self.evaluate(dense_without_validation)
 
+  @test_util.run_deprecated_v1
   def testShapeInferenceKnownShape(self):
     with self.session(use_gpu=False):
       indices = array_ops.placeholder(dtypes.int64)
@@ -187,6 +204,7 @@ class SparseToDenseTest(test.TestCase):
       output = sparse_ops.sparse_to_dense(indices, shape, 1, 0)
       self.assertEqual(output.get_shape().as_list(), [None, None, None])
 
+  @test_util.run_deprecated_v1
   def testShapeInferenceUnknownShape(self):
     with self.session(use_gpu=False):
       indices = array_ops.placeholder(dtypes.int64)
diff --git a/tensorflow/python/kernel_tests/sparse_xent_op_test.py b/tensorflow/python/kernel_tests/sparse_xent_op_test.py
index cc8c7c238f..8f0842f7f5 100644
--- a/tensorflow/python/kernel_tests/sparse_xent_op_test.py
+++ b/tensorflow/python/kernel_tests/sparse_xent_op_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops as ops_lib
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
@@ -80,6 +81,7 @@ class SparseXentTest(test.TestCase):
       self.assertAllClose([0.0, 0.0, 0.0], tf_loss)
       self.assertAllClose([[0.0], [0.0], [0.0]], tf_backprop)
 
+  @test_util.run_deprecated_v1
   def testInvalidLabel(self):
     features = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
                 [1., 2., 3., 4.]]
@@ -152,6 +154,7 @@ class SparseXentTest(test.TestCase):
         nn_ops.sparse_softmax_cross_entropy_with_logits(
             labels=constant_op.constant(0), logits=constant_op.constant(1.0))
 
+  @test_util.run_deprecated_v1
   def testLabelsPlaceholderScalar(self):
     with self.session(use_gpu=True):
       labels = array_ops.placeholder(np.int32)
@@ -187,6 +190,7 @@ class SparseXentTest(test.TestCase):
   def testEmpty(self):
     self._testXent(np.zeros((0, 3)), np.zeros((0,), dtype=np.int32))
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.session(use_gpu=True):
       l = constant_op.constant([3, 0, 1], name="l")
@@ -201,6 +205,7 @@ class SparseXentTest(test.TestCase):
     print("cross entropy gradient err = ", err)
     self.assertLess(err, 5e-8)
 
+  @test_util.run_deprecated_v1
   def testSecondGradient(self):
     images_placeholder = array_ops.placeholder(dtypes.float32, shape=(3, 2))
     labels_placeholder = array_ops.placeholder(dtypes.int32, shape=(3))
@@ -230,17 +235,20 @@ class SparseXentTest(test.TestCase):
     self.assertAllCloseAccordingToType(np_loss, tf_loss)
     self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
 
+  @test_util.run_deprecated_v1
   def testHighDim(self):
     features = [[[1., 1., 1., 1.]], [[1., 2., 3., 4.]]]
     labels = [[3], [0]]
     self._testHighDim(features, labels)
 
+  @test_util.run_deprecated_v1
   def testHighDim2(self):
     features = [[[1., 1., 1., 1.], [2., 2., 2., 2.]],
                 [[1., 2., 3., 4.], [5., 6., 7., 8.]]]
     labels = [[3, 2], [0, 3]]
     self._testHighDim(features, labels)
 
+  @test_util.run_deprecated_v1
   def testScalarHandling(self):
     with self.session(use_gpu=False) as sess:
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
@@ -318,7 +326,7 @@ def sparse_vs_dense_xent_benchmark(batch_size, num_entries, use_gpu):
   # Using sparse_softmax_cross_entropy_with_logits
   with session.Session(config=config) as sess:
     if not use_gpu:
-      with ops_lib.device("/cpu:0"):
+      with test_util.device("/cpu:0"):
         ops = _sparse_vs_dense_xent_benchmark_sparse(labels, logits)
     else:
       ops = _sparse_vs_dense_xent_benchmark_sparse(labels, logits)
diff --git a/tensorflow/python/kernel_tests/sparsemask_op_test.py b/tensorflow/python/kernel_tests/sparsemask_op_test.py
index 6f5dd45b61..b1cd0227bc 100644
--- a/tensorflow/python/kernel_tests/sparsemask_op_test.py
+++ b/tensorflow/python/kernel_tests/sparsemask_op_test.py
@@ -20,12 +20,14 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class SparseMaskTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     values = np.random.rand(4, 4).astype(np.single)
     indices = np.array([0, 2, 3, 4], dtype=np.int32)
diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py
index af90e03966..517db3450f 100644
--- a/tensorflow/python/kernel_tests/split_op_test.py
+++ b/tensorflow/python/kernel_tests/split_op_test.py
@@ -42,6 +42,7 @@ class SplitOpTest(test.TestCase):
       data -= 1j * data
     return data
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     model_input = array_ops.placeholder(dtypes.float32, shape=(1, 10))
 
@@ -85,6 +86,7 @@ class SplitOpTest(test.TestCase):
     with self.cached_session(use_gpu=True) as sess:
       sess.run(result, feed_dict={model_input2: np.ones([4, 2])})
 
+  @test_util.run_deprecated_v1
   def testFailWithoutExplicitNum(self):
     size_splits = array_ops.placeholder(dtype=dtypes.int32, shape=[None])
 
@@ -209,6 +211,7 @@ class SplitOpTest(test.TestCase):
     self.assertAllEqual(result[:, 0:1], inp_grads[0])
     self.assertAllEqual(result[:, 1:4], inp_grads[1])
 
+  @test_util.run_deprecated_v1
   def testOutputShape(self):
     for axis in [1, -1]:
       with self.cached_session(use_gpu=True):
@@ -322,11 +325,13 @@ class SplitOpTest(test.TestCase):
     for i in range(4):
       self.assertAllEqual(result[:, i:i + 1], inp_grads[i])
 
+  @test_util.run_deprecated_v1
   def testGradientsAll(self):
     for dtype in _TEST_DTYPES:
       self._testGradientsSimple(dtype)
       self._testGradientsSimpleVariable(dtype)
 
+  @test_util.run_deprecated_v1
   def testShapeFunctionEdgeCases(self):
     # split_dim greater than rank of input.
     with self.assertRaises(ValueError):
@@ -356,6 +361,7 @@ class SplitOpTest(test.TestCase):
     for s in splits:
       self.assertEqual(None, s.get_shape().ndims)
 
+  @test_util.run_deprecated_v1
   def testVariableShapeFunction(self):
     # size_splits too big
     with self.assertRaises(ValueError):
@@ -366,6 +372,7 @@ class SplitOpTest(test.TestCase):
     assert s0.shape.as_list() == [2]
     assert s1.shape.as_list() == [1]
 
+  @test_util.run_deprecated_v1
   def testNonexistentDimTensor(self):
     x = array_ops.placeholder(dtypes.int32)
     values = np.zeros([5, 30])
diff --git a/tensorflow/python/kernel_tests/stack_op_test.py b/tensorflow/python/kernel_tests/stack_op_test.py
index 0f1fa97c38..ca3357a0ed 100644
--- a/tensorflow/python/kernel_tests/stack_op_test.py
+++ b/tensorflow/python/kernel_tests/stack_op_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import variables
@@ -41,6 +42,7 @@ def np_split_squeeze(array, axis):
 
 class StackOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     np.random.seed(7)
     with self.session(use_gpu=True):
@@ -54,6 +56,7 @@ class StackOpTest(test.TestCase):
           c = array_ops.stack(xs)
           self.assertAllEqual(c.eval(), data)
 
+  @test_util.run_deprecated_v1
   def testSimpleParallelCPU(self):
     np.random.seed(7)
     with self.session(use_gpu=False):
@@ -63,6 +66,7 @@ class StackOpTest(test.TestCase):
         c = array_ops.parallel_stack(xs)
         self.assertAllEqual(c.eval(), data)
 
+  @test_util.run_deprecated_v1
   def testSimpleParallelGPU(self):
     np.random.seed(7)
     with self.session(use_gpu=True):
@@ -72,6 +76,7 @@ class StackOpTest(test.TestCase):
         c = array_ops.parallel_stack(xs)
         self.assertAllEqual(c.eval(), data)
 
+  @test_util.run_deprecated_v1
   def testConst(self):
     np.random.seed(7)
     with self.session(use_gpu=True):
@@ -96,6 +101,7 @@ class StackOpTest(test.TestCase):
         b = array_ops.reshape(a, array_ops.stack([2, 3]))
         self.assertAllEqual(b.get_shape(), [2, 3])
 
+  @test_util.run_deprecated_v1
   def testConstParallelCPU(self):
     np.random.seed(7)
     with self.session(use_gpu=False):
@@ -110,6 +116,7 @@ class StackOpTest(test.TestCase):
         c = array_ops.parallel_stack(data)
         self.assertAllEqual(c.eval(), data)
 
+  @test_util.run_deprecated_v1
   def testConstParallelGPU(self):
     np.random.seed(7)
     with self.session(use_gpu=True):
@@ -124,6 +131,7 @@ class StackOpTest(test.TestCase):
         c = array_ops.parallel_stack(data)
         self.assertAllEqual(c.eval(), data)
 
+  @test_util.run_deprecated_v1
   def testGradientsAxis0(self):
     np.random.seed(7)
     for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
@@ -136,6 +144,7 @@ class StackOpTest(test.TestCase):
         err = gradient_checker.compute_gradient_error(xs, shapes, c, shape)
         self.assertLess(err, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testGradientsAxis1(self):
     np.random.seed(7)
     for shape in (2, 3), (3, 2), (4, 3, 2):
@@ -150,6 +159,7 @@ class StackOpTest(test.TestCase):
         err = gradient_checker.compute_gradient_error(xs, shapes, c, out_shape)
         self.assertLess(err, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testZeroSizeCPU(self):
     # Verify that stack doesn't crash for zero size inputs
     with self.session(use_gpu=False):
@@ -161,6 +171,7 @@ class StackOpTest(test.TestCase):
         p = array_ops.parallel_stack(list(x)).eval()
         self.assertAllEqual(p, x)
 
+  @test_util.run_deprecated_v1
   def testZeroSizeGPU(self):
     # Verify that stack doesn't crash for zero size inputs
     with self.session(use_gpu=True):
@@ -172,6 +183,7 @@ class StackOpTest(test.TestCase):
         p = array_ops.parallel_stack(list(x)).eval()
         self.assertAllEqual(p, x)
 
+  @test_util.run_deprecated_v1
   def testAxis0DefaultCPU(self):
     with self.session(use_gpu=False):
       t = [constant_op.constant([1, 2, 3]), constant_op.constant([4, 5, 6])]
@@ -182,6 +194,7 @@ class StackOpTest(test.TestCase):
     self.assertAllEqual(stacked, expected)
     self.assertAllEqual(parallel_stacked, expected)
 
+  @test_util.run_deprecated_v1
   def testAxis0DefaultGPU(self):
     with self.session(use_gpu=True):
       t = [constant_op.constant([1, 2, 3]), constant_op.constant([4, 5, 6])]
@@ -225,6 +238,7 @@ class StackOpTest(test.TestCase):
 
 class AutomaticStackingTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     with self.session(use_gpu=True):
       self.assertAllEqual(
@@ -255,6 +269,7 @@ class AutomaticStackingTest(test.TestCase):
       self.assertAllEqual([[[0., 0.], [1., 1.]], [[2., 2.], [3., 3.]]],
                           self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def testVariable(self):
     with self.session(use_gpu=True):
       v = variables.Variable(17)
@@ -308,6 +323,7 @@ class AutomaticStackingTest(test.TestCase):
     t_2 = ops.convert_to_tensor([t_0, t_0, t_1], dtype=dtypes.float64)
     self.assertEqual(dtypes.float64, t_2.dtype)
 
+  @test_util.run_deprecated_v1
   def testPlaceholder(self):
     with self.session(use_gpu=True):
       # Test using placeholder with a defined shape.
@@ -326,6 +342,7 @@ class AutomaticStackingTest(test.TestCase):
       self.assertAllEqual(
           [[0, 0, 0], [0, 2, 0], [0, 0, 0]], result_1.eval(feed_dict={ph_1: 2}))
 
+  @test_util.run_deprecated_v1
   def testShapeErrors(self):
     # Static shape error.
     ph_0 = array_ops.placeholder(dtypes.int32, shape=[1])
diff --git a/tensorflow/python/kernel_tests/stack_ops_test.py b/tensorflow/python/kernel_tests/stack_ops_test.py
index dffb260b5f..d50f3f4680 100644
--- a/tensorflow/python/kernel_tests/stack_ops_test.py
+++ b/tensorflow/python/kernel_tests/stack_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -41,6 +42,7 @@ class StackOpTest(test.TestCase):
         c1 = gen_data_flow_ops.stack_pop_v2(h, dtypes.float32)
       self.assertAllClose([[4.0, 5.0]], self.evaluate(c1))
 
+  @test_util.run_deprecated_v1
   def testStackPushPop(self):
     self._testStackPushPop(use_gpu=False)
     self._testStackPushPop(use_gpu=True)
@@ -56,6 +58,7 @@ class StackOpTest(test.TestCase):
         c1 = gen_data_flow_ops.stack_pop_v2(h, dtypes.float32)
       self.assertAllClose(a, self.evaluate(c1))
 
+  @test_util.run_deprecated_v1
   def testStackPushPopSwap(self):
     self._testStackPushPopSwap(use_gpu=False)
     self._testStackPushPopSwap(use_gpu=True)
@@ -93,6 +96,7 @@ class StackOpTest(test.TestCase):
           c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()])
       self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry))
 
+  @test_util.run_deprecated_v1
   def testStackWhileSwap(self):
     self._testStackWhileSwap(use_gpu=False)
     self._testStackWhileSwap(use_gpu=True)
@@ -112,6 +116,7 @@ class StackOpTest(test.TestCase):
       r = c1 + c2
       self.assertAllClose(9.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testMultiStack(self):
     self._testMultiStack(use_gpu=False)
     self._testMultiStack(use_gpu=True)
@@ -135,6 +140,7 @@ class StackOpTest(test.TestCase):
       self.assertAllClose(out1, 4.0)
       self.assertAllClose(out2, 5.0)
 
+  @test_util.run_deprecated_v1
   def testSameNameStacks(self):
     self._testSameNameStacks(use_gpu=False)
     self._testSameNameStacks(use_gpu=True)
@@ -146,6 +152,7 @@ class StackOpTest(test.TestCase):
       c1 = gen_data_flow_ops.stack_close_v2(h)
       self.evaluate(c1)
 
+  @test_util.run_deprecated_v1
   def testCloseStack(self):
     self._testCloseStack(use_gpu=False)
     self._testCloseStack(use_gpu=True)
@@ -159,6 +166,7 @@ class StackOpTest(test.TestCase):
         c1 = gen_data_flow_ops.stack_close_v2(h)
       self.evaluate(c1)
 
+  @test_util.run_deprecated_v1
   def testPushCloseStack(self):
     self._testPushCloseStack(use_gpu=False)
     self._testPushCloseStack(use_gpu=True)
@@ -175,6 +183,7 @@ class StackOpRefTest(test.TestCase):
         c1 = gen_data_flow_ops.stack_pop(h, dtypes.float32)
       self.assertAllClose([[4.0, 5.0]], self.evaluate(c1))
 
+  @test_util.run_deprecated_v1
   def testStackPushPop(self):
     self._testStackPushPop(use_gpu=False)
     self._testStackPushPop(use_gpu=True)
@@ -189,6 +198,7 @@ class StackOpRefTest(test.TestCase):
         c1 = gen_data_flow_ops.stack_pop(h, dtypes.float32)
       self.assertAllClose(a, self.evaluate(c1))
 
+  @test_util.run_deprecated_v1
   def testStackPushPopSwap(self):
     self._testStackPushPopSwap(use_gpu=False)
     self._testStackPushPopSwap(use_gpu=True)
@@ -238,10 +248,12 @@ class StackOpRefTest(test.TestCase):
           c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()])
       self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry))
 
+  @test_util.run_deprecated_v1
   def testStackWhileSwap(self):
     self._testStackWhileSwap(use_gpu=False)
     self._testStackWhileSwap(use_gpu=True)
 
+  @test_util.run_deprecated_v1
   def testMultiStack(self):
     self._testMultiStack(use_gpu=False)
     self._testMultiStack(use_gpu=True)
@@ -255,6 +267,7 @@ class StackOpRefTest(test.TestCase):
       _ = c1 + c2
       self.assertNotEqual(h1.eval()[1], self.evaluate(h2)[1])
 
+  @test_util.run_deprecated_v1
   def testSameNameStacks(self):
     self._testSameNameStacks(use_gpu=False)
     self._testSameNameStacks(use_gpu=True)
@@ -265,6 +278,7 @@ class StackOpRefTest(test.TestCase):
       c1 = gen_data_flow_ops.stack_close(h)
       self.evaluate(c1)
 
+  @test_util.run_deprecated_v1
   def testCloseStack(self):
     self._testCloseStack(use_gpu=False)
     self._testCloseStack(use_gpu=True)
@@ -277,6 +291,7 @@ class StackOpRefTest(test.TestCase):
         c1 = gen_data_flow_ops.stack_close(h)
       self.evaluate(c1)
 
+  @test_util.run_deprecated_v1
   def testPushCloseStack(self):
     self._testPushCloseStack(use_gpu=False)
     self._testPushCloseStack(use_gpu=True)
diff --git a/tensorflow/python/kernel_tests/stage_op_test.py b/tensorflow/python/kernel_tests/stage_op_test.py
index b814843b86..83e06ba48b 100644
--- a/tensorflow/python/kernel_tests/stage_op_test.py
+++ b/tensorflow/python/kernel_tests/stage_op_test.py
@@ -18,6 +18,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -28,6 +29,7 @@ TIMEOUT = 1
 
 class StageTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSimple(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -47,6 +49,7 @@ class StageTest(test.TestCase):
         _, yval = sess.run([stage, y], feed_dict={x: i})
         self.assertAllClose(4 * (i - 1) * (i - 1) * 128, yval, rtol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testMultiple(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -67,6 +70,7 @@ class StageTest(test.TestCase):
         self.assertAllClose(
             4 * (i - 1) * (i - 1) * (i - 1) * 128, yval, rtol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testDictionary(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -110,6 +114,7 @@ class StageTest(test.TestCase):
 
     G.finalize()
 
+  @test_util.run_deprecated_v1
   def testPeek(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -133,6 +138,7 @@ class StageTest(test.TestCase):
       for i in range(10):
         self.assertTrue(sess.run(peek, feed_dict={p: i}) == [i])
 
+  @test_util.run_deprecated_v1
   def testSizeAndClear(self):
     with ops.Graph().as_default() as G:
       with ops.device('/cpu:0'):
@@ -158,6 +164,7 @@ class StageTest(test.TestCase):
       sess.run(clear)
       self.assertEqual(sess.run(size), 0)
 
+  @test_util.run_deprecated_v1
   def testCapacity(self):
     capacity = 3
 
@@ -219,6 +226,7 @@ class StageTest(test.TestCase):
       # It should now be empty
       self.assertTrue(sess.run(size) == 0)
 
+  @test_util.run_deprecated_v1
   def testMemoryLimit(self):
     memory_limit = 512 * 1024  # 512K
     chunk = 200 * 1024  # 256K
diff --git a/tensorflow/python/kernel_tests/string_join_op_test.py b/tensorflow/python/kernel_tests/string_join_op_test.py
index e4371ab5b9..2548e8695f 100644
--- a/tensorflow/python/kernel_tests/string_join_op_test.py
+++ b/tensorflow/python/kernel_tests/string_join_op_test.py
@@ -17,12 +17,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
 
 class StringJoinOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testStringJoin(self):
     input0 = ["a", "b"]
     input1 = "a"
diff --git a/tensorflow/python/kernel_tests/string_length_op_test.py b/tensorflow/python/kernel_tests/string_length_op_test.py
index 06bf28ebce..bfa6ac2454 100644
--- a/tensorflow/python/kernel_tests/string_length_op_test.py
+++ b/tensorflow/python/kernel_tests/string_length_op_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
@@ -32,6 +33,7 @@ class StringLengthOpTest(test.TestCase):
       values = self.evaluate(lengths)
       self.assertAllEqual(values, [[[1, 2], [3, 4], [5, 6]]])
 
+  @test_util.run_deprecated_v1
   def testUnit(self):
     unicode_strings = [u"H\xc3llo", u"\U0001f604"]
     utf8_strings = [s.encode("utf-8") for s in unicode_strings]
@@ -51,6 +53,7 @@ class StringLengthOpTest(test.TestCase):
           'not in: "BYTE", "UTF8_CHAR"'):
         string_ops.string_length(utf8_strings, unit="XYZ")
 
+  @test_util.run_deprecated_v1
   def testLegacyPositionalName(self):
     # Code that predates the 'unit' parameter may have used a positional
     # argument for the 'name' parameter.  Check that we don't break such code.
diff --git a/tensorflow/python/kernel_tests/string_split_op_test.py b/tensorflow/python/kernel_tests/string_split_op_test.py
index 92e13db0f7..0c91deb522 100644
--- a/tensorflow/python/kernel_tests/string_split_op_test.py
+++ b/tensorflow/python/kernel_tests/string_split_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
@@ -39,6 +40,7 @@ class StringSplitOpTest(test.TestCase):
       self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
       self.assertAllEqual(shape, [2, 4])
 
+  @test_util.run_deprecated_v1
   def testStringSplitEmptyDelimiter(self):
     strings = ["hello", "hola", b"\xF0\x9F\x98\x8E"]  # Last string is U+1F60E
 
@@ -81,6 +83,7 @@ class StringSplitOpTest(test.TestCase):
       self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e", b"f", b"g"])
       self.assertAllEqual(shape, [10, 1])
 
+  @test_util.run_deprecated_v1
   def testStringSplitWithDelimiter(self):
     strings = ["hello|world", "hello world"]
 
@@ -103,6 +106,7 @@ class StringSplitOpTest(test.TestCase):
       self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"])
       self.assertAllEqual(shape, [2, 2])
 
+  @test_util.run_deprecated_v1
   def testStringSplitWithDelimiterTensor(self):
     strings = ["hello|world", "hello world"]
 
@@ -121,6 +125,7 @@ class StringSplitOpTest(test.TestCase):
       self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
       self.assertAllEqual(shape, [2, 2])
 
+  @test_util.run_deprecated_v1
   def testStringSplitWithDelimitersTensor(self):
     strings = ["hello.cruel,world", "hello cruel world"]
 
diff --git a/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py b/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py
index 2cc87008da..25f573fc14 100644
--- a/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py
+++ b/tensorflow/python/kernel_tests/string_to_hash_bucket_op_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
@@ -26,6 +27,7 @@ from tensorflow.python.platform import test
 
 class StringToHashBucketOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testStringToOneHashBucketFast(self):
     with self.cached_session():
       input_string = array_ops.placeholder(dtypes.string)
@@ -34,6 +36,7 @@ class StringToHashBucketOpTest(test.TestCase):
 
       self.assertAllEqual([0, 0, 0], result)
 
+  @test_util.run_deprecated_v1
   def testStringToHashBucketsFast(self):
     with self.cached_session():
       input_string = array_ops.placeholder(dtypes.string)
@@ -46,6 +49,7 @@ class StringToHashBucketOpTest(test.TestCase):
       # Fingerprint64('d') -> 4470636696479570465 -> mod 10 -> 5
       self.assertAllEqual([9, 2, 2, 5], result)
 
+  @test_util.run_deprecated_v1
   def testStringToOneHashBucketLegacyHash(self):
     with self.cached_session():
       input_string = array_ops.placeholder(dtypes.string)
@@ -54,6 +58,7 @@ class StringToHashBucketOpTest(test.TestCase):
 
       self.assertAllEqual([0, 0, 0], result)
 
+  @test_util.run_deprecated_v1
   def testStringToHashBucketsLegacyHash(self):
     with self.cached_session():
       input_string = array_ops.placeholder(dtypes.string)
diff --git a/tensorflow/python/kernel_tests/string_to_number_op_test.py b/tensorflow/python/kernel_tests/string_to_number_op_test.py
index 99ee25e125..49ccfd1028 100644
--- a/tensorflow/python/kernel_tests/string_to_number_op_test.py
+++ b/tensorflow/python/kernel_tests/string_to_number_op_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
@@ -45,6 +46,7 @@ class StringToNumberOpTest(test.TestCase):
         with self.assertRaisesOpError(outstr):
           output.eval(feed_dict={input_string: [instr]})
 
+  @test_util.run_deprecated_v1
   def testToFloat(self):
     self._test(dtypes.float32,
                [("0", 0), ("3", 3), ("-1", -1),
@@ -58,6 +60,7 @@ class StringToNumberOpTest(test.TestCase):
                 ("INF", float("INF"))],
                [("10foobar", _ERROR_MESSAGE + "10foobar")])
 
+  @test_util.run_deprecated_v1
   def testToDouble(self):
     self._test(dtypes.float64,
                [("0", 0), ("3", 3), ("-1", -1),
@@ -71,6 +74,7 @@ class StringToNumberOpTest(test.TestCase):
                 ("INF", float("INF"))],
                [("10foobar", _ERROR_MESSAGE + "10foobar")])
 
+  @test_util.run_deprecated_v1
   def testToInt32(self):
     self._test(dtypes.int32,
                [("0", 0), ("3", 3), ("-1", -1),
@@ -84,6 +88,7 @@ class StringToNumberOpTest(test.TestCase):
                    ("2.9", _ERROR_MESSAGE + "2.9"),
                    ("10foobar", _ERROR_MESSAGE + "10foobar")])
 
+  @test_util.run_deprecated_v1
   def testToInt64(self):
     self._test(dtypes.int64,
                [("0", 0), ("3", 3), ("-1", -1),
diff --git a/tensorflow/python/kernel_tests/substr_op_test.py b/tensorflow/python/kernel_tests/substr_op_test.py
index bb2d4a7913..9302152e82 100644
--- a/tensorflow/python/kernel_tests/substr_op_test.py
+++ b/tensorflow/python/kernel_tests/substr_op_test.py
@@ -22,6 +22,7 @@ from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
@@ -319,6 +320,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       (np.int32, "UTF8_CHAR"),
       (np.int64, "UTF8_CHAR"),
   )
+  @test_util.run_deprecated_v1
   def testBadBroadcast(self, dtype, unit):
     test_string = [[b"ten", b"eleven", b"twelve"],
                    [b"thirteen", b"fourteen", b"fifteen"],
@@ -338,6 +340,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       (np.int32, -6, "UTF8_CHAR"),
       (np.int64, -6, "UTF8_CHAR"),
   )
+  @test_util.run_deprecated_v1
   def testOutOfRangeError_Scalar(self, dtype, pos, unit):
     # Scalar/Scalar
     test_string = {
@@ -361,6 +364,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       (np.int32, -4, "UTF8_CHAR"),
       (np.int64, -4, "UTF8_CHAR"),
   )
+  @test_util.run_deprecated_v1
   def testOutOfRangeError_VectorScalar(self, dtype, pos, unit):
     # Vector/Scalar
     test_string = {
@@ -381,6 +385,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       (np.int32, "UTF8_CHAR"),
       (np.int64, "UTF8_CHAR"),
   )
+  @test_util.run_deprecated_v1
   def testOutOfRangeError_MatrixMatrix(self, dtype, unit):
     # Matrix/Matrix
     test_string = {
@@ -414,6 +419,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       (np.int32, "UTF8_CHAR"),
       (np.int64, "UTF8_CHAR"),
   )
+  @test_util.run_deprecated_v1
   def testOutOfRangeError_Broadcast(self, dtype, unit):
     # Broadcast
     test_string = {
@@ -444,6 +450,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       (np.int32, "UTF8_CHAR"),
       (np.int64, "UTF8_CHAR"),
   )
+  @test_util.run_deprecated_v1
   def testMismatchPosLenShapes(self, dtype, unit):
     test_string = {
         "BYTE": [[b"ten", b"eleven", b"twelve"],
@@ -471,6 +478,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
     with self.assertRaises(ValueError):
       string_ops.substr(test_string, position, length)
 
+  @test_util.run_deprecated_v1
   def testWrongDtype(self):
     with self.cached_session():
       with self.assertRaises(TypeError):
@@ -478,6 +486,7 @@ class SubstrOpTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(TypeError):
         string_ops.substr(b"test", 3, 1.0)
 
+  @test_util.run_deprecated_v1
   def testInvalidUnit(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
diff --git a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
index e1b24756f3..56de2e933d 100644
--- a/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_image_op_test.py
@@ -24,6 +24,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.core.framework import summary_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import image_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
@@ -49,6 +50,7 @@ class SummaryV1ImageOpTest(test.TestCase):
         }""" % ((i,) + shape[1:]) for i in xrange(3))
     self.assertProtoEquals(expected, image_summ)
 
+  @test_util.run_deprecated_v1
   def testImageSummary(self):
     for depth in (1, 3, 4):
       for positive in False, True:
@@ -84,6 +86,7 @@ class SummaryV1ImageOpTest(test.TestCase):
           # Check the rest of the proto
           self._CheckProto(image_summ, shape)
 
+  @test_util.run_deprecated_v1
   def testImageSummaryUint8(self):
     np.random.seed(7)
     for depth in (1, 3, 4):
diff --git a/tensorflow/python/kernel_tests/summary_v1_ops_test.py b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
index 1206cb7013..e070f5bf6f 100644
--- a/tensorflow/python/kernel_tests/summary_v1_ops_test.py
+++ b/tensorflow/python/kernel_tests/summary_v1_ops_test.py
@@ -26,6 +26,7 @@ from __future__ import print_function
 from tensorflow.core.framework import summary_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import logging_ops
 from tensorflow.python.platform import test
 from tensorflow.python.summary import summary
@@ -60,6 +61,7 @@ class SummaryV1OpsTest(test.TestCase):
       value { tag: "c2" simple_value: 20.0 }
       """, self._AsSummary(value))
 
+  @test_util.run_deprecated_v1
   def testMergeSummary(self):
     with self.cached_session() as sess:
       const = constant_op.constant(10.0)
diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py
index a187fa115c..3b2a56bd1f 100644
--- a/tensorflow/python/kernel_tests/template_test.py
+++ b/tensorflow/python/kernel_tests/template_test.py
@@ -72,6 +72,7 @@ def variable_scoped_function_with_local_variable():
 
 class TemplateTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_end_to_end(self):
     """This test shows a very simple line model with test_loss.
 
@@ -172,6 +173,7 @@ class TemplateTest(test.TestCase):
     self.assertEqual("s1/dummy:0", v1.name)
     self.assertEqual("s1_1/dummy:0", v3.name)
 
+  @test_util.run_deprecated_v1
   def test_same_unique_name_raise_error(self):
     tmpl1 = template.make_template(
         "_", variable_scoped_function, unique_name_="s1")
@@ -190,6 +192,7 @@ class TemplateTest(test.TestCase):
         template.make_template(
             "_", variable_scoped_function, unique_name_="s1")
 
+  @test_util.run_deprecated_v1
   def test_unique_name_and_reuse(self):
     tmpl1 = template.make_template(
         "_", variable_scoped_function, unique_name_="s1")
@@ -260,6 +263,7 @@ class TemplateTest(test.TestCase):
     self.assertEqual("s1/test/dummy:0", v1.name)
     self.assertEqual("s1_1/test/dummy:0", v3.name)
 
+  @test_util.run_deprecated_v1
   def test_enforces_no_extra_trainable_variables(self):
     tmpl = template.make_template("s", function_with_create, trainable=True)
 
@@ -675,6 +679,7 @@ class TemplateTest(test.TestCase):
     self.assertEqual(1, len(tb.variables))
 
   # TODO(apassos) handle local variables in Eager
+  @test_util.run_deprecated_v1
   def test_local_variables(self):
     # Make sure trainable_variables are created.
     with variable_scope.variable_scope("foo3"):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index bb8645e2d5..76e90ffea3 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -165,6 +165,7 @@ class TensorArrayTest(test.TestCase):
                    [106.0, 107.0], [8.0, 9.0]]), c0)
 
   @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  @test_util.run_deprecated_v1
   def testTensorArrayWriteConcat(self):
     self._testTensorArrayWriteConcat(dtypes.float32)
     self._testTensorArrayWriteConcat(dtypes.float64)
@@ -188,6 +189,7 @@ class TensorArrayTest(test.TestCase):
                           self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
   @test_util.disable_control_flow_v2("b/118890905")
+  @test_util.run_deprecated_v1
   def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros()
 
@@ -204,6 +206,7 @@ class TensorArrayTest(test.TestCase):
                         self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
   @test_util.disable_control_flow_v2("b/118890905")
+  @test_util.run_deprecated_v1
   def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros()
 
@@ -303,6 +306,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(convert([[3.0, 301.0]]), d2)
 
   @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  @test_util.run_deprecated_v1
   def testTensorArraySplitRead(self):
     self._testTensorArraySplitRead(dtypes.float32)
     self._testTensorArraySplitRead(dtypes.float64)
@@ -313,6 +317,7 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArraySplitRead(dtypes.string)
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorGradArrayWriteRead(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -346,6 +351,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[2.0]], g_d1)
       self.assertAllEqual(-2.0, g_d2)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradGrad(self):
     if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2:
       self.skipTest("Legacy TensorArray does not support double derivatives.")
@@ -366,6 +372,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([2.0], session.run(g2))
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorGradArrayDynamicWriteRead(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -408,6 +415,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(3, g_vs)
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorGradAccessTwiceReceiveSameObject(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -424,6 +432,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(t_g_ta_0, t_g_ta_1)
       self.assertAllEqual([[4.0, 5.0]], d_r1_0)
 
+  @test_util.run_deprecated_v1
   def testTensorArrayWriteWrongIndexOrDataTypeFails(self):
     with self.session(use_gpu=True):
       ta = _make_ta(3, "foo", dtype=dtypes.float32)
@@ -457,6 +466,7 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.write(3, 3.0).flow)
 
+  @test_util.run_deprecated_v1
   def testTensorArrayReadWrongIndexOrDataTypeFails(self):
     with self.session(use_gpu=True):
       ta = _make_ta(3, "foo", dtype=dtypes.float32)
@@ -491,6 +501,7 @@ class TensorArrayTest(test.TestCase):
         self.evaluate(ta.read(3))
 
   @test_util.disable_control_flow_v2("v2 allows multiple writes.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayWriteMultipleFails(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -502,6 +513,7 @@ class TensorArrayTest(test.TestCase):
         self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow)
 
   @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  @test_util.run_deprecated_v1
   def testTensorArrayConcatIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -534,6 +546,7 @@ class TensorArrayTest(test.TestCase):
         self.evaluate(w3.concat())
 
   @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  @test_util.run_deprecated_v1
   def testTensorArraySplitIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
       in_eager_mode = context.executing_eagerly()
@@ -598,12 +611,14 @@ class TensorArrayTest(test.TestCase):
         wb1_grad.flow.eval()
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayWriteGradientAddMultipleAdds(self):
     for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
                   dtypes.complex64, dtypes.complex128):
       self._testTensorArrayWriteGradientAddMultipleAdds(dtype)
 
   @test_util.disable_control_flow_v2("Low level legacy TA op test.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradWithShapeKnownElementShape(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -634,6 +649,7 @@ class TensorArrayTest(test.TestCase):
                           sess.run(read_value, feed_dict={value: fed_value}))
 
   @test_util.disable_control_flow_v2("Low level legacy TA op test.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradWithShapeUnknownElementShape(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -720,6 +736,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(c([[3.0, 2.0]]), grad_vals[0])
       self.assertAllEqual(c(-2.0), grad_vals[1])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientWriteRead(self):
     for dtype in (np.float32, np.float64, np.complex64, np.complex128):
       self._testTensorArrayGradientWriteReadType(dtype)
@@ -757,10 +774,12 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
 
   @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientWritePackConcatAndRead(self):
     self._testTensorArrayGradientWritePackConcatAndRead()
 
   @test_util.disable_control_flow_v2("v2 does not support clear_after_read.")
+  @test_util.run_deprecated_v1
   def testTensorArrayReadTwice(self):
     with self.session(use_gpu=True):
       value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
@@ -814,10 +833,12 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(len(grad_vals), 1)
       self.assertAllEqual([[2.0 - 1.5, 3.0 + 1.5], [4.0, 5.0]], grad_vals[0])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientUnpackRead(self):
     self._testTensorArrayGradientUnpackRead()
 
   @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientSplitConcat(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -863,6 +884,7 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(len(grad_vals), 1)
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientDynamicUnpackRead(self):
     self._testTensorArrayGradientDynamicUnpackRead()
 
@@ -1043,6 +1065,7 @@ class TensorArrayTest(test.TestCase):
         grad = gradients_impl.gradients(loop(x), [x])[0]
       self.assertAllClose(31.0, self.evaluate(grad))
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSumOfTwoReadVariablesWithoutRepeatGrad(self):
     with self.session(use_gpu=True) as session:
       a = array_ops.identity(
@@ -1078,6 +1101,7 @@ class TensorArrayTest(test.TestCase):
   def _grad_source_for_name(self, name):
     return tensor_array_grad._GetGradSource(constant_op.constant(0, name=name))
 
+  @test_util.run_deprecated_v1
   def testSkipEagerGetGradSource_Invalid(self):
     with self.assertRaises(ValueError):
       self._grad_source_for_name("")
@@ -1086,6 +1110,7 @@ class TensorArrayTest(test.TestCase):
     with self.assertRaises(ValueError):
       self._grad_source_for_name("foo/bar")
 
+  @test_util.run_deprecated_v1
   def testSkipEagerGetGradSource_NoEnclosingScope(self):
     self.assertEqual("gradients:0", self._grad_source_for_name("gradients"))
     self.assertEqual("gradients_0:0", self._grad_source_for_name("gradients_0"))
@@ -1097,6 +1122,7 @@ class TensorArrayTest(test.TestCase):
     self.assertEqual("gradients_0",
                      self._grad_source_for_name("gradients_0/foo/bar"))
 
+  @test_util.run_deprecated_v1
   def testSkipEagerGetGradSource_EnclosingScope(self):
     self.assertEqual("foo/gradients:0",
                      self._grad_source_for_name("foo/gradients"))
@@ -1111,11 +1137,13 @@ class TensorArrayTest(test.TestCase):
     self.assertEqual("foo/bar/gradients_0",
                      self._grad_source_for_name("foo/bar/gradients_0/baz"))
 
+  @test_util.run_deprecated_v1
   def testSkipEagerGetGradSource_NestedUsesInnermost(self):
     self.assertEqual(
         "foo/gradients/bar/gradients_0",
         self._grad_source_for_name("foo/gradients/bar/gradients_0/baz"))
 
+  @test_util.run_deprecated_v1
   def testSkipEagerWriteShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1141,6 +1169,7 @@ class TensorArrayTest(test.TestCase):
         w0.write(0, c2)
 
   @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  @test_util.run_deprecated_v1
   def testSkipEagerPartlyUnknownShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1212,10 +1241,12 @@ class TensorArrayTest(test.TestCase):
         w1.write(4, c2)
 
   @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
+  @test_util.run_deprecated_v1
   def testUnpackShape(self):
     self._testUnpackShape()
 
   @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
+  @test_util.run_deprecated_v1
   def testSplitShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1246,6 +1277,7 @@ class TensorArrayTest(test.TestCase):
             tensor_shape.TensorShape(
                 ta1.handle.op.get_attr("element_shape")).ndims, None)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerWriteUnknownShape(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1289,10 +1321,12 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
   @test_util.disable_control_flow_v2("b/117943489")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayUnpackDynamic(self):
     self._testTensorArrayUnpackDynamic()
 
   @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArraySplitDynamic(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -1319,6 +1353,7 @@ class TensorArrayTest(test.TestCase):
                                     .ENABLE_TENSOR_ARRAY_V2 else v1_msg):
         ta.stack().eval()
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayEvalEmpty(self):
     self._testTensorArrayEvalEmpty()
 
@@ -1339,10 +1374,12 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([0, 5], self.evaluate(concatenated).shape)
 
   @test_util.disable_control_flow_v2("b/117943489")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
 
   @test_util.disable_control_flow_v2("b/117943489")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayScatterReadAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -1370,6 +1407,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
   @test_util.disable_control_flow_v2("b/117943286")
+  @test_util.run_deprecated_v1
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -1407,6 +1445,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(expected_grad, grad_vals[0])
 
   @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGetsDeviceFromFirstWrite(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       # this initial device will be ignored.
@@ -1456,6 +1495,7 @@ class TensorArrayTest(test.TestCase):
             [s for s in dev_stats[d] if "/TensorArray" in s.node_name])
 
   @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2)
@@ -1486,6 +1526,7 @@ class TensorArrayTest(test.TestCase):
             [s for s in dev_stats[d] if "TensorArray" == s.node_name])
 
   @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayDisabledColocateWithFirstWriteCall(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       ta = tensor_array_ops.TensorArray(
@@ -1568,6 +1609,7 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(size0_v, 2)
       self.assertEqual(size1_v, 4)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradYsInCorrectScope(self):
     n_time = 1
     n_dim = 1
diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py
index a72888c256..5d46176bce 100644
--- a/tensorflow/python/kernel_tests/topk_op_test.py
+++ b/tensorflow/python/kernel_tests/topk_op_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import nn_ops
@@ -181,6 +182,7 @@ class TopKTest(test.TestCase):
     k = constant_op.constant(3)
     self._validateTopK(inputs, k, [19, 18, 17], [11, 3, 7])
 
+  @test_util.run_deprecated_v1
   def testKNegative(self):
     inputs = [[0.1, 0.2], [0.3, 0.4]]
     with self.session(use_gpu=True):
@@ -189,12 +191,14 @@ class TopKTest(test.TestCase):
       with self.assertRaisesOpError("Need k >= 0, got -7"):
         values.eval(feed_dict={k: -7})
 
+  @test_util.run_deprecated_v1
   def testKTooLarge(self):
     inputs = [[0.1, 0.2], [0.3, 0.4]]
     with self.assertRaisesRegexp(ValueError,
                                  r"must have last dimension >= k = 4"):
       nn_ops.top_k(inputs, 4)
 
+  @test_util.run_deprecated_v1
   def testTopKGradients(self):
     with self.session(use_gpu=True) as sess:
       inputs = array_ops.placeholder(dtypes.float32, shape=[2, 5])
diff --git a/tensorflow/python/kernel_tests/trace_op_test.py b/tensorflow/python/kernel_tests/trace_op_test.py
index f1abaefb66..52640c02c2 100644
--- a/tensorflow/python/kernel_tests/trace_op_test.py
+++ b/tensorflow/python/kernel_tests/trace_op_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -34,6 +35,7 @@ class TraceTest(test.TestCase):
       tf_ans = math_ops.trace(x).eval()
     self.assertAllClose(tf_ans, np_ans)
 
+  @test_util.run_deprecated_v1
   def testTrace(self):
     for dtype in [np.int32, np.float32, np.float64]:
       for shape in [[2, 2], [2, 3], [3, 2], [2, 3, 2], [2, 2, 2, 3]]:
diff --git a/tensorflow/python/kernel_tests/unicode_decode_op_test.py b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
index c34145bff1..c165021eea 100644
--- a/tensorflow/python/kernel_tests/unicode_decode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_decode_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl as errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gen_string_ops
 from tensorflow.python.platform import test
 
@@ -79,6 +80,7 @@ class UnicodeDecodeTest(test.TestCase):
       self.assertAllEqual(self.evaluate(row_splits).tolist(), [0, 4])
       self.assertAllEqual(self.evaluate(starts).tolist(), [0, 3, 6, 9])
 
+  @test_util.run_deprecated_v1
   def testStrictError(self):
     text = constant_op.constant([b"\xFEED"])
     _, error, _ = gen_string_ops.unicode_decode_with_offsets(
@@ -97,6 +99,7 @@ class UnicodeDecodeTest(test.TestCase):
     with self.test_session():
       self.assertAllEqual(self.evaluate(utf8_text).tolist(), [65533])
 
+  @test_util.run_deprecated_v1
   def testBadReplacementChar(self):
     text = constant_op.constant([b"\xFE"])
     _, error, _ = gen_string_ops.unicode_decode_with_offsets(
@@ -121,6 +124,7 @@ class UnicodeDecodeTest(test.TestCase):
           codepoint("o")
       ])
 
+  @test_util.run_deprecated_v1
   def testBadErrorPolicy(self):
     text = constant_op.constant(["hippopotamus"])
 
diff --git a/tensorflow/python/kernel_tests/unicode_script_op_test.py b/tensorflow/python/kernel_tests/unicode_script_op_test.py
index 927e5459ed..83cfeb2021 100644
--- a/tensorflow/python/kernel_tests/unicode_script_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_script_op_test.py
@@ -20,12 +20,14 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
 
 class UnicodeScriptOpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testValidScripts(self):
     inputs = [
         ord("a"),
@@ -45,6 +47,7 @@ class UnicodeScriptOpTest(test.TestCase):
               0  # USCRIPT_COMMON (ZYYY)
           ])
 
+  @test_util.run_deprecated_v1
   def testInvalidScript(self):
     inputs = [-100, 0xffffff]
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
index 037ecd104b..a3b4fd0347 100644
--- a/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_transcode_op_test.py
@@ -22,6 +22,7 @@ from absl.testing import parameterized
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
@@ -133,6 +134,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       values = self.evaluate(outputs)
       self.assertAllEqual(values, b"\x00 ")
 
+  @test_util.run_deprecated_v1
   def test_transcode_bad_utf8_with_strict_errors(self):
     bad_string = b"\x00\xff"
     with self.cached_session() as sess:
@@ -145,6 +147,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
           "Invalid formatting on input string"):
         self.evaluate(outputs)
 
+  @test_util.run_deprecated_v1
   def test_transcode_bad_utf8_start_with_strict_errors(self):
     bad_string = b"\xffabcd"
     with self.cached_session() as sess:
@@ -317,12 +320,14 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
       (b"\xfe\xff\x00<\xfe\xff\x00>", "UTF-16", b"<\xef\xbb\xbf>"),
       (b"\xff\xfe<\x00\xff\xfe>\x00", "UTF-16", b"<\xef\xbb\xbf>"),
   )
+  @test_util.run_deprecated_v1
   def test_bom_handling(self, string, input_encoding, expected):
     with self.test_session():
       output = string_ops.unicode_transcode(
           string, input_encoding=input_encoding, output_encoding="UTF-8")
       self.assertAllEqual(output.eval(), expected)
 
+  @test_util.run_deprecated_v1
   def test_invalid_encoding_causes_errors(self):
     strings = [[b"a", b"abc"], [b"ABC", b"DEF"]]
 
@@ -349,6 +354,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
             replace_control_characters=False)
         self.evaluate(outputs)
 
+  @test_util.run_deprecated_v1
   def test_invalid_error_policy_causes_errors(self):
     strings = [[b"a", b"abc"], [b"ABC", b"DEF"]]
 
@@ -378,6 +384,7 @@ class UnicodeTranscodeOpTest(test.TestCase, parameterized.TestCase):
 
       self.assertAllEqual([b"AbCdE", b"HiJkL"], transcoded)
 
+  @test_util.run_deprecated_v1
   def test_cjk_encodings(self):
     strings_ja = [
         b"\x5c\x5c",  # Yen sign
diff --git a/tensorflow/python/kernel_tests/unstack_op_test.py b/tensorflow/python/kernel_tests/unstack_op_test.py
index d314e1eaf9..f5ba475e7a 100644
--- a/tensorflow/python/kernel_tests/unstack_op_test.py
+++ b/tensorflow/python/kernel_tests/unstack_op_test.py
@@ -74,6 +74,7 @@ class UnstackOpTest(test.TestCase):
           cs = [self.evaluate(c) for c in cs]
           self.assertAllEqual(cs, data)
 
+  @test_util.run_deprecated_v1
   def testGradientsAxis0(self):
     for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
       data = np.random.randn(*shape)
@@ -86,6 +87,7 @@ class UnstackOpTest(test.TestCase):
                                                         shapes[i])
           self.assertLess(err, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testGradientsAxis1(self):
     for shape in (2, 3), (3, 2), (4, 3, 2):
       data = np.random.randn(*shape)
@@ -99,6 +101,7 @@ class UnstackOpTest(test.TestCase):
                                                         out_shape)
           self.assertLess(err, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testInferNum(self):
     with self.cached_session():
       for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2):
@@ -107,16 +110,19 @@ class UnstackOpTest(test.TestCase):
         self.assertEqual(type(cs), list)
         self.assertEqual(len(cs), shape[0])
 
+  @test_util.run_deprecated_v1
   def testCannotInferNumFromUnknownShape(self):
     x = array_ops.placeholder(np.float32)
     with self.assertRaisesRegexp(ValueError,
                                  r'Cannot infer num from shape <unknown>'):
       array_ops.unstack(x)
 
+  @test_util.run_deprecated_v1
   def testUnknownShapeOkWithNum(self):
     x = array_ops.placeholder(np.float32)
     array_ops.unstack(x, num=2)
 
+  @test_util.run_deprecated_v1
   def testCannotInferNumFromNoneShape(self):
     x = array_ops.placeholder(np.float32, shape=(None,))
     with self.assertRaisesRegexp(ValueError,
diff --git a/tensorflow/python/kernel_tests/variable_ops_test.py b/tensorflow/python/kernel_tests/variable_ops_test.py
index c63d7f89c7..0f3e261992 100644
--- a/tensorflow/python/kernel_tests/variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/variable_ops_test.py
@@ -60,15 +60,18 @@ class VariableOpTest(test.TestCase):
       # that Variable and Assign have GPU implementations for matching tf.
       self.assertAllEqual(x, self._initFetch(x, tftype, use_gpu=True))
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     self._testTypes(np.arange(0, 20).reshape([4, 5]))
 
+  @test_util.run_deprecated_v1
   def testset_shape(self):
     p = state_ops.variable_op([1, 2], dtypes.float32)
     self.assertEqual([1, 2], p.get_shape())
     p = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
     self.assertEqual(tensor_shape.unknown_shape(), p.get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssign(self):
     value = np.array([[42.0, 43.0]])
     var = state_ops.variable_op(value.shape, dtypes.float32)
@@ -76,6 +79,7 @@ class VariableOpTest(test.TestCase):
     assigned = state_ops.assign(var, value)
     self.assertShapeEqual(value, assigned)
 
+  @test_util.run_deprecated_v1
   def testAssignNoValidateShape(self):
     value = np.array([[42.0, 43.0]])
     var = state_ops.variable_op(value.shape, dtypes.float32)
@@ -83,6 +87,7 @@ class VariableOpTest(test.TestCase):
     assigned = state_ops.assign(var, value, validate_shape=False)
     self.assertShapeEqual(value, assigned)
 
+  @test_util.run_deprecated_v1
   def testAssignNoVarShape(self):
     value = np.array([[42.0, 43.0]])
     var = state_ops.variable_op(value.shape, dtypes.float32, set_shape=False)
@@ -90,6 +95,7 @@ class VariableOpTest(test.TestCase):
     assigned = state_ops.assign(var, value)
     self.assertShapeEqual(value, assigned)
 
+  @test_util.run_deprecated_v1
   def testAssignNoVarShapeNoValidateShape(self):
     value = np.array([[42.0, 43.0]])
     var = state_ops.variable_op(value.shape, dtypes.float32, set_shape=False)
@@ -102,6 +108,7 @@ class VariableOpTest(test.TestCase):
     self.assertEqual(tensor_shape.unknown_shape(), tensor.get_shape())
     return tensor
 
+  @test_util.run_deprecated_v1
   def testAssignNoValueShape(self):
     value = self._NewShapelessTensor()
     shape = [1, 2]
@@ -110,6 +117,7 @@ class VariableOpTest(test.TestCase):
     self.assertEqual(shape, var.get_shape())
     self.assertEqual(shape, assigned.get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignNoValueShapeNoValidateShape(self):
     value = self._NewShapelessTensor()
     shape = [1, 2]
@@ -118,6 +126,7 @@ class VariableOpTest(test.TestCase):
     assigned = state_ops.assign(var, value, validate_shape=False)
     self.assertEqual(tensor_shape.unknown_shape(), assigned.get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignNoShape(self):
     with self.cached_session():
       value = self._NewShapelessTensor()
@@ -126,6 +135,7 @@ class VariableOpTest(test.TestCase):
       self.assertEqual(tensor_shape.unknown_shape(),
                        state_ops.assign(var, value).get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignNoShapeNoValidateShape(self):
     with self.cached_session():
       value = self._NewShapelessTensor()
@@ -136,6 +146,7 @@ class VariableOpTest(test.TestCase):
           state_ops.assign(
               var, value, validate_shape=False).get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignUpdate(self):
     var = state_ops.variable_op([1, 2], dtypes.float32)
     added = state_ops.assign_add(var, [[2.0, 3.0]])
@@ -143,6 +154,7 @@ class VariableOpTest(test.TestCase):
     subbed = state_ops.assign_sub(var, [[12.0, 13.0]])
     self.assertEqual([1, 2], subbed.get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignUpdateNoVarShape(self):
     var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
     added = state_ops.assign_add(var, [[2.0, 3.0]])
@@ -150,6 +162,7 @@ class VariableOpTest(test.TestCase):
     subbed = state_ops.assign_sub(var, [[12.0, 13.0]])
     self.assertEqual([1, 2], subbed.get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignUpdateNoValueShape(self):
     var = state_ops.variable_op([1, 2], dtypes.float32)
     added = state_ops.assign_add(var, self._NewShapelessTensor())
@@ -157,6 +170,7 @@ class VariableOpTest(test.TestCase):
     subbed = state_ops.assign_sub(var, self._NewShapelessTensor())
     self.assertEqual([1, 2], subbed.get_shape())
 
+  @test_util.run_deprecated_v1
   def testAssignUpdateNoShape(self):
     var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
     added = state_ops.assign_add(var, self._NewShapelessTensor())
@@ -164,6 +178,7 @@ class VariableOpTest(test.TestCase):
     subbed = state_ops.assign_sub(var, self._NewShapelessTensor())
     self.assertEqual(tensor_shape.unknown_shape(), subbed.get_shape())
 
+  @test_util.run_deprecated_v1
   def testTemporaryVariable(self):
     with test_util.use_gpu():
       var = gen_state_ops.temporary_variable(
@@ -173,6 +188,7 @@ class VariableOpTest(test.TestCase):
       final = gen_state_ops.destroy_temporary_variable(var, var_name="foo")
       self.assertAllClose([[10.0, 12.0]], self.evaluate(final))
 
+  @test_util.run_deprecated_v1
   def testDestroyNonexistentTemporaryVariable(self):
     with test_util.use_gpu():
       var = gen_state_ops.temporary_variable([1, 2], dtypes.float32)
@@ -180,6 +196,7 @@ class VariableOpTest(test.TestCase):
       with self.assertRaises(errors.NotFoundError):
         self.evaluate(final)
 
+  @test_util.run_deprecated_v1
   def testDuplicateTemporaryVariable(self):
     with test_util.use_gpu():
       var1 = gen_state_ops.temporary_variable(
@@ -192,6 +209,7 @@ class VariableOpTest(test.TestCase):
       with self.assertRaises(errors.AlreadyExistsError):
         self.evaluate(final)
 
+  @test_util.run_deprecated_v1
   def testDestroyTemporaryVariableTwice(self):
     with test_util.use_gpu():
       var = gen_state_ops.temporary_variable([1, 2], dtypes.float32)
@@ -201,6 +219,7 @@ class VariableOpTest(test.TestCase):
       with self.assertRaises(errors.NotFoundError):
         self.evaluate(final)
 
+  @test_util.run_deprecated_v1
   def testTemporaryVariableNoLeak(self):
     with test_util.use_gpu():
       var = gen_state_ops.temporary_variable(
@@ -208,6 +227,7 @@ class VariableOpTest(test.TestCase):
       final = array_ops.identity(var)
       self.evaluate(final)
 
+  @test_util.run_deprecated_v1
   def testTwoTemporaryVariablesNoLeaks(self):
     with test_util.use_gpu():
       var1 = gen_state_ops.temporary_variable(
@@ -217,6 +237,7 @@ class VariableOpTest(test.TestCase):
       final = var1 + var2
       self.evaluate(final)
 
+  @test_util.run_deprecated_v1
   def testAssignDependencyAcrossDevices(self):
     with test_util.use_gpu():
       # The variable and an op to increment it are on the GPU.
@@ -232,6 +253,7 @@ class VariableOpTest(test.TestCase):
           result = math_ops.multiply(var, var)
       self.assertAllClose([4.0], self.evaluate(result))
 
+  @test_util.run_deprecated_v1
   def testIsVariableInitialized(self):
     for use_gpu in [True, False]:
       with self.test_session(use_gpu=use_gpu):
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 3720f736ac..44d4bd5e30 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -152,6 +152,7 @@ class VariableScopeTest(test.TestCase):
   # TypeError: Fetch argument <tf.Variable 'string:0' shape=() dtype=string>
   # has invalid type <class '...ResourceVariable'>, must be a string or Tensor.
   # (Can not convert a ResourceVariable into a Tensor or Operation.)
+  @test_util.run_deprecated_v1
   def testStringDefaultInitializer(self):
     with self.cached_session():
       v = variable_scope.get_variable("string", shape=[], dtype=dtypes.string)
@@ -310,6 +311,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # InvalidArgumentError: /job:moo/replica:0/task:0/device:CPU:0 unknown device.
+  @test_util.run_deprecated_v1
   def testVarScopeCachingDevice(self):
     with self.cached_session():
       caching_device = "/job:moo"
@@ -424,6 +426,7 @@ class VariableScopeTest(test.TestCase):
   # invalid type <class '...ops.resource_variable_ops.ResourceVariable'>, must
   # be a string or Tensor. (Can not convert a ResourceVariable into a Tensor or
   # Operation.)
+  @test_util.run_deprecated_v1
   def testControlDeps(self):
     with self.cached_session() as sess:
       v0 = variable_scope.get_variable(
@@ -450,6 +453,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # AssertionError: True is not false (last assertFalse)
+  @test_util.run_deprecated_v1
   def testEnableResourceVariables(self):
     old = variable_scope._DEFAULT_USE_RESOURCE
     try:
@@ -464,6 +468,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # TypeError: Fetch argument None has invalid type <type 'NoneType'>
+  @test_util.run_deprecated_v1
   def testControlFlow(self):
     with self.cached_session() as sess:
       v0 = variable_scope.get_variable(
@@ -1148,6 +1153,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testGetCollection(self):
     with self.cached_session():
       _ = variable_scope.get_variable("testGetCollection_a", [])
@@ -1204,6 +1210,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testGetTrainableVariablesWithGetVariable(self):
     with self.cached_session():
       _ = variable_scope.get_variable("testGetTrainableVariables_a", [])
@@ -1242,6 +1249,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testGetTrainableVariablesWithVariable(self):
     with self.cached_session():
       _ = variable_scope.variable(1.0, name="testGetTrainableVariables_a")
@@ -1283,6 +1291,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testGetGlobalVariables(self):
     with self.cached_session():
       _ = variable_scope.get_variable("testGetGlobalVariables_a", [])
@@ -1295,6 +1304,7 @@ class VariableScopeTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testGetLocalVariables(self):
     with self.cached_session():
       _ = variable_scope.get_variable(
@@ -1370,6 +1380,7 @@ class VariableScopeWithPartitioningTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testResultNameMatchesRequested(self):
     with variable_scope.variable_scope(
         "scope0", partitioner=axis0_into2_partitioner):
@@ -1444,6 +1455,7 @@ class VariableScopeWithPartitioningTest(test.TestCase):
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
   # obtaining different results in the eager case compared to the graph one
+  @test_util.run_deprecated_v1
   def testScalarIgnoresPartitioner(self):
     with variable_scope.variable_scope(
         "scope0", partitioner=axis0_into2_partitioner):
@@ -1583,6 +1595,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase):
   # dtype=float32> cannot be interpreted as a Tensor. (Tensor
   # Tensor("custom_getter/add:0", shape=(1, 2, 3), dtype=float32) is not an
   # element of this graph.)
+  @test_util.run_deprecated_v1
   def testGetterThatCreatesTwoVariablesAndSumsThem(self):
 
     def custom_getter(getter, name, *args, **kwargs):
@@ -1610,6 +1623,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase):
   # dtype=float32> cannot be interpreted as a Tensor. (Tensor
   # Tensor("sum_getter_2/add:0", shape=(1, 2, 3), dtype=float32) is not an
   # element of this graph.)
+  @test_util.run_deprecated_v1
   def testNestedCustomGetters(self):
 
     def sum_getter(getter, name, *args, **kwargs):
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 14ec46dcb2..08d885e8a8 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_state_ops
@@ -42,6 +43,7 @@ from tensorflow.python.util import compat
 
 class VariablesTestCase(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testInitialization(self):
     with self.cached_session():
       var0 = variables.VariableV1(0.0)
@@ -69,6 +71,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(0.0, self.evaluate(var0))
       self.assertAllClose(1.1, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testInitializationOrder(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([3, 6]), name="rnd")
@@ -107,6 +110,7 @@ class VariablesTestCase(test.TestCase):
       for _ in variables.Variable([0.0, 1.0]):
         pass
 
+  @test_util.run_deprecated_v1
   def testAssignments(self):
     with self.cached_session():
       var = variables.Variable(0.0)
@@ -125,6 +129,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(4.0, self.evaluate(four))
       self.assertAllClose(4.0, self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testResourceAssignments(self):
     with self.session(use_gpu=True):
       var = resource_variable_ops.ResourceVariable(0.0)
@@ -181,12 +186,15 @@ class VariablesTestCase(test.TestCase):
         self.evaluate(count_up_to)
       self.assertEqual(3, self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testCountUpToInt32(self):
     self._countUpToTest(dtypes.int32)
 
+  @test_util.run_deprecated_v1
   def testCountUpToInt64(self):
     self._countUpToTest(dtypes.int64)
 
+  @test_util.run_deprecated_v1
   def testControlDepsNone(self):
     with self.cached_session():
       c = constant_op.constant(1.0)
@@ -200,6 +208,7 @@ class VariablesTestCase(test.TestCase):
       self.assertEqual([], var_x.value().op.control_inputs)
       self.assertEqual([], var_x._ref().op.control_inputs)  # pylint: disable=protected-access
 
+  @test_util.run_deprecated_v1
   def testControlFlow(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(0, name="v0")
@@ -236,6 +245,7 @@ class VariablesTestCase(test.TestCase):
       self.evaluate(v0.initializer)
       self.evaluate(add)
 
+  @test_util.run_deprecated_v1
   def testControlFlowInitialization(self):
     """Expects an error if an initializer is in a control-flow scope."""
     def cond(i, _):
@@ -249,6 +259,7 @@ class VariablesTestCase(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "inside a control-flow"):
       control_flow_ops.while_loop(cond, body, [0, 0])
 
+  @test_util.run_deprecated_v1
   def testUseVariableAsTensor(self):
     with self.cached_session():
       var_x = variables.Variable(2.0)
@@ -258,6 +269,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(3.0, self.evaluate(var_y))
       self.assertAllClose(5.0, math_ops.add(var_x, var_y).eval())
 
+  @test_util.run_deprecated_v1
   def testZeroSizeVarSameAsConst(self):
     with self.cached_session():
       zero_size_var = variables.Variable(array_ops.zeros([0, 2]))
@@ -270,6 +282,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(const_mul.eval(), variable_output)
       self.assertAllClose([[0., 0.], [0., 0.]], variable_output)
 
+  @test_util.run_deprecated_v1
   def testCachingDevice(self):
     with self.cached_session():
       var = variables.Variable(2.0)
@@ -280,6 +293,7 @@ class VariablesTestCase(test.TestCase):
       self.assertFalse(var_cached.device.startswith("/job:foo"))
       self.assertTrue(var_cached.value().device.startswith("/job:foo"))
 
+  @test_util.run_deprecated_v1
   def testCollections(self):
     with self.cached_session():
       var_x = variables.VariableV1(2.0)
@@ -295,6 +309,7 @@ class VariablesTestCase(test.TestCase):
                        variables.global_variables())
       self.assertEqual([var_x, var_z, var_t], variables.trainable_variables())
 
+  @test_util.run_deprecated_v1
   def testCollectionsWithScope(self):
     with self.cached_session():
       with ops.name_scope("scope_1"):
@@ -316,6 +331,7 @@ class VariablesTestCase(test.TestCase):
           getattr(variables.Variable.__add__, attr),
           getattr(ops.Tensor.__add__, attr))
 
+  @test_util.run_deprecated_v1
   def testOperators(self):
     with self.cached_session():
       var_f = variables.Variable([2.0])
@@ -389,12 +405,14 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose([[80.0]], self.evaluate(matmul))
       self.assertAllClose([[20.0, 30.0], [40.0, 60.0]], self.evaluate(rmatmul))
 
+  @test_util.run_deprecated_v1
   def testSession(self):
     with self.cached_session() as sess:
       var = variables.Variable([1, 12])
       variables.global_variables_initializer().run()
       self.assertAllClose([1, 12], self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testColocation(self):
     with ops.device("/job:ps"):
       var = variables.VariableV1(0, name="v")
@@ -403,6 +421,7 @@ class VariablesTestCase(test.TestCase):
     self.assertDeviceEqual("/job:ps", assign_op.device)
     self.assertEqual([b"loc:@v"], assign_op.op.colocation_groups())
 
+  @test_util.run_deprecated_v1
   def testInitializerFunction(self):
     value = [[-42], [133.7]]
     shape = [2, 1]
@@ -440,6 +459,7 @@ class VariablesTestCase(test.TestCase):
           lambda: constant_op.constant(1.),
           constraint=constraint)
 
+  @test_util.run_deprecated_v1
   def testNoRefDataRace(self):
     with self.cached_session():
       a = variables.Variable([1, 2, 3], dtype=dtypes.float32)
@@ -450,6 +470,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllEqual(b.eval(), [3, 4, 5])
       self.assertAllEqual(c.eval(), [5, 6, 7])
 
+  @test_util.run_deprecated_v1
   def testInitializerFunctionDevicePlacement(self):
     with self.cached_session():
       initializer = lambda: constant_op.constant(42.0)
@@ -468,6 +489,7 @@ class VariablesTestCase(test.TestCase):
       for i in v2.initializer.inputs:
         self.assertEqual(expected_group_v2, i.op.colocation_groups())
 
+  @test_util.run_deprecated_v1
   def testVariableDefInitializedInstances(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v_def = variables.Variable(
@@ -511,6 +533,7 @@ class VariablesTestCase(test.TestCase):
           variables.Variable(variable_def=trainable_variable.to_proto())
           .trainable)
 
+  @test_util.run_deprecated_v1
   def testLoad(self):
     with self.cached_session():
       var = variables.Variable(np.zeros((5, 5), np.float32))
@@ -519,6 +542,7 @@ class VariablesTestCase(test.TestCase):
 
       self.assertAllClose(np.ones((5, 5), np.float32), self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testRepr(self):
     var = variables.VariableV1(np.zeros((5, 5), np.float32), name="noop")
     self.assertEqual(
@@ -552,6 +576,7 @@ class IsInitializedTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(0, self.evaluate(uninited).size)
 
+  @test_util.run_deprecated_v1
   def testVariableList(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2], name="v")
@@ -589,6 +614,7 @@ class ObsoleteIsInitializedTest(test.TestCase):
     with ops.Graph().as_default():
       self.assertEqual(None, variables.assert_variables_initialized())
 
+  @test_util.run_deprecated_v1
   def testVariables(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2])
@@ -600,6 +626,7 @@ class ObsoleteIsInitializedTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.evaluate(inited)
 
+  @test_util.run_deprecated_v1
   def testVariableList(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2])
diff --git a/tensorflow/python/kernel_tests/weights_broadcast_test.py b/tensorflow/python/kernel_tests/weights_broadcast_test.py
index c476004b89..677d8f2f22 100644
--- a/tensorflow/python/kernel_tests/weights_broadcast_test.py
+++ b/tensorflow/python/kernel_tests/weights_broadcast_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.platform import test
@@ -51,40 +52,48 @@ class AssertBroadcastableTest(test.TestCase):
           values_placeholder: values,
       })
 
+  @test_util.run_deprecated_v1
   def testScalar(self):
     self._test_valid(weights=5, values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def test1x1x1(self):
     self._test_valid(
         weights=np.asarray((5,)).reshape((1, 1, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def test1x1xN(self):
     self._test_valid(
         weights=np.asarray((5, 7, 11, 3)).reshape((1, 1, 4)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def test1xNx1(self):
     self._test_valid(
         weights=np.asarray((5, 11)).reshape((1, 2, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def test1xNxN(self):
     self._test_valid(
         weights=np.asarray((5, 7, 11, 3, 2, 13, 7, 5)).reshape((1, 2, 4)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testNx1x1(self):
     self._test_valid(
         weights=np.asarray((5, 7, 11)).reshape((3, 1, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testNx1xN(self):
     self._test_valid(
         weights=np.asarray((
             5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3)).reshape((3, 1, 4)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testNxNxN(self):
     self._test_valid(
         weights=np.asarray((
@@ -107,29 +116,35 @@ class AssertBroadcastableTest(test.TestCase):
             values_placeholder: values,
         })
 
+  @test_util.run_deprecated_v1
   def testInvalid1(self):
     self._test_invalid(weights=np.asarray((5,)), values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalid1x1(self):
     self._test_invalid(
         weights=np.asarray((5,)).reshape((1, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidPrefixMatch(self):
     self._test_invalid(
         weights=np.asarray((5, 7, 11, 3, 2, 12)).reshape((3, 2)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidSuffixMatch(self):
     self._test_invalid(
         weights=np.asarray((5, 7, 11, 3, 2, 12, 7, 5)).reshape((2, 4)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidOnesExtraDim(self):
     self._test_invalid(
         weights=np.asarray((5,)).reshape((1, 1, 1, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidPrefixMatchExtraDim(self):
     self._test_invalid(
         weights=np.asarray((
@@ -137,6 +152,7 @@ class AssertBroadcastableTest(test.TestCase):
             2, 17, 11, 3, 5, 7, 11, 3, 2, 12, 7, 5)).reshape((3, 2, 4, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidSuffixMatchExtraDim(self):
     self._test_invalid(
         weights=np.asarray((
@@ -164,18 +180,21 @@ class BroadcastWeightsTest(test.TestCase):
           values_placeholder: values,
       }))
 
+  @test_util.run_deprecated_v1
   def testScalar(self):
     self._test_valid(
         weights=5,
         values=_test_values((3, 2, 4)),
         expected=5 * np.ones((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def test1x1x1(self):
     self._test_valid(
         weights=np.asarray((5,)).reshape((1, 1, 1)),
         values=_test_values((3, 2, 4)),
         expected=5 * np.ones((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def test1x1xN(self):
     weights = np.asarray((5, 7, 11, 3)).reshape((1, 1, 4))
     self._test_valid(
@@ -183,6 +202,7 @@ class BroadcastWeightsTest(test.TestCase):
         values=_test_values((3, 2, 4)),
         expected=np.tile(weights, reps=(3, 2, 1)))
 
+  @test_util.run_deprecated_v1
   def test1xNx1(self):
     weights = np.asarray((5, 11)).reshape((1, 2, 1))
     self._test_valid(
@@ -190,6 +210,7 @@ class BroadcastWeightsTest(test.TestCase):
         values=_test_values((3, 2, 4)),
         expected=np.tile(weights, reps=(3, 1, 4)))
 
+  @test_util.run_deprecated_v1
   def test1xNxN(self):
     weights = np.asarray((5, 7, 11, 3, 2, 13, 7, 5)).reshape((1, 2, 4))
     self._test_valid(
@@ -197,6 +218,7 @@ class BroadcastWeightsTest(test.TestCase):
         values=_test_values((3, 2, 4)),
         expected=np.tile(weights, reps=(3, 1, 1)))
 
+  @test_util.run_deprecated_v1
   def testNx1x1(self):
     weights = np.asarray((5, 7, 11)).reshape((3, 1, 1))
     self._test_valid(
@@ -204,6 +226,7 @@ class BroadcastWeightsTest(test.TestCase):
         values=_test_values((3, 2, 4)),
         expected=np.tile(weights, reps=(1, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testNx1xN(self):
     weights = np.asarray((
         5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3)).reshape((3, 1, 4))
@@ -212,6 +235,7 @@ class BroadcastWeightsTest(test.TestCase):
         values=_test_values((3, 2, 4)),
         expected=np.tile(weights, reps=(1, 2, 1)))
 
+  @test_util.run_deprecated_v1
   def testNxNxN(self):
     weights = np.asarray((
         5, 7, 11, 3, 2, 12, 7, 5, 2, 17, 11, 3,
@@ -234,29 +258,35 @@ class BroadcastWeightsTest(test.TestCase):
             values_placeholder: values,
         })
 
+  @test_util.run_deprecated_v1
   def testInvalid1(self):
     self._test_invalid(weights=np.asarray((5,)), values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalid1x1(self):
     self._test_invalid(
         weights=np.asarray((5,)).reshape((1, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidPrefixMatch(self):
     self._test_invalid(
         weights=np.asarray((5, 7, 11, 3, 2, 12)).reshape((3, 2)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidSuffixMatch(self):
     self._test_invalid(
         weights=np.asarray((5, 7, 11, 3, 2, 12, 7, 5)).reshape((2, 4)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidOnesExtraDim(self):
     self._test_invalid(
         weights=np.asarray((5,)).reshape((1, 1, 1, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidPrefixMatchExtraDim(self):
     self._test_invalid(
         weights=np.asarray((
@@ -264,6 +294,7 @@ class BroadcastWeightsTest(test.TestCase):
             2, 17, 11, 3, 5, 7, 11, 3, 2, 12, 7, 5)).reshape((3, 2, 4, 1)),
         values=_test_values((3, 2, 4)))
 
+  @test_util.run_deprecated_v1
   def testInvalidSuffixMatchExtraDim(self):
     self._test_invalid(
         weights=np.asarray((
diff --git a/tensorflow/python/kernel_tests/where_op_test.py b/tensorflow/python/kernel_tests/where_op_test.py
index 9e074b2304..56c1390411 100644
--- a/tensorflow/python/kernel_tests/where_op_test.py
+++ b/tensorflow/python/kernel_tests/where_op_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -54,6 +55,7 @@ class WhereOpTest(test.TestCase):
       with self.assertRaises(ValueError):
         array_ops.where([False, True], None, [1, 2])
 
+  @test_util.run_deprecated_v1
   def testBasicVec(self):
     x = np.asarray([True, False])
     truth = np.asarray([[0]], dtype=np.int64)
@@ -67,11 +69,13 @@ class WhereOpTest(test.TestCase):
     truth = np.asarray([[2], [4]], dtype=np.int64)
     self._testWhere(x, truth)
 
+  @test_util.run_deprecated_v1
   def testRandomVec(self):
     x = np.random.rand(1000000) > 0.5
     truth = np.vstack([np.where(x)[0].astype(np.int64)]).T
     self._testWhere(x, truth)
 
+  @test_util.run_deprecated_v1
   def testBasicMat(self):
     x = np.asarray([[True, False], [True, False]])
 
@@ -80,6 +84,7 @@ class WhereOpTest(test.TestCase):
 
     self._testWhere(x, truth)
 
+  @test_util.run_deprecated_v1
   def testBasic3Tensor(self):
     x = np.asarray([[[True, False], [True, False]],
                     [[False, True], [False, True]],
@@ -99,36 +104,47 @@ class WhereOpTest(test.TestCase):
     truth = np.vstack(truth).T  # Convert to [num_true, indices].
     self._testWhere(x, truth, expected_err_re)
 
+  @test_util.run_deprecated_v1
   def testRandomBool(self):
     self._testRandom(np.bool)
 
+  @test_util.run_deprecated_v1
   def testRandomInt32(self):
     self._testRandom(np.int32)
 
+  @test_util.run_deprecated_v1
   def testRandomInt64(self):
     self._testRandom(np.int64)
 
+  @test_util.run_deprecated_v1
   def testRandomFloat(self):
     self._testRandom(np.float32)
 
+  @test_util.run_deprecated_v1
   def testRandomDouble(self):
     self._testRandom(np.float64)
 
+  @test_util.run_deprecated_v1
   def testRandomComplex64(self):
     self._testRandom(np.complex64)
 
+  @test_util.run_deprecated_v1
   def testRandomComplex128(self):
     self._testRandom(np.complex128)
 
+  @test_util.run_deprecated_v1
   def testRandomUint8(self):
     self._testRandom(np.uint8)
 
+  @test_util.run_deprecated_v1
   def testRandomInt8(self):
     self._testRandom(np.int8)
 
+  @test_util.run_deprecated_v1
   def testRandomInt16(self):
     self._testRandom(np.int16)
 
+  @test_util.run_deprecated_v1
   def testThreeArgument(self):
     x = np.array([[-2, 3, -1], [1, -3, -3]])
     np_val = np.where(x > 0, x * x, -x)
@@ -136,6 +152,7 @@ class WhereOpTest(test.TestCase):
       tf_val = array_ops.where(constant_op.constant(x) > 0, x * x, -x).eval()
     self.assertAllEqual(tf_val, np_val)
 
+  @test_util.run_deprecated_v1
   def testBatchSelect(self):
     x = np.array([[-2, 3, -1] * 64, [1, -3, -3] * 64] * 8192)  # [16384, 192]
     c_mat = np.array([[False] * 192, [True] * 192] * 8192)  # [16384, 192]
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index e08699922a..09cbeb1a0d 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -43,6 +43,7 @@ from tensorflow.python.platform import test
 
 class WhileV2Test(test.TestCase, parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSingleLoopVar(self):
     x = constant_op.constant(2.)
     ret = while_loop_v2(
@@ -64,6 +65,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(16., eval_result[0])
       self.assertSequenceEqual(sess.run(grad), [32.])
 
+  @test_util.run_deprecated_v1
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
     y = constant_op.constant(3.)
@@ -84,6 +86,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertSequenceEqual(self.evaluate(ret), [45., 3.])
       self.assertSequenceEqual(self.evaluate(grad), [9.])
 
+  @test_util.run_deprecated_v1
   def testMultipleLoopVars(self):
     x = constant_op.constant(5.)
     y = constant_op.constant(3.)
@@ -114,6 +117,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertSequenceEqual(self.evaluate(grady_1), [6.])
       self.assertSequenceEqual(self.evaluate(grady_2), [61.])
 
+  @test_util.run_deprecated_v1
   def testMultipleWhileLoops(self):
     x = constant_op.constant(2.)
     ret1 = while_loop_v2(
@@ -128,6 +132,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertSequenceEqual(self.evaluate(grad), [32.])
       self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
+  @test_util.run_deprecated_v1
   def testDoubleDerivative(self):
     x = constant_op.constant(2.)
     ret = while_loop_v2(
@@ -140,6 +145,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertSequenceEqual(self.evaluate(grad), [32.])
       self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
+  @test_util.run_deprecated_v1
   def testPruning(self):
     x = constant_op.constant(1)
 
@@ -175,6 +181,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     g = GetOptimizedGraph()
     self.assertEqual(len([n for n in g.node if n.op == "Enter"]), 2)
 
+  @test_util.run_deprecated_v1
   def testCaptureExternalTensorInCond(self):
     x = constant_op.constant(2.)
     y = constant_op.constant(1.)
@@ -187,6 +194,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(self.evaluate(ret), 18.)
       self.assertSequenceEqual(self.evaluate(grad), [9.])
 
+  @test_util.run_deprecated_v1
   def testCaptureExternalTensorInBody(self):
     x = constant_op.constant(2.)
     y = constant_op.constant(3.)
@@ -197,6 +205,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(self.evaluate(ret), 18.)
       self.assertSequenceEqual(self.evaluate(grad), [9.])
 
+  @test_util.run_deprecated_v1
   def testLoopWithTensorListPushBack(self):
     x = constant_op.constant(2.)
 
@@ -219,6 +228,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(sess.run(ret[0]), 16.)
       self.assertSequenceEqual(self.evaluate(grad), [32.])
 
+  @test_util.run_deprecated_v1
   def testDuplicateAccumulator(self):
     x = constant_op.constant(2.)
 
@@ -259,6 +269,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       ("PartiallyDefinedShape", [None, 2]),
       ("FullyDefinedShape", [1, 2]),
   )
+  @test_util.run_deprecated_v1
   def testAccumulatorElementShape(self, shape):
 
     def MatchShape(actual_tensor_shape):
@@ -344,6 +355,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
             while2_op.get_attr("body").name, r"foo_while_1_body_\d*")
 
   @test_util.enable_control_flow_v2
+  @test_util.run_deprecated_v1
   def testWhileAndTensorArray(self):
     with self.cached_session() as sess:
       param = constant_op.constant(2.0)
@@ -354,6 +366,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       r = gradients_impl.gradients(r, param)[0]
       self.assertAllClose(21.0, self.evaluate(r))
 
+  @test_util.run_deprecated_v1
   def testNestedWhile(self):
     # Compute sum of geometric progression: n^0 + n^1 + ... + n^m
     # We compute the pow using a while loop.
@@ -377,6 +390,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(self.evaluate(result), 364.)
       self.assertSequenceEqual(self.evaluate(grad), [547.])
 
+  @test_util.run_deprecated_v1
   def testIdentityNodeInBody(self):
 
     def Body(v):
@@ -392,6 +406,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(self.evaluate(ret), 16.)
       self.assertSequenceEqual(self.evaluate(grad), [32.])
 
+  @test_util.run_deprecated_v1
   def testNestedWhileAndTensorArray(self):
     n = constant_op.constant(3.0)
 
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index 77669f08cc..f5d03c2370 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
@@ -88,6 +89,7 @@ class XentTest(test.TestCase):
     self._testSingleClass(True)
     self._testSingleClass(False)
 
+  @test_util.run_deprecated_v1
   def testRankTooLarge(self):
     for dtype in np.float16, np.float32:
       np_features = np.array([[[1., 1., 1., 1.]], [[1., 2., 3.,
@@ -152,12 +154,14 @@ class XentTest(test.TestCase):
       self.assertAllCloseAccordingToType(np_loss, tf_loss)
       self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
         gen_nn_ops.softmax_cross_entropy_with_logits(
             [[0., 1.], [2., 3.]], [[0., 1., 0.], [1., 0., 0.]])
 
+  @test_util.run_deprecated_v1
   def testNotMatrix(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -179,6 +183,7 @@ class XentTest(test.TestCase):
         np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float64),
         np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float64))
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     with self.cached_session() as sess:
       l = constant_op.constant(
@@ -206,6 +211,7 @@ class XentTest(test.TestCase):
     print("cross entropy gradient err = ", err)
     self.assertLess(err, 5e-8)
 
+  @test_util.run_deprecated_v1
   def testGradientLabelWithV2(self):
     with self.cached_session():
       l = constant_op.constant(
@@ -224,6 +230,7 @@ class XentTest(test.TestCase):
 
     self.assertLess(err, 5e-8)
 
+  @test_util.run_deprecated_v1
   def testSecondGradient(self):
     with self.cached_session() as sess:
       l = constant_op.constant(
diff --git a/tensorflow/python/kernel_tests/zero_division_test.py b/tensorflow/python/kernel_tests/zero_division_test.py
index 7c82f9320a..3dd9ec4ba9 100644
--- a/tensorflow/python/kernel_tests/zero_division_test.py
+++ b/tensorflow/python/kernel_tests/zero_division_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.platform import test
 
 class ZeroDivisionTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testZeros(self):
     with test_util.use_gpu():
       for dtype in dtypes.uint8, dtypes.int16, dtypes.int32, dtypes.int64:
diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py
index 45099677e0..d0ec4f4425 100644
--- a/tensorflow/python/layers/base_test.py
+++ b/tensorflow/python/layers/base_test.py
@@ -143,6 +143,7 @@ class BaseLayerTest(test.TestCase):
           synchronization=variable_scope.VariableSynchronization.ON_READ,
           trainable=True)
 
+  @test_util.run_deprecated_v1
   def testReusePartitionedVaraiblesAndRegularizers(self):
     regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3
     partitioner = partitioned_variables.fixed_size_partitioner(3)
@@ -445,6 +446,7 @@ class BaseLayerTest(test.TestCase):
       self.assertTrue(isinstance(result, dict))
       self.assertEqual(set(['label', 'logits']), set(result.keys()))
 
+  @test_util.run_deprecated_v1
   def testActivityRegularizer(self):
     regularizer = math_ops.reduce_sum
     layer = base_layers.Layer(activity_regularizer=regularizer)
@@ -533,6 +535,7 @@ class BaseLayerTest(test.TestCase):
         self.assertEqual(len(layer.trainable_variables), 1)
         self.assertEqual(layer.variables[0].graph, outer_graph)
 
+  @test_util.run_deprecated_v1
   def testGetUpdateFor(self):
 
     class MyLayer(base_layers.Layer):
@@ -577,6 +580,7 @@ class BaseLayerTest(test.TestCase):
     self.assertEqual(len(layer.get_updates_for([intermediate_inputs])), 1)
     self.assertEqual(len(layer.get_updates_for([outputs])), 0)
 
+  @test_util.run_deprecated_v1
   def testGetLossesFor(self):
 
     class MyLayer(base_layers.Layer):
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index d3200fa5b5..a3e493edfe 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.layers import convolutional as conv_layers
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -59,6 +60,7 @@ class ConvTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'kernel_size'):
       conv_layers.conv2d(images, 32, None)
 
+  @test_util.run_deprecated_v1
   def testCreateConv2D(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -87,6 +89,7 @@ class ConvTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 4, 32])
     self.assertListEqual(layer.bias.get_shape().as_list(), [32])
 
+  @test_util.run_deprecated_v1
   def testCreateConv2DChannelsFirst(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, 4, height, width))
@@ -97,6 +100,7 @@ class ConvTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 4, 32])
     self.assertListEqual(layer.bias.get_shape().as_list(), [32])
 
+  @test_util.run_deprecated_v1
   def testUnknownInputChannels(self):
     images = array_ops.placeholder(dtypes.float32, (5, 7, 9, None))
     layer = conv_layers.Conv2D(32, [3, 3], activation=nn_ops.relu)
@@ -140,6 +144,7 @@ class ConvTest(test.TestCase):
     self.assertListEqual(output.get_shape().as_list(),
                          [5, height / 2, width, 32])
 
+  @test_util.run_deprecated_v1
   def testCreateConv1D(self):
     width = 7
     data = random_ops.random_uniform((5, width, 4))
@@ -156,6 +161,7 @@ class ConvTest(test.TestCase):
     output = conv_layers.conv1d(data, 32, 3, activation=nn_ops.relu)
     self.assertListEqual(output.get_shape().as_list(), [5, width - 2, 32])
 
+  @test_util.run_deprecated_v1
   def testCreateConv1DChannelsFirst(self):
     width = 7
     data = random_ops.random_uniform((5, 4, width))
@@ -165,6 +171,7 @@ class ConvTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 4, 32])
     self.assertListEqual(layer.bias.get_shape().as_list(), [32])
 
+  @test_util.run_deprecated_v1
   def testUnknownInputChannelsConv1D(self):
     data = array_ops.placeholder(dtypes.float32, (5, 4, None))
     layer = conv_layers.Conv1D(32, 3, activation=nn_ops.relu)
@@ -180,6 +187,7 @@ class ConvTest(test.TestCase):
                                  'should be defined. Found `None`.'):
       _ = layer.apply(data)
 
+  @test_util.run_deprecated_v1
   def testCreateConv3D(self):
     depth, height, width = 6, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 4))
@@ -191,6 +199,7 @@ class ConvTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 3, 4, 32])
     self.assertListEqual(layer.bias.get_shape().as_list(), [32])
 
+  @test_util.run_deprecated_v1
   def testUnknownInputChannelsConv3D(self):
     volumes = array_ops.placeholder(dtypes.float32, (5, 6, 7, 9, None))
     layer = conv_layers.Conv3D(32, [3, 3, 3], activation=nn_ops.relu)
@@ -199,6 +208,7 @@ class ConvTest(test.TestCase):
                                  'should be defined. Found `None`.'):
       _ = layer.apply(volumes)
 
+  @test_util.run_deprecated_v1
   def testConv2DKernelRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -210,6 +220,7 @@ class ConvTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testConv2DBiasRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -221,6 +232,7 @@ class ConvTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testConv2DNoBias(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -247,6 +259,7 @@ class ConvTest(test.TestCase):
     output = layer.apply(images)
     self.assertListEqual(output.get_shape().as_list(), [5, height - 2, 3, 32])
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DReuse(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
@@ -255,6 +268,7 @@ class ConvTest(test.TestCase):
     conv_layers.conv2d(images, 32, [3, 3], name='conv1', reuse=True)
     self.assertEqual(len(variables.trainable_variables()), 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DReuseFromScope(self):
     with variable_scope.variable_scope('scope'):
       height, width = 7, 9
@@ -265,6 +279,7 @@ class ConvTest(test.TestCase):
       conv_layers.conv2d(images, 32, [3, 3], name='conv1')
       self.assertEqual(len(variables.trainable_variables()), 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DInitializerFromScope(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope(
@@ -283,6 +298,7 @@ class ConvTest(test.TestCase):
         # Check that the bias still got initialized to zeros.
         self.assertAllClose(weights[1], np.zeros((32)))
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DNoReuse(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
@@ -325,6 +341,7 @@ class ConvTest(test.TestCase):
     self.assertEqual(conv3d.kernel_constraint, k_constraint)
     self.assertEqual(conv3d.bias_constraint, b_constraint)
 
+  @test_util.run_deprecated_v1
   def testConv3DChannelsFirst(self):
     # Test case for GitHub issue 15655
     images = array_ops.placeholder(
@@ -358,6 +375,7 @@ class SeparableConv1DTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'kernel_size'):
       conv_layers.separable_conv1d(data, 32, None)
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConv1D(self):
     length = 9
     data = random_ops.random_uniform((5, length, 4))
@@ -379,6 +397,7 @@ class SeparableConv1DTest(test.TestCase):
     self.assertEqual(layer.pointwise_kernel.get_shape().as_list(), [1, 8, 32])
     self.assertEqual(layer.bias.get_shape().as_list(), [32])
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConv1DChannelsFirst(self):
     length = 9
     data = random_ops.random_uniform((5, 4, length))
@@ -404,6 +423,7 @@ class SeparableConv1DTest(test.TestCase):
     output = layer.apply(data)
     self.assertEqual(output.get_shape().as_list(), [5, length // 2, 32])
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConv1DWithStridesChannelsFirst(self):
     data_format = 'channels_first'
     length = 10
@@ -413,6 +433,7 @@ class SeparableConv1DTest(test.TestCase):
     output = layer.apply(data)
     self.assertEqual(output.get_shape().as_list(), [5, 32, length // 2])
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv1DReuse(self):
     length = 10
     data = random_ops.random_uniform((5, length, 3), seed=1)
@@ -421,6 +442,7 @@ class SeparableConv1DTest(test.TestCase):
     conv_layers.separable_conv1d(data, 32, 3, name='sepconv1', reuse=True)
     self.assertEqual(len(variables.trainable_variables()), 3)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv1DReuseFromScope(self):
     with variable_scope.variable_scope('scope'):
       length = 10
@@ -431,6 +453,7 @@ class SeparableConv1DTest(test.TestCase):
       conv_layers.separable_conv1d(data, 32, 3, name='sepconv1')
       self.assertEqual(len(variables.trainable_variables()), 3)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv1DNoReuse(self):
     length = 10
     data = random_ops.random_uniform((5, length, 3), seed=1)
@@ -439,6 +462,7 @@ class SeparableConv1DTest(test.TestCase):
     conv_layers.separable_conv1d(data, 32, 3)
     self.assertEqual(len(variables.trainable_variables()), 6)
 
+  @test_util.run_deprecated_v1
   def testSeparableConv1DDepthwiseRegularizer(self):
     length = 9
     data = random_ops.random_uniform((5, length, 4))
@@ -450,6 +474,7 @@ class SeparableConv1DTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testSeparableConv1DPointwiseRegularizer(self):
     length = 9
     data = random_ops.random_uniform((5, length, 4))
@@ -461,6 +486,7 @@ class SeparableConv1DTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testSeparableConv1DBiasRegularizer(self):
     length = 9
     data = random_ops.random_uniform((5, length, 4))
@@ -472,6 +498,7 @@ class SeparableConv1DTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testSeparableConv1DNoBias(self):
     length = 9
     data = random_ops.random_uniform((5, length, 4))
@@ -522,6 +549,7 @@ class SeparableConv2DTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'kernel_size'):
       conv_layers.separable_conv2d(images, 32, None)
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConv2D(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -562,6 +590,7 @@ class SeparableConv2DTest(test.TestCase):
                          [1, 1, 4, 32])
     self.assertListEqual(layer.bias.get_shape().as_list(), [32])
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConv2DChannelsFirst(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, 4, height, width))
@@ -584,6 +613,7 @@ class SeparableConv2DTest(test.TestCase):
     output = layer.apply(images)
     self.assertListEqual(output.get_shape().as_list(), [5, height, width, 64])
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConvWithStrides(self):
     height, width = 6, 8
     # Test strides tuple
@@ -607,6 +637,7 @@ class SeparableConv2DTest(test.TestCase):
     self.assertListEqual(output.get_shape().as_list(),
                          [5, height / 2, width, 32])
 
+  @test_util.run_deprecated_v1
   def testCreateSeparableConvWithStridesChannelsFirst(self):
     data_format = 'channels_first'
     height, width = 6, 8
@@ -632,6 +663,7 @@ class SeparableConv2DTest(test.TestCase):
     self.assertListEqual(output.get_shape().as_list(),
                          [5, 32, height / 2, width])
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DReuse(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
@@ -641,6 +673,7 @@ class SeparableConv2DTest(test.TestCase):
         images, 32, [3, 3], name='sepconv1', reuse=True)
     self.assertEqual(len(variables.trainable_variables()), 3)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DReuseFromScope(self):
     with variable_scope.variable_scope('scope'):
       height, width = 7, 9
@@ -651,6 +684,7 @@ class SeparableConv2DTest(test.TestCase):
       conv_layers.separable_conv2d(images, 32, [3, 3], name='sepconv1')
       self.assertEqual(len(variables.trainable_variables()), 3)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DInitializerFromScope(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope(
@@ -671,6 +705,7 @@ class SeparableConv2DTest(test.TestCase):
         # Check that the bias still got initialized to zeros.
         self.assertAllClose(weights[2], np.zeros((32)))
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DNoReuse(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
@@ -679,6 +714,7 @@ class SeparableConv2DTest(test.TestCase):
     conv_layers.separable_conv2d(images, 32, [3, 3])
     self.assertEqual(len(variables.trainable_variables()), 6)
 
+  @test_util.run_deprecated_v1
   def testSeparableConv2DDepthwiseRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -690,6 +726,7 @@ class SeparableConv2DTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testSeparableConv2DPointwiseRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -701,6 +738,7 @@ class SeparableConv2DTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testSeparableConv2DBiasRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -712,6 +750,7 @@ class SeparableConv2DTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testSeparableConv2DNoBias(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -768,6 +807,7 @@ class Conv2DTransposeTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'kernel_size'):
       conv_layers.conv2d_transpose(images, 32, None)
 
+  @test_util.run_deprecated_v1
   def testCreateConv2DTranspose(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -839,6 +879,7 @@ class Conv2DTransposeTest(test.TestCase):
     self.assertListEqual(output.get_shape().as_list(),
                          [5, height * 2, width, 32])
 
+  @test_util.run_deprecated_v1
   def testConv2DTransposeKernelRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -850,6 +891,7 @@ class Conv2DTransposeTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testConv2DTransposeBiasRegularizer(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -861,6 +903,7 @@ class Conv2DTransposeTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testConv2DTransposeNoBias(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 4))
@@ -873,6 +916,7 @@ class Conv2DTransposeTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 32, 4])
     self.assertEqual(layer.bias, None)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DTransposeReuse(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
@@ -881,6 +925,7 @@ class Conv2DTransposeTest(test.TestCase):
     conv_layers.conv2d_transpose(images, 32, [3, 3], name='deconv1', reuse=True)
     self.assertEqual(len(variables.trainable_variables()), 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DTransposeReuseFromScope(self):
     with variable_scope.variable_scope('scope'):
       height, width = 7, 9
@@ -891,6 +936,7 @@ class Conv2DTransposeTest(test.TestCase):
       conv_layers.conv2d_transpose(images, 32, [3, 3], name='deconv1')
       self.assertEqual(len(variables.trainable_variables()), 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DTransposeInitializerFromScope(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope(
@@ -909,6 +955,7 @@ class Conv2DTransposeTest(test.TestCase):
         # Check that the bias still got initialized to zeros.
         self.assertAllClose(weights[1], np.zeros((32)))
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv2DTransposeNoReuse(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
@@ -955,6 +1002,7 @@ class Conv3DTransposeTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'kernel_size'):
       conv_layers.conv3d_transpose(volumes, 4, None)
 
+  @test_util.run_deprecated_v1
   def testCreateConv3DTranspose(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 32))
@@ -976,6 +1024,7 @@ class Conv3DTransposeTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 3, 4, 32])
     self.assertListEqual(layer.bias.get_shape().as_list(), [4])
 
+  @test_util.run_deprecated_v1
   def testCreateConv3DTransposeChannelsFirst(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, 32, depth, height, width))
@@ -1019,6 +1068,7 @@ class Conv3DTransposeTest(test.TestCase):
     self.assertListEqual(output.get_shape().as_list(),
                          [5, depth * 2, height, width, 4])
 
+  @test_util.run_deprecated_v1
   def testConv3DTransposeKernelRegularizer(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 32))
@@ -1030,6 +1080,7 @@ class Conv3DTransposeTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testConv3DTransposeBiasRegularizer(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 32))
@@ -1041,6 +1092,7 @@ class Conv3DTransposeTest(test.TestCase):
     self.evaluate([v.initializer for v in layer.variables])
     self.assertListEqual(self.evaluate(layer.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testConv3DTransposeNoBias(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 32))
@@ -1053,6 +1105,7 @@ class Conv3DTransposeTest(test.TestCase):
     self.assertListEqual(layer.kernel.get_shape().as_list(), [3, 3, 3, 4, 32])
     self.assertEqual(layer.bias, None)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv3DTransposeReuse(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 32), seed=1)
@@ -1062,6 +1115,7 @@ class Conv3DTransposeTest(test.TestCase):
         volumes, 4, [3, 3, 3], name='deconv1', reuse=True)
     self.assertEqual(len(variables.trainable_variables()), 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv3DTransposeReuseFromScope(self):
     with variable_scope.variable_scope('scope'):
       depth, height, width = 5, 7, 9
@@ -1072,6 +1126,7 @@ class Conv3DTransposeTest(test.TestCase):
       conv_layers.conv3d_transpose(volumes, 4, [3, 3, 3], name='deconv1')
       self.assertEqual(len(variables.trainable_variables()), 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv3DTransposeInitializerFromScope(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope(
@@ -1091,6 +1146,7 @@ class Conv3DTransposeTest(test.TestCase):
         # Check that the bias still got initialized to zeros.
         self.assertAllClose(weights[1], np.zeros((4)))
 
+  @test_util.run_deprecated_v1
   def testFunctionalConv3DTransposeNoReuse(self):
     depth, height, width = 5, 7, 9
     volumes = random_ops.random_uniform((5, depth, height, width, 32), seed=1)
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index a61639b2db..cf6f0fbb70 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -59,6 +59,7 @@ class DenseTest(test.TestCase):
     dense.apply(random_ops.random_uniform((5, 2)))
     self.assertEqual(dense.name, 'dense_2')
 
+  @test_util.run_deprecated_v1
   def testVariableInput(self):
     with self.cached_session():
       v = variable_scope.get_variable(
@@ -140,6 +141,7 @@ class DenseTest(test.TestCase):
     outputs = dense.apply(inputs)
     self.assertEqual(outputs.get_shape().as_list(), [1, 2, 4, 7])
 
+  @test_util.run_deprecated_v1
   def testCallOnPlaceHolder(self):
     inputs = array_ops.placeholder(dtype=dtypes.float32)
     dense = core_layers.Dense(4, name='my_dense')
@@ -179,6 +181,7 @@ class DenseTest(test.TestCase):
     if not context.executing_eagerly():
       self.assertEqual(outputs.op.name, 'dense2/BiasAdd')
 
+  @test_util.run_deprecated_v1
   def testActivityRegularizer(self):
     regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3
     dense = core_layers.Dense(
@@ -189,6 +192,7 @@ class DenseTest(test.TestCase):
     self.assertEqual(len(loss_keys), 1)
     self.assertListEqual(dense.losses, loss_keys)
 
+  @test_util.run_deprecated_v1
   def testKernelRegularizer(self):
     regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3
     dense = core_layers.Dense(
@@ -200,6 +204,7 @@ class DenseTest(test.TestCase):
     self.evaluate([v.initializer for v in dense.variables])
     self.assertAllEqual(self.evaluate(dense.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testKernelRegularizerWithReuse(self):
     regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3
     inputs = random_ops.random_uniform((5, 3), seed=1)
@@ -212,6 +217,7 @@ class DenseTest(test.TestCase):
     self.assertEqual(
         len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1)
 
+  @test_util.run_deprecated_v1
   def testBiasRegularizer(self):
     regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3
     dense = core_layers.Dense(2, name='my_dense', bias_regularizer=regularizer)
@@ -222,6 +228,7 @@ class DenseTest(test.TestCase):
     self.evaluate([v.initializer for v in dense.variables])
     self.assertAllEqual(self.evaluate(dense.losses), self.evaluate(loss_keys))
 
+  @test_util.run_deprecated_v1
   def testFunctionalDense(self):
     with self.cached_session():
       inputs = random_ops.random_uniform((5, 3), seed=1)
@@ -231,6 +238,7 @@ class DenseTest(test.TestCase):
           len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2)
       self.assertEqual(outputs.op.name, 'my_dense/Relu')
 
+  @test_util.run_deprecated_v1
   def testFunctionalDenseTwice(self):
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
@@ -262,6 +270,7 @@ class DenseTest(test.TestCase):
         vars2 = variables.trainable_variables()
       self.assertEqual(vars1, vars2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalDenseInitializerFromScope(self):
     with variable_scope.variable_scope(
         'scope',
@@ -307,6 +316,7 @@ class DenseTest(test.TestCase):
       core_layers.dense(inputs, 2)
     self.assertEqual(called[0], 2)
 
+  @test_util.run_deprecated_v1
   def testFunctionalDenseInScope(self):
     with self.cached_session():
       with variable_scope.variable_scope('test'):
@@ -393,6 +403,7 @@ class DropoutTest(test.TestCase):
     np_output = self.evaluate(dropped)
     self.assertAllClose(np.ones((5, 3)), np_output)
 
+  @test_util.run_deprecated_v1
   def testDynamicLearningPhase(self):
     with self.cached_session() as sess:
       dp = core_layers.Dropout(0.5, seed=1)
@@ -426,6 +437,7 @@ class DropoutTest(test.TestCase):
     self.assertAlmostEqual(0., np_output.min())
     self.assertAllClose(np_output[:, 0, :], np_output[:, 1, :])
 
+  @test_util.run_deprecated_v1
   def testFunctionalDropout(self):
     with self.cached_session():
       inputs = array_ops.ones((5, 5))
@@ -437,6 +449,7 @@ class DropoutTest(test.TestCase):
       np_output = self.evaluate(dropped)
       self.assertAllClose(np.ones((5, 5)), np_output)
 
+  @test_util.run_deprecated_v1
   def testDynamicRate(self):
     with self.cached_session() as sess:
       rate = array_ops.placeholder(dtype='float32', name='rate')
@@ -452,6 +465,7 @@ class DropoutTest(test.TestCase):
 
 class FlattenTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCreateFlatten(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32')
@@ -476,6 +490,7 @@ class FlattenTest(test.TestCase):
     shape = core_layers.Flatten().compute_output_shape((None, 3, None))
     self.assertEqual(shape.as_list(), [None, None])
 
+  @test_util.run_deprecated_v1
   def testDataFormat5d(self):
     np_input_channels_last = np.arange(
         120, dtype='float32').reshape([1, 5, 4, 3, 2])
@@ -493,6 +508,7 @@ class FlattenTest(test.TestCase):
 
       self.assertAllEqual(np_output_cl, np_output_cf)
 
+  @test_util.run_deprecated_v1
   def testDataFormat4d(self):
     np_input_channels_last = np.arange(
         24, dtype='float32').reshape([1, 4, 3, 2])
@@ -510,16 +526,19 @@ class FlattenTest(test.TestCase):
 
       self.assertAllEqual(np_output_cl, np_output_cf)
 
+  @test_util.run_deprecated_v1
   def testFunctionalFlatten(self):
     x = array_ops.placeholder(shape=(None, 2, 3), dtype='float32')
     y = core_layers.flatten(x, name='flatten')
     self.assertEqual(y.get_shape().as_list(), [None, 6])
 
+  @test_util.run_deprecated_v1
   def testFlattenValueError(self):
     x = array_ops.placeholder(shape=(None,), dtype='float32')
     with self.assertRaises(ValueError):
       core_layers.Flatten()(x)
 
+  @test_util.run_deprecated_v1
   def testFlattenUnknownAxes(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(shape=(5, None, None), dtype='float32')
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index cc3badbde1..07d8e40b75 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.core.protobuf import saver_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.layers import convolutional as conv_layers
 from tensorflow.python.layers import normalization as normalization_layers
 from tensorflow.python.ops import array_ops
@@ -143,6 +144,7 @@ class BNTest(test.TestCase):
 
     return train_vars, loss_val
 
+  @test_util.run_deprecated_v1
   def testHalfPrecision(self):
     ref_vars, ref_loss = self._trainEvalSequence(
         dtype=dtypes.float32,
@@ -228,33 +230,43 @@ class BNTest(test.TestCase):
                                ckpt_b_use_gpu, use_gpu_test_a, use_gpu_test_b,
                                freeze_mode)
 
+  @test_util.run_deprecated_v1
   def testCheckpointFusedCPUAndFusedGPU(self):
     self._testCheckpointCrossDevice(True, False, True, True)
 
+  @test_util.run_deprecated_v1
   def testCheckpointFusedCPUAndFusedCPU(self):
     self._testCheckpointCrossDevice(True, False, True, False)
 
+  @test_util.run_deprecated_v1
   def testCheckpointFusedGPUAndFusedGPU(self):
     self._testCheckpointCrossDevice(True, True, True, True)
 
+  @test_util.run_deprecated_v1
   def testCheckpointNonFusedCPUAndNonFusedGPU(self):
     self._testCheckpointCrossDevice(False, False, False, True)
 
+  @test_util.run_deprecated_v1
   def testCheckpointNonFusedCPUAndNonFusedCPU(self):
     self._testCheckpointCrossDevice(False, False, False, False)
 
+  @test_util.run_deprecated_v1
   def testCheckpointNonFusedGPUAndNonFusedGPU(self):
     self._testCheckpointCrossDevice(False, True, False, True)
 
+  @test_util.run_deprecated_v1
   def testCheckpointNonFusedGPUAndFusedGPU(self):
     self._testCheckpointCrossDevice(False, True, True, True)
 
+  @test_util.run_deprecated_v1
   def testCheckpointNonFusedGPUAndFusedCPU(self):
     self._testCheckpointCrossDevice(False, True, True, False)
 
+  @test_util.run_deprecated_v1
   def testCheckpointNonFusedCPUAndFusedCPU(self):
     self._testCheckpointCrossDevice(False, False, True, False)
 
+  @test_util.run_deprecated_v1
   def testCreateBN(self):
     # Call layer.
     bn = normalization_layers.BatchNormalization(axis=1)
@@ -281,6 +293,7 @@ class BNTest(test.TestCase):
         ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
         bn.trainable_variables)
 
+  @test_util.run_deprecated_v1
   def testCreateFusedBNFloat16(self):
     # Call layer.
     bn = normalization_layers.BatchNormalization(axis=1, fused=True)
@@ -310,6 +323,7 @@ class BNTest(test.TestCase):
         ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
         bn.trainable_variables)
 
+  @test_util.run_deprecated_v1
   def test3DInputAxis1(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -353,6 +367,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def test3DInputAxis2(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -436,6 +451,7 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def test4DInputAxis2(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -477,6 +493,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def test4DInputAxis3(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -518,6 +535,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def test4DInputAxis3Fused(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -601,6 +619,7 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def testNegativeAxis(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -643,6 +662,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def testBooleanLearningPhase(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -683,6 +703,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def testFunctionalNoReuse(self):
     inputs = variables.Variable(
         np.random.random((5, 4, 3, 6)), dtype=dtypes.float32)
@@ -735,6 +756,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def testFunctionalReuse(self):
     inputs1 = variables.Variable(
         np.random.random((5, 4, 3, 6)), dtype=dtypes.float32)
@@ -799,6 +821,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=2)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def testFunctionalReuseFromScope(self):
     inputs = variables.Variable(
         np.random.random((5, 4, 3, 6)), dtype=dtypes.float32)
@@ -813,6 +836,7 @@ class BNTest(test.TestCase):
           inputs, axis=-1, momentum=0.9, epsilon=epsilon, training=training)
       self.assertEqual(len(variables.global_variables()), 5)
 
+  @test_util.run_deprecated_v1
   def testNoCenter(self):
     bn = normalization_layers.BatchNormalization(axis=1, center=False)
     inputs = random_ops.random_uniform((5, 4, 3), seed=1)
@@ -828,6 +852,7 @@ class BNTest(test.TestCase):
     self.assertEqual(len(bn.trainable_variables), 1)
     self.assertEqual(len(bn.non_trainable_variables), 2)
 
+  @test_util.run_deprecated_v1
   def testNoScale(self):
     bn = normalization_layers.BatchNormalization(axis=1, scale=False)
     inputs = random_ops.random_uniform((5, 4, 3), seed=1)
@@ -843,6 +868,7 @@ class BNTest(test.TestCase):
     self.assertEqual(len(bn.trainable_variables), 1)
     self.assertEqual(len(bn.non_trainable_variables), 2)
 
+  @test_util.run_deprecated_v1
   def testRegularizers(self):
     reg = lambda x: 0.1 * math_ops.reduce_sum(x)
     bn = normalization_layers.BatchNormalization(axis=1, beta_regularizer=reg)
@@ -868,6 +894,7 @@ class BNTest(test.TestCase):
     self.assertEqual(bn.gamma_constraint, g_constraint)
     self.assertEqual(bn.beta_constraint, b_constraint)
 
+  @test_util.run_deprecated_v1
   def testRenorm(self):
     shape = (4, 3)
     xt = array_ops.placeholder(dtypes.float32, shape)
@@ -926,6 +953,7 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, yt_val_train, atol=1e-5)
         self.assertAllClose(y_test, yt_val_test, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testAdjustment(self):
     shape = (4, 3)
     xt = array_ops.placeholder(dtypes.float32, shape)
@@ -970,6 +998,7 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, yt_val_train, atol=1e-5)
         self.assertAllClose(y_test, yt_val_test, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testRenormWithAdjustment(self):
     shape = (4, 3)
     xt = array_ops.placeholder(dtypes.float32, shape)
@@ -1040,6 +1069,7 @@ class BNTest(test.TestCase):
       normalization_layers.batch_normalization(
           inp, virtual_batch_size=-1)
 
+  @test_util.run_deprecated_v1
   def testGhostBNVirtualBatchFull(self):
     shape = [6, 5, 4, 3]
     inp = random_ops.random_uniform(shape, seed=1)
@@ -1065,6 +1095,7 @@ class BNTest(test.TestCase):
         inp, virtual_batch_size=3)
     self.assertListEqual(out.shape.as_list(), shape)
 
+  @test_util.run_deprecated_v1
   def testGhostBNUnknownBatchSize(self):
     np_shape = [10, 5, 4]
     tf_shape = [None, 5, 4]
@@ -1080,6 +1111,7 @@ class BNTest(test.TestCase):
 
       self.assertListEqual(list(y.shape), np_shape)
 
+  @test_util.run_deprecated_v1
   def testGhostBN2Dims(self):
     shape = [6, 2]
     virtual_batch_size = 3
@@ -1133,6 +1165,7 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, y_val_train, atol=1e-5)
         self.assertAllClose(y_test, y_val_test, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testGhostBN4DimsAxis3(self):
     shape = [6, 10, 10, 3]
     virtual_batch_size = 2
@@ -1186,6 +1219,7 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, y_val_train, atol=1e-2)
         self.assertAllClose(y_test, y_val_test, atol=1e-2)
 
+  @test_util.run_deprecated_v1
   def testGhostBN4DimsAxis1(self):
     shape = [6, 3, 10, 10]
     virtual_batch_size = 2
@@ -1256,6 +1290,7 @@ class BNTest(test.TestCase):
       normalization_layers.batch_normalization(
           inp, axis=[1, 2, 1])   # duplicate
 
+  @test_util.run_deprecated_v1
   def test3DInputMultiAxis12(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -1297,6 +1332,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def test5DInputMultiAxis123(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -1338,6 +1374,7 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
+  @test_util.run_deprecated_v1
   def testGhostBN5DimsMultiAxis14(self):
     shape = [6, 3, 10, 10, 4]
     virtual_batch_size = 3
diff --git a/tensorflow/python/layers/pooling_test.py b/tensorflow/python/layers/pooling_test.py
index 7533674e5a..cf1fa1e691 100644
--- a/tensorflow/python/layers/pooling_test.py
+++ b/tensorflow/python/layers/pooling_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.layers import pooling as pooling_layers
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
@@ -64,6 +65,7 @@ class PoolingTest(test.TestCase):
     output = layer.apply(images)
     self.assertListEqual(output.get_shape().as_list(), [5, 3, 4, 4])
 
+  @test_util.run_deprecated_v1
   def testCreateMaxPooling2DChannelsFirst(self):
     height, width = 7, 9
     images = random_ops.random_uniform((5, 2, height, width))
@@ -73,6 +75,7 @@ class PoolingTest(test.TestCase):
     output = layer.apply(images)
     self.assertListEqual(output.get_shape().as_list(), [5, 2, 6, 8])
 
+  @test_util.run_deprecated_v1
   def testCreateAveragePooling2DChannelsFirst(self):
     height, width = 5, 6
     images = random_ops.random_uniform((3, 4, height, width))
@@ -83,6 +86,7 @@ class PoolingTest(test.TestCase):
     output = layer.apply(images)
     self.assertListEqual(output.get_shape().as_list(), [3, 4, 4, 5])
 
+  @test_util.run_deprecated_v1
   def testCreateAveragePooling2DChannelsFirstWithNoneBatch(self):
     height, width = 5, 6
     images = array_ops.placeholder(dtype='float32',
diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py
index 739278273b..d154b6759b 100644
--- a/tensorflow/python/ops/bitwise_ops_test.py
+++ b/tensorflow/python/ops/bitwise_ops_test.py
@@ -34,6 +34,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
   def __init__(self, method_name="runTest"):
     super(BitwiseOpTest, self).__init__(method_name)
 
+  @test_util.run_deprecated_v1
   def testBinaryOps(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64]
@@ -70,6 +71,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
             gen_bitwise_ops.population_count(input_tensor))
         self.assertAllEqual(truth, popcnt_result)
 
+  @test_util.run_deprecated_v1
   def testInvertOp(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64]
@@ -94,6 +96,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
           expected = [dtype.max - x for x in inputs]
           self.assertAllEqual(inverted, expected)
 
+  @test_util.run_deprecated_v1
   def testShiftsWithPositiveLHS(self):
     dtype_list = [np.int8, np.int16, np.int32, np.int64,
                   np.uint8, np.uint16, np.uint32, np.uint64]
@@ -108,6 +111,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs))
         self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs))
 
+  @test_util.run_deprecated_v1
   def testShiftsWithNegativeLHS(self):
     dtype_list = [np.int8, np.int16, np.int32, np.int64]
 
@@ -121,6 +125,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs))
         self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs))
 
+  @test_util.run_deprecated_v1
   def testImplementationDefinedShiftsDoNotCrash(self):
     dtype_list = [np.int8, np.int16, np.int32, np.int64]
 
@@ -136,6 +141,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
                   bitwise_ops.right_shift(lhs, rhs)])
 
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16]
diff --git a/tensorflow/python/ops/clip_ops_test.py b/tensorflow/python/ops/clip_ops_test.py
index e9f7941b42..a59a0c22d4 100644
--- a/tensorflow/python/ops/clip_ops_test.py
+++ b/tensorflow/python/ops/clip_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import numerics
 from tensorflow.python.platform import test
@@ -57,6 +58,7 @@ class ClipOpsTest(test.TestCase):
       result, expected = self.evaluate([clipped, dense_clipped])
     self.assertAllClose(result, expected)
 
+  @test_util.run_deprecated_v1
   def testClipTensorByNorm(self):
     # Simple example
     self._testClipTensorByNorm([[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], 4.0,
diff --git a/tensorflow/python/ops/collective_ops_test.py b/tensorflow/python/ops/collective_ops_test.py
index 9c772a9354..0fd9368d21 100644
--- a/tensorflow/python/ops/collective_ops_test.py
+++ b/tensorflow/python/ops/collective_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import collective_ops
 from tensorflow.python.platform import test
 
@@ -49,16 +50,19 @@ class CollectiveOpTest(test.TestCase):
     self.assertAllClose(results[0], expected, rtol=1e-5, atol=1e-5)
     self.assertAllClose(results[1], expected, rtol=1e-5, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testCollectiveReduce(self):
     self._testCollectiveReduce([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1],
                                [0.3, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3],
                                [0.2, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2], True)
 
+  @test_util.run_deprecated_v1
   def testCollectiveAutoGraphKey(self):
     self._testCollectiveReduce([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1],
                                [0.3, 1.3, 2.3, 3.3, 4.3, 5.3, 6.3, 7.3],
                                [0.2, 1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2], False)
 
+  @test_util.run_deprecated_v1
   def testCollectiveReduceScalar(self):
     self._testCollectiveReduce(0.1, 0.3, 0.2, True)
 
@@ -81,6 +85,7 @@ class CollectiveOpTest(test.TestCase):
     self.assertAllClose(results[0], t0, rtol=1e-5, atol=1e-5)
     self.assertAllClose(results[1], t0, rtol=1e-5, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testCollectiveBroadcast(self):
     self._testCollectiveBroadcast([0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1])
 
diff --git a/tensorflow/python/ops/gradient_checker_test.py b/tensorflow/python/ops/gradient_checker_test.py
index 66c7b9a71b..4d2b5efac7 100644
--- a/tensorflow/python/ops/gradient_checker_test.py
+++ b/tensorflow/python/ops/gradient_checker_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
@@ -46,6 +47,7 @@ def _nan_grad(unused_op, grad):
 
 class GradientCheckerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAddSimple(self):
     np.random.seed(1)  # Fix seed to avoid flakiness
     with self.session(use_gpu=False):
@@ -60,6 +62,7 @@ class GradientCheckerTest(test.TestCase):
     tf_logging.info("x1 error = %f", error)
     assert error < 1e-4
 
+  @test_util.run_deprecated_v1
   def testAddSimpleGPU(self):
     np.random.seed(2)  # Fix seed to avoid flakiness
     with self.session(use_gpu=True):
@@ -74,6 +77,7 @@ class GradientCheckerTest(test.TestCase):
     tf_logging.info("x1 error = %f", error)
     assert error < 1e-4
 
+  @test_util.run_deprecated_v1
   def testAddCustomized(self):
     np.random.seed(3)  # Fix seed to avoid flakiness
     with self.cached_session():
@@ -92,6 +96,7 @@ class GradientCheckerTest(test.TestCase):
     tf_logging.info("x2 error = %f", error)
     assert error < 1e-10
 
+  @test_util.run_deprecated_v1
   def testGather(self):
     np.random.seed(4)  # Fix seed to avoid flakiness
     with self.cached_session():
@@ -109,6 +114,7 @@ class GradientCheckerTest(test.TestCase):
     tf_logging.info("gather error = %f", error)
     assert error < 1e-4
 
+  @test_util.run_deprecated_v1
   def testNestedGather(self):
     np.random.seed(5)  # Fix seed to avoid flakiness
     with self.cached_session():
@@ -130,6 +136,7 @@ class GradientCheckerTest(test.TestCase):
     tf_logging.info("nested gather error = %f", error)
     assert error < 1e-4
 
+  @test_util.run_deprecated_v1
   def testComplexMul(self):
     with self.cached_session():
       size = ()
@@ -144,6 +151,7 @@ class GradientCheckerTest(test.TestCase):
       self.assertLess(
           gradient_checker.compute_gradient_error(x, size, y, size), 2e-4)
 
+  @test_util.run_deprecated_v1
   def testComplexConj(self):
     with self.cached_session():
       size = ()
@@ -157,6 +165,7 @@ class GradientCheckerTest(test.TestCase):
       self.assertLess(
           gradient_checker.compute_gradient_error(x, size, y, size), 2e-5)
 
+  @test_util.run_deprecated_v1
   def testEmptySucceeds(self):
     with self.cached_session():
       x = array_ops.placeholder(dtypes.float32)
@@ -279,18 +288,23 @@ class MiniMNISTTest(test.TestCase):
     tf_logging.info("Mini MNIST: %s gradient error = %g", tag, err)
     return err
 
+  @test_util.run_deprecated_v1
   def testInputGradient(self):
     self.assertLess(self._BuildAndTestMiniMNIST(0, "input"), 1e-8)
 
+  @test_util.run_deprecated_v1
   def testHiddenWeightGradient(self):
     self.assertLess(self._BuildAndTestMiniMNIST(1, "hidden_weight"), 1e-8)
 
+  @test_util.run_deprecated_v1
   def testHiddenBiasGradient(self):
     self.assertLess(self._BuildAndTestMiniMNIST(2, "hidden_bias"), 1e-8)
 
+  @test_util.run_deprecated_v1
   def testSoftmaxWeightGradient(self):
     self.assertLess(self._BuildAndTestMiniMNIST(3, "softmax_weight"), 1e-8)
 
+  @test_util.run_deprecated_v1
   def testSoftmaxBiasGradient(self):
     self.assertLess(self._BuildAndTestMiniMNIST(4, "softmax_bias"), 1e-8)
 
diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py
index cd1abbfc13..b48ef67196 100644
--- a/tensorflow/python/ops/histogram_ops_test.py
+++ b/tensorflow/python/ops/histogram_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import histogram_ops
@@ -84,6 +85,7 @@ class HistogramFixedWidthTest(test.TestCase):
   def setUp(self):
     self.rng = np.random.RandomState(0)
 
+  @test_util.run_deprecated_v1
   def test_with_invalid_value_range(self):
     values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
     with self.assertRaisesRegexp(
@@ -92,6 +94,7 @@ class HistogramFixedWidthTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "Dimension must be 2 but is 3"):
       histogram_ops.histogram_fixed_width(values, [1.0, 2.0, 3.0])
 
+  @test_util.run_deprecated_v1
   def test_with_invalid_nbins(self):
     values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
     with self.assertRaisesRegexp(
@@ -146,6 +149,7 @@ class HistogramFixedWidthTest(test.TestCase):
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, self.evaluate(hist))
 
+  @test_util.run_deprecated_v1
   def test_shape_inference(self):
     value_range = [0.0, 5.0]
     values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]]
diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py
index 0ea15b0d23..c481266dd7 100644
--- a/tensorflow/python/ops/image_grad_test.py
+++ b/tensorflow/python/ops/image_grad_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import image_ops
@@ -47,6 +48,7 @@ class ResizeNearestNeighborOpTest(test.TestCase):
         resize_out = self.evaluate(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
+  @test_util.run_deprecated_v1
   def testGradFromResizeToLargerInBothDims(self):
     in_shape = [1, 2, 3, 1]
     out_shape = [1, 4, 6, 1]
@@ -62,6 +64,7 @@ class ResizeNearestNeighborOpTest(test.TestCase):
             input_tensor, in_shape, resize_out, out_shape, x_init_value=x)
       self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradFromResizeToSmallerInBothDims(self):
     in_shape = [1, 4, 6, 1]
     out_shape = [1, 2, 3, 1]
@@ -77,6 +80,7 @@ class ResizeNearestNeighborOpTest(test.TestCase):
             input_tensor, in_shape, resize_out, out_shape, x_init_value=x)
       self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testCompareGpuVsCpu(self):
     in_shape = [1, 4, 6, 3]
     out_shape = [1, 8, 16, 3]
@@ -116,6 +120,7 @@ class ResizeBilinearOpTest(test.TestCase):
       resize_out = self.evaluate(resize_out)
       self.assertEqual(out_shape, list(resize_out.shape))
 
+  @test_util.run_deprecated_v1
   def testGradFromResizeToLargerInBothDims(self):
     in_shape = [1, 2, 3, 1]
     out_shape = [1, 4, 6, 1]
@@ -129,6 +134,7 @@ class ResizeBilinearOpTest(test.TestCase):
           input_tensor, in_shape, resize_out, out_shape, x_init_value=x)
     self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradFromResizeToSmallerInBothDims(self):
     in_shape = [1, 4, 6, 1]
     out_shape = [1, 2, 3, 1]
@@ -142,6 +148,7 @@ class ResizeBilinearOpTest(test.TestCase):
           input_tensor, in_shape, resize_out, out_shape, x_init_value=x)
     self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testCompareGpuVsCpu(self):
     in_shape = [2, 4, 6, 3]
     out_shape = [2, 8, 16, 3]
@@ -160,6 +167,7 @@ class ResizeBilinearOpTest(test.TestCase):
 
       self.assertAllClose(grad[False], grad[True], rtol=1e-4, atol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testTypes(self):
     in_shape = [1, 4, 6, 1]
     out_shape = [1, 2, 3, 1]
@@ -199,6 +207,7 @@ class ResizeBicubicOpTest(test.TestCase):
         resize_out = self.evaluate(resize_out)
         self.assertEqual(out_shape, list(resize_out.shape))
 
+  @test_util.run_deprecated_v1
   def testGradFromResizeToLargerInBothDims(self):
     in_shape = [1, 2, 3, 1]
     out_shape = [1, 4, 6, 1]
@@ -214,6 +223,7 @@ class ResizeBicubicOpTest(test.TestCase):
             input_tensor, in_shape, resize_out, out_shape, x_init_value=x)
       self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradFromResizeToSmallerInBothDims(self):
     in_shape = [1, 4, 6, 1]
     out_shape = [1, 2, 3, 1]
@@ -229,6 +239,7 @@ class ResizeBicubicOpTest(test.TestCase):
             input_tensor, in_shape, resize_out, out_shape, x_init_value=x)
       self.assertLess(err, 1e-3)
 
+  @test_util.run_deprecated_v1
   def testGradOnUnsupportedType(self):
     in_shape = [1, 4, 6, 1]
     out_shape = [1, 2, 3, 1]
@@ -306,6 +317,7 @@ class CropAndResizeOpTest(test.TestCase):
         samples.append(sample)
     return samples
 
+  @test_util.run_deprecated_v1
   def testGradRandomBoxes(self):
     """Test that the gradient is correct for randomly generated boxes.
 
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index 71a574e0a0..e7249333bd 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -211,6 +211,7 @@ class GrayscaleToRGBTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     # Shape inference works and produces expected output where possible
     rgb_shape = [7, None, 19, 3]
@@ -271,6 +272,7 @@ class AdjustGamma(test_util.TensorFlowTestCase):
       else:
         raise AssertionError("Exception not raised: %s" % err_msg)
 
+  @test_util.run_deprecated_v1
   def test_adjust_gamma_less_zero_tensor(self):
     """White image should be returned for gamma equal to zero"""
     with self.cached_session():
@@ -308,6 +310,7 @@ class AdjustGamma(test_util.TensorFlowTestCase):
 
       self.assertAllClose(y_tf, y_np, 1e-6)
 
+  @test_util.run_deprecated_v1
   def test_adjust_gamma_less_one(self):
     """Verifying the output with expected results for gamma
     correction with gamma equal to half"""
@@ -329,6 +332,7 @@ class AdjustGamma(test_util.TensorFlowTestCase):
 
       self.assertAllClose(y_tf, y_np, 1e-6)
 
+  @test_util.run_deprecated_v1
   def test_adjust_gamma_greater_one(self):
     """Verifying the output with expected results for gamma
     correction with gamma equal to two"""
@@ -938,6 +942,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
       y_v[i][2] = b
     return y_v.reshape(x_np.shape)
 
+  @test_util.run_deprecated_v1
   def testAdjustRandomSaturation(self):
     x_shapes = [
         [2, 2, 3],
@@ -996,6 +1001,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
+  @test_util.run_deprecated_v1
   def testLeftRight(self):
     x_np = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[3, 2, 1], [3, 2, 1]], dtype=np.uint8).reshape([2, 3, 1])
@@ -1021,6 +1027,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
+  @test_util.run_deprecated_v1
   def testRandomFlipLeftRight(self):
     x_np = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[3, 2, 1], [3, 2, 1]], dtype=np.uint8).reshape([2, 3, 1])
@@ -1049,6 +1056,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       self.assertGreaterEqual(count_flipped, 20)
       self.assertGreaterEqual(count_unflipped, 20)
 
+  @test_util.run_deprecated_v1
   def testRandomFlipLeftRightWithBatch(self):
     batch_size = 16
     seed = 42
@@ -1113,6 +1121,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
+  @test_util.run_deprecated_v1
   def testUpDown(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[4, 5, 6], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
@@ -1138,6 +1147,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
+  @test_util.run_deprecated_v1
   def testRandomFlipUpDown(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[4, 5, 6], [1, 2, 3]], dtype=np.uint8).reshape([2, 3, 1])
@@ -1166,6 +1176,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       self.assertGreaterEqual(count_flipped, 20)
       self.assertGreaterEqual(count_unflipped, 20)
 
+  @test_util.run_deprecated_v1
   def testRandomFlipUpDownWithBatch(self):
     batch_size = 16
     seed = 42
@@ -1230,6 +1241,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, x_np)
 
+  @test_util.run_deprecated_v1
   def testTranspose(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint8).reshape([2, 3, 1])
     y_np = np.array([[1, 4], [2, 5], [3, 6]], dtype=np.uint8).reshape([3, 2, 1])
@@ -1256,6 +1268,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
+  @test_util.run_deprecated_v1
   def testPartialShapes(self):
     p_unknown_rank = array_ops.placeholder(dtypes.uint8)
     p_unknown_dims_3 = array_ops.placeholder(
@@ -1314,6 +1327,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         rotated = image_ops.rot90(rotated)
       self.assertAllEqual(image, self.evaluate(rotated))
 
+  @test_util.run_deprecated_v1
   def testRot90NumpyEquivalence(self):
     image = np.arange(24, dtype=np.uint8).reshape([2, 4, 3])
     with self.test_session(use_gpu=True):
@@ -1323,6 +1337,7 @@ class FlipTransposeRotateTest(test_util.TensorFlowTestCase):
         y_np = np.rot90(image, k=k)
         self.assertAllEqual(y_np, y_tf.eval({k_placeholder: k}))
 
+  @test_util.run_deprecated_v1
   def testRot90NumpyEquivalenceWithBatch(self):
     image = np.arange(48, dtype=np.uint8).reshape([2, 2, 4, 3])
     with self.test_session(use_gpu=True):
@@ -1411,6 +1426,7 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
       y_tf = self._adjustContrastTf(x_np, contrast_factor)
       self.assertAllClose(y_tf, y_np, rtol=1e-5, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testContrastFactorShape(self):
     x_shape = [1, 2, 2, 3]
     x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
@@ -1474,6 +1490,7 @@ class PerImageWhiteningTest(test_util.TensorFlowTestCase):
     y /= stddev
     return y
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     x_shape = [13, 9, 3]
     x_np = np.arange(0, np.prod(x_shape), dtype=np.int32).reshape(x_shape)
@@ -1574,11 +1591,13 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
     y = image_ops.crop_to_bounding_box(image, 0, 0, height, width)
     self.assertEqual(y.get_shape().as_list(), post_shape)
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
     self._assertReturns(x, x_shape, 0, 0, x, x_shape)
 
+  @test_util.run_deprecated_v1
   def testCrop(self):
     x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
     x_shape = [3, 3, 1]
@@ -1603,6 +1622,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
     y = [1, 2, 4, 5, 7, 8]
     self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     self._assertShapeInference([55, 66, 3], 55, 66, [55, 66, 3])
     self._assertShapeInference([59, 69, 3], 55, 66, [55, 66, 3])
@@ -1616,6 +1636,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
     self._assertShapeInference([None, None, None], 55, 66, [55, 66, None])
     self._assertShapeInference(None, 55, 66, [55, 66, None])
 
+  @test_util.run_deprecated_v1
   def testNon3DInput(self):
     # Input image is not 3D
     x = [0] * 15
@@ -1627,6 +1648,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
                          target_width,
                          "'image' must have either 3 or 4 dimensions.")
 
+  @test_util.run_deprecated_v1
   def testZeroLengthInput(self):
     # Input image has 0-length dimension(s).
     # Each line is a test configuration:
@@ -1658,6 +1680,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
           "assertion failed:",
           use_tensor_inputs_options=[True])
 
+  @test_util.run_deprecated_v1
   def testBadParams(self):
     x_shape = [4, 4, 1]
     x = np.zeros(x_shape)
@@ -1675,6 +1698,7 @@ class CropToBoundingBoxTest(test_util.TensorFlowTestCase):
     for params, err_msg in test_config:
       self._assertRaises(x, x_shape, *params, err_msg=err_msg)
 
+  @test_util.run_deprecated_v1
   def testNameScope(self):
     image = array_ops.placeholder(dtypes.float32, shape=[55, 66, 3])
     y = image_ops.crop_to_bounding_box(image, 0, 0, 55, 66)
@@ -1691,6 +1715,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
     else:
       self.assertEqual(y.get_shape().as_list(), post_shape)
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     x_shapes = [[13, 9, 3], [5, 13, 9, 3]]
     for x_shape in x_shapes:
@@ -1734,6 +1759,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(y_tf, y_np)
       self.assertAllEqual(y_tf.shape, y_np.shape)
 
+  @test_util.run_deprecated_v1
   def testCropping2(self):
     # Test case for 10315
     x_shapes = [[240, 320, 3], [5, 240, 320, 3]]
@@ -1750,6 +1776,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
           self.assertAllEqual(y_tf, y_np)
           self.assertAllEqual(y_tf.shape, y_np.shape)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     # Test no-op fraction=1.0, with 3-D tensors.
     self._assertShapeInference([50, 60, 3], 1.0, [50, 60, 3])
@@ -1810,6 +1837,7 @@ class CentralCropTest(test_util.TensorFlowTestCase):
           with self.assertRaises(ValueError):
             _ = image_ops.central_crop(x, 0.5)
 
+  @test_util.run_deprecated_v1
   def testNameScope(self):
     x_shape = [13, 9, 3]
     x_np = np.ones(x_shape, dtype=np.float32)
@@ -1902,12 +1930,14 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       self.assertAllClose(y, self.evaluate(y_tf))
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
     offset_height, offset_width = [0, 0]
     self._assertReturns(x, x_shape, offset_height, offset_width, x, x_shape)
 
+  @test_util.run_deprecated_v1
   def testPadding(self):
     x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
     x_shape = [3, 3, 1]
@@ -1932,6 +1962,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     y_shape = [3, 4, 1]
     self._assertReturns(x, x_shape, offset_height, offset_width, y, y_shape)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     self._assertShapeInference([55, 66, 3], 55, 66, [55, 66, 3])
     self._assertShapeInference([50, 60, 3], 55, 66, [55, 66, 3])
@@ -1945,6 +1976,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     self._assertShapeInference([None, None, None], 55, 66, [55, 66, None])
     self._assertShapeInference(None, 55, 66, [55, 66, None])
 
+  @test_util.run_deprecated_v1
   def testNon3DInput(self):
     # Input image is not 3D
     x = [0] * 15
@@ -1956,6 +1988,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
                          target_width,
                          "'image' must have either 3 or 4 dimensions.")
 
+  @test_util.run_deprecated_v1
   def testZeroLengthInput(self):
     # Input image has 0-length dimension(s).
     # Each line is a test configuration:
@@ -1988,6 +2021,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
           "all dims of \\'image.shape\\' must be > 0",
           use_tensor_inputs_options=[True])
 
+  @test_util.run_deprecated_v1
   def testBadParams(self):
     x_shape = [3, 3, 1]
     x = np.zeros(x_shape)
@@ -2002,6 +2036,7 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     for config_item in test_config:
       self._assertRaises(x, x_shape, *config_item)
 
+  @test_util.run_deprecated_v1
   def testNameScope(self):
     image = array_ops.placeholder(dtypes.float32, shape=[55, 66, 3])
     y = image_ops.pad_to_bounding_box(image, 0, 0, 55, 66)
@@ -2109,6 +2144,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
     # TODO(wicke, shlens, dga): Restore this test so that it is no longer flaky.
     # self.assertGreaterEqual(min(fraction_object_covered), min_object_covered)
 
+  @test_util.run_deprecated_v1
   def testWholeImageBoundingBox(self):
     height = 40
     width = 50
@@ -2123,6 +2159,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
         aspect_ratio_range=(0.75, 1.33),
         area_range=(0.05, 1.0))
 
+  @test_util.run_deprecated_v1
   def testWithBoundingBox(self):
     height = 40
     width = 50
@@ -2153,6 +2190,7 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase):
         aspect_ratio_range=(0.75, 1.33),
         area_range=(0.05, 1.0))
 
+  @test_util.run_deprecated_v1
   def testSampleDistortedBoundingBoxShape(self):
     with self.test_session(use_gpu=True):
       image_size = constant_op.constant(
@@ -2248,6 +2286,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
     else:
       return False
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     img_shape = [1, 6, 4, 1]
     single_shape = [6, 4, 1]
@@ -2282,6 +2321,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
         newshape = self.evaluate(yshape)
         self.assertAllEqual(single_shape, newshape)
 
+  @test_util.run_deprecated_v1
   def testTensorArguments(self):
     img_shape = [1, 6, 4, 1]
     single_shape = [6, 4, 1]
@@ -2343,6 +2383,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
       _ = image_ops.resize_images(image, [6, None],
                                   image_ops.ResizeMethod.BILINEAR)
 
+  @test_util.run_deprecated_v1
   def testReturnDtype(self):
     target_shapes = [[6, 4], [3, 2], [
         array_ops.placeholder(dtypes.int32),
@@ -2591,6 +2632,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
               value[use_gpu] = self.evaluate(out_op)
           self.assertAllClose(value[True], value[False], rtol=1e-5, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     self._assertShapeInference([50, 60, 3], [55, 66], [55, 66, 3])
     self._assertShapeInference([55, 66, 3], [55, 66], [55, 66, 3])
@@ -2611,6 +2653,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
     self._assertShapeInference([59, 60, None], [55, 66], [55, 66, None])
     self._assertShapeInference([None, None, None], [55, 66], [55, 66, None])
 
+  @test_util.run_deprecated_v1
   def testNameScope(self):
     img_shape = [1, 3, 2, 1]
     with self.test_session(use_gpu=True):
@@ -2661,6 +2704,7 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
                                    preserve_aspect_ratio, use_tensor_inputs)
       self.assertShapeEqual(y, ops.convert_to_tensor(y_tf))
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioMultipleImages(self):
     x_shape = [10, 100, 100, 10]
     x = np.random.uniform(size=x_shape)
@@ -2668,36 +2712,42 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
     self._assertResizeCheckShape(x, x_shape, [250, 250], [10, 250, 250, 10],
                                  preserve_aspect_ratio=False)
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
 
     self._assertResizeEqual(x, x_shape, x, x_shape)
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioSmaller(self):
     x_shape = [100, 100, 10]
     x = np.random.uniform(size=x_shape)
 
     self._assertResizeCheckShape(x, x_shape, [75, 50], [50, 50, 10])
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioSmallerMultipleImages(self):
     x_shape = [10, 100, 100, 10]
     x = np.random.uniform(size=x_shape)
 
     self._assertResizeCheckShape(x, x_shape, [75, 50], [10, 50, 50, 10])
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioLarger(self):
     x_shape = [100, 100, 10]
     x = np.random.uniform(size=x_shape)
 
     self._assertResizeCheckShape(x, x_shape, [150, 200], [150, 150, 10])
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioSameRatio(self):
     x_shape = [1920, 1080, 3]
     x = np.random.uniform(size=x_shape)
 
     self._assertResizeCheckShape(x, x_shape, [3840, 2160], [3840, 2160, 3])
 
+  @test_util.run_deprecated_v1
   def testPreserveAspectRatioSquare(self):
     x_shape = [299, 299, 3]
     x = np.random.uniform(size=x_shape)
@@ -2767,12 +2817,14 @@ class ResizeImageWithPadTest(test_util.TensorFlowTestCase):
     y = image_ops.resize_image_with_pad(image, height, width)
     self.assertEqual(y.get_shape().as_list(), post_shape)
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
 
     self._assertReturns(x, x_shape, x, x_shape)
 
+  @test_util.run_deprecated_v1
   def testPad(self):
     # Reduce vertical dimension
     x = [1, 2, 3, 4, 5, 6, 7, 8]
@@ -2863,12 +2915,14 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
     y = image_ops.resize_image_with_crop_or_pad(image, height, width)
     self.assertEqual(y.get_shape().as_list(), post_shape)
 
+  @test_util.run_deprecated_v1
   def testNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
 
     self._assertReturns(x, x_shape, x, x_shape)
 
+  @test_util.run_deprecated_v1
   def testPad(self):
     # Pad even along col.
     x = [1, 2, 3, 4, 5, 6, 7, 8]
@@ -2906,6 +2960,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
 
     self._assertReturns(x, x_shape, y, y_shape)
 
+  @test_util.run_deprecated_v1
   def testCrop(self):
     # Crop even along col.
     x = [1, 2, 3, 4, 5, 6, 7, 8]
@@ -2943,6 +2998,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
 
     self._assertReturns(x, x_shape, y, y_shape)
 
+  @test_util.run_deprecated_v1
   def testCropAndPad(self):
     # Pad along row but crop along col.
     x = [1, 2, 3, 4, 5, 6, 7, 8]
@@ -2962,6 +3018,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
 
     self._assertReturns(x, x_shape, y, y_shape)
 
+  @test_util.run_deprecated_v1
   def testShapeInference(self):
     self._assertShapeInference([50, 60, 3], 55, 66, [55, 66, 3])
     self._assertShapeInference([55, 66, 3], 55, 66, [55, 66, 3])
@@ -2983,6 +3040,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
     self._assertShapeInference([None, None, None], 55, 66, [55, 66, None])
     self._assertShapeInference(None, 55, 66, [55, 66, None])
 
+  @test_util.run_deprecated_v1
   def testNon3DInput(self):
     # Input image is not 3D
     x = [0] * 15
@@ -2996,6 +3054,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
       self._assertRaises(x, x_shape, target_height, target_width,
                          "'image' must have either 3 or 4 dimensions.")
 
+  @test_util.run_deprecated_v1
   def testZeroLengthInput(self):
     # Input image has 0-length dimension(s).
     target_height, target_width = [1, 1]
@@ -3021,6 +3080,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
           "all dims of \\'image.shape\\' must be > 0",
           use_tensor_inputs_options=[True])
 
+  @test_util.run_deprecated_v1
   def testBadParams(self):
     x_shape = [4, 4, 1]
     x = np.zeros(x_shape)
@@ -3035,6 +3095,7 @@ class ResizeImageWithCropOrPadTest(test_util.TensorFlowTestCase):
     self._assertRaises(x, x_shape, target_height, target_width,
                        "target_width must be > 0")
 
+  @test_util.run_deprecated_v1
   def testNameScope(self):
     image = array_ops.placeholder(dtypes.float32, shape=[50, 60, 3])
     y = image_ops.resize_image_with_crop_or_pad(image, 55, 66)
@@ -3118,6 +3179,7 @@ class JpegTest(test_util.TensorFlowTestCase):
         image1_crop, image2 = self.evaluate([image1_crop, image2])
         self.assertAllEqual(image1_crop, image2)
 
+  @test_util.run_deprecated_v1
   def testCropAndDecodeJpegWithInvalidCropWindow(self):
     with self.cached_session() as sess:
       # Encode it, then decode it, then encode it
@@ -3194,6 +3256,7 @@ class JpegTest(test_util.TensorFlowTestCase):
       # The images should be the same.
       self.assertAllClose(image1, image2)
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     with self.test_session(use_gpu=True) as sess:
       jpeg = constant_op.constant("nonsense")
@@ -3202,6 +3265,7 @@ class JpegTest(test_util.TensorFlowTestCase):
         self.assertEqual(image.get_shape().as_list(),
                          [None, None, channels or None])
 
+  @test_util.run_deprecated_v1
   def testExtractJpegShape(self):
     # Read a real jpeg and verify shape.
     path = ("tensorflow/core/lib/jpeg/testdata/"
@@ -3212,6 +3276,7 @@ class JpegTest(test_util.TensorFlowTestCase):
       [image_shape] = sess.run([image_ops.extract_jpeg_shape(jpeg)])
       self.assertEqual(image_shape.tolist(), [256, 128, 3])
 
+  @test_util.run_deprecated_v1
   def testExtractJpegShapeforCmyk(self):
     # Read a cmyk jpeg image, and verify its shape.
     path = ("tensorflow/core/lib/jpeg/testdata/"
@@ -3293,6 +3358,7 @@ class PngTest(test_util.TensorFlowTestCase):
       self.assertEqual(2, image0.shape[-1])
       self.assertAllEqual(image0, image1)
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     with self.test_session(use_gpu=True):
       png = constant_op.constant("nonsense")
@@ -3337,6 +3403,7 @@ class GifTest(test_util.TensorFlowTestCase):
     self._testValid("scan.gif")
     self._testValid("optimized.gif")
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     with self.test_session(use_gpu=True) as sess:
       gif = constant_op.constant("nonsense")
@@ -3363,6 +3430,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
         self.assertTrue(y_saturate.dtype == output_dtype)
         self.assertAllClose(y_saturate.eval(), y_np, atol=1e-5)
 
+  @test_util.run_deprecated_v1
   def testNoConvert(self):
     # Make sure converting to the same data type creates only an identity op
     with self.test_session(use_gpu=True):
@@ -3372,6 +3440,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
       self.assertEquals(y.op.type, "Identity")
       self.assertEquals(y.op.inputs[0], image)
 
+  @test_util.run_deprecated_v1
   def testConvertBetweenInteger(self):
     # Make sure converting to between integer types scales appropriately
     with self.test_session(use_gpu=True):
@@ -3380,6 +3449,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
       self._convert([0, 2**32], dtypes.int64, dtypes.int32, [0, 1])
       self._convert([0, 1], dtypes.int32, dtypes.int64, [0, 2**32])
 
+  @test_util.run_deprecated_v1
   def testConvertBetweenFloat(self):
     # Make sure converting to between float types does nothing interesting
     with self.test_session(use_gpu=True):
@@ -3388,6 +3458,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
       self._convert([-1.0, 0, 1.0, 200000], dtypes.float64, dtypes.float32,
                     [-1.0, 0, 1.0, 200000])
 
+  @test_util.run_deprecated_v1
   def testConvertBetweenIntegerAndFloat(self):
     # Make sure converting from and to a float type scales appropriately
     with self.test_session(use_gpu=True):
@@ -3396,6 +3467,7 @@ class ConvertImageTest(test_util.TensorFlowTestCase):
       self._convert([0, 1.1 / 255.0, 1], dtypes.float32, dtypes.uint8,
                     [0, 1, 255])
 
+  @test_util.run_deprecated_v1
   def testConvertBetweenInt16AndInt8(self):
     with self.test_session(use_gpu=True):
       # uint8, uint16
@@ -3587,6 +3659,7 @@ class TotalVariationTest(test_util.TensorFlowTestCase):
 
 class FormatTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testFormats(self):
     prefix = "tensorflow/core/lib"
     paths = ("png/testdata/lena_gray.png", "jpeg/testdata/jpeg_merge_test1.jpg",
@@ -3619,6 +3692,7 @@ class FormatTest(test_util.TensorFlowTestCase):
 
 class NonMaxSuppressionTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testSelectFromThreeClusters(self):
     boxes_np = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
                 [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]]
@@ -3634,6 +3708,7 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase):
           boxes, scores, max_output_size, iou_threshold).eval()
       self.assertAllClose(selected_indices, [3, 0, 5])
 
+  @test_util.run_deprecated_v1
   def testInvalidShape(self):
     # The boxes should be 2D of shape [num_boxes, 4].
     with self.assertRaisesRegexp(ValueError,
@@ -3676,6 +3751,7 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase):
       scores = constant_op.constant([0.9])
       image_ops.non_max_suppression(boxes, scores, 3, [[0.5]])
 
+  @test_util.run_deprecated_v1
   def testDataTypes(self):
     # Test case for GitHub issue 20199.
     boxes_np = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
@@ -3720,6 +3796,7 @@ class NonMaxSuppressionTest(test_util.TensorFlowTestCase):
 
 class NonMaxSuppressionPaddedTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testSelectFromThreeClusters(self):
     boxes_np = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9],
                 [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]]
@@ -3752,6 +3829,7 @@ class NonMaxSuppressionPaddedTest(test_util.TensorFlowTestCase):
       self.assertAllClose(selected_indices.eval(), [3, 0, 5])
       self.assertEqual(num_valid.eval(), 3)
 
+  @test_util.run_deprecated_v1
   def testSelectFromContinuousOverLap(self):
     boxes_np = [[0, 0, 1, 1], [0, 0.2, 1, 1.2], [0, 0.4, 1, 1.4],
                 [0, 0.6, 1, 1.6], [0, 0.8, 1, 1.8], [0, 2, 1, 2]]
@@ -3779,6 +3857,7 @@ class NonMaxSuppressionPaddedTest(test_util.TensorFlowTestCase):
 
 class NonMaxSuppressionWithOverlapsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testSelectOneFromThree(self):
     overlaps_np = [
         [1.0, 0.7, 0.2],
@@ -3804,6 +3883,7 @@ class NonMaxSuppressionWithOverlapsTest(test_util.TensorFlowTestCase):
 class VerifyCompatibleImageShapesTest(test_util.TensorFlowTestCase):
   """Tests utility function used by ssim() and psnr()."""
 
+  @test_util.run_deprecated_v1
   def testWrongDims(self):
     img = array_ops.placeholder(dtype=dtypes.float32)
     img_np = np.array((2, 2))
@@ -3813,6 +3893,7 @@ class VerifyCompatibleImageShapesTest(test_util.TensorFlowTestCase):
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(checks, {img: img_np})
 
+  @test_util.run_deprecated_v1
   def testShapeMismatch(self):
     img1 = array_ops.placeholder(dtype=dtypes.float32)
     img2 = array_ops.placeholder(dtype=dtypes.float32)
@@ -3853,6 +3934,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
     """Returns an image or image batch with given shape."""
     return np.random.rand(*shape).astype(np.float32) * max_val
 
+  @test_util.run_deprecated_v1
   def testPSNRSingleImage(self):
     image1 = self._RandomImage((8, 8, 1), 1)
     image2 = self._RandomImage((8, 8, 1), 1)
@@ -3866,6 +3948,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
       tf_psnr = image_ops.psnr(tf_image1, tf_image2, 1.0, "psnr").eval()
       self.assertAllClose(psnr, tf_psnr, atol=0.001)
 
+  @test_util.run_deprecated_v1
   def testPSNRMultiImage(self):
     image1 = self._RandomImage((10, 8, 8, 1), 1)
     image2 = self._RandomImage((10, 8, 8, 1), 1)
@@ -3879,6 +3962,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
       tf_psnr = image_ops.psnr(tf_image1, tf_image2, 1, "psnr").eval()
       self.assertAllClose(psnr, tf_psnr, atol=0.001)
 
+  @test_util.run_deprecated_v1
   def testGoldenPSNR(self):
     q20, q72, q95 = self._LoadTestImages()
 
@@ -3903,6 +3987,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
       self.assertAllClose(psnr2, tf_psnr2, atol=0.001)
       self.assertAllClose(psnr3, tf_psnr3, atol=0.001)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     q20, _, _ = self._LoadTestImages()
     psnr = self._PSNR_NumPy(q20, q20, 1)
@@ -3911,6 +3996,7 @@ class PSNRTest(test_util.TensorFlowTestCase):
       tf_psnr = image_ops.psnr(tf_q20, tf_q20, 1, "psnr").eval()
       self.assertAllClose(psnr, tf_psnr, atol=0.001)
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     img1 = self._RandomImage((10, 8, 8, 1), 255)
     img2 = self._RandomImage((10, 8, 8, 1), 255)
@@ -3952,6 +4038,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     """Returns an image or image batch with given shape."""
     return np.random.rand(*shape).astype(np.float32) * max_val
 
+  @test_util.run_deprecated_v1
   def testAgainstMatlab(self):
     """Tests against values produced by Matlab."""
     img = self._LoadTestImages()
@@ -3989,6 +4076,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       self.assertAllClose(expected, self.evaluate(ssim), atol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testNegative(self):
     """Tests against negative SSIM index."""
     step = np.expand_dims(np.arange(0, 256, 16, dtype=np.uint8), axis=0)
@@ -4003,6 +4091,7 @@ class SSIMTest(test_util.TensorFlowTestCase):
     with self.test_session(use_gpu=True):
       self.assertLess(ssim.eval(), 0)
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     img1 = self._RandomImage((1, 16, 16, 3), 255)
     img2 = self._RandomImage((1, 16, 16, 3), 255)
@@ -4044,6 +4133,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     """Returns an image or image batch with given shape."""
     return np.random.rand(*shape).astype(np.float32) * max_val
 
+  @test_util.run_deprecated_v1
   def testAgainstMatlab(self):
     """Tests against MS-SSIM computed with Matlab implementation.
 
@@ -4060,6 +4150,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
 
     self.assertAllClose(expected, np.squeeze(scores), atol=1e-4)
 
+  @test_util.run_deprecated_v1
   def testUnweightedIsDifferentiable(self):
     img = self._LoadTestImages()
     ph = [array_ops.placeholder(dtype=dtypes.float32) for _ in range(2)]
@@ -4121,6 +4212,7 @@ class MultiscaleSSIMTest(test_util.TensorFlowTestCase):
     self.assertTrue(np.all(msssim >= 0.0))
     self.assertTrue(np.all(msssim <= 1.0))
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     img1 = self._RandomImage((1, 180, 240, 3), 255)
     img2 = self._RandomImage((1, 180, 240, 3), 255)
diff --git a/tensorflow/python/ops/losses/util_test.py b/tensorflow/python/ops/losses/util_test.py
index df2e60e2e4..22a8eaae26 100644
--- a/tensorflow/python/ops/losses/util_test.py
+++ b/tensorflow/python/ops/losses/util_test.py
@@ -20,12 +20,14 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops.losses import util
 from tensorflow.python.platform import test
 
 
 class LossesUtilTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testGetRegularizationLoss(self):
     # Empty regularization collection should evaluate to 0.0.
     with self.cached_session():
diff --git a/tensorflow/python/ops/math_grad_test.py b/tensorflow/python/ops/math_grad_test.py
index d1fe834fc7..88aa48271f 100644
--- a/tensorflow/python/ops/math_grad_test.py
+++ b/tensorflow/python/ops/math_grad_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients
@@ -52,6 +53,7 @@ class SquaredDifferenceOpTest(test.TestCase):
     self.assertLess(left_err, 1e-10)
     self.assertLess(right_err, 1e-10)
 
+  @test_util.run_deprecated_v1
   def testGrad(self):
     self._testGrad([1, 2, 3, 2], [3, 2])
     self._testGrad([2, 4], [3, 2, 4])
@@ -83,6 +85,7 @@ class AbsOpTest(test.TestCase):
           value, shape, output, output.get_shape().as_list())
     self.assertLess(error, max_error)
 
+  @test_util.run_deprecated_v1
   def testComplexAbs(self):
     # Bias random test values away from zero to avoid numeric instabilities.
     self._testGrad(
@@ -99,6 +102,7 @@ class AbsOpTest(test.TestCase):
 
 class MinOrMaxGradientTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testMinGradient(self):
     inputs = constant_op.constant([1.0], dtype=dtypes.float32)
     outputs = math_ops.reduce_min(array_ops.concat([inputs, inputs], 0))
@@ -106,6 +110,7 @@ class MinOrMaxGradientTest(test.TestCase):
       error = gradient_checker.compute_gradient_error(inputs, [1], outputs, [])
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testMaxGradient(self):
     inputs = constant_op.constant([1.0], dtype=dtypes.float32)
     outputs = math_ops.reduce_max(array_ops.concat([inputs, inputs], 0))
@@ -116,6 +121,7 @@ class MinOrMaxGradientTest(test.TestCase):
 
 class MaximumOrMinimumGradientTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testMaximumGradient(self):
     inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32)
     outputs = math_ops.maximum(inputs, 3.0)
@@ -123,6 +129,7 @@ class MaximumOrMinimumGradientTest(test.TestCase):
       error = gradient_checker.compute_gradient_error(inputs, [4], outputs, [4])
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testMinimumGradient(self):
     inputs = constant_op.constant([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32)
     outputs = math_ops.minimum(inputs, 2.0)
@@ -133,6 +140,7 @@ class MaximumOrMinimumGradientTest(test.TestCase):
 
 class ProdGradientTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testProdGradient(self):
     inputs = constant_op.constant([[1., 2.], [3., 4.]],
                                   dtype=dtypes.float32)
@@ -143,6 +151,7 @@ class ProdGradientTest(test.TestCase):
           outputs, outputs.get_shape().as_list())
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testProdGradientForNegativeAxis(self):
     inputs = constant_op.constant([[1., 2.], [3., 4.]],
                                   dtype=dtypes.float32)
@@ -153,6 +162,7 @@ class ProdGradientTest(test.TestCase):
           outputs, outputs.get_shape().as_list())
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testProdGradientComplex(self):
     for dtype in dtypes.complex64, dtypes.complex128:
       inputs = constant_op.constant([[1 + 3j, 2 - 1j], [3j, 4]],
@@ -164,6 +174,7 @@ class ProdGradientTest(test.TestCase):
             outputs, outputs.get_shape().as_list())
         self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testProdGradientForNegativeAxisComplex(self):
     for dtype in dtypes.complex64, dtypes.complex128:
       inputs = constant_op.constant([[1 + 3j, 2 - 1j], [3j, 4]],
@@ -178,6 +189,7 @@ class ProdGradientTest(test.TestCase):
 
 class SegmentMinOrMaxGradientTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testSegmentMinGradient(self):
     data = constant_op.constant([1.0, 2.0, 3.0], dtype=dtypes.float32)
     segment_ids = constant_op.constant([0, 0, 1], dtype=dtypes.int64)
@@ -187,6 +199,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase):
                                                       [2])
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testSegmentMaxGradient(self):
     data = constant_op.constant([1.0, 2.0, 3.0], dtype=dtypes.float32)
     segment_ids = constant_op.constant([0, 0, 1], dtype=dtypes.int64)
@@ -196,6 +209,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase):
                                                       [2])
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testSegmentMinGradientWithTies(self):
     inputs = constant_op.constant([1.0], dtype=dtypes.float32)
     data = array_ops.concat([inputs, inputs], 0)
@@ -206,6 +220,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase):
                                                       [1])
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testSegmentMaxGradientWithTies(self):
     inputs = constant_op.constant([1.0], dtype=dtypes.float32)
     data = array_ops.concat([inputs, inputs], 0)
@@ -219,6 +234,7 @@ class SegmentMinOrMaxGradientTest(test.TestCase):
 
 class FloorModGradientTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testFloorModGradient(self):
     # Making sure the input is not near the discontinuity point where
     # x/y == floor(x/y)
@@ -233,6 +249,7 @@ class FloorModGradientTest(test.TestCase):
 
 class DivNoNanGradientTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasicGradient(self):
     inputs = constant_op.constant(np.arange(-3, 3),
                                   dtype=dtypes.float32)
@@ -244,6 +261,7 @@ class DivNoNanGradientTest(test.TestCase):
           outputs.get_shape().as_list())
       self.assertLess(error, 1e-4)
 
+  @test_util.run_deprecated_v1
   def testGradientWithDenominatorIsZero(self):
     x = constant_op.constant(np.arange(-3, 3),
                              dtype=dtypes.float32)
@@ -263,6 +281,7 @@ class XlogyTest(test.TestCase):
     xlogy_ygrad = self.evaluate(gradients.gradients(math_ops.xlogy(x, y), y)[0])
     return xlogy_xgrad, xlogy_ygrad
 
+  @test_util.run_deprecated_v1
   def testNonZeroValuesGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0.1, dtype=dtype)
@@ -273,6 +292,7 @@ class XlogyTest(test.TestCase):
       self.assertAllClose(xlogy_expected_xgrad, xlogy_xgrad)
       self.assertAllClose(xlogy_expected_ygrad, xlogy_ygrad)
 
+  @test_util.run_deprecated_v1
   def testZeroXGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0., dtype=dtype)
@@ -282,6 +302,7 @@ class XlogyTest(test.TestCase):
       self.assertAllClose(zero, xlogy_xgrad)
       self.assertAllClose(zero, xlogy_ygrad)
 
+  @test_util.run_deprecated_v1
   def testZeroYGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0.1, dtype=dtype)
@@ -290,6 +311,7 @@ class XlogyTest(test.TestCase):
       self.assertAllClose(-np.inf, xlogy_xgrad)
       self.assertAllClose(np.inf, xlogy_ygrad)
 
+  @test_util.run_deprecated_v1
   def testZeroXYGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0., dtype=dtype)
@@ -307,6 +329,7 @@ class XdivyTest(test.TestCase):
     xdivy_ygrad = self.evaluate(gradients.gradients(math_ops.xdivy(x, y), y)[0])
     return xdivy_xgrad, xdivy_ygrad
 
+  @test_util.run_deprecated_v1
   def testNonZeroValuesGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0.1, dtype=dtype)
@@ -317,6 +340,7 @@ class XdivyTest(test.TestCase):
       self.assertAllClose(xdivy_expected_xgrad, xdivy_xgrad)
       self.assertAllClose(xdivy_expected_ygrad, xdivy_ygrad)
 
+  @test_util.run_deprecated_v1
   def testZeroXGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0., dtype=dtype)
@@ -326,6 +350,7 @@ class XdivyTest(test.TestCase):
       self.assertAllClose(zero, xdivy_xgrad)
       self.assertAllClose(zero, xdivy_ygrad)
 
+  @test_util.run_deprecated_v1
   def testZeroYGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0.1, dtype=dtype)
@@ -334,6 +359,7 @@ class XdivyTest(test.TestCase):
       self.assertAllClose(np.inf, xdivy_xgrad)
       self.assertAllClose(-np.inf, xdivy_ygrad)
 
+  @test_util.run_deprecated_v1
   def testZeroXYGrad(self):
     for dtype in [dtypes.float16, dtypes.float32, dtypes.float64]:
       x = constant_op.constant(0., dtype=dtype)
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index cd45b6f136..add1621a56 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -92,6 +92,7 @@ class ReduceTest(test_util.TensorFlowTestCase):
 
 class LogSumExpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testReduceLogSumExp(self):
     for dtype in [np.float16, np.float32, np.double]:
       x_np = np.random.rand(5, 5).astype(dtype)
@@ -120,6 +121,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
         y_tf_np = self.evaluate(y_tf)
         self.assertAllClose(y_tf_np, y_np)
 
+  @test_util.run_deprecated_v1
   def testKeepDims(self):
     for dtype in [np.float16, np.float32, np.double]:
       x_np = np.random.rand(5, 5).astype(dtype)
@@ -129,6 +131,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
         y_np = log(np.sum(exp(x_np), keepdims=True))
         self.assertAllClose(y_tf_np, y_np)
 
+  @test_util.run_deprecated_v1
   def testOverflow(self):
     x = [1000, 1001, 1002, 1003]
     for dtype in [np.float16, np.float32, np.double]:
@@ -146,6 +149,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
         y_np = log(np.sum(exp(x_np - max_np))) + max_np
         self.assertAllClose(y_tf_np, y_np)
 
+  @test_util.run_deprecated_v1
   def testUnderflow(self):
     x = [-1000, -1001, -1002, -1003]
     for dtype in [np.float16, np.float32, np.double]:
@@ -163,6 +167,7 @@ class LogSumExpTest(test_util.TensorFlowTestCase):
         y_np = log(np.sum(exp(x_np - max_np))) + max_np
         self.assertAllClose(y_tf_np, y_np)
 
+  @test_util.run_deprecated_v1
   def testInfinity(self):
     with self.session(use_gpu=True):
       res = math_ops.reduce_logsumexp(-np.inf).eval()
@@ -186,6 +191,7 @@ class RoundTest(test_util.TensorFlowTestCase):
 
 class ModTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     x = [0.5, 0.7, 0.3]
     for dtype in [np.float32, np.double]:
@@ -256,6 +262,7 @@ class ApproximateEqualTest(test_util.TensorFlowTestCase):
         z_tf = self.evaluate(math_ops.approximate_equal(x, y, tolerance=0.0001))
         self.assertAllEqual(z, z_tf)
 
+  @test_util.run_deprecated_v1
   def testApproximateEqualShape(self):
     for dtype in [np.float32, np.double]:
       x = np.array([1, 2], dtype=dtype)
@@ -309,6 +316,7 @@ class ScalarMulTest(test_util.TensorFlowTestCase):
 
 class AccumulateNTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     np.random.seed(12345)
     x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
@@ -317,6 +325,7 @@ class AccumulateNTest(test_util.TensorFlowTestCase):
       self.assertAllClose(sum(x), math_ops.accumulate_n(tf_x).eval())
       self.assertAllClose(x[0] * 5, math_ops.accumulate_n([tf_x[0]] * 5).eval())
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     np.random.seed(54321)
     x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)]
@@ -328,6 +337,7 @@ class AccumulateNTest(test_util.TensorFlowTestCase):
 
 class AddNTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testPartials(self):
     """Test that previously revealed a bug in buffer forwarding for AddN."""
     partials = []
@@ -341,6 +351,7 @@ class AddNTest(test_util.TensorFlowTestCase):
     with self.session(use_gpu=True):
       self.assertAllEqual(res.eval(), 100)
 
+  @test_util.run_deprecated_v1
   def testFloat(self):
     np.random.seed(12345)
     for num_inputs in range(1, 10):
@@ -351,6 +362,7 @@ class AddNTest(test_util.TensorFlowTestCase):
         self.assertAllClose(x[0] * num_inputs,
                             math_ops.add_n([tf_x[0]] * num_inputs).eval())
 
+  @test_util.run_deprecated_v1
   def testInt(self):
     np.random.seed(54321)
     for num_inputs in range(1, 10):
@@ -364,6 +376,7 @@ class AddNTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(x[0] * num_inputs,
                             math_ops.add_n([tf_x[0]] * num_inputs).eval())
 
+  @test_util.run_deprecated_v1
   def testGrad(self):
     np.random.seed(42)
     for num_inputs in range(1, 10):
@@ -392,6 +405,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
     divs = np.arange(-3, 0, .25).reshape(1, 12)
     return nums, divs
 
+  @test_util.run_deprecated_v1
   def testFloorModInt(self):
     nums, divs = self.intTestData()
     with self.cached_session():
@@ -401,6 +415,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       np_result = nums % divs
       self.assertAllEqual(tf_result, np_result)
 
+  @test_util.run_deprecated_v1
   def testFloorModFloat(self):
     nums, divs = self.floatTestData()
     with self.cached_session():
@@ -412,6 +427,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       #               % array_ops.constant(divs)).eval()
       # self.assertAllEqual(tf2_result, tf_result)
 
+  @test_util.run_deprecated_v1
   def testTruncateModInt(self):
     nums, divs = self.intTestData()
     with self.cached_session():
@@ -419,6 +435,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       np_result = np.fmod(nums, divs)
       self.assertAllEqual(tf_result, np_result)
 
+  @test_util.run_deprecated_v1
   def testTruncateModFloat(self):
     nums, divs = self.floatTestData()
     with self.cached_session():
@@ -426,6 +443,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       np_result = np.fmod(nums, divs)
       self.assertAllEqual(tf_result, np_result)
 
+  @test_util.run_deprecated_v1
   def testDivideInt(self):
     nums, divs = self.intTestData()
     with self.cached_session():
@@ -437,12 +455,14 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       #               // array_ops.constant(divs)).eval()
       # self.assertAllEqual(tf2_result, tf_result)
 
+  @test_util.run_deprecated_v1
   def testDivideName(self):
     with self.cached_session():
       op = math_ops.divide(
           array_ops.constant(3), array_ops.constant(4), name="my_cool_divide")
       self.assertEqual(op.name, "my_cool_divide:0")
 
+  @test_util.run_deprecated_v1
   def testRealDiv(self):
     nums, divs = self.floatTestData()
     with self.cached_session():
@@ -450,12 +470,14 @@ class DivAndModTest(test_util.TensorFlowTestCase):
       np_result = np.divide(nums, divs)
       self.assertAllEqual(tf_result, np_result)
 
+  @test_util.run_deprecated_v1
   def testComplexDiv(self):
     foo = array_ops.constant([1. + 3.j])
     with self.cached_session():
       _ = math_ops.divide(foo, 1.).eval()
       _ = math_ops.div(foo, 2.).eval()
 
+  @test_util.run_deprecated_v1
   def testFloorDivGrad(self):
     with self.cached_session():
       a = variables.Variable(2.)
@@ -471,6 +493,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
             [None if x is None else self.evaluate(x) for x in c_grad],
             [None, None])
 
+  @test_util.run_deprecated_v1
   def testConsistent(self):
     nums, divs = self.intTestData()
     with self.cached_session():
@@ -497,6 +520,7 @@ class DivAndModTest(test_util.TensorFlowTestCase):
 
 class DivNoNanTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     for dtype in [np.float32, np.float64]:
       nums = np.arange(-10, 10, .25, dtype=dtype).reshape(80, 1)
diff --git a/tensorflow/python/ops/nn_batchnorm_test.py b/tensorflow/python/ops/nn_batchnorm_test.py
index 31b2790f2b..e978f1d326 100644
--- a/tensorflow/python/ops/nn_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_batchnorm_test.py
@@ -71,6 +71,7 @@ class BatchNormalizationTest(test.TestCase):
                                        gamma if scale_after_normalization else
                                        None, epsilon)
 
+  @test_util.run_deprecated_v1
   def testBatchNorm(self):
     x_shape = [3, 5, 4, 2]
     param_shape = [2]
@@ -169,16 +170,20 @@ class BatchNormalizationTest(test.TestCase):
                                       shift_after_normalization, v,
                                       err_tolerance)
 
+  @test_util.run_deprecated_v1
   def testBatchNormInputGradient(self):
     self._testBatchNormGradientInAllNeedConfigs(0, "x")
 
+  @test_util.run_deprecated_v1
   def testBatchNormMeanGradient(self):
     self._testBatchNormGradientInAllNeedConfigs(1, "mean")
 
+  @test_util.run_deprecated_v1
   def testBatchNormVarianceGradient(self):
     self._testBatchNormGradientInAllNeedConfigs(
         2, "variance", err_tolerance=1e-03)
 
+  @test_util.run_deprecated_v1
   def testBatchNormBetaGradient(self):
     # Since beta does not exist when scale_after_normalization=False, we only
     # test for scale_after_normalization=True.
@@ -187,6 +192,7 @@ class BatchNormalizationTest(test.TestCase):
         self._testBatchNormGradient(3, "beta", scale_after_normalization, True,
                                     v)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGammaGradient(self):
     # If scale_after_normalization is False, backprop for gamma in v1
     # will be 0. In version 2 of the API, if scale_after_normalization is False,
@@ -199,6 +205,7 @@ class BatchNormalizationTest(test.TestCase):
       self._testBatchNormGradient(4, "gamma", True, shift_after_normalization,
                                   2)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradImpl(self):
     x_shape = [7, 5, 4, 6]
     param_shape = [6]
@@ -245,6 +252,7 @@ class BatchNormalizationTest(test.TestCase):
             self.assertAllClose(
                 all_grads[i + len(to_check)], all_grads[i], atol=0.000001)
 
+  @test_util.run_deprecated_v1
   def testBatchNormKeepDims(self):
     """Test for tf.nn.moments(..., keep_dims=True / False).
 
@@ -391,6 +399,7 @@ class SufficientStatisticsTest(test.TestCase):
         if shift:
           self.assertAllClose(np_s, tf_s, atol=0.000001)
 
+  @test_util.run_deprecated_v1
   def testSuffStats(self):
     for has_shape in [True, False]:
       for keep_dims in [True, False]:
@@ -511,6 +520,7 @@ class MomentsTest(test.TestCase):
       self.assertAllCloseAccordingToType(expected_mean, self.evaluate(mean))
       self.assertAllCloseAccordingToType(expected_variance, self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     for keep_dims in [False, True]:
       for dtype in [dtypes.float32, dtypes.float16]:
@@ -519,6 +529,7 @@ class MomentsTest(test.TestCase):
         self.RunMomentTestWithDynamicShape(
             shape=[2, 3, 5, 4], axes=[0], keep_dims=keep_dims, dtype=dtype)
 
+  @test_util.run_deprecated_v1
   def testGlobalNormalization(self):
     for keep_dims in [False, True]:
       for dtype in [dtypes.float32, dtypes.float16]:
@@ -533,6 +544,7 @@ class MomentsTest(test.TestCase):
             keep_dims=keep_dims,
             dtype=dtype)
 
+  @test_util.run_deprecated_v1
   def testAxes(self):
     for keep_dims in [False, True]:
       for dtype in [dtypes.float32, dtypes.float16]:
@@ -573,9 +585,11 @@ class MomentsTest(test.TestCase):
         print("Moments %s gradient err vs input %d = %g" % (from_y, i, err))
         self.assertLess(err, 1e-11)
 
+  @test_util.run_deprecated_v1
   def testMeanGlobalGradient(self):
     self._testGlobalGradient(from_y="mean")
 
+  @test_util.run_deprecated_v1
   def testVarGlobalGradient(self):
     self._testGlobalGradient(from_y="var")
 
diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py
index 8065df4b16..95e05a977b 100644
--- a/tensorflow/python/ops/nn_grad_test.py
+++ b/tensorflow/python/ops/nn_grad_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import nn_grad  # pylint: disable=unused-import
@@ -31,6 +32,7 @@ from tensorflow.python.platform import test
 
 class Relu6OpTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testRelu6GradGrad(self):
     inputs = constant_op.constant(
         [[-2, -1, 1, 3], [5, 7, 8, 9]], dtype=dtypes.float32)
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index f7c8a7a70a..a7ed834c72 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -49,6 +49,7 @@ class ZeroFractionTest(test_lib.TestCase):
     nonzeros = np.count_nonzero(x.flatten())
     return 1.0 - nonzeros / total_elements
 
+  @test_util.run_deprecated_v1
   def testZeroFraction(self):
     x_shape = [5, 17]
     x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32)
@@ -62,21 +63,25 @@ class ZeroFractionTest(test_lib.TestCase):
     eps = 1e-8
     self.assertAllClose(y_tf_np, y_np, eps)
 
+  @test_util.run_deprecated_v1
   def testZeroFractionEmpty(self):
     x = np.zeros(0)
     y = self.evaluate(nn_impl.zero_fraction(x))
     self.assertTrue(np.isnan(y))
 
+  @test_util.run_deprecated_v1
   def testZeroFraction2_27Zeros(self):
     sparsity = nn_impl.zero_fraction(
         array_ops.zeros([int(2**27 * 1.01)], dtype=dtypes.int8))
     self.assertAllClose(1.0, self.evaluate(sparsity))
 
+  @test_util.run_deprecated_v1
   def testZeroFraction2_27Ones(self):
     sparsity = nn_impl.zero_fraction(
         array_ops.ones([int(2**27 * 1.01)], dtype=dtypes.int8))
     self.assertAllClose(0.0, self.evaluate(sparsity))
 
+  @test_util.run_deprecated_v1
   def testUnknownSize(self):
     value = array_ops.placeholder(dtype=dtypes.float32)
     sparsity = nn_impl.zero_fraction(value)
@@ -122,6 +127,7 @@ class SoftmaxTest(test_lib.TestCase, parameterized.TestCase):
     self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps)
 
   @parameterized.parameters(((5, 10),), ((2, 3, 4),))
+  @test_util.run_deprecated_v1
   def testGradient(self, x_shape):
     x_np = np.random.randn(*x_shape).astype(np.float64)
     with self.cached_session():
@@ -157,6 +163,7 @@ class LogPoissonLossTest(test_lib.TestCase):
     self.assertAllClose(y_tf_np, y_np, eps)
     self.assertAllClose(y_tf_np_stirling, y_np_stirling, eps)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     x_shape = [5, 10]
     x_np = np.random.randn(*x_shape).astype(np.float64)
@@ -207,6 +214,7 @@ class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase):
     self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps)
 
   @parameterized.parameters(((5, 10),), ((2, 3, 4),))
+  @test_util.run_deprecated_v1
   def testGradient(self, x_shape):
     x_np = np.random.randn(*x_shape).astype(np.float64)
     with self.cached_session():
@@ -229,6 +237,7 @@ class L2LossTest(test_lib.TestCase):
       value = self.evaluate(l2loss)
       self.assertAllClose(7.0, value)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     x_shape = [20, 7, 3]
     np.random.seed(1)  # Make it reproducible.
@@ -276,6 +285,7 @@ class L2NormalizeTest(test_lib.TestCase):
     y_tf = nn_impl.l2_normalize_v2(x_tf, dim)
     self.assertAllClose(y_np, self.evaluate(y_tf))
 
+  @test_util.run_deprecated_v1
   def testL2NormalizeGradient(self):
     x_shape = [20, 7, 3]
     np.random.seed(1)
@@ -361,6 +371,7 @@ class DropoutTest(test_lib.TestCase):
           sorted_value = np.unique(np.sort(value[i, :]))
           self.assertEqual(sorted_value.size, 1)
 
+  @test_util.run_deprecated_v1
   def testDropoutPlaceholderKeepProb(self):
     # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate
     # that it is producing approximately the right number of ones over a large
@@ -389,6 +400,7 @@ class DropoutTest(test_lib.TestCase):
       print(rel_error)
       self.assertTrue(rel_error < 0.15)
 
+  @test_util.run_deprecated_v1
   def testShapedDropoutUnknownShape(self):
     x_dim = 40
     y_dim = 30
@@ -422,6 +434,7 @@ class DropoutTest(test_lib.TestCase):
       print(rel_error)
       self.assertTrue(rel_error < 0.15)
 
+  @test_util.run_deprecated_v1
   def testInvalidKeepProb(self):
     x_dim = 40
     y_dim = 30
@@ -437,6 +450,7 @@ class DropoutTest(test_lib.TestCase):
     with self.assertRaises(ValueError):
       nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2]))
 
+  @test_util.run_deprecated_v1
   def testShapedDropoutShapeError(self):
     # Runs shaped dropout and verifies an error is thrown on misshapen noise.
     x_dim = 40
@@ -457,6 +471,7 @@ class DropoutTest(test_lib.TestCase):
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[1, 1])
 
+  @test_util.run_deprecated_v1
   def testNoDropoutFast(self):
     x = array_ops.zeros((5,))
     for p in 1, constant_op.constant(1.0):
@@ -938,6 +953,7 @@ class ReluTest(test_lib.TestCase):
     z = self.evaluate(nn_ops.relu(constant_op.constant(x)))
     self.assertAllEqual(y, z)
 
+  @test_util.run_deprecated_v1
   def testNaNs(self):
     # Test that relu(nan) = nan for various sizes.
     for i in range(18):
@@ -966,6 +982,7 @@ class LeakyReluTest(test_lib.TestCase):
     self.assertLessEqual(outputs.max(), 1.0)
     self.assertAllClose(inputs, outputs)
 
+  @test_util.run_deprecated_v1
   def testValues(self):
     for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]:
       np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype)
@@ -977,6 +994,7 @@ class LeakyReluTest(test_lib.TestCase):
       self.assertAllClose(
           outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol)
 
+  @test_util.run_deprecated_v1
   def testName(self):
     np_values = np.array([-2, -1, 0, 1, 2], dtype=np.float64)
     outputs_with_name_set = nn_ops.leaky_relu(
@@ -990,6 +1008,7 @@ class LeakyReluTest(test_lib.TestCase):
 
 class SwishTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testValues(self):
     np_values = np.array(
         [np.linspace(-10.0, 0.0, 100),
@@ -1004,6 +1023,7 @@ class SwishTest(test_lib.TestCase):
 
     self.assertAllClose(actual_outputs, expected_outputs)
 
+  @test_util.run_deprecated_v1
   def testGradients(self):
     shape = [5, 3, 4]
     sigma = 5
diff --git a/tensorflow/python/ops/nn_xent_test.py b/tensorflow/python/ops/nn_xent_test.py
index 7bf18c47fe..3e5c198fc6 100644
--- a/tensorflow/python/ops/nn_xent_test.py
+++ b/tensorflow/python/ops/nn_xent_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import nn_impl
@@ -53,6 +54,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase):
     losses = np.array(self._SigmoidCrossEntropyWithLogits(x, y)).reshape(*sizes)
     return logits, targets, losses
 
+  @test_util.run_deprecated_v1
   def testConstructionNamed(self):
     with self.cached_session():
       logits, targets, _ = self._Inputs()
@@ -82,6 +84,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase):
           tf_loss = self.evaluate(loss)
         self.assertAllClose(np_loss, tf_loss, atol=0.001)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     sizes = [4, 2]
     with self.cached_session():
@@ -92,6 +95,7 @@ class SigmoidCrossEntropyWithLogitsTest(test.TestCase):
     print("logistic loss gradient err = ", err)
     self.assertLess(err, 1e-7)
 
+  @test_util.run_deprecated_v1
   def testGradientAtZero(self):
     with self.cached_session():
       logits = constant_op.constant([0.0, 0.0], dtype=dtypes.float64)
@@ -129,6 +133,7 @@ class WeightedCrossEntropyTest(test.TestCase):
     losses = np.array(self._WeightedCrossEntropy(x, y, q)).reshape(*sizes)
     return logits, targets, q, losses
 
+  @test_util.run_deprecated_v1
   def testConstructionNamed(self):
     with self.cached_session():
       logits, targets, pos_weight, _ = self._Inputs()
@@ -157,6 +162,7 @@ class WeightedCrossEntropyTest(test.TestCase):
         tf_loss = self.evaluate(loss)
       self.assertAllClose(np_loss, tf_loss, atol=0.001)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     sizes = [4, 2]
     with self.cached_session():
diff --git a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
index 243fa34c4b..ef3464f243 100644
--- a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
@@ -90,6 +90,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
           preferred_dtype=dtypes.string,
           expected_dtype=dtypes.int32),
   ])
+  @test_util.run_deprecated_v1
   def testConvertRaggedTensorValue(self,
                                    value,
                                    dtype=None,
@@ -145,6 +146,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
           message=('Tensor conversion requested dtype string for '
                    'Tensor with dtype int32')),
   ])
+  @test_util.run_deprecated_v1
   def testConvertTensorError(self,
                              pylist,
                              message,
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
index 79a2ecd87a..d9d840500c 100644
--- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
@@ -135,6 +135,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
           expected=ragged.constant_value(
               [[[[b'c', b'a'], [b'd', b'd']], [[b'f', b'e']]]], ragged_rank=2)),
   ])
+  @test_util.run_deprecated_v1
   def testRaggedBatchGather(self, descr, params, indices, expected):
     result = ragged.batch_gather(params, indices)
     self.assertEqual(
@@ -144,6 +145,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
         expected = expected.tolist()
       self.assertEqual(result.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testRaggedBatchGatherUnknownRankError(self):
     params = [['a', 'b'], ['c', 'd']]
     indices = array_ops.placeholder(dtypes.int32, shape=None)
@@ -186,6 +188,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
            indices=[[[0]]],
            message='batch shape from indices does not match params shape'),
   ])
+  @test_util.run_deprecated_v1
   def testRaggedBatchGatherStaticError(self,
                                        params,
                                        indices,
diff --git a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
index b3279c1e84..d939d9d634 100644
--- a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
@@ -298,6 +298,7 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase,
           keepdims=True,
           expected=ragged.constant_value([[[1], [4, 6]], [[7, 9], []]])),
   ])  # pyformat: disable
+  @test_util.run_deprecated_v1
   def testBooleanMask(self, descr, data, mask, keepdims, expected):
     actual = ragged.boolean_mask(data, mask, keepdims=keepdims)
     self.assertEqual(
@@ -307,6 +308,7 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase,
         expected = expected.tolist()
       self.assertEqual(actual.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testErrors(self):
     self.assertRaisesRegexp(ValueError,
                             r'mask\.shape\.ndims must be kown statically',
diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
index bddc5d8580..3699f90f46 100644
--- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
@@ -221,6 +221,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=[[b'a00', b'a01'], [], [b'a20', b'a21']]),
   )   # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedConcat(self,
                        descr,
                        rt_inputs,
@@ -266,6 +267,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           error=ValueError,
           message='Dimension 0 in both shapes must be equal'),
   )
+  @test_util.run_deprecated_v1
   def testStaticError(self, rt_inputs, axis, error, message, ragged_ranks=None):
     rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks)
     self.assertRaisesRegexp(error, message, ragged.concat, rt_inputs, axis)
@@ -278,6 +280,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           error=errors.InvalidArgumentError,
           message='Input tensors have incompatible shapes'),
   ])
+  @test_util.run_deprecated_v1
   def testRuntimeError(self, rt_inputs, axis, error, message,
                        ragged_ranks=None):
     rt_inputs = [
@@ -287,6 +290,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.test_session():
       self.assertRaisesRegexp(error, message, concatenated.eval)
 
+  @test_util.run_deprecated_v1
   def testNegativeAxisWithUnknownRankError(self):
     rt_inputs = [
         array_ops.placeholder(dtypes.int64),
@@ -296,6 +300,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         ValueError, r'axis may only be negative if ndims is statically known.',
         ragged.concat, rt_inputs, -1)
 
+  @test_util.run_deprecated_v1
   def testSingleTensorInput(self):
     """Tests ragged_concat with a single tensor input.
 
diff --git a/tensorflow/python/ops/ragged/ragged_const_op_test.py b/tensorflow/python/ops/ragged/ragged_const_op_test.py
index 9c3b2ac88a..2505b23912 100644
--- a/tensorflow/python/ops/ragged/ragged_const_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_const_op_test.py
@@ -133,6 +133,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       dict(pylist=[[b'a', b'b'], [b'c'], [b'd', b'e', b'f']],
            dtype=dtypes.string),
   )
+  @test_util.run_deprecated_v1
   def testRaggedConst(self,
                       pylist,
                       dtype=None,
@@ -258,6 +259,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           exception=ValueError,
           message='inner values have inconsistent shape'),
   )
+  @test_util.run_deprecated_v1
   def testRaggedConstError(self,
                            pylist,
                            dtype=None,
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
index 26e6b8b6d4..305a96df9c 100644
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
+++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
@@ -394,6 +394,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
         result_flat_values = array_ops.reshape(result, [-1])
       self.assertAllEqual(expected_flat_values, result_flat_values)
 
+  @test_util.run_deprecated_v1
   def testUnknownRankError(self):
     x = ragged.constant([[1, 2], [3]])
     y = ragged.from_row_splits(
diff --git a/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py b/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py
index 0c4fd458c2..3ff66973b6 100644
--- a/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py
@@ -105,6 +105,7 @@ class RaggedExpandDimsOpTest(test_util.TensorFlowTestCase,
            expected=EXAMPLE4D_EXPAND_AXIS[4],
            expected_shape=[3, None, None, 2, 1]),
   ])  # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedExpandDims(self,
                            rt_input,
                            axis,
diff --git a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
index 77418ff20d..3c0db9e8fb 100644
--- a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     st = sparse_tensor.SparseTensor(
         indices=[[0, 0], [0, 1], [0, 2], [1, 0], [3, 0]],
@@ -39,6 +40,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(rt.eval().tolist(), [[1, 2, 3], [4], [], [5]])
 
+  @test_util.run_deprecated_v1
   def testEmpty(self):
     st = sparse_tensor.SparseTensor(
         indices=array_ops.zeros([0, 2], dtype=dtypes.int64),
@@ -49,6 +51,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(rt.eval().tolist(), [[], [], [], []])
 
+  @test_util.run_deprecated_v1
   def testBadSparseTensorRank(self):
     st1 = sparse_tensor.SparseTensor(indices=[[0]], values=[0], dense_shape=[3])
     st2 = sparse_tensor.SparseTensor(
@@ -64,6 +67,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
                             ragged.from_sparse, st3)
 
+  @test_util.run_deprecated_v1
   def testGoodPartialSparseTensorRank(self):
     st1 = sparse_tensor.SparseTensor(
         indices=[[0, 0]],
@@ -78,6 +82,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     ragged.from_sparse(st1)
     ragged.from_sparse(st2)
 
+  @test_util.run_deprecated_v1
   def testNonRaggedSparseTensor(self):
     # "index_suffix" means the value of the innermost dimension of the index
     # (i.e., indices[i][-1]).
diff --git a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
index 7c59cd0b77..1d8a00cc18 100644
--- a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.platform import googletest
 class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                              parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     # The examples from ragged.from_tensor.__doc__.
     dt = constant_op.constant([[5, 7, 0], [0, 3, 0], [6, 0, 0]])
@@ -262,6 +263,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                        [[[5, 6], [7]], [[0, 8], []]]]
       },
   )  # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedFromTensor(self,
                            tensor,
                            expected,
@@ -278,6 +280,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
     with self.test_session():
       self.assertEqual(rt.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testHighDimensions(self):
     # Use distinct prime numbers for all dimension shapes in this test, so
     # we can see any errors that are caused by mixing up dimension sizes.
@@ -395,6 +398,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
           'expected': [[], []]
       },
   )
+  @test_util.run_deprecated_v1
   def testEmpty(self, dt_shape, expected, lengths=None, padding=None):
     dt = array_ops.zeros(dt_shape)
     rt = ragged.from_tensor(dt, lengths, padding)
@@ -447,6 +451,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
           'error': (ValueError, r'ragged_rank must be greater than 0; got -1')
       },
   )
+  @test_util.run_deprecated_v1
   def testErrors(self,
                  tensor,
                  lengths=None,
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index c52db9e2a1..62c6819374 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -183,6 +183,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
           indices=[[0, 0, 1], [0, 0, 0], [0, 1, 0]],
           expected=[[b'c', b'd'], [b'a', b'b'], [b'e', b'f']]),
   ])  # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedGatherNd(self, descr, params, indices, expected):
     result = ragged.gather_nd(params, indices)
     self.assertEqual(
@@ -192,6 +193,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
         expected = expected.tolist()
       self.assertEqual(self.evaluate(result).tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testRaggedGatherNdUnknownRankError(self):
     params = ragged.constant([['a', 'b'], ['c', 'd']])
     indices1 = array_ops.placeholder(dtypes.int32, shape=None)
@@ -219,6 +221,7 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
           indices=ragged.constant([[0]]),
           message='The innermost dimension of indices may not be ragged'),
   ])
+  @test_util.run_deprecated_v1
   def testRaggedGatherNdStaticError(self,
                                     params,
                                     indices,
diff --git a/tensorflow/python/ops/ragged/ragged_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
index bb52d05c32..76c90cdfee 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     params = constant_op.constant(['a', 'b', 'c', 'd', 'e'])
     indices = constant_op.constant([3, 1, 2, 1, 0])
@@ -46,6 +47,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
           ragged.gather(ragged_params, ragged_indices).eval().tolist(),
           [[[b'e'], [b'd'], []], [[b'd']], [], [[b'a', b'b', b'c']]])
 
+  @test_util.run_deprecated_v1
   def testTensorParamsAndTensorIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = [2, 0, 2, 1]
@@ -55,6 +57,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
           [b'c', b'a', b'c', b'b'])
       self.assertEqual(type(ragged.gather(params, indices)), ops.Tensor)
 
+  @test_util.run_deprecated_v1
   def testRaggedParamsAndTensorIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = [2, 0, 2, 1]
@@ -63,6 +66,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
           ragged.gather(params, indices).eval().tolist(),
           [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']])
 
+  @test_util.run_deprecated_v1
   def testTensorParamsAndRaggedIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = ragged.constant([[2, 1], [1, 2, 0], [3]])
@@ -71,6 +75,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
           ragged.gather(params, indices).eval().tolist(),
           [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']])
 
+  @test_util.run_deprecated_v1
   def testRaggedParamsAndRaggedIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = ragged.constant([[2, 1], [1, 2, 0], [3]])
@@ -82,6 +87,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
            [[]]]                                        #  [p[3]            ]]
       )  # pyformat: disable
 
+  @test_util.run_deprecated_v1
   def testRaggedParamsAndScalarIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = 1
@@ -89,6 +95,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(
           ragged.gather(params, indices).eval().tolist(), [b'c', b'd', b'e'])
 
+  @test_util.run_deprecated_v1
   def test3DRaggedParamsAnd2DTensorIndices(self):
     params = ragged.constant([[['a', 'b'], []], [['c', 'd'], ['e'], ['f']],
                               [['g']]])
@@ -101,6 +108,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
            [[[b'g']], [[b'g']]]]                                  #  [p2, p2]]
       )  # pyformat: disable
 
+  @test_util.run_deprecated_v1
   def testTensorParamsAnd4DRaggedIndices(self):
     indices = ragged.constant(
         [[[[3, 4], [0, 6]], []], [[[2, 1], [1, 0]], [[2, 5]], [[2, 3]]],
@@ -115,6 +123,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
            [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]],
            [[[b'b', b'a']]]])  # pyformat: disable
 
+  @test_util.run_deprecated_v1
   def testOutOfBoundsError(self):
     tensor_params = ['a', 'b', 'c']
     tensor_indices = [0, 1, 2]
@@ -131,6 +140,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
                               r'indices\[1\] = 3 is not in \[0, 2\)',
                               ragged.gather(ragged_params, ragged_indices).eval)
 
+  @test_util.run_deprecated_v1
   def testUnknownIndicesRankError(self):
     params = ragged.constant([], ragged_rank=1)
     indices = constant_op.constant([0], dtype=dtypes.int64)
diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
index dac86310b9..7a8603c949 100644
--- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
@@ -140,6 +140,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       ),
   ])
 
+  @test_util.run_deprecated_v1
   def testRaggedMap(
       self,
       fn,
@@ -164,6 +165,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       output_values = self.evaluate(output)
       self.assertAllEqual(expected_output, output_values.tolist())
 
+  @test_util.run_deprecated_v1
   def testRaggedMapOnStructure(self):
     batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]])
     # [[10, 20, 30], [40], [50, 60, 70]]
@@ -184,6 +186,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(output.eval().tolist(), [66, 44, 198])
 
   # Test mapping over a dict of RTs can produce a dict of RTs.
+  @test_util.run_deprecated_v1
   def testRaggedMapOnStructure_RaggedOutputs(self):
     batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]])
     # [[10, 20, 30], [40], [50, 60, 70]]
@@ -215,6 +218,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(output['robin'].eval().tolist(),
                           [[11, 21, 31], [41], [51, 61, 71]])
 
+  @test_util.run_deprecated_v1
   def testZip(self):
     x = ragged.constant([[10, 20], [30, 40], [50, 60], [70], [80, 90, 100]],
                         dtypes.int64)
@@ -237,6 +241,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           result, [[[0, 10], [0, 20]], [[1, 30], [1, 40]], [[2, 50], [2, 60]],
                    [[3, 70]], [[4, 80], [4, 90], [4, 100]]])
 
+  @test_util.run_deprecated_v1
   def testBatchGather(self):
     tokens = ragged.constant([['hello', '.', 'there'], ['merhaba'],
                               ['bonjour', '.', 'ca va', '?']])
diff --git a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py b/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
index 798d7c3ce8..b5802cb82d 100644
--- a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
@@ -43,6 +43,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
     with self.test_session():
       self.assertEqual(result.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     """Test the examples in apply_op_to_ragged_values.__doc__."""
     rt = ragged.constant([[1, 2, 3], [], [4, 5], [6]])
@@ -54,6 +55,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
       self.assertEqual(v2.eval().tolist(), [[1, 4, 9], [], [16, 25], [36]])
       self.assertEqual(v3.eval().tolist(), [[6, 7, 8], [], [9, 10], [11]])
 
+  @test_util.run_deprecated_v1
   def testOpWithSingleRaggedTensorArg(self):
     tensor = ragged.constant([[1, 2, 3], [], [4, 5]])
     self.assertRaggedMapInnerValuesReturns(
@@ -61,17 +63,20 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(tensor,),
         expected=[[0, 0, 0], [], [0, 0]])
 
+  @test_util.run_deprecated_v1
   def testOpWithTwoRaggedTensorArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
     self.assertRaggedMapInnerValuesReturns(
         op=math_ops.multiply, args=(x, y), expected=[[3, 2, 12], [], [4, 25]])
 
+  @test_util.run_deprecated_v1
   def testOpWithRaggedTensorAndScalarArgs(self):
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
     self.assertRaggedMapInnerValuesReturns(
         op=math_ops.multiply, args=(5, y), expected=[[5, 10, 15], [], [20, 25]])
 
+  @test_util.run_deprecated_v1
   def testOpWithThreeRaggedTensorArgs(self):
     condition = ragged.constant(
         [[True, True, False], [], [True, False]])  # pyformat: disable
@@ -82,6 +87,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(condition, x, y),
         expected=[[b'a', b'b', b'C'], [], [b'd', b'E']])
 
+  @test_util.run_deprecated_v1
   def testOpWithRaggedTensorListArg(self):
     x = ragged.constant([[1, 2, 3], [], [4, 5]])
     y = ragged.constant([[10, 20, 30], [], [40, 50]])
@@ -90,6 +96,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=([x, y, x],),
         expected=[[12, 24, 36], [], [48, 60]])
 
+  @test_util.run_deprecated_v1
   def testOpWithKeywordArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -98,6 +105,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         kwargs=dict(x=x, y=y),
         expected=[[3, 2, 12], [], [4, 25]])
 
+  @test_util.run_deprecated_v1
   def testOpWithMixedPositionalAndKeywordArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -107,6 +115,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         kwargs=dict(y=y),
         expected=[[3, 2, 12], [], [4, 25]])
 
+  @test_util.run_deprecated_v1
   def testNonElementWiseOp(self):
     x = ragged.constant(
         [[[3, 1, 4], [1, 5, 9], [2, 6, 5]], [], [[3, 5, 8], [9, 7, 9]]],
@@ -119,6 +128,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         },
         expected=[[8, 15, 13], [], [16, 25]])
 
+  @test_util.run_deprecated_v1
   def testOpWithRaggedRankGreaterThanOne(self):
     # ragged_rank=0
     x0 = [3, 1, 4, 1, 5, 9, 2, 6, 5]
@@ -163,6 +173,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
             [[[54, 14], [48, 45]]]    # row 3
         ])  # pyformat: disable
 
+  @test_util.run_deprecated_v1
   def testOpWithRaggedRankThree(self):
     x = ragged.constant([[[3, 1, 4]], [], [[], [1, 5]]])
     y = ragged.constant([[[1, 2, 3]], [], [[], [4, 5]]])
@@ -171,6 +182,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(x, y),
         expected=[[[3, 2, 12]], [], [[], [4, 25]]])
 
+  @test_util.run_deprecated_v1
   def testOpWithInnerValuesOnly(self):
     x = constant_op.constant([[1, 2], [3, 4], [5, 6]])
     y = constant_op.constant(2)
@@ -191,6 +203,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
                             r'Inputs must have identical ragged splits.*',
                             ragged.map_inner_values, math_ops.add, x, y)
 
+  @test_util.run_deprecated_v1
   def testRaggedTensorSplitsMismatchErrorAtRuntime(self):
     splits1 = array_ops.placeholder_with_default(
         constant_op.constant([0, 3, 3, 5], dtypes.int64), None)
diff --git a/tensorflow/python/ops/ragged/ragged_operators_test.py b/tensorflow/python/ops/ragged/ragged_operators_test.py
index a99d788ef7..7fe8159d82 100644
--- a/tensorflow/python/ops/ragged/ragged_operators_test.py
+++ b/tensorflow/python/ops/ragged/ragged_operators_test.py
@@ -27,6 +27,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
   # @TODO(edloper): Test right-handed versions of operators once we add
   # broadcasting support for elementwise ops.
 
+  @test_util.run_deprecated_v1
   def testOrderingOperators(self):
     x = ragged.constant([[1, 5], [3]])
     y = ragged.constant([[4, 5], [1]])
@@ -40,6 +41,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
     if a != b:
       print('%30s %s' % (b, a))
 
+  @test_util.run_deprecated_v1
   def testArithmeticOperators(self):
     x = ragged.constant([[1.0, -2.0], [8.0]])
     y = ragged.constant([[4.0, 4.0], [2.0]])
@@ -75,6 +77,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual((2.0 % y).eval().tolist(), [[2.0, 2.0], [0.0]])
       self.assertEqual((x % 2.0).eval().tolist(), [[1.0, 0.0], [0.0]])
 
+  @test_util.run_deprecated_v1
   def testLogicalOperators(self):
     a = ragged.constant([[True, True], [False]])
     b = ragged.constant([[True, False], [False]])
diff --git a/tensorflow/python/ops/ragged/ragged_range_op_test.py b/tensorflow/python/ops/ragged/ragged_range_op_test.py
index 3c6a6fb75c..644423ecb7 100644
--- a/tensorflow/python/ops/ragged/ragged_range_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_range_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedRangeOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     """Examples from ragged_range.__doc__."""
     with self.test_session():
@@ -38,6 +39,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
       rt3 = ragged.range([0, 5, 8], [3, 3, 12], 2).eval().tolist()
       self.assertEqual(rt3, [[0, 2], [], [8, 10]])
 
+  @test_util.run_deprecated_v1
   def testBasicRanges(self):
     with self.test_session():
       # Specify limits only.
@@ -56,6 +58,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
           [list(range(0, 4, 2)), list(range(3, 4, 3)),
            list(range(5, 15, 4))])
 
+  @test_util.run_deprecated_v1
   def testFloatRanges(self):
     with self.test_session():
       expected = [[0.0, 0.4, 0.8, 1.2, 1.6, 2.0, 2.4, 2.8, 3.2, 3.6], [3.0],
@@ -64,6 +67,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
                             [0.4, 1.5, 2.2]).eval().tolist()
       self.assertEqual(expected, [[round(v, 5) for v in row] for row in actual])
 
+  @test_util.run_deprecated_v1
   def testNegativeDeltas(self):
     with self.test_session():
       self.assertEqual(
@@ -77,6 +81,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
           [list(range(0, 0, -1)), list(range(-3, 0, 1)),
            list(range(5, 0, -2))])
 
+  @test_util.run_deprecated_v1
   def testBroadcast(self):
     with self.test_session():
       # Specify starts and limits, broadcast deltas.
@@ -89,6 +94,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(
           ragged.range(0, 5, 1).eval().tolist(), [list(range(0, 5, 1))])
 
+  @test_util.run_deprecated_v1
   def testEmptyRanges(self):
     rt1 = ragged.range([0, 5, 3], [0, 3, 5])
     rt2 = ragged.range([0, 5, 5], [0, 3, 5], -1)
@@ -96,6 +102,7 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
       self.assertEqual(rt1.eval().tolist(), [[], [], [3, 4]])
       self.assertEqual(rt2.eval().tolist(), [[], [5, 4], []])
 
+  @test_util.run_deprecated_v1
   def testShapeFnErrors(self):
     with self.test_session():
       self.assertRaisesRegexp(ValueError, r'Shape must be at most rank 1.*',
@@ -107,12 +114,14 @@ class RaggedRangeOpTest(test_util.TensorFlowTestCase):
       self.assertRaisesRegexp(ValueError, r'Dimensions must be equal.*',
                               ragged.range, [0], [1, 2])
 
+  @test_util.run_deprecated_v1
   def testKernelErrors(self):
     with self.test_session():
       self.assertRaisesRegexp(errors.InvalidArgumentError,
                               r'Requires delta != 0',
                               ragged.range(0, 0, 0).eval)
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     self.assertEqual(ragged.range(0, 0, 0).shape.as_list(), [1, None])
     self.assertEqual(ragged.range([1, 2, 3]).shape.as_list(), [3, None])
diff --git a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
index 93176c738d..9f51d59ba3 100644
--- a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
@@ -300,6 +300,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=2,
           expected=[[mean(1, 2), mean(3, 4, 5)], [mean(6, 7), 8], [9]]),
   )
+  @test_util.run_deprecated_v1
   def testReduce(self, ragged_reduce_op, rt_input, axis, expected):
     rt_input = ragged.constant(rt_input)
     reduced = ragged_reduce_op(rt_input, axis)
@@ -311,6 +312,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertTrue(
         ((actual == expected) | (np.isnan(actual) & np.isnan(expected))).all())
 
+  @test_util.run_deprecated_v1
   def testMeanNan(self):
     rt_as_list = [[0, 1, 2, 3], [4], [], [5, 6], [7], [8, 9]]
     expected = (
@@ -321,6 +323,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.test_session():
       self.assertEqualWithNan(reduced.eval(), expected)
 
+  @test_util.run_deprecated_v1
   def testMeanWithTensorInputs(self):
     tensor = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]]
     expected = [2.0, 20.0]
@@ -328,6 +331,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.test_session():
       self.assertAllEqual(reduced.eval(), expected)
 
+  @test_util.run_deprecated_v1
   def testErrors(self):
     rt_input = ragged.constant([[1, 2, 3], [4, 5]])
     axis = array_ops.placeholder_with_default(constant_op.constant([0]), None)
diff --git a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
index 4d5a0a5d11..4a705be484 100644
--- a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
@@ -143,6 +143,7 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
           expected=[[2, 3, 0], [4, 1]],
           expected_ragged_rank=1),
   ])  # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRowLengths(self,
                      rt_input,
                      expected,
diff --git a/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py b/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py
index f246bf3552..7f5f4e91bd 100644
--- a/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     splits = [0, 3, 3, 5, 6, 9]
     expected = [0, 0, 0, 2, 2, 3, 4, 4, 4]
@@ -33,12 +34,14 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(segment_ids.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testEmptySplits(self):
     # Note: the splits for an empty ragged tensor contains a single zero.
     segment_ids = ragged.row_splits_to_segment_ids([0])
     with self.test_session():
       self.assertEqual(segment_ids.eval().tolist(), [])
 
+  @test_util.run_deprecated_v1
   def testErrors(self):
     self.assertRaisesRegexp(ValueError, r'Invalid row_splits: \[\]',
                             ragged.row_splits_to_segment_ids, [])
diff --git a/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py
index fa7adf66b0..7e52f2d844 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     segment_ids = [0, 0, 0, 2, 2, 3, 4, 4, 4]
     expected = [0, 3, 3, 5, 6, 9]
@@ -33,6 +34,7 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(splits.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testEmptySegmentIds(self):
     # Note: the splits for an empty ragged tensor contains a single zero.
     segment_ids = ragged.segment_ids_to_row_splits([])
@@ -49,6 +51,7 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
     self.assertRaisesRegexp(ValueError, r'Shape \(1, 1\) must have rank 1',
                             ragged.segment_ids_to_row_splits, [[0]])
 
+  @test_util.run_deprecated_v1
   def testNumSegments(self):
     segment_ids = [0, 0, 0, 2, 2, 3, 4, 4, 4]
     num_segments = 7
@@ -57,6 +60,7 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(splits.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testUnsortedSegmentIds(self):
     # Segment ids are not required to be sorted.
     segment_ids = [0, 4, 3, 2, 4, 4, 2, 0, 0]
diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
index 40a101b4da..9e4877ae3e 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
@@ -110,6 +110,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
       (ragged.segment_mean, mean, [5, 4, 3, 2, 1, 0]),
       (ragged.segment_mean, mean, [0, 0, 0, 10, 10, 10]),
   )
+  @test_util.run_deprecated_v1
   def testRaggedSegment_Int(self, segment_op, combiner, segment_ids):
     rt_as_list = [[0, 1, 2, 3], [4], [], [5, 6], [7], [8, 9]]
     rt = ragged.constant(rt_as_list)
@@ -146,6 +147,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
       (ragged.segment_sqrt_n, sqrt_n, [5, 4, 3, 2, 1, 0]),
       (ragged.segment_sqrt_n, sqrt_n, [0, 0, 0, 10, 10, 10]),
   )
+  @test_util.run_deprecated_v1
   def testRaggedSegment_Float(self, segment_op, combiner, segment_ids):
     rt_as_list = [[0., 1., 2., 3.], [4.], [], [5., 6.], [7.], [8., 9.]]
     rt = ragged.constant(rt_as_list)
@@ -157,6 +159,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
     self.assertNestedListAmostEqual(
         self.evaluate(segmented).tolist(), expected, places=5)
 
+  @test_util.run_deprecated_v1
   def testRaggedRankTwo(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121],],  # row 0
@@ -179,6 +182,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                  []]  # pyformat: disable
     self.assertEqual(self.evaluate(segmented2).tolist(), expected2)
 
+  @test_util.run_deprecated_v1
   def testRaggedSegmentIds(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121],],  # row 0
@@ -201,6 +205,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
         'but segment_ids is ragged and data is not.', ragged.segment_sum, dt,
         segment_ids, 3)
 
+  @test_util.run_deprecated_v1
   def testShapeMismatchError2(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121]],  # row 0
diff --git a/tensorflow/python/ops/ragged/ragged_stack_op_test.py b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
index d474a749f0..4343471694 100644
--- a/tensorflow/python/ops/ragged/ragged_stack_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
@@ -265,6 +265,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=[[[b'a00', b'a01'], [], [b'a20', b'a21']]]),
   )   # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedStack(self,
                       descr,
                       rt_inputs,
@@ -313,6 +314,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testError(self, rt_inputs, axis, error, message):
     self.assertRaisesRegexp(error, message, ragged.stack, rt_inputs, axis)
 
+  @test_util.run_deprecated_v1
   def testSingleTensorInput(self):
     """Tests ragged_stack with a single tensor input.
 
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index fa681c07bb..608fbd6e5b 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -114,6 +114,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   # RaggedTensor class docstring examples
   #=============================================================================
 
+  @test_util.run_deprecated_v1
   def testClassDocStringExamples(self):
     # From section: "Component Tensors"
     rt = ragged.from_row_splits(
@@ -199,6 +200,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   # RaggedTensor Constructor (private)
   #=============================================================================
 
+  @test_util.run_deprecated_v1
   def testRaggedTensorConstruction(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
@@ -243,6 +245,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 # RaggedTensor Factory Ops
 #=============================================================================
 
+  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithDerivedNRows(self):
     # nrows is known at graph creation time.
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -265,6 +268,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
+  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithDerivedNRowsDynamic(self):
     # nrows is not known at graph creation time.
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -288,6 +292,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
+  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithExplicitNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
@@ -309,6 +314,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g'], [], []])
 
+  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithExplicitNRowsEqualToDefault(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
@@ -332,6 +338,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
+  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithEmptyValues(self):
     rt = ragged.from_value_rowids([], [])
     rt_nrows = ragged.nrows(rt)
@@ -343,6 +350,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertEqual(self.evaluate(rt_nrows).tolist(), 0)
     self.assertEqual(self.evaluate(rt).tolist(), [])
 
+  @test_util.run_deprecated_v1
   def testFromRowSplits(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
@@ -368,6 +376,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, err_msg):
       ragged.from_row_splits([], [])
 
+  @test_util.run_deprecated_v1
   def testFromRowStarts(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_starts = constant_op.constant([0, 2, 2, 5, 6], dtypes.int64)
@@ -388,6 +397,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
+  @test_util.run_deprecated_v1
   def testFromRowLimits(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_limits = constant_op.constant([2, 2, 5, 6, 7], dtypes.int64)
@@ -408,6 +418,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
+  @test_util.run_deprecated_v1
   def testFromRowLengths(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_lengths = constant_op.constant([2, 0, 3, 1, 1], dtypes.int64)
@@ -429,6 +440,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
+  @test_util.run_deprecated_v1
   def testFromNestedValueRowIdsWithDerivedNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_value_rowids = [
@@ -453,6 +465,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         self.evaluate(rt).tolist(),
         [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
 
+  @test_util.run_deprecated_v1
   def testFromNestedValueRowIdsWithExplicitNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_value_rowids = [
@@ -509,6 +522,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       ragged.from_nested_value_rowids([1, 2, 3], [[0, 1, 2], [0, 1, 2]],
                                       constant_op.constant([3, 3]))
 
+  @test_util.run_deprecated_v1
   def testFromNestedRowSplits(self):
     inner_values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_row_splits = [
@@ -576,6 +590,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           value_rowids=value_rowids,
           nrows=array_ops.expand_dims(nrows, 0))
 
+  @test_util.run_deprecated_v1
   def testGraphMismatch(self):
     with ops.Graph().as_default():
       values = constant_op.constant([1, 2, 3])
@@ -588,6 +603,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   # Ragged Value & Row-Partitioning Tensor Accessors
   #=============================================================================
 
+  @test_util.run_deprecated_v1
   def testRaggedTensorAccessors_2d(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
@@ -622,6 +638,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           [self.evaluate(s).tolist() for s in rt.nested_row_splits],
           [[0, 2, 2, 5, 6, 7]])
 
+  @test_util.run_deprecated_v1
   def testRaggedTensorAccessors_3d_with_ragged_rank_1(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
@@ -657,6 +674,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           [self.evaluate(s).tolist() for s in rt.nested_row_splits],
           [[0, 2, 2, 5, 6, 7]])
 
+  @test_util.run_deprecated_v1
   def testRaggedTensorAccessors_3d_with_ragged_rank_2(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_row_splits = [
@@ -709,6 +727,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   # RaggedTensor.shape
   #=============================================================================
 
+  @test_util.run_deprecated_v1
   def testShape(self):
     """Tests for RaggedTensor.shape."""
     rt1 = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
@@ -842,6 +861,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[:, -2:], [row[-2:] for row in EXAMPLE_RAGGED_TENSOR_2D]),
       # TODO(edloper): Add tests for strided slices, once support is added.
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemWithRaggedRank1(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     # Ragged tensor
@@ -883,6 +903,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[..., 0, 0, 0], IndexError,
        'Too many indices for RaggedTensor'),
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithRaggedRank1(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
@@ -961,6 +982,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       # TODO(edloper): Add tests slicing inner ragged dimensions, one support
       # is added.
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemWithRaggedRank2(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     rt = ragged.from_nested_row_splits(
@@ -982,6 +1004,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[5], ValueError, '.*out of bounds.*'),
       (SLICE_BUILDER[0, 5], ValueError, '.*out of bounds.*'),
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithRaggedRank2(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
@@ -996,6 +1019,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[2:], []),
       (SLICE_BUILDER[:-3], []),
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemWithEmptyTensor(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     rt = ragged.from_row_splits([], [0])
@@ -1005,6 +1029,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[0], ValueError, '.*out of bounds.*'),
       (SLICE_BUILDER[-1], ValueError, '.*out of bounds.*'),
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithEmptyTensor(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
@@ -1020,6 +1045,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[0, 1], EXAMPLE_RAGGED_TENSOR_2D[0][1]),
       (SLICE_BUILDER[-3, 0], EXAMPLE_RAGGED_TENSOR_2D[-3][0]),
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemWithPlaceholderShapes(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     # Intentionally use an unknown shape for `splits`, to force the code path
@@ -1034,6 +1060,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   @parameterized.parameters(
       (SLICE_BUILDER[..., 2], ValueError,
        'Ellipsis not supported for unknown shape RaggedTensors'),)
+  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithPlaceholderShapes(
       self, slice_spec, expected, message):
     """Test that rt.__getitem__(slice_spec) == expected."""
@@ -1042,6 +1069,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.from_row_splits(values, [0, 1])
     self._TestGetItemException(rt, slice_spec, expected, message)
 
+  @test_util.run_deprecated_v1
   def testGetItemNewAxis(self):
     # rt: [[[['a', 'b'], ['c', 'd']], [], [['e', 'f']]], []]
     splits1 = [0, 3, 3]
@@ -1089,6 +1117,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   #=============================================================================
   # RaggedTensor.__str__
   #=============================================================================
+  @test_util.run_deprecated_v1
   def testRaggedTensorStr(self):
     rt1 = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
     expected1 = ('RaggedTensor(values=Tensor("RaggedFromRowSplits/values:0", '
@@ -1126,6 +1155,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   # RaggedTensor.with_values() and RaggedTensor.with_inner_values().
   #=============================================================================
 
+  @test_util.run_deprecated_v1
   def testWithValues(self):
     rt1 = ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])
     rt2 = ragged.constant([[[1, 2], [3, 4, 5]], [[6]], [], [[], [7]]])
@@ -1147,6 +1177,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   #=============================================================================
   # Session.run
   #=============================================================================
+  @test_util.run_deprecated_v1
   def testSessionRun(self):
     rt1 = ragged.constant([[1, 2, 3], [4]])
     rt2 = ragged.constant([[[], [1, 2]], [[3]]])
@@ -1156,6 +1187,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]])
       self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]])
 
+  @test_util.run_deprecated_v1
   def testSessionRunFeed(self):
     rt1 = ragged.from_row_splits(
         array_ops.placeholder(dtypes.int32),
@@ -1176,6 +1208,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]])
       self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]])
 
+  @test_util.run_deprecated_v1
   def testSessionPartialRunFeed(self):
     # Placeholder inputs.
     a = ragged.from_row_splits(
diff --git a/tensorflow/python/ops/ragged/ragged_tile_op_test.py b/tensorflow/python/ops/ragged/ragged_tile_op_test.py
index 672d212114..f335b15dd1 100644
--- a/tensorflow/python/ops/ragged/ragged_tile_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tile_op_test.py
@@ -181,6 +181,7 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                     [[[5], [6]]]]),
 
   ])  # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedTile(self,
                      descr,
                      rt_input,
@@ -209,6 +210,7 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       with self.test_session():
         self.assertEqual(tiled.eval().tolist(), expected)
 
+  @test_util.run_deprecated_v1
   def testRaggedTileWithTensorInput(self):
     # When the input is a `Tensor`, ragged_tile just delegates to tf.tile.
     dt = constant_op.constant([[1, 2], [3, 4]])
diff --git a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
index cb8e82c00d..69b31ad0e9 100644
--- a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     rt = ragged.constant([[1, 2, 3], [4], [], [5, 6]])
     st = ragged.to_sparse(rt)
@@ -40,6 +41,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(' '.join(repr(st.eval()).split()), expected)
 
+  @test_util.run_deprecated_v1
   def test2DRaggedTensorWithOneRaggedDimension(self):
     rt = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     with self.test_session():
@@ -49,6 +51,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(st.values, b'a b c d e f g'.split())
       self.assertAllEqual(st.dense_shape, [5, 3])
 
+  @test_util.run_deprecated_v1
   def test3DRaggedTensorWithOneRaggedDimension(self):
     rt = ragged.constant([[[1, 2], [3, 4]], [[5, 6], [7, 8], [9, 10]],
                           [[11, 12]], [], [[13, 14]]],
@@ -63,6 +66,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
                           [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
       self.assertAllEqual(st.dense_shape, [5, 3, 2])
 
+  @test_util.run_deprecated_v1
   def test4DRaggedTensorWithOneRaggedDimension(self):
     rt = ragged.constant(
         [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]], [], [[[9, 10], [11, 12]]]],
@@ -88,6 +92,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
           ])
       self.assertAllEqual(st.dense_shape, [3, 2, 2, 2])
 
+  @test_util.run_deprecated_v1
   def test4DRaggedTensorWithTwoRaggedDimensions(self):
     rt = ragged.constant([[[[1, 2], [3, 4]], [[5, 6], [7, 8], [9, 10]]],
                           [[[11, 12]], [], [[13, 14]]], []],
@@ -135,6 +140,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     self.assertEqual(st.values.shape.as_list(), [7])
     self.assertEqual(st.dense_shape.shape.as_list(), [3])
 
+  @test_util.run_deprecated_v1
   def testKernelErrors(self):
     # An empty vector, defined using a placeholder to ensure that we can't
     # determine that it's invalid at graph-construction time.
@@ -173,6 +179,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
       self.assertRaisesRegexp(errors.InvalidArgumentError, empty_splits_error,
                               ragged.to_sparse(bad_rt5).eval)
 
+  @test_util.run_deprecated_v1
   def testGradient(self):
     # rt1.shape == rt2.shape == [2, (D2), (D3), 2].
     rt1 = ragged.constant([[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0]]]],
diff --git a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
index 688676e46c..77499b9cb3 100644
--- a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.platform import googletest
 class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
                                  parameterized.TestCase):
 
+  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     """Example from ragged_to_tensor.__doc__."""
     rt = ragged.constant([[9, 8, 7], [], [6, 5], [4]])
@@ -97,6 +98,7 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
           'expected': [[[[1], [2]], [[9], [9]], [[3], [9]]]],
       },
   )
+  @test_util.run_deprecated_v1
   def testRaggedTensorToTensor(self,
                                rt_input,
                                expected,
@@ -132,6 +134,7 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
           'error': (TypeError, "Expected int32, got 'a' of type 'str' instead"),
       },
   )
+  @test_util.run_deprecated_v1
   def testError(self, rt_input, default, error, ragged_rank=None):
     rt = ragged.constant(rt_input, ragged_rank=ragged_rank)
     with self.assertRaisesRegexp(error[0], error[1]):
diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py
index 03672e4521..de83a54977 100644
--- a/tensorflow/python/ops/ragged/ragged_where_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py
@@ -165,6 +165,7 @@ class RaggedWhereOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           y=ragged.constant_value([[[['a']]], [[['b']]]]),
           expected=ragged.constant_value([[[[], [b'A']]], [[[b'b']]]])),
   ])   # pyformat: disable
+  @test_util.run_deprecated_v1
   def testRaggedWhere(self, condition, expected, x=None, y=None):
     result = ragged.where(condition, x, y)
     self.assertEqual(
diff --git a/tensorflow/python/ops/sort_ops_test.py b/tensorflow/python/ops/sort_ops_test.py
index 8a92f49266..17ce604cbf 100644
--- a/tensorflow/python/ops/sort_ops_test.py
+++ b/tensorflow/python/ops/sort_ops_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import sort_ops
@@ -33,9 +34,11 @@ from tensorflow.python.platform import test
 
 class SortTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testRandom_lowDimensionality(self):
     self._testRandom_lowDimensionality(negative_axis=False)
 
+  @test_util.run_deprecated_v1
   def testRandom_lowDimensionality_negative(self):
     self._testRandom_lowDimensionality(negative_axis=True)
 
@@ -53,6 +56,7 @@ class SortTest(test.TestCase):
             np.sort(arr, axis=sort_axis),
             sort_ops.sort(constant_op.constant(arr), axis=sort_axis).eval())
 
+  @test_util.run_deprecated_v1
   def testRandom_highDimensionality(self):
     np.random.seed(100)
     for _ in range(20):
@@ -65,6 +69,7 @@ class SortTest(test.TestCase):
             np.sort(arr, axis=sort_axis),
             sort_ops.sort(constant_op.constant(arr), axis=sort_axis).eval())
 
+  @test_util.run_deprecated_v1
   def testScalar(self):
     # Create an empty scalar where the static shape is unknown.
     zeros_length_1 = array_ops.zeros(
@@ -77,11 +82,13 @@ class SortTest(test.TestCase):
       with self.assertRaises(errors.InvalidArgumentError):
         sort.eval()
 
+  @test_util.run_deprecated_v1
   def testNegativeOutOfBounds_staticShape(self):
     arr = constant_op.constant([3, 4, 5])
     with self.assertRaises(ValueError):
       sort_ops.sort(arr, axis=-4)
 
+  @test_util.run_deprecated_v1
   def testDescending(self):
     arr = np.random.random((10, 5, 5))
     with self.cached_session():
@@ -90,6 +97,7 @@ class SortTest(test.TestCase):
           sort_ops.sort(
               constant_op.constant(arr), axis=0, direction='DESCENDING').eval())
 
+  @test_util.run_deprecated_v1
   def testSort_staticallyKnownRank_constantTransposition(self):
     # The transposition array should be a constant if the rank of "values" is
     # statically known.
@@ -107,6 +115,7 @@ class SortTest(test.TestCase):
         tensor_util.constant_value(transposition),
         [0, 4, 2, 3, 1])
 
+  @test_util.run_deprecated_v1
   def testArgsort_1d(self):
     arr = np.random.random(42)
     with self.cached_session():
@@ -114,6 +123,7 @@ class SortTest(test.TestCase):
           np.sort(arr),
           array_ops.gather(arr, sort_ops.argsort(arr)).eval())
 
+  @test_util.run_deprecated_v1
   def testArgsort(self):
     arr = np.random.random((5, 6, 7, 8))
     for axis in range(4):
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index 7438cdb3f1..94aaebed95 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -46,6 +46,7 @@ class LBetaTest(test.TestCase):
           0.5, self.evaluate(math_ops.exp(special_math_ops.lbeta(x_one_half))))
       self.assertEqual([], special_math_ops.lbeta(x_one).get_shape())
 
+  @test_util.run_deprecated_v1
   def test_one_dimensional_arg_dynamic(self):
     # Should evaluate to 1 and 1/2.
     x_one = [1, 1.]
@@ -57,6 +58,7 @@ class LBetaTest(test.TestCase):
       self.assertAllClose(0.5,
                           beta_ph.eval(feed_dict={ph: x_one_half}))
 
+  @test_util.run_deprecated_v1
   def test_four_dimensional_arg_with_partial_shape_dynamic(self):
     x_ = np.ones((3, 2, 3, 4))
     # Gamma(1) = 0! = 1
@@ -81,6 +83,7 @@ class LBetaTest(test.TestCase):
           self.evaluate(math_ops.exp(special_math_ops.lbeta(x_one_half))))
       self.assertEqual((2,), special_math_ops.lbeta(x_one_half).get_shape())
 
+  @test_util.run_deprecated_v1
   def test_two_dimensional_arg_dynamic(self):
     # Should evaluate to 1/2.
     x_one_half = [[2, 1.], [2, 1.]]
@@ -288,6 +291,7 @@ class EinsumTest(test.TestCase):
     for case in self.long_cases:
       self.run_test(case)
 
+  @test_util.run_deprecated_v1
   def test_invalid(self):
     for axes in self.invalid_cases:
       inputs = [
@@ -297,6 +301,7 @@ class EinsumTest(test.TestCase):
       with self.assertRaises(ValueError):
         _ = special_math_ops.einsum(axes, *inputs)
 
+  @test_util.run_deprecated_v1
   def test_invalid_keyword_arguments(self):
     m0 = array_ops.placeholder(dtypes.int32, shape=(1, None))
     m1 = array_ops.placeholder(dtypes.int32, shape=(None, 1))
@@ -311,11 +316,13 @@ class EinsumTest(test.TestCase):
           invalid1='value1',
           invalid2='value2')
 
+  @test_util.run_deprecated_v1
   def test_repeated_axis_single_input(self):
     x = array_ops.placeholder(dtypes.float32, shape=[2, 2])
     with self.assertRaises(ValueError):
       _ = special_math_ops.einsum('ii->', x)
 
+  @test_util.run_deprecated_v1
   def test_dim_mismatch(self):
     for axes, input_shapes in self.dim_mismatch_cases:
       inputs = [
diff --git a/tensorflow/python/profiler/internal/run_metadata_test.py b/tensorflow/python/profiler/internal/run_metadata_test.py
index 216cc3dd54..a8859f845b 100644
--- a/tensorflow/python/profiler/internal/run_metadata_test.py
+++ b/tensorflow/python/profiler/internal/run_metadata_test.py
@@ -26,6 +26,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -154,6 +155,7 @@ class RunMetadataTest(test.TestCase):
     # deallocates the memory after matmul started.
     self.assertGreater(random_allocs[1].alloc_micros, mm.all_start_micros)
 
+  @test_util.run_deprecated_v1
   def testCPU(self):
     ops.reset_default_graph()
     with ops.device('/cpu:0'):
@@ -167,6 +169,7 @@ class RunMetadataTest(test.TestCase):
     ret = _extract_node(run_meta, 'MatMul:MatMul')
     self.assertEqual(len(ret), 0)
 
+  @test_util.run_deprecated_v1
   def testLoopCPU(self):
     ops.reset_default_graph()
     with ops.device('/cpu:0'):
diff --git a/tensorflow/python/profiler/pprof_profiler_test.py b/tensorflow/python/profiler/pprof_profiler_test.py
index 11a3487360..120a0d0eaa 100644
--- a/tensorflow/python/profiler/pprof_profiler_test.py
+++ b/tensorflow/python/profiler/pprof_profiler_test.py
@@ -24,6 +24,7 @@ from proto import profile_pb2
 from tensorflow.core.framework import step_stats_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -135,6 +136,7 @@ comment: 9
       profile.ParseFromString(profile_contents)
       self.assertEquals(expected_proto, str(profile))
 
+  @test_util.run_deprecated_v1
   def testProfileWithWhileLoop(self):
     options = config_pb2.RunOptions()
     options.trace_level = config_pb2.RunOptions.FULL_TRACE
diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py
index 680cd71d1f..885f08ca4b 100644
--- a/tensorflow/python/profiler/profile_context_test.py
+++ b/tensorflow/python/profiler/profile_context_test.py
@@ -21,6 +21,7 @@ import os
 
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -35,6 +36,7 @@ builder = option_builder.ProfileOptionBuilder
 
 class ProfilerContextTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasics(self):
     ops.reset_default_graph()
     outfile = os.path.join(test.get_temp_dir(), "dump")
@@ -69,6 +71,7 @@ class ProfilerContextTest(test.TestCase):
       with gfile.Open(outfile, "r") as f:
         self.assertEqual(profile_str, f.read())
 
+  @test_util.run_deprecated_v1
   def testAutoTracingInDeubMode(self):
     ops.reset_default_graph()
     x = lib.BuildFullModel()
@@ -90,6 +93,7 @@ class ProfilerContextTest(test.TestCase):
         for f in gfile.ListDirectory(test.get_temp_dir()):
           self.assertFalse("run_meta" in f)
 
+  @test_util.run_deprecated_v1
   def testDisabled(self):
     ops.reset_default_graph()
     x = lib.BuildFullModel()
diff --git a/tensorflow/python/profiler/profiler_test.py b/tensorflow/python/profiler/profiler_test.py
index eacb7d21e6..e4f7361e5d 100644
--- a/tensorflow/python/profiler/profiler_test.py
+++ b/tensorflow/python/profiler/profiler_test.py
@@ -21,6 +21,7 @@ import os
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -35,6 +36,7 @@ builder = option_builder.ProfileOptionBuilder
 
 class ProfilerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testProfileBasic(self):
     ops.reset_default_graph()
     outfile = os.path.join(test.get_temp_dir(), 'dump')
@@ -171,6 +173,7 @@ class ProfilerTest(test.TestCase):
       checker = advice_pb.checkers['ExpensiveOperationChecker']
       self.assertGreater(len(checker.reports), 0)
 
+  @test_util.run_deprecated_v1
   def testMultipleProfilePerStep(self):
     ops.reset_default_graph()
     opts = (builder(builder.trainable_variables_parameter())
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 3378bf08cd..3b7f0b250e 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -26,6 +26,7 @@ from absl.testing import parameterized
 from tensorflow.python.client import session
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
@@ -93,6 +94,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
     super(SavedModelLoaderTest, self).tearDown()
     shutil.rmtree(test.get_temp_dir(), ignore_errors=True)
 
+  @test_util.run_deprecated_v1
   def test_load_function(self, builder_cls):
     self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
@@ -108,6 +110,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("y:0").eval())
 
+  @test_util.run_deprecated_v1
   def test_load_graph(self, builder_cls):
     self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
@@ -127,6 +130,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(errors.FailedPreconditionError):
         self.evaluate(y)
 
+  @test_util.run_deprecated_v1
   def test_load_with_import_scope(self, builder_cls):
     self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
@@ -157,6 +161,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("baa/x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("baa/y:0").eval())
 
+  @test_util.run_deprecated_v1
   def test_restore_variables(self, builder_cls):
     self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
@@ -174,6 +179,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       loader.restore_variables(sess, tf_saver.Saver())
       self.assertEqual(55, self.evaluate(z))
 
+  @test_util.run_deprecated_v1
   def test_run_init_op(self, builder_cls):
     self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
@@ -206,6 +212,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
     with self.assertRaises(RuntimeError):
       loader.get_meta_graph_def_from_tags(["not_a_graph"])
 
+  @test_util.run_deprecated_v1
   def test_load_saved_model_with_no_variables(self, builder_cls):
     """Test that SavedModel runs saver when there appear to be no variables.
 
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 8fb28039a4..ef0a92fddd 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -282,6 +282,7 @@ class SaveTest(test.TestCase):
         {"out": model(array_ops.ones([1, 4]))},
         self._import_and_infer(save_dir, {"x": [[1., 1., 1., 1.]]}))
 
+  @test_util.run_deprecated_v1
   def test_export_functional_keras_model_after_fit(self):
     x = input_layer.Input((1,))
     y = core.Dense(1, name="y")(x)
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index efcd21cce3..0f18fb1a01 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -199,6 +199,7 @@ class SavedModelTest(SavedModelTestBase):
                                    constants.SAVED_MODEL_FILENAME_PBTXT):
         loader.load(sess, ["foo"], export_dir)
 
+  @test_util.run_deprecated_v1
   def testVerifySessionGraphUsage(self):
     export_dir = self._get_export_dir("test_verify_session_graph_usage")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -219,6 +220,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertEqual(
           42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval())
 
+  @test_util.run_deprecated_v1
   def testSequence(self):
     export_dir = self._get_export_dir("test_sequence")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -236,6 +238,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertRaises(AssertionError, builder.add_meta_graph_and_variables,
                         sess, ["baz"])
 
+  @test_util.run_deprecated_v1
   def testTags(self):
     export_dir = self._get_export_dir("test_tags")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -325,6 +328,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertRaises(RuntimeError, loader.load, sess, ["foo", "baz"],
                         export_dir)
 
+  @test_util.run_deprecated_v1
   def testVariables(self):
     export_dir = self._get_export_dir("test_variables")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -377,6 +381,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertRaises(errors.NotFoundError, loader.load, sess, ["baz"],
                         export_dir)
 
+  @test_util.run_deprecated_v1
   def testGraphWithoutVariables(self):
     export_dir = self._get_export_dir("test_graph_has_variables")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -412,6 +417,7 @@ class SavedModelTest(SavedModelTestBase):
       c = a * b
       self.assertEqual(30.0, self.evaluate(c))
 
+  @test_util.run_deprecated_v1
   def testNoOverwrite(self):
     export_dir = self._get_export_dir("test_no_overwrite")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -436,6 +442,7 @@ class SavedModelTest(SavedModelTestBase):
     self.assertRaises(AssertionError, saved_model_builder._SavedModelBuilder,
                       export_dir)
 
+  @test_util.run_deprecated_v1
   def testSaveAsText(self):
     export_dir = self._get_export_dir("test_astext")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -467,6 +474,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertEqual(
           42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval())
 
+  @test_util.run_deprecated_v1
   def testCollections(self):
     export_dir = self._get_export_dir("test_collections")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -517,6 +525,7 @@ class SavedModelTest(SavedModelTestBase):
 
       self.assertEqual(len(ops.get_collection("foo_vars")), 0)
 
+  @test_util.run_deprecated_v1
   def testSignatureDefs(self):
     export_dir = self._get_export_dir("test_signature_defs")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -604,6 +613,7 @@ class SavedModelTest(SavedModelTestBase):
     self._validate_sig_def_keys(builder, valid_tensor_info,
                                 constants.TRAIN_OP_SIGNATURE_KEY)
 
+  @test_util.run_deprecated_v1
   def testSignatureDefValidationSucceedsWithName(self):
     tensor_with_name = meta_graph_pb2.TensorInfo()
     tensor_with_name.name = "foo"
@@ -617,6 +627,7 @@ class SavedModelTest(SavedModelTestBase):
     builder = saved_model_builder._SavedModelBuilder(export_dir)
     self._validate_outputs_tensor_info_accept(builder, tensor_with_name)
 
+  @test_util.run_deprecated_v1
   def testSignatureDefValidationSucceedsWithCoo(self):
     tensor_with_coo = meta_graph_pb2.TensorInfo()
     # TODO(soergel) test validation of each of the fields of coo_sparse
@@ -631,6 +642,7 @@ class SavedModelTest(SavedModelTestBase):
     builder = saved_model_builder._SavedModelBuilder(export_dir)
     self._validate_outputs_tensor_info_accept(builder, tensor_with_coo)
 
+  @test_util.run_deprecated_v1
   def testAssets(self):
     export_dir = self._get_export_dir("test_assets")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -662,6 +674,7 @@ class SavedModelTest(SavedModelTestBase):
           compat.as_bytes("ignored.txt"))
       self.assertFalse(file_io.file_exists(ignored_asset_path))
 
+  @test_util.run_deprecated_v1
   def testAssetsNameCollisionDiffFile(self):
     export_dir = self._get_export_dir("test_assets_name_collision_diff_file")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -693,6 +706,7 @@ class SavedModelTest(SavedModelTestBase):
           "asset_file_tensor_1:0",
           asset_id=1)
 
+  @test_util.run_deprecated_v1
   def testAssetsNameCollisionSameFilepath(self):
     export_dir = self._get_export_dir("test_assets_name_collision_same_path")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -730,6 +744,7 @@ class SavedModelTest(SavedModelTestBase):
           compat.as_bytes("hello42.txt_1"))
       self.assertFalse(file_io.file_exists(ignored_asset_path))
 
+  @test_util.run_deprecated_v1
   def testAssetsNameCollisionSameFile(self):
     export_dir = self._get_export_dir("test_assets_name_collision_same_file")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -767,6 +782,7 @@ class SavedModelTest(SavedModelTestBase):
           compat.as_bytes("hello42.txt_1"))
       self.assertFalse(file_io.file_exists(ignored_asset_path))
 
+  @test_util.run_deprecated_v1
   def testAssetsNameCollisionManyFiles(self):
     export_dir = self._get_export_dir("test_assets_name_collision_many_files")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -838,6 +854,7 @@ class SavedModelTest(SavedModelTestBase):
       # the main_op, following a restore.
       self.assertEqual(3, ops.get_collection("v")[2].eval())
 
+  @test_util.run_deprecated_v1
   def testTrainOp(self):
     export_dir = self._get_export_dir("test_train_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -865,6 +882,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertIsInstance(
           loader_impl.get_train_op(meta_graph_def), ops.Tensor)
 
+  @test_util.run_deprecated_v1
   def testTrainOpGroup(self):
     export_dir = self._get_export_dir("test_train_op_group")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -892,6 +910,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertIsInstance(
           loader_impl.get_train_op(meta_graph_def), ops.Operation)
 
+  @test_util.run_deprecated_v1
   def testTrainOpAfterVariables(self):
     export_dir = self._get_export_dir("test_train_op_after_variables")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -922,6 +941,7 @@ class SavedModelTest(SavedModelTestBase):
       loader.load(sess, ["pre_foo"], export_dir)
       self.assertFalse(ops.get_collection(constants.TRAIN_OP_KEY))
 
+  @test_util.run_deprecated_v1
   def testMultipleAssets(self):
     export_dir = self._get_export_dir("test_multiple_assets")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -962,6 +982,7 @@ class SavedModelTest(SavedModelTestBase):
       self._validate_assets(export_dir, bar_graph.asset_file_def, "bar.txt",
                             "content_bar", "asset_file_tensor:0")
 
+  @test_util.run_deprecated_v1
   def testDuplicateAssets(self):
     export_dir = self._get_export_dir("test_duplicate_assets")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1008,6 +1029,7 @@ class SavedModelTest(SavedModelTestBase):
       self._validate_assets(export_dir, bar_graph.asset_file_def, "foo.txt",
                             "content_foo", "asset_file_tensor:0")
 
+  @test_util.run_deprecated_v1
   def testOp(self):
     export_dir = self._get_export_dir("test_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1080,6 +1102,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertEqual(b"k1", v1.keys().eval())
       self.assertEqual(3.0, v1.values().eval())
 
+  @test_util.run_deprecated_v1
   def testCustomSaver(self):
     export_dir = self._get_export_dir("test_custom_saver")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1102,6 +1125,7 @@ class SavedModelTest(SavedModelTestBase):
         self.assertEqual(
             saved_graph.saver_def.restore_op_name, "my_saver/restore_all")
 
+  @test_util.run_deprecated_v1
   def testNoCustomSaver(self):
     export_dir = self._get_export_dir("test_no_custom_saver")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1124,6 +1148,7 @@ class SavedModelTest(SavedModelTestBase):
         self.assertEqual(
             saved_graph.saver_def.restore_op_name, "save/restore_all")
 
+  @test_util.run_deprecated_v1
   def testMultipleCustomSavers(self):
     export_dir = self._get_export_dir("test_multiple_custom_savers")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1154,6 +1179,7 @@ class SavedModelTest(SavedModelTestBase):
     _validate_custom_saver("tag_1", "save_1/restore_all")
     _validate_custom_saver("tag_2", "save_2/restore_all")
 
+  @test_util.run_deprecated_v1
   def testImportScope(self):
     export_dir = self._get_export_dir("test_scoped_assets")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1205,6 +1231,7 @@ class SavedModelTest(SavedModelTestBase):
           ops.get_default_graph().get_tensor_by_name(
               "scope_name/constant_tensor_name:0").eval())
 
+  @test_util.run_deprecated_v1
   def testClearDevices(self):
     export_dir = self._get_export_dir("test_clear_devices")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1323,6 +1350,7 @@ class SavedModelV1Test(SavedModelTestBase):
     self.assertEqual(expected_asset_file_name, asset.filename)
     self.assertEqual(expected_asset_tensor_name, asset.tensor_info.name)
 
+  @test_util.run_deprecated_v1
   def testWritingAssetsToCollection(self):
     export_dir = self._get_export_dir("test_writing_assets_to_collection")
     builder = saved_model_builder.SavedModelBuilder(export_dir)
@@ -1355,12 +1383,14 @@ class SavedModelV1Test(SavedModelTestBase):
           compat.as_bytes("ignored.txt"))
       self.assertFalse(file_io.file_exists(ignored_asset_path))
 
+  @test_util.run_deprecated_v1
   def testLegacyInitOpWithNonEmptyCollection(self):
     export_dir = self._get_export_dir(
         "test_legacy_init_op_with_non_empty_collection")
     self._testInitOpsWithNonEmptyCollection(export_dir,
                                             constants.LEGACY_INIT_OP_KEY)
 
+  @test_util.run_deprecated_v1
   def testMainOpWithNonEmptyCollection(self):
     export_dir = self._get_export_dir("test_main_op_with_non_empty_collection")
     self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY)
@@ -1464,6 +1494,7 @@ class SavedModelV1Test(SavedModelTestBase):
     self.assertIn("T", node_def.attr)
     self.assertIn("Tout", node_def.attr)
 
+  @test_util.run_deprecated_v1
   def testLegacyInitOp(self):
     export_dir = self._get_export_dir("test_legacy_init_op")
     builder = saved_model_builder.SavedModelBuilder(export_dir)
diff --git a/tensorflow/python/saved_model/signature_def_utils_test.py b/tensorflow/python/saved_model/signature_def_utils_test.py
index cc0fd8ce05..53c452359f 100644
--- a/tensorflow/python/saved_model/signature_def_utils_test.py
+++ b/tensorflow/python/saved_model/signature_def_utils_test.py
@@ -22,6 +22,7 @@ from tensorflow.core.framework import types_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -59,6 +60,7 @@ def _make_signature(inputs, outputs, name=None):
 
 class SignatureDefUtilsTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBuildSignatureDef(self):
     x = array_ops.placeholder(dtypes.float32, 1, name="x")
     x_tensor_info = utils.build_tensor_info(x)
@@ -89,6 +91,7 @@ class SignatureDefUtilsTest(test.TestCase):
     self.assertEqual(types_pb2.DT_FLOAT, y_tensor_info_actual.dtype)
     self.assertEqual(0, len(y_tensor_info_actual.tensor_shape.dim))
 
+  @test_util.run_deprecated_v1
   def testRegressionSignatureDef(self):
     input1 = constant_op.constant("a", name="input-1")
     output1 = constant_op.constant(2.2, name="output-1")
@@ -114,6 +117,7 @@ class SignatureDefUtilsTest(test.TestCase):
     self.assertEqual(types_pb2.DT_FLOAT, y_tensor_info_actual.dtype)
     self.assertEqual(0, len(y_tensor_info_actual.tensor_shape.dim))
 
+  @test_util.run_deprecated_v1
   def testClassificationSignatureDef(self):
     input1 = constant_op.constant("a", name="input-1")
     output1 = constant_op.constant("b", name="output-1")
@@ -145,6 +149,7 @@ class SignatureDefUtilsTest(test.TestCase):
     self.assertEqual(types_pb2.DT_FLOAT, scores_tensor_info_actual.dtype)
     self.assertEqual(0, len(scores_tensor_info_actual.tensor_shape.dim))
 
+  @test_util.run_deprecated_v1
   def testPredictionSignatureDef(self):
     input1 = constant_op.constant("a", name="input-1")
     input2 = constant_op.constant("b", name="input-2")
@@ -181,11 +186,13 @@ class SignatureDefUtilsTest(test.TestCase):
     self.assertEqual(types_pb2.DT_STRING, output2_tensor_info_actual.dtype)
     self.assertEqual(0, len(output2_tensor_info_actual.tensor_shape.dim))
 
+  @test_util.run_deprecated_v1
   def testTrainSignatureDef(self):
     self._testSupervisedSignatureDef(
         signature_def_utils_impl.supervised_train_signature_def,
         signature_constants.SUPERVISED_TRAIN_METHOD_NAME)
 
+  @test_util.run_deprecated_v1
   def testEvalSignatureDef(self):
     self._testSupervisedSignatureDef(
         signature_def_utils_impl.supervised_eval_signature_def,
@@ -239,11 +246,13 @@ class SignatureDefUtilsTest(test.TestCase):
     self.assertEqual(
         types_pb2.DT_FLOAT, signature_def.outputs["metrics/value"].dtype)
 
+  @test_util.run_deprecated_v1
   def testTrainSignatureDefMissingInputs(self):
     self._testSupervisedSignatureDefMissingInputs(
         signature_def_utils_impl.supervised_train_signature_def,
         signature_constants.SUPERVISED_TRAIN_METHOD_NAME)
 
+  @test_util.run_deprecated_v1
   def testEvalSignatureDefMissingInputs(self):
     self._testSupervisedSignatureDefMissingInputs(
         signature_def_utils_impl.supervised_eval_signature_def,
diff --git a/tensorflow/python/saved_model/simple_save_test.py b/tensorflow/python/saved_model/simple_save_test.py
index 0d0665072a..21c2e9df2f 100644
--- a/tensorflow/python/saved_model/simple_save_test.py
+++ b/tensorflow/python/saved_model/simple_save_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import os
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.saved_model import loader
@@ -53,6 +54,7 @@ class SimpleSaveTest(test.TestCase):
       self.assertEqual(actual_tensor_info.tensor_shape.dim[i].size,
                        expected_tensor.shape[i])
 
+  @test_util.run_deprecated_v1
   def testSimpleSave(self):
     """Test simple_save that uses the default parameters."""
     export_dir = os.path.join(test.get_temp_dir(),
diff --git a/tensorflow/python/summary/summary_test.py b/tensorflow/python/summary/summary_test.py
index cacc28cc59..64f0f315c5 100644
--- a/tensorflow/python/summary/summary_test.py
+++ b/tensorflow/python/summary/summary_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -38,6 +39,7 @@ from tensorflow.python.summary import summary as summary_lib
 
 class SummaryTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testScalarSummary(self):
     with self.cached_session() as s:
       i = constant_op.constant(3)
@@ -51,6 +53,7 @@ class SummaryTest(test.TestCase):
     self.assertEqual(values[0].tag, 'outer/inner')
     self.assertEqual(values[0].simple_value, 3.0)
 
+  @test_util.run_deprecated_v1
   def testScalarSummaryWithFamily(self):
     with self.cached_session() as s:
       i = constant_op.constant(7)
@@ -74,6 +77,7 @@ class SummaryTest(test.TestCase):
     self.assertEqual(values[0].tag, 'family/outer/family/inner_1')
     self.assertEqual(values[0].simple_value, 7.0)
 
+  @test_util.run_deprecated_v1
   def testSummarizingVariable(self):
     with self.cached_session() as s:
       c = constant_op.constant(42.0)
@@ -89,6 +93,7 @@ class SummaryTest(test.TestCase):
     self.assertEqual(value.tag, 'summary')
     self.assertEqual(value.simple_value, 42.0)
 
+  @test_util.run_deprecated_v1
   def testImageSummary(self):
     with self.cached_session() as s:
       i = array_ops.ones((5, 4, 4, 3))
@@ -103,6 +108,7 @@ class SummaryTest(test.TestCase):
     expected = sorted('outer/inner/image/{}'.format(i) for i in xrange(3))
     self.assertEqual(tags, expected)
 
+  @test_util.run_deprecated_v1
   def testImageSummaryWithFamily(self):
     with self.cached_session() as s:
       i = array_ops.ones((5, 2, 3, 1))
@@ -119,6 +125,7 @@ class SummaryTest(test.TestCase):
                       for i in xrange(3))
     self.assertEqual(tags, expected)
 
+  @test_util.run_deprecated_v1
   def testHistogramSummary(self):
     with self.cached_session() as s:
       i = array_ops.ones((5, 4, 4, 3))
@@ -130,6 +137,7 @@ class SummaryTest(test.TestCase):
     self.assertEqual(len(summary.value), 1)
     self.assertEqual(summary.value[0].tag, 'outer/inner')
 
+  @test_util.run_deprecated_v1
   def testHistogramSummaryWithFamily(self):
     with self.cached_session() as s:
       i = array_ops.ones((5, 4, 4, 3))
@@ -148,6 +156,7 @@ class SummaryTest(test.TestCase):
       const = constant_op.constant(10, dtype=dtype)
       summary_lib.histogram('h', const)
 
+  @test_util.run_deprecated_v1
   def testAudioSummary(self):
     with self.cached_session() as s:
       i = array_ops.ones((5, 3, 4))
@@ -162,6 +171,7 @@ class SummaryTest(test.TestCase):
     expected = sorted('outer/inner/audio/{}'.format(i) for i in xrange(3))
     self.assertEqual(tags, expected)
 
+  @test_util.run_deprecated_v1
   def testAudioSummaryWithFamily(self):
     with self.cached_session() as s:
       i = array_ops.ones((5, 3, 4))
@@ -178,6 +188,7 @@ class SummaryTest(test.TestCase):
                       for i in xrange(3))
     self.assertEqual(tags, expected)
 
+  @test_util.run_deprecated_v1
   def testTextSummary(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -193,6 +204,7 @@ class SummaryTest(test.TestCase):
       summ = summary_lib.text('foo', array_ops.constant('one'))
       self.assertEqual(summ.op.type, 'TensorSummaryV2')
 
+  @test_util.run_deprecated_v1
   def testSummaryNameConversion(self):
     c = constant_op.constant(3)
     s = summary_lib.scalar('name with spaces', c)
@@ -204,6 +216,7 @@ class SummaryTest(test.TestCase):
     s3 = summary_lib.scalar('/name/with/leading/slash', c)
     self.assertEqual(s3.op.name, 'name/with/leading/slash')
 
+  @test_util.run_deprecated_v1
   def testSummaryWithFamilyMetaGraphExport(self):
     with ops.name_scope('outer'):
       i = constant_op.constant(11)
diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py
index 20b62e5016..d702ddc0a2 100644
--- a/tensorflow/python/summary/writer/writer_test.py
+++ b/tensorflow/python/summary/writer/writer_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -100,6 +101,7 @@ class FileWriterTestCase(test.TestCase):
     # We should be done.
     self.assertRaises(StopIteration, lambda: next(rr))
 
+  @test_util.run_deprecated_v1
   def testAddingSummaryGraphAndRunMetadata(self):
     test_dir = self._CleanTestDir("basics")
     sw = self._FileWriter(test_dir)
@@ -173,6 +175,7 @@ class FileWriterTestCase(test.TestCase):
     # We should be done.
     self.assertRaises(StopIteration, lambda: next(rr))
 
+  @test_util.run_deprecated_v1
   def testGraphAsNamed(self):
     test_dir = self._CleanTestDir("basics_named_graph")
     with ops.Graph().as_default() as g:
@@ -181,6 +184,7 @@ class FileWriterTestCase(test.TestCase):
     sw.close()
     self._assertEventsWithGraph(test_dir, g, True)
 
+  @test_util.run_deprecated_v1
   def testGraphAsPositional(self):
     test_dir = self._CleanTestDir("basics_positional_graph")
     with ops.Graph().as_default() as g:
@@ -189,6 +193,7 @@ class FileWriterTestCase(test.TestCase):
     sw.close()
     self._assertEventsWithGraph(test_dir, g, True)
 
+  @test_util.run_deprecated_v1
   def testGraphDefAsNamed(self):
     test_dir = self._CleanTestDir("basics_named_graph_def")
     with ops.Graph().as_default() as g:
@@ -198,6 +203,7 @@ class FileWriterTestCase(test.TestCase):
     sw.close()
     self._assertEventsWithGraph(test_dir, g, False)
 
+  @test_util.run_deprecated_v1
   def testGraphDefAsPositional(self):
     test_dir = self._CleanTestDir("basics_positional_graph_def")
     with ops.Graph().as_default() as g:
@@ -207,6 +213,7 @@ class FileWriterTestCase(test.TestCase):
     sw.close()
     self._assertEventsWithGraph(test_dir, g, False)
 
+  @test_util.run_deprecated_v1
   def testGraphAndGraphDef(self):
     with self.assertRaises(ValueError):
       test_dir = self._CleanTestDir("basics_graph_and_graph_def")
@@ -216,12 +223,14 @@ class FileWriterTestCase(test.TestCase):
       sw = self._FileWriter(test_dir, graph=g, graph_def=gd)
       sw.close()
 
+  @test_util.run_deprecated_v1
   def testNeitherGraphNorGraphDef(self):
     with self.assertRaises(TypeError):
       test_dir = self._CleanTestDir("basics_string_instead_of_graph")
       sw = self._FileWriter(test_dir, "string instead of graph object")
       sw.close()
 
+  @test_util.run_deprecated_v1
   def testCloseAndReopen(self):
     test_dir = self._CleanTestDir("close_and_reopen")
     sw = self._FileWriter(test_dir)
@@ -265,6 +274,7 @@ class FileWriterTestCase(test.TestCase):
     # We should be done.
     self.assertRaises(StopIteration, lambda: next(rr))
 
+  @test_util.run_deprecated_v1
   def testNonBlockingClose(self):
     test_dir = self._CleanTestDir("non_blocking_close")
     sw = self._FileWriter(test_dir)
@@ -274,6 +284,7 @@ class FileWriterTestCase(test.TestCase):
     sw.close()
     self._assertRecent(time_before_close)
 
+  @test_util.run_deprecated_v1
   def testUseAfterClose(self):
     test_dir = self._CleanTestDir("use_after_close")
     sw = self._FileWriter(test_dir)
@@ -289,6 +300,7 @@ class FileWriterTestCase(test.TestCase):
     for w in triggered:
       self.assertEqual(w.category, UserWarning)
 
+  @test_util.run_deprecated_v1
   def testWithStatement(self):
     test_dir = self._CleanTestDir("with_statement")
     with self._FileWriter(test_dir) as sw:
@@ -299,6 +311,7 @@ class FileWriterTestCase(test.TestCase):
   # Checks that values returned from session Run() calls are added correctly to
   # summaries.  These are numpy types so we need to check they fit in the
   # protocol buffers correctly.
+  @test_util.run_deprecated_v1
   def testAddingSummariesFromSessionRunCalls(self):
     test_dir = self._CleanTestDir("global_step")
     sw = self._FileWriter(test_dir)
@@ -345,6 +358,7 @@ class FileWriterTestCase(test.TestCase):
     # We should be done.
     self.assertRaises(StopIteration, lambda: next(rr))
 
+  @test_util.run_deprecated_v1
   def testPluginMetadataStrippedFromSubsequentEvents(self):
     test_dir = self._CleanTestDir("basics")
     sw = self._FileWriter(test_dir)
@@ -404,6 +418,7 @@ class FileWriterTestCase(test.TestCase):
     # We should be done.
     self.assertRaises(StopIteration, lambda: next(rr))
 
+  @test_util.run_deprecated_v1
   def testFileWriterWithSuffix(self):
     test_dir = self._CleanTestDir("test_suffix")
     sw = self._FileWriter(test_dir, filename_suffix="_test_suffix")
diff --git a/tensorflow/python/tools/freeze_graph_test.py b/tensorflow/python/tools/freeze_graph_test.py
index 5dc14a6961..efdf7dd2cf 100644
--- a/tensorflow/python/tools/freeze_graph_test.py
+++ b/tensorflow/python/tools/freeze_graph_test.py
@@ -161,9 +161,11 @@ class FreezeGraphTest(test_util.TensorFlowTestCase):
             },)
         builder.save(as_text=True)
 
+  @test_util.run_deprecated_v1
   def testFreezeGraphV1(self):
     self._testFreezeGraph(saver_pb2.SaverDef.V1)
 
+  @test_util.run_deprecated_v1
   def testFreezeGraphV2(self):
     self._testFreezeGraph(saver_pb2.SaverDef.V2)
 
diff --git a/tensorflow/python/tools/optimize_for_inference_test.py b/tensorflow/python/tools/optimize_for_inference_test.py
index 10bfb0dc70..310776ff1b 100644
--- a/tensorflow/python/tools/optimize_for_inference_test.py
+++ b/tensorflow/python/tools/optimize_for_inference_test.py
@@ -128,6 +128,7 @@ class OptimizeForInferenceTest(test.TestCase):
         graph_def, [], [add_name], dtypes.float32.as_datatype_enum)
     self.assertProtoEquals(expected_output, output)
 
+  @test_util.run_deprecated_v1
   def testFoldBatchNorms(self):
     with self.cached_session() as sess:
       inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6]
@@ -171,6 +172,7 @@ class OptimizeForInferenceTest(test.TestCase):
     for node in optimized_graph_def.node:
       self.assertNotEqual("BatchNormWithGlobalNormalization", node.op)
 
+  @test_util.run_deprecated_v1
   def testFoldFusedBatchNorms(self):
     for data_format, use_gpu in [("NHWC", False), ("NCHW", True)]:
       with self.cached_session(use_gpu=use_gpu) as sess:
@@ -222,6 +224,7 @@ class OptimizeForInferenceTest(test.TestCase):
       for node in optimized_graph_def.node:
         self.assertNotEqual("FusedBatchNorm", node.op)
 
+  @test_util.run_deprecated_v1
   def testFuseResizePadAndConv(self):
     with self.cached_session() as sess:
       inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6]
@@ -253,6 +256,7 @@ class OptimizeForInferenceTest(test.TestCase):
       self.assertNotEqual("MirrorPad", node.op)
       self.assertNotEqual("ResizeBilinear", node.op)
 
+  @test_util.run_deprecated_v1
   def testFuseResizeAndConv(self):
     with self.cached_session() as sess:
       inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6]
@@ -282,6 +286,7 @@ class OptimizeForInferenceTest(test.TestCase):
       self.assertNotEqual("MirrorPad", node.op)
 
 
+  @test_util.run_deprecated_v1
   def testFusePadAndConv(self):
     with self.cached_session() as sess:
       inputs = [1, 4, 2, 5, 3, 6, -1, -4, -2, -5, -3, -6]
diff --git a/tensorflow/python/training/adadelta_test.py b/tensorflow/python/training/adadelta_test.py
index 7cbaf1039f..0e5af5a922 100644
--- a/tensorflow/python/training/adadelta_test.py
+++ b/tensorflow/python/training/adadelta_test.py
@@ -166,6 +166,7 @@ class AdadeltaOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_resource=True, use_callable_params=True)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/training/adagrad_da_test.py b/tensorflow/python/training/adagrad_da_test.py
index c7c47206a9..aacfe6faf4 100644
--- a/tensorflow/python/training/adagrad_da_test.py
+++ b/tensorflow/python/training/adagrad_da_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -73,12 +74,15 @@ class AdagradDAOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.094821, -0.189358]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testAdagradDAWithoutRegularizationBasic1(self):
     self.doTestAdagradDAwithoutRegularizationBasic1()
 
+  @test_util.run_deprecated_v1
   def testResourceAdagradDAWithoutRegularizationBasic1(self):
     self.doTestAdagradDAwithoutRegularizationBasic1(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -100,6 +104,7 @@ class AdagradDAOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            rtol=0.01)
 
+  @test_util.run_deprecated_v1
   def testAdagradDAwithoutRegularizationBasic2(self):
     for dtype in [dtypes.float64, dtypes.float32]:
       with self.cached_session() as sess:
@@ -132,6 +137,7 @@ class AdagradDAOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.094821, -0.189358]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testAdagradDAWithL1(self):
     for dtype in [dtypes.float64, dtypes.float32]:
       with self.cached_session() as sess:
@@ -164,6 +170,7 @@ class AdagradDAOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.085339, -0.17989]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testAdagradDAWithL1_L2(self):
     for dtype in [dtypes.float64, dtypes.float32]:
       with self.cached_session() as sess:
diff --git a/tensorflow/python/training/adagrad_test.py b/tensorflow/python/training/adagrad_test.py
index 962e65c41f..da26fcdb7f 100644
--- a/tensorflow/python/training/adagrad_test.py
+++ b/tensorflow/python/training/adagrad_test.py
@@ -96,6 +96,7 @@ class AdagradOptimizerTest(test.TestCase):
   def testBasicLocked(self):
     self.doTestBasic(use_locking=True)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -116,6 +117,7 @@ class AdagradOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -142,6 +144,7 @@ class AdagradOptimizerTest(test.TestCase):
             np.array([2.715679168701172, 3.715679168701172]),
             self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseBasic(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -173,6 +176,7 @@ class AdagradOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([[3.0], [3.715679168701172]]), self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndices(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -203,6 +207,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllClose(aggregated_update_var.eval(),
                               self.evaluate(repeated_index_update_var))
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndicesResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -227,6 +232,7 @@ class AdagradOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(
               self.evaluate(var_repeated), self.evaluate(var_aggregated))
 
+  @test_util.run_deprecated_v1
   def testSparseStability(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -263,6 +269,7 @@ class AdagradOptimizerTest(test.TestCase):
                   -0.01029443
               ]]), self.evaluate(var0))
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -299,6 +306,7 @@ class AdagradOptimizerTest(test.TestCase):
             np.array([2.715679168701172, 3.715679168701172]),
             self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testDynamicShapeVariable_Ok(self):
     with self.cached_session():
       v = variable_scope.get_variable("v", initializer=constant_op.constant(1.),
@@ -307,6 +315,7 @@ class AdagradOptimizerTest(test.TestCase):
       # Creating optimizer should cause no exception.
       adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1)
 
+  @test_util.run_deprecated_v1
   def testDynamicShapeVariableWithCallableInit(self):
     var0 = variable_scope.get_variable("var0",
                                        initializer=constant_op.constant(1.),
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index 87dad0a8a6..b0bae27577 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -102,12 +102,15 @@ class AdamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     self.doTestSparse(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceSparse(self):
     self.doTestSparse(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testSparseDevicePlacement(self):
     for index_dtype in [dtypes.int32, dtypes.int64]:
       with self.cached_session(force_gpu=test.is_gpu_available()):
@@ -121,6 +124,7 @@ class AdamOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
         minimize_op.run()
 
+  @test_util.run_deprecated_v1
   def testSparseRepeatedIndices(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -236,6 +240,7 @@ class AdamOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_resource=True, use_callable_params=True)
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -274,6 +279,7 @@ class AdamOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/training/basic_loops_test.py b/tensorflow/python/training/basic_loops_test.py
index 5f5718e64a..511a8334d5 100644
--- a/tensorflow/python/training/basic_loops_test.py
+++ b/tensorflow/python/training/basic_loops_test.py
@@ -23,6 +23,7 @@ import shutil
 
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.training import basic_loops
 from tensorflow.python.training import supervisor
@@ -37,6 +38,7 @@ def _test_dir(test_name):
 
 class BasicTrainLoopTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasicTrainLoop(self):
     logdir = _test_dir("basic_train_loop")
     sv = supervisor.Supervisor(logdir=logdir)
@@ -55,6 +57,7 @@ class BasicTrainLoopTest(test.TestCase):
           sv, train_fn, args=(sv, "y"), kwargs={"a": "A"})
       self.assertEqual(3, num_calls[0])
 
+  @test_util.run_deprecated_v1
   def testBasicTrainLoopExceptionAborts(self):
     logdir = _test_dir("basic_train_loop_exception_aborts")
     sv = supervisor.Supervisor(logdir=logdir)
@@ -71,6 +74,7 @@ class BasicTrainLoopTest(test.TestCase):
       with self.assertRaisesRegexp(RuntimeError, "Failed"):
         basic_loops.basic_train_loop(sv, train_fn)
 
+  @test_util.run_deprecated_v1
   def testBasicTrainLoopRetryOnAborted(self):
     logdir = _test_dir("basic_train_loop_exception_aborts")
     sv = supervisor.Supervisor(logdir=logdir)
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 03810b57e3..8e54a14f47 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import state_ops
@@ -91,10 +92,12 @@ class MockCheckpointSaverListener(
 
 class SecondOrStepTimerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def test_raise_in_both_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SecondOrStepTimer(every_secs=2.0, every_steps=10)
 
+  @test_util.run_deprecated_v1
   def test_raise_in_none_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SecondOrStepTimer()
@@ -413,11 +416,13 @@ class CheckpointSaverHookTest(test.TestCase):
       basic_session_run_hooks.CheckpointSaverHook(
           self.model_dir, saver=self.scaffold.saver, scaffold=self.scaffold)
 
+  @test_util.run_deprecated_v1
   def test_raise_in_both_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.CheckpointSaverHook(
           self.model_dir, save_secs=10, save_steps=20)
 
+  @test_util.run_deprecated_v1
   def test_raise_in_none_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.CheckpointSaverHook(self.model_dir)
@@ -1143,11 +1148,13 @@ class SummarySaverHookTest(test.TestCase):
       basic_session_run_hooks.SummarySaverHook(
           scaffold=monitored_session.Scaffold(), summary_op=self.summary_op)
 
+  @test_util.run_deprecated_v1
   def test_raise_in_both_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
           save_secs=10, save_steps=20, summary_writer=self.summary_writer)
 
+  @test_util.run_deprecated_v1
   def test_raise_in_none_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
@@ -1478,10 +1485,12 @@ class ProfilerHookTest(test.TestCase):
   def _count_timeline_files(self):
     return len(gfile.Glob(self.filepattern))
 
+  @test_util.run_deprecated_v1
   def test_raise_in_both_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.ProfilerHook(save_secs=10, save_steps=20)
 
+  @test_util.run_deprecated_v1
   def test_raise_in_none_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.ProfilerHook(save_secs=None, save_steps=None)
diff --git a/tensorflow/python/training/checkpoint_management_test.py b/tensorflow/python/training/checkpoint_management_test.py
index b61ed17531..8606ec4a20 100644
--- a/tensorflow/python/training/checkpoint_management_test.py
+++ b/tensorflow/python/training/checkpoint_management_test.py
@@ -62,6 +62,7 @@ class LatestCheckpointWithRelativePaths(test.TestCase):
     finally:
       shutil.rmtree(tempdir)
 
+  @test_util.run_deprecated_v1
   def testNameCollision(self):
     # Make sure we have a clean directory to work in.
     with self.tempDir() as tempdir:
@@ -99,6 +100,7 @@ class LatestCheckpointWithRelativePaths(test.TestCase):
           self.assertIsNotNone(
               checkpoint_management.latest_checkpoint(traindir))
 
+  @test_util.run_deprecated_v1
   def testRelativePath(self):
     # Make sure we have a clean directory to work in.
     with self.tempDir() as tempdir:
@@ -270,6 +272,7 @@ class SaverUtilsTest(test.TestCase):
   def tearDown(self):
     gfile.DeleteRecursively(self._base_dir)
 
+  @test_util.run_deprecated_v1
   def testCheckpointExists(self):
     for sharded in (False, True):
       for version in (saver_pb2.SaverDef.V2, saver_pb2.SaverDef.V1):
@@ -288,6 +291,7 @@ class SaverUtilsTest(test.TestCase):
           ckpt_prefix = checkpoint_management.latest_checkpoint(self._base_dir)
           self.assertTrue(checkpoint_management.checkpoint_exists(ckpt_prefix))
 
+  @test_util.run_deprecated_v1
   def testGetCheckpointMtimes(self):
     prefixes = []
     for version in (saver_pb2.SaverDef.V2, saver_pb2.SaverDef.V1):
@@ -302,6 +306,7 @@ class SaverUtilsTest(test.TestCase):
     self.assertEqual(2, len(mtimes))
     self.assertTrue(mtimes[1] >= mtimes[0])
 
+  @test_util.run_deprecated_v1
   def testRemoveCheckpoint(self):
     for sharded in (False, True):
       for version in (saver_pb2.SaverDef.V2, saver_pb2.SaverDef.V1):
diff --git a/tensorflow/python/training/device_setter_test.py b/tensorflow/python/training/device_setter_test.py
index 85b75502ab..3cff87b326 100644
--- a/tensorflow/python/training/device_setter_test.py
+++ b/tensorflow/python/training/device_setter_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -33,6 +34,7 @@ class DeviceSetterTest(test.TestCase):
       "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]
   })
 
+  @test_util.run_deprecated_v1
   def testCPUOverride(self):
     with ops.device(
         device_setter.replica_device_setter(cluster=self._cluster_spec)):
@@ -47,12 +49,14 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:ps/task:1", w.initializer.device)
       self.assertDeviceEqual("/job:worker/cpu:0", a.device)
 
+  @test_util.run_deprecated_v1
   def testResource(self):
     with ops.device(
         device_setter.replica_device_setter(cluster=self._cluster_spec)):
       v = resource_variable_ops.ResourceVariable([1, 2])
       self.assertDeviceEqual("/job:ps/task:0", v.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksWithClusterSpecClass(self):
     with ops.device(
         device_setter.replica_device_setter(cluster=self._cluster_spec)):
@@ -65,6 +69,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:ps/task:1", w.initializer.device)
       self.assertDeviceEqual("/job:worker", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksPinVariableToJob(self):
     with ops.device(
         device_setter.replica_device_setter(cluster=self._cluster_spec)):
@@ -82,6 +87,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:ps/task:1", x.initializer.device)
       self.assertDeviceEqual("/job:worker", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksUseCpuForPS(self):
     with ops.device(
         device_setter.replica_device_setter(ps_tasks=1, ps_device="/cpu:0")):
@@ -95,6 +101,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:moon/cpu:0", w.initializer.device)
       self.assertDeviceEqual("/job:worker", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksNoMerging(self):
     with ops.device(
         device_setter.replica_device_setter(
@@ -109,6 +116,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:ps", w.initializer.device)
       self.assertDeviceEqual("/job:worker", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksWithClusterSpecDict(self):
     with ops.device(
         device_setter.replica_device_setter(cluster=self._cluster_spec.as_dict(
@@ -122,6 +130,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:ps/task:1", w.initializer.device)
       self.assertDeviceEqual("/job:worker", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksWithClusterDef(self):
     with ops.device(
         device_setter.replica_device_setter(
@@ -135,6 +144,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:ps/task:1", w.initializer.device)
       self.assertDeviceEqual("/job:worker", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksWithDevice(self):
     cluster_spec = server_lib.ClusterSpec({
         "sun": ["sun0:2222", "sun1:2222", "sun2:2222"],
@@ -155,6 +165,7 @@ class DeviceSetterTest(test.TestCase):
       self.assertDeviceEqual("/job:moon/task:1", w.initializer.device)
       self.assertDeviceEqual("/job:sun", a.device)
 
+  @test_util.run_deprecated_v1
   def testPS2TasksWithCPUConstraint(self):
     cluster_spec = server_lib.ClusterSpec({
         "sun": ["sun0:2222", "sun1:2222", "sun2:2222"],
diff --git a/tensorflow/python/training/ftrl_test.py b/tensorflow/python/training/ftrl_test.py
index 70b5db31f8..39b299c64a 100644
--- a/tensorflow/python/training/ftrl_test.py
+++ b/tensorflow/python/training/ftrl_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -68,12 +69,15 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.28432083, -0.56694895]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithoutRegularization(self):
     self.doTestFtrlwithoutRegularization(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceFtrlWithoutRegularization(self):
     self.doTestFtrlwithoutRegularization(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testFtrlwithoutRegularization2(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session() as sess:
@@ -103,6 +107,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.28232238, -0.56096673]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -121,6 +126,7 @@ class FtrlOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session() as sess:
@@ -150,6 +156,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.93460727, -1.86147261]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1_L2(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session() as sess:
@@ -180,6 +187,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.02406147, -0.04830509]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1_L2_L2Shrinkage(self):
     """Test the new FTRL op with support for l2 shrinkage.
 
@@ -217,6 +225,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType(
             np.array([-0.14378493, -0.13229476]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL1_L2_L2ShrinkageSparse(self):
     """Tests the new FTRL op with support for l2 shrinkage on sparse grads."""
     for dtype in [dtypes.half, dtypes.float32]:
@@ -251,6 +260,7 @@ class FtrlOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([[-0.22578995], [2.]], v0_val)
         self.assertAllCloseAccordingToType([[4.], [-0.13229476]], v1_val)
 
+  @test_util.run_deprecated_v1
   def testFtrlWithL2ShrinkageDoesNotChangeLrSchedule(self):
     """Verifies that l2 shrinkage in FTRL does not change lr schedule."""
     for dtype in [dtypes.half, dtypes.float32]:
@@ -335,6 +345,7 @@ class FtrlOptimizerTest(test.TestCase):
   # with Adagrad.
   # So, basing on these two properties, we test if our implementation of
   # FTRL-Proximal performs same updates as Adagrad or GradientDescent.
+  @test_util.run_deprecated_v1
   def testEquivAdagradwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
@@ -355,6 +366,7 @@ class FtrlOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(val0, val2)
       self.assertAllCloseAccordingToType(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivSparseAdagradwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
@@ -378,6 +390,7 @@ class FtrlOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(val0, val2)
       self.assertAllCloseAccordingToType(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivSparseGradientDescentwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
@@ -401,6 +414,7 @@ class FtrlOptimizerTest(test.TestCase):
       self.assertAllCloseAccordingToType(val0, val2)
       self.assertAllCloseAccordingToType(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivGradientDescentwithoutRegularization(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with self.cached_session():
diff --git a/tensorflow/python/training/gradient_descent_test.py b/tensorflow/python/training/gradient_descent_test.py
index 2028e7b4b0..5a6c5cfa74 100644
--- a/tensorflow/python/training/gradient_descent_test.py
+++ b/tensorflow/python/training/gradient_descent_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -35,6 +36,7 @@ from tensorflow.python.training import gradient_descent
 
 class GradientDescentOptimizerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -58,6 +60,7 @@ class GradientDescentOptimizerTest(test.TestCase):
                                            self.evaluate(var1))
         self.assertEqual(0, len(optimizer.variables()))
 
+  @test_util.run_deprecated_v1
   def testBasicResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -83,6 +86,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
                                            self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testBasicCallableParams(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -109,6 +113,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
                                            self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testMinimizeResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -135,6 +140,7 @@ class GradientDescentOptimizerTest(test.TestCase):
             [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -162,6 +168,7 @@ class GradientDescentOptimizerTest(test.TestCase):
             [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0))
         self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRate(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -184,6 +191,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
                                            self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testGradWrtRef(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -195,6 +203,7 @@ class GradientDescentOptimizerTest(test.TestCase):
         for grad, _ in grads_and_vars:
           self.assertAllCloseAccordingToType([1.0], self.evaluate(grad))
 
+  @test_util.run_deprecated_v1
   def testWithGlobalStep(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -218,6 +227,7 @@ class GradientDescentOptimizerTest(test.TestCase):
                                            self.evaluate(var1))
         self.assertAllCloseAccordingToType(1, self.evaluate(global_step))
 
+  @test_util.run_deprecated_v1
   def testSparseBasic(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py
index 327f087138..a3d268a017 100644
--- a/tensorflow/python/training/input_test.py
+++ b/tensorflow/python/training/input_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
@@ -41,6 +42,7 @@ from tensorflow.python.util import compat
 
 class MatchFilenamesOnceTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def test(self):
     temp_dir = self.get_temp_dir()
     filenames = [os.path.join(temp_dir, n) for n in os.listdir(temp_dir)]
@@ -68,6 +70,7 @@ class MatchFilenamesOnceTest(test_lib.TestCase):
 
 class LimitEpochsTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoLimit(self):
     with self.cached_session():
       seven = constant_op.constant(7)
@@ -76,6 +79,7 @@ class LimitEpochsTest(test_lib.TestCase):
       for _ in range(100):
         self.assertEqual(7, self.evaluate(seven_forever))
 
+  @test_util.run_deprecated_v1
   def testLimit(self):
     with self.cached_session():
       love_me = constant_op.constant("Love Me")
@@ -90,6 +94,7 @@ class LimitEpochsTest(test_lib.TestCase):
 
 class InputProducerTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoShuffle(self):
     with self.cached_session():
       input_tensor = [[1, 2, 3, 4],
@@ -114,6 +119,7 @@ class InputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testNoShapeInference(self):
     with self.cached_session():
       # Disable shape inference for the input.
@@ -139,6 +145,7 @@ class InputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testShapeError(self):
     input_tensor = array_ops.placeholder(dtypes.float32, None)
     with self.assertRaisesRegexp(ValueError, "fully defined shape"):
@@ -147,6 +154,7 @@ class InputProducerTest(test_lib.TestCase):
 
 class StringInputProducerTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoShuffle(self):
     with self.cached_session():
       strings = [b"to", b"be", b"or", b"not", b"to", b"be"]
@@ -169,6 +177,7 @@ class StringInputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testShuffle(self):
     with self.cached_session():
       strings = [b"a", b"b", b"c"]
@@ -214,6 +223,7 @@ class StringInputProducerTest(test_lib.TestCase):
       with self.assertRaises(ValueError):
         _ = inp.string_input_producer([])
 
+  @test_util.run_deprecated_v1
   def testNullString(self):
     # Runtime check for empty string list.  This is slightly oblique:
     # The queue runner should die with an assertion error on the null
@@ -233,6 +243,7 @@ class StringInputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       strings = [b"to", b"be", b"or", b"not", b"to", b"be"]
@@ -241,6 +252,7 @@ class StringInputProducerTest(test_lib.TestCase):
       self.assertProtoEquals("s: 'SHARED_NAME_XYZ'",
                              queue.queue_ref.op.node_def.attr["shared_name"])
 
+  @test_util.run_deprecated_v1
   def testConstructionRace(self):
     with self.cached_session() as sess:
       strings = [b"to", b"be", b"or", b"not", b"to", b"be"]
@@ -263,6 +275,7 @@ class StringInputProducerTest(test_lib.TestCase):
 
 class RangeInputProducerTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoShuffle(self):
     with self.cached_session():
       num_epochs = 3
@@ -285,6 +298,7 @@ class RangeInputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testShuffle(self):
     with self.cached_session():
       num_epochs = 200
@@ -324,6 +338,7 @@ class RangeInputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       range_size = 5
@@ -335,6 +350,7 @@ class RangeInputProducerTest(test_lib.TestCase):
 
 class SliceInputProducerTest(test_lib.TestCase):
 
+  @test_util.run_deprecated_v1
   def testNoShuffle(self):
     with self.cached_session() as sess:
       num_epochs = 3
@@ -359,6 +375,7 @@ class SliceInputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testShuffle(self):
     with self.cached_session() as sess:
       num_epochs = 1200
@@ -403,6 +420,7 @@ class SliceInputProducerTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       source_strings = ["A", "B", "D", "G"]
@@ -495,12 +513,15 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testOneThread(self):
     self._testOneThreadHelper(use_dict=False)
 
+  @test_util.run_deprecated_v1
   def testOneThreadDict(self):
     self._testOneThreadHelper(use_dict=True)
 
+  @test_util.run_deprecated_v1
   def testUint32DataTypes(self):
     values = constant_op.constant([0, 1, 2, 3, 4, 5], dtype=dtypes.uint32)
     batched = inp.batch([values], batch_size=2)
@@ -512,6 +533,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testUint64DataTypes(self):
     values = constant_op.constant([0, 1, 2, 3, 4, 5], dtype=dtypes.uint64)
     batched = inp.batch([values], batch_size=2)
@@ -523,6 +545,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testOneThreadDynamicPad(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -553,6 +576,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testOneThreadEnqueueMany(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -588,6 +612,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testManyThreads(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -628,6 +653,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testOneThreadSmallerBatch(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -685,6 +711,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testManyThreadsSmallerBatch(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -740,6 +767,7 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       batch_size = 10
@@ -757,12 +785,14 @@ class BatchTest(test_lib.TestCase):
           "s: 'SHARED_NAME_XYZ'",
           batched[0].op.inputs[0].op.node_def.attr["shared_name"])
 
+  @test_util.run_deprecated_v1
   def testCannotInferRankError(self):
     with self.cached_session():
       x = array_ops.placeholder(dtype=dtypes.int64)
       with self.assertRaisesRegexp(ValueError, "Cannot infer Tensor's rank"):
         inp.batch([x], batch_size=2)
 
+  @test_util.run_deprecated_v1
   def testBatchedSparseTensorInferredShape(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -770,6 +800,7 @@ class BatchTest(test_lib.TestCase):
     batched = inp.batch([sparse], batch_size=2)
     self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testBatchedSparseTensorInferredShapeEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -777,6 +808,7 @@ class BatchTest(test_lib.TestCase):
     batched = inp.batch([sparse], batch_size=2, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testBatchedSparseTensorInferredShapeUnknownRank(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -786,6 +818,7 @@ class BatchTest(test_lib.TestCase):
     batched = inp.batch([sparse], batch_size=2)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -795,6 +828,7 @@ class BatchTest(test_lib.TestCase):
     batched = inp.batch([sparse], batch_size=2, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testSingleElementDict(self):
     x = inp.batch({"c": [12, 12]}, batch_size=8)
     self.assertAllEqual((8, 2), x["c"].get_shape().as_list())
@@ -838,24 +872,31 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
+  @test_util.run_deprecated_v1
   def testMaybeEnqueuePerExample(self):
     self._testKeepInputHelper(1, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadMaybeEnqueuePerExample(self):
     self._testKeepInputHelper(5, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testInvalidKeepInputVector(self):
     # Can't have vector `keep_input` with `enqueue_many=False`.
     with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"):
@@ -877,6 +918,7 @@ class BatchTest(test_lib.TestCase):
                       batch_size=1,
                       enqueue_many=True)
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShape(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -884,6 +926,7 @@ class BatchTest(test_lib.TestCase):
     batched = inp.maybe_batch([sparse], keep_input=True, batch_size=2)
     self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -892,6 +935,7 @@ class BatchTest(test_lib.TestCase):
         [sparse], keep_input=True, batch_size=2, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2])
@@ -900,6 +944,7 @@ class BatchTest(test_lib.TestCase):
         [sparse], keep_input=[True, False], batch_size=2, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -909,6 +954,7 @@ class BatchTest(test_lib.TestCase):
     batched = inp.maybe_batch([sparse], keep_input=True, batch_size=2)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -919,6 +965,7 @@ class BatchTest(test_lib.TestCase):
         [sparse], keep_input=True, batch_size=2, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -929,6 +976,7 @@ class BatchTest(test_lib.TestCase):
         [sparse], keep_input=[True, False], batch_size=2, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchCorrectValues(self):
     sparse_t = sparse_tensor.SparseTensor(
         indices=[[0, 1], [0, 2], [1, 0], [1, 3]],
@@ -1055,12 +1103,15 @@ class BatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testTwoThreads(self):
     self._testTwoThreadsHelper(use_dict=False)
 
+  @test_util.run_deprecated_v1
   def testTwoThreadsDict(self):
     self._testTwoThreadsHelper(use_dict=True)
 
+  @test_util.run_deprecated_v1
   def testMismatchedDictKeys(self):
     with self.assertRaisesRegexp(ValueError, "must have the same keys"):
       inp.batch_join(
@@ -1075,6 +1126,7 @@ class BatchJoinTest(test_lib.TestCase):
           }],
           batch_size=8)
 
+  @test_util.run_deprecated_v1
   def testTwoThreadsDynamicPad(self):
     with self.cached_session() as sess:
       # Two threads, the first generates (0..69, ["a"] * 1..70).
@@ -1152,6 +1204,7 @@ class BatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testTwoThreadsSmallerBatch(self):
     with self.cached_session() as sess:
       extra_elements = 2
@@ -1253,6 +1306,7 @@ class BatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testTwoThreadsDynamicPadSmallerBatch(self):
     with self.cached_session() as sess:
       extra_elements = 2
@@ -1351,6 +1405,7 @@ class BatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       batch_size = 10
@@ -1373,12 +1428,14 @@ class BatchJoinTest(test_lib.TestCase):
           "s: 'SHARED_NAME_XYZ'",
           batched[0].op.inputs[0].op.node_def.attr["shared_name"])
 
+  @test_util.run_deprecated_v1
   def testCannotInferRankError(self):
     with self.cached_session():
       x = array_ops.placeholder(dtype=dtypes.int64)
       with self.assertRaisesRegexp(ValueError, "Cannot infer Tensor's rank"):
         inp.batch_join([[x]], batch_size=2)
 
+  @test_util.run_deprecated_v1
   def testSingleElementDict(self):
     x = inp.batch_join([{"c": [12, 12]}], batch_size=8)
     self.assertAllEqual((8, 2), x["c"].get_shape().as_list())
@@ -1425,24 +1482,31 @@ class BatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputPerExample(self):
     self._testKeepInputHelper(1, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputPerExample(self):
     self._testKeepInputHelper(5, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testInvalidKeepInputVector(self):
     # Can't have vector `keep_input` with `enqueue_many=False`.
     with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"):
@@ -1464,6 +1528,7 @@ class BatchJoinTest(test_lib.TestCase):
                            batch_size=1,
                            enqueue_many=True)
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShape(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -1471,6 +1536,7 @@ class BatchJoinTest(test_lib.TestCase):
     batched = inp.maybe_batch_join([[sparse]], keep_input=True, batch_size=2)
     self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -1479,6 +1545,7 @@ class BatchJoinTest(test_lib.TestCase):
         [[sparse]], keep_input=True, batch_size=2, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2])
@@ -1487,6 +1554,7 @@ class BatchJoinTest(test_lib.TestCase):
         [[sparse]], keep_input=[True, False], batch_size=2, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -1496,6 +1564,7 @@ class BatchJoinTest(test_lib.TestCase):
     batched = inp.maybe_batch_join([[sparse]], keep_input=True, batch_size=2)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -1506,6 +1575,7 @@ class BatchJoinTest(test_lib.TestCase):
         [[sparse]], keep_input=True, batch_size=2, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -1516,6 +1586,7 @@ class BatchJoinTest(test_lib.TestCase):
         [[sparse]], keep_input=[True, False], batch_size=2, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchCorrectValues(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0, 1], [0, 2], [1, 0], [1, 3]],
@@ -1601,12 +1672,15 @@ class ShuffleBatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testOneThread(self):
     self._testOneThreadHelper(use_dict=False)
 
+  @test_util.run_deprecated_v1
   def testOneThreadDict(self):
     self._testOneThreadHelper(use_dict=True)
 
+  @test_util.run_deprecated_v1
   def testOneThreadSmallerBatch(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -1663,6 +1737,7 @@ class ShuffleBatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testManyThreads(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -1710,6 +1785,7 @@ class ShuffleBatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testManyThreadsSmallerBatch(self):
     with self.cached_session() as sess:
       batch_size = 10
@@ -1768,6 +1844,7 @@ class ShuffleBatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       batch_size = 10
@@ -1828,24 +1905,31 @@ class ShuffleBatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputPerExample(self):
     self._testKeepInputHelper(1, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputPerExample(self):
     self._testKeepInputHelper(5, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testInvalidKeepInputVector(self):
     # Can't have vector `keep_input` with `enqueue_many=False`.
     with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"):
@@ -1864,6 +1948,7 @@ class ShuffleBatchTest(test_lib.TestCase):
                               keep_input=array_ops.placeholder(dtypes.bool),
                               enqueue_many=True)
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShape(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -1871,6 +1956,7 @@ class ShuffleBatchTest(test_lib.TestCase):
     batched = inp.maybe_shuffle_batch([sparse], 2, 10, 1, True)
     self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -1879,6 +1965,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         [sparse], 2, 10, 1, True, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2])
@@ -1887,6 +1974,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         [sparse], 2, 10, 1, [True, False], enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -1896,6 +1984,7 @@ class ShuffleBatchTest(test_lib.TestCase):
     batched = inp.maybe_shuffle_batch([sparse], 2, 10, 1, True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -1906,6 +1995,7 @@ class ShuffleBatchTest(test_lib.TestCase):
         [sparse], 2, 10, 1, True, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -2024,12 +2114,15 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testTwoThreads(self):
     self._testTwoThreadsHelper(use_dict=False)
 
+  @test_util.run_deprecated_v1
   def testTwoThreadsDict(self):
     self._testTwoThreadsHelper(use_dict=True)
 
+  @test_util.run_deprecated_v1
   def testTwoThreadsSmallerBatch(self):
     with self.cached_session() as sess:
       # Two threads, the first generates (0..26, "a").
@@ -2133,6 +2226,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testMismatchedDictKeys(self):
     with self.assertRaisesRegexp(ValueError, "must have the same keys"):
       inp.shuffle_batch_join(
@@ -2150,6 +2244,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
           min_after_dequeue=16,
           seed=223607)
 
+  @test_util.run_deprecated_v1
   def testSharedName(self):
     with self.cached_session():
       batch_size = 10
@@ -2214,24 +2309,31 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
+  @test_util.run_deprecated_v1
   def testSingleThreadKeepInputPerExample(self):
     self._testKeepInputHelper(1, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testMultipleThreadKeepInputPerExample(self):
     self._testKeepInputHelper(5, True, keep_input_vector=True)
 
+  @test_util.run_deprecated_v1
   def testInvalidKeepInputVector(self):
     # Can't have vector `keep_input` with `enqueue_many=False`.
     with self.assertRaisesRegexp(ValueError, "`keep_input` cannot be a vector"):
@@ -2253,6 +2355,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
           keep_input=array_ops.placeholder(dtypes.bool),
           enqueue_many=True)
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShape(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -2260,6 +2363,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
     batched = inp.maybe_shuffle_batch_join([[sparse]], 2, 10, 1, True)
     self.assertAllEqual((2,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0]], values=[1.0], dense_shape=[1])
@@ -2268,6 +2372,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
         [[sparse]], 2, 10, 1, True, enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeEnqueueManyPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=[[0], [0]], values=[1.0, 2.0], dense_shape=[2])
@@ -2276,6 +2381,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
         [[sparse]], 2, 10, 1, [True, False], enqueue_many=True)
     self.assertAllEqual((1,), batched.dense_shape.get_shape().as_list())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRank(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -2285,6 +2391,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
     batched = inp.maybe_shuffle_batch_join([[sparse]], 2, 10, 1, True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankEnqueueMany(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
@@ -2295,6 +2402,7 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
         [[sparse]], 2, 10, 1, True, enqueue_many=True)
     self.assertIs(None, batched.dense_shape.get_shape().num_elements())
 
+  @test_util.run_deprecated_v1
   def testMaybeBatchedSparseTensorInferredShapeUnknownRankPerExample(self):
     sparse = sparse_tensor.SparseTensor(
         indices=array_ops.placeholder(dtypes.int64),
diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py
index 9c31c0924f..9de5bc8168 100644
--- a/tensorflow/python/training/learning_rate_decay_test.py
+++ b/tensorflow/python/training/learning_rate_decay_test.py
@@ -61,6 +61,7 @@ class LRDecayTest(test_util.TensorFlowTestCase):
       self.evaluate(step.assign(100))
       self.assertAllClose(self.evaluate(decayed_lr), expected, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testVariables(self):
     step = variables.VariableV1(1)
     assign_1 = step.assign(1)
@@ -100,6 +101,7 @@ class LRDecayTest(test_util.TensorFlowTestCase):
     self.assertAllClose(self.evaluate(decayed_lr), 0.001, 1e-6)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testPiecewiseConstantEdgeCases(self):
     x_int = resource_variable_ops.ResourceVariable(
         0, dtype=variables.dtypes.int32)
diff --git a/tensorflow/python/training/learning_rate_decay_v2_test.py b/tensorflow/python/training/learning_rate_decay_v2_test.py
index 354ddb25be..cb96773e29 100644
--- a/tensorflow/python/training/learning_rate_decay_v2_test.py
+++ b/tensorflow/python/training/learning_rate_decay_v2_test.py
@@ -61,6 +61,7 @@ class LRDecayTestV2(test_util.TensorFlowTestCase):
       self.evaluate(step.assign(100))
       self.assertAllClose(self.evaluate(decayed_lr()), expected, 1e-6)
 
+  @test_util.run_deprecated_v1
   def testVariables(self):
     step = variables.Variable(1)
     assign_1 = step.assign(1)
diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py
index b6cac6addf..ba155fa6c6 100644
--- a/tensorflow/python/training/momentum_test.py
+++ b/tensorflow/python/training/momentum_test.py
@@ -160,6 +160,7 @@ class MomentumOptimizerTest(test.TestCase):
       self.assertStartsWith(optimizer_variables[1].name, "var3")
       self.assertEquals(2, len(optimizer_variables))
 
+  @test_util.run_deprecated_v1
   def testNesterovMomentum(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -186,6 +187,7 @@ class MomentumOptimizerTest(test.TestCase):
           self.assertAllClose(var0_np, self.evaluate(var0))
           self.assertAllClose(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testSparseNesterovMomentum(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -280,6 +282,7 @@ class MomentumOptimizerTest(test.TestCase):
     self.evaluate(sgd_op)
     self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0))
 
+  @test_util.run_deprecated_v1
   def testTensorLearningRateAndMomentum(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -440,6 +443,7 @@ class MomentumOptimizerTest(test.TestCase):
     # pylint: enable=line-too-long
     return db_grad, db_out
 
+  @test_util.run_deprecated_v1
   def testLikeDistBeliefMom01(self):
     with self.cached_session():
       db_grad, db_out = self._dbParamsMom01()
@@ -453,6 +457,7 @@ class MomentumOptimizerTest(test.TestCase):
         mom_update.run(feed_dict={grads0: db_grad[i]})
         self.assertAllClose(np.array(db_out[i]), self.evaluate(var0))
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -534,6 +539,7 @@ class MomentumOptimizerTest(test.TestCase):
             ]),
             self.evaluate(var1)[2])
 
+  @test_util.run_deprecated_v1
   def testSharing(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 2ceb387ec3..9dbcfa52b7 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -37,6 +37,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -504,6 +505,7 @@ class StopAtNSession(monitored_session._WrappedSession):
 class WrappedSessionTest(test.TestCase):
   """_WrappedSession tests."""
 
+  @test_util.run_deprecated_v1
   def test_properties(self):
     with self.cached_session() as sess:
       constant_op.constant(0.0)
@@ -511,6 +513,7 @@ class WrappedSessionTest(test.TestCase):
       self.assertEquals(sess.graph, wrapped_sess.graph)
       self.assertEquals(sess.sess_str, wrapped_sess.sess_str)
 
+  @test_util.run_deprecated_v1
   def test_should_stop_on_close(self):
     with self.cached_session() as sess:
       wrapped_sess = monitored_session._WrappedSession(sess)
@@ -518,6 +521,7 @@ class WrappedSessionTest(test.TestCase):
       wrapped_sess.close()
       self.assertTrue(wrapped_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_should_stop_uses_check_stop(self):
     with self.cached_session() as sess:
       wrapped_sess = StopAtNSession(sess, 3)
@@ -526,6 +530,7 @@ class WrappedSessionTest(test.TestCase):
       self.assertFalse(wrapped_sess.should_stop())
       self.assertTrue(wrapped_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_should_stop_delegates_to_wrapped_session(self):
     with self.cached_session() as sess:
       wrapped_sess0 = StopAtNSession(sess, 4)
@@ -544,6 +549,7 @@ class WrappedSessionTest(test.TestCase):
       wrapped_sess.close()
       self.assertTrue(wrapped_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_run(self):
     with self.cached_session() as sess:
       c = constant_op.constant(0)
@@ -561,6 +567,7 @@ def busy_wait_for_coord_stop(coord):
 class CoordinatedSessionTest(test.TestCase):
   """_CoordinatedSession tests."""
 
+  @test_util.run_deprecated_v1
   def test_properties(self):
     with self.cached_session() as sess:
       constant_op.constant(0.0)
@@ -569,6 +576,7 @@ class CoordinatedSessionTest(test.TestCase):
       self.assertEquals(sess.graph, coord_sess.graph)
       self.assertEquals(sess.sess_str, coord_sess.sess_str)
 
+  @test_util.run_deprecated_v1
   def test_run(self):
     with self.cached_session() as sess:
       c = constant_op.constant(0)
@@ -577,6 +585,7 @@ class CoordinatedSessionTest(test.TestCase):
       coord_sess = monitored_session._CoordinatedSession(sess, coord)
       self.assertEqual(42, coord_sess.run(v, feed_dict={c: 42}))
 
+  @test_util.run_deprecated_v1
   def test_should_stop_on_close(self):
     with self.cached_session() as sess:
       coord = coordinator.Coordinator()
@@ -585,6 +594,7 @@ class CoordinatedSessionTest(test.TestCase):
       coord_sess.close()
       self.assertTrue(coord_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_should_stop_on_coord_stop(self):
     with self.cached_session() as sess:
       coord = coordinator.Coordinator()
@@ -593,6 +603,7 @@ class CoordinatedSessionTest(test.TestCase):
       coord.request_stop()
       self.assertTrue(coord_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_dont_request_stop_on_exception_in_main_thread(self):
     with self.cached_session() as sess:
       c = constant_op.constant(0)
@@ -607,6 +618,7 @@ class CoordinatedSessionTest(test.TestCase):
       self.assertFalse(coord.should_stop())
       self.assertFalse(coord_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_stop_threads_on_close_after_exception(self):
     with self.cached_session() as sess:
       c = constant_op.constant(0)
@@ -654,6 +666,7 @@ class CoordinatedSessionTest(test.TestCase):
       self.assertTrue(coord.should_stop())
       self.assertTrue(coord_sess.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_propagates_exception_trace(self):
     assertion = control_flow_ops.Assert(False, ['This should fail.'])
     with self.cached_session() as sess:
@@ -801,6 +814,7 @@ class RecoverableSessionTest(test.TestCase):
     def create_session(self):
       return self._sess
 
+  @test_util.run_deprecated_v1
   def test_properties(self):
     with self.cached_session() as sess:
       constant_op.constant(0.0)
@@ -809,6 +823,7 @@ class RecoverableSessionTest(test.TestCase):
       self.assertEquals(sess.graph, recoverable_sess.graph)
       self.assertEquals(sess.sess_str, recoverable_sess.sess_str)
 
+  @test_util.run_deprecated_v1
   def test_run(self):
     with self.cached_session() as sess:
       c = constant_op.constant(0)
@@ -817,6 +832,7 @@ class RecoverableSessionTest(test.TestCase):
           self._SessionReturner(sess))
       self.assertEqual(51, recoverable_sess.run(v, feed_dict={c: 51}))
 
+  @test_util.run_deprecated_v1
   def test_recovery(self):
     with self.cached_session() as sess:
 
@@ -863,6 +879,7 @@ class RecoverableSessionTest(test.TestCase):
       with self.assertRaisesRegexp(IndexError, 'pop from empty list'):
         recoverable_sess.run(v, feed_dict={c: -12})
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_coordinator_exception(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -888,6 +905,7 @@ class RecoverableSessionTest(test.TestCase):
       self.assertFalse(session.should_stop())
       self.assertEqual(2, session_creator.number_of_sessions_created)
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_non_preemption_in_coordinator(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -917,6 +935,7 @@ class RecoverableSessionTest(test.TestCase):
       with self.assertRaises(errors_impl.UnknownError):
         session.close()
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_session_getting_stuck(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -941,6 +960,7 @@ class RecoverableSessionTest(test.TestCase):
       self.assertFalse(session.should_stop())
       self.assertEqual(2, session_creator.number_of_sessions_created)
 
+  @test_util.run_deprecated_v1
   def test_step_fn_recovery_from_coordinator_exception_when_run_hooks(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -971,6 +991,7 @@ class RecoverableSessionTest(test.TestCase):
       self.assertFalse(session.should_stop())
       self.assertEqual(2, session_creator.number_of_sessions_created)
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_non_preemption_in_coordinator_when_run_hooks(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -1005,6 +1026,7 @@ class RecoverableSessionTest(test.TestCase):
       with self.assertRaises(errors_impl.UnknownError):
         session.close()
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_session_getting_stuck_when_run_hooks(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -1049,6 +1071,7 @@ class RecoverableSessionTest(test.TestCase):
     # exception.
     return session
 
+  @test_util.run_deprecated_v1
   def test_step_fn_recovery_from_coordinator_exception_with_raw_session(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -1081,6 +1104,7 @@ class RecoverableSessionTest(test.TestCase):
       self.assertFalse(session.should_stop())
       self.assertEqual(2, session_creator.number_of_sessions_created)
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_non_preemption_in_coordinator_with_raw_session(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -1118,6 +1142,7 @@ class RecoverableSessionTest(test.TestCase):
       with self.assertRaises(errors_impl.UnknownError):
         session.close()
 
+  @test_util.run_deprecated_v1
   def test_recovery_from_session_getting_stuck_with_raw_session(self):
     with self.cached_session() as test_session:
       session_creator = CountingSessionCreator(test_session)
@@ -1443,6 +1468,7 @@ class MonitoredSessionTest(test.TestCase):
   # This set of tests, verifies the supervised session behavior when exceptions
   # are raised next to the innermost session run() call.
 
+  @test_util.run_deprecated_v1
   def test_recovery(self):
     logdir = _test_dir(self.get_temp_dir(), 'test_recovery')
     with ops.Graph().as_default():
@@ -1795,6 +1821,7 @@ class MonitoredSessionTest(test.TestCase):
             isinstance(hook.run_metadata_list[0], config_pb2.RunMetadata))
         self.assertGreater(len(hook.run_metadata_list[0].partition_graphs), 0)
 
+  @test_util.run_deprecated_v1
   def test_with_statement_and_close(self):
     # Test case for https://github.com/tensorflow/tensorflow/issues/12224
     # where close() inside the with should have a better error message.
diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py
index 6ce5de6663..b15f7377f0 100644
--- a/tensorflow/python/training/moving_averages_test.py
+++ b/tensorflow/python/training/moving_averages_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.training import saver as saver_lib
 
 class MovingAveragesTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAssignMovingAverageWithoutZeroDebias(self):
     with self.cached_session():
       var = variables.Variable([10.0, 11.0])
@@ -49,6 +50,7 @@ class MovingAveragesTest(test.TestCase):
           [10.0 * 0.25 + 1.0 * (1.0 - 0.25), 11.0 * 0.25 + 2.0 * (1.0 - 0.25)],
           self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testAssignMovingAverage(self):
     with self.cached_session():
       var = variables.Variable([0.0, 0.0])
@@ -62,6 +64,7 @@ class MovingAveragesTest(test.TestCase):
           [1.0 * (1.0 - 0.25) / (1 - 0.25), 2.0 * (1.0 - 0.25) / (1 - 0.25)],
           self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testAssignMovingAverageNewNamingMultipleCalls(self):
     with variable_scope.variable_scope("scope1") as vs1:
       with variable_scope.variable_scope("scope2"):
@@ -76,6 +79,7 @@ class MovingAveragesTest(test.TestCase):
     actual_names = [v.name for v in vs1.global_variables()]
     self.assertSetEqual(set(expected_names), set(actual_names))
 
+  @test_util.run_deprecated_v1
   def testAssignMovingAverageNewNamingMultipleCallsWithReuse(self):
     with variable_scope.variable_scope("scope1") as vs1:
       var = variable_scope.get_variable("Var", shape=[])
@@ -86,6 +90,7 @@ class MovingAveragesTest(test.TestCase):
       moving_averages.assign_moving_average(var, 0.0, 0.99)
       moving_averages.assign_moving_average(var, 0.0, 0.99)
 
+  @test_util.run_deprecated_v1
   def testWeightedMovingAverage(self):
     with self.cached_session() as sess:
       decay = 0.5
@@ -111,6 +116,7 @@ class MovingAveragesTest(test.TestCase):
       denominator_2 = denominator_1 * decay + weight_2 * (1.0 - decay)
       self.assertAllClose(numerator_2 / denominator_2, wma_array)
 
+  @test_util.run_deprecated_v1
   def testWeightedMovingAverageBfloat16(self):
     bfloat16 = pywrap_tensorflow.TF_bfloat16_type()
     with self.cached_session() as sess:
@@ -213,32 +219,38 @@ class ExponentialMovingAverageTest(test.TestCase):
                         (10.0 + 30.0) * (1 - dk)) / _Scale(dk, 2), dim)
     self.assertAllClose(expected, self.evaluate(avg2))
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNoNumUpdates_Scalar(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25)
       self._CheckDecay(ema, actual_decay=0.25, dim=1)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNoNumUpdates_Scalar_Debias(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.25, dim=1)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNoNumUpdates_Vector(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25)
       self._CheckDecay(ema, actual_decay=0.25, dim=5)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNoNumUpdates_Vector_Debias(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.25, dim=5)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNumUpdates_Scalar(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
       ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1)
       self._CheckDecay(ema, actual_decay=0.181818, dim=1)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNumUpdates_Scalar_Debias(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
@@ -246,12 +258,14 @@ class ExponentialMovingAverageTest(test.TestCase):
           0.25, num_updates=1, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.181818, dim=1)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNumUpdates_Vector(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
       ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1)
       self._CheckDecay(ema, actual_decay=0.181818, dim=5)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNumUpdates_Vector_Debias(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
@@ -259,6 +273,7 @@ class ExponentialMovingAverageTest(test.TestCase):
           0.25, num_updates=1, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.181818, dim=5)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesWithControlDeps(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(0, name="v0")
@@ -284,6 +299,7 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual([17.5], self.evaluate(v1_avg))
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testBasicEager(self):
     v0 = variables.Variable(1.0)
     v1 = variables.Variable(2.0)
@@ -339,9 +355,11 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual(ema.average(v1).op.name, ema.average_name(v1))
       self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2))
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNames(self):
     self.averageVariablesNamesHelper(zero_debias=True)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNamesNoDebias(self):
     self.averageVariablesNamesHelper(zero_debias=False)
 
@@ -387,12 +405,15 @@ class ExponentialMovingAverageTest(test.TestCase):
         self.assertEqual(
             ema.average(tensor2).op.name, ema.average_name(tensor2))
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNamesRespectScope(self):
     self.averageVariablesNamesRespectScopeHelper(zero_debias=True)
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesNamesRespectScopeNoDebias(self):
     self.averageVariablesNamesRespectScopeHelper(zero_debias=False)
 
+  @test_util.run_deprecated_v1
   def testSubsetAverageVariablesNames(self):
     with self.cached_session():
       v0 = variables.Variable(10.0, name="v0")
@@ -421,6 +442,7 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual(ema.average(v1).op.name, ema.average_name(v1))
       self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2))
 
+  @test_util.run_deprecated_v1
   def testAverageVariablesDeviceAssignment(self):
     with ops.device("/job:dev_v0"):
       v0 = variables.Variable(10.0, name="v0")
@@ -451,6 +473,7 @@ class ExponentialMovingAverageTest(test.TestCase):
       _ = saver_lib.import_meta_graph(meta_graph)
     return graph_copy
 
+  @test_util.run_deprecated_v1
   def testImportedGraphVariablesToRestore(self):
     g = ops.Graph()
     with g.as_default():
diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py
index 5ed0a30285..e175b5a799 100644
--- a/tensorflow/python/training/optimizer_test.py
+++ b/tensorflow/python/training/optimizer_test.py
@@ -62,6 +62,7 @@ class OptimizerTest(test.TestCase):
       self.assertAllClose([-14., -13.], self.evaluate(var0))
       self.assertAllClose([-6., -5.], self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testAggregationMethod(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -87,6 +88,7 @@ class OptimizerTest(test.TestCase):
         self.assertAllClose([-14., -13.], self.evaluate(var0))
         self.assertAllClose([-6., -5.], self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testPrecomputedGradient(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -230,6 +232,7 @@ class OptimizerTest(test.TestCase):
     with self.assertRaises(NotImplementedError):
       sgd_op.apply_gradients(grads_and_vars)
 
+  @test_util.run_deprecated_v1
   def testTrainOp(self):
     with self.cached_session():
       var0 = variables.Variable([1.0, 2.0])
@@ -241,6 +244,7 @@ class OptimizerTest(test.TestCase):
       opt_op = sgd_op.minimize(cost, global_step, [var0, var1])
       self.assertTrue(opt_op in ops.get_collection(ops.GraphKeys.TRAIN_OP))
 
+  @test_util.run_deprecated_v1
   def testConstraint(self):
     constraint_01 = lambda x: clip_ops.clip_by_value(x, -0.1, 0.)
     constraint_0 = lambda x: clip_ops.clip_by_value(x, 0., 1.)
diff --git a/tensorflow/python/training/proximal_adagrad_test.py b/tensorflow/python/training/proximal_adagrad_test.py
index 9d46a6682d..ce214ac418 100644
--- a/tensorflow/python/training/proximal_adagrad_test.py
+++ b/tensorflow/python/training/proximal_adagrad_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -64,12 +65,15 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       self.assertStartsWith(opt_vars[1].name, var1._shared_name)
       self.assertEqual(2, len(opt_vars))
 
+  @test_util.run_deprecated_v1
   def testProximalAdagradwithoutRegularization(self):
     self.doTestProximalAdagradwithoutRegularization(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceProximalAdagradwithoutRegularization(self):
     self.doTestProximalAdagradwithoutRegularization(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testProximalAdagradwithoutRegularization2(self):
     with self.cached_session() as sess:
       var0 = variables.Variable([1.0, 2.0])
@@ -96,6 +100,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       self.assertAllClose(np.array([-1.60261, -2.296985]), v0_val)
       self.assertAllClose(np.array([3.715679, 2.433051]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -114,6 +119,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testProximalAdagradWithL1(self):
     with self.cached_session() as sess:
       var0 = variables.Variable([1.0, 2.0])
@@ -140,6 +146,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
       self.assertAllClose(np.array([-6.663634, -9.190331]), v0_val)
       self.assertAllClose(np.array([2.959304, 1.029232]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testProximalAdagradWithL1_L2(self):
     with self.cached_session() as sess:
       var0 = variables.Variable([1.0, 2.0])
@@ -206,6 +213,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
     v0_val, v1_val = self.evaluate([var0, var1])
     return v0_val, v1_val
 
+  @test_util.run_deprecated_v1
   def testEquivAdagradwithoutRegularization(self):
     with self.cached_session():
       val0, val1 = self.applyOptimizer(
@@ -223,6 +231,7 @@ class ProximalAdagradOptimizerTest(test.TestCase):
     self.assertAllClose(val0, val2)
     self.assertAllClose(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivSparseAdagradwithoutRegularization(self):
     with self.cached_session():
       val0, val1 = self.applyOptimizer(
diff --git a/tensorflow/python/training/proximal_gradient_descent_test.py b/tensorflow/python/training/proximal_gradient_descent_test.py
index 8797b308eb..25b206605d 100644
--- a/tensorflow/python/training/proximal_gradient_descent_test.py
+++ b/tensorflow/python/training/proximal_gradient_descent_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -62,12 +63,15 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       self.assertAllClose(np.array([-0.9, -1.8]), v0_val)
       self.assertAllClose(np.array([-0.09, -0.18]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testProximalGradientDescentwithoutRegularization(self):
     self.doTestProximalGradientDescentwithoutRegularization(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testResourceProximalGradientDescentwithoutRegularization(self):
     self.doTestProximalGradientDescentwithoutRegularization(use_resource=True)
 
+  @test_util.run_deprecated_v1
   def testProximalGradientDescentwithoutRegularization2(self):
     with self.cached_session() as sess:
       var0 = variables.Variable([1.0, 2.0])
@@ -92,6 +96,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
       self.assertAllClose(np.array([0.1, 0.2]), v0_val)
       self.assertAllClose(np.array([3.91, 2.82]), v1_val)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -111,6 +116,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testProximalGradientDescentWithL1_L2(self):
     with self.cached_session() as sess:
       var0 = variables.Variable([1.0, 2.0])
@@ -174,6 +180,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
     v0_val, v1_val = self.evaluate([var0, var1])
     return v0_val, v1_val
 
+  @test_util.run_deprecated_v1
   def testEquivSparseGradientDescentwithoutRegularization(self):
     with self.cached_session():
       val0, val1 = self.applyOptimizer(
@@ -190,6 +197,7 @@ class ProximalGradientDescentOptimizerTest(test.TestCase):
     self.assertAllClose(val0, val2)
     self.assertAllClose(val1, val3)
 
+  @test_util.run_deprecated_v1
   def testEquivGradientDescentwithoutRegularization(self):
     with self.cached_session():
       val0, val1 = self.applyOptimizer(
diff --git a/tensorflow/python/training/quantize_training_test.py b/tensorflow/python/training/quantize_training_test.py
index 07fd488563..62e783f200 100644
--- a/tensorflow/python/training/quantize_training_test.py
+++ b/tensorflow/python/training/quantize_training_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -52,6 +53,7 @@ class PywrapQuantizeTrainingTest(test.TestCase):
 
   # Test that save/restoring works for EMA variables generated in the
   # quantized training rewrite.
+  @test_util.run_deprecated_v1
   def testQuantizedSaveRestore(self):
     save_path = os.path.join(self.get_temp_dir(), 'quantized_save_restore')
 
diff --git a/tensorflow/python/training/queue_runner_test.py b/tensorflow/python/training/queue_runner_test.py
index 65c2c13d8b..4113cecf55 100644
--- a/tensorflow/python/training/queue_runner_test.py
+++ b/tensorflow/python/training/queue_runner_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import variables
@@ -40,6 +41,7 @@ _MockOp = collections.namedtuple("MockOp", ["name"])
 
 class QueueRunnerTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testBasic(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -60,6 +62,7 @@ class QueueRunnerTest(test.TestCase):
       # The variable should be 3.
       self.assertEqual(3, self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testTwoOps(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -83,6 +86,7 @@ class QueueRunnerTest(test.TestCase):
       self.assertEqual(3, self.evaluate(var0))
       self.assertEqual(30, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testExceptionsCaptured(self):
     with self.cached_session() as sess:
       queue = data_flow_ops.FIFOQueue(10, dtypes.float32)
@@ -99,6 +103,7 @@ class QueueRunnerTest(test.TestCase):
       self.assertTrue("Operation not in the graph" in str(exceptions[0]))
       self.assertTrue("Operation not in the graph" in str(exceptions[1]))
 
+  @test_util.run_deprecated_v1
   def testRealDequeueEnqueue(self):
     with self.cached_session() as sess:
       q0 = data_flow_ops.FIFOQueue(3, dtypes.float32)
@@ -127,6 +132,7 @@ class QueueRunnerTest(test.TestCase):
       with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed"):
         self.evaluate(dequeue1)
 
+  @test_util.run_deprecated_v1
   def testRespectCoordShouldStop(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -151,6 +157,7 @@ class QueueRunnerTest(test.TestCase):
       # The variable should be 0.
       self.assertEqual(0, self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testRequestStopOnException(self):
     with self.cached_session() as sess:
       queue = data_flow_ops.FIFOQueue(10, dtypes.float32)
@@ -163,6 +170,7 @@ class QueueRunnerTest(test.TestCase):
       with self.assertRaisesRegexp(ValueError, "Operation not in the graph"):
         coord.join()
 
+  @test_util.run_deprecated_v1
   def testGracePeriod(self):
     with self.cached_session() as sess:
       # The enqueue will quickly block.
@@ -180,6 +188,7 @@ class QueueRunnerTest(test.TestCase):
       # the queue to be closed and the enqueue to terminate.
       coord.join(stop_grace_period_secs=1.0)
 
+  @test_util.run_deprecated_v1
   def testMultipleSessions(self):
     with self.cached_session() as sess:
       with session.Session() as other_sess:
@@ -195,6 +204,7 @@ class QueueRunnerTest(test.TestCase):
         other_threads = qr.create_threads(other_sess, coord=coord)
         self.assertEqual(len(threads), len(other_threads))
 
+  @test_util.run_deprecated_v1
   def testIgnoreMultiStarts(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -211,6 +221,7 @@ class QueueRunnerTest(test.TestCase):
       new_threads = qr.create_threads(sess, coord=coord)
       self.assertEqual([], new_threads)
 
+  @test_util.run_deprecated_v1
   def testThreads(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -238,6 +249,7 @@ class QueueRunnerTest(test.TestCase):
       self.assertEqual(1, len(exceptions))
       self.assertTrue("Operation not in the graph" in str(exceptions[0]))
 
+  @test_util.run_deprecated_v1
   def testName(self):
     with ops.name_scope("scope"):
       queue = data_flow_ops.FIFOQueue(10, dtypes.float32, name="queue")
@@ -247,6 +259,7 @@ class QueueRunnerTest(test.TestCase):
     self.assertEqual(
         1, len(ops.get_collection(ops.GraphKeys.QUEUE_RUNNERS, "scope")))
 
+  @test_util.run_deprecated_v1
   def testStartQueueRunners(self):
     # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
     zero64 = constant_op.constant(0, dtype=dtypes.int64)
@@ -265,6 +278,7 @@ class QueueRunnerTest(test.TestCase):
       # The variable should be 3.
       self.assertEqual(3, self.evaluate(var))
 
+  @test_util.run_deprecated_v1
   def testStartQueueRunnersRaisesIfNotASession(self):
     zero64 = constant_op.constant(0, dtype=dtypes.int64)
     var = variables.VariableV1(zero64)
@@ -278,6 +292,7 @@ class QueueRunnerTest(test.TestCase):
       with self.assertRaisesRegexp(TypeError, "tf.Session"):
         queue_runner_impl.start_queue_runners("NotASession")
 
+  @test_util.run_deprecated_v1
   def testStartQueueRunnersIgnoresMonitoredSession(self):
     zero64 = constant_op.constant(0, dtype=dtypes.int64)
     var = variables.VariableV1(zero64)
@@ -292,6 +307,7 @@ class QueueRunnerTest(test.TestCase):
           monitored_session.MonitoredSession())
       self.assertFalse(threads)
 
+  @test_util.run_deprecated_v1
   def testStartQueueRunnersNonDefaultGraph(self):
     # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
     graph = ops.Graph()
diff --git a/tensorflow/python/training/rmsprop_test.py b/tensorflow/python/training/rmsprop_test.py
index a9b8954e39..8f029d5310 100644
--- a/tensorflow/python/training/rmsprop_test.py
+++ b/tensorflow/python/training/rmsprop_test.py
@@ -89,6 +89,7 @@ class RMSPropOptimizerTest(test.TestCase):
       var_t[gindex] = var[gindex] - mom_t[gindex]
     return var_t, mg_t, rms_t, mom_t
 
+  @test_util.run_deprecated_v1
   def testDense(self):
     # TODO(yori): Use ParameterizedTest when available
     for (dtype, learning_rate, decay, momentum,
@@ -164,6 +165,7 @@ class RMSPropOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariable(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -187,6 +189,7 @@ class RMSPropOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testMinimizeSparseResourceVariableCentered(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session():
@@ -210,6 +213,7 @@ class RMSPropOptimizerTest(test.TestCase):
                                            self.evaluate(var0),
                                            atol=0.01)
 
+  @test_util.run_deprecated_v1
   def testSparse(self):
     # TODO(yori): Use ParameterizedTest when available
     for (dtype, learning_rate, decay,
@@ -286,6 +290,7 @@ class RMSPropOptimizerTest(test.TestCase):
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testWithoutMomentum(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with test_util.use_gpu():
@@ -354,6 +359,7 @@ class RMSPropOptimizerTest(test.TestCase):
                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0))
             ]), self.evaluate(var1))
 
+  @test_util.run_deprecated_v1
   def testWithMomentum(self):
     for dtype in [dtypes.half, dtypes.float32]:
       with test_util.use_gpu():
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 6b2177b0bb..40b7c0d5a7 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -170,6 +170,7 @@ class SaverTest(test.TestCase):
   def testResourceBasic(self):
     self.basicSaveRestore(resource_variable_ops.ResourceVariable)
 
+  @test_util.run_deprecated_v1
   def testResourceColocation(self):
     partitioner = partitioned_variables.fixed_size_partitioner(num_shards=2)
     with ops_lib.device("/job:ps/device:GPU:0"):
@@ -300,6 +301,7 @@ class SaverTest(test.TestCase):
             not op.name.startswith("saver2/save/"))]
     self.assertEqual(ops_in_saver2_scope_but_not_save_scope, [])
 
+  @test_util.run_deprecated_v1
   def testSaveCopyRestoreWithSaveRelativePaths(self):
     """Save, copy checkpoint dir and restore from copied dir.
 
@@ -369,6 +371,7 @@ class SaverTest(test.TestCase):
       self.assertEqual(b"k1", v2.keys().eval())
       self.assertEqual(30.0, v2.values().eval())
 
+  @test_util.run_deprecated_v1
   def testFilenameTensor(self):
     v0 = variables.VariableV1(0, name="v0")
     filename = b"somerandomfilename"
@@ -387,6 +390,7 @@ class SaverTest(test.TestCase):
             ValueError, "The passed save_path is not a valid checkpoint:"):
           save.restore(sess, "invalid path")
 
+  @test_util.run_deprecated_v1
   def testInt64(self):
     save_path = os.path.join(self.get_temp_dir(), "int64")
 
@@ -462,6 +466,7 @@ class SaverTest(test.TestCase):
       # Verify non-duplicate names work.
       saver_module.Saver({"v0": v0, "v2": v2.saveable})
 
+  @test_util.run_deprecated_v1
   def testBasicsWithListOfVariables(self):
     save_path = os.path.join(self.get_temp_dir(), "basics_with_list")
 
@@ -557,6 +562,7 @@ class SaverTest(test.TestCase):
     # The cached readers should know to re-read the file.
     self._SaveAndLoad("var1", 1.1, 2.2, save_path)
 
+  @test_util.run_deprecated_v1
   def testAllowEmpty(self):
     save_path = os.path.join(self.get_temp_dir(), "allow_empty")
     with self.cached_session() as sess:
@@ -661,6 +667,7 @@ class SaverTest(test.TestCase):
       self.assertAllClose(1.0, one.eval())
       self.assertAllClose([2.0, 2.0, 2.0], twos.eval())
 
+  @test_util.run_deprecated_v1
   def testReshape(self):
     save_path = os.path.join(self.get_temp_dir(), "variables_reshape")
     with session.Session("", graph=ops_lib.Graph()) as sess:
@@ -719,6 +726,7 @@ class SaverTest(test.TestCase):
   def testSaveWithGlobalStepWithPadding(self):
     self.testSaveWithGlobalStep(pad_step_number=True)
 
+  @test_util.run_deprecated_v1
   def testSaveToNonexistingPath(self):
     file_io.write_string_to_file(
         os.path.join(self.get_temp_dir(), "actually_a_file"), "")
@@ -761,6 +769,7 @@ class SaverTest(test.TestCase):
         error_msg_template = "Parent directory of {} doesn't exist, can't save."
         self.assertEqual(error_msg_template.format(save_path), str(exc))
 
+  @test_util.run_deprecated_v1
   def testSaveToURI(self):
     # ParseURI functions don't work on Windows yet.
     # TODO(jhseu): Remove this check when it works.
@@ -982,6 +991,7 @@ class SaveRestoreShardedTest(test.TestCase):
           checkpoint_management.latest_checkpoint(self.get_temp_dir()),
           os.path.join(self.get_temp_dir(), "sharded_basics"))
 
+  @test_util.run_deprecated_v1
   def testSaverDef(self):
     with self.cached_session():
       v0 = variables.VariableV1(123, name="v0")
@@ -1087,9 +1097,11 @@ class SaveRestoreShardedTest(test.TestCase):
               num_shards=3))
       self.assertAllEqual(saved_full, restored_full)
 
+  @test_util.run_deprecated_v1
   def testPartitionedVariable(self):
     self._testPartitionedVariables(use_resource=False)
 
+  @test_util.run_deprecated_v1
   def testPartitionedResourceVariable(self):
     self._testPartitionedVariables(use_resource=True)
 
@@ -1184,6 +1196,7 @@ class MaxToKeepTest(test.TestCase):
       # Deleted by the first helper.
       self.assertFalse(checkpoint_management.checkpoint_exists(s3))
 
+  @test_util.run_deprecated_v1
   def testNonSharded(self):
     save_dir = self._get_test_dir("max_to_keep_non_sharded")
 
@@ -1421,6 +1434,7 @@ class MaxToKeepTest(test.TestCase):
       self.assertTrue(
           gfile.Exists(checkpoint_management.meta_graph_filename(s3)))
 
+  @test_util.run_deprecated_v1
   def testNoMaxToKeep(self):
     save_dir = self._get_test_dir("no_max_to_keep")
     save_dir2 = self._get_test_dir("max_to_keep_0")
@@ -1449,6 +1463,7 @@ class MaxToKeepTest(test.TestCase):
       self.assertEqual([], save2.last_checkpoints)
       self.assertTrue(checkpoint_management.checkpoint_exists(s2))
 
+  @test_util.run_deprecated_v1
   def testNoMetaGraph(self):
     save_dir = self._get_test_dir("no_meta_graph")
 
@@ -1472,6 +1487,7 @@ class KeepCheckpointEveryNHoursTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   @test.mock.patch.object(saver_module, "time")
+  @test_util.run_deprecated_v1
   def testNonSharded(self, mock_time):
     save_dir = self._get_test_dir("keep_checkpoint_every_n_hours")
 
@@ -1591,6 +1607,7 @@ class SaveRestoreWithVariableNameMap(test.TestCase):
       self.assertEqual(20.0, self.evaluate(v1))
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_deprecated_v1
   def testNonReshapeResourceVariable(self):
     self._testNonReshape(resource_variable_ops.ResourceVariable)
 
@@ -1605,6 +1622,7 @@ class MetaGraphTest(test.TestCase):
     gfile.MakeDirs(test_dir)
     return test_dir
 
+  @test_util.run_deprecated_v1
   def testAddCollectionDef(self):
     test_dir = self._get_test_dir("good_collection")
     filename = os.path.join(test_dir, "metafile")
@@ -1754,11 +1772,13 @@ class MetaGraphTest(test.TestCase):
       v1 = sess.graph.get_tensor_by_name("v1:0")
       self.assertEqual(11.0, v1.eval())
 
+  @test_util.run_deprecated_v1
   def testMultiSaverCollection(self):
     test_dir = self._get_test_dir("saver_collection")
     self._testMultiSaverCollectionSave(test_dir)
     self._testMultiSaverCollectionRestore(test_dir)
 
+  @test_util.run_deprecated_v1
   def testClearExtraneousSavers(self):
     test_dir = self._get_test_dir("clear_extraneous_savers")
     filename = os.path.join(test_dir, "metafile")
@@ -1813,6 +1833,7 @@ class MetaGraphTest(test.TestCase):
       self.assertEqual(33, len(meta_graph_def0.graph_def.node))
       self.assertEqual(21, len(meta_graph_def1.graph_def.node))
 
+  @test_util.run_deprecated_v1
   def testBinaryAndTextFormat(self):
     test_dir = self._get_test_dir("binary_and_text")
     filename = os.path.join(test_dir, "metafile")
@@ -1845,6 +1866,7 @@ class MetaGraphTest(test.TestCase):
                                                lambda e: "does not exist"):
         saver_module.import_meta_graph(filename)
 
+  @test_util.run_deprecated_v1
   def testSliceVariable(self):
     test_dir = self._get_test_dir("slice_saver")
     filename = os.path.join(test_dir, "metafile")
@@ -1985,6 +2007,7 @@ class MetaGraphTest(test.TestCase):
       train_op = ops_lib.get_collection("train_op")[0]
       self.evaluate(train_op)
 
+  @test_util.run_deprecated_v1
   def testGraphExtension(self):
     test_dir = self._get_test_dir("graph_extension")
     self._testGraphExtensionSave(test_dir)
@@ -2070,6 +2093,7 @@ class MetaGraphTest(test.TestCase):
       return i + 1, x + r
     self._testWhileLoopAndGradientSerDes(body)
 
+  @test_util.run_deprecated_v1
   def testNestedControlFlowSerDes(self):
     # Test while loop in a cond in a while loop.
     # pylint: disable=g-long-lambda
@@ -2098,6 +2122,7 @@ class MetaGraphTest(test.TestCase):
                                       lambda: math_ops.multiply(x, -1.0))))
     # pylint: enable=g-long-lambda
 
+  @test_util.run_deprecated_v1
   def testStrippedOpListDef(self):
     with self.cached_session():
       # Creates a graph.
@@ -2135,6 +2160,7 @@ class MetaGraphTest(test.TestCase):
         self.assertEqual(o.summary, "")
         self.assertEqual(o.description, "")
 
+  @test_util.run_deprecated_v1
   def testStripDefaultValuedAttrs(self):
     """Verifies that default valued attrs are stripped, unless disabled."""
 
@@ -2171,6 +2197,7 @@ class MetaGraphTest(test.TestCase):
       self.assertIn("T", node_def.attr)
       self.assertIn("Tout", node_def.attr)
 
+  @test_util.run_deprecated_v1
   def testImportIntoNamescope(self):
     # Test that we can import a meta graph into a namescope.
     test_dir = self._get_test_dir("import_into_namescope")
@@ -2241,6 +2268,7 @@ class MetaGraphTest(test.TestCase):
           filename + ".meta", graph=graph_2, import_scope="my_scope")
       self.assertIsInstance(new_saver_3, saver_module.Saver)
 
+  @test_util.run_deprecated_v1
   def testImportIntoImplicitNamescope(self):
     # Test that we can import a meta graph into an implicit namescope.
     test_dir = self._get_test_dir("import_into_namescope")
@@ -2363,6 +2391,7 @@ class CheckpointReaderTest(test.TestCase):
 
   _WRITE_VERSION = saver_pb2.SaverDef.V1
 
+  @test_util.run_deprecated_v1
   def testDebugString(self):
     # Builds a graph.
     v0 = variables.VariableV1(
@@ -2594,6 +2623,7 @@ class ScopedGraphTest(test.TestCase):
 
   # Verifies that we can save the subgraph under "hidden1" and restore it
   # into "new_hidden1" in the new graph.
+  @test_util.run_deprecated_v1
   def testScopedSaveAndRestore(self):
     test_dir = self._get_test_dir("scoped_export_import")
     ckpt_filename = "ckpt"
@@ -2603,6 +2633,7 @@ class ScopedGraphTest(test.TestCase):
 
   # Verifies that we can copy the subgraph under "hidden1" and copy it
   # to different name scope in the same graph or different graph.
+  @test_util.run_deprecated_v1
   def testCopyScopedGraph(self):
     test_dir = self._get_test_dir("scoped_copy")
     saver0_ckpt = os.path.join(test_dir, "saver0.ckpt")
@@ -2659,6 +2690,7 @@ class ScopedGraphTest(test.TestCase):
       saver3.restore(sess, saver0_ckpt)
       self.assertAllClose(expected, sess.run("new_hidden1/relu:0"))
 
+  @test_util.run_deprecated_v1
   def testExportGraphDefWithScope(self):
     test_dir = self._get_test_dir("export_graph_def")
     saver0_ckpt = os.path.join(test_dir, "saver0.ckpt")
@@ -2695,6 +2727,7 @@ class ScopedGraphTest(test.TestCase):
       saver3.restore(sess, saver0_ckpt)
       self.assertAllClose(expected, sess.run("new_hidden1/relu:0"))
 
+  @test_util.run_deprecated_v1
   def testSerializeSaverWithScope(self):
     test_dir = self._get_test_dir("export_graph_def")
     saver1_ckpt = os.path.join(test_dir, "saver1.ckpt")
@@ -2955,6 +2988,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
       # exception" block in Python 3.
       self.assertNotIn("NewCheckpointReader", cs.exception.message)
 
+  @test_util.run_deprecated_v1
   def testGraphChangedForRestoreErrorRaised(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
@@ -2976,6 +3010,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
             "a mismatch between the current graph and the graph"):
           a_saver.restore(sess=sess, save_path=save_path)
 
+  @test_util.run_deprecated_v1
   def testLoadFromObjectBasedGraph(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
diff --git a/tensorflow/python/training/server_lib_multiple_containers_test.py b/tensorflow/python/training/server_lib_multiple_containers_test.py
index f599e9b55b..fb6118942b 100644
--- a/tensorflow/python/training/server_lib_multiple_containers_test.py
+++ b/tensorflow/python/training/server_lib_multiple_containers_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.client import session
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
@@ -33,6 +34,7 @@ class MultipleContainersTest(test.TestCase):
   # TODO(b/34465411): Starting multiple servers with different configurations
   # in the same test is flaky. Move this test case back into
   # "server_lib_test.py" when this is no longer the case.
+  @test_util.run_deprecated_v1
   def testMultipleContainers(self):
     with ops.container("test0"):
       v0 = variables.Variable(1.0, name="v0")
diff --git a/tensorflow/python/training/server_lib_same_variables_clear_container_test.py b/tensorflow/python/training/server_lib_same_variables_clear_container_test.py
index 3a5eb712c6..e0ab21bbd9 100644
--- a/tensorflow/python/training/server_lib_same_variables_clear_container_test.py
+++ b/tensorflow/python/training/server_lib_same_variables_clear_container_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.client import session
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
@@ -32,6 +33,7 @@ class SameVariablesClearContainerTest(test.TestCase):
   # TODO(b/34465411): Starting multiple servers with different configurations
   # in the same test is flaky. Move this test case back into
   # "server_lib_test.py" when this is no longer the case.
+  @test_util.run_deprecated_v1
   def testSameVariablesClearContainer(self):
     # Starts two servers with different names so they map to different
     # resource "containers".
diff --git a/tensorflow/python/training/server_lib_same_variables_clear_test.py b/tensorflow/python/training/server_lib_same_variables_clear_test.py
index 4682f1ab84..7b147af6c5 100644
--- a/tensorflow/python/training/server_lib_same_variables_clear_test.py
+++ b/tensorflow/python/training/server_lib_same_variables_clear_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.client import session
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -32,6 +33,7 @@ class SameVariablesClearTest(test.TestCase):
   # TODO(b/34465411): Starting multiple servers with different configurations
   # in the same test is flaky. Move this test case back into
   # "server_lib_test.py" when this is no longer the case.
+  @test_util.run_deprecated_v1
   def testSameVariablesClear(self):
     server = server_lib.Server.create_local_server()
 
diff --git a/tensorflow/python/training/server_lib_same_variables_no_clear_test.py b/tensorflow/python/training/server_lib_same_variables_no_clear_test.py
index 5aa7f45c2b..1b2d588f44 100644
--- a/tensorflow/python/training/server_lib_same_variables_no_clear_test.py
+++ b/tensorflow/python/training/server_lib_same_variables_no_clear_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -33,6 +34,7 @@ class SameVariablesNoClearTest(test.TestCase):
   # TODO(b/34465411): Starting multiple servers with different configurations
   # in the same test is flaky. Move this test case back into
   # "server_lib_test.py" when this is no longer the case.
+  @test_util.run_deprecated_v1
   def testSameVariablesNoClear(self):
     server = server_lib.Server.create_local_server()
 
diff --git a/tensorflow/python/training/server_lib_sparse_job_test.py b/tensorflow/python/training/server_lib_sparse_job_test.py
index 8c2745b51a..93b06e6216 100644
--- a/tensorflow/python/training/server_lib_sparse_job_test.py
+++ b/tensorflow/python/training/server_lib_sparse_job_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 
@@ -30,6 +31,7 @@ class SparseJobTest(test.TestCase):
   # TODO(b/34465411): Starting multiple servers with different configurations
   # in the same test is flaky. Move this test case back into
   # "server_lib_test.py" when this is no longer the case.
+  @test_util.run_deprecated_v1
   def testSparseJob(self):
     server = server_lib.Server({"local": {37: "localhost:0"}})
     with ops.device("/job:local/task:37"):
diff --git a/tensorflow/python/training/session_manager_test.py b/tensorflow/python/training/session_manager_test.py
index 2b5c3b01de..4294ffa851 100644
--- a/tensorflow/python/training/session_manager_test.py
+++ b/tensorflow/python/training/session_manager_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import variables
@@ -68,6 +69,7 @@ class SessionManagerTest(test.TestCase):
           "", init_fn=lambda sess: sess.run(v.initializer))
       self.assertAllClose([125], sess.run(v))
 
+  @test_util.run_deprecated_v1
   def testPrepareSessionFails(self):
     checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session")
     checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2")
@@ -152,6 +154,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
       self.assertEquals(1, sess.run(v))
 
+  @test_util.run_deprecated_v1
   def testRecoverSession(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session")
@@ -206,6 +209,7 @@ class SessionManagerTest(test.TestCase):
               variables.global_variables()),
           local_init_op=None)
 
+  @test_util.run_deprecated_v1
   def testRecoverSessionWithReadyForLocalInitOp(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(self.get_temp_dir(),
@@ -259,6 +263,7 @@ class SessionManagerTest(test.TestCase):
       self.assertEquals(1, sess.run(v))
       self.assertEquals(1, sess.run(w))
 
+  @test_util.run_deprecated_v1
   def testRecoverSessionWithReadyForLocalInitOpFailsToReadyLocal(self):
     # We use ready_for_local_init_op=tf.report_uninitialized_variables(),
     # which causes recover_session to not run local_init_op, and to return
@@ -315,6 +320,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("w:0")).eval(session=sess))
       self.assertEquals(1, sess.run(v))
 
+  @test_util.run_deprecated_v1
   def testRecoverSessionNoChkptStillRunsLocalInitOp(self):
     # This test checks for backwards compatibility.
     # In particular, we continue to ensure that recover_session will execute
@@ -343,6 +349,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("w:0")).eval(session=sess))
       self.assertEquals(1, sess.run(w))
 
+  @test_util.run_deprecated_v1
   def testRecoverSessionFailsStillRunsLocalInitOp(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(
@@ -386,6 +393,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("w:0")).eval(session=sess))
       self.assertEquals(1, sess.run(w))
 
+  @test_util.run_deprecated_v1
   def testWaitForSessionLocalInit(self):
     server = server_lib.Server.create_local_server()
     with ops.Graph().as_default() as graph:
@@ -437,6 +445,7 @@ class SessionManagerTest(test.TestCase):
         # because of overly restrictive ready_for_local_init_op
         sm.wait_for_session("", max_wait_secs=3)
 
+  @test_util.run_deprecated_v1
   def testWaitForSessionInsufficientReadyForLocalInitCheck(self):
     with ops.Graph().as_default() as graph:
       v = variables.VariableV1(1, name="v")
@@ -454,6 +463,7 @@ class SessionManagerTest(test.TestCase):
                                  "Session was not ready after waiting.*"):
       sm.wait_for_session("", max_wait_secs=3)
 
+  @test_util.run_deprecated_v1
   def testPrepareSessionWithReadyForLocalInitOp(self):
     with ops.Graph().as_default():
       v = variables.VariableV1(1, name="v")
@@ -493,6 +503,7 @@ class SessionManagerTest(test.TestCase):
       self.assertEquals(1, sess.run(w))
       self.assertEquals(3, sess.run(x))
 
+  @test_util.run_deprecated_v1
   def testPrepareSessionWithPartialInitOp(self):
     with ops.Graph().as_default():
       v = variables.VariableV1(1, name="v")
@@ -559,6 +570,7 @@ class SessionManagerTest(test.TestCase):
       self.assertEquals(1, sess.run(w_res))
       self.assertEquals(3, sess.run(x_res))
 
+  @test_util.run_deprecated_v1
   def testPrepareSessionWithCyclicInitializer(self):
     # Regression test. Previously Variable._build_initializer_expr would enter
     # into an infinite recursion when the variable's initial_value involved
@@ -632,6 +644,7 @@ class SessionManagerTest(test.TestCase):
           "Init operations did not make model ready for local_init"):
         sm2.prepare_session("", init_op=None)
 
+  @test_util.run_deprecated_v1
   def testPrepareSessionWithInsufficientReadyForLocalInitCheck(self):
     with ops.Graph().as_default():
       v = variables.VariableV1(1, name="v")
@@ -684,6 +697,7 @@ class ObsoleteSessionManagerTest(test.TestCase):
           "", init_fn=lambda sess: sess.run(v.initializer))
       self.assertAllClose([125], sess.run(v))
 
+  @test_util.run_deprecated_v1
   def testPrepareSessionFails(self):
     checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session")
     checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2")
@@ -745,6 +759,7 @@ class ObsoleteSessionManagerTest(test.TestCase):
           variables.is_variable_initialized(
               sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
 
+  @test_util.run_deprecated_v1
   def testRecoverSession(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session")
diff --git a/tensorflow/python/training/slot_creator_test.py b/tensorflow/python/training/slot_creator_test.py
index 382c15bb55..1f26aaa434 100644
--- a/tensorflow/python/training/slot_creator_test.py
+++ b/tensorflow/python/training/slot_creator_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
@@ -31,6 +32,7 @@ from tensorflow.python.training import slot_creator
 
 class SlotCreatorTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testCreateSlotFromVariable(self):
     with self.cached_session():
       v = variables.Variable([1.0, 2.5], name="var")
@@ -43,6 +45,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float32, slot.dtype.base_dtype)
       self.assertAllEqual([1.0, 2.5], self.evaluate(slot))
 
+  @test_util.run_deprecated_v1
   def testCreateSlotFromTensor(self):
     with self.cached_session():
       v = constant_op.constant([1.0, 2.5], name="const")
@@ -55,6 +58,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float32, slot.dtype.base_dtype)
       self.assertAllEqual([2.0, 5.0], self.evaluate(slot))
 
+  @test_util.run_deprecated_v1
   def testCreateZerosSlotFromVariable(self):
     with self.cached_session():
       v = variables.Variable([1.0, 2.5], name="var")
@@ -69,6 +73,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
       self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
+  @test_util.run_deprecated_v1
   def testCreateZerosSlotFromDynamicShapedVariable(self):
     with self.cached_session():
       dyn_shape = constant_op.constant([2], dtype=dtypes.int32)
@@ -90,6 +95,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
       self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
+  @test_util.run_deprecated_v1
   def testCreateZerosSlotFromTensor(self):
     with self.cached_session():
       v = constant_op.constant([1.0, 2.5], name="const")
@@ -103,6 +109,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float32, slot.dtype.base_dtype)
       self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
+  @test_util.run_deprecated_v1
   def testCreateZerosSlotFromDynamicShapedTensor(self):
     with self.cached_session():
       v = random_ops.random_uniform([2], dtype=dtypes.float64)
@@ -118,6 +125,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
       self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
+  @test_util.run_deprecated_v1
   def testCreateSlotFromVariableRespectsScope(self):
     # See discussion on #2740.
     with self.cached_session():
diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py
index 9dc88d78cc..f6505acc9a 100644
--- a/tensorflow/python/training/supervisor_test.py
+++ b/tensorflow/python/training/supervisor_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import parsing_ops
@@ -420,6 +421,7 @@ class SupervisorTest(test.TestCase):
       with self.assertRaisesRegexp(RuntimeError, "requires a summary writer"):
         sv.summary_computed(sess, sess.run(summ))
 
+  @test_util.run_deprecated_v1
   def testLogdirButExplicitlyNoSummaryWriter(self):
     logdir = self._test_dir("explicit_no_summary_writer")
     with ops.Graph().as_default():
@@ -505,6 +507,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir="", session_manager=sm)
       sv.prepare_or_wait_for_session("")
 
+  @test_util.run_deprecated_v1
   def testInitOp(self):
     logdir = self._test_dir("default_init_op")
     with ops.Graph().as_default():
@@ -514,6 +517,7 @@ class SupervisorTest(test.TestCase):
       self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
       sv.stop()
 
+  @test_util.run_deprecated_v1
   def testInitFn(self):
     logdir = self._test_dir("default_init_op")
     with ops.Graph().as_default():
@@ -527,6 +531,7 @@ class SupervisorTest(test.TestCase):
       self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
       sv.stop()
 
+  @test_util.run_deprecated_v1
   def testInitOpWithFeedDict(self):
     logdir = self._test_dir("feed_dict_init_op")
     with ops.Graph().as_default():
@@ -540,6 +545,7 @@ class SupervisorTest(test.TestCase):
       self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
       sv.stop()
 
+  @test_util.run_deprecated_v1
   def testReadyForLocalInitOp(self):
     server = server_lib.Server.create_local_server()
     logdir = self._test_dir("default_ready_for_local_init_op")
@@ -582,6 +588,7 @@ class SupervisorTest(test.TestCase):
     sv0.stop()
     sv1.stop()
 
+  @test_util.run_deprecated_v1
   def testReadyForLocalInitOpRestoreFromCheckpoint(self):
     server = server_lib.Server.create_local_server()
     logdir = self._test_dir("ready_for_local_init_op_restore")
@@ -713,6 +720,7 @@ class SupervisorTest(test.TestCase):
                                    "Variables not initialized: w"):
         sv.prepare_or_wait_for_session(server.target)
 
+  @test_util.run_deprecated_v1
   def testSetupFail(self):
     logdir = self._test_dir("setup_fail")
     with ops.Graph().as_default():
@@ -723,6 +731,7 @@ class SupervisorTest(test.TestCase):
       variables.VariableV1([1.0, 2.0, 3.0], name="v")
       supervisor.Supervisor(logdir=logdir, is_chief=False)
 
+  @test_util.run_deprecated_v1
   def testDefaultGlobalStep(self):
     logdir = self._test_dir("default_global_step")
     with ops.Graph().as_default():
@@ -732,6 +741,7 @@ class SupervisorTest(test.TestCase):
       self.assertEquals(287, sess.run(sv.global_step))
       sv.stop()
 
+  @test_util.run_deprecated_v1
   def testRestoreFromMetaGraph(self):
     logdir = self._test_dir("restore_from_meta_graph")
     with ops.Graph().as_default():
@@ -753,6 +763,7 @@ class SupervisorTest(test.TestCase):
   # This test is based on the fact that the standard services start
   # right away and get to run once before sv.stop() returns.
   # We still sleep a bit to make the test robust.
+  @test_util.run_deprecated_v1
   def testStandardServicesWithoutGlobalStep(self):
     logdir = self._test_dir("standard_services_without_global_step")
     # Create a checkpoint.
@@ -803,6 +814,7 @@ class SupervisorTest(test.TestCase):
 
   # Same as testStandardServicesNoGlobalStep but with a global step.
   # We should get a summary about the step time.
+  @test_util.run_deprecated_v1
   def testStandardServicesWithGlobalStep(self):
     logdir = self._test_dir("standard_services_with_global_step")
     # Create a checkpoint.
diff --git a/tensorflow/python/training/training_ops_test.py b/tensorflow/python/training/training_ops_test.py
index 929dd74ac6..51f49ca081 100644
--- a/tensorflow/python/training/training_ops_test.py
+++ b/tensorflow/python/training/training_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework.test_util import TensorFlowTestCase
 # Import resource_variable_ops for the variables-to-tensor implicit conversion.
 from tensorflow.python.ops import resource_variable_ops  # pylint: disable=unused-import
@@ -59,6 +60,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       self.assertShapeEqual(out, apply_sgd)
       self.assertAllCloseAccordingToType(x - alpha * delta, out)
 
+  @test_util.run_deprecated_v1
   def testApplyGradientDescent(self):
     for (dtype, use_gpu) in itertools.product(
         [np.float16, np.float32, np.float64], [False, True]):
@@ -127,6 +129,7 @@ class TrainingOpsTest(TensorFlowTestCase):
         self.assertAllClose(linear_update, self.evaluate(linear))
         self.assertAllClose(expected_out, out)
 
+  @test_util.run_deprecated_v1
   def testApplyAdagrad(self):
     for (dtype, use_gpu) in itertools.product(
         [np.float16, np.float32, np.float64], [False, True]):
@@ -136,6 +139,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       grad = np.arange(100).astype(dtype)
       self._testTypesForAdagrad(x, y, lr, grad, use_gpu)
 
+  @test_util.run_deprecated_v1
   def testApplyFtrl(self):
     for dtype in [np.float16, np.float32, np.float64]:
       x = np.arange(100).astype(dtype)
@@ -207,6 +211,7 @@ class TrainingOpsTest(TensorFlowTestCase):
         self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i],
                                            self.evaluate(accum)[index])
 
+  @test_util.run_deprecated_v1
   def testSparseApplyAdagrad(self):
     for (dtype, index_type) in itertools.product(
         [np.float16, np.float32, np.float64], [np.int32, np.int64]):
@@ -220,6 +225,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       indices = np.array([0, 2]).astype(index_type)
       self._testTypesForSparseAdagrad(x, y, lr, grad, indices)
 
+  @test_util.run_deprecated_v1
   def testSparseApplyAdagradDim1(self):
     for (dtype, index_type) in itertools.product(
         [np.float16, np.float32, np.float64], [np.int32, np.int64]):
@@ -233,6 +239,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       indices = np.array([0, 2]).astype(index_type)
       self._testTypesForSparseAdagrad(x, y, lr, grad, indices)
 
+  @test_util.run_deprecated_v1
   def testSparseApplyFtrlDim1(self):
     for (dtype, index_type) in itertools.product(
         [np.float16, np.float32, np.float64], [np.int32, np.int64]):
@@ -248,6 +255,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       indices = np.array([0, 2]).astype(index_type)
       self._testTypesForSparseFtrl(x, y, z, lr, grad, indices)
 
+  @test_util.run_deprecated_v1
   def testApplyAdam(self):
     for dtype, use_gpu in itertools.product(
         [np.float16, np.float32, np.float64], [False, True]):
diff --git a/tensorflow/python/training/training_util_test.py b/tensorflow/python/training/training_util_test.py
index ba64e785ac..3317008fce 100644
--- a/tensorflow/python/training/training_util_test.py
+++ b/tensorflow/python/training/training_util_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import monitored_session
@@ -46,6 +47,7 @@ class GlobalStepTest(test.TestCase):
     self.assertRaisesRegexp(TypeError, 'does not have integer type',
                             training_util.get_global_step, g)
 
+  @test_util.run_deprecated_v1
   def test_invalid_shape(self):
     with ops.Graph().as_default() as g:
       self.assertIsNone(training_util.get_global_step())
@@ -70,6 +72,7 @@ class GlobalStepTest(test.TestCase):
                               training_util.create_global_step, g)
       self._assert_global_step(training_util.create_global_step(ops.Graph()))
 
+  @test_util.run_deprecated_v1
   def test_get_global_step(self):
     with ops.Graph().as_default() as g:
       self.assertIsNone(training_util.get_global_step())
diff --git a/tensorflow/python/util/deprecation_test.py b/tensorflow/python/util/deprecation_test.py
index 34cbca52a1..035c416d79 100644
--- a/tensorflow/python/util/deprecation_test.py
+++ b/tensorflow/python/util/deprecation_test.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import deprecation
@@ -174,6 +175,7 @@ class DeprecationTest(test.TestCase):
                         set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_with_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -214,6 +216,7 @@ class DeprecationTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_with_one_line_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -239,6 +242,7 @@ class DeprecationTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_no_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -488,6 +492,7 @@ class DeprecatedArgsTest(test.TestCase):
       deprecation.deprecated_args(date, instructions, "missing")(_fn)
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_with_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -535,6 +540,7 @@ class DeprecatedArgsTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_with_one_line_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -565,6 +571,7 @@ class DeprecatedArgsTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_no_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -595,6 +602,7 @@ class DeprecatedArgsTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_varargs(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -615,6 +623,7 @@ class DeprecatedArgsTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_kwargs(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -635,6 +644,7 @@ class DeprecatedArgsTest(test.TestCase):
     self._assert_subset(set(["after " + date, instructions]), set(args[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_positional_and_named(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -660,6 +670,7 @@ class DeprecatedArgsTest(test.TestCase):
                         set(args2[1:]))
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_positional_and_named_with_ok_vals(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -692,6 +703,7 @@ class DeprecatedArgsTest(test.TestCase):
     self.assertEqual(0, mock_warning.call_count)
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_deprecated_args_once(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -708,6 +720,7 @@ class DeprecatedArgsTest(test.TestCase):
     self.assertEqual(1, mock_warning.call_count)
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_deprecated_multiple_args_once_each(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -752,6 +765,7 @@ class DeprecatedArgValuesTest(test.TestCase):
       deprecation.deprecated_arg_values(date, instructions)
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_with_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -804,6 +818,7 @@ class DeprecatedArgValuesTest(test.TestCase):
     self.assertEqual(2, mock_warning.call_count)
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_with_one_line_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
@@ -839,6 +854,7 @@ class DeprecatedArgValuesTest(test.TestCase):
     self.assertEqual(2, mock_warning.call_count)
 
   @test.mock.patch.object(logging, "warning", autospec=True)
+  @test_util.run_deprecated_v1
   def test_static_fn_no_doc(self, mock_warning):
     date = "2016-07-04"
     instructions = "This is how you update..."
diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py
index 997a3c5c36..d0d0c5f793 100644
--- a/tensorflow/python/util/nest_test.py
+++ b/tensorflow/python/util/nest_test.py
@@ -482,6 +482,7 @@ class NestTest(parameterized.TestCase, test.TestCase):
     self.assertEqual(nt.a[1][::-1], rev_nt.a[1])
     self.assertEqual(nt.b[::-1], rev_nt.b)
 
+  @test_util.run_deprecated_v1
   def testMapStructureOverPlaceholders(self):
     inp_a = (array_ops.placeholder(dtypes.float32, shape=[3, 4]),
              array_ops.placeholder(dtypes.float32, shape=[3, 7]))
diff --git a/tensorflow/python/util/tf_should_use_test.py b/tensorflow/python/util/tf_should_use_test.py
index cde67c4e4f..65d848cf2a 100644
--- a/tensorflow/python/util/tf_should_use_test.py
+++ b/tensorflow/python/util/tf_should_use_test.py
@@ -24,6 +24,7 @@ import gc
 import sys
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.util import tf_should_use
@@ -39,6 +40,7 @@ def reroute_error():
 
 class TfShouldUseTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testAddShouldUseWarningWhenNotUsed(self):
     c = constant_op.constant(0, name='blah0')
     def in_this_function():
@@ -52,6 +54,7 @@ class TfShouldUseTest(test.TestCase):
     self.assertIn('in_this_function', msg)
     self.assertFalse(gc.garbage)
 
+  @test_util.run_deprecated_v1
   def testAddShouldUseFatalWhenNotUsed(self):
     c = constant_op.constant(0, name='blah0')
     def in_this_function():
@@ -74,6 +77,7 @@ class TfShouldUseTest(test.TestCase):
     error.assert_not_called()
     fatal.assert_not_called()
 
+  @test_util.run_deprecated_v1
   def testAddShouldUseWarningWhenUsedWithAdd(self):
     def add(h):
       _ = h + 1
@@ -81,6 +85,7 @@ class TfShouldUseTest(test.TestCase):
     gc.collect()
     self.assertFalse(gc.garbage)
 
+  @test_util.run_deprecated_v1
   def testAddShouldUseWarningWhenUsedWithGetName(self):
     def get_name(h):
       _ = h.name
@@ -88,6 +93,7 @@ class TfShouldUseTest(test.TestCase):
     gc.collect()
     self.assertFalse(gc.garbage)
 
+  @test_util.run_deprecated_v1
   def testShouldUseResult(self):
     @tf_should_use.should_use_result
     def return_const(value):
@@ -101,6 +107,7 @@ class TfShouldUseTest(test.TestCase):
     gc.collect()
     self.assertFalse(gc.garbage)
 
+  @test_util.run_deprecated_v1
   def testShouldUseResultWhenNotReallyUsed(self):
     @tf_should_use.should_use_result
     def return_const(value):
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index cba6246fef..e7f23a1174 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -39,6 +39,7 @@ from google.protobuf import message
 from google.protobuf import text_format
 
 from tensorflow.python.lib.io import file_io
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import resource_loader
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
@@ -310,6 +311,7 @@ class ApiCompatibilityTest(test.TestCase):
         update_goldens=FLAGS.update_goldens,
         api_version=api_version)
 
+  @test_util.run_deprecated_v1
   def testAPIBackwardsCompatibility(self):
     api_version = 1
     golden_file_pattern = os.path.join(
@@ -328,6 +330,7 @@ class ApiCompatibilityTest(test.TestCase):
         'tensorflow.python.util.lazy_loader.LazyLoader'
         in str(type(tf.contrib)))
 
+  @test_util.run_deprecated_v1
   def testAPIBackwardsCompatibilityV1(self):
     api_version = 1
     golden_file_pattern = os.path.join(
-- 
GitLab


From 70eca712206b644acb2f089dca69b14f68bdac77 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Thu, 29 Nov 2018 15:33:27 -0800
Subject: [PATCH 1071/1554] Do not create unnecessary placeholders when cloning
 the model in DistributionStrategy in predict mode.

PiperOrigin-RevId: 223423532
---
 .../keras/engine/training_distributed.py      | 35 +++++++++++++------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index d16832318c..49050f0a97 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -485,7 +485,17 @@ def experimental_predict_loop(model, iterator, verbose=0, steps=None):
   ]
 
 
-def _clone_and_build_model(model, inputs=None, targets=None):
+def _custom_compile_for_predict(model):
+  """Custom compile for TPU predict mode."""
+  model.total_loss = None
+  model._fit_function = None
+  model._eval_function = None
+  model.train_function = None
+  model.test_function = None
+  model.predict_function = None
+
+
+def _clone_and_build_model(model, inputs=None, targets=None, mode=None):
   """Clone and build the given keras_model."""
   # We need to set the import here since we run into a circular dependency
   # error.
@@ -512,15 +522,18 @@ def _clone_and_build_model(model, inputs=None, targets=None):
 
   if isinstance(targets, tuple):
     targets = nest.flatten(targets)
-  cloned_model.compile(
-      optimizer,
-      model.loss,
-      metrics=metrics_module.clone_metrics(model._compile_metrics),
-      loss_weights=model.loss_weights,
-      sample_weight_mode=model.sample_weight_mode,
-      weighted_metrics=metrics_module.clone_metrics(
-          model._compile_weighted_metrics),
-      target_tensors=targets)
+  if mode == _Mode.PREDICT:
+    _custom_compile_for_predict(cloned_model)
+  else:
+    cloned_model.compile(
+        optimizer,
+        model.loss,
+        metrics=metrics_module.clone_metrics(model._compile_metrics),
+        loss_weights=model.loss_weights,
+        sample_weight_mode=model.sample_weight_mode,
+        weighted_metrics=metrics_module.clone_metrics(
+            model._compile_weighted_metrics),
+        target_tensors=targets)
   return cloned_model
 
 
@@ -529,7 +542,7 @@ def clone_model_on_replicas(model, strategy, make_callback_model=False,
   """Create a cloned model on each replica."""
   with strategy.scope():
     grouped_model = strategy.extended.call_for_each_replica(
-        _clone_and_build_model, args=(model, inputs, targets))
+        _clone_and_build_model, args=(model, inputs, targets, mode))
     if mode is _Mode.TRAIN:
       model._grouped_model_train = grouped_model
     elif mode is _Mode.TEST:
-- 
GitLab


From b4a07298f812aa15a94f8e498f284f8f780bbd20 Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Thu, 29 Nov 2018 15:34:13 -0800
Subject: [PATCH 1072/1554] Update canned estimator with EstimatorV2

PiperOrigin-RevId: 223423661
---
 .../v1/tensorflow.estimator.-baseline-classifier.pbtxt     | 1 -
 .../v1/tensorflow.estimator.-baseline-regressor.pbtxt      | 1 -
 .../golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt | 1 -
 ...rflow.estimator.-d-n-n-linear-combined-classifier.pbtxt | 1 -
 ...orflow.estimator.-d-n-n-linear-combined-regressor.pbtxt | 1 -
 .../golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt  | 1 -
 .../v1/tensorflow.estimator.-linear-classifier.pbtxt       | 1 -
 .../golden/v1/tensorflow.estimator.-linear-regressor.pbtxt | 1 -
 .../v2/tensorflow.estimator.-baseline-classifier.pbtxt     | 5 -----
 .../v2/tensorflow.estimator.-baseline-estimator.pbtxt      | 7 +------
 .../v2/tensorflow.estimator.-baseline-regressor.pbtxt      | 5 -----
 .../golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt | 5 -----
 .../golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt  | 7 +------
 ...rflow.estimator.-d-n-n-linear-combined-classifier.pbtxt | 5 -----
 ...orflow.estimator.-d-n-n-linear-combined-estimator.pbtxt | 7 +------
 ...orflow.estimator.-d-n-n-linear-combined-regressor.pbtxt | 5 -----
 .../golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt  | 5 -----
 .../v2/tensorflow.estimator.-linear-classifier.pbtxt       | 5 -----
 .../golden/v2/tensorflow.estimator.-linear-estimator.pbtxt | 7 +------
 .../golden/v2/tensorflow.estimator.-linear-regressor.pbtxt | 5 -----
 20 files changed, 4 insertions(+), 72 deletions(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
index 225742539d..ee3a72bfce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.BaselineClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifier\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
index 5c51767d56..3874b84d5a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.BaselineRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressor\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
index a142ca3290..b54133b294 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifier\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index 85a20828a0..5a1d85a9b1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifier\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index e05c7ce0a2..db4780e4c0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressor\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
index fc3b1d9813..a44e719099 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressor\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
index d213551c0b..2c8e82517b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.LinearClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifier\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifierV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
index 004dfccb3b..1bdc6124fe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-linear-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.LinearRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressor\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
index 22cbcf08f1..efe9e74697 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.BaselineClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineClassifierV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
index 38b27f735f..382d392f39 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-estimator.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.BaselineEstimator"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineEstimator\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineEstimatorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
index a965042b41..a7300bf06b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-baseline-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.BaselineRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.baseline.BaselineRegressorV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
index f6bd4d2121..a540085aba 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNClassifierV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
index 09e0d38192..d1b29d670a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-estimator.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNEstimator"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNEstimator\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNEstimatorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index 60627cc197..f6c3910a9f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedClassifierV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
index e311f96d3d..b78527279c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-estimator.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedEstimator"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedEstimator\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedEstimatorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index dc6aca25dd..9133f0d3b2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNLinearCombinedRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn_linear_combined.DNNLinearCombinedRegressorV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 7338abc1d9..a58d733302 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.DNNRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.dnn.DNNRegressorV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
index 6559c581fb..47de660a38 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-classifier.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.LinearClassifier"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearClassifierV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
index 2148374fde..66a127606a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-estimator.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.LinearEstimator"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearEstimator\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearEstimatorV2\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
index e6ea074ff8..5c094fe131 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-linear-regressor.pbtxt
@@ -1,7 +1,6 @@
 path: "tensorflow.estimator.LinearRegressor"
 tf_class {
   is_instance: "<class \'tensorflow_estimator.python.estimator.canned.linear.LinearRegressorV2\'>"
-  is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.Estimator\'>"
   is_instance: "<class \'tensorflow_estimator.python.estimator.estimator.EstimatorV2\'>"
   is_instance: "<type \'object\'>"
   member {
@@ -40,10 +39,6 @@ tf_class {
     name: "export_saved_model"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'experimental_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'infer\'], "
   }
-  member_method {
-    name: "export_savedmodel"
-    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
-  }
   member_method {
     name: "get_variable_names"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 57c274383a5edeffc475df5c97b0a9cd994722a3 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Thu, 29 Nov 2018 15:44:13 -0800
Subject: [PATCH 1073/1554] Fixed a bug in the AttrBuilder related experimental
 C APIs, where the string value `attr_name` passed into
 `TF_AttrBuilderSetType()` could get destroyed before a subsequent call to
 `TF_AttrBuilderCheckCanRunOnDevice()` on the same `builder`.

That is a use-after-free error, since `builder` stores a string_view version of
`attr_name`, and uses it in the latter API call.

PiperOrigin-RevId: 223425447
---
 tensorflow/c/c_api_experimental.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 0c593185db..3693cc8599 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -8799,6 +8799,10 @@ void TF_MakeInternalErrorStatus(TF_Status* status, const char* errMsg) {
 // This builder is used in the eager API to build a NodeDef.
 struct TF_AttrBuilder : public tensorflow::AttrBuilder {
   using tensorflow::AttrBuilder::AttrBuilder;
+  // The string buffers to make sure that any `attr_name` we pass into
+  // `builder->Set()` will outlive the subsequent
+  // `TF_AttrBuilderCheckCanRunOnDevice()` call(s) on the same `builder`.
+  std::set<std::string> attr_names;
 };
 
 TF_AttrBuilder* TF_NewAttrBuilder(const char* op_name) {
@@ -8809,13 +8813,15 @@ void TF_DeleteAttrBuilder(TF_AttrBuilder* builder) { delete builder; }
 
 void TF_AttrBuilderSetType(TF_AttrBuilder* builder, const char* attr_name,
                            TF_DataType value) {
-  builder->Set(attr_name, static_cast<tensorflow::DataType>(value));
+  auto iter = builder->attr_names.insert(attr_name).first;
+  builder->Set((*iter).c_str(), static_cast<tensorflow::DataType>(value));
 }
 
 void TF_AttrBuilderSetTypeList(TF_AttrBuilder* builder, const char* attr_name,
                                const TF_DataType* values, int num_values) {
+  auto iter = builder->attr_names.insert(attr_name).first;
   builder->Set(
-      attr_name,
+      (*iter).c_str(),
       tensorflow::gtl::ArraySlice<const tensorflow::DataType>(
           reinterpret_cast<const tensorflow::DataType*>(values), num_values));
 }
-- 
GitLab


From 5261e4dc3fed1ba6b209f203335767cf50797b42 Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama <sguada@google.com>
Date: Thu, 29 Nov 2018 16:10:20 -0800
Subject: [PATCH 1074/1554] Add testNestOutputs to tf.function

PiperOrigin-RevId: 223429927
---
 tensorflow/python/eager/function_test.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index e47c5083a3..d2cd407d80 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -147,6 +147,24 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     out = a_times_b(pair({'a': t}, {'b': t}))
     self.assertAllEqual(out, math_ops.matmul(t, t).numpy())
 
+  def testNestedOutputsGraphMode(self):
+    matmul = def_function.function(math_ops.matmul)
+
+    pair = collections.namedtuple('pair', ['a', 'b'])
+
+    # TODO(b/120222989) remove autograph=False.
+    @def_function.function(autograph=False)
+    def pairs_mul(pair_a, pair_b):
+      return pair(matmul(pair_a.a, pair_b.a), matmul(pair_a.b, pair_b.b))
+
+    a = constant_op.constant([[1.0, 2.0], [1.0, 2.0]])
+    b = constant_op.constant([[3.0, 4.0], [3.0, 4.0]])
+
+    out = pairs_mul(pair(a, b), pair(b, a))
+    expected = pair(math_ops.matmul(a, b).numpy(),
+                    math_ops.matmul(b, a).numpy())
+    self.assertAllClose(out, expected)
+
   def testGraphEagerIsolation(self):
 
     @function.defun
-- 
GitLab


From 0b87511ae234d3d54f06c7b12e8fcfa7361ecb48 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 29 Nov 2018 16:15:20 -0800
Subject: [PATCH 1075/1554] Add publicly available corpus and dictionary for
 decode_wav_fuzz.

PiperOrigin-RevId: 223430684
---
 tensorflow/core/kernels/fuzzing/BUILD           |   4 ++++
 .../02cc44cdfec1d9d0d0c66c5a5f40d3d20e4c4c3a    | Bin 0 -> 11564 bytes
 .../087e1d7fae1c1ddcbaa3b5f822a171ad15498186    | Bin 0 -> 146 bytes
 .../0f61c33027394a0f14d29dcd22f405cad943b7cf    | Bin 0 -> 29936 bytes
 .../10cdebea1659c21a0248f88654ae41f62786abf1    | Bin 0 -> 58657 bytes
 .../126e68def9fd973a100e0f66cadf09448a716b57    | Bin 0 -> 9209 bytes
 .../1275d41ebf8788ce3a949352e4bc654b04012da3    | Bin 0 -> 70 bytes
 .../1a7f1c407fb3864ddb559f88f373a21d1be51584    | Bin 0 -> 35640 bytes
 .../1c3e1c91f187f6bcea86f172ff5bbbd955a9654d    | Bin 0 -> 57 bytes
 .../300fe1e0a47543037cbf0243b6756c9aa48799c4    | Bin 0 -> 72238 bytes
 .../31ec5b0134bedcfe283f4978e6e65b7d35d5d4ad    | Bin 0 -> 26123 bytes
 .../4e7cbb27667bcfca92838aa8020749990013a9b1    | Bin 0 -> 66423 bytes
 .../585e469231d202812bfba8285fb30c8e31c857b9    | Bin 0 -> 70833 bytes
 .../58eab6bc2386e2ef43fe4f55cb6ad3611399d5de    | Bin 0 -> 22 bytes
 .../63448c6a9feb8c72b3e82af4d735ec2e62ddd328    | Bin 0 -> 58203 bytes
 .../6874d5b1c7a64b596c61f24877d422e89bebe58b    | Bin 0 -> 25086 bytes
 .../7501f79cb067da108020579ed654349c7933d22f    | Bin 0 -> 54715 bytes
 .../782051f8120182b860c7fe1b265179cfa2fe03fd    | Bin 0 -> 66439 bytes
 .../793feab2deb35e284a975f6527d76a8be5540fe6    | Bin 0 -> 61 bytes
 .../7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329    | Bin 0 -> 6150 bytes
 .../8210dc595a2652f2f812093b01e239e7918ea065    | Bin 0 -> 55329 bytes
 .../8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7    | Bin 0 -> 71409 bytes
 .../91d787a9298ddc015efa783a92c4bdba8af0d7de    | Bin 0 -> 44 bytes
 .../92c065286f956f086e977556358f6b54b12bcacc    | Bin 0 -> 315 bytes
 .../a35c9bb71792b60a13dea23a41b41847ad4b93d6    | Bin 0 -> 44 bytes
 .../a6ea960c7b4d42772888280277b26e645ceee904    | Bin 0 -> 22470 bytes
 .../aa526aa853333f0bb11804b5243df411452cecd2    | Bin 0 -> 21331 bytes
 .../ca533cd26c7ca6bf69e62351b265ded496fdf1d9    | Bin 0 -> 52741 bytes
 .../f38c61da15f2cb7a39ff02e69f0b00e99f37ec86    | Bin 0 -> 66555 bytes
 .../f88f1012473e6cfcc9b39b2552f682b2f73eff8c    | Bin 0 -> 54757 bytes
 .../fa79819c5de04bc06c69bec3fa7f2e982826ea2f    | Bin 0 -> 20280 bytes
 .../fce08de222896ac3a20657a3b4f42d5b6c54a96a    | Bin 0 -> 69522 bytes
 .../fuzzing/dictionaries/decode_wav.dict        |   4 ++++
 33 files changed, 8 insertions(+)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/02cc44cdfec1d9d0d0c66c5a5f40d3d20e4c4c3a
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/087e1d7fae1c1ddcbaa3b5f822a171ad15498186
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/0f61c33027394a0f14d29dcd22f405cad943b7cf
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/10cdebea1659c21a0248f88654ae41f62786abf1
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/126e68def9fd973a100e0f66cadf09448a716b57
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/1275d41ebf8788ce3a949352e4bc654b04012da3
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/1a7f1c407fb3864ddb559f88f373a21d1be51584
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/1c3e1c91f187f6bcea86f172ff5bbbd955a9654d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/300fe1e0a47543037cbf0243b6756c9aa48799c4
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/31ec5b0134bedcfe283f4978e6e65b7d35d5d4ad
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/4e7cbb27667bcfca92838aa8020749990013a9b1
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/585e469231d202812bfba8285fb30c8e31c857b9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/58eab6bc2386e2ef43fe4f55cb6ad3611399d5de
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/63448c6a9feb8c72b3e82af4d735ec2e62ddd328
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/6874d5b1c7a64b596c61f24877d422e89bebe58b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/7501f79cb067da108020579ed654349c7933d22f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/782051f8120182b860c7fe1b265179cfa2fe03fd
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/793feab2deb35e284a975f6527d76a8be5540fe6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/8210dc595a2652f2f812093b01e239e7918ea065
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/91d787a9298ddc015efa783a92c4bdba8af0d7de
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/92c065286f956f086e977556358f6b54b12bcacc
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/a35c9bb71792b60a13dea23a41b41847ad4b93d6
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/a6ea960c7b4d42772888280277b26e645ceee904
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/aa526aa853333f0bb11804b5243df411452cecd2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/ca533cd26c7ca6bf69e62351b265ded496fdf1d9
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/f38c61da15f2cb7a39ff02e69f0b00e99f37ec86
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/f88f1012473e6cfcc9b39b2552f682b2f73eff8c
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/fa79819c5de04bc06c69bec3fa7f2e982826ea2f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_wav/fce08de222896ac3a20657a3b4f42d5b6c54a96a
 create mode 100644 tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict

diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index bb92057bb6..72088e0de2 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -53,6 +53,10 @@ tf_oss_fuzz_dict("decode_png")
 
 tf_ops_fuzz_target_lib("decode_wav")
 
+tf_oss_fuzz_corpus("decode_wav")
+
+tf_oss_fuzz_dict("decode_wav")
+
 tf_ops_fuzz_target_lib("example_proto_fast_parsing")
 
 tf_ops_fuzz_target_lib("parse_tensor_op")
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/02cc44cdfec1d9d0d0c66c5a5f40d3d20e4c4c3a b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/02cc44cdfec1d9d0d0c66c5a5f40d3d20e4c4c3a
new file mode 100644
index 0000000000000000000000000000000000000000..2a65e0e2559eea919bdbb661370c679fdbdd7e26
GIT binary patch
literal 11564
zcmWIYbaPYDWnc(*40BD(Em06)U|?WmU}O+bXJAn9V_;xn5MW42EJ<Y0Wf%n{guws*
zBq*kt(!t>ha=rH-42ajwNW4;TDo0h0UpIbPRFfb)hz<w|5kVzEaf3@1A&y{ybc1*x
zeIPb57*#DOIMH=MRA7@qSBcDr=mM1!$f6K7$c+#ds3d{07#JXWaPtu|_*e)#K`fZN
zL2Q^Vko^$5Afg~XOb(TXsDg=MQwK5ugi&1&QVGHk{U8$J4iF1jB}5cvK1ddXL41gr
zAQn1?>G}Wv-~Yd0_~-wh|G)qL0^^_mfBydg#^3*c1H-TXzx@CF|I`1^Q2OKlPyavu
z|M36A|M&mj|9|)Y9h8Ky-u{2{|2<S5qVm)KFaN)SO#qqq_5Y9mzrp7J{{Q#?pZ_5D
zf-uCDj0`LctPE@nYz)i{EDUT691PqHTnwC0k_$}pGVn9-GVn3*GVn48Fz_(&Fz|sf
zKLbC509X{n5(KLfVBlxqW8h~HWZ(nqWoKYx;A8-q!^6PIz{bGNz{SADzyY?Ak>US;
zCI(QbGcf%94+<Sfs6ayG$Nz8tK_LMN2N3)7|F2MgeFTTX+y5W_fBygB|GWPm!9M=>
z|HuD7;PCzT9};KGU>iYpf_w#XmjHthg9w8dgE)gYgEWI27%DNSFeox8F{m<VFlaKU
zGiWeqFlaGoF=#UAFlaMqG3YVqGZ-=$GZ-@%G8i!EGU!2ReFg&tBL+hT9R^JX4F+`v
zH3k(1HLz|K24x0i1{DTX234?m>R@w0A}S0@P#F+QkwKn84vZxkBpGBFq!}a`q`;&E
zg9L*(g9w8Vg8+jtgDBX=A`BoX#305X!5{{f;{}H;I|CB~3j;F)69Y5D-~T`V{{pA1
z@BhF2|MLIS|F{2N|Nrp+&HrcrU;Kai|MCCl|DXJS`2XSmyZ`U}zy1Hl|C|4>|G)JA
z(*Mi<FZ{m%W?lGy?*F;}XaAr2fAar{|Hu9xg+dVP;Qs^v5B}f(|KR^4{}2A(^M4<h
zcj*7I|405G`G4&HVK900|MCAP|DXDQ`v1BAm;YY@n|l5Kt^ara-}`_2|DFF2{@?$9
z@BicfPyavt|Kk6v|1bW(`2X_%>;JF+zXRt0P@IEO{LlaY!D$zia{vEl1gB9}u%AIG
z6_k>=82G?p1WI*0;Is%LxfwVZn82xvnE|Aphk=`c6P$9n83Y&v!5EZgL8(EA0R(v%
z*crINsyG=q7(kc>9Hz_+91I)`><l~%TnwxX><p0f%)-FLz{&v1*C1Q|{{Icm$={(Q
zB#;05|NH-+|Df^`l-pRrrhrTU<poY~4g<Lfl<ycASQ)q(*cn*BrbF@z$XB2o#m^uN
zPOFmOQ~^r0Vhmyok_@2q0m9M@Vho}TG7Pc|atw+L$_&~JnhZ+dG^fU(4NkfGU=osQ
zb;0tW6b(zWkhBg;=epq3uEU_mpa)L%It&I3h77vk^bShtARVCGpw0kF&7ianN#me&
zti&M4AO(gJ&@{>m&*#kGwkHF_&;OsHB^oIAzWM*+|D*p8{y+Nv<p1OU5B}c)r^H+T
zZ~VXU|N8&y|F8bP@c;b(^Wby{N{gVhcl!T{|A+n`{D1KOq5p@$@Yw&;;52&X|Ed2c
z{-6AR^8e}or~aS$e-6w)_5b|;E8z5Z`TxcLm;Yb=f9?NOFnQzuEpTeQ|NkL4?LGbf
z{QvX+FaE#z4=GJSB`wI6um8XK{|cNc-~505|1G%GdH?^*e^5ySigQp5gUT9EjDupC
zf#Ki(fB!)x8YoACFe^CyLDCSYl!c@!P#c^HoNGa~1}KC;HNzip{QxSVL82@SY~ZpP
zR3d}QSqSC@r$mq#2Lq^d1DOm;L(JgT4-><`|DbXJlwUw)DX4^nm<lQrKyn~ikm(>d
z!qO(F27uYd!oUc27aKJGKxvc%TqA(;1UmzyE&!!U5pdZr0FG}^c?wF0ppsRBL5e|z
zL5D$$L7PDvT(T-K$T7$;NHKuYumXb;g93v*gDis#gFHCJDl$kjNPx>?MQ|ys2$qv!
z5MkhB0Oe0%a0&;Ne1Z(T;C#Zt0I5qrwG=4dg31L@8PCJO4=oEo^^XXHB-lJ@25|;q
z27Yk3aWjC@B|EtO0foRnP!|6G>;ISkZ=j{()Bn%@zyALTT%$rt(C`02HRzxJEMWHt
zGDv}4FU}wcb_=K!`1$`EwElhl|M~yN|L^|4_5V6F{hs=N?*GOAXaAr3f8qbd|2M(u
z?cM*+|9}2xWZ(pcCMY+HF$jT6c_9Wq1`Y;BhM)i6{eSrX>i;wUkN!XKfA|0G|F`_#
z_<zm+CI1)vpZ|Z+|7HKz|KI!n?El;UpZ|aV|J(n6|JfOY7-SgK8H^b07~C0r8G;x>
z8G;#n8C)5x8T1(x7z7!Z8NUC2^Z)Vx>;I4a-}Hap|K9%%|11Aj|8M<2;s1>P3;r+v
zzwZCe|0n+6{{Q(uAA>rBErTCJ978cfJHrfy<qTUG_Au;c*v+t+VHv{|h6aW#h9Cw@
z23ZCMhG+ke{a^UM;eXoyp#Qf2b^fdVSNt#aU*f;Qf3yDq|8xF#{9pP1@_!ZvbB0WY
z84TwcSQ+&hgBY_I>lr67_A@p!7BU7hYBMr1o@8iZaAsg**!REszxn^y|5pDi{Ac*@
z|KA&bcm7@Scjn(|f2aLj_4mqO?tel5*8Wrczw*Bg!xM&XMrEcQOs32WnSU{>vKX@{
zu{>q&W|m-{$n=*ni}4vlG{fHi0{_GQ&Hel6kNKZvzxjW+{bKo*^YinMo*ybdcz>k-
zkosxy>+0{Tf4Ld*nU1sUWlQBa$@!Y=9(M`PI-VIkEIdEC$~e2&OIWj*Co%5(fB5f>
z-+I4R{80P8`D@{qpw9}QzJIX&5c<LCBje}CU;BQR{Y_z<#<G_!k>eq!4!0oBJ)Wh!
zmV8V3dihTB*7NM)+QX5<c7-{L(dqx`zZd?<{5kl0_V4iDbAAQ?-1z;{*G->mKCXR#
z_nq7OD<7IabAK25UG+bgS&eNedmTqPXFiu5w<3=;uM*!$zN>uNd@FeNaHVrJv1Tzf
z{g3?1`N!wC(r?q>jK9@?HT=;2rttafd+E0wuXA6Yd^6?!(a+m|{QLWashBO7!<6$B
z=M1hi?r5F_-h4iP{!V^g{)@a)Je*vK>@h4MjA#By{r&qp>v!Dm!rzL&nSOEoaQiyp
z<Gi=Mul~Gnf0^{!=l#AfH-3F!_|KZb8O+VW6V9`ON168|Zwj9@e>uN1e;3~$o;O_a
z90{!9OgaA*{;v9c``6K5QosFvfBjYV^YORJ&s^`XykdVL_I%9?=Qn*HwSPSL_nD=X
z(~)N#?{nU-yfS=-d{g-t`HlIV_}B17^UmQ`;525lWM*L~`TPAh$8V0`OMbuqz4v$c
zuMOYzz9hc)c@^?(){|#X!(Yn2pZ`t!Uk&SHE@?hxfhqw#fsOos_%`w_<NLyQkI#xv
zjaP>I0>?krQ%tY_m;4p|qx}2Puae)jzkPmR`pNXY<+I?s{ug>r9zHC6Z1Vj0+iTxi
z|NC-?@t+hD5tS9aCu}EVBXET8DsKhv2A&x1pPZA}Jy>;_yBQAsEB_nz=j`u=zn}d6
z`Mdo0wV%S@O+T}}?R|FiVbJ}ohv%R1y#M#pfn^47nn<pApM<^mACXd_Z2oz?96UF;
zo^aZ89AsO}vY$zZvEl!vf8YOR|26vi<xlkA8-IF!GyhchdglH2=PnPw+_AW$@zD0=
zl`k6@Kl9X!O_eT{4U$<QAuZ}6kjTTzwTJy8+i}(#EE}1PndUL}Gn6sRWcc=9pJDy~
z^8aT4TmK6GUh&QLW6Vp|2Rye_uj}5DeXRC=`9DYAV95~0UCOhStYxo?sS14L&|z(2
zvSF-XlwjP$_=qu?DT(<rvj}S)%RS~RO!pb=7<T`C_)Gr#*$-l`zTdaHo_%@oWv@G1
zU#|T1jn_f8P-CGEla{pdatSv6B-Y1&ul#EJsrb9(uNgxja|P=;_CStJ9Md_B*v_(S
zXUzEj^Dq0ar0?H9O24jo{P+6fiyJN+xVqs<;kQa2M#Ve2hfL}Woz$fy&v4%Vd*{pM
zkIkQozOnqZXZpuJgZnk_Wd3u!sXUAv8O-w;I)0n~a`<uaOU)<$*LxrDy?Nt&);Xrj
z-yTl?ewJ^h+E&x=R{5sq)fS4(X4Ltj_geDh-FFt>{xfXiROFu{Tr0d>sFi;*w-SdM
z)4zXrepP<=`Y!Zk!KW$jlAh0hAba)L`6ZVo+`0GZ6ql`%l<9n{gQf>mxdeIsTD-M+
zVgEwm{oCInY)|>8h-?&g5c1-m!W+uHgrknto1ys6^zXf2wtQatG34W=H*258KKOCf
z=!*L-;pf@^Hj1QctTR1lx?8(bB8d6_2e#+)pJ%?k{PR8Qe!fkjhN2gQviKix@8(wF
zGT@MBj{NKX^U){iPv#%P-{*f)d6)R|-Qx$>t*>?7KKz28$wcCh7K^EY;cS&e{{O$e
zy~=o!_NwdiGNxDjJfi9nhlMxs{^b;4x97aYCB+@a?(vWLyZ8GiZ(hF(dmsH}>KD%s
zGhZFO9eO3<I>#f^KO4oiYd^J^WI9u|mZ$HF+2cj`{=eM)eJ1B#@%fURlDh-~*gaVS
zSxUGX_$2woxzhf=_`LnqycfYA?taexRr~kc*Y9tY?;bk4^!%jTLf_RS>kRMNJ+=x|
zJ;=87jr^SpH&4C}|DP+kUcOD*L4=EY_WxD?wb(KQZ%X)!334ZW7kM@1Vg4h>Ptkwd
zI1X}m{>gr)bA9GPrK1Ng8-D4QO*eOO*>2;Yq{(pLG1ILlcN#x#;eI9FB3CPB!g})W
zt=}3<Wdc8Crz*XZ+|BO#-sGO=?ea(3-}mwC7hNXO#`OBR#g&Kyw{{m@2z;w2T4+{c
zA8k2RX*$cd*OMPJzZU*3#nUUwC!ot+_s8*jB%>gYm`tnM1yy#*0!HcATDP~|ig;f3
zQ<DF=c#L2)%gr~f*Zt4s9({6Z%A>Adr^K1{Tn!zxW=XDR-T0;b<A-l6ex<X9v+ZVB
z#-#E86H`8agZM)EUCO*N{@gErUV6Ui-nOSZJ}hIn$S*9sh-cY<j<>3}d@r}2DLl38
zp2Fw(yl=G^8&1@2lAq6Y<ol%$b)Oi&eq?B4i)E}}GGmBjl@Zw@ZKk+U;jGwHwoRYq
zo*6y#d9mwjAhQDhTVW4guYZR>e0+58=CZ4$XZ0`dezNV?A!#4OPUCc~d!ldtPI$NS
zE&InKztcH3u_QB@GFvb;^VCSxDflZbm72{h^JCF#*(WZ~Oup`5n9sF?UytuK`{f_2
z-tK!i^;+A-)Dy2S<UHqPSgUfwqS4%6S5VmQ$C}smudP13|E<sA%e?(x7-IzM9Fa;z
z1(nANt3?zU&ELg7H+gFQ+W7BI=Jy;k`TudP;izDo@>S{Owws-o)*U;1nC15IAAjUe
z*uJ!5HOvs5`SZ)G?pGH+ZT<C%c_q`P-*W#G*sqGqXzbQ3RyfVS_|K6yW-l(hHvS;=
zYwEv^|F?29@to(M$tV1G@!OSm!q4A68+tVB+^5&K*ebN9J9OI2&<^M2`NaQ3^<~kw
zJHH+>&SsqTSA)Y`FkY5P+e4*Etc_##*9C9(zsvb#_if7$&A;X>&fH~!D#E{b<o>w4
z`g8No>Eq`Po|Qbg;GXlJAX$D#SFadzWr^?~0uR>T_xhmpea8O@jM_h1m@7o0<h%5g
zw7-j3u<ZJJ_m%&9gU`hueZQ&vdC1(spC<WLYO=5&d-%JuTUl35oX@+l=Bnc<qep41
z0csxZmfrjIyE!vnF1(g=Z}Hb1e}6Hu{(t@@iFvEk7tQA;N3>*k#lNz>F?zlI%fUZ9
zU;lsJ^J@XaRFRL0aWV>`8qBh<^B%O_eQ<;0`l;Iqmk*qH_=G|DoSARHZ+8#btDmOc
z{C?HtjUw}XMvp(+zfOAZ!?s@|(z46^iCoR^*RNvUv3#4tF!A5lH!iQUzwKgeQMjkG
zQcjdB<J*lFi(agJ@%~BoLz(NEm)~9NyR`0yqQ(^4H}2oe61ePNH9Zo3+Wu)4)2ly?
zpJ#pR{F^5_TR*_iUL}|}_KU>(_8(b)bN`(Dl={^FjUsc6xTUIw!dYH!M%5o1zIgrE
z`X%yN!kr&CzTOml8Swh%9nMeI;$O5KoNn6M3;MsZzdiMK{Kw5)POMQsE`K-tWhzo?
z^4i2k={%eMn{O{3eK%m}{w4d-?$zYC=NK1DBrBg+bQb*mXW_To-#LF>`|<Uy?}PO>
z`yQ-*lm7kU57!6QH!m{J(3Y|jbeJc(^JT!jswe87%(&z^Qoc<7_UV78z#2U(BSwkq
zjBcMEzdQY(fg|X5!28aZN$*_QN@Tm`Z;M{zocH(2FU!Avzh-{#`t;`I(O3HKxZd-A
zUj1S2ySvYRT;h6aBi>=O$LfaG-9Lg455LU$7ROl0-TdGF%fo+xY^w!dDJRSEayzn!
zF;4rNz^wTH);p`WcRx<~IhE(P*iw;LArAhT>{3i-|JVI^@x}g|^$+tO%Rf$e{p?ln
zJEf0J9}3^d-~4#pfu%!J$h2GkHCMp%fY;~$?q_0Wul&dJdjI!OZe2lk`OnhsTtysK
zTmdXe|L^@&c>CkkuJ51z|KRw>ufdbWn<DgxPmt~9pKo8CzkL01<X8OnL*E_0c75pj
zsQB6ZyYx5pcT&&hUXi-}giA~})lgWKhvn+4P2Uf3&tO0GulZ}!+w8v!_=5x(gxrKC
za<1oIE~>-j|6BFvi#OJ<xPKM?z4u?9r=PEfmsfZJ-zTOSi~;}Cf4Ki>|91KF?w=k1
zS^li~+V_R=OThQoUkPvh?$5b==F*gZAGJO83Y5R_e)#hHubZF=+uBd}UpKv;^xIdE
zLtqYDCs&c+WU<pS(L%nz%s!WXS@d4$6W<rfk5B$o^Y0hFF8o<&3A;V>HP#(}zJ1yJ
zx%GqD*VhaztY?_lvDdO*|E==d<=g+aAKv_bWN`bz_2wJ@e?L;&s`p$aOT3VEE#owv
z`3ycU*S|Xd{??E4JWV_|S=S1*NmPpmi{*1p`@;X}`8W9wo$vZSHh;=vkP<W&zAjM9
z|A|YABZNKYAMeLm@2~!7W2j;^W)oscXXfPEz|r)t_}k)-)o-2N=smf3WBZMXx8}aH
zl?l-gP?;dBByfnymLr|X=*{t$CqJ?MoXk3t{S@O~o~@GQ;tK>bxTpVf{IUL5^_Q?W
z2VRSOO#FX?yOuYfN1gw+&{_U^wv4}OKbd|%`P=r7>+gX-mW<YHOF6!=?PpZ{$Nu}x
z_ngm<USECQ``GWnnun|J{Cwdh+^LhLwn^d**F@H*Tp!tjKmT~i^U313G)n`E=)XwL
zZPMK`GNLiOXBnb@YyM;YdgJ+|m(rgyeh0JpaLs3r;k4lYz&DTW{(t5FZU6TDjrnW)
zbIFeghHLD3TnX&H%udYF%zyr!{Oa>w=k2Z6U9T@bxpV)|Bfl3VOreU46!^q)g?ag7
zd6l{Oe_ef<_Fneqy+5!1W&ajpIwJf>!d|G2E1LBOb2(%B_t4i4ue3f1{3>VAWd6^x
zi(6PwmtT^zgyjM288#h8j^7SHvVT?mpU!liIhf@DTMVZ<XD{o_Kb+qVe?9qW_M3Sx
zOP^1AqVgjDRsCCeHUXKr5<f-EMC}EZ@vY^Q|8f4^+;8`OuKT&<&->q3n4^SDgiW}Y
zvzxFtvT`$>`SI`VueTCk9)4f*YtwJ{|A)DEh&l+Wa+z^;aq_YyFirW>@=J+f8Poj#
zrT<?u&Sx#;>}KD@$oW70-=SY`zSevccq9DU<(>PR6E9c4wt8*%MV{xe*gyWwe2#+0
z_`?O1IoW@deY@~a?$48-2Y<%?xWJUkw}Xq9wVM4O2M>o6%a$JwpNhZP{4o2j`t9Yn
zTmK8Ws(E*CGjn(FJmUGp(au=#_w2uE|F`|0_^<U(DuWYi5Nj<9Khx2F=Kps7jrhs)
zW#`AK@Atgrd)xS?<8AKS1D^z#<9Xh5b?_+jo#8bU_{OuDDUsnalk@*o|BnBY`Ze+I
z4dz9RRSYFe8cesC)!6hHGk<^hWAXpbzr26lf0O@(GqSTxW|_mhjrlbT8{1XZgUqE&
z51GHSEMnfnD9w<^aD!<d^Kquz3~&E={_Fnh@JHkKhM()d=YQ|}-u~s`r;VSKKYsdb
z_#^wr&YyXIJs8ZHHZW%}?_>VV_L*Zgr!yBjrw|7pM>hK!Hci%Y=5A&UmS(1D|KI)V
z|G)j8#J}QyjQ{TcUGe`L<02+mrtOT0j4h0J87}@eW;9{Cz_^V`hq;QWf$1dEXNFn-
zV*V-qjs1P|SKSY$Z#7@-zEphN`o8s}(--HjGrlqXRQjXw{|>`GMiu7AY_i-tc({0*
zc?@_ec=z)ZaK2;v%DRX3I*TzA=l{w7?*8BL&*|@-KU;nq{(1iQ>R+b6v;N-u_v)Yj
z|5N`3{<AVLFx_TKVZO`k#B!4PD3cIV5~B)ZBE#K(Q~&n-*7{lX{leFt&nX|ze`NS_
z^vj8_TYl92F8CYqKc4Y1(|Z;-4o5CY?oyt0JPtf_d7g47a4K;~bJ(*LuxK*3G35Ne
z^?%yGMSs@+uK4}-_k};({^b5q`|JC6&tK91f&YE}-}>*&Aj4?Dl+JXEaW-Qm<1~hr
z|3Uvg{CW2~;@8XX+rHlaoc5XdYyP*b-!y+%|6cu9^Zzae7p4}L$?VmfB3%2qV!2mv
zzvMROj^+H!9>BhcO`r7`(;fyH24M!d|4Dys{~Z7Q@As5HjenH>$p5+bN9Es^e~bQ|
z`#1mpS%xmg^^E<DlNmQK`ZC5c82q>V7yno4Pw}s&9}~ZQ|Dy5b;+GF!bH5dT|MGLr
zpMQVN|4T3`GBdF4=Geli$aS5oj{6q3FZW4K9gYTeQT9I8Y0RG(A2F<Euw%ISFZS=L
zKcD}6`ZN2F-XG;Z|Np%H%lq%mU*&&q{wXp1Wl(0+W$a`8$~c=bmm%^0nt#9kI{dxz
z`|i)0A8y|td~N?4^-bqH+mA&*m;R3bd-xw4!&*iKmYHl89LAg<IXk)Ta$Vyx=Gw<G
zpIwptH)|ma5A%LTA;$R(yZ*QQYyNBWH}G%NU+%v<{_y=h@z>*D<UiMcEB{US-^lQd
z;XgwV<2lB4j2w)944MBI|Fih#`*+=MtzTb$9Q+>pt^2F&H^=V=KLUO}`L*(o(ZAmR
zT8xU!?yRcpDIC_EJ2}5|N^mXXoXgS2eu(WQ>ui=F=47U&jCzbI3?l#K{%!u-_&4yc
z_+R$FK7U{S4g6>M&+*^!f2IHV8748@We{QvVl-jg!obPk@_*?+`F}BgRsLM~b?)b(
zAD!P9f1C2H_<Qq@!k;UD>HgXJSM+}kgFMqE=E<yH>?s_NISe^{IZZjYa_nS3&nC=f
z#463g&TPTd#(0H6oI(G8&Oh#djQ=?QUHN<Xugt%Oe+&Li{&)Oe;QzP(QyDfhd}0t{
z{Kv3^!Hr?ff2RMn|L*>M{paTIHNV#VWc+FIqvCtgcc~v5Kka`_`ThNm*S|CWEg3_Z
zT9`Rlr?72jkLGCSh~VJmsAKPBo635cMVzIUc^A`e#)}Ll47>jG{*V2~{*U#a_CNc7
zCjT7&HUGQxPxybp|H}WD{#P(OV_;+yV-#Y%%P@mMhoR&DuYWE7ZvB=1>+r|!ciOL=
zKUIG2|FQVTogbk;8GoJlwc+>cKSuxT|7S34W<1Aqjrj%3T2=+NHnufv<!owf8(EcD
zx3c)M9AH*vZf1JQn8f&xVJd?H!;Syv{~!Io_<!&J{{M^rANhaj|F!==|AR)5K{IwQ
z{!jnk^grSM`+rsc6#lLKEC09T&$iztzrX+5_iN6tX}{`!t^DQu`}FVXKOg@3|G)UZ
zgW(!uKhs5~smxg{Z&_Mdcd;6>9%EU_V$QOaIg|MTQxuaW(_Kb6#%&BM8D=o_GlVeQ
z{(t_z34;fNI0HX}8bdV0Jceop0R}DxX9hC{R)*{UC;gB6@A&`Gzkq-D{#O4L`g{0K
z&L8GK4}X9AefRgS-xq#=`>p+F_8<Pg8Gk?jUHVV^|EmA~4EGp(7|$?zGi_xOV76lR
zW1hl%i}?)m9OfG4B<2uiGv@0|wM<z|woJzvlNhBKFEXrP=w%3C5M}uN|K|T?|1<v!
z{lEF|^}qlBp8q@ZZ}-3D|N8!g{rmNI|KH7j7ys?~+xWNOZ~EWnzmxvf{H^#q@9&Mj
za{r?LHU8WEPyT=9|7HJA{^w@MVmQXY#HhxY$hd>?AEO472U9Fl9n)T>cT7K-n3=_y
z`I+A`9b#%^a$r(sVr1IIn8T>Vc%5MiLj(gq!}0$U{zv||{BQAJ;s1|+U;lmkcj@1<
ze+~a){>A>Q{nz*}^<VHm?|(M`4F75WQ~hWEukzpGf7}0^{P*Oa)c=V8`TxuQFZh4&
zKP!VAgA+pvLp#GZhRY1M7~V37Gx{=iF&<#N#rT$ygNc`kf$2XZ6B82?1JgUkGmIM<
zXENq9+A>Nqeqq?n(7@o&Aj$CH|Jwg8|8xFF{5Ss3_W%CBJ^yC^tN0iG&-0)6Kfiy@
z|BU|`{xkjO^e^sT>A(7aGyh%q$M|3Lzvh3R|9SuW{;&GK=l|LN&;N6RXB{LNlo+fS
zd>A|!d>E1#@)*h)dKs25Y+*RTaF5{~!*_;%4BU+3jIxYUjNFV&jI50R8D2A7V%W_v
ziy@c6nL(1_`TyhpxBj2^zy5#JfAjyM|G)ja_wUrdBmYkPJN@s%zYG7a|9k!K-#_vH
z=HQX>3IA9A-~Ru^|A+to{+D3TXRu~)V(<jp?!n-}V9(&h;Kks~;K<<35XcbDkik&K
zP{dHg(8I8jVK2i)hC2+;7=ANwGqN&%WBA1Inc*G7V}?@<s~9FT)H9?o*fNMSFfe@h
zf8+nQ|9$_9|EK+r{O|qW;lKHR%l~fw1O7+<&-`EYf71We|M&hs@&D@ocmF{%2%vcw
zWd>sg8wN`TWAI#|I)f|&Hv=OB7lSy1EQ2(7enN;r6g<x&#30XL$Y8@@!(hQ+%izl3
z%@D-k2M#-523rO-20jMR%*v<#kHEQi&;K?5=lq}ezvF-Z|K<NT{onL|>;E19cmF>G
zo+m!_|MLH*;CcM_|DXQ9{U0=2eEmOY{_fHLm;WF9KL?&~2D$Xs{|Dfiy08C#|9|)Y
z@&C8~fB$C%&sOm>C^Bd>=z!-jwHS06Oc@LrK=YfL;JJAj1|bGP@Ju>C0|x_WK9+}p
zl>r3)|7T`kX8_IAfoAAHv><pcUWP$|0W@C)n$3sIZwi6eM}TIIL1u9>ure?*`~lCm
zGcYhR{Qds}JU95~KNABR187wRXhs<{GX+|Cz{dcZ&t(O#viSf1*Z-gYzkt_qy!`*<
z|I7cM{(t)qTK@rB_W@eX@c%y-gCK)AgA9WVg9L*x1IPr>>=!QsXttk`;X8Ou$m{<v
z{y+c!5Im3i?EkC(@Bf3=NPPMK_W$$$ufXdfe*XXc|Lgyc|6l!o@*gzI37TmK&2N4F
z|K|Tws2FI40B9!u<^PZWKmUIRUN7+d|BwG4|9|}d<3DId^Y?!c|1WsXik$&8YX(^@
z!otADzzv>_1I;{w){=l`#zAx9p!r5n$UtT@L9)O9L&BMpftleycqIyGrj(HZH1i2s
zm+}{E6Eg#7Zk&?=G)E`EAPAmiXJwFLaAXK#uwhVR0L^!+FqkvgGT1PfGbl3fF$jZK
zOn_EwNHFj+Ffsi3&&?nTp7CR5`2C+9Jby060E%;n+dysxxd0SnAk`oLzXz|6c=i9~
z|2O~Nfy3p&|HuEIf!8L0W`sY2X8=LI{{0^`vkLMDX!aHqvs?_I^#&j}gHjM^WejM3
zAGCr7G*b#$D+BU}AOmRKhA4RD0ccf+5Q8vyHWM`O4@!-Y*<+BKKrGOV@ZbLo;PnnX
z43K$GkeMLgiZKW?faXL&DNu+3loml4!~(5{0Ig2|`4dEg;s_MJzyE_$0Vr*P*3Wzg
zr?YS1^z-TeC$QgsfL9=a;`smnKmR}f2ZhNyaGC_AlULvs8n6F@QYvV+{_+1O|DS+Y
zY=BbJ<Npu;gJw-ZD@yMDfAIg||40At{lD@5+W+hSuYqUtL9_1{{$Kfj<NwwFpn3Sq
z|1W`O`)~Ze{r~R&2jH3am;YaaX^@GaFn|3YWb3Q{Z~ueVG=bdt<^Qk$|NetkbFhME
zs6p#B1i|ZI1Q-Os`2)00MwCGuJYOsUULyfvgTflLY67%gLX<%S8h)TO0*VPxXoBXM
zL7~hGUO@s{=>%CZ0*X`6nh(&r4bW^nXq63U1|PK21~h-n2wlAdnns3{Brp;biy##c
z8pMLBMW;b3K=BSz1JaF*A!?CHWLXd!qzhy_X!R6mg$zgx*))(|(2PEWhRA|Qh^-J7
zhz&9WvStcdB}fLO1||cdL1G{b5(lXTiGgf{@gZU$9t5NEAv}<Nkjp{(AT&fRhy=-j
tXb^_D3Zw$Wg2{mRAX$jbAQB`CG7BaOQVGH!+dvq`2FZigL_pMnWB`l+j+Oub

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/087e1d7fae1c1ddcbaa3b5f822a171ad15498186 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/087e1d7fae1c1ddcbaa3b5f822a171ad15498186
new file mode 100644
index 0000000000000000000000000000000000000000..329af8a3b9d8928bc197d1361236a82fda888026
GIT binary patch
literal 146
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bG01{!K1f(REB*vsMGCca~
s_4JG6XXej>Uy8n*|GfV5(Jz(X-hOBO@$cK*Z^qwUzTYaoz2EyD00FQh%K!iX

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/0f61c33027394a0f14d29dcd22f405cad943b7cf b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/0f61c33027394a0f14d29dcd22f405cad943b7cf
new file mode 100644
index 0000000000000000000000000000000000000000..8cda165c8c791847ddaffdecd62f503c42a2895f
GIT binary patch
literal 29936
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6l4DG{_EnOL;s&M1~469i2fV)>(S4Tzs~&5
z{NwoN^e>;ECw}n#Jpb$9zo*Qt9Ba6Cb3WtX=T_m3;O=HW%3{E*&fLSgn)5K9ort)^
zIf-<!p8~mDMQoo~zA))BN-?FgtY%%qD$4qm^*<*!-xgjYZUgQ;JgGdtIK9|&SaVtQ
zna?x)_<Q#E&0m(k9sX4R{rW$ixsGK8W7A*G-xa@lf7Slt`}Oh1<F8sDcfEb_hUYEI
zJC2VQU!Hxr`nmR##YfE#BJaXpc0Q?nJn0Gh(;d%PUe~{0^-=#r(W`5Z+V3P@Uw`T3
z`RnK8&dXk0e|hGWuB$pXWA29C|9C(5f$u}f$KFqFKYIQk>0!sCH4n4zN8k0i^ON_-
ztFKQ#yMD_3DE2Y(!>@PMZwudKypelb`S$Z0jyF25YhDGt+Vd*)_377YuR~sbeD3>V
z(o2U|mtS(exc?;Zq0c>sJ7;e{zH{<!*j=YPjCUU0(YUwj-n+ZI@A%(|yqkMJ<e~ec
zjgP)R`u!;E@!Tin&njL_f5rV~=NrFw)*qcehkdU86#lXJgVx8^&o92t_}=wH@yFJ0
zFTYlP<NV?NE9H;LU(vs-{_gw#htZG8hvCa#*T3oi>KN`Yv9Z2p6Xob<KgPO&WfMy>
zOFHu+rdv$ASbnij<dWmA=9<oVf<uT?lADQ7O7O0*yy!xaQ^NCvHANIeYel<6E(pI6
zxghpR{I2+Nu?ZqqgzSZu3VjoPASx&RT})6+S=2~GRaipkjldRuRsI@*rNYO={3Rw!
zd=Q@`7A*QhBwBQqSdheAiO*vDL?gw_#NUfe7c~>^7qH})<>%sm%eR=%hW8s+6K6fA
zKUX3*3(tOTXYM}kWFAJ|FFesao4F@*yYl?zohy(eoG7X;c29J!Xt|iA1e@f22^9%W
z@tdM)qQ6A9h}DR3h}sL+3EdZR6Xq6H777#K=d<ET;GV{v!DGwg!!?a#AIB`t#hmS&
z|2VU___-c)M6=hi#j|CxUSaZOIPu?uv4uI9)q>56t(7&5C7d~hc`b`K+h4Y+YyoUK
z><t{QoWD8J*i~4ySi)GH*x5NBb2;$1@@(Pm<6g-v%;U^EnNNwonBPVqT+l>FRj5v2
z7w>xR>)b1NWBE_;>+)aWb>V)@?!;Qa@`5FvRi0IwHHO`odm-;79zHG$_C(f0Eat4U
z*sgM%=Hli)%UQv(fjyM{1KW94BbG{L2bTM+4eah5w(RDt*O>E|w=t`+y0bTO=x{t|
z^J3LxQDhNf*~etg7{w^S%*xupwwY}QiyfomzvX|P{uckU_V3dF#SCr?pZ?GMKk5JR
z|2_X^{66;c)vwUMMgNyD?q;6J;>6;~9LuzT={d6qi!{@@|F8cQ{V!*D#30J>{@=NO
z_x~?sJj1+$HHs~nbq8}NQ#aFY<}%jPY|89R?9<t;Ilpn;=6=r|$o-ZxkmD5lVvat}
zP)-w$yKHA!&#~&WZ{lF&4B|*;zt6_UA;o=yx0~-VUy#5V!CE0r;eW#XA{xRM1eXgO
z7w8naFT77kUqGI(fM)`iKc^{Y1E(|RclLd(KbWsG`?AC_OEG@=_xNw%zwiI2GoE0~
zU?~1K^UsA}Za;H=F#m}8-v7<!Tl%+s-<E$<|N7$N{CB)>@4vBlEAfu^{gZbQZzEo>
zecAAW^@Yw0>lf3XAAWY_*~{lYUYve$@Y&4APaYOMe*NtK%VlpO-~amX?1R$>fe)4+
zvOWZUxc^@HgZwA1ug2e(d|&)c`0Ja`J)d8E{`ST1o6L9a@6q3OeSQ6v`TK?M&fiyl
zwffTX<?c6)pYlJ`zOVbr{H62L!cV?m4u4htcIB(v*OaeUzb*e!`?Kfgsh>-KrT&rm
z&&$-${F!Mg<70*rh8O=$7~&YOGFmaNWq835$#k0~p3RZ<DAN}PH3kO;4MrX&U8Yt>
zK8BuulmGqu|A4`s;m|*;zjOZh|E>S`=imH)H~(h;z4dp}zmk7le<S|1{nGof<lEhE
z{y&O;KKU{E8|RmspA<en|LpT+|K|rERo@G}-TKDoZTj1eH-}$Uy?FL4^f~{_PcOH;
znDvz5k<f#y4+0;}d9>ltj>l7<=sgL4yz$YThyC~G-kEsQ?#A4k%y*pczPmH|_RpK<
zH!t7xy3Keu_+Hh$mv{B<{ky;Ek?GUGXVp)4Jbv})&*S)K<uBx3o_}%jh1Sc}FMD1+
zd9C~Q>s$ACGu}nNxBlStf&D$_Tb|c)uiRf3zx)4D?u+M_ji2{@*8igXh2@Lomo1-l
zKdt$=>a*<k`#;@&J^UX1<^RVIAIv_SeE;PAkM}w63*Sw7yZP<eck|vaf6w#b!pEs!
z;=f1yWcu~!=d+(@em(j#_y1hRLZ(YhYnZ+>3NlS$KEm>zIgjzlKbwC%49l5pSe~(b
zV7bq%&ZN(n#Aw0vjp-j#8WS&5D-%CU7~2naCXPV%Q1(e26S&@RmvF~$g>zluTFjNq
zmCbdK^DH|H>jI{^OvbD=?2PO+EUy?BGUzbwXEJ5!VBN;X!+wX&j=hcJC|4!#a=sYe
zpWI(~PV*lU+9nby`a$%h_zlS}sZ6P6sZ^<I$s`FW2}VghDIO^!i8-R%g}a1hL{-J@
zBp!%YiPee<iOGqJiGLPbB3>=NNYqS(RkT|Cs^l+8G4TfBje?GXKLq#$Ukc_6$BX_E
zn<Rc%{EK+KxQ&>yXrQQxxTI95%xjq{88?|=+0F826_u4*73V9~DFrL1syM0XXdKg6
zt^QxNTJ@-!yr#bPW9@%hu9{uy^=i3llhm}-{;K{{GuBw5@kb*^bB+eH+Fhk$g>ZQj
z`NMK%vKyqsrPoULNFSAokQ5eA67dt-C-_3hRWw`tg?P7^yT~I!eF0s8enD%Y0|MN9
zJGu3^#5mnJy4h#2m9uVOdCl^GHH<Bbbq4cmMqb9-46_(S8Jhkt`e*q!=+B`)rGJ0@
zarn*n^UF7ZZ^ysQ{PF4+$KMNofB#|rWBw=U@1lRv|9Sq`{>}eA<JXtp=Kr$(NBo!g
z@BaVle<g;^{~i7>{m;%&@W1)rnZGyxto~j8OZI2tkJO*(zhD1Z`S;Y{#=lqoI{tTO
z%xA7-{lK<`{WrS;hbm_|_XnO_ZcUCQtdCi`SaaFZ*q*X}V`XD2VAWw=$ohcw3+quf
z1CC{!g52A97V<sh@8&<kSIi$H_((`zxJ@WgC`CA4ghPZ`m|w_Nz=YR~tB&J1$2~3&
zo>^Sq*&Z_gVq|8#!0?8FoiUa1JmX5nj|_kQ|NZy%Z|)!UUr9fDe;ohm@~idd<L~0%
zK76_S`NqeY?=QZ!cysGj$g7F3#9w#6R(NgsviI4XCzqduK7apG{+0C$muKBiEuINK
zzxk};nb&jc7x^zlUp{+v^3CJ7lJ7X)*1uMNz3=s_*Nkr}-dubm_crzIjW@NgwO-Y~
zjCtw#vijwzSHItcz1#Oz{f++X8Lz*;dHPoIt;6eSFGF6Of1dS1^yTdrzAw05IKR+-
z>G|sGEBDu#uOGhJ@p8|Li7!}R`n>%2LigqMR~O!NyuJE1;vL(&b#LllSG_*-rt@9-
zd;Rwz@66wMy`T3%?_<;l$#+*?v%Kzqz4wjCJFoY*-?zVydN2Cn$A?27*L+g^{QlGN
zPwzjOf6o4F_&NQP`A4%4U*0Eu(E0f0<J*s4Kg7I$`gX}%=XYuE4!#wA^YG=~m)~Ej
zztwu3_}u)-)<<fOI3Be>Qh3b&*z(cc2g~m>-mkj1?e5$=CAVE~>E7IMQ{xW*J@I=<
zceU=kyLJCo>+Q|A{caz)Rd6fzR>ZB1x7cnM-V(p5aO3)og}1-l<9=xP@ce!L`?DS>
zK7RW|>Y2!MvlrU0s^2iZd-!hk`?K$#z5D%E`d#t+Ss#l(r+s<;<?$DTFUvmP`fT=v
z{Y%j2Ri6w$@BLc-GvJT$U&B8&zv_Q2`eXZ#@85?%^M4!v*7>vT@27v0|4;h=<KK+G
ze1CucYhg5HGG}!8-}Be>@6^9_|2{EnVeVxyVmZgc#(t1<ANN|Gi9F0a^LVoP&4f+F
z%EW8MqDB4-Yl?jmFBRV}${_MqNLzTe$Z^qT(c2;cqMt>BMMFf^h-izw6l)We6fPHh
zDX1^fBib&~Cwy4ALwKcdt|*)MCb8!tKZJCJ+=R4*P6+Y~Zsb4ASIk$yYtQ4u!_Fhj
zd4VOGc?atQj;UOp+*i44c_#C=@NX6D6OtBE6!I7PEXXb_BxWE{F7aC8xa3vI*OI%W
zj!IpU%#g?u6BA7rbrpLh79;jq_>-W6&`yzZ@fh)iqVA%pA~{001XF}^gr^E;2xbbf
z38e^^h_s8G7uqf8B5;s*Kko(pFM{iY1%)|<-U|r`-xsPANfMhYep>vH_(sWUnMrc#
z@{)2=at{@Zl!BEGD>JCJsMshiRCuEps=P><O@&n@Sk*^eQmtLZTJ5XGdyN{Ee8nmi
zW%ajeAJmU&_-Te|a;vAQXKC`QyQ#UWddNLi4YIGdsTBJ5he=FN`?734mkYa|;8um(
ziaI<k|10@#$cxF9OZ!Q_XFdDnCdUo!Q#y0`-hG?!t&)>N>8eJDD98U>U#;HWd}Q<f
zut=X~j^M6Ol7I3A7qEGL(0H@q`+Tk>o<rZZ{ah&0#5(DQ{B@Z>mAq5_R=;w6)B7#?
z!~Ylc44w++5(!_+pDcc6^nMb<6`nS(^=$QjZ@t`q&+Jjkn=?NxelL5o_0=t=rIMli
zcb-?AOTQcX#qsNlk3}CoJ-mH)(iit%+aK<^x&M{I_gCN5KVEnn_nqN~`HRWVExsv!
z&3nD+<Dx&;f6II?c=zGq;pf~xbiNsXne|8i^X*45&vt$e{(tf3t$Xt>uY7p^lONws
zbz6B;_Pt+ApTBw`@T-R@;NR5mv%bvwQ^4aTypb!7;lyu`&j;U{{S;tGdee4o!VSk~
z`X46$==;I)GT`o~y8^emuitre{(bQu2KGn4S)Z?Y8u4+*&lx|Kzv6mm@jCe{<GU+Q
zGM+I$iMS?uS>w6*Kh^)cUNK#NaGCMeu?OmJX8n22k<HTZR{K`iJ+61g@0Q;UxU2t-
zgZaUq?!Og3mOS%#*7~vU{jXQQ-@N&v&Zz!B@bi&xYJZ}?zy9_1U&vRxk2T*-ex>}Y
zWcmHK`%}&@IiWR@wQLsOJD588a~XELT<~eRkdb1Okmb+v=Rwc8zh3=&<D2#C`7awk
zX#Bj#%Ei0wFYEgme=l<@u$Z#05xB;i&$fzrA#0V$R;hCCXY9O!x%?k_%ebEZY5Et$
zb&0i{IZN=S#16q$&V6h)EX)6y|Nq0)F1B3c8rv0yxm-3pt-qr_eEuHH%EdAFpX&FU
zzcW~u{4;sZ^?Wa*EaSDOm+xrZU-Q8DmET+CH=n*I{QCR!&DF2hT^>q5Tl1*v&a0a|
zcWysmem3zL$1R_;+b(<DFg+J=@zOon2ZFc#E|#6Va3%D?fjgDA-dvk>KKP2@U549_
zugG2gc`fVCiF@65n{T*X`*OqlTFh0wCk@~27&d>mfAszGmx~@(%OA0REdKQFmGr~q
z_a$G=ecSk&{oTUnPEXc;`Tt|_>viuP{#O5c|7+a`m4{|eCVX4@^Xp57dsdI%emTIX
z@axK}j~`3fzcc@N<@NaKyU?GKO!jOnOvNlI+*QmwzMT1B_Pv|w2-i-o@9Zm?oPP>^
zi(=TpqshMX+o@lgyfHj47z|k+h@>j(%a(Az;m#89XaB-mt8h!Qo7+%?O*vb+Ng|5(
z7^e}J6{iBXreLaQs)W5<ni8vgt7sBuI7=G8x=f<vF5&f3smd$l#YIGTgn6ck{E}ND
z+bgOj)+`<=6fAO5dY|NK;fZ2*Wgd$-GcNk$#iK2$C3Q<YRzg?GSbT!ONA8oH4>+DO
z@BH_d-AyowV=G&N*hHx~{*z1+->tr<b9@nxktpR+`nU0q0*3<U*8iDo{{%j;2>g8i
zNACYK=2&h=E+;-ce!u@Fe_7e9nIhR@g%)u*{XNfgkv*F2HUm=zV+kwc{~zC0FkIu2
z=KI1O!_miF$H>M$TR@X9gz4@7pDZ!#>YP7$D+Tp<Ww{pe%5XRF#>-t-*d+2zs8VdP
z>|5mk*>#FfbVcMJ^LdKAm1I|S*Y8nDl)I<=Uh#*rx#l75O%{$eo*MqL9_p+nJI!s3
zvJ9pevS{v?UoCZ2bd&TWl}qx6MGXXe`S=7@2=s}V$_7ii@Ry6ck+KlF#<h&!Mzl$k
zPjs4)IQL>!7OwSTt_sc)`<R=4?Pp%ff1mFd^W%S9EHNA^9QPTg{iyps^PATDo!_{5
zW(eA_&iEemMURo2ZzGp3qr{KnA1!`Q;Q7iQ&++5`xo@H$Lf-s&CG<Im;SNjVUyl#J
zp6`Br@T180^^CI_+dnyenDJ}MABlg`OkIDD{+RU7>0im$V{hI(U-e%9U+}MW&s838
zdwuJj%14f$3;!Pe^Ztj?hu<&M-!Elo;5y7G{jKP8*@w7K?#w3qDjcu=oc+G~*GiTy
z&T`hBY}16r6|Tyr^6q9>#?rtyn~#U9nKw~XU+T2<6tUAhT)bZ;dsXkNe3Ovi>1F@I
zdq5^qy<4$YNRCU9XA1vvo*f+N+^+=v#7+psu$M7-FdMMVWm5cg>r28<L&jq)O^o&5
zt-h{fuwkzHzUT|@zoTqiJO?=E{r&Vs@<;H$I3_i=E3E1N<UdQjnE%xN?U9e0KE!;o
z`|AC3&R@+R3*QO6b$NI8ecXr1FS}j@fBW!P`@792k1sQSEdTTE_sOpoA7tLI{y6K~
zzu&okfBZ80W&HcbFOy%>e_Z=E<JT+3+sr8pYyUm@XZttyzYymRUT3!Af9L*oFx+FF
z%~r>*#`^c4$S;vU0*o6N=KY<-;vlq4=pAeBzk7eK{BZbD!}yT<AKz~NIQ~SoeSg;f
z*JAT#lVZvHzwOtjzs}4_|Fb{OfBEjU*^fFVS@w4v#cVbV)xQOP%>Cl~lb6F?z>|9q
z>n5hne+0iX{+Pvhf@?jS<KIcYPySN>QS&R}Puj1)zhr*pe_r;{;s@7Hi+83^ES}oG
zUGqWprRKeBH*Id8y}Rat&a<+&oNxRdzQ1F6*Yn!?^Y_oPowd5M^6uRmE@#D06r8ww
zI`>rZfxi9K=YCwjdQs>^$$>Y!a`%2dGUxcsQy(w=xD<bC-|?bT@+Y$oZ#a;C!u{f(
zv$dz<&N7|gJd}Ro=^2MJ4d(*S=A2n{?(<2dvyKn0KACXc@RH3%lT!;0?ccZf@S<~R
zSAwrEx)by8$J2e!jPA}ozw_j_^GB~QyR!Uj!-e!~JFcF;Qg-wFEwO8RZoPXN^}zlL
z*R7W?UVd%(FZkQ`qxI)UUvod)dgJ|p=}$Sw4GvKjImVptRc~BhAAT45^*zHW&Wrrs
zQZ9;bB%X4Hum<pyivE;oR|t|5mRT=zM|z^veo0$VPr;AEHWK?K?}{H5N*CspX_Rx6
z(vnV+_7hPR*d_c+N>BNLs<FJd=vF>k!PU~aie@qjl1~(>)uzcG5j!cOCwfL?iQr@|
zXHGxCTOzsw+xYke?FE$iKMH){KhHUzO`dzXK$f7Q@JyMbYK1Dc;>&pUau##n7x*G7
zFRaY7lFNj<hNFw)9@k&?G-fg8XpU)uY=T=DKmRmj+Q_k$^Ei(Yp90T9E*Z`>>`^=~
z#Z06mg)Z{!;z{B@BRE;=xljbRvglc<hoZ+t?(?g$E@T$tT+07VyjDtGf?2#$a4pXa
zK@;(PQh(*9%d8XA5%CuC5ie1irhH04OY4)#bfa92PDM|}Zsm_E{pzK<6Ad5h%vGPO
zbKP*VA+u?!nZDt7!!WZ0M*7-ZY710fXdcmfVw7$A%FNt)la006bzM!ha)n3oC5lq2
zX<F;Fo~YTXtW(>iW1=}z{+*nw%pQ?uiJ3BtlGCM)<V<A(6pkt_mt7@cEbJ|?LPTA{
zOw@z#3CAx^4&M1ZrkwlP>bN%wEf+19QkUM#pUct9mB>}j*~D?2RgCMtNPx^=>6JoX
z*cn)>S@$rt|Bd{$<exVuCw~{m)&C#<F#XK?wVySGiT88Sm&hO0pR|62F~t4;_A}w%
zuiw?5Ti>RBdGzo8pLIW^ziYl<`uy(W)sOeR&HwuNecr3*ujjmf{MPuH|BL1Cqh1<6
zN_$%MTJw9t|NbA@A2+<ydCmQf`y=<;y>FJi`}NBAxxkB(HzFTqywiJg;Em)5r*G-M
zB!4ITnfT|yA9jZ2|Ji=u`KkCXoDDRCXwAXFtig1G(el6L-?U%HKKs01{VwpW(fh_P
zVP74;WPH{9^6rD(hvbhop9Q{c{J#0ilMkj}+y1WjQ}gxMXPY05f7AaR{`v1){Lk{A
znO_#a`|z^oMZ(*(&+4CNy}kBa@P+(m_W#R&i@sxdcIt`n)4NZmJdSyC^u@1N=bj&U
zT=z))amJIA&mO(xes|*2rB6RzZ+L8bKlr}PgUSbePeWd;d=U5`^=;dSZ!a>Qq~3je
z%k25@ci&!lzbt%a^~m?-tdG5KMPI7E-TiUh$A-6d&uyPdzFP5a#T%t(5)Urly>wUQ
zZrc6$r;5+{o*sH~_L0y7sfTV4T<$eI@_P39*|c}oKR*80`ZfEL`1{U}pTA~*`S$tx
z_YFT3ex3eb@bBA~TOW3OF8wC^E9Kwzzp1}B{SE#9@VCQ{Gv6$KX#eYC{QP&_?|*;4
zGej}7FkSgO`FH%!f}bb;?PQqn|0u(vU;EzAez){J%j@<RhA)y{UU-}LX7S_ocb?wZ
zcHR4?=gk$@Brh?ZesDD4#K(&_Z<{>W`{?`=i3gLe|GXS<ZTFo`4;AjkT|0GI^IGy%
zsdGn8YM)QM%6o11mG_s7E+4<4fB*SI_XkUEUBA|N)A4TQU5C3O_a8r+_;SwsZEsIM
zGkY@kY43A}R})?sJd=OR|E=|l@!KuW&OEJrcjR;6+m4qkZ>-)Ne|zqI#@qMra=!}x
zT>S4X>v84_e<J_+vk7pga(!TZ$6&*BjJ=P2BYP$PNuehK-9r2Vb9uvsIRuk=j)>2b
z;^f}Ixlin^bb|CZ@jzh_u_UPoiJ!vR{0De71<i!hMQSB~ODz!DA&@A%TQp57P3^f}
zxo)q9o%}bEog#^1ONHKvyq9W_4wq?^X%o9D@SfjEWUcf<*~c=L%5$_TbXs(0>egy*
zQ5KeulRYUvQR%w8w^X6j2N?~imy-LHZfmSoo-fhP|BUys&`b#x*(-8PGW#VvB~MGN
zmz|;cKwnVDUM*OuP+_jZH-*Uxy7Gr)b>%0^f0R%Z>Jzb%*)F?F_yGGE_C5TuLdt^E
zghGWk@yl_);@TjfAbD5bL20R+k$AdbHGekW1zr=OdlIe^I>IM}b0pj4n^Ybs&6Ip1
zpu}^Yqn`aEyBWJa%h|t--_QJ#;yNMnQ)IWWq~IcM2X1xVdhT?7e{lnmM(&-QT3j!=
z!~~2)77PCrIKs7>^(SYI;8)=;5i5}n0ZX3cT-m%zLLUVVaNF>92ptw)EZQjLBQsT+
zPbN<KxMYruzVa=#GfI;s7Yp=p`*A<!Wf15U<P+`^$mB3&Vq>|$`Gn^$_a9CMc9s8U
zz6ySQ@+<rA=|4Ju-~ILZm-w&s|7V8EKMy}AeY)^1`CIR&iqGNSo_!8_r}k>elim04
zKW2Gb`i|#W<n77V-rvr8KJkOYr}($8o_O4@x>j*p@A2#>nh$yIlw4!K^6l!j8$~yR
zZf?85a>eJ|qw|qhdal%8<-1;WdCG-@m*OvnTrfM&b>-CUggb7RdCoRoG`;!gZq0q|
z+X<J8E_vU$aYy{V%RRSS=Wd+5p>h4^jgxl|KAQCM&Ku1)6W`l@lKC|CNBiI2?*X4R
zzw!R!WAfoy&#%BI&F?HMEq+lVT<V7GZKZ0}bcHG6QlbVD_atXXtPouya!%rb+##ha
z%14#gt178&RV-1wrm;yUNjp^MyN-&cr%Jx!HpMiR<ErASyOrhTeIz`^x65!VhAEv^
zV3zU|W)fBqc_;XkZ#T~r9xuM{eANQ-VrOK2%O}gu71I^!71|^mEj(9Hfxm=DlH(zZ
zGDib1H{V3=<Lqx3r~lpkN8yj*uh4Jz-~D^`?Xl_8$uB>Bdif`Tq3{2!e`<gHeysj_
z=}X_2)X%z~UVfbN>Do7|fB8(a8AJYV{I=)QzfXmq*L{inmhf%%kJG<z{3`gV{QJs3
zbLKhBi~raCx$?vQ%Z^W;pZ0v5^~w6vypO71c)zp%c=-L?cjF(wz8Zg=@~-b4-^Zx0
ziC=GgeE3o3tJ-(YFKVxkKc4W2^~v<7b<b|Th<uy;rv6#~Q|o7k9`AT+{kr7M=T|RY
z1U*xH;r;&E7oV>)KQI2&^!fL9r+*fV<qS>#F8yBrL-*_B&%NLG|7`op{OR$>zwZyd
zlm8_3b>%0$xA&g2J!-wT?jh@w@cYm2^gQ_c=;fo`4=&$Xbm!$gf%|0-rayl1WX03}
z&lbMWf0_JD{z=j^hgVbI&Ht46h5OslZ+E}k{v!Id=M(>D-7hoVx4%CBs_LcWE5A2g
zug*TLd8GA-<B8?-#y2@1?|<6(`TS?UkCh(<zMlR1>+{}^3Gc#Q2fTd!^62|JUnYMN
ze>d}u=WErM2`_8k7<_d5ZvJcK=d71L&z`>UePjN?`P1A_0bgExx%zq2=X)OxysCUY
z?Rnjcz*lB(R=pK^_wV)MSIKXdyy<$C{95w8?#I}V!Jm$QN&Q~<L*RSHmpvc1zT<hz
z{=wsG<&TR$qkal}H~jYeTidtE-#-2D`<?Rp>(7lp(|=F?<NjOX`>v0NKYaPh`Fq`O
z-=C4+V!lfK`2077xrSviODo4>Zawbv+#h)4czSqV@vw58V0+5)pM{z0f#5GONAdl_
zpZL9bT{uPA3OU;arVDN7S;_K}L6p&pQG!A0-@o5`f9QWH`Sjwm-<OqN^M7{zwD_*`
z%jthLLoH(sOBl=a|5tvw{%HPw|9jW>nC}@sjejlv@$J_#1}PQ+_Dh`iIgHqTGGAoe
z&-9c{o-303BX>LRSAmJb6~d<kXYxPi`@vr=a6`yO!c8hsqCxVsjI)f0q`$bF#2xA5
z3gXI5%B8A*)jw$zX>3q6RpM8Csc=B?vs#(%5rd6JuZ-=DR~c<HYB#!X{Lt*6MX!av
zMS!KURf+i}Lv0-qO)2e<1|g<~=Dn8o)<-P&Tc}xYvT3&tw$ZZBu|IC(X{l}Y&Y02g
zkdCXSm3o2NH#L6sBWkPF?3I&b=8DY_)a1U%x|n$*vnY!?a~I=N1}4U{44eL*__pf9
z<9Dy$GktpSS@`SBZ=ByNzApT7_sj7w&%cWPQ2P1lNA>sBUw(b?dSmv&>RI*EtY>U5
z9bf-^Bl+&pJB^QzKF)mS@b>aMk$0EgJbF{{`sK@cFF(Kd_<Z`)lMgT6|9k)NL%nA=
zUR`^;<Nd{V$6lX!Iq${c=e94RUopPTdb96E$cy|}j<1fs3VM6{t@qorZx_6i`%wEa
z@8i~w0iSk%obXZO^TsbPzdZfA|A+1`^`H4adw);+yZ?V3!|wkl87<jZIg;5^*&{gk
zxYBsCc^`5obN*m&;ZWyx<GUevP~?;NOUYo#t)e>xn)##!G=x8k=}YBF$I0?5{!%)s
z_+M_Zw1AYL%zC-~a?4~F%YK*3Q=Fj`A@46;EiEB?SazQL1%-aOa_MD~$r9Y+x?(Qk
zF_POP>!dQ}mMgtbaaX;lq#@@nwN{c%)=<$~*<ba8I+sSTN`xH0m^r^Y?|vQ`jtIu-
ze-HoJ{%7K!yT6`)Px`j^+mj#5{@O65F>5n1Fns@0@@wx;_Fwydnf(3CFp0U1?H2br
zzVF-y?1@Yve~<jy^jrR448sA2r~iEz9x*ihU-oDA&)452zYBb={xa`}#@`Qr%l{<(
z)ctbz1M8RbzYhHM{T=*0>GSsYMei29fBpW(+uSz>@BBYJ_;B!J;>R~{rCz^$ne_be
z<1-Hm?*~1sew^~;@H77xYoBMo6n&Hb`svHNFJ`}3{QB&BlTRDJO#e3N+qv)Izc_yt
ze*ORc#9N-XcCQ{jfB$^)^IOk6pU1yodQ<r6&$nsc?|=RCQS{C0=ewS>JafL6cKzFR
z!P|njgKuzLU2?_i`oC+lFBY7BbnN%xBZqPivg{Mt{dbS}q1%VA9<bb_yJPnDf4g?=
zU9|7*evJbf`z7|P>`UGqzB7FX-_E(aMfXbXvpRI{Xy4&Q`@Zis+x=+Qs@;F~2=1}l
z)v?F?pz*=WyYFs&xAF9bf=w5<e&6Z5FX`~n6Fw*39}YP1=|KO1^ZNw%OB`@Ju;M`Z
zA@*ZFCzqXMKIwB>=EBY^Pp&V&>2+)A&2`syUEFmp|3b>;byqH3pLXB<WywdrZ{pwg
zeSh@B;fL(^Ek6$Y*7+yTxRfQ8Z6oUg<{YMcCK=|pO!dtFSS8tu*cP+%b8X_j&#Ngg
zK~Pp?mw39AlgxXW)6%o0=1ba11c`Two)l6O`6;ze@tbOzMuEmI^$?BQYA+S9$}N&-
zQ&d&5RBlslQu(V^rx~uDq+PF>rfH+osCQdG+{o4BlySImiD|KUlj%3ZCW9J%S3OJp
zbc1L;4y_xi>y(TYPRjDhtdP=@SSczcI#F!11dqf7;e-5>_-6C+@V??b%>PbkfoPP-
zNB%7CX&j9lt(-cX3LID2zj18hStyh${#sl?G){Q0$U*TgscF(2(wC&pNCivDNh`{P
zN()Le%LvFWP*|aAr|z#7uKG+VL|#WmOVU)FN&J?0x1_gps>}@8!*X?UU!~o}-Gp}u
zT@lU}X%U?(u}|i<>@KNq5-!r)WcSLmDo<85S5r}QRP$9|qAsnfswAb9rCOpjQ<qWq
zwAM$Bi|Qq6@ybpLPvrZRxHabJ9??H;7->4+RK?iO&`R%*)@iNlTK<|7)wNV#C}k)%
z%XLU^ms%|4C0!^hDz{WRULr%(MCbtDJZ=`QBi!KvABA2CxbYq4ixoU193#vt@RD~a
z-*y2-kr450@g&jXLLd0va#eFoWZ%WE%E859%YKSgmX(jaip!ekH`gtWSWXWf6TWJ`
z9KIypFm7h9vs~G{uKeeD!?=prUD&>{S+eIcfB)P1yXt4ymjxe8KQ8+C<-PX1Ij^_9
z=y?(JYSOFf7weuJc_4AW{Qiym{r45`Z@w?_$mq$E$7dfYKIM6-`1<6l=`W>TU3s(W
zUFf@aZ<1elJX~=v<Kd%69FJB%-1hMDqeoA+Jdu2S>+!s2JujEM5q$siy}*ZK?_=KC
zzZQCK^hEAy*2{fwwLb`dTKC2E`<EXTKR^Dk`=RvX%#YAt_J1<|p8w<ZOY{52FK(ai
zy;py4`yt@t@{b2T>U@^_&hq=#ui)<=zs&f0^*hV2UB424KL4Kn{m56d&qbdme~fw8
z_(uKR#`lZg|9-#xgY*Zrw~wAPKh}My`>6U!_A`fPlO8ACZ@OJ|bKZ^8>xZvxxo&WC
z*DdqAW%t+KFSxJzVDkO%cdTv;-Tr<{=yu-ib+@!`ZNF3X@a2<vPi>y|JQaIx_EP%Q
zpBI5IT3&v9`Rm2=m&f03`E=pC;Ged?27h#ZHT_uhZPyp6&(}Xxyi<R_`E$*WwqNsq
zE&i?j|1#4iraS-c{JQ&N@(-q;d_Vnu^!{M~9sD=@ugjmh-^>2AGW4_9v;AN*;ke29
zm%D)X4v#92Ht$WoVEzc+D$Yu_@2peVY}pK16q!ylonz(U$m4jxzK1=Y(}_EhlY!;g
z|Gs~}|K9xL{;THuitn?3X8fG`efihKPxs%+zFqig-!tE*7SHxS4Ssz8{@L5lZrr(U
zcD?(W&$Z60mRH0s&$xK}{JwLH=jNZOJAL_d%h}fRr57YF*qk>ycj3(O(<@J}JoEe9
z^@}l=Z(QoRc;JH4g`?+YoQXJ<dXoQ4=%tU>XW#U`x&7vWTla1o+<kKQ=G~R|Y#!7-
z5WIin?*6-y_pKi0Jv#O<=b`o!-<Jh%o!(u3J?q7U=UFdaKNo&h^vv>A==&L;*}l4b
z;raUN`|_W=e{TCJ@N3iWvVZ6PtN*w8ck%Dle`gspSZZ0d*)MZW<c{PC=UK-Uz>&rN
zjQs}tebz5bHcaZQDIA&{X{=kATUkD`u48?|a*(Z=yI3GYc#d$j&?})2qI)F8WPV8B
zl2Vd(lTna4AeAM#LPAgSmlT)mD!CH*J@N{Q7nGl=W~lvFO;KH<QmU-1q^5XCfm3OQ
z+8Z4^!;MB=#`jEio5q;Bnrt(+H<@qp*!ZK-B%@D8ohC0#MNQoGYqjR8H7d_g6j5N8
zOOw%**)J0;yF=Dij!XWI{1k;53Mb@NNQ+6H7MBsfBN`zrE4WS2LR3;xOWIyqL;9rj
z6zOG>N5nJ5%Oygjo=P*y#7HlcJ}cuYJ6-y?c(TYNVL#D1V&38l#q>lhgi?i83r!UC
z6FAC$g5Q__2G1#uGpwGh4s3O7kt_m?%nTP8tC+tr?Pb{f@6$hf#wMm`jOomsobLs0
zi=>D=74#7}AuvafOW-v3D|R3DGR{5R;=Hr@&Il|N$`V!-IVNf%E+sA_CN0_{TqJmk
zzli^%K)3KwkqN@51Ro333EbzG<h#gK$!@^<glW_Nx<9?Y&i^d`@&1e8hZnCUUrv9{
z_I&%(#z*=08Si&KsCfA5!PNWO_xSHVyYu^Q#C_uj77y1xl6w5%!R9-PHxJ(UbxYu`
z$laf}7Tvgf!}WIAo#NY1Z@J#NaPQQEEf3z^k9_F#<l9rGXVaggJW6>K_H^-cljn+0
zBOf(9xbVR1VZej*`<L$d-Cc8A?)IHqPj4N(z3xutojJFvZ=AYu{dV=e)A#n?6}fx(
z{;Owt?^HfHf1UWV?JqyW5r%vJ8yNanwsEL&MzXDE&SdG~Si!^2cZ6p$*C)=;+%<d^
z{2%x~2-FGw5@-{U7hEpHEb1+BL)utwm)t*DBiZS)67qfWuVp=D&dK=5+bek}`zfxM
z^Ox0>QJ2n_ZkPEbn<4)~&PwK;WVocRRGah%DN%_DVh!T6BrGJ)Np6;ml(djckT4d{
z6MrG0EEOSjO(IT|QLv784$n(oS3WbI9?pgAFW9QsU$EIT|N6J~kJs-bKbQY-{c+>R
z@t=A>&Au)DnEuw`jpe(g?-#!_d^6|8ttTHJWj@;c(Dk9r!@vjq_ov-Ie81-YzI*!j
z9^QL)U*y52hX<Y&y)k@W`7Ywkhu8Do`F~vg@zO`mFU8-V{0#jS_^bEN0*2#EI~Y2D
z-~P7YEBE&$-zC0%`!w%;_dCrG$3F&t+WV>M(~b|*-g&<beB1uc@q^l@$DgNvVf*^-
zTf@($-zk4j{Ehg1_=nLC!5@h~IDTdRX8XhOH}r47pNv0Y|0XibWcto5#uCTE%eI2!
z0#_y%Kl?J4v#itDQ`!C4WLU2;$1$I0Zewv}d(COZ_ep?7a1I|AmkEpa|JXk+Kl#7h
ze>?T%wwKS|ntnR<ndwX07rn1hUu(Yn`c(g^?$eDA%ird`a(^N6a@Cu(_w(PceYf|G
z<g3gVy3bcViFtVcZu705H@I$j-SNNYci-Ut(tB!mm)v5xWpZcn-HCT@-FkU5;r7w{
z43D2ZjKA-GZ_-`sJ88Eb-zd0w_x92|4L5&Zc00fCto!-Gi{CHCTsU`j?V0AYZ_bvS
zDLln^GVw&*$@(+5E=;*{;F{-+Wj7b!+;rpV4bNMzZoj=Na{tfW^LNbe{dn-^(W=K2
zA3Hwgc-;QT<`Ku^2~U1MZGBex)bGiUr<-3{eenKr=F7#83*XIsoAP$aoBOZX-^_R$
z`9b0HyDzK1+I`Fae)&h=&rd&R|N8Zt>%TekYSz=NUszYMYjCFtM2Wr^7n9VL+9I`B
z>YP-i^ir9A*^M&1Qla8LVlTuMrQ2j2q>9A&L>NUTh)Rgpi!T(PC$U&^qvUDHLTL#(
zZ3PL%SqkcM5z_ai_+@OR&BeWhr1{tK84K77F6J-ie#Mr~tjMtc&(B|Xf3y5M@?U|0
z@Bi<AVNC1T>$tnQqdB@*mD!eZC~@U-DRbFz_Hyju_{jNy$3-AqaFKwUfEoXJ9!u^(
zZhjtq-dx^8JX3jD1dj{v7oI5OB{WTVj_7mIC&F1m1;SdQ`66xtySQg?cL^|wT^Bzr
zsvuO$zm@MX?@8`YoOd|&xKHrD7pxWImzpV~BUdQrB6~ziOhQQFkhGIRs#2wrhthVX
zZsiCSY1PH5wJI8l8)W}VpOdbTNtG>?oh!p4BPFv~MpG_S;jY2~Sr&=sLJk7w_}F-L
zx%Y9t=XlG}$FYU2gGGe-A4Be6`=23S9Y25lGXKZEpWnV+{bcna>_f$;bzkDYDtzJl
zB=T|V$CIC)eZKb9{D;fWqu;~7RDC@7(e%sFFJT|u-bB3o`Ev1_b?+@dH+&cVUHm8D
z&%8fO|K9#v{`bo-_MhE9KK*e2)%eHbpUuAoe*}M<|Ni%<`Jei~AHSde;QW^R#o~+H
zSO4$lerf+Z@z3?Q`Zv2TYrfh2eENg)`|Gcc-z<N;`8D-V=-*}kzWw)Oc*oGj#Kc^}
z`2PRa{~iCA{WJcz_Ww7g&8*d|!pzecLzxm-ma#c=IB-m6OJwn5KEZN;V<ztk!5U!}
zp_P2UdEEGx3-|~n2^|tR%E!!C#+S_>EKnoxivK0QvLK(Zil~_Qeu<xwe<W2T&Wf6g
zdWs*Aa+4L7Rge~v+9Rba!!LJ2{*QdB{APt<mFMbbHIHeoSAU?KEzc`0AW<&iBb_AU
zAt@rFCUAo<hrdlgM)0h_5B?1Pe*VP*eL}Z|Ckb8U^W`dK)nxSg|LFg1hUpBo|0;hQ
z{EGkm?a%hVd;Z?~WBlj!pP0Yaf2@8h{$BY@@Mrut<1ZoKKK%(~`o_e>sQ5qs-?sm^
z8JU=cndO;g{-5$E?|0Ile}AfftNsl5G5<%zk89uKex&|%`nms`<0s2^^WRK=v+2!=
zH?QAZcysGb!`nyi-h2r9EdBNKmp30Ly!Cl)^k(f_wYMR!+MjKDto(S!ljqNVzgYQl
z?~BgocF%V_U-KgTRpFbbZ)d+Pc`f?#(TkYZ%I}uG`~F7aHRJ2YZ~nb|^M22Jk9SL6
zGr#hBZT7DJgX+g~@BCg?KKg$9_O<pao>x9!3Ow(1y6lAJamC|u$C;0PJiPj#$AK9K
zcn@VCzH@lzA<qNX_Ui0m-;=QK!@-)vUk)za$GKZ!XV=ctJ8x~Dw#9KH?|Pf{hd0dF
zD7k6(Ch1M<)=ycpZdK(P{!M?k9o){d?d+CcTa35;-sZclaO<Y6#@ps?Q`xa>m-3!H
zyDN5u?Nr~Pw*C5c&fV7gIS#oVQaqS;aMh8WCj-t+zxe%f@wJAVd3QG7oAY48qwpsZ
zPt~8;Kc4qA{PnUATR!dnX#1Z3ZT1`gx7zRhKhFNd^!eXsk8h?wGyh03$g`|ux8m&Q
z*vZb%Y08tqSIR5S)yJ;HA<CV`w@+ZA;0u8s{>ywk0>Z+4VshdOL=}aq_yTwr^P7t7
zl@O4wmO3VJL}HfoP5G}%fyz4-n-$J0FetGpA5kt<kyLe2y{zJ+vQ8yRjaTEh`g>I`
z<yOU0iqb0nYWGy9DfP+8%B0I2kUOC`NrhKUO;u0nfZR!$64|NpQVR95QIfJ^(W3og
z8sc@Le}yasIR$L^=kVJI-Vjm|b`%m2yvhHQZyA4(&<xQ>;?WW(#PcN@q|@Yum8K|p
zD3&UGRt!^NRC}g=PxFN4DYbsZ5E)(RhjI&*)RfOCN+<})XDfIq|5OoCl~*~X=qNu^
zZmWEp;sYfm<rKvZIdhqGnJ@)YHA5{Utu*zmDqJd2Dh<j3iZ|qfWWGs0lu(yamkU>}
zRby6rq%5awrTSY_RbRr$+GvJBfS!i#U%h;z%f>eh=jm<HvDcli&tRl(%xL`GV404J
zMwi+V4Ncue2CEIO>I!H#XeH=u){iubH@dIypz}=Qv6`S-t?F%+c4b8cN$GWBh9a+p
zr-`r%tMSEf_ON?#$a7udI?KVw=ECaF`ksZAO`fxmTY@`-V<w9uWAFdA|3?}4nPxJ!
z|5yHZ`=0<~5VH^qBh$veoj*IiH-9ht@$Kh>U$#GIeR=xs%Bu^nYTn)cnELU-Th3RL
zpQk^sfAQzJ*wa@Jm)sAz$9G@n(UPZAo=<pg{#5AUrn_733O=-ds`1?K>HLSA?`J)j
z{!sSerhDPH58RNw(R-8k?w5z+&%&PfK4p6(bN~GPf+x-|g<reA`u-yQh1g5s*IV8;
zzu)_Q^Lv$#9A6y1&-wY|=j0!Yz8QVF{xRs&{%@&&WEdtgME#5W^YPE}f93xL7@jgH
zGTJi6Gbyls<h;qhM|hfWp+Fu_G-ncr2xm6uTy_uU^?$v7^8Wbx^TnU(|K9yG|6lN5
z`hVE}hYTuA<&4Vz)Bbe+>ios==jGqX|8xJV{rmcR-LL(>B7cAWmH3_Eqsg16uQT6H
zd3*QG?$@VZ?0J0XuJFxIHx}KU`q1b>=IsgBUtBwI^VHoF_q*>OyVrL&|4z)^BM+88
zdGIXYS=HmR2g&!7AO3w>^y=g5Wv|3u=Dhs#`uMw}@AKc=zyJL%@7>LJ*&pwHmHwsn
z+wjM)&s#s1eUksm^?lyA1z+X9Ie!1~ed5pFKl%T<|6cn2;rGISJj?~`9h{pu^x3{M
zo%{dpUo>MU%XHQt7D1*Gh6@a*nNG26Vm-vt#aQxJ=QqP2f&Vc~KbdowBN<QsyYuJp
zp9lZ`G9F|;!o<vQ>hIpa3;wGyYA_uAJMmBcAMHPVe~kY<{BOh1@$cHNxbMy1)_t%0
z(fD2eOXz!z*B@S}y?Fkd?YYhqmj_k%{2#DBX?V`@YWl0L=eiH$Z$(~Pe$Dh&>|O7>
ze{SX66ucF1r}>`G{d4z@-hF%f`OOVCSKscuYjbzjt-R}j*Hmx#-Zr~u@PO}u-`(Du
zYS#^~bKXq4<$P2B+WgBcm#<&VzIp$S`u*7ZTz5a+D8738is<#@H&t%8-2Q%B?XLNK
zxkp!?6g-W5eC$E!gLMz>AOCss<Jp`S?5~txC%%q(qxNCSms?+-f0qB~_CD}^#RrCu
zKR+mZ^!gz1ZqXZ)cdDN!{#^Oj`ETB@q#q?eY=4I{Ol1AV;m`G*^FK#0XD#;*UQ@mt
zUJ2gmyy<*V0_q}f#d9UkO8AINi@g>*E;(DqL5^E~o!os{Gx-_Hi!=(gH)!!_+*eQ6
z=`!jv%eG*#Twxw>a>u|`Ki;t4Y>QQ(RkGO>qY6VV<3lD=CQA+5^uzQ*^o$Kmjenc)
zm{b|;(r(lU*YMCfq!X{VOIKZcwZ?q)9qOOdKdMboF;H5ka96%n-cf#v+#=a0(!azN
zghd4Xgww@~q;#b}OGZna6K|GyD0xoGT1radtSG0bgV-sF_0kox4`jAV@k=h3h>+}+
zu9G_=-ypX_`m1=7h?mfOK?}iI0$%(HylZ%t@fis{7WpsYB&5W5hbxP7J7*<#H*XZ5
z6mKHeZuZ@5bJ_QC1aPcjd&)YQt%7|!hZpxY-ZQ)sJmp*(TpB#L_ydKu3I68m=9b}%
z=A6&%!@Gw^mWz?SjBOt09Nubv5B?jx|G9YC3z+{i?q#lEzrZ<(!;bX{({(0smTfEn
z%vlVV|Em8H{4Mu8@=wd3nLoFE3Hc!ZF6CY8`zh~J-%WV)=B3{AcTcUJt$t?ktn>-f
zqXQ2NAEi8PdU5@g)*H4r<*%l{c>Mh1i~p}~z0r7g?Opx5?6-1n9p0UOU-?1pL)ZHy
z?{hxf_z?Ah<Nf(J46mA>FM4|VNz$_iFZaBWdZ+z<&xhR4a^K(mT>H!R*S{Z1-#2{w
z^yBoORSb)nm>8e`Y5$r0WBt$lf1Lg&|G)V6$M46#+JE={?fx&#ID;vKWg}ZLr#8=f
zzG{J!d|ljOT#eixcoPNFL~e_25ql=~UQ|ddUgET*zr+HO_x$O+`~n+A*u_jm_6S<>
z_wg>`-ORT~fI;NX+}}HXiTpb8v*V}p&(}X(f2{ew;`^HK9N%-kP5Jin+xc%*-yFaB
ze%tcx^f%sbLSNgzJpTOX6W6EX9~nNCe%kzL-zS&POTHw3lm0R3XU{L2-yeQw{aNux
z^{>yrfB%m$eq#z_-pSm-62N+hRf(;c?Gl?9yCM4nwnnz;Z130}v)QnPuuf!o#;nd<
z&-9zIhEb2vm~kV+zkk|)#sBdAUh=c%$Di-tzH9xs^L_faq_1jUWxh`Q^5Apu=U1Or
zeDe9U>SO%J*B@7XV*0H3x$AS$mt$X5zdQV3{+ax9%FpvZ-~2rM^V82&zqEeu`Ss>!
z)6auHdw=!+j{3v<m;0a8|EvG=87vqhnYJ?DVo7J6!n%|7Evq?ODcdSGH}-fAL(U$~
z6PzD7?YOkKZFwH?T;a{<pCB+(AV{EIKuGYJ-~pi!;bX!UBAO!U!qbHOz{4Atg&qre
z3de}-5<M@LEnXv@FCH&`TkL@7X_0J^Op$aEJ`rc(jY4OIz6s3|k`nqX_)T!9AiLlf
z{ug|ze73xcxm!51*$=X2v8XXKF?le4WYA`O#;DEI!^q2M$LPxVf#EenEaPQH4JJvZ
ze#R6=eI^&?<IHE6H#1LU&Sd_}q`|b1@iSu*lLylu#@&qVjAe|*jP(p_|4aUF{rCQ_
z<=@IbrhgXwx&FuZPwQ{N-w%FW`}N`%*Kgb33xCV~vHSDlcgt`4-@AWZ`nl{!`S-+c
z^S|1Dee-4JmxW(mf0_Gb!RHyD_I<SfIPoLrrzaoJel-4M_^Iuq_Q#@+?>^>y>i)$0
zsqSOm$Bd8sA6Y(b__*fN<j*``{(aW|qW-n&+otc&e!TzL_3OZ|pkF>eBY!;qzU_P4
zcbV_i-z>f@`DXLI>HCcDH^007nD}Gw55AxKe(wIY?YGe%_CFJTyZ^TT&GWnT*Q=jv
zezyIL`nlqV@b{drIbUKvKl!-p!}<4f-g&&;`TFF`>=*GbcE3n}vGn<^XB^KIp6Wia
ze|+%Kg-7cieST>2@cjdqhhdLyKmPC}_vxyq>z;-^z4WC0$=fH-pU!yJ^~~*={IhFM
z%btoqJ@91ilh02opLRbre){~$q$htK|9mv<q3na__a5ARa!2k?-|fKLOK<DnNxIW`
zN8s-FyTbQX9*8_ldSvos|I=yDVxP}?@$u!aS9Px^z3zCu^EKNW`!~nlFuz;!p6w(1
zr(2&GzP$bF@<aDm?eFM6_x{NJmH2!45A$Eyf3g1+7>XHu86N-N@W0~!^?%v__WrZ}
z@BIJxzgvHo|0(>f|7-J)wC}sW&HncM+obQVKNx@N{M!2a*`M0Kum7(5_w>IdV>;7G
z=2@)q?Atl^bKd1j;Bn=X637>f6N(p}A#z%Dx0sK3t9ZHiH?a+(CL*rFKZSgS)kKbp
zh=@)RWfuD;YAB{7K3RfU>WS1LsUWFZDJkhU((<y~<-8P96`v}eRdiEaqF|tKL|#|E
zUT%SGwM>|_gOt3amc%S^FY&`-*F}Gd>=pSX5-EB@R7K1|OkM1d=pPXuk*~sAg=>UW
zgmZ+Z3vvlo2u$U_$2W^FlJ7t75#H^*ZhTAvp+b5h`JxVD-C`wT+G1NpcZePkyC?ob
zyii<3{EOI2F<$W;@pa-?#6={WBvd4HB_>GhmzXZ$BGDq=D0WV?R8&m#pvWzeFwt3}
z>Y}%VUkZNVPvu+4<HqH|A;kWHwU0%Hc@kqYLn?zf1Ji%Ce;@wN`OE#c<B#N@N56$Y
z9k}0@emDQQ_}B0MVup0aZpICaD;eK01Th?En83J$v4HUm!_oh&|G)f``5*sZk->w}
zh-n_vTqb{}2aLIlYZzSqPyHMAr}y{6Uy;8!f35#n__O1u+fR`nC%)-_d;0bA*UoPT
zzd!nM_h<MomS3-bmi@Z&>(s9kzxMrV{N?qF_gB|X^`ACBi+-B^-2J2e$H5;He^&qc
z_)GV9?eF;C?Z29TPWy5Cd;9mq?=!#EeC_(u{&~`;;~$@YVEPdJzV%(}+nzU5Ur&9t
z^5wA?U!TW3=YB5v{P6PwFV?=C_G;_vhBx_dzQ2C{I^fN_H(TE3zWe`f+k5K|3Lk#G
zum7<8qswQdubaR9{2upn_OH0#Qh&_<ru<9#ugox;;W0xIV+PYK=CdrTSS8t_*n-%e
zvA$w`!p6jLm?NBX2Ipc<2ChghH7;JRSzNW;_B>a4ym`;_`t!N+<?-tB{Ng&rxt4>6
zgNZ|c!;#}42M_03j=StK>=)UtvTb8)V4J`;iA{vffR&Affw`aQBcne<`9F!j>wYu;
zO8xQj+pe#wU(3F{`1IpL&HEYe4Br{Qvw5ff&h?$$yX?36Zz5jHz7Bs~^ZNL!mY0iO
ze0%=(nesEqXOYh)KYRX6@%g7`Q=dsblY6@HvBx8+hxhLL-|M||^H$nTlN&#;*<EG5
z{PcqA`Q>L1pJ_Scc;??}(=$?MzMlGd^6tsACs&+YcvAVK)rs53Pae-bo^+i3#G(_+
zPPCs`cY@{Qrjt&mr<}cVe#%9*%hFe7Uzv5~%$3Vm!>+sBn0q7YX85gJw;XRL+&O>u
z%)RCJcRVP5xbGq7Beh4p4`)2segEvei+4HhPQ2rFNBxe)oh^52?tZ_!^RCt1(z|8%
z%I;e|Xni>4@sg)*&(FSC`^xaG^oO&b?7yD>{`;rF?;pPz{>uJy`ESqA%dmxEE2BH}
zDwYs78;)+yA}%X#Wo}n4Q7%?)f1X@kX}%^tLH;8CNd7GT*ZdU%DT2?1Ohq(B9mKT6
zJH(fX&lJxPpD*qw@lRrt<Q%Dc(hFr3<-_FV<nPKqRPa~)p=hq$rNX9Grf#Y6QDc%;
zw$3)){d%kQ*BY=Iu^1~D*BWm$$uxa$;%Aa)vdd($Nrj2CX|b8E`9t&T7R;8h7Aq|(
ztwL>lY;W3X*}b;0w%ljRX?D!wspU$`O;*3GldLl>d(FAbt1KQ_uCdTD%`_<2a#h``
zJVW)nhLr9B!}q3V&GpQDji(vO8)fRcs8-6|m6n#fsO+J6PKVW?&}fbUgQkw0gJ`*6
zJO5j*!)!VnlX))jh4McXm?H97@~-Sj+4C}!<vuBVP_ULalU0;iA`>kmBiSliE0Q6)
zRs6lw2ALBwKC;ZRn<YiXVnidv<i)><d5bLOi{O~Ve1>7x|J47h8Q55Uvstq3WUBlB
z>d%IsHeY7Fu6o?@@b{DDFErll`WXG)=;!UPvhQl2?z?Y!XT}YQYwp*~Z;0GJc6ZLb
ztos&^bDzI?uJ)|>X~{Fc7iV6GJ=1^u;_>gNPEVISRJ*(2*3CPO59dEtd6Mz$#q-3c
zJP)tm)w~mU=f&Oi_hjxS-ITey@6w}7k1xw#<GcOpLD7@_PmVv*dl>vk;_=~otv4QC
z)4O@*w%mie&wD?xeGmG1`&+}u=dT@}r#}&TT>kL!gNi2)Unai^e%tWD<*UKZPrpQd
z1^<ls<H0cJ|CQgazqbC)|10=+_0LmZ?Z2)bJ@jGp(1+1OA4U&-=pH@v;oIn;52J@Z
z4BJB=M%Sf{u1gzTmo~aCZFF7Q=(@Dgb!nsP(ni;%jjl@@U6(exE^TyO+UUBp(RFE~
z>(WNorH!sj8(o(+x-M;WUE1inw9$2GqwCT}*QL>XUD|%h6489&DxqhBnnI4kIU)(7
z>|#?z^M&^deBryoo6EC@%bIf+$8XMd?wwpq*kxJvGi>{N;P?AqPQU;C>G-GeZ~7me
z-(^2vd|&d-^{derfiIsw|Ng}Csr_T>hqrIlUtfL6^g91t;-|}B`o8vlE%~zaqw0I*
z_Z=U&K0bKg^>+K4FYm>^uKaHB-Tmu_j~m|Ry!!WI<xA@~P47bA^}msPeeKn$*9YHd
zyeW94|MJ0$gqKyXTwm{ewc$m@vs+Jlo^O41|LxBA%pYRkC%!9soACDI+a+(M-vqw$
ze_8u#(!0>lr@zkmy6m(0hpVsqp6fsHe01vJ{f8GGhClRwz;$2Xp2J<UJ9}<zyK(Va
z%r(vH#n;`hMO-;~sp-<$%R8@%Tz_~y@rLdV*Xs|i>Rq0BZth9>W7CdoJaX#j{$qv5
zZ=C2lb>Y;96RJl;4n*$Rw99Vyti93)9~?e@Wa1I;BP))aI$CpV*3rv{(+-s!EI)8>
z-=jTxyZ7x(+Hril&5q)op1T?Mp4$6z?~gtCyY}p8-L-zN;QrhD&g=_3sB%K$!o$lY
zm;YbvzTkQO{kfd;UKew(Zok=ayXE$T+f8?--<$AY-vhA+jSoIPWPY;onZrxb*ClT%
z-%fgW@q^44mhY2)u>JJ-x#;KqpA&!j{hazE`}_KD7T@>(wEx$|<j8W3`8U%=Mmq+{
z|EmAjF<fB2#@@vJfVY-!8}D`=OP<Hv|G9iQcd*}M|G_ED<Hf_wEz5P0>n@KH|5pKd
z;cH^u(xUPo<qPCuW&TRvmE}-iQ~agKrLt2^Su;oLs%E?TH|0iocj<W&C&U>fmr0Au
zevm#UsV&hazEM0t++MUnI738Ne4?bB)H;c|V!uQt3a=7s5)KfPmP(cVD#t0GBquD}
zEYl%ZteB%@qx4mYQ?*b-Q~QbLZ<T}c#nR3ayy89LdXnifpX8er(-c1|tW`Lzut%Xq
zzD@41`~jtFs*xIeTDP>KwNrFr^!N?e8qYV~Y9?(iXU=1G!{~wDWSs}v&DsUp&N@Hz
zej0~cBw4VTZZK@sx6_N$&C$u%d9C--NZX9T{I7|&k*(oPLv^EThS7#<hO9;v#=niV
zj87Ui7;qan8n_ww7%&?88ErHcFo`mntba_`UROugSeH{bK{sFjui-hP93v&8`v(2G
zv6>%LgH-;iTvv-yH&?Ay+$_6H>bL}hguVC?(a9peMD~lRN<@hBi-ZWg;cerY$o-wG
zmaCXcm|KSX4(Bs=ZPs3<+l)6Eum1n~=iBetzY+}d8MXiK_!<2r@q^R54R2H4-hXTQ
zA?s7prvo37-vz$8_}c6Bo)^W>c06l-zVF%7C(j;LKD_ZD@qzq<9S`n4Y=12CZ1M~D
zSMy#^d%f%Bi|0wt_@6jETKVYeliU}FUvGXb_%iGHnP;6(4?VViB=Bg-lj$#1Uq`+^
z^U~(Uq8Hh(X1(TnbNP+@yG`#IKbC!b@WJf8`diO8hHo|BANaWLOVxMwUpc?0{+j#a
z`1h4R4*r_-*My;osf2YYhauN>&JP@$*(b8TWV*-@!ywM|gr$n@B&#pWGUjULe@ttc
zmNHc`DKfXPp5U0xy@$t>`v->)hY)8t*LCh>o)6r*JkPj|IS;cgWD@%y_-E6vm){?J
z$^Fv*_3u~7ueP5izkT|`_^HDqt_O|xIPSUKw|yA!$l_7(!;S~v9!z?8@4=J@OCQ-h
zJ@<6olj6sWk3$|eKmPeh{E_s-c@KgfPI+YY*#5EgqdoUq?|!@Ec~|2e-vhPB{?GMZ
zPkF=iX8UXNHw)fyy;=Xt`nC0Yn=doIZ~U>~yUn*--+F#5`+53T+aHgAq6|+MtC>Ht
zIIzBBmSVPGna(E4HGyxbP`&74v4<img1mf}xRW{0v5RsvajNm`;&&2K6`myQAe=7v
zobNJ^KF=%O^8&>}SwhDJ-30Fm*b5}`C-F(}^782MaPxlUnaRb(*1_2N|LlKe#@~#&
zjEM}fj4sS-%sPy<|NMSG|8eyD#&7Q5G=G%-7XSDDU)taAKh?h-_;mmM{I}e14!(|l
zyY~IL4>BJV-u-#K=as~(h?l`HdtWKPHhY=;^zOq&4@w>yKHB?m&I5t_nfE^3KlyOY
zLzxH7cbD8;f9?I%O;=Z6eRYlNrowIAJFT}STr0SA<eb{+sN*4r9S`arzIZI{M9OiY
zqc(@9A67itan$1IqNCo&jvU)`-2CM9Q--JioU%F{a+dpI)YZQ?+wQ475`VJhvC+fr
z_oVM?-j=(mcr)|%#(U0>zC8K!H1FB9Xa3KGpH6y`@^tC*`>&S2^?1MbL+2;1FY~|b
z{c`?G;n(ZmkNuwUzl>=C^J1opjEkAKvhi@%an0ch<1*$s$k)#o$J5VM&AFdbn@3b2
zP?S}sQt6!9QH{kKaq5X`t5vtE*r|l8F49QR+h<a3@zcWDVzqgyd7Jrs^DX8P7MT{z
zX1@(5={4%KX)|btYV~VQ*9_CH*Slb#VQ^OGo%$Xn4|!iXSvh986>>5Pj}*@;wJ0u;
z`yjnYa)Wr9*m}_-v8|GcvJP@$GOr{~iCqx8B)(sKnpl=-n23V#cfoZ6hxodA1i4>v
z@$snftq^!4q%N{Sc(veWz8~CAxfHn%a!GU9a&z<D6bKcn5`4?|m*+WmIM*zW&FnlJ
z)?Br`pZS>uT?EYpR|!55S|IEy%p&|*c%tZ2u^jPbVi!ez3eOaclb9{lF8xwUQ_4&F
zs?1mEV96NqPO+(CGsJl$8>RGQw#e?34VMv+_LGv9bP^8{{U+=y(kG@QSt7Mh@}Zca
z@C*Sx{#U$pJT2V$+;_Myaw~EdbD46vaJg|YbJcJvbNX>5^U4VX3-t26<hJL$$R5ZZ
z$!5)Zf@K?vE$e4i4c4zr?F>i%zWDX)$D!}}-;ICn{e9#2w;y(2cYccc#PxadC+?4Z
zAA&xf`FP@E^2d|!XTH^aQ}8<F_1rf-?<Rc6{>1zF)W@**i{89_b?DXMSLa{7fAiu)
z)tC3*UVi`b&ExBr&&r?vzZZS~;Jy5(j;||z^!-%-q47=a>*TNhzw!Pk{89F^{&)D_
z^nWJ*(*Laa_2yUgAN7B`|Nmw<!PL&WkFAWQkm1Rn)L(Hw_y2tLtLsnUzqbDu{~h_$
z{p-sQ>mS}fj{XS#_3!UTrYv@CE+#G)_Vp}lnKPI-GN-aEX1T~RpVf;km#vd+G21FO
zWp*_VUe5oVdOUUfv4UUt!+5$meA#BQd2_txJjSiYw^YDSc#&wIn3I^FXp!(Xfi=AA
zxSBbPIUaF*Vh>|sWhnV;@YnsH)&GhA&oFQ@pJg^;TJ*pC&!iuVzI^>4|L)t{#UH}I
zO#Hs_r{gd0pF6)l`R4ka@5h`UMn7YJE%}r6|0u)r|5N`q{QCAS@Uztiv-kBMAAF7d
zx%ns4_mt0nKBRqk{UPGxv5!8VH+|jlo#W@yp8-EjelGgO|98rN6Q*+3MVxndZ}IE!
z@o{mpH8ZO-{bfACEX3Z&CB!?8cLh%@&pw{F+;=#$If6Nzc;^e{i9HaL6p;{^%X^&H
zo1a@yQFxl@2k}*siBcj`!ID!XCX0uN?G=?4vk+e-`AKGxoQZ6xWTS|opuON{Ax@Fi
zBHW_8M3P1B3I7+Y<Dbquo12|;8QV^lDa?T^A6TnduQ7Eqa4-ZjUS(dx8qU_sF2;G1
z;~-lW%Rk0r|KI%;{B!r0=kKe3djHw~cl-PP2m6<$@4ep`y_)hu@_EU#$<K~F+x^Vy
zncmYikG&qQxa)KC%2n>GmDjUxN8fvNui$Rm?W$Y<Z+^U?eSOkZ-YZis@}J8%m3+Ma
zD92H!qpyxmJf?8G?}XRs&a*~mPoAtja(DmTy<Gd!_U}Bn<j9%hZ%=8SH9A*&?)cf|
zrv*-S94|d#aVF;C-m5>ar(b__CG7I=%k!`LUXQ-{;`ZTtJ08kENqH*qjQjc9=V32a
zzS!|H_|4J}ZeO>3-}lYubJBZ;w}x+P-#_}a>0AD<lE3-?^BIB|e*Am;XWFm$AHm<R
zf8YF*<L^R-3ydWUpZ+EN+xd@;A%HQSse?s?J)M0cs|ZUe^9trrmH<{O_U&Bu{0D{K
Ii1~>F00~zYsQ>@~

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/10cdebea1659c21a0248f88654ae41f62786abf1 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/10cdebea1659c21a0248f88654ae41f62786abf1
new file mode 100644
index 0000000000000000000000000000000000000000..4afe44dd91c3038dfa26d06730b2213c1d98dc50
GIT binary patch
literal 58657
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?)V*Bt+VM@_t0&KEpRzyR{6O--zK2U6zk8hgnB%d*qjwMN9!`HO`s~H?$X7jY
z)jquYu<X74TZ`8zFRPzTdR*{$$<uewo;+Lq{Mw5N&ksFb{ovaDllRZvcYN6Rgm*4C
z10TaE7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C
z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R
z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7
zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z
z5Eu=C(GVC7fngm2%KxkWe`kzke!^VKmd+W<^_SC$<1*`ImPD2-%nq!IoT0ot0=$AR
z1WW~f@b?N#66g|`DD+iyy2L5T5Ggz9v(j&+K1lXSo{_jOaY%ZN0*mTu^%t7ow9jkb
z)AZMnQvasLq{*!PU+bi1s0OFHm)bnFI*s#Ml{$L*-o~}2e@u>=OfVHNJ!QmasBhSB
zG|?o|?3J0nd4ol{#YNKvMmG%J>R;0TX~=4J-}1Cgnq8mWC%a<%y|zu(YpnlUhg)5<
zm}8-0p=@4f#%Q*~<hW6S;aP*}hR;llEo-eWSZiB9w@^2&G|bUorGMLCrD39xvr)Q1
zpI(D*h4u>#Ta7eLJ1sjcZ!KRfRV{ICEuDNVWA&+O@fyE1demR5^=dF^J=7>xy`*qW
zrcY{-#0IhFVpWnZ(v6byMXiNpgd_wm^DX0FE$~_3F#mj>e$H0*yDZs^D*w8Eulu?0
zd;YhF-~aw9|2y-a(LavA^}m<?IQmuWtKC<(FR7pEKh(d|d;RR$h9^u<e>|D+xa-ln
zM>dZZKX88V`$5tJwg*m+44x`HU;QHaCErVtm)~EezC84F&qI~_Pj7#{;dk@WZSVVU
zAK!i^_ssu^$m4>CNA54X-}B(m1D%I<j}JV_fBfJ<;Qh1rEbcMi^|~W=SNDF#BcW%a
zFYKSoJzem)?a|hU8z0SklJr#Ond)=PXT48OJh}Gt<kP^X(a(fly1w>#WAe7*t-#w0
zZ_dA7^lIkoOK+pz*L-06l<>vrTin;jA5-6*f0Oz~`L)Wcq&L$(y!^EG(~gf;ANxKC
zzn6aZ?rq=OU2mtqTk}rrUBTPVH=kcKzlnRD_VVxZb1&b$UHYN)<HwIaAJ4v9_uB1c
z@(ab6k+1SzbG%vfy7?8)%Yf&;&!#;qd6x84^4Xb}DsQ#kdA+~%k>&G>k5fOid|dd+
z^z-~LKHpA%yYh9)C(pOFFK#`Feb{q<&cj8|OkbP6EqeR>&D7TdueQJFe}3$_&da|q
zx4m5YBJ8>Pv(_i^k2D{=ztenk=8e2t6?Y#!c=-6p)9+8GKbi9={sH&>k_WAiH$JU@
z=JoW|!@hfm?v&hFcgOPX<@+j6RzAD^Y~?fmXHL&%K0Ec)<B8ytqNi-n?>xWpqV`qi
ztB23So=ksm?#}<4J8qo0x$SPpquozKo)tc2fBNmo)+dF}_Pm<#uJS|1$IBnBKk|N9
z{QlqjneUgqvwY|OF7N%(kE&lHzOno)|84W<=daa2m;G}8`~Uw%rvEIdtQ%QoGIKG-
zGSvOoW_ZhZjrlxlAp17<HV#g%Xf7MB5}sKCY$8`hCyN_O709T__Q)tnhe$e$zY@7F
zyhKz@GEl}uu2$}x>`xgbnQPLzvJDE;RW_(AXztQnq$#0cp*lgSL-Cg42_+L%Ce3&q
zU43DrCB}b@QVq=vZs{i)954(uK4ARINXlrXL6F{4omkyCJzf3f24{_oOlFuYG&L}v
zXzpZQVjgEvZPjR7ZvVtS*umQIu7jc7GpqF$?&g(dyUfm*FR@g&`CzMPx6}5Xt-hU_
z-F&+|`_*<kZT?yt+Pt(*us&w>+47!6y?MFWX){fWJC>)d=i4r^6Ss4;aj`mLanW4d
z{E+EV<9`N5x{EZfsr*#DDbFbXMy^xtmn^s3bh!_5Ch}SGCUQ5Vze^pEUMyWFHBpK~
zCQ9axq>|`deo^jDj{9s2SeLTB=S<{T$TN}q3g;j86>N`Lb6L|_8QGSy<*+rdu4cKx
zyp5@x@$Y|`e=&a!{eJdG;=d=OJhK7I7v>903mA7Wh%s#a-}OK6f5kt|KQTXQzU=#C
z_PO-))K8tCK7GFU#rDg#&skr#e^vON_VeZ+Nrt10Nlcl{Us)Eio@FUywqlZCeDHt$
zzd8SU7%nmYVcW!!!O_p&%O1r(n=Ow0Atxh`9`8)P^ZXY49lR5H7IXV?Ze`ob`iyN8
z=MElq{&2yW!UshEi8YAvh;9&05SlM|PjHIhZ2k~l6`m6A3a&QJm7Ffz&v}yt>IB;b
zPYZ?%c?zisg$qlHO_FGkl$7$2UM$Ti<u1Nbq(!J&U=^Pu-#LDJA#RZ<kqTjJq16Jp
zeD*wE+)ue<c#HUq`FeQFxE`>lvQB4O`(N<yj^Dw5rvCl?SMlGwe=Gj4_;37w*T1fR
z&HtYL4gT}#=bdk=pMSr5^y=I52ha9B<9S~9BIuRTtCSZN&!<0^etz`n>?it9grB57
zW_a}CLC*ue2jTbZZwFoXxh#EQ+BwbhaTm3()L&b9J?8q`s}Wb;UfOc8^Wuz4yqAw%
ztU3ScO!ui{Ct{CpI9hmE?I6Q}v;$iYmLL9bbmEECC+D5`cC72@^~27G>JBm;EIrV=
z-(l~YouS*)w{>pYxqaF$>3z@lU)rC(-+X`n{;3Cw52_zJd${}P{i7a7E01nFcKi5@
z<1t6y9(3Jrx{rU~?|mx|iXEMI+~|bQu`h?d?^oIvv+vCTfg_&BDvp~R?>_qD@ae<d
zM}8f-dgR`b1xHnn?>_P4<b~5y&b_`c{ZjZ9+iOubCf@44Q-1Hj{j~c(?*`oQymk17
z>&>RymiM?G9)9ffwCL&Ar~J=jpU-?g{pGbc&hI%slzyoHbmi--A1uH7f6xCF_Ve=h
z3tz8(@%ZNZQ|r%-zd8RF{ZsmX{QuYgMhs1i`<UvOZZO_tWMwj8TE(QroXA|yoWNqh
zroiFN70w$ca9wbTAd}!bf%gJY0_XV;^Kauf=6}I!&6CD0%JY!dM?g$?m*`e;X~{aN
zdg(stR_U|StE6?MH%ncS+#_Ki@mt(hJVumRI8E@YK$&2X(0So+qFoXzB<Dyx7n2r~
z5@Q#W7vmHck?51SEOA;wTcS){SiDE9Qgo~EM?rJJ7lMa`-NhD5Xi80$nkpG0-XO9@
zP@G?dSB}Sm+lNz~U5|~Qt(@%z`y$R7uC-jdxXyFga~JR!@Eze76f6`<6P6a{7g{1%
zCbV8yScFsfqJR)zFLxYg5jzXJB&QFLGoLTtbKd2A#sac}xq<>huEGW)n}x3m3JKij
zmlM<x+9LRq-;wVEuMmHjAg{=5@pfr;xgfczvevTuWN*odDEv@(q);ILQYJ%+Pm)9O
zzGSWRbJ+%kO-i*YpH$`5g*EPLEY-}{`lIzjYmb(-_7k1YdN~Gn3^EKV^kelZbOUr}
z>9rX$nKGMSFmpHCYWCQS&1{0{c{628E$ctlhpkJkzgSz_F0xayPqJHM8){c<f8YLy
zgO9U}o1MF_8-q)=V~za|TV5M&Yb9$|Yey?l%Y){3%rwmxSrk|uv-Yx)wOL?oX)SIY
zZN1mp+~%N7zm31OuT_E7bE`P3J_`f0I1_o}c0)gda>Lyw`R4H!cg;o2P8h#5+^wIZ
zTdHee@X%<ov5fIVqaLH{MnXoX^qX`S=)BSSs%x%aXE50)$TYxgs!6TUIzw^83;F_j
zLOLHc*Q)PS`L56}`%zk3W`XQk`HzZ*%I(T?m8UBoSJGD$Rk$dxA%8*koXjPeO|skN
zrpmpQ`7C8ADJecxq+G~eaE@Su@C6Zbkvw5*;q!t@{Jp$$cw2add2~7NuwG$KWR7OG
zU_Q#Umyw4tf-#73HA61rCg$xd^O>!f(ixX9torZvZ~dS7znA}V{^9uL$$PmsQ7>OT
zpYy!)`QK-3&)lAVc(VV=@h2`%8J}Hww(9xz=hDxYKHu>2`Kzlhlb%OCVSD)a?*ChR
zZXCZ+c>Bj)<9oC2MBECvp?y8#TF2FkS3Rx^-H5mzdoA|bitFV!cimij-S?{AmHC$=
zFKb=NzxMOy^SiDOx*zmCc=15_LD;>dyPbDS?@qcSdAs~(^Nlw*s&9GS$-g`E?*BXb
zcbM*cyrXtc>i*jYmmcqXD*9~4v#J-NueQ9lesl1(<7@HPw_mP#e)E~ji=(e>-?qKu
zeHZ+;|Lx4TAKzSkQ}FiX+jDOZzHxt@{(AS@+7G6mE_~Yj>G`LVpFe+A|D5*u>z9tN
z&R<eKP5p4^UCdkdH<qtUUhjV`|7ymwJCA<de|LA?ou)g<cXRKZz9)U3^TC$~><<$j
zINZN;ch#LEx4mx*-I2Ukf8XrB%)OpFDz{3mGhMxYN$pb5<@cAvFG*flaxUoXmeUTW
zBu?_2;61+k=;On`4jwzO=YY|{)B|$+BKFAcDc*Bw&&j>(_DdhKKf-x5?8w5yTMsuL
zt~#{(V9dca2SWC*-1~d?&RrtAj_jPYvuJ0+&eWZYc1G>$-@Sd0@!l<aZT7!Cu;<{M
zLl#GtAN4-=_Nes{`$J|20}dGNzq?=I@bhC%CuL4ap3**b=Ty;Y&(p4_rOqUuU2v}D
zyxax#i+3)azWn&o*9%k5PddNyg6t))E21~v+`0DP(xWMlb)Q&2z4)x-1^27|*M@H|
zzg_ai@^$>{nm60u-hZ$ES>c<&58hvA|0w>~WO%_4!SsmvH47)(QFceJY+gTsC4y4~
z4F#(O9QZBy#QAjj-2`_C#R^>!%oBPoq$OM>;w|PWepft7!cyY8_<iw}5`j{=((j}W
zN_omGm-UqUEPq~Ux$1uPc#SpcDe8~Zcc@=i>rpdS->kk=O<JW|@tph)IWxI1xfD50
zIYv1Nx%;yEa(m^b%dyMvQ_xhblAka4T)s^4g`%gTkHU2MS8`#pQ>ELaBW0({=P8yd
zDJrolev+Rnw?@`pE=K-~`~`)>iiZ_d6&ETnDR9Z(mt7>YRk~67gS4*fP5DCQHL536
z!&J3ZC#WbW2PiI<50eX$m6h2hwOcYx@~q?wsrAyDGOV&8vbSU}$g9XL727TpDE>=E
zPHw5x8WDNnmEx<!<9H<*Yk&Xxd!DtQr<YG!Ku_p2PYy%lFL|bWg74*DDmV)5W2pUp
z|Lfi#AO4*Gf1a72OMx$zkDJ$-o1Oa^??L_?-ZGx^g29rmvU8<=3;OY}3#=3_=i9~h
zkBNun09U+_gve>(EYW3xzt}XH<hkcaS7?gpC@ZQ9pW@--VdnJ}C=<COaZj#W*-=G6
zNm@=#=DeJk{2Iv@;#=kZs#ob<)N|43R9>KNqdQkeMblB8NmJ9{fO(hYb<1e$iDvcM
zy{hV3!DbmQQ@r|}@0i)?%4vFQKQd4^xnve-W8tvMUfnj;a)X(aMZZO(NrZ8SWrh>K
zr?n@C(=O`;HXH0K?Zs^Vn<bkrvtHr!)T!2y)#;tBu*oVNKHY7mi<}HSuRA4~3F#_p
zhG`aRz0i!&7BGx6<u&CsK4l<h&|@^wh(otXBTDnCUb<PZMYw^ZMuJAW_IYh1%_Ay5
z6@M!~RLfMGp{lMnLurph2LDdpc#$=7M^#)DFNy8vna!TYK9A!DM<Um3K3gF-;q5{n
z1pe}83T_jy;x1<6Vt>uuF8EOJ3fDg7D@?kqr&zZ!ZT^$}!|vyn->iT7|6KVe_>cYj
zf_KhuCVra!*M-HBVe?nPw*fC6J>By3&U3@JHD7uDEd49<_sbvAzwiI3{i^u9<W1VE
zws$JuF8o;iDem>km+i0ry-j&P<DK67<S&bU%KTpb_aO6Zwx^8Je~$mLWU1oo5#1{s
z%k|>_uJ5{^mwj&ee&J6Fa~(%9XEDcV_7m(~9LelbOnraE{;)AMaIX;56Hwt6VrOFM
zU}|SdVPs>N%BaJ-kz+QGp}<jreY_r=0&KBt|F|jyj07KY&tuN}{rv0K&$?e;e%tlu
z3&S;r^9-Aqnc15-R<fODVE85T-Q@Q)mMoqNyy;wStbB~K{)_(;{Wa_R>K|8sF8=l4
z&)t8~|DOIz{;Kp*_T#iKpMM<qo&8(nNB<|ex7n|S-erDV_(}THsgHX<X?;2Nb>H`j
z@9RIEem(oS?JLjE9KQvAKl^(2efF!E=T6U<pWb}@{gK_H=Z}9r|M+U>Ywg$iubf~1
zdE5G)^HcSA-(M5HiG9d;?epsOn<F3MKP7(5{9yI*$EQnQ;=bj4-SDC2mCUoWXJT)X
zKW+GY>b=j)e~;51w><gz@Y>z2cOKqNxWE2^{KI(<%I-_uW4yQGzT=|@PwQSLyyARr
ze_!e9p>rYUr(VjutakC!S(`IA&orK6Iq!8==v3vgg2QD;noh5|RCoFEdDSz{r>>vk
zI<xhZ#fh@xTaJsIczkm8nS`?pr`Mjial-19;ThozOjk^=tU5pExcUC4yIJ;!AGv$n
z=!EmJyGOU4Og+EllJ-TD(-lYR4+b2Wb$a*ZsW&Fwh`ZW*X~iX$tIMt}zN&Hg(#46F
zBCZ^|I_av=Wu^;V=h)Bhzi4~S>E@FgrPmf-JaMY*xWQ?;OX=5TZ~9-4zxL^t{G+0$
z%O1zvUv+ES^&i(?+)}%L@X^mlP4^exX1XPPEBqGEt&=zHZnfPuzIXEx)AQ8ltDjDL
z%KJR*<*v71K7IWf^v&i=+J`6a?Z3(YKgYt(c98iq<1SV~-VXwD{5)KoELH!5{$(<J
zVln5M$>Ye?$hwD-fpI#MAafhT{=csOq?vxRE#s2n@#nGU*~tBo`!-*#kf-om0UmBU
zmORF_tjBmZ3Gnduaf-6}bLR8U7L4OR#J!chn1z$Yl5HdBdfuu0fjl=^BN*Em9xxa&
zF#NyxZ!g0a7DcXlUQynOoEKPrGITJMGRL#IvQJ?3`@iMe+|P|aX8-eN(EBI%Tl!b{
zAGQCN|6BjJ`WN%N=3CdN8J}dod;c|J*z+g&YukITcc<U~{BY>QocBxKO?f}(li%0Z
zU(3Iq`QrQK^jD)_%l;o_&Swc^@cz~O{lbqkzo-3U_<!}!&OdF8LhLs<UARAR@8F8#
z*uvJ%-pb=F{899*&@A3m&c|#?Z2Iiq*)MYxavkPz;ExdeB&04fMZ`ojOTtgqSb<k@
zlHykRSyKGs^Cc{07b~2QzbrjPx>;$qma<-{UXX5&)>MsEnuR*127acGEIX}tSely}
z8YUTXnzC86o3Ajss{KJ@rH;3;w56;qlfAS36uT8Ran?q5i=7+YL|ly=_L^_ey`fg8
zs;*gQpk(pGrpeyf;jry_i|r;)jTf7D+xa^yItw~3viG%PvfXAKZF$>tl3}fmmc|pc
zQyM>Y@{D$x9X4NVT4JcH+ob(Wr$f(MuUorORb5t6tXg=x*gd%zwP_j~H123VRX?QQ
zD#algE~BB8q9(1@sIpPTL1n(;Ke^R1k0g`CcZw_!*vxyD|DotP=|%Fp6{VG26`ExC
z%lOH6D+j9|)L5(<Ah%guOYDbajY6xcviegEZ5<2kc$Hu|7uiSh{VG=K9xA~K(-eZ0
zo+|#9H<wM7x+GyQW+%9ik5Oo{_;0C~(yY?vlK;d*#n*|Yifc-pm(iBZlDZ~xfiI4`
zi@TMtROp20BMAxVdWpLNb2$&NS#vJqpCr6bU^dSa?z`Mp+@H93IIpm3ut+mq{k#7+
z<G%|GI!xP`7@4**eEM(yujV($@3_Bx|62Z7{doS#=AGi3r*E#m-|^Y|d;9NxhH$2b
z{~~_9|MBX#57Ps-2G;G2cmALF|C}M2(dJ+3uXDfn{^tM9`Mdvb>ED@ufBy~t8~iu^
z-+{j(f3E%s`u~EVlQEw`=uh`I@z3)<?fvTcr+|3_XEgsl!M}ViTx#q|?AN)|1ibi#
zxn{Fvu-kFR^EPtYGFLLlGGzS^{J;7?E5klUQI@l;SsdBizq#*lrE>LgMzH^45oVh9
z_v??guWLRld{z0?@}G&to`aq9IrFbS^MA<wnDG1Czg2%Wf0O+D;FIZ>!(R@6eE4R`
zE1uV1UuV3Pd|&q=_>=UPr(dprHTquk-Q%0im#B|d-{rlvf0O!3`^Ax`NslK!(tUF0
z+18hrU-P{>`Pk@Q{ayEmbDpcbae5X0<nV*;ho7IsJYVoc@Lusvs~c`N;%=_Gz2~m|
zy`sB5cOC8<JW_pP^ThN~!@ZR^gRbyg<i7ar!nw1DPuiU@I<@WG{>vAxSzOD#wC&vY
zbHP_s?)-Z2_@3pBkju|6C*L@FTld!8D;DR4PTx8ye5U05j|&dx8P15D-gUO(V$0PD
zH}~G+zwz*L#D$a#HCK6WAG;%X=lLy}+q>`kJ~w@{=*_xUdtcbUO8NNWR|VrDrk(%p
z{CNCr@~;hlH~%{HS>Wxi7nLt&yk>p>>0{I9qaQuqCx5u}<-#u)1|6nyMt#Nw3`-bJ
zv;5?A;9AJ`h^d}Yn)M#{0{(FRZ@h{;n%tLoCkSm4u@m~reVyet(-(G4UU{C|YzmCW
z{^<R=`A?NGoynZ(&_CzjN`Er`hcL%;EZ|zp{*`$u(;nu#?B3kpIaV>P{-gTy=r4B0
zDXe-d$NtUy8TMoD?`{95GOl8r{-5plmLF_?Pcf^oi8D|Av;1rFXY+4If8_tn`I-K`
z@5}BltG*}y@nvdf31*o6OYFPkw`V_m{(k$*^K07|r%$)ORWUHJ&tu!g6!d@Nzf{I(
z)*I|=*ypfp{J-SS)qjC3Djcub<yboZvHvdn&GL8QpZcF&U!p(SeNy>q@$=)qBBq55
zY=4$~_xNV;>*T*{f6TuczO#F?>qFWP(Z8&J7JqB|@agU0_m@9)e^L6v{n6yj;}_zu
zeBNLGtoK>y?V6`tk1`&cJUjf%?1{w#i+g?#w4Qdo7W`E7rR#&s%PEfxA9OsP^?dO&
zmWS7GUAZavNc7#I&jKIYUKu@cei-^l?CI1O`(G!%n)o>C?!miRj}JaieYWYr=Ubm{
zFy0Y*r13oJ1^2TVkC#5Se5UpK$_K^Ib3fdFz5e-$=c?~QzQ%q#_hI^r)+frZ)_f`Y
zrS|LE_qAWYfA#<A@K>Kvo7Irj^1uC$z;7#m#WOOq)clY5as6x7&kz6jncuNIXPLv|
z!OF*O#%(L;C~{iRk2jEO7S|=dRl+MoP6^0z^|Pt*q=<{i$0;PrG0XBR#H+2*`ma-B
zaK&(mj+2VBf`!U-odCl|-3(Pu#R8=&)yrzjRBM#{6lN=$sb1E6sV8OlRCm9+qms6Q
zr}7dFEv?zA7iE`7yp`fn3Rb(TlA%y6dqIAsTA}U<BR_Lii^Ybk)i%m9%B)hbQ{hxN
zEjo*LJ<nwUDUn@5uXx?Lo^r0@GUd57tY@A^JvSNxqaiRF0;3@?8UmvsFd71*Aut*O
zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8Umvs
zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF
z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*
zAut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?
z8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Auwb@
F008$Rw`>3a

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/126e68def9fd973a100e0f66cadf09448a716b57 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/126e68def9fd973a100e0f66cadf09448a716b57
new file mode 100644
index 0000000000000000000000000000000000000000..9b2d29856d1483ced1b8ea3cf55f53182947451f
GIT binary patch
literal 9209
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA%pGge;%)Kne
zERM{YO#2w5|J(fg{a4^$%m15<Q<=}O6tT5(Y~{4&Y~_&Wh+x0OR?K#VwV17#y@2fl
z%Sx8{EZ12kuqLv4v3zD0W07GFW}N;1-=Cge^1qM&N&d6*_wC=Iza4(7{+a$K?(czr
zJO8o$Rrvk(=Zc@YKP$hNeSPuy;wQFGejf`yzW8+gv;XIaPo*E$zU6;Y@!IND;|tyA
z=blAAU-vxY`Kss4FN<EydbReY#Y^c|jBmcZ6?yOe;nT;CPtBhWeoFeZ;nTS<wcmKZ
ztNn2OS@rYu59S{ZKdgUV{q^MUSq4+4K4vjiakfS_6INB``%F&EElfF#+Zkps<TEBR
z@i84?@L;H5X!sxc@ASV!265&EtS4Erm_9I?Grsv(__ybe`S07m7XDWIbN7$lzf1o(
z{u%zM{<;0zg)c0hpMCiIZsQyISCd|BeJ=6b_W82sN-vsTta@(ojP>cSM{^%Y-M7EH
z;P%JczIUVV-MshwUgZ7o2g(ocJv{n|?}^T{(igHX<zDQ5?)NhEmFvroFN9uAe`WhR
z=<SpDnxD15@BPX8i}`2V_cdSLzUY2V{dDUi=O_Em!Cw--PW<luv-&6NFV5d_zxVz*
z^<S9jE7L<}4OTt2TWmG#ZX9Wx%G@rz`}m~z75Pu|{}wnc_(kxp;88&jA$egQ;Ub}r
zLKPy{L<7Ve#oma`5#J}4C3;xomT;HQa{*hvZtho{F&wh&Q`y-#^|+$AZFqKY2XhH=
z{$O`y&tw12Uc@25*~j^nlZ(5G$ADLx_aD!69)2D^o+Uhdyl;4_dE9x{^7!)d@YVCz
z3(5-%is*}E2u~0?ASflUnpcb4fpZVXEY3+>m$@>y?RoF;9pMw;o5#DFXDv4acN-@c
zhYUw6$6Izm_PK1iY+Kke*c~_=ICHsVxGTBZILz6kSuQXZ{XhQq;va)Qx__4bk^KAj
z@BjZ>nGUjeu>N65WQ|}u$##(K09z3oKif36`|Mtv#@xqvU-2^vb_kvjd?y$pv`Ofi
z5T9_d@G_D6V!aaIC6XkWrLIVFOBzbViyMpgiTjC95^EHDDpn|_A^JjCKsZV0nV^o)
zMqv-J(-PcL_a!AH<0L+dzY;$v{#=Y*OjhiqSc3R#aW9E-iOmvoBpSr?MO%bt3vmhE
z7ii}1<~z<Cz`KSgmwPVPOs<=pvK-slwz5uP>1FO=%4FQm(8i$2z`~Hi;LY&$e?P+w
zhOGZC|IYoh`@i7dj=u-~$o+Bplk;csZ<$}qe@Ol){BiXM$IrZ<?LVu3czm<^eCWOY
z+qTz}UT%24>{-Cm36J?7&wI@HB>u_oCw5PlK52Qp=27dTOOHN23VPJ=aQZ{#N3M^)
zKid6d<};NSH80y=dA{EMdfjWQ*U7KiUoLpb`1;YC$?pn3y!rI&%j~avzdC-~^KHs^
zm7jsXuKZf~d-Wgozgz#f|B?A)@u%~T=wH!)=l|bkbYwosEX%Tl#g}y>t0$`#%Uh-d
zrXNgSm_1m&v&6BV<Fw=|<Gjuh#^J(ogrl58nO%`>0_$-W0~P_6kIb?xGgv0GSg`1^
z{9|rnwqU-^#KKg@c$l$}=_FGj(?iCij3tc14Ez2a{Ad3E;XmeoR)1su==|>bS@r$e
z*SN0<U!%Tqd<*zq@}ujg{I7>U+`c{jyz8US`+&EFZ*1Q*yjlEq?)&o}?|-iSs{L*A
z*Um3}pJ#v0|HAh5{g-85W_?-k#pKJ)&%3_lf7|%|$dB+}8-JVssrmEpkJR7Cf71R~
z{Q2@H_TS3?KNub|9b!4os?Ju$c8cu_+d8&LRt9D<MsJ3S{~rJN|Lytr<ll^c>;Jj@
z_haZ_WMk@Pyu;wcko&*(-@3nd|NQ=)_1ph<+wZ%--~GPwyZd+2uh<_azNvp(^L4@(
zuTNLralfANeC?AtkM=yA@-X+I@k5@64G#kD%inXpYjNkstp_*X-Q>A_>CU2i3=jAp
zCO+zYJmay%;~9_VJUR2E|B2Jnz0U+**u3<6b^O)mSLLt&zwUcu|8DCC)lascE`OBx
zDD=Vcee1i_w+(N0ygB*S=Y8o%p)ZEta=%;rIR4|okHbH#f4cwD`+fiSmOmwb`Tn{6
zKf*YLrGjk{`)Q75&Kk~i&J<35&YK)>IF4|f<M_-O!o8ncgQtgAhW`!!T7k8K%Y+Jq
z_Y12D=Lxk4T^EuQvK1)go6T#^yOrk_PdcB8z;*#{0V)0?yeqgHI3qY*Ia)bNIj(ao
z<I?2a%Quz(mcTnfCLv`ZZ{b-YilWY<DWb8WQlb|{K8W;*dWxMD^AJBIZY0qsu~MQ}
zVwQxdM7-D^;f+F(LXtufLJ2}UgrbB`3*Qzp6>Jgk77*lL!mG>soky5wAD0E^MRpnX
zmu&mkRN1pRHgXzpZQ@Mi*w3cQx`;WIDU-32;pcyGhA9l<jDH!wF?lmjV_wLT&Z@%7
z#5#%f7V9KdM^**aKGu_Ll^hma*SSS`FY!k5UE|Z>ci~s!|HyZd?<SuQ|1|*#;k6<S
zqI<<|h?~wGZJm#{&dF_^i#Lca7nvavEy65vTlkFdPhoM9O5x=~TZIaR^+hy9yTn)}
ziX>~KqNJ=PUyB!rtq_qH&KA-U5*Jz{R4sBt?6d@f)Jdr?QYKPIB{al)MU_P!3Y7~!
z=3l_Kf_EOz1nxUr<y=}^Yd8;bG_dEe?O|nSy~gasB*eImVIjjAvOqe634<<!HiH3!
zJA(m(CW8QD50f+VCnh^46~@^AzJFtXU;HWf^X`w0Keqh{`g#1P=TGe)TfUWjY5chV
z-SRiLUU9#i@%-DfFV6&@|9-aenf$Ykr@Njkdd&E^^>N&jk|!2Vs-E0@^6`o8)2C1G
zKYRWB<BJ<F8(+PB<?_1gb>(aA*GjL$UcZ06<gNIJJ0Ar<OMNZ-e)GqPpJKnu{#^WP
z`JbPmpJ5Z@1*XqTip(7>c5G4XG8~IJcsScQ*K>+-3Ujcre`U2}{l#*h<q1m&t1eq6
z`*DsboZmSYap`bh<g($4<2u82l`EdBo^uw5HHQSpfA$8Bqnv`=-?*K47I1rWJ>n>0
zzs>5(vXse%@jSy*h8~8y|2_Zr{G0T5*`MIwi+^(d%=mfdhs^iMU)#R8eZKn9^}~sG
z{_jNJ|9;Q*k@2(b*SK%F-(<g~e|`FS!^epC5pS2jE_hY&YQpQoZ+^aA^#1OL4IjR~
zpYgu(edqfl@5|m_cz5w#$op;Yoj!zry#8ta7n5&1-}}DT{;2<H{p;(m1HUKyS^xLU
zzvcgZ88R4-{9pfnF~e;JLx!~sflT7e`<V)u3Yk_iooA9^l4Ueu)L=Tt<j>5`Qphrm
z#h>LlQyzo$zf*rU|9<eR``7DVmVfU5eg4n-zxaQPf4l!k{kHjO|9#Kbd0(_Xn|)+@
zf9H+J>jf`eUW7cic%J%P^Lg*HEl;OC34F}_Xyb$O`vLbl?>@Y1bKmB{`Uh_xzI+`2
zROcDv^LNiTz4-9L;)Tld(UFwVk(AMql+lrtkvWp$#lD?=CZ{{s22NG3TJB%mVmz5V
zJUoxNZ}W8V1ajqbD6{9Xak6h>cjFM^a^acJ`<0iMuY+Gk&`cng|FmF`$QNNYp?(21
zfj|LnfzN!kyr$f}>~^dS9Qu6MgtiK=7Wlzk!)3^m$!);i&K$-Vz>vk1!?up&J7*xz
zB0g2YG~pY94t!5|^ZB&}N(9#ni3_m_x(I$0+AT6y^tRX>iA$2n((h#wWJ0Bjq+dzd
zN!}6vEA~=suJ{M>3ljUKOJpsj6(l7ke~254Pm<U!-XvluTrS)yGDS36L_;`QBwRE>
zY_r4+Ni#`5$psP{#h68_L_P}_2~OeL#jDSo!Lx@apI4C2j*pvn5zi!EX5OjX0XzbH
z`MfOL>Rj`<o^i?Yyx@`N*5&Nv_{}cL{+8t=qvF5QKl;92`L6hD#ZQHw=08?`pYmho
zkJ2AGKWG1F{`UIoq^}iU*uN-#vHyJeW9!GgAB5l4yoq_c>%GV4nP0Yi{P%9<Td&t)
zFI=BCJ@S4S_#pZIh6i^ZUwc0D_02bZZ|dLPe9QZK&U41+kDfC=XL>66WW{6o#|t0p
zJU#jB-;1TMFTJ_@#`LZ1o6J`tZ(hIq|8DL(yZ3A#I6m(F!1KZSlg!tp-ztAR`|;#^
z$<M|=Nq_79xc^r8o&0C!p9{a*fByOP{$DQhYqlbe#q8{Cw^{s|H!!j=E?}@@`1IfA
zzu5muhLub~tby#i*mkfou^(ig%~s47$5zVvhV2lCE;}#lUFLcwKIYvlo-7ZTUNg>M
z6l84q%lvc4kKMmQf6x8({72i5)?Z(L{r&Og`@!!Qzb*gz_{)vYt3E}3I{i`q!=3jD
zAHKXh|7OAK@2@Oge|z2YM(^#)H?ptaytH`X`t0|UyeC_pDm{~btn{erspE@DuQ=Z3
zz6pB$^bO~`D{u4OnZCDv|K?rDyWQ{ZeZ2W~_jkFUpMJ0Xx0XSJ$(U&!<30w7|5g9<
z7(E#q7=#(0Gx9TEU~XYP$=uFzoVA4g2Wvc&KGSaIPfQGqMgMpFllgD+|H9w0KfnLY
zVbNsIVLQflgUyh2F|!HdtG|`M&3-m~SN^u_v%qJeuLfV8zi@y3^ku=Pln-CtAN?Tu
zVba^OH=S>qUfaA{_%iin=u5sA^PXOOa_#A^r>;+yJ$m-Q^MUb0>xWSfDjqaE*nNNH
z{q_eN9zK3_=yA}K+9zwDC_Y>G{KoU37Z+Z-ygvP!?e)D^6W_{xO#giUi{#e_pWl6I
z`&9MO<b%%##ZS*Z*M3p^>ifOmm+9X<|LXofVd!O^$hwi$gN=jZJo{zVT`Wl~J6S~8
zQrXXO-r$WEU=UOkk`-Pr6ew88e~34U_b%T8fh_`o0&@kg2y=;U6FDl}AuJ&LUEnV7
zey(zkSL_TNzU(hpkFhLbsbg8h+{j|j@swMT_YZdnS2w2`rxHgI+b)(Ymd7kvED<aV
zSYNPNaAa}zaCUJPa5{4+vL~?xvp!^zW_4yQV$Ek=%DRDlHP?CW)m*PRKXAoz>vE>B
zU1D9ywvX)<+i!M74t9<*P6n=Z9EUiPxF&LUbMNMy%W;59kM}jN3g2y>7OrwmBaUuX
zO_owtZ;qE-ecY8?S)4aGm^t#;99W{6?lH<Tr?UKI4r4yba)A9e=QJ)3uEQLM*j+g0
za29cSaWn9q<U7ruDez6uSLB>%ikPhUDRBphsS^DXof57RE5-FC?n*w9%8^!=IVIyF
z+b7#A8!Y=r=AMj_%vZ@^i67#p#Ee8WL^cZv3MmL5680BaEcQ+Eg3LDAgVIWpQ^n=Q
zBt=q%o(ujJtQOP|+#whvR3|JhswJu+(j}ZLlqN8fFP*o9=LOek4kmV2RySs5ro)WQ
zOn;ddvMgW`W{qSKWa4F5@VD`o@6W)Wx4-v&Yy1A<$G#t}-}iiz|1SUI`wzFDe}9zy
zwEunn_tu}6zn%DU^z-LWEFYWSulO+S)1*%~K9s*xc;ox(@(b(dJD&7DTysD1zW9Cd
z`#ukZA5D1t`LW^S36C-#IXqhVDDN@%W1&YjkKRA}`RLK3#>di6MW26p?(w4MrR=M>
zFF0Oodmiw-@P*Oqzi-1o2z;9IdB<1b?*ZS^zfSrl@YDO(*`JesO8@Ho_3GD@Uw?n}
zez*Mo<h%6Gj$hA42iQjk*oV{ryZOi)U{9th7#*%3hQsyypKW-i^K9SKH&08SH9l8(
zIr|mM8;5t-Kh%CQ`rP(8>oecyN1wVs{rdRi<J?c{K6ib&|3&2Mhp*h<O~3E@cKDmm
z_lECm-+8{Nezo}O@U8Ou^&dxmN&OA~FT(he(UIvP(*@>PETXI_tlwBO+2c4ra5?jA
z;eE#UkpGeZr{HpdEWvQ$a*;q0WsxM2X(B2j-9ifmZu6hzkK%vNr^Ua9KTCi?@DqO_
z?-cGsT$?!;a+GpNarSWa@+k93@h1uV5R?|K5VjV+FJvYBQTT<hk1&I<fN-*~n#g$(
zQ&D~~R`IuD5@J(Dbw%?<<_q%(FB1wAVin>LDiUfC`Yp&WG+Rhk=!$@sK!Cs#{-1nJ
zyfe9jxz=&|arm;SvC6aVVqMN=!+wZ8lhcozpZg1k6}ueU9u`C9hm3rTL5$%{I?QL8
zr?G^wI<f_`&t~UhXJCK8K9A!l#~h9{jvE|&T<qLE+)X^+c+L0)1ttnC7C0&3AXqLa
zB$z3{EKnt|NpObHGodtLUlC_f5wTff!s7cRJfv%6IAktJtI52Q{wsA<@{L5lq_4Dz
Vj1w_4Dl!7HPI4h~m9hphGXQ{Zo-F_X

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/1275d41ebf8788ce3a949352e4bc654b04012da3 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/1275d41ebf8788ce3a949352e4bc654b04012da3
new file mode 100644
index 0000000000000000000000000000000000000000..b4370a16d58c0b2d52af8fc09ac8606fd7e88d7f
GIT binary patch
literal 70
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU|@7v!@$rH!N|bIAi$84Sdti%#>nvK>-x{8
b->m)?{4e_L{F8eQSJ3Mpu0Pg%=Q<Antk)Tl

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/1a7f1c407fb3864ddb559f88f373a21d1be51584 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/1a7f1c407fb3864ddb559f88f373a21d1be51584
new file mode 100644
index 0000000000000000000000000000000000000000..d28721708d44c0b6227a14286b19f6cdf3be7010
GIT binary patch
literal 35640
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T_3Yy_hF5v-5<Z^)#PLP;YsJ?SUnYLm{p|Rq`fK@Dw=YjV
znSDz7<o22GE8~xh-$DP<84?(GGJN|N_&4s)%ioKCtNfn&EBBY#FSnlwKdOIh{!#Ir
z|Eu<A`%i5j^*)Atiuh9Ub;(!#ZzsQ<{O0`q-;apjNB^$)|M0*4|GxhR{~!I!^;_sy
z?62HkHNV<^r~Wf!-ooC+xu3I~OMv?;=Qnl{_UY`L9Lel&Sc_Q1n3b7Ine<pZ*k*8S
z<o>}YFE~Z;kiZ)LApSJ|$9&v;pLk>W68LQSc=(qH91z?o$RMaG(8%A<KSSV;V1e*U
zkt?FtMSqIcigk#66ibv8lu?oClA0p<St3Dvqe!CQYrZpl)%*+jZt~vcw-RO)l@(<b
zJuC7^SWn19K%4&q-#WgPyr;RfIrp;dV&!FHW<SQ!$lcE;Cpb+gS%^naML<>HH9rHt
zF`qJDIG+sPT;3|)i@axei+K#WCUN{@&tVs0PvPj`dd~HU^9!dG*Kv-0>|Z!;aP@NA
zazEo-#J-lbf;s;`$8XJ_cYoabe(BrH@A1F>|J}jh%6OdN55r#uxBu*a`TpGft@C@w
z&&S{Yf8F=x$d~V5Pkev*bITt-hF+#0OwSnaFyu4nG8F$W`X}`7(!c%xSNs?MU+_Pc
zL4-l)U+ABj-|v4N_;vWV>|fQtX@Bhgr2TRH6ZD7W@6mr>85uz7kEMg9i)Am<=KooL
zXa14-`|7Xhzoh?$%sbd`a&mLc=BnZD<d)#(<DSZ$#Pf=$kmm%~LJkdf4)z`FH5^Mg
zA9Ekz`zNqoaF2i<zXo3>?+0!-uDzT}T-IDWIdwU|b6n#v<haQe!0OJjjX9dRirJS%
zh4mjxDoZ`fJ(ePte3t2KTRFdT{pW1r$Yi%+(_qnN<oU1v|I7cz|9Ag!FuY;f$$Efo
z2J2O39p>juvlw6hkNm&<|J?sA|3d!!{W<f8$&bPxo<GZfx%`Rx-^Jv?yq3|NA>;qw
zzg~Yze|P<M`XlvM|L=-FjDJP`3IDtHXWj1~zs!Dp`1#}4yg!@&#{cvA|MvgH|DXT0
z{a?oD#d4Ijg7qScB&!gsG_xdw{J)OB4S(JLe)}W%ZxLfWYbV=f)&y2J)=4a>%pOc<
z7{!^am}~{_3LX{o5Rw<>5iSz?C{!VGO*BBvQS6P_9Pxc(S)zwUZV7h@Jr}U$>*jvN
z8N(sVK9!w~Q;#c(+lFTccQBU_=MQ#Q_B{6Q>_r>`oPC^MIk~v2cno;8dH?ZD=i%q!
z<5|MP$NPq-n#Y}IEsrlR4_`fhy`a3XpoqRmhVTTT1A<Zlt9iA!9XR)J%;KEHb(t%J
z+n)Ch-w{3mzInW>dDe0>aJO-CamaAAa=c|1WS`5H%eI9rgWZ9{fiss&hP#rhjl-Nx
zn&kpx(f{LrFa9zAqx)y+AIZOe|Nj5KmFXah2kRe}MAitllWYgs4zLxm@v}{1yU*^$
zY0Q0$_Z2^*V29ud!FPfoLYsuH3GoRR3ojG7FV-vZT_Q=6S?Y=;x1^y&ytuJ=pSYj+
zB(X-Zr(%U-8lo?R1%#7?o(bv*Z4~wpJ1xO2bzf3KGEU;N_$%>);?Kp{#bm`!iY17@
z7Wa}Um)I;ZN1{PIU$jMdwh))leSv2FZocEZ0laH?a=GVn&E&c{+VLFic#d{FM?0Rw
zpySE={?nV~uMA#pd!_lI`@7fQBmek*Ykt1)>diBam*VdmzJB_d{G0PP$FKL_4ZdFg
zWczXcN6xRwzdkeca#Zmx=Kse1nC0&u>mL_?IQ~^(KEd&qM}gmoe*&*7S121lb1~z7
zrsM2&e5FFS1lDqivG_BpGi!2m@=X+05t}NZAU#1USKL(ehH#?LZ$VLEE3q>&S}G^h
z_?1JX>x3$}&#@n3i(;4JtmHE0n!_Q&c8%%o|NOrOzn}eb_*co?!7js*z%2Ra<qzTC
z$N!4{|L{MbVIHFbb1b9vU*kW2{?7Ru{Y&tB^0)q<xBo`}d;62=v*?>C&-XoJd!hTP
z^{vYX=?@0)Y~EzPYJb`Bvj4?~=PO=DzwLY<_|EvH>f@IWA3f!HmGWl(TeY{-UmL%y
zd#rv({N}@(n{Hmde);OXtBJR@pX_<Q^zn(iUvGxr*l;uR?uLhlp833ve4X?B$K%FF
z*B|CS-20I0$<G&i-*tWx{<P&y-_v;yIUcQl;`k!`^|m*!-`;*#_1gZa;$yX^+Ryes
zefo6Di=!X5|1M<U|MTs0_8YzD3eTp!`2Oa{=jp%e{@MS@`Z4v(%8%L~B;E(U=lv}E
zv-+>=-}yhbe^PzF;2rmePoLz!U;N$6uz=Z&Y58C0UwJ<!etZ69_2;s$`~LCrycO0L
zKF?#v((||R*T3(g-&%i8XSmMH&ou2{<1g0lTff};c;S8X$17jY{8axP^P}ps`-h<S
zChs@Czx;vybItdzKa2iW|Bn1U?@#M*wIBaKw|!#yY0py5rOf$>x%hADkC<=DUr&8`
z_QRQB600S<6WccC%}h?rWsLRz`v1i-I<ks!__1wfOkvo}RK&u=s>C*hy_)kAmnVlC
zlktDa{~mu&{8aho^(FYHKC=Nw8f*0b=ASPspBp~2fA;E`-1CU%oiA3rdibXAUG;nI
z55GRF`^fc4{FB0`xu42D|NeaWbKU3i&*wkeelh*h`T5SL;!it2R(;5NfB)T#cNOn)
z-ub@MduQ>E<6Y6)k8gIqDSdPH^@&&MFIPR^^HlGN`{U?GVh;-+oO&?-0pI<IyDRT(
zytDm|#U1Y3LbptAExBcKtN3QkO|@G`Zn58Xx~+GI=ia~j>JN83bbEB-5$|L3$4!q#
zpZs_t`^@F}zvucdmb|$3^46=nuToz*y-ImC@729m`mguDzW&DKUCMj458WTme~|pB
z`tja}-VaMZ=zRS8vG|k9r)wW~e46yd{o9)F`9ClI4*#diV9eOUw1y>?Z3de=n<<+F
z+b!0UtnXPTvdM7V;{3y<$)m{^E^tk7lTfZOlSrUQu!xb!W8rdP3t=<i`$9(q^8^;~
z$MWm*&*WRq`-Eo)&mA6p-e0@{{5J$dgdPYf3yTSh3fl`W5{?$h7d<H^EOA5Pi^Mw#
z9?2p}NvXwB=cHyxC(16B%aRk2+auQ_-!6Ykj#I8tcDd|-SsuA;xpVS!6)!6pD_bZ>
zC~sAMs=Qn|LfJz(S9z-PO65%D4@!+nl1l!HS_)3`Hu4|k>l6wUG!?$e@02%`_mJb0
zeI@-z>ZjyF33qXOu??c(qK8Db2`37D7HH*H;p^mC&K1O|z^TBQz|qKV!!E?`#deP6
z4^s}KH^a$)>3`1u?D($sb>gR#50Bq5zdQQ2|LxW{_g{0ox$`FdZP(kiZ_mH&e7oh%
z%-2b;{9kT+vGzsQi}L4f&rF}OJo9?W`{dxG6%UU;kb7|M{+;`h4>BGsdQkh|!u`+p
z%I}5VGrRZpZqMC<yVLKM-93ND{La7Iw{N%JzIp5FO_Q57H}BnCdh_5-p4+qUgxw9f
zD|c7oZq(hTyMcG7+<kO!#RK(6MUSsOX?@oDBL3Bc*LH8tzX^Oh`|awt>)uMd6Mt{^
zWX_{K52rlLeQ5lU=V8Nxfcx_IobOuPxpC{k&389>ZeO~y=pMrZ{)dT=dLPetEb(~8
z<2g^xJn4Vp^mOktffqI}{azh^_4!r#>;JF&-q^p}`a$)R?WfBhB|Zv$aD3nTF7<80
zn;maXzV&%u`cdeM;kVrH7C(;vc<|%!59^=qzx00J|Gni;$zQ&IZvT%kPGPBFTf}~v
zqnWdYGo3Sqlb`b@#~Y3#9OpPbbB1v5=hooq;g#Wk!@pKwt>7}D0^$9_YQlL!Ekf6Y
z<b-Sm3i)R9n)7buxy6&tXCkm&fLlO{{|N62?gq{X4p)v=j#7^69Lu;gdH3>7<-aBH
zPLN4RS;$*>mWZOLvuKKFtf-XeMUf98eWIRXXT?0k4~ZK|^hvCg=#`iyVJZ<X_D6W5
zP^6Hgkc3cz&<>#};nTvmg-iuo1iS?V`IqqO@_y$L=Gn((!FiEghW#bmJ~mbMY>thb
z23(ss6FK&?sj@C&PG!nu>}2@)Uz}kIgE-?~#&1mC%+r_`vZS-BurjeuV!g#WiPe!+
zfwhnIBwHni1=n?MQQk|uk$l(qH27WkmH0pMUF5sT=fi(ZKtgz}NQ3BJu^Zy1;tit9
zMP`UZi!h7a7Cs~VQ&?Q2Qh2%0R-r;+eGv`OE-_Y#BFP%5C@E{n*Wv|YD@5dlvxRho
z#D&%fRg2sZJ1xN=byDh!l!??)2@UaHQDu>bLgj*w`4{l5;GM@af%^_uIhPjK8qR|p
z4eU8=dsx|7uQ59@2{Eo?SjdphV8Wowpv_>w;Lc#cpvfS>*u&(^{E5kqNrf@?zwh7J
z-xq%h{=EBR<Bx4Wf_@(V>G@Op$ChtpUm8E|f4BV2tykPHXFUJ*?8`I3=f9t=d?x>_
z<LRy^iykvRZhaj0q~wXkld2~-pL~3x`}FD4`_EoK|M=p@%f?r4U%9;QdR_Ti`?b>R
zu-ETjFL^8e;m$|F&r)B@zTf<D;-}c}vOgF9TK?x}=x5l(c!B9NlOl5miyd1Oy9~!-
z4j#@n&h?yPoWdMz>|a^!SbwqHXL-UB!m7)b$$p$;3g>suMO-@E7rAV>;<(OmUFC}B
zs^^@=Va*}I@t?hc<0z*f_cv}Qo(0_AT#q=4*l)ACvMgnCVLZ?9l%a>=?tjn!J^v>C
zUG^vV_u`+NKQn&b`62Ut^4GR6ZlAAybp3GRo&P)0_rKq>ePsM>`!()c?l;+Q>0h6I
z-taNveZ<@4uM1ukyqfU(@SC4+7rnpxVZ(>7?`OQPeBb&0$osPQ7v5cb7xI4Fd#4ZK
zAFqF!|Hb4R&-cFXwLj{ATL1d`>%i{`f7bs!^KbcoUxo~ZBmdX`U(9fu!H{7sLm-nl
z^M0lRrb4EbOy`+om}D7E7&VyAG5IsIvlOySWASHs&XmVs{qNMD&A%V~>i+fmm*t=P
zf1m%e{xAOD;@|E+Qon6}+JE2kb>0`P&t@N)-rsp6@_NBbmlq+=EuN=7*L>dlY|GPW
zPXZq^Kic@9{C>c_&btrq+T6E!u>Qf@hc6$;Kh=50`25}TO)oyYuy~>JeE+iz&vc&c
zd-~>S>9fY?3NL5BVtM26?)rz?Pez~HK4*RA`~2ur_orVUpM0GAY2D|pFZaKQeEsm1
z`@8A)UEdCW^ZDNJo$Wi%H`T8eUmd<xe!u?X$S<kC;r~S#Uotu}J!HDTJc~t?HHGyX
zYbJXf=Larlo-Mr3_#X0K65teEE|4V{E?h1WD55NqBr;7zMWkD3p}=kav;0x~@A<U&
z*YIZvFbIC)FXWxVeTZu_=R%HB4k^wau3jExJ}Lerfggg>!WF{S!uN%&gg*+u5cUyf
z5Ec+l7FH8EFJdanFUBhVR!l-{s;I7LzQ}xG9^qv|VM44z9707x4MM*K`GsZ+$qHQ&
z@Dd0Rc*6gauZedicQDsFPCpJ`HZ@jx)?KX2*=*Pkv1fAnar1M3;jm(tW81@G$o!Cz
zk1>cboJoiIEb}y$FjhylVD{PUT<i?&57_5%9Oan9k;ZX@gO7`yyNA1p=NqpXzo5WG
zfyDwR1snv+1%(7N1(*e@1U3oI5PBw*ChRNXEGi;4OH5dNpM-~WjSPp(1!*;zSJHo_
zj!M3f=$G`BHj#0X5s-C~3z4goHIONhQjpvzp(hb7E-l_6E-!IfLQ9fSQc3cxq=3{M
zNfF7J5{?od#4m}>7ikny6|@yt!=J!^i!XqWg-?Q4hFgYnJ9{1*1M7X}bmksrTb2;k
zbhedj)@=H0er#;4@0o0v_AnJOX)|v7@9^*YpPj!a{`~xH@z-TvGr#WoV({h0=lh@E
ze?Ijo_e147kGJe^7QU)}x$ni57Y#2ry)=Hg`$gc3_0RV`d-%lQk;((F`@Z+~-f6yV
zaC`0TZFlD1^|&|j{-%euk6oXveiHk%{n_*9JTD)<-1+Lm>+5fIKRo*Q@>9>3@89Hp
zp7>Sur}dvS<38qhEL&LKu$*IMVQXWJVtvb6!Cuc<#g)Ou%x%u2#&eKMoAVg^5;iur
z#jHzM_p>eK$lyA_J&Wfj&j#Kvd{_9r1;Pcl2;~WP3M&bl3Y8013C<R*6zmgZ5~>uM
zD0D}#MnIkaD{lr*CD#HDId&nodRBARPb{V^%b1=s*#AHN=hRQfAC^B>d>8$8^o#J9
zRi7<BYkoTOVeW^dkCh+KeJJ~|_=C{L;E#bH4BqW}WAMiRwb`qeFPmTOe{K8r&D%|H
zx4hMQSN-nlJInWq@4vrGd%yF&_4{@2e!P?YaOdOl&tYFZzsLPt^Xu`ixxf5=S^kRt
z<@D>$&(lA5{aXCz;J=;!cm1zp2xKZ?UdG(cvVcv3^BI>j_bjexuEktDTx&UIvg@-i
zX8*%+kMlTZ1lKn1b=+q-C$mpwUCk`cG@HTf|B-*W|JfOs7;67-{ulhO|DWi;X@92u
z?)}aC+wSMwZznz%eLVI);N8PFKVL6;o%s6eE6!IZU;KT3?0NO`lh3|Cjd_~*^v=_Z
zPraU+Je}}#%2S1>Wl!v%^gZ46{NjuCFP^=a{_@*P=2zce2ES5$bNQX^2fq*hKg{{W
z`DN`FgD;YwnLi7DDf)8$^ZL(6zf^vE`<?a2zi)HD8Gm>A-u;d3+t05u-_*a&{krZ;
z;TP+#9pC(ZO#JoYkI(<PjPlHfncZ1lu;j7+WnIeV%kIr_hw~2iMP4cXK7IuOCc#$%
z>-c5)@AH2WXcdeUY!hG?;1tN<SLEyC{m*-YZxjE0{&fER{O1Jn1(Nyp@_gog$+d{{
zAjcw(W1J_r9e7{!uHx(EFA$g{*dwGWtRhk^`dn0B)Jnu(c$Ls$A#>qW;Z?#ZA}pfZ
zqNyTI!a+ikLSjO51g{GC2}}^k5j-p?B4{gcfnSmTI&TC|J+~B3A#W!y4^J=GEY3oX
zS8R5yf0;d5wz6`uiLolM*07nfPh<09Wo4Pqtj^-k@|}4xOD|go$5GA)oOYZMoV&Ts
zabM&9#J!xSkoPn1LtZ7`PM+I5vv}qC;`qw=7V+KVi{g92)6T8PZOfg>J&&7>=LJtb
zpO}ETV6R|<;2*)?LIuLL!be3y#p5Maq`ah3rS?l{N*|SaDXA;9O8TCRt}LhQL768q
z9<og`bEGCnZk6nlWS7*HWRQ}U)|5$;*)HuTJzM&g^akm2X*X$E>6uc$Bs|1og)a+i
z<kR6*=c(q7<!<3V$n}8Z7@HXD0p`U_os5wT3IE^z)BAt(KLbPT|3Ckd{<r^s^w00#
zihq~?KV#U$pzweC-)(;u{(ke*^@r#8=<hGS7k$6=Rq$)uSKY6cpP4?Ezb|^X>h1hD
zp|2;u%6zr;<?H9kPq#f<@u2s9{=L0-f^KiRrFl#7=Emzm*KS_jf356>>n)8tdiR_k
zs6MKBto!8A6V_*G&u_e_e&zqB?OpcAk}n&-efxgn$LXIJes%q}`1Ako4~7ZMS6F7T
zZex4JzK7#F$6<~=oO`%sd0BaDc}jS8@l4=x;<n=a&CbdZ#j%8AC1*YNMBX%hKLH&9
z4}o04Jwj!|e}wOed=<SZc24YpsEFuS5g}1tQ3KJPqSM8a#52WbiS>w15HS_b7M#a#
z#uv@2$(zYr$lJ)Xj!To%mg53jAIm8w2d2l2Z43hctbgDAA@g11>&;KvAD4XC_hHWm
z-4EgKo!&Ei;QR3I-MzQ#-hOzS`_AIM!-q{DCx7PqI_>M~uMfYre%tW<*N;O#%YKFb
zUiACKuZExLKc;<m{NDQg$#?M|Nk4x6nELbC&)q)*e>(oW_H*~I3BMcu$ox}dXklE=
zsKI!MVJ^cghP4cmjIB(oSoGQI*;cZtvIn#OW1Gp=z_y!>fxU`Nnw6Pl4|6$l0n;G{
z&i@zx`ut`8d+Cq$UzdLp|Hc2C{X6())9<$5tAFSIX8OJ27tgQPKZ3u9d=38m;iJq)
zpAWI`%ic|X+xmw0waCj?&v>48JT`nJ@o>R|8xM*eN<M0R6#wY?L%v6U9&LZ@^@Q)4
z%F8pah2G}8&3K#fcFNo2xBK4gd-MBk%6qvFu^+a6==r$v)ALXBJ_>v&dmsE>?!EVW
z@efKL89ym~a{0*o!Qs8<`={@xd|>?8@G<Pu!%u5I)qIlpZ26`7>#c9KKhl1c|8f79
z`k#kkCWAQRbjBE_2<AMNeXO1AVVsFv)4Aqx-R6qqmgSM=4dqi5@D@5C%pf8lvRcGk
zw0I$d4&#0%Q<e_aZEQU3ci8OM+c=JLRq`(9i{bsr{e|Z={~@7mB9WpWL{Ey}knEDm
zlxmhrm8zCZl8};Ml+=^rkus8)Bf4F<OISu!RoqVEfq0czt*DTgoVb|yXR#&X)#8gp
z%|uv5tHrNM{*n|EZxG%n=qUI@fKTwHV7_p?=pV63;)lh*h}VnTh$)K(ikgT^N`=b2
zmZ_3)lL?mHEPqx}S*cZVzG9tHuyU%3lbVjkF^$#g|5dA1kE+RQ>T5sN{-@=t*`;2u
zma8^NO-t>s>OVDOjU^g?G;%cOXfUhYRVr2pmp73=EN3RWK{{M|t#ps{QK<+?Veupp
zKcRhsFN9o0v&CPCcZ<1;JQCCw&=u$xv=%xbz|FUlTaQbO(~YB>eFj@O>jswBEDu=2
z*s@q>Fu!KxWxUNWi$Rp3>HngCmVbl(9QsrG_tzhX-;6)Md=vO~{M*bQuYPg-z3}(<
zALc*if0F(#`WOA5=YQ?r{NFQvefe$vFYABAe~JI@|F8a6V%YrO;s4VA><k6}oBy5p
zd-Kog-{rq#e<uD&{h9vz^`Di0PyKEDd*!d=e|N@w=1SHNY+KlWvnz0@a+Y&{;K}9I
z<XFP`n5By~mo1I$DeE^@Hnsv*9oB`c4_Lpj9%VD&SjH*Hy^Uuf-$VXx{v&+F{4s)$
zgye<Wgc5~PgyTgxM3{y7g=_^(c+I%#IF57N<MQB{#r2)-A@eUrX2uH)Zy4AaQyI@Q
zu4Mek@aO;Ee_#LR{!#yx^rQF3@t-cgT7N$NF8=Mqm&>33$o=jA*Uk8q<vIH#&JNDM
zoC|n8gagD&#r;I21UB$q;(H;uT%=fRt(dxKk8qjL89^PP^FnSy_XOJcSMy&JY8C$`
zaZ5yyUyElYPYnNLp=1#zQ58{PkvBp!g&&9%i7*Sr@|*HE3YCisN?(>z6+6PWlH&!N
zKSwB6Cr>yZ3qL#m13qqk6@h&Mf&AjUMLhrbbVawwyi+ieXP0Ua^%njryjM&~GF-}D
z`kwS2nPj=s@<-&~$xfF_l~^hDT0u{(OYObVY1s!7-eS|mu1O@zT$C49T&yrhVVPp6
z(jA3VSzF1&qT57b#TUu&Dmp1<$exyP5Q`Lx7Ecss7WWbF75^c=SaOQArgXBnk5C|g
ztU!~<bqNV+C20>yd9fYBCBkJQ+eA~u_{8>#ZWiN|?3eD7VUo5IPZE|B+9T{I#wRgP
ze4fZ7f%Sac{2YRHLQF!%0t^C|1t*B;h<y}YC(J5vg2#&6m7ANVnR_#P2;-*TqCYSF
zDEc|^7sv13pL*Z(zPbE(`-}09?eASb)qYfepYr3|uaLi2{;L1B`+ni8)Q`=-*Z+<D
zd+hhkU$cMz_%pgXZgh3rpjjO!a8Y1`K)b*Lfs2BVg$@Y2ixi5S6mb{T7EKW;7cLM=
z7F;QCnm>n6h1ZFvoqHwMHI759_n4#@WB<GTRrwS8C-+a%Z?B)PzuA8E_!9it>(hyk
zRUhws`1XFsJCC>9-e|n_e7pFK*PF9%c;7vG|K;P+&tJaCeog&S|9SGKRUhYlZ2P43
zneX$VPYs`JKY4uG|Ec72-j}sszkjp;G3n>kUk`pa|Jm}V{*S<)gTF8R*8Vg1kMm!*
ze;)s<{vY{w@$a`kPJf>MzW$r<kIdiQ|I+_&`=|47*T0+ppD}D>3}O1tRLmU0yqHOo
zNuKE^V<F=?23>{~|F`_NWZ1xPmO-4M=3m{P&R=;yZhdR}Hsc%jx5r;%Kkxq7`C;n&
z`gcd)PIw#g_SKuLH?v-EdiCq2z{|(aUp(`E_UozIvjfj&Jzw+U+RHt!lwKcr)%mjh
z#ikcoFMqrgdoBKU{ks?M{NCMsEBNl!yIt?AKQw>z`6T%%;A7s0<o5^P?S7a3F8}Sx
zH$UE7dYk^<`D53oU!M+sI`%Q{gVy_n?|9$;eXsw)^u5))J8uKu2ELj3dg5!2Hz(e-
zymfh3_g?ekqEF{O|NY|sZQFOZpPzm`_`~+!lChaNlU0Y!jIE2!k^L$AMvga}bGh4i
zW%&2<FXsQppDh?Bd|G6asJqxyvDso=Vjo4KM2?AQi(U}X5or=O6}~C#EqYk&nD~3~
z9C0S`<zjqdkzx`O=Ortowo5&dIw!S7YN6CLsgF_@q-V>>%dU{smvff8Dyt*wDPt*J
zDCI3xAvIm<yp*kUleE9|WGPR{6!GbzYefP?b_@FpM+!d@sug^~zlTqd?>uik?+G3*
zo@DMqt~O3pPB~5kPAkqS9P;d}Y}Z+X*ygd<aX#dl%q`5L!1I7xoaYUXEMGf+f&i01
zG`}x@B)=(tB0r13BY{VP^Mp1C*$Ax_WENx;*upoL_a#pWPd3jRo*3SVynlIH_!9V5
z@uu+p=e@}{lYf^$hLEQSlbC{dySST#loXFlk?aDw6!{DCc?y>miWKY>8Wq0Er_0@x
z;gs>0>6K-d7f~=&xFxqxrc8=W!b!ARSYP;)&`ZH40Z;xSJ`TQ%ydAvZy#Bnec<ymu
z=5pp*$+?81fL)bsFUx9Xe&$A|lZ>8>{){z@YK+_rv;GPEJ^EYe_upSPeu@5m_S^1H
z$M225a(^xR#qnF@_mW>)zjA**`>y$I<(J=|f<N+o@OyXbwfw8;FN<C_z4U$g_<8Nq
z+mFuPKYU00_SIX*Z&}~cxtVc2?JC!m#LM?D&A#;g(!I+cuUKAdzP{wf>ziq}SKm2&
zSL*({2c?f@K2dvq=cUyfzIUtMU;FUy!^sa}AKO1keKGjv{Nv5f4ZkD*M*Z9H{|zG-
z%Q{vz_L=PK*sVCOaj<bN<6Oq|f-9D54`&UhGS?<<H(n#Y0=_rAS9xx7yKswf@8nkC
zHRf~T`^o!>x0G)dKf554&|RSjk>jF<VwPf+ViUw_#j3;v#8^ac3%?UG7OE7?5s>2F
z#9Pg+!m)_uFyq|+asRIT_4vE{kM^I-zq)_?{<`q9>ZeT~e|>0qzxG|<yFG6+-fVq6
z<8|?C+1HV;@?SQ+WPNS_w&=ak$EzPjKbd`c`?2WbqK|t%UHM$_<@lF7UwFTM`l|EY
z<45DqJ-;e`oBYxHyX&vvKdt{0{;&DB@bAq(`+qO}rT$CqSLH8>-+jOL|IYlK_G`{h
z#$V@uHUFOdN93RTe*=bP3|kqJ8SR-3SQFR;*yGrp*iW+^WwmB`$z;cr!FcDt_rIpU
ztNteb`}gm{zf*tJ{(Sm5_4}HyzF&$z3w@UPbm@J`Tgf*9uOGf-e%bb-^ac0JtuLRv
zRDQMN)z4S!UopJ0dAaa~+>8Cs4PS`840`GRa@$Lfm(9;LpIv&|__XF}{<C?{dtNMi
z+5c+mt4ptzzJC5j^4<M+@86w#C-vUp{rq=Q@3P;mdH3yY%^S(r^IloJW_siKhT+YZ
zSA{RVU+_Jbe=74t{HegR*-t+|Ir-$wllG^a&n;evy^MKv=Jo40Ki-PHH~kR%vFp>J
z&zfJ~e2e=T_lNEOWd;wXdCbWyOIY$*{xX}ftYuARx8r!vA<xCg{fBEhR|A(4_hz1S
zK0p2${J;3q1!f6!2xtf}2s8*h7Vr?5%Fiv(AgChjC(<p#D`G9oF7#YrrocCW1A^a$
z-U&O3ych8mOBUCV2$rmn3X`5BBPtgtU#sw3@wZZ}vYGNSB`&4QijNdODqd7fP~4&*
zsbC|&Sk_$Tk@Pd^4bm5+g=Mx$%S+FdDqt7|qaiRF0;3@?8UmvsFt|gY{Kumo+&^Fb
zSo*{F2mg=#KNkL~`s46d{a*xw3sW+a3=;!$8_OnEXEp~m9=0E>j;wyn4vh5-^$h?2
zum2bG_v4?Kzmxuc_`~)m^mofIho5`C%Y3W&!u<L2$HO0r-+y}>_g41ZzIRXFmb|(1
zn(<BQo8NEtzn%E5?R~(9*bk@Q`@ZLY|M1<Fcjw<#yj%5_=Pm2o%Wu}dUi#|Yt0S-X
zzIJ%+`TELhw>L&_cDxpOqxG)jgYd@*AL2i_e&~81@Q&|o`J2ygcD~*JHt}uM+XrtM
z-YtH+<4x>qjaTI_&0c<fvGs-Z%Wto4ylH)#`gZbL>31#fj=y7lKl8olN7K*xU%9@!
z{4D;>``7rN<9{;-f5xj!JuKX8>Fk`c9NXEpvQA;?W$t0hWZcfs#-Pc-!jQt?&G7Yq
zKf?`%tp6_m&i%9dzu@1FzX$%v{c-w}^JnpInP1C)Nd74NarFns&%B@QKdXOue6#v|
z=)L~iw%3zhZg{@zS-{f?kNF?Zd(8MG{>kqrc2Ac+X?eWnQR|~ik3K#Mderc6`a|VM
zu8+Py+WlnaGnE%LFWX;vzTW<N-D|7Y$*<a9E_liK`q7)o?+QP>`Sk0{?5}&jI)2;p
zZOV6*pMk%w{95>X^&j@XTmQKKk@;isr}K~KU(tW(|KDbGWIo9(%d&*UmvtknC#x3A
zTc!l2A534EJy^c8#Ic{_wB#z|yv`BE;lgo*qntyTU6E}9>v0wX76F!z%(5&qSSGVr
zu;{V;V{T%$V7|@7!c@k1n6Z%QBvT>NL&l?wC5*xh`~DsLXa4`;Kjwc{e`EgW{O<Z$
z_5Iq{xUUIcqrP%{3;160qwA;quZKU}zCHfD>!Z*6fVYKjY~M7zS^ReH`|}^~f3E$i
z{cZEt&M$qRXMfKB!uIw3mt|jOeOd6u<jc*^yT0Uq+xY#+kMLg`f1CfQ`Sb9P)ZfQ{
z(*9Wd`SK_B-^%|#7#=bmVmZ#L&Q`^CitP*AI<`nw24*ovZ-$Eh9{>3N?fLiQ-;96j
z|GE74W9VRHW9nwS!{Ee_`@i<zy1#e-{QjNw+y8gl@4LU>{l4<M`*+f>*dHgpsefDZ
zb;1{~PgmY?zn<}Y?UT{PL8FU<Mi&Q-E)E)995lK(XmoMV=;EN!#X+A0Mi&QBb8%1}
zUlY$YE)ULw?4MXmm@64r|F!&a_<ifA%1@`CqCX#e7x<R=h3E6rkLe%c-tBo4^d{xa
z_t$2x@4u3H9rya_tH&>^Upl>7^V;g|zIPkmKYsu0{n_`{?=#-<yo-5v>|MjVz_)I1
za^6gQ`{3Qz_d7oXe|-5-^3%+ZDj%~ya(#;al=-Ru)5A~cpTB>O{*w2__lxkCwVzLX
za{Kt={g(IJKFEDa`aJ8)ov+uviGAPrZS}Y3-`Rd~|LOaC{~rUxDuy<OHUAI&)BM-}
zx93m(?-Rdf{%ZQ=|LfDwcRyBs>-fC(L-xDcH}<bhUN^pe`fAb3eJ^IbIQ(Mu3)dG6
zFM?h?d=dXr^VO8sYHuyywY-yi*Z7w2?bbKBZ~Whsy}9%z{O!`W9&a7qh`la;HRl!g
zYnRubUmtvX>%GoLwNEQQ@B6y;d-o6VA9mj#f9w2K|26Q-t<Mr)a=&POG5Z|<De=>Z
zPZvM=eJc65@B`-um-nLY#NV>Lz5MpbySwjpy<Pw2@0){fH@%I2EAeja`{a-NKh64F
z{Dt}3i|?#IbAFos+VZ>OZ^i#AMsw!rELv=z*l%*UaV%%I;fUb;&pC%{CHDiKKfDk5
zZ233xs|p?vx+b($kX<lTaJHbNpp$^Kz)=Bv!CL}9_{;e}@|y{o3)Kou7pfQP7L*g<
z<u~Tr!CS_6il0m10l$UdR^bHEFQP4CTg43|qov+U6-c+q7|WfPf1t2a@r#nZO1Vn5
zvVxMlqM8D~{25sRnGcfM5&_~`;?u<xL|+M?5H=Fl6-?m|<^RU7D)5PaB7X+|fBxA5
zMgnvB6$N+&tpzU#)Cueom?}6+h*5Z*5WA3zV5UGMzaHOW-Uz-fegT18{x7_9c;a{-
z@o4f^^FHTO6ud1|C0r!jA<QIVA+kaEl~AzIBf(>WVnX3U@<LYy*9-azY6x`l)$?BB
zoyNDGZwudL{^f!e!to+SqW8qkNEk|8l5CXtC9Wv(NZeTbg;=9_mBcQId*a{3E{U;<
ze-O(Pa}nJlY$x<uU?0CA|35xE{)zlH{1v?N+$T8QIqNtkvB$D=aj0@W<vhw+#`%(S
z6W3KPd#)3lk2#&W<hepQ7jcwv_;GAwU&<!N8o}((<jvI2WY4sa@d6_Qb31b^lMdq~
z#!4n_rW*`1|EvF>@&Dxi-hT)G-2EN-`}I%d9|GT(eT)0{<tyvgsL!b%SG?DJx9APq
z>r*d<UW&X>c{cBH)I);@eh=n8_;jD;zT*Aq_XY3E+%vj2>;CG8_aAS2s`Gr)i;7oz
zZ`$6@dw1Zy$w#rz>R)$#%loeS<IT?pzvKSA{+;$`#b5h>KmSVqGx)!ip_|c`aTCJ~
z25p9;|MdRm{n`Cz&R@@eP5(Uqt1up5;$xo8^qz4WgXsTzf8+k9{5|=1=D&{rXa0-*
zpYX5ye<9<0rnk%ttX6E<?71B5oE@B#xVCbC<W}cC!zs?8!Ir`DgQ=BKmci-&rhi6%
zD}O!vp7hP}>zdCeK1zS^dSCWl?Y;He*4JEb+}|#E)Afq`Wx|VT&#j-$dUE6O;U|gD
zET4BgH-Dl1a^K5mFE6~D{4(+7#FtjDKD>&2ee?CAH@DuLey#9|_2rKjw_e_T74&-L
z>n(4-y!C&l{%*y)7w;`TihY{>$^LWJm#be{zu){$?Lnj0Tm?K=c=qzV<e9{)&$p77
zfoB=lf6fHX8yrTQzFZF68Qd?pBDu7=D!7`s<9Oxy*#-Ipcmx>)!vt>e^9Y;~*d%B!
z6fbm8C`I_V@MPi7LhM3zf`<io1Wxj8<}Kvi$$OqxgLf<UcFtV(yDTb9{{L_Onf5#E
zcf_xo-)p`WeR=+w{j=96?N6?sUVJ?CA?y8vcVFK1y^DG)^v3tK+UpaqJ>T}fxBax}
z3*$HIZ(F`T`=a&b)93InFTZ^K68xqAv+ifs&zC=4`84UX?AKr48h`ZvjQb`3d-L!2
zzmNaU`|bL>=ePWyD}Sc^E&jLg|04zq#&Cw@|9bzw{cHcn`+ow%G{$9&y^OOM*D>B_
ze98Ei@d6Voivvq2a|=@m69@Aw=3r(+CK1LMhK&EY|8oB2{C8%MV$k@{|L^smBfo$A
z+W)KR*W8~$KX|`S`1<Yh@lS%EvOli)aN#}6`+#?;Z#CYWd=>ff<@2Y{VxB#D+V%AC
z(;3h1Jy(BO{&LTYFVCx=KYiBmjQM%<i^s2KzWwvA;T^}j>+cS}fAZesgUv_FPhFn`
zK5>2w`r!M4^8?#^{&xr8?tXjfZRI=u5AvU0em?f4;EUsDjZa%XUjAtOsqa(sr!OCE
zKkocc{2}+lh7Zvn3qS4o%<|RtTkiMzA2mN+e%<?3^xNQ%(%+hY!3+sZkt|MZoE)z?
zPH`ONXy^3j`o}e$hn0Vtz&U{<0_K931(Sp<g?mLT#CpV+i*FYDDQYKnQ*4`9zvyG(
z2B9^Ap9P!+VgzahcL>P}?-Pm=auZr0v|Tt`be|Zb_+znqVk^Wv#S+AP#cxV9OTCo-
zCNop^hU_KT4B2p*A5yGR>5@++7D;qTq)055_#q)I@j+BV<gSp5U=#mhzO8)G{D1hv
z`Mr4$axdau!)?m_oa+JCb}nzOCeDQ%wj3cGQXFjT<*Z69wal}aFR{q6=W^cSYT~Zu
z?&WgiT*UE^qmxsItBot4OP-6F%Yo}K*F^3~JS%xO@D=c%;pY_K7MRKZk}sDpn6HKJ
z5Z_%s4SrUEp8{Ee34*r-KJmBma|p~7cqH&ZaEfrRNTNui$VO2W@o0&gk{hL`$O_A^
zk*||)l2=fep<tt+Dt|$iPgX^?PWFrJV>vB_!wPH)o^rV|qSEnFhEgt4k0j?vJQ7n8
zO%dKC7%0%pZ^>`Muh0LA&zH}e_XBr3mk*}~hZwsT+ZERHtXym=Y`SczY*K6sSyY(g
zm|7W|8L}A+8Gkc+GaX@E!tnI}vHt=8WByJ3tMm8tAI3lZzh3{i@SXkpiLZM;7k!-e
zKImQg+wX6<-kx|9^IG%et7jEY#2y`cFyVpggOK~L?n>S9yd`{d-i_Ti!fy)S+IlPh
zw%8rtJ4tso+;P6^dT-f%yN98VnVy||QU5yOo%6@8&mX^3eeL@C=Ii5cm%gw5QTWU4
zZ_$5shN%n-8CNpJGn=w>vc|AWasKAC<~qhTmHQ3%MDBH5dpPSkHMvx{WqD$FSMhoA
zzva8Zdzt4j&t2XCemj9h0tW?7jV=xvT^uyJIB0Zn(CFfz(ZxXnyEsVFP{LedlEeXV
zN3qi)GQvHAX#xiP6@1qG%>w@fUJJ+xDhp|fREd2NFOeve(3iL@o+ExptX|AsoKYfL
z{J!WO5f%|g;fX>rLcaxTgyMyF3l|GJ2y+S_5?s!2!MBY!oHvM9hHocdFW*%@Ie~0}
zU3}WSi+TC@kMlp}Zx*;MXecZo@?4}}bgrnCn1+OxWVpm$@skodl0Fi$5(N^M#4m}x
z6V(&365<oG7nTx<6iFAkD_kg)B4EyUh$o3Bhli2p9hW%g12zR#OO|WQznN#Su(PtW
z{$M%962bC=IiES5=@_Fk(@CbIOedK(GVNq^W-$KW|8L`8hd;M}RsVYR>)9`tUtGV^
zf0h3__Veryg&&u{FaF;0J?^{WcZKiK-`{=f{_68(?&sZ~!alxv|MT6ccN*_MzMJ;;
z_Ur2}e>^+$IQ8MV`!@G~++Y3R>wTX4p7;0PUvdB0{oV(^9xQ#h|53|h^(WJwW<1~V
z;^s^C*Kc2MdGqJ(h4))NPW=4l>x&=DfBXD3{Kx%YmcfismFXW-F3S`)FOGE_D>>$J
zPT;QMP36ntE8zRU_lNH>Zy%2^k35eqPcDxF?-$<he0l<Z1#Su`31stc;k(Ie!W+hG
z#V5wE#Xp-*h)<pG2H#|UNdZQ|Y@tHoYr^3o0-~*=D@7Yc*Nfx}7Yi*C^br^x=NTR6
z86D>t9p}M2&NH}I3XQsMGz11Z1oYoTyq0|({<`M%@mDP`7rprQ{OvR4XOhn%pG|)D
z{F&nOPtT@4lYA!kbmL=>M^X>(-S@xOd*|k@w3{Y3eqOV?%6j?f1=aJ*&mKP0a>nt@
zztg5?q|SUj_4DN2lV?w^IJxko@=2=`w~wDZo_jp$IQxl3CzhRPKe6rv%gIeAolZ|V
zd*%F;i)@#rugtzO>&lrcm#>Chceye5M%2ylTeogG-cGo4{_dH3%kS@aQ2cP;L(WHP
zk9r@@c(D8a*?Sl7a@?JG$Lo&z9g90#?$q4<es||xtGlIl%kGukw|da}aLVH)Pu-rM
zeX;hH;alksXFu70J^%gpPlMk-elz@){pa%Eo}rgv3&U1Mcji?rA#64r-JC^SR@}<m
zu3VyAtla)QxxCVRO?-m<Mf{QcS^Tg0D+E#mp9`6aXo@<BX^D4;FB6|Bo*_P8+)v`4
z#3so(Qum}6$|}l-$;-*#m4B$<ulPgJT)9hyO|4AbQsbk>B&}?nZMysQR_m`dU^QYf
zRxqwL-e{6(`rgFPB+q1*$z+oX6KB(6GhOqC=GQHlEn_WKT2xwv+W6SswAHeEZDVb@
z&y>^bn8j1em6n^Vepx43XIl1}bD39JJhEJ4p<|k9P_E^wx>b3G>URw(-2;a2P0yO^
z4Ro>^wSP1OMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU
z1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtuD>!05S~L;Kv#A-c5QVsvT!
R=+gSpBNRrDP{2Gw0RZrq*DC-3

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/1c3e1c91f187f6bcea86f172ff5bbbd955a9654d b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/1c3e1c91f187f6bcea86f172ff5bbbd955a9654d
new file mode 100644
index 0000000000000000000000000000000000000000..611b38b71d541c68be9e6397dc4366c75a951532
GIT binary patch
literal 57
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bG01{z<0gw;_14tAADvt+|

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/300fe1e0a47543037cbf0243b6756c9aa48799c4 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/300fe1e0a47543037cbf0243b6756c9aa48799c4
new file mode 100644
index 0000000000000000000000000000000000000000..fcf8360b277d0f051e111ce38d6fd4e33a2c17e1
GIT binary patch
literal 72238
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoayQR|0^ELKVJA)=jq93|6VM8ed*2JH>Pi8-(<cLdGq?+|95lW*}Z4`!0~bS2c8es
zpJcu+{Z{$o*^ejROMW)~N%~v&$Njg$@8mx-|6KUh{`1eT_y2O4U$Yf)EN16qyUpUy
zyn&I0aRGxJ!>9i?|Hb}SGOT0@Vhv>9#kPZ$iTxn^Y_?*yIJQ#OH*AMEblG`X?=sgj
z@iFga@nm_x^qO%7qab6;U*?}Xe(e4g`g`uL=RewhwEp_~>+g>@-w%Gj_-*;u$6s!I
zUiB&R)9H`;AMU(Q`0(Z3`8Nw*e}851`rGTCH+pYZzL9<X=B338*Jr<<<UQH)ROy-g
zW2Hw`PaR)Odd2ZJ_f632r*AmlU3r`L&h)+Y`#0}G-tB&O@8iv{yT8l*{PcV6zqJe+
zOvX&>822$q{IB|-$LPt}z#z={oROdT0&@%VN#=Hz<E$m@KUm|L^qF=ue_~=_Ec(CW
zpUi)o{}=w2{rUZG4vQvx4%;!d8*GNGi<wOrU;VB8ZT7R_yYjbXp9MY(eKq*%{Du4L
zr!NaWrF{7E{^$qM50l=Oz3F_@^xEdt!k4KpLtpZ}nD_MJlWR|RJ#~Gu?9sCao)3&4
zT0e|>Q1PJQ!S4Gj@3%kL@bK}YLyv==)IM4JMDf|e=Qo}Qy}0nw<@M>;Y_IRVn)p`k
zWBTX&UnIXi`26lu+o!6JCLer0D1LhOx%P|NSKsdizfAw``B(S<2}3XQMAnV09&8*O
z=h-i_?qW$|*~ucxmdbvP^9FCU0E3{SkgV`}p+Lbx{zJS;ym$E)2y77u6qqY`MVL!;
zo5)e&4q*Y|?*eyu_j8qVykckI@MV9&dW>ZeOC8H1=0+BKj;Gv$ynnbuxVkyjIF&ex
z*mkjGu{>tUVu@f`!1{vCf+LHwhqH^bfYX^nkv)kmnDrrxG^;af5o<o{Qq~RZtGUi|
zujYEq`GG5zTbDD1?Go!swtZ}`*nYDsa<FrhaWZhN<2b~T#5Iw-n|n9sT#f@=dc3cB
zRrqf6v~ZPk8gX>9YO<8FdUL$w>f^5D%Hq7i!OW4z=D-robdOP%IhExva~ShcmILg+
zIj3=Pa2@72#O}f|hqH*wi<^P>B;RTNOo4BLz9Q#DQ^aJ&Pl-E7OqJ-D=#+4kSShYA
zaaZz*RF1T|%qbZc**@7`*<jf}GWTSZWWGuUOZ*T&C1xb5A+lLWP)I@ekg&hVVzF<M
z7i6}{9+Xy+oGLCaCMl9C^jz?-V6~uz;10nUp*mr4Q7usokuKp}p)`S+eCfO;JTJIT
zb1<>Hvbr%dGaY7ZX8Oy#kYxdjFl!`>AQLacg1?Qwe18W1y#2lJTif>+Klc4-{l4d$
z{CD{u-+#FM{QINqr~U8qzqkIp{O!b-qn|&2V)@woe#M7rpC)~}@uB>k!W-XLmtR;v
z-|?jP;hOu2_r>pv-}iYS{Aj}C&yNisPk5C1$l=k-M|qFA9}7LQdG!9#&qt3QH9nSp
zD*F7(bB`B2FJ)i7eZlc!+w*|ug)fX=|9u<&LEzJr&pW;fe-HSU{&mtffuG*L&i<VA
zQ~FouuUEgO{QCQ&_q*lyC*P%icKmwwXA(m{b1#cAizBlp(>@02|2F@A{}uSx^8Y5|
zROT}*MQp7cTRCkxTRG%8BG@mn6|-GoEoLibFJSw?vXW&!%XO9stck2%ET5UhSY()k
z8E60h_owHV{O{v`lK<@defxLlZ-?Kif2RM5`+MNu&VOuw6@I_{x#Fkp&&uy*UtfH_
z_=)Y4-^YTFFFsxW?Eg99Q|X7bZ~5O;ytaDP_(J#jxo45j*FDd8zUn#i%c57aUafs;
z@lyH~<C|}9Mc%uA`1G;kQ}d^TpOQXp_;l_|?Kj@<YCl|mR{cEvgZYQU59^;-e?9qo
zmcf*%k6DaWoUM_~gjJRKK9du33sVl`c7_=Y`HYE7d`w3eJQyk%8vcj=JN++_L7aI3
z>q(X@rVotfjBoxG{_Xi={`>Z?g}>GQ-2J2X@6tbxe};dme{TPF;S0;>XCMB)+xSNQ
z)ub0&pG!QqeZK6u(u<}StDajtV}1JT(cA}8_wDa4xc%|A@7?HoH}5^a7kNMYf%3z9
z505_Ld!qBK^o8t8xfi>i`@IZ(<@)mD3!zujU)jD6di&(P=4b8idw;V2V*VNTea%<5
zFS?&oKi&Gs`N{rs@Rx+I6Tf@^tp3URi}QEf@4bIc{TF8X%Jh&~gH@017F!Lw8%G+a
zGPeuwK0YaaMgG(LzXeVUei6JYcvR3sNM4vnxJc-uP=&}f(Eu?=u{UCK#P^A1i5?cY
zCEO+ST)>vEoBI`K42LZHRCYE_J+3Hj8=f89!CXR|KiFN_^Vq+$7jX!1_HlmY<l?U4
zG2qqa{l_z%ho6U!X9*7<?;D<K9(SI#Jifd<eD(bGg7U(GBKjg3!V`oJ2uca8=GEeM
z;M~J8i*pj!Wv&cvd)_;INB9Kz=JBrPS<B79-NwnqA;Zzi@s?eXeJ)!r+ZMJAb_Wgz
z&Ri}T?n<sU4s$kXmJ5tU|BwH@_{aQ@?w_T9B>(>X`~Uw|rh_aVtbbS%StHm^vK?eQ
zz*fY@&o+(iKD!sEG50avSNx2E9fBtW-wB2YZ4$aB#3x)VyiDZ2Sg*u)i6lv8sVkD)
zl7<rT;>O~A;(p?j#2Uq(iWQ1!h`taO5Ka<$Ca5E{QP@N5v;?=*eMt$)IEl~Vufz|E
zKNn*clNCEDmLUFG+)JWdVza~?i3ag}(H7y^LR>=k1)BN0`Hu4j@UG#><(|tmlj|m@
zEXQ`Xt*ld6dYOBeG8wlsv@vKhurQ=Bcr$$c-_LM^A?v@(zjOcW{xA5q<L`k#a(|ru
z<osFuTjtmDACf-`e_Z{+@iXsd`_Jkh9^b4!A9}C<w(a$#mm8ihdlv9?!ejo&^Byxk
ziGT9@iQUtsPg)+YdDQyo(xZ=$f*v(Ioc>Vxk?W)Hk9I$q`Ap?S&CB*zp0BsRUiaGS
zb@HqBmkVApzJBy(^1H$hZ$AC{GW+Y^ua4jLe4FxJ<!9ipE58>0Uj2vt@76!=e`Nkx
z{OSB7`d9Sd`Tw^W9hpxu%d#wC@nzk}>dC6b@|G!q=?BvnW)GI{EOG4TI4!x#IInYr
zaky|C;V9=&W>;jJz<Qj;fJK1iBeN{a43^0(7A$%!|CpPYEtqdJu`rb}9%d|LI>}VX
z^pNo=V+o@$!@hq9|C#@P_>cLY)!&#uI={PqR(-$rHSTM|*Ql=?-vYju{OI~A|Lfro
zw{MR>@A~NTKHzQP8{0PxZx+9u`~LjL`=4vSYJc1Owew5g=h>h0zp#CM|7F>iSzi`>
zG5K=y^R6%X-!^_f@+17$#^2_DYW_U@BlY+3pR_*~f4=;Q{kQV}4~B<KhggoYs<TzG
zonrgKwvH{5m4R7|(VL;-zsEoRe|!Et`8VU=`hPC}{TMnJ*_gT+?=Uzq<o>Vyx9;!V
zKfixx{r3Oe_WSPdcfYUv?*5(hEB42UZ|dLHe4X&c>(iBY+^=UmU;AXvqdgC&Jj{J)
z{E+8i!-IhP^7owYTHLvD>%q-;H+gPfy0hpW!vp?@iH~|8&v-2Hc*f&7PtH8)f8z9X
z?=yiHHZT2N9e?%tRr%}xulwHEzuWpj^^@(V%O52^3Vm>V-})}~ZNr-#Z%)4Td0+Zb
z=!@aE-0v1Yj{kV@<M0pbpYFf(e&7GS<xk09zJG52k1$SQsbE{geww41vxYOBGli3%
z^Crg|jw2lBI6iZRaPQ~V;OXI&;eW%wR$#5*GNA(D{laR(c|t8h*M;PSYy}GWX7if!
zZsobflg?)%uw8&#K#Kne?+We)&Ik@yj#iFRj_VxDxHNh9@=fKxCGbv=Nl019TX>d;
zqNuZIifF8;l;}l~4<db{o?>UkJj4%)8%gv@td!`Lm?dE<5ij;fc%x9Hkfe}=P=e47
zp(x?g!ncJ?1zQBX1qAt*@apn@=Mm=F$7R8JkzI!UCEGqWRrYL-jhqHtn>Z6W_Oq$7
zE@Dn)%4F<h`1xO)VG4sd<6p*aOy11Xm>067v#PK%u})&W#X5=AkyU}UkM$&5C5Hvq
zb#77KOT3YM*Z4H}UHFywKk{AVyUFLne@#F_c&$i-=w7iK;-=yaqRT~Qh(wDpi`*7I
zBm7fXT%=NXxzJXjLScOo4bd(!R*53X8mTBLYsuH*1!5~i<b|_^bcDo()(BOL+z>l0
z!60=~>Wh?#)KLiy@m^77k%vO%f{*zZ@U7sT$1{QZ4p%vs7S|figB%U)Ic$4a*;%hK
zJ243{u47opkj`Mjpv$1mV8GzcV8EcsAi&te<jnku$&N{dG4{Xj-`L+5e+vG*`(xve
zZ9js39{=h2Q~Sr3Z)IN^Kkk3G{LQUb+%IQ5|Mu+5Gr{M-pRIf*|E%Nbt|yBgGd^y8
z9QUN;iN%wuCpVvbe4_jG>C^kqUO)f%;>OFyS8rdryzY8k`C9w6((AC-?_V!@EB@im
zN5RihU(3GV{Bh!^*zdAG7ynxR=V$0=*u;2&=`)iea|eqZTNJwt$6^j1&Nj~VoMN29
z9Bk}gS?ySVvD{~Q!V<!&%a+N0oMQ^-cg{szI@}kzY`EgM&Tw7jis!25oW)_yA;IyV
zy@BH>ry%z?ZYQ1v+}>P|IEvVBv%0b@WpZIW&+wF?hvDvj&;LFDCjDLZC;0c`pPWB4
ze%|>Z^L_Hywl8j<uYPp>aN?c+JJI*Q-?M#W{A~L*?py9R*>CAzpMKu(G2(s1+vTqd
zUKPBW@cQtZpKlkvzx!dshp+Evysv!U`TofJviBF>U3?eve%pJe58)rLf13Zr<QvcT
zzVEd^>VI1Q`ugj@?+Jg_|2^|>`F~%A42C2B*Z*J4aGSx9VJ$--lQ{E!rUIryrj<<R
znPiw`8BG{9n9eczGqbZ4vP@&~XL-()$6)>M)Su12AN=b6_4=3PpZkBG|Fixt{@>!?
z?mtq$ZGPH+-}80e7p>1`ADQ0Yc_Z?A!AqAHA<r$Ir#{zw-urCJ(`iovA2UDN_@MlL
zz`f495AWLCw|TJs!P|!~AICq{dB*ts-SbT^KD@Aaq4IqHvklL5p6z@3=4t7(#^(wz
zXTM^3<M8hKhuTj@pW8lXedhc8=u`KnUmu@*ocn3r=dLgJzlePO@Rj?!>GxgV4uA9c
z-te96JI^=OuNGe&zEys|{^Q6mslVa>MHpW)Ix;<Ey1+b(MU*v#^&4v@dmQHnE@z%C
zywCU^@?R3*6kINlB^WMTE)poBERrNLO+-bcTWF!cZT_?TQT*@uwD{NXX9+L}e&R3W
zox**HYcuCUj#3UO&K|B_9%Vi${v?4Pg3`hj!q&p~g{*`>3cnEc5oQn;5Kb0W6FDzp
zD#|a$D*jeXLTsw2u4ul<d|@8pWkO*>tU??@MM4cizXkb)W(&y*T@mmS2oQL}|C6tY
zcP4i**E&u=4qrAkR(aN4tjpPK*blL1a{6)ebAREmVwYpv!(zz%kdcouh%uZ=hxsh?
zG?p+{N48-0+3Z~G4D1is=W!h6n8T6Aaf5@8i=DfNyNTx;uNl9fz(j$?0w)C=1j_}5
z1TzJg1*!x#3C<9DCX^=ZE8;9FA~s7*SbU#^hjfh$hs*_OHJMk^f2EE}zLDsc^p!S|
zagq^`b&?B_tCTg6DUni;+$f<Z5iKq)-XSh8aaux4l2KAg@~otQ)Er3>$(a(45+B4b
ziOm;j6jBwm6<EWcz<-M`fRBYwf>(xHhI2c69vcJeedcuL9%fsX5Y}|Im2B2*`fPq|
zY^?8@Y?$^i6)|ZuZu{@>@B5#fzbF3u{B7~qWnVMD?)qZz<;LgxpWlBz^(ps5<vWkJ
z>~9vns(!ie#g!KgFE_n3e!2Ta;EVOo_dR>~#Nd(21F!qO_x9dtzHM-O?d@%M=HK<W
zH}U?ahqaGgpR9fo`?USp^XEJ-AHLlA>ci{nZ*@OB`uOrw&zJAt<bIy`RrRO!pETn>
z=65VxSl+OlV`X7$V~t{c%UZ!+&soKl!NtsN&ZEY2kV~8M82b`7HnzpAOIY`_E#=7I
zI>0@P=O@nw-Y<Mt_`L<f1-A(033m!B37ZO)3swov7OWKP6J!#q6q+b>N3cdfo&PIu
z22Ul|0uDKLA+~x}bJkBRrYy^to-^40KmO;`PsbmYKURDf{dV+=@RwDeEk0|0I`d)f
zhop~{AJ2U#`>^<f(8u79fgcRs?RsPI#{ae1tCugEU+sTw`}WP-O>ein)p}R`?&>?s
z_loboze{_+^S$-^b?<(>ll^e#<MPj8Up>Fa{ao|w@vphR{C-*fivH#F>(0;9KX?6F
z{O91mo&R_JuVV;gDqvp5+|IIqO@i|omoxV)u4t~sTs&NBIcBo!voB`<!*P%EIA;Xc
zHtu!YXE-ObPi0-rEY38W!R-H$f4TqJ8JHMq|8M>m{ICC?=)Y-yrv2{y&HLN#=iF~6
zJ{NsF_CDa<!#6))FM6H$`s*vsS0`WmeSYkD_4AX@zCVq5n)vk2(~D2No|-(J@N~*k
zg{Ngt?4R^K-Szz9i}f#_y_o*;+e_wG-(Lp5Qhjszo$Uv|5C1>R`Na8U?H7YDlAoDB
z3w|m3a{lxB&qu#hetY|!^~b+&bH5pXclqA^jqTgduQK1%zs>!+?n~hp>#rT({C-UQ
z_2Q4u|GA9v%!irXSzfT@vHoRU%I3@N&2fkG4);Y~DgHiw1py|(R|4z!W%=*(e-daF
zj1+7WU>D#N$lzDx>*M{;dxLKi|9$>+{{8&t1o8!v`S$XB=6=bwi1Q%FB93F6C%7GW
zU-Pcw>*X&Hm?YREq$;c;QZ4#iRA1Ce#9w%o&|x8S;Z)&O!YLvwqTHgXB2L0VLXtva
zLURPK3it_35XcccEGQyqD{z5dk^eex1W!G;6i*><Cod0AFV`&2LXKB#cC3GyJz2K0
za<PfADzMhDnX*q~^I~OXna`}w;?MG(c`-{bTL;Hc&Ig=!oDrP6xz2H4<Nn0GoTrfY
zGw(xQCEiY++dQ*)<@w_H%J>%X-Q$bmd&1Mst;lW5oyk3qn~moMPd=ZRfVp6=V1wWv
z!QVm!!nMLjMMA~nB~_%nq*A5!OKD0Um3k?uE451co{X+6r|dzQCo&$gO)_(&CP;3T
z?2}}d)RknAl9tw#Nt4+w?I%53`j+$t>2hf|X<6x+QokfT#A1an3vA@m;Z^6U=8olV
z;XcUqfa4gO80!J%#Y~-ykqim{-~QA4fAc>BL+t-Q|C0W<|9|w)@861lm;XOw*u<dl
zfBN5Te-{3J^V9W*=lAIEFTNLjzx7q{Yui`dua=*gK9;{PdbjHB{5PSmC%?*kwe{ue
z=gCjEJzDXg_kRApy?26cZ@Q&<OY!E$>p|CUUfqAK?1t+tjXQexoFAw@s(GyY<k1t>
zXKBxGyr_QV|EBF-_Q#Sh8^3+~e&fgKpBH|0{kHh?|L+fm3CvemX0dK#d&Rzo<2uJ-
zjy;@vxMg`+d24w}cy{qj;Bn%%;{46d$`Qq}gkvRVJ@-W3G=4t;9RUx4T){m;Wx{`i
z?}>aBy(xB1?189==vNUTQC?94(Ve2x#gfD`#b=52h)xhO70wo%$8W|L&8x|q$y><V
z$g_@1lhc;t0$U%;DJBP|$Bb<Z0{^Uk-~A!;UE}M`Pud@seAxG4&j;NP;qRT^GkoCt
z@b2Bcx9i@1c$@pq;=RL%O&=$J=KDJB>*}u$zqWqc@cq}1LqE%Yh5ugk`^2w?pXooQ
zeRurc`u)jw@gGS)e*T#H^V!ecKLdX{{=D{c_pb@R8~(`rQ)6ghT+XP$c!*&x!z_li
z43dnkOsiP*+3ML=vZ=BMv;Sk8$=1NOn~j0JicOl8nPm@iIdcKiAqLL>7ytVFW&eBW
zkM&=de-i)2|C{|g_-E7aw%@CN=l*8;z2X<ouh&0<zlVGc{`}#i%txOOvG2>?O@7<@
zhWE9|%U92Mo_0Jod?fL3!GjwQiXKWnYJC*{==npwM}Hn|f9&;y@0rTWGp~i-=Df{#
zoAGwa+vK<V-t2qx`)$g5xeu`)wtndOxboBUPxC$sd?<S#{9f+8_j~aVN*@_NDSdMJ
z$o#?Kz32O<@27lV{Mhg@?9;<fYd+O{lK5=-rTXiwZ?!+tewF`m|CjoohhZjzIOBB2
z7^Vp3JeGZ|o$O(piCojU=5gKTisY8%k>(BMQxxzPIv~s-A|SF_#9Xvk^u6dkv3ugv
zCH6>!ODIT0OGrpe6-yF{6*?fWg};$sSU^EAOvqSRPFP0hy+E?SMu8N;mx6^t{=&8*
z>qXi`6hwXtXNp*ez7y>cGZnuft|D<m{IS?<(SDIV!q0>fg_a8i2_F%@Bit+;Bs4?7
zn?Hf?Iu8rC4Oa_SFqbUnZuawRmTa3?*;or$R<M+_o@RAsUB!~llE>o8V#yrB`1b#W
z|Md+0jCRZgtd;DSIHWnlIE>iK*_N>#WPiiq#3{h}mZOZLkbM$cHQOJy@9bfmXSghR
zx_ATlGWneNg!ne{I`GQy%JMStO7LFbJ<S)vU(COnpG&|>AX`9C&_Zahu#{+_=yj1U
z5lPWAqUmDk;`=2INJ>bZm#mazlk$*$FTG4UO}bp#S^A$;w)7mCJF;1FUu1X4I7mxN
zO_SUzDJUf&bxG1sQboc-%v|J=kiO7XL0Q2-fh7JczHnX+o-D2n9M<d-Y&@+0n3b7d
zF}X4yVisl*WN~Fa#n{I1;eYu51OGz)e_+_pD9UKZ@Z^8)|KNYm|D5|>{X5}z*{{z(
zSbx0wzT^9%Z{=Sme*E>$>8<&j4X<CnUiC)gZPS}aueQ9{@GS7@x+fk_^dIX#G`fH0
zZrdG;+oiWA-YUMm|IW9&fA2XwSo?6rqr}HKPi8({{p`i_l9&Hq^}Vrt$MNCo$Nis0
zzGi<D_+j?*`Olc&_x@@!%x5fP`pvX~`5lWQn;iQtj@MiTJXd)3^1S4k#H-J@l9z#J
z8P|W#1kM{AMx4G}4%`{sFSsJPw7Dv{nz-Y5<@wnK`UH3c83e-wZt?R7oDtY0Xf6~l
zbWkWo__*+7;m<<sLUw|O1$YEb@@?iV<lV`8o>zl+EBAKJT=u&xDop<WZ~mF~JM4GF
zubba%z7~CX{+a!=*C*{yuAg3fJoF*!{eyR3-u1nUdMotC_qE#V6R$nr_P@9NwCD@t
zH|uX(zCQb+_2tv&@GmdFeEkyqrT??;XV%Y`KVA7W>9g$DU*8&k^#6?eCI5T#@Atot
z|IYjE`n%`1{GTg-ru;4bxA6ZX1`Eb;hUEWx|G)if|Hu1(0>d=MWsJRyvl!Pg-e-Kt
z_?Phl6Dx}YODJ;-Qwb9X^DO3IW<w?s#u$c-|GEEi{^k64W{_gg_|O0E^`9fZfBf42
ztLfL=pFux(zfbu3?ep<Zf}gTKuJ~}_J<I!mcd2hR-kf|D`SRuSr_W-ZJ$c&o^zhRe
z&+a`}e_8%=&x<e5tDir8*71z_dGm|MuV%je^RD3?$GhwA4!(c#-sFSLN6SxLp9DT}
zehm8H`+@TV+k5_Z2jA{~d+KfFJN^&ypI&}G_NCy9<7bUeTRvX?X#A<~Q}d@UA8kMG
z{80QM_rrz{(H{#x?fJ~|)%IKN_xc|-KV5#^`&IPY;E&SZnt#Cz2~3eJPHdbUuQ^U}
z9OY=|^ym7=HJyi*f11EKfg=Luf|mu8ge--7MJ&X6#FvY27W*k`Cw5b8n^?c-W8ns&
zHG-c7oCRV8Y6W))$qMfiiV|`YS|GGtI9qg|7^C=Ov3p`G#5~0k#C*kXN;FHol>R0&
zQ}%}JCD{zwaG4)ctWxQcPbC&fbV;O0ESC5oAuRDhR6^vgkc(gw|6;zae9`=W_`~_V
zc@J_g;$FjT%Ke<{0oQgeZ>}cJg&ejVAskX1Z0zN%N-VX^vzRZj$g$^g-r{QFuIBFL
za^zgZ@sOjFQ-`aKE1yfAi<!%T>oC_u?nyi=c{lJC@Sox56yO$^$^VismoJ#Fh3^pG
zT|NzdR)L=aS%L|Iw*)@%xAJoc%oKPe@IY{iaIZ+BNTbL`Q5ErMiJOuerKiXW%de5I
zlW&q&P?(`$qo68(L6%QeMYc}%i|k`LErr7hYzm%oxiX^C@lu9TE>e#q=SVyfQxQ!O
z-Xs_((9CbiZ^Ez7|BBC-&ztuHcRQC4rv`@@yBFIP*7K}fY$|NJY^iKgYztXbnB$mQ
z8Jiih84MYJGkP-}VO+xS^#8H{0smwEP5rC$_w*meKmEU6|G4m-{ricpdp;L^oc2EG
zUHaSaZ@Au`coXwl^X02&6;H$-9eXh0f$M{i`>*aw-SNC7d~@E7-8aH-3g6m#EC06G
z9p5`icQ)K{zUz8#*?qf*p^urKoqSRMI^mu3$F9#Gzf^tg`ugVU<8PO~ul`Z^%k6K`
ze|3hb3=0`oGQ~5SvUIY>uuF0N=CtNI#x<4u4fjOubzFNm>p3;KRJdh%Vt7~adGWvH
zyTN;z=P=J*-T;0(fkgrb1x^WE6xblpF7QC$qTpkp1H$eig(4?K+(orTQ$)&z3xtvd
zR|=fw&*4+yb>eB~UdeTh;}GjTCMm|)|89R({)GO?{gd?D>*wolwqHHI1b_DWbmC*x
z$2%Xsz2EW9<L$OL8gD({E`H<n=Ik5ZcaPqG`FQm6moKtkQ@_-Ip8RRm$9W&yK52dC
z`+VqA!zbHM9-sDqD*2rEW$oAR-|T-(`g!%&gWt`6w*0C8Bk<?o?@Pb6|IGd4{MYTD
z$N#GTNB&*>`|Xd@pQpdC|K|H6^LO{Z^#9xb>HORE@8<t!4BHq(n7%U=GlwuQX3}Jm
zXZp!l$as!Hmtn>KE&nYUHZYuJ5ND|QSNEs$SKg0X-`c*-_{RP1@t4@oyFYe*nEJl{
z-O;xb-iEw=^(O1htk;`f{dy_z^6~Q*&%B@gdaCy9z_VG;*Sxs)a?dNJ*9Ts8zHEQ7
z=|$GdA1}pTi@#m}?!`O5cQ@Y(zI*j<*Zb-Z%^!U}Nq!3WnD-(1{lRy;-=)9He|z%H
zk2jazroVUo*!Ahxr-Pr4eT@5{^?u<y-uHjs>whqPZ}slZ+rYPhZ)U!p_?qL*i8n28
zUEbBb*ZjEX)49)ozxaRK_TBC0r(X~Lu>H4WY-Y}6)nPMZ>tb_cf6Bg*;|=Fr?lxW-
z{=NK*`Ty}}3x)}w7TF}~E;dzcwiuV#N0BIzV<Ot37esVKnuJY-Zwh;h9u_+${$4ys
zoJoAS7@t_An1sZ6$qK3MQjet0No|o@C^b#$qtpfI*)sC7D`fTMoaL^{>d1P^SV|X4
zc}rDDO_w?^Wh>nz?Jqr9%2P5$e7fjbkpPk1!v4aM!jFV%1)uQm;S=OL&s)!Xf`^MI
znY)myjZ>9Vj?;kCigOBwJUc7fb=DxZdF*wZ54k3D3-c)OJm41RdBY>i*Uq0Hz$6gO
z@5>*_Z_1y@&m!<h;E~`wp$$SdLTd$?1sMgl@Xh6Y$y36U&GUvQhIb<GU)~nJ1in?g
zDZKxAZ}QFL-zAVC<SD`=rXb!f?j|86#UoQByFe~Q{(^j-!exab1$%`?g|G7IayMl-
zW&CA&W!dFL6buz^$t{#AlVX!_60H{27ycylQm{$DlfQ_MgYP152X8p9KkqA^d)$|~
zoViwVF5xI(S7qDFvYMHnxsmB4qbH+3V-2GkBR9jWe*%Ax{#N?^_t%YIqQ9U0w)@lZ
zd*iR%U(0@R{1*AW<d@d3+@H_BYkph#<@cxHk9;5e-ragF|7!ZnqL)oCeP2F)Ui<X+
zqqFx9-x0rk_15uQ*0*$SW?WCZ%5^32^8HJ*FMYps@AAhhme-oEFS+shX4>u5cMji`
zy1(v0>7$uX)SlmYY4wKh-KzK3KK%P|@<Z6i_D@n@48A%4c=L0^@5sMV|2F)8!^p+5
zj+KplCi^;eD~@X%Y@EwDmvOz|isjnFS;MK!wTat}*NCrx?+x!&o}1h*++y52xfOVg
z`JDKE@_yni<(tLNF32QwS13Z{xTv9+rC6od1hHDNDlq{u7SY?n?}UtnDg|={r1&@S
zR&%RxEMhs#IQM_tzbk(|{_g&x{pa$p?jOIuF8r+eY17AFA6nk8eb@JH&)bYQTVKz3
zUHn@1b>yr3mrXBOU)#SedN1_x>POK}W}n`EEc&?U<DO4fJ{NpB{^iaW-mjm&>U{V3
z(fD)EuZrI$fAs$D`fK=4>;HuRYyK_#d-Ko!-%Ed~|C0Mv`Agz=-|zjuGk>T3n)8$K
z*ZE(~zi0mu`RD%MfMFTKR)%Codu9XH1U3QoICdxY(`-jstyx|&*)e4>-udtSuj%iq
zzls0;{k!n*)L*qfpMFmLzUHg%m*UStpJhH>dSCKZ@{PdjhcB65w!J8Q!ToaU%O@|D
zU+sAH^VRxS46kfnE_@;PV*hi)7h*4iUi!b>_R`~J^K;E-m!390t$CXNY~J&p7t3Du
zzuNlh(yOJfpTCiOcmLh{cPHOTy?1y&|DDvk?00M4eS2H;M)LK%R~E0C-gv%Yc=P2|
z;Y;rqe9z^d$~+N&D)4Of)6Y*%J~{KG{VC^jix*-qV_uzk{rb(1w_@*2Kg53Q`gG{C
z=GQmh;(o^cVf%lX!GmcYb27^kmVB1K%w{ZWS(DlAINo!}b1`!N;hN6Xz@@~!nJ1mk
zkADXLFaC6aSpppb8UhRg4FZn^JOrlla|<*GstEgubc^tcSPQcYJr|fM@J--=;CG>S
z!j2;EMSR7Q#Wf^?B`c)Dq$kOU$_2{TDm+*GtrV+lro2pvOX;%WBgK!37Zno}w<t&|
z*vK!IHJ5oL{Y-j;^aW{QnQhYY(sQK>Bn>6ZB_>H65O)+iEg~b_BbX*&z+b^<&EG8W
zPvEtHoS?Fhrbw087x5B_LJ56|%i=lWhs5f|?8O-+qQ&ov?h#=TaTJ~?BqQ`&utq3e
zc(-t|u!AtC@FBtF{1$xMc*A*vcxCu@^7Zmv<&zW07TCq7&AXVFkN-ITQ~qXw+k%F|
z0wT{v`bFo8T8U{$cu9s!>=i#Lp(E)dAuCZJaY_7=*gH`@5i21+A$wsdkw}qrk-Nf$
zLMZ~~e1~|Fcyf3cdERk}b3R~GV6|kq#{8Rk1`9hYJL?aYQ!EiIKbZ5G)0vJjIy0SQ
zI?8mCX(Q84MrQ`&|NZ|q{&o0s`&ad^SHGV9a{0yeEB#mbuVX*Y{!sXF`TOGUE#Kq5
zD}Gn_9{v5@x9+b#U*>+^{VD9@oA*E8oqDJ7{^Pr8Z*RZ8{_@AOGmldrp1W^z|Hu8+
z55C^#x$k*@|NRyBpWW|$@aw_Shx;G3JXU`)?P<pI9WQRaWPknk^_Dk(-d=dW<>SQ9
zZ@#|xvHZ8sU&DXg|797>7*(16G3ByMVe{fx$FY)QF6RX9D&ADSEWQH14}5?49`p9`
z2=mDE=<?+9DDZya{m!Q+@K@lbfRaEq{}#TRye7P1yjFZ-{9638`Gok?`EKw{=9d&;
z6wDSX6uu@LE+Qb>D!Nj%QFOgXzHqV7B0(R4JiaELYg`_j2iZTdmM~W`u>NcL<M8{|
zPnDleKSh5&_%84*@e9xArytWl#J$_|Cg@GdoA0m9Uf+Ku^E&SJ(^rpQR=;$5wdS?e
z+kNjgynp=u+55Bat>0(7<9QeJ?%2DAcY$x+-sHTQ`1ZlOukUw$2>$r;qvWTVA5}hP
zf8_cU{VDTP|EGta(m#Ly9Q`Hli|-fVFKa)a_~iES#rrMqw|$WNl=OMlmpfmteG~h>
z@!RTe&%d+%;{Mb3_x?WyhE)u03~T-$`ltD?|8LKq{NE>j&HUB$%m3G>pYMLG{MPY#
z?T74lwQuZSo4js({q)tMm-}AKcyai}>KCpr7+wUuc=#gzrRJ+CuhrgKzH50W_pb3R
z-`lNka^LvBDSLD2P59fTZ#~{Ryb*g{{A$iC?$<7_KfgZs_SSoyk7}P*e%|+W?f32<
z;y>)ZKmOMFt^RA^ms_7DzT|$<`eODu{!`+o6Q3@A^7~Zsap4Ef4=(RT--*9vdwcoq
zk#~3B?RvZZ&EGc%-)?#v|5oDN+V{yH_kWu8x%dn7w-?`8f9Cu&{k7$H$KQ(oRgC7$
z(^<6GKC$2AaN}6cZo?75`JZzR*GldOJb!o}^4aol<X06uAaqS=t022zsNiftNkJz8
zX@R2x_JX$re(;y`f8;k4G#9EBnl4l?)Ga6{z{_vUw}ZEg?-W0mzyp2@!L7mxqF+Q?
z#I}kXNJdM&mnx8MlQEV%FaJPcr{WhSdzEsPY-I%{c||n^e)%)90x};YwIu??wZx~3
zDTuxjJ|S!*tSgwpAIkrYUsd1}|3v-_{{Q^51&jpd@+%7P3R(+Z5U3N_B`{TRmJp-x
zIw5u;7r{({NPa!O#k>)GUHk$9x%^*v=kUbwJmS&ht>%5srzm(^s7km<xI>sp#6o0)
z@GGHUp+|zp1jU5Hh2({<3a%IQ71R*u<g4es#5;{|J>M3-%lyj)ErjDmibU^;oslq<
zx+K{s@k?A$;*q$q_zST{@hXX368FTviCq$775^ZXC*~r$Mc7X0wZJ}pLH>VycKj3h
zZTKsA<+)FAx^vcXOk$5^=i*T1e9C#0vyAg4=O(VJT=rZiI3II5bIEgsaxUU1<M89y
z#=ewIj5UJUpUIo4pUIwSA>#!`2Ih9=SSB6DNsN_D+DtbXX8u?IKjZ(&|GobX{<-@*
z^7reX%0C3YFZ&kv?aNozuTh^<KdyMM`EJo0w%4a#3cVD0q4I3r<EV!Q5Bwg?eemf%
z%YDWB)9(x3m$_$jZ`S?Q5AQ$T_EhKjrWX~j^xm|+o%im*dy|i1pVhza`j+=y^~al^
z4}Qn}dHp->&x*hH|9<|J{%7!iD?>M<E#oGJ84TJCNB`;l&HJ<a&z!%W|C;`J{#Ri<
zz{JNqo9R8{HU`oE_x{HHP5FEB@63N4|Ihpv`#<4d_y0o1_e^h@8Cb2@ve|Pv*f~2m
zCvk1%{>ZJ)eTGw<LxU}Y<p)zMqb!5d|4sjl{#O2a_C4vF<JUExPkfaA;Pt-jz1n;0
zx2><a-nhSA@TTh(_sfJA)1F&DoAu<z<HJu9pIJWdcy9he`{llu&t6`5Ir(Mc%ZV?o
zUVV5K`TFMTMQ?7sIsIDU73<3%FK)fO`zq-5%GX=oe0l5tPW|1AcQ4*sd=&dM`;-0W
ztS?u;vVOn$z5K_cAKX7*{#g3M_Xq!v{XZ7|s`}&bSN&fEg9}qKlME9Ba~sPhR%bQ`
zHXgPgtd6XH%npq84D}5E|F8cS^Y`PQn7@<$e)z-oC-isAFNdFdzRP^8_`>}8^2fs;
zir;^G8~0ZB-M)8E-j=+%^P2HZ>YLwh_P?F@uI+umhu9CN-}}DjfB*2^m3QagRlHmE
zmgg<&+skj(zh3(4-K!(7_r7*`?fLr3YqvK>Z+5&Ed875N<b&|X2_ND=xPIt*AMlRv
zZTXweZ+5=j|2FY$)!PSe8Qv{^yW>slYmHasFU?+lezEn1^~-OsZoFxIoBDS0Tj_T#
z?~cD?eLwTP=ttAf`d_)eyZkKv&HLB*pW}Zs27kt@Og${zZ0YPWoD$p}JVLyQyb^qN
z{L=+$1?LL>66_Ra6MZI<A!05%QS6-9Wzi@R0bx&}je<`FrU?`Z?iWfD))HPIR3vm(
zNLN@$_=wOHp-`b)f{ucH0vZBM0(OFn1$PO46x=4HDRM;gwAfVfj}j?TqSAuWhonlS
zlBMjVf+f$1?+{BAQx=;jdR63;@KK>`K}&)Ae2%<9Jg<3<^QQ35;W^6V%FDzX!DGmC
zisu1uCEt5qMP41=`MhPk`*<9AHuJFXN%Q;iPvd_nP$^_4Y$_}){9ovv&?=!$p=CmE
zg=&SHMSMjCMT<nHh}enti>?;^E$S<_QLI4RU*d<va>;2@`O@q%l`^MfKFWy7M#x6X
zUX^i^iI!O-GfgH{X042`><L*;IR`m*`Qr*}lolx`sOYN&Yp7{9Xe6tbtDRH*qjEs`
zjM6nFBjv|R%t{%Gv5JL?uM{fem&$&Swv(DJAuoPc)I_vM^nfU%XsNKSke6Vizy<zr
z{&0RJfj~iVAtNCMp^bvRg3AQ@_?`Fyc(3qy^8|9Ma>;X=afq<*VEw>6ow4+P-#`6-
ztbcF+j{kG(Z|lGKe;NM{{yYBf%U_j0KYp70xcJTY>yFR0pI(04_>uWz>wAlL-{1at
zEBv<d_4Ah<FVvndd&=^p{ZZb-ng<d0Pv8A`JN?$Tn{KztZ}r^Lxy^NF>YaOc4DL4E
zy?;0M-m!b~_wU{R^?>^k<71g8E>FdtoqzV@x$8^jSGBL^zSeka^lsa`$oIM*u6>aC
zc=N;L4}U-Oe02R-@j>ZB;)l5(^gpn?mv|TW?#;Vr@6NvKe}C)4g^v=Sc6>Db$n=r%
z<I4{ZKRo{6^0D=k`WK$B9bb2TOZhSJr`oTkU&nqa|K9RD?vKUaynn0zH#6K~SjHg4
z5c9wPU*TV#Kl6W8{#^Fs()S79UVU-={N<zcN9_+Y-nqO@f9wA);oY;h5$~qG_xf<)
zecZbrZ?3<tdbRk4`16g=ik?4vQTl4)>+m<uZ$G^~`&Qy@;~UF2{%_{Jx&21@?c2Ba
z-Wk7t`>yPr{5!dKq3@F3&3||P{j`rtpZ&gseGUIs^S$o}+fS~a>OUiXR{y;Iv+bAr
zZ^1u5{~Y}L<8S`oJAZ8cI{Z8O@A1Dg{}UM}F^MuaGi$SaV2NinVtv4Jgr$MyKC=$<
za;7?_gG}d{elp%=uwmHzf9Jo5zgK^k{7U{A{X^&b_pj%^?EcL9Ir)?F#|`f}-~WGi
z_uc$=i{5>GSNi_)`}glnKm7Wz|D(aDo=;~!UHSC?Q`+a{pNl_V`}F6d{>Q=(k?#fH
z`MmXh+x~XZ+l6o6zrFVE&U^C@Y9D5OX#B|e>A=T#9~eJqytjP!@6DChQm+rcTKDSg
ztGllbzq<U&>-FK+hh7W3e(-Ygi_quN&%~Z?c_Q%i*3+lYI9}+!^nYdWy6N?-*Z*Fx
ze>3~-=Xa|=WPB3%()hLboA>usKN5d2|5^7p<^L51C8m|k{;ZSOjM;nGf3Vwg9AQ7h
zR?PN|t)E?*Lz=UWYaTZ{&w1_??%!M`TrW5ebEa`^<8J2l<)0<cFIXnTA$&tfN~ldR
zL9k8mgrKg_J)!HuyrL7t_$8zypG%&Tnj!r|T34oCMo{*KY`NS@xkGaA<@U)5$#u#m
z%1X&Dl1Y}Skg1n(mw73@LHdOB1L@7ubEL0JtH>0|u*gcwO_P^Z%vb7DZc`Cby{+<4
zxlO4+@uk9Pg-Z%DimwzIl&>jwsfek@sD`OJs>Z7BQWaIRQVUY6QCp|>NR3-vUENIG
zUp+~^P`yX}v3jnCkmg>^ADS(iuQX<96luh0lxfV>_@%K{(^^YU>%3;2rlw|>#vAoX
z>h|h8)S}g5)k4$`sPd``sGe6%Qj1lyR?}DeuBxSaQrS<5PjR*U3E2i|KgrAD@5Mfd
z?h?5q>>+$m=!M{ZfgFBCzJokM+&Y|H?73{OSdXyAvL>?>GwU+{V{&53Wc<dkpJ4&R
zLWYA3@{Dg7w=-R4GGhv1lx8sh$M#3{SM87e-%`Ks_+tIV_cOz%nIBBvpL|>RX7=kX
zuVdbPeUtTe=iB*jzrH#2TIrR^i@<07Pj)`K`JnB-;QcrErrzVc*KqgPo%-85Z=SpU
z@2cyS=a(EWnP0NHq;sk3lK<r!mp@<TxT1L_`O1kaZdVsx{dv{on%%X!Ya-W^uLoaG
zzJB9+<c%FSL~owHsd0Pfoo{!4-7|QQ^)T`g_v6OL_a3)CseL;8nb?afFO^@vc)jsW
z*ITJ~O7Et<ef6f{O~4zwH<52nz1j1Y^*!5%iVp`rJpM5CgTn{o50W1wKb-rp`s3@5
z_dc?G3i`zVIp)jmuV&v3e=7d=`NQyc#@~y7TmQ2Eo%?6eAKkxq{}%qc^6%}xb^px%
zb^cxRXU^}4U+O;-er*4~^4p59X<y5~>U}-*<<=MPuNS}iep~tN*0-DAa=vwa_5E_~
zQ|!mf@9p0Gev|)Z&YP&W+3%{}$A5VJ;l+o>4?-XAz5n_??8BQ6T^}Vrt@>2)x$=wJ
zSL3fgzuf$C{L7{<E55LO<^5LjUHGTV?;C%w{8wYlXPUsw#qyDPKl6L$QkK;$DJ*s@
zu`Dbs0nFOWzRa(gwlIY;`?E}C&0x3V+`}ct!_OPUdyA)+`vd26jwtrutRGl<S+=o6
zu%2RdW&Oxpz_gcP>%Vz_mi{{VBmBGlw>@8&KJWUt^~0O@_ufBxKly#}`@Z)P?-}3k
zdN<|W<98MB|Gr=HA?@R)kI5hReqj5M^Pcg2)w`~@6W;uI9rAkdE3Q}XUsOH+@{H$s
z!Lx%;S)bi~w)naKiv=$_UL?NIezEHL=Vu|$o<E)Y)cEPOCoE4lKXrUI`Pt8BZqJjS
z&wYO5x!j9wFUnu8ei`uc>5IiL4!j6`$^NS6mHX=%Z>GIn_%`e9oVSnOdc1q{Zpr)Y
zA8I~@eR=vd`un;c=YM|r#qx*kuj9W-|IaeSFh#K#vCU(D%VEgHz}?R6!_&`e#s65~
zx8MaKArS}BJkc*AEh4cZB_b0=YDEr;h>Ar@{E^C&^^!lP5T&$MSwn?MWudaH@?0f<
zB_Sm>B^zZuRde+p8u41T+RQqCb(ZKZ)Q!_M)y>m2(sR&{HV8HRXXs^QVRX*W!?4!C
zUq4Xqpl*?_x2~A(Or6`>x3zw1F4qjxWY_$yDWY>!w@dGk-UK~CJzrfJof}%WHKjFs
zG_*B1HS{!kHDomRX@1k3tSP3MrQxO?q^749sOF;Pt~ObXS=~`RKs{0YyV^5V4%I@H
z1<G%gx|K|oDwW)oHz*%c{-XRsxk6b=Sxvb`d7E;G@=T@MiklVO<>$+F$vu*NBvU2h
zCUaDJskE}p7MVy{VL1^wM%g>kPSRCUZjvDqC&UxQZ;Q2x-W3iM4i(-ev_h~z;5XkK
zUOAptE?v$e?1F4tSQ40LGx{+kFeo!_WIW6m!+4RwogwW1-M=k=j{Tndd-?Chzjyw=
z^n26qMZc|o@B4Y>yTG?6pXYw$djItG{Fe`&Z+=?$<k+L!`>pqVA8S9Ye!l1J^e-O2
zJOAta`T1?)_x(Sme;@mE<JZh@LSMAs|9!#y;_matC+?3QJkr0XfAi$ESC`W-<erT?
z_4?S3L)rTz_cZNh-+gk|+FfzGB6b<=nz2)7C&SLaJLc@1zH9F8w|fltvF&f!FMeS8
zLFprM$KsAJJK=I__34N+v1j|vn_swnw(?Box#$bGE}XrTdwtKH%!e<Y-g))vea;u@
zANjwR{yWDog|U#yg{6kggmW5?KmTNbVu4cr82)$xA%Q)7zxWRGhx0S@>+nzETgEHL
ztH<+&tBdO>S1Q*;&KS<WoE_Y%d`AW3goDNANKTS=mHjT4DjzMsMt-W?QfXfaCecTN
z@%*cKR&qRHPWi9&JO6vY*ZE(}zOZ~b@`d@E@3*^OmwmJU;rZj;_ZvTJ{^b8P`1|9}
zzQ3yfU;hjEyY)}cpI^V9{5Jo+|7Xsx-ajw@O#RdHSK|M7hGM2nrcI1JjI2yYnH-sf
z820_U`tK5GttO)>(=X;Ob{DR9+!B1#_)qdb<=@S($^VzHiEk}WE~h)|Q^xoIx&C_m
z`2Bg;2gSGCua>?rd_MU}=OcwjrytIL5PLuVUc%jzcTU_XyqkYt<5BzLGmoD=dHi(S
zlln*VAAWv#_aVo__y?EnSKT{%$N$c^J9F+Hx!ZMj*B#5-Q*LtK{B}#{{?*4qFV4Kt
z{50YFtUt>cCNO_voy-1>y@tJtjfr(3b12hC2ABVP{z&{%_?h{$_?O^cVTPZKXPDJl
z3t6I>lbC-q?O=Suu$V!Z(U?hzS(4=@t26sjb{mc#9B(;xaa41hW0PTd!)VB$^RMo=
z$q$dO8ee{Vx&5vBhwV?EUjn~%e)s;W{%QH6^ykGt>;CTi<M)^CZ}6Xa|DBjV{h$1Q
z9+M<93(G`~5YA}UUe>3a&pE?cU0IYlk~vk`^*C+$dU<Dav~V==$BDIw-VqcRQ<Vvo
zWS6+Ez^#^~<fojarK{_wy+(VT{%K<gQx=n_h8g<qdb@ScXsy&p)1Ih%Pivy)W?cuv
zS^D2}YIQ~RGITg}Jq^zp2O0AiUo;LiU14_FB*>svCtTx);!kN_$ySNi;%h|Bg?9@n
z3%T*#<K*Mg<-N<hoQILOns+yk6YnLyoqT_IzHz3r-C<qI;m>8u^_tt2KUJ_xaGs#1
zu$t&p(Hs$JVRNAqf}e$g#h4|xNxTu?C&3`KSK3FeTHacoOJTO6l#-N^o^p(;wYr!F
zw`QwWnD#~OeL5Vve7a(KZhCWdB6ViyF4X>^wnl|l<*DLYg(4*um1vc8H9yTenu?l2
znqSo;RMnNwDV&$hl|3ncSAktoR{5jKQssXNhvhy>TS=S~aTe+on9Og%SHZoSeKqrL
zh8h3o{oDA5?N9n2=HG%p>wf(Hw&Ans$B7^QeOUkT`NztS3qPED|LDE=hwbmA-zdB+
zf7<uZ?w;*!hMRwHEV-3@yWv*!P2C%xt_faWd!z6c-yOBP>+U4q-hOM|t!+0SUVC+U
z+T{gTn6G$W`gPIrip2Gz>k`+FUv0Qra;5o7>eZ5~E3a^0R=HSx{_UB&r~J>bo)0-c
z<DBogIcJ5=JUNkcWXi$k2doe99!NVl|M2!>(We5>-a2o4@y_|Ov$<zv&fGpd;Y`GN
z`zy^i1aG~)#d}BV&eS^^_vYST`_Sfz+*5{UVJ{cHNqXD*CjYheYlk<J-Yxr>`9<ve
z>R;CX-ZQ*s;$&XPxRY@+lOoeo20sQ~21y1@hE9e<j2oGbF&$tkXIjGahuMPV7n2aP
z3(FFg4=guXf3ekYeB>15R^zeZnZl*Qk;QtAiI?%=|H}WT{@eX``d9t0=AX&m@;_aF
z{QgM%dGOo$x7IJgA4k3>e_8w~>qGqe&UdwMufL6bSNzuL?a_CvAF4jw{ZRMu<fo)B
zDqn-YuKC*db@G>gpXI+KemVc;#Mg=6S$`${w)=hL_sKsof0X{X{^k02;a?_01>+2+
zpDgp(Gq}=tjrm*o_4zIM#RVSmr}8b}Gvepr{lGPwi<|Q`n-se$hapERM>*$W4r|s&
zj1~<3|7?EG`yu+>=Uec%l5gDKCVjj5L+;mvA9CN`eO~e*<BivgxleXHJaaGa_TwA-
zuNGVkyP$bV=Ti2?;}@S?{Cvs#%CAeGFE(BNd4>0y!}Y+Mo%eP;Hh!x3{NPLemrgJG
zUhI78_K5e9_~V=>E1vFrI^)^5=P#bAKaF}S@J#R7q~{yo?EUQYz2!UaPoZBS-<!XD
z{-W~z-_Ip~*8Hvd=gjzu<p8@emjsU+&uXr<9P2plb6w_c<(|UJAi&OlmA8feq@bHn
zu0Rr>F8^8qPJ!wCN&Nb}K3ref9oVmMdhjgcZsq>SGm$rkx1MJWcMH!g?mryKoUgeJ
zx!-dx;nw3@FHj_`EE*+tSz@|$h0Idvzfx{e%@SwDk4wtQw8~zQOObDt3zyv_`%vMR
zYMEAxuC;!t{ulknhO3O<m}r}=GxatOHM(#7*mRZoWJ_D?3pQ%@iH@rrr#q~**SC9V
z-D|bgve2T{Lc;2Y)eLJ%+mE)=w(qR6%`=Qk3{3Uy^sDvX>6PeqXeVmT)QZz))a}yu
zG?=A#UFU~Zj`{}W9g15O)Z{<Oa7oXT)RXWM_Z5F5#w2Plw4C3Cx1IYZmpa#P_8Y8L
z%pd<<`tAIC(l4i9hJW7wHU3xg&-<U^-`GF?e~SK8{NeiJ_<PH*l;0_TU;H!t&-tI>
zpWYwe-wnT+{>1-D`}6nDyT1&7Fa5gmljB#<ukXJW{yy<HjUk$GI^#LU+YDy^*ZwX2
z<M1c^kK3P5e|Y~*``^N($C}8dz}~@@!P3X1&v56@u3u+=e*D4lllj;AUxvTke!KsU
z|MmR)l&>aVu6+9Z@$JX`AMbti`1Il9=Z}$}7kv5ih3o6YFO{FwKh=J$|J3}&;j7&j
zsn4fA%DzAQ`s)k5XG<Rcf5`FB_hHdv?Pq*1%3f@LcH!}Yhh`64@0;B}bbsB0*AMj{
zvpjwJ{Oc>5x32H(-s!%pd$;SI$-9GZ7QeQ6)A`ono$h;!4`v@0z2E)5{zKhI?@x@M
z-hEj8LGvTe$Lx<sKEM9@<E!L1x9>N;*?d|1ar%3!x07Fgf3^IT_-nB@<!`RP>3UoI
zHs)>n+x@S7UY5TUeEsot(wovZ%x@;VPJcb~_4e1-Uafd}`Nh-cQqMm=b9jFJ`Ge=3
z&lz8=c(MD%wih8Ul%6kodi!zEqu_@N?^oVUxFdEa=T6Jrx%W)&|GHOmx9s-Bn_4%%
zUr)NhdUNzlm(ep_M$dHlG<v4X=$S5~XSxicGhHN~tf0x6F6^Ucx{RLbGJ2-V=$S4*
zN6&N_J<~;g^h}r0GhGOu=`uR<HahY)I`TF;@<!E>H<8;n?utGTdZhID^rMPLx{qc)
zRC#Ff;K==74@w@cesJLdA$ggH2ObnW)Ol?Fl>LRo>*BZR?|!|V@^;$Wk8dpA1io4S
zhWqV;x8?7u-Y@yE=~LU+&hIHdQhr?izW@8m?-##K`}Xqdrq3y#wZ0tvc;TJ-yXo)F
zeOUN;-sh8_ZNGi`ru1#rx9IQ1KWu&<{>RT`&GMV&9P2VRBX&{Fsod(k_56`Srb4&)
zU+^*WJMit~eZsqjC!Om#X8?B$?*smJK^?)_{1UvsIS;TeX8X;0khPpOoh6A$pD~zu
zJDVc=Y}VB*Y%CL*=P^kzu46P|_F;O@_?_98MU#0Eb1(}x%QEJFOesuP8Rsx){xAGH
z`*-#4-@o_$=KIIT+{bFedXe=o+eTJ5=2%8whLryo{{{WeWBA3ef<c6#<p0EfKK~N`
z7c(d^`Y;JFo%uiSU-CcgzwiDW{X6rI!7t7qAzxL$7=3&5W6AGN|E4i~```4x{=e)0
z=Kn|jxBWL~n8|pH@fu?@vliP14j=BDyw~`p1g8s55}YF7&v%7~fp;g57td1OaL|bg
z0=oom@|W`^@Sfnw<oUsE!Sk0VoY#SmQQ)CKwV;;pWDy-v3$eB0t0goguStHDa+2O6
zwMS~c)KbaAl3%6PNbi-IDrc-vr}#~=QE{50rc$0#t@32mnd&{7`?Y+v)@V-ET&0n&
zZlm6&o~o{?&aO65xkK@yf|){ve4_jm`A+!``K|J2<%JZO6^|(%QdlA<EZZf0TGCna
zkR+?r3CZ6QWfDA+lO;Dv&XQu2*(keDzFDDP-d8S5=Dwt@#1!#FaVZH=$xz8B5~szj
z#8!w%3a=5Y5{Tok;XBC7$D7Ne%)OUGgKY-$6^0N09{p2bIK_CFc?W9_+Z5JmEIrJz
zjN$)J{VV@h^N*cj6C*p5F4J13`Am&WQOtZShnSBs=dskV__3^J*~a3|qRJw}qQtU-
zrJq%o&75sATLb$mwsR~=%sEWujG|1r%sMQ6tXJ4Bu{~qk!dAfgj^!(h0gEro8rD3v
zk8D5L%Q>#IuV(we`iu1;Yb&cg>kgJk7B`lD=HE>D%vQ{Q7&RCR89e_#{P*df+P^D*
zCjOrN)AoD+*Vmtqe~kN}`Tp9Q@>i>0L_YI>^5_xs1MWN7H}_uOb5-R^@MXnIi_V*z
znRUYK=!rvX557APabW#{9S1cJdmhy|&U7O4M9GO;$61cI9$S7)^|<|U;o}pJ?>h0|
zWWlNWQyHfePKBMaIqi5_<J8KND^4n(Y&-GiIMazKC$F8ZI{W6F#Rd0^A(wVvI(y0g
z@@U>1fq7HlvcshVm!4eSe5LZ*r5jnduH9C;D|T<o{jU!tpX5FBd~x?h)yu$F_g}t!
zvHpeEi?`3Gyx8&5|Mk?j>hJ%)U;DA*bK{r&UoU_2{2uTl_Git{tv~ntO#6}Wjrohf
z$6fDy-&()Xcys&Bk~itE#a^+ztb8%y#f=wnFDJcXd&Bznz}uPcUcT#pH{spYcU#}h
zeW&m~;e+a@^3Mrh)W50zVEsAiXWlQ9-;@5V`e(~v$#{hEE`!4V-hU4NfBjEoIL5%w
zSji~D^n%Hp<sNGU`vwkWE*b6_+)X^|c%%8&^2PD55D*lS7XB){SY*4%J&~=VvSM|j
ze4;6$D@CP5)kSs*Gl*;!IW2lz%u9TYIKPCqM508PM3=-{iDrrI;^E?Z#6-oC#U6{f
ziKk1Plzbp%Eu$>!B%3IENM?dmzqqtWw16A$W3H<lsvOCj$=v$9Gx=)y!};y_HuAjU
z3gC3)@L-Q-yT$6vI)_DzwVBP7-HqLcJ&FA_`)&?V&fgr?oN`<|+@0J{xTo`U@NVLJ
z&A(ZIQP5M+Q?N?Fh@YSDG>-~*Gp7p2Pc{cO4z_8m^(^MhhnaerMOaFi%b9nx)UYP7
zYO$VTnZUA^#g3(eS&{i7lL&Jf%VSm<_Gpe`P8lvKu3*kY4rUH3j+q?aI9fPex$3xt
zxb|?CasK95$f3c($X>?!mpO<@ok8}W!e6Vu%l>Np`}Obne|Ltg|NH;y{K@;}@#Flr
zzHiUIE&P7(N8_*LKX?DC|GWEF<lpCiPySD4@MLsjQeZA%c4ro2)?$ulE@M8-e4b?i
z+guI@uBqIsdC~^1$mjje8_dVW$2yHAw}+Sc{g?jF=HK@IzR9qj`2h10#`*u={L=q+
z;$z19)_1jU<6f)2V0^mt@y*AV9$$Xs{_xR#|N9>IlkXk9!*J)y?b2I6uD`!3bXELn
z!&T{PUf18;5V;w1ZP6w93z6q0o(Vfscy9Iu#f$8h6t70#@W0t{bKmXK`|}>9K4E^k
z`suRgJ6^`WlzHC$l<BF(6Yt0CA0K*_`DXq{xzDdYiheZwQ1{mS)$<q2UOs$f^7`vb
zxfkKjA3eYS;?c{quNJ-j{ASNPo)2?BWPkkjsrHN1*O^~_f4=@j|GW3kAHTByZup<|
zKlJa{-%`Ije^&p9|9R}!yI=QzR{og!ed>3!?+d<~e%<(;>sR$phaa3jSNz)eOY=w1
zXVdrHZzsJ~eB1d}>SNirzdu*}RR4D4<BE5NZ?az2Jb(1u{^i#fm!3U)!ueF{`SF(*
zUax#}?zPX$&S&CJ3!e5oPkOEO;l}5mU*>+X_`2e|`md$G&i&Z_b^oUuAL2jE`)K=x
z|9kz<iND!@pZKc%Ug*uUS4uDRo}GJP`flr|;IBKsZT{i?r~9AIpTpm0evSJo@%7P{
zd0#jG`1Ef*i!Uo56Yu|(e{TIe_r2}=lpi6#q<>reeEqfi^Q=$PKVAIv`QwK7uiu<`
z+5POm6R)Sy&y8N~cysH`^;gqhYQ8%6%Hh@Y7uL_7J)ZLD`J?P7HqU0g`1D%uo#~su
z&!QjgxUYQA{jU36_4|fTa$f1Z%lk0xi}FvMU!C8ZJ{N!B`Y`E3>U*B|8K1j<cK*5e
zN8nG!PsguvpJYF}eLC|c>f4d8KR(-k{QTDE&BHf2Z#TSAd|m&-{3++-h)0r-U7s<(
zE`7)R;qQluAMbrU^=abgq))8xd0yAN*zv;u_2zd|J~4e4{w4EM_sgz#Z(g&!p7-+h
zi*K)4KSh3L|K0honyHcP21h%43CnZlrL42rEZFyOZsbiAEEfJQv_?RL?**3~J15&>
z4tbt$ya9Z5ypy;)Im<a%IZtq{<KD*A#>vI`ls%u#p8XP6E`Nt`x|oMVl++!Wk8+b0
zJ}6F8G?A~Ay(H(TbX+xCbCNEn0gv8)jmIi;l~@%#Wd&u66q;4*HM6w!bo~rw7-bsr
z=w)c9Yn{?MuRUGIQcuT7$!xXR36nhI0wYU<>AG8Vrs=&lJZZehXr_Loj=0ux%{5xm
zI<IsO>z~(OsB5bIQ&V0`R;Nk7)o6yv5i=3XE~~><Mpj=f;?1~>Qgk0`sc7r!Y8kj1
z-!*w|ywpHSr${|i`IG!nxoE}vs#2OF+Qz!f`bqlxb#1lS)EB8OQ8!SZqqbecTIaUz
zQ=NLPUFz#pZYl7}o|WDzYpk$BL0V2(a-pz4{~zug&dD6_IodfI+47kM8J7HI`fK!8
z`>){NCx1%+1pYbqr|Iw0zxIFU|El=5`_q&U>>n<E;Q1W%?bHv(9}Qo8Kc>HretZ2j
z=i9R%YQOIJx$Doqe=itPnRFOM{xSb~_507C=zoj<Z)9A-+QRvlGl-p?Wi!L=KXZOo
z{aEw!>F?Kn;(l}eQ2x^J;nUkwZzA9LzUhBs_2%iTke3WE{ypP;?)oD7W#h{=FHgOi
z^hW93@^>@dK7M8K{NH1@M=u_-K8|>{>s8d-A8$6l+VDL0$?pe~?sD95xYz#3;OWVy
zOP)=9!T5^hW!cljN1Gp|KKc6i%OlyR6J931x&E5zW$4q?hlY0#-F$!J&aF##{T^68
zY`xEX=kbkC*QQ*3ccta3@b#%TuHRU5{p7VB*U#SEep~0R_q|E?TOaO!ocpx++4*OF
z&k~=`d&2(Y<l~Ab#m`D#zIgrqwcM-f7e3FvJe+;^(4FM_9FONe)qD2g>Ahz=pFe+k
z?$NP_rypH=%=JX>3D2{6uWH{3zuWRE@wxPqrU%ORT<?8+Q1~?LrRb}N&rd#4ePsGz
z`@J1^3-0kg%zk|JvF76$k9ItgdgAo#`g5(9-LJ2{ll{c~_02cd@5jD!efjsX>wWNB
z^|!Jga=(~-@BOj%=k}kQzyJNx^f~*B%#XW&BpFUImN3h)zGi*K(#Tl%ujl_Yro+tV
zm^ZQu@f{TWAP~nJ#c9d5h}n&C5@QhCZ5}znt%6ScCfp3{%*?v~r~di-``n)ge?R|K
z__ONg!5_9icmDkG)ArZxUkm;u{Lf+3WeQ|k!?^iB=ifuWx_*lP>iF~I{|ctV%*o7A
zOdbp;{;>a=_3P)~H;lVkCbL$sHn3%|o3U<Slw$N`mS<&W%V4kN3gEvlbXagD?_4fB
zjtG`2hU)({jE7jhv+1#&V3PQ+`KRdTvu_?>6Td$FX7+>W$I5SlUlqPQ{j%&^^!I@8
zJAOF+=KAaPkB1?Wk%MvH|Kfidf1muG^E=^h2Lm6o1d9vHTIN5D=l`$%$H{QU{dC%?
zjc3v>Y`WBc#pQa{EqfbwA$Bjeb1Z+Dau~fCPX0^(bN*+?ceSq*Kc#$l{EqqE(YO6?
zx4yamn&Zu#H|cM?-mZOn{%z;mEpKMNPI~44a@&iwFS1^gKW}?x`i$k7*Hhjn2Oq6?
zc>ICfgLC)q+?RZi@nF$|+6Nczf4*0KFZ7<-y|;IJ?iSpgez)xI`8(!!{@uQPyY=?X
zTUT$I+^o5I@8;5*2XFG+o_#0mZpdA^yApS!?l#>GygTLYqkAhJs6Q%teDz7|v(6Xs
zuO_^<dvpFx;M>`6SHE5NR^pxbd$SK8Kj?gn`1t1I-cJsnJwB&@X8R)bweZ`~@ArNj
z`MLF1$M0pom49#iwfh(IZ@J&1zb$?j|6cU_$nVR)=l_2GJLoUZ|NsAgG2CbR!V<&g
z#QuT3pF^6HpVNt>h~1D~l>GwRR<^Hfz3hb?N}PI}lAIekZ*YC#KE{*8dxZBH?-E`W
z-q}1(JXSnKJaIfTxDRnvag}qKa_!}e;<Vtj;C#sOnf(yk2G)%%vzgnO`Wde=1TftE
zANT*!zo38L{&M~k{I~b-*1rk=9RA<@AH=YfL7&l*iG$gic?R<qmdmUWY>{koY~NWG
z*(}+Pvf8p*vxc#LVM$>LWD#M_VmraUhC`fFfO89n6~{F8oovo*GuVEy?PlX=D`(YZ
zwPuxN?PU>T;b1w%{GaI^;}-^JhCBb4{rCFc|8MHw^MAbl%>BLNm&mUZKRbRp|9t(!
z^~aj;E55J!&hb6x+mvrFzn%Y9_092{@3$@APJiS5CiJ!a%j3_FK5>0I{*mER>8H(~
z_I+~syyQ#rH|Zafe)jyb`TgN{)}IxBRR8+?`}hAC<2R--=AFzPECH;CSe4kC*)Fk(
zu^X~KU~6QX&i0P&F`Err2<t?aXUyu%^-RARYZ&zyjTtvG{QIZ<SNsp(?<GHLe*F3V
z?Yq{GJKv{&OZuwzRp#r&FAqKke}468#V4Ont3JkmeEo6NC#KJepSwO6eL413^}E9l
z=AX$wr~Ews^Ucr0KR^9k^-Jsbo?mZ%HvK&Kv-em3@2Ee#f4Toj{lEG@pTUALl4&dR
zEtYiFDXcqL-?Eysm9nj3b7PO^Fy!pvJi+;a(~e7v+m`1M&lTQ${s{sz1%d?X1%w2j
z2_6s%5k4ktA)+agE<8=hPbgJrwa{gu$3mXMF(SJ}&x>V?*NEqf$BW+<J0N;mBwHj?
zBwd70#94Tw&{?5xLbHUVggy&?6Wl4tF8GE21)nOPE$?FP7S3$;gREIBYRpVb9*iFu
zv>Bf<YBTjP@-o^nx-x!Xc+C*Yc$rayNs_6bF@;f|$%Xki^BLyN%oCY2ng23rFfC;K
z%$UUF!Ssi5H)A_v8KW^{J;U1nlK)%(z5i?ZxAKqapGAMJ|MC6P`dje#gJ0Kvz4*oT
z+xGXu-*SKK{=E3z^4tFR?q8RFF8fjbJ@MQ8ueM*`e3|)W;g{E6=6+f5dB&%GAMHO*
z{K)y~$;Y!FjXxQFYWt}DvFPKwk2#;ZKk<I5`<VAJ<0Jn^mX8}guK6_iGtZZQpS8cJ
zf35nq>HD)E?|*jvI`Av#m(S10AJ4yU``-3l=6m%wi*HN5*?e#MKI8k%@2)>4{@DA2
z@8`asyMJx_ZS;rz&xGIZzwLkX{4V|V>gSrDZ9k)auJ|GRJ?CrAmzd8_KJNN({{5VH
z9&dNPKKU~HMf{81FVbHueSYg1$1{bex=-vMAAEG-(Yi;UADTRT|G?#8*rVHzKRn5O
zy6Wk=r(sVoJ!yaP_Q~_7GoE!lb9*NL?Ap_^r{Yf!Jem9C^OMS_-A|35K7TUl$)CqR
zA5D8G`{4P#2X~*`k-O7(JMi|>+xmBs?lj&JxcmLC@O_mBA`g=unLOG5blS7n=d)gX
zeEI8D-RnuOJ6`X6&GyFr&9OJk@0Pr0`^f(3)+dH9Z@;?y(EU~WJNnPPKXQL1{$BpW
z{8#p0?0*G@Vg_G^$Nx9{ulRrcU-rMf|E&Kz|3Cik*5BoS3V-YW+WaH!`|fYEzdipp
z>AUL>#-BRBw*G$hr}ppbzw7=z{cp*b&UBJ_7Hd5Fc8>j=cexUHT=}E~@&)6B;)Q33
zoEF_J<|E!JUM~JkY=fwYh^z2VAzxuNk)t9aqEkef#lDFeis^_?mSC28B6UbANUBy!
zO8SkoyzF*4FNIXar;2A4-4vH77$_W(*OjlATOeC46DI8-B`>KZF-zP_{IJ+{(VrrF
zMSh7yik=Wv5pxhz7ds^SN5n_utMFFg8etXT9HHrgT!Iw>Q~B@l&EkvX`_FrXcRQ~e
zACo|+ke*1ssDoIySc#an*jCXUq6ft8iT@BU6c-WyBKA^@S3F02o%j`T5eX*=6$xF5
z2@?Ayrc1a;w1_u~of9n;6%#!ua!Vvkbe5>P=q=%wf?xPk`4;lHad~hEv43FgW07H=
z#MsP`${^0b^k41Yhre_Fa{ul4Bl+jiZ=pZNe_sE-^t<`b#lL?47c-<Yb~A2ZT*>&3
zA&B8P!vw}1j0KEm7>@pD{r}~k%>VfRiVPl%MojaV<}&#+Jz&gbT*Kh<f9l_`KfS*n
z{)+s?`D^{p!k-;K-F}MvIPp#Y+taU?zjl5*`2Eq3yFbH!vHW`dv+UQEU#EVZ__gm>
z<1epYyuZ4Bs{ge4S@hHN=k6c%KMwww__O-g$6va?Yk$Z8ZvWN%bJ~y7-`l?@exLcR
z=4;oN_Ro_(9sl_J1Jj4#_pR?*-}bzj`g-cCl`oII`1(BNIrnqP=ZBvkc(L~7v{zeS
zH@wMz^ZoVn*8y+dz1i|M_uc<@+umD$Q26lcef@{!A6-5xeck-+=l8guvwy|?milA<
zH|1Z_e`SW*438O#7&Dk=F`s2w#VW}b#TLZ&jP(`k6E-G}!yMt9GdLG>GH^w5sd4dg
z&El%%w&%IZ<IQ`X*PqXoFOOH3=NH#0&b1so984Sn9F80ZIe0kNa@=K?VZX?Bm2Dea
z1KR|)No*o)2CQr>49xvZ9~u1_%Ku6HUH6;$SL%<K-*$af{aW_r#it)1YTnOyXZX(e
zoy|M-cc8Vy*>CmVM7)-L9sauJ_3>9NFBiS|_WbQL<!6%5BA-ou_WYUR^H0yFK9hVV
z_jKc9k4I7u@7?#m*L&yYt+bmaH-28TyUKd`=>^sE%g-J@({je~%)is7XQa-2J@xbC
z-IHfet~j~yr1D9t6St3_Jf3?z={Wm|MJJY>Xg{&;1k1@yC!J1DIeX>&l#6VarLWAs
zGV98jE0?c^U3a-L_eRvs@LRWTIo?jVbN=p`d&}?dcu@Rs-$TwvYL9v!&Umo<{@HsM
z?{eIoc*pCG`W=fqTkh1{{eE}nU8}pLcgyaT-M4zs`f$qQB~RU+pMA0RmEl|I4`)Bw
ze?9;G_fLc0KYla(mHp@P-=3kDVGF}nMt9~_EFo+*9NnBnTvpu5+^$@rT&&#wJh{Bm
zd`*0U{6+kc{8{|3`6~oc1fL6;ifD>Dh-ry;h%XbLDV`xdU))dPpTs7~Ia2qe7s@Ki
zhsn#y-<5x;;IH^Y(OkJpg-xwY-BRPD#w4w5oo%}N^;YYzHDEPjF;+0HHQs2FY5Ly8
z&m_-em&s(43KM73Vl!RyhvwHUm@Q*1R$5eAh1&So-n7-Sdu?NFxzCi-?3l$<%axX!
ztbSQ1S!Y`Insb>~Sv;~_W1(Z3X;7}^s=8HqhU#|>Dcu8x?@iB|>zVl)PcxD?%G7mH
zt(3bfEiHFZ*+cW34y!?-(HsK?O&vK0(Q?6d{<mC**>pH2^IYN!<$owJMdY*OUD=be
z=Vd0#eNy<KU@dPZt0=QXCR#>DvQ@NJBtvwo_<N}hGACqwWSM0*ONxrch(?IXi+>aI
z7Fo;}!7+*X48yGdssC3qu(AARvt--JRQLbYpAA24zRY@E^|<5V?<dP&XuR3=G5WjF
z&)Z*R-_<_dci-~Pj2jZy+^?D65V?Kq?wosB_bndhK7aFE?OE~Dl4pJ|&b$zNrvLcG
z<KIu6o-TQ)c6Y(8n|B%?&VQ`(B;(nO=ZQ~w9$vqzc_;ABi@WRZ$=pr4DRXt-rAL<@
zUzWedcl*_Yq9^;G9Dk(uF!+(g<HPq_Z#=xFck|3`xd(Tj_kLje9`y6}w}y|;UpqWc
ze<Jj_{NdvV6;B?%Onwvmw&8=zSA(CQeu?}F{u%SfgJI79E5BWTZT+49SMcxZpQpas
ze_j3E{ny1mhyFij3}8CI5dAmo*Q1{wf1UZA`N#3k>0drSPyFEfdH&bIe@~fPIo5FP
z=6uG%&#l56!QIV%l*NEqow<i~HRoYII}ve-a}w!dKLv8Rir7A}d|}dMlwwL}S<SkJ
zRh0EB>wivezAd~)+y>lxcv5+OaeA@mu;#MpGoNSp@%QZSo4+i7JN&8s`}Kc3a~;bH
z#-_iVzbk(A{;K`O_v_=2$6vKR?t1&;4bNMacN`xrzC8PK^>ghfi;tQgMBatH?0i!D
zc+wO0r#qgpysm%0>ZAULqF2`*wckm+zW&n5^ViSGotM41{_@N#T~~E(#@r3N|M7nA
z1K)>|kG-GVe)Rl7(!-8NYaV9bkG|`3XVIO2yRCNx?pr?=cz*Yp_2d2bCGMTL^Y)h2
zEymk3ZYkZ^a5e4n+za*RTP{joVY#~RO2=j2ORF#GUp{f+*_nT*-k;(-{r2?nb8{~L
zyit5-!<~X#h1b?xOgQ%lvjcUH?_u+UyZ7y$<h){hll|)EE0?#o->JV#ef{%A!^?%Q
zKfh^yr~7Hox4S=se-;1u^40na|L4h{GC#ln!uc)jTg%r?pQS&)`*P;H*RNl{oBv$-
zllU)<!IF`i@gl=ahCK`}Ob1wbxcvl;#7|03kQ0`FCo3jXBe_{DR&=kZzBr?lgsih%
zu6&R3V)gs#uIk4$9dtYNl=bH7a_PO%FE*TRkf8Tj=ef=q-Aj5u^u>&;ExuYCTCFkF
zHF~YTPxqkq9j$HJ2D%NpJ9V6OYIQE^?lv$oW-|U@c))1CiHg~7vt?$|W+G-M%u>u#
zEtIU2?Q@-HyEeJ9xO{Q!wr8?6vEnfoG~H*)ZC+qjVLHQXwZ&@72bM}!cGiz<yB&5q
zXxR7IeX?_~OS9HAUt%0=q;0goFxzOq*<_nN_PqAFHkU1TnD?3n8vi$ZWcb3+)NqPE
zkM1EYPpvyzAzHZ_b!sN61u8wta};mMwaKiI5|DT*(kkRB*dxFuaE8B?-=8m+JC8k+
zS%~4o-|c@N{G0#(*1xd7r~gd-`|<CsKUe<bGw85#vKO-%vOQueV_(S@%iQsA=C2Fi
zM@MZ(M{P$(ZAV9KM@MZ(M{S47sBJil1Lqv>XWZ*~_Vdc}uI9SVK9kLn;}91kw=H)s
z|5VZ4;s-_hgkJJVb2o4lu$ypj@O%|C6N?v15N_ZP<~hVEz|O`h$hw2=HTydDHEgnM
z8El)`0@*y-irD=)*Klv;>kw2DmKV7pQYLap=#zk|K%>A-L0OUeVvLeak`5A+L_LMn
z1r`eY6Iv-cPdr!RvV^Bpg|wTbzNogyM6ouBbjiiix8-wG{;M5U{j9i3&O+v;M6%ds
zaZ8!23T4V^%B+e~a>u2oO6G|#7rQ4eC{-n`FTGLHUZPJtT>Q4!dr?`jyAoezm=xrd
z)~L)?TcaAF#4rC-W`%T-w5F_tLXOf_WnYz@%1ag1<vV1HWrO71l=M}usGL(ht@cIL
zLpea9LhhhkyTS!Udu2oQSK2JPd0KJm!754$_oVYAizRPMTgdNM_$ntZ{XmRCxR-xE
zPa9VUX9(wCjuQ6$ET&AFj87OY{g?U|@|Wvh^8Yyujf}S$O&MMPm;aghbK-ZQ@2Nio
ze;fRp$sobl{9pWU_phe!Vqdeq^8c9m>)-Fme>VKt{zvmq`%l(y9A77XtNZ@pd)cqm
z|4uSmF&+7@`Mc%Yr%#*SbG^OsCjNcsr|8eCKkB@zdFA=s^GW%`()-WvO}lUQp!9zA
z-DS5Q-+Fkf@|NpOhMSJJ58b=<pyQ$0Bdtdl?{B&N=6c4}{g?eO%UzAX(QxbOt@pPA
zZ>_nWe<l3#>MPvWU2l5ca=119*0)<RH$AV}UU_hN<`w>HYpyT4>3O%{VZ@`1`-g8c
z-1NG3{xaj`y;pADIC1yogEtQt?!UTIbzAq=)*Ea$Y;V50opvwae#E_Rcl7Tz+`D?O
z|K9Wa%O72PqWSF1^K&nyU!Q%Y^)leO#xvz-JDyK{W%u^hyQcTY-$lF;ekuGc{K>;7
zI?o+m{&?l~Hvaw9_m|(jd~@ZM_p6xKYHyt18h+&ZZuNWnAJsowe|mh2{+#%6<A>x=
zS>HbVdh*xwU)Y~5KV`pf{POx!;iuHk-Cr+#ulq6UyVv(gKQexu`~CLMzJKo-Ygsed
zudy3)d}iOwn#^?N|KxwC|A{mHWZA}XhwD7|1D;x5As$XH4sJ>QlS2DNg2jR*G^Hj<
zt&sRFsv`PB^nw_##2Km6a=FT>>H!+-RBtOP%5h5-h{=oQisei8$()p(DdQ@2Oq^Nt
zlOQj@F|PryC0_=gHSaO*A6$RARJg0SR&us*UFNyUw@83Z_^HS)kzYcu1lagPd3|^;
z@jT-DE8rzmCaf*8T*!xiD^DoTG~Q%>Zo!R0S4Aqsev0lB&Jf(hf0yqkUq8Q{ptxwF
z<S(g9lI`MbB0u;Zaptq-uzq2C$63PT&U>G`p5rVFD^vIX|9>X`-tb%H56d6T-)z4s
ze{TEv<7d#XX}_oZ?f4(e_>^fgvjwvu<Cnhyzm9)f|JnRg!RP32Pk!+JQvbF5r^}C;
zuaiFUe0uYF=hsu;4*yvByZx`hU%lT9KfioG@qOd>89&;7PyH{=tjqeC<rib!-@qSQ
zpU=Nfd(Zjt<mZ6z-oNsH+x@QorTlZ^_sVZ?zeaz%@%{b}?;lRz4Zh#_e(=YOpT57v
z{|5b^!nl+<h4lq%7Rz5oMuwb!&;Qi_>Hf#Tl)%EwdW`irTN^7Y)0zLX8Fn-6VG&|?
z<2ui)C!ix>%J+l2oQsR=Cf7k8C4ohvOQmJyr^p|d<Cgs|c|h#B2%o5_c&L<$>}NT3
z`6$_aQhOzO#kL4f6lxL46}OeNllm@|Ej?8_MOs>VqVyM;b~zD+Uy2#ZY|3X9805^Q
zwI%XJcZxKL@k;)aDwUZayGk}(=9%Pf@dM%<Qgt#(a>o?Bl*Ltgly=EW$)1s3Ed5ei
zT-IE{PDMz4ow~Bx0;Soq2gQE~F5(U3uH{naw&p#|C(ftEqruhAZpD(tc;|ogf9?OD
z|H}Sm__6uxvCqdpPW<rvL)oWIUzERx{9O2}^4HlPm%mznI{NO%>zS`kygvVq<73}P
z)laKFP5LnB&DB?OZw|g~eYfSk#HYDma=*^~Jo`iHyZE<)Z!f;(`>^lx^dF3Wng84T
zTl{<O_nc2DZ-2a8^g{4u#;cpJ_r0-vBk-#9xx`c6N9XRnzsvHV;Zfe>(#O*uZ+<-K
z@v6r&o)|xS{yg^O;aApgtlsW?Q}gQH^DR$J9v^&g;DPmH@29_??t2#X+~`@xW6uYs
z_eAbF-1mGi=VAAgJ<k`v@P6*_H0trYhc*vuAC^4c@%+V`B_CKnb$z_>{`{L4FSb10
z^JL@G<IkO6AAc|V+3Jhhm%PtqA3wj_@pi*ItB;pIr+?M|_VHWA_mb~r-{XEb{HptN
z=wA_I2g_Wx-E13J)mXkUZv8*)->HA03@?~?Sy|Y)**~)vvSl$>F+E{^!s5dEmTfEN
zeC{&tT+ZukKUoA>9x=19nsCVT<_paf`z&!(GD&i$c%0~D;b7rIBEsS+lCz~7Wsl1K
zlHMaJDd8fvR76U&Mm$AwpJbWjXNk??zeF`eKZur!KayA_<skb<e!XI~;$-<a*~8Kj
zQqv?VC8cGjDl{l5tHh}`tL#(clV2-)UDj3JLUFUQwR)SDvv!>3Wwj0!L8Ytm9`d@1
zhg2*yq_jS0hHKQS-d9kU-Yj-Y#7g9ZaE^$WSd-Xnk%NMd`Et2uv43MxV6kLTV~Jp1
z%NX(h+aLDde||dtn(_P9pUZz${y$-eVZ8Z&>)+46nSb5;-u`XRH_so;zbb#-`+4o>
zyPr?LmwmDPyy45ruQ^}me+&I7|7*o}tuJmLC%;pEWA}R98@CV2Ul;wH{paRi>AyFA
z?)y^sUh1{oi?7cfJzM>}?&Xr#j&J&2J$YXHl>PDM2a*r=JzV<u-Q(oP9FGkiy?bEy
zaQb7>XD^;dzUq0a_Tk-!W$*3ZTD(qqS^aF%<ATRap1ynb<k{-y*IrC`e(3S)2iNYO
zynpV#<HN=$EHB(%O?f5o^1!nVPyRfTd(`_l<N3eWj34%X3i<r~W8?d8ukSz4dOG{*
zj%P2QPkhz%&i3QHkFP$wc`x$r=<83flHRnvlm3wTQTUV9$4l?mzTfdd>C=bLD&JCn
z9Q+maXVGu9pRc}ge(n8~{~_-~@2A6GWq)4(z2)z<f2;p>{Av05_j~4#Uq9l0PWip(
ze<9O3MrMW!|5E<`{C(t4CxZ#wG@b&X`C?p>lO+4ZuL^GwVBq)U+sC&-AWqm-Oj`1m
zRFIU4#3xZFkt;$uLQjQR#Ab`F7VQz4Bs@(hMaV{2T|``Tmsq<*x}=5VFYy<mtRe@5
zk_A@?)ClQ`EtaU2ydae+GhKR;#CtJ8iB!ogl48<R<jyNDRBBgjk(ZD?E2SbiL*k(%
zqwI2pIm$Mwzg2c9Nz2cYx+6MM=%nCH!D7MFf|<h4gx3pJ@txtG#xa*gpQ(sx4YMOl
z5X%GRHl_m%`u}(St^dRD$N3Nc-}Zlp{_Fe~`D^-n%}>wo-@mAS{rX+$_oTnt46m3R
zS?ZY=Fn;?l{eSa6sefnw8vK9ESiro4$%fI9!TSHFzcc>E{AXhR&eq6j!*!5TfTNe?
z)Bl=3BEL(2N&E`>o$@b^QJ1-q<v&X;^G1fr|6c!{@h_C&1EVL?F2<J(tN*wC+x5?!
zL7izkiy=og_g|iaJc-=aoV9ERn13+dV4T40#^%ex&pC(l52rmxCCd%QZbk#fNB_AQ
zRx>rQdb3Hh8Zmn_I{lCM`~1({f5}YWSod=HbD40(apbYOFzxs+{%_vjx&O8?7%<;u
zt>e(<n#bkOCC1syzKxxYYbnoUz9fMX!8Za|`3v~x^V<n*6Wk{}P4t}DVX+$`a|NgI
z>GCY)v|``ER>mRBWx$oemCdcqeUd|sZ3fF6=ABIZOj1k+%wa6om^Uz5{TKZk^?T2+
zi@$gM3ueyd_|7fQyPU_5JA%`K{XVM#+jI5=E;HWQ0+)nuh<+2(6Aux+Cuqj+&8NfX
z$rsBvnRhj}IcFD}4NEBVcIHf$sjPe1pK(@lwR8G#sIcd;<}xo}TEIM;^)&l_PAl#j
zo~^vsdE<D#aZTa6#p%hhhJ6voPR=u&vp9O$wzC{%PGNq^9M6`(^^(_*KaOuFw+n|R
z%Y^?Me|P<?{PFoa?~jTf^M5S+;s0aRw<VuHzb|?t{>tYi`^(N3#xM3gSAS;wr1J5@
z$K_8-p1M94eEH+$@s|@`%z1Y6$%n@wkFPzt`}q2^u9uCka$dW?<$723CiRu=%jy@m
zpC5VN`6Bz(lsCL@gI~XWap~F3CsmJEKR)+#^UHH@=e?i#uHenLm+PMWePsK9_rBeI
zwFg-bGapMlD|&JArOK;UFGQX#do205_p#Iy{--R@1fL~6UGhZxN&1tTr~98VzIgER
z<?G@%VXqxt?SHxUg~;>I&tAVUe?9%p+_%%-dB5*`|L|SL+YN7A-^RV|eRJte$vfTm
zzuwM&eecEEXaAlSJp1_k?CXUei@q8Di2H8*HRaRaciOL=U+jF&^m5KC`!}!Ns=w#|
zaPULMhwJZ*-=@Fmd$Z}S-}@~eq(9|;UjCW!v&rX8U%J1o`!VbH?0=OEZyAgjJpVQQ
zuKY3atHbB2Php=GzleT$_~r69mG5j{qdup7&i(xN)9TL;zP|r)_4l7Y3V)n_&;OnD
zr}3`^!$+3CTzdpA3f~g87nT<OE4Y%MjnAHM7k`hSn{bS%m4vNyzU(}?6>=@IQ)S-D
z9F#Sadn)%qK2_nLyqSELT(ew+e3e4HqOX#o@*^cz#WMMDxdz#LGS6h$6&+Q4)I2n%
zXl>Qz)h^N8tiC|4U9Cs0U(HpcLOV(KrA~?VH_f{m*ERYy8MTk;Eiid%VPGj|zR;xJ
zkWKf1dayE|VuON<f~vwD1!tvR<;%(vO15&FCBKV?32zhpC-7L{qrf47*ZlkVLilp{
zcM67xG>d6Um`I9C){0At?hv{s@QnWtzm%Y$@OP1qqD#eQiJOVv6Hyh;72Yq*F3c@F
zStLWuLHwWSPNB2>bNO!avh&LD?H3dfdn)lrB2p|>SXoe+?<{8kn=Gq6YYQtATO|8r
z&L7-5ye&M<T-qGfY{G0i+19dWa-88<#j%=w9qU8pTg;PL>)7{j?BHzS3gXJ)C}n-d
zWW{*yzsdhPh6zj)m`zxwv&OS|u|_iQV?4*8!SL^2@84H{Ef~@m1sEj%P5LAI`}t4n
zpTR$O{WfN}&J@9D|L^PX)SuN~T|O4Qd;8Y<{nrm?KJEQ_@MqXx$^X0mZT=JcOaEuZ
z&)Q$zznA>^_V?SrOaC_fRsWmw?;C?G^DE{b%zv5xGQIkL{dfG2179zGKK5z-C;iXM
zKX-oC`~2zC#E*6F7~h1vPJI>nqU*WmOYJwI?_A$8ym5Hd{p#Q=omZT1rhPp1egE(K
zf3E(H`xW;y`TLj8tsg5tHh((*`PWy4pKt!`V+dy6&%BYTf$;~!3`TC|Vit4OOxC9?
zHq6}&BL6b~y8b)=PygS_znA|y|5g6|;)nB(Ge7KqO8lz(6Zro%10U0W=2fhBS@l@A
zv)p9KXWhuUhjk{q6Spld1CI#TLyi`9D>fyz@9b?n3xs})ZWCb<Vi35(vzon~>Ds?*
ze^mZ-{XX=|?6=I{`hP$E&iwu8`{mD<KN!9*dgt~o|J|W?=iV)QYx(-XOPSZd-g10M
z_{jUY{)_LI#82PeKYbhc#`N{gSM{&|zFGE8?E}vzy)Wm#w0utd#PTWZgU(z3*Oy)!
zyuJVS+#A7HSDuSKuX%dok<8<o=Qm#~zP<c9?&Yax&QF#<aJ_TkChM)Ww=Uf}a9j4?
z#D{iId!E-l7kt+7#OQIsqk=~}A6Y%-eDd{i^rPty93ON)T=Tf&Y0tCz=O<pgd%^kK
z{At)zw`WtH89nEI`S=y%oBFrq@6A3Wzb|-q_AT>U{x@8&Q(o1*e*8B4-H+FkUV6W1
zddBxe<8jv0Gp`<eSoL}Br>2h~A6wtAd%fYs*XQ$JEPWaOO6&Fdx1T=F`pWx#_qSbN
zpMRPBrR>X-FGs)4`T6a)*WX2dga0c2Q(%l@dBw`duFbiM>n-OWb|<z+ESs6tn2VTg
zSt41Vur22>=la9t#kGRtKD!aeUG~FldhDw?gLxkCy6}DE&E`uG&=NW=94=v{V5UA_
z>$P^Jj)2Zgtw0S|HA6KS4IQnU+OzaHjNL8%*zB;kbl739Xy0L%V#i=F<dEsO!D+q|
zpW_+(zxI5N{!Z(ixm=yx?zo+EWpWm9_-kunv)8KHD&0E8M%#9#U5LXx2N!!eyW_U9
zwtubTt@l|=+dj4PwV!DF!MewKnpLVrzIm<XB3l>7FOEm;n{3}%@3g2lX42oHqoS>?
zxmhhj#aMZRvagD{@*nxD()JQ7L?eY~3pNQR3RMc}3QF)Z^V;x8@#^z!=DWxLTJWUM
zW<gVdCVm@!ZvF)RU;$>q3jz@WPx$-z_;`{yL)kC0=CW*OVq-kXpv@%C?8H>akoj-g
zpW@%=eqH{R@wb&Jgsp(>2g_V$4#ws`UEgazpZE~|KIPq{x0Bx*zBl~D`t{@2bzhcz
zy7M9HeapL+caiVUf6)5G_$lDS$9HevrMy4*zUV{4$B<9{pFBTac|ZGI`CGp?i(fx|
zz3$EQw~yYv{=oV9(dU%UI-e(e-t{T+L&H1qcUJE_-qpNSe!c6(&F4YSwV%~J`}lI@
zJIzmdpW8lFeTaV7_j<;2u_rZ;zCWyZX!r2wL&L}6PiH*8@S^%f$#bvgK`+W*#=c^H
z)A6qI!}SlMA5z}Gcz^KY;m?s@^}aj(bpN&OXXlUd?|$EOzH)qJ_;%>qg>M_aZ~NK(
zOZjKc_pWalU!y<o{Gjyi<C}AD+1_)!U-GW|o%6dtZ}Z<>d@uWX-jAfecm8et_wt|l
zzk|P5{w(=v_Dkk>)*qvPQ4D6x&)NLAZg6*UzvQ~X^@{5ymp6A1&tYCBenEaFzF6Ks
zK2?E@f|rFh2rm)-EXd6NnEMNd58F1D{VY|i>}<2y>e$;kIys+kdUO8d2<7bLFlW2Y
z;=p>Cbv>&I>v?7drvD7g4AcMF|4(Js;W)&_&t1=H&*s4N<*)3|+Al62b>GXr^>{u1
zRr71dw+G&_yr2B`$E%tbY|rJMIXr#*B<AU<r&&)MpIm*s=P~;e$0sFE?mba{n)IaW
zQRjoW`ycPExp(E>{rhJh9Dd;Z(CP8~CyGxVJ)ZxB_esFR?{{5pzqtAB=8c<WH*~Hq
zzxL&-+|`h)Tdz6aWW9Uve&BtRyK8UnyIFVb+U4@g;a5zqKE68ZTE}&ho6L9q-^+d=
z{9x(*Klh^^WIs@N`2SJD)9U9ZUKG5X|BCCa<cIU0l)n~!H~$&=^WyglU*CQf{KWq8
z%*Te$ieC%AZ~QgyZ~Olv4Bd>1OjDUwGy45!`^Wu%2g5f;O=e}5?<_8C)$G0;m$|<4
z{}!1d?k2H7yg~G=P^y3#Uk=X<?ilVnTwGkQIM;Kt@c$GH7w#2S6xI-WEU=OPFyAh|
zUHs<+{s?RpxGb<(@RHCUVHr_3vA<%c#XgG4itZG7FS1QkMm$CGpY$5pOLE-`M-+V&
z?<!nTj8GC$+#tVHeuDys(pIG;rQ3>KiiZ>|6oM6bRSv4N>xk)_>UZmY*Ur^Cpuwkc
zNW)0$rgnzzPrWF^2PRJ~{A~W)ZnAx1^T|fScCF1zn?AcXhYOBnPP3inI|)0#cCK>i
za0zi)>n!bb+<v*Oip^cC5UX(Ooi-<IX4!18eP(ywPS<XS?FSnh>rjgp6AQyo{nxts
zI#pUfG)}1Js@JP8Q)kp%ryZo*p~J7$tY)R;EO%cjO`KgMUQm^vi<g7jkaHf#Mb4vK
zMV##H=B!JZpE9mw`0zjJzw-aX{}TRA{Jr7V%U=z@)&9Kxll=GaAFkiee$4r%_Qmtl
zzK=gXMt!>cN&9o~=PRF8zu0|N{`Bjk<HsrQ+FyTvS^e_yiyhC+o(Vk_c{1e@_rn7Z
z=0CK09Q3sL*~F*k9#4Oy`_S_K*1PNPJifj2_R-sww*&5E-p_ly@7ci@LNB{tJbkhG
zW%)~om-}BGdo$(j)VKHFcD<{7zy8DKj}o5_emwrc<Nd<75pUMMu6`}~=Koun4{tsM
zeLVBY=!?{sr=Qio9{r~HUE$k`FV{Xl{QTo{*XK{40zO^)=<;dF=ks4@|2+Ed43h!N
zVP;XLmj8NxJ$`HaZ2!*p{opsFZ))EfzD0a*`C0gT?l09JYrmcQ8u~@>^U+WDJ{f$@
z{Jiy3!$<Xxt3KL&`uU0b%YiTTUoL%a`5gWE{ik1_EWgx#P5-*|OUD=fug||e{C4@f
z*w0D7mi+4dVerlP>!UB)UsyhG`?&vu{|B}Y?jMvsIe(q}W9Dzazf=D#`4#w6_=nH8
zS6^hmIDg*rso~SlkHw#Yzq0&X`P=R9x4*mp{`~v-pZS0G|I7?Jj5`>z{y+LB|9{v2
z`wU)8Cd}5%6PZ>q?qRs~-}(RS|GbQUm_k@KusE^`us`SM<4omh;Q7m|&HI;&jq?M0
z3HwgALRMWC7UoHes~G+=2r`MVM6hde&E)dr3}rvZ!p@ZQfAXJWKPP`T`C9Yo@`pzs
z3_czClJfoZ_vml&UuS>${%PIEJ0B}Qo&5CY)B8{FKE3^X`Rlsx$v?0Civ4}%m*B6$
zpDjP^zi<AQ{{8)T?(b*5p8Y!So7;EgALoC5{$23*)t|s$lfHlaI`PZNPXZsE-&?%h
z`TE7HiLchYp8oF8$JL)1zvO&sdVk<e`zyB>dCz>G>OA@K=;K4LhZ>J6o=$z4`o`_u
z)%UYLeEzWS<LXZ*K41U(?0fN#s2`txJp8%qm)0MXf2aTVGhAnQ$uNiE+5gD@6aH&3
zI5RZ;kNfxVZ}mUF|I`0hF?2H>XI{@VkHPi-kAIQ>_y3>#zxKZ@!zTtYrim<h98Y+*
z2)-13BK}IeMf{%_znF>WG7&|QJYh$nO#<KeUkJzvUlmoAI4xl;u}%E3*j15_Ld`<P
zLMsJ&`4;gAatCm=a>;YA=03yyit9Va61E1Gc&3RAk_>K)YnlEqc{2ZF(PB^Ih~{MD
z{>t;8H=lO~_b<)`oHIE8bF^?w<G95!o70th9<QjNu$Z27xtxRoo1&fKONAW@FBF&*
zyA&+t17stlKS)|h?U%VMZ>8v<sHJ#c;j6r;{Cm0S@@Ew`DDG3dulQT3Q00(nv-%EA
zF>L|uUM*fN8O<hj3pF{lt!g~#?COH*J?gz0Et-N_hT7e_Ci-4_pR}{I4r{*C=u&^6
zI!i@VWuvmL@<ipys$3ean!TF8)jz8IS2!UvOX8|Xvfwpd0j^-S|4gSD5*Z2@Uob6Y
zmSz6Q$i?{Pzv91pe^~!||Bd)__SdwZaX;j~mw(&(E$sXL?}^`?zMcOv@pJE|haY1<
zS$(<s_0-p>ul3(*e@y-v{LAxC&%YD@GXL!VQS?>h^STcc-d4YgcyZ)e;M3A4i=SM4
zn)BT8McVV6r;(2}9`3#O?2h{#!8;<iBX91xzU!LnHR<b*Z<gGBe&7G$nMcx3e4bo*
zeDZPplUGmPK1q4<<*~t&TTk9S{rddms}paYz2SOe_FD8+-isN}H@#?n75m!bwbSc^
zuXnw<`}W2=kq>7-#(eVp@bk^0SKKfEKAZkz)1zGvH$Ieo)cNSmW9jFNulK)|et-8p
z&xhjo&F>W7#lK5>xAE=5H+x<yyiRzn|F-c%^p|Dd-~0^vt@6j>&&=OuzgPU0{B!oV
z|8LLVCBF~-KJ#bVKY4~(3|Ib({(JZP&(F>uHQz11fBJ6qWB<2XUsilc`5OGS>+AaO
z{lC8d-v3+T_vv46ex3UD`lr&*%AX~_%zi8X(f(J=_@AYa-GYOOBZu9E?LJF3iyLb!
z+X1%yY}V|N96LF;ax?Qk5Sk$RO-xW?i-fiWkGQAUF42!74}@cdW($<^JM!o9-{xN{
z@Km6RKb2RMM}>POry%ETt_I#Ed~5gu_>T+Z3kiu_5&b3ZE+r+MCHYlsiRg8aOyLuP
zdj$LiY6R{GSO{Jc>KEH4B__)zw^44soVu)?bfA=ql%eDcF*T83L1{h*?li7;?$10&
zcscl%@aFI&a7|#}#Bz~oEu#)Y{=cNZ=6`4Xz3^`V10$0a(@e%?3?2V-{vG?v`7ir_
z1Y<B$KVuieV}|*RQH)`XZ<)eanwSk3+5dO`P5yKDSMCp?Z!BM3zs&pO^O5_L)fbCz
zI^PYyPx@x~_28#>?=QR!e6#4)%onlGK0SW&@WlhChwmO4J)QgP^0Up))SqcSDSWu*
zzS@1|`zH5=@1MPAcYp4K2M?D$YJIHtWd7rvM^7H^edzG8>0#C5z^6P<&p&Q|eEjj|
z$AV8XpGH4Vc-i>s?@N=HFJ4}KCHm^Xi?c7jzqEYa{>Jc~z{j$$Z9g@C%l-cUOZV5c
z@0MTLzc74x^jYq+<R|`*LLdIVzx&bfYx|GH-#7pE|BqovW$5_d`ES-g%l}>fZvWx<
z<Me07pY*>8|4%ZOFiWv;vXnF5WID^#%Dj-}6zdeW(`-uYnH>K(A8>u<j^Ha0P!|a2
zyT`-F<Hdc6Gl4yybuz0CTLx=7>k*E%yo`d^g{FzT6@4PQO=Pn01mPZ$deOt8-lCc!
z_l10fehG1lN=UqtVwY8sH&c{Q;!&Ed^ho)T$`=&|)l$_TYEhcAwKwQ$8+<g3HssXj
z*FB`|u9d8rrp2x!q@$rNq^+v`M{~8te2o~*xtj4Bo~qVL4vGgAgyr^2wMzO(a!G6v
z(-vPZ871qWa9Ke`;i~+5Ic}ME5_`lainobv5cw$dU7&%FhnI)1Ltwp7vIv{lEwKtQ
zdr@cMhXR)TT73FE{hZ7kQS4T1YOG?coUB>Qz6{6yl>O}cmi$HS^W;zLpEy2ty_bLI
z^490g+gER1slL7VZo*ru*SxP7U*3Mk{zU5GpL@seG2UN%_u-v$_f?;$J)8gR!1KZv
zY|rOD5q-Sp(b7jUkC-1e-d}r{_fF|;^E;;Z7$07I{N!oCbI})<o_~Ip_ssqo!?T@F
zeV;CP;_+n86ZWUIPqUwCK5u#^_|)%-)f3GpFP~ID&wq9JwfCDhZ!f*i`M~l~_w(<s
z5<mEV_Wp?c9{7Fr_o8p5UqinC`8ofW#&3&%cbItC*|=77Z{~T<YsNQ?=K>caw;Rs{
zo<44Ut`{8h+3&NhWar^J&hwM6PC!v`i$Ep+WZrsiPp(~@YdQMa@3Oz)Fz0OGe9oo8
zQ_gM0`IgO|bpy*$Rt1i5u9e)2dD?m9__px1@ka~%70egSkua5*BDPq}S^SUaKH*d$
zTcM9a_QJM87DBT`FG=c1>qrO4e3bbuby&PvbdHFc$aA430s(x5JZrdoITJX4aXImp
z@}~<j311X05MCtIFQhBHURY41UF4X^O%YYmFQRGU{F2iojU+_HxJ3;`cZwbq<CVNA
zZ7r)TYb#qSn<G;peP1R~-diDGo>i_<MngJOQdKNi*h=`iu#-@`z#YN!BDP{@MK_5o
z5>69h5GdvO$a$VKjB6q1GxjH}i<xs6H5fV=Oqo<zwzG1v?`QX8Kg2qnc>-fELoCDm
z|4ILT{$>4V_qXuR$3KaGZ~o^0`RJSL*XqwLpQe5;`TFLo%-7s6=RYs_)bY{%<JFIM
zKmGZ%?L)%bu-EHfMm<Y-bm#uQdq(#*-W9le?XKU$zfW#G?Rc8;?Ci5!PqH8Fdl>cT
z%A*yJ_CIugu;5<m-NL*7@0C7GeNzA2@YR9W7hVUv;(HPQto`Y>CkGyDK7Rk`(xV5D
zvLDAkk$L*>3FDKOkD?wmKgxOh?+MfMyD#Rwba^T8Qu*cd7eOz6ytH_;_RaCvysztC
zZF(90Lg`uBlmAbqKimH7<1>a=dG8WFp8v%0MfPjO*ArhRe%Af$_@(-5`B%3uPd=G_
zO8VsXneQv(kBr|z|I!%}7<V#!`xp2(?$68Li+`*9p86~Im)S43p9w#ze{B9y@tyyx
z_GkM~Z6EbMhJ1?nQu1}lSN(4%zn%Q%{QcjLh~G#5uK54(zy1Hd{|Emc{mb=R=vVBo
z++Q`n+J2}0Gi2Vv-o?3}vz$wS`zz-+b`kdJ?3^6Q>~C0$Sj3o>nM#@TSUlKfaBSrM
z!6z>`MevZo8vY>uH2%kY+<c#SWBC&JZ25Tjmk1mX+$qQ)s439M-_Jio;E!N|@Jo>^
zqSr-#iq?vCh<y}GloXUvk?E3}BKcV&L42b~qTp-3Gkn$j3;AyH-sQIvW)zhbWfeUu
z@<>=u$U{Jz{{!DTzLmVExwSd>vh8B!Wn*SP#?i>#&nG81O(<E2M^HsTRp2#01HUn!
zGG92K4BuSdD&C8{XLyTw47nz8{A15y7h+H0=-_(J^@;Nfrxe$5j(zN3IBszDa@%r0
z<6OkPmbHR8|3AlX&7XIF-1~m%+syCrzyAN-!QjexoZ%0{Uk11T?0@<G-2JWdd&kem
z-~WHz_vOf!?_W=RfBAFEA3la&rXNht81FFTGw3oD|1bI{^zYKY{r^||7yn=IKbAp+
zLFZrSpPJwAe;xRB__yp|)xT+f?Ea+var_hXhvo0le_t6HSeLSHVCi7#V%f{I`G3~m
znSW&dzWQtWFX_J_^A7f#oZMWqxoWsOxh1&yxTkU_@x0<G<T=5$kVAu=gM9~k4aXAB
z$J___{t2uX+#{gJufdne`+?hyYcHn~mo?W;PF>FL9M?DuIc~BAu)4EsV~%F7V)kWG
zVg1LF%2LmAkEMtupJh7RR?hES|2dmDGTCj|G+4A5dH(DF|MI`_|J{Ea3~!irvL0ZY
z!FrWhhxs|vEXLRWBmXb|KlgvjzmPwFf6n}2@}uyF=g;zAE`Or_cQJV|uVpl6$oT*F
zuh*Z_-(A0*{z(1R|GVN3<6n_~!vC)QS@-+LFSB1Ce*XA1@6YDH@&A1Ozx_Y)|L1>g
z|Ccd(u^eTsV7<s9$tuJu%`C|v|F7e3!(aEm-~I^xTf`X8+R1jAHG$QQbrMS|vj@`|
zMsX%9CR@hL|E7Oceuw@#`b+fpoZr&_t}qs}RI}!@+OkY%DrZb%aQmP3zn-CnA>{vq
ze=h%x{-^va`rG+u<?q1XcYpKzjrv#rUyOl);mZHb|8@Rr{+DBX%6yyUGxIklX{JDi
z!oQ}!rhPx~P2@Z0_lWOjf0qBf`QMRY+W+<cB>t`Zd;Jg3AICq@f9?N$`YZM4%FmNO
zPX3tn<H!%MUnl?E{Xdt9nOTvs?f?FNdVf`Z7yRP?{o(iYKOKMe{dW58{d>_b>z^4v
ziheBm!TK}um+>FjzubST{%Zcq{j2_G#-FTz*Z#LN)G*BXKlxwC-^yP*z8(0o@JrDb
zi!a(=Yrb#&wfFCif9L<&{@w9c>ffusYyTek8}jecKjnXdf42S<`LX`H#E-%s**`n~
zbTjN>y2bRK@%R74zplSye@^<Y`u)ZCitoF=?)sAV#q)E)$0hIC-txUT`5@}%%ggl_
zV=pbd`s&7<Tc2)Cy48Iv@s9ibc@NqjSU;40^yPuxJ@?!5Z!lkfca8OW-1V0?-`_p=
z;M&7g51%|(_+akC4Uae;FL=!MROH#tr`AtaJW_a6{iyCy+as1I3eP28KYTau1J{SL
z_uJkFzCZoW{)5@)s;@`C{rP_VNA6GSUmt#1{66*j^&k6xatx}>i`i5;7jZT4bn=Pw
zYw-ICGKn%sa7YzM|CRnPT`FxW$uCwaGDqZ#h?Ll8iHS1X6mF{&si~<tt4OGBSIty?
zq>`*+uac}%sjQ~>PfkRRLvD^-y8Jc;c9p{#-P-Zm)>>0Eo~mXkhbg%#Jy1HQ%%M`C
z9IiZ9`L>dmVt_)E{43db855beGD5QZWxmO1%2voENcT$ROD&UjkyVr3B>hrKPO4Bm
zPq>%gmnV$tBBv{7H%B>V2zNF2J<j{=-&o(Vykt>kF=x8PpvI8&&+K=__xqpYK7N1i
z_HNso$!}MDi2j`Q<^1QpA7{N^`*!K8jOSCHO?<xZ`ORlW&n7($dBXWv?9tH&U+%5A
zTYp#L-tBu6?r*%Wb^r4{_4|eQlJ17wTlQez!`_F*4?P~{Jh*UA{f^Yl;_ETjS6^?u
z(R0h<?w|XfkLEm*dKmP8;bH1S@rO1KxgYU7e*5^z<5iD(9xi=Y@hJGQ%9D3by<Zu9
z;Q6BV_4(JfZ^yrKe|_+^{JYdoi{D;<UH`5ASNuPXfq}7r@e-pFlLWI5s|3d`PG+vh
zTnl-$cu(+b;C;!TCbU&VT5P76y?Bv$r>L}0J%22pEZ<?i3IQo$9&smW6WNtASET1i
zzm!gwF_OuZmXmgtwwKP43YG8^V-+hDJ0+SYdQMzMX0d#aqNI|lVuk!}SpjKXi6n7L
ziB^fP;u_)-Vz)$&2tOD4AUIoK9lxEx9>GmQKZT5h<AgJXb%d3LSOjkhDvF#HnJMfl
zd{_9FP?A6i?|!ah9KYFru(xvD=X}L|kk^gx4v!fZ9|t?T4clXuHO#flx0n-{g&24K
zb@?sw>&g$8@2=l&e!Ks3!QYI3S%3I`2mQM7z4yzDPxYTZek}Sp=R?^$fj668`M%_T
z{_RQm<Ag_t9)5fH{87x~Cy%5bUw^#v(W(cA59dF2fBN9*@27j7c0GOa#P?CaJ>}az
zH&t)^xXyM{?{@ROQx9)FI{PsB!OZ)o?^i#FcyQo8^Mi&5%nugaE4X8M>*mc>w~}tD
z+>*F!_2|melII;S*1zC*(f&N`S;Z6OCm)`gJU{uY@tOGZoacSdOJ3-_ocE&bMe?f~
zZyMj7cwhRl<nymDQ@)<~y7$|mpKJeC{_p$$k%5IdgqfX*h2h-aH-BFJjrsqF;Vm;S
zM-{gKuNv=No-E$Gd_VbH1^fg{gmw#8iv$Y`3r-MNBPb~JL$FZzxmbeqEcsiC&lTmA
z0+jzK9aDHJFR93{q^D%2bW>47QB^TXF<n7ZK3#s3LZZ?al_2%G>LKc<G~%@wwN`2t
zX+P0@qjytpjb4|Yf!;lx?V9^l6O<n+Em69q)TI)yZm+GauVi4O$E{nZds;V3_m<9H
zomAZ&y2U!YTJzNJsJ>SDuFS2}punxPO!c)|pDMSipsJ_xMulf`1+tT59?N`|nJq0Z
zl`3f@VI}rOL|bIB&_qF3p+w=G!V^UL#J-C+irb1!6n!G<EFLDoCSENvT{u#hT~M00
zmh&UqTviFzA{JGaN37|bf4Dh#W^plc?qg?R7hzk%@`d>qvn8_`qr|^ezqb8Y_&xmV
zuTN(_MSfZJjs2(Luk}AWe<*yn`(p6n+}jgxyWcK(lk?`z>!UA^Ki~7r>M6rx=0_Zl
zmp{4qG~n6Ur{+(;KXrY+?fHl2+%JE={Q2VMv;L<?p2R+Of9P^=`>moI)2`QF3%$y5
z?ZJ(kxBc#l-@SMH-K}T0KHfTfOZ2wjo#wkS_t@_m-uioe+O_(tORnTzesW3tYSxXw
zn@6vET?@SW?Q+E>tBY0_$}T*(7=5YY;_(aB7dS5DoL4>HaQ@19%L`nW;;+<PU2{$H
zrrmAPJE?d4?)uzMeOUZx^&{`cKb{mn+xk@JF~`G)_aEHjyw`Ql`{C}V!LO#im3x2j
zeb<NQAC`Rh^P&1v(wDw(3x06?Wcac1`=TEoej5F5|HJoRg!w7kG0uP7UwCwQgLo5o
zWVoMmyYiU{qzO(GIwQ;~VkOKVtRlQo=!?)7k#piFq*lr%E38oZrF=p~L^VxiipmBx
zGmRdN*P73?C3Lj4+|_eb@|Er={#Gng`l+%+(^}U-PfRafS5>z}uUWrGf2V<=ai_^?
z;|+$V^<U_z>1pa7(D|oRrYB+~V#a9^YT;$kYW~qA-SD5DvR;6mpn;H4jL}lVz4{C~
zCYr4p(i*!p)@mu})ay>xyR6Hk%cA>Vd#2VMtyZ0%x~6(AdVBPz8eTTk)ZeYSQf0oP
zko;QNZn;*4P0I4>x79W&y_0K`JtDJDdYg2#%r)r=QtXmKVtqm{_#OFP@E+xT!Lypz
zPvDoJu<#U-y&~zt#{~EBujYNjv!2(AuYfm>r-hrBi<?7=ErjJOQ#O+eGYjh@)+|;p
z)?2Iwte2TKFf{%*|6jx~jqx)>F5_Ef6&7{I>c1hsF8{dk-SBJR=fhtXe!u$D{a4mc
z-k)246n&rd&FowLSMe_^Kkj@d|0ecj#&g4GTOM0H6o1(I=>HSr7gJu{c^&tr{k7T4
zpl6&<vL5w5)P7X*DDi>bz3+F5ZlAm*e%t02$IXj3Y;H>3T78T0_S`#d_ss7Z-RZy0
ze7Ekd#vO*cwGYxCWjrjo7ksPbI{Wp2YbUPcUA4a%dbi`g_=AP_&F_cY=f1!E-iv#8
z@9n!MbHC@_$2*R<7v5~XvGTghwWzBnt}@;Tywmr9@v;4rM~`<u%766#;jf3&9`imq
z{K)c=;X{Xea(7(sNZn<=SM-4WiTSgD=QCdHf3f<7)ho`|_g}WXWP5$>P50X$Z~wkG
z`o#0;%loXik6%xHJ?WLqt7C6YfB5uy#aHRC)n6pO$^2;gdG^=#-xvNE|7-hK@t5Vd
z(T_*ps=i(Sn)kK;tH}4IzmEPhWHMxV%>0$n|G&r|$DaW|WPVxyUHos&Kk@$@|6~5;
z{4M<b<fq%O`rqw;zy4QYWMI-}{=&q?sQka`|98e%<|oXxZ0Ve#Tz@&8I4-kZW=UkZ
z!tB7R$QjDZBfu;8Lcmnu2Y;`?B!Mo0i9%mRr%RlY43V;vJ}doJ>VssT<Qa+k5{IPM
zD6pumR)3-SP5Zp|JxzZNDfMq^Oq$Hv|FuqPhH7xCd#TM+tJ65IRjH$=?`>Rb`p4v`
z$pljY(^E!#hWdv6MiWgU&0d-Ln>SdLTU<0<V06Rat^OtbpN6bv_bpG`q}lb^eX=XI
z-)q}sy~g^#b-2|vi#Zl57Ru&zW{hS#OpY5R7@jqlZurc^*s|98g0;5wa|?CTO2Zue
zRr<FLRvIQ6IUA)L^yxL|R%pM_u+>P@w9~TF^49XzQq>aI*3!w>GFG3e7O(MJqeuO<
zTCWC!)<cbA)k_N3Wcs8QNo){%E><P!BHbuCU({MyMo2>7GT$=()dHUd4)f3F>E~=^
zzsr)%sPeDt_qw0^zUO~?`2Fv%^1n0x8U5q<TmO6MkE373zS@0t`;z*p{zLscz1Pp4
zZFs`;^v9D4kGmeNdt~!y@dM`vzaJz$V0+;7$l$5M^VKhsU-G>adHMZi>dQk<_dHa&
z|Md3P8-6!0-S)o!_VMj!a?kvqh&(QMc;x=F`#ld1J<xe*_xQk*{KpR-1l~V;&*C2Q
zU9UT0cXjV)JQ8{)`ojLX+|vb*+a7Iwxbe}vCrM9bo~b^!eAfHq#FJ}JPd*KN8vRV@
zrR!^-Hzsc@-U__E@aFvMMXzSQzVtTgea#29PYGX~zQuig{4w?2`8TO=lwYg7N_sQx
z!^=-=KkfKv^|9}R@O$ZZ@80&k-Su|*yEX5`-W9y<eDnD=^P9NWX)pgiKlk$8+oc~$
zKYsk^^YQGvb+6rCCcjX88Tl&jHOHGpubW@-ybO5m`)t~?l4nUzC7+#nsq$9qo!9$I
zA6Y)H_&D`L%g2SEOh3>6;`8nFw<}+#eDZu-`{LG<*oQs$=R92W%=ESC+oHG6-%Nci
z@M`;u{^!S@>%9E?a@)(LFT$RyKWlvw|48$}`#a4yXWq!WRdM&xgNKigJpKN3`ja`2
z;vaC|FL}`Vc;nOhXI@WVJ?y)8=uXL<b$2Z9UcRsLWaYEl&sIM3f9CXT=Ce~zJ)Q_Y
zDSFEG{Lb?mFKS<PzIymP?8)>8=kEN!x#Px}o7?VoJlg#<<XPcU_NU*TY<*JrY|pC+
z?<zlJe7yY8`Xld$#qa;UpZR{-JIi<e@ABRs{iymS;v37)^4~Upe*RkhbJ;KVzyJSV
zWctsN%DRzdCNmdPEJNLYZHBju*O<?<2C{EsZ{y(PisrK6D&d(Wz$S84bh5aiRDq0&
zY>$kRbcm$0_$!g?!b?QeBm-qk<Z9*4$^Mj4lDQ_WE8CzjU1fv1g61yGMVb;C7OE4J
zIuvgyo=`GTWzvk-(bX3=T4MaiDAmx+;Ff-(!2!ci;{(ROjHHZK8U*P*)rr-O)6>;o
zZgAGf$Yh4eLQ@0tiRMn`CFXG!)mDwR<@QhPgB`3L?>ZRTJ+oSG;ci}Ow#)2{`4UTY
zn-8{%b~|nF+3MS=+0D1hvtMnu)8?<Wq0LL{1nXl~pDpiM)SH)^oi@|7xMO+RdcN%v
zJ8?Tl8yBl178lLM%@3I_HU4K{q`OGtn#xbboAQkEZ{#}Ve#vsnO_%#1XCj{^Zz6X?
z`n%Kt>BZ7@QWK>(WTIs5NGgfW<rn4d<hakafORR`d(K3jg*+3vuW<fhUy+hnlE|9Q
z%E-2qEr+dvbv4Tk=50*njDP>j{EPW>==ZZf68}9J<(UmwzA#^4TEMu2L5yMR|E~Xm
z|117!{)zcf^JU*Bv(KfUr+(`E^y%}(FScK{ea`x_{j0+Fw4XQsNHQE{Ok&Dp{>rkD
z^(;#vvlWvB<AeX}|IPW=!*GfD58Eb=432*GUiK*V*=%v_4>=im^mu3To#(gU@8F%t
zvzXhDb1U0c)@N*+ICt=<^M?!06h0vOPpm<VM|6X5g3x@ydxBF0XY+^fs_>L>S8%m)
zuH<y#e$JaLP$$?fcv>)A$Wus7C|p=lY?4HSq@<LG^kQjFDR=ReA}vDI0;~8O`Ofj%
z3vr7?iBt$%3#}H&<+JDU;(p2<!&}5>%-6$X#`S<bm32DP+W&%ocl-|iGxhK9zl#6f
z{af*W#ed`fyZ&|kYyS7_Z}6W_Kks}~{rvmgqgUUaKX|t98PD^&7eTL#UZuRKcs~8P
z^z)-nXFt(@BK#!vF~g%54|*Q(JqW*Ne>>>9&t>Tg)6Qw0kGrUSrT*H=>oM2gUX8f&
z_R^M%ofl_Z;=O$AV$JziXSz=vI}v+)!_mUSY6lq(q#f9Lu>A0cqZ3c8J~{8iw_{yL
zuOD_kRCkc+VCjL@{SJHI><rzWzO8fH&h687N$-2U|I+^S{pS1o_fI`gd{F(+*~8sO
z?;rIzT6uKavD?RI9FIBr_Mq#2(|!E=e(zg(Q0(Zu<3=ZZj(s`weZSJan0;>!2psV|
zR&m_qc=ypChfg2wKJx3x)g$+gEI6uqeD{eTCoi0ya_;qo>6gN<*j|geG4WRSo$`AJ
z?x)@Vc{kvW=dHsxTyHkrw!Fvn@bF`&r$tY{KIMNN`+Vl}=`XLnaemM7q4Y!jrz>Ay
z{b2ds|9k$gu%DN|U-)|Ui^n(LpIU!z{LT5d=%3R6<Nv?@H)3dF+{aYMbc69GBP){$
z(<&w{=0xUt<^&c4HU$oEu5jKsf$M@x1epZi3A`7G5;)I)n136;G5-r*Yo0W2QJ#mq
zJ_2IGyF|B&OH0;C)l2tDw@ROtUL~z7y;<st<Q@qFiQnS3;xVGk!fAqE1<C}Agw6|p
z6YY{%Avs6lxtO$=lo-31ycnmrh(w>nWr@=g+7e~r!s0z*m7-gPKMI-)z7RYl>@K!g
zLQ`t0)Ktk3@dlAKg5vxtymCAi+&-M@?0TRRvDjX)FXF7>TFbSI>pYh|cL9$9-w}R6
z!9t-lVQFE0p(TQ4LhFTvML2~o3JCG_a>sENv9qvCa{BN%^ZD{U=UvWcEFdeGD<~l3
zDr_LKS@^1;kidO@IYAwvErLJ!9r-Ts3h{>t@`}tBZ<l753zC~EYc0D^_LiK8!ViT<
z3I*~nWiq7rBsnDSOV&z1mu*nkq*SZ&NmX85SmVCNQq6p=KUzPu_GnpaKhgQDmt%0p
zAj6<SKUS|oH$ZomUYjA4DYN+nGk3GCW{=I-%qEzgH&eFMvi@Uz*t*pEi?y}wB0DAf
zB)c`Xp?1ag_wA23_&CeB*}40=F}PGa*4Xc`<+ahaR<dTbcC-?;JZOH$Ow)XkMS;~Z
zYcCsFn+4XE*5cOD)_bkZZ4TP>+xT1iS`}D5w~DjsvoJ7=Gm$rLH}o?oH{5NKZys-P
z*IdNxgz-zm-TFDYrMead4~-@p%NS2I>M^=*BxH0-ze#t2&KsSty5{<I29u3~OasiO
zn$#MtGZZ(xpf8{&r1Md8t@=)t?+X30AEmWr7Ra8J|EOrF+^#%VdAjm(C4EIvg^ThU
z@)u;!$y}1zB)eU1s@z+d&r+6>lHyZE%7yF&=Lj|kUl1`D$rH8~J};=m-^)9Pw}n@j
zN0;*s>lNli=4fUM=A%q|8F?5Z7=svBGvqREV&2X&pV^8jopA}ns{d~P*8iFRd-*Tt
zAC6z1yq9|u_43v8InO(v|9!^x%<bt1(AgYMT%IyMyYg(+^X<>2pD%sB;pOvJS6?PQ
zk9xxP@bTUMxAxpPexvaAkGsbAX5ERn6>vlQdc?Jks~4|&To<|#aXt20?6now%Wv+w
zx%Rs6RlO_oFGpV1x{`nG=gsGLT_1Em=zH+uf%1c}dr5aY@0Q-3bVu@b`OW4VZ*ElI
z^173Mcjn#ycl7Tt-T8P&?Vi;Aw+}8o-uG1W*^XybFG62!d2RjX;A_X%;;(PNT=V?q
zGnW@fU)jEGd&m1O_-+5&nQuS7x%#Hy?aR04-X478{yP2j?zgocOg~-twE5HXPbWWr
z{;d8v?eo_!9bcWlq<otC;m*65x9o2$Uzfbz|62aljAwTq{ks3|?z%focarbs-aCCy
z`ab7_FAvxsCOmMsf9LM1J4bGN-xj(fd9VJy*?pONJ$F=Ym0V}Kdi|2xrJ&32FNa@}
zys+e4(Ah1g9ZpG{<T=56eD~4EhkqSBc3{r|ql2jj<n~4Ek=;|g=hB{&d)MukK4gD{
z^Jv(Sg@?BuZaiFdX!XIEgKG|i>|eR}_wJp$M0Oq7IcaCn&V-$*I~VPY+NCgC(M<KS
z=1V;(!>79Y)g6_z6+D%fXlQB8R=p^@MB=R!k5aJOU6l-lV%ZDwE7b~hPZ;@`vsx@R
zT&=cImQiMvf}IMd!fDZ2yz6-`3rLCV5_-k!&h?aY6_+W`E&droegbS<EzJM^r!umz
zK4rP|Z_ih@_bKnozjpmP_v^~{4PO#JfB(Gf>xCb;|1ka&`Puir>-ntbI`8?u&HEbs
zKKZ57i=fwj?=HR7eii@h{u8rjxvyq^SpT))yVEzG&zc_?KW_i>`D_2j)i33q?0IVT
zM&P5+r;8s}yj%BX?Q5Sm9UpFgU-@U*FM}`4ALf3J|C`FXpLNy${vQ*+?)ZNCcgeqJ
z{{sI#{MW{Gkt2@(pK!h?lkjKWIb4A}ih|2Tz6qb_TglnMwO*i0?5Jd%)MN>NvCSfV
z!goX&Wv3~LC^U<o61XHJFMCVvr}}pVBMEb1E#WUB1)>fju0quUy#l|4-%G~GYb!R$
znMnQ=xh--|f>|y|K~46kxSbe>q@~Oc*}JlzrEf@n5#K0QD`l&)MEjKH0fj(uA<>I6
zo7JXj_$jeU7m6<uFO-}nB`I}S;)b}IM7@-Ye6|X|nu^jr$q>O(-o^aFBJN@zMf!vs
zghWLY#TJNf6@M?HEpUXpmFp4zG^y1}l1egCh5|x7vjyKuMo628uID|)d5!B7AHPtq
zP_V#!o*GVP&d)p*La#)(id^Sg%$~?n!*-4HA(t6vAgc#sE5lRf?;LY^+Ie<yJY!zK
zbeKa?Bul1C`lL`c#}DQ(4q<_B!c0PwcwVvJU=Qc{EcjOBj*u#E0!I_OIF~=)GQov{
zX}q)8av0MXXRsdOwBqt)6J`|o|C;eP`)l4;{BwBx**CI&;L;I^my(k75S+kX&eXsv
z!~0(-Sa=h!D%%;x)r`lPi&)i|^Z)Mt8U8Ej??%Setifyo%wd1~zHj>K@RREw=YPiE
z`CoKD$b8!Squ}4F|3!Z{|C;!#=l^&18@v~IQd!o0?|Ap`o#fZAKmLE8_Sx#4#OtfC
z&%A&C`N3zk51ZbVeVq1<{ohO0T+UVO%bC9aiTrKBkjD0#-Irzi-#0&&{nBRK#~Q)P
z&h+GO=I?p`)Y%L9c=?$)9RBotp7h1|&v~XFEPhO4e~o`>|DN)9*8koAlm7<)R{7J(
zz|Ofv$V%KyB#_6Lg^%$OvlK@&=Mpw4hNr)>|IA`~!ZwX<7n920uAh&8moi&%zv27F
zeVS?NuPeWHF}!AZ$kNMD^K;9WwO=-W>;1v;bNe^F&vQSfexCQ^)W29p$$$3WUcHrk
zt@eJ`7lH3$U*$g3ze#>4{H6WJ)*rE7-oAbLvhMY*k77U9{W1Jy_}TKU;M?+#@4otd
zU;0_+jo-8Or#3HMzOs1r_G!qY2@g*_;(y-%cJl}8x5-a;-*voQeqZew`zwtX>JOQ2
zN8eU>u<3E)W6S$@Zce)X`R3mHhn}5!b@sW_eXZ*}S3Is&-kx$l`d;`A%gZ+|MP4hs
zb^liXwT_D-=Rcp{c%}6g*WC-Zg|26u_d4x+!TA>7ql1q>+}m^W+wFai9zC7?<oH9D
zdvdqM?o54n|K;uvD?j?Z+41D!{RI!|pKW?0@ovkjisy<i+TUn@ocQt0+efd8UtM{#
z<1^#mP0VxI?y>CoYx-l_@BPefoYq{^*|?cJnWH)Fd8K*NxJ=l#GcRBXXOH8_6IK_C
z5SYw%<$uM0SJo<?g?vZ2nYmtY@8Vl5&?Jz+`<bJV)r(~cyCmOHq02&Nd7rXe_&e)w
zALA#MA1t%}zyA^P)8_vYCL?AI<~W9xzlDBI__^k94T~weF(d!?j`!7Xc|Y2I7y0G;
zUH9|ik5->@K39HL|9JVW^lPD4EpPk3e)&`R-};Ztmt$|1yqWk>;%ng7Tc1qdr@Y(t
zvF6*E-+BzY|2O_V^+WuZ?f)>=*KBG`tv_GAH+c2_P1YCbA8OyYKVSb~^S=Cj+{bSp
zJ3oATr}*CJ^MhYyjEc<L{=fXO>eHc*THmUFcmHGmSNUtn_cz~}ezE+u_`mY+m7i7L
zcYHtg`!Z7jTQ`&UudYvx?-snz|5E<b_xGM3iC+Rf`F)=9<?Ls}kKf-Ze+c=i`1`=W
zH~(+`IrTaFCHGUw=lfn=dGqac%!`jtzdif@g7dZZ+rl?dFC?A{KjVC3{6*oX^AEL8
zD_{6Lba^=Y+56XD-|D=X{L=VUz}p}1MBgoW&G^Fk>GNj~-kkev`eXX{10O|SZG77Q
zocrB_&k<jkKX||4dK2*W$-C3<x4nJwGVJ-zXAZB`zIQVJWS`7z@T>k)^(UJjU;cdk
z`}j}r?~Gq{zt{e=Vyyq~`0MgF?jH<)jxlcIxW%2vKIQ+`-@E>Gv0mf)$MuNKmw77F
zO{O9iRSrfj362cbXckTm4<Tt;MnzrOk0M`qukub6&Xp9AahHB9Q6%w1N?6uI)=nl-
z^0SD5@D?!xxsNLA)%uk7%jii@l)t1Zs<mDFlIC(1356SS;R-fNyA)r^DadS;`YHQg
zJ=f^9=_bQOjScd3a&{`YT61;J>k4aYs@JF<QghVcRo|`@FBc@2t-MTYy1uM_hUN_g
z4#_iOU!`^`L?~M*YRk+KZxLw~UL*Qj?2OP=?wjm-Ty{cBr8;C8B`@<yuuS^5h2b#A
zUxAxK%6$DCnry4sG}(<=HvdifvE=)ap9X)E8JU>3{rmUT>Ye+Wr|&(!-T2$Xw2fiY
zpQ(Q=88le>SbqO!{FU_G|CbO`5x0YozEB)z$Y0HGhrhk~Q^u^px}4GG@3r6A|9Ti5
z89x0g{Ql<0{J+bYB{+k*6}jHC{{ENuYx!SmwyV4|1@`bU@~-B;EUGVkO8T<6y^t&K
zMIHwc8~Kx}St@H~(uGg(y7IeAJW^s;Z&8`3n4=V|{z!*OuS1JbLqqd|_6q$ZQ#;Fk
zb1_2!%}$lSYMXVh8Fd)F(eYIKt29d`SJPc*kLGseU2;zHJJi?=#VnhwvWzU$1QZ@9
zP13qzC}gt0z*+l;x|zmjt@*m=br))%(G1X1*L61VF(}ZkRf?2w5n3YrSn`kjHO1?)
zQ^hkx?ur^pE|v<Dd@9;6pu^WCvP#}gZMU+pqz{iC>lfCQ+|fd*Vi!dx3o`MBaBtvl
z<vPIjg~^HW5aVSQTdt41MclQ_H-An1cJxQZe=#-(4q?`_{|kSg{dw^B%fAwT8oo1p
zDft@q??2Z<VHx2zPJutGKK8u}``r7xf}xG!`M-;Q%l{tyEA`LmulFym@9)34|9;HK
z#J-086w~Qn&%dnwa^`0;qc58fD=TB)e`TgmZ0=k>ocmdf|IhxjmSHXDOTj=9Grk{;
zU%&19)cf_=?;OV0%r#8)|MmYrVbEvT{72<y_BZLTa^K_rL^C<EakKLOTm6me>x>_o
zf6n~Q{Ql{q)4LgO!`>TxnDaK^Md~Bt`znw2zdZAS<8#tGuO}OBUcan-wdvNHyYY7<
zZ_c`=c5UXhOIL-icwO$gEPQ3^RiT@Tcg62D+?;nY;*7^xh0E)1mfn%P<$bO5>b2`H
zZoa*_`$o$(#mlEIp1gYb?#pLq-z2=Mc-VI9$xWSm7oG&a`uTeGtF<o<J)898*Tca3
zGwu}K%DnaO&eaDSpL}|j{p9)`o@=|W&AF%W-1jx-8_qYCZ=~Mly}ACf?)lxP%8!3N
z<a)mNW6fV)R%vGWpIhG@d-3mi!)x9TX`f0yd4KGC@A~e<TbsA(Z)D#vzFq$Q^Osw{
z`~J`Qv+?V$_u}u^K5hFk{qMK`jEp@DhX33D*8e*HwfX&@=P8dKJv{tu@q6FzcHb_%
z+4U&y4%4j#x0UZZJq&yp`#|*m@_QHVZ-3Bt@7v9{*Y;iKzO(L;+>5Xm=}#8i_q-E&
z=k`6j2etPb?!Lbjck{-Lxz|OnExoLK&h^B+<CSMMt|#9we4uu#?&88zi;hd3ynbfN
z`Mc+f&aFGEdj8Wziz`uAzF&NJPW$Zdv$HOWUlqG@@7&W9l1Dp_PCl{cRMHvGb8F5?
zp0hp2dVcPOUl$)<m~_tS+{KHRZ+>~$@;K^V=k>>zK3qC?ea_vghYKD@Jo)l?%45+d
z?N1jy(|T6&)bZJ^7gyireRlrp^Xbf6p4ZxMSU!|~X8e}@W7)6fKQ({u{A&5#^pA@n
zhrx|$GW%D-Y?<$}SHw#Bj&oSE|KYUdyDQisGDCEh=p#`zF;B6*V*KJ~#N(y3<bSI!
z)>)|+qIFG)PnK1tM~+{~OYN>!jot)<6#ZIlSIvA4G1W^7I`UjfirU$x1?Ddd{-__9
z*Oy))T_?L(ZjQoJ#S8_0xpJA)QW6s8B8dWe{Bwl9NV+S|R7p_YC08%OAtEEpAbMOP
zPsUv?UhcK@Z}BZ+r^MJrj0KMJT@o~qG1F|->(*YR)~nbjyHSQ&cB-tsLbsBp;$^ve
zayR84$azU06MH1wEK(=IDEmatK{iq1k>Ej|TrL^zFMN^0g`&-(hlQsKP8UiO$ro(m
z-p!uH@q_QVc#E`{q^78);4;4be1G_!@y!!BDcH@wk>@LS22UE#Aub1YOP04RIh^)<
zj|5}|zwvG1w&I$_J)i%i=mV)0(gu=~M9YQF3Z50z=6B+@<-EfcD|lN%SL&093$GjN
zt^bGrG5vq~zn%F!+eg-P=2J{{%-zgUOjj6`7!(+tS=>3QximNnSSI~@^-JUr7t<=v
z>wEzMZ}>iONwFm`&td%bSNfOe&;P$CFmK?o72Lu9i|ZSUF2k39iy7vz6mZ?(d&zIX
ze}uQ1`yGcZTP^c@#=}fLY<fJ81et`l^D41k{=4nhmp^Nnn%ILmtvHXftFdlo>S0*>
zSLZjw?{)v!*_R5&NG3>c5`V<+&Ed+@!KBGtz;TJ+TjZtaPcb#|6{1|iEBOC&vvbYk
z&J_?7W0b5D%i)jZuw!H8I>&!lSXAtf_ydW{5+@~WB@T#riar;v7VZ?kEFY}4S$&>r
zkz#|4pyVI1m!j$tCuAPVO_j}%iIKS`tt!<o-X>BkWG{SEe6OsrLWt}Q@fhKMf>k2-
zC6-9HNq>^GmN+AJQ1pSwEn#aRW`U)A+5!?H|Hb^o-U-@r*RXi~H~xL}hyJgde=YvE
z{k{Fm_s55C&EIUl@P80|_vQV)Pj5b-`S|(uxyLGZZEqLdJO9M;^{=-IZw|juefI5<
z<o(Te#P1~CS$mKB3CHVSAO3%K{jB<a;Y*9BB9AXUp8ewF`_?b#zkK<`@R9fZr#H)A
z8NA%~O7lbacdx%k{_*|R{Cwfnn`as?#osr4{q!^WH|KATU+=#ge7*k3_T&7IoL`fF
zeP-z8sN!49|Bd@G%illNKQ8`o{Hwxzg5xib0>2ag1YTFJP&R(%V#fVU$Jy)nN`-C-
ztmP78@n=+L*5v5qn<%UzHdR7FdV*B0xT)w3;Y6X|f}+A!VrOKuR8FYzD~Cwe2~}{P
zV?V?e#V*BJ$z{wnheL$z8q?kX`F{<5Kl|nIuadcgU4|opS@O@zAHu(n{}uiJ;eS5E
zJVpiPSVrr=#()0&o%1*Pm*Ds0Z~Z@S|Be3l_9xS4(Kl0`?|a7fLibhcTbB>g9}M2v
zyvcmk{<7m`|BDUJSG<gV+xb55o$*W6$1fi~ddl-E<<0!JYHz2%Hhx+6SpAOp&4)KP
z-MoDL^3{7+6K`uj+4Fqq;}dtk-VDF7;b!LD4G#}J^LZWlI_LS1$BmD!Kg@l&_aWDl
zpD*^l>-;4AY0I0wr}G|iJX-(6@kRLSZEs$`z5TB0wf$4Y$7)ZtpY4D8^y!orM?Y-;
zUC6-y=iBG(H+s(%o=to4{mqZh(|_0fv;ULzW9pZcAGJS7ybpZO`&ss9^<US&^M7pr
zr22lrJMIskKFNQ-_`8>30kav?^1sf%@_tPG_WaB0&t+ft{p00%E37Ykp2v=*=WpY$
zf8RyFwf>yWaGjZ-Y1+TWU##D^e!2DW!u#frSH7P4ss20WN7ZNd4?*uu-fw(=`2+js
zn(tkI7X7XM9r=6SpVr@MKmLDi`^56oo~4{ine!8K@!!-RG2fKGp8E3ahcm+@R!ep#
zwr$LtnVgu*80-J_|BGXEWEJD^W82J_!mydCh=qq$iERpdHRmTTPYyRG<NuQXJ^r5f
zsq)S1OYl#9W&@5i*69DuKa0K<emnbh;uoPGM*n9s`7;{-pZizr-#Z3pCUd40EcF~0
zxb|>8Wn0d$@lWR;>3<3g;Y=4<R&X5Q;pGWqd%{%CoW{(^G=(9GaUrL*n6$K<7&m_d
z=LS|cmS@Z{EOnehLf=In2u<Zx;Lv2YVz}^k+3&7DMhw@OnOMyjS%0;DN%-{j<IYcQ
zUlV_^GMKR}W@h|X^@HQ<>JL9(Pk;IMmHTG_24nW!EGd7jzec^=_VUtG#m7INO1#_l
z`O&9}_mOWNyfl2#^K|PY$tNu@H^1}xIQIkV+sn_FJ+i(3^ltXOB@dT9Gkksct?gUm
zSEA3D9wy%wyf*pr&KolyuYINeYW-8U2j6Z_y!r0>x9hv^Bt5Noll6Yy2iA{aA11x`
zdFS)y@at`F+dgjiy79Z@_X%Iszo>nw_<HG^^v}0{x)}~Kc`$EbTEXc2|JUz#KlOgU
zXRPPBBcdt6FYYF+F7SzOKkqM|JA5v}tP;Pa7E8NI8_1lI+pdtVxK(MBinMx*W~$~k
zwZF<g6^|=wt8l2tXkON4(QDJ+s{caQMq5hrgL<gCsm4CtwPv5Kw^`Yn+URL$x~S`@
zF{nvumg%_}1sgT%S8GeC+bFM9*e=hd@J2yaSw~G(jajKxCRUtZv_XtbVyc9j)H7*k
z=}?K4!peNJI22jtFeWqUaVm?fk&crIlgJSa=9<X5l4&iY5KB6zE5D#{mvE+F3tt@1
z9Ii_o(>We;iSwHZvxzJcSjpALJo%s0KT8H{=5uU^Ia7E%`1bQmXU}95{*(P<{dcB6
zlUbN~BKhLEC$P-@d;7<wFF!uUewO<A?ms^(Gskm|e0F^nS%ycyum8~hVf`!cp8-=e
z^BRVn-%>x0{t);T_UrpEpFc<bbugAQ8vZ-~Gvcf7r%j(czVH2ak@Y)gGS?rre#Yy6
zYJTc}SNgW|TlLSDzw;SN7zG%1{N4Q9{%70Q+)wjAb$@O8ZvBn-Q^A|g7ZERZy*%{F
z=(YUob#Dwmy!`O}&GhHhk5@lj{pkHu?$_cUi@q-SdiA6I+um2@FJHf?ev$p+_{;sT
zx!)wdy8fK|>5oUBA4)%5@WA+y+B3~pQ(ymjarg1XhfYreU$(vKd-eU*>NjiO&3hyJ
zZ1sJ^+k0=^zWMszpQq_>Hov!jm+@lWgYKKJuasYAxDs?V;)ea*TMrYStax(f@xRB#
zPvf8UJ^%Uo{HG7!*?->o%Jre}P4nCJ@3lVu{ru|tv_F>__c1duocZbZMg8NVkI7#X
ze@ih-b9M2o<IG@L_J8T0gx}@A3;*6{6k<Qkb%5tL4+r;e_99jZmWM1$IJo&lgd4>Q
z#Ttbc^S$Qn=DWgIz@IKSS!|N5sKPJl-=bXt>$xjAb+|PI_KADTJ1GBFS|o2GHB%%_
zppo|(cRG&;|61V{;uj<~iM<tS;M>K?!Pd?4kj0m+i2W8T2h*wlPyWyUzvQ3zKZ$>m
z3}GyOY-!9q|B`;Z`ke5o;FHMb)87*Q9%S0i62<JzaQC0!|Kz{Uzvlhq`{VKd0pnz*
zn+!7lmi|8SbHdNQ-xdEN8DB8ZWVymr%UHtX%(9hr1zSHm1NSw7a^ZS`%iPW!FWDM7
zs<?W%fAVRGoEH@r4&mFzv79B2`3s}{f4hI43=S*?92T7EY>do{7`HP%XWGRYz}d|c
z!nd49m}4$$1iLub8lFahUQs8R<H{`R=aiG>nkAQt=8ANRR)}{>rz#4oMyq(rpBMir
zR3h|3@D+cuz<H6|QrG3V6}QPAmAENlEA&Y)UDQGPy24TwN!2Ne8nX3L-4aX1UWmRI
zyCwBf;g0eJ1ukg~(bN1{JnOkyxOVf^icOZ*mcAoaCUBZ-HM=S6ET-v98`&IrcL<w^
zuN6JSAIQ<i^!$JL|40AVGp=IZ%T)NU>erJWH^1-tw(y(y4~akiOg5mcKTNCto%)&m
zb?&F6kH<bA`04on7-Kr);(up;YyWckuJuXt-LJQYKTi5S>v!hwh#z;pNPiOgsQgLh
z%ek+d-=}@o{=VlM*Y^$Iqkla7A^mg4FX4Z08D21){<rwI%g?T#n|@{f-uv$i%M=a?
zj&hd!|JMJy{WIf-^p_VOPJg`oRp+<f|A_yVf2RH5{9f^c<#*?wnST%c)BET0d)|-g
zA4h*B{L5pMXI{_xmeYi%n@fk?hFR|a>p%DY=>6NlB*V3sf3|=;Zw*@j<7|dThJ$~<
z|FZsV{ojN2Gy6AYqrbvGY`$*&!u0L*cf(&@f3$yFeARw0{^sGUd9RkdzW7$}<NU7z
zKl6T;e4Fy&*-MwF(;h8)82L2+?aR-0UoU_3doA#6!IPzrj^BNI>+{_MPwn34eBJg%
z|NXTW4<Da;*z{=blUvU<UvGJR^@Z(Ik;l^>i9B|Hoc7rDS;L#ZA1gjfemeC|`{kWS
zj`yPOZhkcDjpdhh-!^@F`gPZrj?Z!*8eZl-KJu{Tsoh(%FVf$9KF7cN^|I_m@ypQH
zz3);#t^d5|<HGlP?<L-wyc2uN`S$QT>rb`cYJRQxJNe&)-znd2d=mM*;rq;g!OW5@
zo0%3fod3U`A@u*gpUz+UKY4x0`|<UU!GEQHvwpq$_WA4b?}vVV{k8ko;~!SvLOwtJ
z5c}cBN69aNUlYEne|`Pc?)$E9exKN0vp<Qw|KzU3L%-)+U)jIZc(&wm;3M@%=O3`%
z6}u^a{l)c-w{0G9KiY6V;+Dj<vTKWPn%@z-ck$ktJ9anXFP%N}<)qK4*{4g+*q`M+
z-+gJtwf-CXucn`KIClS_;NjL2Ea!_apSbeoQuVn5rx;H4pENjLbL7q8700{IufDeT
zhViu*7xGU(J&}8|?evp#>6b#TIA77Z6nXyYx!{XOulV2Ky3_i=<hkv8@h_F1#NK2)
zee;m}@tqefA3y)pVi00J&g94>&FsL${%_yU>p#l>>}Q_A{Y$`K=pz3&o)0|Qe3tzD
zLW{(trEH{2rIIC;q?Bd$%gmDgD|JO?hT?AZex1p>>RRhnt|}~1s8YGCm2O~fe9Bne
zc)igzBWt7Y`nPm0Xie5>Ho9ctV{_BygjJCFWRr_VZw&7kJu^9M+HLa9@SnbmL7Abk
zVU)gv?rLpborAh_^(N|gt8Z24m3}T6FXO0KrCO{WtuCo{RaIR*NPV$NwBmF5Qwn*?
z57d~ng0ydG%Bp9mnyK2UJyfsONLRn1VybXQszO3TGEDN4_!hAc@fyiK8A<uKat6|G
zMg4@dg{F)ClDZ?`s8pqtt#C%RT>7%aNzq{82BCGru40Cg8PX4=3dEWOym>6S4sjmk
zQst@P5#rv=evf%AV-VwI#??$Z%uAUi*e-Fo^D*#hv)}$d|JSsiVt=w3RM@h4<_c&F
zmhgY#;o^G8md>oo!2F+;QJl4ab3b=2*LAiT%!`=VS<2a7b6nug<Fn!m<Tl|f;|$`m
z=bFzc!1a!|Ote&nU+%DEuh2>E{p=grzOYZ=-YgI)vPi^0D2negmovv#mQ;qOzsLR^
zU=HJQ<7X8(%Uj4jkCTxjls$+ehU)<LFRrT`rW`h0jyw(ApEx|(<2h1z9tgY_4CnLa
zc*n%f@c!TLe=iwVvOnW~#l3-Z5Bp`dzpQ-BAOF7pS@YBU&n<?2)^rXB&P>iI&RZPK
z9CtZxaB#7wFnj-h_-Fdx$p3u*_5ZH@6ZU@viymhc=N`6q%z=!Y|G)lYX4=Uv&v#BR
zT<{hjEAKX*3p}OVr#Xz+f3x>-UE^IMuugEMKp|fVj|}%eu6CZEytjEuIE&d_+3Yz3
zc$V^5@Z9Cq68s>-C=nvDT(nHEoJWI0p52@4sz8Xu2bpe}OA-%7x`iwSCHcj90=a&1
zdGh)QY!*%w(-GSzw2N;Yj}mVu|1H7Af@%C#yq~#?xu0;caIRq6!aAAlB8Lw5BJNnu
zIu`A}{$H28fBE+Ghw^U`zaISD@=f!z!iVW^-Cyg!Y<RxwndOWAH%C97{Br5@qW7w=
zy<XTq7kOd$YUS$#uU%gEygKyq=?nhnwoeouNk1}p^7Pq)7a}hYJb(T;_P)Sfp?g-3
z)L+=Xt$iQ#ZuhI%&%K^`KKXKA^|tg4*Bc&pG9Ix#*LiX3nfz1lC%#V<pQ=A&d$#q7
z<>QYJJ0DJdbm6hY6QL(zPhY*@de`@f`>W{J_n&k>n7&ha_u-@3Pp$vw8Jz#;|7HGH
z%^1(}l6g7f-~aRfdopUUuHm@Fv5{4PiHG6f|Dy~un65A{X7*xw!63^R#(a@gjNOxc
zBD*eUF?Rz`9?u-^D2_5FyT6}*|M|b3eIf59-n*Q!tj<ga7y|$2|5^B>^y}Nt8@_mb
zpZs(8&#>=IpEtf!f1C4u!6)0#ZXfI4?R(As>gY?(SKP1Xy<Pf&|Fi2?&F}g@D!*U-
z6!5C`(VF{ikJw(!d9(A4!plcb>>gP^?76?|w(N~x*W|AsyZ-f7$%7e>-#)l->(te>
zD<`k%-m<vebSvc6)tezVR$te>vGm5@8}T=n-duC*^PQvj3m&rF|9qY6;=%L(uk5+2
z^VIl7#|y@nEiaj09)4c_?D3POkJdhz`Y_>n+J__GM7~9S{QCCI8>6=q-|c<B{QZG<
z%5P&|d%m`O%k<&Tr=wqve);`%)vqFk9n5x2?ti|2<@~bzoAU4E|Kd!pOwA1X|8M+v
zVSLZ1%Bb>B^jFRg`QOrvIc$&EyqOajiWq#DUNL`WUe8?1@`?E>!;k;|EcZDrdHA>&
za8z?#<`(B?=4a<=VPC~G<-aw9Jc}sDe-0fsN5*}Bm;e3vFZBP0e+&QY`1$ik-5(3)
zhaC2tw^{EpOS7zDdC7W@-H>A+YdK@q-yeTIFqE@4v9D#H%l?uhfaf{?BY~~FmhAT!
zKK)JkpTly4%Zs0le;ZdS`%8|MJWKgJ_*{78xE#3l@bC&a2}_7f6OiV9&Hji(jkjB{
zP$-(eohyND3X1}39Ge(>E8A_Br!0{i8a(fKBDt(t8UA<wvii--q{{tKpipoV-*Fy(
zK3;*#{0e+ZJP$azIL~vR6l9iIEF~_nOh|-p4v#pWxWGexUS1Q<9JZ5eFF4oooZ$83
z)93dWv=C*Hx-0uoR!!n5e+bu3&JN!DLbpZNh-!;y3EvX_CK@7cE|w^i#~Z};p6e1{
zwlIt63c+dI7uhOVkFm*fn6p1;{_@}VukK&o|JjU&%)-nQ7%cu<GHI}Da^-X0VUhZO
z`?uI1lmCa96WHdmbpBuZYx9q*KUV(m{Vx9H)4LU~9=($KaOT^(U*f+6zd!$?_jThp
zr|<0FYChk3f8w?4EA_X(KR)|<^t=D}((hisnHjPe+y0mQcK#OkiS5(buk3%y88<Mm
zWLm=D!{Er|#p=plz~;<S!xGFP&&MsIBC$uJLF|*Tgz!ZXHHi$VK&g)s(&Cn4XT_VO
zIAqvlc%@fLiz(QvMyt<NlU2E{@LH}~zD!A0<A~k~LuUO&S_zt`v|Mz4XlrXuSDU5o
zsqLZv)ws{1*7BvPvthK}W8E12d;>21@7m`z^3@qNGqlZg_GmL`f6&%7xMz}YmS8%^
z=!mYEMu<w6N}ooru8e-UHm}+PrE`kM6(r@ZOI#3A;IHF*#=li)t7xm(Md6)%XE;SU
z61cALDDV_<eqcModWOxE;~aZB8z<|3=E*E)*uHYS<9y7a&wA^B`|l;c!v5MZ3Nh<5
zn*6o?b@1n#Un;-n{t*7E{Q2Og-p?K1!hcWwm;6`hXT(?OFXCU9e%<m}`h&rn6E9kx
zC%i~{+52+Wi^tEyUU0oU@}l{L#0&GM$`9=CSloGZ-{GmqOUIXYp4+`>dX@WD`9s3T
z&JRo8*}uL2*6pLxw;4Yl{@nJX^+)5+y5ArFUi=sKC;xlS=ZBy2zuf#9@qNROmY+|5
z_5O+ad+G1ezXyL6e--%X^<m@ZFF$?%8!(*yTm5_LFWKL#f4~2^=9|kGt<N()H-G#3
zOYGmQzrTJqey#h&`YHR9_{X1b*S@^|<l@7>51%~_dn)#{{R!K%6)!wqCBJTaBmCyq
zi?}D&4>cbrzT)`E{b~PunRiZa|Gv?GQ}Dv(arXVhJN&nn-r9ec^+Ejo;ya&jG+e8^
zrhj9>&BB|{u6bU5ebMRinkx^l_+D|ka_nmF^<_8q-THX<^*y0G>Nh;D9lbu`PT0c(
z56ka`-tM~PbLZh*nLE}uXWmf1sd-E5uF)fx=iaZ*y?pyD=4saRudi;sIrOIdb=#}!
zuV=nn^+D|u<LCHKnjhbNVE&}_rTH7ncgru^-?P1Cd_U*wpWkYKFaJ9Flm9pWzuyd9
zjF%XWGdMAb{%2v}X36K+!YRRS#vIIej)|Aehr@*90qX>&ZieX$R~Y-5PqTbsiDXG(
z_F?|bI)!T=&r#0FEUy^rnT0t%aW`=bbFi_BF<)X@%~Ha4nN5eyfF*~Kk0FGSk>wD}
z7RF6~oPQep6#w1wr~1#2U)Dbsem(q!<=gQe8-54;Rs5^@XWyUs|Ff8km}343{S5s2
z>&wm`QvaSYm@;_(Q~CS<@67)N|0n+|_$U2$`mgL?9RKDsrL*)hmH(gn_upSrh7(L&
zEH{|K{<r;p@w4*x>VNEvhZ$Eh{${+u{F!Y9rw`YE_Bdva|EmA^m>#l!;VR$?=cs4<
z$+nG?i{~%THy%@-2|Nw_f+Bp9A7$KSwWOwtaftpEjh9#|xl??eaHPOG{#Sw;VoxQx
zr5{LcmAEO%Aj_i=qZptNA!{OKB`G2ktthV=sJc{XlDxCLucDyRQw2%+Z!!(iiZXKY
z(JE^+UA0eZ>{ZQGnW0jzx>nU&*-WlXf?0H`Xu5>3)H%tW5<TLM;<F?cNaaZ1lYAhi
zB;q1`N;FLBvCJjuCW(5H!-B?wN`meJ4SegkH8{mM|8v_2@CmmI1@pUcUtnL%_MLqf
zhY{Nx#zlXR{yy|)-~TcuJLc1jyZ#sb|IaX)=@FAA<Nv>IzqkLY`FoV<0-FfC0L!}n
z-G8k9*!}JJTmHx8m&1?T9}T~K89G@$vZ}BIGfFb>GAc0NVG(3K$#m-9#6PS42{QMv
zt!Fo2OJVV1UBZ5!qk(fCXCV7NCa(Wp|9>;}vrT4~XOm|>%}~u~%<_?SHES!&bLLkp
zZ`uEHd+?VCIP<UJ5#c(|F_ANro0De-r#njp!&^oPwqlMi?ES1g%t|c!Y_aU~*d^JQ
zv7To>&&bGV%`%VcA)h(lF)mpS84d+59c~{^UN$)vX%-jOT6TV}A6y$a=dmwhO<{S>
z;>xy(bq8bB?`7Xwzg_vc`_KMA4}b3c{_5x5zsvqlW3XZhXX^cL_1oY_(a+ug_A^al
zWdB?8v;L>=AM<~!|ET}W`<DE5<JV8$_WrQ>we{D<pLf1<eK-7B^r!W&<!{k%Eg#Q*
zoc}fI$BG{g-y^;neLeo|;`g8*96t?y{P}$Q?Sq$zuYZ5&{<`PuhL3u0-o7k+v-jQW
zH}0>3UM0W2^Sb)2%}3R*Ro^v!<a|H&E$YYJUxL3Meee6^_3qX?i_bRStA5V-W%YZ}
zpR4~4FzjL4#!}6q$*A+k{RjWgynhFo*qFrs<^7WRS@kREPw}6rzg>RU{r>QK_n+#2
z4;hk}ZZOXK@AlX6PstyhKNUZtzTErJ^(p_`h3~t+m3%4r*!N!Mebu{!ce~ynd$01Q
z;aT*fmybkVc)Trm7x~)vnb70f$9YdKJ}P=(et+jZq5HWHK0oYxtoBsz>4isM@2|Zl
zey{h=jGOvb|6Y*3&~nM<s^2xatFJD_UCO;IbH(y%<Mp;1tFP5u@;!IzT-eoFx3lk5
z-MnyB_sW%PlWyI;b>QaoTZ?b2-u`=Q!yT9Vxet##`ts=SL&b;BA4)ypf2#gC_u-Al
z_AeFRrF=a4>Bpx9U;cl;`^)xU0&_ci7ROq)tIR8zw{Y0-z2lp~?Zqz2+Ri?S+nGm$
z`!n}K-eA5azC`{uAwCH=8BG~GDN88^nftO?a``fwCB>z@WberrEACagt8_@oRHZ^~
zvxbEBJe{xF4jQgXZu0!fvo*Rj&D7>8-jM55*sq+eoTiwr*r=E%FDlz0<1Y7Hu~x-d
zSxbRM-b|rWQB3iY>~{%Uv0GvR67MATNW2rz5dST<Q?x}?M<QS9n1qH%6~8B6r%<NE
zUMW4vm!dVoP9haz0%FUA*@d17X!95FGV=!U`-x1II4m(kbght^P^qw(up_?{7Z1l?
zj%%C>+;Y5kcxUq@a|d&~@qFan&9{T+BYP^-jsIy(itG$r%3LNKDQvUYgSb|6P3Bz4
znZqf=?#nWn`7!HePF5Z#?oJL#wocX*w)JdhSfm;M{$2I=<Nw!8=Bznv8`(<O0@*Ct
z4LNskDRXUNJHz<t{{dzxP95%Ou0oDrwtcJz**e*;u`_UZvae<R#$3#Nn$@2xjrTRr
zdd@AZ$;`W$Ls+tyX8#lZv*XW(|FX;hY#%u6xWu{Mau#wY@D&T3<oDp&%x=NDgw2pk
zh}WB!olBani`9sI0{bR*CQe&UQFdt-Wu{|HENoRA!tBeKLjQ&QN&PePPv>u=pR2yD
z`ug=-{?Cfv%74E6I{PE?d))VgAM=0y`+4en?&rTBazEetF8izZ=l^dHKcD=>{iX0L
z%lAV+cmMwItM!NQcfKEof64rv|M%1HD?b)}d-`qW_s?G?J}r7T>+O?wb|2?|cKo{g
z>x8fCzlwji_%-S8)PG!mO@3?rF8TYAVL9^xmI&q^hLism{lCe`$TETT3oAG4EapC@
z80Lj+I$Xir+FTnsv^WlN+~mB(Y0jR>+{5U~#K987R>V=lq0PRZ{W;eSzBhumh2{#*
z<`3iB%J)_vTll7Mq2P4hYg`sw%ejKM^>|iu@8USgI*qlO{Ss#-&lbLNzE9j|I2|}m
zIqf)3vzYxi`z`hR#GeCym;GJ*=l<``KiB{M{P+03)c+U1oqjz0D*9Fa+uQHeKhnRK
zd|mzd{pS_m+<rFw?EK#RW#vci54Yc+`@r*g^;ffRJ3e=P*!%A4+wZThJa2e3?|%J*
zd5<KX)IO<ueCyG?C*seGU$ngZ@p9)2(dXjNmORyce&<#9+plldzDj*r^Xl)L&+qEq
zA9{EFP4z3Immgnle6#d}=a>6mt-t>K9P!!q^SMtEA1=QxerfT>;^Uz&+rQ2D{^7g)
z&!xYde)s);`@8mc@Gs||%YKIa^<{2g|IY5twv%NZiwoN|4kqr!+{s)C9D3}N*ef{~
z@bC%r3hD^-@JaA13b698;a<!h#ahlbk28_y6R#iNGu}76Dts4sOL!G{A9IUxb#g3b
zU%;`8JDl$lZxoj^+Xdzs%#|!(SV~y5nWr=U{crgH+J9}vB*rcOFa9b0mHl)2kJH}{
zeLna8@|(?XT;3JFXa4ZweZhO74@n=RKk9t=`A*}l-K!hVl%6y^)_>~!Jn#AECvgvF
z+~4wm`|<TB=}%uj33;;O$%dz=pZ$3*|1$8U;ET*>)1J<Iy7OtmlL-%&+&O<+?(XNi
zT=$ONt-h;uZ|eP)2U!m{J-YsQ^ON=`3m#2>_~miq^L;Nuo*O)6dAi`m+;<(HCx5;5
zRrK5LZ(-kGe)s#?{QJS5*gpcl(|#TNHQ~4F@AMx{Uj@G|{r3BZ!*7;9L4UaaPi0DC
zc4C%dt!3}wVB)yV8ph(tR>V1vvzbkgxr`}<bt|VkPcqMJ?s;5ooaZ>2IG%Ex;A-Hl
z;XfweE_6<KrbwIcRQ_*Vww#t+#oS3eTD&!U69jjOh>E(3$ctu)>58R^JQsW+U?nUn
zwnQ{f=nCHzo+LgIp%UR_p+2Dn!tBC2LjQ$kiR6pTm$)l=Q&LrGqtr>sPh#7Jy7)|a
z=5U|ne#x_+uRy>=C`mX{s7)Y1piMxEUz5j~V>0UlHYqMEE+uv=ma|NMnF?8wSoK(o
zSpTuMvf8ljW((jH<=MpNC-6)lK`>CzQlOSkhKGeSgl!MYW#$*mHmqma?y-w<D6-FC
z%j7u8wUo=3Ly>JV>jyS<PE{@eP6qb<tc7gO9D<w?9Nz4aZ0)S=tetG{*w=6_<i5vq
zgU5|0je82`PqzCkY0PUGJs7q!q%*x`zQH2Rn#xkje3yAIYb(15M?L!zwo7btIBs&4
zaQAS=vN^FFXL-n`&+(F-ft{a?fmMRVgoT;af>oFGGTS?jb`F2GJuE@Y;*1^t4*qWZ
zdF3bjADh3@f0O=X|7!pK`|GN2IX|xd{PUCJ=Yk(QexCYu`{(s<e?Q57xc^@0)8j91
zzg_;W_2c7@l|S`=CjNNx<MA)`f2$Z3n2$02Wm?M|%>0t^@qey=TK_T`6qwhs_OtI~
z*JNMM{*L1n=U=XV?ipO?Ior4m_~iMm_<VWhaxLP%$~Rfyp1?ez98qU+bBTV5ITAZ1
zgJt&02PofH?NDP-6Hq;(d|UCZ+*RpdX)jp|`5g+|lvq?ID1T6zr@U6>xJr|<vEonp
z(~7k!+f*`?A1WCs-%)8-PtmASKdSy#W3fh|x~tk{m3hip$_^@Vs%olc%07zc<zr>T
zq;e&ui1UfB5O<gAk@b)dkiReYN3KnwPRUuRUVg7kp!6E4Q<7W~Y+_49ctz4hSVTVw
zZxFmM@I&AZ{}NtHZf;IT4l_<xZadxx{!ad1Jg%IoY(~t}|AP)})B0WeSDQhLf%o6F
zKmC6M|1A7{;MeURtl$5Bb^a>$W&6jE?+(0u^0x79`WuzkL9c3G3BSJhO7j)ptHUo#
zUi^7J>(z~S8Xr48u)JIUYX0+WPxd~DxZ8er!2_`;ZqEvzsXu-HIPvl8#|}@ko<4li
z_N@Qq-B;INg}&y0lki684d3gzFI8TidwKm;*qaCMQa(QU=>Orx+srqM-*SDN{#oOT
z+Lxc7AAQpMaQ8Le3-9MTFO}acdiVXE$h!~kxIVu6DE~?S^Vu(o-@bkm`TpYD-)|=0
z7kqR1%Jb#LC&`Z&-nV_I{?zkD`J3>!pP!jOzIx~Ue$U54pWVMq`m*WE+Rv9hD8Kpk
zoZ<QD=i$$JpCvr~`K0$r!;{%hZa+Ev^wG05&-|V?KS_F0^Em8L;-iO;N*?WcV13W>
zPTB3Vw<GU<yZ8U@m0P(tVy?Sge|DqtcG}&8_e}5C-}Sz=_xjCieAi2^cVD-@KIht_
ztA$s$Ugfzif79!>(%t-fy!Tav6Ghd<?upJ7Ef<rNV3WKrp(4R4ep56}^q1%su^KTB
zQG4M!q5DE^!ra2jLSX{@d{#UO+|#%-cx-umxTbOJ<Cw*{n6sVpA7>U9Ki6Z9X!bg`
zc(yFoD@?u&C;oddwlD{?TCiEMwX%k>gfpiwuVwLO`^z?!Er3mjy@A7(^EXEty9%oo
zOBkyYJ3HrNE(ac0o-N#c+$*_-d7ODC^C|Hc^V<l73z`V23e^eh;$6>uoqGjuEdL39
zUH&V)F5HjVomdN4Ua-Wo%Cl;-#;_Z6FXX+%!^dU8p2&KL#hi5(+f|O!T-@AeIV(6e
zu!pjLU^~xh#8S!Zz;d6pf!&?Mmff878gm}=HfA+eclJgO9ggR0UaXodiY!7b`<Tob
zqZkF4Sy>y{HnZ(uv14@nxBSo3-{ODP{$2XNn8A(V)Bl<OC;dPEzvthK-^YHw`W5=O
z=>HPN-OMvtoLD@WW0@8(J!ckSk!Cvg|MkD3|K$vi7(^M~|2y~Z{{MxHXP9@eMzJNc
z?qKd@>So%_T*i8uO_`mEeLA}}=Qpm~-0!&qx!-aIa-3pc%+bdg%4x!Jm+dU;IaYo4
zO&pAzK^)2K_u2S3q_|J;cJn>v3lcaZSSzF{{7;x)L__$3;BtZE0-Zwlh4%^R3&`^o
z@J!(H=QQPP;B@Bv&c2WJ2lI7iUzRv#DaJ4V9{&ye_x=BL#uJPg48{Lu{<-kW?Ptyp
z<{vTN`@h+IOaHd-+wyPfUtfHj|Bm<V{WlhGCEoGAfATKkZN%%fFB@L4zR-DL{bKs_
z!_Tffd-?pwi_<R-KAZXY$-|<@ub=&Yx$I5k`(GcPeQ^39@WJv!)`!3k_ungjkpIN>
z)%g38?~A_)e|_`0=ktrt-@f>Llljj5J^I_Oudlx{f4}hE`TNSRR$p4a-2JBUQ~qb#
z_jO;HzjS_D_{sOn;jhZyu6%X-n)3DPx8*--fA;)5^>gX3)IT!+d71i|KQm2be9Tb7
z@Z!G-LmcB(Mk~g(3@;cWnQpVhvpKRJW%|OP#^Au9!N|j;%hbxq$I$a{^1pxoA28T6
z9QtSVcg`RGzxDtA{G0#p=HKkUxBgE0SMsmxZ^WOrUwS{5e7pP2|3~r9CqE{C<NR{-
zlfvicpMAdU|NP*i>U*KLTi^J+O@G_*=J2bk7tfxBKIeb=>E)Ievz{_M5_)j;LExh~
zk2XBo@p$SJy(i(1H$Iy4u>bztI}>l(-I#lm`Hu75cXuY={&~~<=H;7Sw;AsS->bU!
z@~+;!fA=>%GJP8OtorGW$FCm!c^v<&{Ds`h^Dj=m(0aN0WzVZ8uXW#kee3>i#=GeE
z)*qZcu)pVg%kx_9mHX@BcmF@ieewLV@$<gV`d^g4uza!nvgNbxr!^l}eU|-x|EJrp
zhu@>W{QvmjgV~3Z@1MN?@jmB$;kzksH@`jmZr=Ol?|D94_&D`T{P&2TOurueeD?Fq
zuSb98{-4WO$aINm4byi<L8d9pM_Ar7=P^F{XY-GTVL6ix%QKb_Eccn!ne-Wx7%iB-
zG5upoW8!6MW#VTEWBb9*#1Y6I%07u>0@oYv67CqTaIPy{i@B1yvbhd&o@HlYUBEP#
z$(Xf<osqqU<rU*X1|7!zOr|UytlQXl*zd5}vA1y?<*MXe&KJY`llu$LY5qe(+e9Ko
zKZu?bzaiNrl_}LMl`2&&nIs`4!6>OG#Uo`TF-LT}aF?)*sH(V~!~^jvv070fF*$KD
z@y}vQ#H+;@iJFP9idKtXmHZ_sCf*>tQP5HFhX9}8OTm2Mc+o#%lf(~;e-W=2w-Hko
z4HPvImy`;Xc`Z{V<0ca<yIKCMqOww};(WzArC{Y$6(=<vjbj?C)&Hwjs~%O8*VNa3
zto={RRkKUIUM*K`lA4y<U)6tV#u`gB{%GWA&e332yQ@^J5H4>be^|~;c7t@d^jhg2
z>7!B+lEUIiB7Q>q1YZcbie`(y5bqXq7kMP8FQ6;XFK8`vK!BTXC$}D#7^fRYH~S2>
za@GwjuUQ_jhOuR_&R~Ac$jf+}VHSfZL(~67|1AFo{W<if^zW}f4!;?He)%Tw?fAEu
zKVJRf_<P~+??23c%>N|)UGy*dKhOW#zxlss{QC0S{9o4pi2oA*-Tz<xuf(wVzr+8f
z|JfM|{x|<S^Y`YT)xXPs$^K0Ik@_?J_v=3^|DO8W`1i_R$N%n(`OKB9AK13A|7KU<
zP~|M={=k#Vt;w;3^)X8qYc5+F+f&wWtZZxrtU9a<Ss$=|VLi%bz_E-|kb4`?LcWLm
z-TX)Riuq#%9|_3|w+ST*r3lB1aELGq^9$JunDCl$)o~o>xX0zeGmGmx+e7AGjLeJ|
z7~U|jGo~`0XI#nnk>St(zyH4e&HbbPE9pn?kK;dGezpF5{9XLphcA~u-}pH5{l&Ky
zZ*IK`c{TBs`0MW13a>3+_CA~Q<noiy=kH(2zp{Se@~r!*#WUgOH=i{;^LlRmBL9Wx
z%V)1nzIpss@*T(9`q%2O_q~4gn(<A=n~QJc-lo32@uv2*)~ouLF)uw|R=+&;>i3(l
zcl+L|ztMj^<MsD9Pv0uOb$C7PWyp*3&$C{LzP$ay_XXDr=NH;9Jzsr&<^DSJ^}|;?
zUha7@@dfKkpO^n$=)S!E>cX3jw^!drykmQ}?oIvcs@G@UbiONpum3*eo%uVj_wzpJ
zeT@1b`R>YVme>8S_r4K%=k@;f`}X%y??pfS_;Bdsnoo+K-+wy(>HR13&)J_1Kc{~(
z|7iB%%lo7cIv?MBeEae1hnV+I-!6IU{4VX?!MCDs9=_cB^80J`w_2|gpPN6~`bg~&
z$D{T~3Xl08TRyt`VEKK<`&IY0-JN@<<hJWA-J2V3YTV(!Cw?#KuGXD*x9;C+y}kLi
z-|Yjp3U0;Tinz7$7TfK@TjDnrZd|{y@b;H`+z$;Op1;q3f7S!V$8VoVJrj9u_Cotr
z^&6&l58thRfA;;ecfa3Czbk$}>tpffv@g%UJpN+vW!dLjpUu9oe+l}$>XYH;y<f|J
z2K+JpYxt+;SN*R=e{BEp{rm7|{%_;oI)AqP{q%40|4ILU{G0KY@9*z_EsUm2=8P`?
zd;XgKo%*-#-zSDG%)Kl|EazC**bj2<<6g@%k%yUQ9#1yEnXrjinRu;Ow8(#9O|ehn
zrQ-WV8ARR+X$#L5IWF2PdRrtw^s{KNXo$!f5pA)TVr`<5!sUW51@%RGMB7FBgbxdM
z2(J{*6=f6OB=%h7hmfw2n~;{!2|<2A(2@Ved<DGrJU%?^Ji?q8SfZJCus-0J%H_#@
zmAjT_GH(n2R>3|YX(2@+f1%HU?7~7~1`_2GuO*I4UX^?;xl8J()FsIbi7YWO(R5K)
zu~%X-VxNUS2|5Vv6e$;v5nm|kE}ANmBXmnJMJPvjs&Iy2rU08zig1ZYyU2N=-GVLx
z2YL7NUf};CxK3D5m{aJzkbv-gp*oQyvAN=>#Se*Zl&qGSB$qBPDJLcOP{BwkSn04b
zgKCS4jnYDeH;SRki<H?^SXF{mebgn@+EuL8zG}SJs8PvRtWr@{f2;OE{g{TIW|$_o
zdYXEcCcnCyn!Bop++)=s`+A#7p>Kbf#PqZ;%jR>ru<HqKRk*FF!_)G=lK+Ohm|VHE
zpX7VivtMp<+|WLyGnen(w+Y`WIXRTBYIKNl{J-_p>g~-(Ht!FM^l9b@?)oJ8Ctq*@
zo973OHyggs=St!^^ljVEg(6L?lWxdgm-$o4JLPZnE7v!@-;zK4e^JlisbDUV@U{HO
z;%7$hCox>%Y2#YYR{!_b%l-Gv9;LiF^V8z@vNu~_-C|lQ8Ond>dBwT(yP;nkzrOfb
z^x@OP+jl2@asRdb;hvlOUnzWl^<Dkrg|~6v8Ge|*nEc%0o8s5J*PA{r`g8ra%=dzK
zA08fl&izB@oAH-ffAl}!eiZX;=jY)67k}QmH}CSwhvz@}@$FQ%l{aPI`?d7>s}};l
zdYA(KP5nOW%d9^IJYK>ZxzZR;{Py^K@U7WT0fwYEZPzB;aD1l!Ve*f@A1p5e?tZ!}
zaI5?Jok!>27yn^kfApL6`I@H@A9wtm@niWbu7?({lfN>)yYeLC8RL_PYoeDmo{Rre
z{lDuK)Aa|J8E+kXp#EmopXVIeEDdk9Z-w3CdT0D@`Q3oK`rkO1AN=Y5Tk&JbGmmGj
zAN$__diDFwn=k5&>i+{iANi*CC;I#AUvK|~e6{;n^WEfE%D+mM-+#M5<@}NpS|eG@
zX7Rm)sgpmKVaLk_pOy<5DK-gN{w#kU^ql+a)xS5sS-+nDvhjn)&wH#~yxab=zMt{;
zGPeSYDeD@6YrOevtC$zER*7ttD(8O2&MTP9|B<(h>-nFie?eTASj(BS1aC_05Nze#
z$7aK_{Ga*%KV0o%%SEoSU16BZWy90@JL<#d@4>8G9CQDve!uxUgLTP2ljmH|_cF>d
zUVD1^j@JD(4~$>=y;Xkm>3hPjzfa#>{d(Qyq4cvgkGk%>y2*3r_5<c;6Q6P1@;STh
zvd0b6a{(7G-IIMFc-!w{*~tr6LLVHsQ+eynwMplLuL#~{xc&Hw+~uFwvhJL?*L}D7
zhTF9-H_WfaT-AHh@ZFAK^LP74-!Ff;=yA3D5$ng|Pw!qyKU{ua^3~k8jj!3?Eqw0u
zWbK##KNi1U_uk=e^}qMO)_qWUX!c~nx0OG?zGS#(_4w_V1B?p4uDtsAv6THg^Pg8<
zkDtB^{VB<0&&I-3%#y-g#k}LonGa^)yP1w~?d1B-zLLrLr_i@3h8;Yb>`T9$`lZPm
z!}Ef{kmZ3$s<OUp3HKZBECGM^FTAx1w<No{4Mo_Lvz419qIi#S8gW^1DsXEGri!LY
z*vq9UvC6lKCUJ(dr17iEBuefQUN4oZyh2`FM1)6}XPU?_xh1l_qH1Ez;*mnZA}6Ky
zNv;;2D0Wxov4}I{qCZ|d+LBsQx5Q&5bft{NCkTAxKFRrj<0<pbe}CEC1d}+nvL%R3
zl#1g&$t3aJ>U%oJ7x5U0QXZv$8~-S9C~$85pUL)5-~)@m&-Z`i{y$@m<#yz9;?v{z
z`)~4>mA#rNk}Xzf5r@;?^Gp}nquCgkG8ju(8UO$Iwu0drk2K#G?ih|f<~l|;_SpiO
zd?8G4|Nmr(VOQt;$y+I?$1BUVkXMGgi8o&Ey22)rZ$gz~i)G&`2gt5de4;BN|CrBH
z<gFyTs=I!VN}}97<@bs|l+85{X>YP{wDHvNm-SF*HQ8xyW0Yku#gIjFzx-;ctD>8v
zAE{iDKP+k>;LFD+utK0u#8fs|(uKcV<c*Yt&^4}Q{5GOZqI{y$gv7ZQv$All7jso`
zme|MK^lLx!QvUmV$Cw}g<6?>7P~o`GIPFK>_nF_c-tYXz%`-#LhIPjGpf7ri+<Y6k
zbQvXn9RFzXdjii_{&<cb|Id9B{Sflz&nuzNIShAL8vlBH`1O4E>w_OfzOQGT&Dj3Q
z@xzQ?Q~pT&lV<ArbM(ige@_2Oz8-t?=J~4k`u~D|t$VKWc-!k+?^Hf={9O3=@SpcT
zj6VE+q5gg;Lj%`gM(J-wpUXbPeR5|u;aA~!_2=yO)xTD<ba9rm?qr)LEUs`>HkEfb
z!!nizzS(>{T+O_RqWV&&rKgCU=HcS~D%q=gU*(&G1Wzyf7v2Lhk?P%wy+U$aiab;J
zm-Fo4Naubf=qGkUD2Ba^!Gqa=Z7!4IuUlUdei||!V`*Zn|8Dhl9fJ*X)%QhTc>f(`
z<Kj8MIq&bMFOolk|HUz>v0Y(J|0n-h>c#x0_HU1T-1H&lligSEpL70d{#f`<;H}HM
zv+v_ROn%w*BKX^fzuMnzK6!kZ`D6K?Z@*7|wfG?Oe)Y#$-~Rp1{R=t{*!cI2Unak%
z|G4&T#;;e5x0zEI*8Y3)&-QQZe<98tyv}UJ|IYpGV7SK&y5>lY_3uBCUm|}57&kD?
z`#XunL1>xKJJ#HP_x@b@;qaq|@ges=zTNzB{E2M){;dD6#pcZ>#gg}b+pkZ5otcyV
zXMdjm^4)8*A9YN!?C&^=*=!i9e+&GW`^EPsFNeE;C-)xKO-!5r2!3b$F^lm8*LpU`
zzmtBS{H6Y*=2yg@v|oRJ$^6RyyzHaJ53ZjU?@XUqJhgwj=7Z`>&3o5w+T1>Scg+Kx
zXJv0W-}pUzf5-B!=e70c@1JEmYjtJi-Mcqj&WfKXIC1xM?y2Gfefz7={kVShqR@$w
z18;Wa?)`jZ&heY4K3@89DgM;H<3*?BPi7z9a3KAJ`^7(JYfr_UWjeumDE-9KGY)4O
z&IO*$IkV{8=aWij9UokMGU2-6C7X*TrxqOAzi;v3Md#A41YcitC+6Xgr~956-JN@W
z=gDp7k6vGPW%=2L3+dN(Ts?oK?B@AfV%PTEdiONyf&CS(TQ6U{{Mzte@VD(p>(7tA
z=6<;K#`^=)pK^{H9HK08j5*({-nhO#{4VtCdxle-7x}%VTom6(Jmm~w4d5vi{VCI~
z5F{rovtH(o^hBxslD49rf**x#B=$?*6+bMLF3c;_DCa1pC7mSgC!#8_OZb<Rp7H}#
zV|j7Wt$enEtEF=l&14iLpD0wTO_M(&c2Y!7^o+<7!O2|CoPL70M05qV@$n1V3n=q{
z6!^e@o^w8%Joj>eEI~!#nKDP!3RP^ym+|c7Eatv1@I_Q!Sea)fmkD<bM;FIEuD|SQ
z%wo*Z9Mc5Z1h+7L{%Odxkz*_8aULT+1)hamGMsDJqj+A5nMg?rUF6xtlf->SaI(~M
zp$Kke(X&zyMURWz=T~E0$SlUWl>eJ}t(3Y1vv{T8TAmq#CgS^~{>n|4Stq6=;w|JO
zUZONj`ILf|)+dwcM!6cDik^zy${$tw)k}3J8a~#Ut3Frfy5VF)X46zNeZ%jDVP*%6
z^tHLv7O1|^Jfio+DBJXvnYr~Q8*8)cx|(X`3XkMV6s1(twAN`oQL|N9r?yMSM02M6
zJ2_XGJtEB#Gi4Yhr%M~jnaTtx993E_yGp`X*jr$Qh`NNCs0ZH@j$fP{yz_ZXIrp>G
zac>k_E?O?7F1?pOm!p>}k*l1uiQ_n{7}tG~0GYqiD}}zWGq6^(?qO>G8~JO=KW|P>
z{w|KI|3Ca;`kD7@KWhpT@8_Z~kw2<GY5fRei2MERXTrZ<zpFpDzD@n|=->T6>wZXo
z*L=V9`Q68>AMbmc|Ml_vyjRa(&w2m&t?@Je7t7yAy)=H5_O$A?=J$mE{XeolZg{8j
zn)@C1NA9<K-z<Cg>y__wffprjL_W-Tr}yT-8_5q&-_n0c{!aKa@y~-l><r8Qv;DsF
zQ}JIon+0DVpEU;uvj)=%M$7+}f75;)`|R_6^}E2gM(-QHgnf1VlJQmZ%exPDACf=X
zd=~h&@%!d4Pd=D_ZTq|8PtDh3pKX3L{!RaP_~*ZG@juIdW`0@x?!(KT7YT3EKC6G8
z_4e9x!58wM+5a#9E&7h-*{LVOPwzgN@;K(n(HFm7oqK-Zaor>F#~DvfK6~_%``w98
zmp=V?z2UL#{owmD4=Nw@Jq>xW@<HH()VFOPzP-qJl6v>?Ewksp-+g=K{j%_x)g#}R
zvp)8|6@97tcK6419~<7<J-2--`D(?x6>pTDNj$iG_tIUJyJ`31pDI4*dwS@}*+)VT
zq#n9GaJkp;$m`kXXVczU|M>W0>(}g0;_o{@e*T*M<=f}$-#7eF_;vb!!M|@`ZhhGC
zx%8Xtuatk=|EB)l^f&bX!`}`+&U~}{q5ZFm@$=txzyJOH&Je}S!gS^D<lpf>3x1yX
zx07MQ|Dz0xe(ifd``yy_EU(*N7`{k)dEsr|o5hdU-+6jt+jZ}oo;O!qlf1-u`oYnF
z6CW?$ylwJe@1yfiBpyt<{_}FcwcU3%Jyf_CckR?=&1=b5rOq8WseL~2D(|)3SKePT
zx_tbG{{80<-5)Hub^Ti7O~<>HcOCAE+<*LN;>$Vjx4k|6%<ReBr@hY^UQKvm@J#+K
z|F_mJ#&5SgJM*;i-I33MZ#!PHys>(7{O!5-8E@ae%l#_&bMe2otjC!z{E7VM&nCc~
z%JqTu9fJ+iG4?+8jqH{DCxxB}bPMqd%;gOi<`7KgIU+t!ij#W-=RUEw(h1Vv!~=yz
z#FC^UBz_8K^B>^V6f_e~7pax}Eww;ohd`q6ZqYQUG_~h?<+{BZcJkjuc8Vm5Efsnv
z@?NSzI$WkvrcLasz<Yitk+sqbWgp8}D$miX&}q?~savbLMOj!rPWGhyM5XKU-cp59
zA7nJ7UP|(F9B2E)YRA5k>mjcr|2DqMJZ9YUxG(Y;@l4{J#-`5F!xYM>&rr%RpXn2e
z6We~aTDEMq7i=<YQOt>qj~GLk3s_iKlUYibKQljK`NCq$`h``7?GNi>)-5bXtaTjX
zJfXbJJnGyE+$(tt1X2W!^C|G{;GN8KjXRr1l8;HCQ1G|VS`i0Pf6*2(LvasL8R2|k
zc2PMoKe1aP2|||y`uLrAUAeWnR&mT=D`m-Lie)NbE#P>~F2_2HNrLe)!)8V)raHzN
zhUEXp{xSbw^nVLO1=A(w-^^9aElj@{H2$Ce<MOlRYxL(YpO$@o_$BM>jV}UUd_Pxx
zD*9yhx#jbpPYR!Ae(e3!_4VkF-#>YO27Pz@CiFf3XWsA7KR5rJ{B!lU-0#)DxBq$d
z$M4Ub-^+go{g(bM{QK4~$6t;=`G3@ZkN7d`r_Rsy-}Zmr{gLIP<Hx>_z8{l6Jb(A-
z?c;anKTi8%_I1i<y^nqGJ>PSFQ2)69qu$5E?;pOadLQvo^E3CConN@V>3z@qKJWYO
z@9(~~ekuMq^BwzJ=eH*B!rpbfm49>S74PfkZ@#{bdh7g#@3rKs)vqqUx&LnN`+e^w
zzd80Q`E|s*o{y5BCw<cRsPSRd`>pSfe6aZR{gcFJhR+{AMSp(sS@-k54{q<KzP|Rd
z{8i1HKX1HWzkFf!-0AtP7v--ezFz%C`0biEliu$5@bvS+FT$UXzhD2R{Pn5V>Tl-0
z?tPi_?Db>D$0d&+KH2$9|Ap+!s+WE*8(((5+Wad1#gQi*kCY#BJ(~2G>&czRT2FR7
ziFx|^+1(dgUnaeL_~P-4f>(_1_I`}`ocihS`vY(D-paj~`*{8%`^S0jBi}W?O?=z=
zF5yG>#|Iw+KP~&b?d!ttLO-g%@qbDA6#hx-%aU&_KNfti{?_re{A=_#`|t7JEx+e{
zxA}4Jmmfna+ZN7UoK_sO*cP+CWxdRHlud-Skm>3Fo&UHP6qygPzGhQl?_`(aoXtIl
zH;Zo)A3xtl9yuOiz7)X*;R@juf?E7_yxhDwJf&O{*w3;2WV*w&gT;b<3Wq<30J}MB
zCbJ0hRu)Y*Ue-N~+5Z^+e*4SCkjhxk_>JKrgEG@1mI-WD?B(p*?2A}+SZ=Zyup4li
za4ujAWwvDMW<Jc~%yN+FFQYp1Rn~PJcHGN&_wwcO%5m#)m2=1N-4a+QaGUou*JI8O
zZaY3lfe!-z1gZqj3fYS+5;-NDDKtaChEIxDj^AE*l9;dfCeh=<X~M!HKZGlUiUjWP
zS@P}SlNVScI8#VeNL*kWZ#B<;-crG3BBml7f?<3MdA<2$`StnZ`8;@M@T}v};QJ~N
zD%>L6DA>$bz;mCciC<CZuFw;~djcH-wgSccihK)r7I7PKOLL#+-o<;5-&>%PkB#RJ
zmj%yZehFbWkx#-igwF}D7C9rjR_wBvpqQuV8PNjqa}vgqbrLFKib4nZ!uV7LWQ78R
zuL_5Yh>6yU8jCuLToMWs{LOF8|DK;;C`@Fk=mJr0(U&6DVk;&5q~xVqB>2VjMdphf
z5nCYPC%I8#mAIwE3aQUBFJ%SgrpmsTej<5QqENC&dadkt+3zyBGSg*PWn*PoWj$qN
zWfsd+%C^Z(kl!rdFSl2QQCePVl~j%NCMhL}$08*{mO>we1;zNp9mNDiSBM6PPZ0ki
zdO(;<Fq*fBa|+viR%4Dn?j#;T?hLMpT&uWla;+8kz`2JrgljI>39c91rM#DUm-C$D
zy2TmI{e$-sUqA0A?sBf>Tns$Eyz+bp`Ofk07Q89^NJLt+Lexc6MVLb%obN4Pt>97-
z7BP2GUXhPNX9VX9aPc4Fy~OLlm&$9(lfuKudx~3@!+?3-{~7;f7`++F|0e$4`0MAd
z%-<h>E&jRwNAwS=A7{V+`@Z8x^-tlS8Q)qyIemEYVd`h=ZwtQt`10%1)DM1dZC*LP
zV0te9%=2mPlPymypL@M}_-gHoOHad|q&<D}T<hhU7tSv(y!iO?)+>|OEw5wV6u&Ka
zH~0Olj|;yr{owf}^i%7b?3Zbuk9{uw^8L%9FG^oNeq#Li>HUchPd_?*bb7zx?bf$*
z-)nz5`|19Nf_J`er@wvo_Sc*4*OIRVUgy7Jcy;zw_Zz-<@7~3}U-mxh!>x~cpL;$p
z`n3N8^Lw9ni{EW{-~1uq!>@N!-u!&o|6<CE(=Wa~U;p&qqx}yV9$$Zw^yL1-KliuY
zFL_Y#aMMHGhsz$^yf1aH>CW9d6Yd?m-}^w|(XGdxPYa*@e%A1e=jpjeZV#T_`+9%d
zqrFe&KH2w3^MTX7uXp#~%fIh`zx3XcySwjxy*J~5_QM<ZbMGF#9d&2by;BdKJoJ0S
z`$*#9k^58beY(5iZqwaWcYW@8-v4}m&;1+sBJQrdReD|RTG5TVyQPn~o?d*S_4v!f
z<cC%dX54GMyZ_Gn+dprAyIcP7;*(QPPdw&**!zI}k-_5~k5@i9{Iv4f?Puqn89m$e
zH2#^yiyJSsUw?ix`~B~ak)I}iaC_JK*6;nJPr~0?zIT29{4MI6<F~eNr@t|LH~-%F
zeesW~-*f)+F<)f<$vF4_wSV{ihcTuxH8A_Ju47-pS<ZEd>jd`?9)I4sJa4(Xcqa2j
z3or@1=AF&6n|H5(zlgo~42d=g4vBT*tP*{aZ=_RYkIQgL$4VWRc9xqZuO-(m^+rrV
z^qh#Zh=g#FP_fWw!M_5l1#$$h3HgaIh@KMJF47^|FZM+&R4hW|zu+9flfsw8UP^dM
zN=gWeg^D~D{wlIdY`wUs_$kq9(cNO%5_2Rbi|rDgBG@P}OW=i|yvQaoU5Q%p$D&(A
zKa2eo_mJ2kQ7+jp#V_L^Yb~oRV=DDoe3_WEc&X%bncs3NW!0tX#ZQT`h`Wh%iTjA9
zi+vZrE7>i5TP9y-o}_}<5n(0aB#}#Ew<R<s^Tk_4n1p@_oD(P!h!Us})Dzh$UMF=+
z+DK-z)Os;{p(ue>f~_KdM4d!z1+xXN3%(J$DWoWLUqDjezCe*spGd8ErKG1snTVCZ
zVO~bQ7XBCfANk(#M)FGY2JlAl%JQz}ea9yta9!|^h@Qj|Nd`$(u`uDALMucV#f>DI
z#ixiyiS&uQ6|obk5sDHJ;uq$>$p2N)T`W_|T>8I+zSu&MQzFKqPlde&llb%bGI@9K
zXz+IP`3NWo3J6@{+ra-=@VHR4;AOr--ZnmE!C>K^!U@8CLQ91v37d-)h};udC!#LA
zT5yh_tnfim0SQxyLa}oqQ$;F7Tg4WMm5Htsz9#sH--T}*uN2>9zT12*yt3TNTx?uY
zoQK%1FsCpvvg~2M&h?plI=38GF}n(jGNb<g*ng#em;Y}2_4T*^|0zt$EQ^^xFa|J8
z{;$Kp!=%V!%X*vTD-#35mcP}1OaF2H7y2*%f9HQ&#urRCm`WHD{;vIf<#)<o0fw(k
zJ6Jxjc(eF28!=fi<}u{`PygrnZ|?sBCN;Ko9DCWHG4J|6>(BIGQorJVNBs5wm-AQo
z_og3?-*vt{`MmGLf_MJ!j=xiXulC;go%)-mm%h&fpQ%1)eyRM*>D9zn;jj7MM84s8
z^XpaV%kUR@FaEvsc_a8P?H%Xa8!rW(sXeKABJuRwlPOQko;-Yf;c@1Z?5CX1Ha=~7
zvix!W6OCsto_&5A{Y3S#&ZE+Y?2q<8=6bsM>B?u~FaE#OeUtq5?VC5R-@np++5SA`
zh34yHZ{*)tzq<Hh)ys-E`R_vBMZa}=v*eZ7OSYF@uYzCay?*z~=GC8<TV5%@+5g7n
zwdBjb7i(WOy>57W<lX;wyzg(lyYbfiZSlL34^AKXKMH=l^l|0qi(iYrz5OEaDfIoS
zcR?TCecb!;{d>E&POtVp2VI>O`trvcr}v^CUVPm1MfaQ1SF=yQ-kH5seEaq7+V=sU
z#J*ntHuKxkFPlHfeX9M`^6AsZjF0)BG`>yxCH-geFY_O+-`;+0|8DW~`OkH~Qhwk2
zdG=f7*Y96{e%<;t>f5XD<-eBxJ@7yO|K&dpKdrvSe7XO5!k5Oc6TWTy9`-%*%f%17
z-lu-}`#$E~<F`NGcYK-uQ~Zze?+ZUNzFB>__i5p$+n*kNYWfuTant+E_e~!(K5qYT
z<$d&fix1+Tia)RW{NR)QM~(NwZ);vRygvW-`v=ue=RRb=FMe<G;lzi|4-D`1-paq7
z^FHfy$+tP*dA}=toBOTlN7*m#KQI3_{<r^s|F7gf28K|^RL0K?4Gb2HF3ei2xoox^
z!CYs!esR`w{o+~7Zz~utSjWGbN0GaQ`#8@o-p{;De1G{01YJe0i*6L{6!8%LER-N3
zDefnkFL^+`Of*hdLTI~SqoAu`gWz95CZWlKae@+pJNTFIs_{f~mvU$Gz7+T^yiMeq
z&>Mb1J~_T`e1`nX`DXE*;cpZCFLX=TMp#$Ch37HXI-XqqWrESdQX)3O%7Pd9ta-ok
zY~-EGcbxyc;0%$8;yh9prA#F`#XbnN3mzBZ7VDN=ES)V~FIg=fE_PeAQ}m=ryzoUK
zNzuC!)1)jVUBwOwJ><9KKP8|nTrTP$zD@kG*k#d=B2pr=g?9=+65b^IQMg64TYROs
zmFROpNB*b0`FuMCmJ3ISofg*<UnyEI`bAVnghk*Sw-V<{4sot!+;h0YxgK-eVn55i
zi{l&TKdusPR_;5TlQ<T$o3n3ZlVp9y^oY@h=`vFmGZV{Z<|rls#!!Y?4AYnbS<kUu
zWt+fui0vSIIHx980B0M!Gg~_wKSvm+CFf;!Mz&8Z@~lf)-?JWJJI2nyzJ;ZqIfZ2p
zD?7UxM-Rsvb|v;cHc_^ztl=z5OyUeh|93N7XSQX(#^KDdfz6F|Hw!y!ENe0AA=de9
zG8`OSlejr}O1Xb?YH^gY-{OekR^eU4tIfNfdn>0K2N%bE4iB!K+~;`?@?PY7!*4F|
zj&B*y7OoP`C{9!EFTC>n7JR)tSGc%1o!QT@O0(v(7O~xD-_5y~yO_t5>nLjhBgg-S
ze@6f77&w?tGgq;iv#GF6V0+50%Xyu1C08&{Io~mXc|yrTi}+J{ZgV+v-{E29Hx;}j
zv`l!DFstxVL1Tef{#$%2`EKyH2xW`5i1SN?h~*0B3f>Z61MNW-x+BCdtRcKps8rBO
zU^agp9~aLlP9Cm>Jfi%${I_^3xnFUdWShzQjb#qYN|r8`cg!=GS1{W#>oA>W@cbY5
zZ^^$y|KBoXG4?YY_{Z`0`S1AO_P_uC^8aQ2^U!zmZ)snnzHa%l^V9SX8t<LnwY*dK
zVDNFpht=<Iz0-PU{chbm&3Av_biSGS#{Ny;>-5)GUoU-K|4R5(;A`=>4euJ?uXu0t
zKH`1thua^wd|L7;;-k`g@waPUZ+g}I%JlW0*QegRetY-b_V;`rSAX>U@aOH;H=1uR
zy^H+t|AXvD#*aHcoc+N6asCIV4~idaJ|%yS`?CJ)(r@wKHhj_il<?u&2dU32Uw?l6
z@NM;X`R|jz8hn}cdDZ8YpL0G3d@ld|>2vy*WuM(YO1#s0eeI?2>kV%|zPI{h`ZeUc
z{`aJ>7N1vs{PW@Dhsz%xe0=`p%#WqNLVj-iHuKAx&zC>je_8pZ`|I*=U%&BxtNh~h
zS>yA>&(l6<d|vgL=d0y+=N~!WZhR8^aQB_syB%+{-X(un@=5lK?3a#D<sWXp`}(H&
zmGa9~FKb@&zVrA{{~__+*4NWsZF+U&rS6N_FW6rHe{=Y)@H>NdqVEFV8oaiDk@>9s
zS>*GA=WCzMfBN~!%O~5O+<0vMc+caBPmVk(ezN27<wur}`X4qvsJtJ0-{;}0C!Wu(
zpEEsQ@SOXF+>3wD=RAM-T;k=Cmt8MKUNOG8`}WqmJ?~e4komanWA5j?uM*#yzUqBF
z@zv#<!nd?<%fCzil>3$X>-|r|UkZO1{+<3e^M5Nt3gZ!`erA29KmQ;8yYa8?|0;&p
zj4aGoncp#MGIKN8F)A`{U@T(V#<Y`3k-47vJ<}q_FaKZv)%lb7JNtLa|NnpD|Ai_n
z{qO#7>z}v3_5OVM!|~7T|3!vM=2}*E*1t^VjMEs*7!8>oF?q4fVSB{R!tTO4mAQ%W
z)c=qFqZt=5HL--Sr*r0V-eiwp-OKczL5d-l!IvqNHI^fktAlG2=M#=jj%V!Z?04DJ
zI3l=R`A+aP@E+st;>zVR;Wp>l#LFVkE_6`XRaihsSg=W;PvEgYmLR*ZnAisiUa8fR
zD<tB@*NEN{b`V}J@<ME_#9c`?ss9pZ#ks^8#Mnii2woHH5Sb(XR$_z1U2zNX9<gU4
z+k|EdJ`<cG^jPS#kee`{$aayPBDx}*g#QVj5^)jD6I~?QDtb<YPb6RXzfi8Qyr`kL
zodl=EE%6f)+*0$TQl+$|ilr6g7!;xuCMp;!UQ>)znyqwAsZ!ZWr9;I^WwP>PB_pK{
zMSsPqirW;AD*jbmr&z5JD9<j>DnDP2T~1n#O@6Jsu6%>6xAZ1SVaYiXNfK8jrb;f5
z`X;?V##<&snoDYhgoeZvi2})Hsa|OYnE>exl5fRx#rQ<yMShFS5Q~y9lUyq?N&JLZ
zhp2#Xmq0y#pa6@ItgyN8Vj(HPLOyHmsT>CE<!qZ+53+t?>tmn7wwgtP$?Cu2-`+oS
z|K9nZ%GAnonDr6sJeEC-pZ~J_e)^O1*P>r(zZ3p^`+N0&JwxCB`TsWltNDMO;VDx*
zYc=~cjujmL*l(~IvIVnkW(#1?<5<Iajmwt%K4%g8HP(r&`&nPIZeY8~v4Go`*M--U
zCzSgomjHJUw-1jE?{D62K5>3kelb2Lo-nRk92eO6*?+U$V;ACV=e)o%kDZx)KARnD
zDswy2U8bqbP0UtIOpM1FW;5*jzx3btzX$$q`K$H!{BMn45<hu=KK^<Cm(K4gzgmA*
z{;2=%_Kodp+!u#0&p+#YuK(or>Bq-EAGW`r_^$8mhd1lr<h&_=bKp(UoBgk~UMIaW
zd^zQX{>v$^roX=S+VYLeTb;Mjub;lS|NPkt>(}?*%70k!k^58q$LbGJ?|t8uyyboy
z_15xT`1|=EzJ9#*iR-i3r@{}v-o1FY>wVP+;g1(TwtQ0k6!5|4UE14%w_D$OysLRH
z_)+82{g3b7|9m6)D&xh)7aFfs-?@Eo{rLN1@u#?tYu*XG$$k~~GUdg^=TR?2Ud?=+
z`Fh>Uy601#3Ow<DeE;#or$=7|zixUX`6lVrujgN%{(RE%B=m{)Q|sq;FVDWLeqr_O
z{uAk^o1bofy6<V*vkTA9yqNi-?0Lbnc~2KTIrG^5ap2=+Pi{XGe^K^)@6&&ee?MCG
z$o7%QqkWH$KP!H*?YZQ$_fHz0C_lOT$n)XShpbO#KK=QGVHAvpz-R~z&JX|q`=z1^

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/31ec5b0134bedcfe283f4978e6e65b7d35d5d4ad b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/31ec5b0134bedcfe283f4978e6e65b7d35d5d4ad
new file mode 100644
index 0000000000000000000000000000000000000000..868e2672727ac1b7d505faaa22d79edee3252cf5
GIT binary patch
literal 26123
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gbczC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4cP*Mf)+x1g{4Fb
zMX!r=iAai`5lt6M7vC>&KvF{Lykw;$o0Nz2d+BA;Y0~A=&eH#+vZd$9+>y<a`y#tT
z#z9(IYMSI$NkJ(IsY{Z6k}47wV&)=`g!F~B3d#xw3MBDo@rCnp@MLjq;IL+wVB=x^
z$E?i!ipiDv5VJ6gAd4&WDaJO25C6meANUvY{{zE*Mo~sPhA01X{|Enj{^#89>fZ^!
z%YJ?S!TRIX_Z{CCeJlSm@#C*|PH)ZMY<T_p^{O`_Z=2pcdbQ=nhG&6K*FEugqW@U`
zq0#*_ciZk*+%CN}@mBHe{dd0I{d>>h!P<u_9wk1`c{20q>Sr&Wm%RM{s_%{EJB|-u
zKkol5@-_RLzz?&Z&ws}JzV}y?VLoFS({H8?%<otf+2q)FalGa#;JL!Hm**wVBwl^K
zmAni*%eek?CUD;1Fyi#(a^TM3e!&&VrOj2r)x;ggE6>j^&?mqn$RHRdaEqTu;Ecc~
zL35#ap@Tvx!pDUt3x5`37qSyPEWjghl5aC_A@5G!^Sm0oTe-J$=Ca>qQDO4`fAi0@
z-(kNae%<_D^R?*9^Uv&`y*_Dwa{ct;<Dm~(?;pJT@~-b))LWr9zOU6@pLp&0w*S5D
zr$t{Fzgd6V^7YvltuLQGhktqb<?EN=Fa4i&KeK+m{OQW4NuOoE{`%JVqyJ~zFZth_
zf4~2I{CD1O*WW$A<^NpyGv#mbzlHxFF<3B$GbI1l`~U4<`#;|Q6Bwp3E@SLvoW;0~
z@jl~A#=ndgm{?gHSVEawm`a#9m}fBuGaE9AFvc)s{LlTD^DpPWGlLX^#((~Qum2qR
z{o~jEUroQ}{tWuT`+dUKZ=a8U68x0?am9xV?^)glyi0wn@#f^K$d@mlKYbSS?8(!v
zr-z@;cy{l(`pfc{dtQ8bUj6*(vyNxX&zoO7el_#$pLY%KINn`<ckum__a+}~K3ab2
z`Xum)^JCBl-w&K0*xvKMJNS0@+f#2V-|>Ht|Mc?nu`dN*96xJ(+Vb)8N8?X@pPD~?
z`Dpub=ZE4CxgR!si2hjkY0qbtueRTEzt{h$`RVfO-mjwH27i?P*8B@*NMMR&abn}-
zc+GK&<0wZvr$5&}uIW6i{L=)^2^<kH7rZQ(BxEVvD`FwmBfeaGv)E5jJF%N$+r;`s
z9}71Ktr7ez;4Ba$P%F4YNLF~CP?V6H&;p_D!r7wx#2CdNi`^4jA?7KTAm%H6Q=(bw
zrSvzMnX)%zFUe-ehRghrVwFmld@8X>qDvx0VzI;z31Nv3q7ou^g<J%i_!skS<%{P3
z!ynG?&3ll05%(HyQ|{+n54g5-d2=;!F66M~2;q?8U}G<5Rbr`Sp2d8LMUFj}^A=YV
zcQtn}mm}vQj)xqboH|@>T=`t`T+Cb!T!*<Pa!=w}$-9BCfd33XrvSIWO#YXAxqQKV
zEqsUg?(%8yvkLqa$P!Etye067zm=auV5Yz$fd_(9gnLC2MH)pmimHf5OWc&)C_P11
zSbmLsoqUtLg2D_18wFMQ3$lE&DzbI5Ut}N4X(=35U{mmv%asw8j+Zi&a*=u@IY;7=
zn2KnM@Fu}Pfo6V7eiMFu{#SgyeBQhtxZAmWI5jxL*uB`Uu%2h-VpCz$WlLp~Vq3_f
z!W_rc%Gk`1&0xs*o6(!;2;&llr~i-r5BMMRZ|Yy2zo-8&{^|eq`p1Ru?B7p(-SfHV
z<Fxld@6z9Xf5Y|m#G9DcnlE2Ht9T;z=-7h^4_qIF+<$df>W=3v;hXbr?7k6xQ~1`_
zTlu%e?)ctGy0hVq^Ig|_%kJAf41LV>?Bt93*9q^OKX!fo_@(M=*Vi{+AAh^_ef5vR
zUv7Vk{;M-gWmw3#k}00ql%<n3hFyyDH>Wk%F|MiHZ@4FNujAUoS<k7-rNS-C6T`cT
z&x`*p-woc&JcoJi@&@qR2`mygC~!*PqQC}$c7X>17X=>+9T0XGDHJ&=;x4K!nj%sz
zTp*MzxKiLWe-57tuM<x@_e!p79EVu%F-bAT{&)MU@+b69?w_RJUO!)dv;FGvCHS+~
zrxPEmKHmB8?fs5-9&fk3(Rl0mcJUjpH)r4QzI*ik%g3XizkHGXn);>w^W;yfKF<5t
z_DSnA-{(W08a~;6^7yp>Q_1JNFKfSk|7QPV($A~E9{g_pv*l0yAAvsye_#5o{b%kU
z=f7_MJpNbxKl1P5-*11M{yhDC{WsqqnZLXLrT^ddPv_sRe>eX>W7x(R!t|Y~m^p-b
zF_R{fJkw9cLdJ6px(qA+Z~1S@uz}$$gE&LYzq&u2zw&<E`quVs#y9S7kH5rz-u<!j
z!_@cn?~cBm@HXV_t2bG1X1(6@>eowwmye&nc;@}=*Hg7;2cFG(zUIZXmwR3*y*}`&
z^JV*sO)s)u{&*?&TKw($cQ4-gy}S8V@ZGC-yWUrSX#VK)N%B*`$Gi{8?+?D){Vx4o
z{@aspe!RK#HvPTx$F5JmJ{|mY>|@*qt@jJx@xK52UjKvXd#iVM-Uhx6d^7X)#Mc~e
zPP}P(>+-Jdz2?V7pU!>$`^Ep;w(o8~KmB^}hwZ;5V>5Fms}7qPTNj%n`&0If9B(-1
za<}oy@bBec%>R!+TQE%cw8$n=cd@Brv&Fc?K8i$%923zNy&$3^(j;sud{fw4^sv}5
z@%Q36;!NVp#rVV`#Uv!oOIAp2mwF_1PHKzPLaAv|AEhox&z6ywT_LM4=PY+sR!7!T
z#!|Xa%3G>JYP!^UDO>3#X@BX-Ql642;?qUfiUf%47WNm86n-RBEBJ(e51%04dER>7
z6Fgiz$=ropZJer{a-0U7R-98f<k?x-uCoTQ&10|Qe8@GKTbM_I=K;4k&l?_DzIOfu
z0VaWHeqa7bepCKLeing80*?ge32hLv5n3zAEXXLZg>NqJOP&&*Y@RnfF}xFb|MIr*
zCGf4{P2v5|dy{V_|1N<HAx{w|F$M8<aW@GmDIS?3*#&Yb@)zXu6fP?iDcCDCDtwhs
zm%Ay$DdR8GE6Xk~qF|_SOKzb|nG~CZlW4WDzVIiZmx4_Kp8Q389DEmfJ9xu+{dr&U
z+~dB?<;=B`a|uTQyDHmWmetJs%#BPZ89f>O8EY8T7`Yi{{S)|m^taOQzrSw$68-(`
zx80wP-y47B{#y2n<G0A~CBL+O<^Fv3UGv+@FTXzpf8_h%_wLqf`B&3l7QJkG>HG5W
z^V+AkADz8__>TDPtGABdvc9ErGvj*NRjw<Em+xPied+t9dzU|6vAouNeaVg2H`8vf
zzH|7l)cti2N*~R9qW1jGORG10?^eCP_Tk@$lOMu9wttfPV(`uR$D5xUen<X|`nTc#
z8%8dcb*yacGuhX%TX9_DVB=iIxs2-tS1i{a&Kgc-u1(x-yheNld~bNK^4#Qh;TGfG
z$*sU^%;&`SllK#EDc>x9c0negyFw8n$3+drEX69tCWzIFRf!3Rv54LlekWusR4JGv
zAjQ9lx0+jpV-d??#<~CF{$2U&@ptzh?LU`)b^rMNb>U~#Pn$me`q1)z?Yq8rd){We
z+4_3M>*Ck4uOnaOzifKR`r7_&(R-nfS3in=GW+!QW6{S&ANPE^^10y4@h^A2@P7UD
zRp-0MkH(*SepUQ7`J?xD*I&bbTK^~fU-NI_-<yB-|6ck_{g>RY%3l({`+o2Lo%uWM
z*PNe>zs~<^{yqDT$Upc01`Nv>wlX9$+A|xlCa?*x$FV!HpJqGCYR&SJ$&M+5@y>tm
ze@%Z^{Z0J$@85-gr~az_`Sf$@_cdRAzZ8EK`YiM5()*IPl5YfFKYYpjvh79b3+|U&
zUp{%M{A$OmpRd-xVt8fqa^VZP7yF+Zz7TsE^wR(3wwE3+o1bewyY#g2Y0cC8XY-!-
zyjb?K|JBx4mtHM>{rrvOyZi6nzdQL(>b=AJ`R}COWxreV?%Ug%H<GXCy|Q@C^v3fI
z!<#R!3SWA^;Cn9rROX5JQ-No*pMHLF^2wPe?N2$MTf7i^8T0DQ>(_67ycK(I`XTmX
z*QZ0DHNU?37WXr5bPRrU44zhF@Q$Nn@S|h!qhs)+WALM6@S|h!qhs)+WALM6@RW|h
zkE}%;qYcQ>MI5wP#GyL6h+}jS$LJ!C(M24ii#SFXaf~kFz`clr0Sq>aP_~%kf9Su{
z{}LI*nHR90WXWRsz-Z3+=3n98o<HWlZ~t2OTkX%?KYITz{p0v&_^0~k_HP%yuzY^@
z;qSYRZ{%N1da?Dn#B<x{%bqK}XnL{gxy3Wqr@tP}eIRw;{_cX?A8-5KjlOsD-t&8r
z_ro73KfL$w=p()-I?qa9$i9?&vHQ8-%g|S@FF(EzdNuu(?dzboPu^>O*8aZtC+jcf
zpK;&Ue0BSx`#JT~t&g0a>^}#8N%%VPyZ6uPpRB()f5-ja`{&euVWzK451BPs_1JE)
z)v&v9q;V>9yYTMglj2w8Kh6JJ;I!Zu!MlP-1wDl1g?WUFggy#Yh+Gp55OWlJBQ{5T
zpIDaYVUb(HT|&<VZ27vmUvb88$g)pmXXDi4isH86*})ymCB*rI-IYC${X2URhX7|E
z=T}ZH?kXMwUTxlgJkxpjdH8si@bK}z;i=|v=UL0+%ge)8&tESnFDxjcFOnfVLFj;>
zl)!3UEp7+SJsh(*Cvjco%HX!=y~B5ePk?V8?`od4+zi}poLn3-9IYI0*#+6>vgNXE
zVas55;Ber~<&xp9<Z9zEXOm{Rz*zME_}`0v%>U^AS^7uv@87@w|8HeF$l}5Jhb56U
zg6$;RLAC>IMQr?R)7b8_dvO|bALD(+&nVa-ctY@<V2IErp=&~X!o|YNMDC0AN_>|{
zl4O>;BFQajC=oAiEZ!&XCq7B6QS7N$p_qo~3t<7_B%x=5Izk(TJ;Y8+a7*2nl#q;*
z_$>ZP{Gj-AF?KOov6Es6;;+TMB+4Z=OU#jI5YHEF5uPo?C3Ih)nZKLwIBx*&8lGJ4
zxm+{3ZgR?UY-iibI)$Z|xrZr}aXUjBgC+wDLkfd8!`J`)3^y3E{=57;_s{PCf`2>y
z9{3~o$LUYbpT)mrel7nY`J?d1)gK%`^M1Dftp4Hg&Fb@^_xf+!UQc?t;rX&>0Z%79
z=6^izG2@f?C%>QAJze^w<?)(Ft&c7}`uHg5QNzRO50xLeKKlM>_mi2=R9@7)Y=7nX
zdi(2judQAuziNNE;3ebhM{g#-EBx^0)2}bHzwZ6&_-)U(Dc@Co2L8J8YvJ$Jf7t(S
z{p0>e=8wgn&Of4mMgN`uf1A;f`6RO}%Mun}){U&5tXeE@nG%?OFnwY6VEN7x$9|5}
zlB<mKI!73X3&#<Tat>v7MYaj7$5{+m1Xw;Y%d*U1napCrqQ~-&xry0=`8E>^QyJr7
z#zLl(OodDj8ILlSFbXs5`*-l4`TvLinEzS*jrpVVyX$Aw_iJC{z9xK)`pWSw;CsoB
zuAlP19{zCq_W1Lzk3R1M-WI;Gebew}@!PrY&wsrCx%R8}x6NNWzw~{c{W<>&+t>GB
zmVKG^Wx*GdFE>B$`jY=`<M$&!!hdc2ZT_d`&%-}be;@xz`(yFv%b(bPEC2ssc*t~!
z<v6Q4TNT?Wwl8e!*dkdOn8g^q87lsJ{Nw+(=iifmGybjr=kni=p@Wf)shjZ*gA+sU
z=*0Wz#QW&P`{=~`u$p-P!}k9&g9p<*=46&7Ecq;dnax<%vL>_JalGe{=VIjk!!@0&
zflG;dGfz67AO8&gU;OC;vjjQ>Gz1s~8U!8-cnD18=N4!XR1x+Q=@#J?u@+_*dM+?i
z;G4h!!S6!vgdIiRi};Eqi)%;(OIApQNl%gyl?#-wRd}xWTParAOnI3Sm(pd$M~WX6
zFDfP|Zc&g_u#sOZYcBIh`kC|w=?l`rGTWr(rRPc&NE%9*OH7hDAnqu3T0};;M=(vm
zfWLyzn!j1#pTKJYIYDJ1O_3_GFXAN<g%bJ_m&J3$4~f-_*^4tuM2p`S-6O&x;wU^(
zNJi+lV2x0`@NVH^VFzJO;X{JU`7QXi@rLsT@yhV+<m=_T$|om~EwGDEn|CoUAOCUw
zr~J(Vw*?J_1w@{U^o!0FwGz{i@RAId*eiZgLPyd^LRO+c;*$6!v3H_+B343tLiWN^
zB9S8LB6o!gg;E48_zv+T@#OF@^1R~`=X}7Xz-q~Ijrlk83>J1)cGe#(r&uCbelX`V
zr!yU6bY?oqbd>2N(?+J9jLr<k|NH-qM$RCPoQ;1S{@ngm{p;1QXTMy2as5jFRsQSP
z&$B-ieq8>(_<PIuxbKSJ6~0G*fA_8XtIwCYpLc%>`}pSl&v&QZX}tgVZra=1udl!S
z@$Agw)Q9Kp+uZ+gfAxc}_j&Gn-rs+J#r<dZdmsFIu=L^nM=g)lpG<q2@qEXNn=jd4
zzkR*s&7Zdy-f#Ih@$;LnFMcfl?eo|0ANPM*1~W!grhiPiEK}ILIM#8j<e1AjfxC(~
zl`o60fbRp}AHK)DeLTWE@;tgcxjYKIUwFUs=?VN5xGA6{kj=k^?<TJaZy2u?pBTRu
z|7<=XK6Snue3SVl1sDahg$jkQ35Saah_;HZ6m1k;FOn}@EVM|_M<9={iRT)Z2j@Zd
zPpl=(l?<%^TK+ivzV%b(r_)c-pAWtZd`tYo^ZDt=^bc|G_Phytlk(>KYqQt)U&*|V
zd;Rp)<CoPhonEbZZS{8FyAAIjzkl}r?0f6?8Si-B#k@QAuHjwaTemkkZzjHd@b2sT
zogacfzWgZpY34_jkJ%r&K1F}Z{M7&H;ivS^-#<rx$@}8_Mfl6w&nG^)eSGnL%lmB~
z<US>Rp7rI<*K6OzzHj`t`rGsGY`?hw^!>g6kAYzoLmR`I|A+o*{_FqS^C$oJiC;5+
zHU0Ac_37ukA1l9gd|vw@`(5oD``0F~8(%+twdm!(7c*WQezE$6>kEb#K`$P@h<~a1
zYRYT1x0df(-pRdde9QNC>zmv+{%^|OTzV7!cIjJ>w+?T_UKhWb^NRbm%j?gt55B$i
zUgx9Qr<I@geO>##`-k`syYG*`b$+Y=8u;bbXNfPlU$nlMeUAT>`02!_i=X^Hm3&<I
zf%AjQd(n5|Z`s~netYEI-FLg*u7C6Q&B3>u-p0R`c(?X_^2hz3W_>RH!u;*Uch;Xd
zKTUsa`Q7oi;(ryRIrDTDEw)eWH#yummb2S%L~#D+oWr$}`vK1%-iLg){2TdI1rG>a
z6WS`sE*L5}TToKaNkCfQsDQoTErB2W<@_J{%>>PbYK5i?)eChC$_eoD8}sepE#o`I
z&n57H-$HP!aDwO;(H60-;s%n@Qtzb-q}ybS<<84LP}r&XMaf>JTqRpsK}lXwO@Uwj
zjI4mn2T5&-0C6qx>0%0^uY^ws8wu+QrtpXIf8$pb_{2YvKZE~2|7-yxfw}yO0=$CO
zf)@np1a=8b6`Uo+D7;RHUC2c+Qy`LGk8d$=1YZ}wfIu$)7v4EMaXgQBG<mCepYtgS
z-WIA7E)wn#W)iUw*&zH%C|Kx`;4wilp>QF2p{s)H1$_lI1UmWZc`xx!<6F<Sh3_)|
zazP8>c#$H}dtzrK45cnfHcI>wSCn`pZY=&ntWms5Vwc1{@o!?6#8|~Yh~<g7h;9+K
z6M8MMk6)1gAD<onM1C9o3SN2c6P)gxbsUq}W7)YlR5_n=9_1|Ke95_q>nfK$*9p$Y
zoX%YGT%nwcILbKuIJU7bWfNnKVD@M7X6k3MXIjX3fsui^ojI0Chj9{PC6hMO4ThQj
z)&I}<fAW9tzk`47{*L_p`ls>_f$z({#eMtomGx`X=hTlY-fO;F^oH&Ash2`8MP8^p
zoA)^Cp}_;c2Xh~My3cZ7@&5Gtg7;<a8Qq(8fAz!rkGDP5dA{jI#VfrxZExqjJMiA*
zqu6Klue-kGeOLYQ=I4XoaerR_PW!Xsul>KDf2IE!{NKvZ&1lQGiD3qVHp9_>dVll&
z?EW+7ujjv}f1dwU7!NS<G0$dt&$x|2^#8rTaeq_(p8PxWU&sG5|Hb}K_}Bfvknug!
zTV@7UE4FO*Tn={54$etjTe&}St8<^>6z9-j%V7Dz)XFH!;PijfKcl~uzn*<h`sVm`
z&F2#zr9XJRFMF@{-ui9pYpyr$Zx_7jdd2-R;l;G)*3V`=x$*e$lf-A1&pV!*ztDcU
z@8z?X7hX<&nfP+zORHBOUPZpX`FhctTW?OkR(Qqw^2du?FYmqzdcE@XmN#GC`oB|u
zx8mK4_ZA<;KF$7Q|2gZ+)vv7IZ+<WT@#qKl&zC=z{_y?5|6~7;g}<u)IQ&)r7s24d
zl*}Z<#K7FfvWeB1&4G=F?FXwPs~@uiV?9GX!~g&5|Hb_M_$TJ?q`x2ju>A@B-SW%f
z=brB}-zvT^f4=<j@Q32}-`>W(m3_DG-IKQ^Z|=Nie3Sa-_nZB1C%$WYAMhde!|C_F
z@A=<9e0SyD`F9oXR=wqU%lh{6oAs}kzIylS$m_kY9bS9BzVh1bjp3UeuSMQyy({@3
z{BgpE_z$ihy50x8<9l2F=JT7KZ}-1Vd|UPQ!CQuRi{I{e6Z=}@RryP^m!Ds3ePR9b
z+p8OITHmI=o%~k%UCX=U?^xf@d@uUZ^t1j~uJ0~Ci+}U}HU8)L-;BYZ@hVdf3pZOj
zy9}oUcL$FUZz8V*pB?{nfm*@2g1-bih1o=(iDZbFi%t|fCw5shN<=`|Q)r{$Q-Ns$
zg@XHql7zK{7YG#zofXm*77{)pG({*>=$4?PV4r}7K$C!-;9|jDf*%F932BNP5j`z7
zRs5qwij=6dp!6ZBQmJGqJE>sFbK*P162+9oCW>Aa`6PT)C|l4{;69%tZxGLGp5weJ
zymNSt^0@Lc@ka0%@|@y%z+1`po>!4qhj%`28Sg$GN1n|*EPT@ZzWmepUkX$TnF*T;
z3k&}jx+k<ss8eW}&|9He;bswEQ9;onktrf}qWz+)MSqL>ift4t5cik(A+cO?npD0t
zyG*6bDVdKlqOuXP(Xv-%++?C<*2qkgNtIbE<12eYmQ&6_j$Qt^!WyMT$_Xm^YQY+6
znhhGs>g8(ZRR5?PP(Gt{P02|4u@bXVhGMK@q2eor3i+k7AEfQ1rc20+-xW0xEfPH-
z$|zbYtSjUt*eGy;Kb$|DUr8WPP+Z7Jh(Tzhps(OEfj)jGz5w1UJl;Hk+^SsioMs#%
z>^oRLFi&SJ{onUb{~zn$+rQ)g-1^)4FaBS~zk~md|NHV+<<F0wCO<BI^ZmNxbM2>>
zA2)tv{@D87;@$VRKi&$zt$h9bWycG(=gXe5JZXQF_ps(c#QoEEKi*Ei_3ftHt@2wv
zw{&iE-I;pl-W`Lx4R`O~jlFm5p8WlL_kTU$e#H1#=84Nwv1jL>{dn&BlKEBbtGTZ=
z-Wt8z_Ac_h?uTn1WIo>fF!{sZ4?Q1UKURED`jGfx?g#x3Ebk@W1-^Up?%BJu@A}{0
z`f%Z+#HSq}O+PYyWc>K@!^01cKe&8s{iObd=WEB;o!?S^O#G?#tLfLVU&_C?{Equ$
z@i*_^>i^9Qw-}Z&2r<O`@Bdf$m*>y?UzI<X{kZgf!napn96x{gDE(3U!;E(>Z`0rU
zze{-c>}|xmY45#09C#o1?#G+!ud7}yej)yR<FlgY&t8<in)o{Wjq}@2Z_mD!c-#2K
z@{RwSd2epNQGWaO?Y(!#@87;Fdnf-+?p^4+q<8b*oqs>=qta);FJWK9ztw#2`@!~;
z>!<q9h@aIzum5cO<^Egn&(A*x|Ni)!|M$)xo4*eKj{bZ6@67*1#z{<~%+1W&EFW0n
zS&dj9upD7&V7bq%!@QiSj_DxNd8VI?cNuILHvixGFXHdj-zC42e@6e%`TqUuxi7mv
zvwlwgr2KKid(QX&-`#yT|J|Z@U*DC!zx@9Fd(#iUKJ5Q!@Tuq1nNL?f{r{BqdHLt!
z&(}Wv`KbS~@I&N#!FN7yz2COKUG#S0+xKs;y}R??{DazuSsxlda(+7S@!bc;4;t?+
z-~D@Y<+ar7!>`u8I{WJGtHZA@zw&y0`1PUJ0<Ryuoctp6x%4x!r(2!~JiYbw=`)TO
zx-b1-8N6<KJ?r(q*X!TRe*5{|>JJ&81imzWE&k^HebtY|U(A2j{Z09Qg+YmFC9^;4
zBsOFA9`+yX_8dpp&#)D<ePioqm*$Y>tmB%;&CYY4JB9l<R|(e(&cmE(T-&&td42h3
z3G@q=32_MD5RwvV6HE|n6FecPD|Aojx-hTk1TlUIDaq%O=cHyx|B%*|sh1Iyy&+pJ
zw^Htq+<Uowazb*QvWc=%vWsMrWh!LqW!z<6N^g)pA^kvlv-BM4>(VMRMKUb1(sI+}
zWfk+4`jp#Lgj8><JXCH|Do}i>a9ZJ#f{fxTMF!<-%3UgAsxhizs*b9$s=HK0)vVNl
z)N0h$sXbETR##UyQ}<U-QZH2RQGcwSt0APhSM!Hvi{>khSsFzeF&bqWb2WZxtktyE
z($hMxS*NL~*`@JDeUiGp`VO^dwOF+fwF9cWssgI#Rg=_W)vVR@)xN80sh(8!Q{q!x
zEq_9`LE2C9viN(k52Cw7E(v=G9~62axL+WLUy<)1j}W&GXBT@e+bh;1tg)=gEXB;a
z%>S62m@*l^G3;kpz_5_vAcH*P8^-NSmzm6%LKvkP%>S|dQT<i>WB<3*uRFe2fARgy
z@M-1;llLdz7QUJNddusWH(%dmz1{hC{@br_4!u@-W%45MS^txrk8VC_yDxbE&AqAj
zIPW#wJ$9%5_RgE<uK&C0dgb{g$4lmytS;$X>bm5A`NrkXmpQIzUP-=k;)>hVg;#%G
zHMwSYt?ru0_2lcp*ORZ`xE^_9#|_b&r*CT9-g)QS-Cy?%9%Ma?e8l~@@$tRKtxsy7
z&VDBL;>t_q*DqdgeAD$->YdWNX>VV>X?PRx#_mnzn^SN0yk&jQ_Mzg#!4Ho=O#R^S
z!T5vZ2gwiTKCJ%u`s2NiET4it@qdo_viqyqcf+5GzkU8N{GIXl;@{T4?0@I}S@cKu
z@7=$J|E~Od`)}Ppvwxj`*Zi6DJK~r6&x9Y_zpwnZ;%nO1@~?Vd4}H1y#rx~UufE?_
ze!KPU=C_=0U0;2_T>BLJ@$!4Scfa4{znSwU>TUMBs`v39UVnJ;q49&zhkNh8z7PBG
z=0n#<iBGFOReY}eqW0DJ>(4JYza0Ov>C1{QY+rf5m3$Ze>GJ!=-z)#s81tDXFmthd
zWZuvGp1G7|HA@PM9ZM_=3rhgAHnT7DYo;wsVa)z4Q&}_E?Kt;viSh9B2JznF>E-^w
zIh`Yl{Wt3emR^=^ED@}ySY25^G8ZuIW!U;}-k+tvPW}l0ZvSo17pBj<K5qT+=Ka0*
zkKRvyU;MuBeZ+gl_q*OrdH48T#rwbSmwZV3xani^$GsoeKIFV-d|&mh>+OU$KVFBt
zUi^yd)%zD!&%Zq5d0z1B;8WITcb_eO?*C%Ji;fqGFSK8*dj9!Y$g}59=RP%ldhH3z
z)6GvEpG|)D^O@W8<mYpr-*_(fV%v-Im#bd}ynOm%@rwg5LSM4K>Urh<dd8b+Zx_DJ
zdOPRsqqiRK-n?7#e*1@-Phnr4evSUV?#KC`Uw*OtVf*X&Z_@v>3^7bmEJkee*xzy(
zaxrkXbNlf0^IGvg7WgfAK}bl%K{QYFi%5$|tVoH-M3GvNgCe40krIET@?^c_k10ec
ztyR`gVNzMBEUP?M$zMrGNlnQ{Sx?nm{f9=pmaR6k&R?A+x(jvVbWL^hbdB^J^rH<z
z4gVQ>8Ce*eGxRX5HSpID)H|qKr0cCKraM#Tw)SnUpPI`x!!+47e`|{99M$d8JES*3
zPf*WSS4QWC)@@B`%^nSH4NeU`jb05I&3&5RG$(6{X=Z7-sRya)sRgRJsJW|6R%2Fo
zR1Z*3RR6B_OqD~mP-TJgTcvI#Q>98JcjXPrhm^l4zfi7FmQq$zZc*N*9HKl^>9*o#
z1$X)Ra$RzdWFN^?$+*cJm0l{XEVD%>QdU?_M2=DRj<l0>m6V%gh{Or;MDg2Vt)h2@
z1BF9{w+XEfED-q3H-}e_r<F^W^9Z{j+ZL7t=GlyX3<(U%j2jsbGsZApWN>E)`+xUu
z%b#Pv=l)*)d-3m`zc2mX^n1~7>)-o+UimKY?aAl4AGzK?eLer>gXf!{);&4)DEEHr
zec#90PphBrc{}}!$M4SnI)8qCoA`bIPwC&s{@nOA^PA8Y?e~9QFu%C_yzz<q;|GuQ
z@9Ez>dF|EZ^b5IX<4(Okw&PItKFK{zyV-Z2+_iRB+^&dSM!RP0)Y-|f^Y4y1JE!lO
zyZh}P!+mV~TlR||Sbk9Yh}^Nb<I7ICoLYT4;!Nz>zVqf6ZlA3@lX))s!mSHuFXdj}
zb0_oRi>G&9y?USXMfykn@1_6FF-&1BWO8AtVKd>J#^cXFS)f>;ls|?)UO-4-58p4o
z!~Eg=%=|k1Q}~wg%JJ&)eBtWiI?9#GHIXxh^Dk!yw<_OJ0XgAdu{n~Hq+Mmd%caUk
z%de52Dz{YHSAt3OkzhRkYMzxGPnc8wEB(&@9`JSk7qc%cUygiX{^tAb?$>4C?0<Ov
zc=!Fr&ze8^e+~Zr__Ob?>i^gO0{(9O)AQ%o?<c>_fA9a9^Q-sI%Rf{9wEUI$|DB<j
zDU)dvV-F)M(@`czCLxA>|E~VK^l$lpUq(}=U(8+XE?n=pCHSWCpX7haznfo^|1Vz?
z-&&qrPIuO)jPL(*{q^|q`}3|3if_4JEq!75eDagdM+%QlKb-#{_I~`mgu5s2oVZhX
zH~+rIqxQ#V9zT2X`02JM^^fL1{QU6lLym{>4=&xWx_9=D|DA7l=G;AUx9je%JC?Vn
z+~mIb?Uv5{tB-|VoOz@9X~Oqef0i>$VE)EBm;D=i4SN+E6YE0eP^OOzF8}xZk@%(X
zGxKNhFTuaU3_lsqFsri`vP3f{G5==T!T5w>F@rFpF_RLrB+E@!XZEG+HXJ`V-g4~X
zsOC7wCd2ZE(U3vsU)^t$A0A&dzWn%d`&;!7+n+qY1b*xM?)_E$)AC2@&x?Q7{oVP;
z?=Rcm;6L;JJ28FwKl%SWCP`)%mWdo8oYAbktWP<gbB43JvM6&TbE>lIaoY0r^3LXH
z;b`EG6KfH@BPcGWDibQnE^%FfTP;b+PdQ6VSJzK_jrKbI)5a2}EGAD4GxXi{cI%$e
zTB(tyJyG|b)<n(Cx(<f3^uOuU>Wb=R=y2$I8lE!_GUhS9XdG&~!tAn1kU_0ZxW*5~
zpVGXNtrD-r*NB=6?-o)Pa^t(l$;YM3dzW`P4<m0i?`|F^-b;Kt`Tp>H<4k9}!@88i
zpUal(HMc8&s$iGkJV8xiHPNY}IU>@+=0Ya~KMMtmF-vZfcq6_~f<bDpw2xf1ytO=+
z!fZt;B`GC6<rr0KbukTY%~q{2?Tgy`bU1YRbj9@C^ycbB>deqxsQp82jS8>IQ^mCk
zMM^9x(JJX`ej0Z)6*Yx4zp6*5sw<yUI4_$kds6<c0=uHD@<)}W%KsD&%YBr#k~k;g
zEYvG7ncsr1f_pXlYUbSxGyc!}xA70#pY%V>zXgBR{rLTD!)Mcv6F>a>u>RxokCh)6
zemM93(R=X^+uupQQFvMYwC|zaJ=@z1H~-#Pax3|E!>#C>x;H*u6TH6mM&T{KJ8E~=
z-ATT^{nor&+ipI*_UiJq%L}eBU-7>5>!RfqiR(qzC9WO6+HkexO7oS}t0h-gUg5s1
za<TgS+cS4h`JZ7uA98-iIp1@0&I+A*aw6%-l!MO?SRdd$kalqX;qAwwPX(U6b>8;k
zo%3a9bI-_}xqW)VnTYfDSDJ4K-g<kB_m0+`sdqH)&Aq?&q0JMyrwq@+UM_r-^tSa)
z{%h^m4sRyCTlO*Yi`e(ozpVegXL!%V$-IzpC*x)&MW&|=ehj<}k_?&*oeYN<H!>Y#
zI>1!Uw1nvovjxj9CLv}QmL)77SZ=cZVyof!$SKIJ#$&@Xg-eAai}e~4FXO}imH$uu
zxBKt(ulirjKa;=Zf4cto{gL?d;J5Q{tzUvaj(ko2viMWhhxqrM?`q#(e;fO*_^s31
zqwiQhRDHPnq3+|!Pf1@?z6O6?^R@Bo<S+j|%YRAya{kMSuM@ws{!037_xs52lYe6V
zDE)E$%k}TVzf6V-#u-dMS>~~4aHa7Y^SAQr^IPzX3q0aa<y*jK#1A^4a5fh==W8}8
zc2y2Tj#iFx&cz(otdAHi82ta){GRti^t;cu;BO`0xW7&McJ+tcuL(cozP<ar<U__A
zuNQNl?09(QUf%7;H}+pGxEOXp^ODY`?2E@QKDqe$lJ}Kgmp)%?y8QDB?=^?(fj2wv
z?RaebRPp)2m;5iCUi7`#`PA(Z?<4WYIZswR-T8FJvv1E|JX3!f^;F=Q-m^*1H@?~X
z+39=Bcix{uze2t@fBF1H<@>*%Oa83+TlLSG@fFJfc4ICH9ygxVTx&Vjaop#+%-za8
zg_l8qo&PFt3;#($H=$gCBtBjKwE~<1)A^J5^?7}`zOp;8U*YuNS;pPU{gG!PZw_xg
z&l>I)o?G01IFdPEa~pEM=Ul?A$G2XfNLX1kO6;=4bm<D2rP6<;+@zW%&WazGl#^+d
zy&{()-zpa_yGiz;!Y|b_trlHt{Zjoe`i~7)8NV^nHd|-vZ5(QJ-}tfVD)Y&fw$>MH
z)a(--S2<31SZl9u_tLu8YO7_TMXQB`)eox~){?d#ZKZABS!J7N7?&8B>f7m8>%Y@0
z(e2Pq)S9Ukr_HF_rSEAlOYgeQ53L;a4az$dw<@T~f0E&no+qg%;U(@X{zi;R)Lv*g
zzYA|W_fIZ$uHWo8Sgn{p{=4+s`S+w>PQMKQy#H(bujZfkKgGYXfBgRx{i*oF^~dq|
zmR~8qQ~tjAXZWA<Kf^!0Kfb>kelz`v|C9FT@1J*n8U9}Sb>%0=ubyAue=Yod;%^#5
zG~;x}bBwnc%>J+aTl&Z0Pxv3VKcD{a{+sr{g-MS!kxhZUgDr!lk4c~5&YxYs&i?%P
zgX1Ujuk*hQf4lv5{~iD9`S&SbO}<?D^!el4kNZE~`{?oM!^h7bBR?<r^5+ZJ*NI;$
zKdXPL{aF90`HRC>yDw6oPkoeqfA;m)7kbZ@JpTWX<Du`vqQ~0L_+FH~*#7Lo;{^}R
z9=P5&yMO5Zx(BZx>OW?A`t<qNS2k~5-`Ty>eOLEx*E^GU2j47yZS$t{t;0Lr_ZA<_
zJ}i2_`+fa~x{uzU7(c!Hu=<1MN1l(_ACG)~{q@IJ$!~7oZ+x@)vi9Tj_f~HwzyAJe
z`780)VsFadTz}K`w)kz#+xEBnU;Df)e<}F-<Ljh1rEi$uOn9CCdgklxudlsY@$&ME
zr_ZIHe|+Zf{QC0;&pV$pzF6^M_ls>WLS866U-b0$<Df^u4;S9Ayqj=G>`u;|mb-KB
zncV+%ujX#q?TI(FZhXI<bc6M#$n6_<MIQ(~QhI#)QN<(OM>8L)JT!T5<o>S*B@b6W
zxbQ&c;eiJQ4|N`!KV^R*@w)hJ`nz9mr@Wo^_Tw9iH-T@~zu|tn;BEQ4s`pDiZ2Hvp
zwex$*kCY#mzwiIP^83Ya)4sj@y6JPuXRR+sKVEoe{%-oaa~~Fdp7;6WXWMU|zA1g1
z^)32)@eiBdhyU?2S+o3RImfz;&4^u;b1JtwZ#{pckg3ou{ug}A{0@9Od7tpE;YsIu
z&KbZR!~1~0T~J4GHopY#Z_Weki`jm&9%L<NO=n4B(q{~2-p;1TKAUwl3meNs=6OsK
zjO!Rpn0=VuGk$0GWzl3_#2n1R&9aR7A5#j`RmM3Cn*R&`&i-Bf`}gmCzxn?0G54|B
zuwG<6%(ju$jX9Rlmm%f<#eYHn^B8_HtY8pfDEU9}pU=O<|HTYSj6O^POlSVj`<MJr
z`|rCyNB_?JWAKafN61&zFGk<q{8;k))4yp9-~KoKumA7*zxn@>|84)x8D=uxV!X!K
z%&f(>fy0OUChs+VDZ%N2lLV&-`14)iVc^}#<HfU-H=O@If4aafft&p0d<ncKcrtl@
za9i;F<q79?;A0ecC{Qh^B|KR~N7O=Wt@vsQP04GLU!|O+w@B@gnlH6f^04GrsWsAj
zWv0p*E7U1|Q*2b6rl_fur&OyvS#_p*kLG?YU#&HoQ#Ds<q^sMgx2dP9YpS!WO;qks
zyr^KNP$8cvKSjP%zC(Vi{8@P+1!l!#iiZ@I$O+4KNuQQ<mOLcMDs@8gw?vr)kK|;@
zO_H;u*km@!?vrm;=$H4E%aXY-X)7^BJW*UqLR2zT@`=Q0aVxPEB9g*u1giw%_-ptM
z^78TK@+fof<<MZ8!F+|`!@o!W6c|o19%kObn!`4Qbs9?#b1Y-{|5N|U|JD3sXV}EZ
z&ZNt<mT5jyBU2PJAIl-;W6XIhH7tHCt68?OxU;CT$gn7}tYGP9)nzkho6Oe0{)+7!
zOA>PqQ#qq3Q!cX(OCReMwo7c!*tW10u)bsY%3{Fc%d&<wkL@GdPxf+->+Gx9KCu2`
zeaPC%YR|fZC6dLBrJwmXQ$Dj5^B+bH#zF?q{}2Ct`lt5q%Abk9XaBVQ-v9OW=i?va
zK4`wb_NM&R>KBpE{GU8}#QcE!PWH{c*Y{jixe|O?@zSF6CTC`yFgtqU(AtCV4n!PS
ze_+Q!&BLBYHI6f#$UISU;?{AN<E_V*A5%SUe_Z(Z#N)e8JUCf!s{T~QDTPyEr)*9;
zp4K?E^5lw>$|u`S{5j5aV#>*Dr>oAsIcIUf{bI<a-IvZ@^1m!_+2PWGOHVFuzEXMZ
z(v7TJ*KRA_6}z|P{?~_+Px795zPS6M>Sf@o`!C<VSpUN7#oOmoUhH`3|9a|M_4j|@
zul-o@x$(>Xub01heh>H&`?Kcf)}MQRru|6x#{5O#<F0qUZ>`^Gyt(~m$(!`oVy{?U
zR=$|<;>L@(my=$xy<vTO;O)$JFW>dQoAB=HyRGl$zEgOg@Im!c`R9Z$>fcm<u>PF%
zGw+wl?@51F{j+7TWIV!nmqFow?>~qCzy7B(9An^TtYj2ndckDQa*s8FeFKLwmkjp|
z?k1jfywQAX`QrFj2nY&E3x5?}EV5nXp2${FS+P1%KG77>m7-Fj>LR;@8ALXVoEAMU
z<|V#HoL|CQB2gktqD$hfM6<+p@o@1yVxnTnVvoh##M32CN<NUXmQj{<l1-F7Br`#(
zUtC%wTELC>G1pZNRgPrNWNv-lnS8bU;rw=d8+l%F1#miYc(6ya-C}iSox`HV+RSFk
z?#AxJp2Yr|eK&_F=Wh;cPB|_f?oRF}+|zkFcsKFA=HD#9DCjBZDOe?7#Lv%nnn#7Z
znNx-1Cz}Hs2ir8(dKPo$!%V%*A}poM<;=TTYFHCkwOCKFOki2dV#m_KtjK(kNrX9#
z<uR)ado)Kerwo@AS1@NH2Q!Bi$4riI94(x#Ty<PRTzfdnIDd02<j~+?WG`d=%N)d{
z&LI0w;jh)-Wq-B){rdO(zdJ+L|NVb;{^b4g_;LPQ-?wMq7Jk3?qw!brpSyq6|K0s7
z^6&G%C;ulicrv;%DKHl>yEBV1Yca<&moXn^KF_j%Z7zoc*HrG+JZZe&d4u`5_;Psz
zcvE=gd9!((xUX_v;Yi|;;+V_+jje;tj;(_=l4T>4GUG#rGRB#VR~XruXRz?H@v{rC
zx3H<QNwKxC>9b#F-^szmWzRjCM}aSgKUu(But@N%K%BrwfzyIMLejz}BG*L(MZ-mx
ziVBKt7IPHeAg(HLTjH){m(*6N_fpl;sxs$fjAXlIKghn3T`ub{Ya_E#YP+PpWTr%k
z_<yk&aXE=biFAnu2?j}T$=ecq67k~q#7f2fh$f0Uh~5x&5t9{rC0Z#uPvnuXv@na%
zBY}DR4tz?yzqxmDsc|`QNpe|p9$;U_%EGL~aQ4s99|B*Gev<pB{=V#O#+$3J*Ss!!
zt?)YOb>JJzx8L7pyvuo4@NVt9IqyEaRd{>q&9gTj->iJY@TUBA!E3YEZm&FE+P$cM
zw&;n{<Iacr4-VWvfB(Y$hxZj8ggo$k!296p{pkA*_ZHp#ap(W-HMgp7PPpNAJ^5<t
z<=~52=Tpu;Jk51#$_bGZ*~jl6bvt5s*!xh~!JP*(4`?3za_Ib#%wyRnE}VLGrt93v
z^N|<2FMPdVc2VbI_{E(U?_Yd)@%F{)i)StbUMRV6=fc$st``oRuRI@eKKT5p^S3Xk
zT(Y?Q=`!n;<Ck||K5_Z;<(XIZUe&n%_r{Z3wRdjcy?O8a{Rt1ZK5l*b>G|^4I`1t$
z7JfSP`R$k1uU%gSzux(>=<AMeHs6<j-}%G!*SFure<%KX^nVRwG;=A78!Io{A-3i0
z-#H?=g1CQhC-VgJEaI-=YUHfsWa8rH+R53@>BzZ*LxN)(`$G0Yjxf%*oJCxm+)KC(
zdFJy-@J{Bv#jC~F#OKa8hxaJ23IA$=>4F@>DWa#v-$|~N-Y&CR_KRGP{9d^QvNE#g
zWfEnWWh7<9WuMFGDyS)NC|p;Vso1QjuJ}P=ze0$@9Qj>x_hcnxi)0>2Ye^?a@kz~>
zx+(om#z^*?%x#(1GVf)+$%M*=$<3GFqL8V0UC~Twzmm7|dgW)zzm*>;Pg35hq^DRV
z&nqV-yH3VWW`neYw4rp1)G~>4VjN-%MK6fV6y_7E5b))H$NPlak?Rqs7FPk+PA+}!
zzuXMG-Mpr}F+5J(e>qok#Iwt@>9c-gF=I_;<z&@n(PvI%vSh6O|M{=;--~~^{v`a?
z`)&H0^|$jc<)3Oln!aE8_VcUT*Dqf-eR2FU<@32u4?oWT5c)p<UGdxduXA7Rewq7{
z;brj)&KF;vpL{;;nah*=4^1EN-nY4Df6wQ>$U~3E22ahOeR!t+LhohFOTL%)UwnUI
z_p<b5!^`HEO)odReD|{PrP&Lm=NF&VK3n*7_Y;*TcOQRwEdS*H<L8gPpDcb7{p951
zxW_t=Z62?BZ2qMFN&b_zC+bfbp0zwvd%o-i>+6YcFTKC>QRB<)Zwf!_elh<^__vsW
zgXtC166PwFF4jUeK6W_{1}<5iqr5EqxdM4YZ$;e1B_&Tv)yV9TRhHW&ds613^aSZR
z>3>pFq%@?CNQO%GOL&Sai)D(=7r83DN|;ZiT0~fsTkNY?zxY3KVTr}!JmO_ydqtf^
zZA61aO+<x714N&RGK%?&-V<>Woh}+6`d`FK^q#1z*mBVe!lwkY1Wfr`c`tFF<f`VH
z!&SgFn^T-~F=ruHG?y>ud-hdqL9FMQ%b89v?qrl_+Rk*J$)AatX)@C<rUa%QMmI)V
zMp4GK414}}|6BK0<L~r8Cw_nW)%t78FP7gjzlDBl{7(4Y`1{ANj-Qvl@BU`@ZS&V(
zU%I|r{j&F~@weWu=f7O|Qvc=Tr+^Pk?@ZnteC77C^10zN`)9A7$vuyF-uYt1tA}s;
z-c`TX{_yL=x{q9+#6Kx~n)|8j^Y71>Ki7RO|9t+l?HAK8ouBV~D*m+dW7UVO_xIn;
zcvtZ*=bi66y>}MxINlY#{rG0*o6<LDU!QoD{&LmxJx}$XxId15B=)fI!KnxHAMo9e
zxV!Ss#yi{ZSlr>hEp*G|){<K$w~BAp+*G@D<QDsFr`vjWc<%kXul{hyL$^mK9`Qal
zf86v~^vRDWvd>(e|9h_gV#$kZFK@lN`zrO7)2ozM^IqM1rT=>W>+5e!-le=(`_TR2
z{0GU8svqxt=>4$tgU-jVAB#V!e7g2=$EQhO+`p~)p8xaW@9=-R491KtOlw$T*=Df0
zvzf9<u-#%k$@-pkBAX1yEzUn&nmn3(;R4qLHwonmGl>L>1dAAnJQgk&wh%TGzAtn{
zFi&6se=NT~|4hE+yia&`@Z90i=l#VSz<)zPMCgH#vap!2sIa~8BH?I}e9@C)!V)(m
zzDT^2;E^nnl$2U51)60`lwB&9B_|-aN3KV{UH+CFr(C1#a@qf~JaXA`=j7)qURE+z
zwor~x-m3gmdAV|gvWIf6@>J!O%9+X^lp2*JmHZX86rAL3<Uh*SDHJGZDtwjSDQ_t6
zA;%~CO8Sq~PsxQ6?&9`h8$`oJ4~c9OP89kq(8{mE*U7V-D~MBpQ-L#qqmkW)U5MR_
z?HtP=rW{6ZhLiu&|D6BX@m=lf#7`+79=~ILcl2%l+pTZzzvg&z=S}+CuD5I7o`2i<
zcFUWYuajQ+zufj>?Tf4z<<HxmnLcBA=Jk~K$-ze}9v*)n_u$<9JNG3YWIR~(p!UIq
z`=9TX-wVBGcJJ-op1TEir{68Rd;X63oqxA)-)_Bq^VZdyCO2zt-n+T<=E0jhw`bo8
zyBl&>?ykh$sJl&f1Mg0``{>?^2kMWC9$$UZ`mFOs{HqDC?cSV!6Zm%a+tqK^y_I+;
z{@(1v#}7ImBR;<Qxc8I8XOGY6pV_`heJ%WU^!vRZM}BVo)$x1TZ{^<`f9?Lo{9Ep~
z=x>YP#lIK*KJxqW@A<!<{|@@g^Z)<<Ukvw|zOclwIkA6W@8^)_<mYtaC}KBc7iGV|
zww3KGTQ7SdhZ3hArzGb_&Kq1`xR3E9@gCuQ#=C@9g?BcO6OR>75l<Y?4DLf*Rb1s<
zrd)eDqc|-%EjS-?d}cqywt;mc%WUR$rhdk23;_%`|Hu7*^e^b&x4)eK1pn>*yY+9v
zKZpM}{|7NFWzc7|Wa419W}d;kh2=791Y0DV9NTwRMK(*eqpY^9)~sQyUszID0$D^@
zv)E3sui+5q6yV&#VZ||xeJ7hU+YGi}Y`fX`*~(dUS*=-RS$kQ;SU6ZtG5=?J$M}W8
znc>d=W&geY_y3#v_xvBPKXZTY_$Bh|#LteO&OcxOaQ(67`-<;tzH@xf`8MU-OA3}%
zejf?TD!-loR`t#CoA0+R-%fwy{U-Fa{mbLek3MmII{uO2Q|YJ8pZ0xn`Ml&y@;B)p
zlYaL6vibetch;X3e^mea{QLL+7~?mlFy@`i9V`K?hgg-^n%OR~iLo29KVWNQo6h!*
z?J=7TTL|k!mS@cB%=Jva8EY8z7>yY>GW`3e{a5@C-|r<qYkvIs{_VTgk2~L|e@ptR
z_EqNV#4is%2Y-I`X~id>Ppdx0e|-IM)hDLUii2wT=jX1^MPH76RsHVpgZXFj&nZ98
Z|9tcF@Xt>_SN+oZ4Otty=hvH`O#nK_LPY=o

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/4e7cbb27667bcfca92838aa8020749990013a9b1 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/4e7cbb27667bcfca92838aa8020749990013a9b1
new file mode 100644
index 0000000000000000000000000000000000000000..898584d96f6bfcd76ebed9dcde3b7df88442af78
GIT binary patch
literal 66423
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bG01{z<0&gh&4@$$Rp#T5>
z!`P$rXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD
zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinpbY`G
zLF=+nhmD57Xb6mkz-S1N83K-E>KoNI8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*
zAut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?
z8UmvsFd71*Aut*OqaiRF0;3^7-w^mu381f2M{OGofzc2c4S~@R7!85Z5Eu=C(GVC7
zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z
z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7_=eKFlb#i>afue7!85Z5Eu=C(GVC7fzc2c
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C
z(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7{nnkx)65|N6x6zMnhmU1V%$(
yGz3ONU^E0qLtr!nMnhmU1V%$(Gz3OY2>cy6ULE!6Xb6mg(GVC7fzc2kHv|Buph`Ue

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/585e469231d202812bfba8285fb30c8e31c857b9 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/585e469231d202812bfba8285fb30c8e31c857b9
new file mode 100644
index 0000000000000000000000000000000000000000..a4994c208300b6cba20708367337667fbbd8dabc
GIT binary patch
literal 70833
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#P;0`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O+Vxj#;S
za{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W%MH($Jqvg`;W7W?d5;;N
z#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQe5UfE=4JaU&)3^uuX}Cv
zI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()m0t^gul~dSck3VbKQezT
z{&fBk{VV$K{QujGj?5>SWm%T6__A(f^<>pzdCQc*^n>XOvj@v}mN@ovoR(Z=oYy(R
zI9xc6aFlZ>vn#SqU_H)az#_o%ky(~y2Fqj?3l=?=f6PtH7R<MqSeVKf4>J}ron$Ix
zddPT`v4l~WVc)-l|IGhC{Kx#y>Tk>+o!?zQtG-|R8uvBfYt&bcZvo#+esukm|Ml>P
z+qcJ`cYXAEAMm#DjqRI;H;doSeSiMr{m->uwZCot+WDpL^X$+0U)a9B|FZ1MtS<|`
zn0&eUdDoZxZyUcK`4RqW<8SjnHGdxdk^1}iPud@gKVSaD{#*I~2g5_ALoCNx)!C}p
zPO*JqTgMj3%D^nf=*>{^-{T+uzdiq+{G0J_{XdugeheLqY)svZcNm-)a{t%<Tle?w
zpWnZ;e*6D!`+fKKyWdxScmGcM75n4FH}!98zE1e!_36qx?$<M(uYEG-(VmA>9_BtY
ze#rB%;X%NC`FqZHE$-a7_2A~an>@EK-C1;x;Q{}{#7Di4XFQg8Jmc}4Cug4YKXH1x
z_nE*8o0opCj=%c+s{Hl;*L`p7-);S%`pNdw<&P2{g+4gGZ+(~gw&Bf=Hz(ivyf6JI
z^u_R7?stnH$A3KdarlSzPxoJXzwiIv@~7l4-#@qiM;NEDRIn{#Kh4q1S;LvmnZn7>
zd6VM}#}STm9G^Kuxc75w@bvJ?@W0_-E3j5@nNWf7eqlA?JfRk$>q2ruwgQEGvw6*V
zxANTLN#`>W*e<{=AjN-#cLjF?X9R~UM=M7u$90ZnT$;Rl`KI#U5_l)bB&00lEj&v^
zQPf#9MKo4aO7x=02a!HePqDLN9^!|@jU@UcR!a0r%#tvbh!^`Kyiq7pNK!~bC_!k4
zP?Yd#;oCx{f-M5x0)qTYcy)Qd^9b|o<Feqq$S%YFl5HQGDtk7^Mot5+O`M4w``J`k
z7cr+YWioa${QNJ@Foi*!@h{^yCU53x%nMo4Syfn>SSPXGVx7e5$g05F$9j^jlEZ@Q
zI=3kACEiHBYkV5~F8oUTANelw-Q@G(za}6dyjG+^bg$SAaZ~XI(d8mDM50BQMQ#h9
z5&kJGE>bDHTxhFMp|HM)hG>@<t3;7xjZ~DBwd8B@0<je$^1|6dIzr+?YlNyrZit<h
zV30Z~^+n1=>ZpW<c(16k$U~uW!N>dy_*U@F<C(yHhpU`Ri)#(%L5>FY9JW2I?5x+A
zotT6e*D)+)NM|r%&}GnOFko<JFksMR5Mb<Ka%TR-WXGh!82jJ%Z|v`jKLvl@{ju@K
zwjV)1kN@=isr_Tix3VvdANRjo{^r&z?w2#3e|z@jnc(x^&sIK@f7bDI*ONt$86USk
zj(bw_#NtWSlbcUIKGA*p^y&R)ub+Q>apPs<tGBOQUU$8&d~Nhv>2=ua_pg_{75{MO
zqu^($uVvqF{y6bd?04Cpi+?Tu^E31_Y+}5?^qEPKxr4=yEs9-+V=)I0XB+2wPBBhl
z4mS3$tahxwSnjhtVF_W?Wy@qg&M}4aJLe)U9qx-<He7LBXSl9%#dFnj&f>7<kl^^w
z-oSB`Q;_=`w-e6-Zf~wf97XK6SzTF{GPy9GXL!oc!*KV%=l`C6lm0II6a0JePtKni
zKkxjI`9Aq;+ZVUbS3kObIPuQ^o#^}D@7X>wezyG@_bvCE?6>r<Pd{(?81X*h?ef<J
zuL@pGczyWI&$o---~F)R!`JsS-dDcwe1GJ9+4~FcF1`zSzwN!#hwzWrKh6JQ@{Q+v
z-}l-d^*^nDef@Rd_k=&||DO4`{J$?l2E&p6>;ErixXob5u$Cc^Nt}5<Qvp*U(@LiE
zOfpQej3$g4Oy`*Vnb}ziS*Ef0vpi?YW3c{r>d)ri4}Nw3di~4t&;7s8|5^VR|8Mbc
z_aCX>Hb3pZ@A*3Ki`Hkek4*3Hyb*c5;HArpkmnZ9Q=e--?|ruA>9i+-kC`8Bd{BNr
z;9e(au=2jmgY^&IK79E&{;AG0#^>*zZ+h|Jg~bb%=lh>+c&77g-_tiwOP@79S9m%5
z70Vlkch^7Eelq&p_Brb_-{(i4x<CE;_~hf<PwPH+eYyWd<m-p8+}};V@A`K5o6q-#
z?`+?BzNvn-`0DVj^8584M}A5D4gW8~_>$3)=^@hv=2<MFtSPMDSTot<I6rVX^K9XL
z#`lo_k^raRa)B(taN%;1KoMn;B#~(%Dk9xN3k7cTpXHC@f6u4IzlJ|cfI;vRe<AM_
z?n7LgITv!2a!7IZaP{&i^GWe13H%V07OoJs7QQcJCHztNg|Lq>gRp>bvap)Sc@a}l
zelb?@w_*}vQ$=+}^F`(h^9U~!3KL=#;t(nlY7qJ@$S*WoNLJ{IfR{jkz!UzTd`-MF
zxr4dZar$xivZ=Akv+iPD&St}Yh&_|jkDH(S3x^fE9NQiiL*|E!e2hVi;Y>QrXPKw5
zgt0oZ1+&j)=VE7If51ME<0!`*jx>%N9DH2t+&$b)Jl}ZD_yq+f3M>{lDc~SjE+{0J
zDZngHC9p|whR`#iG+|#6XHgNcSz^NC`y@Q1Yh*ZNE=a4%ypsMabyV_=M8Bl3w26$9
zjDW0@T!>tytbt64l!D|&2|bBuacS`mae0Z;5?Ydsl1h?iB?YABNQy|#lyH>zAbv?~
zzDT2xs-UgF8vX?STYLe0EPN8YGTbtp+u8Hj7+CK!r!)63+p>hPrn9YNvu4w0^J8OU
zea~dWw1=sPNt<!oe}{kH|Lpud@#p7ni@z@Wn)!9t7lSW1KHvZR{`09%xgRRudAwzR
zv+z~*%Y84dyl8m2>80_@-7f-Ptbe}m*~2FWk5nFb-S@q>_fGR|gWGFwZ@V-9uE)KJ
z_cuMPeeC*V^^@4A?a!V+=Xv??<<3_hUSEH!`{B{Ym!EpReE%l*^Te;JKdt|y8TT>2
zW7)#;hUFY93tJm&6zf~o3if)=Dy|GJW^QvHHJ*c9+MLJOm$0$1EoNQ9x}R++M+Vmc
z?pZuPc{cEV;k&}`Ef6laMJP|WQ&>sZRH$6AN^rJdrC^^RlTfA5M4>x^H3I7VUwJcl
zD!CSL$gvBt)w7zjequ3YS;q97!T$g8Kc{{={;>S9;=Aa#qhEx-tom&6S@YAG4|6{x
zeXRU=?nBv!#UF$|27e6vVDN6&8-q9gugzY)eA)bJ|7+W~Z{BWtyXCFcyXtpW-&wv_
zeE<Dj+WVdFt>3SE_v4-HhdUpae-8WV`91FEnqQB9&Hd%~%ko$BFQ;F3exClh>(}Bx
z2mkH-zw3V;Lm*QD^D^dkmIZ7QoX@zNxo2@jb1mlL;abZvlU<*EG5a5mdz{BPBe=G4
zuj4+$IhlPb>uP3krr8W;|Bw93{m;(8#8CTx^S|JK{r^P&P5U$LckgfB-*!Leemn8G
z=;N{X0q-8Z`T2U$>%`YzUva)V`Qq>MW6!IfpM3WHY0T5ar+1!SeCqYo<mrT`Q=Te3
zEqh}Br0?mj=NDhBfAQ?a^q1dWGQax%GWeD1o6GNPKlpw4|6$H2&M#}f7<`fZ%=}sK
zOVO9}pVxmr`la&Q+wZJD{(YPK&G@^^_wH|O-+q3T`KJDD?$>o+3cpx??fB;RW8$wD
ze|-MWWt3+=%<Rtcf+dgjFY8h^Uv_VfJDhj8FY-$9_wg$TFbTdASjR8Rf1m%8K&xP+
zV4DED0H;6(zan2B?|<GKe4F_1^QZIg=RYTqFObZ)m*+G0ORhzn2RRmT9OFE}?ZEq*
zcNJeRe}TXx!5$%1VHJ^T(dVN2qE;gQ!mET13z-Y23a=7Q5n&PK7EKj#5)KlQ6cQ7f
zBY0K7Phf&Tj^JTI5kXsl3;c@w*Lfp&>ba$O3VAzud3bucW^opBykfIs{mbmhvXzyK
zO^j86wT8`<eHxn=D=W)<W_1>Smha4qS$f$zIF52Y;I!k6;M~o1j{6$-C+_7ug}k46
zAMz^kcJkcjnZ+y57spq|w}|f^UliXHo_20UZd>k5?s?p7JTG|i`NRaw1$zY>1pf&B
z7Ag>~6+S8wDjqMXBIPBODz#rqQ~IdXOG#a+RnqrlbY(eZ56V1|@sMqjnIknpa;s#Y
zB)g=pB!iT+w5Cj&%ywx%>Dkh^q&G;HOS?(SO3#$~CE+0!D|}gCBcBehI!`rsEO!g{
zL9PcJ$JoSJ4=^ug>ST;$NcjKupWgqQ{}~u!|Nr@y^uPW8qkn$?R{XpC{~5z328I9A
z|8D!U@b{aau0K4#M}L3uz3BU`uYzCOzUqFp{LJ*R{C&~8Rd46N34J~JRpzU$FJC`T
ze!A_^iU+;-^Y87w6LfpiEzMhsH#c4nx_0yG{%d77TyJUI(YxpTK=o10W8Ei@p0GYk
zdw%0Z^(+53ZSS%_mVDXx?c4VoKTiL=@T=>$#h?Fwe=tm7zQQt#bsO6&_B|ZeISzB|
z;oQS5%gf4J%TvO$i)R9l6So!TZ+2FWD2^o@D>>`AC-SE8`w8d>cnIVQ?hz^z{v&)(
z<g4gSv2$V%L`6iuiU^7FiW-RS6rC=XB%UcgORPt9f{3Ybw%|N|GrnkEP2No2Lf%H6
zbzGX9wj3AO`dCgeIWRqDY-14kXZ`!`51H>8UvGZW{<!4Bz7Kmo=za))@ARJG1K)>t
z@9w=__x8iv+;<l59X@ROIQcW**J)o@e|`A1_1lK;zkVG0S@tXZ_oCk?el`3||1s^m
z<M-C@Pri%)Nc!>f$JC$Ce(wGm_|x&{wV%6xP59mLN9LazLkr__Mh(V8409P~F|1{f
zWNc+x#iGwv&$g0Hl|7jKAKOf}2DaU74D3~G(yYuZdzj0a3z!ZuaQ?se*XJ+$-%Ee2
z|GNB>_%Ht7?BBsZn|`<bUi~}wH`DJGzj%JV{t^5=<ZJNf4<BVd`h19eU-oYD+txR{
zuSH(IddBm#<FVl*iH8dw+;~v*Q1VghqxeV9AM!o=^Jx2HuP1!ZR9>EWE%Y|$ZN}S-
zw^QCGzuos{-<#iWQ{Kyci2bnjL(j*RpPql3_fg<O+56!4a__z0i+@o1$oNU=lgmfu
z4-W4=-#>jn<pbl#hL2&N9)4Q$spgZ!XUi|uUvGV@{gL*o{Ez#;)c-sTGa1Agr!&Sd
zMKI^F>|^a@593Van$9(k>o!*;w=9n|Zz!LlfVa>AVFnQak<}vRqQ#=`Mem8-6Q3@z
zM<QH8K_Xg0LSm{|l1Qx30f8<2jr_s_3W8xm#=>&KGD7bKk_9#jqzJwgEEMt=wiQ_~
z(k7xH@>@7l#6t9)Xpfkw_ziIti5udN#b%54i|i48CX^_&TqsERi0~cZX5k>A83Nw?
z34GUiSh#JtTDXF_WI1=UpJ%gV+r-MoTEMb`rJVINt265=mUNap7FQNa<`BlW|1bQn
zXXt0NV=iE=WWU5A%^AjF#9q#}jO`%%8xAK<0nWD^WgLa<lh~@+{;+*#592(;Wy#aU
z8^D*z=fo$(w~^O@SB6)Xmx)(`_X6)}z6kze{?+_k0#*Xq0)m1TLVJa!L<>c)i*$)d
zik=Zo7fTo4FL6LpLh8I^r6ik_hxB{tWzuQV<<id5|D>{|=g8cV&64{fyF<o7T3TwF
z<W@;RDG8}dl75mZ5*A|SB9Dahg|-UH3I+-!@n`Xc^K$THac$tRW|v^&Vg1Lf%>0VU
zmH7~}FpD6IEAuJFHii%X!~Y-n7xMoD!+u6lMmvTl|8xHb|9k%D-0$k&3BSvJeg47v
z<JI>a-xqx=|1$C8uXj#w&EITz{rdH)HzIGF-aLA><;8|)flt>x@pz*DSpT8X{WEvl
z?pWL|y*2Sx@$LP0zTN$M&*8z^hbtZ>KF)bE^XckmFP@jY{Qs)&jpaLz4_`m-|19z~
z`<uWIv!Bm@#{9nbSCe5rV;R$LrVY&RSQOdh*mrTf<|^R1!n2p>CC?;YeZG~v3_Q!Y
z{&Oa9-rz9e^yPBk&ftE*70IQ|Rl(K79mgxr&o0m>z$3^Y7$$IwpGV+~z$QU+p?IN#
zLMg(>g(nMt7Gf8&6Fe-yBXE*$GjAd9PTupp8oXP%w{zyQ-(^u@^8bJH&$QoRzaxI#
z{9g05=*#oZ?4P|pX@7G4^y1^84_WUYy!-O5?_Jbep*Oy-)n1=??fJI<z3rz(Ul_kx
zf7|l)*%z%ZpFW3wdHLn*m*6k`pLIX8e!l$a%BM-6WxxLV*7&3UXWTFO-<yBG|9$*-
z-f!36J-_AuT=_HQZ}Go{{~s|}ForWE|JVEf?O*#p-v1LArZFyK>}8zAxQ_8Y<4eZB
zj2D<#SsYkGnOm4jm^hecF$XgnGKnz8Fl7AC{g?AE=f5+96obZp{(rCk9Qpm@*ZyBk
zzvli7`oa5s!q;z~kAD*Ul>KqVhYRmn-Uqx(eXH^2<g3V+FP}es7W3@M)2^q7pU!x8
z@45QR@|SyFe0g5|{OPlfXUxx=Up#&_^X;E^4evPKU4M7*{gd}5A8bBae(L%p@QL$d
z&<EcSoFCZU^S?XzcK6#;Z!6#Ne~|z5^7FAT1z#LLYkb=B@$yIGPko=7KYjUV`*G)o
z;t#nWHhhTwSomqrXO^$F-*Ug#|ET%t^6TEOqTdF8l>XNI3uZ`QiezzO<K%eFaf;(8
zM?0rK*FUc5JgofF1kMQ@5il3LESMx@Dcmb!A=V?lTzs?GPf<Iun_}C<`b8fLHwdi}
z{4C%s5F=14xI;)*c%M*|keko~q3y!iqWi=c#UG2^6I&tXDV8ASD}Ga=S?Z<qH<_8T
zH)Jo#X2^!i{E%XmN|$^pu}GpzB1K}c#19E!i4UR@B6o#c1e^F5^KIpe=KsSV&hO28
zkb4pL8g5hW=UflCwsU!NHE}NFu;mEhkm6utFK1O^sb!wUe2GPlJ(u$qR}*(NcQ2PC
z=OT`W9G#pxTy0$WT=HDZTn=1^xh8T?;#tYNfv<r73_qs;x4=yPmwdT=!F(-zhxqRD
zY4EcO{1nI%Oc1;!@QJ^bpF?1#z$1YNf>VThMG{3CMK+46h(}A@l-wviMOIjTjeMPa
zle~h$3<VnnRrw3De6lLCb+TV%AIoVe99Cdc@RZAy5tWXYGL&+WdL%hV;*pq&Xo~PA
z!9am#eoKB6etrH|e7=0%ydSvRxqLV^IK<e!*sidiXXRp3Vbf(xWs_oC$fCj=$JEN$
z%#h7s$oQMlo9PJS5{9S$kNpq$AM<bOU!A|F|1kdP|MmLEh41X&Pki0;x#;7x_d)N{
z-+q6?_4dS@nAe&wUp=dMBKGLmg9#5@AB5b0byw<+=Pluz^KR_E5q?wn*4A73x5e)G
z-buQ%;g0iN*L%zE+dT|@%=GN!i~83I@0>q&eg62R>TB26H(wuryYzkakHTMWe~bRB
zGfZVz$heXzp4pV8lQo82it{(8HP<n&soZb4Cvvaj+QV7TsmZ0nEz1+byNb_?|1IAQ
z-pf3PdG7KC@Y@M25;!PuO5mcv27z{g2Lcxb9}682b{8oWIVs{Usx6u#QZ8H|lq|SX
z;52^@p9-%NPdoQYu4^2JSnn}OF~<IP`>XON^iS@eq~BgYUw^ax>hUG`v)88+AFDpz
z`S9)ij&~k!x4qGL>-l!^8?QHK-|)VB^#04oqo2Qgk^P$brT+8ePpdx8``GqL>oecy
zL!TNx*?#i)wEt7d=e#d#zkdH_|6|h6tG^!nZvM07PyHW(KL>wb`mOzE?jPsBZvQ;~
zSN%Wo@8aKYf1Lh2{eAs6-yfO3yZ@#C-}X=E->!c*|372c#u&o%ovD~Pgn2QOCX+nV
zPsT#Va}2r+EB<f!Z^^KM;VgqVL(RXsKb^nwe%$)j_HD*D?r)F3#D3oWvGc>!_x10N
zzMb$k<n60BS#M^&-t_9%OM#b<pTBtK{p{CMwPy#O&3eA(#kH4vUMamk@T&7=`-@F4
zvR?jpDfU|Y?fQ2w-ub<|`Bw1Vt9QHJSAS^!=<`YPQ^3c(56SNjzT5pS{ayaslW%^!
zx%4*uz4OPePrp7L{B-PN+y|}q3*Yg+|NCD5gXw##cX!?fz72dc^Yz5n9B)p%X?g4N
zuI|0&$3>sceg6B!|J%0jZa+W$dhmzsza?Wcb0(_}n;Badn<M*E_Kh5GIOlS=@yhV;
z<zLMIk3U;5O!&0OCQ)~>sbaImxWqn+M2Q>|(H6ZRq9f8IY$|+H*jx0l*fH_<;yL0>
z;>*SO#3IEcB+g4#NNtyTBy~<|i_}7?X;L4hE=bRok(XT|t1ss)cU4wL)>Fn(x=_kn
zszPeI)Ojge=_YA^>B&-_k}2ZTMc0Z1i0l^j7mgHuBvdQ-gntj8Am4f3dfpQ}Ts+C#
zg<Nf%s+@A12Ao!$Q#j<=S=p|$2C>a!uj72kHJMwOM}g-7w>Zxm9$CJ2{saLgfoOhT
z{z!gP{zQHjfky(51m_8D5V8?kE66O!D6oZZF7Hd85}s_HH#{-C6M6sgw(uqJt>R7L
z{m*-oZzlgPfeax}5hgJO@pf@H2`MQanIhQ*aw+l`<nt6RD-<c%D>N#6l~0$uDZ?q_
zFVic_E-#{BsBlYep-h<+n}m~SwXnYMC!v>uO#+_$MSL857kN8)!+HIAU-8`IzRcy!
zwUToQM*+Jk+g_H{%>2xaOeYyV8T}b+7}XfL8D{+x_<QuX((k{&Zu}Dc{p`2hpN`)f
zf93vK_KV}U$nPb;w0`COeD+=Q+sZG$KLvl}`{4KP)@%7!(_a?7Y<lVY^6~T9r?(%S
zy?^+Q`0cB=j^DDrrE@dmdfHX4D~XryUz&aC`=xuAKVGrC)_i@*jn_BRZm+&`_^#Cb
zbq`7(&3vNv{LV|OH+=6_y}$P1--nYQ!alZtlKNus&H2ZhpBsKh{*C&#;r|;(E|zty
zZ0s}H*Rfl1T;pKlT*kSK>jhUV*B;IqPGzo5+-|%^d<A@Oc(3x@<aXf}<KD@wz-!Fs
z#P^f;6K^TsEPi%DCZW4R5hBM$4aF?QD#a#<)rwV#35c<X-WGl*WGqxEm?I#?zlpb+
zTZLm0%VEa3|Kt8$`Rnm__aE&)mw$Et`2BU^XVp)eKK}a9@_y~RzIS`xX1v+@ddBPG
z*Rro8U**4Sddd3Q{%z5Fp^sNTiheTt^!8)X$3-9ae7f?v;LGtZcfRm`{q$AmyT^~l
zpL>2){5JWc_jlJ{!+%=;C;VUYZ{gpYfA;@g`b+(n+^@=C62JR?@Bf|oJMGt;pNzlG
z|7!j{`;W*!_x}bA%NVvYBs1DG8?Yv@39!epJF%Z;JIZR!@{-ApDTDFOfA4=ye^>oY
z{P*wQg@331s{Q%&bL#gsUwyw6e-`>I^Xbz2lDCp?1YSRU$^5eIMd=Ifms?*xd8zzr
z$E%;O*1uwSW%F|33%M8jpBugqdl~f7|K+xq9xt1pYd*X5wDD=p)BI=ip7*?1_Ok!g
z)>oHaEq(p`jpV!g@7}*V`A+J+!~6N~q~2w}Tl4PQ+nP6$ujjq8c+K?2^9{qBFRuz;
zdcWX%F8@^KiTG22XS1Jvesc23nJ4W}IiFj+5PKQ(>dfocZ+^TLdvE$7_G8zlL!ULj
zzWEmSGwu)D|H}*>O!JtNS(dQmv;1W?V_D0Z%x=f=o<p9Ck^2wVbgl+2CGO2U>3n|t
zGx&e;rwhyy=n&8lU=U~!cr4%{FqNNMpg~YY*iWQegjd8`m|f_(z)XQ}0tW=Y3%wI|
z6nQV=E0!#-ArUNDAr&S)Nk&vIP`*~-x#Dl7SY<QiWlCI1mlYo=epI}un4q{tK~lj+
zezB~%%p>V%(i@~NNDIqsla`mBD^(zAC}A!!N#cOGqu6N?8Q~tmGywzt3O;N8W`TbK
zuLa};m4!4#s>HsCmq-*!=u2D{&k;W)Rxf5R&L|NreqVHt2#biL@I)aQq2GcvLh-`8
zg^Pt9ggJ!|2`=Zi;M>L<&Ktxl!?%;Km+vZ{oItj~E<SDE#k_p{$N8V~Hw)YrG!zyP
zc`ni~I#<+6Ohdv;GF)P>_(=&JNgoMWi2{jB;+MqUiRy`13GoTp3rmSailmF&6)qG?
z5wPGp#FNC6!^6n)j!T^L0h<D=CCfGD-^?>u*jd?Gf3TcliD3D`oX?!jbd1rN=_J!p
zrjtw?nRYTdGZ_Ex|F`k4!=Kx~s(-!u_3W3+FRowdzsi3d`+4?<!jH?}7k_X09`{}G
zyTbSA@9(~KfA#q?_w(*gVISYT|M~9JJB{}r-%Wda`}OsgKc1a=oci$GeVh9~?yr9E
z^*+yi&-?rDueks0e(!@{50*aM|ET4$`jcr-GoJ5waq}hn>$k7By!rF?!uu^BCw_kO
z^~I0nzkU81{^R~H%V5T+%Jh#Zmt_i@7son|l^k<9CvaEsrt)R+74UuF`@{E`w~t4d
zN1jKQCznTo_Y3cLK0SfI0yhPe1hV<J@ZID!;SJ-p;uGW7;-AeY#HY@8gKsjwqyVE}
zwosw)HQ{g(0nt{`m7<NJ>qYW~i-i^m`UvFlHSt{I^58tk{)x4OxsrkPU&|kd-?x6M
z{B-&$`t!kefp3Xlcs@V<nEoN|-JUl=Z&KcTe{J^q{wtZ+aj&1gdi=8brPHf5udUwh
zd$-~J<M+?rpM7urKI0wFyO?*!-Zi`neCzfm=gq{o58i!!zw<-z$Cn=^Kh6B8@-h1(
z*Qe-DnV<SUJ^YmZ`TOVSFL_^lzX*R>`}xEtw~sI0Z+XA%gWRX2&$GVV`Fib}*!PX!
zR)2f`o$VL*pT58M|1mJEVrXMn^Z(F4&42xWd;aAAKJjbjuclxAzdrqZ_haR^j?Zg9
zWWTF@WB=OZb>r)&uNJ-B_hQD2!!K69aDBn>BIw1#7x6DOUrl+f_SW)U%R9Mujc@th
zZhe#c#{W&(n@exP-!6UY@z&vu*z4j~b6#=3c6t5z^})Bd-s^l+`?T`&zOQS)cmELo
zVfX#<x6W_%Ujx6~`YiD!_lwpSv(NFL5<i{zbn%nlr;?8gKX86<c`y1-{4Lwt%Wsdo
zyZdg}+x2h$zB%}I)7$vB67SZ&PyV?7)2z?MUzoqW_|Ez>=cnnfEx$YdR{XDGG-sa9
zqQ&-!{U(PS$8vTXjtI{GoO8HVazEht!~2lWmVYC^s^9^kYeHKE*#$!dXA4RSItfS%
z92Kw^ye061znuRgznP%9P_58(p?aZiK{)|leq+8Jyk&f+__+ig@LLFO6;2TSBHALh
zRop-_TI#)2fpnXUvD|t22MRkCzbM(Ol&fSbD=5h;swwcxpOF=i`5>t+5g@K5K3z;f
z^p)@lVIyH(!4&>b{%`!M0-yLN@@Me>=btTLBrum>QGi#_TJVBEoxm=Ese-eF7=_mf
zu?x8fW(q{|>+vn-jo|Cz7ZAwh|H3<mCywV4k0x(5?{hvy!P`Pr!bQRz!b~C-A{&HX
z2?Yy15<Dg-CKN6tFLYIKy`Zn4hCnA@J?|ynX?*MXw(wo%UoL1N94}HNdQa?(grU?W
z$wrA^;))WF#Er#Yh&75=N$iriC;m<Bk{GM_2eCXc7tt-kc0#WO_VEky|KqdcpU7{+
zU%@NSeS*`SvyNjDdn`K_hbre&&ZC@VoG&>yab4xI=Q_dpnA4d{o-34d5l0z^AICQK
zrEFrX5zPKf-c0>W_Dl;IFEBDNw=>5w=`c=WtYp$=y1_8>zxw|f|4;t!{de%s-QSVF
zU;kA8A@F_Kx43U#zOsIe`keZ4#e2<ni{7xkKJ`-QrN|4FXY(FMJv4aW_h9aWPxo2w
zE8d@eU+})nJ)?WG?yr7$|M9k`I?p$~sCcFKrtR&#cL&~^d=&ev{&m;4yzi<%-u!&<
zJMPcx-)VnV{I&o0^RM(jga2C@x*2U5H!;j$&}KOLPw#KupWT1v{Pq0T^w0CZ3gZDL
zKIYj>?-{o-i2lF#H|}rB-;;l5{_FUE=D*ng3IDqP7c#zQddtkfYQ>h#p3A|`*}*x9
zYb*ChZguW6oZ=iBY#A&+m|7WS8Jzxa`e*dF^4GKPN#7j5uK9f8qx1)__hs+Z-dn$I
zea-d8{q2G`U9Y%bCcK#T-1^z9CpR7+ev<gi@_EN|^B3AL_q}}f^1{o>FB4x*d};OS
z!>h>GH(xJ$bL-9N*9xy#U;cP;>*d{7L9bW7-ty+lTmN_J?^e8f@!sO2*r(Z_>_2CH
zx%!p$`_1p=KOX(y{`vC9(jUG*_<!vGvG7;bABVr{{~{P%n39=fm>8JbST?abvpKNw
zu>D|lWc6cqV611TXZZhr{lA#MAOFPso%Hv^AGSZCzgvDe{M_?h=3B)V=FgWu9{y1L
z{@dHQx3cf{y?gSu<jtMejBirk{C>0l?ZkI&?*l%>emMQ!_dWmnhwrYuJO8fY-Kw`d
zZ&}}7ezX4d(pT?Z9eKU?wZm)A*H>P<y)k^V<F&{et#>6Kgg;LB5dXpTL)ZI&cYJTl
z-+X?v^X>k(iEpdkK6uOUZt>e4Z(?6-yefZb_VV+KtuL%!etUJ}P3zm#x0ByWziWAS
z{2lB2neRnEnts;*%JtplXYp^|zsCO@|C=%RGhSutVc}*=XP4oW;O^iN;!Wh0;IreO
zE>J5tSMZl$r!br7Gm#7tbJ2-n=fp0HMu`XrdkSq7d@3+apipqXP?E5g@B*PCp|e7|
z!a~AFgr*3E3f&TP6zmhw5NHyx6I?8~OYo!MHX%)sBci9priy=*NRbki7L-0DRVtM%
zWhWIZc}{$XSfZG+*hJB*BA<kh3S|pg3f$*&<PG9^&2yYLg?A3mQ65)bCf*1hL!MJS
z4|ps2-t#K*>hR9zE#uwC<H)m_hlNj?-<N+H|4V^NAv0l9VPWC_LidDL33UoB6M8FD
zE8HyND=H{jBr-+BPPAWiwdikAU$Kp11>*h^KO~k*PLs-)W|ygyIVJN^MpQOJHd^+o
zjGIif%o>?#GO03aWqf5%$a2a#$g#^GS6HL8NI5}8UoBWeO|wBGS-o8Coa!Hy1IlNV
zt|=KQKUQK^%213|EL41@P$9om_Jg#Y)N~1X@w=iXqD7(yL>Wa(g>{9z1RDh|@Q3q<
z^D7Ai3W^IE2{8z56!aBbCeX+4#23JOg~yvGkXw~Yp3{s&gnb9=2j=OFrT_c>>HlN>
zd;53%pId)h|Hc2y_;>K%@qb_bs{Hx!)8xm+Z@yo5e6IcU^5e#j%pY6dTfF=J_QzY{
zx0SD-zwCIS_I%k>mM86x@*dVah`4|H?#J8dx4zwUyH$Rx=a$ZGt~*oj+`D6Nx8d&n
zyRr9<-IKq6@BXg`+>aO^%RF&;D)#LBvmei0UoyX{eKq&B##^Iz+ulXK*ZpwqgUrX9
zA0~hJ`=RHf>&J=@N*@wG%>AJMf#to#yTEsE-aUJF_Fez`TOTfbl=!scqv=PckBlE*
zet7ud@duZWt)JAt@O<s~y7ODgkBL9kel`6%_DlKqmfvxIEdJ*GTm8S8;TFR(1|f!+
z|NZ|8|ML8q|Eu!nvLBbePx$ugi{s}nAEiHPf0*&k<!$;~|91)Rp1qBDH|@RGhXe29
z-u-xU{dLu=#V^F4Z+uqt{Mn1rR})`{zj1#1>FwFK5^o#dSibRpGw;pqH_C6{zP<O(
z`2E{=W$)zQ$-N7Gm-KG_yYug-eN_7F_a*FW__vzxeLvWKa{W~Q8S%6F=k=d$zubQd
z{`vXm;NKsA^Z(xYWAoSH-_d`M|DE}t$T*2fl)0H%o8<#bJgX7w1C}E!4J`MWb(ohk
z)iE7pI?wcz@h*c6!{+}x|3&=0`n%*;^3UiWI^Vy4J@;kzXV%ZjpOimtc+dI%|GT^I
z=D%C??(4hK_m|(le{cHX*N6Qd4L<dJI`iqur~jYQJ}>`V{Q26aKOgl!7Ji6)FZj;q
zt@qpZw~O8`eEa_GwRd;kn}1OIFzZ9(N6t?NKEC_F_(9{n<-31xuDq6defZV7S7%?{
zeRcTN<yT&>55GS2THy7Amy=(FK9_zb_H@e=fv2~gK7GdVLieTrD}&cfuV=mf_j>)C
z*>69;Tm2#9lfak8uf^ZIzpwg{_>1|^y1yy^uP`Vvtz`CRoy2C$-oyTb-JatJ`x&-k
zwr_0x?9v?4oON9DxY>EmbEk0s<|^TO!FiZ7jcXfsGp{fIEP;N(G9eD(8$wb-ZGs7c
zZGtBRb%pK;T^HsRogl_9Atm`-@|@HR=^xU%GW9ZovNvSQ<yOial6x<=Pfkd#Q#Mgn
zN_LS<vP^|cy^OoeOX&^LC!`-pZ<d}TeO+2brbvcGR$6YFysTorQlE00ijeAUm50i0
zN(G896;3N$Qjk%6rO2RsO}R@&Of^O|Ow~~}R&|%EsG5~pkXnt}I<-e?-0JG;X6pXx
zN$Q2_J?f9ub2WrC_iFyoY|(tBF-xOJBSxc4W3I+8jkTKAT6$XNHS07rHM=z4s83S2
zSKpx)trn{mqIN))S5-juylRqKteUl&zS?(HE!C6CeoB0btL0C~Hc0zPUKW2Z_Ca))
z$R%M9;e$di1osQ%@GJ5i<PqZ5;p}41WqZYXgf*5mnWdOnm-!!)6H_MRH-`NT3m6tM
z9AuDZe8ae%=`xcUQwXCpgZV$UKdQfKf9(I3`gO+_>o2~a89vSYVDkRt+rl@qUvGIG
z^XBWDthYPg&VT#$&7s#yuS{M9KI?z7^U=)*ZTAK5zqvQ{9_PJ=yT|U--`;ui-1UD~
zU9UX9<ao*alGP=hOI?@zFW<QQ`7*~9%`3@QPF!)jy721Ht0vd%uGL)=xt@GI_<Hj7
z8`mRm?6@I%^Yl%P+dJ=kyZh^&!Go-ak&n0^H$J}ixb;cx)7j6&UR-&p{QAY~jc>Z%
zO1)EhH|_1KHw|wB-q^i~d~@o}p0}*;**;W!IQZf5hp8VNJ{W(H{2=+^+=ta4Uw^#!
zk>yj+C;rbdUv_^r`)>GC@wd+(hQBlZUi{nom;LYDKa2k8{=NIR@ZXhxZ~v|PXZEl2
z@0vezen<RL|C#V(`}dXKR(wtSTK-k<>!B~VzIcDV_|^B@%5S&6-TapGt?R4rmusJ5
zKVE)s_wM(b{5NynM7_;^SM@&r!|M+(J~Vz1`f%_4*Y{x`-hAl#DDi34r;5*&U(~)D
zfBpI8=9lANHho#~h3zZvx03I|KV5#`_<QBQ8e=}w1ZFOlkIeg--!qr8tY%4Jv15s4
zVPOei)@Jr)e$BLnDU8{lWh!e1yB+5qE-@Z{-XPvvJiXi>IHz+&vHxcMz|zaIjU|Hh
z6ss%iN9F>iy$oCb&HJ<T*U2B@-|fHc`NH&h*T=0N-n_r}{?Yr%?~C8}y^na$_<q;B
zDeoS?t9bwS{gMx9A2)qW{<!x8+lQR@jPI-7b-kVN=Ev)h*Nb0qy?Xzm>iL&vJkJZB
z9em3A?C!J0&;4I4c+v49@rCw_RnI>^3wieZ>D;HrPp>^;dAj+j<Fm=nem--1p8S07
z^Bd3QUTk|&{&MxpfR|5SEPip|Md(ZRS3R%XU(a|m?d`(1S#RgOee~Aj-J5qy-f#a<
z^C|4h)34Fr*ZnyE^UE)mKWu*;|4sUTmLY~Iip7X+9{XDkLoNpHc5WY@eqJm7#{$0v
zF9->VIEdznei3OAi4`dknJ7{#a!^E6EK=f+RGzGt{4s?nrM1c$DoiR1m1ULZD)}o3
zDXA&hDC?=3tN+l5*Rs`S*7>WmM0cTXoUW;Ep01IegMPF@sNp|DFCz=1bA}#<wFdtB
zfqDmZi*&tp#dK%t+}6IW^;2`XW|$_s=5I|Aouj&4dWZBT=n3li>dNTc(7LTDt=XfY
zt--0Gr_rk+qq$G>o91LqG0iLuH}xPjJ+(kJ7d3aa$!g5%j_LvGiR$0go~d%E7OE^z
zeyh~2WU5rD<gUCy`H=D#<rm5o%2LW|$}P&<ltYwfD&1Dxtl%y`U#?5;k?bRxDj7GK
zqtZ*Im1VZbM9K=wiO4a^-jQ~au99+-43Rh?o+y4>tX1@`aG-Fg@HU|pf&~J<`R4G-
z@w9U3avot9WZS}$z&xALk0F6UnQ<fIVa6E7iwy1zVgK*`ZTWNT_uSvhe=q*M^Y^9S
zn|?3)ZT)-S&nw>rzCHOo_aoQ)r?2O~eDHkp)4C_e9_8L|z3=;2`)T#_J#VLf@%Y{O
zU+2%yZxg@o|0(_Z*q<A}W_}a;qW%8w3+5MhpEo{nfBfK){yqJhC$GJ_oPHtqY}~2W
z$95db-Y2=IX*c`sle^aLirW>j%V^h(ojN-icK+QlXXo@?b9cYpW4MoPf6IRH1IrIe
zACWs2cYN6ims6`xN1Ta0+jrjl!tJw_XEM)4U$}MQ?4{i6d+uaDeDU<ot5@%HzDWPb
z|Go6zIff~Wg-k9iHEbrF(|G*(Ckqq{l=8>$#|sDv?BV;xcbGq%pP65We+u6+UO8Sp
zo-bToTt~T5xh8VPaQ@}&;8x{3Dj+8uEH+1SlC-PrcezygX!$krqeaAM5iwdsj201i
ziioLlOQn4!m_#25#`CY{S;_H)Ipx37@BHrpU*~@@`@-_&$QR~szTfVCUG~lXhv$!X
z-*5b^`IG<G;O~z=`~Ir_fBi4u@76y(e}4Ub^4t9P{+~I&djGuqGxbl)Uy1+U8H$-Q
znKm)@FtRcoWpZQ^V%YcZ>c2~%<=l*>Ouv}B*j>2ZaZB(`<3Gv&lz%tBCjVc)Ccd>i
zxt#8-PZ{6;=lbjM<M-!X9~9qmzgqgj@cHB?osSeAoqjm~LG1nbdkJ?>-Z^on@NWKn
zjYsW|&pdwi<nhyOPwF4dfB5;~-G>|x;~!kQUv=;79sfJu?##J+<Zjp9U3V;RPr1o`
z^V=<*`&S<ey*Tqm^V5Xyv;Hh+n85svbuRli_8RsoHYV1E%%Myl8C?GF`6Ka5;b-R0
z;$MP)g&BS_o?%vJEo6yiPGbJew1e>p!(s+uMq?%=W=WQttj_F9*=;y}aJ=Q%#Zk?1
zj!lN;4Wl80&cC|fCO<sBYJB<e<@UGgAGSYvehK{6`Q7`g`lscO(w`Uqtoyt3kKbRm
zzrlaz{dZ#e^nddIc}$YbEG!c_LO7#Yds&}yKIaT)b!AcJNaj>!*W<M1>*bxz(ZbQd
zA1Br#dPh)POjRaSl3n7u0=HU{lAm&xmaeX!_8RSV`lpR0Oj%5x8fNIb>+RM(qqR~a
zO?#s5J*|nFn{^!wXX$^_snr$L%h2J_^)x(Z9AwO6e9<`6bcNYvlOTgyop6mGia(`!
zC0iw4i?0zi7v3$TEab*_kCTr}m-jC3avnzBYTn&EPP~`+cJlq<`No;fc87H-hd-As
z*K2N9{#3y(!Fht3!fK*ZMRP=?h0TRd2!0j{7Gsv&Ch<mmp9F){UTGh>YI$pUE``~O
zQc6-vdde}X*6Ly!+?uUgVcHk9_vvux^684{x#`W-iPV{)yHNXw+8PyJm8Xhp6^fKt
zRH9YV)%-N?Xew$7X?|6YP*qnxr*K|2SN5d*T?KYUS>=x^OO^jA9G3eiZ6$F|#963U
zU^2f2Uj_GS_SMY08D{*S_iy7Lwm<2An12iYto!l%+lJ4kA18kJ_hJ3V=N~ISF8pxr
z{iFBdAGW`fexvZR{Au4qyL+~`8E*c)vE)|r?S@;?H+64(x+Zvi?Tx})e0S9DuDg?b
zd;6_<x3=AUc<t5YX_psVVZP#h>DNWeD-zd>u1j1yezoCh$(80SsaH#`uDrs1S><B&
z`L}28p7KA#dOqa*jB~!{=A0Ee^W;R*ktqkCAFw{adm!!L{KMOiMV|^hd+WUI#XINA
z&gPzxIdl8;gfkK6?XNW75WMyF7VjOcJ5%py+?#uU?L(U<a!(nag}q$(Ch2YKoBY?>
zuN~e@dbjLj<`=Q=tAAPld(ZHmiIaID<4(rSOo~iT8T=S{86+7r89EsbF>Yi!#&m$G
zoM{QuA7%@dUra*GE-Xt}KCs+m{l!+p@sU%ITaCwtX9|}JM;7ZfCSJye|11BW`fvB&
z>0kA~ntvvL%l~x!@%tn3=fQ91-&(%}e;oOm{AKZ{tPk<;JKxp5z5X`#UGZC|w@2Tx
zeyI9z_e0&slb@2lsC*6ny5?)+*U4Z0eU|@{_~rbU6JIBOXZ@A*+wS*~-zWdX{89Sj
z`j_k9g@2h06^t{OezMGC&)`bqHRf;S*XOt37Z-TMpUSs@&xoIg_XF2#E^f}(Y*Os1
z9EKdN9OaygIjmV9F<LPA|Fii$?}zAjpKrn6O1^P_oAmAK54m3xe#m`$_j$>Oj5l5{
z=04f+@XWou+mCPTzgloH?1JVcolDslk6(Op@$)6`E59y%zSwm6=M~;-4%Y*3cHZ0Z
z*!Zd9^Mfz>Upl?$d$IGW+aumb;*WElta!Tf>5OOJp1*je{xs^Tz%#vPlb&yUv-h*p
z_m=OxKZSmUd~g2p`HRZ;e?OP}S@XB*pEKhtmILg@ToOEPJgd3Za;)RH&vlu*m3s;=
zg8)1KRo)i<lY(wSxdKUiy8LSeI0dHjC-Lj^`fz<^cVNH5>A|y%yOsMR&qUrF-g=%j
z+$}t}xc_h@bH3&_<bKb&gj<hqy+Dz$vS^gpWr^w16*5bu|4O+@HA|coKQ1XJ(<*yK
zE=9goE?jn#>_dfLs%2U&y4L!o`d{=P8?G{bW1?-g&eYpD)abtPW7AdUlPztnFW9Kr
zCpxZjobIsJUf=Geb+6S{%R-A*3kj<qRx_+6Z9m#d+rG2PHqS6FF)-D))34Tlr&pre
zp`EBTQ!7rJQMXIq(_og~b)6qtIqDmfcPMUEP?P^8!zDdWQcuE5+*kaK7?Y^I&~knk
z-gfSvT<ToE*>A8~F@OAb>9_OmNxz(a8UA_y*Z5z}Kkt8ve`Ejn|0()Y@rUb=<L@oM
zQhulWeeuumKj(jje|mp>e>ePQ`V;>r?a$vo@BT9Uz4YtKPmW(bzrO!k`1{1)G=^x#
z>5S(XZ!?(vU;DT8kHeqvKW=|M{o(yL?SBiC9%~|-0(%Er21_54KEs_qyMCSh`SAzG
zPv&3ee;NLE`|bWa{@3&GQ@)yfx$^1r$G0E%f4uk6<I{(apFc)^Uhw757p|`pzf^u!
z|5W?2{!{Z8hp%>Dq&}beDEt2G>#r~Lo-KL&{~^ai--ktywV(05D0{K}*@edo9-2LH
zy>E8^(EW7}UO&`-%<}Z<^RKUL-nzcCd#C%Z?%l3;ChrcuS^V1OP3K#Oce?K_KA3%2
z^nUmI`VVy<y+1L2diP=V2hEQ>AG1Fm`TY9pkFS#7+`ixVX7gq3$La5_-cEk~{nheU
z;;+Ttl)t(Drt59-+nBfQZ}-3Ud0GBa@b$;nNpDKuFu$4bI{o#`*V|uTd$r=_<rhz%
zOFjSi%;EX<=MSEDK4*Nf;>GS4+g^mcP<p=T>FviskAfdAykB`Y;f~myoI5Rd=iW29
z|Lb1O-Ll&gZ))B6em&_1>rIi{H|~l)5PGEa`1GTSN4k$@K2&*V^5Dq*Uk^$iu6}Ug
zfy~1L4+<XYJT`yI{zBq)@!Rxwzur!HJMHbqHx_RK->iSb{dU3I@^@A5mwedtsqJg$
z_mm$gKQ4dY|9$26i{GYwd--+K=akP{Uygpf@Xq|*^mpezEc`t0^U2S)-#&d)`ZnuZ
z^!MT)Hop)5<7cvF`OR{Ubs3uxyC~;WZgt*z{zxHHp<Dbf_?Y<}_;&I>;a$U%&h?x#
zfIEiw0e`!oj^J#53EtnF2iO<0{boJLTF#ozlEkFX7|gt#O_6;z>uMG@mWj;sm?Rk2
zF`6*@FuiB|&g{#g$-Ia;n1!2V8S_7;6sD_;a~L%L7yh07yZZO<-}`>^{o`ZqW3^$u
z$a<J<BdZ&8ETb<&%KwZ1g8t_*{9;(aAi_}cf8sx%e~JH#8I%}(m;{*4{GazP`JeXR
zcYlulo%zS$7w3<Vuc}{+zP<Ue<oBn4(-^+}Z~9;V-}Qg<|0Dm~{+lz*WW2?Ajj@?o
zi){mk5BE*pYy47z(*-99P7(0uyTZf3yOYO@XDM$uXlsYSE`gi;<$MXeCwMY>esEjx
z{N)Mfb>L$ZcqmXUs3kmEL`T#@Y_0fe2~Ejsl3%5qq_;@zk(w{HRPwOoSE)79du67|
z87tH&ep75zoTjL$l&4gyJXv+7dXMIQEnlrQno~7bX{4*$sJE%7s%xsVt4&nyP`s#M
zrcfcDC_hEMQ@%rftNdAcAq8f|V~U3qmdFXqc1fR>be23M$trb1^0!2p1drro$xV{8
zq}XIO%I=eIR_K@amCKU3FKH_=MLbblN<vgJRPu?$X>lvD6(W+tYXqwV;`nR$4)XHx
z=JF_W@8!^7o56gA;lsa2{}dQbF&<{#!J5N1g>@QB4|6PI`2SP?%Kz2;V`tdJ$j+q8
zw3cZ;QzKIpGat(#=3~rxEHx~CEUQ_zvADCSvdFL~v8-U}XVqmhXPeB{!2XKu97_^&
z4pTX!C{r%84oe^F6}C%k&)BxG6|lZz`O0Fz;>)szHIMBh+fVj#j_d5J**>uTVtvTk
z%4*NLgC&y1jisOYH&Z^d74siP4aPzS&;Jkqefp>N@5-Nvzi0on{oeoe_2=Us<34D<
zzxJm5)#?|K&-|Y}dc^#I`%d=Fz1R0#Rk;#;S@F`M^Co9zoiICk;?UZI?+!#9Sbt#0
zLCwRSM>UQ!oya^<a^lu;mgB9*mLF3+Zhu_(_{8J8PCPhSaH{@P#wmqUVW(_PJD%1!
zwesYOlgcOCPW(B}bYjZMYp1KuzBy-c!Tn;$rQMg#Uh=;zaM|I~flE&=Z@yA_?b3~`
zTi0$Y-4(mH<^I=)l27uUdA_*&qUvSftNSnCzF7al>&4sWQ(o+N>Hm7_TlM#U->>~x
z@wxHK{;!w6d43Q05&N^|=hmNley06M_{RK2;Nz}$zHhDHXuP@oX33lM*J7_&URJ)C
z@Z!dcxR;Y&vAtn^d*JQNcQ4=dznk#x>btG)=Dt&SpYTETQ~BqFFY4b^f3W_X^fT|5
z$?r*jR{gVOuw*>Kc$Y!pfA2qs|G)mHG8|*zXRKrtVS2%2&T@}6f_(#rGM5bZ4DKeL
zb-dAhYx&~%R|p6SNeh1!UM#X*<etb@QCYD%Q9jWW(Uqc7qUs{Mgc(FOi<}lcF6Je^
zMx0;5TOv^+OrlHTtwgiLcJXlWJz}C_$zqSi+{Du*PD(zIvX)Vnb&^e#JtQ+hs$X1M
zBwE0Y_c7O14pok1&SY+V-kE&0{Nem|d>eUQaRqQXa(J*uv)y8KW}U;L#oEkf%I?PQ
z!=A+cnteBiDCchuYfd>X9_~)=C*0F{I(Rqnz2@I6z$oY`=qXqwV8qYQcbZ3qyO~pk
z<0qQ~8wcAo)_N9m=EF?A%pxqM%;n6xS!!4lShZMBu}olD%VNjU!K}!9kx7I(jpZ?`
z40|+3F{ccd6jv~3A_p^v6~|1DZyYV0u3U9oLR@<|%Q$~?EacGOU}P_2{mUH0q|PAw
zPvNiC-(`Qb{{8y*{J%Rx*8lx~b^hf2^7wK7Ti>^5-xhwq_oMMw@}IkZ)&Jf7EAsF2
zzbF4EGk7w(F)1(?FuOC0F>5i$GnX+RW<JldfNd^^1J_jU)jVmu-+6=ixcG8;19($-
z<$1GtoVc%YUg1dMkm8uj{*A4J&5o^uHIii`lQQE&hBC&Pj8_=hnP;%@vhlMEu(z<O
zvPrSEu<5g3XWz-e#AVMtnMZ*yhd)`sT(C&+tU#Q=M}gCVK0?yMCL-5G1VzI|mx>CC
zZ5DGB-yp6kaa-c9WS7)dsrORV(yB7&WQ=6HWk1NikzFq9FKZ*SQfj-Ty=10DiTHo9
z7;!m?Mu~KZ1_=g9Z^_#dd=l~E_ryxY{)i@uI*8s7brF*ldnH;aI#1-0u(U9X&?A9)
z{0@9dyuZ13aj9`Ra7l7ma~@z{#>&F1#Blb{(H{a|kA9N-sQ$j}ZN{6cuh+aTd#&&~
z>2=^6%eUX(X1vRJSMYA_yE*SZyj6I6>dmt^AK$Ee!|<m3b-`=1*KV&oUfR8=f41m}
z(&NsD`VS7=KY#zi{fGAz9)vvbe8BtQ>iy{Z4fht^{c-32?KQWmZ%(-3c0KuO>gC{z
zTIW;FK0M8JYRU<b6WPb_A9Xuoc-Z?;*}<I$G7o4T{Br30k<4S+CoY_Nb*Agw$@7sH
zx-WdaV0KaGV)(_K7w=zuc=7hd>WgPC1YRh)aOc9+3$7OqoUc3|az6O{sq?ols9dtR
z{OL05mE)IpUp{g9^W~XW_FmPv{`ba{TeWv?-@SS7{QU_Jw?1xt`sw-d*E;VlJ{EpD
z^!e?V)~{V(1;5_;vgqrMZ#LhTf8Y7T_1CxG#eXOMd-Q(|V>ELqiyJF1+ab2)?B6*e
zxq`TVa3}Kw^DN@7;cDcp<YeOF=Gw{G&gsaxgF}L28v8=_LXI%bx12>>oZL&e4SDAC
zNbpYPy~V4=*Tm<}H;4BquL=KZf$4%A!YQJs#otM;l-@40TK0=vkNjS_1+p@-=VcOQ
zm}Mko#ATn$=_;rxa41|?n5o#TsIK@yVZTC%!W{Wsa`$8<WQ$}TNoz?bNbyO{m%1ta
zPR2;~o6K#Q*D~*AzR85jhRMyB-=dJIcwNy<X}^-U@_OZG%D<H#DNj<~s-&k_CC@7-
zCA&_>PiBL(gS4S^i_|iSb7CA~3q>!8%oOGmsu1wyf5-cT+mY)LrxsTM*G?{d?!Vj&
zyxqK}yfHjZ+<!S&bHuaDv+1*bV=-e*X60nnX3=L(WU^$e{{Q)}^WTeqxc(&k*86Sx
zoAtNzFXf+VKbpQ@`S$ax+t)8&HhppYGUfBRPY*xN{}B2<|6TFh`>%6f?S7g2lHq0X
z3(gl`o}YX^?U~Dy`wvYY@ZPt%XMfM<zQ{w5#|BT$pM7|y{zC6%%uBwP_g{Q}VfV81
zWy8znmrXA>ynOev@uk@drRNu))jnJJboUdLCwCuzc`X0r|KsP6y`L<868+@l<G9B<
zk8K{WdTjor{z?9mwkPUO8J@K~Q+vMb1?%gHZ!f*S^HJl=?QaS{>wYo+N%*&zfrIH4
z(-P(?mM+#pHa>Pa4hAk+o};`h{J8>oLT^Rf#3dz9O4Z2hkyV!4C3{llqx1ynIO%^<
zQ=~Mcj!1?|_Dgt*D~n}{&KJ2Vyh@l)q*_E+lw0hpSikr`abbzY;ymJIVtYlMMQucb
zL`_76MFT{ii86}$i{2A)5}hs@Ao^d#O7xzntJreU3&N)avjj}}TX`>WpX93Mn!{DV
zHJekMb1`QjS2UL|=X>^5Y(cE&nai0@Fz#fOXWGtmpUIzznQ1c9FQx>h9!57tTSigF
zwG4aycmG@WSL5&WKPP^F`qlbt%P*GSGQWj>Yy3|5-T3>*ua2LWzVH5K_igjnUthYu
zT>Y~5tMRwqujjv9_)`Dn<fniSOz%wI9DL>Wvhum%Gy7++p2<Crc;5M9#jA&J`rcK)
z*Z%P9!@7@LpTs{Ye46{I?DOx>mp|8iF8_S~v+WntFP)$7d@BC5^JCSAtoQfd&3ISw
zF6W)^JH2-n?>OESz5V!R=bO?uXJ4OqmHu+o^F2@Xp141bekAs=@WH7E^B?ftkGQ+?
z&c-|2?^xX7zAbdi<kpf~Cbx=j*4$LPb>tTNZKvCMcX;moyRZIm$3wSACm!)WHh<jo
zSoFz{C$i66p8tEU|6<9DYcFrTy89~imD8(~SMy%od!_$+|Lf~-Ox~ruSNqWY;rs{5
zkE$Q<edztL^n=dFuOEv)seHQjamS}gU);Z~`JVst;_vW(x(vpQElg`zV%cV}xwDzF
zNwD2wJ<0l>bt0P#$1To3T$((ZeBlDu1UCug3NwiWiUf-oi98lA7q$>K6TUBWL@-Za
z0e>vNKL1R<<-AXLcJSQc(dYfe8^C`<Kt$+)kg~9tu&A)T@FL-8k$lmUV!{$PB)&+z
zli-mol9ZHMECrf~Pn2CMmnA14w@0o=zFq#79H(5P>~h)vvOIFxa_8jdDqdDHR<=-%
zP~NKiRC&2_gtCWnuJTmnmCBjQACwxEB$fOXwG^D>ZR9`7*C`YzXexY_-zje>?;*!0
z`%3zc)KAHU67J&mVjD!mMGuK=6HXNREYQlY!q>^OoGXY^fm4AqfuoV#hFyr=i|riC
zAEq2eZ-$fq(*K<Q+3{WN>%>nfA0EGBes}b3|J$u^?!V@EbLUO^+pf23-=2Tl`F6{j
znXi*x`M=!uV(p8p7v;~}o|!&ldFJ($_sPLWD;^$yAot+h{X6$1A7ngO^q}^^h5MiH
zmEQ}!XLj%H-JZJzcc<SiyL<kQ`JI2aZ{Kdcee>4Un<h7FZr;1O^ya~vJhx}x3A-C|
zSMILF-Ke`wcLVQEx%=qeiU;bCiXLBm()z6PMf|G?ukGHPe-rq2_S@BO*S(c^C;r~-
z!^aOgA0s}#`MCF!!)K4r>7UuYNPR8*cJ%wbA4h&}{nhb%*>C0F8-MNo#r#|Dx9D$+
z-^ITd{XX*h^6&Y-pZ^Z}%k%&L|6dIEnZB^ZusN}RVDIOU=H%ye;wWM_WEW+>z_yj`
zD_bvnA%_yD9;YPdM$Q{tU$~F)B=H{Mea5?lSA};rj}wm-PZ3WX&kXKETvc4<T&7%m
zIiol&I4w9Ia(re##I}KTBg<^&cBX#DYYYJlH~+`|fAlZt-?zV<{{;W-{k!#V!as-q
zH~$AQEM?GVv}EF7wq~BeyoKd5YXn;)n;hGBRz)^Twxg`Jtk$eytY27CSOQr@ShLtp
zu&?0|=M>=F!ePZRjeRGZGusTdUu?VC_}R)?by=-hWm$V!#8^03PBH&yddK*M!I|OC
z|7HKZ{`dcz`uF@FuRn8t@AxJ1>%`BFpUyvD|8V`W=KG58Yrb=Q&-pgx+skj~zg2y6
z{O0>@%eT|tc)tmKZU6H4^P^8(pN@ZI_*D97^QV2ETs|-PlKf5j$E2S<zifVg_?`7<
z#UItbKL7sxKgRftDU5k1a|cTR>mgPpwq~|VY+~$&><`!)*`~9-V|&bI!xq9ik>we)
zI&(eKZ^jx%Jw{{3jST<(Y5x`f!}oj1&zc{9zJL3!_2bU>>EDvRs(qFDI`PYc&%vKx
zeOmF!=hLc>@gHA*T=j|Rv*PEj&qZI3eO3ML@Pqkh^3N$h&;NY$^YG74KUe+I`n~7Z
zo1aZT5B}`^)&D!{5AR>@e^URi{?BKyV2oti%6y9@oplQ9PS&@q=4_>GtJvJw<2ejD
zdpJ*Ue&Dp@(&Dz|dBk&tH=lokz)XQ4fqDTU!DoU8ghGUm30sJ0ilhrq6Y>*E6<RHH
zS?IBlr*MqOF46O1+2S?g`Qq{7x5W;Ko)*a#$rMQ!;S+Hd-Y9fd=$p_iAt|BHg5Lyp
z3bG4+;eWxW%4f^Fn7f5DoBbeb7K<7)6O#wyM+R-iXN=lRJ&e4Jc8so!9~fRU#4=uH
z)L@ci>Ss)0)Ms*GKF)lGc{B4w=1k_lOd3oJ89y^7F?lfkVcgBw&RE81%vjH`_P^x+
z)_?E+TK=v4WBO;&pX+~o|Fr%V{Qcn9wO=oOas9Uaz3{i(AG<#<ez*L#|GoRyrJu`w
zlz&hBHvg;b*Ee5gep&eC^_RI{7JQ!ZY2Qctj}t#~etPop>__8IhM(F#YJV*H`0iuQ
zr|wU@pXxs5ea!gC|B>b6hL3ALP5#XD<=<!RFX~^bzHR#c?8p0`UB3?e3i{>qGxEpt
z@7un&eV6%O{mtUrl5aNOo4(KZe)GHQkBL9_{^0w$@8|Ac+kPAUVgEDXxBGAV-#oud
zf4%y-=4ac_sGlo-2!GG{n)4;*^OKLeKAe9)=bgvfov%;6%zhF7V)u*m7fYYtddBfg
z;i>Ku`^N_#U3j$a(dUOI58pp<c^LNS_Tvvva-Xhxy6$P%(@Rg<pS*qY{OOEmUC-Q}
z$v?aHwCt()(*sZDKKcBl@@e-|<EPJ`Ok&gF3g*`4+Q^~Bafst4=Os>a_Dtp;Mpq^d
zmKe4ojuH-S_WkV7xo+^i5xgxlS8z6e7~fXDw*uM1H-!rYr}JLpvfx_I6~wK_vzmJs
z$4S;{tkvw7I4gO!@Rjp@;y%Oaz-h{9$8nm)?7!J>soy959QeEJ@8UoAe|P@5{`cp<
z$N#1NzxeI+<Kb7)ukzpCey{$K{=MYu>d)^#ulVNnv*~B&_vSAvKXQM#{r=nsp3kel
zntj{xx$DE;cURwje|_b7!=rik>mST}B=My7N!{aHkLEoQe_s5e<>il;J70)C7k{?o
zsqXVTue#rUeY5se>dTr}f8Tt5SNHzVyX$YNUm3mp_;TZ$r5`-M-2ZC*_2=h^&$ge>
zeTw*S`E~J2i#HY@4}IDGZN~Qx-{pTU{pIw#@Auo^wZDUZIsaVtGwiP~a|8Q#c6YX&
zEb~}g*sgIfaWCdh=1SnuW1qxc$+>`sPoP&&N1%sKf?rX9m46NQV)iK3a<+M#i9DZp
z{rH~ozTs8jyTDt*tHAr1Ta>GlV=?;zj$Pd0e3y8mxSZK8FwbDFWck8U!lKPQo$>E~
z!~fU*YcnP>Zux)lPwB7hpVNPw{&wi|x%ZdfY<}bNuJAqchZpY)-V1$5`WXFD=flr;
z8gK1h-FT+-q~WptQ|IS-&p$tjdpP6%mIvIAuRlqD`ua)8lNC=kJU#vF&vW^gfiDGL
zWImhrbl%gQPZOR@c(CNo`P*`LKi}oLcl2)cU9Edl@3%b2dbsJ)^~al^v_Dz!X!^r1
zj~k!wdlB;7;3><~1uy2l>-aqR>#eV%-*$fs`~LE~-_PdX5B|jd5%`_<>)@{mzg2&y
z|7iLu_;u;G-#;9Fv-}DA!~K6MQxdZivlMGBdk+T_$8FXy7Du)s&Uu{8Y<kRPOd+gW
zIo)}Zd1iCZ<7(qP$I-;`l;Z?f18)ugF#&g>bHX!4+JvX_f8(;{wB#!0PU6wxt>K#>
zxJyJ-)Kx@YG)qiZEKTIO;0pmOVNtOqqIp7B_^$9I@rekP2qz2m2`vz27uFH_FEmRe
zUu?d_UCEo0s!|)JPD*|f+b-0_XUa2&`y}^Ep8b3U0wzL9!jVF40s#VT0#f{%JkA`G
zSs$=TaanOGv0Jg6W%|oh$dbgW$6CbtkF}N6hIKbv0H-L=CO$ubX95X=fr6F-wR|!>
zESw>1dsr?rzhJguJ<E2FU6ezSeGXeD$4RcGT)rHNY?E0(u&HyZatUxUu<vIrWOL>a
z<c#3(W{+fRXKiQgWP8WHhI1kJJ)RppZait+Q#gOJ-DgQ-Ud!mgu$3X5=`HgO7HQU0
zmQv=s%zIf|*+n?&*_W_gVw=NpldFWghclMViRC!ULpFVmm+TDe{A>)Y5-cVx%&Zoy
zx~!Mk-f^^Z__OU{31Sv!?D%)^cjM11KiU7-{FVNj^e6jQ`}f~pSAEO*asB6?pBz6I
z{MhmH)UVq=uYdddN%q72_d=f@e|h`u@^`HtAAhX;ssA(a$CDqAf2sdl#i+o1jOj1a
zTIOKpmyD19bN$o$m&u^OypFY>eJ8sn`+D|w9IrV4a`ki1;5yIQ#%;hS&u_)&%QKg2
z5%*QT$pZHT<_YD9I*Xf2^h?Z<*eMw-vsXSq`Mzq08iSgE>JjDJig)F%N(W1O$y&(o
zP}ru#qB24GgVH?ZwJOI|nv{(df6AX$tX0{jlA-)i$w>K*O1pZBMwR+e^|u;}H44>T
z)h?^dQ_fO$P>EAjQ#DieQ9LgnD;p-2D=|f!Pke>AyHt;?hkStieYrn!Z3=Zt&Pw(2
zdu0Np*GQd`<dR?$TOz_Mk}kp``bl_$;C+D~0(bbA@LF<nb2@UEak6sT@ka1>^8ey-
z<y2)eVxIm#`On>7TEA=mYBOjt@cz5@r~i-OpM}2<{JQ;v_50ti&R@m8Z2$Q2-GR4H
z-Zs8Xf1~m`=vD12;nx>mX};onb@*k;i$Bk2y}I#E<73AMmUru4&40e_$=(MMciZnS
zcp&z~?OEY7^{4M2Cq91t*x_l`(}z#mp7p=H`|A3u(AWHL65i;%;d?#zrOL~5FR#A}
zd-LF3%Eu=k{Xe{ToB3w(Tdt4OKWltZ`||ViqfdGt?!M-G;r(3arSh9a@4mkidH3NR
z*T+{M<v;0vKKn)S+t+U*-(P(D`_1J0f^RNgdA_{(B>C~e`?e3&pL)J1e-r-p^E30u
zSMQwP@A-J>v-_7xUp9SN`}xub<u~7+Gdw^2Jp4KDvxKKVpY%RycryFR?I(wyK6<w1
zncvgqCrM9g9)~?jeDv^9$)jBltnXRgDZ734cI4e}_x|6#ax3>n%yqZx&u&!SPP=>X
zp6UJiyWY3<UcY&b?|RAg?(5dq=UjVqweafJt321`Z+hKUx|@HG_rA*gb@y8D%HLtT
zoqFr!t%<iIZr9z`y~psV_esaouTL@`hdp}skoQsElaiM+-_7~h_vz-x+V_Xw)W1IX
zdghyZZ}r|UeINBc;Qg2PuRompxc^ho7nN@tzxV%K|GVQ)?Vp@q7uQ_gfA!M!=vznc
zc0HW;Wct&nC!vpc9)>*n^sM!b?0bv1?k@|U?Rk3V+3^=IUaGyaeYO47x;H!D$9!D#
zA@N=F>%dos-~9Nv`5XI>DPMa&HoRy4*#CLwmw>PPzM6e){c`wA@HdyACw{N`UHS9>
z_q3n+e{VDXVzy^G_V@134?iyc`us=vZ{436f8_r5|LbP_%JQ6j5@!eJU(N-*9>M|Q
zrQ&`fQUV)zFY&z)TrN^9wpL7Cv`4s1=!~F_(0L&@p?d=D{Hyt|3AKuUlei@!$gjn-
zk|&0LvQV-Jlc<WQu*e&snZgf5ibR-&V);$^8->cn1*I=bsfr!pTgmZ)&7UKbtCJ_3
zkA<I|{{bI2zly*<fk1w7-XflVe7d4rWZo$l$+Jr}h<Xcu72YeRBpEJcFMUsXk4&=M
zY561a?_{S-rAn-ndaa<R)}{7d>9p(v32(9KV%H>+WiHAKD=t=;qp(adROyaFs;sT#
zVbN_OvEqwlcom%#Gh|OoIEY1xMT;kjGmHC(_lo}zUo1IAT2ne%+(#&oKUSbg<hq1}
zw34)kq`cS;;S%98k!_+WVtitIMK_CaO7=_l$uLP<i6;rm3GET~6XTPZCq7T)k-&OB
zZhj8IIw2;ZVgUw$%YqX`bi_W2t`lYzIKgAZ?aIx~)6Bh@J%n-7Z_%HZeiZ$j_>1Fr
z?@zt&dEZ=qy#2-a$M*NGpK3p<zfbw`?N`X(D}U90+kL<ARqDs)-|PQI{yp~l=C9em
zfBf0@_tszIzc+u^{eJPM;r|?_H7rw@WB(ie+V$<`H??2+f4}_w|L57CMSqw46J&^F
z%x0=&KEQl~`2lM*ryKVMPF2=d|L6Y6{gd%;9m6Tcd5n7)=P+}yakHOdGh%aNljHE>
z=HZ_qtSeq5c3+^CQ=X-b$&KYan=Qw5P6?iVJ|%%*!D_*0{9e3QxlFl=dBOyq38#oE
z3h(EY;(W*+&hdu*AzKlfANyiX6&_DsUEXvaVeTNVo7~L;f5psY-pf9e_7%?-s^Bl+
zPY}E(^h3y5xLPzuvQI`?_OA3nNpZ1ff}sMMLJvhQiE4>73AFQU;B04q&GMJA?!U{w
zL;t)P4l%A~l3_aXU+(X|-*0{w{+;l@jv?${?$5$6b3bx@bp4p{LHOP6*K1!*cxC%~
z*K79IEH76*6L~uCiSX0*XO1rtU&KE9`uN(TmyZk|TR)n3fBoHs_Y595Jec&r?$Q3o
zaZkjaia%fRD*4^#_o?rMUOT*4^1SP1)0=;9o!)Z4`SAMxoAh@@?-f35{9yR;^~VRF
z?tHfY+VU;=tMJD?Z~R`fzuEXk|INQwQ(if|u6h0WRp+bUFC$-ezx?!a@hiS}>pt^;
z-}<%mW8&MOHv#XLd_MM7>znoWIX^mnaC}$%8uYpBqucwIw^!aS{*d_P%eN!nCVu|-
zF6_1AEB@D0Uw?U}^IGKXx3}?c;$IoObb7)0Jo4$?Cl6obd}#YR<I9fs39n{8d-8<y
z$>K+@54|3SK9YDW_PFd((?h)nYww!fV!LtYhRmJI_p=_R+;_O+d;9Y3AGfRTWZ$!U
z_~&urQ<JBrPwzc7c^ddg>;Ck643E}7cYZVJZTIVQ&qJQBe(Lz_(v!r;DNj~D3wp8t
z<@(n(Z{NJ-c~kU)>)GdLUav&n9sbbwvGJqmr@Bu@pCUhLect++|4ZkWps$kOGk#|L
zviiC88}C=+@0b4QF(xri`)~X2=wF?Gg8%LRum2bRFYuqqKi>bmO!HY~*{j(iSi@OP
zGsm%j&S;2XiDvBi@BKfCVJp)M76rBgY?Ih-v9_{(;ppKy#;M5hi~R)GEPf{8b3(BK
zM|jP78hNt#{s>GI-Ye=S-XyVAGF3WIcB<T8xhrzl<@gkAlq6JEtGrQam-CZcD4H$8
zA!;M$D*jyDP-2(FHpz)n8>L=L%1MNaxr^sZpOSNttCeC9z0L2)tH?8*OPWiTXD^?U
zfUlsS@GfC)p#=U}JjXdrIev05amVpEbNjKoGUffd{fFi6;lFSHZDY8}q{n)h-GM8K
z`#l#c=RCF@Y~Q(V@P`Rb;1}na#c`4C5vvBvDyG>?*~}R%g{&*sS~w<hS#vMq3})|T
z`^nD9t-*JTZ#~a8&Sz}JEFR2y%xz4XjI9hIjAqPtSxY!NI2}1&v+iP5;PmFT<3Gx$
z!*h*8i2WgJ3Udu(FXJo5os3@@y_lb|#&hoCj^$p#`HUl#^97d`?*hKfyuI8rI0HF2
zIbu2ZI6BzMS$;4nGOuCbWs7Dz$oiV)6{`;yAOAjqK7rr-fqX7Juefe;T5?Ko@^jR%
zu4g*Hu=xM|e+K{T{(k)Z^4Gmz@_!Egz47<qZ?>N%-~7K6e0lga{F~R;y`PtU;`^-k
z_1HK0Z@<3eeLegA@$Vo1&oNG7y!F5JkHAm2?_%Fweq{VQ`}@T28NYx3j{M{Qhx<?X
z@9bahzg&N7{(bvDo~fM4n?dYf^WXpfUNTy-6tOU{Okr8T`i?b?RhmVBNr@qeftQ7q
zvw|m@ZwFr)&n}Mdtmj!OSfx1bajEls;1T0<;oHx%k}HCPh4ldAOorEt!mKwr+PIE!
z6tewgjc0RaJHo2S%E&6n*23<?QNXc?J(AUjsf8hdQI6#bTMfrr4rg`|)(YmAOrFeF
zSbEqGa(VHd<ued?D!5H(g`fn#Ax|sUL$13#w*0{Y2l(6gZws^u6$*<8#|!@8j}{OT
zJR`73;HZEX{}*mX&fn}`IlgmsaJz7-vW2pIVUlJPWjMv~i*Y5>U8ciKlbFA=nselF
zyk^_LqRlMBoWQ(@shL6ZukA0%pF4hN{h0Or?6+-SjlORG+WD>hJNwVwKd=5!`;qu_
z_wUbt|NMLQm-YAZ?`~gre7W+~@JHP5f`4!R$NdlZznL+TMVK{^MUeSD(`1%r)&dr8
z=G}~E{(t+M`1jmjpMO*SE&3P#FX-R(f7bsm{cHGp<G0}NmA_~I&1A@A`ok#k|KP9L
z->!ca{G9q_`?t;?Za+<aKKSYJC;VUQKg)k*|9t;HV(4LVWcFd2`M>7Ri=UIePy5>N
zMdORcmy|C$Uwgm({oeh<_xsi_CqBLTB>C0vhx~8nKSjS5evSR`;7!x(yf-V~MSPg_
z(fsqTFMi*Se*5?J_-CIFhHtODp8R_K>j|$tU#Y)Pd=~WN<fDRzY!7tq=igU;aN)t-
zhpvw~pHw{l^N{7?x<@k4-Ct|HUHj(pOW9|ok2gN-dC>iU>*1dV84s>LV0rBRROy-h
zGr?y?Pg9>1Jf8P>;*<I(ryehQoc4Ijqva1xAI!SH;lbsHmmaQvwC3sL7u_$VUmCu+
z_oDxe--ml2?A{l@4Se(C_3PL5Z*tz9`SAK<{l}J%Q$DABtN*d>=lh@Hzj%L_|F-%)
z`}d#U;(v1g=>D<$&GP%~@5Vn@{yh8>^5??u>OU!e&HpC+IrzKr?;j>#jtEY7_5zj{
zj4cd*{~IvKGtFW>&Ax!`FN->x3r8iVCs!MnA@_M+BY_5ilL8t-y25jXZVTk|H}eY#
zJ`s8%%q7w<oGKh8;v^C*IGb0N`!3f^-nRm$1lt9C`DSxlb6#XGW0&K&!P&sg$UTLV
zk!uInT+Vgu?W|?YlbC{;)S138&0x95dYJVna}mR*zyJT9__yirqu;8(wSLe2?e_cC
z56N#Yzi5AZ_0#R25u+AU661saZU4OfUig#!H}jvz|Ly;GGZZkZum!S9aCmbFu&1)J
zut+h_X5P%2z%I_t$`;LflI1gt2^$l;KKn~{M@}{FgFLxBvRsYqs_Z*BrFi~u=Ww-f
zEMVhj6=Nx5S;p$a*2A`(?I0Tq$0AN%t{0r!I3qbOvo2x&&NQD{n6;Npi+wwLJm(*-
zyWEVtL44x;>ioL=IRd`~rwcao*YhUu-sF!FvKC6?*XOq3=wk0-JIAt#C6CRH;{wMs
zjv}sHUNr$uAywhU!u7(lgk}r=69^Ufz*op~nlpssJo|6<4;*Vbg}4g2TDXpJb#g!C
zzQ~o!Va{=Yr(IA;=m`Hdo_wxLoPwMa*nhCwax?J15tu1>L#Rh&uc(9AdC`X=TSUS|
zjl^z=Wr{x$mlIzvA|d1{Fo*v>zlwm3fS<r|fgGVt!s^1wLe@el!tUaI(qXbjvfHI=
zB-e{~iCz=Z5VRBAD3mU;UNltvv*b;ga=G8Kma_X~E##iazLdQzcVB*re1q&_DG!M?
z;>*SL#0o@ML|%w+hzCo27yB>bEOJ6*j+nfJfy5MXeKAjwjY1&;sk~3P8F>VGEV%D-
zb@1%s+sE%Gutq>tK#o6)&zLWpuaTcsFi>!+fU3Z8{xkeC0tx(rd}X{kd|L!ugpUXx
z7Cs@uBwi}TDa$T9QMym^o>-so8UE`$pSa#|aI#NgwPcyj?87X}^ps%}!$bz(|EYgZ
z|91Vw_0#mnjc?6g^S^%jTK{e9x4B;fzFhvS_j%GMrcdP`65p+Q!~5p)n=NlQzrFnC
z>MOUGZZ8kKNPK?g*{^3s&p$r@_Wb>Gr5C@SS3T!=KKq&0v(HaWo_%}P@#6VQjhBy}
zuYd9UCF83*FXy~m^wR59?<=ua`7htR=y-A9dB^j0FJ<4pc>nD~_51v{N8apuz2>Fr
zi!IM*yl8#(`SqSRoo^lAy?;07z1@4^w~t>HyqfW<>ecpFO|Q<rG=DYe^@=xcZ~Nco
zymR~z`N`!=_*a{+I$z7aF8|8-ZO1qEAA5g%{FeQt{qu@X+diy#ckhk&o3^(X-hF@f
z{oTIz7d}{gT<}5oz4Ke$H{P!^U!}Zy@k;Zx#Ou&kyf5pY+dlPt{PWSW#~M#$pFDhg
z<#F=k|BvQ8wtZsr<j7;gM`jPC?#tY}f9KV0quY+RXWdD<_xrx%!^%f4Ptu<yKX-km
z_hj><n1_+~b?!ReF1VF+JN3@F+mW{nZ?3qJa>Me*z8h&b9d5GTEV<cqYyNGoJNNHI
z-d%Mk=63F_Z8vY+WV`+UHuGKf`+5(19>qNNc=F|G{fpDDCcJii?f$y(wan}ISA8!_
zU-rGa_<GSBfwynoM7{NT@BHz>N1;!epO1Zc`}Od*%ilwOM*WWbbN<)mAKE`+zOVc0
z@#XsGRbOhq<^G8OmG?*MpDn{|=Cf=(9C93)9P2pda9m<P!uppviKUL!kyVbRo%sh#
z76&i40r!8djXaY4TLeA{ZW68&n=7R%$10yJcSu%5cCE}w>4OrhMJI?p6jK#v5kD`!
zT2e#irtB_RbJ>luymIQY_R>m{8WPJSZb{q|*AcT7;TB#aTrbiitR!S76eHXt;x76`
zgkMxgqD`7jx>90-_#^Qv;s?Y^MV<-W5Y81nBx)<BCLtr8B3GlJq986WCGQ|_DyJyJ
zAXP1CD1AhRT}Ddkox~@JX_DcRQQ~_=XNtv1T#<;A@RTT)ye+*<wpaG4%nq4NGI6r%
za?fSjrP!tRN##g6gU;WP_$#$QMqB!;M4;qGsjrgS;&~!lg&9P@iCz{tD|A`#xL~iK
zoWLgDa2{V?4t^tnFu}h<m7=#KQl%B8KTAl7my7KY>lSMkQ57l?)DZ0C-^=@f=L=6X
zZ#wUDo~t~4ybJg`_(S<$@gC%D6gVV2Tck|bM#xquTewa5k3cowd%jA68~oM0k~}`V
zTmlOPX9z6dd&Ha17tg<de=lDZ?>wHr+*`RXb8qIJ%U!_Z&C9?W%kz-uD&IQ+3Bf%6
z2YmAc%!PIfp5f=_o5@?w_m1xnZy-+^Pd@Jj-d#M8xZJs3aZlo9;C;%a&iR)kjZ2-o
zfO7)7Cx<Ur9JdLN4WFuDfiRayvaqP|QejpR17Q!r6#jF3%lXv>XA7|kzZbkM5G$x4
z6d))h&?fLr&{k-Hz<XXro?xCSJhQpxvx~DmVEe)$#>K-qg*}yhEqg2bD)tx-d(Hr^
ze_T7b;<;XP9pSFw_T^+@i)DGv>cg>)V;#F4y99d~+f3HIEOjhPStZ${*<;vKINopu
zaeMPH^St6@W8cR5l#P>P7Mlu77&A8uH|rDDB(`;I+U$?m9<ZEc+Qk&l{G5^T-};~L
zzJ-39_3iD~mM?{$<38R05cPi6+x4$MylQ`a=C#jj+1E;MQr|^?c>F%}E#K=uFKu6b
zd;aU`gvY55EglFxh<)(-LD|EWhZ7#!K05g5$RnS}s!!@38$Rs0U-iJ~G2hc$Pfk8A
zeJuTC@00o`El*ZGJO5JcwajakH{S2~K79GG@x!ur8(tT`RC#grS@6?EkL4c4KN5Nx
z_u}5G%WsU|U3e$?-uwNu_s$;zKCFA!^2YMDz-#R{kKayt_w()3H&b3`zfyY@_DboM
z)ho5v`(O9J7JnoB&gFyg$BYlm?-}0LehB<n^nvyLmA4Px#J{=p#{b>2_p0wtzjc3S
z@Im*Z%*VzLd*6Az(RkJWBH)G8E3G#+Z|1%F{Nm~Jmgird8@&{LE%ny$UCq0Q_ZL3A
z_}KWV{nNyc86P-5JbOR){oZ#yZ=zq#d6n@-{OyA`lisX*&H8%%>-BHM-)g*_@+Ru_
z=2yM1Ti^bA-~K7!i_sVRPbMFBy$}B&{$b6#`ENbmvcLWN#_O%ZJKhgFKB|3u{C@fe
z?oV$&`h58D_T}sP*NJa#zS;k#=8gUv?zcDIvwvp$I_vB0FO{F8KCS;K`Dy)^^dB+5
z9{v3E<KPdapOU}0f7|^I_`T})kKc3t@ci5U--J<(=`-VghTQ)L|9$?)@<00T@n3eo
zjQ^<otNl0qFWcYA|8yBcnWivhGH+$pVsd2o{eK~2D~lipAGa>gB<`17tGRdb7V_`s
zKgj3Go6IwhN0|2@ue?C8@Jx|2!fL|Pg%d>PicAn@5Hb*&Cz3AyO(IkBfuxsofoz?8
ztwN;y5t%wEA&C_ti9%k&%S7ji#fcskx+Bmj*e3i+_=S+YP_f`4{$M_4zGA)@{<r*<
z{0)4|cujau^1k4^%r}A8fOiSsZGjtt9s*2!FL>qnUHL=#*m$RMYjc@!Okm|@jpk_K
zk>>l$GoNcI$83&!oQt?bxR-Mu=9c2I;{Cz<pO0HeK=ikWsj!+5tIz_Wk3uGbC-`RY
zJ>)MHXcu@XASBc!q9VRo;-X}%RF>p6@e`s7!ft}8LMoz>Vs}Jx1q=9c`PTBQ@+a`^
z<1ZBK7yKenDDX`nSvWz|L?lhHj(<Mi9==xo5`HFrae*y@B0>iQ?FByy>=*nb^g}37
z=%1ji;C_Bz-V>Z@?8zKQxY)UEIsdV7uqv`Rvq-YsV_wB_g>^UE2lgeLKe%Q2_Xx5I
z$p{qkR&hshU*Hzu`p!0yHJ>$<Wh&DW#;c5fm^@kTvix9?W_`!9hS`C!^k4fQ-9NYg
zSpJ>)C-wK`Uk`r2{(Iqn6k`NaIg=t&FXLf`Zwz|O?^*A$EoEKGT*CB}NuQ;I<v4R3
zb31b<vpust^KWKv*6nP59Q~ZyTz1?{ya~MXxvjaba@F%3;eE=xjK_}qFV{b=2(Fu)
z8#tvn`8kfWePXp^U&-~5*O7l4-(?;%?s?o7d5m}_aZY1XXX#-IWz=UVWth+OiN%R+
zKU*zZHrop}8MY|qM8-#qA<P9VEUd{aCCs0hAF+I4v1R?jD#P}NbusG}79-X=4so7P
z-ew+kZUyd@yafU&0>}9j_;&D4=DEh5%_GUjBv2^$TWGC_gQ&k~i<qIfhp3EjzA(F}
zoS2{3Es+GFOQ4fecwM=*xmIz^U@K+GWr}4gU@hQy%`V3}i%Ej<F~ep?DW*Ec8iwTm
z$Nn+@U-W+qLj}_%=HJX!%q>j67&QK$|Kswr<!kikFQ1lue)uKp>y0l0Uwl7TeJc87
z_POQrpHB*(W`6Ab)b;h~kKaFee+GSb{3i50|7YIs&_6f-ocwe3x7_d5zqkK+^~dke
zo!`rU2mO}*E&Th|FUMbwKly*ue~<Vv>!;4o_22e?-u;o~qvOZEkG>z1KRkc;=<VZo
z=RZ#SV)k{)XT6Vo?>*mheo+6o|D)c=!|xxyt9l>tQS&qRmz`g@zUh6>{66pd?eFit
zwSFo7IP)F*Tj#eX@50`7yp?}*=oRnl=Wo8gje6_+hVQlHtJSYAzq$Wz?)!c3CcioM
zD*1K9yPl7dpC^6N_^9z=)%&gQk9@HB^!<~>XNJ!oKSh6j@>%!uzYlKjroO)Rviw!e
zn?G;7U%z}|_1x+Ctrz95Cca+%M)>WTH<RA(`0(`e!7sv}kH264ru_A(*XnQPzV3aQ
z^X&Cw#>XX(A3oXnO#g-K%c_@tFB@NWzS{gM{>70e9FLSAay^>#nCr=%$68NzJc)Vw
z`q|wVTVE!<eE8z=i-K2-@AiI-_?-Ib@B0I9^WMt6m-~4BBm2jB?<3zezD<1F`7Yr@
z_s0hx13xYMyzT44??OMSzwv)b`4s+1>dTUEEI$@}um0BYwft-JH~a7L-z~rAe7E^=
z@0TA#D%%#$U7S`Nv)C51zGc14c9czowUFuQ|DFH17!;WgvA$+gVee#@;+)Mrhc}CF
z5+6U`MjkmHVZIc>2H^_f6@ps)b-diXIXtCY6WGtO{A9Yrw1dTheF}#^hXA`dYbLV@
z^Hvs3HeS{}jM@Jf{(k$*#gNKa&-jhuB7-v1B9;kkR_x{M+U$#1by#k)7_b{~ns6>)
z3uU%s>SjL7;>>c8=`W)?^HtV$9CqBxc=z(<@yc=Qag}q&@ZAzvCvcnhG}mL!4sJU>
zM}ZFl{{*T8&kEU#ED|{-oGCOzz=ltXSB~Fac#@c}_$JZg!fC?7B0q#Hgo*_2@LBTh
z;gc6wBREq?R7hN48*eqwe%?~SWg?~`9D-qd3wgcyWcl^^<M}*zXYj1!(ct?k5Gvdv
z+$h-0SHN?hr-@%t=&sNc!FvK70=5Fh{EB=FcouORaZ7Wb=ibG8kKbFMl8=q&4wnVb
zVSWi=H<3@mGlb6xuNFBYx>oG6n4p-a=o!%h@pBT!l64X)Vv0fs`NH^A1!RQ+gs%#R
zi-?KViW-YLid+&36a39@&HtXCUnopus^|hyZqb(_)?zCq{G{ZiS|s?z^hM^291&X}
z;U~FKVwJe1#0sg;GB0HX<)+HMmwqC7RiaR`M|!R7ciHbUxiZsbSY=~nS!F$CWMvl1
zRLZu=O_1L#-!HdUhEZBxYL!%t^d>1KiN_)(LY6`wg$2d<#2v*1MOTOhh))pzB6>iW
zOE8+Zh;s_teO6<RKJFwQLGBE$iCn9=ZgQ>V`oOt|GlXj{*9oo{+@-vid6)B?<hsQf
z&HaP-5??>>Chl^s<y;IrzP$2$2l>wN?-sl%{76Jvv_jNHR7IFWAe`?lU#;L$5f(9b
zQC^XcLT3c$3vlrt;=RP{z?aHv%9Fyw$a{)gmcxK~-v1f@Wf;8~%Ks+*-uUb1ugu>c
ze=YvG{zvo=sUK&*|NFk<NA*wPpBdj;J~@4O@?q*{>u(Fb{rK|h)6@@sZ*5*VzF>MT
z|IG7g?vpJ~ET4P5diZMXi%U<#o}@i}^IYrYnitM5F1+~o^42Sp*DbGO-W0zrcsKX`
ztd9%7F#X{9CG=D4o9vfqpO1Ym{__3Hp)X2bK7L~S`04$L4^KZjd~|xh;qBJ9bKh%!
zI{WGVhk|#$Z>PU~_x9JD?$?s91zzXBVt94-RreddckkZCzF+n}>%*;&dY^kfFZ#6q
z1M_>IcZ=U`c;EaX;KQ$XQ{Mc1+5ck7i_<T@JzxLy-=qBx86IDMlJw;M!$0@8-7k4i
z@Nm;Z-G|E_+`KP!uj$U+I}`35yWjgj;L)weo=*#({eIT)jOXdOM{W<E-TQif+oQct
z=04f?Nb`Zyy{~uo-^;)6f4}tJlDoU_e!Vy2f%d~2_jB(ay&ZLD)xA>>o;>t>#QR9%
z;gS1O?tQwu;%?L3Rd;>vdEWngf6x6J_ag4Dyj6N#?OM@|y1S*1xSn2oqV@R8!{mon
z4`$qJyu1I-``bTnf4f`$@ZytGPftANec1be{gJ`r9gkN&IsCNp+3jcNo*6yc^)&vO
z#ETm*wO@aJGyDDTkCC4ye{g%(`quCLqff%$TE2IE|NJfLo8z~(Z>PU8d^i8z_<iw@
zs^4?|^D$p!{>eD^|FwVj{)aK9F*Pvzv94oZ!CB6AiR%RS4<3Ksxjb*VyLcw^MGG(q
zyyl(FvzvFXfWL^n_za0Q2@Z*M;;a&Vl5eC_Wsl2nNykbZmUfn#C9fsdF7-xCLG+x6
zvxtOnkx;SFXTiS$s|9iduL=2yFo>QK*)Gx{+AsD+EL1E)<iFq?!IQ$5#9m5xN=ixy
zi-n3j7XB)-OKiQksQ4+-YSG<d*%EUkCX4M7o+8*NFiYTtpuET?F<psT@yDWDL_drD
z6!(zWB2g~cFU2q8AZsnFEMqG5S$vt8w0NoHbD7_AD`nNC>cvlqv532gbBX(irHg$R
zzbn}-eOo49W}c*i*b!kR;UtkuVz(tUCG*8wM3{tr37ius5r`6~5Y!XdDPAXaOWH_g
zv($Pqd!Z<SRf4S|e?*-`Yz4Cgt_!{qx+$b6bYDPH;J!eSP@hPxc%`JLM45<{z+qlS
zz83x${2%$=@ka7W^9Jxn@yhb9=6%N}AaGsqkBFYc5lIF~Rk1MPn?fr@7{!ewn#HGx
zMv3%^ycMw%sS%125aJi+zsUbp&|NH3%3S)tgud89ky9eZqECgr1(W#m`7(KT@M!RM
z^Z5uU2nq;X;@iOgSn#+|wBTjFLf$q$Wx-(KpTY^keL_oxCJCF16o}jtStp_{yjpOM
zpsesgQ2_~4i9)e+B2z^wL|erciIs`26TT++h~I^88?O}KWxm^dF1)hb%3N$*Qk;j_
zt}v%CF|zDozs~iUdpfrqS24Q^i!!7B|JZ+}f0zGm`}Os=|NkjW$}EeSKQIO`O#ZLK
zz{8}-V#|7)<tq~d!<N6*e@p*y{ulZ$|9|IyTgDemH<(Ho68^6JedTw`Ujc@%OgmUU
zuz0ihG8-{jG3GJk{ZIer`ETz30wy)KbsT%ypE2+HKkLu*UsAu~e@FcF|CjSu`S+$D
zj^A~@J^8%v!-9AI?~cDyf3No5`JMWkrkB3Y1D~lrXMU;t%IVd_SK+Vu-$cIQc=PL3
z>C5mJc`yFG^m!xrF6|xX+Z!(ho~b>lc_Q)j+mk6z%$_`aeBp8ClkBIQ&o(}7db0d+
z{u7O7FP?pV8vR7|vCgB?hwP8`KjwP6`RU4M;xGQc)P0lu_U)TDuiwAYe%by!<%Q<!
zV{hc&SiidXV%5uvH~H^E-bKH4db8w}*h{vTUax{*=e>US%I4Ldms?&bzuEuB<+bF?
zz87m>Hob0md*t2!cf9X!y}R+&{B7~Ok`GQF`9BJNy!3J9=Zjy9zP<e-@G12Ds&_#j
z-hJHr@%?+dw@$D2KF@e|_F3r5A8(xAi+*_VanBdsZ%SXyKK*)U_Ez!j*SBlm2YeFy
zdi~qXZ%e;y{v`LQ_EXEJPaiWr=6}-oHtCo2pUJ<>f4F{o`?dYM#n0zI*ZoTQeedVl
zZ<SxafBpG&>({7nufCW6So-(C|NQ@#|2X`#`WEx${^to_8oy5Xw()z|_sB07KkRy+
z`r+^Un0Jrg{(Rr@W&Tg`Kgz!^{K)ub_2u5Dg`aMJdi1I3Q{cx<?=#;webD&0{lk^_
z(eEujh<_^nyzcXZPxc=*-V492dEM~({M+vzR6m{jko~^+y~&3YA38rUyw`gx|8~y%
ztj{Ii=6vV<uJmp0x27LuzqtRr{M-27{{Q{IlK&VOLK#yTKQlBiSTMRUYq93C*>VJP
zo#Fb$S<m&0XEDF6V7y=*|7spZ?iTLjJiB;5^D^=M<tq?$6}c|DQM6OUL-?~$f{3KJ
zpJcw|0r4`?IAIB)?ShShu7VAMe+8L@CJV+1N(k=YU&5=#6U|-9oz44F;J5HLk!wP4
z_yzgo_`dNO@-OF`#dn6kP4K_aEnypBT>%%K$6V`pa`~4DMhi=c*a#~NUgWdp{mQeE
zcP`&?{_}z}L?(*!NL`dNmE;urAk;2+T!>q&TXL~<wsgH@wRpJLZP8BAlOplL7lkB6
z?@COQvXpccJ0$dw-;)27fU<D8sDt=6@yBA9ML&v2iOd$>Df~!ylki937SV3;mEu;S
z&jlU%pYrDO?G#up93ggETu*$ZXuaqcQ5_K$fpgqSoGUrRxt4Ly;ST3|%yEnTEc-5w
zZ=C<QO1N3M?{H4ySj=wDzL8Cm^%>J6MjNKfOj*oKESs65m;@L@8D=p|V+v$F$99!%
z0^1?BgY4m)np^>#ZS2l$?QHxUVVstnm)RNFKC#HNE@gesdW7v5I|KU`mVV|GmOZTO
z>}DK29B<f_*!$Q-*`~6FvnVl%GZg*b&2XLBmi-!sGsgxtH`d)O?5we@#jJ-|=d;Oh
zaBxlH=HMyi{>`byQO16YBZ^yvcMY#L?|Sa7oNgRk9QQdqxOQ@%=Q+rGk?#$^xxhQV
zWjtHBN;so9O}W4D%JW<B_3~Wd;^K5>Kf@}`n$KFqcAtGW=U(n&9#5{LtObl5{~P`p
z{jX!-U^>lQ#cIx`!Zv~JDZ4J`b<UMs!93-B#{}jHB?~R$PvyDI<;;DDhn3${@RHCn
z;Z4G<!b=5>1!DPc@vY>$!QUd3E!rZ^FA*Y^E1WBMOMp#qv0$3e9U*>U4dI<arGi!h
zv-#`zxOh%+@^CHW5#`V2zr|b0{fgrx+f3GPEOS^^vUIV$W1hjhg4vE)hv_tf=l{5W
zOa2}D|CS+(v7h0<KaRi8f5-o}|NZ}$|1a~OhrXMCOZyu2b<3BXpQeA%c<=PC<(<L@
zgO4jdtbTv%oz^?+ckAA1zWe*8^UcgR_HX)Lr@y}Xdg<%>SHiCXUyHwOc-Q!T#e1Xo
z5$|h1-2S-b(~?gSAC=yVzg_cs)2rrJrmz3JKK17H+q>_!zvuh7`lIKEKX13b(R_R9
zUF3)VA7no=e%$%t><9jj^FKI!Q2bc)Dfx5Um-SzlevALM;fv;{gb&v~NPTAc`t$3D
zZ>zt{f1mu-;LEJft3I#%obx&0bNS~_pVPlA`|SQv;+@v(YcGXgZ+QFhz11hvuOZ*{
zzbAdQ_`LGtpARQLT>kLj<MS_Pek}bJ@^j<2nP1j?zWmw#%gQg^UzdOT`i=iv<rk;V
z8lNYAp7uH8^QzB0UoF2o|H%1v<CEBjyYJNA?RcB@F8RZfPqJTRzjS;m|8V==*Eh|t
zlwYoTS@W9royUjz4~g%#zMl4K)2kydbzjVW!S?$9o5OE~-x<6UeHZxF;I;jW%xCS-
zBA*vLU;AwS)6Y*{KH2u<#$)rxdmc}Ga^y+zlO2yQKeBw(|FH2v<^9<EJ`Z0#@qBLm
zoay<3=iD#kUi^DL=lR3u5-*Ru?0PBkit)|ex3}KydB6ID%*SmXb3f;OmH5{5RqyMG
zuP)ydzNLLz{$2W~+^^JM?|&NpQuxF0@ASW!|63VS7>_XZGwU<``Ty|Wjem9jS24V1
zWMRI_{Ek_ZnVZRuQIT;2V-eFfrkzZR%=OIgnHDjA`TzQ_&Y#5J*}qf%DEy26zx03T
zfA@b||GfRJ_vgzWj(=wVFEUgz*Rry+{$(;}oW@|rXvp-4$%|zU+aq=sb{E#E%uS4^
z{(t-*&A5oEi6w+Roims7CVK?yUZ(d9QVh8azD%L4u^gdX9bA(*pKx??JY!d9zssh^
z5y9=scY?2h_ZW8<S1y+cw>i%yUKW9Np@YJ%!U96Vf=vQ_0*?i<1lfhf#6C#yO0AY$
zArUXWM)a1jgYa^Z7h-EA?n<&r{g*f^&Lz$u#xC+i@S0$U$Q<#v5*sA$id%^Hh&>b8
zCNx{{ncx(m$3mZl+=Tf=wu|f((G}Sw{7?9lh>K{R=pxZp(Q_hvBKgAqg>r@EMGeL6
zBse8*iJy?*mYOG(Dy1z|EUh5Npb(`nQNdX8nqs8VY^7^TmC9Bs9V%8Tla(JU87Xxr
z`YTRV+@^R`@vq`K#cG8>d3JeL`T26}a?)~a@@wUF<r`$Zr8h|mOU{u<lDH}{RdRvU
zH|Yg3-ZB}|Tv96}G$f`-6i7Bp^-43y1W0d?d@G(S#wQvt@>^tvSd@gB<XVYI;wQv9
zL<NMq1nT(%1z3b+h0TQ*3rPtU@>z3F<uG6`XWPVjko5~&ANv%x)hrTBR{s_M_Wqgs
z_s;)RrdF22tdCgdvFu^|{FmkT)1REb7X3>5o$%+|->d)Y8T$Uu|F`j9&Hw8RPnqIb
ztJ$Y<tl;>^euK@BEtqXHTL60=#~RLST(;czIg8k@u});&&-#*e1KUlG1>C;8F1(&R
zq1-RI1h{*+eRynmfAe<piSw)Si}5+}gmK;CxWLZO{+sO{yAWqP=LL>=?9A-*+3Z+T
zncJD}GEHS}Vzy#pVm!_;n_=JorT@PFJ@9wSU#-9Ae{1}b_{sb8@z49ebbe3y)%vsY
zNBwuVZ){)VzBqh&{#oa9{U^6iKR*8Xu>JkScYSX^yjlMy=S}&W18<7n?0>EGI_Z_+
z%PBAPUru>7{q?oimTzR<>b#A9{q)8C=g(ePzrOcY{=<Tg+@I<{R)2_k@B6OgE%)1~
zx0dh1-_QT>_2aEiT%XN86@K{j?!~)Z@2frtf4umy<&)~CfDb<J(%u%l-TKz!UCn#J
zj~buue|-1;=Nrja880rr(0Hx-&h3Ni$KM}|KgE4q^G@JR_N%a$DK9QQk9r~UYUb<A
z*Xv%^J)iPa;EDg^`;Q+!J^CW}b<-QkH%YI4J^%Xj=aZHvp-;4*T0ggYdG=-X3#(`M
zpGZI5{B--%eNWq-U3h-x#mpCF&kLT-d%Eb!naB2z10OGYa{HP1i?Zi?pZ<IN`_ZyT
zwvR*}?R$LuS@DZ)&n2I|f70+o`N`c!o)4EkWPOq@#U?uW@q$OmkCr`p`sn>5mdEms
z=RJP<<mS`qPg|e1K9hKU`l;pP@Q14&IzOKASnbi@2igz!K79Ca?!!e7zdd^M#Q*7$
z$9a$99{qmQ@p#SS)sL4wS@SIL#i<vIUnIY9d->vZ{=5A5S|5BrKL5DuqyEQjA2L6t
zet!Rj@2lz8j&GZOJpL8_XXYQ%KR16r`d#%W?Qh?|J_bAHdn`sQQ<#{UY?*6Wz1Ukh
zu5)Gb%@;V%AIU4gJ(*LTvyjt?OPyy4pP#@}fztw#0?zz@_*D6Q_>1^e1zrmpig<~d
zh*%1J637z}5Ks`v7t|GT6W<`QP5hbYMG+^FWx|VuZ-{V+mq`XnpOM}nwLxN%7?Y^A
zNRdc^=pnHJ30<jkQan=bl8+@8h&zd<2+tC}AUZ=LOKP=Lg`|Ucv`C1c2!9n{Ccl8-
zF`=iz8--hh_6YU~N(n{@1PW9NS_p3yu@+q~GF`Y?=$0Ux&~u@;!ZM<LV&BD0Bz2^s
zr0z;8NuCq;6zdUtD&Z~dA;Tu4B|TXxT55viREaRj{n9D2FJzjfcS&87Iw-YFl1t*D
zn2i{dScuqPu}*OgiOCWUk~UJSrTQeFivJKZ6k`)j5IHAuO>~Bsh}c3A3lVeC1)|GE
zHVWqn_X~dzE*D`EH4@{HaFJw?TqIsEc3rGW;+f=5sYB9YvJ!G@<>t!Wk}a1NlM|G8
zRghFVrtGP5S^1Jur(%=BEqO=z334Z7_sKk!z9s!u`o2`6M6Ku?VR7N-LJx#uMFPaN
zCHW;zilhim<e$pVAiys0Phh6-H_>{r1)_=~Uj!5Q>v&ChZTaT&&lcblJSeb{e<B|@
z-&tNWUJsu6+~2sxdGGNS@Cx!ca2;oV$FiJ>kExoOk8KY-H~Ti0J|-o`zyC%4Px?3S
zpUeLc21&+O3|Id<{?qug;pgS=2fmm5eEr+upD)7~Mt0^1<|d|ihJ$~X|2F+q{`1*S
z=ih06CjWW#d;6~&KMj5z{=M&S!2g^775=~d>-$&h@8v%Qe-`|){JZ?G&_A{RDGZ4W
zXa1@DZThq9&-Xt;e<%Gt{+Ht)$G@F_SN`4b_xzv#zv_Sb{7n4C@F)K7^S_h-cKiwY
z{o>cr-@pD?{QdL$<4^M+dEcjh@BMM|r`0d-pXa{!f1B}j#g~N7Yd!{lDEUzO@&3n_
zkCGo7K4`u#ew*;d^UawzvTu*Px%WEY^|#kXZ@u2mejWGn@bhcWraseouKuF;Mb-0N
zPnsWYy1)Lu?}Hl;dLODhN_sTs(WOUekJujS-CuV%{BHQYulL;^hCXt7-1>OcqZ<$B
zKDzj1`ZM|GtDkRwDfw3IgUZLl@0;FUc)jYC>Z_>N%ig(vV*O(MS>j{s`?B}ZAGAN-
z`FP~ho6o#ocYf{ty5LLVXZcT-A0<9=e){!U<(v2SZ{N(nNqzJ9#`w+YYwDN(pYMF(
z`yTys-LK2PB!02~jQp|ThxaeWzmNW#G0HJq{2TH|_RsI%Y`@R{wETJSr`K=mzeoRF
z`)|cy{a^3zpWn5ACj5>4C-Z;T|44@IjCsr|EE}0wnWLE=F~%`1XD(&oW!ca4h~dV+
zzQ61KZu)2azxSWg-^;)M{#yEL#;=rL4}aSJoczP{$LAlvejfSB`{UKur=MLv<$e_V
z82RDXyXv=vZ!+G<y{&xv`3=V#o!2$5f?n-;mHPVhYqi%QFF!u_eKF~!!>h|LIbYm=
z68O;Pp2MB9w;$g*c{l8?(;db;kM3yPTXpZ<-Q9Qm??m3sy&v+>{n5rp-yi*c6!v)T
zlk#U3FQ&iZezWt9-#hD%&Y#0RSAPosSo}fjW9#P^UuS&p`l0w^>$jI*E5C96aQ~I^
z$K<c*-&KG2{r|)0$K=EC<*)1C^nY~>cbM2%U$cpF^s^sh-N3SmC7C6ic@fhsrd=$*
z*e7zyaaVIq=RCn7#3{+m#3v<qS6E(jp~xxWdBU0^3Zk{5T_P8RUx-`~dnJBXe7V>J
zkt;&>LQ93d2|o~(6aOwID5flGB%&%TA@oLI3%@FVjlfdjV`BailO;ZgPZA3j{UH)9
zI!i1_Vy?tzv3;VEVrJs+#iomz3HJ+F^2_pb@xSF;%xA;<jjM^Xp3|Qzk(-5QKesb?
zA9pejBkvcUXr9g7let}ae)G;1ND@vIRTsM_I#;w@Oj3eP^1g(M1gH2-(KOLtqFcmj
z#5hFlh3kaw3%Ln%3o8qS3Gnk-@g#6h<Ido*<?-Q~#<7oM7UyElcFupCSzP>Fk2#{*
z>)7JivRJP$`7)gN@4?u@9L#FLX2sUZ8paaNoWi`8#hdLf+f=pyHXZf`4p+|K9BJ$-
ztXeE#tWNCgoR7I2cwBk5aQAVq<QC>}=AF!^#9z#BBM>fVBBUx*C$NimJ@<9)6}++h
zC-`;wukgBXKW2AgEns=U63;5ns?8e1Zp^)q_Yw~umj!zw>me3%)>&*<IZktNbD!m`
z;Ml+(%Km}vJgX5)C9?y|ebxqccMe;2bJlCjdCc3G)mYux8##11p0jzeYO*M@2(j#A
zGG~lp6kuj$ZD8BXwu8lv(edB%KTm&)|5^KY>HlH|H-=CDXa1k`|M>r&e=~j``}yiu
z=-;CMOBi=E&t!38@nnu=TEO(2S%gKJ>D>R<|BC*XGdyAtWqAMZ+`s$(7c!n<-oYBh
zmdv_?xs$1zX*Y8j>uEM+b|&`e?ADy$xNdX5=MLn4%NfXVihVIhA7?113CCTwv#jS>
z_1QOZFmeWQB(vXV<KvLxKEd0~_n0q8;EZ6ckf!iIVSW(};R}Mx1&#}J3f&jpC!{YR
z&sV@Rfy<xMl(T`;ne#jQKGq-1*O`4;;+Ulvzx;drH}K#0|I-;yFlI0m|C{;e!Y{X<
zIX{?x#C-4nX7erm+rDqhzo~zH@p1k;-naMPSiF^Z$NT=tyNI_Duh+h8c)|KY=Y{o)
z>CX>8yYlSi^B*rxzc~18=Hn+1iypsz_W$LwH<9mueR%f4>4U%r%MV!}0zce;ulzy&
z6W3Sc?@PWf{wDnO&F7xaFFt?!;`dGFJNNhKZ@a#}{>uFQ!guHIE5BNOY58*Zo5oN1
zpK0IMeP#aA`Dx)N-!F&1Du283)$MD_*Q?)_|ET@h^YhftrN2`D$o%JJ>SzAUG?no&
zLkYu+|0WD^j8_@07}qkqV2EV8%@WV%$a<9N3xgVi1A_)550fraD<dC6&%eq4{{4Tz
zV9#*qpVi+vfBgT}|NHZA{=b`lv;W@uJLzA^zplR#f7*WO{aEtt?l=D*#Xq0?nEZ|N
z%gs*;pPzsB`Lh4>gO953h2Cy`<MTHCZO5C#uc}@=dlvef|K+EbTVBk1%J4|&!PN(W
zkLEnu@My>5sZaEtgg@T+XwJj_`*ZJ1ylHo1?oH-9&UfG4nSA@_P4k<VZ+hKkyc>M4
z>fXz{diVa_-}K1zY2dT!r#l|Mdi3XU{Il{Gaxc%nIQc^B<?5F`ub#Ztef#yT`@0$M
zqTgG8aQeXhp7SlwYq?kMuZ!RP|0wsx^UKE1`#$S`QU1d6#q!IR&$^%1d|dTe_WS*x
zZoeLWkN)!i<A)DsA5Ol1^8UyBocD$Aro7$!_UyZP@0Y*l`EcRm)GzVhBYraddi3+z
z&ojRs{h9lJE@L6nC8jk@-x&p&rZ692dC#23_~f6>KOTnVOg1dfSU#}aXI5v@XG~(W
zVEV@Nk136bm#LMBpCyd#2Rjo-AbTkLB#sGOZ@5diW4OY(u5c~pO6JPuI>>pJorQG)
z(_AKF)*5z3_8OK~j0+ib822-ovUIR+W8-1J!)C|c#&MLZl6N^@4DV0wFFdFD4+(7(
zi4^@HdQ$v`WS3N?RI^m7RJCN1gp>rMq@EOyl##?7(e1)r!ZM<&;&u`b#H++=MTNxV
z#Kpuvi!Bka7GET4Cc-LOEq+z<m!z0@gYZT{N5LNge1b0p^M&I@|A<WzKP>)5yk6W!
zOj$Hg)I?lTDpcmR4Cp-bVA;*`XBCx|S{3Ij)+q%mr>Z!q>1Z6&SgrnFwOaM4n!KjI
z_G9gTTCSR1>h)^5YLnEo)c&geQ#00BqVY!~M{|w_v)WyyVuf&d6Zyk(X0jWk!==|s
z_edX=ijWi*PZIGH+9&uz$W=63{DpY8n7ha$L45&Tfqp@2p#uWkd^@@IxWqW!IJ((q
zu$8lJV0q2*fHjOQi**L`Yerti+YGZ9L>Zd?FZyTsH|WoyKc#<v{c-rs`18v*fp5pZ
z&HVA|7suZVe}DgB{$u_p>F=U{(f@h=*Z$4_J>%Dx-{$|a{zv?m`0xJz>VGAM&Ho+#
zFa6KXQ1HL`-<iKR|E&I9{!8{};*Zpy>Azq9S^4+W-^RaJ{yP45XUu1=Wc|Rlh5a|X
z0*5MRIrj&iTy9N{C9IEGx>$4B(%7D|eq&{0D`3@OUC8=?^$Y7!HUo}joPyljcoy<K
z<nQJ`!dJ{6Blt*2UbsytQ7A<?UW7x0S(snQR=|YUjH{00ILAFM51v_E-`O5A|6*ij
zyuk2=ft@jx@jT;7#*Ykt{{Q{=^>6MU^<POpdVd`M>GG@f=i~3<-#&b~{Q1VmneQ*Y
zwRm&uRmiJ}uf$(>zgBo{`Lg%foF|u`gg$@&QvQ|o3zuizPc5DaKfn2`;hEQS>lgVi
zL|;C8b@I*Qx03HT-qycXf4%SZtJjQgD&AatBlkA-?Tt6JueDy)zl?e5`Lg=usaL<>
zguUDMR{f3s>lv@Vzj^vr@vXz_X)i-woPVD6LiFYB7rrmJUO2zde(Cw@>nr!ynXezd
z+VOJFi-|8-U;4cK_d@sO^;Z|(biBR#HsT%IyLE5sUst_8^QQA%`Fs8MA@9uJdA*<a
zLGNSK2g!F=UbDRJf4%pO$UCq1x8JwFk9sfq;m3zVAJ=?R{QUmY@lWqRnSajyZ1_3-
zlle!p4`1FVebD*%=HuIsUq8gWfBJUGTjzIa?+(5dee>|;-k0BBtH0HHo%r1R$<{||
zk2oH+KT>$i|Jd@;-3QC>Gv2Sdx9#rSJ0-VWZ|UCLa8u(B|2^@0Nq4pGyt{S(R_pD}
zxBYG(xK(f~_EyBLjknlt7v2)Tsc_@^jfJ<r+~a;|`0)IF{`<2YC_aAsMCzHybF&xP
zud3fLy?gj>_4~8$pS}D2R{CA>`&l20Kc{_p{^ju(gD=ZI-}-F!h5bv==T)B!Kkxlo
z{xjf@@n6F~HNWbAE&5~okMG}yKl6VZ|JM1l?eC|5lmAcp|Ks0`zkGjx|7&41Win@U
z`QP)`^zYQab^kswY+>$YF=9E#!p44(b07Cwo{2onJo9+6`OSn)#LC2L#iB+23u}sf
z5-%0sFUlbDR!CcTw#aeOX3^Us0ivHpgGEC`)`)0}y%cK`l@u-)d?~0e(j(d~(kFaa
zxI=iQaIPqu_$IOEB0q$5h1`U+giZ+Z3vT2;%va1;z-!Or!^6%a%z1$&nt2E71CFU&
zp4?ZtYk4N~w(xHi>=Tj}QWWwR`YgyUEF@+iQ7-XX;<)5h$=8y*q>f5mlFX3E5)%_m
z7j+eTB^D$0S@@HngV0Wqa`71Pg`)1FsUkT-w**s!a)hS}X9#8runDCImx#2BoEO?H
z=pt~CcR%k1{x5>-gaw5;h29GZ2;Uc~6G;-AD}Gx1koZQ)YMDuL>GG0tQgRO!jFf_v
z4l6ULwy4-BEmU};7^=KTnN5XNC0Nx*T~e)G#aivF#(Rw#m3+l26=n6eY9G{(Y4~Y|
zX>zNlsb^{OtGlVWt9r;iRt>VRx2Y8R_J>JKPy4cLK9>u-p5Ru6+lo3oE&nU|Z^(<u
zl}r0czGprA<tE1s?Nd5)`QCk-@U4=QL+PqUhbYJYTVJi--h5>9{;)`&W{%*lPm+J~
z1sAY+e$aTc;ro29B%VXxw*6cv(!@IHhWvGzKb5>w{#L(oebf6b`NRJg^$ea0<`M~C
z%bzTMX7qj%!xf%3uJvs7e{a3qf6weu%9}GkEq*V1v-Q<2rlpdh{CA#LoJ+qO`o;0<
zi;qPgK0UmBchVR4U)vw<xw-$9!uMC-)jwW%8~2^zhxv=i&n><we$9Km>Eoh5*MG}=
zFL?Li;o;}pKXkqsf0^}1|MTrfG0%2>4*q}f=dFA5F0Xue{*xcyPIX&(Q}(@IOP{}b
zA@Hk*Dd6AK@3X$l`cuH;CA^U<jp4*^kIx6+n*9`DNP5$DZNd%5XZjx||LFU{@-pD=
zr@I2Ty071PbpCzu9|rbEzgeHJc^dI?$Ilr*mcQb9Xz@DvE91K>Pcoh{K8d&{dRgPS
z_&?SEyIwI}e{h-c*0BfbZ)W{@&XLX1@K*a)*gdXy#_yKj4Y;fSjf45YpYFdEKbAc6
zc-H!{@BOb=zu&z1qRy!PKk)OBZ)$&{zrX(V_Fu?XyN@;BO@5{Pt7Q58xBFAhFFBz#
zlC^9W-#eH(`Ewa|yj<{UxsZ`!laS@l^5;R%xxZfhd*hq+>-jGmKWO~C$I8XK?Jw*5
z8GkQxE3lZdt`WG#o6ok2c_C|+$X2Ox?q}@0g1P)3dCRz-|7rRc#C3_aoH<MIro;}x
zR?dBFHZ05kng9R8)h@PN<Qm%*hPhleJgvW@K79Tj%*w?v_n+$bo4+$ym;5t%&h>mR
zqb%dKr<d<&-Cy&-_?6#V<u{+cC;a;R^v%_;*Igb;KU?#t>&~m2Ja=wCV17378OJT3
zv)e9v+%P>CaPiVT*$0BR{VtZByl^G-!GSxKx87WvbUyfs;9Z8>kFUsG{&_9y&WU^7
zcbjjxUHfvw{94Ray(bOd?HD$Hw}15g@|TMqSIZx<ek}g<?v?bz<@Y6D&3)VWn*H6v
z=T1-7e)<1n@#}T(9sXATd;e?Q2bG6rPbPd@`Sa^bhI>|z-+np3sPOB`tB)T`*}pUY
zdFA!^>ATRMl1%n&EKJ2LDcn`eJHDLxVD`P6=?K?OuJ7zCnVf$LeT!n)!K2B(^xLUl
zn!GVQFBlA29*CqW>&upKzv0dj@Mr(RTdQzOvYXpbgiSeHxk)05_ZX)UmldZ1x29mK
zXsU$0T$&QAe5+^@XE;k5zq(AK<Sya$QmM)-<i$lqc!YVTiTsjVBHJsfCe|z-DHJSn
zQhJ}{YT=1ucV!-nI5RH#<He&bsU>wwJXS(i%2<4Yz(?+roDVpjGVlENm)%V;iDN5U
zg4jf<IR2AN65p-9r*nJ}kC7<lQTn&>j{=7R=hpw3Z2ts4un7Eo|3~isGv-)sM=mEm
zJ$}FcCVyGktC=F%VucoQIQ>1(bdf!pje#kHv4oZJ|Br7g7_RY1^L^ot;pk(oV`O8W
zEuhI4!u0n4PnH;Vb<Ur>m4bS_vRn&!Ww@Jo<K?a^Y!dk<R4KMt_N{V&>^j9Kx+3zA
z`8-A5O0uiE>-VT6%H30bulPgRT=S6jCJRR!PYr)r4|P_Po#r-1Sq4)KSv2>{ua>$h
zx=H$x$|d>3q6PxKe0%~c1o}iwWrHPM_{&A!NLdJ7<66dVBiba&Cpt|?oO>}V3)gxv
zR|RK@eaub2_A@W#zt4A!`SCw4mKY8dj{A($e$;)R`AzHn&Trg2GX!l|XM7L(qQ}V1
zw~<SiQR2t(j~2ft@O<Tu=lJpe+&9q=A#eV?68fCOaEGPwug8a9&v(B*_)+BhddAs|
z?VlV!%=k6skHkM|rmjCne@y!4^snUWu{Up?uX?ZlFZkEG=PHl4y}tEM<s-+>g?|tK
zdH=)c!|xaB@0T((a2;lp{#Nw4>_glqcV-iQ6^>Vb&VFD0Yb8q;XF2OmwrRrR3Rh)Q
zd3Q4`V`<=<&Bw#l%$q2xFLhdair8r$F5a(_y{h+BzDY>%^s;~9Js=aQ-mTaxB*&%5
zGlhRS&kl}s?pK0-Vkd-R*vl9^m<`zGGAaJL^(En_A>%QYCdT^jR$tdK*f3XpU-X6d
z-%&O$o&%io{(kx*`6KvW9FrQ`71s2B@}H$%%ztYC_Q=OgA7Vb)ef9o1=db3Eh3^F3
zy1YC4KJLThmt8M{zkT?t{oUr1$CsHumjC(o`{Y-P4>Ippf1LI0-|yVNKYp40GX8zz
zm&vc`Kdyb7@#_`iZRQk)wf~;{v;7<UUx;%DuQOZmzjJ>(816C8W~*aYWBvP2<d?`F
z0mcms^Zrg^aS&Q2^o}+6-@QLqemMN7VSLE_k8d}B9DgF)zCY{#Yq5E=NwMVp-}dX%
zUuWi||Jk4CzkK)F>_;7wEc-i-Vm2Fw>fZuC=6>=0$;;s`;K{v*braL(KZ4&Gf6QV$
z!L^>v@$aPHCx5B`sQDG~C+*kYUoyY)KQH@e@q_E9#XHj{7EkTpuKA$)QuE%mn>M%4
z-d*!R=ULfX&NqG!-`}yk>v?Vc`TJ+t&RShrdH3!Om$Tw03QpWToqMYIK;QoAb3d+M
zy(n~|<iML<xqCk!nREQ+sgIX_T#7%n?|9KE`IFg)HylVm;ePSY+1gWaXPHiL9!fv)
z^o+xqhI4^ubIvR}_xYsKS;q%gpG>%Jc**9X$*Bd0_U~JKc+t7EE5X+n-HCbl<LSO<
zMtA3)-+6M|`J>mDU0Hs%;X?Yg9aqm^DZ6?8me{pDx86OCdSHKr>(<K`FTXbY7yNDe
z(faeFuel#?z489Q^rxKT28Sq%9AnP+syD8$55Eij`kvtw=S6;RDHp{z5>GipSOa)U
zMSse)D+I|2%dD5VBRx@Szof0Gr{G6n8;Sjrcf}72r3>@QG|D+jX-OwZ`-!Lu>=OPZ
zrKkKr)mUC!bSs~&;A-hyMKc)%$tMccYSZM8h@BMC6FnocL~t^fGpC>6EfHOTZG8NK
z_5#ZM9|b<}pXZ#<CeOWGAWKkDc&5xzwL%qJ@nt-FIg7dP3w#lk7gpw3$z{S_!_mcY
zkLxdc8nYO4G{-bSHo+~7pMM%MZRFU>d7Q_HPl0D4mkj3`_9&j0VkT0OLKk^<@g#Ad
z5u7abTquHDS@f*bL($_R_xaUW7cz@+F6I9wUMr<8!7N@WxRz&zpo#cCslRg5W!8!5
zh<FS6h?gi$Q$D4jrS-{Vx>2r1r=q7~xAI4oe)Uq_iH47L=Bm%txo$Ywkl8fVOyBUk
zVVKzgBYkZywFRm#G>_;#G0HZ5WoB-@$;R63x~`^Lxxyp)5=AN1G_7@7Pt<Hx)~W5%
zG0~hU|4z<TW{*g-#7r4R$?4KYa;7o?3P+We%dV0z7WNicA)+o}ChEcWgyR<{2k(3y
zQ_lTtb=(_;mW!54sY~zW&*kXlO5`f%Y~nc1D#mqRBtYh`^h%*G><p~ctb3T+|3>~=
z^3R)-lfR4O>i-Xan11H{+RvK8#QVAEOXQF0Pg+017~+0^`<d|X*YE1jt#4DmJo<P4
z&$=Jd-!<PaeSY`x>c{)u=6`+sKJV4@*K^)Kerx>9|Hbn6Q7?@jr9G{Bt@%CSfB%o{
zj~m|Uyykw#{gM0a-Z#tM{d(p5T;N5?8<7t)-s!zL@J8~3)3@|rlD`xFO#JiU4?Dy1
z|7^eS{8aoG&St^a$7jvK!K}e_g3<E7<=?bl$3FYKU;Qrdt<n3&FJWIDzhr#X{POOD
z-G}6lHlGE)ZT!CZ%aaeLU)%n!_*3)s*k_v`jepbs9sc?6Tl~-RpP644zx(jA=S9NX
zw9o3FXT81lT=0ebXZHWge~Z3jd3NfF@YB0braX>$a`eToSLdD|cwF~L{Bg#Ulg}Q#
z<bHSJ)1^;8UT=78dq4QT%!A4YeNRJPtb7poAoXqAhi@-3o}}J=e9P?l?|0u`dA}@t
zX7$MT<*bjrZ$)3KzTN$C-N%NvcF%2}O1@h0Zp9m=XA%!C-@SBK<!;*j_@|1``JNtn
za`utX1F45@4_xjwJo0+>`PsC0);~V}*!ng5llc42kDtG0fBE+L`u7b#6n>rlU-0kS
zms=lpd@lVa`zz(&_P?pWH~kI$|M0iNk2Bvae`x>fV*LDf-S2;YzcWNJvoKxxJNb9~
z&w`&P{_SL#@c$^oqF?*o&wjV`J<IF%7ltpAUS4>c_h#|q^>?1$*mm9frsvHS*Ca17
zo_=sN;Kav^H*cFf*!$@G6Nv|tuK&CoaBcUUO%E0B#a%mfS@T-*RjG4FPHLY|yvloR
z_m%gTj4mI)p@0ARL-z+uZe73Dc+>H2<z0unBKIFZn)q_g`)zMeKQntW_i68QhF23_
z7(A1I%m1zQi}Bkn&(1upe0St?;M<OuEN`sd9DjT6ea74O?{dEi{#^X;E$eaS3x6X2
z`LhXdr*eH@eaB$Kbd0@^eIt7%|4E@I0^LIV0&{u8g*gP1d5(zBlj7vwz`0NCt#pF)
zH}ODW5wRqx2#KG<+5881H3iLt(?x0}e@iV8*&&c9yjwI)DoyRVUb$|shMoL3k)0xm
zVoQbIiM*F;kPeq=lxY*YD)64)No1|`LfOYMmdbOqDs)<OXX@5!Zc!GNkCQzqKT+wr
zyth=L)CU<2sh5)bm2PXSR-P}>&i{<}vCvEj71=9tOfvf=J0(v`te2gk`9NP#$6hU1
zsZe3A!Z(G<3cB)#WOd~y%YT$m6zUVPk=ZW0OZWi$8TLK=u|mp%(}Y5WH}T7Hzv9{;
zpdfiy-a%=poRN6CU^Ra>-vwS1p?eap5<0>sgmWa@<(pI<D9x07BA~=`o}-@qBfA;9
zKFis^jNi}vlHxic@>68Du%zH3ZU@j63Eb)Y{^ABAjodpqwYXk#i3u2qEEfJLaD;0$
z>rc)a!LPzyB32?D0+u|>xw3hcggy!!;I`rI5IQWpShP{fM`o%tpG=(eamgGRedSwf
zXOt#OE*9wH_Tzrc%OKDz$S2$-kjY`l#Kv-g^9j#i?mwIi>?;4yd=>oq<X86J(|>gS
zzWeL*FY#aN|IZASe;$5L`gGx2^0(ek6`#YuJ^LK=PVLo_C%f<8f6Vf>^c~N$$lH^z
zy}zCHeBuX(Pw{VGJ@L3*b*<vI-s9O%G#~QZDY?de<=fS5H;Qfs-Q0GA<%-X_N9QB2
z^jxXG%6Gl$@{|h)FU4ODxnOpl>&mIy33uEs^PFwGXnOO}-J1K_w-YWGUGlzh<Bs@!
zmwRrv&fPe9L*x3-8z=7`d^G9hoj00qCcd}*B=c$NkM_U4-vd5te&hYc$K=Dao?n4a
zn%`MiTKu9!xYP~V+e+1{=?YWCr9=%R?n%y&SRuMZ<ebC<xkE}<l#eQ}S5;Ems#v0U
zO=FWzl6I)hcO4Z?PnCSdZHj3s$5q8ucPq=w`$%|-Z<pa#3{yI-z%1n_%p|NJ@=ow6
z-)^2MJYIa?`Kkrv#m>n5mQR+QE2b;dE3`>CT6nIY0)GjQB*#M*WsU}3ZoY}!$JyU7
zPXD|6kHR0rU!mXbzx(&>+hfzGlV5)N^zu&vL*M^d|J45Y{aF3=(wDw3sh@Q}z5F=k
z)3t9_|MHn;Glu-#_-)Upf1e6Julo}DE#cejAE$rc_*L*z`S+E7=FD@L7yqyObLEHq
zmmQxxKkfNA>y!1Tc^_53@P23i@$mb(@5VoVeKr0#<z3%9zK>B~6Tja0`0%65SGDh)
zU({Y7e>~w4>yznE>z>_w5&1UxP5ra}r`FF7J>K!u`gO^h&#zv*2zsXY!u$QRFFs#q
zeqQ{k>GSXJPX8<z%Nd&fUHZNLhwj(MpL@UW|JnAH`P1W%f8QT^C;v(6>&j1hZ|^;4
zd(?Vw-9y$V;rE~4>3Q(?(aT4>A6&k(=+4V~0{6=vOn?01$%?1{pDlc$|1$ZR{F9_-
z4zH%ZoBt{C3-`C9-|l|7{YCU^&nN!Rx?g6zZ-0IMRn<$$SAK80UY&hf^GNFv#}muv
zjc;;3-v6}m^ZCzyA1gl!d_DX1*XO+-6W)cr4tV+c<<a+dzD)ik{%+<Q&)2Fi6JFN7
zG5F~A-Tc?e&si^ho;`ix`^Nl(^QXC=0=~TXa`p43&-XqYcvbm)+Vi>>fv?Qota>Z-
z?%(Uhuae&^dDHbO`L*PG-H)*!gFhYrlKQ>yhrsuYFMB?2eaG{b{e#EX${!bhM*S4{
zZusr_x3+JSzkT}Q_dDhH*Pk1IrvIM&$Njg&_gx<kfB5p1^Y^;nzCR<s#e9|e@%e8E
za}CR4mR63%+<M&Sxj*p8@$~S#;$h`F!S<BpKMOP01HoTnj^g`;Kk<9>x^RlJ6>_!<
zOc&bDvy$Z{gD9gHqXdK0zkk2?{?PwY^6ABAzb`Al=Kt*aY4Kg>m(%}hhFZoNmN1s-
z|F8UV{n7mW{`ao$G2b(O8vk1S<J+%g3{ort?3Xz2a~QGxWWLC_pXn)^JXa+5NA7ms
zuL2W=D}+x8&g6g2_k+J$;D(Tmgqu{NM1$mM8D|+0Nq=!Ui96EA6~vXBluK3rs(;cb
z(%7JCs>HAOQsIE&XSFijBL*9dUK!gPuQJ+Z)NXX&_@UWBi(U(TivUYys}l1|hT1wJ
zno`;y4MI!}&3i5Ft&dplw@|a*WYcaPY@=nLV}IPn(^A{)oiU@~AstsuEA;}kZ)*JN
zN7PoU*()c>%oUp<sL6eibusfsW>FS(<}Svk3`~q?88-br@om+I$M0UhXZrNuv+&oM
z-#EWld|mkE?w8|Vo_`hnq4e|9kLvHMzx?{(^~UUl)wAlSS<l#BI==q-M)KXGcN!lb
zeVqBu;qB#jBJVD}dGw~@^~;y@UVeV@@%i+pCm&wC|M&jkhkDO$yt?*w$NP)#j=etd
za^8!>&uw2szhZow^=99TkQe!{9A6!M74-J@Tkp4L-!6D3_o4P<-p8#U13vBkIN_tl
z=Z#-petG(J{}0_?>Ob>;_WqvscmMx7hTZ>9GFq~+awM~-vPW?6ai#HO^FHKG=KR6l
z!lBN6jPHivL6J}5FC~K|w~Fo*Xy%g^&=CGCrZ1Hz9Vg4L_)F=i;(xit(gIR~GVA5`
z%Po^#Ec;z9PjQA)guK6WwX}rnVcB`|7Zm#C%B7b{CQERO>x#LE$4G9Itdq)=Tdwp%
z#a;EHl7^hS)LKb4Swlr{Wq;KV>RcMVDiL!0V&?qry!&}%I3gIQ|2_O?`=5z_?*4lI
zJ?Y!tZ%=+K`)k9L#;nc6!0`P~$*;XX*?;Z-W%Bnk!zAW5wp-li_`Y)+uqQHw{5|q(
z({K5IF$@P7p8oe?c*M}~f7zeeKVN^B{4Vgd`pdi@8h=0hE&r4BQ}@f=53FC#|2pv3
z_jmC3q|e*m7rk5b{`LDCZ*$)myz~F?;KRX>i67s*m3sa1WzzG<kIy_PydU(i`f<vW
z!_WL*tbLySQuIy!>!&a8zL@=D@$0kiO+Ib>GX2}6Z|A;;|Kj{r`1Sw$6K{Fm+P!-C
z{QdLE&u=~Rd>;RT=}qCMKi{T(zyI~mN6|O0pYM9k^33^O+VyYO1#b)94!*&0b;%X4
z>;JCJzF6@3(Xro$j~vQ9$g)pl_uoC@hi)Ifdcbmz?vB~p|LxkfchSDL`!x<|?3dWD
zvM+ge_|Ehld^_jv7Tqhk&+5>*qkV@L?fbslZ1<yGt9Jj{Be=(ISH~XngT@Ch@4mbB
z-Nw@!3N~Hb`hBPKzNEuPPxzdCe>mX4rvv>5&hHc4FLA)_z={LqhuDw#oLqL2`J~Tj
znF~9wJh{I7rq`{dH`iU;b#d3Z{0k|U*Il`EecFBZmn9$hzKMU|_x;fihaa-vxBNKp
zTj!rV<5HGXwvDV0m~)u&nPiyXGSxHxW0hntVq474&$WsBKCh<01VLGmUE=9dPBQOh
zPD{_0nlEW55hUIzdQwPD<fqg=#c!%<8U-4=)I&6GtG!gbDz`|UO;J_JQn^jJN#(Cv
zoo2Xpl6Jjjnx>6Tquy=(a3fcfQ^w)OC8ovZO{U)rn+$67UG*&W(+#5aIJ9o4u2V8r
zI4R2~vqDNs0(9-mM6t~hJQ5Rx5Asjqo6XC^`-=B4|2v@tqER9r`LnpEaWrzYa_Vp@
za9m~o#<7WKp-`&$YjFk9IN`k_2gSRjrb%;1Uy?c_6)Ytuttb;JEhx<_BOtp#VTG!l
zy1!bu>NBMfc^w%oNmFqq@mu2ElHSs(GBack%hkz!m39|*6W%3sML1ieMRcyjKAGFH
zyQIEJxJYl4-7C+kJXzITO-0R7%~ySiy0ogQl9W=GYKhiNT}Iv0S|2qos+Xw6D?2GX
zk?&XH)|jJvME|&9r0IN96=Od`E4@Ejr?swY`D;#8*HV3<l%d!x*CD-KYO$1;bfK)M
z+*0Xyi40K_p#yyLxLLT4aEA+g6nZ7##&?)6R`8H;j4-djOWvh?+XWOwLd3JhlSGdT
zec*e`Rn0MxeHXhb2N#De`zcmgRzCJBE^D6OT(>x4IX!qx_^SDG_>y?TxS6@ma%J<n
z@}K7o<0@izVf)5r$)3yn{cr2<s-IzB7JM-Mxai}T_uB8~yx#Vr=S9$~Nw2D3tb20g
zfyDjt`#0|Q-&ef9`M$&>qbExqpM9kGl;@@5>yxjhzm$4)<;|*hq3_<kNq*t+aK*ih
zhmRg{JX-y5+r!I`9zEIeMDp>i$Mc@`yj=1|@cqyC0w0dOk9lYRTIjja6S=2ZFZaFG
z{viBm-51yIUw%~l{P@G}htiKTKSF=m|H=4!{*Tu$&F>q(xP7|!Uj4o8hk%dEKOXp~
z^I7gY%kNvig1>+KGUMyj?<~J|{Yw1#{CoQMBVWxv7k!%iG3H(48})Y^-!Fdu`~C6{
z(jU~`K6=jlSofjsqv|Kw&m5jjdYp8>>2}r4c{fV0AHKHby1~s|x6JRB-CujZ;J)gE
z$@jnCvAQjE`}-}S+j+Ow-O{?X{Z7@xmrv$BwRzg}RP4FgOX*jCUIe~qdHM0>uNTi>
z9)G*#(}nMXf7<>U{L%T<^kdPtU0<X=U;j|?PW}Dn&ow{Ve$D^2__y}|%S@Y??)<y+
z>+X-qKbU^<{q+0M`-A;=@ZapeE`RENFZ<KV(9dGe_JhrY<0j`{?gHLBJgPj}yf^uR
z`6GC%I4jw{vrc8RWiw<^WID}sj+KWakK+OR9`<-nC+<j229{_4`~Lm@d-IR`ubS^G
zzR&)d@pJ0;<zEv&-G3+hcHygi&wQU+Jlp>?`0@SwXKz2dap$_(_3mpv*E+9SUJ<)I
z<Kpr2`_3_*n}4S6^ySknXIsygUXZw8bKdCOg)_%buROi-%<ps8FUDNHajECxfeT6(
zj-H!wCgN1;N&YjTmp)#febf8q_L~Q8-MejY_sQLxcURuCc~JL2@cxmz`|nELw|bcO
z=-9)YhuTkkUlzP|dUyHttQQlWXT5m+T=-ehGs{<@?`M2w`|9$A=j*HQ%YW|vx$URG
zuT8(p{+;`;{@>=`#lKhoon_2msb$q>zsxz2JCY}yXB}4nM;7}t_8aW?S-&vZFsZYq
zaA<O*v2I~*W%<auj`a!4LAGY@Vu1|dIl|RKuY^8`?vWIe`5}EvN=e#HMnUF)RF>ol
z2|dYQQe3jD<VxiC$SWvbP=2PGq4r-jMRkcvsj{+?n&Ke^PNf-YZ*=SoHyU*r-!s{5
z8e{5evd!4uWWLE`<Bvv@j6NB4n!GR-HF4Lk)tal;s60baM1fr{O-55@zf7>~4q00{
zF8Mq1Qxs+>oRC`~Ehc$dTt@tkXoRq=;5I=EQAtTHX?tl6>66k^q?bt^5ziDamk5!1
zD$OVpBfU`itc<Jdbm`;b$s&)0{Y2-8d5bR;(-W}(AD%x^&`;nf{|SCy{u?}}IL@$o
zvO2KUu|={7Ffub-V60;P#<Z7V^S@93>=~Pwo-w8~cXGZLxGj<*@>I}A;Do>&K`w#Q
z+^^Vu*vmNgaEtTK<~t*>P$)}SQRJAYiMW)wjF_}&k8qLTDgGk<j{@DoM@1$GpAvj5
zP$zJoUy|=4S0%dv>l3C;|Lgwr{yP7&{Kxw*f*)SImV7z=IotE?Pa7ZQ-)Fqv`Jm$A
zs|Qo>Yv1F)`|Qr|yAk(|A6PtG`$+2XhX<SQDBe7H<JT>LyCQdg-dc3y@(tJ9Wp|2i
zKfUF8=fb^H54JpbcR%u>)01ycnVwC5lJY3!QP|VP&rO~yK8<|T@ZiD&uZIB-((hlo
z=XZC_ZMoZbZauwq@b<bpnRn*gs=jgR#`W9P_fFs2dspP{;rp+i>Ah3=<otEw&$hq(
z3`ZF5{cm9CW7)=`#u>@Bo;j1Hhhqf~JKqtW$y}c}KXcdcRq%h{{~%B&_)DNoKwfaU
z5VNSa#0_a<xm|MqWQ}B}%Sy=i$-kEMlsPBkBX6(dq3ox)Ud~@uQ$}4nU%Fl9mu!ao
z3pp#9caq_fx>9Y@AEZPjCWtkN&yui^JSVwXGE&k)GC{&vJWu?EgtAnG)HR7XQAWW!
z-Z?xkd0qL;czQS&vcF)fVt>JA&;0A(+CN^ukNjNz!}Z6FAIE>{{WSZw^ke#4hc}k*
zmcC#7&hX8g7q^~#e3bcU^F!B%G7keE^xvO$|M2~q`}^+c-+OrP*?o}*n;sr`R`ka3
zedW7|Hy>Wld*}ag`NvBiIlmNtfATZ*SKzPSKMNR+GwopL{C)e|hOgY;mwcD__U+TW
z_ucO_KOFxU{Aur}s!uyUOnc}3Ht=oxJI4=dpB{gn{)O%9yKfCYn|`PKJ@Gf<_u(H#
zKLmdy{^0nP`J3$z$KTMu0e>?7g#DYyFq7#!vlvSp3oqLWjtg9wT>R|ISkAIeV^3xG
zW0PUM#vI3dp1F<1mF+dB72hWT7Qs1uTwEqB-v49&xcub*a{ukrm)l-Gdu#gX)MutI
zZC~`hMt!aM^6OLmr@Bu!J}iHm_sac+$jeo4(%#R1zxLhUH<GV1U+6wx^(5xu{kzS#
ze%|1^<#osZp5J|g`%CYs-Cc5v<(A2v#djy(xpnL1&4k-W?=w7p_Avgw`@Knbt?#7W
zdVHhc=H1&%?=;-}ecA2&zO(M<3om}Z7<1v=*|ley&%QZZa;ESU<H^JmaVP7~+`2I3
z%7JU1H<sO8d~?%{r#C!ry}JGOuE_mApbJ{={dn-^(W=K2A3Hwgc-;QT<`Ku^2~U1M
zZGBex)bGiUr<-3{eenKr=F7#83*XIsoAP$aoBOZX-^_R$`9b0HyDzK1+I`Fae)&h=
z&rd&R|N8Zt>%TekYSz=NUszYMYjCFtM2Wr^7n9VL+9I`B>YP-i^ir9A*^M&1Qla8L
zVlTuMrQ2j2q>9A&L>NUTh)Rgpi!T(PC$U&^qvUDHLTL#(Z3PL%SqkcM5z_ai_+@OR
z&BeWhr1{tK84K77F6J-ie#Mr~tjMtc&(B|Xf3y5M@?U|0@Bi<AVNC1T>$tnQqdB@*
zmD!eZC~@U-DRbFz_Hyju_{jNy$3-AqaFKwUfEoXJ9!u^(Zhjtq-dx^8JX3jD1dj{v
z7oI5OB{WTVj_7mIC&F1m1;SdQ`66xtySQg?cL^|wT^BzrsvuO$zm@MX?@8`YoOd|&
zxKHrD7pxWImzpV~BUdQrB6~ziOhQQFkhGIRs#2wrhthVXZsiCSY1PH5wJI8l8)W}V
zpOdbTNtG>?oh!p4BPFv~MpG_S;jY2~Sr&=sLJk7w_}F-Lx%Y9t=XlG}$FYU2gGGe-
zA4Be6`=23S9Y25lGXKZEpWnV+{bcna>_f$;bzkDYDtzJlB=T|V$CIC)eZKb9{D;fW
zqu;~7RDC@7(e%sFFJT|u-bB3o`Ev1_b?+@dH+&cVUHm8D&%8fO|K9#v{`bo-_MhE9
zKK*e2)%eHbpUuAoe*}M<|Ni%<`Jei~AHSde;QW^R#o~+HSO4$lerf+Z@z3?Q`Zv2T
zYrfh2eENg)`|Gcc-z<N;`8D-V=-*}kzWw)Oc*oGj#Kc^}`2PRa{~iCA{WJcz_Ww7g
z&8*d|!pzecLzxm-ma#c=IB-m6OJwn5KEZN;V<ztk!5U!}p_P2UdEEGx3-|~n2^|tR
z%E!!C#+S_>EKnoxivK0QvLK(Zil~_Qeu<xwe<W2T&Wf6gdWs*Aa+4L7Rge~v+9Rba
z!!LJ2{*QdB{APt<mFMbbHIHeoSAU?KEzc`0AW<&iBb_AUAt@rFCUAo<hrdlgM)0h_
z5B?1Pe*VP*eL}Z|Ckb8U^W`dK)nxSg|LFg1hUpBo|0;hQ{EGkm?a%hVd;Z?~WBlj!
zpP0Yaf2@8h{$BY@@Mrut<1ZoKKK%(~`o_e>sQ5qs-?sm^8JU=cndO;g{-5$E?|0Il
ze}AfftNsl5G5<%zk89uKex&|%`nms`<0s2^^WRK=v+2!=H?QAZcysGb!`nyi-h2r9
zEdBNKmp30Ly!Cl)^k(f_wYMR!+MjKDto(S!ljqNVzgYQl?~BgocF%V_U-KgTRpFbb
zZ)d+Pc`f?#(TkYZ%I}uG`~F7aHRJ2YZ~nb|^M22Jk9SL6Gr#hBZT7DJgX+g~@BCg?
zKKg$9_O<pao>x9!3Ow(1y6lAJamC|u$C;0PJiPj#$AK9Kcn@VCzH@lzA<qNX_Ui0m
z-;=QK!@-)vUk)za$GKZ!XV=ctJ8x~Dw#9KH?|Pf{hd0dFD7k6(Ch1M<)=ycpZdK(P
z{!M?k9o){d?d+CcTa35;-sZclaO<Y6#@ps?Q`xa>m-3!HyDN5u?Nr~Pw*C5c&fV7g
zIS#oVQaqS;aMh8WCj-t+zxe%f@wJAVd3QG7oAY48qwpsZPt~8;Kc4qA{PnUATR!dn
zX#1Z3ZT1`gx7zRhKhFNd^!eXsk8h?wGyh03$g`|ux8m&Q*vZb%Y08tqSIR5S)yJ;H
zA<CV`w@+ZA;0u8s{>ywk0>Z+4VshdOL=}aq_yTwr^P7t7l@O4wmO3VJL}HfoP5G}%
zfyz4-n-$J0FetGpA5kt<kyLe2y{zJ+vQ8yRjaTEh`g>I`<yOU0iqb0nYWGy9DfP+8
z%B0I2kUOC`NrhKUO;u0nfZR!$64|NpQVR95QIfJ^(W3og8sc@Le}yasIR$L^=kVJI
z-Vjm|b`%m2yvhHQZyA4(&<xQ>;?WW(#PcN@q|@Yum8K|pD3&UGRt!^NRC}g=PxFN4
zDYbsZ5E)(RhjI&*)RfOCN+<})XDfIq|5OoCl~*~X=qNu^ZmWEp;sYfm<rKvZIdhqG
znJ@)YHA5{Utu*zmDqJd2Dh<j3iZ|qfWWGs0lu(yamkU>}Rby6rq%5awrTSY_RbRr$
z+GvJBfS!i#U%h;z%f>eh=jm<HvDcli&tRl(%xL`GV404JMwi+V4Ncue2CEIO>I!H#
zXeH=u){iubH@dIypz}=Qv6`S-t?F%+c4b8cN$GWBh9a+pr-`r%tMSEf_ON?#$a7ud
zI?KVw=ECaF`ksZAO`fxmTY@`-V<w9uWAFdA|3?}4nPxJ!|5yHZ`=0<~5VH^qBh$ve
zoj*IiH-9ht@$Kh>U$#GIeR=xs%Bu^nYTn)cnELU-Th3RLpQk^sfAQzJ*wa@Jm)sAz
z$9G@n(UPZAo=<pg{#5AUrn_733O=-ds`1?K>HLSA?`J)j{!sSerhDPH58RNw(R-8k
z?w5z+&%&PfK4p6(bN~GPf+x-|g<reA`u-yQh1g5s*IV8;zu)_Q^Lv$#9A6y1&-wY|
z=j0!Yz8QVF{xRs&{%@&&WEdtgME#5W^YPE}f93xL7@jgHGTJi6Gbyls<h;qhM|hfW
zp+Fu_G-ncr2xm6uTy_uU^?$v7^8Wbx^TnU(|K9yG|6lN5`hVE}hYTuA<&4Vz)Bbe+
z>ios==jGqX|8xJV{rmcR-LL(>B7cAWmH3_Eqsg16uQT6Hd3*QG?$@VZ?0J0XuJFxI
zHx}KU`q1b>=IsgBUtBwI^VHoF_q*>OyVrL&|4z)^BM+88dGIXYS=HmR2g&!7AO3w>
z^y=g5Wv|3u=Dhs#`uMw}@AKc=zyJL%@7>LJ*&pwHmHwsn+wjM)&s#s1eUksm^?lyA
z1z+X9Ie!1~ed5pFKl%T<|6cn2;rGISJj?~`9h{pu^x3{Mo%{dpUo>MU%XHQt7D1*G
zh6@a*nNG26Vm-vt#aQxJ=QqP2f&Vc~KbdowBN<QsyYuJpp9lZ`G9F|;!o<vQ>hIpa
z3;wGyYA_uAJMmBcAMHPVe~kY<{BOh1@$cHNxbMy1)_t%0(fD2eOXz!z*B@S}y?Fkd
z?YYhqmj_k%{2#DBX?V`@YWl0L=eiH$Z$(~Pe$Dh&>|O7>e{SX66ucF1r}>`G{d4z@
z-hF%f`OOVCSKscuYjbzjt-R}j*Hmx#-Zr~u@PO}u-`(DuYS#^~bKXq4<$P2B+WgBc
zm#<&VzIp$S`u*7ZTz5a+D8738is<#@H&t%8-2Q%B?XLNKxkp!?6g-W5eC$E!gLMz>
zAOCss<Jp`S?5~txC%%q(qxNCSms?+-f0qB~_CD}^#RrCuKR+mZ^!gz1ZqXZ)cdDN!
z{#^Oj`ETB@q#q?eY=4I{Ol1AV;m`G*^FK#0XD#;*UQ@mtUJ2gmyy<*V0_q}f#d9Uk
zO8AINi@g>*E;(DqL5^E~o!os{Gx-_Hi!=(gH)!!_+*eQ6=`!jv%eG*#Twxw>a>u|`
zKi;t4Y>QQ(RkGO>qY6VV<3lD=CQA+5^uzQ*^o$Kmjenc)m{b|;(r(lU*YMCfq!X{V
zOIKZcwZ?q)9qOOdKdMboF;H5ka96%n-cf#v+#=a0(!azNghd4Xgww@~q;#b}OGZna
z6K|GyD0xoGT1radtSG0bgV-sFhmoVl$x4hKCp&ta?C5c_VXsDylN~)y_6P6iak8Vw
z$x0ABPS)kW#?L?JTSkwQeKmTV?C5c_qsPgP9w$3`ob2dvvZKez>W>~LJ9?Zfy^fRp
zHF})v=y9^6$H|T!Cp&ta?C5c_qsPgP9w$3`oGiVLlb!kW=abRnWJiya9X(EV(&%xr
zqsPgP9w$3`ob2dvvMi&=$&MZ;J9?b#=y9^6$H|T!Cp&ta?C5c_qsPgP9w$3`ob2dv
zvZKezjvgmFdYr86=y9?SMvs%dIC`Ay=y9^6$I0H<HhP@w=y9^6$H|T!Cp&ta?C5c_
zqsPgP9w$3`oGdMmlRZ6poGh)6lVunMqaiRF0;3@?8UmvsFd71*Aut*OLns6QxAgV+

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/58eab6bc2386e2ef43fe4f55cb6ad3611399d5de b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/58eab6bc2386e2ef43fe4f55cb6ad3611399d5de
new file mode 100644
index 0000000000000000000000000000000000000000..eb38d110153b75463f4af9c56bf11d5c6d425685
GIT binary patch
literal 22
dcmWIYbaQJ+V`K<-40BD(Em06)U|?Wm002W^1iAnK

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/63448c6a9feb8c72b3e82af4d735ec2e62ddd328 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/63448c6a9feb8c72b3e82af4d735ec2e62ddd328
new file mode 100644
index 0000000000000000000000000000000000000000..6c534ab19cf330994c377b4e158ed98492880589
GIT binary patch
literal 58203
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#P;0`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O+Vxj#;S
za{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W%MH($Jqvg`;W7W?d5;;N
z#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQe5UfE=4JaU&)3^uuX}Cv
zI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()m0t^gul~dSck3VbKQezT
z{&fBk{VV$K{QujGj?5>SWm%T6__A(f^<>pzdCQc*^n>XOvj@v}mN@ovoR(Z=oYy(R
zI9xc6aFlZ>vn#SqU_H)az#_o%ky(~y2Fqj?3l=?=f6PtH7R<MqSeVKf4>J}ron$Ix
zddPT`v4l~WVc)-l|IGhC{Kx#y>Tk>+o!?zQtG-|R8uvBfYt&bcZvo#+esukm|Ml>P
z+qcJ`cYXAEAMm#DjqRI;H;doSeSiMr{m->uwZCot+WDpL^X$+0U)a9B|FZ1MtS<|`
zn0&eUdDoZxZyUcK`4RqW<8SjnHGdxdk^1}iPud@gKVSaD{#*I~2g5_ALoCNx)!C}p
zPO*JqTgMj3%D^nf=*>{^-{T+uzdiq+{G0J_{XdugeheLqY)svZcNm-)a{t%<Tle?w
zpWnZ;e*6D!`+fKKyWdxScmGcM75n4FH}!98zE1e!_36qx?$<M(uYEG-(VmA>9_BtY
ze#rB%;X%NC`FqZHE$-a7_2A~an>@EK-C1;x;Q{}{#7Di4XFQg8Jmc}4Cug4YKXH1x
z_nE*8o0opCj=%c+s{Hl;*L`p7-);S%`pNdw<&P2{g+4gGZ+(~gw&Bf=Hz(ivyf6JI
z^u_R7?stnH$A3KdarlSzPxoJXzwiIv@~7l4-#@qiM;NEDRIn{#Kh4q1S;LvmnZn7>
zd6VM}#}STm9G^Kuxc75w@bvJ?@W0_-E3j5@nNWf7eqlA?JfRk$>q2ruwgQEGvw6*V
zxANTLN#`>W*e<{=AjN-#cLjF?X9R~UM=M7u$90ZnT$;Rl`KI#U5_l)bB&00lEj&v^
zQPf#9MKo4aO7x=02a!HePqDLN9^!|@jU@UcR!a0r%#tvbh!^`Kyiq7pNK!~bC_!k4
zP?Yd#;oCx{f-M5x0)qTYcy)Qd^9b|o<Feqq$S%YFl5HQGDtk7^Mot5+O`M4w``J`k
z7cr+YWioa${QNJ@Foi*!@h{^yCU53x%nMo4Syfn>SSPXGVx7e5$g05F$9j^jlEZ@Q
zI=3kACEiHBYkV5~F8oUTANelw-Q@G(za}6dyjG+^bg$SAaZ~XI(d8mDM50BQMQ#h9
z5&kJGE>bDHTxhFMp|HM)hG>@<t3;7xjZ~DBwd8B@0<je$^1|6dIzr+?YlNyrZit<h
zV30Z~^+n1=>ZpW<c(16k$U~uW!N>dy_*U@F<C(yHhpU`Ri)#(%L5>FY9JW2I?5x+A
zotT6e*D)+)NM|r%&}GnOFko<JFksMR5Mb<Ka%TR-WXGh!82jJ%Z|v`jKLvl@{ju@K
zwjV)1kN@=isr_Tix3VvdANRjo{^r&z?w2#3e|z@jnc(x^&sIK@f7bDI*ONt$86USk
zj(bw_#NtWSlbcUIKGA*p^y&R)ub+Q>apPs<tGBOQUU$8&e69Uj>2=ua_pg_{75{MO
zqu^($uVvqF{y6bd?04Cpi+?Tu^E31_Y+}5?^qEPKxr4=yEs9-+V=)I0XB+2wPBBhl
z4mS3$tahxwSnjhtVF_W?Wy@qg&M}4aJLe)U9qx-<He7LBXSl9%#dFnj&f>7<kl^^w
z-oSB`Q;_=`w-e6-Zf~wf97XK6SzTF{GPy9GXL!oc!*KV%=l`C6lm0II6a0JePtKni
zKkxjI`9Aq;+ZVUbS3kObIPuQ^o#^}D@7X>wezyG@_bvCE?6>r<Pd{(?81X*h?ef<J
zuL@pGczyWI&$o---~F)R!`JsS-dDcwe1GJ9+4~FcF1`zSzwN!#hwzWrKh6JQ@{Q+v
z-}l-d^*^nDef@Rd_k=&||DO4`{J$?l2E&p6>;ErixXob5u$Cc^Nt}5<Qvp*U(@LiE
zOfpQej3$g4Oy`*Vnb}ziS*Ef0vpi?YW3c{r>d)ri4}Nw3di~4t&;7s8|5^VR|8Mbc
z_aCX>Hb3pZ@A*3Ki`Hkek4*3Hyb*c5;HArpkmnZ9Q=e--?|ruA>9i+-kC`8Bd{BNr
z;9lq5hj(r6+dNqR;O)bgkK>=}JY#(R?)jz{A6{6zP<g)p*@kC2&-Oii^R)C?<8y_V
zvtO~iad>zAL+vM{&uyQxKJ$Hk^r`#Pua8eY&i%CRbJv&qUqrrs_{#m=^!u)FhrjuJ
zZ}`sko#&hCSBtL>-zvXf|8eA()Zg&`B8)E?9hn|7U0|NYBFdV=`i(V{J&yAOmov{6
z-e-Id`7a4@3N9DO5)2nE7YP(m7D*DBCZZzJEwoVJHvd`vDE{|+TKsGHvji9fKk*mx
zPT@YpwV87vM=6ICXAf5|k20SWf0DotL22O%VQb<0LRP{bg<lB!2r~!^2qz1xiJTWP
z73CLW6@M!xAvRT1S2SN_zA%sQGNCXbRv`|dBB2JM--7%?vxQ`Zt_XMu1PDCg|H;?H
zJCi$@YaOQ_hcBBNt32y2*5zzA?1$JhIsLf#xxa8&vCFaTVKHQW$jHYS#2C(`!+e%`
z8cP_fBU>>0Y<4bo2KEQ+^Ei%j%;8AmxWU24#m?Qs-Nf^a*Nk6KV4}ccfs+Cbg5`oj
zf|&x$0#yQ=1ZN066G{{I6>%075t}6@EWS^|L%K$WL*|0Cn#?Qdzfwmf-$?XJ`bwL~
zILQdeI?08|RmvL3lt?K^Zj{iIh!&R??+}-lI4z+i$tbBLc~(+DYL29c<V*=ii4WqJ
z#O8}M3aJX(3asHz;J?Kez{kQT!7IZp!?~S3kBx!#K65&A53?;x2x~gqN;YdYeKtQf
zHrDq{HcWe%ikP$+xBYkc_x;b#-xGg+{<iq*vagw6cYQJVa^v&;&+k8<`jq>j@}0+9
z_BRV(RlnT#;>wGLmz!Q1zuf&I@WuM)`<^{~V(>`if!BTCdwcIR-!{0t_V%_r^Y41x
zn|OcI!`jELPgXyPecJx)`E#C^4`1$l_2KpPx4Iu5eSG<;=gaqRaz9V}s`}IVPnvNb
z^E;L;EN@uOv9hqWu|~1JWvyVZ=d9w&;9}-B=TYN1$feDBjC~0k8{1;mC9M0|mU3ip
z9pIkD^OI)-?-#x+{N4iLf?I_0ggb?mgiVFY1*-&S3swsD2{H*)3QZKcBUmG#&i|D+
zgQt>f0f!vB5L-Q~IqN4DQ<h~+&l&9hAOCaer{fRHA1l6#emnX__{*x#7N0dgo%t~L
zL(<2}kLNy=eOUZK=wtB5zz+uRcD*rp<Nw<1)ytR7ulB#Tef#F^rng((YQ3v|clDj+
zd&T$P-=)3Z`QG~dx_3X`$$q%=arx)4ub$uIey;iT_}AQDe!nb#MgMa8b?4{lpSyl7
z{&Vo(&i}jq*D(Y#6)-PjZf9A*Cc*iP%b9x?S2WjRE*`G695dPV*%!0_;kd_noHK%J
z8}~ZyGn|vzr?ReQ7H68xVD|sWzuf=q3``8Q|2O{&{@4Fc^xw2U(|-5<=KXE=bMChj
zpNl>odmr%b;hUeY7rjn={q+^+tCKJOK0o%n`uWLc-=D@jO?-Oi>BXmBPfeaqcsk{&
z!qc)R_D}kr?s|Ul#rhY|UQB=a?IrW8?=OR2slK`V&h~@fhyNeueB%7F_KU$6$<NH6
z1-}%1IsbY6=c8XLzrFp=`s3fXx!;VxyL|8d#`f*!SDA0>-{yW@_oeWQ_1BJXem^Gu
zdhy5S|6E3S=EKbHEH7B{SpTvvW%FhC=D5Rohx;P06n`JTf&i1?D}i<Vvi$e?KMAx7
zMhdnGunTYsWbiBU_3{4ay}`GM|2}^@|9<{+0{H^Te0zC5bHC(T#Ceco5yvsk6Wk8G
zuX$JT_3{@8OcLx7QWaJasTO@MsxN9K;xD{P=&+EvaH{Ys;S>=TQEt&x5hvjwAxR-I
zp*ezA1^fgi2;>MJ78DV*6}Z5!$bX$Tf~THail>mblb465munVhA;&8=JJ!F<o-A8g
zx!A;56<BN7OxdTgd9kvx%x6|-@n`wYyqKkzt%Ktz=L1eV&Ir!kT<5s2aev}o&Qr+y
znfD>D5^pEZZJt@Y@_cc8WqgbH?(s$OJ>hBRR^+zj&g7oQ&BpVBC!bGDz+A9butD&T
z;BTP<;acINBBA2(k}6VOQmIn=r8K3FO1+fSm0BfzPexakQ}&?D6B!TLCYd=>6C}4v
z_DQl!>Pj+5NlR<Wq{(cT_LH71eM@?Sbh)&fw5;?@sb3NvVzI)P1vc{O@T&7vbH{SG
za3AD)z;TRCjP(HXVx~^UNQQ*}Z~y82zxkhmA@={De@Xw_|3CWY_ix3&%m1G-Y+_LO
zKmG5vKMQ}q`RV$@^LzC77vGD%-})-}we73!SIf^#AIskty<7Em{+rO(lV4@N+WPYK
z^W>-79<6xLdq4l)-aA3JH{H^_rFe7W^`L7vukOEAcEk0S#vQ$T&JR={)jZaH^5_Zc
zv$W?oUR1yGf7A9Z`(w$Mjo-d~zwzVr&kMi0ep~$c|Mv&O1m-I&vskyWy<*?Pah>BZ
z#~#i-+_Jo^ytO<fJiB-%@HlZ>asFm!<%r@~!m*OGo_iv18o!@_j(~?iuHYV_GT}eM
z_e8#m-V{40_CQob^s9)FD6go2=uXk;VoBne;<LnhL??)t3TF$><2U1r=GElQ<SpcF
z<XOk1$!W`Rfvu0_6q5tfW5zZHfq&M&@BWbauJQHeC+&|*KJ5Fj=Y#Ht@b^yd89wlR
zc=zt!+jVb0yv==Q@!sLXrjL_9^L?H6b@kVWUt7Oz`2Oq1p`T^H!hbLNed1Tc&-5SD
zzB_(z{r=>;_>ZI?KYvX9`RwQJpMgIee_s2!``3iu4S!_*sWG%LE@#wWJj5`UVHU$$
z21&+Nrd2HZZ1rp_*;Lts+5fT4WNTpC&Bnl9#U{<l%(92MoVkGM5CiA`i+_Fovj4sG
z$NI0!KZ*b1|IPj#{IltI+waxCbAL1aUh#|P*Xtj_-$T9zfBx`M=A+Mt*!N}cCckZc
z!~0s~<*R2rPdgqPK9YF2;K7XtMGqw(wLXe}^!y>;qd$+fKlXaU_e|yGnb$&ZbKYjW
z&3HTIZSvcFZ}z?U{Wj&j+=ti?TR-%CT>0twr+FU*K9s!=elPdl`@Q%FrH_oCls>t9
zWd7jr-t+y__ftMFer)&{_UYlLHJ@reNqn~aQvLPTx7r_Rzsmo(|4aSP!!VOUoN+p1
z3{wPi9?L$~PWCX)M6T&v^SEwvMRLpXNb`pBDGGQC9S~*^5fE7|VlG-N`d;*&*gf&-
z5_=@VB@`s0B_t%KiY1A}3LOyG!r#a*ETAA5CS)uuCoCiMULaXuqd<z_OTj`Re_>mZ
z^&)K|3L?LSGes;!---5!nTp>KSCP0O{#b0bXurrF;b%gLLd%7MgpUZ{5pEU^5}G04
z&7Z(`ori_nhO31um`j#(H~V=uOSVm{Y^((=D_F`|PqR9+u3|}N$zyS4v1ATmeEa{x
z|9XagMmy#L)=KtE9MYU&97gQrY|GdVvcKVQ;uPR~%TdNr$Ucd!n(YtUclI#OGhCKD
zUAzH&nS4%sLVO!}9e8DUWqFx+C3r9Jp5}|-FXms(&m~|bkS!o6Xd$#$SW2`|^twow
zh@|Kl(R8tN@%<79BqgNIOIAv<NqI=WmtH2FCS5M=Ed5U^TY8Sn9oa0oFS0vi9HgbC
zrb%v<6qJ&Xx+Li*sUl$^W-jtbNMC5HpsZk^KoWlzUpOxZPZrk(4r_J^HXhc0%*xEK
zm|U3;F$=Q@vbZvzVr*mh@IU<jfqx<YKQQcP6lJtyc=A8@fAGKOf6o1`{+;l<?APZX
ztUq3T-|>CXxAHF&KmK~>^w#{%hS#rOuX-c$w&~5IS6g0ecoz6{-4l-|`j7P=8r?s0
zx9yI_?b2HlZx!F(f9KoXzxNy-tbMrRQR3sACo`X}e)i&d$;<z*`rcT+<M{CP<NnVg
zU$egn{4o3Z{AbMXdw(?<<};Qt{bt(0{EkJDO^$sR$7`+vo+~_id0z5N;??I{$;-gA
zjO#yV0_P15BTio~2ks2+7hI8C+FTV}P26$3^8D-qeF8j!41!?-xA=Jk&IoK0G#82&
zIw+JPd|Y_4@Mj@*Av?jt0z3jI`8M+w^6un4&#S?^m3up9F8f^;6(;}xH~&og9rioo
z*Uj%WUyHsx|IGf`>y!2;*H14#9{P~={=vI1@A}?Fy%l=n`&#YwiPxTQ``_DsTJ(kS
zoAtLXU!Q%^`ts>>_?MSozJ3Y*(*IfaGwbKepRRnG^jY@nuWyY%`hUj#lK;K=_xs<+
zf9L&n{oV6h{?C;^Q~nnJTloJGg9T$aL-K#U|KI+#|Kt5XfngfsGR9uUS&Zu#?=!w+
z{L6TOiIv5HC6u{^sf3Axc@}dpvmuiRV+=#a|J;8$|8o92Ge|LL{OAAo`p=QyKYs21
z)%0ub&!8W?-zR+i_WAfH!B5#ASA4kep5=YOyVSQDZ%)37eEIVE(`PZyo;>Y(did##
zXZN0~zbt>b=f#)j)z6<k>v+cey!plBS2N%KdDrlc<K6Xl2j4$=Z}P$BqvfZrPXeDf
zKL&m9{lNKw?LGgygKu}gJ@vNo9sdXUPcJ_o`%>`5@w3LKEgvs`H2&20srl2FkG3Co
zeklHs`(eX}=#Pb;_IzgfYWpqsd;O1^pDw@d{VMux@JH!y&A(uV1g1z9CpJ!w*Bqxf
zj&ihf`g8r`n$E+@KTY7Az!3p+!OMb4LYBh4A{Jsj;>*Q1i~SU}6T2z4O{`z^v2cUX
z8o|#3&H^z4wSqf@WQF$$MG3hHEfCr+oGrRfj8Xit*gdfoVxD3NV!q-xC7PvPN`I4?
zDSJcql5B=-xXceJR;hH!rxJ@Kx+GF07EAn)5SI8LDj{-L$VIS;e=*-yzG(hG{NeoG
zya%}#aj)Sv<$liffNMLKH&+wqLJnJw5DqC0HuiE>C6-#|S<IJM<k)jLZ*etoS9AAr
zIdU%Ic*xPosl(OAmCq&5#mwcvb(m`+_avT`yc_rm_|Ncj3UCX|<bTPR%NNYo!gq-8
zE}sTJtH4izEWrf9TLPc>TlqNzW(qtKcpx}MxK|`mq)}v}sET;B#7)VK(o<xG<=4p9
z$v4R>D9li>QBak?Aj>DKB3mc>MfS0rmcn5LHU&?)Tp3a6cqv0E7pX^*b0i*#sfeZs
zZxRd?Xy&)%H{sXkf5qp^=gs?pyPeC2Q-ec{-HYuC>v>i#HWfBqwp2DLwuLMz%yCSu
zjLi($42F!q8NHc~FfL(u`v2Jffd4W7rvBCWd-@OKpZ;I3e_Z&^{{6()J)es{PJ18p
zF8%HIH(YN|yoq_O`SR7XiYH=^jy;(0!1Y1M{a1IT?s(o3zB%v4?i=AZg>P-Wm4938
zj_;kMI~(pe-*vsW?7rQ@(8o;APQIvro$${2W7p@8U#h-#eSP!w@wZFgSN|yd<@UGe
zzdFNIhJ}nPnc|sESvpx`*rhmsb6Rs9<C@C-hI=CSI<7sO^_-eqD%`R>F}$nzy!hYp
z-Qc~<bC~BYZvelYz#@Tz0;dEn3TzN)7kD6WQSh<Q0bzHMLXndq?xNbFDI(><1wzS!
zD+NyT=kTfUI`On~ujIPMaftOElN4j@f49FXe?tG{{z>}n_4D;N+piv9f<JqGI`OgU
z<DC!R-tTzl@pjuAjklg}7r*g(bM_7IyGQT8d_4O3%NN<NsbA_pPyV#(<Ghb;pR_*n
zeLnQ5;gjtrk5BtQm3+?ovi9ruZ}vYX{k;0?!SCijTmIDl5%_cP_od(3f9C#i{_FP7
z<A2rvBmXY`{r1P{&(q)6fAjs3`MdjH`u}bJbpGx7ck}-<hHZ=?Oy8M`nM0TtGifr(
zGyP;NWIV^9%dq1Amj9Lv8yL<qh%?mutNYXWEAPjxZ*AXZeB=K1_)F~P-5)zYOnqPf
z?&#YIZ$sX`dXx2L*6U5Le!UcU`S|&ZXWq|#Jym;l;MuI_YhGM?x#yMA>jSSkU$(#4
z^djr!kC$Sv#ow-f_u`%3yPIzX-@SUb>wWcy=8ry~BtHdw%=?i1{@}aa@6zApzdiZp
z$D2!U)89LP?E3WU)4@;2KE{2}dcW`;@B6>+^*@-tw|aNyZQ$F$H#1*Pe9iIZ#G974
zF7N8zYkpkx>D=ePU;Mvq`|kGh)2|1A*#28GHZy0k>adxyb+I|JKV{#@@rH9QcN?z^
z|6cyZ{Qvl~1;d0-i)<2g7n>?JTZ~KWqezs<F%fOi3nDrqO~R(aH-)`L4~rcWe=nXR
z&LqBEj8805OhV$kWQEjrsYg=hq_#*cl$s{>QR;&9Y#Djk6|(wr&T>~}b!0tdETs#j
zyrn9nrc0fdvXyR<_LrV4<tdpWK3#OJNPx(0VSnLB;YUKXf=~GO@Covr=dI^G!NbLq
z%w5RU#;M9F$7#T6#W{sTo}HELI%^QyJoY-yhg_4ng?SWs9&n5Eyy21MYv)f8U=oPt
z_vMe|H|0;{XAyWL@JMi;&;}tJp|yg{f{X%N_~!Dy<SF6F=6S;t!#k1pFK-K90^cg$
z6yE>5H~D7r?-Ixm@)ThbQxI<#caxBk;*lwmT_Bete?dM^;j%)Jg1thc!dLlpxtlVa
zGX65Xvh4CA3Wf@|<QB@5NwG;diB=2i3x5)NDcB_7$zR0B!FQ3jgEySlpZ68dJ?_g~
z&Ri=wmv9uYtFrB7S<TGP+{kp2(UZ}iv4&BNk(*)GKY_nTe=Gg|`|HLp(cjO0+x_YI
zz42G>uVueDevABG@=NPi?$2l6HNUO=^7~WpN4^h!?{2-8e>MGO(aWZnzAqm?uYG#^
z(b@Zl?}*>Ndh7Tt>svZEGp?sy<+_r1`TnKZm%d-RclqNL%WKWom)v-LGwt^3JBRN|
z-Cy^h^wG>GYR~Vyw0gt$Zq@s1AO3wf`629M`zNU{2H%{2y!pA|cjVuwe;fY4VdP?2
z$I8Y&lYJe#6~{FWHqK?7%eY=}#d7W8tl?DV+QjX~Ys6Q;_lEZ>&rNO@ZZYni+zPzL
zd`^5nc|Y-%^3CFB7i1E;D-<DeT+~p^Qmj&Jf>^Csm6(7Si|B3PcS6QOm4Z0}Qv92E
ztGQJ;7O@;=ocllS-<7`}e|P`U{&V?P_mAIS7k*a#wCUro4=wN4zUzCp=WWKDt*>Xi
zE`BZhI`UQi%chsCukGI!y%+j;^`qz~vrlh77JXdwanGkKp9{Vm|8nOG@7GUXb-sK2
zX#Bb7SH*9WKYD+6{Wbij^?$<uHUAd=z4>SV@1?)gf64u-{3Y?b@Av-SnZMJ1&H2gr
z>-?|g-?RUS{B!?rz_5&AD?>7)J+lF80-FGP9J>?yX||)R)+{fX?3gkb@BH`v*YtPQ
z-^73a{$2QY>aW_LPd}%AU-Q-XOYvu+&oZAby)Sty`9|RN!<Wo2+g_Bu;C{LF<&&4n
zuXeop`D*<uhF3N(7ru~tvH!W@3$d3$Fa2L`d+G7A`MKt^OHUi0);!IBHt%`Qi)AnS
zUu}JL>DAKL&)-PCyZ`R}yOZyv-aEXX|4!;%_PaIjzP+t^Bl&vXD~s1mZ#>^Hy!rB~
z@TK<)zUT5!WuAyX6?iuL>E|aWpPYHp{*?2%#S5{QF|W?Ne*NahTe0`1A7VdteLD16
z^Xr>$aX;h!u>HTx;K4MHIhkb%OFqkAW;2$xtjX+l9Pc^gxfr?sa82iG;8Nn=%#+UN
z$3KJr7k|3IEP)OI4FLv$27$){9s*POxdj>oRfPRSx<z<JtcBTyo(s$r_$F{b@Vn4E
zVMmeoBEDkD;u;dck`+>6(vxIF<pSkv6`m{pR*F?NQ(mUTrF2>Gk>W?ii;4+~TNES}
zY~&Zqn#(+rekQ#^`hv8u%r<Fx>A6w`l7<rI5|bnjh&zg%7LgI|5lj;>;IH7b=5H4G
zC-7Q8PEc7$Q>04li+G7dp@hD~W$_&GLt^z}_Tr2Z(c<?-_lU5FI0{b`k`ekXSR)iK
zyj!?f*g=?6_>kaoeha>Byy3h-yfS<{`Fi=T^2rHg3+&?4=3UIo$A6svDSxxTZ9zj}
z0g>k-{i1V4t;94Wyd=XV_KKgB(2?|!kd-KqxFmi_?478dh?NkZkiD>!NTf)*$X($=
zp%ejgzC%1oJUKj!Jny)~IUleouv)TQWB$!NgN2=yo%ILHDV7M9AI$m8=}gBMotaKD
z9c4Pnw2^5iqcel?|Neg)|2q7+{j2)dt6$H4x%}e#mHw;z*Rh{xe<=L8{C)BFmhW-j
z6~8NdkN*DdTlZI=FLOWd{uK7{&HJD4PQBB3|MA_lx3^zkfBECtna8OQ&)v7V|KtAZ
z2Vd{=-1oe{|Ne^m&+hj=`1N4v!~Kt19;-i@_B7-9ju$syvcG=&ddr(XZ!f&x@^Rwl
zH(y`;SpM7Rui-!L|FR5bjH*okm~vUBuz7K;<5<ZtmvaJl6>lnE7GDA12fjaik9qre
zgn8t7ba`@l6nMYze&^E@_$zQzKuI8*e+%DDUK8FhUMoH^el7mld_sKcd^h+e^Ggaa
z3T6ux3SScr7ZDI`6<sOXD7s!GU$|Ikk)V%29$ypBH7*a%gY2JJOPDJeSpT*Bark}f
zr^-*KpQ1k>d>8nZ_=V^5(~s#N;@<6f6Z9tK&G*-4ukXK-c^&uq>8r;tt6w_3TJzfK
z?Y?&#-amf-?ETsI*6%ak@w|(9ckErmyTG?@Z*tyDeEZ<t*Y`U=1b=+_QS#Hwk18Lt
zKXQGF{*?Kt|I@=y>7T!Uj{cJO#rKQwm$jcyd~*Bv;{BHQ+djyBO8PwO%bl;+zKMO`
z_-*yK=ik|WasTQ2d;cE;!zzY0hBf~W{nPx{|F`E){_hjNX8vmW<^Su`&v!poe(U(W
z_Cxl&+Bf#EO<p&?e)?+B%Y83qyg2+~^$XV*3@?ISJbV%VQuEc6*J^Jq-?hAxd)N4u
z@9ow%xo`a6l)bt1Cj9Nvw;pdD-iW;}el_P6_iLBepI;w*d+WW<N3~BYKkxgx_Ivja
z@gH{IAAjroR{u5d%dO86Uvj@_eKGqS|0(g)iBA_l`F$$+xbOq#2bcGv@5JA-y}kVQ
z$h*7mcD-Hy=I@(>Z#TV-e=G5B?fc}9`#;V4T>ORk+l%k4KXZPX{@U`p<8Q_PDn@hW
z=`31opV)75xN$6Jx8aE3{LeXuYbEyso<F<~`E2<&@~a9S5V|I`RghgURB*PSq@a_4
zw7^jTd%;@*Klsb}Kk}OinhVtmO&6*c>K2p};N>^w+reALcZ#1&-~qpd;8x)T(J!Jc
zVq3)xB%`I?OBG1B$r#I>mw%wJQ}K(Ey-K-Cwz7hfyrP-{zx)|l0htey+7bccTH@2i
z6hvPMpAa?@))h?Q59R;HuPX3~e<FVd|9}430!9LJ`4t6t1+4`y2-FGe5|}DDONdc;
zoe;Z_i(sZeB)=ZtV%`Y8E`9-lT>dY-b9mx-9`R`MR`WjRQxv={R3%&_+#$>)Vj;3Y
z_?1ww&?CWPf?`79Lh?da1=kDu3Tg;+^40TR;+@8~o^K1^W&Y)Y7Q*o&MWXk_&PW(a
zU6O2+_$96=@krcQ{DoMfc$LI1iF@MT#4d@kihmHx6LS&WB5Wu0T3{c)Apbu;JN}9M
zHvAR5^4upl-8t(xCb7q|b8)D0KIJ^hS;qO2a}(E9E_<#MoR2x3x#YP*ITvx1arkj;
zV_(W9#u~xw&*aV2&t%WEknsW|19LlbERzo7B*sc6ZKfLxGykjqpYi|X|K5KG|J?l@
z`TO-x<sSmymwk)-_T?+<*Qn2_A6LBBe7EQg+v`&=g<guhP<b}*anwVD2YwIcKKOK>
z<-X$m>GuWi%iJ@%H|zfDhxZ?Ed#dw%(~F8%dT-j^&U<&@y~#(h&+1=yeari<`s2;d
z2fyR~y#Af`XT@Lpe?R|9|1<c%m7$x_mT?oq3<hn6qyO~&=Kb0IXU<>Ge@*{9|En+_
zVB%w*&Gepe8-wWodw=8pru;qmcjmv2|7ZS-{h#o!`+p(hd#1O{46Igc+3dL-?3^8(
zleo5Wf8<u@KEo-_p~05H@`I_BQI^5!|E7ONe=C1I`=0d8@#~t;Cq7Dl@OoeNUhTd0
z+t$}yZ`|K5c+>TY`(?t5Y0s^n&3ba<@!=<l&n%yJJU4%#{c_*SXD=_jocuEJ<;0g(
zuRgqre0}rvqBpnRoPMqFiuL7>7q?#CeHHY2<?AhPzP$B+r~YomyBF^*K8k&s{mK4w
z)|abaS-;=>UjF0J5AL5Ye=Pms`-A_-{vQi}RsC`JtNt&7!G$TANrs7mxs7ELt23Je
z8xPwLR!3GpW(UT4hI)qo|JVPE`TOxt%->0WKm1|)6Z*U5m&4CJ-(|j4d}024`Qzaa
z#qYnpje9HmZr{5nZ%f|XdCmAH_08`$``=D{*Y-Z(L+pps?|t9%zkm4d%DeOLD&DPn
z%k!4??d3P?UoU<2?$wdkdtW=e_I!Qiwc8t`H#=U7ywQ4B@<I6Hgb(o_Tt9TZ4|vD-
zw*1ZKH#^_%f1CKW>g|KK4DS}d-SH;&wZ^ORmu4?Nzu5Z1`sKG*H{P_qO?^B0t@OK=
zcgNqczMuJC^rPu#{jXf#U49n-=KX8@&+)$*gFoX{rXCh<wsdwGP6_S~9wFXDUI{)s
z{^<g>f^!9b33dvzi9Qp_5HS~>D0WWlvS^fufUu{~M!}~7(*z0y_X{NnYY8t9DiS&?
zq$?~Wd_-u9P^i!?K}W$p0S$pB0XxCPg1ZDi3T_k96geV#T5PKLM~M_EQE5TxLsF$u
z$x?Pw!IJ01cZem5DT_@My(;oa_^42}pryclK1bdlp4U9bc~f}j@EqlF<z?cH;4$Pm
z#q)r-lJ7mQBCihbeBLtNeLRjln|WCHr1^dMr}4iOs1!02HWd~Y{x5V-Xq8Z>&@!R7
zLbbxpBEF)6qD3N8MC?TSMOTad7WEa|C{`fuFY!ZSx#Tpdd}(%>N|{qKA7w;kBV?mx
zugbW|M9Zv^nI@AevsT7e_Jk~_oP!*@{BeafN{f^eRP@z?HPkd4G?LZJ)y}E@Q8}P|
zM(LW8k@90DW~B_pSj9rcR|*yKOJzSu+euBAkQcuzY9d-BdO(y>v{YDE$V;$M-~xX*
ze>lIAK%k(wkdY9B&_+RD!DRw{{7!rUyjOU<c>=jrx#T&`I7HZYuzp~k&RF`t@1OoZ
z*1xxZ$N#zYxAkBAzl?te{~iDL<*&-0A3sfgT>R$yb;sx0PcJ`i{K)*V^}WTr?{9y+
z6@FX!`uWR_7i!O!J!N^){wVKZ&4Y;hr|*8eoqp@vO}AU+w|Z{r+~&G7_0GLJ26r3o
z-oG1r@7O*0`}gkudcggN@v+Pkm#1RS&OiI{-1Q~%tJ+s{Uu(QIdbjOe<a^x@*FMO6
zy!m1Bhrb_sKDvIa_@MM5@x$B?`X5-{OS}tw_vYQRcW2-AzrXe2!bgcuJ3g9zWctYX
z@#TkyA0B^j`PlkN{R_|6j;}kvrTm!qQ|(vNuVcTIe{cC6_s8OI-oMrVn;C8~EMpL2
zi22|DukbI=pZUKke=hrR>HCCluf8~b{_;`!qxOdx?_A!dzx98Y@b1~$h<DT8dwn?Y
zKJML*H`iZRy;}T2{Q1UbMbDqTD19~Yb@&_Sx1ZjgeJk;{@r~sh|2OmA+<v3{_U+qy
z?~LESeOLBQ{+-;r(057i=D$1te%eQ+&wgLRzJ`CR`QG<~?I+hy^`8+xtAAeq+4jr*
zx8R?je-8fr@i+hPoj*2z9sV8t_xRtL|A~x~m_(VInYCFyu*9<(u|8lq!qUKUpIL`_
zIa3|eL8kLeKN;^b*f4DVzw=+j->bh%ekK2m{-N{z``2?{c7JC5ocu}o<A(Q~@BhEM
z`)>ZbMen}8D}8_Y{rmT(AAWt<|Iy%6&!;n=u6+9cDed#}&&8jwefslJ|6}2Y$oGQp
zeBOG$ZGXGy?ZUV3-(Gun=e_v{wGXpCG=Aj#bl~H=4~!o)-dn!=_vXrLsn>^Jt$TI$
z)!kQzUtNCX_4@GZL$3v1KX^I$Md)+sXJSvcJP~+$>*>>H94~ZV`oA)G-Sm3a>wmA;
zznT5^^Sjj_GCm1>Y5ZFJ&HMYRABn%1|E&9)^8X5h64Od%f7VHC#_T=pKiKU#j<BC$
zD`xw~*3T}@A<bFGHIJK}=R9``_iwHet{0q#In%hdaX0h&^3M|J7c3Lv5WXQKCDbOE
zAlN2&LQq%ep3rq+UeO6+{1Q@<&n3@E&5-^ftt(S6BPe@Awp?zd+#$L5a{J_j<T_;&
zWu;^n$t25E$kfZY%e<7{Abmpmf%InSInvjqRb+}}SY)N;rpe1H<}39nx2XuJ-d1_2
z+@@5Z_)_7t!X*V6#aD_9%GZ>;RK!$cRKrvqRby3msfwyusRgOksI60bq{gkTu5PC8
zub!k{sNSRgSUp!mNOP~|56u?MR~oZ4iZo(0$~5L`{L)yfX|1KFbzZYhQ&Y1`<Bj?x
zb$j(4YSC)3Y9VR|RC!eeRL`p>sl}>UtLdwKSJhHIsqCl3r?^`FglvPfpX6on_hKJJ
zcZpmQ_7FZO^g?jIKn}km-$5QBZXM1p_FT4CtVdX5S(90cnRS`}F*z}1GJa#&&#-`D
zA;UoidB!)4+nFvinK6YhN;8=MWBa4}tM<qKZ>e8*e6jxG`<da>%nv5-PrfaDGyC<H
z*D-ItzR7yK^X>e%U*8;ht@O&|Mc}jkCp#bAe9(4Z@cx^7Q}1!!Yq)#tPW|njH_u)F
zch&XE^GlAG%r9A8(z(=i$^Y_=%bzcET+zIeeC5Oyx2p@U{=8~(&F)&=HIeJd*MqMo
zU%zoZ^2UxEqBl?9)VRI#&bPb2?ioDDdKmeL`*GvrdyiY6)IOd4Ozg##m&&hSyx#bx
z>#fu~rFYZbzIxN}Cg6?To5(k(-t2kH`kw7W#fO6*9)Fnn!Qq4P2gwhTAI^PP{qgn3
zdmmXo1%2ZG9P?%OSF`ViKNWxb{9*Vz<L|}4t$*47&i%9KkM7^Qe+&Oz`S<qUx_@T>
zI{&WuGv{~2FZG`ZKem5g`EAA5w6En~^}ZhZa_fut*Nb0$zpeas>)Xw5Ip4az`hL0g
zDfZ*#_jd1ozsY|y=S|ex>~~e~<3GIq@Zv+`2cZx5-hX``_TkNku8$I*R(-1YT=_-q
ztMS*LUv7Rm{$<mb6<^rC@_sA%F8tHw_l>_-{;M(OGfiOTV)@9tpZPs=Da&e>6c#&{
zSQZwR0A_7wU*^|LTbRO_{aL26X0Y3F?%@*S;pYwFy~WeZ{eg2jM-=;S)(<SbEZbNj
zSWmIKvVLSPVA{*D_20ZdOMjjG5&qr&+nz5>pLc!S`r*y{d+#5;pZvb~ec$_t_l)m%
zy_@px@w<xmf8Q_pkoIxY$K;QDKd^nsdC&O1>Rs2{32%P94tc%!71yixFRGq@dB*d+
z;Mu{atk3Q~Tm0Pr#ex?dFA`s9zgYGB^Rtj=&!5hHYW(!t6PBl&pE^F9{Osp5x97>v
z=RUvjT<*oT7v(QkzYKW!^u^*A2VR7}WPjE3%Ki0>H`CrOe4F)l&f7<CJ>I=}x8(iy
z4>g~{zC8UJ{e9h!^FP1*V)?`N*YV$^|7RIun4(yW*ygdn<uK%8;BM#k;pykK;(sjg
zTkwLAkcfk5p6C~m7Lizy5|N1_wIT;aM8zT{{z&D?ddVMCh*Dastf9iBvQSx8d9ISb
zl8};`l8v&Ss=4|Pjd(3vZDyUnI!kmH>c;7s>gMSh={e{}8-yDEGxRdDFgj=GVOVS6
zuOFy)P`60eTUShXrp|5c+gd+0murS;vTOd<6wx`V+og9%Z-Snnp0BQq&JC^Gn$ns*
z8rmA18hRSN8Zw&uG{0$1))dps(r{A`QqxllRC7^tSDUQHtnR2Dpq{AyUG14Fhiak9
z0_C?#-AblPl}hf)8<Y<ze^Gv+T%jzbtft(eyiGYod8X2B#mx%t^7G}o<Q~aBlBtq$
zlQ}BAR9abPi%g`fu$+h-qwF1NC+R9FH^~r*6XJ>Dx5ZjT?+OPBhYD{KS|L~<@SATA
zuN+S+moDcKc0sl+ED6lB8T}X%7?c?|G9G4(VZ6xT&Jgzh?%$R_$9~WKz5Ms$-#dR_
z`n~D*qTklP_x-%`UEtf3&vQR=y?^?8{>ul?H$Sa=a_mv={nq=wkF}pxKi~6q`WKJi
zo&R<I{QNfY`~IKOzmNU7@oVNcp)cC+|Gr>;arb%S6Zgjt9_ioHzj^Z7tIO#Za?i${
zdVOriq3nH<dzyB$?>@O}?XI|85xb0b&Dg24lVRuI9dmY0-!*sl+dYQ+*!H*V7eBE4
zp!5;BV{ylqop3p|`gFvZ*t31-%`e<OTX`n)T=a!o7tUVFy}sv8=EE0H@4R~TKIe<{
zkNn?D|D9u)!dS@U!cxO#!a0q{pMSDIu|O$*41c_UkiZ_kUwnu8!}*!{b@-?7E#sBr
z)#LfX)x~v`E0t>^XAI|G&JJ!>zM}$i!ogy5BqvF`%6^wim5-KRBR^GcskE;IljtMC
zc>dKqD><Gpr~Fs?o&P=H>-;ZfUs%2z`NI6o_uJjC%f8wF@ci-a`;DJ9fAaqt{QdD~
z-(S`Lum1)7-TJ5J&#&K4ew+W^|1;-T@1K`{rv7R9EAjt3Lorh((<a6qMpmYyOpZ)K
z4Ez3F{dWnpR+G_`=@)Ysy9?JlZVA3={3rRJ^6%!?<p0ap#J83wm(!i~DdYS9Tz@@&
z{QkV_gW_B6S4&?QKA-%g^O3@%(+}rAh`k?wFX8UVJ16cG-p#+S@u>arna9tbJbt?E
zN&Tbw4?jP=`;g;d{DVvPtL~k><A3MdojG@p-0ix%>yG8^DL1)qe!Hb}|LS9*7iZpR
zewy%o)}Q4J6PUlT&Sn3`Uc+9+#>BdiIh5%mgUkOte<Xe>{LK7W{7dk!FvCyAGtBC&
zg)Gs`NzA{Qb}&9+Sj-^IXw0O<EXi_{)tP-MyA8(=j<+1UII20$vB|K!VKijW`B(Sb
z<cG&sjW0jG-2PVm!}cf7FM;1Wzk7dG|Fry3`t#zSb$@sM@%zj6H~7!I|4vMw{!jit
zk4ci5g=Hc~2xl~FFY8mz=bYiJt}MzN$(*X}dYra=y}YwIS~wc`<HTA-?+A*Esmg>(
zvP)c7;8sgg@>9;z($)3TUZcHE|Fp4$DT~Qd!wh|Qz1_NJv{q`QX;0L>r!`S?v#x{T
zEd6gfwYs8u89E%go`&a)gN%8MFB*rMt}wf75@b-T6Rz<?@uxJeWUIt$@in66!n=i(
zh1~e=aq@BL^4{fL&cn!C&AXe&iT4uUPQE`p-#F9R?yxT9@aMASdd=<1pDNfTI8RVh
zSWR@QXpV@qu({9)!Oud$V$71;B;JVclVFhAEA1m!EpIK)r7&AjN=ZsdPdP@_T3t+o
zTeDRwO#7ntJ{=BSK3y?AH@&$!kvcPU7i#}dTcg6O@>FrHLXi@SO0-J4nxEz!O+`&1
z&9CYas_M$;6wb@$%AS<JtH7=(tNc-Asq#OC!*U;`tt8HgI1BX(Oy;-XtKeSEzM6SA
z!;Js){%!oj_9y)h^KZeQbw7T8+wj@+<HQgDKCJ)v{A1<Eg&)qnfAn7b!}fR5ZxmjZ
zKka*HchB}V!_B`pmfT9d-Eb@VrtXbT*95Pxy-|3J?~dBtb$60)Z@)F~*0!4uuf4iF
z?ec;v%vZcG{kmv*MdEtVb%|@ouQpsQxzc<k^=ir0l~=egt6Z!;|Mtw?Q~qaI&xf3!
zanASLoU=k_o}5TJGUed&1J(z452PKOe|Y<`=u?4bZ=JWjc;|fC+1xWSXKtUKa3<os
z{gvh$g16q@;=QAFXX+h|dvouveQ5JU?kU5wu$K$pB)x5YlmA-#wZof9@0NYc{37;!
z^)Ksx?-|}RaWXGt+{w6^Ns;L(gC7GggCv6{Lnp%_#*IwJm<}+NGc95I!)(Fwi%E#t
zg=GoL2bP<xzu0OxK5`0jtMS<IOyN@D$YQ<5#LM{bf93yE|Ly)e{j2_0^UvgO`Jb*o
zet#tXJoxSWTkDtLk0W1`zbyWg^&$R!=eyds*WbpzD}L+r_UJp-4^<!TeyIC+@>9|m
zm9N2H*L-dKI{C}L&+=aqznuSa;_JlktiO_e+x<TB`{bXPKT3aG|8o7i@Gq00f^i1Z
zPnLP?8C+?+#{8}P`urCB;sTHOQ~4I~8S(S*e&CwT#m)JeO^RKW!;qtuqnvXwhc)XX
zMhgc2e>T78{Sf``^DX#W$v5tAlfGU3A@^&-54mseJ}>!@@y6@L+$TF8p1GHI`|*wa
zR|_tNUC_Lwb1D1c@rzF`e!k>=<=3Up7n?5syuy3U;d<cB&U-r^8$VTie()v#OQ#oo
zFLpk4d&K)l{Bh2c6;F3Qo$>73^B2$5pGG|uc&7Jk(({dP_I`Hy-twLIr_isE@6BI6
ze^L4V@8^;~YyMXKb7p+Sa)8~KOM=IZXEoPaj&&UOxh`|Ja!=u95MbxO%G<(!QqWB(
zS0IT`mw&AQr@(anBz}EfAFi+L4(wMrJ$ROJw{m~vnaG>NThFtGyM^Z#_aBaA&ez<A
z-0wM;aO?4{7bp@|7L5|SEHPcWLT0J-Unw`KW{I=n$0g-tT4k@urO3C+h0AV|eW>tD
zwM?r;*IK_+|BL=(!&Sy_Otj6`nR*+C8r?U3Y`V&PvZbx{1sgT{M8{Q*(;e2@>)XAw
z?zP%#S!mH}Az}5yYKFC>?MGW_+jmyk<{8E%2B!LU`qldH^h$I)v=g;vYQ<?Y>UQaS
z8qCtWuJc1HM}33x4#lksYVx0CxTNPv>PdKs`-;C2V-mF&TF&pn+s^%yOP%XC`wdnr
z=8yj_{dWF6>6g<l!$0r;8vm>L=lxIdZ|oocKSh5k{&4+q{JrH@%I}oFFa8<+=lsv`
zPw$WK?}pz@f8zh7{rUUn-Cu^kmwsLO$?>b_*Y{rwf1mi9#t_Xoo$(ywZ3eUdYyX!1
zarhJd$L-IjKfM2@{cmB?V@+gJVDDhdVCiGhXSnlc*RQibKmOqO$^7g5FT>w%zukYw
z|9bv?%2$&wS3Z6I`1a%ekM}-$eERV5^T)`~3%>mM!u56Hm&(uTpK3qWe`@~X@YU{%
z)aO$lW#6BD{q=?3vn7xJKje7m`>^P-_A|a0WiPfryYP6yL$e32_s#Afy1(wh>xcS}
zS)M+9{`Hm3Ti17X?{weQz1#K9<lVtHi(lKk>3r+(PWQdV2eS{0-tT^2|Do=q_b0|r
z?>?;lp!t#KWA?`*pI?9d@m2Dh+xHvaY`(1hIQ_lV+sUuLzgqrE{I%Gd@;BGtbiFNp
z8}qjP?f%z3FUwyFzW(?+=}qYy<~I{wr@x;0di(2ZuU5Rg{Nm|zsplV`IXu7q{K50i
z=Zr5_yx9F>+l!DFO3xQPz5O`oQSifs_bcxv+!4EzbEoC*+<PYXf8DFOTXuWmO|2W>
zuP5DLy*YZO%jlUdqi4E&8a>lx^h}r0GhIf{bQwL<W%Nv!(KB5}&vaoJJ=10MOqbC!
zT}IDz0iWqII`TF;@-{m1HahY~)sZ)m+c)ltJ`j4O^!W6nibuMSW<FGTX!795{a+7C
z9<F|H;Q=9enTH1+6g<><Z2pw}g~aRPx9RVGy`A!Q+S`wBEZzjZS^tLn?Si-E@2cJ}
z`LO9z+t<$TDL+zvT>ifQ`^xVZzfJr0^6RG0DWA2z9Q}CVo%y@z@6LT#_<7#vlb>zB
zefp;KZPvHw@5MiCejonF&t%Q=o8=tqGBzW2QO>E{>b&**kwT_IxA<T1G4ng{?c{yJ
zyM`y7>p5otcMR_X{&qnf!P)#0yuUdQurFr&&3cfvoHd;#iAkR^n0Y&!BKvIC)hui*
z6Pf2RNieQsG-38(de8Ws*_TC=c@c9k3pdL$=6_5nOjjA_FlhcS{5$)1_3z)m_x<Mk
z$H&~qYQuVw^)TB;RyXEYMqh@M{}=xS{m*0g#jt`wgrVgB#D6~j68{%7C^7ml2{4`c
zKkr}iKkdKo{v7=~^N+zV&L1IPRlgX0d-G$-?@#}xF?{>q^uPYU>;LBeNB+0{H)oj1
zc#H8GV>7cB+XfCF?wh>V_@xA=3r-T8BH+(=g@=K6Cyy7;Qr>XTi3$R{1a9(|^Cj?}
z;K}6q!EM3wmnWRpfsaw(p+L2umhfZ|9Z?Ihwc@KKG$pS|ewA{P-XgU}YQEG`$-|Og
zrPfIAm6<AMtWc-;O|emNnxdvso>HyyWYwAKJ(~Nqe6`kSPSsqck*;o|-lm?auBpzh
zHc`1l@uGs6LWO*y{1o|4`40K5@@M6R6qpr{DIQW-A}1`{C4E}bS@Mu1tJDd}-x6gK
zJd%?oH%ZQtVw2e@yHCDZp<mutE=%UVq^-mh@kDVc2~o*V$tM!0#jV6vh)4>r5v&r3
z<FDa6$jir@%cIP_mqUYX2J;n$5C0zhQ(!p7c$j$yYYy8K)@dv~%(0B&|4;oZ|5x*m
zonaFrJCiQcTBi9-jZ9I@d@P5Uk1^-5)Uf!mtY+E9;?APVBEzD@vVx_bRhP}2Z8BQ}
z`zy9{EJ@5cOy!KCOu5WDEPbq3*e<a>W81=3!1|8mD~kb(FUuO%JhqQ)KiSJUuCuRZ
z`@s5(^&x93t3B%umPi&imVV~nO!>@K%zqd)7z-IZ|3Ccq>7UxaD}N^bp8eDId;izh
zpO1fx`=I&$+MDuMt6xMu^MCT_5%UA?JJ~n)Uf*+7<x22n#Y>CMo1B?-!tCgYLu(Jd
zI}mYT{ec|^H4l3p)i};{BJ)JaiCf26j<+6LeoXba{c+*r6OZpY@!({^srpkHrxZ?w
zow7OYcv|Ds%9ATjDxYjS@#i?xi76+qovu3j=A6X^_lqHyc3(Ps$^Y_b-W-8>Q{b}0
zr306qT;6=8^4g^vS+}m;R=O*8Z_E9!4<(=EJ@b5V_eIsqz*qNQzJ0O&h1ZL>&!@cD
z@zVeG)VJ#I|Gr=QvEp;%m;GNafAjnv@FVtT&Cjhr_xw!zk?@WAi@?WS?|k1{ztMPe
z`^}O!>956JvAnE&G2z9H7jZ8qy<&UA`u4!vneSe{>wh=l-PLzn-_3oe@IK*#>ZkJ0
z318H|ss3R7Iq7HKFO%Ps{;c|E%V5cPgz+we!vEfX4*!4sPh~j9z|UC8D8lrD$(-dL
zYXtiS4rMMG?it)oJnML)`PTBq@vjgN6p|MHD!f=^yU0C}t)jAGb)tNtDWWSyr9{<5
zb_p|xY!*2!dR)v)e2qB2gttVZM3_XE#9N7GiS6Ry;(NqI#gfGyi@AxXOPrK^AZ0D1
zEbAnjD0@g|f>gh_v`Dmo8}DPTs~oBv$(+gD`n)swYWc(Y?f5qGyy6PrbmZ`0k7m2Y
z>dZQaMT@nW&6M4Z-G@Di{Wbe;4pGkE9M+t2Ts+*J+)uct^K|fT;(N`%S%6W{Q_xee
zO2CMppYJq}3U@Q73dc`22R072X{_}u=FEqgdYMI7N}0=<ceB*6Ca`L;o?@B6vX;e;
zrGr_K`681Da~jKIRvGqaj$%$3E-9{H&O{Do4l9nC9N#!vI9<8wxP-X&aF%iY=2*y~
z!NJI0#`>2zh)JD6_MgIEtG~<sYW@55@A-dshOGbl|LXk7`{nWD{I|Yu&%Q1Ee(y))
zujD^>|EmAH`&Z=O=YLQBPiF9BbYoIrE?{<N7Gu_8j%O}oKFoZcWdYk<4hOEO+^c!g
z2Cm5G{mvWA$Hm7wjU~5-m-+pd{?F#$_Wr)fu%7t<^Ag7S|K9x4|90YI#{1THwQu8I
ztG-}-y7ckQ$Cn;oe&qh}(S85>9`}>)9lgVF=gRHUTR*P9zbbT9{A$Bh>1$rs-`o(n
z8FOvXCHV`H=O&&BJ5zXW_65a@?3Wa;M&Izi*>Q8<?b7@69;H5Ee!BYUvgbQq#=n$#
z-u;y6sl^lT$Lk*-dY1WS{ztjbuRe-?H2hHa*8J7;7t3Bgd}Z?b>r1&8;m;pEzyIRV
z%d@W*z5e`W&pVzEb3bH%{PwB#i`3VdUw(hS{zd=0_s<`{vj1-QpY=cV@7Lc_zdL_c
z|A_y2?AN<r_kULYnEHL{ceC#czM6jB_?_!l^-qT%oIh9m+W1TJN6%-|_uX$Ny;XeM
z`Bv&<*|)zxSNv4}cH`rUcZP4WUe-K+^xXdC*B6(bJ$u6WRO<Qhmls~Id~@!#&&$qd
z;!g{n_B>B|t@Yu?=bvBZezEwv;=B5<rN7So*#33@ryC#QKg|1R`-T5|{m+TN*?*t-
zs{LN*&9hfZFZ7<Bdtv%+>!;waJHKuI;r*xkpU$7d-)DY}`zrDE(U*B&H~;wbZ#|1I
zD<2c@|CN7k{XF-*?faA;A-|-5Tm5|fwfpm|Pt!kL{Pg+bhWD@EoO#*(?7$PRr_s-i
zUhQ~u>&^97(_d=7I`_)q)$|wE&z?P=^62@a>?bzQX1)0ITJN3do4?PZAMLoWe9!%^
z`(5?>hEH-{>AlPQFzt)-Pn}<#-<v)cf8hEs=|k#!p7$A_yMK26x%fxmPsUHjuX3Mc
zKe~N7^Cjxrk*_~K+kgE0*5}Q`H#u)Nyit5z|HAw!=i`V+l8;@VF~2T-$Nb^%hlwBW
zeLVGP;^(AKtnYbV*Sy&A!vFQ=cT+wweHZ>E^HcZBu6J)<v%H@7^7e~wuUS7uerNyP
z`LCL(k?jUYJ9`PsbLOS2v)L@z_i%3HO%yB^{w}meK!ooFmmNDN+hPuRo^QMXe098&
zxH~z^IaoPQaINFs#?{8j#rc#ypUs~A5?3yNhj6->heVXr9hr}ElNCNFPE$0Iuavzc
z=csgCHCuC%E~f#H-hYk9Dsz=s6+C4HWs4M=RqHjgwDolT3}zT*8uI96Xs2tP(mJm_
zUB^;S$4JR+wb==iJmUf*OM~gUTXd%By*4~)yvS&#ex#1L)^g1?TGBeNbPwyF*I%e>
zs{K<_UQ1S|Nx#)-hRG2#5z8*C!&XLCUoGOzxQtSCA8M&+>*{J5xEkLzd2YPaKuV`b
zJyiLV{871R#rvvKnj+f9y3G1X`ulZlwb;}bsVz}AP@ki=UBg=Ew(e7%daYgR>r`$j
z@XDT*-YRRXutGsvPFZrHus{DF?i|j^9Pc^WIU3pWnFSe^{AK!U^jG_@;NK^IO8*4@
zIrpdO@6*5bf9L<I__q7gln?A5E`H$o9Q5tf55^x2Uwl8NzmI-<{Wa&?vma`|?)kav
z&%S>z7*d&Z7)Aau|9SQM&!6aji~nzAT*2DH`Ij?@ot<Sf!|gwFepdZh^YiKN*MH)E
zbNx{M((vKa+f#2M-}t`ie`EFL>8p^J3@`pY<9+V>BKl?H%QY`gy_)n!>D}^oGu}Ra
zW%2yqW4A{y9<n};c(&_R)Y~6#How~NJom}(2b1n{+;O<q{>b3z$)`)6O?<)lisfb5
z)5J%cAEiF|`uNKu*{2g;Cce4;n(1Ze)6|ECcMsisf8)-rOLzSqSUzmM&wS_cjZfF6
zTzz+?<*M-YsW-0QSabd4wH?>b-rRm$=dSm?N%vbH?th&7wD{TiXMWETpU!*2{^aE2
ziYLX-N?*Qs{r<JwtLhg%&%QjIefQ9v<og_t=RehZ_TlNhXFH!ie|qlGv4^K0U3<*+
zMD7XCvw5#--wD6l@+$GU^pmCs%J*FFeSA>(H0-75tB21|K2d#S`e6IL9d`@v@jlFc
zeD$&B;~9^3Jd%3i^z8a`t(V=eufCK0#QpWnH`edRzH)u}_p$4J@LTn_vLAB4n0)X3
zvG(WopPRq`{nGR~`-{wvyMH7ZPBE4+%dx&@ea6zrSog2z|23w=%;%UlvJ3GY6#O6%
z#~a0I$+n2ujd2oV5Zi4YIl--hPW&d^4D8Izy8oyC`TP6ap9g<G|5f<2>gT~9wm*0N
z{PENF*X>^m{v`a*Vbo;`WLm?x`9J61L%+IyivQ~P^W^^uro+t1%u!4p3@84u|C;sd
z=ifJsyICf)R<Jg(Ww4vEZeWyR^kkN2WoOG^ujLBhzb|xHa3$|tE<270mMVto|22$<
zSiZCAv7KO&_^<h==;yO<9$yo`KK*9)gXzc0Z-QSHzC8W1>|6BrfbTngIR57P>-CR^
zA(D}Uao_*qe;R+E{GRhW;co{6AF~9D3(H#OKaA)9ul~o$aK`;~+Nq6a(k^Ve)PKe0
zdetp^8+IXfFSc_mf0%L@y%|paOaF8JXUBK7uM<C|e0cnh`Q6dC{cpFvx&NBu&7C*t
zZ@b>EeS7|G=i4oBX1-2(<^OWqi?uJZUX(v?duIBK<(b!0-X{kit$29+f!u?0_wU@7
ze30>A(SzCt7w&()SAH+_p4q*(cYE#@+?{^6?C$wH=6C+xzJ0s(_RU*YZ<^e!xq0vA
z(whfw^4y+%C+u#>UAemwccbn$-3`1u<?f?<D;}snDtdhNN$a!D7xAwqytaFD{!QT9
z*>6|BUH4Yvo%nmR4<A41e2n<`=HuQ^4xc?fr+;SqBK5WK+tKg$ejNF^^;gI5Wxthw
zZ~V3U7xQnq-=e=Qei#2<^!v!~%fILUe*Qb?FVFw~|9>&uXZpet!{)^PfxVwYnv<W?
ziKB?!kX@Ah0^3%$uWY^Sg&azpdYqD+8#!-qec?XFlf-+3_ZjaJUKQTiJWf1TJViWl
zJTtftaaD1ZbD47O<&5IA;I!a;$nlx|5ZeaUjV!a7+nM?quQ3EL-25N+|Ixpof8YLc
z{uBJS_wUxf3I81a-~1oMu#`cc(UOUS*_wF<^A?uNtPyOHY;tVhSryqV*^aW>vRbo-
zv3_AmVF_drVa;MY!M=t=oKt{v3x^fQH1?fr&TKQ-ezEOl<7X>p)n&D2m1XT^5o6(C
zImP^+=^f)224{vl|Ce!?a6Dk0z|_q!o#6^&Kl5ppFD#KPDa=01zgeen?c+JhIho}Z
zV?DDl$0zP4Zeb2KRx##FOsiQ+*e<i_uo<xAF!C{kFfy_nV%fsD>5ubIgP-ERTmDr4
z`SHv8$HK3Nzp#8e{$s=MfWL}=HUI4UGyi`UlMz$QKcSz2Uw?hs`9td8GX_%z?|&+P
z|Nou&zu^Dme+B=f|4#pv{fpz@e5Q1kUZ(Q@bN~MPYszqfiHqe1Q`rBu-!Fbv{$Bl$
zo$)Z^YR2D;7nnb@t>E<G`p+K6%<*6KA0N|0_AguoT;Uw`Y(LqyadPqe<@v^A$}@qd
zfnQLBPx7OTyR4SfbTJOmzoPLHOC@)T&l8RmSjYcLP($pgB)9Yf$*mGMB^hLS6k-$u
z6e46zq^u-GWTF-2RRdL*Dov7imiJW@RC=l)DgRBTL0VBpPCi;?ji#&iX^p+AxhgYM
z>Q&dOdMlgBl}Rv*P8Cg;5SBV8xl^J?+);d%!~&@t>3fn7#FRu_gincvNj;XiB;6!Y
zFLGGWSWrpOU7&$)9k&Lj80UX(8v#DycA;Q?H|`7Ui`l-j@8U3Go5Q&1@6q3f{_Oi-
z#$?BQnsL|vqW}LHCNn)^(q#Pq*X{TAUp0S^GF@O3VHaRo_rLp()gQaR9e>OJxcqYX
zk^7_Jw=Y8{%STofmS9Fn23|%5<~uBctS6aH{hRn_)jvVz9=7%D25c!TUaU*l?{hS8
z&f^SZ-^axD-|PQxrhc}`?DA~#%%>Tu8I4&!vaV)rWqHp0isdc)Uv3Zn5&>uaH9R6*
z=Q$>FhH`WA%;0oqsbF}^D8W|D@rAvgwTD@WMV~E}eIC0c`!d$^%;y;y8Le67aXsWS
z=R3wF%OS&|z@@|O!^z7g$0E(*!dlDD&-H_A1Lr*UMXV_-uUTB#HnHwttops|TkE$g
zKX?Dx|L5V)z29H`y!&_A|7i?XOyNwu|E+!-{3!al``>=1NsR1&OMce>^!;Q0Z}lJb
zpLySszi#~c>D%5PHovz1y7=?XcdqY-Ka2jf{<Zuq`mN>T*^l$TM*Udv!{K|xSEH}T
zzg_$u^n>H4!H++mZ@+!;GV%5A58Yq)eBJO-@6Fqng>Ux0d;P}!RnV*C*LPl5zqR?O
z`nBr2#*duu$G%1Vxcf`+_oMH9pS<4PdS~(3=6lu88NaN4FZy%!-vNd_OxswhSu`1S
z{<#0(|C#siAQKyt_`kegGC!++1^p@hGxfL2@4DX~e((NM{qG?|64MRFdH>!1I{qp7
zqw}ZYXVjN_AG$u}f4lH~_qUQSB_I3V%e=38m+)@a`(y7_-ZVUme)RH@$P16R1@9tX
z`#uwTT>Cih$;C%S56tiHyeD)&_rd3feUH_i>OH;i=<EHp_r&k@-kEVz|LWfh(id7T
z*<AIzCU^DKrMOGEmu0S4UTwVIc4PImnoGXtPMr(8I_q}!ovND`uIgU7a&6MBySEP9
zoPKNZZPnX<Z*92aazFRskw;%1{e7tT@cBcjC;U&<ALl;2@!0;Q;=7cOM?d}ewBXDC
z?{|OM{!3tPXV2nT%XXD{CG!>z8@_jZGq}CjMOoX~CviLTXmEe#UdS8F*Tk2|-zLN-
z;U=RgV<%-P#UOKEHcKvFX0xQYl$Y#1`C`SrN_UkGDVeHNsBPAe(4MFBRog+sRmn}B
zUwO7hm!_H8JjEMwoeKMvvz5~n(-j*P^W;Tk8)V$&o-5X>I4f%@u*jP!bSjD|UXuMT
zVJmh^EI{I&#2$%v;u+$<#deCei0Vk>OC6KY5UJw#<m(j5l-Mh!C;3veM%YQDLQFtx
znJ~N1GXZV>0$yg`AbvlQsS<}JW{9p8auX^Q787>lcjDsV*voN^Q-NEK_YUuDo@DM|
zZa1Efyu0~!@O)%XWxDY{jY*N6flHaogd>G*7JCraYOcwg3psN*W!QaLCNn=~-OS0#
z<HX&`A<5Rsn!>i8?F@@F<KMrl{(k)bn#r6shixNU30okWCA%T#4lZS`O>AcvKm9+z
zEXAqA9nDq95zMxa^&nd(`!#k34o~*AtlyZ6nNPF&bEWaV=2_3Vg*BOZ7jp<p7Srs1
z!hd%B+3;VMIe_g0haHzV*IUj)?gYMKfs_0mJe%1qSeLLFatZNz^RjbEvvsi=u}@&%
z#LmQN%PGn(&7#b7jERM<ibI%v8B^%LkUyz^X8!5?ZS-^1w^d)ie#`$^@mu-NmtSXp
zM1GI^p73M-&woEpeb4>;_e1XITi<1W_5S?-?cwK>pSZsier5T7=;!X=AAYs|5dO~h
z<M1z;zw`fo`hDfcqHj;X&HVoPtHh^8?`FMy^3Lw#{LhYGcYmGmb^TZI?-su%{hj)c
z>#xagt=}boA2KXwUceH;+{1A4-=hCF85vn7uzq3XW}U^{#}vc7kWGgxm|L4`BZn5p
zA&#4zmpIMYGnsoBU70voV%Ul}N;tIH_p?9ey21BG@V3xg!P)#_d|Ub63S<l46fP8;
z&U=l^f@?Wf5Vs!BYVKVeCt0VlR<mE?tmN6kSI+l|`wXW8rzxi$$7vR`|7O3XexLYr
z;P0}(i~rpJ-TCMG-=F^;|Cjpz;<wX}hhIg%%71(Nz4}M`_mZ!xKfnLH;+xygrk|bP
zo4>65$o=8=`*R<7KCk|2_HD=Kt`B?PU48rg^_AxhkLKO4e=zTn#FN@5b&qd7n)gKf
zdGU*ump@+ad?ET={MnMHy3g;t>VEt6&DvL~FKb@?ee?NU-TOoDuD_{%W%Tmn%Z+cA
ze(?Nq|Eu-apPwT>+kQUxDdNNB*TpX_-dKD*^kw_E8Q(vAm;br+m(%aQ-*11{{to`-
z{BzmQu)n^{4ea09-Pv}s%wut3yT-x9y_h?hD}h6geG+>m=K>x+fnGr!fgU~yenkOR
z{x#f-*`rv?+2(O3@_gd;<9o*YhF68}0&fYg0`FsPQLavo#q0|>c5#REUE+=6a%Q{0
zJcGHC<qJy*i#GFg#=rj!|6lvB&6vcv<^RP$rN6R&PXBTG+o8|r-d}#R`HjoF!uQM{
zUc4`OFZ3blWAsOz4?o{&ytR9E<C)TvhR6C(ouB7D|NJEG;f(uR9&kUt{v`eB>n9;k
zRy^78^z^eo&*fhRz7%|s`E1(Lc~5sfO?Wcl!IC@YZ_C~Ne3$Fq(Yw`mweC&5-|`^q
z;igB|A8&rr{$#<U=?}j=ZhXG)MaXl5rz}qwyqNp0<MZUNx4w#g+x;!<`^)csKbwC)
z_!Ij_;CI@ugTE&HR{fp+qv@;Q*QMWn|8V%t@+asI_y4I(Nz6{nQmnP?JseCNw^_qj
z9NCIE=W#Z(=`oivg|KerbmvLtnaw?qtBvy<M-#_WjuTuByfysC1l)zr3C|R16Q0Wd
zjmwtPlB<|IiARgKhHrx4E)h{tR}p#9EHPcNG?C|mF9fWFMa7ng<_TTlyTX&iCn8iL
zoGjEQv_P0$SV!o;&@7RBvH22rC2vZqN^O)nDfvlkyHFRODbF14liV+P_VX18m<S~a
zM+&tG1PHVVNbzg(ICD&9eZVHgWyPh$ZpCtz=`T|uOA@OdYZ2=|)>c*<*4=CYoT5CN
z`1}N(2_y&x3R()(^2zY9aE7q$VY$rwg4u@kEZaSHQ4U4+Ic%96C%Kk#`En?-O=kVT
zrp~F#CBVtRzMr*_&6z`xGlIjLJ(8`RwVkz-?H&6X&V}6fcy92x@uYE2;rz*VpCyfX
zEu#m+R)%z@x6C(Kq*+s0N}2C6?`3Ue7vZR9U&3~YZ4Spxt`hDZ&R8}lmg6iB+4MPH
zvNN#rvoWwru$Ztgvs$p~vR-C;$I;H=&$fpph*_Mm<KMyGjX$saWdCFHSNd<#pX^`l
z-+zBy^)2Ve^`C!!a{OHIW5>@^zi$7${_XE4*$?;M3w?V0<?Xl2-?e^x{IT+<{?Ei8
zPkuc9rT%XfqXP3WroT*UnS+^MGCuy#^-t?xCW8X=I@W&no$Q+I>)GFNyyE=J)z3YH
z>pW*0w*j9#zZIV^&s?rW+*kP~3)~Z!CzK=VEN(8*FEK}Ar)03qUikp!`>Gvk3~B<Z
zN0e_X-j%y59W3o7YazcwVVe?*$^_*PO7oQ0svK8oQZ`onDSukAR%M$?hVnxtBjr0P
z?dmBSRq998-)bz@C{%Y<yR0%#IZN3=B~DdM)lAt(@w|MjY?xH8#1wHp@fG6kQa!RB
z@&WSq<^IUEDby)BE7i;Il?jwyBXvrWOM*>oi3qPqx(JKtC*ci(_XU0k+~HrsYst;c
z>BwQm$;xfV8^Pbn|BJ_!Q<cq#dHVn4KX-p={jUA1&7j4=`|sMH{y&0$7XCi)>-G=U
z?|;8Ke--<({o}`X2i`t;+xRy9jmqnwSGBK%UtfHs`HJt=;g=;Z{yd-c>c%^bj~yRa
z-mQN%|M|8jdmlvHZNIzVf!GtbXNAwypT2*b`1ti>ho@OjA3kY&*8lSEtLv{qU-Q37
zc%$=%@Acf5DlgByy#6Ze&4YI-AD?{m|M22%=9|TDxjs(+tno$d%g@h`KIwh9`<m~C
z_j8?>%5N6E`~FVk-G_HvA76cx|D^x<>=(svU%!cbfAQ__H<RxRzPWtm`SRkE<i`u|
z+dfo(>iMGlP59f-&&(fRy>ouQ=i{Nz?q4Q-+4N=Y=Sv@y-+X({@ci`i@aMeG5}y8i
z()*<0$?PY$pB#Sr=-HZQeovd9Bt5Bl9QG*j(Zfe2k9Iw<zGr!-?DpB)k$1n{`+xV!
zt=t<i*WIo^yHR;N?e4*QruXabdf(c6{pK~k>m}E_uUlWAbM4X9!mC@a@?4j{>2+J_
zZvH*q`zrU>-D|xoe~0aM>aCNvCf<&?U3Xje9>b&FCmm0}KFNF>_UPF|-bZ;)N?y)<
zH|JyDr<)&Z-yeQc|N7wTnQ!jB)qB76eboDa_g~(>{&4c+{!c+)RK9Ke-v4v`?~XsU
ze{%kI{`2^^?yvU0*Z*fQ39{O=WwO0zO=Vfh`0ZcHzXuF*thYHPaBbyGXDeXb_-DpX
z$se{qtbXnM6aH`O|CfwaEG}#@Y+qPDFz;YtX4}fPhTW6%IM)>J=Uhn~`&r+y>|j01
zwvPP+#|o~=+)KEuIGR`=Ge@yJX4}HKhHDe29LICk(@cI0AO43hIx^p9VP{*&zMM;j
zFPop6kAr6_mkehuyFJ?x)-1N|>?|DOYzLV>{crpy^iTcYlz)Z)cQO28v|#$pQ1}1-
ze`}_HEII6MoaZ=Ca!g=jVmZomn7NBJlkGO!7xqb<E4j`1*729{hw;DTf5p$sFTuyl
zcaEQ5C|&5bfEWL6J{SJW0+m8vgf+x=NUW3mDzQYOLh_tcfy{il?F#9NH{`u!4v9O6
zm<zuXz9ixxnk`x+dQ{X@+)%Pla)Jb><VmUZ(#$eHWM;{oQ(#uoPzaLIl1vxBE%s78
zNy=R|O5wj!oAOe{K>2;Ldt{Ewe3ole`mb_JMO103Y`7$gc(9nO7?;>d(OxkpiG7l5
zq=ck*N(o9;NS>1jlK3LdB5_mVi$tN=0>O{m?Cc*{PO{EpU(Pv)$ARCUpPkp3E112G
z#f5o0%M13)T&g_3x!$l_vrb{|X4L(E?(fdOXZ~GeC}6H*`@*r2Q<(EJI}>{u+Y458
zwmJ?j?nhj%9P3#m7#07w{=dZ_!_>#D!Wzl8l5GNO74sp+LPjy>8Eo=gU${efo^bbZ
zt>JjXmd1LTc?#2RrVq?dSf6t2=Z@m7<a@_=g!d5l1J36hGdPcOt>XI1@syQ~*`8U6
z&4ObGhc4#@PI)dVE<4Wm?4Q{4IOlMm;FA~HFWexUB=l6Ek$*Yg1>O{%y<BTKbvW%f
ze{*_qS#pVRtME+Y@#X!*dyP+4NK$Nvn7zna!F>YGLLs8r;%~)ECHN$xBr?V9L?VTP
z1<wmgid2h#k_?mjC9ziYjKE5scCIGQT^vg}R&!qGI>BAQ6U)uWsmGzoah5HMHIbEp
z{SenL-d-L7_R9>fe$V@P^=IX;oL^nP?){Sb#q)#xtNTaxcY$wCy`J^v+B^P_!5`({
zOT5|oa{Y6KXT?u<JuP}J^6JNHp10+1pTF7vy7T47r)`hcKiK`y_SuG4+OMl#6g|m)
z@Z(<F1L;TS9{qhX;l;gI3~#*Ow!VG%TIAL9m(yMgytDog{!#X0^9Q*P7v6Ebt$uU=
z&C0jy-X*+G_@MS_`{(2@iJ$X66us4X<NkKfho7JOzC8LQ^+ERC?{~XD-uis$i`&<!
zUo*bx{OI_}@>A>k=Py~Gr+g0oqWvZRbMU7rA3nd^`S!t^MQ;|p(S6(dHu%lv*FWDb
z{NVF3?Zf_e9B-xH?0qx&o!k4?w>w^{Jc+!|c%R`>{S(`#&!1R4(Rgy_$*(6*A3t~$
z^T_0p{p06P&Og2RwC2gmhl}pj-hOc7`;Bk6mfZ2aD|<KY?%sRc59U8;c<}uG>-!fU
zEPo{Q%;=@a%Uw_ZKghm2<8JN4x@X^B=DlA0rv2T&_dnkMe8>AP;qA@0cRuX=GUt2F
zuU)_Oe)fEE{gC!n_-*q$%MT)-w7<Oj7W7N}?}2{;|F!<T`kV283R63K9!~{d5sw6W
zD`VLIFowm9Ul~=I)R@ItZ?U^^o#*1?vgKUH(a(9Ar&iEdY=uOX_!Z$@d|x=ZIG8!u
zIr=y{Isb6o<NYOYTQE{UgRh9^8dn|H6kbIk2azX2+Wa=$%Q;tZN%QFNRC3SbR_5gt
z*duH%c2BHdY_X_>sDju+iQUruvNvV=q-@2H2%8B#75px+hCh$bhA&khSJ+2XNsLv@
zN|asHPi%t338^NTLK!*9V<H!Xf`uDJTtwCgF5s)>Rp(>mzscvsm%<yuoy-}@nZ)JC
z?apn++05$BsQPc|->83E{(b!?@~``k*Wc>@mJDzHzxf}<aFOvc3p<AzS0Q%~*Lv1l
z|E2zB{&oF-o<WCU=f6|`su^xGX|O~wUuBG9Y+>5UQqJzj9mltiH<4>H8xPA(mRj~+
z4lxcc_6@8utXo;`GAS_Z{mcAk`R`?a%Nc7~7jvBASjOV_|H1G3zs&#YGWIdGGO@8b
zaR%^6^6uw0<-E-HnRPzfGmf9UmBN=r-U=%6hH~VyNwF)iTd{GmIkEe5+45@g>Tx@A
zadBC5c5|lltP>CveIX_<Je4PmV=G$%n=6|%Yc)$7+jh>W+=1NeoSdv$4Ca5Deu?~+
z`uCMFi#eaM^^eqd|F0jv@_woRVDWz42c0i<KT?0K{JHh#qF;M|3jWyeqw9C<|18Go
z|1y5Q`)2%I>c_9|JHD>^tp4@m55qs!|2+OH{15%R<=4I+wm-xErZXL4wqd;Wx9)e>
zZ~4Eue{=tQ{@w8B>c2Xsl`MN$9<U^^o@ZIa{D)~Xb3fZ8j&Rm1|JDELG1M|Qv8`tN
z$+DDHmqUQlhJ%?Sor9hI3$qE+PG)hA<vcTZ^|%yS9T|-nL>XT&?q%X;$z?5LU(e~s
z^_#Pg%ZR(5>prJ8mp=Cwo|(M+Ik}jx{!RS7^Y{F}lm2S`G5#ydV9jLAe2;k#OB3s2
zRz9|G9FhFKV$zbc#Z(37bMNL*=Lle*%*M^}kc*%93-3|hvpl-ouQ}$hNw8jE@npNt
zS;}L?b)Mzof0jRIex3gP`q!-=3ctSp{lqYX=?POXW8uFqzjD8S{`%oZ^glkvga4lX
zi23yE?ZMZbFKeDFJfHF+^0mvmGw&-t9Qio?Q{u;I@6+BheY*dB>0i!&Uw+Q}YWB(U
z!-@CJ?=HRh@^;?G%&%=fmj0^$o&5Xg&+p$Xzp?zPXZXx&$+3dXp7|VOHWMq;ZiX!k
z+)NfM57;(vFmmnXnZcjVzmI1nmpk_@J_8{Y!DgQ4?AKU6GaImMW9nj%XY6GbW7S}L
z$bO#lB-bO(_3XFV_HzE?TPw6(=s*8Po}FCITx&VAI4sz`*|b>qGtXuE$;8I`l}(FX
zku8CBA=?42PCh%{{p|TnCjW*0ss3yD8~<m;pYFeGfAfFu{KfqH=}(_;wV$#+ihOzc
zgW=D_Uu(aHeNcU~=;fyu^IooaCH9*8)xqaX&)z+~@{IL)($j;Fjy-gHV*gV7?ZG!~
zFKQnt-aU59>-M%gbM7%bT=t0j@v=v<kLEu#dcb@);im76v$q#N>Ui<(W%TnmkAB}(
zxy5}m>ejwHdiNjR-*m6~Ug`txN3$L#-mkxN>DJskKOUZW-u=4gP1381XAMsjpSnML
z{CvVIp0}6Yo&KQn`RkX~FHIkr-%WUP;w{rhfv?5iOTWGOwDjGs*ZOaq-uHg0`6~Ux
z_1E`5dl;TFFJO7Ybo0O4-}=97jPb0&9R8dO*_m1L7!NS~WBAUvhRKL&KNADnG0wN#
zwmeh0{<0rs+rX~Hb&dC|z#)NZzEGZ<+yZ=tLi0qAh{=j2iyjg4lWLJYFE>L@PWG<U
zMM)Rwv$D?eR&uW-BSjVp8VLmo8;UFt=@AwcauebcF%Xj$Uni;|EHCs$C_uzR^sh*y
zkT>s3wi}FLjLj@p*~Ho2F-tOaG3qj1VS2&j$F!Z{^uHDV5*Tw?IXHsZ4VV}HmHWl?
z%k+=Ze{IH0Mq$QfjNHtXEHha4Js-UTaP$tq(K`S~?*N=QdI#X>9e_{lNACa}y#w&y
z=pBHgcL0vw0r*aI^bWw$I{-)T035voaEH+79e{PCcL0vw0XTXG;OHHIqjvy~-T^pz
z2jJ)(fTMQ+J{`RSaP$tq(K`S~?*LphdI#X>9e|^E0FK@PIC=-*=pBHgcL0vw0XTXG
z;OHHIA4l&1G#b4FaP$tq(K`S~?*JUV190>Xz|lJZNACa}y#sLc4#3en0C}YN-|+tx
zcqTkwL`x`7Ku++3z<ur_Rzv2EOm7%Xnd?~Zu~o1?VRz>&;8N%M!I{T-mBWW?BX<GM
z48D7OHasod<-C>r3;8Yh`uI2nxcT?<PUNfLKf!0l+r#V6AJ6xaXY>xh(K`S~?*QC8
zdI#X>9e|^E0FK@PIC=-*=pBIK@?w%AsY1^M{|Z(MY6$KSj1j6678lhL)ez|t&J{`%
zn8}yUTf*~#>of-wyDO_3Gc(g+#%89!%nMl-un4n8vIsKqGA#Jp_{;Za;LqFN`@Xe(
zfAM4AkJj&dzR7==|MC5Y+t0s0%6{7aKL2~`&&%IVd^!60^Cy;%&F@!ynD%MXryC#2
z-zmKDeRcVT_46H1dLOR2pLk#VzW9Bg2f~jgJpTOH@bQF4nU5SEt$dXCnESENBb!I>
zAN_pv=uzWi>8GO4zdZMN(eqOF)!P>wFSb1ocwYFz==I;X;U5G(P5HdztMK=LZ|Pqr
zeG~ZU{p;+{Nk653b^dzwYs#;`KYG7get+^^`e(<lXMZM*-T}xkwC(^@`R4uo+c)!X
zQr|qjF@AIUn)>Dc=R05czDNID_v`X6iC^qLBY$l8;r)y8@1y@_jB*SY|AzdL{qy@b
z+wb!~Eq@;T>Gj+C@6mtP{#!9v|JVEb=XdR&34de%$^75-KaycPV;-{#%SL8a=4hrz
zjB!lMnM+xCS@ttMVz}|I@9(<5oBmn<@BOFr_ww(*zn1=*@hj!m!=JW4C;#yL@%hKE
zpGSW3{&@BE>1WqZxgW(oMt=D9uKI1^n~XPdZ!6z^e#7xb=XK4ipjUfdrM^D>TJ3en
z%a6}}Urc)G@apnQ&KLKe1U~e+=Wyrj?Z<ac-VM9!bcgZIqdOY+R^5AdclRCtJCS#D
z?}t2ef3)$@_eZ}Ug*~48r2JXMi|Mbp-|T$j_s;sG^XIV7)t|yY7JtzC*!ua!*BRft
zeklIf`t9Y{%5R)M+<&F~G5IU{ch%p0|Nk)hG5Iij`Rn>O{a+o!9VRx`*KDF3{p`nB
zH?VACNoGlBUc_{ZX&1{c_K93_+|^vuIZtp1aY}MC@kt5Z6_yuWC~``8p0K8ff@rO1
zm&gU-7a|wLUWwloUoJL5<cg5J&{Cmq!Vg5{#J`IPiYbd4iKq%o2)z;5!mr9-Bd}EX
zn3%uBWQh;rlf;5We~3hj&Jqifm@DyFY@cYPn3?!{vFV~_!u<l4{IdL9{BQXd^V#rz
z<7(op=k(`F<YwX7&+W|J$DPc>$oqvSnrAcjWNufU-@J1Ll7tgQ)y3|K&J`^elayeS
zyf2|5!6|-IG)?rE=oYaWF%D6C;X0xFLT<v`!pcHn0{nbdJPF*>xHEWcd3?C0aqQ!m
z#krWXo%0`O78gI)V~%L{I<|PWEY>Sbz6>Y+doZ>z2eVqRS+TXUhOvY*r!cQ&@n-wW
zHkB=aO^3aK!<F+lM;f~ds}@Tbs}nmr=VLAh9#@_%+<n|DxrKS0c_;HJ@fY*k2!soo
z2&oFy3GCus&wZVH1#c|>34UGvE4(h;kJ+7A3s_#T#Iwq?YO}_$8*?w@y~M-EWx<}v
zdWglGbr#!Ij?-M++-Er}I5x0{vVUMZ&uYX{$?U*#pS6MAox_&hob?)W9`iP4HCA`_
zMh+d0=WJfAnk<SeLM;23%o(E?1(;b`8`w6p?O?HEbo{sc&(q)Hf7bq8`oEaLjp5V(
zng1vKKmNbx-;CeKe!lt@`nTx+62{%kGg+KiJegye7BD?$7GaTQI`{wezoP%;438K@
z8Q%Xp_wWAyg^XvIcd$mWCA02e?qup_+Ra?XdYVm{or!%qyEW%GuG`%2xdXZ1at3mo
zVqeVB#~I3L!f}`FEbBQ|efCWpjGRFn$?W&p_&B7vPw;m0J?0A%I3rjqq$&JQm|sLg
z_=4baf#U+5LidID3F!;S^A+$+;PU4*<!s<|=KRjSkM#%hb!K0dIA$rvFaI9@4gB~0
z|8&L^j2R5Y|7QNV@XPII&JX4vG2i>Y*?dd?w(r~WZ|YxPe4PJ|_wD^R7H=iq@xFiZ
zF5+#(>$NW%Ua-E<d13ux`t!rjt~`7B{Kt#aFAhGN`S{7hqQ|eF{eQXaP2~GuAD(@1
z`XKPZ@<Y~#zz_G|D}Rvx#P!wq`;zaAzX^YR^SS5qi_hP__<fW4&iy_5+pe##zcPQn
z@ZI_Q%CA;mTE5)<rtwq$XWI95Uzxvjep>j+_sik0%HOVhb^Ds~_3F3fKWcyW{5<t@
z>95p3GXHs*`k6m7O=Wz{P{Q!yzX?Me<5fm0#<dJD7$TW&v&6GGvL0pn!l1_Bz@Wj%
z!=%g9%E-si^KbIMfBzpa*fSjZXZ3f^AOFAg|Ni`&|L^AC?7z4EPWo5!uj_BbpSE9m
zKbCyE`_2DH@y{neCV%7na`ThI=jWe&zU=?};G^n$p|@M#_`FSj+wtb`tEv~zo`pW=
zfBEU<mKU?0GCUG`aP>jpqdAW@JlgSi>Jz;u;g2^yn)9&#{@gngZ`$3Mdz1N&^WArM
zCg1*f)BNV;n_jmW?*`wiy7%&~-o1bKH$5_a8u+aG>5j**9{qV7|E&Cl+{^PXPQK83
zx%y?#t0%8@-+q1T{%*#*==atioIbF>=X}faTJDwm>*9C+Kgxaa{Ic=$zR&t!l)tci
zvHY^-v+k!gA6I>r{eJ(a+pmY;qrd$B_~C=uhm-H0y#Mh&=Y8S3DQ`EwJ^OCn`{nO>
zK3w=X^-KKsh@VWq9{qgw^USYDf9C$5%UH;CiD?bfcSb>`Da=P$-ZSShKKW<!kB4D7
zlMTx=mJclVnbn!}8Iu?-n7%RnV@hM<Wol*OX9;8b!Op}H$R5f*iDLrS8}1VB7_M-x
zD_o1YlDV?E4sxDlXJK8yG?&SkwT7LMy@ur#<3a`<#{EpDEFG-d*m&6Qu-UP<aUA8U
z<Xz4e!~2u_3(sl(Lqgj`B1J!lo)o_!*(H@J)hv}NRV|q$Atk{msVBuFWh602bh~hu
zu#BjxxShlU@hY)eQ6VuoaWV1FVoSuU#TSX1iLi=Ri(i%eB`GG}AiPn~QSgTVpWsWu
zeBpS}KVp-_4~u^huNSuwQx**rH4&GT3YB>+Qzhdj6D+$~{;Z<1Qmf*8#X6;6<x~|X
zH64v(8mra+t5&NXRg>4$*M6-1Ps>%aOTAt#S8bA-mfBy{e`>}WOEmsy<Y>;(U{<@U
zRICs#Zz6wK&P;ZLbhz|d=^p8$QW28E;z=TYLi+??2)T-8i@y->7IPPQB&aW-E6^`!
zEp$MDn{Ow#9+w!W8%H<$47PIC4J@x&9<YY7WwFj+e$B|sc$;AsgD6AO|3&{S{|5az
z^r!UiuRjjI8GnBHCh+a}x0yd){o?q0;qUK1%zw=PB>i3VFZw^v|JuL#zi0gV^4t7h
z*8hnA693)*U;VGdu=&5k|E2%g84CV4|2y;d=AYHS%YVuKO#G4hGyV7LKP&&9`rG*T
z%3sI-?u_}&m8>7wwy^(ZSKv_PEa(2flgq8iv4r(8OBZV{TN>L_)^DtAYz3@3tP5Em
zuzq1Z%4Wc^j8l+%8_z<%hy30ANBD~QV+0=w$qTm$B?_eo$BS@?Fbnew*$SBOnsL=}
z9Ot;l<-s$H>pR;+=3k7=j29T*Ft9VGGM;B#$@r1s&;P&wzW&Yqqy8)DNAHj0KV5#c
z{(Ssh{M&~wmp|Y5IP?9*w-#@1y$X3X@s;@N?$-*hEnoINoAc!IlhEhyU&_C-e&O=0
z`>Dk<;paD>H9YfrZv7(vh3Ly?uTH*s{8sWE$J_eX>aX{`e)XF1O~spwZ{*&lzP<6L
z_O;fl`j;^;JzrM8JoW1Lo3MBL-m1URe?8;%_cu@9D!z4iJ?&-4i}TO3UWmTD{lfPJ
z*9+$t+Alp{eSPKrI`j3zS36$rc`@+?>r0=P|6b_6y#DIKn~t|v-$uM+d$;aQ{p+gN
zXWn$aD}S&5KIEPGJFoZiKInan`XKr4%4?R_{jc}F5qanJ{`UL!_fhXfKm7P`=;NAC
zil5(qI{xYXC-cwQpAA2!e=`4Q_TkI>qz^hD-+X-g@#}|}_fOw0dF%Wx?cKq*qHi9)
z-23wTYxTEUuM?k}KiT?7?GeYL_D2ej`5#+8y8B@Hea8D$_qN@gd#B{K>n+`z8*Xac
z;lC$-FX^t<op-nH-)g<R`L^Hf1Gfrp#omgzwec3)?ZR8)Hx+JNzp?Q4mwVg~4IiGr
z&wqc`1I5R0pGZ9ud2aSX`&IQDrgsnDt$u&@{j+z!-%7tLen0DD@#nNJ&%ZqWV(?|z
z=Ubo6zOa7@`n>9s;pe?y%YO#^G5%}#r{-7vuSI`s|MC6%@Mr#S<KH@ew*CF|Z}R_1
z|9||O@t5!K?|&_frcCCHF8_P}n*N>ox9;C3hAqs!EJiHnSlHMPa_-|^%QKOOnP(nP
zHouv$iCCF<tyr|ke_>6rPvWKG`$ZW<-U?|8&lWi@+AMloBtZ1DXs~FA$QluCv6o_P
zqLRYpf-eR2MS4WrMf!vf3wH>w6wVc86W=8ET;zw4u8^CMme2`7e!-3Whxv;63V7{#
ze0bP-ggGy;L^JPTeZVo5%ai*mcP-Ck-WL9?f_*~LLW)BELZ1cMg@wcnB+4aTOB|QH
zD*0M+m()?IOOhE9Sz=<M>7uS;uf$@+J_~;mbP(DpQZ61NzEIR%G*u)==$2rLP>%3a
z;S9k{0XCr&;S!N{k@G^k1ziLV^6uxo!2d;Xov@%Vr_g&L0pa^Xbs|Y(bHz`K9}?dv
zSuHb3E?r(yPD<{ff{{|N(qUx=)fN>SrG*M_6hoC4DYL1tssyY0s7tD~t5~aj)p)N_
zqmr*!rJ}6<R_%lOF%3V>FimdtH1#Y^eswoBcU2F$$Erd0^){74-~KR(>1kh<&F6Aq
z*Av{Ta9dG_r{#Yo{|$LDxpHYg$@i>hzue@wp?ykcF5kOv6TVe)awuKZ=n&=jf9tE&
z+nbMU-X9j})65av^-1zizTg5j&kq`JHhiDYmBe%C+qRzzMVeSA-H^X7^QV$`%HQf&
zu5Wt3C4cz;qMpH1!CWHYYx$GK&y3zrVz|Q7#<iZU{_m}q`|p`ON_lhUr^WAOZ??X=
z#k5p1l>g52igW3AL%%qFeetpA!>5P0?@s#S{%iZgJvaBiQuzMryZXlqZ{xl*{4jqp
z`MJe6#jkm<H+@|6=lXA%?*;EZJUslI`-jdq<1e%R=zqTbDCXJD&%ysM{=9W>-sP1K
z&wujc+o^6VZ_2*+Yw7b>F9d$|Fa`XZ`hC`yS$_(6yo5J$r7@iN?eY2GTeF`63`uX=
zu1&b%_)P!9<R5)MSY8I){d8C0R`>NgkIug@{=>lj=r`;0HBTcx?)W+5$MRQP4=r9N
ze`S1k<w?df#wQWiL@#SR7yqaFf7dIf>klq7-a7U`{mraD&pEPL8s2K(3cJVk&iLK(
zy8(Cgzi}`>_|yHj;>VI_9?x1o_Pzi0>i3&BU(^}Z{|A0P@=fhe^!L}l-u?^uYWK0`
zyUDMVf0Zo1|8{@M`6VZ`MzWU8;(G^ECx0%(j+YBQEf+FUY!b5kS^hleIrrDAe{X!V
zem(zX;|GnO_gJ}jxBX>(KjZIZZUq)o)-?jxc=Op-F)w7T64@$M&i#y?S1_0VBX1ek
z^FK}hg19cRmNRDw-jvuO*vh$&&4y+9KlA^8xZ1^*i(F&7!Z4T1hNty+)Q8XCgIT#a
z=KfRte)D$*>ym#a&$*uOWt3&S_Vn@{t@~>p7{Bs+tNiBE_k>@6pT4>J^}5SL>1S&m
zb=`S&ljqLu2h7hVKI6FMb9UQhj~k}v0xn*<C;LF~w%^6FlNYXpJ~(iv^46Pclg<ZU
z5xmQA`|%aI%RjGW-8pfu`)>0Mw`*T+m|u&zs`sSfyB)*k@Ai+rU;c8@<7)XM){n)X
z-o283xct84tGRC*U$ehk_}uBq+AsfqEPlQ2y~E$?fA4>-`=IjB?8$_0D}R1{$#Bo=
z@!Ky47!`h9dG+yQDf@TkKd-zVKYbVaQ<BM^jfJV0C55|+dB>MCAI!dYGacdD$@QIm
zC6n_{p>I(PJ9sqNmwr3-OOrQ-=LLfy%L9>AWqsKa?l;_70{-k@cxx4INp^D^im)kX
zD>q3*@gCze;<DmY;MNpO6-||}mrGM(m2VYI;tXd=<5!nSl-wn}UMf|2g}k_k2#+w&
zG?8C&OJsXR)x?^`BZY!RPD<~STrE6N?5@mX5og9lf4q3KCAFk(iN{LlN*Rk!5ctS_
zlJfz_Q|6uj{<6CXCUI<KOAwnV6~}*)N#eWJ_jHag;xQ7XJWBsI{!!pi;N1E@lkK0t
z2Nr>!@BhgCf5sfk?a1ZCr^oO2-{dbVdo@!eTddF`4yV88nJ%(NvoSDbFqW_~{{Qi9
z1;aHSX}&MqF&ur&b&PE6vjsHyLYUtE|H%@=uFm<Bw^C4#SC(rbuMBq+Z@k=fg-s&g
zget`r%f3|(kX@(vL{~)qF`uW%TS<0Rcl{ofM7ev)?-hS2n`<7@-elou<Eh~<>!HqS
zveVqgD9d1qA&cgI`PEWaMK?)5Qn@65Skyqkmyb_ig+QN(scf*M3xB!D8z~E+Yh26t
zZA6<y`9!A)iE}SzW#L*c=BnT<v5&dw*M8=u{P+2eF+cvt#S+7z!f~H*+K;;LGrwuQ
z-}#N3XNI5+>x}O~U-THc`8IOtGD`e7{?X$11fH+_@f<(?pZg~IA>_@US3;k281Aq%
z{`L6q>-p~22S18@U(Yz3vHg?dhZ(=7{E_%4&D8bh=#NSNoc@)3J@)3!^HuNl{{{bA
z_gv-iw%51bseI)4x$y7dKkt7Sefa%C{rysg2Cl=5(%*_cmwkx)<j!ouufp-_&)M&*
zf30Nc;w)#~$u><`T;Zy0D(`NFWh@PRv-x<qnt2mN^`%ZrPZ2xK!^QhmvRC!K$~OrK
zo?iAZya!|=)w>mYh2*#td8Y6$=h?xL&izWzPwa$H40{=a2eSd&Tqea|x4tC&G-N!+
z(!^N*-RkQ)1{>z8?~A_h{yWOX#dCmj-rrAOB!2||i(^t_yTY3OPyVyii}_FO-yZq6
z=|jvXyRY6q=ls?DvGAS1TbFld-^YEJ{Icss@V5_twZGeZ^7u0I$MQelexLkm@j>SO
z>W{O&{rjE!_s1`@U&g<0{4)79{l~R$Gk(2dyv>}#u=d}Rf3|;P{|j;M;B{sz{&((g
z2g5z)*=%*}YOH_%iTo1zBfz+UVcy?KEDl1;gx;~{{=4_*$`6MhHH;6r|MBhSkK<2d
z+xKVve=Rm|HYt|8|J#0j`s>V`^gsLa{Fm=uoBgO`l4XC#QOst;Q2krr$J{TzKY2Oa
z1w6U;ux?`7{73LR<BwU4C%D$LIsTpW`{Xb6A2q)s{-pi-`%C6m{^w;MEq-wQw0LLw
z#Nw&_+ch6lUuxdFcGKqe*}H2V=sYWX%lXFd;rlz5cRjDIKY#x$+gYnCEAQUD;c`~|
zM8S!>r*lsgAL!d(eeTEgs~3e%lpJ`oD|hebBXf@5JoWL?k4y2V_8l)eC4Vye@P-5F
zC)_XoIa_-w?kv*@&O_-Zo}O_y({L{EY|fcQ=RTiQI_vo0>XQlA4KLYTG&!~4(Efdk
z4=*~Gb|v`wqB}7Ue>~my%;@gi^E*#&JAd^0vMbBaHe5)*w&UvgD`hv&-x9mF=hnNY
zQ4j2|aNT<O;^o(d|AN15KU#l&^fmXxtvB8unEsS=+~5#pkz>sHUiHTH_2GA+U*9vF
z;=IW3E#;#4M&c=F2x|aOspwCcc7-50VVU(Zccdpu?U%F_^%VRlY$LH>@~-${p>$zh
znMOHBDJ|(FX+IHFfnCDCr1X>@s2a<Qi*Dt!6<jTyt7s;pAo)b0T5X#A5wVjZdZK4U
zmIzMfa_00Cyd|P5u#Jyj&|W~9|D(VM{_~vk+2pyG3uFl@3eS`|s#d6CE53|pFK03L
zeSt5c^1{kIE4fU#YdE?%?s5HPPh%Eij^>yq$R@aj@$*kZri~n1Igj%g@hR{u<dWfB
z!yd)+Qp`k3Qs^SjE}kUrGlG+)o(n~AD~q0$dMJ8a<UYR|>q2HR&ZYd{#A~J0C78u4
z1=sS-5Hu0rC-qluy39H;9T9IKAMp~UY09S*w6s2%OgGBa=v4Gn>{kA$(yv~sJJIm5
z&Rq4mI@b*+8#0@wn&}&UHw-g7V5G0jrM5uzh2{~xCq~((uguJ?H`!R5UDwrAD_3|V
zU!o|bnx?f*>xr7J$~v`OIwqPk<=@G<%IpzomY6BSC^=o)NX}FyK;fv;a@kc9#=_nL
zD@4>K%tSr-o^br)<lvppW6HUot&V%6&~njoDRt?+{J9*xT!~!eoJ}0ZS;e^Siv-C0
zm0l_Ig`I)5nspCT``^f4Oa6Ita`Ja^T>byy57W=QU;9~8n0P-IeTn>0{YmRb7(?9e
zZ$A_M{rX+~x%F-8mq-8Z|5^7#`n%@)rO)p^Uj2CA+x)MO-{-x0{(8>)$8U|F`M+5H
zKI*0MqqL`0uQk6X{O|vf{c*!Po!8v&xIc2g-TP+QyI-$-p9{Pwc_Z>+#yh<?2i{12
zaQc@1OY(QZpNW4S{9$KU{-5plou7*T!r4IURjfHUm^GMAFk1e%{G0ac*k_;jtKS8_
zHG1FpCG4x?myEBPU*3JN`;h$6=Ci=Jjo&wadGf*ZYun!ye`>xS`)u>0@o)OS!$1Fh
zi~m{vGxN*hcOPE%yhwPP_F4V&thd*m3%-#5%>IA*Z_#%w&rUrNetP%Gl*choj=uQy
z>fG}KkLw<ZKhAh^^4X)8-0x0&y7cMC>kW@>?+4$Pc~JSF?`g=3l@9_Rq`qzY@a;v$
zlhnJ9Z<#&+{qEZ<@0W$otRDHkob|Eyt>{bDx4S>C``GZ-?z!z#$yY1ht$3sKOya@i
zyO-{&+)cY5|5Wig-_t`+&OQ=)AobAgfy=#yM_$i9Kb!W>`p3r~Tfb(15`W+M@$=X0
zFW)|2|Gwde!mrc+3;uoka_hs6&!yjFf2I7}{x|jaroW;8AO3duaps%l5AA<ljGzCm
z`~C0lcZMit7N#qIC;yKBS@843znu&d{vTyn^lRVy+3%LVXL;TJ!th1X%L{Mw-YkB+
z{?5}I+pc@x^t`#^n&c(M(+`dYocMV0=53P)dmo*DBJp6-^`DmmuI;|F>7l~CxND~_
zYhFvfDs}G2N$vBAS9!1PzViN((dFYe^zT1^=>A~It?SnsZ#v$syz6jR<o@GF6JO4G
zzwPbmXJ$|4KJ9(Z@M^*fgJ<$@`M<S(F@C${*_o%6?~Z&9eB1Go<&D*w<8RNs&v^U(
zUG7)GpNs#!Wj)S(;ZNj0e>MT`RIU%K?-*>Dj<NT#Z)C6JKPmJ?pj(JvU@mXCFo$3=
z&k^x?Qk>iyIQNOYl}?cUCLSm(B9<f-A@Nf<oBsf>rl6T{x=5|$Z>a?$I|LGicZ;S;
zrKvsFE7$GSu#^8LvQs2cY^l&Yk@r#!(%~|VGHqg41>W;JiL8}gDEnB(QhAP6g-(m^
zOx;?|Ey}|3ak3}nCn{Z+_m(P@`XHkr^-_|b<2c(VRy+2UTn~92`M2?1<}u@*$9<8<
zh-VV#G&XgX9;Q%6eTGto`AnZ!oY?lW)v{%?y<n4Ji(*b>e8d>ST)@J@n#@wd{F(U?
z%NG_~)-S9wY=2l6vu<HAVy)v4=LzL)=27QX;9kjFAdn((oKJyo2k&H_Yuwp9l6*`8
zg@V6@)`~cY`ir)R8H#&|$_VERvx~}!`H9^UNf5du(8uq@>&mUowTfc~TPaH}Q!G;f
zYXQe=b~)BrOcIQb88$OYG1W2FFeLv!_K*4hqW@bMDwr-Y|7NaYZejYxpz;6wAD5pk
zU!y;N`Lyiw!!KE1Z+sE>;`_PkQ_&~0&n=(-d{X!{^JDL)uCGUb{Qk-NGw8eHH=*zO
zKl6Tv{<-<*<e#g*<$kaJz5UOtKYoAi{9gV$=(qH5;orA@IsS6|$^WDNd&G}fKXrbt
z|F-}0?vE@V9Y6Mc^!=Fp;rY8qZy&!q|8d$Ev#(P=>wWBd@A;ndgZjt)AN4*Se*f@Y
z)%%E#nxDD9?EJ#@P49c=_j%uMe}DI_^-J-`neW)&I=?k}7xu2>t^AuquXtZSfAjTi
z)LZ8_e6J;6t$ua+&HZ<C-|u@j`OUFc$*&{c^?a24Jn56hM~x4w-fw+><b%bh@1G<-
zGkpH|Df;u1&$^%geQ<j>_4T!v<*#bq{CVU3`sE9&=T6UWy(oV*@%8FA!f)5Sne=wY
zho_$pei8nB{Qdej<*!e@R(~`1b??iZXRjYKJ}!Cu@X5|+`Y&W(R=xCl+4!>a)#g|6
zFOEFnc%=N0>(QjgTu<&i)_StzNzBvN&+fk1`ZDR|!xxWV6ue@5xA$Yj=hRPs-ye9J
z_g3z`+{g1D*+0&EANj8FZQ|R`cL^W5KR);v_-WbaZC@9D7y42CjsHu^r|?fwUzU7h
z`LW=8^|y|%<zJ(}*?*7!ZuvduyUmY#zx)_d*|u=*;<VzJ#kQFBE$d~rqiiCqg-lQX
z@BGKbpvZiP^);IcdndaT=WOmdyjgsc`1tuY^2qTB^Q8zj2v-QN5Y*zY<K^bf;VI>s
zz<!S9C(|9K9V`~?Q#kxN1lY}4Gnqx0x3Xxm@v`n=%>Kvl_uF4ChE&FS#%~N48I+k8
zu}omIVlQXcW?#gr!*Y|wfZc%8gmVE~D6=I~H}hc@XO@FZe;L)8ud=S=u;X6FyO%GI
zSB_hctDHNA@0P$if!n;NxgK+NaNF@Y3VaawCr~ALR>)ptk;p0GOraS9HhfaNa{Ttf
zlf-<*H;En>P7@Xu`5{~(R3vbR&ysHspS-{t!I?s$LgE73c&mB#^Og!O6EPLx5Dep6
z$m`7~%dgKL&*#BAgJ&I&2H#hKP~jHgM!{yj0-pOkP5g>NcZHq^-V^8$uoWogSL9p3
zvxwV>Tblbk_b%Rh{N4hUd~7^-xGZ=M^GgW3iF^{CA$(4Fwa6LKwPKgW1jRf>&xjU?
zpOY|_tdmd?QxrPL7sjV5AS)Cgd{sDHL`<|+)L7I}<dRUB;BS6w{`dU+LSZ6PMHh&2
zi@p@G7F#LdCnYb{BEc`FFEU@`h}Z%NKgo>}tHdoOR!Du8c_}L>H&yn%^b^Ue5`~gI
z(rabE%YK*1m6<NXDjO@yD(fjDE3;UpQnpQQg8XLre!0CejMDN_tE6hAH%Tc;JQgVt
zvK0C#EGWh&?kFZGx<WKSe1iBF(F4L<g3-K1oKx8Dvl?^saVPNza%XT&<XXjblWQ&4
z2hKg5AzX90PH?^8F6F(<yPW4F*DcOy?jO9D`1*M_ahG!~=VIXT<(20<$aju^x8P0T
zM<UXq6{0SpD#9EB;e2oTY6X{yu!y;f@``*EIwLq=fQ$bS?<HObzEoaQo)jKN-c#JN
z90ttu{?GU?!|2UW{x|XW#$P{wW&Zy7Yw^$ZKcas~{W$ym-}fCqs(%Xq%=p&w$?3zB
z4^ux|e_QbF$CqE9rhf2yYxBzS1=Dl+XP!@UpKN(z`P}Q(!&hrxTzVSzB<<;&=UOk<
zyl{ST;l;<7w_cgNZh0N^ruc2aySeXYeO&m3=?Bj*p`TjcWWP-NeC%`am+xN=eNp=I
z@e||6Pw!8Bc>2-dqtp8hZ@0dk`(FFg*-!UB6uk3&JN@mux4+(Wzm|M0@H+n$!>hBe
zy5I1<d-pE({j&F2A8vir``q(+(Wm_%nBV)nTl{Xr`{oY;AAY@?^5*Bu{ufhToPP1`
z`TD2-9_@d~@c8<Zq$l?u{<**He#wJ^hnpViK3w+T=6$JqO?U3znQ-se{oV%xk8VBo
zd|LSI_p^p)JWtO(a(nRX-q-uv9_@WH_sPCTnh%`reZ9N?UjBXm`=$4m+}(Zm>%AEd
zv>)EMpL_S{?WjAe?wxw@<e}dq-bWG-kKCVf@6+8Ccbo36y6bb#^Zw`id+y)37jbvx
zt<vji*NSe`-7S5@_4MKst;b&;CO@=#Fymh1-Timo-~M^~+uibq7oVJZdg3wf!`=t%
zj|?8~c)aq-;ir|)Za+Ks%;?#!r}57uUfg)8{rdBp+3$aUjQlkDgWJ2-w|?&*eG>lG
a^1bW(=WkKp9KW@FJN=E}yZQIV?~4Iw^;b^-

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/6874d5b1c7a64b596c61f24877d422e89bebe58b b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/6874d5b1c7a64b596c61f24877d422e89bebe58b
new file mode 100644
index 0000000000000000000000000000000000000000..e054ad5f14723fa1bd5829725e38de4a681cb3e8
GIT binary patch
literal 25086
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#P;0`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O+Vxj#;S
za{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W%MH($Jqvg`;W7W?d5;;N
z#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQe5UfE=4JaU&)3^uuX}Cv
zI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()m0t^gul~dSck3VbKQezT
z{&fBk{VV$K{QujGj?5>SWm%T6__A(f^<>pzdCQc*^n>XOvj@v}mN@ovoR(Z=oYy(R
zI9xc6aFlZ>vn#SqU_H)az#_o%ky(~y2Fqj?3l=?=f6PtH7R<MqSeVKf4>J}ron$Ix
zddPT`v4l~WVc)-l|IGhC{Kx#y>Tk>+o!?zQtG-|R8uvBfYt&bcZvo#+esukm|Ml>P
z+qcJ`cYXAEAMm#DjqRI;H;doSeSiMr{m->uwZCot+WDpL^X$+0U)a9B|FZ1MtS<|`
zn0&eUdDoZxZyUcK`4RqW<8SjnHGdxdk^1}iPud@gKVSaD{#*I~2g5_ALoCNx)!C}p
zPO*JqTgMj3%D^nf=*>{^-{T+uzdiq+{G0J_{XdugeheLqY)svZcNm-)a{t%<Tle?w
zpWnZ;e*6D!`+fKKyWdxScmGcM75n4FH}!98zE1e!_36qx?$<M(uYEG-(VmA>9_BtY
ze#rB%;X%NC`FqZHE$-a7_2A~an>@EK-C1;x;Q{}{#7Di4XFQg8Jmc}4Cug4YKXH1x
z_nE*8o0opCj=%c+s{Hl;*L`p7-);S%`pNdw<&P2{g+4gGZ+(~gw&Bf=Hz(ivyf6JI
z^u_R7?stnH$A3KdarlSzPxoJXzwiIv@~7l4-#@qiM;NEDRIn{#Kh4q1S;LvmnZn7>
zd6VM}#}STm9G^Kuxc75w@bvJ?@W0_-E3j5@nNWf7eqlA?JfRk$>q2ruwgQEGvw6*V
zxANTLN#`>W*e<{=AjN-#cLjF?X9R~UM=M7u$90ZnT$;Rl`KI#U5_l)bB&00lEj&v^
zQPf#9MKo4aO7x=02a!HePqDLN9^!|@jU@UcR!a0r%#tvbh!^`Kyiq7pNK!~bC_!k4
zP?Yd#;oCx{f-M5x0)qTYcy)Qd^9b|o<Feqq$S%YFl5HQGDtk7^Mot5+O`M4w``J`k
z7cr+YWioa${QNJ@Foi*!@h{^yCU53x%nMo4Syfn>SSPXGVx7e5$g05F$9j^jlEZ@Q
zI=3kACEiHBYkV5~F8oUTANelw-Q@G(za}6dyjG+^bg$SAaZ~XI(d8mDM50BQMQ#h9
z5&kJGE>bDHTxhFMp|HM)hG>@<t3;7xjZ~DBwd8B@0<je$^1|6dIzr+?YlNyrZit<h
zV30Z~^+n1=>ZpW<c(16k$U~uW!N>dy_*U@F<C(yHhpU`Ri)#(%L5>FY9JW2I?5x+A
zotT6e*D)+)NM|r%&}GnOFko<JFksMR5Mb<Ka%TR-WXGh!82jJ%Z|v`jKLvl@{ju@K
zwjV)1kN@=isr_Tix3VvdANRjo{^r&z?w2#3e|z@jnc(x^&sIK@f7bDI*ONt$86USk
zj(bw_#NtWSlbcUIKGA*p^y&R)ub+Q>apPs<tGBOQUU$8&e69Uj>2=ua_pg_{75{MO
zqu^($uVvqF{y6bd?04Cpi+?Tu^E31_Y+}5?^qEPKxr4=yEs9-+V=)I0XB+2wPBBhl
z4mS3$tahxwSnjhtVF_W?Wy@qg&M}4aJLe)U9qx-<He7LBXSl9%#dFnj&f>7<kl^^w
z-oSB`Q;_=`w-e6-Zf~wf97XK6SzTF{GPy9GXL!oc!*KV%=l`C6lm0II6a0JePtKni
zKkxjI`9Aq;+ZVUbS3kObIPuQ^o#^}D@7X>wezyG@_bvCE?6>r<Pd{(?81X*h?ef<J
zuL@pGczyWI&$o---~F)R!`JsS-dDcwe1GJ9+4~FcF1`zSzwN!#hwzWrKh6JQ@{Q+v
z-}l-d^*^nDef@Rd_k=&||DO4`{J$?l2E&p6>;ErixXob5u$Cc^Nt}5<Qvp*U(@LiE
zOfpQej3$g4Oy`*Vnb}ziS*Ef0vpi?YW3c{r>d)ri4}Nw3di~4t&;7s8|5^VR|8Mbc
z_aCX>Hb3pZ@A*3Ki`Hkek4*3Hyb*c5;HArpkmnZ9Q=e--?|ruA>9i+-kC`8Bd{BNr
z;9lq5hj(r6+dNqR;O)bgkK>=}JY#(R?)jz{A6{6zP<g)p*@kC2&-Oii^R)C?<8y_V
zvtO~iad>zAL+vM{&uyQxKJ$Hk^r`#Pua8eY&i%CRbJv&qUqrrs_{#m=^!u)FhrjuJ
zZ}`sko#&hCSBtL>-zvXf|8eA()Zg&`B8)E?9hn|7U0|NYBFdV=`i(V{J&yAOmov{6
z-e-Id`7a4@3N9DO5)2nE7YP(m7D*DBCZZzJEwoVJHvd`vDE{|+TKsGHvji9fKk*mx
zPT@YpwV87vM=6ICXAf5|k20SWf0DotL22O%VQb<0LRP{bg<lB!2r~!^2qz1xiJTWP
z73CLW6@M!xAvRT1S2SN_zA%sQGNCXbRv`|dBB2JM--7%?vxQ`Zt_XMu1PDCg|H;?H
zJCi$@YaOQ_hcBBNt32y2*5zzA?1$JhIsLf#xxa8&vCFaTVKHQW$jHYS#2C(`!+e%`
z8cP_fBU>>0Y<4bo2KEQ+^Ei%j%;8AmxWU24#m?Qs-Nf^a*Nk6KV4}ccfs+Cbg5`oj
zf|&x$0#yQ=1ZN066G{{I6>%075t}6@EWS^|L%K$WL*|0Cn#?Qdzfwmf-$?XJ`bwL~
zILQdeI?08|RmvL3lt?K^Zj{iIh!&R??+}-lI4z+i$tbBLc~(+DYL29c<V*=ii4WqJ
z#O8}M3aJX(3asHz;J?Kez{kQT!7IZp!?~S3kBx!#K65&A53?;x2x~gqN;YdYeKtQf
zHrDq{HcWe%ikP$+xBYkc_x;b#-xGg+{<iq*vagw6cYQJVa^v&;&+k8<`jq>j@}0+9
z_BRV(RlnT#;>wGLmz!Q1zuf&I@WuM)`<^{~V(>`if!BTCdwcIR-!{0t_V%_r^Y41x
zn|OcI!`jELPgXyPecJx)`E#C^4`1$l_2KpPx4Iu5eSG<;=gaqRaz9V}s`}IVPnvNb
z^E;L;EN@uOv9hqWu|~1JWvyVZ=d9w&;9}-B=TYN1$feDBjC~0k8{1;mC9M0|mU3ip
z9pIkD^OI)-?-#x+{N4iLf?I_0ggb?mgiVFY1*-&S3swsD2{H*)3QZKcBUmG#&i|D+
zgQt>f0f!vB5L-Q~IqN4DQ<h~+&l&9hAOCaer{fRHA1l6#emnX__{*x#7N0dgo%t~L
zL(<2}kLNy=eOUZK=wtB5zz+uRcD*rp<Nw<1)ytR7ulB#Tef#F^rng((YQ3v|clDj+
zd&T$P-=)3Z`QG~dx_3X`$$q%=arx)4ub$uIey;iT_}AQDe!nb#MgMa8b?4{lpSyl7
z{&Vo(&i}jq*D(Y#6)-PjZf9A*Cc*iP%b9x?S2WjRE*`G695dPV*%!0_;kd_noHK%J
z8}~ZyGn|vzr?ReQ7H68xVD|sWzuf=q3``8Q|2O{&{@4Fc^xw2U(|-5<=KXE=bMChj
zpNl>odmr%b;hUeY7rjn={q+^+tCKJOK0o%n`uWLc-=D@jO?-Oi>BXmBPfeaqcsk{&
z!qc)R_D}kr?s|Ul#rhY|UQB=a?IrW8?=OR2slK`V&h~@fhyNeueB%7F_KU$6$<NH6
z1-}%1IsbY6=c8XLzrFp=`s3fXx!;VxyL|8d#`f*!SDA0>-{yW@_oeWQ_1BJXem^Gu
zdhy5S|6E3S=EKbHEH7B{SpTvvW%FhC=D5Rohx;P06n`JTf&i1?D}i<Vvi$e?KMAx7
zMhdnGunTYsWbiBU_3{4ay}`GM|2}^@|9<{+0{H^Te0zC5bHC(T#Ceco5yvsk6Wk8G
zuX$JT_3{@8OcLx7QWaJasTO@MsxN9K;xD{P=&+EvaH{Ys;S>=TQEt&x5hvjwAxR-I
zp*ezA1^fgi2;>MJ78DV*6}Z5!$bX$Tf~THail>mblb465munVhA;&8=JJ!F<o-A8g
zx!A;56<BN7OxdTgd9kvx%x6|-@n`wYyqKkzt%Ktz=L1eV&Ir!kT<5s2aev}o&Qr+y
znfD>D5^pEZZJt@Y@_cc8WqgbH?(s$OJ>hBRR^+zj&g7oQ&BpVBC!bGDz+A9butD&T
z;BTP<;acINBBA2(k}6VOQmIn=r8K3FO1+fSm0BfzPexakQ}&?D6B!TLCYd=>6C}4v
z_DQl!>Pj+5NlR<Wq{(cT_LH71eM@?Sbh)&fw5;?@sb3NvVzI)P1vc{O@T&7vbH{SG
za3AD)z;TRCjP(HXVx~^UNQQ*}Z~y82zxkhmA@={De@Xw_|3CWY_ix3&%m1G-Y+_LO
zKmG5vKMQ}q`RV$@^LzC77vGD%-})-}we73!SIf^#AIskty<7Em{+rO(lV4@N+WPYK
z^W>-79<6xLdq4l)-aA3JH{H^_rFe7W^`L7vukOEAcEk0S#vQ$T&JR={)jZaH^5_Zc
zv$W?oUR1yGf7A9Z`(w$Mjo-d~zwzVr&kMi0ep~$c|Mv&O1m-I&vskyWy<*?Pah>BZ
z#~#i-+_Jo^ytO<fJiB-%@HlZ>asFm!<%r@~!m*OGo_iv18o!@_j(~?iuHYV_GT}eM
z_e8#m-V{40_CQob^s9)FD6go2=uXk;VoBne;<LnhL??)t3TF$><2U1r=GElQ<SpcF
z<XOk1$!W`Rfvu0_6q5tfW5zZHfq&M&@BWbauJQHeC+&|*KJ5Fj=Y#Ht@b^yd89wlR
zc=zt!+jVb0yv==Q@!sLXrjL_9^L?H6b@kVWUt7Oz`2Oq1p`T^H!hbLNed1Tc&-5SD
zzB_(z{r=>;_>ZI?KYvX9`RwQJpMgIee_s2!``3iu4S!_*sWG%LE@#wWJj5`UVHU$$
z21&+Nrd2HZZ1rp_*;Lts+5fT4WNTpC&Bnl9#U{<l%(92MoVkGM5CiA`i+_Fovj4sG
z$NI0!KZ*b1|IPj#{IltI+waxCbAL1aUh#|P*Xtj_-$T9zfBx`M=A+Mt*!N}cCckZc
z!~0s~<*R2rPdgqPK9YF2;K7XtMGqw(wLXe}^!y>;qd$+fKlXaU_e|yGnb$&ZbKYjW
z&3HTIZSvcFZ}z?U{Wj&j+=ti?TR-%CT>0twr+FU*K9s!=elPdl`@Q%FrH_oCls>t9
zWd7jr-t+y__ftMFer)&{_UYlLHJ@reNqn~aQvLPTx7r_Rzsmo(|4aSP!!VOUoN+p1
z3{wPi9?L$~PWCX)M6T&v^SEwvMRLpXNb`pBDGGQC9S~*^5fE7|VlG-N`d;*&*gf&-
z5_=@VB@`s0B_t%KiY1A}3LOyG!r#a*ETAA5CS)uuCoCiMULaXuqd<z_OTj`Re_>mZ
z^&)K|3L?LSGes;!---5!nTp>KSCP0O{#b0bXurrF;b%gLLd%7MgpUZ{5pEU^5}G04
z&7Z(`ori_nhO31um`j#(H~V=uOSVm{Y^((=D_F`|PqR9+u3|}N$zyS4v1ATmeEa{x
z|9XagMmy#L)=KtE9MYU&97gQrY|GdVvcKVQ;uPR~%TdNr$Ucd!n(YtUclI#OGhCKD
zUAzH&nS4%sLVO!}9e8DUWqFx+C3r9Jp5}|-FXms(&m~|bkS!o6Xd$#$SW2`|^twow
zh@|Kl(R8tN@%<79BqgNIOIAv<NqI=WmtH2FCS5M=Ed5U^TY8Sn9oa0oFS0vi9HgbC
zrb%v<6qJ&Xx+Li*sUl$^W-jtbNMC5HpsZk^KoWlzUpOxZPZrk(4r_J^HXhc0%*xEK
zm|U3;F$=Q@vbZvzVr*mh@IU<jfqx<YKQQcP6lJtyc=A8@fAGKOf6o1`{+;l<?APZX
ztUq3T-|>CXxAHF&KmK~>^w#{%hS#rOuX-c$w&~5IS6g0ecoz6{-4l-|`j7P=8r?s0
zx9yI_?b2HlZx!F(f9KoXzxNy-tbMrRQR3sACo`X}e)i&d$;<z*`rcT+<M{CP<NnVg
zU$egn{4o3Z{AbMXdw(?<<};Qt{bt(0{EkJDO^$sR$7`+vo+~_id0z5N;??I{$;-gA
zjO#yV0_P15BTio~2ks2+7hI8C+FTV}P26$3^8D-qeF8j!41!?-xA=Jk&IoK0G#82&
zIw+JPd|Y_4@Mj@*Av?jt0z3jI`8M+w^6un4&#S?^m3up9F8f^;6(;}xH~&og9rioo
z*Uj%WUyHsx|IGf`>y!2;*H14#9{P~={=vI1@A}?Fy%l=n`&#YwiPxTQ``_DsTJ(kS
zoAtLXU!Q%^`ts>>_?MSozJ3Y*(*IfaGwbKepRRnG^jY@nuWyY%`hUj#lK;K=_xs<+
zf9L&n{oV6h{?C;^Q~nnJTloJGg9T$aL-K#U|KI+#|Kt5XfngfsGR9uUS&Zu#?=!w+
z{L6TOiIv5HC6u{^sf3Axc@}dpvmuiRV+=#a|J;8$|8o92Ge|LL{OAAo`p=QyKYs21
z)%0ub&!8W?-zR+i_WAfH!B5#ASA4kep5=YOyVSQDZ%)37eEIVE(`PZyo;>Y(did##
zXZN0~zbt>b=f#)j)z6<k>v+cey!plBS2N%KdDrlc<K6Xl2j4$=Z}P$BqvfZrPXeDf
zKL&m9{lNKw?LGgygKu}gJ@vNo9sdXUPcJ_o`%>`5@w3LKEgvs`H2&20srl2FkG3Co
zeklHs`(eX}=#Pb;_IzgfYWpqsd;O1^pDw@d{VMux@JH!y&A(uV1g1z9CpJ!w*Bqxf
zj&ihf`g8r`n$E+@KTY7Az!3p+!OMb4LYBh4A{Jsj;>*Q1i~SU}6T2z4O{`z^v2cUX
z8o|#3&H^z4wSqf@WQF$$MG3hHEfCr+oGrRfj8Xit*gdfoVxD3NV!q-xC7PvPN`I4?
zDSJcql5B=-xXceJR;hH!rxJ@Kx+GF07EAn)5SI8LDj{-L$VIS;e=*-yzG(hG{NeoG
zya%}#aj)Sv<$liffNMLKH&+wqLJnJw5DqC0HuiE>C6-#|S<IJM<k)jLZ*etoS9AAr
zIdU%Ic*xPosl(OAmCq&5#mwcvb(m`+_avT`yc_rm_|Ncj3UCX|<bTPR%NNYo!gq-8
zE}sTJtH4izEWrf9TLPc>TlqNzW(qtKcpx}MxK|`mq)}v}sET;B#7)VK(o<xG<=4p9
z$v4R>D9li>QBak?Aj>DKB3mc>MfS0rmcn5LHU&?)Tp3a6cqv0E7pX^*b0i*#sfeZs
zZxRd?Xy&)%H{sXkf5qp^=gs?pyPeC2Q-ec{-HYuC>v>i#HWfBqwp2DLwuLMz%yCSu
zjLi($42F!q8NHc~FfL(u`v2Jffd4W7rvBCWd-@OKpZ;I3e_Z&^{{6()J)es{PJ18p
zF8%HIH(YN|yoq_O`SR7XiYH=^jy;(0!1Y1M{a1IT?s(o3zB%v4?i=AZg>P-Wm4938
zj_;kMI~(pe-*vsW?7rQ@(8o;APQIvro$${2W7p@8U#h-#eSP!w@wZFgSN|yd<@UGe
zzdFNIhJ}nPnc|sESvpx`*rhmsb6Rs9<C@C-hI=CSI<7sO^_-eqD%`R>F}$nzy!hYp
z-Qc~<bC~BYZvelYz#@Tz0;dEn3TzN)7kD6WQSh<Q0bzHMLXndq?xNbFDI(><1wzS!
zD+NyT=kTfUI`On~ujIPMaftOElN4j@f49FXe?tG{{z>}n_4D;N+piv9f<JqGI`OgU
z<DC!R-tTzl@pjuAjklg}7r*g(bM_7IyGQT8d_4O3%NN<NsbA_pPyV#(<Ghb;pR_*n
zeLnQ5;gjtrk5BtQm3+?ovi9ruZ}vYX{k;0?!SCijTmIDl5%_cP_od(3f9C#i{_FP7
z<A2rvBmXY`{r1P{&(q)6fAjs3`MdjH`u}bJbpGx7ck}-<hHZ=?Oy8M`nM0TtGifr(
zGyP;NWIV^9%dq1Amj9Lv8yL<qh%?mutNYXWEAPjxZ*AXZeB=K1_)F~P-5)zYOnqPf
z?&#YIZ$sX`dXx2L*6U5Le!UcU`S|&ZXWq|#Jym;l;MuI_YhGM?x#yMA>jSSkU$(#4
z^djr!kC$Sv#ow-f_u`%3yPIzX-@SUb>wWcy=8ry~BtHdw%=?i1{@}aa@6zApzdiZp
z$D2!U)89LP?E3WU)4@;2KE{2}dcW`;@B6>+^*@-tw|aNyZQ$F$H#1*Pe9iIZ#G974
zF7N8zYkpkx>D=ePU;Mvq`|kGh)2|1A*#28GHZy0k>adxyb+I|JKV{#@@rH9QcN?z^
z|6cyZ{Qvl~1;d0-i)<2g7n>?JTZ~KWqezs<F%fOi3nDrqO~R(aH-)`L4~rcWe=nXR
z&LqBEj8805OhV$kWQEjrsYg=hq_#*cl$s{>QR;&9Y#Djk6|(wr&T>~}b!0tdETs#j
zyrn9nrc0fdvXyR<_LrV4<tdpWK3#OJNPx(0VSnLB;YUKXf=~GO@Covr=dI^G!NbLq
z%w5RU#;M9F$7#T6#W{sTo}HELI%^QyJoY-yhg_4ng?SWs9&n5Eyy21MYv)f8U=oPt
z_vMe|H|0;{XAyWL@JMi;&;}tJp|yg{f{X%N_~!Dy<SF6F=6S;t!#k1pFK-K90^cg$
z6yE>5H~D7r?-Ixm@)ThbQxI<#caxBk;*lwmT_Bete?dM^;j%)Jg1thc!dLlpxtlVa
zGX65Xvh4CA3Wf@|<QB@5NwG;diB=2i3x5)NDcB_7$zR0B!FQ3jgEySlpZ68dJ?_g~
z&Ri=wmv9uYtFrB7S<TGP+{kp2(UZ}iv4&BNk(*)GKY_nTe=Gg|`|HLp(cjO0+x_YI
zz42G>uVueDevABG@=NPi?$2l6HNUO=^7~WpN4^h!?{2-8e>MGO(aWZnzAqm?uYG#^
z(b@Zl?}*>Ndh7Tt>svZEGp?sy<+_r1`TnKZm%d-RclqNL%WKWom)v-LGwt^3JBRN|
z-Cy^h^wG>GYR~Vyw0gt$Zq@s1AO3wf`629M`zNU{2H%{2y!pA|cjVuwe;fY4VdP?2
z$I8Y&lYJe#6~{FWHqK?7%eY=}#d7W8tl?DV+QjX~Ys6Q;_lEZ>&rNO@ZZYni+zPzL
zd`^5nc|Y-%^3CFB7i1E;D-<DeT+~p^Qmj&Jf>^Csm6(7Si|B3PcS6QOm4Z0}Qv92E
ztGQJ;7O@;=ocllS-<7`}e|P`U{&V?P_mAIS7k*a#wCUro4=wN4zUzCp=WWKDt*>Xi
zE`BZhI`UQi%chsCukGI!y%+j;^`qz~vrlh77JXdwanGkKp9{Vm|8nOG@7GUXb-sK2
zX#Bb7SH*9WKYD+6{Wbij^?$<uHUAd=z4>SV@1?)gf64u-{3Y?b@Av-SnZMJ1&H2gr
z>-?|g-?RUS{B!?rz_5&AD?>7)J+lF80-FGP9J>?yX||)R)+{fX?3gkb@BH`v*YtPQ
z-^73a{$2QY>aW_LPd}%AU-Q-XOYvu+&oZAby)Sty`9|RN!<Wo2+g_Bu;C{LF<&&4n
zuXeop`D*<uhF3N(7ru~tvH!W@3$d3$Fa2L`d+G7A`MKt^OHUi0);!IBHt%`Qi)AnS
zUu}JL>DAKL&)-PCyZ`R}yOZyv-aEXX|4!;%_PaIjzP+t^Bl&vXD~s1mZ#>^Hy!rB~
z@TK<)zUT5!WuAyX6?iuL>E|aWpPYHp{*?2%#S5{QF|W?Ne*NahTe0`1A7VdteLD16
z^Xr>$aX;h!u>HTx;K4MHIhkb%OFqkAW;2$xtjX+l9Pc^gxfr?sa82iG;8Nn=%#+UN
z$3KJr7k|3IEP)OI4FLv$27$){9s*POxdj>oRfPRSx<z<JtcBTyo(s$r_$F{b@Vn4E
zVMmeoBEDkD;u;dck`+>6(vxIF<pSkv6`m{pR*F?NQ(mUTrF2>Gk>W?ii;4+~TNES}
zY~&Zqn#(+rekQ#^`hv8u%r<Fx>A6w`l7<rI5|bnjh&zg%7LgI|5lj;>;IH7b=5H4G
zC-7Q8PEc7$Q>04li+G7dp@hD~W$_&GLt^z}_Tr2Z(c<?-_lU5FI0{b`k`ekXSR)iK
zyj!?f*g=?6_>kaoeha>Byy3h-yfS<{`Fi=T^2rHg3+&?4=3UIo$A6svDSxxTZ9zj}
z0g>k-{i1V4t;94Wyd=XV_KKgB(2?|!kd-KqxFmi_?478dh?NkZkiD>!NTf)*$X($=
zp%ejgzC%1oJUKj!Jny)~IUleouv)TQWB$!NgN2=yo%ILHDV7M9AI$m8=}gBMotaKD
z9c4Pnw2^5iqcel?|Neg)|2q7+{j2)dt6$H4x%}e#mHw;z*Rh{xe<=L8{C)BFmhW-j
z6~8NdkN*DdTlZI=FLOWd{uK7{&HJD4PQBB3|MA_lx3^zkfBECtna8OQ&)v7V|KtAZ
z2Vd{=-1oe{|Ne^m&+hj=`1N4v!~Kt19;-i@_B7-9ju$syvcG=&ddr(XZ!f&x@^Rwl
zH(y`;SpM7Rui-!L|FR5bjH*okm~vUBuz7K;<5<ZtmvaJl6>lnE7GDA12fjaik9qre
zgn8t7ba`@l6nMYze&^E@_$zQzKuI8*e+%DDUK8FhUMoH^el7mld_sKcd^h+e^Ggaa
z3T6ux3SScr7ZDI`6<sOXD7s!GU$|Ikk)V%29$ypBH7*a%gY2JJOPDJeSpT*Bark}f
zr^-*KpQ1k>d>8nZ_=V^5(~s#N;@<6f6Z9tK&G*-4ukXK-c^&uq>8r;tt6w_3TJzfK
z?Y?&#-amf-?ETsI*6%ak@w|(9ckErmyTG?@Z*tyDeEZ<t*Y`U=1b=+_QS#Hwk18Lt
zKXQGF{*?Kt|I@=y>7T!Uj{cJO#rKQwm$jcyd~*Bv;{BHQ+djyBO8PwO%bl;+zKMO`
z_-*yK=ik|WasTQ2d;cE;!zzY0hBf~W{nPx{|F`E){_hjNX8vmW<^Su`&v!poe(U(W
z_Cxl&+Bf#EO<p&?e)?+B%Y83qyg2+~^$XV*3@?ISJbV%VQuEc6*J^Jq-?hAxd)N4u
z@9ow%xo`a6l)bt1Cj9Nvw;pdD-iW;}el_P6_iLBepI;w*d+WW<N3~BYKkxgx_Ivja
z@gH{IAAjroR{u5d%dO86Uvj@_eKGqS|0(g)iBA_l`F$$+xbOq#2bcGv@5JA-y}kVQ
z$h*7mcD-Hy=I@(>Z#TV-e=G5B?fc}9`#;V4T>ORk+l%k4KXZPX{@U`p<8Q_PDn@hW
z=`31opV)75xN$6Jx8aE3{LeXuYbEyso<F<~`E2<&@~a9S5V|I`RghgURB*PSq@a_4
zw7^jTd%;@*Klsb}Kk}OinhVtmO&6*c>K2p};N>^w+reALcZ#1&-~qpd;8x)T(J!Jc
zVq3)xB%`I?OBG1B$r#I>mw%wJQ}K(Ey-K-Cwz7hfyrP-{zx)|l0htey+7bccTH@2i
z6hvPMpAa?@))h?Q59R;HuPX3~e<FVd|9}430!9LJ`4t6t1+4`y2-FGe5|}DDONdc;
zoe;Z_i(sZeB)=ZtV%`Y8E`9-lT>dY-b9mx-9`R`MR`WjRQxv={R3%&_+#$>)Vj;3Y
z_?1ww&?CWPf?`79Lh?da1=kDu3Tg;+^40TR;+@8~o^K1^W&Y)Y7Q*o&MWXk_&PW(a
zU6O2+_$96=@krcQ{DoMfc$LI1iF@MT#4d@kihmHx6LS&WB5Wu0T3{c)Apbu;JN}9M
zHvAR5^4upl-8t(xCb7q|b8)D0KIJ^hS;qO2a}(E9E_<#MoR2x3x#YP*ITvx1arkj;
zV_(W9#u~xw&*aV2&t%WEknsW|19LlbERzo7B*sc6ZKfLxGykjqpYi|X|K5KG|J?l@
z`TO-x<sSmymwk)-_T?+<*Qn2_A6LBBe7EQg+v`&=g<guhP<b}*anwVD2YwIcKKOK>
z<-X$m>GuWi%iJ@%H|zfDhxZ?Ed#dw%(~F8%dT-j^&U<&@y~#(h&+1=yeari<`s2;d
z2fyR~y#Af`XT@Lpe?R|9|1<c%m7$x_mT?oq3<hn6qyO~&=Kb0IXU<>Ge@*{9|En+_
zVB%w*&Gepe8-wWodw=8pru;qmcjmv2|7ZS-{h#o!`+p(hd#1O{46Igc+3dL-?3^8(
zleo5Wf8<u@KEo-_p~05H@`I_BQI^5!|E7ONe=C1I`=0d8@#~t;Cq7Dl@OoeNUhTd0
z+t$}yZ`|K5c+>TY`(?t5Y0s^n&3ba<@!=<l&n%yJJU4%#{c_*SXD=_jocuEJ<;0g(
zuRgqre0}rvqBpnRoPMqFiuL7>7q?#CeHHY2<?AhPzP$B+r~YomyBF^*K8k&s{mK4w
z)|abaS-;=>UjF0J5AL5Ye=Pms`-A_-{vQi}RsC`JtNt&7!G$TANrs7mxs7ELt23Je
z8xPwLR!3GpW(UT4hI)qo|JVPE`TOxt%->0WKm1|)6Z*U5m&4CJ-(|j4d}024`Qzaa
z#qYnpje9HmZr{5nZ%f|XdCmAH_08`$``=D{*Y-Z(L+pps?|t9%zkm4d%DeOLD&DPn
z%k!4??d3P?UoU<2?$wdkdtW=e_I!Qiwc8t`H#=U7ywQ4B@<I6Hgb(o_Tt9TZ4|vD-
zw*1ZKH#^_%f1CKW>g|KK4DS}d-SH;&wZ^ORmu4?Nzu5Z1`sKG*H{P_qO?^B0t@OK=
zcgNqczMuJC^rPu#{jXf#U49n-=KX8@&+)$*gFoX{rXCh<wsdwGP6_S~9wFXDUI{)s
z{^<g>f^!9b33dvzi9Qp_5HS~>D0WWlvS^fufUu{~M!}~7(*z0y_X{NnYY8t9DiS&?
zq$?~Wd_-u9P^i!?K}W$p0S$pB0XxCPg1ZDi3T_k96geV#T5PKLM~M_EQE5TxLsF$u
z$x?Pw!IJ01cZem5DT_@My(;oa_^42}pryclK1bdlp4U9bc~f}j@EqlF<z?cH;4$Pm
z#q)r-lJ7mQBCihbeBLtNeLRjln|WCHr1^dMr}4iOs1!02HWd~Y{x5V-Xq8Z>&@!R7
zLbbxpBEF)6qD3N8MC?TSMOTad7WEa|C{`fuFY!ZSx#Tpdd}(%>N|{qKA7w;kBV?mx
zugbW|M9Zv^nI@AevsT7e_Jk~_oP!*@{BeafN{f^eRP@z?HPkd4G?LZJ)y}E@Q8}P|
zM(LW8k@90DW~B_pSj9rcR|*yKOJzSu+euBAkQcuzY9d-BdO(y>v{YDE$V;$M-~xX*
ze>lIAK%k(wkdY9B&_+RD!DRw{{7!rUyjOU<c>=jrx#T&`I7HZYuzp~k&RF`t@1OoZ
z*1xxZ$N#zYxAkBAzl?te{~iDL<*&-0A3sfgT>R$yb;sx0PcJ`i{K)*V^}WTr?{9y+
z6@FX!`uWR_7i!O!J!N^){wVKZ&4Y;hr|*8eoqp@vO}AU+w|Z{r+~&G7_0GLJ26r3o
z-oG1r@7O*0`}gkudcggN@v+Pkm#1RS&OiI{-1Q~%tJ+s{Uu(QIdbjOe<a^x@*FMO6
zy!m1Bhrb_sKDvIa_@MM5@x$B?`X5-{OS}tw_vYQRcW2-AzrXe2!bgcuJ3g9zWctYX
z@#TkyA0B^j`PlkN{R_|6j;}kvrTm!qQ|(vNuVcTIe{cC6_s8OI-oMrVn;C8~EMpL2
zi22|DukbI=pZUKke=hrR>HCCluf8~b{_;`!qxOdx?_A!dzx98Y@b1~$h<DT8dwn?Y
zKJML*H`iZRy;}T2{Q1UbMbDqTD19~Yb@&_Sx1ZjgeJk;{@r~sh|2OmA+<v3{_U+qy
z?~LESeOLBQ{+-;r(057i=D$1te%eQ+&wgLRzJ`CR`QG<~?I+hy^`8+xtAAeq+4jr*
zx8R?je-8fr@i+hPoj*2z9sV8t_xRtL|A~x~m_(VInYCFyu*9<(u|8lq!qUKUpIL`_
zIa3|eL8kLeKN;^b*f4DVzw=+j->bh%ekK2m{-N{z``2?{c7JC5ocu}o<A(Q~@BhEM
z`)>ZbMen}8D}8_Y{rmT(AAWt<|Iy%6&!;n=u6+9cDed#}&&8jwefslJ|6}2Y$oGQp
zeBOG$ZGXGy?ZUV3-(Gun=e_v{wGXpCG=Aj#bl~H=4~!o)-dn!=_vXrLsn>^Jt$TI$
z)!kQzUtNCX_4@GZL$3v1KX^I$Md)+sXJSvcJP~+$>*>>H94~ZV`oA)G-Sm3a>wmA;
zznT5^^Sjj_GCm1>Y5ZFJ&HMYRABn%1|E&9)^8X5h64Od%f7VHC#_T=pKiKU#j<BC$
zD`xw~*3T}@A<bFGHIJK}=R9``_iwHet{0q#In%hdaX0h&^3M|J7c3Lv5WXQKCDbOE
zAlN2&LQq%ep3rq+UeO6+{1Q@<&n3@E&5-^ftt(S6BPe@Awp?zd+#$L5a{J_j<T_;&
zWu;^n$t25E$kfZY%e<7{Abmpmf%InSInvjqRb+}}SY)N;rpe1H<}39nx2XuJ-d1_2
z+@@5Z_)_7t!X*V6#aD_9%GZ>;RK!$cRKrvqRby3msfwyusRgOksI60bq{gkTu5PC8
zub!k{sNSRgSUp!mNOP~|56u?MR~oZ4iZo(0$~5L`{L)yfX|1KFbzZYhQ&Y1`<Bj?x
zb$j(4YSC)3Y9VR|RC!eeRL`p>sl}>UtLdwKSJhHIsqCl3r?^`FglvPfpX6on_hKJJ
zcZpmQ_7FZO^g?jIKn}km-$5QBZXM1p_FT4CtVdX5S(90cnRS`}F*z}1GJa#&&#-`D
zA;UoidB!)4+nFvinK6YhN;8=MWBa4}tM<qKZ>e8*e6jxG`<da>%nv5-PrfaDGyC<H
z*D-ItzR7yK^X>e%U*8;ht@O&|Mc}jkCp#bAe9(4Z@cx^7Q}1!!Yq)#tPW|njH_u)F
zch&XE^GlAG%r9A8(z(=i$^Y_=%bzcET+zIeeC5Oyx2p@U{=8~(&F)&=HIeJd*MqMo
zU%zoZ^2UxEqBl?9)VRI#&bPb2?ioDDdKmeL`*GvrdyiY6)IOd4Ozg##m&&hSyx#bx
z>#fu~rFYZbzIxN}Cg6?To5(k(-t2kH`kw7W#fO6*9)Fnn!Qq4P2gwhTAI^PP{qgn3
zdmmXo1%2ZG9P?%OSF`ViKNWxb{9*Vz<L|}4t$*47&i%9KkM7^Qe+&Oz`S<qUx_@T>
zI{&WuGv{~2FZG`ZKem5g`EAA5w6En~^}ZhZa_fut*Nb0$zpeas>)Xw5Ip4az`hL0g
zDfZ*#_jd1ozsY|y=S|ex>~~e~<3GIq@Zv+`2cZx5-hX``_TkNku8$I*R(-1YT=_-q
ztMS*LUv7Rm{$<mb6<^rC@_sA%F8tHw_l>_-{;M(OGfiOTV)@9tpZPs=Da&e>6c#&{
zSQZwR0A_7wU*^|LTbRO_{aL26X0Y3F?%@*S;pYwFy~WeZ{eg2jM-=;S)(<SbEZbNj
zSWmIKvVLSPVA{*D_20ZdOMjjG5&qr&+nz5>pLc!S`r*y{d+#5;pZvb~ec$_t_l)m%
zy_@px@w<xmf8Q_pkoIxY$K;QDKd^nsdC&O1>Rs2{32%P94tc%!71yixFRGq@dB*d+
z;Mu{atk3Q~Tm0Pr#ex?dFA`s9zgYGB^Rtj=&!5hHYW(!t6PBl&pE^F9{Osp5x97>v
z=RUvjT<*oT7v(QkzYKW!^u^*A2VR7}WPjE3%Ki0>H`CrOe4F)l&f7<CJ>I=}x8(iy
z4>g~{zC8UJ{e9h!^FP1*V)?`N*YV$^|7RIun4(yW*ygdn<uK%8;BM#k;pykK;(sjg
zTkwLAkcfk5p6C~m7Lizy5|N1_wIT;aM8zT{{z&D?ddVMCh*Dastf9iBvQSx8d9ISb
zl8};`l8v&Ss=4|Pjd(3vZDyUnI!kmH>c;7s>gMSh={e{}8-yDEGxRdDFgj=GVOVS6
zuOFy)P`60eTUShXrp|5c+gd+0murS;vTOd<6wx`V+og9%Z-Snnp0BQq&JC^Gn$ns*
z8rmA18hRSN8Zw&uG{0$1))dps(r{A`QqxllRC7^tSDUQHtnR2Dpq{AyUG14Fhiak9
z0_C?#-AblPl}hf)8<Y<ze^Gv+T%jzbtft(eyiGYod8X2B#mx%t^7G}o<Q~aBlBtq$
zlQ}BAR9abPi%g`fu$+h-qwF1NC+R9FH^~r*6XJ>Dx5ZjT?+OPBhYD{KS|L~<@SATA
zuN+S+moDcKc0sl+ED6lB8T}X%7?c?|G9G4(VZ6xT&Jgzh?%$R_$9~WKz5Ms$-#dR_
z`n~D*qTklP_x-%`UEtf3&vQR=y?^?8{>ul?H$Sa=a_mv={nq=wkF}pxKi~6q`WKJi
zo&R<I{QNfY`~IKOzmNU7@oVNcp)cC+|Gr>;arb%S6Zgjt9_ioHzj^Z7tIO#Za?i${
zdVOriq3nH<dzyB$?>@O}?XI|85xb0b&Dg24lVRuI9dmY0-!*sl+dYQ+*!H*V7eBE6
zp!5;BV{ylqop3p|`gFvZ*t31-%`e<OTX`n)T=a!o7tUVFy}sv8=EE0H@4R~TKIe<{
zkNn?D|D9u)!dS@U!cxO#!a0q{pMSDIu|O$*41c_UkiZ_kUwnu8!}*!{b@-?7E#sBr
z)#LfX)x~v`E0t>^XAI|G&JJ!>zM}$i!ogy5BqvF`%6^wim5-KRBR^GcskE;IljtMC
zc>dKqD><Gpr~Fs?o&P=H>-;ZfUs%2z`NI6o_uJjC%f8wF@ci-a`;DJ9fAaqt{QdD~
z-(S`Lum1)7-TJ5J&#&K4ew+W^|1;-T@1K`{rv7R9EAjt3Lorh((<a6qMpmYyOpZ)K
z4Ez3F{dWnpR+G_`=@)Ysy9?JlZVA3={3rRJ^6%!?<p0ap#J83wm(!i~DdYS9Tz@@&
z{QkV_gW_B6S4&?QKA-%g^O3@%(+}rAh`k?wFX8UVJ16cG-p#+S@u>arna9tbJbt?E
zN&Tbw4?jP=`;g;d{DVvPtL~k><A3MdojG@p-0ix%>yG8^DL1)qe!Hb}|LS9*7iZpR
zewy%o)}Q4J6PUlT&Sn3`Uc+9+#>BdiIh5%mgUkOte<Xe>{LK7W{7dk!FvCyAGtBC&
zg)Gs`NzA{Qb}&9+Sj-^IXw0O<EXi_{)tP-MyA8(=j<+1UII20$vB|K!VKijW`B(Sb
z<cG&sjW0jG-2PVm!}cf7FM;1Wzk7dG|Fry3`t#zSb$@sM@%zj6H~7!I|4vMw{!jit
zk4ci5g=Hc~2xl~FFY8mz=bYiJt}MzN$(*X}dYra=y}YwIS~wc`<HTA-?+A*Esmg>(
zvP)c7;8sgg@>9;z($)3TUZcHE|Fp4$DT~Qd!wh|Qz1_NJv{q`QX;0L>r!`S?v#x{T
zEd6gfwYs8u89E%go`&a)gN%8MFB*rMt}wf75@b-T6Rz<?@uxJeWUIt$@in66!n=i(
zh1~e=aq@BL^4{fL&cn!C&AXe&iT4uUPQE`p-#F9R?yxT9@aMASdd=<1pDNfTI8RVh
zSWR@QXpV@qu({9)!Oud$V$71;B;JVclVFhAEA1m!EpIK)r7&AjN=ZsdPdP@_T3t+o
zTeDRwO#7ntJ{=BSK3y?AH@&$!kvcPU7i#}dTcg6O@>FrHLXi@SO0-J4nxEz!O+`&1
z&9CYas_M$;6wb@$%AS<JtH7=(tNc-Asq#OC!*U;`tt8HgI1BX(Oy;-XtKeSEzM6SA
z!;Js){%!oj_9y)h^KZeQbw7T8+wj@+<HQgDKCJ)v{A1<Eg&)qnfAn7b!}fR5ZxmjZ
zKka*HchB}V!_B`pmfT9d-Eb@VrtXbT*95Pxy-|3J?~dBtb$60)Z@)F~*0!4uuf4iF
z?ec;v%vZcG{kmv*MdEtVb%|@ouQpsQxzc<k^=ir0l~=egt6Z!;|Mtw?Q~qaI&xf3!
zanASLoU=k_o}5TJGUed&1J(z452PKOe|Y<`=u?4bZ=JWjc;|fC+1xWSXKtUKa3<os
z{gvh$g16q@;=QAFXX+h|dvouveQ5JU?kU5wu$K$pB)x5YlmA-#wZof9@0NYc{37;!
z^)Ksx?-|}RaWXGt+{w6^Ns;L(gC7GggCv6{Lnp%_#*IwJm<}+NGc95I!)(Fwi%E#t
zg=GoL2bP<xzu0OxK5`0jtMS<IOyN@D$YQ<5#LM{bf93yE|Ly)e{j2_0^UvgO`Jb*o
zet#tXJoxSWTkDtLk0W1`zbyWg^&$R!=eyds*WbpzD}L+r_UJp-4^<!TeyIC+@>9|m
zm9N2H*L-dKI{C}L&+=aqznuSa;_JlktiO_e+x<TB`{bXPKT3aG|8o7i@Gq00f^i1Z
zPnLP?8C+?+#{8}P`urCB;sTHOQ~4I~8S(S*e&CwT#m)JeO^RKW!;qtuqnvXwhc)XX
zMhgc2e>T78{Sf``^DX#W$v5tAlfGU3A@^&-54mseJ}>!@@y6@L+$TF8p1GHI`|*wa
zR|_tNUC_Lwb1D1c@rzF`e!k>=<=3Up7n?5syuy3U;d<cB&U-r^8$VTie()v#OQ#oo
zFLpk4d&K)l{Bh2c6;F3Qo$>73^B2$5pGG|uc&7Jk(({dP_I`Hy-twLIr_isE@6BI6
ze^L4V@8^;~YyMXKb7p+Sa)8~KOM=IZXEoPaj&&UOxh`|Ja!=u95MbxO%G<(!QqWB(
zS0IT`mw&AQr@(anBz}EfAFi+L4(wMrJ$ROJw{m~vnaG>NThFtGyM^Z#_aBaA&ez<A
z-0wM;aO?4{7bp@|7L5|SEHPcWLT0J-Unw`KW{I=n$0g-tT4k@urO3C+h0AV|eW>tD
zwM?r;*IK_+|BL=(!&Sy_Otj6`nR*+C8r?U3Y`V&PvZbx{1sgT{M8{Q*(;e2@>)XAw
z?zP%#S!mH}Az}5yYKFC>?MGW_+jmyk<{8E%2B!LU`qldH^h$I)v=g;vYQ<?Y>UQaS
z8qCtWuJc1HM}33x4#lksYVx0CxTNPv>PdKs`-;C2V-mF&TF&pn+s^%yOP%XC`wdnr
z=8yj_{dWF6>6g<l!$0r-8vm>L=lxIdZ|oocKSh5k{&4+q{JrH@%I}oFFa8<+=lsv`
zPw$WK?}pz@f8zh7{rUUn-Cu^kmwsLO$?>b_*Y{rwf1mi9#t_Xoo$(ywZ3eUdYyX!1
zarhJd$L-IjKfM2@{cmB?V@+gJVDDhdVCiGhXSnlc*RQibKmOqO$^7g5FT>w%zukYw
z|9bv?%2$&wS3Z6I`1a%ekM}-$eERV5^T)`~3%>mM!u56Hm&(uTpK3qWe`@~X@YU{%
z)aO$lW#6BD{q=?3vn7xJKje7m`>^P-_A|a0WiPfryYP6yL$e32_s#Afy1(wh>xcS}
zS)M+9{`Hm3Ti17X?{weQz1#K9<lVtHi(lKk>3r+(PWQdV2eS{0-tT^2|Do=q_b0|r
z?>?;lp!t#KWA?`*pI?9d@m2Dh+xHvaY`(1hIQ_lV+sUuLzgqrE{I%Gd@;BGtbiFNp
z8}qjP?f%z3FUwyFzW(?+=}qYy<~I{wr@x;0di(2ZuU5Rg{Nm|zsplV`IXu7q{K50i
z=Zr5_yx9F>+l!DFO3xQPz5O`oQSifs_bcxv+!4EzbEoC*+<PYXf8DFOTXuWmO|2W>
zuP5DLy*YZO%jlUdqi4E&8a>lx^h}r0GhIf{bQwL<W%Nv!(KB5}&vaoJJ=10MOqbC!
zT}IDz0iWqII`TF;@-}2f-rkRnybXzwH<8;n?utGTdZhID^rMPLx{qc)RC#Ff;K==7
z4@w@cesJLdA$ggH2ObnW)Ol?Fl>LRo>*BZR?|!|V@^;$Wk8dpA1io4ShWqV;x8?7u
z-Y@yE=~LU+&hIHdQhr?izW@8m?-##K`}Xqdrq3y#wZ0tvc;TJ-yXo)FeOUN;-sh8_
zZNGi`ru1#rx9IQ1KWu&<{>RT`&GMV&9P2VRBX&{Fsod(k_56`Srb4&)U+^*WJMit~
zeZsqjC!Om#X8?B$?*smJK^?)_{1UvsIS;TeX8X;0khPpOoh6A$pD~zuJDVc=Y}VB*
zY%CL*=P^kzu46P|_F;O@_?_98MU#0Eb1(}x%QEJFOesuP8Rsx){xAGH`*-#4-@o_$
z=KIIT+{bFedXe=o+eTJ5=2%8whLryo{{{WeWBA3ef<c6#<p0EfKK~N`7c(d^`Y;JF
zo%uiSU-CcgzwiDW{X6rI!7t7qAzxL$7=3&5W6AGN|E4i~```4x{=e)0=Kn|jxBWL~
zn8|pH@fu?@vliP14j=BDyw~`p1g8s55}YF7&v%7~fp;g57td1OaL|bg0=oom@|W`^
z@Sfnw<oUsE!Sk0VoY#SmQQ)CKwV;;pWDy-v3$eB0t0goguStHDa+2O6wMS~c)KbaA
zl3%6PNbi-IDrc-vr}#~=QE{50rc$0#t@32mnd&{7`?Y+v)@V-ET&0n&Zlm6&o~o{?
z&aO65xkK@yf|){ve4_jm`A+!``K|J2<%JZO6^|(%QdlA<EZZf0TGCnakR+?r3CZ6Q
zWfDA+lO;Dv&XQu2*(keDzFDDP-d8S5=Dwt@#1!#FaVZH=$xz8B5~szj#8!w%3a=5Y
z5{Tok;XBC7$D7Ne%)OUGgKY-$6^0N09{p2bIK_CFc?W9_+Z5JmEIrJzjN$)J{VV@h
z^N*cj6C*p5F4J13`Am&WQOtZShnSBs=dskV__3^J*~a3|qRJw}qQtU-rJq%o&75sA
zTLb$mwsR~=%sEWujG|1r%sMQ6tXJ4Bu{~qk!dAfgj^!(h0gEro8rD3vk8D5L%Q>#I
zuV(we`iu1;Yb&cg>kgJk7B`lD=HE>D%vQ{Q7&RCR89e_#{P*df+P^D*CjOrN)AoD+
z*Vmtqe~kN}`Tp9Q@>i>0L_YI>^5_xs1MWN7H}_uOb5-R^@MXnIi_V*znRUYK=!rvX
z557APabW#{9S1cJdmhy|&U7O4M9GO;$61cI9$S7)^|<|U;o}pJ?>h0|WWlNWQyHfe
zPKBMaIqi5_<J8KND^4n(Y&-GiIMazKC$F8ZI{W6F#Rd0^A(wVvI(y0g@@U>1fq7Hl
zvcshVm!4eSe5LZ*r5jnduH9C;D|T<o{jU!tpX5FBd~x?h)yu$F_g}t!vHpeEi?`3G
zyx8&5|Mk?j>hJ%)U;DA*bK{r&UoU_2{2uTl_Git{tv~ntO#6}Wjrohf$6fDy-&()X
zcys&Bk~itE#a^+ztb8%y#f=wnFDJcXd&Bznz}uPcUcT#pH{spYcU#}heW&m~;e+a@
z^3Mrh)W50zVEsAiXWlQ9-;@5V`e(~v$#{hEE`!4V-hU4NfBjEoIL5%wSji~D^n%Hp
z<sNGU`vwkWE*b6_+)X^|c%%8&^2PD55D*lS7XB){SY*4%J&~=VvSM|je4;6$D@CP5
z)kSs*Gl*;!IW2lz%u9TYIKPCqM508PM3=-{iDrrI;^E?Z#6-oC#U6{fiKk1Plzbp%
zEu$>!B%3IENM?dmzqqtWw16A$W3H<lsvOCj$=v$9Gx=)y!};y_HuAjU3gC3)@L-Q-
zyT$6vI)_DzwVBP7-HqLcJ&FA_`)&?V&fgr?oN`<|+@0J{xTo`U@NVLJ&A(ZIQP5M+
zQ?N?Fh@YSDG>-~*Gp7p2Pc{cO4z_8m^(^MhhnaerMOaFi%b9nx)UYP7YO$VTnZUA^
z#g3(eS&{i7lL&Jf%VSm<_Gpe`P8lvKu3*kY4rUH3j+q?aI9fPex$3xtxb|?CasK95
z$f3c($X>?!mpO<@ok8}W!e6Vu%l>Np`}Obne|Ltg|NH;y{K@;}@#FlrzHiUIE&P7(
zN8_*LKX?DC|GWEF<lpCiPySD4@MLsjQeZA%c4ro2)?$ulE@M8-e4b?i+guI@uBqIs
YdC~^1$mjje8_dVW$2yHAw}+P*08^I9cK`qY

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7501f79cb067da108020579ed654349c7933d22f b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7501f79cb067da108020579ed654349c7933d22f
new file mode 100644
index 0000000000000000000000000000000000000000..3be6a61cbab76ee7142caae58d99695ac573a46d
GIT binary patch
literal 54715
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#Y5#`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O+Vxj#;S
za{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W%MH($Jqvg`;W7W?d5;;N
z#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQe5UfE=4JaU&)3^uuX}Cv
zI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()m0t^gul~dSck3VbKQezT
z{&fBk{VV$K{QujGj?5>SWm%T6__A(f^<>pzdCQc*^n>XOvj@v}mN@ovoR(Z=oYy(R
zI9xc6aFlZ>vn#SqU_H)az#_o%ky(~y2Fqj?3l=?=f6PtH7R<MqSeVKf4>J}ron$Ix
zddPT`v4l~WVc)-l|IGhC{Kx#y>Tk>+o!?zQtG-|R8uvBfYt&bcZvo#+esukm|Ml>P
z+qcJ`cYXAEAMm#DjqRI;H;doSeSiMr{m->uwZCot+WDpL^X$+0U)a9B|FZ1MtS<|`
zn0&eUdDoZxZyUcK`4RqW<8SjnHGdxdk^1}iPud@gKVSaD{#*I~2g5_ALoCNx)!C}p
zPO*JqTgMj3%D^nf=*>{^-{T+uzdiq+{G0J_{XdugeheLqY)svZcNm-)a{t%<Tle?w
zpWnZ;e*6D!`+fKKyWdxScmGcM75n4FH}!98zE1e!_36qx?$<M(uYEG-(VmA>9_BtY
ze#rB%;X%NC`FqZHE$-a7_2A~an>@EK-C1;x;Q{}{#7Di4XFQg8Jmc}4Cug4YKXH1x
z_nE*8o0opCj=%c+s{Hl;*L`p7-);S%`pNdw<&P2{g+4gGZ+(~gw&Bf=Hz(ivyf6JI
z^u_R7?stnH$A3KdarlSzPxoJXzwiIv@~7l4-#@qiM;NEDRIn{#Kh4q1S;LvmnZn7>
zd6VM}#}STm9G^Kuxc75w@bvJ?@W0_-E3j5@nNWf7eqlA?JfRk$>q2ruwgQEGvw6*V
zxANTLN#`>W*e<{=AjN-#cLjF?X9R~UM=M7u$90ZnT$;Rl`KI#U5_l)bB&00lEj&v^
zQPf#9MKo4aO7x=02a!HePqDLN9^!|@jU@UcR!a0r%#tvbh!^`Kyiq7pNK!~bC_!k4
zP?Yd#;oCx{f-M5x0)qTYcy)Qd^9b|o<Feqq$S%YFl5HQGDtk7^Mot5+O`M4w``J`k
z7cr+YWioa${QNJ@Foi*!@h{^yCU53x%nMo4Syfn>SSPXGVx7e5$g05F$9j^jlEZ@Q
zI=3kACEiHBYkV5~F8oUTANelw-Q@G(za}6dyjG+^bg$SAaZ~XI(d8mDM50BQMQ#h9
z5&kJGE>bDHTxhFMp|HM)hG>@<t3;7xjZ~DBwd8B@0<je$^1|6dIzr+?YlNyrZit<h
zV30Z~^+n1=>ZpW<c(16k$U~uW!N>dy_*U@F<C(yHhpU`Ri)#(%L5>FY9JW2I?5x+A
zotT6e*D)+)NM|r%&}GnOFko<JFksMR5Mb<Ka%TR-WXGh!82jJ%Z|v`jKLvl@{ju@K
zwjV)1kN@=isr_Tix3VvdANRjo{^r&z?w2#3e|z@jnc(x^&sIK@f7bDI*ONt$86USk
zj(bw_#NtWSlbcUIKGA*p^y&R)ub+Q>apPs<tGBOQUU$8&e69Uj>2=ua_pg_{75{MO
zqu^($uVvqF{y6bd?04Cpi+?Tu^E31_Y+}5?^qEPKxr4=yEs9-+V=)I0XB+2wPBBhl
z4mS3$tahxwSnjhtVF_W?Wy@qg&M}4aJLe)U9qx-<He7LBXSl9%#dFnj&f>7<kl^^w
z-oSB`Q;_=`w-e6-Zf~wf97XK6SzTF{GPy9GXL!oc!*KV%=l`C6lm0II6a0JePtKni
zKkxjI`9Aq;+ZVUbS3kObIPuQ^o#^}D@7X>wezyG@_bvCE?6>r<Pd{(?81X*h?ef<J
zuL@pGczyWI&$o---~F)R!`JsS-dDcwe1GJ9+4~FcF1`zSzwN!#hwzWrKh6JQ@{Q+v
z-}l-d^*^nDef@Rd_k=&||DO4`{J$?l2E&p6>;ErixXob5u$Cc^Nt}5<Qvp*U(@LiE
zOfpQej3$g4Oy`*Vnb}ziS*Ef0vpi?YW3c{r>d)ri4}Nw3di~4t&;7s8|5^VR|8Mbc
z_aCX>Hb3pZ@A*3Ki`Hkek4*3Hyb*c5;HArpkmnZ9Q=e--?|ruA>9i+-kC`8Bd{BNr
z;9lq5hj(r6+dNqR;O)bgkK>=}JY#(R?)jz{A6{6zP<g)p*@kC2&-Oii^R)C?<8y_V
zvtO~iad>zAL+vM{&uyQxKJ$Hk^r`#Pua8eY&i%CRbJv&qUqrrs_{#m=^!u)FhrjuJ
zZ}`sko#&hCSBtL>-zvXf|8eA()Zg&`B8)E?9hn|7U0|NYBFdV=`i(V{J&yAOmov{6
z-e-Id`7a4@3N9DO5)2nE7YP(m7D*DBCZZzJEwoVJHvd`vDE{|+TKsGHvji9fKk*mx
zPT@YpwV87vM=6ICXAf5|k20SWf0DotL22O%VQb<0LRP{bg<lB!2r~!^2qz1xiJTWP
z73CLW6@M!xAvRT1S2SN_zA%sQGNCXbRv`|dBB2JM--7%?vxQ`Zt_XMu1PDCg|H;?H
zJCi$@YaOQ_hcBBNt32y2*5zzA?1$JhIsLf#xxa8&vCFaTVKHQW$jHYS#2C(`!+e%`
z8cP_fBU>>0Y<4bo2KEQ+^Ei%j%;8AmxWU24#m>>g-Nf^a*Nk6KV4}ccfs+Cbg5`oj
zf|&x$0#yQ=1ZN066G{{I6>%075t}6@EWS^|L%K$WL*|0Cn#?Qdzfwmf-$?XJ`bwL~
zILQdeI?08|RmvL3lt?K^Zj{iIh!&R??+}-lI4z+i$tbBLc~(+DYL29c<V*=ii4WqJ
z#O8}M3aJX(3asHz;J?Kez{kQT!7IZp!?~S3kBx!#K65&A53?;x2x~gqN;YdYeKtQf
zHrDq{HcWe%ikP$+xBYkc_x;b#-xGg+{<iq*vagw6cYQJVa^v&;&+k8<`jq>j@}0+9
z_BRV(RlnT#;>wGLmz!Q1zuf&I@WuM)`<^{~V(>`if!BTCdwcIR-!{0t_V%_r^Y41x
zn|OcI!`jELPgXyPecJx)`E#C^4`1$l_2KpPx4Iu5eSG<;=gaqRaz9V}s`}IVPnvNb
z^E;L;EN@uOv9hqWu|~1JWvyVZ=d9w&;9}-B=TYN1$feDBjC~0k8{1;mC9M0|mU3ip
z9pIkD^OI)-?-#x+{N4iLf?I_0ggb?mgiVFY1*-&S3swsD2{H*)3QZKcBUmG#&i|D+
zgQt>f0f!vB5L-Q~IqN4DQ<h~+&l&9hAOCaer{fRHA1l6#emnX__{*x#7N0dgo%t~L
zL(<2}kLNy=eOUZK=wtB5zz+uRcD*rp<Nw<1)ytR7ulB#Tef#F^rng((YQ3v|clDj+
zd&T$P-=)3Z`QG~dx_3X`$$q%=arx)4ub$uIey;iT_}AQDe!nb#MgMa8b?4{lpSyl7
z{&Vo(&i}jq*D(Y#6)-PjZf9A*Cc*iP%b9x?S2WjRE*`G695dPV*%!0_;kd_noHK%J
z8}~ZyGn|vzr?ReQ7H68xVD|sWzuf=q3``8Q|2O{&{@4Fc^xw2U(|-5<=KXE=bMChj
zpNl>odmr%b;hUeY7rjn={q+^+tCKJOK0o%n`uWLc-=D@jO?-Oi>BXmBPfeaqcsk{&
z!qc)R_D}kr?s|Ul#rhY|UQB=a?IrW8?=OR2slK`V&h~@fhyNeueB%7F_KU$6$<NH6
z1-}%1IsbY6=c8XLzrFp=`s3fXx!;VxyL|8d#`f*!SDA0>-{yW@_oeWQ_1BJXem^Gu
zdhy5S|6E3S=EKbHEH7B{SpTvvW%FhC=D5Rohx;P06n`JTf&i1?D}i<Vvi$e?KMAx7
zMhdnGunTYsWbiBU_3{4ay}`GM|2}^@|9<{+0{H^Te0zC5bHC(T#Ceco5yvsk6Wk8G
zuX$JT_3{@8OcLx7QWaJasTO@MsxN9K;xD{P=&+EvaH{Ys;S>=TQEt&x5hvjwAxR-I
zp*ezA1^fgi2;>MJ78DV*6}Z5!$bX$Tf~THail>mblb465munVhA;&8=JJ!F<o-A8g
zx!A;56<BN7OxdTgd9kvx%x6|-@n`wYyqKkzt%Ktz=L1eV&Ir!kT<5s2aev}o&Qr+y
znfD>D5^pEZZJt@Y@_cc8WqgbH?(s$OJ>hBRR^+zj&g7oQ&BpVBC!bGDz+A9butD&T
z;BTP<;acINBBA2(k}6VOQmIn=r8K3FO1+fSm0BfzPexakQ}&?D6B!TLCYd=>6C}4v
z_DQl!>Pj+5NlR<Wq{(cT_LH71eM@?Sbh)&fw5;?@sb3NvVzI)P1vc{O@T&7vbH{SG
za3AD)z;TRCjP(HXVx~^UNQQ*}Z~y82zxkhmA@={De@Xw_|3CWY_ix3&%m1G-Y+_LO
zKmG5vKMQ}q`RV$@^LzC77vGD%-})-}we73!SIf^#AIskty<7Em{+rO(lV4@N+WPYK
z^W>-79<6xLdq4l)-aA3JH{H^_rFe7W^`L7vukOEAcEk0S#vQ$T&JR={)jZaH^5_Zc
zv$W?oUR1yGf7A9Z`(w$Mjo-d~zwzVr&kMi0ep~$c|Mv&O1m-I&vskyWy<*?Pah>BZ
z#~#i-+_Jo^ytO<fJiB-%@HlZ>asFm!<%r@~!m*OGo_iv18o!@_j(~?iuHYV_GT}eM
z_e8#m-V{40_CQob^s9)FD6go2=uXk;VoBne;<LnhL??)t3TF$><2U1r=GElQ<SpcF
z<XOk1$!W`Rfvu0_6q5tfW5zZHfq&M&@BWbauJQHeC+&|*KJ5Fj=Y#Ht@b^yd89wlR
zc=zt!+jVb0yv==Q@!sLXrjL_9^L?H6b@kVWUt7Oz`2Oq1p`T^H!hbLNed1Tc&-5SD
zzB_(z{r=>;_>ZI?KYvX9`RwQJpMgIee_s2!``3iu4S!_*sWG%LE@#wWJj5`UVHU$$
z21&+Nrd2HZZ1rp_*;Lts+5fT4WNTpC&Bnl9#U{<l%(92MoVkGM5CiA`i+_Fovj4sG
z$NI0!KZ*b1|IPj#{IltI+waxCbAL1aUh#|P*Xtj_-$T9zfBx`M=A+Mt*!N}cCckZc
z!~0s~<*R2rPdgqPK9YF2;K7XtMGqw(wLXe}^!y>;qd$+fKlXaU_e|yGnb$&ZbKYjW
z&3HTIZSvcFZ}z?U{Wj&j+=ti?TR-%CT>0twr+FU*K9s!=elPdl`@Q%FrH_oCls>t9
zWd7jr-t+y__ftMFer)&{_UYlLHJ@reNqn~aQvLPTx7r_Rzsmo(|4aSP!!VOUoN+p1
z3{wPi9?L$~PWCX)M6T&v^SEwvMRLpXNb`pBDGGQC9S~*^5fE7|VlG-N`d;*&*gf&-
z5_=@VB@`s0B_t%KiY1A}3LOyG!r#a*ETAA5CS)uuCoCiMULaXuqd<z_OTj`Re_>mZ
z^&)K|3L?LSGes;!---5!nTp>KSCP0O{#b0bXurrF;b%gLLd%7MgpUZ{5pEU^5}G04
z&7Z(`ori_nhO31um`j#(H~V=uOSVm{Y^((=D_F`|PqR9+u3|}N$zyS4v1ATmeEa{x
z|9XagMmy#L)=KtE9MYU&97gQrY|GdVvcKVQ;uPR~%TdNr$Ucd!n(YtUclI#OGhCKD
zUAzH&nS4%sLVO!}9e8DUWqFx+C3r9Jp5}|-FXms(&m~|bkS!o6Xd$#$SW2`|^twow
zh@|Kl(R8tN@%<79BqgNIOIAv<NqI=WmtH2FCS5M=Ed5U^TY8Sn9oa0oFS0vi9HgbC
zrb%v<6qJ&Xx+Li*sUl$^W-jtbNMC5HpsZk^KoWlzUpOxZPZrk(4r_J^HXhc0%*xEK
zm|U3;F$=Q@vbZvzVr*mh@IU<jfqx<YKQQcP6lJtyc=A8@fAGKOf6o1`{+;l<?APZX
ztUq3T-|>CXxAHF&KmK~>^w#{%hS#rOuX-c$w&~5IS6g0ecoz6{-4l-|`j7P=8r?s0
zx9yI_?b2HlZx!F(f9KoXzxNy-tbMrRQR3sACo`X}e)i&d$;<z*`rcT+<M{CP<NnVg
zU$egn{4o3Z{AbMXdw(?<<};Qt{bt(0{EkJDO^$sR$7`+vo+~_id0z5N;??I{$;-gA
zjO#yV0_P15BTio~2ks2+7hI8C+FTV}P26$3^8D-qeF8j!41!?-xA=Jk&IoK0G#82&
zIw+JPd|Y_4@Mj@*Av?jt0z3jI`8M+w^6un4&#S?^m3up9F8f^;6(;}xH~&og9rioo
z*Uj%WUyHsx|IGf`>y!2;*H14#9{P~={=vI1@A}?Fy%l=n`&#YwiPxTQ``_DsTJ(kS
zoAtLXU!Q%^`ts>>_?MSozJ3Y*(*IfaGwbKepRRnG^jY@nuWyY%`hUj#lK;K=_xs<+
zf9L&n{oV6h{?C;^Q~nnJTloJGg9T$aL-K#U|KI+#|Kt5XfngfsGR9uUS&Zu#?=!w+
z{L6TOiIv5HC6u{^sf3Axc@}dpvmuiRV+=#a|J;8$|8o92Ge|LL{OAAo`p=QyKYs21
z)%0ub&!8W?-zR+i_WAfH!B5#ASA4kep5=YOyVSQDZ%)37eEIVE(`PZyo;>Y(did##
zXZN0~zbt>b=f#)j)z6<k>v+cey!plBS2N%KdDrlc<K6Xl2j4$=Z}P$BqvfZrPXeDf
zKL&m9{lNKw?LGgygKu}gJ@vNo9sdXUPcJ_o`%>`5@w3LKEgvs`H2&20srl2FkG3Co
zeklHs`(eX}=#Pb;_IzgfYWpqsd;O1^pDw@d{VMux@JH!y&A(uV1g1z9CpJ!w*Bqxf
zj&ihf`g8r`n$E+@KTY7Az!3p+!OMb4LYBh4A{Jsj;>*Q1i~SU}6T2z4O{`z^v2cUX
z8o|#3&H^z4wSqf@WQF$$MG3hHEfCr+oGrRfj8Xit*gdfoVxD3NV!q-xC7PvPN`I4?
zDSJcql5B=-xXceJR;hH!rxJ@Kx+GF07EAn)5SI8LDj{-L$VIS;e=*-yzG(hG{NeoG
zya%}#aj)Sv<$liffNMLKH&+wqLJnJw5DqC0HuiE>C6-#|S<IJM<k)jLZ*etoS9AAr
zIdU%Ic*xPosl(OAmCq&5#mwcvb(m`+_avT`yc_rm_|Ncj3UCX|<bTPR%NNYo!gq-8
zE}sTJtH4izEWrf9TLPc>TlqNzW(qtKcpx}MxK|`mq)}v}sET;B#7)VK(o<xG<=4p9
z$v4R>D9li>QBak?Aj>DKB3mc>MfS0rmcn5LHU&?)Tp3a6cqv0E7pX^*b0i*#sfeZs
zZxRd?Xy&)%H{sXkf5qp^=gs?pyPeC2Q-ec{-HYuC>v>i#HWfBqwp2DLwuLMz%yCSu
zjLi($42F!q8NHc~FfL(u`v2Jffd4W7rvBCWd-@OKpZ;I3e_Z&^{{6()J)es{PJ18p
zF8%HIH(YN|yoq_O`SR7XiYH=^jy;(0!1Y1M{a1IT?s(o3zB%v4?i=AZg>P-Wm4938
zj_;kMI~(pe-*vsW?7rQ@(8o;APQIvro$${2W7p@8U#h-#eSP!w@wZFgSN|yd<@UGe
zzdFNIhJ}nPnc|sESvpx`*rhmsb6Rs9<C@C-hI=CSI<7sO^_-eqD%`R>F}$nzy!hYp
z-Qc~<bC~BYZvelYz#@Tz0;dEn3TzN)7kD6WQSh<Q0bzHMLXndq?xNbFDI(><1wzS!
zD+NyT=kTfUI`On~ujIPMaftOElN4j@f49FXe?tG{{z>}n_4D;N+piv9f<JqGI`OgU
z<DC!R-tTzl@pjuAjklg}7r*g(bM_7IyGQT8d_4O3%NN<NsbA_pPyV#(<Ghb;pR_*n
zeLnQ5;gjtrk5BtQm3+?ovi9ruZ}vYX{k;0?!SCijTmIDl5%_cP_od(3f9C#i{_FP7
z<A2rvBmXY`{r1P{&(q)6fAjs3`MdjH`u}bJbpGx7ck}-<hHZ=?Oy8M`nM0TtGifr(
zGyP;NWIV^9%dq1Amj9Lv8yL<qh%?mutNYXWEAPjxZ*AXZeB=K1_)F~P-5)zYOnqPf
z?&#YIZ$sX`dXx2L*6U5Le!UcU`S|&ZXWq|#Jym;l;MuI_YhGM?x#yMA>jSSkU$(#4
z^djr!kC$Sv#ow-f_u`%3yPIzX-@SUb>wWcy=8ry~BtHdw%=?i1{@}aa@6zApzdiZp
z$D2!U)89LP?E3WU)4@;2KE{2}dcW`;@B6>+^*@-tw|aNyZQ$F$H#1*Pe9iIZ#G974
zF7N8zYkpkx>D=ePU;Mvq`|kGh)2|1A*#28GHZy0k>adxyb+I|JKV{#@@rH9QcN?z^
z|6cyZ{Qvl~1;d0-i)<2g7n>?JTZ~KWqezs<F%fOi3nDrqO~R(aH-)`L4~rcWe=nXR
z&LqBEj8805OhV$kWQEjrsYg=hq_#*cl$s{>QR;&9Y#Djk6|(wr&T>~}b!0tdETs#j
zyrn9nrc0fdvXyR<_LrV4<tdpWK3#OJNPx(0VSnLB;YUKXf=~GO@Covr=dI^G!NbLq
z%w5RU#;M9F$7#T6#W{sTo}HELI%^QyJoY-yhg_4ng?SWs9&n5Eyy21MYv)f8U=oPt
z_vMe|H|0;{XAyWL@JMi;&;}tJp|yg{f{X%N_~!Dy<SF6F=6S;t!#k1pFK-K90^cg$
z6yE>5H~D7r?-Ixm@)ThbQxI<#caxBk;*lwmT_Bete?dM^;j%)Jg1thc!dLlpxtlVa
zGX65Xvh4CA3Wf@|<QB@5NwG;diB=2i3x5)NDcB_7$zR0B!FQ3jgEySlpZ68dJ?_g~
z&Ri=wmv9uYtFrB7S<TGP+{kp2(UZ}iv4&BNk(*)GKY_nTe=Gg|`|HLp(cjO0+x_YI
zz42G>uVueDevABG@=NPi?$2l6HNUO=^7~WpN4^h!?{2-8e>MGO(aWZnzAqm?uYG#^
z(b@Zl?}*>Ndh7Tt>svZEGp?sy<+_r1`TnKZm%d-RclqNL%WKWom)v-LGwt^3JBRN|
z-Cy^h^wG>GYR~Vyw0gt$Zq@s1AO3wf`629M`zNU{2H%{2y!pA|cjVuwe;fY4VdP?2
z$I8Y&lYJe#6~{FWHqK?7%eY=}#d7W8tl?DV+QjX~Ys6Q;_lEZ>&rNO@ZZYni+zPzL
zd`^5nc|Y-%^3CFB7i1E;D-<DeT+~p^Qmj&Jf>^Csm6(7Si|B3PcS6QOm4Z0}Qv92E
ztGQJ;7O@;=ocllS-<7`}e|P`U{&V?P_mAIS7k*a#wCUro4=wN4zUzCp=WWKDt*>Xi
zE`BZhI`UQi%chsCukGI!y%+j;^`qz~vrlh77JXdwanGkKp9{Vm|8nOG@7GUXb-sK2
zX#Bb7SH*9WKYD+6{Wbij^?$<uHUAd=z4>SV@1?)gf64u-{3Y?b@Av-SnZMJ1&H2gr
z>-?|g-?RUS{B!?rz_5&AD?>7)J+lF80-FGP9J>?yX||)R)+{fX?3gkb@BH`v*YtPQ
z-^73a{$2QY>aW_LPd}%AU-Q-XOYvu+&oZAby)Sty`9|RN!<Wo2+g_Bu;C{LF<&&4n
zuXeop`D*<uhF3N(7ru~tvH!W@3$d3$Fa2L`d+G7A`MKt^OHUi0);!IBHt%`Qi)AnS
zUu}JL>DAKL&)-PCyZ`R}yOZyv-aEXX|4!;%_PaIjzP+t^Bl&vXD~s1mZ#>^Hy!rB~
z@TK<)zUT5!WuAyX6?iuL>E|aWpPYHp{*?2%#S5{QF|W?Ne*NahTe0`1A7VdteLD16
z^Xr>$aX;h!u>HTx;K4MHIhkb%OFqkAW;2$xtjX+l9Pc^gxfr?sa82iG;8Nn=%#+UN
z$3KJr7k|3IEP)OI4FLv$27$){9s*POxdj>oRfPRSx<z<JtcBTyo(s$r_$F{b@Vn4E
zVMmeoBEDkD;u;dck`+>6(vxIF<pSkv6`m{pR*F?NQ(mUTrF2>Gk>W?ii;4+~TNES}
zY~&Zqn#(+rekQ#^`hv8u%r<Fx>A6w`l7<rI5|bnjh&zg%7LgI|5lj;>;IH7b=5H4G
zC-7Q8PEc7$Q>04li+G7dp@hD~W$_&GLt^z}_Tr2Z(c<?-_lU5FI0{b`k`ekXSR)iK
zyj!?f*g=?6_>kaoeha>Byy3h-yfS<{`Fi=T^2rHg3+&?4=3UIo$A6svDSxxTZ9zj}
z0g>k-{i1V4t;94Wyd=XV_KKgB(2?|!kd-KqxFmi_?478dh?NkZkiD>!NTf)*$X($=
zp%eiNzC%1oJUKj!Jny)~IUleouv)TQWB$!NgN2=yo%ILHDV7M9AI$m8=}gBMotaKD
z9c4Pnw2^5iqcel?|Neg)|2q7+{j2)dt6$H4x%}e#mHw;z*Rh{xe<=L8{C)BFmhW-j
z6~8NdkN*DdTlZI=FLOWd{uK7{&HJD4PQBB3|MA_lx3^zkfBECtna8OQ&)v7V|KtAZ
z2Vd{=-1oe{|Ne^m&+hj=`1N4v!~Kt19;-i@_B7-9ju$syvcG=&ddr(XZ!f&x@^Rwl
zH(y`;SpM7Rui-!L|FR5bjH*okm~vUBuz7K;<5<ZtmvaJl6>lnE7GDA12fjaik9qre
zgn8t7ba`@l6nMYze&^E@_$zQzKuI8*e+%DDUK8FhUMoH^el7mld_sKcd^h+e^Ggaa
z3T6ux3SScr7ZDI`6<sOXD7s!GU$|Ikk)V%29$ypBH7*a%gY2JJOPDJeSpT*Bark}f
zr^-*KpQ1k>d>8nZ_=V^5(~s#N;@<6f6Z9tK&G*-4ukXK-c^&uq>8r;tt6w_3TJzfK
z?Y?&#-amf-?ETsI*6%ak@w|(9ckErmyTG?@Z*tyDeEZ<t*Y`U=1b=+_QS#Hwk18Lt
zKXQGF{*?Kt|I@=y>7T!Uj{cJO#rKQwm$jcyd~*Bv;{BHQ+djyBO8PwO%bl;+zKMO`
z_-*yK=ik|WasTQ2d;cE;!zzY0hBf~W{nPx{|F`E){_hjNX8vmW<^Su`&v!poe(U(W
z_Cxl&+Bf#EO<p&?e)?+B%Y83qyg2+~^$XV*3@?ISJbV%VQuEc6*J^Jq-?hAxd)N4u
z@9ow%xo`a6l)bt1Cj9Nvw;pdD-iW;}el_P6_iLBepI;w*d+WW<N3~BYKkxgx_Ivja
z@gH{IAAjroR{u5d%dO86Uvj@_eKGqS|0(g)iBA_l`F$$+xbOq#2bcGv@5JA-y}kVQ
z$h*7mcD-Hy=I@(>Z#TV-e=G5B?fc}9`#;V4T>ORk+l%k4KXZPX{@U`p<8Q_PDn@hW
z=`31opV)75xN$6Jx8aE3{LeXuYbEyso<F<~`E2<&@~a9S5V|I`RghgURB*PSq@a_4
zw7^jTd%;@*Klsb}Kk}OinhVtmO&6*c>K2p};N>^w+reALcZ#1&-~qpd;8x)T(J!Jc
zVq3)xB%`I?OBG1B$r#I>mw%wJQ}K(Ey-K-Cwz7hfyrP-{zx)|l0htey+7bccTH@2i
z6hvPMpAa?@))h?Q59R;HuPX3~e<FVd|9}430!9LJ`4t6t1+4`y2-FGe5|}DDONdc;
zoe;Z_i(sZeB)=ZtV%`Y8E`9-lT>dY-b9mx-9`R`MR`WjRQxv={R3%&_+#$>)Vj;3Y
z_?1ww&?CWPf?`79Lh?da1=kDu3Tg;+^40TR;+@8~o^K1^W&Y)Y7Q*o&MWXk_&PW(a
zU6O2+_$96=@krcQ{DoMfc$LI1iF@MT#4d@kihmHx6LS&WB5Wu0T3{c)Apbu;JN}9M
zHvAR5^4upl-8t(xCb7q|b8)D0KIJ^hS;qO2a}(E9E_<#MoR2x3x#YP*ITvx1arkj;
zV_(W9#u~xw&*aV2&t%WEknsW|19LlbERzo7B*sc6ZKfLxGykjqpYi|X|K5KG|J?l@
z`TO-x<sSmymwk)-_T?+<*Qn2_A6LBBe7EQg+v`&=g<guhP<b}*anwVD2YwIcKKOK>
z<-X$m>GuWi%iJ@%H|zfDhxZ?Ed#dw%(~F8%dT-j^&U<&@y~#(h&+1=yeari<`s2;d
z2fyR~y#Af`XT@Lpe?R|9|1<c%m7$x_mT?oq3<hn6qyO~&=Kb0IXU<>Ge@*{9|En+_
zVB%w*&Gepe8-wWodw=8pru;qmcjmv2|7ZS-{h#o!`+p(hd#1O{46Igc+3dL-?3^8(
zleo5Wf8<u@KEo-_p~05H@`I_BQI^5!|E7ONe=C1I`=0d8@#~t;Cq7Dl@OoeNUhTd0
z+t$}yZ`|K5c+>TY`(?t5Y0s^n&3ba<@!=<l&n%yJJU4%#{c_*SXD=_jocuEJ<;0g(
zuRgqre0}rvqBpnRoPMqFiuL7>7q?#CeHHY2<?AhPzP$B+r~YomyBF^*K8k&s{mK4w
z)|abaS-;=>UjF0J5AL5Ye=Pms`-A_-{vQi}RsC`JtNt&7!G$TANrs7mxs7ELt23Je
z8xPwLR!3GpW(UT4hI)qo|JVPE`TOxt%->0WKm1|)6Z*U5m&4CJ-(|j4d}024`Qzaa
z#qYnpje9HmZr{5nZ%f|XdCmAH_08`$``=D{*Y-Z(L+pps?|t9%zkm4d%DeOLD&DPn
z%k!4??d3P?UoU<2?$wdkdtW=e_I!Qiwc8t`H#=U7ywQ4B@<I6Hgb(o_Tt9TZ4|vD-
zw*1ZKH#^_%f1CKW>g|KK4DS}d-SH;&wZ^ORmu4?Nzu5Z1`sKG*H{P_qO?^B0t@OK=
zcgNqczMuJC^rPu#{jXf#U49n-=KX8@&+)$*gFoX{rXCh<wsdwGP6_S~9wFXDUI{)s
z{^<g>f^!9b33dvzi9Qp_5HS~>D0WWlvS^fufUu{~M!}~7(*z0y_X{NnYY8t9DiS&?
zq$?~Wd_-u9P^i!?K}W$p0S$pB0XxCPg1ZDi3T_k96geV#T5PKLM~M_EQE5TxLsF$u
z$x?Pw!IJ01cZem5DT_@My(;oa_^42}pryclK1bdlp4U9bc~f}j@EqlF<z?cH;4$Pm
z#q)r-lJ7mQBCihbeBLtNeLRjln|WCHr1^dMr}4iOs1!02HWd~Y{x5V-Xq8Z>&@!R7
zLbbxpBEF)6qD3N8MC?TSMOTad7WEa|C{`fuFY!ZSx#Tpdd}(%>N|{qKA7w;kBV?mx
zugbW|M9Zv^nI@AevsT7e_Jk~_oP!*@{BeafN{f^eRP@z?HPkd4G?LZJ)y}E@Q8}P|
zM(LW8k@90DW~B_pSj9rcR|*yKOJzSu+euBAkQcuzY9d-BdO(y>v{YDE$V;$M-~xX*
ze>lIAK%k(wkdY9B&_+RD!DRw{{7!rUyjOU<c>=jrx#T&`I7HZYuzp~k&RF`t@1OoZ
z*1xxZ$N#zYxAkBAzl?te{~iDL<*&-0A3sfgT>R$yb;sx0PcJ`i{K)*V^}WTr?{9y+
z6@FX!`uWR_7i!O!J!N^){wVKZ&4Y;hr|*8eoqp@vO}AU+w|Z{r+~&G7_0GLJ26r3o
z-oG1r@7O*0`}gkudcggN@v+Pkm#1RS&OiI{-1Q~%tJ+s{Uu(QIdbjOe<a^x@*FMO6
zy!m1Bhrb_sKDvIa_@MM5@x$B?`X5-{OS}tw_vYQRcW2-AzrXe2!bgcuJ3g9zWctYX
z@#TkyA0B^j`PlkN{R_|6j;}kvrTm!qQ|(vNuVcTIe{cC6_s8OI-oMrVn;C8~EMpL2
zi22|DukbI=pZUKke=hrR>HCCluf8~b{_;`!qxOdx?_A!dzx98Y@b1~$h<DT8dwn?Y
zKJML*H`iZRy;}T2{Q1UbMbDqTD19~Yb@&_Sx1ZjgeJk;{@r~sh|2OmA+<v3{_U+qy
z?~LESeOLBQ{+-;r(057i=D$1te%eQ+&wgLRzJ`CR`QG<~?I+hy^`8+xtAAeq+4jr*
zx8R?je-8fr@i+hPoj*2z9sV8t_xRtL|A~x~m_(VInYCFyu*9<(u|8lq!qUKUpIL`_
zIa3|eL8kLeKN;^b*f4DVzw=+j->bh%ekK2m{-N{z``2?{c7JC5ocu}o<A(Q~@BhEM
z`)>ZbMen}8D}8_Y{rmT(AAWt<|Iy%6&!;n=u6+9cDed#}&&8jwefslJ|6}2Y$oGQp
zeBOG$ZGXGy?ZUV3-(Gun=e_v{wGXpCG=Aj#bl~H=4~!o)-dn!=_vXrLsn>^Jt$TI$
z)!kQzUtNCX_4@GZL$3v1KX^I$Md)+sXJSvcJP~+$>*>>H94~ZV`oA)G-Sm3a>wmA;
zznT5^^Sjj_GCm1>Y5ZFJ&HMYRABn%1|E&9)^8X5h64Od%f7VHC#_T=pKiKU#j<BC$
zD`xw~*3T}@A<bFGHIJK}=R9``_iwHet{0q#In%hdaX0h&^3M|J7c3Lv5WXQKCDbOE
zAlN2&LQq%ep3rq+UeO6+{1Q@<&n3@E&5-^ftt(S6BPe@Awp?zd+#$L5a{J_j<T_;&
zWu;^n$t25E$kfZY%e<7{Abmpmf%InSInvjqRb+}}SY)N;rpe1H<}39nx2XuJ-d1_2
z+@@5Z_)_7t!X*V6#aD_9%GZ>;RK!$cRKrvqRby3msfwyusRgOksI60bq{gkTu5PC8
zub!k{sNSRgSUp!mNOP~|56u?MR~oZ4iZo(0$~5L`{L)yfX|1KFbzZYhQ&Y1`<Bj?x
zb$j(4YSC)3Y9VR|RC!eeRL`p>sl}>UtLdwKSJhHIsqCl3r?^`FglvPfpX6on_hKJJ
zcZpmQ_7FZO^g?jIKn}km-$5QBZXM1p_FT4CtVdX5S(90cnRS`}F*z}1GJa#&&#-`D
zA;UoidB!)4+nFvinK6YhN;8=MWBa4}tM<qKZ>e8*e6jxG`<da>%nv5-PrfaDGyC<H
z*D-ItzR7yK^X>e%U*8;ht@O&|Mc}jkCp#bAe9(4Z@cx^7Q}1!!Yq)#tPW|njH_u)F
zch&XE^GlAG%r9A8(z(=i$^Y_=%bzcET+zIeeC5Oyx2p@U{=8~(&F)&=HIeJd*MqMo
zU%zoZ^2UxEqBl?9)VRI#&bPb2?ioDDdKmeL`*GvrdyiY6)IOd4Ozg##m&&hSyx#bx
z>#fu~rFYZbzIxN}Cg6?To5(k(-t2kH`kw7W#fO6*9)Fnn!Qq4P2gwhTAI^PP{qgn3
zdmmXo1%2ZG9P?%OSF`ViKNWxb{9*Vz<L|}4t$*47&i%9KkM7^Qe+&Oz`S<qUx_@T>
zI{&WuGv{~2FZG`ZKem5g`EAA5w6En~^}ZhZa_fut*Nb0$zpeas>)Xw5Ip4az`hL0g
zDfZ*#_jd1ozsY|y=S|ex>~~e~<3GIq@Zv+`2cZx5-hX``_TkNku8$I*R(-1YT=_-q
ztMS*LUv7Rm{$<mb6<^rC@_sA%F8tHw_l>_-{;M(OGfiOTV)@9tpZPs=Da&e>6c#&{
zSQZwR0A_7wU*^|LTbRO_{aL26X0Y3F?%@*S;pYwFy~WeZ{eg2jM-=;S)(<SbEZbNj
zSWmIKvVLSPVA{*D_20ZdOMjjG5&qr&+nz5>pLc!S`r*y{d+#5;pZvb~ec$_t_l)m%
zy_@px@w<xmf8Q_pkoIxY$K;QDKd^nsdC&O1>Rs2{32%P94tc%!71yixFRGq@dB*d+
z;Mu{atk3Q~Tm0Pr#ex?dFA`s9zgYGB^Rtj=&!5hHYW(!t6PBl&pE^F9{Osp5x97>v
z=RUvjT<*oT7v(QkzYKW!^u^*A2VR7}WPjE3%Ki0>H`CrOe4F)l&f7<CJ>I=}x8(iy
z4>g~{zC8UJ{e9h!^FP1*V)?`N*YV$^|7RIun4(yW*ygdn<uK%8;BM#k;pykK;(sjg
zTkwLAkcfk5p6C~m7Lizy5|N1_wIT;aM8zT{{z&D?ddVMCh*Dastf9iBvQSx8d9ISb
zl8};`l8v&Ss=4|Pjd(3vZDyUnI!kmH>c;7s>gMSh={e{}8-yDEGxRdDFgj=GVOVS6
zuOFy)P`60eTUShXrp|5c+gd+0murS;vTOd<6wx`V+og9%Z-Snnp0BQq&JC^Gn$ns*
z8rmA18hRSN8Zw&uG{0$1))dps(r{A`QqxllRC7^tSDUQHtnR2Dpq{AyUG14Fhiak9
z0_C?#-AblPl}hf)8<Y<ze^Gv+T%jzbtft(eyiGYod8X2B#mx%t^7G}o<Q~aBlBtq$
zlQ}BAR9abPi%g`fu$+h-qwF1NC+R9FH^~r*6XJ>Dx5ZjT?+OPBhYD{KS|L~<@SATA
zuN+S+moDcKc0sl+ED6lB8T}X%7?c?|G9G4(VZ6xT&Jgzh?%$R_$9~WKz5Ms$-#dR_
z`n~D*qTklP_x-%`UEtf3&vQR=y?^?8{>ul?H$Sa=a_mv={nq=wkF}pxKi~6q`WKJi
zo&R<I{QNfY`~IKOzmNU7@oVNcp)cC+|Gr>;arb%S6Zgjt9_ioHzj^Z7tIO#Za?i${
zdVOriq3nH<dzyB$?>@O}?XI|85xb0b&Dg24lVRuI9dmY0-!*sl+dYQ+*!H*V7eBE4
zp!5;BV{ylqop3p|`gFvZ*t31-%`e<OTX`n)T=a!o7tUVFy}sv8=EE0H@4R~TKIe<{
zkNn?D|D9u)!dS@U!cxO#!a0q{pMSDIu|O$*41c_UkiZ_kUwnu8!}*!{b@-?7E#sBr
z)#LfX)x~v`E0t>^XAI|G&JJ!>zM}$i!ogy5BqvF`%6^wim5-KRBR^GcskE;IljtMC
zc>dKqD><Gpr~Fs?o&P=H>-;ZfUs%2z`NI6o_uJjC%f8wF@ci-a`;DJ9fAaqt{QdD~
z-(S`Lum1)7-TJ5J&#&K4ew+W^|1;-T@1K`{rv7R9EAjt3Lorh((<a6qMpmYyOpZ)K
z4Ez3F{dWnpR+G_`=@)Ysy9?JlZVA3={3rRJ^6%!?<p0ap#J83wm(!i~DdYS9Tz@@&
z{QkV_gW_B6S4&?QKA-%g^O3@%(+}rAh`k?wFX8UVJ16cG-p#+S@u>arna9tbJbt?E
zN&Tbw4?jP=`;g;d{DVvPtL~k><A3MdojG@p-0ix%>yG8^DL1)qe!Hb}|LS9*7iZpR
zewy%o)}Q4J6PUlT&Sn3`Uc+9+#>BdiIh5%mgUkOte<Xe>{LK7W{7dk!FvCyAGtBC&
zg)Gs`NzA{Qb}&9+Sj-^IXw0O<EXi_{)tP-MyA8(=j<+1UII20$vB|K!VKijW`B(Sb
z<cG&sjW0jG-2PVm!}cf7FM;1Wzk7dG|Fry3`t#zSb$@sM@%zj6H~7!I|4vMw{!jit
zk4ci5g=Hc~2xl~FFY8mz=bYiJt}MzN$(*X}dYra=y}YwIS~wc`<HTA-?+A*Esmg>(
zvP)c7;8sgg@>9;z($)3TUZcHE|Fp4$DT~Qd!wh|Qz1_NJv{q`QX;0L>r!`S?v#x{T
zEd6gfwYs8u89E%go`&a)gN%8MFB*rMt}wf75@b-T6Rz<?@uxJeWUIt$@in66!n=i(
zh1~e=aq@BL^4{fL&cn!C&AXe&iT4uUPQE`p-#F9R?yxT9@aMASdd=<1pDNfTI8RVh
zSWR@QXpV@qu({9)!Oud$V$71;B;JVclVFhAEA1m!EpIK)r7&AjN=ZsdPdP@_T3t+o
zTeDRwO#7ntJ{=BSK3y?AH@&$!kvcPU7i#}dTcg6O@>FrHLXi@SO0-J4nxEz!O+`&1
z&9CYas_M$;6wb@$%AS<JtH7=(tNc-Asq#OC!*U;`tt8HgI1BX(Oy;-XtKeSEzM6SA
z!;Js){%!oj_9y)h^KZeQbw7T8+wj@+<HQgDKCJ)v{A1<Eg&)qnfAn7b!}fR5ZxmjZ
zKka*HchB}V!_B`pmfT9d-Eb@VrtXbT*95Pxy-|3J?~dBtb$60)Z@)F~*0!4uuf4iF
z?ec;v%vZcG{kmv*MdEtVb%|@ouQpsQxzc<k^=ir0l~=egt6Z!;|Mtw?Q~qaI&xf3!
zanASLoU=k_o}5TJGUed&1J(z452PKOe|Y<`=u?4bZ=JWjc;|fC+1xWSXKtUKa3<os
z{gvh$g16q@;=QAFXX+h|dvouveQ5JU?kU5wu$K$pB)x5YlmA-#wZof9@0NYc{37;!
z^)Ksx?-|}RaWXGt+{w6^Ns;L(gC7GggCv6{Lnp%_#*IwJm<}+NGc95I!)(Fwi%E#t
zg=GoL2bP<xzu0OxK5`0jtMS<IOyN@D$YQ<5#LM{bf93yE|Ly)e{j2_0^UvgO`Jb*o
zet#tXJoxSWTkDtLk0W1`zbyWg^&$R!=eyds*WbpzD}L+r_UJp-4^<!TeyIC+@>9|m
zm9N2H*L-dKI{C}L&+=aqznuSa;_JlktiO_e+x<TB`{bXPKT3aG|8o7i@Gq00f^i1Z
zPnLP?8C+?+#{8}P`urCB;sTHOQ~4I~8S(S*e&CwT#m)JeO^RKW!;qtuqnvXwhc)XX
zMhgc2e>T78{Sf``^DX#W$v5tAlfGU3A@^&-54mseJ}>!@@y6@L+$TF8p1GHI`|*wa
zR|_tNUC_Lwb1D1c@rzF`e!k>=<=3Up7n?5syuy3U;d<cB&U-r^8$VTie()v#OQ#oo
zFLpk4d&K)l{Bh2c6;F3Qo$>73^B2$5pGG|uc&7Jk(({dP_I`Hy-twLIr_isE@6BI6
ze^L4V@8^;~YyMXKb7p+Sa)8~KOM=IZXEoPaj&&UOxh`|Ja!=u95MbxO%G<(!QqWB(
zS0IT`mw&AQr@(anBz}EfAFi+L4(wMrJ$ROJw{m~vnaG>NThFtGyM^Z#_aBaA&ez<A
z-0wM;aO?4{7bp@|7L5|SEHPcWLT0J-Unw`KW{I=n$0g-tT4k@urO3C+h0AV|eW>tD
zwM?r;*IK_+|BL=(!&Sy_Otj6`nR*+C8r?U3Y`V&PvZbx{1sgT{M8{Q*(;e2@>)XAw
z?zP%#S!mH}Az}5yYKFC>?MGW_+jmyk<{8E%2B!LU`qldH^h$I)v=g;vYQ<?Y>UQaS
z8qCtWuJc1HM}33x4#lksYVx0CxTNPv>PdKs`-;C2V-mF&TF&pn+s^%yOP%XC`wdnr
z=8yj_{dWF6>6g<l!$0r;8vm>L=lxIdZ|oocKSh5k{&4+q{JrH@%I}oFFa8<+=lsv`
zPw$WK?}pz@f8zh7{rUUn-Cu^kmwsLO$?>b_*Y{rwf1mi9#t_Xoo$(ywZ3eUdYyX!1
zarhJd$L-IjKfM2@{cmB?V@+gJVDDhdVCiGhXSnlc*RQibKmOqO$^7g5FT>w%zukYw
z|9bv?%2$&wS3Z6I`1a%ekM}-$eERV5^T)`~3%>mM!u56Hm&(uTpK3qWe`@~X@YU{%
z)aO$lW#6BD{q=?3vn7xJKje7m`>^P-_A|a0WiPfryYP6yL$e32_s#Afy1(wh>xcS}
zS)M+9{`Hm3Ti17X?{weQz1#K9<lVtHi(lKk>3r+(PWQdV2eS{0-tT^2|Do=q_b0|r
z?>?;lp!t#KWA?`*pI?9d@m2Dh+xHvaY`(1hIQ_lV+sUuLzgqrE{I%Gd@;BGtbiFNp
z8}qjP?f%z3FUwyFzW(?+=}qYy<~I{wr@x;0di(2ZuU5Rg{Nm|zsplV`IXu7q{K50i
z=Zr5_yx9F>+l!DFO3xQPz5O`oQSifs_bcxv+!4EzbEoC*+<PYXf8DFOTXuWmO|2W>
zuP5DLy(x10#$C||LXVUlpMF&FNcYjqhbj+E9vr#<>p{uG)ekN_ka>9ELBT_v$L3Gj
zUr4+zew+U8*V`#?r@j67#^O!joAqzF-!6Dt{;ul%k`J3cwSDdUp7JB*$K~()zpwm$
z@!Pa-FTZa3obp-g%h8V)-kHCf{_fm|g`ekrKKa@9+ox|z-)4P_{$BjU=J(-${7lv?
zzgf<)E@Lxd7v-GFt<GD|A1P!ibc_E5A2YuL-%j2qylZ&Uxt?<daL4dI;BOby5uD90
z!TX!@0Q+LL->e5&%URP|l9==vgPFIpDYDOIUCqMAGLd;6lLX^BMiXWqruU5BnSEI_
znHMn!vv9L4WB$jK!gQ5!4uj_Z!oRbBSO5O~d*5%qe|*e+tTwC{Sr4;qWOZYXW%Ol8
z`G4_W(EmJ!Ukoc4L>Nl`PyFZeFY$jdgA$_;lK|71|MUJO|I_~a?$6P`GyfR;;`|Zv
zRrQO}w>LkQ{QmTB8pF5$P5<lvyZ&$ff8>ALe{+VJjJFuCF*Y-6v2Eb+;l9axjbBP|
zy5J<iDFXg{S9lnBck+1gEaeRct=ksZC2*6!oG*d*1WzW<4{i&dzdYf*4t$IP4+W|P
zwS*^&=!jZ~trcG_p(%My@~f1S^cJZ-QuC#jN*<Q{Dz!#>ugp|AV}&}!Z;FkI(-bw8
z@|0?oC#%j>@6p_^<*T(ubE@VljdXPz^)~fXbxn14wTa3diWe2k6e{Es<)_GZ%6G_b
zl|L&lq`<6rO!1Jy5;<YnF6q;f&XR{DS*1=${+1|{;E|jxxk+-C6r0RO*?scO3jOlF
za#=F>C2b|9h$o6mNr*~@N<NV|Ep8>YLPSz{jbN2P9DfbpL0&%ITpnfay&M{BGnlV1
zeE9e1p8~@v#>31zSaaB>uufy?VUA@C|9|RV`M;Wf><pV2*_m{i)-ug!YGjIH=3_a;
ze2h7drG~|iWi`t-7Izj^78w>LmK7}hth#LGY?Ij<*k7@oV@YDpVJc@7Wy)pNVd-PN
z!gh)68QT`N0@imdUs()Td|B48=COTb`^jF;ah-iN+XvQPtPfdRS?yVOutc)BvGg<l
zX3A%_V*bOZ!C1)P`Tya+Pyf{ZUHLQd_w1jx-}}G5{(Ssn+y~9~*WQ%BTKyvOng5eV
zkC-2D-^sqY_xhfzDp!IpD_&Z3-sH@z6J|$G99nzu-GPV$>ksTWsCn4)sK#-o6PYJU
zPTV@qa=i7}@?)yU?T-r|pLl%Ni3cYOPSu~vIHhnZ?3B%E$I}|8R-RmOQu$=ti9g4g
zPE0v@?R3@IH|H!axL*vpwENQ8Oa7MyE<0R0aOuhA%~vX~UAmEV>)LIlyJGjY-2eJe
z@=4w^&lh)JRJ{y*b^qnt7wcbmy?Fb4%8MN@{a;UgtN#A)`?Vh{J~w{Z|Ml`W&+h?0
zVt>~B-1>9R&$J&2-<ZD$eBAZU_pS9CjW@U7EP0duTI?0e%gPrMUfg&Q_j1xJwl}PA
z54@fE?&Z7wcN5-SeYf@9+;<A^6F#VZD*v4DMg5!V57wWPe&+o$`90~+s(-c&mW)Rj
z?=mR-@BQcS|JVOihGPu;jFpTcOfQ(sS?;k$uy5c{=91x_!QI5OjyIZbEngh}3IRbO
zY2mNJi$%7J+!NU<Dl1ke$|ss4x>8h1R9$43FoVcuk<+5b#k|DVi1SN$OC(B!NpwlP
zm1vgOE*>tvM@&>KS?sZxn|Qj!Ny!IN)-uYnPO^!zhh!#5^@~f3L<_j_KIXd0p~{iW
znar)vJCm=LKb+r=ZzInut^iI)4iENdwp*;utaDhjSew~Q+1=QE*pt{_v+w2*<^0WI
z%_+yl!`;dKgnK$q2k$1n*Zi9W7zI5AJq4=-jQIKaPV=a6H*>0R{A6=r<6xV{TF+w6
ze3+@1S%jsOxtw`7OATuRs}}1imI*9tS?pLkm=&2XGKny!u{>s#VUOl0=9J-*;tJ+W
z<Y4Bo;+V<tjiZIrm8*_Rh-(jL8Ru_~g&Z0jjO=Bsf0=`r)EQ*|Dg3qiyX>#lzhD2J
z|95A|`oI6L&Y!$r9zV{1>-+ZX+rsbnel-3{{&V-Q`oFt>MgD#M_vHU%22VyeCI#jK
zW_M;WW-aD;<}&8P%;#Aau+8Og;F`+4nkS9-J8v)_7hf)K0B;JfJa0CS6ZcilD;!B2
zQXF&Hzp-_&*|Bx7MzU;VQf7R}P{ufu@d_h5^9&YVHhy*i_7*l(HYv6iHhuQ%>^nJ_
zxa_$n^C<A;@FxqH3l<5U6^Ik~C~#WPM@U-OMC7`NplG=0Qc*#%&0>z?8^l#5ZcE&i
z?2_6l^<Ju4T2<zpjFD`&><8I5vdd-tWo=|uN^O_4m&}wX5&th1BQ7V=D3LDFAi*H%
zEqPmlPa<CYo>-~aAJIfn2hkg%E@HA`uS6?F=ZQQLmKJ6adL%H9-+@nw_c!-0E;TL(
zE=ewH&I9brSXr2r7|#AV`a|IB(NA(8)!&!B&3JS5^_tgZuN7V=y$*b1`S$zUjCVQj
z3f`@KH|O1lw+e4hy?OTL<C~Rl7~Yh>E_iMB+U=FcOS>2K&lWvVdffR?|G|O#=kH&*
z|M0%TgOCTF4|pG3y&rwQ;ohRVKkod$z2;W+%?UT$t|wnjy&QZ|>wL=Dho`wtO*tWQ
zBK!FLqi#nG4|^XfJGk>e<^j!vUk;r=l6frq#D!C@&UBqSc|P(&_l2()%r5F&48OSZ
z;{A&cFW$abeeukNzzZc8?p(Nf!S%v{^OfgA&Ig}Ab^i7Rl}i?vKV4?Ma{Th{%O@^>
zzC82F-m4nd|K50VtM<<AyEpHhzdzyO*2k?+KRsXmTIapR$HGsCKEM6a`nBt;;MY4}
z7Jc3E&F1^^?>m3E{`&U2`0vDjkN&S=jAkxnabx9WJH)n}{X0h_R}l9P?qr@|o<-a>
zT#cNSoJ?HYTst}2IUPB7a7b`WV_(Q#$Pvc*ma~Y9lY0rbA<ujs3Es)Pw|KSqn)uxL
z=I|cnHQ`?^FkO&CI7Rfd_&dp!(%WTL%YKpTk>4w~KvqWfyiB4Dvy7ySxa@N|T?I7-
z4u$IqGZmW^)fGP|>{kd;m?OVS?w+iKY>~_(X)WmlDL$$BQa7dF$r#CglesPPTIRjX
zH<?h`FuD2iTNE-CuPd4<?N{<vUa$O2`M2^T<w?q0mGl&=<ay<!WY@{~$!w5zkT#TV
zky<8kPK-lrq38vXnZkTR6#~Be?|7eZJ90hZ)Z!}O+R3HQ{g<18x0~0LH-^WF`!DBe
zj(B!?HhtD_EM~09temXcEc(oeOqPt*|3CkA{(JEc*Pn#ndcRG7v;KDerTkOvN7MH!
z-+q2|`}*a}rZ0|PrhGp4>EXxuA41>fzbk%w|8?%G-7j-rGQ2E)!TI9L^OMh~J#%?-
z|Dovv-upK9?C<&97kTLM*x;%8vk%YIU+BGzdCB+k{)_J~>|U0>Y<SuHvgzf9m+xLS
zzBGHG^!(zp+Gh)&?tY^3<nH4ykL92IfBgKh_mjm>qMw|69QRn~vCZREkIkRdKgoa6
z_C)<D!?TuWYR{LwV0}ID?WOm3K5Bfq{Y~L#-7n@p3I7%|a4@}MTEbk#(#2ZH#>Xzl
z!N4WUbCj2bKUW}6=&gvGxTNGssT!F*vdVJ1WKYU`l%60RC;d-qij;=b5y?==ehE);
zWwA`r`65?^R|)fpREr3Ua*KTx>lgngE-bNFoJYJ&Y_F)ZsEuflsEMesXn^Q5QARO;
z(R(6JqSHkKME{FeiQW@+6<aQPLHLwlmVhaLEAJ)llU&tYbGQn)W^;;jF6J!cistg=
ze9yj$Er|6zb2-xq#+{7vOxv06Gx;+yGfigt#gxF*!|29n%P7jYmSNBT?tkn4YW$u4
z=fv+%zgmB7`Ni^E=C{yqjo%5s8-M@!)$#Mv_ub#@zHR>c>r2;{t6%nhHU8H7_57C$
zU+TY{{1otk>7B`&gRk6PRz5d;X8-KfGr8vx&pThNc=hm2-@EGf+8=&>Soe|ZllUiv
zPjf$&eg6IV^5?qG<)6=gw*6xIrStQhPsN{heysYC_5S|58Sg6I<-GHKr}xg{9ml((
zw;$i^d{g@7?CTS+(qFE6zUQgl6Zgl_kHj7pJ~;JY{sX@I5qDSK*?4FB9g92Mw}oz*
z+*)$W<W}*`nwx63j@)9u?Q~o34$r-R_thWnc<A=%#3SCv=8u~mi$3}BME04>^MB9v
zUo3fX?d7dkcVDHxa(b2WYTm1Ruk>H<e|`Op$-9*IY9G2koc|#CQT5}!54|6je$e^&
z^<(iTl~30`?)Wt6i~F}V-}8T7{2l&Jm%*5^g=q~-EZYn=cQ#Wt3AS6TCt2UKPGpne
zxW)O0OOr>FFI?c7;3lD5VJ4A4kzf%ck;lU2!WP12!uN%a2<8ba;E(0k=by>9oc9UO
z4xT$a`n<n*1Nd(UhzLCpQWh2y78SM^UL+hXk}rBvOjzQE#21No5<HScl9E!3r9iW6
ziLy)Ovg8Eh_Q>_fx69v><CJTZT`v1ymPamI?wtHw#mh>@$`;BI%3GD6Dlb=#Q1(#H
zRi3K6QaMxkgHofCq>{g)mV%SKjr>RXI)wrSO@*)WJLL`KJ>>XgUrGOw`YE|k!d={6
zY=daH=pm79!ihqk1zP!4_&Rx(a|LlKa4K*na5S>punVz!v7KZ2!<575&2aKx`k(VZ
zJHD%Zo%ku`!{c|%?~cChf4lX~{ns3C?z~BV+x2$s+w*Tb-)?y`^L5fI|CifdtbLL7
zqWpQ=Gt*}*&%B=UJ~{Yk#lzzd<Q|;6f9JmBgNz4@9@IX#aR2kY@_V88%<jFt+jF<z
z?)1B5chBE3zw__*?c1%lZ{E6k)8uB&&3iYO-aL4d=l1M7VRu9B%H5T?8+EtoZs6T1
zcOTtb@j(4i(c`O6TAy{kh<`QVwcVTZZvx-We!KeZy0;SV#NV5J`1nERW5mZdANPK8
z`0VjH{WIGasjr3Kj()%Q<H*mgzdC*|`>p(Y<FDPnn19Rt7X5AUyZHB_-$#C5{yqQq
z^WQ;#dH(<Z|BK;1(-)Q)HYfHE?EM_locx?l97XJg?4s-!*tW8LW$R@x<WS<&<CNsw
z$a#b73->XeB;F&u&v=*cs_@R{apJM!DdLIanZbRCtBR|f%am&`XB4Lerv>Ljj?e6e
z*fy|kWSPy}&eYF%jUj;H=Kr|=kNySy`}UXfpWwf}f4BZk_~-Ed=Kmmur40IvmP{PX
z*32`Qx3FAhjbMvplVkhNs>o)^c9hkY)tWVo^$SZ1OCXB~YZluH_B9;hoC2I%IIK9P
zvF~JaW}CtGi)}X>KU+DgE~_=GENd@|7z+o>Ddzu7?-;)@I5XV&zwE!)|Nehd|DONj
z^=Izy9lu0=o%q@D)A{G?AFe;vd|&Z>&3BIPIp3yyd-?7Bx2kWB-+aGq`F8pn?>C{Z
z?Oz^$e)Ng!)A5fCpGrS%{<QCt%jYFulD|p+nDn#fm(A}Fzq9_V_@nyQ=ik5o#~8mc
zg)#4B?qCUEJ;bWS*35Q^O^n@;{Q+Ae+jO>fY>(M&*g{w*vOHr}XRc@Z%~->z$7syB
zk>TGz?Z4uG_<k?>S@Yx1_ix{|e%$#!{aeykwXZT?Cw_VGIr#IdPb)t8d|LG}{^RS9
zt3EM(R{Y%cx#-KWud3f2elY(`{yF96`JZoo9{&01=c->?zxVul^Rwya!Joaq`hQ3L
z;r+|~PwM~G|M?6SjFC)RnQyVAvrb{%$@-SnoUN2?6`LD-Jcl7?59bNa51e*fTHLlg
zk9e-|=JQVwm?;n>P%j`P_)PGCP>Ap`VG9vWk#yl{LViN2LaT)?3q2O{6pj(uC3;>g
zTf9a*Up!v?w%7sD(<0d-nIh>Td?L=m8->maeG{4`Bqj7&@SEUHL3Y6}{4e-a`D}R?
zbGLA2vma#5Vo_seV)9`8$e_*mj8U7Zhmn`jj?tCz1H)^ESjNkY8cdQ*{fsG$`b;j&
z$C=MCZ)TpzoXPx`NrP!2<7dVsCJ&}RjJp}z8Os=r8S5F={+ImU`tSW;%fFR>O#dwU
zbN!F+pVr@kzaRX%_UpwjuHUx57yg#}WB2F9@0Q>8zjyz-^mEyd^6!b?=6|*Q`sT~b
zFAKlC{xbK=g3mKP?fYo|apFhLPftFc{b>Bj@Kf7I?T<wt-+j#a)cuL~Q{BhBj~O5N
zKeBw>@Nvzj$)9<?{QIo^Mg42lw@u%l{doVg>(_x_LBD){M*evIecSi8?=s)3zgc`+
z^3CRZ)At$QZ+>_EG4aRVAACRe{oMU)+i#;k?0+WwcK>bvo9B1wuU9|U{A~Lf^>f7!
z;qN(LbH2oUe)4hGhx6~}yz_Xw^YzJ>*)QT>?0%8{V(If+&p4hbJk@<-|M=je3y;=4
z`ux!3;rj<J55peae*ED{?$cFI*F6n;dg)2~lebTvKb`Tc>zUg#`DfRjmOT}Jdf>_2
zC!e2GKJ9*L{Pg*gNo+b?!Q9$h8#%N%4sqP%yu@kFp2^(9=*q;w62n%+QNp3kzMuU$
z*A2cmg13d{3eM&a<J-#jRv=sWrf{L)blz)R7F^4@g1Gf~R&(#-ILSJVwVM4BXC==T
zzH+`#+-Eo)I88b2I8L*e{Wtq9_4~x11AmwOUHs?%@6JEh|Ni{<_`lTu7r&i;Jp3y9
zRsP%C@6|ujzn6Sn{rUap72n)`HvR1U-uz|dNA3@|-=F)y^Lh1Gvu`^-cYWCV?&{m`
zudh6Bcr@>R{eyXrB%ahhse63u(Yz<(&x>EQy!`QU=L^y2;?I^m)qQ^FRrlMkZ`Qs_
zeOdGB@0-u>>fRrEcl}NEE2EbmUv7M}^n>S@`(LfU{`?&A+4l3fPZ1w3zb<}h@y6og
zp)cFN&G`P|yZq0kznp&e{eJtq_IL0v=by`dhW+(rZeah;?#{N8Wgd$Q+cgd*?#0~6
zTnQX{?336lIT!Ho3G@o;2=wqt@GA<i@~`1u%pS#B&Nh!Tk>?YyAKx?HH@qr*7kEo}
z6?h+Wi*j{xEM{N8v5PyL?-FkmmowW1fzi{7M^7srJ*{~3wBpgzibqc?9zCsi^t9s9
z(~2cWPb(fht$6gb;?dKJ#YRsn9zCsi^t9r09HXZdkDgXMdRp=5X~m<b6_1`)JbGI3
z=xN2HrxlN$Rvi6(^t9s9(~3t=D<0s}ic<s|ge!zs2x{@y@pAL#@RV{*U_Zz5lj#oA
z4i*dcDIESB0_^6jnam>0TUj*Ocv<%_X8&XO`|U3mLn>oE<2Qzj49ZN4SSGMpv6r)J
zvoB)RVY$g-z;3{4!nuGgl-ZK0oB1${Gs{7yzl`e4S6SC_*l{o8-OHE9E61(JRn8s5
zcS~TMz-`{sT#q?Bxb65H1wIJ;6Q~kAD`YRSNaU1orqB!l8$KysIevTLNn*a@n?#Qb
zrwI#-{1C1XDiXNEXUVsRPhMb+;7lP=A#s6iywyDWc}oSCiI|FT2!`=3<n`v0<=5wr
z=kws5!LyD>gYT<AsBnvLqhK>%0ndG&CVoYsyFyO{?+J7W*a{T$EAlPiS;TF`EzNzN
zdl&CLes6(FJ~o~^Toycs`6YziL_P`65I!foTI7uATCvMwf?}ScXG9Cc&q)|d)=8*{
zDGD9r3*%E2kQE9LzA79pA|_fZYAos~a!Dvm@Hf9T|9gIZp)irDq6<X1MPG_oi>;LK
zlaiNek>D587nv_|L~MbCpX5e~RpOQsE2KWlyp$D`n=1QW`ibOKi9*R9>9w-oWxvbh
z%1oDGm5r5UmGzX7m02uPDcdGDL4LD*zuaCKMrnDeRZ=z5o1~N^9*dL+Sqgm=78K(X
zcN7y8T_GAEK0*A8=mB9a!D!wh&M9p7S&cdRxRZDUxih#Xa;@UJ$+ed21Lq#j5U#mg
zC%9g4m-1fbUCwio>lSA;_YdAneEqzexXZbgb20Gv^2+la<U7Z|TkxjvBN1uQ3Q-qP
z6=4p6aK5*EwSr4USj5~#c||@7oe`Wbz{P)v_Y$uIUn;LDPYMqs?<sCs4g=<S|7ZM{
zVf1Dw|C{)G<FB8;GJk*kwfN`yAJIRgew_XO@B5A))jx%QW_)Y;<n-amhpC^fzb*Lo
z<IAs4Q$P5<wRz?Eg6X;ZGtZ~FPqsX<eD3w?;j6VTE<Fu<lJ@k?bFG(aUO2zF@Z#gk
zTdz!Bx4e#dQ~b8z-Q4%HJ}&&i^n>S@&`+&zvR|fsKK8lz%l9vbz9@b9_=)l3r}rm5
zJpJhK(dqq$w_D%NeXsrL?5F!53f}p?o&NUS+h1?GUrW9gc%A=>;nmq!-Ea8by?Yn?
ze%br154S$*eeU_Z=+piW%<p~PEq=G*ee;Ka55L|`dGqsS|BERvPQUo}eErjZkM=)g
zczpdy(v$lS|J>hpzvMx|!%Yu$A1-@v^S;!*raO1<Ot^RKe(wW;N4Fk(J}rFq`&q*?
zo~P#?xjlGx@9X_-kM=&9`()oE%?D2RzTVw`FaN&({nC3&?(V+(_1=sJ+7EBs&%Jx}
zcGR6!_f9={^3d-Q?<0waNA6F#_v!A6yG?gj-SxTWdH?hMJ@;?ii@3Y;R_S%MYehHe
z?v_5{dV2AR*5fY^lOI|=m~pT1?*2RPZ~wgg?QZ$Qi%(8HJ@J_LVebR>M+T2~JYM<a
z@YBj?x1XJRX7p^=)A(l+FK)cle*O8)?DxMvMt++7!R=k^Tfg^@J_&zo`QG*Y^S7vP
zj^En8o&Lt~-TZsw_r*V|e$V;O$9$3bC*$1z*Z$r6AI6x*)WGb=x{iGXXF1m;t`pop
zc>H<i^1S8l;+f1BEx;u3ns+wOZr;5D{v!6`GbGw1I3(7Ivr6<yzL8FqJubr~9V>NM
z+F5Rvyp~+M)EhAc(Q_isA`-$yLd8O#1^)`H7RV92Cgdl=AbLt<yGVy<zt|VCP_YP+
z|AKP_PYPcWdnw^5DJdZ=7Ao>s_^Ze+vGwAj;-^HbMR$v3OU#j&EVfH{ieRI_EP)q-
z@*<nWbR}xVAB%1g{VeuV+(TlEM7d<Y6u*pvthKDNjH%RT@nvGt;-!+$Wq!-8lvS6i
z7e6J&BJL*6CGI1ZF7{pgu4K3LZJB(Td6EiZM}(DxlSD3w-Imal%olGFVG{Z!a895^
zAWEP@P)}s1c%9TOX(O4<QtQR+g`xyj3AT#-5p@!=70ed6F8D_1rjVl0eE~^<`vOHm
zeIm8um6DzkWg=Drhj|(KTKHe^f8=||8_6rp8^9aIE6cl@_Z^>rz;(etB6<=>BpD=C
z#lnPd3at=f6gQG+7M~&-CDJGIR>V%EMkq=^h+mlhBL7!Gcd<+<bLsyQ`eF-3PKg+c
zJ{9&BOybYy%jDg`qruzF=OdsXC?If&Zv+2h!Q(>Ff|vOUdE5Au1%riu3MUBn2`v?x
zBy28HAaYM+ort>dYQZ^zvcd;N1td%*3dPQeOckjRZ53N2RwlYm_?qA&eiy!Nyi$CZ
z`EK*M@XB&4bFpzraUNp3!kogy$g+q1I@f3J>D+Q$#q269%8dH|WB--@UH-f6*Vo_v
z|EDl1vn*!*z!<<V`M(YW50fH`E$eNTuS^UKTmDx6E&a#&U+BO5|DFGB8DB8nU@Boq
z_`CM^mES3U1sJ|E?O^%9;?3gAY{X>6n8%R!KmDKQzq$VlnAF(TaqMM(#=PtQtUuF#
zN&Sld9r4%yU(R3U-<y6oe%JZ-<nz7{3*Py^JN{1nz1n-{cj|ANUiv-{e5U%G`K9tJ
zr&kkSg}>&16ZwYY&97IbFT-Etz4-Ui=Z)aIw0E3uZ@d(EruL-fiNw=yPo_LEd-Cw{
zg~ypsvY&E3+xWEU$@0hfPc)vrc=q{e^b^&`I*&>pvOn7YnCt20rz@X{zxe-B_f7KK
zw{PCOe*a4QW&87#7n-k+y^()o{p#Y2RWB>v<i87f7yZ`h&5~DQFWFvty$XJv_xjx{
zn^%8cZh58rX8#+P*OD*$UaWoD^t$2gk$3;!@xH(H?#5g5x5e*DJ~(~k|0wwJ(#Msb
zFMcih_V$awr_lGS-UWSl_i^vX_wViAI=$NaJmcBfXQ3~Dym5Lj`r*aLJzsRcDSb8j
z^y{73TgA6u->!Wh@Ja0J^=~u3E&a0jlia7;Pc5H5ea!fn|4HN9q+im1CjT=3;ri|E
z*Y@ugKcD|x_bcW1y`N{lRet^c_2<{EU!%Ug`d<EH>E8qY^Z#G|<M7k!Tg;dHpC^22
z{5s*=#_wU@Bfnhyu<L#5hrjP*-aUT%^L@vc`9H<~DF43jBjcOZmwTTUe!BhX(Wj<Q
zfgd-$&wStXLF4204_Dqtzqj}x{;Bx$y3Y?j*?-h{FZ{OVb;IlPZ@+&~{dDd__WR=Z
zCLd0G=={L&Uhl2^+d1#EK9_u(^PTs*(zm(antqi1;{NmUZ{vUa|M&k&{$pSWWlUxK
z%+SDK!RW%Q#hS}z%Mr|VhU*t+J=ZUu#r(E{@q%^yt9cZ;Tey$&?Be~*%f$DWuRzdM
z<htla(M}N$;m<+|B9h{MlKGMc#LGnEge8Qw3pNV63N{G-6=V{cEEp#!A-IEo39lMY
zG<PX?Ht$P;-@@BOt_i*27vz)U`^IO;znpIt-x>Zk!T&<Hgl&X%1zdO@bFJga<zFTk
zEi5HsBdjcVk<Xg<E6+yWxqQd@&kN2FnJCU9by3Pxl2h!1P`lu9A#Sm5$;Hyy()E(n
z;^AVqMLR`Lio^?F6p|FZD=|&VQqooIkkCVZOa4;=%EINM4&vLyAB$ZU{U{<OGFy13
z@FU?(!XJfOM7zaTid%_37j)!*%A3!(Q((DpgxG0uJ@J*I^`c)ybwpSM&T%VouH+Es
zTE;zxJDlq=$1V1=?7KL=asJ~f;b!H&!#Rm#F}peYMm9;-XH1V6ZI~`IWid0cY-WyP
z5?~Bvn8h%SDUkIX+f}v+Y=_tmvWIhOas_a<u{*Q1v+;9;aawX-W@lvk#3IkSl=VI9
z5w>IO4D4H2`k7N$_OP<En{o7TykS>j?_(2Xo5~u_qQoT5Q1pK{!*ym`_G=u@92?l&
zSa-9qv&OO(vmRod&nCmc!8M7SgQt}HH>Vay8T&1cC~g(rHN4uq>$$gbx^ZxE+~@G%
z+R1&M=OFJzzBl~l0`K^i@oeEL;f&%m<^IAe&u_uk%X5W`i_@9?468J2K5G%%efHg)
zd%25wJh_gt7BF)BZ}?~Qzm9=}=`?c{t2vtr+XS|!?7E!SIahK8^OW-)6PPEIEVPI}
zmFG5>Gxr@HR(?~#OG3+pHwm)}FBLQvh~>Y<x03G$e~VDIXp1<%M2J|faIWAj0XD(K
zf@wl`g!qLugm(&+3R(%w=C9-9;yK01!?ln{ls}jM7H=i@D~^+FGg-f}%wbu{(#7(Q
zc?R<eW;<pbrqc|b|Kt8G`FH65TZSygeue}8IQ~BW9sk?@_y1r1zs!Fg`fmO$?Q7K6
zEnjwin*KrKz0<pvcM2a2KCbw%`u(kUTJNmit$U~W?(dt<H#6VZzv+9O{`%_crLXH>
z3BL+_E&jIQUE})|?~UF^ys!Om`{R~ROFl(>RC+J|cFpTeubN+(zW($2)SK6D@4nmq
zp6}!8kDed?yxsan^X;W~kstnlkp0N`ap#A#ANW7c|KRjN@ng-W<j-+m)_-04E&khv
zFPfhcK3w}C^_k`C&#xc8t^O|meezd>FS9<c`n>XU&gX#7<)1%&PXDs(v-?MhcUrHn
zy%c`E;qAxwR-a72hJ4rmp7hn?^U9BZKAik;`NM;c&%d1cvGiBS&yC+^ep&PR@@M-m
zE5CGqUH<LsH~w#xUz|Q`e4hAu+UJbVt3LC5wfye<Bj?+VPhubLzEgX*<89Wv<PS?e
z$$pXj(($SM!|iup-!#8ce!1#p&1>Fw9v|vIB);4FdfKZ^ua3ObeKGq5+w1>t4!;$C
zXYfw+UEo`T*Y+<mpS3@Wd|vQ;?X&q$KR<c-WZRP)kIf(Nc|7sSktfAZc09iP$nsJD
z!^Q`d_haw-Jbd-U^SSkNrsoTubH9*#@$dPZ=MSGtygc%<>!rvm#y5B0-g>v^{pt@g
zAGdwX{haqz;#<>Ky{{*}x_ndkmiBG=cj=#Uzfym_|7rM3;Sa;V)Bk4vZ)HefJi^q^
ztk3l4|HFSb{?+|o#qgSuh50J;J7!I0ZYDcMMaB(`MNHe6b}}h4*E7FoTEzI}|Leax
ze-eLZ|4#X%@Gt)V(*L3V-T!U<^Y*vipD%wn{+a#1$WX~#%gWCBm&u%Q8iN_5A=4u!
zFP1rMkJwq*U0A0wH!+_2|M7n`<07UemJs%I&Rou$>=CScncg!<G2}A%GKI3na)feq
za82TT!qLg`j9s1mE}I%h1h*^S3BCs2W87U_xm+gP<~*BtSp?dJ4hp*p3kV4dHVO0z
zJQm0jWEU0_`yjz9wOVq8M7;PK(Obd}!plWoh^>{lE6FDHU*fDdmpFqMyT}v4Yl0mj
zbHv|DY>>DsZXw<y_Dp1(&}_kHf>VSZ3w;)H6Xp}yF0xZZS7ej$KjBj%E~0s&i$q&R
z&x!Df<O}~7$`zIuH59j#;FP!}enNs<YMxZ8l(tl{w1OOiLX^Tp1!Ki)ijhjQm98mO
zDqE>^s932?R(`Bxq|~A4uQ*k4o8nQ$zl!S=s}%y}+2vW~=gYCnNz1Xxua(!8Z;<tt
z-Xtk3IY%N%;;O_{$pup1q!-9|%VbD%Nv)93keDJ-AlWR{E6pGiAiY8It$3~&pJ=?u
zZ;=^dQ4(g7Yb7R$pAhR16%g(csOJw9U=flPHWywjBqdnLXU#p8!+^b<Z4>K3)-P;*
z>{HlQvq&&m{a5_k`)BUoJO5LeT3HUWK4P86vWM~WUzXobe{%j>^egRm!k=${ul}!R
z==(qa-^PD6|F1JVWr}C5W}n8fg5w|i4K_ozV7AR{0ql7kYdEiQ*>d0KEMmXLI+1lh
z>r2)RY&SU;aQpJQ@Otuua=+vf;O^n};j!WU&D+f<&acWZ#^=Nn#&wJ10y{tZZ?=2v
zLY(cK7dYmzGqcZUvtvzVZfCm7G?lrD*@}sY@i@b5hJF8+{`>y-z~3!@wf>&}t?^6Z
zC-2Y4Kkxt2`90-V>(9y`_21pTv3-sE;_&79XPwXWpWHtE`1t3;_V*Lt^}YS@X8oI-
zH|1{*yeWFK|FzcZq*sP7r@YXAIpx*#*VkTKzL9yW^EUeR(--%jKYL;Q`rcdl4+}nW
zf2#jj{UPeT@4J$>+;5}aTD}W^KmWtmkGDQ?eKz}4_~F;P7w>kxulgYT@#4ppPpY2+
zKKQ&#dt2~!>sya^HSYyKYJ9r?@!k8MZzNx3ytw#6<F)ELw-2r#e}63g6!&q>JApUZ
zufkrYytw#0>V?RwnXfZnuX|bde9BXSC;pG`KYsZ1=!@XjO>ZRMB)$6e{Oi-7Pg<UY
zKGA+^{oL;5*_YKXte)L}BK>sp)9p|9J#Bk-;rW>tGhdWFFL*ZZ>7plR9@{?-e7x+*
z?PuaI%AW6i`tR}YN6Q}BJ`#Df@A2_x#V@u!mwfjANy8K6CwCutK3w{c^~ubqKc7r~
zyx>vtqh*htK6?L%<+1$Zd5@nyx%qVZ)7Gc0&m^9oerowR{NbvH&W~q2R(tgKf%e0_
z4<A09`*6|2Z;#$Q@qc>cao(f2N53C+JYMs7_2Xqv);tS*aq7k57s)T&UcPvp|1ST%
z)(78@&p+<^sQ+=>hs=+upWlDs`)c~N<J+bmkAH>#nfb@`&&}VDepmfT``h=gkHL=l
z9*Ysn6eeaSTjp9;FZNcB>s*<9^97FcNAe1APv%tTEaY_JQs-I1=O^$~;Ix3GfHVIe
zK2?4n{vv)=f!BhDB3_~<B9=m*1o8v~1QZ1F1$9N-#5YK66MrUpQN&4PneZav8zLOy
zWs-r?XQX#XZIGBG#w2PjQY2C!dPuB5LRadX6pxg<<YS2i;!dI|!n1@gh|ZA6l3FcQ
zA?Y9<EfOLq!e7Og$uA&yOz5fbM&TBrJ%W9LQi4$efdZ9+7Q$OatVP$0Oc!nzx+Taa
z^jzqzu#9M**mrRgNgb&usk@R&lIO%d#d^e^N_b0q$gs(1Nl%uFmYN_rRU%AszjTW1
z3z=r=T~gPi4oWSP<dS$OW+TQV79#dntW%srVzPvTq>a>SsXoc4;y=U;#n?m>M9zs^
z6P+O@BDPS(Ld0Bjf#`CPjly}t{lXuF%SD((jl?)4TqGGJ7m3%4T^Fm8cqX}1>X5XU
ztc2WJxw&$;WXom6<OJni6(p68DSN71R=%Xvso11&OWsj_g4_w&eKJp_Z%MzEzAu$1
zQ7bw}SX}tI&;y}ZkpOXRNq&iwA}N9s`KR(T2(SzM6PPLdO|)KYfvBR$7r_MnI$l#=
zTfX`HvjzAB4+?DLpUB6}cb3<T*Mnz1_cv~F-g~?Syn;LqT*ukpu`FleW2$E6W81^d
z&AyGLk4cH~?|+g1lm5;7=kh;<L6Y$m!`1(e|1|z=_<8yJf$t?hU;lRa=gaVgk)1h$
zxrr&B;o#rpzfFIY|9tk-`FGl%$$uXG-u~;xPlI2FfA9Mn@c-t2h5v8=`u^4Wd-+en
zp9Oy`|1SS4^iS=73PU2pnSUyOoBk~O^ZifI-${Ru|K<3{@o(qfm47$<J^$zbulk=p
zKNEj3{E7ej{O{zy9e;v;zxZ|Z_pd(|fB*dc_|yDH-uLO>dw-n#Y4ywd=eh6w-)4MW
z@g?E&nvcOBN<Nf+y#KM~qvXei51Q|b-zL2Ad~@cF?As%6?!69p{q42UTd%jXU&p;X
z{QTOpsn2wttH0=dQT2S+ljet;?ytY^`{2fd-iIoWk{-=@bm>vrBesWn_t)JGzZ-t<
z>wUL}p^uy%w?1C==*Gjjk1jr${!IS)>gU^EO1@S5pz`tX`=+-SUaxwk`YP)6vUl#E
zSie|*miXBEzU+PU2knn{J|6k><}>ftonL#uF8Gr8S^ksdM~RP|pMHH-`R4uo+c)!X
zQr|qjF@AIUn)>Dc=R05czDNID_v`X6iC^qLBY$l8;r)y8@1y@_jB*SY|AzdL{qy@b
z+wb!~Eq@;T>Gj+C@6mtP{#!9v|JVEb=XdR&34de%$^75-KaycPV;-{#%SL8a=4hrz
zjB!lMnM+xCS@ttMVz}|I@9(<5oBmn<@BOFr_ww(*zn1=*@hj!m!=JW4C;#yL@%hKE
zpGSW3{&@BE>1WqZxgW(oMt=D9uKI1^n~XPdZ!6z^e#7xb=XK4ipjUfdrM^D>TJ3en
z%a6}}Urc)G@apnQ&KLKe1U~e+=Wyrj?Z<ac-VM9!bcgZIqdOY+R^5AdclRCtJCS#D
z?}t2ef3)$@_eZ}Ug*~48r2JXMi|Mbp-|T$j_s;sG^XIV7)t|yY7JtzC*!ua!*BRft
zeklIf`t9Y{%5R)M+<&F~G5IU{ch%p0|Nk)hG5Iij`Rn>O{a+o!9VRx`*KDF3{p`nB
zH?VACNoGlBUc_{ZX&1{c_K93_+|^vuIZtp1aY}MC@kt5Z6_yuWC~``8p0K8ff@rO1
zm&gU-7a|wLUWwloUoJL5<cg5J&{Cmq!Vg5{#J`IPiYbd4iKq%o2)z;5!mr9-Bd}EX
zn3%uBWQh;rlf;5We~3hj&Jqifm@DyFY@cYPn3?!{vFV~_!u<l4{IdL9{BQXd^V#rz
z<7(op=k(`F<YwX7&+W|J$DPc>$oqvSnrAcjWNufU-@J1Ll7tgQ)y3|K&J`^elayeS
zyf2|5!6|-IG)?rE=oYaWF%D6C;X0xFLT<v`!pcHn0{nbdJPF*>xHEWcd3?C0aqQ!m
z#krWXo%0`O78gI)V~%L{I<|PWEY>Sbz6>Y+doZ>z2eVqRS+TXUhOvY*r!cQ&@n-wW
zHkB=aO^3aK!<F+lM;f~ds}@Tbs}nmr=VLAh9#@_%+<n|DxrKS0c_;HJ@fY*k2!soo
z2&oFy3GCus&wZVH1#c|>34UGvE4(h;kJ+7A3s_#T#Iwq?YO}_$8*?w@y~M-EWx<}v
zdWglGbr#!Ij?-M++-Er}I5x0{vVUMZ&uYX{$?U*#pS6MAox_&hob?)W9`iP4HCA`_
zMh+d0=WJfAnk<SeLM;23%o(E?1(;b`8`w6p?O?HEbo{sc&(q)Hf7bq8`oEaLjp5V(
zng1vKKmNbx-;CeKe!lt@`nTx+62{%kGg+KiJegye7BD?$7GaTQI`{wezoP%;438K@
z8Q%Xp_wWAyg^XvIcd$mWCA02e?qup_+Ra?XdYVm{or!%qyEW%GuG`%2xdXZ1at3mo
zVqeVB#~I3L!f}`FEbBQ|efCWpjGRFn$?W&p_&B7vPw;m0J?0A%I3rjqq$&JQm|sLg
z_=4baf#U+5LidID3F!;S^A+$+;PU4*<!s<|=KRjSkM#%hb!K0dIA$rvFaI9@4gB~0
z|8&L^j2R5Y|7QNV@XPII&JX4vG2i>Y*?dd?w(r~WZ|YxPe4PJ|_wD^R7H=iq@xFiZ
zF5+#(>$NW%Ua-E<d13ux`t!rjt~`7B{Kt#aFAhGN`S{7hqQ|eF{eQXaP2~GuAD(@1
z`XKPZ@<Y~#zz_G|D}Rvx#P!wq`;zaAzX^YR^SS5qi_hP__<fW4&iy_5+pe##zcPQn
z@ZI_Q%CA;mTE5)<rtwq$XWI95Uzxvjep>j+_sik0%HOVhb^Ds~_3F3fKWcyW{5<t@
z>95p3GXHs*`k6m7O=Wz{P{Q!yzX?Me<5fm0#<dJD7$TW&v&6GGvL0pn!l1_Bz@Wj%
z!=%g9%E-si^KbIMfBzpa*fSjZXZ3f^AOFAg|Ni`&|L^AC?7z4EPWo5!uj_BbpSE9m
zKbCyE`_2DH@y{neCV%7na`ThI=jWe&zU=?};G^n$p|@M#_`FSj+wtb`tEv~zo`pW=
zfBEU<mKU?0GCUG`aP>jpqdAW@JlgSi>Jz;u;g2^yn)9&#{@gngZ`$3Mdz1N&^WArM
zCg1*f)BNV;n_jmW?*`wiy7%&~-o1bKH$5_a8u+aG>5j**9{qV7|E&Cl+{^PXPQK83
zx%y?#t0%8@-+q1T{%*#*==atioIbF>=X}faTJDwm>*9C+Kgxaa{Ic=$zR&t!l)tci
zvHY^-v+k!gA6I>r{eJ(a+pmY;qrd$B_~C=uhm-H0y#Mh&=Y8S3DQ`EwJ^OCn`{nO>
zK3w=X^-KKsh@VWq9{qgw^USYDf9C$5%UH;CiD?bfcSb>`Da=P$-ZSShKKW<!kB4D7
zlMTx=mJclVnbn!}8Iu?-n7%RnV@hM<Wol*OX9;8b!Op}H$R5f*iDLrS8}1VB7_M-x
zD_o1YlDV?E4sxDlXJK8yG?&SkwT7LMy@ur#<3a`<#{EpDEFG-d*m&6Qu-UP<aUA8U
z<Xz4e!~2u_3(sl(Lqgj`B1J!lo)o_!*(H@J)hv}NRV|q$Atk{msVBuFWh602bh~hu
zu#BjxxShlU@hY)eQ6VuoaWV1FVoSuU#TSX1iLi=Ri(i%eB`GG}AiPn~QSgTVpWsWu
zeBpS}KVp-_4~u^huNSuwQx**rH4&GT3YB>+13IiYSa!4gSw&@~R>k>>bxOg?sVYut
zIvU3`R;&M4tyVp%Ca<Zl{aE{-maAr$dc9h%+9WkCwZE$W)QmNjX#CO0(VU~ftaewa
zSRq{AME<axnd}DXaOt(uJ<>;|A|!>ylSKT4_6fcaauv-Me<9v2<}UI`P+ve-pkL5h
z=zst>-%f5lE-_9wj&AlDY~`#QSYER{U=3r-Vx7VKnvs|BHp46iQHG}fi~d>u4f=EF
zPwC%Ze;j@@{`~Sy;M?(UGk?7L#qsyT-`{_j|Cs+t`n%{~^nafJwSV(}&-nG_xB0)U
z{}KNs{=5Ib`d^7*^M8l`OaHSo6#Q@gcjoWSKdXP2|C0Tg_#^da`tR3&R{lNpxAE_l
zzmEUi8S|MdSwFCCVgJppz@f@n&i#QWms^u#3F~8)F4kPOG`6R#-&on$3Rrbm7qUKJ
z{la>b&46PWry%z>o`rl5`Mdd#@D=mN2tE>$7j6?u6iN|}7vT_L7UmbS6)@p7<ErC0
z&T)^+gJ%}kceaPjzZjVrFEG4eU}sEaJkPk2@gu{Z|9}5|{hRwo{a4bD-XF(*y8LSW
z`S`o|w+~+~f4=c?=KG6pE#BOE74mB0EAiLeuN7WfzU+NA=gH+Kq0isHlz(OY!sS`_
zQ;TQ9&u>0!c;@xo`bGW=(U;F&oqY57t>in7xAm{pU+;VU>NVq=iZ>VE$h}Q{d*e;*
zYpqxHFJoSMzN~(E>ecTzVej_6Rez)ZddBPTZ=SwYeCzOf+RKm^=bvZ25Pf<3h3^Zl
z7tSxVUwXd!`pW%v=Ie*AcD&s4V&V(dmp(85z0iGm{ndpx9dEC`jd;iQZrz*u*Hy32
zyy<*b{$Brm$UF0QUhn69(EAwmLGs;|*DSC5U+;Y*^3Lo1?f32Pquz^t`0?S;$2Ff6
zKfnKU{L}kS=AW}a8-7mzWd70Y!<Y9-A9Oyx`S|wZ*AFr8pT1r4*7;r9yMu2<-#mP|
z_vQE3>Tk7PCq6fSvh|VLBaTPyj}#vBKel{y_rdb}jQ6YVZM!@7PRVW8Te>$l+|;<k
ze^2~g(p{}P?{3|{)p~pLZNJ+GZWY{$y%lk5<1Mz^g}20SD%`k!W8v*D_qZP#K0JS)
z|Ng88ijUtuk$NWb-0X$+tLisQ?;gHe{r>FxXYYQ$m3~+Je%8n0&uL$te|h}H;LEbl
zw?3PFVgC~JdDSPw&wIa?{|xwJ{MYbL&9C}ji~iXD<NNpF&-~xUzjgj>`}^tN<o}cY
z|M)lKFW=wa|5_MLnamko{`dSf{X6w<-M>!^TbO%Uj9AXGu(2QH+{e9^XCe<X&pe)N
zeluYcu`=;mv1pP1!kS{A#7o8Zi!zA3719=-EplA6S@gC@faqt@V9^kfH6q$#FU8tK
zC56icUkd7r^oX{L^a&pp?hsxnoGZ#EzDex4$PXc1AvYl{p%a4qf*bh{^A+<I@Y?hE
z@UZg;b6#MHX5PX2fMY6`C-+tETAs<gE&N*r`-G&06oveSJ`1u73yB#>luNvpI4*fr
z^0nkHsiRVtBr_zk#Kc6?MP0>SiN%P07XBpYAhc7YTs%g6p{ToPsz{E|Ex{C_9O0?L
z8G@MtY(go*B_i!2=Y@6)x(FQP-Oqc0|BK){VL@R|q4z=p!uN&hM3Thjik}ugB)(Cy
zT4s`5y1b;El-xrFBc))a!^#Y*Eh;uj3l-idhAJ;oW>aBR30Cz{msD$4u~z%4@m`}w
zC10^hMOpo=+6VPx8h)B#n%wGX>RFon>TYW8svdHWRfFv7Z7PMn{b3T*)4nX5&*j3d
zC%9GNwxSMC%l}IL8}edu<<fqV?^(}&xyf-u`;^XHzIWdye5>T-P`awoA<FUp)>o^y
zHy_!&KP=LxnIpLCljNU#!3AudA2i-<_&%R2iRaL_Z9f-^G_g*)A%9)wPbKe^ztyi?
z-}HV<{_y`rJ%gu$xkSR(@+XU*8NHvxaD}IhYdu^2-&-&D-!prZ^5)D>i{H!MY<+c$
zX{lr=|DER*=hE+nesTQz;$zW=PY-Y3o%F^1*Y<~dZtj1j@cq?y^^X_c#(ih_Vg6$B
zbBk|^U-MpX`nc%N_1`kz3*LQrc=$Q@51ntuUuON$|9tyV%(I=Jga2RrdF$T1%PSwA
z|K!KFQ{7hHlzs2l(&w*U2>j|{3ivnm`>Zds{uJ<d32)>|V>t2K<MY9{W<LcOlHRmk
zn{dPNnf`~#Kl*;KybQSe>8`-7?(26Roqu2ahk^akZ`S8)o<@A!@pHzH<*&FNTD(sF
z%J}ZelZ<DKPa>{~Ue<Un{!jJ)u2)RgA6#a<b?kxqn^}LJb7ZqLyw$!Hc8}|w@w?@B
z1Mcd7<6wU9r~7Zkk0sAMp0$4Ld;jaz?>BG0s57en5Bz-Oo7$h~@2`Ko{TK4p?qkh&
zlV2(SDp`L2?f#VWOHOEwWG$P;_YS5`{#=F~FBg1TE@Y(GBxL!s{CUuG?ypz>-uPzy
zdj8AC4;nx3v2yWl`^)-%#^1}_3M{6qYXq+G=CiG0UdUP{vQ?^_`x!g0U@reh-ZHM|
zf13UUab039XU-D5DX~Mam2)4P4a@R>=Kue2wTmqmxyE*dVJ?>qPwVff51+pWvvP6F
z{ipi<=I;#FCI3vGb3Nb7D9d>5>E%0G_t!iye&zR8`OT;A3BUe6eRK8eb(e?I&(=KZ
zy7THL&z;*3n4e92#&OH%?6%7uH%!k3T)cEo_JQDSzl&ujFI)+IaNthmtvA;uoe#bu
zc$eY!<12EPe_qSFbK+k2-R2u^*S_2^zZP><?@7aVJBH2Q?H_%={N<v@)$&KIAB#V|
zdnNsF`F+V(bKf?;W`DQvxzm%iU;h7C{CeGchriYT-v3(nLFJ*@lL_Be{`~rq;hxpw
zw_gr0D*U?g>f^^!_V3JpUU@x!`Y!aRB$GWG3sW&m3U?LrjxT3En0@bNI>NP+>pS~O
zCg-0*-=Y|H@My9x{dVe?CT|SS3kE}$2O_D;`m!b5Z@9Ar{Mo<o)+*eR?B+HUVN=dl
zZjy-NJ;rIoWyPt$ttpr)nkr#0m!`xj-zu8K8P1Z%uP&1)xl4GxRI2g{d2taD9$}to
zBERI8$o7h=i8YHy3I&Utl-?(~T6m(^U75!s&Wwxxc=2dUYDwJ^kCo7sG8Ug8@R9o@
z=L3$X%sc=6Wp@)y;@HZTAU07dj{hW+#CNOj=^S6gV<bv>l>Tk}qrjoSx%Gc0+dqL1
zECN5@|B?Iuj5(Iuk;{oskKgaV$zN9XYNklGSfNE6PJhocU1X1DV_?c)EMaB*|Kr;V
zhHE_1d|$X@IQp3D7}?lo3uy9%Func%lO={-o%1JerJx?KEZ0I_8SW<Dc)9Ben?$||
zRf;W^eXATGyH4?mu890&K2MRilI*JP`aLR%a`%+qEB;V6*F2=X$->dbQ^Q}@L!H%R
zr@4($mcbN57R~+gtEH}rZjyeaa!LNMsDXelAD_Sqfj$va*<eW*{&JBwQWiqjxR&wT
zh&GAxiB1y|=U&Xp!nIz^Rl!+eA9K^M{me`G@ADmFe*BM%C5A(V<38iGA9dene$#rt
z^BXtM3_%;#8Q+7x=rMBhZRFBrl=yM{qs8wDJYV_aIez><_f7Oe$eTZ}gg)mm++k__
z>+#{&^WCoxeiZq>o^du~`zOZ_Gk#6^Bk@m~sq4?tACvw${VVx;?9H3!tKRGX3;wn4
zxys{huW!9m`N;8e;orl5-v2QA@cV`O`=tyGT!$H@zZHEh`w;iZo!Nw6h2zzqv)@<$
zTFKJIS<bqXZJMyS!d2N+-rWq#SQ_|d^YL&s^CpVwOP!XUB6gaGi}$N!uj+l3ZxRwb
zz3g9j56DEScPsV^$#E(2OyOV7vx6g@`<0-d*a@K+_A&+!W&^glOp3p5eM$If$asvU
ziLw5>)z@_lHq2Gu7k%OVca)8b=K$xtzn{KH{s{gT$E3z~g*E-3{AZ~b^Pk$kJ@RqW
zhnP=xU%h|M`K$S3;X8r1F7M91kNYtBW!H<~Zy)|@f4BMM@nz<Z<$u2YKKa$+gUtKY
zA7_30_dECRk6&iLjDO$wW%6tKk89s%{CdTBn>mGH?Y}4gZ2!jo7vkK(>&#aC@7&)G
zhI`Dj+3MKUSpWVL`6cp4fN=xEyuXuJ9E6q$y<^S&ckj=Y9}Yii7$0)~<J-+2$Dhcy
z@6Y=GT5R5IQY?A@xBdF`*O@u#fA;73FW<d3`%%Xv%l?j|n9YWv`nSN3xnF#L@^ZKf
zcyjMy-Ndx{kKlL4AF~)waII%^{5$FQ$zSR}YJNrhN&EHpm&~vH&&xhq{NVa&@y_&#
z#Z&vYYd)yH)Vz1?rp@iMch@}7c~<t8^Nrub_jfGsdR|+9{{C6EvsPDD-o1On<*fLL
zf)jU7=bkD)(6_((+>h&5FAAL~Iq+sz?%vNw<{ZCy>f@y!m*P+DJ6?23{$%#y4F}Rs
zxL^Epw)RxqS*8=5htf|xJ>zhu;auR^oHL8geLksl*73pBCljt4Ub4Apa%#b${reUl
zUUV+)O7QhXcVZs?c)IVI(cQV{cb?pK{^<2(SC*e`xR8Er$JO&!%5I*&C3bDkt#?nO
z9@t;uy7ltK%dZXp1%KOqwEq0)Ywm|zZ@fP+{VC_T!6C{b$C&fI>W%B`!|y`BzGpba
zd6C~+%0=;w#8b`?)&QPT(VsHy3PEzhGV5jTNKcg7FKH|4Dfm&?Mq<C@UGc+0>B77+
zjdG4sTGC0<ej=&@yM%v9=_x-@HI^3_-O6VxxLP__(M(1`@`*yV+BEqiVkbrPM9+vU
z5uD8B%;_h1OGH;-8y~-*y?`?RM}ZIg=Q-!I$#X9k$P!c(o+)!wtx&~Qd>PMP&SLKS
z0$)Vsg_U_$a+z?~aCC9p<NC{<#w^Ah%`r`oO>hh2=bwg58#%Ue9_KOQQ{Y+1CBwOf
zJ&Nb0n2D66&_$kIJW1SV1Sd;97mDCk7CkHVQ1rOSeSS68h0J1{OZmTv*Gj2NFpF0T
zuH~5_Xd=E(>aW~%nRQ}1BHlti;w4Jclus#WX?-%8Zj`IhspzTLt^84?U%ga!qTyqm
zx$1Lut{YA^WHwDT(>MHX7-n|BNMD;vZGq|w%_DkGjIvE%nVDN}vavS1uB)k5uJA~{
zL{UmLO>3Rj6E$0vb!xkGOf+Z8zms#7*(1^{F;j+7a=NsUoT*HJ!cnE=va2MFg}nt<
zh^R}LiF)uo;rPYL!8@PFlyg5@9rs3|<)Y<M>e74pb2)ms61mDbn>dcMigDc+36S|K
zy;A54I|FMq>mH`|zmdO|{PX7I<nQ9R`v1cprk{Dg_Oqri@qRA)68WS0lh%(ghPdC~
zekT0;^}G6W>)X^XkN(~Nv+jrVcg^=ppWl7F`tiQE`ClKu&wKU!^_=&Q-x@#jf3f_1
z)Jx+>X-}(OYkp7o-~S`~<A!%SueslGf8>6<_sz0*zh3!17kE+fM&!eccY1FQypjCi
z^ez3D<nM$(6aPH;!_Kh$Kils+KNbIlvsv)<@mX_lFl#WKV6^;i`8Vy?vCls5SHBB<
zYxKVHOW0S(FBxAozr6ck_aXVC&1Zpc8^3S<^5lc**S5ba{?vRu_Sxn~<KOguhkyS2
z7XP#SXXcm1?>@Zjd6Doo?X&vlS#PgB7knZAnf?Fr-=gnWo}GFk{PgaVDUV~G9DVWY
z)w$;f9@jk*f1L5;<g-UFx!;}mbm`NN*Bc(&-VeSn^PuuU-_wv6D<1?tNPXM(;oFOh
zC#iQI-!gmt``x!!-Y*NESv~T7IqPHZThW)QZ+CxO_p#xv-E-TglCM_0Tk%HenZ$$3
zcQ4&lxtn%B{;A?~zNd$toP8wpK<c5}1DAUZkG!6Jem3o$^^cD~wtmh2B>ukh<L9s0
zU%q|5{(Zv_g<q%t7ySG7<<^HCpG&{V{!00`{cq~;O@BlGKm6_R<IFe9AKL%A7(f4A
z_xs=9?+j7QEKFDaPW~PLv*726e>)i_{6EUD=-0mYv)?U!&+@wch2e{&mlxjVy;=Nt
z{hg;bwq5tW>3MU-HOWhirym>*IPvk~&D$mq_C7lQMB>4u>pw3CT-$wT(?f-Oao0{=
z*1VQ{RqEW4liKGKukv2oedYZnqszx{=-+?-(EY)ZTi35O-gLZMdDr2t$o<EUCcd2W
ze%ssA&&;08ecJn+;njo}2G8W*@_%dnV*GZ?volXC-yQiJ__pIE%NwgV$KRfNpYitn
zyWFpWKNtUd%X*yo!k@^0{%iu=sazje-!a%Q9b@lf-^gCce^Tg)K(`RTz+B#NVGhA$
zo+IM(q&T@ZaPAX(E1e+yO*~LoL@Y@vLgJ@zHva)$O+ho^bdg%g-%<-ib_gU2?-osy
zN>h8TSFYQuVJH7hWT!}?*ixZ)BJZUdq{C$zW!l893cTlc5?L#~Q1-EmrScrD3Y`|+
znYy)_Ta<<6<77|DPgJ@t?=4j*^+85M>ZRm<rP~^-mFG*e^FQN#EHqO>MfQpulgxg}
zPRY{}>t$zXKF}A`u~!RLDpZ)O@J(T|g0B1_SzY<b@*gD>h5AHnWVXxh5<b9whJ6oz
ztdO$cG@(%8P5g4)uede{C`jIwcTid?XC$63Sk0f!cY)VL=$?eDgpTkD;T*|!`6iVI
zN;4&&2q^KK=cs4@$Zp22&vN!J<M%Vaq_|Fq{1n+OEGf8%+W~Y%0(UyUzqo-&Blk{D
zEv}bbVgg1Yi-ms*9N}8c`jfLp@T+i_h?Pi(fF;jzu54Z<p^pLwxNUelgboWY7HyRB
zk(nyZCle=qTrx*SU-_2W8KudRiv{|){kR|VG6?hv@(FhdWO5iXv9Vm>e8TgW`wu4r
zyUPDFUj@HD`IY_m^dFtS@BaGyOZ?aR|1(47pNF55K3({h{H^y>#pm#E&prpeQ+u`K
z$?p62AG5qIeaG`G^7iCw?{8;4pZLMyQ~cXkPdsi{U8}gQ_jvXb&4)a9O0KbA`F3^N
zjiQ@DH@Dqjx#Dx~(fP<LJy+_l@?EdGJmtc{OYxUOE|{I?x^n7v!X3BEJZBp(n%;bL
zx8}a~?S#ukm%MM>xFdew<(}KEb2m=j(768d#>u+}A5D6B=Z)r@iSKPc$$Xmnqy2C1
z_khou-*|uVG5PSU=U3p9=64pB7QZMFE_Flpwo<ify22E3DNzH7dy+FGR){VUIVbTz
z?vT<I<)g~$Rh87XDwZf-)7Yewq#dgBT}MUJQzc(<n_`;EaaD2E-OBRvJ`$ed+hw>F
z!<0@dFiZIfGYKn*yc2xNx0`1Qj~Cx}zG?w^u`@Ei<&$OSis=gV3T+aO7M?4pz+b{6
z$?=dynWKT1n{OibarQTi)Bo=NqwvS@SLnC<@BTge_Sp34<d>g5z5J8F(D#4VKea!8
zKUROe^ri1h>Sx_gFF#KCbnTnfzkH_Ij3IwFe%tfu->1UQ>%K&OOZYbX$LZfUeii&w
z{(a@2IrALm#sBO6T=`-DWydGaPkTPj`eglS-bd9hyx-Y>Jp6v{yYY`-UyVObdDr)j
z?_<>0#IH9#KKv;2RqZ?H7q!>NA5VD1`egdkx@R|EM7~XaQ~#|0sr9o%k9Rz^eqHkB
z^Q#vxf}Sb9@P7a7i_h1YpBH~>`uzL5(?1Kwa)zdVmwvDRq5Jjm=icx8f3|&P{`C0c
z-}i^!$$ygiy7H6W+k4O19<|<E_mK5T`2FX1dLI0J^zzZ}2bb?Gy7Tg$!2PlZ(;vTh
zvf}CgXA58Gzf686|0L;|!>g(9=6_23!u{>&x4U0%e-ZuK^NIhn?w1+w+h3o5RrONx
zmEW7LS7)ErJkol^@x=0Z<C~n1_djj?eEzfF$I6caU(bI1^?C2dgm+=D175y<dG!6A
zFOxrsznl5S^R?>BgqO8%3_iMjH~+QrbJk0rXHQ@FzA^vc{Auo|fG;n;T>ZT1^SuuT
zUR6Gy_Pp*z;48B?tKJH|`}cbBtK>IJ-gLc6el7W4_hanG;7`ZDq<$~_A@Dup%bt&0
z-|@U<|KRbp^2f!WQ9lK~8-9EKt?k?7Z=Zhn{Z9G)_2<T)>AxrcasMsxeb>jsAHIC$
z{JrkC@6X6@F<+&AeEu84T*I=MrIlkbw;uO-?hib2JUu+Gcv!hkusvn@&%(_0K=7BC
zqxgQ|PyF7zE}Wulg`DjI(}lM4tYmq~Aj;^)D8V50@89peKlHzpe0uTO@5{=s`9HgU
zT71{}<@CRrp_Z|RC5&bI|0};-e>8u;|Gn#b%=e6+#=jQ-`1WfVgA|JZ`z6l%97b$E
znJ+T#XL`ye&lSo2k-MGutH4Cz3gJ_NGx?wM{ot<_xFKXC;U<+R(I9zR##u&0(qCLo
z;*Ruj1##sj<x<tZ>Yp@<G&ZQ3D)B46R5+mcS*=X>h`~mqSH||ntBkf8wHw_xerR^k
zqSr#-BEVAFs>J+~p|*~Qrj+(agAh|g^Il7P>m!!?E!3<x*|b{++i2P6*dMp?wA40x
zXUu4LNXJ#vO1(hsn;O6R5w+E7_R2{zbH!!|YI0v>UCg|ZS(HVcxr^~B0~6y}hE0D@
zd|UP5@w?aWnLa)EEc|umH_q=BUl)G4`{nqT=U+vCDE<8Oqx$>mFTXx`y)k=X^{o17
z)-$%3j<0{dk$m^)oyNyUA7{REczgMs$h%8#9=)k}{qp6!m!DsJd_Mi@$%hy3|Gj_s
zq299_udcn_@&4kwW3NxVocH4JbK4iuuNdEEz1jC7<VF4~$5%&R1--rf*8A<*w+r6M
zeW?AI_i^jTfKR(WPWY(tdE=LtU!H#5|3mke`p^8Iy}zgZ-T%LiVfX)&jFxPy9Lemd
z>=7J%TxmSnybrmPIe)OXaHw-1<GUevP~?;NOUYo#t)e>xn)##!G=x8k=}YBF$I0?5
z{!%)s_+M_Zw1AYL%zC-~a?4~F%YK*3Q=Fj`A@46;EiEB?SazQL1%-aOa_MD~$r9Y+
zx?(QkF_POP>!dQ}mMgtbaaX;lq#@@nwN{c%)=<$~*<ba8I+sSTN`xH0m^r^Y?|vQ`
zjtIu-e-HoJ{%7K!yT6`)Px`j^+mj#5{@O65F>5n1Fns@0@@wx;_Fwydnf(3CFp0U1
z?H2brzVF-y?1@Yve~<jy^jrR448sA2r~iEz9x*ihU-oDA&)452zYBb={xa`}#@`Qr
z%l{<()ctbz1M8RbzYhHM{T=*0>GSsYMei29fBpW(+uSz>@BBYJ_;B!J;>R~{rCz^$
zne_be<1-Hm?*~1sew^~;@H77xYoBMo6n&Hb`svHNFJ`}3{QB&BlTRDJO#e3N+qv)I
zzc_yte*ORc#9N-XcCQ{jfB$^)^IOk6pU1yodQ<r6&$nsc?|=RCQS{C0=ewS>JafL6
zcKzFR!P|njgKuzLU2?_i`oC+lFBY7BbnN%xBZqPivg{Mt{dbS}q1%VA9<bb_yJPnD
zf4g?=U9|7*evJbf`z7|P>`UGqzB7FX-_E(aMfXbXvpRI{Xy4&Q`@Zis+x=+Qs@;F~
z2=1}l)v?F?pz*=WyYFs&xAF9bf=w5<e&6Z5FX`~n6Fw*39}YP1=|KO1^ZNw%OB`@J
zu;M`ZA@*ZFCzqXMKIwB>=EBY^Pp&V&>2+)A&2`syUEFmp|3b>;byqH3pLXB<Wywdr
zZ{pwgeSh@B;fL(^Ek6$Y*7+yTxRfQ8Z6oUg<{YMcCK=|pO!dtFSS8tu*cP+%b8X_j
z&#NggK~Pp?mw39AlgxXW)6%o0=1ba11c`Two)l6O`6;ze@tbOzMuEmI^$?BQYA+S9
z$}N&-Q&d&5RBlslQu(V^rx~uDq+PF>rfH+osCQdG+{o4BlySImiD|KUlj%3ZCW9J%
zS3OJpbc1L;4y_xi>y(TYPRjDhtdP=@0A2esQEamWkHiGwgZz{DX7lp!zT!R1|4wLu
zXq3oD{w(fk9E}{UoI0Ee99P-DactsQD3mJxT3kUiPI#}#LGdoBY0@0hm!!@}1xv|E
zE6RjQ3raJ~2*@r_SfOgC?ynZE`b;TAUPnet(o~#D{FZpPq_=dc%naGXa&>ZFrQOBd
zgm(#D5zZEA5uGcsPv*AlE~#%4F4Eg%_sX*>PgXToQ&DqN^HpD>F0HDnB&C$4TB0>m
zmr?h$)<=zt>LqIN%1#PT<olJlHRk9Z(LZh&X*%Cj#n{i#O7D->X|3y8{+bijwNzgy
zWhgewbx3cQS}f%yT_`Imw^TY_B16<f=m6h5ZWgX1+~EQrg<c7`@g3%i6+9#yBg`xC
zl6NWJb^%3^5b<pBB+=tSANbyKRdY;a-^H%V!Np<Aeu`C=m5;rO%bMpm*Da1%P7fXv
zzG}W4z9il-Zf35tT-m&?{O5VYxQf_a*uJq@vga~?|J(Yz>Sx%O1s_a5F8cW8z4p5~
zueZJEc@gw#(yQth>z*8WAaTF^{*C+n_Z9DNzAy2}=*g1DXCEm(<$0<2`sAzWFQr~x
zd9&(W=(~4sl3#c{TyZbs;iE?!k5)h2_VDtfM^Cmqk$imX@w{g}FPFR#eE;*kz=vb+
zW8T@n7J6><MDA(U%YARPKL~$X_r>-5mmd{BKmM@$q4eX-kI-NCe=`1_|Ks&b^ZUjx
zZlCVGSATE&A>iZkj|V>Le3tvp^841W;O`&5%=misJIk+KzY>2w|DOK+$XBz^MV}^r
zjCt4iM*ZE!_lw{Ee!u*K^ar)KkDfC>)_th^sQO9vGlyrB9w*&zx?Od1-i^}hhp%n9
zZg6wgE%Uo&_t)MpxUc$P^8N33tZob4{(ej7cHZrEx3q3;zf<+_<&$|&ZJzc#6?<;>
zQu@`O7lAKYUVeP}>&5ez$KP)Gbm6<;pSHgSe{_B|{aEyE*B7bJ*FRLeQ-8ntbIp&o
zU-N%0{;mE0GSeoeJOA$dy8C1D52l}dKmC66{$T$d{5Sir%b&X6%l@=7^t0Hr{a`cU
zxXJmKyMXr&k1CHg?@hj7{s`VG&PulLtW()+*$i0}nNBmEW98w<<9NWnhdrLti93>$
zf#uo%zJI^}-u&bKtLFQP@3Vhq{G9rI`Palx_ut9BUHEF>GvB8c&-OnJetiG_+1t-<
z+_`Rcz5ANawa%-SSHv#QxOn{hzH^M{=AWrMeff0D+1B%=7bGs&oHsgm;mq;VD^IUH
z^ZVTOi!qmPT<W=a;DXYHqvvLvi8z&dlK)KTrH|KV-}Jt@{pNvN_ih{9eRB8a-Ie!j
z9@IS$ynp2G{=1U*tsdq*I`%N<q4pEsmj!Q~-d%n@>&1lUSub8c7k*as%<@&}`x&3v
zzPfzj`TFYn@}IkZZu=?lYt!$tf9L+I|F`*f@$c1tXBjhCYFV|}FLO@hj^qjFS;rN?
zk;VRu{RaDe)-OypOzNyD9GV<ytXr5{Sw6C^V|~JMkgb`!SRg}qj&QZmE1?gfdnCnV
zen{VvQj&I)QII(xl_j}CLQnFS6qoEOxf1z3@(PL<l%J_)sQp(>QC*@^s;sP}rg%t!
zQ)!0U8y!2tjYeI@_e^%1#+bUAY%{hunQ!vg_@mJzqfbViCNE4yP2BZswdSfdD$h_9
zQDB!#lhKsfFB2@gL)KP~Oa6}h6onZIC*)R0i%Fgqml3}s8X+tzxJ}SPR8mq)+Fn{i
z`lR#}>1C2f#52XqB|@a0N;AsDNH3H=E8{9VUHZ6qvdAN0KhZg2-r@_z^h7Mchv!ce
z^b<JBe}dna{|3(~jx(&DtPX5-Y>_MijLZxd7^|4SG3{m8{O{90d&VZFXN>90ot*Cl
zZi}RdJQeg2I3X}ckW1h+_bYZE_A<^r+~T~m`OXL|6v`4-6gehpA}%E^BPK1{BU~hS
ziob~eqd>RtQIQG4rvx7h)Ct_@m*l(1RmpC^`h;oI|GGcDzs~<G|MC8d;D;BlC0|Z|
z&h~u!)5b^n_ZjbZKB##3>cQ0e+V}YHKD+b#Zp3}#2Nn<4K9YL;;lbuRiZ>75_;pL*
zuE^b=w-(*Fe8cs2*`4CsPj9*2xp42)gDnr<-H&|e^yJ%9rf1Wiq&!M_6!vuSbCc(a
zPa_{SJh<?{>tVoy^!u0Y`Q2S}TkiIqTTgEtyuI#D=AAjWs&AaSas77nz0>#h-W9of
z`2MSBdhb*|Ie(q_v+XZG!x4si{~H+kShjJfaYnMOXU=5l;aI`L&Ub`oGS?^0&)hY9
z75pFgKM2$b{t{>tkQZDo#4PG9aYNczZkOCYStHr$vJ&!r@~>q*WzNa?$lEJ<DEld{
zm-Cm^lu?(?mu{E&C7U7tLe5I&on*MAu2h@!2PsjB31SW6vm`7e&q;2UjFhyHOpq`Z
z&l7(kp)3_4bxk5plu@vbcMi`>URORdo*vGH>@V1=*k7>OGynRx_K(-^BR`k_aQ$)P
z$MK(fKh3@^{h0pN;f>|HrSBKNGki1W#jPhFA7wt;{LuBG%)`J3{r9KcKYYLD{=R$q
z_a5GRc3<SdriTZf6}>ThU->TL&4<_X-uZuA{_)aB&M(E^pZpB{75J<7&jN<yOgk7l
zf8YML;Vbv|CEq2!efu=;efK-f563?Sf7<)0>eG%7)82W%4Sd`F&hdlVr^lbCe_{Lj
z?pwpprr#-lPyCJeefWpb55XUaKRAA6{$~5b@i+8uz@LmiVgDvF%w+n`EXESY!ppXT
z;{sPE7eD(lmb0wW*i+g4*ko9*F~>2VXKrJ0WqZwO#rH{oMQ{!u7ncc(_y5>GE<gFd
z+<!au<+hj4-kN?o^_l5Q+ZVmBQD1Am{Q6Y?sqWK_56j=?y>fpc@^aOiwD<GhuYI@o
zjpVD$7rM_^J&AdE|8DcGpEtN}dEN29=Xc-W{?dDDcbD8^xn*)^@!g4cZryr$GvW5p
z`wWktJ&eEaes9uU>pN+;9^WXqdH43xI}JB~Uv@jc@2vaz!i(Q8##}gecI}zwvv1Cp
zoGCoTcrx)s+{yYgw=PV%a^RZhjb%3%-`sTL=?%|YuWrA+D{}u2=z^AeKOVe!wCeH1
z$BvIV9=AWTdBpK}!js=mTc1@v^?UN;>E>5fAH2Vu`Ev2&!gq7uro3J9=KgE;H#6Qw
zeo*-Q?#t@0cHi>9U;fee^V84SzkdDZ`ftv>n)Nj67uHqm8r*3DQKIj~#Uyp5wn#0O
zIww^ry;P=OcB2fhRH(R**b8w*={6Y$sUk5x5k`>-q7vfu;tR#+Ni3G!D0y14P+CGx
zTR}o`mV&xmg!FwWei>V7b8#;rY5ui*#sap2i}}mBU$Lb#D>Cf=^Yho;-z@)*{8wP$
z`~Ul27}GlTI__@nXpSycWwxapN?f^I%3OAwy&QWuK5{<baS;d?TqNKoV8(x*$C5jc
zo1cfDH<$Mi&s1I(!Q;aFg(nJm2~888Bl=wQiEx%sfv}cnzKEN^F76rJT>?yE*ToNu
zDhSo`Z{>T;dy@MT=N(Qx?i0N41#89lrDn?L$Q8=D$R3drlMs?PB<-Y-s#K}up|oA8
zTRB2ST6M8%t%`=?2HC&T=cFrSQe{hJ=gP3iNXaag(Uc2SxT|nLmPO*Zkb}TEJ~m!m
z?tPr^Io@*gacp7hU=d;d$B_Hi{%6Qn$IoBC%>S|P=eKWHKUsYU`%v*|-Iw^U3Sam>
ziG1Ap@#LpxpRav2|KalU==bn1RUZ$2H2re)OV~%ZHxVy?zFhof-FwT=4c~=-7yk+P
zGw%=6zqkLE|NZid{b%=&Pe0s$HU9DVXY+5tAHm<|zyJMd{-^%$$M2^<IKSn7vG^kQ
z)&KjsU)ujp{B!-S{>|>ons0VLpZ?(d{`#xqH_IPyeog%o`ghsCZ~y%m-Z8W>F)^1g
zzW=}Vf5-o2|BU~w{r`<=Gix=gF!MCVP^JWyWo*tI4jhx&5?TD1Pp}-|n8~|Butu0g
zXeHlo9yh+_0zN`XLWcy7@-g$3@n!P|3)Bd_;(y7nEXXIUA}S`nU*f0aA4wI7v!dpr
zp5h0j++>Ai6{Llv_DJc<@XMW$|0AC&zgZzz<+=J<&10JD)gLHl%kxSLNR&(XNGHj7
zNQ#K43Ebez;cpX=5j-pKgFl16pMSAHpU`dLNkUioe7Q<lH5tABKl*>0VLC(YzslbR
zzv6#?`?LM;p1-&L82@?wC+4s9AFJPrzgPYe{2Bkv_)EyQPk#cLzA-T|D*n&^x9$IJ
zMkZ!qW_hNW|EK)P`<?XX-=FH=sy_pM%>NPb<J$MQAE`f`e(wL~_{s9!{5R9zY<hFz
z&FeQ8-rRcA@b=NWHy?sNOMm_R<;}+lZ+%`Hy;=KK?QO`b_Gg<OD?gs`<oUDTFIK+X
z`=ax?-SZvK*SrXSRru!V+u3hRUW>kb^djc9^1G$)zQ2)p&G`E9n}6@#yx;TQ<K2?i
z%&)v&o4xD*p!)ILJHMBekG|i&eXae9=atWw0?&J$E<2%lT=BTvapq$m53fGxabU&)
z-b2}k?;PHF$n(Ipy*hi?_ayB5aIog^mxD|9aqd>w*|qcZ&Rg53ZE@VlyWVE~;SDo3
zN^aV{NqW<|^;6cYTUEJ+f79P>2e<QVJG<rA7UONdxA|@>+`4J2@wR!}RCX-erMzd)
z?uuPuJJolnZNI*qbGP+=jzg}86c45yTy<pU$$)dyFMhvVe68VT-kr_&<~*42DEx`U
zQ}rkIkLNuNf4%I(mQTAs+P>$1oBhWBt@eBWkF!59eg5~^<D2Qv%s-M0@+>RatvLHR
zcCzzxn(}1umGX*n^|32)h;pa#?Gu<N_(GtE|1uwsfUq#1n4I_mQAMFDz5w3E{H7v%
zB?P3arH)A)k(ecYQ~s+`pz=<|W`*+#3`%UuN0dudBvoBhFRS>dtW$|n<JI`B{$ABf
zxmEF$qO^*?+C9~2N_}#&GU+l0<W49~QsGroQ`J*CAa_!xM0TpYltR61l%%X!v}nJW
zhIpOmUm;6DP5~SKIs7()H-uD#9fbr0Z}R`-TgD$GG(+@}c(lX`@qCE}=`?v^r721t
zilqvl6~j~*)t;%}(>$ShO08cpL`GNoq1-|xHRUsk5(+}{*$Q6DKUG9j<yB58I?B(K
z+bSQY_&`ZXIYqHU&Riy4CQQLp%}~ooD@}c?3YSWhN`rEM;tjbVnQxL0CDf(V<-(O~
z)tJ>DDa$Eass7ef)t4}`Hkx4&pr@hxS1;e_vhfYWd3sxP>~-hsGZ?8GGa7$4Sf-<*
z(WQ1oLsNH=!D@r6x&qn_S_wLv^&^eqjqd9^=seSStR|>dt9o0dU0G2<QhJ@3p~!3D
zX(Ft`YJ4%AJ?vf_@?4j=&T{avxv=`PzGq=&ljkhtmf+6dn91VE*!#cj|4{~hrkRZG
z|CRsU{wKf~#4N<Z$h7fq=g*Gs&ELy@eEa#}m+jA4U!K0Z^6J8?ns>K9rha_zmh;u*
z=jqSuU;KG4_Vm@mCHF(_@!i*XwB+fO=M$crKNWho>F(CMf)DMVYCQLQI{)G3`&kdB
zKa_pA>0bEl12<%E^xovX`{kkdv#{sAPuU*H+&_Q6;ED4~;n%LOzQ0I+A@)-E^_I8I
z@Atmn{9ffF#}|k1bAJB#Ir+z;Z$@9Pe+>Gx|6A%G8HR}rQU4<UeEhTgU-^FlhNldQ
zjJAyNObV<YIdAgs5uPSoD3He!&6&g@!kNuEm)(PT{a>%2yg$DFeDP=czjy!4{}=q1
z{vY=LA%hB2IivFbv_GA{I)8EedHFZ;|J?s-|GxfS_iO*J$lsrTC4Oi4X!7Ri>&&-P
z-rjw)`}OG;dmbOUD}3|QjYW5-J~Voed3(b37uOElJazZP{qFn6?)BZxzY}x!$b;oi
z9y|+pR`s~-LGu0Nhku_Iz54ii*(<S^IWPabKK}0L`~3Iz?|;9`dw26)_Q!i)rGKgY
zHvIAH^VW}LpX9%CeV_Mj!B@F&j^BTLpZK%)PyWB|zn6Z0_`UES4|4%~2j?aZeYWpR
z=l;L@7tPqoGMzPuMUbh4;R3^Hrc*4NSP!vuF_!$*`OWZ0;C~F$Pv#uvNXFCu?)*9Y
z=fS_fj0c&IFflWn`n&h<g8yoa8VpDOPW+SqNBd9TALD-y|JyKh{JZum?tAmMb>HiL
zG=A6r68c`_^@kT~FP=YVd#>}u<w4aw{|Bs38lH2!n*OTmx$eXGTank6Uo*WGd)NE!
zpIbRM1#bo1X};%k|J=Q!ci-NAesjak)wetE+T5LWEAM*XHPsuwx6SSuJm7oacenSZ
z+I7S0oHvthIp5U3Hve+V<?C0oZ{EM7en0j;*WFJyim%?jB6|J!O_kd%x4++3yK8=5
z?$MPe1y3U%AA1n`VBJIe$A6yucsAz+`zz(wiLYbcsC}67<<{5dpXEQgy$^g}@qyvv
z&kqV8y*>!MTlB`{o$BX_KUe;B{+stJ=|{;A+uz{~6Ip+8_;Y>d{Lc~0S<AhH*OV`Z
zSAussZ#rL;fV#+A@m$HX5<cS6Vz0%HOU{;YkmHtLCwE`gOn!#)B8@`r4O%=J_tn#N
zx{SKavMpFFSD43}+%a&~k2mZ$+hP@Hm25V}sKSuT_>hT|$x_2M{V=@{J!1n?<KHGc
zCRGNzv>P?TH9WKq>BQ^p(pA@9tubGHhx#Y=k7^TC43yR>+?8*Yca)zZw@CJh^e=G*
zVG%(;;dJpLDP8H$lF<_9#G54^N}iLlmXeY<E6OSAAa+V(y>x}_1DUN-{F2KhA|!jI
z>*S8eH^{A!{wkg%;w3a+&_ZyQfERxP?;4(Ed`3c#MgEI82`Ta2;mYFN&RNOb%^Srh
z#hb{rn|(LiT=sn&0UT@Cp0ZA6t6<;G;l;g;_YAKDPdS$cmj=%*{y?E^g1`B?xn($`
zIp=fx@b2M}<zi$nW1Gi0hqs#FgZ~Eae=c730_OjWdzmZPFK|xcuw#9~be&0@WgANX
za~8wpzv_Pkf6M)j{L}Jh=Fe?kLO#g9OL^D&e#-mQcN5;cd8zmO-BYV)tDhM>D}BQB
z=)eQRM=4L6UR-~r^@i<D`K#$K9zXy1;{U5#Z#3RrdsqK1`>otthj*vnSAJ0Y(Di=F
z`<xFqK16-scz^y4!>i`!i=Li-lJxAs%RO(T-f6$z^C9=M-1m1s*Z#8o_3ww$_YL1Z
z{W$$+6~kgCCdTJ~+J7ehSpReXAE*Dx|1bXi@%!<w_TRmKyZ=iw&R_~**~nJRsm=4A
zuUg<FUl(^6S0nca-bBGPk=vqM#GZ-07ZnnVmpCoyFR?)6J%2hczraQjb}>_tJ%U#J
zeY}f!H}mZgU=X<@YAq5VC@yeM;Em8xkrg6}!u3K9!bKuKMf=1ri*t$n75dJ9f%h@b
zPo8ex`@GV;Pq-vG^4W6Oj<UUA{mh)fRKeuMvYzb``*HSoc5${%Oh*4U{QCDp_-Di~
zq2DpTe1CTTF#d7%$Ah0{KTE%vfByF&|HI=C9v`N?WqNh#S<w^S$6=2jJ>hw-^dji_
zkEhe07C!TPQTS@-YqvK&Zw|kSd$;-1|L@Yji+_Fi+V=j$tMHduuhQQNe-!`h`+3jj
z{I9A%zW$Q<bKuw7A2+_Q{}K3W;cv-5uYU{tcKT)ZyZxUZGb_gnE@^HyuH_sX*j};x
zWOimc^zYlR%|AZ>Z2EtSRi7h?J(g`D>v@)FmH?JSRt1h`u1>BOoaeZrcr<xUc(?Ps
z<L=}>&mG5;%DatkCf`QxHSF(MI+-spEn`~FG@0oJ(|M-NjE@=4GEQbb&hm&wiKUO_
z1A8HlF~5?)3;u_^_c`~mO#09GC-V2{Kd=A3`7`Th`B%M9Q6FmFi@&pfJ^4kzi?1(V
zzG{2<{psU}0{2z!w%opPTk(F;qiK(L9+=-{xWjgL(mlC*Yi{e_EV=&r+SD5icb?u$
zxNmwV?CP1b0Vg9)6rU_Oopnb3wBkv=6K9SKpXfc|bYk=I565qvus`+dbjKNs(`+Zb
zj!ii%d^q%I#|ie+uTP&j&2*OkJjeOtXGPB4JvZ^(yt9YSG@O&VDs(6AZrv^68>!c)
z-srofef#EZ_4}6}X+D|wr2py9XV+fHy;|~e^^0{ccD#J{`q;ZKA7nml{KWG4#ph{X
z>%VLK;`=v+F^73O^D3tIj5nBl+3PuFxSnx!@h%lwCmtf1Cm}1IA=1ddgDaXtlk+Xl
z6u~x;GorlW2P9&p)=96FxhC^R`mxkCsVJFJIaB!p*@;q8lJ_KSWcJ9bD2J#_Q!16u
zk?D{;ELJ2EAT*ypj3<GUfg_fagO^)i3I7}J?`%HI_nDYkd)R++8uMJ`*}=VsYYNvB
zE^BU4uA^)-nHibqvy`xzurV_K{8#XY=lAcQbAFor^89W2=k}kMf9wC>XPC>B$-0-l
zn&TTs4%c#CZDCUhHAz?T<)RxzYDKz4kBM1GC`c>5l2ee`DD_kJzk06GYtv1Ji5eT^
z>*VZIa<%5_p4S!D)>N-iJ*4KS!K=PqDPAr}E?aq-)^vSY{S3_;3LKJW#J)=HRESWv
zP}G*0Bi<s?D!fMYx7Zn>tK2u)^|<VWmP&QVGD=?NlVF+jZwtd=j=usog_QaFIW*Z;
zv1zg!vuysG^kd2QBR>uPCNnZIZ~OP}tJORAH&5Ste7o_thiMzbrax2vSTbm^^s)T@
z&-g3pyZ<jCrXp?!A$_4Z&XB*F-wuC!^QVkigLOHh&EIRkv;XxlIx>9vRrvkQkNJO>
zGfQv=b1QPaXZ`&z@7MCb)@)aKXA11$W8_`Ue_2#t`jqr#aeE<G-itgAA~y0TRkKvq
z%A^aQ;C1D9mw2SauHK?DQ87m;SpAU>lU|1wqlSj&1??63Nv3v|{pMnZ0-BvFf7LeY
zUNh=2dZXj1_E%|^O0K56&K}L}%Dd#8<aemC8H!mpS!Eens0k=MP@1H5#Zbs(fq}F3
z4|Ow*&sy_!&+9JKKBF0+rLOC2;A2prU8@u+;Ucs|__5?4`D=>TWv7Z~h};!5lw2$o
zCizsfUqFYiOJtS2o!V|?VM!kzJ=QO*E4ibEQpGNcP8MY14dLFv-O6=<?F*9=;~~b&
zEVf)9d5gGfnQ#7@`0ePAivMD44jjU)Xa5)eKKt|F@0Wii{xp1N_)_vU?B9Q`g~Bqz
zZJYvsR(<Sy7xuaLcLhTm!}EU^|Cawf_*d$m(_imjT;Jb+bN~IAk%@f``zfZ=zn*_t
z`{m5fWJX^$Ay!t#zW>ThpV-{Fd^q>B82_LBXD!28&X<CLB4&I)7{7ko_o?^mvEMn2
zubFF@>i_Hif5M>8u=$V5&+Ko~U**2X{fTCBWaDP#|F`-Z*Vh?8H2<9Wo%#LKN2hl)
z-iEz5`Y`8hz>CyJ#`jeo?SFaZ1IOp2cV166+`N8S`D)XxHFx9hNZy=vP3_vuYnQGH
zUGciyby@hz)T=@_74M4QYq&Y@V#FDbvkI5j-7LK$dCU7+=hbW1U)+3qbN7vwYl@do
zT|9a9^4*uu&b~=_Rq?Rx){~n$_bxmMe)aS9>Q`%D9C|kC$*+fj_h;NGx|MnB-<_)u
zHa_|EEc?mzJ3QBRUz>AJ;koZ?&NrNID&I)G&3kkGW!>|;Pn944ddT&B@yD9KysXm9
z@;|q}JNDw=^M=>FAJRUReDeO-_ulp0iMKXy)8ELxVSKy%{pT;Ye)s*K^JnANUGK%;
zv3=V1WBT83{}~y37!3cn|E>RZ{%iC5KhIMhJ$iWf+2Z%U-|fC#c(dzK+#RM{3vMgl
zcX}B3F!q7y{pI&A+~5A7@7}kYZ?EmU&V6UyBe@r0FVdeZxbJx<^v>;jb`NUrH{5-H
zEAHlv8*{IVUR!!u`JC&CdB-cyYFtmgU-&@nR^7#grxqQTI(hxfl=FAb7oA&oR`vX+
zixyX+u6)1v@SOJ9-)CoC6u&BV<=(lcCnS${9-Vw*&#9y{p6Aw_lRRg8j`jT93%@Qt
zyfEpU)wzooFW>y~u;p>oz0T{8FMYUl?)sd&RSy?Dj(GCr@s!7+Puia@dZzWP<f-Gc
zTQ9D@&HL>9)#uZhw>+=4->`fr{ml3+`^U0h&3|hC-1*h=yXhYnLk@!*(`5Fqg4r_P
zWv_^p@*U@}X8*%!%Xe3>Lu7{NEYU}zYGR&Zd&T(0&xprMY03XqU97WGFGTB_5}z!q
zOphGDl9$?Dts1=v1}XZr+OC@U8e*!K6m;adloYkIO$*Fl82nK`F0U`WLb^_NvD_Sm
zr-~U0{Bq?or==t$%taCf^!VopeUWrmoT-wayi2ZLf<r_`m_hWoM4pVhT)f<C>EGg8
z#7>E^ix>+W<+~(kAY-Q4s@JW(NUc|~Pj;gWv+Pt^dxdT#O~uP{_vCKMKalg1J|^}^
zxLKr5f>HK~oP%tl#3R9jJh@yl++X-2g$qTSMGp&46`U@VCXz4M#J!t6i{l61bMY2w
zF-c8POTlG)`}zLxJ>#1va8j_Fe<ROV?hKwZo<m#??3OHVS#mh-`5p<#3V!3;!fnMh
zi+eu*Nzn&VE2Ir1CyAB|ofSMQsLk)hZOeIwD^~Efgs#*l5f@%J)?5D%|6}_9^nW|^
zdA5(N>CC5?>X^HkqnNHRC^0B7I<vTQRC8%?7O+hE_v)9(A1<a<oY(mR1m5s{;*w%Z
zV4lPH?XUDN(Vzc+Phj4_V=K6W{}<Oc7F~uf{}waMV=3Ue!S|Big8v9_HTOFXTee!}
z_l$>`eAx7O9tkoDZ|7BFz5I9EuP=YrGBvRWb6Rm8XIEq0%+$lM7Sx^kz3v}7`%=Le
z$pq<5;*a>fIb2yfm^7ISI4<#fi@X&5DW)dALX=B*1^<6;cCLBcxdLKhjFNR?IsCC4
zc5JL%=lBl`i;Ddbe;{#L;-rMF!~ro+(dWX|!kyxm<%88WtIty{Qf!bBl>8(1QdC{y
zgv>*^sj?X|F*4VrRi*mH+eC_m?1gWN@0B%H2$8)Z9wYouuu9~<#1iQ?=}(f@5@*B?
ziarpzC2TFkEU=VMTR=kOznGucJ3(9S8Wykr#=npL(EoMwuf_kizqfz+{`l~%`J3$*
z{ttrhzP!Kp>CNXeA3wi7_gLkw?d_s_=bu=<{`FSj&EXfS&%Qm9yubO5_?@IXYwvMC
z;duS)!~f5&pH<&4d};Ai<ng7)vtOKi-}>eJmoJ|fKJvc*^k(@hgO}T0X@2Pb?)CS`
zKfd3ZpD(<6^GxHV`1^*hpMEC)=KRg^>-~3wuh&1>ew_c2^K0_2&kVgBReX#2zi~fi
z`TNKE$HgCxe^r=IaQx*_;CJGm!0XBt%Er%J%($QFIC~vmsn9KfwOnE>{*3C(njD>c
z6NOd8rb;MCPmszLHx<1hoGA2LP*m7T?2L?-$_X`o<q+vQp$hJE?1$K**rhltxs18y
zaEP#7W4il4|F6OCXTKc&RWf(5%WxzxOa6KJL-_aczoP#?{Lg2Y$Ed&@%V_=A_|Koe
zbN)vE68xV0t^eokztR8R{$%<r`ew@Web3ll=)P)w>+(VRgTXtSH<_>6Uv|9gf3e~D
zikH!EJKqPsGk&T1_~pY#PkCOYyqW)2?d|l}#xLt0tKSj7`S9kZo0qR&zIyL!;%)6G
zd!8?SeB$oco8dP$+|0bY;o+fYKCdHR=RE)Mxbe~Thq({;KID4x^TpnGou7n1ZF$r8
zblyXbN9&(Bz6gK4?ak}Ax8GI0wtuSlSna9yv;9w>KArO7=!fmU3mN$TeEXdJM(?@8
zvuQ8BzxnZb`tQ1b_J6W|O#QO*qxJ`h_kr(uKg<5C{_Fa8{*UdSRNpUn$Nk~cC;9Ie
zfA=yhU^Zh~{@3|e-j9jjo_|^Wx$Nt{f4n?zh4qEc^VqTU{B8X8@4M)?)|AAOL}q@b
zY5y93v3}qB<<`dw@0&kf`FiH3`tO(@RiE8I1id$Tzw!O$5A2_7zIXju^tbwV<nMWZ
zT7Rqk`2V@>6U$F~mU1p-&QHw6e^Y<Nd{h2<>dUhq&J2@SE!myewlQyJa$+uHtpC^l
zFOJcXRgA-rZ8KvE!)B%;79Lh5wkhn@oS(QnIoz0x|4aV&_<Q1~$~Uhs!9Vqx4LH(R
zqyIPmEc#aX?d;cyUxa=b{h!U`&uILA?q9Kg?--n!%$ZiO)PMP`{OSLD(f1GD%YW+l
zy5dLQPxT)f-_*WN{`&tL?~lSCWk2hGhyP9gXYw!o&zfIvepUZb|F`@9Z-x_0?X3IQ
z%2)~+p8QGu758)h&sV>?{uKUe`+xD@kw4wPzWlKM;r-+2kKkYb{(fZ2V%O$k;&NeM
z&$5;|gLxx!D$8P)i!Ae5z1VWuI@uPptzuJVSL5L2{LiV!Q^y}G_=P`=r<=o<Z6=#H
z$7{}G+-iJF1^k2;iS~&(iTQ~Z32zfv!@G{FnZual5yvO?FcwyZlD`Ij-TztrpZNa_
z11IxYW+SFW|I7bO`myNC*AMdVzP(-iA^gk4?;C$Q{__60^ZS!;uHX57%=uyTGxpb#
zKUx2eGCco3^>4$kZ{GqxTYWHlU;pvJ*Vvz%e=>bf`TXZY+K1O4B0e7b=<|8g*B#$E
zelGnP@YCeyqF?-fr~EfzDra59d58BFzYZTC7dKlovpUmX#uLm!?0sB9ywiAB@Wk@$
z<9W+{hclZanA3@OzEGan12IVv34yu1$9cW^xdj!4r-^<LUnQ9+B_b6pIYnZ!c!=0u
zQE4#?@kNrKWERPp$c9QbiWmyo3w{>j6j?38ExJo2S>&GZf5AHb>AbVK**TZ7?PQt4
z9LVy4wVL%BQ#S(#Lonl2<~6M0Y`yGaoF_RBvSqRSV?6f%-Cw~!cYk^QzWS&4pZ$Ng
zzwdvre_8t8`;F17DK8|Smpq&N?8vj-&#a#5JzewI>*0#KJ~yvi<-S^ZJ^Oa_y+`*7
z?zY{oy7m9&#~a$$Ctc;eGUX!wxr|fE$NP_R9CbSS>gdE{3dj3Sc%AM%YjpPH$=V}#
z_ut*iwJ&Y|&Vx&ioH_pXl;&BZbG7G=pIv@h;AF?~(i0YEVlM8z`ty4F^*2|-F8{tf
z|Ellx=$kKYAHKKaq5PASrxMS&pTB(`_G0CW9WR65EdAj2b=&uS-+VqNy=Qo9__p@_
zqfeW@<^L-AoBuzbA&B9}zqfy;{fhq){Qdg(%|AK*E@ZgCSi<n>U(&yw|JWD;7~`2b
iSVY*<**CI^u#_^dU=C#oV6|f3&SlSkQ233QpEv;eGrS}K

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/782051f8120182b860c7fe1b265179cfa2fe03fd b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/782051f8120182b860c7fe1b265179cfa2fe03fd
new file mode 100644
index 0000000000000000000000000000000000000000..a0d8a6ec48c983fc3870d173c5b4c73eb474eecd
GIT binary patch
literal 66439
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bG01{z<0&gh&4@$$Rp#T5>
z!`P$rXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD
zjE2By2#kinXb6mkz-S1JhQPpufZD)WJ!<o42#kinXb23S5MUZUUK#btXb6mkz-S1J
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinhzJ3uTMKS0-*<W#_%QZ?=>6sQ
zF5KV#pzq$dn{Th}yUu-Q-6OddVK363EV%D^C-lzkdv*_M?>F3ie=F|hjT>{Xi(XrL
zS^1pniFwB>&uUyxzF+u2?N;5zg{KxBmpXa<%#`zY&ljCrcUJZMr;8R>qON?u`0$+e
z+23bpT@=46cIDo=rza$jb{?I4V$Z3hGoI(xoRd6fdye(|+zY=hKD;pLoYlFD7cbxZ
z^04J`)V<E@k1u_=bng0`yHyVtJdSws<?)oqqEFhNE_$Z*tmLWVvs*8&zRmmW{MF~v
znYTQzwcoIODE-X%E&Io^U(J7N{@nT1^1JCD7efw%8`EU=uY%b!-(|0emGT|uux9_m
zY0Gz4utQ{q=q%AkqH1EEVtd8-#m|VxOKHjfR$Z*KQZGd7ni8KZt4xm^zmk{QU9B3u
z2?i<pwc4(l`5I!XmlSm5xs(*OvrP-kUl{yRKQ6B?y+XQ9cCp+Xg{O)c3jA{AGN+{^
zB+Nw;1@!pm2z`-sSDdMmpu9`2UV=kJMwmhLxI~_eyIj27Yw6$OTf|O@v5OcB9Ob(t
zXdq*z*{avAy-2NBu}^lR472Q1S$l<UB~8W4a`)tJ%0H0vl0GK(NVr*~PJ&VPiJXIM
zqQoP?gFLxhGTdMIB83Y@n?(-`PZgXllqQld*u=e?J&WT9-*fR6X)#GnQA@#PeEa$S
z@IB+3CvZ}*n|~wESMCg+G@e6T4(ygJZ&`9U?fD)F$O?Yr+rn+dHH&*b|4GpYQY)kl
zBqxcM3!N1_E2z!y#BIxYhbva_wuG+KClME3H`ZJK5C3EO|MY)5^Le(9tm(|BnCh6j
znWLDlFeou7Fgmlib5wI_a2Bvk`uFOW$R94IRh-xP0tDXhed3a0OJJVE`0cOsFVUa>
ze@|fEz+)@8gZ~%THx^xnFaH)Z%ws9wy21C7--7=LZ#DNj4qLWb=J$+;nS9vvcpeEd
z32*0BV!ix#+pjNw)-pA*2Xk6+9%ol$-OSX(u=cOcZ-(FN{;{(!6^xNgklrNzh~JyT
zm8FA8levK762G^|OVOWVYT_$IxrA5n|L11sn#Y|hAST8rStpjmAIo9K#>#b$|FE#A
z*dOr+5|<@TO4v#q5c3p$E?h0#DSlZ#SZ%ZVJk=t_1{p!gKVmOM)g?~IJd~R%n;{b;
zb4^-Rs$aZKq*%ya_@?+?S!0C|*&E_9!v6%TMD9y0k#3XzBxx;iM(m*I1Cd+8)<Vnz
zOZl_~Bt-s;`H8&~wB@d0@%nH4`{)n-UpN0+{BQev`<L&J58s-<*?!^wAo%Xf`+J|>
zd_ME>^XqeuRqoo}F1mOAiRJ5GZx!Agexds8+at;Qo9~F<NxHN49`_TD*S|je|Lpo%
z_5H$^7EeVUUwS<I#mV=rU(SE|@`>Ri@B2@0mcKH1x$TwahwkrQe~<j*`>px;!mBsW
zG+v6oZ}|G@XYy~(-yFZ*e>eDg{gds-`5!sICja`((92Q9x0wGM_hXj7f2@C8{Neal
zh4}=>UmgX1C;kb%u3VvP{LICS`<afj*YTAK-4a;KCC1{<sLrg((aASaSVe5Ago5-0
zsa$bW(Hp{vLcaw?g{{QS$Y`mYP~%q)k**V};6BHGh%Jg;inEf-m}?G)2-`KLyZ`h5
z8vK6t%i&)oa|gQ&M*_3tpO-&`e;@xV`v1fKe1>_93e2&L)_;xv{P{cQZ}cz0@5$f#
zf8PEZ{qOBhrq7~praa&GjO~T)tJb$JAEZARyt8?e`KtY8$IJc~8=kLt8U42Nec(Ie
zm#U9nK790)=T*v^`ES+UPJeCuvhK0^9r2qFZ*IDI`TFIn_pT=1)_$_*`O?QH?tZ-)
zeq+PU%)1*N9(v~UI`VbS^B<2JA6<W#`*80=t|vcV?0whyN%+&2H+@g%J>+<_{)yv@
z@YmbkyncK8UDa#*r;3l&o@zha|Mcn8DKCzG*#5haf&b69&kB0k&!)Y3{^I+aAD^fH
zuKQ>IC+o-5FDpN4e~@?|_@4K(?9b}Iu7BtM*#1fN{epMgA3lAO|9<gzFT(<6Gp6N#
zoqy&1nE37am(`!kzV7?S%kx%PU-&$a9ZS#O#$W%wi+*eUIi2A;Ge6U`e~rIbzi<6=
z>*Iy@%^$CPJ@Zrjcg&Bf&+Z?B-kZGN`2O+-_RlrnyZ$WtTm3uo_q;!?ztw*H|J?S8
z<)=MMIhQi$C+6b6sXt=ADStin<=GEshDofJ>`rXkm^U*yF_$sc|Lgx3$LPo^#^J}d
znK6Z7GgA=@533T}6!vP)Ph6fHZcN7iCI5T;J@HfJo7b1%pZd%O9BHi4|C@gneJlKS
z_UptiLO+cD&t~#xH2y#Luh_qL49-mEOe<LGIWBPR;d;upoMGdi&Og%s6d1ypF0!oP
zIKsor6UO$0shl~DnUQG<LlomePHQn~X*n@&{sztstZppNm}6M#IE941i#`yV%B#Sk
z$!x`N;qS8FU4M)it}!#QnlZBeYW<S%>FdXxpW41A{$gb?V_D41_^;{*$Jf;#e!iam
z^6xA6&jJj_?7LY~{#t*HdbjQ6rKgIIe?FCXx9#(zPZjSY-#mC}_@d|O)<=?0T3&8`
z=k;;!2iCWjpD%l4d;jU(?0ZWdE_-J9`tV!Zx5lqTpD{g5zAbod^5va3W<Fl~O8?dR
zr*03v-JW>!-SuzRcil;PTJa|9{k#vXAHzOOdhhej=gr~Q+upW)-0*edcggP)zN&vw
z`%>}s(l_a!Z~t^N9Axrf-omtk(fR+c-|v3v{eI6_&vQpaQ-WXIO;}yv6W@N`Up#mC
zT!dLAeoHNuc9k}eIU~1SAzg8+(k2yY^%l)k&24Ibm47N8SJYPFP><2Ptj(g=roUDH
zg|3aZl;#KZP<2y{eY$JSK3i|IvNyHS)6jHL*HL3olhiEJb2ADyYSypTmQc4*UaPQO
zo=f44f~>NRnyMPJQmag?IKOCv7@NdY2{)-{($3PM5-Ww3`DSq_vdm#jX42zS7Fi=5
zCle--BN)szk#!~0T1Fw3bWT@(LE$dpOu-huIG#CNmpG<#JmeDRHx*_RStPKMtB-l|
zKdFC~4A#u&*bZ~1@Obd;=b6r)$te6M`^Wn4On)Y`F!Mz6#dA+!nfv$lk4s;Ee2o1p
z_4D0-epY6V=N$R$`Yf^xkA7eOq5s4BSKvPbrfB9h3^~80ejNQF@GI=s_g_ALj{NIj
zEN3+Qcm8L@SKm*YK6!lK`|l#_cg|$4KWzPs*Z<V~)c>yZZRfY@pDlmqGn6n2Fzood
z`M3Sgwy(LL=6~w`+VtJ}8}FxrH=QpcUhH~#=#|lH`Pb{-7=C#9;rpBE&#ND=ez^M4
z`={Km#XlB(UGVklNBg(EugYJ(eo_4*`^E8>`(JavNqlwvIrq~ak3K(?ez@R)@guco
zny;q5{`KPS<BJcSo(8^bd)4>q`>WM&*1ntfM)uk2`-Zpo-nf19^}Rn&)8A}<Z~rdi
z#k>dIH(y^VzsztY=xW3b`@6RuCOld3<jmuLkBgtiKkIw`^Y!^pAHK8yyz`aoL*bj|
zx9i_)ef<0R)%R(CE;H_9W@0$=)9;J=$3-8Lzb5{cVwUFW;#tR;!LscC(mx5m%YPUC
zz0WAbewyn5&u<<M?%(W1tP(5_S(b2c^N9#IiWQ1A3N7Y)&D+g)g|C1=U2wA4Bw0~~
zU(&xty9CyAS90obYY6NU_m+21{;jk~-b8AqNSZ(+?=$Xn9uNMt!YjltNNf^&E7ZWZ
zi<5(`o8=*kFIy4&EmjVuQ~#g*pZ|Z!Kk<JO|0Eg0Sp3-1n0fvs{dn~`;ZwmUk<X{U
zCHy_ew4Ei2*_+|+Kf(XWf1Q8L`^opm<NpK3$xJsHWd1GvedOnapMAe8{zWprV4lfx
zg{hXYgvps@E9(ljes%`#YXarM^#Yf<ojG2zHF8vO^>F{>(-Ju^DlQztw~b>tOC0kT
zM*IJE|2!ESSPVEUIMdk}nHMo`XME1Ii#33=n<s>CIgc>MT-FG7ajrEyjRL)*PBO=p
zS=7%dC(AWUE)~rc=@hLH@03ne6jqH^@svL={!^$#=!M`b{$zpkBDbZk%X2GklRYYN
zQ^Z#2lVG~2gY<QUr7DuDQxr91>!rFSmWsU)eJ^%P>ZQUR<qHa2(i)<t`LlS|bG2~o
z=BpK(EUhhlN32ZXG}mf&Q`T8b)0sB1Ir8ohHW6PddWb)eqmSwN|M35h{;y|T#k`lP
z@L$!hCqHg}-}P<bH}M}5fBczj*fLntnO6Th^)vhH+)qg#k9|Jy)A9c?#&pKT|IYl@
z{^j&t>yzfYUvCe8ob-Lx@66v3Kkj^y{v`BK`IF3-b6+{XPy4R@ea|<p?;E~H|9JRA
z`sa*a!vEefykI!}Z}D%JpItvU{mT5k_um<oDI5|U<t+FAt^alVXT}fdFE2iv{&@MT
z&TqZ{5&td!O#8w4z2XPU@6JCn{~r3M_s`|`ydTv+j{Zvcm&Yj2yq@(frwLCtmkzrP
zv)upJfA0O!`?rHhhHEkZYyo-R8nyt&*$j;g2mgNmW&PXwzX$7Q_HWEae}#Y8eBJtm
z>D%e=hQGT0X#ckOs{LO4&BIsoUM+cj@vYv+`CkQo=KU=BHs!;!mo87IJzDfI@@f9t
zm!IvvUjFF!THx7&Crck4zx(#q=eq}<+P%;Dy6ubp`)e;AK0fuZ>CxOLx1MXh-tzkD
z3)`n6kEcBndF=i;?Xl~#hBtpdR(zKHbn2b<%R7%8??v6+{Aku2%P;G`ZTj}~>#i>y
zpXEL@yv%!i<YCKGySHXvq`&!mj(_*-W!a13m!YqF-=%(9|9Q{Hh41y=OT0IEC-#=}
z?csOUpK8C={95yO^1lhcQ@-8!B=UK~_nH5KnI%~^Gc9H~|9?G0=>L5`oxk*d^7@kZ
z<Le)T|4RR6{d)E7^Vj9y5B>c5Yxl3mKdiome17^N_QQ{ll3xP9CVW-@`ueNg_g&xo
zKC!)Ke-eBD$z6$ue$ThQvVW=ZY{}!mN9vExKVZEpc2oZPi|ZS2+dSZYwBdflEs1Mo
z*B0M2zaw_<;=MC>>~6$gI(z2JNuN`*PnVprKg)Z*`_hVQ{WtbsO+V*w?EXQ)!>uP+
z&KF-kapldW>T?H9F`Vi@X>h#e$eY6}j(443eQoaz<7+Q2<ez?eBKKt5=_lvXFNIuj
zzM^v}^8D3v!55ES@xQ@!r}cr!bKCdgUn)O|y~%j`<{|gvJ1<&3e*US&AjEu}$&pE#
z*@21u-@c#Mf0X~(&pd<smw>;}MgDI*A9%F+Ecy9`7Kuqq*+`d4B}*zvDa-7anI-*K
z>Wa(^#og-tI+JzPwbrX#Ral}>rE*y--N4@Xl(D+;dZTMb)<)m;Z|Pjnnyk}objiZU
z=BCXFt042qCKrv~7~U~@W^&lH+vJ<!KYbU2GDBm-D18at)!Mu|2X*J_P1NyL->T3n
z{ai9$#!<0KwOBn`T~h6;s=9iR`eK!6#pm*;6!MfGs4-~;Y2VV6RnJg0Q?*ljs9vp+
zu6{$sRN;<Pg@lG=nB*n#En*?!HIjWYlJalm45Z(R`Uz<ZO&9$obw|EYsY)qZ;f!p#
z^ks>YqQSxqLhFQG#SA4gq#sBXh&2g#^H_2n;ylWw%2UH5#J!pQ9`jnpAjZp#tC@6|
zmoiJRUE*@*W8l?hzx{vyuW3KU{$w+#ux0bi70?ze;s3<L#r2RaomrKE`9CY8IBNmt
ze(qYX>ufWa7csH3l(W6&xWJvqXT=xDZNgc`8N_AJHJ?*}>m6^IXsHap++oRHp_APE
z**CI%VV}UgSs+qmk%)m%6yIeoXO6EdsSHhjkNrEq9LD9w&nj@1w~%`tCnHBFdk{wq
z*8%QdTvs_vIc&Hbc^bGsad@)FbENP*5O^;b&gaeXj)|S&{lDM;UNWv^f5!cadjsbl
z_RDO4S^1bh{(b+m=BN3eTMYfI=^PH6nVeCaw>X+P?sDAV;9^f<_Wu9y&-A~M|M~vw
z|6Thh?EeZDJ<cl5J#6oo0~tC0fBnbIw3A()@0?(`;4MB@-fcV=cuKiXa~QGzX7A&=
z#=Asdo#0G?LcS6n8SZ~v?L0quZ}XIJ7PGms*>eW)EakD_xy!31_(6nGB1B@jXqjL+
zj|PW4yEoTWfe?ugGTkzlBp!-%3t0+E@{98Xa{c1+<n<BQESxB&BeqXy7vDM_CEiZ{
zTY`%P)A+4;KXVszKjC8GT*0=5bu!yU4jt}A+_9W>EZTqlzb<+I^6lvl<=-NHJ@~oh
zo91VQ57Xbezt(@*@O;-Z%NPA`j($A(<<jRx?^R!Wy|8~S^1|@d%GU>8yS(msb?D{O
z7yQp{pC~+%eq`|E>9YkdL|z_v{`_(5eSy0|_pBbNzp#B<`#$L1?pL#)dp+}f^5wqj
zZRs1XH$3iSJYsvU^WxMq`KR7be4i*jRe#3zZ0i%t#~&YdKAimM!efaiLQle;zIws+
zuJ04~SJAKUKk0rjeW&v7!$-BBTK~^8IRDT8%lxmJF`nfm^K!<&|L6brWYl0?!*Ppa
zBdY)t55vL#M;T@?U146#?8WqgL6$L$`68<ryC?fZc3sY5?gpMbo;lo69A!*)e?R~J
z^M5`2Lf%WfcR6EOotX|W1pd$ev+zgh*SDWHeDV4|`RDGRVc(fPZ+xfzHs}3<Pqv@k
zKGwh6_nQ6H(U+XBxL?nEyYvJ9XV<Tq-}Qe~e!u!D;8p9RHTT^fvAvk{X6GA)mye#<
zJ+gk-bAQ)u*&DyE$zMNq{p+og2QwbOeQ@E{sjF#MPF~Z!WpTUdR>-ZZH$!f$zOH*?
z>5acP;%_d!x#rgAJ4f#qJY>87`8wCdgXjNW*>hLtsqu@B7mP1kUNXNt{Ji|x<0nfW
zt$i@{VZ!sY4@bU<e2e_}_3fKCMsFv++xvd``vdQk-^RZ7d~Nxb>BFB-N535X^84$m
zUquW%nC+O{|9t<-`DOVx<=@Hw#hF~0ni=%}-}vvs_?}UfQRScLubdz9zoi*-*dDQY
zGbb_>G59dOV*bp$p1GFg6Z2JuAOHPX?sHo5@NqBTsOGrLEzZx(&(71rzKUtee`^ML
z7EzA>96D@{jQjpB|NHS@=>HA>7XI1s^XHGcKNidnIqW%av)*NvW?94XlJy?DA;&(}
za>lH`KmL4RC}(YAU&}t1{Ut{L&vX7q0$X`4+3zuY`kV4Uhvf#B7e5>SHm+3mmmDj3
zmhyS<x$wwwIdJXa;T3QamJpdHAkF=n{Sk*6Z?|BfP&9u#R|4A<76sNgHZk^Aw%aUE
zSt2<!c;4|ua#^!7{O|r{^_!PTmHVSWq2MOI<2?L)yaJc`75J2R9&mDTp65O($Sko~
zN?c-@kO<!#9&tW#frtFOye6DEY$w@XaIWP!!RyJV&+jj2A<80kSN5T-n#5E75U!n^
z9lZC2Zi}uF)fUkbz9sxkG(_B7EKw+rH;C&!*CoDeVHVL9g44J!vQ@GkW0U7FXMfK8
z<-hM=-M_s5vl$JUg_$QXSp2tS(qPx*%ICbpBK80FZ?Qin{|_-Iu+3%Z{J-?q<{wvo
zto-5oUHr?ZcPm~!dL{MY%(r#F#D58XfBr@9>&9<R-`T&_e7^Pm#B0}A>TiF4eD?L|
zcmMCD-@Se_Gh{Kg{V)0L{4MSi+o!W%+5eO?ZeU)?w1mNj!I8;})s?+~&6%Z!C745=
zk6T1VVvj_F*e78L;fo?_5*bo~QXeIx#Vy6oiZ@Ad$gs)qO0Sd_Q?OT!R-daTt8!i8
zwOqA)nUbu=5xo<J%=(M85;RX~x#;}R*4CV^HcQ=8+e81Wai2x4<x5j%!)U$7x-t6s
z23-2zwa;tht21h5Xq)Nm(Pq&8psj0g&m`Y0!E}z%5nVBj5S1>KK8;>o8U1o?UbP2G
z=M;}CNXlK8xFDp!U&r^1f2+_|(N?jG!aMoSaEfvya9!b1;3?w#z;=Z744Wy(Irelm
zPS*d-lUdHNedTz^`ItkW_16FP-%Eal{k3BhV%BFg`D^{_;LkU|RDRF>A^cVO^TAKO
zpF6&V|DO6U`LEK?h_BLL#J?{6y5+O<2ZJ{!UbH+<c#-t7_vNk^kDrIV;CgxFMe_@Z
z7v@iuAK2fqxbx_~!&8x$jxX;#w|mj_D)+7OhlG!vAC|nce|!I}+ef8uGk!k&x$Q^m
zkH(*Mzd!!H_%G~F{`Z{E4?pF9x%oBX`-UGaKcD{U{S)=~(%+?j5B@6tD)7<k!^Y2F
ze)|45U^xA^`uEgdvcFgVe*bgLH<vG3pJ#k-{`U2k*uPnSfBkIyTK9?dQ}!qEk3Zk8
zeR=)K#fN_%K6@PYRP1T{6Sik7UU<Ape%<y)_|2~uaZjutYCcYU#qp8*)Bg7|@0{NL
zeWU-T;DycO?E8s#_-`$}wf`>bgZTT!cRt@}xK?>h|Hguwg*TsF^Su1}qSNIyR~}yR
zz2bD`*wx<a%Wmwu_3`fOdqQ{AZ+KigdVRv3u!jd8mfs7#-F3_7&cnMhcdT#DyrF(m
z^On|Kqem{!y<eSs`Sw}N)2!!TU)_3h=uP?SwpZ6*&wRJ)gW4y?&+(r$Kfe3G{7LIe
z^Ea07mS48NXM4-|e$LlFzt#R;{&n;x|8M?(zZtq1FEJcvaAFYs&%(gXlFzY)Q-a-$
zIhgSr6EB+&hY806)(K4A4AU8|F!nQ_X8FPr$&$kC!~C0d3fDfKqnwjjUNP1)3v+zp
zZsHc^U}F_yzQnYerG)J=n+}@+OAaF+LkJ@y%ORF6jGO*A|1|h1{=4N*^`9TVtbZ*0
zdiV>=x8pxH{0{i5_*e7KzCZK-XE7Nu#rzZc8Tj?rmz_VP{yk$bW$^x|^7sGWng0v^
zPySc%Px|llU)jGn{>^7fXX#}s|3CNdzrUsoCz!ZcZZL)YZ~Oh?XXWqJ|JWH1Gp=U*
z&3J+NGusMIAFluGam*b5RsZoZJ!JpFRlpU_QP1|1Z5t;S&tINzJf=JocpCTxMffB?
z%DBsFNlh2y5dA9}FR@f|r}#YKNP%_yuLL#3o=S2{KakuiaZ{2(mPa8*F+d?g)<num
zQbZ<NQC>Arb*a)Md1rZFMM0&f3X<~QWE!LuW#r_eRn};_YM<8FtD37aL#1AIt*W=O
znOvC!v*=XObO~XpbCNqHdc+;YXGtuO%8|Y&`9Mrb#6|d&XqePvnM=}567?d71&sxj
z1l<K1_||c2aEfvM=e7~x6K)p@=6B=1z`mI6JNqsUBepq=i~b({edy1=|7A>e%%>T5
z{V)3epJ6i7BPLD8|9{<nZ~s;E_bAf^HW79KmUaKT|5*L8``hui{Ey2ohab5=8h-mS
zbh3P8RbdHclw{y#RA9ctBFK7@>D0f8e^&hyWbR>G&u+k$!s5lcg#A881Lr)>K=yr1
zT>riP|7PlEo6IiHCeM7Dp_<W{<s<8A)>fA1%&%D9vj64w;4cwy=3m1j!gZcwB4;Q!
zC(jH{ca{o<w~P{O#T;MQ`&oOKm00xIV%g`hOR_IxJ<oiek&)4wWggc<K6AcfT(TT8
z912`I+&-MVY;r8pEH13I?EG9mxHfRkV_(FY!t$EMm2DI24#ukA%f7XKyYh4QpZ$Lx
z{@nZh)z7<sm;Il{V8s;9)cfD+x51C1pS%C<XPU&w{<q|3{ZHRN=KogzQU96uE&1!l
zub;l{{bBQK>#vJH?|kR_Zuqn4PwQXH-=g1IKA!zJ|7+Bb6+axlM|?H<di>kP??FE}
zej5Dv^ZEAM2QL#}|NhYZb<fugANAh6eOdTs@4MG;++PK~N`8Ikb@f}DkE&m*zH9u*
z`F`wM)Q`Kr1b;vJ-uKDt-K}>PpKZQZ{haa3>i42QSN|Pg*u%7qrJ6;PQRk2Q5B{Hd
z{|+*-F^T`n`z7<U>Q~U8;y+V=yZo;E{o(iSKh^&pG9)qGV4U~g?XTmXl0Q0sDt<<N
zx%Z*#Q~tLL-*<m2`BL(+@4d|Xs&@(RcD+CLUgb^0v*<@JABnv1cw6u;^0n_Xp~tn4
z^PXIMRP@07{?2<s_j4b7e%SX|?Wx|=3y;3uUwcpdUhkb5H}$Xny&!#|<&w=+ziV<=
zUtNm3lzUm`isjYD>uonyU#q#~d+yY^u&c9fXWyy1dEu(=l`GdK-MV}0z|HBm7T;FA
z{rA>}J1+NgA0B!1<<Z}ViVvSZlzPJdRQ+-8!yAw7Un;&!`FQlxk53D}{QrLUm+ij<
z=63chj<sx8nO8Dz;jrO*$2Wu9i(QnpoqZCwGmi%MXYPf(!F)}8iTrIsd=hRlnlg4$
zmQoBd_hqx>@?|zlic5LP-jgp@+^cj~>5!7CN`=~H4GHaeI$yONG+dS3<oT6nYjkOv
zsm)WoA=jy}UpZSjO)*`uQ87<mRJK9JUGBMJt%|d<mI8~snL?+cnBpbb?-I6Rx5NS@
z-bw6{cqg7A{#$IPXp5+hM84E92@R1deowwmp-hRrQhJgvMQeneL@LAt#Fh!O3q2Fi
z<}cu7<_+Ta6PYS;SYn3gS|K-~QeiP+M}8+R9*(^n*Ekip<#_M#&gMzx4(4{_`N+GQ
zZwJpu_Ee@D|I?Th*%`Q$xlA}x*k-W@ajoW>%(;*=hf{{#mt`{ZW7f@_tUONKog9*E
zovbNr>)Fn*NHhNZyXx=9|F4<MS##JnvX!s}vRSela_-<#=Gw$|hVj$?1I$vKI^5A*
zg&e_b`&bXMb+TV$XW;N;U(5Q9xtRGht3Ov7?`xj*oLg9vnRhXVuw*gK{wMrr$Da-V
zWtjumK5*D^iF3W>EaXn$D;7A(@4>T~-GX%qn<1AFuQx9{mo!@!s}cJI_D$?eoVJ{z
z?9wdCOvjj5*s3^$*_Sbe{tNk&`e){!&fi8qSAARc_3O9%pB2BA|9ts%_DAIRxbF!+
z=KuWn^VIj;&woGUe!lfx_E+!E|KA>dKKY6JOW{|R?}vWw{{7)s>kr}Yd_NBVlKDIT
z@2B5aek}U-^xMqupTA0cTJ&z#+b8erKF<H__;vT!318QL75{GWYtrAT|G55|{MPzi
z^7kRba^?jr5zIXdC;u(_f0L1sWdiFLR&Lf=%zaET%nR9cxPrO0xi)fWaU9~f$$5#>
zoIR7dhtZXZgC&Noh@*r<n|(j~bFLeFZv<}(%@v%@AI7(p@2x<#@J-=D!Rfr$xGcDq
za|Lnh@vP?F#c`5#8f!KCCC*BoEqvvCpSaI(I&hkD+Hss_G5c@!Tk7|TKL`FU`@8th
z{okE`uK)e{@9}@B|1W+!{do9Q^sD^0x8JLOq<=5@y883`&nv#U{cQT#`Mvqe%8%S1
zZofbGf#>t;uV&wNeD3<N_ubXE-(O#O-tcJN{rU&<9!Wf@eNy-M)}wh(#Ge<xXnFbL
z<<1wP&&8iDd8+&T&a3XXU*D{KmHM*g)!#Rt-_^Z8^zQnb>Q_cDKfc`fX6XmdFZaJ%
zfBpG6;<N4NbDttUTz*~r(&CN9$3tJXf1C0B!*}_gOMf~2?)&}rckS=sU(P?5{S5o-
z%iO^Jo!y;nC(Aq*7q)90Ox%mPlerQ&^w=k{S8^`k;S=Z;)Dh_6li*hrVC7%Ky_h|U
zwVZ7pXCluhUO&EPyl;3__%864@G9^=<`(7Z<XFtUfMXYTINv4SC@yEV3(PZ^D_Op<
zl(1+sPiOr5-|+vn|Jsa6j9dO+{8RcX`{(o@r@tNgeD3|_H=EzMyeoXq{Ncs>g7-on
zl0HU%)cNr9oyJ?cS2vz1J!yEX|J3<;-t*5-;vUYpzvTh<<Lgh-pT2$)@?^!64Np%$
z`}17>W#CJ}7n#qdJ)QS-=hK8I6CNzNbN;s6-OqQq?j5~beOK$=)cY+DvL0@Fbp7$>
zC+$xbJevOS%j3r9`(A`RH+ahObis?c?>atD{(9@H=(pY9!oI)!?)S6#_k%yNe*}J~
z{W|z-!f(~z=|7sj3VvPs?e`Cd-z<NE{&4@F%9O<H#4N>H%ihDm#BrN7jKz_yh;tri
zGn*cB8B+-BR!(=GWS-gF^SIhL&v7(yJmomS)xcZBe@wt#=$!COkv8F}{NK22IW4)0
zxs!Obcx(732<{RQ6?GMn7tIpW6-yI&F8D&gN?25EiD;hC6}~GxNqizgCBn%<eL@R_
z*@bn4{tL|#$rqb1aaZ!Dq^i_LsgshQ#I_4{@tN|>;XcXzl4n0(fq;onl5nI@n?Qg-
zn}8I*CXX}6WY!04Qe0MCO6*oFXPN#o6|y9;>aiBF{$p)rwPD@O7QiXWvx(17;F&;z
zV4$F-KrNpP4-01q+a8w7%rBU2SkJQEV;AL6WS_&9$#IfvDVHyYBHLuv4{Yk3s$2q`
z4D9<^3)!4G1UVx(yxAk!+F9FKJK5f`ui;$CeUIk`j~h=K_Y}^bZ1-8xnAb9TFl=Q=
zXL`$ggGHJ(m8F#VF7sa2R(27NdiEu3m)PcT+~g|Z?%|AOb7DEp@{moR<0U%-J3ku(
zs|1S)3p1+)t1jzhws#!u9R6&3Sb~_v89V+R{N4ET%1`z`Hh-o6CjH6&)&BkW*Hzzg
zeq8_g=O@R{1wVHDJoW4L&+Fg*ev<ug|Gm(s$6wxlyZl}2$HyNlf9n5C{PE<+<6r9k
zRxv6tA7lE<w3a!T`6c7y|6KpH{$(;KFt20nXWz-L$-bWb9mgxqzg+#?Gq}!kws9Nq
z$@5$B`SQ%=TEu;oZ?eEWfq6nXqR!&x68#c$Bz8&$%j}g8P`<C)p~j#lpn634w&Gp6
ztJ1;JUa}VQI~2Alv8YT?{-88Zd9BKEl_q6l#h>!06>C+tsbnZWR5DV&qtdRPqEV%O
zRQ;{SVvRy|SGCJ3^OUoc9aQ2})l|)teH72j$I6CD<w{Hu=M!Hc?k?3M>meT?e_!s8
zT$@6jlCx61{9c(r=`~WPB)KHm#FmKgilmFMh<*~@Ab4Nkhrk{FCA^m0+?<XaW}K|t
zcDxb%o&3LeTsc+QjF_kYPyTcFm)7svzuF9147~rY{ptTB_-Eno1HW$nVEz90tMgZ}
zFWWzUe0SjOledj;)8DAP4tiDlO8E7~SDLT*ULAf}^5W0)S+8!q)A-o&f#u!$SM#55
zd$RXI#NGD03m%9)aeG$yO#SKm$BB<$KX!PU_4MJBwrBk>@4mYJD)cq~n}jzyZ}?u%
zeW~*D+{^2)!rnZ1m-6w+NB<8m-e$g8{FdwE^v@b!)V}=u{OFV3hr6%&UU)y(d8zzn
z(Yx>OMBaUP$Mx~mNBK|spU-|#{Py*m$oCiD{(dw0zTlh7SDr5~K1qJO@V@Os^{1XM
z%HM>){rt@Q@zp!$_j^7b`t1H?(w9wN)_%V9LHW(M=M2wJKM#M-`z+z<&nLZ48lKF4
za{I~Qr;nbkdFJ=D`AO1~n#W;}5+6N$RPt!o1M7R1cgk*`y&ZY?+r9sHuiVPL5p&(`
z`m-CAx6|$(yk~m9{;v0}z1MGE<GWsRz5BZL^*PrbT`j!2^(xPG`I}z1mG0)><GrtP
zf8D*-yYhF~Zl~Tld28bBh}(6yb?-4e>V4Aj^y`z%$6=42J>-3q_oU?I%y)A>_I<kf
zvG)DpH}$U%zMlE!-dnx*OW#Mm4|xCO{p$}WKkok&^hM>{#_#<<*Z=PLQ~M|9Z|6Uc
zf9w8g|9ky^29qGGJzFN*d)8Ezm5ks1rTlxqAjf)}V*=M!&UCf{#*Kew{FMA*`@`zj
z&OhP*w*G&~SjFPP7Q^<1<pc8$7G}1sY-`v(IgfKq;eO7Q#Ic|C9m@{Zvux|wKX9zz
zn#{d~%Zj6k^)Yi4%VV}JoNKr?amsN#XFbj2$ME5Q2%{tOeHM1Mh3w0@WcafAx%oJF
zrgF(}dWilNi4^kYeaUu%F^sX9<tm#v+dF1SrY=TZrYlS@nEaTwGo1dn;$H$|E-ME|
zFuMWs!oPCAn0}f5QTngVn8_&2xQvmTxsqiD%RZ(B44Mqf8P~AXaJ=I>$mz_w=YRX}
zs9(Z=ZvK1EV9aRAu#TaQS&{8CTQ*w&>jf50w$~hQdCm!x2yW!L%5wH!+n<GhZvTz^
z-@~B8G?PV&oq^Mcb2-~ero;bl{Fh_C%wEgApGTZ~37ZAuqrWMCPye0s_u`+me@^{>
z!1|bri`SCZi~B3P44WhSL#{slO+s%4U-B;FbYPEQ7v!AHd58TZ+XRkJ+?@Ph`QPzA
z=Q_br$G(-lnk$7bPe7XgKGz)9+l-bB3;tdBo$y2Q>;KOz->&{h`z83x>HDnDhd-|R
zsPZ}Nd(WS1|2qC|`n}`#lfUN}A{pQQOZmIzA2VYo^E}r5tjk!QF$XjMWin>@z~;fZ
zlQWKE1KUToZqAw9ueg{vX0dcL?_f3K$mE>DWz9W_>nx`hS2j<(fS8D~h^gRuzJ9*#
z0umz0Vt+*Eh-iwO6cH7hC+026Cz2-OCE6mUD7jeXjhwMugp{jjjgYCZpGcgruHaq1
zlf0Ap?D^O5{pJ<reZ;+lJDJx;&{gb<<TlAkqRs*rcrNlZ@NW|A5qu`VA?PQ#Pw<Lh
ziohjaG44Mce>vuGXY#WPeGyFM-_IS!`IK`n&j<eNf_6eZ!t2C-NEAuNNj6IS7gG^4
z5&I>6NG4H<MdgjWq-2b6seq0^uE1A*Wq~?Ddtph@bE2n&JNcuz?y*j1Ud4Q!bt&fw
zo=rTLIX|(@VdY_UX0c)V_D|*So4?ckJ^fqoC+p|OFD)MyzghE2=uOOf#*csB3BJ;O
z`up*fXJs!dU#7kM{%ZAGhL7x@+dtWT-1GkQo8p(kFY;f?yjlKs-kW1DH$6Z4{MQSX
zSC?Npyy$xV|Ao|RtGCMU%ib?|7xwnYtBjWyUN8J0^Eu~Z!dt~xJ6`d;kNKqg$>U@6
z$9*5KeKP&>`OCzw3%-8(T=Vho`&l16KCk*3`eoUB(>I&nq`rUq@ye&CAJ=^3`&{(d
z^0UO(XWy^?nD~9|*JWQH|M300>!0cG-Y;J7`d+KNP55x})1)sYUu(W@`RelZz-O7y
zzdo6M-1+YB+bi#*KL7n%^M(8U!<TK(&poq#e)5IbE8W)#Z)Dz_eHHL($IHm)8jn}q
zn{cP*p7*1;$AR}{ZlqpRJ$L5ZgbUX%s$LPf?r}r&#_St!Zt~sUcuW4K^o{Ntmu_vo
z_x<6HhoN^puUB1ixUlTPgNtdGj$Gbw?dc7LTX%22y3=y|>vgs(yDo`ele@F#{)@XO
zZ!}$gb6)Fw(S=_Z*IeFz_0sj|TSxD9J)HMs`qQW<p^tbThCKT8to4oTdyBX3FAJaT
zd3xvB@fR;%s=cy(wf)t)H#^_Qd|dM(@m=%lz*mRg{P?)}8~cwbUwb|_yl4K{|9R(^
zfUo<$ntg5ka`;Q|H<zC$ey{pn`Sbtxw4eEZZ!`X4wr4u__wLURKQ8|I{73n3-JcnM
z<o@>m>t_7Q@|=AVX9wqB&IP<4!U5u?;(j7h0vmWQ@x2gSE>bMER!m*AN4QMrjG&It
zc_BBUdjjqJtNE`9wTgd}xFsUUuf?;HCx(BrP_hV<sEVkt$Qz-V!Vg4>M3{wQ`Azv7
zh04VRr7ugViXGuw$?<~CpCgp3lP8>yg`b`O0UtNNioiaBKz?!FBA$P8x}sZT-YFQ#
zvr9FIdJBIQ-Ycdg87^fneNTFiOtRc*`6Kf0WT#7|O01N6t)QpYrS@LwwCn>3Z?WlO
z*Cdi<F3JllE>@VMuuL&j>5f9GtgYl>(QP8J;)`T>6`d3_WKT;th((G;izkXRi~ET8
zivJK_EICD5Q#x7PM<|d#R-j4bx`c$ZlC+1Uyx0!m65%qDZK5e+d}4b=H;Zvf_DlE4
zFiBg9Cke|5?Gg49<CB;tK2PM4z<NGzeh$GpAts?>0S1A~f)hk^#6F6y6J`}S!DGek
z%FWHw%)OaCgmKev(Vv%o6#bm|i{p3iPrdJX-&}sY{l)ml_V=!zYCo#KPx<lfSIFNh
zf7O56eZTNk>c{5a>;FdnJ@)(Nui3wU{Mq*R)?eelH-Fdte(|T_{~V?@EK`_c{~P|=
z_3h?2wO{#vzx@6G=h>e{f0z6dWQb(UW~yX9z<h)G0c$j;8}|lIRn}Mk=l;q4lksmI
z!zspjjC&a8FmteRv!7x!Vsm4Y<M86<;h!O_D_$gaU!av!o~4b+jpaO>Eyr|D37&pF
zC4peUYQbmxUc6ViOu34A!UUcPr-&*F@8^}`e8?Wo@rL~&TM?Td`(jQN9#39f-gF*e
z?jWw4+|2@i#mr^i%RZF$70(u`;4k1$5WFY!L&#aUS~NzoPexhxuJl1kaj|EDp#qvh
z4@EACYKb%nwDWA>Y-fMX@|UshzstWv|GXIvF|KBkVLI|(?(e?eZ+;j4o$$YoA?#o7
z&%!TrKXQF^{h07U_}%T-YhO)xW&3*9YxdVHFIPPic{=Zj@YD8ZjxQ2l#6J7__}ZhF
zj|?AMKbm-d{oRH43?4W<nDoHz(f-GAPsE;zKVR`G`Q7LDsqchdJG@x(yz6Dtn}2Ve
z-g3YB@cRFo^mj$?6+UeIVEFO%#|NM8e767E@-6wR@W(xG{9d!a+4x5P&A(SuUOBw3
zdHwlS=d0f@BVTsE{Pc41E53K@KJ$Oy`nB|9;@hA%0q>W5KK51XoAviOKRSMJd{_J$
z^ttS#+xwQcSKcoEkoe`xw<F&se*X9_?6u=7{?}7qe|e?zTIB7wxAAY{Um3h~dcpZT
z^6A|t4`1YbX!|<j%Z~R6uVy}b@`Urr;zzCzy&i@>l6Wlkxa?8WL%j!U@0#6WyK(1+
z%$>{kvmU10cevww`||A{x2x}D-?Mx8=W*gwlc%On?>#kn8u&=-{`7kckJdkTelzK9
z_v>@dL!PdF>iF!^lf=g<PgXt)da?iI`qwpY-@N5{Q}lxC+2?0ouSDJ*{?PZa@uTRc
zx=%)*B0p(;-ujvUOXruMuae(0erEi#`nmNR?^omRm;UH6CNWO?Z~O1)U!8w~|Ly;;
z{}=u*@Sn*)-v7Kz^I2uttJxx0!&y!<$FW>z4PuL7iDvBi@BKfCVJp)M76rBgY?Ih-
zv9_{(;ppKy#;M5hi~R)GEPf{8b3(BKM|jP78hNt#{s>GI-Ye=S-XyVAGF3WIcB<T8
zxhrzl<@gkAlq6JEtGrQam-CZcD4H$8A!;M$D*jyDP-2(FHpz)n8>L=L%1MNaxr^sZ
zpOSNttCeC9z0L2)tH?8*OPWiTXD^?UfUlsS@GfC)p#=U}JjXdrIev05amVpEbNjKo
zGUffd{fFi6;lFSHZDY8}q{n)h-GM8K`#l#c=RCF@Y~Q(V@P`Rb;1}na#c`4C5vvBv
zDyG>?*~}R%g{&*sS~w<hS#vMq3})|T`^nD9t-*JTZ#~a8&Sz}JEFR2y%xz4XjI9hI
zjAqPtSxY!NI2}1&v+iP5;PmFT<3Gx$!*h*8i2WgJ3Udu(FXJo5os3@@y_lb|#&hoC
zj^$p#`HUl#^97d`?*hKfyuI8rI0HF2Ibu2ZI6BzMS$;4nGOuCbWs7Dz$oiV)6{`;y
zAOAjqK7rr-fqX7Juefe;T5?Ko@^jR%u4g*Hu=xM|e+K{T{(k)Z^4Gmz@_!Egz47<q
zZ?>N%-~7K6e0lga{F~R;y`PtU;`^-k_1HK0Z@<3eeLegA@$Vo1&oNG7y!F5JkHAm2
z?_%Fweq{VQ`}@T28NYx3j{M{Qhx<?X@9bahzg&N7{(bvDo~fM4n?dYf^WXpfUNTy-
z6tOU{Okr8T`i?b?RhmVBNr@qeftQ7qvw|m@ZwFr)&n}Mdtmj!OSfx1bajEls;1T0<
z;oHx%k}HCPh4ldAOorEt!mKwr+PIE!6tewgjc0RaJHo2S%E&6n*23<?QNXc?J(AUj
zsf8hdQI6#bTMfrr4rg`|)(YmAOrFeFSbEqGa(VHd<ued?D!5H(g`fn#Ax|sUL$13#
zw*0{Y2l(6gZws^u6$*<8#|!@8j}{OTJR`73;HZEX{}*mX&fn}`IlgmsaJz7-vW2pI
zVUlJPWjMv~i*Y5>U8ciKlbFA=nselFyk^_LqRlMBoWQ(@shL6ZukA0%pF4ONe$4uQ
z_S?3vMqjso?flmMo&D$TpI3jV{Yd<|`}gO+fBrrD%ldoycek%QzFhfg_#^Ij!M`{E
z<NgQy-^>`vBFq}dBFOxnX);SQYXJ*4^KQm7|G)iB{Cn=N&%Y`E7X6F=7xeG?KkNUO
z{x$r)@muis%HOm9W-??l{b7{&fAH7rZ`VHyeop<e{o9DxYBcJ}(GVC7fzc2c4S~@R
z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7
zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z
z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GXw=8x4ce
z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Euoc
LAut*O!z2U%d`{oi

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/793feab2deb35e284a975f6527d76a8be5540fe6 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/793feab2deb35e284a975f6527d76a8be5540fe6
new file mode 100644
index 0000000000000000000000000000000000000000..8a9216e10b777da5e8888e5549ab39346de8acf2
GIT binary patch
literal 61
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84n8+BzAFT7v=1uS6
R#~RNyo*#K_@@^@AFaXUS67B#1

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/7f41ec3a9805c6b8f3656c4f9f6d0ff7dbf8a329
new file mode 100644
index 0000000000000000000000000000000000000000..aaa91f2f45bd5730550211f4a080e6119d15b48e
GIT binary patch
literal 6150
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_uqQuZ
zcjqkNQs?@?na6pR!-s1lcLC1~zI%K&JT2Viyp{Y5`7QYR_&5c)`S<fq<g4I6!Dq(X
z!|Tr<&-aq&9AAmRB7v(y#^Ne6>T)%b8bU>UTY0T{(|E=C%LQKX3-Sl^2lKDzYv)tt
zj}T}RyeROVH-;yT*Ozw|cOrKxw>sA+j#ccI>?=9eahh^{;@-)-jxUJMfv1qmhx-cu
zSCKEGnWBl}PEz}1%N1V99hLbi6DM<5YQ4k*Q7+LC5jNrP!dxQ0LIJ|dME{Aem;NQ=
zDkUg(M{uivf{?dhEB_b%<pQh%%>0%7F9mf);zVV{JEeSN`K2?&9*g7%CkRvvGziY(
z&*XLF?&ny~F_~wN;6;&d!frx0geD1h3q2Ib=HuXA#JQ5|3#S-o0=ELUI=eS}1=nP*
zha8XD<})k)xBvbAm*bynzZw1n{ciiV|8wd$)}LWNb$^t6GyEF=Ip>qf7uK(fKAn13
z^g8JEg*Vq<YrK+v+4~~$_3n3@-adJy^J2>rttY#l=|8u7cH%kv%b@4Wo^5=w{#E6h
zrnj?SJH9;f;_mZnPaB_1ee(5b?TeCEy)XKns6Ku5-0#`yrw?DYzj^W6_VtxlM_wIx
zCH!W_`@|0$-gdmLeShu4#m~FG8~tqfx&61$zj=SWe;xY0>hG05XMV^2dHnAf;~b_&
z=EbZt*}B*(Ie&8+aBOFt#`>I1hQ;dtvVW8QANnWu-=3j~VJ~A4OA6~DmMbjIESDG+
znaWs7n0WtB|7ZM9>)+!49sgJSTlHUtsg5O=rIG0qL)^cf-&Mb!{5bqQ=4Z^Wl|SeF
zDEk%!${j;KbByMV(Y!(ZyutC+_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T
z|MbM=@!`iB&yTz|dAIcKmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6E
zACo@_e3<@m%jfA|0>0RM4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|
z`5KEETP|A$dm_6edlCCmwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z
z+c`Bj*0VM<b1*jivHaQdGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ij
zeY@#R(c9WL9Iy7i^m!fgM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L
z=S?qNUtD`C@T}^2$up%VHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%
zpXYwL^R4Oo^zVkh;{LTUEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?
zdA0eu1YZiU^Bv&&%jv+w!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADK
zP2;-G>Bv#ZmdobA9?B8Uy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz
z&9j&5D~BA17yEYhnVjxi8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t
z%Y|n??^j-4z7BpFK{J6|{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`g
zTHptF4VNKLCbt25J98Lg07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z
z=py)0Xt&5*(c5BkBrZuNOTU*%kO`G8l71y+CwWJFG)0c4$ibB&|BAg7n=Af7{DQ=O
z=@MB>X$471$sgjz;*%t{i#LfF3YQDFicArW7SRxn76}(k5Zf#<L()vrPjZ38Mloj5
zDv{5^MS@fKcJb=-X7KFc$>$a1v*Y9DUBokqmzj4ecL0w7Up_Ajw>sB6u4i1bJTG|U
zxpg@^IexQ?vcF|H$*B16^pC!;SH3I$TJclir}>YS->3YT`J?nl&d=FDn!mmNI_Yc0
z7xpiTU+h0${@D6)?+4*`HE&|x?t1U>dFGcbAOF2u`PS=o*bCREO^>`E20lo>zv021
z$Jd_Ee0}px-<$fkH{bHUp7WgX`J?BI&zYV|K3VZt{_(=cI!{kN`}bn$>q~F$zA=3(
z`zG_1$eY*i{=b|1&h9<i2ab=sKk$68{v`8t>9@)s&wf1lUh=c?PtxDIKkmO3ekcE#
z`RBr~_Md-#z5kcX{F<$ZV=+5B+iezq<_(N2j0+g-7(V^C`7idrl3^uN5Nja&F18)4
zOza2QXR{Tv#j%yLzF|AWq07$8dY8GLiH~_Vizmwirq_%!7zG(y{xbjE@niR|(BE@^
zJ^#`6qxIL<Uw?nR`F`;G#c#{MKK^p!^QupgpH6?&|8VDh!iO*K&c9jk`ui)3*WX_E
zywQ8R@{R24H!m$-xIX*+B=5<Vr%KP{A1ghodg}OM(kqU)xo?7AKYhdb?#kP|cc$;H
z-@kbm@^1ILdmnFp-Thte=cnIm|E*=vU@~S}$GDF{;(yiuJVsB(1_oiq=ZyT!7noa^
zPcpZ&9A_<I|G^s1q|daQ`4bZZW6}Q||78B#{J-$G?9cCib67OlbJ&it-C#3hUCeC4
z`08)vZ?m5b-<7{D`z-KT=&Qk3=P%q}KYdy7DdoeL_eVd7ewg&O>`mvJrq?#F7QRe<
z8Tyj%#k{8%pIm#o>#6ILWsjac@O)tW(E4H2gNg?Y4|d;QdB6R^hKG+I9eNz}r1r_$
zCyLJ&KELri=*5MXF0W6&W_x|_)x@`QAJaeI|04PI!RL3M+CEi%H2L83LGjbG&$VCF
zzWRPI_+|Qc&%e6=PZ)ZcC$es2^<d-RIM05Wbr(w#%T5+iwp8|WoHux*1sDVsg=B@-
z3k3=m@*m<&;=RkaKwyhNpuk+gE5cl&+eD5EcL)mze;2sRyPvC^;}tsthcEjJ)?+M-
zSn607F*mZ<b3El1<o&}P!qv^G#;L?n#I}nii{&v(7E1)n0@fF7793feJ)B*f1)R<t
zitI^j!K@Eiq*<L=i&*nnm$GhPU(I!%do|Z<&JSF%+`618Y?oM9vh8Dg#rB(Bk%OJ1
zjFW+D9mgS#B(90v-Q2r5=W-n2(&K&2tHO7ir-iGW(}<&+Rg<NZ)tlobS08sJR~F|D
z4rY!#HV2kyrhAOC%&9DYnZuZmvK(Ol%{h&WgX=KIA$AvzIh;jYUfc}4C;3kEX9|21
z^c6WLnj$7EeoEXyVyZ;HM5lzS#7c2}iMx_dq;jOyWlqVs$o9$h$_C5+k+~<MB=c1=
zSmKBHDKR5a4Ux@4f<g+yhlKq_7K?q8ydbkp_Mo(q<WzBaF-ehBq342s1*-)$1a}C=
z2-OLTi)x8#h;#|(3Z)6m<V)u*;d#MznuCemmDP=zndvZNGt*z@g)9qLgjpk51etgl
z7W{4e<@+=6=k4!(-`c*v__6Ot>-Rn1<iE@R`2NG~=ieV?Kka{?|Go9+<!>jx9R2+H
z6U)cu_bWb3`!wm(jSuDT6yEs0y8OcW`Hm;O57*pJyf1!V{Jzfv;YSl5e|~KEc*3L1
zM-GoxKFWK{{aEOc&7=2^em;8isPVD%Q_<&No_oCLc`5tq?F)_<+nxtJFMMJ2`tRHD
z4+5X2eBSX@_<O*&^ske?3H<c_b@u0^pVGfNf4%xO<=5XIz27arKlv{Gv*XvZKa&{x
znR{7`Ssa-)nf5VA|F`+~`>(*imj5>yr!t>mDPn8o*ve_k*~%f$5y5_mt(ff!YcX3f
zdjZ=AmX$2?S+288U`=H8V)@K0#v;QU%sBi1zdt>{<bNOkll*7r@7upae>?nE{WJYf
z+}{KLcK&1gtML2n&lNv)e^!1k`}*SZ#ZPRX{5}?ZeDUe}XaCO;pGrThearu*;<eSQ
z#uvKJ&pnHLzV3O(^HtB8UlzTZ^=j=)i<i=`7~gz*EArm`!>5lOpPD}%{FL-*!>4m!
zYQOP*SNq}mv+C#RAIv`-epvs!`s>Nxvkaz8eavF4;%tp<CakK=_nDlSTbObfw=>LO
z$Y)Gs;$u3(;K5MA(C|O>-|2se4C2fSSWmKKF@0b(XMFRo@Ndr_^WV3BE&Q$a=k6c9
zf0zDo{4@Mh{d4=b3tw11Kl||a-NrZauO_|N`ds3<?ek^Nm0mQxSoPfE8SB$ukLEs*
zx^I7X!R?Q?eeXu!yLs>Vy~z9F50oF?dwBE_-xHl@r7vVhE4a}LZnT0Mt>EAl+)KF^
zyPx~L41MML^5YAkSJPkFz7Bf(<h|x+?eBYkvi@TJ8TWn7SGO;^pHn~G`pEgo{&Vn`
zgs&67d;hHd$@+`)ciiv2e@^`uX8OwXkXeIOkL?y)4Z9mh8mBV13-3NYDSk!%)BL{$
zP78h!yeoKA&_hUGm`Aus=%Y}D$TiUbF-NgCVxu)It{QfZ_&%{L(ZeFQgu8^E3)u2?
MbHCz@fmFL30Qs>JDgXcg

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/8210dc595a2652f2f812093b01e239e7918ea065 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/8210dc595a2652f2f812093b01e239e7918ea065
new file mode 100644
index 0000000000000000000000000000000000000000..46316baf29ff126da46159bf6c16fb11eb9cb23e
GIT binary patch
literal 55329
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fb=~&LI0w;jh)-Wq-B){rdO(zdJ+L|NVb;{^b4g*vRnjpY~tz
zKYYKJ{H*!$=li$sT0ic5pZ+cBtJ+tYuM@vK_#FKC)u$Dod_JxE82|D0$5o%0J}Z9i
z`dswo*jLr>4nLTGCjXrB^Zd^@KM()>^mElOt>1foz4_Vn^We|kU;V$M{_y_g{wMYS
z>i>KO3&u#Mt<1Mr(pjgl?qq$-YR*>5wu;S-J)XmmvxoBp=Lb$ZE-h|bo<}@ac=P!u
z2+R}+5~vpt5_~3jKqy4`n6QP2rbxQ*G$B8sRH4;EmxUe+c?!pf>=HdMmMvZ*o-ZCR
zep~E-=xLE`kxY?v5k3)T;f+FPg}w>R5|R@7Eci`ury#rF7ycJ~s(iM*i@94kv)K=_
zX0fO-GckEEeq_*Qe8#BF)WgWjXvgTv_<`XyLoDNEMhzxOrhdj0Mtvq1=Htv~m^U*|
zWX@#%%cQ}yknuBP5|anhAI9B`?TlrN#*Fn0YyV6BZ)I5U*Ya=WAJadJ{#^g#`=|A{
z;O_^&uKjxPi|e=T?}fkR{@DF_F<Nkq7F?qR7vX~I@%OV|kA2j6Z~o5aP48>NH``w=
zeg5f*%j3h3HJ%@NZSrpE+bgfnzG#0M^z`fV;8*>x_B{Xh*#Gh2r^jC0dR6_#?){IC
z|2`&v5cn|t<Cf3UzXW`-`5N+L&F{uPHNPAG^!}UjZ~y;Xru&TF{=5Fq_{YFl!M2F)
zA@emBGqzl|4E98JN%kW4rEFnr-fa0Skt~{Q8(0-s*RuMsY-YO8G>>^DYc=P4p1r(B
zxVLj^aI9x-X69gQ_+$CA=V$!yoIlCGr~FF$bMH_0Prq**Uk`k#{37-F+DDyFHJ@&N
zc>8wKo1(Y1Z#Z7<ed+T$=8fWO`4`+T^k4hDZh3D0<k+L;N79dYABsNYdQ|(!{juiL
z6VIDoy1uyfRNz_F^O9#uPi{Qk_O#-8!Sg524!_*<Uih=w=Wk!Oeb@OB^WF2?;x7@O
zO+U~5a_3vq_vzmaf5rW4V_L@6&f&%0!Y0jro^>VDcE;<>2U%-ajaf=rjM+uGw77S2
zOY&;-a|yl_VCOr)^_SCuhlBqDzcRle-xOX}{;Ps2LRtL%ymEYhc#XN=bNFy7aW!(C
z<C?~Gozsz{k}a3bfjyKXoO>&88t($$o4i@vGdRw$3A0AC>}KI))njF3eaSMPwTHcx
zOPXgd*H;cX4lnlY>@zvtxi)aBa@BJG;uhn{<l*6Y%zc}uizkpPpF^2FmyMHs6T2IS
z5SI(jeBQ6TynG$}GJ<9Tx%{UEgG9avvkCPJs0joLa0`6qtK~K2?q#=QW#G`~yC$?%
zc(uR}?iwybo=k27_IBnl#sG#arX03)9N#$uc^2`h3Z@C)5Om;s!kf>pEl?u3UPxSs
zP0&T~qtI@VxuUnl=15$UOqPBxlOPi+T_pWV%1-i*_+PP?Vsphmh+mM{FI^&QDXkzW
zDfvU(SbUPicJU?=L*a7aR*@;9(IOhc(IVlZ31XWiW=NVz`bjR3*eJ#<S|##XxJYmc
z-!5K#-VB~SJo&tWe0F@?yo-1y@iOyH<qqHx;LGP_;a2CG$MuX$mgfbJJhv`qC&zDg
zQTDeiCm9w0o&M4H^~!g}Un_np{51cu^81t@Gk=u+$oV<@NAtJWUnhO7_`?20@r(WE
z%O6`m?)@PAuI5e5+g<NHKF|EJ<>SA1E8lv(4twGHwCR!e!@vj0_cuJa^Z45HnXhlY
z>3dWE_U2pO*K?jTK7aI_@j25|$tNow%RgTDSm)`<Xa8O-eSPW8-8ZIhW#44J5_$9b
z-T!xU-`Txq`@r#W_XnO2)}Lg)F8x;d<JpfV-%EZr{z>{<_s9LW!tdliGyh!p)&BF(
zulN6QnP0ONaV%zMXS>bf&%A+=g>eCc9mA*pHvh%`S2C<*3Stdp-^I3rm5Kcz`)syi
zwm7y@);Da2ICR;0S?@B}Gx0I+X7OZs!1S7N2BRQj%U|Z7JAUl`75aPbujfD7ezgAj
z`s?qHH{TC_zxZwW*T-LOd|ve_^3&;$`XBDRPx$cV-T5~QUVndO@%r2Ao;P}LSH6*b
z{pO{`3)g4ApX5E+@>J=W{9~m@RZksXOnSxfHup`?>!)ux-(7i|_s;aa_4_yPLf-9u
zckkoPue-m?{rvQM?Z34Q8cfDa>lpVjNc^w*pU3FQ*uWsn_?(fS`2uqb^GW7*mgB4?
z>_1rJne>@<Gk;=YU@ZE-<DblboBtR7mi_ttZw`wldk)(%wi|4Qtc#gV7+?LZ{B8EL
z;k)v;WuFB;3w<^C>imWK>!&XZKBav4^8V-t(GQc}mc8kG)AZWr)xwvlFGFAQy_om(
z;*)DncRh7|vh2~b2c8d%A6h?*dQkD8;lb|vEAO{I*zoZ2qeG8_p42{B`$X~C!sj=h
z2feuP(&hE(*KDuvy_)z|?qmAr`(Gr#KKT6ZQ`@Jik0u{{J}7>A_PO?p+E?H21;0%H
z?)g{u{|Q4c^F-E-tR8F}9Ov0Dv+iO^V%f<e%9hH0j`Id@v;c#kqL8fcdZ9qULjFU%
zNxXOY76@z+2o#trctw~?beqUg;SON|;qL->dG~XbbG%|_;P7RC!Fr5k5lbD*BIZUG
zdyc2vg1mpYL%6y*)i{+nir99sWU)ME$zq9MS-|>&&4MF~vxl>bvw+i?Ly<j+EtvHo
zi!`e<YY}Tc>r&PY?5nxXbFb!l&G~^VmRpxIh3yjSO16D$uh@RGD{`=NlyNd}t>ZYv
zk;FBTyPJDA=Uk2hTzb5(c~$ss^R#f4a~g4Uvud)GvU+p8<m%(D<jUf_!NJUt$L7Ei
z&2*1ZmN}K>FLN04QI-Sjzd5IIac~{xIK=M4F^98=%Zr<V_axtG{!D>yg1#c>L{r3M
z#ZQSlNKBRJm*|vml~^gRFL77$iByiXy38pV7ui19UfE#TKQi}Zlw`h2221=9KP6@)
zsv)vjNKi;Y_>i!_$YQZ?k{4vQ$sUwelAJ0oFD5CHD)e0NuVA&HhTsmt7@<00aZxQ%
z4UsP4T%k09nSANIB|I;<PIEA^yRy15Gcz4#Y-ak)ypUx9i!f^>iy#v(!-Bt!zkGiN
z{=EIY?_1mV7eDs>X#KwDoBVhAAK!nt{rvl*?5F+j^S`(Ly!`FNm!qFQe`5LA{C>rU
zX`d#2y78g>ox&U6SC?N{Ki~1B_u-oRiTB0ti{JNoApB^;<Ij%`A5VCc`N-kX%13#R
zxgQHXvU&9W(a%SZ9yLCeek%I>%X5zxJuhWny?w#)V%zh8=Y=nfUjKa?{z2f=l+Qc9
z3V#pymi~3pH-Vqtzs~-g^i%p*=dV}4ru_Q*qxZYz_b1<_e|G$O_Gc19KXWgOF^eO!
zCeuC!>HjwWe*YEt*Yf`+<5cD|EJbXs99ubUIa@j8IU?9Eu@$pjVJ&7WW-nm-z_OBM
zKFf8M39N~%UM!!P#aLvRgBfT4|M#com;CSJf0F;~{C)d(=x>MLs(+^ciTiut-_Cz*
ze-(be{kh_&?$65aWnW)>zW9mlli$aJk1sx5|Lp%c;#28|wQu>~RJ^u&)%Zg9`MGD2
z&(}TAc)sd6^UI=FvtF%zY4N+|xBc(kzb^e;_M`lJ;<x!<ZNI+xGV{yAFR#DM{j%Wm
zj8FSM+JBt*k@M4&k7qv`e=_{k_EGy|(Z_cmb3S#ylzzqd=G!OUPjw&jK4yI6|M*tq
zJ<G=pAJ=?xf1}hc=O?*PG+Ts2)JDuz{JFTH#4d?#k`tviO1+kplL!}c7tfbICFden
zE5#sso8OUFk!L!WG?y&TUOpuOUqM0PUBcW#3H-Boj&quF{N!Nbj^lCW_G5Qt%KLZw
z56j=ff8YMw#&DBKkM%OU16LCFdoEVad2Bn_zH{B+4-=fgFU~WI<09K5Rt=U_OtYD?
znKM`lSy!;Na7^T~=3c}Z%-+lPlbw@WgYOpKdY)~Z&)AAtJec*E+n6*NTNy$a&6w}9
zmT+`%I&!>b-NmZF>CJ1$f0R#$=Ng9)`$N_g<{HLc##f9x8NV`mF+XFC=iJ2|%e{j0
z8AmGT3oa|(1$>)%d%0(D26Aw6#B%U)bg-4P{9saKUc<u67R`2$^)<^YRv#`t{(S;{
z0>AkK`CNEjaoysy<dop#=cr*_&vbxc@&Eh(4F1{u{rLOkuY14b{~Y{#<L|@YY(GuD
z`F|<+^6+c;H?OaIKQH~n_gU}jv2XI<etpUNdiML{-#`AJW1Pfz>woJXfuC;Q#lE}z
z$oO^k_le&#e*gX*`N#hc_n+|J*}vR>x&GGt`}Ti4Q#q42gV?|3zyJTeWVB)_VqsvJ
z!m@z%9cvn^G>ZU}5<?OLFAFPY1y3~J4!$y;T^!$8&$CppN^#udQs?==BgW^#x1VPv
zR|E$O>jB1@46hl5S#NT*aUJ6*Wc$k+&*sc_gjJE1kyVhbh24jvfMXGRB&!co3qt~<
z9Lp898jiIb&g>$r70fT0JejYs^spb~^5Q+qXCUxYaGTHyK?#0Co>s1hTz7eF`GW-x
z@VD{b7HAVH6c!PV7yQ8=Eg&R#MqrV^Q2{UhFWiirzuCWXeCO)mcHvZI3uXDjB+V$w
zaEjp<<4UHxOoy2!F@I+@=g8xD&9;F>n^}fAfq4;AGlS+|+h3ADcl^-$G3)!;Z`-~a
zeck@G^IQ9O_Mf|dUj3o=Bk||%-=F{f`S<KE>+j{?-M;Sla^<VxkGS6j|K9wM`ycRs
zGh-x+Fl!)-AoF{s$t=yR1uWdmyBW{?|MoZW@43G||EBy~^e_Hj(7)^dtp8v7*YNko
zZ^7Ryf6xA#$&ktPhf(7H!C$k#UH>fjIrYo-Z=FBfewzGz@YCZ@_`lYFmjBBB`Tl>z
z(8J`&?87wkf6bp4KPP>k_O;=Q#utq*DPMHH_I~^Oz59pn_pM(}e0uXq@~htu`QOfe
ziheEp8vEhFo2J)!Z&tpG_%P|C`R89>{JtIi_V4TQ&psav-(Gn=`StqO6JC41Qh%ZN
zEa=I}M+Fbr9_ZZ9zpwn@!h^dHT_1Bksd)V7A<M&ck7S;^zt(!Y_RZy&vd>B%Z+zJE
zp!)&W!#@u)9$bCE^4R^U(lh&Kg3pScramcnJn!+uC-qNGJzn%U?eUgJ%O9FPn00@{
zgUb&uJzW22&C|&*x?f7aG<<RIMgJSW5BEOUy)S+n_~ys!*RSi}<h(oc;q}M*k1Zdk
zd`|gR|6|+F_dmsd@%}FVZU1}r??1oA|K$GB{bTo=<@ed&jeoBEdH5&f&xPOBe^UOM
z|4sOF@OR_iKTN(H5uEPq1uQQZTNwWSH(-!wn#FpWeF58F7Iiikj!I5Xt~M@1?(@7x
z0u2Hu1vG?oh35+07Rcvs<`)utBJ@O<OQc^oRX9q-NhDZsHm@xAU9OqDZv{>XwhQ?3
z&E~e|yvSb0F2`|$vw@qDdkQBb*AA|^oa@-zS<9FwF$FWJGksy2!E%rFFzZp~B8E?Y
z|NlMlZ`0pLzg2&0{hs^V?f0u6lHXo_(f;=8r`taxMlGf!#s~k~{(1eq@F)9k=0A`B
z+yC!oC}37$3uKqz@a7O;Pi19ckz$_ByqPtDU7VejEt>Tt%V!o7HYRp`_LuCAoNC+$
zd2)GVxf<D3*>`YC@%-V=;cDSnz{bxi#!|+zjMay&hiy6AK{ghSMV!1`FF3bxMsi$c
zUBdjGX+E<sYcHD?`*!wt&Ocmtxfywb_{90u`E~ho1bzul7i{LQ=S|?f$sZ$REtJNu
z&uzoe#oohqj%5=|9-AG<1&(JNMO?YOY66@>s=|wf>xE|t%@+J85GwG2uaM_7X9&l6
z_TTIuIM#9saTRj4a2@07<bKF~kt>(OoZ|pbyP%NJ5&msF`COMc1vw|M|6sS}X5fD#
zFjMe`P>;x7Q3tW}q7Oy3h=hw8iQN**6n`QvC%#@pLda8K4*z|A6#*LoKY`-{IYOI+
z)rFIVtc6m9-NpN)!(@$Qw@cSZt{3kTy(Xj~XeYQ)C|zW|XsGyS$(u6ea=&FQW%tQi
z$UTvLDSKJ&zWfyV2HC|@9ujNBmy7F(6^O8iyb$3K50>~Y_Fu$V<b=o^F?k6Ci7Ddx
zVxA%!g+c^Ud7p4I@(A)+aNp(X;Mv8ukKa#Vjex3v9Df#{F<&-cBR{KPpx{&iRe|ID
zXZU3V68HuA%6N77wg|Wg9}zw*d_sgtyi|%)mR)wDbf4rsu|DB5{MUItalPT-WS_)p
z$ugbUhgp{CDZ?g)i44B~Q~#d+?fQ%Br|FLy-<rSXfBp2e{@d1XbH4_Bx%^r0^Q2Eq
zpUOWZzFYN%_s!)uTi$Mdd-=`PS8gxeULJUn`25VXU(br3e|-M!`TOTeFMdC-dd~5D
z_A{+#pP!mM`}VBk#q*aMFCRT$|Kj;e##eV<&Uv}$rPr(8S7NX7U%q+K@#4Vqj_2!M
z%D#Q^{@aJ@_xW#+yxH}7%}do6Tb|E&(faE1>pgEe-#Wf~|8CBEyZ6FxAHOPiHRDy)
ztL?9vUY&hu{%X?e6>r?$_P@<}=lCJ=lgpRzuQp$GzLtGm{+03Dj&JNg_Wt<zE&EIR
z=M|r}eOU4C-W%^XZEr8U`~L3xyM6C3e6aYq;Dhjc=eN3VykBR&N_q9-mF8=S*P*X?
zU)Ddjed_u6=c8qhHJ->mdHDFs<K)NxAI*7e`^4tSk;jIQ%pOSHm$`TU&a2x-w;gZK
zx|4M8_kG8Qm5*GWq(4i3?)psc$>v8f4<qmE+;zNNa4YF{>Ya19BX1erTyZ1ihUJZY
zH_~o8++@93a<l2y{M%l4?%#>LyXsEN?c7`2Zr-@bcKiQr=DY0o^&a*-ih1nu<jd3g
z7pGrMc<uVy{dM7Mnb+~J`d*g4?0a?b^`bWdZ{NI$dh7Mx`Qw9+LZ380AN%t5>)~&g
zzlZ#c`W^Y_{IAPDw132WU-#AH%k|HzzSMrp{Sp5w?~m3$TZY-pXW4i-<Tx@p)^W_?
zxWs;h^)GW0OC75ts~k%^^ADCR4qk2p?*CjHc_jI_2z(OUBwQypS4vfmRX$nnkgSO8
zTA7p52PIaEP7r-4rYg=NeqMaFq=w8**<G^cvKwW2<<w>ErIjQ#B$i3slDH?XBW5eY
zExbm!UZh7@NyttpMz}}BUG#|vzo?Exn>3qrrNjjBN8(q+4~UhDJQKPhoGW@r)K*MQ
zLPk18u0}ycL0n!+-a+0}PEm$Is#?-e`iKm>jFi+niBA&KB*P`6#P^EM6pNF%A`vIy
zDN#J(G5_Ovj~SoDKl%N{?&;DeEsxhcYJGI+(Z@$Yj~X6Mf2jP(_0ji7yPwQ_rt+fZ
zW&117*V|vOdu{bP`BnSN1uq$2KYBCyUEzl}pMHIr{dMnG$8USSP5G|!Gw|1yUkiV)
z{=@!v>mT<&GJh=obp8?jEBf#J|J#g?%qN*;S(dQ)vTkJcWYuDM%ap+MgXs&i2g`Ss
zIQDa#mRx0=*EzyCTsV$!lyfMvE3!>sJ<ejlBEa&IS(ar6%VZV{7Cn}K%uUP|%(t0X
zn93LrGZr$PWGZBO$as{ogi)Ab-@k+Z%>O_9$NbOgZ_FQ^-(5eezF+$q_ch^b)K`vg
z0pCl0bp4e7_3($=x5uA%ee`)B@V4-c?VE--i{H+DfBxhB&$VB*zis~7`K9mk?9cgM
z*uK91vh2&OFAKhye7X5~*O&Zn8^0g<5&moAZ}UGje;)pk`uq4#+8>KQU;f1YTlxP7
z!$YP+EXP^Z*{ax1v3+4%#}>)Tz%0h-%~0{*;~)ROJ^!BkoAGb`KbQZ03>}PYOx=uk
z7@Qb#|JVLo_xJ9f-@mhd`~PnHefRgf-&cNj|4#Z9`{Tqn^>1swPWa;W>B>9q*E624
zeKP0Ko`+K&<~}rj$n&t_LBM_ad(L+)?%cTb;O4uVJhw01S#*!#0sq6qN4<|{JeGJo
z<MEs)XP)#waeBJ<nZOI1mwvB~zxw>D{Pq9WeQ)gFZT+D7$@bIbj}jk+J~+N_eV6*S
z;mwXWC*S(KFa0R=#qeA1cZ(m#e?0hc_=ojR_g{Ly@BiNNr{pi+Kezu!7^kpQuq|Rg
z&C$$R!<o*R!pYBhlj9A?5sq^lpE*Og_j7CT^zh2?zu{jiuvT!HP=WA%VKw1Ap%$U*
zLUKa30)>3DdChsZ^4#J{=Q9!5F2F4y#eal%1$P5y1cxg}D@Q5Eb&h3Rn!J1Yrt;qs
zcqhmtq%7nuJWE7T)LAq|G*(nf^rFZIkv>sRv9n?x;)leIB>E&)O7u$1k}#Es7yBc;
zQ7BSKQb<B5L1>3il<;Zc+d`&-Edt&Gg8WN(b$P$@2=naYvf#YPF2nwkZ6BK|dp5^L
zP6Mt@oQWL!*;H8<F{d(RGIlcj{4dThg+ZM0FXJ~RZ{}&t3t7@xRalu=C$ZjQoy6+M
zs=(UEdXlY@!-DHNw<zx=-blV{d>Z^N{7U>E`7ZL^<n!UbCLkfaR-{36uh<Q7Q}G7T
z<svggqD7cRZVR6g{wXXjQYpM#Xsb}6u)c_fXqOnPM3H2TRFssp<ZJN)u@xfn!r4MP
zLgGSegsMeuh@F;TkUA;#Mao3#sDy@iuc)%fL!olP$NUTUR`AZ_nZSLAtDH-VYYpc?
zjt2G|wmq!utk;;Gn1mSDF)U<AXE0&VWzc3YU~p$JV9;a`VC-RXX8y!v$E3m-```C(
z?C*;|1%KZCvGK>YA3;Bl|MdK+{bS3wvM-Gv_rF{I=GH6jmouJ!d-mm-;Pc<lRz8z|
z*70=LlSPjiAGbb^ds6bm;z`w$n@>JI(S7>#>HTM~pMQLD<7MNkx364YcfGEBt^Hc*
zb=d3oua~?P|8VD{;Ag3?W#4c9IPp{LciEqde=Yy>GxRfTV!XignMskkgT;<5id}|d
zF$WK48|QjXF-~C)HukTqcC5cx?z22$31QV`%Va;!F@^Ix=OQj0?u%SDTyb1yxUO==
zbJcUs;;`nB;P}tpz;TpQkoz0A6VC!}Z>~oiMeMg(U0IegxiFq*c*@YjaQDCG|DJ!7
z{x16y{Cn|F&Yu}S@BFy?ee&0~FK(Z&esukC;+_9H(f7aKvwdXzZ2L9tTkbd6Z|Ps3
ze%|mg;(f&1<*y4~6}+18`tX~dZx_A4`(eY6hhIg%%71(Nz4}M`_mZ!xKfnLH;+xyg
zrk|bPo4>65$o=8=`*R<7KCk|2_HD=Kt`B?PU48rg^_AxhkLKO4e=zTn#FN@5b&qd7
zn)gKfdGU*ump@+ad?ET={MnMHy3g;t>VEt6&DvL~FKb@?ee?NU-TOoDuD_{%W%Tmn
z%Z+cAe(?Nq|Eu-apPwT>+kQUxDdNNB*TpX_-dKD*^kw_E8Q(vAm;br+m(%aQ-*11{
z{to`-{BzmQu)n^{4ea09-Pv}s%wut3yT-x9y_h?hD}h6geG+>m=K>x+fnGr!fgU~y
zenkOR{x#f-*`rv?+2(O3@_gd;<9o*YhF68}0&fYg0`FsPQLavo#q0|>c5#REUE+=6
za%Q{0JcGHC<qJy*i#GFg#=rj!|6lvB&6vcv<^RP$rN6R&PXBTG+o8|r-d}#R`HjoF
z!uQM{Uc4`OFZ3blWAsOz4?o{&ytR9E<C)TvhR6C(ouB7D|NJEG;f(uR9&kUt{v`eB
z>n9;kRy^78^z^eo&*fhRz7%|s`E1(Lc~5sfO?Wcl!IC@YZ_C~Ne3$Fq(Yw`mweC&5
z-|`^q;igB|A8&rr{$#<U=?}j=ZhXG)MaXl5rz}qwyqNp0<MZUNx4w#g+x;!<`^)cs
zKbwC)_!Ij_;CI@ugTE&HR{fp+qv@;Q*QMWn|8V%t@+asI_y4I(Nz6{nQmnP?JseCN
zw^_qj9NCIE=W#Z(=`oivg|KerbmvLtnaw?qtBvy<M-#_WjuTuByfysC1l)zr3C|R1
z6Q0WdjmwtPlB<|IiARgKhHrx4E)h{tR}p#9EHPcNG?C|mF9fWFMa7ng<_TTlyTX&i
zCn8iLoGjEQv_P0$SV!o;&@7RBvH22rC2vZqN^O)nDfvlkyHFRODbF14liV+P_VX18
zm<S~aM+&tG1PHVVNbzg(ICD&9eZVHgWyPh$ZpCtz=`T|uOA@OdYZ2=|)>c*<*4=CY
zoT5CN`1}N(2_y&x3R()(^2zY9aE7q$VY$rwg4u@kEZaSHQ4U4+Ic%96C%Kk#`En?-
zO=kVTrp~F#CBVtRzMr*_&6z`xGlIjLJ(8`RwVkz-?H&6X&V}6fcy92x@uYE2;rz*V
zpCyfXEu#m+R)%z@x6C(Kq*+s0N}2C6?`3Ue7vZR9U&3~YZ4Spxt`hDZ&R8}lmg6iB
z+4MPHvNN#rvoWwru$Ztgvs$p~vR-C;$I;H=&$fpph*_Mm<KMyGjX$saWdCFHSNd<#
zpX^`l-+zBy^)2Ve^`C!!a{OHIW5>@^zi$7${_XE4*$?;M3w?V0<?Xl2-?e^x{IT+<
z{?Ei8Pkuc9rT%XfqXP3WroT*UnS+^MGCuy#^-t?xCW8X=I@W&no$Q+I>)GFNyyE=J
z)z3YH>pW*0w*j9#zZIV^&s?rW+*kP~3)~Z!CzK=VEN(8*FEK}Ar)03qUikp!`>Gvk
z3~B<ZN0e_X-j%y59W3o7YazcwVVe?*$^_*PO7oQ0svK8oQZ`onDSukAR%M$?hVnxt
zBjr0P?dmBSRq998-)bz@C{%Y<yR0%#IZN3=B~DdM)lAt(@w|MjY?xH8#1wHp@fG6k
zQa!RB@&WSq<^IUEDby)BE7i;Il?jwyBXvrWOM*>oi3qPqx(JKtC*ci(_XU0k+~Hrs
zYst;c>BwQm$;xfV8^Pbn|BJ_!Q<cq#dHVn4KX-p={jUA1%`n=#8|~eV_U?vW?@scz
z^fK9A*{3o)WH!mf$*RjemuZ(`m)a+lBjqd^F0ohQuhaq=ZRxKPfsz}gzDjC~=ZS0;
zW)S@*dRgSG&}G5ng1v%r0-Jcld3<>}_>Ba@1pf+Eir$h)l~$DgEFmRcF1AOkTdY|`
zRj5QzL$H&7FYgDQFFet_>AcT*uJZKpF5v6n59NQwdyuzL;E?cakuqT$AzPtr;Wptv
z0@Zx)`6>l&@K^In^7!y_2`m(xA+Uh&5pO<UJpTgzy?j-?^LYMpZ{@zsy_tJ1cL9$#
zF9UBZ&qJQ8eD4G#1oQYG@XZr27uqd&hM${nCT}_4JHA7_fjntE`Mei+ckw*pa_4%*
zJ&Bir_bHb;=U<LAE_Ln#&I#<E9KKv}+$KCWe5!&4!dxQB!lJ@Ug;_-mggpdP_|Nez
z=T{e;EyODPUhuX+te}EWfS`~-o4_|gTcHI4?|Bt@f_bL!%;uWUF3$FV?F)w(7Z2wY
z_Eh$@?5*sp*kd^CIRm)<aqZxW=X%X`gu8~@my?ApmgPCC563!=b?kQR66|GcGg<et
z)UhmOm1K`*k6};Yc*7aQ?ajl?^NN#=eH-gjHcpONY$_~a%-k&8tWQ{z*w(RWvp-^c
zz;c#p7gId*b4JF0>wmuc7W!@0x3^zgz7&3r`*i<9)caX)*T4Sos{Qqu*FLXhUn{*y
zeHZ=V@%zxXe6Rn!w0-&Q`LCxF9;ZIEcp&s3_QC51We-~(PIzeh=-{Ixk9;1hKB;?b
z_^{`G)dQo)d{1vZIr+HsvGkL@PwJnvJX!VZ{7bdhGOtzMc)#QO@a4nC56j+dcwPKb
z<;B%!!A}=GmU|TcNa$(Yi+iswzcGGy;hp4r@AuQ*JAVlHu<l*U8_U-MueIMiemmve
z&$mzCOnIICO6^tHE2URfuhd@ef8GCD{EhHCmk-7tGd?iCXLw)xA@F0-2iEsj-adE}
z|K`#g|98vYtG++|*8QEq2i=b{9~(dHedqN?<5mBQfEQA)wBFdfnfL1Ri>J?9o_~36
z^iuS-)LX-MHSZ$cU-<CiW8<gxPZK|8eBk`>?ET#Ld*AiEiGDTbRmL0fw-4S-db92|
z>+AKe*S`^etMPWqo2b{DU-iCjef#Tu`=@{}MqlhdnS9vwKKz6Dhc)l!zx8;_{`T)1
zueS>Cct7m;sP^&k`{^IJKfV3v^Wn$am#^zzC%(D)X8)U-H~Me5-`;r7{+aFTtgpAf
zRDO>7wEm;yr}babf5iNH^z+k?gFlphO8(;hZTCCi_p0ANe$V;C^Kbiq6Gk<r&y4#S
za{nLv_xT^o|LDKRf7$&q{-g4*_TTisY=0;J(`5{0n!=RHyp>st$&umr|AmaLEP@<-
z+`2rIxL<Ou=HAI$$iJWeAfGF5GS567Vcvth@&duaGeyn_s|imRP7s+ZGC`O@$UtbG
zNV@npiA>1{l3vmUvUT#c3X$?hWa^}ZBvyzd3V8`H6P+g(Cwf@ujzFhioA4{)7ee+z
z#e#?UgZY^Giuq#r-||=TH}EavHQ_zU`-1N>-vnL*-X(mu1#Sp>2r%)z;FaTd<qzd!
z<DJT_&1J$dft8mvnxlnBn(r^qe6FP&vpMc@F5(j5Ue0}(TZ+eu_XqEPK5iia(cdDb
z!fHaSLJNdG3YiF=;G4nskiS%*UErmFkWiP1iuh)Ui;}TYS(4ktPlzfAy9uTWsfb33
z-4V$ZEa1!KTg$J?pTM_|zfiDW@QXm9z&C+p;RI0=ku<?N{`q`+_*(f(_?h^{1-1x^
z2ptf#7yKx&U+|OA51~Y%e}cAx`}uu&PjIHOCvzO(V&}Hy{Kv+@s>tHZBFS=(c@@hQ
z*4=C$*q3nr;FjgzBgiTwBT&d&#T~_cfm?*@JKIFoeAZBwsZ2{4uQL8&@?^Qo@`FX1
z^&QI^W(UU7f9-#C|J?p#`FG}@)ZdqXJ^20l?}h(Sj1f%bOo~jsjE5P%G3YVBXT8U^
zlyxa{3DZ+1eU=WE<IHi)?aZCb_RRLoznQ&Rx3l?i^mA%+*>N-RCh*SZw&uFZRnK#T
z_bKl(9y{*8T>rQtxNdT8;FRLz=Qz&xiPesMCD%h<NB(VmmwC*%=W$=;G2)rTIgL%7
zrH3h$QJ<ldVLsC*7ALm-Y_)9JY%kbk*rJ#d86Po*Fc+|}uqLyVFn?x##PWs3mh}s(
z4BH>p#jIOcj9BY9#Cbw_n|ai^6}VUO76_yW9OqNu+rc}T=Nfl5k0c+HK%wAop|v6o
zqW+>SVus=#qB6qy!tA1QVt!(`L=uE93H0$h@w#$rbFJc-!B)zW%M{C0z*@lZnq7`{
z7Lx?yV}{L)QcQJ>H4MrBkNso*zv%xKh6<)j%)gndm|K{BF=+fh|HtKL%h%}7Up_7S
z{P0WG*Bf61zW9Ew`c(AE>~qWKKc5sn&HUK=sq5>}AHRR{{tWu=_)X}0{?EMMp?_}v
zIr-=6Z@J&Ae{cWu>W|-_JHMCz4*D(qTln{_Uyi>VfAasR{~qyU)=!<E>%Z;)y!#`|
zN5_wSAALV2e|Y}x(c8!G&VQWt#q8^p&w3yG-h001{Gk4E|3|%#hu=SZSM@&PqvmJs
zFFU_*ebf7%`F-B^+uz@PYyDFEappVrx6W@(-i5vEcq{+r&@0~8&)<A~8}-)t4c}|Y
zSF2xLesllb-1qz5O@4FiRr2eIcRe2^KTrCk@loT$s`p#pANgSM>H8;%&kUbGev1D5
z<g@PQe;?f5O?`dsW%;X`H-FxEzkd0`>bcYNTQACAO?<ujjquwwZzjFn@!{#`gI|O{
zAAi68P5J9nuhrkoeck&q=h^GWjE_qmKYX(Dnf?pemsKzQUN*k$e6{&i{EH(`I36iK
z<a#veG1rqjkF}occoOsU^|QM#w!Tby`S8W#7X_~v-|hVv@j3O=-}eXJ=Dn4BFZc2M
zNA{2N-bcP`e4F^T^IgJ+?vD>X27X%hdE3{8--Uiuf8+m>@+th2)R!gSSbi+{Uj41(
zYx&pcZ}#8gzgvFK`EK*$-Y-9fRJJXgyEv^lX0a`1eam{8?I@cFYa!Fq|2zM2F(@)0
zVtvh~!rsX)#W|aM4sRCUBtCw=jXZKZ!h9)$4Z;<|D+IOp>v*|&b9hR*Ca|Al`N?#L
zX$Ol1`xFj;4gq#^)=Xv*=B+H6Y`m;{7_<K|{QdTqiy@V<p79&QMFwT2MJyB8tk}!h
zwb>W3>ag5oF<>{~G~ryp7RqeN)XjXD#hK+G(_cn)=BuphIPAEW@$Tiz<CWvq<0|Kl
z;kzZUPT)50X|Bhd9o%+&jshP9{s~kGo)xkeStN2wI8$hbfDNA%uN=R<@FX!`@lB$~
zh0}zEMSciZ2o(w3;j`r1!zVAWMsTK(sF1k8Hr{HU{k)}u%S22?I0VD^7V>)Y$@1&-
z$Mbpc&fr<cqrvx8AXK<TxKXf~uYl)1PZPhQ&|RS?g7*YE1Z)M0`4#yV@GRms;+Ez<
z&%KNH9>2FhB_A8l9WD!=!~7D$ZX%zAX9%AYUM+G)bgkHBF+nj;(KDh2;^!ocCF>+q
z#1w@N@`drK3djlt2wxQr7ZDS!6*U%h6uBf6Cit7*n*TjNzfhRSRM7>Z+@dc<ti@JJ
z_({o2wMg)b>5I%4IU=?|!cTIe#42%1i4{_xWnRh(%1xDhFa1RFszjk=kMvsE@3P-z
za%HB=u*$~DvdVhO$jU61sg!M#n;^egzF%(dQpug-^MoS>*73g*)DU|r$u0dra;wBm
zNd{RSg&4&Eg$P*_DJw}4nP^3M)j-vyN|WTB<$V<em7Xd{%72q-kXDqDlaE$eqv@)B
zT4S$juF4FRdeyb6-pXcjWfIJyQ$^Dygr&|&?v&^ecNCu`u|O(E`kv$iF(nZf;Zvev
zQjcXWNjFK<iyRg-7E}^+7ii#H$F0FB#`&MyMu1PaT_~8}jr#)oVz%$>yEu&4<}fb$
zd-V6AKl}cdG1)PnX596^=>LC)$xM%!G#UT@b^E>jSIys}Oc&Th*acYD{qO!`^~dgS
z$KUclF25Xp<o;;*?aR=~@{v`AC74l?ftOK%`3{R9>q(|l|0e!f^-qwwhiyH(0b2@-
z7wZ!C`y36N^Ed<9_c3w(_xk^vsh@2!yF8mb^J#`^Mq`$btgBgDS)Mb$VtLE{m)nEC
zM8KJU4UY)dd5(#kq1>E2GdSH@Dj41}O0X4kd|~fr?O|48(PxWgpT{oAzKr!e^La)_
zMr)RNTo3uo`HpePa>#HfaOrUSaPqRru}HJHu-3BkbN%4jz&Ve75o-#|YZh0wO{_Z@
zt9~#0*81(r&)t9a|9SXx@Ap?f@BUr(e;R`oQ#e!af2-dHKZ<_t{<oiL5+nQHlArZI
zegByMTm47<XWqBuuN%L9`nLCn&9AM$F8;jpo$I^d&!RuAe=UEDerx%7_T&7oQ9oAv
zaQGha)#&T-Zx_D@{owd%@Z-<t+ixGdOnm+OL-*G`UpIWzd-L{X;hVkhUcYgF74$0k
z^_|z%Z*4xRey#eh@gwK^v2Rg7?*0<|{pfq&C$D$6-dTLM`Cj#N#xJYii~d~wcYt9J
z(>9iB7EMN-Kkh&Hf9Cx=$i&7Z{x9#B%+IP{L4S(>O#SWhyYBag-@E@*|9i-g#B_sk
z-ha2hj(<x2==`bp8TIAfhptcg-!6RL{jKCn$;ZC;GViP2CA{19{@8n!Hx19CAH94e
z^1|b7!Mn)UzR!dn*FMgBa`92o1M~Yk?+M+{een5V-($6>dQUGr`g(uuJ@I?JcV^tw
zzxwxr^o5p7Hdp<w$z6SQDehA4Wtl6MR~xUl-B^9C=92HZQ|H32&bpm_r|RZ~tGZXN
zT$^<3?yUnir{7w9TlMzeTO01U+|PY@<k6Q$e;+D7eEv}C3I9{|$GH!0Jhp$S_%7w+
z(N8}<E%@^P``ur*{}Pzn*|Rv-vR!3f$-ITbhVLEU3~n!WQPy_$N!-pn8r+|`7xD)4
zHSs0#w+Zn{xXEbB*hyJRG05DP&63NP*(@n8<t2MhzF2Xu(p{xPN~S6mYMV7AwCCx3
z)ppQuRdSQ(SDvlWrD>)%Pw|Far^0^aY~?h?bj3!+Jb6*s1{rs`=Zdu|&dOQ}Eb?Xw
zor+?Lmt?<7*oxf}3y^pxu}9*ac!v0Iv7MqVqB;`!QpY4TM5_2b`8tI%CH6|`Nxl@V
z5q1)(5EBqvCd@AMOhB8zfR~v!h~H0Ss>ET58KP^2+=NPn#e^OCow#^7_Hta~RN$85
zy~8`3Cz(5#+l}WV?{2;wJRjLpnQr_~V^U;i;8Nx?;YeYd#U8}9nrkxWLe3mc8FpWm
z$;^*gH*>P`IB|D!NV0XZrm(GNJHsN)`1kLszaRg<W-@2ZVcW=7!WPJ8$!^HGgG-rf
z6WbZaPyY`vOL6LOM{^Z&1heg9J;>I{evO@h!;^h2>o?|N=F_bHTxq<odDe4oVNGV<
z#T>$t#Wee$@Sh!jHvE@m4q*GhVaFxT^_H`cJAtoQ;3U5X&t`TD)+KC)Ttd9wyzE@k
zY+bBI>=W2Gu`_Yna*DD`vnVqiV`5>e;t*zE#uWN5<WK6KnSVNe8~t4MZPnMW-|~M}
z{8s++<=5FCk>BIKC;XWI^WV=?-*Z3z{gC_l)_2)oy+8kdd-(a}C+;tWUs=8%`nmh}
zhhMEfgunCsIQ&cI@BF`?eqZ^q=-bn8GrxcSD)DL2yIF6aytDf_|Fh%Q-CrksUH?`5
zyTz|bf2aQA`fKuA>vzfDhYZV^7qCPy_b{CNx9I;(Mn;witY28US!Xf#F~u-1WYgga
z=GNxg$f3n?h~p;bB~EkpOy(X&S0)aY7`7sg5)N(l{p`=VZt%Skye%|Wa5jG!-&Vf2
z0@=bhg$o6z^Iqe!;9AZV#I47(ntK<=N!Dqs)$ErzD|xo?mGgb#KEvt2Y07EGahk>K
zzu9l8-zWYY_`B@y;y?F)cmBEl_vgRI|E2!F`0X@QW?~p1fPZu*W^^WoX>=y$^5{$q
zQ8O`G47~rY{ptTB_-Eno1HW$nVEz90tMgZ}FWWzUe0SjOledj;)8DAP4tiDlO8E7~
zSDLT*ULAf}^5W0)S+8!q)A-o&f#u!$SM#55d$RXI#NGD03m%9)aeG$yO#SKm$BB<$
zKX!PU_4MJBwrBk>@4mYJD)cq~n}jzyZ}?u%eW~*D+{^2)!rnZ1m-6w+NB<8m-e$g8
z{FdwE^v@b!)V}=u{OFV3hr6%&UU)y(d8zzn(Yx>OMBaUP$Mx~mNBK|spU-|#{Py*m
z$oCiD{(dw0zTlh7SDr5~K1qJO@V@Os^{1XM%HM>){rt@Q@zp!$_j^7b`t1H?(w9wN
z)_%V9LHW(M=M2wJKM#M-`z+z<&nLZ48lKF4a{I~Qr;nbkdFJ=D`AO1~n#W;}5+6N$
zRPt!o1M7R1cgk*`y&ZY?+r9sHuiVPL5p&(``m-CAx6|$(yk~m9{;v0}z1MGE<GWsR
zz5BZL^*PrbT`j!2^(xPG`I}z1mG0)><GrtPf8D*-yYhF~Zl~Tld28bBh}(6yb?-4e
z>V4Aj^y`z%$6=42J>-3q_oU?I%y)A>_I<kfvG)DpH}$U%zMlE!-dnx*OW#Mm4|xCO
z{p$}WKkok&^hM>{#_#<<*Z=PLQ~M|9Z|6Ucf9w8g|9ky^29qGGJzFN*d)8Ezm5ks1
zrTlxqAjf)}V*=M!&UCf{#*Kew{FMA*`@`zj&OhP*w*G&~SjFPP7Q^<1<pc8$7G}1s
zY-`v(IgfKq;eO7Q#Ic|C9m@{Zvux|wKX9zzn#{d~%Zj6k^)Yi4%VV}JoNKr?amsN#
zXFbj2$ME5Q2%{tOeHM1Mh3w0@WcafAx%oJFrgF(}*0S5P9bwI4+s@9yA<lM?>C^wl
ze?tG%|4sQ<_<t9}FGdTd-wbvC@Bg=E`p1&P?#6kJ^CZUvHYS#%Ooy4fSTosfvwdNo
z#JQ5&jBg!(34a*>JN{Svy!;Y;ynN^Q`GwMjZVPzv@8)yizbsHG^hH=hY=^`;$*&Si
zBq}7&NfpS<m)ow8u6RS<OXiTcgNV8CJK;+r4x-tjMWRPVO~nl*`y?kwa7vz(S})Bk
z^FwBq+&KkiB@Klj87;|l@!MiA#gnAmWup}SE43*vRScBhC%Z@HxXfp{My3BMw^T%x
zrpks(vWN$Zxr%X#ofPdAbCTF6xkgGzdZ(12RE6X@i6DtD;w%z3CB8@$iY*ZQ$j#3F
zf#oFYJoe?Bb9fy1{rTB>jk$u^>sVZvx3j!pzs#k|^PB4pyEW?+=59va|L6Yh{Cno#
zMTP?AI<_wy8##qJKeIEjm$AKIRcEW?;NpJ7<;t<1MS@ZBf9wBS3^Gi8%qpyrY%AF&
zuvRf2Vk~47W1hh#&-H~ngy#u&AJ-a=H*9IFr<tcP?PmJG{Dk!>$A0c8-b%iAd`EZ>
zaX;XE&M|}YDAy{kuN+TV*_iE_mDnsec5vu&Uf`7HlH#)Ce9!)gJ&$t^_X$3Eq5Z-Q
z!bw6;1seI6^IhOg;n~ZzmQ#n*j`KIC7ndcM2)7E)G#+2xPrTRoWQ8Qfc8J-FtQFiR
z;4Bm(nl1iTyi|fuGD;#-%uXayC|K~kprlB(_$SFQsb3OnMb8MV<Z0(>;@ritlw&pL
zb*>ZK1w66bjGTHLnjB}@vRD&Y8Q2eT{o?KA5n#W}@ap%xpI3iY{>u5)_3Pd*sb4%l
z*uT1eWPca<=G5z1Z?3)L{}}vH{=LMTtuNO<S9n(Zbl20O=OV9uyykgZ{`UEs{jWP;
zetg>YX#IoT4{e`qc%}Wi>P69$><2&YwLOr2bnemLClg-Wd&Tg^`)%vnhp$CmJ%2gv
zwZJ>;58)qWKQ@1m`*7hM*W2nh_us61yY5}W`-Bf_pSFKa{*w4P??cgBoj2}p_k8&I
zx$n!PPf{Ob-~E2K`{S+8m%g}to%%K7o6e7ppDaJMzJLCb^?AzY@Gsh5@;?WEn)2cE
zyPa<zyjk>S(Hq^jy>ElxY<~Uo?ZOW}AJabUf5-7w`pw=qli#_$Z+*MtrOK1Y`;7M)
z9@Rgwefs=~#S@Jucb@!u^7Qe8M=_5~9@#&B{^b1An@?+=tbDlWUhVA%H@@Hac5BHU
z@4K>h<L>Ug$NgabgN6ss@4vo(@xk&(LeGp|ioD$Q^#6nGyEE?AKCFB8?PcEU#c$f*
z{d@o8{m*y2?-JhLe0%4^&M$Mm=lt6BTkmJj7uOGIZ-w7Bzq9-x@=5#4yKh0i#Qz@n
zC-7hE->bhF|EDmuv*+<t@D=e$u(vXX{SRYU%=nd2l}U|Rob?vF8`pU*J}z6%WgPvS
zmw9Rhjm1_-REb{^-o^KYlZ%6ygPo&~qm%Ow*FD}}0=ESt1vL1Ic&>5PaZTY>6mk%G
zBBaf4!@Zny6_+%R4o@ZbJZ@!PPJunb=3@86`o$KDN{A|mEtJ?T-7kAnrccUN{D`ob
z&{M(h0&Dp5_-y!61#*RbM3ux?#jHfxMg7DkNSu&rk|~stlRPGJK`2<bQN%@Ljo<>l
zT3&TNM*f?8PJAi6A>7HFk(^0fe%$WdW}MBe?u@Gcmi~?Ux8>j0e<J_7|9Jha{%^_f
z=Kq`jQ4ALuFSD?7sBsl?_i(Lez4c$}Z{}au|K}NW7<T?U^{<-YHj@TR6!TTaD8?41
ztt{p2e%x_<`*;(%CbRLd++?X`@8uBV;9}pvD#N;!<t~!~!`{Elf0qAV_P3m|mUS`5
zDUM|<e*Yi*zW>YouP$RBQ!5i2s}pAck0kGYZd1<7Y@b=@vpwVZ$y+IWS>&ytB5x>1
zKARM~0=pF(7n>8iKbI}9Ca)g1BNrE!HD@<xI?p-*LD3gt^1@Sj!Z^0FC9t`&NwZe7
zw6SgHoXQ=@&Cbcms>NXbr|Fl-Z>fJ@8MB!48C(BIefR(R@hk6_`VSWG*L~3WQuibE
z*UFz;e=hp9_ov{G4L`bm$NtY^oc=H4=euvl-=%*1`o81qs?X|QKmIWMWBt$Lzrz2}
zzgvFo`(gVt>~A{LA!ZxKTYu|*cm0<CoBKES&*$F_f3E(kV_M0whvflF0_%B}Ma+Mg
zHZ%9LP2vb=z4BlEpB_Uka}(QYwx29ZS#>!CIBhtXInp`U*}pKGFzsX(=UC1&gIAAB
zk=2pWh(VO`1>;^OZkAluGWPYHeq6sf`?!p_`?>COYIEsxf8m+QyPuPb`Rd=q-#dTL
z|2yfg)*s`)vJBQt#?1Ga_pmgvE@tIp`^FK;?<*!PIa^Fsa6b2L4t0(I_Q`DA91pqp
zdB5-;<vq)z%l(>T4x0q)1r|@X`<$gbMqKAvF8*ivbLQ9S->-k&`l0aa``=FtGnk$*
z1v3`@`|>OI`{%D8enkJ{V?6lp>5rICzuq2v-TAWSxx(`)FCt&NygT#0;=_@T(?2DC
zoc2ELJ=3TA-<SU7{P*SOysu`TEI*uh-~8^<n=fzYea!sY_G9U<`rpaFkN*7r&GH+|
zuX={htd<-r*zB3lF=jKdGVNyA!obaB!SaA@0|z75UY;5J>HPb6R&u#>-{Lb6QW0$C
zdCq=~<ukJZ%QmJi26@I_W-(R`wukKJIZtvu;#|*un{6-WKfbj>+lBt~Z{*p@<;=B~
zGmFE5-J4B|bwBf5rk_k~tY6u**cI6lSQoM#;OgYF<K54m&t&pn=%4DphQINDR{ZJy
z%l0?__s(C;zn}i}`BwWW>!Zk*r#~3}O#HR>YuE?XH;Z0=dNJ?iidSN<xnCW8&h+fv
z(<{$dpC>&%_~_U}w<q>5#or!$)ApkFk>cHBx4dp|yEEq=!^35dxF0WjB>QOoL!$@G
zcN1><-Z*=E@uQ9x-(E&PfAi?~U6othH=}OtyQ6pi;r&hbs_&&f;C?jgVdDMzJC|<F
zz4PPYndjZFd)_3ys(9A$MDeNnv&YXTyyAI#>D}oMI-kFOY5mglk@?+(Hz(dQeH8dw
z{Jr$sn@>yM?Ru^M#_4_Ur<$+QKU{x(|FehTDf0rBH%vGGyZx>I%f=Yb8qDF(xsaWi
zC6Dm{!#{@ajBA*TnD#R<upQ%k%WcaumFqA2QML{2T3pw7&k7t8sOAghxydcSXDBpJ
z^oW?OShDC5F+Zsm+4FKU<m6=UN?nw6kv=QyEN><EN-|Ppp`ejaps=CH0+AkJQ6V=W
zJ`n>kY4LTU8p85IUxWfgJVgJBL<)KHzGS<>7{=Jla+OV-?H#iuQx~Hy(-o!{Onyw;
z8BYIO@h^cfmz9GfnB9PR;a|C5OutP3DE-%F%w!a1T*k=FT*)$nWgpW522F<LjB8kG
zINos`<aB1;^S}Lf)Gy&bH~+n7FlMx5SjSMutjPA6Et@TX^#Thg+iQ-uJm&;T1UK?r
zWjXt=?a#tLxBo`|?_tnkn#m%?&cJEJxtwh!)8YR&{>w36X0PSm&m+#ggw2BS(chH6
zr~l6Rd-2cOKd1gbV13NR#cRpy#r>6ChRu=vAy*&&CZV^2FL{@7I<QBu3v$lpyu*Hy
zZ34$9ZchHM{O@?5bDiL*W8cbN&6UEJCm_v#pKA{5ZAMFm1^+JmPWU1D_5Wv<Z&!b$
z{Sy4;^nKRn!yi|DRQVkCz30!he;t1}{oe8W$=`Dfk&JKurTks<kD0NPc^>P2)@3Zu
zn1h-BG8wacVDsSI$r;D7f$bw(H|I?5S6oaSvsk*Bcd(jqWO7d7vgV$|b(T|$E1RcX
zKukng#8hxSUq9b=0SS?0u|J}7L^MTCiinEM6Z0136G;>C5^WJvlw2(HM$T9+LdsRN
zM#xmyPb5xQSMV<1N#4nP_WW!3e)Ed*KH^@&oy=<^=qmO_a+~BNQD=b*JQsNy_%{jm
z2tE_w5cCt=CwN6LMc@*z822BJzZ`S8Gx^ztz6hrB@8=HVe9F0(=L7$BK|7%y;dNp^
zB#I>CBpW6Ei>Zj2i2V{jB$KGbqVh&wQZh!kR6s`{SKuqZvOt}ny|AR{Inh(Xo&3>U
z_gJShuVTK=x|H(-&nBMBoS)d{u=21vv)C|w`=|2v&EIMNp8hTPllAlCmzEEU->i8h
z^d{y#<Hx`61YhYs{r&jLv$B_!FVkLrf3^B8!$<bd?Vs#E?s<RuP4P?N7x^z`-YkDR
z@6EB7o1Pzi{_6$HtIIDPUUWVG|3d1u)m!EFW$zcf3w!(HRmRH;uNQuh`JD4H;jQAU
z9j|!a$9&TL<nb~3<GzpAKAC>`{AJ?T1z$gXuKD=){j3ijpI3bi{j%)6>6^`OQs2M*
zc;(a6k83{ieJ=WJ`B~!Yv+vh`O#Hs~>$0zpfB630_0RNo?-#FkeXmvCCVaT~Y0{UH
zuQgw{e0BMH;Iqu<U!P1r?tJ(6?UnaYpZ|WX`NIAF;mfw?=bl+VKlwuJmG0|=H!^R|
zz6yA?<7MP?jmN9*O}JBY&-+o_<G}keH&QREo;!1H!iDP>Rj-I#_qd^XWA=?VH~DUF
zyd{5A`bPJSOSd-P`~Gmp!_d2)*Q+i$Tv&GD!Ns&oM=o!;_Vk9rt-H5h-D$b~^&H!k
zU6;hK$=z9V|Ha*tH<~WLIj?oT=)$jyYcB7<dg*%ft)q9l9?p9*{b|&b&__HELmqv4
z*7`>Fy~SJimxa&vJiYVm_=^`W)n3`Y+Wu<Yo1LIz<31$5YknR0>hPN%A2)wv|1srj
z&&P)M%pdzd@B9+*b>COBudQDWe+mBP^7F*+Rlh5L{{No#Gym^x#$U|#OvnD-{rTa?
z#b2NQDF3bdGvklk-~NByj9*!vvrppe;QY(EfY(DfK)h7kPee*!1Meli7lO-0ipAE7
zsf+dqmkFH_)Db!_<R)}apq+m;|23gj@oy5hL<IS@cvkYn@J|*>7GV-q5fv7BBQ#U^
zfk=@EvrsI*DSx9-xwxS8Whqs$BYZ14Ua<LdgmQKAg!8fRv-3aT<K|Zp*e4LkFV0)U
z^N&wgbc@V81tWQOsRmJR;jhAb#grt&rR=5eN$-(KmOCwfME;%Zbg5K{l~S)2^whf4
z-YcD!eIVg2HeKwRM6%39d11xH3Ud^eDTXTDQAm}wl{_rEO(a%)kqob*lVXPKX$c3h
zNU><~L~&+uAMsxCAL5H8r$}o`CyV<C1@gxVG>Kf7kdRiA_K=hp+aX*cTqd$jG)0V0
zY_I5MF;2;T={^}IX)Ez0VL72a!hT|W67$68i98Zm&&SQrAy_BGBvdTGAaGf5f{2dT
zN6~e{tO6%^thimdxp|toH?xN@Zu%|y^U{x^pA&y^{O<j!_dV~M%a6Ce82{M*-t|-M
zNA>q9Kfe76`FrKB`ft1M7rsjU*!+9_-^jnme&75x`}dDO+y37AYy9`-@4DYF{xtlb
z!?cEF3Ulm#!(Y3;-TbEZEC278zyJR{`?Kipl7E5>k&M|)mCOg2Z!kY#jplUY-oUBK
z`s)AOKe>N0{;gv;#W;^~591tW4mNJ~Q*1_TZftTKUfewVGlX@;i^T2=v~tR`v@yA{
zoM*G;n9eD|)6b_Q5G+_N_>A9+_bQhuS20hRz%$_#QAOeXyi%MG*~2;Bus>ugV)J8P
z%&EfT$*aqo&Lhkn#C4OqS>Ugjxy*anhtj^{*+Lck1^fwu_k?~3ISW^d#z^+bD9hfJ
zJ}4<J_DnETKvU?U$R$xNktTt5o(-Js?5|n=GS>Zf`FH4_H^U*u)l4!>NB+zG-S_*=
z@4~+m{?{>t{mcDX_+{=#u8*!C6FvyPyZw6Ys|l}cU+;R&{+i|Gs%Iik=RFaA+WyS(
zMdFLtXI~#*d-U>=;bZGZ6YsCTyYQaD1BVBb9@stF|2Xc6*i-T6D_$kP`}{ujozQEC
z7fYUZy=;2(@2%5Y?l&J^|9_MIuIRnOhm9W$KfeC>;M1MY_Fr4RC4UwExaW=EYxXxA
z-{`;j_iD;3hu1Z)KfmgH_4{Sy%kGz-UM_yc_io*1{_k7AmVQip8}ugN{gThezG{85
z{yyhN#}AJ0ieH01mwj}5-}3g#+r=LezkK<2<lDs0AK!(&c6`PEdg|*huXJ9Ey#4kz
z{!RQVgO^S(IG;y8z5C?hi<}Q_UuS&T@jl_z%x6!Ya6Vc5$n~Mu!_Y?(kHsFBJ!*QW
z_h9W^vs-L8?%a^MbNPPO!<73DcYJSOzWw8N^_}c{b`SqNPJC+e)b#1SrzTGWA8Fm6
zevjeN`sdDXCcW)`eeQY4)74KMpIv&A_&DXs%4b0@_P<>Jy5{Yhw>)o(UT{78{LJf>
z$h*TI`aU*(6#Z29$>>w$C#}z0Kl6X-{1Wt4@_WY5j9*qiw|?XOYW)4uA3eq-#%cd;
z{~i6S^H1=<{r~m<!v6*SGx^8+pO<Mqt1NppTLf!3%W38~mg}rRY%wg+j6MIo|0gkQ
zWqQG)z;=Lb65B1-R<<u3JzU2)6*+#fpWvFs&m??KC|2MIuQ^X6PZr-Ffr-L<Mg7E^
zB(_SXN(ahLmHR7qMee#BpMs5&gvx4_H=~DEj2>DsdT7PJ(L*an53LwMhgLwQ%SXx4
z5Ez6Za9B`8&{p6Azasy2-Uyz0ZYiEZ-cDW~o?fn5oP`{(*z8#UGJCRYW#wWMV^v_S
zVKZf)#^%M!$}*o>oyDK!JM&_eUbYU7qnr;o?KmSicXOTNzQ+BDdpS=b?`PhJyh^;C
zJhypf@yheX@s;r{;=9Ke#rK4#om-LHmOGPs9yc4$3!Z#FF#&VIUcm;zKZ3u73WRHg
zkBWqf$4jb6c}b;8?U&M&J}UK6Qdeq~^gS6}Sx(u5GEZbYWSeB>NKKI3D%mH=E~zWY
zASErWDU&9%UD{82w)8FO4btV(Zql;SGo^k>c!<RcUl!QNr^BnxQ_UUA-NJp4>jB3x
zHZj%%%!`>i86z1I{=faF_y6X928P)GfBq%?Z~y=3pWnX~|1ST3#;}P&;s5l%+x{&4
z{pP3Z56|z>-(P$$`hM%H;McaVx?e3nGkq+7U-WL(+xc%oUr&CO`D*LS*UyumZhN%i
zLGS(idwcH$-QILd^OoYxjn{*&-MqU0TG<WPTN-!t?m0hDeN^*U_sOFttk2S(-*{2|
z%KuH<yX=o8Up9XG_Wj0>(?2i#>iTW*=l|ay3=^2Iu*_oJ#`cPR565+m!yJ1!_i)Sd
zvhvpQl<@50nZV=3ZN>SUos}bsV+qGf&U)^NylMP?0y+X70=a^Fgvx~f2;UR=Dtc4w
zoY(_V5z((ALZZB)2BJGfr;8<tXNu1f>k*wGVk(?1IFH|qFPc}AH<P!Jw~=QZmnNqz
z#|5@NmQzd)Oph7c7zF-V|GxV}=DWt%o1e5lF8Q$U!=4YiAHv@|y=VBq_u<{UdvDjh
z{qQ#ToyB{H51T$t{>=Av+Sk=zAAW89w&DA)ABTRH{R;oR==X_V4L{R=O#ANmz4iN(
z@8Um_e*F9~_2;vnyMG4$bo_bk=k8w<emDG)`KQLv!nmAKgYgi<T!vW;YZ)XNTbWj|
z=(E+ctz=VW4`%<zHj}M^Z8sYOdlj2BD>KU;=5po&rb7&z|1bXa`OE(I(jV);F8?I{
zi~l$Kcks`q-)+BF|IYo*^n1lGo?owj1b+|t8vOagN12a4A7bB^y_@{D^$qW9k(aNY
z@jUH#Z1_mx;erP@9uz&4eAN0V{?YS?e2@M-+Wy$<3Ewl7muFrJz0G->@iyb_l()%m
z_r2No=J(r__i`U%KWzQb^Ks>;=bz?%6!=i~KKQ-dd++z+ACx{aep346@{#$2!+X#7
zPv1}Z!1%G@W7wyMpVoY;`6Th#@=NvCTi<Gbr2Q)Y<Nh!8KM%u9264vej4@0R%y}&P
zSUcInI1{<1bIs$r%@xTl%OlMj%BLvcEp$MbK}0}gwTQWBvFLlzdt&#*r%UXS2$xWh
zh?bC$m@1Ye5-W5-U<-dEzp#LUV3?4xu$-`r(0hSofsFzwf-eONh5UtWMb?Y7i71Ht
z7S0s05Pc`wBW5aoLtI7ThWKN#*`oa-dxW0}B?>JU3KBjdd`GxhI7n!QfH!{v-*p}q
zZX2!^u3#=%&fV<i*(}*Mv9hriu&iJyXFbj8%({vtoh6UOmBo@dgz@eF3;*jG`Wfw*
z3s@`JFL6k7hH)6Nm$NNnJIMZq!--RX^DRdiM<M$pwraLNY~R_#IL~ld@^tYA@MZEj
z@d@#5<aOYc;g#iO;+5dNz<Zi6g1?x5H9wbtl|Z(DprD1&USTQGLec9YT_TdAXGGJ*
z(#7{n9FUZdIxkr%$tL9?{a$*RbeeRzw6pX-sch*vGIwON<i5!6ka3WfmYOEHRZ>t&
zLh6#FpQMU}g_ya>BO!gEt%9<GfdWbVS$yHV96VWE8#t`lCD?da|1m2wzhZJ_KEy1{
zBFN&(e2TG+;luy%{|Eks{QtnPpHY<2j^WAw-2cJ<p8q-byZU#+@3LQ?f3W^|^?k?p
zMc>N5O#Jxkozq+MHyd8Re!c3A$lInjk6vwgvEf<Z({)cgp6EZ;e`s|7%-yy-7P;c4
zqBn#Sg?<Z)3R{Vtk<n5)p~kNqB3&m`!F`VX5L*<x6lW!uG1nXp5w>egcmL=AHTeDP
zm&3nG<_>lljs#}OKQDg>|33a#^#6zd`3&<I6_{fgt^XST`SW+q-{@b0-;=-f|GfP-
z`rq51OrJ&HOnJWV8QTlpSFLYdK1hEscxUq_^HuxHj+gx}HauVPGWu=j`@na`FI6AE
zeE8@o&#RO-^WUnyo&MVRW!+=-JK{GV-rRKa^7YGC?_Ev2t^H)r^QDhZ-2Hkp{Kkfx
znRholJoL=xb>!=u=RY1dKDz!e_u<}$Tu*+!*!!;Ylklf4Z~C6jd&u!<{S(I*;jg#7
zdHweGyQ<grPZb}lJ=K1;|LN1GQ(heXu>E%-1OK0IpR?cSJy&=(?Zx*uKR!?YUH8xa
zPu7pAUsit9{vh!_@ICKm*`L*aUH{JivHg?k`vvc~KYaQm|NY|cUWNtCW=zZfI{(W1
zG4b2;FRMS7ecktum*=gpzVLY-JC>fmjlceV7yZ`yb2`IyW`3q={~CX>e&71#*2fF)
zn?GLpdgiD4@0cG|pWQzMy*GKk@%`lw?4N7Ccl}xPxB7SF?|FY(f2;lY|GDiG%TIfj
zaxP`gPt3)CQ-8#KQ~rAD%d;QO43k(b*`3(7F>hvaVlHE>|JVO7j?s}-jKhy@Gh+(F
zW~L$*9#$o`DeTpppSV0Z+?b62OaAxxd*Y|cH?J?jKlPanIMP_7|2O|E`d0Yu?AM83
zgnk(PpUvdYX#9WfU$KAh7@V2RnO3mWb6nus!}XMHIm5<3oqweNDKLaHU1V9oafFAL
zCyeb0Q#o@QGb7U!hA75`oYrE}(sE+l{0*ENSlw8jF~_jfaS9227kwZ!l~;j7li7;l
z!rx`TyZ#t4Tw`WpHDhG`)%qpj)7OtXKec^L{Kd*(#<G~1@n6*sj<2gf{CqwA<=<EC
zp9L6<*>|&~{I&iX^={kCOHUOa|9mR(ZrkTapDNx*zIpJ{@I}wlt&b$1w7lH>&g<jc
z53Fx5KVSC9_Wskm+4q(_T=vZH_2IX+Z;fAxK4W^Ad|U9^<jXs6%zV7|mHw;sPu(7T
zyFKydyX)Vs@4A!pwBk+H`*|N&KZbpn^xo&4&zr-qx4muqxZ&%@?~>mqd{zIV_NC(M
zrEk(d-~Q=lILPF|yoG56qx1h?zu*1T`~9A=p68B;rUbvZo3OgTC%*l>zj*HOxd^jL
z{FYiQ?J8{`a|XPSXRFdC6>0Sr%~Z{8YJZh~DjrwVR^d>O(Y&n9qSvOsRsV&qjkc8L
z2lY^OQ;mJPYt24eZ?m#Dwb9eibWzt)V^EXSEYou{3N~uiuhy1Ow^3fJuw9-@;f;c<
zvW}Xn8naTXOsqJ+XoDD=#8e44sb|v8(xDP7g_ZecaVWCPVN7Py<5U(|BONCbCXpi;
z%r%j9CDU3)A(nJbSAId^F5yhU7QQ&1Ib4@GrgJ>x66ZG+W)oQ?u#&5fdGbH0f0hi^
z%;(q+bEfck@a^ZB&YsCA{3rXz`tMACCbKZ}MDoRRPhgq*_x6uVUw(Xy{VetK-G6>o
zW{&3^`Rw{EvJ8)YU;m;1!}?d?KLe&{<~0mCzomX0{UPux?AP~SK7Wq<>tHNrH2in|
zXT(?EPn$k@eBb-;BI|d~WUfDK{fyWD)cn-{uJmo^x9XoQf9ErlFbXj2_`CVH{m-_q
zxu52L>i*jF-TE8vr-C<~FCt#-dU@!T(QEnF>)sfCc=_S`o9WN1AFqD6`qBHR+^@wy
z7JXgt_3B6ax4o~*U%q}({UZCt@t6BwbH7P^b^ST_(;tsMKa_sB;DPZYwP%{IroR65
z;_l;%51pO{zHEEd_v-tr)o<3moA*Zc+3Nd-xA)$-ee?CbKTp%&Y<_S5F5|_#2i-Sc
zUn#%La3$z!#0~qqw;m=uS@Gn|<A0BfpT<Ayd;atF`A;9dv;VyFmFq*{o94Ic-)nvR
z`}x)PX@4#=?qg<RIP=r*i~7e!ACtc({+42v=IY{E$C<&h?Elg~3BSvK7yiA^D8zo6
z>j2Mh9uDr`>_w~+EDu?haB%a92serqiZu!?=6lWC&3A>bfInSuve+b9QH5X9zeT$Q
z)^k^K>Tqia>=XBvcToPVv`F4WYNkk<KqK!n?sOgx{<Xp@#4ku}5_>Duz_*K&gRPt8
zA&W0t5&JDx4yIH8pZuTyf5|`be-i&B8Nyin*wUDJ{w4i*^*P~F!6%W=r@tlqJ;=13
zC5qXb;qE`d|H*%yf6e>J_s8S^1IEcrHyLF9E&YAu=Y*eqzbpPlGQMD*$#R9Mma&A%
znPn^M3buZB2JUMD<-+v>m${ueUa~cERB`oi|K!sWIWH<M9KyGaV>wG4^A|?@|91a8
z85~#)I4n5R*%+A@F>Ysk&a{g)fU}z?gl{>IFvnch2zGI<H9U<1y`oMs$CX*s&nYL%
zHA^lP%@yestq|{&PE{0EjaKoLKQI1Ms6^<6;4A)Qf%77_rLN0!D{hlLDsfZ9R_K#p
zx~PNnb%muWlB!b_HDv3hx+Rv1y%2pbc1!A|!X4!c3S80}qNn+@c-C{ZaP8)+6`L%r
zEqzC<OyD%vYIalBSxnQJHnKVL?hrN+Un_cuKaiu3>G}Wg|BwE!XI#a+m#Od{=#rS5
z-*<gm_)YwW#2<eq8@3GAbf(q+PW{aOI`>o3$77!l{B-<(j4_>Y@xL>_wSPH%*ZQRS
z?$_JHA18gE^*i%-#E&~)q(2FLRQ@FM<=j`!@6*0(f8X<s>-&c9(LWykkp4O2m+-&0
z3@;c?|6Baq<!9H=O}{dK@BMd%WeSG`M>)&=f9rqU{+aPZ`pb(Cr$1i)s`Fd#f5d;w
zKhu72ey{k!^1JiT%)f{J>HTy0J?}^LkE6d5{^c>sGp}cT%W1;X&85R`!z}mz^`Cox
z^#1K&lHpp+KU+Yaw}vf%aW+FE!@<Abe_8*w{_ny1nf)8H(O=;oHea`XVfuFZyWy{{
zKia=7zG}Z0fAjFwyjM$JUwo_gasF3<pLst^zD@b??4`@oX^$2?jC`8^_T^{0ua`gi
zy%u=3;K|ZQ$M3$q_4)3Br*`jizHa-X|Nh#GhmTJ^Y<e{J$*t#_ueZFu`oi|9$m3~`
zL>{|8PJ8V7tl`byj}@OKKb?A~{qoKu$9qwCH$R&7#`4R$Z=1e7{krQ*$7i_@4KMQ^
zA9>jF)b6d>7wK<4pX1;CdRg|O_+{wp-gl{=)_>mfap8Nt_Y&_--if{Ce0%tv^{3iz
zHNV#Uo&0aY@04#hK8bwZ@O|dLU}i~{%}k3K&i`M}5c+@LPv<ZFpS-^0{rLLF;J?zp
zS-)O=`}}qJ_d`Fw{@VTP@eiwSA)lXqi2d;6qvV&suL)n(zrOxz_kGtlzfWwh*`LJT
ze{xsiq2KeZuk2rHJX`WO@R9nX^AA|>irtjI{^I(^+cpolA8oiFaZBP_*|o(t&F_fa
zyLj)+9lIOxm(HH~a?<D2?9(M@?9cL^@4mF+TK|pxSJTfq9J_x|@Nnx1mh;7zPh5F(
zsruZ3Qw*p2PZ}JrIr8T4isN19S6|zE!}!{Z3;Cy?p2$7fcKXS=^h+UEoUiCyiadYy
zT=2!CSNv~q-D!Pb^4#{l_?OC0VsA2@zIn*~_|A)#kDq^PF$ggqXL4kcW_Dm=|F`ew
z^&jPb_A}4m{w3fqbdmoX&j%iDK1+Unp+#cSQZ~}1Qpu7^Qpz&>WoAkLmAWD`Lvgoy
zzs_V`b*=R(R~42hRH<CnN;j}KK4q+Kyx!=Vk+sox{aZR0v?l8`8(p&SvAJn;!Yaso
zvdKlGH->kNo|zmr?Kb&l_)p)(pv=(NFiKxSceOUJ&OzO|dJ}cL)we42N<Wv3mvL0A
zQY}`GR+m(}s;aIYq`p`sTJgF3DTO@c2Wm`OLE5)8Wz{oO%~b8w9;#Psq^sXhF;%!D
zRUx4v876s2e2Z9!c#UMAjHLWqIRojpqJBc!LeoWmN!^ieRH{<SRyZSDE`3?zq-d~k
zgU~u*S207$4Cx0_1!7GC-aM9Ehd7UNsq)nD2yt&_zsJ0mF^KUp<7y@y=B3OMY?rv)
z`51V$*>C@!|7+S$u|L@iDs0(2a|N^oOZY$WaB)3kOJ`PPVE)g_D9&2Ixu3h1>pI&E
z=0!~GEahykIWBPL@mcW&a+`3LaRzbObIs=z;CjbfCR!@PFLzk7SLh`7e)f%QU)U#b
zZx)CYStMd06vcO$%bDXVODaRt-(&v{Fo$uu@v{n?<t^l%$H~YM${xfK!*ziB7uQt|
zQw|$0N1g`mPaK}?@f;~U4+P!|hVyxIyklZ#c>nMBzn6?F*`IO0;@-fyhy60!UsgWm
zkAL6)todpF=N3ahYdVJmXC`M9=PizAj=LN;IJnqTn7#i${4@P;<bS^Z`hVB{3H!f-
zMUS(Ja}V1)=0HZy|6l(xGwo!T=Q}4DE_jQNm3JG@1)fsw(;R|d1n&wS74#627v>Qz
z68b1qA#zPLK+I9>jo2LVePUUnhed7)cL_Zgu;uIKe#IHXA<I6MosCnED~j8OX9ssM
zmk{R<c31X1_V4UP90HtuoL@P)xT|;!c(r-|@l5C8=i%d7!o$b=hNqgxoo6kNFE0;Y
zJ%7ERys)5%zDS1f1fc_hQUa@awYVKP_i)VOoWymRD}&pf_YU6?J^{XYysLTEax-wZ
zadL6UaI|u~Wfx?h%a+Twg)M{Kfy03_mrI7blB<owoK2eL0%Ott<9{#yG5@3cXXziw
zzkmP!|G$;#Ad3g<AC^Sc2)2`K2iXp=6|wQNO=G*y?!{@$eT?@NKcirW;0eKZf+0ei
zgsutk2^R}56S*(eEAd?-Ns?LWiX^wBp+vm6v3Q@jpZFxPMzN=2g<=|_FN6hzlZ2iL
z>IiKV_7FQQ!7X)PQbICL;<NZG@q^;e#n{DU#ZHPPh`$#1k|>whEHOu-K|EiyMR>Ll
zm(YEIX8vxz<Gca9Yj|?G=W@;Dy2&Zav7K!z>lBt=<{qX@#_bGk44Moq3@HrW3}65E
zGu&Xv`tS1Z+&{bj3;yl+d*F}UAE!S#e-{6i`L+Cq<d4E1SATH)%=_8?v-*d}H>=Nw
z-s`_@dp+sphUd$k1w5tt>8cF843Z3*44n*z7&kH<V>-Z8&a{N-53>c!FD4;o7nUU~
zA6Ra({$i`)_{b^9t;S=+GlffqBa8JK6EEY#|CRqw{kQw?^so9~%|DaB<$t>V`2CUi
z^WeAhZ>?W~KaPA&{<8Q})`$4_o$qSjUVj_=uK2Cf+oSJTKU96V`=RdR$xlgNRK5m(
zUGuf^>*O#0KFfbe{Br)wiLVpCv;Io@ZTI`g?~{LG{wV!%{mb?5!oN(03dR{sKUwCn
zXK<zQ8uPdE>+@UiiwivBPvu*{XT;CL`+;jV7dPi?HYs*h4nvMsj&jb$9M-Il7%dq5
z|JnSW_e1o%&$r-jCEvKeP5O5Ahup6TKjgl>`@G~s#v88}bD!*Zc;;T-?Z-FvUoE&8
zc0u!!&ZX>&$1gs)`1z9em0y=WUu?Sk^9t`ZhwFhiJMZmyZ2VO5`N5a`FP&cWz1aEG
z?Gf)I@y9t&Ry^JLbjGu9&tE)Ke;V~v;F;dDNzXUF+56e)d&_s;pF+PvzBhmQ{6*#a
zzn@F~tod8@&zbQR%K>&{E(sntp4D7yIo5I9=eo?@$~}dbL4ckADsKz_NkKQET!AD$
zUH-KKoC4GNllb*{eYn1|JFs8j^x#>>-OBxuXCiM7Z#~Z%?iQX~+<!QdIbU-da=+(X
z!mY=*UZ6-=Su{%Qvcz=h3Yn$Sf2G`{nkCMPAD5JqX_dVqmm=RP7cRR=_MyTr)iSLX
zU2FYP{V)2D4ObbzG0`?#XX<SnYINWDvFR%F$(FX(7i`q*6CGDMPIp*quW$F#y4Px}
zWuZl@g@n})s~OgkwjXV!ZQogCn`ao87?|qY=~wH&(<{;K&`#8vsTHTqsN1FQX)sIg
zy3P-+9Q6&#I~2DnsL6kl;gX&wsVCtj?koOAj7ij9XgR+NZ#(x-E_JTo>^E4gm_Poz
z^xOIOq+d?I4FA0UYy7X~pZ7n-zp;P({}lbH_`~(b@%NTrDZf+xzW8VOpYuP%KfOP`
zzZ-rt{fYmR_UG@PcYhiFUix+AC&#azU*CT%{C(nY8bdVWbjEXxw;9a-ul-y4$Kg--
zAGbfB{_y^r_P>Qmk2R4^fxUw*gQbs2pW)7*UBAx${P=_8C-blKzYKr7{dWHy|Lghp
zDPK*#T>13*<J*t>Ki>Q3@#(|I&mSW{FZlB33)k0)Un)PVf2#dh|Ec+l!&kd6QlC$K
zlzo5p_171A&z3y?|B&OM@57?U+Ryl2l)c#g?84&(56vF9-Z#5{=>EC~uOI3^W_kMb
z`PWxAZ(ZNnz0-YH_iooalXnN-EPieCrt_`CJKgsdAIv^1dcXU9{fD}b-k%shz5B5G
zgXTw`kJ%rOe184)$5+X3Zr^Wwv-z_2<Mj7dZzsS0{%ZLv@z-K+%HLdn)AhFaZOq&D
zxBFlFyexky`1<4Pq&KB+nBPozo&I{}>+P?vy;|||@{6a>rJjF$=J5Rb^9Rp6pEJH#
z@nZLjZ7)JzC_P{F^!DSRN5Kyl-mkoya7XM;&YhOKbMKkl|8=kCZrSaLH??kjzn*l1
z^`^+}8+Sz?2t87IeELzvBi%<cAF4bwd2r<ZuLmU$S3kJ$fRMb*!vhZr9_l<cf6D$s
z;&t)c^mo7BPI)`+?Z-D3Zvx+}f5ZKD!Q1k8RqvO4*z~FGYv=crA1OaBf8YOo<@bx<
zrhR+)b<^jR&stxOe!TF`{N40-=RPd_Jn!?#&$iz_eN*~2>s$2q;vY7@5C7w5vS#_s
za*lNwn-RMx=TvTW-g^E>Ayc7S{4e;J`5pLn@;>2R!;{YSoHKwshW7z~yP%HXY<>ye
z-<${77qk6lJ;++ln$D8Mq|X@4yq!&veKzZA7B-fN%=4He7}qhHF#9mQXZ+6W%c9A=
zh&h;rn`IgEKc*C>tBi9PH2)X=o&CG|_wV2Pe)Ij~WA0<MVZF$Dm~A7g8*?n9FGI@z
zi~oZD=P~?ZSivB|Q1XA`Kc9bz|BD%v7=4%on9ls4_b>UM_TP7Zj{cqb$KV&|kC3mb
zUyQ!J`LX2pr+?EJzWs0dU;p3rfAjw%|J(kXGt6YX#dwXenOTc%1BVayP2Ow#Qi9V3
zCkajw@aMb2!@#?f$BSnvZ#e&b{&ay|0yp`~`4V_f@MQA*;I`oT%M;G)z{e=?P@q~+
zOL($~j;Mv$TJhBqnv&Nfze+htZ;{#~HD79}<YCFLQfs95%1o6rR;W|_rr4-BO;J-R
zPpMXUvg%Cr9?kt)zFKQEr)sX!NLRN}Z&OcI*HmX$o2cBOcu~Pjp+Y`Ueu{jje24s2
z`Lpsu3e1Yf6b~sZkrS5fl0GfzEO|(hRqBM~Z;3Jq9?8j)n<QsRvB_+d-6!9y&@b;R
zmnCyw(pF-Mc%rzJgs5bw<P(Y0;#OiSL?ngR2v!Ni@z?Mj<mKbd<x%F|%b~$GgZT=>
zhkuX$DKMO3Jj}d<HHU2q>ok@g=2*t?|EK<y|Eu}O&ajD*ok^EzEz^9aMy4ocK9)nw
z$C&e2YFPYOR<mqlac5Cwkzr9{S;5lJs>^21Hkqw~{T16emL%pJrgBD6rd(znmOj=i
zY?s)cv29^1V138(mBoO?mt_rW9@|H@pX}uv*V$LIePI2?`jEAi)t+?+OC*aMOF#2(
zrhH~A=0A)YjD-xI{~!MQ^iS>Il|K`I&;Duqz5na$&&NN;eb9V=?M?Zs)h{BS`9FE|
zi1`8co$Q-?ukX34awYh(;-y9BP0q|ZVRrPyp|uC!9f&xv{=kldnuk4)Y8+=ek$Ixz
z#I55j$6JprKc;%z{<!e*iN|-HcyO}dRQ;)pQwpcTPT8DxJgsqR<;fK%l~1;v_;Z};
z#FUfQPFJ0MbI#&|`^At;yDy!+<bQcIZ;rscDR9~0(t%4)E^oe4dF|4TtXtP^E8P{l
zx8?rVhmue7o_W5w`=aV);H&#D-@aJ?!t2G`=Tlznc<KLo>Ra{qf8VeDSn;{>%l@yI
zzj=NS_!0ZF=I7R*dw!<<NchJ5Md0JEcfN0}-)OwK{btFV^w(mqSYB4XnDFApi@2AQ
zUa`GleS6^T%y%!}^}n0&?&`a(@8-T!c%Se=^;7xhgfHsfRDZDkob)sAm&xx*e^&jo
zWw2yC!g!ZK;eYQxhyTC+r!pL4;AgC46k&S7WX^JrHG+KuhccH8_YCePo^`y@d~5mQ
z_*V!B3P}rp6<#c|UF4p~R#928I#E8+6w#HUQljc2yM!4;HjA7VJuc=YzDAs1!doIy
zB21!7;;lrp#CGv;@jYUqV##8U#oWZxB~D5{kg}FhmUWU%lszOfL8@O|S|nP)jrTFv
zRSs2-WX@!6ecqXTwfy1yc6=LoUU3C*I&yfhN3-2xb!MHzqQ%<GX3Fly?!%tM{+fL^
zhbZT74r@+1E*|bq?kC*Sc{+GE@xA8XEWjw}Dd;I!C1AwQ&v%+fg}a$kh2tli0~-h1
zG}d|+bLPWLz04vkrOf5byIE>j6IiuaPq9p3S<7O_(!s3Ae340nIgRBps|<TIM=_@i
zmlRhpXCenPhZV<6j&B?-oUUATTtZxXILkPHb1dZ0;9z7gWBtn<gzx_4ALqaIeS7w8
z;rDw#8h<7Kx%*fB-`&3=|33eF@_#aeC!-sa0&@YgJF^(G7IQpv8S`Q0^DGP4=5jc2
zP32zAlQwWgKJRzlU_LHB)@dxcJ-p2Czx01L|F-w{O@{T%2bh;I&j0u3m;SdCA2Z&!
zzN>v3_geJ@<I|;&Z$7^C`0^w7hmY?2-}ktmeDCNThC5eom)`nu{ry#;tKwH1u1a6?
zy8h;d$jz8*i!RAuh&(s(OxT&ibF(ifUSz+dcs2Tl|ILn@`)-%spZ6&B3G>s{PnSL4
z@iP9U%=7N2OiwMIct2kM_|UV=H}gNreSY;(^rPX2y0_-9p1)Z3^5H9!*I!@Cy$FB)
z==uE@k6xaAwdnQdH+$ake3<(o`{TDywO^#Z&iwNG^Yt(K-@SkS_?7*4!~d-Rp?|;r
zmipcKv-(H;&tt#d{ks3N^2gNgQ@@*iU+~rR>&EX~zp8&a{NVh#;@8Gsnm>9zo4)UU
zJL#?B+s?OAAIrY|{kh_&`nMY&SG+TPll8LZ`J?CdFTcLH^z7La&Zkn(kH5U|dgYsQ
zuYF#2J`;ah@U-W7(rc{`H$MOTGWUzc*A?H@e=YrW?#K48`#;_I5dUG`N82y_-|K%)
z{LTLR#8>V2LT{eEQhK5H?A!~}cUwONf8F_Q^AGPo-T!p{9R5D@Yus0fuaCaW`?~qZ
zr+@2Nd|COJc>k~bbL;21?`_|w{0R9a{oCs2>#yCPXMLLf>Efr)A2+;z{pQTe?q>&{
zcs-4NZuDx$n_F+LzncD1^VPXm4zH%auzvRJ@svl;A7wwWc{c0Cr`LM#OyB%{7X4_)
zedT-Zcir!*-#2`c^Gfes-iK*llz-~{>ipjHx%dOuhe;n&-}AiB_}u-o^UuXU0)H}o
zI)0V=B>U0r)0r<(-;R9!@!9_4=eIs@9=^$WyWx%E>-rbwPdOh)Jd%9u`i%K?={x2R
ze?Lt8c<<w>PZK{UePVsj^Sb86ju-x~H@}<miRruWFPWdZUv|BF^P1)LyqC9Me0$CM
zDe^n}@6LbKOpR<eINI4uSe`R4Wu47t!M=xcBX6Q$vG8}HH3A}hFSzX3IoTF-$n$*T
z4dAQeoy6VAS<b=Ad4g*l_cpFJPA<-;?D=f=?3cK5`8$Nu#XKaUr0&Rkl$)&ZL2;U*
ziF~E(B{@f><Eq)3lXN)^c=Y~jJXV>j#H!#aD=1r}(5zaonWe3#>t`^-DASNfFGD+B
z>y*}c?ddv}dOAi*W~<FknB*B37+D%j*WIEsP4BhgN#jLEGxZ~N#I=@duF;a#d8K<;
z|GfS}T~qC!n(|t*I!*emMl(!~n2A_+Ssk`AvifQfZ^mVmqWe%wMO#-_%fQw6uE}%b
zr3O+uMe3o-pX86qMJwJ{mC_W^Hr8d<PtxD7YpcbkzDR9}x`Fx}we1?#I=6M7>eOrP
zQeUTXOMzGRtn^k{V}%t8(sIg@3x)mp|8VDUPUd*e(azDxmd`B6u;ee(U!%X;e+B<O
z`BVBQ@XxtFO@E*Mwf{T+SH-v8pQe0Z|8VgG&*z|Tr+zU0X!zp$G5vk?+v~46-=6(Y
z`*qLHU4Qodd%=*(q{ArkkNMB5-+xB;;f?OY6CB-#H%#~8T_<ZF-a8!o@ZQIKTr;{4
zZ*(8t=svvBeR!k$@P_q1yx<bk<dVvQRL^AB(aWPpFOMF*JbLu<=+VogIYuv!9=$wz
z&gkXQqnAgIULHMqdGzSz(W93~k6s>4?B&riqnAgIULHMqdGzSz(W93~k6s>a`s2p8
z=CAo*KYgwLw)NZG!E|{vrKhK;Q)<Ad{?QN^4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R
z7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7
zfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z
J5TI)a003&9DSiL|

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/8dffe4c5c26d891b578fd2ea4b9adfc0c96ad5f7
new file mode 100644
index 0000000000000000000000000000000000000000..54a777d22c159918c5e85078560fe363c2a78b10
GIT binary patch
literal 71409
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bG01{z<0&gh&4@$$Rp#T5>
z!`P$rXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD
zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mk
zz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD42=+Ay0zf8@_na=fe&LJh~8g*@524<
z5BlzXyZQFozU$m~);*GY5%wbe$%6ZycS7&nzGwHK_I|_N_qXD1-ncRMy6Cl~mzB@C
zo|t#M@~p=7<oksW)Na*XTzG2HajBEn&rCUg_k7X0b!S!2f4XRKCF;ueix1CfpZ$Gy
z)<yBFVpr~+dwN3hXy?(%C-$65I^%h6%{j?)w&z&S&%N;L;=>D*&RLzic=7VhFArND
zN8RhZ{`k^|OXsf7xm)#c!Q+T0Umj0+Ec&GV>7r*^&q|&;KD+he>f5}}&R=~#oq5ai
zTKf&lhtkiC-?D!!`_=rX=Fgp9Ex()oaWUjDxG_y;|0<X*^Ii6eSSjCe4r}&5oVI*-
z1v^A$h|UsyB&sInDYjRPU;K=Cyp)#wZ`H**EA>LOt|{@!vdZ+x@hf?$-PNkmn_!Tl
zU#sn^nXe(HdPzY?o=ZtlJKMCt{Dr|E_2csT(krCvWEacLQFyADp};R!E^}H+Lc&}m
zQ9zG>j?foLcg2}13Cg?V>LoZtWP}++k4xmqxXZ=My_WthzD4Ym7`uqEz)`+Sf(9~X
znyq@>+KbeB75ij2$}r1Lm9<ysR?<|wEO$@tru+jrFX>}qkA#~=>LeItpU64LCQ3XK
zJjj#FCByxNFH*Qrv|049@KnL+LTMuTf=%4J*|Ru)@I4oAkrtEG6txsw#<!pE58pGs
zc>*T|yZJZreC5vIN#i-h<-l&q@|Gos)1L2<fUMv*zAfBVT(h|6^Pd!bAhkl;Kys33
zxzJg`vx3_EPTaPfcer8&Z%gP(eG+lubz{Bt|L{Mi|4;w7GoNSs$ePZ4im8sdn>mW<
z3WE}Z0;4mFJ4ZE_24?}wq<^n|iTvSWTE%&tFF@c8-zP39wgl!mjNkrB{}TQA|Mvvu
z4Lr7jJNSQbePhvO`0{Tt!#tJ(t{Z$W`7QX5@K$rb<FI9`Wq!|in8}AtkLQsflkj$4
zCDzM-xBdF^XDw3`doZUJ=W%v5*3C>k3~T@D{AT#Q?jJk*Qo$I>1nEuUkNCYgTv<Ap
zG?@!HF7bPdycGQ@rY62ZluLL8|9@_Fu6f+K0%Br}l67J^{IMK%Y^+@8_zw$<iv1CP
zAaPmZq=c=+0WnX}=fc&(o#L0}gVi>x&r>Z@Y>*L@{3G^KR9)hP%tN`UvKcZlGS{S4
zrTWF&M2dy%g>Q=Ql{Hofk-Z@vBm7UWO60!866rSSPm<OWXT%PQJ`lMjY%Rnru#`_*
zKtkldn4j1?L0j$`7O(%tzmNXV|8?`P#s9Xyw}1Km`0%ayo9!3=4}$N$yubJ9&F3>8
zKfgZrSmmzm?V@|<pIE;B^;Y4{;TNjUzCDt>zxj^%ouoT!?{Pojc>U|c|Ie<URo^ds
zY4KF#@ukPJUz~j3`sMtWFP|7b^1lD{X89|Fm)l-xe(3)0_4mj>zTcXkFT8s5Oyi~a
z`-ZQdekT9s{LS&}{da?}*FV{Qod1#YYx1wp480sxe2e+NaX)7H`^Wmn#UGA;RhUn3
z{N+*LcjBMG>&g|%#?M^LxS#1bdmUe?&@F+rTw*N#jOxsq9G!d<g;m6+N+?KAkjfP|
z6}=&xDD+!URM<-FjEt7b2{nG@5a~Lh3hs04huEUnr8q0OjJf7;h_GE_y8A!>ufgwU
zza0KmGIy}ca3nBG{(1RB`1kR@qW?eq&u5s&sK6Y{X#Lmt&!4|@{zm^2{GR-+|L5($
z(f{85Wcn=nX3Fz@&)8n*zG{8z@<IB8!8@BbnXlSkcD(F=vEliOm(g!K-v_=keyRHS
z<-<o$d0wTwng3Sp?ey2iFY6ww-x0t0@aCqQm#<&GdhcrDZS5y}o-ciT;_la*;Wswi
z%)Gnd;h|?fuOnaQJpb{y@zM2%xexa~<a+Y+#ol+FpM*bcdDHiF-b0Q@>z_Ej2!Flp
z&Fi<f-&MV~f2#Oc?Wy*&{ZF4ho$})7hwZ-$8TkKv`>dds{cPHs=P$m$`SE%B@4A2X
zf3ki|{j&0-_6Ldgf$w=g%l@qX>-u;8kL{mS-!FK_{o&Im`R^Bh_cAPCHe*`;*ZEi8
zkBQ%&e_8#x?CZXNygYA(^@Y##*s=8dZT$7`yXd#ppVJwxGxIY|``7r3_50Q@w?1BY
z-~92)*E2uWf5-f&`t1H8=)KANjqfjiVE<h6z3b1Sztz7Zf6x2V`djVC|IcloSbo~G
zlyfO_eqt{EoBAW>oATFFU!MJNW|+ik$?n9qjd?SZ6LT43{lETyag2_vVjO;Kn;BCW
zHZv8m@USYeO<}L*{KVzS;l^bAU-G}l-xEJozIlBK{;AJwz>&rp{lEEV(YL~HXTMJT
zBJ{)P|7<3IM&ti;|BC&4$KcFl&a{H1p5p@79<HZs%NaKQ>HH)8Pk|wv=_1Ptjw3w0
zJYj54n97;cm>HR-Fhnsf<g^x(mX;Ia=5OHK!0N{Gj5&s-j#EhJyXXU<sk{mtn#@)V
z7yd5$-Sx+a;Tkg&s~IEfuhuUKpT2(F`Kj$|;xASPGnU27jQ^^BaC}|;;pgk=FaN%B
z|17{@%)Xl?<*)VEsCV05UV5td_~%oJciTQc`c&~g^38*nhA(=aZha*Aq~+!2cU~Xo
zeqeoj`T4R(w)daj&Azwf;j(9juMfYqeQW$m^cmB`<lBPRCSTrpW9H+vuk>H7f9m$&
z+wF-r-(CN9eb=3&rxkCq-p~8M`Z4Uor1w7WeBK;>z3pw=#|>XMewX|{;j8)=wJ#N4
zFMX5#`SwpY!$Bqw<}FMs7@hzB`u*;w-tYH}^*nb(G$r`O-GtQzKJo46{l#;K&qbJ3
z;<wadX;*0jnKN?R719;ADs57cR&UWv)!e4`SNW&naYbzv4)qw#%i1h@ZTegFU+CIs
zOKE;k4^=nS*r&VJ?6dVYD|=HLJq=A4bsaSZHA&4fJvXCZqh|eTZ3%T7<+Tdi<+&8z
zD99@7sHv(kE49kRit~#$h_Ojbm2i`KChaU8DzQ>nnQs<{BFh}cWF|dMWsx<~aWY{N
zIfB7l6IoX>tz{HqN#}Is7ZmOi&J=9ni{qKYb%|p-$3rf0ep6vKkwpS4x%!wV|C9P>
z$zaWVj_ojK3XcchexB*<nT*1JvVW}q&h%$83o}n7Up)5&mbrg#|G4z!$H&;uQa|7Q
z=VxW+c+Qc}uFoRN@aXsTANoJ6e+B+CV2Wm5!;te^>c`O^0>8q3egEb2=g7Yf#&Sl(
zf9HQjeD(db>66Fzz5gz<e&<Z)`oq@Gc>Pb!PyO#o-*$eh{@L<(K0^tk0K<;In}6H?
zZ2Ow~Y5u3~uT9^rzwv%5c+>eJ;>E6)hh7=ImVdqOjp2uvAHKht{=E9}>W8Zzy?@I6
zTKr?t*9Bj%ezbqv`>Ooq>lf88vR@p3x&Jlyo5WYwpL0L`@#ynI>4ys*7(Y^bruk~>
z>t8SKKEC+S>1p80wpV?xzQ0=iX6?IqZ)BgXzHfMY?~U6xU*G%lH2uxy_xA5HUd(&Y
zee?B|^2-cYg04o~u)lljVZxIYPtH94_qh0J{IkC2KVP5!^x-@E&pThaJ`}!be!Kp?
z*2lk}UwxnU=Q86yW+sL+KmER_e_Zr2`D@~DDQ0P|E}nIq87#~GFa4A7yZm?I-}{V0
z?5DX7@cicC;Qq~C#45q^kYx!6H=l@bqgbI>qtIf$*Sy_)SNIC}(*-AsO_CK=_$B>Y
zv`b(;cO|C|w}!wzac_AC<=;w+<V~bzilhlN@;>8E=kefQE4)Jdg2X1Vw?YkkyEr-6
zx>+8w__7tT-(uxpI`#j_|M~xy{1g8t@lTQ=jKz;FjhW|P(vMf46FwDu68U`kTf*Oi
zOxszan7tY9{uBJ4{MY%{yq|o3JpMmmoXm8SLFV7m-$#B<_}TZn;$I}=3+9<DSD0!U
zOPHKlwz95Z>t|=+z9vvETrY5$+nM7fTO&smR}c43J}r^+qT<3KeA_sdv&1ofVYL5m
z_s^5TfyIEsf-{|sk$Dm0cE;ySyI2D_yLm$Rmh%X6%w>&W7w1~T(<sm@>LhbqnMM7a
za<W{r<WkXGkxtPH@lNSfMPb!w6;Ju|;y;B-gkA`~;!hShFLGPzx;(eyHrb;RH$`lP
zJ_)9aI!IqvSgInaIz>@KwqB}RVyV~*(f4Avq+TlAQNEzSC9NTPnm>zYJy#3YZoXQv
z$<o@=cf`sBPIIkhH)Wm0G@WT9n<MWIVH5GSqKEhcIr^BM{}2EF=>K}gRm^*t3jbC8
zdh+Au_g&u>eiQ#8@yDOZhAo3NooV&IQ$Mr6&i$11@!01BKOO%cV@zjU{O`<f?O#sc
zwLWRS`}Owl$4TF3{m%Ry@#D@H=}$r*l|RXRIro+G`?T-c-}iju`o7_N^pA%>q<_x%
zCH(I#!wZJf{}%st`Pubz)340md;gtbnZhB#QO<Jz-}+y-e`frU{_^6(>5rGc>ipLG
zAMxMv&$J($-z$Ew{O<fS^Y5X5djDL0&-+pR<LIx1e|e1Z%<Ea-a+>gTbLp_#Fw6aa
z{pa2vy?;BHWVjad&lZs9tzipboXyb4aPaT<U)H~^|9h~0X8*=)^jG+Y&DX78n7*C<
zZuqO~kM?hiuiEd$-#mOZ@70pm7vJiAoc~qeXWq|}Z&N-zd+G9Y+M`7eBcJBKefin$
z>*bGruLYhhc(U}-@w;zteZG6(sondWuiL)pzrXh4;p0;en;y-5a_hO~>n*RZzOa2N
z@_5=Kk;m?j(;mA%Yk2ebW5s95Pp95#zr6Fv@m|#3&5vfivHY^`+oo?%zwY|d@mcOe
z!^^zKM;^93wR>y!Mf#i1=lFNOUY5Nmei{0@_g(6z^`G~AT=-t^y~KNycVcfj-yVKv
z{i*g_&960oC;yx9JLTJrPa>Z;e4qI*m|2o#Gt***^Z(Z~g#O?6)A>vPC$BGgKfeAk
z_^<SD)~{FJK7U>Q{m{>^zjpt6{KM*7$mgdYVn6)&DETGuYr<Fcudlz_ec$!X?-Sc=
z_9wCTpWKyr==Xf<EBluk&z3w6e5C&9`~%jzVmIZlzqr2fw#@_XM;q=(+>*Ffc5U%Z
z^E+bqF5WwH$L>b_rL$+gob)+0`*g_}`?I{~yDzP{)_-IF)%0@?$L=2#JluMM<$Uqw
z6Ib3`sy=t%6vL_hlLp6Yj=VX%;&|8j)z|jkFuwNULjLKeCvs1=oqlpI{Zhyk=PNpw
zBF|qv7ku&P75^JtcUm8qJhy!>{-yGh*qe-}Zys_#zVo8x<L94R3_{GunH-s<nH`we
z|LyyE{YUwq{me7Ce+l>tUF849^MOa3&yt^CXpxw-l#O($RI;R!l(NiznOV|*rLM@#
zP~5HFuQOR!U2DC{RfQ!ARVtUY(hcm5PZ_HluQ$48WNq|a|CY`Lt;ssgMwcvnY;M||
zunIDtY;w`)jo}@mXC{YDyG_0s{?m6cC^IxRjMA6TU9HWlb5M7#-b5X5^{ooM($6L1
zWgHc&REyQ4)g{%gs;a98sV`QER(vjhN+D1Aff|!mkoGN2S@jH6GgUjahw9ZD>FPIB
zOcm}(RY+(^hDlx$-y#+wUL)BjBPst@&OrLDsGpFw&~(vXQg`GVm8z7o70$?(OJ9~a
zDH<%?Ahb@{Rm@N_L;8VKfmoA(H;*OPA<m;*sysD3Lfo6#?=i1s3}U>@xSC0ac`365
z+a)e{J_cTG_S^sG|C;tw>`yj>3R^bMTmfyt68=v-TwD*?(wS8mnE$gfinA7Q?&q%M
zy3RI(c@YykOF7$Xjtks*d{%sc+$Nl5oIzanT=O{vxZd%WiI&Rn%N>^N6*|eipM4|S
z7xoF<n*}087Ks=LMe$wca_0ETlFHEZ_t?J!%wb$^{Hy|Jc?-GcaWZm*vIlX*a2??O
z#dVd#l*5M0k*9(C6Ne{zJVy%81A+H~;e6g4@0i#b-v9gk?<M0(_GjF$xHoX_VZY4w
zmz9tC<KOo`Ykr#lxy8`Wn$F?CnaLT&d5fc&<1WVy4lec-X7B$G|4jcI`JeB<{@=BK
z!v3#d(c`S*+{5;cIgpX_|JQ%aOgq`-`OXQ33*O>m<=w_}fv1%FG=~xUZ}vW}YrIPY
z)(OrODC8^Qk>UQw)z0&i_cl)nXEB>An>}X$&r%)>p1ZtSf*(W}B|;>Yi<SwN^Js9$
zvwL%06$p{|Ak!^#N#dbMw~(cvB)>RMAlEN0PhKB^&BBRdI%4~TcJZy_QR407za_X>
zFpb}e_cM1f_Y*D_&J}E1SSPbx<j~<>#2w37$D;k$|Lc<XFW;X2Q2s6A*Mpy1zG;3|
z_%Qvg`)mD|4bOKyvwYG2=IF<hUoL%K^j`I~*9-gSA}<VIt$cmpwae?CSBG9ceZl|S
z_KCtH=|=`no<3XfLgeLv=g%L<-WRwlbkFLM`U~5)weN%8?S3`;xz{t#CtvQX-j=@M
zdc)&R#v``pIxkK=lYi>{#P^BfQ}t(T&$d3XeEjiY=fla5E<BcaBJ?Ee>8lr9@A^J*
ze--`u{*&$p(|0QGK73UBsrCOngY*CVzs&!t8RJ=AGB0QR`+xp_Peu*aH5|7%HnIvZ
z@h}|xf0SVc(-r2$%w9|{7-Sj4m@l%5v3s&lWY^^^=5FB0<C()9#Zks&_xJPfKmXUW
zFXX+%dzUko)tTu4L*W1XKMQ}9etr9S!xyjblYj318TOs&^Tv1TZ*$%+_+<Op?PLAB
zeXrSH9ev69iu?7vw@W|pe|G(<`Cb1<<@c+f0$#N~T65p+5!;J7Z+5;>c=_mw-6QLV
zJ@<Frmc8-on*8-+*T3E>c`)Pg+XokJow}NK<>WQpTNbyQZiU>sdNbt4>g&2UmfrY#
zBmU;nn`>@;zH{_`!9%wDpRaRWJb3>9l|6TLo*KXCc)|Fx<t6jW!_Ui~J$|zE(b@-7
zA0|9c`*7r&$hXLkU*EoYWAt|7yS?w1zd!I!`EBfL&)1f3nLhmabo9&7FTcO8`c=fR
zgV~PB{m=KWoL`oIQ~sU&U!2L6shL6l|Be4HjPDs$8CCv?{>u3w|67_dhwTxYH*+FG
z5rYrYE9TG4>zQj=J~3Zq`0?ML<vyn+4<Gjej%tp}+~WMq{Omj}?5mij{I_P1XA$N2
z&!NNS$hhzC^1mPdh5q00Z{eREKY#wH`(wfUki(wyHtStxX_hrCFIn%g8*=PpEoaR7
z`{U0ChH};>_O<MD*<W%5@I2>#B(Rm&lKmdTr@txxb69S0dGWLHZ{td3f61|uXDOct
zp9_y1mjl-x9$o<_VF{6G0@B>C*&lJJ@pcOq3Ptm`b0x4%VNqa>V-sU<WxLJtlqHfw
zgXbMjB$qWS!~gDIR=;_fRJlJ26bf$QJI=$;$18A|Ux81F=K&`d=Xvgvg3J<&rNkwc
z35oE{;SuK(7kJ3e%WJ}!!*-JG1?O6x6TF^$`uzTa7NRUtcV!>Ss!2TM58>L$*};2X
z=(gw@QEd?|;akGrL_@^Q#S(?`c!Rj!b6w)g7G@D$AvlfuB3mWvF*bP)bN1)VU;g|4
z)&0x+Kbz5zS(teOgT;SKCJlB?u6)irEK>h({}%gW^8XNX0^3}c&i_k)ZT@lf$I2hR
z-^IUtdbi@$qgPTN&U{<<OZ=DM_vc^qzHa>H^qu`%&F5S1PrP=0rT+H!$7f%Ue)s=g
z`rYd{GeZ_*+y9c^&fnrbv3)xGmHkgS;|AuHOiLJi7#x|rSY6o**qm8vSb{m^`M5<?
zB=$%&h<y^45WXm)CXpc(DD_c7THI3Xtay_YhYXtxuk=c3F$H_oX!W^jvMSdVUdvU>
zmnq3=9ML;r$gICeD?#&=mW$30ZEelzYO~ZmwLSE|8uwY$TD~-OHjLJLtQ(`BZ@{Jh
zUHiO7zB;33hPIi`9&HBg58Ap0_e}E55=`e99nlrj2vO-$>C@=dmC-NP=2d&3bWZWO
zf~4Gai3>ss{B?ZK__qpe6>Sx}D7=&J45uhZ0@oED1)d_#4{S$R&#;+toMTUC<7EBM
zJelPT+gFZvoR2y5S#SMs|Gnf_*k3zFA!dC>lfTx#4*q=eOXc_6AHrXiKOg+m`?=#=
z`0uIzlK(3GjQA@3Mf~g1uUkG#e=vA+;zi5zgcnIKdtdH)@%VYz3$B+(UNpaucwzoj
z`GNf%i#w0*J3JM6>G<-_bGsK!uX5iie@OV)`C-XB`?vSsx_wmoHsj~RpWA-4{%HJJ
z_xt1Di~qv@<bTik{P0u$mz!TBzHj)^^7HAh-ak=)Fa2Hm_u#MMuL2*vK5YE_<)`m|
z1BTOotA9`ZCHs5z@Ap5~d~^At^?AnU=5JqriT#`P_t($HuXUeTKV^Rs|M>Im+Lza#
zTzvTV;j_nKPsN_LKVf^e;)Tbn<kxL)gx~yn5%<LUq2}YnR~#R?Kka`n^Umq*-#7Yi
z3SQVe&c2^`hyT{nTl?>_K8U|xeCP9xhHI7A^lvP<S$Ol=HP6ehFFIXbbLHU`-z!d6
zj$Q4&zU;=nTOaSfz9)1?{f5W2qt_?g343_pVfnq#+g-PO?mWCJbI1DT%p2-AHE(I%
zHG1Uo-22tJmv5iNJk5Ik_0_F6hu)OGZhLk8^~`sxKB#?S{2c#D^W(b@%%8NrG=F3H
zZuw>Vd$zZX@8^8|^IPrj<zGjC^8e=l_nV=M@e;#v1}6s5|11pLEcqN;I3?K4n1dP5
zG4ZnbaF}pBV4c9!%`lze3S&R>X_hZ6kt`|9KFq&ar*Q4#Im$Vi<rQN+voOae?j~+w
z4mMUX=1WYgSxVS0v+1xIu;eiEF@!KOvK(UB!no;=^G}1H;=fz|RR8($%lgN{uZO>|
zd=pO+mJ`||>?g)2F;9G+$RmOEeBAsTf^|YnLd60M0+$5~USEF6_bTuitH22!D{fbA
zZk}fD&FmqJn|_P_y!4~!=fqzezk7e`eb4*m^5g9<#y_^dcl}iRQT=_&k8i(1{$Ba3
z{@d>Rg|AXSHveA#H}dbX-#35F{{7?6w!gRj8vnidyYBakKMnuqFs)&k!W{eG@Yk+y
zH@~U<%K!W2@Bcr~{w(^t<ewlzBx5#HCG!F18_W+_qdDEUH*l)5zWP7+Pwt<Lf9n`d
zG0tP$!#IbTgN>X06q^y78=D-57dH?83}IdIBC-1dt(@{KZA@+~=h<vIrgKX0^z$hR
z1PfLRKI8Y|y~<_ERm>A6@Ju*GR8e?8uN3D)_Hd3j><`(B*!<WRbE@!o^6K)Y^9XYX
zaoyx@7WgY>F7saYp|r1fwonCs0e^zvJ)s{$&cfBAF_L{U%CdK*4@!!QJrfKS&=h(o
za!FK6q)DKiX9H(D`)iiJjCKEA{vG<~&2WfuHIod}k^gdk_x*nJyYTOX|8)#u|8jp8
zewq7`>!a((gb%{+ZogjpYQihq*SlV`zh-&4>Y2#Xc~69&wm)-xk@zC^+1JO{9=&{I
z_}Kc<#QW>-F1%;(z~RB92X>G4KaP7M_Eh}&idV_+KEF?WC-mCk#ggY;FPq-{d+YR;
z`^|^f|KFs)D|)Z+VdDqGkFP&I_;ly9{nwUn$zO#(?s?<)n*GhjH~Me>y_)jM;dRaH
z&#yXP{eBtwvis$ymy2KVy<7L0|NGXjr5_XD2E7S*zvT0=uUg-%zt8#6@q^>L;@6<h
zWgp$%x4gabcJYVAFJHbL`8M(M$9G|`9bfUkp8ERBE1lOOZ@;~be-r=8;HA?G&gYR&
z?>>3>BIiTf*BM`Syia&F^VyRpoKF@%a((FaF!YhcW3k6&kD4CpJy?6!>=xUNJ2zzR
zT)v<6Fy+3(9pBrRZ~wSmeJA^#-NQeR6Q7zqHGO*Tsmas8M_Tu%-(z^R{<-s;NpHJf
zpL-tiboEomXP2HNK2CYE@>$S}{V&(Qu6g_BEzg^x7hKOiKl6Ge^6v15zK@L`ML*Sj
zGWrzxN$d00&-`CHzXW}i{GRbM<CoRXt>1XR8h^j^M~^XyaoT^|e@Fl7{1g0d|9}0z
z@PC2-O#boy=VhADD$8EY7Qq_Ma+*1g<vME+TMSDyW6yu@|49s6nO?9cupMBV#CD6d
zmF){h57#kHMUG$WC%9(uGYOv)iWNA*YtGZilg0N(V50C|Q9tn}iLH{U(t)y5<^IZD
zk-IL(r(mNbp|V=#jZ(XupX5T(Y!MDo8!=b$=i-JEyCk+rPL$dx^;%L+B3#T}JYV{h
zoQqto6ocq(en(zKp6OiDT(UfS`IH2F1qFq733Cf2@Xz8o&S}c=lY@ynj>nnXkKL6i
z@89h|EPoIGefw`4!%ZeV*30Y;TuI#TxmY>pvF%{{&UJ%7OmG6fIL|DOi)@crHCR?L
z&1TAG&R{8IUBTAEF_Fufdl6?adoSBhc1~^$zFU0jdA4yrV=HFyVAf-9W71@7We8z3
zW4_B;!qLI$$nlzW7pnrNH?JN4Q9d1>YaBxC4_Q-~YZ!YOUoq}v{L1LX{ERi8a~F3k
z_X^Hu9I2cyxU6^=@NMSp<(|PA$ic}G%fZLd!B)=lgGrHj4GS+@G}}Se*DSACeYp7e
z_X+d~{N@kjbK!Z#b&J!IQ-YJ9qlR@o(*cIX|L^}Z_-FU`<L{Th?){SgbMWttzYl-2
z{WSUJ|E1u|!>{4ryuR-Jyz~>_XT7h-zR7?4^(F7?+3$~k|M-87aT4RL|E+%ne!6`Z
z`|k21<JZ~WCw|ZP{rh+1AOAnxf5Lxf|8oE3`djnw+yC)Q<xJiTV*i@|{{Q!q(Tb&r
zg@I)X%L3MStZA&$ECNhQ3`q>UEUcUrJkfkR_{w;8aeQYz&r-oE#c_{Io#z9O7@rH@
zex8+F5gaV62N-8Eyk-<;y~)wWb&R8s?JsLQn={)HRz+4uRzbEFb{~!cjz#Q|tUgRF
z3<->KELYfSIM#AFvx~4+Fu!E-WWK`E!+wy<i}x&_fxuJ2Z9*#qCHM__TDcx_-Q}_6
z4;DDU-^PDipiQVySVTBp@CSdifRNxBfkgsG1-$sba5HlLX8+3ZovVY}g;SL+l;sPP
zG@~fPDTZH+E1B*x9cG%u{GHXDBah=X+XfbGW*O!L=0!}+44Qv!e@Xt_@k8s!tnX*P
zZTo8Ub^F)OZ|&dNfA0Qy^@rMz#Gku=fByUD-?P80zn6b^`?}-Hm9K_B;(izWd-Ff;
zf589EjFBwDtbr_o%<q{dvoy07uy8Z)W<2x%+uy{$=l=TqoAPhbzxaPa|E~YD{(tFT
z!`~af1%I#nJ^ODaLnhN7Mv4Cif6e}O{j=cb)Gyn=4UNSGqplncfzc2c4S~@R7!85Z
z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R0EGbS=nnSL9qgk!*hhD;kM3X}-N8P(gMD-d
z`{)k#(H-ogJJ?5eu#fIwAKk$|x`Tam2m9y__R$^eqdV9~cd(D{U?1JVKDvW_bO-zB
z4tCJA$tWHTfzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c
z4S|so0tzF=qoe*D4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c
z4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5E$tp
zATZJcVATJkAut*OqaiQ~MnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0q
YLtr!nMnhmU1V%$(Gz3ONU<8B!0CJOm3IG5A

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/91d787a9298ddc015efa783a92c4bdba8af0d7de b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/91d787a9298ddc015efa783a92c4bdba8af0d7de
new file mode 100644
index 0000000000000000000000000000000000000000..826747d852b00397d859fcd65e7404d42cd329f5
GIT binary patch
literal 44
wcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}RumU;s0j7z7wnKAT#7`^(7y0KiQMpa1{>

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/92c065286f956f086e977556358f6b54b12bcacc b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/92c065286f956f086e977556358f6b54b12bcacc
new file mode 100644
index 0000000000000000000000000000000000000000..77b8f518b4c46b2cef89bd7a12f23ea49c8505ab
GIT binary patch
literal 315
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU|@7v!@$rH!N|bGAi$84Sdti%#>nvK>-x_I
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3t%M-cp)OEX;u%6S(Ja1#z9{>E>(X!)`t!
z11|GTG0cAgHD8KNli8bzkp<O!C*D@x#{vn0Nqk~_wLGZ2bhdC7S&kyEvz))!Z?oQF
zUBmvEosj{_?srA6gI-^FbN#i(E9sZLFCt&>ezysmri*M&?Dd?Bcy{mw@XqCC;Cm_X
I_Q|U-0IQK?#{d8T

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a35c9bb71792b60a13dea23a41b41847ad4b93d6 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a35c9bb71792b60a13dea23a41b41847ad4b93d6
new file mode 100644
index 0000000000000000000000000000000000000000..45d6b6fa606fe4a25d00476bd4ff785bb80fbba4
GIT binary patch
literal 44
xcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$tt&B(ySAi$842vQZ~2>{n92^atX

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a6ea960c7b4d42772888280277b26e645ceee904 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/a6ea960c7b4d42772888280277b26e645ceee904
new file mode 100644
index 0000000000000000000000000000000000000000..14954c595882e6d35c5f77f36dfcb9ad315c606c
GIT binary patch
literal 22470
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aApti5{
zug+hueoguH_ebw{%kNLVOaJWn_3Y0ihJNN=7GoAiW=*Dj4ATE?{{8+d@UP|nO~$Fr
zXIP5ZS~<3I+H$sX$a6%nUt%j}yTV$`R?J?&_JL(3%Y2sWEE8B0S-n_3GmEjvFb6Zv
z{{Qb!&oBAk$Nwb%+4=kS@6g{4zg7QC{}cE3z`vdU*#0W~e*1I9Pu-uD-^;$f_<Zpb
z+b6$|1s`90y8hY!bHu084{P7@zo~d_^{VlO?(=icBA>5&p7DIubLN*tuV%ek`_kg2
z^ee_U-`<M6cmMF|W5=iFPX|9GecJHp+?U#Kyx-M+xc;pAdHM(Q4~HMtKd=6J^7kx*
zDN`S_7^^s2Bby1UD)W6NC*~HW9LDVoGZ^w26Pfs!jxcyIR4_FB5B+!gUm}Az^8(hB
zELlt+7|j{q{44z1^T+)6?OzLjtNpqANAKUIe;oe||5X3n{_VmSme0>V{C&6ajr^-g
zFSb6Hcy9ZA*>j~AO)pkGw|K_-^w*=g52Wtf-(7I~<89x&(f4lNdwwtSe)t3BhxZ;H
zeZ==f=UM3s*_U!Jc0c!f8T!ig<;NF7ucp7UeI4}n$$QPu+TZv7Wc|hbGw%DEuWny-
zKc{}W^^x<F{pa8>3126E_x@S^ll2$p@3`N4|D5_S%=DG%A+rXn9@{Op8g@61G)`r1
z7v6n*Qv8bir}=*ioEH2dcvtYKpofsWFpqGN&_|&Pk!zv>Vvb^O#O8?a6U!1kEOJY@
zOX#_PEnheHE6x}WS@x;yY@B*rQQS5>JGg_nggAe&yRzr8e`hb^5a8_N{L0D2UBzR-
ztIhk5XF3l*4<FAG9zNbTJk>nzJZpJ;d3pHi`RfJcg#|_QMKXjZ2ptfV5?IZv#qGek
zhhrA!B(BR`8Qk`~cleI*3GmJ1UCpzWn}NHHlZ!)!qm|<=yCC~qwp_L?Y#Hnh91fhh
zTr%91Tx}fYY|<<j7>oWN|9kO|`5)asOaDmz{rmU-|E)|1Sv*+(uq3iZu$^Q($aa9O
zh>f3Z8ryw#FHU3bW4y2U83j88PYAvf3=!HSbWMm)xLA0Z$bGS1iSH6glFU+9B)KIG
zCE~@6#rwql#3zY0iaix86w?rWAuJ%AB=k&BM`)w4huCQeZmIi{5|VKepT%E^9~6Hs
z#x5o+c2X=s{I$53M7hLfi8&Gt;`yR2!n1|AgzgJ8^LO(d=MCUp!;{NBmun{1O-@;k
z?QC0Fr?B)g_b_ELZf9s?&}3j?NMZ11`1-$};RZw2f0uvf{@MLs@NdW81ApZHIQ_}_
zv-r2nujM}^e-!?>`h(+V-p}@*)jvGGS$#hAUjJ>|>q##+JYV)K;OT_N{Ez27W_%L=
z<o6T1r%Ru-JYMst_0gqAA0GuhYIr#Pq4FcwN8caqelqi!%8Qzp?XNsvZ-2e+wbkq7
zSM4tsykva+=*{GJg&*F0`t@b@*S%jIzwKe*WsqdhWawl##JG{^7}Ei!a;7Crf0!*;
zelZC#yRa-_`M`3M^%q+W$45><ZZ#eoo+(@^99gW_n0Of<{;&Lh>c8E8r+?M|YW|u0
zE&tQ?$M28Cp9jC4e{1~`{Bh)K@|VS*vOdJW?|fJL_WIk{cg1g=-X49&`l0H>-4Asi
zPku`JqVhHP>zc2PUnhV0_gVf+;+OMZPJEsCo%L7JZ@b?|exLjk^GE5A>tC*a7ye~3
zR4~q9`pGhnJ%cNa*O<SRU!UKCUtHi3e=6SsJ|lh}-Va=}xwtuBvq`b5au{;7a+Gr}
z=CEdc#Aw0b|Ig<4ydR?9eZB>MEBVI#ZPK@^KjeN*_#yZ0-RC7AGTwN-nEPbM!!!5t
zZa==U|7yX-unU@(bS`CIJbv-X#m|?#ul&07`C`-MpI3OVIb09C*?DirW8<fa&kw%j
zf9dq1@5RoiZjX2$i9gPHvf}B^r!$^?d;a2?`qL;-5&Ue@^Nnxzes=oa@}2jm(65m1
z&0ju$QThJw=aN5b{#N~SW_-nRfZdo&g2#<#HP>2>bsYD(E_1hXPvK<{VCTQe+rocR
z&`l^;Ac;?xf2{zgz;ym3etljauCMG4>{mEFc$RUua)0ER$eY7k&$EWRh36LcAC6?s
z*W8BO?>U!n>+!7@C=yl{jS{;oF<rVsW~uaFDL1KRiL>IzCFNvVWv|Gk$hXRc%Wjf=
zsPId*OshrLTEA5Pi~eK7RmN{jw9VF;dK-rt-8X)0y2^aArLFY^8#Vhx$5oEg9oE|G
z+r6~zwc2V~XwhmRVfDjmhP9;aM_XyzcUIZv8O9|BruugJ)%x%BN_0E46SZb)#c4C@
zcIkT>%+kBA^Fu2~eS`82#jOfz@}Fe5q~}TMNqC9-ioX$K615jv&hNt8&i#{1o$ELI
z4OT1WkN+<HcK$u-m(wr9Kkxq<|Eu}u{ZH|4>>vL>MSm*(aQ$)oz2#TR@07nU{u%z~
z{Lk=D?~m{AhTlwo;{T-m`TOVHUxvSzeqH&=@vG<8_g@QtpZJ@`5Y0H9@f_oA2DATb
z|Cau7_!Iue?a!w_y#J>CZ(-77O=MGG?_kSd>0{DoxbtV%ud_ct{^0n@{OkNL!{2Vd
z-G9gbdj5UNSCcPSK7Ic9_T&DK_da@j`tb4d$H>nMzWn*Z^>yNx%FpVbYCqP0YX0Kz
z)$WVb=TjeL-=BT`^@ZNEC6E6<<ap@&u;{V&GrkvPFSb9s@OZ&Pvj?vC&F&w%zwW{7
zhx(6Mo<4p4^_9(A*LQaBbl=sz+x5=m-N84DU)#LteCzN|_r1jjvk!~j?|xtZq3)yi
zC&o|jKCJ$r`H|;i_QxZiUw{4aRq~tL_Z#1AzO4N?{k_%O$*;e^TK-D>wb+~TH`m{E
zy)Awl^S1r%{?|S)%U=q<{`flSP3arvHxpi`zn=Me`|E43R=m9Y;^}j#=O3RrJiq?@
z!Sl}Nj4xKa*!^PLi;x#e&lf$t{W$1R@WX}oEAJ-U5xbLfr{(V4dnWgP-K)7<c6;JY
ztsCF3C*5GZDRTS9UC{?ZkCYytepK;D_tDIUDi2K_9J&ANLCM3_4=y~Ad3fMK!9$(L
z=1<vQNW3n7oBr<C+bM6Sz5V#c;!WV2^>4V}E_hr1uIl}g51T%<eeL|7@+0NP<?s8y
zul#=T+q7>lzi#@R@>%Q4(T^A2nZKL<?%aokpXYr(`Puf{r*BH%W_^qPUi`!6_u+s1
zOx7&FS<bO8V>4nG<($f`&RfqPDP$^ii~j{5Grt4hPTnWHYk1PRo^u9p$M8PjZx_@N
zoXs!6`<wFs`(n1=tOr@kS<_jPnDiNgnYXhkvd?B+&BDeqk$E1I1mikJ6J{T#_l)0}
zeOWY_7cmF3aI-99{>PNUbd_-qgXaIjzq5Z=|Ni}Z-*3Kue9V2UHmny}53_A#bz_cY
z^kqo-fAL??|2&3Y3@aE!7)t(6{O9v8@qaOc5~B~30MnWO^Zq6O)BgMJ&(Xg#{}}w@
z{1Nh1^^4KBH$RsA{`7Ae!?*uU|Lgy|{%`(&<bT_LbB39Ww-~Q6HZyCnZQ$_XzR7!y
zUrKPg;3UB*0{(ngco=wh@_6wq<qhY*&z~-^OW-DdIbQ<r37$-zAKVr^e|f@r9rzdp
z9tu<oY6(vk(Gj%}TPwa=LR0dZ<X0&t=`B)wq~=R4l{_r@Rcej&UYV(K#tL<c-xM1a
zrzvVG<tf!FPgb3&-lMr+%U5fS=2Xp98tLjb>TT+&>YD28Y7><^6fY{6DOAWO%1@E+
zl<$z=Dt}g9NP$`LnBpOYC33>DUDBr|oh1)RvPzwh{4G%?!6P|Ya+BmNDK?pnviszl
z75e3U<+5b%OWI0I5l<ACk`R>)m3$&`THH!(g@~l^8o?@oIQ|;GgS>paxjf3;dpR`N
zW-wo2`0(%1KLv(UjE9+bu;#E$VV%a(!yL;P{{Pg!@_#k|*cmo4vNP#2t!0|e)W{UY
z%*S$w`51E^OAU)3%W9TwEbc6-EHW%gEGt<0S#{aW*(S3!u)ktE$CAXH!&J^F%9P8j
z!_vokh3yjCGqx>k1+4E_zOopw__C~F&13t>_LIGw<2w6lwhyeoSRb;svf8umV2NaL
zW9eu9&6Lk<#r%g+gRzjo^Z&ztpZ=-+yYgq^@7X_XzxRKA{rULExDT4|ue~XMwfaTm
zGyf-#9x*@QzLR}(@AW-bRjveIR=l+6yvdnaC(MqXIJEZQy8{si)*sk$Q1h_oQH|qF
zCo)fzoVazI<#_9{<;PTy+aDJ`KJoaj6Aw-noT@*SaZ2G-*eRRSj;A$FtvtEnr1HtO
z6Mv2~otSd++UcsZZ_Zg<aK9LGY4@eGm;5gaTz0s0;L?-Jo3B(}yL2P#*0tM8cg5~)
zx&QT{<deK-o-gjcsCpUr>i)~OFV?^Cdhz!8lovZ*`oEs~R{j0o_iH~^d~W=*|Lf&%
zp5Ft0#Qv=Lx%KCspJ_i5zA=9h__*tx?_29P8gFjDS@I_Rwb(0`mz6Ijytwfq?&YLc
zY;RcK9(X(R-OG3V?<Tyv`flsHx$hL-Cwx%-RQ@^Pi~2X!AFMwo{mlDi@_W*sRsU=m
zEE$h5-epku-}}$u|F8e4496Jw87mn@m|ifMv)p5iVBf%@%q7D;gS&}m9d9(>TD~~`
z6#{}n(!yVb7mI8cxhJw!R937`lutB8bfu`2sJh55VFr=SBBw=<i+PE!5$BiimPnKc
zljxFoE72^mT|8WTkC>=fve;uWH}Q0dladdltYwsCon#Yb56MiB>KB(5i576<eav;0
zLzN?$GnreTcP3vge>lG#-$tHSTmhVp93Je^Y`0jQS?925u{N`rvb(YSuqUy<X5Y;r
z%K4kanp2L8hr5&e3HNlK4&F_CulY9%FbaAKdJ0wv81eJ-o#s*DZst_s_{rwL#=$m?
zwVuVC`7l#2vj|Hmb2;;FmKxRsRxQ?3EE8DPve>b7Fe@@&WD;RcV|mOf!ye61%qhbq
z#TCq%$id8E#W9oP8%GPLD_0$t5Z4~gGS1%|3pq467}?8M|1t+LsWZs_Q}}E3ciCU9
zf4}}c|L@L_^?(0goj-ZMJbs-2*7xn%w}s#D{b>A^{O9gp^?!H&iv0Wh@5%qk44#Z`
zObW~e%<jx$%v#Lx%w^1nna{H<V4KU~z%`Y7HBTDvcivz=F1}pe0NxZ{dERUuC+@49
zS2&V5q&ViXe`D)lvt#REjbz!#q|Eq`p^R}R;}u4B<{2!!Z2as3>@94nY*K73Z2Iii
z*>`d<aoKZE=276w;ZGJY7c3GyD-b8}QQ)+okC3#miO6*kLD6v0rJ{mjo5dW(H;AiB
z+?Kd2*(J4A>b+F8w5rTG86(+l*$=XBWS7hO%i74Sl-e$7FPSM(BK}`2MqEy!Q6gQU
zL4rZjTk^I9pG3U)J+V@;Kcb1E4x%?iUBqO?UWrzU&J%egEG^6;^hjVHzXP8V?{Ds1
zTxwhnT#{VYoCnyKv9d5LF`WH#^oPLLqo3qHs=qILoAKuA>ou>-UMsv#dL8)2^6mGx
z8Sir56}(&fZqB<8Zx!C2dh_hf$2TkAFuW;$UGUoMwc9I?mv%4epDlW#^tkh({(}Se
z&)>gr|KWXw2O$qUAMif7dO!Mp!@Wg!f86<hd(Extn-gxhT~EH6dO7%_*7=mP4^MNQ
znsP$qME3FfN8OGX9`-&|c5vr`%mbPSzZ^P$B=cDIi3_J*o#{Gv@_gil?h9Wpm|fJl
z7=Cf*#rqc@Uc7y=`r?@jffq_H+_`Y|g6o9?=PS>LoDV*K>iq2sDwixSf4a<i<@n{@
zmrq>&e0k=Ty;n7^|Gn|#R_&eJcW>T1e}BTmt&dxuetN$Awa$BskA<HOeSZ6;^=sEx
z!LN6|Ec&|Ro6Yy--*^6S{q^m4@!yI69{pd#7|mSD;>OC$c8F~``*)5=t|0Co+{rw_
zJd3z%xEeVtIhnY)xps24b2@VF;E>>$#=elfkRy!qEoTuIC-)L=L!S9O61<anZ}DpJ
zHSxLg&EY-DYr?-;V7efOaEj<@@pqCdrMJthmi;2vBfnQ}fvk+|d6`5RW*JEtaoOi`
zx(aFv917PJW-2x-sw;j_*sl<xFh_ou+&x(d*&>-o(pu69QhZYLrEW^UlQEM0CUaZn
zwaj~&Z!)2>VRG~3w<u&PURN|z+OOoTyk7a4@^9rw%9E70D(NX!$@9ud$*z;}li48c
zAZ;k!BDGB7oEV4LLeUE%GllttDg=D_-|;@-cI0}*sl`>mwUbMq`!6>GZ#S<gZw!wU
z_g~J{9P#Y(Z2GL<Sj<?HSvgs?S@f9`nJgKr|9}4L{P*G?u0ILC^?sZFX8rB_OZlhT
zkEZWezWw~__Vvq`O<x?pO!<87)5DMRKZL%|e^>nW{_EUVyI<zMWO!Npg7d|f=O>>}
zd*<@w{zKCTy!UPH+28ZIFY?givB6XGXCI!aztDRb^OEo7{TJU~*u5-$+3>RYWz)+I
zFW<dvd};PV>G{QHwa*ql-Tg%6$=%0a9?L)Z|M>Z1?<b3&L_azCIPS5|W1GjT9-BX@
zf0F;C?TPwRhG#9$)SfSU!TNgQ+e`27eAM`I`<uegx?jwH68<e_;9z>iw1l~erHi$Y
zjgMW9gMmwy=O`}=f385D&|48VaY@OOQZ+JrWR>N1$)1$?C_O<sPWqqJ6e$g<Ba)$#
z{Suzy%3_(K^F^)-uM*}HsTL6y<re!Y)-V1~Tv%eUIFERl*j`a*Q5(@9Q4>*N(E!nB
zqKsnxqW46cM5l`ei2fI`61^wtDz;qog77K9ECEygR^Ch8C%LM*=5Q5o&E^#6T+CU>
z70u<#`JR0hTM+Ac=5nSJj5`_SnYJ_CXYyxaW}3|Oiz$JrhtZAEmQj>(EyJGw-T&78
z)%ZL8&xzlkezpGE@{8rS%x|IJ8ov{MH~#+dtK;XT@4LU*ecSx?*O#s@SHJB2YW%JD
z>-jGizSMs?`6=K7(>s$l2Vc3ptbA_x%>LP{XL8RYo_D@j@#^85zIWB{wLkp&u<j$*
zC-F}TpXPol`~3U!<<E7W%Rit0Z2QIZOXuf1pNc>2{8;rN>;3(AGu~Ca%X#PfPVb$?
zJC1inZ$G}-`KI*E+1DptrN3PDe9u$8C+?4<ABjCId~oW){0DsZBkr!ev+>UMI~I4i
zZwuWrxwYh$$*tm>H8<669l6DR+v&F69iDst?yEoC@zCwjiATJT%^x>C7Jc&LiR?3%
z=l`DTzgY6(+RIz7?!HQW<@74$)x1~tUg^Ky|N8nHlXofa)jo88IR8QNqw2?dA9_D5
z{h;&l>&N0xDxa=>-0^AB7x!;#zUTkE_&fZcE`u>+3)32wShg8#?rf%P5^T3vPqMye
zoyaD`af|a0mnM%UU%0?E!A(NB!b~E8BEcd?B9Dd3g)M~5gzpO-5zG@<z#q%6&p(rI
zIqwsm9Xxk<^m%{r2Jqhy5D|JHq%15ZEGld-yhu1&BwzHTn6ShRi7yiGBzPo?BqgO5
zOP!ONA)P3@R4z+SKyHs*k9@oQEjdoPM%m@E|7CgPvgOXn&sDsvWUOqV9HG2b`Kj`9
z<p^aD<y_^d$}5#Kl|LvoDoHB&D{3h?$=k?(l&@1LP|#HPD!)_SP~Jn1Pxh7cAE}>`
z3nkpe?Zq~ThKn8&*(RJQ^jV;lUxlxeXE|38rvj$}X97nfyA8V#yBFIzmOo56jNS|<
z|E2#q|Fh$}+SiGnQa(I>$NcW-+y1v(-`s!A@#fB(^tWAa*S<afw)5?lH#1)+z4Cv#
z?Zw&`Sue_;w>>j`#`4VTDesemk5)W9{y^@*x%+qSOFqbWu;@YUgA4aR-z&ctde7|M
z+q*q?3+_(8TXy&S9rHW?Zr{G$di&<Bt2a$<*4(^zbLq{4H+gQ)z7uvg<gVOZiMvsE
zo9+hQopSfly%i7C9~C{m`lR()=ZpAP6JFcBIsYc`?d-R!->!Qr@lO1`*@urGbUsFW
zeDiVdCx_1-pVL3HeUbWF`0eQTdq0l+-1@8I_p;y0zc>Ec{fqgx+;7p}7Qc&sFZzAt
z_vPR7e?R{n^q1%V|Np-j?lXO1iD7eM|G?hQA<fCp>BLdQZpbdmet~T(+gG+;_CgLN
zPCZUZ&W)TmxV~^7<4NK@!uyPO39kz8Y#t{bE1n{rIG!2Yhq$V^%DGIr_HssXT5wu$
zKIHh!eu!-Y>qeH@%<WA5jMo?f7;gTL`~T=)(7$hgIsXa%+xvIx--Le-|8M>eVpz(c
z&uGcS!EDVugLw<fW!4C`NH#gP@2rY!mTX5^ZCR~Z!&twtq_70Ch_Gg{onT+XA<ikl
zxrM`uV;cKTHfOdOY`@rcv+=W)v+A;1v&ypevWT&8u$*H4&-9M*3xhMmo&U@Jd;RbK
zH}&uNKVE<4{@(FR<kyLx9Y39azW(9*W6k#!-`9NS_@47^%D0!_&VQ@==J?I`+m>&q
zzwv$(`r7{G@#jaMxIP{K$ndH3)8<e6KDm5e@+JA3^p8nDdw$vc{_s2N&x${)e|`S_
z`+tn_8&eqbPUa4l0M<jSN^H$+m)OME4cQ;CHL^`-d&l;e&4w+6bt20%W_9Lzrr(S;
zjCzd5j2jvL{nP#{{)g}PlAkp{{(S%TUF*l4@6*2}eO3D^^L65v2cLsKzxuS|lh3DB
zALBp1{<!KB(`UucU7w4-9Q&&J-QfrG&*Yy|exCpN=I7y`pMI|TrS*HyuQxxNejfbU
z`>X$V)F0lz-2bHhU;UrYV8Iy4w3Yc5OFHWm)}5?xS<Tr>*;cW+vBz^5a`teZ;QYX8
z$EC$>%kzlm3U5CD1c8|XK?3yxLW0i(4+w<_9}~6^(G*D+o+jiclq$4Z=(5mbAy45L
zkzJza#j?d~#Ph}D#czup5IrrDEs`mcF2X0`EWA<Ztk5^1Swd1mp9Q}O?i6Gf{KEf&
zPnFM>cQJPhXEysm)+`n^W+o;N#*YlzjL#UgnR*y`8SNNd89y+*W{73H%&5U6$<)u7
z!l=*W!hD?h4D)8@iOiYIf0;Cx7BYTjOk(n2`op-Jv7NDu(U`HGVeNm(|E>St|F!&E
z`N#CnqCeOF`2K1AE%^JvuWP?v{Nnm;`+MPUxj%M)Ui@zPZU1}suS-9d{V4yQ_-+1I
z+pllF%>1(O%j++5zbyDX<I}#6_8%vH<oxvH<JphKpA0{>eboL~^zq%toKM}Kct6#B
z%=?(}k^dvh#|<CXe46~3=gYs(+F#VaR(;#_{n?NAKf8V%_!aca=V#=P=ij$|Z~HFu
zz51KQw<X_fzBhfJ@%`p^*B=vq?ES&_bKlS1zqb80`osQb!f*HA_P=?4m;QS7bIs4T
zpHV+o{1E=0^EKy7%;zT`cYQeje$G3Ow>w{-e3|_s{>APW=`WT(zx9mcnZi@uC-#pI
zKDzK|-J{PBO&-2~;PNo+(e1|{p5#7V^>p3Su&0-vv_E<K<oVMX&$^zuJ(GWS?P=Ll
z@uvr#%zg6tN#)b-r^Zj8KbiF8&*PturahE>@ciC`yHD=O-RZj>czfw>{X0o_8t(|)
z{eD;YzRClUhe?l2p6q`*?OE*eSuZ}m{Pn8t^`zGwuXny?dt?9R*c;||OWw16WdC&Q
z6T_FcUtNCa{;K^Q{pa2vxxW&BFaKfwEBi0@zXC%sgD=D5{~P{S{J;J$``_Mw*8iRV
zAOCmj@A5x|zx97@{*m^5_qW;Ko`0M4-Sr3KPn};|e?R+E`}g(Vb^o6Jw`5FbI>|hX
zHJ*Jt$9~ScTnRj`d{P4Wf^kCe!ZSopi|!Wl5pNYQ7yl-<LDWRVRrsfnudtfPQ4tZ*
zDWc3`-$V_?bi^l1FiSm=IwTb&RVyVW{YF||cDtOHLaO3Z#j}cTic1s>6pqO2%Gb*+
zkgb*plXj4jm(-G&CGI7DSnRs!Pm#SMzeFNMPl&3BIf$u?9TNQ`;v@1^c&l)Yu!?Yw
z&~!mA!3u$?{P*}~@kR3e=RLx^o!5<zNgz~6Pb6Q|L9AP>L`++3tLP5V17i2Ye~1@~
zi->;_dnv{%o+G|a{EE1Ugp-7dgs#K{iTx7OC0rz0#2dxViI$3ri5?WWB@!k&OH^I-
zmhel#FZ`)|3whkQJUE2dKd|<($S_Z0Y-UJh5NBZeulDc5-#LG||91S5{PXCy&>!PJ
zuYX_q-Tdd`U%&s08PXZM88<MlWPHaE#BiKp0^<(G0>(28NB^__|ME}ffBb(%1`kFf
zrg=<rnf#d^Fy=C@VQ~3B^>5gp-ro;@MgHRawf<+}&yJsNKSh3=_@@8u>DS9&JHH+L
z{^-ZupW(k)e!c!#_Up>8Q@>99+V`vRm)9@eUtK@df7<*k`f2)e_mBD?2Y*cbS^ewd
zFWuj@zvF+m|7!j@?Z@fw?cWo>&-_;Nwd+g!=SiQAe|-Ld=|k}Q)_1LMd)`caJ@wVf
zm&aaweIE0i`?=)v!_N=ASo?C?tF5ma-sHde{`&drfH&{nY<Zje?*F@O@2x*5eE9Xg
z{=@Q*E}xaYZvOW3d)&|2zv6yN{W1TW@-OMXGQ(_!#|%Y`8BDX7&$6sym1K)z3u1f5
z`ik`l8xzN2j&RNyoQpXbxFWgKxOll{an*9$^IYZe<~`5r&*#dQ$E(Zpi|Z8US`Ho#
zCJq4(M~;IWJe+Ge?y}3UUu3(=wvDZUZ35dQHW4-hRyGy}=6<G+jQ$Me|0Mpd`_23-
z^~cL^yS}P^E&KA~(~l1|?`OO-d}sX5=AHUG&?t2FTm3f?uVr6{zpi<G{8h`#MK8WR
zfBQ`NndGy`XOo{jf2R2S)3d41B%jGW-T2t!k<`O`_x<nn-nn@z?WW0%pV#cJvR;0A
zLG}Fdvxm>LoN+w!@3iR|sWV?s{XBX1<k^!ePA)vDeA4Q~?c*np=N?Zw&VFLiiDf6+
zPpmt^a&ps2r_)o;UO7MIBHLx@E3>c6x^m{q<*Q-WU2e?15p^^C)~#EPw-fH1zkBB1
z^7}g;6hGYekn@q+quz%z9_+q<_TI(29Cs()@w%ga$KuYGJ2iK|-`#oF>Tc=XvU_Fs
ztsb;Kobq_dQ@7`5U#xv)_*VME*-!Rg&wu~@)8O}y-wc0c|GE6PXXs_v!myRmop}{Y
z2%8N@H)j!-6}K|CE0-u2E4M#SF0V9S6Q3Y|5q~6q7XNGh3V{^C=R&3;nxYP3TH+nz
z%fx4jXNb=i_mlW1u}N}{)II5ivWoIy@^bQb<sT~eEB;V4SME|_Q!7)q)cB||Nh@1t
zo9=$S)%t4<SdCbW6^v_*H=1OczBlnR$urqyGTEfU#M!jiOxOIO`E?6s%UFw*7L`_^
zHa@mDZME!P+gMxfGvzcpX7SW=rR658U)D+1nU=lgT;^34k1W?%=$K|2lxw-FZdIP4
z`dvdx_kiJh)3fG!X1>PLjO2|nbzM{|<?c#L%Ux9V&^)KZYEWo2$ACdoN6taCT(F)0
zE!SZ-9gfL7m-s^Y9|}wn`7C)?_N452naOgW6h0_e%bUq6$}EwImXVQc6|EJ?5Zx;N
zUTTBP2^k+*X4%b>qGB<k5n}S<-^9E{7V|}LOkzI6FzbKn|J4j^EWg<-*>*D3{eSgm
z!%v$pvtCy{?s)k7$?_K(Z+3l*{%-X1_E*_=wNLlmx4bjshQu}ZYvwmZZXdfl=U&!*
zi^sXo-#k})R{XT&ncs^uFT|edKYsD}_fw~*OCGA-U2yB>oyLdrAFDjcc=qCX;!~c7
z*Y9fH3B2>-?)rN&cav_)T-|r+(WS?i<*)JGe)XW}$^IwDAL%^|ekAeu@V(X>53lLn
zJab#_!QJP*AK1PJ{k;9H;p6ky4$sq{2t6);`1nD^lZP*p-vqyH_~7!@;OD1bBEN!v
z#{BVMnDhV2Z`WU2f9L-d{JZ++sjv25SATc^b@9)k|IZl%m<}*R{|)=~=;y~@XMSh?
zar|@om(R}=Klpy0|8?-+Q|4BVHC($npK<VWtMEo}ce5X5F<@3_?qOZcd6>^mL|o#W
zM7r2dfn2U4wofcym~<JXn9^BRv#wzkW&O(fpOc$!3$GEk0rwuBRGwd)UhFxnxh(q3
z=NW$dJ^TCSFU#K!f2#j}{U6U<$FhR4=`ZK+ieJ6IYJc(l`uOAVSFMk`-oALl^Oofu
z$484V&%Rv!T>Hu5qvi*ZcVRC(pVU5{^o0HCj%O^d>))^XsQ;np)wM_McM`9!zjX5a
z^>cFPWiPJ3Jo8G|Rh^qLcf;;~yr28P_o3ut?<cn(J%5n&u;bC1huQa|@A}+XbSL0$
z>s^8S){h0A-+gBNc>jHgdnfL^y=8Ta@%D^cN;fuKO}jkzLjC!ci;`DZuI{_iaoP9M
z>Pz~UPh5C*=HIFJr}$34J$?M#oXbCN6yMo!r{GrMwKW$L&OJKgemd>c#xrRbHeKq!
z;&Q#}mi;}xhs_V}-nV;_^NR6J_N$w(T;AS(r~WSW_0Ja#FBiW4{HFPx?x#K9?*0t^
zRs7@2SL-kQpC^CH{QUk4=eM+PEnhc%mj3+i%bD+9zkdC0{&VF|;=eQoOGa+SiwrXv
z_At0G9bo0*_7gM`KPf#yPFVh(te8xV<Yuv0(Y>Pj;*3%fvd(h3@;%Cn)$gmjsvpyI
z(CyGu)|;!#rT0d^*l@Z*g5GDH=Q?L}FX{cz7c;K5_-buvwZ>G}=(YYn-Gkb9w6<v*
z=r-u?)N$6S)w!s<+rY?}$@qie0i*pUDrURQmYGSLiI|-*OEFKiP_j<8&vl;d+T_aO
z^2M>+p2^n4ipN~gbe}1=d4XAl=?t^g7OO2ESSnfBSwFJvcG&5lVc%o-$<Dzp&05oZ
ziE*%zw$TE^Y@_{VlWq3c^V;XyT(;a{-fJ3Y{NM1A;R{1k!zubax`(tpweDzzXyt0u
zshOx2sPrh$QM@JBCbL3HK;o%LtB|K)j{uv%8U9v&f4*GqJoZdxA%+isxBq?cZ~p&V
z|HA&B{xkLO$G^A!T=|pFpu@_^Ud(35_K2;FeI;8gbH~4#zb<@#|E2%4`M15lZ!)ZB
zKES+$asIzIzx2PI_?Yp&^<C}TxYw#L7@sbEeDm?8$Cn?uKYVoG|Gvll<a<Z&Fx<Iv
zyY$wN>+i1$T@}CDa8>%6*Y!6yL~h1hTXaeOLgcxLXTr`Do|}C^@gn;r#jDXb{BL&L
z+;_Y5{=7%2Pne&se!A@Wj+gN-WuA9GWqNAy#QX93$A_L}zM20~?(?gUq8|-E)V(!-
z_58)Mmk(c=y#D%9?nU_XN6+uSc=Yn@t3|IrzuEJS=fm6&*&n}ss{JDMb>^4fpRa$>
z|L*<s$FJ<a8~$hg5B>Y~x76>>pVdF&e;)hw?$`aFl|QC_pZeYG`+~2gUpIc|`c?hY
z;Rol>6~8wA()`i$+4Oz)+evQ~-*&#0`dIeu@6Q!K)xX{NxZ<7Ro2-{L&mTRvfBE&r
zrDxBca6XlKe*EQy*DK$gd+qbG^O^Y5f~P&tlU{3mxbgYtm$_dozOMML{%h&4b3e9!
z-T&#vhxiZkKH7fa|6c!d;&1lfC%$UG7kcyTmC_5nXXjp+zT5gK`0LJZn}2x!>Heql
z=kWKLU*o<?e0}s~-q+1PKK)zI;>*g%#QT5cpIblAeQ*0d<wwXb>EBjAUw`fXJnPf+
zPZvLZ{<z`&>o;d!c0W7t#OrDFbE8)~-rRa~{nhlBny=2ia(FfUh4r&%kEc9({wVv2
z&9hlAKE2j^XZq&vv*<@V?knGOzw3Ti{l4LooL749@;*%aqWn|mSLgSp&&40OK1}+M
z`kv=~#^>&zoqsO=5%`nw)A6g^C)tl~pU!-V`gY_i2ao;7&u@L+JbaV$cEcOR*Yz*V
zpK?BqcqIAQ^%?W)(s#@s{(hMF@!rQ%pC*1z`o#L4=XK4C9WVS}Z+<uB6VrF$Uot;+
zzwCPV<~7Ufc`t9j`1YFhQ{;E{-<|)enHt${aI~|Rusmm8$~v3Pf_)F?M&3lhV&U&X
zYXn62UU1p5bFwYwkmvcv8^Bk`JBhoKvz&vK^90vA?rmIcoLrnw+4I@#*)MVB@^=WQ
zi+M;yN!^k8C^uQ*gW@zr6ZuNnOLC4%$5pd6C+Tt;@aX;5c&sv4iB-W<R#3J`p;@(F
zGfP`f*Uw;vQKlh}UWRtM)+w#?+S7F`^>mDs%vPJ7Fv&A6FtRk5uDeBNn%--}lg5jT
zX6i@kh-)p^T%#qe^Gf%y{(1d{x~AGcHRZKrb(-{Bjb@k}F%z-uvN~*KWcAe|-i*sA
zMfahWini|P^7_%`^`pz{N0-+Rx8?Pt)%j?3K3biR?v@+fEjPMbZgjWY=x({u-EyP5
z<p}JSJNt9e&(Ym-qr2sxyX8PTcSa8m89g{;XdfIhy61Cr&nI2?e2#7s8Qml@x=Cbo
zlgQ{Mk<m>e&qg<i4D=?E(Szeg4~`o>IL>hN;JDF)<CsSejvL^E<4|@i5<W!^x~Gxv
H_MJ}v2xxQ7

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/aa526aa853333f0bb11804b5243df411452cecd2 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/aa526aa853333f0bb11804b5243df411452cecd2
new file mode 100644
index 0000000000000000000000000000000000000000..c0cc5c469f9ab41222d76349f960610e01d4cda8
GIT binary patch
literal 21331
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
zKwTD74%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#P;0`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O-=*7p|g
zzQ6tPR`_k@>*p^!UZ_1^_LSvG`=h*vH4h^0pT7I?cKWSvH{EWP-|D%gbDQhV)I0a?
z7~E~Rd;f0iy<_*}@87%s>jC#8#>X;GT%L+OJOAv*bJv&5uWDb-eXa4<=-sw=k?(as
zT>BvN@#crgAO3#m`RMww;)BwM#1C^n=zm~&FYzw$-J5sM-kp8d|Nhp83m+vu?f7W=
zk?AAj$Cn=-et7)B<zwq7^)EbMJHGDxmhxlbPqkl7zmEM<{=MaQ+#id-dH+`bZ)Ui~
zu#7>7A?APozrw#ff9C(H{JHGMrSB8Iz53$#`O8P?kJ=w*ymNV*{?`9p!n<d0Bi>DW
z@Acup`?z;M-dulO^=k18@#h<#6+M6UqV(0o*WquR-+p>~_N~O*#y6I4{NK!bbNh|*
z+qZA;y)%CQ_FdUK`FC>fLf<95oB!_o`)MDQKKp$M`x^eO=6l}{wx3)-)qh6(tp0iZ
zXWK9L--3UB{yF&f$KU+FcmCM?b@+Gm-{XI0{wFd{ViIL;X4Yo;z!J}D#QK2c2ulOY
zeP$iz<xF)<2bs<@{ban$V8gKa|IU9Af3N;7`IY=L`iIW<?_bY-+5MUIbMhzUj~m`|
zzW@L3?z{Q#7QOrWuJrxo_wV1Ee)#oa|3`yQJ)h2ey7KA&r?k(@KNo+#_UX?@{f~to
zBHs(X^Lgw2w*Bp*w+r9Ce|zoSo%iM+)IQAm(D;$_(}9ogJ}`dJcyIac-<vD1rCuL?
zweHo~S9f0>es%ek*XzTt54{$6{ov)~7opFkpNT!)@<ib2t*1|)alFud>Ho^$b<^uv
zum8PX|7P~v&+k@$$oM4irSWU=H}CJOekA^4{<H3H%Ks}2N=z%6{aGim8MF7W|6sT0
zIKqC0t(ff_TR*!rhcss$*F0`^p7Y!(+`qX>xL$A`=1k+-#@)>8%Rft?U$9JwL->Y}
zlu(;sf?%8A2|-<<dqUTRc||9P@k>ZaK9@WvHADJ`w609OjG*id*>bs+a);#J%k7gB
zlIxUBl$DZQB$F&tAyY5oF7r})gY*gM2hy9R=SW|dR*@-^VUd-Vn<g)-n6K2Q+@>O=
zdRyh8a+^|t;!B0o3YQdQ6kjPaC|^_VQV~;)Q4LddRE<^Lr7Egsr52=Cqqa`%ks7zU
zy1JRVzj~5-p?Z({WA$7OA<ey-KQvo3Uun$JDAI`0DASm$@k?W^rnQ!y)_Ki3O-;=%
zjW_C()a})Gs70&As)eW>P~}w>P(81jq!z1Yt){Q`T~$l<q_Uq9pW<ry6S57`ev+5P
z--~?^-6e8K*hBcB&<nx+0y+GOd<S`ixOF(Y*mK!lu^wTKWld%&X4Yl?$K=G6$@q<7
zKf?lsg$xH7<Qd;EZfCm8WX2T2D9vF0kL{1@ui78`zomZN@x}U!?`MWjGe4NTKl!%s
z&Ft4(UdO!o`X=k`&bRa5etmQ3wbCn-7lF_EpX_{e^FiBv!TWFSO})o?ui@^oJN37B
z-aL2x-&NNu&o4P%GQVVXN#|16CI8DeE`PqvaYge=@|6=;+^#OX`tz#EHM?td*F>%-
zUk|>XeEr7t$QwIuh~7MXQ{(o|JKyg9x@Yhp>tW<0?#GRf?>%mPQu}oFGqD#}UMjzS
z@p|K%uD4R}l-^By`|3@@n}9cVZzA8Edb8&(>wC5j6(0_Mc>H1N2Zs;FA0$6WemM7G
z^~cvA?|o$X6!eMzbIh0BU(LQ7{#5+!^M~Q@jK3HEw*F=RJNM6`Ke~VK{w@4><=@+X
z>;9Sj>-@Xs&z#>8ztn#w{Mi0|<+l}I)4rB})%$wr%dIcoUoU?3{kHPkt#3EK<$UY<
z>igx|r`V5|-`l<W{U-m-oHtQ#v)@&{kN@!c!;24%AA~;Kd;j%)*oQYCx;{#LTJ@>o
zbLAJcuf|`0e!2PO_?JyzR(xUm%KNS4yYNq!-#7kV`LD*9&oqIVi{&Hpe&+Ygr7Wvi
zQdsO*Vp&*N0+_X#eVJb~ZD9&y_Gg*On!#?zxra-Pho3iy_ZCkt_Xp1D98v7QSwFD!
zvTS3CU_Hg^%KDMHfN3wo)_?Q<Ed6!zNBDR9Z+pHlecttP>xVb*@4bKYe)9X`_kHgp
z-ZQ@6^=`_$$L}iM|9!vYL)ynpACo`s{lNAi=RM>5s&`#)C%pObI^^}@S6r{&zo>ft
z<r&ZOf@cSxvOc@}Z1Hpd7Ykl=yhwbZ{bJSg&(A`hJ%2j)sqxcmPgtIAe(Lya^0S}M
z+@2>tpZomAbGaAWUX;IF{W9R?(-(_h9C#7>lKoZBEBDtk-b{PD@NL%HId31m^?3K@
z-IDj)Kh%5*`||W_^!If?&j0-Ki{%g7U&nuw{-0%tVTxigVw=bQmcx*XfxDgCho_&{
zivO{|Z@~*fLLv^Ld7@uLT0~++N<=1#)QTJw5fzJ+_#>4k>m`3oAxdejvW5zi%0gvX
z<+)1!N<vC%N;b-Rs^;oHG~%^vwV8GP>MYS+s2itis+*^4r01X?Z4hes&(O=r!swi#
zhheROzkZ<JLER!<Z(T9nnL4+%Z)^S3T&@|W$*%cZQ$**eZkOI6y$O1PdcL|cIyba#
zYf5YOXlQG2YUpY7YRG8r)BL76SyN0iOT$e)NKH>IP|ZcnU2U=&v$~^tfO?|(ceQ7#
z9IAyX3zXj~bt{=FRVukFZ%{s@{6+bNa)q*#vYK*>@;2oV<(W#i6*num%g>kVl6xfk
zNTy20P3EZdQfXzGEi#d^!g3;VjIwv6ousRz+$2LJPKYOp-xg~Xy(t_h94fp`XoX;b
zz;C`eymCCPT)Lb`*ag|Pup}_gX7po7U{Ge<$at7BhVdeUJ44w2yMJ5$9Q!@@_wwJ1
zfA9Q#>G!7Ji+)@G-uLs$cY$wDKF|Hg_5SJW`7a+l-~6=h$+1Vd_gnA#KGuF({d~{c
z>0dm4cmCJ;^Yh!p@B4pB|33ET#;=**guZCM|NDaZ#ogzPPuw3rc%*+%|K`bSuP&!w
z$UPf(>h-Z5hqCuc?rGZ1zWd~^wY%bWMeH)#HDjmFPKKR-cg)#2eb?OGZ}%APW82@d
zU;M!GgVIOjj>R2cc4BlOc61<ibRc%{55zKz4#bWQ#OjR>#EuTcjt<0*4#bWQ#18I(
z*!L3e4~-7Q4w8Xbxj#;Sa{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W
z%MH($Jqvg`;W7W?d5;;N#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQ
ze5UfE=4JaU&)3^uuX}CvI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()
zm0t^gul~dSck3VbKQezT{&fBk{VV#9=_!LB122OlgC;{K!y(3vOvjiGFqJbcVfw>t
z!Sahqh}nf@3CjnTo2<XsYB)Y}3UaIQ*zio@QsKyAy~f1L`0#(_|5N|%{yY7v{#WzQ
z<Zt<(u0MW%B>p`3?fhHom*9^hUz5Kq{*?70{(a}W+PBx=#=a|l>-6^MJJt_XAMSpr
z`*`wG(ifGl!C%*WZTvd<%fHX^UlPBZ|8nB%#P6)Xl78F$KJxqIpO`;Ne_a1^{k!ll
zlc9of2GdWLdF&ZnX}re#t^E4@7X0D@kN8vh7VsJI^YDJ)n$5+{`I=3NU6sR-qm`qa
zb1{cC>mx=B2LFFHzvulB{qFNE_*=<0?r)R6UHu{VYr+q?Z|^=Y`H=C(>&4tBJ0707
zmv{T|jr~^(E{0vuyrgp}`{MD7PcDAG<bCDWrOy|eF8{p3d(Gi`;LXl^J02T9ReXN%
zCI3sO7kw{wK6QJ<`$+t8&XW~ScRro*?A!Af&(xnrJr#JS_iWPhjc@jTcKY7(o%g5E
zuaNJ}Up{|P`Tp<cl0R$yR{e8ke8qBr-Iz;)$Bkz-*IJHs9QU~{bGLF&;bjnD=fBF^
z!hcfGO(<6&iBFe*tpKOMbp9lMeO@1~uj~%&S2#U*mT|Xof8?3So5Nervxd8c=N9)L
zj%3c)+=krmIhSzj@vRpq5>^(C61yxhUAjVMsq|kdH>qZcv*O1k<z!lAugImyx5|af
zZjybd@JqE!t3}sZzf}K={$s;c#&1lt&DNQE8;2U*H-2oo%6zh=t@Q;PHTy)zRgTjg
z*4pday|nJN+G<&7(P|-K^}}k0wWRGwTWQ;OR@vqm#w7-(`gZ!&`tS5gbUU;YwPtF?
zX*23}>3bT?(z~wnLn}vpgYpi=tqN-LpJceC=Sk{Gc!~RpzY${+wHI2>@50;8{gX?b
z>o@xiRx9R@|1SM@{ypiJ(=Wq6@BbSAtNG{sPw{W;AOAl^e=7cP{c-%g<yXq@l)o?j
z8UE+|&+t$0kMHk>-%Nkv|D^r-`{&(XhQF77UHQrJtLNAEUkiVq_?yNM%{ZO$9OG>U
zv;S-Vmi}@06aL5T&!<1U|EB$KVbWtwWK&@8V9Q|XW7221^Jmwuvp+xn;P}b>>-;an
z-)_I%f5-oN{(Z_<lP_03eg63N<NlBLK6-rm@bUA<$j=MD{Q1K5b>f%G&+4CQKh}S0
z{^Ib}?u*ptQy*pDpMCxHh2FCzkN-d9c<B4E=&|-Qz87ULwm-Y@c)>%n2d?+c?jO3p
z?!oJa`j1(jK7Ic6mCak%cXsb|-_^a__0Hto!8eOv+q~&~>+nwZy~PK!4~yRKeqaBg
z?xXi7#!v4)tp1?+k>_Lf$0MI#fBo@Q@|)ZD8{cfcto=Csz17>vufM-q{!09{*qib<
z*WYx#Eq)vGw*Bq?*FG=HUkbkd_&VuL=^N%Z6JDpkp80zF>uax8yuAG4>2s;)AD=lq
zzyAEe^UmjtFIK$R{bJjTkQYkN7d^fGIOtLE!-e-N?<U+4yOVRL<?h^jCij2ctGQct
zd*V&48{e-d-C(^ba{I<z(Fa10lpddcRPjjn(aeV`4^18%x&P}y$-~tTE<BKVc;G?7
zL!HOwPuX8cye@v5{_fY?DQ~B}{rJY>P2ijLZ@Awscw7Fi>iv=rn?ALD?fjndBjv~C
z@B6>6{C@G<v~MrJZu*?^S?kNuj~CvVznlK<+=qpq=Y2l;+4kF~Z%W^0eT)8H{KMw=
z;eY&0)-1nS&ap0IGh!FzoXV}vThAXUWGZxv{{<g2zXRV+-Y2|kc+$C^a|Uq7@IK&g
z7t|4)%`d_GoAUtsVz%F`2U*Km(^-<3^cjPhx3ej-&t_fC!p1U@c^;Dl<2ptYW*?^a
zjNh4kSu~j!F$c47vn*r&$CSc!m2nP(=KsRKvwv6r{{4I3Z@zzg%zdmjtQT1ivu$K`
zV~%C?Wk~sd@n6vYJceHkD;Pu=O8!s$=kqV|e=&m+qYsk+)0zMC{w4p@{`>CF(Z4hQ
z82sY=5%N{_i_y0?KbHLd^luu&xBpH5>;Jp{Z~lMef7^d^hMA1F7_TulGi$MJ;PBzT
z$$O1oN^rX1B*7^H{(M(>7<hN`c=0Ud4d=hlpDwUV;3j`LUjpw5o=lz}+!j24dBS-e
z_!tEq3RDYf2~QT$5w#FoE52GnQ}UYRS1BjyEmC`==1VP=JS_QDYK`<>nW=Kd3U!L#
z6dM(%DQYU^Db*@ZR-LKdqq$$pS8I*tRLxZy>FPG>ZR)A&n(FLo6O}s@FDjTRRLCdF
zPm%AG?~vare^y>dfm!jG;vt14a>BA*(x)YzB@aomN}Z7WEm0=HBRN@eljJNZHkpmG
z`{bJy`sID)vSjW{+Dc3jPZXDu5S0v-d?ImL+)8YPh@|iu!770`{u;i6ynMX5Jj&dA
zIW*X2FkfN#@bA$-1%^|MhnaV<=CDm+oyO9`9LpH~|J1+oe>MNu88$JpGwCv|Wtz{_
z$P~rQ$8w1I7;_#=4T~SkYL;y*?kuV-GAv3gD_Htjb=l0>CbKoLzhXPblEj?DRL&^M
zl*_Ec(#Lv*?GoEFwk>Q0tnXO9vKX-VvaDgvWBbVVlf9hdI{Rw253IjfAF{Tx+OzIp
ziDYqO>1Y1Ul+SF%{D)D4v5>*@|HFTu{;B=D@@L}j**|T+_kVr;`S{1U51Q|<y(xdS
z`bFe3|0j<gF+bqGlYMjV^*vWrt^{9JytL@N$(dOv%#NNowD#b;0}%(-AJ}nF^RVYp
zjpIxwGEbD8xOJT6c<Zs{$5fBo9~VA8@%XM24^9@Gsy~%+O5s%4DVx)dr!`KkJh|ee
z^2xRne~vSqm~!&k>8i7D&RJY=zZi09_ocI!{4Wb!cDQul(v!=ZuT);UbR+B5wcAQ}
z#qMpn|Mj8dle}l1FYdmmdKviY{>!&7*1zz2@%H(Y7du}1zn=P5{r%tfYd=<eZv3+U
z>*a5r-vfTc{;c`A_2-_SX+ILaF@F*Gxa*zoTkAI(Z*IR?@+SSY*ejNol`kf|xbY(H
z<)l|^Z&=?Rcsujm%Xj_nCcL})ZtJ_b?-brAd{F&V{yE```Zv`dtUo9H%==~Xd(xj(
z|7;m78ILgDWl;Fv`_JM3um7nG#~An-D;Y(YUND)n++&Sk-@u{FCBr>~yNPEVZ#3Up
zzBv9B0)j%)!e50Ki)<IUC$d#kR;*5xPc%hzrKpssy2vhJ29eDor$vv8d5NzP=a=x7
zNR$YZ=#qFV(JZlDJY0N_n5bB?*kds_@pOrkk`JV;Wt3%|WD{i%$xM*y7nc@^7I5Qz
z%ypGRl_QxmnOmQCCSNUoIKLg=MxIw(0i2E;9_-O<w^*H7=dft8HnW+syRrMQC$Ya~
z-_0S)`J2O<Q;v&=yOa9~_jH~P-c5Y3`8NwN3VI593RVdi@$>VY=278p=2YSM$>zYu
z!8VPxp2eK`FjFtH2umq*IrDCo8rB3>E!I;k6Ij-=*s*jlD>7eX5@AkbdCV%q9?enA
zDZ?ej70j8)!OUUBF_YsPM+>JbR~?rS*B;I?&fgphIW#yJ*~?h}G6ylKGsyl^_-pld
z*<Y=Hzy3Y{@6M3*fB#>dKY71Aew_c-_wCuYh2QV}X#AD@=k8zie|P_i{QLax$^Xd=
zo{Vlx3d{w}?#yD$TFmjxWz2_}&$BFGo6F(AHI;icPa5xc-e5j1zFgh_-V|PW-fSKx
z?yH<vIFdM|IOei{W9wkEW9wjzWZB51%=nO@jBzI86-IXE87#bP{OkhkEo`c6Qfw`3
z`s~-)cXBXs*>g|kQQ*trPZls2ED}5`5GU|a;IyERkhHLg$aN7x(QwhFqJm<Z#T>;q
zh^tE6mbfd~CAC%Ry;QZds?0eVBiU}*53+A$m&^Lg+Q_Vw+Ae7?nJG~s{$DIcTu!1<
zB3+_Ef<e+-^0ow@M7;Ptu~M-=qKTpoqBlfc#AL-@iB^iv6L};oEzBbHNMIhn1D_J_
zZ|+@OYFrLnl3doD2iTXfvM?(#oc(k3hrrjPpX5HOzb|{6@#gC5HLuHFE4)s69r(ub
z?f17C?{eN1yj%Nj&btq972ckD^X$#XH!I&TyeWTO@Y?LP+bfTkb}#CmEqbE#xbva@
zg9G=^-@kDG;eCY%ArCws@IJVDKl*;dy+wC_-1&cd&8_O26K=R&PrjOZIryU1`INH{
zPjj7`azf-p_VN2i-HsR@_C8d0aOZ){1DXfF96Em_^H}zY3#VS4={k4veB_1h3tumo
zUDUZ4esSl;`xhTxynV6y;+YG97fLSNxp4J@>xBd7E6<0V4?chD{Ot=Wmn<%Sy3Bgz
z_~qS~Ph9?ddFGY9S2eExz47E$?Va0qZ{9n9f5OA9k6WL9dcOR%&U=fGg`W<6e*2~M
zYu8u7uXnyI`nuzr&G+Tscm8nw_3d}@---Vo{a?cv&0Naj#>&ffh;2FhcaBJ|AnqUB
z$vnY4i@0mJ8aXRDnYg&Qc5=3JI&$vdkl>iczL33;BaHJcXAu`C_Y!VHp7}fyypwrv
z@oMok@wxKN;XTT0!oONzx*&&eis)(acakfmx67=S{UX;RzgKR7tc>h=nM4_88A%y&
z+2?Y)3Tg@*3fC28DmE*sD}GSeuMnazM}C*wJy{9aBAG|hTG9zpd{XnJZc4wCF_Qfz
zb6e)M%zK${GNH0za`WZ4C}b*LS2R=FujH+~Uiq2wZ{<hIla#kA=_yvp^U6udu9NYT
z*&yv8Z7AI$wM^oi7>C$G(F-Coh53Xk1bq45@jl^p<a)%Z#Z|zylS`lbFE;~kH?Jvg
z4387{U(VGW@$B+!`mEnr%vh6IIa#$?^qCWxEE%i+fBx(I_u?O}KMB9}ew+Sg{q6iq
z`KQ{Crteq2{ru|o^~;w{UmU+o`F!rv!;kYnguc&zSN!(=>)cnnU*^7Kcv<{{^Tn6v
zC!bGy=JMqJL(>Pm_igUk-}AXI^3db4!Bg{RAD*ed(0dv4lJDjH7vEply)1p%@Ur=3
z)5{Gn-@R;nY4$?t`Ne0o&lW!2{Y2%--N#=Z%Rl-5`1xb+CySp%KRNk0?y=5eo5!mj
zn?I?4lK-UbiTYE9XD!dvo-cdB`g-EqOYiS|)cA7yo5Ih!U(A0J{w-$UV0y*0gt>~P
zi?xuAk6n&~flHR>C@%|tu0Wp9TM;*LNy(E^H8Oi-mF0HHo|O40JwZB7`k&MkDGjM3
zlA)6Q5}xA9Vws}zMXn0366O=B77-TZ7W*pJFaA$lSYoj_k9e8bUQuUJ8_^(96H#H&
z0MTcnjAH(x_e7jTr;7%N{ui+ly(j7_wp{dr@F~G80aN}~-b>skxvIJ5a20UP<`m~#
z%vs14&E?Dao_!Tt5bJs7a;6iEI~nDfwlm#l@@HaZn#}ZzDS@en(T&lTQIv5l!=C@$
z|JME0_&fd2iQk`owf@@ji{-b>Z=v5BzY~5p{{Hc+<L9OCyT93e+x+#{m#!~YzwG^L
z{H^!v`7al~)PFhoDc}RsJCipDU%9=kd~W#6{@JT%a?c~4cfMHh>fxKdch&E;Km7Wz
z?jzSH@lOh$=6)*s{QL9e&vl>6KcD|>`^EH2=jS`0ia+iASoI<6{rz_{-c`KIdFT61
z@14avj(0_GKfc-dru5C(*C$@3zg+cv&r`i8?vJA%i9IZQaO%PQ2YmM<?ykJE@y_-;
z7I(OB3*9ogwd9t`t>T+CH`Q((xy63l>9*b-o_qi9t3TZF(CyKQN4$^CA2&S~ee&ap
z>@%0=|DNl=Sn}f9%UiGRzDj-N^eW}myjS;L>A&9p`uZD_cPa1HK6HOL|3UJj>c@K@
zdOs}vp!4zT$Kp>apRRq}@oCZ*_it;y=l{I;JN%z6gE3<Z(;Ajowi#^hY^H1yY`0iX
zvc6}X$R@*ai}MecCXXgxxWF~RO+vZCOd^3I!6HT?kA=&HEriX4?+YCf%oA9^AIq=L
zKa+1c?-QOKJa>5Xd4KT+@ZS&+5qcn`EG#B0Dr_&jNH|&~U-YDyu*40CFB0!0cqEG?
zC8ZWifmRzO$}W}5k`s{IBiAF}E`LjoQ?5~Vx$J*g9=U9}bMkW)FDn@<TPQ~;Z&iM(
zyj(d#*+V&3d8+bC<xJ%dN{vdAO8$yk3QqDi@*m~v6bckH6~4;vlsA<3kmHkmCH+V0
zr{qEjcX4~M4Wi+qheWmsCklNQXysSo>*QI^6~w8)slb`Q(a3JYF2wG|c8=u_Qx2my
z!^wZ?f6o8x_^$SK;-{1kkKZxBJNmZ&?bbK<Uvs><^Ctam*W0yk&%f<_yXDQy*GaGZ
zUv7J`_C?l<^5<>OOrNnl^Lonr<lv(f505{PdvNalo%@mxG9D~?Q2XG*{m=Kx?}gqo
zyZ82P&)tH%)9;quJ%7ji&cEBYZ@1pQdF$#;lbba+@7-K_^WaUM+q3V4-3_@bcUR(W
z)ZM1Lfp@3eeROZd1NBEmkFP#yeb)IR{?&xnc5lwV34A;I?drGd-b%a^e{c5T;|HCO
z5g*@t-22Jlv&ZN3&um|$z7~Ev`u*OIBR{wP>iE6vxAO0ezjps({w?=g^tZ+D;@^vY
zANhUx_x#__e+T{L`Tzg_FNXU}Usz&BYmU*HW3=WNtvN<(j?tQfw3>s>iTwk6KZi6Y
zKc^E%5xXI~DEkGrt!!V}df5v(lsNS`B{?^8-r)MeeT*lG_XzJZ-X**$yt8?nc&vDe
zc;a|wa3A8T;wtAd<=V>`#c9E5!TFHmGy5U74XhhkW;3@l^)p^$2w=GRKkomde?kAg
z{pI{8_;2sut$!2#IsCu*KZs!|gFd4r69=<3^9<%KESFg$*dp2F*uJwWvRSenWwm9s
zW({Ng!ji%g$Rfg;#dd;y4Tm_V0OuACD~@UGJK3DsX0ZKY+s($$R?e!+YRxLk+RGxw
z!ohNi`9IS;#xD%c40rx7`|tI?|KHTV=l^*9nfrUkFOgp-es=tH{`vZc>yI_xSA1Xd
zo#T7Xw<+IVemnoI>YL*?-)~#Ko&LuAP3UX;m&czUed79b{3FAs(odT|?fc~NdC8aL
zZ_+;|{p|T=^ZUc^tUoLMsQ&f&_wWBP#&1kv%sZJoSOQoNu`01Ovt42nV>e`fz}Cn%
zo$VdlV>TPM5Y~w-&zRMj>zRHt)-dWZ8Z&NW`1eoyulOIn-%Ebh{P^?z+jp%WcfL>m
zmh@HatIXGlUmko8{`~6GicdbDR(*{B`1<3jPfVW`KX-jD`f}{6>UW19%s-QVPWgHM
z=bN90e}4M8>X+8<J-^=kZ2EcdXYa56-%)>f|8oD6`hWF*K7$2gB-2*rTP*3UQ&@Mh
zzGXFMD`i{7=Efe+VaVCTd4lr;ryZ9Tw=K^jo-4fh{1XIb3Iqw%3kV556FeXkB798P
zLPS#}U3i+1pHQmMYN5+QkA*yiV?=g|o)^m&uMy7|j~Bl!c0lyBNVZ6(NV*80h_moU
zp|e8Ygk}jz34Ip)Cb(0OUGNM43qDmoTi(UoEu7ix2U)XN)R>u=JQzPRXfr-z)Mn~o
z<YlyDbY=X&@R}i(@iL<ZlO$6=V+x}_lMC~4<}=KjnI|%5GXG`LU|PudnK6mUgXs_B
zZpL=TGDc&@dWN<CCI7end;iz+Z{;7;Ka2ic|Kt0o^|#>f2fwcUdhv_vx9#tRzvcee
z{dw`b<+uIs-M=pVT=t{<d*ZkGUv0m>`7-m%!Y{AC%>A<9^NdgXKH7hr_>uF|laFUV
z8h<kU)b>&PW6{TVA9Fr+f8zaA_c8Bd#z+2-EFU*~T=Qx2XPz(rK5Ktb|628J)AwgT
z-v8|Sb>LUfFQ1>0Kc0Wz_Py=9%=hYV7T=b9v-#fiea82j-(7!9{IT~3-_LzNcmLY<
z+vpGbp9#O+f7}1&`Ca<!)z39Q+kQs<T=7Hrd(PLKFEO8=eBAZn{QEiYJl^hneez}Y
zi})A2U!=cS`ux^2j%Nx_b)VQjKKSUuqjiryKQwvx{(;NGut&Eae|VDnbk);!Ps5&G
zdeZ*n?UUzEXFThA=Jrhf*|n!-PsN`ecry3N=O>j<yPq0Aeg0(9lRuAtKAQGW_QCUe
z5AHs>BX_6ocHr%$xApHN-D$idaQFLN;rl8NL>?wRGI_H9>9l9D&u6{(`104Qy4RCl
zcf8*Dn(dAKn`3X7-z|C1_L2S5txpVJ-hOrYq5G@$cl4iof8_p3{Js2#`LFE1*#8O)
z#SFd-kN<D@U-AF?zwCc||5^Wc{(t=6t-s6v6#mx#wfRTd_ub!Se|!FI(s$P%j6ZdL
zZT<c1Pwn5=f7ktc`rnc<o#`a=EY^7T?Hv0#?{X#ZxbjH}<O{|L#S706IW4+d%tyRc
zyj=X7*alG(5m(`#LcYRkB1c6;M5l-{i+vL{6w?u(EWs@GMCy=KkW{Udl=K^EdD-o9
zUJ9v-PZiH9x+yMEFi<!muPa|Kw?MX9CQRBvN?uY+VwSj<_+hc@qCZ9Uiu@9Z6g?rT
zBIY2bE_O)tkBE=RSK+O~HNq;wIYQF~xdbZ&rt;t8o5dH&_n-F&?{;1{J|=-sAw7|N
zQ3tVZu@W(Dv8|#zL=TAF6aOJzC@v!YMeL;*uXv95I`J#wA`(s#DiXR96D0OaOqXzx
zXc2D|J11HyDkgeR<d#U7=qyon(ObeV1;6m8@-5_X<MQATV*kL}$0EZ#iLsd>l|h_=
z>A%{)4}a(U<^J38NAl03-$H+k|GfTv>38#=i+}z8FJ?$*>}K4+xRUW5LlDDph6#*2
z7z-HBFdY5Q`v1#6ng8+s6&XAjjhN;!&1Ldudcc^=xQ4;y|J1)>e|mpE{1y3&^Vj;H
zg+Dufy8RUSapIf)x2In(f9?Es@cW}5cYlWeV)^y@XW6eSzfS!+@oV3&#$R5)cz<>M
zRR3x7v*@Sk&)q-je;oWV@n`j~kH2((*Zz+G-Ttfj=d>TEzqfx+{66zr&DX9k?Vl%o
zI{xwb2c{3f?_1xszU_H4_4U+OD_<Ua@%4GkbMEJo&ksL8@M7)DX|J}vZg`Xb=KJgC
zuLIt^d$Z+j?z{i*w!OFhpzz_>`}z;dKe~KY`nvhs&+l<RXa9=(E%nFzZ_2-<|H=%r
z86GnfF=jB$Vm`~VidB*=iY<uk8S5+7Cu~d{hdIJIXK*g&WZ;VAQsd&~n#EPiZO?O+
z$D8*&uRotFUmmY6&o8c1oNGCFIG8vDI2<_+a`14j<+#f(!+w$ND%&=;2DS-olh{Pq
z3|QG%7?}H+J~H|<l>d|XyY4shuhbtezwP>}`nBxKi%&m3)V!bZ&hVY_JDYdv??CgH
z*>CmVM7)-L9sauJ_3>9NFBiS|_WbQL<!6%5BA-ou_WYUR^H0yFK9hVV_jKc9k4I7u
z@7?#m*L&yYt+bmaH-28TyUKd`=>^sE%g-J@({je~%)is7XQa-2J@xbC-IHfet~j~y
zr1D9t6St3_Jf3?z={Wm|MJJY>Xg{&;1k1@yC!J1DIeX>&l#6VarLWAsGV98jE0?c^
zU3a-L_eRvs@LRWTIo?jVbN=p`d&}?dcu@Rs-$TwvYL9v!&Umo<{@HsM?{eIoc*pCG
z`W=fqTkh1{{eE}nU8}pLcgyaT-M4zs`f$qQB~RU+pMA0RmEl|I4`)Bwe?9;G_fLc0
zKYla(mHp@P-=3kDVGF}nMt9~_EFo+*9NnBnTvpu5+^$@rT&&#wJh{Bmd`*0U{6+kc
z{8{|3`6~oc1fL6;ifD>Dh-ry;h%XbLDV`xdU))dPpTs7~Ia2qe7s@Kihsn#y-<5x;
z;IH^Y(OkJpg-xwY-BRPD#w4w5oo%}N^;YYzHDEPjF;+0HHQs2FY5Ly8&m_-em&s(4
z3KM73Vl!RyhvwHUm@Q*1R$5eAh1&So-n7-Sdu?NFxzCi-?3l$<%axX!tbSQ1S!Y`I
znsb>~Sv;~_W1(Z3X;7}^s=8HqhU#|>Dcu8x?@iB|>zVl)PcxD?%G7mHt(3bfEiHFZ
z*+cW34y!?-(HsK?O&vK0(Q?6d{<mC**>pH2^IYN!<$owJMdY*OUD=be=Vd0#eNy<K
zU@dPZt0=QXCR#>DvQ@NJBtvwo_<N}hGACqwWSM0*ONxrch(?IXi+>aI7Fo;}!7+*X
z48yGdssC3qu(AARvt--JRQLbYpAA24zRY@E^|<5V?<dP&XuR3=G5WjF&)Z*R-_<_d
zci-~Pj2jZy+^?D65V?Kq?wosB_bndhK7aFE?OE~Dl4pJ|&b$zNrvLcG<KIu6o-TQ)
zc6Y(8n|B%?&VQ`(B;(nO=ZQ~w9$vqzc_;ABi@WRZ$=pr4DRXt-rAL<@UzWedcl*_Y
zq9^;G9Dk(uF!+(g<HPq_Z#=xFck|3`xd(Tj_kLje9`y6}w}y|;UpqWce<Jj_{NdvV
z6;B?%Onwvmw&8=zSA(CQeu?}F{u%SfgJI79E5BWTZT+49SMcxZpQpase_j3E{ny1m
zhyFij3}8CI5dAmo*Q1{wf1UZA`N#3k>0drSPyFEfdH&bIe@~fPIo5FP=6uG%&#l56
z!QIV%l*NEqow<i~HRoYII}ve-a}w!dKLv8Rir7A}d|}dMlwwL}S<SkJRh0EB>wive
zzAd~)+y>lxcv5+OaeA@mu;#MpGoNSp@%QZSo4+i7JN&8s`}Kc3a~;bH#-_iVzbk(A
z{;K`O_v_=2$6vKR?t1&;4bNMacN`xrzC8PK^>ghfi;tQgMBatH?0i!Dc+wO0r#qgp
zysm%0>ZAULqF2`*wckm+zW&n5^ViSGotM41{_@N#T~~E(#@r3N|M7nA1K)>|kG-GV
ze)Rl7(!-8NYaV9bkG|`3XVIO2yRCNx?pr?=cz*Yp_2d2bCGMTL^Y)h2Eymk3ZYkZ^
za5e4n+za*RTP{joVY#~RO2=j2ORF#GUp{f+*_nT*-k;(-{r2?nb8{~Lyit5-!<~X#
zh1b?xOgQ)GjQi=dQyb5uUD$M~|BB1?s$2H=_#QStxO?C3NzN<AH`%XlzH)hc`<?o`
z)Ym^>G`w8+`tzISce<bUe7pNI_*e0dFJG;{@PD5CDf9FDFPz`fzO{Va^jZ4zyDw+H
zd;R+LyZO(RKZ*a+7%UmN880%-WZ1*t!gPR@hucrkNc^Pq1UX^(cd}wKHIkdfVnz3g
z>Wec<Nys|O<;wRcFIK;=?y7!F(?PdGPg!rSE|=aL{bIxE1_^qfb)M^-(Y>ViLto6e
z+TyFVq175wU8C3f`*aU#-_hEpZJ^tryHm$mr&i~p?rsAkV<zJdh6jxHo2Z!WHd|&U
zZ6;!N!Ysu+)k4WS**@2Kwri6si^~_sZhIzM6DuBbLDPMv+~x&l6{a)HR$HvLd|;_$
zWoP}!w%cK+gNA*N-6uN-yEJP}^CiZ?M%qRT46}{)n@zUaW6x`!YjfFhhk37Qpz(jh
zM}{v9O%12$^XMMZ^3=Mc6{3}^QKx32TA<RSJV)`CT${`aDFKP6BCSH6f;|Fk0%!PJ
z`ThBFx%1dFnS~fW{N4Wd!N2+cZ~Y7Vd-~7RzaRhJ`g7$^K7$S`CwnoQA=@LiGWL~h
zvCJL+X8yYH{r#8z&*tCu{=Uhup7{Xt62|%e-u%-4cH(2k`_^~0Z{uF8zF>U1^zqHd
zmmXh!<o@u{egFF&_ml4(y~A+l%I(ryKd!&ODs)x+YQt6OYhKsi+z`1Lb8XQj`3sTf
zCY}j9Q+RIn1;vZ(mlUr?-|)ZLadY48();rsr9NSPy87v|=R01;zm$32{gmmc#S`zx
z>mMI_micD>N4d|hK8k)c{80DS{MGXp%U(WwW%By#OSu=}&mTR%|Kichv#%Dt{`_Xo
zJDv}7KV*OW_Nn%Z)Yq9`et*9HMgP0^&mX_C|8Dr7^*{9Q*WXgVJAYRHi2r%)*Squo
zZ!<bFpJbM0S;FGWx{=kBRg2{<Qv%ZurZ3DMEZ<q;*w1lVa+Psj=Lq9);W)xk&Y{e%
z$TorXIEw*`0Lw>aS(X_rlUXcS^jQ8eH!)i<-)3TADq}p%SjcpesgUU*<59*EMq!41
z{|^2$|NrnG^FOP<F@JP^cm1sTe(h`A*MzT8Upc-7d@uRY^;7=W!(aD*R{og!ed>3!
z?+d<~e%<(;>sR$phaa3jSNz)eOY=w1XVdrHZzsJ~eB1d}>SNirzdu*}RR4D4<BE5N
zZ?az2Jb(1u{^i#fm!3U)!ueF{`SF(*Uax#}?zPX$&S&CJ3!e5oPkOEO;l}5mU*>+X
z_`2e|`md$G&i&Z_b^oUuAL2jE`)K=x|9kz<iND!@pZKc%Ug*uUS4uDRo}GJP`flr|
z;IBKsZT{i?r~9AIpTpm0evSJo@%7P{d0#jG`1Ef*i!Uo56Yu|(e{TIe_r2}=lpi6#
zq<>reeEqfi^Q=$PKVAIv`QwK7uiu<`+5POm6R)Sy&y8N~cysH`^;gqhYQ8%6%Hh@Y
z7uL_7J)ZLD`J?P7HqU0g`1D%uo#~su&!QjgxUYQA{jU36_4|fTa$f1Z%lk0xi}FvM
zU!C8ZJ{N!B`Y`E3>U*B|8K1j<cK*5eN8nG!PsguvpJYF}eLC|c>f4d8KR(-k{QTDE
z&BHf2Z#TSAd|m&-{3++-h)0r-U7s<(E`7)R;qQluAMbrU^=abgq))8xd0yAN*zv;u
z_2zd|J~4e4{w4EM_sgz#Z(g&!p7-+hi*K)4KSh3L|K0honyHcP21h%43CnZlrL42r
zEZFyOZsbiAEEfJQv_?RL?**3~J15&>4tbt$ya9Z5ypy;)Im<a%IZtq{<KD*A#>vI`
zls%u#p8XP6E`Nt`x|oMVl++!Wk8+b0J}6F8G?A~Ay(H(TbX+xCbCNEn0gv8)jmIi;
zl~@%#Wd&u66q;4*HM6w!bo~rw7-bsr=w)c9Yn{?MuRUGIQcuT7$!xXR36nhI0wYU<
z>AG8Vrs=&lJZZehXr_Loj=0ux%{5xmI<IsO>z~(OsB5bIQ&V0`R;Nk7)o6yv5i=3X
zE~~><Mpj=f;?1~>Qgk0`sc7r!Y8kj1-!*w|ywpHSr${|i`IG!nxoE}vs#2OF+Qz!f
z`bqlxb!~sReS7?Q*GHfC0dEW6*uH6av-s`Y_vb&}|6Kc3``hNPonQJs&;FeMh3)J6
zFU!8n`m*4Q$(Ng-cYVqKw(<LsAK||?{x<(p^XK6oslSi^r2Vn@^W{(Mzm@-gFg#>B
z#B!Wfovn)P6x$cJb!?HW49sGT-V7E0J^u0k+w<?qzZw75|8x28$I!vZ#?;Mthrx*<
z_kZobb${>v`TaZVxBu_9-*<n%`+eni_wS@%u|H0HQ~$Q+>x3^}pRT;)em&#)+9z`!
z?RhxmVeUiYhdd7(9t7N%zvq0{;?9j*4{pA@$#eUX7MuDawI%8X>T}e#Ygp^t)_tl|
zueD2koysi*UfHwKTV;(ERwzizDN8OC_UHe@ox?eq<2^?^M<ZK4vmnEgzf6CP{%Zdf
z{QKlj>7T$q=l(SPefrn_@BCjC-*$hR@`3%s#Sc86gT9^m!T6)$i|@zu_t9^!zvg^<
z_CxL0JwJE-+4t`SLn@OFqsTwzKd*lO`4jzb@&ApCD_C1N|8fSgv$Jewxcz6&&#E75
zem?#E`cK?%t{=)@8a{k_d+N=dMfVsU@IOp^)cbhGV~NKz9?yAl=1Kn(r>Bu`eBboH
zv3m3LRme+*7yq8|K6iZ){j%}pnwO_uO?spBZuz?zZy&$1c>eFP+oKl`SszC{+x05y
z?T<H`Uu}4v`{eh7Nq0H!INWQ0WbpLl(<RR)zF>UC^0Mq{;-k%vQlET%{N<7C(+MvV
z-&}vq^fL5m>O;f3hi<;Vap%^hyM7NWAGY3SzVrCTr)yKLzPr+LRrvbU8`p2FxqkB6
zj_YS{ZojQ_*Zbb2`>hZ6KhAwx{OtTQzh{Y0=RIM6a`JJ-lj3KkFJHWV|61-<^$VY8
zUmniBd+1K`eU8WTpXxpP@buoZozI^?J@@F?!_$whJ?45M_k`!!yjQjFgx_s>mH1rx
zNz()6d#?9BJ}7(|_EPlK!{;ZTs6H}%u>IbSy9M`nA7($k`dIVvj7K{jNj-6TcKx~5
z%kI}#-^qUB{`%${>-S?{xxVauCh)@MrQfULuRgyjfBpY;-y8dPTR*6Nvi)@Vqr^v{
z5039!-=)57c(dcp$+tf5OFs&IG5nVM-Qvgb9}j*U{$c&^W7qrOx9V?Yf4cwD`+fiS
zmOmwb`Tn{6Kf*YLrGjk{`)Q75&Kk~i&J<35&YK)>IF4|f<M_-O!o8ncgQtgAhW`!!
zT7k8K%Y+Jq_Y12D=Lxk4T^EuQvi*?z#pHYMkF`Iy|J?lj@0X^}*<WOS-2EfTaEh^n
TS&sEJ>oXRCLcT`Ex_><YV|c9u

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/ca533cd26c7ca6bf69e62351b265ded496fdf1d9 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/ca533cd26c7ca6bf69e62351b265ded496fdf1d9
new file mode 100644
index 0000000000000000000000000000000000000000..bb026584ca28d5de4e7438c9d21e68726cbf521c
GIT binary patch
literal 52741
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#P;0`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O+Vxj#;S
za{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W%MH($Jqvg`;W7W?d5;;N
z#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQe5UfE=4JaU&)3^uuX}Cv
zI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()m0t^gul~dSck3VbKQezT
z{&fBk{VV$K{QujGj?5>SWm%T6__A(f^<>pzdCQc*^n>XOvj@v}mN@ovoR(Z=oYy(R
zI9xc6aFlZ>vn#SqU_H)az#_o%ky(~y2Fqj?3l=?=f6PtH7R<MqSeVKf4>J}ron$Ix
zddPT`v4l~WVc)-l|IGhC{Kx#y>Tk>+o!?zQtG-|R8uvBfYt&bcZvo#+esukm|Ml>P
z+qcJ`cYXAEAMm#DjqRI;H;doSeSiMr{m->uwZCot+WDpL^X$+0U)a9B|FZ1MtS<|`
zn0&eUdDoZxZyUcK`4RqW<8SjnHGdxdk^1}iPud@gKVSaD{#*I~2g5_ALoCNx)!C}p
zPO*JqTgMj3%D^nf=*>{^-{T+uzdiq+{G0J_{XdugeheLqY)svZcNm-)a{t%<Tle?w
zpWnZ;e*6D!`+fKKyWdxScmGcM75n4FH}!98zE1e!_36qx?$<M(uYEG-(VmA>9_BtY
ze#rB%;X%NC`FqZHE$-a7_2A~an>@EK-C1;x;Q{}{#7Di4XFQg8Jmc}4Cug4YKXH1x
z_nE*8o0opCj=%c+s{Hl;*L`p7-);S%`pNdw<&P2{g+4gGZ+(~gw&Bf=Hz(ivyf6JI
z^u_R7?stnH$A3KdarlSzPxoJXzwiIv@~7l4-#@qiM;NEDRIn{#Kh4q1S;LvmnZn7>
zd6VM}#}STm9G^Kuxc75w@bvJ?@W0_-E3j5@nNWf7eqlA?JfRk$>q2ruwgQEGvw6*V
zxANTLN#`>W*e<{=AjN-#cLjF?X9R~UM=M7u$90ZnT$;Rl`KI#U5_l)bB&00lEj&v^
zQPf#9MKo4aO7x=02a!HePqDLN9^!|@jU@UcR!a0r%#tvbh!^`Kyiq7pNK!~bC_!k4
zP?Yd#;oCx{f-M5x0)qTYcy)Qd^9b|o<Feqq$S%YFl5HQGDtk7^Mot5+O`M4w``J`k
z7cr+YWioa${QNJ@Foi*!@h{^yCU53x%nMo4Syfn>SSPXGVx7e5$g05F$9j^jlEZ@Q
zI=3kACEiHBYkV5~F8oUTANelw-Q@G(za}6dyjG+^bg$SAaZ~XI(d8mDM50BQMQ#h9
z5&kJGE>bDHTxhFMp|HM)hG>@<t3;7xjZ~DBwd8B@0<je$^1|6dIzr+?YlNyrZit<h
zV30Z~^+n1=>ZpW<c(16k$U~uW!N>dy_*U@F<C(yHhpU`Ri)#(%L5>FY9JW2I?5x+A
zotT6e*D)+)NM|r%&}GnOFko<JFksMR5Mb<Ka%TR-WXGh!82jJ%Z|v`jKLvl@{ju@K
zwjV)1kN@=isr_Tix3VvdANRjo{^r&z?w2#3e|z@jnc(x^&sIK@f7bDI*ONt$86USk
zj(bw_#NtWSlbcUIKGA*p^y&R)ub+Q>apPs<tGBOQUU$8&e69Uj>2=ua_pg_{75{MO
zqu^($uVvqF{y6bd?04Cpi+?Tu^E31_Y+}5?^qEPKxr4=yEs9-+V=)I0XB+2wPBBhl
z4mS3$tahxwSnjhtVF_W?Wy@qg&M}4aJLe)U9qx-<He7LBXSl9%#dFnj&f>7<kl^^w
z-oSB`Q;_=`w-e6-Zf~wf97XK6SzTF{GPy9GXL!oc!*KV%=l`C6lm0II6a0JePtKni
zKkxjI`9Aq;+ZVUbS3kObIPuQ^o#^}D@7X>wezyG@_bvCE?6>r<Pd{(?81X*h?ef<J
zuL@pGczyWI&$o---~F)R!`JsS-dDcwe1GJ9+4~FcF1`zSzwN!#hwzWrKh6JQ@{Q+v
z-}l-d^*^nDef@Rd_k=&||DO4`{J$?l2E&p6>;ErixXob5u$Cc^Nt}5<Qvp*U(@LiE
zOfpQej3$g4Oy`*Vnb}ziS*Ef0vpi?YW3c{r>d)ri4}Nw3di~4t&;7s8|5^VR|8Mbc
z_aCX>Hb3pZ@A*3Ki`Hkek4*3Hyb*c5;HArpkmnZ9Q=e--?|ruA>9i+-kC`8Bd{BNr
z;9lq5hj(r6+dNqR;O)bgkK>=}JY#(R?)jz{A6{6zP<g)p*@kC2&-Oii^R)C?<8y_V
zvtO~iad>zAL+vM{&uyQxKJ$Hk^r`#Pua8eY&i%CRbJv&qUqrrs_{#m=^!u)FhrjuJ
zZ}`sko#&hCSBtL>-zvXf|8eA()Zg&`B8)E?9hn|7U0|NYBFdV=`i(V{J&yAOmov{6
z-e-Id`7a4@3N9DO5)2nE7YP(m7D*DBCZZzJEwoVJHvd`vDE{|+TKsGHvji9fKk*mx
zPT@YpwV87vM=6ICXAf5|k20SWf0DotL22O%VQb<0LRP{bg<lB!2r~!^2qz1xiJTWP
z73CLW6@M!xAvRT1S2SN_zA%sQGNCXbRv`|dBB2JM--7%?vxQ`Zt_XMu1PDCg|H;?H
zJCi$@YaOQ_hcBBNt32y2*5zzA?1$JhIsLf#xxa8&vCFaTVKHQW$jHYS#2C(`!+e%`
z8cP_fBU>>0Y<4bo2KEQ+^Ei%j%;8AmxWU24#m?Qs-Nf^a*Nk6KV4}ccfs+Cbg5`oj
zf|&x$0#yQ=1ZN066G{{I6>%075t}6@EWS^|L%K$WL*|0Cn#?Qdzfwmf-$?XJ`bwL~
zILQdeI?08|RmvL3lt?K^Zj{iIh!&R??+}-lI4z+i$tbBLc~(+DYL29c<V*=ii4WqJ
z#O8}M3aJX(3asHz;J?Kez{kQT!7IZp!?~S3kBx!#K65&A53?;x2x~gqN;YdYeKtQf
zHrDq{HcWe%ikP$+xBYkc_x;b#-xGg+{<iq*vagw6cYQJVa^v&;&+k8<`jq>j@}0+9
z_BRV(RlnT#;>wGLmz!Q1zuf&I@WuM)`<^{~V(>`if!BTCdwcIR-!{0t_V%_r^Y41x
zn|OcI!`jELPgXyPecJx)`E#C^4`1$l_2KpPx4Iu5eSG<;=gaqRaz9V}s`}IVPnvNb
z^E;L;EN@uOv9hqWu|~1JWvyVZ=d9w&;9}-B=TYN1$feDBjC~0k8{1;mC9M0|mU3ip
z9pIkD^OI)-?-#x+{N4iLf?I_0ggb?mgiVFY1*-&S3swsD2{H*)3QZKcBUmG#&i|D+
zgQt>f0f!vB5L-Q~IqN4DQ<h~+&l&9hAOCaer{fRHA1l6#emnX__{*x#7N0dgo%t~L
zL(<2}kLNy=eOUZK=wtB5zz+uRcD*rp<Nw<1)ytR7ulB#Tef#F^rng((YQ3v|clDj+
zd&T$P-=)3Z`QG~dx_3X`$$q%=arx)4ub$uIey;iT_}AQDe!nb#MgMa8b?4{lpSyl7
z{&Vo(&i}jq*D(Y#6)-PjZf9A*Cc*iP%b9x?S2WjRE*`G695dPV*%!0_;kd_noHK%J
z8}~ZyGn|vzr?ReQ7H68xVD|sWzuf=q3``8Q|2O{&{@4Fc^xw2U(|-5<=KXE=bMChj
zpNl>odmr%b;hUeY7rjn={q+^+tCKJOK0o%n`uWLc-=D@jO?-Oi>BXmBPfeaqcsk{&
z!qc)R_D}kr?s|Ul#rhY|UQB=a?IrW8?=OR2slK`V&h~@fhyNeueB%7F_KU$6$<NH6
z1-}%1IsbY6=c8XLzrFp=`s3fXx!;VxyL|8d#`f*!SDA0>-{yW@_oeWQ_1BJXem^Gu
zdhy5S|6E3S=EKbHEH7B{SpTvvW%FhC=D5Rohx;P06n`JTf&i1?D}i<Vvi$e?KMAx7
zMhdnGunTYsWbiBU_3{4ay}`GM|2}^@|9<{+0{H^Te0zC5bHC(T#Ceco5yvsk6Wk8G
zuX$JT_3{@8OcLx7QWaJasTO@MsxN9K;xD{P=&+EvaH{Ys;S>=TQEt&x5hvjwAxR-I
zp*ezA1^fgi2;>MJ78DV*6}Z5!$bX$Tf~THail>mblb465munVhA;&8=JJ!F<o-A8g
zx!A;56<BN7OxdTgd9kvx%x6|-@n`wYyqKkzt%Ktz=L1eV&Ir!kT<5s2aev}o&Qr+y
znfD>D5^pEZZJt@Y@_cc8WqgbH?(s$OJ>hBRR^+zj&g7oQ&BpVBC!bGDz+A9butD&T
z;BTP<;acINBBA2(k}6VOQmIn=r8K3FO1+fSm0BfzPexakQ}&?D6B!TLCYd=>6C}4v
z_DQl!>Pj+5NlR<Wq{(cT_LH71eM@?Sbh)&fw5;?@sb3NvVzI)P1vc{O@T&7vbH{SG
za3AD)z;TRCjP(HXVx~^UNQQ*}Z~y82zxkhmA@={De@Xw_|3CWY_ix3&%m1G-Y+_LO
zKmG5vKMQ}q`RV$@^LzC77vGD%-})-}we73!SIf^#AIskty<7Em{+rO(lV4@N+WPYK
z^W>-79<6xLdq4l)-aA3JH{H^_rFe7W^`L7vukOEAcEk0S#vQ$T&JR={)jZaH^5_Zc
zv$W?oUR1yGf7A9Z`(w$Mjo-d~zwzVr&kMi0ep~$c|Mv&O1m-I&vskyWy<*?Pah>BZ
z#~#i-+_Jo^ytO<fJiB-%@HlZ>asFm!<%r@~!m*OGo_iv18o!@_j(~?iuHYV_GT}eM
z_e8#m-V{40_CQob^s9)FD6go2=uXk;VoBne;<LnhL??)t3TF$><2U1r=GElQ<SpcF
z<XOk1$!W`Rfvu0_6q5tfW5zZHfq&M&@BWbauJQHeC+&|*KJ5Fj=Y#Ht@b^yd89wlR
zc=zt!+jVb0yv==Q@!sLXrjL_9^L?H6b@kVWUt7Oz`2Oq1p`T^H!hbLNed1Tc&-5SD
zzB_(z{r=>;_>ZI?KYvX9`RwQJpMgIee_s2!``3iu4S!_*sWG%LE@#wWJj5`UVHU$$
z21&+Nrd2HZZ1rp_*;Lts+5fT4WNTpC&Bnl9#U{<l%(92MoVkGM5CiA`i+_Fovj4sG
z$NI0!KZ*b1|IPj#{IltI+waxCbAL1aUh#|P*Xtj_-$T9zfBx`M=A+Mt*!N}cCckZc
z!~0s~<*R2rPdgqPK9YF2;K7XtMGqw(wLXe}^!y>;qd$+fKlXaU_e|yGnb$&ZbKYjW
z&3HTIZSvcFZ}z?U{Wj&j+=ti?TR-%CT>0twr+FU*K9s!=elPdl`@Q%FrH_oCls>t9
zWd7jr-t+y__ftMFer)&{_UYlLHJ@reNqn~aQvLPTx7r_Rzsmo(|4aSP!!VOUoN+p1
z3{wPi9?L$~PWCX)M6T&v^SEwvMRLpXNb`pBDGGQC9S~*^5fE7|VlG-N`d;*&*gf&-
z5_=@VB@`s0B_t%KiY1A}3LOyG!r#a*ETAA5CS)uuCoCiMULaXuqd<z_OTj`Re_>mZ
z^&)K|3L?LSGes;!---5!nTp>KSCP0O{#b0bXurrF;b%gLLd%7MgpUZ{5pEU^5}G04
z&7Z(`ori_nhO31um`j#(H~V=uOSVm{Y^((=D_F`|PqR9+u3|}N$zyS4v1ATmeEa{x
z|9XagMmy#L)=KtE9MYU&97gQrY|GdVvcKVQ;uPR~%TdNr$Ucd!n(YtUclI#OGhCKD
zUAzH&nS4%sLVO!}9e8DUWqFx+C3r9Jp5}|-FXms(&m~|bkS!o6Xd$#$SW2`|^twow
zh@|Kl(R8tN@%<79BqgNIOIAv<NqI=WmtH2FCS5M=Ed5U^TY8Sn9oa0oFS0vi9HgbC
zrb%v<6qJ&Xx+Li*sUl$^W-jtbNMC5HpsZk^KoWlzUpOxZPZrk(4r_J^HXhc0%*xEK
zm|U3;F$=Q@vbZvzVr*mh@IU<jfqx<YKQQcP6lJtyc=A8@fAGKOf6o1`{+;l<?APZX
ztUq3T-|>CXxAHF&KmK~>^w#{%hS#rOuX-c$w&~5IS6g0ecoz6{-4l-|`j7P=8r?s0
zx9yI_?b2HlZx!F(f9KoXzxNy-tbMrRQR3sACo`X}e)i&d$;<z*`rcT+<M{CP<NnVg
zU$egn{4o3Z{AbMXdw(?<<};Qt{bt(0{EkJDO^$sR$7`+vo+~_id0z5N;??I{$;-gA
zjO#yV0_P15BTio~2ks2+7hI8C+FTV}P26$3^8D-qeF8j!41!?-xA=Jk&IoK0G#82&
zIw+JPd|Y_4@Mj@*Av?jt0z3jI`8M+w^6un4&#S?^m3up9F8f^;6(;}xH~&og9rioo
z*Uj%WUyHsx|IGf`>y!2;*H14#9{P~={=vI1@A}?Fy%l=n`&#YwiPxTQ``_DsTJ(kS
zoAtLXU!Q%^`ts>>_?MSozJ3Y*(*IfaGwbKepRRnG^jY@nuWyY%`hUj#lK;K=_xs<+
zf9L&n{oV6h{?C;^Q~nnJTloJGg9T$aL-K#U|KI+#|Kt5XfngfsGR9uUS&Zu#?=!w+
z{L6TOiIv5HC6u{^sf3Axc@}dpvmuiRV+=#a|J;8$|8o92Ge|LL{OAAo`p=QyKYs21
z)%0ub&!8W?-zR+i_WAfH!B5#ASA4kep5=YOyVSQDZ%)37eEIVE(`PZyo;>Y(did##
zXZN0~zbt>b=f#)j)z6<k>v+cey!plBS2N%KdDrlc<K6Xl2j4$=Z}P$BqvfZrPXeDf
zKL&m9{lNKw?LGgygKu}gJ@vNo9sdXUPcJ_o`%>`5@w3LKEgvs`H2&20srl2FkG3Co
zeklHs`(eX}=#Pb;_IzgfYWpqsd;O1^pDw@d{VMux@JH!y&A(uV1g1z9CpJ!w*Bqxf
zj&ihf`g8r`n$E+@KTY7Az!3p+!OMb4LYBh4A{Jsj;>*Q1i~SU}6T2z4O{`z^v2cUX
z8o|#3&H^z4wSqf@WQF$$MG3hHEfCr+oGrRfj8Xit*gdfoVxD3NV!q-xC7PvPN`I4?
zDSJcql5B=-xXceJR;hH!rxJ@Kx+GF07EAn)5SI8LDj{-L$VIS;e=*-yzG(hG{NeoG
zya%}#aj)Sv<$liffNMLKH&+wqLJnJw5DqC0HuiE>C6-#|S<IJM<k)jLZ*etoS9AAr
zIdU%Ic*xPosl(OAmCq&5#mwcvb(m`+_avT`yc_rm_|Ncj3UCX|<bTPR%NNYo!gq-8
zE}sTJtH4izEWrf9TLPc>TlqNzW(qtKcpx}MxK|`mq)}v}sET;B#7)VK(o<xG<=4p9
z$v4R>D9li>QBak?Aj>DKB3mc>MfS0rmcn5LHU&?)Tp3a6cqv0E7pX^*b0i*#sfeZs
zZxRd?Xy&)%H{sXkf5qp^=gs?pyPeC2Q-ec{-HYuC>v>i#HWfBqwp2DLwuLMz%yCSu
zjLi($42F!q8NHc~FfL(u`v2Jffd4W7rvBCWd-@OKpZ;I3e_Z&^{{6()J)es{PJ18p
zF8%HIH(YN|yoq_O`SR7XiYH=^jy;(0!1Y1M{a1IT?s(o3zB%v4?i=AZg>P-Wm4938
zj_;kMI~(pe-*vsW?7rQ@(8o;APQIvro$${2W7p@8U#h-#eSP!w@wZFgSN|yd<@UGe
zzdFNIhJ}nPnc|sESvpx`*rhmsb6Rs9<C@C-hI=CSI<7sO^_-eqD%`R>F}$nzy!hYp
z-Qc~<bC~BYZvelYz#@Tz0;dEn3TzN)7kD6WQSh<Q0bzHMLXndq?xNbFDI(><1wzS!
zD+NyT=kTfUI`On~ujIPMaftOElN4j@f49FXe?tG{{z>}n_4D;N+piv9f<JqGI`OgU
z<DC!R-tTzl@pjuAjklg}7r*g(bM_7IyGQT8d_4O3%NN<NsbA_pPyV#(<Ghb;pR_*n
zeLnQ5;gjtrk5BtQm3+?ovi9ruZ}vYX{k;0?!SCijTmIDl5%_cP_od(3f9C#i{_FP7
z<A2rvBmXY`{r1P{&(q)6fAjs3`MdjH`u}bJbpGx7ck}-<hHZ=?Oy8M`nM0TtGifr(
zGyP;NWIV^9%dq1Amj9Lv8yL<qh%?mutNYXWEAPjxZ*AXZeB=K1_)F~P-5)zYOnqPf
z?&#YIZ$sX`dXx2L*6U5Le!UcU`S|&ZXWq|#Jym;l;MuI_YhGM?x#yMA>jSSkU$(#4
z^djr!kC$Sv#ow-f_u`%3yPIzX-@SUb>wWcy=8ry~BtHdw%=?i1{@}aa@6zApzdiZp
z$D2!U)89LP?E3WU)4@;2KE{2}dcW`;@B6>+^*@-tw|aNyZQ$F$H#1*Pe9iIZ#G974
zF7N8zYkpkx>D=ePU;Mvq`|kGh)2|1A*#28GHZy0k>adxyb+I|JKV{#@@rH9QcN?z^
z|6cyZ{Qvl~1;d0-i)<2g7n>?JTZ~KWqezs<F%fOi3nDrqO~R(aH-)`L4~rcWe=nXR
z&LqBEj8805OhV$kWQEjrsYg=hq_#*cl$s{>QR;&9Y#Djk6|(wr&T>~}b!0tdETs#j
zyrn9nrc0fdvXyR<_LrV4<tdpWK3#OJNPx(0VSnLB;YUKXf=~GO@Covr=dI^G!NbLq
z%w5RU#;M9F$7#T6#W{sTo}HELI%^QyJoY-yhg_4ng?SWs9&n5Eyy21MYv)f8U=oPt
z_vMe|H|0;{XAyWL@JMi;&;}tJp|yg{f{X%N_~!Dy<SF6F=6S;t!#k1pFK-K90^cg$
z6yE>5H~D7r?-Ixm@)ThbQxI<#caxBk;*lwmT_Bete?dM^;j%)Jg1thc!dLlpxtlVa
zGX65Xvh4CA3Wf@|<QB@5NwG;diB=2i3x5)NDcB_7$zR0B!FQ3jgEySlpZ68dJ?_g~
z&Ri=wmv9uYtFrB7S<TGP+{kp2(UZ}iv4&BNk(*)GKY_nTe=Gg|`|HLp(cjO0+x_YI
zz42G>uVueDevABG@=NPi?$2l6HNUO=^7~WpN4^h!?{2-8e>MGO(aWZnzAqm?uYG#^
z(b@Zl?}*>Ndh7Tt>svZEGp?sy<+_r1`TnKZm%d-RclqNL%WKWom)v-LGwt^3JBRN|
z-Cy^h^wG>GYR~Vyw0gt$Zq@s1AO3wf`629M`zNU{2H%{2y!pA|cjVuwe;fY4VdP?2
z$I8Y&lYJe#6~{FWHqK?7%eY=}#d7W8tl?DV+QjX~Ys6Q;_lEZ>&rNO@ZZYni+zPzL
zd`^5nc|Y-%^3CFB7i1E;D-<DeT+~p^Qmj&Jf>^Csm6(7Si|B3PcS6QOm4Z0}Qv92E
ztGQJ;7O@;=ocllS-<7`}e|P`U{&V?P_mAIS7k*a#wCUro4=wN4zUzCp=WWKDt*>Xi
zE`BZhI`UQi%chsCukGI!y%+j;^`qz~vrlh77JXdwanGkKp9{Vm|8nOG@7GUXb-sK2
zX#Bb7SH*9WKYD+6{Wbij^?$<uHUAd=z4>SV@1?)gf64u-{3Y?b@Av-SnZMJ1&H2gr
z>-?|g-?RUS{B!?rz_5&AD?>7)J+lF80-FGP9J>?yX||)R)+{fX?3gkb@BH`v*YtPQ
z-^73a{$2QY>aW_LPd}%AU-Q-XOYvu+&oZAby)Sty`9|RN!<Wo2+g_Bu;C{LF<&&4n
zuXeop`D*<uhF3N(7ru~tvH!W@3$d3$Fa2L`d+G7A`MKt^OHUi0);!IBHt%`Qi)AnS
zUu}JL>DAKL&)-PCyZ`R}yOZyv-aEXX|4!;%_PaIjzP+t^Bl&vXD~s1mZ#>^Hy!rB~
z@TK<)zUT5!WuAyX6?iuL>E|aWpPYHp{*?2%#S5{QF|W?Ne*NahTe0`1A7VdteLD16
z^Xr>$aX;h!u>HTx;K4MHIhkb%OFqkAW;2$xtjX+l9Pc^gxfr?sa82iG;8Nn=%#+UN
z$3KJr7k|3IEP)OI4FLv$27$){9s*POxdj>oRfPRSx<z<JtcBTyo(s$r_$F{b@Vn4E
zVMmeoBEDkD;u;dck`+>6(vxIF<pSkv6`m{pR*F?NQ(mUTrF2>Gk>W?ii;4+~TNES}
zY~&Zqn#(+rekQ#^`hv8u%r<Fx>A6w`l7<rI5|bnjh&zg%7LgI|5lj;>;IH7b=5H4G
zC-7Q8PEc7$Q>04li+G7dp@hD~W$_&GLt^z}_Tr2Z(c<?-_lU5FI0{b`k`ekXSR)iK
zyj!?f*g=?6_>kaoeha>Byy3h-yfS<{`Fi=T^2rHg3+&?4=3UIo$A6svDSxxTZ9zj}
z0g>k-{i1V4t;94Wyd=XV_KKgB(2?|!kd-KqxFmi_?478dh?NkZkiD>!NTf)*$X($=
zp%ejgzC%1oJUKj!Jny)~IUleouv)TQWB$!NgN2=yo%ILHDV7M9AI$m8=}gBMotaKD
z9c4Pnw2^5iqcel?|Neg)|2q7+{j2)dt6$H4x%}e#mHw;z*Rh{xe<=L8{C)BFmhW-j
z6~8NdkN*DdTlZI=FLOWd{uK7{&HJD4PQBB3|MA_lx3^zkfBECtna8OQ&)v7V|KtAZ
z2Vd{=-1oe{|Ne^m&+hj=`1N4v!~Kt19;-i@_B7-9ju$syvcG=&ddr(XZ!f&x@^Rwl
zH(y`;SpM7Rui-!L|FR5bjH*okm~vUBuz7K;<5<ZtmvaJl6>lnE7GDA12fjaik9qre
zgn8t7ba`@l6nMYze&^E@_$zQzKuI8*e+%DDUK8FhUMoH^el7mld_sKcd^h+e^Ggaa
z3T6ux3SScr7ZDI`6<sOXD7s!GU$|Ikk)V%29$ypBH7*a%gY2JJOPDJeSpT*Bark}f
zr^-*KpQ1k>d>8nZ_=V^5(~s#N;@<6f6Z9tK&G*-4ukXK-c^&uq>8r;tt6w_3TJzfK
z?Y?&#-amf-?ETsI*6%ak@w|(9ckErmyTG?@Z*tyDeEZ<t*Y`U=1b=+_QS#Hwk18Lt
zKXQGF{*?Kt|I@=y>7T!Uj{cJO#rKQwm$jcyd~*Bv;{BHQ+djyBO8PwO%bl;+zKMO`
z_-*yK=ik|WasTQ2d;cE;!zzY0hBf~W{nPx{|F`E){_hjNX8vmW<^Su`&v!poe(U(W
z_Cxl&+Bf#EO<p&?e)?+B%Y83qyg2+~^$XV*3@?ISJbV%VQuEc6*J^Jq-?hAxd)N4u
z@9ow%xo`a6l)bt1Cj9Nvw;pdD-iW;}el_P6_iLBepI;w*d+WW<N3~BYKkxgx_Ivja
z@gH{IAAjroR{u5d%dO86Uvj@_eKGqS|0(g)iBA_l`F$$+xbOq#2bcGv@5JA-y}kVQ
z$h*7mcD-Hy=I@(>Z#TV-e=G5B?fc}9`#;V4T>ORk+l%k4KXZPX{@U`p<8Q_PDn@hW
z=`31opV)75xN$6Jx8aE3{LeXuYbEyso<F<~`E2<&@~a9S5V|I`RghgURB*PSq@a_4
zw7^jTd%;@*Klsb}Kk}OinhVtmO&6*c>K2p};N>^w+reALcZ#1&-~qpd;8x)T(J!Jc
zVq3)xB%`I?OBG1B$r#I>mw%wJQ}K(Ey-K-Cwz7hfyrP-{zx)|l0htey+7bccTH@2i
z6hvPMpAa?@))h?Q59R;HuPX3~e<FVd|9}430!9LJ`4t6t1+4`y2-FGe5|}DDONdc;
zoe;Z_i(sZeB)=ZtV%`Y8E`9-lT>dY-b9mx-9`R`MR`WjRQxv={R3%&_+#$>)Vj;3Y
z_?1ww&?CWPf?`79Lh?da1=kDu3Tg;+^40TR;+@8~o^K1^W&Y)Y7Q*o&MWXk_&PW(a
zU6O2+_$96=@krcQ{DoMfc$LI1iF@MT#4d@kihmHx6LS&WB5Wu0T3{c)Apbu;JN}9M
zHvAR5^4upl-8t(xCb7q|b8)D0KIJ^hS;qO2a}(E9E_<#MoR2x3x#YP*ITvx1arkj;
zV_(W9#u~xw&*aV2&t%WEknsW|19LlbERzo7B*sc6ZKfLxGykjqpYi|X|K5KG|J?l@
z`TO-x<sSmymwk)-_T?+<*Qn2_A6LBBe7EQg+v`&=g<guhP<b}*anwVD2YwIcKKOK>
z<-X$m>GuWi%iJ@%H|zfDhxZ?Ed#dw%(~F8%dT-j^&U<&@y~#(h&+1=yeari<`s2;d
z2fyR~y#Af`XT@Lpe?R|9|1<c%m7$x_mT?oq3<hn6qyO~&=Kb0IXU<>Ge@*{9|En+_
zVB%w*&Gepe8-wWodw=8pru;qmcjmv2|7ZS-{h#o!`+p(hd#1O{46Igc+3dL-?3^8(
zleo5Wf8<u@KEo-_p~05H@`I_BQI^5!|E7ONe=C1I`=0d8@#~t;Cq7Dl@OoeNUhTd0
z+t$}yZ`|K5c+>TY`(?t5Y0s^n&3ba<@!=<l&n%yJJU4%#{c_*SXD=_jocuEJ<;0g(
zuRgqre0}rvqBpnRoPMqFiuL7>7q?#CeHHY2<?AhPzP$B+r~YomyBF^*K8k&s{mK4w
z)|abaS-;=>UjF0J5AL5Ye=Pms`-A_-{vQi}RsC`JtNt&7!G$TANrs7mxs7ELt23Je
z8xPwLR!3GpW(UT4hI)qo|JVPE`TOxt%->0WKm1|)6Z*U5m&4CJ-(|j4d}024`Qzaa
z#qYnpje9HmZr{5nZ%f|XdCmAH_08`$``=D{*Y-Z(L+pps?|t9%zkm4d%DeOLD&DPn
z%k!4??d3P?UoU<2?$wdkdtW=e_I!Qiwc8t`H#=U7ywQ4B@<I6Hgb(o_Tt9TZ4|vD-
zw*1ZKH#^_%f1CKW>g|KK4DS}d-SH;&wZ^ORmu4?Nzu5Z1`sKG*H{P_qO?^B0t@OK=
zcgNqczMuJC^rPu#{jXf#U49n-=KX8@&+)$*gFoX{rXCh<wsdwGP6_S~9wFXDUI{)s
z{^<g>f^!9b33dvzi9Qp_5HS~>D0WWlvS^fufUu{~M!}~7(*z0y_X{NnYY8t9DiS&?
zq$?~Wd_-u9P^i!?K}W$p0S$pB0XxCPg1ZDi3T_k96geV#T5PKLM~M_EQE5TxLsF$u
z$x?Pw!IJ01cZem5DT_@My(;oa_^42}pryclK1bdlp4U9bc~f}j@EqlF<z?cH;4$Pm
z#q)r-lJ7mQBCihbeBLtNeLRjln|WCHr1^dMr}4iOs1!02HWd~Y{x5V-Xq8Z>&@!R7
zLbbxpBEF)6qD3N8MC?TSMOTad7WEa|C{`fuFY!ZSx#Tpdd}(%>N|{qKA7w;kBV?mx
zugbW|M9Zv^nI@AevsT7e_Jk~_oP!*@{BeafN{f^eRP@z?HPkd4G?LZJ)y}E@Q8}P|
zM(LW8k@90DW~B_pSj9rcR|*yKOJzSu+euBAkQcuzY9d-BdO(y>v{YDE$V;$M-~xX*
ze>lIAKwzDqxR8+$gV07nU%_Ppef&;*0lZguym<n-Rk`Fj%{WBZcd&k7p3YeMzwe*^
zKi0puf5-p1^|$q3{J)HU2mc-a_vNq3pC3O>eq8+K`*p|X+D|V(Zv4pnvGu*hyYFv*
zycK?1`TF_Gju&drmpx^9(*7v#Va<bx`={@Iyq$jQ+fBDy<+pln>D=bJGxg5BI|g?f
z?%ux}d+*pi`TO_o|9Zgvi1D$^6PKrA&(1&l@!a(#^Q+odb6;z`HF~%0UF3V+57$1(
ze7yN#@`t}4dOo^-toWexA@Rf95BeWi-b=g-eD~(vvv+6T^}oOM;lf9WPdh%Eeq{Q{
z`0?e3haVn)aQWE!N&O4Y*N(3{zoq<`_*3my)30N{lz(se9rwrLZ{EMv|C<?ZF)U*c
zVu<<Q|F7^b&!73fDt|8faq0VnZ?C>Me*W@N`lI%T8Sh-)roZ)nm+<b{+lY75-g|vG
z@ILO{k2lv}SG`*NLj3v0XGPDSy(oP(F);j%^V?5v&%Tv-+xW)vjsKf@Z*IR)e*5<A
zy?4g%-@Yq*C;v|FUFf@{ck|z!e?RS`(r3RfVPC_))qL;!!4;wY^XE^tpVdFF|7`o^
z{#)?R&p!wM{`i~!_s$=izYhP7{(JoI%>P8jNlc>5&CJ>?A6Vj9jaVPB9ARl-xzDV_
zyqu|y=^)d2rk{*=8EhCf|KIs9;_ubpCBKq?M*q<H{{8E@FS|dpeop?R{Bgs3&iDV{
z-F-Lz-J*A2-<7_<{QmuW(+|Hs?Eh%+spr#~Pgg$u|CIK5`RC%#*FOFEsQ<C>L*#qG
zcRp{u-?qP9^mgIf_iwMgyYt@sgW88#9~wV$emd~+-3P`G8t*ON{d;rewbbjwuhzXf
z`|9qi!>=yC@_K#v^`X}SuOGae{37(Z^fR%iTb>9!z4i3zGmaO!Fa2K`yl#3u>-E3a
z>)*_N`}y7K4;h~XzBGO<{^tFC)sMtq%zxJXP5FO?L5XQ4vp?%3He>c4_8;u_97ovC
zuobg?W9w&^=8)#B<C@3K&U2nSh5I*G3D*nG!<=bc+qj!~efehz^b3{=aR}cKk`ihY
zOb~1nJRztnbWiBIFt6wYF@6at$>);iq-IF}kk*x{ml2e`AzLoDQtpu4d%1mbLUNt5
ziLz3%i)4~zDrD+q++|)$Z;(DA{XlxN^c?By(ke1VGAy#va?|8x74wz)l-pE<RBx+1
zRBlr$P<*LyTH%s{jN&Ur2IXtYT`FR#F{)vzj;gV$yHrKhtki<kYSh-LJyPRVS64Sv
z_g7C+FI4YQf2^LXA*8ui^M_`O<|~a^8bum08f6-DHGXNV)wI^q(>kwNr>UvgrSV36
zlDfV64z*~tShWzf1FF2L0;=a#lhk6>tkv|@zN>1fo>cZz;!|8Le?qoF+E4Pb_<OMr
zqPs*c33~`16nY`JUm%BHk?$an5VsCz7ke(-E7l{dv8>4~#mu_Q|CpSZG8w-y>}OcO
zu#n*(gFNFK#_dd(nar3%7^NA^|FQj1{Z;#8|F_hyJHA+d@%_y3Y32u$_b1;LzM1`c
z%j=joU*BZC-T8L@+plj9y;gc<@*?nA|C61MZa!$cFL?jWy{Y#&?={>#cBlUK&YS11
z|GVmX<@qJYOXioXF6mtAy5xWP#^ukKIj(44NxpL8irdwNSASkLxn_5*?wZK;<m<uL
zlds>n9(iNO4bhvYZ))7$dFR{RU-t|iWIc?0#QnJO@x8~bPimjeekS(f%1h<fFJ5na
z)Ad&BozlB$Z(qG>coXo(?oH&IQ*ZXXWqr@~q2j~A505`g{owGy_=DsJ$q(l~tp51=
z<Gqh8pMpN|e~$UG`>WY^!=H-3ef}`~o$>eL-`2nEf9L*L^hfva-M@wZuKauZZ{0t$
zf1Q8V{F(DR;+Oi*gdf|#ul%;+YueZHuX<k(eYy3;`|HK8zTZ}UyY=nnx14WXUwyw^
z`xN`}@_V~?zu)A)ne!&<ZT7pW_wgTIe|Yhs@q^HZd+)!#5Bu=uL)S-%Ppdvve6IYW
z_SN|7&o4K>9RITE%Ze{-UwOZkd>8)d^83c$EC1CP^O+_vbFqA6-p~A=xs+u!OA3n}
zODqcuO8~PrvoG^&rY%fi%>FD>Su@z}IQMXg@$mBo@!sO;<^I4qog<3<H|qzMUY2bv
z5v-?JU0FXe7clK**!pkYpQXP}{s{kW|836~rq8=RZvF7){k`{(-cNpC{J!sf#Cyi~
zyWUNC_xN4K`@ip(d`SDa>0|Q8y&u><<h*BmU-ho*?SwZ!UWdG1{EF+<`xjNuzdYl4
zUhwSTQ`Tp9pDljw|6;+5ju(k9v|p@x{`pzRv*%CeJ~e)N?Fq}%%}*VlO@8+CncMT^
z=X0OmcrN#1+l%s-t6v7ZeEMSXivur0U$VdIdFB3k#+zwx7rxDUJLm1Aw;u1_yj${q
z`-hrOVPBqpjsCvw$N8UMezE*v`|J2`(*LszF-%b`Mr`xg-*OmoF>tqY`|$MhTJb*?
z_$_!rNJzv%G*9%4NQ+3UNQuZqky??1BBElE5`U!fWWD5%DMTr)Rn|~pQdy`ht2|f9
zUr9(wP02=CPt{!gheo`Xtv0jHU!5ho3w7gkO?C5hjr1J!qYXk0{~3B2Ss0x&^f0V7
z@YfI2JE&Wv>#ZxMJ5%Sj_HC`7n#(oAG}$$OYl`R`)$P(dq&GoNP|sIaM(2jsZB1#-
z9t~{`P7OVcUJV(|eVX4iCu@pnW@)&o2dU|)1**BIxvNc9V^()m4^U53|E~5-l|!{q
zWr6ZrrEVotrAj4t<qgV*l)osyP_9sxQdU!LQQoE;qC8XSw&G?5clr5pU2>0PAIVh7
zxXB!qUMj6DvqdISR#;9%j#2iGw3Bp|l$&IT#0l|4@!MjpqIZP@g+qn639S$;5cthE
zhgXiLl}nfN2)iKL7M29&*^GV+2@J}N8yOEX#xP!FaAyenfA??8pJTu0{$Boj@$a3#
zFa6&1d(m&}-}`=E`7ZG7$>+Hrx!ym0J^$r{=bN9_JvsI$_kQbr-^bcdtDo<AJN=8t
z@6P`^e|~<O_<jFR>EFlx-1s%~o6r~S_kUk7zqtFn@rnE62aojc>EAqg?bYS<3%O_G
zPQ5<1<52cK$vsWG*>|7ZwRTtBu83VmyJqav*~zf;?~XY;r|+7(`|TdXeQf(%_KP1_
zeo*>|+_AXh%TBnQT75d=Ozhdd^X3<BpRGKTc`o|GtqW%_<zC-&C-dQpr*~exdY|(}
z`bYlnrT@+`Okpf!a$%`qGvS=Z<Ig`?pje=kKZZYEKuBN@-!Hzy{Nen}{5t$o_?Gd?
z@#^t>;p*Z#%9YABku!$#FJ}k0D&J87IpJWjIg*p4U1h(^rOHRkuaTcBw^Z6!f=Tp|
zU_Ae7o|PO=m{a~M{m%a$@OAzdvo9=Pj(lPM=KJmL*Ja=Ae|Y|Q_x;Atnm_q}4gUW4
zv+u9!|JVNl{%-x#^XJ#^C%?^q@Bf+etM|{#KU4p-{FV6souQa1lW7xU4<jqnQ6@(w
zA%=bbuKv3OTCB-v%Jhr5i`|9m9k&GEH2#zPPx*KAYx4i)YvNnWlgsJO`jqkgf3CkC
zKYoAS^+EA1_p7BZ44+Sa()mc?(dmctAH?2|zn5_L<ed|D3h(CM*Lc+a_{`&HPaZ$r
z_N4yN{D+?(-hIgNF#f@%`&IYO-toWl?arLLNA7mr-F3(E_LQ64H^1G|xqtPs(2Fx~
zG(S!FKI_kNh6&8ySm&~TW3OSaVq;=m$Q;V_k-_Evo<9=56n<v@EdC|<SD4`^;~8dk
z)<TwO<|O9dOgk8#Ff3*eW;A9}VwPmN$?D9$l--8o2gh5ET^!XM=h$Re-Y^<6==`hu
zZSuq8tHzffUv7V^{$cx*=a;~5o!`B`s()JkDE)cy&$_=m|M>l7`y2dc-hU^iPyZ+X
zpT{K0%)&B}BZM=WwU_lN=X1_*R#z5fj$}?%c0Ep8zFywh94#CT{BdF}qIU$v#Z+ZN
zCD|peD{!kNDfua9Y3b_vX|K^<r+?a5!j#42sbPk`yWVcyGg>P((zGY)-qV_>xmnl2
zaF+fzomyQ{y$l@=T~EVv#zDqB#utr4O;?y*HVHDQ)d|=5q4-mpSF%;&wfGuQbK%`W
z%0g~@_c-~ubb0UcF6Uw7t>)d$<HUQ3Zzta$o^PD#Y<F0fa`<!Ea=qqu<xdst5}YTf
zDXb<sRWwILTG(9Zgy3hPU@>ONZ4z(9_en5F?UnYCtCqKx=TexhD5WH&q^BICYOOA&
z0XhyKO#7ntJ{=BSK3y?AH@&$!kvcPU7i#}dTcg6O@>FrHLXi@SO0-J4nxEz!O+`&1
z&9CYas_M$;6wb@$%AS<JtH7=(tNc-Asq#OC!*U;`tt8HgI1BX(Oy;-XtKeSEzM6SA
z!;Js){%!oj_9y)h^KZeQbw7T8+wj@+<HQgDKCJ)v{A1<Eg&)qnfAn7b!}fR5ZxmjZ
zKka*HchB}V!_B`pmfT9d-Eb@VrtXbT*95Pxy-|3J?~dBtb$60)Z@)F~*0!4uuf4iF
z?ec;v%vZcG{kmv*MdEtVb%|@ouQpsQxzc<k^=ir0l~=egt6Z!;|Mtw?Q~qaI&xf3!
zanASLoU=k_o}5TJGUed&1J(z452PKOe|Y<`=u?4bZ=JWjc;|fC+1xWSXKtUKa3<os
z{gvh$g16q@;=QAFXX+h|dvouveQ5JU?kU5wu$K$pB)x5YlmA-#wZof9@0NYc{37;!
z^)Ksx?-|}RaWXGt+{w6^Ns;L(gC7GggCv6{Lnp%_#*IwJm<}+NGc95I!)(Fwi%E#t
zg=GoL2bP<xzu0OxK5`0jtMS<IOyN@D$YQ<5#LM{bf93yE|Ly)e{j2_0^UvgO`Jb*o
zet#tXJoxSWTkDtLk0W1`zbyWg^&$R!=eyds*WbpzD}L+r_UJp-4^<!TeyIC+@>9|m
zm9N2H*L-dKI{C}L&+=aqznuSa;_JlktiO_e+x<TB`{bXPKT3aG|8o7i@Gq00f^i1Z
zPnLP?8C+?+#{8}P`urCB;sTHOQ~4I~8S(S*e&CwT#m)JeO^RKW!;qtuqnvXwhc)XX
zMhgc2e>T78{Sf``^DX#W$v5tAlfGU3A@^&-54mseJ}>!@@y6@L+$TF8p1GHI`|*wa
zR|_tNUC_Lwb1D1c@rzF`e!k>=<=3Up7n?5syuy3U;d<cB&U-r^8$VTie()v#OQ#oo
zFLpk4d&K)l{Bh2c6;F3Qo$>73^B2$5pGG|uc&7Jk(({dP_I`Hy-twLIr_isE@6BI6
ze^L4V@8^;~YyMXKb7p+Sa)8~KOM=IZXEoPaj&&UOxh`|Ja!=u95MbxO%G<(!QqWB(
zS0IT`mw&AQr@(anBz}EfAFi+L4(wMrJ$ROJw{m~vnaG>NThFtGyM^Z#_aBaA&ez<A
z-0wM;aO?4{7bp@|7L5|SEHPcWLT0J-Unw`KW{I=n$0g-tT4k@urO3C+h0AV|eW>tD
zwM?r;*IK_+|BL=(!&Sy_Otj6`nR*+C8r?U3Y`V&PvZbx{1sgT{M8{Q*(;e2@>)XAw
z?zP%#S!mH}Az}5yYKFC>?MGW_+jmyk<{8E%2B!LU`qldH^h$I)v=g;vYQ<?Y>UQaS
z8qCtWuJc1HM}33x4#lksYVx0CxTNPv>PdKs`-;C2V-mF&TF&pn+s^%yOP%XC`wdnr
z=8yj_{dWF6>6g<l!$0r;8vm>L=lxIdZ|oocKSh5k{&4+q{JrH@%I}oFFa8<+=lsv`
zPw$WK?}pz@f8zh7{rUUn-Cu^kmwsLO$?>b_*Y{rwf1mi9#t_Xoo$(ywZ3eUdYyX!1
zarhJd$L-IjKfM2@{cmB?V@+gJVDDhdVCiGhXSnlc*RQibKmOqO$^7g5FT>w%zukYw
z|9bv?%2$&wS3Z6I`1a%ekM}-$eERV5^T)`~3%>mM!u56Hm&(uTpK3qWe`@~X@YU{%
z)aO$lW#6BD{q=?3vn7xJKje7m`>^P-_A|a0WiPfryYP6yL$e32_s#Afy1(wh>xcS}
zS)M+9{`Hm3Ti17X?{weQz1#K9<lVtHi(lKk>3r+(PWQdV2eS{0-tT^2|Do=q_b0|r
z?>?;lp!t#KWA?`*pI?9d@m2Dh+xHvaY`(1hIQ_lV+sUuLzgqrE{I%Gd@;BGtbiFNp
z8}qjP?f%z3FUwyFzW(?+=}qYy<~I{wr@x;0di(2ZuU5Rg{Nm|zsplV`IXu7q{K50i
z=Zr5_yx9F>+l!DFO3xQPz5O`oQSifs_bcxv+!4EzbEoC*+<PYXf8DFOTXuWmO|2W>
zuP5DLy*YZQ%jlslqldbD8a>oy^iY@4LtRD>bs0U>W%N*&(L-HE4|QP}J=A6NP?ym|
zT}BUe0UzozI`TF;@-{m1HahY~)sZ)m+c)ltJ`j4O^!W6nibuMSW<FGTX!795{a+7C
z9<F|H;Q=9enTH1+6g<><Z2pw}g~aRPx9RVGy`A!Q+S`wBEZzjZS^tLn?Si-E@2cJ}
z`LO9z+t<$TDL+zvT>ifQ`^xVZzfJr0^6RG0DWA2z9Q}CVo%y@z@6LT#_<7#vlb>zB
zefp;KZPvHw@5MiCejonF&t%Q=o8=tqGBzW2QO>E{>b&**kwT_IxA<T1G4ng{?c{yJ
zyM`y7>p5otcMR_X{&qnf!P)#0yuUdQurFr&&3cfvoHd;#iAkR^n0Y&!BKvIC)hui*
z6Pf2RNieQsG-38(de8Ws*_TC=c@c9k3pdL$=6_5nOjjA_FlhcS{5$)1_3z)m_x<Mk
z$H&~qYQuVw^)TB;RyXEYMqh@M{}=xS{m*0g#jt`wgrVgB#D6~j68{%7C^7ml2{4`c
zKkr}iKkdKo{v7=~^N+zV&L1IPRlgX0d-G$-?@#}xF?{>q^uPYU>;LBeNB+0{H)oj1
zc#H8GV>7cB+XfCF?wh>V_@xA=3r-T8BH+(=g@=K6Cyy7;Qr>XTkqQF41a9(|^Cj?}
z;K}6q!EM3wmnWRpfsaw(p+L2umhfZ|9Z?Ihwc@KKG$pS|ewA{P-XgU}YQEG`$-|Og
zrPfIAm6<AMtWc-;O|emNnxdvso>HyyWYwAKJ(~Nqe6`kSPSsqck*;o|-lm?auBpzh
zHc`1l@uGs6LWO*y{1o|4`40K5@@M6R6qpr{DIQW-A}1`{C4E}bS@Mu1tJDd}-x6gK
zJd%?oH%ZQtVw2e@yHCDZp<mutE=%UVq^-mh@kDVc2~o*V$tM!0#jV6vh)4>r5v&r3
z<FDa6$jir@%cIP_mqUYX2J;n$5C0zhQ(!p7c$j$yYYy8K)@dv~%(0B&|4;oZ|5x*m
zonaFrJCiQcTBi9-jZ9I@d@P5Uk1^-5)Uf!mtY+E9;?APVBEzD@vVx_bRhP}2Z8BQ}
z`zy9{EJ@5cOy!KCOu5WDEPbq3*e<a>W81=3!1|8mD~kb(FUuO%JhqQ)KiSJUuCuRZ
z`@s5(^&x93t3B%umPi&imVV~nO!>@K%zqd)7z-IZ|3Ccq>7UxaD}N^bp8eDId;izh
zpO1fx`=I&$+MDuMt6xMu^MCT_5%UA?JJ~n)Uf*+7<x22n#Y>CMo1B?-!tCgYLu(Jd
zI}mYT{ec|^H4l3p)i};{BJ)JaiCf26j<+6LeoXba{c+*r6OZpY@!({^srpkHrxZ?w
zow7OYcv|Ds%9ATjDxYjS@#i?xi76+qovu3j=A6X^_lqHyc3(Ps$^Y_b-W-8>Q{b}0
zr306qT;6=8^4g^vS+}m;R=O*8Z_E9!4<(=EJ@b5V_eIsqz*qNQzJ0O&h1ZL>&!@cD
z@zVeG)VJ#I|Gr=QvEp;%m;GNafAjnv@FVtT&Cjhr_xw!zk?@WAi@?WS?|k1{ztMPe
z`^}O!>956JvAnE&G2z9H7jZ8qy<&UA`u4!vneSe{>wh=l-PLzn-_3oe@IK*#>ZkJ0
z318H|ss3R7Iq7HKFO%Ps{;c|E%V5cPgz+we!vEfX4*!4sPh~j9z|UC8D8lrD$(-dL
zYXtiS4rMMG?it)oJnML)`PTBq@vjgN6p|MHD!f=^yU0C}t)jAGb)tNtDWWSyr9{<5
zb_p|xY!*2!dR)v)e2qB2gttVZM3_XE#9N7GiS6Ry;(NqI#gfGyi@AxXOPrK^AZ0D1
zEbAnjD0@g|f>gh_v`Dmo8}DPTs~oBv$(+gD`n)swYWc(Y?f5qGyy6PrbmZ`0k7m2Y
z>dZQaMT@nW&6M4Z-G@Di{Wbe;4pGkE9M+t2Ts+*J+)uct^K|fT;(N`%S%6W{Q_xee
zO2CMppYJq}3U@Q73dc`22R072X{_}u=FEqgdYMI7N}0=<ceB*6Ca`L;o?@B6vX;e;
zrGr_K`681Da~jKIRvGqaj$%$3E-9{H&O{Do4l9nC9N#!vI9<8wxP-X&aF%iY=2*y~
z!NJI0#`>2zh)JD6_MgIEtG~<sYW@55@A-dshOGbl|LXk7`{nWD{I|Yu&%Q1Ee(y))
zujD^>|EmAH`&Z=O=YLQBPiF9BbYoIrE?{<N7Gu_8j%O}oKFoZcWdYk<4hOEO+^c!g
z2Cm5G{mvWA$Hm7wjU~5-m-+pd{?F#$_Wr)fu%7t<^Ag7S|K9x4|90YI#{1THwQu8I
ztG-}-y7ckQ$Cn;oe&qh}(S85>9`}>)9lgVF=gRHUTR*P9zbbT9{A$Bh>1$rs-`o(n
z8FOvXCHV`H=O&&BJ5zXW_65a@?3Wa;M&Izi*>Q8<?b7@69;H5Ee!BYUvgbQq#=n$#
z-u;y6sl^lT$Lk*-dY1WS{ztjbuRe-?H2hHa*8J7;7t3Bgd}Z?b>r1&8;m;pEzyIRV
z%d@W*z5e`W&pVzEb3bH%{PwB#i`3VdUw(hS{zd=0_s<`{vj1-QpY=cV@7Lc_zdL_c
z|A_y2?AN<r_kULYnEHL{ceC#czM6jB_?_!l^-qT%oIh9m+W1TJN6%-|_uX$Ny;XeM
z`Bv&<*|)zxSNv4}cH`rUcZP4WUe-K+^xXdC*B6(bJ$u6WRO<Qhmls~Id~@!#&&$qd
z;!g{n_B>B|t@Yu?=bvBZezEwv;=B5<rN7So*#33@ryC#QKg|1R`-T5|{m+TN*?*t-
zs{LN*&9hfZFZ7<Bdtv%+>!;waJHKuI;r*xkpU$7d-)DY}`zrDE(U*B&H~;wbZ#|1I
zD<2c@|CN7k{XF-*?faA;A-|-5Tm5|fwfpm|Pt!kL{Pg+bhWD@EoO#*(?7$PRr_s-i
zUhQ~u>&^97(_d=7I`_)q)$|wE&z?P=^62@a>?bzQX1)0ITJN3do4?PZAMLoWe9!%^
z`(5?>hEH-{>AlPQFzt)-Pn}<#-<v)cf8hEs=|k#!p7$A_yMK26x%fxmPsUHjuX3Mc
zKe~N7^Cjxrk*_~K+kgE0*5}Q`H#u)Nyit5z|HAw!=i`V+l8;@VF~2T-$Nb^%hlwBW
zeLVGP;^(AKtnYbV*Sy&A!vFQ=cT+wweHZ>E^HcZBu6J)<v%H@7^7e~wuUS7uerNyP
z`LCL(k?jUYJ9`PsbLOS2v)L@z_i%3HO%yB^{w}meK!ooFmmNDN+hPuRo^QMXe098&
zxH~z^IaoPQaINFs#?{8j#rc#ypUs~A5?3yNhj6->heVXr9hr}ElNCNFPE$0Iuavzc
z=csgCHCuC%E~f#H-hYk9Dsz=s6+C4HWs4M=RqHjgwDolT3}zT*8uI96Xs2tP(mJm_
zUB^;S$4JR+wb==iJmUf*OM~gUTXd%By*4~)yvS&#ex#1L)^g1?TGBeNbPwyF*I%e>
zs{K<_UQ1S|Nx#)-hRG2#5z8*C!&XLCUoGOzxQtSCA8M&+>*{J5xEkLzd2YPaKuV`b
zJyiLV{871R#rvvKnj+f9y3G1X`ulZlwb;}bsVz}AP@ki=UBg=Ew(e7%daYgR>r`$j
z@XDT*-YRRXutGsvPFZrHus{DF?i|j^9Pc^WIU3pWnFSe^{AK!U^jG_@;NK^IO8*4@
zIrpdO@6*5bf9L<I__q7gln?A5E`H$o9Q5tf55^x2Uwl8NzmI-<{Wa&?vma`|?)kav
z&%S>z7*d&Z7)Aau|9SQM&!6aji~nzAT*2DH`Ij?@ot<Sf!|gwFepdZh^YiKN*MH)E
zbNx{M((vKa+f#2M-}t`ie`EFL>8p^J3@`pY<9+V>BKl?H%QY`gy_)n!>D}^oGu}Ra
zW%2yqW4A{y9<n};c(&_R)Y~6#How~NJom}(2b1n{+;O<q{>b3z$)`)6O?<)lisfb5
z)5J%cAEiF|`uNKu*{2g;Cce4;n(1Ze)6|ECcMsisf8)-rOLzSqSUzmM&wS_cjZfF6
zTzz+?<*M-YsW-0QSabd4wH?>b-rRm$=dSm?N%vbH?th&7wD{TiXMWETpU!*2{^aE2
ziYLX-N?*Qs{r<JwtLhg%&%QjIefQ9v<og_t=RehZ_TlNhXFH!ie|qlGv4^K0U3<*+
zMD7XCvw5#--wD6l@+$GU^pmCs%J*FFeSA>(H0-75tB21|K2d#S`e6IL9d`@v@jlFc
zeD$&B;~9^3Jd%3i^z8a`t(V=eufCK0#QpWnH`edRzH)u}_p$4J@LTn_vLAB4n0)X3
zvG(WopPRq`{nGR~`-{wvyMH7ZPBE4+%dx&@ea6zrSog2z|23w=%;%UlvJ3GY6#O6%
z#~a0I$+n2ujd2oV5Zi4YIl--hPW&d^4D8Izy8oyC`TP6ap9g<G|5f<2>gT~9wm*0N
z{PENF*X>^m{v`a*Vbo;`WLm?x`9J61L%+IyivQ~P^W^^uro+t1%u!4p3@84u|C;sd
z=ifJsyICf)R<Jg(Ww4vEZeWyR^kkN2WoOG^ujLBhzb|xHa3$|tE<270mMVto|22$<
zSiZCAv7KO&_^<h==;yO<9$yo`KK*9)gXzc0Z-QSHzC8W1>|6BrfbTngIR57P>-CR^
zA(D}Uao_*qe;R+E{GRhW;co{6AF~9D3(H#OKaA)9ul~o$aK`;~+Nq6a(k^Ve)PKe0
zdetradwdU@AKblf_ax^P<D2YPH($BDz5P!8UFz$fFB)DheEs=N^E=&7d%oTM8T_mG
z$Ct0xU-&;y{*?Lo{TI$}Y2RADZu%_!`Q4W@-@ShQ`rZ8J%Adr4X$+Q(+>93)W-{zy
zaA7*Y%ERp^Xe548dV-v={5x4OnHtH>VzHuoMfJrQr6gpX<#OeFlozYtS9euErs<&D
zp{J}jSC>ogjefD=bb|!F&pOX_&gfp!`=KvpTy62y+R$o^sjks${e8LzweM(c(>Bm;
z(A}xytW&FVQFph2kuj6;2g3tK`%P5LcAG6TlQt7EJ7Jb$o@$|Foot`$JlnO&mBr<Y
zW4Aq%t%((nxuEGjQ*QGDvkKE0W~(h$TRyN<va+*&WZUhq(?P?&$L^DzgI$`nruh=%
zU?Xj#1%}y1`^_fX?6K#y&$YR1xx>8IG|>3J;UmKrhNgy7^m%j-X?beh(F)PZ)u>Z5
zQ7usEQJ$lCORi03g_MBAQ;}97Pr)7mHi0wzt^EFcx!if|nan~AAO3Ft`{3XF|F`~y
z{XP9>>feulZ~eLQC!axwm6N@g&5-R8TN(RGwpiwle=~nw_&(a`7;SWnHabQd9ixp7
z+B7<hMjIVeZ*+vSIB?G4e#X6?XFsnj?`p2=>@(RMISz3#a@%tE@=q1rEq+k6Pv|9|
zG<O3>0lNtY2hUeQGqHHF1mOn$V4g#q0_<$8f~-5(UbC-bU&AKLmch1}Es)KVt%%)^
za}D=az79bpVR?}oB4r|nggyzV3N#Ab6qFUYFUBa@B<UbANz_wFU0|WWKcSVP^Tcx{
zE=zbyRY<!@>WgZNOcZOANS9nJeOo?9<-git)z6Bn<Sb-fN+gSI7PpkSs!*n!rp&4+
zC3jqUs$`z{a<O~jf>Kq|`qCRE?Iruf!^Llly%&`gyDRZkhDkwQX^qNUwKb{%O8oLa
zWmZTRNo&eVDC8(@RrXccsk~HCUA{xMST;!BO-Wznipn|F(`sK-J(L3!D&!8zwJTgu
zv{yD%f2GZ$o2M109;~9Ia8Ei<vRLxAw1xbBg|Bkr(htNKgnRkt^R#hwaE5UH<tSm_
z&tl4?$@qlf(toLcA%D64CI6qp(8ze3(Uj5kfBB!8KPP?{`kwkj@VCLgnG6z)&Hu&!
zcK>SnF7`F+EB}v~zyAH6{Aa_T?SC}?wEtxN#_@IHx4Q2izL))4{qH2B71NRbn!j7V
zefqTNJ=fbCZ{pvFev1CQ`lHUfnpd9BJ)e|6EWQ8y-n9E>4@&P>-(7b5@vVoqDsQ>o
zWVq>g`_R2x4>}&2J<@t~@&1<EZ?0!t-GABtvfS198x6Of-g<v4@Yb5^`B%a(ufD>4
z-SwvDEr(mvZ+*KJbJO#h?Ue_YXI|mIw&wbxo1S+I9!5OMxPSOI!%eSi=Pxr}-h1Wd
zjT3iYK6vwh;r^>TRkw9-ZN0&E!}jK@+iCX#?nm7Fc1Qni!@aBb`tLozzx>g)Cz{XB
zJU{nR`t{jYS}y~hYdlkaw&VHKS9Wi2y=!`Z{9VKw;g`bC!k;{RqVwG0<&RfxZ{y!z
zeSi7g%QshEdB2Kzt@g(Gt>H(m?^eIJ|55$3^{2<T=+B8CH-1R|l=bbyuP1*^|Aqb8
z@>BNv#xJiw6@E(n-2L^^_qrdmzI%P2^dsZfx!-U9?ECkgv6eNH{TjO=$7lA<tjSDQ
z{!jjQ`ky%CPnK;Qceu`TKj5k572@IK;^3C#KPj|dBv>q1LQ`sz)C!5;qAH?4L@$W(
zN}Q28EtjjDsve-RPW85;q8ztWftb8#u2{ZgpUg?wnKG_Y$HbXMKMC^k8}l0QTJmM^
zS@Ry_{=xN^ONG0NYb9q3*JYloe2WCwgrAD+68R<cN`Q?&l-Gyn63-*PzXD!DWy0Db
z%Y}USxAKJYOyf=F=N8;3bXBB6?5F5H;S9k|{CD|&^7ZrE35ts*O8%0%B-t*`Ch~*t
z5obPI4(k`Tcbp|W?!5Q8>p9M{urhW3|Nm$5?+w3I{;>Sf{LS{O^5?dnKYj-Nn)Z9j
z-;V#mj8B<1Gg~klGJg3R@ay=u^`Ff@6?~5V_T&feFZEx`f4cmr`8w$n&!;z^cYZzf
z?eLG4zuW&B{MGx-@bk;}6W=#}pYfyZ_tgK=%(|?PS$;9r{SEw~_4)k!wD+7JPks*g
z?)@wOx83jRU&=oxey{xY_G|RF8{hB$@c!ZS-QfF;?+1Up`04vw{BO|zDU3^*Q&?ZH
zX0iNbWMs(s_xw-&pYDGgObIN^tjAc7v$e6ZGM)K9n_)N89u^^XH?H%%dICBErhGrR
z%elC?ZgL&uQ4&}rx>Q<Lev15YId0kik_W_|i|~n>iib+6$bObnmyeR&C$(3iS8R*$
zM4=XuTya}TJE`wd+0s*`Q>3M(CrW>jX_pgG_@$Vk%%*%+fkDn(T3aGtbf-v@7_a0%
zsZyB<va4jnWu8gy7C#`)Ayp@nBzH`~OIci{M`@S5l<XPl#nLaO#bwPE>{Nu**QqP3
zEl`>*dr<s`;3D2Y?piK&ZfoAdeByjsJQ`f>>{cvUjCcM=|JVNi`LFD6h98^19{YU!
z<HQfoKa_pi^hNo5$j^npDu12*arvwDr=#zFyq@{$#Ow3#I6n4$RQ<H-)1(h`-duep
z_vYZ+)^}UpOMII9CHL#x&$B<IzKee=`1aykz7P98PyfOAm-)ZVzs0}je$V-o^7hBe
zMK1(jX1u!jdfyw%Hv+FppG!RDeRS^L`@1X;8Xo06E`2=x@#e>q9<O>l<B9RJ=g(tb
z9)4y0#_H|PH#M*BJ>T-w<nh4=2Od~I_I~>N>Aq)C&yAjCJobEGdQaq@!+p;Oa~^g-
z+4Fqy3-9L+Poo~cdua2p_F>869nW99S@MDPQ`g4}@6W$^@nXx<Jx?}1J^tMJ_3`(z
zpRK;AeaZV=_VM$(9d9?hv-)`XbNW~PZy&!^d@uQ4_C4;0!>_tOhyE2YcCgH4+s(Ft
zRgL8v<JSMv{+;?4%J71Tmz9N$oBcC;AzK!671I;uCoC?kZ`rnT&gU-U&gHz$_LD`B
z<q<Ors|klZZ@$n>vCk4mC6gp~ipPmg77i9ZBqA)HA~{>SQTC|pFX=s!k`gXrOGTta
zYs6C|_eqvXewNrQ{!3It^n+-r_#=r`QVz0z<ku@!D^8Y=lRYdgAvH~+Qc_xWszQU3
zvPzt4v&ue2KKZq>*JWMhEfhB^TdTKeIcvvhURLW+5mdS=?;)?Ncu2)ULrUv|X1GSJ
z>U{-u>CIxdM65(k2<M21i8YDM7C9*Rm@k)m7W+3A1r|#dHI@kGwTuz}zx`qV{pY9S
zuNl8z{ki;C<^L0g7{;6bxBmV7oB7wh@9p3AeDnOl{HyZUy`R^9zWe#~d)XJu&l|q1
z{F?K1{<qMd^1oJm*ZSi2aq>ImH+HYry>a`X{B_aK*?(^SmHvC<=e{q6@1<Vbz4-d<
z(X-Xh>s~H-?f9ne)syG7PuU-Dejxc^-@~Pk-#t!#%<<UZ(YptB52rsCefHvc<g1>y
zY9HQxSoYrjt;Oq<m(|ZEJuY~><mtO-PoAxQe(lAC=Z7AzesJyn$@}N-J3efD!t%oH
z)s$BPFAqH1@Z`@UxktT^GoJr@&G=#Or;yLzKQ_Mq_WJ(wtf#Y|?s)d{`NUUE?`%KL
z`}pd^oA)B`j=ui%D(OwzJLwOZAB8_zeZ2I3?fV@cls<j<tnw}O$H8Awe-`~#`}yh{
z=hxm(`5*E=^nN=0Rrcrg-&_7(`?vaE$Dfv;f4^t``1K?1=ak=j{ueTxV`OHy@Gs@>
z&)-M>bTXK*P2(vLnlHvBIZ3il{HpL40S10gzI}Wf1mc8U#iS)~Nd-x%NPH4?61gIj
zBlJ|5MQpa%YSA8%Ny5{FQiN=T)kVZbcZs!2q)S>z{t|y7$|`a|C|PiYK#h=|*kXxV
z$qQ1MGSj6eNxT;mlt`7_A}J<4Mee-fLZx=a7I_KTvr;OOGbA2LGRiJjn4@f?`dej(
zlC=CRsXL-Gg-#0I6f71zEto0%OnALu72g@|X&iG|^qGp7)-XG=1hG6|Zeu#Yp#Oj8
z-}*lcf1LmD|K*s?>B>EiS5#0~Oi#L8PC|iA(N6KD!VZNO3QUS!3YPK#vJuiBB(0?O
z%UqVXQgl$%QoOJ5RbEv7z1(#9vkDs&_bJ|2{H;`|a!9pVeTSx)wt#l87O$3!W|O*w
znw;8JH6C?#bwTwW^<Iq@O+hU~?QUHYeJ{OF+F4qMHQ#A;sXtJir6Q`bQCU}cqVi-_
zE{#^rUd`X?A65P<oRFC%aaAN)@EWfGS1{Xurqc|G3<ZoYn3gijGXG@cVtn&o@!!2a
ztbe`#M*KPZYueAaA9CNzzis^%_I>~N#P3eu&VQNsx%bn<kFlSuzFhr!>TA^3`fs&A
zCjSin<@u-Q--&;jfA;?<`YQ5y-G>Qpt6xRDIPxs;Yw454PcA;qdG7ck?Rn1A$j2HF
z_uhMU$Ni4r9g*9SH}_oMb<Opf^!3L#OYT0u@Bi@3Bk3nTPcA$@`8fW`t0!-tq&)fZ
z*x<>nC-0tqeSY%Qi8s&QaJ?~mE&3|&#f;~hUNpaoeeLnu>Gi?ayWZS=d*hwRhqE7J
zK6!rl`DW28?w5a`O@FfK(XNLZAId)JeDvnA^mE47``=2xzx$r&L-G6OcZ%=g-zB}<
z_;%r&J+BpBC%o2w+xQ{+%d+opeg^$k`D5{C=5Mp#D}GD<Is4oHx99JY--mvm`7`aG
zJi{!8EB{6Rz5D&=XXlTa?-t)beYg6t|J$uEE54+B4gT8ob^Z7LU*CW4|1I(R^shI+
zPW^iQQ|V{r&yrtezm@-J|0`zv&r-;4!NJ6l!|uX%pCy~cjWw3-0NZ{xYxYQvot#^_
znfV_GO%VMiCMdB*LR*4I+*53q=tq$U!m&cL1xooH`E&Vi^RE?nDp19r$}7sF!o8AH
zkn=WI1Md>PHGBd5#|83*ghZ~0{t|bWl9JAn{3^CY^two<@Cm^^0{#Lu0(S%~1TP8o
zi*1t<lVy|JC^uhDUDi%IP)bG0Q1XSCnn<vqG@k=^8dp2_XPzUx9DGZ7b9fTCCa`Z}
zxyZDZQHLS_U(#Rmzq9^c__u(8k;#f_CgU=Oj{iCTj{W8Qm;FD2F_@{Jv5Vm`!+gdl
z#xTaWOkpfd%m$3?|2zLC|GE1s_lM9omanc~=6&+{$o<Lci^Vsc?}pzeeKY)e@YB2Z
z7v2WGS@de=i`ZwM9zS{b;(^n{caMyo&V6?I+2&{J&orMDK3sEO?Y{DTll#K=&)&1U
zKlj0dhsz$dKGu6O|8dTvClB{Nba>eGu<CK(Q=X^iAGbd~{&@3a!6%tdqn{_dY<%_i
zrOC?|FR#85eRbf)*%#kmTE6C)!ZVv|KD#*E1GX<5Vq83&Q`l44*Rr>=uVRnku;&cm
z`p30{E1v5$*Aea-ZeLCowpf<utUes;IM%V-u}iR*vCU-N%TmX(lvR>FnmvX+h2srp
z5VtoEGtVndHui0-PuVy*X0fTTgfVloaI-#PO=4Terp^9{?E%YKrd>?&%+DDa|E>S|
z?px@$S>N7%ZTV98IquW_4^i)Dy<PwM!>jh!XI}ffmVK@CCiPwPhsW<j-}1fw^V0U^
zx97i}PI#R9(Bgs6gV+bJACx_8c{t&r?W2Q_jy&>ttoo$xvEjp>`&AE&9`ilD_2lH^
z(#O(I_CBe9((+{0v-2<2Udy~zdE@<#@57f58$T?2x8Zg1OO+Q_p9Mc%^jPju{3D^K
zaWC$@y8OoY-Gz6O@4eqod++=q;KRCiEpIGe3%u5T^Z4zQcR$}geKX~C_A9klVXu^4
zS-nzwz5jLpYw<V2?_53@f6Vy6{GQ=`?T5gRMITt-UwQlBP5hfnZ~Wgad$0Qb^jr6L
z1|M`k%6x46u=kzU8;w`}F9Kdjz0!JP^Jd<w&o7=nZ+ZUZxzS6}*HUi{-_^W}cz@x;
zi;s<;+CNSFnDK%0!?X8u-|v0b^CtS$oL3od#NR%6GwIE`*Q~GCzh3`F{H?~@DQ}`)
zZ+_MLy7ld^_wAnoz8HP6|77xE*Zc4f;vd$$oB!71E&JQQZ@k_ryyN|_<D=Tg$M2_q
z;QsXXqtAyQZ(qKyf1UW|=9~R*YToF-;eLDLJ^N?2ud}}1{!;ll>eKp<lAqRpN&gY^
z>(S3oKMwv-`YHK~`?uZifZwZr|M)%U56{2t|4kUxm_9S^XUP43@ZaZuEdQhb9{*+c
z%lMDVzuJG(|FZp^{7;uLlxYf6Ci7NiEha~X-~Sgfwz3Fv@Nw(%OyYjYwVHb;Zz2DF
z{)2q3yvaQCc!YTm^2!SY3(pieBdjJoT{uBxuE+#o1|b8Xc_QiJ-y||6A4qyh7s%Gh
z*D6HHACak(5|UUUk|^XQyi9bSSe)o#p*sSdf^EXDgkK2R3l$3<;t%Fy<}2ol;eX3t
z$=|@YjMs$sB<~Bp%X|}f4S1LE-4?hZ=pn$w_kvfB-<3a<kBxUKw>Fmv#{^bh)@Y6v
z9%;V6JoCAha?Iwq$GM10gnK#nVQwiNE8ZWx|M|Fu1Vn#}m<p>2u?j5^`Y2=~c!F;R
z-$VXVfp&qH0zyJvA}ZpWB`!+FN@YoI6F(uUAnYcXDx@MBDRxIBSFnIDmv1e<Dt`js
zKK??%e!(vSg#zCMl7$mQO+?ZJ>-gvM?cr<XFX3n67Z=zfC?a$~&|dJPz<$9`LO+BO
zh5iZJ3hw9k<vqcf#-7Y^go~Zqmh&GQ2dg5BGm9k4J?2#`S6FwmePCb0`GZ@Qe~%!m
zkc>bfZxweG_XTbduJ3FUS@T&#S*9{AVZ6%thsl%WF3S%VY1VfvYnUAvOaHb1(fxD#
zkLBN)e^P&6{`KJZ>%SNNM=?e)l`|<a^)eo2_{N~e{GRn5+fvr0%q2`une<sYSdKHt
zF}E{!GTSrTGyi7xX5G%_$I;KJ&1J{U#GAl7pWB-2Dpx(v5#Fb~%XsX#|8o7~ir~7*
zxq(xPlb_=_+b32#_LW=@c^&z;@m=OI<DSQTk;jN<66Z8Fb(S8cP)2=*Qil0VpIDsO
z_OsQpWwX6tlVOWuPGo$<7{Xk@!or%&Qo{V1`4P((7F*UYtTJqWSQoQyVKHK@;}GWw
z<!$Cs=T_ie$y*?hB5<5ffo})zWS(o>**ubbOag_1zlGL{IEea-wul*udx**i=L@ro
z%8B`j-4aO<x&%71kJpu3n`;%v47O61T&7s20@ebK*X(kvvzR0pA2V!blwzu5tYJw0
zf9xOg|3&||FjO#IV*brs#oWU5i$UZ6`9Cf{TfRnr{_<(r=Z9aizTWsE@WuCY)u*CQ
zW}jO=|M{fwY39e?PhDS+{`mcq_h-;|$8SR4^MB_34*hfU&&fYmf6M(|{d@bLSAYEe
z-1)uychGO?-@?Cd{c`-}_>=!f{r89;vwrIQT>owV=iMJ!K01Ev`{?^I`NQ*fkKR6h
zcmCtFFJ@n-eAfHe_ulh8=LhwV`#<V^JpBIQyQ=pQA2mO7f7$tk>zm&9%<uEQ-~RsY
zTkDtNk2Bw~zjc0V@-FOM$6NU~hhFi%e*Wg`+o-qBZ}?tIzFPh2@|*kb=Dy$eZt|OB
zuaaL!yzBWW`FYYOjgJ~1R=wZ){>TT5Pv1XDd}jFk@l*8YC!cjc|NG$fZtCl6FUw!m
zy!rFS`}NBgR?nTD-+EF0YU1nFZ-n2jc{Azljt@^iAN(Tx`S|<wZ^~bvdaeFu?(5!{
zInQ1{W_(=o_~DbC&-7o&zN~ub_p<S2=c~=H;$Iwj!tqG?A=jfxkGY=Qd93wh$CH?+
zub<t0vGrxr%ZD!>zbJUc_-^mVh|j5?{=Pr(Ht((6d%2J2KeB(E_dfDn<J-iyo$nGp
zbboyCG4Ru}&)dE({4VsP`Wye3luzNGq`oZq#`0sq_v&vQU(3Hnf3yD{|K0L?&Uc$1
z_kQ^?q_S<{+{J0dF^g?6>s!{#Y)9EdSPPk+{@?kJi$Rh35bJ9;74}YcDbCs4b9l4(
zCh_s}ZRC;T5#~z~Y!I#xULmN(U&qVMo5NGeHG%yc%TJ~|OgmUC*r#y#a|p1Tvt}}j
zFmGkiWaDMs!<hY#;qSM<Tnwp<^^D&bE;1-HEn=C#X2o94uFbxPRfpv!ivha<rwQi*
zwoqnErf%lLEY2(knf@}WGhbz0$6?34jCU_z9<Lm?9#=Vc4Bst*bpp3}Pjfxy?BKTJ
za}@X>@K2yh@T`!%$Rd$b!kI!d1Z?=Ec;)!*g(r#mif<A<E}SMTEb>FRLa0dK4xc68
z9zJ=2HG(sRM1{l!w((Z;?B^{NTqa^F!XX&Ow~*JHPnKVwKc3HncLvWo9u2;)0-?e!
z!i|E>d<8uBd7AhYh3*PH5xghRAz&*|%&*9|fM*f65w|q=dG1}j_xQa9D*4!W?r>S~
z9Ojo0b`$v|JVW@L@M@7WqHD!2iwTN(ik=ZI5I-kjELkU^BBm&GkS~l+RX|oKK=`U~
zxQLi&t*Ei6qsS$pFu~va*8K1J`Gvwnriv~Q<raM@VlB2(!cR(GszridOkZTa$Puvx
z5`K~!C02=BO01CjEb~%UP;RR1d+8^VS0xH1d!*OOewY0&lPfb_hE+CJmQ~hMMpkCA
zOr>m_+ywc}^8IpqWf-O9rB+GRNN<u-l6WjqB4jD_QCLunPux*VP;`Z8fcOOQFQNy8
zxdfwmi#VsS-DfrC=;Kb}5#-L`n#i?^>n7J)t`D4hI77JRa-HCM!ClIGnRhwQNv>O*
z(cC|HFY)#BZsIQITF%A5<I5}0caZNK|8BvX!jD9xMJq&IL{)@21j6~=^3@716=4x`
z7v&ZCD0D_}z5o~hA>K>84t%M+raUP;jJ&6~WjPF(=l!4YUxv|}q5N;+?~T8H{>uFQ
z@z>&?>wiT5kos}<`@ioyepLSy{+aQu<&)EgCm*JMw*I!@+mA25K281L_txf>;|r$e
z^3Ob<=04f-#PYe<tB0@FzPR)>>`B_wH_x?Ru6g17;=+rMFK@jvdEN3l=1uY2f_HP@
z&-%FV3)2stUqU~%zR7->_W9W7;xFI79Qva4<>M#DkDuP3`0(_j!$+t08{TexJNLcz
zr?a2#e<*n8`*!-<cW-~a>3%KwTHtm5D~4BRUv<CXd-v{L?E7W!vp(GVsQ0<&^P*4t
zKQO=ddAIo8hWE`M0zUkDH|5RGm;EoMyg2>h+w=8L|2^9Okm2$5CrMB4Km2ol+x?OU
z1rIkp)P1<@!Oi<p_nPk9y))t7vHQIb1RmXb?D@3t+3#l!&v>4md*t@u*}bp#w>{eX
zWbTuFk2D`R-TQiX|GoVC{`X7oExEh<?$>)W9%w(jaX<I&(c4jXR^2=G;K@V3N4$?D
z9v-<r<=&^eEABSkU3J&zp6C6~_xIetaWCTT%3Gz^)vguYsJmPGi0kRaCt8ocJWPIQ
z^<c)m#=HCPyubbP_P4v`4=+AB_4LGJ-iN&p*dG}@-tl<llfzFdpWS|T?wQfET~Fhm
zNxZo6Qv3DiH?!aW{uudb@&~tft#AF_Kl&v6t>t^y_s`#=zBztt`*!*p!*}!Vjo%ml
zsQNwUKOgf&=AVpn|6lue?|&F$8dC$aAL}~y6`bW<m$*)F|KRcGoy+r<yNhQsU$g*|
zz-!*wJiB@K3iylIi_ef~li-k8C(bI-C;3J?Rra_HmvpSuVQFW%S@K$P?NV>V6hzO7
zIEzRK7YP*$eHQ#Huv#ET@S2dH2!rS;k?kTKqWxlD#6ra)ME(oT5j-h;N$jPBr=+BW
zuvn<bW8tqNyTsOui;ABTtrp!amMt+yVzSsS;VFWR0<#2O2+E6W64RBa6@M(cMf9`S
zPjL^4EfVFD{Zjlg4zkv=$}*->pT(DnNsE_CK9~6|w^CMJs$Tq*7>l@@IG4DOSi0DE
z@w<}U(zj*uW#&mLh#e7D5>67iBz9XuQ!-z?MTAM{m%urJ5`idz3PC-Qo#J&;x1^0^
zHcPD+vlogISS8ph@<-H3#8xm{;JV-&p_@XALiYtE1?~$J3H6E8idRZ{N|cFM2^{8S
z<ZI!7!T*u(9d9JBG;aWJ6llJR_Z^>rz;(etB6<=>BpD=C#lnPd3at=f6gQG+7M~&-
zCDJGIR>V%EMkq=^h+mlhBL7!Gcd<+<bLsyQ`eF-3PKg+cJ{9&BOybYy%jDg`qruzF
z=OdsXC?If&Zv+2h!Q(>Ff|vOUdE5Au1%riu3MUBn2`v?xBy28HAaYM+ort>dYQZ^z
zvcd;N1td%*3dPQeOckjRZ53N2RwlYm_?qA&eiy!Nyi$CZ`EK*M@XB&4bFpzraUNp3
z!kogy$g+q1I@f3J>D+Q$#q269%8dH|WB--@UH-f6*Vo_v|EDl1vn*!*z!<<V`M(YW
z50fH`E$eNTuS^UKTmDx6E&a#&U+BO5|DFGB8DB8nU@Boq_`CM^mES3U1sJ|E?O^%9
z;?3gAY{X>6n8%R!KmDKQzq$VlnAF(TaqMM(#=PtQtUuF#N&Sld9r4%yU(R3U-<y6o
ze%JZ-<nz7{3*Py^JN{1nz1n-{cj|ANUiv-{e5U%G`K9tJr&kkSg}>&16ZwYY&97Ib
zFT-Etz4-Ui=Z)aIw0E3uZ@d(EruL-fiNw=yPo_LEd-Cw{g~ypsvY&E3+xWEU$@0hf
zPc)vrc=q{e^b^&`I*&>pvOn7YnCt20rz@X{zxe-B_f7KKw{PCOe*a4QW&87#7n-k+
zy^()o{p#Y2RWB>v<i87f7yZ`h&5~DQFWFvty$XJv_xjx{n^%8cZh58rX8#+P*OD*$
zUaWoD^t$2gk$3;!@xH(H?#5g5x5e*DJ~(~k|0wwJ(#MsbFMcih_V$awr_lGS-UWSl
z_i^vX_wViAI=$NaJmcBfXQ3~Dym5Lj`r*aLJzsRcDSb8j^y{73TgA6u->!Wh@Ja0J
z^=~u3E&a0jlia7;Pc5H5ea!fn|4HN9q+im1CjT=3;ri|E*Y@ugKcD|x_bcW1y`N{l
zRet^c_2<{EU!%Ug`d<EP>E8qY^Z#G|<M7k!Tg;dHpC^22{5s*=#_wU@Bfnhyu<L#5
zhrjP*-aUT%^L@vc`9H<~DF43jBjcOZmwTTUe!BhX(Wj<Qfgd-$&wStXLF4204_Dqt
zzqj}x{;Bx$y3Y?j*?-h{FZ{OVb;IlPZ@+&~{dDd__WR=ZCLd0G=={L&Uhl2^+d1#E
zK9_u(^PTs*(zm(antqi1;{NmUZ{vUa|M&k&{$pSWWlUxK%+SDK!RW%Q#hS}z%Mr|V
zhU*t+J=ZUu#r(E{@q%^yt9cZ;Tey$&?Be~*%f$DWuRzdM<htla(M}N$;m<+|B9h{M
zlKGMc#LGnEge8Qw3pNV63N{G-6=V{cEEp#!A-IEo39lMYG<PX?Ht$P;-@@BOt_i*2
z7vz)U`^IO;znpIt-x>Zk!T&<Hgl&X%1zdO@bFJga<zFTkEi5HsBdjcVk<Xg<E6+yW
zxqQd@&kN2FnJCU9by3Pxl2h!1P`lu9A#Sm5$;Hyy()E(n;^AVqMLR`Lio^?F6p|FZ
zD=|&VQqooIkkCVZOa4;=%EINM4&vLyAB$ZU{U{<OGFy13@FU?(!XJfOM7zaTid%_3
z7j)!*%A3!(Q((DpgxG0uJ@J*I^`c)ybwpSM&T%VouH+EsTE;zxJDlq=$1V1=?7KL=
zasJ~f;b!H&!#Rm#F}peYMm9;-XH1V6ZI~`IWid0cY-WyP5?~Bvn8h%SDUkIX+f}v+
zY=_tmvWIhOas_a<u{*Q1v+;9;aawX-W@lvk#3IkSl=VI95w>IO4D4H2`k7N$_OP<E
zn{o7TykS>j?_(2Xo5~u_qQoT5Q1pK{!*ym`_G=u@92?l&Sa-9qv&OO(vmRod&nCmc
z!8M7SgQt}HH>Vay8T&1cC~g(rHN4uq>$$gbx^ZxE+~@G%+R1&M=OFJzzBl~l0`K^i
z@oeEL;f&%m<^IAe&u_uk%X5W`i_@9?468J2K5G%%efHg)d%25wJh_gt7BF)BZ}?~Q
zzm9=}=`?c{t2vtr+XS|!?7E!SIahK8^OW-)6PPEIEVPI}mFG5>Gxr@HR(?~#OG3+p
zHwm)}FBLQvh~>Y<x03G$e~VDIXp1<%M2J|faIWAj0XD(Kf@wl`g!qLugm(&+3R(%w
z=C9-9;yK01!?ln{ls}jM7H=i@D~^+FGg-f}%wbu{(#7(Qc?R<eW;<pbrqc|b|Kt8G
z`FH65TZSygeue}8IQ~BW9sk?@_y1r1zs!Fg`fmO$?Q7K6Enjwin*KrKz0<pvcM2a2
zKCbw%`u(kUTJNmit$U~W?(dt<H#6VZzv+9O{`%_crLXH>3BL+_E&jIQUE})|?~UF^
zys!Om`{R~ROFl(>RC+J|cFpTeubN+(zW($2)SK6D@4nmqp6}!8kDed?yxsan^X;W~
zkstnlkp0N`ap#A#ANW7c|KRjN@ng-W<j-+m)_-04E&khvFPfhcK3w}C^_k`C&#xc8
zt^O|meezd>FS9<c`n>XU&gX#7<)1%&PXDs(v-?MhcUrHny%c`E;qAxwR-a72hJ4rm
zp7hn?^U9BZKAik;`NM;c&%d1cvGiBS&yC+^ep&PR@@M-mE5CGqUH<LsH~w#xUz|Q`
ze4hAu+UJbVt3LC5wfye<Bj?+VPhubLzEgX*<89Wv<PS?e$$pXj(($SM!|iup-!#8c
ze!1#p&1>Fw9v|vIB);4FdfKZ^ua3ObeKGq5+w1>t4!;$CXYfw+UEo`T*Y+<mpS3@W
zd|vQ;?X&q$KR<c-WZRP)kIf(Nc|7sSktfAZc09iP$nsJD!^Q`d_haw-Jbd-U^SSkN
zrsoTubH9*#@$dPZ=MSGtygc%<>!rvm#y5B0-g>v^{pt@gAGdwX{haqz;#<>Ky{{*}
zx_ndkmiBG=cj=#Uzfym_|7rM3;Sa;V)Bk4vZ)HefJi^q^tk3l4|HFSb{?+|o#qgSu
zh50J;J7!I0ZYDcMMaB(`MNHe6b}}h4*E7FoTEzI}|Leaxe-eLZ|4#X%@Gt)V(*L3V
z-T!U<^Y*vipD%wn{+a#1$WX~#%gWCBm&u%Q8iN_5A=4u!FP1rMkJwq*U0A0wH!+_2
z|M7n`<07UemJs%I&Rou$>=CScncg!<G2}A%GKI3na)feqa82TT!qLg`j9s1mE}I%h
z1h*^S3BCs2W87U_xm+gP<~*BtSp?dJ4hp*p3kV4dHVO0zJQm0jWEU0_`yjz9wOVq8
zM7;PK(Obd}!plWoh^>{lE6FDHU*fDdmpFqMyT}v4Yl0mjbHv|DY>>DsZXw<y_Dp1(
z&}_kHf>VSZ3w;)H6Xp}yF0xZZS7ej$KjBj%E~0s&i$q&R&x!Df<O}~7$`zIuH59j#
z;FP!}enNs<YMxZ8l(tl{w1OOiLX^Tp1!Ki)ijhjQm98mODqE>^s932?R(`Bxq|~A4
zuQ*k4o8nQ$zl!S=s}%y}+2vW~=gYCnNz1Xxua(!8Z;<tt-Xtk3IY%N%;;O_{$pup1
zq!-9|%VbD%Nv)93keDJ-AlWR{E6pGiAiY8It$3~&pJ=?uZ;=^dQ4(g7Yb7R$pAhR1
z6%g(csOJw9U=flPHWywjBqdnLXU#p8!+^b<Z4>K3)-P;*>{HlQvq&&m{a5_k`)BUo
zJO5LeT3HUWK4P86vWM~WUzXobe{%j>^egRm!k=${ul}!R==(qa-^PD6|F1JVWr}C5
zW}n8fg5w|i4K_ozV7AR{0ql7kYdEiQ*>d0KEMmXLI+1lh>r2)RY&SU;aQpJQ@Otuu
za=+vf;O^n};j!WU&D+f<&acWZ#^=Nn#&wJ10y{tZZ?=2vLY(cK7dYmzGqcZUvtvzV
zZfCm7G?lrD*@}sY@i@b5hJF8+{`>y-z~3!@wf>&}t?^6ZC-2Y4Kkxt2`90-V>(9y`
z_21pTv3-sE;_&79XPwXWpWHtE`1t3;_V*Lt^}YS@X8oI-H|1{*yeWFK|FzcZq*sP7
zr@YXAIpx*#*VkTKzL9yW^EUeR(--%jKYL;Q`rcdl4+}nWf2#jj{UPeT@4J$>+;5}a
zTD}W^KmWtmkGDQ?eKz}4_~F;P7w>kxulgYT@#4ppPpY2+KKQ&#dt2~!>sya^HSYyK
zYJ9r?@!k8MZzNx3ytw#6<F)ELw-2r#e}63g6!&q>JApUZufkrYytw#0>V?RwnXfZn
zuX|bde9BXSC;pG`KYsZ1=!@XjO>ZRMB)$6e{Oi-7Pg<UYKGA+^{oL;5*_YKXte)L}
zBK>sp)9p|9J#Bk-;rW>tGhdWFFL*ZZ>7plR9@{?-e7x+*?PuaI%AW6i`tR}YN6Q}B
zJ`#Df@A2_x#V@u!mwfjANy8K6CwCutK3w{c^~ubqKc7r~yx>vtqh*htK6?L%<+1$Z
zd5@nyx%qVZ)7Gc0&m^9oerowR{NbvH&W~q2R(tgKf%e0_4<A09`*6|2Z;#$Q@qc>c
zao(f2N53C+JYMs7_2Xqv);tS*aq7k57s)T&UcPvp|1ST%)(78@&p+<^sQ+=>hs=+u
zpWlDs`)c~N<J+bmkAH>#nfb@`&&}VDepmfT``h=gkHL=l9*Ysn6eeaSTjp9;FZNcB
z>s*<9^97FcNAe1APv%tTEaY_JQs-I1=O^$~;Ix3GkTd@uK2?4n{vv)=f!BhDB3_~<
zB9=m*1o8v~1QZ1FMRi5o#5YK66MrUpQN&4PneZav8zLOyWs-r?XQX#XZIGBG#w2Pj
zQY2C!dPuB5LRadX6pxg<<YS2i;!dI|!n1@gh|ZA6l3FcQA?Y9<EfOLq!e7Og$uA&y
zOz5fbM&TBrJ%W9LQi4$efdZ9+7Q$OatVP$0Oc!nzx+Taa^jzqzu#9M**mrRgNgb&u
zsk@R&lIO%d#d^e^N_b0q$gs(1Nl%uFmYN_rRU%AszjTW13z=r=T~gPi4oWSP<dS$O
zW+TQV79#dntW%srVzPvTq>a>SsXoc4;y=U;#n?m>M9zs^6P+O@BDPS(Ld0Bjf#`CP
zjly}t{lXuF%SD((jl?)4TqGGJ7m3%4T^Fm8cqX}1>X5XUtc2WJxw&$;WXom6<OJni
z6(p68DSN71R=%Xvso11&OWsj_g4_w&eKJp_Z%MzEzAu$1Q7bw}SX}tI&;y}ZkpOXR
zNq&iwA}N9s`KR(T2(SzM6PPLdO|)KYfvBR$7r_MnI$l#=TfX`HvjzAB4+?DLpUB6}
zcb3<T*Mnz1_cv~F-g~?Syn;LqT*ukpu`FleW2$E6W81^d&AyGLk4cH~?|+g1lm5;7
z=kh;<L6Y$m!`1(e|1|z=_<8yJf$t?hU;lRa=gaVgk)1h$xrr&B;o#rpzfFIY|9tk-
z`FGl%$$uXG-u~;xPlI2FfA9Mn@c-t2h5v8=`u^4Wd-+enp9Oy`|1SS4^iS=73PU2p
znSUyOoBk~O^ZifI-${Ru|K<3{@o(qfm47$<J^$zbulk=pKNEj3{E7ej{O{zy9e;v;
zzxZ|Z_pd(|fB*dc_|yDH-uLO>dw-n#Y4ywd=eh6w-)4MW@g?E&nvcOBN<Nf+y#KM~
zqvXei51Q|b-zL2Ad~@cF?As%6?!69p{q42UTd%jXU&p;X{QTOpsn2wttH0=dQT2S+
zljet;?ytY^`{2fd-iIoWk{-=@bm>vrBesWn_t)JGzZ-t<>wUL}p^uy%w?1C==*Gjj
zk1jr${!IS)>gU^EO1@S5pz`tX`=+-SUaxwk`YP)6vUl#ESie|*miXBEzU+PU2knn{
zJ|6k><}>ftonL#uF8Gr8S^ksdM~RP|pMHH-`R4uo+c)!XQr|qjF@AIUn)>Dc=R05c
zzDNID_v`X6iC^qLBY$l8;r)y8@1y@_jB*SY|AzdL{qy@b+wb!~Eq@;T>Gj+C@6mtP
z{#!9v|JVEb=XdR&34de%$^75-KaycPV;-{#%SL8a=4hrzjB!lMnM+xCS@ttMVz}|I
z@9(<5oBmn<@BOFr_ww(*zn1=*@hj!m!=JW4C;#yL@%hKEpGSW3{&@BE>1WqZxgW(o
zMt=D9uKI1^n~XPdZ!6z^e#7xb=XK4ipjUfdrM^D>TJ3en%a6}}Urc)G@apnQ&KLKe
z1U~e+=Wyrj?Z<ac-VM9!bcgZIqdOY+R^5AdclRCtJCS#D?}t2ef3)$@_eZ}Ug*~48
zr2JXMi|Mbp-|T$j_s;sG^XIV7)t|yY7JtzC*!ua!*BRfteklIf`t9Y{%5R)M+<&F~
zG5IU{ch%p0|Nk)hG5Iij`Rn>O{a+o!9VRx`*KDF3{p`nBH?VACNoGlBUc_{ZX&1{c
z_K93_+|^vuIZtp1aY}MC@kt5Z6_yuWC~``8p0K8ff@rO1m&gU-7a|wLUWwloUoJL5
z<cg5J&{Cmq!Vg5{#J`IPiYbd4iKq%o2)z;5!mr9-Bd}EXn3%uBWQh;rlf;5We~3hj
z&Jqifm@DyFY@cYPn3?!{vFV~_!u<l4{IdL9{BQXd^V#rz<7(op=k(`F<YwX7&+W|J
z$DPc>$oqvSnrAcjWNufU-@J1Ll7tgQ)y3|K&J`^elayeSyf2|5!6|-IG)?rE=oYaW
zF%D6C;X0xFLT<v`!pcHn0{nbdJPF*>xHEWcd3?C0aqQ!m#krWXo%0`O78gI)V~%L{
zI<|PWEY>Sbz6>Y+doZ>z2eVqRS+TXUhOvY*r!cQ&@n-wWHkB=aO^3aK!<F+lM;f~d
zs}@Tbs}nmr=VLAh9#@_%+<n|DxrKS0c_;HJ@fY*k2!soo2&oFy3GCus&wZVH1#c|>
z34UGvE4(h;kJ+7A3s_#T#Iwq?YO}_$8*?w@y~M-EWx<}vdWglGbr#!Ij?-M++-Er}
zI5x0{vVUMZ&uYX{$?U*#pS6MAox_&hob?)W9`iP4HCA`_Mh+d0=WJfAnk<SeLM;23
z%o(E?1(;b`8`w6p?O?HEbo{sc&(q)Hf7bq8`oEaLjp5V(ng1vKKmNbx-;CeKe!lt@
z`nTx+62{%kGg+KiJegye7BD?$7GaTQI`{wezoP%;438K@8Q%Xp_wWAyg^XvIcd$mW
zCA02e?qup_+Ra?XdYVm{or!%qyEW%GuG`%2xdXZ1at3moVqeVB#~I3L!f}`FEbBQ|
zefCWpjGRFn$?W&p_&B7vPw;m0J?0A%I3rjqq$&JQm|sLg_=4baf#U+5LidID3F!;S
z^A+$+;PU4*<!s<|=KRjSkM#%hb!K0dIA$rvFaI9@4gB~0|8&L^j2R5Y|7QNV@XPII
z&JX4vG2i>Y*?dd?w(r~WZ|YxPe4PJ|_wD^R7H=iq@xFiZF5+#(>$NW%Ua-E<d13ux
z`t!rjt~`7B{Kt#aFAhGN`S{7hqQ|eF{eQXaP2~GuAD(@1`XKPZ@<Y~#zz_G|D}Rvx
z#P!wq`;zaAzX^YR^SS5qi_hP__<fW4&iy_5+pe##zcPQn@ZI_Q%CA;mTE5)<rtwq$
zXWI95Uzxvjep>j+_sik0%HOVhb^Ds~_3F3fKWcyW{5<t@>95p3GXHs*`k6m7O=Wz{
zP{Q!yzX?Me<5fm0#<dJD7$TW&v&6GGvL0pn!l1_Bz@Wj%!=%g9%E-si^KbIMfBzpa
z*fSjZXZ3f^AOFAg|Ni`&|L^AC?7z4EPWo5!uj_BbpSE9mKbCyE`_2DH@y{neCV%7n
za`ThI=jWe&zU=?};G^n$p|@M#_`FSj+wtb`tEv~zo`pW=fBEU<mKU?0GCUG`aP>jp
zqdAW@JlgSi>Jz;u;g2^yn)9&#{@gngZ`$3Mdz1N&^WArMCg1*f)BNV;n_jmW?*`wi
zy7%&~-o1bKH$5_a8u+aG>5j**9{qV7|E&Cl+{^PXPQK83x%y?#t0%8@-+q1T{%*#*
z==atioIbF>=X}faTJDwm>*9C+Kgxaa{Ic=$zR&t!l)tcivHY^-v+k!gA6I>r{eJ(a
z+pmY;qrd$B_~C=uhm-H0y#Mh&=Y8S3DQ`EwJ^OCn`{nO>K3w=X^-KKsh@VWq9{qgw
z^USYDf9C$5%UH;CiD?bfcSb>`Da=P$-ZSShKKW<!kB4D7lMTx=mJclVnbn!}8Iu?-
zn7%RnV@hM<Wol*OX9;8b!Op}H$R5f*iDLrS8}1VB7_M-xD_o1YlDV?E4sxDlXJK8y
zG?&SkwT7LMy@ur#<3a`<#{EpDEFG-d*m&6Qu-UP<aUA8U<Xz4e!~2u_3(sl(Lqgj`
zB1J!lo)o_!*(H@J)hv}NRV|q$Atk{msVBuFWh602bh~huu#BjxxShlU@hY)eQ6Vuo
zaWV1FVoSuU#TSX1iLi=Ri(i%eB`GG}AiPn~QSgTVpWsWueBpS}KVp-_4~u^huNSuw
zQx**rH4&GT3YB>+1G-2jSa!4gSw&@~R>k>>bxOg?sVYutIvU3`R;&M4tyVp%Ca<Zl
z{aE{-maAr$dc9h%+9WkCwZE$W)QmNjX#CO0(VU~ftaewaSRq{AME<axnd}DXaOt(u
zJ<>;|A|!>ylSKT4_6fcaauv-Me<9v2<}UI`P+ve-pkL5h=zst>-%f5lE-_9wj&AlD
zY~`#QSYER{U=3r-Vx7VKnvs|BHp46iQHG}fi~d>u4f=EFPwC%Ze;j@@{`~Sy;M?(U
zGk?7L#qsyT-`{_j|Cs+t`n%{~^nafJwSV(}&-nG_xB0)U{}KNs{=5Ib`d^7*^M8l`
zOaHSo6#Q@gcjoWSKdXP2|C0Tg_#^da`tR3&R{lNpxAE_lzmEUi8S|MdSwFCCVgJpp
zz@f@n&i#QWms^u#3F~8)F4kPOG`6R#-&on$3Rrbm7qUKJ{la>b&46PWry%z>o`rl5
z`Mdd#@D=mN2tE>$7j6?u6iN|}7vT_L7UmbS6)@p7<ErC0&T)^+gJ%}kceaPjzZjVr
zFEG4eU}sEaJkPk2@gu{Z|9}5|{hRwo{a4bD-XF(*y8LSW`S`o|w+~+~f4=c?=KG6p
zE#BOE74mB0EAiLeuN7WfzU+NA=gH+Kq0isHlz(OY!sS`_Q;TQ9&u>0!c;@xo`bGW=
z(U;F&oqY57t>in7xAm{pU+;VU>NVq=iZ>VE$h}Q{d*e;*YpqxHFJoSMzN~(E>ecTz
zVej_6Rez)ZddBPTZ=SwYeCzOf+RKm^=bvZ25Pf<3h3^Zl7tSxVUwXd!`pW%v=Ie*A
zcD&s4V&V(dmp(85z0iGm{ndpx9dEC`jd;iQZrz*u*Hy32yy<*b{$Brm$UF0QUhn69
z(EAwmLGs;|*DSC5U+;Y*^3Lo1?f32Pquz^t`0?S;$2Ff6KfnKU{L}kS=AW}a8-7mz
zWd70Y!<Y9-A9Oyx`S|wZ*AFr8pT1r4*7;r9yMu2<-#mP|_vQE3>Tk7PCq6fSvh|VL
zBaTPyj}#vBKel{y_rdb}jQ6YVZM!@7PRVW8Te>$l+|;<ke^2~g(p{}P?{3|{)p~pL
zZNJ+GZWY{$y%lk5<1Mz^g}20SD%`k!W8v*D_qZP#K0JS)|Ng88ijUtuk$NWb-0X$+
ztLisQ?;gHe{r>FxXYYQ$m3~+Je%8n0&uL$te|h}H;LEblw?3PFVgC~JdDSPw&wIa?
z{|xwJ{MYbL&9C}ji~iXD<NNpF&-~xUzjgj>`}^tN<o}cY|M)lKFW=wa|5_MLnamko
z{`dSf{X6w<-M>!^TbO%Uj9AXGu(2QH+{e9^XCe<X&pe)NeluYcu`=;mv1pP1!kS{A
z#7o8Zi!zA3719=-EplA6S@gC@faqt@V9^kfH6q$#FU8tKC56icUkd7r^oX{L^a&pp
z?hsxnoGZ#EzDex4$PXc1AvYl{p%a4qf*bh{^A+<I@Y?hE@UZg;b6#MHX5PX2fMY6`
zC-+tETAs<gE&N*r`-G&06oveSJ`1u73yB#>luNvpI4*fr^0nkHsiRVtBr_zk#Kc6?
zMP0>SiN%P07XBpYAhc7YTs%g6p{ToPsz{E|Ex{C_9O0?L8G@MtY(go*B_i!2=Y@6)
zx(FQP-Oqc0|BK){VL@R|q4z=p!uN&hM3Thjik}ugB)(CyT4s`5y1b;El-xrFBc))a
z!^#Y*Eh;uj3l-idhAJ;oW>aBR30Cz{msD$4u~z%4@m`}wC10^hMOpo=+6VPx8h)B#
zn%wGX>RFon>TYW8svdHWRfFv7Z7PMn{b3T*)4nX5&*j3dC%9GNwxSMC%l}IL8}edu
z<<fqV?^(}&xyf-u`;^XHzIWdye5>T-P`awoA<FUp)>o^yHy_!&KP=LxnIpLCljNU#
z!3AudA2i-<_&%R2iRaL_Z9f-^G_g*)A%9)wPbKe^ztyi?-}HV<{_y`rJ%gu$xkSR(
z@+XU*8NHvxaD}IhYdu^2-&-&D-!prZ^5)D>i{H!MY<+c$X{lr=|DER*=hE+nesTQz
z;$zW=PY-Y3o%F^1*Y<~dZtj1j@cq?y^^X_c#(ih_Vg6$BbBk|^U-MpX`nc%N_1`kz
z3*LQrc=$Q@51ntuUuON$|9tyV%(I=Jga2RrdF$T1%PSwA|K!KFQ{7hHlzs2l(&w*U
z2>j|{3ivnm`>Zds{uJ<d32)>|V>t2K<MY9{W<LcOlHRmkn{dPNnf`~#Kl*;KybQSe
z>8`-7?(26Roqu2ahk^akZ`S8)o<@A!@pHzH<*&FNTD(sF%J}ZelZ<DKPa>{~Ue<Un
z{!jJ)u2)RgA6#a<b?kxqn^}LJb7ZqLyw$!Hc8}|w@w?@B1Mcd7<6wU9r~7Zkk0sAM
zp0$4Ld;jaz?>BG0s57en5Bz-Oo7$h~@2`Ko{TK4p?qkh&lV2(SDp`L2?f#VWOHOEw
zWG$P;_YS5`{#=F~FBg1TE@Y(GBxL!s{CUuG?ypz>-uPzydj8AC4;nx3v2yWl`^)-%
z#^1}_3M{6qYXq+G=CiG0UdUP{vQ?^_`x!g0U@reh-ZHM|f13UUab039XU-D5DX~Ma
zm2)4P4a@R>=Kue2wTmqmxyE*dVJ?>qPwVff51+pWvvP6F{ipi<=I;#FCI3vGb3Nb7
zD9d>5>E%0G_t!iye&zR8`OT;A3BUe6eRK8eb(e?I&(=KZy7THL&z;*3n4e92#&OH%
z?6%7uH%!k3T)cEo_JQDSzl&ujFI)+IaNthmtvA;uoe#buc$eY!<12EPe_qSFbK+k2
z-R2u^*S_2^zZP><?@7aVJBH2Q?H_%={N<v@)$&KIAB#V|dnNsF`F+V(bKf?;W`DQv
zxzm%iU;h7C{CeGchriYT-v3(nLFJ*@lL_Be{`~rq;hxpww_gr0D*U?g>f^^!_V3Jp
zUU@x!`Y!aRB$GWG3sW&m3U?LrjxT3En0@bNI>NP+>pS~OCg-0*-=Y|H@My9x{dVe?
zCT|SS3kE}$2O_D;`m!b5Z@9Ar{Mo<o)+*eR?B+HUVN=dlZjy-NJ;rIoWyPt$ttpr)
znkr#0m!`xj-zu8K8P1Z%uP&1)xl4GxRI2g{d2taD9$}toBERI8$o7h=i8YHy3I&Ut
zl-?(~T6m(^U75!s&Wwxxc=2dUYDwJ^kCo7sG8Ug8@R9o@=L3$X%sc=6Wp@)y;@HZT
zAU07dj{hW+#CNOj=^S6gV<bv>l>Tk}qrjoSx%Gc0+dqL1ECN5@|B?Iuj5(Iuk;{os
zkKgaV$zN9XYNklGSfNE6PJhocU1X1DV_?c)EMaB*|Kr;VhHE_1d|$X@IQp3D7}?lo
z3uy9%Func%lO={-o%1JerJx?KEZ0I_8SW<Dc)9Ben?$||Rf;W^eXATGyH4?mu890&
zK2MRilI*JP`aLR%a`%+qEB;V6*F2=X$->dbQ^Q}@L!H%Rr@4($mcbN57R~+gtEH}r
zZjyeaa!LNMsDXelAD_Sqfj$va*<eW*{&JBwQWiqjxR&wTh&GAxiB1y|=U&Xp!nIz^
zRl!+eA9K^M{me`G@ADmFe*BM%C5A(V<38iGA9dene$#rt^BXtM3_%;#8Q+7x=rMBh
zZRFBrl=yM{qs8wDJYV_aIez><_f7Oe$eTZ}gg)mm++k__>+#{&^WCoxeiZq>o^du~
z`zOZ_Gk#6^Bk@m~sq4?tACvw${VVx;?9H3!tKRGX3;wn4xys{huW!9m`N;8e;orl5
z-v2QA@cV`O`=tyGT!$H@zZHEh`w;iZo!Nw6h2zzqv)@<$TFKJIS<bqXZJMyS!d2N+
z-rWq#SQ_|d^YL&s^CpVwOP!XUB6gaGi}$N!uj+l3ZxRwbz3g9j56DEScPsV^$#E(2
zOyOV7vx6g@`<0-d*a@K+_A&+!W&^glOp3p5eM$If$asvUiLw5>)z@_lHq2Gu7k%OV
zca)8b=K$xtzn{KH{s{gT$E3z~g*E-3{AZ~b^Pk$kJ@RqWhnP=xU%h|M`K$S3;X8r1
zF7M91kNYtBW!H<~Zy)|@f4BMM@nz<Z<$u2YKKa$+gUtKYA7_30_dECRk6&iLjDO$w
zW%6tKk89s%{CdTBn>mGH?Y}4gZ2!jo7vkK(>&#aC@7&)GhI`Dj+3MKUSpWVL`6cp4
zfN=xEyuXuJ9E6q$y<^S&ckj=Y9}Yii7$0)~<J-+2$Dhcy@6Y=GT5R5IQY?A@xBdF`
z*O@u#fA;73FW<d3`%%Xv%l?j|n9YWv`nSN3xnF#L@^ZKfcyjMy-Ndx{kKlL4AF~)w
zaII%^{5$FQ$zSR}YJNrhN&EHpm&~vH&&xhq{NVa&@y_&##Z&vYYd)yH)Vz1?rp@iM
zch@}7c~<t8^Nrub_jfGsdR|+9{{C6EvsPDD-o1On<*fLLf)jU7=bkD)(6_((+>h&5
zFAAL~Iq+sz?%vNw<{ZCy>f@y!m*P+DJ6?23{$%#y4F}RsxL^Epw)RxqS*8=5htf|x
zJ>zhu;auR^oHL8geLksl*73pBCljt4Ub4Apa%#b${reUlUUV+)O7QhXcVZs?c)IVI
z(cQV{cb?pK{^<2(SC*e`xR8Er$JO&!%5I*&C3bDkt#?nO9@t;uy7ltK%dZXp1%KOq
zwEq0)Ywm|zZ@fP+{VC_T!6C{b$C&fI>W%B`!|y`BzGpbad6C~+%0=;w#8b`?)&QPT
z(VsHy3PEzhGV5jTNKcg7FKH|4Dfm&?Mq<C@UGc+0>B77+jdG4sTGC0<ej=&@yM%v9
z=_x-@HI^3_-O6VxxLP__(M(1`@`*yV+BEqiVkbrPM9+vU5uD8B%;_h1OGH;-8y~-*
zy?`?RM}ZIg=Q-!I$#X9k$P!c(o+)!wtx&~Qd>PMP&SLKS0$)Vsg_U_$a+z?~aCC9p
z<NC{<#w^Ah%`r`oO>hh2=bwg58#%Ue9_KOQQ{Y+1CBwOfJ&Nb0n2D66&_$kIJW1SV
z1Sd;97mDCk7CkHVQ1rOSeSS68h0J1{OZmTv*Gj2NFpF0TuH~5_Xd=E(>aW~%nRQ}1
zBHlti;w4Jclus#WX?-%8Zj`IhspzTLt^84?U%ga!qTyqmx$1Lut{YA^WHwDT(>MHX
z7-n|BNMD;vZGq|w%_DkGjIvE%nVDN}vavS1uB)k5uJA~{L{UmLO>3Rj6E$0vb!xkG
zOf+Z8zms#7*(1^{F;j+7a=NsUoT*HJ!cnE=va2MFg}nt<h^R}LiF)uo;rPYL!8@PF
zlyg5@9rs3|<)Y<M>e74pb2)ms61mDbn>dcMigDc+36S|Ky;A54I|FMq>mH`|zmdO|
z{PX7I<nQ9R`v1cprk{Dg_Oqri@qRA)68WS0lh%(ghPdC~ekT0;^}G6W>)X^XkN(~N
zv+jrVcg^=ppWl7F`tiQE`ClKu&wKU!^_=&Q-x@#jf3f_1)Jx+>X-}(OYkp7o-~S`~
z<A!%SueslGf8>6<_sz0*zh3!17kE+fM&!eccY1FQypjCi^ez3D<nM$(6aPH;!_Kh$
zKils+KNbIlvsv)<@mX_lFl#WKV6^;i`8Vy?vCls5SHBB<YxKVHOW0S(FBxAozr6ck
z_aXVC&1Zpc8^3S<^5lc**S5ba{?vRu_Sxn~<KOguhkyS27XP#SXXcm1?>@Zjd6Doo
z?X&vlS#PgB7knZAnf?Fr-=gnWo}GFk{PgaVDUV~G9DVWY)w$;f9@jk*f1L5;<g-UF
zx!;}mbm`NN*Bc(&-VeSn^PuuU-_wv6D<1?tNPXM(;oFOhC#iQI-!gmt``x!!-Y*NE
zSv~T7IqPHZThW)QZ+CxO_p#xv-E-TglCM_0Tk%HenZ$$3cQ4&lxtn%B{;A?~zNd$t
zoP8wpK<c5}1DAUZkG!6Jem3o$^^cD~wtmh2B>ukh<L9s0U%q|5{(Zv_g<q%t7ySG7
z<<^HCpG&{V{!00`{cq~;O@BlGKm6_R<IFe9AKL%A7(f4A_xs=9?+j7QEKFDaPW~PL
zv*726e>)i_{6EUD=-0mYv)?U!&+@wch2e{&mlxjVy;=Nt{hg;bwq5tW>3MU-HOWhi
zrym>*IPvk~&D$mq_C7lQMB>4u>pw3CT-$wT(?f-Oao0{=*1VQ{RqEW4liKGKukv2o
zedYZnqszx{=-+?-(EY)ZTi35O-gLZMdDr2t$o<EUCcd2We%ssA&&;08ecJn+;njo}
z2G8W*@_%dnV*GZ?volXC-yQiJ__pIE%NwgV$KRfNpYitnyWFpWKNtUd%X*yo!k@^0
z{%iu=sazje-!a%Q9b@lf-^gCce^Tg)K(`RTz+B#NVGhA$o+IM(q&T@ZaPAX(E1e+y
zO*~LoL@Y@vLgJ@zHva)$O+ho^bdg%g-%<-ib_gU2?-osyN>h8TSFYQuVJH7hWT!}?
z*ixZ)BJZUdq{C$zW!l893cTlc5?L#~Q1-EmrScrD3Y`|+nYy)_Ta<<6<77|DPgJ@t
z?=4j*^+85M>ZK(A=+2VSoh73?OGbB=jP5KM-B~iavt(HBEJ;Z$NgUl-GP<*5bZ5!l
zkMBo!mW=K!8QodJJ-V}GbZ5!v&Jvx`oh73?OGbB=48xrz45MH)1V%$(aE1T?h2#bu

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/f38c61da15f2cb7a39ff02e69f0b00e99f37ec86 b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/f38c61da15f2cb7a39ff02e69f0b00e99f37ec86
new file mode 100644
index 0000000000000000000000000000000000000000..0900eb1352b10576bebda11b0e2b943d77877d57
GIT binary patch
literal 66555
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoaw3LlNFEUA1{2Y^YrAie=nB4zVznq8`HP4Z!%wrym|fZ|GT;G?B26|;P|-v1J4KR
zPcmPZeyjZP?8lSuB|jVgB>k=X<NjOWck-W^e=huL|M}<F`+vF2ui1(?7PGUn-DdG;
z-oVJhxPZZq;nRPc|6>0u8CEg{u?Dj5V%x#W#D0){Hd`@U99t>t8@59ny6n8HcbV&%
z_?UOIc(OcTdd)b4QIN6aFZ0hGKX(5L{XO^B^B-+LT7P~0_4mh{?+3qM{I>k-<1aTp
zulf}E>GVha4|m=teE9P2{F?=@zrV70{q1$n8@;zH-^jjx^U~sl>$Bfa@}6vYs`O0$
zvC^Zer;aZsz2bPA`zGl1(>I*&uDs2AXZqgy{hN0o?{>et_wnY}-QVSYe)_%k-&zI@
zCS#^`jQbcQ{#X6aWAtQfU=U_}&dASvfw_hGBy&5<an=&{AFT0A`b@i-KQS>d7X9Dx
zPv*bP{|kT1{`~$oheeY;hwT{K4K_p8#mpv*ul`p4Hv8G|UHRLx&jO!?z8ZXW{=)tB
z)0YLGQa*fnfAoXshe>bC-gLfcdTsM+;mg#Qp)dJf%zJwA$+f4up1M9+_UPFI&j-d2
ztsh1`sCdxuVE6r%_uC(Ac=-6yp~pc_YM-oqqWEm#^Bd2DUR-$T^7`~^w%7MwO?)f&
zG5z!XFOpv$e17+-?Nil9lMg;06hA%tT>C}stMB)MU#5Te{Hy!_grS#tBI`z04>k^t
z^X!*dcd;a~>|_yTOJzUDd4o4vfI(1ENLF~gP@rHT{~_Ka-n)DY1hxnS3d|L}BFrVa
zP2{L>hp>R~cY(XS`?<<FUa>Q9__DuXJ;t(#rH*A0b0dpA$5U=W-ap(ST-}^%oJt%;
zY`a*pSRS)vu|%*eV12=6!I8z;!`a1I!0F7P$ezR&%=(Z;n$?-Lh&7*eDeDII)m-Pf
zS987Q{J<5<t;?Chc8PT*+dj5eY`@tRIoLVMI2pLsaU9}E;+n|a&App*F2?~bJ>J*6
zDtxzjTDZzNjX1hlHCakoy*XZT^>J5nWpUo%VCKkUb6|;Py2mKXoXYZ-IgI%z%K`S^
zoYS~CxDIn1Vt3(~!&$`T#m&HblJ7KsrocBrUy*a7DPpqXr^Fp3rb_fnbV|5NtQ6Om
zxGVWYDo0vf=9G+!Y@ckeY_RMfnR_xyGG8TwC4PvX5;GFj5ZNpwD5M~KNZ4OwvDi1s
z3o_ee4@xUZP8F9IlN3o6dM@}^uv$<<aED-wP@S;2sFtXPNSAP~P@2F@zI5Iao)=uF
zIhfd8S>2eKnGQ2HGyP>=$g+S%m^G3`kcpRJ!QaMTzCQzh-u~YAt?m1ZANzi^e&6#=
z{=58-??2pr{{2z*)BgAQ-&=oP{&wQa(a)bhv3zWPzv9ERPm?~~_)z{%;f?RB%P*{-
z?|9PtaLxV1`{MV-@B2Iuel+3n=f{SRCp^l0<nU<aqrAu5kA)uDJbM4==c7lD8XrqP
z6@C8YxyOs1m$I+kzTkMV?RmiS!WTxb|Go|XAn<9*=N(^#zXyCv|2pZLz)$aAXMaxm
zDgCSS*Q;Mse*OK?``z;Ulkd_$JAOU;Gl`*}xtGP5#gSQ)X&;01f17{5{|fwT`G1pf
zD)SkZBDPkJt(>-;tsL?k5$u=PirKEP7PA$z7qER`S;;b=<vPm*)<jk>me0&$EHcc&
zjI;m$`_uDF{`c`e$$xhKzWqD&x5IDMKhyuj{XOt+=RdZ;3cug}T=7%)XXW>@uP;7d
z{KWRj?_<Ho7oV<w_WvC5sr19zxBPD^UR%9te4+dN+_T8%>z-#kU-g{%Wznlyuhzb_
zcq#pg@y)llBJbTleEQh&srl2vPf4FPd^-1~_8aeawI8lOtA3vT!TiJFhxN~^zn=U(
z%V5gX$1KJw&eq6g!m7%ApUH{2g(-(|JHrfye8xm3KBglK9t;%>4gW*`o&J}|AkMsi
z^(0Fc(+5U##y9^8|MvVb|9$({!ry9t?*7sHcj+I;Kf^!OKevCo@P*~`vk!mYZG0pD
zYSN3X&n2GQK411+=|$6vRnIM+u|ECvXzl~4`}TJi-2Qmm_ipsPoA;jIi@YEHK>6Xl
zhesdrJ<)kq`a<@l+>71M{a%K?a((&nh0v?%uWVljy?ydt^RxE%y+2uhG5?JFzUHgj
z7v0aPpKg8R{AB+*_)EgqiQm0{R{v!E#rZq#_ufCJ{tGjGWqQc0!K%l0i>-#;jU$ay
zncIbTAD<MzBL8Xr-vXxvzX;wHJSylRBrnV(TqN{Ss6ynLXn>fb*c-7q;`_w1L=TJH
z67CXuE?~>o&HaiqhC`NpDmxpe9#<5%4bKklU@jrfAMCE|dF<cWi#P;0`#8UHa&cGj
z81QQI{^Oa>!_ULVvxJ9__YF@qk2}v=9$#J_zIy(8L3v?85q*&i;R!+q1f>L4^J;NB
zaPHxl#W{)VGFJw-J?|a9BYXmU^LSVDtmS6lZsX+Qkl|?Mc*`!xK9?<*Z3|lly90*<
zXD*ivcO_RFhdG-x%LT@w|HuDc{A2z{_s`Nll7IjH{r`U}(?J#w);}zXtPyM{*$%QD
zU@KzdXPd@$pWTbonEM#-D}F}74#5+G?*v1HHVIu5;u9_wUM6y1tXJZ@M3N-4)D=l?
zNkfTvabxj5aX;}%VvS-?#R|nVL|+IC2qy_W6VwsfDC{A2T7p~ZzNCa?oWy7GSK<f7
zpNp}J$%>s6OAvo8?j=z!u~}k{M1y#~Xp8V{Augf&0?qv0e8+hMc-Qdca?j<O$#s)c
zmSa2HR@Nyjz05sKnT*>R+88t$SQt_mycxd!?`OEdkoDi?-?@Kw{}=q*@%O+Vxj#;S
za{es-E%R&n56K^eKd%1Z_?h>!{b%(Lk8f6=553oa+xB|W%MH($Jqvg`;W7W?d5;;N
z#6S7{#O~?RCoPZHJZgP(>CwkWL5~_9PJgKU$o0|pN4uZQe5UfE=4JaU&)3^uuX}Cv
zI{8)m%LOkPUq5;?`CZ|MH=llenf-O|SI2LAzD@bA@-y()m0t^gul~dSck3VbKQezT
z{&fBk{VV$K{QujGj?5>SWm%T6__A(f^<>pzdCQc*^n>XOvj@v}mN@ovoR(Z=oYy(R
zI9xc6aFlZ>vn#SqU_H)az#_o%ky(~y2Fqj?3l=?=f6PtH7R<MqSeVKf4>J}ron$Ix
zddPT`v4l~WVc)-l|IGhC{Kx#y>Tk>+o!?zQtG-|R8uvBfYt&bcZvo#+esukm|Ml>P
z+qcJ`cYXAEAMm#DjqRI;H;doSeSiMr{m->uwZCot+WDpL^X$+0U)a9B|FZ1MtS<|`
zn0&eUdDoZxZyUcK`4RqW<8SjnHGdxdk^1}iPud@gKVSaD{#*I~2g5_ALoCNx)!C}p
zPO*JqTgMj3%D^nf=*>{^-{T+uzdiq+{G0J_{XdugeheLqY)svZcNm-)a{t%<Tle?w
zpWnZ;e*6D!`+fKKyWdxScmGcM75n4FH}!98zE1e!_36qx?$<M(uYEG-(VmA>9_BtY
ze#rB%;X%NC`FqZHE$-a7_2A~an>@EK-C1;x;Q{}{#7Di4XFQg8Jmc}4Cug4YKXH1x
z_nE*8o0opCj=%c+s{Hl;*L`p7-);S%`pNdw<&P2{g+4gGZ+(~gw&Bf=Hz(ivyf3{a
z^u_R7?stnH$A3KdarlSzPxoJXzwiIv@~7l4-#@qiM;NEDRIn{#Kh4q1S;LvmnZn7>
zd6VM}#}STm9G^Kuxc75w@bvJ?@W0_-E3j5@nNWf7eqlA?JfRk$>q2ruwgQEGvw6*V
zxANTLN#`>W*e<{=AjN-#cLjF?X9R~UM=M7u$90ZnT$;Rl`KI#U5_l)bB&00lEj&v^
zQPf#9MKo4aO7x=02a!HePqDLN9^!|@jU@UcR!a0r%#tvbh!^`Kyiq7pNK!~bC_!k4
zP?Yd#;oCx{f-M5x0)qTYcy)Qd^9b|o<Feqq$S%YFl5HQGDtk7^Mot5+O`M4w``J`k
z7cr+YWioa${QNJ@Foi*!@h{^yCU53x%nMo4Syfn>SSPXGVx7e5$g05F$9j^jlEZ@Q
zI=3kACEiHBYkV5~F8oUTANelw-Q@G(za}6dyjG+^bg$SAaZ~XI(d8mDM50BQMQ#h9
z5&kJGE>bDHTxhFMp|HM)hG>@<t3;7xjZ~DBwd8B@0<je$^1|6dIzr+?YlNyrZit<h
zV30Z~^+n1=>ZpW<c(16k$U~uW!N>dy_*U@F<C(yHhpU`Ri)#(%L5>FY9JW2I?5x+A
zotT6e*D)+)NM|r%&}GnOFko<JFksMR5Mb<Ka%TR-WXGh!82jJ%Z|v`jKLvl@{ju@K
zwjV)1kN@=isr_Tix3VvdANRjo{^r&z?w2#3e|z@jnc(x^&sIK@f7bDI*ONt$86USk
zj(bw_#NtWSlbcUIKGA*p^y&R)ub+Q>apPs<tGBOQUU$8&e69Uj>2=ua_pg_{75{MO
zqu^($uVvqF{y6bd?04Cpi+?Tu^E31_Y+}5?^qEPKxr4=yEs9-+V=)I0XB+2wPBBhl
z4mS3$tahxwSnjhtVF_W?Wy@qg&M}4aJLe)U9qx-<He7LBXSl9%#dFnj&f>7<kl^^w
z-oSB`Q;_=`w-e6-Zf~wf97XK6SzTF{GPy9GXL!oc!*KV%=l`C6lm0II6a0JePtKni
zKkxjI`9Aq;+ZVUbS3kObIPuQ^o#^}D@7X>wezyG@_bvCE?6>r<Pd{(?81X*h?ef<J
zuL@pGczyWI&$o---~F)R!`JsS-dDcwe1GJ9+4~FcF1`zSzwN!#hwzWrKh6JQ@{Q+v
z-}l-d^*^nDef@Rd_k=&||DO4`{J$?l2E&p6>;ErixXob5u$Cc^Nt}5<Qvp*U(@LiE
zOfpQej3$g4Oy`*Vnb}ziS*Ef0vpi?YW3c{r>d)ri4}Nw3di~4t&;7s8|5^VR|8Mbc
z_aCX>Hb3pZ@A*3Ki`Hkek4*3Hyb*c5;HArpkmnZ9Q=e--?|ruA>9i+-kC`8Bd{BNr
z;9lq5hj(r6+dNqR;O)bgkK>=}JY#(R?)jz{A6{6zP<g)p*@kC2&-Oii^R)C?<8y_V
zvtO~iad>zAL+vM{&uyQxKJ$Hk^r`#Pua8eY&i%CRbJv&qUqrrs_{#m=^!u)FhrjuJ
zZ}`sko#&hCSBtL>-zvXf|8eA()Zg&`B8)E?9hn|7U0|NYBFdV=`i(V{J&yAOmov{6
z-e-Id`7a4@3N9DO5)2nE7YP(m7D*DBCZZzJEwoVJHvd`vDE{|+TKsGHvji9fKk*mx
zPT@YpwV87vM=6ICXAf5|k20SWf0DotL22O%VQb<0LRP{bg<lB!2r~!^2qz1xiJTWP
z73CLW6@M!xAvRT1S2SN_zA%sQGNCXbRv`|dBB2JM--7%?vxQ`Zt_XMu1PDCg|H;?H
zJCi$@YaOQ_hcBBNt32y2*5zzA?1$JhIsLf#xxa8&vCFaTVKHQW$jHYS#2C(`!+e%`
z8cP_fBU>>0Y<4bo2KEQ+^Ei%j%;8AmxWU24#m?Qs-Nf^a*Nk6KV4}ccfs+Cbg5`oj
zf|&x$0#yQ=1ZN066G{{I6>%075t}6@EWS^|L%K$WL*|0Cn#?Qdzfwmf-$?XJ`bwL~
zILQdeI?08|RmvL3lt?K^Zj{iIh!&R??+}-lI4z+i$tbBLc~(+DYL29c<V*=ii4WqJ
z#O8}M3aJX(3asHz;J?Kez{kQT!7IZp!?~S3kBx!#K65&A53?;x2x~gqN;YdYeKtQf
zHrDq{HcWe%ikP$+xBYkc_x;b#-xGg+{<iq*vagw6cYQJVa^v&;&+k8<`jq>j@}0+9
z_BRV(RlnT#;>wGLmz!Q1zuf&I@WuM)`<^{~V(>`if!BTCdwcIR-!{0t_V%_r^Y41x
zn|OcI!`jELPgXyPecJx)`E#C^4`1$l_2KpPx4Iu5eSG<;=gaqRaz9V}s`}IVPnvNb
z^E;L;EN@uOv9hqWu|~1JWvyVZ=d9w&;9}-B=TYN1$feDBjC~0k8{1;mC9M0|mU3ip
z9pIkD^OI)-?-#x+{N4iLf?I_0ggb?mgiVFY1*-&S3swsD2{H*)3QZKcBUmG#&i|D+
zgQt>f0f!vB5L-Q~IqN4DQ<h~+&l&9hAOCaer{fRHA1l6#emnX__{*x#7N0dgo%t~L
zL(<2}kLNy=eOUZK=wtB5zz+uRcD*rp<Nw<1)ytR7ulB#Tef#F^rng((YQ3v|clDj+
zd&T$P-=)3Z`QG~dx_3X`$$q%=arx)4ub$uIey;iT_}AQDe!nb#MgMa8b?4{lpSyl7
z{&Vo(&i}jq*D(Y#6)-PjZf9A*Cc*iP%b9x?S2WjRE*`G695dPV*%!0_;kd_noHK%J
z8}~ZyGn|vzr?ReQ7H68xVD|sWzuf=q3``8Q|2O{&{@4Fc^xw2U(|-5<=KXE=bMChj
zpNl>odmr%b;hUeY7rjn={q+^+tCKJOK0o%n`uWLc-=D@jO?-Oi>BXmBPfeaqcsk{&
z!qc)R_D}kr?s|Ul#rhY|UQB=a?IrW8?=OR2slK`V&h~@fhyNeueB%7F_KU$6$<NH6
z1-}%1IsbY6=c8XLzrFp=`s3fXx!;VxyL|8d#`f*!SDA0>-{yW@_oeWQ_1BJXem^Gu
zdhy5S|6E3S=EKbHEH7B{SpTvvW%FhC=D5Rohx;P06n`JTf&i1?D}i<Vvi$e?KMAx7
zMhdnGunTYsWbiBU_3{4ay}`GM|2}^@|9<{+0{H^Te0zC5bHC(T#Ceco5yvsk6Wk8G
zuX$JT_3{@8OcLx7QWaJasTO@MsxN9K;xD{P=&+EvaH{Ys;S>=TQEt&x5hvjwAxR-I
zp*ezA1^fgi2;>MJ78DV*6}Z5!$bX$Tf~THail>mblb465munVhA;&8=JJ!F<o-A8g
zx!A;56<BN7OxdTgd9kvx%x6|-@n`wYyqKkzt%Ktz=L1eV&Ir!kT<5s2aev}o&Qr+y
znfD>D5^pEZZJt@Y@_cc8WqgbH?(s$OJ>hBRR^+zj&g7oQ&BpVBC!bGDz+A9butD&T
z;BTP<;acINBBA2(k}6VOQmIn=r8K3FO1+fSm0BfzPexakQ}&?D6B!TLCYd=>6C}4v
z_DQl!>Pj+5NlR<Wq{(cT_LH71eM@?Sbh)&fw5;?@sb3NvVzI)P1vc{O@T&7vbH{SG
za3AD)z;TRCjP(HXVx~^UNQQ*}Z~y82zxkhmA@={De@Xw_|3CWY_ix3&%m1G-Y+_LO
zKmG5vKMQ}q`RV$@^LzC77vGD%-})-}we73!SIf^#AIskty<7Em{+rO(lV4@N+WPYK
z^W>-79<6xLdq4l)-aA3JH{H^_rFe7W^`L7vukOEAcEk0S#vQ$T&JR={)jZaH^5_Zc
zv$W?oUR1yGf7A9Z`(w$Mjo-d~zwzVr&kMi0ep~$c|Mv&O1m-I&vskyWy<*?Pah>BZ
z#~#i-+_Jo^ytO<fJiB-%@HlZ>asFm!<%r@~!m*OGo_iv18o!@_j(~?iuHYV_GT}eM
z_e8#m-V{40_CQob^s9)FD6go2=uXk;VoBne;<LnhL??)t3TF$><2U1r=GElQ<SpcF
z<XOk1$!W`Rfvu0_6q5tfW5zZHfq&M&@BWbauJQHeC+&|*KJ5Fj=Y#Ht@b^yd89wlR
zc=zt!+jVb0yv==Q@!sLXrjL_9^L?H6b@kVWUt7Oz`2Oq1p`T^H!hbLNed1Tc&-5SD
zzB_(z{r=>;_>ZI?KYvX9`RwQJpMgIee_s2!``3iu4S!_*sWG%LE@#wWJj5`UVHU$$
z21&+Nrd2HZZ1rp_*;Lts+5fT4WNTpC&Bnl9#U{<l%(92MoVkGM5CiA`i+_Fovj4sG
z$NI0!KZ*b1|IPj#{IltI+waxCbAL1aUh#|P*Xtj_-$T9zfBx`M=A+Mt*!N}cCckZc
z!~0s~<*R2rPdgqPK9YF2;K7XtMGqw(wLXe}^!y>;qd$+fKlXaU_e|yGnb$&ZbKYjW
z&3HTIZSvcFZ}z?U{Wj&j+=ti?TR-%CT>0twr+FU*K9s!=elPdl`@Q%FrH_oCls>t9
zWd7jr-t+y__ftMFer)&{_UYlLHJ@reNqn~aQvLPTx7r_Rzsmo(|4aSP!!VOUoN+p1
z3{wPi9?L$~PWCX)M6T&v^SEwvMRLpXNb`pBDGGQC9S~*^5fE7|VlG-N`d;*&*gf&-
z5_=@VB@`s0B_t%KiY1A}3LOyG!r#a*ETAA5CS)uuCoCiMULaXuqd<z_OTj`Re_>mZ
z^&)K|3L?LSGes;!---5!nTp>KSCP0O{#b0bXurrF;b%gLLd%7MgpUZ{5pEU^5}G04
z&7Z(`ori_nhO31um`j#(H~V=uOSVm{Y^((=D_F`|PqR9+u3|}N$zyS4v1ATmeEa{x
z|9XagMmy#L)=KtE9MYU&97gQrY|GdVvcKVQ;uPR~%TdNr$Ucd!n(YtUclI#OGhCKD
zUAzH&nS4%sLVO!}9e8DUWqFx+C3r9Jp5}|-FXms(&m~|bkS!o6Xd$#$SW2`|^twow
zh@|Kl(R8tN@%<79BqgNIOIAv<NqI=WmtH2FCS5M=Ed5U^TY8Sn9oa0oFS0vi9HgbC
zrb%v<6qJ&Xx+Li*sUl$^W-jtbNMC5HpsZk^KoWlzUpOxZPZrk(4r_J^HXhc0%*xEK
zm|U3;F$=Q@vbZvzVr*mh@IU<jfqx<YKQQcP6lJtyc=A8@fAGKOf6o1`{+;l<?APZX
ztUq3T-|>CXxAHF&KmK~>^w#{%hS#rOuX-c$w&~5IS6g0ecoz6{-4l-|`j7P=8r?s0
zx9yI_?b2HlZx!F(f9KoXzxNy-tbMrRQR3sACo`X}e)i&d$;<z*`rcT+<M{CP<NnVg
zU$egn{4o3Z{AbMXdw(?<<};Qt{bt(0{EkJDO^$sR$7`+vo+~_id0z5N;??I{$;-gA
zjO#yV0_P15BTio~2ks2+7hI8C+FTV}P26$3^8D-qeF8j!41!?-xA=Jk&IoK0G#82&
zIw+JPd|Y_4@Mj@*Av?jt0z3jI`8M+w^6un4&#S?^m3up9F8f^;6(;}xH~&og9rioo
z*Uj%WUyHsx|IGf`>y!2;*H14#9{P~={=vI1@A}?Fy%l=n`&#YwiPxTQ``_DsTJ(kS
zoAtLXU!Q%^`ts>>_?MSozJ3Y*(*IfaGwbKepRRnG^jY@nuWyY%`hUj#lK;K=_xs<+
zf9L&n{oV6h{?C;^Q~nnJTloJGg9T$aL-K#U|KI+#|Kt5XfngfsGR9uUS&Zu#?=!w+
z{L6TOiIv5HC6u{^sf3Axc@}dpvmuiRV+=#a|J;8$|8o92Ge|LL{OAAo`p=QyKYs21
z)%0ub&!8W?-zR+i_WAfH!B5#ASA4kep5=YOyVSQDZ%)37eEIVE(`PZyo;>Y(did##
zXZN0~zbt>b=f#)j)z6<k>v+cey!plBS2N%KdDrlc<K6Xl2j4$=Z}P$BqvfZrPXeDf
zKL&m9{lNKw?LGgygKu}gJ@vNo9sdXUPcJ_o`%>`5@w3LKEgvs`H2&20srl2FkG3Co
zeklHs`(eX}=#Pb;_IzgfYWpqsd;O1^pDw@d{VMux@JH!y&A(uV1g1z9CpJ!w*Bqxf
zj&ihf`g8r`n$E+@KTY7Az!3p+!OMb4LYBh4A{Jsj;>*Q1i~SU}6T2z4O{`z^v2cUX
z8o|#3&H^z4wSqf@WQF$$MG3hHEfCr+oGrRfj8Xit*gdfoVxD3NV!q-xC7PvPN`I4?
zDSJcql5B=-xXceJR;hH!rxJ@Kx+GF07EAn)5SI8LDj{-L$VIS;e=*-yzG(hG{NeoG
zya%}#aj)Sv<$liffNMLKH&+wqLJnJw5DqC0HuiE>C6-#|S<IJM<k)jLZ*etoS9AAr
zIdU%Ic*xPosl(OAmCq&5#mwcvb(m`+_avT`yc_rm_|Ncj3UCX|<bTPR%NNYo!gq-8
zE}sTJtH4izEWrf9TLPc>TlqNzW(qtKcpx}MxK|`mq)}v}sET;B#7)VK(o<xG<=4p9
z$v4R>D9li>QBak?Aj>DKB3mc>MfS0rmcn5LHU&?)Tp3a6cqv0E7pX^*b0i*#sfeZs
zZxRd?Xy&)%H{sXkf5qp^=gs?pyPeC2Q-ec{-HYuC>v>i#HWfBqwp2DLwuLMz%yCSu
zjLi($42F!q8NHc~FfL(u`v2Jffd4W7rvBCWd-@OKpZ;I3e_Z&^{{6()J)es{PJ18p
zF8%HIH(YN|yoq_O`SR7XiYH=^jy;(0!1Y1M{a1IT?s(o3zB%v4?i=AZg>P-Wm4938
zj_;kMI~(pe-*vsW?7rQ@(8o;APQIvro$${2W7p@8U#h-#eSP!w@wZFgSN|yd<@UGe
zzdFNIhJ}nPnc|sESvpx`*rhmsb6Rs9<C@C-hI=CSI<7sO^_-eqD%`R>F}$nzy!hYp
z-Qc~<bC~BYZvelYz#@Tz0;dEn3TzN)7kD6WQSh<Q0bzHMLXndq?xNbFDI(><1wzS!
zD+NyT=kTfUI`On~ujIPMaftOElN4j@f49FXe?tG{{z>}n_4D;N+piv9f<JqGI`OgU
z<DC!R-tTzl@pjuAjklg}7r*g(bM_7IyGQT8d_4O3%NN<NsbA_pPyV#(<Ghb;pR_*n
zeLnQ5;gjtrk5BtQm3+?ovi9ruZ}vYX{k;0?!SCijTmIDl5%_cP_od(3f9C#i{_FP7
z<A2rvBmXY`{r1P{&(q)6fAjs3`MdjH`u}bJbpGx7ck}-<hHZ=?Oy8M`nM0TtGifr(
zGyP;NWIV^9%dq1Amj9Lv8yL<qh%?mutNYXWEAPjxZ*AXZeB=K1_)F~P-5)zYOnqPf
z?&#YIZ$sX`dXx2L*6U5Le!UcU`S|&ZXWq|#Jym;l;MuI_YhGM?x#yMA>jSSkU$(#4
z^djr!kC$Sv#ow-f_u`%3yPIzX-@SUb>wWcy=8ry~BtHdw%=?i1{@}aa@6zApzdiZp
z$D2!U)89LP?E3WU)4@;2KE{2}dcW`;@B6>+^*@-tw|aNyZQ$F$H#1*Pe9iIZ#G974
zF7N8zYkpkx>D=ePU;Mvq`|kGh)2|1A*#28GHZy0k>adxyb+I|JKV{#@@rH9QcN?z^
z|6cyZ{Qvl~1;d0-i)<2g7n>?JTZ~KWqezs<F%fOi3nDrqO~R(aH-)`L4~rcWe=nXR
z&LqBEj8805OhV$kWQEjrsYg=hq_#*cl$s{>QR;&9Y#Djk6|(wr&T>~}b!0tdETs#j
zyrn9nrc0fdvXyR<_LrV4<tdpWK3#OJNPx(0VSnLB;YUKXf=~GO@Covr=dI^G!NbLq
z%w5RU#;M9F$7#T6#W{sTo}HELI%^QyJoY-yhg_4ng?SWs9&n5Eyy21MYv)f8U=oPt
z_vMe|H|0;{XAyWL@JMi;&;}tJp|yg{f{X%N_~!Dy<SF6F=6S;t!#k1pFK-K90^cg$
z6yE>5H~D7r?-Ixm@)ThbQxI<#caxBk;*lwmT_Bete?dM^;j%)Jg1thc!dLlpxtlVa
zGX65Xvh4CA3Wf@|<QB@5NwG;diB=2i3x5)NDcB_7$zR0B!FQ3jgEySlpZ68dJ?_g~
z&Ri=wmv9uYtFrB7S<TGP+{kp2(UZ}iv4&BNk(*)GKY_nTe=Gg|`|HLp(cjO0+x_YI
zz42G>uVueDevABG@=NPi?$2l6HNUO=^7~WpN4^h!?{2-8e>MGO(aWZnzAqm?uYG#^
z(b@Zl?}*>Ndh7Tt>svZEGp?sy<+_r1`TnKZm%d-RclqNL%WKWom)v-LGwt^3JBRN|
z-Cy^h^wG>GYR~Vyw0gt$Zq@s1AO3wf`629M`zNU{2H%{2y!pA|cjVuwe;fY4VdP?2
z$I8Y&lYJe#6~{FWHqK?7%eY=}#d7W8tl?DV+QjX~Ys6Q;_lEZ>&rNO@ZZYni+zPzL
zd`^5nc|Y-%^3CFB7i1E;D-<DeT+~p^Qmj&Jf>^Csm6(7Si|B3PcS6QOm4Z0}Qv92E
ztGQJ;7O@;=ocllS-<7`}e|P`U{&V?P_mAIS7k*a#wCUro4=wN4zUzCp=WWKDt*>Xi
zE`BZhI`UQi%chsCukGI!y%+j;^`qz~vrlh77JXdwanGkKp9{Vm|8nOG@7GUXb-sK2
zX#Bb7SH*9WKYD+6{Wbij^?$<uHUAd=z4>SV@1?)gf64u-{3Y?b@Av-SnZMJ1&H2gr
z>-?|g-?RUS{B!?rz_5&AD?>7)J+lF80-FGP9J>?yX||)R)+{fX?3gkb@BH`v*YtPQ
z-^73a{$2QY>aW_LPd}%AU-Q-XOYvu+&oZAby)Sty`9|RN!<Wo2+g_Bu;C{LF<&&4n
zuXeop`D*<uhF3N(7ru~tvH!W@3$d3$Fa2L`d+G7A`MKt^OHUi0);!IBHt%`Qi)AnS
zUu}JL>DAKL&)-PCyZ`R}yOZyv-aEXX|4!;%_PaIjzP+t^Bl&vXD~s1mZ#>^Hy!rB~
z@TK<)zUT5!WuAyX6?iuL>E|aWpPYHp{*?2%#S5{QF|W?Ne*NahTe0`1A7VdteLD16
z^Xr>$aX;h!u>HTx;K4MHIhkb%OFqkAW;2$xtjX+l9Pc^gxfr?sa82iG;8Nn=%#+UN
z$3KJr7k|3IEP)OI4FLv$27$){9s*POxdj>oRfPRSx<z<JtcBTyo(s$r_$F{b@Vn4E
zVMmeoBEDkD;u;dck`+>6(vxIF<pSkv6`m{pR*F?NQ(mUTrF2>Gk>W?ii;4+~TNES}
zY~&Zqn#(+rekQ#^`hv8u%r<Fx>A6w`l7<rI5|bnjh&zg%7LgI|5lj;>;IH7b=5H4G
zC-7Q8PEc7$Q>04li+G7dp@hD~W$_&GLt^z}_Tr2Z(c<?-_lU5FI0{b`k`ekXSR)iK
zyj!?f*g=?6_>kaoeha>Byy3h-yfS<{`Fi=T^2rHg3+&?4=3UIo$A6svDSxxTZ9zj}
z0g>k-{i1V4t;94Wyd=XV_KKgB(2?|!kd-KqxFmi_?478dh?NkZkiD>!NTf)*$X($=
zp%eiNzC%1oJUKj!Jny)~IUleouv)TQWB$!NgN2=yo%ILHDV7M9AI$m8=}gBMotaKD
z9c4Pnw2^5iqcel?|Neg)|2q7+{j2)dt6$H4x%}e#mHw;z*Rh{xe<=L8{C)BFmhW-j
z6~8NdkN*DdTlZI=FLOWd{uK7{&HJD4PQBB3|MA_lx3^zkfBECtna8OQ&)v7V|KtAZ
z2Vd{=-1oe{|Ne^m&+hj=`1N4v!~Kt19;-i@_B7-9ju$syvcG=&ddr(XZ!f&x@^Rwl
zH(y`;SpM7Rui-!L|FR5bjH*okm~vUBuz7K;<5<ZtmvaJl6>lnE7GDA12fjaik9qre
zgn8t7ba`@l6nMYze&^E@_$zQzKuI8*e+%DDUK8FhUMoH^el7mld_sKcd^h+e^Ggaa
z3T6ux3SScr7ZDI`6<sOXD7s!GU$|Ikk)V%29$ypBH7*a%gY2JJOPDJeSpT*Bark}f
zr^-*KpQ1k>d>8nZ_=V^5(~s#N;@<6f6Z9tK&G*-4ukXK-c^&uq>8r;tt6w_3TJzfK
z?Y?&#-amf-?ETsI*6%ak@w|(9ckErmyTG?@Z*tyDeEZ<t*Y`U=1b=+_QS#Hwk18Lt
zKXQGF{*?Kt|I@=y>7T!Uj{cJO#rKQwm$jcyd~*Bv;{BHQ+djyBO8PwO%bl;+zKMO`
z_-*yK=ik|WasTQ2d;cE;!zzY0hBf~W{nPx{|F`E){_hjNX8vmW<^Su`&v!poe(U(W
z_Cxl&+Bf#EO<p&?e)?+B%Y83qyg2+~^$XV*3@?ISJbV%VQuEc6*J^Jq-?hAxd)N4u
z@9ow%xo`a6l)bt1Cj9Nvw;pdD-iW;}el_P6_iLBepI;w*d+WW<N3~BYKkxgx_Ivja
z@gH{IAAjroR{u5d%dO86Uvj@_eKGqS|0(g)iBA_l`F$$+xbOq#2bcGv@5JA-y}kVQ
z$h*7mcD-Hy=I@(>Z#TV-e=G5B?fc}9`#;V4T>ORk+l%k4KXZPX{@U`p<8Q_PDn@hW
z=`31opV)75xN$6Jx8aE3{LeXuYbEyso<F<~`E2<&@~a9S5V|I`RghgURB*PSq@a_4
zw7^jTd%;@*Klsb}Kk}OinhVtmO&6*c>K2p};N>^w+reALcZ#1&-~qpd;8x)T(J!Jc
zVq3)xB%`I?OBG1B$r#I>mw%wJQ}K(Ey-K-Cwz7hfyrP-{zx)|l0htey+7bccTH@2i
z6hvPMpAa?@))h?Q59R;HuPX3~e<FVd|9}430!9LJ`4t6t1+4`y2-FGe5|}DDONdc;
zoe;Z_i(sZeB)=ZtV%`Y8E`9-lT>dY-b9mx-9`R`MR`WjRQxv={R3%&_+#$>)Vj;3Y
z_?1ww&?CWPf?`79Lh?da1=kDu3Tg;+^40TR;+@8~o^K1^W&Y)Y7Q*o&MWXk_&PW(a
zU6O2+_$96=@krcQ{DoMfc$LI1iF@MT#4d@kihmHx6LS&WB5Wu0T3{c)Apbu;JN}9M
zHvAR5^4upl-8t(xCb7q|b8)D0KIJ^hS;qO2a}(E9E_<#MoR2x3x#YP*ITvx1arkj;
zV_(W9#u~xw&*aV2&t%WEknsW|19LlbERzo7B*sc6ZKfLxGykjqpYi|X|K5KG|J?l@
z`TO-x<sSmymwk)-_T?+<*Qn2_A6LBBe7EQg+v`&=g<guhP<b}*anwVD2YwIcKKOK>
z<-X$m>GuWi%iJ@%H|zfDhxZ?Ed#dw%(~F8%dT-j^&U<&@y~#(h&+1=yeari<`s2;d
z2fyR~y#Af`XT@Lpe?R|9|1<c%m7$x_mT?oq3<hn6qyO~&=Kb0IXU<>Ge@*{9|En+_
zVB%w*&Gepe8-wWodw=8pru;qmcjmv2|7ZS-{h#o!`+p(hd#1O{46Igc+3dL-?3^8(
zleo5Wf8<u@KEo-_p~05H@`I_BQI^5!|E7ONe=C1I`=0d8@#~t;Cq7Dl@OoeNUhTd0
z+t$}yZ`|K5c+>TY`(?t5Y0s^n&3ba<@!=<l&n%yJJU4%#{c_*SXD=_jocuEJ<;0g(
zuRgqre0}rvqBpnRoPMqFiuL7>7q?#CeHHY2<?AhPzP$B+r~YomyBF^*K8k&s{mK4w
z)|abaS-;=>UjF0J5AL5Ye=Pms`-A_-{vQi}RsC`JtNt&7!G$TANrs7mxs7ELt23Je
z8xPwLR!3GpW(UT4hI)qo|JVPE`TOxt%->0WKm1|)6Z*U5m&4CJ-(|j4d}024`Qzaa
z#qYnpje9HmZr{5nZ%f|XdCmAH_08`$``=D{*Y-Z(L+pps?|t9%zkm4d%DeOLD&DPn
z%k!4??d3P?UoU<2?$wdkdtW=e_I!Qiwc8t`H#=U7ywQ4B@<I6Hgb(o_Tt9TZ4|vD-
zw*1ZKH#^_%f1CKW>g|KK4DS}d-SH;&wZ^ORmu4?Nzu5Z1`sKG*H{P_qO?^B0t@OK=
zcgNqczMuJC^rPu#{jXf#U49n-=KX8@&+)$*gFoX{rXCh<wsdwGP6_S~9wFXDUI{)s
z{^<g>f^!9b33dvzi9Qp_5HS~>D0WWlvS^fufUu{~M!}~7(*z0y_X{NnYY8t9DiS&?
zq$?~Wd_-u9P^i!?K}W$p0S$pB0XxCPg1ZDi3T_k96geV#T5PKLM~M_EQE5TxLsF$u
z$x?Pw!IJ01cZem5DT_@My(;oa_^42}pryclK1bdlp4U9bc~f}j@EqlF<z?cH;4$Pm
z#q)r-lJ7mQBCihbeBLtNeLRjln|WCHr1^dMr}4iOs1!02HWd~Y{x5V-Xq8Z>&@!R7
zLbbxpBEF)6qD3N8MC?TSMOTad7WEa|C{`fuFY!ZSx#Tpdd}(%>N|{qKA7w;kBV?mx
zugbW|M9Zv^nI@AevsT7e_Jk~_oP!*@{BeafN{f^eRP@z?HPkd4G?LZJ)y}E@Q8}P|
zM(LW8k@90DW~B_pSj9rcR|*yKOJzSu+euBAkQcuzY9d-BdO(y>v{YDE$V;$M-~xX*
ze>lIAK%k(wkdY9B&_+RD!DRw{{7!rUyjOU<c>=jrx#T&`I7HZYuzp~k&RF`t@1OoZ
z*1xxZ$N#zYxAkBAzl?te{~iDL<*&-0A3sfgT>R$yb;sx0PcJ`i{K)*V^}WTr?{9y+
z6@FX!`uWR_7i!O!J!N^){wVKZ&4Y;hr|*8eoqp@vO}AU+w|Z{r+~&G7_0GLJ26r3o
z-oG1r@7O*0`}gkudcggN@v+Pkm#1RS&OiI{-1Q~%tJ+s{Uu(QIdbjOe<a^x@*FMO6
zy!m1Bhrb_sKDvIa_@MM5@x$B?`X5-{OS}tw_vYQRcW2-AzrXe2!bgcuJ3g9zWctYX
z@#TkyA0B^j`PlkN{R_|6j;}kvrTm!qQ|(vNuVcTIe{cC6_s8OI-oMrVn;C8~EMpL2
zi22|DukbI=pZUKke=hrR>HCCluf8~b{_;`!qxOdx?_A!dzx98Y@b1~$h<DT8dwn?Y
zKJML*H`iZRy;}T2{Q1UbMbDqTD19~Yb@&_Sx1ZjgeJk;{@r~sh|2OmA+<v3{_U+qy
z?~LESeOLBQ{+-;r(057i=D$1te%eQ+&wgLRzJ`CR`QG<~?I+hy^`8+xtAAeq+4jr*
zx8R?je-8fr@i+hPoj*2z9sV8t_xRtL|A~x~m_(VInYCFyu*9<(u|8lq!qUKUpIL`_
zIa3|eL8kLeKN;^b*f4DVzw=+j->bh%ekK2m{-N{z``2?{c7JC5ocu}o<A(Q~@BhEM
z`)>ZbMen}8D}8_Y{rmT(AAWt<|Iy%6&!;n=u6+9cDed#}&&8jwefslJ|6}2Y$oGQp
zeBOG$ZGXGy?ZUV3-(Gun=e_v{wGXpCG=Aj#bl~H=4~!o)-dn!=_vXrLsn>^Jt$TI$
z)!kQzUtNCX_4@GZL$3v1KX^I$Md)+sXJSvcJP~+$>*>>H94~ZV`oA)G-Sm3a>wmA;
zznT5^^Sjj_GCm1>Y5ZFJ&HMYRABn%1|E&9)^8X5h64Od%f7VHC#_T=pKiKU#j<BC$
zD`xw~*3T}@A<bFGHIJK}=R9``_iwHet{0q#In%hdaX0h&^3M|J7c3Lv5WXQKCDbOE
zAlN2&LQq%ep3rq+UeO6+{1Q@<&n3@E&5-^ftt(S6BPe@Awp?zd+#$L5a{J_j<T_;&
zWu;^n$t25E$kfZY%e<7{Abmpmf%InSInvjqRb+}}SY)N;rpe1H<}39nx2XuJ-d1_2
z+@@5Z_)_7t!X*V6#aD_9%GZ>;RK!$cRKrvqRby3msfwyusRgOksI60bq{gkTu5PC8
zub!k{sNSRgSUp!mNOP~|56u?MR~oZ4iZo(0$~5L`{L)yfX|1KFbzZYhQ&Y1`<Bj?x
zb$j(4YSC)3Y9VR|RC!eeRL`p>sl}>UtLdwKSJhHIsqCl3r?^`FglvPfpX6on_hKJJ
zcZpmQ_7FZO^g?jIKn}km-$5QBZXM1p_FT4CtVdX5S(90cnRS`}F*z}1GJa#&&#-`D
zA;UoidB!)4+nFvinK6YhN;8=MWBa4}tM<qKZ>e8*e6jxG`<da>%nv5-PrfaDGyC<H
z*D-ItzR7yK^X>e%U*8;ht@O&|Mc}jkCp#bAe9(4Z@cx^7Q}1!!Yq)#tPW|njH_u)F
zch&XE^GlAG%r9A8(z(=i$^Y_=%bzcET+zIeeC5Oyx2p@U{=8~(&F)&=HIeJd*MqMo
zU%zoZ^2UxEqBl?9)VRI#&bPb2?ioDDdKmeL`*GvrdyiY6)IOd4Ozg##m&&hSyx#bx
z>#fu~rFYZbzIxN}Cg6?To5(k(-t2kH`kw7W#fO6*9)Fnn!Qq4P2gwhTAI^PP{qgn3
zdmmXo1%2ZG9P?%OSF`ViKNWxb{9*Vz<L|}4t$*47&i%9KkM7^Qe+&Oz`S<qUx_@T>
zI{&WuGv{~2FZG`ZKem5g`EAA5w6En~^}ZhZa_fut*Nb0$zpeas>)Xw5Ip4az`hL0g
zDfZ*#_jd1ozsY|y=S|ex>~~e~<3GIq@Zv+`2cZx5-hX``_TkNku8$I*R(-1YT=_-q
ztMS*LUv7Rm{$<mb6<^rC@_sA%F8tHw_l>_-{;M(OGfiOTV)@9tpZPs=Da&e>6c#&{
zSQZwR0A_7wU*^|LTbRO_{aL26X0Y3F?%@*S;pYwFy~WeZ{eg2jM-=;S)(<SbEZbNj
zSWmIKvVLSPVA{*D_20ZdOMjjG5&qr&+nz5>pLc!S`r*y{d+#5;pZvb~ec$_t_l)m%
zy_@px@w<xmf8Q_pkoIxY$K;QDKd^nsdC&O1>Rs2{32%P94tc%!71yixFRGq@dB*d+
z;Mu{atk3Q~Tm0Pr#ex?dFA`s9zgYGB^Rtj=&!5hHYW(!t6PBl&pE^F9{Osp5x97>v
z=RUvjT<*oT7v(QkzYKW!^u^*A2VR7}WPjE3%Ki0>H`CrOe4F)l&f7<CJ>I=}x8(iy
z4>g~{zC8UJ{e9h!^FP1*V)?`N*YV$^|7RIun4(yW*ygdn<uK%8;BM#k;pykK;(sjg
zTkwLAkcfk5p6C~m7Lizy5|N1_wIT;aM8zT{{z&D?ddVMCh*Dastf9iBvQSx8d9ISb
zl8};`l8v&Ss=4|Pjd(3vZDyUnI!kmH>c;7s>gMSh={e{}8-yDEGxRdDFgj=GVOVS6
zuOFy)P`60eTUShXrp|5c+gd+0murS;vTOd<6wx`V+og9%Z-Snnp0BQq&JC^Gn$ns*
z8rmA18hRSN8Zw&uG{0$1))dps(r{A`QqxllRC7^tSDUQHtnR2Dpq{AyUG14Fhiak9
z0_C?#-AblPl}hf)8<Y<ze^Gv+T%jzbtft(eyiGYod8X2B#mx%t^7G}o<Q~aBlBtq$
zlQ}BAR9abPi%g`fu$+h-qwF1NC+R9FH^~r*6XJ>Dx5ZjT?+OPBhYD{KS|L~<@SATA
zuN+S+moDcKc0sl+ED6lB8T}X%7?c?|G9G4(VZ6xT&Jgzh?%$R_$9~WKz5Ms$-#dR_
z`n~D*qTklP_x-%`UEtf3&vQR=y?^?8{>ul?H$Sa=a_mv={nq=wkF}pxKi~6q`WKJi
zo&R<I{QNfY`~IKOzmNU7@oVNcp)cC+|Gr>;arb%S6Zgjt9_ioHzj^Z7tIO#Za?i${
zdVOriq3nH<dzyB$?>@O}?XI|85xb0b&Dg24lVRuI9dmY0-!*sl+dYQ+*!H*V7eBE4
zp!5;BV{ylqop3p|`gFvZ*t31-%`e<OTX`n)T=a!o7tUVFy}sv8=EE0H@4R~TKIe<{
zkNn?D|D9u)!dS@U!cxO#!a0q{pMSDIu|O$*41c_UkiZ_kUwnu8!}*!{b@-?7E#sBr
z)#LfX)x~v`E0t>^XAI|G&JJ!>zM}$i!ogy5BqvF`%6^wim5-KRBR^GcskE;IljtMC
zc>dKqD><Gpr~Fs?o&P=H>-;ZfUs%2z`NI6o_uJjC%f8wF@ci-a`;DJ9fAaqt{QdD~
z-(S`Lum1)7-TJ5J&#&K4ew+W^|1;-T@1K`{rv7R9EAjt3Lorh((<a6qMpmYyOpZ)K
z4Ez3F{dWnpR+G_`=@)Ysy9?JlZVA3={3rRJ^6%!?<p0ap#J83wm(!i~DdYS9Tz`jC
z>k71&eYABo+PWHTU5&P`Mq5{-t*g=2)oAN#U|UxnKYoAS^+EA1_p7BZ44+Sa()mc?
z(dmctAH?2|zn5_L<ed|D3h(CM*Lc+a_{`&HPaZ$r_N4yN{D+?(-hIgNF#f@%`&IYO
z-toWl?arLLNA7mr-F3(E_LQ64H^1G|xqtPs(2Fx~G(S!FKI_kNh6&8ySm&~TW3OSa
zVq;=m$Q;V_k-_Evo<9=56n<v@EdC|<SD4`^;~8dk)<TwO<|O9dOgk8#Ff3*eW;A9}
zVwPmN$?D9$l--8o2gh5ET^!XM=h$Re-Y^<6==`huZSuq8tHzffUv7V^{$cx*=a;~5
zo!`B`s()JkDE)cy&$_=m|M>l7`y2dc-hU^iPyZ+XpT{K0%)&B}BZM=WwU_lN=X1_*
zR#z5fj$}?%c0Ep8zFywh94#CT{BdF}qIU$v#Z+ZNCD|peD{!kNDfua9Y3b_vX|K^<
zr+?a5!j#42sbPk`yWVcyGg>P((zGY)-qV_>xmnl2aF+fzomyQ{y$l@=T~EVv#zDqB
z#utr4O;?y*HVHDQ)d|=5q4-mpSF%;&wfGuQbK%`W%0g~@_c-~ubb0UcF6Uw7t>)d$
z<HUQ3Zzta$o^PD#Y<F0fa`<!Ea=qqu<xdst5}YTfDXb<sRWwILTG(9Zgy3hPU@>ON
zZ4z(9_en5F?UnYCtCqKx=TexhD5WH&q^BICYOOA&!L8Y<6{dYrd!G)6E}yQLo}1oW
zok*P-x(l^`sI5`qRe7qoR-s6VMI~A#UCmGPj;5lfkmgtQ2vv3Ea|-8Wb7fD;-&J5&
zlvVzyvQ+t>!eP0O(pD1ZM4W|s1t#-b@KtcHW?#*`n_<TPdH**4Vf&N*hxxbQ&$=JK
zzis$z`f=iie;?L=eEzZW<H8T;-amRT{$cw&={E{5%b)f=w7X|}o8ji)8%u5_-)^`S
zeN*?wr)z@O*WM_+#dk;T?z%h4x3}M#cWc|thu2<Ro_2Y`73M46mwsKeydrVE=(@zU
z<5wH5mRxDRl6tk|>dGtJmsKuSpMQJi?kWE>tmi|{&p79MZq8YuGfz$=9hq|Q`2p(#
zya&<_&Of~USoEpDv$xLMUc7U@>}>8CnKQRfPdF2C-u_DS4Z&M)Z}HyIx-<2T#=W`s
z*FLm)BKMTxS=h^kZ<5}&zR7>B{o3Kpq<70cW_}U-zWSH-zxNF9nK+pjGVWyD%%sTl
zl);aImqC(2lcAI05aULsV@wB_%9)lh{b9CX`Nbr}?835y<pax2)?aKj93MFaxz%`V
zc&2cvaAdJwW8!6e_`mZ1ssDEWo&Ht-tNCa0xBO4nAHP2me;)jH{;l;(@W+v_$zK+K
z%K8xhzVlt}+v{&*-xa@gdVBO8>xZfjcR$p9Jozc<i^|vFuWP<Gex3Z~-)H$RiC@lt
zIq`Mkch+A?zwLe>`F-+F%pavcu7A1yUHF&DP{BBZ=_kuP_6)8xUSs}NetmumesO_E
z{Hc5k_>A~@ct3E>=Hljj%_haJ%3;XS%2Cd_n8TX&5u*iz|391G^L~ha_xTq5t>hc`
zw@Kfw{*e1M;fLI}cb}Jh$av%RV(yb256|4oyZ!ja{;LHS!!BrE(z%p<@%Y6j7e8O}
zzVhqR=Zj63e_r9e=5RgmX6L;fkBy%yK0o-9|E1H5z85>6x;^53B>p(($%>~tpU!yp
z?fHvm>QAGd3Ov($HtG4sH+w%jeQ){B`%~yw$oJ+ipTDSl|MzprpEZB0{y8(gVmZKW
z%q79&#<QAhEyp^J`&^f~Te+w3G6=BqU*&D#KPl)Ylq-<Lr^~-qfKy;Pe-ghwuMgK(
zb_ez=oE|*OxLdhD@=WB-;jQOc!`;Gji~A2pGUsb<L+<ySOStv;)(aE~D~m>nU6z<G
zT_Lkn`mdCmRI|ic@#B(mGOe;#<Wl5Y<-%n*$v#y0rCO%dqHC>Rs{cj*vEeG?HzwL<
z>rB0kLyhhmKQ>)uKH1XN`hty`eWK$k$LS7h?e*<mTK8IQwJfw~wUDs-VKu{A()OdR
zwCy{qZ1W7`5(86xJN;_?cX}nd9omUnGqvKh8Fjn#Jq>2*UDx@cm7~5vd57Xw1vU9k
zGF;O0B=sb`#C^rzh%t%U3oYk&;ce&s$)(QqoBal>74ye`mwr3{p7hJ<m*JoHe~tgu
z{PX^&_&4^C|DU2i6@R$?IR4)9E9G~}-xvQ3|8xFl_^0>B_jki@ra$q2(*FGY^X@Oh
z-%G!){N(u6^XvPsg}+byO=E~=oX&WT@iv3m|FwTh|2X^!|Ks-O(;wb{)Bd+G>9HoV
zDX@32Ww7)y=`-B<v+LK{pC5m4{AB)h{+Hozx8Lr+<9|K>KIN;)mn)w>e|-CK|HpeD
zJwAQ-`1xbx=LKK>eBt^!@k`}r^-r}Q>pwMrarkQYMe6gZkFxL2zW(|`@7a>a{~vNZ
z^nF<LSo;~@i?SEnpIvyo;Gx+A*ZXGo58Yq);PpfO$1G2uKL7g4=B?{HyLY<p>fY^o
zXY%ggo5inf-gLfoc&Gc`;)B_TMeldNum4c@(fbqQr*|J#f6)BM^D+D5k<YKc{`e~S
z&F%Y*Z#G}new_Z^>h0v$-(M|%CH`9MP5GPaZ@S(Vzm0j@{&xRspO@t?1z&%Bo%E*k
z4fC4`uhU=8e7*hkwO1=%UVicPxzzKI&m5j#fBxWk=X1svD_-n=vF%033#I3ap5A^O
z^eFh@!uyqX6YhxJ$+^>VckVrt`@inh+%3C3@ut>|@7I%Vu-+88edDg^1EEJsk550U
zc%=Ji=0lZ-CJ&C>|Mj5c;pzt$9>_dA@SxzK&SUeZ>@Or<7r#w^_v`JHx6|H!d}Hw@
z@Xh)++;11WEq_<_e#wVTpW425eoy(4^5gRN{ohx9zxZw1x0hcxeNOqT_2uZt3-8R|
zO@DXp!@|$=KA-$-`|Z;=rEjyoMSm~;Ve|X&KYk`_mftMrSeLOGv5Rs}<yPmd=Z_RJ
z6}rX$f{&Tsfo~`86W%pE>0Hk_1Gr;&AMm#e>Ilx}m*D-)d4PQ}+i%u`tmUlfEJ;lI
zjKR#?*%aAlv#w@gW0}Z2k4b`Y9is`e57T?b@65g|n#_xsgITy)mNEZhN@2RnIEO*=
zf8pQRzpH=${=M%v-#<R)K2{sni>!y)HnO@g$1?gdr2N15FX(?B!!L#v3?d9A|0n+Q
z`Iq>=m_dorhe?3x%>Q}+lK*M{efQ_+-<f|5esTT?`KtQG=-ZngOMZX)H;v)j|EB--
z|6Tt#|3C7-?Y}w0OvYP`*BG0bwb(Xr_;BCky~ZykI9+g(;1mIWzAHQoygPZkc$V^p
zgLcIU>=L-iU(T1ndx9sE=Lfe1&tIN!UI#u#frkRsf?C3pMRY_h#MX+hme7>ECizv$
zNqURa9;x|KOC=9WewA7yy;o+coUuZk;y1-c#c7I~N_k4P%9B-Rs`qH_*Yed`qd8S`
zl}5U{je47Us=B5+yV^wM4#kTKW(pPZiSkqAJLNm%x5}TD7gAtWJf?U^VTqiuY?t(D
zNoUDJlB`lEB!5ekN$^NcmfR#cONvcqqwGHUW`%xvU%4!q`;xX2Q^XU+r6fcpLnWU`
zoEEndTOlGTyhgA}AdbI=?;tN9Z!V8A_g)SSwi(P<7(V=a^iP4|6yss$9jrNQQ&^|5
z^f1RVhW|hHul!%lKX!&qjO<LhOlz6uGc__rG4rt;Vm`*4$5O-M$FiDb8;d)ODvJz@
z63YsfepX#JbGFHB4eYPj&aosh=P;EsiZbOg>#+2(USYe$_Ka-{TLJ4kmai-ZEWRvj
zSo7FEvi)Q)=eW+kn(YJYFV=^wt*rK}J6IxF+*taVe>3GXTQUD()L<-R@cjSq-=}|S
z|E~O*_<QzG+wc8fUw=OSG46xr`)hB?U#)%-`ON>xqesjSxbI}&+<Se`Rh28jmlZE9
zI&X4j)(NwtCl0MW`0hZ&f%OM=9MnAQc~s*#(}~OzB`0njXF1+_Z22+O<Mzjek54?l
z>%@bT1*hsyWt>tt6?V$zwBu=wQ!7udIH`QH?ZltsOedzCymq?k?3;5I7u+v~T-tr<
z>?Qxp0+$^w9k}%5^5!d**Dl@2x^?Zg(p|B8Tkd~-DETDsndghUFRES!zPkVN?Ths<
zyk5M0KIO%Zm;SG(zEyw!_x;+B6`vcw?EiZCo9Fj{AF)4ces2A_=V#iFgm27W1U~M1
z=lj<BjmDeXZ<f4Ce=YWk<z?lI2`_HEh<iEd726xuw+G(NeE0HQ|GNqAuD;v)ZtgpU
z_X!_VKb3z@_@e$z^#|+ENk8*`nf#vgXVpJj21~{xjCUCn{`dZK`2XvFD#I}be#S~h
z5vCVR<}CMEBiJ`^D09hh&){z2S;rgAx0WxCe}#ackhJhu;l(1`Med1g6_pjM6Xg?4
z5nU-NC8{p6OPE1qv&d=D<6>UoYsC2_yd@GP!X&yR-byq}Y!?p~-y<d}mMr#I%uPI9
z;-usQDQg*JStr><*+ViDr256BMWO}Vcpr0J<xu5F=1k_+=bgz{%OB2f$G4H^6;}YK
zBZmijG}|p!XVy6^TCB}%rtEI)KI}>Cui1BVh;shsu;!HG;^FS(e!@MSr-OGB-)sKO
z0*r#5f}Vm^0!IA&e5ZL-xSKguIDWD@uyL?WW36W~XFkl-%PhiD%3RL8o27;|fmMt3
z6w3sbwJdfl9n6Z%7nwwu(^wv}%CJXs6m!aONpS^pCUP)ySaHnc_{Pz~>B?2dCB(Id
zvyAgM$3hMb4o3De*1ya_OzI4>{}ldO{ayA~>))?`&;PqKWc}a&SLaXOFOMJRzx924
z_HE(!dp{a~CI7kmSN-4JzasxW|9kR(GJ_|h8<PTa0kb=^7_%00JaZZIVdnEJ3)tpz
zIB-qnUd@xn`<*wKkBcvtH-I;VSDrVU$BFwY=M|154k?bg?BCcr*zDLkSR+|BGAT1Y
zWGG{t$#{j4op}ZeFB?C*0DB9YDw`Br3!6Usb@rVcOkDQdlX(>Qa`=-4%ms@C&kDo|
zd=xk>=p!U8Y$9@9L{Kzbbg8JI*k&<D@eSgt61OGpN_I(Ym3l8#Ev+hZPR2;KTlRzO
z8`<Tu{<1bQE2Xwe+Dm3il!*TqixHQTXp~5oXpmr#^p?CW!6y+feow4a?2l-osDtPY
zQ5P{;u~(v%qVq%^2}=vJ2t5**$M3+W#QU3j7nd5B1D7P1HRl2LWvndBN(^WJ9Q`5i
z_2?(LkLvHs-e$bH`g+alveyc)lU@hDv3&deZN|HtcLnd(zMJ#z!&`;7r`|k!^YP8f
zHw<seUl+VKd+qkh<E7n;`e%!tC_V0csQ=)={qy%P+<$mq;X%j)&j-8@uHKKn-*9iy
z-5+=U-(GX8`sRciZr78qrd|%dsC7Q&?8DPsr>2|`Igx$*{!zChhKIcml^xu9AoGCc
z!7qo-AIUtHed5BYS7*A;ojf0Tq5Hzu3uYH}E{0#+dGY?mhZk>OtiE{WLg0mx3wJJD
zz2JJ`!1>DaA?Jh7pE`g0g32X}%bzZ@UO9ew_vI6pKVP1CW$#su>wj-NxmA1T_T8KJ
z&flN#aO>mNr=OlLf35T0;$z{bL!aM%Y5m&uRq*SbFN?nJ_-6Bc`S+baTz`H0UHo_A
zzeoSqFh(<%vbeGGvK?Yu&i<Vvk}HV&2X``0FwY|H8m>mpN=_y&Zmyl2?VOIBJ2)gb
zrm-(%FXRa0e9KwH#mT*d+mL5Ij|A^z-dns{d`*1rd~<k@@|y6k7ML!`A)F$5TKt{l
zO6lz~t7X5)^~mp)TOcbVdtN3{hFL~ZMqKu}oUVeJ0*AtNg_(-Yit36V6!t5GD9n-H
zC3jC&Lbgcek+ha{f)t<Be5sq#?_`W*zscN|c`fr^=9^5YY?$19`7H{Wiq{p*l=drm
zE3a36ru<v^k@6(vtx9@|Rr0)YQnKr0{A4yrJ4hQ!w@59MI48y-wovqf$V_2Ap$Y+C
z{&&1jxE;A3acXfDaP8#M=l;vhz}wAh${WMu#Qm3ZHAg(VJexl2Hx@J2WL8dAZ5Dmz
zL?%nd>i?hrI{&@+hwD$mZ@u59zgd4f|5E;`_M_?hm2W@4x_$lfWz!eOFH=6B`}FYR
z{12h;^WPP}z5hD*)$W(MFBx7Izu<iF<@w3y)1J9Jx&P4g0q=dAd-nHy?u$J1cx>?0
z{Mm<R>M!(O#=PWvdH==t7j`d8UpBmKe%bVL!^?Ls8(*5eP<npxS?#lhPj^32d2;vh
zm&fu?{y%>H*!#)iC(%z%K8|~=^VsI`s>kL}>YwC4X?vpnl;K&+GqvZ-Ua-EN`1aEK
zJ0CT^-2SHUv+fu3pM-yl8910;F)d-PV(DTnWaDF(<6z*D<vGgB!k;UUC-hdtO<Yp)
zq*RT}9$96%U9u--K1xrJj+6c;HAPB8>WE~hWWR)`xUyKL=zNi@!mEV&M5;xEMY+Yk
ziuH^C6Bm|PEY2fdCbn19S=2@}NYq4BSTsQNnJA-}zvw*?C(-Gm0iyp!tVHjLx{56q
zy&!x_FiXIczm@kA_erj5t~p!<T(ddFITv#laz%6ba=vF@#TLYRp1GXq1mjLdd8X}5
z_nG{en3*Os{bEXB>S1(av}F`!T+6WMfA_z2e>MJ2|8wH^r(dnVw)|rGE%RIGx5n>;
z-;KY2{Ob65>HF?)cHcIC{q?2m%hfM?zZ!q*{d)e(g)jAAPJRmb!1T`K&B0f0FDsuL
zKC^%J>Y3d0i07RzR=j%nrte+#d+iUuKCJu5^-27b!l${P%0B=8eED<T=km|zKihsW
z{nGjQ&Zpu}J3m%^$a;VO-Hdk??{ePxzSDbW@s8tN(c6!2cD^ZnbN2O#SLrWTJ>T<G
z?}_{4=tp7?3m=?%F#iGH{fN6O?`*uY{f@;Q?%P7QOl~c?Wpb<dX3b5tTSsoO-*&pK
zcZcWRzx(PBcRX}^bm9^3WAn#Nk42yScq04E<@vwo`Y)Eexc2hatGlmKUpc)>c{T6V
zy;u6L_rJdW#^hbfd$kYUAI^V}{HXfz-iO`~OF!s*{Q9x@lgg)SA9sA3^u_(#n(z5P
zFa8eyr^{f>*uu1iC6;Xln>(8+n*`e})|0I7Stqi|aNOej!==fi$rmngO>mP?t}v5G
zph&QYk;r4=a$yT$GvWI}M+EZ(7VyXN>+{d#Th9B0X9v$69(~?lyaD_-1Vn@$2q_DT
z35yEb3ojCm7ReVqDJCp&L*k3XI|&}iB1uW9#ZsU(Oo_5f<+9`i<o3w*$hXVilH-(X
zlwB_SUzSHMTkf3vT*b>u#>y7T5z1SYpDHg`j!^bc&Q+eOyiz$+`GZoUlBANqqLzY_
zyp8-v`8tIH1x<yo@;l`X<vry1WM4`Dk@_jQP{Li@UTlMCxac8~ZNiB{p9Na^Rror2
zmU9JhDsU=rCU7*e+pr6<d$FBk`NNdM=*@8QU;3Z(KRdpweVzCz<-_B5%<qoA?SH%V
z&HdLLZ|=NFf7|tT?c4KjJKt`3GxK%QEB}|<UaWnQ^`iWF+cVQ=EYG~2@;*8EXvM?h
z59A)4yMO1t<b#X{iyqWIxN!gTz4CjZ_ss6Sz1wrQ;O_LhWp~ftF~9Tg_U+rPw{PCM
zdeh`)&CPo^m)<;hljrvAJ7IT2?#kVjxEpo1>2BcNDR&>;Tk$~sQPJb8Pg<XKzKDM{
z;kDhH^KSy*&VIZ4?Yg%T@5JAmefao6=VQdjHy`(Ya`^1=IsG%+7pbp>-;RF2_v6UV
zt-m^cFZ-?hd*iR&znFi^{TBUg@w@o<qTfe;U;aJ+_w(ODe|i4@|No2OKGPSL7&a&N
z5A6LM(wzL9P8>z-hU}v37udG4eP!!qFXT|-)Z>)o+{k%@>kIcWo+REQyw7--@T&06
z=5gY&;wj>Z<C(#Ih^vaLoXeDJFJ}~|1*Zk)LyphvhuAi-Ze*Fw+|Ja`c#R=|;pYFi
z|BwC!{rmQp^Pk|qy??j<P59^V|K|T7hNTSpjFwCs%+|~^n76Q8W{qHrWRqk2&Z@{}
z$##_0merayjP(mk3QHi12x}JG3HCJ{;+z7UTR5yZrm^p2b7q^t_KR&d8$Vk)t1hcG
zt1N3Tix>+B%PHpnOz#-KFgP>Z`M>PH*Z=;1Q~#d-<Mn6m?;XEHex3N)@zeR|>mRN^
z)_h;_ea&}{?>XP5e0%xr{I{xaj^BL0ZTWWk8}B!vukBwRe}43d>(lX%44+CrZT__H
zlgsBNUy{E`|Csc%=a<dz55KeitoWn)*XQ5A|Hl}=F@-VjWbR-IU_HdD#MaDqiA{{%
zko^H$BinSgcWjT@Y}i6rC$c<aR%fnf`psCwsK;o`xRK%CKkdKbfB1ec`C0Sh&-ZWN
zwSL_BKK)zLSGBJ)UnhQf@HzPNt4}LF`FvXSG5+K0kE=d0eOCP3^||QFv9GG%9eyzX
zO#V6L=lP#+ejfh$>F26nTEF-Fdh@gC=fR)7zxsbi{o(z~{ZH!u)&Kbn7L1WhTbXaM
zq_a+8-O2iv)ts%AZ55jvdpw6BXAkEI&JUb+Tw2_=Jdb#;@aFSR5SS?tBv3CPB=}75
zfKZ6=F<}c4O_6lrX+nNNsY0uTE(<*t@)V8{*(G{jEL*%rJYPIs{I=Ku(bFQ?BAFuT
zB77pw!W)Io3VjorB_t*ES@4_SPC<6TFZ?g~RQYUq7jw68X0sn;&0<kwW@7SS{K%lq
z_>57TsfUr5(T>rT@dLwahFHeSj2cXmO#O^0jQUJ2%*UC}FmGm_$ehXimq~+ZA>(Jp
zBqk50Ka9H>+ZoFkjT!41*8Z3L-}>+UU(3Ihe@y=@`g8q{@1NG+g1;a9y7ueEFRtIV
zzZd?N`(yX##qXBi_P=-ky7Y6|kMi${-{ya{{rcw1%r6VSy#6xx%Yx4{KJEKx|8e3+
z&QDK1p8aV2$?#L#N9~VAAK!h<`PBW1_fy@+ypI_l`9HFJ-0*SDr^%mrzWn>F{YCw2
z)wfOGpZ$3Mv+LJ^UqQcoen$Ry{(al`w(m0EtG`)%Tk_53d(-zB-*0|*{W0;!-XDBF
z_x;@cYuj(5KkR=d{C59s|C{G`>91Em*Zgez8TE6;58>}QUvs|1e17tA*N5}(=e+ZH
zyYuzQm)S4kU+jL7{$lC#ThBP2DLmDEV*mKyqYIDLJ^K96<l*}VE)T;V-G2PxN$%5C
zPuD#SdwS_f`;)g%o<E)Otm~QEGx=xNo|Zioe|q4_+$W!(R6gx~YW(#1lSxnhJpTD;
z+C$j~&+k3B`{a(?oxa<Fx0l}5zms&Q@s7aV?{|got2_{SnDofx$^NI)p2a?&_2T2p
zU$5$3PkP<)dgp7lH}-Fiy<vX0<UQL*_D{DyF?@OZ)#ZoouiD?yfA0N}`z!JH@*n2E
zvj1ZLD=-u@_%b~Hzu|wy|Lgy<|Ly%}{oncj@qf4eF8@>bTmRSQA8Frrf1Ca7`L{{m
zU4JnC)cLjb_p?8>e_#Jy_wVU{OU87jlgzVN<Jq@!?B~48mB8c5Cnb<C7$+1jJVWHP
z=x#9|@mBG2@o!=qL`_6og?|e93ag166%i4gBFZfGP1I0KM|`pbv(yu*LsCIfwNg^j
zZ=~gAx664cq$)mDJgexYxJ1D~;fTDhe7)QP*=m_EX$L8JNiB(4;$Gs1#jcC~6xl2C
zOC(bCgs6&`gP6M5A<;h~J|bU*w+hz?s|e=^O&8=6tPq&We~)h#UnJju-Xpx*dENM!
z1VV-MMDj%)#Ja^w#I(h>itZ3SAa+muhj^j5i1-(=mtwr)IpXWYuZW9CI7z5T=t@kG
z*e@|%!bPG*yix3&XsM`}=s}TNB4MJlMAb!a3BMHl!k@~wkjIV7gF}e@18X0P4D%$$
zW`<M-aR#RUYX3g`o%5IbZ^s|WKaYM3{W1RY`uC;Z&3`Wb_4~h=A)T?CaRcK@#&--s
z496KJFz#S1U_8Tc^grwWFaKoz$NyJk@L)7zn#VMk$)D)~V=m(w2ABU+|AzhP{r&J)
z<S))&>wgyh?D*;SQ{=~qZ~EV!e!cv)^V`AikAB?!8UBmq*Xy5Uzpnf`_3Om1eZLxi
zdHv%3)%8>Tr_IlzpQb-||ET|Q@W;fT)xSRe(*0fgJN|e3ujZfAew_Z^{yp*g%x^Vc
zyS}u4p7iPX$LAlIJ_NsSeb@T7=grjDQ(vuodF;j4=P}Q@pG!VJ{QSU+wJ)c=+WNZT
zP5ztjub;mTc=PVfmbba@{=eJy-ui>WhhOjOKP>;~@>%KY=5Ig0$NilBEAF?{AM?K{
z|C0VIGt6dq%uvLb!8D8cEXyiZNwz4qAhu_$uUMb3F>xH`2<M!^xtNoIE0Rl%i<fH_
zS1q?a&s838-t)Zve6D<Xyt+KUxK44d<>299;t=3)<T%K|!?~8@F1rl-MYgMK+t?b|
zCa_In6Jax8Wn*Dr?q~YQ=+998PvY;o-^{;Kf4uy*>#OS5vM(<_{rFJxe#Se)cgF8*
z-l@L>jY4O?)qfN5TK0AL>zdccU$wkk^y1s|x6hQHNj{5wHu>4}XNu21J)8PW@|oPz
zjgLJZNj<!G-~V3kotwAPZkpWqdCl%B>*c2xRL?Izd-zPt8OJmKPMe;QI`j3^&y#mg
zo;|tZ<ieB6C#_E0K7R6e?(wAK>?anTSazcQ#JUqKCpVpRIz8p=mGe_BvR#(GGW*J`
zD`&1;z8ZGj<;L6_Q8&YH-MZy?JK@gxyJzk#zrW)_@xy%&IUlJ#>U}ul!S4HK?_Iph
zad+Y!uRH2@EbeT%Q*-zG-JN%>?v~yyyH|GK>Ot$nDUX*tb$fpH#oAYfZ>2w+{bc|3
z{P*8K4SxUl&G1+DpUZ!HhF*p(3|kr9nOCudu-R~Qa~5$~aVvAXa*1-Wa{Ke-@=EhH
z@d@%5@kjD!@xSJ;5J(YxE@UdADe54mCEg*vOnjzzhWLDOKZ$=5n<VE*-IHD@t0*5P
zFDHLj{-J`u;txf0<t`OAwK8=}jgK0Ww6b-!>F(ECt-scQ)riGd!MN6Vqe-UedlNsC
zJd<4}lT9j2oK1_(bj=@{U$<bkjI~&4QE3%w<70c%R?F_SjkV=IQ%<vE7Edi#T5huX
zWu0W5Y1wPeWnN|R$a0N^j%lVrxt6QyR^=J0-!-Ik4;a2TJ!`IK=4(97NZu$@*G08b
z?yj`7+(l&%&2u`e28Bj*3>Y+Z<QzoH1>5=Gavf&V;h4;Gi7%A@p}-W8&ysg#Ps*N`
znJo87;e&#;yqT<`%o3Ss85zk|(OQuV(XHa|r8dZ%knxdamfb8VDi$LeAto>WP0U+l
zF<%77B<3>=v;L?4U(LYA@|(?)Z6{OR|5txD{IvNp>vh%Rj)%XWEPtW#X4l8)??yjw
zf0cb#`*h!Z%R4h}NL+KjW`0BD_OZKj?q%J#c%1wE&2zP9#ZOC~`Mo&vLhPCT;}?&A
zKXrP#<e}Q#1-EYAX?!^UvC5N-XD^;7KIM6M{jTPnz&kJQuD>U9H|eI#)qR&9U3z?3
z{u<xyR}YGw?0<6nk>11LM-q<@-)p_`@S5JuGq>d)+<o5rf$e+H&)eS`K0bf#@I3vA
z(Btxlj~`S#dH6E<P4L@>4=!H~et!BT@+<gf%pVViIsdQxcKx;Wcm7|&zpH<q`fC4m
z^>_DQ7ylgk|C}*^=>S9Y->_eget!IQ=6B{F$3Lfk`TRWbgYW10UkCp^Wp3qI!?m09
z83#YN3U35=H~Uc*17>yR9@f>IhxzP8#3jy2q>KF&$mJ?x`^55vNtaQIDV=3C>l#*3
z)~~GpIl1|^@EUO&aPQ$s<@v?w#h$~O%c9SGp5e#ev%hctvi$Avr~2>L|MARqEGrnB
z{&N1V_|^NX_7~r;k3SxN)%v*W?Ta@&Z&}`Pe6;xT?90{9wVy0LYJL!T7xuFAN$ulF
zPuQRCc*gR&{{5<t`X7p3U3=7iC-M6FODE4?KPPuy_Tu`>Gp}@A)wvmSH|+k$`?(K%
zA4)#<escTK^9M-}J07ihn0-I`uFsuCcLMIV-W9lS{aE1n-DlR1_urSecjC_5TUNIi
zZ_l`;bYsKSw99iZ)Sqv;D0zkD>b@%-mwhj-zNCNo#D!;P{+)V%itqH>)5p)vx%~4+
z@tqBK3T_o%TXQkt+@mw@r_)YtJd<`|)203^F4wDW+27-P*!<w`eY+<)uNdECzq<L#
z<?Zcv>hDrt|9sK#a^dUGZ<^ofe%kZx?$6*~#Xr7$wf@5YdGe>s&+or*eoOn-@^#Z^
z>Cf-JocZqc>(}q*KUe-F{!3%9WaMVN$S{*(4}%NS0ahMvKS3k$lhPC9gyrAKipkVS
zZWfCb-7Bgu&L|}z>nxWm-=n-({l2=Z`Y}xh-3~ovy}7zvdT;cL4W}C<=zZ3Cu5(8B
zlHLz}G2?2BuhxcEYfN>GUhD7EJ*a(0Yn!%#ZiDVl9cP_dor}7=4UCMLj6WD2FxqdT
zVz%3CnVGbih}j9V6!TOICF^AST<6)YO|C31UmUybnQTq0c+3S&_nC5=7noI;&M;eT
zvD)&1rIMAM^&{JEhn)@@_C0o=>>TXUtToM-7zZ0^8!a%*Hrj7C*=CPDuYIn~Wy>Ar
zy{3W2{|z4*zA!X3oTAU8dq~Su>yB24R<1^!nu%(GN{{j!#anW1GApD6B%X@23V90l
z2(Ss9;cw;l=gZ~JW6xw3V)*cP``-uu=KsI-FYNE>KU4pH{Cn%ql|T6mI;@=R#cYOb
zkJ!rCSF*)2cl?|A>%#Z<U;00rf7|=}Cc}E>1I$Yp=l^^2OaI%6j~VY<-_^d2d#(C{
z@#)gXHy>YmeEE_4!$<f1?|a-&zIXHv!<{R)OK<(S{{E`aRq?A0SEa9cU4L^!<YvsZ
zMVI6+M4p>?ChSb%x!D&KFS1`!yc&JO|7OR{eYZ>R&wG^mg!$>}r^}x2cp3jv=6Uy1
zrl%H9ydSTBeCS!`oB1E*KEL`X`qA)1-COfl&tEKi`S6v=>#r~6UW7k?^!)ydM=#I5
zTJ-w!n?3J%KFs}){qftU+AmUHXMXwp`T7_A@7_Ou{L22j;eXcu(7#`QOa1QrS^Xpa
z=doY!e%=3B`D5z$so%}MFZgQub>nxgU)4VyesKO=@oVER%^y9VP2YFFo%B}mZRcC5
zk7eKf{#@}>{o9R?E8ZEt$$DAy{Lyp!mtS98diLxI=ToWY$6sD}z4Fbu*FG;hpNT&$
zc-r$k>9y8}8=rrEnft}!>x%E{zn1<w_hb9l{hw}pi2pF}qwN>|@AW??{$~Gu;;Z(1
zp*PQ7DZS8pcJ77gyRDyszwZ3D`G@zP?teOe4u7BdHSVj#*GFIGeck-y)4%mBzN~yq
zy#H7Jx%Knh_qOj-euVsz{%!U1_1Estvp!A#bn(;Yj~m{<eskt!_p<{}yq-osH+r?>
z&8;`rUrm3h`Rd#&hgZ{ISU-FAc*>*akFuZGJe&36(`&tVrf>c}i+;4@zVbcyyY6?@
z?;AeJd8PL*@58h&%0G2}b$)O9T>OFS!=w+X?|I&5eD40)`RC#vfj=2P9ly$blKtrR
z>CBg?Z%4lV_-z02^IM-c58vdx-S9^7b^QzTr<{)?9!WlSea8H{^d0kuzaJ)ky!Y|c
zr-`4FKC!;%d0q2j#|!`0o8L|O#PnVGm&{MyFT38odCl^A-pkuBzP)Ds6#1R~cjv!q
zrbf0K9PR8SEYF#jvd(6+VBf>JkvCDWSopip8UYc$7hHDioNS9Z<axgF2JqGKPU7z5
zEazb5Ji)b&dmC39Cl}{a_Ix&b_Dfv3{2jvSVjdDvQg>uN%1u`Ipg2v@M7~n?lANQ`
zan)?iNxGZ{JbM2%9;?h%VpZ^z6_hPfXjZM)%+l7=^)r}ZlxfJLm!X}mbxP~J_H-Rf
zJsl$@v(;uNO!ABij4TbN>u%ARruW+Lr12u7nfj4B;#$i!*Jw%WywW|ae_nr~uBrA<
zO?fR@ohJQOqZuYg%tS1^tPWclS$(yLH{&u&(S4|;qOGf|W#DRj*W|hJQUfWSBK1(^
zPx43Qq80C}N@<E{8|yObC+Y9kwbf!%U!=A~-9UYg+I9_Vo!h!kb?UWtsjpMHrNAqD
zR(h+fvBC-kX*p%dg~I;)f4FlvCv&{#Xy<5T%V!p3Sn`+YuhC!azk+|C{3-nt_~+c8
zroT`B+W(#ZtK!@4Pg6dyf4KO8=X21vQ$HAgG<@;>nEpQc?e*83Z_j?H{krGpu0Q+!
zy<kXX(qR<&$NcBj?>~Q{|1JK%k#PlU3+G?XAa-_^%?!8y%=uaMW6jT}zhD1}`_1)3
z`AfrxPj64XiG1VxrvHuAo2RcrUNXG+_l)<s>x<}@jW5@{JoReQ8>M&4-_3aY_?5-;
ze~;ZBy?Ds_IO5r^S5a?&yxII}!}HuHzaLDx%W=oyUi%}1rzf8-c{cF{<13bzWls|y
zZGM#c<m=-vk7S=tc$xU-`fH|_p-)pE8s0s0^ZkuGw=UiFdtmvn^*-~R$2UG*n{xHt
zm6of**QegNeq+t`lh<}!KYMfgZJoQ`_a@zMeYpQ|?$hFD=b!mKOME)-3Hy_ik1L)O
zKP!Ft;`RI2a<8gi_&odaaQ59pcaraOJf8nl@7af^_nz&1{`~2=N5>wXest|I*AuxX
zJkRF6s(mN?Zp*90=h9D_9w^^)z4!4!;nT2}qOTr4Klw!Uk?Di&_jcSZxX1f2`|;Js
znvZ8Z+VM#0iPN*|&$V85zrOlT_7nHlH{V#lAN$Jn<=@Ay_rY(~-^zZ-{bKUH_s80w
z+kbBU{`X7M=j<;sKkojKWH`lG!Ys%7n)Mk=BV*map8waF4l|!)-pDS*cTn(yKpbxr
zrzP7WW;e!3j6rO-dE^AQ3Oezda5JzoGwc4J`seTObAKNE{rp$q&#IpXf7t%q`SZt5
z+h4bTE%=l0KZjA5DUfLm<L3XIe-HiY`YHaa<Ij`-E0_*5Co@Mec`%&#!~Sd5ub+S4
zFz#lV%v!<Pz?Q*o#=3z~iqVr<o|T;~gT0n3fd9VGVZoKWbGhs|B3P;zs{hw89%A{<
zrpI=IN#eiepQ4}7zIl92{QC5p*$<{4E58YTRrvDs%d&6L-vhqy_~H1Q>#x^89)?Ip
z4#s`|i~ni-ee!$G?}Wb{41CNIEG{f-ng1}J|G)YlCqp=k1Lqv>XWZ*~_Vdc}uI9SV
zK9kLn;}91kw=H)s|5VZ4;s-_hgkJJVb2o4lu$ypj@O%|C6N?v15N_ZP<~hVEz|O`h
z$hw2=HTydDHEgnM8El)`0@*y-irD=)*Klv;>kw2DmKV7pQYLap=#zk|K%>A-L0OUe
zVvLeak`5A+L_LMn1r`eY6Iv-cPdr!RvV^Bpg|wTbzNogyM6ouBbjiiix8-wG{;M5U
z{j9i3&O+v;M6%dsaZ8!23T4V^%B+e~a>u2oO6G|#7rQ4eC{-n`FTGLHUZPJtT>Q4!
zdr?`jyAoezm=xrd)~L)?TcaAF#4rC-W`%T-w5F_tLXOf_WnYz@%1ag1<vV1HWrO71
zl=M}usGL(ht@cILLpea9LhhhkyTS!Udu2oQSK2JPd0KJm!754$_oVYAizRPMTgdNM
z_$ntZ{XmRCxR-xEPa9VUX9(wCjuQ6$ET&AFj87OY{g?U|@|Wvh^8Yyujf}S$O&MMP
zm;aghbK-ZQ@2Nioe;fRp$sobl{9pWU_phe!Vqdeq^8c9m>)-Fme>VKt{zvmq`%l(y
z9A77XtNZ@pd)cqm|4uSmF&+7@`Mc%Yr%#*SbG^OsCjNcsr|8eCKkB@zdFA=s^GW%`
z()-WvO}lUQp!9zA-DS5Q-+Fkf@|NpOhMSJJ58b=<pyQ$0Bdtdl?{B&N=6c4}{g?eO
z%UzAX(QxbOt@pPAZ>_nWe<l3#>MPvWU2l5ca=119*0)<RH$AV}UU_hN<`w>HYpyT4
z>3O%{VZ@`1`-g8c-1NG3{xaj`y;pADIC1yogEtQt?!UTIbzAq=)*Ea$Y;V50opvwa
ze#E_Rcl7Tz+`D?O|K9Wa%O72PqWSF1^K&nyU!Q%Y^)leO#xvz-JDyK{W%u^hyQcTY
z-$lF;ekuGc{K>;7I?o+m{&?l~Hvaw9_m|(jd~@ZM_p6xKYHyt18h+&ZZuNWnAJsow
ze|mh2{+#%6<A>x=S>HbVdh*xwU)Y~5KV`pf{POx!;iuHk-Cr+#ulq6UyVv(gKQexu
z`~CLMzJKo-Ygsedudy3)d}iOwn#^?N|KxwC|A{mHWZA}XhwD7|1D;x5As$XH4sJ>Q
zlS2DNg2jR*G^Hj<t&sRFsv`PB^nw_##2Km6a=FT>>H!+-RBtOP%5h5-h{=oQisei8
z$()p(DdQ@2Oq^NtlOQj@F|PryC0_=gHSaO*A6$RARJg0SR&us*UFNyUw@83Z_^HS)
zkzYcu1lagPd3|^;@jT-DE8rzmCaf*8T*!xiD^DoTG~Q%>Zo!R0S4Aqsev0lB&Jf(h
zf0yqkUq8Q{ptxwF<S(g9lI`MbB0u;Zaptq-uzq2C$63PT&U>G`p5rVFD^vIX|9>X`
z-tb%H56d6T-)z4se{TEv<7d#XX}_oZ?f4(e_>^fgvjwvu<Cnhyzm9)f|JnRg!RP32
zPk!+JQvbF5r^}C;uaiFUe0uYF=hsu;4*yvByZx`hU%lT9KfioG@qOd>89&;7PyH{=
ztjqeC<rib!-@qSQpU=Nfd(Zjt<mZ6z-oNsH+x@QorTlZ^_sVZ?zeaz%@%{b}?;lRz
z4Zh#_e(=YOpT57v{|5b^!nl+<h4lq%7Rz5oMuwb!&;Qi_>Hf#Tl)%EwdW`irTN^7Y
z)0zLX8Fn-6VG&|?<2ui)C!ix>%J+l2oQsR=Cf7k8C4ohvOQmJyr^p|d<Cgs|c|h#B
z2%o5_c&L<$>}NT3`6$_aQhOzO#kL4f6lxL46}OeNllm@|Ej?8_MOs>VqVyM;b~zD+
zUy2#ZY|3X9805^QwI%XJcZxKL@k;)aDwUZayGk}(=9%Pf@dM%<Qgt#(a>o?Bl*Ltg
zly=EW$)1s3Ed5eiT-IE{PDMz4ow~Bx0;Soq2gQE~F5(U3uH{naw&p#|C(ftEqruhA
zZpD(tc;|ogf9?OD|H}Sm__6uxvCqdpPW<rvL)oWIUzERx{9O2}^4HlPm%mznI{NO%
z>zS`kygvVq<73}P)laKFP5LnB&DB?OZw|g~eYfSk#HYDma=*^~Jo`iHyZE<)Z!f;(
z`>^lx^dF3Wng84TTl{<O_nc2DZ-2a8^g{4u#;cpJ_r0-vBk-#9xx`c6N9XRnzsvHV
z;Zfe>(#O*uZ+<-K@v6r&o)|xS{yg^O;aApgtlsW?Q}gQH^DR$J9v^&g;DPmH@29_?
z?t2#X+~`@xW6uYs_eAbF-1mGi=VAAgJ<k`v@P6*_H0trYhc*vuAC^4c@%+V`B_CKn
zb$z_>{`{L4FSb10^JL@G<IkO6AAc|V+3Jhhm%PtqA3wj_@pi*ItB;pIr+?M|_VHWA
z_mb~r-{XEb{HptN=wA_I2g_Wx-E13J)mXkUZv8*)->HA03@?~?Sy|Y)**~)vvSl$>
zF+E{^!s5dEmTfENeC{&tT+ZukKUoA>9x=19nsCVT<_paf`z&!(GD&i$c%0~D;b7rI
zBEsS+lCz~7Wsl1KlHMaJDd8fvR76U&Mm$AwpJbWjXNk??zeF`eKZur!KayA_<skb<
ze!XI~;$-<a*~8KjQqv?VC8cGjDl{l5tHh}`tL#(clV2-)UDj3JLUFUQwR)SDvv!>3
zWwj0!L8Ytm9`d@1hg2*yq_jS0hHKQS-d9kU-Yj-Y#7g9ZaE^$WSd-Xnk%NMd`Et2u
zv43MxV6kLTV~Jp1%NX(h+aLDde||dtn(_P9pUZz${y$-eVZ8Z&>)+46nSb5;-u`XR
zH_so;zbb#-`+4o>yPr?LmwmDPyy45ruQ^}me+&I7|7*o}tuJmLC%;pEWA}R98@CV2
zUl;wH{paRi>AyFA?)y^sUh1{oi?7cfJzM>}?&Xr#j&J&2J$YXHl>PDM2a*r=JzV<u
z-Q(oP9FGkiy?bEyaQb7>XD^;dzUq0a_Tk-!W$*3ZTD(qqS^aF%<ATRap1ynb<k{-y
z*IrC`e(3S)2iNYOynpV#<HN=$EHB(%O?f5o^1!nVPyRfTd(`_l<N3eWj34%X3i<r~
zW8?d8ukSz4dOG{*j%P2QPkhz%&i3QHkFP$wc`x$r=<83flHRnvlm3wTQTUV9$4l?m
zzTfdd>C=bLD&JCn9Q+maXVGu9pRc}ge(n8~{~_-~@2A6GWq)4(z2)z<f2;p>{Av05
z_j~4#Uq9l0PWip(e<9O3MrMW!|5E<`{C(t4CxZ#wG@b&X`C?p>lO+4ZuL^GwVBq)U
z+sC&-AWqm-Oj`1mRFIU4#3xZFkt;$uLQjQR#Ab`F7VQz4Bs@(hMaV{2T|``Tmsq<*
zx}=5VFYy<mtRe@5k_A@?)ClQ`EtaU2ydae+GhKR;#CtJ8iB!ogl48<R<jyNDRBBgj
zk(ZD?E2SbiL*k(%qwI2pIm$Mwzg2c9Nz2cYx+6MM=%nCH!D7MFf|<h4gx3pJ@txtG
z#xa*gpQ(sx4YMOl5X%GRHl_m%`u}(St^dRD$N3Nc-}Zlp{_Fe~`D^-n%}>wo-@mAS
z{rX+$_oTnt46m3RS?ZY=Fn;?l{eSa6sefnw8vK9ESiro4$%fI9!TSHFzcc>E{AXhR
z&eq6j!*!5TfTNe?)Bl=3BEL(2N&E`>o$@b^QJ1-q<v&X;^G1fr|6c!{@h_C&1EVL?
zF2<J(tN*wC+x5?!L7izkiy=og_g|iaJc-=aoV9ERn13+dV4T40#^%ex&pC(l52rmx
zCCd%QZbk#fNB_AQRx>rQdb3Hh8Zmn_I{lCM`~1({f5}YWSod=HbD40(apbYOFzxs+
z{%_vjx&O8?7%<;ut>e(<n#bkOCC1syzKxxYYbnoUz9fMX!8Za|`3v~x^V<n*6Wk{}
zP4t}DVX+$`a|NgI>GCY)v|``ER>mRBWx$oemCdcqeUd|sZ3fF6=ABIZOj1k+%wa6o
zm^Uz5{TKZk^?T2+i@$gM3ueyd_|7fQyPU_5JA%`K{XVM#+jI5=E;HWQ0+)nuh<+2(
z6Aux+Cuqj+&8NfX$rsBvnRhj}IcFD}4NEBVcIHf$sjPe1pK(@lwR8G#sIcd;<}xo}
zTEIM;^)&l_PAl#jo~^vsdE<D#aZTa6#p%hhhJ6voPR=u&vp9O$wzC{%PGNq^9M6`(
z^^(_*KaOuFw+n|R%Y^?Me|P<?{PFoa?~jTf^M5S+;s0aRw<VuHzb|?t{>tYi`^(N3
z#xM3gSAS;wr1J5@$K_8-p1M94eEH+$@s|@`%z1Y6$%n@wkFPzt`}q2^u9uCka$dW?
z<$723CiRu=%jy@mpC5VN`6Bz(lsCL@gI~XWap~F3CsmJEKR)+#^UHH@=e?i#uHenL
zm+PMWePsK9_rBeIwFg-bGapMlD|&JArOK;UFGQX#do205_p#Iy{--R@1fL~6UGhZx
zN&1tTr~98VzIgER<?G@%VXqxt?SHxUg~;>I&tAVUe?9%p+_%%-dB5*`|L|SL+YN7A
z-^RV|eRJte$vfTmzuwM&eecEEXaAlSJp1_k?CXUei@q8Di2H8*HRaRaciOL=U+jF&
z^m5KC`!}!Ns=w#|aPULMhwJZ*-=@Fmd$Z}S-}@~eq(9|;UjCW!v&rX8U%J1o`!VbH
z?0=OEZyAgjJpVQQuKY3atHbB2Php=GzleT$_~r69mG5j{qdup7&i(xN)9TL;zP|r)
z_4l7Y3V)n_&;OnDr}3`^!$+3CTzdpA3f~g87nT<OE4Y%MjnAHM7k`hSn{bS%m4vNy
zzU(}?6>=@IQ)S-D9F#Sadn)%qK2_nLyqSELT(ew+e3e4HqOX#o@*^cz#WMMDxdz#L
zGS6h$6&+Q4)I2n%Xl>Qz)h^N8tiC|4U9Cs0U(HpcLOV(KrA~?VH_f{m*ERYy8MTk;
zEiid%VPGj|zR;xJkWKf1dayE|VuON<f~vwD1!tvR<;%(vO15&FCBKV?32zhpC-7L{
zqrf47*ZlkVLilp{cM67xG>d6Um`I9C){0At?hv{s@QnWtzm%Y$@OP1qqD#eQiJOVv
z6Hyh;72Yq*F3c@FStLWuLHwWSPNB2>bNO!avh&LD?H3dfdn)lrB2p|>SXoe+?<{8k
zn=Gq6YYQtATO|8r&L7-5ye&M<T-qGfY{G0i+19dWa-88<#j%=w9qU8pTg;PL>)7{j
z?BHzS3gXJ)C}n-dWW{*yzsdhPh6zj)m`zxwv&OS|u|_iQV?4*8!SL^2@84H{Ef~@m
z1sEj%P5LAI`}t4npTR$O{WfN}&J@9D|L^PX)SuN~T|O4Qd;8Y<{nrm?KJEQ_@MqXx
z$^X0mZT=JcOaEuZ&)Q$zznA>^_V?SrOaC_fRsWmw?;C?G^DE{b%zv5xGQIkL{dfG2
z179zGKK5z-C;iXMKX-oC`~2zC#E*6F7~h1vPJI>nqU*WmOYJwI?_A$8ym5Hd{p#Q=
zomZT1rhPp1egE(Kf3E(H`xW;y`TLj8tsg5tHh((*`PWy4pKt!`V+dy6&%BYTf$;~!
z3`TC|Vit4OOxC9?Hq6}&BL6b~y8b)=PygS_znA|y|5g6|;)nB(Ge7KqO8lz(6Zro%
z10U0W=2fhBS@l@Av)p9KXWhuUhjk{q6Spld1CI#TLyi`9D>fyz@9b?n3xs})ZWCb<
zVi35(vzon~>Ds?*e^mZ-{XX=|?6=I{`hP$E&iwu8`{mD<KN!9*dgt~o|J|W?=iV)Q
zYx(-XOPSZd-g10M_{jUY{)_LI#82PeKYbhc#`N{gSM{&|zFGE8?E}vzy)Wm#w0utd
z#PTWZgU(z3*Oy)!yuJVS+#A7HSDuSKuX%dok<8<o=Qm#~zP<c9?&Yax&QF#<aJ_Tk
zChM)Ww=Uf}a9j4?#D{iId!E-l7kt+7#OQIsqk=~}A6Y%-eDd{i^rPty93ON)T=Tf&
zY0tCz=O<pgd%^kK{At)zw`WtH89nEI`S=y%oBFrq@6A3Wzb|-q_AT>U{x@8&Q(o1*
ze*8B4-H+FkUV6W1ddBxe<8jv0Gp`<eSoL}Br>2h~A6wtAd%fYs*XQ$JEPWaOO6&Fd
zx1T=F`pWx#_qSbNpMRPBrR>X-FGs)4`T6a)*WX2dga0c2Q(%l@dBw`duFbiM>n-OW
zb|<z+ESs6tn2VTgSt41Vur22>=la9t#kGRtKD!aeUG~FldhDw?gLxkCy6}DE&E`uG
z&=NW=94=v{V5UA_>$P^Jj)2Zgtw0S|HA6KS4IQnU+OzaHjNL8%*zB;kbl739Xy0L%
zV#i=F<dEsO!D+q|pW_+(zxI5N{!Z(ixm=yx?zo+EWpWm9_-kunv)8KHD&0E8M%#9#
zU5LXx2N!!eyW_U9wtubTt@l|=+dj4PwV!DF!MewKnpLVrzIm<XB3l>7FOEm;n{3}%
z@3g2lX42oHqoS>?xmhhj#aMZRvagD{@*nxD()JQ7L?eY~3pNQR3RMc}3QF)Z^V;x8
z@#^z!=DWxLTJWUMW<gVdCVm@!ZvF)RU;$>q3jz@WPx$-z_;`{yL)kC0=CW*OVq-kX
zpv@%C?8H>akoj-gpW@%=eqH{R@wb&Jgsp(>2g_V$4#ws`UEgazpZE~|KIPq{x0Bx*
zzBl~D`t{@2bzhczy7M9HeapL+caiVUf6)5G_$lDS$9HevrMy4*zUV{4$B<9{pFBTa
zc|ZGI`CGp?i(fx|z3$EQw~yYv{=oV9(dU%UI-e(e-t{T+L&H1qcUJE_-qpNSe!c6(
z&F4YSwV%~J`}lI@JIzmdpW8lFeTaV7_j<;2u_rZ;zCWyZX!r2wL&L}6PiH*8@S^%f
z$#bvgK`+W*#=c^H)A6qI!}SlMA5z}Gcz^KY;m?s@^}aj(bpN&OXXlUd?|$EOzH)qJ
z_;%>qg>M_aZ~NK(OZjKc_pWalU!y<o{Gjyi<C}AD+1_)!U-GW|o%6dtZ}Z<>d@uWX
z-jAfecm8et_wt|lzk|P5{w(=v_Dkk>)*qvPQ4D6x&)NLAZg6*UzvQ~X^@{5ymp6A1
z&tYCBenEaFzF6KsK2?E@f|rFh2rm)-EXd6NnEMNd58F1D{VY|i>}<2y>e$;kIys+k
zdUO8d2<7bLFlW2Y;=p>Cbv>&I>v?7drvD7g4AcMF|4(Js;W)&_&t1=H&*s4N<*)3|
z+Al62b>GXr^>{u1Rr71dw+G&_yr2B`$E%tbY|rJMIXr#*B<AU<r&&)MpIm*s=P~;e
z$0sFE?mba{n)IaWQRjoW`ycPExp(E>{rhJh9Dd;Z(CP8~CyGxVJ)ZxB_esFR?{{5p
zzqtAB=8c<WH*~HqzxL&-+|`h)Tdz6aWW9Uve&BtRyK8UnyIFVb+U4@g;a5zqKE68Z
zTE}&ho6L9q-^+d={9x(*Klh^^WIs@N`2SJD)9U9ZUKG5X|BCCa<cIU0l)n~!H~$&=
z^WyglU*CQf{KWq8%*Te$ieC%AZ~QgyZ~Olv4Bd>1OjDUwGy45!`^Wu%2g5f;O=e}5
z?<_8C)$G0;m$|<4{}!1d?k2H7yg~G=P^y3#Uk=X<?ilVnTwGkQIM;Kt@c$GH7w#2S
z6xI-WEU=OPFyAh|UHs<+{s?RpxGb<(@RHCUVHr_3vA<%c#XgG4itZG7FS1QkMm$CG
zpY$5pOLE-`M-+V&?<!nTj8GC$+#tVHeuDys(pIG;rQ3>KiiZ>|6oM6bRSv4N>xk)_
z>UZmY*Ur^CpuwkcNW)0$rgnzzPrWF^2PRJ~{A~W)ZnAx1^T|fScCF1zn?AcXhYOBn
zPP3inI|)0#cCK>ia0zi)>n!bb+<v*Oip^cC5UX(Ooi-<IX4!18eP(ywPS<XS?FSnh
z>rjgp6AQyo{nxtsI#pUfG)}1Js@JP8Q)kp%ryZo*p~J7$tY)R;EO%cjO`KgMUQm^v
zi<g7jkaHf#Mb4vKMV##H=B!JZpE9mw`0zjJzw-aX{}TRA{Jr7V%U=z@)&9Kxll=Ga
zAFkiee$4r%_QmtlzK=gXMt!>cN&9o~=PRF8zu0|N{`Bjk<HsrQ+FyTvS^e_yiyhC+
zo(Vk_c{1e@_rn7Z=0CK09Q3sL*~F*k9#4Oy`_S_K*1PNPJifj2_R-sww*&5E-p_ly
z@7ci@LNB{tJbkhGW%)~om-}BGdo$(j)VKHFcD<{7zy8DKj}o5_emwrc<Nd<75pUMM
zu6`}~=Koun4{tsMeLVBY=!?{sr=Qio9{r~HUE$k`FV{Xl{QTo{*XK{40zO^)=<;dF
z=ks4@|2+Ed43h!NVP;XLmj8NxJ$`HaZ2!*p{opsFZ))EfzD0a*`C0gT?l09JYrmcQ
z8u~@>^U+WDJ{f$@{Jiy3!$<Xxt3KL&`uU0b%YiTTUoL%a`5gWE{ik1_EWgx#P5-*|
zOUD=fug||e{C4@f*w0D7mi+4dVerlP>!UB)UsyhG`?&vu{|B}Y?jMvsIe(q}W9Dza
zzf=D#`4#w6_=nH8S6^hmIDg*rso~SlkHw#Yzq0&X`P=R9x4*mp{`~v-pZS0G|I7?J
zj5`>z{y+LB|9{v2`wU)8Cd}5%6PZ>q?qRs~-}(RS|GbQUm_k@KusE^`us`SM<4omh
z;Q7m|&HI;&jq?M03HwgALRMWC7UoHes~G+=2r`MVM6hde&E)dr3}rvZ!p@ZQfAXJW
zKPP`T`C9Yo@`pzs3_czClJfoZ_vml&UuS>${%PIEJ0B}Qo&5CY)B8{FKE3^X`Rlsx
z$v?0Civ4}%m*B6$pDjP^zi<AQ{{8)T?(b*5p8Y!So7;EgALoC5{$23*)t|s$lfHla
zI`PZNPXZsE-&?%h`TE7HiLchYp8oF8$JL)1zvO&sdVk<e`zyB>dCz>G>OA@K=;K4L
zhZ>J6o=$z4`o`_u)%UYLeEzWS<LXZ*K41U(?0fN#s2`txJp8%qm)0MXf2aTVGhAnQ
z$uNiE+5gD@6aH&3I5RZ;kNfxVZ}mUF|I`0hF?2H>XI{@VkHPi-kAIQ>_y3>#zxKZ@
z!zTtYrim<h98Y+*2)-13BK}IeMf{%_znF>WG7&|QJYh$nO#<KeUkJzvUlmoAI4xl;
zu}%E3*j15_Ld`<PLMsJ&`4;gAatCm=a>;YA=03yyit9Va61E1Gc&3RAk_>K)YnlEq
zc{2ZF(PB^Ih~{MD{>t;8H=lO~_b<)`oHIE8bF^?w<G95!o70th9<QjNu$Z27xtxRo
zo1&fKONAW@FBF&*yA&+t17stlKS)|h?U%VMZ>8v<sHJ#c;j6r;{Cm0S@@Ew`DDG3d
zulQT3Q00(nv-%EAF>L|uUM*fN8O<hj3pF{lt!g~#?COH*J?gz0Et-N_hT7e_Ci-4_
zpR}{I4r{*C=u&^6I!i@VWuvmL@<ipys$3ean!TF8)jz8IS2!UvOX8|Xvfwpd0j^-S
z|4gSD5*Z2@Uob6YmSz6Q$i?{Pzv91pe^~!||Bd)__SdwZaX;j~mw(&(E$sXL?}^`?
zzMcOv@pJE|haY1<S$(<s_0-p>ul3(*e@y-v{LAxC&%YD@GXL!VQS?>h^STcc-d4Yg
zcyZ)e;M3A4i=SM4n)BT8McVV6r;(2}9`3#O?2h{#!8;<iBX91xzU!LnHR<b*Z<gGB
ze&7G$nMcx3e4bo*eDZPplUGmPK1q4<<*~t&TTk9S{rddms}paYz2SOe_FD8+-isN}
zH@#?n75m!bwbSc^uXnw<`}W2=kq>7-#(eVp@bk^0SKKfEKAZkz)1zGvH$Ieo)cNSm
zW9jFNulK)|et-8p&xhjo&F>W7#lK5>xAE=5H+x<yyiRzn|F-c%^p|Dd-~0^vt@6j>
z&&=OuzgPU0{B!oV|8LLVCBF~-KJ#bVKY4~(3|Ib({(JZP&(F>uHQz11fBJ6qWB<2X
zUsilc`5OGS>+AaO{lC8d-v3+T_vv46ex3UD`lr&*%AX~_%zi8X(f(J=_@AYa-GYOO
zBZu9E?LJF3iyLb!+X1%yY}V|N96LF;ax?Qk5Sk$RO-xW?i-fiWkGQAUF42!74}@cd
zW($<^JM!o9-{xN{@Km6RKb2RMM}>POry%ETt_I#Ed~5gu_>T+Z3kiu_5&b3ZE+r+M
zCHYlsiRg8aOyLuPdj$LiY6R{GSO{Jc>KEH4B__)zw^44soVu)?bfA=ql%eDcF*T83
zL1{h*?li7;?$10&cscl%@aFI&a7|#}#Bz~oEu#)Y{=cNZ=6`4Xz3^`V10$0a(@e%?
z3?2V-{vG?v`7ir_1Y<B$KVuieV}|*RQH)`XZ<)eanwSk3+5dO`P5yKDSMCp?Z!BM3
zzs&pO^O5_L)fbCzI^PYyPx@x~_28#>?=QR!e6#4)%onlGK0SW&@WlhChwmO4J)QgP
z^0Up))SqcSDSWu*zS@1|`zH5=@1MPAcYp4K2M?D$YJIHtWd7rvM^7H^edzG8>0#C5
zz^6P<&p&Q|eEjj|$AV8XpGH4Vc-i>s?@N=HFJ4}KCHm^Xi?c7jzqEYa{>Jc~z{j$$
zZ9g@C%l-cUOZV5c@0MTLzc74x^jYq+<R|`*LLdIVzx&bfYx|GH-#7pE|BqovW$5_d
z`ES-g%l}>fZvWx<<Me07pY*>8|4%ZOFiWv;vXnF5WID^#%Dj-}6zdeW(`-uYnH>K(
zA8>u<j^Ha0P!|a2yT`-F<Hdc6Gl4yybuz0CTLx=7>k*E%yo`d^g{FzT6@4PQO=Pn0
z1mPZ$deOt8-lCc!_l10fehG1lN=UqtVwY8sH&c{Q;!&Ed^ho)T$`=&|)l$_TYEhcA
zwKwQ$8+<g3HssXj*FB`|u9d8rrp2x!q@$rNq^+v`M{~8te2o~*xtj4Bo~qVL4vGgA
zgyr^2wMzO(a!G6v(-vPZ871qWa9Ke`;i~+5Ic}ME5_`lainobv5cw$dU7&%FhnI)1
zLtwp7vIv{lEwKtQdr@cMhXR)TT73FE{hZ7kQS4T1YOG?coUB>Qz6{6yl>O}cmi$HS
z^W;zLpEy2ty_bLI^490g+gER1slL7VZo*ru*SxP7U*3Mk{zU5GpL@seG2UN%_u-v$
z_f?;$J)8gR!1KZvY|rOD5q-Sp(b7jUkC-1e-d}r{_fF|;^E;;Z7$07I{N!oCbI})<
zo_~Ip_ssqo!?T@FeV;CP;_+n86ZWUIPqUwCK5u#^_|)%-)f3GpFP~ID&wq9JwfCDh
zZ!f*i`M~l~_w(<s5<mEV_Wp?c9{7Fr_o8p5UqinC`8ofW#&3&%cbItC*|=77Z{~T<
zYsNQ?=K>caw;Rs{o<44Ut`{8h+3&NhWar^J&hwM6PC!v`i$Ep+WZrsiPp(~@YdQMa
z@3Oz)Fz0OGe9oo8Q_gM0`IgO|bpy*$Rt1i5u9e)2dD?m9__px1@ka~%70egSkua5*
zBDPq}S^SUaKH*d$TcM9a_QJM87DBT`FG=c1>qrO4e3bbuby&PvbdHFc$aA430s(x5
zJZrdoITJX4aXImp@}~<j311X05MCtIFQhBHURY41UF4X^O%YYmFQRGU{F2iojU+_H
zxJ3;`cZwbq<CVNAZ7r)TYb#qSn<G;peP1R~-diDGo>i_<MngJOQdKNi*h=`iu#-@`
zz#YN!BDP{@MK_5o5>69h5GdvO$a$VKjB6q1GxjH}i<xs6H5fV=Oqo<zwzG1v?`QX8
zKg2qnc>-fELoCDm|4ILT{$>4V_qXuR$3KaGZ~o^0`RJSL*XqwLpQe5;`TFLo%-7s6
z=RYs_)bY{%<JFIMKmGZ%?L)%bu-EHfMm<Y-bm#uQdq(#*-W9le?XKU$zfW#G?Rc8;
z?Ci5!PqH8Fdl>cT%A*yJ_CIugu;5<m-NL*7@0C7GeNzA2@YR9W7hVUv;(HPQto`Y>
zCkGyDK7Rk`(xV5DvLDAkk$L*>3FDKOkD?wmKgxOh?+MfMyD#Rwba^T8Qu*cd7eOz6
zytH_;_RaCvysztCZF(90Lg`uBlmAbqKimH7<1>a=dG8WFp8v%0MfPjO*ArhRe%Af$
z_@(-5`B%3uPd=G_O8VsXneQv(kBr|z|I!%}7<V#!`xp2(?$68Li+`*9p86~Im)S43
zp9w#ze{B9y@tyyx_GkM~Z6EbMhJ1?nQu1}lSN(4%zn%Q%{QcjLh~G#5uK54(zy1Hd
z{|Emc{mb=R=vVBo++Q`n+J2}0Gi2Vv-o?3}vz$wS`zz-+b`kdJ?3^6Q>~C0$Sj3o>
znM#@TSUlKfaBSrM!6z>`MevZo8vY>uH2%kY+<c#SWBC&JZ25Tjmk1mX+$qQ)s439M
z-_Jio;E!N|@Jo>^qSr-#iq?vCh<y}GloXUvk?E3}BKcV&L42b~qTp-3Gkn$j3;AyH
z-sQIvW)zhbWfeUu@<>=u$U{Jz{{!DTzLmVExwSd>vh8B!Wn*SP#?i>#&nG81O(<E2
zM^HsTRp2#01HUn!GG92K4BuSdD&C8{XLyTw47nz8{A15y7h+H0=-_(J^@;Nfrxe$5
zj(zN3IBszDa@%r0<6OkPmbHR8|3AlX&7XIF-1~m%+syCrzyAN-!QjexoZ%0{Uk11T
z?0@<G-2JWdd&kem-~WHz_vOf!?_W=RfBAFEA3la&rXNht81FFTGw3oD|1bI{^zYKY
z{r^||7yn=IKbAp+LFZrSpPJwAe;xRB__yp|)xT+f?Ea+var_hXhvo0le_t6HSeLSH
zVCi7#V%f{I`G3~mnSW&dzWQtWFX_J_^A7f#oZMWqxoWsOxh1&yxTkU_@x0<G<T=5$
zkVAu=gM9~k4aXAB$J___{t2uX+#{gJufdne`+?hyYcHn~mo?W;PF>FL9M?DuIc~BA
zu)4EsV~%F7V)kWGVg1LF%2LmAkEMtupJh7RR?hES|2dmDGTCj|G+4A5dH(DF|MI`_
z|J{Ea3~!irvL0ZY!FrWhhxs|vEXLRWBmXb|KlgvjzmPwFf6n}2@}uyF=g;zAE`Or_
zcQJV|uVpl6$oT*Fuh*Z_-(A0*{z(1R|GVN3<6n_~!vC)QS@-+LFSB1Ce*XA1@6YDH
z@&A1Ozx_Y)|L1>g|Ccd(u^eTsV7<s9$tuJu%`C|v|F7e3!(aEm-~I^xTf`X8+R1jA
zHG$QQbrMS|vj@`|MsX%9CR@hL|E7Oceuw@#`b+fpoZr&_t}qs}RI}!@+OkY%DrZb%
zaQmP3zn-CnA>{vqe=h%x{-^va`rG+u<?q1XcYpKzjrv#rUyOl);mZHb|8@Rr{+DBX
z%6yyUGxIklX{JDi!oQ}!rhPx~P2@Z0_lWOjf0qBf`QMRY+W+<cB>t`Zd;Jg3AICq@
zf9?N$`YZM4%FmNOPX3tn<H!%MUnl?E{Xdt9nOTvs?f?FNdVf`Z7yRP?{o(iYKOKMe
z{dW58{d>_b>z^4viheBm!TK}um+>FjzubST{%Zcq{j2_G#-FTz*Z#LN)G*BXKlxwC
z-^yP*z8(0o@JrDbi!a(=Yrb#&wfFCif9L<&{@w9c>ffusYyTek8}jecKjnXdf42S<
z`LX`H#E-%s**`n~bTjN>y2bRK@%R74zplSye@^<Y`u)ZCitoF=?)sAV#q)E)$0hIC
z-txUT`5@}%%ggl_V=pbd`s&7<Tc2)Cy48Iv@s9ibc@NqjSU;40^yPuxJ@?!5Z!lkf
zca8OW-1V0?-`_p=;M&7g51%|(_+akC4Uae;FL=!MROH#tr`AtaJW_a6{iyCy+as1I
z3eP28KYTau1J{SL_uJkFzCZoW{)5@)s;@`C{rP_VNA6GSUmt#1{66*j^&k6xatx}>
zi`i5;7jZT4bn=PwYw-ICGKn%sa7YzM|CRnPT`FxW$uCwaGDqZ#h?Ll8iHS1X6mF{&
zsi~<tt4OGBSIty?q>`*+uac}%sjQ~>PfkRRLvD^-y8Jc;c9p{#-P-Zm)>>0Eo~mXk
zhbg%#Jy1HQ%%M`C9IiZ9`L>dmVt_)E{43db855beGD5QZWxmO1%2voENcT$ROD&Uj
zkyVr3B>hrKPO4BmPq>%gmnV$tBBv{7H%B>V2zNF2J<j{=-&o(Vykt>kF=x8PpvI8&
z&+K=__xqpYK7N1i_HNso$!}MDi2j`Q<^1QpA7{N^`*!K8jOSCHO?<xZ`ORlW&n7($
zdBXWv?9tH&U+%5ATYp#L-tBu6?r*%Wb^r4{_4|eQlJ17wTlQez!`_F*4?P~{Jh*UA
z{f^Yl;_ETjS6^?u(R0h<?w|XfkLEm*dKmP8;bH1S@rO1KxgYU7e*5^z<5iD(9xi=Y
z@hJGQ%9D3by<Zu9;Q6BV_4(JfZ^yrKe|_+^{JYdoi{D;<UH`5ASNuPXfq}7r@e-pF
zlLWI5s|3d`PG+vhTnl-$cu(+b;C;!TCbU&VT5P76y?Bv$r>L}0J%22pEZ<?i3IQo$
z9&smW6WNtASET1izm!gwF_OuZmXmgtwwKP43YG8^V-+hDJ0+SYdQMzMX0d#aqNI|l
zVuk!}SpjKXi6n7LiB^fP;u_)-Vz)$&2tOD4AUIoK9lxEx9>GmQKZT5h<AgJXb%d3L
zSOjkhDvF#HnJMfld{_9FP?A6i?|!ah9KYFru(xvD=X}L|kk^gx4v!fZ9|t?T4clXu
zHO#flx0n-{g&24Kb@?sw>&g$8@2=l&e!Ks3!QYI3S%3I`2mQM7z4yzDPxYTZek}Sp
z=R?^$fj668`M%_T{_RQm<Ag_t9)5fH{87x~Cy%5bUw^#v(W(cA59dF2fBN9*@27j7
zc0GOa#P?CaJ>}azH&t)^xXyM{?{@ROQx9)FI{PsB!OZ)o?^i#FcyQo8^Mi&5%nuga
zE4X8M>*mc>w~}tD+>*F!_2|melII;S*1zC*(f&N`S;Z6OCm)`gJU{uY@tOGZoacSd
zOJ3-_ocE&bMe?f~ZyMj7cwhRl<nymDQ@)<~y7$|mpKJeC{_p$$k%5IdgqfX*h2h-a
zH-BFJjrsqF;Vm;SM-{gKuNv=No-E$Gd_VbH1^fg{gmw#8iv$Y`3r-MNBPb~JL$FZz
zxmbeqEcsiC&lTmA0+jzK9aDHJFR93{q^D%2bW>47QB^TXF<n7ZK3#s3LZZ?al_2%G
z>LKc<G~%@wwN`2tX+P0@qjytpjb4|Yf!;lx?V9^l6O<n+Em69q)TI)yZm+GauVi4O
z$E{nZds;V3_m<9HomAZ&y2U!YTJzNJsJ>SDuFS2}punxPO!c)|pDMSipsJ_xMulf`
z1+tT59?N`|nJq0Zl`3f@VI}rOL|bIB&_qF3p+w=G!V^UL#J-C+irb1!6n!G<EFLDo
zCSENvT{u#hT~M00mh&UqTviFzA{JGaN37|bf4Dh#W^plc?qg?R7hzk%@`d>qvn8_`
zqr|^ezqb8Y_&xmVuTN(_MSfZJjs2(Luk}AWe<*yn`(p6n+}jgxyWcK(lk?`z>!UA^
zKi~7r>M6rx=0_Zlmp{4qG~n6Ur{+(;KXrY+?fHl2+%JE={Q2VMv;L<?p2R+Of9P^=
z`>moI)2`QF3%$y5?ZJ(kxBc#l-@SMH-K}T0KHfTfOZ2wjo#wkS_t@_m-uioe+O_(t
zORnTzesW3tYSxXwn@6vET?@SW?Q+E>tBY0_$}T*(7=5YY;_(aB7dS5DoL4>HaQ@19
z%L`nW;;+<PU2{$HrrmAPJE?d4?)uzMeOUZx^&{`cKb{mn+xk@JF~`G)_aEHjyw`Ql
z`{C}V!LO#im3x2jeb<NQAC`Rh^P&1v(wDw(3x06?Wcac1`=TEoej5F5|HJoRg!w7k
zG0uP7UwCwQgLo5oWVoMmyYiU{qzO(GIwQ;~VkOKVtRlQo=!?)7k#piFq*lr%E38oZ
zrF=p~L^VxiipmBxGmRdN*P73?C3Lj4+|_eb@|Er={#Gng`l+%+(^}U-PfRafS5>z}
zuUWrGf2V<=ai_^?;|+$V^<U_z>1pa7(D|oRrYB+~V#a9^YT;$kYW~qA-SD5DvR;6m
zpn;H4jL}lVz4{C~CYr4p(i*!p)@mu})ay>xyR6Hk%cA>Vd#2VMtyZ0%x~6(AdVBPz
z8eTTk)ZeYSQf0oPko;QNZn;*4P0I4>x79W&y_0K`JtDJDdYg2#%r)r=QtXmKVtqm{
z_#OFP@E+xT!LypzPvDoJu<#U-y&~zt#{~EBujYNjv!2(AuYfm>r-hrBi<?7=ErjJO
zQ#O+eGYjh@)+|;p)?2Iwte2TKFf{%*|6jx~jqx)>F5_Ef6&7{I>c1hsF8{dk-SBJR
z=fhtXe!u$D{a4mc-k)246n&rd&FowLSMe_^Kkj@d|0ecj#&g4GTOM0H6o1(I=>HSr
z7gJu{c^&tr{k7T4pl6&<vL5w5)P7X*DDi>bz3+F5ZlAm*e%t02$IXj3Y;H>3T78T0
z_S`#d_ss7Z-RZy0e7Ekd#vO*cwGYxCWjrjo7ksPbI{Wp2YbUPcUA4a%dbi`g_=AP_
z&F_cY=f1!E-iv#8@9n!MbHC@_$2*R<7v5~XvGTghwWzBnt}@;Tywmr9@v;4rM~`<u
z%766#;jf3&9`imq{K)c=;X{Xea(7(sNZn<=SM-4WiTSgD=QCdHf3f<7)ho`|_g}WX
zWP5$>P50X$Z~wkG`o#0;%loXik6%xHJ?WLqt7C6YfB5uy#aHRC)n6pO$^2;gdG^=#
z-xvNE|7-hK@t5Vd(T_*ps=i(Sn)kK;tH}4IzmEPhWHMxV%>0$n|G&r|$DaW|WPVxy
zUHos&Kk@$@|6~5;{4M<b<fq%O`rqw;zy4QYWMI-}{=&q?sQka`|98e%<|oXxZ0Ve#
zTz@&8I4-kZW=UkZ!tB7R$QjDZBfu;8Lcmnu2Y;`?B!Mo0i9%mRr%RlY43V;vJ}doJ
z>VssT<Qa+k5{IPMD6pumR)3-SP5Zp|JxzZNDfMq^Oq$Hv|FuqPhH7xCd#TM+tJ65I
zRjH$=?`>Rb`p4v`$pljY(^E!#hWdv6MiWgU&0d-Ln>SdLTU<0<V06Rat^OtbpN6bv
z_bpG`q}lb^eX=XI-)q}sy~g^#b-2|vi#Zl57Ru&zW{hS#OpY5R7@jqlZurc^*s|98
zg0;5wa|?CTO2ZueRr<FLRvIQ6IUA)L^yxL|R%pM_u+>P@w9~TF^49XzQq>aI*3!w>
zGFG3e7O(MJqeuO<TCWC!)<cbA)k_N3Wcs8QNo){%E><P!BHbuCU({MyMo2>7GT$=(
z)dHUd4)f3F>E~=^zsr)%sPeDt_qw0^zUO~?`2Fv%^1n0x8U5q<TmO6MkE373zS@0t
z`;z*p{zLscz1Pp4ZFs`;^v9D4kGmeNdt~!y@dM`vzaJz$V0+;7$l$5M^VKhsU-G>a
zdHMZi>dQk<_dHa&|Md3P8-6!0-S)o!_VMj!a?kvqh&(QMc;x=F`#ld1J<xe*_xQk*
z{KpR-1l~V;&*C2QU9UT0cXjV)JQ8{)`ojLX+|vb*+a7Iwxbe}vCrM9bo~b^!eAfHq
z#FJ}JPd*KN8vRV@rR!^-Hzsc@-U__E@aFvMMXzSQzVtTgea#29PYGX~zQuig{4w?2
z`8TO=lwYg7N_sQx!^=-=KkfKv^|9}R@O$ZZ@80&k-Su|*yEX5`-W9y<eDnD=^P9NW
zX)pgiKlk$8+oc~$KYsk^^YQGvb+6rCCcjX88Tl&jHOHGpubW@-ybO5m`)t~?l4nUz
zC7+#nsq$9qo!9$IA6Y)H_&D`L%g2SEOh3>6;`8nFw<}+#eDZu-`{LG<*oQs$=R92W
z%=ESC+oHG6-%Nci@M`;u{^!S@>%9E?a@)(LFT$RyKWlvw|48$}`#a4yXWq!WRdM&x
zgNKigJpKN3`ja`2;vaC|FL}`Vc;nOhXI@WVJ?y)8=uXL<b$2Z9UcRsLWaYEl&sIM3
zf9CXT=Ce~zJ)Q_YDSFEG{Lb?mFKS<PzIymP?8)>8=kEN!x#Px}o7?VoJlg#<<XPcU
z_NU*TY<*JrY|pC+?<zlJe7yY8`Xld$#qa;UpZR{-yU{hn5~FK|Ka7+$!}hm>uKQe;
zzA){a=J~jb+E?nYt-Ky{{q5C=D{n7tx!8Ge#wFg%$1c{Ke|4t&)Ugw>$2S};Jgjz*
z;XvAftq02we>ghv#OjmtPJBDob@cjS=R<V|nGTj7Xx;Cy_s!1G?djV(x9!|MZI|@E
z=ld`1Pv398zkmPK1H}i`51l>Sef0iOkE4}Gw;j8Ee8%yZqi+wo?l;}Xzwh_Hl?TO+
z&O2^&!spnRL*Mr+?Tgv>=77Kv&tnzGO^$aT{c-s8;qD{9j$A!*@5q9qs>gSq_;K>W
z=_%)4UzmO={EF?hs2dY+b>At!ci?{7{hxOO?s(ohe8cr-({0OpTn`UFc6wU$^y^dp
z=dsUcKA-;b+8gKh93M(Q)PK72_0<oS-~GSm{|ftg`TK>hSHF0C^Zlvy=f>Zhe~bPp
z{XhQy>whDLCdPeCbxb!HZ!)qnnJ}$l(qc|zu4hhQF<?{R@a78VjT5*oxI~ai@SVVW
zfhd9V{D=9s@f-8M;I-yS;}+$4$m=5@CcH~@tGKjeom9PapLDDAS?N{My3(7au1M~Y
zFp&5yZYv%m$}F5F_*I}xut?~<@VAhn(mcsI63@k?#iYd8#pJ~}#YH6gBrZ#wme7_c
z6Bicm5vvs4D*RE<T=0e9Az^p1#S)rQQ>CU#hKM(atPvFFSK*c8vEcUMRA<)%oruNu
zf_)KZ4cA((U0mn6?70hg4ET=l3kntrr3p(5^9wBzEE8HUEG)t)d{ID%ua`THvxuFA
zU6Rv>$C=NU?>X;sK4Sq{!CXNBAy;7ok<G$a1%(9e^UDe92yGGk$?wQ_fmet>OpsS(
zws^ZVyIhdmR9S1;eX_UYL==7~JW?o-e<_n8#V5%jd0(<t`nha_!X~9!l~1bj>cSfL
zHI{1TYyHvsp|wZLTKkF4XT2PQI|dmB75cGy6}kbsv-H{wnM|3@FPOQTZ8dvr#%4Cb
z^t_p}rIz&{>%-Qi)?cixZ5P=o*(cepu?@8=w!d$G#KFf|#?8*%*NwrY+Ofudhb^y-
zwzZNqtF@z*sO3TPJ7${Zi!2JPj#+!z$l5HhwzL+vj<()wZEkbWrr*Zj+SjVU>bX^%
zRiA}{S)7Tyal4_PLAl{>lYH}di@W9`W+#ka8t&H5(Jj@rFnDM**;vMSqEV00bt568
zQ~FK13v}M-eAPAAuQQlz6l5A;Hr1roXq}<B;RSsGJt3Wsnrqc}s(e@Im;ETMEwe!O
zto%nsL*;hmxysX(k1OdbiYi=`*O0#;drsz(%qH3Ga#Q8r%6yixl#~>oDpD?FFE~fA
zLHL4*xk#R{weWdCCH`LCIlL{r!aTa1cUZ45Co)GfTQDDG+RMnp7{M6CxSAoCaTD`)
zmif$9OzDhE7*_pv`?vnj{NKxeIsb6{^5nhTo2Zwsp3iyS`TXxQwr6fnKY-5Wc;fPu
z@!6GUtDbLvF8zGz^9?Vbzq<M|>3P%>wug`J{=c>7#_=14w}0F<zBlVm#I1lE+Senl
zbzHr8)#JL*jfm^9*J7`&xL$s9*Uh!peXr_WnSVL*veuRSYd>#3zw7#-`$6A>7Y~#l
zgxyQJ+j+P2?xZ`Cx65xf-*|JQ`j*$7{JS&n{=cJthw0A8J8Ji&?!SF->G8g&qR)0b
zt9lXoYRhZuHwRxkz7~Id`{kPFH=nt@IQq)=ZQDEEcfoJ_-_Csd@y*pY1#e%zJ@@wD
z8~4}guXn$#{b2g(!l%ujo_{*|`SWM>&uO2(e(Ct?{3Yen)DL&w#k^&IWBIz|_5RoL
zuVy^E^XS+8cX!v_X}XhqH}~G@d(!tgAAEVh{xIQz!~HvVSKT>s+xxcA9m#w3_s#Ch
z-0Qica;xMz)79&j)Gh^Get$XqlH`RY=Yq~|Iqh&t;v~-r-s8KEK0f^G;IRXH4j3Iw
zJs`I)Vvp>e;ysu4oZP!^zw{ydBb-OWjx0R9^>E|isza*}#vEL8AY}i_y}x(w+$FN>
z$j(VSi*_dLOx?L?XVk9#-P`vV@7=Q3X8+p*dk)SyWN~EqQSW1Kk6ItGKV)_=;DFKo
zyZaRmKR@PlQs$K8DeY5tP8FT@Jnec~>P+(41?O7M%Uw{vc<0jT%a1R8y)fncr1L8;
z$X@cgB6{P^oof#+J(}`Z_lfn>i_bb<aKGw*ZTR-`+a+%-U&p_$d9(fP{rCEx6}}1l
z;Qe*>kK%t#h8GMGOplmfvv9H<Wq0Jt=JgX;A~;3RP_SCSf!~r(oKKhEO>l=$tk4y~
zJfYV@TEbN#-eR8Ocg3S5EG4ds-xpsg5h#@_{Z8tjl&8#cSx>po^5>P7tL|5i*I1*T
zqW(yIhx&E39yMe2&FV|lq*baF&&ltQGm{IGOOfN0W0aGSyDzIRw^weu9J~BJ1x>{&
z`T26s<;xUbD0(XTC`^}sB^M?;Rk}?&Qg*t0o?@wzq7tj(C;7Q@Yh>-^V&uQbUr;!#
zcvw+YaiIc}0+;-K*+nv2r5mL`NbAbplrL0Xqk2L$OjTQTf{KE2fZ}5LFu5RES($B8
zyCu^k&q}_KS}(0B!zvphdrS6$yo%gXvF$>E;=g3%<d#aU5s?>ODZWZPj#rYg_V=&9
z=UMxCdikUU^n_0H<S;b;l4rUn_+I{{f}_wrhT8A<zwZ6<;m`U1=b8Dr6!>EKxOttq
z*}0$b9^}vAE#o;a7%b^3J6GzrpdSyrz)InAzFlnpn0QzYaK#Hrh@2MA5?v<vi%pYB
zo_mgTg{Fv(vZA{1DIP8!W?o-`GLbtH_vE^j9aRLBq~+9P&dZ6(uaSHqzE$q8dX?Tq
zJr|8m<pt_Cx^s0@G#%BMG&Kzln0HxTw~V%)XjZS?tE#RQY?k3N#jD@>j+vdVoTj(-
zBLj7lOJ;#K77nZI)ooKPH<(FT^jkEVL>PBiW;pSCT6=Oh?Xq5Av%$X7Ud-mdS+eOe
z>lIE<ooXFfo!;3Bo2=5|)7@se$jQ+2x>J&wkgl?3m}a5Y3(Xj90mC>`UQ=G<QwDMd
zJw_9aICP6NqBO7SrJDs?gc~?&Bxtm2pVv0hJfiYb@wf6rwM?}cs_JSpl=etu@bBb}
z7g-~BRK-Q{lGuKp+3acT^EhsBBy!E>vlVg^-Y)b(;4gos;5Gp(?qW7B_Sf9)f)53+
zaP4Eh!lcW3igg>)=0Dj#?0#<f&HAVR&y|0I|Jc7Tc<20P;-~3<U05s`Hh&d-8}Q=M
z(=AW$JU4t>^Ofh%(!Vl)zx)yX`~HvGuZqu0-lV;1d#Cd4!jILT;$E+O+5Y<9+m!b+
z-s!zh{<7$&%<tuY4>HeYd&(&N=lCB>mMXp;(Y?a4Trd9b`mXzV+2@w;7yhI$*Krhc
z7IU0tKf&I`k<2c|)b~g14;xbh_X<Hh0Tpf`b|#h%rgo+jMmC12j5@3vIcD=13LF*K
z$Lql<z!uB)kE=q!Nbn)|Jm$RL&%b{Cto!BVw_Sg}FkE9e&#;M^nZ1c)CEIBRhF>z@
zO@2>f$>O=do6hCN%EvhCzxY4VU$efi{&Dr^;$IK`-2E5*@9D4PuSy?fKTiAd`Nx6Z
z*}p}8^na3joBdkoUFOGypQJyX`ndO#)|X>n_kFMUzW&qc*R!A7zViIc@mt{cv#)30
zXTOSh?(~fL>CMOAAK5*6{`lwfkFR#V)_$%3%K7!5x2^9vKUIJC{Wam6*oTbQKCfQC
zIr1U?Q{u<W4^|(4e7f`{?pw~+4IgS=$vjJYCiW)z(}vHd-ut}#_c-lw%aflEuif2x
z=i%Li`|BUbKb-fV?7q}J#(OL7J3e~wwC-iXE6(Tk_m!?5Iu~+&>ZQ!fY8OA9wK;S1
zOyfD0^Im6#PE{T&I9zt5>GYaQb(b%nS3To=>iQ|JGh0ttoG3fK<+#X+$0t{xNjS@J
zdhLlDC#+5xo)Nylbj9?_s`GP>o9}<Rn`M9ak-Nu@PB<UCdvxo`)bm>|X<sxsU2&xT
zV8D@Cr*~hTdSlX!xU0RFR$OAay6oEGs~VRtU7UC+;>w|`ldcL~X1dUIj{W@pi?-LC
zZa%qDdTrsw6Q|0K8=RKAlzv_IrvLT$YoBh(KPq~<>~YNfRkyZX|8f1rEw%dxAN_pP
zbbsM(rd!gt!f)~1I(gIXR@-gkdp933Jx_hU`suW%ywAg4?t1&>)7P&--)z35eR%TT
z{+s;&b1dv^2bn)J?qU_>{U9L6&%?#ZQuRORUnavR7IUtdJdRwAta}(47^gD{GPg18
z|Lgisn&~&&GA=0|e;#|Djocr(Z}Zg(c?!=J;NiAo$zxp0dW>h201tm3rzo30XFmUI
z!8ray+*{d;SvXlN**0>n=bg$Q$a9l5g0Y?90fP|(!~cu__A-27QRJ%U73H1Cd4c68
zLkB}Cb3B_X`vg|M|69Jz{oMFt_CJ3Hy?=7QrGJJ0QTu=Szx98se=)ynzIA<?@k#c(
z_g^E1J%56~w!Qaycl!O$4~IU?dB5b{l=pK!`F(x;wfyUuFTP(+e>M8G?Eg{be3n24
z?_bT|FZ?+3d)hyS|5yL){L{uL#D0U*h5G~d4z4(kEo|-Vtvt@cA4Sg!&Eieve9V@_
zrqBMJ{W3=(*I^z9{s_TOLh2$@L`+1pB>ZHJ6?hdVDQ=aYCB-j3U&2y$vBC-Y%hFS%
zo0Vp3DeIN$1?l!^P1RVXS*T-b;Ai^CveSBprMao0VUi)IDVs&R`3j?}+8;Dl>UbMV
zTguup**n`$v0GsiXKiG+*tyY7#MQ`QulW|;8)|i`>Y8;1N)|tCn(Un&4%?o$*lzOF
zc(Hl6oxii9v!LT5dtW;y+ilj-mbXnO8P@7(X*^LorSVfI&uFLFVe`eNC5F1XP1?_N
zI`q8ty0sfs)nz5cs)e_U-II$^o2Icr<BsN2^+O7-QXG=uG8#%LYSL<rDjQWCROT!G
zlUptGNHR%$r^o_<&Aey%ABvunUL?O;QCi7Wp-FbXjGuhBa<KYAjm4?~a+}4q#C}ND
zD7307t3TDy*0IozR|%GLk$oiJuVSU{p%SbxO(9t6sp4;WbJ<j>OA_{Cc7hA}7=<Q_
z|CV|w%_?m!`A<Ale4SXTxTe&38Ex4tscRw^_~N*`xLf&3g-(b*l8}(Dm$)l1m-7Id
zHRm$^Ny7UCX7fDZzRPXJ{fUc*^9rj5i!{^Kzx#hP{=2}S!?cZwk!dT#r~mf<YJPM4
zj{Dp9ujP-`kLRCk-YLF$`sVuk9iP3wxBu>E2xofuFXGqxAFqD<Fg;*vVBOAm=l_ZS
z&l!>#ZT_YHI`@n3Z~ot$zx)4|{+;>v_uufp!GGib9r!Eq=jxxJ|1TIi8S@#0{&as6
z|2*&0-mjj23Ya%=M)U6z{LAOUrN*Abew{l_z>8m)Yc^X3yB&8tZzHEIb0vc;L)QPm
z|EvGAGVEg%WjV{5#gWbZoBIw|Dpwz81p6-*VWw$+zy4_Zy5_UOSCwBa|Cw0qIoLU$
zGynQC|A*X<3BSMnTlHu2H_6WrKAC<w{N?b+hi{g=;(7h`b;euC_jMnFKS_Ui`sMmp
zqwhuEJ-+FDiTZf;UEW*!H>t0*UmSUw^myVU-6waRZGCz9HQ%d~kB#ot-*taD=ef!o
zr&r-m4nOFA`1wi9^94@??-k#)y5V*s?&hl7d+yrbE4u4**WtdwBh@E1PfQ;*+*^4w
z=nBt8?u*|poI88?q}>UlQ`^q%zkK1E#kJf^+s=JI7kowK&aVfL?^)gmx%~Wc@{Oan
zb#L9hVsT#R^sSS^XG+fhxZrS};f%=XU1uvUwp^WXbMGzw8xJo>Tu8Z4bCvh@u{(ly
zp5KzWz5BlJbJI7A-mH7I_l5ndl#efdRWL4M+WGI!kH_C8|Jv|(^RGjn1>Wv@QTbxV
zYu5LlJ~n+m`qATk@`pQLF8p#~&|xZP)Ms44u!P|>%TG=Ru7zxmnCcm&S?_T#;1B2j
z#;eGq$$g1;g3u-rJE5=K*I8~eePP$+mFKz5roedYkKUh~|5O>%nar6E{d4}U^e5wg
z2y;Bg0<N{}Uzw*e?P0#l?#=z3V-?ftKdL{E{$gjG!m7t|?BC3vVL#^n-u8bg<0{7K
z|Ji<T`N8)06tfDOIP=s$%fBXnHve|?NB+;8pXuNGzU=<8>U;7ZU#51JV20Vh#J)>@
zd-lWU@3+4^zqWmG`gH4C6$2CdJhoj-LH{@YOJ$5^y}`bQeGbdU|4aT{{TIlh!tt72
zj-~S-`|q;fEPogNssGvaCHkY?CzY=jKR^B}Vp_<+_Gig=k8cLQPX4?0$Na0|JG(c#
zKBWB+{mc4i@wc`QpWYsRfB94Q7o{)UA5Gpoej)zK=l%81dY^^fu6fG!DC4oov%}BK
zo>)Avxaapk>uJ|(!B0hBx<0tPobt%<LC51+&lf*qd3gQSm79W(MBg3yEby`ImC+OD
zhoO(eo=$zS|8?T4iI1c19=x0N_~7%@XPX{;zV+z_;~k+#8qcF%a6g;zc<E!yXIihX
zd{F#6_rv|y>z|)^uKF(IYwWjkAEv)(eWLto&6lEIYQL_1U;Fj@SO1?5fAtx)Sq)h&
z|J(lv{I>E}JR>tp&HsoW*S}`{{P2&T`5nu1mN_gQtbFWd+_r*_BBurYcmug+ab4nD
zCA>o9lz=Q(Kbsm)inxe;oI;`;vn;<tyxJPA|2icGR}7cvIH@=*Sg1_b2{3Hb%~0i3
zEKsUay{xuOwMNNLVYZ@~>SfKBdQyf@b@!_~DrqZtDlgH{(weP$QFe*MTPYr;V70p{
z84AU+7vxu}73!Wa@-t_(SZugjZKEus%qj&t6;6fIqO*9{^IR5?64@p6ir1a%Dd#FK
zQ=VJ=GlcvE*tlAl|NT#8WMO^Ea_QfmuWavA-j{#v`gQKtmG2wAB!2$>dD+(sKW_hF
z{3r6W?|s+vS<iLe^L?B5HTZq<OQ#n>ul?R#daL~^{@MK}X3uh8&HS+bYr%J?Z#<th
zKQMmW{^j%6{*S9)%01ch)a;GGN1;y_KdgAS?#<fQK5sfc-2T4u&$3?zUzk74{T%-{
zm32Sss{j2zCVt)V{q*mWf6x8}{(JbZjp-ss9REMzd{HLh&%ATE0(le#mx+86KF_z3
zvx95BK$qB2$vCOW68>VFMf!yAh%(AfQxZ{V7C$9$Nl0GymfBDC?+Qi|=E7RSUqlK-
z9YkD(ss(xlehI&qjFHz?Y>+dN{3&u<<eUVvT#$mA>``$$F%C&fnIE!uWj{;bko+RP
zQLI+VR%MCyDa`{4f#O1<7iBi9P1W#IVwElwUnE{AIZaAZ>afHOaW#p0DHr){6@E1p
zrF)Vgf~CBR`GrN?#XgGk2{{OfiYSUL5Z@~PUPN2q2zM*jBmQYptCb{`WTXrQgm`8P
zzLkuSHW6LVdy4ZK*C{@Jp<bb2f%!Z&oX(t|c`AfniEb6S&bOF7k)?+18s|eUGtNL(
z55`u8r_A3u=JK@j?BaOFynyL2hoVT9OqcXYp=^#H%wZhD0^fw0geLL4V!y#2&huID
zt;iiARo(=SCU$Wyf4*gc3kB17XR+llrZLW7J;G_l<;f<@DDwX`<8SuYys!A@@c6TD
zWc|RUBN8tqCFvnJfxVomfmMe0zfiF7CSFyxGmNVlk24pssxjyP-TgECSJK~&jH_9L
z*#wxw{`P&}^wr@f*FVnxjKA~0=zfs-wD(8Bzg7Q>{%-y?@mJ6P@9Z~tFYu(Ytoz>a
z?%zAfuU~)s|32-r)jNsTS6`ob|NirX&uSkwy({}T?Hl{Qm#n#*tJs$_eg6~r+kzpD
z?KitG%l5x-ek}W?&A5*>f|Z@=$=}T1^Zu!`7xMA)GjTZl>G?eAi}9cHOg~usn8f}X
z|I+?F<?pQjyZ<Nu4gRh2r;~x5bBmCbxS2>Gk1-1$<0ED%j%Lm!Y*GwQe`Wuf#q@-2
z8rv==mA_p-AO9|8w&H%n_l^5B)6`#Ae(hp-&GL|?m!an8mM?3+Z2s2!gX8D+Z+f5S
zeoXy5@5iZsv5b=c?7zKwEBRXO{jM(p-^ISleW-tv{7(2w`;V<ZV!yn7`|@So>suei
zey;mt_{;FK<y*nG<saXD_4~f`v(6j8XYEgIUc7u|@#^i<kVg|9o_fUpy#MXy57uv!
zpYFcvc)R?*+B5c78ZXozGTn~8t?*#e<HX07_wU@Cbp7+qz4s42JN4@9bEo@S*LkjZ
zT&ui2<$m<N@EewwZ(NGJR(R|Ft^R8r7e&s0KELrw>n*Ok7j6q(&p7XO+V_I<Extzw
zAAh*F=jOND`yM@dI{V4-hb;HxZj0TS`tbhC-5*wd^n0`8$;JB%9@IbE^hV;{mRA+e
z6<@T!(f&B`<D0jSUKPK(@@B_p#=o1G=d#^n+4I-*$F$%3ncX<8xu&ymGkG#cbK3Jt
z^QLi`ux)2vz!J_L$CD?lE*2p$neEE|ivO;xRXhv%j&L(`z2M%(w^*P_Ac6NYM<J^h
z%M^A=zN12yh0gLmWx4Qo*55wHPb@!JX8nKvBjl&e|0PUD%o@yb3@d*N{haV~&EFap
zQ+8uU{_h>{tKaf|wEZsf%lEtP=fxkbKIMF_{H*@*@>}WGLa$oh_J95Ir}Dq`ADJ)5
z-Yj`D@uS4oz^}JHnZ8eXx9wxiw==)>7<T_}{C(<&_%GZ4VXUv&)R<a-zIt!)>iwIn
zFVa8MzHxuP{=w#b`TMw!-#&JJ`1Vfmz0v0fzseXDnYaCa`D4|mLm#!iRsZh($NsPK
z*OKpVzBBz|`D^ii<=-nmtG@5}e(d*UrUJHZChuQepBmpSc%T2J{HO2lJwFn^1bp)Q
zJmt&T&xRkrzf=AY@>TKofq!rQ-~4mxbM{N_r;^Y2y}I(|+v}JYAD@1E_WcFtYwfp%
zZ=zmEJQaS%`NsH*!cXTPYM)lV@OkL+aQ3tJufM+4c{BN?@vDHhKi-MHTk@Lmh4a(r
z&mO!v_u2Hv^zR2gioV+TwEsExy9b{mzA%6Ae#7-9;O&!lr{8aT`{HHT^PSHeUaNiY
zWd6xMnc3i1{io_rHb1`n`TF<qpWxpazv_Ok{b$8k|KIV~<!{_S82%h%+{ST>JCA+J
z|E<4w{p(`A#`TZu5t}dbRHmCuMJ%cuj9d~N8LZJPoE#oP(z1+-y0RZdzVKe<ohqCw
zDI()8{aB(%;)#^7tcR?fOrqpx5dq;XVg_;_Ro1KZDeafhlb$GlNmW#9yY?l`<th>i
zH{`+<Y?O8>zLHar*(mi>_P=_r(QDI9hKU**<m=?@RC2ZE>YmpX*49+7Q9Y#QsKKkg
zT`68JNG@A>nbvfDS^W&n8wwnfXT-ir?No?RwoufTnIqmJ(ki@0^tad<p{v|C+4Z>W
zgqBKm$TCV^=96HV^luBpVUE87H-(h>`Z+Y&R<UWa8?$WwoAhJJ_ai?I{w6arF>m|#
z@2k~2_cu@9dwjd`w})vP!=^t||5!3;u=KI~{?GU;>AU|gA*Ld32O)i-IL?s2n%@q8
zd-JD^S%Y;sqs`xIzq9}KFgh}P`c?S-&5!wimorOn26HQNy=VRXFYnj#zt(J5d1ngj
z;bY`o&3{=`U;32vWpR5USKf;}4k9-4Csngl*2<&{pWt=ncb9mi#ID|=GEp%{DOmlH
z4wGJo7NdrS<^}B)`bnmCmi^{ph60+MDu2~B>s~YJFnXiosrFZCmP)RsyUrfX?aI64
zoaA??u^Eb4Hd$pES*QspJW!gXb;VG~WPyRR_78P4jn7*1b<gWA)IOsbprx+sY~W*1
zpk1pJDd8ftMEJ4fANgyF*JY=QXNcStHI!T|6(;#qv|m7nuS;Z=yq(%^WnoDl9zE7C
ztSh;rg;K>XicS_};tk>6z}?Dqfb9#D6XPMq%Ph8BA9;(oYngBUn)vPLkBa|dYz`d4
ztY`lh{yzKj;P01zCH^#gXZTX`HSFJiu7$!f!fl)ae^!0$dl&Y(_jd(D8^iN|7yp+3
zJ@{AZpVMFOUtHhce{=u+n30Kn4f`pk)4!g7S^MS8&tyhlHX&A4#=if`OrO}?xqLYH
zvl#!M{bw!1TF#e(fg)ynKN!D$+xMyW>#^TCjIWt%nCk!Q|9`@u&#?KA%Fpa?(qHAi
z$Nh<Ba%AIX<^Q+(8`swvKQ#ZG`JMUw(?_RwGv0>1H~KK=ZNQ7vN5=P69_@d5<^#v)
zq<3CVHr%{^S@~+ytu=S!??~R9bxrNs%xjmf3SIHK+;v&_%G9euHx=)S-)p!z?_$Im
zkFyGw*WE0=BYDgFTIbbk*I(Rxdvo`VmTQWaPhC8D_43`9&(6L{cvbPR?bef<I`=L-
z34ZnS_3Bq^UmSWi>B+B$f%j+JDY}(;>))NL4>mse^ep?y^*cP*c3+!wPvN=mYtA>E
zZz|tNz0G@b{bk+ryHAxL|9Z&veDTMczr3u{%<@0CzB~5f-}8poydTm&m3;F4*!SM`
z-HEq0Z`0q%zF~a3{Qc)Iw|@8ipYvzq*In<$-?4q#_G9|rZ~qw?dl(G=xBspGb^dGf
z`#;Z99zA+^_}Sw3zTfS>U3jzWQQRG-TMKS0-*<W#_%QZ?=>6sQF5KV#pzq$dn{Th}
zyUu-Q-6OddVK363EV%D^C-lzkdv*_M?>F3ie=F|hjT>{Xi(XrLS^1pniFwB>&uUyx
zzF+u2?N;5zg{KxBmpXa<%#`zY&ljCrcUJZMr;8R>qON?u`0$+e+23bpT@=46cIDo=
zrza$jb{?I4V$Z3hGoI(xoRd6fdye(|+zY=hKD;pLoYlFD7cbxZ^04J`)V<E@k1u_=
zbng0`yHyVtJdSws<?)oqqEFhNE_$Z*tmLWVvs*8&zRmmW{MF~vnYTQzwcoIODE-X%
zE&Io^U(J7N{@nT1^1JCD7efw%8`EU=uY%b!-(|0emGT|uux9_mY0Gz4utQ{q=q%Ak
zqH1EEVtd8-#m|VxOKHjfR$Z*KQZGd7ni8KZt4xm^zmk{QU9B3u2?i<pwc4(l`5I!X
zmlSm5xs(*OvrP-kUl{yRKQ6B?y+XQ9cCp+Xg{O)c3jA{AGN+{^B+Nw;1@!pm2z`-s
zSDdMmpu9`2UV=kJMwmhLxI~_eyIj27Yw6$OTf|O@v5OcB9Ob(tXdq*z*{avAy-2NB
zu}^lR472Q1S$l<UB~8W4a`)tJ%0H0vl0GK(NVr*~PJ&VPiJXIMqQoP?gFLxhGTdMI
zB83Y@n?(-`PZgXllqQld*u=e?J&WT9-*fR6X)#GnQA@#PeEa$S@IB+3CvZ}*n|~wE
zSMCg+G@e6T4(ygJZ&`9U?fD)F$O?Yr+rn+dHH&*b|4GpYQY)klBqxcM3!N1_E2z!y
z#BIxYhbva_wuG+KClME3H`ZJK5C3EO|MY)5^Le(9tm(|BnCh6jnWLDlFeou7Fgmli
zb5wI_a2Bvk`uFOW$R94IRh-xP0tDXhed3a0OJJVE`0cOsFVUa>e@|fEz+)@8gZ~%T
zHx^xnFaH)Z%ws9wy21C7--7=LZ#DNj4qLWb=J$+;nS9vvcpeEd32*0BV!ix#+pjNw
z)-pA*2Xk6+9%ol$-OSX(u=cOcZ-(FN{;{(!6^xNgklrNzh~JyTm8FA8levK762G^|
zOVOWVYT_$IxrA5n|L11sn#Y|hAST8rStpjmAIo9K#>#b$|FE#A*dOr+5|<@TO4v#q
z5c3p$E?h0#DSlZ#SZ%ZVJk=t_1{p!gKVmOM)g?~IJd~R%n;{b;b4^-Rs$aZKq*%ya
z_@?+?S!0C|*&E_9!v6%TMD9y0k#3XzBxx;iM(m*I1Cd+8)<VnzOZl_~Bt-s;`H8&~
zwB@d0@%nH4`{)n-UpN0+{BQev`<L&J58s-<*?!^wAo%Xf`+J|>d_ME>^XqeuRqoo}
zF1mOAiRJ5GZx!Agexds8+at;Qo9~F<NxHN49`_TD*S|je|Lpo%_5H$^7EeVUUwS<I
z#mV=rU(SE|@`>Ri@B2@0mcKH1x$TwahwkrQe~<j*`>px;!mBsWG+v6oZ}|G@XYy~(
z-yFZ*e>eDg{gds-`5!sICja`((92Q9x0wGM_hXj7f2@C8{Nealh4}=>UmgX1C;kb%
zu3VvP{LICS`<afj*YTAK-4a;KCC1{<sLrg((aASaSVe5Ago5-0sa$bW(Hp{vLcaw?
zg{{QS$Y`mYP~%q)k**V};6BHGh%Jg;inEf-m}?G)2-`KLyZ`h58vK6t%i&)oa|gQ&
zM*_3tpO-&`e;@xV`v1fKe1>_93e2&L)_;xv{P{cQZ}cz0@5$f#f8PEZ{qOBhrq7~p
zraa&GjO~T)tJb$JAEZARyt8?e`KtY8$IJc~8=kLt8U42Nec(Iem#U9nK790)=T*v^
z`ES+UPJeCuvhK0^9r2qFZ*IDI`TFIn_pT=1)_$_*`O?QH?tZ-)eq+PU%)1*N9(v~U
zI`VbS^B<2JA6<W#`*80=t|vcV?0whyN%+&2H+@g%J>+<_{)yv@@YmbkyncK8UDa#*
zr;3l&o@zha|Mcn8DKCzG*#5haf&b69&)IMEo+~_?_Tu}SAD^fHuKQ>IC+o-5FDpN4
ze~@?|_@4K(?9b}Iu7BtM*#1fN{epMgA3lAO|9<gzFT(<6Gp6N#oqy&1nE37am(`!k
zzV7?S%kx%PU-&$a9ZS#O#$W%wi+*eUIi2A;Ge6U`e~rIbzi<6=>*Iy@%^$CPJ@Zrj
zcg&Bf&+Z?B-kZGN`2O+-_RlrnyZ$WtTm3uo_q;!?ztw*H|J?S8<)=MMIhQi$C+6b6
zsXt=ADStin<=GEshDofJ>`rXkm^U*yF_$sc|Lgx3$LPo^#^J}dnK6Z7GgA=@533T}
z6!vP)Ph6fHZcN7iCI5T;J@HfJo7b1%pZd%O9BHi4|C@iltb9Itd7>%X=;evtqn9U+
zyvq|uOH}HYs72ojzn%R$@r%$8qyMv+{27h^&;2X*?;V3PlR480mU@m0Tzj~lvMp!W
z_^0!a^gjiLaHfkaD>#nu@bZMQJz*+mPGe?dn!*spxRBFYOj=q_jGMoKa|5dz%QNN}
zmO4%$q3@y(gr@Q;aA-1HF<kh&?0456BZh0tOsr;%tiM{nBz*e%ap$MDuZh1{8O&G~
zGc*3H`oZya^@pFYr@#FB%KftdgE9MVmXyENU!&e_dwJ=p;^UuBCEjiO{OD7~`^YyB
zUK+mWdAjwH<dc?{o8NhTocn?G?d9jo9@*Z1dN=#tl84Kl8NNRJ*7mLOE750650h^T
zUYmS*=Z%?<*S^w!wf?EwgKxJd-h6lc+x1;{lAcz)$$CHU1MA1I50l>eyz_Z;`1Q88
zZ67y$-S}Pd`-HFRU(~)-e7*Ed`sdp}-3$krJeapItzdNi|Lga=pL)OFGuHFm5z&<3
z7k3j@7x=`tpZ6Ef9X=OfR*BzIi=|zq4P?&9ZC6NF+^V!mMOwW@GgWh&+F#|LipLeT
zRXEgRG%stj=(Xu@)qkODqb;TRK|NI6RAZm+TC>mA+pO$OZS*uWUDS2d7}O*+%k<og
zf{mK>tF<N6ZIst4Y?tR!c%vYztfQu?#;nvT6D!Uy+91XzF;&7%>Y22&bg0BiVP(Eq
z9EvP+7?YXwIF&`#NXN;9N#qCyb4_Gj$+VVHh$WrVm0wV}OE^=og)fe04%a1)=^PKa
z#Q9By*+doztmNurp8QYhpCyAe^EtM|oGCmWeEWH(vu832|H=Nb{yWp3$t=t~k$myo
z6IkZ{z5V0TmmeQvKTG|5_n)7Ynd3P}KD$1PEW@MU*MI2$u>KYJ&wweKc@0C(Z>b+g
ze+c{v`}O^o&z~dzIvC3t4ga108S&Nk)22@z-}nBz$oic#nd=W*KjZa3H9z&gD}CGf
zt@>xn-}wwBi~<Zh{%-zl|Fi9D?x*>my1zDkxBkZaso+iLi-;GyULJa7^jiM)x;KU&
zUVixgX8QB$$EzQ%e)Rq+_iOQwMPC<uz53DqZSSk{m#<$`zsP=Z{N?`F+;0+JU4PE~
z^v9#m52YV2cwqcU?V0AQsjq*%xcm6xL#L;KFWX-Az54!Y^_#Wt=Dm@9w)(!|?Y%c{
z-+X=V&(ribo8Q~N%Xl&GLHEtqSIRFlTnV}wal`)Zt%nIuRy;ZL_}}B=r}5AFp8tG(
z{?mu=>_6{(<@!+grupsq_gWwSetz|R+Mmme`<R&+&iwTIqW*Ew$K<bxzonR^xw?4P
zab~bA`@i&0!te6mg@5le3bCK&I>7UrhlBe!dl9Py%R`nW9Nc^&!i{2uVvRzJ`CjvO
z^IhR9;7=EvEH+72RN<HOZ_zG+_1u-5I@}rp`^3HF9h847Es{5pnkkYd(8&9YJDtaa
zf35He@e2~0#NG-u@a^K{VC!ai$l}XZ#D0sFgXz@&C;#XFU-D1<pTs{&hA<XCwlrp*
ze@Q=HeNOmP@JZzJ>2C>t4>E0MiDLF<xcg7=fAU}FU-N$Q{qgw!fN?U@O$M2NOMf5v
zIpJsD?}~qsj4zmHvRq-RWh`NGX4%TRf~}vOf%}?3xp2L}Wo~DVmu!t3Ra`yXKl!vo
z&Wnl*hwyFVSk4m1{Dsl}zuiAi1_u@c4hzn7Hb&+}jN2KXGwosx;OyoJ;akol%rTcW
zf?b?z4Ns#$uc(vEab*_ubIQqb&5}z+b45BuE5tjcQx%0(qg6cR&x`*QDiL}i_=-PS
z;JnCfsq6CGirZw5O57B&75XHYF6tnCU16z;r0Nt!4cU6BZi%I0FGSyq-I98#a7X!q
z0++Og=xP2ep7mTUT)X*d#U@K@OWzSI6FAMan%$Ij7SnX5jcks*JA_Td*NPtE59H`$
zdj3EB|D*rw8CNmyWh(qv_3O!xo8NbRTlh`<hr}O$CL7RcG)$}io%)&mb?&F6kH<bA
z`04on7-Kr);(up;YyWckuJuXt-LJQYKTi5S>v!hwh#z;pNPiOgsQgLh%ek+d-=}@o
z{=VlM*Y^$Iqkla7A^mg4FX4Z08D21){<rwI%g?T#n|@{f-uv$i%M=a?j&hd!|JMJy
z{WIf-^p_VOPJg`oRp+<f|A_yVf2RH5{9f^c<#*?wnST%c)BET0d)|-gA4h*B{L5pM
zXI{_xmeYi%n@fk?hFR|a>p%DY=>6NlB*V3sf3|=;Zw*@j<7|dThJ$~<|FZsV{ojN2
zGy6AYqrbvGY`$*&!u0L*cf(&@f3$yFeARw0{^sGUd9RkdzW7$}<NU7zKl6T;e4Fy&
z*-MwF(;h8)82L2+?aR-0UoU_3doA#6!IPzrj^BNI>+{_MPwn34eBJg%|NXTW4<Da;
z*z{=blUvU<UvGJR^@Z(Ik;l^>i9B|Hoc7rDS;L#ZA1gjfemeC|`{kWSj`yPOZhkcD
zjpdhh-!^@F`gPZrj?Z!*8eZl-KJu{Tsoh(%FVf$9KF7cN^|I_m@ypQHz3);#t^d5|
z<HGlP?<L-wyc2uN`S$QT>rb`cYJRQxJNe&)-znd2d=mM*;rq;g!OW5@o0%3fod3U`
zA@u*gpUz+UKY4x0`|<UU!GEQHvwpq$_WA4b?}vVV{k8ko;~!SvLOwtJ5c}cBN69aN
zUlYEne|`Pc?)$E9exKN0vp<Qw|KzU3L%-)+U)jIZc(&wm;3M@%=O3`%6}u^a{l)c-
zw{0G9KiY6V;+Dj<vTKWPn%@z-ck$ktJ9anXFP%N}<)qK4*{4g+*q`M+-+gJtwf-CX
zucn`KIClS_;NjL2Ea!_apSbeoQuVn5rx;H4pENjLbL7q8700{IufDeThViu*7xGU(
zJ&}8|?evp#>6b#TIA77Z6nXyYx!{XOulV2Ky3_i=<hkv8@h_F1#NK2)ee;m}@tqef
zA3y)pVi00J&g94>&FsL${%_yU>p#l>>}Q_A{Y$`K=pz3&o)0|Qe3tzDLW{(trEH{2
zrIIC;q?Bd$%gmDgD|JO?hT?AZex1p>>RRhnt|}~1s8YGCm2O~fe9Bnec)igzBWt7Y
z`nPm0Xie5>Ho9ctV{_BygjJCFWRr_VZw&7kJu^9M+HLa9@SnbmL7AbkVU)gv?rLpb
zorAh_^(N|gt8Z24m3}T6FXO0KrCO{WtuCo{RaIR*NPV$NwBmF5Qwn*?57d~ng0ydG
z%Bp9mnyK2UJyfsONLRn1VybXQszO3TGEDN4_!hAc@fyiK8A<uKat6|GMg4@dg{F)C
zlDZ?`s8pqtt#C%RT>7%aNzq{82BCGru40Cg8PX4=3dEWOym>6S4sjmkQst@P5#rv=
zevf%AV-VwI#??$Z%uAUi*e-Fo^D*#hv)}$d|JSsiVt=w3RM@h4<_c&FmhgY#;o^G8
zmd>oo!2F+;QJl4ab3b=2*LAiT%!`=VS<2a7b6nug<Fn!m<Tl|f;|$`m=bFzc!1a!|
zOte&nU+%DEuh2>E{p=grzOYZ=-YgI)vPi^0D2negmovv#mQ;qOzsLR^U=HJQ<7X8(
z%Uj4jkCTxjls$+ehU)<LFRrT`rW`h0jyw(ApEx|(<2h1z9tgY_4CnLac*n%f@c!TL
ze=iwVvOnW~#l3-Z5Bp`dzpQ-BAOF7pS@YBU&n<?2)^rXB&P>iI&RZPK9CtZxaB#7w
zFnj-h_-Fdx$p3u*_5ZH@6ZU@viymhc=N`6q%z=!Y|G)lYX4=Uv&v#BRT<{hjEAKX*
z3p}OVr#Xz+f3x>-UE^IMuugEMKp|fVj|}%eu6CZEytjEuIE&d_+3Yz3c$V^5@Z9Cq
z68s>-C=nvDT(nHEoJWI0p52@4sz8Xu2bpe}OA-%7x`iwSCHcj90=a&1dGh)QY!*%w
z(-GSzw2N;Yj}mVu|1H7Af@%C#yq~#?xu0;caIRq6!aAAlB8Lw5BJNnuIu`A}{$H28
zfBE+Ghw^U`zaISD@=f!z!iVW^-Cyg!Y<RxwndOWAH%C97{Br5@qW7w=y<XTq7kOd$
zYUS$#uU%gEygKyq=?nhnwoeouNk1}p^7Pq)7a}hYJb(T;_P)Sfp?g-3)L+=Xt$iQ#
zZuhI%&%K^`KKXKA^|tg4*Bc&pG9Ix#*LiX3nfz1lC%#V<pQ=A&d$#q7<>QYJJ0DJd
zbm6hY6QL(zPhY*@de`@f`>W{J_n&k>n7&ha_u-@3Pp$vw8Jz#;|7HGH%^1(}l6g7f
z-~aRfdopUUuHm@Fv5{4PiHG6f|Dy~un65A{X7*xw!63^R#(a@gjNOxcBD*eUF?Rz`
z9?u-^D2_5FyT6}*|M|b3eIf59-n*Q!tj<ga7y|$2|5^B>^y}Nt8@_mbpZs(8&#>=I
zpEtf!f1C4u!6)0#ZXfI4?R(As>gY?(SKP1Xy<Pf&|Fi2?&F}g@D!*U-6!5C`(VF{i
zkJw(!d9(A4!plcb>>gP^?76?|w(N~x*W|AsyZ-f7$%7e>-#)l->(te>D<`k%-m<ve
zbSvc6)tezVR$te>vGm5@8}T=n-duC*^PQvj3m&rF|9qY6;=%L(uk5+2^VIl7#|y@n
zEiaj09)4c_?D3POkJdhz`Y_>n+J__GM7~9S{QCCI8>6=q-|c<B{QZG<%5P&|d%m`O
z%k<&Tr=wqve);`%)vqFk9n5x2?ti|2<@~bzoAU4E|Kd!pOwA1X|8M+vVSLZ1%Bb>B
z^jFRg`QOrvIc$&EyqOajiWq#DUNL`WUe8?1@`?E>!;k;|EcZDrdHA>&a8z?#<`(B?
z=4a<=VPC~G<-aw9Jc}sDe-0fsN5*}Bm;e3vFZBP0e+&QY`1$ik-5(3)haC2tw^{Ep
zOS7zDdC7W@-H>A+YdK@q-yeTIFqE@4v9D#H%l?uhfaf{?BY~~FmhAT!KK)JkpTly4
z%Zs0le;ZdS`%8|MJWKgJ_*{78xE#3l@bC&a2}_7f6OiV9&Hji(jkjB{P$-(eohyND
z3X1}39Ge(>E8A_Br!0{i8a(fKBDt(t8UA<wVi*OZAut*OqaiRF0;3@?8UmvsFd71*
zAut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?
V8UmvsFbYOPU^E0qLtrq6005bZDI5R*

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/f88f1012473e6cfcc9b39b2552f682b2f73eff8c b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/f88f1012473e6cfcc9b39b2552f682b2f73eff8c
new file mode 100644
index 0000000000000000000000000000000000000000..d74f0a3326d2a644f869ba969426268dc559df25
GIT binary patch
literal 54757
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`gTHptF4VNKLCbt25J98Lg
z07Dj24%<48@0@`=i}+Lp(}ZsbI`BQ=&F9w^C=pyQBre1z=py)0Xt&5*(c5BkBrZuN
zOTU*%kO`G8l71y+CwWKwuh>hmx#AzhFG%c{E|ImAR*;mG{2^{EK1pJ`c$0{saJg`+
z$Q03N5e?yJk#NxjvCR@QB+VrKBo|0*6k`^x68S7#Bshg{7q32V2G1Uzd|p95J3emS
zMLd&unR%yj2k;2+<@2&|t8>lcdd4No^MXg7TbHww<2Sn~`&*WijEetG|LFUA<-6jq
z6+abzn*Uh&eaerSKT3b({G9!x`P=KSlfG7bVgI7|#s2f<kF6i~eh_|F^Csr)uJ<0F
zXMWl8@!z|ZZ@pfJy>NZn^vL^R;DhA*8y?(weC_$n*Eiqvy{UhD^DXb|InNoNKYGsi
zoayQR|0^ELKVJA)=jq93|6VM8ed*2JH>Pi8-(<cLdGq?+|95lW*}Z4`!0~bS2c8es
zpJcu+{Z{$o*^ejROMW)~N%~v&$Njg$@8mx-|6KUh{`1eT_y2O4U$Yf)EN16qyUpUy
zyn&I0aRGxJ!>9i?|Hb}SGOT0@Vhv>9#kPZ$iTxn^Y_?*yIJQ#OH*AMEblG`X?=sgj
z@iFga@nm_x^qO%7qab6;U*?}Xe(e4g`g`uL=RewhwEp_~>+g>@-w%Gj_-*;u$6s!I
zUiB&R)9H`;AMU(Q`0(Z3`8Nw*e}851`rGTCH+pYZzL9<X=B338*Jr<<<UQH)ROy-g
zW2Hw`PaR)Odd2ZJ_f632r*AmlU3r`L&h)+Y`#0}G-tB&O@8iv{yT8l*{PcV6zqJe+
zOvX&>822$q{IB|-$LPt}z#z={oROdT0&@%VN#=Hz<E$m@KUm|L^qF=ue_~=_Ec(CW
zpUi)o{}=w2{rUZG4vQvx4%;!d8*GNGi<wOrU;VB8ZT7R_yYjbXp9MY(eKq*%{Du4L
zr!NaWrF{7E{^$qM50l=Oz3F_@^xEdt!k4KpLtpZ}nD_MJlWR|RJ#~Gu?9sCao)3&4
zT0e|>Q1PJQ!S4Gj@3%kL@bK}YLyv==)IM4JMDf|e=Qo}Qy}0nw<@M>;Y_IRVn)p`k
zWBTX&UnIXi`26lu+o!6JCLer0D1LhOx%P|NSKsdizfAw``B(S<2}3XQMAnV09&8*O
z=h-i_?qW$|*~ucxmdbvP^9FCU0E3{SkgV`}p+Lbx{zJS;ym$E)2y77u6qqY`MVL!;
zo5)e&4q*Y|?*eyu_j8qVykckI@MV9&dW>ZeOC8H1=0+BKj;Gv$ynnbuxVkyjIF&ex
z*mkjGu{>tUVu@f`!1{vCf+LHwhqH^bfYX^nkv)kmnDrrxG^;af5o<o{Qq~RZtGUi|
zujYEq`GG5zTbDD1?Go!swtZ}`*nYDsa<FrhaWZhN<2b~T#5Iw-n|n9sT#f@=dc3cB
zRrqf6v~ZPk8gX>9YO<8FdUL$w>f^5D%Hq7i!OW4z=D-robdOP%IhExva~ShcmILg+
zIj3=Pa2@72#O}f|hqH*wi<^P>B;RTNOo4BLz9Q#DQ^aJ&Pl-E7OqJ-D=#+4kSShYA
zaaZz*RF1T|%qbZc**@7`*<jf}GWTSZWWGuUOZ*T&C1xb5A+lLWP)I@ekg&hVVzF<M
z7i6}{9+Xy+oGLCaCMl9C^jz?-V6~uz;10nUp*mr4Q7usokuKp}p)`S+eCfO;JTJIT
zb1<>Hvbr%dGaY7ZX8Oy#kYxdjFl!`>AQLacg1?Qwe18W1y#2lJTif>+Klc4-{l4d$
z{CD{u-+#FM{QINqr~U8qzqkIp{O!b-qn|&2V)@woe#M7rpC)~}@uB>k!W-XLmtR;v
z-|?jP;hOu2_r>pv-}iYS{Aj}C&yNisPk5C1$l=k-M|qFA9}7LQdG!9#&qt3QH9nSp
zD*F7(bB`B2FJ)i7eZlc!+w*|ug)fX=|9u<&LEzJr&pW;fe-HSU{&mtffuG*L&i<VA
zQ~FouuUEgO{QCQ&_q*lyC*P%icKmwwXA(m{b1#cAizBlp(>@02|2F@A{}uSx^8Y5|
zROT}*MQp7cTRCkxTRG%8BG@mn6|-GoEoLibFJSw?vXW&!%XO9stck2%ET5UhSY()k
z8E60h_owHV{O{v`lK<@defxLlZ-?Kif2RM5`+MNu&VOuw6@I_{x#Fkp&&uy*UtfH_
z_=)Y4-^YTFFFsxW?Eg99Q|X7bZ~5O;ytaDP_(J#jxo45j*FDd8zUn#i%c57aUafs;
z@lyH~<C|}9Mc%uA`1G;kQ}d^TpOQXp_;l_|?Kj@<YCl|mR{cEvgZYQU59^;-e?9qo
zmcf*%k6DaWoUM_~gjJRKK9du33sVl`c7_=Y`HYE7d`w3eJQyk%8vcj=JN++_L7aI3
z>q(X@rVotfjBoxG{_Xi={`>Z?g}>GQ-2J2X@6tbxe};dme{TPF;S0;>XCMB)+xSNQ
z)ub0&pG!QqeZK6u(u<}StDajtV}1JT(cA}8_wDa4xc%|A@7?HoH}5^a7kNMYf%3z9
z505_Ld!qBK^o8t8xfi>i`@IZ(<@)mD3!zujU)jD6di&(P=4b8idw;V2V*VNTea%<5
zFS?&oKi&Gs`N{rs@Rx+I6Tf@^tp3URi}QEf@4bIc{TF8X%Jh&~gH@017F!Lw8%G+a
zGPeuwK0YaaMgG(LzXeVUei6JYcvR3sNM4vnxJc-uP=&}f(Eu?=u{UCK#P^A1i5?cY
zCEO+ST)>vEoBI`K42LZHRCYE_J+3Hj8=f89!CXR|KiFN_^Vq+$7jX!1_HlmY<l?U4
zG2qqa{l_z%ho6U!X9*7<?;D<K9(SI#Jifd<eD(bGg7U(GBKjg3!V`oJ2uca8=GEeM
z;M~J8i*pj!Wv&cvd)_;INB9Kz=JBrPS<B79-NwnqA;Zzi@s?eXeJ)!r+ZMJAb_Wgz
z&Ri}T?n<sU4s$kXmJ5tU|BwH@_{aQ@?w_T9B>(>X`~Uw|rh_aVtbbS%StHm^vK?eQ
zz*fY@&o+(iKD!sEG50avSNx2E9fBtW-wB2YZ4$aB#3x)VyiDZ2Sg*u)i6lv8sVkD)
zl7<rT;>O~A;(p?j#2Uq(iWQ1!h`taO5Ka<$Ca5E{QP@N5v;?=*eMt$)IEl~Vufz|E
zKNn*clNCEDmLUFG+)JWdVza~?i3ag}(H7y^LR>=k1)BN0`Hu4j@UG#><(|tmlj|m@
zEXQ`Xt*ld6dYOBeG8wlsv@vKhurQ=Bcr$$c-_LM^A?v@(zjOcW{xA5q<L`k#a(|ru
z<osFuTjtmDACf-`e_Z{+@iXsd`_Jkh9^b4!A9}C<w(a$#mm8ihdlv9?!ejo&^Byxk
ziGT9@iQUtsPg)+YdDQyo(xZ=$f*v(Ioc>Vxk?W)Hk9I$q`Ap?S&CB*zp0BsRUiaGS
zb@HqBmkVApzJBy(^1H$hZ$AC{GW+Y^ua4jLe4FxJ<!9ipE58>0Uj2vt@76!=e`Nkx
z{OSB7`d9Sd`Tw^W9hpxu%d#wC@nzk}>dC6b@|G!q=?BvnW)GI{EOG4TI4!x#IInYr
zaky|C;V9=&W>;jJz<Qj;fJK1iBeN{a43^0(7A$%!|CpPYEtqdJu`rb}9%d|LI>}VX
z^pNo=V+o@$!@hq9|C#@P_>cLY)!&#uI={PqR(-$rHSTM|*Ql=?-vYju{OI~A|Lfro
zw{MR>@A~NTKHzQP8{0PxZx+9u`~LjL`=4vSYJc1Owew5g=h>h0zp#CM|7F>iSzi`>
zG5K=y^R6%X-!^_f@+17$#^2_DYW_U@BlY+3pR_*~f4=;Q{kQV}4~B<KhggoYs<TzG
zonrgKwvH{5m4R7|(VL;-zsEoRe|!Et`8VU=`hPC}{TMnJ*_gT+?=Uzq<o>Vyx9;!V
zKfixx{r3Oe_WSPdcfYUv?*5(hEB42UZ|dLHe4X&c>(iBY+^=UmU;AXvqdgC&Jj{J)
z{E+8i!-IhP^7owYTHLvD>%q-;H+gPfy0hpW!vp?@iH~|8&v-2Hc*f&7PtH8)f8z9X
z?=yiHHZT2N9e?%tRr%}xulwHEzuWpj^^@(V%O52^3Vm>V-})}~ZNr-#Z%)4Td0+Zb
z=!@aE-0v1Yj{kV@<M0pbpYFf(e&7GS<xk09zJG52k1$SQsbE{geww41vxYOBGli3%
z^Crg|jw2lBI6iZRaPQ~V;OXI&;eW%wR$#5*GNA(D{laR(c|t8h*M;PSYy}GWX7if!
zZsobflg?)%uw8&#K#Kne?+We)&Ik@yj#iFRj_VxDxHNh9@=fKxCGbv=Nl019TX>d;
zqNuZIifF8;l;}l~4<db{o?>UkJj4%)8%gv@td!`Lm?dE<5ij;fc%x9Hkfe}=P=e47
zp(x?g!ncJ?1zQBX1qAt*@apn@=Mm=F$7R8JkzI!UCEGqWRrYL-jhqHtn>Z6W_Oq$7
zE@Dn)%4F<h`1xO)VG4sd<6p*aOy11Xm>067v#PK%u})&W#X5=AkyU}UkM$&5C5Hvq
zb#77KOT3YM*Z4H}UHFywKk{AVyUFLne@#F_c&$i-=w7iK;-=yaqRT~Qh(wDpi`*7I
zBm7fXT%=NXxzJXjLScOo4bd(!R*53X8mTBLYsuH*1!5~i<b|_^bcDo()(BOL+z>l0
z!60=~>Wh?#)KLiy@m^77k%vO%f{*zZ@U7sT$1{QZ4p%vs7S|figB%U)Ic$4a*;%hK
zJ243{u47opkj`Mjpv$1mV8GzcV8EcsAi&te<jnku$&N{dG4{Xj-`L+5e+vG*`(xve
zZ9js39{=h2Q~Sr3Z)IN^Kkk3G{LQUb+%IQ5|Mu+5Gr{M-pRIf*|E%Nbt|yBgGd^y8
z9QUN;iN%wuCpVvbe4_jG>C^kqUO)f%;>OFyS8rdryzY8k`C9w6((AC-?_V!@EB@im
zN5RihU(3GV{Bh!^*zdAG7ynxR=V$0=*u;2&=`)iea|eqZTNJwt$6^j1&Nj~VoMN29
z9Bk}gS?ySVvD{~Q!V<!&%a+N0oMQ^-cg{szI@}kzY`EgM&Tw7jis!25oW)_yA;IyV
zy@BH>ry%z?ZYQ1v+}>P|IEvVBv%0b@WpZIW&+wF?hvDvj&;LFDCjDLZC;0c`pPWB4
ze%|>Z^L_Hywl8j<uYPp>aN?c+JJI*Q-?M#W{A~L*?py9R*>CAzpMKu(G2(s1+vTqd
zUKPBW@cQtZpKlkvzx!dshp+Evysv!U`TofJviBF>U3?eve%pJe58)rLf13Zr<QvcT
zzVEd^>VI1Q`ugj@?+Jg_|2^|>`F~%A42C2B*Z*J4aGSx9VJ$--lQ{E!rUIryrj<<R
znPiw`8BG{9n9eczGqbZ4vP@&~XL-()$6)>M)Su12AN=b6_4=3PpZkBG|Fixt{@>!?
z?mtq$ZGPH+-}80e7p>1`ADQ0Yc_Z?A!AqAHA<r$Ir#{zw-urCJ(`iovA2UDN_@MlL
zz`f495AWLCw|TJs!P|!~AICq{dB*ts-SbT^KD@Aaq4IqHvklL5p6z@3=4t7(#^(wz
zXTM^3<M8hKhuTj@pW8lXedhc8=u`KnUmu@*ocn3r=dLgJzlePO@Rj?!>GxgV4uA9c
z-te96JI^=OuNGe&zEys|{^Q6mslVa>MHpW)Ix;<Ey1+b(MU*v#^&4v@dmQHnE@z%C
zywCU^@?R3*6kINlB^WMTE)poBERrNLO+-bcTWF!cZT_?TQT*@uwD{NXX9+L}e&R3W
zox**HYcuCUj#3UO&K|B_9%Vi${v?4Pg3`hj!q&p~g{*`>3cnEc5oQn;5Kb0W6FDzp
zD#|a$D*jeXLTsw2u4ul<d|@8pWkO*>tU??@MM4cizXkb)W(&y*T@mmS2oQL}|C6tY
zcP4i**E&u=4qrAkR(aN4tjpPK*blL1a{6)ebAREmVwYpv!(zz%kdcouh%uZ=hxsh?
zG?p+{N48-0+3Z~G4D1is=W!h6n8T6Aaf5@8i=DfNyNTx;uNl9fz(j$?0w)C=1j_}5
z1TzJg1*!x#3C<9DCX^=ZE8;9FA~s7*SbU#^hjfh$hs*_OHJMk^f2EE}zLDsc^p!S|
zagq^`b&?B_tCTg6DUni;+$f<Z5iKq)-XSh8aaux4l2KAg@~otQ)Er3>$(a(45+B4b
ziOm;j6jBwm6<EWcz<-M`fRBYwf>(xHhI2c69vcJeedcuL9%fsX5Y}|Im2B2*`fPq|
zY^?8@Y?$^i6)|ZuZu{@>@B5#fzbF3u{B7~qWnVMD?)qZz<;LgxpWlBz^(ps5<vWkJ
z>~9vns(!ie#g!KgFE_n3e!2Ta;EVOo_dR>~#Nd(21F!qO_x9dtzHM-O?d@%M=HK<W
zH}U?ahqaGgpR9fo`?USp^XEJ-AHLlA>ci{nZ*@OB`uOrw&zJAt<bIy`RrRO!pETn>
z=65VxSl+OlV`X7$V~t{c%UZ!+&soKl!NtsN&ZEY2kV~8M82b`7HnzpAOIY`_E#=7I
zI>0@P=O@nw-Y<Mt_`L<f1-A(033m!B37ZO)3swov7OWKP6J!#q6q+b>N3cdfo&PIu
z22Ul|0uDKLA+~x}bJkBRrYy^to-^40KmO;`PsbmYKURDf{dV+=@RwDeEk0|0I`d)f
zhop~{AJ2U#`>^<f(8u79fgcRs?RsPI#{ae1tCugEU+sTw`}WP-O>ein)p}R`?&>?s
z_loboze{_+^S$-^b?<(>ll^e#<MPj8Up>Fa{ao|w@vphR{C-*fivH#F>(0;9KX?6F
z{O91mo&R_JuVV;gDqvp5+|IIqO@i|omoxV)u4t~sTs&NBIcBo!voB`<!*P%EIA;Xc
zHtu!YXE-ObPi0-rEY38W!R-H$f4TqJ8JHMq|8M>m{ICC?=)Y-yrv2{y&HLN#=iF~6
zJ{NsF_CDa<!#6))FM6H$`s*vsS0`WmeSYkD_4AX@zCVq5n)vk2(~D2No|-(J@N~*k
zg{Ngt?4R^K-Szz9i}f#_y_o*;+e_wG-(Lp5Qhjszo$Uv|5C1>R`Na8U?H7YDlAoDB
z3w|m3a{lxB&qu#hetY|!^~b+&bH5pXclqA^jqTgduQK1%zs>!+?n~hp>#rT({C-UQ
z_2Q4u|GA9v%!irXSzfT@vHoRU%I3@N&2fkG4);Y~DgHiw1py|(R|4z!W%=*(e-daF
zj1+7WU>D#N$lzDx>*M{;dxLKi|9$>+{{8&t1o8!v`S$XB=6=bwi1Q%FB93F6C%7GW
zU-Pcw>*X&Hm?YREq$;c;QZ4#iRA1Ce#9w%o&|x8S;Z)&O!YLvwqTHgXB2L0VLXtva
zLURPK3it_35XcccEGQyqD{z5dk^eex1W!G;6i*><Cod0AFV`&2LXKB#cC3GyJz2K0
za<PfADzMhDnX*q~^I~OXna`}w;?MG(c`-{bTL;Hc&Ig=!oDrP6xz2H4<Nn0GoTrfY
zGw(xQCEiY++dQ*)<@w_H%J>%X-Q$bmd&1Mst;lW5oyk3qn~moMPd=ZRfVp6=V1wWv
z!QVm!!nMLjMMA~nB~_%nq*A5!OKD0Um3k?uE451co{X+6r|dzQCo&$gO)_(&CP;3T
z?2}}d)RknAl9tw#Nt4+w?I%53`j+$t>2hf|X<6x+QokfT#A1an3vA@m;Z^6U=8olV
z;XcUqfa4gO80!J%#Y~-ykqim{-~QA4fAc>BL+t-Q|C0W<|9|w)@861lm;XOw*u<dl
zfBN5Te-{3J^V9W*=lAIEFTNLjzx7q{Yui`dua=*gK9;{PdbjHB{5PSmC%?*kwe{ue
z=gCjEJzDXg_kRApy?26cZ@Q&<OY!E$>p|CUUfqAK?1t+tjXQexoFAw@s(GyY<k1t>
zXKBxGyr_QV|EBF-_Q#Sh8^3+~e&fgKpBH|0{kHh?|L+fm3CvemX0dK#d&Rzo<2uJ-
zjy;@vxMg`+d24w}cy{qj;Bn%%;{46d$`Qq}gkvRVJ@-W3G=4t;9RUx4T){m;Wx{`i
z?}>aBy(xB1?189==vNUTQC?94(Ve2x#gfD`#b=52h)xhO70wo%$8W|L&8x|q$y><V
z$g_@1lhc;t0$U%;DJBP|$Bb<Z0{^Uk-~A!;UE}M`Pud@seAxG4&j;NP;qRT^GkoCt
z@b2Bcx9i@1c$@pq;=RL%O&=$J=KDJB>*}u$zqWqc@cq}1LqE%Yh5ugk`^2w?pXooQ
zeRurc`u)jw@gGS)e*T#H^V!ecKLdX{{=D{c_pb@R8~(`rQ)6ghT+XP$c!*&x!z_li
z43dnkOsiP*+3ML=vZ=BMv;Sk8$=1NOn~j0JicOl8nPm@iIdcKiAqLL>7ytVFW&eBW
zkM&=de-i)2|C{|g_-E7aw%@CN=l*8;z2X<ouh&0<zlVGc{`}#i%txOOvG2>?O@7<@
zhWE9|%U92Mo_0Jod?fL3!GjwQiXKWnYJC*{==npwM}Hn|f9&;y@0rTWGp~i-=Df{#
zoAGwa+vK<V-t2qx`)$g5xeu`)wtndOxboBUPxC$sd?<S#{9f+8_j~aVN*@_NDSdMJ
z$o#?Kz32O<@27lV{Mhg@?9;<fYd+O{lK5=-rTXiwZ?!+tewF`m|CjoohhZjzIOBB2
z7^Vp3JeGZ|o$O(piCojU=5gKTisY8%k>(BMQxxzPIv~s-A|SF_#9Xvk^u6dkv3ugv
zCH6>!ODIT0OGrpe6-yF{6*?fWg};$sSU^EAOvqSRPFP0hy+E?SMu8N;mx6^t{=&8*
z>qXi`6hwXtXNp*ez7y>cGZnuft|D<m{IS?<(SDIV!q0>fg_a8i2_F%@Bit+;Bs4?7
zn?Hf?Iu8rC4Oa_SFqbUnZuawRmTa3?*;or$R<M+_o@RAsUB!~llE>o8V#yrB`1b#W
z|Md+0jCRZgtd;DSIHWnlIE>iK*_N>#WPiiq#3{h}mZOZLkbM$cHQOJy@9bfmXSghR
zx_ATlGWneNg!ne{I`GQy%JMStO7LFbJ<S)vU(COnpG&|>AX`9C&_Zahu#{+_=yj1U
z5lPWAqUmDk;`=2INJ>bZm#mazlk$*$FTG4UO}bp#S^A$;w)7mCJF;1FUu1X4I7mxN
zO_SUzDJUf&bxG1sQboc-%v|J=kiO7XL0Q2-fh7JczHnX+o-D2n9M<d-Y&@+0n3b7d
zF}X4yVisl*WN~Fa#n{I1;eYu51OGz)e_+_pD9UKZ@Z^8)|KNYm|D5|>{X5}z*{{z(
zSbx0wzT^9%Z{=Sme*E>$>8<&j4X<CnUiC)gZPS}aueQ9{@GS7@x+fk_^dIX#G`fH0
zZrdG;+oiWA-YUMm|IW9&fA2XwSo?6rqr}HKPi8({{p`i_l9&Hq^}Vrt$MNCo$Nis0
zzGi<D_+j?*`Olc&_x@@!%x5fP`pvX~`5lWQn;iQtj@MiTJXd)3^1S4k#H-J@l9z#J
z8P|W#1kM{AMx4G}4%`{sFSsJPw7Dv{nz-Y5<@wnK`UH3c83e-wZt?R7oDtY0Xf6~l
zbWkWo__*+7;m<<sLUw|O1$YEb@@?iV<lV`8o>zl+EBAKJT=u&xDop<WZ~mF~JM4GF
zubba%z7~CX{+a!=*C*{yuAg3fJoF*!{eyR3-u1nUdMotC_qE#V6R$nr_P@9NwCD@t
zH|uX(zCQb+_2tv&@GmdFeEkyqrT??;XV%Y`KVA7W>9g$DU*8&k^#6?eCI5T#@Atot
z|IYjE`n%`1{GTg-ru;4bxA6ZX1`Eb;hUEWx|G)if|Hu1(0>d=MWsJRyvl!Pg-e-Kt
z_?Phl6Dx}YODJ;-Qwb9X^DO3IW<w?s#u$c-|GEEi{^k64W{_gg_|O0E^`9fZfBf42
ztLfL=pFux(zfbu3?ep<Zf}gTKuJ~}_J<I!mcd2hR-kf|D`SRuSr_W-ZJ$c&o^zhRe
z&+a`}e_8%=&x<e5tDir8*71z_dGm|MuV%je^RD3?$GhwA4!(c#-sFSLN6SxLp9DT}
zehm8H`+@TV+k5_Z2jA{~d+KfFJN^&ypI&}G_NCy9<7bUeTRvX?X#A<~Q}d@UA8kMG
z{80QM_rrz{(H{#x?fJ~|)%IKN_xc|-KV5#^`&IPY;E&SZnt#Cz2~3eJPHdbUuQ^U}
z9OY=|^ym7=HJyi*f11EKfg=Luf|mu8ge--7MJ&X6#FvY27W*k`Cw5b8n^?c-W8ns&
zHG-c7oCRV8Y6W))$qMfiiV|`YS|GGtI9qg|7^C=Ov3p`G#5~0k#C*kXN;FHol>R0&
zQ}%}JCD{zwaG4)ctWxQcPbC&fbV;O0ESC5oAuRDhR6^vgkc(gw|6;zae9`=W_`~_V
zc@J_g;$FjT%Ke<{0oQgeZ>}cJg&ejVAskX1Z0zN%N-VX^vzRZj$g$^g-r{QFuIBFL
za^zgZ@sOjFQ-`aKE1yfAi<!%T>oC_u?nyi=c{lJC@Sox56yO$^$^VismoJ#Fh3^pG
zT|NzdR)L=aS%L|Iw*)@%xAJoc%oKPe@IY{iaIZ+BNTbL`Q5ErMiJOuerKiXW%de5I
zlW&q&P?(`$qo68(L6%QeMYc}%i|k`LErr7hYzm%oxiX^C@lu9TE>e#q=SVyfQxQ!O
z-Xs_((9CbiZ^Ez7|BBC-&ztuHcRQC4rv`@@yBFIP*7K}fY$|NJY^iKgYztXbnB$mQ
z8Jiih84MYJGkP-}VO+xS^#8H{0smwEP5rC$_w*meKmEU6|G4m-{ricpdp;L^oc2EG
zUHaSaZ@Au`coXwl^X02&6;H$-9eXh0f$M{i`>*aw-SNC7d~@E7-8aH-3g6m#EC06G
z9p5`icQ)K{zUz8#*?qf*p^urKoqSRMI^mu3$F9#Gzf^tg`ugVU<8PO~ul`Z^%k6K`
ze|3hb3=0`oGQ~5SvUIY>uuF0N=CtNI#x<4u4fjOubzFNm>p3;KRJdh%Vt7~adGWvH
zyTN;z=P=J*-T;0(fkgrb1x^WE6xblpF7QC$qTpkp1H$eig(4?K+(orTQ$)&z3xtvd
zR|=fw&*4+yb>eB~UdeTh;}GjTCMm|)|89R({)GO?{gd?D>*wolwqHHI1b_DWbmC*x
z$2%Xsz2EW9<L$OL8gD({E`H<n=Ik5ZcaPqG`FQm6moKtkQ@_-Ip8RRm$9W&yK52dC
z`+VqA!zbHM9-sDqD*2rEW$oAR-|T-(`g!%&gWt`6w*0C8Bk<?o?@Pb6|IGd4{MYTD
z$N#GTNB&*>`|Xd@pQpdC|K|H6^LO{Z^#9xb>HORE@8<t!4BHq(n7%U=GlwuQX3}Jm
zXZp!l$as!Hmtn>KE&nYUHZYuJ5ND|QSNEs$SKg0X-`c*-_{RP1@t4@oyFYe*nEJl{
z-O;xb-iEw=^(O1htk;`f{dy_z^6~Q*&%B@gdaCy9z_VG;*Sxs)a?dNJ*9Ts8zHEQ7
z=|$GdA1}pTi@#m}?!`O5cQ@Y(zI*j<*Zb-Z%^!U}Nq!3WnD-(1{lRy;-=)9He|z%H
zk2jazroVUo*!Ahxr-Pr4eT@5{^?u<y-uHjs>whqPZ}slZ+rYPhZ)U!p_?qL*i8n28
zUEbBb*ZjEX)49)ozxaRK_TBC0r(X~Lu>H4WY-Y}6)nPMZ>tb_cf6Bg*;|=Fr?lxW-
z{=NK*`Ty}}3x)}w7TF}~E;dzcwiuV#N0BIzV<Ot37esVKnuJY-Zwh;h9u_+${$4ys
zoJoAS7@t_An1sZ6$qK3MQjet0No|o@C^b#$qtpfI*)sC7D`fTMoaL^{>d1P^SV|X4
zc}rDDO_w?^Wh>nz?Jqr9%2P5$e7fjbkpPk1!v4aM!jFV%1)uQm;S=OL&s)!Xf`^MI
znY)myjZ>9Vj?;kCigOBwJUc7fb=DxZdF*wZ54k3D3-c)OJm41RdBY>i*Uq0Hz$6gO
z@5>*_Z_1y@&m!<h;E~`wp$$SdLTd$?1sMgl@Xh6Y$y36U&GUvQhIb<GU)~nJ1in?g
zDZKxAZ}QFL-zAVC<SD`=rXb!f?j|86#UoQByFe~Q{(^j-!exab1$%`?g|G7IayMl-
zW&CA&W!dFL6buz^$t{#AlVX!_60H{27ycylQm{$DlfQ_MgYP152X8p9KkqA^d)$|~
zoViwVF5xI(S7qDFvYMHnxsmB4qbH+3V-2GkBR9jWe*%Ax{#N?^_t%YIqQ9U0w)@lZ
zd*iR%U(0@R{1*AW<d@d3+@H_BYkph#<@cxHk9;5e-ragF|7!ZnqL)oCeP2F)Ui<X+
zqqFx9-x0rk_15uQ*0*$SW?WCZ%5^32^8HJ*FMYps@AAhhme-oEFS+shX4>u5cMji`
zy1(v0>7$uX)SlmYY4wKh-KzK3KK%P|@<Z6i_D@n@48A%4c=L0^@5sMV|2F)8!^p+5
zj+KplCi^;eD~@X%Y@EwDmvOz|isjnFS;MK!wTat}*NCrx?+x!&o}1h*++y52xfOVg
z`JDKE@_yni<(tLNF32QwS13Z{xTv9+rC6od1hHDNDlq{u7SY?n?}UtnDg|={r1&@S
zR&%RxEMhs#IQM_tzbk(|{_g&x{pa$p?jOIuF8r+eY17AFA6nk8eb@JH&)bYQTVKz3
zUHn@1b>yr3mrXBOU)#SedN1_x>POK}W}n`EEc&?U<DO4fJ{NpB{^iaW-mjm&>U{V3
z(fD)EuZrI$fAs$D`fK=4>;HuRYyK_#d-Ko!-%Ed~|C0Mv`Agz=-|zjuGk>T3n)8$K
z*ZE(~zi0mu`RD%MfMFTKR)%Codu9XH1U3QoICdxY(`-jstyx|&*)e4>-udtSuj%iq
zzls0;{k!n*)L*qfpMFmLzUHg%m*UStpJhH>dSCKZ@{PdjhcB65w!J8Q!ToaU%O@|D
zU+sAH^VRxS46kfnE_@;PV*hi)7h*4iUi!b>_R`~J^K;E-m!390t$CXNY~J&p7t3Du
zzuNlh(yOJfpTCiOcmLh{cPHOTy?1y&|DDvk?00M4eS2H;M)LK%R~E0C-gv%Yc=P2|
z;Y;rqe9z^d$~+N&D)4Of)6Y*%J~{KG{VC^jix*-qV_uzk{rb(1w_@*2Kg53Q`gG{C
z=GQmh;(o^cVf%lX!GmcYb27^kmVB1K%w{ZWS(DlAINo!}b1`!N;hN6Xz@@~!nJ1mk
zkADXLFaC6aSpppb8UhRg4FZn^JOrlla|<*GstEgubc^tcSPQcYJr|fM@J--=;CG>S
z!j2;EMSR7Q#Wf^?B`c)Dq$kOU$_2{TDm+*GtrV+lro2pvOX;%WBgK!37Zno}w<t&|
z*vK!IHJ5oL{Y-j;^aW{QnQhYY(sQK>Bn>6ZB_>H65O)+iEg~b_BbX*&z+b^<&EG8W
zPvEtHoS?Fhrbw087x5B_LJ56|%i=lWhs5f|?8O-+qQ&ov?h#=TaTJ~?BqQ`&utq3e
zc(-t|u!AtC@FBtF{1$xMc*A*vcxCu@^7Zmv<&zW07TCq7&AXVFkN-ITQ~qXw+k%F|
z0wT{v`bFo8T8U{$cu9s!>=i#Lp(E)dAuCZJaY_7=*gH`@5i21+A$wsdkw}qrk-Nf$
zLMZ~~e1~|Fcyf3cdERk}b3R~GV6|kq#{8Rk1`9hYJL?aYQ!EiIKbZ5G)0vJjIy0SQ
zI?8mCX(Q84MrQ`&|NZ|q{&o0s`&ad^SHGV9a{0yeEB#mbuVX*Y{!sXF`TOGUE#Kq5
zD}Gn_9{v5@x9+b#U*>+^{VD9@oA*E8oqDJ7{^Pr8Z*RZ8{_@AOGmldrp1W^z|Hu8+
z55C^#x$k*@|NRyBpWW|$@aw_Shx;G3JXU`)?P<pI9WQRaWPknk^_Dk(-d=dW<>SQ9
zZ@#|xvHZ8sU&DXg|797>7*(16G3ByMVe{fx$FY)QF6RX9svyv+<IzROql=D57afl-
zIv!ngOp8Uwys3Oyd<A?T`2O%c=I!GV=8@;o<;mqy;Qhk;olj5TufR<KC4p@IEqph5
zO?bn2t@y<FwfJZA3Gu1(-Qb(dFDbw%m@QN&d`&o9L_oAvbfsvc=z5WS;bNghf<6Ly
zd`&#pxI8!yvVUSNVXkCg{nzrx;rFedDnFfmivE1?UEo{d7oN{gKc;_(d$;FJ(3_Mu
z-(Q=(zW++*b=>QxuO7dwe(CgT&1<W-``&GM|M>m0_h;W*zt4Eb^DgGyv3Cvc0^hp5
z$$2yJ?Spq;-|zeo{PE>S$xkyss(j4;$n`1uQ|71sPY*w(fBybC`b*vy-!H;n)_y+m
z$?fBd_gmg?`ylrz>GP~FcfMZxCiZ>fx7FXCe`ou}{ipBm{eKJ$s~FlC*8D&8PxD{@
z-=07Dzfb&{`K#%d|F2I!-~Cwmt>g3B583Z(-`Kx4dENN>>8nLA_q~|$;_!>rFI-<R
zya;;n@J0Mf%~w-itG%^+*YZy8UE^E6w_D%jzVUxk_U6)?@V86fdc1XbBlf!Z)tpz{
zuU%e$etq!mt@k<~)jqBKyzlGU@7+Jdf7pG0{H^m_{nx-Rw?0dJ$^D}B#q4wZr^HVu
zK3)9e_o?LL!VjDuT;7Ym6MxJ0_VU{!@9w_a^>+Q6zi$q{-Sjs8t;D;v?~_07|1|4!
z@fYTAFTS(>%=u~hYs>GBzZL(h7|ofdvuLq>V!z4Z#<85;h9iRWKj$2-mD~?_{_sBJ
zv*q8&uPS&z=$g=0L3Y7V!P$b6f=&X`0!Ibx1#b!b;4kO@$ZsZSE>tTtU8r8DTTo7b
zm*1Fg2X7hQDSj@22mBU-TZI!uzlgSoZ520=jFx&YRUq9aV=Q-G{(-_y#V<<sD&;EK
z$_h&IifRh{@@HfPWIjl0O9Y5(iBA_(5Pc<lLfA-HS1^S?l>ZyQs=z1yiToM-|M_PN
z7zxbfR}|nCv=+P|P$#fUV5;COAx7bKLhM2=f|&x5{Ca$gc_a9`_yq)V`M>ba;fdpU
z#G}bu&HJ2BQSi1<m2i=8hcJ_fg~$fsS3<!;j|7hiiV1}a$qQW-TrcP=s3FkFSI>Ki
zcN*V%zAb#0`Iie?2*-;QiQW@CBVj0YNwQJmm$;(DBXMK#7h;X#RT8@-?umaByClXc
z{y{8H%tdsIu$|Crfqndf{Qvmu_$Tt)@K^B4bD!XJ=d9zH#2(Ae#i7dil=CQO8RtvR
zO<Y&G?72>GKIU}hlIIHLT*Ohv;m5I!eJPt5YXq}DlQ&a8lReWy#tV!L%<asvOgfB{
z7%Q2ynQk!5{IC9h#{ZN5d;cB$bN6@T@7F(-e+YbE_ATz)m#?f}qdup8T=8D>-J&;a
zuTQ-cdMWZk<=MQ)Q4b9s_&u2W;M0AU`-=Cc-xs_ubI<7Btoy4U-haI9sm}9FFDhQ?
zy=i+p@7;m-CLhH<tAE|~E$_SPk2gOb{Eqwc`ghu&6@Tsj{roHa&*1-7hHge%#!U<}
z7_=FV{?q%L_h<K?Ie$I>HU0DauflkMiH~_U(|g8k45I(<{f+yZ^7rK5ng2TepZPEL
zf5N}+|Amb2ncgxpuv)QYv*&WKb9Qh};@Zmnkz1Yn45v7U23rQp52jW|Sq7*7oBkR7
zt^D=ud(t<@uWLS^_$d9s>wVdKwfEL<TVHd%aeuquP1h^#mkBSXJ-2=~>&cDBho2-q
zvwYt1-28?1%Y84Oy}a;p^2@}R6JJ`r`tU09_089d-rRa~`nAF<)|WqC+<JNURnY5|
zueZGU^49;I`nwhHUc9&XDE4XgC;QJ?U#@;-{eJU%`Hx3GxPQL<vGj-U5B?wfe=Ph}
z^~d3_`o9PU7p7z;872niHkM7S&TI~BJZwK$9a;UD9T@8w>KXq3U;i)W@5etee<%I@
z@Q3YB=<k+a4nOyNm-$xlh57U4kB2`LzyJ0&?yc;*eea&UEqQb2HRGGqH^1NPe>?GA
z+xvhIu^&#q_kGX*{^7eT@6Nxgc(>{;&s)~Fm*1>^z4X<)S4Up&eeLkt^YxY2Zf}g<
z?07BmM(bV42jPzsKE!`;{m}J3;2q!F@;9H~?0mcbZQ|Riw-4Sjyj%Qs$D7#K8n4P<
zn!Wt|V(Sa*m)~C9c+>he_3h-h((hW{9e>CAe&&18kEWmXzjA$d`C0s%_pk9k$Ny#w
z{)|_ddRVyG(%EG=CAd3ygm@EqCHU<4rwh~y&K3M6*eT2=`b;E4#9VZu*g3JwqER9O
z!k$7K1)mB`6DSniFO(##CA>hWNa(DPuCS2s5uqtUp+dI=9R>RYGz6Li>;xAJ?h^bc
zxJ^h?<cR2Lv8m!8B~qkBr3IxANtH?^OW8>UOP&+oA(kknEH+W}s>mndqe9t&mIC+r
z9C?FyUh^F1P2ruxbCk!Gmx(um$B^d~&ja2{zW2O}ygIz|dCPeB@i_8q=3(KJ=J(~F
z#{W{FQpil$R9IN}ztBCQRYILY%Y@zv)e1L@_=*aO7Kuy|u@mhVT`l@s)K_eySb?~|
z#1Dz(lGCK}rP*aFWlqU_lo6GUkd2nTD&r;-Ewe^unoO$9S{Yy26SACg4sz`B#}(En
zEmBTU(N_!BP}6MCNLDXbJE!_b<$&@TrE5w?%8!+pl`<4#6$=$#DOAWWmHi-XCpBF{
zUi_}8iD;4N0Z~TLQej;oFTqBE3;f~y;rvPhfr8>fMnVii8wGs@mkIRoJMjhZUg7cP
z3FKDglIJwz5MkfJ`hj^mW9k3CfBOGe|K9!`|L4}<)_?K;GX5R>cl_U%zbb!z{51J-
z@tg109iMAIz5KZGBlE}B_ZIKIzy0x6_-*Cu=Px^6s6Aiyl;uhLqr8VT4<hcLzWecZ
z`mJv_-ENiN>ba$Jo9oWhJNNDw+-<mf|8DHPWB26m-@E_o0rw-u$1+b`o{Bv?|Ln(e
z*O$z%YG2KLt?}0A-L`j;?{z<1`yli2=7-52{(k8B==!nZgVKk@4|6~0e_(kp@h<S)
zn|IINoqgB;{?>;JA0<BR_-Oi(=_BLEmmeN}c>KZTW9ujNFFap6zV7^%@?+vpwO>uY
zj{Q>pz2$e@AB(?v|5pERX1K+$j6sMY=70ac!oNI!=Krevx$MWK?-Rbg`r`Qc%SY*t
z+8<`Tb9tNo*8g3?yJv4B-c5V&_2Iz#xOYF^Tz_5lYViy4=Nq3DJ%9G1^wq@I;cuMZ
zetLWMt;E~LH<oYw-^_b+`;GG3w{P#gGk*W}UD-SNcXIDS-zB}9|L*+zX&;q7`+W)f
z8vd>3d*2VXpIkrHe@6VQ{(1dp+b{Ruf`5MgIr#U--~7LK{@DC=_;>W*<9}!VCo)cA
z5@l{?)@J#@63=SH`hevKO9RV&W*z3`Om$2Lna(r)WW38@!?5}P&VLbqul_FimHadM
zhtBu!U(bEn{h9T1@+akw8{Tui|NrjpyZP@Hz5DvE^!?@c@86q#`1N7`M}tp2pU!-`
z^6CGlw9m^w7k|F?>CZ>~kA)v1-wVF;dF%bQ{q3T+3*Wwfd+ptw_vRneKFs>i_>uF|
zfsgM#Fn-W@Z~5-un=7xSULSt7?$y~>cV8WTb@`Ro>%*@Py%u==;N|2Oq0gnCi9Ox&
zMBwSIr%#`8ywH8=|H|NX)9YEU|Gi%SX7=09?^b`v_$2V9@oVun@9(RAB>rOlv+i%o
z|0@hiOe>lFStqd>v-hz7V7KQu!hVLWnC%-|Kf5%CG-n;xJZ^TL^V})izqv}dUT_}f
zOyk<d-OTIDKTDuruuO<U_=b>_P@7<aV4L6xL0zGHLf3_PMJI^yOGrsRmpmsmL;8ob
zu1vj*pzICVa=DdqhveSN?UNId>y%BDm6BZ~lPpsqQ!nE#^HO?)^a<$)(wn8{NMDy$
zktvd4k(HL4CNHa)uhgg9rXr+zTjim0n^J+|ONG-4mlR|aUnw#uUsLW<5mSv(4O4Yg
zjaA*HDyn9s7Nl0AwodJl8n?Q-x|zDadXjpfdXM^J^;``h&ApmGG+Q)ZY0T0n(umP0
z)0nIAOJl93wU(aNdCfXaP0cQiH|mqr?bUavMXSZCg{U1+<y93>J+GRi7OQ5hrmyy0
zRZI1xvY!&4;%fO5vJKLHl9$Ebi+vE?C2~pFL-?T33&H&YIsA%z2YH0JbvV1&bJ<?8
z9$}4TO=c-()@A<3<iwQ8_>Eyd!vcnd3<nwH8Q(B&XS&Q}#uUOR&0zkI?T_lO+8_JB
zrGDM<#rliyXNFHRKbX8f`L^)Q?AKdf$GrLaChP6axAWhAeRJrw(kqh}fzSG%?0j_d
zLEC-7`)}?|y~la4;qI|J^|yE4Ja_%yRo5%eFF9T^zhreu=Tg@t|I0Tnf4<CdMe|DX
zl@nLot}eX#^Qy@;yK8mVM6M@a55As!{l@jk8#``@-aLI%<Mz%w-|qgpXYe5FVdNw3
z$BmEgJ#Kwc`*ij*u@_ffD!+d5dgGg}w^Hwv-c5V^>P^F&fH!t;BHx^Pv*#`Ad$tc1
z9}a$a{9)<`hY!XdBtJ-gIQL=o$JZb4ePsC*^ojp-%$MC?&AuD{RQ&DphvDyxzZd_u
z{$>9=_s^m~x_|HfE&O-o-`jud{+a#j{JZAQoZk_@)PE-Y*#3Rxw-sO0zLtO0`+Dfh
ztuNkRFMjp?w({GpZ#Tc?eCzt^`{mlF*pHXr+r9h!CjZTxH&Ji1-&MVj|M2?5iw}(-
zgg)GR|Mh*?hc_R(K1zIA^{L`><rlTD#$SJax%uVzmrY+*d|~^_`>o`=@K2ZDH~wDv
zuf~|qG=Z6m<s<Wc=J(8{EUQ^kSnOD0Sy)&Cn6;UGnO`$)VG3jRXPL^H!EVR7hf9oy
zpErp27Edqt2hQmnQS852Kd|(&Y-5RFJ;mzE`jNSSX)nXpfAjt<{dMw3_;>qnd%iGz
z-t}?ohd1x<y?^w6^84cVeeWaQGrr&TZpypI?<(H^eZS;G+Q&^FlRxhL!1f{MJ>&bT
zcU^BMy!r7u<n`iLT(920sCxe88PD^AX9u6MKD+yD@pJzd3tn`*NPMCFV%77{&qAI(
ze>(T6@zZNhSe|Zv>iBH(v!Bo0o+m$_`~1dpxfk1Bl)qg4GT`OY7mHsUcoF)N{Z-E^
z_t!JtOnbZVZPwd4Zy&w&c=zVrlK0y`)O-s2^7L!;_jNzc|NQcc<qz9m$A6RlpJj+)
ziefQho5%i^!;p)CyPeyIr=Qn~|FOVt!3#n{A`YT?qF+Q>L}EosL?(*ViX0RX6^oSk
zBb6uXC4WpIN@=aKh6<C)LS<Ryxk~;@LP}~%Hp+Uc=ITE*;<aqGnRWi^EYV%48>efk
zo2P4}=b#^L5Ni0((96ie=$xU4VXc9`exTk#-6CCYT`}F6I=8iNYyH$*t{JAuuK8P2
zMCYh(m);@033`HhzPd6xH?(eRN^ACLXlrn4=xOw7$Y}1<{H8fsQ%o~U!%aO%O;0UQ
z%|*>!ZL%7(x}$o4dZPMwwP&gvs)Z^Gl;0|KE14=)D!D6fP(GymMfrttg|d{gnsSTs
zHsuiInM$`6H!HZy&zI|xdnEfvrb@<5=BV^iX=Rx$GLf>vaw2k!vUjANq^qRdBts-l
zh$o8Q7Hbu~D;y{sD!ff-g<yfeZ@xLaay+eEx|~PY1=+T+Brwlr^kYb1P-fi7c$hJU
z@gjpeL)ibje_Q?>`#tyf^52Vp@BDq~_om;Aep~<E_w&kkfp1Sf&;7{t{^{%aFCRSL
z{Iu@Lu}8W0Tkrcm)_z+3e9znIUp#(y{@3~Q^V`Jl`+rLRKKAFvubJP3zG%Py`-1t!
z-RF%@+#f%9q<>HU=E-ZXE~j6}JsWrG^|2j?viC{uY1+-c`{b^*yW)05>@wOlW2eqe
zhMj+R%-K18*WBH2_ZaSD+uyQZ{J`>q(nsWu#T{RE!sXQJ(-CK4&-R@+zi|6(<(bTL
z(HCxAID0Ag`kp(P4_`dJ^Xk?6oG;Qp@_#S=caC8SV<D3ZOAVU|=QJLF{>cKx0;T*h
z{P6-p0(<y=@g3$5=V#{E;h(~{j8~3VkLL?l7uQj)RIZ7fF`R!nJGfQ(jta;L2aC;-
zoFwfk`&}+oK3aZ_{8YK6(!LT*qK^dQ`B(F-<aolI@?Yt9{`Y{d^S_vVVfk|83-dSM
zZ+E{g`)2>c^T)gIH-6Uq$^UEc_s5@oe^vj#{ul6f>z|%KzkWaYZT@@z&zxVqe_sBX
z`lscu#Q*OM#Y~w@n;3f-S(%P9IWh?`?E821-zCs;ct%sEU(8+XE?n=pCHSWCpX7ha
zznfo^|1Vz?-&&qrPIuO)jPL(*{q^|q`}3|3if_4JEq!75eDagdM+%QlKb-#{_I~`m
zgu5s2oVZhXH~+rIqxQ#V9zT2X`02JM^^fL1{QU6lLym{>4=&xWx_9=D|DA7l=G;AU
zx9je%JC?Vn+~mIb?Uv5{tB-|VoOz@9X~Oqef0i>$VE)EBm;D=i4SN+E6YE0eP^OOz
zF8}xZk@%(XGxKNhFTuaU3_lsqFsri`vP3f{G5==T!T5w>F@rFpF_RLrB+E@!XZEG+
zHXJ`V-g4~XsOC7wCd2ZE(U3vsU)^t$A0A&dzWn%d`&;!7+n+qY1b*xM?)_E$)AC2@
z&x?Q7{oVP;?=Rcm;6L;JJ28FwKl%SWCP`)%mWdo8oYAbktWP<gbB43JvM6&TbE>lI
zaoY0r^3LXH;b`EG6KfH@BPcGWDibQnE^%FfTP;b+PdQ6VSJzK_jrKbI)5a2}EGAD4
zGxXi{cI%$eTB(tyJyG|b)<n(Cx(<f3^uOuU>Wb=R=y2$I8lE!_GUhS9XdG&~!tAn1
zkU_0ZxW*5~pVGXNtrD-r*NB=6?-o)Pa^t(l$;YM3dzW`P4<m0i?`|F^-b;Kt`Tp>H
z<4k9}!@88ipUal(HMc8&s$iGkJV8xiHPNY}IU>@+=0Ya~KMMtmF-vZfcq6_~f<bDp
zw2xf1ytO=+!fZt;B`GC6<rr0KbukTY%~q{2?Tgy`bU1YRbj9@C^ycbB>deqxsQp82
zjS8>IQ^mCkMM^9x(JJX`ewueQ6*Yx4zp6*5sw<yUI4_$kds6<c0=uHD@<)}W%KsD&
z%YBr#k~k;gEYvG7ncsr1f_pXlYUbSxGyc!}xA70#pY%V>zXgBR{rLTD!)Mcv6F>a>
zu>RxokCh)6emM93(R=X^+uupQQFvMYwC|zaJ=@z1H~-#Pax3|E!>#C>x;H*u6TH6m
zM&T{KJ8E~=-ATT^{nor&+ipI*_UiJq%L}eBU-7>5>!RfqiR(qzC9WO6+HkexO7oS}
zt0h-gUg5s1a<TgS+cS4h`JZ7uA98-iIp1@0&I+A*aw6%-l!MO?SRdd$kalqX;qAww
zPX(U6b>8;ko%3a9bI-_}xqW)VnTYfDSDJ4K-g<kB_m0+`sdqH)&Aq?&q0JMyrwq@+
zUM_r-^tSa){%h^m4sRyCTlO*Yi`e(ozpVegXL!%V$-IzpC*x)&MW&|=ehj<}k_?&*
zoeYN<H!>Y#I>1!Uw1nvovjxj9CLv}QmL)77SZ=cZVyof!$SKIJ#$&@Xg-eAai}e~4
zFXO}imH$uuxBKt(ulirjKa;=Zf4cto{gL?d;J5Q{tzUvaj(ko2viMWhhxqrM?`q#(
ze;fO*_^s31qwiQhRDHPnq3+|!Pf1@?z6O6?^R@Bo<S+j|%YRAya{kMSuM@ws{!037
z_xs52lYe6VDE)E$%k}TVzf6V-#u-dMS>~~4aHa7Y^SAQr^IPzX3q0aa<y*jK#LvU~
zfonDwH|J|MDRxy3LylICa?ZsZ)~t^hEg1a&+5DdOL-f1Px8QFj-?+a``gZk)+^-2g
z<i5T8yyQd18?P60pX_*e=3d_I$2az0Ew~tVLGzN%rR<BxFFv{W`I7gQUza{#Y`XmO
z3hy<C>wz~r@9lVO{8aJz!I%6monG|4*!k4$5$_}M$2m_{Jl*+p#<OqFUp!NP8ue7*
znclNW&o{o=``PJx%Xi+NLcc=3H-Gv3MdkaypG*F%`CIkRnei3N0d`|92_83|)m&>i
z)^XhDy3F0mJ%yJ+fSvy;ZwvoPK{ugXfh0a%{<Q*}0@L}E`1N^xxW2MGuwUWy;917q
z%KedNB5w|FJ<l5M7M@$&e>jpkUvnFBzvo=St;e@sph#F*G)nBU#B}KjnWfTyrQD>N
zCC-W;mz0xfmAxXDBHt<(F1tzgp~5fKGOZR}YyDFFFZz!SR~f%C(KcIW>TMiqbl>=~
z=_>QdmbTUxY}D)%9alL{cUWt$Z}-x=*J`U}p+&2Ogw+qL8P<}vA8n;=-&tjwXBd|l
znCjc<SL?siE79%HPSl#I6{pRp+okVmFiY>c&JV2|^$p596t^m<$$ygJlAb52C*dXT
zEB;1|Nz`6wIll{UJNHj6b*|s+H(0HhKmNP)+xhpTUrxUa|GfWe{IBMp_dmtIv48yk
z6#c3A!}Z7U_m*ENzf=Cc_-FW^^FPBsy+6Lc8-6qWiT{)K=kK3)e;NK>`gP?e$FH7W
z-+wLqed2E#Lp0-b#&e9f8O;8#{agCS;ZOJ<w?Ci$@cx_jzlBMUHIYq$y@M@-rH@IU
z;m)63zs~;r_=Dpo^RM&241c@*cK;p!>-qO7UroMT`SkhY+mHJ{-uvkB>BGm*A0t06
z`10oq*Vl<(DnF}#s{L61srie;SGzA#pHF?1eSh}#*B5%vmOTFdkmI56!=lI9&-h-H
zz1aTj!s7)G%^tYkH@ko6{<;URAL>75dHVGE*H<=gUEkTg(|uR>Zr3}LcL(1rer@xn
z^R2@>-S-wB%swo7zx#dthq{m6pBO*A`>^_h=0~27*&mO5e*N{wSIKW~-*0@g`Lg!o
z^!HY8C%^vwYWXYi*J5wV-&}vw^|ttJ%-i<2`(OLKEPpBZ`s3@QH>Gcw-%NO&{(9!?
z?XR!BTJiGoi>J?}o_~Di@cjDo2hTg7Grm~yV)u(}FG5}@Jzw<n_T!*O!4DVSue_UZ
zN9<0{otC?E@0r~Hb+6`b+3krpwQhXBo^*rtCTIe?VRQytVD#9m(POi!evKZRHF|8;
z=&@OY;n*z6Co5=jY!>_Iv00<XW{n=3HF|8;&(UMEMvu*sA3Zi}^w=!I$7YR=yp4{$
zjgGvHj=WKI<W1!Ejk}@`gdQn9KK-cTk?y0J4^<wTJUDXy*MpLWs~=o=KuBKZ;eiJQ
z4|N`!KV^R*@w)hJ`nz9mr@Wo^_Tw9iH-T@~zu|tn;BEQ4s`pDiZ2Hvpwex$*kCY#m
zzwiIP^83Ya)4sj@y6JPuXRR+sKVEoe{%-oaa~~Fdp7;6WXWMU|zA1g1^)32)@eiBd
zhyU?2S+o3RImfz;&4^u;b1JtwZ#{pckg3ou{ug}A{0@9Od7tpE;YsIu&KbZR!~1~0
zT~J4GHopY#Z_Weki`jm&9%L<NO=n4B(q{~2-p;1TKAUwl3meNs=6OsKjO!Rpn0=Vu
zGk$0GWzl3_#2n1R&9aR7A5#j`RmM3Cn*R&`&i-Bf`}gmCzxn?0G54|BuwG<6%(ju$
zjX9Rlmm%f<#eYHn^B8_HtY8pfDEU9}pU=O<|HTYSj6O^POlSVj`<MJr`|rCyNB_?J
zWAKafN61&zFGk<q{8;k))4yp9-~KoKumA7*zxn@>|84)x8D=uxV!X!K%&f(>fy0OU
zChs+VDZ%N2lLV&-`14)iVc^}#<HfU-Hym_mjleE}oBZW`3A`tGGI@S*Tk!nl3Fmd-
zV-$EOP%Wq>JXu6X)Iw~n_-YAF$!n5drJSU<NbQlDFSS(iu;f>%HPU-!rpg&B)G2;b
zY*d`4sHv2vRI5B$b*6fc=6)?-tu>lcHCJh*tJ|ozsi&%Is<W$2RPIo`s9>g0A)hEe
zMZQzMLw>9LS$QD^X2oNQhZL5`3Cng#pO$o%JS52~bwcvDM41GS<YdWBlCz}PWH!p~
zlW$h&m-m&+lDRKwD=|epQCvzwR5DcZiNtAfE3p+KlEP~Qs|4csYxoZG^6}>KD0A=S
z&|sUve1+k|zeoQR7)~)BX5PV?!#0I=8cPpzEMxfpQ~%2U)%;^;*u==rq|3CHX+BdU
zQxr2F%OU1t%y}#|EPgDjS+=pbv#7Gjuqd&tVCiSoWiw}+%+|pEitQXr5_1kyIio03
zF0&3xAL|viOKi{Bwy+hjzGL~yV!-0dvW7K}?IYVy_HvHv?5o*6u>N9w$lA(k&$@#p
zlEsaspZPaaKC>0`A4UzvLI%(O5C47or}pp4pNYR`|Fr$y|Mm6f;~(QbXuiMpru^0F
z7m?5WpFDcR{DAvT_RYQ5_gq!E5`0<l(xUSwXJ(x+J9^^K+Jo;7L>yRuV8=nt!=6Vq
zjx(LeJW+Du)^V2Ot;d!hQ$22fT=@9J<GW5gI9YJ2{#3>(g;QatY)(6#);P8D<cgEZ
zC)-Z^InH!q%E@b|tIobTXK}&(V#uZ4m(E`DzdV{ZM_}F*xa@H0z@;abH(#l|cIige
zt!uZH?uy;pa{uc?$tQWwJYU>>QS~zL)%}-mU#x%O_2TXGDKB=s^nX3|t@``F@7I2;
z_}utq|JTdkJiiD0i2YgfbL-DNKhu6Bd}ID1@Nw5W-?!FpG~V2Pv*b<sYq3`>FDqY6
zcyZ%J+{;O?*xs<dJ@9tsyO;0!-%WUT_1)HYbKfbvPxzqvsr+-o7xizdKUjZG`kD93
z<oBdMtNz(CSTY`Ayvv~QzxSWR|6l)88ICdVGgdN+Fuh<hXSv53!M=e*nM;Oy26q$B
zI^Jl$wR~~>D+C0Eq=ml<FBaJ@a!+KdsH|9>D4%GG=t@y3QFW1B!VDstMNW$z7xNNd
zBhD}3Es-b@CebDFR-##AyLh<x9x+j|n+!7lmi|8SbHdNQ-xdEN8DB8ZWVymr%UHtX
z%(9hr1zSHm1NSw7a^ZS`%iPW!FWDM7s<?W%fAVRGoEH@r4&mFzv79B2`3s}{f4hI4
z3=S*?92T7EY>do{7`HP%XWGRYz}d|c!nd49m}4$$1iLub8lFahUQs8R<H{`R=aiG>
znkAQt=8ANRR)}{>rz#4oMyq(rpBMirR3h|3@D+cuz<H6|QrG3V6}QPAmAENlEA&Y)
zUDQGPy24TwN!2Ne8nX3L-4aX1UWmRIyCwBf;g0eJ1ukg~(bN1{JnOkyxOVf^icOZ*
zmcAoaCUBZ-HM=S6ET-v98`&IrcL<w^uN6JSAIQ<i^!$JL|40AVGp=IZ%T)LeG}UnP
z`>t;bzlr~l_~XxH!<NCC&b0d9sh`<j=YC51c<l3mpN{{JF{U#v{&(iL_AjUJTAwuE
z{d#-&<D~DierNuU_;Kfp^e3T@%AaJuocqf8ecE^J?|Z&+ec$js`p3f`(m!YX68`s=
z;RVC#e~W*+{OtO<=~w3Oz5mXzOyQ8=C}+9<Z~d>^KQn$ve|hoY^vBCzb$;vpkN9u-
zXW9?W?-f5-es}(v`S;L2y?-vh=l!Vuar9ThzdS~H=Jl*^IZb%FxpdfVnC1Sz{&Vk-
z-oG78GF*%KXA8*l*02RI&Sq$2IQaMbFYDjd|2<egvwveY`YZgy=IhokOy5p_H~iK0
zNBg(MSMB%WZyvsy_iD-Oi*NNl&i^X#Gw)}~w<#Z<y>xjx?a`u#kx%pAzWi+W_3}r*
z*8<NLJX!ka_}#a+KHoj?)b4%G*KJ?)-(P$2@bRgKO^@b2x%FK0^_JIHU)Vksc|7fr
z$Yb}%X^&l>HN5%zvEsAjr&I5=U*37-crWVi=0~&MSbkaeZPT}>Uw3`!_$>FK;bq?A
zBM)1i+PyXVBK^(hbNstsFUwvOzYKld`!4m<`p<hlE_|=|UgEvUJF&N%Zx6q-{#5&|
z=GU6PlmAWlo$~F*Cy~z^zR&y@%q+>WnQ1Y@`Ty$~LjUjk>HMYtlh>ELA7B3%{8#!n
z>({GqpT92ue(2}dU%P)j{$cek<nz-Hu^)bXl>8F-HQ}rJ*VkX|zVG_x_lfN_`;*xF
zPwq-Q^n1SbmHkVNXG<OjK2m>l{sHS<v77SOUtHgK+vWlHqYd{XZb@7#ySDhI`5m!)
z7w?_9V|OF|(%Ca#PWqgheY)g~{aN1g-IrEe>%X!8YWg{cWA_gV9&SCsa=!TTi7Rg|
zRi8U>is4lMNrU4xN8TJ>alGsN>T7#%7+-sFA^-H#6S*hbPCq%9ektUN^A(*-k>{_T
z3%+>tivJC+JFO2)p4+|`|5Eu$>`lhgHxIcV-+9sU@$*kD1|jC-OpZ*_%nnTK|Mva7
z{-gZQe&!k6zXbe+F7kil`M{&iXUWemv`9=^%0{|WDp^uVN?B&V%q;1@QdeYVDDGD8
z*O{!VuC-p}s=^Y5DwWGx=?3=3r;OE&*Bf0kvNrmze@o|r)?}S#qe~V(HaBffSOu9+
zHo0i@#_*2OGn2!n-6r1*|LMCJlo=WuM(IoFuGZ$&IjB2VZ=#O3`c{Qr>F1L1GLDK>
zs>SNj>XK?#Rn^sl)EBEnD?XP$rI4rmK#fT&Nc)zita^s3nW~-IL-lHnboCo5rV4kY
zDkL-{!z3?>ZxIU-uaWGNk(7TcXCVDn)K5rTXu9YxsXOwGN>xhP3TI@?r7ugI6b%+`
z5LzefDrP8|A^kwAK&(l?o5zyt5a&@YRh}9iA@0rW_n6l*1~Fb{T+O7zyp&mj?Gl$e
z9|Ny8`|bbpe@*)-_9vS`g)N(Bu7I{+3I8V^F0O}c>CCDO%>P*##aRnD_jA{BU1yuY
zyoiaNrJU_G#|7>@J}bUJZWGQj&LA#(uKAn-T<>_xL`!A(<qk{s3Z3NM&%TlE3;P7_
z%>t1ki$n~BqWCUzIdgnvNo8pId+gr<<}fZdepZ3AyoKEJI2k!Y*@HM@xDIgt;=0OV
z%3;Ih$kV|6iNljUo+E|lfxvsga6WI2cTDUI?;rpE_mXiX`!nuW+#5LeuwQ2T%gV?6
z@$dVeH9yV&++yfwP3Lgn%;b#Xyv5PXahKx;2N!z^v-kgpf2RM9{LlAa|L@vAVgFaK
z=y6tY?qPe!9LUJ||LZ?yrk(8aeCGti1#j`O@^0g~z*EY7n!||wH+vt~HQprx>jY;C
z6!Mku$Z-GTYUlaMdz+_(vzX15&7L!WXDN>b&s|<E!4D#g5+M@HMau-sc{DiW*}b{0
z3WP{}km;7WB=JzBTgXySl3$!Bkn0zhC$EpdX5mCJ9kG2vyZF}eDDigk-x6Fbn8t6#
z`<c6#`w15d=L)thtdrR;a_Dd`;*RC4W6}QW|8>dxmv2vhDE}7m>%q@0-!wlfe3<^$
z{k8tfhUdGUS-$9hbM)iMFPA<qdawH0>xKPukr#%qR=z&)+U0f6t3xlJzTkgu`$XZ9
z^do~OPoFJ#A@cIT^XHFa?+e@&x@YxB{e|t@+V?^4cE6ha-0PX=lP~vGZ%f~Bz2R{u
z;}P3)ofoH`$v^de;`>DLsroavXIr0GKK}Ty^Wo%27amJI5qc8#^wkTlcYU9@zlwf+
z|4H|Q={uEoA3m!6)cSv(!TEpwU*><+jPWclnU^#E{XhS|C!+@I8jf2W8(9UIco+`;
zKguwJ=?e2=W-q1}46=-2%okb3*ge@Nvg>jdb2sqh@yy|l;wWRX`}_I#pa1LG7xG@>
zy~`QP>dbV2A@G0xpM^h4zrOvv;fvSz$v=1h4ExUXdE-0vw>j?@e6s!Q_ObrmzSr!p
zj=to4#r=BT+od1)Kf8X_{I36_^83|K0k2vgt-0^^i0#FkH#^@bynOV;?veGwp8LCQ
z%ij2PP5%0^>tAn`JecwL?Sl)qPF+pAa`Kw)EsNVtw?b}Ry%}<2^>y7FOK<$W5r1>(
z%{8|^-#L1};33=n&)2yw9z6g5%AUJAPmN!6ykLCU@{;-G;pgSg9zR+7Xzhck4-=lJ
zeK_(><Xhy&uW#SHF?u`k-QM@h-ye9V{5JNr=WENiOdtMyI{M}4m)~Dk{VHPE!EDFm
z{^$Ew&M(WqDgRFXFV5u3)Xbp&|Hgk8#`lb>j4J;`f93p;|1Hg!!}f^Hn>mr8h{1>H
z74v82^~|*_pO~*Q{P^$Ba-Y+ZhmU&!M>WS~ZgGBQes-P~_Ek(%{#!H1vxsv1AKe5u
zx(RM{6Wr(~xY12;qnqGHH^Gf=f*aigH@XRKbQ9d@Cb&D_MmNEYZh{-#1UI?~Zgdme
z=q9+)O>i{a1UI_WW^}2|=u(@ZxzuKK)dj9q7s+Ce#oWZxB~D5{kg}FhmUWU%lszOf
zL8@O|S|nP)jrTFvRSs2-WX@!6ecqXTwfy1yc6=LoUU3C*I&yfhN3-2xb!MHzqQ%<G
zX3Fly?!%tM{+fL^hbZT74r@+1E*|bq?kC*Sc{+GE@xA8XEWjw}Dd;I!C1AwQ&v%+f
zg}a$kh2tli0~-h1G}d|+bLPWLz04vkrOf5byIE>j6IiuaPq9p3S<7O_(!s3Ae340n
zIgRBps|<TIM=_@imlRhpXCenPhZV<6j&B?-oUUATTtZxXILkPHb1dZ0;9z7gWBtn<
z#H7w3`%mGo)!$`*wf_D3_x!&*L)QQOe|7%k{qp#6{#)O-XWtfnzxSi@SMr~`f7Sop
z{VVeC^S>wmCo_05x-ls*7cjdsi!p04$1|5PA7(z!vVd(ahXdDC?$tbLyx)0)`MCIU
zc>{P;c;$Jsd7QYfa$ey`;*jE)%l?h6gUybugEf+6Ba<@YLxwWOnT%H$*_mgs@Uro<
z3$VAasj^A2wXo^4UuWOR!Ng_HJ()*=FNZ%_z+A9M@T@?bz(;}8f<8jh!X_fuMFd5|
zMVE>SiftBi6yG4ODsfxlu4I?gR;l+=)zYdm=VXjzyJbJfzL8xn>o02~vr=liq`hRO
zM2Yx+u^4eViAITZi3SM<NpH#95_}Tz;`hW##r}vUiaLni5Ooog6?-LGDLPN&k+8Hd
zi_jy1dHfE1O1!_hcX6q4IdDmGS#us>U&hM9ti*8k&(R+OUypv0`>6iD>}|%ItFPC*
zE_<!;I_Y)b8_T!f-)6kac~|gm?YlYeKD<?Ud+N=zHy_`ue8cdj{B^-=v)68~JYL$p
zsDHNTiPGcFhx!i=+&_Q+!u^N$6&{2<@O;4g;OhP8`wjON-TiUr|Lrxms&7uX;dVXw
zYU<_Si(2PX&OSWNb!y59krUa+?;mwLVtCm5P}#wq2Qm+69{h6X{E^IK*(WZXdUdAj
z+{yEi7rHNey<m1x=VJK9ofq$4e0cHp#p;V^E(BgExp3#g)eEi{4xFz%A96nU{HgP|
zFQ{Cyxcuod>y_h|cV9kn`SazOSN2}jxc>LXlUucSZr{Cm@BIA<54S#UefsJ7^4B`=
zEj|{0I`sMNm)5UcUj@J3`LgKij&C;Kmw(^+!}Zs<-^G6?{(JO)4P!KODT^B`FWVuu
z<?P=%BDsRNe{d)B1oJH7uHkCrtmI_k;^x}P+0N<6xr0N3V;cKH_Ck&@&bORJT%6oX
zxD9#c^GNVc=Do$M#n;5=&NqkmD6a|sYJus39KtE0r^Vk%u9V&`vs(6xT#x)-xdpN^
zvgc(IWte3oWyEEl%jqhpDR3xUSD2~Ttf;Q|L1Di_h{7ECU2^wiC1i_a9!YCSCrI&0
z&6m0<{Z7V6_M6OYnb$J!WxmOT%7)3!m*1k0sd!z{OliN8xAJ=BXUe~oA1O~#-m0Xh
zSS8OZCndX1#!qI0w1c#vbc@t7iF0BcVhcqth|Cn`6RHsK<$uTfgxitp5vLYc0oP70
zeeS>947}aEro1scPTYSvS98R(%d_dTeq%9XO=jg})n?IWPGquVtp5M`uk+uFf4Kf6
z{MP$z`kVE)^DpI}YCoF3U-|a)tJ~KvUp9Sl{4(Y9xla#2&i@elKL1_u+xxF`U+sRG
z`;y^h@e9ruU!I?QKJA&yllu=%AMoC{xo3aR=f228kH-d2&7XaErv5_jWz0*ym-k<M
ze_{8s^ku`#=9f(`H@tlJvhk(a3#I25pVdBF_;mLZl_z%}e|aqb<p1O6kG-EPeiHrU
z<m0%<I*)B0uX=3$r2a|%leQ=7PZ^%IJX3qV>;>!ViEl5xzw=S!%k6IpKkI%m|4I0_
zn1O@o71I*tDwZzRLN-2jISvLcS)QZ3Ed03wc|vbR+{7g%PfFFu?2%QL+a-Ha=A-lk
z={V_sQd6Wfq>f02O7=^5iYtp{iq03gD!fXVPo!EzSd?4rt60DIKXGA+#o|2TWnz0p
zokeX#gG5b4g+&8IpNTSx`HS8YaT1*_8X)>##7gv@sH@m=(F?+-1hWK8`CEA}ai8R>
z=9<G*z%`pwoO3Z}Ay+h)FXwyqRct}5=b6iyPB89dlxNz`bf3weiJ56K(=Vn3rXEH&
zMq5Tv#<dK4{&)Xd_gCZZ^gkzlfBMz>Ys)W|-!i|2erx<r_}%#X$FGi`m%i`*X7_FL
z*I!?{zFhsX_p9-@-mmAsT=-J|<>aS;4@~b&-W+`8_OkN1;WPVZub#<0k9gksV#TY6
zZ~ER<zt{fo>%+Q_T%W{0DSVpysqFLb&zC>feJ=le{<G~D(=VN$?|dr$wDV)thphMa
z-_3Ye@h<0`?>oJB7VkLT6}|oVX6KvIH)mg;c$NNg)$=`1^`5vtj(#Neu<*gD2lF5B
z-H*7t^3KLP+wWN1;l3?&%jDLQTPC-PZ`RyYyLIFi`)#M&dUts4{kyOJaK}TpM<*Wf
zJ~n^c^jP%Ck0-LvT%P}XuK!}mi)$}$y}J7<^_A1Blvnd!-Fv0~djIR|Z%p2$yjT0s
z{o(uv$&acZ?|ta~u=In@$FCoYKdF4W_HoCjNnhN*t@)n+^WyLDf4U6Dj4e!SSYp{`
zu(`9DvPrPrVm-<Fo^>Ld496|bKU|tTntb5`*912S<q9*21d0TU7>PU<E*G{CHWR)t
zbVM*uU;%$Dzdrv=zU916cy{pI;nC;)#T&qXLqJ66fsnGWn6Rj@z3?L8Xpwx;lVZXW
zHzdADyp!OOERvLzS}X-RYBEuFsa%$vfZQIr9{G0pTXLLojk3#S|I6~oWy_tDpR0IT
z$ynJ!IYN1>@>AvI$`Q&Q%DKu@l~*cfDt}OFRFYKkSJYB)lDConC|{>gprEPnReq<u
zp}dD2pX@8?KT<y>7fQH`+ly@w4HrEmvQ0Qq=(9j8zY1R`&vLFHP6bW{&IFD|b{lpf
zb}zPbEPt4C7`+)z{!9OJ{%6N`wXYLDrF?k&j``iuxBYLozPbOJ<ISBn>2JH<u6=v{
zZRgu9Z)UzudgcFe+l#d?vR;%wZ+m9?jOCfvQ{E>BAFX(J{DItqbNBDumwb@%V9|rx
z2N&*tzE^%P^q$$hw|9H)7TleFx9sluJLY%(-M)Rh_4dtMS8tl!thssb=F*!7Z}Qxp
zeJAX0$X&U+5_hBSHr)-pJLT@9dn+EOKPq~B^-1fq&KL2oCcL(LbN)@>+u3hdzg_oM
z;+^<=vkxCX=zNU$_~zr@PY$0wKBs?X`y%zV@Y~Vv_kJAtx%F4a?`6N0e{cM?`xo<X
zx!<C{Eq)jOUiACO@5{gE|9<{E=r7Oz|Nnn6+-Lg262s=i{(-%pLz<JH(}|;q-H=_B
z{Q}!owy$ix?1dakoO+y+oEtfBaDCxE#*@T*g!dWm5?&SF**s1>Ry;*KaXd4)4{=p-
zm2;VL?d6Q(wBWShe8};c{Sey*){QK)ncJEA8Lu$}Fx>nf_y5tqpnu=~a{d$ixA*VX
zzX|^w{@?r`#ITe>pV5+ugV~yS2J;q{%d8P>k!*5o-&qyeEZL5-+Ok@+hOvHONnr_O
z5n;_@JHft&L!489a|?$R$29hxY|d;m*nY9?X5(ioXVqo3W|d{_Wf5cHU^&J7pXnXr
z7Y1jBJO7vc_xj)eZ|dLkf4u(8{k`Lt$gdMWJAOL<eEq}q$C~dezOVVt@jd6;ly5J;
zo&Q$#&GDP>w=LgJf8+fo^tJuV<Ij&iaeX@ek>OM6r_G=CeRBD{<V*55=^vAR_WZK>
z{o!}kpA~;p|N8v<_x~8<H>NP=oy;990j!5umDrluF0qNR8?rxOYh;_w_Kxi_n+;nC
z>qM4k%<9bbOurdx81)#988<Tg`=|X^{14ynB|mF^{Q3UvyVj38-=}{|`l|L-=Ig{S
z4?YKfe)Va^C!bHNKE{81{c+VNrq7C>yFM3vIrdfcyTcFWpUFR`{5=2j&CkO>KmA<w
zOY8TZUvGXk{XF=y_gDY#s6V`ax&KN1zxqF)!GbZ8X)E(BmUPxBtUFoXvYNA%vaMos
zV~^)B<m}-*!TEvHj!TQ%mgf=A72bUQ2?8?(f&}UXgan@n9uNu<J|=7-qA8LtJWa??
zC{<{+&}E^=LY~4gBD+Mz&x?uY*JO*wi{BPIAbMIPTO?B?U4&1>S$Lz+S)p%2vxKCC
zJ_~*m+$qQ|_=W!kpDLd%?_%y2&TRIBtXV8-%uGxkj2{`a8J{s~Gxad?GTJe^GJarq
z%@E6YnNfpDlBu6Dg;Af$h50!18RpH*6PYua|1xPXEoA)6n8f74^oMabV>@FRqcLMW
z!`lCn|6BjP|7-cT@{j4CMSrgU@%_{KTk!XTU)O%U_{H_x_V>cya)0dpy!hSn+y3|N
zUzdI^`%(Tq@!R~bwqM_TnfYbmm)BqBep&E&#;1KB?LSWZ$oc8X$Fm=eKN)^%`>6e~
z=;OPOIiI>e@qViNnD;T`BmYO1j~hO&`84@6&zFCnwZEu;t@^g<`?DYKe|G&k@GIz-
z&(Fvo&%bZ`-u7MQd-XSqZ%e+}d~f<b<NM9;u0JOJ*!zR;=f0o2e{K71^oRY=gx~JJ
z?SJ$9F8%fD=bE2wKcjxG_#yl~=WEWFn9ol>?)q^4{hW6mZ+E^v`7--O{EOW$(qAln
ze(M>>Gli$RPwXEbe01T_x<{WMnml~}z~y1squY-^Jjs2!>gl?tVNWkTX@Bze$@8Z(
zo^?HQdnW(v+S9V9;!h7enfv7Plgg*vPmP~Ge=_OGpT|ERO?xQ&;Q74=cc0vmyVG|&
z@b=Q%`gfA<G~N-o`~9x)eU%3y50f65JlX$r+Oyc_vtE3B`Ri5P>q)OWUhjO(_Qw9r
zu{X@`mb_>C$o}cpCx$O?zq<U;{Z;!r`p>;Ta(^ZMUjD=USN31*e+7nO249B9|2O=v
z_<#Lh_P@RVtp7XzKmPC5-{pS_f9wC+{3GrA?r*cdJ^wc8yXz0epE|#`{(knS_V4Sz
z>;66cZ^@X>bdq@%YdrgQj{Tf>xe|C>`J@E$1>=O`g=dJI7Tqo8Bi<@rF8)nygQ$s!
ztME@DUtu+oqaq@rQ$(4?zKI%&>4;C3V3v9!bx0~ms#Z!$`i-=_>~=XXg;d3-if0ww
z6qhI%C>)X3m9LjuAX_aHChZ_4FR3LlOWaHRu-J9cpCWrjeu+eio)A?La}ZM(J0$u?
z#7E?-@K)g(VHM#Vq3MELf)xT&`S0<~;)~?_&wGS-JFgoblR&7Do=Co^gIKp%iI}$7
zR?!`z2gL4){}3+}7ZLv=_EL;jJV$(;_!V&x2`32^30;W^68j~lOSnk1h&PIz6D<`L
z6Fn$$OC(HmmZ-YuE#a4fU-(n`7V@}pd2k4^e_-unkzt<1*vyd1AkM(_U+v$AzjOX_
z|LyoA`RCDZp+CldUjM%IyZO(>zkdH0Go&+iGj3p9$@q>Th~YTH1jZeV1&n7Hj{ax;
z|K*>||M>rk3?7U|O!JuLGWjz-V9aG)!{G9N>ff+Gy}uv+iu}d-YyHo{pB+Elev14!
z@lF5R)32Amc78kf{n3xRKf`~q{CfSf?AMiFr+%IIweMHsFRx#`zq)>^|Frp8^wad`
z?jQ9(4*r<<v-;P^U%J0*f5-oB|JD3++K<!U+rKA%pZTrkYuA_d&yzkK|M>g^(}&>q
zt?ydj_Pm+;dg`l{FOR+W`aI@2_jAeTho2vKvG(P(S6g2<yvcv_{q^(L0dL;D+445`
z-T!yn-dlfA`0(p}{fFfrT|O&)-Tdw6_qd<4f5rWl`eXh#<zLc&Wro=dj~R*>Gni&E
zpJiFaD#;eb7R2_9^%d(AHYSe49O0ZZI2Us=a7A*daq)7^;;QAg=ef$`&3m5LpU;&q
zk5`xH7uPAywH!PgOdJ9njvNO$csSQ`++~+xzsPo#Z5vwy+XS{rY$9w1tZXa{%>7Is
z8T}c`|4IB^_nY}w>W`P-c70X-TK46|ryn0`-p_bv_|Ev9%{%pXprhEb-|D}KcrE)n
z{B_Oi<F8s?E_(6p`P*m8&m^BkKAZgP`7_1mpPo&9CizV6>Bh$%kE9;nyYGLm_s-2*
zX*W%7{Jds&mG$z|3##XrpFMo0<&5K*f2U2)NS*n5>gUP3C(oW-adP2F<&#z?ZXZ8+
zJok9garP67PAogoeq!ATmXn)KI-Q<!_R9Gw7uhaLUzvSn)|E3?E?*71?s8-9ji{U9
zw{G2Xyq$38{M|G6mfzp;p!ngwhn$bp9`!z)@nHA;v-d9E<+wZXj@KRaI~I4g+^M<y
z{qD}YR(DJ9mfb76Z}p({;grWqp1M6h`(o`Y!?)5O&VI80dj9+Gp9a5w{ATzo`_JXS
zJwq?U7KW{i?#!!LLfC9Lx;cxuthklAUAaWLSh@Xqa(Sisn)n3yi})k?v-n^0R|upC
zJ{K|-(G+zM(-Q9xUnV|NJVSiGxSzy7iA|Dor0z*ClvR`ulb4gfEB{czU-5^cxpJ2Z
zn_8K=rN&2%Nm|)D+jRHqt=3;_z-q)|tYBPgywN1n^u39nNuJ3rlgTC(CeEhCX1eAN
z&97T9TgF<fw5YTSwehjNX{%-T+Q!;)pDCx=F^i{`D=jx!{jyH7&a~_`=Q6Ldcx1W8
zLdP`Ipj^vUb*u6W)$bZox(5v3o1Qh-GxIf`W+ZQvsq3OzDR);|TJEB<hvqpQR)a#L
zIR*@xI&u!8<$~?}Z@CV$>2OTuxx^RB|4?9x$Y;sBvL|KF%S@L0r0_w(THZ`nQD%uu
zw2X{ot7xrAhUixD_fi{VPRRJkGRtn36cvjRjS!O;|0d=wvY0P|V-oWjhFSkp|F33X
zWBJWy$+nZJ?*FSl8-Ch+nf1EramT~oPnN&Xc(dzc^mn75x4+81t9`ofzU7@6HzcmP
zUo*cUa{JibIrp;eTRhHv{^q&bv*M>E&-`ATc_H>p|M82*zn?liUGh-v?t)u4?=(J~
z|5)Wo#<LgC6QA-tyna{nPT-vvch}#OxtnxT=IXvnk1jpFEPsvf_Nxa)Pxe1K{z&g(
z@FR)GhwruCcz8|k=9$}a5AHti{lNA;=;!Tk4IiJsc6grtMCft(!^aOQo;-Y+{3iHq
z!v~kI20uUj68RPUGv<#6!<_$De!KqK`aA!x;NR6hPkpujy865OuZw>U{eR9Fz;u8i
z`fu2;M?XLQI`cd8kK>=yzkGh4_`&z{{I7%mo-(&`tl`?t`HX{~TZK1*yPN$eivhDb
za}Vok&cl3mBH|L~B+|ux3gmJXv3+9s!lcV6#gxvnnsp7UDC<|&|D4=>TX>DQ4Y>F4
zr1JdY^kUCp&1KPNKF{#u@7dose_8%^_*4D&>;HJ>I+hiTO@BFmSN!VzRr`zY*T)}^
zziNHl_4dUZp0_OTI6hi@dG_V%=h{yeA2mOSybF8T`K0#oq$li8cRXWxUH^X7NBs{)
zudY36zms@<{iT!Vub-1UFMDzQ<(XHyuIk*3xf^!><Ne$Rz7HiIdq27T==p=BhaHdB
zJj}iyeb?vCqB{Y1Tki_ow|*?}{O&XB$NTR~+&gjS?JcWYjJIdpQo6C>YTD(w7wXTq
zT$H@Ra&_O8j?2E6R$tP;eB#2hGyhJ#KgD<Y?djv^=3M@HqxjB-I|a82udTV5aPH9=
z_tR;oHl9hlu<26&6_@K(x9sonJ#2n(_rBeeoL7u*vR~bN<?{CSJN0*|uYbO1c)9TP
z=QqvobU*F+cK2uSui_tHzFL3b|2+9q=I8fcIKQQRYx%nAv-Ib8U(S5@`t|E~^Pek!
z691(!STb@mUSycbu!q5g=>RJax1XSq_(|yra>DZOWW{7^BsYu2itZKF7iW}`kad>J
zmG4nrtbSkJRsEQzgKmeOvff->F1<JU#fH-j67)XnJl8p+dr9wyzL;^f#aC-Xt2L&&
zMz8hv=^oU+qqR-jK(|46r;f8ut<FW=-3CU+OvWD!4;bw?Q8C+Xw#-c0OvLPjS&Dh8
zg_3o$eXjFt*CtmMmoJXp_Dr@WRy^i{ru$5}%?r#bOlO#_wpeZXz*5P|&iavUx5G{c
z4f`IuPj(J=Y1W$NON@h!w2c-RW*hA{n{2bkp4UFt=Cb7u^Ip?H<Nt<_3||<U8cxyY
z(LJQ)sdYyyL@QULPR&HMK&3}{j^Zu3HklPt0uoO}T7^6Xdj!}7&hWSL`}5^;=doup
z3o(57yZ!HjfAjy}`WN>1^q;AJKmNV-=gOaa1|3#T_F^_ewnuDb>?_$~nLGZ?{B_~`
z`!D^U&A;vaeUo85^8w~1jPw7!`KABu#K(;Ht?z2z#=Ta3!T5CP<C~8!J-+<N{o$kg
z{`WoZC*M1IhvCkZ+oiXDTz`L6=&JbDhO5%oysp2wA#yY3+M-MH7b4G1JQH@N@Z9VR
ziWk{0DPE1f;eWH^=Dypd_vbxIeZu^7_0wg~cf5>$Df7JhDbrJnC*F_OKR)y<^UeH^
za-Uy)6#Z!Uq3*5utLHD4y?pq}<n`B=axcQ4KYD)u#iN&JUoCq5`OThpJRjzM$o}~4
zQ|%Y2uQR{={(Sw5{&(-6KYnHZ-S9u_f9T(@zomY6{;d8H|MS?dcfaocto$+c`_%7d
z-xqu}{krix*RSfI4nH`5uK2a_m*$V2&!+FY-%fg~__p({)W@=Ke}As{ss8Q8#})4k
z-(<b4dH(3R{mZW}E<Jnpg!8G?^W!fsyk7a{+-skgozKLd7Ch~Fp7dJl!;Q~Bzs&t&
z@pZ*_^<PVWo%^x<>;6wSKE!{R_tEwX|M&Wz6MwV+KJiuiz0jLyuasWsJv;Zp^xf7^
z!C!ZN+x)}(Pxn8aKZn21{2KRF;_IU?^S*BW@#)`s7GG9ACf@%m|J?d{?t9z!DL+Df
zN&mL``TA@3=UJbof4cbT^T!SEU%xr?visSACtgpZpBuf}@#fZ>>#wH2)O>aBmBXv)
zFRY(EdpzaQ^GDfFY@W?}@#(eRJJUCRpG7~~abNkK`(5|D>h}$w<h;^*m-k`X7v-Ni
zzdFA+eJ=jM^<mP7)b~8^Gd_3!?EG`_kHDXdpN?PUKFNM``*h|@)VCvFe|)z8`1!5R
zn}=_5-fnoK_`3du`BTov5sxGvyFO!nUHXps!`}}RKi>Oz>eIx}NuOBX^SrKkvEzmR
z>&@?`d}8`8{7dGi?w4Kf-n?dcJ@4i17vEm9ev16g{=4&EHB%$o4UTs95|-!8OIc^L
zS+MWn+{l|KSS<WqXpMjf-wQ4~c22g%9P&Khcmw$Acqegpa+Y(ja-QH?$GwfKjgyP>
zDSJMfJ^LlDT>cK>bTJQ!D5*O#ALS-1d{CUGXd+)Jdr8hw>9}gP<|JKC10KEq8jn@x
zDzPef$_mOBDKx9rYi4Qd>G~PWFv>LK(aX?I*E*$jUVFNZrJjzFlG$pr6DE1a1xA(z
z({;D#Ow)U9c+z;0(M<hF9dWJYnrpPAbzbQn)<3VmP}fxZr>4A?tWJ}DtI-UTBW5C&
zT~>#!jI6#|#G7#$rRYA?Qqk7c)iQ83zH9Q_c&UMuPLX=3@+bMDa?y(SRi!jVw2gI{
z^^^4X>)L9usV`DnqHdr*M{T=?wa#tbr#kgoyVTdI+*06`JuAIc)>vVMg0!5n<U(P8
z{y*F~oRc};bF_0bvgI=iGA#Ma^w;RG_FuujPyUqt3H)>JPt)J0f9?Oy|5fpA_opcz
z*gstS!1Foi+o>OnKN`OHeoTKK{r386&bMbj)PCLbbJw4J|6VYpGU+gi{A2#}>i3^N
z(f=0z-^jRvwT1I9XAnC(%Vvh#f9CwG`myHc)8DWE#Qo;_q5P%c!>6~W-bB9febfKO
z>dn(vAukzT{Cmdx-1SBD%f^>$UY>e2>5bC6<?m*^ef-Me`M<|*k6t`veH`&@*Q=<v
zKi+J9wc&a0liv>}-Q~FBaIgK5!PApZmpq&Jg7Fp0%d)45k2XI_ee(73mq)TsC%jC2
zbNw~b%h0E(4-M}gy7~Ubom-dg`aQ6G*m|G&&f^=Ou1&f6?n=v5;p<axT)(mA`pIiM
zuAjZR{kG0s?|YN(w?5qeIQMDsv-8jVo+Un=_k{h)$;TB>il3FfeDV7IYq?j|FMOVT
zc{uy-p*zX<IUdh{s`u=}(|gZ$K7ao7+@oU;Pd~c$nCpq$6P{=DUe&%6ez)aS;&bUI
zO%Igsx!(KupzvwfOVL*kpPzi9`pERb_Io?-7Tn`~nEm+bW6j4i9_@G}^~CAf_2*hI
zyI)^@C;N%}>zi+^-;aIe`tt8%*ZbhN>ThK~<bE;v-uq+i&+R`qfB*ZX>2vlMnICum
zNHUyaEMb;oea-rerIE4jU(f$*Ooy4zF>hoS;yWn#K_HGdiqn#95wjcPB*q}N+dOiD
zTLqo?O}H7@nVEI}PyO@v_qjh0{(k<e@MqP}gFkG4?)>@Vr|qxXzZU#S_@Be5%M{48
zhH>+M&cBC#b^R3o)$!-a{}oJ!nUk5Lm^>Ix{9*q!>(|e}Zy0y8OlGZMZD7k_H)Gwv
zD8=Z>EYHf$mcd@j6~KRA=&;~Q-nm?M91$#44AuW@7!R?0XVYUl!6flt^H0&wXWu-&
zCVqYT&Flx$kCop9zbbrr`eoU-=<fmFcl>bt&Gpyo9}hz$BM0NY|Hc0_{yzCV=Xb*2
z4hBAE2^JTYwakAQ&;MWjkCP#s#es7U_cQMGJo|ZNc~^5?XP?RD$Z?2^k=vHLmw&40
zZt;VneL^q!q`4b73fN6JIC#Danu*1WB?ve02lE`_6kum#6=dDP_L_Yi`x-V`whXq-
zY=LZ^Y(?yToNKtZ@^uI*3CoMz5GfNmB=ku@RiIJerl73IeKAJKCP@d0Nur)Y>H-S|
z{t2xVohP0vaaqDsszTaLQeRYCWTIG`M7rc+>D%%-D*x3EtA18oC1)Y?QX*Mwv$&<q
zRfRI;G-XyrDY@g)Qzi4nmy6vK7nG`!)|cKWX)n<y9xi@c?7gV0*j<UQGE55cN^4Z+
zs;yBCP~w;WDYHVlNLo`?LLo<KtFo`kPUWSF>hc}3#j-*2Zc6$pS5(fao>u#!>Y*H<
zP$73vu3h1RqP?=A`YUY~-8`*0^<Whxg?rL@lEsp@r7h(5D}0p`mwq6|Al%D8pQnwh
zgENHlFGmUceil<EO~xk-m;Our3;E0SFZur*hDOHQjHZmP|I7c({5kQv(D&3Ig1-&^
z&18^ZZ2m9)xBFMqcd@ToU-^H`{Ppkm<Ubq!Z2zPAr~N1EH;%6pztw&J@V)HU>VGF0
zt(cDd*Zke`?bD}C@44RIcoY9V^i%Zb)gN`<)x7e2?)jwrVd?$n_om%9dr*46`tGvZ
zk8eG^Re8(xCc{m~+lTJmdeHIE?2*=^i}$zOesewJ>i*0Am*uX;-)Ok?^w#@Zfw$IN
z&%Y9WdG!_U>#jFFZ#mqWe(T$<n46y0Y_B}HJo5_wwKdll-SoU$@G#<0#{I*$8E$%A
zJAaw+^4=>qZ=AUM^1+)24EJB%sk*IuYwHcR8@4xJ-A=m~a6jVSw>$cG8}41b*MIN%
z{pF9YJ<)u2=J~mo(y!0H(s~*2T;rMYvmMW;zPk0+?p@RS<L@Hg2)`757XIYn6P@P{
zFMqspdmI1$>if&@UcR~V%KKH!YqdAdZw)_keYg6({g3LOtv@}!MSo8GxbZ{sr>t)u
zem(hX`Y-IymY=fUH-35jsqj<k=kBkUzSsSj_1)|Hq#qf-&i#J-XWzf~jJ2$p?AO>0
zIX<&*W=&?g@_+Kb)BnU7f3j@jxWjdx`vFfauMiI>7YDZ_|4E_!BEe$85}Hzzq*h4$
z7F7}bA$mcKSK^G+X}MhGRP_Ljb*i@&73H|43dH0^bH(x{`(#ea&XjSLIwsC6`bm(N
z-<a2c*OD)T&zkob_YbbWTq@jETq`+SxGwWt<y$1cCj3-nm&h-nR|0JOp}amkmv|oW
z{T1*MDihWgSuW(mzm+GHXBuxZKeymUp{pVlVn0Rq31<jy;=jxHldqrOPEcGlQSz76
zCCPSiHjy8Ek2v$$a#+8xz2hw5ap%3yUC(irg_WuM|NlRee{cA$@`vS*=5MxNl|Q%r
z{P8pB*R<bL{&xHiW_-%Dnc0HbknzjkfM3VIt^aKPso-<;w<kY%f2sdk{?p}0&DTkv
zcs{-Pyz}d+Z-;-Z{N4W7;IH0qhM!-)pZLD<`-~rLzo-6}X4YkW%<_w|?r-1^t<UG*
zr@iO=c=B_=ckf^MzwLfk|5E-r@q6XBw_l^b-S~e0hxZSs?*`v*d_VZ(#ZTYg;(vqw
zPhnijoWlBoHH+mhBO^o3zvqAI|8)Q3U`k+NW<ADwoUM(OmFdj?*$lgx_OJ-CyK$Z8
z)f3PWFy;HfUCzbDb(8BLkCMP5(WTO|@>ArG%W=#8mpmZ$T!c^5R6JBlMfS6tx_p%E
zKB>JDy<%I0CknNQ<cix$+DUzv%9frgogytQJyH6LOuL+j!Y{=PWj5ur3Jh}Q(%KUF
zqB}*J#CRqDNtMb>kX<DkF7r%sxA*~Z4yihsB)MY>UdrMsJxaUerDV@YFP45OEiP-W
zV5cIazD`|PZGqBk*@NOg1Q+oJa@TUHb6fKs<`d`B;?dx0XSZU>V!ZP|`oH%7&wpiq
zGyK^6_1Nd*A18iz{-NyCrZ39hLw+v&Rr%}ekIP@JKOKGd<Mqr}Ctja_$MLc6qw1$s
zpC)~n^XBR+xi<&jw!Yi)UgFc-FS%dmexCgy^<Dg1!M7LR@_pF%dHN5=zs&z_{w@AJ
z_j}H#l(#=#E_xyOGUL_F*ZbaBz7cp;`ds2E@1t|~-rr?;(C{elap~jfk2gP_^mx_d
z8BdI#J%1ki^6)F`H&$<VzNvY2@A;OeCXWw3IPk#wvG>#8Pxn2GdT#VA<FV%h(|aQK
z9PWEQnDemv$)4wnUwA)vcpCNi-9wv)wGT@k?|A;=&5{qSpSnI?cz^!Qix*p-?s>BD
z>G9{zuaCc%{cQC`?MvS0vX7tN?RdN4oz=(7pVPnUfBX2Y;(N*WvhQ&}9DddPIrOiH
zv4dqU+itcEtZFRZ7`OhP_V3icP=*&wysRv2-0Yv(3)!-mtC*fJKVflUeap6$b3S(&
zcP{63wx2A5ERUF3SWP(OdGm#4ihY(iDw!m?Q#?*|vT(5QArWEm6v^4rjj~5&e@X9=
zl$3B0TPh+YS|gq!xlgi8^0UNd@n51Eq8~&{#UDwml5&v!Bfnm;T5+;`oa|v~38`rk
zm6Fo3QxzJNlvUzXn^pEH^2x82y)Nr2Z=twZ*;>6#%UL^4^RilpilEX}c@KGA#X~9<
z8d6#xG{ZG&RqrdPOK%n%U4=!#Dy)G!?hLyAi**`HZVxXr`Q83oB32?NgmXm1#G1rr
ziyRbu%$LhOi~Spm0*fV!8cPK8TE>X~-~O=w{`1rE*Noq<{#^d6^8X1#4CBrJTmOFk
z&HU@$_x5jlzIpy&{#E(w-p^}4-~D|0z3hwS=M7(0e$Dwh|6Ax!`Clu(YkhJ1IQgCO
z8@t!*-ne~G{<?@KliPs3ojHs#fFX-1hix6lcg{ebMSQA)X~H)I9r&K`=JRU{lnAaD
z5*K0<bP@b0v|D7Z=xwn%5|<>CrQgdW$b?E4Nxzb^le{DTSL~(OT=5U$7bNyem&jU5
zD@aO8{t!17pCqweyh+4RxLmkZWQu6Ch=y>qNVsT%*k*|tl4g>Ak_#j@iZP2;iF_6=
z5}d-fi&vjFgJ%y<KCd939UnLEBA!XS%)C>%19$}Z@_AXf)w$+zJ>!z)dBG#kt;^ZT
z@ta+g{VmH$M#X=pfAoF5@?G)Qik}KU&3~-?KIO;EAEiHXe$M{U{O$GENnb0zuzykf
zV*mN_$JUQ~KM238c@y(?*L#o8Grw&4`0w4yw_dNqUbsGOdgT2u@ImtZ4G-=-zV>|P
z>zi---qgRn`Ih(foac<sA3bM$&h&Kj%=s&C^WK@hw|@WTUC6uL@9urP`E~bqxu2hY
zul=`{L4(PdX&vJ}28sVw|MM6<85<ac8J{!qGhbkCVLr*+&T^c!g#8C=Jd-}tZst!+
z42(tpcl?w2Z}b1c-?BfyN6(xeJ#&8a%=ytX=h;TjoF6@N{@Uo7^Mucw|2E~j%Fn=G
zSAH%0z4{OP->rY#|H%BY_|y4E^snf@^Z#!%Ix?SRmStJO;>)^`)st0=<t<YJ(+{RE
z%pNS?S>o8waawYfabD*L<8a|P!coql%&y2bf%Q0x0gC|3M`l@;87z}oELikd{xLT(
zTQJ{dVqq#{Jj__gbdsr%=^^7$#u7$hhJF7I{xkpo@E`L(tG_XSbbfdJtonZKYuwj_
zuTft)z6E?Q`O)=L{@24FZr>h%-u2PveZbqoH@0sY-YkAQ_x<^g_dnNu)&92mYv-4~
z&$B=0e_{Lj{>!p2v%W0&V)EtY=Urd&zis?}<VX0gjla$R)ckq)N9yn6KWTp~{(Sip
z`)}p{9}Ewf4zV0(RcEVWJH_^eZ5>-AD+99_qc=mve~*9s|MvWQ@^8k!_5WP{`!RGd
zvN3ft-eGWJ$o*gYZ{6R!e}4bY`tAR_?f2c^?|xtT-Tgc1SL}}y-_*aY`8wf?*QYD*
zxL?nBzV^wSM|&Pld6@gq_#w~3h6e%n<?lJ)wYYQR)`OexZt~o|bZ5~$h6nr)6Cd?H
zp7B`X@r=iFo}78o|HSF(-e&?YY+m}kI{xbOtMb?XU-!MSf4B96>L=Szmp@8;6#C%!
zzV%(|+lDtg-kf~v^S<<>&=<pRx!)~*9RKm)$KfB=Kiz-n{l5Qu%b$|JeE;13A7Pxr
zQo*)}{WM22XANgMX9_1j=S_|`97j0LaeU?s;oi@!!PCPl!~ceVt-xBrWkLnQ`-Rnn
z^MqQ2t_#Tt*$Nc$&E_@d-O6)|C!NnkV7maffE51`-WA*poDm$Z9IYIs9M?IPacT1I
z<(tZXOW>U#laR8Ix9}_xMNwzb6wz2wDbb4}A4K{@J;lz7d59kpH<IX+SSisfF-yW!
zB3|r|@J69XAxR+#p#-5FLQ%q}g>MU)3bqJ%3kdQr;nn5+&LhmTkIRDdBD)OxOSXM%
zs_fYu8#xWQHgP6$>}OMDUBsNql*!o1@bkYo!xRQ_#=nf;n7o;%F)w6EXH{WkVx7c#
zi**vKBdY>yAL~iBN)8LI>)fKemv|%juJLK`yYMUVf8@K!cazVD|C)e=@LG`u(Y<0f
z#7)H;M3;-q5Q!FH7P&2aM);?&xJae&a-pq4g~IwG8lqidtP(|%HBwPh){?Kq3&d84
z$O~r+=?IAntr4mgxgmC1f<fw})E6lesiP7a;=Q8EA`gYi1t0S-;9J2vk7ok+9j<aN
zEv_}32RRzpbJ+H<va?=ec487@T*t7GA)Ud5L6<?B!GOV?!GJ-NL4dJ`$(i{RlO2-^
zW9)z5zp=kB{uKOq_s7N`+kOQ7JpR-3r}mF6-^#u;e%$|V`I}p>xL?kA{_WY9XM)dv
zKU?`s{#nP<T~8K0W_;ZGIPOWw6N@KRPi{W>_(b>V)2H{Jy?*}j#f_JZuin0LdENE8
z^0oGBrPpDv-@jh+R{X=AkAk12zLtHz`QyY-vEOBXF8;Or&(F}$u!->k(`P0{<_;D+
zwkUQPj>Q~2oNb)zImI}IIoR00vf8o!V!6-qge8Pkmo1b1IL8#u@0^Rcbhs~a*>J^i
zo#DF570*@AIg7)ZLxST!djrQ&PC@Q(+)g|TxV^a^aTKxNW_4v*%H+a$p5ZA&55wL6
zp8tFPP5Qg+Pw?->KRJJ9{Jir+=KJKYZC~6zU;XI%;lw-tccSlqzi0c%_}TVr+_&6s
zvft9bKK;DmW5oN2x65A_yefD#;q~D+Ki@8TfA_<N4`1KUcwhOx^Zk+cW$!P%yZA2T
z{kHc`AHqLg|1|%L$v2+wecx+;)c>^p_4U_*-xL0<|9b{>H#I{B!;$~%|1V~^&0xr|
zmLZTyoOwS}0aGE<N~ZHnGEB0JCX5<P=a~GN*;xu%rm^_5JZH*du>N=I&*t9`es%wP
z{mb&t{lCxuS^pRRZ}D&UAF1CqKkdKo`8w~5)@QSiOz-c!5qZ7frOS(u=N8XXpKCtv
zeYWN4v?qa&nICO@P<}t)UgzD1cWv(5JXrtW?ZcOk<Dcq0V|@PZ`KA{iURb<PdA|SI
zhG#m@_C0;`wDeiybA^|)U$MM#cz69n?I)wpZJ)C~^L>8wsr%Efk54|%{j~0L*O&WW
zM81Cb%KhE+`>t<?zxjM`_|Ep7=bP$Ri?0sfD!*U<apafO-|+t;j4v4-nI1A-V4lSy
z%9_IZjWv@!j`IVTGtU;@XM7L&F9~o8E*HoW3>PjJ2^3KlNfMbRq9W2Qv{2wS|5^Sh
z{`Y)Z{A>8L1Q-NA@fY$=;XcH*nR6jWDTfqi4_7abGM^NGlE4o^Y2gZCYvKDsR>B{J
zUkLjMGYAU^Ckv~IoEI?_<riZWe=8;-HdRztG+$)CFpuyup)es<Ar7G;p$4Jfg8V|W
zg=B@U2zUtu2t48c$=Ad?lRKDe9j70MFPj>xJnJsj<!m<WhuAYY{kZwLzi?Qw%dzcY
zF=T$o$j2DO7|x`_e3p3{OBkyoTQK`<b}n`X_6O|qIF546;Yj1S!NJGH&fUY^#Pf~U
zj9*Y-qQGK-lL8Kc<$^+jnF7oLRRWs?X9ztLN)z@KaTXO3n<XYJzE8qKx<-aW=7O}E
z%q!`?Qb#4<Nc2nkN}I?y$q2|g$%V*O${NU&NGV8el+cri7MB+95SN!YEukgJD5)fQ
zR#HG}j--g>ObJJc58{`^=8H56sS4T(tl>}Kzr`28$HFJUE5j|rxt%?aje+$(b2@Vm
zvn@*qYdYIXHfuJ0Ha|8t*7r;{OnaD$n6w$U{df5H{m;(d6MugGw)pF^ubE$WeKGiQ
z<MaK`??0dVl>4FboyS}DHw#}?zufoY%8Q1Vn_e2f-2Ec(#ro&_o;`eG@JQu>*L~l6
zd+#*gHn_d^_O?6o?|R&ucz@Hw+Q+U>RzHb-+WzeMbDozEU+#SM;q~>mx*r~WeEF&8
z%lB_`KTrIs`qTPPnsFcVJC-dhZ&=Q;vaq$WMzOwStzfU`tm4YxV&*pIQR6wtrOkPa
zeF+;I+hW!wtozxPa%6BF;GV_vlV=0(7rra}-U8u*TZHn2JB5{mO@+z@s|05YRtokB
zG6_`*O%%E#SR<g$|CKj`r;=*{ha9^QTRp2e>n9ddmSs%O8SMWb|8wf6;}6RpE53_<
zJNiZV%c{>7pEW<7`7rlG(#Oh==RTBuSo}fgWAMko4+igcy)k&>|Jv-;%a_fs_P@4$
z`{wPYw_Dz7y{mqA^_}H=#rNOerM=(z-unHzcR$|Aez@~-`RA~&p5NnsuKD%&*W6!z
zzbt=6|8n|u=jZ94yM8VHbMW8J|GWOzF$6LdFfU_nXIa1|!TF5KnR^yjG}mG-9<H?<
zGuido7qkE2xW{>%GlFXy_d4z~oRit7vaV(pXPV7m_W#Jg-2dzhOboUEH~$O%*Z)uS
z-?Tr|e)s<7{cZPi?za=4i#{HEAMozso1d>2y-s}n^%du<lP~^0KlZ%(`N?PBpT;~*
ze0t~U#iw3RO`cA8I_0Ut)3PV_Px_wjdVcZ6`WMe$On>?9CG)HAFN0sHzPbF)_JiMt
z{~zXj;{3Aqi@_Jk&&;0%zZ88r|9SoAqhBh&z5UMm<KMTr-;BSzeDD6o_U-3anQ!Xf
z=6+rGrSOaO*N$&~KPLWq@yF-?Tt<23!_4k1FIe(e|FSM+^JVwuxWjpe`y#Ite;>bs
z0F&S=fpz?{{P+1k3A74E3bqNb3vdc#@GJ85@&4z%!M6!?>jnRQ{&NEP0?B-Pc|LQ$
z<XXgekYf?YG0qd*4!o~<SMl}o7YIxe>=9BGRuQQdeJ-jmY9-<?yh`Y>khyTG@G9XH
z5f)Ky(Nqy9;UFPNAu*vjf>#Cn1SSaN2p$#`5wsP!z^}-Eoi~D~o?D8ikhhbUho_fo
z7H1*HD>ggUzs#O2TUoi-#8?$rYuHTLr?Gjlva-x)R%h{N`OdtUrI)RP<0$6?PCL#B
z&fQ$+xUX@4;$F^E$orZ1A+Hi|C(mu3S-kRmaeQTbi}>#GMe#l1Y3Ektw&l*`p2y9`
z^MWUzPfWmEuvf4_@Q>hcp#tGr;iDp<;_;FyQeIN2Qv0PerH@Lzl+=}4C4EmuSC&)u
zpv)5)57{P}IZ_iOw@UU&vP<epGDt~FYs#d_Y?t<vo-KV#dV_Siw41c7^h~K=5*}i)
z!j}a$^6BuZ^Hg)ka<_0F<a)qyj7^O70P|v|PR2-vg#T~<>HWX?pMfFv|DS(J|J(mR
z`sep=#lOq{pD}D=Q20On@3ub+f4}+Z`or^k^!FFvi@x9bD)_bStL|6J&rBc7-xs}G
z^>+T7(ASe+Wxm?_^7ZrNr`sN_c+h)4|K8p^LAN*E(!8a3bK~`(Yd5d%zgBj`^_IpR
zy?f3NR3Ftm)_wBm3G1`8=Qmzdzw&?6_AdKl$(N1azJ0&(<Mhu9zq)=~{Q3X)2g3yB
zD=f2Ex3Rrq-@|d8<1oh_&OO|+ysW&nJS9B4cqZ^Taa(czW@qJy;#k75lCz$BB5xYM
zpMZ{lhd{329-%VfKf?D!zKY%yJ16!)R7CWvh>$3+sDbEC(dlAI;+f*J#Ck+0h?ojz
z3(n&=<BR6i<jv$Q<Za|x$EC?>%W;9NkL47T1Jh&1HU@!z*1zxmkom6h_2wt-k4rx6
z`>^MO?uYRAPVX5$@O^mq?%vyVZ$G@veP{9B;lrknlRxu)o%VJ0*N0zQzis&b>&Ky=
zWxv9IFZzArSHsWrAJe`&esBH$<h%Hfq#r+jO#S)n=kA|@KOKKw`?>qqgx?K+Wd5l!
zv@kAb)L=ZsFqdH#!&(MO##W|PEc$HqY%AGR*@M~tvCU*_VB5{cz+S~B&C1NOhq;`&
zfawqe=l_d;eg3llz4XWWuggD)|Kk76{vG_Y>37@j)xUFpGyPuii|5ztAHm;4z6O8(
z@KNTY&xhFeW$z}xZGFT0TIA)cXFN|k9veQAc(~xfjR!>!B_FjuihuO{A>X4vkG4Pd
zdcyZi<>i^zLT_{4X1vXKJLPTi+kJ2Lz4`q%<-Od8*biGj^n6_T>G`L59|b;?y$^mb
z_ul)x_y?trjGvS~xqM{);PBq_{nPhTJ}`c4_!#!-;iom9YCcJPw)|54_13rAA8EhJ
z|G584{m;WNlR=zuI%5n|1altCKGsh5FwR7->0I-;ZgWL)%koI`hVm&2cncj6W)Kk&
zSuJ8NS}gir^q$x~@#zwKB*G;WB%&oGB&LcbiNp#W5ZJ=s$S*9QAQ&cOEG#E1BlKP%
zSzx0;ir`DZLLq-)Taoo5Z6XRHzlAeJEJWXl_K2B^-w;=kxFP;nY_@2>$R6QmLWx4l
zg@S~S2;UKI77h}cA>hrQz;~U8h1-U!g)5j#mUB1zc{WS7O{{FJ1uQFA%2`jdI<u}~
zNoUDpab>Y&4q<%z|HA)zhJHpn<^tAA_DdYnoM9YB?B#6B*bcJ4;c(&<;C#zb#!<*V
ziLILL58HS4FwQevmONd&0eqQ!PJBXq8+jdgWq4(InRq35FYun`i{LNjU(L@YU?q?(
zASh@dv{zV4v{3ZANSBDD=o!&;v2^kM5(gwDq|QrLO0r3LNWYg}CY>f-F6}J+Pbyn_
zj?5j|EV(bTJ7gTBrKP4xZj}_2l90M2=_jcoVIgKN@<>QuXse*CV4y$}e->XjF9%N+
z*9H!2b_q5f)_=^(%&(YSnGZ1wvk0=dGM{2>WBBkt{QrS}A^$%x>}M2Zv}1VkKlgv|
zzvq9>{jUC<@Vo5S=O3&;UVY#3ebKk_FB3ohdgt`k{LO~fuV1fvBl5QC&7)UaUTk<4
z_;lS9k0<(%^&cADKXbS3j>YZLTN7^;-`;=c+ugtS93HHFxZ+Xb<D4fmpRRuP;(5u-
z|F8PqSia-<@b%;V&mv#5zX|*>`}zE5%<p@DH5ukJmNETi+Q9sdMUhR8eHX`Tt^%Gb
zJbQUw@=W5@=Ud6kz_X0&KW75x4GtqtUoHpk4DJ_PkzCqb6<kf+alG>U>;ioPJc0~@
zVFI`Kc?8Y~Y!WmViWfR4lp=gwc(U+kA$B1<!NUSP0w?)4^A_^%<UP-;!Ml}vJ7+HY
zT^1E4|Nl4tO#2=7JL1>P?=@eGzC8cT{@Lr3_9xd*FFqdnkoEq-yD#th-bK9?dgJ?A
z?e&S*o^Si#+kRT~h4Gv9w=G|vebM^z>2vs(mtVeq3I5XmS@$#R=gXh2e46xG_Uo^2
zjX(N-#{H82z4`b1-^YLF{dWD`^IQJUl|NJd7XMrL{}F=)V>m<df4%?T{<Z(({Xc<W
z8sjp?UdCCB>lp7dzGVE%c!7zP#epT1xrM2OiGz6-b1<_ZlL%uBL&pEye>wkh{yQ^B
zF=+hf|M&XOk>5Xl?f=#EYwpjWAH3fueEs(M_$R?n*&kPYxbU9keZafaw;FFwzKVSL
z^7+$eG0&bn?Rt9n>5OOho~yqsf4S$ym*>^bpFZn&#{9hb#p72q-~M^m@Q&l%^>+u~
zKY4HR!RDjor>;)|pEy4TeenIj`GM^{|GR^4cfURLw(=eS2l-DgKOg&2@Wt`7#-}YG
zFMl-t)c2|R)0dC7A9sEz{*e1&!-wdPg`f6(X8CISE%$r<kD8w@zwZ4i`fcz>>2J-y
zV1@*yNERnHPL9_cr#Oysv~&7%{o|U>!^%HR;GDn_0dv92f=NP_!o4CEVm;!^#W#!n
z6txq(DYi|lU-YqXgU}km&jQW@F#@%MJA`C~_X$M_xd|;0+Af?ex=)Nz{IS?Qu@z#T
zVhLit;x{FlrCv&ZlbI=dL-vwvhHSXZ4=Glubjha@izK=vQY02j{E!fq_#i4Fa#zSj
zu!(;$-&Ve8{y+TT{NB6=xfgM-;Wp)d&h>z6JC`?C6X!w>TaFM8DGoOFa#kglTIN~I
zmssT3b2)EuHE~yS_i{OMF5-B|(aEX9)y9?2CC|mo<-m2AYa;g~o|U{C_zL*X@N)`q
z3(Vwy$(PF)%-6zqi0>|+20yF7Pk}7K1i@PZpZHt(IRs`3JQ8>yI7PTuBvGVMWTU8x
zc(lY#$&J!eWQFC|$k)j?$tx(#P_R)@mA@d%C#xb`C;LVAv7DB|VFfk?Pq|zfQR#Ro
zLn#-jN0M_S9*L=lrU-8m3>0YQx8yhB*XMu5=ga5K`+>Wi%ZF2gLyX;v?F#F8RxUOb
zHeI$<HYv7+EGo=#Os$N~4A~5ZjK3MZnT{|nVR-ug*#Ch4G5@Ci)%kn+596QyU$1{$
z_|E?Q#MeEai#|?!AM`H$?e{laZ%@35d9C^K)w7BxVvmkJnDD^$LCF1Acct!l-V(k!
z@5b&M;WveEZM~I$TkMYSouoS(?l|9dy|?VX-NVqwOwUffsDGXC&iP~4=Z{~izIJ_m
z^Y!t!OW#-jDE#I2x9Gn*!&HWaj4PSqnN3+bS!39xIDd0ma~<QF%Ke6WBKJD3J)HHN
znp`T}vOF=otN6V5-}2qyz07l%=Pqvmzn#D$frA341TG3}5NH>8AaGIevCsiwcacJo
zlOpb-+M+2U<-!F*$$~2dPV?vRsqi}Sv~#cIy2f#c^&XQHW9)yozbb!1|K$Ej`t9}e
z^*7tE9$$h#dwn|bvFhWU58vMJc<1qU+Z&Cyo^Kbw@p^Oi4ez^0@4tLJ`uWQj*{`Wz
z>OW8ZwCdx$k8Pi{KJ$G(^r_*K?I({<`#+U@&ik_V>-TT=KPLUW`s=~(=098h)c+Cq
zbMW`2-`aoX{&D{6_Rr&g)&C>^F8=-Y$LY`0-`9Wh{gL^*`(OJ1ZU1!s?fQ4~|1*Yd
zj3G?lnTnZ1m=`l?GRZUjWGrMn$Dqrw;{TTamJAyh&N7HI)cmXa)A=j!$E|N|-)4N{
z{`UAw?C0GdJ3mZ)U;pmt+X-(&-oAR1^=8)VO|O2v6nOdg`HN@X&wf2sdv@U2tmkW9
zTzk3amD1}2uR340zu5F5>*bG^Vz0&Du7CI9o!`5gZw24IdbjI+^@rw<KA$8%1$@l=
zko^AOyWQ{7-{rqO`R2!)OK;QPJAdr@^y|~XPscvSeb9Qp@Ez~_zwh-wn7+4qcjs;3
z+rT$7Ur&6^@#e&vmbWhN>fUR9T=ePO=f7Y4zis>O_Vd%P2Y=Z9TQW8?XR_+BnXz@T
zIkG=x-^lTXb1ruquMGcQ{>A+N__GDWginiX5_K1wDmGh;OYEaal*lm=ZP5!NIwDQN
zrouOcy+sd;9TR^qo+HjAzFdq?EK*ED;=E*q)OM*yQs<<$NG+6_CiPM3g7j<|dD#`R
z`f|>4S7mi%J!LGV3#GiJDx{`MotLteZj$zwo-E}lnIb-2bgf8$$ZlbO;Yi^}LbZZV
z`1kM$@}1|c=RLv0#goik$koQF$|=Wbz-h%fg+rd5mF+rf5ZgTVI?jh&levX?6nGwR
zi}SqUk>zXWPY_@di01d@kK{MyPvmD2cqH&haGuZxAseB!g3N-90$cdz^1kFL;mPKC
z!xO_hk@qie3ts}?D&7>{|GYQ(X7cY6$Pn@rVG>giZx?rykdoq&DUw|vmm+^bK2PDY
zLXm>KLZiZ0`E<FPGMqC0GQG0w@*)a`3b*7I%9Kg5NjQmC3+oGi5_&1vB;d(k#K*yR
zk+*|4oY$ZC70*5H%UsS}D>;{N6tJtZ?PXcb%+K7&bdu4N(Vww~QH_zCVb(u^zej&7
z{r>yw#xK#|&wkte>G-|zSMIN6zc_x2{9f`)>sRj2XWuoyt^D%)Q}9Q=4}R}%y_SD9
z{bkY1rkB1iA3v{sdi&AY`-ks{-@bb5_$}*OIyW<}r(NZ`l6d+4rP-IhU%Ges;}y$m
z&DWRQczrYN_Ub!_?@HZY_n`FA%qMEk@4U2n!}o60`)eQmeK`3c>|^^UsV@fKoPWIe
zx#4%@->82Z{=Z@5Vp+$^#y*pM9lI6BH4Zk;Wt_{nUU0>7?cuE9ROZ^m?Z#`wSHSm%
z_bSg#ZWnGb?w#BUyvBS^d_Q?V@s{$<;%66R61poCA#z;QP|Q-SQfz`)tyq<qfEbJD
zZQ*x9#zK{XIRaArn|Q0aRX7&09A=#RKkna^zaD>g|Iz+)`B(Rk-(MGgR{gZ;<F5}b
z@7KQTd$;Fp#+$9LXS^<cE&Dq1RsPGSm#nYt-xj?W`grxD=qIyJZ$B1&T=a3zrz@We
zz8wE@=L_%GPhWMud;Dnpx#w5KZ<9ZIe|P;g{HOJQ!v8h@7XH2YXaDb|ztn%p{i^&W
z@w@N${@<Cu(|*nQ$@uI1ujb#g|A_o^|8KysjA1K7GNV1S0c!%A0DBy}6Z>hlqpa2}
zFPZF^G8pgt_x{)Pch%p-fB*hn_;>2B+MiE9r+#1a)%Q#BXQ9tBpDw*Gc`Nxw;Pu0o
z%rDzsl)m78x%K6fm&&hpy!!cS{VRr7HZK>xkbANJx#0`3mq9Q6Uv7Kp@v`~3=Ceyr
z8=uxZ&3`uUdC!YwFZ*9@eRb*8($~-5NWQ!O?)|%y@1)*4yr2J0>RtA`HSfN?t$8E)
zdfqFG*Gz9b-!Q!S@~ZHq_Y1z~@=s-+h(8s0Hv8%4CnukrdD8xr^SQ+fv6nHg&b)s8
z=EqyH_og3WKX!dO^jY)kn{RPH<NmPyzs%slG><u%WeH0@%U@<QmbI+O>~<XQIpnz*
zx&LrY=W5_m;@-@Y&gaKJgZ~$Qy1*=f4gn1T27v~F#{wP#Q~9|C8U$5@{Y1J&ctxy*
z*@d19%oO-0a6s_8&^uvAk@q6LV#(qf62X!cQeo1QWJKix<!cq5EB;oBRW?&zro^Rm
zS@DtLN5zYZ35r`3Bo%Dr7t5N<Jd%DUy+Qhdw6M%JX?f|nQU#KR66O+<Bo2r>ik%jb
z5$+L86ENVf;Irm$7WgOdT0l-vSx8f)O6-exiA14<zQkqm9PvY9^<wtoj1tk}_eJ-J
zu!uMcPZW|7`Yl)^6feA6xLDXhm{a(W;BtNozHPkWyg|G&d^`Di`F_s+bMvqC-y1*o
zeJOk|_1f;mRX#a^Y=K>T+PsT-`S_3XKjm*0xGiWXEFkh+q+fKdsFj$8gqLKv#9r}}
z5;~GT60#Bn5|_j;iM<om6R{HF6S5bU5{VQ^7r853D3l^#&Uc6>i6@7Lk>?$kIOhX4
z1y)OzYs|lyXRxrdva|kRImHse@`E{_Ii2YkqchV<rlU+JnKm-*WOQaQ{@?#^<6nnA
zw|`atdiCqsFPC3jztVq||2p>b><@(>m%o2~_UPH_=XEcaymoxk_v*>>+NbP~H$RYk
ru<zm0$L}5|KjwIB@aWwGyNA;si#~htJn~h~TeT01zqh>m@O>EoE$4hU

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/fa79819c5de04bc06c69bec3fa7f2e982826ea2f b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/fa79819c5de04bc06c69bec3fa7f2e982826ea2f
new file mode 100644
index 0000000000000000000000000000000000000000..2fd41a34fe5b503756e93b68aaa4f90301a78950
GIT binary patch
literal 20280
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bGAi$84Sdti%#>nvK>-x{8
z->m)?{4e^=_jBtX4Q40y*Bqvt(d;>Fh3rynn#|rzj4Xj16S(Ja1#z9{>E>(XbK-5~
zeJqe5n8YW>SId*m7S1BeQN(qY^B4PV)?2J=*dMcBWOHJ#=Ul|IgD-%0E;j?;OMx(f
zL_T$1CEhsRi9DZqr1;<P{}p&9JYPghC{I96@Poj8?jlx0=8a5m7)_b$Snsh_us>mU
z=Pck-=la2!$9a{*hifBu0nZG+dwe!LE!^e2mHZ3&E%^HQI0d-*_w!EVtKdJuXU5yZ
z>(3w0_mbxvUx~mXfvZBs;wm!gay60~LPdOAd98WVc*Xh41zzzB@(1$=^RMS?=Tqg6
z5NH#;DDa*)h9`~Jmv<I-B6ljcI@c$TRqU4RD>>G2nsR;O-pRX;FNn{9r;y8s`wIV8
zkuRc|qKV>8Qu}1f6<*05mH8?YCv#V7y~G1iF3}JXHsSBWTq3?g0m92f|B0`c{w3oo
zB`9`BaI1iVkhfqf{}=w{0;~ee{FVGK1$9N@L}kP~rF>-hr8C4Hi{uC=2viF+2+rcq
z<aOlk=UC4%nP-pSMUij9ZbCPNCJA>7Jrv01<KSMzxsvM(rx<4fw*t31yEl6U*JQ4T
z9FN)NGb{eL|NZ`#<DYB48U6(QZu_?XbLuzNpJ6|Bf0TSP{2Ko`=ab17)~|~`oqAXF
zI_UL<H`iZlypn#|`y%r7?suEsK6$0{V#^b)C%c~MKeu~!;yL@vpy$h;ZG5r*Rppze
zx3gb6zC7~c?(=I;8=p*l^7U!$i;`EpFZ!OSK7IAv@7e074_~&wdGXrz^_5pgULANP
z{AR}c#19+ZcD${9f9=D?&%3@G{cQNT{kPD+d4Ifr9s0fM@0CAie#icK{O=g!9HvI*
z#jG>gy4Wi@e{&jeY-gRu`kYOM#p?gEf0O<n`X~0^o}r0hFJll(3hN=3D=f||mlze9
z%2-O6c>hoTXZ%m=-{Sur|5yB5^<RdmjwP3+k?9jd+`pdRRllD6IQ%{4XUwmaKj-`?
z`^NFr_*=oxReu})?fdok``NF@KI*(Te`oWi_qE}h?XQ+T|MbM=@!`iB&yTz|dAIcK
zmDguqv_B1c`t^D6tNvGep8tF7|M>9JV=r#Ks(xel{>R6EACo@_e3<@m%jfA|0>0RM
z4f(O=cjKR$-;IBI|4sR~|9>viea3J9UH@nNV_>XcTg3K|`5KEETP|A$dm_6edlCCm
zwlFqtwtSXI7EQJdtO~4aS$$YGGu>yJ$GnoYn)5x+Ufv_z+c`Bj*0VM<b1*jivHaQd
zGyZqZpXA?Dex?1n_ow@(-#3o02fkE(k@|e?qt2(APd7ijeY@#R(c9WL9Iy7i^m!fg
zM)9@$3+@;CuYF#(JU4%G>{0V0=|{W|MIUlKs(s}CSo7(L=S?qNUtD`C@T}^2$up%V
zHy&?$TJgN#`IBddU+#G?{MqdDw=dhi>->oM?)h!;mx#}%pXYwL^R4Oo^zVkh;{LTU
zEn{ov@M3RalV(59x{_%-<8|hPtTn90ETt^Q?4n#++&j4?dA0eu1YZiU^Bv&&%jv+w
z!GD2Ynct9a3NI`FRY4V@EdG98Ile!<#@z2Yd^nZ38oADKP2;-G>Bv#ZmdobA9?B8U
zy_GkOcLDEB-Yo7J9B0^sS)*BYvv9KNu`;s0WSP&}!`{jz&9j&5D~BA17yEYhnVjxi
z8#q<DYPo-Li}7Ug@bEn5zRlCc6Udd%q0FAk#>u{k-Hk(t%Y|n??^j-4z7BpFK{J6|
z{?md%B432rg!%>41Of%P1wQlD@|tq@vfHsTaOm@06WS`g`W54wZ*N83yMOrfvEx(o
zr-Pr8K5h7P?n~`A-tTHZTz^*mJpF_Dhr<u+pI3i9`FobZl&Ozdj8&Yik<ElvmH9rC
z6LSkw4&!!)84UT1iA;P<M;JU9Di|96hyFYLFOfl<c>(K5mMo?ZjOL7Q{uTc1`D6b3
z_OFG%)&AW5qxbL9KaPKff2x0O|90UE%jahw{=VDzM*h{L7h9i8Jhy$m?77m5rWdQ8
zTRdZZ`s>l$2U7R#?=HCg@wV^X=zBNsJ--)uKm396!+Q^pKH__#^Q`oR>`S>9yPx~L
z41MML^5YAkSJPkFz7Bf(<h|x+?eBYkvi@TJ8TWn7SGO;^pHn~G`pEgo{&Vn`gs&67
zd;hHd$@+`)ciiv2e@^`uX8OwXkXeIOkL?y)4Z9mh8mBV13-3NYDSk!%)BL{$P78h!
zyeoKA&_hUGm`Aus=%Y}D$TiUbF-NgCVspgziDii%7P%$dCG=dtmam)p6=w{GEc;Y;
zHcmaRC~g~`9o)fOLYzO?UD@;4zq1!{2ypgse&yuiuHrG^)#m-jGo6Q@hmU6o4<GLv
zo@yR<p0zx_ygYpM{PlwJ!h$0DA{oLHgboNw39RPT;&$NN!!e6<64zy}3~qbgJA6m@
z1o-ChuI5?G&A{Ep$;BbV(aQ0bU66e)TQ1ucwhVR$4hPO$E*b7ht~L&HHffd%j79&C
z|GoIf{EzORrGF&<{{8#^|5m1hEFP?XSQ1$y*iNz?WIMoC#KzAyjqN_W7pF1zG2U1F
zjDj74Cj{RKh6rsEx+cUYTr9jy<i1$1#CM4#NoJ`llH8Jp67k~3;(g+N;*-Q0#h!{4
zifM?x5Ec+l5_%@6BeYT2L+rE!x72+}3CTE#&*HDd4~jn*V;7SZJ1LeR{#x8iqFiFL
z#2kqR@qE!1;n_l5LiYul`Mdd!^9Jy);mPHm%QchhCZ{aNcDAjoQ&@VLdzdmAw==Xc
zXfm)cq%e3heEr|gaDyT1zstXK|Lp!R__yQlfj@G8oc`qeS^Qh(*YY2dKMH?b{lW1w
z?`QkZ>K`88tUe!lum85~^`w^@o-cbA@N~js{>SqkGd_ub^81P1)1^;Z9<O=S`smW5
zkB@>LH9VaDQ2CMTqwkM)KbiSV<wecQ_E(;-x4&NZ+Uj-ktM-=*UNXLZ^k(wA!Vhmg
z{rWQd>)x-9-}Zc)@?GU;;IAvc7XDuShyCx?Kkk2I{#g9!{3H5T^xygaw;3IoPcqB0
zEMf6w-N@?6s>Sk_DS_z+(-&qBmhUWa?B_Twxym@NbA)lYa2(+%=TK%>WShWxoW+1e
zfaN2zEXxd*$t)HudMy8#o0u(_Z!@tll`$S>EMz*#RLJy@@hD>nqcFq1e+U1W|9|+8
z`JdI_m_ItdyM9)EzxFlmYr@y4uN>b3zL)&y`YHeG;SaZOk3aAF=<`0{ZQ&c+Hw|wV
zzn%O3{KxyBYrkrL+x)fjOW)_&pYy-4eSQCB*_T;g7JM=Ja`W@9FZtg#en0Xf{MW|c
z=6`DbJp3c|_wk>!KNf$!{E7Xy^8XKphfIf9j<c$>Rk597`@*)4Es~XiS&Y$}q2j;C
zKmLDv{yq6O<KOy!F8}=)IvClQx*6{<I5Fh@ul={~@7+JYe`o#n|K0Zc?(cWMul(-*
zo%AdA$BA$1-`0Gc@Wt!Xm3Q2)XFOl~WX_{K52rlLeQ5lU=V8Nxfcx_IobOuPxpC{k
z&389>ZeO~y=pMrZ{)dT=dLPetEb(~8<2g^xJn4Vp^mOktffqI}{azh^_4!r#>;JF&
z-q^p}`a$)R?WfBhB|Zv$aD3nTF7<80n;maXzV&%u`cdeM;kVrH7C(;vc<|%!59^=q
zzx00J|Gni;$zQ&IZvT%kPGPBFTf}~vqnWdYGo3Sqlb`b@#~Y3#9OpPbbB1v5=hooq
z;g#Wk!@pKwt>7}D0^$9_YQlL!Ekf6Y<b-Sm3i)R9n)7buxy6&tXCkm&fLlO{{|N62
z?gq{X4p)v=j#7^69Lu;gdH3>7<-aBHPLN4RS;$*>mWZOLvuKKFtf-XeMUf98eWIRX
zXT?0k4~ZK|^hvCg=#`iyVJZ<X_D6W5P^6Hgkc3cz&<>#};nTvmg-iuo1iS?V`IqqO
z@_y$L=Gn((!FiEghW#bmJ~mbMY>thb23(ss6FK&?sj@C&PG!nu>}2@)Uz}kIgE-?~
z#&1mC%+r_`vZS-BurjeuV!g#WiPe!+fwhnIBwHni1=n?MQQk|uk$l(qH27WkmH0pM
zUF5sT=fi(ZKtgz}NQ3BJu^Zy1;tit9MP`UZi!h7a7Cs~VQ&?Q2Qh2%0R-r;+eGv`O
zE-_Y#BFP%5C@E{n*Wv|YD@5dlvxRho#D&%fRg2sZJ1xN=byDh!l!??)2@UaHQDu>b
zLgj*w`4{l5;GM@af%^_uIhPjK8qR|p4eU8=dsx|7uQ59@2{Eo?SjdphV8Wowpv_>w
z;Lc#cpvfS>*u&(^{E5kqNrf@?zwh7J-xq%h{=EBR<Bx4Wf_@(V>G@Op$ChtpUm8E|
zf4BV2tykPHXFUJ*?8`I3=f9t=d?x>_<LRy^iykvRZhaj0q~wXkld2~-pL~3x`}FD4
z`_EoK|M=p@%f?r4U%9;QdR_Ti`?b>Ru-ETjFL^8e;m$|F&r)B@zTf<D;-}c}vOgF9
zTK?x}=x5l(c!B9NlOl5miyd1Oy9~!-4j#@n&h?yPoWdMz>|a^!SbwqHXL-UB!m7)b
z$$p$;3g>suMO-@E7rAV>;<(OmUFC}Bs^^@=Va*}I@t?hc<0z*f_cv}Qo(0_AT#q=4
z*l)ACvMgnCVLZ?9l%a>=?tjn!J^v>CUG^vV_u`+NKQn&b`62Ut^4GR6ZlAAybp3GR
zo&P)0_rKq>ePsM>`!()c?l;+Q>0h6I-taNveZ<@4uM1ukyqfU(@SC4+7rnpxVZ(>7
z?`OQPeBb&0$osPQ7v5cb7xI4Fd#4ZKAFqF!|Hb4R&-cFXwLj{ATL1d`>%i{`f7bs!
z^KbcoUxo~ZBmdX`U(9fu!H{7sLm-nl^M0lRrb4EbOy`+om}D7E7&VyAG5IsIvlOyS
zWASHs&XmVs{qNMD&A%V~>i+fmm*t=Pf1m%e{xAOD;@|E+Qon6}+JE2kb>0`P&t@N)
z-rsp6@_NBbmlq+=EuN=7*L>dlY|GPWPXZq^Kic@9{C>c_&btrq+T6E!u>Qf@hc6$;
zKh=50`25}TO)oyYuy~>JeE+iz&vc&cd-~>S>9fY?3NL5BVtM26?)rz?Pez~HK4*RA
z`~2ur_orVUpM0GAY2D|pFZaKQeEsm1`@8A)UEdCW^ZDNJo$Wi%H`T8eUmd<xe!u?X
z$S<kC;r~S#Uotu}J!HDTJc~t?HHGyXYbJXf=Larlo-Mr3_#X0K65teEE|4V{E?h1W
zD55NqBr;7zMWkD3p}=kav;0x~@A<U&*YIZvFbIC)FXWxVeTZu_=R%HB4k^wau3jEx
zJ}Lerfggg>!WF{S!uN%&gg*+u5cUyf5Ec+l7FH8EFJdanFUBhVR!l-{s;I7LzQ}xG
z9^qv|VM44z9707x4MM*K`GsZ+$qHQ&@Dd0Rc*6gauZedicQDsFPCpJ`HZ@jx)?KX2
z*=*Pkv1fAnar1M3;jm(tW81@G$o!Czk1>cboJoiIEb}y$FjhylVD{PUT<i?&57_5%
z9Oan9k;ZX@gO7`yyNA1p=NqpXzo5WGfyDwR1snv+1%(7N1(*e@1U3oI5PBw*ChRNX
zEGi;4OH5dNpM-~WjSPp(1!*;zSJHo_j!M3f=$G`BHj#0X5s-C~3z4goHIONhQjpvz
zp(hb7E-l_6E-!IfLQ9fSQc3cxq=3{MNfF7J5{?od#4m}>7ikny6|@yt!=J!^i!XqW
zg-?Q4hFgYnJ9{1*1M7X}bmksrTb2;kbhedj)@=H0er#;4@0o0v_AnJOX)|v7@9^*Y
zpPj!a{`~xH@z-TvGr#WoV({h0=lh@Ee?Ijo_e147kGJe^7QU)}x$ni57Y#2ry)=Hg
z`$gc3_0RV`d-%lQk;((F`@Z+~-f6yVaC`0TZFlD1^|&|j{-%euk6oXveiHk%{n_*9
zJTD)<-1+Lm>+5fIKRo*Q@>9>3@89Hpp7>Sur}dvS<38qhEL&LKu$*IMVQXWJVtvb6
z!Cuc<#g)Ou%x%u2#&eKMoAVg^5;iur#jHzM_p>eK$lyA_J&Wfj&j#Kvd{_9r1;Pcl
z2;~WP3M&bl3Y8013C<R*6zmgZ5~>uMD0D}#MnIkaD{lr*CD#HDId&nodRBARPb{V^
z%b1=s*#AHN=hRQfAC^B>d>8$8^o#J9Ri7<BYkoTOVeW^dkCh+KeJJ~|_=C{L;E#bH
z4BqW}WAMiRwb`qeFPmTOe{K8r&D%|Hx4hMQSN-nlJInWq@4vrGd%yF&_4{@2e!P?Y
zaOdOl&tYFZzsLPt^Xu`ixxf5=S^kRt<@D>$&(lA5{aXCz;J=;!cm1zp2xKZ?UdG(c
zvVcv3^BI>j_bjexuEktDTx&UIvg@-iX8*%+kMlTZ1lKn1b=+q-C$mpwUCk`cG@HTf
z|B-*W|JfOs7;67-{ulhO|DWi;X@92u?)}aC+wSMwZznz%eLVI);N8PFKVL6;o%s6e
zE6!IZU;KT3?0NO`lh3|Cjd_~*^v=_ZPraU+Je}}#%2S1>Wl!v%^gZ46{NjuCFP^=a
z{_@*P=2zce2ES5$bNQX^2fq*hKg{{W`DN`FgD;YwnLi7DDf)8$^ZL(6zf^vE`<?a2
zzi)HD8Gm>A-u;d3+t05u-_*a&{krZ;;TP+#9pC(ZO#JoYkI(<PjPlHfncZ1lu;j7+
zWnIeV%kIr_hw~2iMP4cXK7IuOCc#$%>-c5)@AH2WXcdeUY!hG?;1tN<SLEyC{m*-Y
zZxjE0{&fER{O1Jn1(Nyp@_gog$+d{{Ajcw(W1J_r9e7{!uHx(EFA$g{*dwGWtRhk^
z`dn0B)Jnu(c$Ls$A#>qW;Z?#ZA}pfZqNyTI!a+ikLSjO51g{GC2}}^k5j-p?B4{gc
zfnSmTI&TC|J+~B3A#W!y4^J=GEY3oXS8R5yf0;d5wz6`uiLolM*07nfPh<09Wo4Pq
ztj^-k@|}4xOD|go$5GA)oOYZMoV&TsabM&9#J!xSkoPn1LtZ7`PM+I5vv}qC;`qw=
z7V+KVi{g92)6T8PZOfg>J&&7>=LJtbpO}ETV6R|<;2*)?LIuLL!be3y#p5Maq`ah3
zrS?l{N*|SaDXA;9O8TCRt}LhQL768q9<og`bEGCnZk6nlWS7*HWRQ}U)|5$;*)HuT
zJzM&g^akm2X*X$E>6uc$Bs|1og)a+i<kR6*=c(q7<!<3V$n}8Z7@HXD0p`U_os5wT
z3IE^z)BAt(KLbPT|3Ckd{<r^s^w00#ihq~?KV#U$pzweC-)(;u{(ke*^@r#8=<hGS
z7k$6=Rq$)uSKY6cpP4?Ezb|^X>h1hDp|2;u%6zr;<?H9kPq#f<@u2s9{=L0-f^KiR
zrFl#7=Emzm*KS_jf356>>n)8tdiR_ks6MKBto!8A6V_*G&u_e_e&zqB?OpcAk}n&-
zefxgn$LXIJes%q}`1Ako4~7ZMS6F7TZex4JzK7#F$6<~=oO`%sd0BaDc}jS8@l4=x
z;<n=a&CbdZ#j%8AC1*YNMBX%hKLH&94}o04Jwj!|e}wOed=<SZc24YpsEFuS5g}1t
zQ3KJPqSM8a#52WbiS>w15HS_b7M#a##uv@2$(zYr$lJ)Xj!To%mg53jAIm8w2d2l2
zZ43hctbgDAA@g11>&;KvAD4XC_hHWm-4EgKo!&Ei;QR3I-MzQ#-hOzS`_AIM!-q{D
zCx7PqI_>M~uMfYre%tW<*N;O#%YKFbUiACKuZExLKc;<m{NDQg$#?M|Nk4x6nELbC
z&)q)*e>(oW_H*~I3BMcu$ox}dXklE=sKI!MVJ^cghP4cmjIB(oSoGQI*;cZtvIn#O
zW1Gp=z_y!>fxU`Nnw6Pl4|6$l0n;G{&i@zx`ut`8d+Cq$UzdLp|Hc2C{X6())9<$5
ztAFSIX8OJ27tgQPKZ3u9d=38m;iJq)pAWI`%ic|X+xmw0waCj?&v>48JT`nJ@o>R|
z8xM*eN<M0R6#wY?L%v6U9&LZ@^@Q)4%F8pah2G}8&3K#fcFNo2xBK4gd-MBk%6qvF
zu^+a6==r$v)ALXBJ_>v&dmsE>?!EVW@efKL89ym~a{0*o!Qs8<`={@xd|>?8@G<Pu
z!%u5I)qIlpZ26`7>#c9KKhl1c|8f79`k#kkCWAQRbjBE_2<AMNeXO1AVVsFv)4Aqx
z-R6qqmgSM=4dqi5@D@5C%pf8lvRcGkv{>}L=smG};?pJeNQ6r$NJL9WNK6$=5{VT$
zAh3nMkzZIqK`>0nSXfS2M(DjjvcN`x6v3B*g+l(qwj%3A+C&sYehX)cSctw8?GZB-
zzag$7aYOvE*lf{$kv+oCgc6083k3-u5xyhbEF2^>L%^Fqf$usG3%3nd3s*3gEaz_a
z^K6!En^@Uc3s_dLl(U{@b!J_~lFpLH;>u#l9K!ha|Aqhc4E>CD%mu8K?3XyCIm0-N
z*vr|Lu^nW8!{Nj!!1<P=jH8f!5?eLfAGYu8VVq~UEP1+k1NbueocM(JHu5^~%J9na
zGVx0AUf?~=7r|f5znY&*z)B!nKv2*^Xs@u8XrbtJkuDKQ(KDjyV(H@hB@ReRNS&9g
zlw_0gkbW<{Ogc@vT-sUspH#N=9GN?^S#n=wcgQ$MOG{0Y+$t$3B_VZ5(oa%F!a~ek
z<dKlR&{jcN!9al|{w%(5UJjltt_>X4>=JA|tpAvmnO`xvG9O|VW)Wm@Wj@8&#_-{P
z`2Pd{LjHeX*v}}+Xvgs6fA0U_f6xD%`(6Dz;dj}u&p%jyy!yW5`=W2<UnYM1_0H+7
z`I`-|U%y`UM&xbNn@6v<yx8z8@aeiI9#8Zi>pwKQf97u69gEwgw<g{yzP<m>x4VDu
zIXqbVaK)p<$2m`CK3)Cn#q*Mv|6ldJv3$qz;p@l!pGCfAe-rp&_VfAAnBVvQYBJ1c
zEMxl3w1N2@iz1sG`!0^xTm?K=c=qzV<e9{)&$p77foB=lf6fHX8yrTQzFZF68Qd?p
zBDu7=D!7`s<9Oxy*#-Ipcmx>)!vt>e^9Y;~*d%B!6fbm8C`I_V@MPi7LhM3zf`<io
z1Wxj8<}Kvi$$OqxgLf<UcFtV(yDTb9{{L_Onf5#Ecf_xo-)p`WeR=+w{j=96?N6?s
zUVJ?CA?y8vcVFK1y^DG)^v3tK+UpaqJ>T}fxBax}3*$HIZ(F`T`=a&b)93InFTZ^K
z68xqAv+ifs&zC=4`84UX?AKr48h`ZvjQb`3d-L!2zmNaU`|bL>=ePWyD}Sc^E&jLg
z|04zq#&Cw@|9bzw{cHcn`+ow%G{$9&y^OOM*D>B_e98Ei@d6Voivvq2a|=@m69@Aw
z=3r(+CK1LMhK&EY|8oB2{C8%MV$k@{|L^smBfo$A+W)KR*W8~$KX|`S`1<Yh@lS%E
zvOli)aN#}6`+#?;Z#CYWd=>ff<@2Y{VxB#D+V%AC(;3h1Jy(BO{&LTYFVCx=KYiBm
zjQM%<i^s2KzWwvA;T^}j>+cS}fAZesgUv_FPhFn`K5>2w`r!M4^8?#^{&xr8?tXjf
zZRI=u5AvU0em?f4;EUsDjZa%XUjAtOsqa(sr!OCEKkocc{2}+lh7Zvn3qS4o%<|Rt
zTkiMzA2mN+e%<?3^xNQ%(%+hY!3+sZkt|MZoE)z?PH`ONXy^3j`o}e$hn0Vtz&U{<
z0_K931(Sp<g?mLT#CpV+i*FYDDQYKnQ*4`9zvyG(2B9^Ap9P!+VgzahcL>P}?-Pm=
zauZr0v|Tt`be|Zb_+znqVk^Wv#S+AP#cxV9OTCo-CNop^hU_KT4B2p*A5yGR>5@++
z7D;qTq)055_#q)I@j+BV<gSp5U=#mhzO8)G{D1hv`Mr4$axdau!)?m_oa+JCb}nzO
zCeDQ%wj3cGQXFjT<*Z69wal}aFR{q6=W^cSYT~Zu?&WgiT*UE^qmxsItBot4OP-6F
z%Yo}K*F^3~JS%xO@D=c%;pY_K7MRKZk}sDpn6HKJ5Z_%s4SrUEp8{Ee34*r-KJmBm
za|p~7cqH&ZaEfrRNTNui$VO2W@o0&gk{hL`$O_A^k*||)l2=fep<tt+Dt|$iPgX^?
zPWFrJV>vB_!wPH)o^rV|qSEnFhEgt4k0j?vJQ7n8O%dKC7%0%pZ^>`Muh0LA&zH}e
z_XBr3mk*}~hZwsT+ZERHtXym=Y`SczY*K6sSyY(gm|7W|8L}A+8Gkc+GaX@E!tnI}
zvHt=8WByJ3tMm8tAI3lZzh3{i@SXkpiLZM;7k!-eKImQg+wX6<-kx|9^IG%et7jEY
z#2y`cFyVpggOK~L?n>S9yd`{d-i_Ti!fy)S+IlPhw%8rtJ4tso+;P6^dT-f%yN98V
znVy||QU5yOo%6@8&mX^3eeL@C=Ii5cm%gw5QTWU4Z_$5shN%n-8CNpJGn=w>vc|AW
zasKAC<~qhTmHQ3%MDBH5dpPSkHMvx{WqD$FSMhoAzva8Zdzt4j&t2XCemj9h0tW?7
z30xG|AkZ%GK;WX_W1$1W?jnUECq>*vwMA1z%7qJrk_A@^oaWErQ{i>uY3E+ab&cZ?
z>pdnZ#@PREe^vg3{>lB5^xNy_>u<JSJ-!5g_WE?<W7WqyAHKcc@y_Gzwl^AYJ>M>V
z<Mrn38{T)1-hcUc^z)Z5vR_lb)PJ7*Y1PMhAKN}@edhao=u^Wd+fN>!_J1n*ocCqz
z*YDr#e@yy$_1A;n&40H1ssAJJ=iu*4zqS9&{p0-C?Vrd0s{cp+UHtp)kJF#0zpwx1
z`y=yr_rLW2+y3eN+x73}|7Q%_7(<x8GZiz3FfV4(WRhq4$ymsEjzO1U#s4k;Eg3d2
zoMjMasQFj-r}J0dk6Yi`zRmc?{q6CW*w4E^c7B-pzW&|Ow-ermynXd1>&>j!n_m5T
zDe&^~^B2#&pZ$8O_Uyp3S<lzJxb||-E2Y;5UUj}~f3fLB*2^C+#a@fQUH|UIJHK}~
z-wM8a^={Yu>JQByeLhKk3iz1!A^H8mce~%EzsrAn^39Jom)@qocmCM*>DQ-&pN@Tu
z`=IrH;XB^<f8XnWFnw?J?#|o5w}EeFzMlA+<IRaTEpJ`k)xFpJxaiZl&ws!8f7|xm
z?dPXo5B{+Iw`6Q)&SceLGh^#wb7X(YzLDb%=UnbKUK#$q{EPYj@n;K$37;0(B<e0U
zRcy8xm)J*<D3N0#+M*XkbVQnjO@(g?dy5_xJ0|{KJV%^Ke7P8(SfrSQ#CgdIsqIpa
zq|Qlgky<D<P3oi61?kx`^0F&r_2r!9uFC4jddgTz7fN|cRY*;jIxl4_-6ZWVJz2_A
zGDUp4=vt8gk=?@n!jZy{glYw!@bBRh<U7w>&wGN0izk`8kgJVTl~az>fYXX|3Wq#9
zE8BI}Ahvnzb({~mCUXn(DDXVs7Uy}xBg@y$pCG^_5Y6w)AIWdZpUBT5@JQg1;5?xX
zLN-Ec1(^jI1-9_b<$cLh!jsMOh9`!1BJW?`7QO_&RlF&@|9Nln&E(%DkRjwL!X&03
z-Y)JYAtl8lQzW}UE=B%=e4fH(g(3xeg+_(1^67FnWjJO0WqM`V<wX<>6>iBblqr*9
zlW-EP7S<R3B=l0SNx+l8h>wHsB5wz8IIlnNE1rAYm${s|R&p-kC}3A*+sm?=nV-3l
z=_I2kqd#K}qZ%VO!>oS-e~<oF`u+FUjbEa_pZ&J`)A4)buiRhDesTO3`Mu<q*00>3
z&%SGZTlwYpr{Ir#AN=0kdM*EI`pcr1O)q_4K7L;N^!B5(_YdC@zkT)A@mtonbZ%x`
zPrJ%>CGqn8OS3P1zjW{N$19fCny)Xp@%m=k?bUY<-<7(*?m_9JnNQT7-+5{EhVR{~
z_t!rB`*89@*vIxyQeO<dIsbU`bHne*zfu1-{C~s9#j=i-jeREjI(93LYaDEx%Q%;D
zz2J)F+QV7Hsm!&B+l|+VuYm6j?^T|g+%DW=+&j4yc#Zj-_<r(!;w|Ny#m_FtBy?9O
zLgcupp_rvurPu_qTCplI0WlWQ+rsaJjD;!%a|ERLH}O_;t8gr0Im|fsf84(-e?9*0
z{-gcp@~`e6zrQa0tomuw$6p^>-miVv_ioSIj5k|f&v;$@TK0A1tNfQuFIivPzb$$%
z^zrIP(NAWd-hM3lxai}aPggz{d^!H*&KKUVpT6pR_xREHbI-4e-zI<b{_gr~_)qKq
zg#T;)E&O})&;H*_f2seH`&Ic%;&<Qg{l7DRr~R7qlkwO2U(LT~{}K7;{@;LM8N*hF
zWJY^t1J(pK0rog{C-&29M_H{|UNYG+Wia0P@BOdo@2bCv|Ni~E@bA=LwLhPJPW`^-
ztM8ZM&qAMNK3#fW@>cSV!0U%EnP0ZOD1E{Ga_h?{FO^^Ic=hwu`d19EY+f#WA@^ec
zbHf*6FN0qCzuflH<7M-6&1aXMHa@L+n*VIx^PU&WUiQD*`s&iFrLUjAk$iXm-TQYZ
z-$}i9ct8K0)Vu6=Yu<f(Tk}Tp^}JUWubJL>zF~Ot<yGNJ?-zW}<)6wt5q~Q1Z1&U7
zPfk8L^Q8SL=W~k}VlQJ}oq7HG&5yTY?@d3%e(d^m=(Fb6H{arZ#{FUYf0@C9X&!Sj
z%MzA+mcPtqENfYl+3h&qbI5Zsa{u9)&eg!B#J!m(ozIVd2LCVqbb(m{9ReBx3<3=T
zj|Dsgrt)(OGzh8)`-yam@QPRqvkN^Jm?`j0;DF$Fp?AWLBJV|f#gfG}B!VR?q{5^p
z$%x7Y%GWA9SNyFMt8AvcOo>bBvf?AfkBS!+6BM^7NGjOKFP1fzc_jTzdV}-@X<?ad
z((=-Cr3x5E!DtAKhQMeDjE2By2n_BJDF5;32lvmHKbHRR{lWiZ|Br>gs{T0qRsR>k
z;KG#5B*VnO+{Utr)tSwKjfd?At0Su)vjby2Lp{U)|Lgz7{QdYR=I^AxAO5iY3H{yj
z%i-sq?=s&izA%5j{PFOI;`iU)#=Vt&x9{DPw<T}xyk>lp`sVkW{ck6}YkME?A@;-R
z_rCA>-#>hJ<=y#r74KHP<$25c_VSzcua~}h_v*;&y{{c!d%nK%+U<?en;ox3-e|om
z`5^pp!iV?|t{=MI2fX8ZTmI(ro1Jg>zfF8w_4dJAhIfnK?syaXTH{suOS6}sUu=D0
z{qozZ8*f_QroNs0R{CAbyW{Uz-_Lw6`qA{W{#UN=E<cNZ^Zqsd=lI`@!JqLeQx6L_
zTROW8rv!Hgj}UJnuLPeR|8#*`!MTFJ1UrS<M4yRdh?t8`6gww&Su{#SK-g1gqu^74
zX#$0U`-PH(wS*T46$zac(iIjGJ|Z+lC{*Z{prc@)fQCSmfSurC!CitM1-A)liX0I=
zEjCsBqeP07sI;K;A*oWSWGOqTV99ghJH!&jl*J~BUKRNyd{ih~&{E(&pCfM&&ugCJ
zyeYhMc#iV8@-p#8@EG!(;(5SZ$@iXDkynRzK5rTCJ|0J&%{(l8()_;s)A(NsR0^31
zn+gjH{};L^v`VN`XqnJkp<3Z)5noY3(ISy4B6gzvqN_!Ji~5Rf6e|$-m-r#ETymOJ
zzBIc`rOYXrk20dN5wg*;S7qE}qGi^|Op{5KSu5i!dqS2|&Owe{{<y*#rA5jKD*9@{
z8fuyi8p-PAYUfn{s2or}qjXKlNcpi6vr>j)tYV?!D}@UArLrHS?WCql$cx_<H4!Zm
zJs`>`S}Lq7<R#cBaDhLZKb&7lAW%?T$ViAmXrrL7;4*<eekZ;F-YY!bJb~P*T=JY|
z93t#HSU)gNXDt2S_fP*H>)+eI<Nw_H+xjp5U&g<K|BnCr@>k{0kDn$#E`Ia<y5n>0
zr<Wf$eq{dG`rhK*_qRXZ3csy<{rqLe3$^FVp0Ye?f0XyI=0U{$(|14KPQUf-rrWLZ
zTRpdQZgbt4dgtC9gS!oP@86BRckG`0{d@O+J>Y)C_*mwN%Tuvu=b!y}?)sAXRqd;}
zuQlEpz1#LK^1bedYae7j-uy86!`}}*A6-9Id{Fw3_+jn`{SPefCEf+Td-LwuyR+~5
z-{1Oh;iJT-9Uo0UGJRzH`0~TU505{%d~E%s{)Ojj$Jd?TQhrSQsrIYs*Rfy9zqkC3
z`(yDp@89bG%?!5~mN5u1#Qg97SNNCb&-`DNKbQTu^nJp&S6>`IfB7i=QTxM;cP?+!
z-}=8xc=zmW#Jg$ly*?axANTIZo9nNuUM+qh{(R%JqUX<El)jqyI{c0E+fQ%LzLj{}
z_{Q>$|C@PlZog4}`}Xa<cgFAEzAJkt|4!~*=)0tM^WU9+KkcK^XTL9DU&FuEeDC|g
z_LJ+U`p<};)jzNQZ2RT@Tky}%KL`K*_?!Rt&L5k<4*!n+d;IUr|3t<~Orp%q%-Sp;
zSmIfYSRb$)VQFBw&#c3|oT-lKAk%rKpNw}IY#28G-}x`%@73QWzmk7O|IqpV{p-0e
zyFasjPX46)al?Dg_y6DBeK-HzqIX~4mA=3H{{4H?55GR_|7h^3=hK-_S3dp!l=gY~
z=i<-TKK=Qq|FQ5x<a@z)K5xC>w!dBUcH!IiZ?C<(^WOY}+J{*m8b5M=I`Hw`2gVN?
z?=9c`dvoQr)a%2q*1bCW>h7z<uP(pxdVToyq1OVhAH1CWBJ{cRGqI;zo(Md>_4Mg8
zju*Nw{a+cpZhAfI^}pBa-^_md`Q7Rd8J`5cG=44q=KX!ukHlZhf7bm?`G18$iD@OX
zKkFnmWA+~QAMExVN7&D>6|;R~>t~ndkmju8n#aw~bDlee`!`n!*9*?WoM~L!xSM%>
z`DY3A3zi9S2;UHr5^57n5Ns1XA*d^KPw2WZujm9ZehDeb=aT27W=Q{#)|IK35tO|l
zTQ0X!?vUJjxqWg%a-Fh?vQn~(WRhhnWa?$yWnM~ekUk;(Kzg(E9O>)QDl$bfEV9ya
z)8u6p^OgFP+f;;9Z>v01Zc{2ye5r6+;gW)k;wwc4<!j1aDq^ZJs$r^*s<EoOR7KUS
z)PmG%)YhpzQsY)vS2t7lS5HzeRPRxLte&eOq`6n~hh~fBD~(wiMH(?0Wg2reerc@L
zwARwoI<Hx$sj1ne@kV`;y1n`iwP>|iwGg!fs=TTKs^?Xc)MC}F)%4ZAt7@s9RQ6Ni
zQ(P^7LbgHLPx7+(d$A9qyF@Mtdk7yCdLg)9ActR(?;wv5w+?3)doJ56)+4O3tjR3J
z%(~3~n4Fk08NV^?XIQ|nkl`SMJmVY2?M#=M%$PzLr5VisvHel~Rr_QAx74pYzF2?p
z{mk%b<_DAaC*Kynnf-dp>zFrR-(<br`F8%>uWt^$R(fUfBJf%Nlbw%lK4`lyc>m44
zsrNYVHQYURr~dZNo9C|oyXt!7`6b6o=9jE4>0Ii%<bV0b<<FNnu4rCKzH;J<+tr0v
ze_l1YW_PXbn#lF!>%rHPuiv;Hd1J>7(VM4lYTVv==iA+1_Y59nJ&b(B{kZY*y~nLj
zYM;)2CiddWOXb%uUT=KU^;YVg(z|JIU%hE~6Y$3FP2`(XZ}z-peb4rx;={oYk3UTP
z;PAotgX9Ov59dCt{`mUiy^k!Pf<Ezoj`_0ttJ!zMpNhYI{xJNV@%Q52*1zn3=l)sr
zNB8gDzlHy<{CoRv-9NK`oqyN-ne#j1m-^3yAKSmL{I=q2+Sl^0dS4HHx%I{S>&36W
z-&TIR_3h@joNrxUeZO4$6#Mb=d%JhP-{ilU^Cs$T_PeV0@gH7)c=4g}gV2Y2@4vne
z`|##N*GGv@t3FkHuKc3*)%ferFE_s&|FY@JiZ5(mdB2r>7yjw;`^MiZ|J4}tnI<rE
zv3z9S&-|Xblw~zb3X2^}EDH-u0JAo;FY{}rElgp|{wz~jGuZ7o_i%~v@bd=o-s0)y
z{=hk%BZ~bu>j#!zmTfE%tfyFASwAutFzsd7`fuK!rN2)82>)*XZO<2`&$~Wu{qW}f
zz4wpaPkvwgzVCg+d&c*>-c5P;_+7>OzweiPNc*_yWAewnAJ{(Ryk~r0^{(sfgf~B4
zhrC|=itE+;7gf){JmYy@@a*7I)@OI0Eq?C*V!?}!7l|*lU#xom`B})b=TGN8HGX>S
z3Cq*XPaU64e)jX3+w<h-bD!UMF85;Fi}IJNUk1E<`eN~m11~~fvcKwi<^Focn`v(s
zzRh|&=k24n9`D|~Tk?MUhni1eU!H!A{=V+V`JZ2YvHW5C>-cZd|FaA+Oi?UGZ1dRP
zau{+kaJO^&@bvRq@jn*$EqFmlNW?)jPxOmOi%6_UiO58eT9JbyqGFK}f28tcz2uK6
zL@BLR)=*(mS*R?lJXgtINk~ae$wpaE)m;6DM!c4-HnYxOoh7;pb>nnRb@OzM^c?h~
z4MGk78G0F67@afpFswE3*ALV?s9U7#tt+NGQ|Gq!ZLOc0%QeF^*)@M_is&5G?b17>
zH$hKO&sSGQ=Z4m8O=-;@4Q&lh4Lyxs4H?aSn%^`hYl>-RX}GBesp+W&s=27St4&s8
zR(DhnP)}6<uJ%loL$y$4f%03WZY5KtN+oyY4a$d<zbL;@u27azR#R?K-liO)JX7hm
z;${VR`T25Pa*t#m$yCX>$sCnlDy=NDMJ7^KSWZNaQTC3slXR7on`DT@3Gqbn+hVPv
zcZCCmLxr~qtq?2__{}$mSB|HZOPBKqyCB;ZmIUV6jD8FW49bif84okYFkWPEX9)X$
z_ixLeW54JAUjBRW@14Id{oeF@(QoVD`+i>eF7WNi=eZxb-amak|K)?{o1fM_Irb>`
ze(QbT$J$S;pYM4){fo!%&i^`petw(yeg9AC-^c#k_%-vJ&=>9Ze_t@axcj{EiTmRR
zkM!^9-#mHk)#daHxo6`}y*{?%Q1(8_Jx#mWcc0v~c30f4h+RgzX6)42$*}Y9jyXH0
z@0z>&?H<E@Z2MdGiyv5iQ2L16vAE;QPPm*}eLCVy?AgBa<`-_Ctvr)?F8ac)3uiCo
zUf**k^Wlr9cV4}EpYuifNB-}n|IRT?VJu{FVX0v=;he_f&p%n9SfG?YhCg0FNMH}&
zFTTV4;rz_}I{Z`kmhsB*>hXNx>f$=emC7}dGlugoX9u?`-%$ZM;b5^jl9QxeWxva%
z%16tuk)JBJRN7a9N%WCmJpXE*l^jo)Q~oRc&i@|pb^aH#FDze<d}03P`|a-6W#8<7
zc>Z|z{l?FlKly(R{{Hy0@2~3r*Z%_kZvE5q=hyEizs-N||C#fv_s`2eQ~$L5mH7Xi
zp_nO?X%k})BP-KUCPyYAhJF98{<{QPtI24}^ozNR-G%ENw*=ob{*(Mq`FHbc^8e*)
z;#<p;%jwSgl=1z4uD>2Xet+KeLGdm3tEDdtpHF_$`AFf>>4)<l#NLm;mvHywofCHo
z@8;jvc+~#*%;RTI9zWgor2f(Tho2wbeaP`J{=udDRrk)`@xSx!&YZhP?snbXb;t7d
zl$+c)zunTgfAz7@i!*ODKTY^P>(6qA3C!PE=dyoeuVJrZV`5#%9Ln^O!R7y+KN7zb
zerEnG{w4TVm|?V@8LekV>zUDdX0)Cet!HRi&-`RO!>rC)$P&$*#Qd9S2jdfl#SFrX
z#!O1gk}Nk_o!OVN+i?8gc+0ViqnhI!n+(evMneXje|5i2et3M<`10e+?QhjTY=83n
z68Nq2yZ2Z1Ps<;rKQI1S_jl(XzrSpMga6F?@5J=!|K$Jkm?W85SSE6Wa7MHCvOeW}
z&Kb_?%A(AX%&E$*$7##g%R8H+g`<H#POL@rj-a@hs!XUPyTo+`ZnY#OKjkbfU0pxz
zHQMX+Pa8{^vY0$I%+Pn&+pT*>Yo$h-_C(!#S`#%l>pB?D(*LGYt1GIPp~Io;X?V^!
z$e73YqH(C{3bV^5K?b!t;Tk^_e@gR8wo1GfUn6QRyjw_F$c^tFCm)wC?_J*IJdC{6
zyt{dvcrWqo<om<(jWeC?4(n15e=b|D*W9lBse)aC^8_`8)kLR?=7>lOn+u%~{45kK
z#w@u_;*Iz|2?nXX(mrz4^49WP3bPfZl%$mOlw(w_)x|WpHCwg9v@dG!)8WwN(-qTm
z)0?XksWU@&q4p28H7dL+PZifH6e+Q&M60B$`DxzKRMZsG{Hh+Is;+!a;k<0F>`D2%
z3hauq${$shD*sbBEca2`O5&V|vrw<VWPS_23hveHtC@E*%=ka=-^M>|f71Uj{}%jN
z_v81s4WCUvPW<ri!}^cUKURKR_~G39NAJZyY=0;HM&V`o)4qpx_iS%7-28iE$*ttu
z4Y#6i>fZQtP4N2K8-=&{?x@{ecPIJw_FMCAZM*sK+N;acE-$#ke8v0HuZxyfB(4`-
zm$-KPYQxo%E6rC@ua;b0d4>D3%EjvQZ_nI4<$s3te8~A3=X}r2IV*JL$%&*RQw}~q
zV10o1K-$6ihqoV#J{5TO)_L2Dcg~lc%{?P?=Jx3cXCltqUunJ}c<b#g-aA@%rry!G
zH~0S9hc-{-o-#ZOd%5sU(%aTI`LDHKJG`0nZrR7oFJj+U|FZt~p5Z+cC-Xwaos65A
z6q%kf_%ZM@NHS<LbTS-b+{ko{=>StX(-NjX%oZ%an1q;JSeCGSV7bZqi>-#^Bc~v@
z8jlUn6fPBxEY@pGyo?Y3SN=cs-|oNDzv_QA|4jat|LOYU_ebK-gWt}-wSEcyIPx|5
z%i>R2AL8G4zN>wE{cY^K;<rw3kG^C5Q1#*Nhq{j^KP7!p`5OFn&DX}SlfV4?EdM3(
z%lR)SzE1qk`YY+T-R~p6PyUJdqx8r1FW0{d|1udW7-um3WSPgF!Ij2q%-_nd&u_sm
zF7Svym2Ux`5kC*_2d>#%+?=o3q}WwC3^`gk$~hNvShGH2v|#Z6XY+gB57F;F--5rD
zeB=H$>D$#Ga=#}0ko)%T^O6r4Z@ga2eX`@>nR|J+AK%!2wcujd1<gx3m$EM&zxd?h
z=S$vKeqH)}vFY;9E4<ent_R-iytm`A@l(a;2Ve5Pbb8VEV&_x0N4$^3ALl$-@pR|Y
z8PC2wfALKHY1C7JXL`>jJ>U3d?`Nm)E#G;63jGTC-u&hB7nSe-elGd5=5N(MXU117
z2iT3dBzW9-R&%Z8SjTan>oRvM_Y__R0e1eYye<4E1>J;l1(Nu5`PT|?3QXru;@9W(
z;rhz%z<!0(gJ&6cEB8m9iM%<y^*n31TX=49|KUjHe9djh{ho6Pw;tbmfg)jL(I~OY
z64RwCWR^<*m2#76mN+YZTvAS^RrZQpihQeFxa=m`hYG(`%d}c_t@TUwzvw?UTxI;m
zMB8kgskd>c(S75`rmM^+TiRM*uu-#5bX?^)-C?c0zTHdfUaPH^g%+(A5>`K~W>`zw
zezcXgeP@+zo?%>KV5)DYU#<U6uSB;)J5g(<R-87YZkN8N!7RP&IzO~>)Hf*aP~57Z
zCjUu>OM0H9o`jdUulO4=CQ*B#<@_$Z?c6`P)VY4M-(a<3{`l|GZ|C2WemVUz{PX^=
z@xPjX-v1Q;#{TjDQ}n0e57!^Z-&=mA{7(7%;-BGv&i@Sm^#1t%ZurggC;m^`pTB?J
z{bl%j>DQH?9KU*gegC!a_ldu04AG3!8P74^W-$A|_HXGQhd<$e-2Qy}!~1XA{}v`a
z)<iZ1_71iTmOdtZhC6?D{W|;e;}4FX%)ie6GW_lK+x>U^ujk*Vd^P!U<<sYnZ$Iw;
zc<-afrw<=Le~kRR;LD#cTwf=Csr;<|srF<2r{*sXU+um~eLnS3_Wjw{Utj1wTk`n-
zLym{O4~rgaKjV8*_G0_93y&8(G<)EB-|YUO`|BROeyIPL<>}MsUtih0b$w^|PWN5i
zyIt>0-W`0i__fWO&bJQlbl+QiF#E9R{qFbmAL>4Oe`5Uf?!)R2njd*SW`8{L`SsTy
zUnRe}eZTR|=F8fT)8AXYo&5UytL3l6UyHpde{=m!*W2Q^F>l-7?tks`vizms>yNLK
z-ju#!ely{9`s<mmx4*vjYQ@XTFP=V^dj9d5!}IIUA3X1T&iG=*i`_4_y$E@s^nB6N
z+mC}D1wUMPzw&Ov9kDw(cUtbwy=QX&*S(s%Ww$5Z)VlHgdeRNnn<BSw+!cKw^hoLP
z=|>fhbRW%psPfR{!IAsF9+W&>{ouj_nTH1+6g<><Z2pw}g~aRPx9RVGy`A!Q+S`wB
zEZzjZS^tLn?Si-E@2cJ}`LO9z+t<$TDL+zvT>ifQ`^xVZzfJr0^6RG0DWA2z9Q}CV
zo%y@z@6LT#_<7#vlb>zBefp;KZPvHw@5MiCejonF&t%Q=o8=tqGBzW2QO>E{>b&**
zkwT_IxA<T1G4ng{?c{yJyM`y7>p5otcMR_X{&qnf!P)#0yuUdQurFr&&3cfvoHd;#
ziAkR^n0Y&!BKvIC)hui*6Pf2RNieQsG-38(de8Ws*_TC=c@c9k3pdL$=6_5nOjjA_
zFlhcS{5$)1_3z)m_x<Mk$H&~qYQuVw^)TB;RyXEYMqh@M{}=xS{m*0g#jt`wgrVgB
z#D6~j68{%7C^7ml2{4`cKkr}iKkdKo{v7=~^N+zV&L1IPRlgX0d-G$-?@#}xF?{>q
z^uPYU>;LBeNB+0{H)oj1c#H8GV>7cB+XfCF?wh>V_@xA=3r-T8BH+(=g@=K6Cyy7;
zQr>XT{vd%}0yp`~`4V_f@MQA*;I`oT%M;G)z{e=?P@q~+OL($~j;Mv$TJhBqnv&Nf
zze+htZ;{#~HD79}<YCFLQfs95%1o6rR;W|_rr4-BO;J-RPpMXUvg%Cr9?kt)zFKQE
zr)sX!NLRN}Z&OcI*HmX$o2cBOcu~Pjp+Y`Ueu{jje24s2`Lpsu3e1Yf6b~sZkrS5f
zl0GfzEO|(hRqBM~Z;3Jq9?8j)n<QsRvB_+d-6!9y&@b;RmnCyw(pF-Mc%rzJgs5bw
z<P(Y0;#OiSL?ngR2v!Ni@z?Mj<mKbd<x%F|%b~$GgZT=>hkuX$DKMO3Jj}d<HHU2q
z>ok@g=2*t?|EK<y|Eu}O&ajD*ok^GJ)!kRAubf_`yqfpw-Yfmr`(IywWAZNLz1oNF
z59dEfepLN<??dm0r5|)Ye*IYdN#)bEk2^k1`r`g=&G-DD7k`KU(`7JbY++i%63aG&
z&7IAZO@i$f>q*x4tP|N}IBs$N;nL*M<O>(LCb&r`SC~m8P$XEyNaV3_xv+(>necs~
zBZ7GX3;1LC_4#M=E$4m0vxDajk3R1&-T?j^0wO{Wgp`HFghhqzg%=4&i{y)*6cd)X
zA@N1xodl0$k))*5Vkyw{L!#_bxhy#Wxjk|{^6m1s<T&LTWtYqTm*tVmmOCduSMjov
zv9g77gz{G9r^?HfBa}UqbCstmuT;)d{-D&TB&p=DsHNZ}ZzKOvzD}V)K~v$Y{7!j8
zc@H^0*;mqkq<%^+lyDcf7uz5jE_z60n{cAgXMt9J6~0cM<y=9W3Y-d@2^@{=Hta&|
zUTo)B{xIb*dNZ8-m;UGc&yMeEUnhP_`SAE1^Sh&O``>PTbN@BRn>%mP-*&xS`}X|X
z&bM3M%zT~n%KzoI7i(W+y(oX)_RRDd%QLU1yiX23TJiAs1Gxw1?%%mD`5@!Lq6f7P
zF5Lfoul!!<J+pgn@Alj+xI6uB+1>MZ%<ufWefxIn?VGo*-ZZ&cbMxNKr8f`W<hecj
zPT1X$yK;9W?nd2hx*K?R%H2ozRy<IDRP^}jlh$XQFXCTKcy0IQ{F}hHv)`_MyY8*T
zJMs5sA3lE2`55u>&Bwi;96oz|PXEmIMe1wex1-<h{W$V->#vUA%YG~W-uP?xFXrEJ
zzeRsr{4V~z==YJ|mw(Uy{rq>(U!MQ}|NmmR&-8^QhRuon1A9M*G$%i&6GsudA-gF1
z1-7khU)g%u3ptcH^*ALtH*((K`oevTCyDn6?=#*dyeho2d7OBxc#3%9cxG@P;;P~*
z=Q8Em%NfOK!D+$ykmED^A+`;y8(C&Ew=?xKUSkMgxcNWs|D%6F|Gxd@{3rNt@87L|
z6aG2;zxh9iVJU+?qa_muvo-S!<}ECjStHmY+2q*1vnsM#vK?i$WwmAvWBtOC!V<_L
z!kWc)f_)8#IHv&T77i<pY3w`MoY`iu{bJkA#?MyHs>^E4D$Cl-BF4hOa*Fvs(>um5
z49*O9{xAFQ^}qk$)W7Hdc>S6Cd&e)4UnhQc{B-{L`iJX}HQ!f!U-O;gd(O8h-(G$@
z|E=nq<2T=LTfUwC#`{g^Yx|eSpC5hV`gHsw!>7_un?LRQ<nnpRm*j8KKPLU``DOF_
z!|$v=EB>hd_4)Vj|1rjIOkvDBnLAhlSP!u(u{E<@ViRLGWPiZc$Tpqr9ou6z8@3SE
zi7d~U)tT#=elylE>M<HKZe;lPPy4U<AHLs9e%Ac>^Zna*tsi&3Pyd$mRqd<H*NI;q
zd=CEn>eGr(KA%>7jQ{xh<El?gpA|oMeJ=WP?5pZ`hab#8lYdV6dH&~{pND^b`nl?t
z*6%&P-u!I(dGKfNum0are|Z0L|C9QE^?yEt1!E-BR_0qQ>8w*&ce1`^HD@bjTgB$a
z9?xOO*~58)^8=?Hmln4z&m*2Iy!reS1ZD~Z3DgS+2|g1%AQU2eOxQw1QzTt@nvkDR
zs?cho%R-NZJcVOKc8Q)B%NDN@&lisuzb$q^^t4E}NTx`-2%m_v@J6AtLf?dD2}uck
z7W^i-Q;=Qo3;zo~RX$tZ#oR5N+3W{dvsl!anV38nKQd@DK4a8o>S5$%v}1H-{J`*<
zA(rtnqXv^CQ$J%0qdt=h^Ks@g%$u1fGG{XXWzt|;$oQEtiOGZM594mecE&PBW5#-h
zwf`mmxBh$o*Ya=WAJadJ{#^g#`=|A{;O_^&uKjxPi|e=T?}fkR{@DF_@w?@>{qNns
zF8y5gqx^f~xA|XfzrOi0^UK07ufNRwvf%TKPy0UFf1LP{^V5@$XFnQ$GW^u`QTt=j
z$9ErdK6QWM{Z#ic?_<VC{*NplH+)?4Y4T^DFaJJke^LKh^=;GlXFuNm?D}=!SI{q?
zpOHVFf8X}K?YqqP>Ted`mVC4M-t>LO_nY5ce@y(b_Xpq4eLr{q+V<P%5Br}9zukY^
z|K|B!`s>xtH9y;aM*UpzL->2n*PJggpPzi(_2K;cIqy8)?tFdnW%i5s7rS4izgYVG
z)-#T03Qu*P*grn_=)$9Qk3K&%dHDW;%fqlow;z9alKXVk({)e7o?d#={^aeG=TB!m
z>w4z)O#a!mr)5vYpB{KJ_sQocl~22$8b5vhWYUvAkAFUz_E7f0^Lr2OKDi@zr|)*)
z?WMQ%?<C!6yd!Y;`(5GtDi1^+COtBFvj6F{XR*&`z4-X@*Q>hMlU{ec-uasCjs2Tr
pZ<yaLdC&Hd{nM>a3}4=Ub@`$DtM+&FpL>7g{!0A4{D=9kEC3t{fc^jg

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_wav/fce08de222896ac3a20657a3b4f42d5b6c54a96a b/tensorflow/core/kernels/fuzzing/corpus/decode_wav/fce08de222896ac3a20657a3b4f42d5b6c54a96a
new file mode 100644
index 0000000000000000000000000000000000000000..35a99bc97d93c9beeea6a917453685fabf853ab2
GIT binary patch
literal 69522
zcmWIYbaQJ+V`K<-40BD(Em06)U|?WmU}SJv!@$rH!N|bG01{z<0&gh&4@$$Rp#T5>
z!`P$rXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1J
zhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin
zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD
zjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mk
zz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By
z2#kgRO+$d`)`Hv0_njUFK8$@JdVl%73-`A_=)3pr=G$xgu5;g6_ekzV*o*Wh3+{W~
z3B7asp524m`we&B--^3=<Hp?UqSuyQRzBx?V&3t}vl`cv?-xE$yH$5_;i*N(rA}Ty
zGv)l<^F`;@omD;m>7vDzs4L$uK0K#=_V?LY7saoNUAcGe=?TfBoku61*mElBjOV#E
z=OoYBo?|^f_rkA>4=+qQXLau4#mhIpJZyO!b+7aK<4Yedox48gZq>sDk0YLZc|7H@
z=#%!Ri=JscD|zbp?AD8`Z}UDofA#rv<}J@_?KdnRN<TAx%l@(KSM#5mKX-n${BHWk
z#gN0`#x$Azt6;XwciAgqrF_RZtl9r?+Vb5M>=2nDI!pACsG69k*j_Py@iXG_Qd;uA
zRTt~5)C<wNro<=9D$^s!ujHk6SF1*Ef<cOYt+uOXzJ{3UB?TRME+s|nY|{es7Y2XS
zkIU;zuaK^jT`V_8;i+PV0>50j%xNhJ33HJ|0X_aXLSH1^6=$j>DDRT1m*5bQ5oQoQ
zE|DkWE*CHNTKc#67O_)e>>|bjNBJ%Z8pxPww(50jFH-AO?33Ln!z?>h)?T4oNmKE%
z+&#IQ@(<*^q>qU`5^ff$lVFs6BIh8RDDg<}AWtrr4EGnlNZ~@!X3@jKQw66BrHSMV
zHgWG}&*J#O_guV1T1-+?)KYL6-+sP7e9!pi37iz{=HJNkl{<qcjpq=T1G^>5Tb3M7
zd%i~kvV!0Ews2c<&ElTVe^T^;)Cy?>$w{K+LT3ff3TpE^aocj<;ffW!Euky*NyLTM
zjrG?5!~dB6KmFg%e4gziYdZ5OraI<s<|w8s3`z_NjLt0X9MxPJoCPeC{=NDo@`sCQ
z73X!n0D(7rpSYyh5}4;Oe)}u^OZ4ae-xHWO@Yo9O;Qz(-jYXH?%fH18^H>VFZt%V2
zx8Og*Th0BB!<Ma<`90%dCLcCEo=1XA!rOV3STFzG_Up@^wM<Ry!JJl{$Jy0bH#7Ax
zto^I=o8kAmf9&i_1!E)=q&JB_;`iooW$9qjWG>*i#P2QgQuL>on)nJ)F5wmY|GC+@
z=5gl=h>0;u)`{it$8y-Qv2vZ`KP)UN_DB4I#AS(-61EZt#5_fx3s(zwieHuwR@<yT
zPqj#~K}JyWkJw95b%_%)59OxHX2`_IT$5In>KAVlDHgI9zA3&})>t7#_J(+j@IS#S
zk^2%$q}!xFNm@&s5j!aQK;)LNwGgwwQa)_~36cL|eq!$gZMkb$y#5>iKKeud*Ui5c
z|J(lF{^k4Q!?)&dwqN)^2)_IB{@$lIpU-^!{QBHumAkgLi|(C&V)^>lTZK1=U#LF&
z_DJ&n<~!ndlJ2a%$Nhxk^{)^AKf8WbeZTOf#Z!^Tmmbf4aq@lZm-An~d}8>>`~K6L
z<*y80ZhNKqq5Hen-y{F{ertZd@aoMojhEu@8@_(}nf#mcH^;B{-wnQA|781d{zuNQ
z$-h1`^m0`3E$08m{g~zNAL}0%e>nbCVLrj}mq&r$iGKpGD_1BRKXWnTex~E>b$q2l
zw*=O5iLv-IsxxbHbn;CURuP*jp&&g$Dp%Z8^oDSv&~HIeVJopSGFmDp)cBP{r0aw#
zxX-a4VvAyz;;iH{=9<GH!gh`6?*IJ12EU*Ea`;!t+`%rxk-#kZ=j9LK-^c%o{{Qel
zpJ5)O0&^^*^<U#ZfBw$-8~sc0d-AvbpSS-;|9ktB>9gpYDbM#kV|$_ds`ahQ2k8$6
z?`+;=zG{Ej@v{HLhUY6@M!)TRANbDrrRw9C4<9|{d6n{J{#&)T(_b6Etb43}NBri)
zo11Q4zJB@Yy{n10wV&*HzVz{lyI*gH-`H?7^X`U+ho1Soj(nZ-{Kw<QN7o<bKHU3|
z>&ed-d*5|_68^O1P2bab4>=yKf8zKe{PngsuixH&SM}Qdsp4a`r`pf<KYjXi%8R2P
zw*M|<;Q#aOvw~jsvuSUhzxe*<$LHz4>;BpQ$@($%%gT@1A0*xfzUTcc`?LD5>)-i5
zwtrH6zu+DBhfkm6zhC^_%dmjijA{8_=U;h0CVqSVW%cK>ulxS-^1Kz+7e3Ep$I|n+
z@z=laqTgD7PG`8z%+ECKU*j*<?_0mz`gq}e^T#V+&-_&X9rL5=v-^jj_a^T*zQ6o|
z{d3Ltu0M<ZR{xItJ?~HJZ?zx)Kev5i`DxEm&ZW%xiMjZ1>W`Rj%3n`?dG^DZVG^q)
zyA#_s=FLn_%w>%A|N8&MF*>q}arm)qW=vt&%v8j}!>YtKg}s{d6PG848<X*W$^RaI
zPyAH*=Jh4`r#`a*M;dGN|K^`X-wMB-{W|fB&<~^kvzh!EjsMU6EB5algENyk(+ZY)
zjtg9SxSp~tXW00s^N;jD1%_~@i!3WRj_~mEgt0wgDrZh(W@MVe5XHEV(^^bgT273c
zzkzcDs~gKR<`|YbP9dT1q7Q_o@+xp>GFvfR_`B?P*B>K>Ys^fnW{j-ATE8TG`ucI_
zr?#(&zgQW}SQaxg{;T@I@pbiwpRZ|J5sw-^8UmvsFd71*Aut*OqaiRF0;3@?8Umvs
zFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF
z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UmvsFd71*Aux19V03fV=;o}^%~_+Hvqm>(
zjc(2w-JFGgbJp~ie_y$O7GN-D-_4Tp*ZOPJyKOHoJym@C^QpwUZJ!@~s(2sy=D|zD
z7d=n6K9YRW@^bS#ua9#-u)e+geAy%0`%mv?-&^u<*)zk}hu_-1HGU=fjOk(WZNY1k
zFYmlD^YPkO`mfeMb$jsb_Qad-u7A6}>rT?siZ@yB=Y3%P81`Y(d!KhcZw|lS_O|Wg
zhOZmHOMaj5RsD<Fmx`~KzDfUl`=^`XAd?647N!-9&i{Y?e)m)F_j|^Ao;xC%68z$B
z!s-H_`1bSu;<>};BFrlBTWYbitF(d48M*BW>55yGHmOLfw`itnZd3cK{8RC`qP7Z$
zdW_~}Z5F*Y{jK^hbZxYyG(V_^s+(%;(_L%!*?OCmy{V0!hNg?Ujv9lSq-L3(n^CY)
zvwpR<gu0FLT7~WMTncX#WR-Q)RMnW3T4iFz`9&MV*d(S(xJf;ec9ss6SShT`H;Y4&
zWe#I9lOCtC$QtQ5nJ|eQ!C<b5tSgz;G77PzbGq^i3U>)-3bydY@yy}6#4(-YA(uG6
zsW6+!B7v1$eaw^pN&T~Aux38Tc9=7T$AfP_&vf=oM&Uo%Kh}R|`ZJk@nJ1Djo_hkz
z+`qSfT>A3kW9(<CpYQ(jvodo$=g4Q*XOU%i^!xe`{U6r90{<B>MKiBq$oVbx<LD28
zUtzz#|MK~B<X;D4Iiumf^FJfL`hME<$>aOpe-~N5b0%~BVe4nS{-@@r{&%HsJHJ)`
zZ23E%p@dO@VaMOizwLjvea-ze|5NwZrtj9@cs~`q>3k9KV%N(<uZ&*Hzh3vo@Waaw
z-``AsUj2CW!_|-8KjnTc{;}xmg0ELU+Q036RsQnzi|QBIFOI+5|C;+v;;ZY=xu5=c
z^!cIm!vznFAE`ald^Pp;uNQY8Uwr8FH1K8HtG-v?U#)(#_T9WUvd>oEH@v;~#_gN0
z@BMk2{$}%g`*#^H<~``X`T9!vWriz3S0irN-@WxP;mL|8XCD81T>LctS>N-Yug`z_
z@SXkVov&OU3g0xpUH@L|<KNG(zEAsenQ<R86T_LGeqYo-F8Y}KHSxC;vou#1&pOTw
zmSz8!{z>><{=4w+eMTYn(_9C5e)Di}|7I^@m0)?uvV?=1Peiy;tWc~`XffYw-fq4t
zd<FdJf|JE2$%-oclKw5)C9s~ml2eCULtvk{x4eV$Z>2@@CQ>s+(gYfLpK+)2c<`?k
zULk%#Vw2cgp$5KPoE&W3EDu?H*^1b2v2rk-`v2tr{Qpb-iT{)MC&>`T;>VW8%=0hl
z$E(i?p9(&Sd_Mgx;qO7F?JQBu-VAsD3I0$1>-=lpPrg4M{~s_;X1d8B^Ka?zBR?nn
z?E78uFOu;E^Gud2Otp+9OwKG@Sy!<2vommC6DSw17r4yr%<+<~k)w*Mhx;d=mdJTg
zap4fYZ5+#4;+VfM+W)uv=gHu}V!&a+na;+@yohl-<8!86tO1<eJRy9`d4xIUvPQ6r
zbFJZN6zCOok~yx-qJB;}S*}@fsc5c9r)Y(Er*x{Kuxhl5r~G;GpF$-<F9culCkvbx
zxh-{Fo?CI7>`{rEBDO-G1k*(wq^~P1RgqMkqNpKTFV!uvRP2T5d$C(mFBR@6Ur^wZ
z)(}0-pT)DDtA%SfU#-|=X>I8{Vr2rSxmL5Avd&_f&a{!uk#~o%iTGO4L;QgpeN4~)
zhyQ={e?8+W=DkdX|EhjH`Em36u5Sy!iT{xJ<IiNnmcg3NwEEwvpV?pMeoFdy?DK)2
zj{lD_rZX=7cjmYDFQ@NXpETe7dVBcer0=tSXa0`(ap#NlC!vqZpJcwA`^x!!+IQ{m
zd%kgf-|#*9$HO1eKWF?B{`Z#Q1;gooi+{WP?E1OsSLW}%|IV;X;gH}cXSx4x{jb|U
zGk!>adGX=&$ID-Je(U{@_;2}V+7Hg}6+c*hcmA3A_s~DRe=fi0{iyzN^jE^aJVtru
z^{j6>O?bMwbl7c}<^I3^bMKGdza306T#NZ<3&``<umv#AW@uzM`1kuS>)+P@Jy<`p
ze`7ZKEBwRe>((z!-%fuw{MGeG`?tkc?f2qu9=@9QYRT)1Z}mRT|0?h^?`O%kDIcD_
zba^`M(V~ZuPxIft{A~C2@<+ed0?!saS^DVs-M6<s-#zfu?tRYJZC~`?UwiTJ@u`PR
zkLEtP^<4Ayme*Hb*gh3`JnfOlWB12tk6oWNy!rdF;<MzZQ}482-g)GBFY4~*N3-5o
zep&Zz)3>KzcYW#jEcc<|W!~c>4_ltvy*2wH{mtid{JUQ-%U%?}41L}EF7?y;&wD;D
ze6ROj;=Rc`vA3LW55KeiRQs*w*P6eR|4sOv^6kbak<S~x&-@q6EXlH&X)(k3|LYk-
z|L^<h{H6bs*O$B>U;h~VSNb>W*Q;-zzb^lN=;zm8yMI0YVf8KK^V1KpAAWq4{1W&z
z;j8-B*I(_v@A~HViS0G}li2%D?n*rLd%pFR{Y#B!OCAS4Qh#**0qb3{oATFRT;F)x
z<^lJk4fi8%Nn9(tw)m#`9kF{C@1410cO(AN*)v~G`kb15y5x-gS>E&AmsVWszp?*n
z`Z<SV_YVplZau+rzWDNqD{n4UpF41h;Z*-ggX1+v-W*<WyzBhxYkO}PUwd&O|Mb%n
zxhLCBKRK6vDddXt6`f0w=dYd%zIgPC{|&A?tq)9|+rAh7Qu#^jO~%tV54j)TdC~Im
z^G_`XA?D*uj!e?b4ovL-_Wiv6qx{c)<{8|-1pI|A@_*y`z@yD)$<HseNK9JFM!Hlg
zSyD+#S!Tb?Ea|^eS7c@=?pE*DnXId>wO-|_!V-llmCIV`2KL6MjMa_T8(lN9Hu|oA
zOXq^tWSwTCOBOyhH*HP~-46e#%SS_CGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0q
zLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(
zGz3ONU^E0qLtr!n24x5^4@y^!I%+foMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(
zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONV33BuLdHSr
zuu*4?hQMeDjE2By2#kinC>RZa(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R
K7!3h>hX4Te88qSm

literal 0
HcmV?d00001

diff --git a/tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict b/tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict
new file mode 100644
index 0000000000..eab65386ce
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/dictionaries/decode_wav.dict
@@ -0,0 +1,4 @@
+header_RIFF="RIFF"
+header_WAVE="WAVE"
+section_fmt="fmt "
+section_data="data"
-- 
GitLab


From 26592d4bb9d9e0ca4262c04fb9497e1583981114 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 16:16:11 -0800
Subject: [PATCH 1076/1554] disable a failing test in oss env.

PiperOrigin-RevId: 223430806
---
 tensorflow/python/keras/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 48cdbf1e66..a573fd5cfb 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -754,7 +754,10 @@ py_test(
     size = "enormous",
     srcs = ["engine/training_generator_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],
+    tags = [
+        "no_oss",
+        "notsan",
+    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From 0ec0dbe21a74956ec3a31d775e7679b9e8d2d03e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 16:16:17 -0800
Subject: [PATCH 1077/1554] Improving error reporting in LogDump.

PiperOrigin-RevId: 223430822
---
 tensorflow/lite/toco/tooling_util.cc | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 44fa658f9c..d0cc799424 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -536,12 +536,12 @@ void DumpGraphvizVideoFrame(const Model& model) {
   if (!dump_hashes.count(hash)) {
     LOG(INFO) << "DUMPING GRAPHVIZ VIDEO FRAME: " << dump_id;
     dump_hashes.insert(hash);
-    CHECK(port::file::SetContents(
-              port::file::JoinPath(
-                  dump_options.dump_graphviz,
-                  toco::port::StringF("toco_video_%05d.dot", dump_id)),
-              graphviz_dump, port::file::Defaults())
-              .ok());
+    const auto result = port::file::SetContents(
+        port::file::JoinPath(
+            dump_options.dump_graphviz,
+            toco::port::StringF("toco_video_%05d.dot", dump_id)),
+        graphviz_dump, port::file::Defaults());
+    QCHECK(result.ok()) << result.error_message();
     dump_id++;
   }
 }
@@ -555,14 +555,13 @@ void LogDump(int log_level, const string& message, const Model& model) {
     string graphviz_dump;
 
     DumpGraphviz(model, &graphviz_dump);
-    CHECK(port::file::SetContents(
-              port::file::JoinPath(
-                  dump_options.dump_graphviz,
-                  absl::StrCat("toco_",
-                               absl::StrReplaceAll(message, {{" ", "_"}}),
-                               ".dot")),
-              graphviz_dump, port::file::Defaults())
-              .ok());
+    const auto result = port::file::SetContents(
+        port::file::JoinPath(
+            dump_options.dump_graphviz,
+            absl::StrCat("toco_", absl::StrReplaceAll(message, {{" ", "_"}}),
+                         ".dot")),
+        graphviz_dump, port::file::Defaults());
+    QCHECK(result.ok()) << result.error_message();
   }
 
   if (!VLOG_IS_ON(log_level)) {
-- 
GitLab


From 4ce74c4b95d31bf28301a3b7b3c65a43266d1954 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 29 Nov 2018 16:18:27 -0800
Subject: [PATCH 1078/1554] Internal change.

PiperOrigin-RevId: 223431155
---
 tensorflow/contrib/distribute/python/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 38ce0b2b8e..249258def3 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -436,7 +436,6 @@ cuda_py_test(
 
 cuda_py_test(
     name = "estimator_training_test",
-    size = "large",
     srcs = ["estimator_training_test.py"],
     additional_deps = [
         ":collective_all_reduce_strategy",
-- 
GitLab


From 895ce1cdb897c00d671a93a2263faa31b7ff7006 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 16:28:24 -0800
Subject: [PATCH 1079/1554] Fix build

PiperOrigin-RevId: 223432621
---
 tensorflow/lite/interpreter_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index 2e0dc77dcd..09b8832b4d 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -1099,8 +1099,8 @@ class TestDelegate : public ::testing::Test {
       };
       delegate_.CopyFromBufferHandle =
           [](TfLiteContext* context, TfLiteDelegate* delegate,
-             TfLiteBufferHandle buffer_handle, void* data,
-             size_t size) -> TfLiteStatus {
+             TfLiteBufferHandle buffer_handle,
+             TfLiteTensor* output) -> TfLiteStatus {
         // TODO(ycling): Implement tests to test buffer copying logic.
         return kTfLiteOk;
       };
-- 
GitLab


From 96e9c4e6844254876a3e258ca79ec046c45d7b02 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Thu, 29 Nov 2018 16:39:54 -0800
Subject: [PATCH 1080/1554] Update error messages in tflite_convert.

PiperOrigin-RevId: 223434427
---
 tensorflow/lite/python/convert_saved_model.py   | 13 +++++++++++--
 .../lite/python/convert_saved_model_test.py     | 16 +++++++++++++++-
 tensorflow/lite/python/lite.py                  | 11 ++++++-----
 tensorflow/lite/python/lite_test.py             | 17 +++++++++++++++--
 4 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/tensorflow/lite/python/convert_saved_model.py b/tensorflow/lite/python/convert_saved_model.py
index ad7c87109c..f8d986b746 100644
--- a/tensorflow/lite/python/convert_saved_model.py
+++ b/tensorflow/lite/python/convert_saved_model.py
@@ -199,7 +199,9 @@ def set_tensor_shapes(tensors, shapes):
       integers representing input shapes (e.g., {"foo": : [1, 16, 16, 3]}).
 
   Raises:
-    ValueError: `shapes` contains an invalid tensor.
+    ValueError:
+      `shapes` contains an invalid tensor.
+      `shapes` contains an invalid shape for a valid tensor.
   """
   if shapes:
     tensor_names_to_tensor = {tensor_name(tensor): tensor for tensor in tensors}
@@ -208,7 +210,14 @@ def set_tensor_shapes(tensors, shapes):
         raise ValueError("Invalid tensor \'{}\' found in tensor shapes "
                          "map.".format(name))
       if shape is not None:
-        tensor_names_to_tensor[name].set_shape(shape)
+        tensor = tensor_names_to_tensor[name]
+        try:
+          tensor.set_shape(shape)
+        except ValueError as error:
+          message = ("The shape of tensor '{0}' cannot be changed from {1} to "
+                     "{2}. {3}".format(name, tensor.get_shape(), shape,
+                                       str(error)))
+          raise ValueError(message)
 
 
 def freeze_saved_model(saved_model_dir, input_arrays, input_shapes,
diff --git a/tensorflow/lite/python/convert_saved_model_test.py b/tensorflow/lite/python/convert_saved_model_test.py
index 0d32c34391..76113853ca 100644
--- a/tensorflow/lite/python/convert_saved_model_test.py
+++ b/tensorflow/lite/python/convert_saved_model_test.py
@@ -75,7 +75,8 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
     convert_saved_model.set_tensor_shapes([tensor], {"Placeholder": [1, 3, 5]})
     self.assertEqual([1, 3, 5], tensor.shape.as_list())
 
-  def testSetTensorShapeInvalid(self):
+  def testSetTensorShapeArrayInvalid(self):
+    # Tests set_tensor_shape where the tensor name passed in doesn't exist.
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
 
@@ -87,6 +88,19 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
         str(error.exception))
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
 
+  def testSetTensorShapeDimensionInvalid(self):
+    # Tests set_tensor_shape where the shape passed in is incompatiable.
+    tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
+    self.assertEqual([None, 3, 5], tensor.shape.as_list())
+
+    with self.assertRaises(ValueError) as error:
+      convert_saved_model.set_tensor_shapes([tensor],
+                                            {"Placeholder": [1, 5, 5]})
+    self.assertIn(
+        "The shape of tensor 'Placeholder' cannot be changed from "
+        "(?, 3, 5) to [1, 5, 5].", str(error.exception))
+    self.assertEqual([None, 3, 5], tensor.shape.as_list())
+
   def testSetTensorShapeEmpty(self):
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py
index 1fb5618466..1b20ff2f92 100644
--- a/tensorflow/lite/python/lite.py
+++ b/tensorflow/lite/python/lite.py
@@ -400,15 +400,16 @@ class TFLiteConverter(object):
     # Checks dimensions in input tensor.
     if self._has_valid_tensors():
       for tensor in self._input_tensors:
-        if not tensor.get_shape():
+        shape = tensor.get_shape()
+        if not shape or not shape.as_list():
           raise ValueError("Provide an input shape for input array "
                            "'{0}'.".format(_tensor_name(tensor)))
-        shape = tensor.get_shape().as_list()
-        if None in shape[1:]:
+        shape_list = shape.as_list()
+        if None in shape_list[1:]:
           raise ValueError(
               "None is only supported in the 1st dimension. Tensor '{0}' has "
-              "invalid shape '{1}'.".format(_tensor_name(tensor), shape))
-        elif shape[0] is None:
+              "invalid shape '{1}'.".format(_tensor_name(tensor), shape_list))
+        elif shape_list[0] is None:
           self._set_batch_size(batch_size=1)
 
     # Get quantization stats. Ensures there is one stat per name if the stats
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 8832247f3c..1ae0d3c3ed 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -182,7 +182,20 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     out_tensor = in_tensor + in_tensor
     sess = session.Session()
 
-    # Test invalid shape. None after 1st dimension.
+    # Test None as shape.
+    converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
+                                                  [out_tensor])
+    with self.assertRaises(ValueError) as error:
+      converter.convert()
+    self.assertEqual('Provide an input shape for input array \'Placeholder\'.',
+                     str(error.exception))
+
+  def testSizeEmptyInvalid(self):
+    in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[])
+    out_tensor = in_tensor + in_tensor
+    sess = session.Session()
+
+    # Test empty shape.
     converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
                                                   [out_tensor])
     with self.assertRaises(ValueError) as error:
@@ -190,7 +203,7 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     self.assertEqual('Provide an input shape for input array \'Placeholder\'.',
                      str(error.exception))
 
-  def testBatchSizeInvalid(self):
+  def testSizeInvalid(self):
     in_tensor = array_ops.placeholder(
         shape=[1, None, 16, 3], dtype=dtypes.float32)
     out_tensor = in_tensor + in_tensor
-- 
GitLab


From a196998315720712bd650531aa8273bbc910b991 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 17:12:21 -0800
Subject: [PATCH 1081/1554] * Create Fingerprint() method for HloModule,
 HloInstruction, and HloComputation for hashing to accelerate checking
 identical HLO modules. * Make HloRunner's CreateExecutable method public so
 that an HLO graph can be compiled without running. Make HloRunner's
 ExecuteWithDeviceBuffers method accept an executable that has already been
 compiled. These two changes enable parallelizing compilation of multiple
 configs of each HLO op.

PiperOrigin-RevId: 223439459
---
 .../compiler/xla/service/hlo_computation.cc   |  2 ++
 .../compiler/xla/service/hlo_computation.h    |  6 ++++
 .../compiler/xla/service/hlo_instruction.cc   | 20 +++++++++++
 .../compiler/xla/service/hlo_instruction.h    | 12 ++++++-
 .../compiler/xla/service/hlo_instructions.cc  |  4 +++
 .../compiler/xla/service/hlo_instructions.h   |  2 ++
 tensorflow/compiler/xla/service/hlo_module.h  |  6 ++++
 tensorflow/compiler/xla/service/hlo_runner.cc | 34 +++++++++++++++++++
 tensorflow/compiler/xla/service/hlo_runner.h  | 20 ++++++++---
 9 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index d06c2207cb..ff122b529b 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -711,6 +711,8 @@ bool HloComputation::operator==(const HloComputation& other) const {
   return eq(root_instruction(), other.root_instruction());
 }
 
+uint64 HloComputation::Hash() const { return root_instruction()->Hash(); }
+
 Status HloComputation::ReplaceWithNewInstruction(
     HloInstruction* old_instruction,
     std::unique_ptr<HloInstruction> new_instruction) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index be1ce33696..c584e4c7ca 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -264,6 +264,12 @@ class HloComputation {
   // Return whether `*this` and `other` are functionally equivalent.
   bool operator==(const HloComputation& other) const;
 
+  // Generates a hash value of an HLO computation. Hash considers
+  // information on opcode, shape, operands, and typically a root instruction.
+  // This function returns the same hash value for equivalent HLO computations,
+  // with respect to HloInstruction::Identical() method.
+  uint64 Hash() const;
+
   // Replaces old instruction with newly created instruction. Removes old
   // instruction from computation. Updates uses and root instruction.
   Status ReplaceWithNewInstruction(
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 36a7a5029b..21b1dbc167 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1760,6 +1760,26 @@ bool HloInstruction::IdenticalSlowPath(
   return false;
 }
 
+uint64 HloInstruction::Hash() const {
+  using tensorflow::Hash64Combine;
+
+  uint64 hash_value = Hash64Combine(0, static_cast<uint64>(opcode()));
+  hash_value = Hash64Combine(hash_value, ShapeUtil::Hash(shape()));
+
+  if (!IsCrossModuleAllReduce()) {
+    if (!operands().empty()) {
+      for (size_t i = 0; i < operands().size(); ++i) {
+        hash_value = Hash64Combine(hash_value, operand(i)->Hash());
+      }
+    }
+  }
+
+  hash_value = Hash64Combine(hash_value, InnerHash());
+  return hash_value;
+}
+
+uint64 HloInstruction::InnerHash() const { return 13; }
+
 void HloInstruction::RemoveUser(HloInstruction* user) {
   auto set_it = user_set_.find(user);
   CHECK(set_it != user_set_.end());
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 8569e86b83..a54716217d 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -888,7 +888,7 @@ class HloInstruction {
 
     // Two AllReduces are Identical if they have the same all_reduce_id.
     // Their operands don't have to be Identical.
-    if (!this->IsCrossModuleAllReduce()) {
+    if (!IsCrossModuleAllReduce()) {
       // Use an explicit loop rather than ContainerEquals, because copying
       // around std::functions may be too expensive in some cases.
       for (size_t i = 0; i < operands().size(); ++i) {
@@ -905,6 +905,12 @@ class HloInstruction {
     return IdenticalSlowPath(other, eq_computations);
   }
 
+  // Generates a hash value of an HLO instruction. Hash considers
+  // information on opcode, shape, operands, and typically a root instruction.
+  // This function returns the same hash value for equivalent HLO instructions,
+  // with respect to HloInstruction::Identical() method.
+  uint64 Hash() const;
+
   // Returns whether the instruction has a constant operand.
   bool HasConstantOperand() const;
 
@@ -1613,6 +1619,10 @@ class HloInstruction {
       const std::function<bool(const HloComputation*, const HloComputation*)>&
           eq_computations) const;
 
+  // Generates a hash value specific to a particular type of an instruction.
+  // This function typically considers the inner root instruction.
+  virtual uint64 InnerHash() const;
+
   // Creates an n-ary elementwise operation.
   static std::unique_ptr<HloInstruction> CreateNary(
       const Shape& shape, HloOpcode opcode,
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 6a57b75488..1ea02cf9c0 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -1372,6 +1372,10 @@ bool HloFusionInstruction::IdenticalSlowPath(
                          other.fused_instructions_computation());
 }
 
+uint64 HloFusionInstruction::InnerHash() const {
+  return fused_instructions_computation()->Hash();
+}
+
 std::unique_ptr<HloInstruction> HloFusionInstruction::CloneWithNewOperandsImpl(
     const Shape& shape, absl::Span<HloInstruction* const> new_operands,
     HloCloneContext* context) const {
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index e35c6e9150..b5c28137a1 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -743,6 +743,8 @@ class HloFusionInstruction : public HloInstruction {
       const HloInstruction& other,
       const std::function<bool(const HloComputation*, const HloComputation*)>&
           eq_computations) const override;
+  uint64 InnerHash() const override;
+
   // Implementation for non-common logic of CloneWithNewOperands.
   std::unique_ptr<HloInstruction> CloneWithNewOperandsImpl(
       const Shape& shape, absl::Span<HloInstruction* const> new_operands,
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 66622a1d26..7b9cbf9a53 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -132,6 +132,12 @@ class HloModule {
     return config_.entry_computation_layout();
   }
 
+  // Generates a hash value of an HLO module. Hash considers
+  // information on opcode, shape, operands, and typically a root instruction.
+  // This function returns the same hash value for equivalent HLO modules,
+  // with respect to HloInstruction::Identical() method.
+  uint64 Hash() const { return entry_computation()->Hash(); }
+
   // Gets the computations in this module.
   //
   // Returns a view of HloComputation*s, so you can iterate over this in the
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index 3f0ca342b4..5a9b820a9d 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -205,6 +205,40 @@ StatusOr<ScopedShapedBuffer> HloRunner::ExecuteWithDeviceBuffers(
       /*profile=*/profile);
 }
 
+StatusOr<ScopedShapedBuffer> HloRunner::ExecuteWithDeviceBuffers(
+    std::unique_ptr<Executable> executable,
+    const absl::Span<const ShapedBuffer* const> arguments,
+    ExecutionProfile* profile) {
+  // Get service run options.
+  se::Stream stream(backend().default_stream_executor());
+  stream.Init();
+  ServiceExecutableRunOptions service_run_options =
+      GetServiceRunOptionsForDevice(backend().default_device_ordinal(), &stream,
+                                    nullptr);
+
+  TF_ASSIGN_OR_RETURN(
+      ScopedShapedBuffer retval,
+      executable->ExecuteOnStreamWrapper(&service_run_options,
+                                         /*profile=*/profile, arguments));
+  TF_RETURN_IF_ERROR(stream.BlockHostUntilDone());
+  return std::move(retval);
+}
+
+StatusOr<ScopedShapedBuffer> HloRunner::ExecuteWithDeviceBuffers(
+    std::unique_ptr<Executable> executable,
+    const absl::Span<const ScopedShapedBuffer> arguments,
+    ExecutionProfile* profile) {
+  std::vector<const ShapedBuffer*> argument_pointers;
+  argument_pointers.reserve(arguments.size());
+  for (const auto& argument : arguments) {
+    argument_pointers.push_back(&argument);
+  }
+  return ExecuteWithDeviceBuffers(
+      /*executable=*/std::move(executable),
+      /*arguments=*/argument_pointers,
+      /*profile=*/profile);
+}
+
 StatusOr<std::vector<Literal>> HloRunner::ExecuteReplicated(
     std::unique_ptr<HloModule> module,
     const ReplicatedExecuteOptions& options) {
diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h
index 2e934bf66a..bb792cf8c9 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.h
+++ b/tensorflow/compiler/xla/service/hlo_runner.h
@@ -136,6 +136,21 @@ class HloRunner {
       const absl::Span<const ScopedShapedBuffer> arguments,
       bool run_hlo_passes = true, ExecutionProfile* profile = nullptr);
 
+  StatusOr<ScopedShapedBuffer> ExecuteWithDeviceBuffers(
+      std::unique_ptr<Executable> executable,
+      const absl::Span<const ShapedBuffer* const> arguments,
+      ExecutionProfile* profile = nullptr);
+
+  StatusOr<ScopedShapedBuffer> ExecuteWithDeviceBuffers(
+      std::unique_ptr<Executable> executable,
+      const absl::Span<const ScopedShapedBuffer> arguments,
+      ExecutionProfile* profile = nullptr);
+
+  // Creates an executable object given an HLO module. If run_hlo_passes is
+  // true, the HLO passes will be run as part of compilation.
+  StatusOr<std::unique_ptr<Executable>> CreateExecutable(
+      std::unique_ptr<HloModule> module, bool run_hlo_passes);
+
   // Executes a given HLO module into a set of replicas, and returns a map
   // with the replica number as key, and the corresponding returned literal as
   // value.
@@ -152,11 +167,6 @@ class HloRunner {
   const Backend& backend() const;
 
  private:
-  // Creates an executable object given an HLO module. If run_hlo_passes is
-  // true, the HLO passes will be run before.
-  StatusOr<std::unique_ptr<Executable>> CreateExecutable(
-      std::unique_ptr<HloModule> module, bool run_hlo_passes);
-
   // Creates a ServiceExecutableRunOptions object to configure a run on device,
   // using the provided stream object. If device_assignment is not nullptr, it
   // will be used to configure the replication parameters. Replicated executions
-- 
GitLab


From a245fa2ecaf8d0d2d19eb680f73b5e960b9dd184 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 29 Nov 2018 17:16:48 -0800
Subject: [PATCH 1082/1554] Distribution Strategy: Enable eager mode in Keras
 with MirroredStrategy (Part 1). Will update tests in follow ups.

PiperOrigin-RevId: 223440045
---
 .../distribute/python/examples/keras_mnist.py |  9 +++-
 .../contrib/distribute/python/keras_test.py   |  2 +-
 .../tpu/python/tpu/keras_tpu_variables.py     |  4 ++
 tensorflow/python/framework/func_graph.py     |  2 +-
 tensorflow/python/keras/backend.py            |  2 +-
 .../engine/distributed_training_utils.py      | 41 ++++++++++-------
 tensorflow/python/keras/engine/training.py    | 10 ++--
 .../keras/engine/training_distributed.py      | 46 +++++++++++++++++--
 .../python/keras/optimizer_v2/optimizer_v2.py |  2 +-
 tensorflow/python/training/optimizer.py       | 11 +++--
 10 files changed, 95 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/examples/keras_mnist.py b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
index 0fd3acd045..8b6487252d 100644
--- a/tensorflow/contrib/distribute/python/examples/keras_mnist.py
+++ b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
@@ -102,18 +102,23 @@ def main(_):
   # Build the train and eval datasets from the MNIST data. Also return the
   # input shape which is constructed based on the `image_data_format`
   # i.e channels_first or channels_last.
+  tf.enable_eager_execution()
+
   train_ds, eval_ds, input_shape = get_input_datasets()
   model = get_model(input_shape)
 
   # Instantiate the MirroredStrategy object. If we don't specify `num_gpus` or
   # the `devices` argument then all the GPUs available on the machine are used.
-  strategy = tf.contrib.distribute.MirroredStrategy()
+  strategy = tf.contrib.distribute.MirroredStrategy(['/gpu:0', '/cpu:0'])
+
+  # TODO(priyag): Use RMSPropOptimizer when it works with eager mode.
+  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
 
   # Compile the model by passing the distribution strategy object to the
   # `distribute` argument. `fit`, `evaluate` and `predict` will be distributed
   # based on the strategy instantiated.
   model.compile(loss=tf.keras.losses.categorical_crossentropy,
-                optimizer=tf.train.RMSPropOptimizer(learning_rate=0.001),
+                optimizer=optimizer,
                 metrics=['accuracy'],
                 distribute=strategy)
 
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 6a006cc85f..435f10358f 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import tpu_strategy
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import values
+from tensorflow.python.eager import test
 from tensorflow.python.estimator import keras as keras_lib
 from tensorflow.python.estimator import run_config as run_config_lib
 from tensorflow.python.framework import constant_op
@@ -38,7 +39,6 @@ from tensorflow.python.keras.engine import distributed_training_utils
 from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
 from tensorflow.python.ops.parsing_ops import gen_parsing_ops
 from tensorflow.python.platform import gfile
-from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import rmsprop
diff --git a/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py b/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py
index 28d3a93851..8b0b240dc7 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_tpu_variables.py
@@ -217,6 +217,10 @@ class ReplicatedVariable(object):
   def get(self):
     return self._primary_var
 
+  @property
+  def _in_graph_mode(self):
+    return self._primary_var._in_graph_mode   # pylint: disable=protected-access
+
   def _should_act_as_resource_variable(self):
     """Pass resource_variable_ops.is_resource_variable check."""
     pass
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index a8a7948b99..f74d072e8e 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -113,7 +113,7 @@ class FuncGraph(ops.Graph):
     # this stack from the default graph even in eager mode. Maybe it should be
     # part of the eager context? This would also allow us to remove a
     # get_default_graph() call from the function cache lookup.
-    self._distribution_strategy_stack = graph._distribution_strategy_stack
+    self._distribution_strategy_stack = list(graph._distribution_strategy_stack)
     # We ignore device placements from any outer scopes while tracing the
     # function when possible, to avoid hard-coding them in the function
     # graph. "Default" placements come from the PartitionedCallOp's placement,
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index c7654642d0..7dae203f8a 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3188,7 +3188,7 @@ def function(inputs, outputs, updates=None, name=None, **kwargs):
   Raises:
       ValueError: if invalid kwargs are passed in or if in eager execution.
   """
-  if context.executing_eagerly():
+  if ops.executing_eagerly_outside_functions():
     if kwargs:
       raise ValueError('Session keyword arguments are not support during '
                        'eager execution. You passed: %s' % (kwargs,))
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index c587287ff9..d100182381 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -54,14 +54,18 @@ def set_weights(distribution_strategy, dist_model, weights):
     num_param = len(layer.weights)
     layer_weights = weights[:num_param]
     for sw, w in zip(layer.weights, layer_weights):
-      assign_ops.append(distribution_strategy.unwrap(sw.assign(w)))
-
+      if ops.executing_eagerly_outside_functions():
+        sw.assign(w)
+      else:
+        assign_ops.append(distribution_strategy.unwrap(sw.assign(w)))
     weights = weights[num_param:]
-  K.get_session().run(assign_ops)
+
+  if not ops.executing_eagerly_outside_functions():
+    K.get_session().run(assign_ops)
 
 
 def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs,
-                  grouped_updates, grouped_session_args,
+                  grouped_updates=None, grouped_session_args=None,
                   with_loss_tensor=False):
   """Unwrap and return the list of values contained in the PerDevice parameters.
 
@@ -103,20 +107,25 @@ def unwrap_values(distribution_strategy, grouped_inputs, grouped_outputs,
     all_outputs = flatten_perdevice_values(distribution_strategy,
                                            grouped_outputs)
 
-  all_updates = flatten_perdevice_values(distribution_strategy,
-                                         grouped_updates)
+  if grouped_updates:
+    all_updates = flatten_perdevice_values(distribution_strategy,
+                                           grouped_updates)
+  else:
+    all_updates = None
 
   all_session_args = {}
-  grouped_feed_dict = grouped_session_args.get('feed_dict')
-  if grouped_feed_dict:
-    all_session_args['feed_dict'] = flatten_perdevice_values(
-        distribution_strategy, grouped_feed_dict)
-
-  grouped_fetches = grouped_session_args.get('fetches')
-  if grouped_fetches:
-    all_session_args['fetches'] = flatten_perdevice_values(
-        distribution_strategy, grouped_fetches)
-
+  if grouped_session_args:
+    grouped_feed_dict = grouped_session_args.get('feed_dict')
+    if grouped_feed_dict:
+      all_session_args['feed_dict'] = flatten_perdevice_values(
+          distribution_strategy, grouped_feed_dict)
+
+    grouped_fetches = grouped_session_args.get('fetches')
+    if grouped_fetches:
+      all_session_args['fetches'] = flatten_perdevice_values(
+          distribution_strategy, grouped_fetches)
+
+  # TODO(priyag): Return only non empty/None values
   return all_inputs, all_outputs, all_updates, all_session_args
 
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1094e549d7..a66d8866bd 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -500,9 +500,6 @@ class Model(Network):
         raise NotImplementedError(
             'optimizer must be an instance of '
             'tf.train.Optimizer, not a %s' % type(optimizer))
-      if self.run_eagerly:
-        raise NotImplementedError('DistributionStrategy is not supported '
-                                  'when running a model eagerly.')
       if sample_weight_mode:
         raise NotImplementedError('sample_weight_mode is not supported with '
                                   'DistributionStrategy.')
@@ -1064,7 +1061,9 @@ class Model(Network):
 
     with self._distribution_strategy.scope():
       iterator = self._distribution_strategy.make_dataset_iterator(x)
-      K.get_session().run(iterator.initialize())
+      init_op = iterator.initialize()
+      if not context.executing_eagerly():
+        K.get_session().run(init_op)
 
     training_utils.validate_iterator_input(x, y, sample_weight,
                                            validation_split)
@@ -1795,7 +1794,8 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps)
-    elif isinstance(x, iterator_ops.EagerIterator):
+    elif (isinstance(x, iterator_ops.EagerIterator) and
+          not self._distribution_strategy):
       return training_generator.fit_generator(
           self,
           x,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 49050f0a97..7cf961b9ec 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -19,10 +19,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import enum
+import enum  # pylint: disable=g-bad-import-order
 import numpy as np
 
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -36,7 +38,6 @@ from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.util import nest
 
 
@@ -540,7 +541,7 @@ def _clone_and_build_model(model, inputs=None, targets=None, mode=None):
 def clone_model_on_replicas(model, strategy, make_callback_model=False,
                             inputs=None, targets=None, mode=None):
   """Create a cloned model on each replica."""
-  with strategy.scope():
+  with K.get_graph().as_default(), strategy.scope():
     grouped_model = strategy.extended.call_for_each_replica(
         _clone_and_build_model, args=(model, inputs, targets, mode))
     if mode is _Mode.TRAIN:
@@ -583,6 +584,9 @@ def _get_input_from_iterator(iterator, model):
 
 def _get_execution_function(model, mode):
   """Get function to run one step of distributed model execution."""
+  if context.executing_eagerly():
+    return _get_eager_execution_function(model, mode)
+
   strategy = model._distribution_strategy
   if not model._grouped_model:
     clone_model_on_replicas(
@@ -627,6 +631,40 @@ def _get_execution_function(model, mode):
         **all_session_args)
 
 
+def _get_eager_execution_function(model, mode):
+  """Get function to run one step of distributed model eager execution."""
+  strategy = model._distribution_strategy
+  if not model._grouped_model:
+    clone_model_on_replicas(
+        model, strategy, make_callback_model=(mode == 'train'))
+
+  def _per_device_function(model):
+    f = model._get_execution_function(mode)
+    return (f.inputs, f.outputs)
+
+  # NOTE(priyag): Try creating a new FuncGraph within DS scope instead of using
+  # the global one.
+  with K.get_graph().as_default(), strategy.scope():
+    # Create train ops on each of the devices when we call
+    # `_per_device_fit_function`.
+    (grouped_inputs, grouped_outputs) = strategy.call_for_each_replica(
+        _per_device_function, args=(model._grouped_model,))
+
+    # Unwrap all the per device values returned from `call_for_each_replica`.
+    # Unwrapping per device values gives you a list of values that can be
+    # used to construct a new train function that is composed of inptus/outputs
+    # on all the devices over which the model is distributed.
+    (all_inputs, all_outputs, _, _) = distributed_training_utils.unwrap_values(
+        strategy,
+        grouped_inputs,
+        grouped_outputs)
+
+    return K.function(
+        all_inputs,
+        all_outputs,
+        name='eager_distributed_{}_function'.format(mode))
+
+
 def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
   """Prepare feed values to the model execution function.
 
@@ -653,7 +691,7 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
         None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
     ]
   ins = inputs + targets + sample_weights
-  if mode == 'train' and not isinstance(K.learning_phase(), int):
+  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
     ins += [True]
   return ins
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 0101ea8b97..9c8fff0fe4 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -329,7 +329,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       for grad, var in grads_and_vars:
         scope_name = ("" if ops.executing_eagerly_outside_functions() else
                       "_" + var.op.name)
-        with ops.name_scope("update" + scope_name), ops.colocate_with(var):
+        with ops.name_scope("update" + scope_name):
           update_ops.append(update_grad_to_var(grad, var))
       # control dependencies does not work in per replica mode, please change
       # this once b/118841692 is fixed.
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 900afeed6c..bf9a79660b 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -201,8 +201,7 @@ def _get_processor(v):
       return _TensorProcessor(v)
     else:
       return _DenseResourceVariableProcessor(v)
-  if isinstance(
-      v, resource_variable_ops.ResourceVariable) and not v._in_graph_mode:  # pylint: disable=protected-access
+  if resource_variable_ops.is_resource_variable(v) and not v._in_graph_mode:  # pylint: disable=protected-access
     # True if and only if `v` was initialized eagerly.
     return _DenseResourceVariableProcessor(v)
   if v.op.type == "VarHandleOp":
@@ -682,7 +681,13 @@ class Optimizer(
             "Gradient must be a Tensor, IndexedSlices, or None: %s" % g)
       p = _get_processor(v)
 
-      scope_name = "" if context.executing_eagerly() else v.op.name
+      if context.executing_eagerly() or (
+          resource_variable_ops.is_resource_variable(v) and
+          not v._in_graph_mode):  # pylint: disable=protected-access
+        scope_name = v.name.split(":")[0]
+      else:
+        scope_name = v.op.name
+
       # device_policy is set because non-mirrored tensors will be read in
       # `update_op`. `_resource_apply_dense`, `lr_t`, `beta1_t` and `beta2_t`
       # is an example.
-- 
GitLab


From 814a6c24e4dfcf220a3b9ea429ad88b07fa81818 Mon Sep 17 00:00:00 2001
From: wenxizhu <wenxi.zhu@intel.com>
Date: Fri, 30 Nov 2018 09:24:23 +0800
Subject: [PATCH 1083/1554] Fix format for clang-format check.

---
 tensorflow/core/graph/mkl_layout_pass.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index edea296c3b..1c1a6ce652 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2700,8 +2700,8 @@ Status MklLayoutRewritePass::FuseTransposeMklOpTranspose(
   for (const Edge* e : transpose_to_nchw->out_edges()) {
     if (!e->IsControlEdge()) {
       const int kTransposeWithMklOpOutputSlot = 0;
-      DCHECK((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot,
-                                  e->dst(), e->dst_input()));
+      DCHECK((*g)->AddEdge(new_node, kTransposeWithMklOpOutputSlot, e->dst(),
+                           e->dst_input()));
     }
   }
 
-- 
GitLab


From 341376d20fbfc392431d72cb6a40192bc368bac9 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 29 Nov 2018 17:17:59 -0800
Subject: [PATCH 1084/1554] Internal change.

PiperOrigin-RevId: 223440182
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 2 +-
 tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 32438260b4..5990caa50c 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -58,7 +58,7 @@ PY_TEST_DIR="py_test_dir"
 SKIP_TEST=0
 RELEASE_BUILD=0
 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
-EXTRA_BUILD_FLAGS=${ADDITIONAL_BUILD_ARGS:-}
+EXTRA_BUILD_FLAGS=${EXTRA_BUILD_FLAGS:-}
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 6a701fc5a1..1bca2a6f88 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -58,7 +58,7 @@ PY_TEST_DIR="py_test_dir"
 SKIP_TEST=0
 RELEASE_BUILD=0
 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
-EXTRA_BUILD_FLAGS=${ADDITIONAL_BUILD_ARGS:-}
+EXTRA_BUILD_FLAGS=${EXTRA_BUILD_FLAGS:-}
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
-- 
GitLab


From a386016a3b2fd58631152fbfec4907f48b09ad49 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 29 Nov 2018 17:18:46 -0800
Subject: [PATCH 1085/1554] Fix @tf.function signatures with unspecified
 default arguments

Surprisingly not much impact on the signature+call benchmarks.

Before:

MicroBenchmarks.benchmark_defun_with_signature 8423 examples/sec
MicroBenchmarks.benchmark_defun_with_signature_and_kwargs 7135 examples/sec

After

MicroBenchmarks.benchmark_defun_with_signature 8277 examples/sec
MicroBenchmarks.benchmark_defun_with_signature_and_kwargs 7032 examples/sec

PiperOrigin-RevId: 223440258
---
 tensorflow/python/eager/def_function_test.py |  6 ++++++
 tensorflow/python/eager/function.py          | 11 ++++++++---
 tensorflow/python/eager/function_test.py     |  5 +++--
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index da85735c47..4100a10044 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -214,6 +214,12 @@ class DefFunctionTest(test.TestCase):
         def_function.function(functools.partial(lambda x, y: x + y, 1.))(
             constant_op.constant(2.)))
 
+  def test_unspecified_default_argument(self):
+    wrapped = def_function.function(
+        lambda x, y=2: x + y,
+        input_signature=[tensor_spec.TensorSpec((), dtypes.int32)])
+    self.assertEqual(3, wrapped(constant_op.constant(1)).numpy())
+
   def test_optimizer(self):
     x = constant_op.constant([[3., 4.]])
     y = constant_op.constant([2.])
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index d9c16aa7b8..9d05a660b1 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1088,16 +1088,21 @@ class PolymorphicFunction(object):
       return inputs, kwargs
     else:
       assert not kwargs
+      signature_relevant_inputs = inputs[:len(self._input_signature)]
       try:
-        nest.assert_same_structure(self._input_signature, inputs)
+        nest.assert_same_structure(self._input_signature,
+                                   signature_relevant_inputs)
       except (ValueError, TypeError):
         raise ValueError("Structure of Python function inputs does not match "
                          "input_signature.")
-      if any(not pywrap_tensorflow.IsTensor(arg) for arg in flat_inputs):
+      signature_inputs_flat = nest.flatten(signature_relevant_inputs)
+      if any(not pywrap_tensorflow.IsTensor(arg)
+             for arg in signature_inputs_flat):
         raise ValueError("When input_signature is provided, all inputs to "
                          "the Python function must be Tensors.")
       if any(not spec.is_compatible_with(other)
-             for spec, other in zip(self._flat_input_signature, flat_inputs)):
+             for spec, other in zip(self._flat_input_signature,
+                                    signature_inputs_flat)):
         raise ValueError("Python inputs incompatible with input_signature: "
                          "inputs (%s), input_signature (%s)" %
                          (str(inputs), str(self._input_signature)))
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index d2cd407d80..2f3eff6d6a 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1289,8 +1289,9 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       defined(array_ops.ones([2, 1]))
 
     # Wrong number of arguments.
-    with self.assertRaisesRegexp(ValueError,
-                                 'Structure of Python function inputs.*'):
+    with self.assertRaisesRegexp(
+        ValueError,
+        'Arguments and signature arguments do not match.*'):
       defined(array_ops.ones([2]), array_ops.ones([2]))
     with self.assertRaisesRegexp(ValueError,
                                  'Structure of Python function inputs.*'):
-- 
GitLab


From 48714769a1df93198fecddb39e809871b89e210b Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 29 Nov 2018 17:20:04 -0800
Subject: [PATCH 1086/1554] Register ops that can update/add/sub tensors.

PiperOrigin-RevId: 223440399
---
 .../api_def/base_api/api_def_ScatterNd.pbtxt  |   4 +
 .../base_api/api_def_TensorScatterAdd.pbtxt   |  94 ++++++++++
 .../base_api/api_def_TensorScatterSub.pbtxt   |  94 ++++++++++
 .../api_def_TensorScatterUpdate.pbtxt         | 106 +++++++++++
 tensorflow/core/kernels/BUILD                 |   1 +
 tensorflow/core/kernels/scatter_nd_op.cc      | 164 ++++++++++++++++++
 tensorflow/core/ops/array_ops.cc              |  58 ++++++-
 .../kernel_tests/scatter_nd_ops_test.py       |  51 ++++++
 tensorflow/python/ops/array_grad.py           |  26 +++
 .../tools/api/golden/v1/tensorflow.pbtxt      |  12 ++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  12 ++
 11 files changed, 614 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorScatterAdd.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorScatterSub.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorScatterUpdate.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt
index 0b5917d428..41955cfbfa 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt
@@ -32,6 +32,10 @@ slices within a tensor (initially zero for numeric, empty for string) of
 the given `shape` according to indices.  This operator is the inverse of the
 `tf.gather_nd` operator which extracts values or slices from a given tensor.
 
+This operation is similar to tensor_scatter_add, except that the tensor is
+zero-initialized. Calling `tf.scatter_nd(indices, values, shape)` is identical
+to `tensor_scatter_add(tf.zeros(shape, values.dtype), indices, values)`
+
 If `indices` contains duplicates, then their updates are accumulated (summed).
 
 **WARNING**: The order in which updates are applied is nondeterministic, so the
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorScatterAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorScatterAdd.pbtxt
new file mode 100644
index 0000000000..1634e51c3c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorScatterAdd.pbtxt
@@ -0,0 +1,94 @@
+op {
+  graph_op_name: "TensorScatterAdd"
+  in_arg {
+    name: "tensor"
+    description: <<END
+Tensor to copy/update.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+Index tensor.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+Updates to scatter into output.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+A new tensor copied from tensor and updates added according to the indices.
+END
+  }
+  summary: "Adds sparse `updates` to an existing tensor according to `indices`."
+  description: <<END
+This operation creates a new tensor by adding sparse `updates` to the passed
+in `tensor`.
+This operation is very similar to `tf.scatter_nd_add`, except that the updates
+are added onto an existing tensor (as opposed to a variable). If the memory
+for the existing tensor cannot be re-used, a copy is made and updated.
+
+`indices` is an integer tensor containing indices into a new tensor of shape
+`shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+
+    indices.shape[-1] <= shape.rank
+
+The last dimension of `indices` corresponds to indices into elements
+(if `indices.shape[-1] = shape.rank`) or slices
+(if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+`shape`.  `updates` is a tensor with shape
+
+    indices.shape[:-1] + shape[indices.shape[-1]:]
+
+The simplest form of tensor_scatter_add is to add individual elements to a
+tensor by index. For example, say we want to add 4 elements in a rank-1
+tensor with 8 elements.
+
+In Python, this scatter add operation would look like this:
+
+```python
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    tensor = tf.ones([8], dtype=tf.int32)
+    updated = tf.tensor_scatter_add(tensor, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [1, 12, 1, 11, 10, 1, 1, 13]
+
+We can also, insert entire slices of a higher rank tensor all at once. For
+example, if we wanted to insert two slices in the first dimension of a
+rank-3 tensor with two matrices of new values.
+
+In Python, this scatter add operation would look like this:
+
+```python
+    indices = tf.constant([[0], [2]])
+    updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]],
+                           [[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]]])
+    tensor = tf.ones([4, 4, 4])
+    updated = tf.tensor_scatter_add(tensor, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [[[6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8], [9, 9, 9, 9]],
+     [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+     [[6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8], [9, 9, 9, 9]],
+     [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+
+Note that on CPU, if an out of bound index is found, an error is returned.
+On GPU, if an out of bound index is found, the index is ignored.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorScatterSub.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorScatterSub.pbtxt
new file mode 100644
index 0000000000..851628bce1
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorScatterSub.pbtxt
@@ -0,0 +1,94 @@
+op {
+  graph_op_name: "TensorScatterSub"
+  in_arg {
+    name: "tensor"
+    description: <<END
+Tensor to copy/update.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+Index tensor.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+Updates to scatter into output.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+A new tensor copied from tensor and updates subtracted according to the indices.
+END
+  }
+  summary: "Subtracts sparse `updates` from an existing tensor according to `indices`."
+  description: <<END
+This operation creates a new tensor by subtracting sparse `updates` from the
+passed in `tensor`.
+This operation is very similar to `tf.scatter_nd_sub`, except that the updates
+are subtracted from an existing tensor (as opposed to a variable). If the memory
+for the existing tensor cannot be re-used, a copy is made and updated.
+
+`indices` is an integer tensor containing indices into a new tensor of shape
+`shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+
+    indices.shape[-1] <= shape.rank
+
+The last dimension of `indices` corresponds to indices into elements
+(if `indices.shape[-1] = shape.rank`) or slices
+(if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+`shape`.  `updates` is a tensor with shape
+
+    indices.shape[:-1] + shape[indices.shape[-1]:]
+
+The simplest form of tensor_scatter_sub is to subtract individual elements
+from a tensor by index. For example, say we want to insert 4 scattered elements
+in a rank-1 tensor with 8 elements.
+
+In Python, this scatter subtract operation would look like this:
+
+```python
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    tensor = tf.ones([8], dtype=tf.int32)
+    updated = tf.tensor_scatter_sub(tensor, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [1, -10, 1, -9, -8, 1, 1, -11]
+
+We can also, insert entire slices of a higher rank tensor all at once. For
+example, if we wanted to insert two slices in the first dimension of a
+rank-3 tensor with two matrices of new values.
+
+In Python, this scatter add operation would look like this:
+
+```python
+    indices = tf.constant([[0], [2]])
+    updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]],
+                           [[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]]])
+    tensor = tf.ones([4, 4, 4])
+    updated = tf.tensor_scatter_sub(tensor, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [[[-4, -4, -4, -4], [-5, -5, -5, -5], [-6, -6, -6, -6], [-7, -7, -7, -7]],
+     [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+     [[-4, -4, -4, -4], [-5, -5, -5, -5], [-6, -6, -6, -6], [-7, -7, -7, -7]],
+     [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+
+Note that on CPU, if an out of bound index is found, an error is returned.
+On GPU, if an out of bound index is found, the index is ignored.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorScatterUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorScatterUpdate.pbtxt
new file mode 100644
index 0000000000..5a6ed1e1ce
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorScatterUpdate.pbtxt
@@ -0,0 +1,106 @@
+op {
+  graph_op_name: "TensorScatterUpdate"
+  in_arg {
+    name: "tensor"
+    description: <<END
+Tensor to copy/update.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+Index tensor.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+Updates to scatter into output.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+A new tensor with the given shape and updates applied according
+to the indices.
+END
+  }
+  summary: "Scatter `updates` into an existing tensor according to `indices`."
+  description: <<END
+This operation creates a new tensor by applying sparse `updates` to the passed
+in `tensor`.
+This operation is very similar to `tf.scatter_nd`, except that the updates are
+scattered onto an existing tensor (as opposed to a zero-tensor). If the memory
+for the existing tensor cannot be re-used, a copy is made and updated.
+
+If `indices` contains duplicates, then their updates are accumulated (summed).
+
+**WARNING**: The order in which updates are applied is nondeterministic, so the
+output will be nondeterministic if `indices` contains duplicates -- because
+of some numerical approximation issues, numbers summed in different order
+may yield different results.
+
+`indices` is an integer tensor containing indices into a new tensor of shape
+`shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+
+    indices.shape[-1] <= shape.rank
+
+The last dimension of `indices` corresponds to indices into elements
+(if `indices.shape[-1] = shape.rank`) or slices
+(if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+`shape`.  `updates` is a tensor with shape
+
+    indices.shape[:-1] + shape[indices.shape[-1]:]
+
+The simplest form of scatter is to insert individual elements in a tensor by
+index. For example, say we want to insert 4 scattered elements in a rank-1
+tensor with 8 elements.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+</div>
+
+In Python, this scatter operation would look like this:
+
+```python
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    tensor = tf.ones([8], dtype=tf.int32)
+    updated = tf.tensor_scatter_update(tensor, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [1, 11, 1, 10, 9, 1, 1, 12]
+
+We can also, insert entire slices of a higher rank tensor all at once. For
+example, if we wanted to insert two slices in the first dimension of a
+rank-3 tensor with two matrices of new values.
+
+In Python, this scatter operation would look like this:
+
+```python
+    indices = tf.constant([[0], [2]])
+    updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]],
+                           [[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]]])
+    tensor = tf.ones([4, 4, 4])
+    updated = tf.tensor_scatter_update(tensor, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+     [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+     [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+     [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+
+Note that on CPU, if an out of bound index is found, an error is returned.
+On GPU, if an out of bound index is found, the index is ignored.
+END
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 986b21045d..be21d403e4 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4589,6 +4589,7 @@ tf_kernel_library(
         ":dense_update_functor",
         ":training_op_helpers",
         ":variable_ops",
+        ":inplace_ops",
     ],
 )
 
diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index fd54c6d6d7..63bb793fdc 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/fill_functor.h"
+#include "tensorflow/core/kernels/inplace_ops_functor.h"
 #include "tensorflow/core/kernels/training_op_helpers.h"
 #include "tensorflow/core/kernels/variable_ops.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -121,6 +122,90 @@ class ScatterNdOp : public OpKernel {
   }
 };
 
+template <typename Device, typename T, typename Index,
+          scatter_nd_op::UpdateOp op>
+class TensorScatterOp : public OpKernel {
+ public:
+  explicit TensorScatterOp(OpKernelConstruction* c) : OpKernel(c) {
+    const DataType dt = DataTypeToEnum<T>::v();
+    const DataType index_t = DataTypeToEnum<Index>::v();
+    OP_REQUIRES_OK(c, c->MatchSignature({dt, index_t, dt}, {dt}));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    const Tensor& input = c->input(0);
+    const Tensor& indices = c->input(1);
+    const Tensor& updates = c->input(2);
+
+    OP_REQUIRES(c, indices.shape().dims() >= 1,
+                errors::InvalidArgument(
+                    "Indices shape must have rank at least one. Found:",
+                    indices.shape().DebugString()));
+    OP_REQUIRES(c, updates.shape().dims() >= 1,
+                errors::InvalidArgument(
+                    "Updates shape must have rank at least one. Found:",
+                    updates.shape().DebugString()));
+
+    TensorShape shape = input.shape();
+
+    OP_REQUIRES(
+        c,
+        (shape.num_elements() > 0 || (indices.shape().num_elements() == 0 &&
+                                      updates.shape().num_elements() == 0)),
+        errors::InvalidArgument(
+            "Indices and updates specified for empty output shape"));
+
+    const int64 outer_dims = indices.shape().dims() - 1;
+
+    for (int i = 0; i < outer_dims; ++i) {
+      OP_REQUIRES(c, indices.shape().dim_size(i) == updates.shape().dim_size(i),
+                  errors::InvalidArgument(
+                      "Outer dimensions of indices and update must match. "
+                      "Indices shape: ",
+                      indices.shape().DebugString(),
+                      ", updates shape:", updates.shape().DebugString()));
+    }
+
+    const int64 ix = indices.shape().dim_size(outer_dims);
+    OP_REQUIRES(
+        c, updates.shape().dims() - outer_dims == shape.dims() - ix,
+        errors::InvalidArgument("Inner dimensions of output shape must match "
+                                "inner dimensions of updates shape. Output: ",
+                                shape.DebugString(),
+                                " updates: ", updates.shape().DebugString()));
+    for (int i = 0; i + outer_dims < updates.shape().dims(); ++i) {
+      OP_REQUIRES(
+          c, updates.shape().dim_size(i + outer_dims) == shape.dim_size(ix + i),
+          errors::InvalidArgument(
+              "The inner ", shape.dims() - ix,
+              " dimensions of output.shape=", shape.DebugString(),
+              " must match the inner ", updates.shape().dims() - outer_dims,
+              " dimensions of updates.shape=", updates.shape().DebugString()));
+    }
+
+    std::unique_ptr<Tensor> forwarded_input = c->forward_input(
+        2, 0, input.dtype(), shape, DEVICE_MEMORY, AllocatorAttributes());
+
+    if (forwarded_input == nullptr) {
+      // We were not able to forward the input, so we deep copy the tensor and
+      // set the output.
+      Tensor* out;
+      OP_REQUIRES_OK(c, c->allocate_output(0, input.shape(), &out));
+
+      OP_REQUIRES_OK(c, tensorflow::functor::DoCopy(c->eigen_device<Device>(),
+                                                    input, out));
+      OP_REQUIRES_OK(c,
+                     functor::DoScatterNd<Device, T, Index, op>(
+                         c, indices, updates, shape, out, false /*allocate*/));
+    } else {
+      // Output forwarded, so simply perform the scatter.
+      OP_REQUIRES_OK(c, functor::DoScatterNd<Device, T, Index, op>(
+                            c, indices, updates, shape, forwarded_input.get(),
+                            false /*allocate*/));
+    }
+  }
+};
+
 template <typename Device, typename T, typename Index,
           scatter_nd_op::UpdateOp op>
 class ScatterNdUpdateOp : public OpKernel {
@@ -282,6 +367,56 @@ TF_CALL_bool(REGISTER_SCATTER_ND_ADD_SUB_CPU);
 TF_CALL_bool(REGISTER_SCATTER_ND_UPDATE_CPU);
 TF_CALL_bool(REGISTER_SCATTER_ND_CPU);
 
+#define REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, index_type, \
+                                                          dev)              \
+  REGISTER_KERNEL_BUILDER(Name("TensorScatterUpdate")                       \
+                              .Device(DEVICE_##dev)                         \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<index_type>("Tindices"),      \
+                          TensorScatterOp<dev##Device, type, index_type,    \
+                                          scatter_nd_op::UpdateOp::ASSIGN>)
+
+#define REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, index_type, dev) \
+  REGISTER_KERNEL_BUILDER(Name("TensorScatterAdd")                            \
+                              .Device(DEVICE_##dev)                           \
+                              .TypeConstraint<type>("T")                      \
+                              .TypeConstraint<index_type>("Tindices"),        \
+                          TensorScatterOp<dev##Device, type, index_type,      \
+                                          scatter_nd_op::UpdateOp::ADD>)
+
+#define REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, index_type, dev) \
+  REGISTER_KERNEL_BUILDER(Name("TensorScatterSub")                            \
+                              .Device(DEVICE_##dev)                           \
+                              .TypeConstraint<type>("T")                      \
+                              .TypeConstraint<index_type>("Tindices"),        \
+                          TensorScatterOp<dev##Device, type, index_type,      \
+                                          scatter_nd_op::UpdateOp::SUB>)
+
+#define REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU(type)                    \
+  REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int32, CPU); \
+  REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int64, CPU);
+
+#define REGISTER_SCATTER_ND_TENSOR_ADD_CPU(type)                    \
+  REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int32, CPU); \
+  REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int64, CPU);
+
+#define REGISTER_SCATTER_ND_TENSOR_SUB_CPU(type)                    \
+  REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int32, CPU); \
+  REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int64, CPU);
+
+#define REGISTER_SCATTER_ND_TENSOR_CPU(type)   \
+  REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU(type); \
+  REGISTER_SCATTER_ND_TENSOR_ADD_CPU(type);    \
+  REGISTER_SCATTER_ND_TENSOR_SUB_CPU(type);
+
+// Register TensorScatterUpdate/Add/Sub for all number types.
+TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_TENSOR_CPU);
+// Register only TensorScatterUpdate for string/bool types as well.
+TF_CALL_string(REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU);
+TF_CALL_bool(REGISTER_SCATTER_ND_TENSOR_UPDATE_CPU);
+
+#undef REGISTER_SCATTER_ND_TENSOR_CPU
+
 // Registers GPU kernels.
 #if GOOGLE_CUDA
 
@@ -319,6 +454,25 @@ TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_UPDATE_SYCL);
 #undef REGISTER_SCATTER_ND_UPDATE_SYCL
 #endif  // TENSORFLOW_USE_SYCL
 
+#define REGISTER_SCATTER_ND_TENSOR_UPDATE_GPU(type)                    \
+  REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int32, GPU); \
+  REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE(type, int64, GPU);
+
+#define REGISTER_SCATTER_ND_TENSOR_ADD_GPU(type)                    \
+  REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int32, GPU); \
+  REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE(type, int64, GPU);
+
+#define REGISTER_SCATTER_ND_TENSOR_SUB_GPU(type)                    \
+  REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int32, GPU); \
+  REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE(type, int64, GPU);
+
+#define REGISTER_SCATTER_ND_TENSOR_GPU(type)   \
+  REGISTER_SCATTER_ND_TENSOR_ADD_GPU(type);    \
+  REGISTER_SCATTER_ND_TENSOR_UPDATE_GPU(type); \
+  REGISTER_SCATTER_ND_TENSOR_SUB_GPU(type);
+
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_TENSOR_GPU);
+
 #undef REGISTER_SCATTER_ND_ADD
 #undef REGISTER_SCATTER_ND_ADD_SUB
 #undef REGISTER_SCATTER_ND_ADD_SUB_CPU
@@ -328,6 +482,16 @@ TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_UPDATE_SYCL);
 #undef REGISTER_SCATTER_ND_UPDATE_GPU
 #undef REGISTER_SCATTER_ND_KERNEL
 #undef REGISTER_SCATTER_ND_KERNEL_INDEX
+#undef REGISTER_SCATTER_ND_TENSOR_TYPE_INDEX_TYPE
+#undef REGISTER_SCATTER_ND_TENSOR_CPU
+#undef REGISTER_SCATTER_ND_TENSOR_GPU
+#undef REGISTER_SCATTER_ND_TENSOR_UPDATE_TYPE_INDEX_TYPE
+#undef REGISTER_SCATTER_ND_TENSOR_ADD_TYPE_INDEX_TYPE
+#undef REGISTER_SCATTER_ND_TENSOR_SUB_TYPE_INDEX_TYPE
+#undef REGISTER_SCATTER_ND_TENSOR_UPDATE_GPU
+#undef REGISTER_SCATTER_ND_TENSOR_ADD_GPU
+#undef REGISTER_SCATTER_ND_TENSOR_SUB_GPU
+#undef REGISTER_SCATTER_ND_TENSOR_GPU
 
 #endif  // GOOGLE_CUDA
 
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index e07a35a63d..281e2996ed 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -2881,14 +2881,9 @@ REGISTER_OP("QuantizedInstanceNorm")
 
 namespace {
 
-Status ScatterNdShape(InferenceContext* c) {
-  ShapeHandle indices_shape;
-  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &indices_shape));
-  ShapeHandle updates_shape;
-  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &updates_shape));
-  ShapeHandle output_shape;
-  TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &output_shape));
-
+Status ScatterNdShapeHelper(InferenceContext* c, ShapeHandle indices_shape,
+                            ShapeHandle updates_shape,
+                            ShapeHandle output_shape) {
   if (c->Value(c->NumElements(output_shape)) == 0 &&
       (c->Value(c->NumElements(indices_shape)) > 0 ||
        c->Value(c->NumElements(updates_shape)) > 0)) {
@@ -2943,6 +2938,26 @@ Status ScatterNdShape(InferenceContext* c) {
   return Status::OK();
 }
 
+Status ScatterNdShape(InferenceContext* c) {
+  ShapeHandle indices_shape;
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &indices_shape));
+  ShapeHandle updates_shape;
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &updates_shape));
+  ShapeHandle output_shape;
+  TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &output_shape));
+  return ScatterNdShapeHelper(c, indices_shape, updates_shape, output_shape);
+}
+
+Status ScatterNdTensorShape(InferenceContext* c) {
+  ShapeHandle output_shape;
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &output_shape));
+  ShapeHandle indices_shape;
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &indices_shape));
+  ShapeHandle updates_shape;
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(2), 1, &updates_shape));
+  return ScatterNdShapeHelper(c, indices_shape, updates_shape, output_shape);
+}
+
 }  // namespace
 
 REGISTER_OP("UpperBound")
@@ -2982,6 +2997,33 @@ REGISTER_OP("ScatterNd")
     .Attr("Tindices: {int32, int64}")
     .SetShapeFn(ScatterNdShape);
 
+REGISTER_OP("TensorScatterUpdate")
+    .Input("tensor: T")
+    .Input("indices: Tindices")
+    .Input("updates: T")
+    .Output("output: T")
+    .Attr("T: type")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ScatterNdTensorShape);
+
+REGISTER_OP("TensorScatterAdd")
+    .Input("tensor: T")
+    .Input("indices: Tindices")
+    .Input("updates: T")
+    .Output("output: T")
+    .Attr("T: type")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ScatterNdTensorShape);
+
+REGISTER_OP("TensorScatterSub")
+    .Input("tensor: T")
+    .Input("indices: Tindices")
+    .Input("updates: T")
+    .Output("output: T")
+    .Attr("T: type")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ScatterNdTensorShape);
+
 REGISTER_OP("ScatterNdNonAliasingAdd")
     .Input("input: T")
     .Input("indices: Tindices")
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index 298db1547a..c1241ba87e 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
@@ -698,5 +699,55 @@ class ScatterNdNonAliasingAddTest(ScatterNdTest):
     pass
 
 
+class ScatterNdTensorTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testUpdateAddSub(self):
+    indices = constant_op.constant([[4], [3], [1], [7]])
+    updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32)
+    t = array_ops.ones([8], dtype=dtypes.float32)
+    assigned = array_ops.tensor_scatter_update(t, indices, updates)
+    added = array_ops.tensor_scatter_add(t, indices, updates)
+    subbed = array_ops.tensor_scatter_sub(t, indices, updates)
+
+    self.assertAllEqual(assigned,
+                        constant_op.constant([1, 11, 1, 10, 9, 1, 1, 12]))
+    self.assertAllEqual(added,
+                        constant_op.constant([1, 12, 1, 11, 10, 1, 1, 13]))
+    self.assertAllEqual(subbed,
+                        constant_op.constant([1, -10, 1, -9, -8, 1, 1, -11]))
+
+  def testUpdateAddSubGradients(self):
+
+    with self.cached_session():
+      indices = constant_op.constant([[3], [1]])
+      updates = constant_op.constant([9, 10], dtype=dtypes.float32)
+      x = array_ops.ones([4], dtype=dtypes.float32)
+
+      assigned = array_ops.tensor_scatter_update(x, indices, updates)
+      added = array_ops.tensor_scatter_add(x, indices, updates)
+      subbed = array_ops.tensor_scatter_sub(x, indices, updates)
+
+      err_assigned = gradient_checker.compute_gradient_error(
+          x, [4], assigned, [4])
+      err_added = gradient_checker.compute_gradient_error(x, [4], added, [4])
+      err_subbed = gradient_checker.compute_gradient_error(x, [4], subbed, [4])
+
+      self.assertLess(err_assigned, 2e-4)
+      self.assertLess(err_added, 2e-4)
+      self.assertLess(err_subbed, 2e-4)
+
+      err_assigned_wrt_updates = gradient_checker.compute_gradient_error(
+          updates, [2], assigned, [4])
+      err_added_wrt_updates = gradient_checker.compute_gradient_error(
+          updates, [2], added, [4])
+      err_subbed_wrt_updates = gradient_checker.compute_gradient_error(
+          updates, [2], subbed, [4])
+
+      self.assertLess(err_assigned_wrt_updates, 2e-4)
+      self.assertLess(err_added_wrt_updates, 2e-4)
+      self.assertLess(err_subbed_wrt_updates, 2e-4)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 6edc193361..45e741ef22 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -802,6 +802,32 @@ def _ScatterNdGrad(op, grad):
   return [None, updates_grad, None]
 
 
+@ops.RegisterGradient("TensorScatterUpdate")
+def _TensorScatterUpdateGrad(op, grad):
+  indices = op.inputs[1]
+  updates_grad = array_ops.gather_nd(grad, indices)
+  tensor_grad = array_ops.tensor_scatter_update(
+      array_ops.identity(grad), indices,
+      array_ops.zeros_like(op.inputs[2], dtype=grad.dtype))
+  return [tensor_grad, None, updates_grad]
+
+
+@ops.RegisterGradient("TensorScatterAdd")
+def _TensorScatterAddGrad(op, grad):
+  indices = op.inputs[1]
+  updates_grad = array_ops.gather_nd(grad, indices)
+  tensor_grad = array_ops.identity(grad)
+  return [tensor_grad, None, updates_grad]
+
+
+@ops.RegisterGradient("TensorScatterSub")
+def _TensorScatterSubGrad(op, grad):
+  indices = op.inputs[1]
+  updates_grad = array_ops.gather_nd(grad, indices)
+  tensor_grad = array_ops.identity(grad)
+  return [tensor_grad, None, -updates_grad]
+
+
 @ops.RegisterGradient("ScatterNdNonAliasingAdd")
 def _ScatterNdNonAliasingAddGrad(op, grad):
   indices = op.inputs[1]
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 9c836c5bf8..490b4a847f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -2176,6 +2176,18 @@ tf_module {
     name: "tanh"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "tensor_scatter_add"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "tensor_scatter_sub"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "tensor_scatter_update"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "tensordot"
     argspec: "args=[\'a\', \'b\', \'axes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 2a30688b46..f5e1545657 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -1052,6 +1052,18 @@ tf_module {
     name: "tanh"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "tensor_scatter_add"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "tensor_scatter_sub"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "tensor_scatter_update"
+    argspec: "args=[\'tensor\', \'indices\', \'updates\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "tensordot"
     argspec: "args=[\'a\', \'b\', \'axes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 326d3eec59d054838cb6e05e44b6458cedc395a5 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Thu, 29 Nov 2018 18:11:03 -0800
Subject: [PATCH 1087/1554] Fixed issue with tflite_convert.

PiperOrigin-RevId: 223446248
---
 tensorflow/lite/python/convert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 756c9daabe..198612f6fe 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -335,7 +335,7 @@ def build_toco_convert_protos(input_tensors,
   model.change_concat_input_ranges = change_concat_input_ranges
   for idx, input_tensor in enumerate(input_tensors):
     input_array = model.input_arrays.add()
-    if toco.inference_input_type == lite_constants.QUANTIZED_UINT8:
+    if toco.inference_input_type == _types_pb2.QUANTIZED_UINT8:
       input_array.mean_value, input_array.std_value = quantized_input_stats[idx]
     input_array.name = tensor_name(input_tensor)
     if input_shapes is None:
-- 
GitLab


From 1b41f3573ee7dff801b33fe504335a60fa839abd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 18:23:18 -0800
Subject: [PATCH 1088/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 223447396
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 99 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 99 +++++++++++++++++++
 2 files changed, 198 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 414964ad3f..6f3db65918 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -76017,6 +76017,105 @@ op {
     }
   }
 }
+op {
+  name: "TensorScatterAdd"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "TensorScatterSub"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "TensorScatterUpdate"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorSliceDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index eae87da6b5..08aafc064e 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36431,6 +36431,105 @@ op {
     }
   }
 }
+op {
+  name: "TensorScatterAdd"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "TensorScatterSub"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "TensorScatterUpdate"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorSliceDataset"
   input_arg {
-- 
GitLab


From 3c6b48c7aa8d152439cc54f960e5b463b13093cd Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 29 Nov 2018 18:42:05 -0800
Subject: [PATCH 1089/1554] Bump TFLite iOS examples to use 1.12.0 CocoaPod

PiperOrigin-RevId: 223449051
---
 tensorflow/lite/examples/ios/camera/Podfile | 2 +-
 tensorflow/lite/examples/ios/simple/Podfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/examples/ios/camera/Podfile b/tensorflow/lite/examples/ios/camera/Podfile
index f460693122..96a0d23426 100644
--- a/tensorflow/lite/examples/ios/camera/Podfile
+++ b/tensorflow/lite/examples/ios/camera/Podfile
@@ -2,4 +2,4 @@ platform :ios, '8.0'
 inhibit_all_warnings!
 
 target 'tflite_camera_example'
-       pod 'TensorFlowLite', '1.10.1'
+       pod 'TensorFlowLite', '1.12.0'
diff --git a/tensorflow/lite/examples/ios/simple/Podfile b/tensorflow/lite/examples/ios/simple/Podfile
index ddb77088d9..931b72c1f5 100644
--- a/tensorflow/lite/examples/ios/simple/Podfile
+++ b/tensorflow/lite/examples/ios/simple/Podfile
@@ -2,4 +2,4 @@ platform :ios, '8.0'
 inhibit_all_warnings!
 
 target 'tflite_simple_example'
-       pod 'TensorFlowLite', '1.10.1'
+       pod 'TensorFlowLite', '1.12.0'
-- 
GitLab


From acbd910f8abf6b4b2f946eeac3bc77f3a6c832ce Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 29 Nov 2018 18:55:57 -0800
Subject: [PATCH 1090/1554] Disable runtime error in TF2 if colocation cannot
 be guaranteed

PiperOrigin-RevId: 223450181
---
 tensorflow/python/framework/importer.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index 71ebfd6ceb..98c7aeccc4 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -21,6 +21,7 @@ import contextlib
 
 from tensorflow.core.framework import graph_pb2
 from tensorflow.python import pywrap_tensorflow as c_api
+from tensorflow.python import tf2
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import errors
@@ -253,7 +254,9 @@ def _ProcessNewOps(graph):
     # Find any device in the list of colocated ops that have a device, if it
     # exists.  We assume that if multiple ops have devices, they refer to the
     # same device.  Otherwise, a runtime error will occur since the colocation
-    # property cannot be guaranteed.
+    # property cannot be guaranteed.  Note in TF2 colocations have been removed
+    # from the public API and will be considered a hint, so there is no runtime
+    # error.
     #
     # One possible improvement is to try to check for compatibility of all
     # devices in this list at import time here, which would require
@@ -262,6 +265,10 @@ def _ProcessNewOps(graph):
       try:
         coloc_op = graph._get_operation_by_name_unsafe(coloc_op_name)  # pylint: disable=protected-access
       except KeyError:
+        # Do not error in TF2 if the colocation cannot be guaranteed
+        if tf2.enabled():
+          continue
+
         raise ValueError('Specified colocation to an op that '
                          'does not exist during import: %s in %s' %
                          (coloc_op_name, op.name))
-- 
GitLab


From af4417be821c1e0e1af2d4324fd3e5cac2ac9887 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 29 Nov 2018 19:02:18 -0800
Subject: [PATCH 1091/1554] Avoid inspecting the decorators of a call's target
 function when the target is a lambda - they don't have decorators.

PiperOrigin-RevId: 223450702
---
 .../python/autograph/converters/call_trees.py      |  5 +++++
 .../python/autograph/converters/call_trees_test.py | 14 ++++++++++++++
 .../python/autograph/core/converter_testing.py     |  6 ++++--
 tensorflow/python/autograph/core/naming.py         |  5 ++---
 tensorflow/python/autograph/pyct/inspect_utils.py  |  8 ++++++++
 .../python/autograph/pyct/inspect_utils_test.py    |  7 +++++++
 6 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index 55cea89126..4a8af42e69 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -140,6 +140,11 @@ class CallTreeTransformer(converter.Base):
 
     if target_entity is not None:
 
+      # Currently, lambdas are always converted.
+      # TODO(mdan): Allow markers of the kind f = ag.do_not_convert(lambda: ...)
+      if inspect_utils.islambda(target_entity):
+        return True
+
       # This may be reached when "calling" a callable attribute of an object.
       # For example:
       #
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index 9d760167a9..b2a740020c 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -85,6 +85,20 @@ class CallTreesTest(converter_testing.TestCase):
       tc = TestClass()
       self.assertEquals(3, result.test_fn_2(tc, 1))
 
+  def test_known_called_lambda(self):
+
+    l = lambda x: x
+
+    def test_fn(a):
+      return l(a)
+
+    ns = {'l': l}
+    node, ctx = self.prepare(test_fn, ns)
+    node = call_trees.transform(node, ctx)
+
+    with self.compiled(node, ns) as result:
+      self.assertEquals(1, result.test_fn(1))
+
   def test_py_func_known_function(self):
 
     def test_fn():
diff --git a/tensorflow/python/autograph/core/converter_testing.py b/tensorflow/python/autograph/core/converter_testing.py
index 7b0608d03f..f1374081d3 100644
--- a/tensorflow/python/autograph/core/converter_testing.py
+++ b/tensorflow/python/autograph/core/converter_testing.py
@@ -32,6 +32,7 @@ from tensorflow.python.autograph.core import errors
 from tensorflow.python.autograph.core import function_wrapping
 from tensorflow.python.autograph.lang import special_functions
 from tensorflow.python.autograph.pyct import compiler
+from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import origin_info
 from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import pretty_printer
@@ -43,7 +44,7 @@ def imported_decorator(f):
   return lambda a: f(a) + 1
 
 
-# TODO(mdan): We might be able to use the real namer here.
+# TODO(mdan): We should use the real namer here.
 class FakeNamer(object):
   """A fake namer that uses a global counter to generate unique names."""
 
@@ -61,7 +62,8 @@ class FakeNamer(object):
                              original_fqn,
                              live_entity=None,
                              owner_type=None):
-    del live_entity
+    if inspect_utils.islambda(live_entity):
+      return None, False
     if owner_type is not None:
       return None, False
     return ('renamed_%s' % '_'.join(original_fqn)), True
diff --git a/tensorflow/python/autograph/core/naming.py b/tensorflow/python/autograph/core/naming.py
index 43fcbcfc03..b8d79daeba 100644
--- a/tensorflow/python/autograph/core/naming.py
+++ b/tensorflow/python/autograph/core/naming.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import qual_names
-from tensorflow.python.util import tf_inspect
 
 
 class Namer(object):
@@ -77,8 +77,7 @@ class Namer(object):
     if not self.recursive:
       return None, False
 
-    if (live_entity is not None and tf_inspect.isfunction(live_entity) and
-        live_entity.__name__ == '<lambda>'):
+    if (live_entity is not None and inspect_utils.islambda(live_entity)):
       return None, False
 
     if owner_type is not None and owner_type not in self.partial_types:
diff --git a/tensorflow/python/autograph/pyct/inspect_utils.py b/tensorflow/python/autograph/pyct/inspect_utils.py
index 4d56b93671..2319430d09 100644
--- a/tensorflow/python/autograph/pyct/inspect_utils.py
+++ b/tensorflow/python/autograph/pyct/inspect_utils.py
@@ -46,6 +46,14 @@ if six.PY2:
   SPECIAL_BUILTINS['xrange'] = xrange
 
 
+def islambda(f):
+  if not tf_inspect.isfunction(f):
+    return False
+  if not hasattr(f, '__name__'):
+    return False
+  return f.__name__ == '<lambda>'
+
+
 def isbuiltin(f):
   """Returns True if the argument is a built-in function."""
   if f in SPECIAL_BUILTINS.values():
diff --git a/tensorflow/python/autograph/pyct/inspect_utils_test.py b/tensorflow/python/autograph/pyct/inspect_utils_test.py
index 622e3bafc0..7e8466d58e 100644
--- a/tensorflow/python/autograph/pyct/inspect_utils_test.py
+++ b/tensorflow/python/autograph/pyct/inspect_utils_test.py
@@ -95,6 +95,13 @@ def free_factory():
 
 class InspectUtilsTest(test.TestCase):
 
+  def test_islambda(self):
+    def test_fn():
+      pass
+
+    self.assertTrue(inspect_utils.islambda(lambda x: x))
+    self.assertFalse(inspect_utils.islambda(test_fn))
+
   def test_getnamespace_globals(self):
     ns = inspect_utils.getnamespace(factory)
     self.assertEqual(ns['free_function'], free_function)
-- 
GitLab


From 19a1dd5268a67bb2fb0c1b034e9c0d6f63db230b Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 29 Nov 2018 19:09:19 -0800
Subject: [PATCH 1092/1554] [XLA] Make pattern_matchers work with gMock.

This lets us unify the HLO pattern matchers and the HLO gmock matchers (in a later patch).

Unifying these two APIs is useful because then we don't have to learn two APIs,
and we don't have to implement features twice.

This change:

 - Adds and tests the DescribeTo and MatchAndExplain APIs (this is the major change)

 - Uses these new gmock matchers in a few tests as a proof of concept.

 - Rewrites the is-constant-scalar API to use a true matcher rather than a std::function predicate matcher.  This is necessary to get a user-friendly DescribeTo message rather than "I don't know what this std::function does."

 - Adds EffectiveScalarConstant helpers along with the old ScalarConstant helpers and then uses these within while_loop_simplifier.

 - Adds some missing simple op matchers: Tuple, Convolution, Pad, etc.

 - Adds a Parameter(n) matcher.

 - Adds Op().Is(), which matches a particular HloInstruction*, which is used in while_loop_simplifier.

 - Updates documentation to reflect new functions (both added here and added in earlier patches).

 - Tightens up the documentation.  It was getting pretty long, and I made it longer.

 - Changes implementation of FooAnyOrder so that it returns an Op rather than an AnyOf.  This lets you do AddAnyOrder(...).IsScalar(), whereas before this was a compile error.

 - Changes the implementation of FooAnyOrder so it uses a custom matcher rather than an AnyOf, in service of better DescribeTo messages.

 - Implements "and" folding, i.e.

     AllOf<AllOf<A, B...>, X, Y, ...> => AllOf<A, B, ..., X, Y, ...>

   in the service of better DescribeTo messages.

PiperOrigin-RevId: 223451504
---
 tensorflow/compiler/xla/service/BUILD         |   33 +
 .../xla/service/hlo_constant_folding_test.cc  |   33 +-
 .../compiler/xla/service/hlo_parser_test.cc   |   34 +-
 .../xla/service/layout_assignment_test.cc     |   54 +-
 .../compiler/xla/service/pattern_matcher.h    | 1264 ++++++++++++++---
 .../xla/service/pattern_matcher_gmock.h       |   92 ++
 .../xla/service/pattern_matcher_gmock_test.cc |   76 +
 .../xla/service/pattern_matcher_test.cc       |  474 ++++++-
 .../xla/service/while_loop_simplifier.cc      |   27 +-
 9 files changed, 1839 insertions(+), 248 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/pattern_matcher_gmock.h
 create mode 100644 tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 1bd04d2785..429b4e490c 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -408,9 +408,36 @@ tf_cc_test(
         ":hlo",
         ":pattern_matcher",
         "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:test",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "pattern_matcher_gmock",
+    testonly = 1,
+    hdrs = ["pattern_matcher_gmock.h"],
+    deps = [
+        ":pattern_matcher",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/core:test",
+    ],
+)
+
+tf_cc_test(
+    name = "pattern_matcher_gmock_test",
+    srcs = ["pattern_matcher_gmock_test.cc"],
+    deps = [
+        ":hlo",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
     ],
 )
 
@@ -2631,6 +2658,8 @@ tf_cc_test(
         ":hlo",
         ":hlo_matchers",
         ":layout_assignment",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_layout",
         "//tensorflow/compiler/xla:shape_util",
@@ -2775,6 +2804,8 @@ tf_cc_test(
         ":hlo_matchers",
         ":hlo_parser",
         ":hlo_pass",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:test",
@@ -3535,6 +3566,8 @@ tf_cc_test(
         ":hlo_casting_utils",
         ":hlo_matchers",
         ":hlo_parser",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
         "//tensorflow/compiler/xla:test_helpers",
         "//tensorflow/compiler/xla:window_util",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
index d12f920722..4f81dc94e5 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
@@ -22,21 +22,22 @@ limitations under the License.
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/types.h"
 
-namespace op = xla::testing::opcode_matchers;
-
 namespace xla {
 namespace {
 
+namespace m = xla::match;
+
 using HloConstantFoldingTest = HloTestBase;
 
 TEST_F(HloConstantFoldingTest, ConvertF32ToS64) {
@@ -49,13 +50,14 @@ TEST_F(HloConstantFoldingTest, ConvertF32ToS64) {
   auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Convert(input));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Convert().WithOperand(0, m::Op().Is(input))));
 
   HloConstantFolding const_folder;
   TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(module.get()));
   EXPECT_TRUE(result);
 
-  EXPECT_THAT(computation->root_instruction(), op::Constant());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Constant()));
   EXPECT_EQ(computation->root_instruction()->literal().GetFirstElement<int64>(),
             42);
 }
@@ -70,13 +72,14 @@ TEST_F(HloConstantFoldingTest, ConvertS64ToF32) {
   auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Convert(input));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Convert().WithOperand(0, m::Op().Is(input))));
 
   HloConstantFolding const_folder;
   TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(module.get()));
   EXPECT_TRUE(result);
 
-  EXPECT_THAT(computation->root_instruction(), op::Constant());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Constant()));
   EXPECT_EQ(computation->root_instruction()->literal().GetFirstElement<float>(),
             42.0f);
 }
@@ -91,13 +94,14 @@ TEST_F(HloConstantFoldingTest, ConvertF32ArrayToS64Array) {
   auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Convert(input));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Convert().WithOperand(0, m::Op().Is(input))));
 
   HloConstantFolding const_folder;
   TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(module.get()));
   EXPECT_TRUE(result);
 
-  EXPECT_THAT(computation->root_instruction(), op::Constant());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Constant()));
   EXPECT_EQ(computation->root_instruction()->literal().Get<int64>({0}), 42);
   EXPECT_EQ(computation->root_instruction()->literal().Get<int64>({1}), 19);
 }
@@ -138,7 +142,7 @@ TEST_F(HloConstantFoldingTest, Concatenate) {
     EXPECT_TRUE(result);
 
     HloInstruction* root = computation->root_instruction();
-    EXPECT_THAT(root, op::Constant());
+    EXPECT_THAT(root, GmockMatch(m::Constant()));
     EXPECT_TRUE(ShapeUtil::Equal(root->shape(), shape));
   }
 }
@@ -165,7 +169,7 @@ TEST_F(HloConstantFoldingTest, Slice) {
   EXPECT_TRUE(result);
 
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
   EXPECT_TRUE(ShapeUtil::Equal(root->shape(), shape));
 }
 
@@ -190,7 +194,7 @@ TEST_F(HloConstantFoldingTest, TransposeConstantFold) {
   EXPECT_TRUE(result);
 
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
   EXPECT_TRUE(ShapeUtil::Compatible(root->shape(), shape));
 
   using NativeT = typename primitive_util::PrimitiveTypeToNative<F32>::type;
@@ -240,7 +244,8 @@ TEST_F(HloConstantFoldingTest, ConstantFoldReduceNoLayout) {
   TF_ASSERT_OK_AND_ASSIGN(bool result, const_folder.Run(m.get()));
   EXPECT_FALSE(result);
 
-  EXPECT_THAT(m->entry_computation()->root_instruction(), op::Reduce());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::Reduce()));
 }
 
 const char* const kConstantFoldLargePad = R"(
@@ -260,7 +265,7 @@ TEST_F(HloConstantFoldingTest, DoesNotFoldLargePad) {
   EXPECT_FALSE(result);
 
   EXPECT_THAT(module->entry_computation()->root_instruction(),
-              op::Pad(op::Constant(), op::Constant()));
+              GmockMatch(m::Pad(m::Constant(), m::Constant())));
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index f13f7504ee..ab71f011ac 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -21,7 +21,8 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
-#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
@@ -29,7 +30,7 @@ limitations under the License.
 namespace xla {
 namespace {
 
-namespace op = ::xla::testing::opcode_matchers;
+namespace m = ::xla::match;
 using absl::string_view;
 
 struct TestData {
@@ -1893,7 +1894,8 @@ ENTRY ReduceR3ToR2 {
 )";
   TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(original));
   ASSERT_NE(module->entry_computation(), nullptr);
-  EXPECT_THAT(module->entry_computation()->root_instruction(), op::Reduce());
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              GmockMatch(m::Reduce()));
 }
 
 TEST_F(HloParserTest, ParseSharding) {
@@ -1953,7 +1955,7 @@ TEST(HloParserSingleOpTest, SingleOp) {
   const HloComputation* computation = module->entry_computation();
   ASSERT_NE(computation, nullptr);
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(op::Parameter(0), op::Parameter(1)));
+              GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(1))));
 }
 
 TEST(HloParserSingleOpTest, SingleOpNoShapeProducesError) {
@@ -1981,7 +1983,7 @@ TEST(HloParserSingleOpTest, SingleOpNoNames) {
   const HloComputation* computation = module->entry_computation();
   ASSERT_NE(computation, nullptr);
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(op::Parameter(0), op::Parameter(1)));
+              GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(1))));
 }
 
 TEST(HloParserSingleOpTest, CanonicalOp) {
@@ -1990,7 +1992,7 @@ TEST(HloParserSingleOpTest, CanonicalOp) {
   const HloComputation* computation = module->entry_computation();
   ASSERT_NE(computation, nullptr);
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(op::Parameter(0), op::Parameter(1)));
+              GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(1))));
   EXPECT_EQ(
       computation->root_instruction()->ToString(HloPrintOptions::Canonical()),
       text);
@@ -2044,7 +2046,11 @@ TEST(HloParserSingleOpTest, SingleOpWithNested) {
   const HloComputation* computation = module->entry_computation();
   ASSERT_NE(computation, nullptr);
   EXPECT_THAT(computation->root_instruction(),
-              op::Fusion(op::Parameter(0), op::Parameter(1)));
+              GmockMatch(m::Op()
+                             .WithOpcode(HloOpcode::kFusion)
+                             .WithNumOperands(2)
+                             .WithOperand(0, m::Parameter(0))
+                             .WithOperand(1, m::Parameter(1))));
 }
 
 TEST(HloParserSingleOpTest, SingleOpWithNested_DoesNotExist) {
@@ -2088,7 +2094,7 @@ TEST(HloParserSingleOpTest, ConvolutionTrivialFeatureGroupCount) {
   const HloComputation* computation = module->entry_computation();
   ASSERT_NE(computation, nullptr);
   EXPECT_THAT(computation->root_instruction(),
-              op::Convolution(op::Parameter(0), op::Parameter(1)));
+              GmockMatch(m::Convolution(m::Parameter(0), m::Parameter(1))));
   auto* convolution =
       Cast<HloConvolutionInstruction>(computation->root_instruction());
   EXPECT_EQ(convolution->feature_group_count(), 1);
@@ -2152,8 +2158,10 @@ ENTRY %axpy.v5 (alpha: f32[], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
       module->schedule().is_computation_scheduled(module->entry_computation()));
   EXPECT_THAT(
       module->schedule().sequence(module->entry_computation()).instructions(),
-      ::testing::ElementsAre(op::Parameter(), op::Broadcast(), op::Parameter(),
-                             op::Multiply(), op::Parameter(), op::Add()));
+      ::testing::ElementsAre(
+          GmockMatch(m::Parameter()), GmockMatch(m::Broadcast()),
+          GmockMatch(m::Parameter()), GmockMatch(m::Multiply()),
+          GmockMatch(m::Parameter()), GmockMatch(m::Add())));
 }
 
 TEST_F(HloParserTest, IsScheduledIsTrueDifferentOrder) {
@@ -2179,8 +2187,10 @@ ENTRY %axpy.v5 (alpha: f32[], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
       module->schedule().is_computation_scheduled(module->entry_computation()));
   EXPECT_THAT(
       module->schedule().sequence(module->entry_computation()).instructions(),
-      ::testing::ElementsAre(op::Parameter(), op::Parameter(), op::Parameter(),
-                             op::Broadcast(), op::Multiply(), op::Add()));
+      ::testing::ElementsAre(
+          GmockMatch(m::Parameter()), GmockMatch(m::Parameter()),
+          GmockMatch(m::Parameter()), GmockMatch(m::Broadcast()),
+          GmockMatch(m::Multiply()), GmockMatch(m::Add())));
 }
 
 TEST_F(HloParserTest, CustomCallWrongNumberofOperandConstraints) {
diff --git a/tensorflow/compiler/xla/service/layout_assignment_test.cc b/tensorflow/compiler/xla/service/layout_assignment_test.cc
index 61d8a0a4e6..311bd78905 100644
--- a/tensorflow/compiler/xla/service/layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment_test.cc
@@ -31,6 +31,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
 #include "tensorflow/compiler/xla/shape_layout.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
@@ -42,11 +44,10 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 
-namespace op = xla::testing::opcode_matchers;
-
 namespace xla {
 namespace {
 
+namespace m = xla::match;
 using ::testing::ElementsAre;
 
 class LayoutAssignmentTest : public HloTestBase {
@@ -342,7 +343,8 @@ TEST_F(LayoutAssignmentTest, ConflictingLayoutTuple) {
 
   // Verify the structure of the HLO graph.
   EXPECT_THAT(root,
-              op::Tuple(op::Tuple(constant), op::Tuple(op::Copy(constant))));
+              GmockMatch(m::Tuple(m::Tuple(m::Op().Is(constant)),
+                                  m::Tuple(m::Copy(m::Op().Is(constant))))));
 }
 
 TEST_F(LayoutAssignmentTest, ElementwiseAndReshape) {
@@ -946,9 +948,11 @@ TEST_F(LayoutAssignmentTest, CopySliceOperandToAvoidImplicitLayoutChange) {
   HloInstruction* root =
       compiled_module->entry_computation()->root_instruction();
   Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {4, 5}, {1, 0});
-  EXPECT_THAT(root, op::Add(op::Parameter(),
-                            op::Slice(AllOf(op::Copy(op::Parameter(1)),
-                                            op::ShapeWithLayout(shape_copy)))));
+  EXPECT_THAT(
+      root,
+      GmockMatch(m::Add(
+          m::Parameter(),
+          m::Slice(m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy)))));
 }
 
 TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) {
@@ -976,10 +980,11 @@ TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) {
       compiled_module->entry_computation()->root_instruction();
   Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {4, 5}, {1, 0});
   EXPECT_THAT(root,
-              op::Add(op::Parameter(),
-                      op::DynamicSlice(AllOf(op::Copy(op::Parameter(1)),
-                                             op::ShapeWithLayout(shape_copy)),
-                                       op::Parameter(2))));
+              GmockMatch(m::Add(
+                  m::Parameter(),
+                  m::DynamicSlice(
+                      m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy),
+                      m::Parameter(2)))));
 }
 
 TEST_F(LayoutAssignmentTest, CopyConcatOperandToAvoidImplicitLayoutChange) {
@@ -1007,11 +1012,12 @@ TEST_F(LayoutAssignmentTest, CopyConcatOperandToAvoidImplicitLayoutChange) {
   HloInstruction* root =
       compiled_module->entry_computation()->root_instruction();
   Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {3, 5}, {1, 0});
-  EXPECT_THAT(root,
-              op::Add(op::Parameter(),
-                      op::Concatenate(AllOf(op::Copy(op::Parameter(1)),
-                                            op::ShapeWithLayout(shape_copy)),
-                                      op::Parameter(2))));
+  EXPECT_THAT(
+      root,
+      GmockMatch(m::Add(
+          m::Parameter(),
+          m::Concatenate(m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy),
+                         m::Parameter(2)))));
 }
 
 TEST_F(LayoutAssignmentTest,
@@ -1038,7 +1044,8 @@ TEST_F(LayoutAssignmentTest,
           .ConsumeValueOrDie();
   HloInstruction* root =
       compiled_module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Convolution(op::Parameter(0), op::Parameter(1)));
+  EXPECT_THAT(root,
+              GmockMatch(m::Convolution(m::Parameter(0), m::Parameter(1))));
 }
 
 TEST_F(LayoutAssignmentTest, PropagatingLayoutFromResultToOperand) {
@@ -1062,8 +1069,9 @@ TEST_F(LayoutAssignmentTest, PropagatingLayoutFromResultToOperand) {
   HloInstruction* root =
       compiled_module->entry_computation()->root_instruction();
   Shape shape_copy = ShapeUtil::MakeShapeWithLayout(F32, {4, 5}, {0, 1});
-  EXPECT_THAT(root, op::Slice(AllOf(op::Copy(op::Parameter(0)),
-                                    op::ShapeWithLayout(shape_copy))));
+  EXPECT_THAT(root,
+              GmockMatch(m::Slice(
+                  m::Copy(m::Parameter(0)).WithShapeEqualTo(&shape_copy))));
 }
 
 TEST_F(LayoutAssignmentTest, TupleCopyOnLayoutMismatch) {
@@ -1149,7 +1157,7 @@ ENTRY %CustomCallWithNotLayoutConstrained (p: f32[42,2,3]) -> f32[1,2,3,4] {
     AssignLayouts(m.get(), &computation_layout);
 
     HloInstruction* root = m->entry_computation()->root_instruction();
-    ASSERT_THAT(root, op::CustomCall(op::Parameter()));
+    ASSERT_THAT(root, GmockMatch(m::CustomCall(m::Parameter())));
     ExpectLayoutIs(root->shape(), {3, 2, 0, 1});
     ExpectLayoutIs(root->operand(0)->shape(), {0, 2, 1});
   }
@@ -1165,7 +1173,7 @@ ENTRY %CustomCallWithNotLayoutConstrained (p: f32[42,2,3]) -> f32[1,2,3,4] {
     AssignLayouts(m.get(), &computation_layout);
 
     HloInstruction* root = m->entry_computation()->root_instruction();
-    ASSERT_THAT(root, op::CustomCall(op::Parameter()));
+    ASSERT_THAT(root, GmockMatch(m::CustomCall(m::Parameter())));
     ExpectLayoutIs(root->shape(), {0, 2, 3, 1});
     ExpectLayoutIs(root->operand(0)->shape(), {0, 1, 2});
   }
@@ -1196,7 +1204,7 @@ ENTRY %CustomCallWithLayoutConstraints (p0: f32[4,4], p1: f32[2,3]) -> f32[1,2,3
   // The custom call should be partially encapsulated in kCopy instructions
   // because of the layout mismatches.
   ASSERT_THAT(m->entry_computation()->root_instruction(),
-              op::Copy(op::CustomCall(op::Copy(), op::Parameter())));
+              GmockMatch(m::Copy(m::CustomCall(m::Copy(), m::Parameter()))));
 
   const HloInstruction* custom_call =
       m->entry_computation()->root_instruction()->operand(0);
@@ -1222,7 +1230,7 @@ ENTRY %CustomCallLayoutConstrainedZeroOperands () -> f32[1,2,3,4] {
   AssignLayouts(m.get(), &computation_layout);
 
   ASSERT_THAT(m->entry_computation()->root_instruction(),
-              op::Copy(op::CustomCall()));
+              GmockMatch(m::Copy(m::CustomCall())));
 
   const HloInstruction* custom_call =
       m->entry_computation()->root_instruction()->operand(0);
@@ -1256,7 +1264,7 @@ ENTRY %CustomCallLayoutConstrainedTupleOperand (p0: f32[4,4], p1: f32[2,3]) -> f
   ExpectLayoutIs(root->shape(), {2, 1, 0, 3});
 
   ASSERT_THAT(m->entry_computation()->root_instruction(),
-              op::Copy(op::CustomCall(op::Tuple())));
+              GmockMatch(m::Copy(m::CustomCall(m::Tuple()))));
 
   const HloInstruction* custom_call =
       m->entry_computation()->root_instruction()->operand(0);
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index f196d9b7f5..fb1645d9b2 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_
 #define TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_
 
+#include "absl/strings/str_replace.h"
 #include "absl/strings/string_view.h"
 #include "absl/utility/utility.h"
 #include "tensorflow/compiler/xla/layout_util.h"
@@ -44,32 +45,45 @@ namespace xla {
 //
 // This pattern will match Add instructions whose first operand is a constant.
 //
-// Each pattern type has the following modifiers:
+// Each pattern type has the following modifiers, which are described where
+// nontrivial.
 //
 //   Op():
-//     - WithName: match operations with the given name
-//     - WithOpcode: match operations with the given opcode
-//     - WithShape: match operations whose shape matches the given pattern
-//     - WithOperand: match operations whose operand matches the given pattern
+//     - Is: is the given HloInstruction* (i.e. pointer equality)
+//     - WithName
+//     - WithOpcode
+//     - WithoutOpcode: anything other than the given opcode
+//     - WithShape: instr's shape matches the given pattern
+//     - WithShapeEqualTo: instr's shape is equal to the given Shape
+//     - WithShapeCompatibleTo: instr's shape is compatible with the given Shape
+//     - WithNumOperands
+//     - WithOperand: operand at the given index matches the given pattern
+//     - IsConstant
+//     - IsNonConstant
+//     - IsConstantScalar/IsEffectiveConstantScalar: Optionally accepts a value,
+//       e.g. IsConstantScalar() or IsConstantScalar(42).
+//     - WithFusionKind
+//     - WithTupleIndex: get-tuple-element operations with the given tuple index
 //
 //   Shape():
-//     - EqualTo: matches shapes that are equal to the argument
-//     - CompatibleTo: matches shapes that are compatible to the argument
-//     - IsScalar/IsArray/IsTuple: matches scalar/array/tuple shapes
-//     - IsDenseArray/IsSparseArray: matches arrays with dense/sparse format
-//     - WithLayout: match shapes whose layout matches the given pattern
-//     - WithLayoutEqualTo: matches shapes whose layouts equal the argument
-//     - WithSubshape: matches tuple shapes whose subshape matches the given
-//       pattern
-//     - WithSubshapeEqualTo: matches shapes with a subshape equal the argument
-//     - WithElementType: matches array/scalar shapes with the given element
-//       type
-//     - WithRank: matches array/scalar types with the given rank
+//     - EqualTo
+//     - CompatibleTo
+//     - IsScalar/IsEffectiveScalar/IsArray/IsTuple
+//     - IsDenseArray/IsSparseArray
+//     - WithLayout: layout shape's layout matches the given pattern (e.g.
+//       Layout().WithDenseFormat())
+//     - WithLayoutEqualTo: shape's layout equals the argument (i.e. another
+//       Layout, but not the result of Layout().foo())
+//     - WithSubshape: shape is a tuple whose subshape matches the given pattern
+//       (e.g. Shape().IsScalar()).
+//     - WithSubshapeEqualTo: shape is a tuple with a subshape equal to the arg
+//       (i.e. another Shape, but not the result of Shape().foo())
+//     - WithElementType: shape is an array/scalar with the given elem type
+//     - WithRank: shape is an array/scalar with the given rank
 //
 //  Layout():
-//     - EqualTo: matches layouts that are equal to the argument
-//     - WithDenseFormat/WithSparseFormat: matches layouts with dense/sparse
-//       format
+//     - EqualTo
+//     - WithDenseFormat/WithSparseFormat
 //
 // Op(), Shape(), and Layout() may be passed an argument of type
 // HloInstruction**, Shape**, or Layout**, respectively, or const versions of
@@ -82,53 +96,55 @@ namespace xla {
 //   CHECK(Match(foo,
 //               match::Op().WithOperand(0, match::Op(&matched_operand))));
 //
-// Helpers are provided for common nullary, unary, binary, and ternary
-// instructions. These helpers can be called with no arguments, in which case
-// they will match any instruction matching the opcode. They may also be called
-// with matches for the operands and with an optional capture. (The capture must
-// be the first argument.) Some examples of these helpers and their equivalents
-// are provided below.
-//
+// Helpers are provided for most HLO instructions. These helpers can be called
+// with no arguments, in which case they will match any instruction matching the
+// opcode. They may also be called with matches for the operands and with an
+// optional capture. (The capture must be the first argument.) Some examples of
+// these helpers and their equivalents are provided below.
+
 // Example nullary instruction:
 //   Parameter()                    == Op().WithOpcode(HloOpcode::kParameter)
 //   Parameter(&a)                  == Op(&a).WithOpcode(HloOpcode::kParameter)
 //
 // Example unary instruction:
-//   Abs()                             == Op().WithOpcode(HloOpcode::kAbs)
-//   Abs(Op(&a))                       == Op().WithOpcode(HloOpcode::kAbs)
-//                                            .WithOperand(0, Op(&a)))
-//   Abs(&a, Op(&b))                   == Op(&a).WithOpcode(HloOpcode::kAbs)
-//                                              .WithOperand(0, Op(&b))
+//   Abs()                          == Op().WithOpcode(HloOpcode::kAbs)
+//   Abs(Op(&a))                    == Op().WithOpcode(HloOpcode::kAbs)
+//                                         .WithOperand(0, Op(&a)))
+//   Abs(&a, Op(&b))                == Op(&a).WithOpcode(HloOpcode::kAbs)
+//                                           .WithOperand(0, Op(&b))
+//
+// Commutative binary instructions have a special form that accepts either order
+// of args, e.g.:
 //
-// Example binary instruction:
-//   Add()                             == Op().WithOpcode(HloOpcode::kAdd)
-//   Add(Op(&a), Op(&b))               == Op().WithOpcode(HloOpcode::kAdd)
-//                                            .WithOperand(0, Op(&a))
-//                                            .WithOperand(1, Op(&b))
-//   Add(&a, Op(&b), Op(&c))           == Op(&a).WithOpcode(HloOpcode::kAdd)
-//                                              .WithOperand(0, Op(&b))
-//                                              .WithOperand(1, Op(&c))
+//   AddAnyOrder(Parameter(1), Abs()) ==
+//     Op().WithOpcode(HloOpcode::kAdd)
+//         .WithBinaryOperandsAnyOrder(Op().WithParameterNum(1), Abs());
 //
-// Example ternary instruction:
-//   Clamp()                           == Op().WithOpcode(HloOpcode::kClamp)
-//   Clamp(Op(&a), Op(&b), Op(&c))     == Op().WithOpcode(HloOpcode::kClamp)
-//                                            .WithOperand(0, Op(&a))
-//                                            .WithOperand(1, Op(&b))
-//                                            .WithOperand(2, Op(&c))
-//   Clamp(&a, Op(&b), Op(&c), Op(&d)) == Op(&a).WithOpcode(HloOpcode::kClamp)
-//                                              .WithOperand(0, Op(&b))
-//                                              .WithOperand(1, Op(&c))
-//                                              .WithOperand(2, Op(&d))
+//   MultiplyAnyOrder(&a, Parameter(), Abs())  // Captures the mul in `a`.
 //
+// The following additional helpers are provided.  In all cases, `&a` is
+// optional.
+//
+//   ConstantScalar(&a)               == Op(&a).IsConstantScalar();
+//   ConstantScalar(&a, v)            == Op(&a).IsConstantScalar(v);
+//   ConstantEffectiveScalar(&a)      == Op(&a).IsConstantEffectiveScalar();
+//   ConstantEffectiveScalar(&a, v)   == Op(&a).IsConstantEffectiveScalar(&a, v)
+//   NonConstant(&a)                  == Op(&a).IsNonConstant()
+//   GetTupleElement(&a, b, index)    == Op(&a).WithTupleIndex(index)
+//                                             .WithOperand(0, b);
+//   Parameter(&a, n)                 == Op(&a).WithParameterNum(n);
 
 struct MatchOption {
   // If true, actually capture matched item into the user pointer.
   bool capture;
+
+  // An explanation for why we failed to match is streamed here, if not-null.
+  std::ostream* explain_os;
 };
 
 template <typename Value, typename Pattern>
 bool Match(Value* value, const Pattern& pattern,
-           MatchOption option = {/*.capture=*/true}) {
+           MatchOption option = {/*.capture=*/true, /*.explain_os=*/nullptr}) {
   if (option.capture) {
     auto new_option = option;
     new_option.capture = false;
@@ -143,6 +159,77 @@ namespace match {
 
 namespace detail {
 
+// Macro for streaming to option.explain_os if it's not null.
+//
+//   EXPLAIN << "value of foo(): " << foo()
+//
+#pragma push_macro("EXPLAIN")
+#define EXPLAIN \
+  if (option.explain_os) *option.explain_os
+
+// kIndentInc is the additional number of spaces that we indent by when we
+// increase the indent "by one".
+enum {
+  kIndentInc = 2,
+};
+
+// Writes a newline and then `indent` spaces.
+//
+// We follow an unintuitive convention in this file's pretty-printers: Indents
+// are performed by the caller, not the callee.  For example, if you want to
+// print
+//
+//   foo:
+//    - bar
+//
+// you'd do:
+//
+//  Foo::DescribeTo(std::ostream* os, int64 indent) {
+//    *os << "foo:";
+//    Indent(os, indent)  // Create a newline at the *current* indent level.
+//    *os << " - ";
+//    bar.DescribeTo(os, indent + 3);  // + 3 because strlen(" * ") == 3.
+//  }
+//
+//  Bar::DescribeTo(std::ostream* os, int64 indent) { *os << "bar"; }
+//
+// Notice that Bar::DescribeTo() does not call Indent; the indenting is
+// performed by Foo.  This convention allows the caller to decide whether a
+// matcher is preceded by a newline, which is important e.g. for the AllOf
+// matcher.
+//
+// (Incidentally, indenting in Match's explanations is handled differently.
+// Indents are a common case in DescribeTo [we're printing a whole tree], but
+// they're a special case in Match [we're printing only a path through the tree
+// that encounters a failing node]. Indents in Match only appear when we
+// encounter a failing disjunction, so we just handle them as a special case
+// there.)
+inline void Indent(std::ostream* os, int64 indent) {
+  *os << "\n";
+  for (int64 i = 0; i < indent; ++i) {
+    *os << " ";
+  }
+}
+
+// SFINAE template that determines whether T declares a static member
+// kIsTrivialMatcher.
+//
+// Trivial matchers get special treatment.  For example, when printing
+// a conjunction of matchers, we don't print "and" after a trivial matcher. This
+// yields e.g.
+//    "a shape compatible with f32[1,2]"
+// rather than
+//    "a shape AND compatible with f32[1,2]"
+template <typename T, typename Dummy = void>
+struct IsTrivialMatcher {
+  static constexpr bool value = false;
+};
+template <typename T>
+struct IsTrivialMatcher<T,
+                        typename std::enable_if<T::kIsTrivialMatcher>::type> {
+  static constexpr bool value = true;
+};
+
 template <typename Item, typename... Patterns>
 class AllOfPattern {
  public:
@@ -162,10 +249,19 @@ class AllOfPattern {
     return matched;
   }
 
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    DescribeToImpl(os, std::integral_constant<size_t, 0>(), indent);
+  }
+
+  // Accessor for patterns_.  Please don't use this outside of this file.
+  const std::tuple<Patterns...>& patterns() const { return patterns_; }
+
  private:
   template <typename ItemType, size_t index>
   bool MatchImpl(ItemType* item, MatchOption option,
                  std::integral_constant<size_t, index>) const {
+    // We don't need to do any EXPLAINing here; it's all correctly handled by
+    // our sub-matchers (if any fail).
     return std::get<index>(patterns_).Match(item, option) &&
            MatchImpl(item, option, std::integral_constant<size_t, index + 1>());
   }
@@ -176,6 +272,73 @@ class AllOfPattern {
     return true;
   }
 
+  // Pretty-printing a conjunction has some special cases to make it easy to
+  // read in the simple (common) case.
+  //
+  // If sizeof...(Patterns) == 1, prints as e.g.
+  //
+  //   a shape
+  //
+  // If sizeof...(Patterns) == 2 and patterns_[0] is a trivial matcher (e.g. "a
+  // shape") prints as
+  //
+  //   a shape compatible with f32[1,2]
+  //
+  // If sizeof...(Patterns) > 2 and patterns_[0] is a trivial matcher, prints as
+  //
+  //   a shape:
+  //    * compatible with f32[1,2] AND
+  //    * that represents a scalar
+  //
+  // Otherwise prints as:
+  //
+  //   all of:
+  //    * foo AND
+  //    * bar
+  //
+  template <size_t index>
+  void DescribeToImpl(std::ostream* os, std::integral_constant<size_t, index>,
+                      int64 indent) const {
+    constexpr bool first_is_trivial =
+        IsTrivialMatcher<typename std::remove_reference<decltype(
+            std::get<0>(patterns_))>::type>::value;
+    constexpr bool is_last = index == sizeof...(Patterns) - 1;
+    const auto& submatcher = std::get<index>(patterns_);
+
+    auto print_bulleted_item = [&] {
+      *os << " * ";
+      submatcher.DescribeTo(os, indent + 3);
+      if (!is_last) {
+        *os << " AND";
+        Indent(os, indent);
+      }
+    };
+
+    if (index == 0) {
+      if (first_is_trivial || is_last) {
+        submatcher.DescribeTo(os, indent + kIndentInc);
+        if (sizeof...(Patterns) > 2) {
+          *os << ":";
+          Indent(os, indent);
+        }
+      } else {
+        *os << "all of:";
+        Indent(os, indent);
+        print_bulleted_item();
+      }
+    } else if (first_is_trivial && index == 1 && sizeof...(Patterns) == 2) {
+      *os << " ";
+      submatcher.DescribeTo(os, indent);
+    } else {
+      print_bulleted_item();
+    }
+    DescribeToImpl(os, std::integral_constant<size_t, index + 1>(), indent);
+  }
+
+  void DescribeToImpl(std::ostream* os,
+                      std::integral_constant<size_t, sizeof...(Patterns)>,
+                      int64 indent) const {}
+
   std::tuple<Patterns...> patterns_;
 };
 
@@ -183,10 +346,6 @@ class AllOfPattern {
 
 // Returns a pattern that represents the conjunction of all input patterns. All
 // patterns need to match in order to have the AllOf pattern match.
-//
-// TODO(timshen): Currently AllOf is still nested, e.g. AllOf<AllOf<A>, B> is
-// not AllOf<A, B>. We might want to flatten the AllOf type structure if the
-// C++ compile error message gets annoying.
 template <typename Item, typename... Patterns>
 detail::AllOfPattern<typename std::remove_const<Item>::type, Patterns...> AllOf(
     const Patterns&... patterns) {
@@ -194,6 +353,25 @@ detail::AllOfPattern<typename std::remove_const<Item>::type, Patterns...> AllOf(
                               Patterns...>(patterns...);
 }
 
+// AllOf<AllOf<A, B...>, X, Y, ...> => AllOf<A, B, ..., X, Y, ...>.
+//
+// This transformation is necessary for good pretty-printing.
+template <typename Item, typename... InnerPs, typename... OuterPs>
+detail::AllOfPattern<typename std::remove_const<Item>::type, InnerPs...,
+                     OuterPs...>
+AllOf(const detail::AllOfPattern<Item, InnerPs...>& inner_p,
+      const OuterPs&... outer_ps) {
+  // Invoke constructor of AllOfPattern<Item, InnerPs..., OuterPs...>.
+  auto make_all_of = [](const InnerPs&... inner_ps,
+                        const OuterPs&... outer_ps) {
+    return detail::AllOfPattern<typename std::remove_const<Item>::type,
+                                InnerPs..., OuterPs...>(inner_ps...,
+                                                        outer_ps...);
+  };
+  return absl::apply(make_all_of, std::tuple_cat(inner_p.patterns(),
+                                                 std::make_tuple(outer_ps...)));
+}
+
 namespace detail {
 
 template <typename LayoutType, typename Impl>
@@ -204,8 +382,18 @@ class LayoutPattern;
 class LayoutPatternBaseImpl {
  public:
   bool Match(const ::xla::Layout* layout, MatchOption option) const {
-    return layout != nullptr;
+    if (layout == nullptr) {
+      EXPLAIN << "Layout is null";
+      return false;
+    }
+    return true;
   }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "a layout";
+  }
+
+  static constexpr bool kIsTrivialMatcher = true;
 };
 
 // A LayoutPattern implementation that matches only if the layout equals a
@@ -216,7 +404,17 @@ class LayoutPatternEqualImpl {
       : layout_(layout) {}
 
   bool Match(const ::xla::Layout* layout, MatchOption option) const {
-    return LayoutUtil::Equal(*layout_, *layout);
+    if (!LayoutUtil::Equal(*layout_, *layout)) {
+      EXPLAIN << "Layout " << LayoutUtil::HumanString(*layout)
+              << " is not equal to expected "
+              << LayoutUtil::HumanString(*layout_);
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "equal to " << LayoutUtil::HumanString(*layout_);
   }
 
  private:
@@ -230,7 +428,16 @@ class LayoutPatternFormatImpl {
   explicit constexpr LayoutPatternFormatImpl(Format format) : format_(format) {}
 
   bool Match(const ::xla::Layout* layout, MatchOption option) const {
-    return layout->format() == format_;
+    if (layout->format() != format_) {
+      EXPLAIN << "Layout has format " << Format_Name(layout->format())
+              << " but expected " << Format_Name(format_);
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with format " << Format_Name(format_);
   }
 
  private:
@@ -242,11 +449,13 @@ template <typename LayoutType, typename Impl>
 class LayoutPattern {
  private:
   template <typename NewImpl>
-  LayoutPattern<LayoutType, AllOfPattern<::xla::Layout, Impl, NewImpl>>
-  AppendImpl(NewImpl new_impl) const {
-    return LayoutPattern<LayoutType,
-                         AllOfPattern<::xla::Layout, Impl, NewImpl>>(
-        AllOf<Layout>(impl_, std::move(new_impl)), matched_layout_);
+  auto AppendImpl(NewImpl new_impl) const
+      -> LayoutPattern<LayoutType,
+                       decltype(AllOf<Layout>(std::declval<Impl>(),
+                                              std::move(new_impl)))> {
+    auto new_allof = AllOf<Layout>(impl_, std::move(new_impl));
+    return LayoutPattern<LayoutType, decltype(new_allof)>(std::move(new_allof),
+                                                          matched_layout_);
   }
 
  public:
@@ -276,6 +485,10 @@ class LayoutPattern {
     return false;
   }
 
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    impl_.DescribeTo(os, indent);
+  }
+
   // Modifies the pattern to match only if the layout equals the given proto.
   // The layout must outlive the returned pattern.
   constexpr auto EqualTo(const ::xla::Layout* layout) const
@@ -306,19 +519,48 @@ class AnyOfPattern {
   explicit AnyOfPattern(const Patterns&... patterns) : patterns_(patterns...) {}
 
   bool Match(const Item* item, MatchOption option) const {
-    return MatchImpl(item, option, std::integral_constant<size_t, 0>());
+    return MatchImpl(item, option);
   }
 
   bool Match(Item* item, MatchOption option) const {
-    return MatchImpl(item, option, std::integral_constant<size_t, 0>());
+    return MatchImpl(item, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "any of:";
+    Indent(os, indent);
+    DescribeToImpl(os, std::integral_constant<size_t, 0>(), indent);
   }
 
  private:
+  template <typename ItemType>
+  bool MatchImpl(ItemType* item, MatchOption option) const {
+    // If we're generating an explanation, buffer it until we know we failed.
+    absl::optional<std::stringstream> explanation;
+    MatchOption new_option = option;
+    if (option.explain_os) {
+      new_option.explain_os = &explanation.emplace();
+    }
+    bool rv = MatchRecursiveImpl(item, new_option,
+                                 std::integral_constant<size_t, 0>());
+    if (!rv && option.explain_os) {
+      EXPLAIN << "None of the following matchers succeeded:";
+      EXPLAIN << explanation->str();
+    }
+    return rv;
+  }
+
   template <typename ItemType, size_t index>
-  bool MatchImpl(ItemType* item, MatchOption option,
-                 std::integral_constant<size_t, index>) const {
+  bool MatchRecursiveImpl(ItemType* item, MatchOption option,
+                          std::integral_constant<size_t, index>) const {
     auto new_option = option;
     new_option.capture = false;
+
+    absl::optional<std::stringstream> explanation;
+    if (option.explain_os) {
+      new_option.explain_os = &explanation.emplace();
+    }
+
     // Try to match the sub-pattern without capturing behavior.
     if (std::get<index>(patterns_).Match(item, new_option)) {
       // Capture the branch.
@@ -337,20 +579,46 @@ class AnyOfPattern {
         // AnyOf will be a runtime number indicate which sub-pattern is matched.
         // Then we run another pass to do captures only with the help of the
         // trace.
-        bool ret = std::get<index>(patterns_).Match(item, option);
-        DCHECK(ret);
+        bool matched = std::get<index>(patterns_).Match(item, option);
+        DCHECK(matched);
       }
       return true;
     }
-    return MatchImpl(item, option, std::integral_constant<size_t, index + 1>());
+    if (option.explain_os) {
+      EXPLAIN << "\nMatcher #" << index + 1;
+      EXPLAIN << "\n - ";
+      std::get<index>(patterns_).DescribeTo(option.explain_os, /*indent=*/3);
+      EXPLAIN << "\nfailed with";
+      EXPLAIN << "\n - ";
+      EXPLAIN << absl::StrReplaceAll(explanation->str(), {{"\n", "\n   "}});
+    }
+    return MatchRecursiveImpl(item, option,
+                              std::integral_constant<size_t, index + 1>());
   }
 
   template <typename ItemType>
-  bool MatchImpl(ItemType* item, MatchOption option,
-                 std::integral_constant<size_t, sizeof...(Patterns)>) const {
+  bool MatchRecursiveImpl(
+      ItemType* item, MatchOption option,
+      std::integral_constant<size_t, sizeof...(Patterns)>) const {
     return false;
   }
 
+  template <size_t index>
+  void DescribeToImpl(std::ostream* os, std::integral_constant<size_t, index>,
+                      int64 indent) const {
+    *os << " - ";
+    std::get<index>(patterns_).DescribeTo(os, indent + 3);
+    if (index != sizeof...(Patterns) - 1) {
+      *os << " OR";
+      Indent(os, indent);
+    }
+    DescribeToImpl(os, std::integral_constant<size_t, index + 1>(), indent);
+  }
+
+  void DescribeToImpl(std::ostream* os,
+                      std::integral_constant<size_t, sizeof...(Patterns)>,
+                      int64 indent) const {}
+
   std::tuple<Patterns...> patterns_;
 };
 
@@ -395,8 +663,17 @@ class ShapePattern;
 class ShapePatternBaseImpl {
  public:
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
+    if (shape == nullptr) {
+      EXPLAIN << "Shape is null";
+    }
     return shape != nullptr;
   }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "a shape";
+  }
+
+  static constexpr bool kIsTrivialMatcher = true;
 };
 
 // A ShapePattern implementation that matches only if the shape equals a Shape
@@ -407,7 +684,16 @@ class ShapePatternEqualImpl {
       : shape_(shape) {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::Equal(*shape_, *shape);
+    if (!ShapeUtil::Equal(*shape_, *shape)) {
+      EXPLAIN << "Shape not equal to "
+              << ShapeUtil::HumanStringWithLayout(*shape_);
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "equal to " << ShapeUtil::HumanStringWithLayout(*shape_);
   }
 
  private:
@@ -422,7 +708,16 @@ class ShapePatternCompatibleImpl {
       : shape_(shape) {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::Compatible(*shape_, *shape);
+    if (!ShapeUtil::Compatible(*shape_, *shape)) {
+      EXPLAIN << "Shape not compatible with "
+              << ShapeUtil::HumanString(*shape_);
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "compatible with " << ShapeUtil::HumanString(*shape_);
   }
 
  private:
@@ -437,7 +732,16 @@ class ShapePatternElementTypeImpl {
       : element_type_(element_type) {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return shape->element_type() == element_type_;
+    if (shape->element_type() != element_type_) {
+      EXPLAIN << "Shape does not have element type "
+              << PrimitiveType_Name(element_type_);
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with element type " << PrimitiveType_Name(element_type_);
   }
 
  private:
@@ -450,7 +754,15 @@ class ShapePatternIsScalarImpl {
   explicit constexpr ShapePatternIsScalarImpl() {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::IsScalar(*shape);
+    if (!ShapeUtil::IsScalar(*shape)) {
+      EXPLAIN << "Shape is not a scalar";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "that represents a scalar";
   }
 };
 
@@ -460,7 +772,15 @@ class ShapePatternIsArrayImpl {
   explicit constexpr ShapePatternIsArrayImpl() {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::IsArray(*shape);
+    if (!ShapeUtil::IsArray(*shape)) {
+      EXPLAIN << "Shape is not an array";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "that represents an array";
   }
 };
 
@@ -470,7 +790,34 @@ class ShapePatternIsTupleImpl {
   explicit constexpr ShapePatternIsTupleImpl() {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::IsTuple(*shape);
+    if (!ShapeUtil::IsTuple(*shape)) {
+      EXPLAIN << "Shape is not a tuple";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "that represents a tuple";
+  }
+};
+
+// A ShapePattern implementation that matches only if the shape is an effective
+// scalar.
+class ShapePatternEffectiveScalarImpl {
+ public:
+  explicit constexpr ShapePatternEffectiveScalarImpl() {}
+
+  bool Match(const ::xla::Shape* shape, MatchOption option) const {
+    if (!ShapeUtil::IsEffectiveScalar(*shape)) {
+      EXPLAIN << "Shape is not an effective scalar";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "that is an effective scalar";
   }
 };
 
@@ -481,7 +828,23 @@ class ShapePatternRankImpl {
   explicit constexpr ShapePatternRankImpl(int64 rank) : rank_(rank) {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::Rank(*shape) == rank_;
+    if (ShapeUtil::Rank(*shape) != rank_) {
+      if (rank_ == 0) {
+        EXPLAIN << "Shape is not a scalar";
+      } else {
+        EXPLAIN << "Shape does not have rank " << rank_;
+      }
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    if (rank_ == 0) {
+      *os << "that is a scalar";
+    } else {
+      *os << "that has " << rank_ << " dimension" << (rank_ != 1 ? "s" : "");
+    }
   }
 
  private:
@@ -503,8 +866,21 @@ class ShapePatternLayoutImpl {
   }
 
   bool Match(Shape* shape, MatchOption option) const {
-    return LayoutUtil::HasLayout(*shape) &&
-           layout_.Match(shape->mutable_layout(), option);
+    if (!LayoutUtil::HasLayout(*shape)) {
+      EXPLAIN << "Shape does not have a layout";
+      return false;
+    }
+    if (!layout_.Match(shape->mutable_layout(), option)) {
+      EXPLAIN << "\nin layout";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with";
+    Indent(os, indent + kIndentInc);
+    layout_.DescribeTo(os, indent + kIndentInc);
   }
 
  private:
@@ -522,17 +898,40 @@ class ShapePatternSubshapeImpl {
       : index_(index), subshape_(subshape) {}
 
   bool Match(const ::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::IndexIsValid(*shape, index_) &&
-           subshape_.Match(&ShapeUtil::GetSubshape(*shape, index_), option);
+    return MatchImpl(shape, option);
   }
 
   bool Match(::xla::Shape* shape, MatchOption option) const {
-    return ShapeUtil::IndexIsValid(*shape, index_) &&
-           subshape_.Match(ShapeUtil::GetMutableSubshape(shape, index_),
-                           option);
+    return MatchImpl(shape, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with subshape at index " << index_.ToString() << " which is";
+    Indent(os, indent + kIndentInc);
+    subshape_.DescribeTo(os, indent + kIndentInc);
   }
 
  private:
+  Shape* GetSubshape(Shape* shape) const {
+    return ShapeUtil::GetMutableSubshape(shape, index_);
+  }
+  const Shape* GetSubshape(const Shape* shape) const {
+    return &ShapeUtil::GetSubshape(*shape, index_);
+  }
+
+  template <typename ShapeType>
+  bool MatchImpl(ShapeType* shape, MatchOption option) const {
+    if (!ShapeUtil::IndexIsValid(*shape, index_)) {
+      EXPLAIN << "No subshape at " << index_.ToString();
+      return false;
+    }
+    if (!subshape_.Match(GetSubshape(shape), option)) {
+      EXPLAIN << "\nin subshape at " << index_.ToString();
+      return false;
+    }
+    return true;
+  }
+
   ShapeIndexView index_;
   ShapePattern<SubshapeType, SubshapeImpl> subshape_;
 };
@@ -542,10 +941,12 @@ template <typename ShapeType, typename Impl>
 class ShapePattern {
  private:
   template <typename NewImpl>
-  ShapePattern<ShapeType, AllOfPattern<::xla::Shape, Impl, NewImpl>> AppendImpl(
-      NewImpl new_impl) const {
-    return ShapePattern<ShapeType, AllOfPattern<::xla::Shape, Impl, NewImpl>>(
-        AllOf<Shape>(impl_, std::move(new_impl)), matched_shape_);
+  auto AppendImpl(NewImpl new_impl) const
+      -> ShapePattern<ShapeType, decltype(AllOf<Shape>(std::declval<Impl>(),
+                                                       std::move(new_impl)))> {
+    auto new_all_of = AllOf<Shape>(impl_, std::move(new_impl));
+    return ShapePattern<ShapeType, decltype(new_all_of)>(std::move(new_all_of),
+                                                         matched_shape_);
   }
 
  public:
@@ -560,6 +961,11 @@ class ShapePattern {
       }
       return true;
     }
+    if (shape) {
+      EXPLAIN << "\nin "
+              << (shape->has_layout() ? ShapeUtil::HumanStringWithLayout(*shape)
+                                      : ShapeUtil::HumanString(*shape));
+    }
     return false;
   }
 
@@ -571,9 +977,16 @@ class ShapePattern {
       }
       return true;
     }
+    EXPLAIN << "\nin "
+            << (shape->has_layout() ? ShapeUtil::HumanStringWithLayout(*shape)
+                                    : ShapeUtil::HumanString(*shape));
     return false;
   }
 
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    return impl_.DescribeTo(os, indent);
+  }
+
   // Modifies the pattern to match only if the shape equals the given proto.
   // The layout must outlive the returned pattern.
   constexpr auto EqualTo(const ::xla::Shape* shape) const
@@ -612,6 +1025,11 @@ class ShapePattern {
     return AppendImpl(ShapePatternIsTupleImpl());
   }
 
+  constexpr auto IsEffectiveScalar() const
+      -> decltype(this->AppendImpl(ShapePatternEffectiveScalarImpl())) {
+    return AppendImpl(ShapePatternEffectiveScalarImpl());
+  }
+
   // Modifies the pattern to match only if the shape has the given rank.
   constexpr auto WithRank(int64 rank) const
       -> decltype(this->AppendImpl(ShapePatternRankImpl(rank))) {
@@ -706,6 +1124,15 @@ Shape(::xla::Shape** matched_shape) {
 
 namespace detail {
 
+// Overloads to get a const or non-const operand out of an instruction.
+inline HloInstruction* HloOperand(HloInstruction* instr, int64 idx) {
+  return instr->mutable_operand(idx);
+}
+inline const HloInstruction* HloOperand(const HloInstruction* instr,
+                                        int64 idx) {
+  return instr->operand(idx);
+}
+
 template <typename HloInstructionType, typename Impl>
 class HloInstructionPattern;
 
@@ -714,8 +1141,18 @@ class HloInstructionPattern;
 class HloInstructionPatternBaseImpl {
  public:
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return inst != nullptr;
+    if (inst == nullptr) {
+      EXPLAIN << "HloInstruction* is null";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "an HloInstruction";
   }
+
+  static constexpr bool kIsTrivialMatcher = true;
 };
 
 // An HloInstructionPattern implementation that matches only if the instruction
@@ -726,13 +1163,44 @@ class HloInstructionPatternNameImpl {
       : name_(name) {}
 
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return inst->name() == name_;
+    if (inst->name() != name_) {
+      EXPLAIN << "HloInstruction not named \"" << name_ << "\"";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "named \"" << name_ << "\"";
   }
 
  private:
   absl::string_view name_;
 };
 
+// An HloInstructionPattern implementation that matches only if the instruction
+// equals a particular pointer.
+class HloInstructionIsImpl {
+ public:
+  explicit HloInstructionIsImpl(const HloInstruction* inst) : inst_(inst) {}
+
+  bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
+    if (inst != inst_) {
+      EXPLAIN << "HloInstruction " << inst << " is not " << inst_ << " ("
+              << inst_->ToShortString() << ")";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "which is " << inst_ << " (" << inst_->ToShortString() << ")";
+  }
+
+ private:
+  const HloInstruction* inst_;
+};
+
 // An HloInstructionPattern implementation that matches only if the instruction
 // has a given opcode.
 class HloInstructionPatternOpcodeImpl {
@@ -742,7 +1210,25 @@ class HloInstructionPatternOpcodeImpl {
       : opcode_(opcode), invert_(invert) {}
 
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return (invert_ ^ (inst->opcode() == opcode_));
+    if (invert_ && inst->opcode() == opcode_) {
+      EXPLAIN << "HloInstruction has opcode " << HloOpcodeString(opcode_)
+              << ", expected anything else";
+      return false;
+    }
+    if (!invert_ && inst->opcode() != opcode_) {
+      EXPLAIN << "HloInstruction doesn't have opcode "
+              << HloOpcodeString(opcode_);
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    if (!invert_) {
+      *os << "with opcode " << HloOpcodeString(opcode_);
+    } else {
+      *os << "with any opcode other than " << HloOpcodeString(opcode_);
+    }
   }
 
  private:
@@ -757,8 +1243,17 @@ class HloInstructionPatternNumOperandsImpl {
   explicit constexpr HloInstructionPatternNumOperandsImpl(int64 num_operands)
       : num_operands_(num_operands) {}
 
-  bool Match(const ::xla::HloInstruction* inst, MatchOption /*option*/) const {
-    return inst->operand_count() == num_operands_;
+  bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
+    if (inst->operand_count() != num_operands_) {
+      EXPLAIN << "HloInstruction doesn't have " << num_operands_ << " operands";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with " << num_operands_ << " operand"
+        << (num_operands_ != 1 ? "s" : "");
   }
 
  private:
@@ -775,11 +1270,25 @@ class HloInstructionPatternShapeImpl {
       : shape_(shape) {}
 
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return shape_.Match(&inst->shape(), option);
+    if (!shape_.Match(&inst->shape(), option)) {
+      EXPLAIN << "\nin output shape";
+      return false;
+    }
+    return true;
   }
 
   bool Match(::xla::HloInstruction* inst, MatchOption option) const {
-    return shape_.Match(inst->mutable_shape(), option);
+    if (!shape_.Match(inst->mutable_shape(), option)) {
+      EXPLAIN << "\nin output shape";
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "outputting";
+    Indent(os, indent + kIndentInc);
+    shape_.DescribeTo(os, indent + kIndentInc);
   }
 
  private:
@@ -797,20 +1306,197 @@ class HloInstructionPatternOperandImpl {
       : operand_index_(operand_index), operand_(operand) {}
 
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return operand_index_ < inst->operand_count() &&
-           operand_.Match(inst->operand(operand_index_), option);
+    return MatchImpl(inst, option);
   }
 
   bool Match(::xla::HloInstruction* inst, MatchOption option) const {
-    return operand_index_ < inst->operand_count() &&
-           operand_.Match(inst->mutable_operand(operand_index_), option);
+    return MatchImpl(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with operand " << operand_index_ << " which is:";
+    Indent(os, indent + kIndentInc);
+    operand_.DescribeTo(os, indent + kIndentInc);
   }
 
  private:
+  template <typename HloInstructionType>
+  bool MatchImpl(HloInstructionType* inst, MatchOption option) const {
+    if (operand_index_ >= inst->operand_count()) {
+      EXPLAIN << "desired operand index " << operand_index_
+              << " is out of bounds";
+      return false;
+    }
+    if (!operand_.Match(HloOperand(inst, operand_index_), option)) {
+      EXPLAIN << "\nin operand " << operand_index_;
+      return false;
+    }
+    return true;
+  }
+
   int64 operand_index_;
   HloInstructionPattern<OperandType, OperandImpl> operand_;
 };
 
+// Matches a binary instruction whose operands come in any order.
+template <typename OperandType1, typename OperandImpl1, typename OperandType2,
+          typename OperandImpl2>
+class HloInstructionPatternBinaryOperandsAnyOrderImpl {
+ public:
+  explicit constexpr HloInstructionPatternBinaryOperandsAnyOrderImpl(
+      const HloInstructionPattern<OperandType1, OperandImpl1>& op1,
+      const HloInstructionPattern<OperandType2, OperandImpl2>& op2)
+      : op1_(op1), op2_(op2) {}
+
+  bool Match(HloInstruction* inst, MatchOption option) const {
+    return MatchImpl(inst, option);
+  }
+
+  bool Match(const HloInstruction* inst, MatchOption option) const {
+    return MatchImpl(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with two operands in either order:";
+    Indent(os, indent);
+    *os << " - ";
+    op1_.DescribeTo(os, indent + 3);
+    Indent(os, indent);
+    *os << " - ";
+    op2_.DescribeTo(os, indent + 3);
+  }
+
+ private:
+  HloInstruction* operand(HloInstruction* inst, int64 idx) const {
+    return inst->mutable_operand(idx);
+  }
+  const HloInstruction* operand(const HloInstruction* inst, int64 idx) const {
+    return inst->operand(idx);
+  }
+
+  template <typename HloInstructionType>
+  bool MatchImpl(HloInstructionType* inst, MatchOption option) const {
+    // We could implement this using AnyOf and AllOf matchers, but the templates
+    // get pretty difficult to debug, since any compile error herein becomes
+    // not-an-error via SFINAE.  Also this way lets us give better messages on
+    // failure.
+    if (inst->operand_count() != 2) {
+      EXPLAIN << "HloInstruction did not have two operands";
+      return false;
+    }
+
+    // If we're not generating explanations, this is pretty simple.
+    if (!option.explain_os) {
+      auto try_match = [&](int64 idx1, int64 idx2) {
+        MatchOption new_option = option;
+        new_option.capture = false;
+        if (op1_.Match(operand(inst, idx1), new_option) &&
+            op2_.Match(operand(inst, idx2), new_option)) {
+          if (option.capture) {
+            bool matched = op1_.Match(operand(inst, idx1), option) &&
+                           op2_.Match(operand(inst, idx2), option);
+            DCHECK(matched);
+          }
+          return true;
+        }
+        return false;
+      };
+      return try_match(0, 1) || try_match(1, 0);
+    }
+
+    // If we are generating explanations, we have some work to do in order to
+    // generate a helpful error.
+    //
+    // First, try all four operand/matcher combinations, recording the
+    // failure explanations separately from option.explain_os. matches[i][j]
+    // tells us if matcher_i matches operand j.
+    bool matches[/*matcher*/ 2][/*operand*/ 2];
+    std::stringstream explanations[/*matcher*/ 2][/*operand*/ 2];
+    for (int i = 0; i < 2; ++i) {
+      for (int j = 0; j < 2; ++j) {
+        MatchOption new_option = option;
+        new_option.capture = false;
+        new_option.explain_os = &explanations[i][j];
+        matches[i][j] = i == 0 ? op1_.Match(operand(inst, j), new_option)
+                               : op2_.Match(operand(inst, j), new_option);
+      }
+    }
+
+    // Check if the match succeeded.
+    for (int i = 0; i < 2; ++i) {
+      if (matches[0][i] && matches[1][(i + 1) % 2]) {
+        // Rerun the matches with capture enabled if necessary.
+        if (option.capture) {
+          auto* operand1 = operand(inst, i);
+          auto* operand2 = operand(inst, (i + 1) % 2);
+          bool matched =
+              op1_.Match(operand1, option) && op2_.Match(operand2, option);
+          DCHECK(matched);
+        }
+        return true;
+      }
+    }
+
+    auto describe_matcher = [&](int matcher_idx) {
+      EXPLAIN << "\n - ";
+      if (matcher_idx == 0) {
+        op1_.DescribeTo(option.explain_os, /*indent=*/3);
+      } else {
+        CHECK_EQ(matcher_idx, 1);
+        op2_.DescribeTo(option.explain_os, /*indent=*/3);
+      }
+      for (int i = 0; i < 2; ++i) {
+        if (matches[matcher_idx][/*operand*/ i]) {
+          continue;
+        }
+        EXPLAIN << "\ndoes not match " << (i == 0 ? "LHS" : "RHS") << ":\n";
+        EXPLAIN << " - ";
+        EXPLAIN << absl::StrReplaceAll(
+            explanations[matcher_idx][/*operand*/ i].str(), {{"\n", "\n   "}});
+      }
+    };
+
+    // If we failed to match, one of the following is true:
+    //  1. op1 (op2) matches neither LHS nor RHS, or
+    //  2. op1 and op2 both match LHS (RHS), but neither matches RHS (LHS).
+    // We print different explanations depending on which case we're in.
+
+    // Case 1.
+    bool wrote_explanation = false;
+    for (int i = 0; !wrote_explanation && i < 2; ++i) {
+      if (!matches[i][0] && !matches[i][1]) {
+        EXPLAIN << "HloInstruction's operands (ignoring order) did not match "
+                << (i == 0 ? "first" : "second") << " matcher.  Specifically,";
+        describe_matcher(i);
+        wrote_explanation = true;
+      }
+    }
+
+    // Case 2.
+    for (int i = 0; !wrote_explanation && i < 2; ++i) {
+      if (matches[/*matcher*/ 0][/*operand*/ i] &&
+          matches[/*matcher*/ 1][/*operand*/ i]) {
+        CHECK(!matches[0][(i + 1) % 2]);
+        CHECK(!matches[1][(i + 1) % 2]);
+        CHECK(!wrote_explanation);
+        EXPLAIN << "HloInstruction's " << (i == 1 ? "LHS" : "RHS")
+                << " operand did not match either of the two matchers.  "
+                   "Specifically,";
+        describe_matcher(0);
+        EXPLAIN << "\nand";
+        describe_matcher(1);
+        wrote_explanation = true;
+      }
+    }
+
+    CHECK(wrote_explanation);
+    return false;
+  }
+
+  HloInstructionPattern<OperandType1, OperandImpl1> op1_;
+  HloInstructionPattern<OperandType2, OperandImpl2> op2_;
+};
+
 // An HloInstructionPattern implementation that matches only if the instruction
 // is a fusion node with a particular kind.
 class HloInstructionPatternFusionKindImpl {
@@ -820,14 +1506,32 @@ class HloInstructionPatternFusionKindImpl {
       : kind_(kind) {}
 
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return inst->opcode() == HloOpcode::kFusion && inst->fusion_kind() == kind_;
+    return MatchImpl(inst, option);
   }
 
   bool Match(::xla::HloInstruction* inst, MatchOption option) const {
-    return inst->opcode() == HloOpcode::kFusion && inst->fusion_kind() == kind_;
+    return MatchImpl(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "with fusion kind " << ToString(kind_);
   }
 
  private:
+  template <typename HloInstructionType>
+  bool MatchImpl(HloInstructionType* inst, MatchOption option) const {
+    if (inst->opcode() != HloOpcode::kFusion) {
+      EXPLAIN << "HloInstruction does not have fusion kind " << ToString(kind_)
+              << "; it's not a fusion";
+      return false;
+    }
+    if (inst->fusion_kind() != kind_) {
+      EXPLAIN << "HloInstruction does not have fusion kind " << ToString(kind_);
+      return false;
+    }
+    return true;
+  }
+
   ::xla::HloInstruction::FusionKind kind_;
 };
 
@@ -839,47 +1543,153 @@ class HloInstructionPatternTupleIndexImpl {
       : tuple_index_(tuple_index) {}
 
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
-    return inst->opcode() == HloOpcode::kGetTupleElement &&
-           inst->tuple_index() == tuple_index_;
+    return MatchImpl(inst, option);
   }
 
   bool Match(::xla::HloInstruction* inst, MatchOption option) const {
-    return inst->opcode() == HloOpcode::kGetTupleElement &&
-           inst->tuple_index() == tuple_index_;
+    return MatchImpl(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "which is a GTE with index " << tuple_index_;
   }
 
  private:
+  template <typename HloInstructionType>
+  bool MatchImpl(HloInstructionType* inst, MatchOption option) const {
+    if (inst->opcode() != HloOpcode::kGetTupleElement) {
+      EXPLAIN << "HloInstruction is not a GTE with index " << tuple_index_
+              << "; it's not a GTE at all";
+      return false;
+    }
+    if (inst->tuple_index() != tuple_index_) {
+      EXPLAIN << "HloInstruction is not a GTE with index " << tuple_index_;
+      return false;
+    }
+    return true;
+  }
+
   int64 tuple_index_;
 };
 
-template <typename ItemType, typename Predicate>
-class HloPredicatePatternImpl {
+class HloInstructionPatternParameterNumImpl {
  public:
-  explicit HloPredicatePatternImpl(Predicate pred) : pred_(std::move(pred)) {}
+  explicit constexpr HloInstructionPatternParameterNumImpl(int64 parameter_num)
+      : parameter_num_(parameter_num) {}
 
-  bool Match(const ItemType* item, MatchOption option) const {
-    return pred_(item);
+  bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
+    return MatchImpl(inst, option);
   }
 
-  bool Match(ItemType* item, MatchOption option) const { return pred_(item); }
+  bool Match(::xla::HloInstruction* inst, MatchOption option) const {
+    return MatchImpl(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "which is parameter " << parameter_num_;
+  }
 
  private:
-  Predicate pred_;
+  template <typename HloInstructionType>
+  bool MatchImpl(HloInstructionType* inst, MatchOption option) const {
+    if (inst->opcode() != HloOpcode::kParameter ||
+        inst->parameter_number() != parameter_num_) {
+      EXPLAIN << "HloInstruction is not parameter " << parameter_num_;
+      return false;
+    }
+    return true;
+  }
+
+  int64 parameter_num_;
 };
 
-struct PatternFriend;
+// Matches a constant scalar or effective scalar, optionally with a given value.
+template <typename ScalarTy>
+class HloConstantScalarImpl {
+ public:
+  explicit constexpr HloConstantScalarImpl(bool match_effective_scalar)
+      : val_(absl::nullopt), match_effective_scalar_(match_effective_scalar) {}
+
+  constexpr HloConstantScalarImpl(ScalarTy val, bool match_effective_scalar)
+      : val_(val), match_effective_scalar_(match_effective_scalar) {}
+
+  bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
+    return MatchImpl(inst, option);
+  }
+
+  bool Match(::xla::HloInstruction* inst, MatchOption option) const {
+    return MatchImpl(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "which is a constant "
+        << (match_effective_scalar_ ? "effective " : "") << "scalar";
+    if (val_.has_value()) {
+      *os << " with value " << *val_;
+    }
+  }
+
+ private:
+  template <typename InstTy>
+  bool MatchImpl(InstTy* inst, MatchOption option) const {
+    const auto* const_inst = DynCast<HloConstantInstruction>(inst);
+    if (!const_inst) {
+      EXPLAIN << "HloInstruction is not a constant";
+      return false;
+    }
+    if (match_effective_scalar_ &&
+        !ShapeUtil::IsEffectiveScalar(inst->shape())) {
+      EXPLAIN << "HloInstruction is not an effective scalar";
+      return false;
+    }
+    if (!match_effective_scalar_ && !ShapeUtil::IsScalar(inst->shape())) {
+      EXPLAIN << "HloInstruction is not a scalar";
+      return false;
+    }
+    if (!val_.has_value()) {
+      return true;
+    }
+
+    // Check that literal == static_cast<LitearlTy>(val) and
+    // val == static_cast<ValTy>(literal).  This is sufficient to ensure that
+    // the two constant scalars are actually "equal".
+    auto val_literal = LiteralUtil::CreateR0(*val_);
+    auto literal_r0_or = const_inst->literal().Reshape({});
+    auto val_as_literal_ty_or =
+        val_literal.Convert(const_inst->shape().element_type());
+    if (!literal_r0_or.ok() || !val_as_literal_ty_or.ok()) {
+      EXPLAIN << "could not construct relevant Literals (how did this happen?)";
+      return false;
+    }
+    auto literal_r0 = std::move(literal_r0_or).ValueOrDie();
+    auto val_as_literal_ty = std::move(val_as_literal_ty_or).ValueOrDie();
+    auto literal_r0_as_val_ty_or =
+        literal_r0.Convert(val_literal.shape().element_type());
+    bool rv = literal_r0_as_val_ty_or.ok() &&  //
+              literal_r0_as_val_ty_or.ValueOrDie() == val_literal &&
+              literal_r0 == val_as_literal_ty;
+    if (!rv) {
+      EXPLAIN << "HloInstruction's constant value " << literal_r0.ToString()
+              << " did not match expected value " << *val_;
+    }
+    return rv;
+  }
+
+  absl::optional<ScalarTy> val_;
+  bool match_effective_scalar_;
+};
 
 // A pattern that matches HloInstructions.
 template <typename HloInstructionType, typename Impl>
 class HloInstructionPattern {
  private:
   template <typename NewImpl>
-  HloInstructionPattern<HloInstructionType,
-                        AllOfPattern<::xla::HloInstruction, Impl, NewImpl>>
-  AppendImpl(NewImpl new_impl) const {
-    return HloInstructionPattern<
-        HloInstructionType, AllOfPattern<::xla::HloInstruction, Impl, NewImpl>>(
-        AllOf<HloInstruction>(impl_, std::move(new_impl)), matched_inst_);
+  auto AppendImpl(NewImpl new_impl) const -> HloInstructionPattern<
+      HloInstructionType, decltype(AllOf<HloInstruction>(
+                              std::declval<Impl>(), std::move(new_impl)))> {
+    auto new_allof = AllOf<HloInstruction>(impl_, std::move(new_impl));
+    return HloInstructionPattern<HloInstructionType, decltype(new_allof)>(
+        std::move(new_allof), matched_inst_);
   }
 
  public:
@@ -895,6 +1705,12 @@ class HloInstructionPattern {
       }
       return true;
     }
+    if (inst != nullptr) {
+      EXPLAIN << "\nin "
+              << inst->ToString(HloPrintOptions()
+                                    .set_print_metadata(false)
+                                    .set_print_percent(false));
+    }
     return false;
   }
 
@@ -906,6 +1722,10 @@ class HloInstructionPattern {
       }
       return true;
     }
+    EXPLAIN << "\nin "
+            << inst->ToString(HloPrintOptions()
+                                  .set_print_metadata(false)
+                                  .set_print_percent(false));
     return false;
   }
 
@@ -935,12 +1755,47 @@ class HloInstructionPattern {
     return AppendImpl(HloInstructionPatternOpcodeImpl(opcode, true));
   }
 
+  constexpr auto Is(const HloInstruction* instr) const
+      -> decltype(this->AppendImpl(HloInstructionIsImpl(instr))) {
+    return AppendImpl(HloInstructionIsImpl(instr));
+  }
+
   // Modifies the pattern to match only if the instruction is a constant.
   constexpr auto IsConstant() const
       -> decltype(this->WithOpcode(HloOpcode::kConstant)) {
     return WithOpcode(HloOpcode::kConstant);
   }
 
+  constexpr auto IsConstantScalar() const -> decltype(this->AppendImpl(
+      HloConstantScalarImpl</*Dummy*/ int>(/*match_effective_scalar=*/false))) {
+    return AppendImpl(
+        HloConstantScalarImpl</*Dummy*/ int>(/*match_effective_scalar=*/false));
+  }
+
+  // This does not check that T has the same type as the instruction, so e.g.
+  // IsConstantScalar(1.0) may match a constant of shape int32[].
+  template <typename ScalarTy>
+  constexpr auto IsConstantScalar(const ScalarTy& val) const
+      -> decltype(this->AppendImpl(HloConstantScalarImpl<ScalarTy>(
+          val, /*match_effective_scalar=*/false))) {
+    return AppendImpl(
+        HloConstantScalarImpl<ScalarTy>(val, /*match_effective_scalar=*/false));
+  }
+
+  constexpr auto IsConstantEffectiveScalar() const -> decltype(this->AppendImpl(
+      HloConstantScalarImpl</*Dummy*/ int>(/*match_effective_scalar=*/true))) {
+    return AppendImpl(
+        HloConstantScalarImpl</*Dummy*/ int>(/*match_effective_scalar=*/true));
+  }
+
+  template <typename ScalarTy>
+  constexpr auto IsConstantEffectiveScalar(const ScalarTy& val) const
+      -> decltype(this->AppendImpl(HloConstantScalarImpl<ScalarTy>(
+          val, /*match_effective_scalar=*/true))) {
+    return AppendImpl(
+        HloConstantScalarImpl<ScalarTy>(val, /*match_effective_scalar=*/true));
+  }
+
   // Modifies the pattern to match only if the instruction is not a constant.
   constexpr auto IsNonConstant() const
       -> decltype(this->WithoutOpcode(HloOpcode::kConstant)) {
@@ -957,6 +1812,22 @@ class HloInstructionPattern {
         HloInstructionPatternShapeImpl<ShapeType, ShapeImpl>(shape));
   }
 
+  // Make this a templated function to work around gcc 4.9.4 template infinite
+  // recursion bug.
+  template <typename Dummy = void>
+  constexpr auto WithShapeEqualTo(const ::xla::Shape* shape)
+      -> decltype(this->WithShape(Shape().EqualTo(shape))) {
+    return WithShape(Shape().EqualTo(shape));
+  }
+
+  // Make this a templated function to work around gcc 4.9.4 template infinite
+  // recursion bug.
+  template <typename Dummy = void>
+  constexpr auto WithShapeCompatibleTo(const ::xla::Shape* shape)
+      -> decltype(this->WithShape(Shape().CompatibleTo(shape))) {
+    return WithShape(Shape().CompatibleTo(shape));
+  }
+
   // Modifies the pattern to match only if the instruction has an operand that
   // matches the given pattern.
   template <typename OperandType, typename OperandImpl>
@@ -971,6 +1842,20 @@ class HloInstructionPattern {
             operand_index, operand));
   }
 
+  template <typename OperandType1, typename OperandImpl1, typename OperandType2,
+            typename OperandImpl2>
+  constexpr auto WithBinaryOperandsAnyOrder(
+      const HloInstructionPattern<OperandType1, OperandImpl1>& op1,
+      const HloInstructionPattern<OperandType2, OperandImpl2>& op2) const
+      -> decltype(this->AppendImpl(
+          HloInstructionPatternBinaryOperandsAnyOrderImpl<
+              OperandType1, OperandImpl1, OperandType2, OperandImpl2>(op1,
+                                                                      op2))) {
+    return AppendImpl(
+        HloInstructionPatternBinaryOperandsAnyOrderImpl<
+            OperandType1, OperandImpl1, OperandType2, OperandImpl2>(op1, op2));
+  }
+
   // Modifies the pattern to match only if the instruction is a fusion node with
   // the given kind.
   constexpr auto WithFusionKind(HloInstruction::FusionKind kind) const
@@ -985,17 +1870,18 @@ class HloInstructionPattern {
     return AppendImpl(HloInstructionPatternTupleIndexImpl(tuple_index));
   }
 
- private:
-  template <typename Predicate>
-  constexpr auto WithPredicate(Predicate pred) const -> decltype(
-      this->AppendImpl(HloPredicatePatternImpl<HloInstruction, Predicate>(
-          std::move(pred)))) {
-    return AppendImpl(
-        HloPredicatePatternImpl<HloInstruction, Predicate>(std::move(pred)));
+  // Modifies the pattern to match only if the instruction is a parameter
+  // with the given parameter number.
+  constexpr auto WithParameterNum(int64 parameter_num) const -> decltype(
+      this->AppendImpl(HloInstructionPatternParameterNumImpl(parameter_num))) {
+    return AppendImpl(HloInstructionPatternParameterNumImpl(parameter_num));
   }
 
-  friend struct PatternFriend;
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    impl_.DescribeTo(os, indent);
+  }
 
+ private:
   Impl impl_;
   HloInstructionType** matched_inst_;
 };
@@ -1090,6 +1976,7 @@ XLA_UNOP_PATTERN(Reverse)
 XLA_UNOP_PATTERN(SendDone)
 XLA_UNOP_PATTERN(Sign)
 XLA_UNOP_PATTERN(Sin)
+XLA_UNOP_PATTERN(Slice)
 XLA_UNOP_PATTERN(Sort)
 XLA_UNOP_PATTERN(Tanh)
 XLA_UNOP_PATTERN(Transpose)
@@ -1127,25 +2014,32 @@ XLA_UNOP_PATTERN(Transpose)
 #define XLA_COMMUTATIVE_BINOP_PATTERN(NAME)                                 \
   XLA_BINOP_PATTERN(NAME)                                                   \
                                                                             \
-  template <typename Lhs, typename Rhs>                                     \
-  inline auto NAME##AnyOrder(Lhs&& lhs, Rhs&& rhs)                          \
-      ->decltype(AnyOf<HloInstruction>(NAME(lhs, rhs), NAME(rhs, lhs))) {   \
-    return AnyOf<HloInstruction>(NAME(lhs, rhs), NAME(rhs, lhs));           \
-  }                                                                         \
-                                                                            \
   template <typename HloInstructionType, typename Lhs, typename Rhs>        \
   inline auto NAME##AnyOrder(HloInstructionType** matched_inst, Lhs&& lhs,  \
                              Rhs&& rhs)                                     \
-      ->decltype(AnyOf<HloInstructionType>(NAME(matched_inst, lhs, rhs),    \
-                                           NAME(matched_inst, rhs, lhs))) { \
-    return AnyOf<HloInstructionType>(NAME(matched_inst, lhs, rhs),          \
-                                     NAME(matched_inst, rhs, lhs));         \
+      ->decltype(Op(matched_inst)                                           \
+                     .WithOpcode(HloOpcode::k##NAME)                        \
+                     .WithBinaryOperandsAnyOrder(std::forward<Lhs>(lhs),    \
+                                                 std::forward<Rhs>(rhs))) { \
+    return Op(matched_inst)                                                 \
+        .WithOpcode(HloOpcode::k##NAME)                                     \
+        .WithBinaryOperandsAnyOrder(std::forward<Lhs>(lhs),                 \
+                                    std::forward<Rhs>(rhs));                \
+  }                                                                         \
+  template <typename Lhs, typename Rhs>                                     \
+  inline auto NAME##AnyOrder(Lhs&& lhs, Rhs&& rhs)                          \
+      ->decltype(NAME##AnyOrder<HloInstruction>(                            \
+          nullptr, std::forward<Lhs>(lhs), std::forward<Rhs>(rhs))) {       \
+    return NAME##AnyOrder<HloInstruction>(nullptr, std::forward<Lhs>(lhs),  \
+                                          std::forward<Rhs>(rhs));          \
   }
 XLA_COMMUTATIVE_BINOP_PATTERN(Add)
 XLA_BINOP_PATTERN(Atan2)
 XLA_BINOP_PATTERN(Divide)
 XLA_BINOP_PATTERN(Complex)
+XLA_BINOP_PATTERN(Convolution)
 XLA_BINOP_PATTERN(Dot)
+XLA_BINOP_PATTERN(DynamicSlice)
 XLA_COMMUTATIVE_BINOP_PATTERN(Eq)
 XLA_BINOP_PATTERN(Gather)
 XLA_BINOP_PATTERN(Ge)
@@ -1157,6 +2051,7 @@ XLA_COMMUTATIVE_BINOP_PATTERN(Minimum)
 XLA_COMMUTATIVE_BINOP_PATTERN(Multiply)
 XLA_COMMUTATIVE_BINOP_PATTERN(Ne)
 XLA_BINOP_PATTERN(Outfeed)
+XLA_BINOP_PATTERN(Pad)
 XLA_BINOP_PATTERN(Power)
 XLA_BINOP_PATTERN(Remainder)
 XLA_BINOP_PATTERN(Send)
@@ -1257,31 +2152,9 @@ inline auto WithOperands(Matcher&& m, int64 operand_num, FirstArg&& first_arg,
 // We could implement all ops as "variadic" ops, but it would make the
 // already-bad compile errors even worse.
 XLA_VARIADIC_OP_PATTERN(Concatenate);
+XLA_VARIADIC_OP_PATTERN(CustomCall);
 XLA_VARIADIC_OP_PATTERN(Reduce);
-
-namespace detail {
-struct PatternFriend {
-  template <typename T>
-  static auto ConstantScalar(T constant) -> decltype(
-      Constant()
-          .WithShape(match::Shape().IsScalar())
-          .WithPredicate(
-              std::declval<std::function<bool(const HloInstruction*)>>())) {
-    std::function<bool(const HloInstruction*)> pred =
-        [constant](const HloInstruction* instr) {
-          const auto& literal = Cast<HloConstantInstruction>(instr)->literal();
-          auto status_or_const = LiteralUtil::CreateR0(constant).Convert(
-              literal.shape().element_type());
-          return status_or_const.ok() &&
-                 literal == status_or_const.ConsumeValueOrDie();
-        };
-
-    return Constant()
-        .WithShape(match::Shape().IsScalar())
-        .WithPredicate(std::move(pred));
-  }
-};
-}  // namespace detail
+XLA_VARIADIC_OP_PATTERN(Tuple);
 
 // Helpers for matching non-constant instructions.
 inline auto NonConstant() -> decltype(Op().IsNonConstant()) {
@@ -1320,14 +2193,71 @@ inline auto GetTupleElement(HloInstructionType** matched_inst, Arg&& arg,
       .WithTupleIndex(tuple_index);
 }
 
-template <typename T>
-inline auto ConstantScalar(T constant)
-    -> decltype(detail::PatternFriend::ConstantScalar(constant)) {
-  return detail::PatternFriend::ConstantScalar(constant);
+// Add overloads for Parameter which take an int64 specifying the parameter
+// number.
+inline auto Parameter(int64 parameter_num) -> decltype(
+    Op().WithOpcode(HloOpcode::kParameter).WithParameterNum(parameter_num)) {
+  return Op().WithOpcode(HloOpcode::kParameter).WithParameterNum(parameter_num);
+}
+template <typename HloInstructionType>
+inline auto Parameter(HloInstructionType** matched_inst, int64 parameter_num)
+    -> decltype(Op(matched_inst)
+                    .WithOpcode(HloOpcode::kParameter)
+                    .WithParameterNum(parameter_num)) {
+  return Op(matched_inst)
+      .WithOpcode(HloOpcode::kParameter)
+      .WithParameterNum(parameter_num);
+}
+
+inline auto ConstantScalar() -> decltype(Op().IsConstantScalar()) {
+  return Op().IsConstantScalar();
+}
+
+template <typename HloInstructionType>
+inline auto ConstantScalar(HloInstructionType** matched_inst)
+    -> decltype(Op(matched_inst).IsConstantScalar()) {
+  return Op(matched_inst).IsConstantScalar();
+}
+
+template <typename ScalarTy>
+inline auto ConstantScalar(ScalarTy val)
+    -> decltype(Op().IsConstantScalar(val)) {
+  return Op().IsConstantScalar(val);
+}
+
+template <typename HloInstructionType, typename ScalarTy>
+inline auto ConstantScalar(HloInstructionType** matched_inst, ScalarTy val)
+    -> decltype(Op(matched_inst).IsConstantScalar(val)) {
+  return Op(matched_inst).IsConstantScalar(val);
+}
+
+inline auto ConstantEffectiveScalar() -> decltype(Op().IsConstantScalar()) {
+  return Op().IsConstantEffectiveScalar();
+}
+
+template <typename HloInstructionType>
+inline auto ConstantEffectiveScalar(HloInstructionType** matched_inst)
+    -> decltype(Op(matched_inst).IsConstantScalar()) {
+  return Op(matched_inst).IsConstantEffectiveScalar();
+}
+
+template <typename ScalarTy>
+inline auto ConstantEffectiveScalar(ScalarTy val)
+    -> decltype(Op().IsConstantEffectiveScalar(val)) {
+  return Op().IsConstantEffectiveScalar(val);
+}
+
+template <typename HloInstructionType, typename ScalarTy>
+inline auto ConstantEffectiveScalar(HloInstructionType** matched_inst,
+                                    ScalarTy val)
+    -> decltype(Op(matched_inst).IsConstantEffectiveScalar(val)) {
+  return Op(matched_inst).IsConstantEffectiveScalar(val);
 }
 
 }  // namespace match
 
 }  // namespace xla
 
+#undef EXPLAIN
+#pragma pop_macro("EXPLAIN")
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_H_
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_gmock.h b/tensorflow/compiler/xla/service/pattern_matcher_gmock.h
new file mode 100644
index 0000000000..8fe2d10a11
--- /dev/null
+++ b/tensorflow/compiler/xla/service/pattern_matcher_gmock.h
@@ -0,0 +1,92 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_GMOCK_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_GMOCK_H_
+
+#include <ostream>
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+
+namespace pattern_matcher_gmock_detail {
+template <typename Pattern>
+class GmockMatcher {
+ public:
+  explicit GmockMatcher(Pattern p) : pattern_(std::move(p)) {}
+
+  // In service of better error messages, list out the overloads explicitly
+  // rather than just using a template.  gMock's polymorphism plus
+  // pattern_matcher yields some pretty gnarly stuff.
+  bool MatchAndExplain(const Layout& l,
+                       ::testing::MatchResultListener* listener) const {
+    return MatchAndExplainImpl(&l, listener);
+  }
+  bool MatchAndExplain(const Layout* l,
+                       ::testing::MatchResultListener* listener) const {
+    return MatchAndExplainImpl(l, listener);
+  }
+
+  bool MatchAndExplain(const Shape& s,
+                       ::testing::MatchResultListener* listener) const {
+    return MatchAndExplainImpl(&s, listener);
+  }
+  bool MatchAndExplain(const Shape* s,
+                       ::testing::MatchResultListener* listener) const {
+    return MatchAndExplainImpl(s, listener);
+  }
+
+  bool MatchAndExplain(const HloInstruction& instr,
+                       ::testing::MatchResultListener* listener) const {
+    return MatchAndExplainImpl(&instr, listener);
+  }
+  bool MatchAndExplain(const HloInstruction* instr,
+                       ::testing::MatchResultListener* listener) const {
+    return MatchAndExplainImpl(instr, listener);
+  }
+
+  void DescribeTo(std::ostream* os) const { pattern_.DescribeTo(os); }
+
+  void DescribeNegationTo(std::ostream* os) const {
+    *os << "is NOT: ";
+    DescribeTo(os);
+  }
+
+ private:
+  template <typename T>
+  bool MatchAndExplainImpl(const T* t,
+                           ::testing::MatchResultListener* listener) const {
+    MatchOption options{/*.capture=*/true, /*.explain_os=*/listener->stream()};
+    return Match(t, pattern_, options);
+  }
+
+  Pattern pattern_;
+};
+}  // namespace pattern_matcher_gmock_detail
+
+template <typename Pattern>
+::testing::PolymorphicMatcher<
+    pattern_matcher_gmock_detail::GmockMatcher<Pattern>>
+GmockMatch(Pattern&& p) {
+  return ::testing::MakePolymorphicMatcher(
+      pattern_matcher_gmock_detail::GmockMatcher<Pattern>(
+          std::forward<Pattern>(p)));
+}
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_PATTERN_MATCHER_GMOCK_H_
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
new file mode 100644
index 0000000000..9ca2fb05c1
--- /dev/null
+++ b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
@@ -0,0 +1,76 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace {
+
+namespace m = ::xla::match;
+using ::testing::Eq;
+using ::testing::Not;
+
+template <typename MatchedTy>
+string Describe(const ::testing::Matcher<MatchedTy>& m) {
+  std::stringstream ss;
+  m.DescribeTo(&ss);
+  return ss.str();
+}
+
+template <typename MatchedTy>
+string Explain(
+    const MatchedTy& val,
+    const ::testing::Matcher<typename std::remove_cv<MatchedTy>::type>& m) {
+  ::testing::StringMatchResultListener listener;
+  EXPECT_THAT(val, ::testing::Not(m));  // For the error message.
+  EXPECT_FALSE(m.MatchAndExplain(val, &listener));
+  return listener.str();
+}
+
+// This file tests the GmockMatch function.  The actual explanation and
+// description returned by matchers is tested in pattern_matchers_test.
+TEST(PatternMatcherGmock, MatchShape) {
+  Shape s = ShapeUtil::MakeShape(F32, {10, 100});
+  // You can pass const Shape& or a const Shape*.
+  EXPECT_THAT(s, GmockMatch(m::Shape()));
+  EXPECT_THAT(&s, Not(GmockMatch(m::Shape().WithElementType(F16))));
+  EXPECT_THAT(Describe<Shape>(GmockMatch(m::Shape().IsArray())),
+              "a shape that represents an array");
+}
+
+TEST(PatternMatcherGmock, MatchLayout) {
+  Layout l = LayoutUtil::MakeLayout({0, 1});
+  EXPECT_THAT(l, GmockMatch(m::Layout()));
+  EXPECT_THAT(&l, Not(GmockMatch(m::Layout().WithSparseFormat())));
+  EXPECT_THAT(Describe<Layout>(GmockMatch(m::Layout().WithSparseFormat())),
+              "a layout with format SPARSE");
+}
+
+TEST(PatternMatchGmock, MatchInstruction) {
+  auto instr =
+      HloInstruction::CreateParameter(0, ShapeUtil::MakeShape(F32, {42}), "p");
+  EXPECT_THAT(instr.get(), GmockMatch(m::Parameter()));
+  EXPECT_THAT(*instr, GmockMatch(m::Parameter(0)));
+  EXPECT_THAT(*instr, Not(GmockMatch(m::Parameter(1))));
+  EXPECT_THAT(Describe<HloInstruction*>(GmockMatch(m::Parameter())),
+              "an HloInstruction with opcode parameter");
+}
+
+}  // anonymous namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_test.cc
index 3f74273517..13886fa6f5 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher_test.cc
+++ b/tensorflow/compiler/xla/service/pattern_matcher_test.cc
@@ -14,14 +14,18 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace xla {
 namespace {
 
+namespace m = match;
+
 TEST(PatternMatcherTest, AddOp) {
   constexpr char kModuleStr[] = R"(HloModule two_plus_two_module
     ENTRY %two_plus_two_computation () -> f32[] {
@@ -229,23 +233,74 @@ TEST(PatternMatcherTest, AnyOf) {
 }
 
 TEST(PatternMatcherTest, ConstantScalar) {
-  constexpr char kModuleStr[] = R"(
-    HloModule test_module ENTRY test { ROOT constant = f16[] constant(42) })";
-  TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, ParseHloString(kModuleStr));
-  auto* root = hlo_module->entry_computation()->root_instruction();
-
-  EXPECT_TRUE(Match(root, match::ConstantScalar(42)));
-  EXPECT_FALSE(Match(root, match::ConstantScalar(41)));
-  EXPECT_FALSE(Match(root, match::ConstantScalar(0)));
-}
+  using match::ConstantEffectiveScalar;
+  using match::ConstantScalar;
+  using match::Op;
+  using match::Tuple;
 
-TEST(PatternMatcherTest, NoMatchConstantScalar) {
   constexpr char kModuleStr[] = R"(
-    HloModule test_module ENTRY test { ROOT v = f16[] parameter(0) })";
+    HloModule test_module
+    ENTRY test {
+      a = s32[] constant(1)
+      b = s32[1,1] constant(s32[1,1]{{2}})
+      c = s32[1,2] constant(s32[1,2]{{2,2}})
+      d = f32[] constant(1)
+      e = f32[] constant(1.25)
+      ROOT tuple = (s32[], s32[1,1], s32[1,2], f32[], f32[]) tuple(a,b,c,d,e)
+    })";
   TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, ParseHloString(kModuleStr));
   auto* root = hlo_module->entry_computation()->root_instruction();
 
-  EXPECT_FALSE(Match(root, match::ConstantScalar(42)));
+  const HloInstruction* a = root->operand(0);
+  const HloInstruction* b = root->operand(1);
+  const HloInstruction* c = root->operand(2);
+  const HloInstruction* d = root->operand(3);
+  const HloInstruction* e = root->operand(4);
+  EXPECT_TRUE(Match(a, ConstantScalar()));
+  EXPECT_TRUE(Match(a, ConstantScalar(1)));
+  EXPECT_TRUE(Match(a, ConstantEffectiveScalar()));
+  EXPECT_TRUE(Match(a, ConstantEffectiveScalar(1)));
+  EXPECT_FALSE(Match(a, ConstantScalar(2)));
+  EXPECT_FALSE(Match(a, ConstantScalar(2.01)));
+  EXPECT_FALSE(Match(a, ConstantEffectiveScalar(2)));
+  EXPECT_FALSE(Match(a, ConstantEffectiveScalar(1.01)));
+
+  EXPECT_FALSE(Match(b, ConstantScalar()));
+  EXPECT_FALSE(Match(b, ConstantScalar(2)));
+  EXPECT_TRUE(Match(b, ConstantEffectiveScalar()));
+  EXPECT_TRUE(Match(b, ConstantEffectiveScalar(2)));
+
+  EXPECT_FALSE(Match(c, ConstantScalar()));
+  EXPECT_FALSE(Match(c, ConstantScalar(2)));
+  EXPECT_FALSE(Match(c, ConstantEffectiveScalar()));
+  EXPECT_FALSE(Match(c, ConstantEffectiveScalar(2)));
+
+  EXPECT_TRUE(Match(d, ConstantScalar(1)));
+  EXPECT_TRUE(Match(d, ConstantEffectiveScalar(1)));
+  EXPECT_TRUE(Match(d, ConstantScalar(1.0)));
+  EXPECT_TRUE(Match(d, ConstantEffectiveScalar(1.0)));
+
+  EXPECT_TRUE(Match(e, ConstantScalar(1.25f)));
+  EXPECT_TRUE(Match(e, ConstantScalar(1.25)));
+  EXPECT_TRUE(Match(e, ConstantEffectiveScalar(1.25)));
+  EXPECT_FALSE(Match(e, ConstantScalar(1)));
+  EXPECT_FALSE(Match(e, ConstantEffectiveScalar(1)));
+
+  const HloInstruction* instr = nullptr;
+  EXPECT_TRUE(Match(a, ConstantScalar(&instr)));
+  EXPECT_EQ(instr, a);
+
+  instr = nullptr;
+  EXPECT_TRUE(Match(a, ConstantScalar(&instr, 1)));
+  EXPECT_EQ(instr, a);
+
+  instr = nullptr;
+  EXPECT_TRUE(Match(a, ConstantEffectiveScalar(&instr)));
+  EXPECT_EQ(instr, a);
+
+  instr = nullptr;
+  EXPECT_TRUE(Match(a, ConstantEffectiveScalar(&instr, 1)));
+  EXPECT_EQ(instr, a);
 }
 
 TEST(PatternMatcherTest, MultiplyAnyOrder) {
@@ -267,6 +322,15 @@ TEST(PatternMatcherTest, MultiplyAnyOrder) {
       root, MultiplyAnyOrder(&instr, ConstantScalar(42), ConstantScalar(52))));
   EXPECT_TRUE(Match(
       root, MultiplyAnyOrder(&instr, ConstantScalar(52), ConstantScalar(42))));
+
+  // Check that MultiplyAnyOrder exposes the same API as Op(), so we can call
+  // e.g. IsNonConstant() on it.
+  EXPECT_TRUE(Match(
+      root, MultiplyAnyOrder(&instr, ConstantScalar(42), ConstantScalar(52))
+                .IsNonConstant()));
+  EXPECT_TRUE(
+      Match(root, MultiplyAnyOrder(ConstantScalar(42), ConstantScalar(52))
+                      .IsNonConstant()));
 }
 
 TEST(PatternMatcherTest, AnyOfShortCircuit) {
@@ -315,14 +379,22 @@ TEST(PatternMatcherTest, AllOf) {
   TF_ASSERT_OK_AND_ASSIGN(auto hlo_module, ParseHloString(kModuleStr));
   auto* root = hlo_module->entry_computation()->root_instruction();
 
+  auto f16_scalar = ShapeUtil::MakeShape(F16, {});
+  auto f16_pattern = Constant().WithShapeEqualTo(&f16_scalar);
+  auto f16_compatible_pattern = Constant().WithShapeCompatibleTo(&f16_scalar);
   auto scalar_pattern = Constant().WithShape(match::Shape().IsScalar());
-  auto f16_pattern = Constant().WithShape(match::Shape().WithElementType(F16));
   ASSERT_TRUE(Match(root, scalar_pattern));
   ASSERT_TRUE(Match(root, f16_pattern));
-  EXPECT_TRUE(Match(root, AllOf<HloInstruction>(scalar_pattern, f16_pattern)));
-  EXPECT_TRUE(Match(root, AllOf<HloInstruction>(f16_pattern, scalar_pattern)));
+  ASSERT_TRUE(Match(root, f16_compatible_pattern));
+  EXPECT_TRUE(Match(root, AllOf<HloInstruction>(scalar_pattern, f16_pattern,
+                                                f16_compatible_pattern)));
+  EXPECT_TRUE(
+      Match(root, AllOf<HloInstruction>(f16_pattern, f16_compatible_pattern,
+                                        scalar_pattern)));
   EXPECT_FALSE(
       Match(root, AllOf<HloInstruction>(Broadcast(Op()), f16_pattern)));
+  EXPECT_FALSE(Match(
+      root, AllOf<HloInstruction>(Broadcast(Op()), f16_compatible_pattern)));
   EXPECT_FALSE(
       Match(root, AllOf<HloInstruction>(Broadcast(Op()), scalar_pattern)));
 }
@@ -431,5 +503,377 @@ TEST(PatternMatcherTest, TestConcat) {
                         Reshape(ConstantScalar(4)))));
 }
 
+template <typename Pattern>
+string Description(const Pattern& pattern) {
+  std::stringstream ss;
+  pattern.DescribeTo(&ss);
+  return ss.str();
+}
+
+template <typename Elem, typename Pattern>
+string Explanation(Elem* elem, const Pattern& pattern) {
+  std::stringstream ss;
+  MatchOption options{/*.capture=*/true, /*.explain_os=*/&ss};
+  Match(elem, pattern, options);
+  return ss.str();
+}
+template <typename Elem, typename Pattern>
+string Explanation(const std::unique_ptr<Elem>& elem, const Pattern& pattern) {
+  return Explanation(elem.get(), pattern);
+}
+template <typename Elem, typename Pattern>
+string Explanation(const Elem& elem, const Pattern& pattern) {
+  return Explanation(&elem, pattern);
+}
+
+// Helper macro for checking a pattern's description and the explanation printed
+// when attempting to match (and presumably failing) on a given object.
+//
+// We use a macro rather than a function because we want good line numbers in
+// errors.  We use this rather than writing a helper that returns a pair of
+// (description, explanation) and doing something like
+//
+//   EXPECT_THAT(DescAndExplanation(...), ::testing::Pair(..., ...));
+//
+// because EXPECT_EQ prints a unified diff if multiline string comparison fails,
+// while EXPECT_THAT does not.  This unified diff makes the errors much easier
+// to read.
+#define EXPECT_DESC_AND_EXPLANATION(elem, pattern, expected_desc,    \
+                                    expected_explanation)            \
+  do {                                                               \
+    EXPECT_EQ(Description(pattern), (expected_desc));                \
+    EXPECT_EQ(Explanation((elem), (pattern)), expected_explanation); \
+  } while (0)
+
+TEST(PatternMatcherTest, LayoutDescribeToAndExplain) {
+  auto layout = LayoutUtil::MakeLayout({1, 2});
+  auto layout2 = LayoutUtil::MakeLayout({2, 2});
+
+  EXPECT_DESC_AND_EXPLANATION(static_cast<const Layout*>(nullptr), m::Layout(),
+                              "a layout", "Layout is null");
+  EXPECT_DESC_AND_EXPLANATION(layout2, m::Layout().EqualTo(&layout),
+                              "a layout equal to {1,2}",
+                              "Layout {2,2} is not equal to expected {1,2}");
+  EXPECT_DESC_AND_EXPLANATION(layout2, m::Layout().WithSparseFormat(),
+                              "a layout with format SPARSE",
+                              "Layout has format DENSE but expected SPARSE");
+  EXPECT_DESC_AND_EXPLANATION(layout,
+                              m::Layout().EqualTo(&layout).WithSparseFormat(),
+                              "a layout:\n"
+                              " * equal to {1,2} AND\n"
+                              " * with format SPARSE",
+                              "Layout has format DENSE but expected SPARSE");
+}
+
+TEST(PatternMatcherTest, ShapeDescribeToAndExplain) {
+  auto shape = ShapeUtil::MakeShapeWithLayout(F32, {1, 2}, {0, 1});
+  auto layout = shape.layout();
+
+  EXPECT_DESC_AND_EXPLANATION(static_cast<const Shape*>(nullptr), m::Shape(),
+                              "a shape", "Shape is null");
+  EXPECT_DESC_AND_EXPLANATION(
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 2}, {1, 0}),
+      m::Shape().EqualTo(&shape), "a shape equal to f32[1,2]{0,1}",
+      "Shape not equal to f32[1,2]{0,1}\n"
+      "in f32[1,2]{1,0}");
+  EXPECT_DESC_AND_EXPLANATION(ShapeUtil::MakeShape(F32, {2, 2}),
+                              m::Shape().CompatibleTo(&shape),
+                              "a shape compatible with f32[1,2]",
+                              "Shape not compatible with f32[1,2]\n"
+                              "in f32[2,2]{1,0}");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithElementType(F16),
+                              "a shape with element type F16",
+                              "Shape does not have element type F16\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().IsScalar(),
+                              "a shape that represents a scalar",
+                              "Shape is not a scalar\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(ShapeUtil::MakeNil(), m::Shape().IsArray(),
+                              "a shape that represents an array",
+                              "Shape is not an array\n"
+                              "in ()");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().IsTuple(),
+                              "a shape that represents a tuple",
+                              "Shape is not a tuple\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().IsEffectiveScalar(),
+                              "a shape that is an effective scalar",
+                              "Shape is not an effective scalar\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithRank(42),
+                              "a shape that has 42 dimensions",
+                              "Shape does not have rank 42\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithRank(0),
+                              "a shape that is a scalar",
+                              "Shape is not a scalar\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(shape, m::Shape().WithRank(1).IsArray(),
+                              "a shape:\n"
+                              " * that has 1 dimension AND\n"
+                              " * that represents an array",
+                              "Shape does not have rank 1\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(ShapeUtil::MakeNil(),
+                              m::Shape().IsArray().WithRank(1),
+                              "a shape:\n"
+                              " * that represents an array AND\n"
+                              " * that has 1 dimension",
+                              "Shape is not an array\n"
+                              "in ()");
+  EXPECT_DESC_AND_EXPLANATION(
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 2}, {1, 0}),
+      m::Shape().WithLayoutEqualTo(&layout),
+      "a shape with\n  a layout equal to {0,1}",
+      "Layout {1,0} is not equal to expected {0,1}\n"
+      "in f32[1,2]{1,0}");
+  EXPECT_DESC_AND_EXPLANATION(
+      shape, m::Shape().WithLayout(m::Layout().WithSparseFormat()),
+      "a shape with\n  a layout with format SPARSE",
+      "Layout has format DENSE but expected SPARSE\n"
+      "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(shape,
+                              m::Shape().WithSubshapeEqualTo({10}, &shape),
+                              "a shape with subshape at index {10} which is\n"
+                              "  a shape equal to f32[1,2]{0,1}",
+                              "No subshape at {10}\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(
+      ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {2, 2})}),
+      m::Shape().WithSubshapeEqualTo({0}, &shape),
+      "a shape with subshape at index {0} which is\n"
+      "  a shape equal to f32[1,2]{0,1}",
+      "Shape not equal to f32[1,2]{0,1}\n"
+      "in f32[2,2]{1,0}\n"
+      "in subshape at {0}\n"
+      "in (f32[2,2])");
+  EXPECT_DESC_AND_EXPLANATION(shape,
+                              m::Shape().WithSubshapeCompatibleTo({10}, &shape),
+                              "a shape with subshape at index {10} which is\n"
+                              "  a shape compatible with f32[1,2]",
+                              "No subshape at {10}\n"
+                              "in f32[1,2]{0,1}");
+  EXPECT_DESC_AND_EXPLANATION(
+      ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {2, 2})}),
+      m::Shape().WithSubshapeCompatibleTo({0}, &shape),
+      "a shape with subshape at index {0} which is\n"
+      "  a shape compatible with f32[1,2]",
+      "Shape not compatible with f32[1,2]\n"
+      "in f32[2,2]{1,0}\n"
+      "in subshape at {0}\n"
+      "in (f32[2,2])");
+  EXPECT_DESC_AND_EXPLANATION(
+      ShapeUtil::MakeTupleShape({ShapeUtil::MakeTupleShape({shape})}),
+      m::Shape().WithSubshape({0, 0}, m::Shape().IsScalar()),
+      "a shape with subshape at index {0,0} which is\n"
+      "  a shape that represents a scalar",
+      "Shape is not a scalar\n"
+      "in f32[1,2]{0,1}\n"
+      "in subshape at {0,0}\n"
+      "in ((f32[1,2]))");
+}
+
+std::unique_ptr<HloInstruction> SetName(absl::string_view name,
+                                        std::unique_ptr<HloInstruction> instr) {
+  instr->SetAndSanitizeName(string(name));
+  return instr;
+}
+
+TEST(PatternMatcherTest, HloInstructionDescribeToAndExplain) {
+  std::unique_ptr<HloInstruction> iota =
+      SetName("i", HloInstruction::CreateIota(ShapeUtil::MakeShape(S32, {42}),
+                                              /*iota_dimension=*/0));
+  std::unique_ptr<HloInstruction> constant =
+      SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(0)));
+
+  EXPECT_DESC_AND_EXPLANATION(static_cast<const HloInstruction*>(nullptr),
+                              m::Op(), "an HloInstruction",
+                              "HloInstruction* is null");
+  EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithName("foo"),
+                              "an HloInstruction named \"foo\"",
+                              "HloInstruction not named \"foo\"\n"
+                              "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithOpcode(HloOpcode::kAdd),
+                              "an HloInstruction with opcode add",
+                              "HloInstruction doesn't have opcode add\n"
+                              "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(
+      constant, m::Op().IsNonConstant(),
+      "an HloInstruction with any opcode other than constant",
+      "HloInstruction has opcode constant, expected anything else\n"
+      "in c = s32[] constant(0)");
+  EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithNumOperands(42),
+                              "an HloInstruction with 42 operands",
+                              "HloInstruction doesn't have 42 operands\n"
+                              "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(iota, m::Op().WithShape(m::Shape().IsTuple()),
+                              "an HloInstruction outputting\n"
+                              "  a shape that represents a tuple",
+                              "Shape is not a tuple\n"
+                              "in s32[42]{0}\n"
+                              "in output shape\n"
+                              "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(
+      iota, m::Op().WithOperand(2, m::Op().WithOpcode(HloOpcode::kAdd)),
+      "an HloInstruction with operand 2 which is:\n"
+      "  an HloInstruction with opcode add",
+      "desired operand index 2 is out of bounds\n"
+      "in i = s32[42]{0} iota(), iota_dimension=0");
+
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("a", HloInstruction::CreateBinary(ShapeUtil::MakeShape(S32, {}),
+                                                HloOpcode::kAdd, constant.get(),
+                                                constant.get())),
+      m::Op().WithOperand(1, m::Op().IsNonConstant()),
+      "an HloInstruction with operand 1 which is:\n"
+      "  an HloInstruction with any opcode other than constant",
+      "HloInstruction has opcode constant, expected anything else\n"
+      "in c = s32[] constant(0)\n"
+      "in operand 1\n"
+      "in a = s32[] add(s32[] c, s32[] c)");
+  EXPECT_DESC_AND_EXPLANATION(
+      iota, m::Op().WithFusionKind(HloInstruction::FusionKind::kLoop),
+      "an HloInstruction with fusion kind kLoop",
+      "HloInstruction does not have fusion kind kLoop; it's not a fusion\n"
+      "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(
+      iota, m::Op().WithTupleIndex(42),
+      "an HloInstruction which is a GTE with index 42",
+      "HloInstruction is not a GTE with index 42; it's not a GTE at all\n"
+      "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(iota, m::Op().IsConstantScalar(),
+                              "an HloInstruction which is a constant scalar",
+                              "HloInstruction is not a constant\n"
+                              "in i = s32[42]{0} iota(), iota_dimension=0");
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("c", HloInstruction::CreateConstant(
+                       LiteralUtil::CreateR1<int>({1, 2}))),
+      m::Op().IsConstantEffectiveScalar(),
+      "an HloInstruction which is a constant effective scalar",
+      "HloInstruction is not an effective scalar\n"
+      "in c = s32[2]{0} constant({1, 2})");
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(10))),
+      m::Op().IsConstantScalar(42),
+      "an HloInstruction which is a constant scalar with value 42",
+      "HloInstruction's constant value 10 did not match expected value 42\n"
+      "in c = s32[] constant(10)");
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(2.25))),
+      m::Op().IsConstantEffectiveScalar(1.25),
+      "an HloInstruction which is a constant effective scalar with value 1.25",
+      "HloInstruction's constant value 2.25 did not match expected value 1.25\n"
+      "in c = f64[] constant(2.25)");
+  EXPECT_DESC_AND_EXPLANATION(
+      constant, m::Op().Is(iota.get()),
+      absl::StrCat("an HloInstruction which is 0x", absl::Hex(iota.get()), " (",
+                   iota->ToShortString(), ")"),
+      absl::StrCat("HloInstruction 0x", absl::Hex(constant.get()), " is not 0x",
+                   absl::Hex(iota.get()), " (", iota->ToShortString(), ")\n",
+                   "in c = s32[] constant(0)"));
+}
+
+TEST(PatternMatcherTest, HloInstructionMatcherAnyOrderDescribeTo) {
+  auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("a", HloInstruction::CreateBinary(
+                       scalar_s32, HloOpcode::kAdd,
+                       SetName("b", HloInstruction::CreateConstant(
+                                        LiteralUtil::CreateR0(0)))
+                           .get(),
+                       SetName("c", HloInstruction::CreateConstant(
+                                        LiteralUtil::CreateR0(0)))
+                           .get())),
+      m::AddAnyOrder(m::Op().WithName("b"), m::Op().WithName("bar")),
+      "an HloInstruction:\n"
+      " * with opcode add AND\n"
+      " * with two operands in either order:\n"
+      "    - an HloInstruction named \"b\"\n"
+      "    - an HloInstruction named \"bar\"",
+      "HloInstruction's operands (ignoring order) did not match second "
+      "matcher.  Specifically,\n"
+      " - an HloInstruction named \"bar\"\n"
+      "does not match LHS:\n"
+      " - HloInstruction not named \"bar\"\n"
+      "   in b = s32[] constant(0)\n"
+      "does not match RHS:\n"
+      " - HloInstruction not named \"bar\"\n"
+      "   in c = s32[] constant(0)\n"
+      "in a = s32[] add(s32[] b, s32[] c)");
+
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("a",
+              HloInstruction::CreateBinary(
+                  scalar_s32, HloOpcode::kAdd,
+                  HloInstruction::CreateParameter(0, scalar_s32, "p").get(),
+                  SetName("c", HloInstruction::CreateConstant(
+                                   LiteralUtil::CreateR0(0)))
+                      .get())),
+      m::AddAnyOrder(m::Op().IsConstantScalar(), m::Op().IsConstant()),
+      "an HloInstruction:\n"
+      " * with opcode add AND\n"
+      " * with two operands in either order:\n"
+      "    - an HloInstruction which is a constant scalar\n"
+      "    - an HloInstruction with opcode constant",
+      "HloInstruction's LHS operand did not match either of the two matchers.  "
+      "Specifically,\n"
+      " - an HloInstruction which is a constant scalar\n"
+      "does not match LHS:\n"
+      " - HloInstruction is not a constant\n"
+      "   in p = s32[] parameter(0)\n"
+      "and\n"
+      " - an HloInstruction with opcode constant\n"
+      "does not match LHS:\n"
+      " - HloInstruction doesn't have opcode constant\n"
+      "   in p = s32[] parameter(0)\n"
+      "in a = s32[] add(s32[] p, s32[] c)");
+}
+
+TEST(PatternMatcherTest, AnyOfMatcherDescribeToAndExplain) {
+  EXPECT_DESC_AND_EXPLANATION(
+      SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))),
+      m::AnyOf<HloInstruction>(m::Op().WithName("foo"),
+                               m::Op().WithName("bar")),
+      "any of:\n"
+      " - an HloInstruction named \"foo\" OR\n"
+      " - an HloInstruction named \"bar\"",
+      "None of the following matchers succeeded:\n"
+      "Matcher #1\n"
+      " - an HloInstruction named \"foo\"\n"
+      "failed with\n"
+      " - HloInstruction not named \"foo\"\n"
+      "   in c = s32[] constant(0)\n"
+      "Matcher #2\n"
+      " - an HloInstruction named \"bar\"\n"
+      "failed with\n"
+      " - HloInstruction not named \"bar\"\n"
+      "   in c = s32[] constant(0)");
+}
+
+TEST(PatternMatcherTest, Parameter) {
+  auto param =
+      HloInstruction::CreateParameter(1, ShapeUtil::MakeShape(F32, {}), "p1");
+  auto non_param =
+      SetName("c", HloInstruction::CreateConstant(LiteralUtil::CreateR0(0)));
+  EXPECT_FALSE(Match(param.get(), m::Parameter(0)));
+  EXPECT_TRUE(Match(param.get(), m::Parameter()));
+  EXPECT_TRUE(Match(param.get(), m::Parameter(1)));
+  EXPECT_FALSE(Match(non_param.get(), m::Parameter()));
+  EXPECT_FALSE(Match(non_param.get(), m::Parameter(1)));
+
+  EXPECT_DESC_AND_EXPLANATION(non_param, m::Parameter(1),
+                              "an HloInstruction:\n"
+                              " * with opcode parameter AND\n"
+                              " * which is parameter 1",
+                              "HloInstruction doesn't have opcode parameter\n"
+                              "in c = s32[] constant(0)");
+  EXPECT_EQ(Explanation(HloInstruction::CreateParameter(
+                            0, ShapeUtil::MakeShape(F32, {}), "p0"),
+                        m::Parameter(1)),
+            "HloInstruction is not parameter 1\n"
+            "in p0 = f32[] parameter(0)");
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index c4790a7f19..d30f67dd81 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -526,16 +526,14 @@ static StatusOr<bool> TryPropagateConstant(HloInstruction* while_op) {
   // performance by forcing us to copy constants.
   absl::flat_hash_map<int, const HloInstruction*> index_to_constant;
   for (int i = 0; i < root_operands.size(); i++) {
-    HloInstruction* instr = root_operands[i];
-    if (instr->opcode() == HloOpcode::kGetTupleElement &&
-        instr->tuple_index() == i && instr->operand(0) == while_body_param &&
-        ShapeUtil::IsScalar(instr->shape())) {
-      auto tuple_element = while_init->operand(i);
-      if (tuple_element->IsConstant()) {
-        VLOG(3) << "Found loop invariant tuple element " << i << " "
-                << tuple_element->ToString();
-        index_to_constant[i] = tuple_element;
-      }
+    const HloInstruction* init_tuple_elem = nullptr;
+    if (Match(root_operands[i],
+              m::GetTupleElement(m::Op().Is(while_body_param), i)
+                  .WithShape(m::Shape().IsScalar())) &&
+        Match(while_init->operand(i), m::Constant(&init_tuple_elem))) {
+      VLOG(3) << "Found loop invariant tuple element " << i << " "
+              << init_tuple_elem->ToString();
+      index_to_constant[i] = init_tuple_elem;
     }
   }
 
@@ -793,16 +791,11 @@ static StatusOr<HloInstruction*> TryMergeInductionVariables(
   // Maps the tuple index of each induction variable to its constant increment.
   absl::flat_hash_map<int64, const HloConstantInstruction*> induction_vars;
   for (int64 i = 0; i < while_body_root->operand_count(); ++i) {
-    const auto& elem_shape = while_body_root->operand(i)->shape();
-    if (!ShapeUtil::IsEffectiveScalar(elem_shape) ||
-        elem_shape.element_type() != elem_ty) {
-      continue;
-    }
-
     HloInstruction* constant;
     if (!Match(while_body_root->mutable_operand(i),
                m::AddAnyOrder(m::GetTupleElement(m::Parameter(), i),
-                              m::Constant(&constant)))) {
+                              m::ConstantScalar(&constant))
+                   .WithShape(m::Shape().WithElementType(elem_ty)))) {
       continue;
     }
     if (!trip_counter && constant->literal().IsAll(1) &&
-- 
GitLab


From 2f2931b682634e6c5c88d6bb9fb8ddd971439bc8 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Nov 2018 20:50:18 -0800
Subject: [PATCH 1093/1554] Changing the prefetch_to_device implementation to
 copy_to_device.prefetch instead. This removes the dependency on
 FunctionBufferingResource which we want to deprecate.

PiperOrigin-RevId: 223459089
---
 tensorflow/contrib/eager/python/datasets.py   |  12 -
 .../contrib/eager/python/datasets_test.py     |  13 -
 .../kernel_tests/prefetch_to_device_test.py   |  54 ++--
 .../data/experimental/ops/prefetching_ops.py  | 239 +-----------------
 4 files changed, 19 insertions(+), 299 deletions(-)

diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index db77a39626..34614b86a7 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import prefetching_ops
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -53,17 +52,6 @@ class Iterator(iterator_ops.EagerIterator):
       TypeError: If `dataset` is an unsupported type.
       RuntimeError: When invoked without eager execution enabled.
     """
-    # pylint: disable=protected-access
-    if (isinstance(dataset, prefetching_ops._PrefetchToDeviceDataset)
-        or (isinstance(dataset, dataset_ops.DatasetV1Adapter)
-            and isinstance(
-                dataset._dataset, prefetching_ops._PrefetchToDeviceDataset))):
-      raise TypeError(
-          "`tf.data.experimental.prefetch_to_device()` is not compatible with "
-          "`tf.contrib.eager.Iterator`. Use `for ... in dataset:` to iterate "
-          "over the dataset instead.")
-    # pylint: enable=protected-access
-
     if not context.context().device_spec.device_type:
       is_remote_device = False
     else:
diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py
index 6a508fc6ba..257d02057a 100644
--- a/tensorflow/contrib/eager/python/datasets_test.py
+++ b/tensorflow/contrib/eager/python/datasets_test.py
@@ -26,7 +26,6 @@ import numpy as np
 from tensorflow.contrib import lookup
 from tensorflow.contrib.eager.python import datasets
 from tensorflow.python.data import Dataset
-from tensorflow.python.data.experimental.ops import prefetching_ops
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.experimental.ops import unique
 from tensorflow.python.eager import test
@@ -208,18 +207,6 @@ class IteratorTest(test.TestCase):
         y = math_ops.add(x, x)
     self.assertAllEqual([0., 2.], y.numpy())
 
-  def testTensorsExplicitPrefetchToDevice(self):
-    ds = Dataset.from_tensor_slices([0., 1.])
-    ds = ds.apply(prefetching_ops.prefetch_to_device(test.gpu_device_name()))
-
-    with self.assertRaisesRegexp(TypeError, 'prefetch_to_device'):
-      datasets.Iterator(ds)
-
-    for i, x in enumerate(ds):
-      with ops.device(test.gpu_device_name()):
-        x = math_ops.add(x, x)
-        self.assertEqual(float(i) + float(i), x.numpy())
-
   def testOverrideThreadPool(self):
 
     def get_thread_id(_):
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index 56befc9e7d..c64bb2d299 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -37,12 +37,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
-    # NOTE(mrry): This device block creates the "host" dataset and iterator on
-    # /cpu:0, and ensures that the prefetching is across devices. In typical use
-    # this would not be necessary, because the GPU device would not support any
-    # of the dataset-related ops.
-    with ops.device("/cpu:0"):
+    with ops.device("/cpu:1"):
       iterator = device_dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
     self.assertEqual(host_dataset.output_types, iterator.output_types)
@@ -51,12 +48,11 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
     self.assertEqual(host_dataset.output_classes, iterator.output_classes)
 
-    next_element = iterator.get_next()
     self.assertEqual(dtypes.int64, next_element.dtype)
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -69,12 +65,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.prefetch_to_device(
             "/job:localhost/replica:0/task:0/device:CPU:0"))
 
-    # NOTE(mrry): This device block creates the "host" dataset and iterator on
-    # /cpu:0, and ensures that the prefetching is across devices. In typical use
-    # this would not be necessary, because the GPU device would not support any
-    # of the dataset-related ops.
-    with ops.device("/cpu:0"):
+    with ops.device("/cpu:1"):
       iterator = device_dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
     self.assertEqual(host_dataset.output_types, iterator.output_types)
@@ -83,11 +76,10 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
     self.assertEqual(host_dataset.output_classes, iterator.output_classes)
 
-    next_element = iterator.get_next()
     self.assertEqual(dtypes.int64, next_element.dtype)
     self.assertEqual([], next_element.shape)
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -99,12 +91,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
-    # NOTE(mrry): This device block creates the "host" dataset and iterator on
-    # /cpu:0, and ensures that the prefetching is across devices. In typical use
-    # this would not be necessary, because the GPU device would not support any
-    # of the dataset-related ops.
-    with ops.device("/cpu:0"):
+    with ops.device("/cpu:1"):
       iterator = device_dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
     self.assertEqual(host_dataset.output_types, iterator.output_types)
@@ -113,12 +102,11 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
     self.assertEqual(host_dataset.output_classes, iterator.output_classes)
 
-    next_element = iterator.get_next()
     self.assertEqual(dtypes.int64, next_element["a"].dtype)
     self.assertEqual([], next_element["a"].shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -134,12 +122,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
-    # NOTE(mrry): This device block creates the "host" dataset and iterator on
-    # /cpu:0, and ensures that the prefetching is across devices. In typical use
-    # this would not be necessary, because the GPU device would not support any
-    # of the dataset-related ops.
-    with ops.device("/cpu:0"):
+    with ops.device("/cpu:1"):
       iterator = device_dataset.make_one_shot_iterator()
+      next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
     self.assertEqual(host_dataset.output_types, iterator.output_types)
@@ -148,11 +133,10 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
     self.assertEqual(host_dataset.output_classes, iterator.output_classes)
 
-    next_element = iterator.get_next()
     self.assertEqual(dtypes.int64, next_element.dtype)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
@@ -172,7 +156,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     iterator = device_dataset.make_one_shot_iterator()
     next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -184,12 +168,9 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
-    # NOTE(mrry): This device block creates the "host" dataset and iterator on
-    # /cpu:0, and ensures that the prefetching is across devices. In typical use
-    # this would not be necessary, because the GPU device would not support any
-    # of the dataset-related ops.
-    with ops.device("/cpu:0"):
+    with ops.device("/cpu:1"):
       iterator = device_dataset.make_initializable_iterator()
+      next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
     self.assertEqual(host_dataset.output_types, iterator.output_types)
@@ -198,12 +179,11 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
     self.assertEqual(host_dataset.output_classes, iterator.output_classes)
 
-    next_element = iterator.get_next()
     self.assertEqual(dtypes.int64, next_element.dtype)
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
@@ -224,7 +204,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     iterator = device_dataset.make_initializable_iterator()
     next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 1c4e14ce32..0c14c589f6 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -17,13 +17,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import warnings
-
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
-from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import device as framework_device
 from tensorflow.python.framework import dtypes
@@ -90,239 +87,6 @@ def function_buffering_resource_reset(function_buffer_resource, name=None):
       function_buffer_resource=function_buffer_resource, name=name)
 
 
-# pylint: disable=protected-access
-class _PrefetchToDeviceIterator(object):
-  """A replacement for `tf.data.Iterator` that prefetches to another device.
-
-  Args:
-    input_dataset: The input dataset
-    one_shot: If true, we make a one shot iterator that's already initialized.
-    device: A fully specified device string where we want to prefetch to
-    buffer_size: Size of the prefetching buffer.
-    shared_name: (Optional.) If non-empty, the returned iterator will be
-        shared under the given name across multiple sessions that share the
-        same devices (e.g. when using a remote server).
-
-  Returns:
-    An Iterator type object.
-  """
-
-  def __init__(self,
-               input_dataset,
-               one_shot,
-               device,
-               buffer_size,
-               shared_name=None):
-    self._input_dataset = input_dataset
-    self._get_next_call_count = 0
-    self._one_shot = one_shot
-    if shared_name is None:
-      shared_name = ""
-
-    if self._one_shot:
-      self._input_iterator = input_dataset.make_one_shot_iterator()
-    else:
-      self._input_iterator = iterator_ops.Iterator.from_structure(
-          self._input_dataset.output_types, self._input_dataset.output_shapes,
-          shared_name, self._input_dataset.output_classes)
-    input_iterator_handle = self._input_iterator.string_handle()
-
-    @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
-    # handle is a scalar `tf.Tensor` of type `tf.string`
-    def _prefetch_fn(handle):
-      """Prefetches one element from `input_iterator`."""
-      remote_iterator = iterator_ops.Iterator.from_string_handle(
-          handle, self._input_iterator.output_types,
-          self._input_iterator.output_shapes,
-          self._input_iterator.output_classes)
-      ret = remote_iterator.get_next()
-      return nest.flatten(sparse.serialize_sparse_tensors(ret))
-
-    self._prefetch_fn = _prefetch_fn._get_concrete_function_internal()  # pylint: disable=protected-access
-
-    iterator_device = ged_ops.experimental_iterator_get_device(
-        self._input_iterator._iterator_resource)
-
-    with ops.device(device):
-      self._buffering_resource = function_buffering_resource(
-          f=self._prefetch_fn,
-          target_device=iterator_device,
-          string_arg=input_iterator_handle,
-          buffer_size=buffer_size,
-          shared_name=shared_name,
-          output_types=nest.flatten(
-              sparse.as_dense_types(self._input_dataset.output_types,
-                                    self._input_dataset.output_classes)))
-
-    if not self._one_shot:
-      reset_op = function_buffering_resource_reset(self._buffering_resource)
-      with ops.control_dependencies([reset_op]):
-        self._initializer = self._input_iterator.make_initializer(
-            self._input_dataset)
-
-  def get_next(self, name=None):
-    """See `tf.data.Iterator.get_next`."""
-    self._get_next_call_count += 1
-    if self._get_next_call_count > iterator_ops.GET_NEXT_CALL_WARNING_THRESHOLD:
-      warnings.warn(iterator_ops.GET_NEXT_CALL_WARNING_MESSAGE)
-
-    flat_ret = ged_ops.experimental_function_buffering_resource_get_next(
-        self._buffering_resource,
-        output_types=nest.flatten(
-            sparse.as_dense_types(self.output_types, self.output_classes)),
-        name=name)
-
-    ret = sparse.deserialize_sparse_tensors(
-        nest.pack_sequence_as(self.output_types, flat_ret),
-        self.output_types, self.output_shapes, self.output_classes)
-
-    for tensor, shape in zip(
-        nest.flatten(ret), nest.flatten(self.output_shapes)):
-      if isinstance(tensor, ops.Tensor):
-        tensor.set_shape(shape)
-
-    return ret
-
-  @property
-  def initializer(self):
-    if self._one_shot:
-      raise NotImplementedError("Can't initialize a one_shot_iterator")
-    return self._initializer
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-
-class _PrefetchToDeviceEagerIterator(iterator_ops.EagerIterator):
-  """A replacement for `tf.data.Iterator` that prefetches to another device.
-
-  Args:
-    input_dataset: The input dataset
-    one_shot: If true, we make a one shot iterator that's already initialized.
-    device: A fully specified device string where we want to prefetch to
-    buffer_size: Size of the prefetching buffer.
-    shared_name: (Optional.) If non-empty, the returned iterator will be
-        shared under the given name across multiple sessions that share the
-        same devices (e.g. when using a remote server).
-
-  Returns:
-    An Iterator type object.
-  """
-
-  def __init__(self,
-               input_dataset,
-               device,
-               buffer_size):
-    with ops.device("/device:CPU:0"):
-      super(_PrefetchToDeviceEagerIterator, self).__init__(input_dataset)
-      input_iterator_handle = gen_dataset_ops.iterator_to_string_handle(
-          self._resource)
-
-    self._device = device
-
-    @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
-    def _prefetch_fn(handle):
-      """Prefetches one element from `input_iterator`."""
-      remote_iterator = iterator_ops.Iterator.from_string_handle(
-          handle, self.output_types, self.output_shapes, self.output_classes)
-      ret = remote_iterator.get_next()
-      return nest.flatten(sparse.serialize_sparse_tensors(ret))
-
-    self._prefetch_fn = _prefetch_fn._get_concrete_function_internal()  # pylint: disable=protected-access
-
-    with ops.device(device):
-      self._buffering_resource = function_buffering_resource(
-          f=self._prefetch_fn,
-          output_types=self._flat_output_types,
-          target_device=ged_ops.experimental_iterator_get_device(
-              self._resource),
-          string_arg=input_iterator_handle,
-          buffer_size=buffer_size,
-          shared_name=iterator_ops._generate_shared_name(
-              "function_buffer_resource"))
-
-  def _next_internal(self):
-    """Returns a nested structure of `tf.Tensor`s containing the next element.
-    """
-    # This runs in sync mode as iterators use an error status to communicate
-    # that there is no more data to iterate over.
-    # TODO(b/77291417): Fix
-    with context.execution_mode(context.SYNC):
-      with ops.device(self._device):
-        flat_ret = ged_ops.experimental_function_buffering_resource_get_next(
-            function_buffer_resource=self._buffering_resource,
-            output_types=self._flat_output_types)
-      return self._element_structure._from_tensor_list(flat_ret)
-# pylint: enable=protected-access
-
-
-class _PrefetchToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
-  """A `Dataset` whose iterator prefetches elements to another device."""
-
-  def __init__(self, input_dataset, device, buffer_size):
-    super(_PrefetchToDeviceDataset, self).__init__(input_dataset)
-    self._input_dataset = input_dataset
-    self._device = device
-    self._buffer_size = buffer_size if buffer_size is not None else 1
-
-  # The static analysis cannot tell that the eager iterator's superclass has
-  # a `next()` method.
-  # pylint: disable=non-iterator-returned
-  def __iter__(self):
-    """Creates an `Iterator` for enumerating the elements of this dataset.
-
-    The returned iterator implements the Python iterator protocol and therefore
-    can only be used in eager mode.
-
-    Returns:
-      An `Iterator` over the elements of this dataset.
-
-    Raises:
-      RuntimeError: If eager execution is enabled.
-    """
-    if context.executing_eagerly():
-      return _PrefetchToDeviceEagerIterator(self._input_dataset, self._device,
-                                            self._buffer_size)
-    else:
-      raise RuntimeError("dataset.__iter__() is only supported when eager "
-                         "execution is enabled.")
-  # pylint: enable=non-iterator-returned
-
-  def make_one_shot_iterator(self):
-    if context.executing_eagerly():
-      return _PrefetchToDeviceEagerIterator(self._input_dataset, self._device,
-                                            self._buffer_size)
-    else:
-      return _PrefetchToDeviceIterator(self._input_dataset, one_shot=True,
-                                       device=self._device,
-                                       buffer_size=self._buffer_size)
-
-  def make_initializable_iterator(self, shared_name=None):
-    return _PrefetchToDeviceIterator(
-        self._input_dataset,
-        one_shot=False,
-        device=self._device,
-        buffer_size=self._buffer_size,
-        shared_name=shared_name)
-
-  def _as_variant_tensor(self):
-    # TODO(mrry): Raise this error earlier (e.g. when one of the Dataset
-    # transformation methods is called.
-    # TODO(mrry): Investigate support for chaining further transformations after
-    # the prefetch, including GPU support.
-    raise NotImplementedError("`prefetch_to_device()` must be the last "
-                              "transformation in a dataset pipeline.")
-
-
 @tf_export("data.experimental.prefetch_to_device")
 def prefetch_to_device(device, buffer_size=None):
   """A transformation that prefetches dataset values to the given `device`.
@@ -340,7 +104,8 @@ def prefetch_to_device(device, buffer_size=None):
     `tf.data.Dataset.apply`.
   """
   def _apply_fn(dataset):
-    return _PrefetchToDeviceDataset(dataset, device, buffer_size)
+    return _CopyToDeviceDataset(
+        dataset, target_device=device).prefetch(buffer_size)
 
   return _apply_fn
 
-- 
GitLab


From ecf27945d551bea2ebccb1262d28f2b89ecd61bd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 21:04:42 -0800
Subject: [PATCH 1094/1554] QuantizedDepthwiseConvKernel should be able to
 handle dilation.

PiperOrigin-RevId: 223460312
---
 .../internal/optimized/depthwiseconv_uint8.h  | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
index 5317cea884..d3dca799a7 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
@@ -1499,22 +1499,26 @@ void QuantizedDepthwiseConvAccumRow(int stride, int dilation_factor,
     int out_x_loop_end_unclampled = 0;
     if (kAllowStrided) {
       if (stride == 2) {
-        out_x_loop_start_unclampled = (pad_width - filter_x + 1) / 2;
+        out_x_loop_start_unclampled =
+            (pad_width - dilation_factor * filter_x + 1) / 2;
         out_x_loop_end_unclampled =
-            (pad_width + input_width - filter_x + 1) / 2;
+            (pad_width + input_width - dilation_factor * filter_x + 1) / 2;
       } else if (stride == 4) {
-        out_x_loop_start_unclampled = (pad_width - filter_x + 3) / 4;
+        out_x_loop_start_unclampled =
+            (pad_width - dilation_factor * filter_x + 3) / 4;
         out_x_loop_end_unclampled =
-            (pad_width + input_width - filter_x + 3) / 4;
+            (pad_width + input_width - dilation_factor * filter_x + 3) / 4;
       } else {
         out_x_loop_start_unclampled =
-            (pad_width - filter_x + stride - 1) / stride;
-        out_x_loop_end_unclampled =
-            (pad_width + input_width - filter_x + stride - 1) / stride;
+            (pad_width - dilation_factor * filter_x + stride - 1) / stride;
+        out_x_loop_end_unclampled = (pad_width + input_width -
+                                     dilation_factor * filter_x + stride - 1) /
+                                    stride;
       }
     } else {
-      out_x_loop_start_unclampled = pad_width - filter_x;
-      out_x_loop_end_unclampled = pad_width + input_width - filter_x;
+      out_x_loop_start_unclampled = pad_width - dilation_factor * filter_x;
+      out_x_loop_end_unclampled =
+          pad_width + input_width - dilation_factor * filter_x;
     }
     // The kernel will have to iterate on the segment of the
     // output row that starts at out_x_loop_start and out_x_loop_end.
@@ -1525,7 +1529,8 @@ void QuantizedDepthwiseConvAccumRow(int stride, int dilation_factor,
 
     int32* acc_buffer_ptr =
         acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth;
-    const int in_x_origin = (out_x_loop_start * stride) - pad_width + filter_x;
+    const int in_x_origin =
+        (out_x_loop_start * stride) - pad_width + dilation_factor * filter_x;
     const uint8* input_ptr = input_data + in_x_origin * input_depth;
     const int num_output_pixels = out_x_loop_end - out_x_loop_start;
     QuantizedDepthwiseConvKernel<
@@ -1703,8 +1708,7 @@ inline void DepthwiseConvGeneral(
                                         FIXED_DEPTH_MULTIPLIER)           \
   if (!row_accum_func && (stride_width == 1 || ALLOW_STRIDED) &&          \
       (input_depth == FIXED_INPUT_DEPTH || FIXED_INPUT_DEPTH == 0) &&     \
-      depth_multiplier == FIXED_DEPTH_MULTIPLIER &&                       \
-      dilation_width_factor == 1 && dilation_height_factor == 1) {        \
+      depth_multiplier == FIXED_DEPTH_MULTIPLIER) {                       \
     row_accum_func =                                                      \
         QuantizedDepthwiseConvAccumRow<ALLOW_STRIDED, FIXED_INPUT_DEPTH,  \
                                        FIXED_DEPTH_MULTIPLIER>;           \
-- 
GitLab


From 4186d29da6cdfaaa5e43c46c76ee07c74e7edb13 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Thu, 29 Nov 2018 21:20:48 -0800
Subject: [PATCH 1095/1554] Move training loop from device to host in
 TPUStrategy.

PiperOrigin-RevId: 223461384
---
 .../contrib/distribute/python/keras_test.py   | 10 ++-
 .../contrib/distribute/python/tpu_strategy.py | 71 +++++++++++++------
 .../contrib/tpu/python/tpu/training_loop.py   |  4 +-
 3 files changed, 60 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 435f10358f..1d00281974 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1267,9 +1267,17 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         # We have initialized the model to the same weight for the distribution
         # and non-distribution run.
         model.set_weights(initial_weights)
+        # TODO(b/120245072): Also use gradient_descent_keras.SGD for
+        # TPUStrategy.
+        # pylint: disable=line-too-long
+        if with_distribution and with_distribution.__class__.__name__ == 'TPUStrategy':
+        # pylint: enable=line-too-long
+          optimizer = gradient_descent.GradientDescentOptimizer(0.5)
+        else:
+          optimizer = gradient_descent_keras.SGD(0.5)
         model.compile(
             loss=keras.losses.mean_squared_error,
-            optimizer=gradient_descent_keras.SGD(0.5),
+            optimizer=optimizer,
             distribute=with_distribution)
 
         training_inputs, eval_inputs, predict_inputs = (
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 1f302fdde8..39ed8f7cf1 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -276,9 +276,9 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       initial_loop_values = {}
     initial_loop_values = nest.flatten(initial_loop_values)
     ctx = values.MultiStepContext()
-    def run_fn(*args, **kwargs):
+
+    def run_fn():
       """Single step on the TPU device."""
-      del args, kwargs
       fn_inputs = dequeue_fn()
       if not isinstance(fn_inputs, tuple):
         fn_inputs = (fn_inputs,)
@@ -290,11 +290,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       else:
         return fn_result
 
-    # TODO(sourabhbajaj): The input to while loop should be based on the output
-    # type of the step_fn
-    def iterate_on_tpu():
-      return training_loop.repeat(iterations, run_fn, initial_loop_values)
-
     # We capture the control_flow_context at this point, before we run `fn`
     # inside a while_loop and TPU replicate context. This is useful in cases
     # where we might need to exit these contexts and get back to the outer
@@ -304,24 +299,56 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self._outer_control_flow_context = (
         ops.get_default_graph()._get_control_flow_context())  # pylint: disable=protected-access
 
-    replicate_inputs = [[]] * self._num_replicas_in_sync
-    replicate_outputs = tpu.replicate(iterate_on_tpu, replicate_inputs)
+    def rewrite_fn(*args):
+      """The rewritten step fn running on TPU."""
+      del args
+      replicate_inputs = [[]] * self._num_replicas_in_sync
+      replicate_outputs = tpu.replicate(run_fn, replicate_inputs)
+
+      # If run_fn has tensor outputs, tpu.replicate returns a list of list. We
+      # will flatten it in this case. If run_fn has no tensor outputs,
+      # tpu.replicate returns a list of no_ops, we will keep the output as it
+      # is.
+      if isinstance(replicate_outputs[0], list):
+        replicate_outputs = nest.flatten(replicate_outputs)
+
+      return replicate_outputs
+
+    # TODO(sourabhbajaj): The input to while loop should be based on the output
+    # type of the step_fn
+    assert isinstance(initial_loop_values, list)
+    initial_loop_values = initial_loop_values * self._num_replicas_in_sync
+
+    # Put the while loop op on host 0.
+    with ops.device(self.get_host_cpu_device(0)):
+      replicate_outputs = training_loop.repeat(iterations, rewrite_fn,
+                                               initial_loop_values)
+
     del self._outer_control_flow_context
     ctx.run_op = control_flow_ops.group(replicate_outputs, enqueue_ops)
 
-    # Filter out any ops from the outputs, typically this would be the case
-    # when there were no tensor outputs.
-    last_step_tensor_outputs = [x for x in replicate_outputs
-                                if not isinstance(x, ops.Operation)]
-
-    # Outputs are currently of the structure (grouped by device)
-    # [[output0_device0, output1_device0, output2_device0],
-    #  [output0_device1, output1_device1, output2_device1]]
-    # Convert this to the following structure instead: (grouped by output)
-    # [[output0_device0, output0_device1],
-    #  [output1_device0, output1_device1],
-    #  [output2_device0, output2_device1]]
-    last_step_tensor_outputs = [list(x) for x in zip(*last_step_tensor_outputs)]
+    if isinstance(replicate_outputs, list):
+      # Filter out any ops from the outputs, typically this would be the case
+      # when there were no tensor outputs.
+      last_step_tensor_outputs = [
+          x for x in replicate_outputs if not isinstance(x, ops.Operation)
+      ]
+
+      # Outputs are currently of the structure (flattened)
+      # [output0_device0, output1_device0, output2_device0,
+      #  output0_device1, output1_device1, output2_device1,
+      #  ...]
+      # Convert this to the following structure instead: (grouped by output)
+      # [[output0_device0, output0_device1],
+      #  [output1_device0, output1_device1],
+      #  [output2_device0, output2_device1]]
+      output_num = len(last_step_tensor_outputs) // self._num_replicas_in_sync
+      last_step_tensor_outputs = [
+          last_step_tensor_outputs[i::output_num] for i in range(output_num)
+      ]
+    else:
+      # no tensors returned.
+      last_step_tensor_outputs = []
 
     # Convert replicate_outputs to the original dict structure of
     # last_step_outputs.
diff --git a/tensorflow/contrib/tpu/python/tpu/training_loop.py b/tensorflow/contrib/tpu/python/tpu/training_loop.py
index b6c350ecd7..0187b4bec6 100644
--- a/tensorflow/contrib/tpu/python/tpu/training_loop.py
+++ b/tensorflow/contrib/tpu/python/tpu/training_loop.py
@@ -166,8 +166,8 @@ def while_loop(condition, body, inputs=None, infeed_queue=None, name=None):
   # control dependencies from any side-effecting operations.
   if input_arity == 0:
     inputs = [array_ops.constant(0)]
-  return control_flow_ops.while_loop(condition_wrapper, body_wrapper, inputs,
-                                     name="")
+  return control_flow_ops.while_loop(
+      condition_wrapper, body_wrapper, inputs, name="", parallel_iterations=1)
 
 
 def repeat(n, body, inputs=None, infeed_queue=None, name=None):
-- 
GitLab


From 2520a5e545b2703793facb1cc27598fc2fc083ce Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Nov 2018 21:40:35 -0800
Subject: [PATCH 1096/1554] Getting rid of all FunctionBufferingResource code.

PiperOrigin-RevId: 223462860
---
 ...xperimentalFunctionBufferingResource.pbtxt |  58 ---
 ...ntalFunctionBufferingResourceGetNext.pbtxt |  25 -
 ...mentalFunctionBufferingResourceReset.pbtxt |  13 -
 .../data/experimental/prefetching_kernels.cc  | 428 ------------------
 .../core/ops/compat/ops_history.v1.pbtxt      |  62 ---
 .../core/ops/experimental_dataset_ops.cc      |  21 -
 .../data/experimental/kernel_tests/BUILD      |  21 -
 .../function_buffering_resource_test.py       | 252 -----------
 .../data/experimental/ops/prefetching_ops.py  |  53 ---
 9 files changed, 933 deletions(-)
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResource.pbtxt
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceGetNext.pbtxt
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceReset.pbtxt
 delete mode 100644 tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResource.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResource.pbtxt
deleted file mode 100644
index 66511eff60..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResource.pbtxt
+++ /dev/null
@@ -1,58 +0,0 @@
-op {
-  graph_op_name: "ExperimentalFunctionBufferingResource"
-  in_arg {
-    name: "string_arg"
-    description: <<END
-String argument to the function call.
-END
-  }
-  in_arg {
-    name: "target_device"
-    description: <<END
-Target device to execute the function on.
-END
-  }
-  out_arg {
-    name: "resource"
-    description: <<END
-Handle to the resource created.
-END
-  }
-  attr {
-    name: "shared_name"
-    description: <<END
-If non-empty, this resource will be shared under the given name across
-multiple sessions.
-END
-  }
-  attr {
-    name: "container"
-    description: <<END
-If non-empty, this resource is placed in the given container.
-Otherwise, a default container is used.
-END
-  }
-  attr {
-    name: "f"
-    description: <<END
-Function to be executed.
-END
-  }
-  attr {
-    name: "buffer_size"
-    description: <<END
-Size of the buffer.
-END
-  }
-  attr {
-    name: "output_types"
-    description: <<END
-The type list for the return values.
-END
-  }
-  summary: <<END
-Creates a resource that fills up a buffer by making function calls.
-END
-  visibility: HIDDEN
-}
-
diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceGetNext.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceGetNext.pbtxt
deleted file mode 100644
index bf4b66b22b..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceGetNext.pbtxt
+++ /dev/null
@@ -1,25 +0,0 @@
-op {
-  graph_op_name: "ExperimentalFunctionBufferingResourceGetNext"
-  in_arg {
-    name: "function_buffer_resource"
-    description: <<END
-The FunctionBufferingResource handle.
-END
-  }
-  out_arg {
-    name: "output"
-    description: <<END
-A list of return values.
-END
-  }
-  attr {
-    name: "output_types"
-    description: <<END
-The type list for the return values.
-END
-  }
-  summary: <<END
-Gets the next element from a FunctionBufferingResource.
-END
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceReset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceReset.pbtxt
deleted file mode 100644
index 729718ddb3..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_ExperimentalFunctionBufferingResourceReset.pbtxt
+++ /dev/null
@@ -1,13 +0,0 @@
-op {
-  graph_op_name: "ExperimentalFunctionBufferingResourceReset"
-  in_arg {
-    name: "function_buffer_resource"
-    description: <<END
-The FunctionBufferingResource handle.
-END
-  }
-  summary: <<END
-Resets the FunctionBufferingResource.
-END
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/kernels/data/experimental/prefetching_kernels.cc b/tensorflow/core/kernels/data/experimental/prefetching_kernels.cc
index 2c6179d9f5..af02452098 100644
--- a/tensorflow/core/kernels/data/experimental/prefetching_kernels.cc
+++ b/tensorflow/core/kernels/data/experimental/prefetching_kernels.cc
@@ -27,434 +27,6 @@ namespace tensorflow {
 namespace data {
 namespace {
 
-struct BufferElement {
-  // The producer sets `status` if getting the input element fails.
-  Status status;
-  // The buffered data element.
-  std::vector<Tensor> value;
-};
-
-using FunctionBufferCallback = std::function<void(const BufferElement&)>;
-
-class FunctionBufferingResource : public ResourceBase {
- public:
-  FunctionBufferingResource(FunctionLibraryRuntime* lib,
-                            std::unique_ptr<ProcessFunctionLibraryRuntime> pflr,
-                            const NameAttrList& func, int64 buffer_size,
-                            const string& source_device,
-                            const string& target_device,
-                            const std::vector<Tensor>& func_args,
-                            const DataTypeVector& output_types)
-      : lib_(lib),
-        pflr_(std::move(pflr)),
-        func_(func),
-        buffer_size_(buffer_size),
-        source_device_(source_device),
-        target_device_(target_device),
-        func_args_(func_args),
-        output_types_(output_types),
-        handle_(kInvalidHandle),
-        is_buffering_(false),
-        end_of_sequence_(false),
-        cancelled_(false) {}
-
-  ~FunctionBufferingResource() override {
-    Cancel();
-  }
-
-  string DebugString() override {
-    return strings::StrCat("FunctionBufferingResource. Size: ", buffer_size_,
-                           "; target_device: ", target_device_);
-  }
-
-  // Instantiates the function the first time it's called. After that it caches
-  // the handle.
-  Status Instantiate() LOCKS_EXCLUDED(mu_) {
-    mutex_lock l(mu_);
-    // Re-use existing handle if it's been set, effectively caching it.
-    if (handle_ != kInvalidHandle) {
-      return Status::OK();
-    }
-    AttrValueMap attr_values = func_.attr();
-    FunctionLibraryRuntime::InstantiateOptions opts;
-    opts.target = target_device_;
-    return lib_->Instantiate(func_.name(), AttrSlice(&attr_values), opts,
-                             &handle_);
-  }
-
-  // Returns true if we've got to the end of the sequence and exhausted the
-  // buffer.
-  bool Finished() LOCKS_EXCLUDED(mu_) {
-    mutex_lock l(mu_);
-    return end_of_sequence_ && buffer_.empty();
-  }
-
-  // Cancels any buffering / prefetching going on.
-  void Cancel() LOCKS_EXCLUDED(mu_) {
-    mutex_lock l(mu_);
-    cancelled_ = true;
-    while (is_buffering_) {
-      cond_var_.wait(l);
-    }
-  }
-
-  // Cancels all pending operations and then clears out the state.
-  void Reset() LOCKS_EXCLUDED(mu_) {
-    Cancel();
-    mutex_lock l(mu_);
-    buffer_.clear();
-    requests_.clear();
-    is_buffering_ = false;
-    end_of_sequence_ = false;
-    cancelled_ = false;
-  }
-
-  // If the buffer has anything, runs `callback` on the first element in the
-  // buffer, else schedules the `callback` to be called. Requires `args` and
-  // `lib` in case more function calls need to be scheduled.
-  void MaybeGet(FunctionBufferCallback callback) LOCKS_EXCLUDED(mu_) {
-    bool start_buffering = false;
-    bool produced_output = false;
-    BufferElement buffer_element;
-    {
-      mutex_lock l(mu_);
-      if (!is_buffering_ && !end_of_sequence_) {
-        start_buffering = true;
-      }
-      if (!buffer_.empty()) {
-        produced_output = true;
-        std::swap(buffer_element, buffer_.front());
-        buffer_.pop_front();
-      } else {
-        produced_output = false;
-        requests_.push_back(std::move(callback));
-      }
-    }
-    if (produced_output) {
-      callback(buffer_element);
-    }
-    if (start_buffering) {
-      FillBuffer();
-    }
-  }
-
- private:
-  void FillBuffer() LOCKS_EXCLUDED(mu_) {
-    FunctionLibraryRuntime::Handle handle;
-    std::vector<FunctionBufferCallback> cancellation_callbacks;
-    std::vector<BufferElement> cancellation_buffer_elements;
-    bool cancelled = false;
-    {
-      mutex_lock l(mu_);
-      handle = handle_;
-      if (cancelled_) {
-        cancelled = true;
-        // Run through and fulfill all pending requests, if possible.
-        while (!requests_.empty()) {
-          if (!buffer_.empty()) {
-            cancellation_buffer_elements.push_back(std::move(buffer_.front()));
-            buffer_.pop_front();
-            cancellation_callbacks.push_back(std::move(requests_.front()));
-            requests_.pop_front();
-          } else {
-            LOG(ERROR) << "Buffer ran out of elements and we couldn't satisfy: "
-                       << requests_.size() << " requests";
-            break;
-          }
-        }
-        is_buffering_ = false;
-      } else {
-        is_buffering_ = true;
-      }
-    }
-    if (cancelled) {
-      for (int i = 0; i < cancellation_callbacks.size(); ++i) {
-        cancellation_callbacks[i](cancellation_buffer_elements[i]);
-      }
-      cond_var_.notify_all();
-      return;
-    }
-    FunctionLibraryRuntime::Options opts;
-    // Copied from CapturedFunction::generate_step_id();
-    opts.step_id = -std::abs(static_cast<int64>(random::New64()));
-    opts.source_device = source_device_;
-    AllocatorAttributes arg_alloc_attr;
-    arg_alloc_attr.set_on_host(true);
-    opts.args_alloc_attrs.push_back(arg_alloc_attr);
-    for (const auto& dtype : output_types_) {
-      AllocatorAttributes ret_alloc_attrs;
-      if (DataTypeAlwaysOnHost(dtype)) {
-        ret_alloc_attrs.set_on_host(true);
-      }
-      opts.rets_alloc_attrs.push_back(ret_alloc_attrs);
-    }
-    if (opts.source_device != target_device_) {
-      opts.remote_execution = true;
-    }
-    opts.create_rendezvous = true;
-    auto* rets = new std::vector<Tensor>;
-    lib_->Run(opts, handle, func_args_, rets,
-              [this, rets](const Status& status) {
-                FunctionBufferCallback callback = nullptr;
-                BufferElement buffer_front;
-                bool restart_buffering = false;
-                {
-                  mutex_lock l(mu_);
-                  BufferElement buffer_element;
-                  buffer_element.status = status;
-                  if (status.ok()) {
-                    buffer_element.value.swap(*rets);
-                  } else {
-                    end_of_sequence_ = true;
-                    is_buffering_ = false;
-                  }
-                  buffer_.push_back(std::move(buffer_element));
-                  if (!requests_.empty()) {
-                    buffer_front = std::move(buffer_.front());
-                    buffer_.pop_front();
-                    callback = std::move(requests_.front());
-                    requests_.pop_front();
-                  }
-                  if (buffer_.size() < buffer_size_ && !end_of_sequence_) {
-                    restart_buffering = true;
-                  } else {
-                    // When the buffer is full, we don't want to call
-                    // FillBuffer() unless we're in cancellation phase in which
-                    // case FillBuffer() will do the final cleanup post
-                    // cancellation.
-                    if (cancelled_) {
-                      restart_buffering = true;
-                    }
-                    is_buffering_ = false;
-                  }
-                }
-                if (callback != nullptr) {
-                  callback(buffer_front);
-                }
-                if (restart_buffering) {
-                  FillBuffer();
-                }
-              });
-  }
-
-  mutex mu_;
-  FunctionLibraryRuntime* lib_;
-  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
-  NameAttrList func_;
-  const int64 buffer_size_;
-  const string source_device_;
-  const string target_device_;
-  const std::vector<Tensor> func_args_;
-  const DataTypeVector output_types_;
-  FunctionLibraryRuntime::Handle handle_ GUARDED_BY(mu_);
-  std::deque<BufferElement> buffer_ GUARDED_BY(mu_);
-  std::deque<FunctionBufferCallback> requests_ GUARDED_BY(mu_);
-  bool is_buffering_ GUARDED_BY(mu_);
-  bool end_of_sequence_ GUARDED_BY(mu_);
-  bool cancelled_ GUARDED_BY(mu_);
-  condition_variable cond_var_;
-};
-
-class FunctionBufferResourceHandleOp : public OpKernel {
- public:
-  explicit FunctionBufferResourceHandleOp(OpKernelConstruction* ctx)
-      : OpKernel(ctx), flib_def_(nullptr) {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("buffer_size", &buffer_size_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("container", &container_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("shared_name", &name_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
-  }
-
-  ~FunctionBufferResourceHandleOp() override {
-    if (cinfo_.resource_is_private_to_kernel()) {
-      if (!cinfo_.resource_manager()
-               ->Delete<FunctionBufferingResource>(cinfo_.container(),
-                                                   cinfo_.name())
-               .ok()) {
-        // Do nothing; the resource can have been deleted by session resets.
-      }
-    }
-  }
-
-  void Compute(OpKernelContext* ctx) override {
-    const Tensor* string_arg;
-    OP_REQUIRES_OK(ctx, ctx->input("string_arg", &string_arg));
-    std::vector<Tensor> func_args;
-    func_args.push_back(*string_arg);
-
-    const string& source_device = ctx->device()->name();
-
-    // Obtain and canonicalize target_device.
-    const Tensor* target_arg;
-    OP_REQUIRES_OK(ctx, ctx->input("target_device", &target_arg));
-    string target_device;
-    OP_REQUIRES_OK(ctx, DeviceNameUtils::CanonicalizeDeviceName(
-                            target_arg->scalar<string>()(), source_device,
-                            &target_device));
-
-    FunctionLibraryRuntime* lib = ctx->function_library();
-    OP_REQUIRES(ctx, lib != nullptr,
-                errors::Internal("No function library is provided."));
-
-    mutex_lock l(mu_);
-    if (!initialized_) {
-      OP_REQUIRES_OK(ctx, cinfo_.Init(ctx->resource_manager(), def()));
-      FunctionLibraryRuntime* clone_lib;
-      std::unique_ptr<ProcessFunctionLibraryRuntime> pflr;
-      OP_REQUIRES_OK(ctx, lib->Clone(&flib_def_, &pflr, &clone_lib));
-      // Create the resource.
-      FunctionBufferingResource* buffer;
-      OP_REQUIRES_OK(
-          ctx,
-          ctx->resource_manager()->LookupOrCreate<FunctionBufferingResource>(
-              cinfo_.container(), cinfo_.name(), &buffer,
-              [clone_lib, &pflr, &source_device, &target_device, func_args,
-               this](FunctionBufferingResource** ptr) {
-                *ptr = new FunctionBufferingResource(
-                    clone_lib, std::move(pflr), func_, buffer_size_,
-                    source_device, target_device, func_args, output_types_);
-                return Status::OK();
-              }));
-      core::ScopedUnref s(buffer);
-      OP_REQUIRES_OK(ctx, buffer->Instantiate());
-      initialized_ = true;
-    }
-
-    OP_REQUIRES_OK(ctx, MakeResourceHandleToOutput(
-                            ctx, 0, cinfo_.container(), cinfo_.name(),
-                            MakeTypeIndex<FunctionBufferingResource>()));
-  }
-
- private:
-  mutex mu_;
-  ContainerInfo cinfo_ GUARDED_BY(mu_);
-  bool initialized_ GUARDED_BY(mu_) = false;
-  std::unique_ptr<FunctionLibraryDefinition> flib_def_;
-  NameAttrList func_;
-  int64 buffer_size_;
-  string container_;
-  string name_;
-  DataTypeVector output_types_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResource")
-                            .Device(DEVICE_CPU)
-                            .HostMemory("resource")
-                            .HostMemory("string_arg")
-                            .HostMemory("target_device"),
-                        FunctionBufferResourceHandleOp);
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResource")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("resource")
-                            .HostMemory("string_arg")
-                            .HostMemory("target_device"),
-                        FunctionBufferResourceHandleOp);
-#if TENSORFLOW_USE_SYCL
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResource")
-                            .Device(DEVICE_SYCL)
-                            .HostMemory("resource")
-                            .HostMemory("string_arg")
-                            .HostMemory("target_device"),
-                        FunctionBufferResourceHandleOp);
-#endif  // TENSORFLOW_USE_SYCL
-
-// Prefetches and fills up a buffer by calling a function that provides the
-// elements to buffer.
-class FunctionBufferingResourceGetNextOp : public AsyncOpKernel {
- public:
-  explicit FunctionBufferingResourceGetNextOp(OpKernelConstruction* ctx)
-      : AsyncOpKernel(ctx) {}
-
-  ~FunctionBufferingResourceGetNextOp() override {}
-
-  void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
-    ResourceHandle handle;
-    OP_REQUIRES_OK_ASYNC(
-        ctx, HandleFromInput(ctx, "function_buffer_resource", &handle), done);
-    FunctionBufferingResource* buffer = nullptr;
-    OP_REQUIRES_OK_ASYNC(
-        ctx, LookupResource<FunctionBufferingResource>(ctx, handle, &buffer),
-        done);
-
-    if (buffer->Finished()) {
-      buffer->Unref();
-      ctx->SetStatus(errors::OutOfRange("end_of_sequence"));
-      done();
-      return;
-    }
-
-    FunctionBufferCallback callback =
-        [ctx, buffer, done](const BufferElement& buffer_element) {
-          Status s = buffer_element.status;
-          if (!s.ok()) {
-            ctx->SetStatus(s);
-            buffer->Unref();
-            done();
-            return;
-          }
-          for (size_t i = 0; i < buffer_element.value.size(); ++i) {
-            ctx->set_output(i, buffer_element.value[i]);
-          }
-          buffer->Unref();
-          done();
-        };
-    buffer->MaybeGet(std::move(callback));
-  }
-};
-
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResourceGetNext")
-                            .Device(DEVICE_CPU)
-                            .HostMemory("function_buffer_resource"),
-                        FunctionBufferingResourceGetNextOp);
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResourceGetNext")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("function_buffer_resource"),
-                        FunctionBufferingResourceGetNextOp);
-#if TENSORFLOW_USE_SYCL
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResourceGetNext")
-                            .Device(DEVICE_SYCL)
-                            .HostMemory("function_buffer_resource"),
-                        FunctionBufferingResourceGetNextOp);
-#endif  // TENSORFLOW_USE_SYCL
-
-// Resets the FunctionBufferingResource, cancelling all pending requests and
-// clearing out the buffer.
-class FunctionBufferingResourceResetOp : public OpKernel {
- public:
-  explicit FunctionBufferingResourceResetOp(OpKernelConstruction* ctx)
-      : OpKernel(ctx) {}
-
-  ~FunctionBufferingResourceResetOp() override {}
-
-  void Compute(OpKernelContext* ctx) override {
-    ResourceHandle handle;
-    OP_REQUIRES_OK(ctx,
-                   HandleFromInput(ctx, "function_buffer_resource", &handle));
-    FunctionBufferingResource* buffer = nullptr;
-    OP_REQUIRES_OK(
-        ctx, LookupResource<FunctionBufferingResource>(ctx, handle, &buffer));
-    core::ScopedUnref s(buffer);
-
-    buffer->Reset();
-  }
-};
-
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResourceReset")
-                            .Device(DEVICE_CPU)
-                            .HostMemory("function_buffer_resource"),
-                        FunctionBufferingResourceResetOp);
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResourceReset")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("function_buffer_resource"),
-                        FunctionBufferingResourceResetOp);
-#if TENSORFLOW_USE_SYCL
-REGISTER_KERNEL_BUILDER(Name("ExperimentalFunctionBufferingResourceReset")
-                            .Device(DEVICE_SYCL)
-                            .HostMemory("function_buffer_resource"),
-                        FunctionBufferingResourceResetOp);
-#endif  // TENSORFLOW_USE_SYCL
-
 class IteratorGetDeviceOp : public OpKernel {
  public:
   using OpKernel::OpKernel;
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 6f3db65918..735ba6b0f8 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21725,68 +21725,6 @@ op {
     minimum: 1
   }
 }
-op {
-  name: "ExperimentalFunctionBufferingResource"
-  input_arg {
-    name: "string_arg"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "target_device"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "resource"
-    type: DT_RESOURCE
-  }
-  attr {
-    name: "shared_name"
-    type: "string"
-  }
-  attr {
-    name: "container"
-    type: "string"
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "buffer_size"
-    type: "int"
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-  }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalFunctionBufferingResourceGetNext"
-  input_arg {
-    name: "function_buffer_resource"
-    type: DT_RESOURCE
-  }
-  output_arg {
-    name: "output"
-    type_list_attr: "output_types"
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalFunctionBufferingResourceReset"
-  input_arg {
-    name: "function_buffer_resource"
-    type: DT_RESOURCE
-  }
-  is_stateful: true
-}
 op {
   name: "ExperimentalIdentityIndexedDataset"
   input_arg {
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index aebe0bf984..2fd6cd0cd6 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -131,27 +131,6 @@ REGISTER_OP("ExperimentalIteratorGetDevice")
     .Output("device: string")
     .SetShapeFn(shape_inference::ScalarShape);
 
-REGISTER_OP("ExperimentalFunctionBufferingResource")
-    .Input("string_arg: string")
-    .Input("target_device: string")
-    .Output("resource: resource")
-    .Attr("shared_name: string")
-    .Attr("container: string")
-    .Attr("f: func")
-    .Attr("buffer_size: int")
-    .Attr("output_types: list(type)")
-    .SetShapeFn(shape_inference::UnknownShape);
-
-REGISTER_OP("ExperimentalFunctionBufferingResourceGetNext")
-    .Input("function_buffer_resource: resource")
-    .Attr("output_types: list(type)")
-    .Output("output: output_types")
-    .SetShapeFn(shape_inference::UnknownShape);
-
-REGISTER_OP("ExperimentalFunctionBufferingResourceReset")
-    .Input("function_buffer_resource: resource")
-    .SetShapeFn(shape_inference::UnknownShape);
-
 REGISTER_OP("ExperimentalMaxIntraOpParallelismDataset")
     .Input("input_dataset: variant")
     .Input("max_intra_op_parallelism: int64")
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 6b22f9b8a8..d7ca5a70e4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -149,27 +149,6 @@ py_test(
     ],
 )
 
-cuda_py_test(
-    name = "function_buffering_resource_test",
-    size = "small",
-    srcs = ["function_buffering_resource_test.py"],
-    additional_deps = [
-        "//tensorflow/python/data/experimental/ops:prefetching_ops",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python/data/kernel_tests:test_base",
-        "//tensorflow/python/eager:function",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
-    ],
-    tags = ["no_windows_gpu"],
-)
-
 py_test(
     name = "get_single_element_test",
     size = "small",
diff --git a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py b/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
deleted file mode 100644
index a78fb4d229..0000000000
--- a/tensorflow/python/data/experimental/kernel_tests/function_buffering_resource_test.py
+++ /dev/null
@@ -1,252 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for the private `FunctionBufferingResource` used in prefetching."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import threading
-
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.data.experimental.ops import prefetching_ops
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.eager import function
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_spec
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.platform import test
-
-
-class FunctionBufferingResourceTest(test_base.DatasetTestBase):
-
-  def setUp(self):
-    self._event = threading.Event()
-
-  def _create_ds_and_iterator(self, device0, initializable=False):
-
-    def gen():
-      for i in range(1, 10):
-        yield [float(i)]
-        if i == 6:
-          self._event.set()
-
-    with ops.device(device0):
-      ds = dataset_ops.Dataset.from_generator(gen, (dtypes.float32))
-      if initializable:
-        ds_iterator = ds.make_initializable_iterator()
-      else:
-        ds_iterator = ds.make_one_shot_iterator()
-      return (ds, ds_iterator)
-
-  def _create_ops(self, ds, ds_iterator, buffer_name, device0, device1):
-    ds_iterator_handle = ds_iterator.string_handle()
-
-    @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
-    def _remote_fn(h):
-      remote_iterator = iterator_ops.Iterator.from_string_handle(
-          h, ds.output_types, ds.output_shapes)
-      return remote_iterator.get_next()
-
-    target = constant_op.constant(device0)
-    with ops.device(device1):
-      buffer_resource_handle = prefetching_ops.function_buffering_resource(
-          f=_remote_fn.get_concrete_function(),
-          output_types=[dtypes.float32],
-          target_device=target,
-          string_arg=ds_iterator_handle,
-          buffer_size=3,
-          shared_name=buffer_name)
-
-    with ops.device(device1):
-      prefetch_op = prefetching_ops.function_buffering_resource_get_next(
-          function_buffer_resource=buffer_resource_handle,
-          output_types=[dtypes.float32])
-      reset_op = prefetching_ops.function_buffering_resource_reset(
-          function_buffer_resource=buffer_resource_handle)
-      destroy_op = resource_variable_ops.destroy_resource_op(
-          buffer_resource_handle, ignore_lookup_error=True)
-
-    return (prefetch_op, reset_op, destroy_op)
-
-  def _prefetch_fn_helper_one_shot(self, buffer_name, device0, device1):
-    worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-
-    ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=False)
-    prefetch_op, _, destroy_op = self._create_ops(ds, ds_iterator, buffer_name,
-                                                  device0, device1)
-
-    with self.test_session(config=worker_config) as sess:
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [1.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [2.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [3.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [4.0])
-      self._event.wait()
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [5.0])
-      self.evaluate(destroy_op)
-
-  @test_util.run_deprecated_v1
-  def testSameDeviceCPU(self):
-    self._prefetch_fn_helper_one_shot("same_device_cpu",
-                                      "/job:localhost/replica:0/task:0/cpu:0",
-                                      "/job:localhost/replica:0/task:0/cpu:0")
-
-  @test_util.run_deprecated_v1
-  def testDifferentDeviceCPU(self):
-    self._prefetch_fn_helper_one_shot("diff_device_cpu",
-                                      "/job:localhost/replica:0/task:0/cpu:0",
-                                      "/job:localhost/replica:0/task:0/cpu:1")
-
-  def testDifferentDeviceCPUGPU(self):
-    if not test_util.is_gpu_available():
-      self.skipTest("No GPU available")
-
-    self._prefetch_fn_helper_one_shot("cpu_gpu",
-                                      "/job:localhost/replica:0/task:0/cpu:0",
-                                      "/job:localhost/replica:0/task:0/gpu:0")
-
-  @test_util.run_deprecated_v1
-  def testReinitialization(self):
-    worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-
-    device0 = "/job:localhost/replica:0/task:0/cpu:0"
-    device1 = "/job:localhost/replica:0/task:0/cpu:1"
-    ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=True)
-    prefetch_op, reset_op, destroy_op = self._create_ops(
-        ds, ds_iterator, "reinit", device0, device1)
-
-    with self.test_session(config=worker_config) as sess:
-      self.evaluate(ds_iterator.initializer)
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [1.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [2.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [3.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [4.0])
-      self._event.wait()
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [5.0])
-      # Lets reset the function buffering resource and reinitialize the
-      # iterator. Should be able to go through this again.
-      self._event.clear()
-      self.evaluate(reset_op)
-      self.evaluate(ds_iterator.initializer)
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [1.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [2.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [3.0])
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [4.0])
-      self._event.wait()
-      elem = self.evaluate(prefetch_op)
-      self.assertEqual(elem, [5.0])
-      self.evaluate(destroy_op)
-
-  @test_util.run_deprecated_v1
-  def testReinitializationOutOfRange(self):
-    worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-
-    device0 = "/job:localhost/replica:0/task:0/cpu:0"
-    device1 = "/job:localhost/replica:0/task:0/cpu:1"
-    ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=True)
-    prefetch_op, reset_op, destroy_op = self._create_ops(
-        ds, ds_iterator, "reinit", device0, device1)
-
-    with self.test_session(config=worker_config) as sess:
-      self.evaluate(ds_iterator.initializer)
-      for i in range(1, 10):
-        elem = self.evaluate(prefetch_op)
-        self.assertEqual(elem, [float(i)])
-      # Try fetching after its over twice to test out end of sequence.
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
-
-      # Now reset everything and try it out again.
-      self._event.clear()
-      self.evaluate(reset_op)
-      self.evaluate(ds_iterator.initializer)
-      for i in range(1, 10):
-        elem = self.evaluate(prefetch_op)
-        self.assertEqual(elem, [float(i)])
-      # Try fetching after its over twice to test out end of sequence.
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
-
-      self.evaluate(destroy_op)
-
-  def testStringsGPU(self):
-    if not test_util.is_gpu_available():
-      self.skipTest("No GPU available")
-
-    device0 = "/job:localhost/replica:0/task:0/cpu:0"
-    device1 = "/job:localhost/replica:0/task:0/gpu:0"
-
-    ds = dataset_ops.Dataset.from_tensor_slices(["a", "b", "c"])
-    ds_iterator = ds.make_one_shot_iterator()
-    ds_iterator_handle = ds_iterator.string_handle()
-
-    @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
-    def _remote_fn(h):
-      remote_iterator = iterator_ops.Iterator.from_string_handle(
-          h, ds.output_types, ds.output_shapes)
-      return remote_iterator.get_next()
-
-    target = constant_op.constant(device0)
-    with ops.device(device1):
-      buffer_resource_handle = prefetching_ops.function_buffering_resource(
-          f=_remote_fn.get_concrete_function(),
-          output_types=[dtypes.string],
-          target_device=target,
-          string_arg=ds_iterator_handle,
-          buffer_size=3,
-          shared_name="strings")
-
-    with ops.device(device1):
-      prefetch_op = prefetching_ops.function_buffering_resource_get_next(
-          function_buffer_resource=buffer_resource_handle,
-          output_types=[dtypes.string])
-      destroy_op = resource_variable_ops.destroy_resource_op(
-          buffer_resource_handle, ignore_lookup_error=True)
-
-    with self.cached_session() as sess:
-      self.assertEqual([b"a"], self.evaluate(prefetch_op))
-      self.assertEqual([b"b"], self.evaluate(prefetch_op))
-      self.assertEqual([b"c"], self.evaluate(prefetch_op))
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(prefetch_op)
-
-      self.evaluate(destroy_op)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 0c14c589f6..9e3ca56929 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -34,59 +34,6 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-def function_buffering_resource(string_arg,
-                                target_device,
-                                f,
-                                buffer_size,
-                                output_types,
-                                container="",
-                                shared_name=None,
-                                name=None):
-  """Creates a FunctionBufferingResource.
-
-  A FunctionBufferingResource fills up a buffer by calling a function `f` on
-  `target_device`. `f` should take in only a single string argument as input.
-
-  Args:
-    string_arg: The single string argument to the function.
-    target_device: The device to run `f` on.
-    f: The function to be executed.
-    buffer_size: Size of the buffer to be populated.
-    output_types: The output types generated by the function.
-    container: (Optional) string. Defaults to "".
-    shared_name: (Optional) string.
-    name: (Optional) string to name the op.
-
-  Returns:
-    Handle to a FunctionBufferingResource.
-  """
-  if shared_name is None:
-    shared_name = ""
-  return ged_ops.experimental_function_buffering_resource(
-      string_arg=string_arg,
-      target_device=target_device,
-      shared_name=shared_name,
-      f=f,
-      buffer_size=buffer_size,
-      container=container,
-      name=name,
-      output_types=output_types)
-
-
-def function_buffering_resource_get_next(function_buffer_resource,
-                                         output_types,
-                                         name=None):
-  return ged_ops.experimental_function_buffering_resource_get_next(
-      function_buffer_resource=function_buffer_resource,
-      output_types=output_types,
-      name=name)
-
-
-def function_buffering_resource_reset(function_buffer_resource, name=None):
-  return ged_ops.experimental_function_buffering_resource_reset(
-      function_buffer_resource=function_buffer_resource, name=name)
-
-
 @tf_export("data.experimental.prefetch_to_device")
 def prefetch_to_device(device, buffer_size=None):
   """A transformation that prefetches dataset values to the given `device`.
-- 
GitLab


From ba14318eee9d2bf11d49eadde9a525b5b2cfea02 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 29 Nov 2018 22:05:27 -0800
Subject: [PATCH 1097/1554] Refactor Interpreter & Subgraph. Preparing to
 implement control flow ops.

* Expose a few Subgraph-related functions in Interpreter.
  E.g. AddSubgraphs, subgraphs_size, subgraph, primary_subgraph.
* Moved the logic which checks if dynamic tensors exist from
  ModifyGraphWithDelegate to PrepareOpsStartingAt.
  * This is a no-op refactoring with better code structure.
  * Exposing `Subgraph::HasDynamicTensors()` which will be required
    for control flow ops.

PiperOrigin-RevId: 223464887
---
 tensorflow/lite/core/subgraph.cc | 26 ++++++++++----------------
 tensorflow/lite/core/subgraph.h  | 22 +++++++++++++++++++++-
 tensorflow/lite/interpreter.cc   | 16 +++++++++++++++-
 tensorflow/lite/interpreter.h    | 29 ++++++++++++++++++++++++++---
 4 files changed, 72 insertions(+), 21 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
index 05a60962b1..90361faeae 100644
--- a/tensorflow/lite/core/subgraph.cc
+++ b/tensorflow/lite/core/subgraph.cc
@@ -109,11 +109,13 @@ class InterpreterInfo : public GraphInfo {
 };
 
 Subgraph::Subgraph(ErrorReporter* error_reporter,
-                   TfLiteExternalContext** external_contexts)
+                   TfLiteExternalContext** external_contexts,
+                   std::vector<std::unique_ptr<Subgraph>>* subgraphs)
     : context_(&owned_context_),
       error_reporter_(error_reporter),
       next_execution_plan_index_to_prepare_(0),
-      external_contexts_(external_contexts) {
+      external_contexts_(external_contexts),
+      subgraphs_(subgraphs) {
   context_->impl_ = static_cast<void*>(this);
   context_->ResizeTensor = ResizeTensor;
   context_->ReportError = ReportErrorC;
@@ -559,6 +561,9 @@ TfLiteStatus Subgraph::ResizeInputTensor(int tensor_index,
 
 TfLiteStatus Subgraph::PrepareOpsStartingAt(
     int first_execution_plan_index, int* last_execution_plan_index_prepared) {
+  if (first_execution_plan_index == 0) {
+    has_dynamic_tensors_ = false;
+  }
   for (int execution_plan_index = first_execution_plan_index;
        execution_plan_index < execution_plan_.size(); execution_plan_index++) {
     int node_index = execution_plan_[execution_plan_index];
@@ -577,7 +582,8 @@ TfLiteStatus Subgraph::PrepareOpsStartingAt(
     // stop for dynamic temporary tensors since they won't affect the
     // sizes of other tensors in the graph.
     if (HasDynamicTensor(*context_, node.outputs)) {
-      break;
+      has_dynamic_tensors_ = true;
+      return kTfLiteOk;
     }
   }
   return kTfLiteOk;
@@ -929,19 +935,7 @@ TfLiteStatus Subgraph::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
     int last_execution_plan_index_prepared;
     TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
                                      0, &last_execution_plan_index_prepared));
-
-    bool has_dynamic_tensors = true;
-    // Dynamic tensors exist if not all nodes can be prepared.
-    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
-      // If all the nodes can be prepared, check if the last node has dynamic
-      // tensors.
-      int node_index = execution_plan_[last_execution_plan_index_prepared];
-      TfLiteNode& node = nodes_and_registration_[node_index].first;
-      if (!HasDynamicTensor(*context_, node.outputs)) {
-        has_dynamic_tensors = false;
-      }
-    }
-    if (has_dynamic_tensors) {
+    if (has_dynamic_tensors_) {
       ReportError(
           "Attempting to use a delegate that only supports static-sized "
           "tensors with a graph that has dynamic-sized tensors.");
diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index 6b44a4731c..d8542f9513 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -34,7 +34,9 @@ class Subgraph {
   friend class Interpreter;
 
   Subgraph(ErrorReporter* error_reporter,
-           TfLiteExternalContext** external_contexts);
+           TfLiteExternalContext** external_contexts,
+           std::vector<std::unique_ptr<Subgraph>>* subgraphs);
+
   Subgraph(const Subgraph&) = delete;
 
   // Subgraphs should be movable but not copyable.
@@ -243,6 +245,15 @@ class Subgraph {
 
   profiling::Profiler* GetProfiler() { return profiler_; }
 
+  // Returns a pointer to vector of subgraphs.
+  // WARNING: This is an experimental API and subject to change.
+  std::vector<std::unique_ptr<Subgraph>>* GetSubgraphs() { return subgraphs_; }
+
+  // True if all tensors in the graph has static size after calling
+  // `AllocateTensors` function.
+  // Before `AllocateTensors` is called, this will always return true;
+  bool HasDynamicTensors() { return has_dynamic_tensors_; }
+
  private:
   // Prevent 'context_' from accessing functions that are only available to
   // delegated kernels.
@@ -474,6 +485,15 @@ class Subgraph {
 
   // Profiler for this interpreter instance.
   profiling::Profiler* profiler_ = nullptr;
+
+  // A pointer to vector of subgraphs. The vector is owned by the interpreter.
+  std::vector<std::unique_ptr<Subgraph>>* subgraphs_ = nullptr;
+
+  // True if all tensors in the graph has static size after calling
+  // `PrepareOpsStartingAt` function (which is called by the `AllocateTensors`
+  // public function).
+  // The value is invalid before `PrepareOpStartingAt` is called.
+  bool has_dynamic_tensors_ = true;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 326aff5ce4..e2129ed46d 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -35,7 +35,8 @@ namespace tflite {
 Interpreter::Interpreter(ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  subgraphs_.emplace_back(new Subgraph(error_reporter_, external_contexts_));
+  // There's always at least 1 subgraph which is the primary subgraph.
+  AddSubgraphs(1);
   context_ = primary_subgraph().context();
 
   // Reserve some space for the tensors to avoid excessive resizing.
@@ -73,6 +74,19 @@ void Interpreter::ReserveNodes(int count) {
   primary_subgraph().nodes_and_registration().reserve(count);
 }
 
+void Interpreter::AddSubgraphs(int subgraphs_to_add,
+                               int* first_new_subgraph_index) {
+  const size_t base_index = subgraphs_.size();
+  if (first_new_subgraph_index) *first_new_subgraph_index = base_index;
+
+  subgraphs_.reserve(base_index + subgraphs_to_add);
+  for (int i = 0; i < subgraphs_to_add; ++i) {
+    Subgraph* subgraph =
+        new Subgraph(error_reporter_, external_contexts_, &subgraphs_);
+    subgraphs_.emplace_back(subgraph);
+  }
+}
+
 TfLiteStatus Interpreter::AddNodeWithParameters(
     const std::vector<int>& inputs, const std::vector<int>& outputs,
     const char* init_data, size_t init_data_size, void* builtin_data,
diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h
index 405cf640b9..6192d56ca2 100644
--- a/tensorflow/lite/interpreter.h
+++ b/tensorflow/lite/interpreter.h
@@ -422,18 +422,41 @@ class Interpreter {
   void SetExternalContext(TfLiteExternalContextType type,
                           TfLiteExternalContext* ctx);
 
- private:
-  friend class InterpreterBuilder;
-  friend class InterpreterTest;
+  // Adds `subgraphs_to_add` subgraphs, preserving pre-existing Subgraph
+  // entries. The value pointed to by `first_new_subgraph_index` will be set to
+  // the index of the first new subgraph if `first_new_subgraph_index` is
+  // non-null.
+  // WARNING: This is an experimental API and subject to change.
+  void AddSubgraphs(int subgraphs_to_add,
+                    int* first_new_subgraph_index = nullptr);
 
+  // Return the number of subgraphs in the model.
+  // WARNING: This is an experimental API and subject to change.
+  size_t subgraphs_size() const { return subgraphs_.size(); }
+
+  // Get a pointer to a subgraph if in bounds.
+  // WARNING: This is an experimental API and subject to change.
+  Subgraph* subgraph(int subgraph_index) {
+    if (subgraph_index < 0 ||
+        static_cast<size_t>(subgraph_index) >= subgraphs_size())
+      return nullptr;
+    return &*subgraphs_[subgraph_index];
+  }
+
+  // WARNING: Experimental interface, subject to change
   Subgraph& primary_subgraph() {
     return *subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
+  // WARNING: Experimental interface, subject to change
   const Subgraph& primary_subgraph() const {
     return *subgraphs_.front();  // Safe as subgraphs_ always has 1 entry.
   }
 
+ private:
+  friend class InterpreterBuilder;
+  friend class InterpreterTest;
+
   // Set the value of an external context.
   static void SetExternalContext(struct TfLiteContext* context,
                                  TfLiteExternalContextType type,
-- 
GitLab


From de80b2f523630e9295a2b359a40938fac6cb3d1a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 22:21:42 -0800
Subject: [PATCH 1098/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 223466360
---
 tensorflow/core/ops/ops.pbtxt | 62 -----------------------------------
 1 file changed, 62 deletions(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 08aafc064e..699646651e 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10188,68 +10188,6 @@ op {
     minimum: 1
   }
 }
-op {
-  name: "ExperimentalFunctionBufferingResource"
-  input_arg {
-    name: "string_arg"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "target_device"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "resource"
-    type: DT_RESOURCE
-  }
-  attr {
-    name: "shared_name"
-    type: "string"
-  }
-  attr {
-    name: "container"
-    type: "string"
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "buffer_size"
-    type: "int"
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-  }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalFunctionBufferingResourceGetNext"
-  input_arg {
-    name: "function_buffer_resource"
-    type: DT_RESOURCE
-  }
-  output_arg {
-    name: "output"
-    type_list_attr: "output_types"
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalFunctionBufferingResourceReset"
-  input_arg {
-    name: "function_buffer_resource"
-    type: DT_RESOURCE
-  }
-  is_stateful: true
-}
 op {
   name: "ExperimentalIdentityIndexedDataset"
   input_arg {
-- 
GitLab


From 9bed21b402bba91254e341b0645c8a1d3d399f5a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Nov 2018 22:51:26 -0800
Subject: [PATCH 1099/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 223468236

---
 tensorflow/go/op/wrappers.go | 1073 ++++++++++++++++------------------
 1 file changed, 511 insertions(+), 562 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 406382977b..ad979c58be 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5645,148 +5645,6 @@ func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size
 	return op.Output(0)
 }
 
-// MapUnstageAttr is an optional argument to MapUnstage.
-type MapUnstageAttr func(optionalAttr)
-
-// MapUnstageCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageCapacity(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageContainer(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageSharedName(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the values associated with the key
-//
-// from the underlying container.   If the underlying container
-// does not contain this key, the op will block until it does.
-func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapUnstage",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstage", err)
-		return
-	}
-	return values
-}
-
-// MapPeekAttr is an optional argument to MapPeek.
-type MapPeekAttr func(optionalAttr)
-
-// MapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekCapacity(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekMemoryLimit(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapPeekContainer(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapPeekSharedName(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op peeks at the values at the specified key.  If the
-//
-// underlying container does not contain this key
-// this op will block until it does.
-func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapPeek",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapPeek", err)
-		return
-	}
-	return values
-}
-
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -25662,6 +25520,77 @@ func Roll(scope *Scope, input tf.Output, shift tf.Output, axis tf.Output) (outpu
 	return op.Output(0)
 }
 
+// MapPeekAttr is an optional argument to MapPeek.
+type MapPeekAttr func(optionalAttr)
+
+// MapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekCapacity(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekMemoryLimit(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapPeekContainer(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapPeekSharedName(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op peeks at the values at the specified key.  If the
+//
+// underlying container does not contain this key
+// this op will block until it does.
+func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapPeek",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapPeek", err)
+		return
+	}
+	return values
+}
+
 // Looks up keys in a table, outputs the corresponding values.
 //
 // The tensor `keys` must of the same type as the keys of the table.
@@ -30173,370 +30102,24 @@ func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_strea
 	return scope.AddOperation(opspec)
 }
 
-// Gets the next element from a FunctionBufferingResource.
-//
-// Arguments:
-//	function_buffer_resource: The FunctionBufferingResource handle.
-//	output_types: The type list for the return values.
+// StringSplitV2Attr is an optional argument to StringSplitV2.
+type StringSplitV2Attr func(optionalAttr)
+
+// StringSplitV2Maxsplit sets the optional maxsplit attribute to value.
 //
-// Returns A list of return values.
-func ExperimentalFunctionBufferingResourceGetNext(scope *Scope, function_buffer_resource tf.Output, output_types []tf.DataType) (output []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types}
-	opspec := tf.OpSpec{
-		Type: "ExperimentalFunctionBufferingResourceGetNext",
-		Input: []tf.Input{
-			function_buffer_resource,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("ExperimentalFunctionBufferingResourceGetNext", err)
-		return
+// value: An `int`. If `maxsplit > 0`, limit of the split of the result.
+// If not specified, defaults to -1
+func StringSplitV2Maxsplit(value int64) StringSplitV2Attr {
+	return func(m optionalAttr) {
+		m["maxsplit"] = value
 	}
-	return output
 }
 
-//     Adds v into specified rows of x.
-//
-//     Computes y = x; y[i, :] += v; return y.
+// Split elements of `source` based on `sep` into a `SparseTensor`.
 //
-// Arguments:
-//	x: A `Tensor` of type T.
-//	i: A vector. Indices into the left-most dimension of `x`.
-//	v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size.
-//
-// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`.
-func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "InplaceAdd",
-		Input: []tf.Input{
-			x, i, v,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Restore a Reader to its initial clean state.
-//
-// Arguments:
-//	reader_handle: Handle to a Reader.
-//
-// Returns the created operation.
-func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReaderResetV2",
-		Input: []tf.Input{
-			reader_handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// A dataset that splits the elements of its input into multiple elements.
-func UnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "UnbatchDataset",
-		Input: []tf.Input{
-			input_dataset,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// OrderedMapStageAttr is an optional argument to OrderedMapStage.
-type OrderedMapStageAttr func(optionalAttr)
-
-// OrderedMapStageCapacity sets the optional capacity attribute to value.
-//
-// value: Maximum number of elements in the Staging Area. If > 0, inserts
-// on the container will block when the capacity is reached.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapStageContainer sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container. Otherwise,
-// a default container is used.
-// If not specified, defaults to ""
-func OrderedMapStageContainer(value string) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// OrderedMapStageSharedName sets the optional shared_name attribute to value.
-//
-// value: It is necessary to match this name to the matching Unstage Op.
-// If not specified, defaults to ""
-func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Stage (key, values) in the underlying container which behaves like a ordered
-//
-// associative container.   Elements are ordered by key.
-//
-// Arguments:
-//	key: int64
-//
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
-//
-//
-// Returns the created operation.
-func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "OrderedMapStage",
-		Input: []tf.Input{
-			key, indices, tf.OutputList(values),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// RpcAttr is an optional argument to Rpc.
-type RpcAttr func(optionalAttr)
-
-// RpcProtocol sets the optional protocol attribute to value.
-//
-// value: RPC protocol to use.  Empty string means use the default protocol.
-// Options include 'grpc'.
-// If not specified, defaults to ""
-func RpcProtocol(value string) RpcAttr {
-	return func(m optionalAttr) {
-		m["protocol"] = value
-	}
-}
-
-// RpcFailFast sets the optional fail_fast attribute to value.
-//
-// value: `boolean`. If `true` (default), then failures to connect
-// (i.e., the server does not immediately respond) cause an RPC failure.
-// If not specified, defaults to true
-func RpcFailFast(value bool) RpcAttr {
-	return func(m optionalAttr) {
-		m["fail_fast"] = value
-	}
-}
-
-// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
-//
-// value: `int`. If `0` (default), then the kernel will run the RPC
-// request and only time out if the RPC deadline passes or the session times out.
-// If this value is greater than `0`, then the op will raise an exception if
-// the RPC takes longer than `timeout_in_ms`.
-// If not specified, defaults to 0
-func RpcTimeoutInMs(value int64) RpcAttr {
-	return func(m optionalAttr) {
-		m["timeout_in_ms"] = value
-	}
-}
-
-// Perform batches of RPC requests.
-//
-// This op asynchronously performs either a single RPC request, or a batch
-// of requests.  RPC requests are defined by three main parameters:
-//
-//   - `address` (the host+port or BNS address of the request)
-//   - `method` (the RPC method name for the request)
-//   - `request` (the serialized proto string, or vector of strings,
-//      of the RPC request argument).
-//
-// For example, if you have an RPC service running on port localhost:2345,
-// and its interface is configured with the following proto declaration:
-//
-// ```
-// service MyService {
-//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
-//   }
-// };
-// ```
-//
-// then call this op with arguments:
-//
-// ```
-// address = "localhost:2345"
-// method = "MyService/MyMethod"
-// ```
-//
-// The `request` tensor is a string tensor representing serialized `MyRequestProto`
-// strings; and the output string tensor `response` will have the same shape
-// and contain (upon successful completion) corresponding serialized
-// `MyResponseProto` strings.
-//
-// For example, to send a single, empty, `MyRequestProto`, call
-// this op with `request = ""`.  To send 5 **parallel** empty requests,
-// call this op with `request = ["", "", "", "", ""]`.
-//
-// More generally, one can create a batch of `MyRequestProto` serialized protos
-// from regular batched tensors using the `encode_proto` op, and convert
-// the response `MyResponseProto` serialized protos to batched tensors
-// using the `decode_proto` op.
-//
-// **NOTE** Working with serialized proto strings is faster than instantiating
-// actual proto objects in memory, so no performance degradation is expected
-// compared to writing custom kernels for this workflow.
-//
-// If the connection fails or the remote worker returns an error
-// status, the op reraises this exception locally.
-//
-// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph.
-//
-// Arguments:
-//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `method` and `request`.
-//	method: `0-D` or `1-D`.  The method address on the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `request`.
-//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `method`.
-//
-// Returns Same shape as `request`. Serialized proto strings: the rpc responses.
-func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Rpc",
-		Input: []tf.Input{
-			address, method, request,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StackPushV2Attr is an optional argument to StackPushV2.
-type StackPushV2Attr func(optionalAttr)
-
-// StackPushV2SwapMemory sets the optional swap_memory attribute to value.
-//
-// value: Swap `elem` to CPU. Default to false.
-// If not specified, defaults to false
-func StackPushV2SwapMemory(value bool) StackPushV2Attr {
-	return func(m optionalAttr) {
-		m["swap_memory"] = value
-	}
-}
-
-// Push an element onto the stack.
-//
-// Arguments:
-//	handle: The handle to a stack.
-//	elem: The tensor to be pushed onto the stack.
-//
-// Returns The same tensor as the input 'elem'.
-func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StackPushV2",
-		Input: []tf.Input{
-			handle, elem,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Resets the FunctionBufferingResource.
-//
-// Arguments:
-//	function_buffer_resource: The FunctionBufferingResource handle.
-//
-// Returns the created operation.
-func ExperimentalFunctionBufferingResourceReset(scope *Scope, function_buffer_resource tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ExperimentalFunctionBufferingResourceReset",
-		Input: []tf.Input{
-			function_buffer_resource,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// StringSplitV2Attr is an optional argument to StringSplitV2.
-type StringSplitV2Attr func(optionalAttr)
-
-// StringSplitV2Maxsplit sets the optional maxsplit attribute to value.
-//
-// value: An `int`. If `maxsplit > 0`, limit of the split of the result.
-// If not specified, defaults to -1
-func StringSplitV2Maxsplit(value int64) StringSplitV2Attr {
-	return func(m optionalAttr) {
-		m["maxsplit"] = value
-	}
-}
-
-// Split elements of `source` based on `sep` into a `SparseTensor`.
-//
-// Let N be the size of source (typically N will be the batch size). Split each
-// element of `source` based on `sep` and return a `SparseTensor`
-// containing the split tokens. Empty tokens are ignored.
+// Let N be the size of source (typically N will be the batch size). Split each
+// element of `source` based on `sep` and return a `SparseTensor`
+// containing the split tokens. Empty tokens are ignored.
 //
 // For example, N = 2, source[0] is 'hello world' and source[1] is 'a b c',
 // then the output will be
@@ -30686,106 +30269,401 @@ func EncodeProto(scope *Scope, sizes tf.Output, values []tf.Output, field_names
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodeProto",
+		Type: "EncodeProto",
+		Input: []tf.Input{
+			sizes, tf.OutputList(values),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a TensorArray for storing the gradients of values in the given handle.
+//
+// If the given TensorArray gradient already exists, returns a reference to it.
+//
+// Locks the size of the original TensorArray by disabling its dynamic size flag.
+//
+// **A note about the input flow_in:**
+//
+// The handle flow_in forces the execution of the gradient lookup to occur
+// only after certain other operations have occurred.  For example, when
+// the forward TensorArray is dynamically sized, writes to this TensorArray
+// may resize the object.  The gradient TensorArray is statically sized based
+// on the size of the forward TensorArray when this operation executes.
+// Furthermore, the size of the forward TensorArray is frozen by this call.
+// As a result, the flow is used to ensure that the call to generate the gradient
+// TensorArray only happens after all writes are executed.
+//
+// In the case of dynamically sized TensorArrays, gradient computation should
+// only be performed on read operations that have themselves been chained via
+// flow to occur only after all writes have executed. That way the final size
+// of the forward TensorArray is known when this operation is called.
+//
+// **A note about the source attribute:**
+//
+// TensorArray gradient calls use an accumulator TensorArray object.  If
+// multiple gradients are calculated and run in the same session, the multiple
+// gradient nodes may accidentally flow through the same accumulator TensorArray.
+// This double counts and generally breaks the TensorArray gradient flow.
+//
+// The solution is to identify which gradient call this particular
+// TensorArray gradient is being called in.  This is performed by identifying
+// a unique string (e.g. "gradients", "gradients_1", ...) from the input
+// gradient Tensor's name.  This string is used as a suffix when creating
+// the TensorArray gradient object here (the attribute `source`).
+//
+// The attribute `source` is added as a suffix to the forward TensorArray's
+// name when performing the creation / lookup, so that each separate gradient
+// calculation gets its own TensorArray accumulator.
+//
+// Arguments:
+//	handle: The handle to the forward TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	source: The gradient source string, used to decide which gradient TensorArray
+// to return.
+func TensorArrayGradV3(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"source": source}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGradV3",
+		Input: []tf.Input{
+			handle, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Creates a dataset that splits a SparseTensor into elements row-wise.
+func SparseTensorSliceDataset(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseTensorSliceDataset",
+		Input: []tf.Input{
+			indices, values, dense_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns x / y element-wise for real types.
+//
+// If `x` and `y` are reals, this will return the floating-point division.
+//
+// *NOTE*: `Div` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func RealDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RealDiv",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+//     Adds v into specified rows of x.
+//
+//     Computes y = x; y[i, :] += v; return y.
+//
+// Arguments:
+//	x: A `Tensor` of type T.
+//	i: A vector. Indices into the left-most dimension of `x`.
+//	v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size.
+//
+// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`.
+func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "InplaceAdd",
+		Input: []tf.Input{
+			x, i, v,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Restore a Reader to its initial clean state.
+//
+// Arguments:
+//	reader_handle: Handle to a Reader.
+//
+// Returns the created operation.
+func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReaderResetV2",
+		Input: []tf.Input{
+			reader_handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// A dataset that splits the elements of its input into multiple elements.
+func UnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "UnbatchDataset",
+		Input: []tf.Input{
+			input_dataset,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// OrderedMapStageAttr is an optional argument to OrderedMapStage.
+type OrderedMapStageAttr func(optionalAttr)
+
+// OrderedMapStageCapacity sets the optional capacity attribute to value.
+//
+// value: Maximum number of elements in the Staging Area. If > 0, inserts
+// on the container will block when the capacity is reached.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapStageContainer sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container. Otherwise,
+// a default container is used.
+// If not specified, defaults to ""
+func OrderedMapStageContainer(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapStageSharedName sets the optional shared_name attribute to value.
+//
+// value: It is necessary to match this name to the matching Unstage Op.
+// If not specified, defaults to ""
+func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Stage (key, values) in the underlying container which behaves like a ordered
+//
+// associative container.   Elements are ordered by key.
+//
+// Arguments:
+//	key: int64
+//
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+//
+// Returns the created operation.
+func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "OrderedMapStage",
 		Input: []tf.Input{
-			sizes, tf.OutputList(values),
+			key, indices, tf.OutputList(values),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Creates a TensorArray for storing the gradients of values in the given handle.
+// RpcAttr is an optional argument to Rpc.
+type RpcAttr func(optionalAttr)
+
+// RpcProtocol sets the optional protocol attribute to value.
 //
-// If the given TensorArray gradient already exists, returns a reference to it.
+// value: RPC protocol to use.  Empty string means use the default protocol.
+// Options include 'grpc'.
+// If not specified, defaults to ""
+func RpcProtocol(value string) RpcAttr {
+	return func(m optionalAttr) {
+		m["protocol"] = value
+	}
+}
+
+// RpcFailFast sets the optional fail_fast attribute to value.
 //
-// Locks the size of the original TensorArray by disabling its dynamic size flag.
+// value: `boolean`. If `true` (default), then failures to connect
+// (i.e., the server does not immediately respond) cause an RPC failure.
+// If not specified, defaults to true
+func RpcFailFast(value bool) RpcAttr {
+	return func(m optionalAttr) {
+		m["fail_fast"] = value
+	}
+}
+
+// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
 //
-// **A note about the input flow_in:**
+// value: `int`. If `0` (default), then the kernel will run the RPC
+// request and only time out if the RPC deadline passes or the session times out.
+// If this value is greater than `0`, then the op will raise an exception if
+// the RPC takes longer than `timeout_in_ms`.
+// If not specified, defaults to 0
+func RpcTimeoutInMs(value int64) RpcAttr {
+	return func(m optionalAttr) {
+		m["timeout_in_ms"] = value
+	}
+}
+
+// Perform batches of RPC requests.
 //
-// The handle flow_in forces the execution of the gradient lookup to occur
-// only after certain other operations have occurred.  For example, when
-// the forward TensorArray is dynamically sized, writes to this TensorArray
-// may resize the object.  The gradient TensorArray is statically sized based
-// on the size of the forward TensorArray when this operation executes.
-// Furthermore, the size of the forward TensorArray is frozen by this call.
-// As a result, the flow is used to ensure that the call to generate the gradient
-// TensorArray only happens after all writes are executed.
+// This op asynchronously performs either a single RPC request, or a batch
+// of requests.  RPC requests are defined by three main parameters:
 //
-// In the case of dynamically sized TensorArrays, gradient computation should
-// only be performed on read operations that have themselves been chained via
-// flow to occur only after all writes have executed. That way the final size
-// of the forward TensorArray is known when this operation is called.
+//   - `address` (the host+port or BNS address of the request)
+//   - `method` (the RPC method name for the request)
+//   - `request` (the serialized proto string, or vector of strings,
+//      of the RPC request argument).
 //
-// **A note about the source attribute:**
+// For example, if you have an RPC service running on port localhost:2345,
+// and its interface is configured with the following proto declaration:
 //
-// TensorArray gradient calls use an accumulator TensorArray object.  If
-// multiple gradients are calculated and run in the same session, the multiple
-// gradient nodes may accidentally flow through the same accumulator TensorArray.
-// This double counts and generally breaks the TensorArray gradient flow.
+// ```
+// service MyService {
+//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
+//   }
+// };
+// ```
 //
-// The solution is to identify which gradient call this particular
-// TensorArray gradient is being called in.  This is performed by identifying
-// a unique string (e.g. "gradients", "gradients_1", ...) from the input
-// gradient Tensor's name.  This string is used as a suffix when creating
-// the TensorArray gradient object here (the attribute `source`).
+// then call this op with arguments:
 //
-// The attribute `source` is added as a suffix to the forward TensorArray's
-// name when performing the creation / lookup, so that each separate gradient
-// calculation gets its own TensorArray accumulator.
+// ```
+// address = "localhost:2345"
+// method = "MyService/MyMethod"
+// ```
+//
+// The `request` tensor is a string tensor representing serialized `MyRequestProto`
+// strings; and the output string tensor `response` will have the same shape
+// and contain (upon successful completion) corresponding serialized
+// `MyResponseProto` strings.
+//
+// For example, to send a single, empty, `MyRequestProto`, call
+// this op with `request = ""`.  To send 5 **parallel** empty requests,
+// call this op with `request = ["", "", "", "", ""]`.
+//
+// More generally, one can create a batch of `MyRequestProto` serialized protos
+// from regular batched tensors using the `encode_proto` op, and convert
+// the response `MyResponseProto` serialized protos to batched tensors
+// using the `decode_proto` op.
+//
+// **NOTE** Working with serialized proto strings is faster than instantiating
+// actual proto objects in memory, so no performance degradation is expected
+// compared to writing custom kernels for this workflow.
+//
+// If the connection fails or the remote worker returns an error
+// status, the op reraises this exception locally.
+//
+// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph.
 //
 // Arguments:
-//	handle: The handle to the forward TensorArray.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	source: The gradient source string, used to decide which gradient TensorArray
-// to return.
-func TensorArrayGradV3(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
+//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `method` and `request`.
+//	method: `0-D` or `1-D`.  The method address on the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `request`.
+//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `method`.
+//
+// Returns Same shape as `request`. Serialized proto strings: the rpc responses.
+func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"source": source}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGradV3",
+		Type: "Rpc",
 		Input: []tf.Input{
-			handle, flow_in,
+			address, method, request,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Creates a dataset that splits a SparseTensor into elements row-wise.
-func SparseTensorSliceDataset(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseTensorSliceDataset",
-		Input: []tf.Input{
-			indices, values, dense_shape,
-		},
+// StackPushV2Attr is an optional argument to StackPushV2.
+type StackPushV2Attr func(optionalAttr)
+
+// StackPushV2SwapMemory sets the optional swap_memory attribute to value.
+//
+// value: Swap `elem` to CPU. Default to false.
+// If not specified, defaults to false
+func StackPushV2SwapMemory(value bool) StackPushV2Attr {
+	return func(m optionalAttr) {
+		m["swap_memory"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns x / y element-wise for real types.
+// Push an element onto the stack.
 //
-// If `x` and `y` are reals, this will return the floating-point division.
+// Arguments:
+//	handle: The handle to a stack.
+//	elem: The tensor to be pushed onto the stack.
 //
-// *NOTE*: `Div` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func RealDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Returns The same tensor as the input 'elem'.
+func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "RealDiv",
+		Type: "StackPushV2",
 		Input: []tf.Input{
-			x, y,
+			handle, elem,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -34005,3 +33883,74 @@ func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output
 	}
 	return scope.AddOperation(opspec)
 }
+
+// MapUnstageAttr is an optional argument to MapUnstage.
+type MapUnstageAttr func(optionalAttr)
+
+// MapUnstageCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageCapacity(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageContainer(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageSharedName(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns the values associated with the key
+//
+// from the underlying container.   If the underlying container
+// does not contain this key, the op will block until it does.
+func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapUnstage",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstage", err)
+		return
+	}
+	return values
+}
-- 
GitLab


From 9ac3f6152458c762a620e61d9fbc248b2357dfb6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 01:07:01 -0800
Subject: [PATCH 1100/1554] compat: Update forward compatibility horizon to
 2018-11-30

PiperOrigin-RevId: 223479454
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index d9613021a9..f83af0a583 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -26,7 +26,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 29)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 30)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 7d931d7b85f65b6145643fdc638aabbce779ab21 Mon Sep 17 00:00:00 2001
From: Neargye <neargye@gmail.com>
Date: Mon, 23 Apr 2018 00:12:31 +0500
Subject: [PATCH 1101/1554] change toolchain to clang

---
 tensorflow/contrib/android/cmake/build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/android/cmake/build.gradle b/tensorflow/contrib/android/cmake/build.gradle
index 17a57b99fd..9e7fd317f3 100644
--- a/tensorflow/contrib/android/cmake/build.gradle
+++ b/tensorflow/contrib/android/cmake/build.gradle
@@ -22,8 +22,8 @@ android {
         }
         externalNativeBuild {
             cmake {
-                arguments '-DANDROID_TOOLCHAIN=gcc',
-                          '-DANDROID_STL=gnustl_static'
+                arguments '-DANDROID_TOOLCHAIN=clang',
+                          '-DANDROID_STL=c++_static'
             }
         }
     }
-- 
GitLab


From f56a6058a44edb2dc6172a0723fb92fd63e7a36f Mon Sep 17 00:00:00 2001
From: neargye <neargye@gmail.com>
Date: Fri, 30 Nov 2018 16:57:33 +0500
Subject: [PATCH 1102/1554] clean-up

---
 tensorflow/contrib/android/cmake/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/android/cmake/build.gradle b/tensorflow/contrib/android/cmake/build.gradle
index 9e7fd317f3..ddec08894f 100644
--- a/tensorflow/contrib/android/cmake/build.gradle
+++ b/tensorflow/contrib/android/cmake/build.gradle
@@ -70,7 +70,7 @@ if (ndkDir == null || ndkDir == "") {
     ndkDir = System.getenv('ANDROID_NDK_HOME')
 }
 
-if(! Os.isFamily(Os.FAMILY_WINDOWS)) {
+if (!Os.isFamily(Os.FAMILY_WINDOWS)) {
     // This script is for non-Windows OS. For Windows OS, MANUALLY build
     // (or copy the built) libs/headers to the
     //    ${TENSORFLOW_ROOT_DIR}/tensorflow/contrib/makefile/gen
-- 
GitLab


From e318a39eae23b4699f923f08958c774611454236 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 30 Nov 2018 08:12:30 -0800
Subject: [PATCH 1103/1554] Implement TensorListConcat and TensorListSplit.
 Remove GPU kernel registrations for TensorListSetItem and TensorListGetItem
 for string element_type since that triggers a non-DMA-copy of string which is
 not supported.

PiperOrigin-RevId: 223517409
---
 .../base_api/api_def_TensorListConcat.pbtxt   |  12 +
 .../base_api/api_def_TensorListSplit.pbtxt    |  13 +
 .../python_api/api_def_TensorListConcat.pbtxt |   4 +
 .../python_api/api_def_TensorListSplit.pbtxt  |   4 +
 tensorflow/core/kernels/list_kernels.cc       |  44 ++-
 tensorflow/core/kernels/list_kernels.cu.cc    |  15 +-
 tensorflow/core/kernels/list_kernels.h        | 310 +++++++++++++++---
 tensorflow/core/ops/list_ops.cc               | 249 +++++++++-----
 .../python/kernel_tests/list_ops_test.py      | 211 ++++++++++++
 .../kernel_tests/tensor_array_ops_test.py     |  61 ++--
 tensorflow/python/ops/list_ops.py             |  36 +-
 tensorflow/python/ops/tensor_array_ops.py     |  31 +-
 12 files changed, 814 insertions(+), 176 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListConcat.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListSplit.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListConcat.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListSplit.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListConcat.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListConcat.pbtxt
new file mode 100644
index 0000000000..909c09aa12
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorListConcat.pbtxt
@@ -0,0 +1,12 @@
+op {
+  graph_op_name: "TensorListConcat"
+  summary: "Concats all tensors in the list along the 0th dimension."
+  description: <<END
+Requires that all tensors have the same shape except the first dimension.
+
+input_handle: The input list.
+tensor: The concated result.
+lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient.
+
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListSplit.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListSplit.pbtxt
new file mode 100644
index 0000000000..24156cb8c4
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorListSplit.pbtxt
@@ -0,0 +1,13 @@
+op {
+  graph_op_name: "TensorListSplit"
+  summary: "Splits a tensor into a list."
+  description: <<END
+list[i] corresponds to lengths[i] tensors from the input tensor.
+The tensor must have rank at least 1 and contain exactly sum(lengths) elements.
+
+tensor: The input tensor.
+element_shape: A shape compatible with that of elements in the tensor.
+lengths: Vector of sizes of the 0th dimension of tensors in the list.
+output_handle: The list.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorListConcat.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorListConcat.pbtxt
new file mode 100644
index 0000000000..c7b6fd106c
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_TensorListConcat.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "TensorListConcat"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorListSplit.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorListSplit.pbtxt
new file mode 100644
index 0000000000..091297db07
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_TensorListSplit.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "TensorListSplit"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 5f244b1b10..42fad1d4b0 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -483,9 +483,19 @@ REGISTER_KERNEL_BUILDER(Name("TensorListGetItem").Device(DEVICE_CPU),
 
 #if GOOGLE_CUDA
 
-REGISTER_KERNEL_BUILDER(
-    Name("TensorListGetItem").Device(DEVICE_GPU).HostMemory("index"),
-    TensorListGetItem);
+#define REGISTER_TENSOR_LIST_GET_ITEM_GPU(T)                      \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("index"),               \
+                          TensorListGetItem);
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
+REGISTER_TENSOR_LIST_GET_ITEM_GPU(bfloat16)
+#undef REGISTER_TENSOR_LIST_GET_ITEM_GPU
 
 #endif  // GOOGLE_CUDA
 
@@ -537,9 +547,19 @@ REGISTER_KERNEL_BUILDER(Name("TensorListSetItem").Device(DEVICE_CPU),
 
 #if GOOGLE_CUDA
 
-REGISTER_KERNEL_BUILDER(
-    Name("TensorListSetItem").Device(DEVICE_GPU).HostMemory("index"),
-    TensorListSetItem);
+#define REGISTER_TENSOR_LIST_SET_ITEM_GPU(T)                      \
+  REGISTER_KERNEL_BUILDER(Name("TensorListSetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("index"),               \
+                          TensorListSetItem);
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_SET_ITEM_GPU);
+REGISTER_TENSOR_LIST_SET_ITEM_GPU(bfloat16)
+#undef REGISTER_TENSOR_LIST_SET_ITEM_GPU
 
 #endif  // GOOGLE_CUDA
 
@@ -660,7 +680,11 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListGather")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListGather<CPUDevice, T>)
+                          TensorListGather<CPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListConcat<CPUDevice, T>)
 
 TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_STACK_CPU);
 REGISTER_TENSOR_LIST_STACK_CPU(quint8);
@@ -680,7 +704,11 @@ REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListScatter")               \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListScatter<CPUDevice, T>)
+                          TensorListScatter<CPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListSplit<CPUDevice, T>)
 
 TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_CPU);
 REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint8);
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index a00bf700ca..23f552642c 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -45,7 +45,12 @@ typedef Eigen::GpuDevice GPUDevice;
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("indices"),             \
-                          TensorListGather<GPUDevice, T>)
+                          TensorListGather<GPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("lengths"),             \
+                          TensorListConcat<GPUDevice, T>)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_STACK_GPU);
 REGISTER_TENSOR_LIST_STACK_GPU(bfloat16);
@@ -82,7 +87,13 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("element_shape")        \
                               .HostMemory("indices"),             \
-                          TensorListScatter<GPUDevice, T>)
+                          TensorListScatter<GPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape")        \
+                              .HostMemory("lengths"),             \
+                          TensorListSplit<GPUDevice, T>)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
 REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bfloat16);
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index d4adc068a2..686679474c 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/coding.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/util/tensor_ops_util.h"
 #include "tensorflow/core/util/util.h"
 
@@ -77,26 +78,30 @@ class TensorListStack : public OpKernel {
   ~TensorListStack() {}
 
   void Compute(OpKernelContext* c) override {
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
+    const TensorList* tensor_list =
+        c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, tensor_list != nullptr,
                 errors::InvalidArgument(
                     "Input handle is not a list. Saw: '",
                     c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-    OP_REQUIRES(c, !l->tensors.empty() || l->element_shape.IsFullyDefined(),
-                errors::InvalidArgument("Tried to stack elements of a empty ",
-                                        "list with non-fully-defined shape: ",
-                                        l->element_shape.DebugString()));
+    OP_REQUIRES(
+        c, element_dtype_ == tensor_list->element_dtype,
+        errors::InvalidArgument(
+            "Invalid data types; op elements ", DataTypeString(element_dtype_),
+            " but list elements ", DataTypeString(tensor_list->element_dtype)));
+    OP_REQUIRES(
+        c,
+        !tensor_list->tensors.empty() ||
+            tensor_list->element_shape.IsFullyDefined(),
+        errors::InvalidArgument("Tried to stack elements of a empty ",
+                                "list with non-fully-defined shape: ",
+                                tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
-      OP_REQUIRES(c, l->tensors.size() == num_elements_,
-                  errors::InvalidArgument("Operation expected a list with ",
-                                          num_elements_,
-                                          " elements but got a list with ",
-                                          l->tensors.size(), " elements."));
+      OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
+                  errors::InvalidArgument(
+                      "Operation expected a list with ", num_elements_,
+                      " elements but got a list with ",
+                      tensor_list->tensors.size(), " elements."));
     }
     // Compute the shape of the output tensor.
     // If `element_shape` is fully-defined it gets used. It is assumed that all
@@ -105,11 +110,11 @@ class TensorListStack : public OpKernel {
     // tensor is used and it is checked that all other tensors have the same
     // shape.
     TensorShape resulting_shape;
-    if (!l->element_shape.AsTensorShape(&resulting_shape)) {
-      const Tensor& t = l->tensors[0];
+    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
+      const Tensor& t = tensor_list->tensors[0];
       resulting_shape = t.shape();
-      for (int i = 1; i < l->tensors.size(); ++i) {
-        const Tensor& t = l->tensors[i];
+      for (int i = 1; i < tensor_list->tensors.size(); ++i) {
+        const Tensor& t = tensor_list->tensors[i];
         OP_REQUIRES(c, t.shape() == resulting_shape,
                     errors::InvalidArgument(
                         "Tried to stack tensors with unequal shapes: ",
@@ -117,7 +122,7 @@ class TensorListStack : public OpKernel {
                         t.shape().DebugString()));
       }
     }
-    resulting_shape.InsertDim(0, l->tensors.size());
+    resulting_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
     OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
     if (output->NumElements() == 0) {
@@ -125,8 +130,8 @@ class TensorListStack : public OpKernel {
     }
 
     ConstMatrixVector inputs_flat;
-    inputs_flat.reserve(l->tensors.size());
-    for (const auto& t : l->tensors) {
+    inputs_flat.reserve(tensor_list->tensors.size());
+    for (const auto& t : tensor_list->tensors) {
       inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
           t.shaped<T, 2>({1, t.NumElements()})));
     }
@@ -146,6 +151,200 @@ class TensorListStack : public OpKernel {
   DataType element_dtype_;
 };
 
+template <typename Device, typename T>
+class TensorListConcat : public OpKernel {
+ public:
+  using ConstMatrixVector =
+      std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>;
+  explicit TensorListConcat(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  ~TensorListConcat() {}
+
+  void Compute(OpKernelContext* c) override {
+    // Check that the input Variant tensor is indeed a TensorList and has the
+    // correct element type.
+    const TensorList* tensor_list =
+        c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, tensor_list != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(
+        c, element_dtype_ == tensor_list->element_dtype,
+        errors::InvalidArgument(
+            "Invalid data types; op elements ", DataTypeString(element_dtype_),
+            " but list elements ", DataTypeString(tensor_list->element_dtype)));
+    // If the TensorList is empty, its element_shape must be fully defined
+    // except for the first dimension.
+    PartialTensorShape shape_except_first_dim;
+    if (!tensor_list->element_shape.unknown_rank()) {
+      OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1,
+                  errors::InvalidArgument(
+                      "Concat requires elements to be at least vectors, ",
+                      "found scalars instead."));
+      shape_except_first_dim = PartialTensorShape(
+          gtl::ArraySlice<int64>(tensor_list->element_shape.dim_sizes())
+              .subspan(1));
+    }
+    OP_REQUIRES(c,
+                !tensor_list->tensors.empty() ||
+                    shape_except_first_dim.IsFullyDefined(),
+                errors::InvalidArgument(
+                    "All except the first dimension must be fully defined ",
+                    "when concating an empty tensor list. element_shape: ",
+                    tensor_list->element_shape.DebugString()));
+    // 1. Compute the shape of the output tensor.
+    // If `shape_except_first_dim` is fully-defined we just prepend the leading
+    // dim to it. Otherwise we use the shape of the first element tensor and
+    // check to make sure shapes of all tensors are compatible.
+    TensorShape output_shape;
+    if (!shape_except_first_dim.AsTensorShape(&output_shape)) {
+      const Tensor& element_tensor = tensor_list->tensors[0];
+      OP_REQUIRES(
+          c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()),
+          errors::InvalidArgument("Concat saw a scalar shape at index ", 0,
+                                  " but requires at least vectors."));
+      output_shape =
+          TensorShape(gtl::ArraySlice<int64>(element_tensor.shape().dim_sizes())
+                          .subspan(1));
+      for (int i = 1; i < tensor_list->tensors.size(); ++i) {
+        const Tensor& element_tensor = tensor_list->tensors[i];
+        OP_REQUIRES(
+            c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()),
+            errors::InvalidArgument("Concat saw a scalar shape at index ", i,
+                                    " but requires at least vectors."));
+        TensorShape actual_shape(
+            gtl::ArraySlice<int64>(element_tensor.shape().dim_sizes())
+                .subspan(1));
+        OP_REQUIRES(c, actual_shape.dim_sizes() == output_shape.dim_sizes(),
+                    errors::InvalidArgument(
+                        "Tried to concat tensors with unequal shapes: ",
+                        output_shape.DebugString(), " vs ",
+                        actual_shape.DebugString()));
+      }
+    }
+    // 2. Build the lengths_tensor and leading dim of the output tensor by
+    // iterating over all element tensors.
+    Tensor* lengths_tensor = nullptr;
+    OP_REQUIRES_OK(
+        c,
+        c->allocate_output(
+            1, TensorShape({static_cast<int64>(tensor_list->tensors.size())}),
+            &lengths_tensor));
+    auto lengths_tensor_vec = lengths_tensor->vec<int64>();
+    int64 leading_dim = 0;
+    for (size_t i = 0; i < tensor_list->tensors.size(); i++) {
+      int64 dim = tensor_list->tensors[i].shape().dim_size(0);
+      leading_dim += dim;
+      lengths_tensor_vec(i) = dim;
+    }
+    output_shape.InsertDim(0, leading_dim);
+    Tensor* output;
+    // 3. Allocate the output tensor and fill it up with the concated element
+    // tensors.
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
+    if (output->NumElements() == 0) {
+      return;
+    }
+
+    ConstMatrixVector inputs_flat;
+    inputs_flat.reserve(tensor_list->tensors.size());
+    for (const auto& element_tensor : tensor_list->tensors) {
+      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+          element_tensor.shaped<T, 2>({1, element_tensor.NumElements()})));
+    }
+    auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
+
+#if GOOGLE_CUDA
+    if (std::is_same<Device, Eigen::GpuDevice>::value) {
+      ConcatGPU<T>(c, inputs_flat, output, &output_flat);
+      return;
+    }
+#endif  // GOOGLE_CUDA
+    ConcatCPU<T>(c->device(), inputs_flat, &output_flat);
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
+template <typename Device, typename T>
+class TensorListSplit : public OpKernel {
+ public:
+  TensorListSplit(OpKernelConstruction* c) : OpKernel(c) {}
+
+  void Compute(OpKernelContext* c) override {
+    Tensor* output_tensor;
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr));
+    PartialTensorShape element_shape;
+    OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(1), &element_shape));
+    OP_REQUIRES(c, element_shape.unknown_rank() || element_shape.dims() >= 1,
+                errors::InvalidArgument(
+                    "TensorListSplit requires element_shape to be at least of ",
+                    "rank 1, but saw: ", element_shape.DebugString()));
+    TensorList output_list;
+    const Tensor& input_tensor = c->input(0);
+    output_list.element_dtype = input_tensor.dtype();
+    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input_tensor.shape()),
+                errors::InvalidArgument(
+                    "Tensor must be at least a vector, but saw shape: ",
+                    input_tensor.shape().DebugString()));
+    TensorShape tensor_shape_without_first_dim(input_tensor.shape());
+    tensor_shape_without_first_dim.RemoveDim(0);
+    PartialTensorShape element_shape_without_first_dim;
+    if (!element_shape.unknown_rank()) {
+      element_shape_without_first_dim =
+          PartialTensorShape(element_shape.dim_sizes());
+      element_shape_without_first_dim.RemoveDim(0);
+    }
+    OP_REQUIRES(c,
+                element_shape_without_first_dim.IsCompatibleWith(
+                    tensor_shape_without_first_dim),
+                errors::InvalidArgument(
+                    "tensor shape ", input_tensor.shape().DebugString(),
+                    " is not compatible with element_shape ",
+                    element_shape.DebugString()));
+    output_list.element_shape = element_shape;
+    const Tensor& lengths = c->input(2);
+    OP_REQUIRES(c, TensorShapeUtils::IsVector(lengths.shape()),
+                errors::InvalidArgument(
+                    "Expected lengths to be a vector, received shape: ",
+                    lengths.shape().DebugString()));
+    output_list.tensors.reserve(lengths.shape().dim_size(0));
+    int64 start = 0;
+    int64 end = 0;
+    for (int i = 0; i < lengths.shape().dim_size(0); ++i) {
+      int64 length = lengths.vec<int64>()(i);
+      OP_REQUIRES(
+          c, length >= 0,
+          errors::InvalidArgument("Invalid value in lengths: ", length));
+      end = start + length;
+      OP_REQUIRES(c, end <= input_tensor.shape().dim_size(0),
+                  errors::InvalidArgument("Attempting to slice [", start, ", ",
+                                          end, "] from tensor with length ",
+                                          input_tensor.shape().dim_size(0)));
+      Tensor tmp = input_tensor.Slice(start, end);
+      start = end;
+      // TODO(apassos) maybe not always align; but weird compiler bugs seem to
+      // prevent this.
+      Tensor aligned;
+      OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned));
+      aligned.flat<T>().device(c->eigen_device<Device>()) =
+          tmp.unaligned_flat<T>();
+      output_list.tensors.emplace_back(aligned);
+    }
+    OP_REQUIRES(c, end == input_tensor.shape().dim_size(0),
+                errors::InvalidArgument(
+                    "Unused values in tensor. Length of tensor: ",
+                    input_tensor.shape().dim_size(0), " Values used: ", end));
+    output_tensor->scalar<Variant>()() = std::move(output_list);
+  }
+};
+
 template <typename Device, typename T>
 class TensorListGather : public OpKernel {
  public:
@@ -156,22 +355,25 @@ class TensorListGather : public OpKernel {
   }
 
   void Compute(OpKernelContext* c) override {
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
+    const TensorList* tensor_list =
+        c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, tensor_list != nullptr,
                 errors::InvalidArgument(
                     "Input handle is not a list. Saw: '",
                     c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
+    OP_REQUIRES(
+        c, element_dtype_ == tensor_list->element_dtype,
+        errors::InvalidArgument(
+            "Invalid data types; op elements ", DataTypeString(element_dtype_),
+            " but list elements ", DataTypeString(tensor_list->element_dtype)));
     Tensor indices = c->input(1);
-    OP_REQUIRES(c,
-                indices.NumElements() > 0 || l->element_shape.IsFullyDefined(),
-                errors::InvalidArgument("Tried to gather 0-elements from "
-                                        "a list with non-fully-defined shape: ",
-                                        l->element_shape.DebugString()));
+    OP_REQUIRES(
+        c,
+        indices.NumElements() > 0 ||
+            tensor_list->element_shape.IsFullyDefined(),
+        errors::InvalidArgument("Tried to gather 0-elements from "
+                                "a list with non-fully-defined shape: ",
+                                tensor_list->element_shape.DebugString()));
     // Compute the shape of the output tensor.
     // If `element_shape` is fully-defined it gets used. It is assumed that all
     // requested tensors have the same shape.
@@ -179,17 +381,17 @@ class TensorListGather : public OpKernel {
     // tensor is used and it is checked that all other tensors have the same
     // shape.
     TensorShape resulting_shape;
-    if (!l->element_shape.AsTensorShape(&resulting_shape)) {
+    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
       const int i = indices.flat<int32>()(0);
       OP_REQUIRES(
-          c, i < l->tensors.size(),
+          c, i < tensor_list->tensors.size(),
           errors::InvalidArgument("Index ", i, " out o range; list only has ",
-                                  l->tensors.size(), " elements."));
-      const Tensor& t = l->tensors[i];
+                                  tensor_list->tensors.size(), " elements."));
+      const Tensor& t = tensor_list->tensors[i];
       resulting_shape = t.shape();
       for (int index = 1; index < indices.NumElements(); ++index) {
         const int i = indices.flat<int32>()(index);
-        const Tensor& t = l->tensors[i];
+        const Tensor& t = tensor_list->tensors[i];
         OP_REQUIRES(c, t.shape() == resulting_shape,
                     errors::InvalidArgument(
                         "Tried to gather elements with unequal shapes: ",
@@ -205,14 +407,14 @@ class TensorListGather : public OpKernel {
     }
 
     ConstMatrixVector inputs_flat;
-    inputs_flat.reserve(l->tensors.size());
+    inputs_flat.reserve(tensor_list->tensors.size());
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
       OP_REQUIRES(
-          c, i < l->tensors.size(),
+          c, i < tensor_list->tensors.size(),
           errors::InvalidArgument("Index ", i, " out o range; list only has ",
-                                  l->tensors.size(), " elements."));
-      const Tensor& t = l->tensors[i];
+                                  tensor_list->tensors.size(), " elements."));
+      const Tensor& t = tensor_list->tensors[i];
       inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
           t.shaped<T, 2>({1, t.NumElements()})));
     }
@@ -290,13 +492,13 @@ class TensorListScatter : public OpKernel {
     PartialTensorShape element_shape;
     OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(2), &element_shape));
     TensorList output_list;
-    const Tensor& t = c->input(0);
-    output_list.element_dtype = t.dtype();
-    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(t.shape()),
+    const Tensor& input_tensor = c->input(0);
+    output_list.element_dtype = input_tensor.dtype();
+    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input_tensor.shape()),
                 errors::InvalidArgument(
                     "Tensor must be at least a vector, but saw shape: ",
-                    t.shape().DebugString()));
-    TensorShape output_shape(t.shape());
+                    input_tensor.shape().DebugString()));
+    TensorShape output_shape(input_tensor.shape());
     output_shape.RemoveDim(0);
     OP_REQUIRES(c, element_shape.IsCompatibleWith(output_shape),
                 errors::InvalidArgument(
@@ -306,11 +508,11 @@ class TensorListScatter : public OpKernel {
     output_list.tensors.reserve(indices.NumElements());
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
-      OP_REQUIRES(c, i < t.shape().dim_size(0),
-                  errors::InvalidArgument("Trying to scatter index ", i,
-                                          " from tensor with ",
-                                          t.shape().dim_size(0), " rows."));
-      Tensor tmp = t.Slice(i, i + 1);
+      OP_REQUIRES(c, i < input_tensor.shape().dim_size(0),
+                  errors::InvalidArgument(
+                      "Trying to scatter index ", i, " from tensor with ",
+                      input_tensor.shape().dim_size(0), " rows."));
+      Tensor tmp = input_tensor.Slice(i, i + 1);
       TensorShape tmp_shape = tmp.shape();
       tmp_shape.RemoveDim(0);
       OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape),
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index 88d6d14c30..01ebcd1543 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -28,13 +28,14 @@ REGISTER_OP("EmptyTensorList")
     .Attr("shape_type: {int32, int64}")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       c->set_output(0, c->Scalar());
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s;
-      TF_RETURN_IF_ERROR(
-          c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(0, &s));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
+          0, &element_shape));
       c->set_output_handle_shapes_and_types(
-          0, std::vector<shape_inference::ShapeAndType>{{s, t}});
+          0, std::vector<shape_inference::ShapeAndType>{
+                 {element_shape, element_dtype}});
       return Status::OK();
     });
 
@@ -45,9 +46,9 @@ REGISTER_OP("TensorListPushBack")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       c->set_output(0, c->Scalar());
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s = c->UnknownShape();
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape = c->UnknownShape();
 
       auto* handle_data = c->input_handle_shapes_and_types(0);
       if (handle_data != nullptr && handle_data->size() != 1) {
@@ -57,18 +58,21 @@ REGISTER_OP("TensorListPushBack")
       if (handle_data != nullptr) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
-        if (list_shape_type.dtype != t) {
+        if (list_shape_type.dtype != element_dtype) {
           return errors::InvalidArgument(
               "Trying to push to list with wrong element dtype. List has type ",
               DataTypeString(list_shape_type.dtype),
-              " but trying to push element with type ", DataTypeString(t));
+              " but trying to push element with type ",
+              DataTypeString(element_dtype));
         }
         shape_inference::ShapeHandle ignored;
-        TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored));
-        s = list_shape_type.shape;
+        TF_RETURN_IF_ERROR(
+            c->Merge(element_shape, list_shape_type.shape, &ignored));
+        element_shape = list_shape_type.shape;
       }
       c->set_output_handle_shapes_and_types(
-          0, std::vector<shape_inference::ShapeAndType>{{s, t}});
+          0, std::vector<shape_inference::ShapeAndType>{
+                 {element_shape, element_dtype}});
       return Status::OK();
     });
 
@@ -89,9 +93,9 @@ REGISTER_OP("TensorListPushBackBatch")
 
       c->set_output(0, input_handles);
 
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s = c->UnknownShape();
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape = c->UnknownShape();
 
       auto* handle_data = c->input_handle_shapes_and_types(0);
       if (handle_data != nullptr && handle_data->size() != 1) {
@@ -101,18 +105,21 @@ REGISTER_OP("TensorListPushBackBatch")
       if (handle_data != nullptr) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
-        if (list_shape_type.dtype != t) {
+        if (list_shape_type.dtype != element_dtype) {
           return errors::InvalidArgument(
               "Trying to push to list with wrong element dtype. List has type ",
               DataTypeString(list_shape_type.dtype),
-              " but trying to push element with type ", DataTypeString(t));
+              " but trying to push element with type ",
+              DataTypeString(element_dtype));
         }
         shape_inference::ShapeHandle ignored;
-        TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored));
-        s = list_shape_type.shape;
+        TF_RETURN_IF_ERROR(
+            c->Merge(element_shape, list_shape_type.shape, &ignored));
+        element_shape = list_shape_type.shape;
       }
       c->set_output_handle_shapes_and_types(
-          0, std::vector<shape_inference::ShapeAndType>{{s, t}});
+          0, std::vector<shape_inference::ShapeAndType>{
+                 {element_shape, element_dtype}});
       return Status::OK();
     });
 
@@ -127,9 +134,9 @@ REGISTER_OP("TensorListPopBack")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s = c->UnknownShape();
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle tensor_shape = c->UnknownShape();
       auto* handle_data = c->input_handle_shapes_and_types(0);
       if (handle_data != nullptr && handle_data->size() != 1) {
         return errors::InvalidArgument(
@@ -138,19 +145,21 @@ REGISTER_OP("TensorListPopBack")
       if (handle_data != nullptr) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
-        if (list_shape_type.dtype != t) {
+        if (list_shape_type.dtype != element_dtype) {
           return errors::InvalidArgument(
               "Trying to read from list with wrong element dtype. List has "
               "type ",
               DataTypeString(list_shape_type.dtype),
-              " but trying to push element with type ", DataTypeString(t));
+              " but trying to push element with type ",
+              DataTypeString(element_dtype));
         }
         shape_inference::ShapeHandle ignored;
-        TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored));
+        TF_RETURN_IF_ERROR(
+            c->Merge(tensor_shape, list_shape_type.shape, &ignored));
         c->set_output_handle_shapes_and_types(0, *handle_data);
-        s = list_shape_type.shape;
+        tensor_shape = list_shape_type.shape;
       }
-      c->set_output(1, s);
+      c->set_output(1, tensor_shape);
       c->set_output(0, c->Scalar());
       return Status::OK();
     });
@@ -161,9 +170,9 @@ REGISTER_OP("TensorListStack")
     .Attr("element_dtype: type")
     .Attr("num_elements: int = -1")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s = c->UnknownShape();
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape = c->UnknownShape();
       auto* handle_data = c->input_handle_shapes_and_types(0);
       if (handle_data != nullptr && handle_data->size() != 1) {
         return errors::InvalidArgument(
@@ -172,16 +181,17 @@ REGISTER_OP("TensorListStack")
       if (handle_data != nullptr) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
-        if (list_shape_type.dtype != t) {
+        if (list_shape_type.dtype != element_dtype) {
           return errors::InvalidArgument(
               "Trying to read from list with wrong element dtype. List has "
               "type ",
               DataTypeString(list_shape_type.dtype), " but expectec type ",
-              DataTypeString(t));
+              DataTypeString(element_dtype));
         }
         shape_inference::ShapeHandle ignored;
-        TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored));
-        s = list_shape_type.shape;
+        TF_RETURN_IF_ERROR(
+            c->Merge(element_shape, list_shape_type.shape, &ignored));
+        element_shape = list_shape_type.shape;
       }
       int expected_num_elements = -1;
       TF_RETURN_IF_ERROR(c->GetAttr("num_elements", &expected_num_elements));
@@ -192,11 +202,88 @@ REGISTER_OP("TensorListStack")
         num_elements = c->MakeShape({expected_num_elements});
       }
       shape_inference::ShapeHandle result;
-      TF_RETURN_IF_ERROR(c->Concatenate(num_elements, s, &result));
+      TF_RETURN_IF_ERROR(c->Concatenate(num_elements, element_shape, &result));
       c->set_output(0, result);
       return Status::OK();
     });
 
+REGISTER_OP("TensorListConcat")
+    .Input("input_handle: variant")
+    .Output("tensor: element_dtype")
+    .Output("lengths: int64")
+    .Attr("element_dtype: type")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape = c->UnknownShape();
+      auto* handle_data = c->input_handle_shapes_and_types(0);
+      if (handle_data != nullptr && handle_data->size() != 1) {
+        return errors::InvalidArgument(
+            "Trying to read from list with wrong variant data.");
+      }
+      if (handle_data != nullptr) {
+        const shape_inference::ShapeAndType& list_shape_type =
+            (*handle_data)[0];
+        if (list_shape_type.dtype != element_dtype) {
+          return errors::InvalidArgument(
+              "Trying to read from list with wrong element dtype. List has "
+              "type ",
+              DataTypeString(list_shape_type.dtype), " but expected type ",
+              DataTypeString(element_dtype));
+        }
+        shape_inference::ShapeHandle ignored;
+        TF_RETURN_IF_ERROR(
+            c->Merge(element_shape, list_shape_type.shape, &ignored));
+        element_shape = list_shape_type.shape;
+      }
+      if (c->RankKnown(element_shape)) {
+        shape_inference::ShapeHandle result;
+        TF_RETURN_IF_ERROR(c->Subshape(element_shape, 1, &result));
+        TF_RETURN_IF_ERROR(
+            c->Concatenate(c->MakeShape({c->UnknownDim()}), result, &result));
+        c->set_output(0, result);
+      } else {
+        c->set_output(0, c->UnknownShape());
+      }
+      c->set_output(1, c->MakeShape({c->UnknownDim()}));
+      return Status::OK();
+    });
+
+REGISTER_OP("TensorListSplit")
+    .Input("tensor: element_dtype")
+    .Input("element_shape: shape_type")
+    .Input("lengths: int64")
+    .Output("output_handle: variant")
+    .Attr("element_dtype: type")
+    .Attr("shape_type: {int32, int64}")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      c->set_output(0, c->Scalar());
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle tensor_shape = c->input(0);
+      shape_inference::ShapeHandle ignored;
+      // Check that tensor is at least a vector.
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(tensor_shape, 1, &ignored));
+      // Check that lengths is a vector.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &ignored));
+      shape_inference::ShapeHandle element_shape_from_tensor_shape;
+      TF_RETURN_IF_ERROR(
+          c->Subshape(tensor_shape, 1, &element_shape_from_tensor_shape));
+      TF_RETURN_IF_ERROR(c->Concatenate(c->MakeShape({c->UnknownDim()}),
+                                        element_shape_from_tensor_shape,
+                                        &element_shape_from_tensor_shape));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
+          1, &element_shape));
+      TF_RETURN_IF_ERROR(c->Merge(element_shape_from_tensor_shape,
+                                  element_shape,
+                                  &element_shape_from_tensor_shape));
+      c->set_output_handle_shapes_and_types(
+          0, std::vector<shape_inference::ShapeAndType>{
+                 {element_shape, element_dtype}});
+      return Status::OK();
+    });
+
 REGISTER_OP("TensorListFromTensor")
     .Input("tensor: element_dtype")
     .Input("element_shape: shape_type")
@@ -205,17 +292,20 @@ REGISTER_OP("TensorListFromTensor")
     .Attr("shape_type: {int32, int64}")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       c->set_output(0, c->Scalar());
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s = c->input(0);
-      shape_inference::ShapeHandle o;
-      TF_RETURN_IF_ERROR(c->Subshape(s, 1, &o));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle tensor_shape = c->input(0);
+      shape_inference::ShapeHandle tensor_shape_except_first_dim;
+      TF_RETURN_IF_ERROR(
+          c->Subshape(tensor_shape, 1, &tensor_shape_except_first_dim));
       shape_inference::ShapeHandle element_shape;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
           1, &element_shape));
-      TF_RETURN_IF_ERROR(c->Merge(o, element_shape, &o));
+      TF_RETURN_IF_ERROR(c->Merge(tensor_shape_except_first_dim, element_shape,
+                                  &tensor_shape_except_first_dim));
       c->set_output_handle_shapes_and_types(
-          0, std::vector<shape_inference::ShapeAndType>{{element_shape, t}});
+          0, std::vector<shape_inference::ShapeAndType>{
+                 {element_shape, element_dtype}});
       return Status::OK();
     });
 
@@ -241,13 +331,14 @@ REGISTER_OP("TensorListReserve")
     .Attr("shape_type: {int32, int64}")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       c->set_output(0, c->Scalar());
-      shape_inference::ShapeHandle s;
-      TF_RETURN_IF_ERROR(
-          c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(0, &s));
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
+          0, &element_shape));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       c->set_output_handle_shapes_and_types(
-          0, std::vector<shape_inference::ShapeAndType>{{s, t}});
+          0, std::vector<shape_inference::ShapeAndType>{
+                 {element_shape, element_dtype}});
       return Status::OK();
     });
 
@@ -257,17 +348,17 @@ REGISTER_OP("TensorListGetItem")
     .Output("item: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       auto* handle_data = c->input_handle_shapes_and_types(0);
       shape_inference::ShapeHandle element_shape = c->UnknownShape();
       if (handle_data != nullptr) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         element_shape = list_shape_type.shape;
-        if (list_shape_type.dtype != t) {
+        if (list_shape_type.dtype != element_dtype) {
           return errors::InvalidArgument("Expected list with element dtype ",
-                                         DataTypeString(t),
+                                         DataTypeString(element_dtype),
                                          " but got list with element dtype ",
                                          DataTypeString(list_shape_type.dtype));
         }
@@ -283,17 +374,19 @@ REGISTER_OP("TensorListSetItem")
     .Output("output_handle: variant")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       auto* handle_data = c->input_handle_shapes_and_types(0);
       c->set_output(0, c->Scalar());
       if (handle_data == nullptr) {
-        c->set_output_handle_shapes_and_types(0, {{c->UnknownShape(), t}});
+        c->set_output_handle_shapes_and_types(
+            0, {{c->UnknownShape(), element_dtype}});
         return Status::OK();
       }
       const shape_inference::ShapeAndType& list_shape_type = (*handle_data)[0];
-      shape_inference::ShapeHandle s = c->input(2);
-      TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &s));
+      shape_inference::ShapeHandle item_shape = c->input(2);
+      TF_RETURN_IF_ERROR(
+          c->Merge(item_shape, list_shape_type.shape, &item_shape));
       c->set_output_handle_shapes_and_types(0, *handle_data);
       return Status::OK();
     });
@@ -304,17 +397,17 @@ REGISTER_OP("TensorListGather")
     .Output("values: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
       auto* handle_data = c->input_handle_shapes_and_types(0);
       shape_inference::ShapeHandle element_shape = c->UnknownShape();
       if (handle_data != nullptr) {
         const shape_inference::ShapeAndType& list_shape_type =
             (*handle_data)[0];
         element_shape = list_shape_type.shape;
-        if (list_shape_type.dtype != t) {
+        if (list_shape_type.dtype != element_dtype) {
           return errors::InvalidArgument("Expected list with element dtype ",
-                                         DataTypeString(t),
+                                         DataTypeString(element_dtype),
                                          " but got list with element dtype ",
                                          DataTypeString(list_shape_type.dtype));
         }
@@ -333,12 +426,13 @@ REGISTER_OP("TensorListScatter")
     .Attr("element_dtype: type")
     .Attr("shape_type: {int32, int64}")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
-      shape_inference::ShapeHandle s;
-      TF_RETURN_IF_ERROR(
-          c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(2, &s));
-      c->set_output_handle_shapes_and_types(0, {{s, t}});
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape(
+          2, &element_shape));
+      c->set_output_handle_shapes_and_types(0,
+                                            {{element_shape, element_dtype}});
       c->set_output(0, c->Scalar());
       return Status::OK();
     });
@@ -354,28 +448,29 @@ REGISTER_OP("TensorListConcatLists")
       TF_RETURN_IF_ERROR(c->Merge(input_a, input_b, &input_a));
       c->set_output(0, input_a);
 
-      DataType t;
-      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &t));
+      DataType element_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
 
       auto* handle_data_a = c->input_handle_shapes_and_types(0);
       auto* handle_data_b = c->input_handle_shapes_and_types(1);
       if (handle_data_a == nullptr && handle_data_b == nullptr) {
-        c->set_output_handle_shapes_and_types(0, {{c->UnknownShape(), t}});
+        c->set_output_handle_shapes_and_types(
+            0, {{c->UnknownShape(), element_dtype}});
         return Status::OK();
       }
       shape_inference::ShapeAndType list_shape_type_a =
           (handle_data_a) ? handle_data_a->at(0) : handle_data_b->at(0);
       const shape_inference::ShapeAndType& list_shape_type_b =
           (handle_data_b) ? handle_data_b->at(0) : handle_data_a->at(0);
-      if (list_shape_type_a.dtype != t) {
+      if (list_shape_type_a.dtype != element_dtype) {
         return errors::InvalidArgument("input_a.type != element_dtype: ",
                                        DataTypeString(list_shape_type_a.dtype),
-                                       " vs. ", DataTypeString(t));
+                                       " vs. ", DataTypeString(element_dtype));
       }
-      if (list_shape_type_b.dtype != t) {
+      if (list_shape_type_b.dtype != element_dtype) {
         return errors::InvalidArgument("input_b.type != element_dtype: ",
                                        DataTypeString(list_shape_type_b.dtype),
-                                       " vs. ", DataTypeString(t));
+                                       " vs. ", DataTypeString(element_dtype));
       }
       TF_RETURN_IF_ERROR(c->Merge(list_shape_type_a.shape,
                                   list_shape_type_b.shape,
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 8df1156438..41c2708fbc 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -880,6 +880,217 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(self.evaluate(result_0), [6., 8.])
     self.assertAllEqual(self.evaluate(result_1), [10., 12.])
 
+  def testSkipEagerConcatShapeInference(self):
+
+    def BuildTensor(element_shape):
+      l = list_ops.empty_tensor_list(
+          element_dtype=dtypes.float32, element_shape=element_shape)
+      return list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+
+    self.assertIsNone(BuildTensor(None).shape.rank)
+    self.assertAllEqual(BuildTensor([None, 2, 3]).shape.as_list(), [None, 2, 3])
+    self.assertAllEqual(
+        BuildTensor([None, 2, None]).shape.as_list(), [None, 2, None])
+    self.assertAllEqual(BuildTensor([1, 2, 3]).shape.as_list(), [None, 2, 3])
+
+  def testConcatWithFullyDefinedElementShape(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=[2, 2])
+    l = list_ops.tensor_list_push_back(l, [[0., 1.], [2., 3.]])
+    l = list_ops.tensor_list_push_back(l, [[4., 5.], [6., 7.]])
+    t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(
+        self.evaluate(t), [[0., 1.], [2., 3.], [4., 5.], [6., 7.]])
+
+  def testConcatWithNonFullyDefinedElementShape(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=[None, 2])
+    l = list_ops.tensor_list_push_back(l, [[0., 1.]])
+    l = list_ops.tensor_list_push_back(l, [[2., 3.], [4., 5.]])
+    t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [[0., 1.], [2., 3.], [4., 5.]])
+
+  def testConcatWithMismatchingTensorShapesFails(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=None)
+    l = list_ops.tensor_list_push_back(l, [[0., 1.]])
+    l = list_ops.tensor_list_push_back(l, [[2.], [4.]])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Tried to concat tensors with unequal shapes: "
+        r"\[2\] vs \[1\]"):
+      t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testConcatEmptyListWithFullyDefinedElementShape(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=[5, 2])
+    t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t).shape, (0, 2))
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=[None, 2])
+    t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t).shape, (0, 2))
+
+  def testConcatEmptyListWithUnknownElementShapeFails(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=None)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "All except the first dimension must be fully"
+        " defined when concating an empty tensor list"):
+      t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testConcatEmptyListWithPartiallyDefinedElementShapeFails(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=[2, None])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "All except the first dimension must be fully"
+        " defined when concating an empty tensor list"):
+      t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testConcatListWithScalarElementShapeFails(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=tensor_shape.scalar())
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Concat requires elements to be at least vectors, "
+        "found scalars instead"):
+      t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testConcatListWithScalarElementsFails(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=None)
+    l1 = list_ops.tensor_list_push_back(l, 1.)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError, "Concat saw a scalar shape at index 0"
+        " but requires at least vectors"):
+      t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32)
+      self.evaluate(t)
+    l1 = list_ops.tensor_list_push_back(l, [1.])
+    l1 = list_ops.tensor_list_push_back(l1, 2.)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError, "Concat saw a scalar shape at index 1"
+        " but requires at least vectors"):
+      t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testEvenSplit(self):
+
+    def RunTest(input_tensor, lengths, expected_stacked_output):
+      l = list_ops.tensor_list_split(
+          input_tensor, element_shape=None, lengths=lengths)
+      self.assertAllEqual(
+          list_ops.tensor_list_stack(l, element_dtype=dtypes.float32),
+          expected_stacked_output)
+
+    RunTest([1., 2., 3.], [1, 1, 1], [[1.], [2.], [3.]])
+    RunTest([1., 2., 3., 4.], [2, 2], [[1., 2.], [3., 4.]])
+    RunTest([[1., 2.], [3., 4.]], [1, 1], [[[1., 2.]], [[3., 4.]]])
+
+  def testUnevenSplit(self):
+    l = list_ops.tensor_list_split([1., 2., 3., 4., 5],
+                                   element_shape=None,
+                                   lengths=[3, 2])
+    self.assertAllEqual(list_ops.tensor_list_length(l), 2)
+    self.assertAllEqual(
+        list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32),
+        [1., 2., 3.])
+    self.assertAllEqual(
+        list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32),
+        [4., 5.])
+
+  def testSkipEagerSplitWithInvalidTensorShapeFails(self):
+    with self.cached_session():
+      tensor = array_ops.placeholder(dtype=dtypes.float32)
+      l = list_ops.tensor_list_split(tensor, element_shape=None, lengths=[1])
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"Tensor must be at least a vector, but saw shape: \[\]"):
+        l.eval({tensor: 1})
+
+  def testSkipEagerSplitWithInvalidLengthsShapeFails(self):
+    with self.cached_session():
+      lengths = array_ops.placeholder(dtype=dtypes.int64)
+      l = list_ops.tensor_list_split([1., 2.],
+                                     element_shape=None,
+                                     lengths=lengths)
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"Expected lengths to be a vector, received shape: \[\]"):
+        l.eval({lengths: 1})
+
+  def testSplitWithInvalidLengthsFails(self):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r"Invalid value in lengths: -1"):
+      l = list_ops.tensor_list_split([1., 2.],
+                                     element_shape=None,
+                                     lengths=[1, -1])
+      self.evaluate(l)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Attempting to slice \[0, 3\] from tensor with length 2"):
+      l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[3])
+      self.evaluate(l)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Unused values in tensor. Length of tensor: 2 Values used: 1"):
+      l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1])
+      self.evaluate(l)
+
+  def testSkipEagerSplitWithScalarElementShapeFails(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 r"Shapes must be equal rank, but are 1 and 0"):
+      l = list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1])
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"TensorListSplit requires element_shape to be at least of rank 1, "
+          r"but saw: \[\]"):
+        element_shape = array_ops.placeholder(dtype=dtypes.int32)
+        l = list_ops.tensor_list_split([1., 2.],
+                                       element_shape=element_shape,
+                                       lengths=[1, 1])
+        l.eval({element_shape: []})
+
+  def testEagerOnlySplitWithScalarElementShapeFails(self):
+    if context.executing_eagerly():
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"TensorListSplit requires element_shape to be at least of rank 1, "
+          r"but saw: \[\]"):
+        list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1])
+
+  def testSkipEagerSplitWithIncompatibleTensorShapeAndElementShapeFails(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 r"Shapes must be equal rank, but are 2 and 1"):
+      l = list_ops.tensor_list_split([[1.], [2.]],
+                                     element_shape=[1],
+                                     lengths=[1, 1])
+
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"):
+        element_shape = array_ops.placeholder(dtype=dtypes.int32)
+        l = list_ops.tensor_list_split([[1.], [2.]],
+                                       element_shape=element_shape,
+                                       lengths=[1, 1])
+        l.eval({element_shape: [1]})
+
+  def testEagerOnlySplitWithIncompatibleTensorShapeAndElementShapeFails(self):
+    if context.executing_eagerly():
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"):
+        list_ops.tensor_list_split([[1.], [2.]],
+                                   element_shape=[1],
+                                   lengths=[1, 1])
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 76e90ffea3..ed6d716ef1 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -123,10 +123,7 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayWritePack(dtypes.int64)
     self._testTensorArrayWritePack(dtypes.complex64)
     self._testTensorArrayWritePack(dtypes.complex128)
-    if not (test.is_gpu_available() and
-            tensor_array_ops.ENABLE_TENSOR_ARRAY_V2):
-      # TODO(b/119684648): Enable this.
-      self._testTensorArrayWritePack(dtypes.string)
+    self._testTensorArrayWritePack(dtypes.string)
 
   def testTensorArrayWritePack(self):
     self._testTensorArrayWritePackMaybeLegacy()
@@ -164,7 +161,6 @@ class TensorArrayTest(test.TestCase):
           convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0],
                    [106.0, 107.0], [8.0, 9.0]]), c0)
 
-  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
   @test_util.run_deprecated_v1
   def testTensorArrayWriteConcat(self):
     self._testTensorArrayWriteConcat(dtypes.float32)
@@ -255,10 +251,7 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayUnpackRead(dtypes.int64)
     self._testTensorArrayUnpackRead(dtypes.complex64)
     self._testTensorArrayUnpackRead(dtypes.complex128)
-    if not (test.is_gpu_available() and
-            tensor_array_ops.ENABLE_TENSOR_ARRAY_V2):
-      # TODO(b/119684648): Enable this.
-      self._testTensorArrayUnpackRead(dtypes.string)
+    self._testTensorArrayUnpackRead(dtypes.string)
 
   def testTensorArrayUnpackRead(self):
     self._testTensorArrayUnpackReadMaybeLegacy()
@@ -305,7 +298,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(convert([]).reshape(0, 2), d1)
       self.assertAllEqual(convert([[3.0, 301.0]]), d2)
 
-  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   @test_util.run_deprecated_v1
   def testTensorArraySplitRead(self):
     self._testTensorArraySplitRead(dtypes.float32)
@@ -512,7 +504,6 @@ class TensorArrayTest(test.TestCase):
           "it has already been written to."):
         self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow)
 
-  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
   @test_util.run_deprecated_v1
   def testTensorArrayConcatIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
@@ -545,7 +536,6 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaisesOpError("shape"):
         self.evaluate(w3.concat())
 
-  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   @test_util.run_deprecated_v1
   def testTensorArraySplitIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
@@ -559,22 +549,32 @@ class TensorArrayTest(test.TestCase):
           lengths = array_ops.placeholder(dtypes.int64)
           ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1})
 
-      with self.assertRaisesOpError(
-          r"Expected sum of lengths to be equal to values.shape\[0\], "
-          r"but sum of lengths is 1 and value's shape is: \[3\]"):
+      error_msg = ("Unused values in tensor. Length of tensor: 3 Values used: 1"
+                   if tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and
+                   not in_eager_mode else
+                   r"Expected sum of lengths to be equal to values.shape\[0\], "
+                   r"but sum of lengths is 1 and value's shape is: \[3\]")
+      with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.split([1.0, 2.0, 3.0], [1]).flow)
 
       ta = _make_ta(1, "baz")
-      with self.assertRaisesOpError(
-          r"Expected value to be at least a vector, but received shape: \[\]"):
-        self.evaluate(ta.split(1.0, [1]).flow)
+      if tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 and not in_eager_mode:
+        with self.assertRaisesRegexp(
+            ValueError, "Shape must be at least rank 1 but is rank 0"):
+          self.evaluate(ta.split(1.0, [1]).flow)
+      else:
+        with self.assertRaisesOpError(
+            r"Expected value to be at least a vector, but received shape: \[\]"
+        ):
+          self.evaluate(ta.split(1.0, [1]).flow)
 
-      ta = _make_ta(2, "buz")
-      with self.assertRaisesOpError(
-          r"TensorArray's size is not equal to the size of lengths "
-          r"\(2 vs. 1\), and the TensorArray is not marked as "
-          r"dynamically resizeable"):
-        self.evaluate(ta.split([1.0], [1]).flow)
+      if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2 or in_eager_mode:
+        ta = _make_ta(2, "buz")
+        with self.assertRaisesOpError(
+            r"TensorArray's size is not equal to the size of lengths "
+            r"\(2 vs. 1\), and the TensorArray is not marked as "
+            r"dynamically resizeable"):
+          self.evaluate(ta.split([1.0], [1]).flow)
 
   def _testTensorArrayWriteGradientAddMultipleAdds(self, dtype):
     with self.cached_session(use_gpu=True):
@@ -773,7 +773,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0])
       self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1])
 
-  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
   @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientWritePackConcatAndRead(self):
     self._testTensorArrayGradientWritePackConcatAndRead()
@@ -837,7 +836,6 @@ class TensorArrayTest(test.TestCase):
   def testSkipEagerTensorArrayGradientUnpackRead(self):
     self._testTensorArrayGradientUnpackRead()
 
-  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   @test_util.run_deprecated_v1
   def testSkipEagerTensorArrayGradientSplitConcat(self):
     with self.session(use_gpu=True) as session:
@@ -1168,7 +1166,6 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaises(ValueError):
         w0.write(0, c2)
 
-  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   @test_util.run_deprecated_v1
   def testSkipEagerPartlyUnknownShape(self):
     with self.session(use_gpu=True):
@@ -1245,7 +1242,6 @@ class TensorArrayTest(test.TestCase):
   def testUnpackShape(self):
     self._testUnpackShape()
 
-  @test_util.disable_control_flow_v2("b/118343962 (TensorArray.split)")
   @test_util.run_deprecated_v1
   def testSplitShape(self):
     with self.session(use_gpu=True):
@@ -1273,9 +1269,10 @@ class TensorArrayTest(test.TestCase):
         self.assertEqual((2, 2), w0.read(1).get_shape())
       else:
         self.assertEqual(r0.get_shape().ndims, None)
-        self.assertEqual(
-            tensor_shape.TensorShape(
-                ta1.handle.op.get_attr("element_shape")).ndims, None)
+        if not tensor_array_ops.ENABLE_TENSOR_ARRAY_V2:
+          self.assertEqual(
+              tensor_shape.TensorShape(
+                  ta1.handle.op.get_attr("element_shape")).ndims, None)
 
   @test_util.run_deprecated_v1
   def testSkipEagerWriteUnknownShape(self):
@@ -1325,7 +1322,7 @@ class TensorArrayTest(test.TestCase):
   def testSkipEagerTensorArrayUnpackDynamic(self):
     self._testTensorArrayUnpackDynamic()
 
-  @test_util.disable_control_flow_v2("b/118343594 (TensorArray.concat)")
+  @test_util.disable_control_flow_v2("b/117943489")
   @test_util.run_deprecated_v1
   def testSkipEagerTensorArraySplitDynamic(self):
     with self.session(use_gpu=True) as sess:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index 89ff48ebd9..dbaae886d4 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -30,7 +30,7 @@ from tensorflow.python.ops.gen_list_ops import *
 # pylint: enable=wildcard-import
 
 
-ops.NotDifferentiable("TensorListConcat")
+ops.NotDifferentiable("TensorListConcatLists")
 ops.NotDifferentiable("TensorListElementShape")
 ops.NotDifferentiable("TensorListLength")
 ops.NotDifferentiable("TensorListPushBackBatch")
@@ -65,6 +65,21 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
+def tensor_list_concat(input_handle, element_dtype, name=None):
+  # Ignore the lengths output of TensorListConcat. It is only used during
+  # gradient computation.
+  return gen_list_ops.tensor_list_concat(
+      input_handle=input_handle, element_dtype=element_dtype, name=name)[0]
+
+
+def tensor_list_split(tensor, element_shape, lengths, name=None):
+  return gen_list_ops.tensor_list_split(
+      tensor=tensor,
+      element_shape=_build_element_shape(element_shape),
+      lengths=lengths,
+      name=name)
+
+
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
@@ -86,6 +101,25 @@ def _TensorListStackGrad(unused_op, dtensor):
   return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:])
 
 
+@ops.RegisterGradient("TensorListConcat")
+def _TensorListConcatGrad(op, dtensor, unused_dlengths):
+  # TODO(srbs): We lose the element_shape information in tensor_list_concat.
+  # Consider providing that as an output of TensorListConcat?
+  if dtensor.shape.rank is None:
+    element_shape = None
+  else:
+    element_shape = [None] + dtensor.shape.as_list()[1:]
+  return tensor_list_split(
+      dtensor,
+      element_shape=_build_element_shape(element_shape),
+      lengths=op.outputs[1])
+
+
+@ops.RegisterGradient("TensorListSplit")
+def _TensorListSplitGrad(op, dlist):
+  return tensor_list_concat(dlist, element_dtype=op.inputs[0].dtype), None, None
+
+
 @ops.RegisterGradient("TensorListFromTensor")
 def _TensorListFromTensorGrad(op, dlist):
   """Gradient for TensorListFromTensor."""
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index e3375ad0ab..d151694951 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -586,7 +586,11 @@ class _GraphTensorArrayV2(object):
 
   def concat(self, name=None):
     """See TensorArray."""
-    raise NotImplementedError("TensorArray.concat")
+    value = list_ops.tensor_list_concat(
+        input_handle=self._flow, element_dtype=self._dtype, name=name)
+    if self._element_shape and self._element_shape[0].dims is not None:
+      value.set_shape([None] + self._element_shape[0].dims[1:])
+    return value
 
   @tf_should_use.should_use_result
   def unstack(self, value, name=None):
@@ -630,7 +634,30 @@ class _GraphTensorArrayV2(object):
   @tf_should_use.should_use_result
   def split(self, value, lengths, name=None):
     """See TensorArray."""
-    raise NotImplementedError("TensorArray.split")
+    with ops.name_scope(name, "TensorArraySplit", [self._flow, value, lengths]):
+      value = ops.convert_to_tensor(value, name="value")
+      lengths_64 = math_ops.to_int64(lengths)
+      if self._infer_shape and not context.executing_eagerly():
+        clengths = tensor_util.constant_value(lengths_64)
+        if value.shape.dims is not None:
+          if clengths is not None and clengths.max() == clengths.min():
+            self._merge_element_shape(
+                tensor_shape.TensorShape([clengths[0]]).concatenate(
+                    value.shape[1:]))
+      flow_out = list_ops.tensor_list_split(
+          tensor=value,
+          lengths=lengths_64,
+          element_shape=self._element_shape[0] if self._element_shape else None,
+          name=name)
+      ta = TensorArray(
+          dtype=self._dtype,
+          handle=self.handle,
+          flow=flow_out,
+          colocate_with_first_write_call=self._colocate_with_first_write_call)
+      ta._infer_shape = self._infer_shape
+      ta._element_shape = self._element_shape
+      ta._colocate_with = self._colocate_with
+      return ta
 
   def size(self, name=None):
     """See TensorArray."""
-- 
GitLab


From d9833ee20411543db72c98a833bda75639c96763 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 08:18:58 -0800
Subject: [PATCH 1104/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 223518108
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 52 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 52 +++++++++++++++++++
 2 files changed, 104 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 735ba6b0f8..170b839443 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -75673,6 +75673,25 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "TensorListConcat"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  output_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+}
 op {
   name: "TensorListConcatLists"
   input_arg {
@@ -75933,6 +75952,39 @@ op {
     type: "type"
   }
 }
+op {
+  name: "TensorListSplit"
+  input_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  input_arg {
+    name: "element_shape"
+    type_attr: "shape_type"
+  }
+  input_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorListStack"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 699646651e..f194d608be 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36087,6 +36087,25 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "TensorListConcat"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  output_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+}
 op {
   name: "TensorListConcatLists"
   input_arg {
@@ -36347,6 +36366,39 @@ op {
     type: "type"
   }
 }
+op {
+  name: "TensorListSplit"
+  input_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  input_arg {
+    name: "element_shape"
+    type_attr: "shape_type"
+  }
+  input_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "TensorListStack"
   input_arg {
-- 
GitLab


From f5291176d424b05e96c9ebdc4a3f4965ff5f7a1a Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Fri, 30 Nov 2018 08:37:01 -0800
Subject: [PATCH 1105/1554] [XLA] Document the iota_dimension attribute for
 kIota

PiperOrigin-RevId: 223520378
---
 tensorflow/compiler/xla/g3doc/operation_semantics.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index e0807518bc..d888b1f23f 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -1466,10 +1466,11 @@ Builds a constant literal on device rather than a potentially large host
 transfer. Creates a rank 1 array of values starting at zero and incrementing by
 one.
 
-Arguments | Type            | Semantics
---------- | --------------- | ------------------------------------
-`type`    | `PrimitiveType` | type U
-`size`    | `int64`         | The number of elements in the array.
+Arguments        | Type            | Semantics
+---------------- | --------------- | ------------------------------------
+`type`           | `PrimitiveType` | type U
+`size`           | `int64`         | The number of elements in the array.
+`iota_dimension` | `int64`         | The dimension to increment along.
 
 ## Map
 
-- 
GitLab


From 3e66cee177a072657ad2aea628ad395446d1479e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 08:48:24 -0800
Subject: [PATCH 1106/1554] Rename WriteToTensor to make it explicit the fact
 that it set the shape to {size}

PiperOrigin-RevId: 223521732
---
 tensorflow/lite/interpreter_test.cc                  |  2 +-
 tensorflow/lite/java/src/main/native/tensor_jni.cc   |  2 +-
 tensorflow/lite/kernels/gather.cc                    |  2 +-
 tensorflow/lite/kernels/hashtable_lookup.cc          |  2 +-
 tensorflow/lite/kernels/skip_gram.cc                 |  4 ++--
 tensorflow/lite/kernels/test_util.h                  |  2 +-
 tensorflow/lite/models/smartreply/ops/normalize.cc   |  2 +-
 tensorflow/lite/models/smartreply/predictor.cc       |  2 +-
 tensorflow/lite/string_util.cc                       |  7 +++++--
 tensorflow/lite/string_util.h                        | 12 +++++++++---
 tensorflow/lite/string_util_test.cc                  |  6 +++---
 .../lite/tools/benchmark/benchmark_tflite_model.cc   |  2 +-
 12 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index 09b8832b4d..dd3d85e7cd 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -566,7 +566,7 @@ TEST(BasicInterpreter, ThreeStepAllocate) {
     DynamicBuffer buf;
     StringRef str_ref = GetString(input, 0);
     buf.AddString(str_ref);
-    buf.WriteToTensor(output);
+    buf.WriteToTensorAsVector(output);
     return kTfLiteOk;
   };
 
diff --git a/tensorflow/lite/java/src/main/native/tensor_jni.cc b/tensorflow/lite/java/src/main/native/tensor_jni.cc
index 035bec6a44..82d2679de9 100644
--- a/tensorflow/lite/java/src/main/native/tensor_jni.cc
+++ b/tensorflow/lite/java/src/main/native/tensor_jni.cc
@@ -278,7 +278,7 @@ void WriteMultiDimensionalStringArray(JNIEnv* env, jobject src,
   tflite::DynamicBuffer dst_buffer;
   PopulateStringDynamicBuffer(env, src, &dst_buffer, tensor->dims->size);
   if (!env->ExceptionCheck()) {
-    dst_buffer.WriteToTensor(tensor);
+    dst_buffer.WriteToTensor(tensor, /*new_shape=*/nullptr);
   }
 }
 
diff --git a/tensorflow/lite/kernels/gather.cc b/tensorflow/lite/kernels/gather.cc
index 61884d6a12..f205daae13 100644
--- a/tensorflow/lite/kernels/gather.cc
+++ b/tensorflow/lite/kernels/gather.cc
@@ -118,7 +118,7 @@ TfLiteStatus GatherStrings(TfLiteContext* context, const TfLiteTensor* input,
     const auto string_ref = GetString(input, pos);
     buffer.AddString(string_ref.str, string_ref.len);
   }
-  buffer.WriteToTensor(output);
+  buffer.WriteToTensorAsVector(output);
   return kTfLiteOk;
 }
 
diff --git a/tensorflow/lite/kernels/hashtable_lookup.cc b/tensorflow/lite/kernels/hashtable_lookup.cc
index b6ae7a3d1a..da1116cf85 100644
--- a/tensorflow/lite/kernels/hashtable_lookup.cc
+++ b/tensorflow/lite/kernels/hashtable_lookup.cc
@@ -137,7 +137,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     }
   }
   if (output->type == kTfLiteString) {
-    buf.WriteToTensor(output);
+    buf.WriteToTensorAsVector(output);
   }
 
   return kTfLiteOk;
diff --git a/tensorflow/lite/kernels/skip_gram.cc b/tensorflow/lite/kernels/skip_gram.cc
index f20719ecaf..265ba18a3e 100644
--- a/tensorflow/lite/kernels/skip_gram.cc
+++ b/tensorflow/lite/kernels/skip_gram.cc
@@ -107,7 +107,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   // Generate n-grams recursively.
   tflite::DynamicBuffer buf;
   if (words.size() < params->ngram_size) {
-    buf.WriteToTensor(GetOutput(context, node, 0));
+    buf.WriteToTensorAsVector(GetOutput(context, node, 0));
     return kTfLiteOk;
   }
 
@@ -145,7 +145,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     }
   }
 
-  buf.WriteToTensor(GetOutput(context, node, 0));
+  buf.WriteToTensorAsVector(GetOutput(context, node, 0));
   return kTfLiteOk;
 }
 }  // namespace
diff --git a/tensorflow/lite/kernels/test_util.h b/tensorflow/lite/kernels/test_util.h
index a49d6c2cae..dadabb86ab 100644
--- a/tensorflow/lite/kernels/test_util.h
+++ b/tensorflow/lite/kernels/test_util.h
@@ -199,7 +199,7 @@ class SingleOpModel {
     for (const string& s : content) {
       buf.AddString(s.data(), s.length());
     }
-    buf.WriteToTensor(tensor);
+    buf.WriteToTensor(tensor, /*new_shape=*/nullptr);
   }
 
   // Populate the tensor given its index.
diff --git a/tensorflow/lite/models/smartreply/ops/normalize.cc b/tensorflow/lite/models/smartreply/ops/normalize.cc
index 8480260f27..3cb11cc055 100644
--- a/tensorflow/lite/models/smartreply/ops/normalize.cc
+++ b/tensorflow/lite/models/smartreply/ops/normalize.cc
@@ -92,7 +92,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 
   tflite::DynamicBuffer buf;
   buf.AddString(result.data(), result.length());
-  buf.WriteToTensor(GetOutput(context, node, 0));
+  buf.WriteToTensorAsVector(GetOutput(context, node, 0));
   return kTfLiteOk;
 }
 
diff --git a/tensorflow/lite/models/smartreply/predictor.cc b/tensorflow/lite/models/smartreply/predictor.cc
index 7db2502977..59bf4a3cf1 100644
--- a/tensorflow/lite/models/smartreply/predictor.cc
+++ b/tensorflow/lite/models/smartreply/predictor.cc
@@ -49,7 +49,7 @@ void ExecuteTfLite(const std::string& sentence,
     TfLiteTensor* input = interpreter->tensor(interpreter->inputs()[0]);
     tflite::DynamicBuffer buf;
     buf.AddString(sentence.data(), sentence.length());
-    buf.WriteToTensor(input);
+    buf.WriteToTensorAsVector(input);
     interpreter->AllocateTensors();
 
     interpreter->Invoke();
diff --git a/tensorflow/lite/string_util.cc b/tensorflow/lite/string_util.cc
index 1b33f5bcba..6efa11d60c 100644
--- a/tensorflow/lite/string_util.cc
+++ b/tensorflow/lite/string_util.cc
@@ -96,8 +96,7 @@ int DynamicBuffer::WriteToBuffer(char** buffer) {
   return bytes;
 }
 
-void DynamicBuffer::WriteToTensor(TfLiteTensor* tensor) {
-  // Set tensor content pointer to tensor_buffer, and release original data.
+void DynamicBuffer::WriteToTensorAsVector(TfLiteTensor* tensor) {
   auto dims = TfLiteIntArrayCreate(1);
   dims->data[0] = offset_.size() - 1;  // Store number of strings.
   WriteToTensor(tensor, dims);
@@ -108,6 +107,10 @@ void DynamicBuffer::WriteToTensor(TfLiteTensor* tensor,
   char* tensor_buffer;
   int bytes = WriteToBuffer(&tensor_buffer);
 
+  if (new_shape == nullptr) {
+    new_shape = TfLiteIntArrayCopy(tensor->dims);
+  }
+
   // Set tensor content pointer to tensor_buffer, and release original data.
   TfLiteTensorReset(tensor->type, tensor->name, new_shape, tensor->params,
                     tensor_buffer, bytes, kTfLiteDynamic, tensor->allocation,
diff --git a/tensorflow/lite/string_util.h b/tensorflow/lite/string_util.h
index c9b74482f7..f076db76f2 100644
--- a/tensorflow/lite/string_util.h
+++ b/tensorflow/lite/string_util.h
@@ -74,12 +74,18 @@ class DynamicBuffer {
   // The function allocates space for the buffer but does NOT take ownership.
   int WriteToBuffer(char** buffer);
 
-  // Fill content into a string tensor, with the given new_shape. The new
-  // shape must match the number of strings in this object.
+  // Fill content into a string tensor, with the given new_shape. The new shape
+  // must match the number of strings in this object. Caller relinquishes
+  // ownership of new_shape. If 'new_shape' is nullptr, keep the tensor's
+  // existing shape.
   void WriteToTensor(TfLiteTensor* tensor, TfLiteIntArray* new_shape);
 
   // Fill content into a string tensor. Set shape to {num_strings}.
-  void WriteToTensor(TfLiteTensor* tensor);
+  void WriteToTensorAsVector(TfLiteTensor* tensor);
+
+  // Deprecated. Use WriteToTensorAsVector() or pass in the new shpe.
+  // TODO(b/120230709): remove when people migrate away.
+  void WriteToTensor(TfLiteTensor* tensor) { WriteToTensorAsVector(tensor); }
 
  private:
   // Data buffer to store contents of strings, not including headers.
diff --git a/tensorflow/lite/string_util_test.cc b/tensorflow/lite/string_util_test.cc
index 377cdd77eb..ba4aee79fd 100644
--- a/tensorflow/lite/string_util_test.cc
+++ b/tensorflow/lite/string_util_test.cc
@@ -55,7 +55,7 @@ TEST(StringUtil, TestStringUtil) {
   new_shape->data[0] = 2;
   new_shape->data[1] = 1;
   buf0.WriteToTensor(t0, new_shape);
-  buf1.WriteToTensor(t1);
+  buf1.WriteToTensorAsVector(t1);
 
   // Check tensor shapes.
   EXPECT_EQ(t0->dims->size, 2);
@@ -99,7 +99,7 @@ TEST(StringUtil, TestAddJoinedString) {
 
   DynamicBuffer buf;
   buf.AddJoinedString({{s0, 3}, {s1, 4}, {s2, 0}, {s3, 3}}, ' ');
-  buf.WriteToTensor(t0);
+  buf.WriteToTensorAsVector(t0);
 
   ASSERT_EQ(GetStringCount(t0), 1);
   StringRef str_ref;
@@ -115,7 +115,7 @@ TEST(StringUtil, TestEmptyList) {
   t0->type = kTfLiteString;
   t0->allocation_type = kTfLiteDynamic;
   DynamicBuffer buf;
-  buf.WriteToTensor(t0);
+  buf.WriteToTensorAsVector(t0);
 
   ASSERT_EQ(GetStringCount(t0), 0);
   ASSERT_EQ(t0->bytes, 8);
diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index 7768b75f76..16f70870b6 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -279,7 +279,7 @@ void BenchmarkTfLiteModel::PrepareInputsAndOutputs() {
       FillRandomString(&buffer, sizes, []() {
         return "we're have some friends over saturday to hang out in the yard";
       });
-      buffer.WriteToTensor(interpreter->tensor(i));
+      buffer.WriteToTensor(interpreter->tensor(i), /*new_shape=*/nullptr);
     } else {
       TFLITE_LOG(FATAL) << "Don't know how to populate tensor " << t->name
                         << " of type " << t->type;
-- 
GitLab


From 5d45c7e715eb1ea8b69133da52228b34ccb3d29d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 08:52:00 -0800
Subject: [PATCH 1107/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 223522179

---
 tensorflow/go/op/wrappers.go | 142 +++++++++++++++++------------------
 1 file changed, 71 insertions(+), 71 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index ad979c58be..b0ed1b151f 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5645,6 +5645,77 @@ func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size
 	return op.Output(0)
 }
 
+// MapUnstageAttr is an optional argument to MapUnstage.
+type MapUnstageAttr func(optionalAttr)
+
+// MapUnstageCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageCapacity(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageContainer(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageSharedName(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns the values associated with the key
+//
+// from the underlying container.   If the underlying container
+// does not contain this key, the op will block until it does.
+func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapUnstage",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstage", err)
+		return
+	}
+	return values
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -33883,74 +33954,3 @@ func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output
 	}
 	return scope.AddOperation(opspec)
 }
-
-// MapUnstageAttr is an optional argument to MapUnstage.
-type MapUnstageAttr func(optionalAttr)
-
-// MapUnstageCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageCapacity(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageContainer(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageSharedName(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the values associated with the key
-//
-// from the underlying container.   If the underlying container
-// does not contain this key, the op will block until it does.
-func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapUnstage",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstage", err)
-		return
-	}
-	return values
-}
-- 
GitLab


From 5b14577d42842871f1cb0eb8dfe77d32db1eb654 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 11:11:23 -0800
Subject: [PATCH 1108/1554] Use Starlark version of http_archive

The native http_archive and new_http_archive will be removed in 0.20.0

PiperOrigin-RevId: 223544202
---
 WORKSPACE | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 0c7bc085b5..7cc08e0164 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,5 +1,7 @@
 workspace(name = "org_tensorflow")
 
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
 http_archive(
     name = "io_bazel_rules_closure",
     sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae",
@@ -57,9 +59,9 @@ android_workspace()
 # Please add all new TensorFlow dependencies in workspace.bzl.
 tf_workspace()
 
-new_http_archive(
+http_archive(
     name = "inception_v1",
-    build_file = "models.BUILD",
+    build_file = "//:models.BUILD",
     sha256 = "7efe12a8363f09bc24d7b7a450304a15655a57a7751929b2c1593a71183bb105",
     urls = [
         "http://storage.googleapis.com/download.tensorflow.org/models/inception_v1.zip",
@@ -67,9 +69,9 @@ new_http_archive(
     ],
 )
 
-new_http_archive(
+http_archive(
     name = "mobile_ssd",
-    build_file = "models.BUILD",
+    build_file = "//:models.BUILD",
     sha256 = "bddd81ea5c80a97adfac1c9f770e6f55cbafd7cce4d3bbe15fbeb041e6b8f3e8",
     urls = [
         "http://storage.googleapis.com/download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_android_export.zip",
@@ -77,9 +79,9 @@ new_http_archive(
     ],
 )
 
-new_http_archive(
+http_archive(
     name = "mobile_multibox",
-    build_file = "models.BUILD",
+    build_file = "//:models.BUILD",
     sha256 = "859edcddf84dddb974c36c36cfc1f74555148e9c9213dedacf1d6b613ad52b96",
     urls = [
         "http://storage.googleapis.com/download.tensorflow.org/models/mobile_multibox_v1a.zip",
@@ -87,9 +89,9 @@ new_http_archive(
     ],
 )
 
-new_http_archive(
+http_archive(
     name = "stylize",
-    build_file = "models.BUILD",
+    build_file = "//:models.BUILD",
     sha256 = "3d374a730aef330424a356a8d4f04d8a54277c425e274ecb7d9c83aa912c6bfa",
     urls = [
         "http://storage.googleapis.com/download.tensorflow.org/models/stylize_v1.zip",
@@ -97,9 +99,9 @@ new_http_archive(
     ],
 )
 
-new_http_archive(
+http_archive(
     name = "speech_commands",
-    build_file = "models.BUILD",
+    build_file = "//:models.BUILD",
     sha256 = "c3ec4fea3158eb111f1d932336351edfe8bd515bb6e87aad4f25dbad0a600d0c",
     urls = [
         "http://storage.googleapis.com/download.tensorflow.org/models/speech_commands_v0.01.zip",
-- 
GitLab


From 92e236dd24a5e52b44fbfccd75f4833b68e14b75 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Fri, 30 Nov 2018 11:12:00 -0800
Subject: [PATCH 1109/1554] Fix for outside compilation.

When generating Placeholder node for host computation/outside compilation nodes, we have to generate different Placeholder node for different src_output edges.

PiperOrigin-RevId: 223544322
---
 tensorflow/compiler/jit/encapsulate_util.cc   | 21 +++++----
 .../compiler/jit/encapsulate_util_test.cc     | 46 +++++++++++++++++--
 2 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_util.cc b/tensorflow/compiler/jit/encapsulate_util.cc
index bcc3213285..1f4b9c90a4 100644
--- a/tensorflow/compiler/jit/encapsulate_util.cc
+++ b/tensorflow/compiler/jit/encapsulate_util.cc
@@ -241,7 +241,7 @@ Status ProcessDataEdgeBetweenOutsideCompilationAndHostComputation(
 
   // Remove the edge from host to outside compilation. Add a placeholder as
   // outside compilation node input.
-  std::map<string, Node*> placeholders;
+  std::map<std::pair<string, int>, Node*> placeholders;
   for (int i = 0; i < edges.size(); i++) {
     Node* dst = g->FindNodeId(edges[i].dst_node_id);
     const Edge* e;
@@ -253,9 +253,10 @@ Status ProcessDataEdgeBetweenOutsideCompilationAndHostComputation(
     // Find or create placeholder node.
     string new_name =
         edges[i].is_host_to_outside_compilation
-            ? absl::StrCat(src->name(), "_host_to_oc_placeholder")
-            : absl::StrCat(src->name(), "_oc_to_host_placeholder");
-    auto iter = placeholders.find(new_name);
+            ? absl::StrCat(src->name(), "_host_to_oc_placeholder_", src_output)
+            : absl::StrCat(src->name(), "_oc_to_host_placeholder_", src_output);
+    auto placeholder_index = std::make_pair(src->name(), src_output);
+    auto iter = placeholders.find(placeholder_index);
     Node* placeholder_node;
     if (iter == placeholders.end()) {
       NodeDefBuilder placeholder_builder(new_name, "Placeholder");
@@ -288,7 +289,7 @@ Status ProcessDataEdgeBetweenOutsideCompilationAndHostComputation(
       Status s;
       placeholder_node = g->AddNode(placeholder_def, &s);
       TF_RETURN_IF_ERROR(s);
-      placeholders[new_name] = placeholder_node;
+      placeholders[placeholder_index] = placeholder_node;
     } else {
       placeholder_node = iter->second;
     }
@@ -642,7 +643,7 @@ Status PreprocessDataEdgesBetweenOutsideCompilations(
 
   // Remove the edge from host to outside compilation. Add a placeholder as
   // outside compilation node input.
-  std::map<string, Node*> placeholders;
+  std::map<std::pair<string, int>, Node*> placeholders;
   for (int i = 0; i < edges.size(); i++) {
     Node* dst = g->FindNodeId(edges[i].dst_node_id);
     const Edge* e;
@@ -652,8 +653,10 @@ Status PreprocessDataEdgesBetweenOutsideCompilations(
     g->RemoveEdge(e);
 
     // Find or create placeholder node.
-    string new_name = absl::StrCat(src->name(), "_oc_to_oc_placeholder");
-    auto iter = placeholders.find(new_name);
+    string new_name =
+        absl::StrCat(src->name(), "_oc_to_oc_placeholder_", src_output);
+    auto placeholder_index = std::make_pair(src->name(), src_output);
+    auto iter = placeholders.find(placeholder_index);
     Node* placeholder_node;
     if (iter == placeholders.end()) {
       NodeDefBuilder placeholder_builder(new_name, "Placeholder");
@@ -673,7 +676,7 @@ Status PreprocessDataEdgesBetweenOutsideCompilations(
       Status s;
       placeholder_node = g->AddNode(placeholder_def, &s);
       TF_RETURN_IF_ERROR(s);
-      placeholders[new_name] = placeholder_node;
+      placeholders[placeholder_index] = placeholder_node;
     } else {
       placeholder_node = iter->second;
     }
diff --git a/tensorflow/compiler/jit/encapsulate_util_test.cc b/tensorflow/compiler/jit/encapsulate_util_test.cc
index 25c32cef01..3b8b49cb92 100644
--- a/tensorflow/compiler/jit/encapsulate_util_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_util_test.cc
@@ -153,23 +153,33 @@ TEST(PreprocessForEncapsulationTest, ControlEdges) {
 TEST(PreprocessForEncapsulationTest, DataEdges) {
   // Build the graph:
   // "const_0" and "const_1" in host computation
+  // "identityn0" = ("const_0", "const_1") in host computation 0
   // "add0" = "const_0" + "const_1" in XLA computation 0
   // "add1" = "add0" + "const_0" in XLA computation 0 & outside compilation 0
   // "identity0" = "add1" in XLA computation 0
   // "add2" = "add1" + "identity0" in host computation
   // "add3" = "add1" + "add2" in XLA computation 1
-  // "add4" = "identity0" + "add2" in XLA computation 1 & outside compilation 1
+  // "add4" = "identity0" + "add2" in XLA computation 1 & outside compilation 0
+  // "add5" = "identityn0"[0] + "identityn0"[1] in XLA computation 1 &
+  //                                               outside compilation 0
+  // "identityn1" = ("identityn0"[0], "identityn0"[1]) in XLA computation 1 &
+  //                                                   outside compilation 0
   // "identity1" = "add4" in XLA computation 1
   // "identity2" = "identity1" in host computation
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output const_0 = ops::Const(s.WithOpName("const_0"), 1, {});
   Output const_1 = ops::Const(s.WithOpName("const_1"), 2, {});
+  auto identityn0 =
+      ops::IdentityN(s.WithOpName("identityn_0"), {const_0, const_1});
   Output add0 = ops::Add(s.WithOpName("add0"), const_0, const_1);
   Output add1 = ops::Add(s.WithOpName("add1"), add0, const_0);
   Output identity0 = ops::Identity(s.WithOpName("identity0"), add1);
   Output add2 = ops::Add(s.WithOpName("add2"), add1, identity0);
   Output add3 = ops::Add(s.WithOpName("add3"), add1, add2);
   Output add4 = ops::Add(s.WithOpName("add4"), identity0, add2);
+  Output add5 = ops::Add(s.WithOpName("add5"), identityn0[0], identityn0[1]);
+  auto identityn1 = ops::IdentityN(s.WithOpName("identityn_1"),
+                                   {identityn0[0], identityn0[1]});
   Output identity1 = ops::Identity(s.WithOpName("identity1"), add4);
   Output identity2 = ops::Identity(s.WithOpName("identity2"), add4);
   Graph g(OpRegistry::Global());
@@ -180,6 +190,8 @@ TEST(PreprocessForEncapsulationTest, DataEdges) {
   Node *add0_node = node_index["add0"], *add1_node = node_index["add1"],
        *identity0_node = node_index["identity0"],
        *add3_node = node_index["add3"], *add4_node = node_index["add4"],
+       *add5_node = node_index["add5"],
+       *identityn1_node = node_index["identityn_1"],
        *identity1_node = node_index["identity1"];
   add0_node->AddAttr("_xla", "0");
   add1_node->AddAttr("_xla", "0");
@@ -188,6 +200,10 @@ TEST(PreprocessForEncapsulationTest, DataEdges) {
   add3_node->AddAttr("_xla", "1");
   add4_node->AddAttr("_xla", "1");
   add4_node->AddAttr("_oc", "0");
+  add5_node->AddAttr("_xla", "1");
+  add5_node->AddAttr("_oc", "0");
+  identityn1_node->AddAttr("_xla", "1");
+  identityn1_node->AddAttr("_oc", "0");
   identity1_node->AddAttr("_xla", "1");
 
   TF_CHECK_OK(PreprocessForEncapsulation(&g, "_xla", "_oc"));
@@ -205,8 +221,9 @@ TEST(PreprocessForEncapsulationTest, DataEdges) {
   EXPECT_NE(bridge_identity0_add4, nullptr);
   // Step 3: add placeholder for edges between host computation and outside
   // compilation.
-  EXPECT_EQ(bridge_add1_add3->def().input(0), "add1_oc_to_host_placeholder");
-  Node *add1_oc_to_host_placeholder = node_index["add1_oc_to_host_placeholder"];
+  EXPECT_EQ(bridge_add1_add3->def().input(0), "add1_oc_to_host_placeholder_0");
+  Node *add1_oc_to_host_placeholder =
+      node_index["add1_oc_to_host_placeholder_0"];
   TF_CHECK_OK(GetNodeAttr(add1_oc_to_host_placeholder->attrs(),
                           kOutsideCompilationToHostOriginalNodeAttrName, &str));
   EXPECT_EQ(str, "add1");
@@ -217,15 +234,34 @@ TEST(PreprocessForEncapsulationTest, DataEdges) {
   add4_node = node_index["add4"];
   ASSERT_NE(add4_node, nullptr);
   EXPECT_EQ(add4_node->def().input(0),
-            "bridge_identity0_add4_host_to_oc_placeholder");
+            "bridge_identity0_add4_host_to_oc_placeholder_0");
   Node *identity0_host_to_oc_placeholder =
-      node_index["bridge_identity0_add4_host_to_oc_placeholder"];
+      node_index["bridge_identity0_add4_host_to_oc_placeholder_0"];
   TF_CHECK_OK(GetNodeAttr(identity0_host_to_oc_placeholder->attrs(),
                           kHostToOutsideCompilationOriginalNodeAttrName, &str));
   EXPECT_EQ(str, "bridge_identity0_add4");
   TF_CHECK_OK(GetNodeAttr(identity0_host_to_oc_placeholder->attrs(),
                           kHostToOutsideCompilationSrcOutputAttrName, &i));
   EXPECT_EQ(i, 0);
+
+  // Check different placeholder nodes are created for different src_output.
+  Node *placeholder0 = node_index["identityn_0_host_to_oc_placeholder_0"],
+       *placeholder1 = node_index["identityn_0_host_to_oc_placeholder_1"];
+  EXPECT_NE(placeholder0, nullptr);
+  EXPECT_NE(placeholder1, nullptr);
+  // Check we only have 2 placeholder nodes created for "identityn_0".
+  int placeholder_count = 0;
+  for (Node *n : g.nodes()) {
+    if (HasNodeAttr(n->def(), kHostToOutsideCompilationOriginalNodeAttrName)) {
+      string attr;
+      TF_CHECK_OK(GetNodeAttr(
+          n->attrs(), kHostToOutsideCompilationOriginalNodeAttrName, &attr));
+      if (attr == "identityn_0") {
+        ++placeholder_count;
+      }
+    }
+  }
+  EXPECT_EQ(placeholder_count, 2);
 }
 
 TEST(PostprocessForEncapsulationTest, ControlEdges) {
-- 
GitLab


From f23efde5109d0f387ecdc535deaf9fb45de5d391 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 30 Nov 2018 11:13:46 -0800
Subject: [PATCH 1110/1554] Keep the old instruction alive when replacing
 instructions in GetDimensionSizeRewriter.

Otherwise it could create use-after-free if we delete and iterate
through instructions at the same time.

PiperOrigin-RevId: 223544664
---
 .../xla/service/hlo_get_dimension_size_rewriter.cc       | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
index 631b3ad735..c919dbd82d 100644
--- a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
@@ -39,7 +39,7 @@ StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
   uint32 size = instr->operand(0)->shape().dimensions(instr->dimension());
   HloInstruction* new_instr = computation->AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(size)));
-  TF_RETURN_IF_ERROR(computation->ReplaceInstruction(instr, new_instr));
+  TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(new_instr));
   return true;
 }
 
@@ -50,12 +50,7 @@ StatusOr<bool> HloGetDimensionSizeRewriter::Run(HloModule* module) {
   HloProto proto;
   *proto.mutable_hlo_module() = module->ToProto();
   for (auto* computation : module->computations()) {
-    // Replacing instructions will change the instruction list in the
-    // computation. So instead of iterating computation->instructions()
-    // directly, we make a copy of the list to avoid use-after-free.
-    std::vector<HloInstruction*> instrs(computation->instruction_count());
-    absl::c_copy(computation->instructions(), instrs.begin());
-    for (auto instruction : instrs) {
+    for (auto instruction : computation->instructions()) {
       TF_ASSIGN_OR_RETURN(bool replaced, ReplaceGetSize(instruction));
       changed = changed || replaced;
     }
-- 
GitLab


From a46412b9384b82593f69df04df49bfbe2c7f245c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 12:03:04 -0800
Subject: [PATCH 1111/1554] Release quantize-trained inception v1, v2, v4
 trained on ImageNet.

PiperOrigin-RevId: 223553382
---
 tensorflow/lite/g3doc/models.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/g3doc/models.md b/tensorflow/lite/g3doc/models.md
index 537e285490..62b3f17c79 100644
--- a/tensorflow/lite/g3doc/models.md
+++ b/tensorflow/lite/g3doc/models.md
@@ -76,8 +76,11 @@ Mobilenet_V1_1.0_128_quant  | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tf
 Mobilenet_V1_1.0_160_quant  | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_160_quant.tgz)  | 4.3 Mb     | 66.9%          | 86.7%          | 37.4 ms
 Mobilenet_V1_1.0_192_quant  | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_192_quant.tgz)  | 4.3 Mb     | 69.1%          | 88.1%          | 51.9 ms
 Mobilenet_V1_1.0_224_quant  | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz)  | 4.3 Mb     | 70.0%          | 89.0%          | 70.2 ms
-Mobilenet_v2_1.0_224_quant  | [paper](https://arxiv.org/abs/1806.08342), [tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/mobilenet_v2_1.0_224_quant.tgz)              | 3.4 Mb     | 70.8%          | 89.9%          | 80.3 ms
-Inception_v3_quant          | [paper](https://arxiv.org/abs/1806.08342),[tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/inception_v3_quant.tgz)                       | 23 Mb      | 77.5%          | 93.7%          | 637 ms
+Mobilenet_V2_1.0_224_quant  | [paper](https://arxiv.org/abs/1806.08342), [tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/mobilenet_v2_1.0_224_quant.tgz)              | 3.4 Mb     | 70.8%          | 89.9%          | 80.3 ms
+Inception_V1_quant          | [paper](https://arxiv.org/abs/1409.4842), [tflite&pb](http://download.tensorflow.org/models/inception_v1_224_quant_20181026.tgz)                          | 6.4 Mb     | 70.1%          | 89.8%          | 154.5 ms
+Inception_V2_quant          | [paper](https://arxiv.org/abs/1512.00567), [tflite&pb](http://download.tensorflow.org/models/inception_v2_224_quant_20181026.tgz)                         | 11 Mb      | 73.5%          | 91.4%          | 235.0 ms
+Inception_V3_quant          | [paper](https://arxiv.org/abs/1806.08342),[tflite&pb](http://download.tensorflow.org/models/tflite_11_05_08/inception_v3_quant.tgz)                       | 23 Mb      | 77.5%          | 93.7%          | 637 ms
+Inception_V4_quant          | [paper](https://arxiv.org/abs/1602.07261), [tflite&pb](http://download.tensorflow.org/models/inception_v4_299_quant_20181026.tgz)                         | 41 Mb      | 79.5%          | 93.9%          | 1250.8 ms
 
 ## Other models
 
-- 
GitLab


From 07d5d08579bbbff910653a59163b4f8f180d16ac Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 13:09:55 -0800
Subject: [PATCH 1112/1554] Support for testing string models with the flex
 delegate.

PiperOrigin-RevId: 223563214
---
 tensorflow/lite/delegates/flex/test_util.cc | 45 ++++++++++++++++++---
 tensorflow/lite/delegates/flex/test_util.h  |  2 +
 tensorflow/lite/string_util_test.cc         | 31 ++++++++++++++
 3 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/test_util.cc b/tensorflow/lite/delegates/flex/test_util.cc
index 08feb349e6..aa24675a7b 100644
--- a/tensorflow/lite/delegates/flex/test_util.cc
+++ b/tensorflow/lite/delegates/flex/test_util.cc
@@ -25,6 +25,29 @@ namespace testing {
 
 bool FlexModelTest::Invoke() { return interpreter_->Invoke() == kTfLiteOk; }
 
+void FlexModelTest::SetStringValues(int tensor_index,
+                                    const std::vector<string>& values) {
+  DynamicBuffer dynamic_buffer;
+  for (const string& s : values) {
+    dynamic_buffer.AddString(s.data(), s.size());
+  }
+  dynamic_buffer.WriteToTensor(interpreter_->tensor(tensor_index),
+                               /*new_shape=*/nullptr);
+}
+
+std::vector<string> FlexModelTest::GetStringValues(int tensor_index) const {
+  std::vector<string> result;
+
+  TfLiteTensor* tensor = interpreter_->tensor(tensor_index);
+  auto num_strings = GetStringCount(tensor->data.raw);
+  for (size_t i = 0; i < num_strings; ++i) {
+    auto ref = GetString(tensor->data.raw, i);
+    result.push_back(string(ref.str, ref.len));
+  }
+
+  return result;
+}
+
 void FlexModelTest::SetShape(int tensor_index, const std::vector<int>& values) {
   ASSERT_EQ(interpreter_->ResizeInputTensor(tensor_index, values), kTfLiteOk);
   ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
@@ -95,12 +118,22 @@ void FlexModelTest::AddTfOp(TfOpType op, const std::vector<int>& inputs,
     return " attr{ key: '" + key + "' value {" + value + "}}";
   };
 
-  // Crude type attribution, will need fleshing out as more tests are added.
-  // TODO(b/113613439): Use nodedef string utilities to properly handle
-  // all types.
-  string type_attribute = attr("T", "type: DT_FLOAT");
-  if (interpreter_->tensor(inputs[0])->type == kTfLiteInt32) {
-    type_attribute = attr("T", "type: DT_INT32");
+  string type_attribute;
+  switch (interpreter_->tensor(inputs[0])->type) {
+    case kTfLiteInt32:
+      type_attribute = attr("T", "type: DT_INT32");
+      break;
+    case kTfLiteFloat32:
+      type_attribute = attr("T", "type: DT_FLOAT");
+      break;
+    case kTfLiteString:
+      type_attribute = attr("T", "type: DT_STRING");
+      break;
+    default:
+      // TODO(b/113613439): Use nodedef string utilities to properly handle all
+      // types.
+      LOG(FATAL) << "Type not supported";
+      break;
   }
 
   if (op == kUnpack) {
diff --git a/tensorflow/lite/delegates/flex/test_util.h b/tensorflow/lite/delegates/flex/test_util.h
index 4d3f5ad096..2cc2dc30e9 100644
--- a/tensorflow/lite/delegates/flex/test_util.h
+++ b/tensorflow/lite/delegates/flex/test_util.h
@@ -63,11 +63,13 @@ class FlexModelTest : public ::testing::Test {
   void SetValues(int tensor_index, const std::vector<float>& values) {
     SetTypedValues<float>(tensor_index, values);
   }
+  void SetStringValues(int tensor_index, const std::vector<string>& values);
 
   // Returns the tensor's values at the given index.
   std::vector<float> GetValues(int tensor_index) {
     return GetTypedValues<float>(tensor_index);
   }
+  std::vector<string> GetStringValues(int tensor_index) const;
 
   // Sets the tensor's shape at the given index.
   void SetShape(int tensor_index, const std::vector<int>& values);
diff --git a/tensorflow/lite/string_util_test.cc b/tensorflow/lite/string_util_test.cc
index ba4aee79fd..cbf1d7b226 100644
--- a/tensorflow/lite/string_util_test.cc
+++ b/tensorflow/lite/string_util_test.cc
@@ -121,6 +121,37 @@ TEST(StringUtil, TestEmptyList) {
   ASSERT_EQ(t0->bytes, 8);
 }
 
+TEST(StringUtil, TestShapes) {
+  Interpreter interpreter;
+  interpreter.AddTensors(1);
+  TfLiteTensor* t0 = interpreter.tensor(0);
+  t0->type = kTfLiteString;
+  t0->allocation_type = kTfLiteDynamic;
+  t0->dims = TfLiteIntArrayCreate(2);
+  t0->dims->data[0] = 2;
+  t0->dims->data[1] = 1;
+
+  // Not setting a new shape: number of strings must match
+  DynamicBuffer buf;
+  buf.AddString("ABC", 3);
+  buf.AddString("X", 1);
+  buf.WriteToTensor(t0, nullptr);
+
+  ASSERT_EQ(t0->dims->size, 2);
+  EXPECT_EQ(t0->dims->data[0], 2);
+  EXPECT_EQ(t0->dims->data[1], 1);
+
+  auto new_shape = TfLiteIntArrayCreate(2);
+  new_shape->data[0] = 1;
+  new_shape->data[1] = 2;
+
+  buf.WriteToTensor(t0, new_shape);
+
+  ASSERT_EQ(t0->dims->size, 2);
+  EXPECT_EQ(t0->dims->data[0], 1);
+  EXPECT_EQ(t0->dims->data[1], 2);
+}
+
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-- 
GitLab


From e7a123f4b361b8104b783d8b1407b45dc087491c Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 30 Nov 2018 13:33:25 -0800
Subject: [PATCH 1113/1554] Add max_bazel_version to not use bleeding edge
 Bazel in TF.

PiperOrigin-RevId: 223566575
---
 configure.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/configure.py b/configure.py
index f087da002d..57a03bd17f 100644
--- a/configure.py
+++ b/configure.py
@@ -452,11 +452,12 @@ def convert_version_to_int(version):
   return int(version_str)
 
 
-def check_bazel_version(min_version):
-  """Check installed bazel version is at least min_version.
+def check_bazel_version(min_version, max_version):
+  """Check installed bazel version is between min_version and max_version.
 
   Args:
     min_version: string for minimum bazel version.
+    max_version: string for maximum bazel version.
 
   Returns:
     The bazel version detected.
@@ -474,6 +475,7 @@ def check_bazel_version(min_version):
 
   min_version_int = convert_version_to_int(min_version)
   curr_version_int = convert_version_to_int(curr_version)
+  max_version_int = convert_version_to_int(max_version)
 
   # Check if current bazel version can be detected properly.
   if not curr_version_int:
@@ -487,6 +489,10 @@ def check_bazel_version(min_version):
     print('Please upgrade your bazel installation to version %s or higher to '
           'build TensorFlow!' % min_version)
     sys.exit(0)
+  if curr_version_int > max_version_int:
+    print('Please downgrade your bazel installation to version %s or lower to '
+          'build TensorFlow!' % min_version)
+    sys.exit(0)
   return curr_version
 
 
@@ -1559,7 +1565,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.15.0')
+  check_bazel_version('0.15.0', '0.19.2')
 
   reset_tf_configure_bazelrc()
   # Explicitly import tools/bazel.rc, this is needed for Bazel 0.19.0 or later
-- 
GitLab


From 32715d16b4d810d8fe711855ee68a5279481e252 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 30 Nov 2018 13:39:56 -0800
Subject: [PATCH 1114/1554] Fix tests in list_ops_test for 2.0.

PiperOrigin-RevId: 223567747
---
 tensorflow/python/kernel_tests/list_ops_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 41c2708fbc..489f6c9b00 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -880,6 +880,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(self.evaluate(result_0), [6., 8.])
     self.assertAllEqual(self.evaluate(result_1), [10., 12.])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerConcatShapeInference(self):
 
     def BuildTensor(element_shape):
@@ -1004,6 +1005,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32),
         [4., 5.])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSplitWithInvalidTensorShapeFails(self):
     with self.cached_session():
       tensor = array_ops.placeholder(dtype=dtypes.float32)
@@ -1013,6 +1015,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           r"Tensor must be at least a vector, but saw shape: \[\]"):
         l.eval({tensor: 1})
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSplitWithInvalidLengthsShapeFails(self):
     with self.cached_session():
       lengths = array_ops.placeholder(dtype=dtypes.int64)
@@ -1042,6 +1045,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1])
       self.evaluate(l)
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSplitWithScalarElementShapeFails(self):
     with self.assertRaisesRegexp(ValueError,
                                  r"Shapes must be equal rank, but are 1 and 0"):
@@ -1065,6 +1069,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           r"but saw: \[\]"):
         list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1])
 
+  @test_util.run_deprecated_v1
   def testSkipEagerSplitWithIncompatibleTensorShapeAndElementShapeFails(self):
     with self.assertRaisesRegexp(ValueError,
                                  r"Shapes must be equal rank, but are 2 and 1"):
-- 
GitLab


From e53c859fcc2e6d685e6c104f894013a0b522ea2e Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Fri, 30 Nov 2018 13:48:51 -0800
Subject: [PATCH 1115/1554] Apply review fixes

---
 tensorflow/tools/dockerfiles/assembler.py | 33 ++++++++++++-----------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tools/dockerfiles/assembler.py b/tensorflow/tools/dockerfiles/assembler.py
index 8d97e1d7dc..9d8a59aebc 100644
--- a/tensorflow/tools/dockerfiles/assembler.py
+++ b/tensorflow/tools/dockerfiles/assembler.py
@@ -1,3 +1,17 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
 """Multipurpose TensorFlow Docker Helper.
 
 - Assembles Dockerfiles
@@ -213,19 +227,6 @@ class TfDockerTagValidator(cerberus.Validator):
       self.partials = kwargs['partials']
     super(cerberus.Validator, self).__init__(*args, **kwargs)
 
-  def _validate_isvariant(self, isvariant, field, value):
-    """Validate that a variant references an existing variant spec.
-
-    Args:
-      isvariant: Value of the rule, a bool
-      field: The field being validated
-      value: The field's value
-    The rule's arguments are validated against this schema:
-    {'type': 'boolean'}
-    """
-    if isvariant and value not in self.root_document.get('variants', dict()):
-      self._error(field, '{} is not an existing variant.'.format(value))
-
   def _validate_ispartial(self, ispartial, field, value):
     """Validate that a partial references an existing partial spec.
 
@@ -261,7 +262,7 @@ def eprint(*args, **kwargs):
 
 
 def aggregate_all_slice_combinations(spec, slice_set_names):
-  """Figure out all of the variant groupings for a spec."""
+  """Figure out all of the possible slice groupings for a tag spec."""
   slice_sets = copy.deepcopy(spec['slice_sets'])
 
   for name in slice_set_names:
@@ -336,9 +337,9 @@ def gather_tag_args(slices, cli_input_args, required_args):
   args = update_args_dict(args, cli_input_args)
   for arg in required_args:
     if arg not in args:
-      eprint(('> Error: {} is not a valid variant, and also isn\'t an arg '
+      eprint(('> Error: {} is not a valid slice_set, and also isn\'t an arg '
               'provided on the command line. If it is an arg, please specify '
-              'it with --arg. If not, check the variants list.'.format(arg)))
+              'it with --arg. If not, check the slice_sets list.'.format(arg)))
       exit(1)
 
   return args
-- 
GitLab


From 8f2fd6fde3236b034a3a7edcb48c5858bad0692e Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 30 Nov 2018 13:40:08 -0800
Subject: [PATCH 1116/1554] [TF:XLA] Bump open source llvm revision to r347980

PiperOrigin-RevId: 223567779
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index dde10ef73d..ba981f43a8 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "f4791ba3e166918bca82df34e2f854e8e188d6055888c64cb28743fd43f2d0d7",
-        strip_prefix = "llvm-b2a42b2112a511a5077fd747fb21e45349cff08d",
+        sha256 = "ba61783c407c8decb2df1ca1c7019cbd0bb38419509ae8870fa7ce6bc30b6cfb",
+        strip_prefix = "llvm-cdc3118297348c68b25dff92b504581f5dd78000",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/b2a42b2112a511a5077fd747fb21e45349cff08d.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/b2a42b2112a511a5077fd747fb21e45349cff08d.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/cdc3118297348c68b25dff92b504581f5dd78000.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/cdc3118297348c68b25dff92b504581f5dd78000.tar.gz",
         ],
     )
 
-- 
GitLab


From 028c2468e5d8e57d5b8d76bc1b28ac14d2efda7c Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 30 Nov 2018 13:44:49 -0800
Subject: [PATCH 1117/1554] [XLA] Add DumpAllPathsFromTo graph dumper function.

PiperOrigin-RevId: 223568539
---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 81 ++++++++++++++++++-
 .../compiler/xla/service/hlo_graph_dumper.h   |  6 ++
 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 804feff290..0745c735e5 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <deque>
 #include <map>
 #include <memory>
+#include <queue>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -578,7 +579,7 @@ bool HloDotDumper::ShouldShowSubcomputation(const HloComputation* subcomp) {
 
   // Show the subcomputation if we're showing any of its members.
   return std::any_of(
-      computation_->instructions().begin(), computation_->instructions().end(),
+      subcomp->instructions().begin(), subcomp->instructions().end(),
       [&](const HloInstruction* instr) { return filter_.Show(instr); });
 }
 
@@ -1298,7 +1299,8 @@ namespace {
 
 // Gets a NodeFilter that includes roughly all instructions whose distance from
 // root is <= radius.
-NodeFilter MakeNodeFilter(const HloInstruction* root, int64 radius) {
+NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
+                                      int64 radius) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
   std::unordered_map<const HloInstruction*, NodeFilterResult> nodes;
@@ -1405,6 +1407,56 @@ NodeFilter MakeNodeFilter(const HloInstruction* root, int64 radius) {
   });
 }
 
+// Gets a node filter that includes nodes on all paths from `from` to `to`.  If
+// the all-paths set contains more than max_nodes elements, includes the nodes
+// on the shortest paths and sets hit_limit to true.
+NodeFilter MakeNodeFromToFilter(const HloInstruction* from,
+                                const HloInstruction* to, int64 max_nodes,
+                                bool* hit_limit) {
+  *hit_limit = false;
+
+  // Elements in the queue are paths through the graph.
+  std::deque<std::vector<const HloInstruction*>> queue;
+  queue.push_front({from});
+
+  // Compute the set of nodes we want to show using a slightly-modified
+  // Djikstra's algorithm.  The only real difference is, rather than stopping
+  // when we find a (shortest) path, we continue until we've found max_nodes
+  // nodes on some path.
+  std::unordered_set<const HloInstruction*> visited;
+  std::unordered_set<const HloInstruction*> to_display = {from, to};
+  while (!queue.empty() && to_display.size() < max_nodes) {
+    std::vector<const HloInstruction*> path = std::move(queue.front());
+    queue.pop_front();
+    if (!visited.insert(path.back()).second) {
+      continue;
+    }
+
+    for (const auto* user : path.back()->users()) {
+      if (user == to) {
+        auto it = path.begin();
+        for (; it != path.end() && to_display.size() < max_nodes; ++it) {
+          to_display.insert(*it);
+        }
+        if (it != path.end()) {
+          *hit_limit = true;
+        }
+      } else if (!visited.count(user)) {
+        auto new_path = path;
+        new_path.push_back(user);
+        queue.push_back(std::move(new_path));
+      }
+    }
+  }
+
+  return NodeFilter([=](const HloInstruction* instr) {
+    if (instr == from || instr == to) {
+      return kHighlightNode;
+    }
+    return to_display.count(instr) ? kNormalNode : kHideNode;
+  });
+}
+
 string SaveGraph(const string& graph,
                  GraphRendererInterface::GraphKind graph_kind,
                  const string& dest_path) {
@@ -1484,7 +1536,7 @@ string DumpNeighborhoodAround(const HloInstruction& node, int radius,
   auto debug_options = node.GetModule()->config().debug_options();
   string label =
       StrCat("Neighborhood of ", radius, " nodes around ", node.name());
-  NodeFilter filter = MakeNodeFilter(&node, radius);
+  NodeFilter filter = MakeNodeRadiusAroundFilter(&node, radius);
   string graph =
       HloDotDumper(node.parent(), label, debug_options, show_backend_config,
                    /*profile=*/nullptr, filter)
@@ -1492,6 +1544,29 @@ string DumpNeighborhoodAround(const HloInstruction& node, int radius,
   return ExportGraph(graph, GraphRendererInterface::DOT_GRAPH, debug_options);
 }
 
+string DumpAllPathsFromTo(const HloInstruction& from, const HloInstruction& to,
+                          int64 max_nodes, bool show_backend_config) {
+  CHECK_EQ(from.parent(), to.parent()) << "Nodes must be in same computation!";
+  auto debug_options = from.GetModule()->config().debug_options();
+
+  bool hit_limit = false;
+  NodeFilter filter = MakeNodeFromToFilter(&from, &to, max_nodes, &hit_limit);
+  string label;
+  if (!hit_limit) {
+    label = StrCat("All paths from ", from.name(), " to ", to.name());
+  } else {
+    label = StrCat(max_nodes, " nodes on the shortest paths from ", from.name(),
+                   " to ", to.name(),
+                   "<br/><br/>***SHOWING ONLY A SUBSET OF ALL PATHS BETWEEN "
+                   "NODES***<br/><br/>");
+  }
+  string graph =
+      HloDotDumper(from.parent(), label, debug_options, show_backend_config,
+                   /*profile=*/nullptr, filter)
+          .Dump();
+  return ExportGraph(graph, GraphRendererInterface::DOT_GRAPH, debug_options);
+}
+
 void DumpText(const HloModule& module, const string& label,
               const string& directory_path, bool do_prefix) {
   Env* env = Env::Default();
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.h b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
index 8d5945aba8..de1eefab77 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.h
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.h
@@ -66,6 +66,12 @@ string DumpGraph(const HloComputation& computation, const string& label,
 string DumpNeighborhoodAround(const HloInstruction& node, int radius,
                               bool show_backend_config = false);
 
+// Dumps nodes on any of the paths from `from` to `to`.  If there are more than
+// max_nodes on all paths, restricts to the max_nodes nodes on the shortest
+// paths.
+string DumpAllPathsFromTo(const HloInstruction& from, const HloInstruction& to,
+                          int64 max_nodes, bool show_backend_config = false);
+
 // Dumps the HloModule::ToString() as a file into the provided directory path
 // suffixed with the provided label.
 //
-- 
GitLab


From 8650885a7a1e037919d8870b77a24e2e33b80a0c Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 30 Nov 2018 13:49:36 -0800
Subject: [PATCH 1118/1554] Move some testing utils from
 estimator_training_test to multi_worker_test_base, preparing for
 independent-worker mode in keras_multi_worker_test

PiperOrigin-RevId: 223569411
---
 .../python/estimator_training_test.py         | 101 +++--------------
 .../python/multi_worker_test_base.py          | 102 +++++++++++++++++-
 2 files changed, 117 insertions(+), 86 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 0f35657a80..8c0f594202 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -24,7 +24,6 @@ import json
 import os
 import sys
 import tempfile
-import threading
 from absl.testing import parameterized
 import numpy as np
 
@@ -70,57 +69,19 @@ PS = dc._TaskType.PS
 original_run_std_server = dc._run_std_server
 
 
-class MockOsEnv(dict):
-
-  def __init__(self, *args):
-    self._thread_local = threading.local()
-    super(MockOsEnv, self).__init__(*args)
-
-  def get(self, key, default):
-    if not hasattr(self._thread_local, "dict"):
-      self._thread_local.dict = dict()
-    if key == "TF_CONFIG":
-      return dict.get(self._thread_local.dict, key, default)
-    else:
-      return dict.get(self, key, default)
-
-  def __getitem__(self, key):
-    if not hasattr(self._thread_local, "dict"):
-      self._thread_local.dict = dict()
-    if key == "TF_CONFIG":
-      return dict.__getitem__(self._thread_local.dict, key)
-    else:
-      return dict.__getitem__(self, key)
-
-  def __setitem__(self, key, val):
-    if not hasattr(self._thread_local, "dict"):
-      self._thread_local.dict = dict()
-    if key == "TF_CONFIG":
-      return dict.__setitem__(self._thread_local.dict, key, val)
-    else:
-      return dict.__setitem__(self, key, val)
-
-
-class DistributeCoordinatorIntegrationTest(test.TestCase,
-                                           parameterized.TestCase):
+class DistributeCoordinatorIntegrationTest(
+    multi_worker_test_base.IndependentWorkerTestBase, parameterized.TestCase):
 
   @classmethod
   def setUpClass(cls):
     """Create a local cluster with 2 workers."""
+    super(DistributeCoordinatorIntegrationTest, cls).setUpClass()
     cls._cluster_spec = multi_worker_test_base.create_in_process_cluster(
         num_workers=3, num_ps=2, has_eval=True)
 
   def setUp(self):
     self._model_dir = tempfile.mkdtemp()
-    self._mock_os_env = MockOsEnv()
-    self._mock_context = test.mock.patch.object(os, "environ",
-                                                self._mock_os_env)
     super(DistributeCoordinatorIntegrationTest, self).setUp()
-    self._mock_context.__enter__()
-
-  def tearDown(self):
-    self._mock_context.__exit__(None, None, None)
-    super(DistributeCoordinatorIntegrationTest, self).tearDown()
 
   def dataset_input_fn(self, x, y, batch_size, shuffle):
 
@@ -143,8 +104,8 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
   def _extract_loss_and_global_step(self, event_folder):
     """Returns the loss and global step in last event."""
     event_paths = glob.glob(os.path.join(event_folder, "events*"))
-    self.assertGreater(len(event_paths), 0,
-                       msg="Event file not found in dir %s" % event_folder)
+    self.assertNotEmpty(
+        event_paths, msg="Event file not found in dir %s" % event_folder)
 
     loss = None
     global_step_count = None
@@ -362,47 +323,15 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
     self._barrier.wait()
     return ret
 
-  def _task_thread(self, train_distribute, eval_distribute, tf_config):
-    os.environ["TF_CONFIG"] = json.dumps(tf_config)
+  def _independent_worker_fn(
+      self,
+      train_distribute,
+      eval_distribute,
+  ):
     with test.mock.patch.object(dc, "_run_std_server",
                                 self._mock_run_std_server):
       self._complete_flow(train_distribute, eval_distribute)
 
-  def _run_task_in_thread(self, cluster_spec, task_type, task_id,
-                          train_distribute, eval_distribute):
-    if task_type:
-      tf_config = {
-          "cluster": cluster_spec,
-          "task": {
-              "type": task_type,
-              "index": task_id
-          }
-      }
-    else:
-      tf_config = {
-          "cluster": cluster_spec,
-          "task": {
-              "type": task_type,
-              "index": task_id
-          }
-      }
-    t = threading.Thread(
-        target=self._task_thread,
-        args=(train_distribute, eval_distribute, tf_config))
-    t.start()
-    return t
-
-  def _run_multiple_tasks_in_threads(self, cluster_spec, train_distribute,
-                                     eval_distribute):
-    threads = {}
-    for task_type in cluster_spec.keys():
-      threads[task_type] = []
-      for task_id in range(len(cluster_spec[task_type])):
-        t = self._run_task_in_thread(cluster_spec, task_type, task_id,
-                                     train_distribute, eval_distribute)
-        threads[task_type].append(t)
-    return threads
-
   @combinations.generate(
       combinations.combine(
           mode=["graph"],
@@ -443,8 +372,9 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
       # 3 workers and 1 evaluator.
       self._barrier = dc._Barrier(4)
 
-    threads = self._run_multiple_tasks_in_threads(
-        cluster_spec, train_distribute, eval_distribute)
+    threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn,
+                                                 cluster_spec, train_distribute,
+                                                 eval_distribute)
     for task_type, ts in threads.items():
       if task_type == PS:
         continue
@@ -482,8 +412,9 @@ class DistributeCoordinatorIntegrationTest(test.TestCase,
         num_workers=3, num_ps=0, has_eval=True)
     # 3 workers and 1 evaluator.
     self._barrier = dc._Barrier(4)
-    threads = self._run_multiple_tasks_in_threads(
-        cluster_spec, train_distribute, eval_distribute)
+    threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn,
+                                                 cluster_spec, train_distribute,
+                                                 eval_distribute)
     threads[WORKER][0].join()
     threads[EVALUATOR][0].join()
 
diff --git a/tensorflow/contrib/distribute/python/multi_worker_test_base.py b/tensorflow/contrib/distribute/python/multi_worker_test_base.py
index 8eec3dc0f6..147c9b83f8 100644
--- a/tensorflow/contrib/distribute/python/multi_worker_test_base.py
+++ b/tensorflow/contrib/distribute/python/multi_worker_test_base.py
@@ -18,8 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import contextlib
 import copy
+import json
+import os
 import threading
 import numpy as np
 
@@ -271,7 +274,6 @@ class MultiWorkerTestBase(test.TestCase):
 
     return config
 
-
   def _run_client(self, client_fn, task_type, task_id, num_gpus, *args,
                   **kwargs):
     result = client_fn(task_type, task_id, num_gpus, *args, **kwargs)
@@ -303,3 +305,101 @@ class MultiWorkerTestBase(test.TestCase):
     for t in threads:
       t.join()
     self.assertEqual(self._result, len(threads))
+
+
+class MockOsEnv(collections.Mapping):
+  """A class that allows per-thread TF_CONFIG."""
+
+  def __init__(self, *args):
+    self._dict = dict()
+    self._thread_local = threading.local()
+    super(MockOsEnv, self).__init__(*args)
+
+  def get(self, key, default=None):
+    if not hasattr(self._thread_local, 'dict'):
+      self._thread_local.dict = dict()
+    if key == 'TF_CONFIG':
+      return dict.get(self._thread_local.dict, key, default)
+    else:
+      return dict.get(self._dict, key, default)
+
+  def __getitem__(self, key):
+    if not hasattr(self._thread_local, 'dict'):
+      self._thread_local.dict = dict()
+    if key == 'TF_CONFIG':
+      return dict.__getitem__(self._thread_local.dict, key)
+    else:
+      return dict.__getitem__(self._dict, key)
+
+  def __setitem__(self, key, val):
+    if not hasattr(self._thread_local, 'dict'):
+      self._thread_local.dict = dict()
+    if key == 'TF_CONFIG':
+      return dict.__setitem__(self._thread_local.dict, key, val)
+    else:
+      return dict.__setitem__(self._dict, key, val)
+
+  def __iter__(self):
+    if not hasattr(self._thread_local, 'dict'):
+      self._thread_local.dict = dict()
+    for x in self._thread_local.dict.items():
+      yield x
+    for x in self._dict.items():
+      yield x
+
+  def __len__(self):
+    if not hasattr(self._thread_local, 'dict'):
+      self._thread_local.dict = dict()
+    return self._thread_local.dict.__len__() + self._dict.__len__()
+
+
+class IndependentWorkerTestBase(test.TestCase):
+  """Testing infra for independent workers."""
+
+  def setUp(self):
+    self._mock_os_env = MockOsEnv()
+    self._mock_context = test.mock.patch.object(os, 'environ',
+                                                self._mock_os_env)
+    super(IndependentWorkerTestBase, self).setUp()
+    self._mock_context.__enter__()
+
+  def tearDown(self):
+    self._mock_context.__exit__(None, None, None)
+    super(IndependentWorkerTestBase, self).tearDown()
+
+  def _task_thread(self, task_fn, tf_config, *args, **kwargs):
+    os.environ['TF_CONFIG'] = json.dumps(tf_config)
+    task_fn(*args, **kwargs)
+
+  def _run_task_in_thread(self, task_fn, cluster_spec, task_type, task_id,
+                          *args, **kwargs):
+    if task_type:
+      tf_config = {
+          'cluster': cluster_spec,
+          'task': {
+              'type': task_type,
+              'index': task_id
+          }
+      }
+    else:
+      tf_config = {
+          'cluster': cluster_spec,
+      }
+    t = threading.Thread(
+        target=self._task_thread,
+        args=(task_fn, tf_config) + args,
+        kwargs=kwargs)
+    t.start()
+    return t
+
+  def run_multiple_tasks_in_threads(self, task_fn, cluster_spec, *args,
+                                    **kwargs):
+    # The task_fn should create std_server by itself.
+    threads = {}
+    for task_type in cluster_spec.keys():
+      threads[task_type] = []
+      for task_id in range(len(cluster_spec[task_type])):
+        t = self._run_task_in_thread(task_fn, cluster_spec, task_type, task_id,
+                                     *args, **kwargs)
+        threads[task_type].append(t)
+    return threads
-- 
GitLab


From 2ed052bf501112e96319dc990f37e2801e575449 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Fri, 30 Nov 2018 14:34:09 -0800
Subject: [PATCH 1119/1554] Add Jupyter tutorial readme

---
 .../tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile | 1 +
 .../tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile       | 1 +
 .../tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile | 1 +
 .../tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile       | 1 +
 .../tools/dockerfiles/partials/jupyter.partial.Dockerfile      | 1 +
 tensorflow/tools/dockerfiles/readme-for-jupyter.md             | 3 +++
 6 files changed, 8 insertions(+)
 create mode 100644 tensorflow/tools/dockerfiles/readme-for-jupyter.md

diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
index ecc8fabcce..9577920e13 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
@@ -105,6 +105,7 @@ RUN apt-get install -y --no-install-recommends wget
 WORKDIR /tf/tensorflow-tutorials
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
 RUN apt-get autoremove -y && apt-get remove -y wget
 WORKDIR /tf
 EXPOSE 8888
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
index 166e255289..b663cca8e2 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
@@ -61,6 +61,7 @@ RUN apt-get install -y --no-install-recommends wget
 WORKDIR /tf/tensorflow-tutorials
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
 RUN apt-get autoremove -y && apt-get remove -y wget
 WORKDIR /tf
 EXPOSE 8888
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
index b77ba52f25..17c1d5b904 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
@@ -137,6 +137,7 @@ RUN apt-get install -y --no-install-recommends wget
 WORKDIR /tf/tensorflow-tutorials
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
 RUN apt-get autoremove -y && apt-get remove -y wget
 WORKDIR /tf
 EXPOSE 8888
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index 9d7340abf3..cb0eb5a3d3 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -92,6 +92,7 @@ RUN apt-get install -y --no-install-recommends wget
 WORKDIR /tf/tensorflow-tutorials
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
 RUN apt-get autoremove -y && apt-get remove -y wget
 WORKDIR /tf
 EXPOSE 8888
diff --git a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
index 51b22d1fdf..c4ec6095c0 100644
--- a/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/jupyter.partial.Dockerfile
@@ -6,6 +6,7 @@ RUN apt-get install -y --no-install-recommends wget
 WORKDIR /tf/tensorflow-tutorials
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_classification.ipynb
 RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/basic_text_classification.ipynb
+COPY readme-for-jupyter.md README.md
 RUN apt-get autoremove -y && apt-get remove -y wget
 WORKDIR /tf
 EXPOSE 8888
diff --git a/tensorflow/tools/dockerfiles/readme-for-jupyter.md b/tensorflow/tools/dockerfiles/readme-for-jupyter.md
new file mode 100644
index 0000000000..f104a7533b
--- /dev/null
+++ b/tensorflow/tools/dockerfiles/readme-for-jupyter.md
@@ -0,0 +1,3 @@
+Want more tutorials like these?
+
+Check out tensorflow.org/tutorials!
-- 
GitLab


From e4ff419499fca61a6f213bc7c4777005371eadea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 14:30:37 -0800
Subject: [PATCH 1120/1554] Automated rollback of commit
 b17d53c0cd07bbae46a55dc9c12d3f48a1c19604

PiperOrigin-RevId: 223576240
---
 tensorflow/python/ops/parallel_for/control_flow_ops_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index c2484766aa..cc20d7ca6a 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -1133,7 +1133,7 @@ class TensorArrayTest(PForTest):
     # y = x * x. Hence dy/dx = 2 * x.
     actual_grad = 2.0 * x
     with session.Session() as sess:
-      actual_grad, computed_grad = self.evaluate([t1, actual_grad])
+      actual_grad, computed_grad = sess.run([t1, actual_grad])
       self.assertAllClose(actual_grad, computed_grad)
 
 
@@ -1287,7 +1287,7 @@ class ControlFlowTest(PForTest):
     expected_output = array_ops.transpose(expected_output, [1, 0])
 
     with session.Session() as sess:
-      out, expected = self.evaluate([out, expected_output])
+      out, expected = sess.run([out, expected_output])
       self.assertAllClose(expected, out)
 
   def test_tensor_array_as_loop_variable(self):
@@ -1475,7 +1475,7 @@ class Benchmarks(test.Benchmark):
     sess = session.Session()
     with sess:
       init = variables.global_variables_initializer()
-      self.evaluate(init)
+      sess.run(init)
       run_fn = sess.make_callable(targets)
       run_fn()  # Warm up
       begin = time.time()
-- 
GitLab


From 71eff2925951e70eff7dfa623e662825526b4c52 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 30 Nov 2018 14:34:20 -0800
Subject: [PATCH 1121/1554] Add a test that validates renames specified in
 tf_upgrade_v2.py. Fix a few issues found running the test.

PiperOrigin-RevId: 223576827
---
 tensorflow/tools/compatibility/renames_v2.py  |   9 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  20 ++-
 .../tools/compatibility/tf_upgrade_v2_test.py | 143 ++++++++++++++++--
 3 files changed, 147 insertions(+), 25 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5a27eb241c..62032449a4 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -153,7 +153,6 @@ renames = {
     'tf.erf': 'tf.math.erf',
     'tf.erfc': 'tf.math.erfc',
     'tf.expm1': 'tf.math.expm1',
-    'tf.extract_image_patches': 'tf.compat.v1.extract_image_patches',
     'tf.fake_quant_with_min_max_args': 'tf.quantization.fake_quant_with_min_max_args',
     'tf.fake_quant_with_min_max_args_gradient': 'tf.quantization.fake_quant_with_min_max_args_gradient',
     'tf.fake_quant_with_min_max_vars': 'tf.quantization.fake_quant_with_min_max_vars',
@@ -260,6 +259,11 @@ renames = {
     'tf.lbeta': 'tf.math.lbeta',
     'tf.lgamma': 'tf.math.lgamma',
     'tf.lin_space': 'tf.linspace',
+    'tf.lite.constants.FLOAT': 'tf.compat.v1.lite.constants.FLOAT',
+    'tf.lite.constants.INT32': 'tf.compat.v1.lite.constants.INT32',
+    'tf.lite.constants.INT64': 'tf.compat.v1.lite.constants.INT64',
+    'tf.lite.constants.QUANTIZED_UINT8': 'tf.compat.v1.lite.constants.QUANTIZED_UINT8',
+    'tf.lite.constants.STRING': 'tf.compat.v1.lite.constants.STRING',
     'tf.local_variables': 'tf.compat.v1.local_variables',
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log': 'tf.math.log',
@@ -405,7 +409,6 @@ renames = {
     'tf.quantized_concat': 'tf.quantization.quantized_concat',
     'tf.random.get_seed': 'tf.compat.v1.random.get_seed',
     'tf.random.set_random_seed': 'tf.compat.v1.random.set_random_seed',
-    'tf.random.stateless_multinomial': 'tf.compat.v1.random.stateless_multinomial',
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
@@ -415,7 +418,6 @@ renames = {
     'tf.read_file': 'tf.io.read_file',
     'tf.real': 'tf.math.real',
     'tf.reciprocal': 'tf.math.reciprocal',
-    'tf.reduce_join': 'tf.compat.v1.reduce_join',
     'tf.regex_replace': 'tf.strings.regex_replace',
     'tf.report_uninitialized_variables': 'tf.compat.v1.report_uninitialized_variables',
     'tf.reset_default_graph': 'tf.compat.v1.reset_default_graph',
@@ -503,7 +505,6 @@ renames = {
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
     'tf.sparse.reduce_max_sparse': 'tf.compat.v1.sparse.reduce_max_sparse',
     'tf.sparse.reduce_sum_sparse': 'tf.compat.v1.sparse.reduce_sum_sparse',
-    'tf.sparse_add': 'tf.compat.v1.sparse_add',
     'tf.sparse_fill_empty_rows': 'tf.sparse.fill_empty_rows',
     'tf.sparse_mask': 'tf.sparse.mask',
     'tf.sparse_matmul': 'tf.compat.v1.sparse_matmul',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 83ef73b8fa..992080025d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -47,11 +47,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.expand_dims": {
             "dim": "axis",
         },
-        "tf.batch_to_space_nd": {
+        "tf.batch_to_space": {
             "block_size": "block_shape",
         },
         "tf.constant": {
-            "verify_shapes": "verify_shapes_is_now_always_true",
+            "verify_shape": "verify_shape_is_now_always_true",
         },
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
@@ -131,9 +131,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "m": "mean",
             "v": "variance",
         },
-        "tf.manip.batch_to_space_nd": {
-            "block_size": "block_shape",
-        },
         "tf.nn.dilation2d": {
             "filter": "filters",
             "rates": "dilations",
@@ -187,8 +184,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "newpath": "dst",
         },
         "tf.gfile.Rename": {
-            "oldpath": "src",
-            "newpath": "dst",
+            "oldname": "src",
+            "newname": "dst",
         },
         "tf.gfile.Walk": {
             "in_order": "topdown",
@@ -293,6 +290,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.manual_symbol_renames = {
         "tf.batch_to_space_nd":
             "tf.batch_to_space",
+        "tf.extract_image_patches":
+            "tf.image.extract_image_patches",
         "tf.gfile.Copy":
             "tf.io.gfile.copy",
         "tf.gfile.DeleteRecursively":
@@ -409,10 +408,14 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.batch_to_space",
         "tf.quantize_v2":
             "tf.quantization.quantize",
+        "tf.sparse_add":
+            "tf.sparse.add",
         "tf.sparse_concat":
             "tf.sparse.concat",
         "tf.sparse_split":
             "tf.sparse.split",
+        "tf.random.stateless_multinomial":
+            "tf.random.stateless_categorical",
         "tf.string_to_hash_bucket":
             "tf.strings.to_hash_bucket",
         "tf.string_to_number":
@@ -421,6 +424,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.random.categorical",
         "tf.random.multinomial":
             "tf.random.categorical",
+        "tf.reduce_join":
+            "tf.strings.reduce_join",
         "tf.load_file_system_library":
             "tf.load_library",
         "tf.pywrap_tensorflow":
@@ -480,7 +485,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "input", "filter", "strides", "padding", "rate", "name",
             "data_format"
         ],
-        "tf.manip.batch_to_space_nd": ["input", "crops", "block_size", "name"],
         "tf.multinomial": [
             "logits", "num_samples", "seed", "name", "output_dtype"
         ],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index dbddf170d6..6cf870595a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_export
+from tensorflow.python.util import tf_inspect
 from tensorflow.tools.common import public_api
 from tensorflow.tools.common import traverse
 from tensorflow.tools.compatibility import ast_edits
@@ -62,6 +63,25 @@ def get_v2_names(symbol):
   return list(names_v2)
 
 
+def get_func_and_args_from_str(call_str):
+  """Parse call string to get function and argument names.
+
+  Args:
+    call_str: Call string must be in the form:
+              `tf.foo(arg1=val1, arg2=val2, ...)`.
+
+  Returns:
+    (function_name, list of arg names) tuple.
+  """
+  open_paren_index = call_str.find("(")
+  close_paren_index = call_str.rfind(")")
+
+  function_name = call_str[:call_str.find("(")]
+  args = call_str[open_paren_index+1:close_paren_index].split(",")
+  args = [arg.split("=")[0].strip() for arg in args]
+  return function_name, args
+
+
 class TestUpgrade(test_util.TensorFlowTestCase):
   """Test various APIs that have been changed in 2.0.
 
@@ -70,6 +90,22 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   work when run with current TensorFlow.
   """
 
+  @classmethod
+  def setUpClass(cls):
+    cls.v2_symbols = {}
+    if not hasattr(tf.compat, "v2"):
+      return
+
+    def symbol_collector(unused_path, unused_parent, children):
+      for child in children:
+        _, attr = tf_decorator.unwrap(child[1])
+        api_names_v2 = get_v2_names(attr)
+        for name in api_names_v2:
+          cls.v2_symbols["tf." + name] = attr
+
+    visitor = public_api.PublicAPIVisitor(symbol_collector)
+    traverse.traverse(tf.compat.v2, visitor)
+
   def _upgrade(self, old_file_text):
     in_file = six.StringIO(old_file_text)
     out_file = six.StringIO()
@@ -104,18 +140,6 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     if not hasattr(tf.compat, "v2"):
       return
 
-    v2_symbols = set([])
-
-    def symbol_collector(unused_path, unused_parent, children):
-      for child in children:
-        _, attr = tf_decorator.unwrap(child[1])
-        api_names_v2 = get_v2_names(attr)
-        for name in api_names_v2:
-          v2_symbols.add("tf." + name)
-
-    visitor = public_api.PublicAPIVisitor(symbol_collector)
-    traverse.traverse(tf.compat.v2, visitor)
-
     # Converts all symbols in the v1 namespace to the v2 namespace, raising
     # an error if the target of the conversion is not in the v2 namespace.
     def conversion_visitor(unused_path, unused_parent, children):
@@ -126,7 +150,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           _, _, _, text = self._upgrade("tf." + name)
           if (text and
               not text.startswith("tf.compat.v1") and
-              text not in v2_symbols):
+              text not in self.v2_symbols):
             self.assertFalse(
                 True, "Symbol %s generated from %s not in v2 API" % (
                     text, name))
@@ -136,6 +160,61 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     visitor.private_map["tf.compat"] = ["v1", "v2"]
     traverse.traverse(tf.compat.v1, visitor)
 
+  def testKeywordArgNames(self):
+    if not hasattr(tf.compat, "v2"):
+      return
+
+    all_keyword_renames = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_keyword_renames)
+    v2_name_exceptions = {"verify_shape_is_now_always_true"}
+
+    # Visitor that verifies V1 argument names, converts to V2 and checks
+    # V2 argument names.
+    def conversion_visitor(unused_path, unused_parent, children):
+      for child in children:
+        _, attr = tf_decorator.unwrap(child[1])
+        names_v1 = get_v1_names(attr)
+
+        for name in names_v1:
+          name = "tf.%s" % name
+          if name not in all_keyword_renames:
+            continue
+          arg_names_v1 = tf_inspect.getargspec(attr)[0]
+          keyword_renames = all_keyword_renames[name]
+          self.assertEqual(type(keyword_renames), dict)
+
+          # Assert that v1 function has valid v1 argument names.
+          for from_name, _ in keyword_renames.items():
+            self.assertIn(
+                from_name, arg_names_v1,
+                "%s not found in %s arguments: %s" %
+                (from_name, name, str(arg_names_v1)))
+
+          # Assert that arg names after converting to v2 are present in
+          # v2 function.
+          # 1. First, create an input of the form:
+          #    tf.foo(arg1=val1, arg2=val2, ...)
+          args = ",".join(
+              ["%s=%d" % (from_name, from_index)
+               for from_index, from_name in enumerate(keyword_renames.keys())])
+          text_input = "%s(%s)" % (name, args)
+          # 2. Convert the input to V2.
+          _, _, _, text = self._upgrade(text_input)
+          new_function_name, new_args = get_func_and_args_from_str(text)
+          # 3. Verify V2 function and arguments.
+          # Note: If we rename arguments, new function must be available in 2.0.
+          # We should not be using compat.v1 in this case.
+          self.assertIn(new_function_name, self.v2_symbols)
+          args_v2 = tf_inspect.getargspec(self.v2_symbols[new_function_name])[0]
+          args_v2.extend(v2_name_exceptions)
+          for new_arg in new_args:
+            self.assertIn(new_arg, args_v2)
+
+    visitor = public_api.PublicAPIVisitor(conversion_visitor)
+    visitor.do_not_descend_map["tf"].append("contrib")
+    visitor.private_map["tf.compat"] = ["v1", "v2"]
+    traverse.traverse(tf.compat.v1, visitor)
+
   def testRenameConstant(self):
     text = "tf.MONOLITHIC_BUILD\n"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
@@ -320,6 +399,44 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testBatchToSpace(self):
+    text = "tf.batch_to_space_nd(input, block_shape, crops, name)"
+    expected_text = "tf.batch_to_space(input, block_shape, crops, name)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = "tf.batch_to_space(input, crops, block_size, name)"
+    expected_text = (
+        "tf.batch_to_space(input=input, crops=crops, block_shape=block_size, "
+        "name=name)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = "tf.manip.batch_to_space_nd(input, block_shape, crops, name)"
+    expected_text = "tf.batch_to_space(input, block_shape, crops, name)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testExtractImagePatches(self):
+    text = (
+        "tf.extract_image_patches(images, ksizes=ksizes, strides=strides,"
+        "rates=rates, padding=padding, name=name)")
+    expected_text = (
+        "tf.image.extract_image_patches(images, sizes=ksizes, strides=strides,"
+        "rates=rates, padding=padding, name=name)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testStatelessMultinomial(self):
+    text = (
+        "tf.random.stateless_multinomial(logits, num_samples, seed, "
+        "output_dtype=dtype, name=name)")
+    expected_text = (
+        "tf.random.stateless_categorical(logits, num_samples, seed, "
+        "dtype=dtype, name=name)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 767631fbcb3890d4d42b84bcac12682fb2e8a79a Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 30 Nov 2018 14:39:16 -0800
Subject: [PATCH 1122/1554] Fix OSS TF gpu build by correcting the TRT header
 file path.

PiperOrigin-RevId: 223577606
---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc            | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc            | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes.h             | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc       | 2 +-
 .../contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h     | 2 +-
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.h             | 2 +-
 tensorflow/contrib/tensorrt/log/trt_logger.h                    | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin.h                 | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h         | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc   | 2 +-
 tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h           | 2 +-
 tensorflow/contrib/tensorrt/resources/trt_allocator.h           | 2 +-
 tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h     | 2 +-
 tensorflow/contrib/tensorrt/resources/trt_resources.h           | 2 +-
 tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc                | 2 +-
 tensorflow/contrib/tensorrt/tensorrt_test.cc                    | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index ecbd4ea802..3b32f72bc1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -59,7 +59,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 938cadc7c4..fee095668e 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -48,7 +48,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 // Check if the types are equal. Cast to int first so that failure log message
 // would work!
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index daa3111192..54e19b7395 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -34,7 +34,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 4790622e83..443033379f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -44,7 +44,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda.h"
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
index f658e45569..189e9c939b 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index b801480a30..b545f497f3 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -31,7 +31,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/contrib/tensorrt/log/trt_logger.h
index 58d9d05d01..96ccacb791 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.h
@@ -20,7 +20,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h b/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
index 167e8197a7..754920b60c 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
index 51393d2092..bbae9fb65c 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
@@ -27,7 +27,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
index 2346cb9ba0..129bdcdbc2 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h b/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
index 5ded702c41..274ce42fec 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index b03fe7b8b5..f857a9de05 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -22,7 +22,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
index e8f08ad9f4..65466c9741 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
@@ -26,7 +26,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 0be5d44f7a..aac9e5c7bd 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -31,7 +31,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
index ad9703325f..f30dba59ad 100644
--- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
+++ b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
@@ -23,7 +23,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace shape_inference {
diff --git a/tensorflow/contrib/tensorrt/tensorrt_test.cc b/tensorflow/contrib/tensorrt/tensorrt_test.cc
index 102a1d3891..769982c645 100644
--- a/tensorflow/contrib/tensorrt/tensorrt_test.cc
+++ b/tensorflow/contrib/tensorrt/tensorrt_test.cc
@@ -22,7 +22,7 @@ limitations under the License.
 #if GOOGLE_TENSORRT
 #include "cuda/include/cuda.h"
 #include "cuda/include/cuda_runtime_api.h"
-#include "tensorrt/NvInfer.h"
+#include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace {
-- 
GitLab


From 9152bfceb4f60fefb4979a6c03afd91ca47f3fcd Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Fri, 30 Nov 2018 14:58:30 -0800
Subject: [PATCH 1123/1554] Add Apache license notification

---
 tensorflow/tools/dockerfiles/tests/build-cpu.sh  | 15 +++++++++++++++
 tensorflow/tools/dockerfiles/tests/build-gpu.sh  | 16 ++++++++++++++++
 tensorflow/tools/dockerfiles/tests/import-gpu.sh | 16 ++++++++++++++++
 tensorflow/tools/dockerfiles/tests/import.sh     | 16 ++++++++++++++++
 4 files changed, 63 insertions(+)

diff --git a/tensorflow/tools/dockerfiles/tests/build-cpu.sh b/tensorflow/tools/dockerfiles/tests/build-cpu.sh
index 337239dc38..bcdc4c2139 100755
--- a/tensorflow/tools/dockerfiles/tests/build-cpu.sh
+++ b/tensorflow/tools/dockerfiles/tests/build-cpu.sh
@@ -1,5 +1,20 @@
 #!/usr/bin/env bash
 
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
 # Download and build TensorFlow.
 set -euxo pipefail
 git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow
diff --git a/tensorflow/tools/dockerfiles/tests/build-gpu.sh b/tensorflow/tools/dockerfiles/tests/build-gpu.sh
index 7a1e7ad340..76b25d5a74 100755
--- a/tensorflow/tools/dockerfiles/tests/build-gpu.sh
+++ b/tensorflow/tools/dockerfiles/tests/build-gpu.sh
@@ -1,4 +1,20 @@
 #!/usr/bin/env bash
+
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
 # Download and build TensorFlow.
 set -euxo pipefail
 git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow
diff --git a/tensorflow/tools/dockerfiles/tests/import-gpu.sh b/tensorflow/tools/dockerfiles/tests/import-gpu.sh
index 214e5ccf2c..6559210dcb 100755
--- a/tensorflow/tools/dockerfiles/tests/import-gpu.sh
+++ b/tensorflow/tools/dockerfiles/tests/import-gpu.sh
@@ -1,2 +1,18 @@
 #!/usr/bin/env bash
+
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
 python -c 'import tensorflow as tf; tf.test.is_gpu_available() or exit(1)'
diff --git a/tensorflow/tools/dockerfiles/tests/import.sh b/tensorflow/tools/dockerfiles/tests/import.sh
index 79998aad77..b73bd86a85 100755
--- a/tensorflow/tools/dockerfiles/tests/import.sh
+++ b/tensorflow/tools/dockerfiles/tests/import.sh
@@ -1,3 +1,19 @@
 #!/usr/bin/env bash
+
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
 set -euxo pipefail
 python -c 'import tensorflow as tf'
-- 
GitLab


From b3dec2b7c8f23310d11c264474301191142d9364 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 30 Nov 2018 15:36:58 -0800
Subject: [PATCH 1124/1554] Adding cross entropy losses to the new V2 losses
 module.

PiperOrigin-RevId: 223586455
---
 tensorflow/python/keras/losses.py             | 139 ++++++++-
 tensorflow/python/keras/losses_test.py        | 272 ++++++++++++++++++
 .../golden/v1/tensorflow.keras.losses.pbtxt   |   6 +-
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   6 +-
 .../golden/v2/tensorflow.keras.losses.pbtxt   |   6 +-
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   6 +-
 6 files changed, 417 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index f6953f8499..9600ae9a69 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -28,6 +28,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.losses_utils import compute_weighted_loss
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops.losses import losses_impl
@@ -268,6 +269,130 @@ class MeanSquaredLogarithmicError(Loss):
     return mean_squared_logarithmic_error(y_true, y_pred)
 
 
+class BinaryCrossEntropy(Loss):
+  """Computes the binary cross entropy loss between the labels and predictions.
+
+  Usage:
+
+  ```python
+  bce = tf.keras.losses.BinaryCrossEntropy()
+  loss = bce([0., 0., 1., 1.], [1., 1., 1., 0.])
+  print('Loss: ', loss.numpy())  # Loss: 12.007
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.BinaryCrossEntropy())
+  ````
+
+  Args:
+    from_logits: Whether `output` is expected to be a logits tensor. By default,
+      we consider that `output` encodes a probability distribution.
+    label_smoothing: If greater than `0` then smooth the labels.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               from_logits=False,
+               label_smoothing=0,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(BinaryCrossEntropy, self).__init__(reduction=reduction, name=name)
+    self.from_logits = from_logits
+    self.label_smoothing = label_smoothing
+
+  def call(self, y_true, y_pred):
+    """Invokes the `BinaryCrossEntropy` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Binary cross entropy losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+
+    if self.label_smoothing > 0:
+      y_true = y_true * (1 - self.label_smoothing) + 0.5 * self.label_smoothing
+
+    return binary_crossentropy(y_true, y_pred, from_logits=self.from_logits)
+
+
+class CategoricalCrossEntropy(Loss):
+  """Computes categorical cross entropy loss between the `y_true` and `y_pred`.
+
+  Usage:
+
+  ```python
+  cce = tf.keras.losses.CategoricalCrossEntropy()
+  loss = cce(
+    [[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
+    [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]])
+  print('Loss: ', loss.numpy())  # Loss: 0.3239
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.keras.losses.CategoricalCrossEntropy())
+  ````
+
+  Args:
+    from_logits: Whether `output` is expected to be a logits tensor. By default,
+      we consider that `output` encodes a probability distribution.
+    label_smoothing: If greater than `0` then smooth the labels. This option is
+      currently not supported when `y_pred` is a sparse input (not one-hot).
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               from_logits=False,
+               label_smoothing=0,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(CategoricalCrossEntropy, self).__init__(
+        reduction=reduction, name=name)
+    self.from_logits = from_logits
+    self.label_smoothing = label_smoothing
+
+  def call(self, y_true, y_pred):
+    """Invokes the `CategoricalCrossEntropy` instance.
+
+    Args:
+      y_true: Ground truth values.
+      y_pred: The predicted values.
+
+    Returns:
+      Categorical cross entropy losses.
+    """
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = ops.convert_to_tensor(y_true)
+    is_sparse = y_pred.shape != y_true.shape
+
+    if is_sparse:
+      return sparse_categorical_crossentropy(
+          y_true, y_pred, from_logits=self.from_logits)
+    else:
+      y_true = math_ops.cast(y_true, y_pred.dtype)
+      if self.label_smoothing > 0:
+        num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype)
+        smooth_positives = 1.0 - self.label_smoothing
+        smooth_negatives = self.label_smoothing / num_classes
+        y_true = y_true * smooth_positives + smooth_negatives
+
+      return categorical_crossentropy(
+          y_true, y_pred, from_logits=self.from_logits)
+
+
 @tf_export('keras.metrics.mean_squared_error',
            'keras.metrics.mse',
            'keras.metrics.MSE',
@@ -355,20 +480,22 @@ def logcosh(y_true, y_pred):
 
 @tf_export('keras.metrics.categorical_crossentropy',
            'keras.losses.categorical_crossentropy')
-def categorical_crossentropy(y_true, y_pred):
-  return K.categorical_crossentropy(y_true, y_pred)
+def categorical_crossentropy(y_true, y_pred, from_logits=False):
+  return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
 
 
 @tf_export('keras.metrics.sparse_categorical_crossentropy',
            'keras.losses.sparse_categorical_crossentropy')
-def sparse_categorical_crossentropy(y_true, y_pred):
-  return K.sparse_categorical_crossentropy(y_true, y_pred)
+def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False):
+  return K.sparse_categorical_crossentropy(
+      y_true, y_pred, from_logits=from_logits)
 
 
 @tf_export('keras.metrics.binary_crossentropy',
            'keras.losses.binary_crossentropy')
-def binary_crossentropy(y_true, y_pred):
-  return K.mean(K.binary_crossentropy(y_true, y_pred), axis=-1)
+def binary_crossentropy(y_true, y_pred, from_logits=False):
+  return K.mean(
+      K.binary_crossentropy(y_true, y_pred, from_logits=from_logits), axis=-1)
 
 
 @tf_export('keras.metrics.kullback_leibler_divergence',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index cbf3c3524c..a1349b4af4 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -27,6 +27,7 @@ from tensorflow.python import keras
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 
@@ -499,5 +500,276 @@ class CosineProximityTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class BinaryCrossEntropyTest(test.TestCase):
+
+  def test_config(self):
+    bce_obj = keras.losses.BinaryCrossEntropy(
+        reduction=losses_impl.ReductionV2.SUM, name='bce_1')
+    self.assertEqual(bce_obj.name, 'bce_1')
+    self.assertEqual(bce_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_all_correct_unweighted(self):
+    y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]],
+                                  dtype=dtypes.float32)
+    bce_obj = keras.losses.BinaryCrossEntropy()
+    loss = bce_obj(y_true, y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[100.0, -100.0, -100.0],
+                                   [-100.0, 100.0, -100.0],
+                                   [-100.0, -100.0, 100.0]])
+    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    loss = bce_obj(y_true, logits)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    bce_obj = keras.losses.BinaryCrossEntropy()
+    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
+    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = bce_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 8.0004, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([10., 10., 10., -10., 10, -10],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    loss = bce_obj(y_true, logits)
+    self.assertAlmostEqual(self.evaluate(loss), 5., 3)
+
+  def test_scalar_weighted(self):
+    bce_obj = keras.losses.BinaryCrossEntropy()
+    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
+    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float32)
+    loss = bce_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 18.4010, 3)
+
+    # Test with logits.
+    y_true = array_ops.ones((32, 1))
+    logits = array_ops.ones((32, 1), dtype=dtypes.float32)
+    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    loss = bce_obj(y_true, logits, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), 0.7205, 3)
+
+  def test_sample_weighted(self):
+    bce_obj = keras.losses.BinaryCrossEntropy()
+    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
+    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
+                                  shape=(2, 3),
+                                  dtype=dtypes.float64)
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = bce_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 21.4907, 3)
+
+    # Test with logits.
+    y_true = constant_op.constant([[0, 0, 1], [1, 0, 0], [0, 1, 0]])
+    logits = constant_op.constant(
+        [[100.0, -100.0, -100.0], [-100.0, 100.0, -100.0],
+         [-100.0, -100.0, 100.0]],
+        dtype=dtypes.float64)
+    weights = constant_op.constant([3, 2, 8])
+    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    loss = bce_obj(y_true, logits, sample_weight=weights)
+    self.assertAlmostEqual(self.evaluate(loss), 288.8888, 3)
+
+  def test_no_reduction(self):
+    y_true = constant_op.constant(((1, 0, 1), (1, 1, 0), (0, 1, 1)))
+    logits = constant_op.constant(((100.0, -100.0, 100.0),
+                                   (100.0, -100.0, 100.0),
+                                   (100.0, 100.0, -100.0)))
+    bce_obj = keras.losses.BinaryCrossEntropy(
+        from_logits=True, reduction=losses_impl.ReductionV2.NONE)
+    loss = bce_obj(y_true, logits)
+    self.assertAllClose((0., 66.6666, 66.6666), self.evaluate(loss), 3)
+
+  def test_label_smoothing(self):
+    logits = constant_op.constant([[100.0, -100.0, -100.0]])
+    y_true = constant_op.constant([[1, 0, 1]])
+    label_smoothing = 0.1
+    # Loss: max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    # Label smoothing: z' = z * (1 - L) + 0.5L
+    #                  1  = 1 - 0.5L
+    #                  0  = 0.5L
+    # Applying the above two fns to the given input:
+    # (100 - 100 * (1 - 0.5 L)  + 0 +
+    #  0   + 100 * (0.5 L)      + 0 +
+    #  0   + 100 * (1 - 0.5 L)  + 0) * (1/3)
+    #  = (100 + 50L) * 1/3
+    bce_obj = keras.losses.BinaryCrossEntropy(
+        from_logits=True, label_smoothing=label_smoothing)
+    loss = bce_obj(y_true, logits)
+    expected_value = (100.0 + 50.0 * label_smoothing) / 3.0
+    self.assertAlmostEqual(self.evaluate(loss), expected_value, 3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class CategoricalCrossEntropyTest(test.TestCase):
+
+  def test_config(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy(
+        reduction=losses_impl.ReductionV2.SUM, name='bce_1')
+    self.assertEqual(cce_obj.name, 'bce_1')
+    self.assertEqual(cce_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_all_correct_unweighted(self):
+    y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]],
+                                  dtype=dtypes.int64)
+    y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
+                                  dtype=dtypes.float32)
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    loss = cce_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
+    y_pred = constant_op.constant(
+        [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
+    loss = cce_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), .3239, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits)
+    self.assertAlmostEqual(self.evaluate(loss), .0573, 3)
+
+  def test_scalar_weighted(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
+    y_pred = constant_op.constant(
+        [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
+    loss = cce_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), .7449, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), .1317, 3)
+
+  def test_sample_weighted(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
+    y_pred = constant_op.constant(
+        [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
+    sample_weight = constant_op.constant([[1.2], [3.4], [5.6]], shape=(3, 1))
+    loss = cce_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 1.0696, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3)
+
+  def test_no_reduction(self):
+    y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(
+        from_logits=True, reduction=losses_impl.ReductionV2.NONE)
+    loss = cce_obj(y_true, logits)
+    self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
+
+  def test_label_smoothing(self):
+    logits = constant_op.constant([[100.0, -100.0, -100.0]])
+    y_true = constant_op.constant([[1, 0, 0]])
+    label_smoothing = 0.1
+    # Softmax Cross Entropy Loss: -\sum_i p_i \log q_i
+    # where for a softmax activation
+    # \log q_i = x_i - \log \sum_j \exp x_j
+    #          = x_i - x_max - \log \sum_j \exp (x_j - x_max)
+    # For our activations, [100, -100, -100]
+    # \log ( exp(0) + exp(-200) + exp(-200) ) = 0
+    # so our log softmaxes become: [0, -200, -200]
+    # Label smoothing: z' = z * (1 - L) + L/n
+    #                  1  = 1 - L + L/n
+    #                  0  = L/n
+    # Applying the above two fns to the given input:
+    # -0 * (1 - L + L/n) + 200 * L/n + 200 * L/n = 400 L/n
+    cce_obj = keras.losses.CategoricalCrossEntropy(
+        from_logits=True, label_smoothing=label_smoothing)
+    loss = cce_obj(y_true, logits)
+    expected_value = 400.0 * label_smoothing / 3.0
+    self.assertAlmostEqual(self.evaluate(loss), expected_value, 3)
+
+  def test_all_correct_unweighted_sparse(self):
+    y_true = constant_op.constant([[0], [1], [2]], dtype=dtypes.int64)
+    y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
+                                  dtype=dtypes.float32)
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    loss = cce_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted_sparse(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    y_true = constant_op.constant([0, 1, 2])
+    y_pred = constant_op.constant(
+        [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
+    loss = cce_obj(y_true, y_pred)
+    self.assertAlmostEqual(self.evaluate(loss), .3239, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits)
+    self.assertAlmostEqual(self.evaluate(loss), .0573, 3)
+
+  def test_scalar_weighted_sparse(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    y_true = constant_op.constant([[0], [1], [2]])
+    y_pred = constant_op.constant(
+        [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
+    loss = cce_obj(y_true, y_pred, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), .7449, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits, sample_weight=2.3)
+    self.assertAlmostEqual(self.evaluate(loss), .1317, 3)
+
+  def test_sample_weighted_sparse(self):
+    cce_obj = keras.losses.CategoricalCrossEntropy()
+    y_true = constant_op.constant([[0], [1], [2]])
+    y_pred = constant_op.constant(
+        [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
+    sample_weight = constant_op.constant([[1.2], [3.4], [5.6]], shape=(3, 1))
+    loss = cce_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 1.0696, 3)
+
+    # Test with logits.
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    loss = cce_obj(y_true, logits, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3)
+
+  def test_no_reduction_sparse(self):
+    y_true = constant_op.constant([[0], [1], [2]])
+    logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
+    cce_obj = keras.losses.CategoricalCrossEntropy(
+        from_logits=True, reduction=losses_impl.ReductionV2.NONE)
+    loss = cce_obj(y_true, logits)
+    self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index c6638bd5e0..1cc472b436 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -38,11 +38,11 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "categorical_hinge"
@@ -122,7 +122,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "squared_hinge"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index cc90d0e709..8cab17edc5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -70,7 +70,7 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "categorical_accuracy"
@@ -78,7 +78,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "cosine"
@@ -154,7 +154,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "sparse_top_k_categorical_accuracy"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index 5204a29a82..6b330c2c1a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -42,11 +42,11 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "categorical_hinge"
@@ -126,7 +126,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "squared_hinge"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index cc90d0e709..8cab17edc5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -70,7 +70,7 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "categorical_accuracy"
@@ -78,7 +78,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "cosine"
@@ -154,7 +154,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
   }
   member_method {
     name: "sparse_top_k_categorical_accuracy"
-- 
GitLab


From eb376a3dc2841f08ad2b54c08864e4c2653dbb9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 15:36:59 -0800
Subject: [PATCH 1125/1554] Don't do ReadVariableOp in embedding ops.

PiperOrigin-RevId: 223586459
---
 tensorflow/python/ops/embedding_ops.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py
index 9ce024ad96..b398601e6f 100644
--- a/tensorflow/python/ops/embedding_ops.py
+++ b/tensorflow/python/ops/embedding_ops.py
@@ -554,7 +554,10 @@ def safe_embedding_lookup_sparse(embedding_weights,
 
   dtype = sparse_weights.dtype if sparse_weights is not None else None
   embedding_weights = [
-      ops.convert_to_tensor(w, dtype=dtype) for w in embedding_weights
+      w if (isinstance(w, resource_variable_ops.ResourceVariable)
+            and dtype in (None, w.dtype))
+      else ops.convert_to_tensor(w, dtype=dtype)
+      for w in embedding_weights
   ]
 
   with ops.name_scope(name, 'embedding_lookup',
-- 
GitLab


From 6d389946f1f3b7b521e1555071ab9e60d2fa207c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 15:42:41 -0800
Subject: [PATCH 1126/1554] Change CopyToBufferHandle to match the signature of
 CopyFromBufferHandle.

PiperOrigin-RevId: 223587373
---
 tensorflow/lite/c/c_api_internal.h  | 6 +++---
 tensorflow/lite/interpreter_test.cc | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index 6280bf825d..5cfa64cc7b 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -496,12 +496,12 @@ typedef struct _TfLiteDelegate {
                                        TfLiteBufferHandle buffer_handle,
                                        TfLiteTensor* tensor);
 
-  // Copy the data from raw memory to delegate buffer handle.
-  // This can be null if the delegate doesn't use its own buffer.
+  // Copy the data from raw memory of the given 'tensor' to delegate buffer
+  // handle. This can be null if the delegate doesn't use its own buffer.
   TfLiteStatus (*CopyToBufferHandle)(TfLiteContext* context,
                                      TfLiteDelegate* delegate,
                                      TfLiteBufferHandle buffer_handle,
-                                     void* data, size_t size);
+                                     TfLiteTensor* tensor);
 
   // Free the Delegate Buffer Handle. Note: This only frees the handle, but
   // this doesn't release the underlying resource (e.g. textures). The
diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc
index dd3d85e7cd..78b5d1b887 100644
--- a/tensorflow/lite/interpreter_test.cc
+++ b/tensorflow/lite/interpreter_test.cc
@@ -1090,10 +1090,10 @@ class TestDelegate : public ::testing::Test {
         TfLiteIntArrayFree(nodes_to_separate);
         return kTfLiteOk;
       };
-      delegate_.CopyToBufferHandle =
-          [](TfLiteContext* context, TfLiteDelegate* delegate,
-             TfLiteBufferHandle buffer_handle, void* data,
-             size_t size) -> TfLiteStatus {
+      delegate_.CopyToBufferHandle = [](TfLiteContext* context,
+                                        TfLiteDelegate* delegate,
+                                        TfLiteBufferHandle buffer_handle,
+                                        TfLiteTensor* tensor) -> TfLiteStatus {
         // TODO(ycling): Implement tests to test buffer copying logic.
         return kTfLiteOk;
       };
-- 
GitLab


From 6b5288391847ba0ebe6067674eddaf3c8f416f05 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 30 Nov 2018 15:43:18 -0800
Subject: [PATCH 1127/1554] A more useful error message for incompatible
 object-based checkpoints loaded using variable names

PiperOrigin-RevId: 223587442
---
 tensorflow/python/training/saver.py      | 38 +++++++++++++++++++-----
 tensorflow/python/training/saver_test.py |  2 +-
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index a29926a57d..e76500e776 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1899,16 +1899,40 @@ def saver_from_object_based_checkpoint(
     builder = BulkSaverBuilder()
 
   saveables = builder._ValidateAndSliceInputs(var_list)  # pylint: disable=protected-access
+  current_names = set()
+  for saveable in saveables:
+    for spec in saveable.specs:
+      current_names.add(spec.name)
+  previous_names = set(names_to_keys.keys())
+  missing_names = current_names - previous_names
+  if missing_names:
+    extra_names = previous_names - current_names
+    intersecting_names = previous_names.intersection(current_names)
+    raise errors.NotFoundError(
+        None, None,
+        message=(
+            "\n\nExisting variables not in the checkpoint: %s\n\n"
+            "Variables names when this checkpoint was written which don't "
+            "exist now: %s\n\n"
+            "(%d variable name(s) did match)\n\n"
+            "Could not find some variables in the checkpoint (see names "
+            "above). Saver was attempting to load an object-based checkpoint "
+            "(saved using tf.train.Checkpoint or tf.keras.Model.save_weights) "
+            "using variable names. If the checkpoint was written with eager "
+            "execution enabled, it's possible that variable names have "
+            "changed (for example missing a '_1' suffix). It's also "
+            "possible that there are new variables which did not exist "
+            "when the checkpoint was written. You can construct a "
+            "Saver(var_list=...) with only the variables which previously "
+            "existed, and if variable names have changed you may need to "
+            "make this a dictionary with the old names as keys. If you're "
+            "using an Estimator, you'll need to return a tf.train.Saver "
+            "inside a tf.train.Scaffold from your model_fn.")
+        % (", ".join(sorted(missing_names)), ", ".join(sorted(extra_names)),
+           len(intersecting_names)))
   for saveable in saveables:
     for spec in saveable.specs:
-      if spec.name not in names_to_keys:
-        raise errors.NotFoundError(
-            None, None,
-            message=("Attempting to load an object-based checkpoint using "
-                     "variable names, but could not find %s in the "
-                     "checkpoint.") % spec.name)
       spec.name = names_to_keys[spec.name]
-
   if cached_saver is None:
     return Saver(saveables)
   return cached_saver
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 40b7c0d5a7..9b822860ea 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -3042,7 +3042,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
       self.assertEqual(before_second_restore_ops,
                        restore_graph.get_operations())
       with self.assertRaisesRegexp(errors.NotFoundError,
-                                   "could not find a_variable"):
+                                   "Could not find some variables"):
         saver.restore(sess=sess, save_path=second_path)
 
   def testLoadFromObjectBasedEager(self):
-- 
GitLab


From 5b8c65d9c6d917b3ab99fbdc577a4a499ab13808 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 30 Nov 2018 15:45:52 -0800
Subject: [PATCH 1128/1554] Add unified LSTM to keras as the alternative
 implementation of LSTM.

The UnifiedLSTM will probably be exposed as tf.keras.LSTM in 2.0. The whole test case of existing LSTM layer has been added to ensure the correctness of the implementation.

PiperOrigin-RevId: 223587809
---
 tensorflow/python/keras/BUILD                 |   1 +
 .../python/keras/layers/cudnn_recurrent.py    |  14 +-
 tensorflow/python/keras/layers/recurrent.py   | 472 ++++++++++-
 .../python/keras/layers/unified_rnn_test.py   | 779 ++++++++++--------
 tensorflow/python/keras/testing_utils.py      |   4 +
 5 files changed, 887 insertions(+), 383 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index a573fd5cfb..7eee8a8bc8 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -523,6 +523,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
 )
 
 py_test(
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent.py b/tensorflow/python/keras/layers/cudnn_recurrent.py
index 81f292817f..16692753af 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent.py
@@ -26,6 +26,7 @@ from tensorflow.python.keras import constraints
 from tensorflow.python.keras import initializers
 from tensorflow.python.keras import regularizers
 from tensorflow.python.keras.engine.input_spec import InputSpec
+from tensorflow.python.keras.layers import recurrent
 from tensorflow.python.keras.layers.recurrent import RNN
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_cudnn_rnn_ops
@@ -80,11 +81,6 @@ class _CuDNNRNN(RNN):
     self._num_inputs = None
     self._vector_shape = constant_op.constant([-1])
 
-  def _canonical_to_params(self, weights, biases):
-    weights = [array_ops.reshape(x, self._vector_shape) for x in weights]
-    biases = [array_ops.reshape(x, self._vector_shape) for x in biases]
-    return array_ops.concat(weights + biases, axis=0)
-
   def call(self, inputs, mask=None, training=None, initial_state=None):
     if isinstance(mask, list):
       mask = mask[0]
@@ -279,7 +275,7 @@ class CuDNNGRU(_CuDNNRNN):
     input_h = initial_state[0]
     input_h = array_ops.expand_dims(input_h, axis=0)
 
-    params = self._canonical_to_params(
+    params = recurrent._canonical_to_params(    # pylint: disable=protected-access
         weights=[
             self.kernel[:, self.units:self.units * 2],
             self.kernel[:, :self.units],
@@ -296,7 +292,7 @@ class CuDNNGRU(_CuDNNRNN):
             self.bias[self.units * 3:self.units * 4],
             self.bias[self.units * 5:],
         ],
-    )
+        shape=self._vector_shape)
 
     outputs, h, _, _ = gen_cudnn_rnn_ops.cudnn_rnn(
         inputs,
@@ -474,7 +470,7 @@ class CuDNNLSTM(_CuDNNRNN):
     input_h = array_ops.expand_dims(input_h, axis=0)
     input_c = array_ops.expand_dims(input_c, axis=0)
 
-    params = self._canonical_to_params(
+    params = recurrent._canonical_to_params(    # pylint: disable=protected-access
         weights=[
             self.kernel[:, :self.units],
             self.kernel[:, self.units:self.units * 2],
@@ -495,7 +491,7 @@ class CuDNNLSTM(_CuDNNRNN):
             self.bias[self.units * 6:self.units * 7],
             self.bias[self.units * 7:],
         ],
-    )
+        shape=self._vector_shape)
 
     outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn(
         inputs,
diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 5d0efc2f16..d9502dfc5b 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -19,9 +19,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import uuid
+
 import numpy as np
 
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import activations
 from tensorflow.python.keras import backend as K
@@ -33,6 +39,7 @@ from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_cudnn_rnn_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.checkpointable import base as checkpointable
@@ -746,35 +753,12 @@ class RNN(Layer):
            training=None,
            initial_state=None,
            constants=None):
-    # input shape: `(samples, time (padded with zeros), input_dim)`
-    # note that the .build() method of subclasses MUST define
-    # self.input_spec and self.state_spec with complete input shapes.
-    if isinstance(inputs, list):
-      # get initial_state from full input spec
-      # as they could be copied to multiple GPU.
-      if self._num_constants is None:
-        initial_state = inputs[1:]
-      else:
-        initial_state = inputs[1:-self._num_constants]
-        constants = inputs[-self._num_constants:]
-      if len(initial_state) == 0:
-        initial_state = None
-      inputs = inputs[0]
-    if initial_state is not None:
-      pass
-    elif self.stateful:
-      initial_state = self.states
-    else:
-      initial_state = self.get_initial_state(inputs)
+    inputs, initial_state, constants = self._process_inputs(
+        inputs, initial_state, constants)
 
     if isinstance(mask, list):
       mask = mask[0]
 
-    if len(initial_state) != len(self.states):
-      raise ValueError(
-          'Layer has ' + str(len(self.states)) + ' states but was passed ' +
-          str(len(initial_state)) + ' initial states.')
-
     if nest.is_sequence(inputs):
       # In the case of nested input, use the first element for shape check.
       input_shape = K.int_shape(nest.flatten(inputs)[0])
@@ -854,6 +838,34 @@ class RNN(Layer):
     else:
       return output
 
+  def _process_inputs(self, inputs, initial_state, constants):
+    # input shape: `(samples, time (padded with zeros), input_dim)`
+    # note that the .build() method of subclasses MUST define
+    # self.input_spec and self.state_spec with complete input shapes.
+    if isinstance(inputs, list):
+      # get initial_state from full input spec
+      # as they could be copied to multiple GPU.
+      if self._num_constants is None:
+        initial_state = inputs[1:]
+      else:
+        initial_state = inputs[1:-self._num_constants]
+        constants = inputs[-self._num_constants:]
+      if len(initial_state) == 0:
+        initial_state = None
+      inputs = inputs[0]
+    if initial_state is not None:
+      pass
+    elif self.stateful:
+      initial_state = self.states
+    else:
+      initial_state = self.get_initial_state(inputs)
+
+    if len(initial_state) != len(self.states):
+      raise ValueError('Layer has ' + str(len(self.states)) +
+                       ' states but was passed ' + str(len(initial_state)) +
+                       ' initial states.')
+    return inputs, initial_state, constants
+
   def reset_states(self, states=None):
     if not self.stateful:
       raise AttributeError('Layer must be stateful.')
@@ -2521,6 +2533,416 @@ class LSTM(RNN):
     return cls(**config)
 
 
+class UnifiedLSTM(LSTM):
+  """Long Short-Term Memory layer - Hochreiter 1997.
+
+  `UnifiedLSTM` unifies the implementations between standard `LSTM` layer and
+  `CuDNNLSTM` layer. Based on available runtime hardware and constrains,
+  `UnifiedLSTM` will choose different implementations to maximize the
+  performance. For instance, if GPU is available and all the parameters meet the
+  requirement of CuDNN kernel, `UnifiedLSTM` will use CuDNN kernel for the
+  calculation.
+
+  Arguments:
+    units: Positive integer, dimensionality of the output space.
+    activation: Activation function to use.
+        Default: hyperbolic tangent (`tanh`). If you pass `None`, no activation
+          is applied
+        (ie. "linear" activation: `a(x) = x`).
+    recurrent_activation: Activation function to use for the recurrent step.
+        Default: hard sigmoid (`hard_sigmoid`). If you pass `None`, no
+          activation is applied
+        (ie. "linear" activation: `a(x) = x`).
+    use_bias: Boolean, whether the layer uses a bias vector.
+    kernel_initializer: Initializer for the `kernel` weights matrix, used for
+      the linear transformation of the inputs..
+    recurrent_initializer: Initializer for the `recurrent_kernel` weights
+      matrix, used for the linear transformation of the recurrent state..
+    bias_initializer: Initializer for the bias vector.
+    unit_forget_bias: Boolean. If True, add 1 to the bias of the forget gate at
+      initialization. Setting it to true will also force
+      `bias_initializer="zeros"`. This is recommended in [Jozefowicz et
+          al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
+    kernel_regularizer: Regularizer function applied to the `kernel` weights
+      matrix.
+    recurrent_regularizer: Regularizer function applied to the
+      `recurrent_kernel` weights matrix.
+    bias_regularizer: Regularizer function applied to the bias vector.
+    activity_regularizer: Regularizer function applied to the output of the
+      layer (its "activation")..
+    kernel_constraint: Constraint function applied to the `kernel` weights
+      matrix.
+    recurrent_constraint: Constraint function applied to the `recurrent_kernel`
+      weights matrix.
+    bias_constraint: Constraint function applied to the bias vector.
+    dropout: Float between 0 and 1. Fraction of the units to drop for the linear
+      transformation of the inputs.
+    recurrent_dropout: Float between 0 and 1. Fraction of the units to drop for
+      the linear transformation of the recurrent state.
+    implementation: Implementation mode, either 1 or 2. Mode 1 will structure
+      its operations as a larger number of smaller dot products and additions,
+      whereas mode 2 will batch them into fewer, larger operations. These modes
+      will have different performance profiles on different hardware and for
+      different applications.
+    return_sequences: Boolean. Whether to return the last output. in the output
+      sequence, or the full sequence.
+    return_state: Boolean. Whether to return the last state in addition to the
+      output.
+    go_backwards: Boolean (default False). If True, process the input sequence
+      backwards and return the reversed sequence.
+    stateful: Boolean (default False). If True, the last state for each sample
+      at index i in a batch will be used as initial state for the sample of
+      index i in the following batch.
+    unroll: Boolean (default False). If True, the network will be unrolled, else
+      a symbolic loop will be used. Unrolling can speed-up a RNN, although it
+      tends to be more memory-intensive. Unrolling is only suitable for short
+      sequences.
+  """
+
+  def __init__(self,
+               units,
+               activation='tanh',
+               recurrent_activation='hard_sigmoid',
+               use_bias=True,
+               kernel_initializer='glorot_uniform',
+               recurrent_initializer='orthogonal',
+               bias_initializer='zeros',
+               unit_forget_bias=True,
+               kernel_regularizer=None,
+               recurrent_regularizer=None,
+               bias_regularizer=None,
+               activity_regularizer=None,
+               kernel_constraint=None,
+               recurrent_constraint=None,
+               bias_constraint=None,
+               dropout=0.,
+               recurrent_dropout=0.,
+               implementation=1,
+               return_sequences=False,
+               return_state=False,
+               go_backwards=False,
+               stateful=False,
+               time_major=False,
+               unroll=False,
+               **kwargs):
+    # return_runtime is a flag for testing, which shows the real backend
+    # implementation chosen by grappler in graph mode.
+    self.return_runtime = kwargs.pop('return_runtime', False)
+
+    super(UnifiedLSTM, self).__init__(
+        units,
+        activation=activation,
+        recurrent_activation=recurrent_activation,
+        use_bias=use_bias,
+        kernel_initializer=kernel_initializer,
+        recurrent_initializer=recurrent_initializer,
+        bias_initializer=bias_initializer,
+        unit_forget_bias=unit_forget_bias,
+        kernel_regularizer=kernel_regularizer,
+        recurrent_regularizer=recurrent_regularizer,
+        bias_regularizer=bias_regularizer,
+        activity_regularizer=activity_regularizer,
+        kernel_constraint=kernel_constraint,
+        recurrent_constraint=recurrent_constraint,
+        bias_constraint=bias_constraint,
+        dropout=dropout,
+        recurrent_dropout=recurrent_dropout,
+        implementation=implementation,
+        return_sequences=return_sequences,
+        return_state=return_state,
+        go_backwards=go_backwards,
+        stateful=stateful,
+        time_major=time_major,
+        unroll=unroll,
+        **kwargs)
+
+    self.state_spec = [
+        InputSpec(shape=(None, dim)) for dim in (self.units, self.units)
+    ]
+    self._num_constants = None
+    self._num_inputs = None
+    self.could_use_cudnn = (
+        activation == 'tanh' and dropout == 0 and not unroll and use_bias and
+        unit_forget_bias)
+
+  def build(self, input_shape):
+    super(UnifiedLSTM, self).build(input_shape)
+    if self.could_use_cudnn:
+      # Add a new set of bias for CuDNN implementation only. Standard LSTM only
+      # has bias for recurrent kernel, while CuDNN LSTM has an extra set for
+      # input gate as well.
+      self.cudnn_bias = self.add_weight(
+          shape=(self.units * 4,),
+          name='cudnn_bias',
+          use_resource=True,
+          initializer=self.bias_initializer,
+          regularizer=self.bias_regularizer,
+          constraint=self.bias_constraint)
+    self.built = True
+
+  def call(self, inputs, mask=None, training=None, initial_state=None):
+    # LSTM does not support constants. Ignore it during process.
+    inputs, initial_state, _ = self._process_inputs(inputs, initial_state, None)
+
+    if isinstance(mask, list):
+      mask = mask[0]
+
+    input_shape = K.int_shape(inputs)
+    timesteps = input_shape[0] if self.time_major else input_shape[1]
+
+    if mask is not None or not self.could_use_cudnn:
+      # CuDNN does not support masking, fall back to use the normal LSTM.
+      kwargs = {'training': training}
+
+      def step(inputs, states):
+        return self.cell.call(inputs, states, **kwargs)
+
+      last_output, outputs, states = K.rnn(
+          step,
+          inputs,
+          initial_state,
+          constants=None,
+          go_backwards=self.go_backwards,
+          mask=mask,
+          unroll=self.unroll,
+          input_length=timesteps,
+          time_major=self.time_major,
+          zero_output_for_mask=self.zero_output_for_mask)
+      runtime = constant_op.constant(
+          'unknown', dtype=dtypes.string, name='runtime')
+    else:
+      # Use the new defun approach for backend implementation swap.
+      # Note that different implementations need to have same function
+      # signature, eg, the tensor parameters need to have same shape and dtypes.
+      # Since the CuDNN has an extra set of bias, those bias will be passed to
+      # both normal and CuDNN implementations.
+      if self.go_backwards:
+        # Reverse time axis.
+        inputs = K.reverse(inputs, 1)
+
+      combined_bias = array_ops.concat([self.cudnn_bias, self.cell.bias], 0)
+
+      # Each time a defun function is called, we will give a unique identifiable
+      # API name, so that the grappler won't get confused when it sees multiple
+      # LSTM layer added into same graph, and it will be able to pair up the
+      # different implementations across them.
+      experimental_api_name = 'lstm_' + str(uuid.uuid4())
+      standard_lstm_attributes = {
+          'experimental_api_implements': experimental_api_name,
+          'experimental_api_preferred_device': 'CPU',
+      }
+      cudnn_lstm_attributes = {
+          'experimental_api_implements': experimental_api_name,
+          'experimental_api_preferred_device': 'GPU',
+      }
+      defun_standard_lstm = function.defun_with_attributes(
+          standard_lstm, attributes=standard_lstm_attributes)
+      defun_cudnn_lstm = function.defun_with_attributes(
+          cudnn_lstm, attributes=cudnn_lstm_attributes)
+
+      if ops.executing_eagerly_outside_functions():
+        # Under eager context, the device placement is already known. Prefer the
+        # GPU implementation here.
+        if context.num_gpus() > 0:
+          last_output, outputs, new_h, new_c, runtime = defun_cudnn_lstm(
+              inputs, initial_state[0], initial_state[1], self.cell.kernel,
+              self.cell.recurrent_kernel, combined_bias, self.time_major)
+        else:
+          last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
+              inputs, initial_state[0], initial_state[1], self.cell.kernel,
+              self.cell.recurrent_kernel, combined_bias, self.activation,
+              self.recurrent_activation, self.time_major)
+      else:
+        # Call the normal LSTM impl and register the CuDNN impl function. The
+        # grappler will kick in during session execution to optimize the graph.
+        last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
+            inputs, initial_state[0], initial_state[1], self.cell.kernel,
+            self.cell.recurrent_kernel, combined_bias, self.activation,
+            self.recurrent_activation, self.time_major)
+
+        function.register(defun_cudnn_lstm, inputs, initial_state[0],
+                          initial_state[1], self.cell.kernel,
+                          self.cell.recurrent_kernel, combined_bias,
+                          self.time_major)
+      states = [new_h, new_c]
+
+    if self.stateful:
+      updates = []
+      for i in range(len(states)):
+        updates.append(state_ops.assign(self.states[i], states[i]))
+      self.add_update(updates, inputs)
+
+    if self.return_sequences:
+      output = outputs
+    else:
+      output = last_output
+
+    if self.return_state:
+      return [output] + states
+    elif self.return_runtime:
+      return output, runtime
+    else:
+      return output
+
+  @property
+  def trainable_weights(self):
+    if self.trainable:
+      weights = []
+      weights += self.cell.trainable_weights
+      if getattr(self, 'cudnn_bias', None) is not None:
+        weights += [self.cudnn_bias]
+      return weights
+    return []
+
+  @property
+  def non_trainable_weights(self):
+    if not self.trainable:
+      weights = []
+      weights += self.cell.non_trainable_weights
+      if getattr(self, 'cudnn_bias', None) is not None:
+        weights += [self.cudnn_bias]
+      return weights
+    return []
+
+  @property
+  def losses(self):
+    losses = []
+    losses += self.cell.losses
+    return losses + self._losses
+
+  @property
+  def updates(self):
+    updates = []
+    updates += self.cell.updates
+    return updates + self._updates
+
+  def get_weights(self):
+    weights = []
+    weights += self.cell.weights
+    if getattr(self, 'cudnn_bias', None) is not None:
+      weights += [self.cudnn_bias]
+    return K.batch_get_value(weights)
+
+  def set_weights(self, weights):
+    tuples = []
+    cell_weights = weights[:len(self.cell.weights)]
+    if cell_weights:
+      tuples.append((self.cell.weights, cell_weights))
+    if getattr(self, 'cudnn_bias', None) is not None:
+      cudnn_bias_weights = weights[len(self.cell.weights):]
+      if cudnn_bias_weights:
+        tuples.append((self.cudnn_bias, cudnn_bias_weights))
+    K.batch_set_value(tuples)
+
+
+def _canonical_to_params(weights, biases, shape):
+  """Utility function convert variable to CuDNN compatible parameter."""
+  weights = [array_ops.reshape(x, shape) for x in weights]
+  biases = [array_ops.reshape(x, shape) for x in biases]
+  return array_ops.concat(weights + biases, axis=0)
+
+
+def standard_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias,
+                  activation, recurrent_activation, time_major):
+  """LSTM with standard kernel implementation.
+
+  This implementation can be run on all types for hardware.
+
+  This implementation lifts out all the layer weights and make them function
+  parameters. It has same number of tensor input params as the CuDNN
+  counterpart. The RNN step logic has been simplified, eg dropout and mask is
+  removed since CuDNN implementation does not support that.
+
+  Note that the first half of the bias tensor should be ignored by this impl.
+  The CuDNN impl need an extra set of input gate bias. In order to make the both
+  function take same shape of parameter, that extra set of bias is also feed
+  here.
+
+  Args:
+    inputs: input tensor of LSTM layer.
+    init_h: initial state tensor for the cell output.
+    init_c: initial state tensor for the cell hidden state.
+    kernel: weights for cell kernel.
+    recurrent_kernel: weights for cell recurrent kernel.
+    bias: weights for cell kernel bias and recurrent bias. Only recurrent bias
+      is used in this case.
+    activation: Activation function to use for output.
+    recurrent_activation: Activation function to use for hidden recurrent state.
+    time_major: boolean, whether the inputs are in the format of
+      [time, batch, feature] or [batch, time, feature].
+
+  Returns:
+    last_output: output tensor for the last timestep, which has shape
+      [batch, units].
+    outputs: output tensor for all timesteps, which has shape
+      [batch, time, units].
+    state_0: the cell output, which has same shape as init_h.
+    state_1: the cell hidden state, which has same shape as init_c.
+    runtime: constant string tensor which indicate real runtime hardware. This
+      value is for testing purpose and should be used by user.
+  """
+  input_shape = K.int_shape(inputs)
+  timesteps = input_shape[0] if time_major else input_shape[1]
+
+  # Only use the second half of the bias weights.
+  _, real_bias = array_ops.split(bias, 2)
+
+  def step(cell_inputs, cell_states):
+    """Step function that will be used by Keras RNN backend."""
+    h_tm1 = cell_states[0]  # previous memory state
+    c_tm1 = cell_states[1]  # previous carry state
+
+    z = K.dot(cell_inputs, kernel)
+    z += K.dot(h_tm1, recurrent_kernel)
+    z = K.bias_add(z, real_bias)
+
+    z0, z1, z2, z3 = array_ops.split(z, 4, axis=1)
+
+    i = recurrent_activation(z0)
+    f = recurrent_activation(z1)
+    c = f * c_tm1 + i * activation(z2)
+    o = recurrent_activation(z3)
+
+    h = o * activation(c)
+    return h, [h, c]
+
+  last_output, outputs, new_states = K.rnn(
+      step,
+      inputs, [init_h, init_c],
+      constants=None,
+      unroll=False,
+      time_major=time_major,
+      input_length=timesteps)
+  return last_output, outputs, new_states[0], new_states[
+      1], constant_op.constant('cpu', dtype=dtypes.string, name='runtime')
+
+
+def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias,
+               time_major):
+  """LSTM with CuDNN implementation which is only available for GPU."""
+  if not time_major:
+    inputs = array_ops.transpose(inputs, perm=(1, 0, 2))
+  input_h = array_ops.expand_dims(input_h, axis=0)
+  input_c = array_ops.expand_dims(input_c, axis=0)
+
+  weights = array_ops.split(kernel, 4, axis=1)
+  weights += array_ops.split(recurrent_kernel, 4, axis=1)
+  params = _canonical_to_params(
+      weights=weights,
+      biases=array_ops.split(bias, 8),
+      shape=constant_op.constant([-1]))
+
+  outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn(
+      inputs, input_h=input_h, input_c=input_c, params=params)
+  if not time_major:
+    outputs = array_ops.transpose(outputs, perm=[1, 0, 2])
+  h = h[0]
+  c = c[0]
+  last_output = outputs[:, -1, :]
+  return last_output, outputs, h, c, constant_op.constant(
+      'cudnn', dtype=dtypes.string, name='runtime')
+
+
 def _generate_dropout_mask(ones, rate, training=None, count=1):
   def dropped_inputs():
     return K.dropout(ones, rate)
diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_rnn_test.py
index e26e47000d..fd94bc9c2f 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_rnn_test.py
@@ -18,34 +18,22 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import time
 
+import numpy as np
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
-from tensorflow.python.eager import context
-from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
-from tensorflow.python.keras import activations
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import constraints
-from tensorflow.python.keras import initializers
-from tensorflow.python.keras import regularizers
 from tensorflow.python.keras import testing_utils
-from tensorflow.python.keras.engine.input_spec import InputSpec
 from tensorflow.python.keras.layers.cudnn_recurrent import CuDNNLSTM
-from tensorflow.python.keras.layers.recurrent import RNN
-from tensorflow.python.keras.utils import tf_utils
+from tensorflow.python.keras.layers.recurrent import UnifiedLSTM
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import gen_cudnn_rnn_ops
 from tensorflow.python.ops import gen_math_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import test
@@ -53,21 +41,17 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import gradient_descent
 
 
-class RNNTest(test.TestCase):
-
-  rewrites = rewriter_config_pb2.RewriterConfig()
-  rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
-  customer_optimizer = rewrites.custom_optimizers.add()
-  customer_optimizer.name = 'ExperimentalImplementationSelector'
-  rewrites.min_graph_nodes = -1
-  graph_options = config_pb2.GraphOptions(rewrite_options=rewrites)
-  config = config_pb2.ConfigProto(graph_options=graph_options)
+# Global config for grappler setting that is used for graph mode test.
+_rewrites = rewriter_config_pb2.RewriterConfig()
+_rewrites.function_optimization = rewriter_config_pb2.RewriterConfig.OFF
+_customer_optimizer = _rewrites.custom_optimizers.add()
+_customer_optimizer.name = 'ExperimentalImplementationSelector'
+_rewrites.min_graph_nodes = -1
+_graph_options = config_pb2.GraphOptions(rewrite_options=_rewrites)
+_config = config_pb2.ConfigProto(graph_options=_graph_options)
 
-  def setUp(self):
-    self.config = RNNTest.config
 
-  def tearDown(self):
-    ops.reset_default_graph()
+class UnifiedRNNTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def test_unifiedRNN(self):
@@ -78,7 +62,7 @@ class RNNTest(test.TestCase):
     batch = 100
     epoch = 1
 
-    with self.cached_session(config=self.config, use_gpu=True) as sess:
+    with self.cached_session(config=_config, use_gpu=True) as sess:
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=batch,
           test_samples=0,
@@ -86,7 +70,7 @@ class RNNTest(test.TestCase):
           num_classes=output_shape)
       y_train = keras.utils.to_categorical(y_train, output_shape)
 
-      layer = UnifiedLSTM(rnn_state_size)
+      layer = UnifiedLSTM(rnn_state_size, return_runtime=True)
 
       inputs = array_ops.placeholder(
           dtypes.float32, shape=(None, timestep, input_shape), name='inputs')
@@ -126,7 +110,7 @@ class RNNTest(test.TestCase):
     batch = 100
     epoch = 1
 
-    with self.cached_session(config=self.config, use_gpu=True) as sess:
+    with self.cached_session(config=_config, use_gpu=True) as sess:
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=batch,
           test_samples=0,
@@ -134,7 +118,7 @@ class RNNTest(test.TestCase):
           num_classes=output_shape)
       y_train = keras.utils.to_categorical(y_train, output_shape)
 
-      layer = UnifiedLSTM(rnn_state_size)
+      layer = UnifiedLSTM(rnn_state_size, return_runtime=True)
 
       inputs = array_ops.placeholder(
           dtypes.float32, shape=(None, timestep, input_shape), name='inputs')
@@ -172,7 +156,7 @@ class RNNTest(test.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
-  @test_util.run_in_graph_and_eager_modes(config=config)
+  @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_keras_model_with_lstm(self):
     input_shape = 10
     rnn_state_size = 8
@@ -193,10 +177,420 @@ class RNNTest(test.TestCase):
     inputs = keras.layers.Input(
         shape=[timestep, input_shape], dtype=dtypes.float32)
 
-    outputs, unused_runtime = layer(inputs)
+    outputs = layer(inputs)
     model = keras.models.Model(inputs, outputs)
     model.compile('rmsprop', loss='mse')
     model.fit(x_train, y_train, epochs=epoch)
+    model.evaluate(x_train, y_train)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_return_sequences_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    testing_utils.layer_test(
+        UnifiedLSTM,
+        kwargs={
+            'units': units,
+            'return_sequences': True
+        },
+        input_shape=(num_samples, timesteps, embedding_dim))
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_static_shape_inference_LSTM(self):
+    # Github issue: 15165
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+
+    model = keras.models.Sequential()
+    inputs = keras.layers.Dense(
+        embedding_dim, input_shape=(timesteps, embedding_dim))
+    model.add(inputs)
+    layer = UnifiedLSTM(units, return_sequences=True)
+    model.add(layer)
+    outputs = model.layers[-1].output
+    self.assertEquals(outputs.get_shape().as_list(), [None, timesteps, units])
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_dynamic_behavior_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    layer = UnifiedLSTM(units, input_shape=(None, embedding_dim))
+    model = keras.models.Sequential()
+    model.add(layer)
+    model.compile(gradient_descent.GradientDescentOptimizer(0.001), 'mse')
+    x = np.random.random((num_samples, timesteps, embedding_dim))
+    y = np.random.random((num_samples, units))
+    model.train_on_batch(x, y)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_dropout_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    testing_utils.layer_test(
+        UnifiedLSTM,
+        kwargs={
+            'units': units,
+            'dropout': 0.1,
+            'recurrent_dropout': 0.1
+        },
+        input_shape=(num_samples, timesteps, embedding_dim))
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_implementation_mode_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    for mode in [1, 2]:
+      testing_utils.layer_test(
+          UnifiedLSTM,
+          kwargs={
+              'units': units,
+              'implementation': mode
+          },
+          input_shape=(num_samples, timesteps, embedding_dim))
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_constraints_LSTM(self):
+    embedding_dim = 4
+    layer_class = UnifiedLSTM
+    k_constraint = keras.constraints.max_norm(0.01)
+    r_constraint = keras.constraints.max_norm(0.01)
+    b_constraint = keras.constraints.max_norm(0.01)
+    layer = layer_class(
+        5,
+        return_sequences=False,
+        weights=None,
+        input_shape=(None, embedding_dim),
+        kernel_constraint=k_constraint,
+        recurrent_constraint=r_constraint,
+        bias_constraint=b_constraint)
+    layer.build((None, None, embedding_dim))
+    self.assertEqual(layer.cell.kernel.constraint, k_constraint)
+    self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
+    self.assertEqual(layer.cell.bias.constraint, b_constraint)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_with_masking_layer_LSTM(self):
+    layer_class = UnifiedLSTM
+    inputs = np.random.random((2, 3, 4))
+    targets = np.abs(np.random.random((2, 3, 5)))
+    targets /= targets.sum(axis=-1, keepdims=True)
+    model = keras.models.Sequential()
+    model.add(keras.layers.Masking(input_shape=(3, 4)))
+    model.add(layer_class(units=5, return_sequences=True, unroll=False))
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+    model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_stacking_LSTM(self):
+    inputs = np.random.random((2, 3, 4))
+    targets = np.abs(np.random.random((2, 3, 5)))
+    targets /= targets.sum(axis=-1, keepdims=True)
+    model = keras.models.Sequential()
+    model.add(UnifiedLSTM(10, return_sequences=True, unroll=False))
+    model.add(UnifiedLSTM(5, return_sequences=True, unroll=False))
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+    model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_masking_with_stacking_LSTM(self):
+    inputs = np.random.random((2, 3, 4))
+    targets = np.abs(np.random.random((2, 3, 5)))
+    targets /= targets.sum(axis=-1, keepdims=True)
+    model = keras.models.Sequential()
+    model.add(keras.layers.Masking(input_shape=(3, 4)))
+    model.add(UnifiedLSTM(10, return_sequences=True, unroll=False))
+    model.add(UnifiedLSTM(5, return_sequences=True, unroll=False))
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+    model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_from_config_LSTM(self):
+    layer_class = UnifiedLSTM
+    for stateful in (False, True):
+      l1 = layer_class(units=1, stateful=stateful)
+      l2 = layer_class.from_config(l1.get_config())
+      assert l1.get_config() == l2.get_config()
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_specify_initial_state_keras_tensor(self):
+    num_states = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+
+    # Test with Keras tensor
+    inputs = keras.Input((timesteps, embedding_dim))
+    initial_state = [keras.Input((units,)) for _ in range(num_states)]
+    layer = UnifiedLSTM(units)
+    if len(initial_state) == 1:
+      output = layer(inputs, initial_state=initial_state[0])
+    else:
+      output = layer(inputs, initial_state=initial_state)
+    assert initial_state[0] in layer._inbound_nodes[0].input_tensors
+
+    model = keras.models.Model([inputs] + initial_state, output)
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+
+    inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    initial_state = [
+        np.random.random((num_samples, units)) for _ in range(num_states)
+    ]
+    targets = np.random.random((num_samples, units))
+    model.train_on_batch([inputs] + initial_state, targets)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def DISABLED_test_specify_initial_state_non_keras_tensor(self):
+    num_states = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+
+    # Test with non-Keras tensor
+    inputs = keras.Input((timesteps, embedding_dim))
+    initial_state = [
+        keras.backend.random_normal_variable((num_samples, units), 0, 1)
+        for _ in range(num_states)
+    ]
+    layer = UnifiedLSTM(units)
+    output = layer(inputs, initial_state=initial_state)
+
+    model = keras.models.Model(inputs, output)
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+
+    inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    targets = np.random.random((num_samples, units))
+    model.train_on_batch(inputs, targets)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_reset_states_with_values(self):
+    num_states = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+
+    layer = UnifiedLSTM(units, stateful=True)
+    layer.build((num_samples, timesteps, embedding_dim))
+    layer.reset_states()
+    assert len(layer.states) == num_states
+    assert layer.states[0] is not None
+    self.assertAllClose(
+        keras.backend.eval(layer.states[0]),
+        np.zeros(keras.backend.int_shape(layer.states[0])),
+        atol=1e-4)
+    state_shapes = [keras.backend.int_shape(state) for state in layer.states]
+    values = [np.ones(shape) for shape in state_shapes]
+    if len(values) == 1:
+      values = values[0]
+    layer.reset_states(values)
+    self.assertAllClose(
+        keras.backend.eval(layer.states[0]),
+        np.ones(keras.backend.int_shape(layer.states[0])),
+        atol=1e-4)
+
+    # Test with invalid data
+    with self.assertRaises(ValueError):
+      layer.reset_states([1] * (len(layer.states) + 1))
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_specify_state_with_masking(self):
+    num_states = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+
+    inputs = keras.Input((timesteps, embedding_dim))
+    _ = keras.layers.Masking()(inputs)
+    initial_state = [keras.Input((units,)) for _ in range(num_states)]
+    output = UnifiedLSTM(units)(inputs, initial_state=initial_state)
+
+    model = keras.models.Model([inputs] + initial_state, output)
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+
+    inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    initial_state = [
+        np.random.random((num_samples, units)) for _ in range(num_states)
+    ]
+    targets = np.random.random((num_samples, units))
+    model.train_on_batch([inputs] + initial_state, targets)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_return_state(self):
+    num_states = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+
+    inputs = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim))
+    layer = UnifiedLSTM(units, return_state=True, stateful=True)
+    outputs = layer(inputs)
+    state = outputs[1:]
+    assert len(state) == num_states
+    model = keras.models.Model(inputs, state[0])
+
+    inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    state = model.predict(inputs)
+    self.assertAllClose(keras.backend.eval(layer.states[0]), state, atol=1e-4)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_state_reuse(self):
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+
+    inputs = keras.Input(batch_shape=(num_samples, timesteps, embedding_dim))
+    layer = UnifiedLSTM(units, return_state=True, return_sequences=True)
+    outputs = layer(inputs)
+    output, state = outputs[0], outputs[1:]
+    output = UnifiedLSTM(units)(output, initial_state=state)
+    model = keras.models.Model(inputs, output)
+
+    inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    model.predict(inputs)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_initial_states_as_other_inputs(self):
+    timesteps = 3
+    embedding_dim = 4
+    units = 3
+    num_samples = 2
+    num_states = 2
+    layer_class = UnifiedLSTM
+
+    # Test with Keras tensor
+    main_inputs = keras.Input((timesteps, embedding_dim))
+    initial_state = [keras.Input((units,)) for _ in range(num_states)]
+    inputs = [main_inputs] + initial_state
+
+    layer = layer_class(units)
+    output = layer(inputs)
+    assert initial_state[0] in layer._inbound_nodes[0].input_tensors
+
+    model = keras.models.Model(inputs, output)
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=gradient_descent.GradientDescentOptimizer(0.01))
+
+    main_inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    initial_state = [
+        np.random.random((num_samples, units)) for _ in range(num_states)
+    ]
+    targets = np.random.random((num_samples, units))
+    model.train_on_batch([main_inputs] + initial_state, targets)
+
+
+class LSTMLayerGraphOnlyTest(test.TestCase):
+
+  def test_statefulness_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    layer_class = UnifiedLSTM
+    with self.cached_session(config=_config):
+      model = keras.models.Sequential()
+      model.add(
+          keras.layers.Embedding(
+              4,
+              embedding_dim,
+              mask_zero=True,
+              input_length=timesteps,
+              batch_input_shape=(num_samples, timesteps)))
+      layer = layer_class(
+          units, return_sequences=False, stateful=True, weights=None)
+      model.add(layer)
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(0.01), loss='mse')
+      out1 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertEqual(out1.shape, (num_samples, units))
+
+      # train once so that the states change
+      model.train_on_batch(
+          np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
+      out2 = model.predict(np.ones((num_samples, timesteps)))
+
+      # if the state is not reset, output should be different
+      self.assertNotEqual(out1.max(), out2.max())
+
+      # check that output changes after states are reset
+      # (even though the model itself didn't change)
+      layer.reset_states()
+      out3 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertNotEqual(out2.max(), out3.max())
+
+      # check that container-level reset_states() works
+      model.reset_states()
+      out4 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertAllClose(out3, out4, atol=1e-5)
+
+      # check that the call to `predict` updated the states
+      out5 = model.predict(np.ones((num_samples, timesteps)))
+      self.assertNotEqual(out4.max(), out5.max())
+
+      # Check masking
+      layer.reset_states()
+
+      left_padded_input = np.ones((num_samples, timesteps))
+      left_padded_input[0, :1] = 0
+      left_padded_input[1, :2] = 0
+      out6 = model.predict(left_padded_input)
+
+      layer.reset_states()
+
+      right_padded_input = np.ones((num_samples, timesteps))
+      right_padded_input[0, -1:] = 0
+      right_padded_input[1, -2:] = 0
+      out7 = model.predict(right_padded_input)
+
+      self.assertAllClose(out7, out6, atol=1e-5)
+
+  def test_regularizers_LSTM(self):
+    embedding_dim = 4
+    layer_class = UnifiedLSTM
+    with self.cached_session(config=_config):
+      layer = layer_class(
+          5,
+          return_sequences=False,
+          weights=None,
+          input_shape=(None, embedding_dim),
+          kernel_regularizer=keras.regularizers.l1(0.01),
+          recurrent_regularizer=keras.regularizers.l1(0.01),
+          bias_regularizer='l2',
+          activity_regularizer='l1')
+      layer.build((None, None, 2))
+      self.assertEqual(len(layer.losses), 3)
+      x = keras.backend.variable(np.ones((2, 3, 2)))
+      layer(x)
+      self.assertEqual(len(layer.get_losses_for(x)), 1)
+
+
+class UnifiedRNNPerformanceTest(test.TestCase):
 
   def _measure_performance(self, test_config, model, x_train, y_train):
     batch = test_config['batch']
@@ -241,7 +635,7 @@ class RNNTest(test.TestCase):
     inputs = keras.layers.Input(
         shape=[timestep, input_shape], dtype=dtypes.float32)
 
-    outputs, _ = layer(inputs)
+    outputs = layer(inputs)
     model = keras.models.Model(inputs, outputs)
     model.compile('sgd', 'mse')
 
@@ -272,7 +666,7 @@ class RNNTest(test.TestCase):
                  'Normal LSTM', sec_per_epoch)
     return sec_per_epoch
 
-  @test_util.run_in_graph_and_eager_modes(config=config, use_gpu=True)
+  @test_util.run_in_graph_and_eager_modes(config=_config, use_gpu=True)
   def test_performance_with_standard_cudnn_impl(self):
     if not test.is_gpu_available():
       self.skipTest('performance test will only run on GPU')
@@ -325,318 +719,5 @@ class RNNTest(test.TestCase):
     #     'Expect the performance of Unified LSTM is more than 5 times of '
     #     'normal LSTM, but got {0:.2f}'.format(unified_vs_normal))
 
-
-class UnifiedLSTM(RNN):
-
-  def __init__(self,
-               units,
-               activation='tanh',
-               recurrent_activation='hard_sigmoid',
-               kernel_initializer='glorot_uniform',
-               recurrent_initializer='orthogonal',
-               bias_initializer='zeros',
-               unit_forget_bias=True,
-               kernel_regularizer=None,
-               recurrent_regularizer=None,
-               bias_regularizer=None,
-               activity_regularizer=None,
-               kernel_constraint=None,
-               recurrent_constraint=None,
-               bias_constraint=None,
-               return_sequences=False,
-               return_state=False,
-               go_backwards=False,
-               stateful=False,
-               time_major=False,
-               **kwargs):
-    super(RNN, self).__init__(**kwargs)  # pylint: disable=bad-super-call
-    self.units = units
-    cell_spec = collections.namedtuple('cell', ['state_size', 'output_size'])
-    self.cell = cell_spec(
-        state_size=(self.units, self.units), output_size=self.units)
-    self.activation = activations.get(activation)
-    self.recurrent_activation = activations.get(recurrent_activation)
-    self.kernel_initializer = initializers.get(kernel_initializer)
-    self.recurrent_initializer = initializers.get(recurrent_initializer)
-    self.bias_initializer = initializers.get(bias_initializer)
-    self.unit_forget_bias = unit_forget_bias
-
-    self.kernel_regularizer = regularizers.get(kernel_regularizer)
-    self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
-    self.bias_regularizer = regularizers.get(bias_regularizer)
-    self.activity_regularizer = regularizers.get(activity_regularizer)
-
-    self.kernel_constraint = constraints.get(kernel_constraint)
-    self.recurrent_constraint = constraints.get(recurrent_constraint)
-    self.bias_constraint = constraints.get(bias_constraint)
-
-    self.return_sequences = return_sequences
-    self.return_state = return_state
-    self.go_backwards = go_backwards
-    self.stateful = stateful
-    self.time_major = time_major
-    self._num_constants = None
-    self._num_inputs = None
-    self._states = None
-    self.input_spec = [InputSpec(ndim=3)]
-    self.state_spec = [
-        InputSpec(shape=(None, dim)) for dim in (self.units, self.units)
-    ]
-
-  @tf_utils.shape_type_conversion
-  def build(self, input_shape):
-    super(UnifiedLSTM, self).build(input_shape)
-    if isinstance(input_shape, list):
-      input_shape = input_shape[0]
-    input_dim = int(input_shape[-1])
-
-    self.kernel = self.add_weight(
-        shape=(input_dim, self.units * 4),
-        name='kernel',
-        dtype=dtypes.float32,
-        use_resource=True,
-        initializer=self.kernel_initializer,
-        regularizer=self.kernel_regularizer,
-        constraint=self.kernel_constraint)
-    self.recurrent_kernel = self.add_weight(
-        shape=(self.units, self.units * 4),
-        name='recurrent_kernel',
-        dtype=dtypes.float32,
-        use_resource=True,
-        initializer=self.recurrent_initializer,
-        regularizer=self.recurrent_regularizer,
-        constraint=self.recurrent_constraint)
-
-    # Normal LSTM has 4 bias instead of 8.
-    if self.unit_forget_bias:
-
-      def bias_initializer(_, *args, **kwargs):
-        return array_ops.concat([
-            self.bias_initializer((self.units * 5,), *args, **kwargs),
-            initializers.Ones()((self.units,), *args, **kwargs),
-            self.bias_initializer((self.units * 2,), *args, **kwargs),
-        ],
-                                axis=0)
-    else:
-      bias_initializer = self.bias_initializer
-    self.bias = self.add_weight(
-        shape=(self.units * 8,),
-        name='bias',
-        dtype=dtypes.float32,
-        use_resource=True,
-        initializer=bias_initializer,
-        regularizer=self.bias_regularizer,
-        constraint=self.bias_constraint)
-    self.built = True
-
-  def call(self, inputs, mask=None, training=None, initial_state=None):
-    if isinstance(inputs, list):
-      initial_state = inputs[1:]
-      inputs = inputs[0]
-    elif initial_state is not None:
-      pass
-    elif self.stateful:
-      initial_state = self.states
-    else:
-      initial_state = self.get_initial_state(inputs)
-
-    if len(initial_state) != len(self.states):
-      raise ValueError('Layer has ' + str(len(self.states)) +
-                       ' states but was passed ' + str(len(initial_state)) +
-                       ' initial states.')
-
-    if self.go_backwards:
-      # Reverse time axis.
-      inputs = K.reverse(inputs, 1)
-
-    if ops.executing_eagerly_outside_functions():
-      if context.num_gpus() > 0:
-        outputs, [new_h, new_c], runtime = cudnn_lstm(
-            inputs, initial_state[0], initial_state[1], self.kernel,
-            self.recurrent_kernel, self.bias, self.units)
-      else:
-        outputs, [new_h, new_c], runtime = normal_lstm(
-            inputs, initial_state[0], initial_state[1], self.kernel,
-            self.recurrent_kernel, self.bias, self.units, self.activation,
-            self.recurrent_activation)
-    else:
-      outputs, [new_h, new_c], runtime = normal_lstm(
-          inputs, initial_state[0], initial_state[1], self.kernel,
-          self.recurrent_kernel, self.bias, self.units, self.activation,
-          self.recurrent_activation)
-
-      function.register(cudnn_lstm, inputs, initial_state[0], initial_state[1],
-                        self.kernel, self.recurrent_kernel, self.bias,
-                        self.units)
-
-    states = [new_h, new_c]
-
-    if self.stateful:
-      updates = []
-      for i in range(len(states)):
-        updates.append(state_ops.assign(self.states[i], states[i]))
-      self.add_update(updates, inputs)
-
-    if self.return_sequences:
-      output = outputs
-    else:
-      output = outputs[:, -1, :]
-
-    if self.return_state:
-      return [output] + states
-    else:
-      return output, runtime
-
-  @tf_utils.shape_type_conversion
-  def compute_output_shape(self, input_shape):
-    if isinstance(input_shape, list):
-      input_shape = input_shape[0]
-
-    if _is_multiple_state(self.cell.state_size):
-      state_size = self.cell.state_size
-    else:
-      state_size = [self.cell.state_size]
-
-    if getattr(self.cell, 'output_size', None) is not None:
-      output_dim = tensor_shape.as_shape(self.cell.output_size).as_list()
-    else:
-      # Note that state_size[0] could be a tensor_shape or int.
-      output_dim = tensor_shape.as_shape(state_size[0]).as_list()
-
-    if self.return_sequences:
-      output_shape = tuple([input_shape[0], input_shape[1]] + output_dim)
-    else:
-      output_shape = tuple([input_shape[0]] + output_dim)
-
-    if self.return_state:
-      state_shape = [
-          tuple([input_shape[0]] + tensor_shape.as_shape(dim).as_list())
-          for dim in state_size
-      ]
-      return [output_shape] + state_shape
-    else:
-      return output_shape
-
-  @property
-  def trainable_weights(self):
-    if self.trainable and self.built:
-      return [self.kernel, self.recurrent_kernel, self.bias]
-    return []
-
-  @property
-  def non_trainable_weights(self):
-    if not self.trainable and self.built:
-      return [self.kernel, self.recurrent_kernel, self.bias]
-    return []
-
-  @property
-  def losses(self):
-    return super(RNN, self).losses
-
-  def get_losses_for(self, inputs=None):
-    return super(RNN, self).get_losses_for(inputs=inputs)   # pylint: disable=bad-super-call
-
-  def get_weights(self):
-    return super(RNN, self).get_weights()  # pylint: disable=bad-super-call
-
-
-def _canonical_to_params(weights, biases, shape):
-  weights = [array_ops.reshape(x, shape) for x in weights]
-  biases = [array_ops.reshape(x, shape) for x in biases]
-  return array_ops.concat(weights + biases, axis=0)
-
-
-def _is_multiple_state(state_size):
-  """Check whether the state_size contains multiple states."""
-  return (hasattr(state_size, '__len__') and
-          not isinstance(state_size, tensor_shape.TensorShape))
-
-
-@function.defun_with_attributes(
-    attributes={
-        'experimental_api_implements': 'lstm',
-        'experimental_api_preferred_device': 'CPU'
-    })
-def normal_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias, units,
-                activation, recurrent_activation):
-  input_shape = K.int_shape(inputs)
-  timesteps = input_shape[1]
-
-  def step(cell_inputs, cell_states):
-    h_tm1 = cell_states[0]  # previous memory state
-    c_tm1 = cell_states[1]  # previous carry state
-
-    # Only use the second half of the bias weights.
-    _, real_bias = array_ops.split(bias, 2)
-
-    z = K.dot(cell_inputs, kernel)
-    z += K.dot(h_tm1, recurrent_kernel)
-    z = K.bias_add(z, real_bias)
-
-    z0 = z[:, :units]
-    z1 = z[:, units:2 * units]
-    z2 = z[:, 2 * units:3 * units]
-    z3 = z[:, 3 * units:]
-
-    i = recurrent_activation(z0)
-    f = recurrent_activation(z1)
-    c = f * c_tm1 + i * activation(z2)
-    o = recurrent_activation(z3)
-
-    h = o * activation(c)
-    return h, [h, c]
-
-  _, outputs, new_states = K.rnn(
-      step,
-      inputs, [init_h, init_c],
-      constants=None,
-      unroll=False,
-      input_length=timesteps)
-  return outputs, new_states, constant_op.constant(
-      'cpu', dtype=dtypes.string, name='runtime')
-
-
-@function.defun_with_attributes(
-    attributes={
-        'experimental_api_implements': 'lstm',
-        'experimental_api_preferred_device': 'GPU'
-    })
-def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias, units):
-  inputs = array_ops.transpose(inputs, perm=(1, 0, 2))
-  input_h = array_ops.expand_dims(input_h, axis=0)
-  input_c = array_ops.expand_dims(input_c, axis=0)
-
-  params = _canonical_to_params(
-      weights=[
-          kernel[:, :units],
-          kernel[:, units:units * 2],
-          kernel[:, units * 2:units * 3],
-          kernel[:, units * 3:],
-          recurrent_kernel[:, :units],
-          recurrent_kernel[:, units:units * 2],
-          recurrent_kernel[:, units * 2:units * 3],
-          recurrent_kernel[:, units * 3:],
-      ],
-      biases=[
-          bias[:units],
-          bias[units:units * 2],
-          bias[units * 2:units * 3],
-          bias[units * 3:units * 4],
-          bias[units * 4:units * 5],
-          bias[units * 5:units * 6],
-          bias[units * 6:units * 7],
-          bias[units * 7:],
-      ],
-      shape=constant_op.constant([-1]))
-
-  outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn(
-      inputs, input_h=input_h, input_c=input_c, params=params)
-  outputs = array_ops.transpose(outputs, perm=[1, 0, 2])
-  h = h[0]
-  c = c[0]
-  return outputs, [h, c], constant_op.constant(
-      'cudnn', dtype=dtypes.string, name='runtime')
-
-
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index d342131a52..7827e16627 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -149,6 +149,10 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
     np.testing.assert_allclose(output, actual_output, rtol=1e-3)
 
   # test training mode (e.g. useful for dropout tests)
+  # Rebuild the model to avoid the graph being reused between predict() and
+  # train(). This was causing some error for layer with Defun as it body.
+  # See b/120160788 for more details. This should be mitigated after 2.0.
+  model = keras.models.Model(x, layer(x))
   model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'])
   model.train_on_batch(input_data, actual_output)
 
-- 
GitLab


From 29c1506fecd37c853f4c5d53e0a199c04f213436 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 30 Nov 2018 16:10:51 -0800
Subject: [PATCH 1129/1554] Change the variable names to be more readable

---
 .../data/kernel_tests/list_files_test.py      | 33 ++++++++++---------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/list_files_test.py b/tensorflow/python/data/kernel_tests/list_files_test.py
index 789f1ab6de..d4e0cdeb97 100644
--- a/tensorflow/python/data/kernel_tests/list_files_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_test.py
@@ -82,27 +82,27 @@ class ListFilesTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.list_files(
         path.join(self.tmp_dir, '*'), shuffle=True, seed=37)
 
-    full_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename))
+    expected_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename))
                       for filename in filenames]
 
-    all_produced_filenames = []
+    all_actual_filenames = []
     for _ in range(3):
-      produced_filenames = []
+      actual_filenames = []
       next_element = self.getNext(dataset, requires_initialization=True)
       try:
         while True:
-          produced_filenames.append(self.evaluate(next_element()))
+          actual_filenames.append(self.evaluate(next_element()))
       except errors.OutOfRangeError:
         pass
-      all_produced_filenames.append(produced_filenames)
+      all_actual_filenames.append(actual_filenames)
 
     # Each run should produce the same set of filenames, which may be
-    # different from the order of `full_filenames`.
-    self.assertItemsEqual(full_filenames, all_produced_filenames[0])
+    # different from the order of `expected_filenames`.
+    self.assertItemsEqual(expected_filenames, all_actual_filenames[0])
     # However, the different runs should produce filenames in the same order
     # as each other.
-    self.assertEqual(all_produced_filenames[0], all_produced_filenames[1])
-    self.assertEqual(all_produced_filenames[0], all_produced_filenames[2])
+    self.assertEqual(all_actual_filenames[0], all_actual_filenames[1])
+    self.assertEqual(all_actual_filenames[0], all_actual_filenames[2])
 
   # TODO(b/117581999): eager mode assertion fail wrapped, debug.
   def tesSkipEagerEmptyDirectoryInitializer(self):
@@ -169,16 +169,17 @@ class ListFilesTest(test_base.DatasetTestBase):
         path.join(self.tmp_dir, '*'), shuffle=False).repeat(2)
     next_element = self.getNext(dataset)
 
-    full_filenames = []
-    produced_filenames = []
+    expected_filenames = []
+    actual_filenames = []
     for filename in filenames * 2:
-      full_filenames.append(compat.as_bytes(path.join(self.tmp_dir, filename)))
-      produced_filenames.append(compat.as_bytes(self.evaluate(next_element())))
+      expected_filenames.append(
+          compat.as_bytes(path.join(self.tmp_dir, filename)))
+      actual_filenames.append(compat.as_bytes(self.evaluate(next_element())))
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(next_element())
-    self.assertItemsEqual(full_filenames, produced_filenames)
-    self.assertEqual(produced_filenames[:len(filenames)],
-                     produced_filenames[len(filenames):])
+    self.assertItemsEqual(expected_filenames, actual_filenames)
+    self.assertEqual(actual_filenames[:len(filenames)],
+                     actual_filenames[len(filenames):])
 
   def testMultiplePatternsAsList(self):
     filenames = ['a.txt', 'b.py', 'c.py', 'd.pyc']
-- 
GitLab


From 644e58cd69263cd87f678d1d90ca7b7d4bfc9634 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 30 Nov 2018 15:47:27 -0800
Subject: [PATCH 1130/1554] [TF:XLA] Don't auto-cluster the Multinomial op

Multinomial is also a stateful random op.  This was an omission in cl/219368223.

PiperOrigin-RevId: 223588036
---
 tensorflow/compiler/jit/mark_for_compilation_pass.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 25796435a5..6618e3a58a 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -86,7 +86,7 @@ bool IsDummyImplOp(absl::string_view op_name) {
 bool IsStatefulRandomOp(absl::string_view op_name) {
   return op_name == "RandomUniform" || op_name == "RandomShuffle" ||
          op_name == "RandomUniformInt" || op_name == "RandomStandardNormal" ||
-         op_name == "TruncatedNormal";
+         op_name == "TruncatedNormal" || op_name == "Multinomial";
 }
 
 bool OpProducesOrConsumesVariant(const Node& node) {
-- 
GitLab


From cb57c4dd8d9cadcb0c25c0940d7cb4f08d5aa530 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 30 Nov 2018 16:32:34 -0800
Subject: [PATCH 1131/1554] Fix the coding style

---
 tensorflow/python/data/kernel_tests/list_files_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/kernel_tests/list_files_test.py b/tensorflow/python/data/kernel_tests/list_files_test.py
index d4e0cdeb97..093d6cb3ca 100644
--- a/tensorflow/python/data/kernel_tests/list_files_test.py
+++ b/tensorflow/python/data/kernel_tests/list_files_test.py
@@ -83,7 +83,7 @@ class ListFilesTest(test_base.DatasetTestBase):
         path.join(self.tmp_dir, '*'), shuffle=True, seed=37)
 
     expected_filenames = [compat.as_bytes(path.join(self.tmp_dir, filename))
-                      for filename in filenames]
+                          for filename in filenames]
 
     all_actual_filenames = []
     for _ in range(3):
-- 
GitLab


From 36f5fe96cc1febe25284383e14b12dd8007dab9b Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 30 Nov 2018 16:28:06 -0800
Subject: [PATCH 1132/1554] [XLA] Delete dead function in hlo_graph_dumper.

No functional change.

PiperOrigin-RevId: 223593972
---
 tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 0745c735e5..4bfa211121 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -112,11 +112,6 @@ class NodeFilter {
            result == kSomeUsersOmitted;
   }
 
-  bool ShowFusionSubcomputation(const HloInstruction* instr) const {
-    CHECK_EQ(instr->opcode(), HloOpcode::kFusion);
-    return Show(instr) && !SomeOrAllOperandsOmitted(instr);
-  }
-
  private:
   std::function<NodeFilterResult(const HloInstruction* instr)> filter_;
 };
-- 
GitLab


From f30d3d01e97dc2e6efc2c916dd9c7dc12c7fa9be Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 30 Nov 2018 16:30:47 -0800
Subject: [PATCH 1133/1554] [tf.data] Fix handling of `NestedStructure` objects
 with nested `NestedStructure` objects.

PiperOrigin-RevId: 223594419
---
 tensorflow/python/data/util/structure.py      | 16 ++++++---
 tensorflow/python/data/util/structure_test.py | 33 +++++++++++++++++++
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 3cf67b0745..01c7c87947 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -316,15 +316,23 @@ class NestedStructure(Structure):
                        % (len(self._flat_types), len(flat_value)))
 
     flat_ret = []
-    for sub_value, structure in zip(flat_value, self._flat_nested_structure):
-      flat_ret.append(structure._from_tensor_list([sub_value]))
+    i = 0
+    for structure in self._flat_nested_structure:
+      num_flat_values = len(structure._flat_types)
+      sub_value = flat_value[i:i + num_flat_values]
+      flat_ret.append(structure._from_tensor_list(sub_value))
+      i += num_flat_values
 
     return nest.pack_sequence_as(self._nested_structure, flat_ret)
 
   def _from_compatible_tensor_list(self, flat_value):
     flat_ret = []
-    for sub_value, structure in zip(flat_value, self._flat_nested_structure):
-      flat_ret.append(structure._from_compatible_tensor_list([sub_value]))
+    i = 0
+    for structure in self._flat_nested_structure:
+      num_flat_values = len(structure._flat_types)
+      sub_value = flat_value[i:i + num_flat_values]
+      flat_ret.append(structure._from_compatible_tensor_list(sub_value))
+      i += num_flat_values
 
     return nest.pack_sequence_as(self._nested_structure, flat_ret)
 
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index e9e2f5be0a..e917747ec6 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -359,5 +359,38 @@ class StructureTest(test.TestCase, parameterized.TestCase):
     self.assertTrue(expected_structure.is_compatible_with(actual_structure))
     self.assertTrue(actual_structure.is_compatible_with(expected_structure))
 
+  def testNestedNestedStructure(self):
+    # Although `Structure.from_value()` will not construct one, a nested
+    # structure containing nested `NestedStructure` objects can occur if a
+    # structure is constructed manually.
+    s = structure.NestedStructure(
+        (structure.TensorStructure(dtypes.int64, []),
+         structure.NestedStructure(
+             (structure.TensorStructure(dtypes.float32, []),
+              structure.TensorStructure(dtypes.string, [])))))
+
+    int64_t = constant_op.constant(37, dtype=dtypes.int64)
+    float32_t = constant_op.constant(42.0)
+    string_t = constant_op.constant("Foo")
+
+    nested_tensors = (int64_t, (float32_t, string_t))
+
+    tensor_list = s._to_tensor_list(nested_tensors)
+    for expected, actual in zip([int64_t, float32_t, string_t], tensor_list):
+      self.assertIs(expected, actual)
+
+    (actual_int64_t, (actual_float32_t, actual_string_t)) = s._from_tensor_list(
+        tensor_list)
+    self.assertIs(int64_t, actual_int64_t)
+    self.assertIs(float32_t, actual_float32_t)
+    self.assertIs(string_t, actual_string_t)
+
+    (actual_int64_t, (actual_float32_t, actual_string_t)) = (
+        s._from_compatible_tensor_list(tensor_list))
+    self.assertIs(int64_t, actual_int64_t)
+    self.assertIs(float32_t, actual_float32_t)
+    self.assertIs(string_t, actual_string_t)
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 61c7cbca28f5095c87f1c02e1b9a3aa0096a340c Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 30 Nov 2018 16:40:07 -0800
Subject: [PATCH 1134/1554] Add functions to switch between 1.x and 2.x global
 behavior.

PiperOrigin-RevId: 223595880
---
 tensorflow/api_template.__init__.py           |  5 ++-
 tensorflow/python/compat/BUILD                |  5 ++-
 tensorflow/python/compat/compat.py            | 43 +++++++++++++++++++
 tensorflow/python/framework/ops.py            | 14 +++++-
 tensorflow/python/tf2.py                      | 17 +++++++-
 .../tools/api/golden/v1/tensorflow.pbtxt      | 12 ++++++
 .../tools/api/golden/v2/tensorflow.pbtxt      |  4 --
 tensorflow/tools/compatibility/renames_v2.py  |  4 ++
 8 files changed, 95 insertions(+), 9 deletions(-)

diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py
index f13623b0d5..d81cf067eb 100644
--- a/tensorflow/api_template.__init__.py
+++ b/tensorflow/api_template.__init__.py
@@ -35,8 +35,9 @@ _tf_api_dir = _os.path.dirname(_os.path.dirname(bitwise.__file__))  # pylint: di
 if _tf_api_dir not in __path__:
   __path__.append(_tf_api_dir)
 
-# Calls to enable and disable features.
-enable_eager_execution()  # pylint: disable=undefined-variable
+# Enable TF2 behaviors
+from tensorflow.python.compat import compat as _compat  # pylint: disable=g-import-not-at-top
+_compat.enable_v2_behavior()
 
 # These symbols appear because we import the python package which
 # in turn imports from tensorflow.core and tensorflow.python. They
diff --git a/tensorflow/python/compat/BUILD b/tensorflow/python/compat/BUILD
index e0a1c8e057..9f2ce8c676 100644
--- a/tensorflow/python/compat/BUILD
+++ b/tensorflow/python/compat/BUILD
@@ -9,7 +9,10 @@ py_library(
     srcs = ["compat.py"],
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
-    deps = ["//tensorflow/python:util"],
+    deps = [
+        "//tensorflow/python:tf2",
+        "//tensorflow/python:util",
+    ],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index f83af0a583..156680e081 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -23,6 +23,12 @@ from __future__ import division
 from __future__ import print_function
 
 import datetime
+
+from tensorflow.python import tf2
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import variable_scope
+
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
@@ -132,3 +138,40 @@ def forward_compatibility_horizon(year, month, day):
     yield
   finally:
     _FORWARD_COMPATIBILITY_HORIZON = old_compat_date
+
+
+@tf_export(v1=["enable_v2_behavior"])
+def enable_v2_behavior():
+  """Enables TensorFlow 2.x behaviors.
+
+  This function can be called at the beginning of the program (before `Tensors`,
+  `Graphs` or other structures have been created, and before devices have been
+  initialized. It switches all global behaviors that are different between
+  TensorFlow 1.x and 2.x to behave as intended for 2.x.
+
+  This function is called in the main TensorFlow `__init__.py` file, user should
+  not need to call it, except during complex migrations.
+  """
+  tf2.enable()  # Switches TensorArrayV2 and control flow V2
+  ops.enable_eager_execution()
+  tensor_shape.enable_v2_tensorshape()  # Also switched by tf2
+  variable_scope.enable_resource_variables()
+
+
+@tf_export(v1=["disable_v2_behavior"])
+def disable_v2_behavior():
+  """Enables TensorFlow 2.x behaviors.
+
+  This function can be called at the beginning of the program (before `Tensors`,
+  `Graphs` or other structures have been created, and before devices have been
+  initialized. It switches all global behaviors that are different between
+  TensorFlow 1.x and 2.x to behave as intended for 1.x.
+
+  User can call this function to disable 2.x behavior during complex migrations.
+  """
+  tf2.disable()  # Switches TensorArrayV2 and control flow V2
+  ops.disable_eager_execution()
+  tensor_shape.disable_v2_tensorshape()  # Also switched by tf2
+  variable_scope.disable_resource_variables()
+
+
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b5175d3c93..6c1a9c6f73 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5393,7 +5393,7 @@ def inside_function():
   return get_default_graph().building_function
 
 
-@tf_export("enable_eager_execution")
+@tf_export(v1=["enable_eager_execution"])
 def enable_eager_execution(config=None,
                            device_policy=None,
                            execution_mode=None):
@@ -5464,6 +5464,17 @@ def enable_eager_execution(config=None,
         server_def=None)
 
 
+@tf_export(v1=["disable_eager_execution"])
+def disable_eager_execution():
+  """Disables eager execution.
+
+  This function can only be called before any Graphs, Ops, or Tensors have been
+  created. It can be used at the beginning of the program for complex migration
+  projects from TensorFlow 1.x to 2.x.
+  """
+  context.default_execution_mode = context.GRAPH_MODE
+
+
 def enable_eager_execution_internal(config=None,
                                     device_policy=None,
                                     execution_mode=None,
@@ -5471,6 +5482,7 @@ def enable_eager_execution_internal(config=None,
   """Enables eager execution for the lifetime of this program.
 
   Most of the doc string for enable_eager_execution is relevant here as well.
+
   Args:
     config: See enable_eager_execution doc string
     device_policy: See enable_eager_execution doc string
diff --git a/tensorflow/python/tf2.py b/tensorflow/python/tf2.py
index c9782a7119..a8b0d6b7c5 100644
--- a/tensorflow/python/tf2.py
+++ b/tensorflow/python/tf2.py
@@ -25,6 +25,21 @@ from __future__ import print_function
 import os
 
 
+_force_enable = False
+
+
+def enable():
+  """Enables v2 behaviors."""
+  global _force_enable
+  _force_enable = True
+
+
+def disable():
+  """Disables v2 behaviors (TF2_BEHAVIOR env variable is still respected)."""
+  global _force_enable
+  _force_enable = False
+
+
 def enabled():
   """Returns True iff TensorFlow 2.0 behavior should be enabled."""
-  return os.getenv("TF2_BEHAVIOR") is not None
+  return _force_enable or os.getenv("TF2_BEHAVIOR") is not None
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index 490b4a847f..584c74f99d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -1052,10 +1052,18 @@ tf_module {
     name: "dimension_value"
     argspec: "args=[\'dimension\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "disable_eager_execution"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "disable_resource_variables"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "disable_v2_behavior"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "disable_v2_tensorshape"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
@@ -1096,6 +1104,10 @@ tf_module {
     name: "enable_resource_variables"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "enable_v2_behavior"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "enable_v2_tensorshape"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index f5e1545657..cb38ae0b49 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -608,10 +608,6 @@ tf_module {
     name: "einsum"
     argspec: "args=[\'equation\'], varargs=inputs, keywords=kwargs, defaults=None"
   }
-  member_method {
-    name: "enable_eager_execution"
-    argspec: "args=[\'config\', \'device_policy\', \'execution_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "ensure_shape"
     argspec: "args=[\'x\', \'shape\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 62032449a4..b1b5d9e8c4 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -126,7 +126,9 @@ renames = {
     'tf.digamma': 'tf.math.digamma',
     'tf.dimension_at_index': 'tf.compat.v1.dimension_at_index',
     'tf.dimension_value': 'tf.compat.v1.dimension_value',
+    'tf.disable_eager_execution': 'tf.compat.v1.disable_eager_execution',
     'tf.disable_resource_variables': 'tf.compat.v1.disable_resource_variables',
+    'tf.disable_v2_behavior': 'tf.compat.v1.disable_v2_behavior',
     'tf.disable_v2_tensorshape': 'tf.compat.v1.disable_v2_tensorshape',
     'tf.distributions.Bernoulli': 'tf.compat.v1.distributions.Bernoulli',
     'tf.distributions.Beta': 'tf.compat.v1.distributions.Beta',
@@ -147,7 +149,9 @@ renames = {
     'tf.distributions.Uniform': 'tf.compat.v1.distributions.Uniform',
     'tf.distributions.kl_divergence': 'tf.compat.v1.distributions.kl_divergence',
     'tf.div': 'tf.compat.v1.div',
+    'tf.enable_eager_execution': 'tf.compat.v1.enable_eager_execution',
     'tf.enable_resource_variables': 'tf.compat.v1.enable_resource_variables',
+    'tf.enable_v2_behavior': 'tf.compat.v1.enable_v2_behavior',
     'tf.enable_v2_tensorshape': 'tf.compat.v1.enable_v2_tensorshape',
     'tf.encode_base64': 'tf.io.encode_base64',
     'tf.erf': 'tf.math.erf',
-- 
GitLab


From 1053d116fa5d81144e809a4e7846ce4e6606d69d Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 30 Nov 2018 16:40:26 -0800
Subject: [PATCH 1135/1554] [XLA] Convert algebraic_simplifier_test to use
 pattern_matcher.

Refactoring away from HLO matchers.

PiperOrigin-RevId: 223595932
---
 tensorflow/compiler/xla/service/BUILD         |   3 +
 .../xla/service/algebraic_simplifier_test.cc  | 466 +++++++++++-------
 .../compiler/xla/service/pattern_matcher.h    |   4 +
 3 files changed, 287 insertions(+), 186 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 429b4e490c..0907593ce7 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1589,7 +1589,10 @@ tf_cc_test(
         ":hlo",
         ":hlo_casting_utils",
         ":hlo_matchers",
+        ":hlo_parser",
         ":hlo_pass",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 48f689c96a..6ff1c9ab42 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -27,9 +27,11 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
-#include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
@@ -42,8 +44,7 @@ namespace xla {
 namespace {
 
 using ::testing::ElementsAre;
-
-namespace op = xla::testing::opcode_matchers;
+namespace m = match;
 
 AlgebraicSimplifierOptions::ValidBitcastCallback bitcasting_callback() {
   return [](const Shape&, const Shape&) { return true; };
@@ -197,7 +198,7 @@ TEST_F(AlgebraicSimplifierTest, TwoReducesToOne) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   HloInstruction* root = m->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Reduce(param, zero));
+  EXPECT_THAT(root, GmockMatch(m::Reduce(m::Parameter(0), m::Op().Is(zero))));
   EXPECT_EQ(root->dimensions(), std::vector<int64>({0, 2, 3}));
 }
 
@@ -219,7 +220,7 @@ TEST_F(AlgebraicSimplifierTest, AddConstOnLHS) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Add(param0, op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(0), m::Constant())));
 }
 
 // Test that [(A + C1) + C2] => [A + (C1 + C2)] for constants C1 and C2.
@@ -245,7 +246,9 @@ TEST_F(AlgebraicSimplifierTest, AddReassociateMergeConstants) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Add(param0, op::Add(constant1, constant2)));
+  EXPECT_THAT(root, GmockMatch(m::Add(
+                        m::Op().Is(param0),
+                        m::Add(m::Op().Is(constant1), m::Op().Is(constant2)))));
 }
 
 TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR0Operand) {
@@ -303,7 +306,8 @@ TEST_F(AlgebraicSimplifierTest, InlineTrivialMap) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Add(param0, op::Broadcast(zero)));
+  EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(0),
+                                      m::Broadcast(m::Op().Is(zero)))));
 }
 
 TEST_F(AlgebraicSimplifierTest, AddBroadcastZeroR1Operand) {
@@ -336,11 +340,11 @@ TEST_F(AlgebraicSimplifierTest, ConstantToBroadcast) {
 
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Constant())));
   EXPECT_EQ(3.14f, root->operand(0)->literal().GetFirstElement<float>());
 }
 
@@ -352,11 +356,11 @@ TEST_F(AlgebraicSimplifierTest, ConstantNotToBroadcast) {
 
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_FALSE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
 }
 
 TEST_F(AlgebraicSimplifierTest, IotaToBroadcast) {
@@ -367,11 +371,11 @@ TEST_F(AlgebraicSimplifierTest, IotaToBroadcast) {
 
   auto computation = m->AddEntryComputation(builder.Build());
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Iota());
+  EXPECT_THAT(root, GmockMatch(m::Iota()));
 }
 
 // Test that A - 0 is simplified to A
@@ -413,7 +417,8 @@ TEST_F(AlgebraicSimplifierTest, SubConstCanonicalization) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Add(param0, op::Negate(constant)));
+  EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(0),
+                                      m::Negate(m::Op().Is(constant)))));
 }
 
 // Test that (A/B)/C is simplified to A/(B*C).
@@ -435,13 +440,16 @@ TEST_F(AlgebraicSimplifierTest, LhsDivOfDiv) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Divide(op::Divide(param0, param1), param2));
+              GmockMatch(m::Divide(m::Divide(m::Parameter(0), m::Parameter(1)),
+                                   m::Parameter(2))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Divide(param0, op::Multiply(param1, param2)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Divide(m::Parameter(0),
+                           m::Multiply(m::Parameter(1), m::Parameter(2)))));
 }
 
 // Test that A/(B/C) is simplified to (A*C)/B.
@@ -462,14 +470,18 @@ TEST_F(AlgebraicSimplifierTest, RhsDivOfDiv) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Divide(param0, op::Divide(param1, param2)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Divide(m::Parameter(0),
+                           m::Divide(m::Parameter(1), m::Parameter(2)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Divide(op::Multiply(param0, param2), param1));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Divide(m::Multiply(m::Parameter(0), m::Parameter(2)),
+                           m::Parameter(1))));
 }
 
 // Test that (A/B)/(C/D) is simplified to (A*D)/(B*C).
@@ -496,14 +508,16 @@ TEST_F(AlgebraicSimplifierTest, DivOfDivAndDiv) {
 
   EXPECT_THAT(
       computation->root_instruction(),
-      op::Divide(op::Divide(param0, param1), op::Divide(param2, param3)));
+      GmockMatch(m::Divide(m::Divide(m::Parameter(0), m::Parameter(1)),
+                           m::Divide(m::Parameter(2), m::Parameter(3)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(
       computation->root_instruction(),
-      op::Divide(op::Multiply(param0, param3), op::Multiply(param1, param2)));
+      GmockMatch(m::Divide(m::Multiply(m::Parameter(0), m::Parameter(3)),
+                           m::Multiply(m::Parameter(1), m::Parameter(2)))));
 }
 
 // Test that A/exp(B) is simplified to A*exp(-B).
@@ -523,13 +537,14 @@ TEST_F(AlgebraicSimplifierTest, DivOfExp) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Divide(param0, op::Exp(param1)));
+              GmockMatch(m::Divide(m::Parameter(0), m::Exp(m::Parameter(1)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(param0, op::Exp(op::Negate(param1))));
+              GmockMatch(m::Multiply(m::Parameter(0),
+                                     m::Exp(m::Negate(m::Parameter(1))))));
 }
 
 // Test that A/pow(B,C) is simplified to A*pow(B,-C).
@@ -550,14 +565,18 @@ TEST_F(AlgebraicSimplifierTest, DivOfPower) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Divide(param0, op::Power(param1, param2)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Divide(m::Parameter(0),
+                           m::Power(m::Parameter(1), m::Parameter(2)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(param0, op::Power(param1, op::Negate(param2))));
+              GmockMatch(m::Multiply(
+                  m::Parameter(0),
+                  m::Power(m::Parameter(1), m::Negate(m::Parameter(2))))));
 }
 
 // Test that broadcasting is done on the right step when simplifying A/pow(B,C)
@@ -579,14 +598,18 @@ TEST_F(AlgebraicSimplifierTest, DivOfBroadcastingPower) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Divide(param0, op::Power(param1, param2)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Divide(m::Parameter(0),
+                           m::Power(m::Parameter(1), m::Parameter(2)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   ASSERT_THAT(computation->root_instruction(),
-              op::Multiply(param0, op::Power(param1, op::Negate(param2))));
+              GmockMatch(m::Multiply(
+                  m::Parameter(0),
+                  m::Power(m::Parameter(1), m::Negate(m::Parameter(2))))));
 }
 
 // A / Const => A * InvertedConst
@@ -608,7 +631,7 @@ TEST_F(AlgebraicSimplifierTest, DivideByConstant) {
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(param0, op::Constant()));
+              GmockMatch(m::Multiply(m::Parameter(0), m::Constant())));
 }
 
 // pow(pow(A, X), Y) => pow(A, X*Y)
@@ -630,8 +653,10 @@ TEST_F(AlgebraicSimplifierTest, PowerOfPower) {
   auto computation = m->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
-  EXPECT_THAT(computation->root_instruction(),
-              op::Power(base, op::Multiply(exp1, exp2)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Power(m::Op().Is(base),
+                          m::Multiply(m::Op().Is(exp1), m::Op().Is(exp2)))));
 }
 
 // Don't simplify pow(pow(A, X), Y) => pow(A, X*Y) if X and Y are complex
@@ -794,7 +819,7 @@ TEST_F(AlgebraicSimplifierTest, SelectMakeTuple) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Add(param1, param2));
+  EXPECT_THAT(root, GmockMatch(m::Add(m::Parameter(1), m::Parameter(2))));
 }
 
 // Test that exp(A)/exp(B) is simplified to exp(A-B)
@@ -815,14 +840,16 @@ TEST_F(AlgebraicSimplifierTest, ExpDiv) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Divide(op::Exp(param0), op::Exp(param1)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Divide(m::Exp(m::Parameter(0)), m::Exp(m::Parameter(1)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Exp(op::Subtract(param0, param1)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Exp(m::Subtract(m::Parameter(0), m::Parameter(1)))));
 }
 
 // Test that exp(A)*exp(B) is simplified to exp(A+B)
@@ -844,13 +871,14 @@ TEST_F(AlgebraicSimplifierTest, ExpMul) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(op::Exp(param0), op::Exp(param1)));
+              GmockMatch(m::Multiply(m::Exp(m::Parameter(0)),
+                                     m::Exp(m::Parameter(1)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Exp(op::Add(param0, param1)));
+              GmockMatch(m::Exp(m::Add(m::Parameter(0), m::Parameter(1)))));
 }
 
 // Test that pow(exp(A), B) is simplified to exp(A*B)
@@ -870,13 +898,14 @@ TEST_F(AlgebraicSimplifierTest, PowExp) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Power(op::Exp(param0), param1));
+              GmockMatch(m::Power(m::Exp(m::Parameter(0)), m::Parameter(1))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Exp(op::Multiply(param0, param1)));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Exp(m::Multiply(m::Parameter(0), m::Parameter(1)))));
 }
 
 // Test that ln(pow(A, B)) is simplified to ln(A)*B
@@ -896,13 +925,14 @@ TEST_F(AlgebraicSimplifierTest, LnPow) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Log(op::Power(param0, param1)));
+              GmockMatch(m::Log(m::Power(m::Parameter(0), m::Parameter(1)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(),
-              op::Multiply(op::Log(param0), param1));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Multiply(m::Log(m::Parameter(0)), m::Parameter(1))));
 }
 
 // Test that ln(exp(A)) is simplified to A
@@ -919,7 +949,8 @@ TEST_F(AlgebraicSimplifierTest, LnExp) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Log(op::Exp(param0)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Log(m::Exp(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -948,12 +979,14 @@ TEST_F(AlgebraicSimplifierTest, LnExpDiv) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Log(op::Divide(op::Exp(param0), op::Exp(param1))));
+              GmockMatch(m::Log(m::Divide(m::Exp(m::Parameter(0)),
+                                          m::Exp(m::Parameter(1))))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Subtract(param0, param1));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Subtract(m::Parameter(0), m::Parameter(1))));
 }
 
 // Test that pow(A, 0) where A is a scalar is simplified to the scalar
@@ -971,13 +1004,14 @@ TEST_F(AlgebraicSimplifierTest, Pow0Scalar) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Power(param0, zero));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Power(m::Parameter(0), m::Op().Is(zero))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Constant());
+  EXPECT_THAT(root, GmockMatch(m::Constant()));
   EXPECT_EQ(root->literal().GetFirstElement<float>(), 1);
 }
 
@@ -995,13 +1029,14 @@ TEST_F(AlgebraicSimplifierTest, Pow0Vector) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Power(param0, zero));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Power(m::Parameter(0), m::Op().Is(zero))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast());
+  EXPECT_THAT(root, GmockMatch(m::Broadcast()));
   EXPECT_TRUE(ShapeUtil::Equal(root->shape(), r1f32))
       << ShapeUtil::HumanString(root->shape());
   EXPECT_EQ(root->dimensions().size(), 0);
@@ -1023,7 +1058,8 @@ TEST_F(AlgebraicSimplifierTest, Pow1) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Power(param0, one));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Power(m::Parameter(0), m::Op().Is(one))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1045,12 +1081,14 @@ TEST_F(AlgebraicSimplifierTest, Pow2) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Power(param0, two));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Power(m::Parameter(0), m::Op().Is(two))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Multiply(param0, param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Multiply(m::Parameter(0), m::Parameter(0))));
 }
 
 // Test that pow(A, -1) is simplified to 1/A.
@@ -1067,13 +1105,14 @@ TEST_F(AlgebraicSimplifierTest, PowNegative1) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Power(param0, negative_one));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Power(m::Parameter(0), m::Op().Is(negative_one))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   HloInstruction* root = computation->root_instruction();
-  EXPECT_THAT(root, op::Divide(op::Broadcast(), param0));
+  EXPECT_THAT(root, GmockMatch(m::Divide(m::Broadcast(), m::Parameter(0))));
   EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kBroadcast);
   EXPECT_EQ(root->operand(0)->operand(0)->literal().GetFirstElement<float>(),
             1);
@@ -1116,10 +1155,10 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedConvolution) {
   m->AddEntryComputation(builder.Build());
   HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::Convolution(lhs, rhs));
+              GmockMatch(m::Convolution(m::Op().Is(lhs), m::Op().Is(rhs))));
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::Broadcast(op::Constant()));
+              GmockMatch(m::Broadcast(m::Constant())));
 }
 
 TEST_F(AlgebraicSimplifierTest, ZeroSizedReduceWindow) {
@@ -1158,10 +1197,10 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedReduceWindow) {
   m->AddEntryComputation(builder.Build());
   HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::ReduceWindow(param, op::Constant()));
+              GmockMatch(m::ReduceWindow(m::Parameter(0), m::Constant())));
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::Broadcast(op::Constant()));
+              GmockMatch(m::Broadcast(m::Constant())));
 }
 
 TEST_F(AlgebraicSimplifierTest, ZeroSizedPad) {
@@ -1184,11 +1223,11 @@ TEST_F(AlgebraicSimplifierTest, ZeroSizedPad) {
       padding));
   m->AddEntryComputation(builder.Build());
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::Pad(param, op::Constant()));
+              GmockMatch(m::Pad(m::Parameter(0), m::Constant())));
   HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::Broadcast(op::Constant()));
+              GmockMatch(m::Broadcast(m::Constant())));
 }
 
 TEST_F(AlgebraicSimplifierTest, ReshapeBroadcast) {
@@ -1209,7 +1248,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeBroadcast) {
   m->AddEntryComputation(std::move(computation));
 
   EXPECT_THAT(m->entry_computation()->root_instruction(),
-              op::Reshape(op::Broadcast(op::Reshape(op))));
+              GmockMatch(m::Reshape(m::Broadcast(m::Reshape(m::Op().Is(op))))));
 
   HloPassFix<AlgebraicSimplifier> simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1228,7 +1267,8 @@ TEST_F(AlgebraicSimplifierTest, ConvertBetweenSameType) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Convert(input));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Convert(m::Op().Is(input))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1248,7 +1288,8 @@ TEST_F(AlgebraicSimplifierTest, RemoveCopy) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1269,21 +1310,24 @@ TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
   *copy->mutable_shape()->mutable_layout() =
       LayoutUtil::MakeLayout({1, 2, 0, 3});
   auto computation = m->AddEntryComputation(builder.Build());
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options(non_bitcasting_callback());
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier1(options);
   ASSERT_FALSE(simplifier1.Run(m.get()).ValueOrDie());
   // Verify that the copy is not replaced.
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options2(bitcasting_callback());
   options2.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier2(options2);
   ASSERT_TRUE(simplifier2.Run(m.get()).ValueOrDie());
   // Verify that the copy is replaced.
-  EXPECT_THAT(computation->root_instruction(), op::Bitcast(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
 }
 
 // Test that unary concatenates are removed.
@@ -1298,7 +1342,8 @@ TEST_F(AlgebraicSimplifierTest, RemoveUnaryConcatenate) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Concatenate(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Concatenate(m::Parameter(0))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1327,15 +1372,17 @@ TEST_F(AlgebraicSimplifierTest, RemoveEmptyConcatenateOperands) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(
-      computation->root_instruction(),
-      op::Concatenate(empty_literal, param0, param0, empty_slice, param1));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Concatenate(
+                  m::Op().Is(empty_literal), m::Parameter(0), m::Parameter(0),
+                  m::Op().Is(empty_slice), m::Parameter(1))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Concatenate(param0, param0, param1));
+              GmockMatch(m::Concatenate(m::Parameter(0), m::Parameter(0),
+                                        m::Parameter(1))));
 }
 
 // Test that reduce of concat is simplified.
@@ -1383,8 +1430,9 @@ TEST_F(AlgebraicSimplifierTest, SimplifyReduceOfConcat) {
 
   EXPECT_THAT(
       computation->root_instruction(),
-      op::Map(op::Map(op::Reduce(param0, zero), op::Reduce(param1, zero)),
-              op::Reduce(param2, zero)));
+      GmockMatch(m::Map(m::Map(m::Reduce(m::Parameter(0), m::Op().Is(zero)),
+                               m::Reduce(m::Parameter(1), m::Op().Is(zero))),
+                        m::Reduce(m::Parameter(2), m::Op().Is(zero)))));
 }
 
 // Test a concatenate with only empty operands is removed.
@@ -1407,7 +1455,8 @@ TEST_F(AlgebraicSimplifierTest, OnlyEmptyConcatenateOperands) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Concatenate(empty_literal, empty_slice));
+              GmockMatch(m::Concatenate(m::Op().Is(empty_literal),
+                                        m::Op().Is(empty_slice))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1434,7 +1483,8 @@ TEST_F(AlgebraicSimplifierTest, ConcatenateOfBroadcastBecomesPad) {
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
-  EXPECT_THAT(computation->root_instruction(), op::Pad(param0, param1));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Pad(m::Parameter(0), m::Parameter(1))));
 }
 
 TEST_F(AlgebraicSimplifierTest, SimplifyConcatenateOfSlices) {
@@ -1495,10 +1545,10 @@ TEST_F(AlgebraicSimplifierTest, SimplifyConcatenateOfSlices) {
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  auto s = m::Slice(m::Parameter(0));
   EXPECT_THAT(
       computation->root_instruction(),
-      op::Concatenate(op::Slice(param0), op::Slice(param0), op::Slice(param0),
-                      op::Slice(param0), op::Slice(param0), op::Slice(param1)));
+      GmockMatch(m::Concatenate(s, s, s, s, s, m::Slice(m::Parameter(1)))));
   // The operand 3 should be a merge of 'slice3', 'slice4' and 'slice5', so its
   // shape should have dimensions {50, 30}.
   EXPECT_TRUE(
@@ -1524,7 +1574,8 @@ TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) {
   *param0->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({0, 1});
   *copy->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({1, 0});
 
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options(non_bitcasting_callback());
   options.set_is_layout_sensitive(true);
@@ -1532,7 +1583,8 @@ TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) {
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   // Copy has not been removed.
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 }
 
 // Test that a simplification which preserves layouts is performed if layout
@@ -1552,7 +1604,8 @@ TEST_F(AlgebraicSimplifierTest, CopyWithSameLayout) {
   *param0->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({0, 1});
   *copy->mutable_shape()->mutable_layout() = LayoutUtil::MakeLayout({0, 1});
 
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options(non_bitcasting_callback());
   options.set_is_layout_sensitive(true);
@@ -1581,7 +1634,8 @@ TEST_F(AlgebraicSimplifierTest, NoBitcastAdded) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options(non_bitcasting_callback());
   options.set_is_layout_sensitive(true);
@@ -1589,7 +1643,8 @@ TEST_F(AlgebraicSimplifierTest, NoBitcastAdded) {
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   // Reshape is not replaced with a bitcast.
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Parameter(0))));
 }
 
 // Test transforming reshapes and transposes of rng.
@@ -1619,7 +1674,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeOfTransposeOfRngToRng) {
 
   // Verify that that reshape(transpose(rng)) is replace by a single rng of the
   // same shape as the reshape.
-  EXPECT_THAT(computation->root_instruction(), op::Rng());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Rng()));
   EXPECT_TRUE(ShapeUtil::Equal(computation->root_instruction()->shape(),
                                reshape_shape));
 }
@@ -1661,8 +1716,9 @@ TEST_F(AlgebraicSimplifierTest, ReshapeReplacedWithBitcast) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Tuple(transformable_reshape, dimensions_wrong_reshape,
-                        layout_wrong_reshape));
+              GmockMatch(m::Tuple(m::Op().Is(transformable_reshape),
+                                  m::Op().Is(dimensions_wrong_reshape),
+                                  m::Op().Is(layout_wrong_reshape))));
 
   AlgebraicSimplifierOptions options(bitcasting_callback());
   options.set_is_layout_sensitive(true);
@@ -1672,7 +1728,8 @@ TEST_F(AlgebraicSimplifierTest, ReshapeReplacedWithBitcast) {
   // Verify that only the first reshape is replaced.
   EXPECT_THAT(
       computation->root_instruction(),
-      op::Tuple(op::Bitcast(), dimensions_wrong_reshape, layout_wrong_reshape));
+      GmockMatch(m::Tuple(m::Bitcast(), m::Op().Is(dimensions_wrong_reshape),
+                          m::Op().Is(layout_wrong_reshape))));
 }
 
 // Regression test for a bug where if we failed to sink a reshape, we'd set the
@@ -1741,7 +1798,8 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Transpose(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Transpose(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options(bitcasting_callback());
   options.set_is_layout_sensitive(true);
@@ -1749,7 +1807,8 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) {
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that the reshape is replaced.
-  EXPECT_THAT(computation->root_instruction(), op::Bitcast(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
 }
 
 TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) {
@@ -1769,7 +1828,8 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Transpose(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Transpose(m::Parameter(0))));
 
   AlgebraicSimplifierOptions options(bitcasting_callback());
   options.set_is_layout_sensitive(true);
@@ -1777,7 +1837,8 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) {
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that the reshape is replaced.
-  EXPECT_THAT(computation->root_instruction(), op::Bitcast(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
 }
 
 TEST_F(AlgebraicSimplifierTest, ReshapesMerged) {
@@ -1797,12 +1858,13 @@ TEST_F(AlgebraicSimplifierTest, ReshapesMerged) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Reshape(param0)));
+              GmockMatch(m::Reshape(m::Reshape(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Parameter(0))));
 }
 
 TEST_F(AlgebraicSimplifierTest, CopiesMerged) {
@@ -1823,14 +1885,16 @@ TEST_F(AlgebraicSimplifierTest, CopiesMerged) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Copy(op::Copy(param0)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Copy(m::Parameter(0)))));
 
   AlgebraicSimplifierOptions options(non_bitcasting_callback());
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Copy(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Parameter(0))));
 }
 
 TEST_F(AlgebraicSimplifierTest, TransposesMerged) {
@@ -1849,12 +1913,14 @@ TEST_F(AlgebraicSimplifierTest, TransposesMerged) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Transpose(transpose1));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Transpose(m::Op().Is(transpose1))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Transpose(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Transpose(m::Parameter(0))));
   EXPECT_EQ(std::vector<int64>({2, 1, 0}),
             computation->root_instruction()->dimensions());
 }
@@ -1873,12 +1939,13 @@ TEST_F(AlgebraicSimplifierTest, ReshapeAndBroadcastMerged) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Broadcast(op::Reshape(param0)));
+              GmockMatch(m::Broadcast(m::Reshape(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Broadcast(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Broadcast(m::Parameter(0))));
 }
 
 // Test merging broadcast and reshape.
@@ -1895,12 +1962,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshapeMerged) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param0)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Broadcast(param0));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Broadcast(m::Parameter(0))));
 }
 
 TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x1_3) {
@@ -1916,13 +1984,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x1_3) {
   auto computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 }
 
 TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4_6x1x1x4) {
@@ -1938,12 +2006,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4_6x1x1x4) {
   HloComputation* computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Broadcast(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Broadcast(m::Parameter(0))));
   EXPECT_THAT(computation->root_instruction()->dimensions(),
               ::testing::ElementsAre(3));
 }
@@ -1961,12 +2030,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_1_3x2x1_6x1x1x1) {
   HloComputation* computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Broadcast(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Broadcast(m::Parameter(0))));
   const std::vector<int64> broadcast_dims =
       computation->root_instruction()->dimensions();
   EXPECT_EQ(1, broadcast_dims.size());
@@ -1986,13 +2056,13 @@ TEST_F(AlgebraicSimplifierTest, BroadcastAndReshape_4_3x2x4x2_6x8) {
   HloComputation* computation = m->AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
-              op::Reshape(op::Broadcast(param)));
+              GmockMatch(m::Reshape(m::Broadcast(m::Parameter(0)))));
 }
 
 TEST_F(AlgebraicSimplifierTest, IotaAndReshapeMerged) {
@@ -2005,12 +2075,13 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshapeMerged) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Iota());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota()));
   EXPECT_TRUE(
       ShapeUtil::Equal(computation->root_instruction()->shape(), result_shape));
 }
@@ -2024,13 +2095,13 @@ TEST_F(AlgebraicSimplifierTest, IotaEffectiveScalar) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Iota());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota()));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   auto root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Constant())));
   EXPECT_EQ(0.0f, root->operand(0)->literal().GetFirstElement<float>());
   EXPECT_TRUE(
       ShapeUtil::Equal(computation->root_instruction()->shape(), result_shape));
@@ -2046,12 +2117,14 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_1_3x2_6) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 
   AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 }
 
 TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4_6x1x1x4) {
@@ -2064,12 +2137,13 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4_6x1x1x4) {
 
   HloComputation* computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Iota());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota()));
   EXPECT_EQ(Cast<HloIotaInstruction>(computation->root_instruction())
                 ->iota_dimension(),
             3);
@@ -2085,12 +2159,13 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_1_3x2x2_6x1x1x2) {
 
   HloComputation* computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Iota());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Iota()));
   const int64 iota_dim =
       Cast<HloIotaInstruction>(computation->root_instruction())
           ->iota_dimension();
@@ -2107,12 +2182,14 @@ TEST_F(AlgebraicSimplifierTest, IotaAndReshape_4_3x2x4x2_6x8) {
 
   HloComputation* computation = m->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 
   AlgebraicSimplifier simplifier(default_options_);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Iota()));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Iota())));
 }
 
 TEST_F(AlgebraicSimplifierTest, RemoveNoopPad) {
@@ -2135,7 +2212,8 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopPad) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
@@ -2179,12 +2257,14 @@ TEST_F(AlgebraicSimplifierTest, NegativePadding) {
     return false;
   };
 
-  EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero))));
   EXPECT_TRUE(has_negative_padding(pad));
 
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Slice(op::Pad(param, zero)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Slice(m::Pad(m::Parameter(0), m::Op().Is(zero)))));
   EXPECT_FALSE(
       has_negative_padding(computation->root_instruction()->operand(0)));
 }
@@ -2213,12 +2293,14 @@ TEST_F(AlgebraicSimplifierTest, TrivialInteriorPadding) {
 
   AlgebraicSimplifier simplifier(default_options_);
 
-  ASSERT_THAT(computation->root_instruction(), op::Pad(param, zero));
+  ASSERT_THAT(computation->root_instruction(),
+              GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero))));
   ASSERT_TRUE(HasInteriorPadding(pad->padding_config()));
 
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Pad(param, zero));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Pad(m::Parameter(0), m::Op().Is(zero))));
   EXPECT_FALSE(
       HasInteriorPadding(computation->root_instruction()->padding_config()));
 }
@@ -2234,7 +2316,8 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopReshape) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Parameter(0))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
@@ -2256,7 +2339,8 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSlice) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Slice(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Slice(m::Parameter(0))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
@@ -2284,12 +2368,14 @@ TEST_F(AlgebraicSimplifierTest, SliceOfSliceToSlice) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Slice(op::Slice(param)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Slice(m::Slice(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Slice(param));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Slice(m::Parameter(0))));
   EXPECT_EQ(computation->root_instruction()->slice_starts(0), 3);
   EXPECT_EQ(computation->root_instruction()->slice_starts(1), 5);
   EXPECT_EQ(computation->root_instruction()->slice_limits(0), dim0 - 2);
@@ -2315,12 +2401,14 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeToReshapeOfSlice) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Slice(op::Reshape(param)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Slice(m::Reshape(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
 
-  EXPECT_THAT(computation->root_instruction(), op::Reshape(op::Slice(param)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Slice(m::Parameter(0)))));
 }
 
 TEST_F(AlgebraicSimplifierTest, SliceOfReshapeUnchanged) {
@@ -2339,7 +2427,8 @@ TEST_F(AlgebraicSimplifierTest, SliceOfReshapeUnchanged) {
   auto module = CreateNewVerifiedModule();
   HloComputation* computation = module->AddEntryComputation(builder.Build());
 
-  EXPECT_THAT(computation->root_instruction(), op::Slice(op::Reshape(param)));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Slice(m::Reshape(m::Parameter(0)))));
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie());
@@ -2380,10 +2469,10 @@ TEST_F(AlgebraicSimplifierTest, ReplacePermutationSortWithScatter) {
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
   EXPECT_THAT(root,
-              op::Tuple(op::Iota(),
-                        op::Scatter(op::Iota(),
-                                    op::Concatenate(op::Iota(), op::Reshape()),
-                                    op::Reshape())));
+              GmockMatch(m::Tuple(
+                  m::Iota(),
+                  m::Scatter(m::Iota(), m::Concatenate(m::Iota(), m::Reshape()),
+                             m::Reshape()))));
 }
 
 TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortIfNonIntegral) {
@@ -2451,7 +2540,8 @@ TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
-              op::Tuple(keys, values0, values1));
+              GmockMatch(m::Tuple(m::Op().Is(keys), m::Op().Is(values0),
+                                  m::Op().Is(values1))));
 }
 
 // Test that A && True is simplified to A
@@ -2753,7 +2843,8 @@ TEST_P(ConvInputPaddingTest, DoTest) {
     ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
     auto* conv = module->entry_computation()->root_instruction();
     SCOPED_TRACE(module->ToString());
-    ASSERT_THAT(conv, op::Convolution(op::Parameter(), op::Parameter()));
+    ASSERT_THAT(conv,
+                GmockMatch(m::Convolution(m::Parameter(), m::Parameter())));
     EXPECT_EQ(window_util::ToString(conv->window()),
               absl::StrCat("size=3x3 ", testcase.expected_conv_window));
   }
@@ -2870,7 +2961,8 @@ TEST_P(ConvFilterPaddingTest, DoIt) {
     ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
     auto* conv = module->entry_computation()->root_instruction();
     SCOPED_TRACE(module->ToString());
-    ASSERT_THAT(conv, op::Convolution(op::Parameter(), op::Parameter()));
+    ASSERT_THAT(conv,
+                GmockMatch(m::Convolution(m::Parameter(), m::Parameter())));
     EXPECT_EQ(window_util::ToString(conv->window()),
               absl::StrFormat("size=%dx%d %s",
                               conv->operand(1)->shape().dimensions(2),
@@ -3142,10 +3234,9 @@ TEST_F(AlgebraicSimplifierTest, ScalarBroadcastToSlice) {
 
   // Running simplification again should not result in any further changes.
   ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie());
-
-  root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(scalar_param));
-  EXPECT_TRUE(ShapeUtil::Equal(root->shape(), slice_shape));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Broadcast(m::Op().Is(scalar_param))
+                             .WithShapeEqualTo(&slice_shape)));
 }
 
 // Test that reshape(transpose(broadcast(/*scalar value*/))) simplifies to a
@@ -3176,10 +3267,9 @@ TEST_F(AlgebraicSimplifierTest, ScalarBroadcastToTransposeReshape) {
 
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
-
-  root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(forty_two));
-  EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reshape_shape));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Broadcast(m::Op().Is(forty_two))
+                             .WithShapeEqualTo(&reshape_shape)));
 }
 
 // Test that ReduceWindow(Pad(op, x), y) can simplify to ReduceWindow(op, x).
@@ -3248,7 +3338,8 @@ TEST_F(AlgebraicSimplifierTest, FoldPadIntoReduceWindow) {
 
   // Verify the result
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::ReduceWindow(operand, op::Constant()));
+  EXPECT_THAT(root,
+              GmockMatch(m::ReduceWindow(m::Op().Is(operand), m::Constant())));
   EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reduce_window_shape))
       << ShapeUtil::HumanString(root->shape()) << " vs "
       << ShapeUtil::HumanString(reduce_window_shape);
@@ -3333,7 +3424,8 @@ TEST_F(AlgebraicSimplifierTest, FoldConvertedPadIntoReduceWindow) {
 
   // Verify the result
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::ReduceWindow(op::Convert(parameter), op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::ReduceWindow(m::Convert(m::Parameter(0)),
+                                               m::Constant())));
   EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reduce_window_shape))
       << ShapeUtil::HumanString(root->shape()) << " vs "
       << ShapeUtil::HumanString(reduce_window_shape);
@@ -3414,7 +3506,7 @@ TEST_F(AlgebraicSimplifierTest, ConstantTupleBecomesTupleOfConstants) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
-              op::Tuple(op::Constant(), op::Constant()));
+              GmockMatch(m::Tuple(m::Constant(), m::Constant())));
 }
 
 // A dynamic-slice is trivial if its start indices are all zeroes and the size
@@ -3436,7 +3528,7 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicSlice) {
   auto computation = m->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
-  EXPECT_THAT(computation->root_instruction(), op::Parameter());
+  EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Parameter()));
 }
 
 // A dynamic-update-slice is trivial if its start indices are all zeroes and the
@@ -3470,7 +3562,7 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
-              op::DynamicSlice(op::Parameter(), op::Parameter()));
+              GmockMatch(m::DynamicSlice(m::Parameter(), m::Parameter())));
 }
 
 // Test that two consecutive broadcasts can be merged to one.
@@ -3492,7 +3584,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcasts) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Constant())));
   EXPECT_THAT(root->dimensions(), ElementsAre(2));
 }
 
@@ -3518,7 +3610,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcasts2) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(op::Parameter(0)));
+  EXPECT_THAT(root, GmockMatch(m::Broadcast(m::Parameter(0))));
   EXPECT_THAT(root->dimensions(), ElementsAre(1, 3));
 }
 
@@ -3538,7 +3630,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcastAndIota) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Iota());
+  EXPECT_THAT(root, GmockMatch(m::Iota()));
   EXPECT_EQ(Cast<HloIotaInstruction>(root)->iota_dimension(), 2);
 }
 
@@ -3559,7 +3651,7 @@ TEST_F(AlgebraicSimplifierTest, MergeBroadcastAndIota2) {
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   root = computation->root_instruction();
-  EXPECT_THAT(root, op::Iota());
+  EXPECT_THAT(root, GmockMatch(m::Iota()));
   EXPECT_EQ(Cast<HloIotaInstruction>(root)->iota_dimension(), 2);
 }
 
@@ -3581,7 +3673,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadLow) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Reshape(op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::Reshape(m::Constant())));
 }
 
 TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) {
@@ -3602,7 +3694,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Reshape(op::Constant()));
+  EXPECT_THAT(root, GmockMatch(m::Reshape(m::Constant())));
 }
 
 TEST_F(AlgebraicSimplifierTest, SliceOfPadMidNonScalar) {
@@ -3642,7 +3734,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidScalar) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Parameter());
+  EXPECT_THAT(root, GmockMatch(m::Parameter()));
 }
 
 TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) {
@@ -3664,7 +3756,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Parameter(1));
+  EXPECT_THAT(root, GmockMatch(m::Parameter(1)));
 }
 
 TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) {
@@ -3686,7 +3778,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Slice(op::Parameter(2)));
+  EXPECT_THAT(root, GmockMatch(m::Slice(m::Parameter(2))));
   EXPECT_EQ(root->slice_starts(0), 1);
   EXPECT_EQ(root->slice_limits(0), 2);
 }
@@ -3708,7 +3800,7 @@ TEST_F(AlgebraicSimplifierTest, NegateNegate) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Parameter(0));
+  EXPECT_THAT(root, GmockMatch(m::Parameter(0)));
 }
 
 TEST_F(AlgebraicSimplifierTest, NotNot) {
@@ -3728,7 +3820,7 @@ TEST_F(AlgebraicSimplifierTest, NotNot) {
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Parameter(0));
+  EXPECT_THAT(root, GmockMatch(m::Parameter(0)));
 }
 
 struct PadReduceWindowEffectiveBroadcastCase {
@@ -3832,10 +3924,10 @@ TEST_P(PadReduceWindowEffectiveBroadcastTest, DoIt) {
       ShapeUtil::Equal(computation->root_instruction()->shape(), output_shape));
 
   if (param.should_become_broadcast) {
-    EXPECT_THAT(computation->root_instruction(), op::Broadcast(::testing::_));
+    EXPECT_THAT(computation->root_instruction(), GmockMatch(m::Broadcast()));
   } else {
     EXPECT_THAT(computation->root_instruction(),
-                op::ReduceWindow(::testing::_, zero));
+                GmockMatch(m::ReduceWindow(m::Op(), m::Op().Is(zero))));
   }
 }
 
@@ -3989,11 +4081,12 @@ TEST_P(DotOfConcatSimplificationTest, ConstantLHS) {
   EXPECT_TRUE(
       ShapeUtil::Equal(computation->root_instruction()->shape(), dot_shape));
 
-  auto match_dot_0 = op::Dot(op::Slice(op::Constant()), op::Parameter(0));
-  auto match_dot_1 = op::Dot(op::Slice(op::Constant()), op::Parameter(1));
-  auto match_dot_2 = op::Dot(op::Slice(op::Constant()), op::Parameter(2));
-  EXPECT_THAT(computation->root_instruction(),
-              op::Add(op::Add(match_dot_0, match_dot_1), match_dot_2));
+  auto match_dot_0 = m::Dot(m::Slice(m::Constant()), m::Parameter(0));
+  auto match_dot_1 = m::Dot(m::Slice(m::Constant()), m::Parameter(1));
+  auto match_dot_2 = m::Dot(m::Slice(m::Constant()), m::Parameter(2));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Add(m::Add(match_dot_0, match_dot_1), match_dot_2)));
 }
 
 // Test that we transform
@@ -4052,13 +4145,14 @@ TEST_P(DotOfConcatSimplificationTest, ConstantRHS) {
   EXPECT_TRUE(
       ShapeUtil::Equal(computation->root_instruction()->shape(), dot_shape));
 
-  auto match_dot_0 = op::Dot(op::Parameter(0), op::Slice(op::Constant()));
-  auto match_dot_1 = op::Dot(op::Parameter(1), op::Slice(op::Constant()));
-  auto match_dot_2 = op::Dot(op::Parameter(2), op::Slice(op::Constant()));
-  auto match_dot_3 = op::Dot(op::Parameter(3), op::Slice(op::Constant()));
-  EXPECT_THAT(computation->root_instruction(),
-              op::Add(op::Add(op::Add(match_dot_0, match_dot_1), match_dot_2),
-                      match_dot_3));
+  auto match_dot_0 = m::Dot(m::Parameter(0), m::Slice(m::Constant()));
+  auto match_dot_1 = m::Dot(m::Parameter(1), m::Slice(m::Constant()));
+  auto match_dot_2 = m::Dot(m::Parameter(2), m::Slice(m::Constant()));
+  auto match_dot_3 = m::Dot(m::Parameter(3), m::Slice(m::Constant()));
+  EXPECT_THAT(
+      computation->root_instruction(),
+      GmockMatch(m::Add(m::Add(m::Add(match_dot_0, match_dot_1), match_dot_2),
+                        match_dot_3)));
 }
 
 DotOfConcatTestSpec kDotOfConcatTestSpecs[] = {
@@ -4175,8 +4269,8 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) {
               HloOpcode::kDynamicSlice);
   } else {
     EXPECT_THAT(computation->root_instruction(),
-                op::DynamicSlice(op::Dot(op::Constant(), op::Constant()),
-                                 op::Concatenate()));
+                GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()),
+                                           m::Concatenate())));
   }
 }
 
@@ -4245,8 +4339,8 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) {
               HloOpcode::kDynamicSlice);
   } else {
     EXPECT_THAT(computation->root_instruction(),
-                op::DynamicSlice(op::Dot(op::Constant(), op::Constant()),
-                                 op::Concatenate()));
+                GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()),
+                                           m::Concatenate())));
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index fb1645d9b2..54d7b52a3e 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -1922,6 +1922,7 @@ Op(::xla::HloInstruction** matched_inst) {
 XLA_NULLOP_PATTERN(Constant)
 XLA_NULLOP_PATTERN(Parameter)
 XLA_NULLOP_PATTERN(Iota)
+XLA_NULLOP_PATTERN(Rng)
 #undef XLA_NULLOP_PATTERN
 
 // Helpers for unary instructions.
@@ -2053,6 +2054,7 @@ XLA_COMMUTATIVE_BINOP_PATTERN(Ne)
 XLA_BINOP_PATTERN(Outfeed)
 XLA_BINOP_PATTERN(Pad)
 XLA_BINOP_PATTERN(Power)
+XLA_BINOP_PATTERN(ReduceWindow)
 XLA_BINOP_PATTERN(Remainder)
 XLA_BINOP_PATTERN(Send)
 XLA_BINOP_PATTERN(Subtract)
@@ -2099,6 +2101,7 @@ XLA_BINOP_PATTERN(ShiftRightLogical)
         .WithOperand(2, std::forward<Arg2>(arg2));                     \
   }
 XLA_TERNOP_PATTERN(Clamp);
+XLA_TERNOP_PATTERN(Scatter);
 XLA_TERNOP_PATTERN(Select);
 #undef XLA_TERNOP_PATTERN
 
@@ -2153,6 +2156,7 @@ inline auto WithOperands(Matcher&& m, int64 operand_num, FirstArg&& first_arg,
 // already-bad compile errors even worse.
 XLA_VARIADIC_OP_PATTERN(Concatenate);
 XLA_VARIADIC_OP_PATTERN(CustomCall);
+XLA_VARIADIC_OP_PATTERN(Map)
 XLA_VARIADIC_OP_PATTERN(Reduce);
 XLA_VARIADIC_OP_PATTERN(Tuple);
 
-- 
GitLab


From 236ed9a095baa0131f9256869a64ca6cfc286b99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 16:41:38 -0800
Subject: [PATCH 1136/1554] Add comment to DistributedDelegate explaining why
 it has to define each overloaded operator separately.

PiperOrigin-RevId: 223596114
---
 tensorflow/python/distribute/values.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index f3f7676f04..81787445e7 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -100,6 +100,15 @@ class DistributedValues(object):
   # DistributionStrategy implementations.
 
 
+# NOTE(josh11b,apassos): It would be great if we could inspect the values this was
+# initialized with and use that to generate the overloaded operators here.
+# Unfortunately, Python's rules for special methods don't allow this, see
+# https://docs.python.org/3/reference/datamodel.html#special-method-names
+# "if a class defines a method named __getitem__(), and x is an instance of
+# this class, then x[i] is roughly equivalent to type(x).__getitem__(x, i)."
+# In particular, these special methods don't go through __getattr__, and
+# it will only use those methods if they are defined in the class, not the
+# object.
 class DistributedDelegate(DistributedValues):
   """A map from device to values; acts as the same type as the values."""
 
-- 
GitLab


From 594044db0505faa4ca41edb1d394d7c643c026ef Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 30 Nov 2018 16:42:19 -0800
Subject: [PATCH 1137/1554] Internal change.

PiperOrigin-RevId: 223596219
---
 .../windows/cpu/pip/build_tf_windows.sh       | 21 ++++++++++++++++---
 .../windows/gpu/pip/build_tf_windows.sh       | 21 ++++++++++++++++---
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 5990caa50c..46f5bdef09 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -58,7 +58,8 @@ PY_TEST_DIR="py_test_dir"
 SKIP_TEST=0
 RELEASE_BUILD=0
 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
-EXTRA_BUILD_FLAGS=${EXTRA_BUILD_FLAGS:-}
+PROJECT_NAME=""
+EXTRA_BUILD_FLAGS=""
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
@@ -74,6 +75,20 @@ for ARG in "$@"; do
     --release_build) RELEASE_BUILD=1 ;;
     --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;;
     --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;;
+    --extra_build_flags)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      EXTRA_BUILD_FLAGS="$1"
+      ;;
+    --project_name)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      PROJECT_NAME="$1"
+      ;;
     *)
   esac
 done
@@ -89,10 +104,10 @@ fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   python tensorflow/tools/ci_build/update_version.py --nightly
-  if [ -z ${EXTRA_PIP_FLAGS} ]; then
+  if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
-    EXTRA_PIP_FLAGS="${EXTRA_PIP_FLAGS} --nightly_flag"
+    EXTRA_PIP_FLAGS="--project_name=${PROJECT_NAME} --nightly_flag"
   fi
 fi
 
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 1bca2a6f88..3aec8d6584 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -58,7 +58,8 @@ PY_TEST_DIR="py_test_dir"
 SKIP_TEST=0
 RELEASE_BUILD=0
 TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
-EXTRA_BUILD_FLAGS=${EXTRA_BUILD_FLAGS:-}
+PROJECT_NAME=""
+EXTRA_BUILD_FLAGS=""
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
@@ -74,6 +75,20 @@ for ARG in "$@"; do
     --release_build) RELEASE_BUILD=1 ;;
     --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;;
     --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;;
+    --extra_build_flags)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      EXTRA_BUILD_FLAGS="$1"
+      ;;
+    --project_name)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      PROJECT_NAME="$1"
+      ;;
     *)
   esac
 done
@@ -89,10 +104,10 @@ fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   python tensorflow/tools/ci_build/update_version.py --nightly
-  if [ -z ${EXTRA_PIP_FLAGS} ]; then
+  if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
-    EXTRA_PIP_FLAGS="${EXTRA_PIP_FLAGS} --nightly_flag"
+    EXTRA_PIP_FLAGS="--project_name=${PROJECT_NAME} --nightly_flag"
   fi
 fi
 
-- 
GitLab


From 568712e56a8b1ad70c712fd41dba454b255ff6fb Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Fri, 30 Nov 2018 16:43:26 -0800
Subject: [PATCH 1138/1554] [tf.data] Fix MaxIntraOpParallelismDatasetOp's
 AsGraphDefInternal

PiperOrigin-RevId: 223596355
---
 .../experimental/threadpool_dataset_op.cc     |  4 +-
 .../kernel_tests/override_threadpool_test.py  | 38 +++++++++++--------
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 335f2b7a4b..287ca6abc6 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -248,7 +248,7 @@ class MaxIntraOpParallelismDatasetOp : public UnaryDatasetOpKernel {
   class Dataset : public DatasetBase {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
-            int max_intra_op_parallelism)
+            int64 max_intra_op_parallelism)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           max_intra_op_parallelism_(max_intra_op_parallelism) {
@@ -332,7 +332,7 @@ class MaxIntraOpParallelismDatasetOp : public UnaryDatasetOpKernel {
     };
 
     const DatasetBase* const input_;
-    const int max_intra_op_parallelism_;
+    const int64 max_intra_op_parallelism_;
   };
 };
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index 7116e7549f..de2788dfcb 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -22,6 +22,7 @@ import threading
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.core.framework import graph_pb2
 from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.experimental.ops import unique
@@ -55,21 +56,20 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
     iterator = dataset.make_initializable_iterator()
     next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
-      self.evaluate(iterator.initializer)
-      thread_ids = []
-      try:
-        while True:
-          thread_ids.append(self.evaluate(next_element))
-      except errors.OutOfRangeError:
-        pass
-      self.assertLen(thread_ids, len(set(thread_ids)))
-      self.assertNotEmpty(thread_ids)
-      if num_threads:
-        # NOTE(mrry): We don't control the thread pool scheduling, and
-        # so cannot guarantee that all of the threads in the pool will
-        # perform work.
-        self.assertLessEqual(len(thread_ids), num_threads)
+    self.evaluate(iterator.initializer)
+    thread_ids = []
+    try:
+      while True:
+        thread_ids.append(self.evaluate(next_element))
+    except errors.OutOfRangeError:
+      pass
+    self.assertLen(thread_ids, len(set(thread_ids)))
+    self.assertNotEmpty(thread_ids)
+    if num_threads:
+      # NOTE(mrry): We don't control the thread pool scheduling, and
+      # so cannot guarantee that all of the threads in the pool will
+      # perform work.
+      self.assertLessEqual(len(thread_ids), num_threads)
 
   @parameterized.named_parameters(
       ("1", 1, None),
@@ -124,6 +124,14 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
 
     self._testNumThreadsHelper(num_threads, override_threadpool_fn)
 
+  def testMaxIntraOpParallelismAsGraphDefInternal(self):
+    dataset = dataset_ops.Dataset.from_tensors(0)
+    dataset = dataset_ops._MaxIntraOpParallelismDataset(dataset, 1)
+    graph = graph_pb2.GraphDef().FromString(
+        self.evaluate(dataset._as_serialized_graph()))
+    self.assertTrue(
+        any([node.op != "MaxIntraOpParallelismDataset" for node in graph.node]))
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 4f3d18dc2fa32e9714a1da3118a1c0402809315c Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 30 Nov 2018 16:58:50 -0800
Subject: [PATCH 1139/1554] Add a deprecation warning to Saver when executing
 eagerly

This is probably long overdue. Saver has lots of subtle bugs which make it unpleasant.

PiperOrigin-RevId: 223598061
---
 tensorflow/contrib/eager/python/saver.py |  5 +++++
 tensorflow/python/training/saver.py      | 20 ++++++++++++++++----
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py
index f9c716360c..1d0d6c6c14 100644
--- a/tensorflow/contrib/eager/python/saver.py
+++ b/tensorflow/contrib/eager/python/saver.py
@@ -115,6 +115,11 @@ def restore_variables_on_create(save_path, map_func=None):
 
 class Saver(object):
   """A tf.train.Saver adapter for use when eager execution is enabled.
+
+  `Saver`'s name-based checkpointing strategy is fragile. Please switch to
+  `tf.train.Checkpoint` or `tf.keras.Model.save_weights`, which perform a more
+  robust object-based saving. These APIs will load checkpoints written by
+  `Saver`.
   """
 
   def __init__(self, var_list):
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index e76500e776..4cd09f8a1d 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1077,16 +1077,28 @@ class Saver(object):
     @compatibility(eager)
     When eager execution is enabled, `var_list` must specify a `list` or `dict`
     of variables to save. Otherwise, a `RuntimeError` will be raised.
+
+    Although Saver works in some cases when executing eagerly, it is
+    fragile. Please switch to `tf.train.Checkpoint` or
+    `tf.keras.Model.save_weights`, which perform a more robust object-based
+    saving. These APIs will load checkpoints written by `Saver`.
     @end_compatibility
     """
     if defer_build and var_list:
       raise ValueError(
           "If `var_list` is provided then build cannot be deferred. "
           "Either set defer_build=False or var_list=None.")
-    if context.executing_eagerly() and var_list is None:
-      raise RuntimeError(
-          "When eager execution is enabled, `var_list` must specify a list or "
-          "dict of variables to save")
+    if context.executing_eagerly():
+      logging.warning(
+          "Saver is deprecated, please switch to tf.train.Checkpoint or "
+          "tf.keras.Model.save_weights for training checkpoints. When "
+          "executing eagerly variables do not necessarily have unique names, "
+          "and so the variable.name-based lookups Saver performs are "
+          "error-prone.")
+      if var_list is None:
+        raise RuntimeError(
+            "When eager execution is enabled, `var_list` must specify a list "
+            "or dict of variables to save")
     self._var_list = var_list
     self._reshape = reshape
     self._sharded = sharded
-- 
GitLab


From a3ccedb38eff4eab5f2954358bec29f1712e265c Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 30 Nov 2018 17:19:29 -0800
Subject: [PATCH 1140/1554] Change softmax_cross_entropy_with_logits_v2 to use
 axis instead of dim

PiperOrigin-RevId: 223600271
---
 tensorflow/python/ops/nn_ops.py               | 84 ++++++++++++++++---
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  2 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |  5 ++
 .../tools/compatibility/tf_upgrade_v2_test.py |  7 ++
 5 files changed, 85 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 74e88b7653..91d292ab69 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -36,13 +36,14 @@ from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
-
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_nn_ops import *
 # pylint: enable=wildcard-import
-
 from tensorflow.python.util import deprecation
+from tensorflow.python.util.deprecation import deprecated_args
+from tensorflow.python.util.deprecation import deprecated_argument_lookup
+
 from tensorflow.python.util.tf_export import tf_export
 
 # Aliases for some automatically-generated names.
@@ -2308,9 +2309,8 @@ def _ensure_xent_args(name, sentinel, labels, logits):
     raise ValueError("Both labels and logits must be provided.")
 
 
-@tf_export("nn.softmax_cross_entropy_with_logits",
-           v1=["nn.softmax_cross_entropy_with_logits_v2"])
-def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
+@tf_export("nn.softmax_cross_entropy_with_logits", v1=[])
+def softmax_cross_entropy_with_logits_v2(labels, logits, axis=-1, name=None):
   """Computes softmax cross entropy between `logits` and `labels`.
 
   Measures the probability error in discrete classification tasks in which the
@@ -2332,7 +2332,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
 
   A common use case is to have logits and labels of shape
   `[batch_size, num_classes]`, but higher dimensions are supported, with
-  the `dim` argument specifying the class dimension.
+  the `axis` argument specifying the class dimension.
 
   `logits` and `labels` must have the same dtype (either `float16`, `float32`,
   or `float64`).
@@ -2350,8 +2350,64 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
       `[batch_size, num_classes]`, each row of `labels[i]` must be a valid
       probability distribution.
     logits: Unscaled log probabilities.
-    dim: The class dimension. Defaulted to -1 which is the last dimension.
+    axis: The class dimension. Defaulted to -1 which is the last dimension.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` that contains the softmax cross entropy loss. Its type is the
+    same as `logits` and its shape is the same as `labels` except that it does
+    not have the last dimension of `labels`.
+  """
+  return softmax_cross_entropy_with_logits_v2_helper(
+      labels=labels, logits=logits, axis=axis, name=name)
+
+
+@tf_export(v1=["nn.softmax_cross_entropy_with_logits_v2"])
+@deprecated_args(None, "dim is deprecated, use axis instead", "dim")
+def softmax_cross_entropy_with_logits_v2_helper(
+    labels, logits, axis=-1, name=None, dim=None):
+  """Computes softmax cross entropy between `logits` and `labels`.
+
+  Measures the probability error in discrete classification tasks in which the
+  classes are mutually exclusive (each entry is in exactly one class).  For
+  example, each CIFAR-10 image is labeled with one and only one label: an image
+  can be a dog or a truck, but not both.
+
+  **NOTE:**  While the classes are mutually exclusive, their probabilities
+  need not be.  All that is required is that each row of `labels` is
+  a valid probability distribution.  If they are not, the computation of the
+  gradient will be incorrect.
+
+  If using exclusive `labels` (wherein one and only
+  one class is true at a time), see `sparse_softmax_cross_entropy_with_logits`.
+
+  **WARNING:** This op expects unscaled logits, since it performs a `softmax`
+  on `logits` internally for efficiency.  Do not call this op with the
+  output of `softmax`, as it will produce incorrect results.
+
+  A common use case is to have logits and labels of shape
+  `[batch_size, num_classes]`, but higher dimensions are supported, with
+  the `axis` argument specifying the class dimension.
+
+  `logits` and `labels` must have the same dtype (either `float16`, `float32`,
+  or `float64`).
+
+  Backpropagation will happen into both `logits` and `labels`.  To disallow
+  backpropagation into `labels`, pass label tensors through `tf.stop_gradient`
+  before feeding it to this function.
+
+  **Note that to avoid confusion, it is required to pass only named arguments to
+  this function.**
+
+  Args:
+    labels: Each vector along the class dimension should hold a valid
+      probability distribution e.g. for the case in which labels are of shape
+      `[batch_size, num_classes]`, each row of `labels[i]` must be a valid
+      probability distribution.
+    logits: Unscaled log probabilities.
+    axis: The class dimension. Defaulted to -1 which is the last dimension.
     name: A name for the operation (optional).
+    dim: Deprecated alias for axis.
 
   Returns:
     A `Tensor` that contains the softmax cross entropy loss. Its type is the
@@ -2361,6 +2417,8 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
   # TODO(pcmurray) Raise an error when the labels do not sum to 1. Note: This
   # could break users who call this with bad labels, but disregard the bad
   # results.
+  axis = deprecated_argument_lookup("axis", axis, "dim", dim)
+  del dim
 
   with ops.name_scope(name, "softmax_cross_entropy_with_logits",
                       [logits, labels]) as name:
@@ -2377,7 +2435,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
     shape = logits.get_shape()
 
     # Move the dim to the end if dim is not the last dimension.
-    if dim is not -1:
+    if axis != -1:
 
       def _move_dim_to_end(tensor, dim_index, rank):
         return array_ops.transpose(
@@ -2387,8 +2445,8 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
                 math_ops.range(dim_index + 1, rank), [dim_index]
             ], 0))
 
-      precise_logits = _move_dim_to_end(precise_logits, dim, input_rank)
-      labels = _move_dim_to_end(labels, dim, input_rank)
+      precise_logits = _move_dim_to_end(precise_logits, axis, input_rank)
+      labels = _move_dim_to_end(labels, axis, input_rank)
 
     input_shape = array_ops.shape(precise_logits)
 
@@ -2402,7 +2460,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
     cost, unused_backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
         precise_logits, labels, name=name)
 
-    # The output cost shape should be the input minus dim.
+    # The output cost shape should be the input minus axis.
     output_shape = array_ops.slice(input_shape, [0],
                                    [math_ops.subtract(input_rank, 1)])
     cost = array_ops.reshape(cost, output_shape)
@@ -2412,7 +2470,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, dim=-1, name=None):
     if not context.executing_eagerly(
     ) and shape is not None and shape.dims is not None:
       shape = shape.as_list()
-      del shape[dim]
+      del shape[axis]
       cost.set_shape(shape)
 
     if convert_to_float32:
@@ -2490,7 +2548,7 @@ def softmax_cross_entropy_with_logits(
     labels = array_ops.stop_gradient(labels, name="labels_stop_gradient")
 
   return softmax_cross_entropy_with_logits_v2(
-      labels=labels, logits=logits, dim=dim, name=name)
+      labels=labels, logits=logits, axis=dim, name=name)
 
 
 @tf_export("nn.sparse_softmax_cross_entropy_with_logits")
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 48501e1b58..560b492fe9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -326,7 +326,7 @@ tf_module {
   }
   member_method {
     name: "softmax_cross_entropy_with_logits_v2"
-    argspec: "args=[\'labels\', \'logits\', \'dim\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], "
+    argspec: "args=[\'labels\', \'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\', \'None\'], "
   }
   member_method {
     name: "softplus"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 4779448760..f0c92ca83c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -262,7 +262,7 @@ tf_module {
   }
   member_method {
     name: "softmax_cross_entropy_with_logits"
-    argspec: "args=[\'labels\', \'logits\', \'dim\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], "
+    argspec: "args=[\'labels\', \'logits\', \'axis\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\'], "
   }
   member_method {
     name: "softplus"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 992080025d..53a4875d5e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -56,6 +56,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
+        "tf.nn.softmax_cross_entropy_with_logits_v2": {
+            "dim": "axis"
+        },
         "tf.linalg.l2_normalize": {
             "dim": "axis",
         },
@@ -442,6 +445,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.data.Iterator",
         "tf.nn.fused_batch_norm":
             "tf.compat.v1.nn.fused_batch_norm",
+        "tf.nn.softmax_cross_entropy_with_logits_v2":
+            "tf.nn.softmax_cross_entropy_with_logits",
         "tf.losses.Reduction.MEAN":
             "tf.compat.v1.losses.Reduction.MEAN",
         "tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 6cf870595a..1512f0bc36 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -437,6 +437,13 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testSoftMaxCrossEntropyWithLogitsV2(self):
+    text = "tf.nn.softmax_cross_entropy_with_logits_v2(labels, logits, dim=2)"
+    expected_text = (
+        "tf.nn.softmax_cross_entropy_with_logits(labels, logits, axis=2)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 8561c63f323c9bc6d34343d57fa9cb152014cf40 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 30 Nov 2018 17:38:33 -0800
Subject: [PATCH 1141/1554] Internal change.

PiperOrigin-RevId: 223602111
---
 tensorflow/python/ops/nn_impl.py              | 106 +++++++++++++++++-
 tensorflow/python/ops/nn_test.py              |  10 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |   2 +-
 .../tools/compatibility/tf_upgrade_v2.py      |   3 +
 4 files changed, 113 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 26bd13eaee..8f74f831c1 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -1462,7 +1462,111 @@ def _compute_sampled_logits(weights,
     return out_logits, out_labels
 
 
-@tf_export("nn.nce_loss")
+@tf_export("nn.nce_loss", v1=[])
+def nce_loss_v2(weights,
+                biases,
+                labels,
+                inputs,
+                num_sampled,
+                num_classes,
+                num_true=1,
+                sampled_values=None,
+                remove_accidental_hits=False,
+                name="nce_loss"):
+  """Computes and returns the noise-contrastive estimation training loss.
+
+  See [Noise-contrastive estimation: A new estimation principle for
+  unnormalized statistical
+  models](http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf).
+  Also see our [Candidate Sampling Algorithms
+  Reference](https://www.tensorflow.org/extras/candidate_sampling.pdf)
+
+  A common use case is to use this method for training, and calculate the full
+  sigmoid loss for evaluation or inference as in the following example:
+
+  ```python
+  if mode == "train":
+    loss = tf.nn.nce_loss(
+        weights=weights,
+        biases=biases,
+        labels=labels,
+        inputs=inputs,
+        ...)
+  elif mode == "eval":
+    logits = tf.matmul(inputs, tf.transpose(weights))
+    logits = tf.nn.bias_add(logits, biases)
+    labels_one_hot = tf.one_hot(labels, n_classes)
+    loss = tf.nn.sigmoid_cross_entropy_with_logits(
+        labels=labels_one_hot,
+        logits=logits)
+    loss = tf.reduce_sum(loss, axis=1)
+  ```
+
+  Note: when doing embedding lookup on `weights` and `bias`, "div" partition
+  strategy will be used. Support for other partition strategy will be added
+  later.
+
+  Note: By default this uses a log-uniform (Zipfian) distribution for sampling,
+  so your labels must be sorted in order of decreasing frequency to achieve
+  good results.  For more details, see
+  `tf.nn.log_uniform_candidate_sampler`.
+
+  Note: In the case where `num_true` > 1, we assign to each target class
+  the target probability 1 / `num_true` so that the target probabilities
+  sum to 1 per-example.
+
+  Note: It would be useful to allow a variable number of target classes per
+  example.  We hope to provide this functionality in a future release.
+  For now, if you have a variable number of target classes, you can pad them
+  out to a constant number by either repeating them or by padding
+  with an otherwise unused class.
+
+  Args:
+    weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor`
+      objects whose concatenation along dimension 0 has shape [num_classes,
+      dim].  The (possibly-partitioned) class embeddings.
+    biases: A `Tensor` of shape `[num_classes]`.  The class biases.
+    labels: A `Tensor` of type `int64` and shape `[batch_size, num_true]`. The
+      target classes.
+    inputs: A `Tensor` of shape `[batch_size, dim]`.  The forward activations of
+      the input network.
+    num_sampled: An `int`.  The number of negative classes to randomly sample
+      per batch. This single sample of negative classes is evaluated for each
+      element in the batch.
+    num_classes: An `int`. The number of possible classes.
+    num_true: An `int`.  The number of target classes per training example.
+    sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`,
+      `sampled_expected_count`) returned by a `*_candidate_sampler` function.
+      (if None, we default to `log_uniform_candidate_sampler`)
+    remove_accidental_hits:  A `bool`.  Whether to remove "accidental hits"
+      where a sampled class equals one of the target classes.  If set to `True`,
+      this is a "Sampled Logistic" loss instead of NCE, and we are learning to
+      generate log-odds instead of log probabilities.  See our [Candidate
+      Sampling Algorithms Reference]
+        (https://www.tensorflow.org/extras/candidate_sampling.pdf). Default is
+          False.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `batch_size` 1-D tensor of per-example NCE losses.
+  """
+  # TODO(yuefengz): get partition_strategy from either variables or distribution
+  # strategies.
+  return nce_loss(
+      weights,
+      biases,
+      labels,
+      inputs,
+      num_sampled,
+      num_classes,
+      num_true=num_true,
+      sampled_values=sampled_values,
+      remove_accidental_hits=remove_accidental_hits,
+      partition_strategy="div",
+      name=name)
+
+
+@tf_export(v1=["nn.nce_loss"])
 def nce_loss(weights,
              biases,
              labels,
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index a7ed834c72..92ca9deb1b 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -790,7 +790,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     exp_nce_loss = np.sum(
         _SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1)
 
-    got_nce_loss = nn_impl.nce_loss(
+    got_nce_loss = nn_impl.nce_loss_v2(
         weights=constant_op.constant(weights),
         biases=constant_op.constant(biases),
         labels=constant_op.constant(labels, shape=(batch_size, 1)),
@@ -798,15 +798,14 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
         num_sampled=4,
         num_classes=num_classes,
         num_true=1,
-        sampled_values=sampled_vals,
-        partition_strategy="div")
+        sampled_values=sampled_vals)
 
     self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
 
     # Test with sharded weights and sharded biases.
     weight_shards, bias_shards = self._ShardTestEmbeddings(
         weights, biases, num_shards=3)
-    got_nce_loss = nn_impl.nce_loss(
+    got_nce_loss = nn_impl.nce_loss_v2(
         weights=[constant_op.constant(shard) for shard in weight_shards],
         biases=[constant_op.constant(shard) for shard in bias_shards],
         labels=constant_op.constant(labels, shape=(batch_size, 1)),
@@ -814,8 +813,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
         num_sampled=4,
         num_classes=num_classes,
         num_true=1,
-        sampled_values=sampled_vals,
-        partition_strategy="div")
+        sampled_values=sampled_vals)
 
     self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4)
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index f0c92ca83c..63bf24b5d5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -210,7 +210,7 @@ tf_module {
   }
   member_method {
     name: "nce_loss"
-    argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'partition_strategy\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'False\', \'mod\', \'nce_loss\'], "
+    argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'False\', \'nce_loss\'], "
   }
   member_method {
     name: "normalize_moments"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 53a4875d5e..4fb2a3f4d6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -745,6 +745,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "was renamed to \"filters\"",
         "tf.nn.erosion2d":
         "WARNING: <function name> now requires a data_format argument",
+        "tf.nn.nce_loss":
+        "WARNING: `partition_strategy` has been removed from `tf.nn.nce_loss` "
+        " The 'div' strategy is used by default.",
         "tf.zeros_like": tf_01s_like_no_optimize_comment,
         "tf.ones_like": tf_01s_like_no_optimize_comment,
     }
-- 
GitLab


From fedf3f1c2affa16127080ff979b216f0b7963d7d Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 30 Nov 2018 17:40:25 -0800
Subject: [PATCH 1142/1554] Update MirroredStrategy for its new constructor
 without cluster resolver.

PiperOrigin-RevId: 223602268
---
 .../python/collective_all_reduce_strategy.py  |  38 ++-
 .../collective_all_reduce_strategy_test.py    |   3 +-
 .../python/cross_device_ops_test.py           |   8 +-
 .../python/estimator_training_test.py         |  40 ++-
 .../distribute/python/mirrored_strategy.py    |  24 +-
 .../python/mirrored_strategy_multigpu_test.py |  40 +--
 .../python/distribute/mirrored_strategy.py    | 279 ++++++++++++------
 7 files changed, 264 insertions(+), 168 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 6e9f9facd0..e988b63a28 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -67,10 +67,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   def __init__(self, container_strategy, num_gpus_per_worker):
     distribute_lib.DistributionStrategyExtended.__init__(
         self, container_strategy)
+    self._cross_device_ops = None
     self._num_gpus_per_worker = num_gpus_per_worker
-    self._initialize_local_worker(container_strategy, num_gpus_per_worker)
+    self._initialize_local_worker(num_gpus_per_worker)
 
-  def _initialize_local_worker(self, container_strategy, num_gpus_per_worker):
+  def _initialize_local_worker(self, num_gpus_per_worker):
     """Initializes the object for local training."""
     self._is_chief = True
     self._num_workers = 1
@@ -84,13 +85,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     self._worker_device = device_util.canonicalize("/device:CPU:0")
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
-    super(CollectiveAllReduceExtended, self).__init__(
-        container_strategy,
-        devices=local_devices,
-        cross_device_ops=cross_device_ops_lib.CollectiveAllReduce(
-            num_workers=1,
-            num_gpus_per_worker=num_gpus_per_worker,
-            collective_keys=self._collective_keys))
+    self._initialize_local(local_devices)
+    self._cross_tower_ops = cross_device_ops_lib.CollectiveAllReduce(
+        num_workers=self._num_workers,
+        num_gpus_per_worker=num_gpus_per_worker,
+        collective_keys=self._collective_keys)
 
     self._cluster_spec = None
     self._task_type = None
@@ -99,8 +98,8 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     logging.info("CollectiveAllReduceStrategy with local_devices = %r",
                  local_devices)
 
-  def _initialize_multi_worker(self, container_strategy, num_gpus_per_worker,
-                               cluster_spec, task_type, task_id):
+  def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec,
+                               task_type, task_id):
     """Initializes the object for multi-worker training."""
     if task_type is None or task_id is None:
       raise ValueError("When `cluster_spec` is given, you must also specify "
@@ -128,13 +127,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
       local_devices = [self._worker_device]
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
-    super(CollectiveAllReduceExtended, self).__init__(
-        container_strategy,
-        devices=local_devices,
-        cross_device_ops=cross_device_ops_lib.CollectiveAllReduce(
-            num_workers=self._num_workers,
-            num_gpus_per_worker=num_gpus_per_worker,
-            collective_keys=self._collective_keys))
+    self._initialize_local(local_devices)
+    self._cross_tower_ops = cross_device_ops_lib.CollectiveAllReduce(
+        num_workers=self._num_workers,
+        num_gpus_per_worker=num_gpus_per_worker,
+        collective_keys=self._collective_keys)
 
     # Add a default device so that ops without specified devices will not end up
     # on other workers.
@@ -268,9 +265,8 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
       # If a `cluster_spec` is already passed in, do nothing here.
       # TODO(yuefengz): check `cluster_spec` is the same if this object has
       # already been initialized with a `cluster_spec`.
-      self._initialize_multi_worker(
-          self._container_strategy(), self._num_gpus_per_worker, cluster_spec,
-          task_type, task_id)
+      self._initialize_multi_worker(self._num_gpus_per_worker, cluster_spec,
+                                    task_type, task_id)
 
     if session_config:
       session_config.CopyFrom(self._update_config_proto(session_config))
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index eba3585a55..8a9e583f0a 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -82,7 +82,8 @@ class CollectiveAllReduceStrategyTestBase(
         instance_key_with_id_start=num_gpus * 10000 +
         CollectiveAllReduceStrategyTestBase.collective_key_base)
     distribution.extended._collective_keys = collective_keys
-    distribution.extended._cross_device_ops._collective_keys = collective_keys
+    distribution.extended._inferred_cross_device_ops._collective_keys = (
+        collective_keys)
     if task_type and task_id is not None:
       return distribution, 'grpc://' + self._cluster_spec[task_type][
           task_id], session_config
diff --git a/tensorflow/contrib/distribute/python/cross_device_ops_test.py b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
index 3602cc9209..d6e9521c1c 100644
--- a/tensorflow/contrib/distribute/python/cross_device_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_device_ops_test.py
@@ -392,18 +392,16 @@ class MultiWorkerCrossDeviceOpsTest(multi_worker_test_base.MultiWorkerTestBase,
           # pylint: disable=g-long-lambda
           combinations.NamedDistribution(
               "CoreMirroredCPU",
-              lambda: mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=0),
+              lambda: mirrored_strategy.CoreMirroredStrategy(["/device:CPU:0"]),
               required_gpus=0),
           combinations.NamedDistribution(
               "CoreMirrored1GPU",
-              lambda: mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=1),
+              lambda: mirrored_strategy.CoreMirroredStrategy(["/device:GPU:0"]),
               required_gpus=1),
           combinations.NamedDistribution(
               "CoreMirrored2GPUs",
               lambda: mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=2),
+                  ["/device:GPU:0", "/device:GPU:1"]),
               required_gpus=2),
       ],
       mode=["graph"])
diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 8c0f594202..b369a7fefe 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -248,6 +248,12 @@ class DistributeCoordinatorIntegrationTest(
     ])
     self.assertAllEqual((BATCH_SIZE, LABEL_DIMENSION), predicted_proba.shape)
 
+  def _get_strategy_object(self, strategy_cls):
+    if strategy_cls == mirrored_strategy.CoreMirroredStrategy:
+      return strategy_cls(mirrored_strategy.all_local_devices())
+    else:
+      return strategy_cls(num_gpus_per_worker=context.num_gpus())
+
   @combinations.generate(
       combinations.combine(
           mode=["graph"],
@@ -266,12 +272,10 @@ class DistributeCoordinatorIntegrationTest(
           required_gpus=[0, 1]))
   def test_complete_flow_standalone_client(self, train_distribute_cls,
                                            eval_distribute_cls):
-    train_distribute = train_distribute_cls(
-        num_gpus_per_worker=context.num_gpus())
+    train_distribute = self._get_strategy_object(train_distribute_cls)
 
     if eval_distribute_cls:
-      eval_distribute = eval_distribute_cls(
-          num_gpus_per_worker=context.num_gpus())
+      eval_distribute = self._get_strategy_object(eval_distribute_cls)
     else:
       eval_distribute = None
 
@@ -298,12 +302,10 @@ class DistributeCoordinatorIntegrationTest(
           required_gpus=[0, 1]))
   def test_estimator_standalone_client(self, train_distribute_cls,
                                        eval_distribute_cls):
-    train_distribute = train_distribute_cls(
-        num_gpus_per_worker=context.num_gpus())
+    train_distribute = self._get_strategy_object(train_distribute_cls)
 
     if eval_distribute_cls:
-      eval_distribute = eval_distribute_cls(
-          num_gpus_per_worker=context.num_gpus())
+      eval_distribute = self._get_strategy_object(eval_distribute_cls)
     else:
       eval_distribute = None
 
@@ -347,16 +349,14 @@ class DistributeCoordinatorIntegrationTest(
           required_gpus=[0, 1]))
   def test_complete_flow_indepedent_worker_between_graph(
       self, train_distribute_cls, eval_distribute_cls):
-    train_distribute = train_distribute_cls(
-        num_gpus_per_worker=context.num_gpus())
-
     if (context.num_gpus() < 2 and eval_distribute_cls ==
         collective_all_reduce_strategy.CollectiveAllReduceStrategy):
       self.skipTest("`CollectiveAllReduceStrategy` needs at least two towers.")
 
+    train_distribute = self._get_strategy_object(train_distribute_cls)
+
     if eval_distribute_cls:
-      eval_distribute = eval_distribute_cls(
-          num_gpus_per_worker=context.num_gpus())
+      eval_distribute = self._get_strategy_object(eval_distribute_cls)
     else:
       eval_distribute = None
 
@@ -399,12 +399,10 @@ class DistributeCoordinatorIntegrationTest(
           required_gpus=[0, 1]))
   def test_complete_flow_indepedent_worker_in_graph(self, train_distribute_cls,
                                                     eval_distribute_cls):
-    train_distribute = train_distribute_cls(
-        num_gpus_per_worker=context.num_gpus())
+    train_distribute = self._get_strategy_object(train_distribute_cls)
 
     if eval_distribute_cls:
-      eval_distribute = eval_distribute_cls(
-          num_gpus_per_worker=context.num_gpus())
+      eval_distribute = self._get_strategy_object(eval_distribute_cls)
     else:
       eval_distribute = None
 
@@ -453,7 +451,7 @@ class RunConfigTest(test.TestCase):
       run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               train_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=2)))
+                  ["/device:GPU:0", "/device:GPU:1"])))
 
   def test_should_run_distribute_coordinator(self):
     """Tests that should_run_distribute_coordinator return a correct value."""
@@ -477,11 +475,11 @@ class RunConfigTest(test.TestCase):
       config_with_train_distribute = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               train_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=2)))
+                  ["/device:GPU:0", "/device:GPU:1"])))
       config_with_eval_distribute = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               eval_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=2)))
+                  ["/device:GPU:0", "/device:GPU:1"])))
     self.assertTrue(
         dc_training.should_run_distribute_coordinator(
             config_with_train_distribute))
@@ -495,7 +493,7 @@ class RunConfigTest(test.TestCase):
       config = run_config_lib.RunConfig(
           experimental_distribute=DistributeConfig(
               train_distribute=mirrored_strategy.CoreMirroredStrategy(
-                  num_gpus_per_worker=2)))
+                  ["/device:GPU:0", "/device:GPU:1"])))
     self.assertFalse(dc_training.should_run_distribute_coordinator(config))
 
   def test_init_run_config_duplicate_distribute(self):
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 4a594f056e..20f1a08d42 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -30,6 +30,7 @@ from tensorflow.python.distribute import values
 _call_for_each_replica = mirrored_strategy._call_for_each_replica
 _reduce_non_distributed_value = mirrored_strategy._reduce_non_distributed_value
 _create_mirrored_variable = mirrored_strategy._create_mirrored_variable
+all_local_devices = mirrored_strategy.all_local_devices
 CoreMirroredStrategy = mirrored_strategy.MirroredStrategy
 CoreMirroredExtended = mirrored_strategy.MirroredExtended
 # pylint: enable=protected-access,invalid-name
@@ -115,8 +116,13 @@ class MirroredExtended(CoreMirroredExtended):
                num_gpus_per_worker=None,
                cross_device_ops=None,
                auto_shard_dataset=False):
-    super(MirroredExtended, self).__init__(
-        container_strategy, devices, num_gpus_per_worker, cross_device_ops)
+    if devices is None:
+      devices = mirrored_strategy.all_local_devices(num_gpus_per_worker)
+    elif num_gpus_per_worker is not None:
+      raise ValueError(
+          "Must only specify one of `devices` and `num_gpus_per_worker`.")
+    super(MirroredExtended, self).__init__(container_strategy, devices,
+                                           cross_device_ops)
     self._auto_shard_dataset = auto_shard_dataset
 
   def _make_dataset_iterator(self, dataset):
@@ -131,22 +137,22 @@ class MirroredExtended(CoreMirroredExtended):
     Returns:
       An `InputIterator` which returns inputs for each step of the computation.
     """
-    if self._cluster_spec:
-      worker_device_pairs = self._worker_devices
-    else:
+    if self._local_mode:
       worker = device_util.canonicalize("/device:CPU:0")
       worker_device_pairs = [(worker, self._devices)]
+    else:
+      worker_device_pairs = self._worker_devices
     return values.DatasetIterator(dataset, worker_device_pairs)
 
   def _distribute_dataset(self, dataset_fn):
-    if self._cluster_spec:
+    if self._local_mode:
+      return values.PerReplicaDataset(
+          self._call_dataset_fn(dataset_fn), self._devices)
+    else:
       return values.MultiWorkerDataset(
           functools.partial(self._call_dataset_fn, dataset_fn),
           self._worker_devices,
           auto_shard=self._auto_shard_dataset)
-    else:
-      return values.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._devices)
 
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index fee37daa42..69c49114a6 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -757,21 +757,23 @@ class MirroredStrategyNameScopeTest(test.TestCase):
       self.assertEqual("c/replica_1:0", c1.name)
 
 
-@combinations.generate(combinations.combine(
-    distribution=[
-        combinations.NamedDistribution(
-            "Mirrored3Devices",
-            # pylint: disable=g-long-lambda
-            lambda: mirrored_strategy.MirroredStrategy(
-                ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]),
-            required_gpus=2),
-        combinations.NamedDistribution(
-            "CoreMirrored3Devices",
-            # pylint: disable=g-long-lambda
-            lambda: mirrored_strategy.CoreMirroredStrategy(
-                ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]),
-            required_gpus=2)],
-    mode=["graph", "eager"]))
+@combinations.generate(
+    combinations.combine(
+        distribution=[
+            combinations.NamedDistribution(
+                "Mirrored3Devices",
+                # pylint: disable=g-long-lambda
+                lambda: mirrored_strategy.MirroredStrategy(
+                    ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]),
+                required_gpus=2),
+            combinations.NamedDistribution(
+                "CoreMirrored3Devices",
+                # pylint: disable=g-long-lambda
+                lambda: mirrored_strategy.CoreMirroredStrategy(
+                    ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"]),
+                required_gpus=2)
+        ],
+        mode=["graph", "eager"]))
 class MirroredThreeDeviceDistributionTest(
     strategy_test_lib.DistributionTestBase,
     parameterized.TestCase):
@@ -1283,14 +1285,14 @@ class MirroredStrategyDefunTest(test.TestCase):
             combinations.NamedDistribution(
                 "Mirrored",
                 # pylint: disable=g-long-lambda
-                lambda: mirrored_strategy.CoreMirroredStrategy(
-                    num_gpus_per_worker=context.num_gpus()),
+                lambda: mirrored_strategy.MirroredStrategy(num_gpus_per_worker=
+                                                           context.num_gpus()),
                 required_gpus=1),
             combinations.NamedDistribution(
                 "CoreMirrored",
                 # pylint: disable=g-long-lambda
                 lambda: mirrored_strategy.CoreMirroredStrategy(
-                    num_gpus_per_worker=context.num_gpus()),
+                    mirrored_strategy.all_local_devices()),
                 required_gpus=1)
         ],
         mode=["graph"]))
@@ -1374,7 +1376,7 @@ class MultiWorkerMirroredStrategyTestWithChief(
 
   def testMinimizeLossGraphCoreMirroredStrategy(self):
     strategy = mirrored_strategy.CoreMirroredStrategy(
-        num_gpus_per_worker=context.num_gpus())
+        mirrored_strategy.all_local_devices())
     strategy.configure(cluster_spec=self._cluster_spec)
     self._test_minimize_loss_graph(strategy, learning_rate=0.05)
 
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index d6d40df5ce..81d25cc559 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -296,6 +296,132 @@ def _create_mirrored_variable(devices, real_mirrored_creator, *args, **kwargs):
   return result
 
 
+def _is_device_list_local(devices):
+  """Checks whether the devices list is for local or multi-worker.
+
+  Args:
+    devices: a list of device strings, either local for remote devices.
+
+  Returns:
+    a boolean indicating whether these device strings are for local or for
+    remote.
+
+  Raises:
+    ValueError: if device strings are not consistent.
+  """
+  all_local = None
+  for d in devices:
+    d_spec = tf_device.DeviceSpec().parse_from_string(d)
+    is_local = d_spec.job in (None, "localhost")
+
+    if all_local is None:  # Determine all_local from first device.
+      all_local = is_local
+
+    if all_local:
+      if not is_local:
+        raise ValueError("Local device string cannot have job specified other "
+                         "than 'localhost'")
+    else:
+      if is_local:
+        raise ValueError("Remote device string must have job specified.")
+      if d_spec.task is None:
+        raise ValueError("Remote device string must have task specified.")
+  return all_local
+
+
+def _cluster_spec_to_device_list(cluster_spec, num_gpus_per_worker):
+  """Returns a device list given a cluster spec."""
+  cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
+  devices = []
+  for task_type in ("chief", "worker"):
+    for task_id in range(len(cluster_spec.as_dict().get(task_type, []))):
+      if num_gpus_per_worker is 0:
+        devices.append("/job:%s/task:%d" % (task_type, task_id))
+      else:
+        devices.extend([
+            "/job:%s/task:%d/device:GPU:%i" % (task_type, task_id, gpu_id)
+            for gpu_id in range(num_gpus_per_worker)
+        ])
+  return devices
+
+
+def _group_device_list(devices):
+  """Groups the devices list by task_type and task_id.
+
+  Args:
+    devices: a list of device strings for remote devices.
+
+  Returns:
+    a dict of list of device strings mapping from task_type to a list of devices
+    for the task_type in the asceding order of task_id.
+  """
+  assert not _is_device_list_local(devices)
+  device_dict = {}
+
+  for d in devices:
+    d_spec = tf_device.DeviceSpec().parse_from_string(d)
+
+    # Create an entry for the task_type.
+    if d_spec.job not in device_dict:
+      device_dict[d_spec.job] = []
+
+    # Fill the device list for task_type until it covers the task_id.
+    while len(device_dict[d_spec.job]) <= d_spec.task:
+      device_dict[d_spec.job].append([])
+
+    device_dict[d_spec.job][d_spec.task].append(d)
+
+  return device_dict
+
+
+def _infer_num_gpus_per_worker(devices):
+  """Infers the number of GPUs on each worker.
+
+  Currently to make multi-worker cross device ops work, we need all workers to
+  have the same number of GPUs.
+
+  Args:
+    devices: a list of device strings, can be either local devices or remote
+      devices.
+
+  Returns:
+    number of GPUs per worker.
+
+  Raises:
+    ValueError if workers have different number of GPUs or GPU indices are not
+    consecutive and starting from 0.
+  """
+  if _is_device_list_local(devices):
+    return len([d for d in devices if "GPU" in d.upper()])
+  else:
+    device_dict = _group_device_list(devices)
+    num_gpus = None
+    for _, devices_in_task in device_dict.items():
+      for device_in_task in devices_in_task:
+        if num_gpus is None:
+          num_gpus = len([d for d in device_in_task if "GPU" in d.upper()])
+
+        # Verify other workers have the same number of GPUs.
+        elif (
+            num_gpus != len([d for d in device_in_task if "GPU" in d.upper()])):
+          raise ValueError("All workers should have the same number of GPUs.")
+
+        for d in device_in_task:
+          d_spec = tf_device.DeviceSpec().parse_from_string(d)
+          if (d_spec.device_type.upper() == "GPU" and
+              d_spec.device_index >= num_gpus):
+            raise ValueError("Device_index on a worker should be consecutive "
+                             "and start from 0.")
+    return num_gpus
+
+
+def all_local_devices(num_gpus=None):
+  if num_gpus is None:
+    num_gpus = context.num_gpus()
+  return (tuple("/device:GPU:%d" % i for i in range(num_gpus)) or
+          ("/device:CPU:0",))
+
+
 class MirroredStrategy(distribute_lib.DistributionStrategy):
   """Mirrors vars to distribute across multiple devices and machines.
 
@@ -306,51 +432,40 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
 
   Args:
     devices: a list of device strings.
-    num_gpus_per_worker: number of GPUs per worker.
     cross_device_ops: optional, a descedant of `CrossDeviceOps`. If this is not
       set, nccl will be use by default.
   """
 
-  def __init__(self,
-               devices=None,
-               num_gpus_per_worker=None,
-               cross_device_ops=None):
-    extended = MirroredExtended(self, devices, num_gpus_per_worker,
-                                cross_device_ops)
+  def __init__(self, devices=None, cross_device_ops=None):
+    extended = MirroredExtended(
+        self, devices=devices, cross_device_ops=cross_device_ops)
     super(MirroredStrategy, self).__init__(extended)
 
 
 class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   """Implementation of MirroredStrategy."""
 
-  def __init__(self,
-               container_strategy,
-               devices=None,
-               num_gpus_per_worker=None,
-               cross_device_ops=None):
+  def __init__(self, container_strategy, devices=None, cross_device_ops=None):
     super(MirroredExtended, self).__init__(container_strategy)
+    if devices is None:
+      devices = all_local_devices()
+    if not devices:
+      raise ValueError("Got an empty `devices` list. Please make sure the "
+                       "`devices` you pass in is not empty.")
     self._cross_device_ops = cross_device_ops
-    # Remember num GPUs which might be needed by `configure` method.
-    self._num_gpus = num_gpus_per_worker
+    self._initialize_strategy(devices)
 
-    self._initialize_local(self._num_gpus, devices)
+  def _initialize_strategy(self, devices):
+    # The _initialize_strategy method is intended to be used by distribute
+    # coordinator as well.
+    if _is_device_list_local(devices):
+      self._initialize_local(devices)
+    else:
+      self._initialize_multi_worker(devices)
 
-  def _initialize_local(self, num_gpus, devices):
+  def _initialize_local(self, devices):
     """Initializes the object for local training."""
-    self._cluster_spec = None
-    # Convert `num_gpus` into `devices`, shouldn't specify both.
-    if devices is None:
-      if num_gpus is None:
-        num_gpus = context.num_gpus()
-      if num_gpus == 0:
-        devices = ["/device:CPU:0"]
-      else:
-        devices = ["/device:GPU:%d" % d for d in range(num_gpus)]
-    elif num_gpus is not None:
-      raise ValueError("Must only specify one of `devices` and `num_gpus`.")
-    self._num_gpus = num_gpus
-    # TODO(yuefengz): consider setting the default device.
-
+    self._local_mode = True
     assert devices, "Must specify at least one device."
     assert len(set(devices)) == len(devices), (
         "No duplicates allowed in `devices` argument.")
@@ -360,32 +475,30 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     self._device_index = values.PerReplica(
         {d: i for i, d in enumerate(devices)})
 
-  def _initialize_multi_worker(self, num_gpus, cluster_spec):
+    self._inferred_cross_device_ops = cross_device_ops_lib.choose_the_best(
+        devices)
+
+  def _initialize_multi_worker(self, devices):
     """Initializes the object for multi-worker training."""
-    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-    self._cluster_spec = cluster_spec
+    self._local_mode = False
 
+    assert devices, "Must specify at least one device."
+    assert len(set(devices)) == len(devices), (
+        "No duplicates allowed in `devices` argument.")
+    # TODO(josh11b): Require at least 2 devices?
+    self._devices = [device_util.resolve(d) for d in devices]
+    self._canonical_device_set = set(self._devices)
+    self._device_index = values.PerReplica(
+        {d: i for i, d in enumerate(devices)})
+
+    device_dict = _group_device_list(devices)
     self._workers = []
+    self._worker_devices = []
     for job in ["chief", "worker"]:
-      for task in range(len(cluster_spec.as_dict().get(job, []))):
-        self._workers.append("/job:%s/task:%d" % (job, task))
-
-    if num_gpus is None:
-      raise ValueError("`num_gpus` is required if `cluster_spec` is given.")
-    if num_gpus > 0:
-      self._worker_devices = [
-          (worker, [
-              device_util.canonicalize(worker + "/device:GPU:%d" % gpu)
-              for gpu in range(num_gpus)
-          ]) for worker in self._workers
-      ]
-    else:
-      self._worker_devices = [
-          (worker, [device_util.canonicalize(worker, "/device:CPU:0")])
-          for worker in self._workers
-      ]
-
-    devices = nest.flatten([l for _, l in self._worker_devices])
+      for task in range(len(device_dict.get(job, []))):
+        worker = "/job:%s/task:%d" % (job, task)
+        self._workers.append(worker)
+        self._worker_devices.append((worker, device_dict[job][task]))
 
     # Setting `_default_device` will add a device scope in the
     # distribution.scope. We set the default device to the first worker. When
@@ -396,14 +509,8 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode.
     self._default_device = self._workers[0]
 
-    assert devices, "Must specify at least one device."
-    assert len(set(devices)) == len(devices), (
-        "No duplicates allowed in `devices` argument.")
-    # TODO(josh11b): Require at least 2 devices?
-    self._devices = [device_util.resolve(d) for d in devices]
-    self._canonical_device_set = set(self._devices)
-    self._device_index = values.PerReplica(
-        {d: i for i, d in enumerate(devices)})
+    self._inferred_cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce(
+        self._workers, _infer_num_gpus_per_worker(self._devices))
 
   def _create_variable(self, next_creator, *args, **kwargs):
     """Create a mirrored variable. See `DistributionStrategy.scope`."""
@@ -444,21 +551,22 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
                                      **kwargs)
 
   def _distribute_dataset(self, dataset_fn):
-    if self._cluster_spec:
+    if self._local_mode:
+      return values.PerReplicaDataset(
+          self._call_dataset_fn(dataset_fn), self._devices)
+    else:
       return values.MultiWorkerDataset(
           functools.partial(self._call_dataset_fn, dataset_fn),
           self._worker_devices,
           auto_shard=False)
-    else:
-      return values.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._devices)
 
   def _make_dataset_iterator(self, dataset):
-    if self._cluster_spec:
-      worker_device_pairs = self._worker_devices
-    else:
+    if self._local_mode:
       worker = device_util.canonicalize("/device:CPU:0")
       worker_device_pairs = [(worker, self._devices)]
+    else:
+      worker_device_pairs = self._worker_devices
+
     return values.DatasetIterator(dataset, worker_device_pairs,
                                   self._num_replicas_in_sync)
 
@@ -467,13 +575,14 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
       input_fn,
       replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
     input_contexts = []
-    if self._cluster_spec:
-      num_workers = len(self._worker_devices)
-      worker_device_pairs = self._worker_devices
-    else:
+    if self._local_mode:
       num_workers = 1
       worker = device_util.canonicalize("/device:CPU:0")
       worker_device_pairs = [(worker, self._devices)]
+    else:
+      num_workers = len(self._worker_devices)
+      worker_device_pairs = self._worker_devices
+
     for i in range(num_workers):
       input_contexts.append(distribute_lib.InputContext(
           num_input_pipelines=num_workers,
@@ -568,23 +677,12 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
       session_config.CopyFrom(self._update_config_proto(session_config))
 
     if cluster_spec:
-      self._initialize_multi_worker(self._num_gpus, cluster_spec)
-
-    if self._cross_device_ops is None:
-      if self._cluster_spec:
-        # It currently cannot detect the toplogy of remote workers. So we
-        # hard-code the multi-worker all-reduce algorithm for now.
-        if len(self._workers) == 1:
-          # The default is "nccl".
-          self._cross_device_ops = (
-              cross_device_ops_lib.AllReduceCrossDeviceOps())
-        else:
-          # The default is hierarchical reduce and broadcast.
-          self._cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce(
-              self._workers, self._num_gpus)
-      else:
-        self._cross_device_ops = cross_device_ops_lib.choose_the_best(
-            self._devices, session_config=session_config)
+      # TODO(yuefengz): remove the following code once cluster_resolver is
+      # added.
+      num_gpus_per_worker = _infer_num_gpus_per_worker(self._devices)
+      multi_worker_devices = _cluster_spec_to_device_list(
+          cluster_spec, num_gpus_per_worker)
+      self._initialize_multi_worker(multi_worker_devices)
 
   def _update_config_proto(self, config_proto):
     updated_config = copy.deepcopy(config_proto)
@@ -592,10 +690,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     return updated_config
 
   def _get_cross_device_ops(self):
-    if self._cross_device_ops is None:
-      self._cross_device_ops = (
-          cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps())
-    return self._cross_device_ops
+    return self._cross_device_ops or self._inferred_cross_device_ops
 
   def _reduce_to(self, reduce_op, value, destinations):
     assert not isinstance(value, values.Mirrored)
-- 
GitLab


From 9a1a45e4f837613b5732fec690e7cfc28c0affa0 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 30 Nov 2018 17:49:00 -0800
Subject: [PATCH 1143/1554] Convert lr to learning_rate when deserializing.

PiperOrigin-RevId: 223602997
---
 tensorflow/python/keras/optimizer_v2/BUILD    |  5 +-
 .../python/keras/optimizer_v2/adadelta.py     | 13 +++
 .../python/keras/optimizer_v2/adagrad.py      | 38 ++++++++-
 tensorflow/python/keras/optimizer_v2/adam.py  | 12 +++
 .../python/keras/optimizer_v2/adam_test.py    | 18 ++++-
 .../python/keras/optimizer_v2/adamax_test.py  |  5 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |  5 +-
 .../keras/optimizer_v2/optimizer_v2_test.py   | 81 +++++++++++++++++++
 8 files changed, 166 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 6b805781f0..b8f0124941 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -172,8 +172,9 @@ cuda_py_test(
 
 py_test(
     name = "optimizer_v2_test",
-    size = "medium",
+    size = "large",
     srcs = ["optimizer_v2_test.py"],
+    shard_count = 4,
     tags = [
         "no_windows",
     ],
@@ -189,6 +190,8 @@ py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/keras",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index e1d7ecb558..55b4eba105 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.training import training_ops
 
@@ -89,10 +91,21 @@ class Adadelta(optimizer_v2.OptimizerV2):
     self._set_hyper('epsilon', epsilon)
 
   def _create_slots(self, var_list):
+    # Separate for-loops to respect the ordering of slot variables from v1.
     for v in var_list:
       self.add_slot(v, 'accum_grad')
+    for v in var_list:
       self.add_slot(v, 'accum_var')
 
+  def set_weights(self, weights):
+    params = self.weights
+    # Override set_weights for backward compatibility of Keras V1 optimizer
+    # since it does not include iteration at head of the weight list. Set
+    # iteration to 0.
+    if len(params) == len(weights) + 1:
+      weights = [np.array(0)] + weights
+    super(Adadelta, self).set_weights(weights)
+
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
index 0896f95f94..670cad70e6 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
@@ -78,8 +80,8 @@ class Adagrad(optimizer_v2.OptimizerV2):
     functions.
     @end_compatibility
     """
-    if initial_accumulator_value <= 0.0:
-      raise ValueError('initial_accumulator_value must be positive: %s' %
+    if initial_accumulator_value < 0.0:
+      raise ValueError('initial_accumulator_value must be non-negative: %s' %
                        initial_accumulator_value)
     if epsilon < 1e-7:
       raise ValueError('epsilon must be larger than 1e-7: %s' % epsilon)
@@ -96,6 +98,38 @@ class Adagrad(optimizer_v2.OptimizerV2):
           self._initial_accumulator_value, dtype=dtype)
       self.add_slot(var, 'accumulator', init)
 
+  def set_weights(self, weights):
+    params = self.weights
+    # Override set_weights for backward compatibility of Keras V1 optimizer
+    # since it does not include iteration at head of the weight list. Set
+    # iteration to 0.
+    if len(params) == len(weights) + 1:
+      weights = [np.array(0)] + weights
+    super(Adagrad, self).set_weights(weights)
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    """Creates an optimizer from its config.
+
+    This method is the reverse of `get_config`,
+    capable of instantiating the same optimizer from the config
+    dictionary.
+
+    Arguments:
+        config: A Python dictionary, typically the output of get_config.
+        custom_objects: A Python dictionary mapping names to additional Python
+          objects used to create this optimizer, such as a function used for a
+          hyperparameter.
+
+    Returns:
+        An optimizer instance.
+    """
+    if 'initial_accumulator_value' not in config:
+      config['initial_accumulator_value'] = 0.
+    if 'lr' in config:
+      config['learning_rate'] = config.pop('lr')
+    return cls(**config)
+
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index a3f12909aa..06f8e52802 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -139,10 +139,22 @@ class Adam(optimizer_v2.OptimizerV2):
 
   def _create_slots(self, var_list):
     # Create slots for the first and second moments.
+    # Separate for-loops to respect the ordering of slot variables from v1.
     for var in var_list:
       self.add_slot(var, 'm')
+    for var in var_list:
       self.add_slot(var, 'v')
 
+  def set_weights(self, weights):
+    params = self.weights
+    # If the weights are generated by Keras V1 optimizer, it includes vhats
+    # even without amsgrad, i.e, V1 optimizer has 3x + 1 variables, while V2
+    # optimizer has 2x + 1 variables. Filter vhats out for compatibility.
+    num_vars = int((len(params) - 1) / 2)
+    if len(weights) == 3 * num_vars + 1:
+      weights = weights[:len(params)]
+    super(Adam, self).set_weights(weights)
+
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index b1b17f8f08..958bbed21c 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -347,15 +348,26 @@ class AdamOptimizerTest(test.TestCase):
       v2 = resource_variable_ops.ResourceVariable(1.)
       opt = adam.Adam(1.)
       opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
-      # There should be iteration, hyper variables, and two unique slot
-      # variables for v1 and v2 respectively.
-      self.assertEqual(10, len(set(opt.variables())))
+      # There should be iteration, and two unique slot variables for v1 and v2.
+      self.assertEqual(5, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
 
   def testAmsgradWithError(self):
     with self.assertRaisesRegexp(ValueError,
                                  "Amsgrad is currently not supported"):
       adam.Adam(learning_rate=1., beta_1=0.9, beta_2=0.99, amsgrad=True)
 
+  def testSetWeightsFromV1AdamWithoutMinimize(self):
+    keras_v1_adam = optimizers.Adam()
+    keras_v2_adam = adam.Adam()
+    keras_v2_adam.set_weights(keras_v1_adam.get_weights())
+    keras_v1_iteration = keras_v1_adam.iterations
+    keras_v2_iteration = keras_v2_adam.iterations
+    self.evaluate(variables.global_variables_initializer())
+    self.assertEqual(
+        self.evaluate(keras_v1_iteration), self.evaluate(keras_v2_iteration))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index 7521a6196e..baf131fbb0 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -359,9 +359,8 @@ class AdamaxOptimizerTest(test.TestCase):
       v2 = resource_variable_ops.ResourceVariable(1.)
       opt = adamax.Adamax(1.)
       opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
-      # There should be iteration, hyper variables, and two unique slot
-      # variables for v1 and v2 respectively.
-      self.assertEqual(10, len(set(opt.variables())))
+      # There should be iteration, and two unique slot variables for v1 and v2.
+      self.assertEqual(5, len(set(opt.variables())))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 9c8fff0fe4..b26b3cefc8 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -304,6 +304,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       grads_and_vars = zip(reduced_grads, var_list)
 
     with ops.init_scope():
+      self._prepare()
       self._create_slots(var_list)
     update_ops = []
 
@@ -325,7 +326,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
         return update_op
 
     with ops.name_scope(name, self._name) as name:
-      self._prepare()
       for grad, var in grads_and_vars:
         scope_name = ("" if ops.executing_eagerly_outside_functions() else
                       "_" + var.op.name)
@@ -426,7 +426,6 @@ class OptimizerV2(optimizer_v1.Optimizer):
             trainable=False,
             initializer=value,
             aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA)
-        self._weights.append(self._hyper[name])
     self._prepared = True
 
   @property
@@ -475,6 +474,8 @@ class OptimizerV2(optimizer_v1.Optimizer):
     Returns:
         An optimizer instance.
     """
+    if "lr" in config:
+      config["learning_rate"] = config.pop("lr")
     return cls(**config)
 
   def _serialize_hyperparameter(self, hyperparameter_name):
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 305267d73e..8fcf40af5d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -18,8 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import tempfile
+
+from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
@@ -29,18 +34,23 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import callbacks
+from tensorflow.python.keras import optimizers
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import input_layer
+from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import sequential
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
 from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
+from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
+from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 
 
@@ -476,6 +486,77 @@ class OptimizerTest(test.TestCase):
         float(backend.get_value(model.optimizer.lr)), 0.01, atol=1e-4)
 
 
+class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
+
+  # TODO(tanzheny): remove test_numeric after algorithm for Momentum, Adam and
+  # NAdam has been unified: currently these three algorithms behave differently.
+  @parameterized.named_parameters(
+      ('adadelta', 'adadelta', True, True), ('adagrad', 'adagrad', True, True),
+      ('adam', 'adam', True, False), ('adamax', 'adamax', True, True),
+      ('nadam', 'nadam', True, False), ('momentum', 'momentum', False, False),
+      ('sgd', 'sgd', False, True))
+  def testOptimizersCompatibility(self, opt_str, test_weights, test_numeric):
+    np.random.seed(1331)
+    with self.cached_session():
+      train_samples = 20
+      input_dim = 3
+      num_classes = 2
+      (x, y), _ = testing_utils.get_test_data(
+          train_samples=train_samples,
+          test_samples=10,
+          input_shape=(input_dim,),
+          num_classes=num_classes)
+      y = keras.utils.to_categorical(y)
+
+      num_hidden = 5
+      model = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+
+      old_mode = os.environ.get('TF2_BEHAVIOR', None)
+      # Disable tf2 to create V1 optimizer.
+      disable_tf2()
+      if opt_str == 'momentum':
+        opt_v1 = optimizers.SGD(momentum=0.9)
+      else:
+        opt_v1 = optimizers.get(opt_str)
+
+      # Test compile and fit with v1 optimizer.
+      model.compile(opt_v1, loss='categorical_crossentropy', metrics=[])
+      model.fit(x, y, batch_size=5, epochs=1)
+      model_dir = tempfile.mkdtemp()
+      gfile.MakeDirs(model_dir)
+      file_name = os.path.join(model_dir, 'model.h5')
+      model.save(file_name)
+
+      enable_tf2()
+      # Test load and fit with v2 optimizer.
+      model_2 = saving.load_model(file_name)
+      opt_v2 = model_2.optimizer
+      self.assertIsInstance(opt_v2, optimizer_v2.OptimizerV2)
+      # set_weights is called inside load_model but exception is swallowed,
+      # this call checks the weights can be set correctly.
+      if test_weights:
+        opt_v2.set_weights(opt_v1.get_weights())
+      if test_numeric:
+        hist_1 = model.fit(x, y, batch_size=5, epochs=1, shuffle=False)
+        hist_2 = model_2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
+        self.assertAllClose(model.get_weights(), model_2.get_weights())
+        self.assertAllClose(model.get_weights(), model_2.get_weights())
+        self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'])
+
+      if old_mode is not None:
+        os.environ['TF2_BEHAVIOR'] = old_mode
+
+
+def disable_tf2():
+  if 'TF2_BEHAVIOR' in os.environ:
+    del os.environ['TF2_BEHAVIOR']
+
+
+def enable_tf2():
+  os.environ['TF2_BEHAVIOR'] = 'enabled'
+
+
 # Note: These tests are kept in a separate class to avoid bugs in some
 # distributions of Python that break AutoGraph which is used by tf.function.
 class OptimizerWithFunctionTest(test.TestCase):
-- 
GitLab


From 2f2f76a5b8e2119b551db582c14da7332668b81a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 18:09:48 -0800
Subject: [PATCH 1144/1554] Added a version of `compute_gradient` and
 `compute_gradient_error` in gradient_checker.py for functions as arguments.

PiperOrigin-RevId: 223604860
---
 tensorflow/python/BUILD                       |  31 ++
 tensorflow/python/ops/gradient_checker.py     |  23 +-
 tensorflow/python/ops/gradient_checker_v2.py  | 311 ++++++++++++++++++
 .../python/ops/gradient_checker_v2_test.py    | 286 ++++++++++++++++
 4 files changed, 642 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/python/ops/gradient_checker_v2.py
 create mode 100644 tensorflow/python/ops/gradient_checker_v2_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 92aac5fc0a..64a3b413cc 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -102,6 +102,7 @@ py_library(
         ":framework_for_generated_wrappers",
         ":functional_ops",
         ":gradient_checker",
+        ":gradient_checker_v2",
         ":graph_util",
         ":histogram_ops",
         ":image_ops",
@@ -1086,6 +1087,7 @@ py_library(
         ":cond_v2",
         ":framework_test_lib",
         ":gradient_checker",
+        ":gradient_checker_v2",
         ":platform_test",
         ":util",
         ":while_v2",
@@ -3178,6 +3180,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "gradient_checker_v2",
+    srcs = ["ops/gradient_checker_v2.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":array_ops",
+        ":framework_for_generated_wrappers",
+        ":gradients",
+        ":platform",
+        "//third_party/py/numpy",
+    ],
+)
+
 # This target is deprecated.
 py_library(
     name = "ops",
@@ -3243,6 +3258,22 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "gradient_checker_v2_test",
+    size = "medium",
+    srcs = ["ops/gradient_checker_v2_test.py"],
+    additional_deps = [
+        ":array_ops",
+        ":client_testlib",
+        ":framework_for_generated_wrappers",
+        ":math_ops",
+        ":nn_grad",
+        ":nn_ops",
+        ":platform",
+        "//third_party/py/numpy",
+    ],
+)
+
 cuda_py_test(
     name = "gradients_test",
     size = "medium",
diff --git a/tensorflow/python/ops/gradient_checker.py b/tensorflow/python/ops/gradient_checker.py
index 3926ff6900..683f78ce9b 100644
--- a/tensorflow/python/ops/gradient_checker.py
+++ b/tensorflow/python/ops/gradient_checker.py
@@ -157,7 +157,8 @@ def _compute_numeric_jacobian(x, x_shape, x_data, y, y_shape, delta,
   # as delta. Convert to float32 here. Since numeric_jacobian is expected to
   # be the groundtruth to compare against, it shouldn't lose any information.
   if x.dtype == dtypes.bfloat16:
-    x = math_ops.cast(x, dtypes.float32)
+    x = math_ops.cast(x, dtypes.float32)  # TODO(wangpeng): Now that the new x
+            # is an output of the old x, isn't feeding to the new x a mistake?
   if y.dtype == dtypes.bfloat16:
     y = math_ops.cast(y, dtypes.float32)
   if x_data.dtype == dtypes.bfloat16.as_numpy_dtype:
@@ -300,7 +301,6 @@ def compute_gradient(x,
       as the initial value.
     delta: (optional) the amount of perturbation.
     init_targets: list of targets to run to initialize model params.
-      TODO(mrry): remove this argument.
     extra_feed_dict: dict that allows fixing specified tensor values
       during the Jacobian calculation.
 
@@ -310,6 +310,7 @@ def compute_gradient(x,
     where "x_size" is the number of elements in x and "y_size" is the
     number of elements in y. If x is a list, returns a list of two numpy arrays.
   """
+  # TODO(mrry): remove argument `init_targets`
   if extra_feed_dict is None:
     extra_feed_dict = {}
 
@@ -327,6 +328,16 @@ def compute_gradient(x,
     return ret
 
 
+def _compute_error(grad):
+  if isinstance(grad, tuple):
+    grad = [grad]
+  error = 0
+  for j_t, j_n in grad:
+    if j_t.size or j_n.size:  # Handle zero size tensors correctly
+      error = np.maximum(error, np.fabs(j_t - j_n).max())
+  return error
+
+
 @tf_export(v1=["test.compute_gradient_error"])
 def compute_gradient_error(x,
                            x_shape,
@@ -369,10 +380,4 @@ def compute_gradient_error(x,
   """
   grad = compute_gradient(x, x_shape, y, y_shape, x_init_value, delta,
                           init_targets, extra_feed_dict=extra_feed_dict)
-  if isinstance(grad, tuple):
-    grad = [grad]
-  error = 0
-  for j_t, j_n in grad:
-    if j_t.size or j_n.size:  # Handle zero size tensors correctly
-      error = np.maximum(error, np.fabs(j_t - j_n).max())
-  return error
+  return _compute_error(grad)
diff --git a/tensorflow/python/ops/gradient_checker_v2.py b/tensorflow/python/ops/gradient_checker_v2.py
new file mode 100644
index 0000000000..cd49bb3d17
--- /dev/null
+++ b/tensorflow/python/ops/gradient_checker_v2.py
@@ -0,0 +1,311 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Gradient checker for functions.
+
+The gradient checker verifies numerically that an function properly
+computes the gradients
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util.tf_export import tf_export
+
+
+def _product(t):
+  if isinstance(t, int):
+    return t
+  else:
+    y = 1
+    for x in t:
+      y *= x
+    return y
+
+
+def _to_numpy(a):
+  """Converts tensors to numpy arrays.
+
+  Converts Tensors and EagerTensors to numpy arrays.
+  When eager execution is enabled, converts IndexedSlices
+  to IndexedSlicesValue with numpy indices/values
+
+  Args:
+    a: any value.
+
+  Returns:
+    If a is EagerTensor or Tensor, returns the evaluation of a by calling
+    numpy() or run().
+    If a is IndexedSlices and eager execution is enabled, calls numpy() on a's
+    fields. Otherwise returns a unchanged.
+  """
+  if isinstance(a, ops.EagerTensor):
+    return a.numpy()
+  if isinstance(a, ops.Tensor):
+    sess = ops.get_default_session()
+    return sess.run(a)
+  if isinstance(a, ops.IndexedSlices) and context.executing_eagerly():
+    return ops.IndexedSlicesValue(
+        indices=[x.numpy() for x in a.indices],
+        values=[x.numpy() for x in a.values],
+        dense_shape=a.dense_shape)
+  return a
+
+
+def _prepare(f, xs_dtypes):
+  """Return a function that executes 'f'.
+
+    In TF 2.x, this is the same as `f`.
+    In TF 1.x, returns a Python function that executes the graph defined by `f`
+    in a Session.
+
+  Args:
+    f: the function.
+    xs_dtypes: dtypes of f's arguments.
+
+  Returns:
+    a function that will be evaluated in both graph and eager mode
+  """
+  if context.executing_eagerly():
+    return f
+  xs = [array_ops.placeholder(x_dtype) for x_dtype in xs_dtypes]
+  y = f(*xs)
+  sess = ops.get_default_session()
+  def decorated(*xs_data):
+    xs_data = [_to_numpy(a) for a in xs_data]
+    return sess.run(y, feed_dict=dict(zip(xs, xs_data)))
+  return decorated
+
+
+def _compute_theoretical_jacobian(f, y_shape, y_dtype, xs, param):
+  """Computes the theoretical Jacobian for f regarding xs[param].
+
+  One can think of the relation among f, xs and y as y = f(xs).
+
+  Args:
+    f: the function.
+    y_shape: the shape of the result.
+    y_dtype: the dtype of the result.
+    xs: a list of tensors.
+    param: the index of the target parameter.
+
+  Returns:
+    A 2-d numpy array representing the Jacobian. It has "x_size" rows
+    and "y_size" columns where "x_size" is the number of elements in xs[param]
+    and "y_size" is the number of elements in the result.
+
+  Raises:
+    ValueError: If result is empty but the gradient is nonzero.
+  """
+  x = xs[param]
+  # Complex vectors are treated as vectors of twice as many reals.
+  x_shape = tuple(x.shape) + (2,) if x.dtype.is_complex else x.shape
+  y_factor = 2 if y_dtype.is_complex else 1
+
+  # To compute the jacobian, we treat x and y as one-dimensional vectors.
+  x_size = _product(x_shape)
+  x_val_size = _product(x_shape[1:])  # This is used for sparse gradients
+  y_size = _product(y_shape) * y_factor
+
+  # Allocate 2-D Jacobian, with x dimensions smashed into the first
+  # dimension and y dimensions smashed into the second.
+  jacobian = np.zeros((x_size, y_size), dtype=x.dtype.real_dtype.as_numpy_dtype)
+
+  # For each of the entry of dy, we set this to be 1 and
+  # everything else to be 0 and compute the gradients -- this will give us one
+  # one column of the Jacobian matrix.
+  dy_data = np.zeros(y_shape, dtype=y_dtype.as_numpy_dtype)
+  dy_data_flat = dy_data.ravel().view(y_dtype.real_dtype.as_numpy_dtype)
+  grad_fn_unprep = backprop.gradients_function(f, [param])
+  grad_fn = _prepare(lambda dy, *xs: grad_fn_unprep(*xs, dy=dy),
+                     [y_dtype] + [x.dtype for x in xs])
+  for col in range(y_size):
+    dy_data_flat[col] = 1
+    grad = _to_numpy(grad_fn(dy_data, *xs)[0])
+    dy_data_flat[col] = 0
+    if isinstance(grad, ops.IndexedSlicesValue):
+      for i, v in zip(grad.indices, grad.values):
+        r_begin = i * x_val_size
+        r_end = r_begin + x_val_size
+        jacobian[r_begin:r_end, col] += v.flat
+    else:
+      jacobian[:, col] = grad.ravel().view(jacobian.dtype)
+
+  # If the output is empty, run the gradients at least once and make sure
+  # they produce zeros.
+  if y_size == 0:  # don't use 'not y_size', because y_size may not be an int
+    grad = _to_numpy(grad_fn(dy_data, *xs)[0])
+    if grad.shape != x.shape:
+      raise ValueError("Empty gradient has wrong shape: expected %s, got %s" %
+                       (x.shape, grad.shape))
+    if np.any(grad):
+      raise ValueError("Empty tensor with nonzero gradients")
+
+  logging.vlog(1, "Theoretical Jacobian =\n%s", jacobian)
+  return jacobian
+
+
+def _compute_numeric_jacobian(f, y_size, y_dtype, xs, param,
+                              delta):
+  """Computes the numeric Jacobian for f regarding xs[param].
+
+  One can think of the relation among f, xs and y as y = f(xs).
+
+  Args:
+    f: the function.
+    y_size: the number of elements of the result.
+    y_dtype: the dtype of the result.
+    xs: a list of tensors.
+    param: the index of the target parameter.
+    delta: the amount of perturbation we give to the input.
+
+  Returns:
+    A 2-d numpy array representing the Jacobian. It has "x_size" rows
+    and "y_size" columns where "x_size" is the number of elements in xs[param]
+    and "y_size" is the number of elements in the result.
+  """
+  # bfloat16 doesn't have enough bits to represent high precision numbers such
+  # as delta. Convert to float32 here. Since numeric_jacobian is expected to
+  # be the groundtruth to compare against, it shouldn't lose any information.
+  x_shape = xs[param].shape
+  x_dtype = xs[param].dtype
+  if y_dtype == dtypes.bfloat16:
+    f = lambda *xs: math_ops.cast(f(*xs), dtypes.float32)
+    y_dtype = dtypes.float32
+
+  # To compute the jacobian, we treat x and y as one-dimensional vectors
+  x_size = _product(x_shape) * (2 if x_dtype.is_complex else 1)
+  y_size = y_size * (2 if y_dtype.is_complex else 1)
+  x_dtype = x_dtype.real_dtype.as_numpy_dtype
+  y_dtype = y_dtype.real_dtype.as_numpy_dtype
+
+  xs_dtypes = [x.dtype for x in xs]
+  # Converts xs to numpy arrays to do in-place perturbation.
+  # Calls asarray() to avoid copying in ravel() later.
+  xs = [np.asarray(_to_numpy(x)) for x in xs]
+  x = xs[param]
+
+  # Make sure we have the right types
+  scale = np.asarray(2 * delta, dtype=y_dtype)[()]
+
+  jacobian = np.zeros((x_size, y_size), dtype=x_dtype)
+  # For each of the entry of x, we slightly perturbs this by adding and
+  # subtracting a delta and then compute difference between the outputs. This
+  # will give us one row of the Jacobian matrix.
+
+  f = _prepare(f, xs_dtypes)
+  for row in range(x_size):
+    original = x.ravel().view(x_dtype)[row]
+    x.ravel().view(x_dtype)[row] += delta
+    y_pos = _to_numpy(f(*xs))
+    x.ravel().view(x_dtype)[row] = original
+    x.ravel().view(x_dtype)[row] -= delta
+    y_neg = _to_numpy(f(*xs))
+    x.ravel().view(x_dtype)[row] = original
+    diff = (y_pos - y_neg) / scale
+    jacobian[row, :] = diff.ravel().view(y_dtype)
+
+  logging.vlog(1, "Numeric Jacobian =\n%s", jacobian)
+  return jacobian
+
+
+def _compute_gradient(f,
+                      y_shape,
+                      y_dtype,
+                      xs,
+                      param,
+                      delta):
+  """Computes the theoretical and numerical jacobian."""
+  x = xs[param]
+  t = x.dtype
+  allowed_types = [dtypes.float16, dtypes.bfloat16, dtypes.float32,
+                   dtypes.float64, dtypes.complex64, dtypes.complex128]
+  assert t.base_dtype in allowed_types, ("Cannot compute gradient for"
+                                         "unsupported type %s of argument %s" %
+                                         (t.name, param))
+  t2 = y_dtype
+  assert t2.base_dtype in allowed_types, ("Cannot compute gradient for"
+                                          "unsupported type %s of y" % t2.name)
+  y_size = _product(y_shape)
+  jacob_t = _compute_theoretical_jacobian(f, y_shape, y_dtype,
+                                          xs, param)
+  jacob_n = _compute_numeric_jacobian(f, y_size, y_dtype, xs,
+                                      param, delta)
+  return jacob_t, jacob_n
+
+
+def _compute_gradient_list(f, xs, delta):
+  """Compute gradients for a list of x values."""
+  # convert xs to tensors so that dtype and shape have uniform types
+  xs = list(map(ops.convert_to_tensor, xs))
+  # run the function to get info of the result
+  xs_dtypes = [x.dtype for x in xs]
+  f_temp = _prepare(f, xs_dtypes)
+  y = f_temp(*xs)
+  return zip(*[_compute_gradient(f, y.shape, dtypes.as_dtype(y.dtype),
+                                 xs, i, delta) for i in range(len(xs))])
+
+
+@tf_export("test.compute_gradient", v1=[])
+def compute_gradient(f, x, delta=1e-3):
+  """Computes the theoretical and numeric Jacobian of f.
+
+  With y = f(x), computes the theoretical and numeric Jacobian dy/dx.
+
+  Args:
+    f: the function.
+    x: a list of tensors.
+    delta: (optional) perturbation used to compute numeric Jacobian.
+
+  Returns:
+    A pair of lists, where the first is a list of 2-d numpy arrays representing
+    the theoretical Jacobians for each argument, and the second list is the
+    numerical ones. Each 2-d array has "x_size" rows
+    and "y_size" columns where "x_size" is the number of elements in the
+    corresponding argument and "y_size" is the number of elements in f(x).
+
+  Raises:
+    ValueError: If result is empty but the gradient is nonzero.
+  """
+  return _compute_gradient_list(f, x, delta)
+
+
+def max_error(grad1, grad2):
+  """Computes maximum elementwise gap.
+
+  Computes the maximum elementwise gap between two lists of tensors of the same
+  shape.
+
+  Args:
+    grad1: a lists of tensors.
+    grad2: a lists of tensors with the same shape as grad1.
+
+  Returns:
+    The maximum elementwise gap between the two.
+  """
+  error = 0
+  for j_t, j_n in zip(grad1, grad2):
+    if j_t.size or j_n.size:  # Handle zero size tensors correctly
+      error = np.maximum(error, np.fabs(j_t - j_n).max())
+  return error
diff --git a/tensorflow/python/ops/gradient_checker_v2_test.py b/tensorflow/python/ops/gradient_checker_v2_test.py
new file mode 100644
index 0000000000..e3351e2920
--- /dev/null
+++ b/tensorflow/python/ops/gradient_checker_v2_test.py
@@ -0,0 +1,286 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for compute_gradient.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import custom_gradient
+from tensorflow.python.ops import \
+gradient_checker_v2 as gradient_checker
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+# needs this to register gradient for SoftmaxCrossEntropyWithLogits:
+import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
+from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class GradientCheckerTest(test.TestCase):
+
+  def testAddSimple(self):
+    # if context.executing_eagerly():
+    #   return
+    np.random.seed(1)  # Fix seed to avoid flakiness
+    size = (2, 3)
+    x1 = constant_op.constant(2.0, shape=size, name="x1")
+    x2 = constant_op.constant(3.0, shape=size, name="x2")
+    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        lambda x1: math_ops.add(x1, x2), [x1]))
+    tf_logging.info("x1 error = %f", error)
+    assert error < 1e-4
+
+  def testAddCustomized(self):
+    np.random.seed(3)  # Fix seed to avoid flakiness
+    size = (2, 3)
+    x1 = constant_op.constant(
+        2.0, shape=size, dtype=dtypes.float64, name="x1")
+    x2 = np.asarray(np.arange(6, dtype=np.float64).reshape(2, 3))
+    # checkint gradients for x2 using a special delta
+    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        lambda x2: math_ops.add(x1, x2),
+        [x2], delta=1e-2))
+    tf_logging.info("x2 error = %f", error)
+    assert error < 1e-10
+
+  def testGather(self):
+    np.random.seed(4)  # Fix seed to avoid flakiness
+    def f(params):
+      index_values = [1, 3]
+      indices = constant_op.constant(index_values, name="i")
+      return array_ops.gather(params, indices, name="y")
+    p_shape = (4, 2)
+    p_size = 8
+    params = constant_op.constant(
+        np.arange(p_size).astype(np.float), shape=p_shape, name="p")
+    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        f, [params]))
+    tf_logging.info("gather error = %f", error)
+    assert error < 1e-4
+
+  def testNestedGather(self):
+    np.random.seed(5)  # Fix seed to avoid flakiness
+    def f(params):
+      index_values = [1, 3, 5, 6]
+      indices = constant_op.constant(index_values, name="i")
+      y = array_ops.gather(params, indices, name="y")
+      index_values2 = [0, 2]
+      indices2 = constant_op.constant(index_values2, name="i2")
+      return array_ops.gather(y, indices2, name="y2")
+    p_shape = (8, 2)
+    p_size = 16
+    params = constant_op.constant(
+        np.arange(p_size).astype(np.float), shape=p_shape, name="p")
+    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        f, [params]))
+    tf_logging.info("nested gather error = %f", error)
+    assert error < 1e-4
+
+  def testComplexMul(self):
+    if not context.executing_eagerly():
+      return
+    c = constant_op.constant(5 + 7j, dtype=dtypes.complex64)
+    def f(x):
+      return c * x
+    x = constant_op.constant(11 - 13j, dtype=dtypes.complex64)
+    analytical, numerical = gradient_checker.compute_gradient(
+        f, [x], delta=0.1)
+    correct = np.array([[5, 7], [-7, 5]])
+    self.assertAllEqual(correct, analytical[0])
+    self.assertAllClose(correct, numerical[0], rtol=1e-4)
+    self.assertLess(
+        gradient_checker.max_error(*gradient_checker.compute_gradient(
+            f, [x], delta=0.1)), 2e-4)
+
+  def testComplexConj(self):
+    def f(x):
+      return math_ops.conj(x)
+    x = constant_op.constant(11 - 13j, dtype=dtypes.complex64)
+    analytical, numerical = gradient_checker.compute_gradient(
+        f, [x], delta=0.1)
+    correct = np.array([[1, 0], [0, -1]])
+    self.assertAllEqual(correct, analytical[0])
+    self.assertAllClose(correct, numerical[0], rtol=2e-5)
+    self.assertLess(
+        gradient_checker.max_error(*gradient_checker.compute_gradient(
+            f, [x], delta=0.1)), 2e-5)
+
+  def testEmptySucceeds(self):
+    def f(x):
+      return array_ops.identity(x)
+    x = constant_op.constant(np.random.random_sample((0, 3)),
+                             dtype=dtypes.float32)
+    for grad in gradient_checker.compute_gradient(f, [x]):
+      self.assertEqual(grad[0].shape, (0, 0))
+    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        f, [x]))
+    self.assertEqual(error, 0)
+
+  def testEmptyFails(self):
+    # if not context.executing_eagerly():
+    #   return
+    @custom_gradient.custom_gradient
+    def id_bad_grad(x):
+      y = array_ops.identity(x)
+      def grad_fn(dy):
+        # dx = constant_op.constant(np.zeros((1, 4)), dtype=dtypes.float32)
+        dx = array_ops.transpose(dy)
+        return dx
+      return y, grad_fn
+    def f(x):
+      return id_bad_grad(x)
+    x = constant_op.constant(np.random.random_sample((0, 3)),
+                             dtype=dtypes.float32)
+    bad = r"Empty gradient has wrong shape: expected \(0, 3\), got \(3, 0\)"
+    with self.assertRaisesRegexp(ValueError, bad):
+      gradient_checker.compute_gradient(f, [x])
+
+  def testNaNGradFails(self):
+    @custom_gradient.custom_gradient
+    def id_nan_grad(x):
+      y = array_ops.identity(x)
+      def grad_fn(dy):
+        dx = np.nan * dy
+        # dx = dy
+        return dx
+      return y, grad_fn
+    def f(x):
+      return id_nan_grad(x)
+    x = constant_op.constant(np.random.random_sample((1, 1)),
+                             dtype=dtypes.float32)
+    error = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        f, [x]))
+    # Typical test would assert error < max_err, so assert this test would
+    # raise AssertionError, since NaN is not < 1.0.
+    with self.assertRaisesRegexp(AssertionError, "False is not true"):
+      self.assertTrue(error < 1.0)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class MiniMNISTTest(test.TestCase):
+
+  # Gradient checker for MNIST.
+  def _BuildAndTestMiniMNIST(self, param_index, tag):
+    # Fix seed to avoid occasional flakiness
+    np.random.seed(6)
+
+    # Hyperparameters
+    batch = 3
+    inputs = 16
+    features = 32
+    classes = 10
+
+    # Define the parameters
+    inp_data = np.random.random_sample(inputs * batch)
+    hidden_weight_data = np.random.randn(inputs * features) / np.sqrt(inputs)
+    hidden_bias_data = np.random.random_sample(features)
+    sm_weight_data = np.random.randn(features * classes) / np.sqrt(features)
+    sm_bias_data = np.random.random_sample(classes)
+
+    # special care for labels since they need to be normalized per batch
+    label_data = np.random.random(batch * classes).reshape((batch, classes))
+    s = label_data.sum(axis=1)
+    label_data /= s[:, None]
+
+    # We treat the inputs as "parameters" here
+    inp = constant_op.constant(
+        inp_data.tolist(),
+        shape=[batch, inputs],
+        dtype=dtypes.float64,
+        name="inp")
+    hidden_weight = constant_op.constant(
+        hidden_weight_data.tolist(),
+        shape=[inputs, features],
+        dtype=dtypes.float64,
+        name="hidden_weight")
+    hidden_bias = constant_op.constant(
+        hidden_bias_data.tolist(),
+        shape=[features],
+        dtype=dtypes.float64,
+        name="hidden_bias")
+    softmax_weight = constant_op.constant(
+        sm_weight_data.tolist(),
+        shape=[features, classes],
+        dtype=dtypes.float64,
+        name="softmax_weight")
+    softmax_bias = constant_op.constant(
+        sm_bias_data.tolist(),
+        shape=[classes],
+        dtype=dtypes.float64,
+        name="softmax_bias")
+
+    # List all the parameter so that we can test them one at a time
+    all_params = [
+        inp, hidden_weight, hidden_bias, softmax_weight, softmax_bias
+    ]
+
+    # Now, Building MNIST
+    def f(inp, hidden_weight, hidden_bias, softmax_weight, softmax_bias):
+      features = nn_ops.relu(
+          nn_ops.xw_plus_b(inp, hidden_weight, hidden_bias), name="features")
+      logits = nn_ops.xw_plus_b(
+          features, softmax_weight, softmax_bias, name="logits")
+      labels = constant_op.constant(
+          label_data.tolist(),
+          shape=[batch, classes],
+          dtype=dtypes.float64,
+          name="labels")
+      cost = nn_ops.softmax_cross_entropy_with_logits(
+          labels=labels, logits=logits, name="cost")
+      return cost
+
+    def f_restricted(x):
+      xs = all_params
+      i = param_index
+      # use x for the i-th parameter
+      xs = xs[0:i]+[x]+xs[i+1:]
+      return f(*xs)
+    # Test the gradients.
+    err = gradient_checker.max_error(*gradient_checker.compute_gradient(
+        f_restricted, [all_params[param_index]], delta=1e-5))
+
+    tf_logging.info("Mini MNIST: %s gradient error = %g", tag, err)
+    return err
+
+  def testInputGradient(self):
+    # if context.executing_eagerly():
+    #   return
+    self.assertLess(self._BuildAndTestMiniMNIST(0, "input"), 1e-8)
+
+  def testHiddenWeightGradient(self):
+    self.assertLess(self._BuildAndTestMiniMNIST(1, "hidden_weight"), 1e-8)
+
+  def testHiddenBiasGradient(self):
+    self.assertLess(self._BuildAndTestMiniMNIST(2, "hidden_bias"), 1e-8)
+
+  def testSoftmaxWeightGradient(self):
+    self.assertLess(self._BuildAndTestMiniMNIST(3, "softmax_weight"), 1e-8)
+
+  def testSoftmaxBiasGradient(self):
+    self.assertLess(self._BuildAndTestMiniMNIST(4, "softmax_bias"), 1e-8)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 32f729c59e8ebae55b6c460cf863eec3bbe51256 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Nov 2018 18:12:37 -0800
Subject: [PATCH 1145/1554] Fix formatting for StatsCalculator string output.

PiperOrigin-RevId: 223605102
---
 tensorflow/core/util/stats_calculator.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/util/stats_calculator.cc b/tensorflow/core/util/stats_calculator.cc
index eb07754650..bce650f245 100644
--- a/tensorflow/core/util/stats_calculator.cc
+++ b/tensorflow/core/util/stats_calculator.cc
@@ -53,7 +53,7 @@ std::string StatsCalculator::HeaderString(const std::string& title) const {
          << " ==============================" << std::endl;
 
   InitField(stream, 24) << "[node type]";
-  InitField(stream, 9) << "[start]";
+  InitField(stream, 17) << "[start]";
   InitField(stream, 9) << "[first]";
   InitField(stream, 9) << "[avg ms]";
   InitField(stream, 8) << "[%]";
@@ -77,7 +77,7 @@ std::string StatsCalculator::ColumnString(const Detail& detail,
 
   std::stringstream stream;
   InitField(stream, 24) << detail.type;
-  InitField(stream, 9) << start_ms;
+  InitField(stream, 17) << start_ms;
   InitField(stream, 9) << first_time_ms;
   InitField(stream, 9) << avg_time_ms;
   InitField(stream, 7) << percentage << "%";
-- 
GitLab


From 6be205455f719fb30b819d295e7535ece1ad13a1 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 30 Nov 2018 19:19:19 -0800
Subject: [PATCH 1146/1554] Add rate to v1 dropout.

This change does change observable behavior in the case where one passes a tf.constant with value 1 to keep_prob. Before, this returned the input (y is x), now it will execute the actual dropout (even though it's pointless) since the input is transformed, and constant_value will not recognize the true value of rate=0 (since it runs before constant folding).

RELNOTES: Dropout now takes `rate` argument, keep_prob is deprecated.
PiperOrigin-RevId: 223609371
---
 tensorflow/python/ops/nn_ops.py               | 58 ++++++++-----------
 tensorflow/python/ops/nn_test.py              | 20 +++++--
 .../tools/api/golden/v1/tensorflow.nn.pbtxt   |  2 +-
 3 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 91d292ab69..cec532ea86 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2915,11 +2915,15 @@ def _get_noise_shape(x, noise_shape):
 
 
 @tf_export(v1=["nn.dropout"])
-def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):  # pylint: disable=invalid-name
+@deprecation.deprecated_args(None, "Please use `rate` instead of `keep_prob`. "
+                             "Rate should be set to `rate = 1 - keep_prob`.",
+                             "keep_prob")
+def dropout(x, keep_prob=None, noise_shape=None, seed=None, name=None,
+            rate=None):  # pylint: disable=invalid-name
   """Computes dropout.
 
-  With probability `keep_prob`, outputs the input element scaled up by
-  `1 / keep_prob`, otherwise outputs `0`.  The scaling is so that the expected
+  For each element of `x`, with probability `rate`, outputs `0`, and otherwise
+  scales up the input by `1 / (1-rate)`. The scaling is such that the expected
   sum is unchanged.
 
   By default, each element is kept or dropped independently.  If `noise_shape`
@@ -2932,48 +2936,34 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):  # pylint: di
 
   Args:
     x: A floating point tensor.
-    keep_prob: A scalar `Tensor` with the same type as x. The probability
-      that each element is kept.
+    keep_prob: (deprecated) A deprecated alias for `(1-rate)`.
     noise_shape: A 1-D `Tensor` of type `int32`, representing the
       shape for randomly generated keep/drop flags.
     seed: A Python integer. Used to create random seeds. See
-      `tf.set_random_seed`
-      for behavior.
+      `tf.set_random_seed` for behavior.
     name: A name for this operation (optional).
+    rate: A scalar `Tensor` with the same type as `x`. The probability that each
+      element of `x` is discarded.
 
   Returns:
     A Tensor of the same shape of `x`.
 
   Raises:
-    ValueError: If `keep_prob` is not in `(0, 1]` or if `x` is not a floating
+    ValueError: If `rate` is not in `[0, 1)` or if `x` is not a floating
       point tensor.
   """
-  with ops.name_scope(name, "dropout", [x]) as name:
-    x = ops.convert_to_tensor(x, name="x")
-    if not x.dtype.is_floating:
-      raise ValueError("x has to be a floating point tensor since it's going to"
-                       " be scaled. Got a %s tensor instead." % x.dtype)
-    if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1:
-      raise ValueError("keep_prob must be a scalar tensor or a float in the "
-                       "range (0, 1], got %g" % keep_prob)
-
-    # Early return if nothing needs to be dropped.
-    if isinstance(keep_prob, float) and keep_prob == 1:
-      return x
-    if context.executing_eagerly():
-      if isinstance(keep_prob, ops.EagerTensor):
-        if keep_prob.numpy() == 1:
-          return x
-    else:
-      keep_prob = ops.convert_to_tensor(
-          keep_prob, dtype=x.dtype, name="keep_prob")
-      keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
+  try:
+    keep = 1. - keep_prob if keep_prob is not None else None
+  except TypeError:
+    raise ValueError("keep_prob must be a floating point number or Tensor "
+                     "(got %r)" % keep_prob)
 
-      # Do nothing if we know keep_prob == 1
-      if tensor_util.constant_value(keep_prob) == 1:
-        return x
+  rate = deprecation.deprecated_argument_lookup(
+      "rate", rate,
+      "keep_prob", keep)
 
-    rate = 1 - keep_prob
+  if rate is None:
+    raise ValueError("You must provide a rate to dropout.")
 
   return dropout_v2(x, rate, noise_shape=noise_shape, seed=seed, name=name)
 
@@ -3018,12 +3008,12 @@ def dropout_v2(x, rate, noise_shape=None, seed=None, name=None):  # pylint: disa
     if not x.dtype.is_floating:
       raise ValueError("x has to be a floating point tensor since it's going to"
                        " be scaled. Got a %s tensor instead." % x.dtype)
-    if isinstance(rate, numbers.Real) and not 0 <= rate < 1:
+    if isinstance(rate, numbers.Real) and not (rate >= 0 and rate < 1):
       raise ValueError("rate must be a scalar tensor or a float in the "
                        "range [0, 1), got %g" % rate)
 
     # Early return if nothing needs to be dropped.
-    if isinstance(rate, float) and rate == 0:
+    if isinstance(rate, numbers.Real) and rate == 0:
       return x
     if context.executing_eagerly():
       if isinstance(rate, ops.EagerTensor):
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 92ca9deb1b..0336d0d27c 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -450,6 +450,17 @@ class DropoutTest(test_lib.TestCase):
     with self.assertRaises(ValueError):
       nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2]))
 
+  def testInvalidRate(self):
+    x_dim = 40
+    y_dim = 30
+    t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
+    with self.assertRaises(ValueError):
+      nn_ops.dropout_v2(t, -1.0)
+    with self.assertRaises(ValueError):
+      nn_ops.dropout_v2(t, 1.1)
+    with self.assertRaises(ValueError):
+      nn_ops.dropout_v2(t, [0.0, 1.0])
+
   @test_util.run_deprecated_v1
   def testShapedDropoutShapeError(self):
     # Runs shaped dropout and verifies an error is thrown on misshapen noise.
@@ -471,12 +482,13 @@ class DropoutTest(test_lib.TestCase):
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[1, 1])
 
-  @test_util.run_deprecated_v1
   def testNoDropoutFast(self):
     x = array_ops.zeros((5,))
-    for p in 1, constant_op.constant(1.0):
-      y = nn_ops.dropout(x, keep_prob=p)
-      self.assertTrue(x is y)
+    y = nn_ops.dropout(x, keep_prob=1)
+    self.assertTrue(x is y)
+
+    y = nn_ops.dropout_v2(x, rate=0)
+    self.assertTrue(x is y)
 
   def testDropoutWithIntegerInputs(self):
     x = constant_op.constant([1, 1, 1, 1, 1])
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 560b492fe9..3697912ec2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -154,7 +154,7 @@ tf_module {
   }
   member_method {
     name: "dropout"
-    argspec: "args=[\'x\', \'keep_prob\', \'noise_shape\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'x\', \'keep_prob\', \'noise_shape\', \'seed\', \'name\', \'rate\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "dynamic_rnn"
-- 
GitLab


From fe9a9dbb8d0ef118b15beb4724f256190fd04d13 Mon Sep 17 00:00:00 2001
From: "Wen-Heng (Jack) Chung" <whchung@gmail.com>
Date: Fri, 30 Nov 2018 16:30:48 -0600
Subject: [PATCH 1147/1554] Split convolution invocation into preparation and
 actual invocation

- split DoConvolve into:

PrepareForConvolution
DoConvolve

- split DoConvolveBackwardData into:

PrepareForConvolutionBackwardData
DoConvolveBackwardData

- split DoConvolveBackwardFilter into:

PrepareForConvolutionBackwardFilter
DoConvolveBackwardFilter

PrepareForConvolutionXXX would allocate scratch memory.
DoConolveXXX would invoke actual convolution algorithms.

Implement forward convoution, backward input convolution, backward filter
convolution on CUDA path.
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 459 +++++++++++++++-----
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 196 ++++++++-
 tensorflow/stream_executor/dnn.h            | 180 ++++++--
 tensorflow/stream_executor/stream.cc        | 172 ++++++--
 4 files changed, 827 insertions(+), 180 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 1f2e2f48bb..387afefc21 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2380,7 +2380,7 @@ bool ShouldIncludeWinogradNonfusedAlgo(
 }  // namespace
 
 template <class T>
-port::Status CudnnSupport::DoConvolveImpl(
+port::Status CudnnSupport::PrepareForConvolutionImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -2389,6 +2389,34 @@ port::Status CudnnSupport::DoConvolveImpl(
     const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
     dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
+  CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  SE_ASSIGN_OR_RETURN(*algorithm_desc,
+                      GetCudnnConvolutionForwardAlgorithm(
+                          stream, cudnn, algorithm_config, input_nd, filter,
+                          conv, output_nd, scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveImpl(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<T>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<T>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
+    dnn::DataType accumulator_type, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
@@ -2411,12 +2439,6 @@ port::Status CudnnSupport::DoConvolveImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionForwardAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, output_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -2432,7 +2454,7 @@ port::Status CudnnSupport::DoConvolveImpl(
   // memory. See nvbugs/2138754, b/80018418.
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
+      if (algorithm_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (input_descriptor.ndims() < 3) {
@@ -2457,7 +2479,8 @@ port::Status CudnnSupport::DoConvolveImpl(
     }());
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -2469,18 +2492,19 @@ port::Status CudnnSupport::DoConvolveImpl(
       /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(),
       /*srcData=*/input_data.opaque(), /*filterDesc=*/filter.handle(),
       /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvForwardAlgo(algo_desc), /*workSpace=*/scratch.opaque(),
-      /*workSpaceSizeInBytes=*/scratch.size(), /*beta=*/beta,
+      /*algo=*/ToConvForwardAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(), /*beta=*/beta,
       /*yDesc=*/output_nd.handle(), /*y=*/output_data->opaque()));
 
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
   }
 
   return port::Status::OK();
@@ -2877,7 +2901,7 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolve(
+bool CudnnSupport::PrepareForConvolution(
     Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -2886,12 +2910,70 @@ bool CudnnSupport::DoConvolve(
     const dnn::BatchDescriptor& output_descriptor,
     DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(PrepareForConvolutionImpl<float>(
+                        stream, batch_descriptor, input_data, filter_descriptor,
+                        filter_data, convolution_descriptor, output_descriptor,
+                        output_data, dnn::DataType::kFloat, scratch_allocator,
+                        algorithm_config, algorithm_desc, scratch_memory),
+                    /*report_error=*/true);
+}
+
+bool CudnnSupport::PrepareForConvolution(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<double>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<double>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(PrepareForConvolutionImpl<double>(
+                        stream, batch_descriptor, input_data, filter_descriptor,
+                        filter_data, convolution_descriptor, output_descriptor,
+                        output_data, dnn::DataType::kDouble, scratch_allocator,
+                        algorithm_config, algorithm_desc, scratch_memory),
+                    /*report_error=*/true);
+}
+
+bool CudnnSupport::PrepareForConvolution(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<Eigen::half>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<Eigen::half>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(
+      PrepareForConvolutionImpl<Eigen::half>(
+          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output_data, acc_type,
+          scratch_allocator, algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolve(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<float>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<float>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kFloat, scratch_allocator,
-                     algorithm_config, output_profile_result),
+                     output_data, dnn::DataType::kFloat, algorithm_desc,
+                     scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -2902,14 +2984,14 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<double>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
+    DeviceMemory<double>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kDouble, scratch_allocator,
-                     algorithm_config, output_profile_result),
+                     output_data, dnn::DataType::kDouble, algorithm_desc,
+                     scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -2920,8 +3002,9 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
+    DeviceMemory<Eigen::half>* output_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
@@ -2930,7 +3013,7 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, acc_type, scratch_allocator, algorithm_config,
+                     output_data, acc_type, algorithm_desc, scratch_memory,
                      output_profile_result),
       /*report_error=*/!output_profile_result);
 }
@@ -3066,7 +3149,7 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
 }
 
 template <class T>
-port::Status CudnnSupport::DoConvolveBackwardDataImpl(
+port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<T>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3076,6 +3159,36 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
+  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  SE_ASSIGN_OR_RETURN(
+      *algorithm_desc,
+      GetCudnnConvolutionBackwardDataAlgorithm(
+          stream, cudnn, algorithm_config, in_back_nd, filter, conv,
+          out_back_nd, scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveBackwardDataImpl(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<T>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<T> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3099,12 +3212,6 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionBackwardDataAlgorithm(
-                          stream, cudnn, algorithm_config, in_back_nd, filter,
-                          conv, out_back_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3116,7 +3223,8 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     }
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3126,44 +3234,44 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
   // zero-initialized, nvbugs/2254619.
   if (CUDNN_VERSION >= 7000 && CUDNN_VERSION < 7300 &&
-      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-      cudnn_type == CUDNN_DATA_HALF && algo_desc.tensor_ops_enabled() &&
+      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
+      cudnn_type == CUDNN_DATA_HALF && algorithm_desc.tensor_ops_enabled() &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputInputYX &&
       output_descriptor.layout() == dnn::DataLayout::kBatchDepthYX &&
       (convolution_descriptor.vertical_filter_stride() > 1 ||
        convolution_descriptor.horizontal_filter_stride() > 1)) {
-    stream->ThenMemZero(&scratch, scratch.size());
+    stream->ThenMemZero(scratch_memory, scratch_memory->size());
   }
 
-  RETURN_IF_CUDNN_ERROR(
-      cudnnConvolutionBackwardData(cudnn.handle(),
-                                   /*alpha=*/alpha,
-                                   /*wDesc=*/filter.handle(),
-                                   /*w=*/filter_data.opaque(),
-                                   /*dyDesc=*/out_back_nd.handle(),
-                                   /*dy=*/backward_output_data.opaque(),
-                                   /*convDesc=*/conv.handle(),
-                                   /*algo=*/ToConvBackwardDataAlgo(algo_desc),
-                                   /*workSpace=*/scratch.opaque(),
-                                   /*workSpaceSizeInBytes=*/scratch.size(),
-                                   /*beta=*/beta,
-                                   /*dxDesc=*/in_back_nd.handle(),
-                                   /*dx=*/backward_input_data->opaque()));
+  RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardData(
+      cudnn.handle(),
+      /*alpha=*/alpha,
+      /*wDesc=*/filter.handle(),
+      /*w=*/filter_data.opaque(),
+      /*dyDesc=*/out_back_nd.handle(),
+      /*dy=*/backward_output_data.opaque(),
+      /*convDesc=*/conv.handle(),
+      /*algo=*/ToConvBackwardDataAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(),
+      /*beta=*/beta,
+      /*dxDesc=*/in_back_nd.handle(),
+      /*dx=*/backward_input_data->opaque()));
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
   }
 
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<double>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3173,17 +3281,17 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<double>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<float>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3193,17 +3301,17 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<float>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3213,22 +3321,86 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<Eigen::half>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, acc_type, scratch_allocator, algorithm_config,
-          output_profile_result),
+          algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<double>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<double>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kDouble, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<float>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<float>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kFloat, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<Eigen::half>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<Eigen::half>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
+                                 output_descriptor, backward_output_data,
+                                 convolution_descriptor, input_descriptor,
+                                 backward_input_data, acc_type, algorithm_desc,
+                                 scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
 template <class T>
-port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
+port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3238,6 +3410,36 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
+  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  SE_ASSIGN_OR_RETURN(
+      *algorithm_desc,
+      GetCudnnConvolutionBackwardFilterAlgorithm(
+          stream, cudnn, algorithm_config, input_nd, filter, conv, out_back_nd,
+          scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<T>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<T> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3261,12 +3463,6 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionBackwardFilterAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, out_back_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3282,7 +3478,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   // results. See nvbugs/2072856
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
+      if (algorithm_desc.algo_id() !=
+          CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (output_descriptor.height() > 1 && output_descriptor.width() > 1) {
@@ -3308,7 +3505,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     }());
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3324,7 +3522,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   //
   // See nvbugs/2379553.
   if (CUDNN_VERSION >= 7100 && CUDNN_VERSION < 7300 &&
-      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
+      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
       cudnn_type == CUDNN_DATA_HALF &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
@@ -3342,9 +3540,9 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
       /*diffDesc=*/out_back_nd.handle(),
       /*diffData=*/backward_output_data.opaque(),
       /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardFilterAlgo(algo_desc),
-      /*workSpace=*/scratch.opaque(),
-      /*workSpaceSizeInBytes=*/scratch.size(),
+      /*algo=*/ToConvBackwardFilterAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(),
       /*beta=*/beta,
       /*gradDesc=*/filter.handle(),
       /*dw=*/backward_filter_data->opaque()));
@@ -3352,16 +3550,16 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
   }
 
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<double>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3371,18 +3569,17 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<double>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
+      PrepareForConvolutionBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble,
-
-          scratch_allocator, algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          backward_filter_data, dnn::DataType::kDouble, scratch_allocator,
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3392,18 +3589,17 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<float>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(DoConvolveBackwardFilterImpl(
-                        stream, input_descriptor, input_data, output_descriptor,
-                        backward_output_data, convolution_descriptor,
-                        filter_descriptor, backward_filter_data,
-
-                        dnn::DataType::kFloat, scratch_allocator,
-                        algorithm_config, output_profile_result),
-                    /*report_error=*/!output_profile_result);
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(
+      PrepareForConvolutionBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kFloat, scratch_allocator,
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<Eigen::half>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3413,20 +3609,83 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<Eigen::half>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
+      PrepareForConvolutionBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, acc_type, scratch_allocator, algorithm_config,
-          output_profile_result),
+          algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<double>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<double>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kDouble, algorithm_desc,
+          scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<float>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<float>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kFloat, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<Eigen::half>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<Eigen::half>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(DoConvolveBackwardFilterImpl(
+                        stream, input_descriptor, input_data, output_descriptor,
+                        backward_output_data, convolution_descriptor,
+                        filter_descriptor, backward_filter_data, acc_type,
+                        algorithm_desc, scratch_memory, output_profile_result),
+                    /*report_error=*/!output_profile_result);
+}
+
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardBiasImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 0641be140d..0b8f4d035c 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -252,6 +252,43 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<float>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<double>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half>* output_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
                   const DeviceMemory<float>& input_data,
                   const dnn::FilterDescriptor& filter_descriptor,
@@ -259,8 +296,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<float>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -270,8 +307,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<double>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -281,8 +318,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<Eigen::half>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoFusedConvolve(
@@ -384,7 +421,20 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool DoConvolveBackwardData(
+  bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<float>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolutionBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -394,6 +444,32 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_input_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool DoConvolveBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -404,8 +480,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -416,11 +492,11 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<double>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -430,9 +506,10 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::ProfileResult* output_profile_result) override;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<float>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -442,9 +519,10 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::ProfileResult* output_profile_result) override;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<Eigen::half>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -454,6 +532,43 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<double>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<double>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<float>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<float>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<Eigen::half>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardBias(
@@ -663,8 +778,8 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<U>* offset_backprop);
 
   template <class T>
-  port::Status DoConvolveImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+  port::Status PrepareForConvolutionImpl(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
       const DeviceMemory<T>& input_data,
       const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<T>& filter_data,
@@ -673,6 +788,19 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveImpl(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<T>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<T>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <typename ElementType, typename BiasType, typename ScaleType>
@@ -693,7 +821,7 @@ class CudnnSupport : public dnn::DnnSupport {
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
-  port::Status DoConvolveBackwardDataImpl(
+  port::Status PrepareForConvolutionBackwardDataImpl(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<T>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -703,19 +831,45 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveBackwardDataImpl(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<T>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
-  port::Status DoConvolveBackwardFilterImpl(
+  port::Status PrepareForConvolutionBackwardFilterImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor_in,
       DeviceMemory<T> backward_output_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveBackwardFilterImpl(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<T>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index c044a356ef..322232d263 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -730,6 +730,7 @@ class PoolingDescriptor {
 class AlgorithmDesc {
  public:
   typedef int64 Index;
+  AlgorithmDesc() : AlgorithmDesc(0, false) {}
   AlgorithmDesc(Index a, bool use_tensor_ops) {
     proto_.set_algo_id(a);
     proto_.set_math_type(use_tensor_ops ? AlgorithmProto::TENSOR_OP_MATH
@@ -1175,6 +1176,43 @@ class DnnSupport {
     return false;
   }
 
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<float>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<double>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half>* output_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
   // Enqueues a single-precision convolution operation onto the stream.
   //
   // Arguments (all borrowed):
@@ -1188,10 +1226,10 @@ class DnnSupport {
   //  output_descriptor: dimensions of the output layer.
   //  output_data: un-owned device memory region in which to place the
   //    convolution result.
-  //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
-  //    space in order to speed up the convolution operation.
-  //  algorithm_config: specifies which algorithm should be used for the
+  //  algorithm_desc: specifies which algorithm should be used for the
   //    operation.
+  //  scratch: un-owned device memory for scratch space in order to speed up
+  //    the convolution operation.
   //  output_profile_result: the output profile result for this call. The
   //    profiling is only enabled when this is not nullptr.
   //
@@ -1216,8 +1254,9 @@ class DnnSupport {
       const DeviceMemory<float>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      DeviceMemory<float>* output_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a double-precision convolution operation onto the stream.
@@ -1229,8 +1268,9 @@ class DnnSupport {
       const DeviceMemory<double>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      DeviceMemory<double>* output_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) = 0;
 
   // Enqueues a half-precision convolution operation onto the stream.
@@ -1243,8 +1283,8 @@ class DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the forward convolution pass.
@@ -1300,6 +1340,45 @@ class DnnSupport {
       const BatchDescriptor& output_descriptor,
       DeviceMemory<float>* output_data) = 0;
 
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<float>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
   // Enqueues a single-precision backward convolution (for data) operation onto
   // the stream.
   //
@@ -1319,15 +1398,15 @@ class DnnSupport {
   //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
   //    space in order to speed up the convolution operation.
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<float>& filter_data,
-      const BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<float> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1337,28 +1416,67 @@ class DnnSupport {
       std::vector<AlgorithmDesc>* out_algorithms);
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
-      const BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<double> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      ProfileResult* output_profile_result) = 0;
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<float>& input_data,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<float>* backward_filter_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<double>& input_data,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<double>* backward_filter_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
       const BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half> backward_output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      ProfileResult* output_profile_result) = 0;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) = 0;
 
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
@@ -1387,8 +1505,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1405,8 +1523,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardFilter(
@@ -1417,8 +1535,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a single-precision backward convolution (for bias) operation onto
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 3edc66cde8..4503127bee 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -549,11 +549,16 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(),
-          /*output_profile_result=*/nullptr));
+          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolve(
+          this, input_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output, algorithm_desc,
+          &scratch_memory, nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -576,11 +581,16 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(),
-          /*output_profile_result=*/nullptr));
+          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolve(
+          this, input_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output, algorithm_desc,
+          &scratch_memory, nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -758,10 +768,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -789,10 +807,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -820,10 +846,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -969,10 +1003,17 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardData(
+          this, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -999,11 +1040,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1032,11 +1082,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1065,11 +1124,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1096,10 +1164,17 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardData(
+          this, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1138,10 +1213,17 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardFilter(
+          this, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1168,11 +1250,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1201,11 +1292,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1232,10 +1332,17 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardFilter(
+          this, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1262,11 +1369,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
-- 
GitLab


From e9158f0268e71b1c5e3384e6552e6083263aa4a8 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 30 Nov 2018 19:38:06 -0800
Subject: [PATCH 1148/1554] [XLA:GPU] (Mostly) Disable fast-math optimizations

We remain performance neutral on the set of tracked benchmarks by allowing
fast-math like optimizations in two places:

 1. We allow LLVM to generate div.approx when necessary by setting
    -nvptx-prec-divf32=0

 2. By default we emit llvm.min/llvm.max that has non IEEE behavior with NaNs.

PiperOrigin-RevId: 223610526
---
 tensorflow/compiler/xla/debug_options_flags.cc       | 12 ++++++------
 .../compiler/xla/service/gpu/elemental_ir_emitter.cc | 10 ++++++++++
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc    |  3 ---
 .../gpu/llvm_gpu_backend/nvptx_backend_lib.cc        | 10 +++-------
 .../compiler/xla/service/gpu/tests/gpu_ftz_test.cc   | 12 ++++++------
 .../compiler/xla/tests/client_library_test_base.h    |  2 +-
 tensorflow/compiler/xla/xla.proto                    |  6 +++++-
 7 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/debug_options_flags.cc b/tensorflow/compiler/xla/debug_options_flags.cc
index a40330a9b1..d7e7b9e621 100644
--- a/tensorflow/compiler/xla/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/debug_options_flags.cc
@@ -54,7 +54,7 @@ void SetDebugOptionsDefaults(DebugOptions* flags) {
   // TODO(jlebar): Disable fastmath once doing so is not a performance
   // regression.
   flags->set_xla_cpu_enable_fast_math(true);
-  flags->set_xla_gpu_enable_fast_math(true);
+  flags->set_xla_gpu_enable_fast_min_max(true);
 
   flags->set_xla_force_host_platform_device_count(1);
 }
@@ -160,11 +160,11 @@ void AllocateFlags() {
           "Enable unsafe fast-math optimizations in the CPU compiler; "
           "this may produce faster code at the expense of some accuracy."),
       tensorflow::Flag(
-          "xla_gpu_enable_fast_math",
-          bool_setter_for(&DebugOptions::set_xla_cpu_enable_fast_math),
-          flag_values->xla_cpu_enable_fast_math(),
-          "Enable unsafe fast-math optimizations in the GPU compiler; "
-          "this may produce faster code at the expense of some accuracy."),
+          "xla_gpu_enable_fast_min_max",
+          bool_setter_for(&DebugOptions::set_xla_gpu_enable_fast_min_max),
+          flag_values->xla_gpu_enable_fast_min_max(),
+          "Enable fast floating point min/max lowering that does not propagate "
+          "NaNs."),
       tensorflow::Flag(
           "xla_llvm_enable_alias_scope_metadata",
           bool_setter_for(
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
index 6dcdaf1cfe..2ab754a471 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
@@ -161,6 +161,16 @@ StatusOr<llvm::Value*> GpuElementalIrEmitter::EmitFloatBinaryOp(
   PrimitiveType lhs_input_type = op->operand(0)->shape().element_type();
   PrimitiveType rhs_input_type = op->operand(1)->shape().element_type();
   PrimitiveType output_type = op->shape().element_type();
+  HloOpcode opcode = op->opcode();
+
+  if (hlo_module_config_.debug_options().xla_gpu_enable_fast_min_max() &&
+      (opcode == HloOpcode::kMaximum || opcode == HloOpcode::kMinimum)) {
+    return llvm_ir::EmitCallToIntrinsic(
+        opcode == HloOpcode::kMaximum ? llvm::Intrinsic::maxnum
+                                      : llvm::Intrinsic::minnum,
+        {lhs_value, rhs_value}, {lhs_value->getType()}, b_);
+  }
+
   switch (op->opcode()) {
     case HloOpcode::kRemainder: {
       return EmitLibdeviceMathCall("__nv_fmod", {lhs_value, rhs_value},
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index 31591914cc..6693f66d62 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -63,9 +63,6 @@ IrEmitter::IrEmitter(const HloModuleConfig& hlo_module_config,
                 &ir_emitter_context->buffer_assignment(), &b_, module_,
                 is_nested),
       hlo_module_config_(hlo_module_config) {
-  b_.setFastMathFlags(llvm_ir::GetFastMathFlags(
-      /*fast_math_enabled=*/hlo_module_config.debug_options()
-          .xla_gpu_enable_fast_math()));
 }
 
 Status IrEmitter::DefaultAction(HloInstruction* hlo) {
diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
index 364f69a69d..24f07e6897 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
@@ -177,13 +177,6 @@ std::unique_ptr<llvm::TargetMachine> GetTargetMachine(
   }
 
   TargetOptions target_options = InitTargetOptionsFromCodeGenFlags();
-  llvm_ir::SetTargetOptions(
-      /*fast_math_enabled=*/hlo_module_config.debug_options()
-          .xla_gpu_enable_fast_math(),
-      &target_options);
-
-  // Enable FMA synthesis.
-  target_options.AllowFPOpFusion = FPOpFusion::Fast;
 
   // Set the verbose assembly options.
   target_options.MCOptions.AsmVerbose = false;
@@ -465,6 +458,9 @@ void GPUBackendInit(const HloModuleConfig& hlo_module_config) {
   // between those loads.
   FeedLLVMWithFlags({"-memdep-block-scan-limit=500"});
 
+  // Use div.approx -- it matters for some float-division heavy benchmarks.
+  FeedLLVMWithFlags({"-nvptx-prec-divf32=0"});
+
   llvm_ir::InitializeLLVMCommandLineOptions(hlo_module_config);
 
   // Initialize the NVPTX target; it's the only target we link with, so call its
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc
index d0ccd8619b..5e524faab1 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc
@@ -75,16 +75,16 @@ class GpuFtzDisabledTest : public GpuFtzTest {
 // Check that we emit mul.ftz.f32 when in ftz mode, and plain mul.f32 otherwise.
 TEST_F(GpuFtzEnabledTest, MultiplyFtz) {
   CompileAndVerifyPtx(CreateBinaryOpModule(HloOpcode::kMultiply), R"(
-    CHECK-NOT: mul.f32
-    CHECK: mul.ftz.f32
-    CHECK-NOT: mul.f32
+    CHECK-NOT: mul.rn.f32
+    CHECK: mul.rn.ftz.f32
+    CHECK-NOT: mul.rn.f32
   )");
 }
 TEST_F(GpuFtzDisabledTest, MultiplyFtz) {
   CompileAndVerifyPtx(CreateBinaryOpModule(HloOpcode::kMultiply), R"(
-    CHECK-NOT: mul.ftz.f32
-    CHECK: mul.f32
-    CHECK-NOT: mul.ftz.f32
+    CHECK-NOT: mul.rn.ftz.f32
+    CHECK: mul.rn.f32
+    CHECK-NOT: mul.rn.ftz.f32
   )");
 }
 
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 34148e5886..65a23dd883 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -76,7 +76,7 @@ class ClientLibraryTestBase : public ::testing::Test {
   void SetFastMathDisabled(bool disabled) {
     auto* opts = execution_options_.mutable_debug_options();
     opts->set_xla_cpu_enable_fast_math(!disabled);
-    opts->set_xla_gpu_enable_fast_math(!disabled);
+    opts->set_xla_gpu_enable_fast_min_max(!disabled);
   }
 
   void SetSeed(uint64 seed) { execution_options_.set_seed(seed); }
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index f745fb8506..bdeb1728fa 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -193,7 +193,11 @@ message DebugOptions {
   //  - Assuming that operations never produce or consume NaN or +/- Inf.
   //  - Assuming that +0 and -0 are indistinguishable.
   bool xla_cpu_enable_fast_math = 99;
-  bool xla_gpu_enable_fast_math = 100;
+
+  // When true we lower the Minimum and Maximum hlos in the GPU backend such
+  // that Min(NotNaN, NaN) = Min(NaN, NotNaN) = NotNaN.  In other words, if flag
+  // this is true we don't propagate NaNs through Min and Max.
+  bool xla_gpu_enable_fast_min_max = 100;
 
   // Crashes the program when any kind of verification fails, instead of just
   // logging the failures. One example is cross checking of convolution results
-- 
GitLab


From b69dce1d9fd31ed88abff8bf8bbbf87023655533 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 30 Nov 2018 20:31:37 -0800
Subject: [PATCH 1149/1554] Rename the unified_rnn_test to unified_lstm_test,
 since it only test the LSTM layer.

We will probably have new test for unified_gru later.

PiperOrigin-RevId: 223613402
---
 tensorflow/python/keras/BUILD                             | 4 ++--
 .../layers/{unified_rnn_test.py => unified_lstm_test.py}  | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)
 rename tensorflow/python/keras/layers/{unified_rnn_test.py => unified_lstm_test.py} (99%)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 7eee8a8bc8..228f47961c 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -515,9 +515,9 @@ py_test(
 )
 
 cuda_py_test(
-    name = "unified_rnn_test",
+    name = "unified_lstm_test",
     size = "medium",
-    srcs = ["layers/unified_rnn_test.py"],
+    srcs = ["layers/unified_lstm_test.py"],
     additional_deps = [
         ":keras",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/keras/layers/unified_rnn_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
similarity index 99%
rename from tensorflow/python/keras/layers/unified_rnn_test.py
rename to tensorflow/python/keras/layers/unified_lstm_test.py
index fd94bc9c2f..9bd1062ec6 100644
--- a/tensorflow/python/keras/layers/unified_rnn_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -51,10 +51,10 @@ _graph_options = config_pb2.GraphOptions(rewrite_options=_rewrites)
 _config = config_pb2.ConfigProto(graph_options=_graph_options)
 
 
-class UnifiedRNNTest(test.TestCase):
+class UnifiedLSTMTest(test.TestCase):
 
   @test_util.run_deprecated_v1
-  def test_unifiedRNN(self):
+  def test_unifiedLSTM(self):
     input_shape = 10
     rnn_state_size = 8
     output_shape = 8
@@ -99,7 +99,7 @@ class UnifiedRNNTest(test.TestCase):
         existing_loss = loss_value
 
   @test_util.run_deprecated_v1
-  def test_unifiedRNN_with_cond(self):
+  def test_unifiedLSTM_with_cond(self):
     # This test is to demonstrate the graph rewrite of grappler plugin under
     # the condition that the function returns different number of internal
     # states.
@@ -590,7 +590,7 @@ class LSTMLayerGraphOnlyTest(test.TestCase):
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
 
-class UnifiedRNNPerformanceTest(test.TestCase):
+class UnifiedLSTMPerformanceTest(test.TestCase):
 
   def _measure_performance(self, test_config, model, x_train, y_train):
     batch = test_config['batch']
-- 
GitLab


From 0a8e110e8e4558fd99244316051ce0ff729119e9 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Sat, 1 Dec 2018 00:52:39 -0800
Subject: [PATCH 1150/1554] Use Keras v2 optimizer for TPUStrategy in
 keras_test.py.

PiperOrigin-RevId: 223625692
---
 tensorflow/contrib/distribute/python/keras_test.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 1d00281974..435f10358f 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1267,17 +1267,9 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         # We have initialized the model to the same weight for the distribution
         # and non-distribution run.
         model.set_weights(initial_weights)
-        # TODO(b/120245072): Also use gradient_descent_keras.SGD for
-        # TPUStrategy.
-        # pylint: disable=line-too-long
-        if with_distribution and with_distribution.__class__.__name__ == 'TPUStrategy':
-        # pylint: enable=line-too-long
-          optimizer = gradient_descent.GradientDescentOptimizer(0.5)
-        else:
-          optimizer = gradient_descent_keras.SGD(0.5)
         model.compile(
             loss=keras.losses.mean_squared_error,
-            optimizer=optimizer,
+            optimizer=gradient_descent_keras.SGD(0.5),
             distribute=with_distribution)
 
         training_inputs, eval_inputs, predict_inputs = (
-- 
GitLab


From 6866480d9f97d14c6fc5e2488e396181be3b43c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 1 Dec 2018 01:02:22 -0800
Subject: [PATCH 1151/1554] compat: Update forward compatibility horizon to
 2018-12-01

PiperOrigin-RevId: 223626306
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 156680e081..a7e8bef670 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 11, 30)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 1)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From d15b49732f9bec02571f20b2c8c3a0b62d9888fb Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Sat, 1 Dec 2018 01:38:43 -0800
Subject: [PATCH 1152/1554] Fix defaulting of axis in TF 2.0 softmax/entropy
 code.

PiperOrigin-RevId: 223628497
---
 tensorflow/python/ops/nn_ops.py                    | 4 +++-
 tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index cec532ea86..225904854f 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2365,7 +2365,7 @@ def softmax_cross_entropy_with_logits_v2(labels, logits, axis=-1, name=None):
 @tf_export(v1=["nn.softmax_cross_entropy_with_logits_v2"])
 @deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def softmax_cross_entropy_with_logits_v2_helper(
-    labels, logits, axis=-1, name=None, dim=None):
+    labels, logits, axis=None, name=None, dim=None):
   """Computes softmax cross entropy between `logits` and `labels`.
 
   Measures the probability error in discrete classification tasks in which the
@@ -2419,6 +2419,8 @@ def softmax_cross_entropy_with_logits_v2_helper(
   # results.
   axis = deprecated_argument_lookup("axis", axis, "dim", dim)
   del dim
+  if axis is None:
+    axis = -1
 
   with ops.name_scope(name, "softmax_cross_entropy_with_logits",
                       [logits, labels]) as name:
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
index 3697912ec2..40e20f8c91 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.pbtxt
@@ -326,7 +326,7 @@ tf_module {
   }
   member_method {
     name: "softmax_cross_entropy_with_logits_v2"
-    argspec: "args=[\'labels\', \'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'-1\', \'None\', \'None\'], "
+    argspec: "args=[\'labels\', \'logits\', \'axis\', \'name\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "softplus"
-- 
GitLab


From 35c19c850a2c0cfb13cda56a493fa9163936d66c Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Sat, 1 Dec 2018 07:59:21 -0800
Subject: [PATCH 1153/1554] Java tests: Fix nightly builds.

For reasons not quite clear to me, the nightly builds (which are currently
using bazel 0.16.1) see failing Java tests with errors like:
  java.lang.NoSuchMethodError: java.nio.DoubleBuffer.flip()Ljava/nio/DoubleBuffer;
when calling "flip()" or "clear()" on a java.nio.DoubleBuffer object.

This is symptomatic of cross compiling in Java (e.g.
javac -source=1.9 -target=1.8) and this change works around that problem.

This does not happen with bazel 0.17.0 on the same test machine.
(Bringing me to the "reasons not quite clear to me" mentioned above).

PiperOrigin-RevId: 223643679
---
 .../test/java/org/tensorflow/TensorTest.java  | 46 ++++++++++++++-----
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java b/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
index 1bd00a763d..3229cce277 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.DoubleBuffer;
@@ -100,7 +101,7 @@ public class TensorTest {
                     : ByteOrder.LITTLE_ENDIAN)
             .asDoubleBuffer()
             .put(doubles);
-    buf.flip();
+    flipBuffer(buf);
     try (Tensor<Double> t = Tensor.create(new long[] {doubles.length}, buf)) {
       double[] actual = new double[doubles.length];
       assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
@@ -179,30 +180,30 @@ public class TensorTest {
       {
         ByteBuffer bbuf = ByteBuffer.allocate(1024).order(ByteOrder.nativeOrder());
 
-        bbuf.clear(); // FLOAT
+        clearBuffer(bbuf); // FLOAT
         tfloats.writeTo(bbuf);
         assertEquals(tfloats.numBytes(), bbuf.position());
-        bbuf.flip();
+        flipBuffer(bbuf);
         assertEquals(floats[0], bbuf.asFloatBuffer().get(0), EPSILON);
-        bbuf.clear(); // DOUBLE
+        clearBuffer(bbuf); // DOUBLE
         tdoubles.writeTo(bbuf);
         assertEquals(tdoubles.numBytes(), bbuf.position());
-        bbuf.flip();
+        flipBuffer(bbuf);
         assertEquals(doubles[0], bbuf.asDoubleBuffer().get(0), EPSILON);
-        bbuf.clear(); // INT32
+        clearBuffer(bbuf); // INT32
         tints.writeTo(bbuf);
         assertEquals(tints.numBytes(), bbuf.position());
-        bbuf.flip();
+        flipBuffer(bbuf);
         assertEquals(ints[0], bbuf.asIntBuffer().get(0));
-        bbuf.clear(); // INT64
+        clearBuffer(bbuf); // INT64
         tlongs.writeTo(bbuf);
         assertEquals(tlongs.numBytes(), bbuf.position());
-        bbuf.flip();
+        flipBuffer(bbuf);
         assertEquals(longs[0], bbuf.asLongBuffer().get(0));
-        bbuf.clear(); // BOOL
+        clearBuffer(bbuf); // BOOL
         tbools.writeTo(bbuf);
         assertEquals(tbools.numBytes(), bbuf.position());
-        bbuf.flip();
+        flipBuffer(bbuf);
         assertEquals(bools[0], bbuf.get(0) != 0);
       }
 
@@ -254,7 +255,7 @@ public class TensorTest {
                         : ByteOrder.LITTLE_ENDIAN)
                 .asDoubleBuffer();
         tdoubles.writeTo(foreignBuf);
-        foreignBuf.flip();
+        flipBuffer(foreignBuf);
         double[] actual = new double[foreignBuf.remaining()];
         foreignBuf.get(actual);
         assertArrayEquals(doubles, actual, EPSILON);
@@ -547,4 +548,25 @@ public class TensorTest {
       // expected.
     }
   }
+
+  // Workaround for cross compiliation
+  // (e.g., javac -source 1.9 -target 1.8).
+  //
+  // In Java 8 and prior, subclasses of java.nio.Buffer (e.g., java.nio.DoubleBuffer) inherited the
+  // "flip()" and "clear()" methods from java.nio.Buffer resulting in the signature:
+  //   Buffer flip();
+  // In Java 9 these subclasses had their own methods like:
+  //   DoubleBuffer flip();
+  // As a result, compiling for 1.9 source for a target of JDK 1.8 would result in errors at runtime
+  // like:
+  //
+  // java.lang.NoSuchMethodError: java.nio.DoubleBuffer.flip()Ljava/nio/DoubleBuffer
+  private static void flipBuffer(Buffer buf) {
+    buf.flip();
+  }
+
+  // See comment for flipBuffer()
+  private static void clearBuffer(Buffer buf) {
+    buf.clear();
+  }
 }
-- 
GitLab


From 0e672efe02940c803f7201d515c9ed0f36135e90 Mon Sep 17 00:00:00 2001
From: John Lin <johnlinp@gmail.com>
Date: Sun, 2 Dec 2018 00:04:22 +0800
Subject: [PATCH 1154/1554] Remove redundant word "the"

---
 .../tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile  | 2 +-
 tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile   | 2 +-
 tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile | 2 +-
 tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile         | 2 +-
 .../tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile  | 2 +-
 tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile   | 2 +-
 tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile | 2 +-
 tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile         | 2 +-
 tensorflow/tools/dockerfiles/spec.yml                           | 2 +-
 tensorflow/tools/docs/parser.py                                 | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
index 9577920e13..14ddf08199 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
index 2f4a3d6beb..16973b47af 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
index b663cca8e2..d8fabadec2 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu-jupyter.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
index 323f89155b..857b5e2047 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/cpu.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
index 17c1d5b904..9ecaec38c2 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
index bcac1f7015..c79bc3cf4c 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
index cb0eb5a3d3..acfe4d8607 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
index e8e6ceafe2..f36a21eaf0 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
@@ -16,7 +16,7 @@
 # THIS IS A GENERATED DOCKERFILE.
 #
 # This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the the TensorFlow dockerfiles documentation
+# throughout. Please refer to the TensorFlow dockerfiles documentation
 # for more information.
 
 ARG UBUNTU_VERSION=16.04
diff --git a/tensorflow/tools/dockerfiles/spec.yml b/tensorflow/tools/dockerfiles/spec.yml
index 5049e8dcfb..4826ddd8e2 100644
--- a/tensorflow/tools/dockerfiles/spec.yml
+++ b/tensorflow/tools/dockerfiles/spec.yml
@@ -17,7 +17,7 @@ header: |
     # THIS IS A GENERATED DOCKERFILE.
     #
     # This file was assembled from multiple pieces, whose use is documented
-    # throughout. Please refer to the the TensorFlow dockerfiles documentation
+    # throughout. Please refer to the TensorFlow dockerfiles documentation
     # for more information.
 
 # A combinatorial explosion of Docker images and Dockerfiles.
diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py
index 83b4bf8128..6dc18ee8dc 100644
--- a/tensorflow/tools/docs/parser.py
+++ b/tensorflow/tools/docs/parser.py
@@ -39,7 +39,7 @@ def is_free_function(py_object, full_name, index):
   """Check if input is a free function (and not a class- or static method).
 
   Args:
-    py_object: The the object in question.
+    py_object: The object in question.
     full_name: The full name of the object, like `tf.module.symbol`.
     index: The {full_name:py_object} dictionary for the public API.
 
-- 
GitLab


From d2c1774b55da71d0998435663ffab88cb58afc08 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 1 Dec 2018 09:41:49 -0800
Subject: [PATCH 1155/1554] Add a file containing all files that will not be
 used outside the OSS repository.

PiperOrigin-RevId: 223647591
---
 tensorflow/opensource_only.files | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 tensorflow/opensource_only.files

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
new file mode 100644
index 0000000000..8ff1645b98
--- /dev/null
+++ b/tensorflow/opensource_only.files
@@ -0,0 +1,17 @@
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
\ No newline at end of file
-- 
GitLab


From f1edf205b290c7fdeefe0b73d27c50a3cab67dcb Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Sat, 1 Dec 2018 10:10:16 -0800
Subject: [PATCH 1156/1554] [tf.data] Add internal method for testing whether a
 dataset has a `Dataset.repeat(None)`.

This utility function can be used to generate a warning when an indefinitely repeated `Dataset` is passed to an API that expects a finite `Dataset`.

PiperOrigin-RevId: 223648690
---
 .../python/data/experimental/__init__.py      |  2 +
 .../data/experimental/kernel_tests/BUILD      | 12 +++
 .../has_indefinite_repeat_test.py             | 53 ++++++++++++
 tensorflow/python/data/experimental/ops/BUILD | 11 +++
 .../experimental/ops/has_indefinite_repeat.py | 81 +++++++++++++++++++
 .../v1/tensorflow.data.experimental.pbtxt     |  4 +
 .../v2/tensorflow.data.experimental.pbtxt     |  4 +
 7 files changed, 167 insertions(+)
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py
 create mode 100644 tensorflow/python/data/experimental/ops/has_indefinite_repeat.py

diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 8a1048513a..a72dd379e1 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -35,6 +35,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@TFRecordWriter
 @@ThreadingOptions
 
+@@assume_finite
 @@bucket_by_sequence_length
 @@choose_from_datasets
 @@copy_to_device
@@ -82,6 +83,7 @@ from tensorflow.python.data.experimental.ops.grouping import bucket_by_sequence_
 from tensorflow.python.data.experimental.ops.grouping import group_by_reducer
 from tensorflow.python.data.experimental.ops.grouping import group_by_window
 from tensorflow.python.data.experimental.ops.grouping import Reducer
+from tensorflow.python.data.experimental.ops.has_indefinite_repeat import assume_finite
 from tensorflow.python.data.experimental.ops.interleave_ops import choose_from_datasets
 from tensorflow.python.data.experimental.ops.interleave_ops import parallel_interleave
 from tensorflow.python.data.experimental.ops.interleave_ops import sample_from_datasets
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index d7ca5a70e4..f2b2e2d0c1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -211,6 +211,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "has_indefinite_repeat_test",
+    srcs = ["has_indefinite_repeat_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python/data/experimental/ops:has_indefinite_repeat",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 py_test(
     name = "ignore_errors_test",
     srcs = ["ignore_errors_test.py"],
diff --git a/tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py
new file mode 100644
index 0000000000..6b2569c317
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py
@@ -0,0 +1,53 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental.has_indefinite_repeat()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.data.experimental.ops import has_indefinite_repeat
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class HasIndefiniteRepeat(test_base.DatasetTestBase, parameterized.TestCase):
+  """Tests for `tf.data.experimental.has_indefinite_repeat()`."""
+
+  @parameterized.named_parameters(
+      ("NoRepeat", dataset_ops.Dataset.range(10), False),
+      ("FiniteRepeat", dataset_ops.Dataset.range(10).repeat(2), False),
+      ("FiniteRepeatNotAtEnd", dataset_ops.Dataset.range(10).repeat(2).skip(1),
+       False),
+      ("InfiniteRepeat", dataset_ops.Dataset.range(10).repeat(), True),
+      ("InfiniteRepeatNotAtEnd", dataset_ops.Dataset.range(10).repeat().skip(1),
+       True),
+      ("InfiniteRepeatThenFiniteRepeat",
+       dataset_ops.Dataset.range(10).repeat().repeat(2), True),
+      ("ConcatenateFiniteAndInfinite",
+       dataset_ops.Dataset.range(10).repeat(2).concatenate(
+           dataset_ops.Dataset.range(10).repeat()), True),
+      ("AssumeFinite", dataset_ops.Dataset.range(10).repeat().apply(
+          has_indefinite_repeat.assume_finite()), False),
+  )
+  def testHasIndefiniteRepeat(self, dataset, expected_result):
+    self.assertEqual(
+        has_indefinite_repeat.has_indefinite_repeat(dataset), expected_result)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 50f5127833..c4a4781aad 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -28,6 +28,16 @@ py_library(
     ],
 )
 
+py_library(
+    name = "has_indefinite_repeat",
+    srcs = ["has_indefinite_repeat.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:tensor_util",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_library(
     name = "iterator_ops",
     srcs = [
@@ -428,6 +438,7 @@ py_library(
         ":filter_for_shard_ops",
         ":get_single_element",
         ":grouping",
+        ":has_indefinite_repeat",
         ":indexed_dataset_ops",
         ":interleave_ops",
         ":map_defun",
diff --git a/tensorflow/python/data/experimental/ops/has_indefinite_repeat.py b/tensorflow/python/data/experimental/ops/has_indefinite_repeat.py
new file mode 100644
index 0000000000..cf5e3f811d
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/has_indefinite_repeat.py
@@ -0,0 +1,81 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Finiteness analysis of `Dataset` objects."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.util.tf_export import tf_export
+
+
+class _AssumeFiniteDataset(dataset_ops.UnaryUnchangedStructureDataset):
+
+  def __init__(self, input_dataset):
+    super(_AssumeFiniteDataset, self).__init__(input_dataset)
+    self._input_dataset = input_dataset
+
+  def _as_variant_tensor(self):
+    return self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
+
+
+@tf_export("data.experimental.assume_finite")
+def assume_finite():
+  """Assume that the input is finite, even if it contains `Dataset.repeat()`.
+
+  Training libraries may analyze a `tf.data.Dataset` to determine if it is
+  finite or infinite (e.g. because it contains an indefinite
+  `tf.data.Dataset.repeat` transformation). Since that analysis may be
+  imprecise, this transformation allows the user to annotate a dataset
+  explicitly as being finite.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    `tf.data.Dataset.apply`.
+  """
+
+  def _apply_fn(dataset):
+    return _AssumeFiniteDataset(dataset)
+
+  return _apply_fn
+
+
+def has_indefinite_repeat(dataset):
+  """Returns `True` if `dataset` or any of its inputs is `Dataset.repeat()`.
+
+  NOTE: For simplicity, this analysis does not attempt to analyze nested
+  datasets (e.g. in a function passed to `tf.data.Dataset.flat_map`). If the
+  analysis is incorrect, you can apply `tf.data.experimental.assume_finite()`
+  to the dataset to override it.
+
+  Args:
+    dataset: A `tf.data.Dataset`.
+
+  Returns:
+    `True` if `dataset` or any of its inputs is repeated indefinitely.
+  """
+  # pylint: disable=protected-access
+  if isinstance(dataset, dataset_ops.DatasetV1Adapter):
+    return has_indefinite_repeat(dataset._dataset)
+  elif isinstance(dataset, dataset_ops.RepeatDataset):
+    count = tensor_util.constant_value(dataset._count)
+    return count == -1 or has_indefinite_repeat(dataset._inputs()[0])
+  elif isinstance(dataset, _AssumeFiniteDataset):
+    return False
+  else:
+    return any(
+        has_indefinite_repeat(input_dataset)
+        for input_dataset in dataset._inputs())
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index ad10b82283..27ea063b66 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -52,6 +52,10 @@ tf_module {
     name: "Counter"
     argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"<dtype: \'int64\'>\"], "
   }
+  member_method {
+    name: "assume_finite"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "bucket_by_sequence_length"
     argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index ad10b82283..27ea063b66 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -52,6 +52,10 @@ tf_module {
     name: "Counter"
     argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"<dtype: \'int64\'>\"], "
   }
+  member_method {
+    name: "assume_finite"
+    argspec: "args=[], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "bucket_by_sequence_length"
     argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
-- 
GitLab


From a4f06947c61ef6f69057f1207c2532a3551ff4f0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 1 Dec 2018 11:54:43 -0800
Subject: [PATCH 1157/1554] Make grouped convolutions use additional spatial
 dimension instead of a reduce window.

PiperOrigin-RevId: 223651863
---
 .../convolution_feature_group_converter.cc    | 154 ++++-------
 tensorflow/compiler/xla/tests/BUILD           |  26 ++
 .../xla/tests/grouped_convolution_test.cc     | 245 ++++++++++++++++++
 3 files changed, 319 insertions(+), 106 deletions(-)
 create mode 100644 tensorflow/compiler/xla/tests/grouped_convolution_test.cc

diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
index 09c3f32860..07d6680a72 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
@@ -271,130 +271,72 @@ Status ConvolutionVisitor::HandleConvolution(HloInstruction* convolution) {
         convolution, std::move(new_convolution)));
   } else {
     int64 activation_input_feature_dim = dim_numbers.input_feature_dimension();
-    auto activation = convolution->mutable_operand(0);
 
     int64 output_feature =
         filter->shape().dimensions(kernel_output_feature_dim);
 
-    int64 input_feature =
-        activation->shape().dimensions(activation_input_feature_dim);
-
     // If group_count == output_feature, then we map those grouped convolutions
-    // onto depthwise convolution + reduce. E.g., we would turn
+    // onto depthwise convolution. This is done by adding an additional spatial
+    // dimension to the activations, kernel, and the output.
+    // E.g., we would turn
     // [2, 12]{B, IF} conv [3, 4]{IF, OF} into
-    // [2, 12]{B, IF} depth conv [1, 12]{IF, OF}, and then use a reduce window
-    // of {1, 3} on the generated [2, 12] output to produce the final result of
-    // [2, 4].
+    // [3, 2, 4]{S, B, IF} depth conv [3, 1, 4]{S, IF, OF}, where S is the
+    // additional spatial dimension. The generated convolution output will be
+    // [1, 2, 4]{S, B, OF} and then reshape the output back to [2, 4] {B, OF}.
+
     if (group_count == output_feature && !filter_expansion_) {
-      Shape reshaped_filter_shape = filter->shape();
+      auto filter = convolution->mutable_operand(1);
+      auto activation = convolution->mutable_operand(0);
 
-      if (kernel_input_feature_dim < kernel_output_feature_dim) {
-        // Transpose IF and OF on the kernel.
-        std::vector<int64> filter_dims;
-        for (int64 i = 0; i < dim_numbers.kernel_spatial_dimensions().size();
-             ++i) {
-          filter_dims.push_back(dim_numbers.kernel_spatial_dimensions(i));
-        }
-        filter_dims.push_back(kernel_output_feature_dim);
-        filter_dims.push_back(kernel_input_feature_dim);
-
-        Shape transposed_filter = filter->shape();
-        auto& dimensions = *transposed_filter.mutable_dimensions();
-        std::swap(dimensions[kernel_input_feature_dim],
-                  dimensions[kernel_output_feature_dim]);
-
-        filter = add(HloInstruction::CreateTranspose(transposed_filter, filter,
-                                                     filter_dims));
-      } else {
-        // For depthwise convolutions, we want the kernel input feature
-        // dimension to be smaller than the output feature dimension. If that's
-        // not the case, we swap the dimensions.
+      // Add spatial dimension to the activation, and reshape.
+      Shape reshaped_activation_shape = activation->shape();
+      ShapeUtil::AppendMajorDimension(group_size, &reshaped_activation_shape);
 
-        auto& dimensions = *reshaped_filter_shape.mutable_dimensions();
-        std::swap(dimensions[kernel_input_feature_dim],
-                  dimensions[kernel_output_feature_dim]);
+      int64 new_spatial_dim = reshaped_activation_shape.dimensions().size() - 1;
 
-        dim_numbers.set_kernel_input_feature_dimension(
-            kernel_output_feature_dim);
+      reshaped_activation_shape.set_dimensions(activation_input_feature_dim,
+                                               group_count);
+      activation = add(
+          HloInstruction::CreateReshape(reshaped_activation_shape, activation));
 
-        dim_numbers.set_kernel_output_feature_dimension(
-            kernel_input_feature_dim);
-        std::swap(kernel_output_feature_dim, kernel_input_feature_dim);
-      }
+      // Add spatial dimension to the filter, and reshape.
+      Shape reshaped_filter_shape = filter->shape();
+      ShapeUtil::AppendMajorDimension(1, &reshaped_filter_shape);
 
-      reshaped_filter_shape.set_dimensions(kernel_input_feature_dim, 1);
-      reshaped_filter_shape.set_dimensions(kernel_output_feature_dim,
-                                           group_count * group_size);
-      auto reshaped_filter =
+      filter =
           add(HloInstruction::CreateReshape(reshaped_filter_shape, filter));
 
-      Shape reshaped_convolution_shape = convolution->shape();
-      reshaped_convolution_shape.set_dimensions(
-          dim_numbers.output_feature_dimension(), group_count * group_size);
-      auto new_convolution = add(HloInstruction::CreateConvolve(
-          reshaped_convolution_shape, convolution->mutable_operand(0),
-          reshaped_filter, /*feature_group_count=*/input_feature,
-          convolution->window(), dim_numbers, convolution->precision_config()));
-
-      // Create the reduce window.
-      Window window;
-      for (int64 i = 0; i < new_convolution->shape().dimensions_size(); ++i) {
-        auto* dim = window.add_dimensions();
-        dim->set_padding_low(0);
-        dim->set_padding_high(0);
-        dim->set_window_dilation(1);
-        dim->set_base_dilation(1);
-        if (i == dim_numbers.output_feature_dimension()) {
-          dim->set_stride(group_size);
-          dim->set_size(group_size);
-        } else {
-          dim->set_stride(1);
-          dim->set_size(1);
-        }
-      }
+      Shape new_output_shape = convolution->shape();
+      ShapeUtil::AppendMajorDimension(1, &new_output_shape);
+
+      // Edit convolution dimension numbers. Note that kernel_input_feature_dim
+      // now becomes a spatial dimension, and the newly added dimension of size
+      // 1 is the new kernel_input_feature_dim.
+      dim_numbers.add_input_spatial_dimensions(new_spatial_dim);
+      dim_numbers.add_kernel_spatial_dimensions(kernel_input_feature_dim);
+      dim_numbers.set_kernel_input_feature_dimension(new_spatial_dim);
+      dim_numbers.add_output_spatial_dimensions(new_spatial_dim);
+
+      // Add window for the new spatial dimension.
+      Window new_window = convolution->window();
+      auto* dim = new_window.add_dimensions();
+      dim->set_window_dilation(1);
+      dim->set_base_dilation(1);
+      dim->set_stride(1);
+      dim->set_size(group_size);
 
-      auto reduce_window_shape = new_convolution->shape();
-      reduce_window_shape.set_dimensions(dim_numbers.output_feature_dimension(),
-                                         group_count);
-
-      auto zero_literal = LiteralUtil::CreateR0(0.0f);
-      TF_ASSIGN_OR_RETURN(zero_literal, zero_literal.Convert(F32));
-      auto zero = add(HloInstruction::CreateConstant(std::move(zero_literal)));
-
-      auto reduce_function = [&]() -> HloComputation* {
-        HloComputation::Builder b("add_computation");
-        Shape shape = ShapeUtil::MakeShape(F32, {});
-        auto lhs =
-            b.AddInstruction(HloInstruction::CreateParameter(0, shape, "lhs"));
-        auto rhs =
-            b.AddInstruction(HloInstruction::CreateParameter(1, shape, "rhs"));
-        auto scalar_op = b.AddInstruction(
-            HloInstruction::CreateBinary(shape, HloOpcode::kAdd, lhs, rhs));
-        return computation_->parent()->AddEmbeddedComputation(
-            b.Build(scalar_op));
-      };
-
-      // Ensure that data input to reduce window is of type F32.
-      if (primitive_util::BitWidth(new_convolution->shape().element_type()) <
-          primitive_util::BitWidth(F32)) {
-        Shape convert_shape = new_convolution->shape();
-        convert_shape.set_element_type(F32);
-        new_convolution = add(HloInstruction::CreateBitcastConvert(
-            convert_shape, new_convolution));
-      }
-
-      auto reduce_window = add(HloInstruction::CreateReduceWindow(
-          reduce_window_shape, new_convolution, zero, window,
-          reduce_function()));
+      auto new_convolution = add(HloInstruction::CreateConvolve(
+          new_output_shape, activation, filter, group_count, new_window,
+          dim_numbers, convolution->precision_config()));
 
-      Shape convert_back_shape = reduce_window->shape();
-      convert_back_shape.set_element_type(activation->shape().element_type());
+      // Delete the extra spatial dimension, and reshape.
+      Shape reshaped_convolution_shape =
+          ShapeUtil::DeleteDimension(new_spatial_dim, new_convolution->shape());
+      auto reshaped_convolution = HloInstruction::CreateReshape(
+          reshaped_convolution_shape, new_convolution);
 
-      // Convert reduced data back to the original data type.
-      auto reduce_window_converted = HloInstruction::CreateBitcastConvert(
-          convert_back_shape, reduce_window);
       TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
-          convolution, std::move(reduce_window_converted)));
+          convolution, std::move(reshaped_convolution)));
 
     } else {
       // The filter expansion mechanism adds zeroes in the kernel.
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 2c18e2fd10..48bbc4e472 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -324,6 +324,32 @@ xla_test(
     ],
 )
 
+xla_test(
+    name = "grouped_convolution_test",
+    timeout = "long",
+    srcs = ["grouped_convolution_test.cc"],
+    blacklisted_backends = [
+        # disabled because of a break b/119590850.
+        "gpu",
+        # disabled because it times out.
+        "cpu",
+    ],
+    shard_count = 50,
+    deps = [
+        "//tensorflow/compiler/xla:execution_options_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/service:bfloat16_normalization",
+        "//tensorflow/compiler/xla/service:despecializer",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
 xla_test(
     name = "check_execution_arity_test",
     srcs = ["check_execution_arity_test.cc"],
diff --git a/tensorflow/compiler/xla/tests/grouped_convolution_test.cc b/tensorflow/compiler/xla/tests/grouped_convolution_test.cc
new file mode 100644
index 0000000000..8f7049910e
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/grouped_convolution_test.cc
@@ -0,0 +1,245 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/execution_options_util.h"
+#include "tensorflow/compiler/xla/service/bfloat16_normalization.h"
+#include "tensorflow/compiler/xla/service/despecializer.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+
+namespace xla {
+namespace {
+
+string GetFloatDataType(bool use_bfloat16) {
+  return use_bfloat16 ? "bf16" : "f32";
+}
+
+struct GroupedConvolution2DSpec {
+  int64 input_feature, output_feature, window, stride, pad, lhs_dilate;
+  int64 group_size, group_count;
+  std::vector<int64> activation_dims;
+  std::vector<int64> activation_layout;
+  std::vector<int64> kernel_dims;
+  std::vector<int64> kernel_layout;
+  std::vector<int64> output_dims;
+  std::vector<int64> output_layout;
+};
+
+class GroupedConvolution2DTest
+    : public HloTestBase,
+      public ::testing::WithParamInterface<
+          ::testing::tuple<GroupedConvolution2DSpec, bool>> {};
+
+static std::vector<GroupedConvolution2DSpec> GetConv2DTestCases() {
+  std::vector<GroupedConvolution2DSpec> config_set;
+  // Add to this set if you want a new test configuration.
+  // Rule : the penultimate number must be divisible by the last number.
+  std::vector<std::vector<int64>> config_options = {{8, 2, 2, 1, 1024, 128},
+                                                    {512, 3, 3, 144, 1024, 16},
+                                                    {256, 3, 3, 129, 512, 64},
+                                                    {64, 1, 2, 127, 32, 8},
+                                                    {256, 3, 3, 256, 1024, 4}};
+
+  for (auto option : config_options) {
+    int64 output_feature = option[0];
+    int64 activation_size = option[1];
+    int64 kernel_size = option[2];
+    int64 batch = option[3];
+    int64 input_feature = option[4];
+    int64 group_size = option[5];
+
+    std::vector<int64> kernel_layout = {3, 2, 1, 0};
+    GroupedConvolution2DSpec config;
+    config.group_size = group_size;
+    config.group_count = input_feature / group_size;
+    config.output_feature = output_feature;
+    config.window = kernel_size;
+
+    config.activation_dims = {batch, activation_size, activation_size,
+                              input_feature};
+    config.activation_layout = {3, 0, 2, 1};
+
+    config.kernel_dims = {kernel_size, kernel_size, group_size, output_feature};
+    config.kernel_layout = {3, 2, 1, 0};
+
+    if (activation_size == 1 && kernel_size == 2) {
+      // Test for outer dim.
+      config.output_dims = {batch, activation_size + kernel_size - 1,
+                            activation_size + kernel_size, output_feature};
+    } else if (output_feature == 256) {
+      // Restrict dilation-based tests only to one feature configuration.
+      config.stride = activation_size - 1;
+      config.pad = 0;
+      config.lhs_dilate = output_feature / 32;
+      config.output_dims = {batch, output_feature / 32,
+                            activation_size - kernel_size + 1, output_feature};
+    } else {
+      config.stride = config.pad = config.lhs_dilate = -1;
+      config.output_dims = {batch, activation_size - kernel_size + 1,
+                            activation_size - kernel_size + 1, output_feature};
+    }
+
+    // Try this layout for all kernel shapes.
+    config.output_layout = {3, 0, 2, 1};
+    config_set.push_back(config);
+
+    // Try other layouts only for certain kernel shapes.
+    if (kernel_size % 2 == 0) {
+      config.activation_layout = {0, 3, 2, 1};
+      config_set.push_back(config);
+
+      config.output_layout = {0, 3, 2, 1};
+      config_set.push_back(config);
+
+      config.activation_layout = {3, 0, 2, 1};
+      config_set.push_back(config);
+    }
+  }
+
+  return config_set;
+}
+
+string GroupedConvolution2DTestDataToString(
+    const ::testing::TestParamInfo<
+        ::testing::tuple<GroupedConvolution2DSpec, bool>>& data) {
+  const auto& spec = ::testing::get<0>(data.param);
+  const string data_type = GetFloatDataType(::testing::get<1>(data.param));
+  string str = absl::StrCat(
+      "activation_dims_", absl::StrJoin(spec.activation_dims, "x"),
+      "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"),
+      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_",
+      absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_",
+      absl::StrJoin(spec.output_dims, "x"), "_output_layout_",
+      absl::StrJoin(spec.output_layout, "_"), data_type);
+  // -1 indicates non-existence.
+  if (spec.stride != -1) {
+    absl::StrAppend(&str, "_lhs_dilation_", spec.lhs_dilate, "x1");
+  }
+
+  // Test names are not allowed to contain the '-' character.
+  absl::c_replace(str, '-', 'n');
+  return str;
+}
+
+string BuildHloTextGroupedConvolution2D(const GroupedConvolution2DSpec& spec,
+                                        bool use_bfloat16) {
+  const string data_type = GetFloatDataType(use_bfloat16);
+  if (spec.activation_dims[1] == 1 && spec.kernel_dims[1] == 2) {
+    // Check for outer dim.
+    return absl::StrFormat(
+        R"(
+    HloModule TensorFlowDepthwiseConv
+
+    ENTRY main {
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+          window={size=%dx%d  pad=1_1x%d_%d rhs_dilate=1x%d}, dim_labels=b01f_01io->b01f,
+          feature_group_count=%d
+    }
+    )",
+        data_type, absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), data_type,
+        absl::StrJoin(spec.output_dims, ","),
+        absl::StrJoin(spec.output_layout, ","), data_type,
+        absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+        spec.window, spec.window, spec.window, spec.group_count);
+
+  } else if (spec.stride == -1) {
+    // Check for basic, non-dilated cases.
+    return absl::StrFormat(
+        R"(
+      HloModule TensorFlowDepthwiseConv
+
+      ENTRY main {
+        activation = %s[%s]{%s} parameter(0)
+        kernel = %s[%s]{%s} parameter(1)
+        ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+            window={size=%dx%d}, dim_labels=b01f_01io->b01f,
+            feature_group_count=%d
+      }
+      )",
+        data_type, absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), data_type,
+        absl::StrJoin(spec.output_dims, ","),
+        absl::StrJoin(spec.output_layout, ","), data_type,
+        absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+        spec.group_count);
+  } else {
+    // Check for base dilations.
+    return absl::StrFormat(
+        R"(
+    HloModule TensorFlowDepthwiseConv
+
+    ENTRY main {
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+          window={size=%dx%d stride=%dx1 pad=%d_%dx0_0 lhs_dilate=%dx1}, 
+          dim_labels=b01f_01io->b01f, feature_group_count=%d
+    }
+    )",
+        data_type, absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), data_type,
+        absl::StrJoin(spec.output_dims, ","),
+        absl::StrJoin(spec.output_layout, ","), data_type,
+        absl::StrJoin(spec.activation_dims, ","),
+        absl::StrJoin(spec.activation_layout, ","), data_type,
+        absl::StrJoin(spec.kernel_dims, ","),
+        absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+        spec.stride, 0, 0, spec.lhs_dilate, spec.group_count);
+  }
+}
+
+XLA_TEST_P(GroupedConvolution2DTest, DoIt) {
+  const GroupedConvolution2DSpec& spec = ::testing::get<0>(GetParam());
+  bool use_bfloat16 = ::testing::get<1>(GetParam());
+  const string hlo_text = BuildHloTextGroupedConvolution2D(spec, use_bfloat16);
+
+  EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01},
+                            [](HloModule* module) -> Status {
+                              BFloat16MixedPrecisionRemoval remover;
+                              TF_RETURN_IF_ERROR(remover.Run(module).status());
+                              Despecializer despecializer;
+                              return despecializer.Run(module).status();
+                            }));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    GroupedConvolution2DTestWithRandomIndices, GroupedConvolution2DTest,
+    ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()),
+                       ::testing::Bool()),
+    GroupedConvolution2DTestDataToString);
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 8584f21392772170b007ee3b3fbfed17fe19e32f Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 1 Dec 2018 20:58:09 +0000
Subject: [PATCH 1158/1554] Fix SparseDenseCwise's broadcasting issue

This fix tries to address the issue raised in 24072.
In `sparse_dense_cwise_mul/add` operations the broadcasting
only support dense to sparse, though the validation was not
captured.

This fix fixes the validation in SparseDenseBinaryOpShared
so that error could be thrown correctly.

This fix fixes 24072.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/sparse_dense_binary_op_shared.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
index ac48202ada..3a6b66302f 100644
--- a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
+++ b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
@@ -88,12 +88,11 @@ class SparseDenseBinaryOpShared : public OpKernel {
     const auto rhs_dims = BCast::FromShape(dense_t->shape());
     BCast b(lhs_dims, rhs_dims, false);  // false for keeping the same num dims.
 
-    // True iff (size(lhs) > size(rhs)), or (sizes equal, lhs cwise rhs).
+    // True iff (size(lhs) >= size(rhs)) and all dims in lhs is smaller or equal to dims in rhs (from right to left).
     auto VecGreaterEq = [](ArraySlice<int64> lhs, ArraySlice<int64> rhs) {
-      if (lhs.size() > rhs.size()) return true;
       if (lhs.size() < rhs.size()) return false;
-      for (size_t i = 0; i < lhs.size(); ++i) {
-        if (lhs[i] < rhs[i]) return false;
+      for (size_t i = 0; i < rhs.size(); ++i) {
+        if (lhs[lhs.size() - 1 - i] < rhs[rhs.size() - 1 - i]) return false;
       }
       return true;
     };
-- 
GitLab


From a7cd4dbea9f276160ebadf82178e77ae5c8d557e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 1 Dec 2018 21:02:04 +0000
Subject: [PATCH 1159/1554] Add test case for sparse_dense_cwise shape
 validation.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/sparse_ops_test.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index 75f65e6251..f58832a89e 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
@@ -798,6 +799,17 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase):
                                                result_tensor.values).eval()
     self.assertAllEqual(result_np, res_densified)
 
+  @test_util.run_deprecated_v1
+  def testCwiseShapeValidation(self):
+    # Test case for GitHub 24072.
+    with self.session(use_gpu=False):
+      a = array_ops.ones([3, 4, 1], dtype=dtypes.int32)
+      b = sparse_tensor.SparseTensor([[0, 0, 1, 0], [0, 0, 3, 0]], [10, 20], [1, 1, 4, 2])
+      c = a * b
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "broadcasts dense to sparse only; got incompatible shapes"):
+        c.eval()
+
   @test_util.run_deprecated_v1
   def testCwiseDivAndMul(self):
     np.random.seed(1618)
-- 
GitLab


From 676c6ea316f4dda962d8b2e29855c040020533f5 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 1 Dec 2018 21:04:41 +0000
Subject: [PATCH 1160/1554] Pylint fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/sparse_ops_test.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py
index f58832a89e..50b6239185 100644
--- a/tensorflow/python/kernel_tests/sparse_ops_test.py
+++ b/tensorflow/python/kernel_tests/sparse_ops_test.py
@@ -804,10 +804,12 @@ class SparseMathOpsTest(test_util.TensorFlowTestCase):
     # Test case for GitHub 24072.
     with self.session(use_gpu=False):
       a = array_ops.ones([3, 4, 1], dtype=dtypes.int32)
-      b = sparse_tensor.SparseTensor([[0, 0, 1, 0], [0, 0, 3, 0]], [10, 20], [1, 1, 4, 2])
+      b = sparse_tensor.SparseTensor(
+          [[0, 0, 1, 0], [0, 0, 3, 0]], [10, 20], [1, 1, 4, 2])
       c = a * b
-      with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "broadcasts dense to sparse only; got incompatible shapes"):
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          "broadcasts dense to sparse only; got incompatible shapes"):
         c.eval()
 
   @test_util.run_deprecated_v1
-- 
GitLab


From 28e034bd293db09a0ecc707f71f65fe6c5dd2943 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 1 Dec 2018 21:28:19 +0000
Subject: [PATCH 1161/1554] Fix `Experimental clang-format Check`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/sparse_dense_binary_op_shared.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
index 3a6b66302f..d7460363fc 100644
--- a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
+++ b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
@@ -88,7 +88,8 @@ class SparseDenseBinaryOpShared : public OpKernel {
     const auto rhs_dims = BCast::FromShape(dense_t->shape());
     BCast b(lhs_dims, rhs_dims, false);  // false for keeping the same num dims.
 
-    // True iff (size(lhs) >= size(rhs)) and all dims in lhs is smaller or equal to dims in rhs (from right to left).
+    // True iff (size(lhs) >= size(rhs)) and all dims in lhs is smaller or equal
+    // to dims in rhs (from right to left).
     auto VecGreaterEq = [](ArraySlice<int64> lhs, ArraySlice<int64> rhs) {
       if (lhs.size() < rhs.size()) return false;
       for (size_t i = 0; i < rhs.size(); ++i) {
-- 
GitLab


From cc8e28b72b52718db5f22830e1d529d8e077b537 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 2 Dec 2018 01:55:11 +0000
Subject: [PATCH 1162/1554] Fix broken link in lite api docs

This fix fixes broken link in lite apis docs

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/lite/g3doc/apis.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/g3doc/apis.md b/tensorflow/lite/g3doc/apis.md
index e9fa24bff1..60ce11d685 100644
--- a/tensorflow/lite/g3doc/apis.md
+++ b/tensorflow/lite/g3doc/apis.md
@@ -347,7 +347,7 @@ interpreter.runForMultipleInputsOutputs(inputs, map_of_indices_to_outputs);
 where each entry in `inputs` corresponds to an input tensor and
 `map_of_indices_to_outputs` maps indices of output tensors to the
 corresponding output data. In both cases the tensor indices should correspond to
-the values given to the [TensorFlow Lite Optimized Converter](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/toco/g3doc/cmdline_examples.md)
+the values given to the [TensorFlow Lite Optimized Converter](convert/cmdline_examples.md)
 when the model was created. Be aware that the order of tensors in `input` must
 match the order given to the `TensorFlow Lite Optimized Converter`.
 
-- 
GitLab


From c297ff16bc798fb78e06460d00bcef52853601fd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 2 Dec 2018 01:02:28 -0800
Subject: [PATCH 1163/1554] compat: Update forward compatibility horizon to
 2018-12-02

PiperOrigin-RevId: 223684627
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index a7e8bef670..8ec90ee759 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 1)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 2)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 3188c2a38b9327e58940e7040a4ba7c72c014718 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 2 Dec 2018 03:42:28 -0800
Subject: [PATCH 1164/1554] Fixing LD_LIBRARY_PATH env variable.

PiperOrigin-RevId: 223692658
---
 .../tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
index 85b9d94313..03de89b717 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -31,7 +31,7 @@ ENV PATH /usr/local/cuda/bin:${PATH}
 # -Wl,-rpath-link=/usr/local/cuda/lib64/stubs to all binaries transitively
 # depending on libcuda. Optimally, builds targeting cuda would do that
 # internally.
-ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs
 
 LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
 
-- 
GitLab


From b18da39f10f998a21bfcda43184ba3a80243dde4 Mon Sep 17 00:00:00 2001
From: Serge Panev <spanev@nvidia.com>
Date: Sun, 2 Dec 2018 19:24:56 +0100
Subject: [PATCH 1165/1554] Fix with clang-format-3.6.0

---
 tensorflow/core/util/tensor_format.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/util/tensor_format.h b/tensorflow/core/util/tensor_format.h
index 68674cb282..a296fb447e 100644
--- a/tensorflow/core/util/tensor_format.h
+++ b/tensorflow/core/util/tensor_format.h
@@ -498,7 +498,8 @@ inline TensorShape ShapeFromFormat(TensorFormat format, int64 N,
   dim_sizes[GetTensorBatchDimIndex(dims, format)] = N;
   for (int dim = 0; static_cast<size_t>(dim) < spatial.size(); dim++) {
     auto dim_size = spatial[dim];
-    if (format == FORMAT_NHWC_VECT_W && static_cast<size_t>(dim) == spatial.size() - 1) {
+    if (format == FORMAT_NHWC_VECT_W &&
+        static_cast<size_t>(dim) == spatial.size() - 1) {
       CHECK_EQ(0, dim_size % 4)
           << "FORMAT_NHWC_VECT_W requires W to be a multiple of 4, but W="
           << dim_size;
-- 
GitLab


From bf70c447d68070e81c1dbc2ebf0b6b0d8ae2c597 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Sun, 2 Dec 2018 11:02:30 -0800
Subject: [PATCH 1166/1554] [tf.data] Consolidating experimental C++ kernels.

PiperOrigin-RevId: 223710135
---
 tensorflow/c/c_api_experimental.cc            |   26 +-
 tensorflow/contrib/data/python/ops/sliding.py |    4 +-
 ...perimentalBytesProducedStatsDataset.pbtxt} |    3 +-
 ...i_def_ExperimentalDatasetToTFRecord.pbtxt} |    2 +-
 ...perimentalDenseToSparseBatchDataset.pbtxt} |    3 +-
 ...f_ExperimentalGroupByReducerDataset.pbtxt} |    2 +-
 ...ef_ExperimentalGroupByWindowDataset.pbtxt} |    3 +-
 ...def_ExperimentalLatencyStatsDataset.pbtxt} |    3 +-
 ..._def_ExperimentalMapAndBatchDataset.pbtxt} |    2 +-
 ...perimentalParallelInterleaveDataset.pbtxt} |    3 +-
 ...def_ExperimentalParseExampleDataset.pbtxt} |    3 +-
 ...xperimentalSetStatsAggregatorDataset.pbtxt |    4 +
 ...ef_ExperimentalSlidingWindowDataset.pbtxt} |    3 +-
 ...t => api_def_ExperimentalSqlDataset.pbtxt} |    3 +-
 ...ef_ExperimentalStatsAggregatorHandle.pbtxt |    5 +
 ..._ExperimentalStatsAggregatorSummary.pbtxt} |    3 +-
 ... api_def_ExperimentalUnbatchDataset.pbtxt} |    3 +-
 .../base_api/api_def_MapAndBatchDataset.pbtxt |   53 -
 .../api_def_SetStatsAggregatorDataset.pbtxt   |    3 -
 .../api_def_StatsAggregatorHandle.pbtxt       |    4 -
 .../api_def_BytesProducedStatsDataset.pbtxt   |    4 -
 .../api_def_DenseToSparseBatchDataset.pbtxt   |    4 -
 .../api_def_GroupByWindowDataset.pbtxt        |    4 -
 .../api_def_LatencyStatsDataset.pbtxt         |    4 -
 .../api_def_MapAndBatchDataset.pbtxt          |    4 -
 .../api_def_ParallelInterleaveDataset.pbtxt   |    4 -
 .../api_def_ParseExampleDataset.pbtxt         |    4 -
 .../api_def_SetStatsAggregatorDataset.pbtxt   |    4 -
 .../python_api/api_def_SlideDataset.pbtxt     |    4 -
 .../python_api/api_def_SqlDataset.pbtxt       |    4 -
 .../api_def_StatsAggregatorHandle.pbtxt       |    4 -
 .../api_def_StatsAggregatorSummary.pbtxt      |    4 -
 .../python_api/api_def_UnbatchDataset.pbtxt   |    4 -
 .../optimizers/data/graph_test_utils.cc       |    2 +-
 .../optimizers/data/latency_all_edges.cc      |    2 +-
 .../optimizers/data/latency_all_edges_test.cc |    7 +-
 .../optimizers/data/make_numa_aware.cc        |    2 +-
 .../optimizers/data/make_numa_aware_test.cc   |    6 +-
 .../optimizers/data/map_and_batch_fusion.cc   |    2 +-
 .../data/map_and_batch_fusion_test.cc         |   21 +-
 tensorflow/core/kernels/data/BUILD            |  156 --
 .../core/kernels/data/experimental/BUILD      |  222 ++-
 .../dense_to_sparse_batch_dataset_op.cc       |    5 +-
 .../group_by_reducer_dataset_op.cc            |    5 +-
 .../group_by_window_dataset_op.cc             |    5 +-
 .../experimental/identity_indexed_dataset.cc  |  163 --
 .../data/experimental/indexed_dataset.h       |  119 --
 ...dexed_dataset.cc => indexed_dataset_op.cc} |  249 ++-
 .../map_and_batch_dataset_op.cc               |   10 +-
 .../parallel_interleave_dataset_op.cc         | 1085 +++++++++++
 .../parse_example_dataset_op.cc               |    5 +-
 .../set_stats_aggregator_dataset_op.cc}       |    5 +-
 .../sliding_window_dataset_op.cc}             |   11 +-
 .../kernels/data/{ => experimental}/sql/BUILD |    0
 .../{ => experimental}/sql/driver_manager.cc  |    4 +-
 .../{ => experimental}/sql/driver_manager.h   |    8 +-
 .../{ => experimental}/sql/query_connection.h |    6 +-
 .../sql/sqlite_query_connection.cc            |    4 +-
 .../sql/sqlite_query_connection.h             |    8 +-
 .../sql_dataset_op.cc}                        |    7 +-
 .../stats_aggregator_ops.cc                   |   10 +-
 .../{ => experimental}/stats_dataset_ops.cc   |   10 +-
 .../to_tf_record_op.cc}                       |    4 +-
 .../{ => experimental}/unbatch_dataset_op.cc  |    2 +-
 .../data/parallel_interleave_dataset_op.cc    | 1051 +----------
 .../core/ops/compat/ops_history.v1.pbtxt      | 1622 ++++++++---------
 tensorflow/core/ops/dataset_ops.cc            |  230 ---
 .../core/ops/experimental_dataset_ops.cc      |  214 ++-
 .../python/data/experimental/ops/batching.py  |    8 +-
 .../python/data/experimental/ops/grouping.py  |   68 +-
 .../data/experimental/ops/parsing_ops.py      |    4 +-
 .../python/data/experimental/ops/readers.py   |    8 +-
 .../data/experimental/ops/stats_aggregator.py |    6 +-
 .../python/data/experimental/ops/stats_ops.py |   12 +-
 .../python/data/experimental/ops/writers.py   |    4 +-
 tensorflow/python/data/ops/dataset_ops.py     |    2 +-
 tensorflow/python/data/ops/readers.py         |    3 +-
 77 files changed, 2583 insertions(+), 2981 deletions(-)
 rename tensorflow/core/api_def/base_api/{api_def_BytesProducedStatsDataset.pbtxt => api_def_ExperimentalBytesProducedStatsDataset.pbtxt} (56%)
 rename tensorflow/core/api_def/base_api/{api_def_DatasetToTFRecord.pbtxt => api_def_ExperimentalDatasetToTFRecord.pbtxt} (91%)
 rename tensorflow/core/api_def/base_api/{api_def_DenseToSparseBatchDataset.pbtxt => api_def_ExperimentalDenseToSparseBatchDataset.pbtxt} (89%)
 rename tensorflow/core/api_def/base_api/{api_def_GroupByReducerDataset.pbtxt => api_def_ExperimentalGroupByReducerDataset.pbtxt} (97%)
 rename tensorflow/core/api_def/base_api/{api_def_GroupByWindowDataset.pbtxt => api_def_ExperimentalGroupByWindowDataset.pbtxt} (82%)
 rename tensorflow/core/api_def/base_api/{api_def_LatencyStatsDataset.pbtxt => api_def_ExperimentalLatencyStatsDataset.pbtxt} (58%)
 rename tensorflow/core/api_def/base_api/{api_def_MapAndBatchDatasetV2.pbtxt => api_def_ExperimentalMapAndBatchDataset.pbtxt} (96%)
 rename tensorflow/core/api_def/base_api/{api_def_ParallelInterleaveDataset.pbtxt => api_def_ExperimentalParallelInterleaveDataset.pbtxt} (90%)
 rename tensorflow/core/api_def/base_api/{api_def_ParseExampleDataset.pbtxt => api_def_ExperimentalParseExampleDataset.pbtxt} (96%)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalSetStatsAggregatorDataset.pbtxt
 rename tensorflow/core/api_def/base_api/{api_def_SlideDataset.pbtxt => api_def_ExperimentalSlidingWindowDataset.pbtxt} (88%)
 rename tensorflow/core/api_def/base_api/{api_def_SqlDataset.pbtxt => api_def_ExperimentalSqlDataset.pbtxt} (87%)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorHandle.pbtxt
 rename tensorflow/core/api_def/base_api/{api_def_StatsAggregatorSummary.pbtxt => api_def_ExperimentalStatsAggregatorSummary.pbtxt} (56%)
 rename tensorflow/core/api_def/base_api/{api_def_UnbatchDataset.pbtxt => api_def_ExperimentalUnbatchDataset.pbtxt} (57%)
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_MapAndBatchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_SetStatsAggregatorDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_StatsAggregatorHandle.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_BytesProducedStatsDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_DenseToSparseBatchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_GroupByWindowDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_LatencyStatsDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_MapAndBatchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ParallelInterleaveDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ParseExampleDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_SetStatsAggregatorDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_SlideDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_SqlDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_StatsAggregatorHandle.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_StatsAggregatorSummary.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_UnbatchDataset.pbtxt
 rename tensorflow/core/kernels/data/{ => experimental}/dense_to_sparse_batch_dataset_op.cc (98%)
 rename tensorflow/core/kernels/data/{ => experimental}/group_by_reducer_dataset_op.cc (99%)
 rename tensorflow/core/kernels/data/{ => experimental}/group_by_window_dataset_op.cc (99%)
 delete mode 100644 tensorflow/core/kernels/data/experimental/identity_indexed_dataset.cc
 delete mode 100644 tensorflow/core/kernels/data/experimental/indexed_dataset.h
 rename tensorflow/core/kernels/data/experimental/{indexed_dataset.cc => indexed_dataset_op.cc} (62%)
 rename tensorflow/core/kernels/data/{ => experimental}/map_and_batch_dataset_op.cc (99%)
 create mode 100644 tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
 rename tensorflow/core/kernels/data/{ => experimental}/parse_example_dataset_op.cc (99%)
 rename tensorflow/core/kernels/data/{stats_aggregator_dataset_op.cc => experimental/set_stats_aggregator_dataset_op.cc} (98%)
 rename tensorflow/core/kernels/data/{slide_dataset_op.cc => experimental/sliding_window_dataset_op.cc} (97%)
 rename tensorflow/core/kernels/data/{ => experimental}/sql/BUILD (100%)
 rename tensorflow/core/kernels/data/{ => experimental}/sql/driver_manager.cc (88%)
 rename tensorflow/core/kernels/data/{ => experimental}/sql/driver_manager.h (81%)
 rename tensorflow/core/kernels/data/{ => experimental}/sql/query_connection.h (92%)
 rename tensorflow/core/kernels/data/{ => experimental}/sql/sqlite_query_connection.cc (97%)
 rename tensorflow/core/kernels/data/{ => experimental}/sql/sqlite_query_connection.h (84%)
 rename tensorflow/core/kernels/data/{sql_dataset_ops.cc => experimental/sql_dataset_op.cc} (96%)
 rename tensorflow/core/kernels/data/{ => experimental}/stats_aggregator_ops.cc (95%)
 rename tensorflow/core/kernels/data/{ => experimental}/stats_dataset_ops.cc (97%)
 rename tensorflow/core/kernels/data/{writer_ops.cc => experimental/to_tf_record_op.cc} (97%)
 rename tensorflow/core/kernels/data/{ => experimental}/unbatch_dataset_op.cc (99%)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 3693cc8599..38e29aa74a 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -6530,7 +6530,7 @@ library {
       }
     }
     node_def {
-      name: "ParallelInterleaveDataset/cycle_length"
+      name: "ExperimentalParallelInterleaveDataset/cycle_length"
       op: "Const"
       attr {
         key: "dtype"
@@ -6551,7 +6551,7 @@ library {
       }
     }
     node_def {
-      name: "ParallelInterleaveDataset/block_length"
+      name: "ExperimentalParallelInterleaveDataset/block_length"
       op: "Const"
       attr {
         key: "dtype"
@@ -6572,7 +6572,7 @@ library {
       }
     }
     node_def {
-      name: "ParallelInterleaveDataset/sloppy"
+      name: "ExperimentalParallelInterleaveDataset/sloppy"
       op: "Const"
       attr {
         key: "dtype"
@@ -6593,7 +6593,7 @@ library {
       }
     }
     node_def {
-      name: "ParallelInterleaveDataset/buffer_output_elements"
+      name: "ExperimentalParallelInterleaveDataset/buffer_output_elements"
       op: "Const"
       attr {
         key: "dtype"
@@ -6614,7 +6614,7 @@ library {
       }
     }
     node_def {
-      name: "ParallelInterleaveDataset/prefetch_input_elements"
+      name: "ExperimentalParallelInterleaveDataset/prefetch_input_elements"
       op: "Const"
       attr {
         key: "dtype"
@@ -6635,14 +6635,14 @@ library {
       }
     }
     node_def {
-      name: "ParallelInterleaveDataset"
-      op: "ParallelInterleaveDataset"
+      name: "ExperimentalParallelInterleaveDataset"
+      op: "ExperimentalParallelInterleaveDataset"
       input: "RepeatDataset:handle:0"
-      input: "ParallelInterleaveDataset/cycle_length:output:0"
-      input: "ParallelInterleaveDataset/block_length:output:0"
-      input: "ParallelInterleaveDataset/sloppy:output:0"
-      input: "ParallelInterleaveDataset/buffer_output_elements:output:0"
-      input: "ParallelInterleaveDataset/prefetch_input_elements:output:0"
+      input: "ExperimentalParallelInterleaveDataset/cycle_length:output:0"
+      input: "ExperimentalParallelInterleaveDataset/block_length:output:0"
+      input: "ExperimentalParallelInterleaveDataset/sloppy:output:0"
+      input: "ExperimentalParallelInterleaveDataset/buffer_output_elements:output:0"
+      input: "ExperimentalParallelInterleaveDataset/prefetch_input_elements:output:0"
       attr {
         key: "Targuments"
         value {
@@ -6742,7 +6742,7 @@ library {
     node_def {
       name: "ShuffleDataset_2"
       op: "ShuffleDataset"
-      input: "ParallelInterleaveDataset:handle:0"
+      input: "ExperimentalParallelInterleaveDataset:handle:0"
       input: "ShuffleDataset_2/buffer_size_1:output:0"
       input: "ShuffleDataset_2/seed_2:output:0"
       input: "ShuffleDataset_2/seed2_2:output:0"
diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py
index bcc383587c..a3ab6b8d72 100644
--- a/tensorflow/contrib/data/python/ops/sliding.py
+++ b/tensorflow/contrib/data/python/ops/sliding.py
@@ -22,7 +22,7 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.util import deprecation
 
 
@@ -41,7 +41,7 @@ class _SlideDataset(dataset_ops.UnaryDataset):
         window_shift, dtype=dtypes.int64, name="window_shift")
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.slide_dataset(
+    return ged_ops.experimental_sliding_window_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         window_size=self._window_size,
         window_shift=self._window_shift,
diff --git a/tensorflow/core/api_def/base_api/api_def_BytesProducedStatsDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalBytesProducedStatsDataset.pbtxt
similarity index 56%
rename from tensorflow/core/api_def/base_api/api_def_BytesProducedStatsDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalBytesProducedStatsDataset.pbtxt
index 73df11b2f7..dc296162ae 100644
--- a/tensorflow/core/api_def/base_api/api_def_BytesProducedStatsDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalBytesProducedStatsDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
-  graph_op_name: "BytesProducedStatsDataset"
+  graph_op_name: "ExperimentalBytesProducedStatsDataset"
+  visibility: HIDDEN
   summary: "Records the bytes size of each element of `input_dataset` in a StatsAggregator."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetToTFRecord.pbtxt
similarity index 91%
rename from tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetToTFRecord.pbtxt
index e1b8a9abdd..085d20d7bf 100644
--- a/tensorflow/core/api_def/base_api/api_def_DatasetToTFRecord.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetToTFRecord.pbtxt
@@ -1,5 +1,5 @@
 op {
-  graph_op_name: "DatasetToTFRecord"
+  graph_op_name: "ExperimentalDatasetToTFRecord"
   visibility: HIDDEN
   in_arg {
     name: "input_dataset"
diff --git a/tensorflow/core/api_def/base_api/api_def_DenseToSparseBatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalDenseToSparseBatchDataset.pbtxt
similarity index 89%
rename from tensorflow/core/api_def/base_api/api_def_DenseToSparseBatchDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalDenseToSparseBatchDataset.pbtxt
index e275cfdd3d..8ebd6d88a8 100644
--- a/tensorflow/core/api_def/base_api/api_def_DenseToSparseBatchDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalDenseToSparseBatchDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
-  graph_op_name: "DenseToSparseBatchDataset"
+  graph_op_name: "ExperimentalDenseToSparseBatchDataset"
+  visibility: HIDDEN
   in_arg {
     name: "input_dataset"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_GroupByReducerDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalGroupByReducerDataset.pbtxt
similarity index 97%
rename from tensorflow/core/api_def/base_api/api_def_GroupByReducerDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalGroupByReducerDataset.pbtxt
index 067ad4018b..dd132802fa 100644
--- a/tensorflow/core/api_def/base_api/api_def_GroupByReducerDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalGroupByReducerDataset.pbtxt
@@ -1,5 +1,5 @@
 op {
-  graph_op_name: "GroupByReducerDataset"
+  graph_op_name: "ExperimentalGroupByReducerDataset"
   visibility: HIDDEN
   in_arg {
     name: "input_dataset"
diff --git a/tensorflow/core/api_def/base_api/api_def_GroupByWindowDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalGroupByWindowDataset.pbtxt
similarity index 82%
rename from tensorflow/core/api_def/base_api/api_def_GroupByWindowDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalGroupByWindowDataset.pbtxt
index ea6bcd4695..6e4c12ed81 100644
--- a/tensorflow/core/api_def/base_api/api_def_GroupByWindowDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalGroupByWindowDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
-  graph_op_name: "GroupByWindowDataset"
+  graph_op_name: "ExperimentalGroupByWindowDataset"
+  visibility: HIDDEN
   attr {
     name: "key_func"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_LatencyStatsDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalLatencyStatsDataset.pbtxt
similarity index 58%
rename from tensorflow/core/api_def/base_api/api_def_LatencyStatsDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalLatencyStatsDataset.pbtxt
index 78d946b0b4..e7351b9d70 100644
--- a/tensorflow/core/api_def/base_api/api_def_LatencyStatsDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalLatencyStatsDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
-  graph_op_name: "LatencyStatsDataset"
+  graph_op_name: "ExperimentalLatencyStatsDataset"
+  visibility: HIDDEN
   summary: "Records the latency of producing `input_dataset` elements in a StatsAggregator."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_MapAndBatchDatasetV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalMapAndBatchDataset.pbtxt
similarity index 96%
rename from tensorflow/core/api_def/base_api/api_def_MapAndBatchDatasetV2.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalMapAndBatchDataset.pbtxt
index 81ef92cae0..bc4270670c 100644
--- a/tensorflow/core/api_def/base_api/api_def_MapAndBatchDatasetV2.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalMapAndBatchDataset.pbtxt
@@ -1,5 +1,5 @@
 op {
-  graph_op_name: "MapAndBatchDatasetV2"
+  graph_op_name: "ExperimentalMapAndBatchDataset"
   visibility: HIDDEN
   in_arg {
     name: "input_dataset"
diff --git a/tensorflow/core/api_def/base_api/api_def_ParallelInterleaveDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalParallelInterleaveDataset.pbtxt
similarity index 90%
rename from tensorflow/core/api_def/base_api/api_def_ParallelInterleaveDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalParallelInterleaveDataset.pbtxt
index d6889b54a0..dd70e33284 100644
--- a/tensorflow/core/api_def/base_api/api_def_ParallelInterleaveDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalParallelInterleaveDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
-  graph_op_name: "ParallelInterleaveDataset"
+  graph_op_name: "ExperimentalParallelInterleaveDataset"
+  visibility: HIDDEN
   attr {
     name: "f"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ParseExampleDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalParseExampleDataset.pbtxt
similarity index 96%
rename from tensorflow/core/api_def/base_api/api_def_ParseExampleDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalParseExampleDataset.pbtxt
index 3de2f18fc2..2de13c5cee 100644
--- a/tensorflow/core/api_def/base_api/api_def_ParseExampleDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalParseExampleDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
-  graph_op_name: "ParseExampleDataset"
+  graph_op_name: "ExperimentalParseExampleDataset"
+  visibility: HIDDEN
   in_arg {
     name: "dense_defaults"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalSetStatsAggregatorDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalSetStatsAggregatorDataset.pbtxt
new file mode 100644
index 0000000000..6e6b2f81b3
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalSetStatsAggregatorDataset.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ExperimentalSetStatsAggregatorDataset"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalSlidingWindowDataset.pbtxt
similarity index 88%
rename from tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalSlidingWindowDataset.pbtxt
index ddde3ee5b4..dc62750b66 100644
--- a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalSlidingWindowDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
-  graph_op_name: "SlideDataset"
+  graph_op_name: "ExperimentalSlidingWindowDataset"
+  visibility: HIDDEN
   in_arg {
     name: "window_size"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_SqlDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalSqlDataset.pbtxt
similarity index 87%
rename from tensorflow/core/api_def/base_api/api_def_SqlDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalSqlDataset.pbtxt
index 7570d5da56..35cddbd061 100644
--- a/tensorflow/core/api_def/base_api/api_def_SqlDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalSqlDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
-  graph_op_name: "SqlDataset"
+  graph_op_name: "ExperimentalSqlDataset"
+  visibility: HIDDEN
   in_arg {
     name: "driver_name"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorHandle.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorHandle.pbtxt
new file mode 100644
index 0000000000..8a770d462d
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorHandle.pbtxt
@@ -0,0 +1,5 @@
+op {
+  graph_op_name: "ExperimentalStatsAggregatorHandle"
+  visibility: HIDDEN
+  summary: "Creates a statistics manager resource."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_StatsAggregatorSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorSummary.pbtxt
similarity index 56%
rename from tensorflow/core/api_def/base_api/api_def_StatsAggregatorSummary.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorSummary.pbtxt
index bcaf9fea1a..ffe0103689 100644
--- a/tensorflow/core/api_def/base_api/api_def_StatsAggregatorSummary.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalStatsAggregatorSummary.pbtxt
@@ -1,4 +1,5 @@
 op {
-  graph_op_name: "StatsAggregatorSummary"
+  graph_op_name: "ExperimentalStatsAggregatorSummary"
+  visibility: HIDDEN
   summary: "Produces a summary of any statistics recorded by the given statistics manager."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_UnbatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalUnbatchDataset.pbtxt
similarity index 57%
rename from tensorflow/core/api_def/base_api/api_def_UnbatchDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalUnbatchDataset.pbtxt
index 324fadac0a..c89e1fd0bd 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnbatchDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalUnbatchDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
-  graph_op_name: "UnbatchDataset"
+  graph_op_name: "ExperimentalUnbatchDataset"
+  visibility: HIDDEN
   summary: "A dataset that splits the elements of its input into multiple elements."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_MapAndBatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_MapAndBatchDataset.pbtxt
deleted file mode 100644
index e230c51edf..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_MapAndBatchDataset.pbtxt
+++ /dev/null
@@ -1,53 +0,0 @@
-op {
-  graph_op_name: "MapAndBatchDataset"
-  visibility: HIDDEN
-  in_arg {
-    name: "input_dataset"
-    description: <<END
-A variant tensor representing the input dataset.
-END
-  }
-  in_arg {
-    name: "other_arguments"
-    description: <<END
-A list of tensors, typically values that were captured when building a closure
-for `f`.
-END
-  }
-  in_arg {
-    name: "batch_size"
-    description: <<END
-A scalar representing the number of elements to accumulate in a
-batch. It determines the number of concurrent invocations of `f` that process
-elements from `input_dataset` in parallel.
-END
-  }
-  in_arg {
-    name: "num_parallel_batches"
-    description: <<END
-A scalar representing the number of batches to create in parallel. Processing
-multiple batches in parallel benefits workloads prone to stragglers.
-END
-  }
-  in_arg {
-    name: "drop_remainder"
-    description: <<END
-A scalar representing whether the last batch should be dropped in case its size
-is smaller than desired.
-END
-  }
-  attr {
-    name: "f"
-    description: <<END
-A function to apply to the outputs of `input_dataset`.
-END
-  }
-  summary: "Creates a dataset that fuses mapping with batching."
-  description: <<END
-Creates a dataset that applies `f` to the outputs of `input_dataset` and then
-batches `batch_size` of them.
-
-Unlike a "MapDataset", which applies `f` sequentially, this dataset invokes up
-to `batch_size * num_parallel_batches` copies of `f` in parallel.
-END
-}
diff --git a/tensorflow/core/api_def/base_api/api_def_SetStatsAggregatorDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SetStatsAggregatorDataset.pbtxt
deleted file mode 100644
index 77123e143b..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_SetStatsAggregatorDataset.pbtxt
+++ /dev/null
@@ -1,3 +0,0 @@
-op {
-  graph_op_name: "SetStatsAggregatorDataset"
-}
diff --git a/tensorflow/core/api_def/base_api/api_def_StatsAggregatorHandle.pbtxt b/tensorflow/core/api_def/base_api/api_def_StatsAggregatorHandle.pbtxt
deleted file mode 100644
index 9b30d64afe..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_StatsAggregatorHandle.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "StatsAggregatorHandle"
-  summary: "Creates a statistics manager resource."
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_BytesProducedStatsDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_BytesProducedStatsDataset.pbtxt
deleted file mode 100644
index fcf541f903..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_BytesProducedStatsDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "BytesProducedStatsDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_DenseToSparseBatchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_DenseToSparseBatchDataset.pbtxt
deleted file mode 100644
index 0a8e068afb..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_DenseToSparseBatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "DenseToSparseBatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_GroupByWindowDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_GroupByWindowDataset.pbtxt
deleted file mode 100644
index 8d40208e61..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_GroupByWindowDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "GroupByWindowDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_LatencyStatsDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_LatencyStatsDataset.pbtxt
deleted file mode 100644
index 94bf6106ad..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_LatencyStatsDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "LatencyStatsDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_MapAndBatchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_MapAndBatchDataset.pbtxt
deleted file mode 100644
index cffd2910fb..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_MapAndBatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "MapAndBatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ParallelInterleaveDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ParallelInterleaveDataset.pbtxt
deleted file mode 100644
index 93cd5719fe..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ParallelInterleaveDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ParallelInterleaveDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ParseExampleDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ParseExampleDataset.pbtxt
deleted file mode 100644
index 45826b6fdc..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ParseExampleDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ParseExampleDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_SetStatsAggregatorDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_SetStatsAggregatorDataset.pbtxt
deleted file mode 100644
index 3a8c1036ca..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_SetStatsAggregatorDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "SetStatsAggregatorDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_SlideDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_SlideDataset.pbtxt
deleted file mode 100644
index 867116c5da..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_SlideDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "SlideDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_SqlDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_SqlDataset.pbtxt
deleted file mode 100644
index 2ab4c3e441..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_SqlDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "SqlDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_StatsAggregatorHandle.pbtxt b/tensorflow/core/api_def/python_api/api_def_StatsAggregatorHandle.pbtxt
deleted file mode 100644
index f7bed36602..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_StatsAggregatorHandle.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "StatsAggregatorHandle"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_StatsAggregatorSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_StatsAggregatorSummary.pbtxt
deleted file mode 100644
index 8b1bab2440..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_StatsAggregatorSummary.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "StatsAggregatorSummary"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_UnbatchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_UnbatchDataset.pbtxt
deleted file mode 100644
index 1e5415749f..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_UnbatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "UnbatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
index affaeafb0f..9d8b388a3a 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
@@ -41,7 +41,7 @@ NodeDef MakeMapAndBatchNode(StringPiece name, StringPiece input_node_name,
                             StringPiece drop_remainder_node_name,
                             StringPiece function_name) {
   return test::function::NDef(
-      name, "MapAndBatchDatasetV2",
+      name, "ExperimentalMapAndBatchDataset",
       {string(input_node_name), "", string(batch_size_node_name),
        string(num_parallel_calls_node_name), string(drop_remainder_node_name)},
       {{"f", FunctionDefHelper::FunctionRef(string(function_name))},
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index 16b2efb3ed..52b4b785a3 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -31,7 +31,7 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-constexpr char kInsertOpName[] = "LatencyStatsDataset";
+constexpr char kInsertOpName[] = "ExperimentalLatencyStatsDataset";
 
 NodeDef MakeLatencyNode(const NodeDef& node, MutableGraphView* graph) {
   NodeDef new_node;
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
index 6789cf5bd6..d428d04a66 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
@@ -57,9 +57,10 @@ TEST(LatencyAllEdgesTest, AddLatenciesAfterTensorMapPrefetch) {
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
 
-  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("LatencyStatsDataset", output));
-  std::vector<int> latency_node_indices =
-      graph_utils::FindAllGraphNodesWithOp("LatencyStatsDataset", output);
+  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("ExperimentalLatencyStatsDataset",
+                                              output));
+  std::vector<int> latency_node_indices = graph_utils::FindAllGraphNodesWithOp(
+      "ExperimentalLatencyStatsDataset", output);
   EXPECT_EQ(latency_node_indices.size(), 3);
   std::vector<NodeDef> dataset_nodes = {std::move(from_tensor_node),
                                         std::move(map_node),
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index e5de981822..72c27a1d4a 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -44,7 +44,7 @@ Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
   std::set<string> nodes_to_delete;
 
   for (const NodeDef& node : item.graph.node()) {
-    if (node.op() != "MapAndBatchDatasetV2") continue;
+    if (node.op() != "ExperimentalMapAndBatchDataset") continue;
 
     auto* numa_node = graph.AddNode(MakeNumaAwareNode(node, &graph));
     graph.UpdateFanouts(node.name(), numa_node->name());
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware_test.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware_test.cc
index 5d52bd6208..4b83fb6ef1 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware_test.cc
@@ -57,7 +57,8 @@ TEST(MakeNumaAwareTest, ReplaceSimple) {
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
 
   EXPECT_FALSE(graph_utils::ContainsGraphNodeWithName("map_and_batch", output));
-  EXPECT_FALSE(graph_utils::ContainsNodeWithOp("MapAndBatchDatasetV2", output));
+  EXPECT_FALSE(graph_utils::ContainsNodeWithOp("ExperimentalMapAndBatchDataset",
+                                               output));
   EXPECT_TRUE(graph_utils::ContainsNodeWithOp(
       "ExperimentalNumaMapAndBatchDataset", output));
 }
@@ -91,7 +92,8 @@ TEST(MapAndBatchNumaAawareReplacementTest, ReplaceWithExtraChild) {
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
 
   EXPECT_FALSE(graph_utils::ContainsGraphNodeWithName("map_and_batch", output));
-  EXPECT_FALSE(graph_utils::ContainsNodeWithOp("MapAndBatchDatasetV2", output));
+  EXPECT_FALSE(graph_utils::ContainsNodeWithOp("ExperimentalMapAndBatchDataset",
+                                               output));
   EXPECT_TRUE(graph_utils::ContainsNodeWithOp(
       "ExperimentalNumaMapAndBatchDataset", output));
   EXPECT_TRUE(graph_utils::ContainsNodeWithOp("CacheDataset", output));
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 800050b840..9a7a751ff7 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -30,7 +30,7 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-constexpr char kFusedOpName[] = "MapAndBatchDatasetV2";
+constexpr char kFusedOpName[] = "ExperimentalMapAndBatchDataset";
 
 NodeDef MakeMapAndBatchNode(const NodeDef& map_node, const NodeDef& batch_node,
                             MutableGraphView* graph) {
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion_test.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion_test.cc
index eed558de7e..ef4e64826f 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion_test.cc
@@ -84,9 +84,10 @@ TEST(MapAndBatchFusionTest, FuseMapAndBatchNodesIntoOne) {
       graph_utils::ContainsGraphNodeWithName(map_node->name(), output));
   EXPECT_FALSE(
       graph_utils::ContainsGraphNodeWithName(batch_node->name(), output));
-  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("MapAndBatchDatasetV2", output));
-  NodeDef map_and_batch_node = output.node(
-      graph_utils::FindGraphNodeWithOp("MapAndBatchDatasetV2", output));
+  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("ExperimentalMapAndBatchDataset",
+                                              output));
+  NodeDef map_and_batch_node = output.node(graph_utils::FindGraphNodeWithOp(
+      "ExperimentalMapAndBatchDataset", output));
   EXPECT_EQ(map_and_batch_node.input_size(), 5);
   EXPECT_EQ(map_and_batch_node.input(0), map_node->input(0));
   EXPECT_EQ(map_and_batch_node.input(1), map_node->input(1));
@@ -169,9 +170,10 @@ TEST(MapAndBatchFusionTest, FuseMapAndBatchV2NodesIntoOne) {
       graph_utils::ContainsGraphNodeWithName(map_node->name(), output));
   EXPECT_FALSE(
       graph_utils::ContainsGraphNodeWithName(batch_node->name(), output));
-  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("MapAndBatchDatasetV2", output));
-  NodeDef map_and_batch_node = output.node(
-      graph_utils::FindGraphNodeWithOp("MapAndBatchDatasetV2", output));
+  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("ExperimentalMapAndBatchDataset",
+                                              output));
+  NodeDef map_and_batch_node = output.node(graph_utils::FindGraphNodeWithOp(
+      "ExperimentalMapAndBatchDataset", output));
   EXPECT_EQ(map_and_batch_node.input_size(), 5);
   EXPECT_EQ(map_and_batch_node.input(0), map_node->input(0));
   EXPECT_EQ(map_and_batch_node.input(1), map_node->input(1));
@@ -252,9 +254,10 @@ TEST(MapAndBatchFusionTest, FuseParallelMapAndBatchNodesIntoOne) {
       graph_utils::ContainsGraphNodeWithName(map_node->name(), output));
   EXPECT_FALSE(
       graph_utils::ContainsGraphNodeWithName(batch_node->name(), output));
-  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("MapAndBatchDatasetV2", output));
-  NodeDef map_and_batch_node = output.node(
-      graph_utils::FindGraphNodeWithOp("MapAndBatchDatasetV2", output));
+  EXPECT_TRUE(graph_utils::ContainsNodeWithOp("ExperimentalMapAndBatchDataset",
+                                              output));
+  NodeDef map_and_batch_node = output.node(graph_utils::FindGraphNodeWithOp(
+      "ExperimentalMapAndBatchDataset", output));
   EXPECT_EQ(map_and_batch_node.input_size(), 5);
   EXPECT_EQ(map_and_batch_node.input(0), map_node->input(0));
   EXPECT_EQ(map_and_batch_node.input(1), map_node->input(1));
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index f1eeda20d3..23398b71a4 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -13,16 +13,6 @@ load(
     "tf_cc_test",
 )
 
-tf_kernel_library(
-    name = "stats_aggregator_ops",
-    srcs = ["stats_aggregator_ops.cc"],
-    deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-    ],
-)
-
 # TODO(mrry): Remove this empty forwarding library.
 cc_library(
     name = "dataset",
@@ -139,17 +129,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "slide_dataset_op",
-    srcs = ["slide_dataset_op.cc"],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "padded_batch_dataset_op",
     srcs = ["padded_batch_dataset_op.cc"],
@@ -161,44 +140,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "dense_to_sparse_batch_dataset_op",
-    srcs = ["dense_to_sparse_batch_dataset_op.cc"],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
-tf_kernel_library(
-    name = "group_by_reducer_dataset_op",
-    srcs = ["group_by_reducer_dataset_op.cc"],
-    deps = [
-        ":captured_function",
-        "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
-tf_kernel_library(
-    name = "group_by_window_dataset_op",
-    srcs = ["group_by_window_dataset_op.cc"],
-    deps = [
-        ":captured_function",
-        ":window_dataset",
-        "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "filter_dataset_op",
     srcs = ["filter_dataset_op.cc"],
@@ -238,21 +179,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "map_and_batch_dataset_op",
-    srcs = ["map_and_batch_dataset_op.cc"],
-    deps = [
-        ":captured_function",
-        ":dataset_utils",
-        "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels:inplace_ops",
-    ],
-)
-
 cc_library(
     name = "parallel_map_iterator",
     srcs = ["parallel_map_iterator.cc"],
@@ -267,16 +193,6 @@ cc_library(
     ],
 )
 
-tf_kernel_library(
-    name = "parse_example_dataset_op",
-    srcs = ["parse_example_dataset_op.cc"],
-    deps = [
-        ":parallel_map_iterator",
-        "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:framework",
-    ],
-)
-
 tf_kernel_library(
     name = "parallel_map_dataset_op",
     srcs = ["parallel_map_dataset_op.cc"],
@@ -359,7 +275,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -430,28 +345,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "stats_dataset_ops",
-    srcs = ["stats_dataset_ops.cc"],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
-    ],
-)
-
-tf_kernel_library(
-    name = "stats_aggregator_dataset_op",
-    srcs = ["stats_aggregator_dataset_op.cc"],
-    deps = [
-        "//tensorflow/core:core_cpu_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "random_dataset_op",
     srcs = ["random_dataset_op.cc"],
@@ -527,17 +420,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "unbatch_dataset_op",
-    srcs = ["unbatch_dataset_op.cc"],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "zip_dataset_op",
     srcs = ["zip_dataset_op.cc"],
@@ -571,20 +453,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "sql_dataset_ops",
-    srcs = [
-        "sql_dataset_ops.cc",
-    ],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels/data/sql",
-    ],
-)
-
 tf_kernel_library(
     name = "iterator_ops",
     srcs = ["iterator_ops.cc"],
@@ -700,16 +568,12 @@ tf_kernel_library(
         ":cache_dataset_ops",
         ":concatenate_dataset_op",
         ":dataset_ops",
-        ":dense_to_sparse_batch_dataset_op",
         ":filter_by_component_dataset_op",
         ":filter_dataset_op",
         ":flat_map_dataset_op",
         ":generator_dataset_op",
-        ":group_by_reducer_dataset_op",
-        ":group_by_window_dataset_op",
         ":interleave_dataset_op",
         ":iterator_ops",
-        ":map_and_batch_dataset_op",
         ":map_dataset_op",
         ":map_defun_op",
         ":model_dataset_op",
@@ -719,7 +583,6 @@ tf_kernel_library(
         ":padded_batch_dataset_op",
         ":parallel_interleave_dataset_op",
         ":parallel_map_dataset_op",
-        ":parse_example_dataset_op",
         ":prefetch_dataset_op",
         ":random_dataset_op",
         ":range_dataset_op",
@@ -728,36 +591,17 @@ tf_kernel_library(
         ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
-        ":slide_dataset_op",
         ":sparse_tensor_slice_dataset_op",
-        ":sql_dataset_ops",
-        ":stats_aggregator_dataset_op",
-        ":stats_aggregator_ops",
-        ":stats_dataset_ops",
         ":take_dataset_op",
         ":tensor_dataset_op",
         ":tensor_queue_dataset_op",
         ":tensor_slice_dataset_op",
-        ":unbatch_dataset_op",
         ":window_dataset_op",
-        ":writer_ops",
         ":zip_dataset_op",
         "//tensorflow/core/kernels/data/experimental:dataset_kernels",
     ],
 )
 
-tf_kernel_library(
-    name = "writer_ops",
-    srcs = ["writer_ops.cc"],
-    deps = [
-        ":dataset_utils",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels:ops_util",
-    ],
-)
-
 tf_kernel_library(
     name = "map_defun_op",
     srcs = ["map_defun_op.cc"],
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 958c42a22a..e8d87b8dd0 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -11,35 +11,31 @@ load(
     "tf_kernel_library",
 )
 
-cc_library(
-    name = "indexed_dataset_headers",
-    hdrs = ["indexed_dataset.h"],
+tf_kernel_library(
+    name = "assert_next_dataset_op",
+    srcs = ["assert_next_dataset_op.cc"],
     deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//third_party/eigen3",
     ],
 )
 
 tf_kernel_library(
-    name = "indexed_dataset",
-    srcs = [
-        "identity_indexed_dataset.cc",
-        "indexed_dataset.cc",
-    ],
+    name = "csv_dataset_op",
+    srcs = ["csv_dataset_op.cc"],
     deps = [
-        ":indexed_dataset_headers",
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//third_party/eigen3",
+        "//tensorflow/core:lib_internal",
     ],
 )
 
 tf_kernel_library(
-    name = "prefetching_kernels",
-    srcs = ["prefetching_kernels.cc"],
+    name = "dense_to_sparse_batch_dataset_op",
+    srcs = ["dense_to_sparse_batch_dataset_op.cc"],
     deps = [
-        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -59,13 +55,29 @@ tf_kernel_library(
 )
 
 tf_kernel_library(
-    name = "csv_dataset_op",
-    srcs = ["csv_dataset_op.cc"],
+    name = "group_by_reducer_dataset_op",
+    srcs = ["group_by_reducer_dataset_op.cc"],
     deps = [
-        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:captured_function",
+    ],
+)
+
+tf_kernel_library(
+    name = "group_by_window_dataset_op",
+    srcs = ["group_by_window_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:captured_function",
+        "//tensorflow/core/kernels/data:window_dataset",
     ],
 )
 
@@ -79,6 +91,18 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "indexed_dataset_op",
+    srcs = ["indexed_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/kernels/data:dataset_utils",
+        "//third_party/eigen3",
+    ],
+)
+
 tf_kernel_library(
     name = "lmdb_dataset_op",
     srcs = ["lmdb_dataset_op.cc"],
@@ -92,12 +116,38 @@ tf_kernel_library(
 )
 
 tf_kernel_library(
-    name = "threadpool_dataset_op",
-    srcs = ["threadpool_dataset_op.cc"],
+    name = "map_and_batch_dataset_op",
+    srcs = ["map_and_batch_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels:inplace_ops",
+        "//tensorflow/core/kernels/data:captured_function",
+        "//tensorflow/core/kernels/data:dataset_utils",
+    ],
+)
+
+tf_kernel_library(
+    name = "matching_files_dataset_op",
+    srcs = ["matching_files_dataset_op.cc"],
     deps = [
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:dataset",
+    ],
+)
+
+tf_kernel_library(
+    name = "non_serializable_dataset_op",
+    srcs = ["non_serializable_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:framework",
         "//third_party/eigen3",
     ],
 )
@@ -118,23 +168,48 @@ tf_kernel_library(
 )
 
 tf_kernel_library(
-    name = "unique_dataset_op",
-    srcs = ["unique_dataset_op.cc"],
+    name = "parallel_interleave_dataset_op",
+    srcs = ["parallel_interleave_dataset_op.cc"],
     deps = [
-        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//third_party/eigen3",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:captured_function",
+        "//tensorflow/core/kernels/data:dataset_utils",
     ],
 )
 
 tf_kernel_library(
-    name = "assert_next_dataset_op",
-    srcs = ["assert_next_dataset_op.cc"],
+    name = "parse_example_dataset_op",
+    srcs = ["parse_example_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core/kernels/data:parallel_map_iterator",
+    ],
+)
+
+tf_kernel_library(
+    name = "prefetching_kernels",
+    srcs = ["prefetching_kernels.cc"],
     deps = [
+        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
-        "//third_party/eigen3",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_kernel_library(
+    name = "set_stats_aggregator_dataset_op",
+    srcs = ["set_stats_aggregator_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib_internal",
     ],
 )
 
@@ -148,24 +223,94 @@ tf_kernel_library(
 )
 
 tf_kernel_library(
-    name = "non_serializable_dataset_op",
-    srcs = ["non_serializable_dataset_op.cc"],
+    name = "sliding_window_dataset_op",
+    srcs = ["sliding_window_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_kernel_library(
+    name = "sql_dataset_op",
+    srcs = [
+        "sql_dataset_op.cc",
+    ],
     deps = [
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
-        "//third_party/eigen3",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data/experimental/sql",
     ],
 )
 
 tf_kernel_library(
-    name = "matching_files_dataset_op",
-    srcs = ["matching_files_dataset_op.cc"],
+    name = "stats_aggregator_ops",
+    srcs = ["stats_aggregator_ops.cc"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "stats_dataset_ops",
+    srcs = ["stats_dataset_ops.cc"],
+    deps = [
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "to_tf_record_op",
+    srcs = ["to_tf_record_op.cc"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels:ops_util",
+        "//tensorflow/core/kernels/data:dataset_utils",
+    ],
+)
+
+tf_kernel_library(
+    name = "threadpool_dataset_op",
+    srcs = ["threadpool_dataset_op.cc"],
     deps = [
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//third_party/eigen3",
+    ],
+)
+
+tf_kernel_library(
+    name = "unbatch_dataset_op",
+    srcs = ["unbatch_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels/data:dataset",
+    ],
+)
+
+tf_kernel_library(
+    name = "unique_dataset_op",
+    srcs = ["unique_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//third_party/eigen3",
     ],
 )
 
@@ -174,16 +319,29 @@ tf_kernel_library(
     deps = [
         ":assert_next_dataset_op",
         ":csv_dataset_op",
+        ":dense_to_sparse_batch_dataset_op",
         ":directed_interleave_dataset_op",
+        ":group_by_reducer_dataset_op",
+        ":group_by_window_dataset_op",
         ":ignore_errors_dataset_op",
-        ":indexed_dataset",
+        ":indexed_dataset_op",
         ":lmdb_dataset_op",
+        ":map_and_batch_dataset_op",
         ":matching_files_dataset_op",
         ":non_serializable_dataset_op",
         ":numa_map_and_batch_dataset_op",
+        ":parallel_interleave_dataset_op",
+        ":parse_example_dataset_op",
         ":prefetching_kernels",
+        ":set_stats_aggregator_dataset_op",
         ":sleep_dataset_op",
+        ":sliding_window_dataset_op",
+        ":sql_dataset_op",
+        ":stats_aggregator_ops",
+        ":stats_dataset_ops",
         ":threadpool_dataset_op",
+        ":to_tf_record_op",
+        ":unbatch_dataset_op",
         ":unique_dataset_op",
     ],
 )
diff --git a/tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
similarity index 98%
rename from tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
index d684d23b24..a9b3fb6551 100644
--- a/tensorflow/core/kernels/data/dense_to_sparse_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
@@ -304,8 +304,9 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("DenseToSparseBatchDataset").Device(DEVICE_CPU),
-                        DenseToSparseBatchDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalDenseToSparseBatchDataset").Device(DEVICE_CPU),
+    DenseToSparseBatchDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc b/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
similarity index 99%
rename from tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
index a01085491e..1c298cfdd6 100644
--- a/tensorflow/core/kernels/data/group_by_reducer_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
@@ -438,8 +438,9 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
   NameAttrList finalize_func_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("GroupByReducerDataset").Device(DEVICE_CPU),
-                        GroupByReducerDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalGroupByReducerDataset").Device(DEVICE_CPU),
+    GroupByReducerDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc b/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
similarity index 99%
rename from tensorflow/core/kernels/data/group_by_window_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
index d9f504a170..98603d5a73 100644
--- a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
@@ -523,8 +523,9 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
   NameAttrList window_size_func_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("GroupByWindowDataset").Device(DEVICE_CPU),
-                        GroupByWindowDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalGroupByWindowDataset").Device(DEVICE_CPU),
+    GroupByWindowDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/experimental/identity_indexed_dataset.cc b/tensorflow/core/kernels/data/experimental/identity_indexed_dataset.cc
deleted file mode 100644
index d10a3dea11..0000000000
--- a/tensorflow/core/kernels/data/experimental/identity_indexed_dataset.cc
+++ /dev/null
@@ -1,163 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/kernels/data/experimental/indexed_dataset.h"
-#include "tensorflow/core/lib/core/errors.h"
-
-namespace tensorflow {
-namespace data {
-namespace {
-
-class IdentityIndexedDatasetOp : public IndexedDatasetOpKernel {
- public:
-  using IndexedDatasetOpKernel::IndexedDatasetOpKernel;
-
-  void MakeIndexedDataset(OpKernelContext* ctx,
-                          IndexedDataset** output) override {
-    uint64 size = -1;
-    OP_REQUIRES_OK(ctx, ParseScalarArgument<uint64>(ctx, "size", &size));
-    OP_REQUIRES(ctx, size > 0, errors::InvalidArgument("`size` must be > 0"));
-    *output = new Dataset(ctx, size);
-  }
-
-  class Dataset : public IndexedDataset {
-   public:
-    Dataset(OpKernelContext* ctx, uint64 size)
-        : IndexedDataset(DatasetContext(ctx)), size_(size) {}
-
-    Status MaterializeDataset(
-        std::shared_ptr<MaterializedIndexedDataset>* materialized) override {
-      materialized->reset(new Materialized(this));
-      return Status::OK();
-    }
-
-    const DataTypeVector& output_dtypes() const override {
-      static DataTypeVector* dtypes = new DataTypeVector({DT_UINT64});
-      return *dtypes;
-    }
-
-    const std::vector<PartialTensorShape>& output_shapes() const override {
-      static std::vector<PartialTensorShape>* shapes =
-          new std::vector<PartialTensorShape>({{}});
-      return *shapes;
-    }
-
-    std::unique_ptr<IteratorBase> MakeIteratorInternal(
-        const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::IdentityIndexedDataset")}));
-    }
-
-    string DebugString() const override {
-      return "IdentityIndexedDataset::Dataset";
-    }
-
-    Status AsGraphDefInternal(SerializationContext* ctx,
-                              DatasetGraphDefBuilder* b,
-                              Node** node) const override {
-      return errors::Unimplemented(
-          "identity_indexed_dataset.AsGraphDefInternal");
-    }
-
-   private:
-    class Iterator : public DatasetIterator<Dataset> {
-     public:
-      explicit Iterator(const Params& params)
-          : DatasetIterator<Dataset>(params) {}
-      Status GetNextInternal(IteratorContext* ctx,
-                             std::vector<Tensor>* out_tensors,
-                             bool* end_of_sequence) override {
-        mutex_lock l(mu_);
-        if (cur_ < dataset()->size_) {
-          out_tensors->emplace_back(ctx->allocator({}), DT_UINT64,
-                                    TensorShape({}));
-          out_tensors->back().scalar<uint64>()() = cur_++;
-          *end_of_sequence = false;
-          return Status::OK();
-        }
-        *end_of_sequence = true;
-        return Status::OK();
-      }
-
-     protected:
-      std::shared_ptr<model::Node> CreateNode(
-          IteratorContext* ctx, model::Node::Args args) const override {
-        return model::MakeKnownRatioNode(std::move(args),
-                                         /*ratio=*/1);
-      }
-
-     private:
-      mutex mu_;
-      uint64 cur_ GUARDED_BY(mu_);
-    };
-
-    class Materialized : public MaterializedIndexedDataset {
-     public:
-      explicit Materialized(Dataset* dataset) : dataset_(dataset) {
-        dataset->Ref();
-      }
-
-      ~Materialized() override {
-        // TODO(saeta): Pull this into MaterializedIndexedDataset
-        dataset_->Unref();
-      }
-
-      const DataTypeVector& output_dtypes() const override {
-        return dataset_->output_dtypes();
-      }
-
-      const std::vector<PartialTensorShape>& output_shapes() const override {
-        return dataset_->output_shapes();
-      }
-
-      Status Get(IteratorContext&& ctx, uint64 index,
-                 std::vector<Tensor>* out_tensors) const override {
-        LOG(INFO) << "Materialized(" << dataset_->size_ << ")::Get(" << index
-                  << ")";
-        if (index >= dataset_->size_) {
-          // Note: use InvalidArgument instead of OutOfRange error because many
-          // things consider OutOfRange to be a "clean termination" error.
-          return errors::InvalidArgument(
-              "Index ", index,
-              " is out of range for this dataset. (Size is: ", dataset_->size_,
-              ".)");
-        }
-        out_tensors->emplace_back(ctx.allocator({}), DT_UINT64,
-                                  TensorShape({}));
-        out_tensors->back().scalar<uint64>()() = index;
-        return Status::OK();
-      }
-
-      Status Size(uint64* size) const override {
-        *size = dataset_->size_;
-        return Status::OK();
-      }
-
-     private:
-      const Dataset* const dataset_;  // Not owned.
-    };
-
-    const uint64 size_;
-    std::shared_ptr<Materialized> materialized_;
-  };
-};
-
-REGISTER_KERNEL_BUILDER(
-    Name("ExperimentalIdentityIndexedDataset").Device(DEVICE_CPU),
-    IdentityIndexedDatasetOp);
-
-}  // namespace
-}  // namespace data
-}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/experimental/indexed_dataset.h b/tensorflow/core/kernels/data/experimental/indexed_dataset.h
deleted file mode 100644
index 27a8360cbc..0000000000
--- a/tensorflow/core/kernels/data/experimental/indexed_dataset.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
-#define TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
-
-#include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/op_kernel.h"
-
-namespace tensorflow {
-namespace data {
-
-// TODO(saeta): Urgh, this is ugly.
-class MaterializedIndexedDataset {
- public:
-  virtual ~MaterializedIndexedDataset() = default;
-
-  // Retrieve the element at a given index. The output tensors are stored in
-  // out_tensors.
-  //
-  // If `index` is greater than `Size()`, tensorflow::errors::OutOfRangeError is
-  // returned.
-  //
-  // Get is thread-safe.
-  virtual Status Get(IteratorContext&& ctx, uint64 index,
-                     std::vector<Tensor>* out_tensors) const = 0;
-
-  // Size determines the number of elements in this IndexedDataset.
-  //
-  // Size is thread-safe.
-  virtual Status Size(uint64* size) const = 0;
-
-  // Returns a vector of DataType values, representing the respective
-  // element types of each tuple component in the outputs of this dataset.
-  virtual const DataTypeVector& output_dtypes() const = 0;
-
-  // Returns a vector of tensor shapes, representing the respective
-  // (and possibly partially defined) shapes of each tuple component
-  // in the outputs of this dataset.
-  virtual const std::vector<PartialTensorShape>& output_shapes() const = 0;
-};
-
-// IndexedDataset represents a dataset that supports random access in addition
-// to iterator-based sequential access.
-//
-// Note: IndexedDatasets are HIGHLY experimental at this time. Expect
-// significant (backwards incompatible) changes!
-class IndexedDataset : public DatasetBase {
- public:
-  IndexedDataset(DatasetContext&& ctx) : DatasetBase(std::move(ctx)) {}
-
-  // Materialize (if necessary) the dataset, and return a pointer.
-  // TODO(saeta): Add in `IteratorContext* ctx` when materializing.
-  virtual Status MaterializeDataset(
-      std::shared_ptr<MaterializedIndexedDataset>* materialized) = 0;
-};
-
-// IndexedDatasetOpKernel abstracts away interfacing IndexedDatasets with the
-// rest of the TensorFlow runtime.
-//
-// Most IndexedDataset's will be private members of classes inheriting from this
-// class.
-class IndexedDatasetOpKernel : public OpKernel {
- public:
-  IndexedDatasetOpKernel(OpKernelConstruction* ctx) : OpKernel(ctx) {}
-  void Compute(OpKernelContext* ctx) final;
-
- protected:
-  // Subclasses should implement this method. It will be called during Compute
-  // execution.
-  virtual void MakeIndexedDataset(OpKernelContext* ctx,
-                                  IndexedDataset** output) = 0;
-
-  template <typename T>
-  Status ParseScalarArgument(OpKernelContext* ctx,
-                             const StringPiece& argument_name, T* output) {
-    const Tensor* argument_t;
-    TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t));
-    if (!TensorShapeUtils::IsScalar(argument_t->shape())) {
-      return errors::InvalidArgument(argument_name, " must be a scalar");
-    }
-    *output = argument_t->scalar<T>()();
-    return Status::OK();
-  }
-};
-
-// Validates and extracts an `IndexedDataset` object from `tensor`.
-//
-// `tensor` must have been written by a call to
-// `StoreIndexedDatasetInVariantTensor`
-//
-// The retrieved pointer isa  borrowed reference to the dataset, which is owned
-// by the tensor. The consumer must either acquire its own reference to the
-// dataset by calling `(*out_dataset)->Ref()`, or ensure that `tensor` is not
-// destroyed or mutated while the retrieved pointer is in use.
-Status GetIndexedDatasetFromVariantTensor(const Tensor& tensor,
-                                          IndexedDataset** out_dataset);
-
-// Stores an `IndexedDataset` object in `tensor.`
-//
-// The ownership of `dataset` is transferred to `tensor`.
-Status StoreIndexedDatasetInVariantTensor(IndexedDataset* dataset,
-                                          Tensor* tensor);
-
-}  // namespace data
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
diff --git a/tensorflow/core/kernels/data/experimental/indexed_dataset.cc b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
similarity index 62%
rename from tensorflow/core/kernels/data/experimental/indexed_dataset.cc
rename to tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
index 75ea462f40..a07eaebdf9 100644
--- a/tensorflow/core/kernels/data/experimental/indexed_dataset.cc
+++ b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
@@ -12,10 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/core/kernels/data/experimental/indexed_dataset.h"
 
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 
@@ -23,42 +25,79 @@ namespace tensorflow {
 namespace data {
 namespace {
 
-Status VerifyTypesMatch(const DataTypeVector& expected,
-                        const DataTypeVector& received) {
-  if (expected.size() != received.size()) {
-    return errors::InvalidArgument(
-        "Number of components does not match: expected ", expected.size(),
-        " types but got ", received.size(), ".");
-  }
-  for (size_t i = 0; i < expected.size(); ++i) {
-    if (expected[i] != received[i]) {
-      return errors::InvalidArgument("Data type mismatch at component ", i,
-                                     ": expected ", DataTypeString(expected[i]),
-                                     " but got ", DataTypeString(received[i]),
-                                     ".");
-    }
-  }
-  return Status::OK();
-}
+// TODO(saeta): Urgh, this is ugly.
+class MaterializedIndexedDataset {
+ public:
+  virtual ~MaterializedIndexedDataset() = default;
+
+  // Retrieve the element at a given index. The output tensors are stored in
+  // out_tensors.
+  //
+  // If `index` is greater than `Size()`, tensorflow::errors::OutOfRangeError is
+  // returned.
+  //
+  // Get is thread-safe.
+  virtual Status Get(IteratorContext&& ctx, uint64 index,
+                     std::vector<Tensor>* out_tensors) const = 0;
+
+  // Size determines the number of elements in this IndexedDataset.
+  //
+  // Size is thread-safe.
+  virtual Status Size(uint64* size) const = 0;
+
+  // Returns a vector of DataType values, representing the respective
+  // element types of each tuple component in the outputs of this dataset.
+  virtual const DataTypeVector& output_dtypes() const = 0;
+
+  // Returns a vector of tensor shapes, representing the respective
+  // (and possibly partially defined) shapes of each tuple component
+  // in the outputs of this dataset.
+  virtual const std::vector<PartialTensorShape>& output_shapes() const = 0;
+};
 
-Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
-                              const std::vector<PartialTensorShape>& received) {
-  if (expected.size() != received.size()) {
-    return errors::InvalidArgument(
-        "Number of components does not match: expected ", expected.size(),
-        " shapes but got ", received.size(), ".");
-  }
-  for (size_t i = 0; i < expected.size(); ++i) {
-    if (!expected[i].IsCompatibleWith(received[i])) {
-      return errors::InvalidArgument("Incompatible shapes at component ", i,
-                                     ": expected ", expected[i].DebugString(),
-                                     " but got ", received[i].DebugString(),
-                                     ".");
+// IndexedDataset represents a dataset that supports random access in addition
+// to iterator-based sequential access.
+//
+// Note: IndexedDatasets are HIGHLY experimental at this time. Expect
+// significant (backwards incompatible) changes!
+class IndexedDataset : public DatasetBase {
+ public:
+  explicit IndexedDataset(DatasetContext&& ctx) : DatasetBase(std::move(ctx)) {}
+
+  // Materialize (if necessary) the dataset, and return a pointer.
+  // TODO(saeta): Add in `IteratorContext* ctx` when materializing.
+  virtual Status MaterializeDataset(
+      std::shared_ptr<MaterializedIndexedDataset>* materialized) = 0;
+};
+
+// IndexedDatasetOpKernel abstracts away interfacing IndexedDatasets with the
+// rest of the TensorFlow runtime.
+//
+// Most IndexedDataset's will be private members of classes inheriting from this
+// class.
+class IndexedDatasetOpKernel : public OpKernel {
+ public:
+  explicit IndexedDatasetOpKernel(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+  void Compute(OpKernelContext* ctx) final;
+
+ protected:
+  // Subclasses should implement this method. It will be called during Compute
+  // execution.
+  virtual void MakeIndexedDataset(OpKernelContext* ctx,
+                                  IndexedDataset** output) = 0;
+
+  template <typename T>
+  Status ParseScalarArgument(OpKernelContext* ctx,
+                             const StringPiece argument_name, T* output) {
+    const Tensor* argument_t;
+    TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t));
+    if (!TensorShapeUtils::IsScalar(argument_t->shape())) {
+      return errors::InvalidArgument(argument_name, " must be a scalar");
     }
+    *output = argument_t->scalar<T>()();
+    return Status::OK();
   }
-
-  return Status::OK();
-}
+};
 
 class MaterializedDatasetResource : public ResourceBase {
  public:
@@ -164,8 +203,6 @@ class IndexedDatasetVariantWrapper {
   IndexedDataset* const dataset_;  // Owns one reference.
 };
 
-}  // namespace
-
 Status GetIndexedDatasetFromVariantTensor(const Tensor& tensor,
                                           IndexedDataset** out_dataset) {
   if (!(tensor.dtype() == DT_VARIANT ||
@@ -211,8 +248,6 @@ void IndexedDatasetOpKernel::Compute(OpKernelContext* ctx) {
   }
 }
 
-namespace {
-
 class MaterializedHandleOp : public OpKernel {
  public:
   explicit MaterializedHandleOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -370,6 +405,144 @@ REGISTER_KERNEL_BUILDER(
     Name("ExperimentalIndexedDatasetGet").Device(DEVICE_CPU),
     IndexedDatasetGet);
 
+class IdentityIndexedDatasetOp : public IndexedDatasetOpKernel {
+ public:
+  using IndexedDatasetOpKernel::IndexedDatasetOpKernel;
+
+  void MakeIndexedDataset(OpKernelContext* ctx,
+                          IndexedDataset** output) override {
+    uint64 size = -1;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument<uint64>(ctx, "size", &size));
+    OP_REQUIRES(ctx, size > 0, errors::InvalidArgument("`size` must be > 0"));
+    *output = new Dataset(ctx, size);
+  }
+
+  class Dataset : public IndexedDataset {
+   public:
+    Dataset(OpKernelContext* ctx, uint64 size)
+        : IndexedDataset(DatasetContext(ctx)), size_(size) {}
+
+    Status MaterializeDataset(
+        std::shared_ptr<MaterializedIndexedDataset>* materialized) override {
+      materialized->reset(new Materialized(this));
+      return Status::OK();
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      static DataTypeVector* dtypes = new DataTypeVector({DT_UINT64});
+      return *dtypes;
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      static std::vector<PartialTensorShape>* shapes =
+          new std::vector<PartialTensorShape>({{}});
+      return *shapes;
+    }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::IdentityIndexedDataset")}));
+    }
+
+    string DebugString() const override {
+      return "IdentityIndexedDataset::Dataset";
+    }
+
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** node) const override {
+      return errors::Unimplemented(
+          "identity_indexed_dataset.AsGraphDefInternal");
+    }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {}
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+        if (cur_ < dataset()->size_) {
+          out_tensors->emplace_back(ctx->allocator({}), DT_UINT64,
+                                    TensorShape({}));
+          out_tensors->back().scalar<uint64>()() = cur_++;
+          *end_of_sequence = false;
+          return Status::OK();
+        }
+        *end_of_sequence = true;
+        return Status::OK();
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
+
+     private:
+      mutex mu_;
+      uint64 cur_ GUARDED_BY(mu_);
+    };
+
+    class Materialized : public MaterializedIndexedDataset {
+     public:
+      explicit Materialized(Dataset* dataset) : dataset_(dataset) {
+        dataset->Ref();
+      }
+
+      ~Materialized() override {
+        // TODO(saeta): Pull this into MaterializedIndexedDataset
+        dataset_->Unref();
+      }
+
+      const DataTypeVector& output_dtypes() const override {
+        return dataset_->output_dtypes();
+      }
+
+      const std::vector<PartialTensorShape>& output_shapes() const override {
+        return dataset_->output_shapes();
+      }
+
+      Status Get(IteratorContext&& ctx, uint64 index,
+                 std::vector<Tensor>* out_tensors) const override {
+        LOG(INFO) << "Materialized(" << dataset_->size_ << ")::Get(" << index
+                  << ")";
+        if (index >= dataset_->size_) {
+          // Note: use InvalidArgument instead of OutOfRange error because many
+          // things consider OutOfRange to be a "clean termination" error.
+          return errors::InvalidArgument(
+              "Index ", index,
+              " is out of range for this dataset. (Size is: ", dataset_->size_,
+              ".)");
+        }
+        out_tensors->emplace_back(ctx.allocator({}), DT_UINT64,
+                                  TensorShape({}));
+        out_tensors->back().scalar<uint64>()() = index;
+        return Status::OK();
+      }
+
+      Status Size(uint64* size) const override {
+        *size = dataset_->size_;
+        return Status::OK();
+      }
+
+     private:
+      const Dataset* const dataset_;  // Not owned.
+    };
+
+    const uint64 size_;
+    std::shared_ptr<Materialized> materialized_;
+  };
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalIdentityIndexedDataset").Device(DEVICE_CPU),
+    IdentityIndexedDatasetOp);
+
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
similarity index 99%
rename from tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index ac001c22b9..203c977b4f 100644
--- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -318,7 +318,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         while (num_calls_ > 0) {
           cond_var_->wait(l);
         }
-        CHECK_EQ(num_calls_, 0);
+        DCHECK_EQ(num_calls_, 0);
         TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
         TF_RETURN_IF_ERROR(
             writer->WriteScalar(full_name("call_counter"), call_counter_));
@@ -810,11 +810,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
   NameAttrList func_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("MapAndBatchDataset").Device(DEVICE_CPU),
-                        MapAndBatchDatasetOp);
-
-REGISTER_KERNEL_BUILDER(Name("MapAndBatchDatasetV2").Device(DEVICE_CPU),
-                        MapAndBatchDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalMapAndBatchDataset").Device(DEVICE_CPU),
+    MapAndBatchDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
new file mode 100644
index 0000000000..0230f90aba
--- /dev/null
+++ b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
@@ -0,0 +1,1085 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <atomic>
+#include <deque>
+#include <utility>
+
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/stats_aggregator.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/data/captured_function.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/gtl/cleanup.h"
+#include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+// See documentation in ../../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &interleave_func_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+  }
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 cycle_length = 0;
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument(ctx, "cycle_length", &cycle_length));
+    OP_REQUIRES(ctx, cycle_length > 0,
+                errors::InvalidArgument("`cycle_length` must be > 0"));
+
+    int64 block_length = 0;
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument(ctx, "block_length", &block_length));
+    OP_REQUIRES(ctx, block_length > 0,
+                errors::InvalidArgument("`block_length` must be > 0"));
+
+    bool sloppy = false;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "sloppy", &sloppy));
+
+    int64 buffer_output_elements = 0;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "buffer_output_elements",
+                                            &buffer_output_elements));
+    OP_REQUIRES(
+        ctx, buffer_output_elements > 0,
+        errors::InvalidArgument("`buffer_output_elements` must be > 0"));
+
+    int64 prefetch_input_elements = 0;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "prefetch_input_elements",
+                                            &prefetch_input_elements));
+    OP_REQUIRES(
+        ctx, prefetch_input_elements >= 0,
+        errors::InvalidArgument("`prefetch_input_elements` must be >= 0"));
+
+    std::unique_ptr<CapturedFunction> captured_func;
+    OP_REQUIRES_OK(
+        ctx, CapturedFunction::Create(interleave_func_, ctx, "other_arguments",
+                                      &captured_func));
+
+    *output =
+        new Dataset(ctx, input, interleave_func_, std::move(captured_func),
+                    cycle_length, block_length, sloppy, buffer_output_elements,
+                    prefetch_input_elements, output_types_, output_shapes_);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input,
+            const NameAttrList& func,
+            std::unique_ptr<CapturedFunction> captured_func, int64 cycle_length,
+            int64 block_length, bool sloppy, int64 buffer_output_elements,
+            int64 prefetch_input_elements, const DataTypeVector& output_types,
+            const std::vector<PartialTensorShape>& output_shapes)
+        : DatasetBase(DatasetContext(ctx)),
+          input_(input),
+          interleave_func_(func),
+          captured_func_(std::move(captured_func)),
+          cycle_length_(cycle_length),
+          block_length_(block_length),
+          sloppy_(sloppy),
+          buffer_output_elements_(buffer_output_elements),
+          prefetch_input_elements_(prefetch_input_elements),
+          output_types_(output_types),
+          output_shapes_(output_shapes) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::ParallelInterleave")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return output_types_;
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() const override {
+      return "ParallelInterleaveDatasetOp::Dataset";
+    }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      TF_RETURN_IF_ERROR(b->AddFunction(ctx, interleave_func_.name()));
+      Node* input_node;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_node));
+      Node* cycle_length_node;
+      TF_RETURN_IF_ERROR(b->AddScalar(cycle_length_, &cycle_length_node));
+      Node* block_length_node;
+      TF_RETURN_IF_ERROR(b->AddScalar(block_length_, &block_length_node));
+      Node* sloppy_node;
+      TF_RETURN_IF_ERROR(b->AddScalar(sloppy_, &sloppy_node));
+      Node* buffer_output_elements_node;
+      TF_RETURN_IF_ERROR(
+          b->AddScalar(buffer_output_elements_, &buffer_output_elements_node));
+      Node* prefetch_input_elements_node;
+      TF_RETURN_IF_ERROR(b->AddScalar(prefetch_input_elements_,
+                                      &prefetch_input_elements_node));
+      DataTypeVector other_arguments_types;
+      other_arguments_types.reserve(captured_func_->captured_inputs().size());
+      std::vector<Node*> other_arguments;
+      other_arguments.reserve(captured_func_->captured_inputs().size());
+      for (const Tensor& t : captured_func_->captured_inputs()) {
+        Node* node;
+        TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
+        other_arguments.emplace_back(node);
+        other_arguments_types.emplace_back(t.dtype());
+      }
+      AttrValue f;
+      b->BuildAttrValue(interleave_func_, &f);
+      AttrValue other_arguments_types_attr;
+      b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
+
+      TF_RETURN_IF_ERROR(b->AddDataset(
+          this,
+          {{0, input_node},
+           {2, cycle_length_node},
+           {3, block_length_node},
+           {4, sloppy_node},
+           {5, buffer_output_elements_node},
+           {6, prefetch_input_elements_node}},
+          {{1, other_arguments}},
+          {{"f", f}, {"Targuments", other_arguments_types_attr}}, output));
+      return Status::OK();
+    }
+
+   private:
+    int64 num_threads() const {
+      return cycle_length_ + prefetch_input_elements_;
+    }
+
+    // Parallel interleave's implementation is designed around a few principles:
+    //  1. Thread creation is relatively expensive. (Not reusing
+    //     threads causes a number of indirect costs such as poorer tcmalloc
+    //     performance due to thread-local caches, etc.) We allocate a fixed
+    //     number of threads at the start and never change. This is why we've
+    //     fused functionality that is theoretically orthogonal (i.e.
+    //     .prefetch()) into the implementation.
+    //  2. Drop-in replacement for standard interleave. The goal will be to
+    //     auto-opt people into an optimized implementation without any work
+    //     on the customer's part. We thus go through great pains to maintain
+    //     identical iteration orders, full determinism (disabled only via a
+    //     flag, etc.)
+    //  3. Performance across a variety of environments and I/O envelopes.
+    //
+    // The actual implementation centers around a collection of worker threads
+    // and their corresponding worker state (tracked in the `workers_` vector).
+    // Worker threads repeatedly receive a vector of Tensors that are used as
+    // input to the flat-map function (`captured_func_`). The output of this
+    // function must be a dataset. The worker thread then repeatedly calls
+    // `GetNext()`, maintaining a buffer of elements to minimize the likelihood
+    // that a caller will block waiting for an element to be produced.
+    //
+    // Pointers to these worker states are kept in 2 disjoint data structures:
+    //  1. `interleave_indices_` is a vector containing indices of WorkerStates
+    //     in `workers_` that we are interleaving. Worker threads backing these
+    //     WorkerStates should be regularly producing values.
+    //  2. `staging_indices_` is a deque containing indices of WorkerStates in
+    //     `workers_` that we will move to `interleave_indices_` when an
+    //     iterator in `interleave_indices_` is exhausted.
+    //
+    // The client calls `GetNext[Internal]()` to retrieve an output element. The
+    // internal implementation updates the state of `interleave_indices_` and
+    // `staging_indices_` as output iterators (run by the worker threads) are
+    // exhausted.
+    //
+    // `input_impl_` is the input iterator that generates arguments for the
+    // flat-map function (`captured_func_`). It is set to an iterator at
+    // Iterator construction, and is fixed until we consume all input elements.
+    // Once it is exhausted, we reset the unique_ptr to eagerly deallocate
+    // memory.
+    //
+    // A few invariants are maintained:
+    //  1. No element in interleave_indices_ should be a -1 unless
+    //     `staging_indices_` is empty and `input_impl_` is empty.
+    //  2. Every `worker_` element is pointed to by at most one element of the
+    //     union of `interleave_indices_` and `staging_indices_`.
+    //  3. Unless `input_impl_` is empty, every `worker_` must be pointed to by
+    //     an element in `interleave_indices_` or `staging_indices_`.
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            workers_(dataset()->num_threads()),
+            worker_thread_states_(dataset()->num_threads()) {}
+
+      ~Iterator() override {
+        mutex_lock l(mu_);
+        cancelled_ = true;
+        // Notify all workers in case they are blocked.
+        for (auto& worker : workers_) {
+          worker.cond_var.notify_all();
+        }
+      }
+
+      Status Initialize(IteratorContext* ctx) override {
+        TF_RETURN_IF_ERROR(
+            dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
+      }
+
+      // It is implemented so that it matches the deterministic interleave
+      // unless getting the next element would block and we are allowed to be
+      // sloppy.
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+        TF_RETURN_IF_ERROR(EnsureWorkerThreadsStarted(ctx));
+        while (!cancelled_) {
+          // Wait for an item to become available, blocking if necessary. If we
+          // are allowed to be sloppy, we can skip over input datasets that do
+          // not have an item readily available.
+          bool can_produce_elements = false;
+          bool must_wait_for_input = true;
+          for (int64 i = 0; i < interleave_indices_.size(); ++i) {
+            int64 index = (next_index_ + i) % interleave_indices_.size();
+            int64 current_worker_index = interleave_indices_[index];
+            if (current_worker_index < 0) {
+              continue;  // Empty interleave elements.
+            }
+            WorkerState* current_worker = &workers_[current_worker_index];
+            can_produce_elements |= current_worker->MayHaveElements();
+            if (!current_worker->outputs.empty()) {
+              // We have an element!
+              next_index_ = index;
+              const bool element_acquired_sloppily =
+                  dataset()->sloppy_ && i > 1;
+              if (!element_acquired_sloppily) {
+                // If the element was acquired in the regular (non-sloppy)
+                // order, then advance the current block and cycle pointers to
+                // the next element in the regular order.
+                block_count_++;
+                if (block_count_ == dataset()->block_length_) {
+                  next_index_ = (index + 1) % interleave_indices_.size();
+                  block_count_ = 0;
+                }
+              } else {
+                block_count_ = 0;
+              }
+              *end_of_sequence = false;
+              Status s = current_worker->outputs.front().status;
+              current_worker->outputs.front().output.swap(*out_tensors);
+              current_worker->outputs.pop_front();
+              current_worker->cond_var.notify_one();
+              return s;
+            } else if (current_worker->is_producing && !dataset()->sloppy_) {
+              // current_worker.outputs.empty(), and we must wait for this
+              // iterator.
+              if (next_index_ != index) {
+                // We have advanced to a new iterator; reset block counts.
+                next_index_ = index;
+                block_count_ = 0;
+              }
+              break;
+            } else if (!current_worker->is_producing) {
+              // This iterator has reached end of input.
+              interleave_indices_[index] = -1;
+              if (input_impl_) {
+                // Start prefetching a new iterator.
+                std::vector<Tensor> args;
+                bool end_of_input = false;
+                Status s = input_impl_->GetNext(ctx, &args, &end_of_input);
+                if (end_of_input) {
+                  input_impl_.reset();
+                } else {
+                  current_worker->SetInputs(s, std::move(args));
+                  staging_indices_.emplace_back(current_worker_index);
+                }
+              }
+
+              if (!staging_indices_.empty()) {
+                // Move a worker from `staging_indices_` to
+                // `interleave_indices_`.
+                interleave_indices_[index] = staging_indices_.front();
+                staging_indices_.pop_front();
+
+                next_index_ = (index + 1) % interleave_indices_.size();
+                block_count_ = 0;
+                // Restart the inner [for] loop
+                can_produce_elements = true;
+                must_wait_for_input = false;
+                break;
+              }
+            }
+          }
+
+          if (!can_produce_elements && !input_impl_) {
+            // No potential for future values.
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+
+          if (must_wait_for_input) {
+            // Wait for elements to become available.
+            RecordStop(ctx);
+            if (dataset()->sloppy_) {
+              sloppy_cond_var_.wait(l);
+            } else {
+              workers_[interleave_indices_[next_index_]].cond_var.wait(l);
+            }
+            RecordStart(ctx);
+          }
+        }
+        return errors::Cancelled(
+            "ParallelInterleaveDatasetOp::Dataset::Iterator::GetNext");
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeAsyncInterleaveManyNode(std::move(args),
+                                                  /*parameters=*/{});
+      }
+
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        // The order of locking is important here to avoid deadlock.
+        mutex_lock l(mu_);
+        mutex_lock ckpt_l(ckpt_mu_);
+        if (input_impl_) {
+          TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+        } else {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_exhausted"), ""));
+        }
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("next_index"), next_index_));
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("block_count"), block_count_));
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("workers_size"), workers_.size()));
+        for (int i = 0; i < workers_.size(); ++i) {
+          TF_RETURN_IF_ERROR(WriteWorkerStateLocked(writer, i));
+        }
+        for (int i = 0; i < worker_thread_states_.size(); ++i) {
+          TF_RETURN_IF_ERROR(WriteWorkerThreadStateLocked(writer, i));
+        }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("interleave_size"),
+                                               interleave_indices_.size()));
+        for (int i = 0; i < interleave_indices_.size(); ++i) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat("interleave_indices_", i)),
+              interleave_indices_[i]));
+        }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("staging_size"),
+                                               staging_indices_.size()));
+        for (int i = 0; i < staging_indices_.size(); ++i) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat("staging_indices_", i)),
+              staging_indices_[i]));
+        }
+        if (!worker_threads_.empty()) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("worker_threads_running"), ""));
+        }
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        // The order of locking is important here to avoid deadlock.
+        mutex_lock l(mu_);
+        mutex_lock ckpt_l(ckpt_mu_);
+        if (!reader->Contains(full_name("input_exhausted"))) {
+          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+        } else {
+          input_impl_.reset();
+        }
+        int64 temp;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("next_index"), &temp));
+        next_index_ = size_t(temp);
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("block_count"), &temp));
+        block_count_ = size_t(temp);
+
+        // Restore WorkerStates.
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("workers_size"), &temp));
+        if (temp != dataset()->num_threads()) {
+          return errors::Internal("Expected ", dataset()->num_threads(),
+                                  " worker states but found ", temp, ".");
+        }
+        for (size_t i = 0; i < dataset()->num_threads(); ++i) {
+          TF_RETURN_IF_ERROR(ReadWorkerStateLocked(reader, i, ctx));
+        }
+        for (size_t i = 0; i < dataset()->num_threads(); ++i) {
+          TF_RETURN_IF_ERROR(ReadWorkerThreadStateLocked(reader, i, ctx));
+        }
+
+        // Restore `interleave_indices_`.
+        std::set<int64> all_indices;
+        {
+          int64 interleave_size;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("interleave_size"),
+                                                &interleave_size));
+          interleave_indices_.reserve(interleave_size);
+          for (int64 i = 0; i < interleave_size; ++i) {
+            int64 temp;
+            TF_RETURN_IF_ERROR(reader->ReadScalar(
+                full_name(strings::StrCat("interleave_indices_", i)), &temp));
+            if (temp >= 0 && all_indices.find(temp) != all_indices.end()) {
+              return errors::Internal(
+                  "Duplicate entry for ", temp,
+                  " found when reading interleave and staging indices.");
+            }
+            if (temp >= 0) {
+              all_indices.insert(temp);
+            }
+            interleave_indices_.emplace_back(temp);
+          }
+        }
+
+        // Restore `staging_indices_`.
+        {
+          int64 staging_size;
+          TF_RETURN_IF_ERROR(
+              reader->ReadScalar(full_name("staging_size"), &staging_size));
+          for (int i = 0; i < staging_size; ++i) {
+            int64 temp;
+            TF_RETURN_IF_ERROR(reader->ReadScalar(
+                full_name(strings::StrCat("staging_indices_", i)), &temp));
+            if (all_indices.find(temp) != all_indices.end()) {
+              return errors::Internal(
+                  "Duplicate entry for ", temp,
+                  " found when reading interleave and staging indices.");
+            }
+            if (temp >= 0) {
+              all_indices.insert(temp);
+            }
+            staging_indices_.emplace_back(temp);
+          }
+        }
+
+        // Start Worker threads.
+        if (reader->Contains(full_name("worker_threads_running"))) {
+          worker_threads_.reserve(dataset()->num_threads());
+          for (size_t i = 0; i < dataset()->num_threads(); ++i) {
+            std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
+            worker_threads_.emplace_back(ctx->env()->StartThread(
+                {}, strings::StrCat("tf_data_parallel_interleave_worker_", i),
+                [this, new_ctx, i]() { WorkerThread(new_ctx, i); }));
+          }
+        }
+        return Status::OK();
+      }
+
+     private:
+      // OutputElem contains the information from a call to GetNext by an output
+      // iterator.
+      struct OutputElem {
+        // The output iterator sets `status` if getting the output element
+        // fails.
+        Status status;
+        // The buffered data element.
+        std::vector<Tensor> output;
+
+        explicit OutputElem(const Status& s) : status(s) {}
+      };
+
+      // Worker threads operate on their relevant WorkerState structs.
+      //
+      // WorkerState's fields are all protected by mu_;
+      struct WorkerState {
+        // The arguments to be used to construct an output iterator.
+        std::vector<Tensor> input;
+        // The buffered output elements.
+        std::deque<OutputElem> outputs;
+        // Set to true iff the worker thread expects to append more elements to
+        // outputs. is_producing can be false despite !outputs.empty().
+        // Concretely, all output elements will have been consumed only when:
+        // is_producing == false && outputs.empty();
+        bool is_producing = false;
+        // Condition variable used to coordinate between threads. The worker
+        // thread waits on this condition variable when it is either (1) waiting
+        // for the main thread to add arguments to `input`, or (2) waiting for
+        // the main thread to consume an element of `outputs`. The main thread
+        // waits on cond_var if it is waiting for the worker thread to produce
+        // an element into `outputs` (this implies sloppy_==false).
+        condition_variable cond_var;
+
+        inline bool MayHaveElements() const {
+          return is_producing || !outputs.empty();
+        }
+
+        // Sets inputs for a worker thread and notifies it to start processing.
+        void SetInputs(const Status& s, std::vector<Tensor> input_arguments) {
+          if (s.ok()) {
+            DCHECK(!MayHaveElements())
+                << "Tried to start inputs, despite already producing!";
+            input = std::move(input_arguments);
+            is_producing = true;
+            cond_var.notify_one();
+          } else {
+            outputs.emplace_back(s);
+          }
+        }
+      };
+
+      // The internal state of a worker thread that is not already captured
+      // in its `WorkerState`.
+      //
+      // This is needed only for checkpointing purposes. We keep this
+      // separate from `WorkerState` and guard its fields using a separate
+      // lock `ckpt_mu_` so as to not affect the performance of main pipeline.
+      struct WorkerThreadState {
+        // The output element that has been produced from the input iterator
+        // and is waiting to be added to `WorkerState.outputs`.
+        OutputElem output_elem;
+
+        // Whether the input iterator returned an `end_of_sequence`.
+        bool end_of_sequence = false;
+
+        // Status returned from `MakeIteratorFromInputElement`.
+        Status iterator_creation_status;
+
+        // The arguments to be used to construct `iterator`.
+        std::vector<Tensor> input;
+
+        std::unique_ptr<IteratorBase> iterator;
+
+        WorkerThreadState() : output_elem(Status::OK()) {}
+      };
+
+      Status EnsureWorkerThreadsStarted(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        if (worker_threads_.empty()) {
+          worker_threads_.reserve(dataset()->num_threads());
+          for (int64 i = 0; i < dataset()->num_threads(); ++i) {
+            std::vector<Tensor> args;
+            bool end_of_input = false;
+            Status s = input_impl_->GetNext(ctx, &args, &end_of_input);
+            if (end_of_input) {
+              input_impl_.reset();
+              return Status::OK();
+            }
+            workers_[i].SetInputs(s, std::move(args));
+            std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
+            worker_threads_.emplace_back(ctx->env()->StartThread(
+                {}, strings::StrCat("tf_data_parallel_interleave_worker_", i),
+                [this, new_ctx, i]() { WorkerThread(new_ctx, i); }));
+            if (i < dataset()->cycle_length_) {
+              interleave_indices_.push_back(i);
+            } else {
+              staging_indices_.push_back(i);
+            }
+          }
+          DCHECK(interleave_indices_.size() == dataset()->cycle_length_);
+          DCHECK(staging_indices_.size() ==
+                 dataset()->prefetch_input_elements_);
+        }
+        return Status::OK();
+      }
+
+      // Produces elements into the worker's output buffers.
+      void WorkerThread(const std::shared_ptr<IteratorContext>& ctx,
+                        const int64 thread_index) {
+        // Notes on checkpointing thread local state, i.e., `WorkerThreadState`:
+        //
+        // 1. Any local state that may need to be checkpointed should be kept
+        //    in `worker_thread_states_[thread_index]`.
+        // 2. `WorkerThreadState` should contain state that is needed only for
+        //    checkpointing, i.e., if we were to remove checkpointing support,
+        //    we could keep that state as local variables in this thread.
+        // 3. This thread should only read/write state at `thread_index`
+        //    and should not access other thread states.
+        // 4. When restoring from checkpoint, threads are started only after
+        //    the restore is complete.
+        // 5. Once restored from a checkpoint, the local state is edited only
+        //    by this thread. 3 & 4 allow making assumptions like temporarily
+        //    caching local state in this thread and using it outside a lock
+        //    e.g. `make_new_iterator`.
+        // 6. `ckpt_mu_` should be wisely used to create *consistent*
+        //    checkpoint markers.
+
+        // std::function arguments are copy-constructable, so we pass raw
+        // pointers, and then immediately wrap them to ensure correct ownership.
+        RecordStart(ctx.get());
+        auto cleanup = gtl::MakeCleanup([this, thread_index, ctx] {
+          mutex_lock l(mu_);
+          workers_[thread_index].cond_var.notify_all();
+          RecordStop(ctx.get());
+        });
+        bool make_new_iterator;
+        {
+          tf_shared_lock l(ckpt_mu_);
+          // Decide whether a new iterator should be built.
+          // 1. If there is an existing iterator, we use it.
+          // 2. If there was an error in iterator creation that could not be
+          //    notified to the client we attempt to send that to the client
+          //    first.
+          make_new_iterator =
+              worker_thread_states_[thread_index].iterator == nullptr &&
+              worker_thread_states_[thread_index].iterator_creation_status.ok();
+        }
+        // Even though `make_new_iterator` has cached values from
+        // `worker_thread_states_[thread_index]` which is guarded by ckpt_mu_,
+        // it is safe to *read* `make_new_iterator`outside of a lock without
+        // worrying about concurrent changes to values in
+        // `worker_thread_states_[thread_index]`. See comment at the start of
+        // this function for details.
+        while (true) {
+          // Whether creation of the iterator succeeded.
+          Status iterator_creation_status;
+          // 1. Build a new iterator or use the existing one.
+          if (make_new_iterator) {
+            // 1a. Get new input tensors or use the exiting ones.
+            bool read_new_input;
+            {
+              tf_shared_lock l(ckpt_mu_);
+              // worker_thread_states_[thread_index].input will be non-empty
+              // if checkpointing happened at CHECKPOINT_MARKER_A.
+              read_new_input =
+                  worker_thread_states_[thread_index].input.empty();
+            }
+
+            if (read_new_input) {
+              mutex_lock l(mu_);
+              while (!cancelled_ && !workers_[thread_index].is_producing) {
+                RecordStop(ctx.get());
+                workers_[thread_index].cond_var.wait(l);
+                RecordStart(ctx.get());
+              }
+              if (cancelled_) return;
+              // Copy the input tensors so that we do not need to block on `mu_`
+              // when building the iterator.
+              // We keep a copy of the input tensors in
+              // `WorkerThreadState.input` till the iterator is in use. This is
+              // used in `RestoreInternal` to re-build the iterator.
+              // TODO(b/78046638): Explore ways to avoid tracking the input
+              // tensors.
+              tf_shared_lock ckpt_l(ckpt_mu_);
+              worker_thread_states_[thread_index].input.swap(
+                  workers_[thread_index].input);
+              // CHECKPOINT_MARKER_A
+              // We have the input tensors but have not built the iterator yet.
+            }
+
+            // 1b. Run the user defined function to produce a new iterator.
+            {
+              tf_shared_lock l(ckpt_mu_);
+              worker_thread_states_[thread_index].iterator_creation_status =
+                  MakeIteratorFromInputElement(
+                      ctx.get(), worker_thread_states_[thread_index].input,
+                      thread_index, *instantiated_captured_func_, prefix(),
+                      &worker_thread_states_[thread_index].iterator);
+              iterator_creation_status =
+                  worker_thread_states_[thread_index].iterator_creation_status;
+              if (!iterator_creation_status.ok()) {
+                worker_thread_states_[thread_index].input.clear();
+              }
+              // CHECKPOINT_MARKER_B
+              // Either an iterator has been successfully built and placed in
+              // `worker_thread_states_[thread_index].iterator` or it failed and
+              // a non-OK status has been put in
+              // `worker_thread_states_[thread_index].iterator_creation_status`.
+            }
+          } else {
+            tf_shared_lock l(ckpt_mu_);
+            iterator_creation_status =
+                worker_thread_states_[thread_index].iterator_creation_status;
+            // Mark that we have used up the restored iterator.
+            make_new_iterator = true;
+          }
+          // 2. Start producing elements or send error state to client if
+          //    iterator creation failed.
+          if (!iterator_creation_status.ok()) {
+            mutex_lock l(mu_);
+            // Wait for space in the prefetch queue.
+            while (!cancelled_ && workers_[thread_index].outputs.size() ==
+                                      dataset()->buffer_output_elements_) {
+              RecordStop(ctx.get());
+              workers_[thread_index].cond_var.wait(l);
+              RecordStart(ctx.get());
+            }
+            if (cancelled_) return;
+            tf_shared_lock ckpt_l(ckpt_mu_);
+            workers_[thread_index].outputs.emplace_back(
+                iterator_creation_status);
+            workers_[thread_index].is_producing = false;
+            worker_thread_states_[thread_index].iterator_creation_status =
+                Status::OK();
+            // CHECKPOINT_MARKER_C
+            // Non-OK iterator creation status has been notified to the
+            // client.
+            workers_[thread_index].cond_var.notify_one();
+          } else {
+            bool end_of_sequence = false;
+            while (!end_of_sequence) {
+              // 3.a Produce an element!
+              {
+                tf_shared_lock ckpt_l(ckpt_mu_);
+                if (worker_thread_states_[thread_index]
+                        .output_elem.status.ok() &&
+                    worker_thread_states_[thread_index]
+                        .output_elem.output.empty() &&
+                    !worker_thread_states_[thread_index].end_of_sequence) {
+                  worker_thread_states_[thread_index].output_elem.status =
+                      worker_thread_states_[thread_index].iterator->GetNext(
+                          ctx.get(),
+                          &worker_thread_states_[thread_index]
+                               .output_elem.output,
+                          &worker_thread_states_[thread_index].end_of_sequence);
+                  end_of_sequence =
+                      worker_thread_states_[thread_index].end_of_sequence;
+                } else {
+                  end_of_sequence =
+                      worker_thread_states_[thread_index].end_of_sequence;
+                }
+                // CHECKPOINT_MARKER_D
+                // An element has been read or an error or end_of_sequence has
+                // been received from the input iterator and is waiting to be
+                // sent to client.
+              }
+
+              // 3.b Make it available to the client.
+              {
+                mutex_lock l(mu_);
+
+                // Wait for space in the prefetch queue.
+                while (!cancelled_ && workers_[thread_index].outputs.size() ==
+                                          dataset()->buffer_output_elements_) {
+                  RecordStop(ctx.get());
+                  workers_[thread_index].cond_var.wait(l);
+                  RecordStart(ctx.get());
+                }
+                if (cancelled_) return;
+
+                tf_shared_lock ckpt_l(ckpt_mu_);
+                workers_[thread_index].is_producing = !end_of_sequence;
+
+                // Output the element.
+
+                // Move the temporary state in WorkerThreadState to WorkerState
+                // and mark it as used.
+                if (end_of_sequence) {
+                  worker_thread_states_[thread_index].iterator.reset();
+                  worker_thread_states_[thread_index].input.clear();
+                  worker_thread_states_[thread_index].end_of_sequence = false;
+                } else {
+                  workers_[thread_index].outputs.emplace_back(
+                      worker_thread_states_[thread_index].output_elem.status);
+                  workers_[thread_index].outputs.back().output.swap(
+                      worker_thread_states_[thread_index].output_elem.output);
+                }
+                worker_thread_states_[thread_index].output_elem.status =
+                    Status::OK();
+                if (dataset()->sloppy_) {
+                  sloppy_cond_var_.notify_one();
+                } else {
+                  workers_[thread_index].cond_var.notify_one();
+                }
+                // CHECKPOINT_MARKER_E
+                // Output element or iterator status has been sent to the
+                // client.
+              }
+            }
+          }
+        }
+      }
+
+      Status WriteWorkerStateLocked(IteratorStateWriter* writer, int index)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        string prefix = strings::StrCat("worker_", index);
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name(strings::StrCat(prefix, "_input_size")),
+            workers_[index].input.size()));
+        for (int i = 0; i < workers_[index].input.size(); ++i) {
+          TF_RETURN_IF_ERROR(writer->WriteTensor(
+              full_name(strings::StrCat(prefix, "_input_", i)),
+              workers_[index].input[i]));
+        }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name(strings::StrCat(prefix, "_outputs_size")),
+            workers_[index].outputs.size()));
+        for (int i = 0; i < workers_[index].outputs.size(); ++i) {
+          TF_RETURN_IF_ERROR(WriteOutputElemLocked(
+              writer, workers_[index].outputs[i],
+              full_name(strings::StrCat(prefix, "_outputs_", i))));
+        }
+        if (workers_[index].is_producing) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(prefix, "_is_producing")), ""));
+        }
+        return Status::OK();
+      }
+
+      Status ReadWorkerStateLocked(IteratorStateReader* reader, int index,
+                                   IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        string worker_prefix = strings::StrCat("worker_", index);
+        // Restore inputs.
+        int64 input_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(worker_prefix, "_input_size")),
+            &input_size));
+        workers_[index].input.reserve(input_size);
+        for (int i = 0; i < input_size; ++i) {
+          workers_[index].input.emplace_back();
+          TF_RETURN_IF_ERROR(reader->ReadTensor(
+              full_name(strings::StrCat(worker_prefix, "_input_", i)),
+              &workers_[index].input.back()));
+        }
+        int64 outputs_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(worker_prefix, "_outputs_size")),
+            &outputs_size));
+        for (int i = 0; i < outputs_size; ++i) {
+          workers_[index].outputs.emplace_back(Status::OK());
+          TF_RETURN_IF_ERROR(ReadOutputElemLocked(
+              reader, &workers_[index].outputs.back(),
+              full_name(strings::StrCat(worker_prefix, "_outputs_", i))));
+        }
+        if (reader->Contains(
+                full_name(strings::StrCat(worker_prefix, "_is_producing")))) {
+          workers_[index].is_producing = true;
+        } else {
+          workers_[index].is_producing = false;
+        }
+        return Status::OK();
+      }
+
+      Status WriteWorkerThreadStateLocked(IteratorStateWriter* writer,
+                                          int index)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        string prefix = strings::StrCat("worker_thread_", index);
+        if (worker_thread_states_[index].iterator != nullptr) {
+          TF_RETURN_IF_ERROR(
+              SaveInput(writer, worker_thread_states_[index].iterator));
+        } else {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(prefix, "_iterator_exhausted")), ""));
+        }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name(strings::StrCat(prefix, "_input_size")),
+            worker_thread_states_[index].input.size()));
+        for (int i = 0; i < worker_thread_states_[index].input.size(); ++i) {
+          TF_RETURN_IF_ERROR(writer->WriteTensor(
+              full_name(strings::StrCat(prefix, "_input_", i)),
+              worker_thread_states_[index].input[i]));
+        }
+        TF_RETURN_IF_ERROR(WriteStatusLocked(
+            writer, strings::StrCat(prefix, "_iterator_creation_status"),
+            worker_thread_states_[index].iterator_creation_status));
+        TF_RETURN_IF_ERROR(WriteOutputElemLocked(
+            writer, worker_thread_states_[index].output_elem,
+            full_name(strings::StrCat(prefix, "_output"))));
+        if (worker_thread_states_[index].end_of_sequence) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(prefix, "_end_of_sequence")), ""));
+        }
+        return Status::OK();
+      }
+
+      Status ReadWorkerThreadStateLocked(IteratorStateReader* reader, int index,
+                                         IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        string worker_prefix = strings::StrCat("worker_thread_", index);
+        // Restore inputs.
+        int64 input_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(worker_prefix, "_input_size")),
+            &input_size));
+        worker_thread_states_[index].input.reserve(input_size);
+        for (int i = 0; i < input_size; ++i) {
+          worker_thread_states_[index].input.emplace_back();
+          TF_RETURN_IF_ERROR(reader->ReadTensor(
+              full_name(strings::StrCat(worker_prefix, "_input_", i)),
+              &worker_thread_states_[index].input.back()));
+        }
+        // Restore iterator.
+        if (reader->Contains(full_name(
+                strings::StrCat(worker_prefix, "_iterator_exhausted")))) {
+          worker_thread_states_[index].iterator.reset();
+        } else {
+          std::unique_ptr<IteratorBase> iterator;
+          Status s = MakeIteratorFromInputElement(
+              ctx, worker_thread_states_[index].input, index,
+              *instantiated_captured_func_, prefix(), &iterator);
+          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, iterator));
+          worker_thread_states_[index].iterator.swap(iterator);
+        }
+        TF_RETURN_IF_ERROR(ReadStatusLocked(
+            reader, strings::StrCat(worker_prefix, "_iterator_creation_status"),
+            &worker_thread_states_[index].iterator_creation_status));
+        TF_RETURN_IF_ERROR(ReadOutputElemLocked(
+            reader, &worker_thread_states_[index].output_elem,
+            full_name(strings::StrCat(worker_prefix, "_output"))));
+        if (reader->Contains(full_name(
+                strings::StrCat(worker_prefix, "_end_of_sequence")))) {
+          worker_thread_states_[index].end_of_sequence = true;
+        } else {
+          worker_thread_states_[index].end_of_sequence = false;
+        }
+        return Status::OK();
+      }
+
+      Status WriteOutputElemLocked(IteratorStateWriter* writer,
+                                   const OutputElem& output_elem,
+                                   const string& prefix)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        TF_RETURN_IF_ERROR(WriteStatusLocked(
+            writer, strings::StrCat(prefix, "_status"), output_elem.status));
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(strings::StrCat(prefix, "_output_size"),
+                                output_elem.output.size()));
+        for (int i = 0; i < output_elem.output.size(); ++i) {
+          TF_RETURN_IF_ERROR(writer->WriteTensor(
+              strings::StrCat(prefix, "_output_", i), output_elem.output[i]));
+        }
+        return Status::OK();
+      }
+
+      Status ReadOutputElemLocked(IteratorStateReader* reader,
+                                  OutputElem* output_elem, const string& prefix)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        TF_RETURN_IF_ERROR(ReadStatusLocked(
+            reader, strings::StrCat(prefix, "_status"), &output_elem->status));
+        int64 output_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            strings::StrCat(prefix, "_output_size"), &output_size));
+        output_elem->output.reserve(output_size);
+        for (int i = 0; i < output_size; ++i) {
+          output_elem->output.emplace_back();
+          TF_RETURN_IF_ERROR(
+              reader->ReadTensor(strings::StrCat(prefix, "_output_", i),
+                                 &output_elem->output.back()));
+        }
+        return Status::OK();
+      }
+
+      Status WriteStatusLocked(IteratorStateWriter* writer,
+                               const string& prefix, const Status& status)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name(strings::StrCat(prefix, "_code")),
+                                static_cast<int64>(status.code())));
+        if (!status.ok()) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name(strings::StrCat(prefix, "_msg")),
+                                  status.error_message()));
+        }
+        return Status::OK();
+      }
+
+      Status ReadStatusLocked(IteratorStateReader* reader, const string& prefix,
+                              Status* status)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
+        int64 code_int;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(prefix, "_code")), &code_int));
+        error::Code code = static_cast<error::Code>(code_int);
+
+        if (code != error::Code::OK) {
+          string error_message;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              full_name(strings::StrCat(prefix, "_msg")), &error_message));
+          *status = Status(code, error_message);
+        } else {
+          *status = Status::OK();
+        }
+        return Status::OK();
+      }
+
+      // Mutex & condition variable to guard mutable iterator internals and
+      // coordinate among worker threads and client thread[s].
+      mutex mu_ ACQUIRED_BEFORE(ckpt_mu_);
+      // The main thread waits on this condition variable if running in sloppy
+      // mode and no values are available.
+      condition_variable sloppy_cond_var_;
+      // Mutex used to wait for a consistent state while checkpointing.
+      // Only Save and Restore require an exclusive lock on this mutex. In
+      // other scenarios we just acquire a shared lock so the pipeline's
+      // performance should not be affected in the absence of checkpointing.
+      // A thread must not wait on any condition variable while holding
+      // `ckpt_mu_` in either shared or exclusive modes.
+      mutex ckpt_mu_;
+
+      // The iterator producing elements which are converted to datasets by
+      // the dataset()->captured_func_ then interleaved together.
+      // input_impl_ is reset when we have exhausted its input.
+      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+
+      // The WorkerState structs the worker threads operate on.
+      // workers_ elements are in at most one of interleave_ and staging_.
+      std::vector<WorkerState> workers_ GUARDED_BY(mu_);
+
+      // Stores the temporary state of WorkerThreads which is not stored in
+      // WorkerState. This is used for checkpointing purposes only.
+      std::vector<WorkerThreadState> worker_thread_states_ GUARDED_BY(ckpt_mu_);
+
+      // Indices in `workers_` of iterators to interleave.
+      std::vector<int64> interleave_indices_ GUARDED_BY(mu_);
+      // Indices in `workers_` of prefetched iterators.
+      std::deque<int64> staging_indices_ GUARDED_BY(mu_);
+
+      // The index into output_elements_ for next element to produce.
+      size_t next_index_ GUARDED_BY(mu_) = 0;
+      // The number of items produced so far within the block
+      size_t block_count_ GUARDED_BY(mu_) = 0;
+      // Flag to instruct the worker threads to exit.
+      bool cancelled_ GUARDED_BY(mu_) = false;
+      // The worker threads. This must be last to ensure the
+      // threads have exited before any other members are deallocated.
+      // TODO(b/65178177): Avoid allocating additional threads.
+      std::vector<std::unique_ptr<Thread>> worker_threads_ GUARDED_BY(mu_);
+    };
+
+    const DatasetBase* const input_;
+    const NameAttrList interleave_func_;
+    const std::unique_ptr<CapturedFunction> captured_func_;
+    const int64 cycle_length_;
+    const int64 block_length_;
+    const bool sloppy_;
+    const int64 buffer_output_elements_;
+    const int64 prefetch_input_elements_;
+    const DataTypeVector output_types_;
+    const std::vector<PartialTensorShape> output_shapes_;
+  };
+
+  DataTypeVector output_types_;
+  std::vector<PartialTensorShape> output_shapes_;
+  NameAttrList interleave_func_;
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalParallelInterleaveDataset").Device(DEVICE_CPU),
+    ParallelInterleaveDatasetOp);
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
similarity index 99%
rename from tensorflow/core/kernels/data/parse_example_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
index c4ab4b28df..4cec555ac1 100644
--- a/tensorflow/core/kernels/data/parse_example_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
@@ -386,8 +386,9 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
   std::vector<std::size_t> elements_per_stride_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("ParseExampleDataset").Device(DEVICE_CPU),
-                        ParseExampleDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalParseExampleDataset").Device(DEVICE_CPU),
+    ParseExampleDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/stats_aggregator_dataset_op.cc b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
similarity index 98%
rename from tensorflow/core/kernels/data/stats_aggregator_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
index a21b3fc16b..80e10d1ea7 100644
--- a/tensorflow/core/kernels/data/stats_aggregator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
@@ -203,8 +203,9 @@ class SetStatsAggregatorDatasetOp : public UnaryDatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("SetStatsAggregatorDataset").Device(DEVICE_CPU),
-                        SetStatsAggregatorDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalSetStatsAggregatorDataset").Device(DEVICE_CPU),
+    SetStatsAggregatorDatasetOp);
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/slide_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
similarity index 97%
rename from tensorflow/core/kernels/data/slide_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
index e67c5272b6..a6c02471e3 100644
--- a/tensorflow/core/kernels/data/slide_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
@@ -29,9 +29,9 @@ namespace {
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
-class SlideDatasetOp : public UnaryDatasetOpKernel {
+class SlidingWindowDatasetOp : public UnaryDatasetOpKernel {
  public:
-  explicit SlideDatasetOp(OpKernelConstruction* ctx)
+  explicit SlidingWindowDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {}
 
   void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
@@ -99,7 +99,7 @@ class SlideDatasetOp : public UnaryDatasetOpKernel {
     }
 
     string DebugString() const override {
-      return strings::StrCat("SlideDatasetOp(", window_size_, ", ",
+      return strings::StrCat("SlidingWindowDatasetOp(", window_size_, ", ",
                              window_shift_, ", ", window_stride_, ")::Dataset");
     }
 
@@ -295,8 +295,9 @@ class SlideDatasetOp : public UnaryDatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("SlideDataset").Device(DEVICE_CPU),
-                        SlideDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalSlidingWindowDataset").Device(DEVICE_CPU),
+    SlidingWindowDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/sql/BUILD b/tensorflow/core/kernels/data/experimental/sql/BUILD
similarity index 100%
rename from tensorflow/core/kernels/data/sql/BUILD
rename to tensorflow/core/kernels/data/experimental/sql/BUILD
diff --git a/tensorflow/core/kernels/data/sql/driver_manager.cc b/tensorflow/core/kernels/data/experimental/sql/driver_manager.cc
similarity index 88%
rename from tensorflow/core/kernels/data/sql/driver_manager.cc
rename to tensorflow/core/kernels/data/experimental/sql/driver_manager.cc
index 783d1e6cb2..58174f69a4 100644
--- a/tensorflow/core/kernels/data/sql/driver_manager.cc
+++ b/tensorflow/core/kernels/data/experimental/sql/driver_manager.cc
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/core/kernels/data/sql/driver_manager.h"
-#include "tensorflow/core/kernels/data/sql/sqlite_query_connection.h"
+#include "tensorflow/core/kernels/data/experimental/sql/driver_manager.h"
+#include "tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.h"
 
 namespace tensorflow {
 namespace data {
diff --git a/tensorflow/core/kernels/data/sql/driver_manager.h b/tensorflow/core/kernels/data/experimental/sql/driver_manager.h
similarity index 81%
rename from tensorflow/core/kernels/data/sql/driver_manager.h
rename to tensorflow/core/kernels/data/experimental/sql/driver_manager.h
index c5428f396b..6afadf91a4 100644
--- a/tensorflow/core/kernels/data/sql/driver_manager.h
+++ b/tensorflow/core/kernels/data/experimental/sql/driver_manager.h
@@ -12,10 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
-#define TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
+#ifndef TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_DRIVER_MANAGER_H_
+#define TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_DRIVER_MANAGER_H_
 
-#include "tensorflow/core/kernels/data/sql/query_connection.h"
+#include "tensorflow/core/kernels/data/experimental/sql/query_connection.h"
 
 namespace tensorflow {
 namespace data {
@@ -38,4 +38,4 @@ class DriverManager {
 }  // namespace data
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
+#endif  // TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_DRIVER_MANAGER_H_
diff --git a/tensorflow/core/kernels/data/sql/query_connection.h b/tensorflow/core/kernels/data/experimental/sql/query_connection.h
similarity index 92%
rename from tensorflow/core/kernels/data/sql/query_connection.h
rename to tensorflow/core/kernels/data/experimental/sql/query_connection.h
index 2fd229a9bf..10c6643679 100644
--- a/tensorflow/core/kernels/data/sql/query_connection.h
+++ b/tensorflow/core/kernels/data/experimental/sql/query_connection.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
-#define TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
+#ifndef TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_QUERY_CONNECTION_H_
+#define TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_QUERY_CONNECTION_H_
 
 #include "tensorflow/core/framework/tensor.h"
 
@@ -67,4 +67,4 @@ class QueryConnection {
 }  // namespace data
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
+#endif  // TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_QUERY_CONNECTION_H_
diff --git a/tensorflow/core/kernels/data/sql/sqlite_query_connection.cc b/tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.cc
similarity index 97%
rename from tensorflow/core/kernels/data/sql/sqlite_query_connection.cc
rename to tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.cc
index 1d374898dc..cadceee8f5 100644
--- a/tensorflow/core/kernels/data/sql/sqlite_query_connection.cc
+++ b/tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/core/kernels/data/sql/sqlite_query_connection.h"
+#include "tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.h"
 
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -106,7 +106,7 @@ void SqliteQueryConnection::FillTensorWithResultSetEntry(
       break;
     // Error preemptively thrown by SqlDatasetOp::MakeDataset in this case.
     default:
-      LOG(FATAL)
+      LOG(ERROR)
           << "Use of unsupported TensorFlow data type by 'SqlQueryConnection': "
           << DataTypeString(data_type) << ".";
   }
diff --git a/tensorflow/core/kernels/data/sql/sqlite_query_connection.h b/tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.h
similarity index 84%
rename from tensorflow/core/kernels/data/sql/sqlite_query_connection.h
rename to tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.h
index 175492c49d..61df29065e 100644
--- a/tensorflow/core/kernels/data/sql/sqlite_query_connection.h
+++ b/tensorflow/core/kernels/data/experimental/sql/sqlite_query_connection.h
@@ -12,12 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
-#define TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
+#ifndef TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_SQLITE_QUERY_CONNECTION_H_
+#define TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_SQLITE_QUERY_CONNECTION_H_
 
 #include <memory>
 
-#include "tensorflow/core/kernels/data/sql/query_connection.h"
+#include "tensorflow/core/kernels/data/experimental/sql/query_connection.h"
 #include "tensorflow/core/lib/db/sqlite.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -53,4 +53,4 @@ class SqliteQueryConnection : public QueryConnection {
 }  // namespace data
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
+#endif  // TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_SQL_SQLITE_QUERY_CONNECTION_H_
diff --git a/tensorflow/core/kernels/data/sql_dataset_ops.cc b/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
similarity index 96%
rename from tensorflow/core/kernels/data/sql_dataset_ops.cc
rename to tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
index f01ecf84af..c16d8ed02c 100644
--- a/tensorflow/core/kernels/data/sql_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
@@ -17,8 +17,8 @@ limitations under the License.
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/data/sql/driver_manager.h"
-#include "tensorflow/core/kernels/data/sql/query_connection.h"
+#include "tensorflow/core/kernels/data/experimental/sql/driver_manager.h"
+#include "tensorflow/core/kernels/data/experimental/sql/query_connection.h"
 #include "tensorflow/core/lib/io/inputbuffer.h"
 #include "tensorflow/core/lib/io/record_reader.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -214,7 +214,8 @@ class SqlDatasetOp : public DatasetOpKernel {
   std::vector<PartialTensorShape> output_shapes_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("SqlDataset").Device(DEVICE_CPU), SqlDatasetOp);
+REGISTER_KERNEL_BUILDER(Name("ExperimentalSqlDataset").Device(DEVICE_CPU),
+                        SqlDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/stats_aggregator_ops.cc b/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
similarity index 95%
rename from tensorflow/core/kernels/data/stats_aggregator_ops.cc
rename to tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
index 2d51467616..894465e181 100644
--- a/tensorflow/core/kernels/data/stats_aggregator_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
@@ -141,10 +141,12 @@ class StatsAggregatorSummaryOp : public OpKernel {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("StatsAggregatorHandle").Device(DEVICE_CPU),
-                        StatsAggregatorHandleOp);
-REGISTER_KERNEL_BUILDER(Name("StatsAggregatorSummary").Device(DEVICE_CPU),
-                        StatsAggregatorSummaryOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalStatsAggregatorHandle").Device(DEVICE_CPU),
+    StatsAggregatorHandleOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalStatsAggregatorSummary").Device(DEVICE_CPU),
+    StatsAggregatorSummaryOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/stats_dataset_ops.cc b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
similarity index 97%
rename from tensorflow/core/kernels/data/stats_dataset_ops.cc
rename to tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
index da0039773c..f0a0d23eb1 100644
--- a/tensorflow/core/kernels/data/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
@@ -255,10 +255,12 @@ class BytesProducedStatsDatasetOp : public UnaryDatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("LatencyStatsDataset").Device(DEVICE_CPU),
-                        LatencyStatsDatasetOp);
-REGISTER_KERNEL_BUILDER(Name("BytesProducedStatsDataset").Device(DEVICE_CPU),
-                        BytesProducedStatsDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalLatencyStatsDataset").Device(DEVICE_CPU),
+    LatencyStatsDatasetOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalBytesProducedStatsDataset").Device(DEVICE_CPU),
+    BytesProducedStatsDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/writer_ops.cc b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
similarity index 97%
rename from tensorflow/core/kernels/data/writer_ops.cc
rename to tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
index c00089d06f..7728baf150 100644
--- a/tensorflow/core/kernels/data/writer_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
@@ -101,8 +101,8 @@ class ToTFRecordOp : public AsyncOpKernel {
   BackgroundWorker background_worker_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("DatasetToTFRecord").Device(DEVICE_CPU),
-                        ToTFRecordOp);
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalDatasetToTFRecord").Device(DEVICE_CPU), ToTFRecordOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/unbatch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
similarity index 99%
rename from tensorflow/core/kernels/data/unbatch_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
index af7f676370..2626ec3ed7 100644
--- a/tensorflow/core/kernels/data/unbatch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
@@ -221,7 +221,7 @@ class UnbatchDatasetOp : public UnaryDatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("UnbatchDataset").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("ExperimentalUnbatchDataset").Device(DEVICE_CPU),
                         UnbatchDatasetOp);
 
 }  // namespace
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index b5a1679082..5d33d1e795 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
-#include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
@@ -36,1050 +35,6 @@ namespace {
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
-class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
- public:
-  explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx)
-      : UnaryDatasetOpKernel(ctx) {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &interleave_func_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
-  }
-
-  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                   DatasetBase** output) override {
-    int64 cycle_length = 0;
-    OP_REQUIRES_OK(ctx,
-                   ParseScalarArgument(ctx, "cycle_length", &cycle_length));
-    OP_REQUIRES(ctx, cycle_length > 0,
-                errors::InvalidArgument("`cycle_length` must be > 0"));
-
-    int64 block_length = 0;
-    OP_REQUIRES_OK(ctx,
-                   ParseScalarArgument(ctx, "block_length", &block_length));
-    OP_REQUIRES(ctx, block_length > 0,
-                errors::InvalidArgument("`block_length` must be > 0"));
-
-    bool sloppy = false;
-    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "sloppy", &sloppy));
-
-    int64 buffer_output_elements = 0;
-    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "buffer_output_elements",
-                                            &buffer_output_elements));
-    OP_REQUIRES(
-        ctx, buffer_output_elements > 0,
-        errors::InvalidArgument("`buffer_output_elements` must be > 0"));
-
-    int64 prefetch_input_elements = 0;
-    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "prefetch_input_elements",
-                                            &prefetch_input_elements));
-    OP_REQUIRES(
-        ctx, prefetch_input_elements >= 0,
-        errors::InvalidArgument("`prefetch_input_elements` must be >= 0"));
-
-    std::unique_ptr<CapturedFunction> captured_func;
-    OP_REQUIRES_OK(
-        ctx, CapturedFunction::Create(interleave_func_, ctx, "other_arguments",
-                                      &captured_func));
-
-    *output =
-        new Dataset(ctx, input, interleave_func_, std::move(captured_func),
-                    cycle_length, block_length, sloppy, buffer_output_elements,
-                    prefetch_input_elements, output_types_, output_shapes_);
-  }
-
- private:
-  class Dataset : public DatasetBase {
-   public:
-    Dataset(OpKernelContext* ctx, const DatasetBase* input,
-            const NameAttrList& func,
-            std::unique_ptr<CapturedFunction> captured_func, int64 cycle_length,
-            int64 block_length, bool sloppy, int64 buffer_output_elements,
-            int64 prefetch_input_elements, const DataTypeVector& output_types,
-            const std::vector<PartialTensorShape>& output_shapes)
-        : DatasetBase(DatasetContext(ctx)),
-          input_(input),
-          interleave_func_(func),
-          captured_func_(std::move(captured_func)),
-          cycle_length_(cycle_length),
-          block_length_(block_length),
-          sloppy_(sloppy),
-          buffer_output_elements_(buffer_output_elements),
-          prefetch_input_elements_(prefetch_input_elements),
-          output_types_(output_types),
-          output_shapes_(output_shapes) {
-      input_->Ref();
-    }
-
-    ~Dataset() override { input_->Unref(); }
-
-    std::unique_ptr<IteratorBase> MakeIteratorInternal(
-        const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::ParallelInterleave")}));
-    }
-
-    const DataTypeVector& output_dtypes() const override {
-      return output_types_;
-    }
-
-    const std::vector<PartialTensorShape>& output_shapes() const override {
-      return output_shapes_;
-    }
-
-    string DebugString() const override {
-      return "ParallelInterleaveDatasetOp::Dataset";
-    }
-
-   protected:
-    Status AsGraphDefInternal(SerializationContext* ctx,
-                              DatasetGraphDefBuilder* b,
-                              Node** output) const override {
-      TF_RETURN_IF_ERROR(b->AddFunction(ctx, interleave_func_.name()));
-      Node* input_node;
-      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_node));
-      Node* cycle_length_node;
-      TF_RETURN_IF_ERROR(b->AddScalar(cycle_length_, &cycle_length_node));
-      Node* block_length_node;
-      TF_RETURN_IF_ERROR(b->AddScalar(block_length_, &block_length_node));
-      Node* sloppy_node;
-      TF_RETURN_IF_ERROR(b->AddScalar(sloppy_, &sloppy_node));
-      Node* buffer_output_elements_node;
-      TF_RETURN_IF_ERROR(
-          b->AddScalar(buffer_output_elements_, &buffer_output_elements_node));
-      Node* prefetch_input_elements_node;
-      TF_RETURN_IF_ERROR(b->AddScalar(prefetch_input_elements_,
-                                      &prefetch_input_elements_node));
-      DataTypeVector other_arguments_types;
-      other_arguments_types.reserve(captured_func_->captured_inputs().size());
-      std::vector<Node*> other_arguments;
-      other_arguments.reserve(captured_func_->captured_inputs().size());
-      for (const Tensor& t : captured_func_->captured_inputs()) {
-        Node* node;
-        TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
-        other_arguments.emplace_back(node);
-        other_arguments_types.emplace_back(t.dtype());
-      }
-      AttrValue f;
-      b->BuildAttrValue(interleave_func_, &f);
-      AttrValue other_arguments_types_attr;
-      b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
-
-      TF_RETURN_IF_ERROR(b->AddDataset(
-          this,
-          {{0, input_node},
-           {2, cycle_length_node},
-           {3, block_length_node},
-           {4, sloppy_node},
-           {5, buffer_output_elements_node},
-           {6, prefetch_input_elements_node}},
-          {{1, other_arguments}},
-          {{"f", f}, {"Targuments", other_arguments_types_attr}}, output));
-      return Status::OK();
-    }
-
-   private:
-    int64 num_threads() const {
-      return cycle_length_ + prefetch_input_elements_;
-    }
-
-    // Parallel interleave's implementation is designed around a few principles:
-    //  1. Thread creation is relatively expensive. (Not reusing
-    //     threads causes a number of indirect costs such as poorer tcmalloc
-    //     performance due to thread-local caches, etc.) We allocate a fixed
-    //     number of threads at the start and never change. This is why we've
-    //     fused functionality that is theoretically orthogonal (i.e.
-    //     .prefetch()) into the implementation.
-    //  2. Drop-in replacement for standard interleave. The goal will be to
-    //     auto-opt people into an optimized implementation without any work
-    //     on the customer's part. We thus go through great pains to maintain
-    //     identical iteration orders, full determinism (disabled only via a
-    //     flag, etc.)
-    //  3. Performance across a variety of environments and I/O envelopes.
-    //
-    // The actual implementation centers around a collection of worker threads
-    // and their corresponding worker state (tracked in the `workers_` vector).
-    // Worker threads repeatedly receive a vector of Tensors that are used as
-    // input to the flat-map function (`captured_func_`). The output of this
-    // function must be a dataset. The worker thread then repeatedly calls
-    // `GetNext()`, maintaining a buffer of elements to minimize the likelihood
-    // that a caller will block waiting for an element to be produced.
-    //
-    // Pointers to these worker states are kept in 2 disjoint data structures:
-    //  1. `interleave_indices_` is a vector containing indices of WorkerStates
-    //     in `workers_` that we are interleaving. Worker threads backing these
-    //     WorkerStates should be regularly producing values.
-    //  2. `staging_indices_` is a deque containing indices of WorkerStates in
-    //     `workers_` that we will move to `interleave_indices_` when an
-    //     iterator in `interleave_indices_` is exhausted.
-    //
-    // The client calls `GetNext[Internal]()` to retrieve an output element. The
-    // internal implementation updates the state of `interleave_indices_` and
-    // `staging_indices_` as output iterators (run by the worker threads) are
-    // exhausted.
-    //
-    // `input_impl_` is the input iterator that generates arguments for the
-    // flat-map function (`captured_func_`). It is set to an iterator at
-    // Iterator construction, and is fixed until we consume all input elements.
-    // Once it is exhausted, we reset the unique_ptr to eagerly deallocate
-    // memory.
-    //
-    // A few invariants are maintained:
-    //  1. No element in interleave_indices_ should be a -1 unless
-    //     `staging_indices_` is empty and `input_impl_` is empty.
-    //  2. Every `worker_` element is pointed to by at most one element of the
-    //     union of `interleave_indices_` and `staging_indices_`.
-    //  3. Unless `input_impl_` is empty, every `worker_` must be pointed to by
-    //     an element in `interleave_indices_` or `staging_indices_`.
-    class Iterator : public DatasetIterator<Dataset> {
-     public:
-      explicit Iterator(const Params& params)
-          : DatasetIterator<Dataset>(params),
-            workers_(dataset()->num_threads()),
-            worker_thread_states_(dataset()->num_threads()) {}
-
-      ~Iterator() override {
-        mutex_lock l(mu_);
-        cancelled_ = true;
-        // Notify all workers in case they are blocked.
-        for (auto& worker : workers_) {
-          worker.cond_var.notify_all();
-        }
-      }
-
-      Status Initialize(IteratorContext* ctx) override {
-        TF_RETURN_IF_ERROR(
-            dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
-        return dataset()->captured_func_->Instantiate(
-            ctx, &instantiated_captured_func_);
-      }
-
-      // It is implemented so that it matches the deterministic interleave
-      // unless getting the next element would block and we are allowed to be
-      // sloppy.
-      Status GetNextInternal(IteratorContext* ctx,
-                             std::vector<Tensor>* out_tensors,
-                             bool* end_of_sequence) override {
-        mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(EnsureWorkerThreadsStarted(ctx));
-        while (!cancelled_) {
-          // Wait for an item to become available, blocking if necessary. If we
-          // are allowed to be sloppy, we can skip over input datasets that do
-          // not have an item readily available.
-          bool can_produce_elements = false;
-          bool must_wait_for_input = true;
-          for (int64 i = 0; i < interleave_indices_.size(); ++i) {
-            int64 index = (next_index_ + i) % interleave_indices_.size();
-            int64 current_worker_index = interleave_indices_[index];
-            if (current_worker_index < 0) {
-              continue;  // Empty interleave elements.
-            }
-            WorkerState* current_worker = &workers_[current_worker_index];
-            can_produce_elements |= current_worker->MayHaveElements();
-            if (!current_worker->outputs.empty()) {
-              // We have an element!
-              next_index_ = index;
-              const bool element_acquired_sloppily =
-                  dataset()->sloppy_ && i > 1;
-              if (!element_acquired_sloppily) {
-                // If the element was acquired in the regular (non-sloppy)
-                // order, then advance the current block and cycle pointers to
-                // the next element in the regular order.
-                block_count_++;
-                if (block_count_ == dataset()->block_length_) {
-                  next_index_ = (index + 1) % interleave_indices_.size();
-                  block_count_ = 0;
-                }
-              } else {
-                block_count_ = 0;
-              }
-              *end_of_sequence = false;
-              Status s = current_worker->outputs.front().status;
-              current_worker->outputs.front().output.swap(*out_tensors);
-              current_worker->outputs.pop_front();
-              current_worker->cond_var.notify_one();
-              return s;
-            } else if (current_worker->is_producing && !dataset()->sloppy_) {
-              // current_worker.outputs.empty(), and we must wait for this
-              // iterator.
-              if (next_index_ != index) {
-                // We have advanced to a new iterator; reset block counts.
-                next_index_ = index;
-                block_count_ = 0;
-              }
-              break;
-            } else if (!current_worker->is_producing) {
-              // This iterator has reached end of input.
-              interleave_indices_[index] = -1;
-              if (input_impl_) {
-                // Start prefetching a new iterator.
-                std::vector<Tensor> args;
-                bool end_of_input = false;
-                Status s = input_impl_->GetNext(ctx, &args, &end_of_input);
-                if (end_of_input) {
-                  input_impl_.reset();
-                } else {
-                  current_worker->SetInputs(s, std::move(args));
-                  staging_indices_.emplace_back(current_worker_index);
-                }
-              }
-
-              if (!staging_indices_.empty()) {
-                // Move a worker from `staging_indices_` to
-                // `interleave_indices_`.
-                interleave_indices_[index] = staging_indices_.front();
-                staging_indices_.pop_front();
-
-                next_index_ = (index + 1) % interleave_indices_.size();
-                block_count_ = 0;
-                // Restart the inner [for] loop
-                can_produce_elements = true;
-                must_wait_for_input = false;
-                break;
-              }
-            }
-          }
-
-          if (!can_produce_elements && !input_impl_) {
-            // No potential for future values.
-            *end_of_sequence = true;
-            return Status::OK();
-          }
-
-          if (must_wait_for_input) {
-            // Wait for elements to become available.
-            RecordStop(ctx);
-            if (dataset()->sloppy_) {
-              sloppy_cond_var_.wait(l);
-            } else {
-              workers_[interleave_indices_[next_index_]].cond_var.wait(l);
-            }
-            RecordStart(ctx);
-          }
-        }
-        return errors::Cancelled(
-            "ParallelInterleaveDatasetOp::Dataset::Iterator::GetNext");
-      }
-
-     protected:
-      std::shared_ptr<model::Node> CreateNode(
-          IteratorContext* ctx, model::Node::Args args) const override {
-        return model::MakeAsyncInterleaveManyNode(std::move(args),
-                                                  /*parameters=*/{});
-      }
-
-      Status SaveInternal(IteratorStateWriter* writer) override {
-        // The order of locking is important here to avoid deadlock.
-        mutex_lock l(mu_);
-        mutex_lock ckpt_l(ckpt_mu_);
-        if (input_impl_) {
-          TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
-        } else {
-          TF_RETURN_IF_ERROR(
-              writer->WriteScalar(full_name("input_exhausted"), ""));
-        }
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar(full_name("next_index"), next_index_));
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar(full_name("block_count"), block_count_));
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar(full_name("workers_size"), workers_.size()));
-        for (int i = 0; i < workers_.size(); ++i) {
-          TF_RETURN_IF_ERROR(WriteWorkerStateLocked(writer, i));
-        }
-        for (int i = 0; i < worker_thread_states_.size(); ++i) {
-          TF_RETURN_IF_ERROR(WriteWorkerThreadStateLocked(writer, i));
-        }
-        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("interleave_size"),
-                                               interleave_indices_.size()));
-        for (int i = 0; i < interleave_indices_.size(); ++i) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat("interleave_indices_", i)),
-              interleave_indices_[i]));
-        }
-        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("staging_size"),
-                                               staging_indices_.size()));
-        for (int i = 0; i < staging_indices_.size(); ++i) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat("staging_indices_", i)),
-              staging_indices_[i]));
-        }
-        if (!worker_threads_.empty()) {
-          TF_RETURN_IF_ERROR(
-              writer->WriteScalar(full_name("worker_threads_running"), ""));
-        }
-        return Status::OK();
-      }
-
-      Status RestoreInternal(IteratorContext* ctx,
-                             IteratorStateReader* reader) override {
-        // The order of locking is important here to avoid deadlock.
-        mutex_lock l(mu_);
-        mutex_lock ckpt_l(ckpt_mu_);
-        if (!reader->Contains(full_name("input_exhausted"))) {
-          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
-        } else {
-          input_impl_.reset();
-        }
-        int64 temp;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("next_index"), &temp));
-        next_index_ = size_t(temp);
-        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("block_count"), &temp));
-        block_count_ = size_t(temp);
-
-        // Restore WorkerStates.
-        TF_RETURN_IF_ERROR(
-            reader->ReadScalar(full_name("workers_size"), &temp));
-        if (temp != dataset()->num_threads()) {
-          return errors::Internal("Expected ", dataset()->num_threads(),
-                                  " worker states but found ", temp, ".");
-        }
-        for (size_t i = 0; i < dataset()->num_threads(); ++i) {
-          TF_RETURN_IF_ERROR(ReadWorkerStateLocked(reader, i, ctx));
-        }
-        for (size_t i = 0; i < dataset()->num_threads(); ++i) {
-          TF_RETURN_IF_ERROR(ReadWorkerThreadStateLocked(reader, i, ctx));
-        }
-
-        // Restore `interleave_indices_`.
-        std::set<int64> all_indices;
-        {
-          int64 interleave_size;
-          TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("interleave_size"),
-                                                &interleave_size));
-          interleave_indices_.reserve(interleave_size);
-          for (int64 i = 0; i < interleave_size; ++i) {
-            int64 temp;
-            TF_RETURN_IF_ERROR(reader->ReadScalar(
-                full_name(strings::StrCat("interleave_indices_", i)), &temp));
-            if (temp >= 0 && all_indices.find(temp) != all_indices.end()) {
-              return errors::Internal(
-                  "Duplicate entry for ", temp,
-                  " found when reading interleave and staging indices.");
-            }
-            if (temp >= 0) {
-              all_indices.insert(temp);
-            }
-            interleave_indices_.emplace_back(temp);
-          }
-        }
-
-        // Restore `staging_indices_`.
-        {
-          int64 staging_size;
-          TF_RETURN_IF_ERROR(
-              reader->ReadScalar(full_name("staging_size"), &staging_size));
-          for (int i = 0; i < staging_size; ++i) {
-            int64 temp;
-            TF_RETURN_IF_ERROR(reader->ReadScalar(
-                full_name(strings::StrCat("staging_indices_", i)), &temp));
-            if (all_indices.find(temp) != all_indices.end()) {
-              return errors::Internal(
-                  "Duplicate entry for ", temp,
-                  " found when reading interleave and staging indices.");
-            }
-            if (temp >= 0) {
-              all_indices.insert(temp);
-            }
-            staging_indices_.emplace_back(temp);
-          }
-        }
-
-        // Start Worker threads.
-        if (reader->Contains(full_name("worker_threads_running"))) {
-          worker_threads_.reserve(dataset()->num_threads());
-          for (size_t i = 0; i < dataset()->num_threads(); ++i) {
-            std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-            worker_threads_.emplace_back(ctx->env()->StartThread(
-                {}, strings::StrCat("tf_data_parallel_interleave_worker_", i),
-                [this, new_ctx, i]() { WorkerThread(new_ctx, i); }));
-          }
-        }
-        return Status::OK();
-      }
-
-     private:
-      // OutputElem contains the information from a call to GetNext by an output
-      // iterator.
-      struct OutputElem {
-        // The output iterator sets `status` if getting the output element
-        // fails.
-        Status status;
-        // The buffered data element.
-        std::vector<Tensor> output;
-
-        explicit OutputElem(const Status& s) : status(s) {}
-      };
-
-      // Worker threads operate on their relevant WorkerState structs.
-      //
-      // WorkerState's fields are all protected by mu_;
-      struct WorkerState {
-        // The arguments to be used to construct an output iterator.
-        std::vector<Tensor> input;
-        // The buffered output elements.
-        std::deque<OutputElem> outputs;
-        // Set to true iff the worker thread expects to append more elements to
-        // outputs. is_producing can be false despite !outputs.empty().
-        // Concretely, all output elements will have been consumed only when:
-        // is_producing == false && outputs.empty();
-        bool is_producing = false;
-        // Condition variable used to coordinate between threads. The worker
-        // thread waits on this condition variable when it is either (1) waiting
-        // for the main thread to add arguments to `input`, or (2) waiting for
-        // the main thread to consume an element of `outputs`. The main thread
-        // waits on cond_var if it is waiting for the worker thread to produce
-        // an element into `outputs` (this implies sloppy_==false).
-        condition_variable cond_var;
-
-        inline bool MayHaveElements() const {
-          return is_producing || !outputs.empty();
-        }
-
-        // Sets inputs for a worker thread and notifies it to start processing.
-        void SetInputs(const Status& s, std::vector<Tensor> input_arguments) {
-          if (s.ok()) {
-            DCHECK(!MayHaveElements())
-                << "Tried to start inputs, despite already producing!";
-            input = std::move(input_arguments);
-            is_producing = true;
-            cond_var.notify_one();
-          } else {
-            outputs.emplace_back(s);
-          }
-        }
-      };
-
-      // The internal state of a worker thread that is not already captured
-      // in its `WorkerState`.
-      //
-      // This is needed only for checkpointing purposes. We keep this
-      // separate from `WorkerState` and guard its fields using a separate
-      // lock `ckpt_mu_` so as to not affect the performance of main pipeline.
-      struct WorkerThreadState {
-        // The output element that has been produced from the input iterator
-        // and is waiting to be added to `WorkerState.outputs`.
-        OutputElem output_elem;
-
-        // Whether the input iterator returned an `end_of_sequence`.
-        bool end_of_sequence = false;
-
-        // Status returned from `MakeIteratorFromInputElement`.
-        Status iterator_creation_status;
-
-        // The arguments to be used to construct `iterator`.
-        std::vector<Tensor> input;
-
-        std::unique_ptr<IteratorBase> iterator;
-
-        WorkerThreadState() : output_elem(Status::OK()) {}
-      };
-
-      Status EnsureWorkerThreadsStarted(IteratorContext* ctx)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-        if (worker_threads_.empty()) {
-          worker_threads_.reserve(dataset()->num_threads());
-          for (int64 i = 0; i < dataset()->num_threads(); ++i) {
-            std::vector<Tensor> args;
-            bool end_of_input = false;
-            Status s = input_impl_->GetNext(ctx, &args, &end_of_input);
-            if (end_of_input) {
-              input_impl_.reset();
-              return Status::OK();
-            }
-            workers_[i].SetInputs(s, std::move(args));
-            std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-            worker_threads_.emplace_back(ctx->env()->StartThread(
-                {}, strings::StrCat("tf_data_parallel_interleave_worker_", i),
-                [this, new_ctx, i]() { WorkerThread(new_ctx, i); }));
-            if (i < dataset()->cycle_length_) {
-              interleave_indices_.push_back(i);
-            } else {
-              staging_indices_.push_back(i);
-            }
-          }
-          DCHECK(interleave_indices_.size() == dataset()->cycle_length_);
-          DCHECK(staging_indices_.size() ==
-                 dataset()->prefetch_input_elements_);
-        }
-        return Status::OK();
-      }
-
-      // Produces elements into the worker's output buffers.
-      void WorkerThread(const std::shared_ptr<IteratorContext>& ctx,
-                        const int64 thread_index) {
-        // Notes on checkpointing thread local state, i.e., `WorkerThreadState`:
-        //
-        // 1. Any local state that may need to be checkpointed should be kept
-        //    in `worker_thread_states_[thread_index]`.
-        // 2. `WorkerThreadState` should contain state that is needed only for
-        //    checkpointing, i.e., if we were to remove checkpointing support,
-        //    we could keep that state as local variables in this thread.
-        // 3. This thread should only read/write state at `thread_index`
-        //    and should not access other thread states.
-        // 4. When restoring from checkpoint, threads are started only after
-        //    the restore is complete.
-        // 5. Once restored from a checkpoint, the local state is edited only
-        //    by this thread. 3 & 4 allow making assumptions like temporarily
-        //    caching local state in this thread and using it outside a lock
-        //    e.g. `make_new_iterator`.
-        // 6. `ckpt_mu_` should be wisely used to create *consistent*
-        //    checkpoint markers.
-
-        // std::function arguments are copy-constructable, so we pass raw
-        // pointers, and then immediately wrap them to ensure correct ownership.
-        RecordStart(ctx.get());
-        auto cleanup = gtl::MakeCleanup([this, thread_index, ctx] {
-          mutex_lock l(mu_);
-          workers_[thread_index].cond_var.notify_all();
-          RecordStop(ctx.get());
-        });
-        bool make_new_iterator;
-        {
-          tf_shared_lock l(ckpt_mu_);
-          // Decide whether a new iterator should be built.
-          // 1. If there is an existing iterator, we use it.
-          // 2. If there was an error in iterator creation that could not be
-          //    notified to the client we attempt to send that to the client
-          //    first.
-          make_new_iterator =
-              worker_thread_states_[thread_index].iterator == nullptr &&
-              worker_thread_states_[thread_index].iterator_creation_status.ok();
-        }
-        // Even though `make_new_iterator` has cached values from
-        // `worker_thread_states_[thread_index]` which is guarded by ckpt_mu_,
-        // it is safe to *read* `make_new_iterator`outside of a lock without
-        // worrying about concurrent changes to values in
-        // `worker_thread_states_[thread_index]`. See comment at the start of
-        // this function for details.
-        while (true) {
-          // Whether creation of the iterator succeeded.
-          Status iterator_creation_status;
-          // 1. Build a new iterator or use the existing one.
-          if (make_new_iterator) {
-            // 1a. Get new input tensors or use the exiting ones.
-            bool read_new_input;
-            {
-              tf_shared_lock l(ckpt_mu_);
-              // worker_thread_states_[thread_index].input will be non-empty
-              // if checkpointing happened at CHECKPOINT_MARKER_A.
-              read_new_input =
-                  worker_thread_states_[thread_index].input.empty();
-            }
-
-            if (read_new_input) {
-              mutex_lock l(mu_);
-              while (!cancelled_ && !workers_[thread_index].is_producing) {
-                RecordStop(ctx.get());
-                workers_[thread_index].cond_var.wait(l);
-                RecordStart(ctx.get());
-              }
-              if (cancelled_) return;
-              // Copy the input tensors so that we do not need to block on `mu_`
-              // when building the iterator.
-              // We keep a copy of the input tensors in
-              // `WorkerThreadState.input` till the iterator is in use. This is
-              // used in `RestoreInternal` to re-build the iterator.
-              // TODO(b/78046638): Explore ways to avoid tracking the input
-              // tensors.
-              tf_shared_lock ckpt_l(ckpt_mu_);
-              worker_thread_states_[thread_index].input.swap(
-                  workers_[thread_index].input);
-              // CHECKPOINT_MARKER_A
-              // We have the input tensors but have not built the iterator yet.
-            }
-
-            // 1b. Run the user defined function to produce a new iterator.
-            {
-              tf_shared_lock l(ckpt_mu_);
-              worker_thread_states_[thread_index].iterator_creation_status =
-                  MakeIteratorFromInputElement(
-                      ctx.get(), worker_thread_states_[thread_index].input,
-                      thread_index, *instantiated_captured_func_, prefix(),
-                      &worker_thread_states_[thread_index].iterator);
-              iterator_creation_status =
-                  worker_thread_states_[thread_index].iterator_creation_status;
-              if (!iterator_creation_status.ok()) {
-                worker_thread_states_[thread_index].input.clear();
-              }
-              // CHECKPOINT_MARKER_B
-              // Either an iterator has been successfully built and placed in
-              // `worker_thread_states_[thread_index].iterator` or it failed and
-              // a non-OK status has been put in
-              // `worker_thread_states_[thread_index].iterator_creation_status`.
-            }
-          } else {
-            tf_shared_lock l(ckpt_mu_);
-            iterator_creation_status =
-                worker_thread_states_[thread_index].iterator_creation_status;
-            // Mark that we have used up the restored iterator.
-            make_new_iterator = true;
-          }
-          // 2. Start producing elements or send error state to client if
-          //    iterator creation failed.
-          if (!iterator_creation_status.ok()) {
-            mutex_lock l(mu_);
-            // Wait for space in the prefetch queue.
-            while (!cancelled_ && workers_[thread_index].outputs.size() ==
-                                      dataset()->buffer_output_elements_) {
-              RecordStop(ctx.get());
-              workers_[thread_index].cond_var.wait(l);
-              RecordStart(ctx.get());
-            }
-            if (cancelled_) return;
-            tf_shared_lock ckpt_l(ckpt_mu_);
-            workers_[thread_index].outputs.emplace_back(
-                iterator_creation_status);
-            workers_[thread_index].is_producing = false;
-            worker_thread_states_[thread_index].iterator_creation_status =
-                Status::OK();
-            // CHECKPOINT_MARKER_C
-            // Non-OK iterator creation status has been notified to the
-            // client.
-            workers_[thread_index].cond_var.notify_one();
-          } else {
-            bool end_of_sequence = false;
-            while (!end_of_sequence) {
-              // 3.a Produce an element!
-              {
-                tf_shared_lock ckpt_l(ckpt_mu_);
-                if (worker_thread_states_[thread_index]
-                        .output_elem.status.ok() &&
-                    worker_thread_states_[thread_index]
-                        .output_elem.output.empty() &&
-                    !worker_thread_states_[thread_index].end_of_sequence) {
-                  worker_thread_states_[thread_index].output_elem.status =
-                      worker_thread_states_[thread_index].iterator->GetNext(
-                          ctx.get(),
-                          &worker_thread_states_[thread_index]
-                               .output_elem.output,
-                          &worker_thread_states_[thread_index].end_of_sequence);
-                  end_of_sequence =
-                      worker_thread_states_[thread_index].end_of_sequence;
-                } else {
-                  end_of_sequence =
-                      worker_thread_states_[thread_index].end_of_sequence;
-                }
-                // CHECKPOINT_MARKER_D
-                // An element has been read or an error or end_of_sequence has
-                // been received from the input iterator and is waiting to be
-                // sent to client.
-              }
-
-              // 3.b Make it available to the client.
-              {
-                mutex_lock l(mu_);
-
-                // Wait for space in the prefetch queue.
-                while (!cancelled_ && workers_[thread_index].outputs.size() ==
-                                          dataset()->buffer_output_elements_) {
-                  RecordStop(ctx.get());
-                  workers_[thread_index].cond_var.wait(l);
-                  RecordStart(ctx.get());
-                }
-                if (cancelled_) return;
-
-                tf_shared_lock ckpt_l(ckpt_mu_);
-                workers_[thread_index].is_producing = !end_of_sequence;
-
-                // Output the element.
-
-                // Move the temporary state in WorkerThreadState to WorkerState
-                // and mark it as used.
-                if (end_of_sequence) {
-                  worker_thread_states_[thread_index].iterator.reset();
-                  worker_thread_states_[thread_index].input.clear();
-                  worker_thread_states_[thread_index].end_of_sequence = false;
-                } else {
-                  workers_[thread_index].outputs.emplace_back(
-                      worker_thread_states_[thread_index].output_elem.status);
-                  workers_[thread_index].outputs.back().output.swap(
-                      worker_thread_states_[thread_index].output_elem.output);
-                }
-                worker_thread_states_[thread_index].output_elem.status =
-                    Status::OK();
-                if (dataset()->sloppy_) {
-                  sloppy_cond_var_.notify_one();
-                } else {
-                  workers_[thread_index].cond_var.notify_one();
-                }
-                // CHECKPOINT_MARKER_E
-                // Output element or iterator status has been sent to the
-                // client.
-              }
-            }
-          }
-        }
-      }
-
-      Status WriteWorkerStateLocked(IteratorStateWriter* writer, int index)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        string prefix = strings::StrCat("worker_", index);
-        TF_RETURN_IF_ERROR(writer->WriteScalar(
-            full_name(strings::StrCat(prefix, "_input_size")),
-            workers_[index].input.size()));
-        for (int i = 0; i < workers_[index].input.size(); ++i) {
-          TF_RETURN_IF_ERROR(writer->WriteTensor(
-              full_name(strings::StrCat(prefix, "_input_", i)),
-              workers_[index].input[i]));
-        }
-        TF_RETURN_IF_ERROR(writer->WriteScalar(
-            full_name(strings::StrCat(prefix, "_outputs_size")),
-            workers_[index].outputs.size()));
-        for (int i = 0; i < workers_[index].outputs.size(); ++i) {
-          TF_RETURN_IF_ERROR(WriteOutputElemLocked(
-              writer, workers_[index].outputs[i],
-              full_name(strings::StrCat(prefix, "_outputs_", i))));
-        }
-        if (workers_[index].is_producing) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat(prefix, "_is_producing")), ""));
-        }
-        return Status::OK();
-      }
-
-      Status ReadWorkerStateLocked(IteratorStateReader* reader, int index,
-                                   IteratorContext* ctx)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        string worker_prefix = strings::StrCat("worker_", index);
-        // Restore inputs.
-        int64 input_size;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            full_name(strings::StrCat(worker_prefix, "_input_size")),
-            &input_size));
-        workers_[index].input.reserve(input_size);
-        for (int i = 0; i < input_size; ++i) {
-          workers_[index].input.emplace_back();
-          TF_RETURN_IF_ERROR(reader->ReadTensor(
-              full_name(strings::StrCat(worker_prefix, "_input_", i)),
-              &workers_[index].input.back()));
-        }
-        int64 outputs_size;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            full_name(strings::StrCat(worker_prefix, "_outputs_size")),
-            &outputs_size));
-        for (int i = 0; i < outputs_size; ++i) {
-          workers_[index].outputs.emplace_back(Status::OK());
-          TF_RETURN_IF_ERROR(ReadOutputElemLocked(
-              reader, &workers_[index].outputs.back(),
-              full_name(strings::StrCat(worker_prefix, "_outputs_", i))));
-        }
-        if (reader->Contains(
-                full_name(strings::StrCat(worker_prefix, "_is_producing")))) {
-          workers_[index].is_producing = true;
-        } else {
-          workers_[index].is_producing = false;
-        }
-        return Status::OK();
-      }
-
-      Status WriteWorkerThreadStateLocked(IteratorStateWriter* writer,
-                                          int index)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        string prefix = strings::StrCat("worker_thread_", index);
-        if (worker_thread_states_[index].iterator != nullptr) {
-          TF_RETURN_IF_ERROR(
-              SaveInput(writer, worker_thread_states_[index].iterator));
-        } else {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat(prefix, "_iterator_exhausted")), ""));
-        }
-        TF_RETURN_IF_ERROR(writer->WriteScalar(
-            full_name(strings::StrCat(prefix, "_input_size")),
-            worker_thread_states_[index].input.size()));
-        for (int i = 0; i < worker_thread_states_[index].input.size(); ++i) {
-          TF_RETURN_IF_ERROR(writer->WriteTensor(
-              full_name(strings::StrCat(prefix, "_input_", i)),
-              worker_thread_states_[index].input[i]));
-        }
-        TF_RETURN_IF_ERROR(WriteStatusLocked(
-            writer, strings::StrCat(prefix, "_iterator_creation_status"),
-            worker_thread_states_[index].iterator_creation_status));
-        TF_RETURN_IF_ERROR(WriteOutputElemLocked(
-            writer, worker_thread_states_[index].output_elem,
-            full_name(strings::StrCat(prefix, "_output"))));
-        if (worker_thread_states_[index].end_of_sequence) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat(prefix, "_end_of_sequence")), ""));
-        }
-        return Status::OK();
-      }
-
-      Status ReadWorkerThreadStateLocked(IteratorStateReader* reader, int index,
-                                         IteratorContext* ctx)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        string worker_prefix = strings::StrCat("worker_thread_", index);
-        // Restore inputs.
-        int64 input_size;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            full_name(strings::StrCat(worker_prefix, "_input_size")),
-            &input_size));
-        worker_thread_states_[index].input.reserve(input_size);
-        for (int i = 0; i < input_size; ++i) {
-          worker_thread_states_[index].input.emplace_back();
-          TF_RETURN_IF_ERROR(reader->ReadTensor(
-              full_name(strings::StrCat(worker_prefix, "_input_", i)),
-              &worker_thread_states_[index].input.back()));
-        }
-        // Restore iterator.
-        if (reader->Contains(full_name(
-                strings::StrCat(worker_prefix, "_iterator_exhausted")))) {
-          worker_thread_states_[index].iterator.reset();
-        } else {
-          std::unique_ptr<IteratorBase> iterator;
-          Status s = MakeIteratorFromInputElement(
-              ctx, worker_thread_states_[index].input, index,
-              *instantiated_captured_func_, prefix(), &iterator);
-          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, iterator));
-          worker_thread_states_[index].iterator.swap(iterator);
-        }
-        TF_RETURN_IF_ERROR(ReadStatusLocked(
-            reader, strings::StrCat(worker_prefix, "_iterator_creation_status"),
-            &worker_thread_states_[index].iterator_creation_status));
-        TF_RETURN_IF_ERROR(ReadOutputElemLocked(
-            reader, &worker_thread_states_[index].output_elem,
-            full_name(strings::StrCat(worker_prefix, "_output"))));
-        if (reader->Contains(full_name(
-                strings::StrCat(worker_prefix, "_end_of_sequence")))) {
-          worker_thread_states_[index].end_of_sequence = true;
-        } else {
-          worker_thread_states_[index].end_of_sequence = false;
-        }
-        return Status::OK();
-      }
-
-      Status WriteOutputElemLocked(IteratorStateWriter* writer,
-                                   const OutputElem& output_elem,
-                                   const string& prefix)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        TF_RETURN_IF_ERROR(WriteStatusLocked(
-            writer, strings::StrCat(prefix, "_status"), output_elem.status));
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar(strings::StrCat(prefix, "_output_size"),
-                                output_elem.output.size()));
-        for (int i = 0; i < output_elem.output.size(); ++i) {
-          TF_RETURN_IF_ERROR(writer->WriteTensor(
-              strings::StrCat(prefix, "_output_", i), output_elem.output[i]));
-        }
-        return Status::OK();
-      }
-
-      Status ReadOutputElemLocked(IteratorStateReader* reader,
-                                  OutputElem* output_elem, const string& prefix)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        TF_RETURN_IF_ERROR(ReadStatusLocked(
-            reader, strings::StrCat(prefix, "_status"), &output_elem->status));
-        int64 output_size;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            strings::StrCat(prefix, "_output_size"), &output_size));
-        output_elem->output.reserve(output_size);
-        for (int i = 0; i < output_size; ++i) {
-          output_elem->output.emplace_back();
-          TF_RETURN_IF_ERROR(
-              reader->ReadTensor(strings::StrCat(prefix, "_output_", i),
-                                 &output_elem->output.back()));
-        }
-        return Status::OK();
-      }
-
-      Status WriteStatusLocked(IteratorStateWriter* writer,
-                               const string& prefix, const Status& status)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar(full_name(strings::StrCat(prefix, "_code")),
-                                static_cast<int64>(status.code())));
-        if (!status.ok()) {
-          TF_RETURN_IF_ERROR(
-              writer->WriteScalar(full_name(strings::StrCat(prefix, "_msg")),
-                                  status.error_message()));
-        }
-        return Status::OK();
-      }
-
-      Status ReadStatusLocked(IteratorStateReader* reader, const string& prefix,
-                              Status* status)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_, ckpt_mu_) {
-        int64 code_int;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            full_name(strings::StrCat(prefix, "_code")), &code_int));
-        error::Code code = static_cast<error::Code>(code_int);
-
-        if (code != error::Code::OK) {
-          string error_message;
-          TF_RETURN_IF_ERROR(reader->ReadScalar(
-              full_name(strings::StrCat(prefix, "_msg")), &error_message));
-          *status = Status(code, error_message);
-        } else {
-          *status = Status::OK();
-        }
-        return Status::OK();
-      }
-
-      // Mutex & condition variable to guard mutable iterator internals and
-      // coordinate among worker threads and client thread[s].
-      mutex mu_ ACQUIRED_BEFORE(ckpt_mu_);
-      // The main thread waits on this condition variable if running in sloppy
-      // mode and no values are available.
-      condition_variable sloppy_cond_var_;
-      // Mutex used to wait for a consistent state while checkpointing.
-      // Only Save and Restore require an exclusive lock on this mutex. In
-      // other scenarios we just acquire a shared lock so the pipeline's
-      // performance should not be affected in the absence of checkpointing.
-      // A thread must not wait on any condition variable while holding
-      // `ckpt_mu_` in either shared or exclusive modes.
-      mutex ckpt_mu_;
-
-      // The iterator producing elements which are converted to datasets by
-      // the dataset()->captured_func_ then interleaved together.
-      // input_impl_ is reset when we have exhausted its input.
-      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
-
-      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
-
-      // The WorkerState structs the worker threads operate on.
-      // workers_ elements are in at most one of interleave_ and staging_.
-      std::vector<WorkerState> workers_ GUARDED_BY(mu_);
-
-      // Stores the temporary state of WorkerThreads which is not stored in
-      // WorkerState. This is used for checkpointing purposes only.
-      std::vector<WorkerThreadState> worker_thread_states_ GUARDED_BY(ckpt_mu_);
-
-      // Indices in `workers_` of iterators to interleave.
-      std::vector<int64> interleave_indices_ GUARDED_BY(mu_);
-      // Indices in `workers_` of prefetched iterators.
-      std::deque<int64> staging_indices_ GUARDED_BY(mu_);
-
-      // The index into output_elements_ for next element to produce.
-      size_t next_index_ GUARDED_BY(mu_) = 0;
-      // The number of items produced so far within the block
-      size_t block_count_ GUARDED_BY(mu_) = 0;
-      // Flag to instruct the worker threads to exit.
-      bool cancelled_ GUARDED_BY(mu_) = false;
-      // The worker threads. This must be last to ensure the
-      // threads have exited before any other members are deallocated.
-      // TODO(b/65178177): Avoid allocating additional threads.
-      std::vector<std::unique_ptr<Thread>> worker_threads_ GUARDED_BY(mu_);
-    };
-
-    const DatasetBase* const input_;
-    const NameAttrList interleave_func_;
-    const std::unique_ptr<CapturedFunction> captured_func_;
-    const int64 cycle_length_;
-    const int64 block_length_;
-    const bool sloppy_;
-    const int64 buffer_output_elements_;
-    const int64 prefetch_input_elements_;
-    const DataTypeVector output_types_;
-    const std::vector<PartialTensorShape> output_shapes_;
-  };
-
-  DataTypeVector output_types_;
-  std::vector<PartialTensorShape> output_shapes_;
-  NameAttrList interleave_func_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("ParallelInterleaveDataset").Device(DEVICE_CPU),
-                        ParallelInterleaveDatasetOp);
-
 // The motivation for creating an alternative implementation of parallel
 // interleave is to decouple the degree of parallelism from the cycle length.
 // This makes it possible to change the degree of parallelism (e.g. through
@@ -1094,9 +49,9 @@ REGISTER_KERNEL_BUILDER(Name("ParallelInterleaveDataset").Device(DEVICE_CPU),
 // The above design choices were made with automated optimizations in mind,
 // isolating the degree of parallelism as the single tunable knob of this
 // implementation.
-class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
+class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
  public:
-  explicit ParallelInterleaveDatasetV2Op(OpKernelConstruction* ctx)
+  explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &interleave_func_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
@@ -1747,7 +702,7 @@ class ParallelInterleaveDatasetV2Op : public UnaryDatasetOpKernel {
 };
 
 REGISTER_KERNEL_BUILDER(Name("ParallelInterleaveDatasetV2").Device(DEVICE_CPU),
-                        ParallelInterleaveDatasetV2Op);
+                        ParallelInterleaveDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 170b839443..c8a5a9ef86 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12076,33 +12076,6 @@ op {
     type: "list(float)"
   }
 }
-op {
-  name: "BytesProducedStatsDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "tag"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "CTCBeamSearchDecoder"
   input_arg {
@@ -17326,21 +17299,6 @@ op {
     minimum: 1
   }
 }
-op {
-  name: "DatasetToTFRecord"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "filename"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "compression_type"
-    type: DT_STRING
-  }
-}
 op {
   name: "DebugGradientIdentity"
   input_arg {
@@ -18487,69 +18445,6 @@ op {
     }
   }
 }
-op {
-  name: "DenseToSparseBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "row_shape"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
-op {
-  name: "DenseToSparseBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "row_shape"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "DenseToSparseSetOperation"
   input_arg {
@@ -21626,6 +21521,33 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalBytesProducedStatsDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalCSVDataset"
   input_arg {
@@ -21691,6 +21613,53 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalDatasetToTFRecord"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "filename"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "compression_type"
+    type: DT_STRING
+  }
+}
+op {
+  name: "ExperimentalDenseToSparseBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "row_shape"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
 op {
   name: "ExperimentalDirectedInterleaveDataset"
   input_arg {
@@ -21725,6 +21694,145 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalGroupByReducerDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "key_func_other_arguments"
+    type_list_attr: "Tkey_func_other_arguments"
+  }
+  input_arg {
+    name: "init_func_other_arguments"
+    type_list_attr: "Tinit_func_other_arguments"
+  }
+  input_arg {
+    name: "reduce_func_other_arguments"
+    type_list_attr: "Treduce_func_other_arguments"
+  }
+  input_arg {
+    name: "finalize_func_other_arguments"
+    type_list_attr: "Tfinalize_func_other_arguments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "key_func"
+    type: "func"
+  }
+  attr {
+    name: "init_func"
+    type: "func"
+  }
+  attr {
+    name: "reduce_func"
+    type: "func"
+  }
+  attr {
+    name: "finalize_func"
+    type: "func"
+  }
+  attr {
+    name: "Tkey_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tinit_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Treduce_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tfinalize_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalGroupByWindowDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "key_func_other_arguments"
+    type_list_attr: "Tkey_func_other_arguments"
+  }
+  input_arg {
+    name: "reduce_func_other_arguments"
+    type_list_attr: "Treduce_func_other_arguments"
+  }
+  input_arg {
+    name: "window_size_func_other_arguments"
+    type_list_attr: "Twindow_size_func_other_arguments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "key_func"
+    type: "func"
+  }
+  attr {
+    name: "reduce_func"
+    type: "func"
+  }
+  attr {
+    name: "window_size_func"
+    type: "func"
+  }
+  attr {
+    name: "Tkey_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Treduce_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Twindow_size_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
 op {
   name: "ExperimentalIdentityIndexedDataset"
   input_arg {
@@ -21836,6 +21944,54 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalMapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalMapDataset"
   input_arg {
@@ -22017,6 +22173,209 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalParallelInterleaveDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "cycle_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "block_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sloppy"
+    type: DT_BOOL
+  }
+  input_arg {
+    name: "buffer_output_elements"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "prefetch_input_elements"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalParseExampleDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "dense_defaults"
+    type_list_attr: "Tdense"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "sparse_keys"
+    type: "list(string)"
+    has_minimum: true
+  }
+  attr {
+    name: "dense_keys"
+    type: "list(string)"
+    has_minimum: true
+  }
+  attr {
+    name: "sparse_types"
+    type: "list(type)"
+    has_minimum: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "Tdense"
+    type: "list(type)"
+    has_minimum: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "dense_shapes"
+    type: "list(shape)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalParseExampleDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "dense_defaults"
+    type_list_attr: "Tdense"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "sparse_keys"
+    type: "list(string)"
+    has_minimum: true
+  }
+  attr {
+    name: "dense_keys"
+    type: "list(string)"
+    has_minimum: true
+  }
+  attr {
+    name: "sparse_types"
+    type: "list(type)"
+    has_minimum: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "Tdense"
+    type: "list(type)"
+    has_minimum: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "dense_shapes"
+    type: "list(shape)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "sloppy"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ExperimentalPrivateThreadPoolDataset"
   input_arg {
@@ -22044,6 +22403,42 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalSetStatsAggregatorDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "stats_aggregator"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "counter_prefix"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
 op {
   name: "ExperimentalSleepDataset"
   input_arg {
@@ -22071,6 +22466,107 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalSlidingWindowDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "window_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "window_shift"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "window_stride"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalSqlDataset"
+  input_arg {
+    name: "driver_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "data_source_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "query"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalStatsAggregatorHandle"
+  output_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalStatsAggregatorSummary"
+  input_arg {
+    name: "iterator"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "summary"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
 op {
   name: "ExperimentalThreadPoolDataset"
   input_arg {
@@ -22136,6 +22632,29 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalUnbatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalUniqueDataset"
   input_arg {
@@ -26360,207 +26879,6 @@ op {
     }
   }
 }
-op {
-  name: "GroupByReducerDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "key_func_other_arguments"
-    type_list_attr: "Tkey_func_other_arguments"
-  }
-  input_arg {
-    name: "init_func_other_arguments"
-    type_list_attr: "Tinit_func_other_arguments"
-  }
-  input_arg {
-    name: "reduce_func_other_arguments"
-    type_list_attr: "Treduce_func_other_arguments"
-  }
-  input_arg {
-    name: "finalize_func_other_arguments"
-    type_list_attr: "Tfinalize_func_other_arguments"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "key_func"
-    type: "func"
-  }
-  attr {
-    name: "init_func"
-    type: "func"
-  }
-  attr {
-    name: "reduce_func"
-    type: "func"
-  }
-  attr {
-    name: "finalize_func"
-    type: "func"
-  }
-  attr {
-    name: "Tkey_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Tinit_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Treduce_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Tfinalize_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
-op {
-  name: "GroupByWindowDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "key_func_other_arguments"
-    type_list_attr: "Tkey_func_other_arguments"
-  }
-  input_arg {
-    name: "reduce_func_other_arguments"
-    type_list_attr: "Treduce_func_other_arguments"
-  }
-  input_arg {
-    name: "window_size_func_other_arguments"
-    type_list_attr: "Twindow_size_func_other_arguments"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "key_func"
-    type: "func"
-  }
-  attr {
-    name: "reduce_func"
-    type: "func"
-  }
-  attr {
-    name: "window_size_func"
-    type: "func"
-  }
-  attr {
-    name: "Tkey_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Treduce_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Twindow_size_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
-op {
-  name: "GroupByWindowDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "key_func_other_arguments"
-    type_list_attr: "Tkey_func_other_arguments"
-  }
-  input_arg {
-    name: "reduce_func_other_arguments"
-    type_list_attr: "Treduce_func_other_arguments"
-  }
-  input_arg {
-    name: "window_size_func_other_arguments"
-    type_list_attr: "Twindow_size_func_other_arguments"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "key_func"
-    type: "func"
-  }
-  attr {
-    name: "reduce_func"
-    type: "func"
-  }
-  attr {
-    name: "window_size_func"
-    type: "func"
-  }
-  attr {
-    name: "Tkey_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Treduce_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Twindow_size_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "GuaranteeConst"
   input_arg {
@@ -29025,119 +29343,59 @@ op {
   }
 }
 op {
-  name: "LRN"
-  input_arg {
-    name: "input"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "depth_radius"
-    type: "int"
-    default_value {
-      i: 5
-    }
-  }
-  attr {
-    name: "bias"
-    type: "float"
-    default_value {
-      f: 1
-    }
-  }
-  attr {
-    name: "alpha"
-    type: "float"
-    default_value {
-      f: 1
-    }
-  }
-  attr {
-    name: "beta"
-    type: "float"
-    default_value {
-      f: 0.5
-    }
-  }
-  attr {
-    name: "T"
-    type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
-    allowed_values {
-      list {
-        type: DT_HALF
-        type: DT_BFLOAT16
-        type: DT_FLOAT
-      }
-    }
-  }
-}
-op {
-  name: "LRNGrad"
-  input_arg {
-    name: "input_grads"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "input_image"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "output_image"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "depth_radius"
-    type: "int"
-    default_value {
-      i: 5
-    }
-  }
-  attr {
-    name: "bias"
-    type: "float"
-    default_value {
-      f: 1
-    }
-  }
-  attr {
-    name: "alpha"
-    type: "float"
-    default_value {
-      f: 1
-    }
-  }
-  attr {
-    name: "beta"
-    type: "float"
-    default_value {
-      f: 0.5
-    }
-  }
-  attr {
-    name: "T"
-    type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_HALF
-      }
-    }
-  }
-}
-op {
+  name: "LRN"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "depth_radius"
+    type: "int"
+    default_value {
+      i: 5
+    }
+  }
+  attr {
+    name: "bias"
+    type: "float"
+    default_value {
+      f: 1
+    }
+  }
+  attr {
+    name: "alpha"
+    type: "float"
+    default_value {
+      f: 1
+    }
+  }
+  attr {
+    name: "beta"
+    type: "float"
+    default_value {
+      f: 0.5
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+      }
+    }
+  }
+}
+op {
   name: "LRNGrad"
   input_arg {
     name: "input_grads"
@@ -29191,38 +29449,71 @@ op {
     }
     allowed_values {
       list {
-        type: DT_HALF
-        type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "LatencyStatsDataset"
+  name: "LRNGrad"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "input_grads"
+    type_attr: "T"
   }
   input_arg {
-    name: "tag"
-    type: DT_STRING
+    name: "input_image"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_image"
+    type_attr: "T"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "depth_radius"
+    type: "int"
+    default_value {
+      i: 5
+    }
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "bias"
+    type: "float"
+    default_value {
+      f: 1
+    }
+  }
+  attr {
+    name: "alpha"
+    type: "float"
+    default_value {
+      f: 1
+    }
+  }
+  attr {
+    name: "beta"
+    type: "float"
+    default_value {
+      f: 0.5
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
@@ -30682,102 +30973,6 @@ op {
   }
   is_stateful: true
 }
-op {
-  name: "MapAndBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "num_parallel_batches"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "drop_remainder"
-    type: DT_BOOL
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
-op {
-  name: "MapAndBatchDatasetV2"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "num_parallel_calls"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "drop_remainder"
-    type: DT_BOOL
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "MapClear"
   attr {
@@ -39046,62 +39241,6 @@ op {
     type: "type"
   }
 }
-op {
-  name: "ParallelInterleaveDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "cycle_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "block_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "sloppy"
-    type: DT_BOOL
-  }
-  input_arg {
-    name: "buffer_output_elements"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "prefetch_input_elements"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "ParallelInterleaveDatasetV2"
   input_arg {
@@ -39596,153 +39735,6 @@ op {
     has_minimum: true
   }
 }
-op {
-  name: "ParseExampleDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "num_parallel_calls"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "dense_defaults"
-    type_list_attr: "Tdense"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "sparse_keys"
-    type: "list(string)"
-    has_minimum: true
-  }
-  attr {
-    name: "dense_keys"
-    type: "list(string)"
-    has_minimum: true
-  }
-  attr {
-    name: "sparse_types"
-    type: "list(type)"
-    has_minimum: true
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_INT64
-        type: DT_STRING
-      }
-    }
-  }
-  attr {
-    name: "Tdense"
-    type: "list(type)"
-    has_minimum: true
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_INT64
-        type: DT_STRING
-      }
-    }
-  }
-  attr {
-    name: "dense_shapes"
-    type: "list(shape)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
-op {
-  name: "ParseExampleDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "num_parallel_calls"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "dense_defaults"
-    type_list_attr: "Tdense"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "sparse_keys"
-    type: "list(string)"
-    has_minimum: true
-  }
-  attr {
-    name: "dense_keys"
-    type: "list(string)"
-    has_minimum: true
-  }
-  attr {
-    name: "sparse_types"
-    type: "list(type)"
-    has_minimum: true
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_INT64
-        type: DT_STRING
-      }
-    }
-  }
-  attr {
-    name: "Tdense"
-    type: "list(type)"
-    has_minimum: true
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_INT64
-        type: DT_STRING
-      }
-    }
-  }
-  attr {
-    name: "dense_shapes"
-    type: "list(shape)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "sloppy"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
-}
 op {
   name: "ParseSequenceExample"
   input_arg {
@@ -61035,42 +61027,6 @@ op {
     }
   }
 }
-op {
-  name: "SetStatsAggregatorDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "stats_aggregator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "tag"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "counter_prefix"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "Shape"
   input_arg {
@@ -61917,41 +61873,6 @@ op {
     }
   }
 }
-op {
-  name: "SlideDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "window_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "window_shift"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "window_stride"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "Snapshot"
   input_arg {
@@ -71110,38 +71031,6 @@ op {
     }
   }
 }
-op {
-  name: "SqlDataset"
-  input_arg {
-    name: "driver_name"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "data_source_name"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "query"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "Sqrt"
   input_arg {
@@ -72702,40 +72591,6 @@ op {
     }
   }
 }
-op {
-  name: "StatsAggregatorHandle"
-  output_arg {
-    name: "handle"
-    type: DT_RESOURCE
-  }
-  attr {
-    name: "container"
-    type: "string"
-    default_value {
-      s: ""
-    }
-  }
-  attr {
-    name: "shared_name"
-    type: "string"
-    default_value {
-      s: ""
-    }
-  }
-  is_stateful: true
-}
-op {
-  name: "StatsAggregatorSummary"
-  input_arg {
-    name: "iterator"
-    type: DT_RESOURCE
-  }
-  output_arg {
-    name: "summary"
-    type: DT_STRING
-  }
-  is_stateful: true
-}
 op {
   name: "StopGradient"
   input_arg {
@@ -77256,29 +77111,6 @@ op {
     type: "type"
   }
 }
-op {
-  name: "UnbatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "UnbatchGrad"
   input_arg {
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index e7212b7004..f00274496c 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -83,13 +83,6 @@ REGISTER_OP("GeneratorDataset")
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
-REGISTER_OP("UnbatchDataset")
-    .Input("input_dataset: variant")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("ZipDataset")
     .Input("input_datasets: N * variant")
     .Output("handle: variant")
@@ -142,57 +135,6 @@ REGISTER_OP("SkipDataset")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("BytesProducedStatsDataset")
-    .Input("input_dataset: variant")
-    .Input("tag: string")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle tag_shape;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape));
-      return shape_inference::ScalarShape(c);
-    });
-
-REGISTER_OP("LatencyStatsDataset")
-    .Input("input_dataset: variant")
-    .Input("tag: string")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle tag_shape;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape));
-      return shape_inference::ScalarShape(c);
-    });
-
-REGISTER_OP("ParseExampleDataset")
-    .Input("input_dataset: variant")
-    .Input("num_parallel_calls: int64")
-    .Input("dense_defaults: Tdense")
-    .Output("handle: variant")
-    .Attr("sparse_keys: list(string) >= 0")
-    .Attr("dense_keys: list(string) >= 0")
-    .Attr("sparse_types: list({float,int64,string}) >= 0")
-    .Attr("Tdense: list({float,int64,string}) >= 0")
-    .Attr("dense_shapes: list(shape) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")  // Output components will be
-                                              // sorted by key (dense_keys and
-                                              // sparse_keys combined) here.
-    .Attr("sloppy: bool = false")
-    .SetShapeFn(shape_inference::ScalarShape);
-
-REGISTER_OP("SetStatsAggregatorDataset")
-    .Input("input_dataset: variant")
-    .Input("stats_aggregator: resource")
-    .Input("tag: string")
-    .Input("counter_prefix: string")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("MapDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
@@ -217,58 +159,6 @@ REGISTER_OP("ParallelMapDataset")
     .Attr("sloppy: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
-REGISTER_OP("MapAndBatchDataset")
-    .Input("input_dataset: variant")
-    .Input("other_arguments: Targuments")
-    .Input("batch_size: int64")
-    .Input("num_parallel_batches: int64")
-    .Input("drop_remainder: bool")
-    .Output("handle: variant")
-    .Attr("f: func")
-    .Attr("Targuments: list(type) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      // Use index from the end to retrieve the Input shapes,
-      // so that to avoid guessing the length of "other_arguments".
-      // batch_size, num_parallel_batches, and drop_remainder are 0-D scalars.
-      shape_inference::ShapeHandle unused;
-      TF_RETURN_IF_ERROR(
-          c->WithRank(c->input(c->num_inputs() - 3), 0, &unused));
-      TF_RETURN_IF_ERROR(
-          c->WithRank(c->input(c->num_inputs() - 2), 0, &unused));
-      TF_RETURN_IF_ERROR(
-          c->WithRank(c->input(c->num_inputs() - 1), 0, &unused));
-
-      return shape_inference::ScalarShape(c);
-    });
-
-REGISTER_OP("MapAndBatchDatasetV2")
-    .Input("input_dataset: variant")
-    .Input("other_arguments: Targuments")
-    .Input("batch_size: int64")
-    .Input("num_parallel_calls: int64")
-    .Input("drop_remainder: bool")
-    .Output("handle: variant")
-    .Attr("f: func")
-    .Attr("Targuments: list(type) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      // Use index from the end to retrieve the Input shapes,
-      // so that to avoid guessing the length of "other_arguments".
-      // batch_size, num_parallel_calls, and drop_remainder are 0-D scalars.
-      shape_inference::ShapeHandle unused;
-      TF_RETURN_IF_ERROR(
-          c->WithRank(c->input(c->num_inputs() - 3), 0, &unused));
-      TF_RETURN_IF_ERROR(
-          c->WithRank(c->input(c->num_inputs() - 2), 0, &unused));
-      TF_RETURN_IF_ERROR(
-          c->WithRank(c->input(c->num_inputs() - 1), 0, &unused));
-
-      return shape_inference::ScalarShape(c);
-    });
-
 REGISTER_OP("PrefetchDataset")
     .Input("input_dataset: variant")
     .Input("buffer_size: int64")
@@ -316,21 +206,6 @@ REGISTER_OP("InterleaveDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
-REGISTER_OP("ParallelInterleaveDataset")
-    .Input("input_dataset: variant")
-    .Input("other_arguments: Targuments")
-    .Input("cycle_length: int64")
-    .Input("block_length: int64")
-    .Input("sloppy: bool")
-    .Input("buffer_output_elements: int64")
-    .Input("prefetch_input_elements: int64")
-    .Output("handle: variant")
-    .Attr("f: func")
-    .Attr("Targuments: list(type) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("ParallelInterleaveDatasetV2")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
@@ -345,43 +220,6 @@ REGISTER_OP("ParallelInterleaveDatasetV2")
     .Attr("sloppy: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
-REGISTER_OP("GroupByReducerDataset")
-    .Input("input_dataset: variant")
-    .Input("key_func_other_arguments: Tkey_func_other_arguments")
-    .Input("init_func_other_arguments: Tinit_func_other_arguments")
-    .Input("reduce_func_other_arguments: Treduce_func_other_arguments")
-    .Input("finalize_func_other_arguments: Tfinalize_func_other_arguments")
-    .Output("handle: variant")
-    .Attr("key_func: func")
-    .Attr("init_func: func")
-    .Attr("reduce_func: func")
-    .Attr("finalize_func: func")
-    .Attr("Tkey_func_other_arguments: list(type) >= 0")
-    .Attr("Tinit_func_other_arguments: list(type) >= 0")
-    .Attr("Treduce_func_other_arguments: list(type) >= 0")
-    .Attr("Tfinalize_func_other_arguments: list(type) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()
-    .SetShapeFn(shape_inference::ScalarShape);
-
-REGISTER_OP("GroupByWindowDataset")
-    .Input("input_dataset: variant")
-    .Input("key_func_other_arguments: Tkey_func_other_arguments")
-    .Input("reduce_func_other_arguments: Treduce_func_other_arguments")
-    .Input(
-        "window_size_func_other_arguments: Twindow_size_func_other_arguments")
-    .Output("handle: variant")
-    .Attr("key_func: func")
-    .Attr("reduce_func: func")
-    .Attr("window_size_func: func")
-    .Attr("Tkey_func_other_arguments: list(type) >= 0")
-    .Attr("Treduce_func_other_arguments: list(type) >= 0")
-    .Attr("Twindow_size_func_other_arguments: list(type) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("FilterDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
@@ -447,23 +285,6 @@ REGISTER_OP("BatchDatasetV2")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("SlideDataset")
-    .Input("input_dataset: variant")
-    .Input("window_size: int64")
-    .Input("window_shift: int64")
-    .Input("window_stride: int64")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle unused;
-      // window_size, window_shift, and window_stride should be scalars.
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
-      return shape_inference::ScalarShape(c);
-    });
-
 // TODO(mrry): Validate that `padded_shapes` are all vectors, the lengths of
 // `output_types` and `output_shapes` are `N` the `output_shapes` are (as far as
 // possible to tell statically) compatible with `padded_shapes`, and that
@@ -504,22 +325,6 @@ REGISTER_OP("PaddedBatchDatasetV2")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("DenseToSparseBatchDataset")
-    .Input("input_dataset: variant")
-    .Input("batch_size: int64")
-    .Input("row_shape: int64")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle unused;
-      // batch_size should be a scalar.
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
-      // row_shape should be a 1-D vector.
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
-      return shape_inference::ScalarShape(c);
-    });
-
 REGISTER_OP("RangeDataset")
     .Input("start: int64")
     .Input("stop: int64")
@@ -622,24 +427,6 @@ REGISTER_OP("TextLineDataset")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("SqlDataset")
-    .Input("driver_name: string")
-    .Input("data_source_name: string")
-    .Input("query: string")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
-                      // stateful to inhibit constant folding.
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle unused;
-      // driver_name, data_source_name, and query should be scalars.
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
-      return shape_inference::ScalarShape(c);
-    });
-
 REGISTER_OP("FixedLengthRecordDataset")
     .Input("filenames: string")
     .Input("header_bytes: int64")
@@ -826,17 +613,6 @@ REGISTER_OP("DeserializeIterator")
     .Input("serialized: variant")
     .SetShapeFn(shape_inference::NoOutputs);
 
-REGISTER_OP("StatsAggregatorHandle")
-    .Output("handle: resource")
-    .SetShapeFn(shape_inference::ScalarShape)
-    .Attr("container: string = ''")
-    .Attr("shared_name: string = ''");
-
-REGISTER_OP("StatsAggregatorSummary")
-    .Input("iterator: resource")
-    .Output("summary: string")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("PrependFromQueueAndPaddedBatchDataset")
     .Input("input_dataset: variant")
     .Input("batch_size: int64")
@@ -867,12 +643,6 @@ REGISTER_OP("EnqueueInQueueDataset")
     // reading from queue handle (is that even possible?).
     .SetShapeFn(shape_inference::NoOutputs);
 
-REGISTER_OP("DatasetToTFRecord")
-    .Input("input_dataset: variant")
-    .Input("filename: string")
-    .Input("compression_type: string")
-    .SetShapeFn(shape_inference::NoOutputs);
-
 REGISTER_OP("DatasetToGraph")
     .Input("input_dataset: variant")
     .Output("graph: string")
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 2fd6cd0cd6..922f6e18a5 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -17,14 +17,17 @@ limitations under the License.
 
 namespace tensorflow {
 
-REGISTER_OP("ExperimentalDirectedInterleaveDataset")
-    .Input("selector_input_dataset: variant")
-    .Input("data_input_datasets: N * variant")
+REGISTER_OP("ExperimentalBytesProducedStatsDataset")
+    .Input("input_dataset: variant")
+    .Input("tag: string")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .Attr("N: int >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle tag_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape));
+      return shape_inference::ScalarShape(c);
+    });
 
 REGISTER_OP("ExperimentalCSVDataset")
     .Input("filenames: string")
@@ -68,6 +71,74 @@ REGISTER_OP("ExperimentalCSVDataset")
       return shape_inference::ScalarShape(c);
     });
 
+REGISTER_OP("ExperimentalDatasetToTFRecord")
+    .Input("input_dataset: variant")
+    .Input("filename: string")
+    .Input("compression_type: string")
+    .SetShapeFn(shape_inference::NoOutputs);
+
+REGISTER_OP("ExperimentalDenseToSparseBatchDataset")
+    .Input("input_dataset: variant")
+    .Input("batch_size: int64")
+    .Input("row_shape: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // batch_size should be a scalar.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
+      // row_shape should be a 1-D vector.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
+REGISTER_OP("ExperimentalDirectedInterleaveDataset")
+    .Input("selector_input_dataset: variant")
+    .Input("data_input_datasets: N * variant")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .Attr("N: int >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("ExperimentalGroupByReducerDataset")
+    .Input("input_dataset: variant")
+    .Input("key_func_other_arguments: Tkey_func_other_arguments")
+    .Input("init_func_other_arguments: Tinit_func_other_arguments")
+    .Input("reduce_func_other_arguments: Treduce_func_other_arguments")
+    .Input("finalize_func_other_arguments: Tfinalize_func_other_arguments")
+    .Output("handle: variant")
+    .Attr("key_func: func")
+    .Attr("init_func: func")
+    .Attr("reduce_func: func")
+    .Attr("finalize_func: func")
+    .Attr("Tkey_func_other_arguments: list(type) >= 0")
+    .Attr("Tinit_func_other_arguments: list(type) >= 0")
+    .Attr("Treduce_func_other_arguments: list(type) >= 0")
+    .Attr("Tfinalize_func_other_arguments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("ExperimentalGroupByWindowDataset")
+    .Input("input_dataset: variant")
+    .Input("key_func_other_arguments: Tkey_func_other_arguments")
+    .Input("reduce_func_other_arguments: Treduce_func_other_arguments")
+    .Input(
+        "window_size_func_other_arguments: Twindow_size_func_other_arguments")
+    .Output("handle: variant")
+    .Attr("key_func: func")
+    .Attr("reduce_func: func")
+    .Attr("window_size_func: func")
+    .Attr("Tkey_func_other_arguments: list(type) >= 0")
+    .Attr("Treduce_func_other_arguments: list(type) >= 0")
+    .Attr("Twindow_size_func_other_arguments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalIgnoreErrorsDataset")
     .Input("input_dataset: variant")
     .Output("handle: variant")
@@ -75,6 +146,44 @@ REGISTER_OP("ExperimentalIgnoreErrorsDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalLatencyStatsDataset")
+    .Input("input_dataset: variant")
+    .Input("tag: string")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle tag_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &tag_shape));
+      return shape_inference::ScalarShape(c);
+    });
+
+REGISTER_OP("ExperimentalMapAndBatchDataset")
+    .Input("input_dataset: variant")
+    .Input("other_arguments: Targuments")
+    .Input("batch_size: int64")
+    .Input("num_parallel_calls: int64")
+    .Input("drop_remainder: bool")
+    .Output("handle: variant")
+    .Attr("f: func")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      // Use index from the end to retrieve the Input shapes,
+      // so that to avoid guessing the length of "other_arguments".
+      // batch_size, num_parallel_calls, and drop_remainder are 0-D scalars.
+      shape_inference::ShapeHandle unused;
+      TF_RETURN_IF_ERROR(
+          c->WithRank(c->input(c->num_inputs() - 3), 0, &unused));
+      TF_RETURN_IF_ERROR(
+          c->WithRank(c->input(c->num_inputs() - 2), 0, &unused));
+      TF_RETURN_IF_ERROR(
+          c->WithRank(c->input(c->num_inputs() - 1), 0, &unused));
+
+      return shape_inference::ScalarShape(c);
+    });
+
 REGISTER_OP("ExperimentalMapDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
@@ -105,6 +214,48 @@ REGISTER_OP("ExperimentalNonSerializableDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalParallelInterleaveDataset")
+    .Input("input_dataset: variant")
+    .Input("other_arguments: Targuments")
+    .Input("cycle_length: int64")
+    .Input("block_length: int64")
+    .Input("sloppy: bool")
+    .Input("buffer_output_elements: int64")
+    .Input("prefetch_input_elements: int64")
+    .Output("handle: variant")
+    .Attr("f: func")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("ExperimentalParseExampleDataset")
+    .Input("input_dataset: variant")
+    .Input("num_parallel_calls: int64")
+    .Input("dense_defaults: Tdense")
+    .Output("handle: variant")
+    .Attr("sparse_keys: list(string) >= 0")
+    .Attr("dense_keys: list(string) >= 0")
+    .Attr("sparse_types: list({float,int64,string}) >= 0")
+    .Attr("Tdense: list({float,int64,string}) >= 0")
+    .Attr("dense_shapes: list(shape) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")  // Output components will be
+                                              // sorted by key (dense_keys and
+                                              // sparse_keys combined) here.
+    .Attr("sloppy: bool = false")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("ExperimentalSetStatsAggregatorDataset")
+    .Input("input_dataset: variant")
+    .Input("stats_aggregator: resource")
+    .Input("tag: string")
+    .Input("counter_prefix: string")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalSleepDataset")
     .Input("input_dataset: variant")
     .Input("sleep_microseconds: int64")
@@ -119,6 +270,59 @@ REGISTER_OP("ExperimentalSleepDataset")
       return shape_inference::ScalarShape(c);
     });
 
+REGISTER_OP("ExperimentalSlidingWindowDataset")
+    .Input("input_dataset: variant")
+    .Input("window_size: int64")
+    .Input("window_shift: int64")
+    .Input("window_stride: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // window_size, window_shift, and window_stride should be scalars.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
+REGISTER_OP("ExperimentalSqlDataset")
+    .Input("driver_name: string")
+    .Input("data_source_name: string")
+    .Input("query: string")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+                      // stateful to inhibit constant folding.
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // driver_name, data_source_name, and query should be scalars.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
+REGISTER_OP("ExperimentalStatsAggregatorHandle")
+    .Output("handle: resource")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Attr("container: string = ''")
+    .Attr("shared_name: string = ''");
+
+REGISTER_OP("ExperimentalStatsAggregatorSummary")
+    .Input("iterator: resource")
+    .Output("summary: string")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("ExperimentalUnbatchDataset")
+    .Input("input_dataset: variant")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalUniqueDataset")
     .Input("input_dataset: variant")
     .Output("handle: variant")
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index d8985fd13b..1422966d27 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -34,7 +34,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_array_ops
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.util.tf_export import tf_export
@@ -365,7 +365,7 @@ class _UnbatchDataset(dataset_ops.UnaryDataset):
     self._input_dataset = input_dataset
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.unbatch_dataset(
+    return ged_ops.experimental_unbatch_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         **dataset_ops.flat_structure(self))
 
@@ -455,7 +455,7 @@ class _DenseToSparseBatchDataset(dataset_ops.UnaryDataset):
     self._row_shape = row_shape
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.dense_to_sparse_batch_dataset(
+    return ged_ops.experimental_dense_to_sparse_batch_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._batch_size,
         row_shape=convert.partial_shape_to_tensor(self._row_shape),
@@ -589,7 +589,7 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
     input_resource = self._input_dataset._as_variant_tensor()
-    return gen_dataset_ops.map_and_batch_dataset_v2(
+    return ged_ops.experimental_map_and_batch_dataset(
         input_resource,
         self._map_func.captured_inputs,
         f=self._map_func,
diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index db10ea3b7f..5c68d91536 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -28,7 +28,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
@@ -237,29 +237,6 @@ def bucket_by_sequence_length(element_length_func,
     return _apply_fn
 
 
-def _map_x_dataset(map_func):
-  """A transformation that maps `map_func` across its input.
-
-  This transformation is similar to `tf.data.Dataset.map`, but in addition to
-  supporting dense and sparse tensor inputs, it also supports dataset inputs.
-
-  Args:
-    map_func: A function mapping a nested structure of tensors and/or datasets
-      (having shapes and types defined by `self.output_shapes` and
-     `self.output_types`) to another nested structure of tensors and/or
-     datasets.
-
-  Returns:
-    Dataset: A `Dataset`.
-  """
-
-  def _apply_fn(dataset):
-    """Function from `Dataset` to `Dataset` that applies the transformation."""
-    return _MapXDataset(dataset, map_func)
-
-  return _apply_fn
-
-
 class _GroupByReducerDataset(dataset_ops.UnaryDataset):
   """A `Dataset` that groups its input and performs a reduction."""
 
@@ -386,7 +363,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
     return self._output_types
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.group_by_reducer_dataset(
+    return ged_ops.experimental_group_by_reducer_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._key_func.captured_inputs,
         self._init_func.captured_inputs,
@@ -482,7 +459,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
     return self._output_types
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.group_by_window_dataset(
+    return ged_ops.experimental_group_by_window_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._key_func.captured_inputs,
         self._reduce_func.captured_inputs,
@@ -522,42 +499,3 @@ class Reducer(object):
   @property
   def finalize_func(self):
     return self._finalize_func
-
-
-class _MapXDataset(dataset_ops.UnaryDataset):
-  """A `Dataset` that maps a function over elements in its input."""
-
-  def __init__(self, input_dataset, map_func):
-    """See `map_x_dataset()` for details."""
-    super(_MapXDataset, self).__init__(input_dataset)
-    self._input_dataset = input_dataset
-
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
-        map_func, self._transformation_name(), dataset=input_dataset)
-    self._output_classes = wrapped_func.output_classes
-    self._output_shapes = wrapped_func.output_shapes
-    self._output_types = wrapped_func.output_types
-    self._map_func = wrapped_func.function
-
-  def _as_variant_tensor(self):
-    input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
-    return gen_dataset_ops.map_dataset(
-        input_t,
-        self._map_func.captured_inputs,
-        f=self._map_func,
-        **dataset_ops.flat_structure(self))
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
-
-  def _transformation_name(self):
-    return "tf.data.experimental.map_x_dataset()"
diff --git a/tensorflow/python/data/experimental/ops/parsing_ops.py b/tensorflow/python/data/experimental/ops/parsing_ops.py
index a63eb8c516..44cb0b8a2c 100644
--- a/tensorflow/python/data/experimental/ops/parsing_ops.py
+++ b/tensorflow/python/data/experimental/ops/parsing_ops.py
@@ -22,7 +22,7 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.util.tf_export import tf_export
 
@@ -80,7 +80,7 @@ class _ParseExampleDataset(dataset_ops.UnaryDataset):
             ]))
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.parse_example_dataset(
+    return gen_experimental_dataset_ops.experimental_parse_example_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._num_parallel_calls,
         self._dense_defaults,
diff --git a/tensorflow/python/data/experimental/ops/readers.py b/tensorflow/python/data/experimental/ops/readers.py
index 1ba26ed5b9..f6faf04998 100644
--- a/tensorflow/python/data/experimental/ops/readers.py
+++ b/tensorflow/python/data/experimental/ops/readers.py
@@ -37,7 +37,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.lib.io import file_io
-from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.ops import io_ops
 from tensorflow.python.platform import gfile
@@ -971,10 +970,9 @@ class SqlDatasetV2(dataset_ops.DatasetSource):
     self._output_types = output_types
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.sql_dataset(self._driver_name,
-                                       self._data_source_name, self._query,
-                                       nest.flatten(self.output_types),
-                                       nest.flatten(self.output_shapes))
+    return gen_experimental_dataset_ops.experimental_sql_dataset(
+        self._driver_name, self._data_source_name, self._query,
+        nest.flatten(self.output_types), nest.flatten(self.output_shapes))
 
   @property
   def output_classes(self):
diff --git a/tensorflow/python/data/experimental/ops/stats_aggregator.py b/tensorflow/python/data/experimental/ops/stats_aggregator.py
index 5274c816a4..f54881b150 100644
--- a/tensorflow/python/data/experimental/ops/stats_aggregator.py
+++ b/tensorflow/python/data/experimental/ops/stats_aggregator.py
@@ -17,7 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -69,7 +69,7 @@ class StatsAggregator(object):
 
   def __init__(self):
     """Creates a `StatsAggregator`."""
-    self._resource = gen_dataset_ops.stats_aggregator_handle()
+    self._resource = ged_ops.experimental_stats_aggregator_handle()
 
   # TODO(b/116314787): Update this/add support for V2 summary API.
   def get_summary(self):
@@ -81,4 +81,4 @@ class StatsAggregator(object):
     Returns:
       A scalar string `tf.Tensor` that summarizes the aggregated statistics.
     """
-    return gen_dataset_ops.stats_aggregator_summary(self._resource)
+    return ged_ops.experimental_stats_aggregator_summary(self._resource)
diff --git a/tensorflow/python/data/experimental/ops/stats_ops.py b/tensorflow/python/data/experimental/ops/stats_ops.py
index 95689433bd..15a9d24546 100644
--- a/tensorflow/python/data/experimental/ops/stats_ops.py
+++ b/tensorflow/python/data/experimental/ops/stats_ops.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
@@ -66,8 +66,10 @@ def bytes_produced_stats(tag):
   """
 
   def _apply_fn(dataset):
-    return _StatsDataset(dataset, gen_dataset_ops.bytes_produced_stats_dataset,
-                         tag)
+    return _StatsDataset(
+        dataset,
+        gen_experimental_dataset_ops.experimental_bytes_produced_stats_dataset,
+        tag)
 
   return _apply_fn
 
@@ -89,7 +91,9 @@ def latency_stats(tag):
   """
 
   def _apply_fn(dataset):
-    return _StatsDataset(dataset, gen_dataset_ops.latency_stats_dataset, tag)
+    return _StatsDataset(
+        dataset,
+        gen_experimental_dataset_ops.experimental_latency_stats_dataset, tag)
 
   return _apply_fn
 
diff --git a/tensorflow/python/data/experimental/ops/writers.py b/tensorflow/python/data/experimental/ops/writers.py
index cc0a80336c..aef6da5140 100644
--- a/tensorflow/python/data/experimental/ops/writers.py
+++ b/tensorflow/python/data/experimental/ops/writers.py
@@ -22,7 +22,7 @@ from tensorflow.python.data.util import convert
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -56,5 +56,5 @@ class TFRecordWriter(object):
           "`dataset` must produce scalar `DT_STRING` tensors whereas it "
           "produces shape {0} and types {1}".format(dataset.output_shapes,
                                                     dataset.output_types))
-    return gen_dataset_ops.dataset_to_tf_record(
+    return gen_experimental_dataset_ops.experimental_dataset_to_tf_record(
         dataset._as_variant_tensor(), self._filename, self._compression_type)  # pylint: disable=protected-access
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index c57ddab15f..7070a84b70 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -2992,7 +2992,7 @@ class _SetStatsAggregatorDataset(UnaryUnchangedStructureDataset):
     self._counter_prefix = counter_prefix
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.set_stats_aggregator_dataset(
+    return ged_ops.experimental_set_stats_aggregator_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._stats_aggregator._resource,  # pylint: disable=protected-access
         self._prefix,
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index a93f3e44d1..29f5f20282 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -158,7 +159,7 @@ class ParallelInterleaveDataset(dataset_ops.InterleaveDataset):
 
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
-    return gen_dataset_ops.parallel_interleave_dataset(
+    return ged_ops.experimental_parallel_interleave_dataset(
         self._input_dataset._as_variant_tensor(),
         self._map_func.captured_inputs,
         self._cycle_length,
-- 
GitLab


From 043fb15805bf1d1e3649b52e89980635942ac163 Mon Sep 17 00:00:00 2001
From: AG Ramesh <ag.ramesh@intel.com>
Date: Sun, 2 Dec 2018 11:33:18 -0800
Subject: [PATCH 1167/1554] Temporarily disable transpose optimization.

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 3 ++-
 tensorflow/core/graph/mkl_layout_pass_test.cc | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index efbeb06e8f..7b12f0a5bc 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2901,6 +2901,7 @@ bool MklLayoutRewritePass::RunPass(std::unique_ptr<Graph>* g) {
 
   DumpGraph("After running MklLayoutRewritePass(NodeMerge)", &**g);
 
+#ifdef ENABLE_TRANSPOSE_OPTIMIZATION
   order.clear();
   GetReversePostOrder(**g, &order);  // This will give us topological sort.
   for (Node* n : order) {
@@ -2921,8 +2922,8 @@ bool MklLayoutRewritePass::RunPass(std::unique_ptr<Graph>* g) {
       }
     }
   }
-
   DumpGraph("After running MklLayoutRewritePass(NodeFusion)", &**g);
+#endif  // ENABLE_TRANSPOSE_OPTIMIZATION
 
   order.clear();
   GetReversePostOrder(**g, &order);  // This will give us topological sort.
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index f4f2ab2a97..22f8791870 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -455,6 +455,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_ConvBpropInput_FilterFwd) {
             "E:3->G:4;F->G;F:control->DMT/_3:control;G->Z;X->Y:1;X->Z:1");
 }
 
+#ifdef ENABLE_TRANSPOSE_OPTIMIZATION
 TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Positive) {
   InitGraph(
       "node { name: 'Input0' op: 'Input'}"
@@ -763,6 +764,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_TransposeConv2DTranspose_Negative) {
       "Transpose0:control->DMT/"
       "_1:control;Transpose1->Relu;Transpose1:control->DMT/_2:control");
 }
+#endif  // ENABLE_TRANSPOSE_OPTIMIZATION
 
 /////////////////////////////////////////////////////////////////////
 //  Unit tests related to rewriting node to Mkl node
-- 
GitLab


From e4e93916658de21aa4b22ca37e2629a664585852 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Sun, 2 Dec 2018 11:35:07 -0800
Subject: [PATCH 1168/1554] Fix API definition filename for
 TensorListPushBackBatch.

PiperOrigin-RevId: 223711401
---
 ...tPushBackBatch.pbtxt => api_def_TensorListPushBackBatch.pbtxt} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tensorflow/core/api_def/python_api/{api_defTensorListPushBackBatch.pbtxt => api_def_TensorListPushBackBatch.pbtxt} (100%)

diff --git a/tensorflow/core/api_def/python_api/api_defTensorListPushBackBatch.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorListPushBackBatch.pbtxt
similarity index 100%
rename from tensorflow/core/api_def/python_api/api_defTensorListPushBackBatch.pbtxt
rename to tensorflow/core/api_def/python_api/api_def_TensorListPushBackBatch.pbtxt
-- 
GitLab


From 823cf73e5f55ccca6178b85a9cb8dc2d77b5582b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Sun, 2 Dec 2018 12:09:47 -0800
Subject: [PATCH 1169/1554] [tf.data] Moving the "visibility: HIDDEN" API
 definition file attribute from Python API (overrides) to the base API.

PiperOrigin-RevId: 223712881
---
 tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt   | 1 +
 tensorflow/core/api_def/base_api/api_def_CacheDataset.pbtxt   | 1 +
 .../core/api_def/base_api/api_def_ConcatenateDataset.pbtxt    | 1 +
 .../api_def/base_api/api_def_DatasetToSingleElement.pbtxt     | 1 +
 .../core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt | 1 +
 tensorflow/core/api_def/base_api/api_def_FilterDataset.pbtxt  | 1 +
 .../api_def/base_api/api_def_FixedLengthRecordDataset.pbtxt   | 1 +
 .../api_def/base_api/api_def_FixedLengthRecordDatasetV2.pbtxt | 1 +
 tensorflow/core/api_def/base_api/api_def_FlatMapDataset.pbtxt | 1 +
 .../core/api_def/base_api/api_def_GeneratorDataset.pbtxt      | 1 +
 .../core/api_def/base_api/api_def_InterleaveDataset.pbtxt     | 1 +
 tensorflow/core/api_def/base_api/api_def_MapDataset.pbtxt     | 1 +
 .../core/api_def/base_api/api_def_PaddedBatchDataset.pbtxt    | 1 +
 .../core/api_def/base_api/api_def_ParallelMapDataset.pbtxt    | 1 +
 .../core/api_def/base_api/api_def_PrefetchDataset.pbtxt       | 1 +
 .../api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt       | 1 +
 tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt  | 1 +
 tensorflow/core/api_def/base_api/api_def_RangeDataset.pbtxt   | 1 +
 tensorflow/core/api_def/base_api/api_def_RepeatDataset.pbtxt  | 1 +
 tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt    | 1 +
 .../api_def/base_api/api_def_ShuffleAndRepeatDataset.pbtxt    | 1 +
 tensorflow/core/api_def/base_api/api_def_ShuffleDataset.pbtxt | 1 +
 tensorflow/core/api_def/base_api/api_def_SkipDataset.pbtxt    | 1 +
 .../api_def/base_api/api_def_SparseTensorSliceDataset.pbtxt   | 1 +
 .../core/api_def/base_api/api_def_TFRecordDataset.pbtxt       | 1 +
 tensorflow/core/api_def/base_api/api_def_TakeDataset.pbtxt    | 1 +
 tensorflow/core/api_def/base_api/api_def_TensorDataset.pbtxt  | 1 +
 .../core/api_def/base_api/api_def_TensorSliceDataset.pbtxt    | 1 +
 .../core/api_def/base_api/api_def_TextLineDataset.pbtxt       | 1 +
 tensorflow/core/api_def/base_api/api_def_ZipDataset.pbtxt     | 1 +
 tensorflow/core/api_def/python_api/api_def_BatchDataset.pbtxt | 4 ----
 tensorflow/core/api_def/python_api/api_def_CacheDataset.pbtxt | 4 ----
 .../core/api_def/python_api/api_def_ConcatenateDataset.pbtxt  | 4 ----
 .../api_def/python_api/api_def_DatasetToSingleElement.pbtxt   | 4 ----
 .../api_def/python_api/api_def_EnqueueInQueueDataset.pbtxt    | 4 ----
 .../core/api_def/python_api/api_def_FilterDataset.pbtxt       | 4 ----
 .../api_def/python_api/api_def_FixedLengthRecordDataset.pbtxt | 4 ----
 .../python_api/api_def_FixedLengthRecordDatasetV2.pbtxt       | 4 ----
 .../core/api_def/python_api/api_def_FlatMapDataset.pbtxt      | 4 ----
 .../core/api_def/python_api/api_def_GeneratorDataset.pbtxt    | 4 ----
 .../core/api_def/python_api/api_def_InterleaveDataset.pbtxt   | 4 ----
 tensorflow/core/api_def/python_api/api_def_MapDataset.pbtxt   | 4 ----
 .../core/api_def/python_api/api_def_PaddedBatchDataset.pbtxt  | 4 ----
 .../core/api_def/python_api/api_def_ParallelMapDataset.pbtxt  | 4 ----
 .../core/api_def/python_api/api_def_PrefetchDataset.pbtxt     | 4 ----
 .../api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt       | 4 ----
 .../core/api_def/python_api/api_def_RandomDataset.pbtxt       | 4 ----
 tensorflow/core/api_def/python_api/api_def_RangeDataset.pbtxt | 4 ----
 .../core/api_def/python_api/api_def_RepeatDataset.pbtxt       | 4 ----
 tensorflow/core/api_def/python_api/api_def_ScanDataset.pbtxt  | 4 ----
 .../api_def/python_api/api_def_ShuffleAndRepeatDataset.pbtxt  | 4 ----
 .../core/api_def/python_api/api_def_ShuffleDataset.pbtxt      | 4 ----
 tensorflow/core/api_def/python_api/api_def_SkipDataset.pbtxt  | 4 ----
 .../api_def/python_api/api_def_SparseTensorSliceDataset.pbtxt | 4 ----
 .../core/api_def/python_api/api_def_TFRecordDataset.pbtxt     | 4 ----
 tensorflow/core/api_def/python_api/api_def_TakeDataset.pbtxt  | 4 ----
 .../core/api_def/python_api/api_def_TensorDataset.pbtxt       | 4 ----
 .../core/api_def/python_api/api_def_TensorSliceDataset.pbtxt  | 4 ----
 .../core/api_def/python_api/api_def_TextLineDataset.pbtxt     | 4 ----
 tensorflow/core/api_def/python_api/api_def_ZipDataset.pbtxt   | 4 ----
 60 files changed, 30 insertions(+), 120 deletions(-)
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_BatchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_CacheDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ConcatenateDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_DatasetToSingleElement.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_EnqueueInQueueDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_FilterDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDatasetV2.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_FlatMapDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_GeneratorDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_InterleaveDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_MapDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_PaddedBatchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ParallelMapDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_PrefetchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_RandomDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_RangeDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_RepeatDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ScanDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ShuffleAndRepeatDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ShuffleDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_SkipDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_SparseTensorSliceDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_TFRecordDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_TakeDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_TensorDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_TensorSliceDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_TextLineDataset.pbtxt
 delete mode 100644 tensorflow/core/api_def/python_api/api_def_ZipDataset.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt
index 639d962874..32def912f8 100644
--- a/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_BatchDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "BatchDataset"
+  visibility: HIDDEN
   in_arg {
     name: "batch_size"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_CacheDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_CacheDataset.pbtxt
index 6889b8ea14..9f7088b900 100644
--- a/tensorflow/core/api_def/base_api/api_def_CacheDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_CacheDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "CacheDataset"
+  visibility: HIDDEN
   in_arg {
     name: "filename"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ConcatenateDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ConcatenateDataset.pbtxt
index 67281f9547..7997d8daaf 100644
--- a/tensorflow/core/api_def/base_api/api_def_ConcatenateDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ConcatenateDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "ConcatenateDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that concatenates `input_dataset` with `another_dataset`."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_DatasetToSingleElement.pbtxt b/tensorflow/core/api_def/base_api/api_def_DatasetToSingleElement.pbtxt
index 2b9dffd883..27d7d6b986 100644
--- a/tensorflow/core/api_def/base_api/api_def_DatasetToSingleElement.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_DatasetToSingleElement.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "DatasetToSingleElement"
+  visibility: HIDDEN
   in_arg {
     name: "dataset"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt
index 9722f5ede3..051cf14c0e 100644
--- a/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt
@@ -1,3 +1,4 @@
 op {
   graph_op_name: "EnqueueInQueueDataset"
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_FilterDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_FilterDataset.pbtxt
index fd60c0f378..776529bc59 100644
--- a/tensorflow/core/api_def/base_api/api_def_FilterDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FilterDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "FilterDataset"
+  visibility: HIDDEN
   in_arg {
     name: "other_arguments"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDataset.pbtxt
index 651b84d0d6..3b14243258 100644
--- a/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "FixedLengthRecordDataset"
+  visibility: HIDDEN
   in_arg {
     name: "filenames"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDatasetV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDatasetV2.pbtxt
index ad82eddb58..def9f85e02 100644
--- a/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDatasetV2.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FixedLengthRecordDatasetV2.pbtxt
@@ -1,3 +1,4 @@
 op {
   graph_op_name: "FixedLengthRecordDatasetV2"
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_FlatMapDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_FlatMapDataset.pbtxt
index 1936119c50..1e20e85325 100644
--- a/tensorflow/core/api_def/base_api/api_def_FlatMapDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FlatMapDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "FlatMapDataset"
+  visibility: HIDDEN
   attr {
     name: "f"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_GeneratorDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_GeneratorDataset.pbtxt
index 4f1cf3e686..06e9a6463e 100644
--- a/tensorflow/core/api_def/base_api/api_def_GeneratorDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_GeneratorDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "GeneratorDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that invokes a function to generate elements."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_InterleaveDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_InterleaveDataset.pbtxt
index bec2828e24..597edf5fb2 100644
--- a/tensorflow/core/api_def/base_api/api_def_InterleaveDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_InterleaveDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "InterleaveDataset"
+  visibility: HIDDEN
   attr {
     name: "f"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_MapDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_MapDataset.pbtxt
index 76d63ec247..4f235f4946 100644
--- a/tensorflow/core/api_def/base_api/api_def_MapDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_MapDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "MapDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_PaddedBatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_PaddedBatchDataset.pbtxt
index d243dfe8b6..53f4d94ecc 100644
--- a/tensorflow/core/api_def/base_api/api_def_PaddedBatchDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_PaddedBatchDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "PaddedBatchDataset"
+  visibility: HIDDEN
   in_arg {
     name: "batch_size"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ParallelMapDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ParallelMapDataset.pbtxt
index 313494dd73..5343605edd 100644
--- a/tensorflow/core/api_def/base_api/api_def_ParallelMapDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ParallelMapDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "ParallelMapDataset"
+  visibility: HIDDEN
   in_arg {
     name: "num_parallel_calls"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_PrefetchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_PrefetchDataset.pbtxt
index e158eedc6f..a71336a285 100644
--- a/tensorflow/core/api_def/base_api/api_def_PrefetchDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_PrefetchDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "PrefetchDataset"
+  visibility: HIDDEN
   in_arg {
     name: "buffer_size"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
index d4549340fa..228c4047d2 100644
--- a/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
@@ -1,3 +1,4 @@
 op {
   graph_op_name: "PrependFromQueueAndPaddedBatchDataset"
+  visibility: HIDDEN
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt
index 0466b40f85..103e608ae8 100644
--- a/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "RandomDataset"
+  visibility: HIDDEN
   in_arg {
     name: "seed"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_RangeDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_RangeDataset.pbtxt
index a9e14b8a05..4ac5050040 100644
--- a/tensorflow/core/api_def/base_api/api_def_RangeDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RangeDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "RangeDataset"
+  visibility: HIDDEN
   in_arg {
     name: "start"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_RepeatDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_RepeatDataset.pbtxt
index fc6169cd32..b2fcab1538 100644
--- a/tensorflow/core/api_def/base_api/api_def_RepeatDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RepeatDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "RepeatDataset"
+  visibility: HIDDEN
   in_arg {
     name: "count"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt
index e83d4a9e96..f0dc1a3e0d 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "ScanDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset successively reduces `f` over the elements of `input_dataset`."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_ShuffleAndRepeatDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ShuffleAndRepeatDataset.pbtxt
index fb425b24a4..9ea1cc8bab 100644
--- a/tensorflow/core/api_def/base_api/api_def_ShuffleAndRepeatDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ShuffleAndRepeatDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "ShuffleAndRepeatDataset"
+  visibility: HIDDEN
   in_arg {
     name: "buffer_size"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ShuffleDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ShuffleDataset.pbtxt
index ea5c52c0ee..c7f4836a3a 100644
--- a/tensorflow/core/api_def/base_api/api_def_ShuffleDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ShuffleDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "ShuffleDataset"
+  visibility: HIDDEN
   in_arg {
     name: "buffer_size"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_SkipDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SkipDataset.pbtxt
index 44e5bac79b..f830049d05 100644
--- a/tensorflow/core/api_def/base_api/api_def_SkipDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SkipDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "SkipDataset"
+  visibility: HIDDEN
   in_arg {
     name: "count"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_SparseTensorSliceDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SparseTensorSliceDataset.pbtxt
index ffb8058349..4203eca73a 100644
--- a/tensorflow/core/api_def/base_api/api_def_SparseTensorSliceDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SparseTensorSliceDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "SparseTensorSliceDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that splits a SparseTensor into elements row-wise."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_TFRecordDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_TFRecordDataset.pbtxt
index 80f64cebb1..30e425794b 100644
--- a/tensorflow/core/api_def/base_api/api_def_TFRecordDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_TFRecordDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "TFRecordDataset"
+  visibility: HIDDEN
   in_arg {
     name: "filenames"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_TakeDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_TakeDataset.pbtxt
index 8808dc6b1f..eadcb6cd05 100644
--- a/tensorflow/core/api_def/base_api/api_def_TakeDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_TakeDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "TakeDataset"
+  visibility: HIDDEN
   in_arg {
     name: "count"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorDataset.pbtxt
index 050e174aac..c086d7420c 100644
--- a/tensorflow/core/api_def/base_api/api_def_TensorDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_TensorDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "TensorDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that emits `components` as a tuple of tensors once."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_TensorSliceDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorSliceDataset.pbtxt
index a26a98fd7f..30cb803b26 100644
--- a/tensorflow/core/api_def/base_api/api_def_TensorSliceDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_TensorSliceDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "TensorSliceDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that emits each dim-0 slice of `components` once."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_TextLineDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_TextLineDataset.pbtxt
index 6b63050996..31ef3e3335 100644
--- a/tensorflow/core/api_def/base_api/api_def_TextLineDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_TextLineDataset.pbtxt
@@ -1,5 +1,6 @@
 op {
   graph_op_name: "TextLineDataset"
+  visibility: HIDDEN
   in_arg {
     name: "filenames"
     description: <<END
diff --git a/tensorflow/core/api_def/base_api/api_def_ZipDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ZipDataset.pbtxt
index 7495693ccc..3c81996359 100644
--- a/tensorflow/core/api_def/base_api/api_def_ZipDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ZipDataset.pbtxt
@@ -1,4 +1,5 @@
 op {
   graph_op_name: "ZipDataset"
+  visibility: HIDDEN
   summary: "Creates a dataset that zips together `input_datasets`."
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_BatchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_BatchDataset.pbtxt
deleted file mode 100644
index 4289c1daf9..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_BatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "BatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_CacheDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_CacheDataset.pbtxt
deleted file mode 100644
index 2bbb4ff9e3..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_CacheDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "CacheDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ConcatenateDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ConcatenateDataset.pbtxt
deleted file mode 100644
index c005a4da0f..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ConcatenateDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ConcatenateDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_DatasetToSingleElement.pbtxt b/tensorflow/core/api_def/python_api/api_def_DatasetToSingleElement.pbtxt
deleted file mode 100644
index e3d34cc15b..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_DatasetToSingleElement.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "DatasetToSingleElement"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_EnqueueInQueueDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_EnqueueInQueueDataset.pbtxt
deleted file mode 100644
index 051cf14c0e..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_EnqueueInQueueDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "EnqueueInQueueDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_FilterDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_FilterDataset.pbtxt
deleted file mode 100644
index 6f91b84218..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_FilterDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "FilterDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDataset.pbtxt
deleted file mode 100644
index d0703471d3..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "FixedLengthRecordDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDatasetV2.pbtxt b/tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDatasetV2.pbtxt
deleted file mode 100644
index def9f85e02..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_FixedLengthRecordDatasetV2.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "FixedLengthRecordDatasetV2"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_FlatMapDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_FlatMapDataset.pbtxt
deleted file mode 100644
index 9de61ac263..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_FlatMapDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "FlatMapDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_GeneratorDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_GeneratorDataset.pbtxt
deleted file mode 100644
index 9dcfa0f7d2..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_GeneratorDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "GeneratorDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_InterleaveDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_InterleaveDataset.pbtxt
deleted file mode 100644
index ef1b06b19c..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_InterleaveDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "InterleaveDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_MapDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_MapDataset.pbtxt
deleted file mode 100644
index 0b1d2f2c73..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_MapDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "MapDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_PaddedBatchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_PaddedBatchDataset.pbtxt
deleted file mode 100644
index c6223b3132..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_PaddedBatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "PaddedBatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ParallelMapDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ParallelMapDataset.pbtxt
deleted file mode 100644
index 09d200dd24..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ParallelMapDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ParallelMapDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_PrefetchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_PrefetchDataset.pbtxt
deleted file mode 100644
index ec4e214eb5..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_PrefetchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "PrefetchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
deleted file mode 100644
index 228c4047d2..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "PrependFromQueueAndPaddedBatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_RandomDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_RandomDataset.pbtxt
deleted file mode 100644
index a5f6f8c6f1..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_RandomDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "RandomDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_RangeDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_RangeDataset.pbtxt
deleted file mode 100644
index 4cd8296b22..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_RangeDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "RangeDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_RepeatDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_RepeatDataset.pbtxt
deleted file mode 100644
index be301da838..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_RepeatDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "RepeatDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ScanDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ScanDataset.pbtxt
deleted file mode 100644
index e71b655c22..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ScanDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ScanDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ShuffleAndRepeatDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ShuffleAndRepeatDataset.pbtxt
deleted file mode 100644
index 7b0d2994f0..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ShuffleAndRepeatDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ShuffleAndRepeatDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ShuffleDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ShuffleDataset.pbtxt
deleted file mode 100644
index 8f0be9197a..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ShuffleDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ShuffleDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_SkipDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_SkipDataset.pbtxt
deleted file mode 100644
index 96a551c5b6..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_SkipDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "SkipDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_SparseTensorSliceDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_SparseTensorSliceDataset.pbtxt
deleted file mode 100644
index 19c0c7f199..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_SparseTensorSliceDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "SparseTensorSliceDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_TFRecordDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_TFRecordDataset.pbtxt
deleted file mode 100644
index 3c270ada3c..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_TFRecordDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "TFRecordDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_TakeDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_TakeDataset.pbtxt
deleted file mode 100644
index 711b335dc1..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_TakeDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "TakeDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorDataset.pbtxt
deleted file mode 100644
index 5bc3920c56..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_TensorDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "TensorDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorSliceDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorSliceDataset.pbtxt
deleted file mode 100644
index 89ad016483..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_TensorSliceDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "TensorSliceDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_TextLineDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_TextLineDataset.pbtxt
deleted file mode 100644
index 08d785191b..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_TextLineDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "TextLineDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/python_api/api_def_ZipDataset.pbtxt b/tensorflow/core/api_def/python_api/api_def_ZipDataset.pbtxt
deleted file mode 100644
index dd1459521f..0000000000
--- a/tensorflow/core/api_def/python_api/api_def_ZipDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "ZipDataset"
-  visibility: HIDDEN
-}
-- 
GitLab


From cdf821289eaa7d791ebef56ee9d72b058ed4b49f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 2 Dec 2018 12:18:32 -0800
Subject: [PATCH 1170/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 223713287
---
 .../core/ops/compat/ops_history.v1.pbtxt      |  120 ++
 tensorflow/core/ops/ops.pbtxt                 | 1514 ++++++++---------
 2 files changed, 853 insertions(+), 781 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index c8a5a9ef86..a8ad79dd01 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21660,6 +21660,37 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalDenseToSparseBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "row_shape"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalDirectedInterleaveDataset"
   input_arg {
@@ -21833,6 +21864,68 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalGroupByWindowDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "key_func_other_arguments"
+    type_list_attr: "Tkey_func_other_arguments"
+  }
+  input_arg {
+    name: "reduce_func_other_arguments"
+    type_list_attr: "Treduce_func_other_arguments"
+  }
+  input_arg {
+    name: "window_size_func_other_arguments"
+    type_list_attr: "Twindow_size_func_other_arguments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "key_func"
+    type: "func"
+  }
+  attr {
+    name: "reduce_func"
+    type: "func"
+  }
+  attr {
+    name: "window_size_func"
+    type: "func"
+  }
+  attr {
+    name: "Tkey_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Treduce_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Twindow_size_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalIdentityIndexedDataset"
   input_arg {
@@ -21944,6 +22037,33 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalLatencyStatsDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalMapAndBatchDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index f194d608be..8cfd84be88 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -4944,33 +4944,6 @@ op {
     type: "list(float)"
   }
 }
-op {
-  name: "BytesProducedStatsDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "tag"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "CTCBeamSearchDecoder"
   input_arg {
@@ -7915,21 +7888,6 @@ op {
     minimum: 1
   }
 }
-op {
-  name: "DatasetToTFRecord"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "filename"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "compression_type"
-    type: DT_STRING
-  }
-}
 op {
   name: "DebugGradientIdentity"
   input_arg {
@@ -8599,37 +8557,6 @@ op {
     }
   }
 }
-op {
-  name: "DenseToSparseBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "row_shape"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "DenseToSparseSetOperation"
   input_arg {
@@ -10089,6 +10016,33 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalBytesProducedStatsDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalCSVDataset"
   input_arg {
@@ -10155,15 +10109,33 @@ op {
   is_stateful: true
 }
 op {
-  name: "ExperimentalDirectedInterleaveDataset"
+  name: "ExperimentalDatasetToTFRecord"
   input_arg {
-    name: "selector_input_dataset"
+    name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "data_input_datasets"
+    name: "filename"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "compression_type"
+    type: DT_STRING
+  }
+}
+op {
+  name: "ExperimentalDenseToSparseBatchDataset"
+  input_arg {
+    name: "input_dataset"
     type: DT_VARIANT
-    number_attr: "N"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "row_shape"
+    type: DT_INT64
   }
   output_arg {
     name: "handle"
@@ -10181,30 +10153,17 @@ op {
     has_minimum: true
     minimum: 1
   }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 1
-  }
 }
 op {
-  name: "ExperimentalIdentityIndexedDataset"
+  name: "ExperimentalDirectedInterleaveDataset"
   input_arg {
-    name: "size"
-    type: DT_UINT64
-  }
-  output_arg {
-    name: "handle"
+    name: "selector_input_dataset"
     type: DT_VARIANT
   }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalIgnoreErrorsDataset"
   input_arg {
-    name: "input_dataset"
+    name: "data_input_datasets"
     type: DT_VARIANT
+    number_attr: "N"
   }
   output_arg {
     name: "handle"
@@ -10222,69 +10181,75 @@ op {
     has_minimum: true
     minimum: 1
   }
-}
-op {
-  name: "ExperimentalIndexedDatasetGet"
-  input_arg {
-    name: "materialized"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "index"
-    type: DT_UINT64
-  }
-  output_arg {
-    name: "components"
-    type_list_attr: "output_types"
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
+    name: "N"
+    type: "int"
     has_minimum: true
     minimum: 1
   }
-  is_stateful: true
 }
 op {
-  name: "ExperimentalIndexedDatasetMaterialize"
+  name: "ExperimentalGroupByReducerDataset"
   input_arg {
-    name: "dataset"
+    name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "materialized"
-    type: DT_RESOURCE
+    name: "key_func_other_arguments"
+    type_list_attr: "Tkey_func_other_arguments"
   }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalIteratorGetDevice"
   input_arg {
-    name: "resource"
-    type: DT_RESOURCE
+    name: "init_func_other_arguments"
+    type_list_attr: "Tinit_func_other_arguments"
   }
-  output_arg {
-    name: "device"
-    type: DT_STRING
+  input_arg {
+    name: "reduce_func_other_arguments"
+    type_list_attr: "Treduce_func_other_arguments"
   }
-  is_stateful: true
-}
-op {
-  name: "ExperimentalLMDBDataset"
   input_arg {
-    name: "filenames"
-    type: DT_STRING
+    name: "finalize_func_other_arguments"
+    type_list_attr: "Tfinalize_func_other_arguments"
   }
   output_arg {
     name: "handle"
     type: DT_VARIANT
   }
+  attr {
+    name: "key_func"
+    type: "func"
+  }
+  attr {
+    name: "init_func"
+    type: "func"
+  }
+  attr {
+    name: "reduce_func"
+    type: "func"
+  }
+  attr {
+    name: "finalize_func"
+    type: "func"
+  }
+  attr {
+    name: "Tkey_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tinit_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Treduce_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Tfinalize_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
   attr {
     name: "output_types"
     type: "list(type)"
@@ -10300,33 +10265,281 @@ op {
   is_stateful: true
 }
 op {
-  name: "ExperimentalMapDataset"
+  name: "ExperimentalGroupByWindowDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
+    name: "key_func_other_arguments"
+    type_list_attr: "Tkey_func_other_arguments"
+  }
+  input_arg {
+    name: "reduce_func_other_arguments"
+    type_list_attr: "Treduce_func_other_arguments"
+  }
+  input_arg {
+    name: "window_size_func_other_arguments"
+    type_list_attr: "Twindow_size_func_other_arguments"
   }
   output_arg {
     name: "handle"
     type: DT_VARIANT
   }
   attr {
-    name: "f"
+    name: "key_func"
     type: "func"
   }
   attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
+    name: "reduce_func"
+    type: "func"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "window_size_func"
+    type: "func"
+  }
+  attr {
+    name: "Tkey_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Treduce_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "Twindow_size_func_other_arguments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalIdentityIndexedDataset"
+  input_arg {
+    name: "size"
+    type: DT_UINT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalIgnoreErrorsDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalIndexedDatasetGet"
+  input_arg {
+    name: "materialized"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "index"
+    type: DT_UINT64
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "output_types"
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalIndexedDatasetMaterialize"
+  input_arg {
+    name: "dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "materialized"
+    type: DT_RESOURCE
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalIteratorGetDevice"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "device"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalLMDBDataset"
+  input_arg {
+    name: "filenames"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalLatencyStatsDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalMapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalMapDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
   }
   attr {
     name: "output_shapes"
@@ -10365,8 +10578,233 @@ op {
     type: "string"
   }
   attr {
-    name: "shared_name"
-    type: "string"
+    name: "shared_name"
+    type: "string"
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalMaxIntraOpParallelismDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "max_intra_op_parallelism"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalNonSerializableDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalNumaMapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalParallelInterleaveDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "cycle_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "block_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sloppy"
+    type: DT_BOOL
+  }
+  input_arg {
+    name: "buffer_output_elements"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "prefetch_input_elements"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ExperimentalParseExampleDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "dense_defaults"
+    type_list_attr: "Tdense"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "sparse_keys"
+    type: "list(string)"
+    has_minimum: true
+  }
+  attr {
+    name: "dense_keys"
+    type: "list(string)"
+    has_minimum: true
+  }
+  attr {
+    name: "sparse_types"
+    type: "list(type)"
+    has_minimum: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "Tdense"
+    type: "list(type)"
+    has_minimum: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "dense_shapes"
+    type: "list(shape)"
+    has_minimum: true
   }
   attr {
     name: "output_types"
@@ -10380,16 +10818,22 @@ op {
     has_minimum: true
     minimum: 1
   }
-  is_stateful: true
+  attr {
+    name: "sloppy"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "ExperimentalMaxIntraOpParallelismDataset"
+  name: "ExperimentalPrivateThreadPoolDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "max_intra_op_parallelism"
+    name: "num_threads"
     type: DT_INT64
   }
   output_arg {
@@ -10410,11 +10854,23 @@ op {
   }
 }
 op {
-  name: "ExperimentalNonSerializableDataset"
+  name: "ExperimentalSetStatsAggregatorDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "stats_aggregator"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "counter_prefix"
+    type: DT_STRING
+  }
   output_arg {
     name: "handle"
     type: DT_VARIANT
@@ -10431,42 +10887,22 @@ op {
     has_minimum: true
     minimum: 1
   }
+  is_stateful: true
 }
 op {
-  name: "ExperimentalNumaMapAndBatchDataset"
+  name: "ExperimentalSleepDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "num_parallel_calls"
+    name: "sleep_microseconds"
     type: DT_INT64
   }
-  input_arg {
-    name: "drop_remainder"
-    type: DT_BOOL
-  }
   output_arg {
     name: "handle"
     type: DT_VARIANT
   }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
   attr {
     name: "output_types"
     type: "list(type)"
@@ -10481,13 +10917,21 @@ op {
   }
 }
 op {
-  name: "ExperimentalPrivateThreadPoolDataset"
+  name: "ExperimentalSlidingWindowDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "num_threads"
+    name: "window_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "window_shift"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "window_stride"
     type: DT_INT64
   }
   output_arg {
@@ -10508,14 +10952,18 @@ op {
   }
 }
 op {
-  name: "ExperimentalSleepDataset"
+  name: "ExperimentalSqlDataset"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "driver_name"
+    type: DT_STRING
   }
   input_arg {
-    name: "sleep_microseconds"
-    type: DT_INT64
+    name: "data_source_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "query"
+    type: DT_STRING
   }
   output_arg {
     name: "handle"
@@ -10533,6 +10981,41 @@ op {
     has_minimum: true
     minimum: 1
   }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalStatsAggregatorHandle"
+  output_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalStatsAggregatorSummary"
+  input_arg {
+    name: "iterator"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "summary"
+    type: DT_STRING
+  }
+  is_stateful: true
 }
 op {
   name: "ExperimentalThreadPoolDataset"
@@ -10599,6 +11082,29 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalUnbatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalUniqueDataset"
   input_arg {
@@ -12749,184 +13255,46 @@ op {
         type: DT_INT64
         type: DT_BFLOAT16
         type: DT_UINT16
-        type: DT_HALF
-        type: DT_UINT32
-        type: DT_UINT64
-      }
-    }
-  }
-}
-op {
-  name: "GreaterEqual"
-  input_arg {
-    name: "x"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "y"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "z"
-    type: DT_BOOL
-  }
-  attr {
-    name: "T"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_INT64
-        type: DT_BFLOAT16
-        type: DT_UINT16
-        type: DT_HALF
-        type: DT_UINT32
-        type: DT_UINT64
-      }
-    }
-  }
-}
-op {
-  name: "GroupByReducerDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "key_func_other_arguments"
-    type_list_attr: "Tkey_func_other_arguments"
-  }
-  input_arg {
-    name: "init_func_other_arguments"
-    type_list_attr: "Tinit_func_other_arguments"
-  }
-  input_arg {
-    name: "reduce_func_other_arguments"
-    type_list_attr: "Treduce_func_other_arguments"
-  }
-  input_arg {
-    name: "finalize_func_other_arguments"
-    type_list_attr: "Tfinalize_func_other_arguments"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "key_func"
-    type: "func"
-  }
-  attr {
-    name: "init_func"
-    type: "func"
-  }
-  attr {
-    name: "reduce_func"
-    type: "func"
-  }
-  attr {
-    name: "finalize_func"
-    type: "func"
-  }
-  attr {
-    name: "Tkey_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Tinit_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Treduce_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Tfinalize_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "GroupByWindowDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "key_func_other_arguments"
-    type_list_attr: "Tkey_func_other_arguments"
-  }
+  name: "GreaterEqual"
   input_arg {
-    name: "reduce_func_other_arguments"
-    type_list_attr: "Treduce_func_other_arguments"
+    name: "x"
+    type_attr: "T"
   }
   input_arg {
-    name: "window_size_func_other_arguments"
-    type_list_attr: "Twindow_size_func_other_arguments"
+    name: "y"
+    type_attr: "T"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "key_func"
-    type: "func"
-  }
-  attr {
-    name: "reduce_func"
-    type: "func"
-  }
-  attr {
-    name: "window_size_func"
-    type: "func"
-  }
-  attr {
-    name: "Tkey_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Treduce_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "Twindow_size_func_other_arguments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "z"
+    type: DT_BOOL
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_INT64
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
 }
 op {
@@ -14469,33 +14837,6 @@ op {
     }
   }
 }
-op {
-  name: "LatencyStatsDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "tag"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "LeakyRelu"
   input_arg {
@@ -15366,102 +15707,6 @@ op {
   }
   is_stateful: true
 }
-op {
-  name: "MapAndBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "num_parallel_batches"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "drop_remainder"
-    type: DT_BOOL
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
-op {
-  name: "MapAndBatchDatasetV2"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "num_parallel_calls"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "drop_remainder"
-    type: DT_BOOL
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "MapClear"
   attr {
@@ -19461,83 +19706,27 @@ op {
   name: "ParallelDynamicStitch"
   input_arg {
     name: "indices"
-    type: DT_INT32
-    number_attr: "N"
-  }
-  input_arg {
-    name: "data"
-    type_attr: "T"
-    number_attr: "N"
-  }
-  output_arg {
-    name: "merged"
-    type_attr: "T"
-  }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "T"
-    type: "type"
-  }
-}
-op {
-  name: "ParallelInterleaveDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "cycle_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "block_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "sloppy"
-    type: DT_BOOL
-  }
-  input_arg {
-    name: "buffer_output_elements"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "prefetch_input_elements"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    type: DT_INT32
+    number_attr: "N"
   }
-  attr {
-    name: "f"
-    type: "func"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+    number_attr: "N"
   }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
+  output_arg {
+    name: "merged"
+    type_attr: "T"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
+    name: "N"
+    type: "int"
     has_minimum: true
     minimum: 1
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
   }
 }
 op {
@@ -19795,83 +19984,6 @@ op {
     has_minimum: true
   }
 }
-op {
-  name: "ParseExampleDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "num_parallel_calls"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "dense_defaults"
-    type_list_attr: "Tdense"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "sparse_keys"
-    type: "list(string)"
-    has_minimum: true
-  }
-  attr {
-    name: "dense_keys"
-    type: "list(string)"
-    has_minimum: true
-  }
-  attr {
-    name: "sparse_types"
-    type: "list(type)"
-    has_minimum: true
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_INT64
-        type: DT_STRING
-      }
-    }
-  }
-  attr {
-    name: "Tdense"
-    type: "list(type)"
-    has_minimum: true
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_INT64
-        type: DT_STRING
-      }
-    }
-  }
-  attr {
-    name: "dense_shapes"
-    type: "list(shape)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "sloppy"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
-}
 op {
   name: "ParseSequenceExample"
   input_arg {
@@ -29327,42 +29439,6 @@ op {
     }
   }
 }
-op {
-  name: "SetStatsAggregatorDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "stats_aggregator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "tag"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "counter_prefix"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "Shape"
   input_arg {
@@ -29826,41 +29902,6 @@ op {
     }
   }
 }
-op {
-  name: "SlideDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "window_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "window_shift"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "window_stride"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "Snapshot"
   input_arg {
@@ -33051,38 +33092,6 @@ op {
     }
   }
 }
-op {
-  name: "SqlDataset"
-  input_arg {
-    name: "driver_name"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "data_source_name"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "query"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "Sqrt"
   input_arg {
@@ -33975,40 +33984,6 @@ op {
     }
   }
 }
-op {
-  name: "StatsAggregatorHandle"
-  output_arg {
-    name: "handle"
-    type: DT_RESOURCE
-  }
-  attr {
-    name: "container"
-    type: "string"
-    default_value {
-      s: ""
-    }
-  }
-  attr {
-    name: "shared_name"
-    type: "string"
-    default_value {
-      s: ""
-    }
-  }
-  is_stateful: true
-}
-op {
-  name: "StatsAggregatorSummary"
-  input_arg {
-    name: "iterator"
-    type: DT_RESOURCE
-  }
-  output_arg {
-    name: "summary"
-    type: DT_STRING
-  }
-  is_stateful: true
-}
 op {
   name: "StopGradient"
   input_arg {
@@ -37137,29 +37112,6 @@ op {
     type: "type"
   }
 }
-op {
-  name: "UnbatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "UnbatchGrad"
   input_arg {
-- 
GitLab


From 0d6234717c962dc82d24192f3f2e36b62bfac810 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Sun, 2 Dec 2018 12:26:01 -0800
Subject: [PATCH 1171/1554] Detect functions that are not convertible because
 their source code is inaccessible. Whitelist namedtuple objects for calling
 without conversion.

PiperOrigin-RevId: 223713634
---
 .../python/autograph/converters/call_trees.py | 17 +++++++++--
 .../autograph/converters/call_trees_test.py   | 16 ++++++++++
 .../python/autograph/impl/conversion.py       | 14 +++++++++
 .../python/autograph/pyct/inspect_utils.py    | 14 +++++++++
 .../autograph/pyct/inspect_utils_test.py      | 30 +++++++++++++++++--
 tensorflow/python/eager/function_test.py      |  3 +-
 6 files changed, 88 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/autograph/converters/call_trees.py b/tensorflow/python/autograph/converters/call_trees.py
index 4a8af42e69..9b85fc8367 100644
--- a/tensorflow/python/autograph/converters/call_trees.py
+++ b/tensorflow/python/autograph/converters/call_trees.py
@@ -22,7 +22,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from collections import namedtuple
+import collections
 
 import gast
 
@@ -35,7 +35,7 @@ from tensorflow.python.autograph.pyct import templates
 from tensorflow.python.util import tf_inspect
 
 
-class FunctionInfo(namedtuple('FunctionInfo', ('dtype',))):
+class FunctionInfo(collections.namedtuple('FunctionInfo', ('dtype',))):
   pass
 
 
@@ -116,12 +116,19 @@ class CallTreeTransformer(converter.Base):
   def _function_is_compilable(self, target_entity):
     """Determines whether an entity can be compiled at all."""
     # TODO(mdan): Expand.
+
     if target_entity.__module__ is None:
       # Functions like builtins and NumPy don't expose a module.
       # Those in general should not be compiled.
       return False
+
     if inspect_utils.isbuiltin(target_entity):
       return False
+
+    if inspect_utils.isnamedtuple(target_entity):
+      # namedtuple doesn't expose its source code, making it uncompilable.
+      return False
+
     return True
 
   def _should_compile(self, node, fqn):
@@ -301,7 +308,13 @@ class CallTreeTransformer(converter.Base):
         # safe for graph mode.
         return node
 
+      elif inspect_utils.isnamedtuple(target_entity):
+        # Although not compilable, we assume they are safe for graph mode.
+        node = self.generic_visit(node)
+        return node
+
       else:
+        # TODO(mdan): Instert dynamic conversion here instead.
         raise NotImplementedError(
             'py_func with return values (unknown function)')
     else:
diff --git a/tensorflow/python/autograph/converters/call_trees_test.py b/tensorflow/python/autograph/converters/call_trees_test.py
index b2a740020c..454d75d755 100644
--- a/tensorflow/python/autograph/converters/call_trees_test.py
+++ b/tensorflow/python/autograph/converters/call_trees_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
+
 import numpy as np
 
 from tensorflow.python.autograph.converters import call_trees
@@ -99,6 +101,20 @@ class CallTreesTest(converter_testing.TestCase):
     with self.compiled(node, ns) as result:
       self.assertEquals(1, result.test_fn(1))
 
+  def test_known_called_namedtuple(self):
+
+    nt = collections.namedtuple('TestNamedTuple', ['a'])
+
+    def test_fn(a):
+      return nt(a)
+
+    ns = {'nt': nt}
+    node, ctx = self.prepare(test_fn, ns)
+    node = call_trees.transform(node, ctx)
+
+    with self.compiled(node, ns) as result:
+      self.assertEquals(nt(1), result.test_fn(1))
+
   def test_py_func_known_function(self):
 
     def test_fn():
diff --git a/tensorflow/python/autograph/impl/conversion.py b/tensorflow/python/autograph/impl/conversion.py
index 055769c73a..f8decd24e8 100644
--- a/tensorflow/python/autograph/impl/conversion.py
+++ b/tensorflow/python/autograph/impl/conversion.py
@@ -73,6 +73,7 @@ def is_whitelisted_for_graph(o):
   Returns:
     Boolean
   """
+  # TODO(b/120224672): Fix this.
   if isinstance(o, functools.partial):
     # tf_inspect.getmodule(functools.partial(...)) otherwise returns None since
     # functools.partial objects do not have a __module__ attribute.
@@ -82,8 +83,21 @@ def is_whitelisted_for_graph(o):
   for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
     if m.__name__.startswith(prefix):
       return True
+
   if hasattr(o, 'autograph_info__'):
     return True
+
+  if inspect_utils.isnamedtuple(o):
+    # Due to the way they're constructed, namedtuple types cannot be converted
+    # because they don't expose source code. But we assume they are safe for
+    # graph mode since they are just containers.
+    if tf_inspect.isclass(o) and len(o.__bases__) > 1:
+      logging.log_first_n(
+          logging.level_warning(),
+          'Entity {} looks like a namedtuple subclass. If it has any custom'
+          ' methods, they will not be converted by AutoGraph.'.format(o), 1)
+    return True
+
   return False
 
 
diff --git a/tensorflow/python/autograph/pyct/inspect_utils.py b/tensorflow/python/autograph/pyct/inspect_utils.py
index 2319430d09..7c819f364f 100644
--- a/tensorflow/python/autograph/pyct/inspect_utils.py
+++ b/tensorflow/python/autograph/pyct/inspect_utils.py
@@ -54,6 +54,20 @@ def islambda(f):
   return f.__name__ == '<lambda>'
 
 
+def isnamedtuple(f):
+  """Returns True if the argument is a namedtuple-like."""
+  if not (tf_inspect.isclass(f) and issubclass(f, tuple)):
+    return False
+  if not hasattr(f, '_fields'):
+    return False
+  fields = getattr(f, '_fields')
+  if not isinstance(fields, tuple):
+    return False
+  if not all(isinstance(f, str) for f in fields):
+    return False
+  return True
+
+
 def isbuiltin(f):
   """Returns True if the argument is a built-in function."""
   if f in SPECIAL_BUILTINS.values():
diff --git a/tensorflow/python/autograph/pyct/inspect_utils_test.py b/tensorflow/python/autograph/pyct/inspect_utils_test.py
index 7e8466d58e..a2c39056d1 100644
--- a/tensorflow/python/autograph/pyct/inspect_utils_test.py
+++ b/tensorflow/python/autograph/pyct/inspect_utils_test.py
@@ -18,7 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from functools import wraps
+import collections
+import functools
 import imp
 import types
 import weakref
@@ -46,7 +47,7 @@ def wrapping_decorator():
     def replacement(*_):
       return None
 
-    @wraps(f)
+    @functools.wraps(f)
     def wrapper(*args, **kwargs):
       return replacement(*args, **kwargs)
     return wrapper
@@ -102,6 +103,31 @@ class InspectUtilsTest(test.TestCase):
     self.assertTrue(inspect_utils.islambda(lambda x: x))
     self.assertFalse(inspect_utils.islambda(test_fn))
 
+  def test_isnamedtuple(self):
+    nt = collections.namedtuple('TestNamedTuple', ['a', 'b'])
+
+    class NotANamedTuple(tuple):
+      pass
+
+    self.assertTrue(inspect_utils.isnamedtuple(nt))
+    self.assertFalse(inspect_utils.isnamedtuple(NotANamedTuple))
+
+  def test_isnamedtuple_confounder(self):
+    """This test highlights false positives when detecting named tuples."""
+
+    class NamedTupleLike(tuple):
+      _fields = ('a', 'b')
+
+    self.assertTrue(inspect_utils.isnamedtuple(NamedTupleLike))
+
+  def test_isnamedtuple_subclass(self):
+    """This test highlights false positives when detecting named tuples."""
+
+    class NamedTupleSubclass(collections.namedtuple('Test', ['a', 'b'])):
+      pass
+
+    self.assertTrue(inspect_utils.isnamedtuple(NamedTupleSubclass))
+
   def test_getnamespace_globals(self):
     ns = inspect_utils.getnamespace(factory)
     self.assertEqual(ns['free_function'], free_function)
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 2f3eff6d6a..8d24bcc0fe 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -152,8 +152,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
     pair = collections.namedtuple('pair', ['a', 'b'])
 
-    # TODO(b/120222989) remove autograph=False.
-    @def_function.function(autograph=False)
+    @def_function.function()
     def pairs_mul(pair_a, pair_b):
       return pair(matmul(pair_a.a, pair_b.a), matmul(pair_a.b, pair_b.b))
 
-- 
GitLab


From bd962d8cdfcda01a23c7051fa05e3db86dd9c30f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 2 Dec 2018 12:45:50 -0800
Subject: [PATCH 1172/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 223714609

---
 tensorflow/go/op/wrappers.go | 1785 +++++++++++++++-------------------
 1 file changed, 763 insertions(+), 1022 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index b0ed1b151f..69de0269ff 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5645,77 +5645,6 @@ func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size
 	return op.Output(0)
 }
 
-// MapUnstageAttr is an optional argument to MapUnstage.
-type MapUnstageAttr func(optionalAttr)
-
-// MapUnstageCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageCapacity(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageContainer(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageSharedName(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the values associated with the key
-//
-// from the underlying container.   If the underlying container
-// does not contain this key, the op will block until it does.
-func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapUnstage",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstage", err)
-		return
-	}
-	return values
-}
-
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -5781,34 +5710,6 @@ func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// Creates a dataset that passes a sliding window over `input_dataset`.
-//
-// Arguments:
-//
-//	window_size: A scalar representing the number of elements in the
-// sliding window.
-//	window_shift: A scalar representing the steps moving the sliding window
-// forward in one iteration. It must be positive.
-//	window_stride: A scalar representing the stride of the input elements of the sliding window.
-// It must be positive.
-//
-//
-func SlideDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, window_shift tf.Output, window_stride tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "SlideDataset",
-		Input: []tf.Input{
-			input_dataset, window_size, window_shift, window_stride,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // EditDistanceAttr is an optional argument to EditDistance.
 type EditDistanceAttr func(optionalAttr)
 
@@ -6448,6 +6349,71 @@ func Asin(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// SparseToDenseAttr is an optional argument to SparseToDense.
+type SparseToDenseAttr func(optionalAttr)
+
+// SparseToDenseValidateIndices sets the optional validate_indices attribute to value.
+//
+// value: If true, indices are checked to make sure they are sorted in
+// lexicographic order and that there are no repeats.
+// If not specified, defaults to true
+func SparseToDenseValidateIndices(value bool) SparseToDenseAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
+	}
+}
+
+// Converts a sparse representation into a dense tensor.
+//
+// Builds an array `dense` with shape `output_shape` such that
+//
+// ```
+// # If sparse_indices is scalar
+// dense[i] = (i == sparse_indices ? sparse_values : default_value)
+//
+// # If sparse_indices is a vector, then for each i
+// dense[sparse_indices[i]] = sparse_values[i]
+//
+// # If sparse_indices is an n by d matrix, then for each i in [0, n)
+// dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i]
+// ```
+//
+// All other values in `dense` are set to `default_value`.  If `sparse_values` is a
+// scalar, all sparse indices are set to this single value.
+//
+// Indices should be sorted in lexicographic order, and indices must not
+// contain any repeats. If `validate_indices` is true, these properties
+// are checked during execution.
+//
+// Arguments:
+//	sparse_indices: 0-D, 1-D, or 2-D.  `sparse_indices[i]` contains the complete
+// index where `sparse_values[i]` will be placed.
+//	output_shape: 1-D.  Shape of the dense output tensor.
+//	sparse_values: 1-D.  Values corresponding to each row of `sparse_indices`,
+// or a scalar value to be used for all sparse indices.
+//	default_value: Scalar value to set for indices not specified in
+// `sparse_indices`.
+//
+// Returns Dense output tensor of shape `output_shape`.
+func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Output, sparse_values tf.Output, default_value tf.Output, optional ...SparseToDenseAttr) (dense tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseToDense",
+		Input: []tf.Input{
+			sparse_indices, output_shape, sparse_values, default_value,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the sum along sparse segments of a tensor.
 //
 // Like `SparseSegmentSum`, but allows missing ids in `segment_ids`. If an id is
@@ -6983,61 +6949,6 @@ func Reciprocal(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// ParseExampleDatasetAttr is an optional argument to ParseExampleDataset.
-type ParseExampleDatasetAttr func(optionalAttr)
-
-// ParseExampleDatasetSloppy sets the optional sloppy attribute to value.
-// If not specified, defaults to false
-func ParseExampleDatasetSloppy(value bool) ParseExampleDatasetAttr {
-	return func(m optionalAttr) {
-		m["sloppy"] = value
-	}
-}
-
-// Transforms `input_dataset` containing `Example` protos as vectors of DT_STRING into a dataset of `Tensor` or `SparseTensor` objects representing the parsed features.
-//
-// Arguments:
-//
-//
-//	dense_defaults: A dict mapping string keys to `Tensor`s.
-// The keys of the dict must match the dense_keys of the feature.
-//	sparse_keys: A list of string keys in the examples features.
-// The results for these keys will be returned as `SparseTensor` objects.
-//	dense_keys: A list of Ndense string Tensors (scalars).
-// The keys expected in the Examples features associated with dense values.
-//	sparse_types: A list of `DTypes` of the same length as `sparse_keys`.
-// Only `tf.float32` (`FloatList`), `tf.int64` (`Int64List`),
-// and `tf.string` (`BytesList`) are supported.
-//	dense_shapes: List of tuples with the same length as `dense_keys`.
-// The shape of the data for each dense feature referenced by `dense_keys`.
-// Required for any input tensors identified by `dense_keys`.  Must be
-// either fully defined, or may contain an unknown first dimension.
-// An unknown first dimension means the feature is treated as having
-// a variable number of blocks, and the output shape along this dimension
-// is considered unknown at graph build time.  Padding is applied for
-// minibatch elements smaller than the maximum number of blocks for the
-// given feature along this dimension.
-//	output_types: The type list for the return values.
-//	output_shapes: The list of shapes being produced.
-func ParseExampleDataset(scope *Scope, input_dataset tf.Output, num_parallel_calls tf.Output, dense_defaults []tf.Output, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ParseExampleDatasetAttr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes, "output_types": output_types, "output_shapes": output_shapes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ParseExampleDataset",
-		Input: []tf.Input{
-			input_dataset, num_parallel_calls, tf.OutputList(dense_defaults),
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Returns a batched matrix tensor with new batched diagonal values.
 //
 // Given `input` and `diagonal`, this operation returns a tensor with the
@@ -8110,28 +8021,6 @@ func LeakyRelu(scope *Scope, features tf.Output, optional ...LeakyReluAttr) (act
 	return op.Output(0)
 }
 
-// Writes the given dataset to the given file using the TFRecord format.
-//
-// Arguments:
-//	input_dataset: A variant tensor representing the dataset to write.
-//	filename: A scalar string tensor representing the filename to use.
-//	compression_type: A scalar string tensor containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-//
-// Returns the created operation.
-func DatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "DatasetToTFRecord",
-		Input: []tf.Input{
-			input_dataset, filename, compression_type,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Computes rectified linear 6: `min(max(features, 0), 6)`.
 func Relu6(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
@@ -9738,6 +9627,35 @@ func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, opt
 	return output
 }
 
+// Convert JSON-encoded Example records to binary protocol buffer strings.
+//
+// This op translates a tensor containing Example records, encoded using
+// the [standard JSON
+// mapping](https://developers.google.com/protocol-buffers/docs/proto3#json),
+// into a tensor containing the same records encoded as binary protocol
+// buffers. The resulting tensor can then be fed to any of the other
+// Example-parsing ops.
+//
+// Arguments:
+//	json_examples: Each string is a JSON object serialized according to the JSON
+// mapping of the Example proto.
+//
+// Returns Each string is a binary Example protocol buffer corresponding
+// to the respective element of `json_examples`.
+func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodeJSONExample",
+		Input: []tf.Input{
+			json_examples,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Transforms a Tensor into a serialized TensorProto proto.
 //
 // Arguments:
@@ -11714,81 +11632,21 @@ func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, opti
 	return op.Output(0)
 }
 
-// RandomUniformIntAttr is an optional argument to RandomUniformInt.
-type RandomUniformIntAttr func(optionalAttr)
-
-// RandomUniformIntSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
+// FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D.
+type FusedResizeAndPadConv2DAttr func(optionalAttr)
 
-// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
+// FusedResizeAndPadConv2DResizeAlignCorners sets the optional resize_align_corners attribute to value.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+// value: If true, the centers of the 4 corner pixels of the input and output tensors are
+// aligned, preserving the values at the corner pixels. Defaults to false.
+// If not specified, defaults to false
+func FusedResizeAndPadConv2DResizeAlignCorners(value bool) FusedResizeAndPadConv2DAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["resize_align_corners"] = value
 	}
 }
 
-// Outputs random integers from a uniform distribution.
-//
-// The generated values are uniform integers in the range `[minval, maxval)`.
-// The lower bound `minval` is included in the range, while the upper bound
-// `maxval` is excluded.
-//
-// The random integers are slightly biased unless `maxval - minval` is an exact
-// power of two.  The bias is small for values of `maxval - minval` significantly
-// smaller than the range of the output (either `2^32` or `2^64`).
-//
-// Arguments:
-//	shape: The shape of the output tensor.
-//	minval: 0-D.  Inclusive lower bound on the generated integers.
-//	maxval: 0-D.  Exclusive upper bound on the generated integers.
-//
-// Returns A tensor of the specified shape filled with uniform random integers.
-func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "RandomUniformInt",
-		Input: []tf.Input{
-			shape, minval, maxval,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D.
-type FusedResizeAndPadConv2DAttr func(optionalAttr)
-
-// FusedResizeAndPadConv2DResizeAlignCorners sets the optional resize_align_corners attribute to value.
-//
-// value: If true, the centers of the 4 corner pixels of the input and output tensors are
-// aligned, preserving the values at the corner pixels. Defaults to false.
-// If not specified, defaults to false
-func FusedResizeAndPadConv2DResizeAlignCorners(value bool) FusedResizeAndPadConv2DAttr {
-	return func(m optionalAttr) {
-		m["resize_align_corners"] = value
-	}
-}
-
-// Performs a resize and padding as a preprocess during a convolution.
+// Performs a resize and padding as a preprocess during a convolution.
 //
 // It's often possible to do spatial transformations more efficiently as part of
 // the packing stage of a convolution, so this op allows for an optimized
@@ -15276,75 +15134,6 @@ func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.O
 	return scope.AddOperation(opspec)
 }
 
-// Return the shape of s0 op s1 with broadcast.
-//
-// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
-// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
-func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BroadcastArgs",
-		Input: []tf.Input{
-			s0, s1,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
-type DataFormatDimMapAttr func(optionalAttr)
-
-// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
-//
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
-	return func(m optionalAttr) {
-		m["src_format"] = value
-	}
-}
-
-// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
-//
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
-	return func(m optionalAttr) {
-		m["dst_format"] = value
-	}
-}
-
-// Returns the dimension index in the destination data format given the one in
-//
-// the source data format.
-//
-// Arguments:
-//	x: A Tensor with each element as a dimension index in source data format.
-// Must be in the range [-4, 4).
-//
-// Returns A Tensor with each element as a dimension index in destination data format.
-func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DataFormatDimMap",
-		Input: []tf.Input{
-			x,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Retrieves the tree ensemble resource stamp token, number of trees and growing statistics.
 //
 // Arguments:
@@ -18575,23 +18364,6 @@ func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Ou
 	return op.Output(0), op.Output(1)
 }
 
-// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
-func BytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "BytesProducedStatsDataset",
-		Input: []tf.Input{
-			input_dataset, tag,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Check if the input matches the regex pattern.
 //
 // The input is a string tensor of any shape. The pattern is the
@@ -18707,6 +18479,29 @@ func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output
 	return op.Output(0)
 }
 
+// Adds a value to the current value of a variable.
+//
+// Any ReadVariableOp with a control dependency on this op is guaranteed to
+// see the incremented value or a subsequent newer one.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value by which the variable will be incremented.
+//
+// Returns the created operation.
+func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "AssignAddVariableOp",
+		Input: []tf.Input{
+			resource, value,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // QuantizedReluAttr is an optional argument to QuantizedRelu.
 type QuantizedReluAttr func(optionalAttr)
 
@@ -22417,6 +22212,34 @@ func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, option
 	return op.Output(0)
 }
 
+// Saves tensors in V2 checkpoint format.
+//
+// By default, saves the named tensors in full.  If the caller wishes to save
+// specific slices of full tensors, "shape_and_slices" should be non-empty strings
+// and correspondingly well-formed.
+//
+// Arguments:
+//	prefix: Must have a single element. The prefix of the V2 checkpoint to which we
+// write the tensors.
+//	tensor_names: shape {N}. The names of the tensors to be saved.
+//	shape_and_slices: shape {N}.  The slice specs of the tensors to be saved.
+// Empty strings indicate that they are non-partitioned tensors.
+//	tensors: `N` tensors to save.
+//
+// Returns the created operation.
+func SaveV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, tensors []tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SaveV2",
+		Input: []tf.Input{
+			prefix, tensor_names, shape_and_slices, tf.OutputList(tensors),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // UnicodeTranscodeAttr is an optional argument to UnicodeTranscode.
 type UnicodeTranscodeAttr func(optionalAttr)
 
@@ -25198,6 +25021,75 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (
 	return op.Output(0)
 }
 
+// Return the shape of s0 op s1 with broadcast.
+//
+// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
+// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
+func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BroadcastArgs",
+		Input: []tf.Input{
+			s0, s1,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
+type DataFormatDimMapAttr func(optionalAttr)
+
+// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
+//
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["src_format"] = value
+	}
+}
+
+// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
+//
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
+	}
+}
+
+// Returns the dimension index in the destination data format given the one in
+//
+// the source data format.
+//
+// Arguments:
+//	x: A Tensor with each element as a dimension index in source data format.
+// Must be in the range [-4, 4).
+//
+// Returns A Tensor with each element as a dimension index in destination data format.
+func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DataFormatDimMap",
+		Input: []tf.Input{
+			x,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // CumprodAttr is an optional argument to Cumprod.
 type CumprodAttr func(optionalAttr)
 
@@ -27530,39 +27422,10 @@ func MatrixSquareRoot(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Convert JSON-encoded Example records to binary protocol buffer strings.
-//
-// This op translates a tensor containing Example records, encoded using
-// the [standard JSON
-// mapping](https://developers.google.com/protocol-buffers/docs/proto3#json),
-// into a tensor containing the same records encoded as binary protocol
-// buffers. The resulting tensor can then be fed to any of the other
-// Example-parsing ops.
-//
-// Arguments:
-//	json_examples: Each string is a JSON object serialized according to the JSON
-// mapping of the Example proto.
-//
-// Returns Each string is a binary Example protocol buffer corresponding
-// to the respective element of `json_examples`.
-func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeJSONExample",
-		Input: []tf.Input{
-			json_examples,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SvdAttr is an optional argument to Svd.
-type SvdAttr func(optionalAttr)
-
-// SvdComputeUv sets the optional compute_uv attribute to value.
+// SvdAttr is an optional argument to Svd.
+type SvdAttr func(optionalAttr)
+
+// SvdComputeUv sets the optional compute_uv attribute to value.
 //
 // value: If true, left and right singular vectors will be
 // computed and returned in `u` and `v`, respectively.
@@ -28595,6 +28458,49 @@ func ReaderReadUpToV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Out
 	return op.Output(0), op.Output(1)
 }
 
+//     Adds v into specified rows of x.
+//
+//     Computes y = x; y[i, :] += v; return y.
+//
+// Arguments:
+//	x: A `Tensor` of type T.
+//	i: A vector. Indices into the left-most dimension of `x`.
+//	v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size.
+//
+// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`.
+func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "InplaceAdd",
+		Input: []tf.Input{
+			x, i, v,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Restore a Reader to its initial clean state.
+//
+// Arguments:
+//	reader_handle: Handle to a Reader.
+//
+// Returns the created operation.
+func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReaderResetV2",
+		Input: []tf.Input{
+			reader_handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // BatchAttr is an optional argument to Batch.
 type BatchAttr func(optionalAttr)
 
@@ -29880,71 +29786,6 @@ func CropAndResizeGradBoxes(scope *Scope, grads tf.Output, image tf.Output, boxe
 	return op.Output(0)
 }
 
-// Saves tensors in V2 checkpoint format.
-//
-// By default, saves the named tensors in full.  If the caller wishes to save
-// specific slices of full tensors, "shape_and_slices" should be non-empty strings
-// and correspondingly well-formed.
-//
-// Arguments:
-//	prefix: Must have a single element. The prefix of the V2 checkpoint to which we
-// write the tensors.
-//	tensor_names: shape {N}. The names of the tensors to be saved.
-//	shape_and_slices: shape {N}.  The slice specs of the tensors to be saved.
-// Empty strings indicate that they are non-partitioned tensors.
-//	tensors: `N` tensors to save.
-//
-// Returns the created operation.
-func SaveV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, tensors []tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SaveV2",
-		Input: []tf.Input{
-			prefix, tensor_names, shape_and_slices, tf.OutputList(tensors),
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// StatsAggregatorHandleAttr is an optional argument to StatsAggregatorHandle.
-type StatsAggregatorHandleAttr func(optionalAttr)
-
-// StatsAggregatorHandleContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StatsAggregatorHandleContainer(value string) StatsAggregatorHandleAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// StatsAggregatorHandleSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StatsAggregatorHandleSharedName(value string) StatsAggregatorHandleAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Creates a statistics manager resource.
-func StatsAggregatorHandle(scope *Scope, optional ...StatsAggregatorHandleAttr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StatsAggregatorHandle",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Greedily selects a subset of bounding boxes in descending order of score,
 //
 // pruning away boxes that have high intersection-over-union (IOU) overlap
@@ -30124,6 +29965,46 @@ func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Outpu
 	return op.Output(0)
 }
 
+// Computes the gradient for the inverse of `x` wrt its input.
+//
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "InvGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// List of the given size with empty elements.
+//
+// element_shape: the shape of the future elements of the list
+// num_elements: the number of elements to reserve
+// handle: the output list
+// element_dtype: the desired type of elements in the list.
+func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Output, element_dtype tf.DataType) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	opspec := tf.OpSpec{
+		Type: "TensorListReserve",
+		Input: []tf.Input{
+			element_shape, num_elements,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // A substitute for `InterleaveDataset` on a fixed list of `N` datasets.
 //
 // Arguments:
@@ -30149,6 +30030,66 @@ func ExperimentalDirectedInterleaveDataset(scope *Scope, selector_input_dataset
 	return op.Output(0)
 }
 
+// RandomUniformIntAttr is an optional argument to RandomUniformInt.
+type RandomUniformIntAttr func(optionalAttr)
+
+// RandomUniformIntSeed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random integers from a uniform distribution.
+//
+// The generated values are uniform integers in the range `[minval, maxval)`.
+// The lower bound `minval` is included in the range, while the upper bound
+// `maxval` is excluded.
+//
+// The random integers are slightly biased unless `maxval - minval` is an exact
+// power of two.  The bias is small for values of `maxval - minval` significantly
+// smaller than the range of the output (either `2^32` or `2^64`).
+//
+// Arguments:
+//	shape: The shape of the output tensor.
+//	minval: 0-D.  Inclusive lower bound on the generated integers.
+//	maxval: 0-D.  Exclusive upper bound on the generated integers.
+//
+// Returns A tensor of the specified shape filled with uniform random integers.
+func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomUniformInt",
+		Input: []tf.Input{
+			shape, minval, maxval,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Add the quantile summaries to each quantile stream resource.
 //
 // An op that adds a list of quantile summaries to a quantile stream resource. Each
@@ -30445,424 +30386,24 @@ func RealDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-//     Adds v into specified rows of x.
-//
-//     Computes y = x; y[i, :] += v; return y.
-//
-// Arguments:
-//	x: A `Tensor` of type T.
-//	i: A vector. Indices into the left-most dimension of `x`.
-//	v: A `Tensor` of type T. Same dimension sizes as x except the first dimension, which must be the same as i's size.
-//
-// Returns A `Tensor` of type T. An alias of `x`. The content of `y` is undefined if there are duplicates in `i`.
-func InplaceAdd(scope *Scope, x tf.Output, i tf.Output, v tf.Output) (y tf.Output) {
+// Creates a dataset that concatenates `input_dataset` with `another_dataset`.
+func ConcatenateDataset(scope *Scope, input_dataset tf.Output, another_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "InplaceAdd",
+		Type: "ConcatenateDataset",
 		Input: []tf.Input{
-			x, i, v,
+			input_dataset, another_dataset,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Restore a Reader to its initial clean state.
-//
-// Arguments:
-//	reader_handle: Handle to a Reader.
-//
-// Returns the created operation.
-func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReaderResetV2",
-		Input: []tf.Input{
-			reader_handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// A dataset that splits the elements of its input into multiple elements.
-func UnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "UnbatchDataset",
-		Input: []tf.Input{
-			input_dataset,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// OrderedMapStageAttr is an optional argument to OrderedMapStage.
-type OrderedMapStageAttr func(optionalAttr)
-
-// OrderedMapStageCapacity sets the optional capacity attribute to value.
-//
-// value: Maximum number of elements in the Staging Area. If > 0, inserts
-// on the container will block when the capacity is reached.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapStageContainer sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container. Otherwise,
-// a default container is used.
-// If not specified, defaults to ""
-func OrderedMapStageContainer(value string) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// OrderedMapStageSharedName sets the optional shared_name attribute to value.
-//
-// value: It is necessary to match this name to the matching Unstage Op.
-// If not specified, defaults to ""
-func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Stage (key, values) in the underlying container which behaves like a ordered
-//
-// associative container.   Elements are ordered by key.
-//
-// Arguments:
-//	key: int64
-//
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
-//
-//
-// Returns the created operation.
-func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "OrderedMapStage",
-		Input: []tf.Input{
-			key, indices, tf.OutputList(values),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// RpcAttr is an optional argument to Rpc.
-type RpcAttr func(optionalAttr)
-
-// RpcProtocol sets the optional protocol attribute to value.
-//
-// value: RPC protocol to use.  Empty string means use the default protocol.
-// Options include 'grpc'.
-// If not specified, defaults to ""
-func RpcProtocol(value string) RpcAttr {
-	return func(m optionalAttr) {
-		m["protocol"] = value
-	}
-}
-
-// RpcFailFast sets the optional fail_fast attribute to value.
-//
-// value: `boolean`. If `true` (default), then failures to connect
-// (i.e., the server does not immediately respond) cause an RPC failure.
-// If not specified, defaults to true
-func RpcFailFast(value bool) RpcAttr {
-	return func(m optionalAttr) {
-		m["fail_fast"] = value
-	}
-}
-
-// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
-//
-// value: `int`. If `0` (default), then the kernel will run the RPC
-// request and only time out if the RPC deadline passes or the session times out.
-// If this value is greater than `0`, then the op will raise an exception if
-// the RPC takes longer than `timeout_in_ms`.
-// If not specified, defaults to 0
-func RpcTimeoutInMs(value int64) RpcAttr {
-	return func(m optionalAttr) {
-		m["timeout_in_ms"] = value
-	}
-}
-
-// Perform batches of RPC requests.
-//
-// This op asynchronously performs either a single RPC request, or a batch
-// of requests.  RPC requests are defined by three main parameters:
-//
-//   - `address` (the host+port or BNS address of the request)
-//   - `method` (the RPC method name for the request)
-//   - `request` (the serialized proto string, or vector of strings,
-//      of the RPC request argument).
-//
-// For example, if you have an RPC service running on port localhost:2345,
-// and its interface is configured with the following proto declaration:
-//
-// ```
-// service MyService {
-//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
-//   }
-// };
-// ```
-//
-// then call this op with arguments:
-//
-// ```
-// address = "localhost:2345"
-// method = "MyService/MyMethod"
-// ```
-//
-// The `request` tensor is a string tensor representing serialized `MyRequestProto`
-// strings; and the output string tensor `response` will have the same shape
-// and contain (upon successful completion) corresponding serialized
-// `MyResponseProto` strings.
-//
-// For example, to send a single, empty, `MyRequestProto`, call
-// this op with `request = ""`.  To send 5 **parallel** empty requests,
-// call this op with `request = ["", "", "", "", ""]`.
-//
-// More generally, one can create a batch of `MyRequestProto` serialized protos
-// from regular batched tensors using the `encode_proto` op, and convert
-// the response `MyResponseProto` serialized protos to batched tensors
-// using the `decode_proto` op.
-//
-// **NOTE** Working with serialized proto strings is faster than instantiating
-// actual proto objects in memory, so no performance degradation is expected
-// compared to writing custom kernels for this workflow.
-//
-// If the connection fails or the remote worker returns an error
-// status, the op reraises this exception locally.
-//
-// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph.
-//
-// Arguments:
-//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `method` and `request`.
-//	method: `0-D` or `1-D`.  The method address on the RPC server.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `request`.
-//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
-// If this tensor has more than 1 element, then multiple parallel rpc requests
-// are sent.  This argument broadcasts with `address` and `method`.
-//
-// Returns Same shape as `request`. Serialized proto strings: the rpc responses.
-func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Rpc",
-		Input: []tf.Input{
-			address, method, request,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StackPushV2Attr is an optional argument to StackPushV2.
-type StackPushV2Attr func(optionalAttr)
-
-// StackPushV2SwapMemory sets the optional swap_memory attribute to value.
-//
-// value: Swap `elem` to CPU. Default to false.
-// If not specified, defaults to false
-func StackPushV2SwapMemory(value bool) StackPushV2Attr {
-	return func(m optionalAttr) {
-		m["swap_memory"] = value
-	}
-}
-
-// Push an element onto the stack.
-//
-// Arguments:
-//	handle: The handle to a stack.
-//	elem: The tensor to be pushed onto the stack.
-//
-// Returns The same tensor as the input 'elem'.
-func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StackPushV2",
-		Input: []tf.Input{
-			handle, elem,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset that concatenates `input_dataset` with `another_dataset`.
-func ConcatenateDataset(scope *Scope, input_dataset tf.Output, another_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "ConcatenateDataset",
-		Input: []tf.Input{
-			input_dataset, another_dataset,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Adds a value to the current value of a variable.
-//
-// Any ReadVariableOp with a control dependency on this op is guaranteed to
-// see the incremented value or a subsequent newer one.
-//
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value by which the variable will be incremented.
-//
-// Returns the created operation.
-func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AssignAddVariableOp",
-		Input: []tf.Input{
-			resource, value,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Records the latency of producing `input_dataset` elements in a StatsAggregator.
-func LatencyStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "LatencyStatsDataset",
-		Input: []tf.Input{
-			input_dataset, tag,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SparseToDenseAttr is an optional argument to SparseToDense.
-type SparseToDenseAttr func(optionalAttr)
-
-// SparseToDenseValidateIndices sets the optional validate_indices attribute to value.
-//
-// value: If true, indices are checked to make sure they are sorted in
-// lexicographic order and that there are no repeats.
-// If not specified, defaults to true
-func SparseToDenseValidateIndices(value bool) SparseToDenseAttr {
-	return func(m optionalAttr) {
-		m["validate_indices"] = value
-	}
-}
-
-// Converts a sparse representation into a dense tensor.
-//
-// Builds an array `dense` with shape `output_shape` such that
-//
-// ```
-// # If sparse_indices is scalar
-// dense[i] = (i == sparse_indices ? sparse_values : default_value)
-//
-// # If sparse_indices is a vector, then for each i
-// dense[sparse_indices[i]] = sparse_values[i]
-//
-// # If sparse_indices is an n by d matrix, then for each i in [0, n)
-// dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i]
-// ```
-//
-// All other values in `dense` are set to `default_value`.  If `sparse_values` is a
-// scalar, all sparse indices are set to this single value.
-//
-// Indices should be sorted in lexicographic order, and indices must not
-// contain any repeats. If `validate_indices` is true, these properties
-// are checked during execution.
-//
-// Arguments:
-//	sparse_indices: 0-D, 1-D, or 2-D.  `sparse_indices[i]` contains the complete
-// index where `sparse_values[i]` will be placed.
-//	output_shape: 1-D.  Shape of the dense output tensor.
-//	sparse_values: 1-D.  Values corresponding to each row of `sparse_indices`,
-// or a scalar value to be used for all sparse indices.
-//	default_value: Scalar value to set for indices not specified in
-// `sparse_indices`.
-//
-// Returns Dense output tensor of shape `output_shape`.
-func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Output, sparse_values tf.Output, default_value tf.Output, optional ...SparseToDenseAttr) (dense tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseToDense",
-		Input: []tf.Input{
-			sparse_indices, output_shape, sparse_values, default_value,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors.
+// Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors.
 //
 // The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
 // `filter` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
@@ -30973,67 +30514,21 @@ func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtyp
 //	batch_size: A scalar representing the number of elements to accumulate in a
 // batch.
 //	padded_shapes: A list of int64 tensors representing the desired padded shapes
-// of the corresponding output components. These shapes may be partially
-// specified, using `-1` to indicate that a particular dimension should be
-// padded to the maximum size of all batch elements.
-//	padding_values: A list of scalars containing the padding value to use for
-// each of the outputs.
-//
-func PaddedBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "PaddedBatchDataset",
-		Input: []tf.Input{
-			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values),
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset that batches input elements into a SparseTensor.
-//
-// Arguments:
-//	input_dataset: A handle to an input dataset. Must have a single component.
-//	batch_size: A scalar representing the number of elements to accumulate in a
-// batch.
-//	row_shape: A vector representing the dense shape of each row in the produced
-// SparseTensor. The shape may be partially specified, using `-1` to indicate
-// that a particular dimension should use the maximum size of all batch elements.
-//
-//
-func DenseToSparseBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, row_shape tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "DenseToSparseBatchDataset",
-		Input: []tf.Input{
-			input_dataset, batch_size, row_shape,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Deprecated. Use TensorArrayGradV3
+// of the corresponding output components. These shapes may be partially
+// specified, using `-1` to indicate that a particular dimension should be
+// padded to the maximum size of all batch elements.
+//	padding_values: A list of scalars containing the padding value to use for
+// each of the outputs.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArrayGradV3
-func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output) {
+func PaddedBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"source": source}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGradV2",
+		Type: "PaddedBatchDataset",
 		Input: []tf.Input{
-			handle, flow_in,
+			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values),
 		},
 		Attrs: attrs,
 	}
@@ -31646,21 +31141,6 @@ func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQu
 	return op.Output(0)
 }
 
-// Produces a summary of any statistics recorded by the given statistics manager.
-func StatsAggregatorSummary(scope *Scope, iterator tf.Output) (summary tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "StatsAggregatorSummary",
-		Input: []tf.Input{
-			iterator,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Compute the pairwise cross product.
 //
 // `a` and `b` must be the same shape; they can either be simple 3-element vectors,
@@ -31854,30 +31334,6 @@ func OptionalHasValue(scope *Scope, optional tf.Output) (has_value tf.Output) {
 	return op.Output(0)
 }
 
-// Creates a dataset that executes a SQL query and emits rows of the result set.
-//
-// Arguments:
-//	driver_name: The database type. Currently, the only supported type is 'sqlite'.
-//	data_source_name: A connection string to connect to the database.
-//	query: A SQL query to execute.
-//
-//
-func SqlDataset(scope *Scope, driver_name tf.Output, data_source_name tf.Output, query tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "SqlDataset",
-		Input: []tf.Input{
-			driver_name, data_source_name, query,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Returns the value stored in an Optional variant or raises an error if none exists.
 func OptionalGetValue(scope *Scope, optional tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
 	if scope.Err() != nil {
@@ -32293,46 +31749,6 @@ func ParallelDynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output)
 	return op.Output(0)
 }
 
-// Computes the gradient for the inverse of `x` wrt its input.
-//
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "InvGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// List of the given size with empty elements.
-//
-// element_shape: the shape of the future elements of the list
-// num_elements: the number of elements to reserve
-// handle: the output list
-// element_dtype: the desired type of elements in the list.
-func TensorListReserve(scope *Scope, element_shape tf.Output, num_elements tf.Output, element_dtype tf.DataType) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"element_dtype": element_dtype}
-	opspec := tf.OpSpec{
-		Type: "TensorListReserve",
-		Input: []tf.Input{
-			element_shape, num_elements,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // PriorityQueueV2Attr is an optional argument to PriorityQueueV2.
 type PriorityQueueV2Attr func(optionalAttr)
 
@@ -32775,95 +32191,330 @@ func QueueCloseV2(scope *Scope, handle tf.Output, optional ...QueueCloseV2Attr)
 	return scope.AddOperation(opspec)
 }
 
-// Computes inverse hyperbolic tangent of x element-wise.
-func Atanh(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Atanh",
-		Input: []tf.Input{
-			x,
-		},
+// Computes inverse hyperbolic tangent of x element-wise.
+func Atanh(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Atanh",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns true if queue is closed.
+//
+// This operation returns true if the queue is closed and false if the queue
+// is open.
+//
+// Arguments:
+//	handle: The handle to a queue.
+func QueueIsClosedV2(scope *Scope, handle tf.Output) (is_closed tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "QueueIsClosedV2",
+		Input: []tf.Input{
+			handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the absolute value of a tensor.
+//
+// Given a tensor `x`, this operation returns a tensor containing the absolute
+// value of each element in `x`. For example, if x is an input element and y is
+// an output element, this operation computes \\(y = |x|\\).
+func Abs(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Abs",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StackV2Attr is an optional argument to StackV2.
+type StackV2Attr func(optionalAttr)
+
+// StackV2StackName sets the optional stack_name attribute to value.
+//
+// value: Overrides the name used for the temporary stack resource. Default
+// value is the name of the 'Stack' op (which is guaranteed unique).
+// If not specified, defaults to ""
+func StackV2StackName(value string) StackV2Attr {
+	return func(m optionalAttr) {
+		m["stack_name"] = value
+	}
+}
+
+// A stack that produces elements in first-in last-out order.
+//
+// Arguments:
+//	max_size: The maximum size of the stack if non-negative. If negative, the stack
+// size is unlimited.
+//	elem_type: The type of the elements on the stack.
+//
+// Returns The handle to the stack.
+func StackV2(scope *Scope, max_size tf.Output, elem_type tf.DataType, optional ...StackV2Attr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"elem_type": elem_type}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StackV2",
+		Input: []tf.Input{
+			max_size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// OrderedMapStageAttr is an optional argument to OrderedMapStage.
+type OrderedMapStageAttr func(optionalAttr)
+
+// OrderedMapStageCapacity sets the optional capacity attribute to value.
+//
+// value: Maximum number of elements in the Staging Area. If > 0, inserts
+// on the container will block when the capacity is reached.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapStageContainer sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container. Otherwise,
+// a default container is used.
+// If not specified, defaults to ""
+func OrderedMapStageContainer(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapStageSharedName sets the optional shared_name attribute to value.
+//
+// value: It is necessary to match this name to the matching Unstage Op.
+// If not specified, defaults to ""
+func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Stage (key, values) in the underlying container which behaves like a ordered
+//
+// associative container.   Elements are ordered by key.
+//
+// Arguments:
+//	key: int64
+//
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+//
+// Returns the created operation.
+func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "OrderedMapStage",
+		Input: []tf.Input{
+			key, indices, tf.OutputList(values),
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// RpcAttr is an optional argument to Rpc.
+type RpcAttr func(optionalAttr)
+
+// RpcProtocol sets the optional protocol attribute to value.
+//
+// value: RPC protocol to use.  Empty string means use the default protocol.
+// Options include 'grpc'.
+// If not specified, defaults to ""
+func RpcProtocol(value string) RpcAttr {
+	return func(m optionalAttr) {
+		m["protocol"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns true if queue is closed.
-//
-// This operation returns true if the queue is closed and false if the queue
-// is open.
+// RpcFailFast sets the optional fail_fast attribute to value.
 //
-// Arguments:
-//	handle: The handle to a queue.
-func QueueIsClosedV2(scope *Scope, handle tf.Output) (is_closed tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: `boolean`. If `true` (default), then failures to connect
+// (i.e., the server does not immediately respond) cause an RPC failure.
+// If not specified, defaults to true
+func RpcFailFast(value bool) RpcAttr {
+	return func(m optionalAttr) {
+		m["fail_fast"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "QueueIsClosedV2",
-		Input: []tf.Input{
-			handle,
-		},
+}
+
+// RpcTimeoutInMs sets the optional timeout_in_ms attribute to value.
+//
+// value: `int`. If `0` (default), then the kernel will run the RPC
+// request and only time out if the RPC deadline passes or the session times out.
+// If this value is greater than `0`, then the op will raise an exception if
+// the RPC takes longer than `timeout_in_ms`.
+// If not specified, defaults to 0
+func RpcTimeoutInMs(value int64) RpcAttr {
+	return func(m optionalAttr) {
+		m["timeout_in_ms"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the absolute value of a tensor.
+// Perform batches of RPC requests.
 //
-// Given a tensor `x`, this operation returns a tensor containing the absolute
-// value of each element in `x`. For example, if x is an input element and y is
-// an output element, this operation computes \\(y = |x|\\).
-func Abs(scope *Scope, x tf.Output) (y tf.Output) {
+// This op asynchronously performs either a single RPC request, or a batch
+// of requests.  RPC requests are defined by three main parameters:
+//
+//   - `address` (the host+port or BNS address of the request)
+//   - `method` (the RPC method name for the request)
+//   - `request` (the serialized proto string, or vector of strings,
+//      of the RPC request argument).
+//
+// For example, if you have an RPC service running on port localhost:2345,
+// and its interface is configured with the following proto declaration:
+//
+// ```
+// service MyService {
+//   rpc MyMethod(MyRequestProto) returns (MyResponseProto) {
+//   }
+// };
+// ```
+//
+// then call this op with arguments:
+//
+// ```
+// address = "localhost:2345"
+// method = "MyService/MyMethod"
+// ```
+//
+// The `request` tensor is a string tensor representing serialized `MyRequestProto`
+// strings; and the output string tensor `response` will have the same shape
+// and contain (upon successful completion) corresponding serialized
+// `MyResponseProto` strings.
+//
+// For example, to send a single, empty, `MyRequestProto`, call
+// this op with `request = ""`.  To send 5 **parallel** empty requests,
+// call this op with `request = ["", "", "", "", ""]`.
+//
+// More generally, one can create a batch of `MyRequestProto` serialized protos
+// from regular batched tensors using the `encode_proto` op, and convert
+// the response `MyResponseProto` serialized protos to batched tensors
+// using the `decode_proto` op.
+//
+// **NOTE** Working with serialized proto strings is faster than instantiating
+// actual proto objects in memory, so no performance degradation is expected
+// compared to writing custom kernels for this workflow.
+//
+// If the connection fails or the remote worker returns an error
+// status, the op reraises this exception locally.
+//
+// See the `TryRpc` op if you prefer to handle RPC failures manually in the graph.
+//
+// Arguments:
+//	address: `0-D` or `1-D`.  The address (i.e. host_name:port) of the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `method` and `request`.
+//	method: `0-D` or `1-D`.  The method address on the RPC server.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `request`.
+//	request: `0-D` or `1-D`.  Serialized proto strings: the rpc request argument.
+// If this tensor has more than 1 element, then multiple parallel rpc requests
+// are sent.  This argument broadcasts with `address` and `method`.
+//
+// Returns Same shape as `request`. Serialized proto strings: the rpc responses.
+func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, optional ...RpcAttr) (response tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Abs",
+		Type: "Rpc",
 		Input: []tf.Input{
-			x,
+			address, method, request,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StackV2Attr is an optional argument to StackV2.
-type StackV2Attr func(optionalAttr)
+// StackPushV2Attr is an optional argument to StackPushV2.
+type StackPushV2Attr func(optionalAttr)
 
-// StackV2StackName sets the optional stack_name attribute to value.
+// StackPushV2SwapMemory sets the optional swap_memory attribute to value.
 //
-// value: Overrides the name used for the temporary stack resource. Default
-// value is the name of the 'Stack' op (which is guaranteed unique).
-// If not specified, defaults to ""
-func StackV2StackName(value string) StackV2Attr {
+// value: Swap `elem` to CPU. Default to false.
+// If not specified, defaults to false
+func StackPushV2SwapMemory(value bool) StackPushV2Attr {
 	return func(m optionalAttr) {
-		m["stack_name"] = value
+		m["swap_memory"] = value
 	}
 }
 
-// A stack that produces elements in first-in last-out order.
+// Push an element onto the stack.
 //
 // Arguments:
-//	max_size: The maximum size of the stack if non-negative. If negative, the stack
-// size is unlimited.
-//	elem_type: The type of the elements on the stack.
+//	handle: The handle to a stack.
+//	elem: The tensor to be pushed onto the stack.
 //
-// Returns The handle to the stack.
-func StackV2(scope *Scope, max_size tf.Output, elem_type tf.DataType, optional ...StackV2Attr) (handle tf.Output) {
+// Returns The same tensor as the input 'elem'.
+func StackPushV2(scope *Scope, handle tf.Output, elem tf.Output, optional ...StackPushV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"elem_type": elem_type}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StackV2",
+		Type: "StackPushV2",
 		Input: []tf.Input{
-			max_size,
+			handle, elem,
 		},
 		Attrs: attrs,
 	}
@@ -33254,6 +32905,25 @@ func TensorArraySizeV3(scope *Scope, handle tf.Output, flow_in tf.Output) (size
 	return op.Output(0)
 }
 
+// Deprecated. Use TensorArrayGradV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayGradV3
+func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source string) (grad_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"source": source}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGradV2",
+		Input: []tf.Input{
+			handle, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseReduceMaxAttr is an optional argument to SparseReduceMax.
 type SparseReduceMaxAttr func(optionalAttr)
 
@@ -33954,3 +33624,74 @@ func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output
 	}
 	return scope.AddOperation(opspec)
 }
+
+// MapUnstageAttr is an optional argument to MapUnstage.
+type MapUnstageAttr func(optionalAttr)
+
+// MapUnstageCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageCapacity(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageContainer(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageSharedName(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns the values associated with the key
+//
+// from the underlying container.   If the underlying container
+// does not contain this key, the op will block until it does.
+func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapUnstage",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstage", err)
+		return
+	}
+	return values
+}
-- 
GitLab


From 5e39bde8ea53ed2d0de4e21288778f534593b3a3 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Sun, 2 Dec 2018 18:45:46 -0800
Subject: [PATCH 1173/1554] Removing no-longer-needed test.

PiperOrigin-RevId: 223731958
---
 tensorflow/python/framework/function_def_to_graph.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/function_def_to_graph.py b/tensorflow/python/framework/function_def_to_graph.py
index 1803cb906d..4d1aabde06 100644
--- a/tensorflow/python/framework/function_def_to_graph.py
+++ b/tensorflow/python/framework/function_def_to_graph.py
@@ -174,9 +174,7 @@ def function_def_to_graph_def(fdef, input_shapes=None):
   # Update inputs of all nodes in graph.
   for node_def in graph_def.node:
     for i in range(len(node_def.input)):
-      # TODO(apassos): how can it not be there?
-      if node_def.input[i] in nested_to_flat_tensor_name:
-        node_def.input[i] = nested_to_flat_tensor_name[node_def.input[i]]
+      node_def.input[i] = nested_to_flat_tensor_name[node_def.input[i]]
 
   return graph_def, nested_to_flat_tensor_name
 
-- 
GitLab


From bd9a4ff87899e71c95739b45fcc1bb9f0c10fbe3 Mon Sep 17 00:00:00 2001
From: tomguluson92 <314913739@qq.com>
Date: Mon, 3 Dec 2018 10:50:10 +0800
Subject: [PATCH 1174/1554] [Proto/decode]word order fixed

---
 tensorflow/core/util/proto/decode.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/util/proto/decode.h b/tensorflow/core/util/proto/decode.h
index cbcb203ee7..248490a2fa 100644
--- a/tensorflow/core/util/proto/decode.h
+++ b/tensorflow/core/util/proto/decode.h
@@ -318,7 +318,7 @@ inline int ReadPackedPrimitives(const void* bufp, const size_t len,
   return count;
 }
 
-// Reads a primitive value field from a serialized proto.
+// Reads a primitive field value from a serialized proto.
 // The value is parsed from the serialized format, then static_cast
 // to the desired type for TensorFlow and stored.
 template <class ValueType, class TensorType,
-- 
GitLab


From bb058ae291d565aa985e3fa8437ee026ad5c10f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 2 Dec 2018 19:01:41 -0800
Subject: [PATCH 1175/1554] FloatDepthwiseConvKernel support dilation.

PiperOrigin-RevId: 223732659
---
 .../internal/optimized/depthwiseconv_float.h  | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h
index 25b66d4b55..c77715de57 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_float.h
@@ -793,22 +793,26 @@ void FloatDepthwiseConvAccumRow(int stride, int dilation_factor,
     int out_x_loop_end_unclampled = 0;
     if (kAllowStrided) {
       if (stride == 2) {
-        out_x_loop_start_unclampled = (pad_width - filter_x + 1) / 2;
+        out_x_loop_start_unclampled =
+            (pad_width - dilation_factor * filter_x + 1) / 2;
         out_x_loop_end_unclampled =
-            (pad_width + input_width - filter_x + 1) / 2;
+            (pad_width + input_width - dilation_factor * filter_x + 1) / 2;
       } else if (stride == 4) {
-        out_x_loop_start_unclampled = (pad_width - filter_x + 3) / 4;
+        out_x_loop_start_unclampled =
+            (pad_width - dilation_factor * filter_x + 3) / 4;
         out_x_loop_end_unclampled =
-            (pad_width + input_width - filter_x + 3) / 4;
+            (pad_width + input_width - dilation_factor * filter_x + 3) / 4;
       } else {
         out_x_loop_start_unclampled =
-            (pad_width - filter_x + stride - 1) / stride;
-        out_x_loop_end_unclampled =
-            (pad_width + input_width - filter_x + stride - 1) / stride;
+            (pad_width - dilation_factor * filter_x + stride - 1) / stride;
+        out_x_loop_end_unclampled = (pad_width + input_width -
+                                     dilation_factor * filter_x + stride - 1) /
+                                    stride;
       }
     } else {
-      out_x_loop_start_unclampled = pad_width - filter_x;
-      out_x_loop_end_unclampled = pad_width + input_width - filter_x;
+      out_x_loop_start_unclampled = pad_width - dilation_factor * filter_x;
+      out_x_loop_end_unclampled =
+          pad_width + input_width - dilation_factor * filter_x;
     }
     // The kernel will have to iterate on the segment of the
     // output row that starts at out_x_loop_start and out_x_loop_end.
@@ -819,7 +823,8 @@ void FloatDepthwiseConvAccumRow(int stride, int dilation_factor,
 
     float* acc_buffer_ptr =
         acc_buffer + (out_x_loop_start - out_x_buffer_start) * output_depth;
-    const int in_x_origin = (out_x_loop_start * stride) - pad_width + filter_x;
+    const int in_x_origin =
+        (out_x_loop_start * stride) - pad_width + dilation_factor * filter_x;
     const float* input_ptr = input_data + in_x_origin * input_depth;
     const int num_output_pixels = out_x_loop_end - out_x_loop_start;
     FloatDepthwiseConvKernel<kAllowStrided, kFixedInputDepth,
@@ -936,8 +941,7 @@ inline void DepthwiseConv(
                                         FIXED_DEPTH_MULTIPLIER)           \
   if (!row_accum_func && (stride_width == 1 || ALLOW_STRIDED) &&          \
       (input_depth == FIXED_INPUT_DEPTH || FIXED_INPUT_DEPTH == 0) &&     \
-      depth_multiplier == FIXED_DEPTH_MULTIPLIER &&                       \
-      dilation_height_factor == 1 && dilation_width_factor == 1) {        \
+      depth_multiplier == FIXED_DEPTH_MULTIPLIER) {                       \
     row_accum_func =                                                      \
         FloatDepthwiseConvAccumRow<ALLOW_STRIDED, FIXED_INPUT_DEPTH,      \
                                    FIXED_DEPTH_MULTIPLIER>;               \
-- 
GitLab


From 2e2f34b0922f98798befbceb14a150f816390ed3 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Sun, 2 Dec 2018 21:28:07 -0800
Subject: [PATCH 1176/1554] Fix function test to trigger and catch correct
 exception

In v1 testInputSignatureForFunctionWithNonTensorInputsNotAllowed was
raising a TypeError when it should have triggered a ValueError. This
was due to the signature being invalid. Once the signature is fixed the
test works in both v1 and v2 mode.

PiperOrigin-RevId: 223740912
---
 tensorflow/python/eager/function_test.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 8d24bcc0fe..71afbd24d8 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1301,7 +1301,6 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       defined.get_concrete_function(
           tensor_spec.TensorSpec(shape=(3,), dtype=dtypes.float32))
 
-  @test_util.run_deprecated_v1
   def testInputSignatureForFunctionWithNonTensorInputsNotAllowed(self):
 
     def foo(a, training=True):
@@ -1310,10 +1309,16 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       else:
         return -1.0 * a
 
-    signature = [tensor_spec.TensorSpec([], dtypes.float32)] * 2
+    signature = [
+        tensor_spec.TensorSpec([], dtypes.float32),
+        tensor_spec.TensorSpec([], dtypes.bool),
+    ]
     defined = def_function.function(foo, input_signature=signature)
     a = constant_op.constant(1.0)
-    with self.assertRaises(TypeError):
+    with self.assertRaisesRegexp(
+        ValueError,
+        'When input_signature is provided, all inputs to '
+        'the Python function must be Tensors.'):
       defined(a, training=True)
 
   def testInputSignatureWithKeywordPositionalArgs(self):
-- 
GitLab


From 2ce0bec9da49d1848ff72233e187b2d12e59b182 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Sun, 2 Dec 2018 23:41:53 -0800
Subject: [PATCH 1177/1554] Modify MatMul tests to use gradient_checker_v2

Note the gradient checker still need a session, hence we are not
completely running in eager mode.

PiperOrigin-RevId: 223749090
---
 .../python/kernel_tests/matmul_op_test.py     | 52 ++++++++-----------
 1 file changed, 21 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py
index 983f463f5e..2a04d452fe 100644
--- a/tensorflow/python/kernel_tests/matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/matmul_op_test.py
@@ -24,8 +24,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -41,11 +40,11 @@ class MatVecTest(test_lib.TestCase):
   def testTwoByTwoCase(self):
     a = np.array([[1, 2], [3, 4]])
     b = np.array([5, 6])
-    with self.cached_session():
-      c = math_ops.matvec(a, b)
-      self.assertAllEqual((2,), c.shape)
-      c_ = self.evaluate(c)
-    self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c_)
+
+    c = math_ops.matvec(a, b)
+    self.assertAllEqual((2,), c.shape)
+
+    self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c)
 
 
 def _AddTest(test, op_name, testcase_name, fn):
@@ -85,17 +84,13 @@ def _GetMatMulTest(a_np_, b_np_, use_static_shape_, **kwargs_):
     # np.matrix(a_np_) * np.matrix(b_np_)
     effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_)
     effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_)
-    with self.session(use_gpu=use_gpu) as sess:
+    with test_util.device(use_gpu=use_gpu):
       if use_static_shape_:
         a = constant_op.constant(effective_a_np)
         b = constant_op.constant(effective_b_np)
-        res = math_ops.matmul(a, b, **kwargs_)
-        tf_val = self.evaluate(res)
+        tf_val = math_ops.matmul(a, b, **kwargs_)
       else:
-        a = array_ops.placeholder(a_np_.dtype)
-        b = array_ops.placeholder(b_np_.dtype)
-        res = math_ops.matmul(a, b, **kwargs_)
-        tf_val = sess.run(res, feed_dict={a: effective_a_np, b: effective_b_np})
+        tf_val = math_ops.matmul(effective_a_np, effective_b_np, **kwargs_)
 
     self.assertAllCloseAccordingToType(
         tf_val,
@@ -119,27 +114,22 @@ def _GetMatMulGradientTest(a_np_, b_np_, use_static_shape_, **kwargs_):
       self.skipTest("Skipping infeasible gradient test.")
 
     # Transpose and possibly conjugate a_np_ and b_np_ according to the
-    # attributes such that tf.matmul(effective_a_np, effective_b_np, **kwargs)
-    # results in a valid matrix multiplication and produces the same result as
+    # attributes such that tf.matmul(a, b, **kwargs) results in a valid matrix
+    # multiplication and produces the same result as
     # np.matrix(a_np_) * np.matrix(b_np_)
-    effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_)
-    effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_)
+    a = _GetTransposedMatrices(a_np_, "a", kwargs_)
+    b = _GetTransposedMatrices(b_np_, "b", kwargs_)
 
     epsilon = np.finfo(a_np_.dtype).eps
     delta = epsilon**(1.0 / 3.0)
     tol = 20 * delta
     with self.session(use_gpu=True):
-      a = constant_op.constant(effective_a_np)
-      b = constant_op.constant(effective_b_np)
-      res = math_ops.matmul(a, b, **kwargs_)
-      for x, x_init in [a, effective_a_np], [b, effective_b_np]:
-        theoretical, numerical = gradient_checker.compute_gradient(
-            x,
-            x_init.shape,
-            res, [a_np_.shape[0], b_np_.shape[1]],
-            x_init_value=x_init,
-            delta=delta)
-        self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
+      theoretical, numerical = gradient_checker_v2.compute_gradient(
+          lambda x: math_ops.matmul(x, b, **kwargs_), [a])
+      self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
+      theoretical, numerical = gradient_checker_v2.compute_gradient(
+          lambda x: math_ops.matmul(a, x, **kwargs_), [b])
+      self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
 
   return Test
 
@@ -223,8 +213,8 @@ class MatMulInfixOperatorTest(test_lib.TestCase):
     b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]])
     c = infix_matmul(a, b)
     d = math_ops.matmul(a, b)
-    with self.cached_session():
-      self.assertAllEqual(c.eval(), self.evaluate(d))
+
+    self.assertAllEqual(c, d)
 
 
 if __name__ == "__main__":
-- 
GitLab


From be070a9ce0be8bb5b723682ed1f42810c38b0baa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 00:36:24 -0800
Subject: [PATCH 1178/1554] Implement Fill.

PiperOrigin-RevId: 223753923
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md |  12 ++
 tensorflow/lite/kernels/BUILD                 |  14 ++
 tensorflow/lite/kernels/fill.cc               | 141 ++++++++++++++++++
 tensorflow/lite/kernels/fill_test.cc          |  94 ++++++++++++
 .../internal/optimized/optimized_ops.h        |   1 +
 .../internal/reference/reference_ops.h        |  10 ++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/testing/generate_examples.py  |  30 ++++
 tensorflow/lite/toco/tflite/operator.cc       |   2 +
 tensorflow/lite/toco/tflite/operator_test.cc  |   1 +
 11 files changed, 308 insertions(+)
 create mode 100644 tensorflow/lite/kernels/fill.cc
 create mode 100644 tensorflow/lite/kernels/fill_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 33cee1ab20..8507521694 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -237,6 +237,7 @@ def generated_test_models():
         "equal",
         "exp",
         "expand_dims",
+        "fill",
         "floor",
         "floor_div",
         "floor_mod",
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index 6976f058d1..dcfda72137 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -989,6 +989,18 @@ Outputs {
 }
 ```
 
+**FILL**
+
+```
+Inputs {
+  0: A Tensor. Must be one of the following types: int32, int64. 1-D. Represents the shape of the output tensor.
+  1: A Tensor. 0-D (scalar). Value to fill the returned tensor.
+}
+Outputs {
+  0: A tensor of the same type as value (input1).
+}
+```
+
 And these are TensorFlow Lite operations that are present but not ready for
 custom models yet:
 
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index c1b005b580..8d5c3b406f 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -183,6 +183,7 @@ cc_library(
         "exp.cc",
         "expand_dims.cc",
         "fake_quant.cc",
+        "fill.cc",
         "floor.cc",
         "floor_div.cc",
         "floor_mod.cc",
@@ -1410,6 +1411,19 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "fill_test",
+    size = "small",
+    srcs = ["fill_test.cc"],
+    tags = ["tflite_not_portable_ios"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/lite/kernels/fill.cc b/tensorflow/lite/kernels/fill.cc
new file mode 100644
index 0000000000..079ee44f37
--- /dev/null
+++ b/tensorflow/lite/kernels/fill.cc
@@ -0,0 +1,141 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/internal/tensor_ctypes.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace fill {
+
+namespace {
+
+constexpr int kDimsTensor = 0;
+constexpr int kValueTensor = 1;
+constexpr int kOutputTensor = 0;
+
+template <typename T>
+TfLiteStatus ResizeOutputImpl(TfLiteContext* context, const TfLiteTensor* dims,
+                              TfLiteTensor* output) {
+  TfLiteIntArray* output_shape = TfLiteIntArrayCreate(dims->dims->data[0]);
+  for (int i = 0; i < output_shape->size; ++i) {
+    T data = GetTensorData<T>(dims)[i];
+    if (data < 0) {
+      context->ReportError(context, "Fill dimensions must be >= 0", dims->type);
+      return kTfLiteError;
+    }
+    output_shape->data[i] = data;
+  }
+  return context->ResizeTensor(context, output, output_shape);
+}
+
+TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* dims,
+                          TfLiteTensor* output) {
+  switch (dims->type) {
+    case kTfLiteInt32:
+      return ResizeOutputImpl<int32_t>(context, dims, output);
+    case kTfLiteInt64:
+      return ResizeOutputImpl<int64_t>(context, dims, output);
+    default:
+      context->ReportError(
+          context,
+          "Fill only currently supports int32, int64 for input 0, "
+          "got %d.",
+          dims->type);
+      return kTfLiteError;
+  }
+}
+
+}  // namespace
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  const TfLiteTensor* dims = GetInput(context, node, kDimsTensor);
+  const TfLiteTensor* value = GetInput(context, node, kValueTensor);
+
+  // Make sure the 1st input tensor is 1-D.
+  TF_LITE_ENSURE_EQ(context, NumDimensions(dims), 1);
+
+  // Make sure the 1st input tensor is int32 or int64.
+  const auto dtype = dims->type;
+  TF_LITE_ENSURE(context, dtype == kTfLiteInt32 || dtype == kTfLiteInt64);
+
+  // Make sure the 2nd input tensor is a scalar.
+  TF_LITE_ENSURE_EQ(context, NumDimensions(value), 0);
+
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  output->type = value->type;
+
+  if (IsConstantTensor(dims)) {
+    TF_LITE_ENSURE_OK(context, ResizeOutput(context, dims, output));
+  } else {
+    SetTensorToDynamic(output);
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* value = GetInput(context, node, kValueTensor);
+
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  if (IsDynamicTensor(output)) {
+    const TfLiteTensor* dims = GetInput(context, node, kDimsTensor);
+    TF_LITE_ENSURE_OK(context, ResizeOutput(context, dims, output));
+  }
+#define TF_LITE_FILL(data_type)                                               \
+  reference_ops::Fill(GetTensorShape(value), GetTensorData<data_type>(value), \
+                      GetTensorShape(output),                                 \
+                      GetTensorData<data_type>(output))
+  switch (output->type) {
+    case kTfLiteInt32:
+      TF_LITE_FILL(int32_t);
+      break;
+    case kTfLiteInt64:
+      TF_LITE_FILL(int64_t);
+      break;
+    case kTfLiteFloat32:
+      TF_LITE_FILL(float);
+      break;
+    default:
+      context->ReportError(
+          context,
+          "Fill only currently supports int32, int64, float32 for input 1,"
+          "got %d.",
+          value->type);
+      return kTfLiteError;
+  }
+#undef TF_LITE_FILL
+  return kTfLiteOk;
+}
+
+}  // namespace fill
+
+TfLiteRegistration* Register_FILL() {
+  static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr,
+                                 fill::Prepare, fill::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/fill_test.cc b/tensorflow/lite/kernels/fill_test.cc
new file mode 100644
index 0000000000..08044d76f9
--- /dev/null
+++ b/tensorflow/lite/kernels/fill_test.cc
@@ -0,0 +1,94 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+using ::testing::IsEmpty;
+
+class FillOpModel : public SingleOpModel {
+ public:
+  explicit FillOpModel(const TensorData& input1, const TensorData& input2) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(input1);
+    SetBuiltinOp(BuiltinOperator_FILL, BuiltinOptions_FillOptions,
+                 CreateFillOptions(builder_).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  int input1() { return input1_; }
+  int input2() { return input2_; }
+  int output() { return output_; }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+TEST(FillOpModel, FillInt32) {
+  FillOpModel m({TensorType_INT32, {2}}, {TensorType_INT32});
+  m.PopulateTensor<int32_t>(m.input1(), {2, 3});
+  m.PopulateTensor<int32_t>(m.input2(), {-11});
+  m.Invoke();
+  EXPECT_THAT(m.ExtractVector<int32_t>(m.output()),
+              ElementsAreArray({-11, -11, -11, -11, -11, -11}));
+  EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({2, 3}));
+}
+
+TEST(FillOpModel, FillInt64) {
+  FillOpModel m({TensorType_INT32, {2}}, {TensorType_INT64});
+  m.PopulateTensor<int32_t>(m.input1(), {2, 4});
+  m.PopulateTensor<int64_t>(m.input2(), {2 ^ 45});
+  m.Invoke();
+  EXPECT_THAT(m.ExtractVector<int64_t>(m.output()),
+              ElementsAreArray({2 ^ 45, 2 ^ 45, 2 ^ 45, 2 ^ 45, 2 ^ 45, 2 ^ 45,
+                                2 ^ 45, 2 ^ 45}));
+  EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({2, 4}));
+}
+
+TEST(FillOpModel, FillFloat) {
+  FillOpModel m({TensorType_INT64, {3}}, {TensorType_FLOAT32});
+  m.PopulateTensor<int64_t>(m.input1(), {2, 2, 2});
+  m.PopulateTensor<float>(m.input2(), {4.0});
+  m.Invoke();
+  EXPECT_THAT(m.ExtractVector<float>(m.output()),
+              ElementsAreArray({4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0}));
+  EXPECT_THAT(m.GetTensorShape(m.output()), ElementsAreArray({2, 2, 2}));
+}
+
+TEST(FillOpModel, FillOutputScalar) {
+  FillOpModel m({TensorType_INT64, {0}}, {TensorType_FLOAT32});
+  m.PopulateTensor<float>(m.input2(), {4.0});
+  m.Invoke();
+  EXPECT_THAT(m.ExtractVector<float>(m.output()), ElementsAreArray({4.0}));
+  EXPECT_THAT(m.GetTensorShape(m.output()), IsEmpty());
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index df335e9e92..c7691e2763 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -64,6 +64,7 @@ using reference_ops::DepthConcatenation;
 using reference_ops::Dequantize;
 using reference_ops::Div;
 using reference_ops::FakeQuant;
+using reference_ops::Fill;
 using reference_ops::Gather;
 using reference_ops::Greater;
 using reference_ops::GreaterEqual;
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index be766ea452..ea3ab06da1 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -4613,6 +4613,16 @@ inline void BroadcastPrelu4DSlow(const PreluParams& params,
   }
 }
 
+template <typename T>
+void Fill(const RuntimeShape& value_shape, const T* value_data,
+          const RuntimeShape& output_shape, T* output_data) {
+  TFLITE_DCHECK_EQ(value_shape.DimensionsCount(), 0);
+  const int flat_size = output_shape.FlatSize();
+  for (int i = 0; i < flat_size; ++i) {
+    output_data[i] = *value_data;
+  }
+}
+
 }  // namespace reference_ops
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 7f0b9239ee..0ad90a9753 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -127,6 +127,7 @@ TfLiteRegistration* Register_FLOOR_MOD();
 TfLiteRegistration* Register_RANGE();
 TfLiteRegistration* Register_LEAKY_RELU();
 TfLiteRegistration* Register_SQUARED_DIFFERENCE();
+TfLiteRegistration* Register_FILL();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -264,6 +265,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RANGE, Register_RANGE());
   AddBuiltin(BuiltinOperator_LEAKY_RELU, Register_LEAKY_RELU());
   AddBuiltin(BuiltinOperator_SQUARED_DIFFERENCE, Register_SQUARED_DIFFERENCE());
+  AddBuiltin(BuiltinOperator_FILL, Register_FILL());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 55796846b3..7bd4c009b4 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3528,6 +3528,36 @@ def make_range_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_fill_tests(zip_path):
+  """Make a set of tests to do fill."""
+
+  test_parameters = [{
+      "dims_dtype": [tf.int32, tf.int64],
+      "dims_shape": [[], [1], [3], [3, 3]],
+      "value_dtype": [tf.int32, tf.int64, tf.float32],
+  }]
+
+  def build_graph(parameters):
+    """Build the fill op testing graph."""
+    input1 = tf.placeholder(
+        dtype=parameters["dims_dtype"],
+        name="dims",
+        shape=parameters["dims_shape"])
+    input2 = tf.placeholder(
+        dtype=parameters["value_dtype"], name="value", shape=[])
+    out = tf.fill(input1, input2)
+    return [input1, input2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input1 = create_tensor_data(parameters["dims_dtype"],
+                                parameters["dims_shape"], 1)
+    input2 = create_scalar_data(parameters["value_dtype"])
+    return [input1, input2], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input1, input2])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def _make_logical_tests(op):
   """Make a set of tests to do logical operations."""
 
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 1c3bee3c5a..c4030a1b1a 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1667,6 +1667,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       "ZEROS_LIKE", OperatorType::kZerosLike));
   ops.push_back(
       MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
+  ops.push_back(
+      MakeUnique<SimpleOperator<FillOperator>>("FILL", OperatorType::kFill));
   return ops;
 }
 }  // namespace
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 09e4435a18..6154d94692 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -151,6 +151,7 @@ TEST_F(OperatorTest, SimpleOperators) {
                                                    OperatorType::kZerosLike);
   CheckSimpleOperator<FloorModOperator>("FLOOR_MOD", OperatorType::kFloorMod);
   CheckSimpleOperator<RangeOperator>("RANGE", OperatorType::kRange);
+  CheckSimpleOperator<FillOperator>("FILL", OperatorType::kFill);
 }
 
 TEST_F(OperatorTest, BuiltinAdd) {
-- 
GitLab


From d8b89170d98d200d3538a6435159f561aa888015 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 01:02:35 -0800
Subject: [PATCH 1179/1554] compat: Update forward compatibility horizon to
 2018-12-03

PiperOrigin-RevId: 223756500
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 8ec90ee759..811bf31f39 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 2)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 3)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 6798656ce64201afe7d8d9ed5445cd5114bfd9b4 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 3 Dec 2018 02:28:11 -0800
Subject: [PATCH 1180/1554] Make behavior of tf.convert_to_tensor consistent in
 both graph and eager mode. Move so that tests for tf.convert_to_tensor are
 run in both graph and eager mode

PiperOrigin-RevId: 223765440
---
 tensorflow/python/framework/ops.py            |   7 +-
 tensorflow/python/framework/ops_test.py       | 103 +++++++++---------
 tensorflow/python/keras/backend.py            |   3 +-
 .../keras/engine/training_dataset_test.py     |  20 ++--
 .../python/keras/model_subclassing_test.py    |   4 +-
 5 files changed, 70 insertions(+), 67 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 6c1a9c6f73..31986fbc41 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -1113,10 +1113,9 @@ def internal_convert_to_tensor(value,
   if ctx is None: ctx = context.context()
   if isinstance(value, EagerTensor):
     if ctx.executing_eagerly():
-      # Fast path for EagerTensors that don't need any conversion.
-      # Note that we don't check that value's dtype matches the dtype
-      # argument.  We expect that the C runtime will do that checking
-      # when we execute the kernel.
+      if dtype is not None:
+        dtype = dtypes.as_dtype(dtype)
+        value = _TensorTensorConversionFunction(value, dtype=dtype)
       return value
     else:
       graph = get_default_graph()
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 04a9ed05fd..c957ffaecf 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -146,15 +146,14 @@ class TensorAndShapeTest(test_util.TensorFlowTestCase):
 
 class IndexedSlicesTest(test_util.TensorFlowTestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_in_graph_and_eager_modes
   def testToTensor(self):
-    with self.cached_session():
-      values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
-      indices = constant_op.constant([0, 2])
-      dense_shape = constant_op.constant([3, 2])
-      x = ops.IndexedSlices(values, indices, dense_shape)
-      tensor = ops.convert_to_tensor(x, name="tensor")
-      self.assertAllEqual(tensor.eval(), [[2, 3], [0, 0], [5, 7]])
+    values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
+    indices = constant_op.constant([0, 2])
+    dense_shape = constant_op.constant([3, 2])
+    x = ops.IndexedSlices(values, indices, dense_shape)
+    tensor = ops.convert_to_tensor(x, name="tensor")
+    self.assertAllEqual(self.evaluate(tensor), [[2, 3], [0, 0], [5, 7]])
 
   @test_util.run_deprecated_v1
   def testNegation(self):
@@ -325,12 +324,12 @@ class OperationTest(test_util.TensorFlowTestCase):
     op = test_ops.a()
     self.assertEqual(tensor_shape.unknown_shape(), op.get_shape())
 
+  @test_util.run_in_graph_and_eager_modes
   def testConvertToTensorNestedArray(self):
-    with self.cached_session():
-      values = [[2], [3], [5], [7]]
-      tensor = ops.convert_to_tensor(values)
-      self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, self.evaluate(tensor))
+    values = [[2], [3], [5], [7]]
+    tensor = ops.convert_to_tensor(values)
+    self.assertAllEqual((4, 1), tensor.get_shape().as_list())
+    self.assertAllEqual(values, self.evaluate(tensor))
 
   def testShapeTuple(self):
     with self.cached_session():
@@ -346,57 +345,61 @@ class OperationTest(test_util.TensorFlowTestCase):
       converted = ops.convert_to_tensor(1)
       self.assertTrue(isinstance(converted, ops.EagerTensor))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_in_graph_and_eager_modes
   def testConvertToTensorNestedTuple(self):
-    with self.cached_session():
-      values = ((2,), (3,), (5,), (7,))
-      tensor = ops.convert_to_tensor(values)
-      self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, ops.convert_to_tensor(values).eval())
+    values = ((2,), (3,), (5,), (7,))
+    tensor = ops.convert_to_tensor(values)
+    self.assertAllEqual((4, 1), tensor.get_shape().as_list())
+    self.assertAllEqual(values, self.evaluate(ops.convert_to_tensor(values)))
 
+  @test_util.run_in_graph_and_eager_modes
   def testConvertToTensorNestedTensors(self):
-    with self.cached_session():
-      values = ((2,), (3,), (5,), (7,))
-      tensor = ops.convert_to_tensor(
-          [constant_op.constant(row) for row in values])
-      self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, self.evaluate(tensor))
-      tensor = ops.convert_to_tensor(
-          [[constant_op.constant(v) for v in row] for row in values])
-      self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(values, self.evaluate(tensor))
+    values = ((2,), (3,), (5,), (7,))
+    tensor = ops.convert_to_tensor(
+        [constant_op.constant(row) for row in values])
+    self.assertAllEqual((4, 1), tensor.get_shape().as_list())
+    self.assertAllEqual(values, self.evaluate(tensor))
+    tensor = ops.convert_to_tensor(
+        [[constant_op.constant(v) for v in row] for row in values])
+    self.assertAllEqual((4, 1), tensor.get_shape().as_list())
+    self.assertAllEqual(values, self.evaluate(tensor))
 
+  @test_util.run_in_graph_and_eager_modes
   def testConvertToTensorNestedMix(self):
-    with self.cached_session():
-      values = ([2], (3,), [constant_op.constant(5)], constant_op.constant([7]))
-      tensor = ops.convert_to_tensor(values)
-      self.assertAllEqual((4, 1), tensor.get_shape().as_list())
-      self.assertAllEqual(((2,), (3,), (5,), (7,)), self.evaluate(tensor))
+    values = ([2], (3,), [constant_op.constant(5)], constant_op.constant([7]))
+    tensor = ops.convert_to_tensor(values)
+    self.assertAllEqual((4, 1), tensor.get_shape().as_list())
+    self.assertAllEqual(((2,), (3,), (5,), (7,)), self.evaluate(tensor))
 
+  @test_util.run_in_graph_and_eager_modes
   def testConvertToTensorPreferred(self):
-    with self.cached_session():
-      values = [2, 3, 5, 7]
-      tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.float32)
-      self.assertEqual(dtypes.float32, tensor.dtype)
+    values = [2, 3, 5, 7]
+    tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.float32)
+    self.assertEqual(dtypes.float32, tensor.dtype)
 
-    with self.cached_session():
-      # Convert empty tensor to anything.
-      values = []
-      tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64)
-      self.assertEqual(dtypes.int64, tensor.dtype)
+    # Convert empty tensor to anything.
+    values = []
+    tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64)
+    self.assertEqual(dtypes.int64, tensor.dtype)
 
-    with self.cached_session():
-      # The preferred dtype is a type error and will convert to
-      # float32 instead.
-      values = [1.23]
-      tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64)
-      self.assertEqual(dtypes.float32, tensor.dtype)
+    # The preferred dtype is a type error and will convert to
+    # float32 instead.
+    values = [1.23]
+    tensor = ops.convert_to_tensor(values, preferred_dtype=dtypes.int64)
+    self.assertEqual(dtypes.float32, tensor.dtype)
 
+  @test_util.run_in_graph_and_eager_modes
   def testConvertToInvalidTensorType(self):
     with self.assertRaises(TypeError):
       # Forcing an invalid dtype should fail with a type error.
       values = [1.23]
-      _ = ops.convert_to_tensor(values, dtype=dtypes.int64)
+      ops.convert_to_tensor(values, dtype=dtypes.int64)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testConvertToTensorFromInvalidTensor(self):
+    tensor = constant_op.constant(42.0, dtype=dtypes.float32)
+    with self.assertRaises(ValueError):
+      ops.convert_to_tensor(tensor, dtype=dtypes.int32)
 
   @test_util.run_deprecated_v1
   def testNoConvert(self):
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 7dae203f8a..420c457a0c 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3161,7 +3161,8 @@ class EagerExecutionFunction(object):
         if value is None:
           raise ValueError(
               'You must feed a value for placeholder %s' % (tensor,))
-      value = ops.convert_to_tensor(value, dtype=tensor.dtype)
+      if not isinstance(value, ops.Tensor):
+        value = ops.convert_to_tensor(value, dtype=tensor.dtype)
       if value.dtype != tensor.dtype:
         # Temporary workaround due to `convert_to_tensor` not casting floats.
         # See b/119637405
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 8020326377..e48df241e9 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -53,8 +53,8 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics)
 
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
@@ -112,8 +112,8 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics)
 
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
@@ -133,8 +133,8 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics)
 
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(2)
     dataset = dataset.batch(10)
@@ -157,8 +157,8 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics)
 
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
@@ -180,8 +180,8 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics)
 
-    inputs = np.zeros((10, 3))
-    targets = np.zeros((10, 4))
+    inputs = np.zeros((10, 3), np.float32)
+    targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index ca7dded60d..92eb1a6185 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -456,8 +456,8 @@ class ModelSubclassingTest(test.TestCase):
       model = SimpleTestModel(num_classes=num_classes, use_dp=True, use_bn=True)
       model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
 
-      x = np.ones((num_samples, input_dim))
-      y = np.zeros((num_samples, num_classes))
+      x = np.ones((num_samples, input_dim), dtype=np.float32)
+      y = np.zeros((num_samples, num_classes), dtype=np.float32)
       dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
       dataset = dataset.repeat(100)
       dataset = dataset.batch(10)
-- 
GitLab


From c4afcb2911c4427245a0f704b23f177a37e8643a Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 3 Dec 2018 06:55:25 -0800
Subject: [PATCH 1181/1554] [XLA] Rename xla/client/lib/numeric.* to
 xla/client/lib/matrix.*.

This more accurately describes what it now contains (no Iota any more), and prepares for moving more matrix helpers out of TF2XLA into the XLA client library.

PiperOrigin-RevId: 223788293
---
 tensorflow/compiler/tf2xla/kernels/BUILD             |  4 +---
 tensorflow/compiler/tf2xla/kernels/conv_ops.cc       |  2 +-
 tensorflow/compiler/tf2xla/kernels/diag_op.cc        |  2 +-
 .../compiler/tf2xla/kernels/image_resize_ops.cc      |  1 -
 tensorflow/compiler/tf2xla/kernels/resampler_ops.cc  |  1 -
 .../compiler/tf2xla/kernels/stateless_random_ops.cc  |  1 -
 tensorflow/compiler/tf2xla/kernels/topk_op.cc        |  1 -
 tensorflow/compiler/tf2xla/lib/BUILD                 |  4 ++--
 tensorflow/compiler/tf2xla/lib/qr.cc                 |  2 +-
 tensorflow/compiler/tf2xla/lib/triangular_solve.cc   |  2 +-
 tensorflow/compiler/xla/client/lib/BUILD             | 12 ++++++------
 .../xla/client/lib/{numeric.cc => matrix.cc}         |  3 ++-
 .../compiler/xla/client/lib/{numeric.h => matrix.h}  |  6 +++---
 .../client/lib/{numeric_test.cc => matrix_test.cc}   |  3 ++-
 14 files changed, 20 insertions(+), 24 deletions(-)
 rename tensorflow/compiler/xla/client/lib/{numeric.cc => matrix.cc} (98%)
 rename tensorflow/compiler/xla/client/lib/{numeric.h => matrix.h} (90%)
 rename tensorflow/compiler/xla/client/lib/{numeric_test.cc => matrix_test.cc} (97%)

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index d85b4f5ae0..dcd38764be 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -144,7 +144,7 @@ tf_kernel_library(
         "//tensorflow/compiler/xla/client/lib:arithmetic",
         "//tensorflow/compiler/xla/client/lib:constants",
         "//tensorflow/compiler/xla/client/lib:math",
-        "//tensorflow/compiler/xla/client/lib:numeric",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/client/lib:pooling",
         "//tensorflow/compiler/xla/client/lib:prng",
         "//tensorflow/compiler/xla/client/lib:sorting",
@@ -196,7 +196,6 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client/lib:arithmetic",
         "//tensorflow/compiler/xla/client/lib:constants",
-        "//tensorflow/compiler/xla/client/lib:numeric",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
@@ -216,7 +215,6 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client/lib:arithmetic",
         "//tensorflow/compiler/xla/client/lib:constants",
-        "//tensorflow/compiler/xla/client/lib:numeric",
         "//tensorflow/core:framework",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:conv_ops",
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
index d820528a43..eafdba876a 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
@@ -22,7 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/core/framework/node_def_util.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/diag_op.cc b/tensorflow/compiler/tf2xla/kernels/diag_op.cc
index 49c12fc232..ee79cbc70d 100644
--- a/tensorflow/compiler/tf2xla/kernels/diag_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/diag_op.cc
@@ -19,7 +19,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/framework/op_kernel.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
index 0c7ca602bf..5a10c52ba8 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/array4d.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/register_types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
index 769e0cd140..54d34a38ab 100644
--- a/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/resampler_ops.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/array4d.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc
index 5db52781be..50653d7b39 100644
--- a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/lib/prng.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/topk_op.cc b/tensorflow/compiler/tf2xla/kernels/topk_op.cc
index 8a0c94cfae..ee3bdf3394 100644
--- a/tensorflow/compiler/tf2xla/kernels/topk_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/topk_op.cc
@@ -15,7 +15,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 #include "tensorflow/compiler/xla/client/lib/sorting.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD
index 1ce3930fd1..3575e480c1 100644
--- a/tensorflow/compiler/tf2xla/lib/BUILD
+++ b/tensorflow/compiler/tf2xla/lib/BUILD
@@ -99,7 +99,7 @@ cc_library(
         "//tensorflow/compiler/xla/client/lib:arithmetic",
         "//tensorflow/compiler/xla/client/lib:constants",
         "//tensorflow/compiler/xla/client/lib:math",
-        "//tensorflow/compiler/xla/client/lib:numeric",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/core:lib",
     ],
 )
@@ -140,7 +140,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/client/lib:constants",
-        "//tensorflow/compiler/xla/client/lib:numeric",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/core:lib",
     ],
 )
diff --git a/tensorflow/compiler/tf2xla/lib/qr.cc b/tensorflow/compiler/tf2xla/lib/qr.cc
index 6b3f2b6e06..aa1c0e4743 100644
--- a/tensorflow/compiler/tf2xla/lib/qr.cc
+++ b/tensorflow/compiler/tf2xla/lib/qr.cc
@@ -24,7 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
index 6524c2a9b1..4bc3796e4f 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
@@ -21,7 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index c5733bc66d..dc188b67a6 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -104,9 +104,9 @@ xla_test(
 )
 
 cc_library(
-    name = "numeric",
-    srcs = ["numeric.cc"],
-    hdrs = ["numeric.h"],
+    name = "matrix",
+    srcs = ["matrix.cc"],
+    hdrs = ["matrix.h"],
     deps = [
         ":arithmetic",
         ":constants",
@@ -118,11 +118,11 @@ cc_library(
 )
 
 xla_test(
-    name = "numeric_test",
-    srcs = ["numeric_test.cc"],
+    name = "matrix_test",
+    srcs = ["matrix_test.cc"],
     tags = ["enable_for_xla_interpreter"],
     deps = [
-        ":numeric",
+        ":matrix",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
diff --git a/tensorflow/compiler/xla/client/lib/numeric.cc b/tensorflow/compiler/xla/client/lib/matrix.cc
similarity index 98%
rename from tensorflow/compiler/xla/client/lib/numeric.cc
rename to tensorflow/compiler/xla/client/lib/matrix.cc
index 377654220b..f78d8509d8 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix.cc
@@ -13,13 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
+
 #include <numeric>
 #include <vector>
 
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
 
 namespace xla {
 
diff --git a/tensorflow/compiler/xla/client/lib/numeric.h b/tensorflow/compiler/xla/client/lib/matrix.h
similarity index 90%
rename from tensorflow/compiler/xla/client/lib/numeric.h
rename to tensorflow/compiler/xla/client/lib/matrix.h
index f62fdab4b0..7af3e68f20 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.h
+++ b/tensorflow/compiler/xla/client/lib/matrix.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_
-#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
 
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/types.h"
@@ -42,4 +42,4 @@ XlaOp LowerTriangle(XlaOp x);
 
 }  // namespace xla
 
-#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
diff --git a/tensorflow/compiler/xla/client/lib/numeric_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc
similarity index 97%
rename from tensorflow/compiler/xla/client/lib/numeric_test.cc
rename to tensorflow/compiler/xla/client/lib/matrix_test.cc
index 7d6aedd494..8eacc583d1 100644
--- a/tensorflow/compiler/xla/client/lib/numeric_test.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc
@@ -13,7 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/client/lib/numeric.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
+
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
-- 
GitLab


From caabd16692077c7e1e705c6bf5b5e0606b680a00 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 07:27:39 -0800
Subject: [PATCH 1182/1554] Bumping CUDA 10 kokoro image to latest build.

PiperOrigin-RevId: 223791989
---
 third_party/toolchains/preconfig/generate/containers.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 0309b8ffca..7099b9bf3e 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,4 @@
 container_digests = {
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:c26138f4c38c754da2bad44a8a068523abf7fbd71d58a57ce92e5342c5431bf5",
-    "cuda10.0-cudnn7-ubuntu14.04": "sha256:7737d770599de8435115bfdf56977002319316a6735ab081f82506cb51443f9d",
+    "cuda10.0-cudnn7-ubuntu14.04": "sha256:66e7d592c8149291d5562a0f3093655a15b09c22e0eb30a87b3b6469b7a30ffc",
 }
-- 
GitLab


From 6bccef267ae0e430a66992524a8e5ef7118695e0 Mon Sep 17 00:00:00 2001
From: Roy Frostig <frostig@google.com>
Date: Mon, 3 Dec 2018 08:16:09 -0800
Subject: [PATCH 1183/1554] [XLA] Add multi-replica allocation and execution to
 the Python client library.

Implemented only for the local XLA service, with guards against use in XRT mode.

PiperOrigin-RevId: 223798412
---
 .../xla/python/local_computation_builder.cc   | 112 ++++++++++--------
 .../xla/python/local_computation_builder.h    |   7 +-
 .../xla/python/local_computation_builder.i    |  31 +++++
 tensorflow/compiler/xla/python/xla_client.py  | 100 +++++++++++++---
 4 files changed, 184 insertions(+), 66 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index c0b57e7d26..ac1c623882 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -148,14 +148,19 @@ static StatusOr<ScopedShapedBuffer> ToBuffer(LocalClient* client,
 
 /* static */
 StatusOr<LocalShapedBuffer*> LocalShapedBuffer::FromLiteral(
-    const Literal& argument, const absl::optional<Shape>& shape_with_layout) {
+    const Literal& argument, const absl::optional<Shape>& shape_with_layout,
+    int replica_number) {
   LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(int device_ordinal,
+                      client->ReplicaNumberToDeviceOrdinal(replica_number));
+  VLOG(1) << "Creating shaped buffer from literal on replica/ordinal: "
+          << replica_number << "/" << device_ordinal;
   StatusOr<ScopedShapedBuffer> buf = [&] {
     if (shape_with_layout) {
       Literal relaid = argument.Relayout(shape_with_layout.value());
-      return ToBuffer(client, /*device_ordinal=*/0, relaid);
+      return ToBuffer(client, device_ordinal, relaid);
     }
-    return ToBuffer(client, /*device_ordinal=*/0, argument);
+    return ToBuffer(client, device_ordinal, argument);
   }();
   TF_RETURN_IF_ERROR(buf.status());
   return new LocalShapedBuffer(std::move(buf).ValueOrDie());
@@ -309,69 +314,80 @@ CompiledLocalComputation::CompiledLocalComputation(
     std::unique_ptr<LocalExecutable> executable)
     : executable_(std::move(executable)) {}
 
-StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
-    absl::Span<LocalShapedBuffer* const> argument_handles) {
+StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::Execute(
+    absl::Span<const std::vector<LocalShapedBuffer*>> argument_handles) {
   LocalClient* client = GetOrCreateLocalClient();
+  const int num_replicas = GetReplicaCount();
+
+  if (argument_handles.size() != num_replicas) {
+    return InvalidArgument(
+        "Attempted to execute with %d replicas when replica count is %d",
+        argument_handles.size(), num_replicas);
+  }
 
-  VLOG(1) << "Execution requested with " << GetReplicaCount() << " replicas.";
+  VLOG(1) << "Executing with " << num_replicas << " replicas.";
 
   // Each replica populates a StatusOr result, but only the output value of
   // replica zero is returned.
-  std::vector<StatusOr<ScopedShapedBuffer>> results(GetReplicaCount());
+  std::vector<StatusOr<ScopedShapedBuffer>> results(num_replicas);
   {
     tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "xlarun",
-                                        GetReplicaCount());
-
-    for (int replica = 0; replica < GetReplicaCount(); ++replica) {
-      pool.Schedule([this, client, replica, &argument_handles, &results] {
-        StatusOr<int> device_ordinal_status =
-            client->ReplicaNumberToDeviceOrdinal(replica);
-        if (!device_ordinal_status.ok()) {
-          results[replica] = device_ordinal_status.status();
-          return;
-        }
-        const int device_ordinal = device_ordinal_status.ValueOrDie();
-        VLOG(3) << "Replica " << replica
-                << " mapped to device ordinal for execution: "
-                << device_ordinal;
-
-        std::vector<const ShapedBuffer*> argument_buffers;
-        argument_buffers.reserve(argument_handles.size());
-        for (auto& handle : argument_handles) {
-          argument_buffers.push_back(handle->shaped_buffer());
-        }
-
-        DeviceAssignment device_assignment =
-            client->backend()
-                .computation_placer()
-                ->AssignDevices(GetReplicaCount(), /*computation_count=*/1)
-                .ConsumeValueOrDie();
-
-        ExecutableRunOptions options;
-        options.set_device_ordinal(device_ordinal);
-        options.set_allocator(client->backend().memory_allocator());
-        options.set_intra_op_thread_pool(
-            client->backend().eigen_intra_op_thread_pool_device());
-        options.set_device_assignment(&device_assignment);
-        StatusOr<ScopedShapedBuffer> result_buffer_status =
-            executable_->Run(argument_buffers, options);
-
-        results[replica] = std::move(result_buffer_status);
-      });
+                                        num_replicas);
+
+    for (int replica = 0; replica < num_replicas; ++replica) {
+      pool.Schedule(
+          [this, client, replica, num_replicas, &argument_handles, &results] {
+            StatusOr<int> device_ordinal_status =
+                client->ReplicaNumberToDeviceOrdinal(replica);
+            if (!device_ordinal_status.ok()) {
+              results[replica] = device_ordinal_status.status();
+              return;
+            }
+            const int device_ordinal = device_ordinal_status.ValueOrDie();
+            VLOG(3) << "Replica " << replica
+                    << " mapped to device ordinal for execution: "
+                    << device_ordinal;
+
+            std::vector<const ShapedBuffer*> argument_buffers;
+            argument_buffers.reserve(argument_handles[replica].size());
+            for (auto& handle : argument_handles[replica]) {
+              argument_buffers.push_back(handle->shaped_buffer());
+            }
+
+            DeviceAssignment device_assignment =
+                client->backend()
+                    .computation_placer()
+                    ->AssignDevices(num_replicas, /*computation_count=*/1)
+                    .ConsumeValueOrDie();
+
+            ExecutableRunOptions options;
+            options.set_device_ordinal(device_ordinal);
+            options.set_allocator(client->backend().memory_allocator());
+            options.set_intra_op_thread_pool(
+                client->backend().eigen_intra_op_thread_pool_device());
+            options.set_device_assignment(&device_assignment);
+            StatusOr<ScopedShapedBuffer> result_buffer_status =
+                executable_->Run(argument_buffers, options);
+
+            results[replica] = std::move(result_buffer_status);
+          });
     }
   }
 
-  for (int replica = 0; replica < GetReplicaCount(); ++replica) {
-    const auto& statusor = results[replica];
+  std::vector<LocalShapedBuffer*> wrapped_results(num_replicas);
+  for (int replica = 0; replica < num_replicas; ++replica) {
+    auto& statusor = results[replica];
     if (!statusor.ok()) {
       return InternalError(
           "Failed running replica %d (other replicas may have failed as well): "
           "%s.",
           replica, statusor.status().ToString());
     }
+    wrapped_results[replica] =
+        new LocalShapedBuffer(std::move(statusor).ValueOrDie());
   }
 
-  return new LocalShapedBuffer(std::move(results[0]).ValueOrDie());
+  return new LocalShapedBufferTuple(std::move(wrapped_results));
 }
 
 static StatusOr<Shape> GetReturnValueShape(const XlaComputation& computation) {
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index c9b7ae824a..8247b05775 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -71,7 +71,8 @@ StatusOr<Literal> TransferFromOutfeedLocalReplica(const Shape& shape,
 class LocalShapedBuffer {
  public:
   static StatusOr<LocalShapedBuffer*> FromLiteral(
-      const Literal& argument, const absl::optional<Shape>& shape_with_layout);
+      const Literal& argument, const absl::optional<Shape>& shape_with_layout,
+      int replica_number);
 
   LocalShapedBuffer(ScopedShapedBuffer shaped_buffer);
   StatusOr<Literal> ToLiteral() const;
@@ -172,8 +173,8 @@ class CompiledLocalComputation {
  public:
   CompiledLocalComputation(std::unique_ptr<LocalExecutable> executable);
 
-  StatusOr<LocalShapedBuffer*> Execute(
-      absl::Span<LocalShapedBuffer* const> argument_handles);
+  StatusOr<LocalShapedBufferTuple*> Execute(
+      absl::Span<const std::vector<LocalShapedBuffer*> > argument_handles);
 
  private:
   std::unique_ptr<LocalExecutable> executable_;
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 5c2538dcc3..88349b788e 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -363,6 +363,37 @@ tensorflow::ImportNumpy();
   $1 = temps;
 }
 
+%typemap(in) absl::Span<const std::vector<xla::swig::LocalShapedBuffer*> >
+    (std::vector<std::vector<LocalShapedBuffer*> > temps) {
+  if (!PySequence_Check($input)) {
+    PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
+    SWIG_fail;
+  }
+  const int size = PySequence_Size($input);
+  temps.reserve(size);
+  for (int i = 0; i < size; ++i) {
+    PyObject* o = PySequence_GetItem($input, i);
+    std::vector<LocalShapedBuffer*> vec;
+    const int vec_size = PySequence_Size(o);
+    vec.reserve(vec_size);
+    for (int j = 0; j < vec_size; ++j) {
+      PyObject* vec_elt = PySequence_GetItem(o, j);
+      LocalShapedBuffer* lsbp;
+      if ((SWIG_ConvertPtr(vec_elt, (void**) &lsbp, $descriptor(xla::swig::LocalShapedBuffer*),
+                           SWIG_POINTER_EXCEPTION)) == -1) {
+        Py_DECREF(vec_elt);
+        Py_DECREF(o);
+        SWIG_fail;
+      }
+      vec.push_back(lsbp);
+      Py_DECREF(vec_elt);
+    }
+    temps.push_back(vec);
+    Py_DECREF(o);
+  }
+  $1 = temps;
+}
+
 %typemap(in) absl::Span<xla::swig::XrtAllocation* const>
     (std::vector<XrtAllocation*> temps) {
   if (!PySequence_Check($input)) {
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index e5fba0d7ac..4dbbeebd0b 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -222,24 +222,33 @@ class LocalBuffer(object):
   means the referent is in device memory.
   """
 
-  def __init__(self, c_buffer, backend):
+  def __init__(self, c_buffer, backend, replica):
     self.c_buffer = c_buffer
     self._backend = backend
+    self._replica = replica
     if backend.backend_type == BackendType.XRT:
       self._delete = c_api.DeleteXrtAllocation
     else:
       self._delete = c_api.DeleteLocalShapedBuffer
 
   @staticmethod
-  def from_pyval(pyval, backend=XLA_LOCAL_BACKEND):
+  def from_pyval(pyval, replica=0, backend=XLA_LOCAL_BACKEND):
     """Allocate and copy to XLA the given python value."""
     pyval = require_numpy_array_layout(pyval)
+    num_replicas = get_replica_count()
+    if not 0 <= replica < num_replicas:
+      raise ValueError(
+          'Attempt to place buffer on replica {} when the replica count is {}'
+          .format(replica, num_replicas))
     if backend.backend_type == BackendType.XRT:
+      if replica != 0:
+        raise NotImplementedError(
+            'Multi-replica execution is not yet supported via the XRT backend.')
       cbuf = c_api.XrtAllocation.FromLiteral(
           pyval, _maybe_encode_string(backend.target))
     else:
-      cbuf = c_api.LocalShapedBuffer.FromLiteral(pyval, None)
-    return LocalBuffer(cbuf, backend)
+      cbuf = c_api.LocalShapedBuffer.FromLiteral(pyval, None, replica)
+    return LocalBuffer(cbuf, backend, replica)
 
   def to_py(self):
     return self.c_buffer.ToLiteral()
@@ -247,6 +256,9 @@ class LocalBuffer(object):
   def shape(self):
     return _wrap_shape(self.c_buffer.shape())
 
+  def replica(self):
+    return self._replica
+
   def delete(self):
     if self.c_buffer is not None:
       self._delete(self.c_buffer)
@@ -263,7 +275,8 @@ class LocalBuffer(object):
     self.delete()
     size = result.size()
     destructured = tuple(
-        LocalBuffer(result.Release(i), backend=self._backend)
+        LocalBuffer(
+            result.Release(i), replica=self._replica, backend=self._backend)
         for i in xrange(size))
     return destructured
 
@@ -576,22 +589,79 @@ class LocalComputation(object):
         layout_fn=layout_fn)
 
   def Execute(self, arguments=()):
-    """Execute with LocalBuffer arguments and return value."""
+    """Execute on one replica with LocalBuffer arguments and return value."""
+    return self.ExecutePerReplica([arguments])[0]
+
+  def ExecutePerReplica(self, arguments=None):
+    """Execute on many replicas with LocalBuffer arguments and return value.
+
+    Args:
+      arguments: A sequence of sequences of LocalBuffers. The i'th inner
+        sequence comprises the arguments for execution on the i'th replica.
+
+    Returns:
+      A list of the computation's outputs on each replica, as a LocalBuffer. If
+      a shallow sequence of arguments was passed in for `arguments`, then the
+      sole, zero'th replica's output is returned instead, as a LocalBuffer.
+    """
     if not self._is_compiled:
       raise ValueError('Cannot execute an uncompiled local XLA computation.')
-    arguments = tuple(arguments)
-    if any(arg.is_deleted() for arg in arguments):
-      raise ValueError('Executing with deleted local buffer argument')
-    return LocalBuffer(
-        self._c_computation.Execute([arg.c_buffer for arg in arguments]),
-        backend=self._backend)
+    if arguments is None:
+      arguments = ((),) * get_replica_count()
+    else:
+      arguments = [list(replica_args) for replica_args in arguments]
+
+    # Check arguments
+    for replica, replica_args in enumerate(arguments):
+      for arg in replica_args:
+        if arg.is_deleted():
+          raise ValueError('Executing with deleted local buffer argument')
+        if arg.replica() != replica:
+          raise ValueError(
+              'Executing on replica {} with argument from replica {}'.format(
+                  replica, arg.replica()))
+
+    # Pull out argument buffer handles
+    stripped_args = [
+        [arg.c_buffer for arg in replica_args] for replica_args in arguments
+    ]
+
+    # Execute
+    if self._backend.backend_type == BackendType.XRT:
+      if len(stripped_args) > 1:
+        raise NotImplementedError(
+            'Multi-replica execution is not yet supported via the XRT backend.')
+      output_buffers = [self._c_computation.Execute(stripped_args[0])]
+    else:
+      output_buffer_tup = self._c_computation.Execute(stripped_args)
+      size = output_buffer_tup.size()
+      output_buffers = [output_buffer_tup.Release(i) for i in xrange(size)]
+
+    # Wrap output handles in LocalBuffer instances
+    return tuple(
+        LocalBuffer(output_buffer, backend=self._backend, replica=replica)
+        for replica, output_buffer in enumerate(output_buffers))
 
   def ExecuteWithPythonValues(self, arguments=()):
-    """Execute with Python values as arguments and return value."""
-    arguments = tuple(
-        LocalBuffer.from_pyval(arg, backend=self._backend) for arg in arguments)
+    """Execute on one replica with Python values as arguments and output."""
+
+    def put(arg):
+      return LocalBuffer.from_pyval(arg, backend=self._backend)
+
+    arguments = [put(arg) for arg in arguments]
     return self.Execute(arguments).to_py()
 
+  def ExecuteWithPythonValuesPerReplica(self, arguments):
+    """Execute on many replicas with Python values as arguments and output."""
+
+    def put(arg, replica):
+      return LocalBuffer.from_pyval(arg, replica, backend=self._backend)
+
+    arguments = [[put(arg, replica)
+                  for arg in replica_args]
+                 for replica, replica_args in enumerate(arguments)]
+    return [out.to_py() for out in self.ExecutePerReplica(arguments)]
+
   def __del__(self):
     self._delete(self._c_computation)
 
-- 
GitLab


From fa0bc68ec9b5bfd66cb01a5f12d8a21b8077ebfd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 08:40:58 -0800
Subject: [PATCH 1184/1554] Flex delegate handles strings properly.

PiperOrigin-RevId: 223801600
---
 tensorflow/lite/delegates/flex/BUILD          |  2 ++
 tensorflow/lite/delegates/flex/buffer_map.cc  |  5 +++
 tensorflow/lite/delegates/flex/delegate.cc    | 31 +++++++++++++++++--
 .../lite/delegates/flex/delegate_test.cc      | 19 ++++++++++++
 4 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD
index 63e86899da..75083bf95a 100644
--- a/tensorflow/lite/delegates/flex/BUILD
+++ b/tensorflow/lite/delegates/flex/BUILD
@@ -83,8 +83,10 @@ cc_library(
         ":delegate_data",
         ":kernel",
         ":util",
+        "@com_google_absl//absl/strings:strings",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite:kernel_api",
+        "//tensorflow/lite:string_util",
         "//tensorflow/lite:util",
     ] + select({
         "//tensorflow:android": [
diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc
index 9a6c5e74a7..262ca9e089 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map.cc
@@ -93,6 +93,11 @@ class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
 class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
   explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor) {
+    if (tensor->data.raw == nullptr) {
+      num_strings_ = 0;
+      data_ = nullptr;
+      return;
+    }
     num_strings_ = GetStringCount(tensor->data.raw);
     data_ = tensorflow::cpu_allocator()->Allocate<string>(num_strings_);
 
diff --git a/tensorflow/lite/delegates/flex/delegate.cc b/tensorflow/lite/delegates/flex/delegate.cc
index 12dcaa11d3..ca7314fbae 100644
--- a/tensorflow/lite/delegates/flex/delegate.cc
+++ b/tensorflow/lite/delegates/flex/delegate.cc
@@ -16,12 +16,14 @@ limitations under the License.
 
 #include <vector>
 
+#include "absl/strings/str_cat.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/lite/context_util.h"
 #include "tensorflow/lite/delegates/flex/buffer_map.h"
 #include "tensorflow/lite/delegates/flex/kernel.h"
 #include "tensorflow/lite/delegates/flex/util.h"
+#include "tensorflow/lite/string_util.h"
 #include "tensorflow/lite/util.h"
-#include "tensorflow/core/lib/core/status.h"
 
 namespace tflite {
 namespace flex {
@@ -68,11 +70,34 @@ TfLiteStatus CopyFromBufferHandle(TfLiteContext* context,
   }
 
   tensorflow::Tensor t = buffer_map->GetTensor(buffer_handle);
+
+  if (output->type == kTfLiteString) {
+    if (t.dtype() != tensorflow::DT_STRING) {
+      context->ReportError(context,
+                           "Inconsistent type for TF string tensor index %d.",
+                           buffer_handle);
+      return kTfLiteError;
+    }
+    DynamicBuffer dynamic_buffer;
+
+    auto tf_data = t.flat<string>();
+    for (int i = 0; i < t.NumElements(); ++i) {
+      dynamic_buffer.AddString(tf_data(i).data(), tf_data(i).size());
+    }
+
+    dynamic_buffer.WriteToTensor(output, /*new_shape=*/nullptr);
+    return kTfLiteOk;
+  }
+
   tensorflow::StringPiece t_data = t.tensor_data();
 
   if (output->bytes != t_data.size()) {
-    context->ReportError(
-        context, "Not enough space to store TensorFlow's aligned buffer.");
+    context->ReportError(context,
+                         absl::StrCat("The given ", output->bytes,
+                                      " bytes are not enough to store "
+                                      "TensorFlow's aligned buffer of size ",
+                                      t_data.size(), " bytes.")
+                             .c_str());
     return kTfLiteError;
   }
 
diff --git a/tensorflow/lite/delegates/flex/delegate_test.cc b/tensorflow/lite/delegates/flex/delegate_test.cc
index e13029d9a5..1b2f476f03 100644
--- a/tensorflow/lite/delegates/flex/delegate_test.cc
+++ b/tensorflow/lite/delegates/flex/delegate_test.cc
@@ -93,6 +93,25 @@ TEST_F(DelegateTest, NonFloatTypeInference) {
   ASSERT_EQ(GetType(2), kTfLiteInt32);
 }
 
+TEST_F(DelegateTest, StringInference) {
+  AddTensors(3, {0, 1}, {2}, kTfLiteString, {2});
+
+  AddTfOp(testing::kAdd, {0, 1}, {2});
+
+  ConfigureDelegate();
+
+  SetShape(0, {2, 2});
+  SetStringValues(0, {"1", "2", "3", "4"});
+  SetShape(1, {2, 2});
+  SetStringValues(1, {"4", "3", "2", "1"});
+
+  ASSERT_TRUE(Invoke());
+
+  ASSERT_THAT(GetShape(2), ElementsAre(2, 2));
+  ASSERT_THAT(GetStringValues(2), ElementsAre("14", "23", "32", "41"));
+  ASSERT_EQ(GetType(2), kTfLiteString);
+}
+
 TEST_F(DelegateTest, MixedGraph) {
   AddTensors(9, {0, 3}, {8}, kTfLiteFloat32, {3});
 
-- 
GitLab


From 6e79807bad57e8726830a0ff9ba0daeda7e1386d Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Mon, 3 Dec 2018 08:49:25 -0800
Subject: [PATCH 1185/1554] [tf.data optimization] Add fake sinks for dataset
 graph functions so the real retvals can be modified during optimization
 passes.

PiperOrigin-RevId: 223802719
---
 tensorflow/core/kernels/data/BUILD            |  2 +-
 .../core/kernels/data/optimize_dataset_op.cc  | 51 +++++++++++++++++++
 .../optimization/optimize_dataset_test.py     | 15 ++++++
 3 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 23398b71a4..55a34c9c62 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -523,7 +523,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:graph_view",
@@ -532,6 +531,7 @@ tf_kernel_library(
         "//tensorflow/core/grappler/clusters:virtual_cluster",
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
         "//tensorflow/core/grappler/optimizers/data",
+        "//tensorflow/core/grappler/optimizers/data:function_utils",
         "//tensorflow/core/grappler/optimizers/data:graph_utils",
     ],
 )
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index cac6c43565..781150b069 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/graph_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/grappler_item_builder.h"
+#include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
 #include "tensorflow/core/lib/core/refcount.h"
@@ -217,6 +218,39 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       std::unique_ptr<IteratorBase> input_impl_;
     };
 
+    void AddFakeSinks(FunctionDef* function_def) {
+      int counter = 0;
+      for (const auto& output : function_def->signature().output_arg()) {
+        NodeDef* node = function_def->add_node_def();
+        tensorflow::grappler::function_utils::SetUniqueFunctionNodeName(
+            strings::StrCat("FakeSink", counter++), function_def, node);
+        node->set_op("Identity");
+        node->add_input(function_def->ret().at(output.name()));
+        (*node->mutable_attr())["T"].set_type(output.type());
+
+        (*function_def->mutable_ret())[output.name()] =
+            strings::StrCat(node->name(), ":output:0");
+      }
+    }
+
+    void RemoveFakeSinks(FunctionDef* function_def) {
+      // Map from identity node names to their input tensor strings
+      std::map<string, string> identity_map;
+      for (const auto& node : function_def->node_def()) {
+        if (node.op() == "Identity" && node.input_size() == 1) {
+          identity_map[node.name()] = node.input(0);
+        }
+      }
+      for (const auto& output_arg : function_def->signature().output_arg()) {
+        const string& tensor = function_def->ret().at(output_arg.name());
+        const string& output_node = tensor.substr(0, tensor.find(':'));
+        if (identity_map.find(output_node) != identity_map.end()) {
+          (*function_def->mutable_ret())[output_arg.name()] =
+              identity_map.at(output_node);
+        }
+      }
+    }
+
     Status ApplyOptimizations(OpKernelContext* ctx, GraphDef* graph_def,
                               string* output_node) {
       // Add an identity node as the fetch node, otherwise we might get
@@ -230,6 +264,15 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       (*node->mutable_attr())["T"].set_type(DT_VARIANT);
       *output_node = node->name();
 
+      // Add fake sink node to graph and functions to allow rewriting the actual
+      // sink nodes.
+      // TODO(b/118820916): When MetaOptimizer adds provisions for function
+      // retvals to be optimizable, we will no longer need this.
+      for (auto& function_def :
+           *graph_def->mutable_library()->mutable_function()) {
+        AddFakeSinks(&function_def);
+      }
+
       // Create metagraph.
       MetaGraphDef meta_graph_def;
       (*meta_graph_def.mutable_graph_def()) = *graph_def;
@@ -283,6 +326,14 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer(
           *grappler_item, config, ctx->device(), &cluster, graph_def));
 
+      // Remove fake sinks after optimizations are done.
+      // TODO(b/118820916): When MetaOptimizer adds provisions for function
+      // retvals to be optimizable, we will no longer need this.
+      for (auto& function_def :
+           *graph_def->mutable_library()->mutable_function()) {
+        RemoveFakeSinks(&function_def);
+      }
+
       return Status::OK();
     }
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index df26a2c0cd..22705e20bf 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -80,6 +80,21 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
     self.assertDatasetProduces(dataset, expected_output=[0])
 
+  def testOptimizationNestedDatasetWithModifiedRetval(self):
+
+    def flat_map_fn(_):
+      dataset = dataset_ops.Dataset.from_tensors(0)
+      dataset = dataset.apply(optimization.assert_next(["MapAndBatch"]))
+      # Should be fused by map and batch fusion
+      dataset = dataset.map(lambda x: x)
+      dataset = dataset.batch(1)
+      return dataset
+
+    dataset = dataset_ops.Dataset.range(1)
+    dataset = dataset.flat_map(flat_map_fn)
+    dataset = dataset_ops._OptimizeDataset(dataset, ["map_and_batch_fusion"])
+    self.assertDatasetProduces(dataset, expected_output=[[0]])
+
   def testOptimizationThreadPoolDataset(self):
     dataset = dataset_ops.Dataset.range(10).batch(10)
 
-- 
GitLab


From a02c524cb328dea373b77a03ec4f448f6f386674 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 3 Dec 2018 16:57:57 +0000
Subject: [PATCH 1186/1554] Update comment based on feedback

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/sparse_dense_binary_op_shared.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
index d7460363fc..a4e89f439e 100644
--- a/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
+++ b/tensorflow/core/kernels/sparse_dense_binary_op_shared.cc
@@ -88,7 +88,7 @@ class SparseDenseBinaryOpShared : public OpKernel {
     const auto rhs_dims = BCast::FromShape(dense_t->shape());
     BCast b(lhs_dims, rhs_dims, false);  // false for keeping the same num dims.
 
-    // True iff (size(lhs) >= size(rhs)) and all dims in lhs is smaller or equal
+    // True iff (size(lhs) >= size(rhs)) and all dims in lhs is greater or equal
     // to dims in rhs (from right to left).
     auto VecGreaterEq = [](ArraySlice<int64> lhs, ArraySlice<int64> rhs) {
       if (lhs.size() < rhs.size()) return false;
-- 
GitLab


From 244da6a4ffa5ff8de80d18baceedcf947ef6b68e Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 3 Dec 2018 09:01:37 -0800
Subject: [PATCH 1187/1554] Make TF2_BEHAVIOR=0 disable TF2 behavior.

Prior to this change, the mere presence of a TF2_BEHAVIOR
environment variable would enable TF2 behavior. With this,
setting that environment variable to "0" will disable it.

PiperOrigin-RevId: 223804383
---
 tensorflow/python/tf2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/tf2.py b/tensorflow/python/tf2.py
index a8b0d6b7c5..75748f8f2c 100644
--- a/tensorflow/python/tf2.py
+++ b/tensorflow/python/tf2.py
@@ -42,4 +42,4 @@ def disable():
 
 def enabled():
   """Returns True iff TensorFlow 2.0 behavior should be enabled."""
-  return _force_enable or os.getenv("TF2_BEHAVIOR") is not None
+  return _force_enable or os.getenv("TF2_BEHAVIOR", "0") != "0"
-- 
GitLab


From 3667478d73da230972b9a6432753e329ecfe7aef Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 3 Dec 2018 09:42:29 -0800
Subject: [PATCH 1188/1554]  replacing CHECK by DCHECK

---
 tensorflow/core/graph/mkl_layout_pass.cc      | 16 ++++++++--------
 tensorflow/core/graph/mkl_layout_pass_test.cc | 10 +++++-----
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index de1a982b9d..eca9afeada 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -776,7 +776,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   // merged with 'm'. If input 'm' is Conv2D, then check if there exists Pad
   // node that can be merged with 'm'.
   static Node* GetPadOrConv2D(const Node* m) {
-    CHECK_NOTNULL(m);
+    DCHECK(m);
     Node* n = nullptr;
 
     const Node* conv_node;
@@ -790,7 +790,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
         }
       }
     } else {
-      CHECK_EQ(m->type_string(), csinfo_.conv2d);
+      DCHECK_EQ(m->type_string(), csinfo_.conv2d);
       // If m is conv2D, Go over all input edges
       // and search for Pad  Node.
       for (const Edge* e : m->in_edges()) {
@@ -2182,7 +2182,7 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 
 Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
                                                 Node* m, Node* n) {
-  CHECK(((m->type_string() == csinfo_.pad &&
+  DCHECK(((m->type_string() == csinfo_.pad &&
           n->type_string() == csinfo_.conv2d)) ||
         ((n->type_string() == csinfo_.pad &&
           m->type_string() == csinfo_.conv2d)));
@@ -2250,7 +2250,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
       PadDataInputEdges++;
     }
   }
-  CHECK_EQ(PadDataInputEdges, 2);
+  DCHECK_EQ(PadDataInputEdges, 2);
 
   // Conv2D must have 2 data inputs: pad output and Filter
   int ConvDataInputEdges = 0;
@@ -2259,7 +2259,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
       ConvDataInputEdges++;
     }
   }
-  CHECK_EQ(ConvDataInputEdges, 2);
+  DCHECK_EQ(ConvDataInputEdges, 2);
 
   // We will use the node name of Conv2D as the name of new node
   // Build new node. We use same name as original node, but change the op
@@ -2282,7 +2282,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
   // Create node.
   Node* new_node;
   TF_CHECK_OK(nb.Finalize(&**g, &new_node));
-  CHECK_NOTNULL(new_node);
+  DCHECK(new_node);
 
   // Incoming data edges from 'pred' node and 'succ' node to new 'new_node'
   // node are already copied in BuildNode.
@@ -2319,8 +2319,8 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
       // Conv2D has only 1 output (at slot 0) and merged node also has only 1
       // output (at slot 0).
       const int kPadWithConv2DOutputSlot = 0;
-      CHECK_NOTNULL((*g)->AddEdge(new_node, kPadWithConv2DOutputSlot, e->dst(),
-                                  e->dst_input()));
+      (*g)->AddEdge(new_node, kPadWithConv2DOutputSlot, e->dst(),
+                                  e->dst_input());
     }
   }
 
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index fa059f1194..5ba98308ae 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -515,7 +515,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
 // _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
 // A1:control->E:control (only one control edge)
 TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A1' op: 'Input'}"
       "node { name: 'A' op: 'Input'}"
@@ -567,7 +567,7 @@ TEST_F(MklLayoutPassTest, Input_ControlEdge_PadWithConv2D_Positive) {
 // _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
 // E:control->A1:control (only one control edge)
 TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithConv2D_Positive) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A1' op: 'Input'}"
       "node { name: 'A' op: 'Input'}"
@@ -612,7 +612,7 @@ TEST_F(MklLayoutPassTest, Output_ControlEdge_PadWithConv2D_Positive) {
 // After layout pass
 // _MklPadWithConv2D(A, A, B, DMT/_0, DMT/_1, DMT/_2)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_Input) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A' op: 'Input'}"
       "node { name: 'B' op: 'Int32Input'}"
@@ -648,7 +648,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_Input) {
 // After layout pass - No merging, since Pad and Conv2D both
 // feed to the same node (Z)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_InOutput) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A' op: 'Input'}"
       "node { name: 'B' op: 'Int32Input'}"
@@ -678,7 +678,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Common_InOutput) {
 // C=Pad(A,B); E=Conv2D(C,D); Z=Zeta(E,Y)
 // After layout pass - No merging
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Negative) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A' op: 'Input'}"
       "node { name: 'B' op: 'Int32Input'}"
-- 
GitLab


From 2ff90ad8d1599b9c5207d813b5e07ac480d588ab Mon Sep 17 00:00:00 2001
From: mbhuiyan <mohammad.ashraf.bhuiyan@intel.com>
Date: Mon, 3 Dec 2018 09:45:37 -0800
Subject: [PATCH 1189/1554]  replacing CHECK BY DCHECK

---
 tensorflow/core/graph/mkl_layout_pass_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 88b592a5fc..082278ee3e 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -472,7 +472,7 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_ConvBpropInput_FilterFwd) {
 // After layout pass
 // _MklPadWithConv2D(A, D, B, DMT/_0, DMT/_1, DMT/_2)
 TEST_F(MklLayoutPassTest, NodeMerge_PadWithConv2D_Positive) {
-  CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+  DCHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
   InitGraph(
       "node { name: 'A' op: 'Input'}"
       "node { name: 'B' op: 'Int32Input'}"
-- 
GitLab


From da61525e1eca27c73c83547df973602e2b35a943 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 09:45:27 -0800
Subject: [PATCH 1190/1554] Use custom py_binary in
 tensorflow/contrib/eager/python/examples/densenet/BUILD

Address #22390#issuecomment-439610881

PiperOrigin-RevId: 223811309
---
 tensorflow/contrib/eager/python/examples/densenet/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index 2dc196f550..e2154fcc5f 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -3,6 +3,7 @@ licenses(["notice"])  # Apache 2.0
 package(default_visibility = ["//tensorflow:internal"])
 
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+load("//tensorflow:tensorflow.bzl", "py_binary")
 
 py_binary(
     name = "densenet",
-- 
GitLab


From 45c3bd5af035508ddadaf114e63ad8a01114d275 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 3 Dec 2018 09:52:48 -0800
Subject: [PATCH 1191/1554] Only update global Eigen thread count when OpenMP
 is enabled

PiperOrigin-RevId: 223812446
---
 tensorflow/lite/kernels/eigen_support.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/eigen_support.cc b/tensorflow/lite/kernels/eigen_support.cc
index 44e0086ad8..bad5975a7c 100644
--- a/tensorflow/lite/kernels/eigen_support.cc
+++ b/tensorflow/lite/kernels/eigen_support.cc
@@ -34,6 +34,15 @@ static_assert(
     "kDefaultArenaAlignment doesn't comply with Eigen alignment requirement.");
 #endif  // EIGEN_DONT_ALIGN
 
+// Helper routine for updating the global Eigen thread count used for OpenMP.
+void SetEigenNbThreads(int threads) {
+#if defined(EIGEN_HAS_OPENMP)
+  // The global Eigen thread count is only used when OpenMP is enabled. As this
+  // call causes problems with tsan, make it only when OpenMP is available.
+  Eigen::setNbThreads(context->recommended_num_threads);
+#endif  // defined(EIGEN_HAS_OPENMP)
+}
+
 // We have a single global threadpool for all convolution operations. This means
 // that inferences started from different threads may block each other, but
 // since the underlying resource of CPU cores should be consumed by the
@@ -78,7 +87,7 @@ void InitDevice(TfLiteContext* context, RefCountedEigenContext* ptr) {
 }
 
 TfLiteStatus Refresh(TfLiteContext* context) {
-  Eigen::setNbThreads(context->recommended_num_threads);
+  SetEigenNbThreads(context->recommended_num_threads);
 
   auto* ptr = GetEigenContext(context);
   if (ptr != nullptr) {
@@ -94,7 +103,7 @@ void IncrementUsageCounter(TfLiteContext* context) {
   auto* ptr = GetEigenContext(context);
   if (ptr == nullptr) {
     if (context->recommended_num_threads != -1) {
-      Eigen::setNbThreads(context->recommended_num_threads);
+      SetEigenNbThreads(context->recommended_num_threads);
     }
     ptr = new RefCountedEigenContext;
     ptr->type = kTfLiteEigenContext;
-- 
GitLab


From f63601af637c1520225e3c83d436611990c2ccbc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 09:53:21 -0800
Subject: [PATCH 1192/1554] Internal change.

PiperOrigin-RevId: 223812539
---
 tensorflow/python/feature_column/feature_column_v2.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 2af2b9f254..6308926494 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -4696,7 +4696,10 @@ def deserialize_feature_column(config,
         'Expected FeatureColumn class, instead found: {}'.format(cls))
 
   # Always deserialize the FeatureColumn, in order to get the name.
-  new_instance = cls._from_config(cls_config, columns_by_name=columns_by_name)  # pylint: disable=protected-access
+  new_instance = cls._from_config(  # pylint: disable=protected-access
+      cls_config,
+      custom_objects=custom_objects,
+      columns_by_name=columns_by_name)
 
   # If the name already exists, re-use the column from columns_by_name,
   # (new_instance remains unused).
-- 
GitLab


From 9c8b5bf6e73a3323c824b63acad380cc0aad735a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 3 Dec 2018 09:53:46 -0800
Subject: [PATCH 1193/1554] Skeleton loading for SavedModels in 2.x

Doesn't do anything useful yet, just a bunch of TODOs.

PiperOrigin-RevId: 223812626
---
 tensorflow/contrib/distribute/python/BUILD    |  1 +
 tensorflow/python/saved_model/BUILD           | 61 +++++++++++++++++--
 tensorflow/python/saved_model/constants.py    |  3 +
 tensorflow/python/saved_model/load.py         | 61 +++++++++++++++++++
 tensorflow/python/saved_model/load_test.py    | 51 ++++++++++++++++
 tensorflow/python/saved_model/save.py         | 18 ++++++
 .../saved_model/saved_object_graph.proto      | 38 ++++++++++++
 .../python/training/checkpointable/util.py    | 19 +++---
 8 files changed, 238 insertions(+), 14 deletions(-)
 create mode 100644 tensorflow/python/saved_model/load.py
 create mode 100644 tensorflow/python/saved_model/load_test.py
 create mode 100644 tensorflow/python/saved_model/saved_object_graph.proto

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 249258def3..4c9c35da5a 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -591,6 +591,7 @@ py_library(
         "//tensorflow/contrib/distribute/python:tpu_strategy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
+        "//tensorflow/python/eager:test",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/keras",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 31b5a59da2..16a31bee61 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -12,6 +12,8 @@ licenses(["notice"])  # Apache 2.0
 exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos")
 
 py_library(
     name = "saved_model",
@@ -21,6 +23,7 @@ py_library(
     deps = [
         ":builder",
         ":constants",
+        ":load",
         ":loader",
         ":main_op",
         ":save",
@@ -89,7 +92,7 @@ py_library(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:lib",
         "//tensorflow/python:platform",
-        "//tensorflow/python:training",
+        "//tensorflow/python:saver",
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
@@ -168,14 +171,15 @@ py_test(
         ":signature_def_utils",
         ":tag_constants",
         "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:lib",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:saver_test_utils",
+        "//tensorflow/python:session",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:test_ops",
         "//tensorflow/python:training",
@@ -266,6 +270,14 @@ py_test(
     ],
 )
 
+tf_proto_library(
+    name = "saved_object_graph",
+    srcs = ["saved_object_graph.proto"],
+    cc_api_version = 2,
+    protodeps = tf_additional_all_protos(),
+    visibility = ["//tensorflow:internal"],
+)
+
 py_library(
     name = "save",
     srcs = [
@@ -273,16 +285,24 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        ":loader",
+        ":constants",
+        ":saved_object_graph_py",
         ":signature_constants",
         ":signature_def_utils",
+        ":tag_constants",
         ":utils",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
+        "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:def_function",
-        "//tensorflow/python/eager:test",
+        "//tensorflow/python/eager:function",
+        "//tensorflow/python/training/checkpointable:base",
+        "//tensorflow/python/training/checkpointable:util",
     ],
 )
 
@@ -291,13 +311,42 @@ py_test(
     srcs = ["save_test.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":loader",
         ":save",
         ":signature_constants",
         ":tag_constants",
         "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/eager:test",
         "@absl_py//absl/testing:parameterized",
     ],
 )
 
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
+py_library(
+    name = "load",
+    srcs = [
+        "load.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":loader",
+        ":saved_object_graph_py",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:util",
+        "//tensorflow/python/training/checkpointable:tracking",
+    ],
+)
+
+py_test(
+    name = "load_test",
+    srcs = ["load_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":load",
+        ":save",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:tensor_spec",
+        "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python/training/checkpointable:tracking",
+    ],
+)
diff --git a/tensorflow/python/saved_model/constants.py b/tensorflow/python/saved_model/constants.py
index 0cd5588a8f..90511a409e 100644
--- a/tensorflow/python/saved_model/constants.py
+++ b/tensorflow/python/saved_model/constants.py
@@ -29,6 +29,9 @@ tf_export(
         "saved_model.ASSETS_DIRECTORY", "saved_model.constants.ASSETS_DIRECTORY"
     ]).export_constant(__name__, "ASSETS_DIRECTORY")
 
+# Subdirectory name containing unmanaged files from higher-level APIs.
+EXTRA_ASSETS_DIRECTORY = "assets.extra"
+
 # CollectionDef key containing SavedModel assets.
 ASSETS_KEY = "saved_model_assets"
 tf_export(
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
new file mode 100644
index 0000000000..397086e106
--- /dev/null
+++ b/tensorflow/python/saved_model/load.py
@@ -0,0 +1,61 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Import a checkpointable object from a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.python.lib.io import file_io
+from tensorflow.python.saved_model import constants
+from tensorflow.python.saved_model import saved_object_graph_pb2
+from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.util import compat
+
+
+def _recreate_object_graph(object_graph_proto):
+  """Recreates Python objects from an ObjectGraph proto."""
+  objects = []
+  for _ in object_graph_proto.nodes:
+    # TODO(allenl): re-create variables and other types
+    objects.append(tracking.Checkpointable())
+  for obj, object_proto in zip(objects, object_graph_proto.nodes):
+    for reference in object_proto.children:
+      setattr(obj, reference.local_name, objects[reference.node_id])
+  return objects[0]
+
+
+def load(export_dir):
+  """Load a SavedModel from `export_dir`."""
+  object_graph_filename = os.path.join(
+      compat.as_bytes(export_dir),
+      compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY),
+      compat.as_bytes("object_graph.pb"))
+  if file_io.file_exists(object_graph_filename):
+    # If there is an object graph associated with the SavedModel, we'll create a
+    # root object from that.
+    object_graph_string = file_io.FileIO(object_graph_filename, "rb").read()
+    object_graph_proto = (
+        saved_object_graph_pb2.SavedObjectGraph())
+    object_graph_proto.ParseFromString(object_graph_string)
+    root = _recreate_object_graph(object_graph_proto)
+  else:
+    raise NotImplementedError(
+        "Currently only SavedModels exported with `tf.saved_model.save` may be "
+        "imported. Other SavedModels may eventually be supported via load().")
+  # TODO(allenl): load functions from the SavedModel into the eager context
+  return root
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
new file mode 100644
index 0000000000..bfa201ad63
--- /dev/null
+++ b/tensorflow/python/saved_model/load_test.py
@@ -0,0 +1,51 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for checkpointable object SavedModel loading."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.python.eager import def_function
+from tensorflow.python.eager import test
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import tensor_spec
+from tensorflow.python.saved_model import load
+from tensorflow.python.saved_model import save
+from tensorflow.python.training.checkpointable import tracking
+
+
+class LoadTest(test.TestCase):
+
+  def test_structure_import(self):
+    root = tracking.Checkpointable()
+    root.f = def_function.function(
+        lambda x: 2. * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+    root.dep_one = tracking.Checkpointable()
+    root.dep_two = tracking.Checkpointable()
+    root.dep_two.dep = tracking.Checkpointable()
+    root.dep_three = root.dep_two.dep
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    save.save(root, save_dir)
+    imported = load.load(save_dir)
+    self.assertIs(imported.dep_three, imported.dep_two.dep)
+    self.assertIsNot(imported.dep_one, imported.dep_two)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index d52251e49f..bc3645fa38 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -32,6 +32,7 @@ from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.saved_model import constants
+from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.saved_model import tag_constants
@@ -400,6 +401,22 @@ def _make_graph_def(root, signature_functions, object_saver):
   return graph_def, signatures, saver_def
 
 
+def _write_object_graph(obj, export_dir):
+  """Save a SavedObjectGraph proto for `obj`."""
+  # SavedObjectGraph is similar to the CheckpointableObjectGraph proto in the
+  # checkpoint. It will eventually go into the SavedModel.
+  object_proto = util.make_object_graph_without_attributes(
+      obj, proto=saved_object_graph_pb2.SavedObjectGraph())
+  extra_asset_dir = os.path.join(
+      compat.as_bytes(export_dir),
+      compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY))
+  file_io.recursive_create_dir(extra_asset_dir)
+  object_graph_filename = os.path.join(
+      extra_asset_dir, compat.as_bytes("object_graph.pb"))
+  file_io.write_string_to_file(object_graph_filename,
+                               object_proto.SerializeToString())
+
+
 @tf_export("saved_model.save", v1=["saved_model.experimental.save"])
 def save(obj, export_dir, signatures=None):
   # pylint: disable=line-too-long
@@ -580,3 +597,4 @@ def save(obj, export_dir, signatures=None):
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))
   file_io.write_string_to_file(path, saved_model.SerializeToString())
+  _write_object_graph(obj, export_dir)
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
new file mode 100644
index 0000000000..89f82b90d6
--- /dev/null
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+import "tensorflow/core/protobuf/checkpointable_object_graph.proto";
+
+option cc_enable_arenas = true;
+
+package tensorflow;
+
+// A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It
+// describes the directed graph of Python objects (or equivalent in other
+// languages) that make up a model, with nodes[0] at the root.
+
+// SavedObjectGraph shares some structure with CheckpointableObjectGraph, but
+// ObjectGraph belongs to the SavedModel and contains pointers to functions and
+// type information, while CheckpointableObjectGraph lives in the checkpoint and
+// contains pointers only to variable values.
+
+// NOTE: This protocol buffer format is experimental and subject to change.
+
+message SavedObjectGraph {
+  message SavedObject {
+    // Objects which this object depends on: named edges in the dependency
+    // graph.
+    repeated CheckpointableObjectGraph.CheckpointableObject.ObjectReference
+        children = 1;
+    // Removed when forking from CheckpointableObjectGraph.
+    reserved "attributes";
+    reserved 2;
+    // Slot variables owned by this object. This describes the three-way
+    // (optimizer, variable, slot variable) relationship; none of the three
+    // depend on the others directly.
+    repeated
+        CheckpointableObjectGraph.CheckpointableObject.SlotVariableReference
+            slot_variables = 3;
+  }
+
+  repeated SavedObject nodes = 1;
+}
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 394cc33ad0..36d3a7bebd 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -648,10 +648,12 @@ def _add_attributes_to_object_graph(
   return named_saveable_objects, feed_additions
 
 
-def _make_object_graph_proto(checkpointable_objects, node_ids, slot_variables):
+def _fill_object_graph_proto(checkpointable_objects, node_ids, slot_variables,
+                             object_graph_proto=None):
   """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
-  object_graph_proto = (
-      checkpointable_object_graph_pb2.CheckpointableObjectGraph())
+  if object_graph_proto is None:
+    object_graph_proto = (
+        checkpointable_object_graph_pb2.CheckpointableObjectGraph())
   for checkpoint_id, checkpointable in enumerate(checkpointable_objects):
     assert node_ids[checkpointable] == checkpoint_id
     object_proto = object_graph_proto.nodes.add()
@@ -676,7 +678,7 @@ def _serialize_gathered_objects(
       checkpointable_objects=checkpointable_objects,
       node_ids=node_ids,
       object_names=object_names)
-  object_graph_proto = _make_object_graph_proto(
+  object_graph_proto = _fill_object_graph_proto(
       checkpointable_objects=checkpointable_objects,
       node_ids=node_ids,
       slot_variables=slot_variables)
@@ -764,12 +766,12 @@ def list_objects(root_checkpointable):
   return checkpointable_objects
 
 
-def make_object_graph_without_attributes(root_checkpointable):
-  """Construct a CheckpointableObjectGraph proto with no variable values."""
+def make_object_graph_without_attributes(root_checkpointable, proto=None):
+  """Fill an object graph proto, ignoring variable values."""
   checkpointable_objects, node_ids, slot_variables = _find_objects(
       root_checkpointable)
-  return _make_object_graph_proto(
-      checkpointable_objects, node_ids, slot_variables)
+  return _fill_object_graph_proto(
+      checkpointable_objects, node_ids, slot_variables, proto)
 
 
 def gather_initializers(root_checkpointable):
@@ -1924,3 +1926,4 @@ class Checkpoint(tracking.Checkpointable):
     # initialization when executing eagerly.
     self._maybe_create_save_counter()
     return status
+
-- 
GitLab


From 62db4a3ccf04d599c8a9067436c8629f1b7d1e9d Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 3 Dec 2018 09:56:03 -0800
Subject: [PATCH 1194/1554] Introduce Operation._add_while_inputs to allow
 adding inputs to a While op.

This is in preparation for changing while_v2 to rewrite the forward
pass to output intermediates needed by the gradient, instead of
outputting all intermediates. Since While ops always have the same
inputs and output types, we need to be able to add inputs in addition
to adding outputs.

PiperOrigin-RevId: 223812986
---
 tensorflow/c/python_api.cc              | 13 +++++++++++++
 tensorflow/c/python_api.h               |  8 ++++++++
 tensorflow/core/graph/graph.cc          | 16 ++++++++++++++++
 tensorflow/core/graph/graph.h           |  6 ++++++
 tensorflow/python/BUILD                 |  1 -
 tensorflow/python/framework/ops.py      | 25 +++++++++++++++++++++++++
 tensorflow/python/framework/ops_test.py | 25 +++++++++++++++++++++++++
 7 files changed, 93 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc
index 247236b760..98d8393332 100644
--- a/tensorflow/c/python_api.cc
+++ b/tensorflow/c/python_api.cc
@@ -160,4 +160,17 @@ void SetHandleShapeAndType(TF_Graph* graph, TF_Output output, const void* proto,
   ic->set_output_handle_shapes_and_types(output.index, shapes_and_types);
 }
 
+void AddWhileInputHack(TF_Graph* graph, TF_Output new_src, TF_Operation* dst,
+                       TF_Status* status) {
+  mutex_lock l(graph->mu);
+  status->status = graph->graph.AddWhileInputHack(&new_src.oper->node,
+                                                  new_src.index, &dst->node);
+  if (status->status.ok()) {
+    // This modification only updates the destination node for
+    // the purposes of running this graph in a session. Thus, we don't
+    // record the source node as being modified.
+    RecordMutation(graph, *dst, "adding input tensor");
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/c/python_api.h b/tensorflow/c/python_api.h
index 5cce84020b..44779ca656 100644
--- a/tensorflow/c/python_api.h
+++ b/tensorflow/c/python_api.h
@@ -34,6 +34,7 @@ void SetAttr(TF_Graph* graph, TF_Operation* op, const char* attr_name,
 
 void SetRequestedDevice(TF_Graph* graph, TF_Operation* op, const char* device);
 
+// Updates 'dst' to consume 'new_src'.
 void UpdateEdge(TF_Graph* graph, TF_Output new_src, TF_Input dst,
                 TF_Status* status);
 
@@ -65,6 +66,13 @@ std::string GetHandleShapeAndType(TF_Graph* graph, TF_Output output);
 // because I couldn't get SWIG to work otherwise.
 void SetHandleShapeAndType(TF_Graph* graph, TF_Output output, const void* proto,
                            size_t proto_len, TF_Status* status);
+
+// This method is used to add a new input edge to 'dst', which must be a While
+// op. The While op's "T" attribute must have already been updated to include
+// the new edge. This is used to construct tf.while_loop gradients.
+void AddWhileInputHack(TF_Graph* graph, TF_Output new_src, TF_Operation* dst,
+                       TF_Status* status);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_C_PYTHON_API_H_
diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index 466310d874..550e3ef915 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -548,6 +548,22 @@ Status Graph::UpdateEdge(Node* new_src, int new_src_index, Node* dst,
   return Status::OK();
 }
 
+Status Graph::AddWhileInputHack(Node* new_src, int new_src_index, Node* dst) {
+  if (dst->type_string() != "While") {
+    return errors::Internal(
+        "dst argument to AddWhileEdgeHack should be a While op, got: ",
+        dst->DebugString());
+  }
+  TF_RETURN_IF_ERROR(IsValidOutputTensor(new_src, new_src_index));
+  int dst_index = dst->in_edges().size();
+  TF_RETURN_IF_ERROR(IsValidInputTensor(dst, dst_index));
+  AddEdge(new_src, new_src_index, dst, dst_index);
+  dst->MaybeCopyOnWrite();
+  dst->props_->node_def.add_input(
+      strings::StrCat(new_src->name(), ":", new_src_index));
+  return Status::OK();
+}
+
 Status Graph::AddFunctionLibrary(const FunctionDefLibrary& fdef_lib) {
   // Need a new-enough consumer to support the functions we add to the graph.
   if (fdef_lib.function_size() > 0 && versions_->min_consumer() < 12) {
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index af0b123706..667eaba24c 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -493,11 +493,17 @@ class Graph {
   // the corresponding NodeDef to reflect the change.
   // REQUIRES: The control edge must exist.
   void RemoveControlEdge(const Edge* e);
+
   // Updates the input to a node.  The existing edge to `dst` is removed and an
   // edge from `new_src` to `dst` is created. The NodeDef associated with `dst`
   // is also updated.
   Status UpdateEdge(Node* new_src, int new_src_index, Node* dst, int dst_index);
 
+  // Like AddEdge but updates dst's NodeDef. Used to add an input edge to a
+  // "While" op during gradient construction, see AddInputWhileHack in
+  // python_api.h for more details.
+  Status AddWhileInputHack(Node* new_src, int new_src_index, Node* dst);
+
   // Adds the function and gradient definitions in `fdef_lib` to this graph's op
   // registry. Ignores duplicate functions, and returns a bad status if an
   // imported function differs from an existing function or op with the same
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 64a3b413cc..17e228a562 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2183,7 +2183,6 @@ py_library(
         ":control_flow_util_v2",
         ":dtypes",
         ":framework_ops",
-        ":framework_test_lib",
         ":function_def_to_graph",
         ":functional_ops_gen",
         ":gradients_impl",
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 31986fbc41..0c30e43cbb 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2086,6 +2086,31 @@ class Operation(object):
         tensor._as_tf_output(),  # pylint: disable=protected-access
         self._tf_input(index))
 
+  def _add_while_inputs(self, tensors):
+    """See AddWhileInputHack in python_api.h.
+
+    NOTE: This is for TF internal use only. Please don't use it.
+
+    Args:
+      tensors: list of Tensors
+
+    Raises:
+      TypeError: if tensor is not a Tensor,
+        or if input tensor type is not convertible to dtype.
+      ValueError: if the Tensor is from a different graph.
+    """
+    for tensor in tensors:
+      if not isinstance(tensor, Tensor):
+        raise TypeError("tensor must be a Tensor: %s" % tensor)
+      _assert_same_graph(self, tensor)
+
+      # Reset cached inputs.
+      self._inputs_val = None
+      c_api.AddWhileInputHack(
+          self._graph._c_graph,  # pylint: disable=protected-access
+          tensor._as_tf_output(),  # pylint: disable=protected-access
+          self._c_op)
+
   def _add_control_inputs(self, ops):
     """Add a list of new control inputs to this operation.
 
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index c957ffaecf..f60c568f32 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -604,6 +604,31 @@ class OperationTest(test_util.TensorFlowTestCase):
     ):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
+  @test_util.enable_control_flow_v2
+  def testAddWhileInput(self):
+    @eager_function.defun
+    def test():
+      output = control_flow_ops.while_loop(lambda x: x < 3, lambda x: x + 1,
+                                           [1])
+      while_op = output.op.inputs[0].op
+      self.assertEqual(while_op.type, "While")
+      orig_num_inputs = len(while_op.inputs)
+
+      new_input1 = constant_op.constant(1.0)
+      new_input2 = constant_op.constant(True)
+
+      while_op._set_type_list_attr("T",
+                                   [t.dtype for t in while_op.inputs] +
+                                   [new_input1.dtype, new_input2.dtype])
+
+      while_op._add_while_inputs([new_input1, new_input2])
+      # Can't add an edge beyond what's specified by "T"
+      with self.assertRaises(errors.OutOfRangeError):
+        while_op._add_while_inputs([new_input2])
+      self.assertEqual(len(while_op.inputs), orig_num_inputs + 2)  # pylint: disable=g-deprecated-assert
+
+    test()
+
   @test_util.run_deprecated_v1
   def testOpDef(self):
     x = constant_op.constant(0)
-- 
GitLab


From 686b8afd8ba74b95d0533076e215c8708ddec422 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Mon, 3 Dec 2018 09:57:26 -0800
Subject: [PATCH 1195/1554] [XLA] Support bitcast in the HLO Evaluator.

PiperOrigin-RevId: 223813266
---
 tensorflow/compiler/xla/service/BUILD         |  2 ++
 .../compiler/xla/service/hlo_evaluator.cc     | 11 +++++++
 .../compiler/xla/service/hlo_evaluator.h      |  2 ++
 .../xla/service/hlo_evaluator_test.cc         | 29 +++++++++++++++++++
 4 files changed, 44 insertions(+)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 0907593ce7..1666227134 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -281,10 +281,12 @@ tf_cc_test(
         "//tensorflow/compiler/xla/service:hlo_element_type_converter",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",  # fixdeps: keep
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 51a3fba176..3a7652a8dc 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_query.h"
 #include "tensorflow/compiler/xla/service/shape_inference.h"
 #include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/window_util.h"
@@ -396,6 +397,16 @@ StatusOr<Literal> HloEvaluator::EvaluateDotOp(
   return Evaluate(cloned_instruction.get());
 }
 
+Status HloEvaluator::HandleBitcast(HloInstruction* bitcast) {
+  const Literal& operand_literal = GetEvaluatedLiteralFor(bitcast->operand(0));
+  Literal result(bitcast->shape());
+  TF_RET_CHECK(operand_literal.size_bytes() == result.size_bytes());
+  memcpy(result.untyped_data(), operand_literal.untyped_data(),
+         operand_literal.size_bytes());
+  evaluated_[bitcast] = std::move(result);
+  return Status::OK();
+}
+
 Status HloEvaluator::HandleParameter(HloInstruction* parameter) {
   CHECK_LT(parameter->parameter_number(), arg_literals_.size());
   const Literal* input_literal = arg_literals_[parameter->parameter_number()];
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index d847900010..45ed8131dc 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -144,6 +144,8 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // Operations that are type-agnostic or always return a specific type, such as
   // HandleIsFinite where boolean is always returned.
   //
+  Status HandleBitcast(HloInstruction* bitcast) override;
+
   Status HandleParameter(HloInstruction* parameter) override;
 
   Status HandleConstant(HloInstruction* constant) override;
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index d95b6ad04f..4eaaab20ea 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <vector>
 
 #include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/reference_util.h"
@@ -35,6 +36,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
+#include "tensorflow/compiler/xla/tests/test_utils.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -2765,6 +2767,33 @@ ENTRY main {
   EXPECT_TRUE(LiteralTestUtil::Equal(arg, actual));
 }
 
+TEST_P(HloEvaluatorTest, Bitcast) {
+  // Regression test for b/114735354.
+  constexpr absl::string_view hlo_text_base = R"(
+HloModule Bitcast
+
+ENTRY main {
+  param = %s[32,121]{1,0} parameter(0)
+  ROOT bitcast = %s[121,32,1]{0,1,2} bitcast(%s[32,121]{1,0} param)
+}
+)";
+  string hlo_text;
+  if (use_bfloat16_) {
+    hlo_text = absl::StrFormat(hlo_text_base, "bf16", "bf16", "bf16");
+  } else {
+    hlo_text = absl::StrFormat(hlo_text_base, "f32", "f32", "f32");
+  }
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual = Evaluate({&args[0]});
+  if (use_bfloat16_) {
+    EXPECT_TRUE(
+        absl::c_equal(args[0].data<bfloat16>(), actual.data<bfloat16>()));
+  } else {
+    EXPECT_TRUE(absl::c_equal(args[0].data<float>(), actual.data<float>()));
+  }
+}
+
 INSTANTIATE_TEST_CASE_P(HloEvaluatorTest_Instantiation, HloEvaluatorTest,
                         ::testing::ValuesIn(use_bf16_params));
 
-- 
GitLab


From c8e908002e7839acb6b95b85c8bf4a0723e5e4d4 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 3 Dec 2018 09:58:13 -0800
Subject: [PATCH 1196/1554] Disabling auto-tuning for CopyToDeviceDataset since
 ModelDataset cannot currently be placed on a GPU. This was causing the placer
 to place everything on the CPU instead.

PiperOrigin-RevId: 223813375
---
 .../kernel_tests/copy_to_device_test.py       | 53 +++++++++++--------
 .../kernel_tests/prefetch_to_device_test.py   |  9 ++--
 .../data/experimental/ops/prefetching_ops.py  |  9 ++--
 3 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index b0516573f5..1acd26270f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -56,7 +56,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -83,7 +83,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual((4,), next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
@@ -109,7 +109,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -136,7 +136,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -163,7 +163,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element["a"].shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -190,7 +190,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element["a"].shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         self.assertEqual({"a": i}, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -222,7 +222,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(dtypes.int64, next_element.dtype)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
@@ -257,7 +257,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual(dtypes.int64, next_element.dtype)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       for i in range(10):
         actual = self.evaluate(next_element)
         self.assertAllEqual([i], actual.values)
@@ -278,7 +278,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
@@ -297,7 +298,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
@@ -330,7 +332,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       for i in range(10):
         x, y, z = self.evaluate(next_element)
@@ -352,7 +355,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -370,7 +374,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       self.assertAllEqual([0, 1, 2, 3], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -388,7 +393,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -406,7 +412,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       self.assertAllEqual([b"a", b"b", b"c"], self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -427,7 +434,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         iterator = back_to_cpu_dataset.make_initializable_iterator()
         next_element = iterator.get_next()
 
-      with self.cached_session() as sess:
+      with self.cached_session(
+          config=config_pb2.ConfigProto(allow_soft_placement=False)):
         self.evaluate(iterator.initializer)
         for i in range(10):
           self.assertEqual(i, self.evaluate(next_element))
@@ -455,7 +463,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
@@ -486,7 +494,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     self.assertEqual([], next_element.shape)
 
     worker_config = config_pb2.ConfigProto(device_count={"CPU": 2})
-    with self.test_session(config=worker_config) as sess:
+    with self.test_session(config=worker_config):
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
@@ -508,7 +516,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
@@ -530,7 +539,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       iterator = device_dataset.make_initializable_iterator()
       next_element = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
@@ -553,7 +563,8 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
       elem_has_value_t = next_elem.has_value()
       elem_value_t = next_elem.get_value()
 
-    with self.cached_session() as sess:
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       # Before initializing the iterator, evaluating the optional fails with
       # a FailedPreconditionError.
       with self.assertRaises(errors.FailedPreconditionError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index c64bb2d299..d280321829 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -153,10 +153,12 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/gpu:0"))
 
-    iterator = device_dataset.make_one_shot_iterator()
+    iterator = device_dataset.make_initializable_iterator()
     next_element = iterator.get_next()
 
-    with self.cached_session():
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
+      self.evaluate(iterator.initializer)
       for i in range(10):
         self.assertEqual(i, self.evaluate(next_element))
       with self.assertRaises(errors.OutOfRangeError):
@@ -204,7 +206,8 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     iterator = device_dataset.make_initializable_iterator()
     next_element = iterator.get_next()
 
-    with self.cached_session():
+    with self.cached_session(
+        config=config_pb2.ConfigProto(allow_soft_placement=False)):
       self.evaluate(iterator.initializer)
       for i in range(5):
         self.assertEqual(i, self.evaluate(next_element))
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 9e3ca56929..5560d1f750 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -51,8 +51,8 @@ def prefetch_to_device(device, buffer_size=None):
     `tf.data.Dataset.apply`.
   """
   def _apply_fn(dataset):
-    return _CopyToDeviceDataset(
-        dataset, target_device=device).prefetch(buffer_size)
+    return dataset.apply(
+        copy_to_device(target_device=device)).prefetch(buffer_size)
 
   return _apply_fn
 
@@ -71,8 +71,11 @@ def copy_to_device(target_device, source_device="/cpu:0"):
   """
 
   def _apply_fn(dataset):
+    options = dataset_ops.Options()
+    options.experimental_autotune = False
     return _CopyToDeviceDataset(
-        dataset, target_device=target_device, source_device=source_device)
+        dataset, target_device=target_device,
+        source_device=source_device).with_options(options)
 
   return _apply_fn
 
-- 
GitLab


From 97ffac50fe31f99a64983b1947b22589df15dbc6 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 3 Dec 2018 10:07:45 -0800
Subject: [PATCH 1197/1554] Split the flaky test_implementation_mode into 3
 tests.

This test case usually took too long to execute, and cause that test shard to time out. Use paramsterized test so that each implementation mode are now split into separate test case.

PiperOrigin-RevId: 223815273
---
 tensorflow/python/keras/BUILD                 |  2 ++
 tensorflow/python/keras/layers/lstm_test.py   | 17 ++++++++-------
 .../python/keras/layers/unified_lstm_test.py  | 21 ++++++++++---------
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 228f47961c..64ca3ac4e0 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -499,6 +499,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -520,6 +521,7 @@ cuda_py_test(
     srcs = ["layers/unified_lstm_test.py"],
     additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index 732bbcfa18..118c4f86c2 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -30,7 +31,7 @@ from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @tf_test_util.run_all_in_graph_and_eager_modes
-class LSTMLayerTest(test.TestCase):
+class LSTMLayerTest(test.TestCase, parameterized.TestCase):
 
   def test_return_sequences_LSTM(self):
     num_samples = 2
@@ -83,17 +84,17 @@ class LSTMLayerTest(test.TestCase):
                 'recurrent_dropout': 0.1},
         input_shape=(num_samples, timesteps, embedding_dim))
 
-  def test_implementation_mode_LSTM(self):
+  @parameterized.parameters([0, 1, 2])
+  def test_implementation_mode_LSTM(self, implementation_mode):
     num_samples = 2
     timesteps = 3
     embedding_dim = 4
     units = 2
-    for mode in [0, 1, 2]:
-      testing_utils.layer_test(
-          keras.layers.LSTM,
-          kwargs={'units': units,
-                  'implementation': mode},
-          input_shape=(num_samples, timesteps, embedding_dim))
+    testing_utils.layer_test(
+        keras.layers.LSTM,
+        kwargs={'units': units,
+                'implementation': implementation_mode},
+        input_shape=(num_samples, timesteps, embedding_dim))
 
   def test_constraints_LSTM(self):
     embedding_dim = 4
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 9bd1062ec6..00dd3f0c04 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import time
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
@@ -51,7 +52,7 @@ _graph_options = config_pb2.GraphOptions(rewrite_options=_rewrites)
 _config = config_pb2.ConfigProto(graph_options=_graph_options)
 
 
-class UnifiedLSTMTest(test.TestCase):
+class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
 
   @test_util.run_deprecated_v1
   def test_unifiedLSTM(self):
@@ -242,20 +243,20 @@ class UnifiedLSTMTest(test.TestCase):
         },
         input_shape=(num_samples, timesteps, embedding_dim))
 
+  @parameterized.parameters([0, 1, 2])
   @test_util.run_in_graph_and_eager_modes(config=_config)
-  def test_implementation_mode_LSTM(self):
+  def test_implementation_mode_LSTM(self, implementation_mode):
     num_samples = 2
     timesteps = 3
     embedding_dim = 4
     units = 2
-    for mode in [1, 2]:
-      testing_utils.layer_test(
-          UnifiedLSTM,
-          kwargs={
-              'units': units,
-              'implementation': mode
-          },
-          input_shape=(num_samples, timesteps, embedding_dim))
+    testing_utils.layer_test(
+        UnifiedLSTM,
+        kwargs={
+            'units': units,
+            'implementation': implementation_mode
+        },
+        input_shape=(num_samples, timesteps, embedding_dim))
 
   @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_constraints_LSTM(self):
-- 
GitLab


From 12a97c3917ec1afb42d402892ba7259c71b6eed3 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 3 Dec 2018 10:08:23 -0800
Subject: [PATCH 1198/1554] [tf.contrib.bigtable] Switch to returning DatasetV1
 instances from API methods.

This change is necessary before the V1 and V2 interfaces diverge any further.

PiperOrigin-RevId: 223815371
---
 .../contrib/bigtable/python/ops/bigtable_api.py  | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
index 7c87b0daeb..9f97934193 100644
--- a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
+++ b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
@@ -222,7 +222,7 @@ class BigtableTable(object):
       A `tf.data.Dataset`. containing `tf.string` Tensors corresponding to all
       of the row keys matching that prefix.
     """
-    return _BigtablePrefixKeyDataset(self, prefix)
+    return dataset_ops.DatasetV1Adapter(_BigtablePrefixKeyDataset(self, prefix))
 
   def sample_keys(self):
     """Retrieves a sampling of row keys from the Bigtable table.
@@ -234,7 +234,7 @@ class BigtableTable(object):
     Returns:
       A `tf.data.Dataset` returning string row keys.
     """
-    return _BigtableSampleKeysDataset(self)
+    return dataset_ops.DatasetV1Adapter(_BigtableSampleKeysDataset(self))
 
   def scan_prefix(self, prefix, probability=None, columns=None, **kwargs):
     """Retrieves row (including values) from the Bigtable service.
@@ -279,7 +279,8 @@ class BigtableTable(object):
     """
     probability = _normalize_probability(probability)
     normalized = _normalize_columns(columns, kwargs)
-    return _BigtableScanDataset(self, prefix, "", "", normalized, probability)
+    return dataset_ops.DatasetV1Adapter(
+        _BigtableScanDataset(self, prefix, "", "", normalized, probability))
 
   def scan_range(self, start, end, probability=None, columns=None, **kwargs):
     """Retrieves rows (including values) from the Bigtable service.
@@ -324,7 +325,8 @@ class BigtableTable(object):
     """
     probability = _normalize_probability(probability)
     normalized = _normalize_columns(columns, kwargs)
-    return _BigtableScanDataset(self, "", start, end, normalized, probability)
+    return dataset_ops.DatasetV1Adapter(
+        _BigtableScanDataset(self, "", start, end, normalized, probability))
 
   def parallel_scan_prefix(self,
                            prefix,
@@ -380,7 +382,8 @@ class BigtableTable(object):
     """
     probability = _normalize_probability(probability)
     normalized = _normalize_columns(columns, kwargs)
-    ds = _BigtableSampleKeyPairsDataset(self, prefix, "", "")
+    ds = dataset_ops.DatasetV1Adapter(
+        _BigtableSampleKeyPairsDataset(self, prefix, "", ""))
     return self._make_parallel_scan_dataset(ds, num_parallel_scans, probability,
                                             normalized)
 
@@ -442,7 +445,8 @@ class BigtableTable(object):
     """
     probability = _normalize_probability(probability)
     normalized = _normalize_columns(columns, kwargs)
-    ds = _BigtableSampleKeyPairsDataset(self, "", start, end)
+    ds = dataset_ops.DatasetV1Adapter(
+        _BigtableSampleKeyPairsDataset(self, "", start, end))
     return self._make_parallel_scan_dataset(ds, num_parallel_scans, probability,
                                             normalized)
 
-- 
GitLab


From dab4e7a725b0c92d5bad8dca683ba055d4a66584 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 3 Dec 2018 10:19:16 -0800
Subject: [PATCH 1199/1554] [XLA:Python] Run computations on the current thread
 if there is only one replica.

Avoids creating a thread pool (expensive) in the common single-replica case.

PiperOrigin-RevId: 223817129
---
 .../xla/python/local_computation_builder.cc   | 87 ++++++++++---------
 1 file changed, 47 insertions(+), 40 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index ac1c623882..c1381779c9 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -330,48 +330,55 @@ StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::Execute(
   // Each replica populates a StatusOr result, but only the output value of
   // replica zero is returned.
   std::vector<StatusOr<ScopedShapedBuffer>> results(num_replicas);
-  {
+  auto execute = [this, client, num_replicas, &argument_handles,
+                  &results](int replica) {
+    StatusOr<int> device_ordinal_status =
+        client->ReplicaNumberToDeviceOrdinal(replica);
+    if (!device_ordinal_status.ok()) {
+      results[replica] = device_ordinal_status.status();
+      return;
+    }
+    const int device_ordinal = device_ordinal_status.ValueOrDie();
+    VLOG(3) << "Replica " << replica
+            << " mapped to device ordinal for execution: " << device_ordinal;
+
+    std::vector<const ShapedBuffer*> argument_buffers;
+    argument_buffers.reserve(argument_handles[replica].size());
+    for (auto& handle : argument_handles[replica]) {
+      argument_buffers.push_back(handle->shaped_buffer());
+    }
+
+    DeviceAssignment device_assignment =
+        client->backend()
+            .computation_placer()
+            ->AssignDevices(num_replicas, /*computation_count=*/1)
+            .ConsumeValueOrDie();
+
+    ExecutableRunOptions options;
+    options.set_device_ordinal(device_ordinal);
+    options.set_allocator(client->backend().memory_allocator());
+    options.set_intra_op_thread_pool(
+        client->backend().eigen_intra_op_thread_pool_device());
+    options.set_device_assignment(&device_assignment);
+    StatusOr<ScopedShapedBuffer> result_buffer_status =
+        executable_->Run(argument_buffers, options);
+
+    results[replica] = std::move(result_buffer_status);
+  };
+
+  if (num_replicas == 1) {
+    // Fast-path if there is only one replica — run the computation on the
+    // current thread.
+    execute(0);
+  } else {
+    // TODO(phawkins): don't recreate the threadpool for each execution.
     tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "xlarun",
-                                        num_replicas);
-
-    for (int replica = 0; replica < num_replicas; ++replica) {
-      pool.Schedule(
-          [this, client, replica, num_replicas, &argument_handles, &results] {
-            StatusOr<int> device_ordinal_status =
-                client->ReplicaNumberToDeviceOrdinal(replica);
-            if (!device_ordinal_status.ok()) {
-              results[replica] = device_ordinal_status.status();
-              return;
-            }
-            const int device_ordinal = device_ordinal_status.ValueOrDie();
-            VLOG(3) << "Replica " << replica
-                    << " mapped to device ordinal for execution: "
-                    << device_ordinal;
-
-            std::vector<const ShapedBuffer*> argument_buffers;
-            argument_buffers.reserve(argument_handles[replica].size());
-            for (auto& handle : argument_handles[replica]) {
-              argument_buffers.push_back(handle->shaped_buffer());
-            }
-
-            DeviceAssignment device_assignment =
-                client->backend()
-                    .computation_placer()
-                    ->AssignDevices(num_replicas, /*computation_count=*/1)
-                    .ConsumeValueOrDie();
-
-            ExecutableRunOptions options;
-            options.set_device_ordinal(device_ordinal);
-            options.set_allocator(client->backend().memory_allocator());
-            options.set_intra_op_thread_pool(
-                client->backend().eigen_intra_op_thread_pool_device());
-            options.set_device_assignment(&device_assignment);
-            StatusOr<ScopedShapedBuffer> result_buffer_status =
-                executable_->Run(argument_buffers, options);
-
-            results[replica] = std::move(result_buffer_status);
-          });
+                                        num_replicas - 1);
+
+    for (int replica = 0; replica < num_replicas - 1; ++replica) {
+      pool.Schedule([&execute, replica] { execute(replica); });
     }
+    execute(num_replicas - 1);
   }
 
   std::vector<LocalShapedBuffer*> wrapped_results(num_replicas);
-- 
GitLab


From f8f17cebcc89921b5f1a204dab1f9fdb47b120c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 10:30:47 -0800
Subject: [PATCH 1200/1554] Remove python-side colocation device check that
 only matches on strings instead of actual devices.

PiperOrigin-RevId: 223818954
---
 tensorflow/python/framework/ops.py      | 40 ++-----------------------
 tensorflow/python/framework/ops_test.py | 37 -----------------------
 2 files changed, 3 insertions(+), 74 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 0c30e43cbb..1a26984809 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -43,7 +43,6 @@ from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import error_interpolation
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import op_def_registry
 from tensorflow.python.framework import registry
@@ -3331,36 +3330,6 @@ class Graph(object):
     self._create_op_helper(ret, compute_device=compute_device)
     return ret
 
-  def _make_colocation_conflict_message(self, op, colocation_op):
-    """Return detailed error message about device conflict due to colocation."""
-    # Example error message:
-    #   Tried to colocate op 'a' (defined at file1.py:149) having device
-    #   '/device:GPU:0' with op 'b' (defined at file2:96) which had an
-    #   incompatible device '/device:CPU:0'.
-    #
-    #   No node-device colocations were active during op 'a' creation.
-    #   Device assignments active during op 'a' creation:
-    #     with tf.device(/device:GPU:0): file1.py:148>
-    #
-    #   Node-device colocations active during op 'b' creation:
-    #     with tf.colocate_with(a): file2.py:93>
-    #   Device assignments active during op 'b' creation:
-    #     with tf.device(/cpu:0): file2.py:94
-    op_info = error_interpolation.compute_field_dict(op)
-    coloc_op_info = error_interpolation.compute_field_dict(colocation_op)
-    msg = ("Tried to colocate op '{op_name}'{op_loc} having device '{op_dev}' "
-           "with op '{coloc_op_name}'{coloc_op_loc} which had an incompatible "
-           "device '{coloc_op_dev}'.\n\n{op_summary}\n\n{coloc_op_summary}"
-           .format(op_name=op.name,
-                   op_loc=op_info["defined_at"],
-                   op_dev=op.device,
-                   op_summary=op_info["devs_and_colocs"],
-                   coloc_op_name=colocation_op.name,
-                   coloc_op_loc=coloc_op_info["defined_at"],
-                   coloc_op_dev=colocation_op.device,
-                   coloc_op_summary=coloc_op_info["devs_and_colocs"]))
-    return msg
-
   def _create_op_helper(self, op, compute_device=True):
     """Common logic for creating an op in this graph."""
     # Apply any additional attributes requested. Do not overwrite any existing
@@ -3413,12 +3382,9 @@ class Graph(object):
       for colocation_op in self._colocation_stack.peek_objs():
         all_colocation_groups.extend(colocation_op.colocation_groups())
         if colocation_op.device:
-          if (op.device and pydev.canonical_name(op.device) !=
-              pydev.canonical_name(colocation_op.device)):
-            msg = self._make_colocation_conflict_message(op, colocation_op)
-            logging.warning(msg)
-          else:
-            op._set_device(colocation_op.device)  # pylint: disable=protected-access
+          # pylint: disable=protected-access
+          op._set_device(colocation_op.device)
+          # pylint: enable=protected-access
 
       all_colocation_groups = sorted(set(all_colocation_groups))
       # pylint: disable=protected-access
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index f60c568f32..7baa02b446 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2850,43 +2850,6 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
       b = variables.Variable([3.0], name="b")
     self.assertEqual([b"loc:@a"], b.op.colocation_groups())
 
-  @test_util.run_deprecated_v1
-  def testInconsistentDeviceWithinColocate(self):
-    with ops.device("/device:GPU:0"):
-      a = constant_op.constant([2.0], name="a")
-      with ops.colocate_with(a.op):
-        # This is allowed due to legacy but clearly wrong, since we
-        # should really be colocating with 'a'.  We allow devices to
-        # override colocate_with, but we log warnings to suggest that
-        # this is probably unintentional or misguided.
-        with ops.device("/cpu:0"):
-          b = constant_op.constant([3.0], name="b")
-
-    self.assertEqual("/device:CPU:0", b.device)
-
-  @test_util.run_deprecated_v1
-  def testMakeColocationConflictMessage(self):
-    """Test that provides an example of a complicated error message."""
-    # We could test the message with any ops, but this test will be more
-    # instructive with a real colocation conflict.
-    with ops.device("/device:GPU:0"):
-      a = constant_op.constant([2.0], name="a")
-      with ops.colocate_with(a.op):
-        with ops.device("/cpu:0"):
-          b = constant_op.constant([3.0], name="b")
-    # The definition-location of the nodes will be wrong because of running
-    # from within a TF unittest.  The rest of the info should be correct.
-    message = ops.get_default_graph()._make_colocation_conflict_message(a.op,
-                                                                        b.op)
-    self.assertRegexpMatches(message,
-                             r"Tried to colocate op 'a' \(defined at.*\)")
-    self.assertRegexpMatches(message, "No node-device.*'a'")
-    self.assertRegexpMatches(message, "Device assignments active.*'a'")
-    self.assertRegexpMatches(message, "GPU:0")
-    self.assertRegexpMatches(message, "Node-device colocations active.*'b'")
-    self.assertRegexpMatches(message, "Device assignments active.*'b'")
-    self.assertRegexpMatches(message, "cpu:0")
-
 
 class DeprecatedTest(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 556e5ca31154d1036a9992a8dcf8350574ee9a96 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 3 Dec 2018 10:38:59 -0800
Subject: [PATCH 1201/1554] gradient_checker: Convert arguments to tensors.

Without this change, the test added to gradient_checker_v2_test.py
would fail with an error like:

AttributeError: 'numpy.ndarray' object has no attribute '_id'

in the call to "tape.watch(x)".

Since gradient_checker_v2.compute_gradients() requires that the "x" argument be
a list of Tensors, it seems reasonable to ensure that the "f" argument is
provided with Tensors.

While at it, also check that the "x" argument is a list to provide a clearer
error message than would be obtained without this check.

PiperOrigin-RevId: 223820537
---
 tensorflow/python/ops/gradient_checker_v2.py      | 13 ++++++++++---
 tensorflow/python/ops/gradient_checker_v2_test.py | 14 ++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/gradient_checker_v2.py b/tensorflow/python/ops/gradient_checker_v2.py
index cd49bb3d17..cf84484112 100644
--- a/tensorflow/python/ops/gradient_checker_v2.py
+++ b/tensorflow/python/ops/gradient_checker_v2.py
@@ -88,14 +88,18 @@ def _prepare(f, xs_dtypes):
     a function that will be evaluated in both graph and eager mode
   """
   if context.executing_eagerly():
-    return f
+
+    def decorated_eager(*xs_data):
+      return f(*map(ops.convert_to_tensor, xs_data))
+
+    return decorated_eager
   xs = [array_ops.placeholder(x_dtype) for x_dtype in xs_dtypes]
   y = f(*xs)
   sess = ops.get_default_session()
-  def decorated(*xs_data):
+  def decorated_graph(*xs_data):
     xs_data = [_to_numpy(a) for a in xs_data]
     return sess.run(y, feed_dict=dict(zip(xs, xs_data)))
-  return decorated
+  return decorated_graph
 
 
 def _compute_theoretical_jacobian(f, y_shape, y_dtype, xs, param):
@@ -288,6 +292,9 @@ def compute_gradient(f, x, delta=1e-3):
   Raises:
     ValueError: If result is empty but the gradient is nonzero.
   """
+  if not isinstance(x, list):
+    raise ValueError(
+        "`x` must be a list of Tensors (arguments to `f`), not a %s" % type(x))
   return _compute_gradient_list(f, x, delta)
 
 
diff --git a/tensorflow/python/ops/gradient_checker_v2_test.py b/tensorflow/python/ops/gradient_checker_v2_test.py
index e3351e2920..ce9ff47d61 100644
--- a/tensorflow/python/ops/gradient_checker_v2_test.py
+++ b/tensorflow/python/ops/gradient_checker_v2_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -177,6 +178,19 @@ class GradientCheckerTest(test.TestCase):
     with self.assertRaisesRegexp(AssertionError, "False is not true"):
       self.assertTrue(error < 1.0)
 
+  def testGradGrad(self):
+
+    def f(x):
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        y = math_ops.square(x)
+        z = math_ops.square(y)
+      return tape.gradient(z, x)
+
+    analytical, numerical = gradient_checker.compute_gradient(f, [2.0])
+    self.assertAllEqual([[[48.]]], analytical)
+    self.assertAllClose([[[48.]]], numerical, rtol=1e-4)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class MiniMNISTTest(test.TestCase):
-- 
GitLab


From 3360c72f32894bc12f5592398d8becbc726ab2d1 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 3 Dec 2018 10:41:16 -0800
Subject: [PATCH 1202/1554] One line configure change for message on wrong
 bazel version.

PiperOrigin-RevId: 223820935
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 57a03bd17f..3a7999fbd6 100644
--- a/configure.py
+++ b/configure.py
@@ -491,7 +491,7 @@ def check_bazel_version(min_version, max_version):
     sys.exit(0)
   if curr_version_int > max_version_int:
     print('Please downgrade your bazel installation to version %s or lower to '
-          'build TensorFlow!' % min_version)
+          'build TensorFlow!' % max_version)
     sys.exit(0)
   return curr_version
 
-- 
GitLab


From 3b905b0e0089e8ffb6213213985a9155728267f2 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Mon, 3 Dec 2018 10:52:38 -0800
Subject: [PATCH 1203/1554] Respect np scalars dtype when converting to
 tensors.

Before this change:
Eager mode would always try to infer a dtype and convert it to int32 (since TF
prefers that), but graph would use the numpy dtype directly.
Eager would do this even if converting lists of scalars, but graph would
downcast.

After this change:
Eager and graph behave the same.
tf.convert_to_tensor(np.int64(1)).dtype == tf.int64
tf.convert_to_tensor([np.int64(1)]).dtype == tf.int32

PiperOrigin-RevId: 223823113
---
 tensorflow/python/eager/pywrap_tensor.cc |  8 ++++++++
 tensorflow/python/eager/tensor_test.py   | 12 ++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 0d0f70d543..206b96eef6 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -220,6 +220,14 @@ TFE_TensorHandle* ConvertToEagerTensor(PyObject* value, PyObject* dtype) {
       return nullptr;
     }
   }
+  tensorflow::Safe_PyObjectPtr value_decrefer;
+  if (PyArray_CheckAnyScalarExact(value)) {
+    // Convert numpy scalars to numpy arrays.
+    value = PyArray_FromScalar(value, nullptr);
+    // The returned value needs to be DECREF'd, but the original value was
+    // created in python code, and doesn't need to be DECREF'd.
+    value_decrefer.reset(value);
+  }
   if (PyArray_Check(value)) {
     int desired_np_dtype = -1;
     if (desired_dtype >= 0) {
diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index 25442ff048..0ee2ff68c2 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -95,6 +95,18 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     t = _create_tensor(values)
     self.assertAllEqual(values, t)
 
+  @test_util.assert_no_new_pyobjects_executing_eagerly
+  def testNumpyDtypeSurvivesThroughTensorConversion(self):
+    scalar_creators = [np.int32, np.int64, np.float32, np.float64]
+    conversion_functions = [ops.convert_to_tensor, constant_op.constant]
+
+    for scalar_creator in scalar_creators:
+      for conversion_function in conversion_functions:
+        np_val = scalar_creator(3)
+        tensor_val = conversion_function(np_val)
+        self.assertEqual(tensor_val.numpy().dtype, np_val.dtype)
+        self.assertEqual(tensor_val.numpy(), np_val)
+
   def testNumpyValueWithCast(self):
     values = np.array([3.0], dtype=np.float32)
     t = _create_tensor(values, dtype=dtypes.float64)
-- 
GitLab


From 124e8eebc72994a3f1153a3f6465f3914523fc1a Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 3 Dec 2018 10:52:40 -0800
Subject: [PATCH 1204/1554] Fix Keras V1 Adam optimizer and check consistency
 with V2 Adam.

PiperOrigin-RevId: 223823119
---
 tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py | 2 +-
 tensorflow/python/keras/optimizers.py                     | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 8fcf40af5d..1702a52a8b 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -492,7 +492,7 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
   # NAdam has been unified: currently these three algorithms behave differently.
   @parameterized.named_parameters(
       ('adadelta', 'adadelta', True, True), ('adagrad', 'adagrad', True, True),
-      ('adam', 'adam', True, False), ('adamax', 'adamax', True, True),
+      ('adam', 'adam', True, True), ('adamax', 'adamax', True, True),
       ('nadam', 'nadam', True, False), ('momentum', 'momentum', False, False),
       ('sgd', 'sgd', False, True))
   def testOptimizersCompatibility(self, opt_str, test_weights, test_numeric):
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 9c8020dc05..10466eb573 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -23,6 +23,7 @@ import six
 from six.moves import zip  # pylint: disable=redefined-builtin
 
 from tensorflow.python import tf2
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2
 from tensorflow.python.keras.optimizer_v2 import adagrad as adagrad_v2
@@ -482,7 +483,7 @@ class Adam(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [state_ops.assign_add(self.iterations, 1)]
+    self.updates = []
 
     lr = self.lr
     if self.initial_decay > 0:
@@ -490,7 +491,8 @@ class Adam(Optimizer):
           1. / (1. + self.decay * math_ops.cast(self.iterations,
                                                 K.dtype(self.decay))))
 
-    t = math_ops.cast(self.iterations, K.floatx()) + 1
+    with ops.control_dependencies([state_ops.assign_add(self.iterations, 1)]):
+      t = math_ops.cast(self.iterations, K.floatx())
     lr_t = lr * (
         K.sqrt(1. - math_ops.pow(self.beta_2, t)) /
         (1. - math_ops.pow(self.beta_1, t)))
-- 
GitLab


From 215beaca8c28f22a2d6b66b16b4770851491a792 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 3 Dec 2018 11:26:56 -0800
Subject: [PATCH 1205/1554] Prevent segments with no inputs

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 3b32f72bc1..3e599b9174 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -585,6 +585,13 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
       }
     }
   }
+  // We don't support segments with no inputs. Fall back to native TF here to
+  // avoid crash later. Constant folding should've folded the ops that make up
+  // these segments.
+  if (inputs.size() == 0) {
+    return tensorflow::errors::Internal("Segment has no inputs (possible "
+                                        "constfold failure)");
+  }
 
   const bool calibrate_int8 =
       (info.precision_mode == INT8MODE && info.use_calibration);
-- 
GitLab


From e6bfd12d734518b6c7e3ffd32eae0f7828125530 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 3 Dec 2018 11:10:07 -0800
Subject: [PATCH 1206/1554] PartitionedCallOp function inlining.

PiperOrigin-RevId: 223826687
---
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/function_optimizer.cc | 466 ++++++++++++++++--
 .../optimizers/function_optimizer_test.cc     | 182 +++++++
 tensorflow/core/grappler/utils/functions.cc   |   6 +-
 tensorflow/core/grappler/utils/functions.h    |   6 +-
 .../core/grappler/utils/grappler_test.cc      |   4 +-
 6 files changed, 612 insertions(+), 53 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 9149ee275d..68f70bf0df 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -149,6 +149,7 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
+        "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 9faab6614c..3ffe8f1a0b 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/algorithm/container.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
@@ -264,6 +265,10 @@ class FunctionOptimizerContext {
     return tensor_mapping_;
   }
 
+  const gtl::FlatMap<string, std::vector<string>>& control_overrides() const {
+    return control_overrides_;
+  }
+
   const GraphView& graph_view() const { return graph_view_; }
 
   const string& grappler_item_id() const { return grappler_item_id_; }
@@ -292,6 +297,13 @@ class FunctionOptimizerContext {
     specialized_functions_.emplace(sig, specialized_func);
   }
 
+  void AddTensorMapping(const SafeTensorId& from, const SafeTensorId& to) {
+    auto inserted = tensor_mapping_.insert({from, to});
+    DCHECK(inserted.second)
+        << "Failed to insert duplicated tensor mapping: "
+        << "from=" << from.ToString() << " to=" << to.ToString();
+  }
+
   void AddTensorMapping(const string& func_node,
                         const FunctionSpecialization& specialized_func) {
     for (const auto& pair : specialized_func.output_mapping) {
@@ -306,6 +318,14 @@ class FunctionOptimizerContext {
     }
   }
 
+  void AddControlOverrides(const NodeDef& func_node,
+                           const std::vector<string>& control_overrides) {
+    control_overrides_[func_node.name()].reserve(control_overrides.size());
+    for (const string& control_override : control_overrides) {
+      control_overrides_[func_node.name()].push_back(control_override);
+    }
+  }
+
  private:
   void InitializeTrulyConstNodes(const GrapplerItem& item) {
     gtl::FlatSet<string> feed_nodes;
@@ -377,6 +397,14 @@ class FunctionOptimizerContext {
   gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>
       tensor_mapping_;
 
+  // When we inline a function into the optimized graph, we no longer have the
+  // function call node to anchor control dependencies. Instead we must expand
+  // each function call control output edge into multiple control dependencies
+  // to all side-effectful ops inside the function body.
+  //
+  // Invalidated function call node name -> Inlined side-effectful nodes
+  gtl::FlatMap<string, std::vector<string>> control_overrides_;
+
   // Use graph view to find active outputs of the function caller nodes.
   GraphView graph_view_;
 
@@ -426,7 +454,7 @@ bool HasTrulyConstInputs(const NodeDef& node,
   const auto is_truly_const = [&ctx](const string& input) {
     return ctx.IsTrulyConst(NodeName(input));
   };
-  return std::any_of(node.input().begin(), node.input().end(), is_truly_const);
+  return absl::c_any_of(node.input(), is_truly_const);
 }
 
 bool HasUnusedOutputs(const NodeDef& func_node, const FunctionDef& func,
@@ -799,22 +827,35 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
 // inputs into that, and we insert another IdentityN node to hook all function
 // outputs to it.
 
-// Returns true iff `node` is a direct function call of `func`, and we know how
-// to inline it into the main graph.
-bool IsInlinableDirectFunctionCall(const FunctionOptimizerContext& ctx,
-                                   const FunctionDef& func,
-                                   const NodeDef& node) {
+// Returns `Status::OK()` iff `node` is a direct function call of `func`, and we
+// know how to inline it into the main graph, otherwise returns and error
+// indicating why the function call is not inlinable.
+Status IsInlinableDirectFunctionCall(const FunctionOptimizerContext& ctx,
+                                     const FunctionDef& func,
+                                     const NodeDef& func_node) {
   // Indirect function calls (PartitionedCallOp) have automatic control
   // dependencies and inlined separately from direct function calls.
-  bool is_direct_function_call = IsDirectFunctionCall(func, node);
+  if (!IsDirectFunctionCall(func, func_node)) {
+    return errors::InvalidArgument("Unsupported function call type: ",
+                                   SummarizeNodeDef(func_node));
+  }
 
   // For direct function  calls we insert IdentityN nodes before/after inlined
   // function body to preserve function call semantics (all inputs evaluated
   // before function evaluation starts, and all function body nodes finished
   // before output consumed by other nodes).
-  bool has_inputs = func.signature().input_arg_size() > 0;
+  if (func.signature().input_arg_size() == 0) {
+    return errors::FailedPrecondition(
+        "Can't inline direct function call with empty inputs: ",
+        SummarizeNodeDef(func_node));
+  }
+
   // TODO(ezhulenev): Relax constraint on output args?
-  bool has_outputs = func.signature().output_arg_size() > 0;
+  if (func.signature().output_arg_size() == 0) {
+    return errors::FailedPrecondition(
+        "Can't inline direct function call with empty outputs: ",
+        SummarizeNodeDef(func_node));
+  }
 
   // Function must execute all the nodes in a function body that might have side
   // effects. After inlining these nodes into the main graph, we can no longer
@@ -825,21 +866,32 @@ bool IsInlinableDirectFunctionCall(const FunctionOptimizerContext& ctx,
   //
   // Indirect function calls (via PartitionedCallOp) have automatic dependency
   // tracking, and allow us to safely inline functions with side effects.
-  bool free_of_side_effects =
-      std::all_of(func.node_def().begin(), func.node_def().end(),
-                  [&ctx](const NodeDef& node) {
-                    return IsFreeOfSideEffect(node, &ctx.function_library());
-                  });
-
-  bool marked_noinline = MarkedNoInline(func);
-  bool marked_specialized = MarkedSpecialized(func);
+  bool has_side_effects =
+      absl::c_any_of(func.node_def(), [&ctx](const NodeDef& node) {
+        return !IsFreeOfSideEffect(node, &ctx.function_library());
+      });
+  if (has_side_effects) {
+    return errors::FailedPrecondition(
+        "Can't inline function with side-effects in the function body: ",
+        SummarizeNodeDef(func_node));
+  }
 
   // We ignore `_noinline` marker in aggressive mode.
   bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
+  if (MarkedNoInline(func) && !aggressive) {
+    return errors::FailedPrecondition(
+        "Can't inline function marked with '_noinline': ",
+        SummarizeNodeDef(func_node));
+  }
+
+  // Function specialization and inlining must be mutually exclusive.
+  if (MarkedSpecialized(func)) {
+    return errors::FailedPrecondition(
+        "Can't inline function created in Grappler function specialization: ",
+        SummarizeNodeDef(func_node));
+  }
 
-  return is_direct_function_call && has_inputs && has_outputs &&
-         free_of_side_effects && !marked_specialized &&
-         (!marked_noinline || aggressive);
+  return Status::OK();
 }
 
 // Create an IdentityN node to hook the function inputs to: this ensures that
@@ -886,17 +938,11 @@ NodeDef InlinedFunctionOutputsNode(const NodeDef& func_node,
 
 Status InlineDirectFunctionCall(const NodeDef& func_node,
                                 const FunctionDef& func,
-                                const FunctionOptimizerContext& ctx,
                                 const int graph_def_version,
+                                const FunctionOptimizerContext& ctx,
                                 GraphDef* optimized_graph) {
   VLOG(2) << "Inline direct function call: " << SummarizeNodeDef(func_node);
-
-  // Indirect function calls (via PartitionedCallOp) have automatic control
-  // dependencies, and doesn't need IdentityN nodes before/after inlined
-  // function body, and we inline them separately.
-  if (!IsDirectFunctionCall(func, func_node)) {
-    return errors::InvalidArgument("Can't inline indirect function call");
-  }
+  TF_RETURN_IF_ERROR(IsInlinableDirectFunctionCall(ctx, func, func_node));
 
   const AttrSlice func_instantiation_attr =
       FunctionInstantiationAttributes(func, func_node);
@@ -959,23 +1005,35 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
     // Make sure the node is placed.
     func_body_node.set_device(func_node.device());
 
-    // Check if a body node is itself a function call and can be inlined.
-    const FunctionDef* func_body_node_func =
-        FindFunctionCall(ctx, func_body_node);
-    if (func_body_node_func != nullptr &&
-        IsInlinableDirectFunctionCall(ctx, *func_body_node_func,
-                                      func_body_node)) {
-      // Recursively inline function calls.
-      TF_RETURN_IF_ERROR(
-          InlineDirectFunctionCall(func_body_node, *func_body_node_func, ctx,
-                                   graph_def_version, optimized_graph));
-    } else {
+    // Move the function body node to the optimized graph.
+    const auto move_node_to_optimized_graph = [&]() {
       // Annotate the node with the function attributes.
       for (const auto& attr : func.attr()) {
         func_body_node.mutable_attr()->insert(attr);
       }
       // Move the node to the main graph.
       optimized_graph->add_node()->Swap(&func_body_node);
+    };
+
+    // Check if a body node is itself a function call and can be inlined.
+    const FunctionDef* func_body_node_func =
+        FindFunctionCall(ctx, func_body_node);
+
+    if (func_body_node_func != nullptr) {
+      Status inlinable = IsInlinableDirectFunctionCall(
+          ctx, *func_body_node_func, func_body_node);
+      if (inlinable.ok()) {
+        TF_RETURN_IF_ERROR(
+            InlineDirectFunctionCall(func_body_node, *func_body_node_func,
+                                     graph_def_version, ctx, optimized_graph));
+      } else {
+        VLOG(2) << "Can't inline nested direct function call: "
+                << inlinable.error_message();
+        move_node_to_optimized_graph();
+      }
+
+    } else {
+      move_node_to_optimized_graph();
     }
   }
 
@@ -1082,9 +1140,255 @@ Status InlineSymbolicGradient(const NodeDef& node,
   return Status::OK();
 }
 
+// -------------------------------------------------------------------------- //
+// Inline indirect functions calls (aka PartitionedCallOp).
+//
+// When we inline indirect function calls, we instantiate the function body from
+// its FunctionDef and caller node attributes, and embed the instantiated graph
+// into the "main graph".
+//
+// In contrast to direct function calls, `PartitionedCallOp` has automatic
+// dependency tracking via input/output control edges, and we relax some of the
+// constraints that we have for direct function call inlining.
+//
+// "When a `PartitionedCallOp` function has a resource (DT_RESOURCE data type)
+// input argument it "captures" the mutable resource.  This is implemented by
+// automatically adding a incoming control edge from the previous side-effectful
+// op touching that resource, and an outgoing control edge to the next
+// side-effectful op using the same resource. This serializes the mutations of
+// the resource to make graph execution deterministic.
+//
+// Function call inlining must preserve side effect visibility:
+//
+// 1) All side effects to the captured resources, that happened before function
+//    call must be visible to the function body nodes using that resources.
+// 2) All side effects to the captured resources, that happened inside function
+//    body, must be visible to every op/function using that resource after the
+//    function call completed.
+
+// To guarantee that these properties are preserved after inlining we do:
+//
+// 1) Forward all input control dependencies from the function call node to the
+//    inlined function inputs (Identity nodes).
+// 2) Each side-effectful op inside function body adds itself as a control
+//    dependency to all the nodes in output control set of function call node.
+//
+// We do not add any other control dependencies to/from function body nodes,
+// because they are pure functions of input tensors, and can be freely
+// reordered.
+
+// Returns `Status::OK()` iff `node` is an indirect function call of `func`, and
+// we know how to inline it into the main graph, otherwise returns and error
+// indicating why the function call is not inlinable.
+Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
+                                       const FunctionDef& func,
+                                       const NodeDef& func_node) {
+  // We inline direct function calls above, using different rules.
+  if (!IsIndirectFunctionCall(func, func_node)) {
+    return errors::InvalidArgument("Unsupported function call type: ",
+                                   SummarizeNodeDef(func_node));
+  }
+
+  // TODO(ezhulenev): Enable it by default.
+  if (ctx.opt_level() != RewriterConfig::AGGRESSIVE) {
+    return errors::FailedPrecondition(
+        "Indirect function inlining supported only in aggressive mode");
+  }
+
+  if (MarkedNoInline(func)) {
+    return errors::FailedPrecondition(
+        "Can't inline function marked with '_noinline': ",
+        SummarizeNodeDef(func_node));
+  }
+
+  // Function specialization and inlining must be mutually exclusive.
+  if (MarkedSpecialized(func)) {
+    return errors::FailedPrecondition(
+        "Can't inline function created in Grappler function specialization: ",
+        SummarizeNodeDef(func_node));
+  }
+
+  // We can't inline functions that are in a fetch set, because it would
+  // invalidate fetch tensors (function call node fully inlined and doesn't
+  // exist in the optimized graph).
+  if (ctx.IsFetchNode(func_node.name())) {
+    return errors::FailedPrecondition(
+        "Can't inline function in a Grappler item fetch set: ",
+        SummarizeNodeDef(func_node));
+  }
+
+  return Status::OK();
+}
+
+Status InlineIndirectFunctionCall(const NodeDef& func_node,
+                                  const FunctionDef& func,
+                                  const int graph_def_version,
+                                  FunctionOptimizerContext* ctx,
+                                  GraphDef* optimized_graph) {
+  VLOG(2) << "Inline indirect function call: " << SummarizeNodeDef(func_node);
+  TF_RETURN_IF_ERROR(IsInlinableIndirectFunctionCall(*ctx, func, func_node));
+
+  const AttrSlice func_instantiation_attr =
+      FunctionInstantiationAttributes(func, func_node);
+
+  GrapplerFunctionItem item;
+  Status item_status = MakeGrapplerFunctionItem(func, func_instantiation_attr,
+                                                ctx->function_library(),
+                                                graph_def_version, &item);
+
+  if (!item_status.ok()) {
+    return errors::InvalidArgument("Failed to inline function ", func_node.op(),
+                                   " instantiated by ", func_node.name(),
+                                   ". Error: ", item_status.error_message());
+  }
+
+  GraphView::InputPort control_input_port =
+      ctx->graph_view().GetInputPort(func_node.name(), Graph::kControlSlot);
+  GraphView::OutputPort control_output_port =
+      ctx->graph_view().GetOutputPort(func_node.name(), Graph::kControlSlot);
+
+  // Nodes that have side effects to the captured resources.
+  std::vector<string> happens_before;
+  absl::c_transform(
+      ctx->graph_view().GetFanin(control_input_port),
+      std::back_inserter(happens_before),
+      [](const GraphView::OutputPort port) { return port.node->name(); });
+
+  // Nodes that must observe side effects to the captured resources.
+  std::vector<string> happens_after;
+  absl::c_transform(
+      ctx->graph_view().GetFanout(control_output_port),
+      std::back_inserter(happens_after),
+      [](const GraphView::InputPort port) { return port.node->name(); });
+
+  // Regular (positional) inputs to the function call.
+  std::vector<SafeTensorId> inputs;
+  for (const string& input : func_node.input()) {
+    SafeTensorId tensor_id = ParseTensorName(input);
+    if (tensor_id.index() == Graph::kControlSlot) break;
+    inputs.push_back(tensor_id);
+  }
+
+  // If we have a node inside the function body without inputs (e.g. Const), we
+  // must attach a control dependency to it, to make sure that if a function
+  // call happens inside a loop, the node will be evaluated in correct frame.
+  //
+  // If the function call node has no inputs and no control dependencies, it
+  // means that it can't be a function call inside a loop, and we can safely
+  // insert that node without inputs into the main graph.
+  //
+  // TODO(ezhulenev): Use FrameMap (see grappler/utils/frame.h) to find out if
+  // the function is called inside a loop.
+  std::vector<string> empty_inputs_hook;
+  if (!item.inputs().empty()) {
+    const InputArgExpansion& arg0 = item.inputs()[0];
+    DCHECK(!arg0.placeholders.empty());
+    empty_inputs_hook.push_back(AsControlDependency(AddPrefixToNodeName(
+        arg0.placeholders[0], /*prefix=*/func_node.name())));
+  } else if (!happens_before.empty()) {
+    empty_inputs_hook.push_back(AsControlDependency(happens_before[0]));
+  }
+
+  // Mapping from input placeholder name to function input position.
+  int idx = 0;
+  absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
+  for (const InputArgExpansion& input_arg : item.inputs()) {
+    for (const string& placeholder : input_arg.placeholders) {
+      input_placeholders_idx[placeholder] = idx++;
+    }
+  }
+
+  const string prefix = strings::StrCat(func_node.name(), "/");
+
+  // Keep track of side-effectful ops inside function body. Each outgoing
+  // control edge from the function call node, must be replaced with control
+  // edges from inlined side-effectful ops.
+  std::vector<string> side_effectful_nodes;
+
+  for (NodeDef& func_body_node : *item.mutable_function_body().mutable_node()) {
+    if (item.IsInputPlaceholder(func_body_node.name())) {
+      // Turn input placeholders into identity node.
+      DCHECK_EQ(0, func_body_node.input_size());
+      func_body_node.set_op("Identity");
+      (*func_body_node.mutable_attr())["T"] = func_body_node.attr().at("dtype");
+      func_body_node.mutable_attr()->erase("dtype");
+      func_body_node.mutable_attr()->erase("shape");
+      int input_idx = input_placeholders_idx[func_body_node.name()];
+      func_body_node.add_input(strings::StrCat(inputs[input_idx].ToString()));
+
+      // All side effects must happen before inputs can start executing.
+      for (const string& hb_node : happens_before) {
+        func_body_node.add_input(AsControlDependency(hb_node));
+      }
+
+    } else {
+      // Update inputs of the regular function body nodes.
+      for (string& input : *func_body_node.mutable_input()) {
+        input = AddPrefixToNodeName(input, /*prefix=*/func_node.name());
+      }
+      if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty()) {
+        *func_body_node.add_input() = empty_inputs_hook[0];
+      }
+    }
+
+    // Add the function node name as a prefix 1) to node name to avoid
+    // collisions; 2) to frame name to avoid multiple LoopCond nodes in one
+    // frame after inlining.
+    TF_RETURN_IF_ERROR(
+        AddPrefixAndSuffixToNode(prefix, /*suffix=*/"", &func_body_node));
+
+    // TODO(ezhulenev): Call PartitionedCallOp to get placement for all function
+    // body nodes, for now just place it on function caller node device.
+    func_body_node.set_device(func_node.device());
+
+    // If the function body has a side-effectful op, we double check that the
+    // function call node has an output control edge, otherwise we can't safely
+    // do inlining and guarantee that node will be executed.
+    // TODO(ezhulenev): If we don't have `happens_after` dependencies does it
+    // mean that no one is interested in observing side effects and we can
+    // safely inline it?
+    if (!IsFreeOfSideEffect(func_body_node, &ctx->function_library())) {
+      DCHECK(!happens_after.empty());
+      if (happens_after.empty()) {
+        // NOTE: If this happens file a bug to TF.Eager team.
+        return errors::Internal(
+            "Can't inline a function with a stateful op and empty output "
+            "control edge set. Function body node: ",
+            SummarizeNodeDef(func_body_node));
+      }
+
+      side_effectful_nodes.push_back(func_body_node.name());
+    }
+
+    // TODO(ezhulenev): Inline nested indirect function calls.
+
+    // Move the node to the main graph.
+    optimized_graph->add_node()->Swap(&func_body_node);
+  }
+
+  // Indirect function call is fully inlined into the optimized graph, and we do
+  // not copy the original function call node, so we have to setup tensor
+  // mapping from old output tensors, to the outputs of inlined nodes.
+  int output_idx = 0;
+  for (const OutputArgExpansion& output : item.outputs()) {
+    for (const string& output_tensor : output.output_tensors) {
+      const SafeTensorId from_tensor(func_node.name(), output_idx++);
+      const SafeTensorId to_tensor = ParseTensorName(
+          AddPrefixToNodeName(output_tensor, /*prefix=*/func_node.name()));
+      ctx->AddTensorMapping(from_tensor, to_tensor);
+    }
+  }
+
+  // After inlining we'll have to forward all control dependencies from function
+  // call node to all side-effectful ops inside function body.
+  ctx->AddControlOverrides(func_node, side_effectful_nodes);
+
+  return Status::OK();
+}
+
 }  // namespace
 
-Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
+Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
                                    GraphDef* optimized_graph) {
   // Nothing to do here.
   if (item.graph.library().function_size() == 0) {
@@ -1149,17 +1453,38 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     if (func != nullptr) {
       const string& func_name = func->signature().name();
+      const int graph_def_version = item.graph.versions().producer();
+
+      const bool is_direct_func = IsDirectFunctionCall(*func, node);
+      const bool is_indirect_func = IsIndirectFunctionCall(*func, node);
 
       // 2a. Inline direct function call if it's inlinable.
-      if (inline_func && IsInlinableDirectFunctionCall(ctx, *func, node)) {
-        // Inline function body into the optimized graph}
-        TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(InlineDirectFunctionCall(
-            node, *func, ctx, item.graph.versions().producer(),
-            optimized_graph));
-        continue;
+      if (inline_func && is_direct_func) {
+        Status inlinable = IsInlinableDirectFunctionCall(ctx, *func, node);
+        if (inlinable.ok()) {
+          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(InlineDirectFunctionCall(
+              node, *func, graph_def_version, ctx, optimized_graph));
+          continue;
+        } else {
+          VLOG(2) << "Can't inline direct function call: "
+                  << inlinable.error_message();
+        }
       }
 
-      // 2b. Specialize it to it's instantiation context if can't be inlined,
+      // 2b. Inline indirect function call if it's inlinable.
+      if (inline_func && is_indirect_func) {
+        Status inlinable = IsInlinableIndirectFunctionCall(ctx, *func, node);
+        if (inlinable.ok()) {
+          TF_SKIP_ERROR_IF_GRAPH_UNMODIFIED(InlineIndirectFunctionCall(
+              node, *func, graph_def_version, &ctx, optimized_graph));
+          continue;
+        } else {
+          VLOG(2) << "Can't inline indirect function call: "
+                  << inlinable.error_message();
+        }
+      }
+
+      // 2c. Specialize it to its instantiation context if can't be inlined,
       // and it has something worth specializing.
       bool specialization_worthy = IsParametrized(*func) ||
                                    HasTrulyConstInputs(node, ctx) ||
@@ -1205,6 +1530,53 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     }
   }
 
+  // Function inlining instantiates function body directly into the optimized
+  // graph, and we might end up with control dependencies to the nodes that no
+  // longer exist in a graph. We need to apply control overrides to all
+  // invalidated nodes, and rewire control dependencies to the inlined
+  // side-effectful function body nodes.
+
+  // TODO(ezhulenev): With nested function call inlining, single pass over
+  // `control_overrides` might not bring the graph into a valid state,
+  // continue until it converges and all invalidated control dependencies
+  // removed.
+
+  if (!ctx.control_overrides().empty()) {
+    for (NodeDef& node : *optimized_graph->mutable_node()) {
+      // Keep track of new control inputs to the node.
+      gtl::FlatSet<string> add_ctrl_inputs;
+
+      // Remove all invalidated control inputs.
+      for (int idx = 0; idx < node.input_size(); ++idx) {
+        // TODO(ezhulenev): Use non-allocating TensorId after migrating
+        // `control_overrides()` to absl::flat_hash_set.
+        SafeTensorId input_tensor = ParseTensorName(node.input(idx));
+
+        auto overrides = ctx.control_overrides().find(input_tensor.node());
+        if (overrides != ctx.control_overrides().end()) {
+          // If this happens it's a bug in the function inlining.
+          if (input_tensor.index() != Graph::kControlSlot) {
+            return errors::Internal(
+                "Illegal input edge from inlined function call node");
+          }
+          // Remove control dependency to the inlined function call node.
+          node.mutable_input()->SwapElements(idx, node.input_size() - 1);
+          node.mutable_input()->RemoveLast();
+
+          // Keep track of all overrides.
+          for (const string& override : overrides->second) {
+            add_ctrl_inputs.insert(AsControlDependency(override));
+          }
+        }
+      }
+
+      // Add overrides to the node inputs.
+      for (const string& ctrl_input : add_ctrl_inputs) {
+        node.add_input(ctrl_input);
+      }
+    }
+  }
+
   *optimized_graph->mutable_versions() = item.graph.versions();
   *optimized_graph->mutable_library() =
       options_.enable_trim_function_library
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 9bb51c2641..98552e9449 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -699,6 +699,188 @@ TEST_F(FunctionOptimizerTest, InlineSymbolicGradient_NoInlineFunc) {
   CompareGraphs(item.graph, output);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionSimpleFunction) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Build a graph to compute c = MyMul(a, b)
+  GrapplerItem item;
+  item.fetch = {"d"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("c", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}},
+            kDevice),
+       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {mul_func} /* Function library */);
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       // Function must be inlined and all nodes placed on a valid device.
+       NDef("c/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("c/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("c/mul", "Mul", {"c/x", "c/y"}, {{"T", DT_FLOAT}}, kDevice),
+
+       NDef("d", "Identity", {"c/mul:0"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {mul_func});
+
+  CompareGraphs(expected, optimized_graph);
+
+  Tensor pi = test::AsScalar<float>(3.14f);
+  item.feed.emplace_back("a", pi);
+  item.feed.emplace_back("b", pi);
+
+  GrapplerItem optimized(item, std::move(optimized_graph));
+  auto tensors_expected = EvaluateFetchNodes(item);
+  auto tensors = EvaluateFetchNodes(optimized);
+  ASSERT_EQ(tensors_expected.size(), 1);
+  ASSERT_EQ(tensors.size(), tensors_expected.size());
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  const Tensor kOne = test::AsScalar<float>(1.0);
+  const Tensor kTwo = test::AsScalar<float>(2.0);
+  const TensorShape scalar = TensorShape({});
+
+  // Compute `x*y` and add `1.0` to the variable.
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T", "v: resource"}, {"z:T"}, {"T: {float, double}"},
+      {{{"one"}, "Const", {}, {{"value", kOne}, {"dtype", DT_FLOAT}}},
+       {{"add"},
+        "AssignAddVariableOp",
+        {"v", "one:output:0"},
+        {{"dtype", DT_FLOAT}}},
+       {{"mul"}, "Mul", {"x", "y", "^add"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Build a graph to compute:
+  //   a = Placeholder
+  //   b = Placeholder
+  //   v = VarHandleOp(init = a)
+  //   f1 = MyMul(a, b, v)
+  //   f2 = MyMul(f1, f1, v)
+  //   return [f2, v]
+  GrapplerItem item;
+  item.fetch = {"out_1", "out_2"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       // Initialize variable with one of the placeholders.
+       NDef("v", "VarHandleOp", {}, {{"dtype", DT_FLOAT}, {"shape", scalar}}),
+       NDef("init_v", "AssignVariableOp", {"v", "a"}, {{"dtype", DT_FLOAT}},
+            kDevice),
+
+       // Call function first time.
+       NDef("f1", "PartitionedCall", {"a", "b", "v", "^init_v"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT, DT_RESOURCE}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       // Call function second time.
+       NDef("f2", "PartitionedCall", {"f1", "f1", "v", "^f1"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT, DT_RESOURCE}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       // Return result of multiplication and a current value of the variable.
+       NDef("out_1", "Identity", {"f2"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("out_2", "ReadVariableOp", {"v", "^f2"}, {{"dtype", DT_FLOAT}},
+            kDevice)},
+
+      // Function library.
+      {mul_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       // Initialize variable with one of the placeholders.
+       NDef("v", "VarHandleOp", {}, {{"dtype", DT_FLOAT}, {"shape", scalar}}),
+       NDef("init_v", "AssignVariableOp", {"v", "a"}, {{"dtype", DT_FLOAT}},
+            kDevice),
+
+       // Function body of a first function call inlined into the graph.
+       NDef("f1/x", "Identity", {"a:0", "^init_v"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/y", "Identity", {"b:0", "^init_v"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/v", "Identity", {"v:0", "^init_v"}, {{"T", DT_RESOURCE}},
+            kDevice),
+       NDef("f1/one", "Const", {"^f1/x"},
+            {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
+       NDef("f1/add", "AssignAddVariableOp", {"f1/v", "f1/one"},
+            {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("f1/mul", "Mul", {"f1/x", "f1/y", "^f1/add"}, {{"T", DT_FLOAT}},
+            kDevice),
+
+       // Function body of a second function call also inlined into the graph,
+       // and input nodes read directly from the inlined nodes of the first
+       // function call.
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f1/add"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f1/add"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("f2/v", "Identity", {"v:0", "^f1/add"}, {{"T", DT_RESOURCE}},
+            kDevice),
+       NDef("f2/one", "Const", {"^f2/x"},
+            {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
+       NDef("f2/add", "AssignAddVariableOp", {"f2/v", "f2/one"},
+            {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("f2/mul", "Mul", {"f2/x", "f2/y", "^f2/add"}, {{"T", DT_FLOAT}},
+            kDevice),
+
+       // Return values read directly from inlined nodes.
+       NDef("out_1", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("out_2", "ReadVariableOp", {"v", "^f2/add"}, {{"dtype", DT_FLOAT}},
+            kDevice)},
+
+      // Function library.
+      {mul_func});
+
+  CompareGraphs(expected, optimized_graph);
+
+  item.feed.emplace_back("a", kOne);
+  item.feed.emplace_back("b", kTwo);
+
+  auto tensors_expected = EvaluateFetchNodes(item);
+  ASSERT_EQ(tensors_expected.size(), 2);
+  EXPECT_EQ(tensors_expected[0].flat<float>()(0), 4.0);  // mul
+  EXPECT_EQ(tensors_expected[1].flat<float>()(0), 3.0);  // read variable
+
+  GrapplerItem optimized(item, std::move(optimized_graph));
+  auto tensors = EvaluateFetchNodes(optimized);
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 7756c73967..57863a71f3 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -94,7 +94,7 @@ void GrapplerFunctionConnectivity::RegisterInputArgExpansion(
   for (int i = 0; i < placeholders.size(); ++i) {
     const string& placeholder = input_arg_expansion.placeholders[i];
     input_arg_placeholders_.insert(
-        {placeholder, InputArgPlaceholder{input_name, /*position=*/i}});
+        {placeholder, InputArgPlaceholder{input_name, /*input_position=*/i}});
   }
   input_arg_expansions_.insert(
       {std::move(input_name), std::move(input_arg_expansion)});
@@ -248,8 +248,8 @@ Status GrapplerFunctionConnectivity::AsFunctionDefInput(
     const InputArgPlaceholder* placeholder =
         FindOrNull(input_arg_placeholders_, node_name);
     if (placeholder != nullptr) {
-      *func_def_input =
-          strings::StrCat(placeholder->input_name, ":", placeholder->position);
+      *func_def_input = strings::StrCat(placeholder->input_name, ":",
+                                        placeholder->input_position);
       return Status::OK();
     }
   }
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index ba9950e484..038cf5f527 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -111,8 +111,10 @@ class GrapplerFunctionConnectivity {
   std::unordered_map<string, tensorflow::NameRangeMap> function_body_outputs_;
 
   struct InputArgPlaceholder {
-    string input_name;
-    int position;
+    string input_name;   // Name of the function input argument.
+    int input_position;  // Index of a tensor in the function input argument
+                         // expansion, it can be greater than `0` if input
+                         // argument is a list of tensors (aka list(type)).
   };
 
   // Mapping from input arg placeholder to the function input tensor.
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 6266733f3e..804fce5ce8 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -116,7 +116,9 @@ void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
     EXPECT_EQ(want.node(i).name(), got.node(i).name());
     ASSERT_EQ(want.node(i).input_size(), got.node(i).input_size());
     for (int j = 0; j < want.node(i).input_size(); ++j) {
-      EXPECT_TRUE(IsSameInput(want.node(i).input(j), got.node(i).input(j)));
+      const TensorId want_tensor = ParseTensorName(want.node(i).input(j));
+      const TensorId got_tensor = ParseTensorName(got.node(i).input(j));
+      EXPECT_EQ(want_tensor.ToString(), got_tensor.ToString());
     }
   }
 }
-- 
GitLab


From 4ee6be5d0a70e9674374d19d0f2513511a90543b Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 3 Dec 2018 11:26:44 -0800
Subject: [PATCH 1207/1554] Support exporting tables from tf.saved_model.save

Adds a mechanism to track assets, then uses it to fill AssetFileDefs and the main_op in the SavedModel.

As with all of tf.saved_model.save, currently this only targets serving and the v1 loader API.

PiperOrigin-RevId: 223830077
---
 tensorflow/python/ops/lookup_ops.py           |   9 +-
 tensorflow/python/saved_model/BUILD           |   2 +
 tensorflow/python/saved_model/builder_impl.py |  49 +--
 tensorflow/python/saved_model/save.py         | 278 ++++++++++++------
 tensorflow/python/saved_model/save_test.py    | 101 +++++--
 .../training/checkpointable/tracking.py       |  37 +++
 6 files changed, 337 insertions(+), 139 deletions(-)

diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 397d56ef40..0ef1ec8c00 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -39,6 +39,7 @@ from tensorflow.python.ops import string_ops
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_lookup_ops import *
 # pylint: enable=wildcard-import
+from tensorflow.python.training.checkpointable import base as checkpointable_base
 from tensorflow.python.training.checkpointable import tracking as checkpointable
 from tensorflow.python.util import compat
 from tensorflow.python.util.deprecation import deprecated
@@ -160,7 +161,7 @@ class InitializableLookupTableBase(LookupInterface):
     self._default_value = ops.convert_to_tensor(
         default_value, dtype=self._value_dtype)
     self._default_value.get_shape().merge_with(tensor_shape.scalar())
-    self._initializer = initializer
+    self._initializer = self._track_checkpointable(initializer, "_initializer")
     self._resource_handle = self.create_resource()
     self._init_op = self.initialize()
 
@@ -309,7 +310,7 @@ class HashTable(InitializableLookupTableBase):
     return exported_keys, exported_values
 
 
-class TableInitializerBase(object):
+class TableInitializerBase(checkpointable_base.CheckpointableBase):
   """Base class for lookup table initializers."""
 
   def __init__(self, key_dtype, value_dtype):
@@ -522,12 +523,14 @@ class TextFileInitializer(TableInitializerBase):
     if (vocab_size is not None) and (vocab_size <= 0):
       raise ValueError("Invalid vocab_size %s." % vocab_size)
 
-    self._filename = filename
     self._key_index = key_index
     self._value_index = value_index
     self._vocab_size = vocab_size
     self._delimiter = delimiter
     self._name = name
+    self._filename = self._track_checkpointable(
+        checkpointable.TrackableAsset(filename),
+        "_filename")
 
     super(TextFileInitializer, self).__init__(key_dtype, value_dtype)
 
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 16a31bee61..53d0640542 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -285,7 +285,9 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":builder",
         ":constants",
+        ":loader",
         ":saved_object_graph_py",
         ":signature_constants",
         ":signature_def_utils",
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index fc84979b6e..f37d283a2a 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -108,26 +108,6 @@ class _SavedModelBuilder(object):
     # weights.
     self._has_saved_variables = False
 
-  def _copy_assets_to_destination_dir(self, asset_filename_map):
-    """Copy all assets from source path to destination path."""
-    assets_destination_dir = saved_model_utils.get_or_create_assets_dir(
-        self._export_dir)
-
-    # Copy each asset from source path to destination path.
-    for asset_basename, asset_source_filepath in asset_filename_map.items():
-      asset_destination_filepath = os.path.join(
-          compat.as_bytes(assets_destination_dir),
-          compat.as_bytes(asset_basename))
-
-      # Only copy the asset file to the destination if it does not already
-      # exist. This is to ensure that an asset with the same name defined as
-      # part of multiple graphs is only copied the first time.
-      if not file_io.file_exists(asset_destination_filepath):
-        file_io.copy(asset_source_filepath, asset_destination_filepath)
-
-    tf_logging.info("Assets written to: %s",
-                    compat.as_text(assets_destination_dir))
-
   def _save_and_write_assets(self, meta_graph_def, assets_list=None):
     """Saves asset to the meta graph and writes asset files to disk.
 
@@ -145,7 +125,7 @@ class _SavedModelBuilder(object):
       return
 
     # Copy assets from source path to destination path.
-    self._copy_assets_to_destination_dir(asset_filename_map)
+    copy_assets_to_destination_dir(asset_filename_map, self._export_dir)
 
   def _tag_and_add_meta_graph(self, meta_graph_def, tags, signature_def_map):
     """Tags the meta graph def and adds it to the SavedModel.
@@ -470,7 +450,7 @@ class SavedModelBuilder(_SavedModelBuilder):
       return
 
     # Copy assets from source path to destination path.
-    self._copy_assets_to_destination_dir(asset_filename_map)
+    copy_assets_to_destination_dir(asset_filename_map, self._export_dir)
 
   def _maybe_add_main_op(self, main_op):
     """Adds main op to the SavedModel.
@@ -656,7 +636,7 @@ def _maybe_save_assets(write_fn, assets_to_add=None):
     if not asset_source_filepath:
       raise ValueError("Invalid asset filepath tensor %s" % asset_tensor)
 
-    asset_filename = _get_asset_filename_to_add(
+    asset_filename = get_asset_filename_to_add(
         asset_source_filepath, asset_filename_map)
 
     # Call the passed-in function that builds AssetFileDef proto and adds it
@@ -675,7 +655,7 @@ def _maybe_save_assets(write_fn, assets_to_add=None):
   return asset_filename_map
 
 
-def _get_asset_filename_to_add(asset_filepath, asset_filename_map):
+def get_asset_filename_to_add(asset_filepath, asset_filename_map):
   """Get a unique basename to add to the SavedModel if this file is unseen.
 
   Assets come from users as full paths, and we save them out to the
@@ -762,6 +742,27 @@ def _add_asset_to_metagraph(meta_graph_def, asset_filename, asset_tensor):
   asset_proto.tensor_info.name = asset_tensor.name
 
 
+def copy_assets_to_destination_dir(asset_filename_map, destination_dir):
+  """Copy all assets from source path to destination path."""
+  assets_destination_dir = saved_model_utils.get_or_create_assets_dir(
+      destination_dir)
+
+  # Copy each asset from source path to destination path.
+  for asset_basename, asset_source_filepath in asset_filename_map.items():
+    asset_destination_filepath = os.path.join(
+        compat.as_bytes(assets_destination_dir),
+        compat.as_bytes(asset_basename))
+
+    # Only copy the asset file to the destination if it does not already
+    # exist. This is to ensure that an asset with the same name defined as
+    # part of multiple graphs is only copied the first time.
+    if not file_io.file_exists(asset_destination_filepath):
+      file_io.copy(asset_source_filepath, asset_destination_filepath)
+
+  tf_logging.info("Assets written to: %s",
+                  compat.as_text(assets_destination_dir))
+
+
 def _add_asset_to_collection(asset_filename, asset_tensor):
   """Builds an asset proto and adds it to the asset collection of the graph.
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index bc3645fa38..a66e19b199 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -19,18 +19,24 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import functools
 import os
 
+from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.core.protobuf import saved_model_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
-from tensorflow.python.eager import function
+from tensorflow.python.eager import function as defun
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.saved_model import builder_impl
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import signature_constants
@@ -38,6 +44,7 @@ from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.saved_model import utils_impl
 from tensorflow.python.training.checkpointable import base
+from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
@@ -106,7 +113,7 @@ def _canonicalize_signatures(signatures):
         signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures}
   concrete_signatures = {}
   for serving_key, signature_function in signatures.items():
-    if isinstance(signature_function, (function.PolymorphicFunction,
+    if isinstance(signature_function, (defun.PolymorphicFunction,
                                        def_function.PolymorphicFunction)):
       input_signature = signature_function._input_signature  # pylint: disable=protected-access
       if input_signature is None:
@@ -117,7 +124,7 @@ def _canonicalize_signatures(signatures):
              "converted to concrete functions using "
              "`f.get_concrete_function(...)`.").format(signature_function))
       signature_function = signature_function.get_concrete_function()
-    elif not isinstance(signature_function, function.Function):
+    elif not isinstance(signature_function, defun.Function):
       raise ValueError(
           ("Expected a TensorFlow function to generate a signature for, but "
            "got {}. Python functions may be decorated with "
@@ -174,64 +181,65 @@ def _tensor_dict_to_tensorinfo(tensor_dict):
           for key, value in tensor_dict.items()}
 
 
-def _map_captured_resources_to_created_resources(
+def _map_captures_to_created_tensors(
     original_captures, resource_map):
-  """Maps eager resources captured by a function to Graph resources for export.
+  """Maps eager tensors captured by a function to Graph resources for export.
 
   Args:
-    original_captures: A dictionary mapping from resource tensors captured by
-      the function to interior placeholders for those resources (inside the
-      function body).
+    original_captures: A dictionary mapping from tensors captured by the
+      function to interior placeholders for those tensors (inside the function
+      body).
     resource_map: A dictionary mapping from resource tensors owned by the eager
       context to resource tensors in the exported graph.
 
   Returns:
-    A dictionary mapping from interior placeholders in the function body to
-    exterior stand-in resource tensors which belong to the exported graph.
+    A list of stand-in tensors which belong to the exported graph, corresponding
+    to the function's captures.
 
   Raises:
     AssertionError: If the function references a resource which is not part of
       `resource_map`.
   """
-  export_captures = {}
+  export_captures = []
   for exterior, interior in original_captures.items():
     mapped_resource = resource_map.get(exterior, None)
     if mapped_resource is None:
-      raise AssertionError(
-          ("Tried to export a function which references untracked stateful "
-           "object {}. Stateful TensorFlow objects (e.g. tf.Variable) must "
-           "be tracked by the main object. Objects may be tracked by "
-           "assigning them to an attribute of another tracked object, or to "
-           "an attribute of the main object directly.")
-          .format(interior))
-    export_captures[interior] = mapped_resource
+      if exterior.dtype == dtypes.resource:
+        raise AssertionError(
+            ("Tried to export a function which references untracked stateful "
+             "object {}. Stateful TensorFlow objects (e.g. tf.Variable) must "
+             "be tracked by the main object. Objects may be tracked by "
+             "assigning them to an attribute of another tracked object, or to "
+             "an attribute of the main object directly.")
+            .format(interior))
+      else:
+        # This is a captured Tensor, but it's not a resource. We'll just add it
+        # to the graph as a constant.
+        mapped_resource = constant_op.constant(exterior.numpy())
+    export_captures.append(mapped_resource)
   return export_captures
 
 
-def _map_function_inputs_to_created_inputs(
-    function_inputs, export_captures, signature_key, function_name):
-  """Creates exterior placeholders in the exported graph for function inputs.
+def _map_function_arguments_to_created_inputs(
+    function_arguments, signature_key, function_name):
+  """Creates exterior placeholders in the exported graph for function arguments.
 
   Functions have two types of inputs: tensors captured from the outside (eager)
   context, and arguments to the function which we expect to receive from the
-  user at each call. `_map_captured_resources_to_created_resources` replaces
+  user at each call. `_map_captures_to_created_tensors` replaces
   captured tensors with stand-ins (typically these are resource dtype tensors
   associated with variables). `_map_function_inputs_to_created_inputs` runs over
-  every input, either captured or argument. For captures, it uses the mapped
-  resource from `export_captures`. For arguments, it creates a new placeholder
-  which will belong to the exported graph rather than the function body.
+  every argument, creating a new placeholder for each which will belong to the
+  exported graph rather than the function body.
 
   Args:
-    function_inputs: A list of all placeholders in the function body.
-    export_captures: A dictionary mapping from interior placeholders in the
-      function body to exterior stand-in resource tensors which belong to the
-      exported graph (see `_map_captured_resources_to_created_resources`).
+    function_arguments: A list of argument placeholders in the function body.
     signature_key: The name of the signature being exported, for error messages.
     function_name: The name of the function, for error messages.
 
   Returns:
     A tuple of (mapped_inputs, exterior_placeholders)
-      mapped_inputs: A list with entries corresponding to `function_inputs`
+      mapped_inputs: A list with entries corresponding to `function_arguments`
         containing all of the inputs of the function gathered from the exported
         graph (both captured resources and arguments).
       exterior_argument_placeholders: A dictionary mapping from argument names
@@ -249,12 +257,7 @@ def _map_function_inputs_to_created_inputs(
   # MetaGraph.
   exterior_argument_placeholders = {}
   mapped_inputs = []
-  for placeholder in function_inputs:
-    mapped_resource_tensor = export_captures.get(placeholder, None)
-    if mapped_resource_tensor is not None:
-      # This is a captured resource.
-      mapped_inputs.append(mapped_resource_tensor)
-      continue
+  for placeholder in function_arguments:
     # `export_captures` contains an exhaustive set of captures, so if we don't
     # find the input there then we now know we have an argument.
     user_input_name = compat.as_str_any(
@@ -287,6 +290,20 @@ def _map_function_inputs_to_created_inputs(
   return mapped_inputs, exterior_argument_placeholders
 
 
+def _call_function_with_mapped_captures(function, args, resource_map):
+  """Calls `function` in the exported graph, using mapped resource captures."""
+  export_captures = _map_captures_to_created_tensors(
+      function.graph.captures, resource_map)
+  mapped_inputs = args + export_captures
+  # Calls the function quite directly, since we have new captured resource
+  # tensors we need to feed in which weren't part of the original function
+  # definition.
+  # pylint: disable=protected-access
+  outputs = function._build_call_outputs(
+      function._inference_function.call(context.context(), mapped_inputs))
+  return outputs
+
+
 def _generate_signatures(signature_functions, resource_map):
   """Validates and calls `signature_functions` in the default graph.
 
@@ -316,35 +333,77 @@ def _generate_signatures(signature_functions, resource_map):
     SignatureDefs as part of that MetaGraph.
   """
   signatures = {}
-  for signature_key, func in sorted(signature_functions.items()):
-    # Register the inference function for this signature in the exported
-    # graph. There is no direct use for the gradient of this function, so we
-    # don't generate/register a gradient function here (but may end up with one
-    # if another function relies on it). Users can still take symbolic gradients
-    # of the function on import, the gradient just won't be in the saved
-    # graph. When exporting a signature which already computes gradients, this
-    # stops us from taking needless second-order gradients.
-    func.add_to_graph(register_gradient_functions=False)
-    export_captures = _map_captured_resources_to_created_resources(
-        func.graph.captures, resource_map)
+  for signature_key, function in sorted(signature_functions.items()):
+    if function.graph.captures:
+      argument_inputs = function.graph.inputs[:-len(function.graph.captures)]
+    else:
+      argument_inputs = function.graph.inputs
     mapped_inputs, exterior_argument_placeholders = (
-        _map_function_inputs_to_created_inputs(
-            func.inputs, export_captures, signature_key, func.name))
-    # Calls the function quite directly, since we have new captured resource
-    # tensors we need to feed in which weren't part of the original function
-    # definition.
-    # pylint: disable=protected-access
+        _map_function_arguments_to_created_inputs(
+            argument_inputs, signature_key, function.name))
     outputs = _normalize_outputs(
-        func._build_call_outputs(
-            func._inference_function.call(context.context(), mapped_inputs)),
-        func.name, signature_key)
-    # pylint: enable=protected-access
+        _call_function_with_mapped_captures(
+            function, mapped_inputs, resource_map),
+        function.name, signature_key)
     signatures[signature_key] = signature_def_utils.build_signature_def(
         _tensor_dict_to_tensorinfo(exterior_argument_placeholders),
         _tensor_dict_to_tensorinfo(outputs))
   return signatures
 
 
+def _trace_resource_initializers(accessible_objects):
+  """Create concrete functions from `TrackableResource` objects."""
+  resource_initializers = []
+
+  def _wrap_initializer(obj):
+    obj.initialize()
+    return constant_op.constant(1.)  # Dummy control output
+
+  for obj in accessible_objects:
+    if isinstance(obj, tracking.TrackableResource):
+      resource_initializers.append(def_function.function(
+          functools.partial(_wrap_initializer, obj),
+          # All inputs are captures.
+          input_signature=[]).get_concrete_function())
+  return resource_initializers
+
+
+_AssetInfo = collections.namedtuple(
+    "_AssetInfo", [
+        # List of AssetFileDef protocol buffers
+        "asset_defs",
+        # Map from asset variable resource Tensors to their init ops
+        "asset_initializers_by_resource",
+        # Map from base asset filenames to full paths
+        "asset_filename_map"])
+
+
+def _process_asset(trackable_asset, asset_info, resource_map):
+  """Add `trackable_asset` to `asset_info` and `resource_map`."""
+  original_variable = trackable_asset.asset_path
+  with context.eager_mode():
+    original_path = original_variable.numpy()
+  path = builder_impl.get_asset_filename_to_add(
+      asset_filepath=original_path,
+      asset_filename_map=asset_info.asset_filename_map)
+  asset_variable = asset_info.asset_filename_map.get(path, None)
+  if asset_variable is None:
+    asset_path_initializer = array_ops.placeholder(
+        shape=original_variable.shape,
+        dtype=dtypes.string,
+        name="asset_path_initializer")
+    asset_variable = resource_variable_ops.ResourceVariable(
+        asset_path_initializer)
+    asset_info.asset_filename_map[path] = original_path
+    asset_def = meta_graph_pb2.AssetFileDef()
+    asset_def.filename = path
+    asset_def.tensor_info.name = asset_path_initializer.name
+    asset_info.asset_defs.append(asset_def)
+    asset_info.asset_initializers_by_resource[original_variable.handle] = (
+        asset_variable.initializer)
+  resource_map[original_variable.handle] = asset_variable.handle
+
+
 def _map_resources(accessible_objects):
   """Makes new resource handle ops corresponding to existing resource tensors.
 
@@ -358,34 +417,83 @@ def _map_resources(accessible_objects):
       to create replacements for.
 
   Returns:
-    A tuple of (object_map, resource_map):
+    A tuple of (object_map, resource_map, asset_info):
       object_map: A dictionary mapping from object in `accessible_objects` to
         replacement objects created to hold the new resource tensors.
       resource_map: A dictionary mapping from resource tensors extracted from
         `accessible_objects` to newly created resource tensors.
+      asset_info: An _AssetInfo tuple describing external assets referenced from
+        accessible_objects.
   """
-  # TODO(allenl, rohanj): Map generic resources rather than just variables.
   # TODO(allenl): Handle MirroredVariables and other types of variables which
   # may need special casing.
   object_map = {}
   resource_map = {}
+  asset_info = _AssetInfo(
+      asset_defs=[],
+      asset_initializers_by_resource={},
+      asset_filename_map={})
   for obj in accessible_objects:
-    if resource_variable_ops.is_resource_variable(obj):
+    if isinstance(obj, tracking.TrackableResource):
+      new_resource = obj.create_resource()
+      resource_map[obj.resource_handle] = new_resource
+    elif resource_variable_ops.is_resource_variable(obj):
       new_variable = resource_variable_ops.copy_to_graph_uninitialized(obj)
       object_map[obj] = new_variable
       resource_map[obj.handle] = new_variable.handle
-  return object_map, resource_map
+    if isinstance(obj, tracking.TrackableAsset):
+      _process_asset(obj, asset_info, resource_map)
+  return object_map, resource_map, asset_info
 
 
-def _make_graph_def(root, signature_functions, object_saver):
-  """Generates and exports call ops for `signature_functions`."""
+def _fill_meta_graph_def(meta_graph_def, obj, signature_functions,
+                         object_saver):
+  """Generates a MetaGraph which calls `signature_functions`.
+
+  Args:
+    meta_graph_def: The MetaGraphDef proto to fill.
+    obj: The checkpointable object being exported.
+    signature_functions: A dictionary mapping signature keys to concrete
+      functions containing signatures to add to the MetaGraph.
+    object_saver: A CheckpointableSaver to add to the MetaGraph.
+
+  Returns:
+    asset_filename_map, a dictionary mapping from asset base names to
+    user-specified full asset paths, which should be copied to the SavedModel's
+    assets/ directory.
+  """
   signatures = {}
   # List objects from the eager context to make sure Optimizers give us the
   # right Graph-dependent variables.
-  accessible_objects = util.list_objects(root)
+  accessible_objects = util.list_objects(obj)
+  resource_initializer_functions = _trace_resource_initializers(
+      accessible_objects)
   exported_graph = ops.Graph()
+  resource_initializer_ops = []
   with exported_graph.as_default():
-    object_map, resource_map = _map_resources(accessible_objects)
+    object_map, resource_map, asset_info = _map_resources(accessible_objects)
+    for resource_initializer_function in resource_initializer_functions:
+      asset_dependencies = []
+      for capture in resource_initializer_function.graph.external_captures:
+        asset_initializer = asset_info.asset_initializers_by_resource.get(
+            capture, None)
+        if asset_initializer is not None:
+          asset_dependencies.append(asset_initializer)
+      with ops.control_dependencies(asset_dependencies):
+        resource_initializer_ops.append(
+            _call_function_with_mapped_captures(
+                resource_initializer_function, [], resource_map))
+    with ops.control_dependencies(resource_initializer_ops):
+      init_op = control_flow_ops.no_op()
+    # Add the same op to the main_op collection and to the init_op
+    # signature. The collection is for compatibility with older loader APIs;
+    # only one will be executed.
+    meta_graph_def.collection_def[constants.MAIN_OP_KEY].node_list.value.append(
+        init_op.name)
+    meta_graph_def.signature_def[constants.INIT_OP_SIGNATURE_KEY].CopyFrom(
+        signature_def_utils.op_signature_def(
+            init_op, constants.INIT_OP_SIGNATURE_KEY))
+
   # Saving an object-based checkpoint again gathers variables. We need to do the
   # gathering from the eager context so Optimizers save the right set of
   # variables, but want any operations associated with the save/restore to be in
@@ -394,11 +502,19 @@ def _make_graph_def(root, signature_functions, object_saver):
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
     saver_def = saver.to_proto()
+    meta_graph_def.saver_def.CopyFrom(saver_def)
   graph_def = exported_graph.as_graph_def(add_shapes=True)
   # Clean reference cycles so repeated export()s don't make work for the garbage
   # collector.
   ops.dismantle_graph(exported_graph)
-  return graph_def, signatures, saver_def
+
+  meta_graph_def.graph_def.CopyFrom(graph_def)
+  meta_graph_def.meta_info_def.tags.append(tag_constants.SERVING)
+  meta_graph_def.asset_file_def.extend(asset_info.asset_defs)
+  for signature_key, signature in signatures.items():
+    meta_graph_def.signature_def[signature_key].CopyFrom(signature)
+  meta_graph.strip_graph_default_valued_attrs(meta_graph_def)
+  return asset_info.asset_filename_map
 
 
 def _write_object_graph(obj, export_dir):
@@ -573,26 +689,24 @@ def save(obj, export_dir, signatures=None):
     # Note that we run this before saving the checkpoint, since looping over
     # attributes may have the side effect of creating variables in some cases.
     signatures = _find_function_to_export(obj)
-  object_saver = util.CheckpointableSaver(obj)
-  utils_impl.get_or_create_variables_dir(export_dir)
-  object_saver.save(utils_impl.get_variables_path(export_dir))
 
   signatures = _canonicalize_signatures(signatures)
-  graph_def, signatures, saver_def = _make_graph_def(
-      obj, signatures, object_saver)
-  saved_model = saved_model_pb2.SavedModel()
-  saved_model.saved_model_schema_version = (
-      constants.SAVED_MODEL_SCHEMA_VERSION)
-  meta_graph_def = saved_model.meta_graphs.add()
-  meta_graph_def.meta_info_def.tags.append(tag_constants.SERVING)
-  meta_graph_def.saver_def.CopyFrom(saver_def)
   # TODO(allenl): Factor out some subset of SavedModelBuilder which is 2.x
   # compatible (no sessions) and share it with this export API rather than
   # making a SavedModel proto and writing it directly.
-  meta_graph_def.graph_def.MergeFrom(graph_def)
-  for signature_key, signature in signatures.items():
-    meta_graph_def.signature_def[signature_key].MergeFrom(signature)
-  meta_graph.strip_graph_default_valued_attrs(meta_graph_def)
+  saved_model = saved_model_pb2.SavedModel()
+  meta_graph_def = saved_model.meta_graphs.add()
+  object_saver = util.CheckpointableSaver(obj)
+  asset_filename_map = _fill_meta_graph_def(
+      meta_graph_def, obj, signatures, object_saver)
+  saved_model.saved_model_schema_version = (
+      constants.SAVED_MODEL_SCHEMA_VERSION)
+  # So far we've just been generating protocol buffers with no I/O. Now we write
+  # the checkpoint, copy assets into the assets directory, and write out the
+  # SavedModel proto itself.
+  utils_impl.get_or_create_variables_dir(export_dir)
+  object_saver.save(utils_impl.get_variables_path(export_dir))
+  builder_impl.copy_assets_to_destination_dir(asset_filename_map, export_dir)
   path = os.path.join(
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index ef0a92fddd..9e5b9b9717 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -23,6 +23,7 @@ import sys
 
 import numpy
 
+from tensorflow.python.client import session as session_lib
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
@@ -35,7 +36,9 @@ from tensorflow.python.keras.engine import input_layer
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
 from tensorflow.python.keras.layers import merge
+from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import loader
@@ -66,26 +69,27 @@ class _ModelWithOptimizer(training.Model):
     return {"loss": loss}
 
 
-class SaveTest(test.TestCase):
+def _import_and_infer(
+    save_dir, inputs,
+    signature_key=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY):
+  """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
+  graph = ops.Graph()
+  with graph.as_default(), session_lib.Session() as session:
+    model = loader.load(session, [tag_constants.SERVING], save_dir)
+    signature = model.signature_def[signature_key]
+    assert set(inputs.keys()) == set(signature.inputs.keys())
+    feed_dict = {}
+    for arg_name in inputs.keys():
+      feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = (
+          inputs[arg_name])
+    output_dict = {}
+    for output_name, output_tensor_info in signature.outputs.items():
+      output_dict[output_name] = graph.get_tensor_by_name(
+          output_tensor_info.name)
+    return session.run(output_dict, feed_dict=feed_dict)
 
-  def _import_and_infer(
-      self, save_dir, inputs,
-      signature_key=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY):
-    """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
-    graph = ops.Graph()
-    with graph.as_default(), self.session(graph) as session:
-      model = loader.load(session, [tag_constants.SERVING], save_dir)
-      signature = model.signature_def[signature_key]
-      self.assertEqual(set(inputs.keys()), set(signature.inputs.keys()))
-      feed_dict = {}
-      for arg_name in inputs.keys():
-        feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = (
-            inputs[arg_name])
-      output_dict = {}
-      for output_name, output_tensor_info in signature.outputs.items():
-        output_dict[output_name] = graph.get_tensor_by_name(
-            output_tensor_info.name)
-      return session.run(output_dict, feed_dict=feed_dict)
+
+class SaveTest(test.TestCase):
 
   def test_method_save_signature(self):
     root = tracking.Checkpointable()
@@ -97,7 +101,7 @@ class SaveTest(test.TestCase):
     save.save(root, save_dir, root.f)
     self.assertEqual(
         {"output_0": 2.},
-        self._import_and_infer(save_dir, {"x": 1.}))
+        _import_and_infer(save_dir, {"x": 1.}))
 
   def test_method_save_concrete(self):
     root = tracking.Checkpointable()
@@ -112,7 +116,7 @@ class SaveTest(test.TestCase):
             tensor_spec.TensorSpec(None, dtypes.float32))})
     self.assertEqual(
         {"out": 2.},
-        self._import_and_infer(
+        _import_and_infer(
             save_dir, {"z": 1.}, signature_key="non_default_key"))
 
   def test_non_concrete_error(self):
@@ -169,7 +173,7 @@ class SaveTest(test.TestCase):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     save.save(root, save_dir, to_save)
     self.assertAllEqual({"output_0": 12.},
-                        self._import_and_infer(save_dir, {"x": 2.}))
+                        _import_and_infer(save_dir, {"x": 2.}))
 
   def test_optimizer(self):
     x = constant_op.constant([[3., 4.]])
@@ -182,7 +186,7 @@ class SaveTest(test.TestCase):
     self.assertNotEqual(first_loss, second_loss)
     self.assertAllClose(
         second_loss,
-        self._import_and_infer(save_dir, {"x": [[3., 4.]], "y": [2.]}))
+        _import_and_infer(save_dir, {"x": [[3., 4.]], "y": [2.]}))
 
   def test_trivial_save_exception(self):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
@@ -197,8 +201,8 @@ class SaveTest(test.TestCase):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     save.save(model, save_dir)
     self.assertIn("loss",
-                  self._import_and_infer(save_dir,
-                                         {"x": [[3., 4.]], "y": [2.]}))
+                  _import_and_infer(save_dir,
+                                    {"x": [[3., 4.]], "y": [2.]}))
 
   def test_single_function_default_signature(self):
     model = tracking.Checkpointable()
@@ -207,7 +211,7 @@ class SaveTest(test.TestCase):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     save.save(model, save_dir)
     self.assertAllClose({"output_0": 3.},
-                        self._import_and_infer(save_dir, {}))
+                        _import_and_infer(save_dir, {}))
 
   def test_ambiguous_signatures(self):
     model = _ModelWithOptimizer()
@@ -246,7 +250,7 @@ class SaveTest(test.TestCase):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     save.save(to_save, save_dir)
     self.assertAllClose({"output_0": 7.},
-                        self._import_and_infer(save_dir, {"x": 3.}))
+                        _import_and_infer(save_dir, {"x": 3.}))
 
   def test_default_attr_stripping(self):
 
@@ -280,7 +284,7 @@ class SaveTest(test.TestCase):
     save.save(model, save_dir)
     self.assertAllClose(
         {"out": model(array_ops.ones([1, 4]))},
-        self._import_and_infer(save_dir, {"x": [[1., 1., 1., 1.]]}))
+        _import_and_infer(save_dir, {"x": [[1., 1., 1., 1.]]}))
 
   @test_util.run_deprecated_v1
   def test_export_functional_keras_model_after_fit(self):
@@ -294,7 +298,7 @@ class SaveTest(test.TestCase):
     save.save(model, save_dir)
     self.assertAllClose(
         {"y": model(constant_op.constant([[1.], [2.]]))},
-        self._import_and_infer(save_dir, {"input_1": [[1.], [2.]]}))
+        _import_and_infer(save_dir, {"input_1": [[1.], [2.]]}))
 
   def test_export_multi_input_functional_keras_model(self):
     x1 = input_layer.Input((2,), name="x1")
@@ -307,12 +311,49 @@ class SaveTest(test.TestCase):
     outputs = model([array_ops.ones([1, 2]), 2. * array_ops.ones([1, 2])])
     self.assertAllClose(
         {"dense": outputs[0], "dense_1": outputs[1]},
-        self._import_and_infer(
+        _import_and_infer(
             save_dir,
             {"x1": [[1., 1.]],
              "x2": [[2., 2.]]}))
 
 
+class AssetTests(test.TestCase):
+
+  def setUp(self):
+    super(AssetTests, self).setUp()
+    self._vocab_path = os.path.join(self.get_temp_dir(), "vocab.txt")
+    with open(self._vocab_path, "w") as f:
+      f.write("alpha\nbeta\ngamma\n")
+
+  def test_table(self):
+    initializer = lookup_ops.TextFileInitializer(
+        self._vocab_path,
+        key_dtype=dtypes.string,
+        key_index=lookup_ops.TextFileIndex.WHOLE_LINE,
+        value_dtype=dtypes.int64,
+        value_index=lookup_ops.TextFileIndex.LINE_NUMBER)
+    root = util.Checkpoint(table=lookup_ops.HashTable(
+        initializer, default_value=-1))
+    root.table_user = def_function.function(
+        root.table.lookup,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.string)])
+    self.assertEqual(
+        2,
+        self.evaluate(root.table_user(constant_op.constant("gamma"))))
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    save.save(root, save_dir)
+    file_io.delete_file(self._vocab_path)
+    self.assertAllClose(
+        {"output_0": [2, 0]},
+        _import_and_infer(save_dir, {"keys": ["gamma", "alpha"]}))
+    second_dir = os.path.join(self.get_temp_dir(), "second_dir")
+    # Asset paths should track the location the SavedModel is loaded from.
+    file_io.rename(save_dir, second_dir)
+    self.assertAllClose(
+        {"output_0": [2, 1]},
+        _import_and_infer(second_dir, {"keys": ["gamma", "beta"]}))
+
+
 class MemoryTests(test.TestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index c85b208d47..4e96aee0c5 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -17,6 +17,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import data_structures
 from tensorflow.python.util import tf_contextlib
@@ -145,3 +149,36 @@ class TrackableResource(base.CheckpointableBase):
     if self._resource_handle is None:
       self._resource_handle = self.create_resource()
     return self._resource_handle
+
+
+class TrackableAsset(base.CheckpointableBase):
+  """Base class for asset files which need to be tracked."""
+
+  def __init__(self, path):
+    """Record the full path to the asset."""
+    # We use a variable here so that @tf.functions do not capture a literal
+    # value. The init_scope prevents functions from capturing `path` in an
+    # initialization graph, since it is transient and should not end up in a
+    # serialized function body. When serialized in a SavedModel, the variable
+    # will be set during the loading process to its location in the assets/
+    # directory.
+    with ops.init_scope():
+      if context.executing_eagerly():
+        self._path = self._no_dependency(
+            resource_variable_ops.ResourceVariable(
+                path, dtype=dtypes.string,
+                name="asset_path"))
+      else:
+        # Adding a variable is too disruptive when v1-style graph building,
+        # since things may get fed and local variable initializers would then
+        # need to be run.
+        self._path = path
+
+  @property
+  def asset_path(self):
+    """Fetch the current asset path."""
+    return self._path
+
+ops.register_tensor_conversion_function(
+    TrackableAsset,
+    lambda asset, **kw: ops.internal_convert_to_tensor(asset.asset_path, **kw))
-- 
GitLab


From 61160457a356cbb56280764896c8a4ec77bfe715 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 11:26:56 -0800
Subject: [PATCH 1208/1554] Internal Change

PiperOrigin-RevId: 223830118
---
 .../python_api/api_def_BesselI0e.pbtxt        |   4 +-
 .../python_api/api_def_BesselI1e.pbtxt        |   4 +-
 .../core/api_def/python_api/api_def_Erf.pbtxt |   8 +-
 .../core/api_def/python_api/api_def_Neg.pbtxt |   7 +-
 .../api_def/python_api/api_def_Sign.pbtxt     |   7 +-
 .../api_def/python_api/api_def_Sqrt.pbtxt     |   7 +-
 .../api_def/python_api/api_def_Square.pbtxt   |   7 +-
 .../api_def/python_api/api_def_Tanh.pbtxt     |  10 +-
 tensorflow/python/BUILD                       |  12 +
 tensorflow/python/framework/python_op_gen.cc  |  28 ++-
 .../framework/python_op_gen_internal.cc       |   6 +-
 tensorflow/python/ops/array_ops.py            |   4 +-
 tensorflow/python/ops/math_ops.py             | 232 +-----------------
 .../ops/ragged/ragged_elementwise_ops.py      |   5 +
 tensorflow/python/ops/sparse_ops.py           |  47 ++++
 tensorflow/python/ops/sparse_ops_test.py      |  22 +-
 tensorflow/python/util/dispatch.py            | 192 +++++++++++++++
 tensorflow/python/util/dispatch_test.py       | 120 +++++++++
 18 files changed, 483 insertions(+), 239 deletions(-)
 create mode 100644 tensorflow/python/util/dispatch.py
 create mode 100644 tensorflow/python/util/dispatch_test.py

diff --git a/tensorflow/core/api_def/python_api/api_def_BesselI0e.pbtxt b/tensorflow/core/api_def/python_api/api_def_BesselI0e.pbtxt
index 7965af4916..fdbe5282bc 100644
--- a/tensorflow/core/api_def/python_api/api_def_BesselI0e.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_BesselI0e.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "BesselI0e"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.bessel_i0e"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_BesselI1e.pbtxt b/tensorflow/core/api_def/python_api/api_def_BesselI1e.pbtxt
index dffd296f6d..3f08cd766d 100644
--- a/tensorflow/core/api_def/python_api/api_def_BesselI1e.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_BesselI1e.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "BesselI1e"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.bessel_i1e"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt b/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt
index 391167254e..21ae77e9ed 100644
--- a/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt
@@ -1,4 +1,10 @@
 op {
   graph_op_name: "Erf"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.erf"
+  }
+  endpoint {
+    name: "erf"
+    deprecation_version: 2
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Neg.pbtxt b/tensorflow/core/api_def/python_api/api_def_Neg.pbtxt
index 0e2bb9b950..ac166561ee 100644
--- a/tensorflow/core/api_def/python_api/api_def_Neg.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Neg.pbtxt
@@ -1,4 +1,9 @@
 op {
   graph_op_name: "Neg"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.negative"
+  }
+  endpoint {
+    name: "negative"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt b/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt
index c2ee91dd12..fb427cdb19 100644
--- a/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt
@@ -1,4 +1,9 @@
 op {
   graph_op_name: "Sign"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.sign"
+  }
+  endpoint {
+    name: "sign"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt b/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt
index 59e2dfe836..16a4d9a7bc 100644
--- a/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt
@@ -1,4 +1,9 @@
 op {
   graph_op_name: "Sqrt"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.sqrt"
+  }
+  endpoint {
+    name: "sqrt"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Square.pbtxt b/tensorflow/core/api_def/python_api/api_def_Square.pbtxt
index 7b39ae25fa..0bd2f1bf41 100644
--- a/tensorflow/core/api_def/python_api/api_def_Square.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Square.pbtxt
@@ -1,4 +1,9 @@
 op {
   graph_op_name: "Square"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.square"
+  }
+  endpoint {
+    name: "square"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_Tanh.pbtxt b/tensorflow/core/api_def/python_api/api_def_Tanh.pbtxt
index c946e0a794..80d11d2785 100644
--- a/tensorflow/core/api_def/python_api/api_def_Tanh.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_Tanh.pbtxt
@@ -1,4 +1,12 @@
 op {
   graph_op_name: "Tanh"
-  visibility: HIDDEN
+  endpoint {
+    name: "math.tanh"
+  }
+  endpoint {
+    name: "nn.tanh"
+  }
+  endpoint {
+    name: "tanh"
+  }
 }
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 17e228a562..ec85e5cb60 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -525,6 +525,17 @@ py_test(
     ],
 )
 
+py_test(
+    name = "dispatch_test",
+    srcs = ["util/dispatch_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":client_testlib",
+        ":platform",
+        ":util",
+    ],
+)
+
 py_test(
     name = "keyword_args_test",
     srcs = ["util/keyword_args_test.py"],
@@ -2858,6 +2869,7 @@ py_test(
         ":framework_test_lib",
         ":sparse_ops",
         ":sparse_tensor",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 465016b808..d91f7b0bdd 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -142,6 +142,7 @@ class GenEagerPythonOp : public python_op_gen_internal::GenPythonOp {
   void AddEagerAttrs(const string& indentation);
   void AddEagerExecute(const string& indentation,
                        const string& num_outputs_expr);
+  void AddDispatch(const string& prefix);
 
   void AddAttrForArg(const string& attr, int arg_index) {
     gtl::InsertIfNotPresent(&inferred_attrs_, attr,
@@ -356,9 +357,14 @@ string GenEagerPythonOp::Code() {
 
 void GenEagerPythonOp::HandleGraphMode(const string& function_setup) {
   strings::StrAppend(&result_, "  # Add nodes to the TensorFlow graph.\n");
-  strings::StrAppend(&result_, function_setup,
-                     "  _, _, _op = _op_def_lib._apply_op_helper(\n");
-  AddBodyNoReturn("        ");
+  strings::StrAppend(&result_, function_setup);
+  if (api_def_.visibility() == ApiDef::VISIBLE) {
+    strings::StrAppend(&result_, "  try:\n  ");
+  }
+  strings::StrAppend(&result_, "  _, _, _op = _op_def_lib._apply_op_helper(\n");
+  AddBodyNoReturn(strings::StrCat("        \"", op_def_.name(), "\", "));
+  AddDispatch("  ");
+
   if (num_outs_ > 0) {
     strings::StrAppend(&result_, "  _result = _op.outputs[:]\n");
     // Special case handling for stateful op with single list output
@@ -628,6 +634,7 @@ void GenEagerPythonOp::AddEagerFunctionTeardown(
 bool GenEagerPythonOp::AddEagerFastPathAndGraphCode(
     const string& parameters, const std::vector<string>& output_sizes,
     const string& eager_not_allowed_error) {
+  strings::StrAppend(&result_, "@_dispatch.add_dispatch_list\n");
   AddExport();
   AddDefLine(function_name_, parameters);
   AddDocStringDescription();
@@ -758,6 +765,7 @@ void GenEagerPythonOp::AddEagerFastPathExecute() {
   strings::StrAppend(&result_, "      except _core._SymbolicException:\n");
   strings::StrAppend(&result_,
                      "        pass  # Add nodes to the TensorFlow graph.\n");
+  AddDispatch("      ");
 
   // Any errors thrown from execute need to be unwrapped from
   // _NotOkStatusException.
@@ -898,6 +906,19 @@ void GenEagerPythonOp::AddEagerExecute(const string& indentation,
                      WordWrap(return_prefix, return_args, kRightMargin), "\n");
 }
 
+void GenEagerPythonOp::AddDispatch(const string& prefix) {
+  if (api_def_.visibility() != ApiDef::VISIBLE) return;
+
+  strings::StrAppend(&result_, prefix, "except (TypeError, ValueError):\n");
+  strings::StrAppend(&result_, prefix, "  result = _dispatch.dispatch(\n");
+  AddBodyNoReturn(strings::StrCat(prefix, "        ", function_name_, ", "));
+  strings::StrAppend(&result_, prefix,
+                     "  if result is not "
+                     "_dispatch.OpDispatcher.NOT_SUPPORTED:\n");
+  strings::StrAppend(&result_, prefix, "    return result\n");
+  strings::StrAppend(&result_, prefix, "  raise\n");
+}
+
 string GetPythonOps(const OpList& ops, const ApiDefMap& api_defs,
                     const std::vector<string>& hidden_ops, bool require_shapes,
                     const string& source_file_name = "") {
@@ -937,6 +958,7 @@ from tensorflow.python.framework import op_def_registry as _op_def_registry
 from tensorflow.python.framework import ops as _ops
 from tensorflow.python.framework import op_def_library as _op_def_library
 from tensorflow.python.util.deprecation import deprecated_endpoints
+from tensorflow.python.util import dispatch as _dispatch
 from tensorflow.python.util.tf_export import tf_export
 
 )");
diff --git a/tensorflow/python/framework/python_op_gen_internal.cc b/tensorflow/python/framework/python_op_gen_internal.cc
index 65b9ad5c6a..cbdeecfbfb 100644
--- a/tensorflow/python/framework/python_op_gen_internal.cc
+++ b/tensorflow/python/framework/python_op_gen_internal.cc
@@ -804,8 +804,8 @@ void GenPythonOp::AddDocStringOutputs() {
 }
 
 void GenPythonOp::AddBody(const string& prefix) {
-  const string apply_prefix =
-      strings::StrCat(prefix, "_result = _op_def_lib.apply_op(");
+  const string apply_prefix = strings::StrCat(
+      prefix, "_result = _op_def_lib.apply_op(\"", op_def_.name(), "\", ");
   AddBodyNoReturn(apply_prefix);
   if (num_outs_ > 1) {
     strings::StrAppend(&result_, prefix, "_result = _", op_def_.name(),
@@ -815,7 +815,7 @@ void GenPythonOp::AddBody(const string& prefix) {
 }
 
 void GenPythonOp::AddBodyNoReturn(const string& apply_prefix) {
-  string args = strings::StrCat("\"", op_def_.name(), "\", ");
+  string args;
   for (size_t i = 0; i < param_names_.size(); ++i) {
     strings::StrAppend(&args, AvoidPythonReserved(param_names_[i].GetName()),
                        "=", param_names_[i].GetRenameTo(), ", ");
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 496d3855b5..7b6242b1cd 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -41,6 +41,7 @@ from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops.gen_array_ops import *
 from tensorflow.python.ops.gen_array_ops import reverse_v2 as reverse  # pylint: disable=unused-import
 from tensorflow.python.util import deprecation
+from tensorflow.python.util import dispatch
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 # pylint: enable=wildcard-import
@@ -403,7 +404,7 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32):
       input, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)):
     input = ops.convert_to_tensor(input)
     np_out_type = out_type.as_numpy_dtype
-    num_elements = np.prod(input._shape_tuple(), dtype=np_out_type)  # pylint: disable=protected-acces:
+    num_elements = np.prod(input._shape_tuple(), dtype=np_out_type)  # pylint: disable=protected-access
     return ops.convert_to_tensor(num_elements, dtype=out_type)
   with ops.name_scope(name, "Size", [input]) as name:
     if isinstance(input, (sparse_tensor.SparseTensor,
@@ -3243,6 +3244,7 @@ gather.__doc__ = gather_v2.__doc__ = gen_array_ops.gather_v2.__doc__
 
 
 @tf_export("batch_gather")
+@dispatch.add_dispatch_support
 def batch_gather(params, indices, name=None):
   """Gather slices from `params` according to `indices` with leading batch dims.
 
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index c3feb1870d..f576b00371 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -43,6 +43,7 @@ from tensorflow.python.ops.gen_math_ops import *
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
+from tensorflow.python.util import dispatch
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
@@ -165,6 +166,7 @@ def argmin_v2(input,
 # pylint: disable=anomalous-backslash-in-string,protected-access
 # pylint: disable=g-docstring-has-escape
 @tf_export("math.abs", "abs")
+@dispatch.add_dispatch_support
 def abs(x, name=None):  # pylint: disable=redefined-builtin
   r"""Computes the absolute value of a tensor.
 
@@ -189,22 +191,10 @@ def abs(x, name=None):  # pylint: disable=redefined-builtin
       of type `float32` or `float64`, respectively.
   """
   with ops.name_scope(name, "Abs", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      if x.values.dtype.is_complex:
-        x_abs = gen_math_ops.complex_abs(
-            x.values, Tout=x.values.dtype.real_dtype, name=name)
-        return sparse_tensor.SparseTensor(
-            indices=x.indices, values=x_abs, dense_shape=x.dense_shape)
-      x_abs = gen_math_ops._abs(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_abs, dense_shape=x.dense_shape)
-    else:
-      x = ops.convert_to_tensor(x, name="x")
-      if x.dtype.is_complex:
-        return gen_math_ops.complex_abs(x, Tout=x.dtype.real_dtype, name=name)
-      return gen_math_ops._abs(x, name=name)
-
-
+    x = ops.convert_to_tensor(x, name="x")
+    if x.dtype.is_complex:
+      return gen_math_ops.complex_abs(x, Tout=x.dtype.real_dtype, name=name)
+    return gen_math_ops._abs(x, name=name)
 # pylint: enable=g-docstring-has-escape
 
 
@@ -291,31 +281,7 @@ _sub.__doc__ = (
     gen_math_ops.sub.__doc__ + ("" if _sub.__doc__ is None else _sub.__doc__))
 
 
-# pylint: disable=g-docstring-has-escape
-@tf_export("math.negative", "negative")
-def negative(x, name=None):
-  """Computes numerical negative value element-wise.
-
-  I.e., \\(y = -x\\).
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`, `int32`, `int64`, `complex64`, `complex128`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`.
-  """
-  with ops.name_scope(name, "Neg", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_neg = gen_math_ops.neg(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_neg, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.neg(x, name=name)
-
-
-# pylint: enable=g-docstring-has-escape
+negative = gen_math_ops.neg
 
 
 # pylint: disable=g-docstring-has-escape
@@ -341,105 +307,6 @@ def _neg(x, name=None):
 # pylint: enable=g-docstring-has-escape
 
 
-@tf_export("math.sign", "sign")
-def sign(x, name=None):
-  """Returns an element-wise indication of the sign of a number.
-
-  `y = sign(x) = -1` if `x < 0`; 0 if `x == 0` or `tf.is_nan(x)`; 1 if `x > 0`.
-
-  Zero is returned for NaN inputs.
-
-  For complex numbers, `y = sign(x) = x / |x|` if `x != 0`, otherwise `y = 0`.
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`, `int32`, `int64`, `complex64`, `complex128`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`.
-
-  @compatibility(numpy)
-  Equivalent to numpy.sign except for the behavior for input values of NaN.
-  @end_compatibility
-  """
-  with ops.name_scope(name, "Sign", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_sign = gen_math_ops.sign(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_sign, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.sign(x, name=name)
-
-
-@tf_export("math.square", "square")
-def square(x, name=None):
-  r"""Computes square of x element-wise.
-
-  I.e., \\(y = x * x = x^2\\).
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`, `int32`, `int64`, `complex64`, `complex128`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`. Has the same type as `x`.
-  """
-  with ops.name_scope(name, "Square", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_square = gen_math_ops.square(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_square, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.square(x, name=name)
-
-
-@tf_export("math.sqrt", "sqrt")
-def sqrt(x, name=None):
-  r"""Computes square root of x element-wise.
-
-  I.e., \\(y = \sqrt{x} = x^{1/2}\\).
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`, `complex64`, `complex128`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`.
-  """
-  with ops.name_scope(name, "Sqrt", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_sqrt = gen_math_ops.sqrt(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_sqrt, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.sqrt(x, name=name)
-
-
-@tf_export("math.erf", v1=["math.erf", "erf"])
-@deprecation.deprecated_endpoints("erf")
-def erf(x, name=None):
-  """Computes the Gauss error function of `x` element-wise.
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`.
-  """
-  with ops.name_scope(name, "Erf", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_erf = gen_math_ops.erf(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_erf, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.erf(x, name=name)
-
-
 @tf_export(v1=["math.scalar_mul", "scalar_mul"])
 def scalar_mul(scalar, x, name=None):
   """Multiplies a scalar times a `Tensor` or `IndexedSlices` object.
@@ -1757,7 +1624,7 @@ def reduce_variance(input_tensor, axis=None, keepdims=False, name=None):
   name = name if name else "reduce_variance"
   with ops.name_scope(name):
     means = reduce_mean(input_tensor, axis=axis, keepdims=True)
-    squared_deviations = square(input_tensor - means)
+    squared_deviations = gen_math_ops.square(input_tensor - means)
     return reduce_mean(squared_deviations, axis=axis, keepdims=keepdims)
 
 
@@ -1804,7 +1671,7 @@ def reduce_std(input_tensor, axis=None, keepdims=False, name=None):
   name = name if name else "reduce_std"
   with ops.name_scope(name):
     variance = reduce_variance(input_tensor, axis=axis, keepdims=keepdims)
-    return sqrt(variance)
+    return gen_math_ops.sqrt(variance)
 
 
 @tf_export("math.reduce_prod", "reduce_prod", v1=[])
@@ -2915,27 +2782,6 @@ def log_sigmoid(x, name=None):
     return gen_math_ops.neg(gen_nn_ops.softplus(-x), name=name)
 
 
-@tf_export("math.tanh", "nn.tanh", "tanh")
-def tanh(x, name=None):
-  """Computes hyperbolic tangent of `x` element-wise.
-
-  Args:
-    x: A Tensor or SparseTensor with type `float16`, `float32`, `double`,
-      `complex64`, or `complex128`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A Tensor or SparseTensor respectively with the same type as `x`.
-  """
-  with ops.name_scope(name, "Tanh", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_tanh = gen_math_ops.tanh(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_tanh, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.tanh(x, name=name)
-
-
 @tf_export("math.bincount", v1=[])
 def bincount(arr,
              weights=None,
@@ -3762,63 +3608,3 @@ def polyval(coeffs, x, name=None):
     for c in coeffs[1:]:
       p = c + p * x
     return p
-
-
-@tf_export("math.bessel_i0e")
-def bessel_i0e(x, name=None):
-  """Computes the Bessel i0e function of `x` element-wise.
-
-  Exponentially scaled modified Bessel function of order 0 defined as
-  `bessel_i0e(x) = exp(-abs(x)) bessel_i0(x)`.
-
-  This function is faster and numerically stabler than `bessel_i0(x)`.
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`.
-
-  @compatibility(scipy)
-  Equivalent to scipy.special.i0e
-  @end_compatibility
-  """
-  with ops.name_scope(name, "bessel_i0e", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_i0e = gen_math_ops.bessel_i0e(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_i0e, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.bessel_i0e(x, name=name)
-
-
-@tf_export("math.bessel_i1e")
-def bessel_i1e(x, name=None):
-  """Computes the Bessel i1e function of `x` element-wise.
-
-  Exponentially scaled modified Bessel function of order 1 defined as
-  `bessel_i1e(x) = exp(-abs(x)) bessel_i1(x)`.
-
-  This function is faster and numerically stabler than `bessel_i1(x)`.
-
-  Args:
-    x: A `Tensor` or `SparseTensor`. Must be one of the following types: `half`,
-      `float32`, `float64`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` or `SparseTensor`, respectively. Has the same type as `x`.
-
-  @compatibility(scipy)
-  Equivalent to scipy.special.i1e
-  @end_compatibility
-  """
-  with ops.name_scope(name, "bessel_i1e", [x]) as name:
-    if isinstance(x, sparse_tensor.SparseTensor):
-      x_i1e = gen_math_ops.bessel_i1e(x.values, name=name)
-      return sparse_tensor.SparseTensor(
-          indices=x.indices, values=x_i1e, dense_shape=x.dense_shape)
-    else:
-      return gen_math_ops.bessel_i1e(x, name=name)
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
index a497500a6a..59b7dd1661 100644
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
@@ -269,6 +269,10 @@ def _add_elementwise_ops_to_this_module(specs, verbose=False):
       op_name = canonical_name
     else:
       op_name = original_op.__name__
+
+    # Temporary hack (will be removed once dispatch is added for RaggedTensors):
+    if op_name == 'neg': op_name = 'negative'
+
     if verbose:
       print('Adding ragged_elementwise_op: tf.ragged.%s (based on tf.%s)' %
             (op_name, canonical_name))
@@ -382,3 +386,4 @@ _TF_ELEMENTWISE_OPS = [
     (parsing_ops.string_to_number, 'string_tensor'),
 ]
 _add_elementwise_ops_to_this_module(_TF_ELEMENTWISE_OPS)
+
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 245080cb26..bbb0ab1378 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -44,6 +44,9 @@ from tensorflow.python.ops.gen_sparse_ops import *
 # pylint: enable=wildcard-import
 from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
+from tensorflow.python.util import dispatch
+from tensorflow.python.util import tf_inspect
+from tensorflow.python.util.tf_export import get_canonical_name_for_symbol
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -2673,3 +2676,47 @@ def _take_many_sparse_from_tensors_map(sparse_map_op,
   output_shape.set_shape([rank])
 
   return sparse_tensor.SparseTensor(output_indices, output_values, output_shape)
+
+
+class _UnaryMapValueDispatcher(dispatch.OpDispatcher):
+  """OpDispatcher for unary ops that maps base function across sparse values."""
+
+  def __init__(self, original_func):
+    self._original_func = original_func
+    func_name = get_canonical_name_for_symbol(original_func)
+    arg_names = tf_inspect.getfullargspec(original_func)[0]
+    self._x = arg_names[0]
+    original_func.__doc__ = (
+        original_func.__doc__.rstrip() + "\n\n" +
+        ("    If `{x}` is a `SparseTensor`, returns\n"
+         "    `SparseTensor({x}.indices, tf.{func}({x}.values, ...), "
+         "{x}.dense_shape)`").format(x=self._x, func=func_name))
+
+  def handle(self, args, kwargs):
+    if args:
+      x, args = args[0], args[1:]
+    else:
+      x = kwargs.pop(self._x, None)
+    if isinstance(x, sparse_tensor.SparseTensor):
+      return sparse_tensor.SparseTensor(
+          indices=x.indices,
+          values=self._original_func(x.values, *args, **kwargs),
+          dense_shape=x.dense_shape)
+    else:
+      return self.NOT_SUPPORTED
+
+
+_UNARY_OPS = [
+    # TODO(b/120307967) Add dispatchers for additional TensorFlow ops.
+    math_ops.abs,
+    math_ops.negative,
+    math_ops.sign,
+    math_ops.square,
+    math_ops.sqrt,
+    math_ops.erf,
+    math_ops.tanh,
+    math_ops.bessel_i0e,
+    math_ops.bessel_i1e,
+]
+for unary_op in _UNARY_OPS:
+  _UnaryMapValueDispatcher(unary_op).register(unary_op)
diff --git a/tensorflow/python/ops/sparse_ops_test.py b/tensorflow/python/ops/sparse_ops_test.py
index 4ee1569249..031069a0f0 100644
--- a/tensorflow/python/ops/sparse_ops_test.py
+++ b/tensorflow/python/ops/sparse_ops_test.py
@@ -18,18 +18,20 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import googletest
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class SparseOpsTest(test_util.TensorFlowTestCase):
+class SparseOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testSparseEye(self):
     def test_one(n, m, as_tensors):
@@ -77,5 +79,23 @@ class SparseOpsTest(test_util.TensorFlowTestCase):
           d = sparse_ops.sparse_to_dense(s.indices, s.dense_shape, s.values)
           self.assertAllEqual(self.evaluate(d), expected_after)
 
+  @parameterized.parameters([
+      (math_ops.abs, [1.0, -1.0, 3.0, -4.0], [1.0, 1.0, 3.0, 4.0]),
+      (math_ops.negative, [1.0, -1.0, 3.0, -4.0], [-1.0, 1.0, -3.0, 4.0]),
+      (math_ops.sign, [3.0, -2.0, 0.0, -4.0], [1.0, -1.0, 0.0, -1.0]),
+      (math_ops.square, [1.0, -1.0, 3.0, -4.0], [1.0, 1.0, 9.0, 16.0]),
+  ])
+  def testUnarySparseDispatch(self, op, values, expected):
+    st = sparse_tensor.SparseTensor(
+        indices=[[0, 0], [0, 1], [2, 0], [2, 4]],
+        values=values,
+        dense_shape=[3, 6])
+    result = op(st)
+    result_value = self.evaluate(result)
+    self.assertAllEqual(result_value.indices, st.indices)
+    self.assertAllEqual(result_value.values, expected)
+    self.assertAllEqual(result_value.dense_shape, st.dense_shape)
+
+
 if __name__ == '__main__':
   googletest.main()
diff --git a/tensorflow/python/util/dispatch.py b/tensorflow/python/util/dispatch.py
new file mode 100644
index 0000000000..e7a56b5922
--- /dev/null
+++ b/tensorflow/python/util/dispatch.py
@@ -0,0 +1,192 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Type-based dispatch for TensorFlow ops.
+
+"Operation dispatchers" can be used to override the behavior for TensorFlow ops
+when they are called with otherwise unsupported argument types.  In particular,
+when an operation is called with arguments that would cause it to raise a
+TypeError, it falls back on its registered operation dispatchers.  If any
+registered dispatchers can handle the arguments, then its result is returned.
+Otherwise, the original TypeError is raised.
+
+By default, dispatch support is added to the generated op wrappers for any
+visible ops by default.  Ops that are implemented in Python can opt in to
+dispatch support using the `add_dispatch_support` decorator.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
+
+# Private function attribute used to store a list of dispatchers.
+DISPATCH_ATTR = "_tf_dispatchers"
+
+
+class OpDispatcher(object):
+  """Abstract base class for TensorFlow operator dispatchers.
+
+  Each operation dispatcher acts as an override handler for a single
+  TensorFlow operation, and its results are used when the handler indicates
+  that it can handle the operation's arguments (by returning any value other
+  than `OpDispatcher.NOT_SUPPORTED`).
+  """
+
+  # Sentinel value that can be returned to indicate that an operation
+  # dispatcher does not support a given set of arguments.
+  NOT_SUPPORTED = object()
+
+  def handle(self, args, kwargs):  # pylint: disable=unused-argument
+    """Handle this dispatcher's operation with the specified arguments.
+
+    If this operation dispatcher can handle the given arguments, then
+    return an appropriate value (or raise an appropriate exception).
+
+    Args:
+      args: The arguments to the operation.
+      kwargs: They keyword arguments to the operation.
+
+    Returns:
+      The result of the operation, or `OpDispatcher.NOT_SUPPORTED` if this
+      dispatcher can not handle the given arguments.
+    """
+    return self.NOT_SUPPORTED
+
+  def register(self, op):
+    """Register this dispatcher as a handler for `op`.
+
+    Args:
+      op: Python function: the TensorFlow operation that should be handled. Must
+        have a dispatch list (which is added automatically for generated ops,
+        and can be added to Python ops using the `add_dispatch_support`
+        decorator).
+    """
+    if not hasattr(op, DISPATCH_ATTR):
+      raise AssertionError("Dispatching not enabled for %s" % op)
+    getattr(op, DISPATCH_ATTR).append(self)
+
+
+def dispatch(op, *args, **kwargs):
+  """Returns the result from the first successful dispatcher for a given op.
+
+  Calls the `handle` method of each `OpDispatcher` that has been registered
+  to handle `op`, and returns the value from the first successful handler.
+
+  Args:
+    op: Python function: the operation to dispatch for.
+    *args: The arguments to the operation.
+    **kwargs: They keyword arguments to the operation.
+
+  Returns:
+    The result of the operation, or `NOT_SUPPORTED` if no registered
+    dispatcher can handle the given arguments.
+  """
+  for dispatcher in getattr(op, DISPATCH_ATTR):
+    result = dispatcher.handle(args, kwargs)
+    if result is not OpDispatcher.NOT_SUPPORTED:
+      return result
+  return OpDispatcher.NOT_SUPPORTED
+
+
+class _TypeBasedDispatcher(OpDispatcher):
+  """Dispatcher that handles op if any arguments have a specified type.
+
+  Checks the types of the arguments and keyword arguments (including elements
+  of lists or tuples), and if any argument values have the indicated type(s),
+  then delegates to an override function.
+  """
+
+  def __init__(self, override_func, types):
+    self._types = types
+    self._override_func = override_func
+
+  def _handles(self, args, kwargs):
+    for arg in itertools.chain(args, kwargs.values()):
+      if (isinstance(arg, self._types) or
+          (isinstance(arg, (list, tuple)) and
+           any(isinstance(elt, self._types) for elt in arg))):
+        return True
+    return False
+
+  def handle(self, args, kwargs):
+    if self._handles(args, kwargs):
+      return self._override_func(*args, **kwargs)
+    else:
+      return self.NOT_SUPPORTED
+
+
+# pylint: disable=g-doc-return-or-yield
+def dispatch_for_types(op, *types):
+  """Decorator to declare that a Python function overrides an op for a type.
+
+  The decorated function is used to override `op` if any of the arguments or
+  keyword arguments (including elements of lists or tuples) have one of the
+  specified types.
+
+  Example:
+
+  ```python
+  @dispatch_for_types(math_ops.add, RaggedTensor, RaggedTensorValue)
+  def ragged_add(x, y, name=None): ...
+  ```
+
+  Args:
+    op: Python function: the operation that should be overridden.
+    *types: The argument types for which this function should be used.
+  """
+
+  def decorator(func):
+    if tf_inspect.getargspec(func) != tf_inspect.getargspec(op):
+      raise AssertionError("The decorated function's signature must exactly "
+                           "match the signature of the overridden op.")
+    _TypeBasedDispatcher(func, types).register(op)
+    return func
+
+  return decorator
+
+
+# pylint: enable=g-doc-return-or-yield
+
+
+def add_dispatch_list(target):
+  """Decorator that adds a dispatch_list attribute to an op."""
+  assert not hasattr(target, DISPATCH_ATTR)
+  setattr(target, DISPATCH_ATTR, [])
+  return target
+
+
+def add_dispatch_support(target):
+  """Decorator that adds a dispatch handling wrapper to an op."""
+  add_dispatch_list(target)
+
+  def wrapper(*args, **kwargs):
+    """Call target, and fall back on dispatchers if there is a TypeError."""
+    try:
+      return target(*args, **kwargs)
+    except (TypeError, ValueError):
+      # Note: convert_to_eager_tensor currently raises a ValueError, not a
+      # TypeError, when given unexpected types.  So we need to catch both.
+      result = dispatch(wrapper, *args, **kwargs)
+      if result is not OpDispatcher.NOT_SUPPORTED:
+        return result
+      else:
+        raise
+
+  setattr(wrapper, DISPATCH_ATTR, [])
+  return tf_decorator.make_decorator(target, wrapper)
diff --git a/tensorflow/python/util/dispatch_test.py b/tensorflow/python/util/dispatch_test.py
new file mode 100644
index 0000000000..b7c5c8eca8
--- /dev/null
+++ b/tensorflow/python/util/dispatch_test.py
@@ -0,0 +1,120 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for operator dispatch."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import googletest
+from tensorflow.python.util import dispatch
+from tensorflow.python.util.tf_export import tf_export
+
+
+class CustomTensor(object):
+  """A fake composite tensor class, for testing type-based dispatching."""
+
+  def __init__(self, tensor, score):
+    self.tensor = ops.convert_to_tensor(tensor)
+    self.score = score
+
+
+@tf_export("test_op")
+@dispatch.add_dispatch_support
+def test_op(x, y, z):
+  """A fake op for testing dispatch of Python ops."""
+  return x + (2 * y) + (3 * z)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class DispatchTest(test_util.TensorFlowTestCase):
+
+  def testAddDispatchForTypes_With_CppOp(self):
+    original_handlers = gen_math_ops.add._tf_dispatchers[:]
+
+    # Override the behavior of gen_math_ops.add.
+    @dispatch.dispatch_for_types(gen_math_ops.add, CustomTensor)
+    def custom_add(x, y, name=None):  # pylint: disable=unused-variable
+      return CustomTensor(gen_math_ops.add(x.tensor, y.tensor, name),
+                          (x.score+y.score) / 2.0)
+    self.assertEqual(len(math_ops.add._tf_dispatchers),
+                     len(original_handlers) + 1)
+
+    # Test that we see the overridden behavior when using CustomTensors.
+    x = CustomTensor([1, 2, 3], 2.0)
+    y = CustomTensor([7, 8, 2], 0.0)
+    x_plus_y = gen_math_ops.add(x, y)
+    self.assertAllEqual(self.evaluate(x_plus_y.tensor), [8, 10, 5])
+    self.assertNear(x_plus_y.score, 1.0, 0.001)
+
+    # Test that we still get the right behavior when using normal Tensors.
+    a = [1, 2, 3]
+    b = [4, 5, 6]
+    a_plus_b = gen_math_ops.add(a, b)
+    self.assertAllEqual(a_plus_b, [5, 7, 9])
+
+    # Test that we still get a TypeError or ValueError if we pass some
+    # type that's not supported by any dispatcher.
+    with self.assertRaises((TypeError, ValueError)):
+      gen_math_ops.add(a, None)
+
+    # Clean up
+    gen_math_ops.add._tf_dispatchers = original_handlers
+
+  def testAddDispatchForTypes_With_PythonOp(self):
+    original_handlers = test_op._tf_dispatchers[:]
+
+    @dispatch.dispatch_for_types(test_op, CustomTensor)
+    def override_for_test_op(x, y, z):  # pylint: disable=unused-variable
+      return CustomTensor(test_op(x.tensor, y.tensor, z.tensor),
+                          (x.score + y.score + z.score) / 3.0)
+
+    x = CustomTensor([1, 2, 3], 0.2)
+    y = CustomTensor([7, 8, 2], 0.4)
+    z = CustomTensor([0, 1, 2], 0.6)
+
+    result = test_op(x, y, z)
+    self.assertAllEqual(self.evaluate(result.tensor), [15, 21, 13])
+    self.assertNear(result.score, 0.4, 0.001)
+
+    # Clean up
+    test_op._tf_dispatchers = original_handlers
+
+  def testDispatchForTypes_SignatureMismatch(self):
+    with self.assertRaisesRegexp(AssertionError, "The decorated function's "
+                                 "signature must exactly match.*"):
+      @dispatch.dispatch_for_types(test_op, CustomTensor)
+      def override_for_test_op(a, b, c):  # pylint: disable=unused-variable
+        return CustomTensor(test_op(a.tensor, b.tensor, c.tensor),
+                            (a.score + b.score + c.score) / 3.0)
+
+  def testDispatchForTypes_OpDoesNotSupportDispatch(self):
+    def some_op(x, y):
+      return x + y
+
+    with self.assertRaisesRegexp(AssertionError, "Dispatching not enabled for"):
+      @dispatch.dispatch_for_types(some_op, CustomTensor)
+      def override_for_some_op(x, y):  # pylint: disable=unused-variable
+        return x if x.score > 0 else y
+
+
+if __name__ == "__main__":
+  googletest.main()
+
+
-- 
GitLab


From 43344c5014b2d33074feb6ff537261f9827453a6 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 3 Dec 2018 11:42:11 -0800
Subject: [PATCH 1209/1554] Update Eigen to commit 2e50f4a5542a.

PiperOrigin-RevId: 223833239
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ba981f43a8..83fd940a63 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "6e505fa8bf8d234d0338679b390cb89f850d870214c751b01a5db7f647e4d438",
-        strip_prefix = "eigen-eigen-135398e50bed",
+        sha256 = "37a483ec219c43219b6e0fc07e799277a4a36abb2b9f4162cfcd256aa211eae8",
+        strip_prefix = "eigen-eigen-2e50f4a5542a",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/135398e50bed.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/135398e50bed.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/2e50f4a5542a.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/2e50f4a5542a.tar.gz",
         ],
     )
 
-- 
GitLab


From 2b145abb819b2c0e266e01b86c370c7c506ac95d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 3 Dec 2018 11:58:05 -0800
Subject: [PATCH 1210/1554] Add test wrappers to limit tests to v1, v2, gpu &
 cuda

PiperOrigin-RevId: 223836130
---
 tensorflow/python/framework/test_util.py | 133 +++++++++++++++++++++++
 1 file changed, 133 insertions(+)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 155e45d16c..fc1a5fbe85 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1041,6 +1041,139 @@ def run_deprecated_v1(func=None):
   return decorator
 
 
+def run_v1_only(reason, func=None):
+  """Execute the decorated test only if running in v1 mode.
+
+  This function is intended to be applied to tests that exercise v1 only
+  functionality. If the test is run in v2 mode it will simply be skipped.
+
+  Args:
+    reason: string giving a reason for limiting the test to v1 only.
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+
+  Returns:
+    Returns a decorator that will conditionally skip the decorated test method.
+  """
+
+  def decorator(f):
+    if tf_inspect.isclass(f):
+      raise ValueError("`run_v1_only` only supports test methods.")
+
+    def decorated(self, *args, **kwargs):
+      if tf2.enabled():
+        self.skipTest(reason)
+
+      f(self, *args, **kwargs)
+
+    return decorated
+
+  if func is not None:
+    return decorator(func)
+
+  return decorator
+
+
+def run_v2_only(func=None):
+  """Execute the decorated test only if running in v2 mode.
+
+  This function is intended to be applied to tests that exercise v2 only
+  functionality. If the test is run in v1 mode it will simply be skipped.
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+
+  Returns:
+    Returns a decorator that will conditionally skip the decorated test method.
+  """
+
+  def decorator(f):
+    if tf_inspect.isclass(f):
+      raise ValueError("`run_v2_only` only supports test methods.")
+
+    def decorated(self, *args, **kwargs):
+      if not tf2.enabled():
+        self.skipTest("Test is only comptaible in v2")
+
+      f(self, *args, **kwargs)
+
+    return decorated
+
+  if func is not None:
+    return decorator(func)
+
+  return decorator
+
+
+def run_gpu_only(func=None):
+  """Execute the decorated test only if a GPU is available.
+
+  This function is intended to be applied to tests that require the precense
+  of a GPU. If a GPU is absent, it will simply be skipped.
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+
+  Returns:
+    Returns a decorator that will conditionally skip the decorated test method.
+  """
+
+  def decorator(f):
+    if tf_inspect.isclass(f):
+      raise ValueError("`run_gpu_only` only supports test methods.")
+
+    def decorated(self, *args, **kwargs):
+      if not is_gpu_available():
+        self.skipTest("Test requires GPU")
+
+      f(self, *args, **kwargs)
+
+    return decorated
+
+  if func is not None:
+    return decorator(func)
+
+  return decorator
+
+
+def run_cuda_only(func=None):
+  """Execute the decorated test only if a GPU is available.
+
+  This function is intended to be applied to tests that require the precense
+  of a CUDA GPU. If a CUDA GPU is absent, it will simply be skipped.
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+
+  Returns:
+    Returns a decorator that will conditionally skip the decorated test method.
+  """
+
+  def decorator(f):
+    if tf_inspect.isclass(f):
+      raise ValueError("`run_cuda_only` only supports test methods.")
+
+    def decorated(self, *args, **kwargs):
+      if not is_gpu_available(cuda_only=True):
+        self.skipTest("Test requires CUDA GPU")
+
+      f(self, *args, **kwargs)
+
+    return decorated
+
+  if func is not None:
+    return decorator(func)
+
+  return decorator
+
+
 @tf_export("test.is_gpu_available")
 def is_gpu_available(cuda_only=False, min_cuda_compute_capability=None):
   """Returns whether TensorFlow can access a GPU.
-- 
GitLab


From 410838cfddfab95748d834c56e0a031b3921b1fe Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Mon, 3 Dec 2018 12:10:23 -0800
Subject: [PATCH 1211/1554] Update sync_replica_optimizer docstring to add a
 deprecation message.

PiperOrigin-RevId: 223838900
---
 tensorflow/python/training/sync_replicas_optimizer.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index fbde8fe3c2..172c141150 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -44,6 +44,9 @@ from tensorflow.python.util.tf_export import tf_export
 class SyncReplicasOptimizer(optimizer.Optimizer):
   """Class to synchronize, aggregate gradients and pass them to the optimizer.
 
+  This class is deprecated. For synchrononous training, please use [Distribution
+  Strategies](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/distribute).
+
   In a typical asynchronous training environment, it's common to have some
   stale gradients. For example, with a N-replica asynchronous training,
   gradients will be applied to the variables N times independently. Depending
@@ -142,9 +145,9 @@ class SyncReplicasOptimizer(optimizer.Optimizer):
 
   @deprecation.deprecated(
       None,
-      "The `SyncReplicaOptimizer` is deprecated. For synchrononous training, "
-      "please use [Distribution Strategies](https://github.com/tensorflow/"
-      "tensorflow/tree/master/tensorflow/contrib/distribute).",
+      "The `SyncReplicaOptimizer` class is deprecated. For synchrononous "
+      "training, please use [Distribution Strategies](https://github.com/"
+      "tensorflow/tensorflow/tree/master/tensorflow/contrib/distribute).",
       warn_once=True)
   def __init__(self,
                opt,
-- 
GitLab


From db22ae4d912dd5d25a0394d4c596e02a2107a24e Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 3 Dec 2018 12:16:22 -0800
Subject: [PATCH 1212/1554] Import rename for Estimator modules.

PiperOrigin-RevId: 223840000
---
 .../serialization/checkpoint_input_pipeline_hook_test.py      | 4 ++--
 tensorflow/python/feature_column/feature_column_v2_test.py    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
index 140ed517ef..8cc66d0c29 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/checkpoint_input_pipeline_hook_test.py
@@ -21,8 +21,6 @@ from __future__ import print_function
 from tensorflow.python.data.experimental.ops import iterator_ops
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.estimator import estimator
-from tensorflow.python.estimator import model_fn
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -33,6 +31,8 @@ from tensorflow.python.platform import test
 from tensorflow.python.training import checkpoint_management
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.training import training_util
+from tensorflow_estimator.python.estimator import estimator
+from tensorflow_estimator.python.estimator import model_fn
 
 
 class CheckpointInputPipelineHookTest(test_base.DatasetTestBase):
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 083cd526e4..0755c0b6ac 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -31,7 +31,6 @@ from tensorflow.python import keras
 from tensorflow.python.client import session
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
-from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column as fc_old
 from tensorflow.python.feature_column import feature_column_v2 as fc
 from tensorflow.python.framework import constant_op
@@ -50,6 +49,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.training import coordinator
 from tensorflow.python.training import queue_runner_impl
 from tensorflow.python.training import rmsprop
+from tensorflow_estimator.python.estimator.inputs import numpy_io
 
 
 def _initialized_session(config=None):
-- 
GitLab


From 5973383f52bc66f26b59d68279d8e8e820b1d6a1 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 3 Dec 2018 12:20:56 -0800
Subject: [PATCH 1213/1554] Change batchnorm expander to use GetDimensionSize.

This is necessary if the input of batchnorm has dynamic
dimension. Otherwise the mean calculation would be incorrect.

PiperOrigin-RevId: 223840830
---
 .../xla/service/batchnorm_expander.cc         | 65 +++++++++++--------
 .../xla/service/batchnorm_expander_test.cc    | 18 ++++-
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  2 +-
 .../xla/service/gpu/nvptx_compiler.cc         |  3 +-
 4 files changed, 58 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/xla/service/batchnorm_expander.cc b/tensorflow/compiler/xla/service/batchnorm_expander.cc
index f70f6ddfec..0e6ca1871b 100644
--- a/tensorflow/compiler/xla/service/batchnorm_expander.cc
+++ b/tensorflow/compiler/xla/service/batchnorm_expander.cc
@@ -107,19 +107,37 @@ class BatchNormExpanderVisitor : public DfsHloVisitorWithDefault {
   }
 
   std::unique_ptr<HloInstruction> Mean(
-      int64 element_count, HloInstruction* operand,
+      HloInstruction* element_count, HloInstruction* operand,
       const std::function<HloInstruction*(std::unique_ptr<HloInstruction>)>&
           add_instruction) {
-    HloInstruction* elem_count_recip =
-        add_instruction(HloInstruction::CreateBroadcast(
-            operand->shape(),
-            add_instruction(HloInstruction::CreateConvert(
-                ShapeUtil::MakeShape(operand->shape().element_type(), {}),
-                add_instruction(HloInstruction::CreateConstant(
-                    LiteralUtil::CreateR0<float>(1.0 / element_count))))),
-            {}));
-    return HloInstruction::CreateBinary(operand->shape(), HloOpcode::kMultiply,
-                                        operand, elem_count_recip);
+    auto broadcast = add_instruction(
+        HloInstruction::CreateBroadcast(operand->shape(), element_count, {}));
+    return HloInstruction::CreateBinary(operand->shape(), HloOpcode::kDivide,
+                                        operand, broadcast);
+  }
+
+  std::unique_ptr<HloInstruction> DynamicElementCountPerFeature(
+      HloInstruction* operand, int64 feature_index,
+      const std::function<HloInstruction*(std::unique_ptr<HloInstruction>)>&
+          add_instruction) {
+    auto elements_per_feature_u32 = add_instruction(
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(1)));
+
+    for (int64 i = 0; i < ShapeUtil::Rank(operand->shape()); ++i) {
+      if (i == feature_index) {
+        continue;
+      }
+      auto dynamic_dimension_size =
+          add_instruction(HloInstruction::CreateGetDimensionSize(
+              ShapeUtil::MakeShape(U32, {}), operand, i));
+      elements_per_feature_u32 = add_instruction(HloInstruction::CreateBinary(
+          ShapeUtil::MakeShape(U32, {}), HloOpcode::kMultiply,
+          dynamic_dimension_size, elements_per_feature_u32));
+    }
+
+    return HloInstruction::CreateConvert(
+        ShapeUtil::MakeShape(operand->shape().element_type(), {}),
+        elements_per_feature_u32);
   }
 
   // Replaces the existing HLO instruction old_instruction, with
@@ -195,9 +213,6 @@ Status BatchNormExpanderVisitor::HandleBatchNormTraining(
   const Shape operand_shape = operand->shape();
   PrimitiveType ptype = operand_shape.element_type();
   int64 feature_index = batch_norm->feature_index();
-  const int64 feature_count = operand_shape.dimensions(feature_index);
-  const int64 size_in_elements = ShapeUtil::ElementsIn(operand_shape);
-  int64 elements_per_feature_int64 = size_in_elements / feature_count;
 
   HloInstruction* scale = batch_norm->mutable_operand(1);
   HloInstruction* offset = batch_norm->mutable_operand(2);
@@ -220,6 +235,9 @@ Status BatchNormExpanderVisitor::HandleBatchNormTraining(
     }
   }
 
+  auto elements_per_feature =
+      add(DynamicElementCountPerFeature(operand, feature_index, add));
+
   auto scale_broadcasted = add(
       HloInstruction::CreateBroadcast(operand_shape, scale, {feature_index}));
 
@@ -243,13 +261,13 @@ Status BatchNormExpanderVisitor::HandleBatchNormTraining(
       add_reduce_computation));
 
   // E[X].
-  auto mean = add(Mean(elements_per_feature_int64, sum, add));
+  auto mean = add(Mean(elements_per_feature, sum, add));
 
   auto mean_broadcasted = add(
       HloInstruction::CreateBroadcast(operand_shape, mean, {feature_index}));
 
   // E[X^2].
-  auto square_mean = add(Mean(elements_per_feature_int64, squared_sum, add));
+  auto square_mean = add(Mean(elements_per_feature, squared_sum, add));
 
   // E^2[X].
   auto mean_square =
@@ -458,9 +476,8 @@ Status BatchNormExpanderVisitor::HandleBatchNormGrad(
 
   int64 feature_index = batch_norm->feature_index();
 
-  const int64 size_in_elements = ShapeUtil::ElementsIn(activation_shape);
-  const int64 feature_count = activation_shape.dimensions(feature_index);
-  const int64 elements_per_feature_int64 = size_in_elements / feature_count;
+  auto elements_per_feature =
+      add(DynamicElementCountPerFeature(activation, feature_index, add));
 
   auto zero_literal = LiteralUtil::CreateR0(0.0f);
   TF_ASSIGN_OR_RETURN(zero_literal, zero_literal.Convert(ptype));
@@ -553,15 +570,9 @@ Status BatchNormExpanderVisitor::HandleBatchNormGrad(
       add_binary(activation_shape, HloOpcode::kMultiply, scale_broadcasted,
                  rsqrt_var_add_epsilon_broadcasted);
 
-  scale_times_rsqrt_var_add_epsilon = add(
-      Mean(elements_per_feature_int64, scale_times_rsqrt_var_add_epsilon, add));
+  scale_times_rsqrt_var_add_epsilon =
+      add(Mean(elements_per_feature, scale_times_rsqrt_var_add_epsilon, add));
 
-  auto elements_per_feature_literal =
-      LiteralUtil::CreateR0<float>(elements_per_feature_int64);
-  TF_ASSIGN_OR_RETURN(elements_per_feature_literal,
-                      elements_per_feature_literal.Convert(ptype));
-  auto elements_per_feature = add(
-      HloInstruction::CreateConstant(std::move(elements_per_feature_literal)));
   auto i1 = add_binary(activation_shape, HloOpcode::kMultiply, grad_output,
                        add(HloInstruction::CreateBroadcast(
                            activation_shape, elements_per_feature, {})));
diff --git a/tensorflow/compiler/xla/service/batchnorm_expander_test.cc b/tensorflow/compiler/xla/service/batchnorm_expander_test.cc
index 08cf802617..8e8fbbd935 100644
--- a/tensorflow/compiler/xla/service/batchnorm_expander_test.cc
+++ b/tensorflow/compiler/xla/service/batchnorm_expander_test.cc
@@ -36,7 +36,21 @@ limitations under the License.
 namespace xla {
 namespace {
 
-using BatchNormExpanderTest = HloTestBase;
+class BatchNormExpanderTest : public HloTestBase {
+ protected:
+  // BatchNorm should have a dynamic sized dividor for mean operations.
+  int64 CountGetDimensionSize(const HloModule& module) {
+    int64 count = 0;
+    for (HloComputation* comp : module.computations()) {
+      for (HloInstruction* inst : comp->instructions()) {
+        if (inst->opcode() == HloOpcode::kGetDimensionSize) {
+          count++;
+        }
+      }
+    }
+    return count;
+  }
+};
 
 // Test that we expand BatchNormTraining.
 TEST_F(BatchNormExpanderTest, BatchNormTraining) {
@@ -68,6 +82,7 @@ TEST_F(BatchNormExpanderTest, BatchNormTraining) {
                              /*rewrite_grad_op=*/true);
   ASSERT_TRUE(rewriter.Run(module.get()).ValueOrDie());
   root = computation->root_instruction();
+  EXPECT_EQ(CountGetDimensionSize(*module), 3);
   // Make sure this operation is expanded.
   EXPECT_EQ(root->opcode(), HloOpcode::kTuple);
 }
@@ -110,6 +125,7 @@ TEST_F(BatchNormExpanderTest, BatchNormGrad) {
                              /*rewrite_grad_op=*/true);
   ASSERT_TRUE(rewriter.Run(module.get()).ValueOrDie());
   root = computation->root_instruction();
+  EXPECT_EQ(CountGetDimensionSize(*module), 3);
   // Make sure this operation is expanded.
   EXPECT_EQ(root->opcode(), HloOpcode::kTuple);
 }
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 2bf24c15c1..6374822c81 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -250,7 +250,6 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
       &pipeline, module->config().debug_options(),
       ReducePrecisionInsertion::PassTiming::BEFORE_OPTIMIZATION);
 
-  pipeline.AddPass<HloGetDimensionSizeRewriter>();
   pipeline.AddPass<MapInliner>();
 
   // TODO(b/65775800): Fix wrong output bug in Call and remove the CallInliner
@@ -270,6 +269,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
         /*rewrite_training_op=*/true,
         /*rewrite_inference_op=*/true,
         /*rewrite_grad_op=*/true);
+    pipeline.AddPass<HloGetDimensionSizeRewriter>();
     AlgebraicSimplifierOptions options(
         [](const Shape&, const Shape&) { return false; });
     options.set_enable_dot_strength_reduction(false);
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 637b861f70..e934cbda17 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -143,7 +143,6 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
                          Compiler* compiler) {
   {
     HloPassPipeline pipeline("optimization");
-    pipeline.AddPass<HloGetDimensionSizeRewriter>();
     pipeline.AddInvariantChecker<HloVerifier>(/*layout_sensitive=*/false,
                                               /*allow_mixed_precision=*/false);
     pipeline.AddPass<GpuHloSupportChecker>();
@@ -175,6 +174,8 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
           /*rewrite_inference_op=*/true,
           /*rewrite_grad_op=*/true);
 
+      pipeline.AddPass<HloGetDimensionSizeRewriter>();
+
       // BatchNormExpander can create zero-sized ops, so zero-sized HLO
       // elimination has to come after that pass.
       pipeline.AddPass<ZeroSizedHloElimination>();
-- 
GitLab


From 375778c0e7c1d4c233c437bbe151cb584077f1f8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 12:21:30 -0800
Subject: [PATCH 1214/1554] internal cleanup

PiperOrigin-RevId: 223840906
---
 tensorflow/contrib/learn/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 238504f6d6..14065fcee5 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -274,6 +274,7 @@ py_test(
     name = "estimator_test",
     size = "medium",
     srcs = ["python/learn/estimators/estimator_test.py"],
+    shard_count = 2,
     srcs_version = "PY2AND3",
     tags = [
         "manual",
-- 
GitLab


From b380b48883a7695825c4201b7eac925deb7898b2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 12:31:32 -0800
Subject: [PATCH 1215/1554] Quick fix for subgraph.h when delegates are
 present.

PiperOrigin-RevId: 223842387
---
 tensorflow/lite/core/subgraph.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h
index d8542f9513..2a7c3a7c32 100644
--- a/tensorflow/lite/core/subgraph.h
+++ b/tensorflow/lite/core/subgraph.h
@@ -219,8 +219,9 @@ class Subgraph {
       TF_LITE_ENSURE(context_, t->delegate != nullptr);
       TF_LITE_ENSURE(context_, t->buffer_handle != kTfLiteNullBufferHandle);
       TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle != nullptr);
-      TF_LITE_ENSURE(context_, t->delegate->CopyFromBufferHandle(
-                                   context_, t->delegate, t->buffer_handle, t));
+      // TODO(b/120420546): we must add a test that exercise this code.
+      TF_LITE_ENSURE_STATUS(t->delegate->CopyFromBufferHandle(
+          context_, t->delegate, t->buffer_handle, t));
       t->data_is_stale = false;
     }
     return kTfLiteOk;
-- 
GitLab


From 64d96c3df527cc44c92653afa2a75692ead7016d Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Mon, 3 Dec 2018 12:43:17 -0800
Subject: [PATCH 1216/1554] Replace deprecated method assertEquals() to
 assertEqual() in all keras py tests.

PiperOrigin-RevId: 223844581
---
 tensorflow/python/keras/backend_test.py       | 14 ++++----
 .../keras/engine/training_utils_test.py       | 36 +++++++++----------
 tensorflow/python/keras/layers/lstm_test.py   |  2 +-
 .../python/keras/layers/unified_lstm_test.py  | 10 +++---
 .../python/keras/layers/wrappers_test.py      |  4 +--
 tensorflow/python/keras/models_test.py        | 12 +++----
 .../keras/optimizer_v2/adadelta_test.py       |  8 ++---
 .../python/keras/optimizer_v2/adagrad_test.py |  4 +--
 .../optimizer_v2/gradient_descent_test.py     | 16 ++++-----
 9 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index fa32b1ecd7..6401e39e53 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1069,13 +1069,13 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
                                                              initial_states,
                                                              **kwargs)
         # check static shape inference
-        self.assertEquals(last_output.get_shape().as_list(),
-                          [num_samples, output_dim])
-        self.assertEquals(outputs.get_shape().as_list(),
-                          [num_samples, timesteps, output_dim])
+        self.assertEqual(last_output.get_shape().as_list(),
+                         [num_samples, output_dim])
+        self.assertEqual(outputs.get_shape().as_list(),
+                         [num_samples, timesteps, output_dim])
         for state in new_states:
-          self.assertEquals(state.get_shape().as_list(),
-                            [num_samples, output_dim])
+          self.assertEqual(state.get_shape().as_list(),
+                           [num_samples, output_dim])
 
         last_output_list[i].append(keras.backend.eval(last_output))
         outputs_list[i].append(keras.backend.eval(outputs))
@@ -1173,7 +1173,7 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
         self.assertEqual(outputs.get_shape().as_list(),
                          [num_samples, timesteps, output_dim])
         # for state in new_states:
-        #   self.assertEquals(state.get_shape().as_list(),
+        #   self.assertEqual(state.get_shape().as_list(),
         #                     [num_samples, output_dim])
         self.assertEqual(new_states[0].get_shape().as_list(),
                          [num_samples, output_dim])
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index d6a92dec7c..4087af5c00 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -37,7 +37,7 @@ class TrainingUtilTest(test.TestCase):
     a = np.ones([10, 10])
     iterator, steps_per_epoch = training_utils.convert_to_iterator(
         x=a, batch_size=batch_size)
-    self.assertEquals(steps_per_epoch, 5)
+    self.assertEqual(steps_per_epoch, 5)
 
     expected_batch = a[:batch_size, :]
     actual_batch, = iterator.get_next()
@@ -49,7 +49,7 @@ class TrainingUtilTest(test.TestCase):
     a = ops.convert_to_tensor(np.ones([10, 10]))
     iterator, steps_per_epoch = training_utils.convert_to_iterator(
         x=a, batch_size=batch_size)
-    self.assertEquals(steps_per_epoch, 5)
+    self.assertEqual(steps_per_epoch, 5)
 
     expected_batch = a[:batch_size, :]
     actual_batch, = iterator.get_next()
@@ -62,7 +62,7 @@ class TrainingUtilTest(test.TestCase):
     b = np.ones([10, 10])
     iterator, steps_per_epoch = training_utils.convert_to_iterator(
         x=a, y=b, batch_size=batch_size)
-    self.assertEquals(steps_per_epoch, 5)
+    self.assertEqual(steps_per_epoch, 5)
 
     expected_x = a[:batch_size, :]
     expected_y = b[:batch_size, :]
@@ -78,7 +78,7 @@ class TrainingUtilTest(test.TestCase):
     sw = ops.convert_to_tensor(np.ones([10]))
     iterator, steps_per_epoch = training_utils.convert_to_iterator(
         x=a, y=b, sample_weights=sw, batch_size=batch_size)
-    self.assertEquals(steps_per_epoch, 5)
+    self.assertEqual(steps_per_epoch, 5)
 
     expected_x = a[:batch_size, :]
     expected_y = b[:batch_size, :]
@@ -94,7 +94,7 @@ class TrainingUtilTest(test.TestCase):
     x = {'1': np.ones([10, 100]), '2': [np.zeros([10, 10]), np.ones([10, 20])]}
     iterator, steps_per_epoch = training_utils.convert_to_iterator(
         x=x, batch_size=batch_size)
-    self.assertEquals(steps_per_epoch, 5)
+    self.assertEqual(steps_per_epoch, 5)
 
     expected_x1 = x['1'][:batch_size, :]
     expected_x2_0 = x['2'][0][:batch_size, :]
@@ -115,7 +115,7 @@ class TrainingUtilTest(test.TestCase):
     a = np.ones([10, 10])
     iterator, steps_per_epoch = training_utils.convert_to_iterator(
         x=a, batch_size=batch_size, epochs=2)
-    self.assertEquals(steps_per_epoch, 5)
+    self.assertEqual(steps_per_epoch, 5)
 
     expected_batch = a[:batch_size, :]
     # loop through one whole epoch
@@ -133,20 +133,20 @@ class TrainingUtilTest(test.TestCase):
   def test_nested_all(self):
     nested_data = {'a': True, 'b': [True, True, (False, True)]}
     all_true = training_utils._nested_all(nested_data, lambda x: x)
-    self.assertEquals(all_true, False)
+    self.assertEqual(all_true, False)
 
     nested_data = {'a': True, 'b': [True, True, (True, True)]}
     all_true = training_utils._nested_all(nested_data, lambda x: x)
-    self.assertEquals(all_true, True)
+    self.assertEqual(all_true, True)
 
   def test_nested_any(self):
     nested_data = [False, {'a': False, 'b': (False, True)}]
     any_true = training_utils._nested_any(nested_data, lambda x: x)
-    self.assertEquals(any_true, True)
+    self.assertEqual(any_true, True)
 
     nested_data = [False, {'a': False, 'b': (False, False)}]
     any_true = training_utils._nested_any(nested_data, lambda x: x)
-    self.assertEquals(any_true, False)
+    self.assertEqual(any_true, False)
 
   def test_check_array_lengths(self):
     training_utils.check_array_lengths(None, None, None)
@@ -166,28 +166,28 @@ class ModelInputsTest(test.TestCase):
   def test_single_thing(self):
     a = np.ones(10)
     model_inputs = training_utils.ModelInputs(a)
-    self.assertEquals(['input_1'], model_inputs.get_input_names())
+    self.assertEqual(['input_1'], model_inputs.get_input_names())
     vals = model_inputs.get_symbolic_inputs()
     self.assertTrue(tensor_util.is_tensor(vals))
     vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
-    self.assertEquals(1, len(vals))
+    self.assertEqual(1, len(vals))
     self.assertTrue(tensor_util.is_tensor(vals[0]))
 
   def test_single_thing_eager(self):
     with context.eager_mode():
       a = np.ones(10)
       model_inputs = training_utils.ModelInputs(a)
-      self.assertEquals(['input_1'], model_inputs.get_input_names())
+      self.assertEqual(['input_1'], model_inputs.get_input_names())
       val = model_inputs.get_symbolic_inputs()
       self.assertTrue(tf_utils.is_symbolic_tensor(val))
       vals = model_inputs.get_symbolic_inputs(return_single_as_list=True)
-      self.assertEquals(1, len(vals))
+      self.assertEqual(1, len(vals))
       self.assertTrue(tf_utils.is_symbolic_tensor(vals[0]))
 
   def test_list(self):
     a = [np.ones(10), np.ones(20)]
     model_inputs = training_utils.ModelInputs(a)
-    self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names())
+    self.assertEqual(['input_1', 'input_2'], model_inputs.get_input_names())
     vals = model_inputs.get_symbolic_inputs()
     self.assertTrue(tensor_util.is_tensor(vals[0]))
     self.assertTrue(tensor_util.is_tensor(vals[1]))
@@ -196,7 +196,7 @@ class ModelInputsTest(test.TestCase):
     with context.eager_mode():
       a = [np.ones(10), np.ones(20)]
       model_inputs = training_utils.ModelInputs(a)
-      self.assertEquals(['input_1', 'input_2'], model_inputs.get_input_names())
+      self.assertEqual(['input_1', 'input_2'], model_inputs.get_input_names())
       vals = model_inputs.get_symbolic_inputs()
       self.assertTrue(tf_utils.is_symbolic_tensor(vals[0]))
       self.assertTrue(tf_utils.is_symbolic_tensor(vals[1]))
@@ -204,7 +204,7 @@ class ModelInputsTest(test.TestCase):
   def test_dict(self):
     a = {'b': np.ones(10), 'a': np.ones(20)}
     model_inputs = training_utils.ModelInputs(a)
-    self.assertEquals(['a', 'b'], model_inputs.get_input_names())
+    self.assertEqual(['a', 'b'], model_inputs.get_input_names())
     vals = model_inputs.get_symbolic_inputs()
     self.assertTrue(tensor_util.is_tensor(vals['a']))
     self.assertTrue(tensor_util.is_tensor(vals['b']))
@@ -213,7 +213,7 @@ class ModelInputsTest(test.TestCase):
     with context.eager_mode():
       a = {'b': np.ones(10), 'a': np.ones(20)}
       model_inputs = training_utils.ModelInputs(a)
-      self.assertEquals(['a', 'b'], model_inputs.get_input_names())
+      self.assertEqual(['a', 'b'], model_inputs.get_input_names())
       vals = model_inputs.get_symbolic_inputs()
       self.assertTrue(tf_utils.is_symbolic_tensor(vals['a']))
       self.assertTrue(tf_utils.is_symbolic_tensor(vals['b']))
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index 118c4f86c2..3f89cc398e 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -57,7 +57,7 @@ class LSTMLayerTest(test.TestCase, parameterized.TestCase):
     layer = keras.layers.LSTM(units, return_sequences=True)
     model.add(layer)
     outputs = model.layers[-1].output
-    self.assertEquals(outputs.get_shape().as_list(), [None, timesteps, units])
+    self.assertEqual(outputs.get_shape().as_list(), [None, timesteps, units])
 
   def test_dynamic_behavior_LSTM(self):
     num_samples = 2
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 00dd3f0c04..d229d14312 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -91,9 +91,9 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
             predict: y_train
         })
         if test.is_gpu_available():
-          self.assertEquals(runtime_value, b'cudnn')
+          self.assertEqual(runtime_value, b'cudnn')
         else:
-          self.assertEquals(runtime_value, b'cpu')
+          self.assertEqual(runtime_value, b'cpu')
         # Make sure the loss is updated for every epoch
         # (layer weights properly updated).
         self.assertNotEqual(existing_loss, loss_value)
@@ -149,9 +149,9 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
             predict: y_train
         })
         if test.is_gpu_available():
-          self.assertEquals(runtime_value, b'cudnn')
+          self.assertEqual(runtime_value, b'cudnn')
         else:
-          self.assertEquals(runtime_value, b'cpu')
+          self.assertEqual(runtime_value, b'cpu')
         # Make sure the loss is updated for every epoch
         # (layer weights properly updated).
         self.assertNotEqual(existing_loss, loss_value)
@@ -212,7 +212,7 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
     layer = UnifiedLSTM(units, return_sequences=True)
     model.add(layer)
     outputs = model.layers[-1].output
-    self.assertEquals(outputs.get_shape().as_list(), [None, timesteps, units])
+    self.assertEqual(outputs.get_shape().as_list(), [None, timesteps, units])
 
   @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_dynamic_behavior_LSTM(self):
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 427314faf1..b9196416dd 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -192,8 +192,8 @@ class TimeDistributedTest(test.TestCase):
     x = keras.layers.Input(shape=(3, 2))
     layer = keras.layers.TimeDistributed(keras.layers.BatchNormalization())
     _ = layer(x)
-    self.assertEquals(len(layer.updates), 2)
-    self.assertEquals(len(layer.trainable_weights), 2)
+    self.assertEqual(len(layer.updates), 2)
+    self.assertEqual(len(layer.trainable_weights), 2)
     layer.trainable = False
     assert not layer.updates
     assert not layer.trainable_weights
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 8af3cc05f5..b0872ae3ab 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -84,21 +84,21 @@ class TestModelCloning(test.TestCase):
       # With placeholder creation
       new_model = keras.models.clone_model(model)
       # update ops from batch norm needs to be included
-      self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2)
+      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch(val_a, val_out)
 
       # On top of new tensor
       input_a = keras.Input(shape=(4,))
       new_model = keras.models.clone_model(model, input_tensors=input_a)
-      self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2)
+      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch(val_a, val_out)
 
       # On top of new, non-Keras tensor
       input_a = keras.backend.variable(val_a)
       new_model = keras.models.clone_model(model, input_tensors=input_a)
-      self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2)
+      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch(None, val_out)
 
@@ -168,7 +168,7 @@ class TestModelCloning(test.TestCase):
     with self.cached_session():
       # With placeholder creation
       new_model = keras.models.clone_model(model)
-      self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2)
+      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch([val_a, val_b], val_out)
 
@@ -177,7 +177,7 @@ class TestModelCloning(test.TestCase):
       input_b = keras.Input(shape=(4,), name='b')
       new_model = keras.models.clone_model(
           model, input_tensors=[input_a, input_b])
-      self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2)
+      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch([val_a, val_b], val_out)
 
@@ -186,7 +186,7 @@ class TestModelCloning(test.TestCase):
       input_b = keras.backend.variable(val_b)
       new_model = keras.models.clone_model(
           model, input_tensors=[input_a, input_b])
-      self.assertEquals(len(new_model.get_updates_for(new_model.inputs)), 2)
+      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch(None, val_out)
 
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index 6409f3ead4..0fb67d0cd1 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -75,16 +75,16 @@ class AdadeltaOptimizerTest(test.TestCase):
             slot = [None] * 2
             slot_update = [None] * 2
             slot[0] = adadelta_opt.get_slot(var0, "accum_grad")
-            self.assertEquals(slot[0].get_shape(), var0.get_shape())
+            self.assertEqual(slot[0].get_shape(), var0.get_shape())
 
             slot_update[0] = adadelta_opt.get_slot(var0, "accum_var")
-            self.assertEquals(slot_update[0].get_shape(), var0.get_shape())
+            self.assertEqual(slot_update[0].get_shape(), var0.get_shape())
 
             slot[1] = adadelta_opt.get_slot(var1, "accum_grad")
-            self.assertEquals(slot[1].get_shape(), var1.get_shape())
+            self.assertEqual(slot[1].get_shape(), var1.get_shape())
 
             slot_update[1] = adadelta_opt.get_slot(var1, "accum_var")
-            self.assertEquals(slot_update[1].get_shape(), var1.get_shape())
+            self.assertEqual(slot_update[1].get_shape(), var1.get_shape())
 
           # Fetch params to validate initial values
           self.assertAllClose(var0_init, self.evaluate(var0))
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index 8004907bf5..b2c290178f 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -372,9 +372,9 @@ class AdagradOptimizerTest(test.TestCase):
         ada_update2 = ada_opt.apply_gradients(
             zip([grads0, grads1], [var0, var1]))
         slot0 = ada_opt.get_slot(var0, "accumulator")
-        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        self.assertEqual(slot0.get_shape(), var0.get_shape())
         slot1 = ada_opt.get_slot(var1, "accumulator")
-        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        self.assertEqual(slot1.get_shape(), var1.get_shape())
         variables.global_variables_initializer().run()
 
         # Fetch params to validate initial values.
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index dae2a47735..b6dc87e2e1 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -291,9 +291,9 @@ class MomentumOptimizerTest(test.TestCase):
 
         # Check we have slots
         slot0 = mom_opt.get_slot(var0, "momentum")
-        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        self.assertEqual(slot0.get_shape(), var0.get_shape())
         slot1 = mom_opt.get_slot(var1, "momentum")
-        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        self.assertEqual(slot1.get_shape(), var1.get_shape())
 
         # Step 1: the momentum accumulators where 0. So we should see a normal
         # update: v -= grad * learning_rate
@@ -474,9 +474,9 @@ class MomentumOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
         # Check we have slots
         slot0 = mom_opt.get_slot(var0, "momentum")
-        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        self.assertEqual(slot0.get_shape(), var0.get_shape())
         slot1 = mom_opt.get_slot(var1, "momentum")
-        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        self.assertEqual(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
@@ -536,9 +536,9 @@ class MomentumOptimizerTest(test.TestCase):
 
         # Check we have slots
         slot0 = mom_opt.get_slot(var0, "momentum")
-        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        self.assertEqual(slot0.get_shape(), var0.get_shape())
         slot1 = mom_opt.get_slot(var1, "momentum")
-        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        self.assertEqual(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
         self.assertAllClose([0, 0], self.evaluate(var0)[0])
@@ -609,9 +609,9 @@ class MomentumOptimizerTest(test.TestCase):
         variables.global_variables_initializer().run()
 
         slot0 = mom_opt.get_slot(var0, "momentum")
-        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        self.assertEqual(slot0.get_shape(), var0.get_shape())
         slot1 = mom_opt.get_slot(var1, "momentum")
-        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        self.assertEqual(slot1.get_shape(), var1.get_shape())
 
         # Fetch params to validate initial values
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
-- 
GitLab


From c9e111804974f21dbe297855ab217e964526baa2 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Mon, 3 Dec 2018 12:48:30 -0800
Subject: [PATCH 1217/1554] Add search_hints option.

PiperOrigin-RevId: 223845447
---
 tensorflow/tools/docs/generate2.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py
index 3dfd0dca33..fba909d26d 100644
--- a/tensorflow/tools/docs/generate2.py
+++ b/tensorflow/tools/docs/generate2.py
@@ -47,20 +47,24 @@ flags.DEFINE_string(
     "output_dir", "/tmp/out",
     "A directory, where the docs will be output to.")
 
+flags.DEFINE_bool("search_hints", True,
+                  "Include meta-data search hints at the top of each file.")
 
-def build_docs(output_dir, code_url_prefix):
+
+def build_docs(output_dir, code_url_prefix, search_hints=True):
   """Build api docs for tensorflow v2.
 
   Args:
     output_dir: A string path, where to put the files.
     code_url_prefix: prefix for "Defined in" links.
+    search_hints: Bool. Include meta-data search hints at the top of each file.
   """
   base_dir = path.dirname(tf.__file__)
   doc_generator = generate_lib.DocGenerator(
       root_title="TensorFlow 2.0 Preview",
       py_modules=[("tf", tf)],
       base_dir=base_dir,
-      search_hints=True,
+      search_hints=search_hints,
       code_url_prefix=code_url_prefix,
       site_path="api_docs/")
 
@@ -70,7 +74,8 @@ def build_docs(output_dir, code_url_prefix):
 def main(argv):
   del argv
   build_docs(output_dir=FLAGS.output_dir,
-             code_url_prefix=FLAGS.code_url_prefix)
+             code_url_prefix=FLAGS.code_url_prefix,
+             search_hints=FLAGS.search_hints)
 
 
 if __name__ == "__main__":
-- 
GitLab


From a63bbdbe3e2afdadc0229e4a0107831bbd33e1c0 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 3 Dec 2018 13:03:24 -0800
Subject: [PATCH 1218/1554] [tf.data] Add a `Structure._batch()` method for
 creating structures of batches.

This change switches `Dataset.batch()`, `map_and_batch()`, and `sliding_window_batch()` to use the new helper. As a cascading side effect, all subclasses of `MapDataset` and `FlatMapDataset` are also switched to use `Structure` for representing their structure.

PiperOrigin-RevId: 223847841
---
 tensorflow/contrib/data/python/ops/BUILD      |   3 +-
 tensorflow/contrib/data/python/ops/sliding.py |  21 +--
 tensorflow/python/data/experimental/ops/BUILD |   1 +
 .../python/data/experimental/ops/batching.py  |  43 +++---
 .../data/experimental/ops/prefetching_ops.py  |  16 +--
 tensorflow/python/data/ops/dataset_ops.py     | 132 +++++++++---------
 tensorflow/python/data/ops/optional_ops.py    |   4 +
 tensorflow/python/data/ops/readers.py         |   7 +-
 tensorflow/python/data/util/structure.py      |  27 ++++
 tensorflow/python/data/util/structure_test.py |  27 ++++
 tensorflow/python/distribute/input_ops.py     |   2 +-
 11 files changed, 171 insertions(+), 112 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 34dc2379d0..0fb406f116 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -188,8 +188,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:function",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py
index a3ab6b8d72..9ebdca317f 100644
--- a/tensorflow/contrib/data/python/ops/sliding.py
+++ b/tensorflow/contrib/data/python/ops/sliding.py
@@ -18,10 +18,9 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 from tensorflow.python.util import deprecation
 
@@ -40,29 +39,31 @@ class _SlideDataset(dataset_ops.UnaryDataset):
     self._window_shift = ops.convert_to_tensor(
         window_shift, dtype=dtypes.int64, name="window_shift")
 
+    # pylint: disable=protected-access
+    input_structure = structure.Structure._from_legacy_structure(
+        input_dataset.output_types, input_dataset.output_shapes,
+        input_dataset.output_classes)
+    self._output_structure = input_structure._batch(None)
+
   def _as_variant_tensor(self):
     return ged_ops.experimental_sliding_window_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         window_size=self._window_size,
         window_shift=self._window_shift,
         window_stride=self._window_stride,
-        **dataset_ops.flat_structure(self))
+        **dataset_ops.flat_structure(structure=self._output_structure))
 
   @property
   def output_classes(self):
-    return self._input_dataset.output_classes
+    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    input_shapes = self._input_dataset.output_shapes
-    return nest.pack_sequence_as(input_shapes, [
-        tensor_shape.vector(None).concatenate(s)
-        for s in nest.flatten(self._input_dataset.output_shapes)
-    ])
+    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._input_dataset.output_types
+    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
 
 @deprecation.deprecated_args(
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index c4a4781aad..a0ab302335 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -135,6 +135,7 @@ py_library(
         "//tensorflow/python/data/util:convert",
         "//tensorflow/python/data/util:nest",
         "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python/data/util:structure",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 1422966d27..58eab0f794 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
@@ -569,13 +570,16 @@ class _RestructuredDataset(dataset_ops.UnaryDataset):
     return self._output_shapes
 
 
-class _MapAndBatchDataset(dataset_ops.MapDataset):
+class _MapAndBatchDataset(dataset_ops.UnaryDataset):
   """A `Dataset` that maps a function over a batch of elements."""
 
   def __init__(self, input_dataset, map_func, batch_size, num_parallel_calls,
                drop_remainder):
     """See `Dataset.map()` for details."""
-    super(_MapAndBatchDataset, self).__init__(input_dataset, map_func)
+    super(_MapAndBatchDataset, self).__init__(input_dataset)
+    self._input_dataset = input_dataset
+    self._map_func = dataset_ops.StructuredFunctionWrapper(
+        map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset)
     self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
     self._num_parallel_calls_t = ops.convert_to_tensor(
@@ -583,36 +587,37 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
     self._drop_remainder_t = ops.convert_to_tensor(
         drop_remainder, dtype=dtypes.bool, name="drop_remainder")
 
-    self._batch_size = batch_size
-    self._drop_remainder = drop_remainder
+    constant_drop_remainder = tensor_util.constant_value(self._drop_remainder_t)
+    if constant_drop_remainder:
+      # NOTE(mrry): `constant_drop_remainder` may be `None` (unknown statically)
+      # or `False` (explicitly retaining the remainder).
+      self._output_structure = self._map_func.output_structure._batch(  # pylint: disable=protected-access
+          tensor_util.constant_value(self._batch_size_t))
+    else:
+      self._output_structure = self._map_func.output_structure._batch(None)  # pylint: disable=protected-access
 
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
-    input_resource = self._input_dataset._as_variant_tensor()
     return ged_ops.experimental_map_and_batch_dataset(
-        input_resource,
-        self._map_func.captured_inputs,
-        f=self._map_func,
+        self._input_dataset._as_variant_tensor(),
+        self._map_func.function.captured_inputs,
+        f=self._map_func.function,
         batch_size=self._batch_size_t,
         num_parallel_calls=self._num_parallel_calls_t,
         drop_remainder=self._drop_remainder_t,
-        **dataset_ops.flat_structure(self))
-    # pylint: enable=protected-access
+        **dataset_ops.flat_structure(structure=self._output_structure))
+
+  @property
+  def output_classes(self):
+    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    dim = self._batch_size if self._drop_remainder else None
-    return nest.pack_sequence_as(self._output_shapes, [
-        tensor_shape.vector(dim).concatenate(s)
-        for s in nest.flatten(self._output_shapes)
-    ])
+    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._output_types
-
-  def _transformation_name(self):
-    return "tf.data.experimental.map_and_batch()"
+    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
 
 @tf_export("data.experimental.map_and_batch")
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 5560d1f750..2a5a7205e8 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -242,36 +242,32 @@ class _MapOnGpuDataset(dataset_ops.UnaryDataset):
     self._input_dataset = input_dataset
     self._use_inter_op_parallelism = use_inter_op_parallelism
 
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+    self._map_func = dataset_ops.StructuredFunctionWrapper(
         map_func,
         self._transformation_name(),
         dataset=input_dataset,
         defun_kwargs={"experimental_ints_on_device": True})
-    self._output_classes = wrapped_func.output_classes
-    self._output_shapes = wrapped_func.output_shapes
-    self._output_types = wrapped_func.output_types
-    self._map_func = wrapped_func.function
 
   def _as_variant_tensor(self):
     input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
     return ged_ops.experimental_map_dataset(
         input_t,
-        self._map_func.captured_inputs,
-        f=self._map_func,
+        self._map_func.function.captured_inputs,
+        f=self._map_func.function,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
         **dataset_ops.flat_structure(self))
 
   @property
   def output_classes(self):
-    return self._output_classes
+    return self._map_func.output_classes
 
   @property
   def output_shapes(self):
-    return self._output_shapes
+    return self._map_func.output_shapes
 
   @property
   def output_types(self):
-    return self._output_types
+    return self._map_func.output_types
 
   def _transformation_name(self):
     return "map_on_gpu()"
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 7070a84b70..0843790768 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -297,7 +297,8 @@ class DatasetV2(object):
     `tf.constant` operations. For large datasets (> 1 GB), this can waste
     memory and run into byte limits of graph serialization. If `tensors`
     contains one or more large NumPy arrays, consider the alternative described
-    in [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
+    in [this
+    guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
 
     Args:
       tensors: A nested structure of tensors.
@@ -316,7 +317,8 @@ class DatasetV2(object):
     `tf.constant` operations. For large datasets (> 1 GB), this can waste
     memory and run into byte limits of graph serialization. If `tensors`
     contains one or more large NumPy arrays, consider the alternative described
-    in [this guide](https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
+    in [this guide](
+    https://tensorflow.org/guide/datasets#consuming_numpy_arrays).
 
     Args:
       tensors: A nested structure of tensors, each having the same size in the
@@ -1888,6 +1890,9 @@ class DatasetStructure(structure_lib.Structure):
   def _to_legacy_output_classes(self):
     return self
 
+  def _batch(self, batch_size):
+    raise NotImplementedError("Batching for `tf.data.Dataset` objects.")
+
 
 # pylint: disable=protected-access
 structure_lib.Structure._register_custom_converter(DatasetV2,
@@ -2028,9 +2033,11 @@ class StructuredFunctionWrapper(object):
     return self._function
 
 
-def flat_structure(dataset):
+def flat_structure(dataset=None, structure=None):
   """Helper for setting `output_shapes` and `output_types` attrs of Dataset ops.
 
+  Either `dataset` or `structure` must be passed to this function.
+
   Most Dataset op constructors expect `output_shapes` and `output_types`
   arguments that represent the flattened structure of an element. This helper
   function generates these attrs as a keyword argument dictionary, allowing
@@ -2038,15 +2045,17 @@ def flat_structure(dataset):
   `**flat_structure(self)` to the op constructor.
 
   Args:
-    dataset: A `tf.data.Dataset`.
+    dataset: (Optional.) A `tf.data.Dataset`.
+    structure: (Optional.) A `Structure`.
 
   Returns:
     A dictionary of keyword arguments that can be passed to many Dataset op
     constructors.
   """
   # pylint: disable=protected-access
-  structure = structure_lib.Structure._from_legacy_structure(
-      dataset.output_types, dataset.output_shapes, dataset.output_classes)
+  if structure is None:
+    structure = structure_lib.Structure._from_legacy_structure(
+        dataset.output_types, dataset.output_shapes, dataset.output_classes)
   return {
       "output_shapes": structure._flat_shapes,
       "output_types": structure._flat_types,
@@ -2421,37 +2430,37 @@ class BatchDataset(UnaryDataset):
     self._drop_remainder = ops.convert_to_tensor(
         drop_remainder, dtype=dtypes.bool, name="drop_remainder")
 
-  def _as_variant_tensor(self):
-    # TODO(jsimsa): Switch to using v2 only any time after 6/30/2018.
-    if smart_cond.smart_constant_value(self._drop_remainder) is False:
-      return gen_dataset_ops.batch_dataset(
-          self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-          batch_size=self._batch_size,
-          **flat_structure(self))
+    # pylint: disable=protected-access
+    input_structure = structure_lib.Structure._from_legacy_structure(
+        input_dataset.output_types, input_dataset.output_shapes,
+        input_dataset.output_classes)
+    constant_drop_remainder = tensor_util.constant_value(self._drop_remainder)
+    if constant_drop_remainder:
+      # NOTE(mrry): `constant_drop_remainder` may be `None` (unknown statically)
+      # or `False` (explicitly retaining the remainder).
+      self._output_structure = input_structure._batch(
+          tensor_util.constant_value(self._batch_size))
     else:
-      return gen_dataset_ops.batch_dataset_v2(
-          self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-          batch_size=self._batch_size,
-          drop_remainder=self._drop_remainder,
-          **flat_structure(self))
+      self._output_structure = input_structure._batch(None)
+
+  def _as_variant_tensor(self):
+    return gen_dataset_ops.batch_dataset_v2(
+        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        batch_size=self._batch_size,
+        drop_remainder=self._drop_remainder,
+        **flat_structure(structure=self._output_structure))
 
   @property
   def output_classes(self):
-    return self._input_dataset.output_classes
+    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    input_shapes = self._input_dataset.output_shapes
-    return nest.pack_sequence_as(input_shapes, [
-        tensor_shape.vector(
-            tensor_util.constant_value(self._batch_size) if smart_cond.
-            smart_constant_value(self._drop_remainder) else None).concatenate(s)
-        for s in nest.flatten(self._input_dataset.output_shapes)
-    ])
+    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._input_dataset.output_types
+    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
 
 def _is_padded_shape_compatible_with(padded_shape, input_component_shape):
@@ -2677,34 +2686,29 @@ class MapDataset(UnaryDataset):
     super(MapDataset, self).__init__(input_dataset)
     self._input_dataset = input_dataset
     self._use_inter_op_parallelism = use_inter_op_parallelism
-
-    wrapped_func = StructuredFunctionWrapper(
+    self._map_func = StructuredFunctionWrapper(
         map_func, self._transformation_name(), dataset=input_dataset)
-    self._output_classes = wrapped_func.output_classes
-    self._output_shapes = wrapped_func.output_shapes
-    self._output_types = wrapped_func.output_types
-    self._map_func = wrapped_func.function
 
   def _as_variant_tensor(self):
     input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
     return gen_dataset_ops.map_dataset(
         input_t,
-        self._map_func.captured_inputs,
-        f=self._map_func,
+        self._map_func.function.captured_inputs,
+        f=self._map_func.function,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
-        **flat_structure(self))
+        **flat_structure(structure=self._map_func.output_structure))
 
   @property
   def output_classes(self):
-    return self._output_classes
+    return self._map_func.output_classes
 
   @property
   def output_shapes(self):
-    return self._output_shapes
+    return self._map_func.output_shapes
 
   @property
   def output_types(self):
-    return self._output_types
+    return self._map_func.output_types
 
   def _transformation_name(self):
     return "Dataset.map()"
@@ -2726,16 +2730,15 @@ class ParallelMapDataset(MapDataset):
         num_parallel_calls, dtype=dtypes.int32, name="num_parallel_calls")
 
   def _as_variant_tensor(self):
-    input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
     # pylint: disable=protected-access
+    input_t = self._input_dataset._as_variant_tensor()
     return gen_dataset_ops.parallel_map_dataset(
         input_t,
-        self._map_func.captured_inputs,
-        f=self._map_func,
+        self._map_func.function.captured_inputs,
+        f=self._map_func.function,
         num_parallel_calls=self._num_parallel_calls,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
-        **flat_structure(self))
-    # pylint: enable=protected-access
+        **flat_structure(structure=self._map_func.output_structure))
 
 
 class FlatMapDataset(UnaryDataset):
@@ -2746,35 +2749,30 @@ class FlatMapDataset(UnaryDataset):
     super(FlatMapDataset, self).__init__(input_dataset)
     self._input_dataset = input_dataset
 
-    wrapped_func = StructuredFunctionWrapper(
+    self._map_func = StructuredFunctionWrapper(
         map_func, self._transformation_name(), dataset=input_dataset)
-    if not isinstance(wrapped_func.output_structure, DatasetStructure):
+    if not isinstance(self._map_func.output_structure, DatasetStructure):
       raise TypeError("`map_func` must return a `Dataset` object.")
-    # pylint: disable=protected-access
-    element_structure = wrapped_func.output_structure._element_structure
-    self._output_classes = element_structure._to_legacy_output_classes()
-    self._output_types = element_structure._to_legacy_output_types()
-    self._output_shapes = element_structure._to_legacy_output_shapes()
-    self._map_func = wrapped_func.function
+    self._output_structure = self._map_func.output_structure._element_structure  # pylint: disable=protected-access
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.flat_map_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        self._map_func.captured_inputs,
-        f=self._map_func,
-        **flat_structure(self))
+        self._map_func.function.captured_inputs,
+        f=self._map_func.function,
+        **flat_structure(structure=self._output_structure))
 
   @property
   def output_classes(self):
-    return self._output_classes
+    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
   @property
   def output_shapes(self):
-    return self._output_shapes
+    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
   @property
   def output_types(self):
-    return self._output_types
+    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
   def _transformation_name(self):
     return "Dataset.flat_map()"
@@ -2793,13 +2791,14 @@ class InterleaveDataset(FlatMapDataset):
         block_length, dtype=dtypes.int64, name="block_length")
 
   def _as_variant_tensor(self):
+    # pylint: disable=protected-access
     return gen_dataset_ops.interleave_dataset(
-        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        self._map_func.captured_inputs,  # pylint: disable=protected-access
+        self._input_dataset._as_variant_tensor(),
+        self._map_func.function.captured_inputs,
         self._cycle_length,
         self._block_length,
-        f=self._map_func,  # pylint: disable=protected-access
-        **flat_structure(self))
+        f=self._map_func.function,
+        **flat_structure(structure=self._output_structure))
 
   def _transformation_name(self):
     return "Dataset.interleave()"
@@ -2822,14 +2821,15 @@ class ParallelInterleaveDataset(FlatMapDataset):
         num_parallel_calls, dtype=dtypes.int64, name="num_parallel_calls")
 
   def _as_variant_tensor(self):
+    # pylint: disable=protected-access
     return gen_dataset_ops.parallel_interleave_dataset_v2(
-        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        self._map_func.captured_inputs,  # pylint: disable=protected-access
+        self._input_dataset._as_variant_tensor(),
+        self._map_func.function.captured_inputs,
         self._cycle_length,
         self._block_length,
         self._num_parallel_calls,
-        f=self._map_func,  # pylint: disable=protected-access
-        **flat_structure(self))
+        f=self._map_func.function,
+        **flat_structure(structure=self._output_structure))
 
   def _transformation_name(self):
     return "Dataset.interleave()"
diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py
index 4113b7ed31..15ec755c67 100644
--- a/tensorflow/python/data/ops/optional_ops.py
+++ b/tensorflow/python/data/ops/optional_ops.py
@@ -191,6 +191,10 @@ class OptionalStructure(structure.Structure):
   def _to_legacy_output_classes(self):
     return self
 
+  def _batch(self, batch_size):
+    raise NotImplementedError(
+        "Batching for `tf.data.experimental.Optional` objects.")
+
 
 # pylint: disable=protected-access
 structure.Structure._register_custom_converter(Optional,
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index 29f5f20282..70a3b1b1cb 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -161,15 +161,14 @@ class ParallelInterleaveDataset(dataset_ops.InterleaveDataset):
     # pylint: disable=protected-access
     return ged_ops.experimental_parallel_interleave_dataset(
         self._input_dataset._as_variant_tensor(),
-        self._map_func.captured_inputs,
+        self._map_func.function.captured_inputs,
         self._cycle_length,
         self._block_length,
         self._sloppy,
         self._buffer_output_elements,
         self._prefetch_input_elements,
-        f=self._map_func,
-        **dataset_ops.flat_structure(self))
-    # pylint: enable=protected-access
+        f=self._map_func.function,
+        **dataset_ops.flat_structure(structure=self._output_structure))
 
   def _transformation_name(self):
     return "tf.data.experimental.parallel_interleave()"
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 01c7c87947..874e8abc6d 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -144,6 +144,19 @@ class Structure(object):
     """
     return self._from_tensor_list(flat_value)
 
+  @abc.abstractmethod
+  def _batch(self, batch_size):
+    """Returns a structure representing a batch of objects with this structure.
+
+    Args:
+      batch_size: An `int` representing the number of elements in a batch,
+        or `None` if the batch size may vary.
+
+    Returns:
+      A `Structure` representing a batch of objects with this structure.
+    """
+    raise NotImplementedError("Structure._batch()")
+
   @staticmethod
   def from_value(value):
     """Returns a `Structure` that represents the given `value`.
@@ -355,6 +368,10 @@ class NestedStructure(Structure):
     return nest.map_structure(
         lambda s: s._to_legacy_output_classes(), self._nested_structure)
 
+  def _batch(self, batch_size):
+    return NestedStructure(nest.map_structure(
+        lambda s: s._batch(batch_size), self._nested_structure))
+
 
 class TensorStructure(Structure):
   """Represents structural information about a `tf.Tensor`."""
@@ -413,6 +430,11 @@ class TensorStructure(Structure):
   def _to_legacy_output_classes(self):
     return ops.Tensor
 
+  def _batch(self, batch_size):
+    return TensorStructure(
+        self._dtype,
+        tensor_shape.TensorShape([batch_size]).concatenate(self._shape))
+
 
 class SparseTensorStructure(Structure):
   """Represents structural information about a `tf.SparseTensor`."""
@@ -470,3 +492,8 @@ class SparseTensorStructure(Structure):
 
   def _to_legacy_output_classes(self):
     return sparse_tensor_lib.SparseTensor
+
+  def _batch(self, batch_size):
+    return SparseTensorStructure(
+        self._dtype,
+        tensor_shape.TensorShape([batch_size]).concatenate(self._dense_shape))
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index e917747ec6..314a459eff 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -391,6 +391,33 @@ class StructureTest(test.TestCase, parameterized.TestCase):
     self.assertIs(float32_t, actual_float32_t)
     self.assertIs(string_t, actual_string_t)
 
+  @parameterized.named_parameters(
+      ("Tensor", structure.TensorStructure(dtypes.float32, []), 32,
+       structure.TensorStructure(dtypes.float32, [32])),
+      ("TensorUnknown", structure.TensorStructure(dtypes.float32, []), None,
+       structure.TensorStructure(dtypes.float32, [None])),
+      ("SparseTensor", structure.SparseTensorStructure(dtypes.float32, [None]),
+       32, structure.SparseTensorStructure(dtypes.float32, [32, None])),
+      ("SparseTensorUnknown",
+       structure.SparseTensorStructure(dtypes.float32, [4]), None,
+       structure.SparseTensorStructure(dtypes.float32, [None, 4])),
+      ("Nest", structure.NestedStructure({
+          "a": structure.TensorStructure(dtypes.float32, []),
+          "b": (structure.SparseTensorStructure(dtypes.int32, [2, 2]),
+                structure.TensorStructure(dtypes.string, []))}), 128,
+       structure.NestedStructure({
+           "a": structure.TensorStructure(dtypes.float32, [128]),
+           "b": (structure.SparseTensorStructure(dtypes.int32, [128, 2, 2]),
+                 structure.TensorStructure(dtypes.string, [128]))})),
+  )
+  def testBatch(self, element_structure, batch_size,
+                expected_batched_structure):
+    batched_structure = element_structure._batch(batch_size)
+    self.assertTrue(
+        batched_structure.is_compatible_with(expected_batched_structure))
+    self.assertTrue(
+        expected_batched_structure.is_compatible_with(batched_structure))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/distribute/input_ops.py b/tensorflow/python/distribute/input_ops.py
index d7974942a1..2ded209701 100644
--- a/tensorflow/python/distribute/input_ops.py
+++ b/tensorflow/python/distribute/input_ops.py
@@ -84,7 +84,7 @@ def auto_shard_dataset(dataset, num_shards, index):
       elif hasattr(dataset, "_map_func"):
         # TODO(priyag): Make this check more robust by enforcing some common
         # property on all map/flatmap/interleave datasets.
-        map_func_def = dataset._map_func.definition
+        map_func_def = dataset._map_func.function.definition
         for node in map_func_def.node_def:
           if node.op in _READER_DATASET_OPS:
             found_reader_op = True
-- 
GitLab


From 09f575f7e6f291112f30e1c1fa14ca5f4baede3b Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 3 Dec 2018 13:03:31 -0800
Subject: [PATCH 1219/1554] [XLA] Fix constness bug in pattern_matcher.h

PiperOrigin-RevId: 223847869
---
 tensorflow/compiler/xla/service/pattern_matcher.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index 54d7b52a3e..cc06cf6ec7 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -2029,10 +2029,10 @@ XLA_UNOP_PATTERN(Transpose)
   }                                                                         \
   template <typename Lhs, typename Rhs>                                     \
   inline auto NAME##AnyOrder(Lhs&& lhs, Rhs&& rhs)                          \
-      ->decltype(NAME##AnyOrder<HloInstruction>(                            \
+      ->decltype(NAME##AnyOrder<const HloInstruction>(                      \
           nullptr, std::forward<Lhs>(lhs), std::forward<Rhs>(rhs))) {       \
-    return NAME##AnyOrder<HloInstruction>(nullptr, std::forward<Lhs>(lhs),  \
-                                          std::forward<Rhs>(rhs));          \
+    return NAME##AnyOrder<const HloInstruction>(                            \
+        nullptr, std::forward<Lhs>(lhs), std::forward<Rhs>(rhs));           \
   }
 XLA_COMMUTATIVE_BINOP_PATTERN(Add)
 XLA_BINOP_PATTERN(Atan2)
-- 
GitLab


From 99031de5b77df3356fd8cd77e525f86de10ca2e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 13:17:27 -0800
Subject: [PATCH 1220/1554] Internal change

PiperOrigin-RevId: 223850140
---
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index ae7db35b47..4b90b596b2 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -104,6 +104,7 @@ py_test(
     srcs = [
         "estimators_test.py",
     ],
+    shard_count = 3,
     srcs_version = "PY2AND3",
     tags = [
         "no_mac",
-- 
GitLab


From 5c25275ebadab9986db708a29ad08c5d5891083c Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 3 Dec 2018 13:17:42 -0800
Subject: [PATCH 1221/1554] Limit tests exercising list resizing until it has
 been fixed

PiperOrigin-RevId: 223850169
---
 .../operators/data_structures_test.py         |  2 +-
 .../autograph/utils/tensor_list_test.py       | 32 ++++++++-----------
 .../kernel_tests/tensor_array_ops_test.py     | 17 +++++-----
 3 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/tensorflow/python/autograph/operators/data_structures_test.py b/tensorflow/python/autograph/operators/data_structures_test.py
index 9397b9acb8..c5a3a3d1ca 100644
--- a/tensorflow/python/autograph/operators/data_structures_test.py
+++ b/tensorflow/python/autograph/operators/data_structures_test.py
@@ -106,7 +106,7 @@ class ListTest(test.TestCase):
     with self.cached_session() as sess:
       self.assertAllEqual(self.evaluate(t), [[1, 2, 3]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def test_append_tensorarray(self):
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l1 = data_structures.list_append(l, 1)
diff --git a/tensorflow/python/autograph/utils/tensor_list_test.py b/tensorflow/python/autograph/utils/tensor_list_test.py
index 0bd3007249..bbbc3bf691 100644
--- a/tensorflow/python/autograph/utils/tensor_list_test.py
+++ b/tensorflow/python/autograph/utils/tensor_list_test.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.autograph.utils import tensor_list as tl
-from tensorflow.python.client.session import Session
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -35,7 +34,7 @@ class TensorListTest(test.TestCase):
   def _shape(self, shape_tuple):
     return constant(shape_tuple, dtypes.int32)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def test_dynamic_list_append(self):
     l = []
     l = tl.dynamic_list_append(l, 1)
@@ -44,19 +43,16 @@ class TensorListTest(test.TestCase):
     l = list_ops.empty_tensor_list(self._shape(()), dtypes.int32)
     l = tl.dynamic_list_append(l, 1)
     s = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
-    with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(s), [1])
+    self.assertAllEqual(s, [1])
 
     l = tensor_array_ops.TensorArray(dtypes.int32, size=0, dynamic_size=True)
     l = tl.dynamic_list_append(l, 1)
     s = l.stack()
-    with self.cached_session() as sess:
-      self.assertAllEqual(self.evaluate(s), [1])
+    self.assertAllEqual(s, [1])
 
     l = tl.TensorList(self._shape(()), dtypes.int32)
     l = tl.dynamic_list_append(l, 1)
-    with self.cached_session() as sess:
-      self.assertAllEqual(sess.run(l[0]), 1)
+    self.assertAllEqual(l[0], 1)
 
   def test_list_append_python(self):
     with context.eager_mode():
@@ -94,13 +90,12 @@ class TensorListTest(test.TestCase):
     c3 = l.count()
     a2 = l.pop()
     c4 = l.count()
-    with Session() as sess:
-      c1, c2, c3, c4, a, a2 = self.evaluate([c1, c2, c3, c4, a, a2])
-      self.assertEqual(c1, 1)
-      self.assertEqual(c2, 2)
-      self.assertEqual(c3, 1)
-      self.assertEqual(c4, 0)
-      self.assertEqual(a, a2)
+    c1, c2, c3, c4, a, a2 = self.evaluate([c1, c2, c3, c4, a, a2])
+    self.assertEqual(c1, 1)
+    self.assertEqual(c2, 2)
+    self.assertEqual(c3, 1)
+    self.assertEqual(c4, 0)
+    self.assertEqual(a, a2)
 
   def test_list_index_tf(self):
     a = constant(3.0)
@@ -110,10 +105,9 @@ class TensorListTest(test.TestCase):
     l0 = l[0]
     l[0] = b
     l1 = l[0]
-    with self.cached_session() as sess:
-      l0, l1, a, b = self.evaluate([l0, l1, a, b])
-      self.assertEqual(l0, a)
-      self.assertEqual(l1, b)
+    l0, l1, a, b = self.evaluate([l0, l1, a, b])
+    self.assertEqual(l0, a)
+    self.assertEqual(l1, b)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index ed6d716ef1..884c04eb7a 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -185,7 +185,7 @@ class TensorArrayTest(test.TestCase):
                           self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
   @test_util.disable_control_flow_v2("b/118890905")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/118890905")
   def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros()
 
@@ -202,7 +202,7 @@ class TensorArrayTest(test.TestCase):
                         self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
   @test_util.disable_control_flow_v2("b/118890905")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/118890905")
   def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros()
 
@@ -1019,6 +1019,7 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(self.evaluate(ta.read(i)), i)
 
   @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
+  @test_util.run_v1_only("b/117943489")
   def testSkipEagerWhileLoopDynamicWritePackGradients(self):
     self._testWhileLoopWritePackGradients(
         dynamic_size=True, dtype=dtypes.float32)
@@ -1238,7 +1239,7 @@ class TensorArrayTest(test.TestCase):
         w1.write(4, c2)
 
   @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def testUnpackShape(self):
     self._testUnpackShape()
 
@@ -1318,12 +1319,12 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
   @test_util.disable_control_flow_v2("b/117943489")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayUnpackDynamic(self):
     self._testTensorArrayUnpackDynamic()
 
   @test_util.disable_control_flow_v2("b/117943489")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArraySplitDynamic(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -1371,12 +1372,12 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([0, 5], self.evaluate(concatenated).shape)
 
   @test_util.disable_control_flow_v2("b/117943489")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
 
   @test_util.disable_control_flow_v2("b/117943489")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayScatterReadAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -1404,7 +1405,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
   @test_util.disable_control_flow_v2("b/117943286")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/117943286")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
-- 
GitLab


From d41bc2083ebd856718068ca06e70f61302360f9b Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 3 Dec 2018 13:18:27 -0800
Subject: [PATCH 1222/1554] Fix a bug in Toco FuseBinaryIntoFollowingAffine
 transformation

It incorrectly removes an array even if it has other consumers.
`GetOpWithInput` function returns the first op when there are
multiple consumer ops.

The transformation can be further optimized, but this CL is
strictly better than before.

PiperOrigin-RevId: 223850263
---
 tensorflow/lite/testing/generate_examples.py           | 10 +++++++---
 .../fuse_binary_into_following_affine.cc               | 10 ++++++----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 7bd4c009b4..3448c58ed2 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -1452,23 +1452,27 @@ def make_conv_with_shared_weights_tests(zip_path):
     input_shape, filter_shape = get_tensor_shapes(parameters)
     input_tensor = tf.placeholder(
         dtype=tf.float32, name="input", shape=input_shape)
+    input_tensors = [input_tensor]
 
     # Construct a constant weights tensor which will be used by both Conv2D.
     filter_tensor = tf.constant(
         create_tensor_data(np.float32, filter_shape), dtype=tf.float32)
-    input_tensors = [input_tensor]
+
+    # Ensure that FuseBinaryIntoFollowingAffine works with an input which
+    # is shared by multiple affine ops.
+    conv_input = input_tensor + 0.1
 
     # Construct 2 Conv2D operations which use exactly the same input and
     # weights.
     result1 = tf.nn.conv2d(
-        input_tensor,
+        conv_input,
         filter_tensor,
         strides=parameters["strides"],
         dilations=parameters["dilations"],
         padding=parameters["padding"],
         data_format=parameters["data_format"])
     result2 = tf.nn.conv2d(
-        input_tensor,
+        conv_input,
         filter_tensor,
         strides=parameters["strides"],
         dilations=parameters["dilations"],
diff --git a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
index 6b4765b23c..436b639253 100644
--- a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
+++ b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
@@ -221,9 +221,8 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op,
   Operator* following_op = GetOpWithInput(*model, binary_op->outputs[0]);
 
   if (!following_op) {
-    AddMessageF(
-        "Not fusing %s because it is not consumed by exactly one other op",
-        LogName(*binary_op));
+    AddMessageF("Not fusing %s because it is not consumed by any op",
+                LogName(*binary_op));
     return ::tensorflow::Status::OK();
   }
 
@@ -288,7 +287,10 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op,
   AddMessageF("Fusing %s into the following %s", LogName(*binary_op),
               LogName(*following_op));
 
-  model->EraseArray(binary_op->outputs[0]);
+  if (CountOpsWithInput(*model, binary_op->outputs[0]) == 1) {
+    model->EraseArray(binary_op->outputs[0]);
+  }
+
   following_op->inputs[0] = binary_op->inputs[index_of_variable_input];
   const auto& old_constant_param_name =
       binary_op->inputs[index_of_constant_input];
-- 
GitLab


From 47fe933d89146a0488563bdfde2c0675a750ed16 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Mon, 3 Dec 2018 13:22:18 -0800
Subject: [PATCH 1223/1554] Don't trigger NaN errors when computing gradients
 for pow(0, x) in eager mode.

PiperOrigin-RevId: 223850837
---
 tensorflow/python/BUILD                       |  3 +
 tensorflow/python/eager/BUILD                 | 12 ++++
 .../python/eager/execution_callbacks.py       | 45 ++++++++++++++-
 .../python/eager/execution_callbacks_test.py  | 55 +++++++++++++++++++
 tensorflow/python/ops/math_grad.py            |  7 ++-
 tensorflow/python/ops/math_grad_test.py       | 23 ++++++++
 6 files changed, 140 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/python/eager/execution_callbacks_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index ec85e5cb60..d990ebb7f1 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3393,6 +3393,9 @@ cuda_py_test(
         ":client_testlib",
         ":framework_for_generated_wrappers",
         ":math_ops",
+        "//tensorflow/python/eager:backprop",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:execution_callbacks",
         "//third_party/py/numpy",
     ],
     tags = ["no_windows_gpu"],
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 5a18afa0fc..f43cf9327a 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -255,6 +255,18 @@ py_library(
     ],
 )
 
+py_test(
+    name = "execution_callbacks_test",
+    srcs = ["execution_callbacks_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":execution_callbacks",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
 py_library(
     name = "graph_only_ops",
     srcs = ["graph_only_ops.py"],
diff --git a/tensorflow/python/eager/execution_callbacks.py b/tensorflow/python/eager/execution_callbacks.py
index 80ff4459d6..28b6b84a82 100644
--- a/tensorflow/python/eager/execution_callbacks.py
+++ b/tensorflow/python/eager/execution_callbacks.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import contextlib
 import functools
 
 import numpy as np
@@ -28,8 +29,13 @@ from tensorflow.python.eager import core
 from tensorflow.python.eager import execute
 from tensorflow.python.platform import tf_logging as logging
 
-_DEFAULT_CALLBACK_ACTION = "raise"
-_VALID_CALLBACK_ACTIONS = (None, "ignore", "print", "raise", "warn")
+IGNORE = "ignore"
+PRINT = "print"
+RAISE = "raise"
+WARN = "warn"
+
+_DEFAULT_CALLBACK_ACTION = RAISE
+_VALID_CALLBACK_ACTIONS = (None, IGNORE, PRINT, RAISE, WARN)
 
 
 # TODO(cais): Consider moving this exception class to errors_impl.py.
@@ -335,3 +341,38 @@ def seterr(inf_or_nan=None):
           functools.partial(inf_nan_callback, action=inf_or_nan))
 
   return old_settings
+
+
+@contextlib.contextmanager
+def errstate(inf_or_nan=None):
+  """Context manager setting error state.
+
+  Example:
+  ```
+  c = tf.log(0.)  # -inf
+
+  with errstate(inf_or_nan="raise"):
+    tf.log(0.)  # <-- Raises InfOrNanError.
+  ```
+
+  Args:
+    inf_or_nan: Set action for infinity (`inf`) and NaN (`nan`) values.
+      Possible values: `{IGNORE, PRINT, RAISE, WARN}`.
+      `IGNORE`: take no action when `inf` values appear.
+      `PRINT`: print a warning to `stdout`.
+      `RAISE`: raise an `InfOrNanError`.
+      `WARN`: print a warning using `tf.logging.warn`.
+      A value of `None` leads to no change in the action of the condition.
+
+  Yields:
+    None.
+
+  Raises:
+    ValueError: If the value of any keyword arguments is invalid.
+  """
+  if not context.executing_eagerly():
+    yield
+  else:
+    old_settings = seterr(inf_or_nan=inf_or_nan)
+    yield
+    seterr(**old_settings)
diff --git a/tensorflow/python/eager/execution_callbacks_test.py b/tensorflow/python/eager/execution_callbacks_test.py
new file mode 100644
index 0000000000..5594ab5f12
--- /dev/null
+++ b/tensorflow/python/eager/execution_callbacks_test.py
@@ -0,0 +1,55 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for eager execution_callbacks."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import execution_callbacks
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+def log_zero():
+  """Computes `log(0.0)`."""
+  return math_ops.log(constant_op.constant(0.))
+
+
+class ExecutionCallbacksTest(test.TestCase):
+
+  def test_errstate_inf_raise(self):
+    with execution_callbacks.errstate(inf_or_nan=execution_callbacks.RAISE):
+      with self.assertRaises(execution_callbacks.InfOrNanError):
+        log_zero()
+
+  def test_errstate_inf_ignore(self):
+    with execution_callbacks.errstate(inf_or_nan=execution_callbacks.IGNORE):
+      self.assertEqual(-float("inf"), log_zero().numpy())
+
+  def test_errstate_nesting(self):
+    with execution_callbacks.errstate(inf_or_nan=execution_callbacks.RAISE):
+      with execution_callbacks.errstate(inf_or_nan=execution_callbacks.IGNORE):
+        self.assertEqual(-float("inf"), log_zero().numpy())
+
+      with self.assertRaises(execution_callbacks.InfOrNanError):
+        log_zero()
+
+
+if __name__ == "__main__":
+  ops.enable_eager_execution()
+  test.main()
diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py
index 35278d9680..c7ec1c57d1 100644
--- a/tensorflow/python/ops/math_grad.py
+++ b/tensorflow/python/ops/math_grad.py
@@ -1041,11 +1041,12 @@ def _PowGrad(op, grad):
   # Avoid false singularity at x = 0
   if x.dtype.is_complex:
     # real(x) < 0 is fine for the complex case
-    log_x = array_ops.where(
-        math_ops.not_equal(x, 0), math_ops.log(x), array_ops.zeros_like(x))
+    mask = math_ops.not_equal(x, 0)
   else:
     # There's no sensible real value to return if x < 0, so return 0
-    log_x = array_ops.where(x > 0, math_ops.log(x), array_ops.zeros_like(x))
+    mask = x > 0
+  safe_x = array_ops.where(mask, x, array_ops.ones_like(x))
+  log_x = array_ops.where(mask, math_ops.log(safe_x), array_ops.zeros_like(x))
   gy = array_ops.reshape(math_ops.reduce_sum(grad * z * log_x, ry), sy)
   return gx, gy
 
diff --git a/tensorflow/python/ops/math_grad_test.py b/tensorflow/python/ops/math_grad_test.py
index 88aa48271f..822f89768c 100644
--- a/tensorflow/python/ops/math_grad_test.py
+++ b/tensorflow/python/ops/math_grad_test.py
@@ -20,6 +20,9 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.eager import execution_callbacks
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -370,5 +373,25 @@ class XdivyTest(test.TestCase):
       self.assertAllClose(zero, xdivy_ygrad)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class PowGradTest(test.TestCase):
+
+  def test_zero_grad_tf_gradients(self):
+    if context.executing_eagerly():
+      self.skipTest("tf.gradients not supported in eager.")
+
+    x = constant_op.constant([-1., 0., 1.])
+    g = self.evaluate(gradients.gradients(math_ops.pow(x, 2), x)[0])
+    self.assertAllClose([-2., 0., 2.], g)
+
+  def test_zero_grad_tape(self):
+    with execution_callbacks.errstate(inf_or_nan=execution_callbacks.RAISE):
+      x = constant_op.constant([-1, 0., 1.])
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        g = tape.gradient(math_ops.pow(x, 2), x)
+      g = self.evaluate(g)
+      self.assertAllClose([-2., 0., 2.], g)
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 871037e91a31f8481147bf31cc38c8ec608acc18 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 13:44:35 -0800
Subject: [PATCH 1224/1554] Check static batch size consistency in Input Layer
 and fit/evaluate/predict.

PiperOrigin-RevId: 223854825
---
 tensorflow/python/keras/engine/network.py     | 95 +++++++++++--------
 tensorflow/python/keras/engine/sequential.py  | 38 +-------
 tensorflow/python/keras/engine/training.py    | 64 +++++++++++--
 .../python/keras/engine/training_test.py      | 63 ++++++++++++
 .../python/keras/engine/training_utils.py     | 52 ++++++++++
 5 files changed, 230 insertions(+), 82 deletions(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 93ae667c8b..7e6cc7bfee 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -38,6 +38,7 @@ from tensorflow.python.keras import backend
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.engine import base_layer_utils
 from tensorflow.python.keras.engine import saving
+from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import layer_utils
 from tensorflow.python.keras.utils import tf_utils
@@ -174,43 +175,7 @@ class Network(base_layer.Layer):
       self.outputs = list(outputs)
     else:
       self.outputs = [outputs]
-
-    # Check for redundancy in inputs.
-    if len(set(self.inputs)) != len(self.inputs):
-      raise ValueError('The list of inputs passed to the model '
-                       'is redundant. '
-                       'All inputs should only appear once.'
-                       ' Found: ' + str(self.inputs))
-    for x in self.inputs:
-      # Check that x has appropriate `_keras_history` metadata.
-      if not hasattr(x, '_keras_history'):
-        cls_name = self.__class__.__name__
-        raise ValueError('Input tensors to a ' + cls_name + ' ' +
-                         'must come from `tf.keras.Input`. '
-                         'Received: ' + str(x) +
-                         ' (missing previous layer metadata).')
-      # Check that x is an input tensor.
-      # pylint: disable=protected-access
-      layer, node_index, tensor_index = x._keras_history
-      if len(layer._inbound_nodes) > 1 or (
-          layer._inbound_nodes and layer._inbound_nodes[0].inbound_layers):
-        cls_name = self.__class__.__name__
-        logging.warning(cls_name + ' inputs must come from '
-                        '`tf.keras.Input` (thus holding past layer metadata), '
-                        'they cannot be the output of '
-                        'a previous non-Input layer. '
-                        'Here, a tensor specified as '
-                        'input to "' + self.name + '" was not an Input tensor, '
-                        'it was generated by layer ' + layer.name + '.\n'
-                        'Note that input tensors are '
-                        'instantiated via `tensor = tf.keras.Input(shape)`.\n'
-                        'The tensor that caused the issue was: ' + str(x.name))
-    for x in self.outputs:
-      if not hasattr(x, '_keras_history'):
-        cls_name = self.__class__.__name__
-        raise ValueError('Output tensors to a ' + cls_name + ' must be '
-                         'the output of a TensorFlow `Layer` '
-                         '(thus holding past layer metadata). Found: ' + str(x))
+    self._validate_graph_inputs_and_outputs()
 
     self._base_init(name=name)
     self._compute_previous_mask = (
@@ -1692,6 +1657,62 @@ class Network(base_layer.Layer):
                               positions=positions,
                               print_fn=print_fn)
 
+  def _validate_graph_inputs_and_outputs(self):
+    """Validates the inputs and outputs of a Graph Network."""
+    # Check for redundancy in inputs.
+    if len(set(self.inputs)) != len(self.inputs):
+      raise ValueError('The list of inputs passed to the model '
+                       'is redundant. '
+                       'All inputs should only appear once.'
+                       ' Found: ' + str(self.inputs))
+
+    for x in self.inputs:
+      # Check that x has appropriate `_keras_history` metadata.
+      if not hasattr(x, '_keras_history'):
+        cls_name = self.__class__.__name__
+        raise ValueError('Input tensors to a ' + cls_name + ' ' +
+                         'must come from `tf.keras.Input`. '
+                         'Received: ' + str(x) +
+                         ' (missing previous layer metadata).')
+      # Check that x is an input tensor.
+      # pylint: disable=protected-access
+      layer, _, _ = x._keras_history
+      if len(layer._inbound_nodes) > 1 or (
+          layer._inbound_nodes and layer._inbound_nodes[0].inbound_layers):
+        cls_name = self.__class__.__name__
+        logging.warning(cls_name + ' inputs must come from '
+                        '`tf.keras.Input` (thus holding past layer metadata), '
+                        'they cannot be the output of '
+                        'a previous non-Input layer. '
+                        'Here, a tensor specified as '
+                        'input to "' + self.name + '" was not an Input tensor, '
+                        'it was generated by layer ' + layer.name + '.\n'
+                        'Note that input tensors are '
+                        'instantiated via `tensor = tf.keras.Input(shape)`.\n'
+                        'The tensor that caused the issue was: ' + str(x.name))
+
+    # Check compatibility of batch sizes of Input Layers.
+    input_batch_sizes = [
+        training_utils.get_static_batch_size(x._keras_history[0])
+        for x in self.inputs
+    ]
+    consistent_batch_size = None
+    for batch_size in input_batch_sizes:
+      if batch_size is not None:
+        if (consistent_batch_size is not None and
+            batch_size != consistent_batch_size):
+          raise ValueError('The specified batch sizes of the Input Layers'
+                           ' are incompatible. Found batch sizes: {}'.format(
+                               input_batch_sizes))
+        consistent_batch_size = batch_size
+
+    for x in self.outputs:
+      if not hasattr(x, '_keras_history'):
+        cls_name = self.__class__.__name__
+        raise ValueError('Output tensors to a ' + cls_name + ' must be '
+                         'the output of a TensorFlow `Layer` '
+                         '(thus holding past layer metadata). Found: ' + str(x))
+
 
 def _is_hdf5_filepath(filepath):
   return (filepath.endswith('.h5') or filepath.endswith('.keras') or
diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 6e2f76cd80..3255613f6a 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -25,6 +25,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import layers as layer_module
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras.engine.network import Network
@@ -150,7 +151,7 @@ class Sequential(Model):
         assert len(layer._inbound_nodes[-1].output_tensors) == 1
         set_inputs = True
       else:
-        batch_shape, dtype = get_input_shape_and_dtype(layer)
+        batch_shape, dtype = training_utils.get_input_shape_and_dtype(layer)
         if batch_shape:
           # Instantiate an input layer.
           x = Input(
@@ -359,38 +360,3 @@ class Sequential(Model):
     if self.layers and hasattr(self.layers[0], 'input_spec'):
       return self.layers[0].input_spec
     return None
-
-
-def get_input_shape_and_dtype(layer):
-  """Retrieve input shape and input dtype of layer if applicable.
-
-  Args:
-    layer: Layer (or model) instance.
-
-  Returns:
-    Tuple (input_shape, input_dtype). Both could be None if the layer
-      does not have a defined input shape.
-
-  Raises:
-    ValueError: in case an empty Sequential or Graph Network is passed.
-  """
-  if ((isinstance(layer, Model) and layer._is_graph_network)
-      or isinstance(layer, Sequential)):
-    # We were passed a model as first layer.
-    # This requires a specific way to figure out the
-    # input shape and dtype.
-    if not layer.layers:
-      raise ValueError('Cannot add an empty model '
-                       'to a `Sequential` model.')
-    # In case of nested models: recover the first layer
-    # of the deepest model to infer input shape and dtype.
-    layer = layer.layers[0]
-    while ((isinstance(layer, Model) and layer._is_graph_network)
-           or isinstance(layer, Sequential)):
-      layer = layer.layers[0]
-
-  if hasattr(layer, '_batch_input_shape'):
-    batch_shape = layer._batch_input_shape
-    dtype = layer.dtype
-    return batch_shape, dtype
-  return None, None
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index a66d8866bd..4fa2c5576f 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -27,6 +27,7 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import losses
@@ -1688,9 +1689,8 @@ class Model(Network):
                 self._distribution_strategy, first_x_value, steps_per_epoch,
                 batch_size, is_training=True))
 
-    # Backwards compatibility
-    if batch_size is None and steps_per_epoch is None:
-      batch_size = 32
+    batch_size = self._validate_or_infer_batch_size(batch_size, steps_per_epoch,
+                                                    x)
 
     x, y, sample_weights = self._standardize_user_data(
         x,
@@ -1923,9 +1923,7 @@ class Model(Network):
         steps, batch_size = distributed_training_utils.get_input_params(
             self._distribution_strategy, first_x_value, steps, batch_size)
 
-    # Backwards compatibility.
-    if batch_size is None and steps is None:
-      batch_size = 32
+    batch_size = self._validate_or_infer_batch_size(batch_size, steps, x)
 
     x, y, sample_weights = self._standardize_user_data(
         x,
@@ -2037,9 +2035,7 @@ class Model(Network):
         steps, batch_size = distributed_training_utils.get_input_params(
             self._distribution_strategy, first_x_value, steps, batch_size)
 
-    # Backwards compatibility.
-    if batch_size is None and steps is None:
-      batch_size = 32
+    batch_size = self._validate_or_infer_batch_size(batch_size, steps, x)
 
     # Validate and standardize user data.
     if self._distribution_strategy:
@@ -2483,6 +2479,56 @@ class Model(Network):
     self._replicated_model = DistributedCallbackModel(first_replicated_model)
     self._replicated_model.set_original_model(self)
 
+  def _validate_or_infer_batch_size(self, batch_size, steps, x):
+    """Validates that the `batch_size` provided is consistent with InputLayer.
+
+    It's possible that the user specified a static batch size in their
+    InputLayer. If so, this method checks the provided `batch_size` and `x`
+    arguments are consistent with this static batch size. Also, if
+    `batch_size` is `None`, this method will attempt to infer the batch size
+    from the static batch size of the InputLayer.
+
+    Arguments:
+      batch_size: The batch_size provided as an argument to
+        fit/evaluate/predict.
+      steps: The steps provided as an argument to fit/evaluate/predict.
+      x: The data passed as `x` to fit/evaluate/predict.
+
+    Returns:
+      The validated batch_size, auto-inferred from the first layer if not
+      provided.
+    """
+    first_layer = super(Model,
+                        self).layers[0]  # Avoids the override in Sequential.
+    static_batch_size = training_utils.get_static_batch_size(first_layer)
+    if static_batch_size is not None:
+
+      # Check `batch_size` argument is consistent with InputLayer.
+      if batch_size is not None and batch_size != static_batch_size:
+        raise ValueError('The `batch_size` argument value ' + str(batch_size) +
+                         ' is incompatible with the specified batch size '
+                         'of your Input Layer: ' + str(static_batch_size))
+
+      # Check Dataset/Iterator batch size is consistent with InputLayer.
+      if isinstance(x, (dataset_ops.DatasetV2, iterator_ops.Iterator,
+                        iterator_ops.EagerIterator)):
+        ds_batch_size = tensor_shape.as_dimension(
+            nest.flatten(x.output_shapes)[0][0]).value
+        if ds_batch_size is not None and ds_batch_size != static_batch_size:
+          raise ValueError('The batch output shape of your `Dataset` is ' +
+                           str(ds_batch_size) + ' which is incompatible '
+                           'with the specified batch size of your Input '
+                           'Layer: ' + str(static_batch_size))
+
+      # Set inferred batch size from the InputLayer.
+      if steps is None:
+        batch_size = static_batch_size
+
+    if batch_size is None and steps is None:
+      # Backwards compatibility
+      batch_size = 32
+    return batch_size
+
 
 class DistributedCallbackModel(Model):
   """Model that is used for callbacks with DistributionStrategy."""
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 4674bce244..1b04be914f 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -629,6 +629,69 @@ class TrainingTest(test.TestCase):
               epochs=1,
               batch_size=5)
 
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_static_batch_in_input_layer(self):
+
+    class Counter(keras.callbacks.Callback):
+
+      def __init__(self):
+        self.batches = 0
+
+      def on_batch_end(self, batch, logs=None):
+        self.batches += 1
+
+    x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32')
+
+    for batch_size, expected_batches in [(None, 2), (4, 16)]:
+      inputs = keras.Input(batch_size=batch_size, shape=(10,))
+      outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
+      model = keras.Model(inputs, outputs)
+
+      model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
+      counter = Counter()
+      model.fit(x, y, callbacks=[counter])
+      self.assertEqual(counter.batches, expected_batches)
+
+      model = keras.Sequential(
+          [keras.layers.Dense(1, batch_input_shape=(batch_size, 10))])
+      model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
+      counter = Counter()
+      model.fit(x, y, callbacks=[counter])
+      self.assertEqual(counter.batches, expected_batches)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_static_batch_in_input_layer_consistency_checks(self):
+    x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32')
+
+    inputs = keras.Input(batch_size=2, shape=(10,))
+    outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
+    model = keras.Model(inputs, outputs)
+    model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
+    with self.assertRaisesRegexp(ValueError,
+                                 'incompatible with the specified batch size'):
+      model.fit(x, y, batch_size=4)
+
+    data = dataset_ops.DatasetV2.from_tensor_slices((x, y))
+    data = data.batch(4, drop_remainder=True)
+    with self.assertRaisesRegexp(ValueError,
+                                 'incompatible with the specified batch size'):
+      model.fit(data, steps_per_epoch=16)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_compatible_batch_size_functional_model(self):
+
+    class MyLayer(keras.layers.Layer):
+
+      def call(self, inputs):
+        return array_ops.concat(inputs, axis=0)
+
+    input1 = keras.Input(batch_size=2, shape=(10,))
+    input2 = keras.Input(batch_size=3, shape=(10,))
+    outputs = MyLayer()([input1, input2])
+    with self.assertRaisesRegexp(ValueError,
+                                 'specified batch sizes of the Input Layers'):
+      keras.Model([input1, input2], outputs)
+
 
 class TestExceptionsAndWarnings(test.TestCase):
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 347582aa95..4b9d562472 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -30,6 +30,7 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import losses
@@ -1120,3 +1121,54 @@ class ModelInputs(object):
   def as_list(self):
     """Returning the inputs as a list."""
     return self._flattened_inputs
+
+
+# Allow use of methods not exposed to the user.
+# pylint: disable=protected-access
+def get_input_shape_and_dtype(layer):
+  """Retrieves input shape and input dtype of layer if applicable.
+
+  Args:
+    layer: Layer (or model) instance.
+
+  Returns:
+    Tuple (input_shape, input_dtype). Both could be None if the layer
+      does not have a defined input shape.
+
+  Raises:
+    ValueError: in case an empty Sequential or Graph Network is passed.
+  """
+
+  def _is_graph_model(layer):
+    return ((hasattr(layer, '_is_graph_network') and layer._is_graph_network) or
+            layer.__class__.__name__ == 'Sequential')
+
+  # In case of nested models: recover the first layer
+  # of the deepest model to infer input shape and dtype.
+  # Subclassed Models may not have been built so can't be checked.
+  while _is_graph_model(layer):
+    if not layer.layers:
+      raise ValueError('An empty Model cannot be used as a Layer.')
+    layer = layer.layers[0]
+
+  if hasattr(layer, '_batch_input_shape'):
+    return layer._batch_input_shape, layer.dtype
+  return None, None
+
+
+# pylint: enable=protected-access
+
+
+def get_static_batch_size(layer):
+  """Gets the static batch size of a Layer.
+
+  Arguments:
+    layer: a `Layer` instance.
+
+  Returns:
+    The static batch size of a Layer.
+  """
+  batch_input_shape, _ = get_input_shape_and_dtype(layer)
+  if batch_input_shape is not None:
+    return tensor_shape.as_dimension(batch_input_shape[0]).value
+  return None
-- 
GitLab


From 73a84a2289dacaf0827c439165307544ed7597a8 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Mon, 3 Dec 2018 13:58:30 -0800
Subject: [PATCH 1225/1554] Part of #22562: update cuda_blas.cc, cuda_fft.cc
 and cuda_rng.cc to use dynamic loading for cuda libraries. Remove explicit
 dependency on cublas, cufft and curand library in stream_executor.

PiperOrigin-RevId: 223857214
---
 tensorflow/stream_executor/BUILD             |   3 -
 tensorflow/stream_executor/cuda/cuda_blas.cc | 122 +++++++++++++------
 tensorflow/stream_executor/cuda/cuda_fft.cc  |  81 +++++++++---
 tensorflow/stream_executor/cuda/cuda_rng.cc  |  36 ++++++
 4 files changed, 187 insertions(+), 55 deletions(-)

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 2526e1adaa..4c764a7b09 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -108,11 +108,8 @@ cc_library(
         "@local_config_cuda//cuda:cuda_headers",
     ] + if_cuda_is_configured([
         "//tensorflow/core:cuda",
-        "@local_config_cuda//cuda:cublas",
         "@local_config_cuda//cuda:cuda_driver",
         "@local_config_cuda//cuda:cudnn",
-        "@local_config_cuda//cuda:cufft",
-        "@local_config_cuda//cuda:curand",
     ]),
     alwayslink = 1,
 )
diff --git a/tensorflow/stream_executor/cuda/cuda_blas.cc b/tensorflow/stream_executor/cuda/cuda_blas.cc
index 7fabb35e28..957f6c98da 100644
--- a/tensorflow/stream_executor/cuda/cuda_blas.cc
+++ b/tensorflow/stream_executor/cuda/cuda_blas.cc
@@ -58,6 +58,11 @@ limitations under the License.
 #include "tensorflow/stream_executor/cuda/cuda_stream.h"
 #include "tensorflow/stream_executor/cuda/cuda_timer.h"
 #include "tensorflow/stream_executor/device_memory.h"
+
+#ifndef PLATFORM_GOOGLE
+#include "tensorflow/stream_executor/dso_loader.h"
+#endif
+
 #include "tensorflow/stream_executor/lib/env.h"
 #include "tensorflow/stream_executor/lib/initialize.h"
 #include "tensorflow/stream_executor/lib/status.h"
@@ -76,21 +81,8 @@ PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuBlasPlugin);
 
 namespace wrap {
 
-#define STREAM_EXECUTOR_CUBLAS_WRAP(__name)                         \
-  struct WrapperShim__##__name {                                    \
-    static const char *kName;                                       \
-    template <typename... Args>                                     \
-    cublasStatus_t operator()(CUDAExecutor *parent, Args... args) { \
-      cuda::ScopedActivateExecutorContext sac{parent};              \
-      return ::__name(args...);                                     \
-    }                                                               \
-  } __name;                                                         \
-  const char *WrapperShim__##__name::kName = #__name;
-
-#define STREAM_EXECUTOR_CUBLAS_V2_WRAP(__name) \
-  STREAM_EXECUTOR_CUBLAS_WRAP(__name)
-
-#define CUBLAS_BLAS_ROUTINE_EACH(__macro) \
+// clang-format off
+#define CUBLAS_ROUTINE_EACH(__macro)      \
   __macro(cublasSnrm2)                    \
   __macro(cublasDnrm2)                    \
   __macro(cublasScnrm2)                   \
@@ -262,6 +254,58 @@ namespace wrap {
   __macro(cublasCdgmm)                    \
   __macro(cublasZdgmm)
 
+// clang-format off
+
+#ifdef PLATFORM_GOOGLE
+#define STREAM_EXECUTOR_CUBLAS_WRAP(__name)                         \
+  struct WrapperShim__##__name {                                    \
+    static const char *kName;                                       \
+    template <typename... Args>                                     \
+    cublasStatus_t operator()(CUDAExecutor *parent, Args... args) { \
+      cuda::ScopedActivateExecutorContext sac{parent};              \
+      return ::__name(args...);                                     \
+    }                                                               \
+  } __name;                                                         \
+  const char *WrapperShim__##__name::kName = #__name;
+
+#define STREAM_EXECUTOR_CUBLAS_V2_WRAP(__name) \
+  STREAM_EXECUTOR_CUBLAS_WRAP(__name)
+
+#else
+
+#define STREAM_EXECUTOR_CUBLAS_WRAP(__name)                               \
+  struct DynLoadShim__##__name {                                          \
+    static const char* kName;                                             \
+    using FuncPtrT = std::add_pointer<decltype(::__name)>::type;          \
+    static void* GetDsoHandle() {                                         \
+      auto s = internal::CachedDsoLoader::GetCublasDsoHandle();           \
+      return s.ValueOrDie();                                              \
+    }                                                                     \
+    static FuncPtrT LoadOrDie() {                                         \
+      void* f;                                                            \
+      auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \
+                                                          kName, &f);     \
+      CHECK(s.ok()) << "could not find " << kName                         \
+                    << " in cublas DSO; dlerror: " << s.error_message();  \
+      return reinterpret_cast<FuncPtrT>(f);                               \
+    }                                                                     \
+    static FuncPtrT DynLoad() {                                           \
+      static FuncPtrT f = LoadOrDie();                                    \
+      return f;                                                           \
+    }                                                                     \
+    template <typename... Args>                                           \
+    cublasStatus_t operator()(CUDAExecutor* parent, Args... args) {       \
+      cuda::ScopedActivateExecutorContext sac{parent};                    \
+      return DynLoad()(args...);                                          \
+    }                                                                     \
+  } __name;                                                               \
+  const char* DynLoadShim__##__name::kName = #__name;
+
+#define STREAM_EXECUTOR_CUBLAS_V2_WRAP(__name) \
+  STREAM_EXECUTOR_CUBLAS_WRAP(__name)
+
+#endif
+
 STREAM_EXECUTOR_CUBLAS_V2_WRAP(cublasCreate)
 STREAM_EXECUTOR_CUBLAS_V2_WRAP(cublasDestroy)
 STREAM_EXECUTOR_CUBLAS_V2_WRAP(cublasSetStream)
@@ -271,7 +315,7 @@ STREAM_EXECUTOR_CUBLAS_WRAP(cublasSgemmBatched)
 STREAM_EXECUTOR_CUBLAS_WRAP(cublasDgemmBatched)
 STREAM_EXECUTOR_CUBLAS_WRAP(cublasCgemmBatched)
 STREAM_EXECUTOR_CUBLAS_WRAP(cublasZgemmBatched)
-CUBLAS_BLAS_ROUTINE_EACH(STREAM_EXECUTOR_CUBLAS_V2_WRAP)
+CUBLAS_ROUTINE_EACH(STREAM_EXECUTOR_CUBLAS_V2_WRAP)
 
 #if CUDA_VERSION >= 7050
 STREAM_EXECUTOR_CUBLAS_WRAP(cublasSgemmEx)
@@ -424,7 +468,8 @@ class ScopedCublasMathMode {
   // Note that when false is returned, an appropriate error has already been
   // logged.
   bool Init(cublasMath_t new_mode) {
-    cublasStatus_t ret = wrap::cublasGetMathMode(parent_, handle_, &old_mode_);
+    cublasStatus_t ret =
+        wrap::cublasGetMathMode(parent_, handle_, &old_mode_);
     if (ret != CUBLAS_STATUS_SUCCESS) {
       LOG(ERROR) << "failed to get old cublas math mode: " << ToString(ret);
       return ok_ = false;
@@ -442,7 +487,8 @@ class ScopedCublasMathMode {
   // successful in the first place.
   ~ScopedCublasMathMode() {
     if (ok_) {
-      cublasStatus_t ret = wrap::cublasSetMathMode(parent_, handle_, old_mode_);
+      cublasStatus_t ret =
+          wrap::cublasSetMathMode(parent_, handle_, old_mode_);
       if (ret != CUBLAS_STATUS_SUCCESS) {
         LOG(ERROR) << "failed to set former cublas math mode: "
                    << ToString(ret);
@@ -675,16 +721,16 @@ bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
                           const DeviceMemory<std::complex<float>> &x, int incx,
                           DeviceMemory<float> *result) {
   return DoBlasInternal(
-      wrap::cublasScasum, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasScasum, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasAsum(Stream *stream, uint64 elem_count,
                           const DeviceMemory<std::complex<double>> &x, int incx,
                           DeviceMemory<double> *result) {
   return DoBlasInternal(
-      wrap::cublasDzasum, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasDzasum, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasAxpy(Stream *stream, uint64 elem_count, float alpha,
@@ -835,16 +881,16 @@ bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
                           const DeviceMemory<std::complex<float>> &x, int incx,
                           DeviceMemory<float> *result) {
   return DoBlasInternal(
-      wrap::cublasScnrm2, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasScnrm2, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasNrm2(Stream *stream, uint64 elem_count,
                           const DeviceMemory<std::complex<double>> &x, int incx,
                           DeviceMemory<double> *result) {
   return DoBlasInternal(
-      wrap::cublasDznrm2, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasDznrm2, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasRot(Stream *stream, uint64 elem_count,
@@ -1060,48 +1106,48 @@ bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
                            const DeviceMemory<std::complex<float>> &x, int incx,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
-      wrap::cublasIcamax, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasIcamax, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamax(Stream *stream, uint64 elem_count,
                            const DeviceMemory<std::complex<double>> &x,
                            int incx, DeviceMemory<int> *result) {
   return DoBlasInternal(
-      wrap::cublasIzamax, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasIzamax, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            const DeviceMemory<float> &x, int incx,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
-      wrap::cublasIsamin, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasIsamin, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            const DeviceMemory<double> &x, int incx,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
-      wrap::cublasIdamin, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasIdamin, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            const DeviceMemory<std::complex<float>> &x, int incx,
                            DeviceMemory<int> *result) {
   return DoBlasInternal(
-      wrap::cublasIcamin, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasIcamin, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasIamin(Stream *stream, uint64 elem_count,
                            const DeviceMemory<std::complex<double>> &x,
                            int incx, DeviceMemory<int> *result) {
   return DoBlasInternal(
-      wrap::cublasIzamin, stream, false /* = pointer_mode_host */, elem_count,
-      CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
+      wrap::cublasIzamin, stream, false /* = pointer_mode_host */,
+      elem_count, CUDAComplex(CUDAMemory(x)), incx, CUDAMemoryMutable(result));
 }
 
 bool CUDABlas::DoBlasGbmv(Stream *stream, blas::Transpose trans, uint64 m,
diff --git a/tensorflow/stream_executor/cuda/cuda_fft.cc b/tensorflow/stream_executor/cuda/cuda_fft.cc
index cbf388a0f8..acac7d6368 100644
--- a/tensorflow/stream_executor/cuda/cuda_fft.cc
+++ b/tensorflow/stream_executor/cuda/cuda_fft.cc
@@ -23,6 +23,11 @@ limitations under the License.
 #include "tensorflow/stream_executor/cuda/cuda_platform_id.h"
 #include "tensorflow/stream_executor/cuda/cuda_stream.h"
 #include "tensorflow/stream_executor/device_memory.h"
+
+#ifndef PLATFORM_GOOGLE
+#include "tensorflow/stream_executor/dso_loader.h"
+#endif
+
 #include "tensorflow/stream_executor/lib/env.h"
 #include "tensorflow/stream_executor/lib/initialize.h"
 #include "tensorflow/stream_executor/lib/status.h"
@@ -38,6 +43,7 @@ PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuFftPlugin);
 
 namespace wrap {
 
+#ifdef PLATFORM_GOOGLE
 // This macro wraps a global identifier, given by __name, in a callable
 // structure that loads the DLL symbol out of the DSO handle in a thread-safe
 // manner on first use. This dynamic loading technique is used to avoid DSO
@@ -52,22 +58,69 @@ namespace wrap {
     }                                                            \
   } __name;
 
-#define CUFFT_ROUTINE_EACH(__macro)                                            \
-  __macro(cufftDestroy) __macro(cufftSetStream) __macro(cufftPlan1d)           \
-      __macro(cufftPlan2d) __macro(cufftPlan3d) __macro(cufftPlanMany)         \
-          __macro(cufftExecD2Z) __macro(cufftExecZ2D) __macro(cufftExecC2C)    \
-              __macro(cufftExecC2R) __macro(cufftExecZ2Z)                      \
-                  __macro(cufftExecR2C) __macro(cufftCreate)                   \
-                      __macro(cufftSetAutoAllocation)                          \
-                          __macro(cufftSetWorkArea) __macro(cufftGetSize1d)    \
-                              __macro(cufftMakePlan1d) __macro(cufftGetSize2d) \
-                                  __macro(cufftMakePlan2d)                     \
-                                      __macro(cufftGetSize3d)                  \
-                                          __macro(cufftMakePlan3d)             \
-                                              __macro(cufftGetSizeMany)        \
-                                                  __macro(cufftMakePlanMany)
+#else
+
+#define STREAM_EXECUTOR_CUFFT_WRAP(__name)                                \
+  struct DynLoadShim__##__name {                                          \
+    static const char *kName;                                             \
+    using FuncPtrT = std::add_pointer<decltype(::__name)>::type;          \
+    static void *GetDsoHandle() {                                         \
+      auto s = internal::CachedDsoLoader::GetCufftDsoHandle();            \
+      return s.ValueOrDie();                                              \
+    }                                                                     \
+    static FuncPtrT LoadOrDie() {                                         \
+      void *f;                                                            \
+      auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \
+                                                          kName, &f);     \
+      CHECK(s.ok()) << "could not find " << kName                         \
+                    << " in cufft DSO; dlerror: " << s.error_message();   \
+      return reinterpret_cast<FuncPtrT>(f);                               \
+    }                                                                     \
+    static FuncPtrT DynLoad() {                                           \
+      static FuncPtrT f = LoadOrDie();                                    \
+      return f;                                                           \
+    }                                                                     \
+    template <typename... Args>                                           \
+    cufftResult operator()(CUDAExecutor *parent, Args... args) {          \
+      cuda::ScopedActivateExecutorContext sac{parent};                    \
+      return DynLoad()(args...);                                          \
+    }                                                                     \
+  } __name;                                                               \
+  const char *DynLoadShim__##__name::kName = #__name;
+
+#endif
+
+// clang-format off
+
+#define CUFFT_ROUTINE_EACH(__macro)                                     \
+  __macro(cufftDestroy)                                                 \
+  __macro(cufftSetStream)                                               \
+  __macro(cufftPlan1d)                                                  \
+  __macro(cufftPlan2d)                                                  \
+  __macro(cufftPlan3d)                                                  \
+  __macro(cufftPlanMany)                                                \
+  __macro(cufftExecD2Z)                                                 \
+  __macro(cufftExecZ2D)                                                 \
+  __macro(cufftExecC2C)                                                 \
+  __macro(cufftExecC2R)                                                 \
+  __macro(cufftExecZ2Z)                                                 \
+  __macro(cufftExecR2C)                                                 \
+  __macro(cufftCreate)                                                  \
+  __macro(cufftSetAutoAllocation)                                       \
+  __macro(cufftSetWorkArea)                                             \
+  __macro(cufftGetSize1d)                                               \
+  __macro(cufftMakePlan1d)                                              \
+  __macro(cufftGetSize2d)                                               \
+  __macro(cufftMakePlan2d)                                              \
+  __macro(cufftGetSize3d)                                               \
+  __macro(cufftMakePlan3d)                                              \
+  __macro(cufftGetSizeMany)                                             \
+  __macro(cufftMakePlanMany)
+
+// clang-format on
 
 CUFFT_ROUTINE_EACH(STREAM_EXECUTOR_CUFFT_WRAP)
+#undef CUFFT_ROUTINE_EACH
 
 }  // namespace wrap
 
diff --git a/tensorflow/stream_executor/cuda/cuda_rng.cc b/tensorflow/stream_executor/cuda/cuda_rng.cc
index 88c4f15792..7f92071932 100644
--- a/tensorflow/stream_executor/cuda/cuda_rng.cc
+++ b/tensorflow/stream_executor/cuda/cuda_rng.cc
@@ -21,6 +21,11 @@ limitations under the License.
 #include "tensorflow/stream_executor/cuda/cuda_platform_id.h"
 #include "tensorflow/stream_executor/cuda/cuda_stream.h"
 #include "tensorflow/stream_executor/device_memory.h"
+
+#ifndef PLATFORM_GOOGLE
+#include "tensorflow/stream_executor/dso_loader.h"
+#endif
+
 #include "tensorflow/stream_executor/lib/env.h"
 #include "tensorflow/stream_executor/lib/initialize.h"
 #include "tensorflow/stream_executor/lib/status.h"
@@ -61,6 +66,7 @@ PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(kCuRandPlugin);
 
 namespace wrap {
 
+#ifdef PLATFORM_GOOGLE
 #define STREAM_EXECUTOR_CURAND_WRAP(__name)                         \
   struct WrapperShim__##__name {                                    \
     template <typename... Args>                                     \
@@ -70,6 +76,36 @@ namespace wrap {
     }                                                               \
   } __name;
 
+#else
+#define STREAM_EXECUTOR_CURAND_WRAP(__name)                               \
+  struct DynLoadShim__##__name {                                          \
+    static const char *kName;                                             \
+    using FuncPtrT = std::add_pointer<decltype(::__name)>::type;          \
+    static void *GetDsoHandle() {                                         \
+      auto s = internal::CachedDsoLoader::GetCurandDsoHandle();           \
+      return s.ValueOrDie();                                              \
+    }                                                                     \
+    static FuncPtrT LoadOrDie() {                                         \
+      void *f;                                                            \
+      auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \
+                                                          kName, &f);     \
+      CHECK(s.ok()) << "could not find " << kName                         \
+                    << " in curand DSO; dlerror: " << s.error_message();  \
+      return reinterpret_cast<FuncPtrT>(f);                               \
+    }                                                                     \
+    static FuncPtrT DynLoad() {                                           \
+      static FuncPtrT f = LoadOrDie();                                    \
+      return f;                                                           \
+    }                                                                     \
+    template <typename... Args>                                           \
+    curandStatus_t operator()(CUDAExecutor *parent, Args... args) {       \
+      cuda::ScopedActivateExecutorContext sac{parent};                    \
+      return DynLoad()(args...);                                          \
+    }                                                                     \
+  } __name;                                                               \
+  const char *DynLoadShim__##__name::kName = #__name;
+#endif
+
 STREAM_EXECUTOR_CURAND_WRAP(curandCreateGenerator);
 STREAM_EXECUTOR_CURAND_WRAP(curandDestroyGenerator);
 STREAM_EXECUTOR_CURAND_WRAP(curandSetStream);
-- 
GitLab


From cb3dbf7e836fa2db53a778584c61cc04e94a4a4e Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 3 Dec 2018 14:01:53 -0800
Subject: [PATCH 1226/1554] Update README.md doc for tf_upgrade_v2.py.

PiperOrigin-RevId: 223857830
---
 tensorflow/tools/compatibility/BUILD          | 10 +--
 tensorflow/tools/compatibility/README.md      | 63 ++++++++++++-------
 ...{test_file_v1_10.py => test_file_v1_12.py} |  0
 3 files changed, 45 insertions(+), 28 deletions(-)
 rename tensorflow/tools/compatibility/testdata/{test_file_v1_10.py => test_file_v1_12.py} (100%)

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 152a79f542..05d924c092 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -126,22 +126,22 @@ py_test(
 genrule(
     name = "generate_upgraded_file_v2",
     testonly = 1,
-    srcs = ["testdata/test_file_v1_10.py"],
+    srcs = ["testdata/test_file_v1_12.py"],
     outs = [
         "test_file_v2_0.py",
         "report_v2.txt",
     ],
     cmd = ("$(location :tf_upgrade_v2)" +
-           " --infile $(location testdata/test_file_v1_10.py)" +
+           " --infile $(location testdata/test_file_v1_12.py)" +
            " --outfile $(location test_file_v2_0.py)" +
            " --reportfile $(location report_v2.txt)"),
     tools = [":tf_upgrade_v2"],
 )
 
 py_test(
-    name = "test_file_v1_10",
+    name = "test_file_v1_12",
     size = "small",
-    srcs = ["testdata/test_file_v1_10.py"],
+    srcs = ["testdata/test_file_v1_12.py"],
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
@@ -164,6 +164,6 @@ exports_files(
         "tf_upgrade.py",
         "renames_v2.py",
         "testdata/test_file_v0_11.py",
-        "testdata/test_file_v1_10.py",
+        "testdata/test_file_v1_12.py",
     ],
 )
diff --git a/tensorflow/tools/compatibility/README.md b/tensorflow/tools/compatibility/README.md
index aabc7b253d..6ff42b1fef 100644
--- a/tensorflow/tools/compatibility/README.md
+++ b/tensorflow/tools/compatibility/README.md
@@ -1,60 +1,77 @@
 # TensorFlow Python API Upgrade Utility
 
 This tool allows you to upgrade your existing TensorFlow Python scripts.
-This script can be run on a single Python file:
+Specifically: \
+`tf_upgrade_v2.py`: upgrades code from TensorFlow 1.12 to TensorFlow 2.0 preview. \
+`tf_upgrade.py`: upgrades code to TensorFlow 1.0 from TensorFlow 0.11.
+
+## Running the script from pip package
+
+First, install TensorFlow pip package. See
+https://www.tensorflow.org/install/pip.
+
+Upgrade script can be run on a single Python file:
 
 ```
-tf_upgrade.py --infile foo.py --outfile foo-upgraded.py
+tf_upgrade_v2 --infile foo.py --outfile foo-upgraded.py
 ```
 
 It will print a list of errors it finds that it can't fix. You can also run
 it on a directory tree:
 
 ```
+# upgrade the .py files and copy all the other files to the outtree
+tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded
+
 # just upgrade the .py files
-tf_upgrade.py --intree coolcode --outtree coolcode-upgraded
-# after upgrade the .py files, then copy all the other files to the outtree
-tf_upgrade.py --intree coolcode --outtree coolcode-upgraded --copyotherfiles True
+tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded --copyotherfiles False
 ```
 
-In either case, it will also dump out a report e.g. which will detail changes
+
+## Report
+
+The script will also dump out a report e.g. which will detail changes
 e.g.:
 
 ```
-third_party/tensorflow/tools/compatibility/test_file_v0.11.py Line 125
+'tensorflow/tools/compatibility/testdata/test_file_v1_12.py' Line 65
+--------------------------------------------------------------------------------
+
+Added keyword 'input' to reordered function 'tf.argmax'
+Renamed keyword argument from 'dimension' to 'axis'
 
-Renamed keyword argument from `dim` to `axis`
-Renamed keyword argument from `squeeze_dims` to `axis`
+    Old:         tf.argmax([[1, 3, 2]], dimension=0))
+                                        ~~~~~~~~~~
+    New:         tf.argmax(input=[[1, 3, 2]], axis=0))
 
-    Old:                   [[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(),
-                                        ~~~~    ~~~~~~~~~~~~~
-    New:                   [[1, 2, 3]], axis=1), axis=[1]).eval(),
-                                        ~~~~~    ~~~~~
 ```
 
 ## Caveats
 
 - Don't update parts of your code manually before running this script. In
-particular, functions that have had reordered arguments like `tf.concat`
-or `tf.split` will cause the script to incorrectly add keyword arguments that
-mismap arguments.
+particular, functions that have had reordered arguments like `tf.argmax`
+or `tf.batch_to_space` will cause the script to incorrectly add keyword
+arguments that mismap arguments.
 
 - This script wouldn't actually reorder arguments. Instead, the script will add
 keyword arguments to functions that had their arguments reordered.
 
 - This script is not able to upgrade all functions. One notable example is
-`tf.reverse()` which has been changed to take a list of indices rather than
-a tensor of bools. If the script detects this, it will report this to stdout
+`tf.nn.conv2d` that no longer takes `use_cudnn_on_gpu` argument.
+If the script detects this, it will report this to stdout
 (and in the report), and you can fix it manually. For example if you have
-`tf.reverse(a, [False, True, True])` you will need to manually change it to
-`tf.reverse(a, [1, 2])`.
+`tf.nn.conv2d(inputs, filters, strides, padding, use_cudnn_on_gpu=True)`
+you will need to manually change it to
+`tf.nn.conv2d(input, filters, strides, padding)`.
 
 - There are some syntaxes that are not handleable with this script as this
-script was designed to use only standard python packages. If the script fails
-with "A necessary keyword argument failed to be inserted." or
+script was designed to use only standard python packages.
+There is an alternative available for TensorFlow 0.* to 1.0 upgrade script.
+If the script fails with "A necessary keyword argument failed to be inserted." or
 "Failed to find keyword lexicographically. Fix manually.", you can try
 [@machrisaa's fork of this script](https://github.com/machrisaa/tf0to1).
 [@machrisaa](https://github.com/machrisaa) has used the
 [RedBaron Python refactoring engine](https://redbaron.readthedocs.io/en/latest/)
 which is able to localize syntactic elements more reliably than the built-in
-`ast` module this script is based upon.
+`ast` module this script is based upon. Note that the alternative script is not
+available for TensorFlow 2.0 upgrade.
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_10.py b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
similarity index 100%
rename from tensorflow/tools/compatibility/testdata/test_file_v1_10.py
rename to tensorflow/tools/compatibility/testdata/test_file_v1_12.py
-- 
GitLab


From b8ff1a2764c80eea6859541bc4ef6523410b4708 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Mon, 3 Dec 2018 14:05:48 -0800
Subject: [PATCH 1227/1554] [grpc]: Allow subclasses to customize Session
 creation.

PiperOrigin-RevId: 223858759
---
 .../core/distributed_runtime/rpc/grpc_session.cc       | 10 +++++++---
 tensorflow/core/distributed_runtime/rpc/grpc_session.h |  3 +++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
index 1ad40fe297..32063fecbb 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
@@ -92,6 +92,12 @@ void ReEncodeConsts(GraphDef* gdef) {
 }
 }  // namespace
 
+void GrpcSession::SetHandleAndGraphVersion(string handle, int64 graph_version) {
+  mutex_lock l(mu_);
+  handle_ = std::move(handle);
+  current_graph_version_ = graph_version;
+}
+
 Status GrpcSession::Handle(string* out_handle) {
   mutex_lock l(mu_);
   if (handle_.empty()) {
@@ -117,9 +123,7 @@ Status GrpcSession::CreateImpl(CallOptions* call_options,
   CreateSessionResponse resp;
   Status s = master_->CreateSession(call_options, &req, &resp);
   if (s.ok()) {
-    mutex_lock l(mu_);
-    swap(handle_, *(resp.mutable_session_handle()));
-    current_graph_version_ = resp.graph_version();
+    SetHandleAndGraphVersion(resp.session_handle(), resp.graph_version());
   }
   return s;
 }
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.h b/tensorflow/core/distributed_runtime/rpc/grpc_session.h
index 63795117f9..e00cf97e38 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.h
@@ -106,6 +106,9 @@ class GrpcSession : public Session {
  protected:
   // Takes ownership of `*master`.
   void SetRemoteMaster(std::unique_ptr<MasterInterface> master);
+  // Allows subclasses to customize Session creation.
+  void SetHandleAndGraphVersion(string handle, int64 graph_version)
+      LOCKS_EXCLUDED(mu_);
 
  private:
   SessionOptions options_;
-- 
GitLab


From 5de9e18915d629b9fcb9f447d6ad6ea33a5d4ada Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Mon, 3 Dec 2018 14:14:25 -0800
Subject: [PATCH 1228/1554] [XLA:TPU] Fix error message to avoid messages such
 as: Requested device count (2) exceeds the number of available devices on the
 target (2)

PiperOrigin-RevId: 223860317
---
 tensorflow/compiler/xla/service/service.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 3b336d5c9d..5ec7fe2ade 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -746,9 +746,9 @@ Status Service::GetDeviceHandles(const GetDeviceHandlesRequest* arg,
   }
   if (available_device_count < arg->device_count() * replica_count) {
     return ResourceExhausted(
-        "Requested device count (%d) exceeds the number of available devices "
-        "on the target (%d)",
-        arg->device_count(), available_device_count);
+        "Requested logical device count (%d) with replica count (%d) exceeds "
+        "the number of available physical devices on the target (%d)",
+        arg->device_count(), replica_count, available_device_count);
   }
 
   for (int64 i = 0; i < arg->device_count(); ++i) {
-- 
GitLab


From 0aafc65dc709cb5c42d4c4990cf87cd7919735e6 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 3 Dec 2018 14:20:29 -0800
Subject: [PATCH 1229/1554] [XLA] Use pattern matcher in hlo_graph_dumper.

In place of bespoke pattern matching.

No functional change.

PiperOrigin-RevId: 223861467
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../compiler/xla/service/hlo_graph_dumper.cc  | 44 +++++++------------
 2 files changed, 17 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 1666227134..81e71eee52 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3134,6 +3134,7 @@ cc_library(
         ":hlo_casting_utils",
         ":hlo_execution_profile",
         ":hlo_tfgraph_builder",
+        ":pattern_matcher",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:types",
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 4bfa211121..302eca656b 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_tfgraph_builder.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/window_util.h"
@@ -236,34 +237,28 @@ string HtmlLikeStringSanitize(absl::string_view s) {
 // it to a short string lets us tell the user what the subcomputation is without
 // drawing it as a graph.
 optional<string> MatchTrivialComputation(const HloComputation* computation) {
+  namespace m = match;
+
   if (computation->instruction_count() != 3) {
     return nullopt;
   }
-
   HloInstruction* root = computation->root_instruction();
-  if (root->operand_count() != 2) {
-    return nullopt;
-  }
-
-  // Check that both of the operands to the root are parameters.
-  const HloInstruction* operand0 = root->operand(0);
-  const HloInstruction* operand1 = root->operand(1);
-  if (operand0->opcode() != HloOpcode::kParameter ||
-      operand1->opcode() != HloOpcode::kParameter) {
+  const HloInstruction *param0, *param1;
+  if (!Match(root, m::Op()
+                       .WithNumOperands(2)
+                       .WithShape(m::Shape().IsEffectiveScalar())
+                       .WithBinaryOperandsAnyOrder(
+                           m::Parameter(&param0, 0)
+                               .WithShape(m::Shape().IsEffectiveScalar()),
+                           m::Parameter(&param1, 1)
+                               .WithShape(m::Shape().IsEffectiveScalar())))) {
     return nullopt;
   }
 
-  // Check that the two operands of root are param0 and param1.  All of the
-  // opcodes we recognize are commutative, so we're OK with either order.
-  auto n0 = operand0->parameter_number();
-  auto n1 = operand1->parameter_number();
-  if (!(n0 == 0 && n1 == 1) && !(n1 == 0 && n0 == 1)) {
-    return nullopt;
-  }
-
-  // If the params are reversed, check that the operation being performed is
-  // commutative.
-  if (n0 == 1) {
+  // If the params are reversed (i.e. operand0 is param1 and operand1 is
+  // param0), check that the operation being performed is commutative.
+  if (root->operand(0) == param1) {
+    CHECK_EQ(root->operand(1), param0);
     switch (root->opcode()) {
       case HloOpcode::kLe:
       case HloOpcode::kGe:
@@ -275,13 +270,6 @@ optional<string> MatchTrivialComputation(const HloComputation* computation) {
     }
   }
 
-  // Check that the root and params are all effective scalars.
-  if (!ShapeUtil::IsEffectiveScalar(root->shape()) ||
-      !ShapeUtil::IsEffectiveScalar(operand0->shape()) ||
-      !ShapeUtil::IsEffectiveScalar(operand1->shape())) {
-    return nullopt;
-  }
-
   // If we recognize the root's opcode, we've successfully pattern-matched!
   switch (root->opcode()) {
     case HloOpcode::kAdd:
-- 
GitLab


From 26b5771b3018f76cb680122caa3bba8ac7e8aa7b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 3 Dec 2018 14:20:51 -0800
Subject: [PATCH 1230/1554] [TF:XLA] Bump open source llvm revision to r348124

PiperOrigin-RevId: 223861539
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 83fd940a63..7e5f84be16 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -472,11 +472,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "ba61783c407c8decb2df1ca1c7019cbd0bb38419509ae8870fa7ce6bc30b6cfb",
-        strip_prefix = "llvm-cdc3118297348c68b25dff92b504581f5dd78000",
+        sha256 = "34170a4aa07e434dd537d98a705dcf1b3901f73820fe1d6b9370e8c1c94e9157",
+        strip_prefix = "llvm-0487bd8f42c8b38166ff825d56014d0ff49db604",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/cdc3118297348c68b25dff92b504581f5dd78000.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/cdc3118297348c68b25dff92b504581f5dd78000.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0487bd8f42c8b38166ff825d56014d0ff49db604.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/0487bd8f42c8b38166ff825d56014d0ff49db604.tar.gz",
         ],
     )
 
-- 
GitLab


From b1bc0b3cde4193b8762d112adc42a5e81bf93126 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 3 Dec 2018 14:30:02 -0800
Subject: [PATCH 1231/1554] Add publicly available corpus and dictionary for
 decode_json_example.

PiperOrigin-RevId: 223863238
---
 tensorflow/core/kernels/fuzzing/BUILD         |  4 ++
 .../013a29ea098a178f8a36741c9fd91144          | 48 +++++++++++++++++++
 .../0875575fb76d630ccb19c5da8aab66b2          |  1 +
 .../7e7f58fc443a11a0a2c5d9b643b7e99b          |  1 +
 .../849a23936269a261c0370b5e9abe2416          |  1 +
 .../85282c1696d98b9843ce3e8bd1cd899f          |  1 +
 .../90388b9c8093d8adedad0644b618da87          | 33 +++++++++++++
 .../9fa2f86ea6d3ade36e961247c3026f8d          | 33 +++++++++++++
 .../c4f18ca60a84e9869a28faf6f65dc758          | 32 +++++++++++++
 .../d456ee029700adef5d28438593010223          |  1 +
 .../e9f0ff6ee8d691ae69d2ecb4710030a2          | 36 ++++++++++++++
 .../dictionaries/decode_json_example.dict     |  6 +++
 12 files changed, 197 insertions(+)
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/013a29ea098a178f8a36741c9fd91144
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/0875575fb76d630ccb19c5da8aab66b2
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/7e7f58fc443a11a0a2c5d9b643b7e99b
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/849a23936269a261c0370b5e9abe2416
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/85282c1696d98b9843ce3e8bd1cd899f
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/90388b9c8093d8adedad0644b618da87
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/9fa2f86ea6d3ade36e961247c3026f8d
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/c4f18ca60a84e9869a28faf6f65dc758
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/d456ee029700adef5d28438593010223
 create mode 100644 tensorflow/core/kernels/fuzzing/corpus/decode_json_example/e9f0ff6ee8d691ae69d2ecb4710030a2
 create mode 100644 tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict

diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index 72088e0de2..2d8b734535 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -64,3 +64,7 @@ tf_ops_fuzz_target_lib("parse_tensor_op")
 tf_ops_fuzz_target_lib("decode_compressed")
 
 tf_ops_fuzz_target_lib("decode_json_example")
+
+tf_oss_fuzz_corpus("decode_json_example")
+
+tf_oss_fuzz_dict("decode_json_example")
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/013a29ea098a178f8a36741c9fd91144 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/013a29ea098a178f8a36741c9fd91144
new file mode 100644
index 0000000000..06fd804480
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/013a29ea098a178f8a36741c9fd91144
@@ -0,0 +1,48 @@
+{
+  features: {
+    feature: {
+      age: {
+        float_list: {
+          value: 29.0
+        }
+      }
+    },
+    feature: {
+      movie: {
+        bytes_list: {
+          value: "VGhlIFNoYXdzaGFuayBSZWRlbXB0aW9u",
+          value: "RmlnaHQgQ2x1Yg=="
+        }
+      }
+    },
+    feature: {
+      movie_ratings: {
+        float_list: {
+          value: 9.0,
+          value: 9.7
+        }
+      }
+    },
+    feature: {
+      suggestion: {
+        bytes_list: {
+          value: "SW5jZXB0aW9u"
+        }
+      }
+    },
+    feature: {
+      suggestion_purchased: {
+        float_list: {
+          value: 1.0
+        }
+      }
+    },
+    feature: {
+      purchase_price: {
+        float_list: {
+          value: 9.99
+        }
+      }
+    }
+  }
+}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/0875575fb76d630ccb19c5da8aab66b2 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/0875575fb76d630ccb19c5da8aab66b2
new file mode 100644
index 0000000000..4ae686974e
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/0875575fb76d630ccb19c5da8aab66b2
@@ -0,0 +1 @@
+{features:{feature:{age:{float_list:{value:[29.0,2,3,4]}}},feature:{movie_ratings:{float_list:{value:[9.0,9.7]}}},feature:{suggestion_purchased:{float_list:{value:[1.0,2,3,4,5]}}},feature:{purchase_price:{float_list:{value:[9.99,8.88,7.77,6.66,5.55],value:[4.44,3.33,2.22,1.11],value:[1.11,2.22,3.33],value:[4.44,5.55],value:0}}}}}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/7e7f58fc443a11a0a2c5d9b643b7e99b b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/7e7f58fc443a11a0a2c5d9b643b7e99b
new file mode 100644
index 0000000000..150f8710f7
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/7e7f58fc443a11a0a2c5d9b643b7e99b
@@ -0,0 +1 @@
+{features:{feature:{age:{float_list:{value:29.0}}},feature:{movie_ratings:{float_list:{value:[9.0,9.7]}}},feature:{suggestion_purchased:{float_list:{value:1.0}}},feature:{purchase_price:{float_list:{value:9.99}}}}}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/849a23936269a261c0370b5e9abe2416 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/849a23936269a261c0370b5e9abe2416
new file mode 100644
index 0000000000..fcfdfedd1b
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/849a23936269a261c0370b5e9abe2416
@@ -0,0 +1 @@
+{features:{feature:{age:{float_list:{value:29.0}}},feature:{movie_ratings:{float_list:{value:[[[[[[9.0,9.7]]]]]],value:[[[9.0,-9.2]]]}}},feature:{suggestion_purchased:{float_list:{value:[1.0,[2,3,[4,5,6,[7,8,9,0]]]]}}},feature:{purchase_price:{float_list:{value:9.99}}}}}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/85282c1696d98b9843ce3e8bd1cd899f b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/85282c1696d98b9843ce3e8bd1cd899f
new file mode 100644
index 0000000000..7c9981d482
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/85282c1696d98b9843ce3e8bd1cd899f
@@ -0,0 +1 @@
+{features:{feature:{age:{float_list:{value:29.0}}},feature:{movie_ratings:{float_list:{value:9.0,value:9.7}}},feature:{suggestion_purchased:{float_list:{value:1.0}}},feature:{purchase_price:{float_list:{value:9.99}}}}}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/90388b9c8093d8adedad0644b618da87 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/90388b9c8093d8adedad0644b618da87
new file mode 100644
index 0000000000..a1315bb8f9
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/90388b9c8093d8adedad0644b618da87
@@ -0,0 +1,33 @@
+{
+  features: {
+    feature: {
+      age: {
+        float_list: {
+          value: 29.0
+        }
+      }
+    },
+    feature: {
+      movie_ratings: {
+        float_list: {
+          value: [[[[[[9.0,9.7]]]]]],
+          value: [[[9.0, -9.2]]]
+        }
+      }
+    },
+    feature: {
+      suggestion_purchased: {
+        float_list: {
+          value: [1.0, [2, 3, [4, 5, 6, [7, 8, 9, 0]]]]
+        }
+      }
+    },
+    feature: {
+      purchase_price: {
+        float_list: {
+          value: 9.99
+        }
+      }
+    }
+  }
+}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/9fa2f86ea6d3ade36e961247c3026f8d b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/9fa2f86ea6d3ade36e961247c3026f8d
new file mode 100644
index 0000000000..d4f9494bbd
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/9fa2f86ea6d3ade36e961247c3026f8d
@@ -0,0 +1,33 @@
+{
+  features: {
+    feature: {
+      age: {
+        float_list: {
+          value: 29.0
+        }
+      }
+    },
+    feature: {
+      movie_ratings: {
+        float_list: {
+          value: 9.0,
+          value: 9.7
+        }
+      }
+    },
+    feature: {
+      suggestion_purchased: {
+        float_list: {
+          value: 1.0
+        }
+      }
+    },
+    feature: {
+      purchase_price: {
+        float_list: {
+          value: 9.99
+        }
+      }
+    }
+  }
+}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/c4f18ca60a84e9869a28faf6f65dc758 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/c4f18ca60a84e9869a28faf6f65dc758
new file mode 100644
index 0000000000..e8ba267eb2
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/c4f18ca60a84e9869a28faf6f65dc758
@@ -0,0 +1,32 @@
+{
+  features: {
+    feature: {
+      age: {
+        float_list: {
+          value: 29.0
+        }
+      }
+    },
+    feature: {
+      movie_ratings: {
+        float_list: {
+          value: [9.0,9.7]
+        }
+      }
+    },
+    feature: {
+      suggestion_purchased: {
+        float_list: {
+          value: 1.0
+        }
+      }
+    },
+    feature: {
+      purchase_price: {
+        float_list: {
+          value: 9.99
+        }
+      }
+    }
+  }
+}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/d456ee029700adef5d28438593010223 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/d456ee029700adef5d28438593010223
new file mode 100644
index 0000000000..3428a1e0fc
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/d456ee029700adef5d28438593010223
@@ -0,0 +1 @@
+{features:{feature:{age:{float_list:{value:29.0}}},feature:{movie:{bytes_list:{value:"VGhlIFNoYXdzaGFuayBSZWRlbXB0aW9u",value:"RmlnaHQgQ2x1Yg=="}}},feature:{movie_ratings:{float_list:{value:9.0,value:9.7}}},feature:{suggestion:{bytes_list:{value:"SW5jZXB0aW9u"}}},feature:{suggestion_purchased:{float_list:{value:1.0}}},feature:{purchase_price:{float_list:{value:9.99}}}}}
diff --git a/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/e9f0ff6ee8d691ae69d2ecb4710030a2 b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/e9f0ff6ee8d691ae69d2ecb4710030a2
new file mode 100644
index 0000000000..ef0923c450
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/corpus/decode_json_example/e9f0ff6ee8d691ae69d2ecb4710030a2
@@ -0,0 +1,36 @@
+{
+  features: {
+    feature: {
+      age: {
+        float_list: {
+          value: [29.0, 2, 3, 4]
+        }
+      }
+    },
+    feature: {
+      movie_ratings: {
+        float_list: {
+          value: [9.0,9.7]
+        }
+      }
+    },
+    feature: {
+      suggestion_purchased: {
+        float_list: {
+          value: [1.0, 2, 3, 4, 5]
+        }
+      }
+    },
+    feature: {
+      purchase_price: {
+        float_list: {
+          value: [9.99, 8.88, 7.77, 6.66, 5.55],
+          value: [4.44, 3.33, 2.22, 1.11],
+          value: [1.11, 2.22, 3.33],
+          value: [4.44, 5.55],
+          value: 0
+        }
+      }
+    }
+  }
+}
diff --git a/tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict b/tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict
new file mode 100644
index 0000000000..5fe4ca23d1
--- /dev/null
+++ b/tensorflow/core/kernels/fuzzing/dictionaries/decode_json_example.dict
@@ -0,0 +1,6 @@
+"features"
+"feature"
+"bytes_list"
+"float_list"
+"int64_list"
+"value"
-- 
GitLab


From 54b5a2163bc2c5a13db8de39fc99ae558fc854a4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 14:32:54 -0800
Subject: [PATCH 1232/1554] batch_matmul_op_test.py: Updated to pass in TF2, by
 using gradient_checker_v2 and removing placeholder nodes.

PiperOrigin-RevId: 223863787
---
 .../kernel_tests/batch_matmul_op_test.py      | 91 ++++++++-----------
 1 file changed, 38 insertions(+), 53 deletions(-)

diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
index a0ad8151b2..aebbc81f8f 100644
--- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
@@ -20,9 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.framework import constant_op
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -76,25 +75,18 @@ class BatchMatmulOpTest(test.TestCase):
 
   # Compares _tfpBatchMatmul(x, y, alpha, adj) and _npBatchMatMul(x, y, alpha,
   # adj)
-  def _compare(self, x_in, y_in, adjoint_a, adjoint_b, static_shape=True):
+  def _compare(self, x_in, y_in, adjoint_a, adjoint_b):
     x_t_shape = x_in.shape[:-2] + (x_in.shape[-1], x_in.shape[-2])
     y_t_shape = y_in.shape[:-2] + (y_in.shape[-1], y_in.shape[-2])
     x = x_in if not adjoint_a else x_in.reshape(x_t_shape)
     y = y_in if not adjoint_b else y_in.reshape(y_t_shape)
     is_floating = x.dtype != np.int32
     tol = 100 * np.finfo(x.dtype).eps if is_floating else 0
-    with self.cached_session(use_gpu=is_floating) as sess:
-      if static_shape:
-        z0 = math_ops.matmul(x, y, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
-        z0_val = self.evaluate(z0)
-      else:
-        x_ph = array_ops.placeholder(x.dtype)
-        y_ph = array_ops.placeholder(y.dtype)
-        z0 = math_ops.matmul(
-            x_ph, y_ph, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
-        z0_val = sess.run(z0, feed_dict={x_ph: x, y_ph: y})
+    with test_util.device(use_gpu=is_floating):
+      z0 = math_ops.matmul(
+          x, y, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
       z1 = self._npBatchMatmul(x, y, adjoint_a, adjoint_b)
-      self.assertAllClose(z0_val, z1, rtol=tol, atol=tol)
+      self.assertAllClose(z0, z1, rtol=tol, atol=tol)
 
   def _rand(self, shape, dtype):
     vals = np.array(np.random.normal(-10, 10, np.prod(shape)), dtype=dtype)
@@ -103,42 +95,41 @@ class BatchMatmulOpTest(test.TestCase):
       vals += 1j * imag
     return vals.reshape(shape)
 
-  def _testNonEmpty(self, dtype, adjoint_a, adjoint_b, use_static_shape):
+  def _testNonEmpty(self, dtype, adjoint_a, adjoint_b):
 
-    def compareNonEmpty(self, a_shape, b_shape):
+    def CompareNonEmpty(self, a_shape, b_shape):
       self._compare(
           self._rand(a_shape, dtype),
-          self._rand(b_shape, dtype), adjoint_a, adjoint_b, use_static_shape)
+          self._rand(b_shape, dtype), adjoint_a, adjoint_b)
 
-    compareNonEmpty(self, [1, 2, 3], [1, 3, 5])
-    compareNonEmpty(self, [1, 2, 3], [1, 3, 1])
-    compareNonEmpty(self, [1, 1, 3], [1, 3, 5])
-    compareNonEmpty(self, [1, 2, 3], [1, 3, 5])
-    compareNonEmpty(self, [7, 1, 3], [7, 3, 5])
-    compareNonEmpty(self, [7, 2, 3], [7, 3, 1])
-    compareNonEmpty(self, [7, 2, 3], [7, 3, 5])
-    compareNonEmpty(self, [10, 64, 75], [10, 75, 30])
-    compareNonEmpty(self, [5, 7, 2, 3], [5, 7, 3, 5])
+    CompareNonEmpty(self, [1, 2, 3], [1, 3, 5])
+    CompareNonEmpty(self, [1, 2, 3], [1, 3, 1])
+    CompareNonEmpty(self, [1, 1, 3], [1, 3, 5])
+    CompareNonEmpty(self, [1, 2, 3], [1, 3, 5])
+    CompareNonEmpty(self, [7, 1, 3], [7, 3, 5])
+    CompareNonEmpty(self, [7, 2, 3], [7, 3, 1])
+    CompareNonEmpty(self, [7, 2, 3], [7, 3, 5])
+    CompareNonEmpty(self, [10, 64, 75], [10, 75, 30])
+    CompareNonEmpty(self, [5, 7, 2, 3], [5, 7, 3, 5])
 
-  def _testEmpty(self, dtype, adjoint_a, adjoint_b, use_static_shape):
+  def _testEmpty(self, dtype, adjoint_a, adjoint_b):
 
-    def compareEmpty(self, a_shape, b_shape):
+    def CompareEmpty(self, a_shape, b_shape):
       self._compare(
           np.zeros(a_shape).astype(dtype),
-          np.zeros(b_shape).astype(dtype), adjoint_a, adjoint_b,
-          use_static_shape)
+          np.zeros(b_shape).astype(dtype), adjoint_a, adjoint_b)
 
-    compareEmpty(self, [0, 3, 2], [0, 2, 4])
-    compareEmpty(self, [3, 0, 2], [3, 2, 5])
-    compareEmpty(self, [3, 3, 2], [3, 2, 0])
+    CompareEmpty(self, [0, 3, 2], [0, 2, 4])
+    CompareEmpty(self, [3, 0, 2], [3, 2, 5])
+    CompareEmpty(self, [3, 3, 2], [3, 2, 0])
 
 
-def _GetBatchMatmulOpTest(dtype, adjoint_a, adjoint_b, use_static_shape):
+def _GetBatchMatmulOpTest(dtype, adjoint_a, adjoint_b):
 
   def Test(self):
     np.random.seed(42)
-    self._testNonEmpty(dtype, adjoint_a, adjoint_b, use_static_shape)
-    self._testEmpty(dtype, adjoint_a, adjoint_b, use_static_shape)
+    self._testNonEmpty(dtype, adjoint_a, adjoint_b)
+    self._testEmpty(dtype, adjoint_a, adjoint_b)
 
   return Test
 
@@ -154,17 +145,13 @@ class BatchMatmulGradientTest(test.TestCase):
     y = y_in if not adjoint_b else y_in.reshape(y_t_shape)
     epsilon = np.finfo(x.dtype).eps
     delta = epsilon**(1.0 / 3.0)
-    with self.cached_session(use_gpu=True):
-      inx = constant_op.constant(x)
-      iny = constant_op.constant(y)
-      z = math_ops.matmul(inx, iny, adjoint_a, adjoint_b)
-      loss = math_ops.reduce_sum(z)
-      ((x_jacob_t, x_jacob_n),
-       (y_jacob_t, y_jacob_n)) = gradient_checker.compute_gradient(
-           [inx, iny], [x.shape, y.shape],
-           loss, [1],
-           x_init_value=[x, y],
-           delta=delta)
+    def Loss(x, y):
+      z = math_ops.matmul(x, y, adjoint_a, adjoint_b)
+      return math_ops.reduce_sum(z)
+    with self.session(use_gpu=True):
+      ((x_jacob_t, y_jacob_t),
+       (x_jacob_n, y_jacob_n)) = gradient_checker_v2.compute_gradient(
+           Loss, [x, y], delta=delta)
       tol = 20 * delta
       self.assertAllClose(x_jacob_t, x_jacob_n, rtol=tol, atol=tol)
       self.assertAllClose(y_jacob_t, y_jacob_n, rtol=tol, atol=tol)
@@ -202,11 +189,9 @@ if __name__ == "__main__":
     for adjoint_a_ in False, True:
       for adjoint_b_ in False, True:
         name = "%s_%s_%s" % (dtype_.__name__, adjoint_a_, adjoint_b_)
-        for use_static_shape in True, False:
-          setattr(BatchMatmulOpTest,
-                  "testBatchMatmulOp_" + name + ("_%s" % use_static_shape),
-                  _GetBatchMatmulOpTest(dtype_, adjoint_a_, adjoint_b_,
-                                        use_static_shape))
+        setattr(BatchMatmulOpTest,
+                "testBatchMatmulOp_" + name,
+                _GetBatchMatmulOpTest(dtype_, adjoint_a_, adjoint_b_))
         if dtype_ is not np.int32:
           setattr(BatchMatmulGradientTest, "testBatchMatmulGradient_" + name,
                   _GetBatchMatmulGradientTest(dtype_, adjoint_a_, adjoint_b_))
-- 
GitLab


From 84069b7bf0ea49724feb09b77d989a3a6f557e44 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Mon, 3 Dec 2018 14:38:11 -0800
Subject: [PATCH 1233/1554] Adds tf.lite.constants symbols to the converter.

PiperOrigin-RevId: 223864922
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 4fb2a3f4d6..21cbf7664c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -453,6 +453,16 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.losses.Reduction.SUM_BY_NONZERO_WEIGHTS",
         "tf.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS":
             "tf.compat.v1.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS",
+        "tf.lite.constants.FLOAT":
+            "tf.float32",
+        "tf.lite.constants.INT32":
+            "tf.int32",
+        "tf.lite.constants.INT64":
+            "tf.int64",
+        "tf.lite.constants.STRING":
+            "tf.string",
+        "tf.lite.constants.QUANTIZED_UINT8":
+            "tf.uint8",
     }
     # pylint: enable=line-too-long
 
-- 
GitLab


From 38e7c9ff27b175b02626aff8a29d7fd5258fdfce Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 3 Dec 2018 14:48:57 -0800
Subject: [PATCH 1234/1554] Add V2 APIs for cross entropy losses.

PiperOrigin-RevId: 223867047
---
 tensorflow/python/keras/losses.py             | 22 ++++---
 tensorflow/python/keras/losses_test.py        | 66 +++++++++----------
 ...ow.keras.losses.-binary-crossentropy.pbtxt | 22 +++++++
 ...ras.losses.-categorical-crossentropy.pbtxt | 22 +++++++
 .../golden/v1/tensorflow.keras.losses.pbtxt   |  8 +++
 ...ow.keras.losses.-binary-crossentropy.pbtxt | 22 +++++++
 ...ras.losses.-categorical-crossentropy.pbtxt | 22 +++++++
 .../golden/v2/tensorflow.keras.losses.pbtxt   |  8 +++
 8 files changed, 149 insertions(+), 43 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 9600ae9a69..4c584d0ff0 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -269,13 +269,14 @@ class MeanSquaredLogarithmicError(Loss):
     return mean_squared_logarithmic_error(y_true, y_pred)
 
 
-class BinaryCrossEntropy(Loss):
+@tf_export('keras.losses.BinaryCrossentropy')
+class BinaryCrossentropy(Loss):
   """Computes the binary cross entropy loss between the labels and predictions.
 
   Usage:
 
   ```python
-  bce = tf.keras.losses.BinaryCrossEntropy()
+  bce = tf.keras.losses.BinaryCrossentropy()
   loss = bce([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Loss: ', loss.numpy())  # Loss: 12.007
   ```
@@ -284,7 +285,7 @@ class BinaryCrossEntropy(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.BinaryCrossEntropy())
+  model.compile('sgd', loss=tf.keras.losses.BinaryCrossentropy())
   ````
 
   Args:
@@ -301,12 +302,12 @@ class BinaryCrossEntropy(Loss):
                label_smoothing=0,
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
-    super(BinaryCrossEntropy, self).__init__(reduction=reduction, name=name)
+    super(BinaryCrossentropy, self).__init__(reduction=reduction, name=name)
     self.from_logits = from_logits
     self.label_smoothing = label_smoothing
 
   def call(self, y_true, y_pred):
-    """Invokes the `BinaryCrossEntropy` instance.
+    """Invokes the `BinaryCrossentropy` instance.
 
     Args:
       y_true: Ground truth values.
@@ -324,13 +325,14 @@ class BinaryCrossEntropy(Loss):
     return binary_crossentropy(y_true, y_pred, from_logits=self.from_logits)
 
 
-class CategoricalCrossEntropy(Loss):
+@tf_export('keras.losses.CategoricalCrossentropy')
+class CategoricalCrossentropy(Loss):
   """Computes categorical cross entropy loss between the `y_true` and `y_pred`.
 
   Usage:
 
   ```python
-  cce = tf.keras.losses.CategoricalCrossEntropy()
+  cce = tf.keras.losses.CategoricalCrossentropy()
   loss = cce(
     [[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
     [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]])
@@ -341,7 +343,7 @@ class CategoricalCrossEntropy(Loss):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.CategoricalCrossEntropy())
+  model.compile('sgd', loss=tf.keras.losses.CategoricalCrossentropy())
   ````
 
   Args:
@@ -359,13 +361,13 @@ class CategoricalCrossEntropy(Loss):
                label_smoothing=0,
                reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
                name=None):
-    super(CategoricalCrossEntropy, self).__init__(
+    super(CategoricalCrossentropy, self).__init__(
         reduction=reduction, name=name)
     self.from_logits = from_logits
     self.label_smoothing = label_smoothing
 
   def call(self, y_true, y_pred):
-    """Invokes the `CategoricalCrossEntropy` instance.
+    """Invokes the `CategoricalCrossentropy` instance.
 
     Args:
       y_true: Ground truth values.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index a1349b4af4..d2791cdcd3 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -501,10 +501,10 @@ class CosineProximityTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class BinaryCrossEntropyTest(test.TestCase):
+class BinaryCrossentropyTest(test.TestCase):
 
   def test_config(self):
-    bce_obj = keras.losses.BinaryCrossEntropy(
+    bce_obj = keras.losses.BinaryCrossentropy(
         reduction=losses_impl.ReductionV2.SUM, name='bce_1')
     self.assertEqual(bce_obj.name, 'bce_1')
     self.assertEqual(bce_obj.reduction, losses_impl.ReductionV2.SUM)
@@ -512,7 +512,7 @@ class BinaryCrossEntropyTest(test.TestCase):
   def test_all_correct_unweighted(self):
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]],
                                   dtype=dtypes.float32)
-    bce_obj = keras.losses.BinaryCrossEntropy()
+    bce_obj = keras.losses.BinaryCrossentropy()
     loss = bce_obj(y_true, y_true)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
@@ -520,12 +520,12 @@ class BinaryCrossEntropyTest(test.TestCase):
     logits = constant_op.constant([[100.0, -100.0, -100.0],
                                    [-100.0, 100.0, -100.0],
                                    [-100.0, -100.0, 100.0]])
-    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
   def test_unweighted(self):
-    bce_obj = keras.losses.BinaryCrossEntropy()
+    bce_obj = keras.losses.BinaryCrossentropy()
     y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
     y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
                                   shape=(2, 3),
@@ -537,12 +537,12 @@ class BinaryCrossEntropyTest(test.TestCase):
     logits = constant_op.constant([10., 10., 10., -10., 10, -10],
                                   shape=(2, 3),
                                   dtype=dtypes.float32)
-    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), 5., 3)
 
   def test_scalar_weighted(self):
-    bce_obj = keras.losses.BinaryCrossEntropy()
+    bce_obj = keras.losses.BinaryCrossentropy()
     y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
     y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
                                   shape=(2, 3),
@@ -553,12 +553,12 @@ class BinaryCrossEntropyTest(test.TestCase):
     # Test with logits.
     y_true = array_ops.ones((32, 1))
     logits = array_ops.ones((32, 1), dtype=dtypes.float32)
-    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits, sample_weight=2.3)
     self.assertAlmostEqual(self.evaluate(loss), 0.7205, 3)
 
   def test_sample_weighted(self):
-    bce_obj = keras.losses.BinaryCrossEntropy()
+    bce_obj = keras.losses.BinaryCrossentropy()
     y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
     y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
                                   shape=(2, 3),
@@ -574,7 +574,7 @@ class BinaryCrossEntropyTest(test.TestCase):
          [-100.0, -100.0, 100.0]],
         dtype=dtypes.float64)
     weights = constant_op.constant([3, 2, 8])
-    bce_obj = keras.losses.BinaryCrossEntropy(from_logits=True)
+    bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits, sample_weight=weights)
     self.assertAlmostEqual(self.evaluate(loss), 288.8888, 3)
 
@@ -583,7 +583,7 @@ class BinaryCrossEntropyTest(test.TestCase):
     logits = constant_op.constant(((100.0, -100.0, 100.0),
                                    (100.0, -100.0, 100.0),
                                    (100.0, 100.0, -100.0)))
-    bce_obj = keras.losses.BinaryCrossEntropy(
+    bce_obj = keras.losses.BinaryCrossentropy(
         from_logits=True, reduction=losses_impl.ReductionV2.NONE)
     loss = bce_obj(y_true, logits)
     self.assertAllClose((0., 66.6666, 66.6666), self.evaluate(loss), 3)
@@ -601,7 +601,7 @@ class BinaryCrossEntropyTest(test.TestCase):
     #  0   + 100 * (0.5 L)      + 0 +
     #  0   + 100 * (1 - 0.5 L)  + 0) * (1/3)
     #  = (100 + 50L) * 1/3
-    bce_obj = keras.losses.BinaryCrossEntropy(
+    bce_obj = keras.losses.BinaryCrossentropy(
         from_logits=True, label_smoothing=label_smoothing)
     loss = bce_obj(y_true, logits)
     expected_value = (100.0 + 50.0 * label_smoothing) / 3.0
@@ -609,10 +609,10 @@ class BinaryCrossEntropyTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class CategoricalCrossEntropyTest(test.TestCase):
+class CategoricalCrossentropyTest(test.TestCase):
 
   def test_config(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy(
+    cce_obj = keras.losses.CategoricalCrossentropy(
         reduction=losses_impl.ReductionV2.SUM, name='bce_1')
     self.assertEqual(cce_obj.name, 'bce_1')
     self.assertEqual(cce_obj.reduction, losses_impl.ReductionV2.SUM)
@@ -622,18 +622,18 @@ class CategoricalCrossEntropyTest(test.TestCase):
                                   dtype=dtypes.int64)
     y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
                                   dtype=dtypes.float32)
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     loss = cce_obj(y_true, y_pred)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
     # Test with logits.
     logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
   def test_unweighted(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -642,12 +642,12 @@ class CategoricalCrossEntropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), .0573, 3)
 
   def test_scalar_weighted(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -656,12 +656,12 @@ class CategoricalCrossEntropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits, sample_weight=2.3)
     self.assertAlmostEqual(self.evaluate(loss), .1317, 3)
 
   def test_sample_weighted(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -671,14 +671,14 @@ class CategoricalCrossEntropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits, sample_weight=sample_weight)
     self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3)
 
   def test_no_reduction(self):
     y_true = constant_op.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(
+    cce_obj = keras.losses.CategoricalCrossentropy(
         from_logits=True, reduction=losses_impl.ReductionV2.NONE)
     loss = cce_obj(y_true, logits)
     self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
@@ -699,7 +699,7 @@ class CategoricalCrossEntropyTest(test.TestCase):
     #                  0  = L/n
     # Applying the above two fns to the given input:
     # -0 * (1 - L + L/n) + 200 * L/n + 200 * L/n = 400 L/n
-    cce_obj = keras.losses.CategoricalCrossEntropy(
+    cce_obj = keras.losses.CategoricalCrossentropy(
         from_logits=True, label_smoothing=label_smoothing)
     loss = cce_obj(y_true, logits)
     expected_value = 400.0 * label_smoothing / 3.0
@@ -709,18 +709,18 @@ class CategoricalCrossEntropyTest(test.TestCase):
     y_true = constant_op.constant([[0], [1], [2]], dtype=dtypes.int64)
     y_pred = constant_op.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]],
                                   dtype=dtypes.float32)
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     loss = cce_obj(y_true, y_pred)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
     # Test with logits.
     logits = constant_op.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
   def test_unweighted_sparse(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     y_true = constant_op.constant([0, 1, 2])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -729,12 +729,12 @@ class CategoricalCrossEntropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits)
     self.assertAlmostEqual(self.evaluate(loss), .0573, 3)
 
   def test_scalar_weighted_sparse(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     y_true = constant_op.constant([[0], [1], [2]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -743,12 +743,12 @@ class CategoricalCrossEntropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits, sample_weight=2.3)
     self.assertAlmostEqual(self.evaluate(loss), .1317, 3)
 
   def test_sample_weighted_sparse(self):
-    cce_obj = keras.losses.CategoricalCrossEntropy()
+    cce_obj = keras.losses.CategoricalCrossentropy()
     y_true = constant_op.constant([[0], [1], [2]])
     y_pred = constant_op.constant(
         [[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]], dtype=dtypes.float32)
@@ -758,14 +758,14 @@ class CategoricalCrossEntropyTest(test.TestCase):
 
     # Test with logits.
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(from_logits=True)
+    cce_obj = keras.losses.CategoricalCrossentropy(from_logits=True)
     loss = cce_obj(y_true, logits, sample_weight=sample_weight)
     self.assertAlmostEqual(self.evaluate(loss), 0.31829, 3)
 
   def test_no_reduction_sparse(self):
     y_true = constant_op.constant([[0], [1], [2]])
     logits = constant_op.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
-    cce_obj = keras.losses.CategoricalCrossEntropy(
+    cce_obj = keras.losses.CategoricalCrossentropy(
         from_logits=True, reduction=losses_impl.ReductionV2.NONE)
     loss = cce_obj(y_true, logits)
     self.assertAllClose((0.001822, 0.000459, 0.169846), self.evaluate(loss), 3)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt
new file mode 100644
index 0000000000..2f7da93f6f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-binary-crossentropy.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.BinaryCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..b3a7cd8097
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.CategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index 1cc472b436..9e26ddbdca 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -1,5 +1,13 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "BinaryCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanAbsoluteError"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt
new file mode 100644
index 0000000000..2f7da93f6f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-binary-crossentropy.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.BinaryCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.BinaryCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
new file mode 100644
index 0000000000..b3a7cd8097
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-categorical-crossentropy.pbtxt
@@ -0,0 +1,22 @@
+path: "tensorflow.keras.losses.CategoricalCrossentropy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.losses.CategoricalCrossentropy\'>"
+  is_instance: "<class \'tensorflow.python.keras.losses.Loss\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'from_logits\', \'label_smoothing\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'0\', \'sum_over_batch_size\', \'None\'], "
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index 6b330c2c1a..c198096d25 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -1,5 +1,13 @@
 path: "tensorflow.keras.losses"
 tf_module {
+  member {
+    name: "BinaryCrossentropy"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CategoricalCrossentropy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "MeanAbsoluteError"
     mtype: "<type \'type\'>"
-- 
GitLab


From b288a1bec52302830ff068c2ef34418a2894e7dc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 14:56:41 -0800
Subject: [PATCH 1235/1554] Automated rollback of commit
 54b5a2163bc2c5a13db8de39fc99ae558fc854a4

PiperOrigin-RevId: 223868582
---
 .../kernel_tests/batch_matmul_op_test.py      | 91 +++++++++++--------
 1 file changed, 53 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
index aebbc81f8f..a0ad8151b2 100644
--- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
@@ -20,8 +20,9 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import gradient_checker_v2
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -75,18 +76,25 @@ class BatchMatmulOpTest(test.TestCase):
 
   # Compares _tfpBatchMatmul(x, y, alpha, adj) and _npBatchMatMul(x, y, alpha,
   # adj)
-  def _compare(self, x_in, y_in, adjoint_a, adjoint_b):
+  def _compare(self, x_in, y_in, adjoint_a, adjoint_b, static_shape=True):
     x_t_shape = x_in.shape[:-2] + (x_in.shape[-1], x_in.shape[-2])
     y_t_shape = y_in.shape[:-2] + (y_in.shape[-1], y_in.shape[-2])
     x = x_in if not adjoint_a else x_in.reshape(x_t_shape)
     y = y_in if not adjoint_b else y_in.reshape(y_t_shape)
     is_floating = x.dtype != np.int32
     tol = 100 * np.finfo(x.dtype).eps if is_floating else 0
-    with test_util.device(use_gpu=is_floating):
-      z0 = math_ops.matmul(
-          x, y, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
+    with self.cached_session(use_gpu=is_floating) as sess:
+      if static_shape:
+        z0 = math_ops.matmul(x, y, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
+        z0_val = self.evaluate(z0)
+      else:
+        x_ph = array_ops.placeholder(x.dtype)
+        y_ph = array_ops.placeholder(y.dtype)
+        z0 = math_ops.matmul(
+            x_ph, y_ph, adjoint_a=adjoint_a, adjoint_b=adjoint_b)
+        z0_val = sess.run(z0, feed_dict={x_ph: x, y_ph: y})
       z1 = self._npBatchMatmul(x, y, adjoint_a, adjoint_b)
-      self.assertAllClose(z0, z1, rtol=tol, atol=tol)
+      self.assertAllClose(z0_val, z1, rtol=tol, atol=tol)
 
   def _rand(self, shape, dtype):
     vals = np.array(np.random.normal(-10, 10, np.prod(shape)), dtype=dtype)
@@ -95,41 +103,42 @@ class BatchMatmulOpTest(test.TestCase):
       vals += 1j * imag
     return vals.reshape(shape)
 
-  def _testNonEmpty(self, dtype, adjoint_a, adjoint_b):
+  def _testNonEmpty(self, dtype, adjoint_a, adjoint_b, use_static_shape):
 
-    def CompareNonEmpty(self, a_shape, b_shape):
+    def compareNonEmpty(self, a_shape, b_shape):
       self._compare(
           self._rand(a_shape, dtype),
-          self._rand(b_shape, dtype), adjoint_a, adjoint_b)
+          self._rand(b_shape, dtype), adjoint_a, adjoint_b, use_static_shape)
 
-    CompareNonEmpty(self, [1, 2, 3], [1, 3, 5])
-    CompareNonEmpty(self, [1, 2, 3], [1, 3, 1])
-    CompareNonEmpty(self, [1, 1, 3], [1, 3, 5])
-    CompareNonEmpty(self, [1, 2, 3], [1, 3, 5])
-    CompareNonEmpty(self, [7, 1, 3], [7, 3, 5])
-    CompareNonEmpty(self, [7, 2, 3], [7, 3, 1])
-    CompareNonEmpty(self, [7, 2, 3], [7, 3, 5])
-    CompareNonEmpty(self, [10, 64, 75], [10, 75, 30])
-    CompareNonEmpty(self, [5, 7, 2, 3], [5, 7, 3, 5])
+    compareNonEmpty(self, [1, 2, 3], [1, 3, 5])
+    compareNonEmpty(self, [1, 2, 3], [1, 3, 1])
+    compareNonEmpty(self, [1, 1, 3], [1, 3, 5])
+    compareNonEmpty(self, [1, 2, 3], [1, 3, 5])
+    compareNonEmpty(self, [7, 1, 3], [7, 3, 5])
+    compareNonEmpty(self, [7, 2, 3], [7, 3, 1])
+    compareNonEmpty(self, [7, 2, 3], [7, 3, 5])
+    compareNonEmpty(self, [10, 64, 75], [10, 75, 30])
+    compareNonEmpty(self, [5, 7, 2, 3], [5, 7, 3, 5])
 
-  def _testEmpty(self, dtype, adjoint_a, adjoint_b):
+  def _testEmpty(self, dtype, adjoint_a, adjoint_b, use_static_shape):
 
-    def CompareEmpty(self, a_shape, b_shape):
+    def compareEmpty(self, a_shape, b_shape):
       self._compare(
           np.zeros(a_shape).astype(dtype),
-          np.zeros(b_shape).astype(dtype), adjoint_a, adjoint_b)
+          np.zeros(b_shape).astype(dtype), adjoint_a, adjoint_b,
+          use_static_shape)
 
-    CompareEmpty(self, [0, 3, 2], [0, 2, 4])
-    CompareEmpty(self, [3, 0, 2], [3, 2, 5])
-    CompareEmpty(self, [3, 3, 2], [3, 2, 0])
+    compareEmpty(self, [0, 3, 2], [0, 2, 4])
+    compareEmpty(self, [3, 0, 2], [3, 2, 5])
+    compareEmpty(self, [3, 3, 2], [3, 2, 0])
 
 
-def _GetBatchMatmulOpTest(dtype, adjoint_a, adjoint_b):
+def _GetBatchMatmulOpTest(dtype, adjoint_a, adjoint_b, use_static_shape):
 
   def Test(self):
     np.random.seed(42)
-    self._testNonEmpty(dtype, adjoint_a, adjoint_b)
-    self._testEmpty(dtype, adjoint_a, adjoint_b)
+    self._testNonEmpty(dtype, adjoint_a, adjoint_b, use_static_shape)
+    self._testEmpty(dtype, adjoint_a, adjoint_b, use_static_shape)
 
   return Test
 
@@ -145,13 +154,17 @@ class BatchMatmulGradientTest(test.TestCase):
     y = y_in if not adjoint_b else y_in.reshape(y_t_shape)
     epsilon = np.finfo(x.dtype).eps
     delta = epsilon**(1.0 / 3.0)
-    def Loss(x, y):
-      z = math_ops.matmul(x, y, adjoint_a, adjoint_b)
-      return math_ops.reduce_sum(z)
-    with self.session(use_gpu=True):
-      ((x_jacob_t, y_jacob_t),
-       (x_jacob_n, y_jacob_n)) = gradient_checker_v2.compute_gradient(
-           Loss, [x, y], delta=delta)
+    with self.cached_session(use_gpu=True):
+      inx = constant_op.constant(x)
+      iny = constant_op.constant(y)
+      z = math_ops.matmul(inx, iny, adjoint_a, adjoint_b)
+      loss = math_ops.reduce_sum(z)
+      ((x_jacob_t, x_jacob_n),
+       (y_jacob_t, y_jacob_n)) = gradient_checker.compute_gradient(
+           [inx, iny], [x.shape, y.shape],
+           loss, [1],
+           x_init_value=[x, y],
+           delta=delta)
       tol = 20 * delta
       self.assertAllClose(x_jacob_t, x_jacob_n, rtol=tol, atol=tol)
       self.assertAllClose(y_jacob_t, y_jacob_n, rtol=tol, atol=tol)
@@ -189,9 +202,11 @@ if __name__ == "__main__":
     for adjoint_a_ in False, True:
       for adjoint_b_ in False, True:
         name = "%s_%s_%s" % (dtype_.__name__, adjoint_a_, adjoint_b_)
-        setattr(BatchMatmulOpTest,
-                "testBatchMatmulOp_" + name,
-                _GetBatchMatmulOpTest(dtype_, adjoint_a_, adjoint_b_))
+        for use_static_shape in True, False:
+          setattr(BatchMatmulOpTest,
+                  "testBatchMatmulOp_" + name + ("_%s" % use_static_shape),
+                  _GetBatchMatmulOpTest(dtype_, adjoint_a_, adjoint_b_,
+                                        use_static_shape))
         if dtype_ is not np.int32:
           setattr(BatchMatmulGradientTest, "testBatchMatmulGradient_" + name,
                   _GetBatchMatmulGradientTest(dtype_, adjoint_a_, adjoint_b_))
-- 
GitLab


From 3661ae91df08852ebb49db0463a21628026fcdcc Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Mon, 3 Dec 2018 15:15:55 -0800
Subject: [PATCH 1236/1554] Use session config when creating XLA devices

---
 tensorflow/compiler/jit/xla_gpu_device.cc | 31 +++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 944f732b99..fb13635414 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 
@@ -52,8 +53,34 @@ Status XlaGpuDeviceFactory::CreateDevices(
     VLOG(1) << "Failed to create XLA_GPU device: " << platform.status();
     return Status::OK();
   }
-
-  for (int i = 0; i < platform.ValueOrDie()->VisibleDeviceCount(); ++i) {
+  const auto& allowed_gpus =
+      session_options.config.gpu_options().visible_device_list();
+  std::unordered_set<int> gpu_ids;
+  int num_visible_devices = platform.ValueOrDie()->VisibleDeviceCount();
+  if (allowed_gpus.empty()) {
+    for (int i = 0; i < num_visible_devices; ++i) gpu_ids.insert(i);
+  } else {
+    const std::vector<string> visible_devices =
+        str_util::Split(allowed_gpus, ',');
+    // copied from gpu/gpu_device.cc Should be redundant since code would fail
+    // there before it gets to here.
+    for (const string& platform_gpu_id_str : visible_devices) {
+      int32 platform_gpu_id;
+      if (!strings::safe_strto32(platform_gpu_id_str, &platform_gpu_id)) {
+        return errors::InvalidArgument(
+            "Could not parse entry in 'visible_device_list': '",
+            platform_gpu_id_str, "'. visible_device_list = ", allowed_gpus);
+      }
+      if (platform_gpu_id < 0 || platform_gpu_id >= num_visible_devices) {
+        return errors::InvalidArgument(
+            "'visible_device_list' listed an invalid GPU id '", platform_gpu_id,
+            "' but visible device count is ", num_visible_devices);
+      }
+      gpu_ids.insert(platform_gpu_id);
+    }
+  }
+  for (int i = 0; i < num_visible_devices; ++i) {
+    if (gpu_ids.count(i) == 0) continue;
     XlaDevice::Options options;
     options.platform = platform.ValueOrDie();
     options.device_name_prefix = name_prefix;
-- 
GitLab


From 50204e4205f3c613ddd6a3ce2ba14f60d6aaaf80 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 3 Dec 2018 15:51:06 -0800
Subject: [PATCH 1237/1554] Run placer for the inlined function body.

PiperOrigin-RevId: 223878376
---
 .../common_runtime/graph_execution_state.cc   |   7 ++
 tensorflow/core/grappler/BUILD                |   3 +
 tensorflow/core/grappler/grappler_item.cc     |  64 +++++++++-
 tensorflow/core/grappler/grappler_item.h      |  28 ++++-
 .../core/grappler/grappler_item_test.cc       |  28 +++++
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../optimizers/arithmetic_optimizer.cc        |   2 +-
 .../grappler/optimizers/function_optimizer.cc | 110 ++++++++++++++++--
 .../optimizers/function_optimizer_test.cc     |  53 +++++++++
 .../grappler/optimizers/meta_optimizer.cc     |   8 +-
 .../optimizers/meta_optimizer_test.cc         |   2 +-
 .../core/grappler/utils/grappler_test.cc      |   2 +
 12 files changed, 291 insertions(+), 17 deletions(-)

diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 9a56c67162..880806f120 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -595,6 +595,13 @@ Status GraphExecutionState::OptimizeGraph(
     grappler::GrapplerItem item;
     item.id = "tf_graph";
     graph_->ToGraphDef(&item.graph);
+
+    // It's ok to skip invalid device annotations in Grappler.
+    Status inferred_devices = item.InferDevicesFromGraph();
+    if (!inferred_devices.ok()) {
+      VLOG(3) << inferred_devices.error_message();
+    }
+
     // TODO(b/114748242): Add a unit test to test this bug fix.
     if (flib_def_) {
       *item.graph.mutable_library() = flib_def_->ToProto();
diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 7982b35853..f353d789d4 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -107,6 +107,8 @@ cc_library(
         ":utils",
         "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
     ],
 )
 
@@ -141,6 +143,7 @@ tf_cc_test(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
     ],
 )
diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index 2c490f3966..f7cda35368 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -19,10 +19,13 @@ limitations under the License.
 #include <unordered_set>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/str_join.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -38,7 +41,8 @@ GrapplerItem::GrapplerItem(const GrapplerItem& other, GraphDef* graph_def) {
   restore_op = other.restore_op;
   save_restore_loc_tensor = other.save_restore_loc_tensor;
   queue_runners = other.queue_runners;
-  allowed_optimizations = other.allowed_optimizations;
+  devices_ = other.devices_;
+  allowed_optimizations_ = other.allowed_optimizations_;
   graph.Swap(graph_def);
 }
 
@@ -111,6 +115,64 @@ std::unordered_set<string> GrapplerItem::NodesToPreserve() const {
   return result;
 }
 
+const std::unordered_set<string>& GrapplerItem::devices() const {
+  return devices_;
+}
+
+Status GrapplerItem::AddDevice(const string& device) {
+  DeviceNameUtils::ParsedName name;
+
+  if (!DeviceNameUtils::ParseFullName(device, &name)) {
+    return errors::InvalidArgument("Invalid device name: device=", device);
+
+  } else if (!name.has_job || !name.has_replica || !name.has_task ||
+             !name.has_type || !name.has_id) {
+    return errors::InvalidArgument("Not a fully defined device name: device=",
+                                   device);
+  }
+
+  devices_.insert(DeviceNameUtils::ParsedNameToString(name));
+  return Status::OK();
+}
+
+Status GrapplerItem::AddDevices(const GrapplerItem& other) {
+  std::vector<absl::string_view> invalid_devices;
+  for (const string& device : other.devices()) {
+    Status added = AddDevice(device);
+    if (!added.ok()) invalid_devices.emplace_back(device);
+  }
+  return invalid_devices.empty()
+             ? Status::OK()
+             : errors::InvalidArgument("Skipped invalid devices: [",
+                                       absl::StrJoin(invalid_devices, ", "),
+                                       "]");
+}
+
+Status GrapplerItem::InferDevicesFromGraph() {
+  absl::flat_hash_set<absl::string_view> invalid_devices;
+  for (const NodeDef& node : graph.node()) {
+    Status added = AddDevice(node.device());
+    if (!added.ok()) invalid_devices.insert(node.device());
+  }
+  VLOG(2) << "Inferred device set: [" << absl::StrJoin(devices_, ", ") << "]";
+  return invalid_devices.empty()
+             ? Status::OK()
+             : errors::InvalidArgument("Skipped invalid devices: [",
+                                       absl::StrJoin(invalid_devices, ", "),
+                                       "]");
+}
+
+void GrapplerItem::ClearDevices() { devices_.clear(); }
+
+const GrapplerItem::AllowedOptimizations& GrapplerItem::allowed_optimizations()
+    const {
+  return allowed_optimizations_;
+}
+
+GrapplerItem::AllowedOptimizations& GrapplerItem::allowed_optimizations() {
+  return allowed_optimizations_;
+}
+
 std::vector<const NodeDef*> ComputeTransitiveFanin(
     const GraphDef& graph, const std::vector<string>& terminal_nodes) {
   bool ill_formed = false;
diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h
index a0748abfe6..aea7d27792 100644
--- a/tensorflow/core/grappler/grappler_item.h
+++ b/tensorflow/core/grappler/grappler_item.h
@@ -85,7 +85,33 @@ struct GrapplerItem {
     bool non_differentiable_rewrites = true;
   };
 
-  AllowedOptimizations allowed_optimizations;
+  const std::unordered_set<string>& devices() const;
+  // Adds a device to a set of available devices, only if it's a valid fully
+  // defined device name. Returns `Status::OK()` if successfully added a device,
+  // and an error otherwise.
+  Status AddDevice(const string& device);
+  // Adds all valid devices from the other Grappler item to the device set.
+  Status AddDevices(const GrapplerItem& other);
+  // Adds all valid devices from the nodes of the graph to the device set.
+  // Returns `Status::OK()` if all device annotations found in a graph are valid
+  // fully defined device names, and an error otherwise.
+  Status InferDevicesFromGraph();
+  // Clears a set of available devices.
+  void ClearDevices();
+
+  const AllowedOptimizations& allowed_optimizations() const;
+  AllowedOptimizations& allowed_optimizations();
+
+ private:
+  // TODO(ezhulenev) Make GrapplerItem a class and hide all public data members.
+  // TODO(ezhulenev): Migrate all unordered collections to absl.
+
+  // A set of fully defined device names that can be used to place the nodes of
+  // the `graph`.
+  // Example of a fully defined name: "/job:work/replica:1/task:1/device:CPU:0"
+  std::unordered_set<string> devices_;
+
+  AllowedOptimizations allowed_optimizations_;
 };
 
 // Return the transitive fanin of a set of terminal nodes.
diff --git a/tensorflow/core/grappler/grappler_item_test.cc b/tensorflow/core/grappler/grappler_item_test.cc
index 72a9f481ca..a8fbe35682 100644
--- a/tensorflow/core/grappler/grappler_item_test.cc
+++ b/tensorflow/core/grappler/grappler_item_test.cc
@@ -14,7 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -44,6 +46,32 @@ TEST_F(GrapplerItemTest, Basic) {
   EXPECT_EQ(main_ops, graph_nodes);
 }
 
+TEST_F(GrapplerItemTest, InferDevices) {
+  using test::function::NDef;
+
+  const string cpu0 = "/job:work/replica:1/task:1/device:CPU:0";
+  const string cpu1 = "/job:work/replica:1/task:1/device:CPU:1";
+  const string cpu2 = "/device:CPU:2";
+
+  GrapplerItem item;
+  item.graph = test::function::GDef(
+      {
+          NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu0),
+          NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu1),
+          NDef("c", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu2),
+      },
+      {} /* Empty function library */);
+
+  ASSERT_FALSE(item.InferDevicesFromGraph().ok());
+
+  EXPECT_EQ(item.devices().size(), 2);
+  EXPECT_NE(item.devices().find(cpu0), item.devices().end());
+  EXPECT_NE(item.devices().find(cpu1), item.devices().end());
+
+  item.ClearDevices();
+  EXPECT_EQ(item.devices().size(), 0);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 68f70bf0df..40eab8b9f0 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -141,6 +141,7 @@ cc_library(
     deps = [
         ":graph_optimizer",
         "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:core_cpu_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index e3ac89b50d..e41b1cf684 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -3572,7 +3572,7 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
 
   // Disable restricted graph rewrites.
   options_.unary_ops_composition &=
-      item.allowed_optimizations.non_differentiable_rewrites;
+      item.allowed_optimizations().non_differentiable_rewrites;
 
   if (options_.dedup_computations) {
     DedupComputations();
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 3ffe8f1a0b..2dd4ff10e4 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -22,8 +22,11 @@ limitations under the License.
 #include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
+#include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/function.h"
@@ -111,10 +114,26 @@ AttrSlice FunctionInstantiationAttributes(const FunctionDef& func,
   }
 }
 
-class FakeCPUDevice : public Device {
+// This is a fake device that should not be used for any op kernel execution,
+// the only purpose of this device is to be passed as a part of DeviceSet to the
+// Placer.
+class FakeDevice : public Device {
  public:
-  FakeCPUDevice(Env* env, const DeviceAttributes& attr) : Device(env, attr) {}
+  FakeDevice(Env* env, const string& device) : Device(env, attr(device)) {}
+  explicit FakeDevice(const string& device) : FakeDevice(nullptr, device) {}
   Status Sync() override { return Status::OK(); }
+
+ private:
+  static DeviceAttributes attr(const string& device) {
+    DeviceNameUtils::ParsedName parsed_name;
+    bool parsed = DeviceNameUtils::ParseFullName(device, &parsed_name);
+    DCHECK(parsed) << "Failed to parse full device name: " << device;
+
+    DeviceAttributes attr;
+    attr.set_name(device);
+    attr.set_device_type(parsed_name.type);
+    return attr;
+  }
 };
 
 // -------------------------------------------------------------------------- //
@@ -240,6 +259,7 @@ class FunctionOptimizerContext {
         graph_version_(item.graph.versions().producer()),
         opt_level_(opt_level),
         function_library_(OpRegistry::Global(), item.graph.library()),
+        available_device_names_(item.devices().begin(), item.devices().end()),
         graph_view_(&item.graph) {
     InitializeTrulyConstNodes(item);
     InitializeFetchNodes(item);
@@ -275,6 +295,18 @@ class FunctionOptimizerContext {
 
   const gtl::FlatSet<string>& fetch_tensors() const { return fetch_tensors_; }
 
+  const DeviceSet* devices() const {
+    // Create fake devices lazily only if we need a DeviceSet.
+    if (available_devices_.empty() && !available_device_names_.empty()) {
+      for (const string& name : available_device_names_) {
+        auto device = absl::make_unique<FakeDevice>(name);
+        available_device_set_.AddDevice(device.get());
+        available_devices_.push_back(std::move(device));
+      }
+    }
+    return &available_device_set_;
+  }
+
   bool IsFetchNode(const string& node_name) const {
     return fetch_nodes_.find(node_name) != fetch_nodes_.end();
   }
@@ -350,11 +382,8 @@ class FunctionOptimizerContext {
   void InitializeFunctionLibraryRuntime() {
     if (!flr_) {
       Env* env = Env::Default();
-      DeviceAttributes attr;
-      attr.set_name("/device:CPU:0");
-      attr.set_device_type("CPU");
       std::vector<std::unique_ptr<Device>> devices;
-      devices.push_back(absl::make_unique<FakeCPUDevice>(env, attr));
+      devices.push_back(absl::make_unique<FakeDevice>(env, "/device:CPU:0"));
       device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
       OptimizerOptions optimizer_opts;
       optimizer_opts.set_do_function_inlining(true);
@@ -375,6 +404,16 @@ class FunctionOptimizerContext {
   std::unique_ptr<ProcessFunctionLibraryRuntime> process_flr_;
   FunctionLibraryRuntime* flr_ = nullptr;
 
+  // Fully defined names of the devices available to the GrapplerItem.
+  const gtl::FlatSet<string> available_device_names_;
+
+  // List of available `FakedDevices` (lazily initialized, see devices()).
+  mutable std::vector<std::unique_ptr<Device>> available_devices_;
+
+  // DeviceSet of fake devices (`FakeDevice`) constructed from
+  // available_devices_ (lazily initialized).
+  mutable DeviceSet available_device_set_;
+
   // Nodes that are Const and not in feed.
   std::unordered_map<string, const NodeDef*> truly_const_nodes_;
   // Specialized functions.
@@ -1305,7 +1344,58 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // edges from inlined side-effectful ops.
   std::vector<string> side_effectful_nodes;
 
-  for (NodeDef& func_body_node : *item.mutable_function_body().mutable_node()) {
+  // ------------------------------------------------------------------------ //
+  // First we need to assign device placements to all function body nodes.
+
+  GraphDef placed_graph_def;
+
+  const DeviceSet* devices = ctx->devices();
+
+  if (devices->devices().empty()) {
+    // If there are no devices available for placer, we just put all nodes to
+    // the same device as a function caller node. This can happen if Grappler is
+    // running "offline", without active runtime session, for example as a part
+    // of a batch job for graph analysis/optimization.
+    VLOG(3) << "Assign function call node device to all function body nodes. "
+            << "Device: " << func_node.device();
+    placed_graph_def = item.mutable_function_body();
+    for (NodeDef& node : *placed_graph_def.mutable_node()) {
+      node.set_device(func_node.device());
+    }
+  } else {
+    // If we are running in an active runtime session, Grappler will get the
+    // graph after initial placing is done, and we should have devices for the
+    // placer.
+    VLOG(3) << "Run placer for instantiated function body. Devices: ["
+            << absl::StrJoin(
+                   devices->devices(), ", ",
+                   [](string* out, const Device* d) { out->append(d->name()); })
+            << "]";
+
+    // Construct a Graph object from the instantiated function body.
+    GraphConstructorOptions opts;
+    Graph graph(ctx->function_library());
+    TF_RETURN_IF_ERROR(
+        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
+
+    // Use function caller node device as a default for placer.
+    const Device* default_device =
+        devices->FindDeviceByName(func_node.device());
+
+    Placer placer(&graph, devices, nullptr, /* No session options */
+                  default_device);
+    TF_RETURN_IF_ERROR(placer.Run());
+
+    // Convert Graph back to the GraphDef.
+    graph.ToGraphDef(&placed_graph_def);
+  }
+
+  // ------------------------------------------------------------------------ //
+  // After all nodes placed we need to prepare them for inlining into the
+  // optimized graph: turn placeholders into identities, update nodes
+  // connectivity, etc...
+
+  for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
     if (item.IsInputPlaceholder(func_body_node.name())) {
       // Turn input placeholders into identity node.
       DCHECK_EQ(0, func_body_node.input_size());
@@ -1337,10 +1427,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     TF_RETURN_IF_ERROR(
         AddPrefixAndSuffixToNode(prefix, /*suffix=*/"", &func_body_node));
 
-    // TODO(ezhulenev): Call PartitionedCallOp to get placement for all function
-    // body nodes, for now just place it on function caller node device.
-    func_body_node.set_device(func_node.device());
-
     // If the function body has a side-effectful op, we double check that the
     // function call node has an output control edge, otherwise we can't safely
     // do inlining and guarantee that node will be executed.
@@ -1362,7 +1448,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
     // TODO(ezhulenev): Inline nested indirect function calls.
 
-    // Move the node to the main graph.
+    // Move the node to the optimized graph.
     optimized_graph->add_node()->Swap(&func_body_node);
   }
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 98552e9449..93a2fcda7b 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -881,6 +881,59 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+  // Add device placement spec to the function body node.
+  (*mul_func.mutable_node_def())[0].set_device("/device:CPU:1");
+
+  // We need fully defined device names to run the placer for inlined function.
+  const string cpu0 = "/job:work/replica:1/task:1/device:CPU:0";
+  const string cpu1 = "/job:work/replica:1/task:1/device:CPU:1";
+
+  // Build a graph to compute c = MyMul(a, b)
+  GrapplerItem item;
+  item.fetch = {"d"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu0),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu1),
+       NDef("c", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}},
+            cpu0),
+       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, cpu0)},
+      // Function library.
+      {mul_func});
+  ASSERT_TRUE(item.InferDevicesFromGraph().ok());
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu0),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, cpu1),
+
+       // Function must be inlined and `mul` node placed on a requested device.
+       NDef("c/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, cpu1),
+       NDef("c/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, cpu1),
+       NDef("c/mul", "Mul", {"c/x", "c/y"}, {{"T", DT_FLOAT}}, cpu1),
+
+       NDef("d", "Identity", {"c/mul:0"}, {{"T", DT_FLOAT}}, cpu0)},
+      // Function library.
+      {mul_func});
+
+  CompareGraphs(expected, optimized_graph);
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 304ddc7710..5560e8d55f 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -524,7 +524,13 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       // can't perform non-differentiable rewrites.
       if (differentiable_functions.find(func_name) !=
           differentiable_functions.end()) {
-        func_item.allowed_optimizations.non_differentiable_rewrites = false;
+        func_item.allowed_optimizations().non_differentiable_rewrites = false;
+      }
+
+      // Function item is allowed to use all devices from the main graph.
+      Status added_devices = func_item.AddDevices(item);
+      if (!added_devices.ok()) {
+        VLOG(3) << added_devices.error_message();
       }
 
       // Optimize function body graph.
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index b60aa25667..42b867b6ac 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -108,7 +108,7 @@ class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
                   GraphDef* optimized_graph) override {
     *optimized_graph = item.graph;
     if (allowed_optimizations_) {
-      allowed_optimizations_->insert({item.id, item.allowed_optimizations});
+      allowed_optimizations_->insert({item.id, item.allowed_optimizations()});
     }
     return Status::OK();
   }
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 804fce5ce8..576494cad5 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -114,6 +114,8 @@ void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
   for (int i = 0; i < want.node_size(); ++i) {
     EXPECT_EQ(want.node(i).op(), got.node(i).op());
     EXPECT_EQ(want.node(i).name(), got.node(i).name());
+    EXPECT_EQ(want.node(i).device(), got.node(i).device());
+
     ASSERT_EQ(want.node(i).input_size(), got.node(i).input_size());
     for (int j = 0; j < want.node(i).input_size(); ++j) {
       const TensorId want_tensor = ParseTensorName(want.node(i).input(j));
-- 
GitLab


From 85e68a20257cdaf176d467fc8a91ba0f1e55462e Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 3 Dec 2018 15:56:28 -0800
Subject: [PATCH 1238/1554] [TF:XLA] Bump open source abseil revision to
 926bfeb9fff223429c12224b7514243886323e8d

PiperOrigin-RevId: 223879243
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7e5f84be16..55c3dd564a 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "3cf6132129ba87f0781c383bfaf381b7174b5818e81fffcc5d04bb451154f0f2",
-        strip_prefix = "abseil-cpp-f95179062eb65ce40895cc76f1398cce25394369",
+        sha256 = "e64ffc34ce6ba78946a6ea828c0ad96ac5bd162ec71c30f701c5cb3ebf8418a7",
+        strip_prefix = "abseil-cpp-926bfeb9fff223429c12224b7514243886323e8d",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/926bfeb9fff223429c12224b7514243886323e8d.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/926bfeb9fff223429c12224b7514243886323e8d.tar.gz",
         ],
     )
 
-- 
GitLab


From 1221a9f8ea7583646a0e5d664d6111d9be2e12d8 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 3 Dec 2018 15:58:00 -0800
Subject: [PATCH 1239/1554] [tf.data] Moving the experimental RandomDataset and
 ScanDataset C++ kernels to `experimental/` and removing both the Python and
 C++ implementation of EnqueueInQueueDataset and
 PrependFromQueueAndPaddedBatch.

PiperOrigin-RevId: 223879566
---
 tensorflow/contrib/training/BUILD             |  23 -
 tensorflow/contrib/training/__init__.py       |   5 +-
 .../python/training/tensor_queue_dataset.py   | 201 ------
 .../training/tensor_queue_dataset_test.py     | 355 ----------
 .../api_def_EnqueueInQueueDataset.pbtxt       |   4 -
 ...> api_def_ExperimentalRandomDataset.pbtxt} |   2 +-
 ... => api_def_ExperimentalScanDataset.pbtxt} |   2 +-
 ...rependFromQueueAndPaddedBatchDataset.pbtxt |   4 -
 .../optimizers/data/hoist_random_uniform.cc   |   2 +-
 .../data/hoist_random_uniform_test.cc         |   2 +-
 tensorflow/core/kernels/data/BUILD            |  38 -
 .../core/kernels/data/experimental/BUILD      |  26 +
 .../{ => experimental}/random_dataset_op.cc   |   2 +-
 .../{ => experimental}/scan_dataset_op.cc     |   3 +-
 .../kernels/data/tensor_queue_dataset_op.cc   | 657 ------------------
 .../core/ops/compat/ops_history.v1.pbtxt      | 208 ++----
 tensorflow/core/ops/dataset_ops.cc            |  58 --
 .../core/ops/experimental_dataset_ops.cc      |  28 +
 tensorflow/python/data/experimental/ops/BUILD |   4 +-
 .../data/experimental/ops/random_ops.py       |   4 +-
 .../python/data/experimental/ops/scan_ops.py  |   4 +-
 21 files changed, 142 insertions(+), 1490 deletions(-)
 delete mode 100644 tensorflow/contrib/training/python/training/tensor_queue_dataset.py
 delete mode 100644 tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt
 rename tensorflow/core/api_def/base_api/{api_def_RandomDataset.pbtxt => api_def_ExperimentalRandomDataset.pbtxt} (90%)
 rename tensorflow/core/api_def/base_api/{api_def_ScanDataset.pbtxt => api_def_ExperimentalScanDataset.pbtxt} (73%)
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
 rename tensorflow/core/kernels/data/{ => experimental}/random_dataset_op.cc (98%)
 rename tensorflow/core/kernels/data/{ => experimental}/scan_dataset_op.cc (98%)
 delete mode 100644 tensorflow/core/kernels/data/tensor_queue_dataset_op.cc

diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD
index 00295f57f6..f6427ae05a 100644
--- a/tensorflow/contrib/training/BUILD
+++ b/tensorflow/contrib/training/BUILD
@@ -26,7 +26,6 @@ py_library(
         "python/training/resample.py",
         "python/training/sampling_ops.py",
         "python/training/sequence_queueing_state_saver.py",
-        "python/training/tensor_queue_dataset.py",
         "python/training/training.py",
         "python/training/tuner.py",
     ],
@@ -287,28 +286,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "tensor_queue_dataset_test",
-    size = "large",
-    srcs = ["python/training/tensor_queue_dataset_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
-        ":training_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:gradients",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:random_seed",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variables",
-        "//tensorflow/python/data",
-        "//tensorflow/python/data/experimental/kernel_tests/serialization:dataset_serialization_test_base",
-        "//third_party/py/numpy",
-    ],
-)
-
 tf_proto_library(
     name = "protos_all",
     srcs = glob(["**/*.proto"]),
diff --git a/tensorflow/contrib/training/__init__.py b/tensorflow/contrib/training/__init__.py
index 3547e71184..87ce57ef06 100644
--- a/tensorflow/contrib/training/__init__.py
+++ b/tensorflow/contrib/training/__init__.py
@@ -59,8 +59,6 @@ from tensorflow.contrib.training.python.training.hparam import *
 from tensorflow.contrib.training.python.training.resample import *
 from tensorflow.contrib.training.python.training.sampling_ops import *
 from tensorflow.contrib.training.python.training.sequence_queueing_state_saver import *
-from tensorflow.contrib.training.python.training.tensor_queue_dataset import enqueue_in_queue_dataset
-from tensorflow.contrib.training.python.training.tensor_queue_dataset import prepend_from_queue_and_padded_batch_dataset
 from tensorflow.contrib.training.python.training.training import add_gradients_summaries
 from tensorflow.contrib.training.python.training.training import clip_gradient_norms
 from tensorflow.contrib.training.python.training.training import clip_gradient_norms_fn
@@ -79,7 +77,6 @@ _allowed_symbols = [
     'FeedingQueueRunner', 'get_or_create_eval_step', 'StopAfterNEvalsHook',
     'SummaryAtEndHook', 'wait_for_new_checkpoint', 'add_gradients_summaries',
     'clip_gradient_norms', 'clip_gradient_norms_fn', 'create_train_op',
-    'multiply_gradients', 'enqueue_in_queue_dataset',
-    'prepend_from_queue_and_padded_batch_dataset', 'train']
+    'multiply_gradients', 'train']
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/training/python/training/tensor_queue_dataset.py b/tensorflow/contrib/training/python/training/tensor_queue_dataset.py
deleted file mode 100644
index 8896a95327..0000000000
--- a/tensorflow/contrib/training/python/training/tensor_queue_dataset.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Python wrappers for Datasets and Iterators."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import convert
-from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
-from tensorflow.python.ops import gen_dataset_ops
-from tensorflow.python.util import nest as tf_nest
-
-
-class _PrependFromQueueAndPaddedBatchDataset(dataset_ops.UnaryDataset):
-  """A `Dataset` that prepends a queue to another `Dataset`.
-
-  A vector of handles to the queue is returned as the first component of
-  the associated iterator.  This vector can be passed to
-  `enqueue_in_queue_dataset` to add new elements to the queue.
-  """
-
-  def __init__(self, input_dataset, batch_size, padded_shapes, padding_values):
-    """Initialize `PrependFromQueueAndPaddedBatchDataset`."""
-    super(_PrependFromQueueAndPaddedBatchDataset, self).__init__(input_dataset)
-    if sparse.any_sparse(input_dataset.output_classes):
-      raise TypeError(
-          "Batching of padded sparse tensors is not currently supported")
-    self._input_dataset = input_dataset
-    self._batch_size = ops.convert_to_tensor(
-        batch_size, dtype=dtypes.int64, name="batch_size")
-    if padded_shapes is None:
-      self._padded_shapes = nest.map_structure(
-          convert.partial_shape_to_tensor, input_dataset.output_shapes)
-    else:
-      self._padded_shapes = nest.map_structure_up_to(
-          input_dataset.output_shapes, convert.partial_shape_to_tensor,
-          padded_shapes)
-    # pylint: disable=protected-access
-    padding_values = (
-        padding_values if padding_values is not None else
-        dataset_ops._default_padding(input_dataset))
-    self._padding_values = nest.map_structure_up_to(
-        input_dataset.output_shapes, dataset_ops._padding_value_to_tensor,
-        padding_values, input_dataset.output_types)
-    # pylint: enable=protected-access
-
-  def _as_variant_tensor(self):
-    # pylint: disable=protected-access
-    return gen_dataset_ops.prepend_from_queue_and_padded_batch_dataset(
-        self._input_dataset._as_variant_tensor(),
-        batch_size=self._batch_size,
-        padded_shapes=[
-            ops.convert_to_tensor(s, dtype=dtypes.int64)
-            for s in nest.flatten(self._padded_shapes)
-        ],
-        padding_values=nest.flatten(self._padding_values),
-        output_shapes=nest.flatten(
-            sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
-    # pylint: enable=protected-access
-
-  @property
-  def output_classes(self):
-    return (ops.Tensor, self._input_dataset.output_classes)
-
-  def _as_batch_shape(self, shape_like):
-    return tensor_shape.vector(None).concatenate(
-        tensor_util.constant_value_as_shape(shape_like))
-
-  @property
-  def output_shapes(self):
-    # First output is a variant representing the Queue
-    return (tensor_shape.vector(None),
-            nest.map_structure(self._as_batch_shape, self._padded_shapes))
-
-  @property
-  def output_types(self):
-    # First output is a variant representing the Queue
-    return (dtypes.variant, self._input_dataset.output_types)
-
-
-def prepend_from_queue_and_padded_batch_dataset(batch_size,
-                                                padding_values=None,
-                                                padded_shapes=None):
-  """A transformation that prepends a queue to a `Dataset` and batches results.
-
-  A vector of handles to the queue is returned as the first component of the
-  associated iterator.  This vector can be passed to `enqueue_in_queue_dataset`
-  to add new elements to the queue.
-
-  Below is an example of how this dataset might be used to split incoming
-  variable-length sequences into "head" and "rest" parts, where "rest" parts
-  are re-enqueued back into the dataset.  A more realistic example would
-  perform some calculation on the "head" and modify some components of "rest"
-  with the result (before re-enqueueing).
-
-  ```python
-  dataset = tf.data.Dataset.from_tensor_slices([2*x for x in range(10)])
-  # Make a dataset of variable-length vectors and their lengths.
-  dataset = dataset.map(lambda count: (count, tf.ones((count,))))
-  # Emit a queue we can prepend to, and counts/values as padded batch.
-  dataset = dataset.apply(
-      tf.contrib.training.prepend_from_queue_and_padded_batch_dataset(
-        batch_size=10))
-  dataset = dataset.prefetch(1)
-
-  iterator = dataset.make_one_shot_iterator()
-  queue, (count, padded_value) = iterator.get_next()
-
-  # Split the padded_value into two pieces: head and rest
-  rest_indices = tf.squeeze(tf.where(count > 3), axis=1)
-  bound = tf.minimum(3, tf.reduce_max(count))
-  value_head = padded_value[:, :bound]
-  count_rest = tf.gather(count - 3, rest_indices)
-  value_rest = tf.gather(padded_value[:, bound:], rest_indices)
-  queue_rest = tf.gather(queue, rest_indices)
-  enqueue_rest_op = tf.contrib.training.enqueue_in_queue_dataset(
-    queue_rest, (count_rest, value_rest))
-  with tf.control_dependencies([enqueue_rest_op]):
-    calculation = fn(value_head)
-
-  while True:  # Will raise OutOfRange when finished with all pieces.
-    session.run(calculation)
-  ```
-
-  Args:
-    batch_size: `int64` scalar tensor.  The batch size to use when performing
-      padded batching.
-    padding_values: (optional) Nested tuple of scalar tensors.  If provided,
-      the structure and dtypes of padding_values should match that of
-      incoming dataset's `output_types`.
-    padded_shapes: (optional) Nested tuple of `int64` vector tensors.
-      If provided, the structure must match that of the incoming dataset's
-      `output_types`.  If not provided, the incoming dataset's `output_shapes`
-      is used.  Any unknown (`None` or `-1`) dimensions in the shapes are
-      treated as being unique per-batch: for each batch time, an unknown
-      dimension is replaced with the maximum given value of this dimension
-      across all tensors for the given component in the batch.
-
-  Returns:
-    A `Dataset` transformation function, which can be passed to
-    `tf.data.Dataset.apply`.
-  """
-
-  def _apply_fn(dataset):
-    return _PrependFromQueueAndPaddedBatchDataset(
-        dataset,
-        batch_size=batch_size,
-        padding_values=padding_values,
-        padded_shapes=padded_shapes)
-
-  return _apply_fn
-
-
-def enqueue_in_queue_dataset(queue, components):
-  """Enqueue components into queue from `PrependFromQueueAndPaddedBatchDataset`.
-
-  The components' dtypes and shapes must be compatible with the `output_shapes`
-  attribute of the `dataset` created by
-  `prepend_from_queue_and_padded_batch_dataset`.  This operation supports both
-  non-batched and batched modes.
-
-  For more details, see the example in the docstring for
-  `prepend_from_queue_and_padded_batch_dataset`.
-
-  Args:
-    queue: `variant` scalar or vector tensor.
-      The tensor emitted by the first component of the iterator associated with
-      `prepend_from_queue_and_padded_batch_dataset`.  If this is a scalar,
-      then the `components` input tensors should not have a prepended batch
-      dimension.
-    components: Nested tuple of tensors, each with a leading batch dimension
-      if `queue` is a vector.  The structure, dtypes, and shapes
-      (excluding batch dimension) must match the nested tuples
-      `dataset.output_types[1]` and `dataset.output_shapes[1]` (the non-queue
-      output types and shapes) of the `dataset` emitted by
-      the original `prepend_from_queue_and_padded_batch_dataset` call.
-
-  Returns:
-    An `Operation` that enqueues `components` into the dataset(s) associated
-    with entries of `queue`.
-  """
-  return gen_dataset_ops.enqueue_in_queue_dataset(
-      queue=queue, components=tf_nest.flatten(components))
diff --git a/tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py b/tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py
deleted file mode 100644
index c1657fec7b..0000000000
--- a/tensorflow/contrib/training/python/training/tensor_queue_dataset_test.py
+++ /dev/null
@@ -1,355 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for TensorQueueDataset."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.training.python.training import tensor_queue_dataset as tqd
-from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import string_ops
-from tensorflow.python.platform import test
-
-
-class PrependFromQueueAndPaddedBatchDatasetTest(test.TestCase):
-
-  def testNoEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1))
-    self.assertEqual((dtypes.variant, dtypes.int32), dataset.output_types)
-    self.assertAllEqual(([None],) * 2,
-                        [x.as_list() for x in dataset.output_shapes])
-    iterator = dataset.make_one_shot_iterator()
-    _, value = iterator.get_next()
-    self.assertEqual([0], self.evaluate(value))
-    self.assertEqual([1], self.evaluate(value))
-    self.assertEqual([2], self.evaluate(value))
-    with self.assertRaisesOpError("End of sequence"):
-      self.evaluate(value)
-
-  def testBatchedNoEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=2))
-    iterator = dataset.make_one_shot_iterator()
-    _, value = iterator.get_next()
-    self.assertAllEqual([0, 1], self.evaluate(value))
-    self.assertAllEqual([2], self.evaluate(value))
-    with self.assertRaisesOpError("End of sequence"):
-      self.evaluate(value)
-
-  def testBatchedWithBiggerPaddingNoEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([[0], [1], [2]])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(
-            batch_size=2, padded_shapes=[3]))
-    iterator = dataset.make_one_shot_iterator()
-    _, value = iterator.get_next()
-    self.assertAllEqual([[0, 0, 0], [1, 0, 0]], self.evaluate(value))
-    self.assertAllEqual([[2, 0, 0]], self.evaluate(value))
-    with self.assertRaisesOpError("End of sequence"):
-      self.evaluate(value)
-
-  def testBatchedWithBiggerPaddingOneEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([[0], [1], [2]])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(
-            batch_size=1, padded_shapes=[3]))
-    iterator = dataset.make_one_shot_iterator()
-    queue_handle, value = iterator.get_next()
-    enqueue_negative = tqd.enqueue_in_queue_dataset(queue_handle, -value)
-    with self.cached_session() as sess:
-      self.assertAllEqual([[0, 0, 0]], sess.run(value))
-      value_1, _ = sess.run([value, enqueue_negative])
-      self.assertAllEqual([[1, 0, 0]], value_1)
-      value_2, _ = sess.run([value, enqueue_negative])
-      self.assertAllEqual([[-1, 0, 0]], value_2)
-      value_3 = sess.run(value)
-      self.assertAllEqual([[1, 0, 0]], value_3)
-      value_4, _ = sess.run([value, enqueue_negative])
-      self.assertAllEqual([[2, 0, 0]], value_4)
-      value_5 = sess.run(value)
-      self.assertAllEqual([[-2, 0, 0]], value_5)
-      with self.assertRaisesOpError("End of sequence"):
-        sess.run(value)
-
-  def testOneEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1))
-    iterator = dataset.make_one_shot_iterator()
-    queue_handle, value = iterator.get_next()
-    enqueue_negative = tqd.enqueue_in_queue_dataset(queue_handle, -value)
-    with self.cached_session() as sess:
-      self.assertEqual([0], sess.run(value))
-      value_1, _ = sess.run([value, enqueue_negative])
-      self.assertEqual([1], value_1)
-      value_2, _ = sess.run([value, enqueue_negative])
-      self.assertEqual([-1], value_2)
-      value_3 = sess.run(value)
-      self.assertEqual([1], value_3)
-      value_4, _ = sess.run([value, enqueue_negative])
-      self.assertEqual([2], value_4)
-      value_5 = sess.run(value)
-      self.assertEqual([-2], value_5)
-      with self.assertRaisesOpError("End of sequence"):
-        sess.run(value)
-
-  def testBatchedOneEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=2))
-    iterator = dataset.make_one_shot_iterator()
-    queue_handle, value = iterator.get_next()
-    enqueue_negative = tqd.enqueue_in_queue_dataset(queue_handle, -value)
-    enqueue_zeroth = tqd.enqueue_in_queue_dataset([queue_handle[0]],
-                                                  array_ops.expand_dims(
-                                                      value[0], axis=0))
-    with self.cached_session() as sess:
-      value_0, _ = sess.run([value, enqueue_negative])
-      self.assertAllEqual([0, 1], value_0)
-      value_1, _ = sess.run([value, enqueue_zeroth])
-      self.assertAllEqual([0, -1], value_1)
-      value_2, _ = sess.run([value, enqueue_negative])
-      self.assertAllEqual([0, 2], value_2)
-      self.assertAllEqual([0, -2], sess.run(value))
-      with self.assertRaisesOpError("End of sequence"):
-        sess.run(value)
-
-  def testManyEnqueue(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1))
-    iterator = dataset.make_one_shot_iterator()
-    queue_handle, value = iterator.get_next()
-    enqueue_many_more = [
-        tqd.enqueue_in_queue_dataset(queue_handle, value + 100 + i)
-        for i in range(1000)
-    ]
-    with self.cached_session() as sess:
-      value_0, _ = sess.run((value, enqueue_many_more))
-      self.assertEqual([0], value_0)
-      rest = []
-      for _ in range(1000):
-        rest.append(sess.run(value))
-      self.assertEquals([[100 + i] for i in range(1000)], sorted(rest))
-      # Going back to the original input.
-      value_1, _ = sess.run((value, enqueue_many_more))
-      self.assertEqual(1, value_1)
-      rest = []
-      for _ in range(1000):
-        rest.append(sess.run(value))
-      self.assertEquals([[100 + i + 1] for i in range(1000)], sorted(rest))
-      with self.assertRaisesOpError("End of sequence"):
-        sess.run(value)
-
-  def testEnqueueWithPrefetch(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1))
-    # Prefetching will request additional values before they are
-    # available to the queue.
-    dataset = dataset.prefetch(buffer_size=3)
-    iterator = dataset.make_one_shot_iterator()
-    queue_handle, value = iterator.get_next()
-    enqueue = tqd.enqueue_in_queue_dataset(queue_handle, value + 1)
-    with self.cached_session() as sess:
-      i = 0
-      while i < 4:
-        received, _ = sess.run((value, enqueue))
-        if received.size > 0:
-          self.assertAllEqual([i], received)
-          i += 1
-      received_last = False
-      while True:
-        try:
-          received = sess.run(value)
-          if received.size > 0:
-            self.assertAllEqual([4], received)
-            received_last = True
-        except errors.OutOfRangeError:
-          break
-      self.assertTrue(received_last)
-
-  def testDatasetWithPaddedShapeSmallerThanInputFails(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([[0, 0, 0]]).repeat(None)
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(
-            batch_size=1, padded_shapes=[2]))
-    iterator = dataset.make_one_shot_iterator()
-    _, value = iterator.get_next()
-    with self.cached_session() as sess:
-      with self.assertRaisesOpError(
-          r"Incompatible input shapes at component 0 between "
-          r"input dataset this dataset: \[3\] vs. \[2\]"):
-        sess.run(value)
-
-  def testEnqueueWithIncompatibleInputsFailsWithInformativeError(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0]).repeat(None)
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1))
-    iterator = dataset.make_one_shot_iterator()
-    queue_handle, value = iterator.get_next()
-
-    enqueue_bad_structure = tqd.enqueue_in_queue_dataset(
-        queue_handle, (value, value))
-    enqueue_bad_dtype = tqd.enqueue_in_queue_dataset(queue_handle,
-                                                     np.array(
-                                                         [1.0],
-                                                         dtype=np.float32))
-    enqueue_bad_shape_no_batch_dim = tqd.enqueue_in_queue_dataset(
-        queue_handle, ([1],))
-    enqueue_bad_shape = tqd.enqueue_in_queue_dataset(queue_handle,
-                                                     np.array(
-                                                         [[1]], dtype=np.int32))
-
-    with self.cached_session() as sess:
-      with self.assertRaisesOpError(
-          "mismatched number of tensors.  Queue expects 1 tensors but "
-          "tried to insert 2"):
-        sess.run(enqueue_bad_structure)
-      with self.assertRaisesOpError(r"Expected component 0 to have batched "
-                                    r"shape \[1,...\], but saw shape: \[\]"):
-        sess.run(enqueue_bad_shape_no_batch_dim)
-      with self.assertRaisesOpError(
-          r"mismatched shapes at component 0.  Attempted to insert tensor "
-          r"with shape \[1\] but queue expected shape: \[\]"):
-        sess.run(enqueue_bad_shape)
-      with self.assertRaisesOpError(
-          r"mismatched dtypes at component 0.  Attempted to insert tensor "
-          r"of type float but queue expected type: int32"):
-        sess.run(enqueue_bad_dtype)
-
-  def testEnqueueWithPaddedBatchFailsWithInformativeError(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2])
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=1))
-    with self.assertRaisesRegexp(
-        TypeError, r"Unable to create padding for field of type 'variant'"):
-      dataset.padded_batch(batch_size=10, padded_shapes=[1])
-
-  def testOneEnqueueWithPadding(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 2, 4, 6])
-    # Make a dataset of variable-length vectors and their lengths.
-    dataset = dataset.map(
-        lambda c: (c, c * array_ops.ones((c,), dtype=c.dtype)))
-    # Emit a queue we can prepend to, and counts/values as padded
-    # batch.
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=3))
-
-    iterator = dataset.make_one_shot_iterator()
-    queue, (count, padded_value) = iterator.get_next()
-
-    # Split the padded_value into two pieces: head and rest
-    rest_indices = array_ops.squeeze(array_ops.where(count > 2), axis=1)
-    bound = math_ops.minimum(2, math_ops.reduce_max(count))
-    value_head = padded_value[:, :bound]
-    count_rest = array_ops.gather(count - 2, rest_indices)
-    value_rest = array_ops.gather(padded_value, rest_indices)[:, bound:]
-    queue_rest = array_ops.gather(queue, rest_indices)
-    enqueue_rest_op = tqd.enqueue_in_queue_dataset(queue_rest,
-                                                   (count_rest, value_rest))
-    with ops.control_dependencies([enqueue_rest_op]):
-      calc = array_ops.identity(value_head)
-
-    with self.cached_session() as sess:
-      self.assertAllEqual([[0, 0], [2, 2], [4, 4]], sess.run(calc))
-      self.assertAllEqual([[4, 4], [6, 6]], sess.run(calc))
-      self.assertAllEqual([[6, 6]], sess.run(calc))
-      self.assertAllEqual([[6, 6]], sess.run(calc))
-      # Get some final batches due to prefetching.
-      for _ in range(3):
-        try:
-          self.assertAllEqual(
-              np.empty(shape=(0, 0), dtype=np.int32), sess.run(calc))
-        except errors.OutOfRangeError as e:
-          self.assertTrue(str(e).startswith("End of sequence"))
-
-  def testNonstandardPadding(self):
-    dataset = dataset_ops.Dataset.from_tensor_slices([0, 2, 4, 6])
-    # Make a dataset of variable-length vectors and their lengths.
-    dataset = dataset.map(
-        lambda c: (c, c * array_ops.ones((c,), dtype=c.dtype)))
-    # Emit a queue we can prepend to, and counts/values as padded
-    # batch.
-    dataset = dataset.apply(
-        tqd.prepend_from_queue_and_padded_batch_dataset(
-            batch_size=3, padding_values=(
-                0,
-                -1,
-            )))
-
-    iterator = dataset.make_one_shot_iterator()
-    _, (unused_count, padded_value) = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertAllEqual([[-1, -1, -1, -1], [2, 2, -1, -1], [4, 4, 4, 4]],
-                          sess.run(padded_value))
-      self.assertAllEqual([[6] * 6], sess.run(padded_value))
-      with self.assertRaisesOpError("End of sequence"):
-        sess.run(padded_value)
-
-
-# TODO(ebrevdo): Figure out how to use run_core_tests to test state
-# saving of an iterator that's had some tensors enqueued into its queue.
-class PrependFromQueueAndPaddedBatchDatasetSerializationTest(
-    dataset_serialization_test_base.DatasetSerializationTestBase):
-
-  def testPrependFromQueueAndPaddedBatch(self):
-
-    def build_dataset(seq_lens):
-      return dataset_ops.Dataset.from_tensor_slices(seq_lens).map(
-          lambda x: array_ops.fill([x], x)).apply(
-              tqd.prepend_from_queue_and_padded_batch_dataset(batch_size=4))
-
-    seq_lens1 = np.random.randint(1, 20, size=(32,)).astype(np.int32)
-    seq_lens2 = np.random.randint(21, 40, size=(32,)).astype(np.int32)
-    self.run_core_tests(lambda: build_dataset(seq_lens1),
-                        lambda: build_dataset(seq_lens2), 8)
-
-  def testPrependFromQueueAndPaddedBatchNonDefaultPadding(self):
-
-    def build_dataset(seq_lens):
-
-      def fill_tuple(x):
-        filled = array_ops.fill([x], x)
-        return (filled, string_ops.as_string(filled))
-
-      padded_shape = [-1]
-      return dataset_ops.Dataset.from_tensor_slices(seq_lens).map(
-          fill_tuple).apply(
-              tqd.prepend_from_queue_and_padded_batch_dataset(
-                  batch_size=4,
-                  padded_shapes=(padded_shape, padded_shape),
-                  padding_values=(-1, "<end>")))
-
-    seq_lens1 = np.random.randint(1, 20, size=(32,)).astype(np.int32)
-    seq_lens2 = np.random.randint(21, 40, size=(32,)).astype(np.int32)
-    self.run_core_tests(lambda: build_dataset(seq_lens1),
-                        lambda: build_dataset(seq_lens2), 8)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt
deleted file mode 100644
index 051cf14c0e..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_EnqueueInQueueDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "EnqueueInQueueDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalRandomDataset.pbtxt
similarity index 90%
rename from tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalRandomDataset.pbtxt
index 103e608ae8..f5d7bc4adb 100644
--- a/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalRandomDataset.pbtxt
@@ -1,5 +1,5 @@
 op {
-  graph_op_name: "RandomDataset"
+  graph_op_name: "ExperimentalRandomDataset"
   visibility: HIDDEN
   in_arg {
     name: "seed"
diff --git a/tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalScanDataset.pbtxt
similarity index 73%
rename from tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt
rename to tensorflow/core/api_def/base_api/api_def_ExperimentalScanDataset.pbtxt
index f0dc1a3e0d..4742cf4d57 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScanDataset.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalScanDataset.pbtxt
@@ -1,5 +1,5 @@
 op {
-  graph_op_name: "ScanDataset"
+  graph_op_name: "ExperimentalScanDataset"
   visibility: HIDDEN
   summary: "Creates a dataset successively reduces `f` over the elements of `input_dataset`."
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
deleted file mode 100644
index 228c4047d2..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_PrependFromQueueAndPaddedBatchDataset.pbtxt
+++ /dev/null
@@ -1,4 +0,0 @@
-op {
-  graph_op_name: "PrependFromQueueAndPaddedBatchDataset"
-  visibility: HIDDEN
-}
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 5af9fbadf7..60755256d8 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -67,7 +67,7 @@ NodeDef MakeStatelessMap(const NodeDef& map_node, const NodeDef& zip_node,
 NodeDef MakeRandomDataset(const NodeDef& random_uniform_node,
                           MutableGraphView* graph) {
   NodeDef random_dataset;
-  random_dataset.set_op("RandomDataset");
+  random_dataset.set_op("ExperimentalRandomDataset");
   graph_utils::SetUniqueGraphNodeName("RandomDataset", graph->graph(),
                                       &random_dataset);
 
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform_test.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform_test.cc
index 455459e3f6..b6a29a442e 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform_test.cc
@@ -55,7 +55,7 @@ TEST(HoistRandomUniform, SimpleHoisting) {
   const int zip_dataset_id =
       graph_utils::FindGraphNodeWithOp("ZipDataset", output);
   const int random_dataset_id =
-      graph_utils::FindGraphNodeWithOp("RandomDataset", output);
+      graph_utils::FindGraphNodeWithOp("ExperimentalRandomDataset", output);
   const int batch_random_id =
       graph_utils::FindGraphNodeWithOp("BatchDatasetV2", output);
   ASSERT_NE(random_dataset_id, -1);
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 55a34c9c62..e2ab77632d 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -223,19 +223,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "scan_dataset_op",
-    srcs = ["scan_dataset_op.cc"],
-    deps = [
-        ":captured_function",
-        "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "flat_map_dataset_op",
     srcs = ["flat_map_dataset_op.cc"],
@@ -345,17 +332,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "random_dataset_op",
-    srcs = ["random_dataset_op.cc"],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "range_dataset_op",
     srcs = ["range_dataset_op.cc"],
@@ -399,17 +375,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "tensor_queue_dataset_op",
-    srcs = ["tensor_queue_dataset_op.cc"],
-    deps = [
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_kernel_library(
     name = "tensor_slice_dataset_op",
     srcs = ["tensor_slice_dataset_op.cc"],
@@ -584,17 +549,14 @@ tf_kernel_library(
         ":parallel_interleave_dataset_op",
         ":parallel_map_dataset_op",
         ":prefetch_dataset_op",
-        ":random_dataset_op",
         ":range_dataset_op",
         ":reader_dataset_ops",
         ":repeat_dataset_op",
-        ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
         ":sparse_tensor_slice_dataset_op",
         ":take_dataset_op",
         ":tensor_dataset_op",
-        ":tensor_queue_dataset_op",
         ":tensor_slice_dataset_op",
         ":window_dataset_op",
         ":zip_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index e8d87b8dd0..7433303f77 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -203,6 +203,30 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "random_dataset_op",
+    srcs = ["random_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_kernel_library(
+    name = "scan_dataset_op",
+    srcs = ["scan_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:captured_function",
+    ],
+)
+
 tf_kernel_library(
     name = "set_stats_aggregator_dataset_op",
     srcs = ["set_stats_aggregator_dataset_op.cc"],
@@ -333,6 +357,8 @@ tf_kernel_library(
         ":parallel_interleave_dataset_op",
         ":parse_example_dataset_op",
         ":prefetching_kernels",
+        ":random_dataset_op",
+        ":scan_dataset_op",
         ":set_stats_aggregator_dataset_op",
         ":sleep_dataset_op",
         ":sliding_window_dataset_op",
diff --git a/tensorflow/core/kernels/data/random_dataset_op.cc b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
similarity index 98%
rename from tensorflow/core/kernels/data/random_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/random_dataset_op.cc
index 816405fea9..b849aa3690 100644
--- a/tensorflow/core/kernels/data/random_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
@@ -152,7 +152,7 @@ class RandomDatasetOp : public DatasetOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("RandomDataset").Device(DEVICE_CPU),
+REGISTER_KERNEL_BUILDER(Name("ExperimentalRandomDataset").Device(DEVICE_CPU),
                         RandomDatasetOp);
 
 }  // namespace
diff --git a/tensorflow/core/kernels/data/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
similarity index 98%
rename from tensorflow/core/kernels/data/scan_dataset_op.cc
rename to tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 12d07f1919..31358bdefc 100644
--- a/tensorflow/core/kernels/data/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -271,7 +271,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
   NameAttrList func_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("ScanDataset").Device(DEVICE_CPU), ScanDatasetOp);
+REGISTER_KERNEL_BUILDER(Name("ExperimentalScanDataset").Device(DEVICE_CPU),
+                        ScanDatasetOp);
 
 }  // namespace
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc b/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc
deleted file mode 100644
index 7fd1c4c9e0..0000000000
--- a/tensorflow/core/kernels/data/tensor_queue_dataset_op.cc
+++ /dev/null
@@ -1,657 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <deque>
-
-#include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/partial_tensor_shape.h"
-#include "tensorflow/core/framework/resource_mgr.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/variant.h"
-#include "tensorflow/core/framework/variant_encode_decode.h"
-#include "tensorflow/core/util/batch_util.h"
-
-namespace tensorflow {
-namespace data {
-namespace {
-
-bool IsGreaterEqualToOrCompatibleWith(const PartialTensorShape& a,
-                                      const PartialTensorShape& b) {
-  // Returns true if dims[a] >= dims[b], or are compatible.
-  if (a.unknown_rank()) return true;
-  if (a.dims() != b.dims()) return false;
-  for (int d = 0; d < a.dims(); ++d) {
-    if (a.dim_size(d) == -1 || b.dim_size(d) == -1) continue;
-    if (a.dim_size(d) < b.dim_size(d)) return false;
-  }
-  return true;
-}
-
-DataTypeVector PrependQueueType(const DataTypeVector& dtypes) {
-  DataTypeVector out;
-  out.reserve(dtypes.size() + 1);
-  out.push_back(DT_VARIANT);  // The queue component.
-  for (const DataType& d : dtypes) out.push_back(d);
-  return out;
-}
-
-std::vector<PartialTensorShape> PrependQueueShapeWithBatch(
-    const std::vector<PartialTensorShape>& shapes) {
-  std::vector<PartialTensorShape> out;
-  out.reserve(shapes.size() + 1);
-  out.emplace_back(PartialTensorShape({-1}));  // The queue component.
-  for (PartialTensorShape s : shapes) {
-    s.InsertDim(0, -1);  // Unknown batch size.
-    out.push_back(std::move(s));
-  }
-  return out;
-}
-
-class EnqueueInQueueDatasetOp;
-
-class PrependFromQueueAndPaddedBatchDataset : public DatasetBase {
- public:
-  PrependFromQueueAndPaddedBatchDataset(
-      OpKernelContext* ctx, const int64 batch_size, const DatasetBase* input,
-      const DataTypeVector& dtypes,
-      const std::vector<PartialTensorShape>& shapes,
-      std::vector<Tensor> padding_values)
-      : DatasetBase(DatasetContext(ctx)),
-        batch_size_(batch_size),
-        input_(input),
-        dtypes_(dtypes),
-        shapes_(shapes),
-        padding_values_(std::move(padding_values)),
-        dtypes_with_queue_(PrependQueueType(dtypes)),
-        batched_shapes_with_queue_(PrependQueueShapeWithBatch(shapes)) {
-    input_->Ref();
-  }
-
-  ~PrependFromQueueAndPaddedBatchDataset() override { input_->Unref(); }
-
-  std::unique_ptr<IteratorBase> MakeIteratorInternal(
-      const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(new Iterator(
-        {this, strings::StrCat(prefix, "::PrependFromQueueAndPaddedBatch")}));
-  }
-
-  const DataTypeVector& output_dtypes() const override {
-    return dtypes_with_queue_;
-  }
-  const std::vector<PartialTensorShape>& output_shapes() const override {
-    return batched_shapes_with_queue_;
-  }
-
-  string DebugString() const override {
-    return "PrependFromQueueAndPaddedBatchDatasetOp::Dataset";
-  }
-
- protected:
-  Status AsGraphDefInternal(SerializationContext* ctx,
-                            DatasetGraphDefBuilder* b,
-                            Node** output) const override {
-    Node* input_graph = nullptr;
-    TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph));
-    Node* batch_size = nullptr;
-    TF_RETURN_IF_ERROR(b->AddScalar(batch_size_, &batch_size));
-
-    std::vector<Node*> padded_shapes;
-    padded_shapes.reserve(shapes_.size());
-    for (int i = 0; i < shapes_.size(); i++) {
-      Node* node;
-      Tensor t(DT_INT64, TensorShape({shapes_[i].dims()}));
-      for (int j = 0; j < shapes_[i].dims(); j++) {
-        t.vec<int64>()(j) = shapes_[i].dim_size(j);
-      }
-      TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
-      padded_shapes.emplace_back(node);
-    }
-
-    std::vector<Node*> padding_values;
-    padding_values.reserve(padding_values_.size());
-    for (const Tensor& t : padding_values_) {
-      Node* node;
-      TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
-      padding_values.emplace_back(node);
-    }
-
-    AttrValue output_types;
-    b->BuildAttrValue(dtypes_, &output_types);
-
-    AttrValue output_shapes;
-    b->BuildAttrValue(batched_shapes_with_queue_, &output_shapes);
-
-    AttrValue N;
-    b->BuildAttrValue<int64>(shapes_.size(), &N);
-
-    TF_RETURN_IF_ERROR(b->AddDataset(this, {{0, input_graph}, {1, batch_size}},
-                                     {{2, padded_shapes}, {3, padding_values}},
-                                     {{"Toutput_types", output_types},
-                                      {"output_shapes", output_shapes},
-                                      {"N", N}},
-                                     output));
-
-    return Status::OK();
-  }
-
- private:
-  friend class EnqueueInQueueDatasetOp;
-
-  class Iterator
-      : public DatasetIterator<PrependFromQueueAndPaddedBatchDataset> {
-   public:
-    explicit Iterator(const Params& params)
-        : DatasetIterator<PrependFromQueueAndPaddedBatchDataset>(params) {}
-
-    ~Iterator() override { queue_->Unref(); }
-
-    Status Initialize(IteratorContext* ctx) override {
-      std::unique_ptr<IteratorBase> iterator;
-      TF_RETURN_IF_ERROR(
-          dataset()->input_->MakeIterator(ctx, prefix(), &iterator));
-      queue_ = new TensorQueue(std::move(iterator), dataset()->dtypes_,
-                               dataset()->shapes_);
-      return Status::OK();
-    }
-
-    Status GetNextInternal(IteratorContext* ctx,
-                           std::vector<Tensor>* out_tensors,
-                           bool* end_of_sequence) override {
-      std::vector<std::vector<Tensor>> batch;
-      TF_RETURN_IF_ERROR(queue_->GetNext(ctx, dataset()->batch_size_, &batch,
-                                         end_of_sequence));
-      const auto& dtypes = dataset()->dtypes_;
-      const auto& shapes = dataset()->shapes_;
-      const auto& input_shapes = dataset()->input_->output_shapes();
-      const auto& padding_values = dataset()->padding_values_;
-      const int64 batch_size = batch.size();
-      out_tensors->reserve(dtypes.size());
-
-      std::vector<TensorShape> max_shapes;  // Of non-queue components.
-      for (int i = 0; i < dtypes.size(); ++i) {
-        const PartialTensorShape& shape = shapes[i];
-        TensorShape out_shape({batch_size});
-        for (int r = 0; r < shape.dims(); ++r) {
-          if (shape.dim_size(r) >= 0) {
-            // padded_shape[r] is known.
-            out_shape.AddDim(shape.dim_size(r));
-          } else {
-            // padded_shape[r] is unknown, find the maximum across
-            // the batch.
-            int64 dim = 0;
-            for (int b = 0; b < batch.size(); ++b) {
-              dim = std::max(dim, batch[b][i].dim_size(r));
-            }
-            out_shape.AddDim(dim);
-          }
-        }
-        max_shapes.push_back(std::move(out_shape));
-      }
-
-      out_tensors->emplace_back(ctx->allocator({}), DT_VARIANT,
-                                TensorShape({batch_size}));
-      if (!batch.empty()) {
-        auto queues = out_tensors->back().flat<Variant>();
-        Variant& queue_inserter = queues(0);
-        queue_inserter = TensorQueueInserter();
-        queue_inserter.get<TensorQueueInserter>()->set_queue(queue_);
-        for (int b = 1; b < batch.size(); ++b) {
-          // Copy the TensorQueueInserter.  Each copy increments the
-          // Ref on the queue_.
-          queues(b) = queues(0);
-        }
-      }
-
-      for (int i = 0; i < max_shapes.size(); ++i) {
-        out_tensors->emplace_back(ctx->allocator({}), dtypes[i], max_shapes[i]);
-        Tensor& component = out_tensors->back();
-        // Try hard to take the fast path.
-        if (shapes[i].IsFullyDefined() &&
-            shapes[i].IsIdenticalTo(input_shapes[i])) {
-          // Take the fast path if we know all the shapes statically.
-          for (int64 b = 0; b < batch.size(); ++b) {
-            TF_RETURN_IF_ERROR(
-                batch_util::CopyElementToSlice(batch[b][i], &component, b));
-          }
-        } else {
-          TF_RETURN_IF_ERROR(
-              batch_util::SetElementZero(&component, padding_values[i]));
-          for (int64 b = 0; b < batch.size(); ++b) {
-            if (batch[b][i].shape() == max_shapes[i]) {
-              TF_RETURN_IF_ERROR(
-                  batch_util::CopyElementToSlice(batch[b][i], &component, b));
-            } else {
-              TF_RETURN_IF_ERROR(batch_util::CopyElementToLargerSlice(
-                  batch[b][i], &component, b));
-            }
-          }
-        }
-      }
-
-      // end_of_sequence was set before we populated out_tensors, so
-      // it's ok to return now.
-      return Status::OK();
-    }
-
-   protected:
-    // Work around bug in MSVC that disallows access to protected
-    // members of Iterator from within TensorQueue.
-    class TensorQueue;
-    friend class TensorQueue;
-
-    class TensorQueue : public core::RefCounted {
-     public:
-      TensorQueue(std::unique_ptr<IteratorBase> input_impl,
-                  const DataTypeVector& dtypes,
-                  const std::vector<PartialTensorShape>& shapes)
-          : dtypes_(dtypes),
-            shapes_(shapes),
-            input_impl_(std::move(input_impl)) {}
-
-      void MaybeWaitForNotificationLocked(mutex_lock* lock)
-          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-        // This essentially just releases the lock and immediately relocks.
-        cv_.wait_for(*lock, std::chrono::milliseconds(0));
-      }
-
-      void NotifyLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) { cv_.notify_all(); }
-
-      Status GetNext(IteratorContext* ctx, const int64 batch_size,
-                     std::vector<std::vector<Tensor>>* batch,
-                     bool* end_of_sequence) {
-        mutex_lock lock(mu_);
-
-        *end_of_sequence = false;
-
-        for (int64 b = 0; b < batch_size;) {
-          if (!entries_.empty()) {
-            batch->push_back(std::move(entries_.front()));
-            entries_.pop_front();
-            ++b;
-            continue;
-          } else {
-            if (input_impl_) {
-              // There's still input coming in.
-              std::vector<Tensor> tensors;
-              bool input_end;
-              TF_RETURN_IF_ERROR(
-                  input_impl_->GetNext(ctx, &tensors, &input_end));
-              if (!input_end) {
-                batch->push_back(std::move(tensors));
-                ++b;
-                continue;
-              } else {
-                input_impl_.reset();
-              }
-            }
-            if (!input_impl_) {
-              // There's no more input coming in.
-              if (RefCountIsOne()) {
-                // No TensorQueueInserters in the wild.
-                if (batch->empty()) {
-                  *end_of_sequence = true;
-                }
-                break;
-              } else {
-                MaybeWaitForNotificationLocked(&lock);
-                // If there's data available, try to add entries again.
-                // Otherwise return a smaller batch and hope the next
-                // iterator request has a non-empty or unused queue_.
-                if (entries_.empty()) {
-                  break;
-                }
-              }
-            }
-          }
-        }  // for (int64 b = ... batch_size)
-        return Status::OK();
-      }
-
-      Status Insert(const std::vector<Tensor>& tensors) {
-        if (tensors.size() != dtypes_.size()) {
-          return errors::InvalidArgument(
-              "TensorQueue::Insert: mismatched number of tensors.  Queue "
-              "expects ",
-              dtypes_.size(), " tensors but tried to insert ", tensors.size());
-        }
-        for (int i = 0; i < tensors.size(); ++i) {
-          if (tensors[i].dtype() != dtypes_[i]) {
-            return errors::InvalidArgument(
-                "TensorQueue::Insert: mismatched dtypes at component ", i,
-                ".  Attempted "
-                "to insert tensor of type ",
-                DataTypeString(tensors[i].dtype()),
-                " but queue expected type: ", DataTypeString(dtypes_[i]));
-          }
-          if (!shapes_[i].IsCompatibleWith(tensors[i].shape())) {
-            return errors::InvalidArgument(
-                "TensorQueue::Insert: mismatched shapes at component ", i,
-                ".  Attempted "
-                "to insert tensor with shape ",
-                tensors[i].shape().DebugString(),
-                " but queue expected shape: ", shapes_[i].DebugString());
-          }
-        }
-        mutex_lock lock(mu_);
-        entries_.push_back(tensors);
-        NotifyLocked();
-        return Status::OK();
-      }
-
-      Status Save(Iterator* iter, IteratorStateWriter* writer) {
-        mutex_lock lock(mu_);
-        if (input_impl_) {
-          TF_RETURN_IF_ERROR(iter->SaveInput(writer, input_impl_));
-        } else {
-          TF_RETURN_IF_ERROR(
-              writer->WriteScalar(iter->full_name("input_exhausted"), ""));
-        }
-        TF_RETURN_IF_ERROR(writer->WriteScalar(iter->full_name("entries_size"),
-                                               entries_.size()));
-        for (int64 b = 0; b < entries_.size(); ++b) {
-          for (int i = 0; i < dtypes_.size(); ++i) {
-            TF_RETURN_IF_ERROR(
-                writer->WriteTensor(strings::StrCat(iter->full_name("entries"),
-                                                    "[", b, "][", i, "]"),
-                                    entries_[b][i]));
-          }
-        }
-        return Status::OK();
-      }
-
-      Status Restore(Iterator* iter, IteratorContext* ctx,
-                     IteratorStateReader* reader) {
-        mutex_lock l(mu_);
-        if (reader->Contains(iter->full_name("input_exhausted"))) {
-          input_impl_.reset();
-        } else {
-          TF_RETURN_IF_ERROR(iter->dataset_input()->MakeIterator(
-              ctx, iter->prefix(), &input_impl_));
-          TF_RETURN_IF_ERROR(iter->RestoreInput(ctx, reader, input_impl_));
-        }
-        entries_.clear();
-        int64 entries_size = -1;
-        TF_RETURN_IF_ERROR(
-            reader->ReadScalar(iter->full_name("entries_size"), &entries_size));
-        if (entries_size < 0) {
-          return errors::DataLoss(
-              "Expected entries_size key '", iter->full_name("entries_size"),
-              "' to have nonnegative value, but saw: ", entries_size);
-        }
-        for (int64 b = 0; b < entries_size; ++b) {
-          std::vector<Tensor> entry;
-          for (int i = 0; i < dtypes_.size(); ++i) {
-            Tensor value;
-            TF_RETURN_IF_ERROR(
-                reader->ReadTensor(strings::StrCat(iter->full_name("entries"),
-                                                   "[", b, "][", i, "]"),
-                                   &value));
-            entry.push_back(std::move(value));
-          }
-          entries_.push_back(std::move(entry));
-        }
-        return Status::OK();
-      }
-
-      mutex* mu() { return &mu_; }
-
-     private:
-      DataTypeVector dtypes_;
-      std::vector<PartialTensorShape> shapes_;
-
-      mutex mu_;
-      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
-      std::deque<std::vector<Tensor>> entries_ GUARDED_BY(mu_);
-      condition_variable cv_ GUARDED_BY(mu_);
-    };
-
-    const DatasetBase* dataset_input() const { return dataset()->input_; }
-
-    std::shared_ptr<model::Node> CreateNode(
-        IteratorContext* ctx, model::Node::Args args) const override {
-      return model::MakeKnownRatioNode(std::move(args), dataset()->batch_size_);
-    }
-
-    Status SaveInternal(IteratorStateWriter* writer) override {
-      return queue_->Save(this, writer);
-    }
-
-    Status RestoreInternal(IteratorContext* ctx,
-                           IteratorStateReader* reader) override {
-      return queue_->Restore(this, ctx, reader);
-    }
-
-   public:
-    class TensorQueueInserter {
-     public:
-      TensorQueueInserter() : queue_(nullptr) {}
-
-      void set_queue(TensorQueue* queue) {
-        queue_ = queue;
-        queue_->Ref();
-      }
-
-      TensorQueueInserter(const TensorQueueInserter& rhs) {
-        queue_ = rhs.queue_;
-        queue_->Ref();
-      }
-
-      TensorQueueInserter(TensorQueueInserter&& rhs) {
-        queue_ = rhs.queue_;
-        rhs.queue_ = nullptr;
-      }
-
-      TensorQueueInserter& operator=(const TensorQueueInserter& rhs) = delete;
-
-      string TypeName() const { return "tensorflow::TensorQueueInserter"; }
-      string DebugString() const { return TypeName(); }
-
-      void Encode(VariantTensorData*) const {}
-      bool Decode(const VariantTensorData&) { return false; }
-
-      ~TensorQueueInserter() {
-        if (queue_) {
-          mutex_lock lock(*queue_->mu());
-          queue_->Unref();
-          queue_->NotifyLocked();
-          queue_ = nullptr;
-        }
-      }
-
-      Status Insert(const std::vector<Tensor>& tensors) const {
-        CHECK(queue_);
-        return queue_->Insert(tensors);
-      }
-
-     private:
-      mutable TensorQueue* queue_;
-    };
-
-   private:
-    TensorQueue* queue_;
-  };
-
- private:
-  const int64 batch_size_;
-  const DatasetBase* input_;
-  const DataTypeVector dtypes_;
-  const std::vector<PartialTensorShape> shapes_;
-  const std::vector<Tensor> padding_values_;
-  const DataTypeVector dtypes_with_queue_;
-  const std::vector<PartialTensorShape> batched_shapes_with_queue_;
-};
-
-class PrependFromQueueAndPaddedBatchDatasetOp : public UnaryDatasetOpKernel {
- public:
-  explicit PrependFromQueueAndPaddedBatchDatasetOp(OpKernelConstruction* ctx)
-      : UnaryDatasetOpKernel(ctx) {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("Toutput_types", &output_types_));
-  }
-
-  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                   DatasetBase** output) override {
-    int64 batch_size = 0;
-    OP_REQUIRES_OK(ctx,
-                   ParseScalarArgument<int64>(ctx, "batch_size", &batch_size));
-    OP_REQUIRES(
-        ctx, batch_size > 0,
-        errors::InvalidArgument("Batch size must be greater than zero."));
-
-    OpInputList padded_shape_tensors;
-    OP_REQUIRES_OK(ctx,
-                   ctx->input_list("padded_shapes", &padded_shape_tensors));
-    std::vector<PartialTensorShape> padded_shapes;
-    padded_shapes.reserve(padded_shape_tensors.size());
-    OP_REQUIRES(ctx,
-                padded_shape_tensors.size() == input->output_shapes().size(),
-                errors::InvalidArgument("Number of padded shapes (",
-                                        padded_shape_tensors.size(),
-                                        ") must match the number of components "
-                                        "in the input dataset's elements (",
-                                        input->output_shapes().size(), ")"));
-    for (const Tensor& padded_shape_t : padded_shape_tensors) {
-      OP_REQUIRES(ctx, TensorShapeUtils::IsVector(padded_shape_t.shape()),
-                  errors::InvalidArgument("All padded shapes must be vectors"));
-      PartialTensorShape padded_shape;
-      OP_REQUIRES_OK(ctx, PartialTensorShape::MakePartialShape(
-                              padded_shape_t.vec<int64>().data(),
-                              padded_shape_t.NumElements(), &padded_shape));
-      padded_shapes.push_back(std::move(padded_shape));
-    }
-
-    OP_REQUIRES(
-        ctx, input->output_dtypes() == output_types_,
-        errors::InvalidArgument("Input dataset and this dataset "
-                                "have different output_types: ",
-                                DataTypeVectorString(input->output_dtypes()),
-                                " and ", DataTypeVectorString(output_types_)));
-
-    for (int i = 0; i < input->output_shapes().size(); ++i) {
-      // Exclude the queue from the tensor_shapes calculation.
-      const PartialTensorShape& tensor_shape = padded_shapes[i];
-      OP_REQUIRES(
-          ctx,
-          IsGreaterEqualToOrCompatibleWith(tensor_shape,
-                                           input->output_shapes()[i]),
-          errors::InvalidArgument("Incompatible input shapes at component ", i,
-                                  " between input dataset this dataset: ",
-                                  input->output_shapes()[i].DebugString(),
-                                  " vs. ", tensor_shape.DebugString()));
-    }
-
-    OpInputList padding_values_list;
-    OP_REQUIRES_OK(ctx,
-                   ctx->input_list("padding_values", &padding_values_list));
-    std::vector<Tensor> padding_values;
-    OP_REQUIRES(ctx,
-                padding_values_list.size() == input->output_shapes().size(),
-                errors::InvalidArgument(
-                    "Number of padding values (", padding_values_list.size(),
-                    ") must match the number of components in the input "
-                    "dataset's elements (",
-                    input->output_shapes().size(), ")"));
-    for (int i = 0; i < padding_values_list.size(); ++i) {
-      const Tensor& padding_value_t = padding_values_list[i];
-      OP_REQUIRES(
-          ctx, TensorShapeUtils::IsScalar(padding_value_t.shape()),
-          errors::InvalidArgument(
-              "All padding values must be scalars; but at component ", i,
-              " saw shape: ", padding_value_t.shape().DebugString()));
-      OP_REQUIRES(ctx, padding_value_t.dtype() == input->output_dtypes()[i],
-                  errors::InvalidArgument(
-                      "Mismatched type between padding value ", i,
-                      " and input dataset's component ", i, ": ",
-                      DataTypeString(padding_value_t.dtype()), " vs. ",
-                      DataTypeString(input->output_dtypes()[i])));
-      padding_values.push_back(padding_value_t);
-    }
-
-    *output = new PrependFromQueueAndPaddedBatchDataset(
-        ctx, batch_size, input, output_types_, padded_shapes,
-        std::move(padding_values));
-  }
-
- private:
-  DataTypeVector output_types_;
-};
-
-REGISTER_KERNEL_BUILDER(
-    Name("PrependFromQueueAndPaddedBatchDataset").Device(DEVICE_CPU),
-    PrependFromQueueAndPaddedBatchDatasetOp);
-
-class EnqueueInQueueDatasetOp : public OpKernel {
- public:
-  explicit EnqueueInQueueDatasetOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
-  void Compute(OpKernelContext* ctx) override {
-    using TensorQueueInserter =
-        PrependFromQueueAndPaddedBatchDataset::Iterator::TensorQueueInserter;
-
-    // TODO(ebrevdo): accept list of sequence lengths to do proper
-    // sub-slicing of tensors for placement into the queue?
-    const Tensor& tensor_queue_t = ctx->input(0);
-    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(tensor_queue_t.shape()),
-                errors::InvalidArgument("queue must be a vector, saw shape: ",
-                                        tensor_queue_t.shape().DebugString()));
-    std::vector<const TensorQueueInserter*> inserters;
-    const int64 batch_size = tensor_queue_t.NumElements();
-    inserters.reserve(batch_size);
-    const Variant* variants = tensor_queue_t.flat<Variant>().data();
-    for (int i = 0; i < batch_size; ++i) {
-      const auto* inserter = variants[i].get<TensorQueueInserter>();
-      OP_REQUIRES(ctx, inserter != nullptr,
-                  errors::InvalidArgument(
-                      "Could not access TensorQueueInserter from queue[", i,
-                      "].  Received variant: ", variants[i].DebugString()));
-      inserters.push_back(inserter);
-    }
-
-    OpInputList components;
-    OP_REQUIRES_OK(ctx, ctx->input_list("components", &components));
-    for (int i = 0; i < components.size(); ++i) {
-      OP_REQUIRES(
-          ctx,
-          components[i].dims() > 0 && components[i].dim_size(0) == batch_size,
-          errors::InvalidArgument(
-              "Expected component ", i, " to have batched shape [", batch_size,
-              ",...], but saw shape: ", components[i].shape().DebugString()));
-    }
-    std::vector<TensorShape> element_shapes;
-    for (int i = 0; i < components.size(); ++i) {
-      TensorShape element_shape = components[i].shape();
-      element_shape.RemoveDim(0);
-      element_shapes.push_back(std::move(element_shape));
-    }
-    for (int64 b = 0; b < batch_size; ++b) {
-      std::vector<Tensor> tensors;
-      tensors.reserve(components.size());
-      for (int i = 0; i < components.size(); ++i) {
-        Tensor t(components[i].dtype(), element_shapes[i]);
-        OP_REQUIRES_OK(ctx,
-                       batch_util::CopySliceToElement(components[i], &t, b));
-        tensors.push_back(std::move(t));
-      }
-      // TODO(ebrevdo): Acquire the lock once for all inserters with
-      // the same underlying queue?  Add InsertLocked?
-      OP_REQUIRES_OK(ctx, inserters[b]->Insert(tensors));
-    }
-  }
-};
-
-REGISTER_KERNEL_BUILDER(Name("EnqueueInQueueDataset").Device(DEVICE_CPU),
-                        EnqueueInQueueDatasetOp);
-
-}  // namespace
-}  // namespace data
-}  // namespace tensorflow
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index a8ad79dd01..e0ec389d28 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21048,24 +21048,6 @@ op {
     type: DT_STRING
   }
 }
-op {
-  name: "EnqueueInQueueDataset"
-  input_arg {
-    name: "queue"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "components"
-    type_list_attr: "Tcomponents"
-  }
-  attr {
-    name: "Tcomponents"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "EnsureShape"
   input_arg {
@@ -22523,6 +22505,80 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalRandomDataset"
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "seed2"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalScanDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "initial_state"
+    type_list_attr: "Tstate"
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Tstate"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalSetStatsAggregatorDataset"
   input_arg {
@@ -40784,48 +40840,6 @@ op {
     minimum: 1
   }
 }
-op {
-  name: "PrependFromQueueAndPaddedBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "padded_shapes"
-    type: DT_INT64
-    number_attr: "N"
-  }
-  input_arg {
-    name: "padding_values"
-    type_list_attr: "Toutput_types"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "Toutput_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "PreventGradient"
   input_arg {
@@ -45102,34 +45116,6 @@ op {
   }
   is_stateful: true
 }
-op {
-  name: "RandomDataset"
-  input_arg {
-    name: "seed"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "seed2"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "RandomGamma"
   input_arg {
@@ -57369,52 +57355,6 @@ op {
     }
   }
 }
-op {
-  name: "ScanDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "initial_state"
-    type_list_attr: "Tstate"
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Tstate"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "ScatterAdd"
   input_arg {
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index f00274496c..b2ea637464 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -172,18 +172,6 @@ REGISTER_OP("PrefetchDataset")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("ScanDataset")
-    .Input("input_dataset: variant")
-    .Input("initial_state: Tstate")
-    .Input("other_arguments: Targuments")
-    .Output("handle: variant")
-    .Attr("f: func")
-    .Attr("Tstate: list(type) >= 1")
-    .Attr("Targuments: list(type) >= 0")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("FlatMapDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
@@ -343,22 +331,6 @@ REGISTER_OP("RangeDataset")
       return shape_inference::ScalarShape(c);
     });
 
-REGISTER_OP("RandomDataset")
-    .Input("seed: int64")
-    .Input("seed2: int64")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
-                      // stateful to inhibit constant folding.
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle unused;
-      // buffer_size, seed, and seed2 should be scalars.
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
-      return shape_inference::ScalarShape(c);
-    });
-
 REGISTER_OP("ShuffleDataset")
     .Input("input_dataset: variant")
     .Input("buffer_size: int64")
@@ -613,36 +585,6 @@ REGISTER_OP("DeserializeIterator")
     .Input("serialized: variant")
     .SetShapeFn(shape_inference::NoOutputs);
 
-REGISTER_OP("PrependFromQueueAndPaddedBatchDataset")
-    .Input("input_dataset: variant")
-    .Input("batch_size: int64")
-    .Input("padded_shapes: N * int64")
-    .Input("padding_values: Toutput_types")
-    .Output("handle: variant")
-    .Attr("Toutput_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .Attr("N: int >= 1")
-    // TODO(ebrevdo): Validate that `padded_shapes` are all vectors, the lengths
-    // of `Toutput_types` and `output_shapes` are `N`, that the
-    // length of `output_types` is `N`, the `output_shapes` are
-    // (as far as possible to tell statically) compatible with `padded_shapes`,
-    // and that `padding_values` are all scalars.
-    .SetShapeFn([](shape_inference::InferenceContext* c) {
-      shape_inference::ShapeHandle unused;
-      // batch_size should be a scalar.
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
-      return shape_inference::ScalarShape(c);
-    });
-
-REGISTER_OP("EnqueueInQueueDataset")
-    .Input("queue: variant")
-    .Input("components: Tcomponents")
-    .Attr("Tcomponents: list(type) >= 1")
-    .SetIsStateful()  // To avoid CSE on multiple calls to Enqueue.
-    // TODO(ebrevdo): SetShapeFn to test input dtypes and shapes by
-    // reading from queue handle (is that even possible?).
-    .SetShapeFn(shape_inference::NoOutputs);
-
 REGISTER_OP("DatasetToGraph")
     .Input("input_dataset: variant")
     .Output("graph: string")
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 922f6e18a5..dbd577400e 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -246,6 +246,34 @@ REGISTER_OP("ExperimentalParseExampleDataset")
     .Attr("sloppy: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalRandomDataset")
+    .Input("seed: int64")
+    .Input("seed2: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetIsStateful()  // TODO(b/65524810): Source dataset ops must be marked
+                      // stateful to inhibit constant folding.
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // buffer_size, seed, and seed2 should be scalars.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
+REGISTER_OP("ExperimentalScanDataset")
+    .Input("input_dataset: variant")
+    .Input("initial_state: Tstate")
+    .Input("other_arguments: Targuments")
+    .Output("handle: variant")
+    .Attr("f: func")
+    .Attr("Tstate: list(type) >= 1")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalSetStatsAggregatorDataset")
     .Input("input_dataset: variant")
     .Input("stats_aggregator: resource")
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index a0ab302335..bc54109eed 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -64,8 +64,8 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:experimental_dataset_ops_gen",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:random_seed",
         "//tensorflow/python:tensor_shape",
@@ -297,7 +297,7 @@ py_library(
     srcs = ["scan_ops.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:experimental_dataset_ops_gen",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:function",
         "//tensorflow/python/data/ops:dataset_ops",
diff --git a/tensorflow/python/data/experimental/ops/random_ops.py b/tensorflow/python/data/experimental/ops/random_ops.py
index 7bf703502b..a4359f356b 100644
--- a/tensorflow/python/data/experimental/ops/random_ops.py
+++ b/tensorflow/python/data/experimental/ops/random_ops.py
@@ -24,7 +24,7 @@ from tensorflow.python.data.util import random_seed
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -38,7 +38,7 @@ class RandomDatasetV2(dataset_ops.DatasetSource):
     self._seed, self._seed2 = random_seed.get_seed(seed)
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.random_dataset(
+    return gen_experimental_dataset_ops.experimental_random_dataset(
         seed=self._seed,
         seed2=self._seed2,
         **dataset_ops.flat_structure(self))
diff --git a/tensorflow/python/data/experimental/ops/scan_ops.py b/tensorflow/python/data/experimental/ops/scan_ops.py
index 1194238e2f..0b7b2d142f 100644
--- a/tensorflow/python/data/experimental/ops/scan_ops.py
+++ b/tensorflow/python/data/experimental/ops/scan_ops.py
@@ -24,7 +24,7 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -130,7 +130,7 @@ class _ScanDataset(dataset_ops.UnaryDataset):
 
   def _as_variant_tensor(self):
     input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
-    return gen_dataset_ops.scan_dataset(
+    return gen_experimental_dataset_ops.experimental_scan_dataset(
         input_t,
         nest.flatten(sparse.serialize_sparse_tensors(self._initial_state)),
         self._scan_func.captured_inputs,
-- 
GitLab


From 2938010901ce224b0c43c071a2815f64bfdc7c7f Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 3 Dec 2018 16:17:59 -0800
Subject: [PATCH 1240/1554] Allow non-TableInitializerBases as table
 initializers again

PiperOrigin-RevId: 223882977
---
 tensorflow/python/ops/lookup_ops.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 0ef1ec8c00..758cb8041d 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -161,7 +161,9 @@ class InitializableLookupTableBase(LookupInterface):
     self._default_value = ops.convert_to_tensor(
         default_value, dtype=self._value_dtype)
     self._default_value.get_shape().merge_with(tensor_shape.scalar())
-    self._initializer = self._track_checkpointable(initializer, "_initializer")
+    if isinstance(initializer, checkpointable_base.CheckpointableBase):
+      self._initializer = self._track_checkpointable(
+          initializer, "_initializer")
     self._resource_handle = self.create_resource()
     self._init_op = self.initialize()
 
-- 
GitLab


From 8c6b4484a0965c4dca0ad2f59469acadcabb2ff2 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 3 Dec 2018 16:20:34 -0800
Subject: [PATCH 1241/1554] Automated rollback of commit
 2ce0bec9da49d1848ff72233e187b2d12e59b182

PiperOrigin-RevId: 223883381
---
 .../python/kernel_tests/matmul_op_test.py     | 52 +++++++++++--------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py
index 2a04d452fe..983f463f5e 100644
--- a/tensorflow/python/kernel_tests/matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/matmul_op_test.py
@@ -24,7 +24,8 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import gradient_checker_v2
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -40,11 +41,11 @@ class MatVecTest(test_lib.TestCase):
   def testTwoByTwoCase(self):
     a = np.array([[1, 2], [3, 4]])
     b = np.array([5, 6])
-
-    c = math_ops.matvec(a, b)
-    self.assertAllEqual((2,), c.shape)
-
-    self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c)
+    with self.cached_session():
+      c = math_ops.matvec(a, b)
+      self.assertAllEqual((2,), c.shape)
+      c_ = self.evaluate(c)
+    self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c_)
 
 
 def _AddTest(test, op_name, testcase_name, fn):
@@ -84,13 +85,17 @@ def _GetMatMulTest(a_np_, b_np_, use_static_shape_, **kwargs_):
     # np.matrix(a_np_) * np.matrix(b_np_)
     effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_)
     effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_)
-    with test_util.device(use_gpu=use_gpu):
+    with self.session(use_gpu=use_gpu) as sess:
       if use_static_shape_:
         a = constant_op.constant(effective_a_np)
         b = constant_op.constant(effective_b_np)
-        tf_val = math_ops.matmul(a, b, **kwargs_)
+        res = math_ops.matmul(a, b, **kwargs_)
+        tf_val = self.evaluate(res)
       else:
-        tf_val = math_ops.matmul(effective_a_np, effective_b_np, **kwargs_)
+        a = array_ops.placeholder(a_np_.dtype)
+        b = array_ops.placeholder(b_np_.dtype)
+        res = math_ops.matmul(a, b, **kwargs_)
+        tf_val = sess.run(res, feed_dict={a: effective_a_np, b: effective_b_np})
 
     self.assertAllCloseAccordingToType(
         tf_val,
@@ -114,22 +119,27 @@ def _GetMatMulGradientTest(a_np_, b_np_, use_static_shape_, **kwargs_):
       self.skipTest("Skipping infeasible gradient test.")
 
     # Transpose and possibly conjugate a_np_ and b_np_ according to the
-    # attributes such that tf.matmul(a, b, **kwargs) results in a valid matrix
-    # multiplication and produces the same result as
+    # attributes such that tf.matmul(effective_a_np, effective_b_np, **kwargs)
+    # results in a valid matrix multiplication and produces the same result as
     # np.matrix(a_np_) * np.matrix(b_np_)
-    a = _GetTransposedMatrices(a_np_, "a", kwargs_)
-    b = _GetTransposedMatrices(b_np_, "b", kwargs_)
+    effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_)
+    effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_)
 
     epsilon = np.finfo(a_np_.dtype).eps
     delta = epsilon**(1.0 / 3.0)
     tol = 20 * delta
     with self.session(use_gpu=True):
-      theoretical, numerical = gradient_checker_v2.compute_gradient(
-          lambda x: math_ops.matmul(x, b, **kwargs_), [a])
-      self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
-      theoretical, numerical = gradient_checker_v2.compute_gradient(
-          lambda x: math_ops.matmul(a, x, **kwargs_), [b])
-      self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
+      a = constant_op.constant(effective_a_np)
+      b = constant_op.constant(effective_b_np)
+      res = math_ops.matmul(a, b, **kwargs_)
+      for x, x_init in [a, effective_a_np], [b, effective_b_np]:
+        theoretical, numerical = gradient_checker.compute_gradient(
+            x,
+            x_init.shape,
+            res, [a_np_.shape[0], b_np_.shape[1]],
+            x_init_value=x_init,
+            delta=delta)
+        self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
 
   return Test
 
@@ -213,8 +223,8 @@ class MatMulInfixOperatorTest(test_lib.TestCase):
     b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]])
     c = infix_matmul(a, b)
     d = math_ops.matmul(a, b)
-
-    self.assertAllEqual(c, d)
+    with self.cached_session():
+      self.assertAllEqual(c.eval(), self.evaluate(d))
 
 
 if __name__ == "__main__":
-- 
GitLab


From 5f882df5dafb98580c6d78273b2a40c9a36dceee Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Mon, 3 Dec 2018 16:22:10 -0800
Subject: [PATCH 1242/1554] Remove

tensorflow_estimator/contrib/estimator/python/estimator/hooks.py, and /hooks_test.py

as they are exist as

tensorflow_estimator/python/estimator/hooks.py, and /hooks_test.py.

Also export tf.estimator.experimental.make_stop_at_checkpoint_step_hook.

PiperOrigin-RevId: 223883621
---
 .../api/golden/v1/tensorflow.estimator.experimental.pbtxt     | 4 ++++
 .../api/golden/v2/tensorflow.estimator.experimental.pbtxt     | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
index 2a9a0346d7..f0fd7ce782 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.pbtxt
@@ -28,6 +28,10 @@ tf_module {
     name: "make_early_stopping_hook"
     argspec: "args=[\'estimator\', \'should_stop_fn\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'60\', \'None\'], "
   }
+  member_method {
+    name: "make_stop_at_checkpoint_step_hook"
+    argspec: "args=[\'estimator\', \'last_step\', \'wait_after_file_check_secs\'], varargs=None, keywords=None, defaults=[\'30\'], "
+  }
   member_method {
     name: "stop_if_higher_hook"
     argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
index 2a9a0346d7..f0fd7ce782 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.pbtxt
@@ -28,6 +28,10 @@ tf_module {
     name: "make_early_stopping_hook"
     argspec: "args=[\'estimator\', \'should_stop_fn\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'60\', \'None\'], "
   }
+  member_method {
+    name: "make_stop_at_checkpoint_step_hook"
+    argspec: "args=[\'estimator\', \'last_step\', \'wait_after_file_check_secs\'], varargs=None, keywords=None, defaults=[\'30\'], "
+  }
   member_method {
     name: "stop_if_higher_hook"
     argspec: "args=[\'estimator\', \'metric_name\', \'threshold\', \'eval_dir\', \'min_steps\', \'run_every_secs\', \'run_every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'60\', \'None\'], "
-- 
GitLab


From d7d9aaf9bdcf2587f3c2c9e3afd5c6554f9bc92f Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Mon, 3 Dec 2018 16:22:57 -0800
Subject: [PATCH 1243/1554] TensorFlow DistributionStrategy: Fix bugs to get DS
 working in Keras in eager.

PiperOrigin-RevId: 223883750
---
 .../contrib/distribute/python/combinations.py |   2 +-
 .../distribute/python/examples/keras_mnist.py |  10 +-
 .../contrib/distribute/python/keras_test.py   | 131 ++++++++++--------
 .../python/mirrored_strategy_multigpu_test.py |  32 ++++-
 tensorflow/python/distribute/device_util.py   |   2 +-
 .../python/distribute/mirrored_strategy.py    |   2 +-
 tensorflow/python/distribute/values.py        |   2 +
 tensorflow/python/keras/engine/training.py    |   6 +-
 .../python/keras/engine/training_arrays.py    |  25 +++-
 tensorflow/python/training/optimizer.py       |   6 +-
 10 files changed, 141 insertions(+), 77 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index c5ce29a436..365ce5cdec 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -192,7 +192,7 @@ def _augment_with_special_arguments(test_method):
         kwargs_to_pass[arg] = kwargs[arg]
 
     if mode == "eager":
-      with ops.Graph().as_default(), context.eager_mode():
+      with context.eager_mode():
         if distribution:
           kwargs_to_pass["distribution"] = distribution.strategy
         test_method(**kwargs_to_pass)
diff --git a/tensorflow/contrib/distribute/python/examples/keras_mnist.py b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
index 8b6487252d..60fda99664 100644
--- a/tensorflow/contrib/distribute/python/examples/keras_mnist.py
+++ b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
@@ -20,6 +20,10 @@ from __future__ import print_function
 import tensorflow as tf
 
 
+from tensorflow.python.distribute import mirrored_strategy
+from tensorflow.python.keras.optimizer_v2 import rmsprop
+
+
 NUM_CLASSES = 10
 
 
@@ -109,10 +113,10 @@ def main(_):
 
   # Instantiate the MirroredStrategy object. If we don't specify `num_gpus` or
   # the `devices` argument then all the GPUs available on the machine are used.
-  strategy = tf.contrib.distribute.MirroredStrategy(['/gpu:0', '/cpu:0'])
+  # TODO(priyag): Use `tf.distribute.MirroredStrategy` once available.
+  strategy = mirrored_strategy.MirroredStrategy(['/gpu:0', '/cpu:0'])
 
-  # TODO(priyag): Use RMSPropOptimizer when it works with eager mode.
-  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
+  optimizer = rmsprop.RMSProp(learning_rate=0.001)
 
   # Compile the model by passing the distribution strategy object to the
   # `distribute` argument. `fit`, `evaluate` and `predict` will be distributed
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 435f10358f..00a220880b 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.distribute.python import tpu_strategy
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import values
+from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.estimator import keras as keras_lib
 from tensorflow.python.estimator import run_config as run_config_lib
@@ -276,47 +277,60 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
   return training_inputs, eval_inputs, predict_inputs
 
 
-strategies = [combinations.default_strategy,
-              combinations.one_device_strategy,
-              combinations.mirrored_strategy_with_gpu_and_cpu,
-              combinations.mirrored_strategy_with_two_gpus,
-              combinations.core_mirrored_strategy_with_gpu_and_cpu,
-              combinations.core_mirrored_strategy_with_two_gpus,
-              combinations.tpu_strategy,  # steps_per_run=2
-              combinations.tpu_strategy_one_step]
+strategies_minus_tpu = [
+    combinations.default_strategy,
+    combinations.one_device_strategy,
+    combinations.mirrored_strategy_with_gpu_and_cpu,
+    combinations.mirrored_strategy_with_two_gpus,
+    combinations.core_mirrored_strategy_with_gpu_and_cpu,
+    combinations.core_mirrored_strategy_with_two_gpus]
+
+tpu_strategies = [
+    combinations.tpu_strategy,  # steps_per_run=2
+    combinations.tpu_strategy_one_step]
 
 
 def strategy_minus_tpu_combinations():
   return combinations.combine(
-      distribution=[combinations.default_strategy,
-                    combinations.one_device_strategy,
-                    combinations.mirrored_strategy_with_gpu_and_cpu,
-                    combinations.mirrored_strategy_with_two_gpus,
-                    combinations.core_mirrored_strategy_with_gpu_and_cpu,
-                    combinations.core_mirrored_strategy_with_two_gpus],
-      mode=['graph'])
+      distribution=strategies_minus_tpu,
+      mode=['graph', 'eager'])
 
 
-def strategy_combinations():
+def tpu_strategy_combinations():
   return combinations.combine(
-      distribution=strategies,
+      distribution=tpu_strategies,
       mode=['graph'])
 
 
-def strategy_and_optimizer_combinations():
-  return combinations.combine(
-      distribution=strategies,
-      optimizer=[combinations.adagrad_optimizer_v1_fn,
-                 combinations.adam_optimizer_v1_fn,
-                 combinations.gradient_descent_optimizer_v1_fn,
-                 combinations.rmsprop_optimizer_v1_fn],
-      mode=['graph'])
+def all_strategy_combinations():
+  return strategy_minus_tpu_combinations() + tpu_strategy_combinations()
 
 
-def strategy_and_inputs():
+# TODO(priyag): Add v2 optimizers here.
+def strategy_and_optimizer_combinations():
+  return combinations.times(
+      all_strategy_combinations(),
+      combinations.combine(
+          optimizer=[combinations.adagrad_optimizer_v1_fn,
+                     combinations.adam_optimizer_v1_fn,
+                     combinations.gradient_descent_optimizer_v1_fn,
+                     combinations.rmsprop_optimizer_v1_fn]))
+
+
+def strategy_and_input_combinations():
+  return (
+      combinations.times(
+          combinations.combine(distribution=strategies_minus_tpu),
+          combinations.combine(mode=['graph'], use_numpy=[True, False])
+          + combinations.combine(mode=['eager'], use_numpy=[False])) +
+      combinations.times(
+          combinations.combine(distribution=tpu_strategies),
+          combinations.combine(mode=['graph'], use_numpy=[True, False])))
+
+
+def strategy_for_numpy_input_combinations():
   return combinations.combine(
-      distribution=strategies,
-      use_numpy=[True, False],
+      distribution=strategies_minus_tpu + tpu_strategies,
       mode=['graph'])
 
 
@@ -471,7 +485,7 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
 class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                               parameterized.TestCase):
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_creating_var_with_numpy_arrays(self, distribution):
     with self.cached_session():
       x = np.asarray(np.random.random((64, 3)), dtype=np.float32)
@@ -480,7 +494,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # Verify that the numpy value is copied to the variable.
       self.assertAllEqual(x, val)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
     # that use per_core_batch_size
@@ -511,7 +525,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_63_samples, steps=None, batch_size=None)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_with_steps_no_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -557,7 +571,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
           distributed_training_utils.get_input_params(
               distribution, input_63_samples, steps=1, batch_size=None)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_no_steps_with_batch_size(self,
                                                              distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -591,7 +605,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=None, batch_size=3)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_with_steps_with_batch_size(self,
                                                                distribution):
     with self.cached_session():
@@ -608,7 +622,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         distributed_training_utils.get_input_params(
             distribution, input_64_samples, steps=10, batch_size=13)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calling_model_with_numpy_arrays(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -639,7 +653,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # with batch_size
       model.predict(inputs, batch_size=8)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calling_model_with_nested_numpy_arrays(self, distribution):
     with self.cached_session():
       model = multi_input_output_model()
@@ -673,7 +687,8 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
       # with batch_size
       model.predict(inputs, batch_size=8)
 
-  @combinations.generate(strategy_minus_tpu_combinations())
+  @combinations.generate(combinations.combine(
+      distribution=strategies_minus_tpu, mode=['graph']))
   def test_numpy_with_sample_weights(self, distribution):
     model = get_model()
     optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
@@ -687,7 +702,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
     model.fit(inputs, targets, sample_weight=sample_weights, epochs=1,
               steps_per_epoch=2, verbose=1)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_flatten_predict_outputs(self, distribution):
     with self.cached_session():
       model = multi_input_output_model()
@@ -715,7 +730,7 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
 class TestDistributionStrategyWithDatasets(test.TestCase,
                                            parameterized.TestCase):
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_calling_model_on_same_dataset(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -734,7 +749,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
                 validation_data=dataset, validation_steps=2)
       model.predict(get_predict_dataset(distribution), steps=2)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_model_interleaved_eval_same_as_direct_eval(self, distribution):
     with self.cached_session():
       user_controlled_model = get_model()
@@ -782,7 +797,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       distribution=[
           combinations.mirrored_strategy_with_gpu_and_cpu,
           combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_fit_with_tuple_and_dict_dataset_inputs(self, distribution):
     with self.cached_session():
       model = multi_input_output_model()
@@ -814,7 +829,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
       model.fit(dataset_dict, epochs=1, steps_per_epoch=2, verbose=1)
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_fit_eval_and_predict_methods_on_dataset(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -867,7 +882,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       distribution=[
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_two_gpus],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_dataset_wrong_input_shape(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -889,7 +904,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
 
   @combinations.generate(combinations.combine(
       distribution=[combinations.mirrored_strategy_with_two_gpus],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_dataset_no_batch_input_validation(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -930,7 +945,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
       distribution=[
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_two_gpus],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_learning_phase_value(self, distribution):
     # TODO(anjalisridhar): Modify this test to use Lambdas since we can compare
     # meaningful values. Currently we don't pass the learning phase if the
@@ -1002,7 +1017,7 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
       distribution=[
           combinations.mirrored_strategy_with_gpu_and_cpu,
           combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_validating_dataset_input_tensors_with_shape_mismatch(self,
                                                                 distribution):
     with self.cached_session():
@@ -1025,7 +1040,7 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
       distribution=[
           combinations.mirrored_strategy_with_gpu_and_cpu,
           combinations.core_mirrored_strategy_with_gpu_and_cpu],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_validating_dataset_input_tensors_with_dtype_mismatch(self,
                                                                 distribution):
     with self.cached_session():
@@ -1048,7 +1063,7 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
       distribution=[
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_two_gpus],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_unsupported_features(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -1097,7 +1112,7 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
       distribution=[
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_two_gpus],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_calling_with_unsupported_predefined_callbacks(self, distribution):
     with self.cached_session():
       model = get_model()
@@ -1139,7 +1154,7 @@ class TestDistributionStrategyWithLossMasking(test.TestCase,
       distribution=[
           combinations.mirrored_strategy_with_two_gpus,
           combinations.core_mirrored_strategy_with_two_gpus],
-      mode=['graph']))
+      mode=['graph', 'eager']))
   def test_masking(self, distribution):
     with self.cached_session():
       np.random.seed(1337)
@@ -1163,7 +1178,7 @@ class TestDistributionStrategyWithLossMasking(test.TestCase,
 class TestDistributionStrategyWithNormalizationLayer(
     test.TestCase, parameterized.TestCase):
 
-  @combinations.generate(strategy_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_batchnorm_correctness(self, distribution):
     with self.cached_session():
       model = keras.models.Sequential()
@@ -1195,7 +1210,8 @@ class TestDistributionStrategyWithNormalizationLayer(
 class TestDistributionStrategyCorrectness(test.TestCase,
                                           parameterized.TestCase):
 
-  @combinations.generate(strategy_combinations())
+  # TODO(b/120186218): Enable this for eager once metrics are working.
+  @combinations.generate(strategy_for_numpy_input_combinations())
   def test_metric_correctness(self, distribution):
     with self.cached_session():
       keras.backend.set_image_data_format('channels_last')
@@ -1227,7 +1243,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       history = model.fit(x=train_dataset, epochs=1, steps_per_epoch=10)
       self.assertEqual(history.history['binary_accuracy'], [1.0])
 
-  @combinations.generate(strategy_and_inputs())
+  @combinations.generate(strategy_and_input_combinations())
   def test_correctness(self, distribution, use_numpy):
     with self.cached_session():
       tolerance = 1e-5
@@ -1292,10 +1308,13 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       # within some limits of tolerance.
       self.assertAllClose(
           wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance)
-      self.assertAllClose(
-          eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance)
-      self.assertAllClose(
-          predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance)
+
+      # TODO(b/120186218): Enable this once metrics are working with eager.
+      if not context.executing_eagerly():
+        self.assertAllClose(
+            eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance)
+        self.assertAllClose(
+            predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 69c49114a6..66512f983e 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -38,6 +38,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.engine import training as keras_training
@@ -190,6 +191,27 @@ class MirroredStrategyVariableCreatorStackTest(
     mode=["graph", "eager"]))
 class MirroredStrategyVariableCreationTest(test.TestCase):
 
+  # TODO(priyag): Modify more tests to use this helper and check more
+  # properties.
+  def _test_mv_properties(self, var, name):
+    self.assertIsInstance(var, values.MirroredVariable)
+    self.assertEqual(name, var.name)
+    for d in var.devices:
+      self.assertEqual(d, var.get(d).device)
+
+  def testVariableInFuncGraph(self, distribution):
+    def model_fn():
+      v = variable_scope.variable(2.0, name="bar")
+      ds_context.get_replica_context().merge_call(lambda _: _)
+      return v
+
+    with func_graph.FuncGraph("fg").as_default(), distribution.scope():
+      v1 = variable_scope.variable(1.0, name="foo")
+      v2 = distribution.extended.call_for_each_replica(model_fn)
+
+    self._test_mv_properties(v1, "foo:0")
+    self._test_mv_properties(v2, "bar:0")
+
   def testSingleVariable(self, distribution):
     def model_fn():
       # This variable should be created only once across the threads because of
@@ -201,8 +223,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     with distribution.scope():
       result = distribution.extended.call_for_each_replica(model_fn)
-      self.assertIsInstance(result, values.MirroredVariable)
-      self.assertEqual("foo:0", result.name)
+      self._test_mv_properties(result, "foo:0")
 
   def testUnnamedVariable(self, distribution):
     def model_fn():
@@ -212,9 +233,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
 
     with distribution.scope():
       result = distribution.extended.call_for_each_replica(model_fn)
-      self.assertIsInstance(result, values.MirroredVariable)
-      # Default name of "Variable" will be used.
-      self.assertEqual("Variable:0", result.name)
+      self._test_mv_properties(result, "Variable:0")
 
   def testMultipleVariables(self, distribution):
     def model_fn():
@@ -227,8 +246,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     with distribution.scope():
       result = distribution.extended.call_for_each_replica(model_fn)
       for i, v in enumerate(result):
-        self.assertIsInstance(v, values.MirroredVariable)
-        self.assertEqual("foo" + str(i) + ":0", v.name)
+        self._test_mv_properties(v, "foo" + str(i) + ":0")
 
   def testMultipleVariablesWithSameCanonicalName(self, distribution):
     def model_fn():
diff --git a/tensorflow/python/distribute/device_util.py b/tensorflow/python/distribute/device_util.py
index 70e1ca4b5d..34474582ad 100644
--- a/tensorflow/python/distribute/device_util.py
+++ b/tensorflow/python/distribute/device_util.py
@@ -50,7 +50,7 @@ def canonicalize(d, default=None):
   # Fill in missing device fields using defaults.
   result = tf_device.DeviceSpec(
       replica=0, task=0, device_type="CPU", device_index=0)
-  if context.executing_eagerly():
+  if ops.executing_eagerly_outside_functions():
     result.job = "localhost"
   if default:
     result.merge_from(tf_device.DeviceSpec.from_string(default))
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 81d25cc559..0775920a7d 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -520,7 +520,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     def _real_mirrored_creator(devices, *args, **kwargs):  # pylint: disable=g-missing-docstring
       index = {}
       for i, d in enumerate(devices):
-        with ops.device(d):
+        with ops.init_scope(), ops.device(d):
           if i > 0:
             # Give replicas meaningful distinct names:
             var0name = index[devices[0]].name.split(":")[0]
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 81787445e7..e1e1f9a208 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -113,6 +113,8 @@ class DistributedDelegate(DistributedValues):
   """A map from device to values; acts as the same type as the values."""
 
   def __getattr__(self, name):
+    # TODO(priyag): This needs to be made robust against pitfalls from mix use
+    # __getattr__ and @property. See b/120402273.
     return getattr(self.get(), name)
 
   # pylint: disable=multiple-statements
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 4fa2c5576f..697286b644 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1947,7 +1947,8 @@ class Model(Network):
         self._distribution_strategy):
       return training_distributed.experimental_test_loop(
           self, iterator=x, verbose=verbose, steps=steps)
-    elif isinstance(x, iterator_ops.EagerIterator):
+    elif (isinstance(x, iterator_ops.EagerIterator) and
+          not self._distribution_strategy):
       return training_generator.evaluate_generator(
           self,
           x,
@@ -2056,7 +2057,8 @@ class Model(Network):
         self._distribution_strategy):
       return training_distributed.experimental_predict_loop(
           self, x, verbose=verbose, steps=steps)
-    elif isinstance(x, iterator_ops.EagerIterator):
+    elif (isinstance(x, iterator_ops.EagerIterator) and
+          not self._distribution_strategy):
       return training_generator.predict_generator(
           self,
           x,
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 390357303e..e8a8e0fc02 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -23,6 +23,7 @@ import functools
 
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
@@ -193,8 +194,22 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
     Feed values for the model in the given mode.
   """
   if model._distribution_strategy:
-    return training_distributed._prepare_feed_values(model, inputs, targets,
-                                                     sample_weights, mode)
+    def get_distributed_inputs():
+      return training_distributed._prepare_feed_values(
+          model, inputs, targets, sample_weights, mode)
+
+    # In the eager case, we want to call the input method per step, so return
+    # a lambda from here that can be called. Note that this is applicable only
+    # in Distribution Strategy case as it follows the same code path for both
+    # eager and graph modes.
+    # TODO(priyag,omalleyt): Either we should move the training DS with
+    # EagerIterator to use training_generator code path, or figure out how to
+    # set a symbolic Iterator out of a Dataset when in eager mode.
+    if context.executing_eagerly():
+      return get_distributed_inputs
+    else:
+      return get_distributed_inputs()
+
   inputs = training_utils.ModelInputs(inputs).as_list()
   targets = targets or []
   sample_weights = sample_weights or []
@@ -309,7 +324,7 @@ def model_iteration(model,
   progbar.params['verbose'] = verbose
 
   # Find beforehand arrays that need sparse-to-dense conversion.
-  if issparse is not None:
+  if issparse is not None and not use_steps:
     indices_for_conversion_to_dense = []
     feed = _get_model_feed(model, mode)
     for i, (input_data, feed_tensor) in enumerate(zip(ins, feed)):
@@ -350,7 +365,9 @@ def model_iteration(model,
 
         # Get outputs.
         try:
-          batch_outs = f(ins)
+          # `ins` can be callable in DistributionStrategy + eager case.
+          actual_inputs = ins() if callable(ins) else ins
+          batch_outs = f(actual_inputs)
         except errors.OutOfRangeError:
           logging.warning('Your dataset iterator ran out of data; '
                           'interrupting training. Make sure that your dataset '
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index bf9a79660b..a9508b862a 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -663,8 +663,10 @@ class Optimizer(
         ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     var_list = [v for _, v in grads_and_vars]
     grads_and_vars = zip(reduced_grads, var_list)
+
     # Note that this is called in a cross-replica context.
-    self._create_slots(var_list)
+    with ops.init_scope():
+      self._create_slots(var_list)
 
     def update(v, g):
       """Apply gradients to a replica variable."""
@@ -754,7 +756,7 @@ class Optimizer(
       # `_resource_apply_dense`.
       distributed_container = var._distributed_container()
       assert distributed_container is not None
-      if context.executing_eagerly():
+      if ops.executing_eagerly_outside_functions():
         key = distributed_container._unique_id
       else:
         key = (distributed_container.graph, distributed_container._shared_name)
-- 
GitLab


From ace119643ea8fd04a5f7c66a16f2e92d6da6953e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 16:27:09 -0800
Subject: [PATCH 1244/1554] Makes the tf2 upgrade script handle
 `tf.parse_single_example` and `tf.parse_example` without inserting a
 compat.v1.

PiperOrigin-RevId: 223884411
---
 tensorflow/tools/compatibility/tf_upgrade_v2.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 21cbf7664c..571367d112 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -443,6 +443,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.io.decode_csv",
         "tf.data.Iterator":
             "tf.compat.v1.data.Iterator",
+        "tf.parse_example":
+            "tf.io.parse_example",
+        "tf.parse_single_example":
+            "tf.io.parse_single_example",
         "tf.nn.fused_batch_norm":
             "tf.compat.v1.nn.fused_batch_norm",
         "tf.nn.softmax_cross_entropy_with_logits_v2":
@@ -551,6 +555,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.strings.length": ["input", "name", "unit"],
         "tf.transpose": ["a", "perm", "name", "conjugate"],
         "tf.tuple": ["tensors", "name", "control_inputs"],
+        "tf.parse_example": [
+            "serialized", "features", "name", "example_names"
+        ],
+        "tf.parse_single_example": [
+            "serialized", "features", "name", "example_names"
+        ],
         "tf.io.parse_example": [
             "serialized", "features", "name", "example_names"
         ],
-- 
GitLab


From d52e08daca3260e0e6e58e028001902e0980c62d Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Mon, 3 Dec 2018 16:37:41 -0800
Subject: [PATCH 1245/1554] Keep tf.train.warm_start api only in v1.

PiperOrigin-RevId: 223886117
---
 tensorflow/python/training/warm_starting_util.py      | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt | 4 ----
 tensorflow/tools/compatibility/renames_v2.py          | 1 +
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/training/warm_starting_util.py b/tensorflow/python/training/warm_starting_util.py
index 19dc04e8fb..8c97f101da 100644
--- a/tensorflow/python/training/warm_starting_util.py
+++ b/tensorflow/python/training/warm_starting_util.py
@@ -360,7 +360,7 @@ def _get_grouped_variables(vars_to_warm_start):
   return grouped_variables
 
 
-@tf_export("train.warm_start")
+@tf_export(v1=["train.warm_start"])
 def warm_start(ckpt_to_initialize_from,
                vars_to_warm_start=".*",
                var_name_to_vocab_info=None,
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 2cc3b7ee1d..3ff4b69d39 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -188,8 +188,4 @@ tf_module {
     name: "summary_iterator"
     argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "warm_start"
-    argspec: "args=[\'ckpt_to_initialize_from\', \'vars_to_warm_start\', \'var_name_to_vocab_info\', \'var_name_to_prev_var_name\'], varargs=None, keywords=None, defaults=[\'.*\', \'None\', \'None\'], "
-  }
 }
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index b1b5d9e8c4..fcbb0fd97d 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -646,6 +646,7 @@ renames = {
     'tf.train.start_queue_runners': 'tf.compat.v1.train.start_queue_runners',
     'tf.train.string_input_producer': 'tf.compat.v1.train.string_input_producer',
     'tf.train.update_checkpoint_state': 'tf.compat.v1.train.update_checkpoint_state',
+    'tf.train.warm_start': 'tf.compat.v1.train.warm_start',
     'tf.train.write_graph': 'tf.io.write_graph',
     'tf.trainable_variables': 'tf.compat.v1.trainable_variables',
     'tf.truncated_normal': 'tf.random.truncated_normal',
-- 
GitLab


From 5ecf02d73c80a807f919564bb4a46e5f582e2bfb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 16:55:41 -0800
Subject: [PATCH 1246/1554] Implement overlap_and_add using lower tensorflow
 primitives.

Also adds a test of unknown shapes.

PiperOrigin-RevId: 223888942
---
 .../signal/reconstruction_ops_test.py         |  57 +++++-
 .../python/ops/signal/reconstruction_ops.py   | 173 +++++++++---------
 2 files changed, 140 insertions(+), 90 deletions(-)

diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index d9b45f67c3..5f77308c2a 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -53,16 +53,63 @@ class ReconstructionOpsTest(test.TestCase):
                             "100000000000000"]
 
   def test_all_ones(self):
-    signal = constant_op.constant(np.ones((3, 5)), dtype=dtypes.int64)
+    signal = array_ops.ones([3, 5])
     reconstruction = reconstruction_ops.overlap_and_add(signal, 2)
 
-    with self.session(use_gpu=True) as sess:
+    self.assertEqual(reconstruction.shape.as_list(), [9])
+
+    with self.session(use_gpu=True):
       output = self.evaluate(reconstruction)
 
       expected_output = np.array([1, 1, 2, 2, 3, 2, 2, 1, 1])
 
       self.assertAllClose(output, expected_output)
 
+  def test_unknown_shapes(self):
+    signal = array_ops.placeholder(dtype=dtypes.int32, shape=[None, None, None])
+    frame_step = array_ops.placeholder(dtype=dtypes.int32, shape=[])
+    reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step)
+
+    self.assertEqual(reconstruction.shape.as_list(), [None, None])
+
+    with self.session(use_gpu=True) as sess:
+      output = sess.run(reconstruction,
+                        feed_dict={signal: np.ones([4, 3, 5]), frame_step: 2})
+
+      expected_output = np.array([[1, 1, 2, 2, 3, 2, 2, 1, 1]] * 4)
+
+      self.assertAllClose(output, expected_output)
+
+  def test_unknown_rank(self):
+    signal = array_ops.placeholder(dtype=dtypes.int32, shape=None)
+    frame_step = array_ops.placeholder(dtype=dtypes.int32, shape=[])
+    reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step)
+
+    self.assertEqual(reconstruction.shape, None)
+
+    with self.session(use_gpu=True) as sess:
+      output = sess.run(reconstruction,
+                        feed_dict={signal: np.ones([4, 3, 5]), frame_step: 2})
+
+      expected_output = np.array([[1, 1, 2, 2, 3, 2, 2, 1, 1]] * 4)
+
+      self.assertAllClose(output, expected_output)
+
+  def test_fast_path(self):
+    signal = array_ops.placeholder(dtype=dtypes.int32, shape=[3, 5])
+    frame_step = 5
+    reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step)
+
+    self.assertEqual(reconstruction.name, "overlap_and_add/fast_path:0")
+
+    with self.session(use_gpu=True) as sess:
+      output = sess.run(reconstruction,
+                        feed_dict={signal: np.ones([3, 5])})
+
+      expected_output = np.ones([15])
+
+      self.assertAllClose(output, expected_output)
+
   @test_util.run_deprecated_v1
   def test_simple(self):
     def make_input(frame_length, num_frames=3):
@@ -100,7 +147,7 @@ class ReconstructionOpsTest(test.TestCase):
                                   dtype=dtypes.int64)
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
-    with self.session(use_gpu=True) as sess:
+    with self.session(use_gpu=True):
       output = self.evaluate(reconstruction)
       string_output = [np.base_repr(x, self.bases[0]) for x in output]
 
@@ -110,7 +157,7 @@ class ReconstructionOpsTest(test.TestCase):
     signal = constant_op.constant(self.powers, dtype=dtypes.int64)
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
-    with self.session(use_gpu=True) as sess:
+    with self.session(use_gpu=True):
       output = self.evaluate(reconstruction)
 
       accumulator = True
@@ -126,7 +173,7 @@ class ReconstructionOpsTest(test.TestCase):
     signal = constant_op.constant(input_matrix, dtype=dtypes.float32)
     reconstruction = reconstruction_ops.overlap_and_add(signal, self.frame_hop)
 
-    with self.session(use_gpu=True) as sess:
+    with self.session(use_gpu=True):
       output = self.evaluate(reconstruction)
 
       string_output = [np.base_repr(int(x), self.bases[0]) for x in
diff --git a/tensorflow/python/ops/signal/reconstruction_ops.py b/tensorflow/python/ops/signal/reconstruction_ops.py
index 0fc7fec239..53c48d9040 100644
--- a/tensorflow/python/ops/signal/reconstruction_ops.py
+++ b/tensorflow/python/ops/signal/reconstruction_ops.py
@@ -18,46 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops.signal import shape_ops
-from tensorflow.python.ops.signal import util_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _shuffle_to_front(input_tensor, k):
-  """Shuffles the last `k` indices of `input_tensor` to the front.
-
-  Transposes `input_tensor` to have the last `k` indices at the front. The input
-  may have arbitrary rank and unknown shape.
-
-  Args:
-    input_tensor: A `Tensor` of arbitrary rank and unknown shape.
-    k: A scalar `Tensor` specifying how many indices to shuffle.
-
-  Returns:
-    A transposed version of `input_tensor` with `k` indices shuffled to the
-    front.
-
-  Raises:
-    ValueError: If `input_tensor` is not at least rank `k` or `k` is not scalar.
-  """
-  k = ops.convert_to_tensor(k, name="k")
-  k.shape.with_rank(0)
-  k_static = tensor_util.constant_value(k)
-  if k_static is not None:
-    input_tensor.shape.with_rank_at_least(k_static)
-
-  rank = array_ops.rank(input_tensor)
-  outer_indices, inner_indices = array_ops.split(math_ops.range(rank),
-                                                 [rank - k, k])
-  permutation = array_ops.concat([inner_indices, outer_indices], 0)
-
-  return array_ops.transpose(input_tensor, perm=permutation)
-
-
 @tf_export("signal.overlap_and_add")
 def overlap_and_add(signal, frame_step, name=None):
   """Reconstructs a signal from a framed representation.
@@ -80,8 +48,8 @@ def overlap_and_add(signal, frame_step, name=None):
     frames of `signal`'s inner-most two dimensions.
 
   Raises:
-    ValueError: If `signal`'s rank is less than 2, `frame_step` is not a scalar
-      integer or `frame_step` is greater than `frame_length`.
+    ValueError: If `signal`'s rank is less than 2, or `frame_step` is not a
+      scalar integer.
   """
   with ops.name_scope(name, "overlap_and_add", [signal, frame_step]):
     signal = ops.convert_to_tensor(signal, name="signal")
@@ -97,56 +65,91 @@ def overlap_and_add(signal, frame_step, name=None):
     # All dimensions that are not part of the overlap-and-add. Can be empty for
     # rank 2 inputs.
     outer_dimensions = signal_shape[:-2]
+    outer_rank = array_ops.size(outer_dimensions)
+
+    def full_shape(inner_shape):
+      return array_ops.concat([outer_dimensions, inner_shape], 0)
 
-    # If frame_length and frame_step are known at graph construction time, check
-    # frame_step is less than or equal to frame_length.
-    frame_step_static = tensor_util.constant_value(frame_step)
-    if (frame_step_static is not None and signal.shape.ndims is not None and
-        signal.shape.dims[-1].value is not None):
-      if frame_step_static > signal.shape.dims[-1].value:
-        raise ValueError(
-            "frame_step (%d) must be less than or equal to "
-            "frame_length (%d)" % (
-                frame_step_static, signal.shape.dims[-1].value))
-      # If frame_length is equal to frame_step, there's no overlap so just
-      # reshape the tensor.
-      if frame_step_static == signal.shape.dims[-1].value:
-        return array_ops.reshape(signal, array_ops.concat(
-            [outer_dimensions, [-1]], 0))
-
-    signal_rank = array_ops.rank(signal)
-    frames = signal_shape[-2]
     frame_length = signal_shape[-1]
+    frames = signal_shape[-2]
 
-    subframe_length = util_ops.gcd(frame_length, frame_step)
-    subframe_step = frame_step // subframe_length
-    subframes_per_frame = frame_length // subframe_length
-    output_size = frame_step * (frames - 1) + frame_length
-    output_subframes = output_size // subframe_length
-
-    # To avoid overlap-adding sample-by-sample, we overlap-add at the "subframe"
-    # level, where a subframe is gcd(frame_length, frame_step). Reshape signal
-    # from [..., frames, frame_length] into [..., subframes, subframe_length].
-    subframe_shape = array_ops.concat(
-        [outer_dimensions, [-1, subframe_length]], 0)
-    subframe_signal = array_ops.reshape(signal, subframe_shape)
-
-    # Now we shuffle the last [subframes, subframe_length] dimensions to the
-    # front.
-    # TODO(rjryan): Add an axis argument to unsorted_segment_sum so we can
-    # avoid this pair of transposes.
-    subframe_signal = _shuffle_to_front(subframe_signal, 2)
-
-    # Use unsorted_segment_sum to add overlapping subframes together.
-    segment_ids = array_ops.reshape(shape_ops.frame(
-        math_ops.range(output_subframes), subframes_per_frame, subframe_step,
-        pad_end=False), [-1])
-    result = math_ops.unsorted_segment_sum(subframe_signal, segment_ids,
-                                           num_segments=output_subframes)
-
-    # result is a [subframes, subframe_length, ...outer_dimensions] tensor. We
-    # return a [...outer_dimensions, output_size] tensor with a transpose and
-    # reshape.
-    result_shape = array_ops.concat([outer_dimensions, [output_size]], 0)
-    return array_ops.reshape(_shuffle_to_front(result, signal_rank - 2),
-                             result_shape)
+    # Compute output length.
+    output_length = frame_length + frame_step * (frames - 1)
+
+    # If frame_length is equal to frame_step, there's no overlap so just
+    # reshape the tensor.
+    frame_step_static = tensor_util.constant_value(frame_step)
+    if (frame_step_static is not None and
+        frame_step_static == signal.shape.dims[-1].value):
+      output_shape = full_shape([output_length])
+      return array_ops.reshape(signal, output_shape, name="fast_path")
+
+    # The following code is documented using this example:
+    #
+    # frame_step = 2
+    # signal.shape = (3, 5)
+    # a b c d e
+    # f g h i j
+    # k l m n o
+
+    # Compute the number of segments, per frame.
+    segments = -(-frame_length // frame_step)  # Divide and round up.
+
+    # Pad the frame_length dimension to a multiple of the frame step.
+    # Pad the frames dimension by `segments` so that signal.shape = (6, 6)
+    # a b c d e 0
+    # f g h i j 0
+    # k l m n o 0
+    # 0 0 0 0 0 0
+    # 0 0 0 0 0 0
+    # 0 0 0 0 0 0
+    paddings = [[0, segments], [0, segments * frame_step - frame_length]]
+    outer_paddings = array_ops.zeros([outer_rank, 2], dtypes.int32)
+    paddings = array_ops.concat([outer_paddings, paddings], 0)
+    signal = array_ops.pad(signal, paddings)
+
+    # Reshape so that signal.shape = (3, 6, 2)
+    # ab cd e0
+    # fg hi j0
+    # kl mn o0
+    # 00 00 00
+    # 00 00 00
+    # 00 00 00
+    shape = full_shape([frames + segments, segments, frame_step])
+    signal = array_ops.reshape(signal, shape)
+
+    # Transpose dimensions so that signal.shape = (3, 6, 2)
+    # ab fg kl 00 00 00
+    # cd hi mn 00 00 00
+    # e0 j0 o0 00 00 00
+    perm = array_ops.concat(
+        [math_ops.range(outer_rank), outer_rank + [1, 0, 2]], 0)
+    signal = array_ops.transpose(signal, perm)
+
+    # Reshape so that signal.shape = (18, 2)
+    # ab fg kl 00 00 00 cd hi mn 00 00 00 e0 j0 o0 00 00 00
+    shape = full_shape([(frames + segments) * segments, frame_step])
+    signal = array_ops.reshape(signal, shape)
+
+    # Truncate so that signal.shape = (15, 2)
+    # ab fg kl 00 00 00 cd hi mn 00 00 00 e0 j0 o0
+    signal = signal[..., :(frames + segments - 1) * segments, :]
+
+    # Reshape so that signal.shape = (3, 5, 2)
+    # ab fg kl 00 00
+    # 00 cd hi mn 00
+    # 00 00 e0 j0 o0
+    shape = full_shape([segments, (frames + segments - 1), frame_step])
+    signal = array_ops.reshape(signal, shape)
+
+    # Now, reduce over the columns, to achieve the desired sum.
+    signal = math_ops.reduce_sum(signal, -3)
+
+    # Flatten the array.
+    shape = full_shape([(frames + segments - 1) * frame_step])
+    signal = array_ops.reshape(signal, shape)
+
+    # Truncate to final length.
+    signal = signal[..., :output_length]
+
+    return signal
-- 
GitLab


From 8ecabf5888da937f61d1b8cc0aedc99b453b821c Mon Sep 17 00:00:00 2001
From: tomguluson92 <314913739@qq.com>
Date: Tue, 4 Dec 2018 09:01:18 +0800
Subject: [PATCH 1247/1554] Update decode.h

---
 tensorflow/core/util/proto/decode.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/util/proto/decode.h b/tensorflow/core/util/proto/decode.h
index 248490a2fa..8dde14dffc 100644
--- a/tensorflow/core/util/proto/decode.h
+++ b/tensorflow/core/util/proto/decode.h
@@ -318,7 +318,7 @@ inline int ReadPackedPrimitives(const void* bufp, const size_t len,
   return count;
 }
 
-// Reads a primitive field value from a serialized proto.
+// Reads a value of a primitive type field from a serialized proto.
 // The value is parsed from the serialized format, then static_cast
 // to the desired type for TensorFlow and stored.
 template <class ValueType, class TensorType,
-- 
GitLab


From 29be54c3c0bd881025ce2ddb72dbd9386ea56060 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Mon, 3 Dec 2018 16:57:53 -0800
Subject: [PATCH 1248/1554] Clean up the continue statements transformer - use
 plain constants instead of TF constants, and use the new state tracking
 mechanics. This also removes some bugs now captured by the reference tests.
 Also reorganizing the reference tests into a more manageable structure.

PiperOrigin-RevId: 223889283
---
 .../converters/continue_statements.py         | 91 ++++++++-----------
 1 file changed, 40 insertions(+), 51 deletions(-)

diff --git a/tensorflow/python/autograph/converters/continue_statements.py b/tensorflow/python/autograph/converters/continue_statements.py
index 584cdc1efd..05e19e59fc 100644
--- a/tensorflow/python/autograph/converters/continue_statements.py
+++ b/tensorflow/python/autograph/converters/continue_statements.py
@@ -24,94 +24,93 @@ from tensorflow.python.autograph.pyct import templates
 from tensorflow.python.autograph.pyct.static_analysis.annos import NodeAnno
 
 
-# Tags for local state.
-CONTROL_VAR_NAME = 'control_var_name'
-CONTINUE_USED = 'continue_used'
-GUARD_CREATED = 'guard_created'
-CREATE_GUARD_NEXT = 'create_guard_next'
+class _Continue(object):
+
+  def __init__(self):
+    self.used = False
+    self.control_var_name = None
+    self.create_guard = False
+    self.guard_created = False
+
+  def __repr__(self):
+    return 'used: %s, var: %s' % (self.used, self.control_var_name)
 
 
 class ContinueCanonicalizationTransformer(converter.Base):
   """Canonicalizes continue statements into additional conditionals."""
 
   def visit_Continue(self, node):
-    self.set_local(CONTINUE_USED, True)
+    self.state[_Continue].used = True
     template = """
-      var_name = tf.constant(True)
+      var_name = True
     """
     return templates.replace(
-        template, var_name=self.get_local(CONTROL_VAR_NAME))
+        template, var_name=self.state[_Continue].control_var_name)
 
   def _postprocess_statement(self, node):
     # Example of how the state machine below works:
     #
-    #   1| stmt           # State: CONTINUE_USED = False
+    #   1| stmt           # State: Continue_.used = False
     #    |                # Action: none
     #   2| if cond:
-    #   3|   continue     # State: CONTINUE_USED = True,
-    #    |                #        GUARD_CREATED = False,
-    #    |                #        CREATE_GUARD_NEXT = False
-    #    |                # Action: set CREATE_GUARD_NEXT = True
-    #   4| stmt           # State: CONTINUE_USED = True,
-    #    |                #        GUARD_CREATED = False,
-    #    |                #        CREATE_GUARD_NEXT = True
+    #   3|   continue     # State: Continue_.used = True,
+    #    |                #        Continue_.guard_created = False,
+    #    |                #        Continue_.create_guard = False
+    #    |                # Action: Continue_.create_guard = True
+    #   4| stmt           # State: Continue_.used = True,
+    #    |                #        Continue_.guard_created = False,
+    #    |                #        Continue_.create_guard = True
     #    |                # Action: create `if not continue_used`,
-    #    |                #         set GUARD_CREATED = True
-    #   5| stmt           # State: CONTINUE_USED = True, GUARD_CREATED = True
+    #    |                #         set Continue_.guard_created = True
+    #   5| stmt           # State: Continue_.used = True,
+    #    |                #        Continue_.guard_created = True
     #    |                # Action: none (will be wrapped under previously
     #    |                #         created if node)
 
-    if self.get_local(CONTINUE_USED, False):
-      if self.get_local(GUARD_CREATED, False):
+    if self.state[_Continue].used:
+      if self.state[_Continue].guard_created:
         return node, None
 
-      elif not self.get_local(CREATE_GUARD_NEXT, False):
-        self.set_local(CREATE_GUARD_NEXT, True)
+      elif not self.state[_Continue].create_guard:
+        self.state[_Continue].create_guard = True
         return node, None
 
       else:
-        self.set_local(GUARD_CREATED, True)
+        self.state[_Continue].guard_created = True
         template = """
           if not var_name:
             original_node
         """
         cond, = templates.replace(
             template,
-            var_name=self.get_local(CONTROL_VAR_NAME),
+            var_name=self.state[_Continue].control_var_name,
             original_node=node)
         return cond, cond.body
     return node, None
 
   def _visit_loop_body(self, node, nodes):
-    self.enter_local_scope()
+    self.state[_Continue].enter()
     scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
     continue_var = self.ctx.namer.new_symbol('continue_', scope.referenced)
-    self.set_local(CONTROL_VAR_NAME, continue_var)
+    self.state[_Continue].control_var_name = continue_var
 
     nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
 
-    if self.get_local(CONTINUE_USED, False):
+    if self.state[_Continue].used:
       template = """
-        var_name = tf.constant(False)
+        var_name = False
       """
       control_var_init = templates.replace(template, var_name=continue_var)
       nodes = control_var_init + nodes
 
-    self.exit_local_scope()
+    self.state[_Continue].exit()
     return nodes
 
-  def _visit_non_loop_body(self, nodes):
-    self.enter_local_scope(inherit=(CONTROL_VAR_NAME,))
-    nodes = self.visit_block(nodes, after_visit=self._postprocess_statement)
-    continue_used = self.get_local(CONTINUE_USED, False)
-    self.exit_local_scope(keep=(CONTINUE_USED,))
-    return nodes, continue_used
-
   def visit_While(self, node):
     node.test = self.visit(node.test)
     node.body = self._visit_loop_body(node, node.body)
     # A continue in the else clause applies to the containing scope.
-    node.orelse, _ = self._visit_non_loop_body(node.orelse)
+    node.orelse = self.visit_block(node.orelse)
     return node
 
   def visit_For(self, node):
@@ -119,21 +118,11 @@ class ContinueCanonicalizationTransformer(converter.Base):
     node.iter = self.generic_visit(node.iter)
     node.body = self._visit_loop_body(node, node.body)
     # A continue in the else clause applies to the containing scope.
-    node.orelse, _ = self._visit_non_loop_body(node.orelse)
-    return node
-
-  def visit_If(self, node):
-    node.test = self.generic_visit(node.test)
-    node.body, continue_used_body = self._visit_non_loop_body(node.body)
-    node.orelse, continue_used_orelse = self._visit_non_loop_body(node.orelse)
-    self.set_local(CONTINUE_USED, continue_used_body or continue_used_orelse)
-    return node
-
-  def visit_With(self, node):
-    node.items = self.visit_block(node.items)
-    node.body, _ = self._visit_non_loop_body(node.body)
+    node.orelse = self.visit_block(node.orelse)
     return node
 
 
 def transform(node, ctx):
-  return ContinueCanonicalizationTransformer(ctx).visit(node)
+  transformer = ContinueCanonicalizationTransformer(ctx)
+  node = transformer.visit(node)
+  return node
-- 
GitLab


From 61676fc59fabd49c3d49713291dac276ae46059f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 17:06:24 -0800
Subject: [PATCH 1249/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 223890656
---
 tensorflow/core/ops/ops.pbtxt | 208 ++++++++++++----------------------
 1 file changed, 74 insertions(+), 134 deletions(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 8cfd84be88..9328a3a1b8 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -9761,24 +9761,6 @@ op {
     type: DT_STRING
   }
 }
-op {
-  name: "EnqueueInQueueDataset"
-  input_arg {
-    name: "queue"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "components"
-    type_list_attr: "Tcomponents"
-  }
-  attr {
-    name: "Tcomponents"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "EnsureShape"
   input_arg {
@@ -10853,6 +10835,80 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalRandomDataset"
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "seed2"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ExperimentalScanDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "initial_state"
+    type_list_attr: "Tstate"
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Tstate"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalSetStatsAggregatorDataset"
   input_arg {
@@ -20670,48 +20726,6 @@ op {
     minimum: 1
   }
 }
-op {
-  name: "PrependFromQueueAndPaddedBatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "batch_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "padded_shapes"
-    type: DT_INT64
-    number_attr: "N"
-  }
-  input_arg {
-    name: "padding_values"
-    type_list_attr: "Toutput_types"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "Toutput_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "PreventGradient"
   input_arg {
@@ -22983,34 +22997,6 @@ op {
   }
   is_stateful: true
 }
-op {
-  name: "RandomDataset"
-  input_arg {
-    name: "seed"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "seed2"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
 op {
   name: "RandomGamma"
   input_arg {
@@ -27956,52 +27942,6 @@ op {
     }
   }
 }
-op {
-  name: "ScanDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "initial_state"
-    type_list_attr: "Tstate"
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Tstate"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "ScatterAdd"
   input_arg {
-- 
GitLab


From 95af78cf8aa34cea0b67aec79014553cc4dc75f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 17:12:33 -0800
Subject: [PATCH 1250/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 223891488

---
 tensorflow/go/op/wrappers.go | 429 +++++++++++++++++------------------
 1 file changed, 202 insertions(+), 227 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 69de0269ff..5cd06299b9 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5588,63 +5588,6 @@ func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncomp
 	return op.Output(0)
 }
 
-// MapSizeAttr is an optional argument to MapSize.
-type MapSizeAttr func(optionalAttr)
-
-// MapSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapSizeCapacity(value int64) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapSizeMemoryLimit(value int64) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapSizeContainer(value string) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapSizeSharedName(value string) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op returns the number of elements in the underlying container.
-func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapSize",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -10714,6 +10657,31 @@ func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_ke
 	return sparse_indices, sparse_values, sparse_shapes, dense_values
 }
 
+// Compute the pairwise cross product.
+//
+// `a` and `b` must be the same shape; they can either be simple 3-element vectors,
+// or any shape where the innermost dimension is 3. In the latter case, each pair
+// of corresponding 3-element vectors is cross-multiplied independently.
+//
+// Arguments:
+//	a: A tensor containing 3-element vectors.
+//	b: Another tensor, of same type and shape as `a`.
+//
+// Returns Pairwise cross product of the vectors in `a` and `b`.
+func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Cross",
+		Input: []tf.Input{
+			a, b,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // CudnnRNNAttr is an optional argument to CudnnRNN.
 type CudnnRNNAttr func(optionalAttr)
 
@@ -13935,6 +13903,126 @@ func ResourceApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.
 	return scope.AddOperation(opspec)
 }
 
+// SubstrAttr is an optional argument to Substr.
+type SubstrAttr func(optionalAttr)
+
+// SubstrUnit sets the optional unit attribute to value.
+//
+// value: The unit that is used to create the substring.  One of: `"BYTE"` (for
+// defining position and length by bytes) or `"UTF8_CHAR"` (for the UTF-8
+// encoded Unicode code points).  The default is `"BYTE"`. Results are undefined if
+// `unit=UTF8_CHAR` and the `input` strings do not contain structurally valid
+// UTF-8.
+// If not specified, defaults to "BYTE"
+func SubstrUnit(value string) SubstrAttr {
+	return func(m optionalAttr) {
+		m["unit"] = value
+	}
+}
+
+// Return substrings from `Tensor` of strings.
+//
+// For each string in the input `Tensor`, creates a substring starting at index
+// `pos` with a total length of `len`.
+//
+// If `len` defines a substring that would extend beyond the length of the input
+// string, then as many characters as possible are used.
+//
+// A negative `pos` indicates distance within the string backwards from the end.
+//
+// If `pos` specifies an index which is out of range for any of the input strings,
+// then an `InvalidArgumentError` is thrown.
+//
+// `pos` and `len` must have the same shape, otherwise a `ValueError` is thrown on
+// Op creation.
+//
+// *NOTE*: `Substr` supports broadcasting up to two dimensions. More about
+// broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+//
+// ---
+//
+// Examples
+//
+// Using scalar `pos` and `len`:
+//
+// ```python
+// input = [b'Hello', b'World']
+// position = 1
+// length = 3
+//
+// output = [b'ell', b'orl']
+// ```
+//
+// Using `pos` and `len` with same shape as `input`:
+//
+// ```python
+// input = [[b'ten', b'eleven', b'twelve'],
+//          [b'thirteen', b'fourteen', b'fifteen'],
+//          [b'sixteen', b'seventeen', b'eighteen']]
+// position = [[1, 2, 3],
+//             [1, 2, 3],
+//             [1, 2, 3]]
+// length =   [[2, 3, 4],
+//             [4, 3, 2],
+//             [5, 5, 5]]
+//
+// output = [[b'en', b'eve', b'lve'],
+//           [b'hirt', b'urt', b'te'],
+//           [b'ixtee', b'vente', b'hteen']]
+// ```
+//
+// Broadcasting `pos` and `len` onto `input`:
+//
+// ```
+// input = [[b'ten', b'eleven', b'twelve'],
+//          [b'thirteen', b'fourteen', b'fifteen'],
+//          [b'sixteen', b'seventeen', b'eighteen'],
+//          [b'nineteen', b'twenty', b'twentyone']]
+// position = [1, 2, 3]
+// length =   [1, 2, 3]
+//
+// output = [[b'e', b'ev', b'lve'],
+//           [b'h', b'ur', b'tee'],
+//           [b'i', b've', b'hte'],
+//           [b'i', b'en', b'nty']]
+// ```
+//
+// Broadcasting `input` onto `pos` and `len`:
+//
+// ```
+// input = b'thirteen'
+// position = [1, 5, 7]
+// length =   [3, 2, 1]
+//
+// output = [b'hir', b'ee', b'n']
+// ```
+//
+// Arguments:
+//	input: Tensor of strings
+//	pos: Scalar defining the position of first character in each substring
+//	len: Scalar defining the number of characters to include in each substring
+//
+// Returns Tensor of substrings
+func Substr(scope *Scope, input tf.Output, pos tf.Output, len tf.Output, optional ...SubstrAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Substr",
+		Input: []tf.Input{
+			input, pos, len,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Exits the current frame to its parent frame.
 //
 // Exit makes its input `data` available to the parent frame.
@@ -20885,151 +20973,6 @@ func BitwiseAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// SubstrAttr is an optional argument to Substr.
-type SubstrAttr func(optionalAttr)
-
-// SubstrUnit sets the optional unit attribute to value.
-//
-// value: The unit that is used to create the substring.  One of: `"BYTE"` (for
-// defining position and length by bytes) or `"UTF8_CHAR"` (for the UTF-8
-// encoded Unicode code points).  The default is `"BYTE"`. Results are undefined if
-// `unit=UTF8_CHAR` and the `input` strings do not contain structurally valid
-// UTF-8.
-// If not specified, defaults to "BYTE"
-func SubstrUnit(value string) SubstrAttr {
-	return func(m optionalAttr) {
-		m["unit"] = value
-	}
-}
-
-// Return substrings from `Tensor` of strings.
-//
-// For each string in the input `Tensor`, creates a substring starting at index
-// `pos` with a total length of `len`.
-//
-// If `len` defines a substring that would extend beyond the length of the input
-// string, then as many characters as possible are used.
-//
-// A negative `pos` indicates distance within the string backwards from the end.
-//
-// If `pos` specifies an index which is out of range for any of the input strings,
-// then an `InvalidArgumentError` is thrown.
-//
-// `pos` and `len` must have the same shape, otherwise a `ValueError` is thrown on
-// Op creation.
-//
-// *NOTE*: `Substr` supports broadcasting up to two dimensions. More about
-// broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-//
-// ---
-//
-// Examples
-//
-// Using scalar `pos` and `len`:
-//
-// ```python
-// input = [b'Hello', b'World']
-// position = 1
-// length = 3
-//
-// output = [b'ell', b'orl']
-// ```
-//
-// Using `pos` and `len` with same shape as `input`:
-//
-// ```python
-// input = [[b'ten', b'eleven', b'twelve'],
-//          [b'thirteen', b'fourteen', b'fifteen'],
-//          [b'sixteen', b'seventeen', b'eighteen']]
-// position = [[1, 2, 3],
-//             [1, 2, 3],
-//             [1, 2, 3]]
-// length =   [[2, 3, 4],
-//             [4, 3, 2],
-//             [5, 5, 5]]
-//
-// output = [[b'en', b'eve', b'lve'],
-//           [b'hirt', b'urt', b'te'],
-//           [b'ixtee', b'vente', b'hteen']]
-// ```
-//
-// Broadcasting `pos` and `len` onto `input`:
-//
-// ```
-// input = [[b'ten', b'eleven', b'twelve'],
-//          [b'thirteen', b'fourteen', b'fifteen'],
-//          [b'sixteen', b'seventeen', b'eighteen'],
-//          [b'nineteen', b'twenty', b'twentyone']]
-// position = [1, 2, 3]
-// length =   [1, 2, 3]
-//
-// output = [[b'e', b'ev', b'lve'],
-//           [b'h', b'ur', b'tee'],
-//           [b'i', b've', b'hte'],
-//           [b'i', b'en', b'nty']]
-// ```
-//
-// Broadcasting `input` onto `pos` and `len`:
-//
-// ```
-// input = b'thirteen'
-// position = [1, 5, 7]
-// length =   [3, 2, 1]
-//
-// output = [b'hir', b'ee', b'n']
-// ```
-//
-// Arguments:
-//	input: Tensor of strings
-//	pos: Scalar defining the position of first character in each substring
-//	len: Scalar defining the number of characters to include in each substring
-//
-// Returns Tensor of substrings
-func Substr(scope *Scope, input tf.Output, pos tf.Output, len tf.Output, optional ...SubstrAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Substr",
-		Input: []tf.Input{
-			input, pos, len,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a Dataset that returns pseudorandom numbers.
-//
-// Arguments:
-//	seed: A scalar seed for the random number generator. If either seed or
-// seed2 is set to be non-zero, the random number generator is seeded
-// by the given seed.  Otherwise, a random seed is used.
-//	seed2: A second scalar seed to avoid seed collision.
-//
-//
-func RandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "RandomDataset",
-		Input: []tf.Input{
-			seed, seed2,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Inverse real-valued fast Fourier transform.
 //
 // Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
@@ -31141,31 +31084,6 @@ func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQu
 	return op.Output(0)
 }
 
-// Compute the pairwise cross product.
-//
-// `a` and `b` must be the same shape; they can either be simple 3-element vectors,
-// or any shape where the innermost dimension is 3. In the latter case, each pair
-// of corresponding 3-element vectors is cross-multiplied independently.
-//
-// Arguments:
-//	a: A tensor containing 3-element vectors.
-//	b: Another tensor, of same type and shape as `a`.
-//
-// Returns Pairwise cross product of the vectors in `a` and `b`.
-func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Cross",
-		Input: []tf.Input{
-			a, b,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Constructs an Optional variant from a tuple of tensors.
 func OptionalFromValue(scope *Scope, components []tf.Output) (optional tf.Output) {
 	if scope.Err() != nil {
@@ -33695,3 +33613,60 @@ func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.Data
 	}
 	return values
 }
+
+// MapSizeAttr is an optional argument to MapSize.
+type MapSizeAttr func(optionalAttr)
+
+// MapSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapSizeCapacity(value int64) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapSizeMemoryLimit(value int64) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapSizeContainer(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapSizeSharedName(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of elements in the underlying container.
+func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapSize",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
-- 
GitLab


From 697339bc7206b04583576218277cc039f7eb39e1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 17:25:19 -0800
Subject: [PATCH 1251/1554] Support uint8 Relu6

PiperOrigin-RevId: 223893131
---
 tensorflow/lite/kernels/activations.cc      | 19 +++++++++++++++--
 tensorflow/lite/kernels/activations_test.cc | 23 +++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index 82072bccb2..a766542560 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -345,9 +345,24 @@ TfLiteStatus Relu6Eval(TfLiteContext* context, TfLiteNode* node) {
       for (; in < in_end; in++, out++) *out = std::min(std::max(0.f, *in), 6.f);
       return kTfLiteOk;
     } break;
+    case kTfLiteUInt8: {
+      ActivationParams params;
+      params.activation_type = FusedActivationFunctionType::kRelu6;
+      params.quantized_activation_min = std::max(
+          0, output->params.zero_point +
+                 static_cast<int32>(roundf(0.f / output->params.scale)));
+      params.quantized_activation_max = std::min(
+          255, output->params.zero_point +
+                   static_cast<int32>(roundf(6.f / output->params.scale)));
+      optimized_ops::ReluX(params, GetTensorShape(input),
+                           GetTensorData<uint8>(input), GetTensorShape(output),
+                           GetTensorData<uint8>(output));
+      return kTfLiteOk;
+    } break;
     default:
-      context->ReportError(context, "Only float32 supported currently, got %s.",
-                           TfLiteTypeGetName(input->type));
+      context->ReportError(
+          context, "Only float32 and uint8 supported currently, got %s.",
+          TfLiteTypeGetName(input->type));
       return kTfLiteError;
   }
 }
diff --git a/tensorflow/lite/kernels/activations_test.cc b/tensorflow/lite/kernels/activations_test.cc
index 1de3dbc44f..67f137baff 100644
--- a/tensorflow/lite/kernels/activations_test.cc
+++ b/tensorflow/lite/kernels/activations_test.cc
@@ -170,6 +170,29 @@ TEST(FloatActivationsOpTest, Tanh) {
                              })));
 }
 
+TEST(QuantizedActivationsOpTest, Relu6) {
+  const float kMin = -1;
+  const float kMax = 127.f / 128.f;
+  QuantizedActivationsOpModel m(
+      BuiltinOperator_RELU6,
+      /*input=*/{TensorType_UINT8, {1, 2, 4, 1}, 8 * kMin, 8 * kMax},
+      /*output=*/{TensorType_UINT8, {1, 2, 4, 1}, 8 * kMin, 8 * kMax});
+  m.SetInput<uint8_t>({
+      0, -6, 2, 4,   //
+      3, -2, 10, 1,  //
+  });
+  m.Invoke();
+  EXPECT_THAT(m.GetDequantizedOutput<uint8_t>(),
+              ElementsAreArray(ArrayFloatNear(
+                  {
+                      0, 0, 2, 4,  //
+                      3, 0, 6, 1,  //
+                  },
+                  kQuantizedTolerance)));
+  EXPECT_THAT(m.GetOutput<uint8_t>(),
+              ElementsAreArray({128, 128, 160, 192, 176, 128, 224, 144}));
+}
+
 TEST(QuantizedActivationsOpTest, Tanh) {
   const float kMin = -1;
   const float kMax = 127.f / 128.f;
-- 
GitLab


From 81eaf1d3f22665ad004bf521b893b9b6fc8e928b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 3 Dec 2018 17:40:08 -0800
Subject: [PATCH 1252/1554] Shuffle output_sizes when testing NCHW.

The output shape needs to reflect that the input has been transposed to NCHW.
Without this fix the test fails with XLA somewhere inside
compute_gradient_error.

I have not done the analysis to figure out how this test passes with TF without
XLA.

PiperOrigin-RevId: 223895440
---
 tensorflow/python/kernel_tests/pooling_ops_3d_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
index 24fb51fc47..347e092dee 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py
@@ -253,6 +253,7 @@ class PoolingTest(test.TestCase):
         ksize = test_util.NHWCToNCHW(ksize)
         strides = test_util.NHWCToNCHW(strides)
         t = test_util.NHWCToNCHW(t)
+        output_sizes = test_util.NHWCToNCHW(output_sizes)
 
       t = pool_func(
           t,
-- 
GitLab


From e2b36c5026b856ca63041085c3183f0f7d490e71 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 3 Dec 2018 17:52:06 -0800
Subject: [PATCH 1253/1554] [tf.data] Remove `Dataset.make_one_shot_iterator()`
 from the V2 API.

Add `tf.compat.v1.data.make_one_shot_iterator(dataset)` to enable the use of V2
`Dataset` objects in a legacy V1 pipeline.

PiperOrigin-RevId: 223896743
---
 tensorflow/contrib/data/python/ops/readers.py |  14 +-
 tensorflow/contrib/eager/python/evaluator.py  |   5 +-
 .../examples/densenet/densenet_graph_test.py  |   3 +-
 .../python/examples/gan/mnist_graph_test.py   |   3 +-
 .../examples/generative_examples/cvae.ipynb   |   2 +-
 .../examples/resnet50/resnet50_graph_test.py  |   3 +-
 .../eager/python/examples/revnet/main.py      |  13 +-
 .../examples/rnn_ptb/rnn_ptb_graph_test.py    |   5 +-
 .../hadoop/python/ops/hadoop_dataset_ops.py   |  11 +-
 tensorflow/contrib/ignite/README.md           |  45 +++---
 .../python/tests/ignite_dataset_test.py       |   3 +-
 .../kinesis/python/ops/kinesis_dataset_ops.py |  11 +-
 tensorflow/contrib/lookup/lookup_ops_test.py  |   5 +-
 .../python/metrics/classification_test.py     |  11 +-
 .../python/loss_scale_manager_test.py         |   2 +-
 .../python/loss_scale_optimizer_test.py       |   4 +-
 .../tensorrt/test/quantization_mnist_test.py  |   4 +-
 tensorflow/contrib/tpu/python/tpu/datasets.py |   2 +-
 .../python/tpu/tpu_estimator_signals_test.py  |   2 +-
 tensorflow/contrib/tpu/tpu_estimator.md       |   9 +-
 .../regression/custom_regression.py           |   5 +-
 .../get_started/regression/dnn_regression.py  |   5 +-
 .../linear_regression_categorical.py          |   5 +-
 .../reading_data/fully_connected_reader.py    |   2 +-
 tensorflow/lite/tutorials/mnist_tflite.py     |   4 +-
 tensorflow/python/data/__init__.py            |   1 +
 .../data/benchmarks/filter_benchmark.py       |   2 +-
 .../python/data/benchmarks/map_benchmark.py   |   4 +-
 .../benchmarks/autotune_benchmark.py          |   8 +-
 .../benchmarks/csv_dataset_benchmark.py       |   3 +-
 .../benchmarks/map_and_batch_benchmark.py     |   4 +-
 .../benchmarks/map_vectorization_benchmark.py |   4 +-
 .../benchmarks/matching_files_benchmark.py    |   3 +-
 .../benchmarks/optimize_benchmark.py          |   4 +-
 .../bucket_by_sequence_length_test.py         |   8 +-
 .../kernel_tests/copy_to_device_test.py       |  16 +--
 .../experimental/kernel_tests/counter_test.py |   9 +-
 .../directed_interleave_dataset_test.py       |   4 +-
 .../kernel_tests/filter_dataset_op_test.py    |   2 +-
 .../kernel_tests/group_by_reducer_test.py     |   6 +-
 .../make_batched_features_dataset_test.py     |  93 ++++++------
 .../kernel_tests/make_csv_dataset_test.py     |  11 +-
 .../make_tf_record_dataset_test.py            |  18 +--
 .../kernel_tests/map_and_batch_test.py        |  18 +--
 .../kernel_tests/matching_files_test.py       |  13 +-
 .../optimization/hoist_random_uniform_test.py |   2 +-
 .../optimization/map_vectorization_test.py    |   2 +-
 .../optimization/model_dataset_test.py        |   2 +-
 .../kernel_tests/parallel_interleave_test.py  |   2 +-
 .../kernel_tests/prefetch_to_device_test.py   |   8 +-
 .../kernel_tests/rejection_resample_test.py   |  10 +-
 .../experimental/kernel_tests/scan_test.py    |   8 +-
 .../kernel_tests/shuffle_and_repeat_test.py   |   2 +-
 .../experimental/kernel_tests/unbatch_test.py |  14 +-
 .../python/data/experimental/ops/readers.py   |  24 ++--
 .../data/experimental/ops/stats_aggregator.py |   1 -
 .../data/experimental/ops/stats_options.py    |   4 +-
 .../data/kernel_tests/from_generator_test.py  |   5 +-
 .../data/kernel_tests/interleave_test.py      |   2 +-
 .../kernel_tests/iterator_checkpoint_test.py  |  10 +-
 .../kernel_tests/iterator_cluster_test.py     |   4 +-
 .../python/data/kernel_tests/iterator_test.py |  53 +++----
 .../python/data/kernel_tests/map_test.py      |  11 +-
 .../python/data/kernel_tests/shuffle_test.py  |  10 +-
 .../python/data/kernel_tests/test_base.py     |   2 +-
 tensorflow/python/data/ops/dataset_ops.py     | 135 +++++++++++-------
 tensorflow/python/distribute/values.py        |   6 +-
 tensorflow/python/grappler/datasets_test.py   |  20 +--
 .../python/keras/engine/sequential_test.py    |   2 +-
 tensorflow/python/keras/engine/training.py    |   2 +-
 .../keras/engine/training_dataset_test.py     |  10 +-
 .../keras/engine/training_eager_test.py       |   6 +-
 .../python/keras/engine/training_generator.py |   2 +-
 .../python/keras/engine/training_test.py      |   2 +-
 .../python/keras/engine/training_utils.py     |   2 +-
 .../python/keras/model_subclassing_test.py    |   2 +-
 .../keras/utils/multi_gpu_utils_test.py       |   2 +-
 .../training/basic_session_run_hooks_test.py  |   2 +-
 tensorflow/python/training/saver_test.py      |   2 +-
 .../tools/api/golden/v1/tensorflow.data.pbtxt |   4 +
 .../golden/v2/tensorflow.data.-dataset.pbtxt  |   4 -
 ...ow.data.-fixed-length-record-dataset.pbtxt |   4 -
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   4 -
 .../tensorflow.data.-text-line-dataset.pbtxt  |   4 -
 ...rflow.data.experimental.-csv-dataset.pbtxt |   4 -
 ...ow.data.experimental.-random-dataset.pbtxt |   4 -
 ...rflow.data.experimental.-sql-dataset.pbtxt |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |   1 +
 88 files changed, 402 insertions(+), 411 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index 4601376dff..aa42782807 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -355,7 +355,7 @@ def read_batch_features(file_pattern,
       shuffle=randomize_input,
       num_epochs=num_epochs,
       shuffle_buffer_size=capacity)
-  iterator = dataset.make_one_shot_iterator()
+  iterator = dataset_ops.make_one_shot_iterator(dataset)
   outputs = iterator.get_next()
   return outputs
 
@@ -379,15 +379,13 @@ class LMDBDataset(dataset_ops.DatasetSource):
     (key value) pairs sequentially.
     For example:
     ```python
+    tf.enable_eager_execution()
+
     dataset = tf.contrib.lmdb.LMDBDataset("/foo/bar.mdb")
-    iterator = dataset.make_one_shot_iterator()
-    next_element = iterator.get_next()
+
     # Prints the (key, value) pairs inside a lmdb file.
-    while True:
-      try:
-        print(sess.run(next_element))
-      except tf.errors.OutOfRangeError:
-        break
+    for key, value in dataset:
+      print(key, value)
     ```
     Args:
       filenames: A `tf.string` tensor containing one or more filenames.
diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py
index 7949a3f6da..51443d2482 100644
--- a/tensorflow/contrib/eager/python/evaluator.py
+++ b/tensorflow/contrib/eager/python/evaluator.py
@@ -22,6 +22,7 @@ import six
 
 from tensorflow.contrib.eager.python import datasets
 from tensorflow.contrib.eager.python import metrics
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import errors_impl
@@ -164,8 +165,8 @@ class Evaluator(object):
         self.__call__(example, *args, **kwargs)
       return self.all_metric_results(summary_logdir)
     # Graph construction
-    call_op = self.__call__(dataset.make_one_shot_iterator().get_next(), *args,
-                            **kwargs)
+    call_op = self.__call__(
+        dataset_ops.make_one_shot_iterator(dataset).get_next(), *args, **kwargs)
     init_op = self.init_variables()
     results_op = self.all_metric_results(summary_logdir)
     return (init_op, call_op, results_op)
diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py
index 4b3cb624bc..24f6b007b5 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py
+++ b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py
@@ -119,7 +119,8 @@ class DensenetBenchmark(tf.test.Benchmark):
       with tf.Graph().as_default():
         np_images, np_labels = random_batch(batch_size)
         dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat()
-        (images, labels) = dataset.make_one_shot_iterator().get_next()
+        (images, labels) = tf.compat.v1.data.make_one_shot_iterator(
+            dataset).get_next()
 
         model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks,
                                   self.output_classes,
diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py b/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py
index 12b39b0cde..e73841fbf7 100644
--- a/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py
+++ b/tensorflow/contrib/eager/python/examples/gan/mnist_graph_test.py
@@ -42,7 +42,8 @@ class MnistGraphGanBenchmark(tf.test.Benchmark):
     # Generate some random data.
     images_data = np.random.randn(batch_size, 784).astype(np.float32)
     dataset = tf.data.Dataset.from_tensors(images_data)
-    images = dataset.repeat().make_one_shot_iterator().get_next()
+    images = tf.compat.v1.data.make_one_shot_iterator(
+        dataset.repeat()).get_next()
 
     # Create the models and optimizers
     generator = mnist.Generator(data_format())
diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb
index ca27a85a22..1a08cc0fd0 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/cvae.ipynb
@@ -470,7 +470,7 @@
         "\n",
         "  if epoch % 1 == 0:\n",
         "    loss = tfe.metrics.Mean()\n",
-        "    for test_x in test_dataset.make_one_shot_iterator():\n",
+        "    for test_x in test_dataset:\n",
         "      loss(compute_loss(model, test_x))\n",
         "    elbo = -loss.result()\n",
         "    display.clear_output(wait=False)\n",
diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py
index f3bb978875..fb7975d8fe 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py
+++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py
@@ -142,7 +142,8 @@ class ResNet50Benchmarks(tf.test.Benchmark):
       with tf.Graph().as_default():
         np_images, np_labels = random_batch(batch_size)
         dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat()
-        (images, labels) = dataset.make_one_shot_iterator().get_next()
+        images, labels = tf.compat.v1.data.make_one_shot_iterator(
+            dataset).get_next()
 
         model = resnet50.ResNet50(data_format())
         logits = model(images, training=True)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py
index b702e91f92..9585f3565f 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/main.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/main.py
@@ -72,14 +72,11 @@ def main(_):
     train_one_iter(model, x, y, optimizer, global_step=global_step)
 
     if global_step.numpy() % config.log_every == 0:
-      it_test = ds_test.make_one_shot_iterator()
-      acc_test, loss_test = evaluate(model, it_test)
+      acc_test, loss_test = evaluate(model, ds_test)
 
       if FLAGS.validate:
-        it_train = ds_train_one_shot.make_one_shot_iterator()
-        it_validation = ds_validation.make_one_shot_iterator()
-        acc_train, loss_train = evaluate(model, it_train)
-        acc_validation, loss_validation = evaluate(model, it_validation)
+        acc_train, loss_train = evaluate(model, ds_train_one_shot)
+        acc_validation, loss_validation = evaluate(model, ds_validation)
         print("Iter {}, "
               "training set accuracy {:.4f}, loss {:.4f}; "
               "validation set accuracy {:.4f}, loss {:.4f}; "
@@ -218,11 +215,11 @@ def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   return logits, loss
 
 
-def evaluate(model, iterator):
+def evaluate(model, dataset):
   """Compute accuracy with the given dataset iterator."""
   mean_loss = tfe.metrics.Mean()
   accuracy = tfe.metrics.Accuracy()
-  for x, y in iterator:
+  for x, y in dataset:
     logits, _ = model(x, training=False)
     loss = model.compute_loss(logits=logits, labels=y)
     accuracy(
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py
index 63b5c4c54d..770484abed 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py
@@ -82,7 +82,7 @@ class PTBBenchmark(tf.test.Benchmark):
         tf.ones(
             [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE],
             dtype=tf.int64)).repeat(num_iters + num_warmup)
-    inputs = dataset.make_one_shot_iterator().get_next()
+    inputs = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next()
 
     with tf.device(tf.test.gpu_device_name()):
       outputs = model(inputs, training=True)
@@ -124,7 +124,8 @@ class PTBBenchmark(tf.test.Benchmark):
             dtype=tf.int64)).repeat(num_iters + num_warmup)
     # inputs and labels have the same shape
     dataset = tf.data.Dataset.zip((dataset, dataset))
-    (inputs, labels) = dataset.make_one_shot_iterator().get_next()
+    (inputs, labels) = tf.compat.v1.data.make_one_shot_iterator(
+        dataset).get_next()
 
     with tf.device(tf.test.gpu_device_name()):
       optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
diff --git a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py
index bf398b838d..d3fcc8cb2a 100644
--- a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py
+++ b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py
@@ -40,15 +40,12 @@ class SequenceFileDataset(dataset_ops.DatasetSource):
     For example:
 
     ```python
+    tf.enable_eager_execution()
+
     dataset = tf.contrib.hadoop.SequenceFileDataset("/foo/bar.seq")
-    iterator = dataset.make_one_shot_iterator()
-    next_element = iterator.get_next()
     # Prints the (key, value) pairs inside a hadoop sequence file.
-    while True:
-      try:
-        print(sess.run(next_element))
-      except tf.errors.OutOfRangeError:
-        break
+    for key, value in dataset:
+      print(key, value)
     ```
 
     Args:
diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md
index c7db0b77e2..5a8c650fb9 100644
--- a/tensorflow/contrib/ignite/README.md
+++ b/tensorflow/contrib/ignite/README.md
@@ -54,14 +54,12 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL
 ```python
 >>> import tensorflow as tf
 >>> from tensorflow.contrib.ignite import IgniteDataset
->>> 
+>>> tf.enable_eager_execution()
+>>>
 >>> dataset = IgniteDataset(cache_name="SQL_PUBLIC_KITTEN_CACHE")
->>> iterator = dataset.make_one_shot_iterator()
->>> next_obj = iterator.get_next()
 >>>
->>> with tf.Session() as sess:
->>>   for _ in range(3):
->>>     print(sess.run(next_obj))
+>>> for element in dataset:
+>>>   print(element)
 
 {'key': 1, 'val': {'NAME': b'WARM KITTY'}}
 {'key': 2, 'val': {'NAME': b'SOFT KITTY'}}
@@ -74,23 +72,22 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL
 ```python
 >>> import tensorflow as tf
 >>> from tensorflow.contrib.ignite import IgniteDataset
->>> 
+>>> tf.enable_eager_execution()
+>>>
 >>> dataset = IgniteDataset(cache_name="IMAGES")
->>> iterator = dataset.make_one_shot_iterator()
->>> next_obj = iterator.get_next()
 >>>
->>> with tf.Session() as sess:
->>>   print(sess.run(next_obj))
+>>> for element in dataset.take(1):
+>>>   print(element)
 
 {
-    'key': 'kitten.png', 
+    'key': 'kitten.png',
     'val': {
         'metadata': {
             'file_name': b'kitten.png',
             'label': b'little ball of fur',
-            width: 800, 
+            width: 800,
             height: 600
-        }, 
+        },
         'pixels': [0, 0, 0, 0, ..., 0]
     }
 }
@@ -100,13 +97,11 @@ jdbc:ignite:thin://localhost/> INSERT INTO KITTEN_CACHE VALUES (3, 'LITTLE BALL
 ```python
 >>> import tensorflow as tf
 >>> from tensorflow.contrib.ignite import IgniteDataset
->>> 
+>>>
 >>> dataset = IgniteDataset(cache_name="IMAGES").map(lambda obj: obj['val']['pixels'])
->>> iterator = dataset.make_one_shot_iterator()
->>> next_obj = iterator.get_next()
 >>>
->>> with tf.Session() as sess:
->>>   print(sess.run(next_obj))
+>>> for element in dataset:
+>>>   print(element)
 
 [0, 0, 0, 0, ..., 0]
 ```
@@ -126,18 +121,18 @@ Ignite Dataset allows using these two aspects of distributed neural network trai
 ```python
 >>> import tensorflow as tf
 >>> from tensorflow.contrib.ignite import IgniteDataset
->>> 
+>>>
 >>> dataset = IgniteDataset("IMAGES")
 >>>
 >>> # Compute gradients locally on every worker node.
->>> gradients = []    
+>>> gradients = []
 >>> for i in range(5):
 >>>     with tf.device("/job:WORKER/task:%d" % i):
->>>         device_iterator = dataset.make_one_shot_iterator()
+>>>         device_iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
 >>>         device_next_obj = device_iterator.get_next()
 >>>         gradient = compute_gradient(device_next_obj)
->>>         gradients.append(gradient)        
->>>        
+>>>         gradients.append(gradient)
+>>>
 >>> # Aggregate them on master node.
 >>> result_gradient = tf.reduce_sum(gradients)
 >>>
@@ -145,7 +140,7 @@ Ignite Dataset allows using these two aspects of distributed neural network trai
 >>>     print(sess.run(result_gradient))
 ```
 
-High-level TensorFlow API for [distributed training](https://www.tensorflow.org/api_docs/python/tf/contrib/distribute/DistributionStrategy) is supported as well. 
+High-level TensorFlow API for [distributed training](https://www.tensorflow.org/api_docs/python/tf/contrib/distribute/DistributionStrategy) is supported as well.
 
 ### Distributed File System
 
diff --git a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
index ef29b5f14a..ff5d4c458c 100644
--- a/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
+++ b/tensorflow/contrib/ignite/python/tests/ignite_dataset_test.py
@@ -21,6 +21,7 @@ import os
 
 from tensorflow.contrib.ignite import IgniteDataset
 from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.platform import test
@@ -65,7 +66,7 @@ class IgniteDatasetTest(test.TestCase):
     self.assertEqual(dtypes.string, dataset.output_types["val"]["NAME"])
     self.assertEqual(dtypes.int64, dataset.output_types["val"]["VAL"])
 
-    it = dataset.make_one_shot_iterator()
+    it = dataset_ops.make_one_shot_iterator(dataset)
     ne = it.get_next()
 
     with session.Session() as sess:
diff --git a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
index 75806dbbeb..c392adbb1d 100644
--- a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
+++ b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
@@ -34,15 +34,12 @@ class KinesisDataset(dataset_ops.DatasetSource):
 
   For example, we can construct and use the KinesisDataset as follows:
   ```python
+  tf.enable_eager_execution()
+
   dataset = tf.contrib.kinesis.KinesisDataset(
       "kinesis_stream_name", read_indefinitely=False)
-  next = dataset.make_one_shot_iterator().get_next()
-  with tf.Session() as sess:
-    while True:
-      try:
-        print(sess.run(nxt))
-      except tf.errors.OutOfRangeError:
-        break
+  for element in dataset:
+    print(element)
   ```
 
   Since Kinesis is a data streaming service, data may not be available
diff --git a/tensorflow/contrib/lookup/lookup_ops_test.py b/tensorflow/contrib/lookup/lookup_ops_test.py
index 5e99ef4605..9b2c2dd87c 100644
--- a/tensorflow/contrib/lookup/lookup_ops_test.py
+++ b/tensorflow/contrib/lookup/lookup_ops_test.py
@@ -25,6 +25,7 @@ import six
 from tensorflow.contrib import lookup
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import counter
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -2737,7 +2738,7 @@ class MutableHashTableBenchmark(test.Benchmark):
 
   def benchmark_many_repeated_scalar_insert_scalar(self):
     table = self._create_table()
-    c = counter.Counter().make_one_shot_iterator().get_next()
+    c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next()
     value = variables.Variable(1.0)
     insert = table.insert(c, value)
     size = table.size()
@@ -2758,7 +2759,7 @@ class MutableHashTableBenchmark(test.Benchmark):
 
   def benchmark_many_repeated_batch_32_insert_scalar(self):
     table = self._create_table()
-    c = counter.Counter().make_one_shot_iterator().get_next()
+    c = dataset_ops.make_one_shot_iterator(counter.Counter()).get_next()
     value = variables.Variable([1.0] * 32)
     insert = table.insert(32 * c + list(range(32)), value)
     size = table.size()
diff --git a/tensorflow/contrib/metrics/python/metrics/classification_test.py b/tensorflow/contrib/metrics/python/metrics/classification_test.py
index d6a670f97b..e789d2cb9d 100644
--- a/tensorflow/contrib/metrics/python/metrics/classification_test.py
+++ b/tensorflow/contrib/metrics/python/metrics/classification_test.py
@@ -291,12 +291,11 @@ class F1ScoreTest(test.TestCase):
 
     labels = labels.astype(np.float32)
     predictions = predictions.astype(np.float32)
-    tf_predictions, tf_labels = (dataset_ops.Dataset
-                                 .from_tensor_slices((predictions, labels))
-                                 .repeat()
-                                 .batch(batch_size)
-                                 .make_one_shot_iterator()
-                                 .get_next())
+    tf_predictions, tf_labels = dataset_ops.make_one_shot_iterator(
+        dataset_ops.Dataset
+        .from_tensor_slices((predictions, labels))
+        .repeat()
+        .batch(batch_size)).get_next()
     f1, f1_op = classification.f1_score(tf_labels, tf_predictions,
                                         num_thresholds=3)
 
diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py b/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py
index 1b0383d24c..c922d0cd11 100644
--- a/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py
+++ b/tensorflow/contrib/mixed_precision/python/loss_scale_manager_test.py
@@ -29,7 +29,7 @@ from tensorflow.python.platform import test
 
 def _GetExampleIter(inputs):
   dataset = dataset_ops.Dataset.from_tensor_slices(inputs)
-  return dataset.make_one_shot_iterator()
+  return dataset_ops.make_one_shot_iterator(dataset)
 
 
 class FixedLossScaleManagerTest(test.TestCase):
diff --git a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py b/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py
index 9009df0eef..33f9a43e80 100644
--- a/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py
+++ b/tensorflow/contrib/mixed_precision/python/loss_scale_optimizer_test.py
@@ -132,7 +132,7 @@ class LossScaleOptimizerTest(test.TestCase):
 
     x = variable_scope.get_variable("x", initializer=1., dtype=dtypes.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices([np.nan, np.inf, 0.1])
-    itr = dataset.make_one_shot_iterator()
+    itr = dataset_ops.make_one_shot_iterator(dataset)
 
     lr = 1
     opt = gd.GradientDescentOptimizer(lr)
@@ -182,7 +182,7 @@ class LossScaleOptimizerTest(test.TestCase):
 
     x = variable_scope.get_variable("x", initializer=1., dtype=dtypes.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices([np.nan, np.inf, 0.1])
-    itr = dataset.make_one_shot_iterator()
+    itr = dataset_ops.make_one_shot_iterator(dataset)
 
     lr = 1
     init_loss_scale = 8
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index e7d6ec4ad3..31cbef89e2 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -191,7 +191,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
               batch_size=batch_size,
               num_parallel_calls=8))
       dataset = dataset.repeat(count=1)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = data.make_one_shot_iterator(dataset)
       features, labels = iterator.get_next()
       return features, labels
 
@@ -205,7 +205,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
               batch_size=batch_size,
               num_parallel_calls=8))
       dataset = dataset.repeat(count=num_epochs)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = data.make_one_shot_iterator(dataset)
       features, labels = iterator.get_next()
       return features, labels
 
diff --git a/tensorflow/contrib/tpu/python/tpu/datasets.py b/tensorflow/contrib/tpu/python/tpu/datasets.py
index d61c824eab..8d6245390f 100644
--- a/tensorflow/contrib/tpu/python/tpu/datasets.py
+++ b/tensorflow/contrib/tpu/python/tpu/datasets.py
@@ -156,7 +156,7 @@ def StreamingFilesDataset(files,
 
     source_dataset = source_dataset.prefetch(1)
 
-    source_iterator = source_dataset.make_one_shot_iterator()
+    source_iterator = dataset_ops.make_one_shot_iterator(source_dataset)
     source_handle = source_iterator.string_handle()
 
   @function.Defun(dtypes.string)
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
index 3786e52b94..55235556de 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
@@ -71,7 +71,7 @@ class TPUEstimatorStoppingSignalsTest(test.TestCase):
 
     with ops.Graph().as_default():
       dataset = input_fn(params)
-      features = dataset.make_one_shot_iterator().get_next()
+      features = dataset_lib.make_one_shot_iterator(dataset).get_next()
 
       # With tf.data.Dataset.batch, the batch is None, i.e., dynamic shape.
       self.assertIsNone(features['a'].shape.as_list()[0])
diff --git a/tensorflow/contrib/tpu/tpu_estimator.md b/tensorflow/contrib/tpu/tpu_estimator.md
index b6514e19dc..552febd80b 100644
--- a/tensorflow/contrib/tpu/tpu_estimator.md
+++ b/tensorflow/contrib/tpu/tpu_estimator.md
@@ -89,12 +89,9 @@ handle training:
 
         dataset = tf.data.TFRecordDataset(
             filename, buffer_size=FLAGS.dataset_reader_buffer_size)
-        dataset = dataset.map(parser).cache().repeat().batch(batch_size)
-        images, labels = dataset.make_one_shot_iterator().get_next()
-        # set_shape to give inputs statically known shapes.
-        images.set_shape([batch_size, 28 * 28])
-        labels.set_shape([batch_size])
-        return images, labels
+        dataset = dataset.map(parser).cache().repeat().batch(
+            batch_size, drop_remainder=True)
+        return dataset
       return input_fn
 
 
diff --git a/tensorflow/examples/get_started/regression/custom_regression.py b/tensorflow/examples/get_started/regression/custom_regression.py
index 2e34362c5c..7b7cbb7866 100644
--- a/tensorflow/examples/get_started/regression/custom_regression.py
+++ b/tensorflow/examples/get_started/regression/custom_regression.py
@@ -100,12 +100,11 @@ def main(argv):
         # that the examples are well mixed.
         train.shuffle(1000).batch(128)
         # Repeat forever
-        .repeat().make_one_shot_iterator().get_next())
+        .repeat())
 
   # Build the validation input_fn.
   def input_test():
-    return (test.shuffle(1000).batch(128)
-            .make_one_shot_iterator().get_next())
+    return test.shuffle(1000).batch(128)
 
   # The first way assigns a unique weight to each category. To do this you must
   # specify the category's vocabulary (values outside this specification will
diff --git a/tensorflow/examples/get_started/regression/dnn_regression.py b/tensorflow/examples/get_started/regression/dnn_regression.py
index 951c93b52e..94669a5082 100644
--- a/tensorflow/examples/get_started/regression/dnn_regression.py
+++ b/tensorflow/examples/get_started/regression/dnn_regression.py
@@ -45,12 +45,11 @@ def main(argv):
         # that the examples are well mixed.
         train.shuffle(1000).batch(128)
         # Repeat forever
-        .repeat().make_one_shot_iterator().get_next())
+        .repeat())
 
   # Build the validation input_fn.
   def input_test():
-    return (test.shuffle(1000).batch(128)
-            .make_one_shot_iterator().get_next())
+    return test.shuffle(1000).batch(128)
 
   # The first way assigns a unique weight to each category. To do this you must
   # specify the category's vocabulary (values outside this specification will
diff --git a/tensorflow/examples/get_started/regression/linear_regression_categorical.py b/tensorflow/examples/get_started/regression/linear_regression_categorical.py
index e2ad415fbc..5312272a95 100644
--- a/tensorflow/examples/get_started/regression/linear_regression_categorical.py
+++ b/tensorflow/examples/get_started/regression/linear_regression_categorical.py
@@ -45,12 +45,11 @@ def main(argv):
         # that the examples are well mixed.
         train.shuffle(1000).batch(128)
         # Repeat forever
-        .repeat().make_one_shot_iterator().get_next())
+        .repeat())
 
   # Build the validation input_fn.
   def input_test():
-    return (test.shuffle(1000).batch(128)
-            .make_one_shot_iterator().get_next())
+    return test.shuffle(1000).batch(128)
 
   # The following code demonstrates two of the ways that `feature_columns` can
   # be used to build a model with categorical inputs.
diff --git a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py
index 7402247448..5c52a2c846 100644
--- a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py
+++ b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py
@@ -126,7 +126,7 @@ def inputs(train, batch_size, num_epochs):
     dataset = dataset.repeat(num_epochs)
     dataset = dataset.batch(batch_size)
 
-    iterator = dataset.make_one_shot_iterator()
+    iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
   return iterator.get_next()
 
 
diff --git a/tensorflow/lite/tutorials/mnist_tflite.py b/tensorflow/lite/tutorials/mnist_tflite.py
index 002365717f..6cc5846163 100644
--- a/tensorflow/lite/tutorials/mnist_tflite.py
+++ b/tensorflow/lite/tutorials/mnist_tflite.py
@@ -34,8 +34,8 @@ flags = flags.FLAGS
 def test_image_generator():
   # Generates an iterator over images
   with tf.Session() as sess:
-    input_data = dataset.test(
-        flags.data_dir).make_one_shot_iterator().get_next()
+    input_data = tf.compat.v1.data.make_one_shot_iterator(dataset.test(
+        flags.data_dir)).get_next()
     try:
       while True:
         yield sess.run(input_data)
diff --git a/tensorflow/python/data/__init__.py b/tensorflow/python/data/__init__.py
index 7536ba668a..2da98bbb8e 100644
--- a/tensorflow/python/data/__init__.py
+++ b/tensorflow/python/data/__init__.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 # pylint: disable=unused-import
 from tensorflow.python.data import experimental
 from tensorflow.python.data.ops.dataset_ops import Dataset
+from tensorflow.python.data.ops.dataset_ops import make_one_shot_iterator
 from tensorflow.python.data.ops.iterator_ops import Iterator
 from tensorflow.python.data.ops.readers import FixedLengthRecordDataset
 from tensorflow.python.data.ops.readers import TextLineDataset
diff --git a/tensorflow/python/data/benchmarks/filter_benchmark.py b/tensorflow/python/data/benchmarks/filter_benchmark.py
index b9acdc7227..a6d86fe221 100644
--- a/tensorflow/python/data/benchmarks/filter_benchmark.py
+++ b/tensorflow/python/data/benchmarks/filter_benchmark.py
@@ -36,7 +36,7 @@ class FilterBenchmark(test.Benchmark):
     with ops.Graph().as_default():
       dataset = (
           dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
 
       with session.Session() as sess:
diff --git a/tensorflow/python/data/benchmarks/map_benchmark.py b/tensorflow/python/data/benchmarks/map_benchmark.py
index 48294eeb89..65d945cdae 100644
--- a/tensorflow/python/data/benchmarks/map_benchmark.py
+++ b/tensorflow/python/data/benchmarks/map_benchmark.py
@@ -58,7 +58,7 @@ class MapBenchmark(test.Benchmark):
                 dataset,
                 map_fn,
                 use_inter_op_parallelism=use_inter_op_parallelism)
-          iterator = dataset.make_one_shot_iterator()
+          iterator = dataset_ops.make_one_shot_iterator(dataset)
           next_element = iterator.get_next()
 
           with session.Session() as sess:
@@ -108,7 +108,7 @@ class MapBenchmark(test.Benchmark):
               dataset,
               map_fn,
               use_inter_op_parallelism=use_inter_op_parallelism)
-          iterator = dataset.make_one_shot_iterator()
+          iterator = dataset_ops.make_one_shot_iterator(dataset)
           next_element = iterator.get_next()
 
           with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
index b00e918338..b48ef95666 100644
--- a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
@@ -39,7 +39,7 @@ class AutotuneBenchmark(test.Benchmark):
                                                                1))).repeat()
     dataset = dataset.map(
         math_ops.matmul, num_parallel_calls=optimization.AUTOTUNE)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     deltas = []
@@ -76,7 +76,7 @@ class AutotuneBenchmark(test.Benchmark):
     options = dataset_ops.Options()
     options.experimental_numa_aware = numa_aware
     dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     deltas = []
@@ -108,7 +108,7 @@ class AutotuneBenchmark(test.Benchmark):
         lambda _: dataset,
         cycle_length=10,
         num_parallel_calls=optimization.AUTOTUNE)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     deltas = []
@@ -161,7 +161,7 @@ class AutotuneBenchmark(test.Benchmark):
         cycle_length=2)
 
     dataset = dataset.map(f3, num_parallel_calls=optimization.AUTOTUNE)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     deltas = []
diff --git a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
index 7eebf49c38..03345ce4e6 100644
--- a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
@@ -27,6 +27,7 @@ import numpy as np
 
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import readers
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers as core_readers
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import gfile
@@ -64,7 +65,7 @@ class CsvDatasetBenchmark(test.Benchmark):
     dataset = dataset.skip(self._num_per_iter - 1)
     deltas = []
     for _ in range(10):
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       with session.Session() as sess:
         start = time.time()
         # NOTE: This depends on the underlying implementation of skip, to have
diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
index 1e8dd0f63d..3e0701ee4f 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
@@ -143,7 +143,7 @@ class MapAndBatchBenchmark(test.Benchmark):
         chained_dataset = dataset.map(
             math_ops.matmul,
             num_parallel_calls=num_calls).batch(batch_size=batch_size)
-        chained_iterator = chained_dataset.make_one_shot_iterator()
+        chained_iterator = dataset_ops.make_one_shot_iterator(chained_dataset)
         chained_get_next = chained_iterator.get_next()
 
         chained_deltas = []
@@ -164,7 +164,7 @@ class MapAndBatchBenchmark(test.Benchmark):
                 math_ops.matmul,
                 num_parallel_calls=num_calls,
                 batch_size=batch_size))
-        fused_iterator = fused_dataset.make_one_shot_iterator()
+        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
         fused_get_next = fused_iterator.get_next()
 
         fused_deltas = []
diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
index 0c3ac8b371..47ec6391f7 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
@@ -116,13 +116,13 @@ class MapVectorizationBenchmark(test.Benchmark):
     num_elems = int(np.sum([np.prod(x) for x in input_size]))
     name_template = "{}__batch_size_{}_input_element_size_{}_{}"
     unoptimized = input_dataset.map(map_fn).batch(batch_size)
-    unoptimized_op = unoptimized.make_one_shot_iterator().get_next()
+    unoptimized_op = dataset_ops.make_one_shot_iterator(unoptimized).get_next()
 
     optimized = input_dataset.map(map_fn).batch(batch_size)
     options = dataset_ops.Options()
     options.experimental_map_vectorization = True
     optimized = optimized.with_options(options)
-    optimized_op = optimized.make_one_shot_iterator().get_next()
+    optimized_op = dataset_ops.make_one_shot_iterator(optimized).get_next()
 
     unoptimized_time = self._run(
         unoptimized_op,
diff --git a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
index 2eb5561b11..c53f8dd7c5 100644
--- a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
@@ -26,6 +26,7 @@ import numpy as np
 
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import matching_files
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import test
@@ -59,7 +60,7 @@ class MatchingFilesBenchmark(test.Benchmark):
     for _ in range(iters):
       with ops.Graph().as_default():
         dataset = matching_files.MatchingFilesDataset(patterns)
-        next_element = dataset.make_one_shot_iterator().get_next()
+        next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
         with session.Session() as sess:
           sub_deltas = []
diff --git a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
index 0eca97d26d..2f9b89111f 100644
--- a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
@@ -49,7 +49,7 @@ class OptimizationBenchmark(test.Benchmark):
         options.experimental_map_fusion = True
         dataset = dataset.with_options(options)
 
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
 
       with session.Session() as sess:
@@ -91,7 +91,7 @@ class OptimizationBenchmark(test.Benchmark):
         options = dataset_ops.Options()
         options.experimental_map_and_filter_fusion = True
         dataset = dataset.with_options(options)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
 
       with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index af20e50fb9..8264dee3c1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -105,7 +105,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
               boundaries,
               batch_sizes,
               no_padding=no_padding))
-      batch, = dataset.make_one_shot_iterator().get_next()
+      batch, = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
       with self.cached_session() as sess:
         batches = []
@@ -155,7 +155,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
             grouping.bucket_by_sequence_length(
                 element_len, boundaries, batch_sizes,
                 pad_to_bucket_boundary=True))
-    batch, = dataset.make_one_shot_iterator().get_next()
+    batch, = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       batches = []
@@ -192,7 +192,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
             grouping.bucket_by_sequence_length(
                 element_len, boundaries, batch_sizes,
                 pad_to_bucket_boundary=True))
-    batch, = dataset.make_one_shot_iterator().get_next()
+    batch, = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       batches = []
@@ -295,7 +295,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
 
     def _compute_batches(dataset):
       """Computes actual batch outputs of dataset and stores in a set."""
-      batch = dataset.make_one_shot_iterator().get_next()
+      batch = dataset_ops.make_one_shot_iterator(dataset).get_next()
       all_sparse_tensors = set()
       with self.cached_session() as sess:
         with self.assertRaises(errors.OutOfRangeError):
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index 1acd26270f..78b8bda1b7 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -42,7 +42,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -69,7 +69,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -95,7 +95,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:0"))
 
     with ops.device("/cpu:0"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -122,7 +122,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1")).prefetch(1)
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -149,7 +149,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -176,7 +176,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1")).prefetch(1)
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -209,7 +209,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -244,7 +244,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1")).prefetch(1)
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
diff --git a/tensorflow/python/data/experimental/kernel_tests/counter_test.py b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
index b370e0029e..49e1f2272b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/counter_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/counter_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import counter
 from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
@@ -29,14 +30,14 @@ class CounterTest(test_base.DatasetTestBase):
   @test_util.run_deprecated_v1
   def testCounter(self):
     """Test dataset construction using `count`."""
-    iterator = (counter.Counter(start=3, step=4)
-                .make_one_shot_iterator())
+    iterator = dataset_ops.make_one_shot_iterator(
+        counter.Counter(start=3, step=4))
     get_next = iterator.get_next()
     self.assertEqual([], get_next.shape.as_list())
     self.assertEqual(dtypes.int64, get_next.dtype)
 
-    negative_iterator = (counter.Counter(start=0, step=-1)
-                         .make_one_shot_iterator())
+    negative_iterator = dataset_ops.make_one_shot_iterator(
+        counter.Counter(start=0, step=-1))
     negative_get_next = negative_iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
index 269c27dde2..ec2c5407b4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
@@ -67,7 +67,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
         for i in range(num_datasets)
     ], weights)
     dataset = dataset.take(num_samples)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -106,7 +106,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
     choice_array = np.random.randint(3, size=(15,), dtype=np.int64)
     choice_dataset = dataset_ops.Dataset.from_tensor_slices(choice_array)
     dataset = interleave_ops.choose_from_datasets(datasets, choice_dataset)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py
index 4f8cb1246f..357b5f1b49 100644
--- a/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/filter_dataset_op_test.py
@@ -47,7 +47,7 @@ class FilterBenchmark(test.Benchmark):
       if optimize_dataset:
         dataset = dataset.apply(optimization.optimize(["filter_fusion"]))
 
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
 
       with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
index c7366f6641..8507df3d3a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_reducer_test.py
@@ -37,7 +37,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
 
   def checkResults(self, dataset, shapes, values):
     self.assertEqual(shapes, dataset.output_shapes)
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
     with self.cached_session() as sess:
       for expected in values:
         got = self.evaluate(get_next)
@@ -130,7 +130,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
           grouping.group_by_reducer(lambda x: x, reducer))
       self.assertEqual([None], dataset.output_shapes[0].as_list())
       self.assertIs(None, dataset.output_shapes[1].ndims)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       get_next = iterator.get_next()
       with self.cached_session() as sess:
         x, y = self.evaluate(get_next)
@@ -194,7 +194,7 @@ class GroupByReducerTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.zip(
         (dataset_ops.Dataset.range(10), dataset_ops.Dataset.range(10))).apply(
             grouping.group_by_reducer(lambda x, y: np.int64(0), reducer))
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
     with self.cached_session() as sess:
       x, y = self.evaluate(get_next)
       self.assertAllEqual(x, np.asarray([x for x in range(10)]))
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
index 48916471b3..3a0ff018e8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
@@ -21,6 +21,7 @@ import numpy as np
 
 from tensorflow.python.data.experimental.kernel_tests import reader_dataset_ops_test_base
 from tensorflow.python.data.experimental.ops import readers
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers as core_readers
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
@@ -41,11 +42,12 @@ class MakeBatchedFeaturesDatasetTest(
         with ops.Graph().as_default() as g:
           with self.session(graph=g) as sess:
             # Basic test: read from file 0.
-            self.outputs = self.make_batch_feature(
-                filenames=self.test_filenames[0],
-                label_key="label",
-                num_epochs=num_epochs,
-                batch_size=batch_size).make_one_shot_iterator().get_next()
+            self.outputs = dataset_ops.make_one_shot_iterator(
+                self.make_batch_feature(
+                    filenames=self.test_filenames[0],
+                    label_key="label",
+                    num_epochs=num_epochs,
+                    batch_size=batch_size)).get_next()
             self.verify_records(
                 sess,
                 batch_size,
@@ -58,11 +60,12 @@ class MakeBatchedFeaturesDatasetTest(
         with ops.Graph().as_default() as g:
           with self.session(graph=g) as sess:
             # Basic test: read from file 1.
-            self.outputs = self.make_batch_feature(
-                filenames=self.test_filenames[1],
-                label_key="label",
-                num_epochs=num_epochs,
-                batch_size=batch_size).make_one_shot_iterator().get_next()
+            self.outputs = dataset_ops.make_one_shot_iterator(
+                self.make_batch_feature(
+                    filenames=self.test_filenames[1],
+                    label_key="label",
+                    num_epochs=num_epochs,
+                    batch_size=batch_size)).get_next()
             self.verify_records(
                 sess,
                 batch_size,
@@ -75,11 +78,12 @@ class MakeBatchedFeaturesDatasetTest(
         with ops.Graph().as_default() as g:
           with self.session(graph=g) as sess:
             # Basic test: read from both files.
-            self.outputs = self.make_batch_feature(
-                filenames=self.test_filenames,
-                label_key="label",
-                num_epochs=num_epochs,
-                batch_size=batch_size).make_one_shot_iterator().get_next()
+            self.outputs = dataset_ops.make_one_shot_iterator(
+                self.make_batch_feature(
+                    filenames=self.test_filenames,
+                    label_key="label",
+                    num_epochs=num_epochs,
+                    batch_size=batch_size)).get_next()
             self.verify_records(
                 sess,
                 batch_size,
@@ -91,10 +95,11 @@ class MakeBatchedFeaturesDatasetTest(
         with ops.Graph().as_default() as g:
           with self.session(graph=g) as sess:
             # Basic test: read from both files.
-            self.outputs = self.make_batch_feature(
-                filenames=self.test_filenames,
-                num_epochs=num_epochs,
-                batch_size=batch_size).make_one_shot_iterator().get_next()
+            self.outputs = dataset_ops.make_one_shot_iterator(
+                self.make_batch_feature(
+                    filenames=self.test_filenames,
+                    num_epochs=num_epochs,
+                    batch_size=batch_size)).get_next()
             self.verify_records(sess, batch_size, num_epochs=num_epochs)
             with self.assertRaises(errors.OutOfRangeError):
               self._next_actual_batch(sess)
@@ -130,18 +135,18 @@ class MakeBatchedFeaturesDatasetTest(
       # Test that shuffling with same seed produces the same result.
       with ops.Graph().as_default() as g:
         with self.session(graph=g) as sess:
-          outputs1 = self.make_batch_feature(
+          outputs1 = dataset_ops.make_one_shot_iterator(self.make_batch_feature(
               filenames=self.test_filenames[0],
               num_epochs=num_epochs,
               batch_size=batch_size,
               shuffle=True,
-              shuffle_seed=5).make_one_shot_iterator().get_next()
-          outputs2 = self.make_batch_feature(
+              shuffle_seed=5)).get_next()
+          outputs2 = dataset_ops.make_one_shot_iterator(self.make_batch_feature(
               filenames=self.test_filenames[0],
               num_epochs=num_epochs,
               batch_size=batch_size,
               shuffle=True,
-              shuffle_seed=5).make_one_shot_iterator().get_next()
+              shuffle_seed=5)).get_next()
           for _ in range(total_records // batch_size):
             batch1 = self._run_actual_batch(outputs1, sess)
             batch2 = self._run_actual_batch(outputs2, sess)
@@ -151,18 +156,18 @@ class MakeBatchedFeaturesDatasetTest(
       # Test that shuffling with different seeds produces a different order.
       with ops.Graph().as_default() as g:
         with self.session(graph=g) as sess:
-          outputs1 = self.make_batch_feature(
+          outputs1 = dataset_ops.make_one_shot_iterator(self.make_batch_feature(
               filenames=self.test_filenames[0],
               num_epochs=num_epochs,
               batch_size=batch_size,
               shuffle=True,
-              shuffle_seed=5).make_one_shot_iterator().get_next()
-          outputs2 = self.make_batch_feature(
+              shuffle_seed=5)).get_next()
+          outputs2 = dataset_ops.make_one_shot_iterator(self.make_batch_feature(
               filenames=self.test_filenames[0],
               num_epochs=num_epochs,
               batch_size=batch_size,
               shuffle=True,
-              shuffle_seed=15).make_one_shot_iterator().get_next()
+              shuffle_seed=15)).get_next()
           all_equal = True
           for _ in range(total_records // batch_size):
             batch1 = self._run_actual_batch(outputs1, sess)
@@ -178,14 +183,14 @@ class MakeBatchedFeaturesDatasetTest(
         for parser_num_threads in [2, 4]:
           with ops.Graph().as_default() as g:
             with self.session(graph=g) as sess:
-              self.outputs = self.make_batch_feature(
-                  filenames=self.test_filenames,
-                  label_key="label",
-                  num_epochs=num_epochs,
-                  batch_size=batch_size,
-                  reader_num_threads=reader_num_threads,
-                  parser_num_threads=parser_num_threads).make_one_shot_iterator(
-                  ).get_next()
+              self.outputs = dataset_ops.make_one_shot_iterator(
+                  self.make_batch_feature(
+                      filenames=self.test_filenames,
+                      label_key="label",
+                      num_epochs=num_epochs,
+                      batch_size=batch_size,
+                      reader_num_threads=reader_num_threads,
+                      parser_num_threads=parser_num_threads)).get_next()
               self.verify_records(
                   sess,
                   batch_size,
@@ -197,13 +202,13 @@ class MakeBatchedFeaturesDatasetTest(
 
           with ops.Graph().as_default() as g:
             with self.session(graph=g) as sess:
-              self.outputs = self.make_batch_feature(
-                  filenames=self.test_filenames,
-                  num_epochs=num_epochs,
-                  batch_size=batch_size,
-                  reader_num_threads=reader_num_threads,
-                  parser_num_threads=parser_num_threads).make_one_shot_iterator(
-                  ).get_next()
+              self.outputs = dataset_ops.make_one_shot_iterator(
+                  self.make_batch_feature(
+                      filenames=self.test_filenames,
+                      num_epochs=num_epochs,
+                      batch_size=batch_size,
+                      reader_num_threads=reader_num_threads,
+                      parser_num_threads=parser_num_threads)).get_next()
               self.verify_records(
                   sess,
                   batch_size,
@@ -217,12 +222,12 @@ class MakeBatchedFeaturesDatasetTest(
       for num_epochs in [1, 10]:
         with ops.Graph().as_default():
           # Basic test: read from file 0.
-          outputs = self.make_batch_feature(
+          outputs = dataset_ops.make_one_shot_iterator(self.make_batch_feature(
               filenames=self.test_filenames[0],
               label_key="label",
               num_epochs=num_epochs,
               batch_size=batch_size,
-              drop_final_batch=True).make_one_shot_iterator().get_next()
+              drop_final_batch=True)).get_next()
           for tensor in nest.flatten(outputs):
             if isinstance(tensor, ops.Tensor):  # Guard against SparseTensor.
               self.assertEqual(tensor.shape[0], batch_size)
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index bcbaf1a7c4..e80accee33 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -25,6 +25,7 @@ import numpy as np
 
 from tensorflow.python.data.experimental.ops import readers
 from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -83,7 +84,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
       expected_output,
       expected_keys,
   ):
-    nxt = dataset.make_one_shot_iterator().get_next()
+    nxt = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     for expected_features in self._next_expected_batch(
         expected_output,
@@ -613,8 +614,8 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
               shuffle_seed=5,
               num_epochs=2,
           )
-          outputs1 = dataset1.make_one_shot_iterator().get_next()
-          outputs2 = dataset2.make_one_shot_iterator().get_next()
+          outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
+          outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
           for _ in range(total_records // batch_size):
             batch1 = nest.flatten(self.evaluate(outputs1))
             batch2 = nest.flatten(self.evaluate(outputs2))
@@ -644,8 +645,8 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
               shuffle_seed=6,
               num_epochs=2,
           )
-          outputs1 = dataset1.make_one_shot_iterator().get_next()
-          outputs2 = dataset2.make_one_shot_iterator().get_next()
+          outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
+          outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
           all_equal = False
           for _ in range(total_records // batch_size):
             batch1 = nest.flatten(self.evaluate(outputs1))
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
index 0bb7b7c5f3..64798feaf8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 from tensorflow.python.data.experimental.kernel_tests import reader_dataset_ops_test_base
 from tensorflow.python.data.experimental.ops import readers
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -122,14 +123,15 @@ class MakeTFRecordDatasetTest(
 
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
-        outputs = readers.make_tf_record_dataset(
-            file_pattern=file_pattern,
-            num_epochs=num_epochs,
-            batch_size=batch_size,
-            parser_fn=fn,
-            num_parallel_reads=num_parallel_reads,
-            drop_final_batch=drop_final_batch,
-            shuffle=False).make_one_shot_iterator().get_next()
+        outputs = dataset_ops.make_one_shot_iterator(
+            readers.make_tf_record_dataset(
+                file_pattern=file_pattern,
+                num_epochs=num_epochs,
+                batch_size=batch_size,
+                parser_fn=fn,
+                num_parallel_reads=num_parallel_reads,
+                drop_final_batch=drop_final_batch,
+                shuffle=False)).get_next()
         self._verify_records(
             sess, outputs, batch_size, file_index, num_epochs=num_epochs,
             interleave_cycle_length=num_parallel_reads,
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index fc97f63358..cad04a70e3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -147,7 +147,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     if drop_remainder:
       self.assertEqual([4, 1], iterator.output_shapes.as_list())
@@ -177,7 +177,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
 
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.cached_session() as sess:
@@ -199,7 +199,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     elements = []
     for _ in range(100):
@@ -229,7 +229,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     elements = []
     for _ in range(100):
@@ -358,7 +358,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -398,7 +398,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -449,7 +449,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
 
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       for _ in range(10):
@@ -466,7 +466,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testShortCircuit(self, structure, map_fn, num_parallel_calls):
     dataset = self.structuredDataset(structure).repeat().apply(
         batching.map_and_batch(map_fn, batch_size=10))
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       if isinstance(structure, tuple):
@@ -509,7 +509,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
     with self.cached_session() as sess:
       for i in range(10):
diff --git a/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
index 787a7a91a4..0ee7616d35 100644
--- a/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/matching_files_test.py
@@ -23,6 +23,7 @@ import tempfile
 
 from tensorflow.python.data.experimental.ops import matching_files
 from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
@@ -49,7 +50,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
     dataset = matching_files.MatchingFilesDataset(
         os.path.join(self.tmp_dir, '*'))
     with self.cached_session() as sess:
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       with self.assertRaises(errors.NotFoundError):
         sess.run(next_element)
 
@@ -60,7 +61,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
     dataset = matching_files.MatchingFilesDataset(
         os.path.join(self.tmp_dir, '*'))
     with self.cached_session() as sess:
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       with self.assertRaises(errors.NotFoundError):
         sess.run(next_element)
 
@@ -74,7 +75,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
     dataset = matching_files.MatchingFilesDataset(
         os.path.join(self.tmp_dir, '*'))
     with self.cached_session() as sess:
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
       expected_filenames = []
       actual_filenames = []
@@ -97,7 +98,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
     dataset = matching_files.MatchingFilesDataset(
         os.path.join(self.tmp_dir, '*.py'))
     with self.cached_session() as sess:
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       expected_filenames = []
       actual_filenames = []
       for filename in filenames[1:-1]:
@@ -119,7 +120,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
     dataset = matching_files.MatchingFilesDataset(
         os.path.join(self.tmp_dir, 'b*.py*'))
     with self.cached_session() as sess:
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       expected_filenames = []
       actual_filenames = []
       for filename in filenames[1:3]:
@@ -156,7 +157,7 @@ class MatchingFilesTest(test_base.DatasetTestBase):
 
     dataset = matching_files.MatchingFilesDataset(patterns)
     with self.cached_session() as sess:
-      next_element = dataset.make_one_shot_iterator().get_next()
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       expected_filenames = [
           compat.as_bytes(filename)
           for filename in filenames
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
index 0aacf8bb07..5f3a8683fb 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
@@ -70,7 +70,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
       iterator = dataset.__iter__()
       get_next = iterator._next_internal  # pylint: disable=protected-access
     else:
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       get_next = iterator.get_next
     for _ in range(5):
       result = self.evaluate(get_next())
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index d979aaa5a0..c2e08e2cd8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -376,7 +376,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch(
         5, drop_remainder=True)
     _, optimized = self._get_test_datasets(base_dataset, map_fn)
-    nxt = optimized.make_one_shot_iterator().get_next()
+    nxt = dataset_ops.make_one_shot_iterator(optimized).get_next()
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r"indices = 10 is not in \[0, 5\)"):
       self.evaluate(nxt)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index d3c121491a..0f0274b41f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -37,7 +37,7 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options.experimental_autotune = True
     dataset = dataset.with_options(options)
 
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
index 77f0dc8e81..c09bfa695e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
@@ -603,7 +603,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
     dataset = dataset.apply(
         interleave_ops.parallel_interleave(
             interleave_fn, cycle_length=16, block_length=2, sloppy=sloppy))
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     with self.cached_session() as sess:
       output_values = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index d280321829..19d5cdb80c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -38,7 +38,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -66,7 +66,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
             "/job:localhost/replica:0/task:0/device:CPU:0"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -92,7 +92,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -123,7 +123,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
diff --git a/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py b/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py
index 675209d189..76f68f50c8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/rejection_resample_test.py
@@ -48,7 +48,7 @@ def _time_resampling(
           initial_dist=init_dist,
           seed=142))
 
-  get_next = dataset.make_one_shot_iterator().get_next()
+  get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
   with test_obj.test_session() as sess:
     start_time = time.time()
@@ -73,12 +73,12 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices(classes).shuffle(
         200, seed=21).map(lambda c: (c, string_ops.as_string(c))).repeat()
 
-    get_next = dataset.apply(
+    get_next = dataset_ops.make_one_shot_iterator(dataset.apply(
         resampling.rejection_resample(
             target_dist=target_dist,
             initial_dist=initial_dist,
             class_func=lambda c, _: c,
-            seed=27)).make_one_shot_iterator().get_next()
+            seed=27))).get_next()
 
     with self.cached_session() as sess:
       returned = []
@@ -117,7 +117,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase):
             target_dist=target_dist,
             initial_dist=init_dist))
 
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       returned = []
@@ -149,7 +149,7 @@ class RejectionResampleTest(test_base.DatasetTestBase, parameterized.TestCase):
             target_dist=target_dist,
             initial_dist=init_dist))
 
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       returned = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index 43a765b9e0..3b978b004b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -67,9 +67,9 @@ class ScanTest(test_base.DatasetTestBase):
 
   @test_util.run_in_graph_and_eager_modes
   def testFibonacci(self):
-    iterator = dataset_ops.Dataset.from_tensors(1).repeat(None).apply(
-        scan_ops.scan([0, 1], lambda a, _: ([a[1], a[0] + a[1]], a[1]))
-    ).make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(
+        dataset_ops.Dataset.from_tensors(1).repeat(None).apply(
+            scan_ops.scan([0, 1], lambda a, _: ([a[1], a[0] + a[1]], a[1]))))
 
     if context.executing_eagerly():
       next_element = iterator.get_next
@@ -134,7 +134,7 @@ class ScanTest(test_base.DatasetTestBase):
     self.assertIs(None, dataset.output_shapes[0][1].ndims)
     self.assertEqual([], dataset.output_shapes[1].as_list())
 
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
index ce63da6bf9..9528f83291 100644
--- a/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/shuffle_and_repeat_test.py
@@ -35,7 +35,7 @@ class ShuffleAndRepeatTest(test_base.DatasetTestBase):
         shuffle_ops.shuffle_and_repeat(buffer_size=5, count=count, seed=seed))
 
   def _gen_outputs(self, ds_fn, num_outputs, verify_exhausted=True):
-    get_next = ds_fn().make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(ds_fn()).get_next()
     outputs = []
     with self.cached_session() as sess:
       for _ in range(num_outputs):
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index 9c6830d993..b6b12bd155 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -63,7 +63,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     self.assertEqual(expected_types, data.output_types)
 
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     op = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -84,7 +84,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     self.assertEqual(expected_types, data.output_types)
 
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     op = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -104,7 +104,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     data = data.batch(5)
     data = data.apply(batching.unbatch())
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -126,7 +126,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     data = data.batch(5)
     data = data.apply(batching.unbatch())
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -149,7 +149,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     self.assertEqual(expected_types, data.output_types)
 
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     op = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -170,7 +170,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     self.assertAllEqual(expected_types, data.output_types)
 
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     op = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -187,7 +187,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
         (constant_op.constant([]), constant_op.constant([], shape=[0, 4]),
          constant_op.constant([], shape=[0, 4, 0])))
     data = data.apply(batching.unbatch())
-    iterator = data.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(data)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/ops/readers.py b/tensorflow/python/data/experimental/ops/readers.py
index f6faf04998..a4c260dde7 100644
--- a/tensorflow/python/data/experimental/ops/readers.py
+++ b/tensorflow/python/data/experimental/ops/readers.py
@@ -572,7 +572,9 @@ class CsvDatasetV2(dataset_ops.DatasetSource):
 
     We can construct a CsvDataset from it as follows:
     ```python
-    dataset = tf.data.experimental.CsvDataset(
+    tf.enable_eager_execution()
+
+     dataset = tf.data.experimental.CsvDataset(
         "my_file*.csv",
         [tf.float32,  # Required field, use dtype or empty tensor
          tf.constant([0.0], dtype=tf.float32),  # Optional field, default to 0.0
@@ -584,13 +586,8 @@ class CsvDatasetV2(dataset_ops.DatasetSource):
 
     The expected output of its iterations is:
     ```python
-    next_element = dataset.make_one_shot_iterator().get_next()
-    with tf.Session() as sess:
-      while True:
-        try:
-          print(sess.run(next_element))
-        except tf.errors.OutOfRangeError:
-          break
+    for element in dataset:
+      print(element)
 
     >> (4.28e10, 5.55e6, 12)
     >> (-5.3e14, 0.0, 2)
@@ -938,17 +935,14 @@ class SqlDatasetV2(dataset_ops.DatasetSource):
     For example:
 
     ```python
+    tf.enable_eager_execution()
+
     dataset = tf.data.experimental.SqlDataset("sqlite", "/foo/bar.sqlite3",
                                               "SELECT name, age FROM people",
                                               (tf.string, tf.int32))
-    iterator = dataset.make_one_shot_iterator()
-    next_element = iterator.get_next()
     # Prints the rows of the result set of the above query.
-    while True:
-      try:
-        print(sess.run(next_element))
-      except tf.errors.OutOfRangeError:
-        break
+    for element in dataset:
+      print(element)
     ```
 
     Args:
diff --git a/tensorflow/python/data/experimental/ops/stats_aggregator.py b/tensorflow/python/data/experimental/ops/stats_aggregator.py
index f54881b150..d5fcc033ab 100644
--- a/tensorflow/python/data/experimental/ops/stats_aggregator.py
+++ b/tensorflow/python/data/experimental/ops/stats_aggregator.py
@@ -47,7 +47,6 @@ class StatsAggregator(object):
   options = dataset_ops.Options()
   options.experimental_stats = tf.data.experimental.StatsOptions(aggregator)
   dataset = dataset.with_options(options)
-  iterator = dataset.make_one_shot_iterator()
   ```
 
   To get a protocol buffer summary of the currently aggregated statistics,
diff --git a/tensorflow/python/data/experimental/ops/stats_options.py b/tensorflow/python/data/experimental/ops/stats_options.py
index 6df608c608..6e884aa08a 100644
--- a/tensorflow/python/data/experimental/ops/stats_options.py
+++ b/tensorflow/python/data/experimental/ops/stats_options.py
@@ -32,14 +32,12 @@ class StatsOptions(options.OptionsBase):
   pattern:
 
   ```python
-  aggretator = tf.data.experimental.StatsAggregator()
+  aggregator = tf.data.experimental.StatsAggregator()
 
   options = tf.data.Options()
   options.experimental_stats = tf.data.experimental.StatsOptions()
   options.experimental_stats.aggregator = aggregator
   dataset = dataset.with_options(options)
-
-  iterator = dataset.make_one_shot_iterator()
   ```
 
   Note: a `StatsAggregator` object can be attached either duing construction or
diff --git a/tensorflow/python/data/kernel_tests/from_generator_test.py b/tensorflow/python/data/kernel_tests/from_generator_test.py
index d58e3c2364..e113b45496 100644
--- a/tensorflow/python/data/kernel_tests/from_generator_test.py
+++ b/tensorflow/python/data/kernel_tests/from_generator_test.py
@@ -56,11 +56,10 @@ class FromGeneratorTest(test_base.DatasetTestBase):
           sess.run(get_next)
 
   def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats):
-    iterator = (
+    iterator = dataset_ops.make_one_shot_iterator(
         dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64)
         .repeat(num_repeats)
-        .prefetch(5)
-        .make_one_shot_iterator())
+        .prefetch(5))
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/kernel_tests/interleave_test.py b/tensorflow/python/data/kernel_tests/interleave_test.py
index cd1d850304..c3450e6525 100644
--- a/tensorflow/python/data/kernel_tests/interleave_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_test.py
@@ -116,7 +116,7 @@ def _make_coordinated_sloppy_dataset(input_values, cycle_length, block_length,
   dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
       2).interleave(interleave_fn, cycle_length, block_length,
                     num_parallel_calls).with_options(options)
-  iterator = dataset.make_one_shot_iterator()
+  iterator = dataset_ops.make_one_shot_iterator(dataset)
   get_next = iterator.get_next()
   return get_next, coordination_events
 
diff --git a/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
index fc4164c81a..2d9d554e5e 100644
--- a/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
@@ -40,7 +40,7 @@ class IteratorCheckpointingTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map(
         math_ops.square).batch(2)
     iterator = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
+    ) else dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator.get_next())
     checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
@@ -61,16 +61,16 @@ class IteratorCheckpointingTest(test_base.DatasetTestBase):
         [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
     dataset = dataset.map(math_ops.square).batch(2)
     iterator_1 = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
+    ) else dataset_ops.make_one_shot_iterator(dataset)
     get_next_1 = iterator_1.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator_1.get_next())
     iterator_2 = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
+    ) else dataset_ops.make_one_shot_iterator(dataset)
     get_next_2 = iterator_2.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator_2.get_next())
     dataset_2 = dataset_ops.Dataset.range(10)
     iterator_3 = iter(dataset_2) if context.executing_eagerly(
-    ) else dataset_2.make_one_shot_iterator()
+    ) else dataset_ops.make_one_shot_iterator(dataset_2)
     get_next_3 = iterator_3.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator_3.get_next())
     checkpoint = checkpointable_utils.Checkpoint(
@@ -93,7 +93,7 @@ class IteratorCheckpointingTest(test_base.DatasetTestBase):
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
     dataset = dataset_ops.Dataset.range(3)
     iterator = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_one_shot_iterator()
+    ) else dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next if context.executing_eagerly(
     ) else functools.partial(self.evaluate, iterator.get_next())
     checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
diff --git a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
index c1f856ec62..6e8210f904 100644
--- a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
@@ -47,7 +47,7 @@ class IteratorClusterTest(test.TestCase):
 
     with ops.device("/job:worker/replica:0/task:0/cpu:1"):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
-      iterator_3 = dataset_3.make_one_shot_iterator()
+      iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
       iterator_3_handle = iterator_3.string_handle()
 
     with ops.device("/job:worker/replica:0/task:0/cpu:0"):
@@ -62,7 +62,7 @@ class IteratorClusterTest(test.TestCase):
   def _testRemoteIteratorHelper(self, device0, device1, target):
     with ops.device(device1):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
-      iterator_3 = dataset_3.make_one_shot_iterator()
+      iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
       iterator_3_handle = iterator_3.string_handle()
 
     @function.Defun(dtypes.string)
diff --git a/tensorflow/python/data/kernel_tests/iterator_test.py b/tensorflow/python/data/kernel_tests/iterator_test.py
index b836a6aecf..32332a0790 100644
--- a/tensorflow/python/data/kernel_tests/iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_test.py
@@ -61,7 +61,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     side = constant_op.constant(0.)
     add = lambda x: x + side
     dataset = dataset_ops.Dataset.from_tensor_slices(component).map(add)
-    value = dataset.make_one_shot_iterator().get_next()
+    value = dataset_ops.make_one_shot_iterator(dataset).get_next()
     self.assertIsNone(gradients_impl.gradients(value, component)[0])
     self.assertIsNone(gradients_impl.gradients(value, side)[0])
     self.assertIsNone(gradients_impl.gradients(value, [component, side])[0])
@@ -75,7 +75,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(
         ValueError, r"`Dataset.make_one_shot_iterator\(\)` does not support "
         "datasets that capture stateful objects.+myvar"):
-      dataset.make_one_shot_iterator()
+      dataset_ops.make_one_shot_iterator(dataset)
 
   @test_util.run_deprecated_v1
   def testOneShotIterator(self):
@@ -86,9 +86,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     def _map_fn(x, y, z):
       return math_ops.square(x), math_ops.square(y), math_ops.square(z)
 
-    iterator = (
+    iterator = dataset_ops.make_one_shot_iterator(
         dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn)
-        .repeat(14).make_one_shot_iterator())
+        .repeat(14))
     get_next = iterator.get_next()
 
     self.assertEqual([c.shape[1:] for c in components],
@@ -113,9 +113,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     def _map_fn(x, y, z):
       return math_ops.square(x), math_ops.square(y), math_ops.square(z)
 
-    iterator = (
+    iterator = dataset_ops.make_one_shot_iterator(
         dataset_ops.Dataset.from_tensor_slices(tensor_components)
-        .map(_map_fn).repeat(14).make_one_shot_iterator())
+        .map(_map_fn).repeat(14))
     get_next = iterator.get_next()
 
     self.assertEqual([c.shape[1:] for c in components],
@@ -140,9 +140,9 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       def _map_fn(x, y, z):
         return math_ops.square(x), math_ops.square(y), math_ops.square(z)
 
-      iterator = (
+      iterator = dataset_ops.make_one_shot_iterator(
           dataset_ops.Dataset.from_tensor_slices(components)
-          .map(_map_fn).repeat(14).make_one_shot_iterator())
+          .map(_map_fn).repeat(14))
       return iterator.get_next()
 
     server = server_lib.Server.create_local_server()
@@ -169,7 +169,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
   @test_util.run_deprecated_v1
   def testOneShotIteratorNonBlocking(self):
     dataset = dataset_ops.Dataset.from_tensors([1, 2, 3]).map(lambda x: x * x)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     next_element = iterator.get_next()
 
     # Create a session with a single thread to ensure that the
@@ -211,7 +211,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensors(
         array_ops.check_numerics(
             constant_op.constant(1.0) / constant_op.constant(0.0), "oops"))
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -404,8 +404,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
     dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40])
 
-    iterator_3 = dataset_3.make_one_shot_iterator()
-    iterator_4 = dataset_4.make_one_shot_iterator()
+    iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
+    iterator_4 = dataset_ops.make_one_shot_iterator(dataset_4)
 
     handle_placeholder = array_ops.placeholder(dtypes.string, shape=[])
     feedable_iterator = iterator_ops.Iterator.from_string_handle(
@@ -461,8 +461,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
       dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40])
 
-      iterator_3 = dataset_3.make_one_shot_iterator()
-      iterator_4 = dataset_4.make_one_shot_iterator()
+      iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
+      iterator_4 = dataset_ops.make_one_shot_iterator(dataset_4)
 
       handle_placeholder = array_ops.placeholder(dtypes.string, shape=[])
       feedable_iterator = iterator_ops.Iterator.from_string_handle(
@@ -522,7 +522,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
   @test_util.run_deprecated_v1
   def testIteratorStringHandleReuseTensorObject(self):
     dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
-    one_shot_iterator = dataset.make_one_shot_iterator()
+    one_shot_iterator = dataset_ops.make_one_shot_iterator(dataset)
     initializable_iterator = dataset.make_initializable_iterator()
     structure_iterator = iterator_ops.Iterator.from_structure(
         dataset.output_types)
@@ -564,10 +564,10 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
         handle_placeholder, dtypes.int32)
 
     with self.cached_session() as sess:
-      handle_int_scalar = sess.run(
-          dataset_int_scalar.make_one_shot_iterator().string_handle())
-      handle_float_vector = sess.run(
-          dataset_float_vector.make_one_shot_iterator().string_handle())
+      handle_int_scalar = sess.run(dataset_ops.make_one_shot_iterator(
+          dataset_int_scalar).string_handle())
+      handle_float_vector = sess.run(dataset_ops.make_one_shot_iterator(
+          dataset_float_vector).string_handle())
 
       self.assertEqual(1,
                        sess.run(
@@ -596,7 +596,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
     with ops.device("/job:localhost/replica:0/task:0/cpu:1"):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
-      iterator_3 = dataset_3.make_one_shot_iterator()
+      iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
       iterator_3_handle = iterator_3.string_handle()
 
     @function.Defun(dtypes.string)
@@ -669,7 +669,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     for device in worker_devices:
       with ops.device(device):
         src = dataset_ops.Dataset.from_tensor_slices([device])
-        itr = src.make_one_shot_iterator()
+        itr = dataset_ops.make_one_shot_iterator(src)
         itr_handles.append(itr.string_handle())
 
     targets = dataset_ops.Dataset.from_tensor_slices(worker_devices)
@@ -705,7 +705,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
     with ops.device("/job:localhost/replica:0/task:0/cpu:0"):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
-      iterator_3 = dataset_3.make_one_shot_iterator()
+      iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3)
       iterator_3_handle = iterator_3.string_handle()
 
     def _encode_raw(byte_array):
@@ -815,7 +815,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
   @test_util.run_deprecated_v1
   def testRepeatedGetNextWarning(self):
-    iterator = dataset_ops.Dataset.range(10).make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset_ops.Dataset.range(10))
     warnings.simplefilter("always")
     with warnings.catch_warnings(record=True) as w:
       for _ in range(100):
@@ -858,8 +858,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
                             expected_output_classes, expected_output_types,
                             expected_output_shapes):
     tf_value = tf_value_fn()
-    iterator = dataset_ops.Dataset.from_tensors(
-        tf_value).make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(
+        dataset_ops.Dataset.from_tensors(tf_value))
 
     self.assertTrue(expected_element_structure.is_compatible_with(
         iterator._element_structure))
@@ -872,7 +872,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
   def testIteratorGetNextName(self):
     with ops.Graph().as_default():
-      iterator = dataset_ops.Dataset.from_tensors(37.0).make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(
+          dataset_ops.Dataset.from_tensors(37.0))
       next_element = iterator.get_next(name="overridden_name")
       self.assertEqual("overridden_name", next_element.op.name)
 
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index a9c4d79429..df522c4351 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -77,7 +77,7 @@ def _make_coordinated_sloppy_dataset(num_elements, num_parallel_calls):
   options.experimental_deterministic = False
   dataset = dataset_ops.Dataset.range(num_elements).map(
       map_fn, num_parallel_calls).with_options(options)
-  iterator = dataset.make_one_shot_iterator()
+  iterator = dataset_ops.make_one_shot_iterator(dataset)
   next_element = iterator.get_next()
   return next_element, coordination_events
 
@@ -544,8 +544,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset_tuple = dataset_tuple.map(preprocess_tuple)
     dataset_namedtuple = dataset_namedtuple.map(preprocess_namedtuple)
 
-    next_tuple = dataset_tuple.make_one_shot_iterator().get_next()
-    next_namedtuple = dataset_namedtuple.make_one_shot_iterator().get_next()
+    next_tuple = dataset_ops.make_one_shot_iterator(dataset_tuple).get_next()
+    next_namedtuple = dataset_ops.make_one_shot_iterator(
+        dataset_namedtuple).get_next()
 
     # make sure both datasets contain the same data
     with self.cached_session() as sess:
@@ -966,7 +967,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return tids
 
     dataset = make_dataset_fn(dataset, _map_fn)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -987,7 +988,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testShortCircuit(self, structure, map_fn, num_parallel_calls):
     dataset = self.structuredDataset(structure).repeat().map(
         map_fn, num_parallel_calls=num_parallel_calls)
-    get_next = dataset.make_one_shot_iterator().get_next()
+    get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
     with self.cached_session() as sess:
       if isinstance(structure, tuple):
diff --git a/tensorflow/python/data/kernel_tests/shuffle_test.py b/tensorflow/python/data/kernel_tests/shuffle_test.py
index 05d5d814c0..ae892a7921 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_test.py
@@ -118,9 +118,8 @@ class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
   @test_util.run_deprecated_v1
   def testSkipEagerSeedZero(self):
     """Test for same behavior when the seed is a Python or Tensor zero."""
-    iterator = (
-        dataset_ops.Dataset.range(10).shuffle(10, seed=0)
-        .make_one_shot_iterator())
+    iterator = dataset_ops.make_one_shot_iterator(
+        dataset_ops.Dataset.range(10).shuffle(10, seed=0))
     get_next = iterator.get_next()
 
     elems = []
@@ -197,7 +196,7 @@ class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
         dataset = dataset_ops.Dataset.range(10).shuffle(
             10, seed=op_level_seed, reshuffle_each_iteration=reshuffle).repeat(
                 3)
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         next_element = iterator.get_next()
 
         run_results = []
@@ -226,7 +225,8 @@ class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
       if initializable:
         iterators = [dataset.make_initializable_iterator() for _ in range(2)]
       else:
-        iterators = [dataset.make_one_shot_iterator() for _ in range(2)]
+        iterators = [dataset_ops.make_one_shot_iterator(dataset)
+                     for _ in range(2)]
 
       results = []
       with self.session(graph=g) as sess:
diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index 03fc0da149..ca6ecdf198 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -65,7 +65,7 @@ class DatasetTestBase(test.TestCase):
         iterator = dataset.make_initializable_iterator()
         self.evaluate(iterator.initializer)
       else:
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
       get_next = iterator.get_next()
       return lambda: get_next
 
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 0843790768..c045299275 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -199,55 +199,6 @@ class DatasetV2(object):
       raise RuntimeError("dataset.__iter__() is only supported when eager "
                          "execution is enabled.")
 
-  def make_one_shot_iterator(self):
-    """Creates an `Iterator` for enumerating the elements of this dataset.
-
-    Note: The returned iterator will be initialized automatically.
-    A "one-shot" iterator does not currently support re-initialization.
-
-    Returns:
-      An `Iterator` over the elements of this dataset.
-    """
-    if context.executing_eagerly():
-      dataset = self._apply_options()
-      return iterator_ops.EagerIterator(dataset)
-
-    graph_level_seed, op_level_seed = core_random_seed.get_seed(None)
-
-    # NOTE(mrry): We capture by value here to ensure that `_make_dataset()` is
-    # a 0-argument function.
-    @function.Defun(capture_by_value=True)
-    def _make_dataset():
-      """Factory function for a dataset."""
-      # NOTE(mrry): `Defun` does not capture the graph-level seed from the
-      # enclosing graph, so if a graph-level seed is present we set the local
-      # graph seed based on a combination of the graph- and op-level seeds.
-      if graph_level_seed is not None:
-        assert op_level_seed is not None
-        core_random_seed.set_random_seed(
-            (graph_level_seed + 87654321 * op_level_seed) % (2 ** 63 - 1))
-
-      dataset = self._apply_options()
-      return dataset._as_variant_tensor()  # pylint: disable=protected-access
-
-    try:
-      _make_dataset.add_to_graph(ops.get_default_graph())
-    except ValueError as err:
-      if "Cannot capture a stateful node" in str(err):
-        raise ValueError(
-            "Failed to create a one-shot iterator for a dataset. "
-            "`Dataset.make_one_shot_iterator()` does not support datasets that "
-            "capture stateful objects, such as a `Variable` or `LookupTable`. "
-            "In these cases, use `Dataset.make_initializable_iterator()`. "
-            "(Original error: %s)" % err)
-      else:
-        six.reraise(ValueError, err)
-
-    return iterator_ops.Iterator(
-        gen_dataset_ops.one_shot_iterator(
-            dataset_factory=_make_dataset, **flat_structure(self)),
-        None, self.output_types, self.output_shapes, self.output_classes)
-
   @abc.abstractproperty
   def output_classes(self):
     """Returns the class of each component of an element of this dataset.
@@ -378,17 +329,19 @@ class DatasetV2(object):
 
     ```python
     import itertools
+    tf.enable_eager_execution()
 
     def gen():
       for i in itertools.count(1):
         yield (i, [1] * i)
 
-    ds = Dataset.from_generator(
+    ds = tf.data.Dataset.from_generator(
         gen, (tf.int64, tf.int64), (tf.TensorShape([]), tf.TensorShape([None])))
-    value = ds.make_one_shot_iterator().get_next()
 
-    sess.run(value)  # (1, array([1]))
-    sess.run(value)  # (2, array([1, 1]))
+    for value in ds.take(2):
+      print value
+    # (1, array([1]))
+    # (2, array([1, 1]))
     ```
 
     NOTE: The current implementation of `Dataset.from_generator()` uses
@@ -1345,6 +1298,60 @@ class DatasetV1(DatasetV2):
   def __init__(self):
     pass
 
+  @deprecation.deprecated(
+      None, "Use `for ... in dataset:` to iterate over a dataset. If using "
+      "`tf.estimator`, return the `Dataset` object directly from your input "
+      "function. As a last resort, you can use "
+      "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
+  def make_one_shot_iterator(self):
+    """Creates an `Iterator` for enumerating the elements of this dataset.
+
+    Note: The returned iterator will be initialized automatically.
+    A "one-shot" iterator does not currently support re-initialization.
+
+    Returns:
+      An `Iterator` over the elements of this dataset.
+    """
+    if context.executing_eagerly():
+      dataset = self._apply_options()
+      return iterator_ops.EagerIterator(dataset)
+
+    graph_level_seed, op_level_seed = core_random_seed.get_seed(None)
+
+    # NOTE(mrry): We capture by value here to ensure that `_make_dataset()` is
+    # a 0-argument function.
+    @function.Defun(capture_by_value=True)
+    def _make_dataset():
+      """Factory function for a dataset."""
+      # NOTE(mrry): `Defun` does not capture the graph-level seed from the
+      # enclosing graph, so if a graph-level seed is present we set the local
+      # graph seed based on a combination of the graph- and op-level seeds.
+      if graph_level_seed is not None:
+        assert op_level_seed is not None
+        core_random_seed.set_random_seed(
+            (graph_level_seed + 87654321 * op_level_seed) % (2 ** 63 - 1))
+
+      dataset = self._apply_options()
+      return dataset._as_variant_tensor()  # pylint: disable=protected-access
+
+    try:
+      _make_dataset.add_to_graph(ops.get_default_graph())
+    except ValueError as err:
+      if "Cannot capture a stateful node" in str(err):
+        raise ValueError(
+            "Failed to create a one-shot iterator for a dataset. "
+            "`Dataset.make_one_shot_iterator()` does not support datasets that "
+            "capture stateful objects, such as a `Variable` or `LookupTable`. "
+            "In these cases, use `Dataset.make_initializable_iterator()`. "
+            "(Original error: %s)" % err)
+      else:
+        six.reraise(ValueError, err)
+
+    return iterator_ops.Iterator(
+        gen_dataset_ops.one_shot_iterator(
+            dataset_factory=_make_dataset, **flat_structure(self)),
+        None, self.output_types, self.output_shapes, self.output_classes)
+
   @staticmethod
   @functools.wraps(DatasetV2.from_tensors)
   def from_tensors(tensors):
@@ -1562,8 +1569,26 @@ class DatasetV1Adapter(DatasetV1):
   def __iter__(self):
     return iter(self._dataset)
 
-  def make_one_shot_iterator(self):
-    return self._dataset.make_one_shot_iterator()
+
+@tf_export(v1=["data.make_one_shot_iterator"])
+def make_one_shot_iterator(dataset):
+  """Creates a `tf.data.Iterator` for enumerating the elements of a dataset.
+
+  Note: The returned iterator will be initialized automatically.
+  A "one-shot" iterator does not support re-initialization.
+
+  Args:
+    dataset: A `tf.data.Dataset`.
+
+  Returns:
+    A `tf.data.Iterator` over the elements of this dataset.
+  """
+  try:
+    # Call the defined `make_one_shot_iterator()` if there is one, because some
+    # datasets (e.g. for prefetching) override its behavior.
+    return dataset.make_one_shot_iterator()
+  except AttributeError:
+    return DatasetV1Adapter(dataset).make_one_shot_iterator()
 
 
 @tf_export("data.Options")
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index e1e1f9a208..8687d1d945 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -1171,7 +1171,7 @@ class PerReplicaDataset(object):
     # Eager mode prefetching would error out in constructor. Only remaining
     # case is non-prefetching in eager mode. We delegate to
     # PerReplicaDataIterator to handle that case.
-    dataset_iterator = self._dataset.make_one_shot_iterator()
+    dataset_iterator = dataset_ops.make_one_shot_iterator(self._dataset)
     return PerReplicaDataIterator(
         dataset_iterator, self._devices, prefetch_on_device=False)
 
@@ -1304,7 +1304,7 @@ class MultiWorkerDataset(object):
     iterators = []
     for worker, dataset in self._datasets:
       with ops.device(worker):
-        iterators.append((worker, dataset.make_one_shot_iterator()))
+        iterators.append((worker, dataset_ops.make_one_shot_iterator(dataset)))
     return MultiWorkerDataIterator(iterators, self._worker_device_pairs)
 
   def make_initializable_iterator(self):
@@ -1523,7 +1523,7 @@ class _SingleWorkerDatasetIterator(object):
         # TODO(priyag): Measure the performance of this approach vs calling
         # get_next on the original dataset N times.
         dataset = self._dataset.batch(len(self._devices), drop_remainder=True)
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
       else:
         iterator = multi_device_iterator_ops.MultiDeviceIterator(
             self._dataset, self._devices)
diff --git a/tensorflow/python/grappler/datasets_test.py b/tensorflow/python/grappler/datasets_test.py
index bd870ad8de..6937301ab2 100644
--- a/tensorflow/python/grappler/datasets_test.py
+++ b/tensorflow/python/grappler/datasets_test.py
@@ -48,7 +48,7 @@ class GrapplerTest(test.TestCase):
     for test_case in test_cases:
       with ops.Graph().as_default() as g:
         dataset = dataset_ops.Dataset.from_tensors(test_case['tensor'])
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -73,7 +73,7 @@ class GrapplerTest(test.TestCase):
     for test_case in test_cases:
       with ops.Graph().as_default() as g:
         dataset = dataset_ops.Dataset.from_tensor_slices(test_case['tensor'])
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -109,7 +109,7 @@ class GrapplerTest(test.TestCase):
             make_generator(test_case['tensor']),
             dtypes.int64,
             output_shapes=test_case['shape'])
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -122,7 +122,7 @@ class GrapplerTest(test.TestCase):
   def testRange(self):
     with ops.Graph().as_default() as g:
       dataset = dataset_ops.Dataset.range(42)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       get_next = iterator.get_next()
       train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
       train_op.append(get_next)
@@ -148,7 +148,7 @@ class GrapplerTest(test.TestCase):
       with ops.Graph().as_default() as g:
         dataset = dataset_ops.Dataset.from_tensors(test_case['tensor'])
         dataset = fn(dataset, test_case['tensor'], test_case['shape'])
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -252,7 +252,7 @@ class GrapplerTest(test.TestCase):
       with ops.Graph().as_default() as g:
         dataset = dataset_ops.Dataset.from_tensors(test_case['tensor'])
         dataset = dataset.batch(42)
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -281,7 +281,7 @@ class GrapplerTest(test.TestCase):
       with ops.Graph().as_default() as g:
         dataset = dataset_ops.Dataset.from_tensors(test_case['tensor'])
         dataset = dataset.padded_batch(42, padded_shapes=test_case['shape'][1:])
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -318,7 +318,7 @@ class GrapplerTest(test.TestCase):
           return dataset_fn
 
         dataset = dataset.flat_map(make_dataset(test_case['tensor']))
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -353,7 +353,7 @@ class GrapplerTest(test.TestCase):
 
         dataset = dataset.interleave(
             make_dataset(test_case['tensor']), cycle_length=42)
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
@@ -382,7 +382,7 @@ class GrapplerTest(test.TestCase):
       with ops.Graph().as_default() as g:
         dataset = dataset_ops.Dataset.from_tensors(test_case['tensor'])
         dataset = dataset.map(array_ops.transpose)
-        iterator = dataset.make_one_shot_iterator()
+        iterator = dataset_ops.make_one_shot_iterator(dataset)
         get_next = iterator.get_next()
         train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
         train_op.append(get_next)
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index fbf893c663..b6d2510897 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -124,7 +124,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     model.fit(iterator, epochs=1, steps_per_epoch=steps_per_epoch)
     self.assertTrue(model.built)
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 697286b644..38a3928a38 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1151,7 +1151,7 @@ class Model(Network):
 
     if isinstance(x, dataset_ops.DatasetV2):
       if context.executing_eagerly():
-        x = x.make_one_shot_iterator()
+        x = iter(x)
       else:
         if x in self._dataset_iterator_cache:
           x = self._dataset_iterator_cache[x]
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index e48df241e9..1b2d0e88e3 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -58,7 +58,7 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
     model.evaluate(iterator, steps=2, verbose=1)
@@ -117,7 +117,7 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
     # Finalize graph to make sure we are not appending another iterator
@@ -138,7 +138,7 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
     dataset = dataset.repeat(2)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     with test.mock.patch.object(logging, 'warning') as mock_log:
       model.fit(iterator, epochs=1, steps_per_epoch=3, verbose=0)
@@ -325,7 +325,7 @@ class TestMetricsWithDatasetIterators(test.TestCase):
     y = np.random.randint(2, size=(100, 1)).astype(np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     outs = model.evaluate(iterator, steps=10)
     self.assertEqual(np.around(outs[1], decimals=1), 0.5)
     self.assertEqual(np.around(outs[2], decimals=1), 0.5)
@@ -334,7 +334,7 @@ class TestMetricsWithDatasetIterators(test.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     outs = model.evaluate(iterator, steps=10)
     self.assertEqual(outs[1], 0.)
     self.assertEqual(outs[2], 0.)
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index d769143106..c1a9154ff5 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -137,10 +137,10 @@ class TrainingTest(test.TestCase):
     x = keras.backend.zeros(shape=(10, 3))
     y = keras.backend.zeros(shape=(10, 4))
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     validation_dataset = dataset_ops.Dataset.from_tensor_slices(
         (x, y)).repeat(10).batch(5)
-    validation_iterator = validation_dataset.make_one_shot_iterator()
+    validation_iterator = dataset_ops.make_one_shot_iterator(validation_dataset)
 
     with self.assertRaisesRegexp(
         ValueError, r'specify .* `steps_per_epoch`'):
@@ -230,7 +230,7 @@ class CorrectnessTest(test.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
     dataset = dataset.repeat(100)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     history = model.fit(iterator, epochs=1, steps_per_epoch=10)
     self.assertAlmostEqual(history.history['loss'][-1], 0.6173, 4)
 
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index e7310a7bb9..7b2bfcfd86 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -69,7 +69,7 @@ def fit_generator(model,
 
   if (isinstance(validation_data, dataset_ops.DatasetV2) and
       context.executing_eagerly()):
-    validation_data = validation_data.make_one_shot_iterator()
+    validation_data = iter(validation_data)
   val_gen = (data_utils.is_generator_or_sequence(validation_data) or
              isinstance(validation_data, iterator_ops.EagerIterator))
   if (val_gen and not isinstance(validation_data, data_utils.Sequence) and
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 1b04be914f..8e3b61be0c 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -563,7 +563,7 @@ class TrainingTest(test.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
     dataset = dataset.repeat(10)
     dataset = dataset.batch(10)
-    iterator = dataset.make_one_shot_iterator()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
     model.fit(iterator, epochs=1, steps_per_epoch=2)
 
     if context.executing_eagerly():
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 4b9d562472..b7c82bfde3 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -177,7 +177,7 @@ def convert_to_iterator(x=None,
   if shuffle:
     dataset = dataset.shuffle(buffer_size=10000)
   dataset = dataset.repeat(epochs)
-  iterator = dataset.make_one_shot_iterator()
+  iterator = dataset_ops.make_one_shot_iterator(dataset)
 
   return iterator, steps_per_epoch
 
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index 92eb1a6185..620275e50f 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -461,7 +461,7 @@ class ModelSubclassingTest(test.TestCase):
       dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
       dataset = dataset.repeat(100)
       dataset = dataset.batch(10)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
 
       model.fit(iterator, epochs=2, steps_per_epoch=10, verbose=0)
       _ = model.evaluate(iterator, steps=10, verbose=0)
diff --git a/tensorflow/python/keras/utils/multi_gpu_utils_test.py b/tensorflow/python/keras/utils/multi_gpu_utils_test.py
index 1780ab6587..8c1abd6324 100644
--- a/tensorflow/python/keras/utils/multi_gpu_utils_test.py
+++ b/tensorflow/python/keras/utils/multi_gpu_utils_test.py
@@ -158,7 +158,7 @@ class TestMultiGPUModel(test.TestCase):
       dataset = data.Dataset.from_tensor_slices((x_train, y_train))
       dataset = dataset.repeat()
       dataset = dataset.batch(4)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = data.make_one_shot_iterator(dataset)
 
       inputs, targets = iterator.get_next()
 
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 8e54a14f47..08942c5bb6 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -1372,7 +1372,7 @@ class FinalOpsHookTest(test.TestCase):
   def test_final_ops_triggers_out_of_range_error(self):
     with ops.Graph().as_default():
       dataset = dataset_ops.Dataset.range(1)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       read_ops = iterator.get_next()
       final_ops = read_ops
 
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 9b822860ea..5d621ba4ff 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -2364,7 +2364,7 @@ class MetaGraphTest(test.TestCase):
   def testPreserveDatasetAndFunctions(self):
     with ops_lib.Graph().as_default() as g:
       dataset = dataset_ops.Dataset.range(10).map(lambda x: x * x)
-      iterator = dataset.make_one_shot_iterator()
+      iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
       _ = array_ops.identity(next_element, name="output")
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
index 509bbae833..bb14216e60 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
@@ -28,4 +28,8 @@ tf_module {
     name: "experimental"
     mtype: "<type \'module\'>"
   }
+  member_method {
+    name: "make_one_shot_iterator"
+    argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
index 39a6e1ee71..760d56aa06 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
@@ -65,10 +65,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
index ef367238d0..c46e276664 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -68,10 +68,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
index a8fc6fbec1..5a19f474fa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -67,10 +67,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
index 697f371344..71be8ed677 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
@@ -68,10 +68,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
index 17ac098910..823c5c4d18 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -68,10 +68,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
index f005d36e1a..28050c2758 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -68,10 +68,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
index b0c0b73ad6..d707ac7320 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -68,10 +68,6 @@ tf_class {
     name: "make_initializable_iterator"
     argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "make_one_shot_iterator"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index fcbb0fd97d..696cf7c41f 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -108,6 +108,7 @@ renames = {
     'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
+    'tf.data.make_one_shot_iterator': 'tf.compat.v1.data.make_one_shot_iterator',
     'tf.debugging.is_finite': 'tf.math.is_finite',
     'tf.debugging.is_inf': 'tf.math.is_inf',
     'tf.debugging.is_nan': 'tf.math.is_nan',
-- 
GitLab


From 49a770d1d8c65d791aa2fbadeec5267acc99d9a4 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Mon, 3 Dec 2018 17:59:44 -0800
Subject: [PATCH 1254/1554] Correctly reduce loss metric in Dist strategy +
 Keras + eager case.

PiperOrigin-RevId: 223897668
---
 tensorflow/contrib/distribute/python/keras_test.py | 14 +++++---------
 .../python/keras/engine/training_distributed.py    |  3 ++-
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 00a220880b..2b2c840daa 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -27,7 +27,6 @@ from tensorflow.contrib.distribute.python import tpu_strategy
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import values
-from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.estimator import keras as keras_lib
 from tensorflow.python.estimator import run_config as run_config_lib
@@ -1210,8 +1209,7 @@ class TestDistributionStrategyWithNormalizationLayer(
 class TestDistributionStrategyCorrectness(test.TestCase,
                                           parameterized.TestCase):
 
-  # TODO(b/120186218): Enable this for eager once metrics are working.
-  @combinations.generate(strategy_for_numpy_input_combinations())
+  @combinations.generate(all_strategy_combinations())
   def test_metric_correctness(self, distribution):
     with self.cached_session():
       keras.backend.set_image_data_format('channels_last')
@@ -1309,12 +1307,10 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       self.assertAllClose(
           wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance)
 
-      # TODO(b/120186218): Enable this once metrics are working with eager.
-      if not context.executing_eagerly():
-        self.assertAllClose(
-            eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance)
-        self.assertAllClose(
-            predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance)
+      self.assertAllClose(
+          eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance)
+      self.assertAllClose(
+          predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 7cf961b9ec..b0a2dfe00d 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -657,7 +657,8 @@ def _get_eager_execution_function(model, mode):
     (all_inputs, all_outputs, _, _) = distributed_training_utils.unwrap_values(
         strategy,
         grouped_inputs,
-        grouped_outputs)
+        grouped_outputs,
+        with_loss_tensor=(mode != 'predict'))
 
     return K.function(
         all_inputs,
-- 
GitLab


From 0781147372a4b0e85203088380bb803735c7b6f1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 18:03:37 -0800
Subject: [PATCH 1255/1554] Fix a typo -- invalid function name.

PiperOrigin-RevId: 223898343
---
 tensorflow/python/ops/random_ops.py                    | 4 ++--
 tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py
index f2df87cf2d..62e2f6d102 100644
--- a/tensorflow/python/ops/random_ops.py
+++ b/tensorflow/python/ops/random_ops.py
@@ -357,9 +357,9 @@ def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None):
     return multinomial_categorical_impl(logits, num_samples, output_dtype, seed)
 
 
-@tf_export("random.categorical", v1=[])
+@tf_export("random.categorical")
 def categorical(logits, num_samples, dtype=None, seed=None, name=None):
-  """Draws samples from a multinomial distribution.
+  """Draws samples from a categorical distribution.
 
   Example:
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
index d788f6dfca..107534e086 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.random"
 tf_module {
+  member_method {
+    name: "categorical"
+    argspec: "args=[\'logits\', \'num_samples\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+  }
   member_method {
     name: "gamma"
     argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
-- 
GitLab


From 4b6547557148ac0e8ca8eb6ac401a96c1e5b8a44 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 3 Dec 2018 18:06:57 -0800
Subject: [PATCH 1256/1554] [XLA] Add WithOneUse(r) matchers to
 pattern_matcher.h

These are useful for guarding certain complex algebraic simplifications that
would make the graph more complex if an instruction has more than one use(r).

PiperOrigin-RevId: 223898824
---
 .../compiler/xla/service/pattern_matcher.h    | 77 +++++++++++++++++++
 .../xla/service/pattern_matcher_test.cc       | 55 +++++++++++++
 2 files changed, 132 insertions(+)

diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index cc06cf6ec7..90ea5dc51e 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -64,6 +64,9 @@ namespace xla {
 //       e.g. IsConstantScalar() or IsConstantScalar(42).
 //     - WithFusionKind
 //     - WithTupleIndex: get-tuple-element operations with the given tuple index
+//     - WithOneUse: Instruction is used as an operand exactly once.
+//     - WithOneUser: Instruction is used by exactly one other instruction, but
+//       is possibly used more than once as an operand (e.g. multiply(x,x)).
 //
 //   Shape():
 //     - EqualTo
@@ -1603,6 +1606,64 @@ class HloInstructionPatternParameterNumImpl {
   int64 parameter_num_;
 };
 
+// Superclass that contains common code used by Op::WithOneUse() and
+// Op::WithOneUser().
+class HloInstructionPatternOneUseOrUserImpl {
+ protected:
+  bool MatchOneUser(const HloInstruction* inst, MatchOption option) const {
+    if (inst->user_count() != 1) {
+      EXPLAIN << "HloInstruction has " << inst->user_count()
+              << " users, but expected exactly one.";
+      if (inst->user_count() > 1) {
+        EXPLAIN << "\nAll users:";
+        for (const HloInstruction* user : inst->users()) {
+          EXPLAIN << "\n - " << user->ToShortString();
+        }
+      }
+      return false;
+    }
+    return true;
+  }
+};
+
+class HloInstructionPatternOneUseImpl
+    : public HloInstructionPatternOneUseOrUserImpl {
+ public:
+  bool Match(const HloInstruction* inst, MatchOption option) const {
+    if (!MatchOneUser(inst, option)) {
+      return false;
+    }
+
+    int64 use_count = absl::c_count_if(
+        inst->users()[0]->operands(),
+        [&](const HloInstruction* operand) { return operand == inst; });
+    if (use_count != 1) {
+      EXPLAIN << "HloInstruction is used " << use_count
+              << " times by its user, but is expected to be used just once: "
+              << inst->users()[0]->ToShortString();
+      return false;
+    }
+    return true;
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "which has exactly one use";
+  }
+};
+
+class HloInstructionPatternOneUserImpl
+    : public HloInstructionPatternOneUseOrUserImpl {
+ public:
+  bool Match(const HloInstruction* inst, MatchOption option) const {
+    return MatchOneUser(inst, option);
+  }
+
+  void DescribeTo(std::ostream* os, int64 indent = 0) const {
+    *os << "which has exactly one user (but possibly is used multiple times by "
+           "that instruction)";
+  }
+};
+
 // Matches a constant scalar or effective scalar, optionally with a given value.
 template <typename ScalarTy>
 class HloConstantScalarImpl {
@@ -1877,6 +1938,22 @@ class HloInstructionPattern {
     return AppendImpl(HloInstructionPatternParameterNumImpl(parameter_num));
   }
 
+  // Modifies the pattern to match if the instruction is used exactly once.
+  // Does not match if the instruction is used twice by the same user (e.g.
+  // multiply(x,x)).
+  constexpr auto WithOneUse() const
+      -> decltype(this->AppendImpl(HloInstructionPatternOneUseImpl())) {
+    return AppendImpl(HloInstructionPatternOneUseImpl());
+  }
+
+  // Modifies the pattern to match if the instruction is used by exactly one
+  // other instruction.  Will match if the instruction is used twice, so long as
+  // it's by the same user (e.g.  multiply(x,x)).
+  constexpr auto WithOneUser() const
+      -> decltype(this->AppendImpl(HloInstructionPatternOneUserImpl())) {
+    return AppendImpl(HloInstructionPatternOneUserImpl());
+  }
+
   void DescribeTo(std::ostream* os, int64 indent = 0) const {
     impl_.DescribeTo(os, indent);
   }
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_test.cc
index 13886fa6f5..3dd18898c8 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher_test.cc
+++ b/tensorflow/compiler/xla/service/pattern_matcher_test.cc
@@ -875,5 +875,60 @@ TEST(PatternMatcherTest, Parameter) {
             "in p0 = f32[] parameter(0)");
 }
 
+TEST(PatternMatcherTest, OneUseAndOneUser) {
+  auto param =
+      HloInstruction::CreateParameter(0, ShapeUtil::MakeShape(F32, {}), "p0");
+
+  EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse()));
+  EXPECT_DESC_AND_EXPLANATION(
+      param, m::Op().WithOneUse(),
+      "an HloInstruction which has exactly one use",
+      "HloInstruction has 0 users, but expected exactly one.\n"
+      "in p0 = f32[] parameter(0)");
+
+  EXPECT_FALSE(Match(param.get(), m::Op().WithOneUser()));
+  EXPECT_DESC_AND_EXPLANATION(
+      param, m::Op().WithOneUser(),
+      "an HloInstruction which has exactly one user (but possibly is used "
+      "multiple times by that instruction)",
+      "HloInstruction has 0 users, but expected exactly one.\n"
+      "in p0 = f32[] parameter(0)");
+
+  {
+    auto reshape =
+        SetName("r", HloInstruction::CreateReshape(
+                         ShapeUtil::MakeShape(F32, {1}), param.get()));
+    EXPECT_TRUE(Match(param.get(), m::Op().WithOneUse()));
+    EXPECT_TRUE(Match(param.get(), m::Op().WithOneUser()));
+
+    auto reshape1 =
+        SetName("r1", HloInstruction::CreateReshape(
+                          ShapeUtil::MakeShape(F32, {1}), param.get()));
+    EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse()));
+    EXPECT_FALSE(Match(param.get(), m::Op().WithOneUser()));
+
+    const char* kMultipleUserExplanation =
+        "HloInstruction has 2 users, but expected exactly one.\n"
+        "All users:\n"
+        " - %r = reshape(%p0)\n"
+        " - %r1 = reshape(%p0)\n"
+        "in p0 = f32[] parameter(0)";
+    EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUse()),
+              kMultipleUserExplanation);
+    EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUser()),
+              kMultipleUserExplanation);
+  }
+
+  auto add = SetName("add", HloInstruction::CreateBinary(
+                                ShapeUtil::MakeShape(F32, {}), HloOpcode::kAdd,
+                                param.get(), param.get()));
+  EXPECT_TRUE(Match(param.get(), m::Op().WithOneUser()));
+  EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse()));
+  EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUse()),
+            "HloInstruction is used 2 times by its user, but is expected to be "
+            "used just once: %add = add(%p0, %p0)\n"
+            "in p0 = f32[] parameter(0)");
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From b9a02e33c5917c9aa8040592cc469613ab1a657d Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 3 Dec 2018 18:09:45 -0800
Subject: [PATCH 1257/1554] [XLA] Simplify A*C + B*C => (A+B)*C when safe.

"Safe" means:

 - A, B, and C are integers, OR
 - A, B, and C are floating-point types, and C is a power of 2 (including
   2^(-k)).

PiperOrigin-RevId: 223899195
---
 .../xla/service/algebraic_simplifier.cc       |  74 +++++++++++
 ..._simplifier_proof_distributive_property.py |  82 ++++++++++++
 .../xla/service/algebraic_simplifier_test.cc  | 122 ++++++++++++++++++
 3 files changed, 278 insertions(+)
 create mode 100644 tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index a348bcf0a2..985c5af1c4 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/algebraic_simplifier.h"
 
 #include <algorithm>
+#include <cmath>
 #include <iterator>
 #include <memory>
 #include <numeric>
@@ -68,6 +69,45 @@ bool IsAll(const HloInstruction* op, int8 value) {
   }
 }
 
+// Checks whether `op` is a floating-point constant or broadcast of a constant
+// of the form +/- 2^k for some integer k positive, negative, or zero.  Such
+// values are interesting because multiplying by a power of 2 just moves the
+// exponent.
+bool IsAllFpConstantPowerOf2(const HloInstruction* op) {
+  // Unwrap the broadcast if necessary.
+  const HloInstruction* c;
+  if (!Match(op, m::ConstantEffectiveScalar(&c)) &&
+      !Match(op, m::Broadcast(m::Constant(&c).WithShape(
+                     m::Shape().IsEffectiveScalar())))) {
+    return false;
+  }
+  auto val = [&]() -> absl::optional<double> {
+    switch (c->shape().element_type()) {
+      case BF16:
+        return static_cast<double>(c->literal().GetFirstElement<bfloat16>());
+      case F16:
+        return static_cast<double>(c->literal().GetFirstElement<Eigen::half>());
+      case F32:
+        return c->literal().GetFirstElement<float>();
+      case F64:
+        return c->literal().GetFirstElement<double>();
+      default:
+        // Cowardly refuse to consider complex types.
+        return absl::nullopt;
+    }
+  }();
+  if (!val) {
+    return false;
+  }
+
+  int exp;
+  double mantissa = std::frexp(*val, &exp);
+  // frexp returns a value in the range (-1; -0.5] U [0.5, 1).  A return value
+  // of +/-0.5 therefore indicates that the floating point value is a power of
+  // 2.
+  return mantissa == 0.5 || mantissa == -0.5;
+}
+
 // Returns whether the given transpose produces a result which is bit-wise
 // identical to its operand and thus may be replaced with a bitcast.
 bool TransposeIsBitcast(const HloInstruction* transpose) {
@@ -415,6 +455,40 @@ Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add) {
                                           sum_of_constants));
   }
 
+  // A*C + B*C => (A+B)*C
+  //
+  //  - If A, B, and C are integers, do this unconditionally. Proof of
+  //    correctness: https://rise4fun.com/Alive/u9X.
+  //
+  //  - If A, B, and C are floating point, do this if C is a scalar constant or
+  //    broadcast of scalar constant and is equal to +/- 2^k for some (possibly
+  //    negative) integer k.
+  //
+  //    Multiplying by a power of 2 just moves the exponent, so our answer is
+  //    exact modulo rounding of intermediate results so long as
+  //
+  //     - none of the three products has an exponent which underflows (so the
+  //       result is 0 or denormal), and
+  //     - none of the three products overflows to inf.
+  //
+  //    Proof: See algebraic_simplifier_proof_distributive_property.py.
+  //
+  //    We deem these differences in rounding, underflow, and overflow
+  //    acceptable in the ML context.
+  HloInstruction *b, *c;
+  if (((Match(lhs, m::Multiply(m::Op(&a), m::Op(&c))) &&
+        Match(rhs, m::MultiplyAnyOrder(m::Op().Is(c), m::Op(&b)))) ||
+       (Match(lhs, m::Multiply(m::Op(&c), m::Op(&a))) &&
+        Match(rhs, m::MultiplyAnyOrder(m::Op().Is(c), m::Op(&b))))) &&
+      (ShapeUtil::ElementIsIntegral(add->shape()) ||
+       IsAllFpConstantPowerOf2(c))) {
+    return ReplaceWithNewInstruction(
+        add, HloInstruction::CreateBinary(
+                 add->shape(), HloOpcode::kMultiply,
+                 computation_->AddInstruction(HloInstruction::CreateBinary(
+                     add->shape(), HloOpcode::kAdd, a, b)),
+                 c));
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py b/tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py
new file mode 100644
index 0000000000..5da13da041
--- /dev/null
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_proof_distributive_property.py
@@ -0,0 +1,82 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Proof that transforming (A*C)+(B*C) <=> (A+B)*C is "safe" if C=2^k.
+
+Specifically, for all floating-point values A, B, and C, if
+
+ - C is equal to +/- 2^k for some (possibly negative) integer k, and
+ - A, B, C, A*C, B*C, and A+B are not subnormal, zero, or inf,
+
+then there exists a rounding mode rm in [RTZ, RNE] such that
+
+ (A*C) + (B*C) == (A+B) * C  (computed with rounding mode rm).
+
+Informally, this means that the equivalence holds for powers of 2 C, modulo
+flushing to zero or inf, and modulo rounding of intermediate results.
+
+Requires z3 python bindings; try `pip install z3-solver`.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import z3
+
+# We do float16 because it lets the solver run much faster.  These results
+# should generalize to fp32 and fp64, and you can verify this by changing the
+# value of FLOAT_TY (and then waiting a while).
+FLOAT_TY = z3.Float16
+
+a = z3.FP("a", FLOAT_TY())
+b = z3.FP("b", FLOAT_TY())
+c = z3.FP("c", FLOAT_TY())
+
+s = z3.Solver()
+
+# C must be a power of 2, i.e. significand bits must all be 0.
+s.add(z3.Extract(FLOAT_TY().sbits() - 1, 0, z3.fpToIEEEBV(c)) == 0)
+
+for rm in [z3.RTZ(), z3.RNE()]:
+  z3.set_default_rounding_mode(rm)
+  before = a * c + b * c
+  after = (a + b) * c
+
+  # Check that before == after, allowing that 0 == -0.
+  s.add(
+      z3.Not(
+          z3.Or(
+              before == after,  #
+              z3.And(z3.fpIsZero(before), z3.fpIsZero(after)))))
+
+  for x in [
+      (a * c),
+      (b * c),
+      (a + b),
+  ]:
+    s.add(z3.Not(z3.fpIsSubnormal(x)))
+    s.add(z3.Not(z3.fpIsZero(x)))
+    s.add(z3.Not(z3.fpIsInf(x)))
+
+if s.check() == z3.sat:
+  m = s.model()
+  print("Counterexample found!")
+  print(m)
+  print("a*c:       ", z3.simplify(m[a] * m[c]))
+  print("b*c:       ", z3.simplify(m[b] * m[c]))
+  print("a+b:       ", z3.simplify(m[a] + m[b]))
+  print("a*c + b*c: ", z3.simplify(m[a] * m[c] + m[b] * m[c]))
+  print("(a+b) * c: ", z3.simplify((m[a] + m[b]) * m[c]))
+else:
+  print("Proved!")
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 628d805f1c..14ce519b6a 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -80,6 +80,128 @@ TEST_F(AlgebraicSimplifierTest, AddZero) {
   EXPECT_EQ(root, param0);
 }
 
+TEST_F(AlgebraicSimplifierTest, FactorIntegerAddition) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p0 = s32[8] parameter(0)
+      p1 = s32[8] parameter(1)
+      p2 = s32[8] parameter(2)
+      x = s32[8] multiply(p0, p2)
+      y = s32[8] multiply(p1, p2)
+      ROOT sum = s32[8] add(x, y)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  AlgebraicSimplifier simplifier(default_options_);
+  ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  EXPECT_THAT(
+      m->entry_computation()->root_instruction(),
+      GmockMatch(m::MultiplyAnyOrder(
+          m::AddAnyOrder(m::Parameter(0), m::Parameter(1)), m::Parameter(2))));
+}
+
+// A*C + B*C => (A+B)*C if C is a floating-point power of 2.
+TEST_F(AlgebraicSimplifierTest, FactorFpAddition) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p0 = f32[] parameter(0)
+      p1 = f32[] parameter(1)
+      c = f32[] constant(0.125)
+      x = f32[] multiply(p0, c)
+      y = f32[] multiply(p1, c)
+      ROOT sum = f32[] add(x, y)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::MultiplyAnyOrder(
+                  m::AddAnyOrder(m::Parameter(0), m::Parameter(1)),
+                  m::ConstantScalar(0.125))));
+}
+
+// A*C + B*C => (A+B)*C if C is a broadcast of a floating-point power of 2.
+TEST_F(AlgebraicSimplifierTest, FactorFpAdditionWithBroadcast) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p0 = f32[4] parameter(0)
+      p1 = f32[4] parameter(1)
+      c = f32[] constant(0.125)
+      b = f32[4] broadcast(c), dimensions={}
+      x = f32[4] multiply(p0, b)
+      y = f32[4] multiply(p1, b)
+      ROOT sum = f32[4] add(x, y)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::MultiplyAnyOrder(
+                  m::AddAnyOrder(m::Parameter(0), m::Parameter(1)),
+                  m::Broadcast(m::ConstantScalar(0.125)))));
+}
+
+// A*C + B*C => (A+B)*C simplification should not happen if C is not a
+// floating-point power of 2.
+TEST_F(AlgebraicSimplifierTest, FactorFpAdditionNotPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p0 = f32[] parameter(0)
+      p1 = f32[] parameter(1)
+      c = f32[] constant(0.3)
+      x = f32[] multiply(p0, c)
+      y = f32[] multiply(p1, c)
+      ROOT sum = f32[] add(x, y)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  EXPECT_FALSE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+}
+
+// A*C + B*C => (A+B)*C simplification should not happen if A, B, and C are
+// complex numbers.
+TEST_F(AlgebraicSimplifierTest, FactorFpAdditionComplex) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p0 = c64[8] parameter(0)
+      p1 = c64[8] parameter(1)
+      p2 = c64[8] parameter(2)
+      x = c64[8] multiply(p0, p2)
+      y = c64[8] multiply(p1, p2)
+      ROOT sum = c64[8] add(x, y)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  EXPECT_FALSE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+}
+
+// A*C + B*C => (A+B)*C simplification is OK if A, B, and C are complex.
+TEST_F(AlgebraicSimplifierTest, FactorFpAdditionBfloat16) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p0 = bf16[4] parameter(0)
+      p1 = bf16[4] parameter(1)
+      c = bf16[] constant(0.125)
+      b = bf16[4] broadcast(c), dimensions={}
+      x = bf16[4] multiply(p0, b)
+      y = bf16[4] multiply(p1, b)
+      ROOT sum = bf16[4] add(x, y)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::MultiplyAnyOrder(
+                  m::AddAnyOrder(m::Parameter(0), m::Parameter(1)),
+                  m::Broadcast(m::ConstantScalar(0.125)))));
+}
+
 // Test that A * 0 is simplified to 0
 TEST_F(AlgebraicSimplifierTest, MulZero) {
   auto m = CreateNewVerifiedModule();
-- 
GitLab


From 48a49df614f0392c2c548e6d01d14d01f7c47714 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 3 Dec 2018 18:40:22 -0800
Subject: [PATCH 1258/1554] [XLA] Use a uniform HloInstruction stringification
 function within pattern_matcher.h.

Previously we used HloInstruction::ToString with custom options in some places,
and HloInstruction::ToShortString() in other places.

PiperOrigin-RevId: 223902223
---
 .../compiler/xla/service/pattern_matcher.h    | 25 ++++++++++---------
 .../xla/service/pattern_matcher_test.cc       | 13 +++++-----
 2 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index 90ea5dc51e..432aa1ea0b 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -1136,6 +1136,13 @@ inline const HloInstruction* HloOperand(const HloInstruction* instr,
   return instr->operand(idx);
 }
 
+// Pretty-printer for HloInstruction.  Sort of like ToShortString, but with
+// fewer %s and more shapes.
+inline string InstToString(const HloInstruction* inst) {
+  return inst->ToString(
+      HloPrintOptions().set_print_metadata(false).set_print_percent(false));
+}
+
 template <typename HloInstructionType, typename Impl>
 class HloInstructionPattern;
 
@@ -1190,14 +1197,14 @@ class HloInstructionIsImpl {
   bool Match(const ::xla::HloInstruction* inst, MatchOption option) const {
     if (inst != inst_) {
       EXPLAIN << "HloInstruction " << inst << " is not " << inst_ << " ("
-              << inst_->ToShortString() << ")";
+              << InstToString(inst_) << ")";
       return false;
     }
     return true;
   }
 
   void DescribeTo(std::ostream* os, int64 indent = 0) const {
-    *os << "which is " << inst_ << " (" << inst_->ToShortString() << ")";
+    *os << "which is " << inst_ << " (" << InstToString(inst_) << ")";
   }
 
  private:
@@ -1617,7 +1624,7 @@ class HloInstructionPatternOneUseOrUserImpl {
       if (inst->user_count() > 1) {
         EXPLAIN << "\nAll users:";
         for (const HloInstruction* user : inst->users()) {
-          EXPLAIN << "\n - " << user->ToShortString();
+          EXPLAIN << "\n - " << InstToString(user);
         }
       }
       return false;
@@ -1640,7 +1647,7 @@ class HloInstructionPatternOneUseImpl
     if (use_count != 1) {
       EXPLAIN << "HloInstruction is used " << use_count
               << " times by its user, but is expected to be used just once: "
-              << inst->users()[0]->ToShortString();
+              << InstToString(inst->users()[0]);
       return false;
     }
     return true;
@@ -1767,10 +1774,7 @@ class HloInstructionPattern {
       return true;
     }
     if (inst != nullptr) {
-      EXPLAIN << "\nin "
-              << inst->ToString(HloPrintOptions()
-                                    .set_print_metadata(false)
-                                    .set_print_percent(false));
+      EXPLAIN << "\nin " << InstToString(inst);
     }
     return false;
   }
@@ -1783,10 +1787,7 @@ class HloInstructionPattern {
       }
       return true;
     }
-    EXPLAIN << "\nin "
-            << inst->ToString(HloPrintOptions()
-                                  .set_print_metadata(false)
-                                  .set_print_percent(false));
+    EXPLAIN << "\nin " << InstToString(inst);
     return false;
   }
 
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_test.cc
index 3dd18898c8..186ef0c791 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher_test.cc
+++ b/tensorflow/compiler/xla/service/pattern_matcher_test.cc
@@ -767,10 +767,11 @@ TEST(PatternMatcherTest, HloInstructionDescribeToAndExplain) {
       "in c = f64[] constant(2.25)");
   EXPECT_DESC_AND_EXPLANATION(
       constant, m::Op().Is(iota.get()),
-      absl::StrCat("an HloInstruction which is 0x", absl::Hex(iota.get()), " (",
-                   iota->ToShortString(), ")"),
+      absl::StrCat("an HloInstruction which is 0x", absl::Hex(iota.get()),
+                   " (i = s32[42]{0} iota(), iota_dimension=0)"),
       absl::StrCat("HloInstruction 0x", absl::Hex(constant.get()), " is not 0x",
-                   absl::Hex(iota.get()), " (", iota->ToShortString(), ")\n",
+                   absl::Hex(iota.get()),
+                   " (i = s32[42]{0} iota(), iota_dimension=0)\n"
                    "in c = s32[] constant(0)"));
 }
 
@@ -910,8 +911,8 @@ TEST(PatternMatcherTest, OneUseAndOneUser) {
     const char* kMultipleUserExplanation =
         "HloInstruction has 2 users, but expected exactly one.\n"
         "All users:\n"
-        " - %r = reshape(%p0)\n"
-        " - %r1 = reshape(%p0)\n"
+        " - r = f32[1]{0} reshape(f32[] p0)\n"
+        " - r1 = f32[1]{0} reshape(f32[] p0)\n"
         "in p0 = f32[] parameter(0)";
     EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUse()),
               kMultipleUserExplanation);
@@ -926,7 +927,7 @@ TEST(PatternMatcherTest, OneUseAndOneUser) {
   EXPECT_FALSE(Match(param.get(), m::Op().WithOneUse()));
   EXPECT_EQ(Explanation(param.get(), m::Op().WithOneUse()),
             "HloInstruction is used 2 times by its user, but is expected to be "
-            "used just once: %add = add(%p0, %p0)\n"
+            "used just once: add = f32[] add(f32[] p0, f32[] p0)\n"
             "in p0 = f32[] parameter(0)");
 }
 
-- 
GitLab


From 85c4d28aed41682a2d0a7290d934184d5dbad283 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 3 Dec 2018 20:26:35 -0800
Subject: [PATCH 1259/1554] [tf.data] Remove
 `Dataset.make_initializable_iterator()` from the V2 API.

Add `tf.compat.v1.data.make_initializable_iterator(dataset)` to enable the use of V2
`Dataset` objects in a legacy V1 pipeline.

PiperOrigin-RevId: 223911205
---
 tensorflow/contrib/bigtable/README.md         |  23 +--
 .../python/kernel_tests/bigtable_ops_test.py  |  18 +--
 .../kernel_tests/assert_element_shape_test.py |  18 +--
 .../kernel_tests/lmdb_dataset_op_test.py      |   3 +-
 .../kernel_tests/slide_dataset_op_test.py     |  34 ++---
 .../linear_regression_graph_test.py           |   2 +-
 .../hadoop/python/kernel_tests/hadoop_test.py |   3 +-
 .../contrib/tpu/python/tpu/datasets_test.py   |  10 +-
 .../contrib/tpu/python/tpu/keras_support.py   |   2 +-
 .../contrib/tpu/python/tpu/tpu_estimator.py   |   2 +-
 .../autograph/operators/control_flow.py       |   2 +-
 tensorflow/python/data/__init__.py            |   1 +
 .../python/data/benchmarks/batch_benchmark.py |   2 +-
 .../from_tensor_slices_benchmark.py           |   8 +-
 .../python/data/benchmarks/range_benchmark.py |   2 +-
 .../benchmarks/map_and_batch_benchmark.py     |   2 +-
 .../benchmarks/unbatch_benchmark.py           |   4 +-
 .../kernel_tests/copy_to_device_test.py       |  26 ++--
 .../dense_to_sparse_batch_test.py             |  20 ++-
 .../directed_interleave_dataset_test.py       |   2 +-
 .../kernel_tests/enumerate_dataset_test.py    |   5 +-
 .../kernel_tests/group_by_window_test.py      |  35 ++---
 .../kernel_tests/ignore_errors_test.py        |   6 +-
 .../make_batched_features_dataset_test.py     |   2 +-
 .../make_tf_record_dataset_test.py            |   2 +-
 .../kernel_tests/map_and_batch_test.py        |  10 +-
 .../optimization/optimize_dataset_test.py     |   4 +-
 .../kernel_tests/override_threadpool_test.py  |   2 +-
 .../kernel_tests/parallel_interleave_test.py  |  13 +-
 .../kernel_tests/prefetch_to_device_test.py   |   6 +-
 .../experimental/kernel_tests/scan_test.py    |   9 +-
 .../dataset_serialization_test_base.py        |   3 +-
 .../range_dataset_serialization_test.py       |   4 +-
 .../experimental/kernel_tests/sleep_test.py   |   2 +-
 .../kernel_tests/sql_dataset_test_base.py     |   3 +-
 .../kernel_tests/stats_dataset_ops_test.py    |  28 ++--
 .../kernel_tests/stats_dataset_test_base.py   |   3 +-
 .../experimental/kernel_tests/unbatch_test.py |   4 +-
 .../experimental/kernel_tests/unique_test.py  |   2 +-
 .../kernel_tests/dataset_checkpoint_test.py   |  22 +--
 .../data/kernel_tests/from_generator_test.py  |  77 +++++-----
 .../from_sparse_tensor_slices_test.py         |   4 +-
 .../data/kernel_tests/from_tensors_test.py    |   2 +-
 .../kernel_tests/iterator_checkpoint_test.py  |   2 +-
 .../kernel_tests/iterator_cluster_test.py     |   2 +-
 .../python/data/kernel_tests/iterator_test.py |  17 +--
 .../python/data/kernel_tests/map_test.py      | 138 ++++++++----------
 .../python/data/kernel_tests/shuffle_test.py  |   8 +-
 .../python/data/kernel_tests/test_base.py     |   2 +-
 tensorflow/python/data/ops/dataset_ops.py     | 129 ++++++++++------
 tensorflow/python/distribute/values.py        |   5 +-
 tensorflow/python/keras/engine/training.py    |   2 +-
 .../tools/api/golden/v1/tensorflow.data.pbtxt |   4 +
 .../golden/v2/tensorflow.data.-dataset.pbtxt  |   4 -
 ...ow.data.-fixed-length-record-dataset.pbtxt |   4 -
 .../tensorflow.data.-t-f-record-dataset.pbtxt |   4 -
 .../tensorflow.data.-text-line-dataset.pbtxt  |   4 -
 ...rflow.data.experimental.-csv-dataset.pbtxt |   4 -
 ...ow.data.experimental.-random-dataset.pbtxt |   4 -
 ...rflow.data.experimental.-sql-dataset.pbtxt |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |   1 +
 61 files changed, 376 insertions(+), 394 deletions(-)

diff --git a/tensorflow/contrib/bigtable/README.md b/tensorflow/contrib/bigtable/README.md
index 2c44abed5e..79052bee35 100644
--- a/tensorflow/contrib/bigtable/README.md
+++ b/tensorflow/contrib/bigtable/README.md
@@ -51,25 +51,18 @@ BIGTABLE_TABLE_NAME = '<FILL_ME_IN>'
 PREFIX = 'train-'
 
 def main():
+  tf.enable_eager_execution()
+
   client = tf.contrib.cloud.BigtableClient(GCP_PROJECT_ID, BIGTABLE_INSTANCE_ID)
   table = client.table(BIGTABLE_TABLE_NAME)
   dataset = table.keys_by_prefix_dataset(PREFIX)
-  iterator = dataset.make_initializable_iterator()
-  get_next_op = iterator.get_next()
 
-  with tf.Session() as sess:
-    print('Initializing the iterator.')
-    sess.run(iterator.initializer)
-    print('Retrieving rows:')
-    row_index = 0
-    while True:
-      try:
-        row_key = sess.run(get_next_op)
-        print('Row key %d: %s' % (row_index, row_key))
-        row_index += 1
-      except tf.errors.OutOfRangeError:
-        print('Finished reading data!')
-        break
+  print('Retrieving rows:')
+  row_index = 0
+  for row_key in dataset:
+    print('Row key %d: %s' % (row_index, row_key))
+    row_index += 1
+  print('Finished reading data!')
 
 if __name__ == '__main__':
   main()
diff --git a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py
index 316da9ebe1..197f5578eb 100644
--- a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py
+++ b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py
@@ -57,7 +57,7 @@ class BigtableOpsTest(test.TestCase):
     sess.run(write_op)
 
   def runReadKeyTest(self, read_ds):
-    itr = read_ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(read_ds)
     n = itr.get_next()
     expected = list(self.COMMON_ROW_KEYS)
     expected.reverse()
@@ -78,7 +78,7 @@ class BigtableOpsTest(test.TestCase):
     self.runReadKeyTest(self._table.keys_by_range_dataset("r1", "r4"))
 
   def runScanTest(self, read_ds):
-    itr = read_ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(read_ds)
     n = itr.get_next()
     expected_keys = list(self.COMMON_ROW_KEYS)
     expected_keys.reverse()
@@ -120,7 +120,7 @@ class BigtableOpsTest(test.TestCase):
   def testLookup(self):
     ds = self._table.keys_by_prefix_dataset("r")
     ds = ds.apply(self._table.lookup_columns(cf1="c1"))
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     n = itr.get_next()
     expected_keys = list(self.COMMON_ROW_KEYS)
     expected_values = list(self.COMMON_VALUES)
@@ -141,7 +141,7 @@ class BigtableOpsTest(test.TestCase):
 
   def testSampleKeys(self):
     ds = self._table.sample_keys()
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     n = itr.get_next()
     expected_key = self.COMMON_ROW_KEYS[0]
     with self.cached_session() as sess:
@@ -161,7 +161,7 @@ class BigtableOpsTest(test.TestCase):
         sess.run(n)
 
   def runSampleKeyPairsTest(self, ds, expected_key_pairs):
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     n = itr.get_next()
     with self.cached_session() as sess:
       self._writeCommonValues(sess)
@@ -218,7 +218,7 @@ class BigtableOpsTest(test.TestCase):
   def testSampleKeyPairsPrefixAndStartKey(self):
     ds = bigtable_api._BigtableSampleKeyPairsDataset(
         self._table, prefix="r", start="r1", end="")
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     with self.cached_session() as sess:
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(itr.initializer)
@@ -226,14 +226,14 @@ class BigtableOpsTest(test.TestCase):
   def testSampleKeyPairsPrefixAndEndKey(self):
     ds = bigtable_api._BigtableSampleKeyPairsDataset(
         self._table, prefix="r", start="", end="r3")
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     with self.cached_session() as sess:
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(itr.initializer)
 
   def testParallelScanPrefix(self):
     ds = self._table.parallel_scan_prefix(prefix="r", cf1="c1")
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     n = itr.get_next()
     with self.cached_session() as sess:
       self._writeCommonValues(sess)
@@ -251,7 +251,7 @@ class BigtableOpsTest(test.TestCase):
 
   def testParallelScanRange(self):
     ds = self._table.parallel_scan_range(start="r1", end="r4", cf1="c1")
-    itr = ds.make_initializable_iterator()
+    itr = dataset_ops.make_initializable_iterator(ds)
     n = itr.get_next()
     with self.cached_session() as sess:
       self._writeCommonValues(sess)
diff --git a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py
index 0456463a19..6c5f8c6b00 100644
--- a/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/assert_element_shape_test.py
@@ -46,7 +46,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase):
     result = dataset.apply(batching.assert_element_shape(expected_shapes))
     self.assertEqual(expected_shapes, result.output_shapes)
 
-    iterator = result.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(result)
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
@@ -88,7 +88,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase):
     result = dataset.apply(batching.assert_element_shape(expected_shapes))
     self.assertEqual(expected_shapes, result.output_shapes)
 
-    iterator = result.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(result)
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
@@ -115,9 +115,8 @@ class AssertElementShapeTest(test_base.DatasetTestBase):
 
     wrong_shapes = (tensor_shape.TensorShape(2),
                     tensor_shape.TensorShape((3, 10)))
-    iterator = (
-        dataset.apply(batching.assert_element_shape(wrong_shapes))
-        .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset.apply(batching.assert_element_shape(wrong_shapes)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
@@ -142,7 +141,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase):
                      tensor_shape.TensorShape((3, 4)))
     self.assertEqual(actual_shapes, result.output_shapes)
 
-    iterator = result.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(result)
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
@@ -184,7 +183,7 @@ class AssertElementShapeTest(test_base.DatasetTestBase):
     result = dataset.apply(batching.assert_element_shape(expected_shapes))
     self.assertEqual(expected_shapes, result.output_shapes)
 
-    iterator = result.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(result)
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
@@ -211,9 +210,8 @@ class AssertElementShapeTest(test_base.DatasetTestBase):
 
     wrong_shapes = (tensor_shape.TensorShape(2),
                     tensor_shape.TensorShape((None, 10)))
-    iterator = (
-        dataset.apply(batching.assert_element_shape(wrong_shapes))
-        .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset.apply(batching.assert_element_shape(wrong_shapes)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
     with self.cached_session() as sess:
diff --git a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
index d2a72272db..b9840b1ff1 100644
--- a/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/lmdb_dataset_op_test.py
@@ -23,6 +23,7 @@ import shutil
 
 from tensorflow.contrib.data.python.ops import readers
 from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -48,7 +49,7 @@ class LMDBDatasetTest(test_base.DatasetTestBase):
     num_repeats = 2
 
     dataset = readers.LMDBDataset(filenames).repeat(num_repeats)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
index c5a7862322..2527706709 100644
--- a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
@@ -63,13 +63,13 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
     # RepeatDataset(count) ->
     # _SlideDataset(window_size, window_shift, window_stride).
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn)
         .repeat(count).apply(
             sliding.sliding_window_batch(
                 window_size=window_size_t,
                 window_shift=window_shift_t,
-                window_stride=window_stride_t)).make_initializable_iterator())
+                window_stride=window_stride_t)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -127,13 +127,13 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
     # RepeatDataset(count) -> _SlideDataset(window_size, stride, window_stride).
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn)
         .repeat(count).apply(
             sliding.sliding_window_batch(
                 window_size=window_size_t,
                 stride=stride_t,
-                window_stride=window_stride_t)).make_initializable_iterator())
+                window_stride=window_stride_t)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -173,12 +173,12 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     window_shift_t = array_ops.placeholder(dtypes.int64, shape=[])
     window_stride_t = array_ops.placeholder(dtypes.int64, shape=[])
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(10).map(lambda x: x).repeat(count_t).apply(
             sliding.sliding_window_batch(
                 window_size=window_size_t,
                 window_shift=window_shift_t,
-                window_stride=window_stride_t)).make_initializable_iterator())
+                window_stride=window_stride_t)))
     init_op = iterator.initializer
 
     with self.cached_session() as sess:
@@ -204,9 +204,9 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return sparse_tensor.SparseTensorValue(
           indices=[[0]], values=(i * [1]), dense_shape=[1])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
-        sliding.sliding_window_batch(
-            window_size=5, window_shift=3)).make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(_sparse).apply(
+            sliding.sliding_window_batch(window_size=5, window_shift=3)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -233,9 +233,9 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           values=array_ops.fill([math_ops.to_int32(i)], i),
           dense_shape=[i])
 
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
-        sliding.sliding_window_batch(
-            window_size=5, window_shift=3)).make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(_sparse).apply(
+            sliding.sliding_window_batch(window_size=5, window_shift=3)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -265,11 +265,10 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return sparse_tensor.SparseTensorValue(
           indices=[[0]], values=(i * [1]), dense_shape=[1])
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(10).map(_sparse).apply(
             sliding.sliding_window_batch(window_size=4, window_shift=2)).apply(
-                sliding.sliding_window_batch(window_size=3, window_shift=1))
-        .make_initializable_iterator())
+                sliding.sliding_window_batch(window_size=3, window_shift=1)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -305,11 +304,10 @@ class SlideDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       yield [4.0, 5.0, 6.0]
       yield [7.0, 8.0, 9.0, 10.0]
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_generator(
             generator, dtypes.float32, output_shapes=[None]).apply(
-                sliding.sliding_window_batch(window_size=3, window_shift=1))
-        .make_initializable_iterator())
+                sliding.sliding_window_batch(window_size=3, window_shift=1)))
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py
index 557ad42752..d412b25b36 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_graph_test.py
@@ -36,7 +36,7 @@ class GraphLinearRegressionBenchmark(tf.test.Benchmark):
         noise_level=0.01,
         batch_size=batch_size,
         num_batches=num_batches)
-    iterator = dataset.make_initializable_iterator()
+    iterator = tf.compat.v1.data.make_initializable_iterator(dataset)
     x, y = iterator.get_next()
 
     model = linear_regression.LinearModel()
diff --git a/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py b/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py
index f7f1189bb9..bc941ae9f2 100644
--- a/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py
+++ b/tensorflow/contrib/hadoop/python/kernel_tests/hadoop_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import os
 
 from tensorflow.contrib.hadoop.python.ops import hadoop_dataset_ops
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -47,7 +48,7 @@ class SequenceFileDatasetTest(test.TestCase):
 
     dataset = hadoop_dataset_ops.SequenceFileDataset(filenames).repeat(
         num_repeats)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/contrib/tpu/python/tpu/datasets_test.py b/tensorflow/contrib/tpu/python/tpu/datasets_test.py
index b58d05eac5..52d87b8004 100644
--- a/tensorflow/contrib/tpu/python/tpu/datasets_test.py
+++ b/tensorflow/contrib/tpu/python/tpu/datasets_test.py
@@ -70,7 +70,7 @@ class DatasetsTest(test.TestCase):
     dataset = datasets.StreamingFilesDataset(
         os.path.join(self.get_temp_dir(), 'text_line.*.txt'), filetype='text')
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     self._sess.run(iterator.initializer)
     get_next = iterator.get_next()
 
@@ -94,7 +94,7 @@ class DatasetsTest(test.TestCase):
     dataset = datasets.StreamingFilesDataset(
         os.path.join(self.get_temp_dir(), 'tf_record*'), filetype='tfrecord')
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     self._sess.run(iterator.initializer)
     get_next = iterator.get_next()
 
@@ -121,7 +121,7 @@ class DatasetsTest(test.TestCase):
 
     dataset = datasets.StreamingFilesDataset(filenames, filetype='tfrecord')
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     self._sess.run(iterator.initializer)
     get_next = iterator.get_next()
 
@@ -154,7 +154,7 @@ class DatasetsTest(test.TestCase):
         os.path.join(self.get_temp_dir(), 'fixed_length*'),
         filetype=FixedLengthFile)
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     self._sess.run(iterator.initializer)
     get_next = iterator.get_next()
 
@@ -177,7 +177,7 @@ class DatasetsTest(test.TestCase):
     dataset = datasets.StreamingFilesDataset(
         dataset_ops.Dataset.range(10), filetype=gen_dataset)
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     self._sess.run(iterator.initializer)
     get_next = iterator.get_next()
 
diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index cf3b2e68e9..7f280eabd2 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -729,7 +729,7 @@ class TPUDatasetInfeedManager(TPUInfeedManager):
     dummy_x_shape[0] *= tpu_assignment.num_towers
     dummy_y_shape = dataset.output_shapes[1].as_list()
     dummy_y_shape[0] *= tpu_assignment.num_towers
-    self._iterator = dataset.make_initializable_iterator()
+    self._iterator = dataset_ops.make_initializable_iterator(dataset)
     K.get_session().run(self._iterator.initializer)
 
     self._get_next_ops = []
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 7171587ff7..a9dc542ae5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -3081,7 +3081,7 @@ class _Inputs(object):
 
     The initializer must be run before calling `features_and_labels`.
     """
-    self._iterator = self._dataset.make_initializable_iterator()
+    self._iterator = dataset_ops.make_initializable_iterator(self._dataset)
     return self._iterator.initializer
 
   def features_and_labels(self):
diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index 1a35efedfa..670897744a 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -123,7 +123,7 @@ def _dataset_for_stmt(ds, extra_test, body, init_state):
         (dataset_ops.Dataset.from_tensors(tag).repeat(), ds))
   ds_with_epoch = epoch_numbers.flat_map(lambda i: tag_with(ds, i))
 
-  iterator = ds_with_epoch.make_initializable_iterator()
+  iterator = dataset_ops.make_initializable_iterator(ds_with_epoch)
   with ops.control_dependencies((iterator.initializer,)):
     epoch_number, iterate = iterator.get_next()
 
diff --git a/tensorflow/python/data/__init__.py b/tensorflow/python/data/__init__.py
index 2da98bbb8e..75ba88f303 100644
--- a/tensorflow/python/data/__init__.py
+++ b/tensorflow/python/data/__init__.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 # pylint: disable=unused-import
 from tensorflow.python.data import experimental
 from tensorflow.python.data.ops.dataset_ops import Dataset
+from tensorflow.python.data.ops.dataset_ops import make_initializable_iterator
 from tensorflow.python.data.ops.dataset_ops import make_one_shot_iterator
 from tensorflow.python.data.ops.iterator_ops import Iterator
 from tensorflow.python.data.ops.readers import FixedLengthRecordDataset
diff --git a/tensorflow/python/data/benchmarks/batch_benchmark.py b/tensorflow/python/data/benchmarks/batch_benchmark.py
index b61ac86eb5..e063849f70 100644
--- a/tensorflow/python/data/benchmarks/batch_benchmark.py
+++ b/tensorflow/python/data/benchmarks/batch_benchmark.py
@@ -42,7 +42,7 @@ class BatchBenchmark(test.Benchmark):
 
     dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat(
         ).batch(batch_size_placeholder)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     for non_zeros_per_row in non_zeros_per_row_values:
diff --git a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
index 74a2d271ad..d7f1a4e7af 100644
--- a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
+++ b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
@@ -41,7 +41,7 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data)
         .repeat(num_epochs + 1).batch(batch_size))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with session.Session() as sess:
@@ -77,7 +77,7 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data)
         .repeat(num_epochs + 1).batch(batch_size))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with session.Session() as sess:
@@ -116,7 +116,7 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100))
         .repeat(num_epochs + 1))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with session.Session() as sess:
@@ -154,7 +154,7 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size)
         .cache().repeat(num_epochs + 1))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with session.Session() as sess:
diff --git a/tensorflow/python/data/benchmarks/range_benchmark.py b/tensorflow/python/data/benchmarks/range_benchmark.py
index 25f63b79a2..a5020e2873 100644
--- a/tensorflow/python/data/benchmarks/range_benchmark.py
+++ b/tensorflow/python/data/benchmarks/range_benchmark.py
@@ -39,7 +39,7 @@ class RangeBenchmark(test.Benchmark):
     # costs).
     dataset = dataset_ops.Dataset.range(num_elements).skip(
         num_elements - 1).take(1).with_options(options)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
index 3e0701ee4f..fbd06a5a78 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
@@ -54,7 +54,7 @@ class MapAndBatchBenchmark(test.Benchmark):
 
     dataset = dataset.apply(batching.map_and_batch(
         lambda _: dense_value, batch_size_placeholder))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     for shape in shapes:
diff --git a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
index c40d479823..c36a32534d 100644
--- a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
@@ -42,7 +42,7 @@ class UnbatchBenchmark(test.Benchmark):
       dataset = dataset.batch(batch_size_placeholder)
       dataset = dataset.apply(batching.unbatch())
       dataset = dataset.skip(elems_per_trial)
-      iterator = dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(dataset)
       next_element = iterator.get_next()
 
       with session.Session() as sess:
@@ -78,7 +78,7 @@ class UnbatchBenchmark(test.Benchmark):
       dataset = dataset.batch(batch_size_placeholder)
       dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices)
       dataset = dataset.skip(elems_per_trial)
-      iterator = dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(dataset)
       next_element = iterator.get_next()
 
       with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
index 78b8bda1b7..b8166fe833 100644
--- a/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/copy_to_device_test.py
@@ -275,7 +275,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0"))
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -295,7 +295,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0")).prefetch(1)
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -329,7 +329,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     device_dataset = device_dataset.with_options(options)
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -352,7 +352,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0"))
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -371,7 +371,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0")).prefetch(1)
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -390,7 +390,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0"))
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -409,7 +409,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0"))
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -431,7 +431,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
           prefetching_ops.copy_to_device("/cpu:0", source_device="/gpu:0"))
 
       with ops.device("/cpu:0"):
-        iterator = back_to_cpu_dataset.make_initializable_iterator()
+        iterator = dataset_ops.make_initializable_iterator(back_to_cpu_dataset)
         next_element = iterator.get_next()
 
       with self.cached_session(
@@ -449,7 +449,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -480,7 +480,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/cpu:1")).prefetch(1)
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -513,7 +513,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0"))
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -536,7 +536,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.copy_to_device("/gpu:0")).prefetch(1)
 
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     with self.cached_session(
@@ -558,7 +558,7 @@ class CopyToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.copy_to_device("/gpu:0"))
     with ops.device("/gpu:0"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_elem = iterator_ops.get_next_as_optional(iterator)
       elem_has_value_t = next_elem.has_value()
       elem_value_t = next_elem.get_value()
diff --git a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
index 4b84446be8..22e057a284 100644
--- a/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/dense_to_sparse_batch_test.py
@@ -34,11 +34,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
   @test_util.run_deprecated_v1
   def testDenseToSparseBatchDataset(self):
     components = np.random.randint(12, size=(100,)).astype(np.int32)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components)
         .map(lambda x: array_ops.fill([x], x)).apply(
-            batching.dense_to_sparse_batch(4, [12]))
-        .make_initializable_iterator())
+            batching.dense_to_sparse_batch(4, [12])))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -63,11 +62,10 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
   @test_util.run_deprecated_v1
   def testDenseToSparseBatchDatasetWithUnknownShape(self):
     components = np.random.randint(5, size=(40,)).astype(np.int32)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components)
         .map(lambda x: array_ops.fill([x, x], x)).apply(
-            batching.dense_to_sparse_batch(
-                4, [5, None])).make_initializable_iterator())
+            batching.dense_to_sparse_batch(4, [5, None])))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -98,16 +96,16 @@ class DenseToSparseBatchTest(test_base.DatasetTestBase):
   def testDenseToSparseBatchDatasetWithInvalidShape(self):
     input_tensor = array_ops.constant([[1]])
     with self.assertRaisesRegexp(ValueError, "Dimension -2 must be >= 0"):
-      dataset_ops.Dataset.from_tensors(input_tensor).apply(
-          batching.dense_to_sparse_batch(4, [-2])).make_initializable_iterator()
+      dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.from_tensors(input_tensor).apply(
+              batching.dense_to_sparse_batch(4, [-2])))
 
   @test_util.run_deprecated_v1
   def testDenseToSparseBatchDatasetShapeErrors(self):
     input_tensor = array_ops.placeholder(dtypes.int32)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensors(input_tensor).apply(
-            batching.dense_to_sparse_batch(4, [12]))
-        .make_initializable_iterator())
+            batching.dense_to_sparse_batch(4, [12])))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
index ec2c5407b4..2144342066 100644
--- a/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/directed_interleave_dataset_test.py
@@ -38,7 +38,7 @@ class DirectedInterleaveDatasetTest(test_base.DatasetTestBase):
     ]
     dataset = interleave_ops._DirectedInterleaveDataset(selector_dataset,
                                                         input_datasets)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
index 3c2e1bb7f3..25742098f1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/enumerate_dataset_test.py
@@ -35,8 +35,9 @@ class EnumerateDatasetTest(test_base.DatasetTestBase):
     components = (["a", "b"], [1, 2], [37.0, 38])
     start = constant_op.constant(20, dtype=dtypes.int64)
 
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components).apply(
-        enumerate_ops.enumerate_dataset(start)).make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensor_slices(components).apply(
+            enumerate_ops.enumerate_dataset(start)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
index 1e54091c7d..cbb79e55f5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/group_by_window_test.py
@@ -65,7 +65,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
             lambda x, y, z: 0,
             lambda k, bucket: self._dynamicPad(k, bucket, 32), 32))
 
-    iterator = bucketed_dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(bucketed_dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -101,7 +101,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
             lambda x, y, z: math_ops.cast(x % 2, dtypes.int64),
             lambda k, bucket: self._dynamicPad(k, bucket, 32), 32))
 
-    iterator = bucketed_dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(bucketed_dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -173,7 +173,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
             lambda d: math_ops.cast(d["x"] % 2, dtypes.int64),
             lambda k, bucket: _dynamic_pad_fn(k, bucket, 32), 32))
 
-    iterator = bucketed_dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(bucketed_dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -207,7 +207,7 @@ class GroupByWindowTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.from_tensor_slices(components).apply(
         grouping.group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(20),
                                  None, window_size_func))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -229,11 +229,11 @@ class GroupByWindowTest(test_base.DatasetTestBase):
   @test_util.run_deprecated_v1
   def testSimple(self):
     components = np.random.randint(100, size=(200,)).astype(np.int64)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components).map(lambda x: x * x)
         .apply(
             grouping.group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(4),
-                                     4)).make_initializable_iterator())
+                                     4)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -258,10 +258,10 @@ class GroupByWindowTest(test_base.DatasetTestBase):
   def testImmediateOutput(self):
     components = np.array(
         [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0], dtype=np.int64)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components).repeat(-1).apply(
             grouping.group_by_window(lambda x: x % 3, lambda _, xs: xs.batch(4),
-                                     4)).make_initializable_iterator())
+                                     4)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -280,10 +280,10 @@ class GroupByWindowTest(test_base.DatasetTestBase):
   @test_util.run_deprecated_v1
   def testSmallGroups(self):
     components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components).apply(
             grouping.group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(4),
-                                     4)).make_initializable_iterator())
+                                     4)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -298,10 +298,9 @@ class GroupByWindowTest(test_base.DatasetTestBase):
 
   @test_util.run_deprecated_v1
   def testEmpty(self):
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(4).apply(
-            grouping.group_by_window(lambda _: 0, lambda _, xs: xs, 0))
-        .make_initializable_iterator())
+            grouping.group_by_window(lambda _: 0, lambda _, xs: xs, 0)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -324,11 +323,10 @@ class GroupByWindowTest(test_base.DatasetTestBase):
           padded_shapes=(tensor_shape.TensorShape([]),
                          constant_op.constant([5], dtype=dtypes.int64) * -1))
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components)
         .map(lambda x: (x, ops.convert_to_tensor([x * x]))).apply(
-            grouping.group_by_window(lambda x, _: x % 2, reduce_func,
-                                     32)).make_initializable_iterator())
+            grouping.group_by_window(lambda x, _: x % 2, reduce_func, 32)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -351,13 +349,12 @@ class GroupByWindowTest(test_base.DatasetTestBase):
               4, padded_shapes=ops.convert_to_tensor([(key + 1) * 10])),
       ))
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensor_slices(components)
         .map(lambda x: array_ops.fill([math_ops.cast(x, dtypes.int32)], x))
         .apply(grouping.group_by_window(
             lambda x: math_ops.cast(array_ops.shape(x)[0] // 10, dtypes.int64),
-            reduce_func, 4))
-        .make_initializable_iterator())
+            reduce_func, 4)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
index bd323592e4..81f580fccb 100644
--- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py
@@ -44,7 +44,7 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
         dataset_ops.Dataset.from_tensor_slices(components)
         .map(lambda x: array_ops.check_numerics(x, "message")).apply(
             error_ops.ignore_errors()))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -63,7 +63,7 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
         dataset_ops.Dataset.from_tensor_slices(components).map(
             lambda x: array_ops.check_numerics(x, "message"),
             num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors()))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -91,7 +91,7 @@ class IgnoreErrorsTest(test_base.DatasetTestBase):
         dataset_ops.Dataset.from_tensor_slices(filenames).map(
             io_ops.read_file,
             num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors()))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
index 3a0ff018e8..7c78810494 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_batched_features_dataset_test.py
@@ -114,7 +114,7 @@ class MakeBatchedFeaturesDatasetTest(
         core_readers.TFRecordDataset(self.test_filenames)
         .map(lambda x: parsing_ops.parse_single_example(x, features))
         .repeat(10).batch(2))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     next_element = iterator.get_next()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
index 64798feaf8..ab2feb6426 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_tf_record_dataset_test.py
@@ -187,7 +187,7 @@ class MakeTFRecordDatasetTest(
             num_parallel_reads=num_parallel_reads,
             shuffle=True,
             shuffle_seed=seed)
-        iterator = dataset.make_initializable_iterator()
+        iterator = dataset_ops.make_initializable_iterator(dataset)
         next_element = iterator.get_next()
 
         self.evaluate(iterator.initializer)
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index cad04a70e3..e6e24c3db1 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -78,7 +78,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -262,7 +262,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
 
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -296,7 +296,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
 
     init_op = iterator.initializer
     with self.cached_session() as sess:
@@ -325,7 +325,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       options = dataset_ops.Options()
       options.experimental_numa_aware = True
       dataset = dataset.with_options(options)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
 
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -482,7 +482,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     captured_t = array_ops.placeholder(dtypes.int64, shape=[])
     dataset = self.structuredDataset(None).repeat().apply(
         batching.map_and_batch(lambda x: captured_t, batch_size=10))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 22705e20bf..2e31df716a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -45,7 +45,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
     dataset = dataset_ops.Dataset.from_tensors(input_t)
     dataset = dataset_ops._OptimizeDataset(dataset, [])
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -58,7 +58,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
     dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
     dataset = dataset_ops._OptimizeDataset(dataset, [])
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
index de2788dfcb..aa81663a18 100644
--- a/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/override_threadpool_test.py
@@ -53,7 +53,7 @@ class OverrideThreadpoolTest(test_base.DatasetTestBase,
             lambda x: script_ops.py_func(get_thread_id, [x], dtypes.int64),
             num_parallel_calls=32).apply(unique.unique()))
     dataset = override_threadpool_fn(dataset)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     self.evaluate(iterator.initializer)
diff --git a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
index c09bfa695e..113326c028 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parallel_interleave_test.py
@@ -86,7 +86,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                                                self.block_length, self.sloppy,
                                                self.buffer_output_elements,
                                                self.prefetch_input_elements)))
-    self.iterator = self.dataset.make_initializable_iterator()
+    self.iterator = dataset_ops.make_initializable_iterator(self.dataset)
     self.init_op = self.iterator.initializer
     self.next_element = self.iterator.get_next()
 
@@ -630,9 +630,8 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
           sparse_ops.sparse_to_dense(x.indices, x.dense_shape, x.values))
 
     dataset = dataset_ops.Dataset.range(10).map(_map_fn)
-    iterator = dataset.apply(
-        interleave_ops.parallel_interleave(
-            _interleave_fn, cycle_length=1)).make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset.apply(
+        interleave_ops.parallel_interleave(_interleave_fn, cycle_length=1)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -701,7 +700,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                                                self.buffer_output_elements,
                                                self.prefetch_input_elements)))
 
-    self.iterator = self.dataset.make_initializable_iterator()
+    self.iterator = dataset_ops.make_initializable_iterator(self.dataset)
     self.init_op = self.iterator.initializer
     self.next_element = self.iterator.get_next()
 
@@ -750,7 +749,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
                                                self.buffer_output_elements,
                                                self.prefetch_input_elements)))
 
-    self.iterator = self.dataset.make_initializable_iterator()
+    self.iterator = dataset_ops.make_initializable_iterator(self.dataset)
     self.init_op = self.iterator.initializer
     self.next_element = self.iterator.get_next()
 
@@ -789,7 +788,7 @@ class ParallelInterleaveTest(test_base.DatasetTestBase):
             buffer_output_elements=1,
             prefetch_input_elements=0))
     dataset = dataset.batch(32)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     results = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
index 19d5cdb80c..80bd43e9ad 100644
--- a/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/prefetch_to_device_test.py
@@ -153,7 +153,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/gpu:0"))
 
-    iterator = device_dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(device_dataset)
     next_element = iterator.get_next()
 
     with self.cached_session(
@@ -171,7 +171,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
         prefetching_ops.prefetch_to_device("/cpu:1"))
 
     with ops.device("/cpu:1"):
-      iterator = device_dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(device_dataset)
       next_element = iterator.get_next()
 
     self.assertEqual(host_dataset.output_types, device_dataset.output_types)
@@ -203,7 +203,7 @@ class PrefetchToDeviceTest(test_base.DatasetTestBase):
     device_dataset = host_dataset.apply(
         prefetching_ops.prefetch_to_device("/gpu:0"))
 
-    iterator = device_dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(device_dataset)
     next_element = iterator.get_next()
 
     with self.cached_session(
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index 3b978b004b..03af7ecd2f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -48,8 +48,8 @@ class ScanTest(test_base.DatasetTestBase):
     start = array_ops.placeholder(dtypes.int32, shape=[])
     step = array_ops.placeholder(dtypes.int32, shape=[])
     take = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = self._counting_dataset(
-        start, make_scan_fn(step)).take(take).make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(self._counting_dataset(
+        start, make_scan_fn(step)).take(take))
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -98,9 +98,8 @@ class ScanTest(test_base.DatasetTestBase):
     start = array_ops.placeholder(dtypes.int32, shape=[])
     step = array_ops.placeholder(dtypes.int32, shape=[])
     take = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = self._counting_dataset(
-        _sparse(start),
-        make_scan_fn(step)).take(take).make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(self._counting_dataset(
+        _sparse(start), make_scan_fn(step)).take(take))
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
index 7f435b8239..e65aa44d06 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
@@ -23,6 +23,7 @@ import os
 import numpy as np
 
 from tensorflow.python.data.experimental.ops import iterator_ops as contrib_iterator_ops
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -578,7 +579,7 @@ class DatasetSerializationTestBase(test.TestCase):
     return np.linspace(0, num_outputs, num_samples, dtype=int)
 
   def _build_graph(self, ds_fn, sparse_tensors=False):
-    iterator = ds_fn().make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(ds_fn())
 
     saveable = contrib_iterator_ops.make_saveable_from_iterator(iterator)
     ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
index aeb338dfd5..34419a3149 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/range_dataset_serialization_test.py
@@ -56,8 +56,8 @@ class RangeDatasetSerializationTest(
   def testSaveRestore(self):
 
     def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(start, stop))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
diff --git a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
index 8c9e8225d6..46b22f80b6 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sleep_test.py
@@ -36,7 +36,7 @@ class SleepTest(test_base.DatasetTestBase):
     sleep_microseconds = 100
     dataset = dataset_ops.Dataset.range(10).apply(
         sleep.sleep(sleep_microseconds))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py
index 6aaaa90c65..809e09c804 100644
--- a/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/sql_dataset_test_base.py
@@ -24,6 +24,7 @@ import sqlite3
 
 from tensorflow.python.data.experimental.ops import readers
 from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
@@ -35,7 +36,7 @@ class SqlDatasetTestBase(test_base.DatasetTestBase):
   def _createSqlDataset(self, output_types, num_repeats=1):
     dataset = readers.SqlDataset(self.driver_name, self.data_source_name,
                                  self.query, output_types).repeat(num_repeats)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
     return init_op, get_next
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index b89aa20432..8a300364f9 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -67,7 +67,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).apply(
             stats_ops.bytes_produced_stats("bytes_produced"))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -93,7 +93,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset = dataset_ops.Dataset.range(100).apply(
         stats_ops.latency_stats("record_latency"))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -114,7 +114,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset = dataset_ops.Dataset.range(100).map(
         lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).prefetch(-1)
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -142,7 +142,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).map(
         lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).prefetch(0)
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -165,7 +165,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset = dataset_ops.Dataset.range(101).filter(
         lambda x: math_ops.equal(math_ops.mod(x, 3), 0))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -264,7 +264,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset = dataset_ops.Dataset.range(100).apply(
         stats_ops.latency_stats("record_latency"))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -285,7 +285,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
   def testNoAggregatorRegistered(self, dataset_transformation):
     dataset = dataset_ops.Dataset.range(100).apply(
         stats_ops.latency_stats("record_latency"))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -302,7 +302,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         stats_ops.latency_stats("record_latency")).apply(
             stats_ops.latency_stats("record_latency_2"))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -328,7 +328,7 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
         stats_ops.latency_stats("record_latency")).apply(
             stats_ops.latency_stats("record_latency"))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
@@ -349,8 +349,8 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset = dataset_ops.Dataset.range(100).apply(
         stats_ops.latency_stats("record_latency"))
     dataset = dataset_transformation(dataset, aggregator)
-    iterator_0 = dataset.make_initializable_iterator()
-    iterator_1 = dataset.make_initializable_iterator()
+    iterator_0 = dataset_ops.make_initializable_iterator(dataset)
+    iterator_1 = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator_0.get_next() + iterator_1.get_next()
     summary_t = aggregator.get_summary()
 
@@ -374,8 +374,8 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
     dataset2 = dataset_ops.Dataset.range(100).apply(
         stats_ops.latency_stats("record_latency"))
     dataset2 = dataset_transformation(dataset2, aggregator, prefix="dataset2")
-    iterator_0 = dataset.make_initializable_iterator()
-    iterator_1 = dataset2.make_initializable_iterator()
+    iterator_0 = dataset_ops.make_initializable_iterator(dataset)
+    iterator_1 = dataset_ops.make_initializable_iterator(dataset2)
     next_element = iterator_0.get_next() + iterator_1.get_next()
     summary_t = aggregator.get_summary()
 
@@ -435,7 +435,7 @@ class FeatureStatsDatasetTest(
 
     dataset = dataset_transformation(
         dataset_fn(), aggregator, prefix="record_stats")
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
index c5bf926759..ab1d1c3028 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
@@ -22,6 +22,7 @@ import numpy as np
 from tensorflow.core.framework import summary_pb2
 from tensorflow.python.data.experimental.ops import stats_aggregator
 from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 
 
@@ -93,7 +94,7 @@ class StatsDatasetTestBase(test_base.DatasetTestBase):
     aggregator = stats_aggregator.StatsAggregator()
     dataset = dataset_fn()
     dataset = dataset_transformation(dataset, aggregator)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
     summary_t = aggregator.get_summary()
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index b6b12bd155..cef5e8d269 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -43,7 +43,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     placeholder = array_ops.placeholder(dtypes.int32)
     dataset = dataset_ops.Dataset.from_tensors(placeholder).apply(
         batching.unbatch())
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_elem = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -206,7 +206,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     ph2 = array_ops.placeholder(dtypes.int32, shape=None)
     data = dataset_ops.Dataset.from_tensors((ph1, ph2))
     data = data.apply(batching.unbatch())
-    iterator = data.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(data)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/experimental/kernel_tests/unique_test.py b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
index ddec968858..1d9941d7f4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unique_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unique_test.py
@@ -44,7 +44,7 @@ class UniqueTest(test_base.DatasetTestBase):
     current_test_case = []
     dataset = dataset_ops.Dataset.from_generator(lambda: current_test_case,
                                                  dtype).apply(unique.unique())
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py b/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py
index cdaa4fd4d5..6dcd94ea02 100644
--- a/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_checkpoint_test.py
@@ -64,8 +64,8 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
   def testSaveRestore(self):
 
     def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(start, stop))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
@@ -114,7 +114,7 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
 
     def _build_graph(start, stop, num_epochs):
       dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
-      iterator = dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(dataset)
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
@@ -161,7 +161,7 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
 
     def _build_graph(start, stop):
       dataset = dataset_ops.Dataset.range(start, stop)
-      iterator = dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(dataset)
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
@@ -200,7 +200,7 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
 
     def _build_graph(start, stop):
       dataset = dataset_ops.Dataset.range(start, stop)
-      iterator = dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(dataset)
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
@@ -233,8 +233,8 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
   def testMultipleSaves(self):
 
     def _build_graph(start, stop):
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(start, stop))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
@@ -276,8 +276,8 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
   def testSaveRestoreWithRepeat(self):
 
     def _build_graph(start, stop, num_epochs):
-      iterator = dataset_ops.Dataset.range(
-          start, stop).repeat(num_epochs).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(start, stop).repeat(num_epochs))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
@@ -321,8 +321,8 @@ class DatasetCheckpointTest(test_base.DatasetTestBase):
   def testSaveRestoreExhaustedIterator(self):
 
     def _build_graph(start, stop, num_epochs):
-      iterator = dataset_ops.Dataset.range(
-          start, stop).repeat(num_epochs).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(start, stop).repeat(num_epochs))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = self._save_op(iterator._iterator_resource)
diff --git a/tensorflow/python/data/kernel_tests/from_generator_test.py b/tensorflow/python/data/kernel_tests/from_generator_test.py
index e113b45496..a6625534e7 100644
--- a/tensorflow/python/data/kernel_tests/from_generator_test.py
+++ b/tensorflow/python/data/kernel_tests/from_generator_test.py
@@ -38,11 +38,10 @@ class FromGeneratorTest(test_base.DatasetTestBase):
                          output_types=None):
     if output_types is None:
       output_types = dtypes.int64
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_generator(generator, output_types=output_types)
         .repeat(num_repeats)
-        .prefetch(5)
-        .make_initializable_iterator())
+        .prefetch(5))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -129,11 +128,10 @@ class FromGeneratorTest(test_base.DatasetTestBase):
           output_shapes=([None], [3]))
               .repeat(num_inner_repeats).prefetch(5))
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(num_outer_repeats)
         .interleave(interleave_fn, cycle_length=10,
-                    block_length=len(input_list))
-        .make_initializable_iterator())
+                    block_length=len(input_list)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -188,11 +186,10 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       return dataset_ops.Dataset.from_generator(
           generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2)
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(num_parallel_iterators)
         .interleave(
-            interleave_fn, cycle_length=num_parallel_iterators, block_length=1)
-        .make_initializable_iterator())
+            interleave_fn, cycle_length=num_parallel_iterators, block_length=1))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -212,9 +209,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield [3]
 
     for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]:
-      iterator = (dataset_ops.Dataset.from_generator(
-          generator, output_types=dtype, output_shapes=[1])
-                  .make_initializable_iterator())
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.from_generator(
+              generator, output_types=dtype, output_shapes=[1]))
       init_op = iterator.initializer
       get_next = iterator.get_next()
 
@@ -236,9 +233,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield b"bar"
       yield u"baz"
 
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.string, output_shapes=[])
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_generator(
+            generator, output_types=dtypes.string, output_shapes=[]))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -258,9 +255,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield "ERROR"
       yield np.array([7, 8, 9], dtype=np.int64)
 
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.int64, output_shapes=[3])
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_generator(
+            generator, output_types=dtypes.int64, output_shapes=[3]))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -282,9 +279,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield np.array([7, 8, 9, 10], dtype=np.int64)
       yield np.array([11, 12, 13], dtype=np.int64)
 
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.int64, output_shapes=[3])
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_generator(
+            generator, output_types=dtypes.int64, output_shapes=[3]))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -307,9 +304,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield 6, 7, 8
       yield 9, 10
 
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=(dtypes.int64, dtypes.int64))
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_generator(
+            generator, output_types=(dtypes.int64, dtypes.int64)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -333,9 +330,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield 1
       yield [2, 3]
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_generator(
-            generator, output_types=dtypes.int64).make_initializable_iterator())
+            generator, output_types=dtypes.int64))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -354,9 +351,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       yield 1
       yield 2
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_generator(
-            generator, output_types=dtypes.int64).make_initializable_iterator())
+            generator, output_types=dtypes.int64))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -384,9 +381,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
       def __del__(self):
         event.set()
 
-    iterator = dataset_ops.Dataset.from_generator(
-        GeneratorWrapper,
-        output_types=dtypes.int64).take(2).make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_generator(
+            GeneratorWrapper, output_types=dtypes.int64).take(2))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -413,10 +410,8 @@ class FromGeneratorTest(test_base.DatasetTestBase):
           generator_with_arg, output_types=dtypes.int64, output_shapes=(),
           args=(elem,))
 
-    iterator = (dataset_ops.Dataset
-                .range(5)
-                .flat_map(flat_map_fn)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(5).flat_map(flat_map_fn))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -441,12 +436,11 @@ class FromGeneratorTest(test_base.DatasetTestBase):
           generator_with_arg, output_types=(dtypes.int64, dtypes.string),
           output_shapes=((), ()), args=(elem, message))
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.zip(
             (dataset_ops.Dataset.range(5),
              dataset_ops.Dataset.from_tensors("Hi!").repeat(None)))
-        .flat_map(flat_map_fn)
-        .make_initializable_iterator())
+        .flat_map(flat_map_fn))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -478,10 +472,9 @@ class FromGeneratorTest(test_base.DatasetTestBase):
                                 stateful=True)
 
     dummy = constant_op.constant(37)
-    iterator = (dataset_ops._GeneratorDataset(dummy, lambda x: x,
-                                              lambda x: x, finalize_fn)
-                .take(2)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops._GeneratorDataset(
+            dummy, lambda x: x, lambda x: x, finalize_fn).take(2))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
index 80ed26e7fb..ef608ebb67 100644
--- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
@@ -36,8 +36,8 @@ class FromSparseTensorSlicesTest(test_base.DatasetTestBase):
   def testSkipEagerFromSparseTensorSlices(self):
     """Test a dataset based on slices of a `tf.SparseTensor`."""
     st = array_ops.sparse_placeholder(dtypes.float64)
-    iterator = (dataset_ops.Dataset.from_sparse_tensor_slices(st)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_sparse_tensor_slices(st))
     init_op = iterator.initializer
     get_next = sparse_tensor.SparseTensor(*iterator.get_next())
 
diff --git a/tensorflow/python/data/kernel_tests/from_tensors_test.py b/tensorflow/python/data/kernel_tests/from_tensors_test.py
index ce70637572..ab3c15263f 100644
--- a/tensorflow/python/data/kernel_tests/from_tensors_test.py
+++ b/tensorflow/python/data/kernel_tests/from_tensors_test.py
@@ -246,7 +246,7 @@ class FromTensorsTest(test_base.DatasetTestBase):
         dataset = dataset.map(lambda x: x + var_1.read_value())
       sess.run(var_1.initializer)
 
-      iterator = dataset.make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(dataset)
       sess.run(iterator.initializer)
 
       with self.assertRaisesRegexp(
diff --git a/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
index 2d9d554e5e..91b356691b 100644
--- a/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_checkpoint_test.py
@@ -113,7 +113,7 @@ class IteratorCheckpointingTest(test_base.DatasetTestBase):
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
     dataset = dataset_ops.Dataset.range(10)
     iterator = iter(dataset) if context.executing_eagerly(
-    ) else dataset.make_initializable_iterator()
+    ) else dataset_ops.make_initializable_iterator(dataset)
     get_next = iterator.get_next
     checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
     for i in range(5):
diff --git a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
index 6e8210f904..728bed20a1 100644
--- a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
@@ -161,7 +161,7 @@ class IteratorClusterTest(test.TestCase):
         dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn)
         .repeat(None).prefetch(10000))
 
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
diff --git a/tensorflow/python/data/kernel_tests/iterator_test.py b/tensorflow/python/data/kernel_tests/iterator_test.py
index 32332a0790..916cf8bb45 100644
--- a/tensorflow/python/data/kernel_tests/iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_test.py
@@ -289,9 +289,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
   @test_util.run_deprecated_v1
   def testNotInitializedError(self):
     components = (np.array(1), np.array([1, 2, 3]), np.array(37.0))
-    iterator = (
-        dataset_ops.Dataset.from_tensors(components)
-        .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(components))
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -523,7 +522,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
   def testIteratorStringHandleReuseTensorObject(self):
     dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
     one_shot_iterator = dataset_ops.make_one_shot_iterator(dataset)
-    initializable_iterator = dataset.make_initializable_iterator()
+    initializable_iterator = dataset_ops.make_initializable_iterator(dataset)
     structure_iterator = iterator_ops.Iterator.from_structure(
         dataset.output_types)
 
@@ -687,7 +686,7 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
     with ops.device("/job:client"):
       client_dataset = dataset_ops.Dataset.zip((targets, handles)).map(map_fn)
-      itr = client_dataset.make_initializable_iterator()
+      itr = dataset_ops.make_initializable_iterator(client_dataset)
       n = itr.get_next()
 
     with session.Session(s3.target, config=config) as sess:
@@ -777,8 +776,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
     def _build_range_dataset_graph():
       start = 1
       stop = 10
-      iterator = dataset_ops.Dataset.range(start,
-                                           stop).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(start, stop))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       save_op = _save_op(iterator._iterator_resource)
@@ -787,8 +786,8 @@ class IteratorTest(test.TestCase, parameterized.TestCase):
 
     def _build_reader_dataset_graph():
       filenames = ["test"]  # Does not exist but we don't care in this test.
-      iterator = readers.FixedLengthRecordDataset(
-          filenames, 1, 0, 0).make_initializable_iterator()
+      iterator = dataset_ops.make_initializable_iterator(
+          readers.FixedLengthRecordDataset(filenames, 1, 0, 0))
       init_op = iterator.initializer
       get_next_op = iterator.get_next()
       save_op = _save_op(iterator._iterator_resource)
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index df522c4351..fdce791447 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -100,7 +100,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     count = array_ops.placeholder(dtypes.int64, shape=[])
 
     dataset = self._buildMapDataset(components, count)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -166,7 +166,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = self._buildParallelMapDataset(
         components, count, num_parallel_calls, output_buffer_size)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -235,7 +235,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = self._buildParallelMapDataset(components, 1000, 100, 100)
     # NOTE(mrry): Also test that the prefetching thread is cancelled correctly.
     dataset = dataset.prefetch(100)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -250,7 +250,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = (dataset_ops.Dataset.from_tensor_slices(components)
                .map(lambda x: array_ops.check_numerics(x, "message"),
                     num_parallel_calls=2))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -265,7 +265,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = (dataset_ops.Dataset.from_tensor_slices(components)
                .map(lambda x: array_ops.check_numerics(x, "message"),
                     num_parallel_calls=2))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -286,7 +286,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = (dataset_ops.Dataset.from_tensor_slices(components)
                .map(lambda x: array_ops.check_numerics(x, "message"))
                .prefetch(2))
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -312,8 +312,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return dataset_ops.Dataset.range(10).map(_map_fn)
 
     def _build_graph():
-      captured_iterator = dataset_ops.Dataset.range(
-          10).make_initializable_iterator()
+      captured_iterator = dataset_ops.make_initializable_iterator(
+          dataset_ops.Dataset.range(10))
       ds = _build_ds(captured_iterator)
       iterator = ds.make_initializable_iterator()
       init_op = iterator.initializer
@@ -343,10 +343,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     input_sentences = dataset_ops.Dataset.from_tensor_slices(
         ["brain brain tank salad surgery", "surgery brain"])
 
-    iterator = (input_sentences
-                .map(lambda x: string_ops.string_split([x]).values)
-                .map(table.lookup)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        input_sentences
+        .map(lambda x: string_ops.string_split([x]).values).map(table.lookup))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -363,8 +362,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     queue = data_flow_ops.FIFOQueue(200, dtypes.int64, shapes=[])
     enqueue_op = queue.enqueue_many(elements)
     close_op = queue.close()
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(-1)
-                .map(lambda _: queue.dequeue()).make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(0).repeat(-1)
+        .map(lambda _: queue.dequeue()))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -387,9 +387,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     enqueue_op = queue.enqueue_many(elements)
     close_op = queue.close()
 
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(-1)
-                .map(lambda _: (queue.dequeue(), queue_2.dequeue()))
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(0).repeat(-1)
+        .map(lambda _: (queue.dequeue(), queue_2.dequeue())))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -406,9 +406,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testCaptureVariable(self):
     counter_var = variable_scope.get_variable(
         "counter", (), dtypes.int32, use_resource=True)
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10)
-                .map(lambda _: counter_var.assign_add(1))
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(0).repeat(10)
+        .map(lambda _: counter_var.assign_add(1)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -426,9 +426,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testCaptureUninitializedVariableError(self):
     counter_var = variable_scope.get_variable(
         "counter", (), dtypes.int32, use_resource=True)
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10)
-                .map(lambda _: counter_var.assign_add(1))
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(0).repeat(10)
+        .map(lambda _: counter_var.assign_add(1)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -438,9 +438,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(get_next)
 
   def testSeededStatefulOperatorIsProperlyStateful(self):
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10)
-                .map(lambda _: random_ops.random_uniform((), seed=11)).batch(2)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(0).repeat(10)
+        .map(lambda _: random_ops.random_uniform((), seed=11)).batch(2))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -462,11 +462,11 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertAllClose(random_values, random_values_2)
 
   def testStatefulMapKeepsStateAcrossIterators(self):
-    iterator = (dataset_ops.Dataset.from_tensors(0).repeat(10)
-                .map(lambda _: random_ops.random_uniform((), seed=11))
-                .repeat(1000)
-                .batch(10)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.from_tensors(0).repeat(10)
+        .map(lambda _: random_ops.random_uniform((), seed=11))
+        .repeat(1000)
+        .batch(10))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -491,9 +491,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       counter_var.assign_add(1)
       return x
 
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(increment_fn)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(increment_fn))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -509,10 +508,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual(10, sess.run(counter_var))
 
   def testMapDict(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(lambda x: {"foo": x * 2, "bar": x ** 2})
-                .map(lambda d: d["foo"] + d["bar"])
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10)
+        .map(lambda x: {"foo": x * 2, "bar": x ** 2})
+        .map(lambda d: d["foo"] + d["bar"]))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -560,10 +559,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testUseStepContainerInMap(self):
     row = np.arange(6)
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensors(row)
-        .map(lambda elems: functional_ops.map_fn(lambda x: x * x, elems))
-        .make_initializable_iterator())
+        .map(lambda elems: functional_ops.map_fn(lambda x: x * x, elems)))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -599,9 +597,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           pred_fn_pairs, default=multiply, exclusive=True)
 
     def build_dataset(row, num):
-      iterator = (
+      iterator = dataset_ops.make_initializable_iterator(
           dataset_ops.Dataset.from_tensor_slices(row).map(
-              lambda x: control_map_fn(x, num)).make_initializable_iterator())
+              lambda x: control_map_fn(x, num)))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       return init_op, get_next
@@ -638,11 +636,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     def build_dataset(row, num):
       # pylint: disable=g-long-lambda
-      iterator = (
+      iterator = dataset_ops.make_initializable_iterator(
           dataset_ops.Dataset.from_tensors(row).map(
-              lambda elems: functional_ops.map_fn(lambda x:
-                                                  control_map_fn(x, num), elems)
-              ).make_initializable_iterator())
+              lambda elems: functional_ops.map_fn(
+                  lambda x: control_map_fn(x, num), elems)))
       init_op = iterator.initializer
       get_next = iterator.get_next()
       return init_op, get_next
@@ -686,11 +683,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     row = np.arange(6)
     num = 2
     # pylint: disable=g-long-lambda
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.from_tensors(row).map(
-            lambda elems: functional_ops.map_fn(lambda x:
-                                                control_map_fn(x, num), elems)
-            ).make_initializable_iterator())
+            lambda elems: functional_ops.map_fn(
+                lambda x: control_map_fn(x, num), elems)))
     # pylint: enable=g-long-lambda
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -720,11 +716,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return script_ops.py_func(_map_py_func, [x], x.dtype)
 
     buffer_size_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(100)
         .map(_map_fn)
-        .prefetch(buffer_size_placeholder)
-        .make_initializable_iterator())
+        .prefetch(buffer_size_placeholder))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -760,9 +755,9 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           sess.run(get_next)
 
   def testReturnList(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(lambda x: [x, constant_op.constant(37.0)])
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10)
+        .map(lambda x: [x, constant_op.constant(37.0)]))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -781,9 +776,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return script_ops.py_func(
           _map_py_func, [x_tensor], [dtypes.int64, dtypes.float64])
 
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(_map_fn)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(_map_fn))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -802,9 +796,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           values=(i * np.array([1])),
           dense_shape=np.array([1, 1]))
 
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(_sparse)
-                .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(_sparse))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -829,9 +822,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertTrue(sparse_tensor.is_sparse(i))
       return sparse_ops.sparse_concat(0, [i, i])
 
-    iterator = (
-        dataset_ops.Dataset.range(10).map(_sparse).map(_check)
-        .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(_sparse).map(_check))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -851,11 +843,10 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       else:
         return i
 
-    iterator = (
+    iterator = dataset_ops.make_initializable_iterator(
         dataset_ops.Dataset.range(105)
         .map(lambda x: script_ops.py_func(raising_py_func, [x], dtypes.int64),
-             num_parallel_calls=2)
-        .make_initializable_iterator())
+             num_parallel_calls=2))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -867,9 +858,8 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(get_next)
 
   def testConstantOutput(self):
-    iterator = (
-        dataset_ops.Dataset.range(10).map(lambda x: [x, "hello", 10])
-        .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).map(lambda x: [x, "hello", 10]))
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
@@ -940,7 +930,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       return const_tensor
 
     dataset = dataset.map(broken_function)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
 
     with self.cached_session() as sess:
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "BrokenConst"):
@@ -1005,7 +995,7 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     captured_t = array_ops.placeholder(dtypes.int64, shape=[])
     dataset = self.structuredDataset(None).repeat().map(
         lambda x: captured_t, num_parallel_calls=num_parallel_calls)
-    iterator = dataset.make_initializable_iterator()
+    iterator = dataset_ops.make_initializable_iterator(dataset)
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
diff --git a/tensorflow/python/data/kernel_tests/shuffle_test.py b/tensorflow/python/data/kernel_tests/shuffle_test.py
index ae892a7921..13df870938 100644
--- a/tensorflow/python/data/kernel_tests/shuffle_test.py
+++ b/tensorflow/python/data/kernel_tests/shuffle_test.py
@@ -130,9 +130,8 @@ class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
         sess.run(get_next)
 
     seed_placeholder = array_ops.placeholder(dtypes.int64, shape=[])
-    iterator = (
-        dataset_ops.Dataset.range(10).shuffle(10, seed=seed_placeholder)
-        .make_initializable_iterator())
+    iterator = dataset_ops.make_initializable_iterator(
+        dataset_ops.Dataset.range(10).shuffle(10, seed=seed_placeholder))
     get_next = iterator.get_next()
 
     with self.cached_session() as sess:
@@ -223,7 +222,8 @@ class ShuffleTest(test_base.DatasetTestBase, parameterized.TestCase):
           10, reshuffle_each_iteration=reshuffle).repeat(3)
 
       if initializable:
-        iterators = [dataset.make_initializable_iterator() for _ in range(2)]
+        iterators = [dataset_ops.make_initializable_iterator(dataset)
+                     for _ in range(2)]
       else:
         iterators = [dataset_ops.make_one_shot_iterator(dataset)
                      for _ in range(2)]
diff --git a/tensorflow/python/data/kernel_tests/test_base.py b/tensorflow/python/data/kernel_tests/test_base.py
index ca6ecdf198..85f6c9de23 100644
--- a/tensorflow/python/data/kernel_tests/test_base.py
+++ b/tensorflow/python/data/kernel_tests/test_base.py
@@ -62,7 +62,7 @@ class DatasetTestBase(test.TestCase):
       return iterator._next_internal  # pylint: disable=protected-access
     else:
       if requires_initialization:
-        iterator = dataset.make_initializable_iterator()
+        iterator = dataset_ops.make_initializable_iterator(dataset)
         self.evaluate(iterator.initializer)
       else:
         iterator = dataset_ops.make_one_shot_iterator(dataset)
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index c045299275..f6a5772bfd 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -135,51 +135,6 @@ class DatasetV2(object):
           options.experimental_stats.counter_prefix)
     return dataset
 
-  def make_initializable_iterator(self, shared_name=None):
-    """Creates an `Iterator` for enumerating the elements of this dataset.
-
-    Note: The returned iterator will be in an uninitialized state,
-    and you must run the `iterator.initializer` operation before using it:
-
-    ```python
-    dataset = ...
-    iterator = dataset.make_initializable_iterator()
-    # ...
-    sess.run(iterator.initializer)
-    ```
-
-    Args:
-      shared_name: (Optional.) If non-empty, the returned iterator will be
-        shared under the given name across multiple sessions that share the
-        same devices (e.g. when using a remote server).
-
-    Returns:
-      An `Iterator` over the elements of this dataset.
-
-    Raises:
-      RuntimeError: If eager execution is enabled.
-    """
-    if context.executing_eagerly():
-      raise RuntimeError(
-          "dataset.make_initializable_iterator is not supported when eager "
-          "execution is enabled.")
-    dataset = self._apply_options()
-    if shared_name is None:
-      shared_name = ""
-    if compat.forward_compatible(2018, 8, 3):
-      iterator_resource = gen_dataset_ops.iterator_v2(
-          container="", shared_name=shared_name, **flat_structure(self))
-    else:
-      iterator_resource = gen_dataset_ops.iterator(
-          container="", shared_name=shared_name, **flat_structure(self))
-    with ops.colocate_with(iterator_resource):
-      initializer = gen_dataset_ops.make_iterator(
-          dataset._as_variant_tensor(),  # pylint: disable=protected-access
-          iterator_resource)
-    return iterator_ops.Iterator(iterator_resource, initializer,
-                                 dataset.output_types, dataset.output_shapes,
-                                 dataset.output_classes)
-
   def __iter__(self):
     """Creates an `Iterator` for enumerating the elements of this dataset.
 
@@ -1352,6 +1307,56 @@ class DatasetV1(DatasetV2):
             dataset_factory=_make_dataset, **flat_structure(self)),
         None, self.output_types, self.output_shapes, self.output_classes)
 
+  @deprecation.deprecated(
+      None, "Use `for ... in dataset:` to iterate over a dataset. If using "
+      "`tf.estimator`, return the `Dataset` object directly from your input "
+      "function. As a last resort, you can use "
+      "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
+  def make_initializable_iterator(self, shared_name=None):
+    """Creates an `Iterator` for enumerating the elements of this dataset.
+
+    Note: The returned iterator will be in an uninitialized state,
+    and you must run the `iterator.initializer` operation before using it:
+
+    ```python
+    dataset = ...
+    iterator = dataset.make_initializable_iterator()
+    # ...
+    sess.run(iterator.initializer)
+    ```
+
+    Args:
+      shared_name: (Optional.) If non-empty, the returned iterator will be
+        shared under the given name across multiple sessions that share the
+        same devices (e.g. when using a remote server).
+
+    Returns:
+      An `Iterator` over the elements of this dataset.
+
+    Raises:
+      RuntimeError: If eager execution is enabled.
+    """
+    if context.executing_eagerly():
+      raise RuntimeError(
+          "dataset.make_initializable_iterator is not supported when eager "
+          "execution is enabled.")
+    dataset = self._apply_options()
+    if shared_name is None:
+      shared_name = ""
+    if compat.forward_compatible(2018, 8, 3):
+      iterator_resource = gen_dataset_ops.iterator_v2(
+          container="", shared_name=shared_name, **flat_structure(self))
+    else:
+      iterator_resource = gen_dataset_ops.iterator(
+          container="", shared_name=shared_name, **flat_structure(self))
+    with ops.colocate_with(iterator_resource):
+      initializer = gen_dataset_ops.make_iterator(
+          dataset._as_variant_tensor(),  # pylint: disable=protected-access
+          iterator_resource)
+    return iterator_ops.Iterator(iterator_resource, initializer,
+                                 dataset.output_types, dataset.output_shapes,
+                                 dataset.output_classes)
+
   @staticmethod
   @functools.wraps(DatasetV2.from_tensors)
   def from_tensors(tensors):
@@ -1563,9 +1568,6 @@ class DatasetV1Adapter(DatasetV1):
   def output_types(self):
     return self._dataset.output_types
 
-  def make_initializable_iterator(self, shared_name=None):
-    return self._dataset.make_initializable_iterator(shared_name)
-
   def __iter__(self):
     return iter(self._dataset)
 
@@ -1591,6 +1593,37 @@ def make_one_shot_iterator(dataset):
     return DatasetV1Adapter(dataset).make_one_shot_iterator()
 
 
+@tf_export(v1=["data.make_initializable_iterator"])
+def make_initializable_iterator(dataset):
+  """Creates a `tf.data.Iterator` for enumerating the elements of a dataset.
+
+  Note: The returned iterator will be in an uninitialized state,
+  and you must run the `iterator.initializer` operation before using it:
+
+  ```python
+  dataset = ...
+  iterator = dataset.make_initializable_iterator()
+  # ...
+  sess.run(iterator.initializer)
+  ```
+
+  Args:
+    dataset: A `tf.data.Dataset`.
+
+  Returns:
+    A `tf.data.Iterator` over the elements of `dataset`.
+
+  Raises:
+    RuntimeError: If eager execution is enabled.
+  """
+  try:
+    # Call the defined `make_one_shot_iterator()` if there is one, because some
+    # datasets (e.g. for prefetching) override its behavior.
+    return dataset.make_initializable_iterator()
+  except AttributeError:
+    return DatasetV1Adapter(dataset).make_initializable_iterator()
+
+
 @tf_export("data.Options")
 class Options(options_lib.OptionsBase):
   """Represents options for tf.data.Dataset.
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 8687d1d945..01a1680a24 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -1186,7 +1186,7 @@ class PerReplicaDataset(object):
       dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
           self._dataset, self._devices)
     else:
-      dataset_iterator = self._dataset.make_initializable_iterator()
+      dataset_iterator = dataset_ops.make_initializable_iterator(self._dataset)
     return PerReplicaDataIterator(
         dataset_iterator,
         self._devices,
@@ -1311,7 +1311,8 @@ class MultiWorkerDataset(object):
     iterators = []
     for worker, dataset in self._datasets:
       with ops.device(worker):
-        iterators.append((worker, dataset.make_initializable_iterator()))
+        iterators.append(
+            (worker, dataset_ops.make_initializable_iterator(dataset)))
     return MultiWorkerDataIterator(iterators, self._worker_device_pairs)
 
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 38a3928a38..1416f5960f 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1156,7 +1156,7 @@ class Model(Network):
         if x in self._dataset_iterator_cache:
           x = self._dataset_iterator_cache[x]
         else:
-          iterator = x.make_initializable_iterator()
+          iterator = dataset_ops.make_initializable_iterator(x)
           self._dataset_iterator_cache[x] = iterator
           x = iterator
         K.get_session().run(x.initializer)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
index bb14216e60..aa47468059 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.pbtxt
@@ -28,6 +28,10 @@ tf_module {
     name: "experimental"
     mtype: "<type \'module\'>"
   }
+  member_method {
+    name: "make_initializable_iterator"
+    argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "make_one_shot_iterator"
     argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
index 760d56aa06..ac8dd2de7f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
@@ -61,10 +61,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
index c46e276664..f157351243 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-fixed-length-record-dataset.pbtxt
@@ -64,10 +64,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
index 5a19f474fa..690da98b1a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-t-f-record-dataset.pbtxt
@@ -63,10 +63,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
index 71be8ed677..fe0bc1a4db 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-text-line-dataset.pbtxt
@@ -64,10 +64,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
index 823c5c4d18..261129b132 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-csv-dataset.pbtxt
@@ -64,10 +64,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
index 28050c2758..0b34bbc942 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-random-dataset.pbtxt
@@ -64,10 +64,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
index d707ac7320..0e61890eee 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sql-dataset.pbtxt
@@ -64,10 +64,6 @@ tf_class {
     name: "list_files"
     argspec: "args=[\'file_pattern\', \'shuffle\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "make_initializable_iterator"
-    argspec: "args=[\'self\', \'shared_name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "map"
     argspec: "args=[\'self\', \'map_func\', \'num_parallel_calls\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 696cf7c41f..b8d57d5e11 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -108,6 +108,7 @@ renames = {
     'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
     'tf.cumprod': 'tf.math.cumprod',
+    'tf.data.make_initializable_iterator': 'tf.compat.v1.data.make_initializable_iterator',
     'tf.data.make_one_shot_iterator': 'tf.compat.v1.data.make_one_shot_iterator',
     'tf.debugging.is_finite': 'tf.math.is_finite',
     'tf.debugging.is_inf': 'tf.math.is_inf',
-- 
GitLab


From 8bf493d03d8cf3ca4618e1ba925a95a71bdc9b35 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 3 Dec 2018 20:44:45 -0800
Subject: [PATCH 1260/1554] Automated rollback of commit
 85e68a20257cdaf176d467fc8a91ba0f1e55462e

PiperOrigin-RevId: 223912775
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 55c3dd564a..7e5f84be16 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "e64ffc34ce6ba78946a6ea828c0ad96ac5bd162ec71c30f701c5cb3ebf8418a7",
-        strip_prefix = "abseil-cpp-926bfeb9fff223429c12224b7514243886323e8d",
+        sha256 = "3cf6132129ba87f0781c383bfaf381b7174b5818e81fffcc5d04bb451154f0f2",
+        strip_prefix = "abseil-cpp-f95179062eb65ce40895cc76f1398cce25394369",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/926bfeb9fff223429c12224b7514243886323e8d.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/926bfeb9fff223429c12224b7514243886323e8d.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
         ],
     )
 
-- 
GitLab


From 6eee77214daee11f3188cfa0c6a20bed249a609d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 21:23:40 -0800
Subject: [PATCH 1261/1554] [XLA] separate out an Execute from
 ExecutePerReplica [JAX] reduce the creation of XLA Shape protos on every call
 (which is slow)

PiperOrigin-RevId: 223915944
---
 .../xla/python/local_computation_builder.cc   | 45 ++++++++++++++++++-
 .../xla/python/local_computation_builder.h    |  8 +++-
 .../xla/python/local_computation_builder.i    |  1 +
 tensorflow/compiler/xla/python/xla_client.py  | 13 ++++--
 4 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index c1381779c9..6e2ee86632 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -314,7 +314,50 @@ CompiledLocalComputation::CompiledLocalComputation(
     std::unique_ptr<LocalExecutable> executable)
     : executable_(std::move(executable)) {}
 
-StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::Execute(
+StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
+    absl::Span<LocalShapedBuffer* const> argument_handles) {
+  LocalClient* client = GetOrCreateLocalClient();
+  StatusOr<int> device_ordinal_status = client->ReplicaNumberToDeviceOrdinal(0);
+  StatusOr<ScopedShapedBuffer> result_buffer_status;
+  if (!device_ordinal_status.ok()) {
+    result_buffer_status = device_ordinal_status.status();
+  } else {
+    const int device_ordinal = device_ordinal_status.ValueOrDie();
+    VLOG(3) << "Replica 0 mapped to device ordinal for execution: "
+            << device_ordinal;
+
+    std::vector<const ShapedBuffer*> argument_buffers;
+    argument_buffers.reserve(argument_handles.size());
+    for (auto& handle : argument_handles) {
+      argument_buffers.push_back(handle->shaped_buffer());
+    }
+
+    DeviceAssignment device_assignment =
+        client->backend()
+            .computation_placer()
+            ->AssignDevices(1, /*computation_count=*/1)
+            .ConsumeValueOrDie();
+
+    ExecutableRunOptions options;
+    options.set_device_ordinal(device_ordinal);
+    options.set_allocator(client->backend().memory_allocator());
+    options.set_intra_op_thread_pool(
+        client->backend().eigen_intra_op_thread_pool_device());
+    options.set_device_assignment(&device_assignment);
+
+    result_buffer_status = executable_->Run(argument_buffers, options);
+  }
+
+  if (!result_buffer_status.ok()) {
+    return InternalError(
+        "Failed running replica 0 (other replicas may have failed as well): "
+        "%s.",
+        result_buffer_status.status().ToString());
+  }
+  return new LocalShapedBuffer(std::move(result_buffer_status).ValueOrDie());
+}
+
+StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::ExecutePerReplica(
     absl::Span<const std::vector<LocalShapedBuffer*>> argument_handles) {
   LocalClient* client = GetOrCreateLocalClient();
   const int num_replicas = GetReplicaCount();
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 8247b05775..149e44570d 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -173,7 +173,13 @@ class CompiledLocalComputation {
  public:
   CompiledLocalComputation(std::unique_ptr<LocalExecutable> executable);
 
-  StatusOr<LocalShapedBufferTuple*> Execute(
+  StatusOr<LocalShapedBuffer*> Execute(
+      absl::Span<LocalShapedBuffer* const> argument_handles);
+
+  // Execute on many replicas. Takes a sequence of argument lists (one argument
+  // list per replica) and returns a tuple of results (one result per replica).
+  // The number of argument lists must be equal to the replica count.
+  StatusOr<LocalShapedBufferTuple*> ExecutePerReplica(
       absl::Span<const std::vector<LocalShapedBuffer*> > argument_handles);
 
  private:
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 88349b788e..d23d693c1e 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -1029,6 +1029,7 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::XrtAllocationTuple::size;
 %unignore xla::swig::CompiledLocalComputation;
 %unignore xla::swig::CompiledLocalComputation::Execute;
+%unignore xla::swig::CompiledLocalComputation::ExecutePerReplica;
 %unignore xla::swig::CompiledXrtComputation;
 %unignore xla::swig::CompiledXrtComputation::Execute;
 %unignore xla::swig::LocalComputation;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 4dbbeebd0b..c91a2aaf56 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -588,9 +588,16 @@ class LocalComputation(object):
         compile_options=compile_options,
         layout_fn=layout_fn)
 
-  def Execute(self, arguments=()):
+  def GetReturnValueShape(self):
+    return _wrap_shape(self._c_computation.GetReturnValueShape())
+
+  def Execute(self, arguments=(), check_for_deleted_args=True):
     """Execute on one replica with LocalBuffer arguments and return value."""
-    return self.ExecutePerReplica([arguments])[0]
+    if check_for_deleted_args and any(arg.is_deleted() for arg in arguments):
+      raise ValueError('Executing with deleted local buffer argument')
+    raw_args = [arg.c_buffer for arg in arguments]
+    output_buffer = self._c_computation.Execute(raw_args)
+    return LocalBuffer(output_buffer, backend=self._backend, replica=0)
 
   def ExecutePerReplica(self, arguments=None):
     """Execute on many replicas with LocalBuffer arguments and return value.
@@ -633,7 +640,7 @@ class LocalComputation(object):
             'Multi-replica execution is not yet supported via the XRT backend.')
       output_buffers = [self._c_computation.Execute(stripped_args[0])]
     else:
-      output_buffer_tup = self._c_computation.Execute(stripped_args)
+      output_buffer_tup = self._c_computation.ExecutePerReplica(stripped_args)
       size = output_buffer_tup.size()
       output_buffers = [output_buffer_tup.Release(i) for i in xrange(size)]
 
-- 
GitLab


From 7018e50143526de7a58703cb0b25babbc9bd48db Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 3 Dec 2018 21:42:44 -0800
Subject: [PATCH 1262/1554] Add MirrorPad op.

PiperOrigin-RevId: 223917407
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/c/c_api_internal.c            |   2 +-
 tensorflow/lite/c/c_api_internal.h            |   2 +-
 tensorflow/lite/kernels/BUILD                 |  12 +
 .../lite/kernels/internal/tensor_ctypes.h     |   5 +
 tensorflow/lite/kernels/mirror_pad.cc         | 374 ++++++++++++++++++
 tensorflow/lite/kernels/mirror_pad_test.cc    | 189 +++++++++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/testing/generate_examples.py  |  82 ++++
 .../propagate_fixed_sizes.cc                  |  48 +++
 tensorflow/lite/toco/import_tensorflow.cc     |  26 ++
 tensorflow/lite/toco/model.h                  |  20 +-
 tensorflow/lite/toco/tflite/operator.cc       |  25 ++
 tensorflow/lite/toco/tflite/operator_test.cc  |   8 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 15 files changed, 794 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/kernels/mirror_pad.cc
 create mode 100644 tensorflow/lite/kernels/mirror_pad_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 8507521694..5eaf719494 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -265,6 +265,7 @@ def generated_test_models():
         "maximum",
         "mean",
         "minimum",
+        "mirror_pad",
         "mul",
         "neg",
         "not_equal",
diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c
index 598d74be84..2923dbad4e 100644
--- a/tensorflow/lite/c/c_api_internal.c
+++ b/tensorflow/lite/c/c_api_internal.c
@@ -59,7 +59,7 @@ void TfLiteIntArrayPrint(const char* s, TfLiteIntArray* a) {
   printf("]\n");
 }
 
-TfLiteIntArray* TfLiteIntArrayCopy(TfLiteIntArray* src) {
+TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src) {
   if (!src) return NULL;
   TfLiteIntArray* ret = TfLiteIntArrayCreate(src->size);
   if (ret) {
diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index 5cfa64cc7b..1cd84eff5c 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -96,7 +96,7 @@ int TfLiteIntArrayEqualsArray(TfLiteIntArray* a, int b_size, int b_data[]);
 
 // Create a copy of an array passed as `src`.
 // You are expected to free memory with TfLiteIntArrayFree
-TfLiteIntArray* TfLiteIntArrayCopy(TfLiteIntArray* src);
+TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src);
 
 // Free memory of array `v`.
 void TfLiteIntArrayFree(TfLiteIntArray* v);
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 8d5c3b406f..00d9d1feae 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -198,6 +198,7 @@ cc_library(
         "lstm.cc",
         "maximum_minimum.cc",
         "mfcc.cc",
+        "mirror_pad.cc",
         "mul.cc",
         "neg.cc",
         "one_hot.cc",
@@ -1437,3 +1438,14 @@ filegroup(
 )
 
 tflite_portable_test_suite()
+
+tf_cc_test(
+    name = "mirror_pad_test",
+    srcs = ["mirror_pad_test.cc"],
+    deps = [
+        ":builtin_ops",
+        ":test_util",
+        "//tensorflow/lite:framework",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/kernels/internal/tensor_ctypes.h b/tensorflow/lite/kernels/internal/tensor_ctypes.h
index b4822d5701..4a94b703f8 100644
--- a/tensorflow/lite/kernels/internal/tensor_ctypes.h
+++ b/tensorflow/lite/kernels/internal/tensor_ctypes.h
@@ -53,6 +53,11 @@ inline bool* GetTensorData(TfLiteTensor* tensor) {
   return tensor != nullptr ? tensor->data.b : nullptr;
 }
 
+template <>
+inline int8_t* GetTensorData(TfLiteTensor* tensor) {
+  return tensor != nullptr ? tensor->data.int8 : nullptr;
+}
+
 template <typename T>
 inline const T* GetTensorData(const TfLiteTensor* tensor);
 
diff --git a/tensorflow/lite/kernels/mirror_pad.cc b/tensorflow/lite/kernels/mirror_pad.cc
new file mode 100644
index 0000000000..e74e47f7a3
--- /dev/null
+++ b/tensorflow/lite/kernels/mirror_pad.cc
@@ -0,0 +1,374 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/quantization_util.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace mirror_pad {
+namespace {
+
+// Simple class that represents a mirror padded tensor - which is the output
+// from the Op.
+struct PaddedTensor {
+  // If not null that means this is a scalar value.
+  // Note: This is not owned by default. It will point to the value
+  // in the input tensor.
+  const void* value = nullptr;
+  // If this tensor is not one value, then this vector will have
+  // all the tensors that belongs to this tensor.
+  // Pointers are owned.
+  std::vector<std::unique_ptr<PaddedTensor>> values;
+  // Pointers to PaddedTensors that are padded on the left of the current
+  // tensor.
+  std::vector<PaddedTensor*> left_pad_ptrs;
+  // Pointers to PaddedTensors that are padded on the right of the current
+  // tensor.
+  std::vector<PaddedTensor*> right_pad_ptrs;
+
+  // Returns mutable pointer to the tensor identified by 'indices'.
+  PaddedTensor* GetMutable(const std::vector<int>& indices) {
+    auto* result = this;
+    for (int i = 0; i < indices.size(); ++i) {
+      if (indices[i] >= result->values.size()) {
+        return nullptr;
+      }
+      result = result->values[indices[i]].get();
+      if (result == nullptr) break;
+    }
+    return result;
+  }
+};
+
+// Util method to initialize the memory of the padded tensor.
+void InitializeTensorMemory(const TfLiteIntArray* const dims, int dim_index,
+                            int dims_size, PaddedTensor* padded_tensor) {
+  if (dim_index >= dims_size) {
+    return;
+  }
+  padded_tensor->values.reserve(dims->data[dim_index]);
+  for (int i = 0; i < dims->data[dim_index]; ++i) {
+    padded_tensor->values.emplace_back(new PaddedTensor());
+    InitializeTensorMemory(dims, dim_index + 1, dims_size,
+                           padded_tensor->values.back().get());
+  }
+}
+
+// Returns pointer to the value at the specified index in 'data'.
+inline const void* GetValuePointerAtIndex(const void* data, int index,
+                                          const TfLiteType data_type) {
+  switch (data_type) {
+    case kTfLiteFloat32:
+      return static_cast<const float*>(data) + index;
+    case kTfLiteInt32:
+      return static_cast<const int32_t*>(data) + index;
+    case kTfLiteUInt8:
+      return static_cast<const uint8_t*>(data) + index;
+    case kTfLiteInt64:
+      return static_cast<const int64_t*>(data) + index;
+    case kTfLiteBool:
+      return static_cast<const bool*>(data) + index;
+    case kTfLiteInt16:
+      return static_cast<const int16_t*>(data) + index;
+    case kTfLiteInt8:
+      return static_cast<const int8_t*>(data) + index;
+    // Unsupported types ?
+    default:
+      return nullptr;
+  }
+  return nullptr;
+}
+
+// Util method that increment index in the N-d array.
+void IncrementTensorIndex(const TfLiteIntArray* dims,
+                          std::vector<int>* tensor_index_ptr) {
+  int dimension_index = dims->size - 1;
+  auto& tensor_index = *tensor_index_ptr;
+  tensor_index[dimension_index]++;
+  while (dimension_index >= 0 &&
+         tensor_index[dimension_index] == dims->data[dimension_index]) {
+    tensor_index[dimension_index] = 0;
+    dimension_index--;
+    if (dimension_index >= 0) tensor_index[dimension_index]++;
+  }
+}
+
+// Fills the 'padded_tensor' with data from 'input_tensor'.
+TfLiteStatus InitFromInputTensor(const TfLiteTensor* input_tensor,
+                                 PaddedTensor* padded_tensor) {
+  const auto* dims = input_tensor->dims;
+  const auto data_type = input_tensor->type;
+  const void* data = static_cast<const void*>(input_tensor->data.raw_const);
+  // Either invalid input or unsupported type.+
+  if (data == nullptr) {
+    return kTfLiteError;
+  }
+  // Index of current processing tensor.
+  std::vector<int> tensor_index(dims->size, 0);
+  int flat_index = 0;
+  const int num_elements = NumElements(input_tensor);
+  while (flat_index < num_elements) {
+    auto* tensor = padded_tensor->GetMutable(tensor_index);
+    if (tensor == nullptr) {
+      return kTfLiteError;
+    }
+    tensor->value = GetValuePointerAtIndex(data, flat_index, data_type);
+    IncrementTensorIndex(dims, &tensor_index);
+    ++flat_index;
+  }
+
+  return kTfLiteOk;
+}
+
+template <typename T>
+inline void GetPadding(const T* data, int offset, int64_t* left_pad,
+                       int64_t* right_pad) {
+  *left_pad = static_cast<int64_t>(*(data + offset * 2));
+  *right_pad = static_cast<int64_t>(*(data + offset * 2 + 1));
+}
+
+inline TfLiteStatus GetPadding(const TfLiteTensor* padding_matrix,
+                               int dimension, int64_t* left_pad,
+                               int64_t* right_pad) {
+  switch (padding_matrix->type) {
+    case kTfLiteInt32:
+      GetPadding(padding_matrix->data.i32, dimension, left_pad, right_pad);
+      break;
+    case kTfLiteInt64:
+      GetPadding(padding_matrix->data.i64, dimension, left_pad, right_pad);
+      break;
+    default:
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus ValidateTensor(const TfLiteTensor* padding_matrix, int offset,
+                            int dimension_index, PaddedTensor* padded_tensor,
+                            TfLiteContext* context) {
+  if (dimension_index >= padding_matrix->dims->data[0]) {
+    return kTfLiteOk;
+  }
+
+  int64_t left_pad = 0, right_pad = 0;
+  TF_LITE_ENSURE_STATUS(
+      GetPadding(padding_matrix, dimension_index, &left_pad, &right_pad));
+  // If we are not going to include border we must have enough values
+  // to use.
+  if (left_pad + offset > padded_tensor->values.size()) {
+    context->ReportError(
+        context, "Not enough values for Mirror Pad, required %d, available %d.",
+        left_pad + offset, padded_tensor->values.size());
+    return kTfLiteError;
+  }
+  if (right_pad + offset > padded_tensor->values.size()) {
+    context->ReportError(
+        context, "Not enough values for Mirror Pad, required %d, available %d.",
+        right_pad + offset, padded_tensor->values.size());
+    return kTfLiteError;
+  }
+  if (!padded_tensor->values.empty()) {
+    ValidateTensor(padding_matrix, offset, dimension_index + 1,
+                   padded_tensor->values[0].get(), context);
+  }
+  return kTfLiteOk;
+}
+
+// Fills 'padded_tensor' with the padding information based on
+// 'padding_matrix'.
+// 'dimension_index' represents which dimension the function is operating on.
+TfLiteStatus PadTensor(const TfLiteTensor* padding_matrix, int offset,
+                       int dimension_index, PaddedTensor* padded_tensor,
+                       TfLiteContext* context) {
+  if (dimension_index >= padding_matrix->dims->data[0]) return kTfLiteOk;
+
+  int64_t left_pad = 0, right_pad = 0;
+  TF_LITE_ENSURE_STATUS(
+      GetPadding(padding_matrix, dimension_index, &left_pad, &right_pad));
+
+  for (int i = left_pad + offset - 1; i >= offset && left_pad > 0;
+       --i, --left_pad) {
+    padded_tensor->left_pad_ptrs.push_back(padded_tensor->values[i].get());
+  }
+  for (int i = padded_tensor->values.size() - (1 + offset);
+       i >= 0 && right_pad > 0; --i, --right_pad) {
+    padded_tensor->right_pad_ptrs.push_back(padded_tensor->values[i].get());
+  }
+
+  for (auto& tensor : padded_tensor->values) {
+    TF_LITE_ENSURE_STATUS(PadTensor(padding_matrix, offset, dimension_index + 1,
+                                    tensor.get(), context));
+  }
+  return kTfLiteOk;
+}
+
+// Fills 'output_data' with data from 'padded_tensor'.
+// The function does this recursively by setting left padding first then
+// original data, followed by the right padding.
+template <typename T>
+int FillOutput(const PaddedTensor* padded_tensor, T* output_data,
+               int index_in_output) {
+  if (padded_tensor == nullptr || output_data == nullptr) {
+    return -1;
+  }
+  if (padded_tensor->value != nullptr) {
+    output_data[index_in_output] = *static_cast<const T*>(padded_tensor->value);
+    return index_in_output + 1;
+  }
+  for (const auto* tensor : padded_tensor->left_pad_ptrs) {
+    index_in_output = FillOutput(tensor, output_data, index_in_output);
+  }
+  for (const auto& tensor : padded_tensor->values) {
+    index_in_output = FillOutput(tensor.get(), output_data, index_in_output);
+  }
+  for (const auto* tensor : padded_tensor->right_pad_ptrs) {
+    index_in_output = FillOutput(tensor, output_data, index_in_output);
+  }
+  return index_in_output;
+}
+
+// Returns the shape of the final output after padding.
+std::unique_ptr<TfLiteIntArray, void (*)(TfLiteIntArray*)> GetPaddedOutputShape(
+    const TfLiteTensor* input, const TfLiteTensor* padding_matrix) {
+  const int input_dims = NumDimensions(input);
+  std::unique_ptr<TfLiteIntArray, void (*)(TfLiteIntArray*)> shape(
+      TfLiteIntArrayCreate(input_dims), TfLiteIntArrayFree);
+
+  int64_t left_pad = 0, right_pad = 0;
+  for (int i = 0; i < input_dims; ++i) {
+    GetPadding(padding_matrix, i, &left_pad, &right_pad);
+    shape->data[i] = SizeOfDimension(input, i) + left_pad + right_pad;
+  }
+  return shape;
+}
+
+}  // namespace
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input_tensor = GetInput(context, node, 0);
+  const TfLiteTensor* padding_matrix = GetInput(context, node, 1);
+  auto* params =
+      reinterpret_cast<TfLiteMirrorPaddingParams*>(node->builtin_data);
+
+  if (params == nullptr) {
+    return kTfLiteError;
+  }
+  const int input_dims = NumDimensions(input_tensor);
+
+  TfLiteTensor* output_tensor = GetOutput(context, node, 0);
+  if (IsDynamicTensor(output_tensor)) {
+    auto output_size = GetPaddedOutputShape(input_tensor, padding_matrix);
+    if (output_size == nullptr) {
+      return kTfLiteError;
+    }
+    TF_LITE_ENSURE_STATUS(
+        context->ResizeTensor(context, output_tensor, output_size.release()));
+  }
+
+  PaddedTensor padded_tensor;
+  // Initialize memory.
+  InitializeTensorMemory(input_tensor->dims, 0, input_dims, &padded_tensor);
+  // Set the values from the input_tensor.
+  TF_LITE_ENSURE_STATUS(InitFromInputTensor(input_tensor, &padded_tensor));
+
+  const int offset =
+      params->mode != TfLiteMirrorPaddingMode::kTfLiteMirrorPaddingReflect ? 0
+                                                                           : 1;
+  // Make sure padding values are sufficient and valid to use.
+  TF_LITE_ENSURE_STATUS(
+      ValidateTensor(padding_matrix, offset, 0, &padded_tensor, context));
+  // Apply padding.
+  TF_LITE_ENSURE_STATUS(
+      PadTensor(padding_matrix, offset, 0, &padded_tensor, context));
+
+  // Fill the output tensor from the padded tensor.
+  TfLiteStatus status = kTfLiteOk;
+
+#define TF_LITE_MIRROR_PAD(type) \
+  FillOutput(&padded_tensor, GetTensorData<type>(output_tensor), 0);
+
+  switch (output_tensor->type) {
+    case kTfLiteFloat32: {
+      TF_LITE_MIRROR_PAD(float);
+      break;
+    }
+    case kTfLiteInt32: {
+      TF_LITE_MIRROR_PAD(int32_t);
+      break;
+    }
+    case kTfLiteUInt8: {
+      TF_LITE_MIRROR_PAD(uint8_t);
+      break;
+    }
+    case kTfLiteInt64: {
+      TF_LITE_MIRROR_PAD(int64_t);
+      break;
+    }
+    default:
+      status = kTfLiteError;
+      break;
+  }
+#undef TF_LITE_MIRROR_PAD
+  return status;
+}
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  return nullptr;
+}
+
+void Free(TfLiteContext* context, void* buffer) {}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input_tensor = GetInput(context, node, 0);
+  const TfLiteTensor* padding_matrix = GetInput(context, node, 1);
+  TfLiteTensor* output_tensor = GetOutput(context, node, 0);
+
+  TF_LITE_ENSURE_EQ(context, NumDimensions(padding_matrix), 2);
+  TF_LITE_ENSURE_EQ(context, SizeOfDimension(padding_matrix, 0),
+                    NumDimensions(input_tensor));
+
+  if (!IsConstantTensor(padding_matrix)) {
+    SetTensorToDynamic(output_tensor);
+    return kTfLiteOk;
+  }
+  // We have constant padding, so we can infer output size.
+
+  auto output_size = GetPaddedOutputShape(input_tensor, padding_matrix);
+  if (output_size == nullptr) {
+    return kTfLiteError;
+  }
+  return context->ResizeTensor(context, output_tensor, output_size.release());
+}
+
+}  // namespace mirror_pad
+TfLiteRegistration* Register_MIRROR_PAD() {
+  static TfLiteRegistration r = {mirror_pad::Init, mirror_pad::Free,
+                                 mirror_pad::Prepare, mirror_pad::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/mirror_pad_test.cc b/tensorflow/lite/kernels/mirror_pad_test.cc
new file mode 100644
index 0000000000..fd09e6e449
--- /dev/null
+++ b/tensorflow/lite/kernels/mirror_pad_test.cc
@@ -0,0 +1,189 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+template <typename T>
+class BaseMirrorPadOpModel : public SingleOpModel {
+ public:
+  BaseMirrorPadOpModel(const TensorData& input,
+                       const TensorData& padding_matrix,
+                       const TensorData& output,
+                       const tflite::MirrorPadMode mode) {
+    input_id_ = AddInput(input);
+    padding_matrix_id_ = AddInput(padding_matrix);
+    output_id_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_MIRROR_PAD, BuiltinOptions_MirrorPadOptions,
+                 CreateMirrorPadOptions(builder_, mode).Union());
+    BuildInterpreter({GetShape(input_id_), GetShape(padding_matrix_id_)});
+  }
+
+  int input_tensor_id() { return input_id_; }
+  int padding_matrix_tensor_id() { return padding_matrix_id_; }
+
+  std::vector<T> GetOutput() { return ExtractVector<T>(output_id_); }
+
+ protected:
+  int input_id_;
+  int padding_matrix_id_;
+  int output_id_;
+};
+
+TEST(MirrorPadTest, EmptyPad) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {0, 0, 0, 0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
+}
+
+TEST(MirrorPadTest, PadOneSide_right_Reflect) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {0, 1, 0, 1});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 2, 3, 2, 4, 5, 6, 5, 1, 2, 3, 2}));
+}
+
+TEST(MirrorPadTest, PadOneSide_left_Reflect) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {1, 0, 1, 0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({5, 4, 5, 6, 2, 1, 2, 3, 5, 4, 5, 6}));
+}
+
+TEST(MirrorPadTest, PadOneSide_right_Symmetric) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {0, 1, 0, 1});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 2, 3, 3, 4, 5, 6, 6, 4, 5, 6, 6}));
+}
+
+TEST(MirrorPadTest, PadOneSide_left_Symmetric) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {1, 0, 1, 0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 1, 2, 3, 1, 1, 2, 3, 4, 4, 5, 6}));
+}
+
+TEST(MirrorPadTest, PadBothSides_Symmetric) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {1, 1, 1, 1});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 1, 2, 3, 3, 1, 1, 2, 3, 3,
+                                4, 4, 5, 6, 6, 4, 4, 5, 6, 6}));
+}
+
+TEST(MirrorPadTest, PadBothSides_Reflect) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {1, 1, 1, 1});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({5, 4, 5, 6, 5, 2, 1, 2, 3, 2,
+                                5, 4, 5, 6, 5, 2, 1, 2, 3, 2}));
+}
+
+TEST(MirrorPadTest, PadBothSides_Symmetric_Whole) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {2, 2, 3, 3});
+  model.Invoke();
+  EXPECT_THAT(
+      model.GetOutput(),
+      ElementsAreArray({6, 5, 4, 4, 5, 6, 6, 5, 4, 3, 2, 1, 1, 2, 3, 3, 2, 1,
+                        3, 2, 1, 1, 2, 3, 3, 2, 1, 6, 5, 4, 4, 5, 6, 6, 5, 4,
+                        6, 5, 4, 4, 5, 6, 6, 5, 4, 3, 2, 1, 1, 2, 3, 3, 2, 1}));
+}
+
+TEST(MirrorPadTest, PadBothSides_Reflect_Whole) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {1, 1, 2, 2});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({6, 5, 4, 5, 6, 5, 4, 3, 2, 1, 2, 3, 2, 1,
+                                6, 5, 4, 5, 6, 5, 4, 3, 2, 1, 2, 3, 2, 1}));
+}
+
+TEST(MirrorPadTest, Pad_Symmetric) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {2, 3}}, {TensorType_INT32, {2, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3, 4, 5, 6});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {1, 1, 2, 2});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({2, 1, 1, 2, 3, 3, 2, 2, 1, 1, 2, 3, 3, 2,
+                                5, 4, 4, 5, 6, 6, 5, 5, 4, 4, 5, 6, 6, 5}));
+}
+
+TEST(MirrorPadTest, Pad_1D_Reflect) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {3}}, {TensorType_INT32, {1, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_REFLECT);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {0, 2});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 2, 1}));
+}
+
+TEST(MirrorPadTest, Pad_1D_Symmetric) {
+  BaseMirrorPadOpModel<int> model(
+      {TensorType_INT32, {3}}, {TensorType_INT32, {1, 2}},
+      {TensorType_INT32, {}}, tflite::MirrorPadMode_SYMMETRIC);
+  model.PopulateTensor<int>(model.input_tensor_id(), {1, 2, 3});
+  model.PopulateTensor<int>(model.padding_matrix_tensor_id(), {0, 2});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 3, 2}));
+}
+
+}  // namespace
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 0ad90a9753..3c60d281b3 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -128,6 +128,7 @@ TfLiteRegistration* Register_RANGE();
 TfLiteRegistration* Register_LEAKY_RELU();
 TfLiteRegistration* Register_SQUARED_DIFFERENCE();
 TfLiteRegistration* Register_FILL();
+TfLiteRegistration* Register_MIRROR_PAD();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -266,6 +267,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LEAKY_RELU, Register_LEAKY_RELU());
   AddBuiltin(BuiltinOperator_SQUARED_DIFFERENCE, Register_SQUARED_DIFFERENCE());
   AddBuiltin(BuiltinOperator_FILL, Register_FILL());
+  AddBuiltin(BuiltinOperator_MIRROR_PAD, Register_MIRROR_PAD());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 3448c58ed2..5c90410a19 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3613,6 +3613,88 @@ def make_logical_xor_tests(zip_path):
   return _make_logical_tests(tf.logical_xor)(zip_path)
 
 
+def make_mirror_pad_tests(zip_path):
+  """Make a set of tests to do mirror_pad."""
+
+  test_parameters = [
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [2, 1]]],
+          "mode": ["REFLECT"],
+          "type": ["const"]
+      },
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [1, 1]]],
+          "mode": ["REFLECT"],
+          "type": ["const"]
+      },
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [2, 1]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["placeholder"]
+      },
+      {
+          "input_shape": [[2, 3]],
+          "padding_matrix": [[[1, 1], [2, 1]]],
+          "mode": ["REFLECT"],
+          "type": ["placeholder"]
+      },
+      {
+          "input_shape": [[3]],
+          "padding_matrix": [[[0, 2]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["placeholder"]
+      },
+      {
+          "input_shape": [[3]],
+          "padding_matrix": [[[0, 2]]],
+          "mode": ["SYMMETRIC"],
+          "type": ["const"]
+      },
+      {
+          "input_shape": [[3]],
+          "padding_matrix": [[[0, 2]]],
+          "mode": ["REFLECT"],
+          "type": ["const"]
+      },
+  ]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.placeholder(
+        dtype=tf.int32, name="input", shape=parameters["input_shape"])
+    if parameters["type"] != "const":
+      padding_matrix = tf.placeholder(
+          dtype=tf.int32,
+          name="padding",
+          shape=[len(parameters["input_shape"]), 2])
+      input_tensors = [input_tensor, padding_matrix]
+    else:
+      padding_matrix = tf.constant(np.array(parameters["padding_matrix"]))
+      input_tensors = [input_tensor]
+    output = tf.pad(
+        input_tensor, paddings=padding_matrix, mode=parameters["mode"])
+
+    return input_tensors, [output]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [create_tensor_data(tf.int32, parameters["input_shape"])]
+    if parameters["type"] != "const":
+      input_values.append(np.array(parameters["padding_matrix"]))
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_success=7)
+
+
 def make_unroll_batch_matmul_tests(zip_path):
   """Make a set of tests to test unroll_batch_matmul."""
 
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index fc2ed07aa0..0e653f08a0 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1783,6 +1783,51 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) {
   }
 }
 
+void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
+  CHECK_EQ(op->inputs.size(), 2);
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  const auto& padding_matrix = model->GetArray(op->inputs[1]);
+
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+
+  auto& output_array = model->GetArray(op->outputs[0]);
+  // If output already computed or padding matrix is non
+  // const then return.
+  if (output_array.has_shape() ||
+      !IsConstantParameterArray(*model, op->inputs[1])) {
+    return;
+  }
+  Shape output_shape = input_array.shape();
+  std::vector<int>& dims = *output_shape.mutable_dims();
+
+  std::vector<int64_t> padding;
+  if (padding_matrix.data_type == ArrayDataType::kInt32) {
+    const auto& data = padding_matrix.GetBuffer<ArrayDataType::kInt32>().data;
+    for (auto elem : data) {
+      padding.push_back(static_cast<int64_t>(elem));
+    }
+  } else if (padding_matrix.data_type == ArrayDataType::kInt64) {
+    const auto& data = padding_matrix.GetBuffer<ArrayDataType::kInt64>().data;
+    for (auto elem : data) {
+      padding.push_back(elem);
+    }
+  } else {
+    CHECK(padding_matrix.data_type == ArrayDataType::kInt64 ||
+          padding_matrix.data_type == ArrayDataType::kInt32);
+  }
+  CHECK_EQ(padding_matrix.shape().dimensions_count(), 2);
+  CHECK_EQ(input_array.shape().dimensions_count(),
+           padding_matrix.shape().dims(0));
+  for (int i = 0; i < input_array.shape().dimensions_count(); ++i) {
+    dims[i] += padding[i * 2] + padding[i * 2 + 1];
+  }
+
+  output_array.copy_shape(output_shape);
+}
+
 }  // namespace
 
 ::tensorflow::Status PropagateFixedSizes::Run(Model* model,
@@ -2055,6 +2100,9 @@ void ProcessUnpackOperator(Model* model, UnpackOperator* op) {
     case OperatorType::kUnpack:
       ProcessUnpackOperator(model, static_cast<UnpackOperator*>(op));
       break;
+    case OperatorType::kMirrorPad:
+      ProcessMirrorPadOperator(model, static_cast<MirrorPadOperator*>(op));
+      break;
     default:
       // Unimplemented, another graph transformation should drop it.
       LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(op->type);
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index aa6b4a3bc5..0b2f810394 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1153,6 +1153,31 @@ tensorflow::Status ConvertConcatOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertMirrorPadOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  if (node.op() != "MirrorPad") {
+    LOG(FATAL) << "Expected MirrorPad.";
+  }
+  const int num_inputs = GetInputsCount(node, tf_import_flags);
+  CHECK_EQ(num_inputs, 2);
+  auto* op = new MirrorPadOperator;
+  for (int i = 0; i < num_inputs; ++i) {
+    op->inputs.push_back(node.input(i));
+  }
+  op->outputs.push_back(node.name());
+  const auto mode = GetStringAttr(node, "mode");
+  if (mode == "REFLECT") {
+    op->mode = toco::MirrorPadMode::kReflect;
+  } else if (mode == "SYMMETRIC") {
+    op->mode = toco::MirrorPadMode::kSymmetric;
+  }
+
+  model->operators.emplace_back(op);
+
+  return tensorflow::Status::OK();
+}
+
 static constexpr int kAnyNumInputs = -1;
 
 enum FlexSupport { kFlexOk, kFlexNotOk };
@@ -2389,6 +2414,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Unpack", ConvertUnpackOperator},
       {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1>},
       {"UnidirectionalSequenceLstm", ConvertUnidirectionalSequenceLstm},
+      {"MirrorPad", ConvertMirrorPadOperator},
   });
 }
 
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index d4fe62ac75..f19355968f 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -156,7 +156,8 @@ enum class OperatorType : uint8 {
   kZerosLike,
   kResizeNearestNeighbor,
   kLeakyRelu,
-  kAbs
+  kAbs,
+  kMirrorPad
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1932,6 +1933,23 @@ struct TensorFlowZerosLikeOperator : Operator {
   TensorFlowZerosLikeOperator() : Operator(OperatorType::kZerosLike) {}
 };
 
+enum class MirrorPadMode { kNone, kSymmetric, kReflect };
+
+// MirrorPad Operator:
+//
+// Inputs:
+// Inputs[0]: required: input tensor to be padded.
+// Inputs[1]: required: 2 Column matrix specifying padding sizes. The number of
+// rows must be the same as the rank of the input.
+// Inputs[2]: required: REFLECT or SYMMETRIC.
+//
+// TensorFlow equivalent: MirrorPad.
+struct MirrorPadOperator : Operator {
+  MirrorPadOperator() : Operator(OperatorType::kMirrorPad) {}
+  // mode is either SYMMETRIC or REFLECT.
+  MirrorPadMode mode;
+};
+
 // Alloc's are used for transient arrays only. An Alloc specifies which interval
 // of the "transient_data" workspace buffer passed to inference functions, is to
 // be used for the transient array at hand. The 'start' and 'end' values are
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index c4030a1b1a..e0faed4927 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1275,6 +1275,29 @@ class SquaredDifference
   int GetVersion(const Operator& op) const override { return 1; }
 };
 
+class MirrorPad
+    : public BuiltinOperator<MirrorPadOperator, ::tflite::MirrorPadOptions,
+                             ::tflite::BuiltinOptions_MirrorPadOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateMirrorPadOptions(
+        *builder, op.mode == MirrorPadMode::kReflect
+                      ? ::tflite::MirrorPadMode::MirrorPadMode_REFLECT
+                      : ::tflite::MirrorPadMode::MirrorPadMode_SYMMETRIC);
+  }
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->mode = options.mode() == ::tflite::MirrorPadMode::MirrorPadMode_REFLECT
+                   ? MirrorPadMode::kReflect
+                   : MirrorPadMode::kSymmetric;
+  }
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
 std::unique_ptr<flexbuffers::Builder> WriteFlexOpOptions(
     const string& tensorflow_node_def) {
   auto fbb = absl::make_unique<flexbuffers::Builder>();
@@ -1581,6 +1604,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   ops.push_back(MakeUnique<SquaredDifference>(
       ::tflite::BuiltinOperator_SQUARED_DIFFERENCE,
       OperatorType::kSquaredDifference));
+  ops.push_back(MakeUnique<MirrorPad>(::tflite::BuiltinOperator_MIRROR_PAD,
+                                      OperatorType::kMirrorPad));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 6154d94692..14ec89cd73 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -616,6 +616,14 @@ TEST_F(OperatorTest, TestShouldExportAsFlexOp) {
   EXPECT_FALSE(ShouldExportAsFlexOp(true, "RFFT"));
 }
 
+TEST_F(OperatorTest, BuiltinMirrorPad) {
+  MirrorPadOperator op;
+  op.mode = MirrorPadMode::kReflect;
+  auto output_toco_op = SerializeAndDeserialize(
+      GetOperator("MIRROR_PAD", OperatorType::kMirrorPad), op);
+  EXPECT_EQ(op.mode, output_toco_op->mode);
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index d0cc799424..fc9a0d8af5 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -415,6 +415,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor)
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
+    HANDLE_OPERATORTYPENAME_CASE(MirrorPad)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From 92854f2426c2b91f2b09831696ffebb5b793933d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 3 Dec 2018 22:45:18 -0800
Subject: [PATCH 1263/1554] Enable depthwise convolution tests on the CPU

PiperOrigin-RevId: 223921992
---
 tensorflow/compiler/xla/tests/BUILD           |  1 -
 .../compiler/xla/tests/conv_depthwise_test.cc | 20 +++++++++----------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 48bbc4e472..f7f090fe4a 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -305,7 +305,6 @@ xla_test(
     srcs = ["conv_depthwise_test.cc"],
     blacklisted_backends = [
         # disabled because of a break b/119590850.
-        "cpu",
         "gpu",
     ],
     shard_count = 50,
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
index 60ce576ceb..bc9bd8a269 100644
--- a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
@@ -136,7 +136,7 @@ string BuildHloTextDepthwiseConvolution2D(
   if (spec.activation_dims[1] == 1 && spec.kernel_dims[1] == 2) {
     return absl::StrFormat(
         R"(
-    HloModule TensorFlowDepthwiseConv, is_scheduled=true
+    HloModule TensorFlowDepthwiseConv
 
     ENTRY main {
       activation = %s[%s]{%s} parameter(0)
@@ -161,7 +161,7 @@ string BuildHloTextDepthwiseConvolution2D(
   } else if (spec.stride == -1) {
     return absl::StrFormat(
         R"(
-      HloModule TensorFlowDepthwiseConv, is_scheduled=true
+      HloModule TensorFlowDepthwiseConv
 
       ENTRY main {
         activation = %s[%s]{%s} parameter(0)
@@ -185,7 +185,7 @@ string BuildHloTextDepthwiseConvolution2D(
   } else {
     return absl::StrFormat(
         R"(
-    HloModule TensorFlowDepthwiseConv, is_scheduled=true
+    HloModule TensorFlowDepthwiseConv
 
     ENTRY main {
       activation = %s[%s]{%s} parameter(0)
@@ -215,13 +215,13 @@ XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
   const string hlo_text =
       BuildHloTextDepthwiseConvolution2D(spec, use_bfloat16);
 
-  EXPECT_TRUE(RunAndCompareNoHloPasses(
-      hlo_text, ErrorSpec{0.01, 0.01}, [](HloModule* module) -> Status {
-        BFloat16MixedPrecisionRemoval remover;
-        TF_RETURN_IF_ERROR(remover.Run(module).status());
-        Despecializer despecializer;
-        return despecializer.Run(module).status();
-      }));
+  EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01},
+                            [](HloModule* module) -> Status {
+                              BFloat16MixedPrecisionRemoval remover;
+                              TF_RETURN_IF_ERROR(remover.Run(module).status());
+                              Despecializer despecializer;
+                              return despecializer.Run(module).status();
+                            }));
 }
 
 INSTANTIATE_TEST_CASE_P(
-- 
GitLab


From f5b4fa5b43d238d1d3e93870bcb65ae7b8f38b47 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 01:02:22 -0800
Subject: [PATCH 1264/1554] compat: Update forward compatibility horizon to
 2018-12-04

PiperOrigin-RevId: 223933783
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 811bf31f39..0b6ff30488 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 3)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 4)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From e58a84208a46cf749f92d8170ed81794ecfa3137 Mon Sep 17 00:00:00 2001
From: Ilya Biryukov <ibiryukov@google.com>
Date: Tue, 4 Dec 2018 01:29:47 -0800
Subject: [PATCH 1265/1554] Update downloadable clang to r347933

PiperOrigin-RevId: 223937748
---
 third_party/clang_toolchain/download_clang.bzl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/third_party/clang_toolchain/download_clang.bzl b/third_party/clang_toolchain/download_clang.bzl
index a941ee1c99..7ced902747 100644
--- a/third_party/clang_toolchain/download_clang.bzl
+++ b/third_party/clang_toolchain/download_clang.bzl
@@ -39,15 +39,15 @@ def download_clang(repo_ctx, out_folder):
 
     # Latest CLANG_REVISION and CLANG_SUB_REVISION of the Chromiums's release
     # can be found in https://chromium.googlesource.com/chromium/src/tools/clang/+/master/scripts/update.py
-    CLANG_REVISION = "346388"
-    CLANG_SUB_REVISION = 3
+    CLANG_REVISION = "347933"
+    CLANG_SUB_REVISION = 1
 
     package_version = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
 
     checksums = {
-        "Linux_x64": "d47b7ac4756c3f8e3bbfa0e81bf199ec8e9faa3a6b11573f0705e9c04af7ad51",
-        "Mac": "de2b0c701e19cda633ea02804866dd24d8506afb8cae51fbcce3415b76f4ded3",
-        "Win": "c7d27f13b41aa9eaaf9760903962e9b2b0f8261058df0d35170711dc60545a7d",
+        "Linux_x64": "cae3643fdf5d46fc9bc8731212bb37573547148d90b64b083165e090133d11b0",
+        "Mac": "083a0e91a38c06e568652313ac7372b17a101268f7d65533d721ca30413442b4",
+        "Win": "43160487cfc7e88076a369a2b6e8e4a0f42e104c28d8903f3aaa62d630aba949",
     }
 
     platform_folder = _get_platform_folder(repo_ctx.os.name)
-- 
GitLab


From 566923394a39f16e0a008fc602416a7aec9a30c9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 02:29:42 -0800
Subject: [PATCH 1266/1554] Fix exported version of keras's model_to_estimator.

PiperOrigin-RevId: 223944247
---
 tensorflow/python/keras/estimator/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/estimator/__init__.py b/tensorflow/python/keras/estimator/__init__.py
index 3c1a63d6df..dcd0600897 100644
--- a/tensorflow/python/keras/estimator/__init__.py
+++ b/tensorflow/python/keras/estimator/__init__.py
@@ -65,7 +65,7 @@ def model_to_estimator(
     raise NotImplementedError(
         'tf.keras.estimator.model_to_estimator function not available in your '
         'installation.')
-  keras_lib.model_to_estimator(
+  return keras_lib.model_to_estimator(
       keras_model=keras_model,
       keras_model_path=keras_model_path,
       custom_objects=custom_objects,
-- 
GitLab


From d6decf3b9adabf64d492bb0e2674d8d928637b2f Mon Sep 17 00:00:00 2001
From: Jason Zaman <jason@perfinion.com>
Date: Mon, 13 Aug 2018 00:29:28 +0800
Subject: [PATCH 1267/1554] systemlibs: Unbundle protobuf

Use system_link_files for protobuf.bzl. The protobuf.bzl file is taken
verbatim from the protobuf repo. This version of protobuf.bzl will only
be used when opting into using the system version of protobuf and is off
by default.

Signed-off-by: Jason Zaman <jason@perfinion.com>
---
 tensorflow/workspace.bzl                     |  12 +
 third_party/systemlibs/protobuf.BUILD        | 103 +++++
 third_party/systemlibs/protobuf.bzl          | 417 +++++++++++++++++++
 third_party/systemlibs/syslibs_configure.bzl |   3 +
 4 files changed, 535 insertions(+)
 create mode 100644 third_party/systemlibs/protobuf.BUILD
 create mode 100644 third_party/systemlibs/protobuf.bzl

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7e5f84be16..7063031628 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -357,6 +357,10 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         name = "protobuf_archive",
         sha256 = PROTOBUF_SHA256,
         strip_prefix = PROTOBUF_STRIP_PREFIX,
+        system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
+        system_link_files = {
+            "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
+        },
         urls = PROTOBUF_URLS,
     )
 
@@ -367,6 +371,10 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         name = "com_google_protobuf",
         sha256 = PROTOBUF_SHA256,
         strip_prefix = PROTOBUF_STRIP_PREFIX,
+        system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
+        system_link_files = {
+            "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
+        },
         urls = PROTOBUF_URLS,
     )
 
@@ -374,6 +382,10 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         name = "com_google_protobuf_cc",
         sha256 = PROTOBUF_SHA256,
         strip_prefix = PROTOBUF_STRIP_PREFIX,
+        system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
+        system_link_files = {
+            "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
+        },
         urls = PROTOBUF_URLS,
     )
 
diff --git a/third_party/systemlibs/protobuf.BUILD b/third_party/systemlibs/protobuf.BUILD
new file mode 100644
index 0000000000..e9244d172b
--- /dev/null
+++ b/third_party/systemlibs/protobuf.BUILD
@@ -0,0 +1,103 @@
+load("@protobuf_archive//:protobuf.bzl",
+    "proto_gen",
+    "py_proto_library",
+    "cc_proto_library",
+)
+
+licenses(["notice"])
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+HEADERS = [
+    "google/protobuf/any.pb.h",
+    "google/protobuf/any.proto",
+    "google/protobuf/arena.h",
+    "google/protobuf/compiler/importer.h",
+    "google/protobuf/descriptor.h",
+    "google/protobuf/descriptor.pb.h",
+    "google/protobuf/descriptor.proto",
+    "google/protobuf/duration.pb.h",
+    "google/protobuf/duration.proto",
+    "google/protobuf/dynamic_message.h",
+    "google/protobuf/empty.pb.h",
+    "google/protobuf/empty.proto",
+    "google/protobuf/field_mask.pb.h",
+    "google/protobuf/field_mask.proto",
+    "google/protobuf/io/coded_stream.h",
+    "google/protobuf/io/zero_copy_stream.h",
+    "google/protobuf/io/zero_copy_stream_impl_lite.h",
+    "google/protobuf/map.h",
+    "google/protobuf/repeated_field.h",
+    "google/protobuf/text_format.h",
+    "google/protobuf/timestamp.pb.h",
+    "google/protobuf/timestamp.proto",
+    "google/protobuf/util/json_util.h",
+    "google/protobuf/util/type_resolver_util.h",
+    "google/protobuf/wrappers.pb.h",
+    "google/protobuf/wrappers.proto",
+]
+
+genrule(
+    name = "link_headers",
+    outs = HEADERS,
+    cmd = """
+      for i in $(OUTS); do
+        f=$${i#$(@D)/}
+        mkdir -p $(@D)/$${f%/*}
+        ln -sf $(INCLUDEDIR)/$$f $(@D)/$$f
+      done
+    """,
+)
+
+cc_library(
+    name = "protobuf",
+    hdrs = HEADERS,
+    linkopts = ["-lprotobuf"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "protobuf_headers",
+    hdrs = HEADERS,
+    linkopts = ["-lprotobuf"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "protoc_lib",
+    linkopts = ["-lprotoc"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "protoc",
+    outs = ["protoc.bin"],
+    cmd = "ln -s $$(which protoc) $@",
+    executable = 1,
+    visibility = ["//visibility:public"],
+)
+
+cc_proto_library(
+    name = "cc_wkt_protos",
+    hdrs = HEADERS,
+    protoc = ":protoc",
+    internal_bootstrap_hack = 1,
+    visibility = ["//visibility:public"],
+)
+
+proto_gen(
+    name = "protobuf_python_genproto",
+    includes = ["."],
+    protoc = "@protobuf_archive//:protoc",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "protobuf_python",
+    data = [":link_headers"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/protobuf.bzl b/third_party/systemlibs/protobuf.bzl
new file mode 100644
index 0000000000..78f19c621a
--- /dev/null
+++ b/third_party/systemlibs/protobuf.bzl
@@ -0,0 +1,417 @@
+def _GetPath(ctx, path):
+  if ctx.label.workspace_root:
+    return ctx.label.workspace_root + '/' + path
+  else:
+    return path
+
+def _IsNewExternal(ctx):
+  # Bazel 0.4.4 and older have genfiles paths that look like:
+  #   bazel-out/local-fastbuild/genfiles/external/repo/foo
+  # After the exec root rearrangement, they look like:
+  #   ../repo/bazel-out/local-fastbuild/genfiles/foo
+  return ctx.label.workspace_root.startswith("../")
+
+def _GenDir(ctx):
+  if _IsNewExternal(ctx):
+    # We are using the fact that Bazel 0.4.4+ provides repository-relative paths
+    # for ctx.genfiles_dir.
+    return ctx.genfiles_dir.path + (
+        "/" + ctx.attr.includes[0] if ctx.attr.includes and ctx.attr.includes[0] else "")
+  # This means that we're either in the old version OR the new version in the local repo.
+  # Either way, appending the source path to the genfiles dir works.
+  return ctx.var["GENDIR"] + "/" + _SourceDir(ctx)
+
+def _SourceDir(ctx):
+  if not ctx.attr.includes:
+    return ctx.label.workspace_root
+  if not ctx.attr.includes[0]:
+    return _GetPath(ctx, ctx.label.package)
+  if not ctx.label.package:
+    return _GetPath(ctx, ctx.attr.includes[0])
+  return _GetPath(ctx, ctx.label.package + '/' + ctx.attr.includes[0])
+
+def _CcHdrs(srcs, use_grpc_plugin=False):
+  ret = [s[:-len(".proto")] + ".pb.h" for s in srcs]
+  if use_grpc_plugin:
+    ret += [s[:-len(".proto")] + ".grpc.pb.h" for s in srcs]
+  return ret
+
+def _CcSrcs(srcs, use_grpc_plugin=False):
+  ret = [s[:-len(".proto")] + ".pb.cc" for s in srcs]
+  if use_grpc_plugin:
+    ret += [s[:-len(".proto")] + ".grpc.pb.cc" for s in srcs]
+  return ret
+
+def _CcOuts(srcs, use_grpc_plugin=False):
+  return _CcHdrs(srcs, use_grpc_plugin) + _CcSrcs(srcs, use_grpc_plugin)
+
+def _PyOuts(srcs, use_grpc_plugin=False):
+  ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs]
+  if use_grpc_plugin:
+    ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs]
+  return ret
+
+def _RelativeOutputPath(path, include, dest=""):
+  if include == None:
+    return path
+
+  if not path.startswith(include):
+    fail("Include path %s isn't part of the path %s." % (include, path))
+
+  if include and include[-1] != '/':
+    include = include + '/'
+  if dest and dest[-1] != '/':
+    dest = dest + '/'
+
+  path = path[len(include):]
+  return dest + path
+
+def _proto_gen_impl(ctx):
+  """General implementation for generating protos"""
+  srcs = ctx.files.srcs
+  deps = []
+  deps += ctx.files.srcs
+  source_dir = _SourceDir(ctx)
+  gen_dir = _GenDir(ctx)
+  if source_dir:
+    import_flags = ["-I" + source_dir, "-I" + gen_dir]
+  else:
+    import_flags = ["-I."]
+
+  for dep in ctx.attr.deps:
+    import_flags += dep.proto.import_flags
+    deps += dep.proto.deps
+
+  args = []
+  if ctx.attr.gen_cc:
+    args += ["--cpp_out=" + gen_dir]
+  if ctx.attr.gen_py:
+    args += ["--python_out=" + gen_dir]
+
+  inputs = srcs + deps
+  if ctx.executable.plugin:
+    plugin = ctx.executable.plugin
+    lang = ctx.attr.plugin_language
+    if not lang and plugin.basename.startswith('protoc-gen-'):
+      lang = plugin.basename[len('protoc-gen-'):]
+    if not lang:
+      fail("cannot infer the target language of plugin", "plugin_language")
+
+    outdir = gen_dir
+    if ctx.attr.plugin_options:
+      outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir
+    args += ["--plugin=protoc-gen-%s=%s" % (lang, plugin.path)]
+    args += ["--%s_out=%s" % (lang, outdir)]
+    inputs += [plugin]
+
+  if args:
+    ctx.action(
+        inputs=inputs,
+        outputs=ctx.outputs.outs,
+        arguments=args + import_flags + [s.path for s in srcs],
+        executable=ctx.executable.protoc,
+        mnemonic="ProtoCompile",
+        use_default_shell_env=True,
+    )
+
+  return struct(
+      proto=struct(
+          srcs=srcs,
+          import_flags=import_flags,
+          deps=deps,
+      ),
+  )
+
+proto_gen = rule(
+    attrs = {
+        "srcs": attr.label_list(allow_files = True),
+        "deps": attr.label_list(providers = ["proto"]),
+        "includes": attr.string_list(),
+        "protoc": attr.label(
+            cfg = "host",
+            executable = True,
+            single_file = True,
+            mandatory = True,
+        ),
+        "plugin": attr.label(
+            cfg = "host",
+            allow_files = True,
+            executable = True,
+        ),
+        "plugin_language": attr.string(),
+        "plugin_options": attr.string_list(),
+        "gen_cc": attr.bool(),
+        "gen_py": attr.bool(),
+        "outs": attr.output_list(),
+    },
+    output_to_genfiles = True,
+    implementation = _proto_gen_impl,
+)
+"""Generates codes from Protocol Buffers definitions.
+
+This rule helps you to implement Skylark macros specific to the target
+language. You should prefer more specific `cc_proto_library `,
+`py_proto_library` and others unless you are adding such wrapper macros.
+
+Args:
+  srcs: Protocol Buffers definition files (.proto) to run the protocol compiler
+    against.
+  deps: a list of dependency labels; must be other proto libraries.
+  includes: a list of include paths to .proto files.
+  protoc: the label of the protocol compiler to generate the sources.
+  plugin: the label of the protocol compiler plugin to be passed to the protocol
+    compiler.
+  plugin_language: the language of the generated sources
+  plugin_options: a list of options to be passed to the plugin
+  gen_cc: generates C++ sources in addition to the ones from the plugin.
+  gen_py: generates Python sources in addition to the ones from the plugin.
+  outs: a list of labels of the expected outputs from the protocol compiler.
+"""
+
+def cc_proto_library(
+        name,
+        srcs=[],
+        deps=[],
+        cc_libs=[],
+        include=None,
+        protoc="@com_google_protobuf//:protoc",
+        internal_bootstrap_hack=False,
+        use_grpc_plugin=False,
+        default_runtime="@com_google_protobuf//:protobuf",
+        **kargs):
+  """Bazel rule to create a C++ protobuf library from proto source files
+
+  NOTE: the rule is only an internal workaround to generate protos. The
+  interface may change and the rule may be removed when bazel has introduced
+  the native rule.
+
+  Args:
+    name: the name of the cc_proto_library.
+    srcs: the .proto files of the cc_proto_library.
+    deps: a list of dependency labels; must be cc_proto_library.
+    cc_libs: a list of other cc_library targets depended by the generated
+        cc_library.
+    include: a string indicating the include path of the .proto files.
+    protoc: the label of the protocol compiler to generate the sources.
+    internal_bootstrap_hack: a flag indicate the cc_proto_library is used only
+        for bootstraping. When it is set to True, no files will be generated.
+        The rule will simply be a provider for .proto files, so that other
+        cc_proto_library can depend on it.
+    use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
+        when processing the proto files.
+    default_runtime: the implicitly default runtime which will be depended on by
+        the generated cc_library target.
+    **kargs: other keyword arguments that are passed to cc_library.
+
+  """
+
+  includes = []
+  if include != None:
+    includes = [include]
+
+  if internal_bootstrap_hack:
+    # For pre-checked-in generated files, we add the internal_bootstrap_hack
+    # which will skip the codegen action.
+    proto_gen(
+        name=name + "_genproto",
+        srcs=srcs,
+        deps=[s + "_genproto" for s in deps],
+        includes=includes,
+        protoc=protoc,
+        visibility=["//visibility:public"],
+    )
+    # An empty cc_library to make rule dependency consistent.
+    native.cc_library(
+        name=name,
+        **kargs)
+    return
+
+  grpc_cpp_plugin = None
+  if use_grpc_plugin:
+    grpc_cpp_plugin = "//external:grpc_cpp_plugin"
+
+  gen_srcs = _CcSrcs(srcs, use_grpc_plugin)
+  gen_hdrs = _CcHdrs(srcs, use_grpc_plugin)
+  outs = gen_srcs + gen_hdrs
+
+  proto_gen(
+      name=name + "_genproto",
+      srcs=srcs,
+      deps=[s + "_genproto" for s in deps],
+      includes=includes,
+      protoc=protoc,
+      plugin=grpc_cpp_plugin,
+      plugin_language="grpc",
+      gen_cc=1,
+      outs=outs,
+      visibility=["//visibility:public"],
+  )
+
+  if default_runtime and not default_runtime in cc_libs:
+    cc_libs = cc_libs + [default_runtime]
+  if use_grpc_plugin:
+    cc_libs = cc_libs + ["//external:grpc_lib"]
+
+  native.cc_library(
+      name=name,
+      srcs=gen_srcs,
+      hdrs=gen_hdrs,
+      deps=cc_libs + deps,
+      includes=includes,
+      **kargs)
+
+def internal_gen_well_known_protos_java(srcs):
+  """Bazel rule to generate the gen_well_known_protos_java genrule
+
+  Args:
+    srcs: the well known protos
+  """
+  root = Label("%s//protobuf_java" % (REPOSITORY_NAME)).workspace_root
+  pkg = PACKAGE_NAME + "/" if PACKAGE_NAME else ""
+  if root == "":
+    include = " -I%ssrc " % pkg
+  else:
+    include = " -I%s/%ssrc " % (root, pkg)
+  native.genrule(
+    name = "gen_well_known_protos_java",
+    srcs = srcs,
+    outs = [
+        "wellknown.srcjar",
+    ],
+    cmd = "$(location :protoc) --java_out=$(@D)/wellknown.jar" +
+          " %s $(SRCS) " % include +
+          " && mv $(@D)/wellknown.jar $(@D)/wellknown.srcjar",
+    tools = [":protoc"],
+  )
+
+def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
+  """Macro to copy files to a different directory and then create a filegroup.
+
+  This is used by the //:protobuf_python py_proto_library target to work around
+  an issue caused by Python source files that are part of the same Python
+  package being in separate directories.
+
+  Args:
+    srcs: The source files to copy and add to the filegroup.
+    strip_prefix: Path to the root of the files to copy.
+    dest: The directory to copy the source files into.
+    **kwargs: extra arguments that will be passesd to the filegroup.
+  """
+  outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs]
+
+  native.genrule(
+      name = name + "_genrule",
+      srcs = srcs,
+      outs = outs,
+      cmd = " && ".join(
+          ["cp $(location %s) $(location %s)" %
+           (s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]),
+  )
+
+  native.filegroup(
+      name = name,
+      srcs = outs,
+      **kwargs)
+
+def py_proto_library(
+        name,
+        srcs=[],
+        deps=[],
+        py_libs=[],
+        py_extra_srcs=[],
+        include=None,
+        default_runtime="@com_google_protobuf//:protobuf_python",
+        protoc="@com_google_protobuf//:protoc",
+        use_grpc_plugin=False,
+        **kargs):
+  """Bazel rule to create a Python protobuf library from proto source files
+
+  NOTE: the rule is only an internal workaround to generate protos. The
+  interface may change and the rule may be removed when bazel has introduced
+  the native rule.
+
+  Args:
+    name: the name of the py_proto_library.
+    srcs: the .proto files of the py_proto_library.
+    deps: a list of dependency labels; must be py_proto_library.
+    py_libs: a list of other py_library targets depended by the generated
+        py_library.
+    py_extra_srcs: extra source files that will be added to the output
+        py_library. This attribute is used for internal bootstrapping.
+    include: a string indicating the include path of the .proto files.
+    default_runtime: the implicitly default runtime which will be depended on by
+        the generated py_library target.
+    protoc: the label of the protocol compiler to generate the sources.
+    use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin
+        when processing the proto files.
+    **kargs: other keyword arguments that are passed to cc_library.
+
+  """
+  outs = _PyOuts(srcs, use_grpc_plugin)
+
+  includes = []
+  if include != None:
+    includes = [include]
+
+  grpc_python_plugin = None
+  if use_grpc_plugin:
+    grpc_python_plugin = "//external:grpc_python_plugin"
+    # Note: Generated grpc code depends on Python grpc module. This dependency
+    # is not explicitly listed in py_libs. Instead, host system is assumed to
+    # have grpc installed.
+
+  proto_gen(
+      name=name + "_genproto",
+      srcs=srcs,
+      deps=[s + "_genproto" for s in deps],
+      includes=includes,
+      protoc=protoc,
+      gen_py=1,
+      outs=outs,
+      visibility=["//visibility:public"],
+      plugin=grpc_python_plugin,
+      plugin_language="grpc"
+  )
+
+  if default_runtime and not default_runtime in py_libs + deps:
+    py_libs = py_libs + [default_runtime]
+
+  native.py_library(
+      name=name,
+      srcs=outs+py_extra_srcs,
+      deps=py_libs+deps,
+      imports=includes,
+      **kargs)
+
+def internal_protobuf_py_tests(
+    name,
+    modules=[],
+    **kargs):
+  """Bazel rules to create batch tests for protobuf internal.
+
+  Args:
+    name: the name of the rule.
+    modules: a list of modules for tests. The macro will create a py_test for
+        each of the parameter with the source "google/protobuf/%s.py"
+    kargs: extra parameters that will be passed into the py_test.
+
+  """
+  for m in modules:
+    s = "python/google/protobuf/internal/%s.py" % m
+    native.py_test(
+        name="py_%s" % m,
+        srcs=[s],
+        main=s,
+        **kargs)
+
+
+def check_protobuf_required_bazel_version():
+  """For WORKSPACE files, to check the installed version of bazel.
+
+  This ensures bazel supports our approach to proto_library() depending on a
+  copied filegroup. (Fixed in bazel 0.5.4)
+  """
+  expected = apple_common.dotted_version("0.5.4")
+  current = apple_common.dotted_version(native.bazel_version)
+  if current.compare_to(expected) < 0:
+    fail("Bazel must be newer than 0.5.4")
diff --git a/third_party/systemlibs/syslibs_configure.bzl b/third_party/systemlibs/syslibs_configure.bzl
index dbf4fd6e32..645d242c96 100644
--- a/third_party/systemlibs/syslibs_configure.bzl
+++ b/third_party/systemlibs/syslibs_configure.bzl
@@ -15,6 +15,8 @@ VALID_LIBS = [
     "boringssl",
     "com_github_googleapis_googleapis",
     "com_github_googlecloudplatform_google_cloud_cpp",
+    "com_google_protobuf",
+    "com_google_protobuf_cc",
     "com_googlesource_code_re2",
     "curl",
     "cython",
@@ -32,6 +34,7 @@ VALID_LIBS = [
     "org_sqlite",
     "pcre",
     "png_archive",
+    "protobuf_archive",
     "six_archive",
     "snappy",
     "swig",
-- 
GitLab


From 8cc14c541e165528237e2bdb74a4e5ef9981675c Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Tue, 4 Dec 2018 05:39:27 -0800
Subject: [PATCH 1268/1554] Make sure that we can fuse diamond shaped graphs.

We have some logic which does a global analysis of the graph to
determine if we want to allow duplication of specific instructions.
This logic however does not take into account whether actually
duplication is needed because the analysis runs on the original
graph and is not recomputed when nodes are fused. For the GPU
and CPU backends, we run the fusion pass twice: once where we
disallow *all* duplications, and once where we allow *some*
duplications. The global analysis is therefore not needed if we
disallow *all* duplications, and then we can successfully fuse
diamond-shaped graphs because we do a reverse postorder traversal
through the graph.

PiperOrigin-RevId: 223961479
---
 .../xla/service/instruction_fusion.cc         | 13 +++--
 .../xla/service/instruction_fusion_test.cc    | 50 +++++++++++++++++++
 2 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index 2297edcbe1..7559ed1bab 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -457,8 +457,13 @@ StatusOr<bool> InstructionFusion::Run(HloModule* module) {
     computation_ = computation;
     reachability_ = HloReachabilityMap::Build(computation_);
 
-    HloInstructionSet do_not_duplicate =
-        ComputeGloballyUnfusible(computation_->MakeInstructionPostOrder());
+    HloInstructionSet do_not_duplicate;
+    // If we allow duplications, we need to compute which instructions we do not
+    // want to duplicate based on a global analysis of the graph.
+    if (may_duplicate_) {
+      do_not_duplicate =
+          ComputeGloballyUnfusible(computation_->MakeInstructionPostOrder());
+    }
     auto fusion_queue = GetFusionQueue(computation_);
 
     // Instruction fusion effectively fuses edges in the computation graph
@@ -566,8 +571,8 @@ HloInstruction* InstructionFusion::FuseIntoMultiOutput(
 bool InstructionFusion::MultiOutputFusionCreatesCycle(
     HloInstruction* producer, HloInstruction* consumer) {
   auto is_reachable = [&](const HloInstruction* a, const HloInstruction* b) {
-    // A consumer operand may have been multii-output fused into a parallel
-    // consumer and thus be missing  from the oridinal reachability map.
+    // A consumer operand may have been multi-output fused into a parallel
+    // consumer and thus be missing from the original reachability map.
     if (!reachability_->IsPresent(a) || !reachability_->IsPresent(b)) {
       reachability_ = HloReachabilityMap::Build(consumer->parent());
     }
diff --git a/tensorflow/compiler/xla/service/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/instruction_fusion_test.cc
index 6b48312649..58b7135cea 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion_test.cc
@@ -394,6 +394,56 @@ TEST_F(InstructionFusionTest, AllowEffectiveUnaryDuplication) {
           .ValueOrDie());
 }
 
+TEST_F(InstructionFusionTest, FuseDiamondGraphsNoDuplication) {
+  auto module = ParseHloString(R"(
+  HloModule test_module
+  ENTRY Test {
+    p0 = f32[100] parameter(0)
+    p1 = f32[100] parameter(1)
+    add = f32[100] add(p0, p1)
+    slice1 = f32[99] slice(add), slice={[0:99:1]}
+    slice2 = f32[99] slice(add), slice={[1:100:1]}
+    ROOT add2 = f32[99] add(slice1, slice2)
+  })")
+                    .ValueOrDie();
+  EXPECT_TRUE(
+      InstructionFusion(InstructionFusion::IsExpensive, /*may_duplicate=*/false)
+          .Run(module.get())
+          .ValueOrDie())
+      << module->ToString();
+
+  HloInstruction* root = module->entry_computation()->root_instruction();
+  // 'add' would originally need to be duplicated if fused. However after its
+  // two users 'slice1' and 'slice2' are fused into 'add2', 'add' has only one
+  // user and can now be also fused.
+  EXPECT_THAT(root, op::Fusion(op::Parameter(), op::Parameter()));
+}
+
+TEST_F(InstructionFusionTest, FuseDiamondGraphsAllowDuplication) {
+  auto module = ParseHloString(R"(
+  HloModule test_module
+  ENTRY Test {
+    p0 = f32[100] parameter(0)
+    p1 = f32[100] parameter(1)
+    add = f32[100] add(p0, p1)
+    slice1 = f32[99] slice(add), slice={[0:99:1]}
+    slice2 = f32[99] slice(add), slice={[1:100:1]}
+    ROOT add2 = f32[99] add(slice1, slice2)
+  })")
+                    .ValueOrDie();
+  EXPECT_TRUE(
+      InstructionFusion(InstructionFusion::IsExpensive, /*may_duplicate=*/true)
+          .Run(module.get())
+          .ValueOrDie())
+      << module->ToString();
+
+  HloInstruction* root = module->entry_computation()->root_instruction();
+  // 'add' would originally need to be duplicated if fused. However after its
+  // two users 'slice1' and 'slice2' are fused into 'add2', 'add' has only one
+  // user and can now be also fused.
+  EXPECT_THAT(root, op::Fusion(op::Parameter(), op::Parameter()));
+}
+
 TEST_F(InstructionFusionTest,
        WideningConvertsAreAlwaysDuplicableIntoConsumers) {
   auto module = ParseHloString(R"(
-- 
GitLab


From 4beea30ec8a231a12b9183f631608a6af64b8172 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 05:47:46 -0800
Subject: [PATCH 1269/1554] Include import library
 (_pywrap_tensorflow_internal.lib) for _pywrap_tensorflow_internal.pyd in pip
 package

To fix #23740

PiperOrigin-RevId: 223962144
---
 tensorflow/python/BUILD            | 15 ++++++++++++++-
 tensorflow/tools/pip_package/BUILD | 11 ++++-------
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d990ebb7f1..a558045e4a 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4181,11 +4181,24 @@ genrule(
 
 # Get the import library of  _pywrap_tensorflow_internal.dll
 filegroup(
-    name = "pywrap_tensorflow_import_lib_file",
+    name = "get_pywrap_tensorflow_import_lib_file",
     srcs = [":_pywrap_tensorflow_internal.so"],
     output_group = "interface_library",
 )
 
+# Rename the import library for _pywrap_tensorflow_internal.pyd to _pywrap_tensorflow_internal.lib
+# (It was _pywrap_tensorflow_internal.so.if.lib).
+genrule(
+    name = "pywrap_tensorflow_import_lib_file",
+    srcs = [":get_pywrap_tensorflow_import_lib_file"],
+    outs = ["_pywrap_tensorflow_internal.lib"],
+    cmd = select({
+        "//tensorflow:windows": "cp -f $< $@",
+        "//conditions:default": "touch $@",  # Just a placeholder for Unix platforms
+    }),
+    visibility = ["//visibility:public"],
+)
+
 # Create a cc_import rule for the import library of _pywrap_tensorflow_internal.dll
 # so that custom ops' dynamic libraries can link against it.
 cc_import(
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index fa372dcd74..82c6bf383f 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -88,6 +88,9 @@ COMMON_PIP_DEPS = [
     "//tensorflow/contrib/timeseries:timeseries_pip",
     "//tensorflow/contrib/tpu",
     "//tensorflow/examples/tutorials/mnist:package",
+    "//tensorflow/lite/python:interpreter_test_data",
+    "//tensorflow/lite/python:tflite_convert",
+    "//tensorflow/lite/toco/python:toco_from_protos",
     # "//tensorflow/python/autograph/converters:converters",
     # "//tensorflow/python/autograph/core:core",
     "//tensorflow/python/autograph/core:test_lib",
@@ -124,7 +127,7 @@ COMMON_PIP_DEPS = [
 py_binary(
     name = "simple_console_for_windows",
     srcs = ["simple_console_for_windows.py"],
-    data = COMMON_PIP_DEPS,
+    data = COMMON_PIP_DEPS + ["//tensorflow/python:pywrap_tensorflow_import_lib_file"],
     srcs_version = "PY2AND3",
     deps = ["//tensorflow:tensorflow_py"],
 )
@@ -228,15 +231,9 @@ sh_binary(
     data = select({
         "//tensorflow:windows": [
             ":simple_console_for_windows",
-            "//tensorflow/lite/python:interpreter_test_data",
-            "//tensorflow/lite/python:tflite_convert",
-            "//tensorflow/lite/toco/python:toco_from_protos",
         ],
         "//conditions:default": COMMON_PIP_DEPS + [
             ":simple_console",
-            "//tensorflow/lite/python:interpreter_test_data",
-            "//tensorflow/lite/python:tflite_convert",
-            "//tensorflow/lite/toco/python:toco_from_protos",
         ],
     }) + if_mkl_ml(["//third_party/mkl:intel_binary_blob"]),
 )
-- 
GitLab


From 5828c4084ca025f7943e097cb2ddfd5e17817d64 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Tue, 4 Dec 2018 06:13:55 -0800
Subject: [PATCH 1270/1554] Add a test case that shows exponential compilation
 behavior on the GPU/CPU backends.

PiperOrigin-RevId: 223964674
---
 .../xla/tests/array_elementwise_ops_test.cc   | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 0615f9425c..f6be27bee2 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -350,6 +350,44 @@ TEST_P(ArrayElementwiseOpTestParamCount, AddManyValues) {
                              error_spec_);
 }
 
+// TODO(b/119692968): This test runs OOM on the GPU and CPU backend.
+XLA_TEST_F(ArrayElementwiseOpTest,
+           DISABLED_ON_GPU(DISABLED_ON_CPU(DeeplyNestedAddWithSlices))) {
+  XlaBuilder builder(TestName());
+  std::vector<float> values(30, 0.0);
+  auto a_literal = LiteralUtil::CreateR1<float>(values);
+  auto a = Parameter(&builder, 0, a_literal.shape(), "x");
+  auto b_literal = LiteralUtil::CreateR1<float>(values);
+  auto b = Parameter(&builder, 1, b_literal.shape(), "x");
+
+  // Construct a sequence of diamond-shaped gadgets like this:
+  //
+  //      add
+  //    /    \
+  //  slice  slice
+  //     \   /
+  //      add
+  //
+  // Each 'left' slice removes the last element, each 'right' slice removes the
+  // first element. In this way, we index into the add with different
+  // multi-dimensional index arrays, which defeats the caching we use to avoid
+  // exponential compile time.
+  std::function<XlaOp(int64)> generate_recursive =
+      [&](int64 slice_size) -> XlaOp {
+    if (slice_size == values.size()) {
+      return Add(a, b);
+    }
+    XlaOp param = generate_recursive(slice_size + 1);
+    auto slice1 = Slice(param, {0}, {slice_size}, {1});
+    auto slice2 = Slice(param, {1}, {slice_size + 1}, {1});
+    return Add(slice1, slice2);
+  };
+  generate_recursive(1);
+  auto a_data = client_->TransferToServer(a_literal).ConsumeValueOrDie();
+  auto b_data = client_->TransferToServer(b_literal).ConsumeValueOrDie();
+  ComputeAndCompareR1<float>(&builder, {0.0}, {a_data.get(), b_data.get()});
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantF32s) {
   XlaBuilder builder(TestName());
   auto a = ConstantR1<float>(&builder, {-2.5f, 3.14f, 2.25f, -10.0f, 6.0f});
-- 
GitLab


From f5dfd16cf97a876d49930d9dd1e59092fc3fd40f Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 4 Dec 2018 07:39:43 -0800
Subject: [PATCH 1271/1554] [TF:XLA] Remove the transpose_x/y and conjugate_x/y
 arguments from the TF2XLA BatchDot helper.

There's no need for a single large function that does everything. We can instead compose these operations out of the individual pieces, simplifying the BatchDot API.

PiperOrigin-RevId: 223975055
---
 .../tf2xla/kernels/batch_matmul_op.cc         |  9 ++++--
 tensorflow/compiler/tf2xla/lib/batch_dot.cc   | 23 +++++---------
 tensorflow/compiler/tf2xla/lib/batch_dot.h    | 11 ++-----
 tensorflow/compiler/tf2xla/lib/cholesky.cc    | 14 ++-------
 tensorflow/compiler/tf2xla/lib/qr.cc          | 31 +++++--------------
 .../compiler/tf2xla/lib/triangular_solve.cc   | 23 +++++++-------
 tensorflow/compiler/tf2xla/lib/util.cc        |  4 +++
 tensorflow/compiler/tf2xla/lib/util.h         |  4 +++
 tensorflow/compiler/tf2xla/lib/util_test.cc   |  2 +-
 9 files changed, 46 insertions(+), 75 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
index 4cfe946b2e..7ba91765fd 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
+
+#include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 
@@ -28,9 +30,10 @@ class BatchMatMulOp : public XlaOpKernel {
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    auto result = BatchDot(ctx->Input(0), ctx->Input(1),
-                           /*transpose_x=*/adj_x_, /*transpose_y=*/adj_y_,
-                           /*conjugate_x=*/adj_x_, /*conjugate_y=*/adj_y_);
+    auto result = BatchDot(MaybeTransposeInMinorDims(
+                               MaybeConjugate(ctx->Input(0), adj_x_), adj_x_),
+                           MaybeTransposeInMinorDims(
+                               MaybeConjugate(ctx->Input(1), adj_y_), adj_y_));
     ctx->SetOutput(0, result);
   }
 
diff --git a/tensorflow/compiler/tf2xla/lib/batch_dot.cc b/tensorflow/compiler/tf2xla/lib/batch_dot.cc
index 5400e8834c..dfc840698a 100644
--- a/tensorflow/compiler/tf2xla/lib/batch_dot.cc
+++ b/tensorflow/compiler/tf2xla/lib/batch_dot.cc
@@ -26,8 +26,7 @@ limitations under the License.
 
 namespace tensorflow {
 
-xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y, bool transpose_x,
-                    bool transpose_y, bool conjugate_x, bool conjugate_y,
+xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y,
                     xla::PrecisionConfig::Precision precision) {
   xla::XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
@@ -61,15 +60,14 @@ xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y, bool transpose_x,
       batch_dimension_numbers.push_back(i);
     }
 
-    int x_inner_dim = transpose_x ? (ndims - 2) : (ndims - 1);
-    int y_inner_dim = transpose_y ? (ndims - 1) : (ndims - 2);
+    int x_inner_dim = ndims - 1;
+    int y_inner_dim = ndims - 2;
     if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) {
       return errors::InvalidArgument(
           "Dimensions ", x_inner_dim, " and ", y_inner_dim,
           " of arguments to BatchedDot must be equal: ",
-          xla::ShapeUtil::HumanString(x_shape), " transpose: ", transpose_x,
-          " vs. ", xla::ShapeUtil::HumanString(y_shape),
-          " transpose: ", transpose_y);
+          xla::ShapeUtil::HumanString(x_shape), " vs. ",
+          xla::ShapeUtil::HumanString(y_shape));
     }
 
     // Check for zero lhs/rhs dim size.
@@ -79,8 +77,8 @@ xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y, bool transpose_x,
       for (int i = 0; i < batch_dimension_numbers.size(); ++i) {
         dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]);
       }
-      int x_outer_dim = transpose_x ? (ndims - 1) : (ndims - 2);
-      int y_outer_dim = transpose_y ? (ndims - 2) : (ndims - 1);
+      int x_outer_dim = ndims - 2;
+      int y_outer_dim = ndims - 1;
       dimensions.push_back(x_shape.dimensions(x_outer_dim));
       dimensions.push_back(y_shape.dimensions(y_outer_dim));
       return xla::Broadcast(
@@ -89,13 +87,6 @@ xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y, bool transpose_x,
           dimensions);
     }
 
-    if (x_shape.element_type() == xla::C64 && conjugate_x) {
-      x = xla::Conj(x);
-    }
-    if (y_shape.element_type() == xla::C64 && conjugate_y) {
-      y = xla::Conj(y);
-    }
-
     xla::PrecisionConfig precision_proto;
     precision_proto.add_operand_precision(precision);
     precision_proto.add_operand_precision(precision);
diff --git a/tensorflow/compiler/tf2xla/lib/batch_dot.h b/tensorflow/compiler/tf2xla/lib/batch_dot.h
index 6edd63a4d3..6cfa698593 100644
--- a/tensorflow/compiler/tf2xla/lib/batch_dot.h
+++ b/tensorflow/compiler/tf2xla/lib/batch_dot.h
@@ -25,12 +25,7 @@ namespace tensorflow {
 
 // Multiplies all slices of `Tensor` `x` and `y` (each slice can be
 // viewed as an element of a batch), and arranges the individual results
-// in a single output tensor of the same batch size. Each of the
-// individual slices can optionally be transposed before multiplication by
-// setting the `transpose_x` or `transpose_y` flag to `true`. Similarly, each
-// can be elementwise-complex-conjugated by setting the `conjugate_x` or
-// `conjugate_y` flag to `true`. To apply a Hermitian adjoint to `x`, set both
-// `transpose_x` and `conjugate_x` to `true`, and analogously for `y`.
+// in a single output tensor of the same batch size.
 //
 // The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]`
 // and `[..., r_y, c_y]`.
@@ -44,9 +39,7 @@ namespace tensorflow {
 //
 //     output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :])
 xla::XlaOp BatchDot(
-    xla::XlaOp x, xla::XlaOp y, bool transpose_x = false,
-    bool transpose_y = false, bool conjugate_x = false,
-    bool conjugate_y = false,
+    xla::XlaOp x, xla::XlaOp y,
     xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/lib/cholesky.cc b/tensorflow/compiler/tf2xla/lib/cholesky.cc
index ab3d0a5668..dfc014395b 100644
--- a/tensorflow/compiler/tf2xla/lib/cholesky.cc
+++ b/tensorflow/compiler/tf2xla/lib/cholesky.cc
@@ -101,10 +101,7 @@ xla::XlaOp CholeskyUnblocked(xla::XlaOp a,
       // a[..., i, i]
       auto a_ii = DynamicSliceInMinorDims(body_a, {i, i}, {1, 1});
       // np.dot(row, np.swapaxes(row, -1, -2))
-      auto diag_dot = BatchDot(row, row,
-                               /*transpose_x=*/false,
-                               /*transpose_y=*/true, /*conjugate_x=*/false,
-                               /*conjugate_y=*/false, precision);
+      auto diag_dot = BatchDot(row, TransposeInMinorDims(row), precision);
       // l[..., i, i] = np.sqrt(a[..., i, i] - np.dot(row,
       //                                              np.swapaxes(row, -1, -2)))
       auto l_ii =
@@ -122,10 +119,7 @@ xla::XlaOp CholeskyUnblocked(xla::XlaOp a,
       // The columns in [i, n] are zeroed out in `row`, so we just have to
       // zero out rows above i+1 after the BatchDot. np.dot(l[..., :, :i],
       // r.T)
-      auto dot = BatchDot(body_l, row,
-                          /*transpose_x=*/false,
-                          /*transpose_y=*/true, /*conjugate_x=*/false,
-                          /*conjugate_y=*/false, precision);
+      auto dot = BatchDot(body_l, TransposeInMinorDims(row), precision);
       // np.dot(l[..., i+1:, :i], r.T)
       auto dot_ip1 =
           xla::Select(xla::Le(mask_range_col, i), mask_zeros_col, dot);
@@ -185,9 +179,7 @@ xla::XlaOp Cholesky(xla::XlaOp a, int64 block_size,
         // a[i:, i:i+k] -= np.dot(l[i:, :i], np.transpose(l[i:i+k, :i]))
         auto lhs = SliceInMinorDims(l, {i, 0}, {n, i});
         auto rhs = SliceInMinorDims(l, {i, 0}, {i + k, i});
-        auto delta = BatchDot(lhs, rhs, /*transpose_x=*/false,
-                              /*transpose_y=*/true, /*conjugate_x=*/false,
-                              /*conjugate_y=*/false, precision);
+        auto delta = BatchDot(lhs, TransposeInMinorDims(rhs), precision);
         auto before = SliceInMinorDims(a, {i, i}, {n, i + k});
         a = UpdateSliceInMinorDims(a, before - delta, {i, i});
       }
diff --git a/tensorflow/compiler/tf2xla/lib/qr.cc b/tensorflow/compiler/tf2xla/lib/qr.cc
index aa1c0e4743..440ad13321 100644
--- a/tensorflow/compiler/tf2xla/lib/qr.cc
+++ b/tensorflow/compiler/tf2xla/lib/qr.cc
@@ -191,12 +191,8 @@ xla::StatusOr<QRBlockResult> QRBlock(
     auto v_broadcast = xla::Reshape(v, shape);
     // a[:, :] -= tau * np.dot(v[:, np.newaxis],
     //                          np.dot(v[np.newaxis, :], a[:, :]))
-    auto vva =
-        BatchDot(v_broadcast, a, /*transpose_x=*/false, /*transpose_y=*/false,
-                 /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
-    vva =
-        BatchDot(v_broadcast, vva, /*transpose_x=*/true, /*transpose_y=*/false,
-                 /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
+    auto vva = BatchDot(v_broadcast, a, precision);
+    vva = BatchDot(TransposeInMinorDims(v_broadcast), vva, precision);
     a = a - xla::Mul(tau, vva,
                      /*broadcast_dimensions=*/batch_dim_indices);
 
@@ -278,12 +274,9 @@ xla::StatusOr<xla::XlaOp> ComputeWYRepresentation(
     auto beta = DynamicSliceInMinorDims(taus, {j}, {1});
 
     // yv has shape [..., n, 1]
-    auto yv = BatchDot(y, v, /*transpose_x=*/true, /*transpose_y=*/false,
-                       /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
+    auto yv = BatchDot(TransposeInMinorDims(y), v, precision);
     // wyv has shape [..., m, 1]
-    auto wyv =
-        BatchDot(w, yv, /*transpose_x=*/false, /*transpose_y=*/false,
-                 /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
+    auto wyv = BatchDot(w, yv, precision);
 
     auto z = xla::Mul(
         -beta, v + wyv,
@@ -375,23 +368,15 @@ xla::StatusOr<QRDecompositionResult> QRDecomposition(
 
     // a[i:, i+k:] += np.dot(Y, np.dot(W.T, a[i:, i+k:]))
     auto a_panel = SliceInMinorDims(a, {i, i + k}, {m, n});
-    auto a_update =
-        BatchDot(w, a_panel, /*transpose_x=*/true, /*transpose_y=*/false,
-                 /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
-    a_update =
-        BatchDot(y, a_update, /*transpose_x=*/false, /*transpose_y=*/false,
-                 /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
+    auto a_update = BatchDot(TransposeInMinorDims(w), a_panel, precision);
+    a_update = BatchDot(y, a_update, precision);
     a_panel = a_panel + a_update;
     a = UpdateSliceInMinorDims(a, a_panel, {i, i + k});
 
     // q[:, i:] += np.dot(np.dot(q[:, i:], W), Y.T))
     auto q_panel = SliceInMinorDims(q, {0, i}, {m, m});
-    auto q_update =
-        BatchDot(q_panel, w, /*transpose_x=*/false, /*transpose_y=*/false,
-                 /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
-    q_update = BatchDot(q_update, y, /*transpose_x=*/false,
-                        /*transpose_y=*/true, /*conjugate_x=*/false,
-                        /*conjugate_y=*/false, precision);
+    auto q_update = BatchDot(q_panel, w, precision);
+    q_update = BatchDot(q_update, TransposeInMinorDims(y), precision);
     q_panel = q_panel + q_update;
     q = UpdateSliceInMinorDims(q, q_panel, {0, i});
   }
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
index 4bc3796e4f..df8489b62e 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
@@ -311,13 +311,13 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks(
         auto a_row =
             MaybeConjugate(SliceInMinorDims(a, start, end), conjugate_a);
         if (left_side) {
-          remainder = b_row - BatchDot(a_row, x, transpose_a, false,
-                                       /*conjugate_x=*/false,
-                                       /*conjugate_y=*/false, precision);
+          remainder =
+              b_row - BatchDot(MaybeTransposeInMinorDims(a_row, transpose_a), x,
+                               precision);
         } else {
-          remainder = b_row - BatchDot(x, a_row, false, transpose_a,
-                                       /*conjugate_x=*/false,
-                                       /*conjugate_y=*/false, precision);
+          remainder =
+              b_row - BatchDot(x, MaybeTransposeInMinorDims(a_row, transpose_a),
+                               precision);
         }
       }
 
@@ -327,13 +327,12 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks(
           xla::ConstantR0WithType(builder, xla::S32, j * block_size);
       std::vector<xla::XlaOp> update_starts = {start_index, zero};
       if (left_side) {
-        x_update =
-            BatchDot(inv_block, remainder, transpose_a, false,
-                     /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
+        x_update = BatchDot(MaybeTransposeInMinorDims(inv_block, transpose_a),
+                            remainder, precision);
       } else {
-        x_update =
-            BatchDot(remainder, inv_block, false, transpose_a,
-                     /*conjugate_x=*/false, /*conjugate_y=*/false, precision);
+        x_update = BatchDot(remainder,
+                            MaybeTransposeInMinorDims(inv_block, transpose_a),
+                            precision);
         std::swap(update_starts[0], update_starts[1]);
       }
       x = DynamicUpdateSliceInMinorDims(x, x_update, /*starts=*/update_starts);
diff --git a/tensorflow/compiler/tf2xla/lib/util.cc b/tensorflow/compiler/tf2xla/lib/util.cc
index 804671fbc7..610d6882ba 100644
--- a/tensorflow/compiler/tf2xla/lib/util.cc
+++ b/tensorflow/compiler/tf2xla/lib/util.cc
@@ -239,6 +239,10 @@ xla::XlaOp TransposeInMinorDims(xla::XlaOp x) {
   });
 }
 
+xla::XlaOp MaybeTransposeInMinorDims(xla::XlaOp x, bool transpose) {
+  return transpose ? TransposeInMinorDims(x) : x;
+}
+
 xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate) {
   xla::XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
diff --git a/tensorflow/compiler/tf2xla/lib/util.h b/tensorflow/compiler/tf2xla/lib/util.h
index 80e9e5b002..d0980717c9 100644
--- a/tensorflow/compiler/tf2xla/lib/util.h
+++ b/tensorflow/compiler/tf2xla/lib/util.h
@@ -72,6 +72,10 @@ xla::XlaOp DynamicUpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update,
 // Transposes a stack of matrices `x` by swapping the last two dimensions.
 xla::XlaOp TransposeInMinorDims(xla::XlaOp x);
 
+// Transposes `x` in its minor dimensions if `transpose` is true, otherwise
+// returns `x` unchanged.
+xla::XlaOp MaybeTransposeInMinorDims(xla::XlaOp x, bool transpose);
+
 // Applies a complex conjugation operation if `a` is complex and `conjugate_a`
 // is true, otherwise returns its argument.
 xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate);
diff --git a/tensorflow/compiler/tf2xla/lib/util_test.cc b/tensorflow/compiler/tf2xla/lib/util_test.cc
index 442fe92c34..820f9a4972 100644
--- a/tensorflow/compiler/tf2xla/lib/util_test.cc
+++ b/tensorflow/compiler/tf2xla/lib/util_test.cc
@@ -126,7 +126,7 @@ XLA_TEST_F(UtilTest, RowBatchDot) {
 
   auto l_index = DynamicSliceInMinorDims(
       a, {index, xla::ConstantR0<int32>(&builder, 0)}, {1, n});
-  BatchDot(l_index, row, /*transpose_x=*/false, /*transpose_y=*/true);
+  BatchDot(l_index, TransposeInMinorDims(row));
 
   ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
                              {a_data.get(), row_data.get(), index_data.get()});
-- 
GitLab


From e185d1cc064b836867d0f33dd8c864ccf9a23570 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 4 Dec 2018 08:02:18 -0800
Subject: [PATCH 1272/1554] [tf.data] Add upgrade script warning about
 `Dataset.make_{initializable,one_shot}_iterator().

PiperOrigin-RevId: 223978130
---
 .../tools/compatibility/tf_upgrade_v2.py      | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 571367d112..c37bab029c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -786,11 +786,32 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "only effects core estimator. If you are using "
         "tf.contrib.learn.Estimator, please switch to using core estimator.")
 
+    make_initializable_iterator_deprecation = (
+        "(Manual edit required) The "
+        "`tf.data.Dataset.make_initializable_iterator()` method has been "
+        "removed. If you are using the Estimator API, you can return a dataset "
+        "directly from your input functions without creating an iterator. "
+        "As a last resort, please replace calls to that method on `dataset` "
+        "with a call to "
+        "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
+
+    make_one_shot_iterator_deprecation = (
+        "(Manual edit required) The "
+        "`tf.data.Dataset.make_one_shot_iterator()` method has been "
+        "removed. If you are using eager execution, you can iterate over "
+        "`dataset` using a Python `for` loop. If you are using the Estimator "
+        "API, you can return a dataset directly from your input functions "
+        "without creating an iterator. As a last resort, please replace calls "
+        "to that method on `dataset` with a call to "
+        "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
+
     # Specify warnings for functions that aren't restricted to the tf.x.y.z
     # format. This should only be used for methods with unique names, e.g.
     # export_savedmodel, which is only defined in Estimator objects.
     self.unrestricted_function_warnings = {
         "export_savedmodel": export_saved_model_renamed,
+        "make_initializable_iterator": make_initializable_iterator_deprecation,
+        "make_one_shot_iterator": make_one_shot_iterator_deprecation,
     }
 
   @staticmethod
-- 
GitLab


From 8f740400a1e41020bc35852a2e611ca237e38fa8 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Tue, 4 Dec 2018 08:07:30 -0800
Subject: [PATCH 1273/1554] Improve documentation of v1 initializers

PiperOrigin-RevId: 223979317
---
 tensorflow/python/ops/init_ops.py | 78 ++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 27 deletions(-)

diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py
index 03d2201a9a..c0a4bcd51d 100644
--- a/tensorflow/python/ops/init_ops.py
+++ b/tensorflow/python/ops/init_ops.py
@@ -55,6 +55,15 @@ class Initializer(object):
   """
 
   def __call__(self, shape, dtype=None, partition_info=None):
+    """Returns a tensor object initialized as specified by the initializer.
+
+    Args:
+      shape: Shape of the tensor.
+      dtype: Optional dtype of the tensor. If not provided use the initializer
+        dtype.
+      partition_info: Optional information about the possible partitioning of a
+        tensor.
+    """
     raise NotImplementedError
 
   def get_config(self):
@@ -143,7 +152,8 @@ class Constant(Initializer):
     value: A Python scalar, list or tuple of values, or a N-dimensional numpy
       array. All elements of the initialized variable will be set to the
       corresponding value in the `value` argument.
-    dtype: The data type.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer.
     verify_shape: Boolean that enables verification of the shape of `value`. If
       `True`, the initializer will throw an error if the shape of `value` is not
       compatible with the shape of the initialized tensor.
@@ -239,7 +249,8 @@ class RandomUniform(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer.
   """
 
   def __init__(self, minval=0, maxval=None, seed=None, dtype=dtypes.float32):
@@ -275,7 +286,8 @@ class RandomNormal(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type. Only floating point types are supported.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
   """
 
   def __init__(self, mean=0.0, stddev=1.0, seed=None, dtype=dtypes.float32):
@@ -316,7 +328,8 @@ class TruncatedNormal(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type. Only floating point types are supported.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
   """
 
   def __init__(self, mean=0.0, stddev=1.0, seed=None, dtype=dtypes.float32):
@@ -369,8 +382,9 @@ class UniformUnitScaling(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type. Only floating point types are supported.
-    
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Sussillo et al., 2014](https://arxiv.org/abs/1412.6558)
       ([pdf](http://arxiv.org/pdf/1412.6558.pdf))
@@ -437,7 +451,8 @@ class VarianceScaling(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type. Only floating point types are supported.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
 
   Raises:
     ValueError: In case of an invalid value for the "scale", mode" or
@@ -483,7 +498,7 @@ class VarianceScaling(Initializer):
     else:
       scale /= max(1., (fan_in + fan_out) / 2.)
     if self.distribution == "normal" or self.distribution == "truncated_normal":
-      # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)
+    # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)
       stddev = math.sqrt(scale) / .87962566103423978
       return random_ops.truncated_normal(
           shape, 0.0, stddev, dtype, seed=self.seed)
@@ -534,8 +549,9 @@ class Orthogonal(Initializer):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type.
-  
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Saxe et al., 2014](https://openreview.net/forum?id=_wzZwKpTDF_9C)
       ([pdf](https://arxiv.org/pdf/1312.6120.pdf))
@@ -592,8 +608,9 @@ class ConvolutionDeltaOrthogonal(Initializer):
       `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
-    dtype: The data type.
-    
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
       ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
@@ -652,8 +669,9 @@ class ConvolutionOrthogonal(Initializer):
       `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
-    dtype: The data type.
-    
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
       ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
@@ -721,8 +739,9 @@ class ConvolutionOrthogonal2D(ConvolutionOrthogonal):
       a factor of `gain`.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
-    dtype: The data type.
-    
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
       ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
@@ -862,8 +881,9 @@ class ConvolutionOrthogonal1D(ConvolutionOrthogonal):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type.
-    
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
       ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
@@ -982,8 +1002,9 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal):
       `gain` after applying this convolution.
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
-    dtype: The data type.
-    
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
+
   References:
       [Xiao et al., 2018](http://proceedings.mlr.press/v80/xiao18a.html)
       ([pdf](http://proceedings.mlr.press/v80/xiao18a/xiao18a.pdf))
@@ -1132,7 +1153,8 @@ class Identity(Initializer):
 
   Args:
     gain: Multiplicative factor to apply to the identity matrix.
-    dtype: The type of the output.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
   """
 
   def __init__(self, gain=1.0, dtype=dtypes.float32):
@@ -1170,9 +1192,10 @@ class GlorotUniform(VarianceScaling):
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed`
       for behavior.
-    dtype: The data type. Only floating point types are supported.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
 
-  References: 
+  References:
       [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html)
       ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf))
   """
@@ -1208,9 +1231,10 @@ class GlorotNormal(VarianceScaling):
   Args:
     seed: A Python integer. Used to create random seeds. See
       `tf.set_random_seed` for behavior.
-    dtype: The data type. Only floating point types are supported.
+    dtype: Default data type, used if no `dtype` argument is provided when
+      calling the initializer. Only floating point types are supported.
 
-  References: 
+  References:
       [Glorot et al., 2010](http://proceedings.mlr.press/v9/glorot10a.html)
       ([pdf](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf))
   """
@@ -1264,7 +1288,7 @@ def lecun_normal(seed=None):
       An initializer.
 
   References:
-      - Self-Normalizing Neural Networks, 
+      - Self-Normalizing Neural Networks,
       [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks)
       ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf))
       - Efficient Backprop,
@@ -1289,7 +1313,7 @@ def lecun_uniform(seed=None):
       An initializer.
 
   References:
-      - Self-Normalizing Neural Networks, 
+      - Self-Normalizing Neural Networks,
       [Klambauer et al., 2017](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks)
       ([pdf](https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf))
       - Efficient Backprop,
-- 
GitLab


From e68e9432bf224987106123e08cc61b9686f256ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 08:17:35 -0800
Subject: [PATCH 1274/1554] Internal change.

PiperOrigin-RevId: 223980636
---
 tensorflow/core/BUILD | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c268605711..0e4923be70 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1803,7 +1803,7 @@ cc_library(
 # registration of ops to prune code size.
 cc_library(
     name = "android_tensorflow_lib_selective_registration",
-    srcs = if_android(["//tensorflow/core:android_srcs"]),
+    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
     copts = tf_copts(android_optimization_level_override = None) + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
@@ -1814,11 +1814,9 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        ":android_tensorflow_lib_lite",
         ":protos_all_cc_impl",
-        "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_set",
-        "@double_conversion//:double-conversion",
-        "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
     ],
     alwayslink = 1,
@@ -1828,7 +1826,7 @@ cc_library(
 # no proto_rtti.
 cc_library(
     name = "android_tensorflow_lib_selective_registration_nortti",
-    srcs = if_android(["//tensorflow/core:android_srcs"]),
+    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
     copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
@@ -1839,11 +1837,9 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        ":android_tensorflow_lib_lite_nortti",
         ":protos_all_cc_impl",
-        "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_set",
-        "@double_conversion//:double-conversion",
-        "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
     ],
     alwayslink = 1,
-- 
GitLab


From 7fa9d6a1f2d5916f3873ea38ed59c69af4c54e70 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 4 Dec 2018 08:52:57 -0800
Subject: [PATCH 1275/1554] Make `TensorBuffer::data()` non-virtual and move
 the pointer into the base class.

All existing `TensorBuffer` subclasses already store a pointer to
their buffer. Accessing that pointer by calling a virtual method is
inefficient. We currently generate the following instruction sequence
at the callsite (when compiling for x86_64 with a recent version of Clang):

1dd002:         mov    (%rdi),%rax                 tensor.h:655
1dd005:         callq  *0x10(%rax)                 tensor.h:655

...and the following implementation for `Buffer::data()`:

236520:         mov    0x10(%rdi),%rax             tensor.h:888
236524:         retq                               tensor.h:888

With this change, we generate a single `mov` instruction inline at the call
site, and avoid any branching.

PiperOrigin-RevId: 223985477
---
 tensorflow/c/c_api.cc                         | 57 ++++++++++---------
 tensorflow/compiler/jit/xla_launch_util.h     | 10 ++--
 .../common_runtime/gpu/gpu_event_mgr_test.cc  |  4 +-
 tensorflow/core/framework/tensor.cc           | 17 +++---
 tensorflow/core/framework/tensor.h            | 17 ++++--
 tensorflow/core/framework/tensor_test.cc      | 21 +++++++
 tensorflow/lite/delegates/flex/buffer_map.cc  | 57 ++++++++++---------
 tensorflow/python/lib/core/py_func.cc         |  6 +-
 8 files changed, 110 insertions(+), 79 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index f13e8777df..94d18eb8b0 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -136,16 +136,22 @@ const char* TF_Message(const TF_Status* s) {
 namespace {
 class TF_ManagedBuffer : public TensorBuffer {
  public:
-  void* data_;
-  size_t len_;
-  void (*deallocator_)(void* data, size_t len, void* arg);
-  void* deallocator_arg_;
+  TF_ManagedBuffer(void* data, size_t len,
+                   void (*deallocator)(void* data, size_t len, void* arg),
+                   void* deallocator_arg)
+      : TensorBuffer(data),
+        len_(len),
+        deallocator_(deallocator),
+        deallocator_arg_(deallocator_arg) {}
+
+  const size_t len_;
+  void (*const deallocator_)(void* data, size_t len, void* arg);
+  void* const deallocator_arg_;
 
   ~TF_ManagedBuffer() override {
-    (*deallocator_)(data_, len_, deallocator_arg_);
+    (*deallocator_)(data(), len_, deallocator_arg_);
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return len_; }
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -199,8 +205,7 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims,
     dimvec[i] = static_cast<tensorflow::int64>(dims[i]);
   }
 
-  TF_ManagedBuffer* buf = new TF_ManagedBuffer;
-  buf->len_ = len;
+  TF_ManagedBuffer* buf = nullptr;
   if (dtype != TF_STRING && dtype != TF_RESOURCE &&
       tensorflow::DataTypeCanUseMemcpy(static_cast<DataType>(dtype)) &&
       reinterpret_cast<intptr_t>(data) % std::max(1, EIGEN_MAX_ALIGN_BYTES) !=
@@ -212,17 +217,15 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims,
     //
     // Other types have the same representation, so copy only if it is safe to
     // do so.
-    buf->data_ = allocate_tensor("TF_NewTensor", len);
-    std::memcpy(buf->data_, data, len);
-    buf->deallocator_ = deallocate_buffer;
-    buf->deallocator_arg_ = nullptr;
+    buf = new TF_ManagedBuffer(allocate_tensor("TF_NewTensor", len), len,
+                               deallocate_buffer, nullptr);
+    std::memcpy(buf->data(), data, len);
     // Free the original buffer.
     deallocator(data, len, deallocator_arg);
   } else {
-    buf->data_ = data;
-    buf->deallocator_ = deallocator;
-    buf->deallocator_arg_ = deallocator_arg;
+    buf = new TF_ManagedBuffer(data, len, deallocator, deallocator_arg);
   }
+
   TF_Tensor* ret = new TF_Tensor{dtype, TensorShape(dimvec), buf};
   size_t elem_size = TF_DataTypeSize(dtype);
   if (elem_size > 0 && len < (elem_size * ret->shape.num_elements())) {
@@ -477,9 +480,9 @@ static TF_Tensor* EmptyTensor(TF_DataType dtype, const TensorShape& shape) {
   CHECK_EQ(nelems, 0);
   static_assert(sizeof(int64_t) == sizeof(tensorflow::int64),
                 "64-bit int types should match in size");
-  return TF_NewTensor(dtype, reinterpret_cast<const int64_t*>(dims.data()),
-                      shape.dims(), reinterpret_cast<void*>(&empty), 0,
-                      [](void*, size_t, void*) {}, nullptr);
+  return TF_NewTensor(
+      dtype, reinterpret_cast<const int64_t*>(dims.data()), shape.dims(),
+      reinterpret_cast<void*>(&empty), 0, [](void*, size_t, void*) {}, nullptr);
 }
 
 // Non-static for testing.
@@ -1592,18 +1595,20 @@ TF_AttrMetadata TF_OperationGetAttrMetadata(TF_Operation* oper,
     break;                                            \
   }
 
-      LIST_CASE(s, TF_ATTR_STRING, metadata.total_size = 0;
-                for (int i = 0; i < attr->list().s_size();
-                     ++i) { metadata.total_size += attr->list().s(i).size(); });
+      LIST_CASE(
+          s, TF_ATTR_STRING, metadata.total_size = 0;
+          for (int i = 0; i < attr->list().s_size();
+               ++i) { metadata.total_size += attr->list().s(i).size(); });
       LIST_CASE(i, TF_ATTR_INT);
       LIST_CASE(f, TF_ATTR_FLOAT);
       LIST_CASE(b, TF_ATTR_BOOL);
       LIST_CASE(type, TF_ATTR_TYPE);
-      LIST_CASE(shape, TF_ATTR_SHAPE, metadata.total_size = 0;
-                for (int i = 0; i < attr->list().shape_size(); ++i) {
-                  const auto& s = attr->list().shape(i);
-                  metadata.total_size += s.unknown_rank() ? 0 : s.dim_size();
-                });
+      LIST_CASE(
+          shape, TF_ATTR_SHAPE, metadata.total_size = 0;
+          for (int i = 0; i < attr->list().shape_size(); ++i) {
+            const auto& s = attr->list().shape(i);
+            metadata.total_size += s.unknown_rank() ? 0 : s.dim_size();
+          });
       LIST_CASE(tensor, TF_ATTR_TENSOR);
       LIST_CASE(tensor, TF_ATTR_FUNC);
 #undef LIST_CASE
diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 437db019a0..706811f55e 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -199,19 +199,17 @@ class XlaTensorBuffer : public TensorBuffer {
  public:
   XlaTensorBuffer(const void* ptr, size_t expected_size, size_t actual_size,
                   Allocator* allocator)
-      : expected_size_(expected_size),
+      : TensorBuffer(const_cast<void*>(ptr)),
+        expected_size_(expected_size),
         actual_size_(actual_size),
-        allocator_(allocator) {
-    data_ = const_cast<void*>(ptr);
-  }
+        allocator_(allocator) {}
 
   ~XlaTensorBuffer() override {
     if (data_) {
-      allocator_->DeallocateRaw(data_);
+      allocator_->DeallocateRaw(data());
     }
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return expected_size_; }
 
   TensorBuffer* root_buffer() override { return this; }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
index d2adf699f5..fe32147557 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
@@ -78,7 +78,8 @@ static std::atomic_int_fast64_t live_tensor_bytes(0);
 // A TensorBuffer that counts live memory usage for testing
 class TestTensorBuffer : public TensorBuffer {
  public:
-  explicit TestTensorBuffer(size_t bytes) : bytes_(bytes) {
+  explicit TestTensorBuffer(size_t bytes)
+      : TensorBuffer(nullptr), bytes_(bytes) {
     live_tensor_bytes += bytes_;
   }
   ~TestTensorBuffer() override { live_tensor_bytes -= bytes_; }
@@ -86,7 +87,6 @@ class TestTensorBuffer : public TensorBuffer {
   size_t size() const override { return bytes_; }
 
   // Not used in this test
-  void* data() const override { return nullptr; }
   TensorBuffer* root_buffer() override { return nullptr; }
   void FillAllocationDescription(AllocationDescription* arg) const override {}
 
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index c7ddc6c21e..7e841489eb 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -68,7 +68,8 @@ namespace {
 // An un-templated base class for Buffer.
 class BufferBase : public TensorBuffer {
  public:
-  explicit BufferBase(Allocator* alloc) : alloc_(alloc) {}
+  explicit BufferBase(Allocator* alloc, void* data_ptr)
+      : TensorBuffer(data_ptr), alloc_(alloc) {}
 
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -106,7 +107,6 @@ class Buffer : public BufferBase {
   Buffer(Allocator* a, int64 n);
   Buffer(Allocator* a, int64 n, const AllocationAttributes& allocation_attr);
 
-  void* data() const override { return data_; }
   size_t size() const override { return sizeof(T) * elem_; }
 
  private:
@@ -442,20 +442,20 @@ struct ProtoHelper<Eigen::half> {
 
 template <typename T>
 Buffer<T>::Buffer(Allocator* a, int64 n)
-    : BufferBase(a), data_(a->Allocate<T>(n)), elem_(n) {}
+    : BufferBase(a, a->Allocate<T>(n)), elem_(n) {}
 
 template <typename T>
 Buffer<T>::Buffer(Allocator* a, int64 n,
                   const AllocationAttributes& allocation_attr)
-    : BufferBase(a), data_(a->Allocate<T>(n, allocation_attr)), elem_(n) {}
+    : BufferBase(a, a->Allocate<T>(n, allocation_attr)), elem_(n) {}
 
 template <typename T>
 Buffer<T>::~Buffer() {
-  if (data_) {
+  if (data()) {
     if (LogMemory::IsEnabled()) {
       RecordDeallocation();
     }
-    alloc_->Deallocate<T>(data_, elem_);
+    alloc_->Deallocate<T>(static_cast<T*>(data()), elem_);
   }
 }
 
@@ -764,7 +764,9 @@ class SubBuffer : public TensorBuffer {
  public:
   // This buffer is an alias to buf[delta, delta + n).
   SubBuffer(TensorBuffer* buf, int64 delta, int64 n)
-      : root_(buf->root_buffer()), data_(buf->base<T>() + delta), elem_(n) {
+      : TensorBuffer(buf->base<T>() + delta),
+        root_(buf->root_buffer()),
+        elem_(n) {
     // Sanity check. The caller should ensure the sub buffer is valid.
     CHECK_LE(root_->base<T>(), this->base<T>());
     T* root_limit = root_->base<T>() + root_->size() / sizeof(T);
@@ -775,7 +777,6 @@ class SubBuffer : public TensorBuffer {
     root_->Ref();
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return sizeof(T) * elem_; }
   TensorBuffer* root_buffer() override { return root_; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 3177bbe7e9..6e03cf9f6f 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -636,10 +636,15 @@ class Tensor {
 // Interface to access the raw ref-counted data buffer.
 class TensorBuffer : public core::RefCounted {
  public:
+  explicit TensorBuffer(void* data_ptr) : data_(data_ptr) {}
   ~TensorBuffer() override {}
 
   // data() points to a memory region of size() bytes.
-  virtual void* data() const = 0;
+  //
+  // NOTE(mrry): The `data()` method is not virtual for performance reasons.
+  // It can be called multiple times when the contents of a `Tensor` are
+  // accessed, and so making it non-virtual allows the body to be inlined.
+  void* data() const { return data_; }
   virtual size_t size() const = 0;
 
   // If this TensorBuffer is sub-buffer of another TensorBuffer,
@@ -657,6 +662,9 @@ class TensorBuffer : public core::RefCounted {
 
   // Whether this TensorBuffer owns the underlying memory.
   virtual bool OwnsMemory() const { return true; }
+
+ private:
+  void* const data_;
 };
 
 template <typename T>
@@ -874,6 +882,7 @@ inline Tensor::Tensor(Tensor&& other)
 
 class Tensor::HostScalarTensorBufferBase : public TensorBuffer {
  public:
+  using TensorBuffer::TensorBuffer;
   void FillAllocationDescription(AllocationDescription* proto) const final;
 };
 
@@ -884,8 +893,7 @@ template <typename T>
 struct Tensor::ValueAndTensorBuffer {
   class HostScalarTensorBuffer : public Tensor::HostScalarTensorBufferBase {
    public:
-    HostScalarTensorBuffer(void* data) : data_(data) {}
-    void* data() const final { return const_cast<void*>(data_); }
+    HostScalarTensorBuffer(void* data) : HostScalarTensorBufferBase(data) {}
     size_t size() const final { return sizeof(T); }
     TensorBuffer* root_buffer() final { return this; }
 
@@ -904,8 +912,7 @@ struct Tensor::ValueAndTensorBuffer {
     }
 
    private:
-    ~HostScalarTensorBuffer() override { static_cast<T*>(data_)->~T(); }
-    void* const data_;
+    ~HostScalarTensorBuffer() override { static_cast<T*>(data())->~T(); }
   };
 
   T value;
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 4fa9d1df67..713f91fe04 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -1491,5 +1491,26 @@ void BM_CreateAndMoveCtrWithBuf(int iters) {
 }
 BENCHMARK(BM_CreateAndMoveCtrWithBuf);
 
+// Benchmark creating and destroy a host-scalar tensor, using the allocator
+// interface.
+void BM_CreateAndDestroyHostScalarNonOptimized(int iters) {
+  TensorShape shape({});
+  Allocator* allocator = cpu_allocator();
+  while (--iters) {
+    Tensor a(allocator, DT_FLOAT, shape);
+    a.scalar<float>()() = 37.0;
+  }
+}
+BENCHMARK(BM_CreateAndDestroyHostScalarNonOptimized);
+
+// Benchmark creating and destroy a host-scalar tensor, using the specialized
+// constructor.
+void BM_CreateAndDestroyHostScalarOptimized(int iters) {
+  while (--iters) {
+    Tensor a(37.0);
+  }
+}
+BENCHMARK(BM_CreateAndDestroyHostScalarOptimized);
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc
index 262ca9e089..0d0c953636 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map.cc
@@ -26,6 +26,8 @@ namespace flex {
 namespace {
 // A tensor buffer that is allocated, deallocated and populated by TF Lite.
 class BaseTfLiteTensorBuffer : public tensorflow::TensorBuffer {
+  using tensorflow::TensorBuffer::TensorBuffer;
+
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(
       tensorflow::AllocationDescription* proto) const override {
@@ -60,31 +62,29 @@ class BaseTfLiteTensorBuffer : public tensorflow::TensorBuffer {
 // representation in TFLITE and TF, so we just need use memcpy().
 class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
-  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor) {
+  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor)
+      : BaseTfLiteTensorBuffer(tensorflow::cpu_allocator()->AllocateRaw(
+            EIGEN_MAX_ALIGN_BYTES, tensor->bytes)) {
     // TODO(ahentz): if we can guarantee that TF Lite allocated tensors with
     // the same alignment as TensorFlow (EIGEN_MAX_ALIGN_BYTES), then we can
     // potentially eliminate the copy below.
     len_ = tensor->bytes;
-    data_ =
-        tensorflow::cpu_allocator()->AllocateRaw(EIGEN_MAX_ALIGN_BYTES, len_);
 
     LogAllocation();
 
-    if (data_) {
-      std::memcpy(data_, tensor->data.raw, tensor->bytes);
+    if (data()) {
+      std::memcpy(data(), tensor->data.raw, tensor->bytes);
     }
   }
 
   ~TfLiteTensorBuffer() override {
     LogDeallocation();
-    tensorflow::cpu_allocator()->DeallocateRaw(data_);
+    tensorflow::cpu_allocator()->DeallocateRaw(data());
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return len_; }
 
  private:
-  void* data_;
   size_t len_;
 };
 
@@ -92,19 +92,30 @@ class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
 // TF's so we need perform the conversion here.
 class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
-  explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor) {
-    if (tensor->data.raw == nullptr) {
-      num_strings_ = 0;
-      data_ = nullptr;
-      return;
-    }
-    num_strings_ = GetStringCount(tensor->data.raw);
-    data_ = tensorflow::cpu_allocator()->Allocate<string>(num_strings_);
+  explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor)
+      : StringTfLiteTensorBuffer(tensor, tensor->data.raw != nullptr
+                                             ? GetStringCount(tensor->data.raw)
+                                             : 0) {}
+
+  ~StringTfLiteTensorBuffer() override {
+    LogDeallocation();
+    tensorflow::cpu_allocator()->Deallocate<string>(
+        static_cast<string*>(data()), num_strings_);
+  }
+
+  size_t size() const override { return num_strings_ * sizeof(string); }
 
+ private:
+  StringTfLiteTensorBuffer(const TfLiteTensor* tensor, int num_strings)
+      : BaseTfLiteTensorBuffer(
+            num_strings != 0
+                ? tensorflow::cpu_allocator()->Allocate<string>(num_strings)
+                : nullptr),
+        num_strings_(num_strings) {
     LogAllocation();
 
-    if (data_) {
-      string* p = data_;
+    if (data()) {
+      string* p = static_cast<string*>(data());
       for (size_t i = 0; i < num_strings_; ++p, ++i) {
         auto ref = GetString(tensor->data.raw, i);
         p->assign(ref.str, ref.len);
@@ -112,16 +123,6 @@ class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
     }
   }
 
-  ~StringTfLiteTensorBuffer() override {
-    LogDeallocation();
-    tensorflow::cpu_allocator()->Deallocate<string>(data_, num_strings_);
-  }
-
-  void* data() const override { return data_; }
-  size_t size() const override { return num_strings_ * sizeof(string); }
-
- private:
-  string* data_;
   int num_strings_;
 };
 
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index 9364aec373..97bebe8617 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -302,15 +302,14 @@ Status DoCallPyFunc(PyCall* call, bool* out_log_on_error) {
 class NumpyTensorBuffer : public TensorBuffer {
  public:
   NumpyTensorBuffer(PyArrayObject* array, size_t len, void* data)
-      : array_(array), len_(len), data_(data) {}
+      : TensorBuffer(data), array_(array), len_(len) {}
 
   ~NumpyTensorBuffer() override {
     // Note: The session::run wrapper is responsible for freeing this while
     // holding the GIL.
-    DelayedNumpyDecref(data_, len_, array_);
+    DelayedNumpyDecref(data(), len_, array_);
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return len_; }
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -329,7 +328,6 @@ class NumpyTensorBuffer : public TensorBuffer {
  private:
   PyArrayObject* array_;
   size_t len_;
-  void* data_;
 };
 
 Status PyObjectToString(PyObject* obj, string* str) {
-- 
GitLab


From 7b1169cd951730e89ec4b019dd1c135e22eed29f Mon Sep 17 00:00:00 2001
From: Chris Antaki <ChrisAntaki@gmail.com>
Date: Tue, 4 Dec 2018 08:57:57 -0800
Subject: [PATCH 1276/1554] Removes line from README

As @terrytangyuan pointed out in #23647, the line being removed makes a suggestion that can't be followed
---
 tensorflow/contrib/tfprof/README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/tfprof/README.md b/tensorflow/contrib/tfprof/README.md
index b29d1acacf..f40e76f554 100644
--- a/tensorflow/contrib/tfprof/README.md
+++ b/tensorflow/contrib/tfprof/README.md
@@ -1,7 +1,5 @@
 # tfprof: TensorFlow Profiler and Beyond
 
-<h1>Please use `tf.profiler.xxx` instead of `tf.contrib.tfprof.xxx`</h1>
-
 <h1>Full Document in
 <a href="https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/profiler/README.md">tensorflow/core/profiler/README.md</a><h1>
 
-- 
GitLab


From 508331bda52cba5446a5a76c8c3aaac787d455bb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 09:00:45 -0800
Subject: [PATCH 1277/1554] Remove dimension switching for depthwise
 convolutions from the feature group converter.

PiperOrigin-RevId: 223986482
---
 .../convolution_feature_group_converter.cc    | 32 -------------------
 1 file changed, 32 deletions(-)

diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
index 07d6680a72..95c7724c3c 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_feature_group_converter.cc
@@ -205,38 +205,6 @@ Status ConvolutionVisitor::HandleConvolution(HloInstruction* convolution) {
     // If the code generator handles depthwise separable convolutions
     // inherently, then no filter expansion is needed.
     if (!filter_expansion_ && depthwise_separable) {
-      const int64 old_kernel_input_feature_dimension =
-          dim_numbers.kernel_input_feature_dimension();
-      const int64 old_kernel_output_feature_dimension =
-          dim_numbers.kernel_output_feature_dimension();
-
-      // For depthwise convolutions, we want the kernel input feature dimension
-      // to be smaller than the output feature dimension. If that's not the
-      // case, we swap the dimensions.
-      if (old_kernel_input_feature_dimension >
-          old_kernel_output_feature_dimension) {
-        Shape reshaped_filter_shape = filter->shape();
-        auto& dimensions = *reshaped_filter_shape.mutable_dimensions();
-        std::swap(dimensions[old_kernel_input_feature_dimension],
-                  dimensions[old_kernel_output_feature_dimension]);
-
-        auto reshaped_filter =
-            add(HloInstruction::CreateReshape(reshaped_filter_shape, filter));
-
-        dim_numbers.set_kernel_input_feature_dimension(
-            old_kernel_output_feature_dimension);
-
-        dim_numbers.set_kernel_output_feature_dimension(
-            old_kernel_input_feature_dimension);
-
-        auto new_convolution = HloInstruction::CreateConvolve(
-            convolution->shape(), convolution->mutable_operand(0),
-            reshaped_filter, group_count, convolution->window(), dim_numbers,
-            convolution->precision_config());
-
-        TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
-            convolution, std::move(new_convolution)));
-      }
       return Status::OK();
     }
     // We want to repeat 'filter' in the 'input_feature_dim' dimension
-- 
GitLab


From 3ec030892d2a92e86ad7015b211b31e6007df415 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 09:34:48 -0800
Subject: [PATCH 1278/1554] Automated rollback of commit
 e68e9432bf224987106123e08cc61b9686f256ce

PiperOrigin-RevId: 223991983
---
 tensorflow/core/BUILD | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 0e4923be70..c268605711 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1803,7 +1803,7 @@ cc_library(
 # registration of ops to prune code size.
 cc_library(
     name = "android_tensorflow_lib_selective_registration",
-    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
+    srcs = if_android(["//tensorflow/core:android_srcs"]),
     copts = tf_copts(android_optimization_level_override = None) + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
@@ -1814,9 +1814,11 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        ":android_tensorflow_lib_lite",
         ":protos_all_cc_impl",
+        "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@double_conversion//:double-conversion",
+        "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
     ],
     alwayslink = 1,
@@ -1826,7 +1828,7 @@ cc_library(
 # no proto_rtti.
 cc_library(
     name = "android_tensorflow_lib_selective_registration_nortti",
-    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
+    srcs = if_android(["//tensorflow/core:android_srcs"]),
     copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
@@ -1837,9 +1839,11 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        ":android_tensorflow_lib_lite_nortti",
         ":protos_all_cc_impl",
+        "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@double_conversion//:double-conversion",
+        "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
     ],
     alwayslink = 1,
-- 
GitLab


From b3710b27cc6098c86204b5ca3459280cbae1161c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 09:48:01 -0800
Subject: [PATCH 1279/1554] TOCO should be OK with using constant arrays as
 outputs.

PiperOrigin-RevId: 223994547
---
 tensorflow/lite/testing/generate_examples.py | 13 ++++++++-----
 tensorflow/lite/toco/toco_tooling.cc         |  1 +
 tensorflow/lite/toco/tooling_util.cc         |  3 +++
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 5c90410a19..4c731a7d18 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -815,6 +815,7 @@ def make_constant_tests(zip_path):
   test_parameters = [{
       "dtype": [tf.float32, tf.int32],
       "input_shape": [[], [1], [2], [1, 1, 1, 1], [2, 2, 2, 2]],
+      "constant_is_also_output": [True, False],
   }]
 
   def build_graph(parameters):
@@ -824,17 +825,19 @@ def make_constant_tests(zip_path):
         shape=parameters["input_shape"])
     constant = tf.constant(
         create_tensor_data(parameters["dtype"], parameters["input_shape"]))
-    # This maximum node is here to avoid the situation where a graph output is
-    # a constant, which is an error in toco.
-    out = tf.maximum(dummy_input, constant)
-    return [dummy_input], [out]
+    out = [tf.maximum(dummy_input, constant)]
+    if parameters["constant_is_also_output"]:
+      out.append(constant)
+
+    return [dummy_input], out
 
   def build_inputs(parameters, sess, inputs, outputs):
     dummy_input = np.zeros(
         parameters["input_shape"], dtype=_TF_TYPE_INFO[parameters["dtype"]][0])
     return [dummy_input], sess.run(outputs, feed_dict={inputs[0]: dummy_input})
 
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs,
+                    expected_tf_success=20)
 
 
 def make_binary_op_tests(zip_path, binary_operator):
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index d8b111d037..55a454e66d 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -309,6 +309,7 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
   // Fix any issues with IO edges. This must happen after any transform that
   // may modify the structure of the edges.
   FixEdgeArrays(model);
+  FixOperatorOrdering(model);
 
   if (quantize_output) {
     // If the user specified default min/max ranges we need to set all arrays
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index fc9a0d8af5..af4cd386a2 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -899,6 +899,9 @@ void CheckNonExistentIOArrays(const Model& model) {
         << "\" is not consumed by any op in this graph. " << general_comment;
   }
   for (const string& output_array : model.flags.output_arrays()) {
+    if (IsConstantParameterArray(model, output_array)) {
+      continue;  // It is OK to request that a constant be an output.
+    }
     QCHECK(GetOpWithOutput(model, output_array))
         << "Specified output array \"" << output_array
         << "\" is not produced by any op in this graph. " << general_comment;
-- 
GitLab


From a4e5957bc8065e2b8b5322d90d8b846525e83d16 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 4 Dec 2018 09:48:03 -0800
Subject: [PATCH 1280/1554] [XLA] [TF:XLA] Move BatchDot into the XLA client
 library.

BatchDot is a helper for performing batched matrix multiplications where all except the minor two dimensions are batch dimensions. It is used by a number of other linear algebra routines.

PiperOrigin-RevId: 223994553
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   1 -
 .../tf2xla/kernels/batch_matmul_op.cc         |  12 +-
 tensorflow/compiler/tf2xla/lib/BUILD          |  20 +---
 tensorflow/compiler/tf2xla/lib/batch_dot.cc   | 106 ------------------
 tensorflow/compiler/tf2xla/lib/batch_dot.h    |  47 --------
 tensorflow/compiler/tf2xla/lib/cholesky.cc    |   2 +-
 tensorflow/compiler/tf2xla/lib/qr.cc          |   1 -
 .../compiler/tf2xla/lib/triangular_solve.cc   |   1 -
 tensorflow/compiler/tf2xla/lib/util_test.cc   |   2 +-
 tensorflow/compiler/xla/client/lib/BUILD      |   4 +
 tensorflow/compiler/xla/client/lib/matrix.cc  |  81 ++++++++++++-
 tensorflow/compiler/xla/client/lib/matrix.h   |  20 ++++
 12 files changed, 114 insertions(+), 183 deletions(-)
 delete mode 100644 tensorflow/compiler/tf2xla/lib/batch_dot.cc
 delete mode 100644 tensorflow/compiler/tf2xla/lib/batch_dot.h

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index dcd38764be..fa51a72aea 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -121,7 +121,6 @@ tf_kernel_library(
         ":while_op",
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:xla_compiler",
-        "//tensorflow/compiler/tf2xla/lib:batch_dot",
         "//tensorflow/compiler/tf2xla/lib:broadcast",
         "//tensorflow/compiler/tf2xla/lib:cholesky",
         "//tensorflow/compiler/tf2xla/lib:qr",
diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
index 7ba91765fd..0a68988de1 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
@@ -13,11 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
-
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 
 namespace tensorflow {
 namespace {
@@ -30,10 +29,11 @@ class BatchMatMulOp : public XlaOpKernel {
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    auto result = BatchDot(MaybeTransposeInMinorDims(
-                               MaybeConjugate(ctx->Input(0), adj_x_), adj_x_),
-                           MaybeTransposeInMinorDims(
-                               MaybeConjugate(ctx->Input(1), adj_y_), adj_y_));
+    auto result =
+        xla::BatchDot(MaybeTransposeInMinorDims(
+                          MaybeConjugate(ctx->Input(0), adj_x_), adj_x_),
+                      MaybeTransposeInMinorDims(
+                          MaybeConjugate(ctx->Input(1), adj_y_), adj_y_));
     ctx->SetOutput(0, result);
   }
 
diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD
index 3575e480c1..34b95a112e 100644
--- a/tensorflow/compiler/tf2xla/lib/BUILD
+++ b/tensorflow/compiler/tf2xla/lib/BUILD
@@ -17,20 +17,6 @@ filegroup(
 
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 
-cc_library(
-    name = "batch_dot",
-    srcs = ["batch_dot.cc"],
-    hdrs = ["batch_dot.h"],
-    deps = [
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/core:lib",
-    ],
-)
-
 cc_library(
     name = "broadcast",
     srcs = ["broadcast.cc"],
@@ -52,7 +38,6 @@ cc_library(
     srcs = ["cholesky.cc"],
     hdrs = ["cholesky.h"],
     deps = [
-        ":batch_dot",
         ":triangular_solve",
         ":util",
         ":while_loop",
@@ -63,6 +48,7 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client/lib:constants",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/core:lib",
     ],
 )
@@ -87,7 +73,6 @@ cc_library(
     srcs = ["qr.cc"],
     hdrs = ["qr.h"],
     deps = [
-        ":batch_dot",
         ":util",
         ":while_loop",
         "//tensorflow/compiler/xla:literal_util",
@@ -129,7 +114,6 @@ cc_library(
     srcs = ["triangular_solve.cc"],
     hdrs = ["triangular_solve.h"],
     deps = [
-        ":batch_dot",
         ":util",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -191,7 +175,6 @@ xla_test(
     name = "util_test",
     srcs = ["util_test.cc"],
     deps = [
-        ":batch_dot",
         ":util",
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -202,6 +185,7 @@ xla_test(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/tf2xla/lib/batch_dot.cc b/tensorflow/compiler/tf2xla/lib/batch_dot.cc
deleted file mode 100644
index dfc840698a..0000000000
--- a/tensorflow/compiler/tf2xla/lib/batch_dot.cc
+++ /dev/null
@@ -1,106 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
-
-#include <memory>
-#include <vector>
-
-#include "tensorflow/compiler/xla/client/xla_builder.h"
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/status_macros.h"
-#include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/core/lib/core/errors.h"
-
-namespace tensorflow {
-
-xla::XlaOp BatchDot(xla::XlaOp x, xla::XlaOp y,
-                    xla::PrecisionConfig::Precision precision) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape x_shape, builder->GetShape(x));
-    TF_ASSIGN_OR_RETURN(xla::Shape y_shape, builder->GetShape(y));
-
-    // Check that both tensors have the same number of dimensions. There must be
-    // at least two (the batch dimensions can be empty).
-    if (xla::ShapeUtil::Rank(x_shape) != xla::ShapeUtil::Rank(y_shape)) {
-      return errors::InvalidArgument(
-          "Arguments to BatchedDot have different ranks: ",
-          xla::ShapeUtil::HumanString(x_shape), " vs. ",
-          xla::ShapeUtil::HumanString(y_shape));
-    }
-    const int ndims = xla::ShapeUtil::Rank(x_shape);
-    if (ndims < 2) {
-      return errors::InvalidArgument(
-          "Arguments to BatchedDot must have rank >= 2: ", ndims);
-    }
-
-    // The batch dimensions must be equal and the matrix dimensions must be
-    // valid.
-    std::vector<int64> batch_dimension_numbers;
-    for (int i = 0; i < ndims - 2; ++i) {
-      if (x_shape.dimensions(i) != y_shape.dimensions(i)) {
-        return errors::InvalidArgument(
-            "Dimension ", i, " of inputs to BatchedDot must be equal: ",
-            xla::ShapeUtil::HumanString(x_shape), " vs ",
-            xla::ShapeUtil::HumanString(y_shape));
-      }
-      batch_dimension_numbers.push_back(i);
-    }
-
-    int x_inner_dim = ndims - 1;
-    int y_inner_dim = ndims - 2;
-    if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) {
-      return errors::InvalidArgument(
-          "Dimensions ", x_inner_dim, " and ", y_inner_dim,
-          " of arguments to BatchedDot must be equal: ",
-          xla::ShapeUtil::HumanString(x_shape), " vs. ",
-          xla::ShapeUtil::HumanString(y_shape));
-    }
-
-    // Check for zero lhs/rhs dim size.
-    if (xla::ShapeUtil::IsZeroElementArray(x_shape) ||
-        xla::ShapeUtil::IsZeroElementArray(y_shape)) {
-      std::vector<int64> dimensions(batch_dimension_numbers.size());
-      for (int i = 0; i < batch_dimension_numbers.size(); ++i) {
-        dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]);
-      }
-      int x_outer_dim = ndims - 2;
-      int y_outer_dim = ndims - 1;
-      dimensions.push_back(x_shape.dimensions(x_outer_dim));
-      dimensions.push_back(y_shape.dimensions(y_outer_dim));
-      return xla::Broadcast(
-          xla::ConstantLiteral(builder,
-                               xla::LiteralUtil::Zero(x_shape.element_type())),
-          dimensions);
-    }
-
-    xla::PrecisionConfig precision_proto;
-    precision_proto.add_operand_precision(precision);
-    precision_proto.add_operand_precision(precision);
-
-    xla::DotDimensionNumbers dot_dnums;
-    dot_dnums.add_lhs_contracting_dimensions(x_inner_dim);
-    dot_dnums.add_rhs_contracting_dimensions(y_inner_dim);
-    for (auto batch_dimension_number : batch_dimension_numbers) {
-      dot_dnums.add_lhs_batch_dimensions(batch_dimension_number);
-      dot_dnums.add_rhs_batch_dimensions(batch_dimension_number);
-    }
-
-    return xla::DotGeneral(x, y, dot_dnums, &precision_proto);
-  });
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/lib/batch_dot.h b/tensorflow/compiler/tf2xla/lib/batch_dot.h
deleted file mode 100644
index 6cfa698593..0000000000
--- a/tensorflow/compiler/tf2xla/lib/batch_dot.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_TF2XLA_LIB_BATCH_DOT_H_
-#define TENSORFLOW_COMPILER_TF2XLA_LIB_BATCH_DOT_H_
-
-#include "tensorflow/compiler/xla/client/xla_builder.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-
-namespace tensorflow {
-
-// Multiplies slices of two tensors in batches.
-
-// Multiplies all slices of `Tensor` `x` and `y` (each slice can be
-// viewed as an element of a batch), and arranges the individual results
-// in a single output tensor of the same batch size.
-//
-// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]`
-// and `[..., r_y, c_y]`.
-//
-// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where:
-//
-//     r_o = c_x if transpose_x else r_x
-//     c_o = r_y if transpose_y else c_y
-//
-// It is computed as:
-//
-//     output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :])
-xla::XlaOp BatchDot(
-    xla::XlaOp x, xla::XlaOp y,
-    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_TF2XLA_LIB_BATCH_DOT_H_
diff --git a/tensorflow/compiler/tf2xla/lib/cholesky.cc b/tensorflow/compiler/tf2xla/lib/cholesky.cc
index dfc014395b..d580e8b330 100644
--- a/tensorflow/compiler/tf2xla/lib/cholesky.cc
+++ b/tensorflow/compiler/tf2xla/lib/cholesky.cc
@@ -18,11 +18,11 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
-#include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
 #include "tensorflow/compiler/tf2xla/lib/triangular_solve.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/lib/while_loop.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/tf2xla/lib/qr.cc b/tensorflow/compiler/tf2xla/lib/qr.cc
index 440ad13321..1b374cf58c 100644
--- a/tensorflow/compiler/tf2xla/lib/qr.cc
+++ b/tensorflow/compiler/tf2xla/lib/qr.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
-#include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/lib/while_loop.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
index df8489b62e..d2d4076401 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
-#include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
diff --git a/tensorflow/compiler/tf2xla/lib/util_test.cc b/tensorflow/compiler/tf2xla/lib/util_test.cc
index 820f9a4972..d86b01a2d5 100644
--- a/tensorflow/compiler/tf2xla/lib/util_test.cc
+++ b/tensorflow/compiler/tf2xla/lib/util_test.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #include <numeric>
 #include <vector>
 
-#include "tensorflow/compiler/tf2xla/lib/batch_dot.h"
 #include "tensorflow/compiler/xla/array2d.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/test.h"
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index dc188b67a6..cc33293a63 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -110,7 +110,11 @@ cc_library(
     deps = [
         ":arithmetic",
         ":constants",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/compiler/xla/client/lib/matrix.cc b/tensorflow/compiler/xla/client/lib/matrix.cc
index f78d8509d8..77d9ec5ece 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix.cc
@@ -21,6 +21,11 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/util.h"
 
 namespace xla {
 
@@ -71,7 +76,7 @@ XlaOp Triangle(XlaOp x, bool lower) {
         AsInt64Slice(shape.dimensions()).subspan(/*pos=*/0, /*len=*/n_dims - 2);
     auto a = Iota(builder, U32, n);
     auto b = Iota(builder, U32, m);
-    xla::XlaOp indicator;
+    XlaOp indicator;
     if (lower) {
       indicator = Ge(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0});
     } else {
@@ -87,4 +92,78 @@ XlaOp UpperTriangle(XlaOp x) { return Triangle(x, false); }
 
 XlaOp LowerTriangle(XlaOp x) { return Triangle(x, true); }
 
+XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape x_shape, builder->GetShape(x));
+    TF_ASSIGN_OR_RETURN(Shape y_shape, builder->GetShape(y));
+
+    // Check that both tensors have the same number of dimensions. There must be
+    // at least two (the batch dimensions can be empty).
+    if (ShapeUtil::Rank(x_shape) != ShapeUtil::Rank(y_shape)) {
+      return InvalidArgument(
+          "Arguments to BatchDot have different ranks: %s vs. %s",
+          ShapeUtil::HumanString(x_shape), ShapeUtil::HumanString(y_shape));
+    }
+    const int ndims = ShapeUtil::Rank(x_shape);
+    if (ndims < 2) {
+      return InvalidArgument(
+          "Arguments to BatchDot must have rank >= 2: got %d", ndims);
+    }
+
+    // The batch dimensions must be equal and the matrix dimensions must be
+    // valid.
+    std::vector<int64> batch_dimension_numbers;
+    for (int i = 0; i < ndims - 2; ++i) {
+      if (x_shape.dimensions(i) != y_shape.dimensions(i)) {
+        return InvalidArgument(
+            "Dimension %d of inputs to BatchDot must be equal: shapes %s vs %s",
+            i, ShapeUtil::HumanString(x_shape),
+            ShapeUtil::HumanString(y_shape));
+      }
+      batch_dimension_numbers.push_back(i);
+    }
+
+    int x_inner_dim = ndims - 1;
+    int y_inner_dim = ndims - 2;
+    if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) {
+      return InvalidArgument(
+          "Dimensions %d and %d of arguments to BatchDot must be equal: "
+          "shapes %s vs %s",
+          x_inner_dim, y_inner_dim, ShapeUtil::HumanString(x_shape),
+          ShapeUtil::HumanString(y_shape));
+    }
+
+    // Check for zero lhs/rhs dim size.
+    if (ShapeUtil::IsZeroElementArray(x_shape) ||
+        ShapeUtil::IsZeroElementArray(y_shape)) {
+      std::vector<int64> dimensions(batch_dimension_numbers.size());
+      for (int i = 0; i < batch_dimension_numbers.size(); ++i) {
+        dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]);
+      }
+      int x_outer_dim = ndims - 2;
+      int y_outer_dim = ndims - 1;
+      dimensions.push_back(x_shape.dimensions(x_outer_dim));
+      dimensions.push_back(y_shape.dimensions(y_outer_dim));
+      return Broadcast(
+          ConstantLiteral(builder, LiteralUtil::Zero(x_shape.element_type())),
+          dimensions);
+    }
+
+    PrecisionConfig precision_proto;
+    precision_proto.add_operand_precision(precision);
+    precision_proto.add_operand_precision(precision);
+
+    DotDimensionNumbers dot_dnums;
+    dot_dnums.add_lhs_contracting_dimensions(x_inner_dim);
+    dot_dnums.add_rhs_contracting_dimensions(y_inner_dim);
+    for (auto batch_dimension_number : batch_dimension_numbers) {
+      dot_dnums.add_lhs_batch_dimensions(batch_dimension_number);
+      dot_dnums.add_rhs_batch_dimensions(batch_dimension_number);
+    }
+
+    return DotGeneral(x, y, dot_dnums, &precision_proto);
+  });
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/matrix.h b/tensorflow/compiler/xla/client/lib/matrix.h
index 7af3e68f20..82e8514411 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.h
+++ b/tensorflow/compiler/xla/client/lib/matrix.h
@@ -40,6 +40,26 @@ XlaOp UpperTriangle(XlaOp x);
 // Get the lower triangle part of the last two dimensions
 XlaOp LowerTriangle(XlaOp x);
 
+// Multiplies slices of two tensors in batches.
+
+// Multiplies all slices of `Tensor` `x` and `y` (each slice can be
+// viewed as an element of a batch), and arranges the individual results
+// in a single output tensor of the same batch size.
+//
+// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]`
+// and `[..., r_y, c_y]`.
+//
+// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where:
+//
+//     r_o = c_x if transpose_x else r_x
+//     c_o = r_y if transpose_y else c_y
+//
+// It is computed as:
+//
+//     output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :])
+xla::XlaOp BatchDot(
+    xla::XlaOp x, xla::XlaOp y,
+    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
-- 
GitLab


From 48f1d6fa940e34365e22874479d751efa706504b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 4 Dec 2018 09:48:05 -0800
Subject: [PATCH 1281/1554] Accept const buffers as args; NFC

Makes it explicit that we do not modify arg buffers in place.

PiperOrigin-RevId: 223994560
---
 tensorflow/compiler/aot/codegen.cc                     |  2 +-
 tensorflow/compiler/aot/codegen_test_h.golden          |  6 +++---
 tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h | 10 ++++++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc
index e0ac7130a6..ab1c1be344 100644
--- a/tensorflow/compiler/aot/codegen.cc
+++ b/tensorflow/compiler/aot/codegen.cc
@@ -178,7 +178,7 @@ Status GenArgMethods(const tf2xla::Config& config,
     TF_RETURN_IF_ERROR(
         AddRewritesForShape(i, xla::Shape(ps.parameters(i)), &rewrites));
     const string code = R"(
-  void set_arg{{NAME}}_data(void* data) {
+  void set_arg{{NAME}}_data(const void* data) {
     set_arg_data({{I}}, data);
   }
   {{TYPE}}* arg{{NAME}}_data() {
diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden
index a2cdab5d1a..968afad65e 100644
--- a/tensorflow/compiler/aot/codegen_test_h.golden
+++ b/tensorflow/compiler/aot/codegen_test_h.golden
@@ -114,7 +114,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction {
   //   with dim indices specifying which value. No bounds checking is performed
   //   on dim indices.
 
-  void set_arg0_data(void* data) {
+  void set_arg0_data(const void* data) {
     set_arg_data(0, data);
   }
   float* arg0_data() {
@@ -132,7 +132,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction {
         arg_data(0)))[dim0][dim1];
   }
 
-  void set_arg_myfeed_data(void* data) {
+  void set_arg_myfeed_data(const void* data) {
     set_arg_data(0, data);
   }
   float* arg_myfeed_data() {
@@ -150,7 +150,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction {
         arg_data(0)))[dim0][dim1];
   }
 
-  void set_arg1_data(void* data) {
+  void set_arg1_data(const void* data) {
     set_arg_data(1, data);
   }
   tensorflow::int64* arg1_data() {
diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index a1d359e97c..c7341cf8b9 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -206,8 +206,14 @@ class XlaCompiledCpuFunction {
   //
   // Aliasing of argument and result buffers is not allowed, and results in
   // undefined behavior.
-  void set_arg_data(size_t index, void* data) {
-    buffer_table_[arg_index_table_[index]] = data;
+  void set_arg_data(size_t index, const void* data) {
+    // The const_cast is safe because the generated code does not write to arg
+    // buffers.
+    //
+    // buffer_table_ contains pointers to buffers that _will_ be written to by
+    // generated code so it would be misleading to make buffer_table_ a `const
+    // void**`.
+    buffer_table_[arg_index_table_[index]] = const_cast<void*>(data);
   }
 
   // ------------------------------
-- 
GitLab


From dfbbc2d4de667e0f9fe07035d0c413d3e4bd8364 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 09:48:13 -0800
Subject: [PATCH 1282/1554] Introduce a new XRTExecute flag which allows an
 exploded (in terms of its handles) tuple to be returned. This prevents
 clients which are interested in the tuple element handles to do extra RPCs to
 get them.

PiperOrigin-RevId: 223994592
---
 .../compiler/xrt/kernels/xrt_execute_op.cc    | 37 +++++++--
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 76 +++++++++++++++++++
 tensorflow/compiler/xrt/xrt.proto             |  4 +
 3 files changed, 109 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 8c6191ddc0..751329eefc 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -228,14 +228,35 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
-
-  Tensor* output_tensor;
-  TF_RETURN_IF_ERROR(
-      context->allocate_output(0, TensorShape({}), &output_tensor));
-  int64 key;
-  TF_RETURN_IF_ERROR(output_tuple->Intern(rm, &key));
-  output_tensor->scalar<int64>()() = key;
-
+  if (config_proto.return_exploded_tuple() &&
+      xla::ShapeUtil::IsTuple(output_tuple->on_device_shape())) {
+    int64 tuple_element_count =
+        xla::ShapeUtil::TupleElementCount(output_tuple->on_device_shape());
+    Tensor* output_tensor;
+    TF_RETURN_IF_ERROR(context->allocate_output(
+        0, TensorShape({tuple_element_count}), &output_tensor));
+
+    for (int64 i = 0; i < tuple_element_count; ++i) {
+      xla::ShapeIndex shape_index;
+      shape_index.push_back(i);
+
+      XRTTupleAllocation* suballocation;
+      TF_RETURN_IF_ERROR(XRTTupleAllocation::MakeSubBuffer(
+          output_tuple, shape_index, &suballocation,
+          /*alias_parent_allocation=*/false));
+      int64 key;
+      TF_RETURN_IF_ERROR(suballocation->Intern(rm, &key));
+      output_tensor->vec<int64>()(i) = key;
+    }
+    output_tuple->Unref();
+  } else {
+    Tensor* output_tensor;
+    TF_RETURN_IF_ERROR(
+        context->allocate_output(0, TensorShape({}), &output_tensor));
+    int64 key;
+    TF_RETURN_IF_ERROR(output_tuple->Intern(rm, &key));
+    output_tensor->scalar<int64>()() = key;
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index b9262c1843..aa0eabfce7 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -175,6 +175,18 @@ xla::XlaComputation AddAndTuple() {
   return builder.Build().ValueOrDie();
 }
 
+xla::XlaComputation AddAndSubTuple() {
+  xla::XlaBuilder builder("AddAndSubTuple");
+  auto p0 = xla::Parameter(&builder, 0, xla::ShapeUtil::MakeShape(xla::F32, {}),
+                           "P0");
+  auto p1 = xla::Parameter(&builder, 1, xla::ShapeUtil::MakeShape(xla::F32, {}),
+                           "P1");
+  auto sum = xla::Add(p0, p1);
+  auto sub = xla::Sub(p0, p1);
+  xla::Tuple(&builder, {sum, sub});
+  return builder.Build().ValueOrDie();
+}
+
 void StoreComputationSnapshot(const xla::XlaComputation& computation,
                               xla::HloSnapshot* dst) {
   auto snapshot = computation.Snapshot().ValueOrDie();
@@ -681,6 +693,70 @@ TEST(RawApiTest, CompileAndExecuteReturnTuple) {
   EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response));
 }
 
+TEST(RawApiTest, CompileAndExecuteReturnExplodedTuple) {
+  xrt::XLAAllocation p0;
+  p0.set_device_ordinal(0);
+  *p0.mutable_value() = xla::LiteralUtil::CreateR0<float>(12.0f).ToProto();
+
+  xrt::XLAAllocation p1;
+  p1.set_device_ordinal(0);
+  *p1.mutable_value() = xla::LiteralUtil::CreateR0<float>(3.0f).ToProto();
+
+  xrt::XLAComputation c;
+  auto config = c.mutable_config();
+  auto shapes = config->mutable_program_shape();
+  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {}).ToProto();
+  *shapes->add_parameters() = xla::ShapeUtil::MakeShape(xla::F32, {}).ToProto();
+  *shapes->mutable_result() =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {}),
+                                      xla::ShapeUtil::MakeShape(xla::F32, {})})
+          .ToProto();
+  StoreComputationSnapshot(AddAndSubTuple(), c.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(true);
+  e.set_release_compilation_handle(true);
+  e.set_return_exploded_tuple(true);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto computation =
+      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
+  auto c_handle = ops::XRTCompile(root, computation);
+  auto p0_value =
+      ops::Const(root.WithDevice("/device:CPU:0"), p0.SerializeAsString());
+  auto p0_handle = ops::XRTAllocate(root, p0_value);
+  auto p1_value =
+      ops::Const(root.WithDevice("/device:CPU:0"), p1.SerializeAsString());
+  auto p1_handle = ops::XRTAllocate(root, p1_value);
+  auto result = ops::XRTExecute(root, c_handle.handle, e_config,
+                                {Output(p0_handle), Output(p1_handle)});
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({result}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  auto handles_vec = outputs.front().vec<int64>();
+  EXPECT_EQ(handles_vec.size(), 2);
+
+  const float kResults[2] = {15.0f, 9.0f};
+  for (int64 i = 0; i < handles_vec.size(); ++i) {
+    auto read_back = ops::XRTReadLiteralAndRelease(root, Input(handles_vec(i)));
+    std::vector<Tensor> voutputs;
+    TF_EXPECT_OK(session.Run({read_back}, &voutputs));
+    EXPECT_EQ(voutputs.size(), 1);
+
+    xla::LiteralProto response;
+    EXPECT_TRUE(response.ParseFromString(voutputs[0].scalar<string>()()));
+
+    auto expected = xla::LiteralUtil::CreateR0<float>(kResults[i]);
+    EXPECT_TRUE(CompareLiteralToLiteralProto(expected, response));
+  }
+}
+
 TEST(RawApiTest, LeakCompilationReference) {
   xrt::XLAComputation c;
   auto config = c.mutable_config();
diff --git a/tensorflow/compiler/xrt/xrt.proto b/tensorflow/compiler/xrt/xrt.proto
index e149f2f435..378bb9246f 100644
--- a/tensorflow/compiler/xrt/xrt.proto
+++ b/tensorflow/compiler/xrt/xrt.proto
@@ -101,4 +101,8 @@ message XRTExecutionConfig {
   bool release_input_handles = 5;
   // If true, release the handle to the computation after running.
   bool release_compilation_handle = 6;
+  // If set to true, and the result shape is a tuple, then instead of returning
+  // a single tuple allocation the execution will return a vector of
+  // allocations, one for each of the first-level elements of the result tuple.
+  bool return_exploded_tuple = 7;
 }
-- 
GitLab


From 551248d4db31779d144c9c40cacbc2e5ad09f2a3 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Tue, 4 Dec 2018 09:49:51 -0800
Subject: [PATCH 1283/1554] Added the functionality of input_data_types flag to
 the Python API.

PiperOrigin-RevId: 223994946
---
 tensorflow/lite/python/convert.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 198612f6fe..0974bdf3d0 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -335,9 +335,11 @@ def build_toco_convert_protos(input_tensors,
   model.change_concat_input_ranges = change_concat_input_ranges
   for idx, input_tensor in enumerate(input_tensors):
     input_array = model.input_arrays.add()
+    input_array.name = tensor_name(input_tensor)
+    input_array.data_type = convert_dtype_to_tflite_type(input_tensor.dtype)
+
     if toco.inference_input_type == _types_pb2.QUANTIZED_UINT8:
       input_array.mean_value, input_array.std_value = quantized_input_stats[idx]
-    input_array.name = tensor_name(input_tensor)
     if input_shapes is None:
       shape = input_tensor.get_shape()
     else:
-- 
GitLab


From 1db31998360be29d233b66ac78fb16c3ebcdae65 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 4 Dec 2018 09:52:20 -0800
Subject: [PATCH 1284/1554] [TF:XLA] Don't ignore data_format in XLA's
 {Avg|Max}Pool3d{Grad} lowering

I didn't add a dedicated test for this because soon we'll be exercising this via
python/kernel_tests/pooling_ops_3d_test.py (that's how I found this problem).

PiperOrigin-RevId: 223995407
---
 .../compiler/tf2xla/kernels/pooling_ops.cc    | 31 ++++++++++---------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
index a259da6383..06c6cc37ec 100644
--- a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
@@ -152,7 +152,12 @@ class MaxPoolOp : public PoolingOp {
  public:
   MaxPoolOp(OpKernelConstruction* ctx, int num_spatial_dims)
       : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims,
-                  /*reduction_type=*/ctx->input_type(0)) {}
+                  /*reduction_type=*/ctx->input_type(0)) {
+    string data_format_str;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str));
+    OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_),
+                errors::InvalidArgument("Invalid data format"));
+  }
 
   void Compile(XlaOpKernelContext* ctx) override {
     auto ksize_or_error = GetKernelSize(ctx);
@@ -180,10 +185,6 @@ class MaxPool2DOp : public MaxPoolOp {
  public:
   explicit MaxPool2DOp(OpKernelConstruction* ctx)
       : MaxPoolOp(ctx, /*num_spatial_dims=*/2) {
-    string data_format_str;
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str));
-    OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_),
-                errors::InvalidArgument("Invalid data format"));
   }
 };
 REGISTER_XLA_OP(Name("MaxPool"), MaxPool2DOp);
@@ -204,7 +205,12 @@ class AvgPoolOp : public PoolingOp {
   AvgPoolOp(OpKernelConstruction* ctx, int num_spatial_dims)
       : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims,
                   /*reduction_type=*/
-                  XlaHelpers::SumAccumulationType(ctx->input_type(0))) {}
+                  XlaHelpers::SumAccumulationType(ctx->input_type(0))) {
+    string data_format_str;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str));
+    OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_),
+                errors::InvalidArgument("Invalid data format"));
+  }
 
   void Compile(XlaOpKernelContext* ctx) override {
     auto ksize_or_error = GetKernelSize(ctx);
@@ -241,10 +247,6 @@ class AvgPool2DOp : public AvgPoolOp {
  public:
   explicit AvgPool2DOp(OpKernelConstruction* ctx)
       : AvgPoolOp(ctx, /*num_spatial_dims=*/2) {
-    string data_format_str;
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format_str));
-    OP_REQUIRES(ctx, FormatFromString(data_format_str, &data_format_),
-                errors::InvalidArgument("Invalid data format"));
   }
 };
 REGISTER_XLA_OP(Name("AvgPool"), AvgPool2DOp);
@@ -390,6 +392,11 @@ class AvgPoolGradOp : public XlaOpKernel {
     OP_REQUIRES(ctx, ksize_[0] == 1 && stride_[0] == 1,
                 errors::Unimplemented(
                     "Pooling is not yet supported on the batch dimension."));
+
+    string data_format;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format));
+    OP_REQUIRES(ctx, FormatFromString(data_format, &data_format_),
+                errors::InvalidArgument("Invalid data format"));
   }
 
   int num_dims() const { return num_spatial_dims_ + 2; }
@@ -449,10 +456,6 @@ class AvgPool2DGradOp : public AvgPoolGradOp {
  public:
   explicit AvgPool2DGradOp(OpKernelConstruction* ctx)
       : AvgPoolGradOp(ctx, /*num_spatial_dims=*/2) {
-    string data_format;
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("data_format", &data_format));
-    OP_REQUIRES(ctx, FormatFromString(data_format, &data_format_),
-                errors::InvalidArgument("Invalid data format"));
   }
 };
 REGISTER_XLA_OP(
-- 
GitLab


From d8b33e9e4a9a3a02c3856279a18715efc45cbb8c Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 4 Dec 2018 09:55:22 -0800
Subject: [PATCH 1285/1554] Split conv_ops_fused

PiperOrigin-RevId: 223995825
---
 tensorflow/core/kernels/conv_ops_fused.cc     | 885 +----------------
 .../kernels/conv_ops_fused_image_transform.cc | 902 ++++++++++++++++++
 2 files changed, 911 insertions(+), 876 deletions(-)
 create mode 100644 tensorflow/core/kernels/conv_ops_fused_image_transform.cc

diff --git a/tensorflow/core/kernels/conv_ops_fused.cc b/tensorflow/core/kernels/conv_ops_fused.cc
index a0484e9235..798a7325cd 100644
--- a/tensorflow/core/kernels/conv_ops_fused.cc
+++ b/tensorflow/core/kernels/conv_ops_fused.cc
@@ -14,897 +14,30 @@ limitations under the License.
 ==============================================================================*/
 
 // Implements convolution operations with other kernels baked into the
-// processing, to optimize latency and memory usage.
+// processing, to optimize latency and memory usage:
+//  - Conv2D + BiasAdd + <Activation>
+//  - Conv2D + FusedBatchNorm + <Activation>
+//
+// Activation: Relu, Relu6, Elu, etc...
+//
+// Kernels for convolutions fused with image transformations (resize and mirror
+// padding) defined in `conv_ops_fused_image_transform.cc`.
 
 #define EIGEN_USE_THREADS
 
-#include <string.h>
-#include <map>
+#include <string>
 #include <vector>
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/framework/tensor_slice.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/conv_ops.h"
-#include "tensorflow/core/kernels/gemm_functors.h"
-#include "tensorflow/core/kernels/image_resizer_state.h"
 #include "tensorflow/core/kernels/ops_util.h"
-#include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/util/mirror_pad_mode.h"
-#include "tensorflow/core/util/padding.h"
 #include "tensorflow/core/util/tensor_format.h"
 
 namespace tensorflow {
-
-namespace {
-
-// We don't want to allocate a buffer to hold all the patches if the size is
-// going to be extremely large, so break it into chunks if it's bigger than
-// a limit. Each chunk will be processed serially, so we can refill the
-// buffer for the next chunk and reuse it, keeping maximum memory size down.
-// In this case, we've picked 16 megabytes as a reasonable limit for Android and
-// other platforms using Eigen, and 1MB for iOS devices, from experimentation.
-#if defined(__APPLE__) && defined(IS_MOBILE_PLATFORM)
-const size_t kMaxChunkSize = (1 * 1024 * 1024);
-#else
-const size_t kMaxChunkSize = (16 * 1024 * 1024);
-#endif
-const size_t kResizeCacheSize = (8 * 1024 * 1024);
-
-// Lookup method used when resizing.
-enum SamplingMode {
-  BILINEAR = 0,
-  NEAREST = 1,
-};
-
-// Simple utility function used by FusedConv to multithread basic workloads. To
-// use it, pass begin and end values for the full workload and a std::function
-// that receives a subset of that through the begin and end values for each
-// worker's task. The division of the full workload into worker tasks is handled
-// by the multithreading logic. Here's an example of how to use it:
-// std::vector<float> my_vector(100);
-// ...
-// FusedConvParallelFor(context, 0, 100,
-//   [&my_vector](int64 task_begin, int64 task_end) {
-//     for (int64 current = task_begin; current != task_end; ++current) {
-//       my_vector[current] *= 10.0f;
-//     }
-// });
-void FusedConvParallelFor(
-    OpKernelContext* context, int64 begin, int64 end,
-    const std::function<void(int64, int64)>& task_function) {
-// On iOS, the thread management imposes a very big performance penalty, so
-// just call the function directly with no multithreading.
-#if defined(__APPLE__) && defined(IS_MOBILE_PLATFORM)
-  task_function(begin, end);
-#else
-  auto& worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
-  thread::ThreadPool* thread_pool = worker_threads.workers;
-  const int64 total_elements = end - begin;
-  // This is a bit of an arbitrary number, but was found to work well for
-  // typical models we've been profiling on various devices.
-  const int64 element_cost = 10000000;
-  thread_pool->ParallelFor(
-      total_elements, element_cost,
-      [begin, task_function](int64 begin_offset, int64 end_offset) {
-        const int64 task_begin = begin + begin_offset;
-        const int64 task_end = begin + end_offset;
-        task_function(task_begin, task_end);
-      });
-#endif
-}
-
-// Holds the state needed for the resizing subtasks.
-template <class T1>
-struct ResizeTaskParameters {
-  ResizeTaskParameters() : st(false) {}
-
-  int cache_height;
-  T1* resize_cache;
-  int cache_line_width;
-  int input_width;
-  int input_depth;
-  int top_padding;
-  int pad_offset;
-  int64 resized_height;
-  ImageResizerState st;
-  const T1* input_batch_start;
-  int64 cache_start_x;
-  int64 cache_end_x;
-  int left_padding;
-  int64 resized_width;
-  int64 padded_width;
-  int64 padded_height;
-};
-
-template <class T1>
-struct PerCacheLineParameters {
-  PerCacheLineParameters() {}
-  PerCacheLineParameters(const PerCacheLineParameters<T1>& other)
-      : cache_line_start(other.cache_line_start),
-        input_top_row_start(other.input_top_row_start),
-        input_bottom_row_start(other.input_bottom_row_start),
-        y_lerp(other.y_lerp) {}
-
-  T1* cache_line_start;
-  const T1* input_top_row_start;
-  const T1* input_bottom_row_start;
-  T1 y_lerp;
-};
-
-// Helper class to simplify bilinear filtering
-template <class T1>
-struct SampleRect {
-  EIGEN_ALWAYS_INLINE SampleRect(const T1* in_top_left, const T1* in_top_right,
-                                 const T1* in_bottom_left,
-                                 const T1* in_bottom_right)
-      : top_left(in_top_left),
-        top_right(in_top_right),
-        bottom_left(in_bottom_left),
-        bottom_right(in_bottom_right) {}
-
-  EIGEN_ALWAYS_INLINE T1 BilinearSample(int channel, T1 x_lerp,
-                                        T1 y_lerp) const {
-    const T1 top =
-        top_left[channel] + (top_right[channel] - top_left[channel]) * x_lerp;
-    const T1 bottom = bottom_left[channel] +
-                      (bottom_right[channel] - bottom_left[channel]) * x_lerp;
-    return top + (bottom - top) * y_lerp;
-  }
-
-  const T1* top_left;
-  const T1* top_right;
-  const T1* bottom_left;
-  const T1* bottom_right;
-};
-
-// Calculates parameters which remain constant through a resize cache row.
-template <class T1>
-EIGEN_ALWAYS_INLINE PerCacheLineParameters<T1> CalculatePerCacheLineParameters(
-    int64 cache_height, int64 cache_y, T1* resize_cache, int64 cache_line_width,
-    int64 input_width, int64 input_depth, int64 top_padding, int64 pad_offset,
-    int64 resized_height, const ImageResizerState& st,
-    const T1* input_batch_start) {
-  PerCacheLineParameters<T1> result;
-  // The cache is organized so that the real y values of the resized image map
-  // onto the actual cache values through a modulo scheme. This means that as we
-  // progress downwards through the image, we keep reusing a small cache and so
-  // keep memory usage down.
-  int64 cache_index_y;
-  if (cache_y < 0) {
-    cache_index_y = cache_height + (cache_y % cache_height);
-  } else {
-    cache_index_y = cache_y % cache_height;
-  }
-  result.cache_line_start =
-      resize_cache + (cache_index_y * cache_line_width * input_depth);
-  // This part is implementing the mirror padding that happens before resizing.
-  float in_y = (cache_y - top_padding);
-  if (in_y < 0) {
-    in_y = -(in_y + 1.0f - pad_offset);
-  } else if (in_y >= resized_height) {
-    in_y = (resized_height * 2.0f) - (in_y + 1.0f + pad_offset);
-  }
-  // Here's where do do the actual resize.
-  in_y *= st.height_scale;
-  const int64 top_y_index = static_cast<int64>(std::floor(in_y));
-  const int64 bottom_y_index =
-      std::min(static_cast<int64>(std::ceil(in_y)), (st.in_height - 1));
-  // Lerp is used for bilinear filtering when that's needed.
-  result.y_lerp = static_cast<T1>(in_y - top_y_index);
-  // Which rows of the original input image to pull the values from.
-  result.input_top_row_start =
-      input_batch_start + (top_y_index * input_width * input_depth);
-  result.input_bottom_row_start =
-      input_batch_start + (bottom_y_index * input_width * input_depth);
-  return result;
-}
-
-template <class T1>
-struct PerCachePixelParameters {
-  PerCachePixelParameters() {}
-  PerCachePixelParameters(const PerCachePixelParameters<T1>& other)
-      : cache_line_pixel(other.cache_line_pixel),
-        left_x_index(other.left_x_index),
-        right_x_index(other.right_x_index),
-        x_lerp(other.x_lerp) {}
-
-  T1* cache_line_pixel;
-  int64 left_x_index;
-  int64 right_x_index;
-  T1 x_lerp;
-};
-
-// Pulls out common parameters used for every resized pixel.
-template <class T1>
-EIGEN_ALWAYS_INLINE PerCachePixelParameters<T1>
-CalculatePerCachePixelParameters(int64 cache_x, int64 cache_start_x,
-                                 T1* cache_line_start, int64 input_depth,
-                                 int64 left_padding, int64 pad_offset,
-                                 int64 resized_width,
-                                 const ImageResizerState& st) {
-  PerCachePixelParameters<T1> result;
-  // Figure out where we're going to store the results of our transform.
-  const int cache_index_x = cache_x - cache_start_x;
-  result.cache_line_pixel = cache_line_start + (cache_index_x * input_depth);
-  // Implement mirror padding by flipping in_x if it's off the edge.
-  float in_x = (cache_x - left_padding);
-  if (in_x < 0) {
-    in_x = -(in_x + 1.0f - pad_offset);
-  } else if (in_x >= resized_width) {
-    in_x = (resized_width * 2.0f) - (in_x + 1.0f + pad_offset);
-  }
-  // Resize the x parameters.
-  in_x *= st.width_scale;
-  // Get the x coordinates for the left and right pixels to pull from.
-  result.left_x_index = static_cast<int64>(std::floor(in_x));
-  result.right_x_index =
-      std::min(static_cast<int64>(std::ceil(in_x)), (st.in_width - 1));
-  // This x_lerp is used to blend pixels in bilinear filtering.
-  result.x_lerp = static_cast<T1>(in_x - result.left_x_index);
-  return result;
-}
-
-// Combines bilinear resizing and mirror padding into the im2col transformation
-// stage of convolution.
-template <class T1, class T2, class T3, class TGemmFunctor,
-          SamplingMode SampleMode>
-class FusedResizeAndPadConvFunctor {
- public:
-  void operator()(OpKernelContext* context, const Tensor& input,
-                  int input_batches, int resized_height, int resized_width,
-                  int padded_height, int padded_width, int input_depth,
-                  const T2* filter_data, int filter_height, int filter_width,
-                  int filter_count, int stride_rows, int stride_cols,
-                  Padding padding, T3* output_data, int output_height,
-                  int output_width, const ImageResizerState& st,
-                  int top_padding, int bottom_padding, int left_padding,
-                  int right_padding, int pad_offset) {
-    if ((input_batches <= 0) || (padded_width <= 0) || (padded_height <= 0) ||
-        (input_depth <= 0)) {
-      LOG(WARNING) << "Conv2D was called with bad input dimensions: "
-                   << input_batches << ", " << padded_height << ", "
-                   << padded_width << ", " << input_depth;
-      return;
-    }
-    if ((filter_width <= 0) || (filter_height <= 0) || (filter_count <= 0)) {
-      LOG(WARNING) << "Conv2D was called with bad filter dimensions: "
-                   << filter_width << ", " << filter_height << ", "
-                   << filter_count;
-      return;
-    }
-    if ((output_width <= 0) || (output_height <= 0)) {
-      LOG(WARNING) << "Conv2D was called with bad output width or height: "
-                   << output_width << ", " << output_height;
-      return;
-    }
-    OP_REQUIRES(
-        context, ((SampleMode == NEAREST) || (SampleMode == BILINEAR)),
-        errors::InvalidArgument("Bad sample mode passed in", SampleMode));
-
-    // These calculations define how the patches will be positioned within the
-    // input image. The actual definitions are quite complex, and rely on the
-    // previously-calculated output size.
-    int filter_left_offset;
-    int filter_top_offset;
-    if (padding == VALID) {
-      filter_left_offset =
-          ((output_width - 1) * stride_cols + filter_width - padded_width + 1) /
-          2;
-      filter_top_offset = ((output_height - 1) * stride_rows + filter_height -
-                           padded_height + 1) /
-                          2;
-    } else {
-      filter_left_offset =
-          ((output_width - 1) * stride_cols + filter_width - padded_width) / 2;
-      filter_top_offset =
-          ((output_height - 1) * stride_rows + filter_height - padded_height) /
-          2;
-    }
-
-    ResizeTaskParameters<T1> task_params;
-    task_params.input_depth = input_depth;
-    task_params.top_padding = top_padding;
-    task_params.pad_offset = pad_offset;
-    task_params.resized_height = resized_height;
-    task_params.st = st;
-    task_params.left_padding = left_padding;
-    task_params.resized_width = resized_width;
-    task_params.padded_width = padded_width;
-    task_params.padded_height = padded_height;
-
-    // The im2col buffer has # of patches rows, and # of filters cols.
-    // It's laid out like this, in row major order in memory:
-    //        < filter value count >
-    //   ^   +---------------------+
-    // patch |                     |
-    // count |                     |
-    //   v   +---------------------+
-    // Each patch row contains a filter_width x filter_height patch of the
-    // input, with the depth channel as the most contiguous in memory, followed
-    // by the width, then the height. This is the standard memory order in the
-    // image world if it helps to visualize it.
-    const int filter_value_count = filter_width * filter_height * input_depth;
-
-    OP_REQUIRES(context, (filter_value_count * sizeof(T1)) <= kMaxChunkSize,
-                errors::InvalidArgument("Im2Col patch too large for buffer"));
-    const size_t patches_per_chunk =
-        kMaxChunkSize / (filter_value_count * sizeof(T1));
-    // Because memory allocation is very expensive on mobile platforms, try to
-    // allocate a persistent buffer that will be kept around between calls. We
-    // use TensorFlow's resource management to ensure that the memory will be
-    // released when the session is over.
-    Im2ColBufferResource<T1, kMaxChunkSize>* im2col_buffer_resource;
-    std::function<Status(Im2ColBufferResource<T1, kMaxChunkSize>**)> creator =
-        [](Im2ColBufferResource<T1, kMaxChunkSize>** resource) {
-          *resource = new Im2ColBufferResource<T1, kMaxChunkSize>();
-          return Status::OK();
-        };
-    OP_REQUIRES_OK(context, context->resource_manager()->LookupOrCreate(
-                                "Conv2d", "im2col_buffer",
-                                &im2col_buffer_resource, creator));
-
-    // Create a resize cache memory buffer that will hold the rows of
-    // transformed and mirror padded input pixels, ready to be copied
-    // into filter patches by im2col.
-    // It's laid out like this, in row major order in memory:
-    //         < cache line width >
-    //   ^    +--------------------+
-    // cache  |                    |
-    // height |                    |
-    //   v    +--------------------+
-    // Each cache row contains a cache_line_width number of resized pixels,
-    // each with input_depth channels. The cache height is typically less than
-    // the full height the resized image would be, so it's filled up
-    // incrementally as we progress downwards through the input creating im2col
-    // patches.
-    task_params.cache_start_x = -filter_left_offset;
-    task_params.cache_end_x =
-        (((output_width - 1) * stride_cols) - filter_left_offset) +
-        filter_width;
-    task_params.cache_line_width =
-        task_params.cache_end_x - task_params.cache_start_x;
-    task_params.cache_height =
-        kResizeCacheSize / (task_params.cache_line_width * input_depth);
-    const int needed_resize_cache_count =
-        filter_height * task_params.cache_line_width * input_depth;
-    OP_REQUIRES(context,
-                (needed_resize_cache_count * sizeof(T1)) <= kResizeCacheSize,
-                errors::InvalidArgument("Input too large for resize cache"));
-    Im2ColBufferResource<T1, kResizeCacheSize>* resize_cache_resource;
-    std::function<Status(Im2ColBufferResource<T1, kResizeCacheSize>**)>
-        resize_creator =
-            [](Im2ColBufferResource<T1, kResizeCacheSize>** resource) {
-              *resource = new Im2ColBufferResource<T1, kResizeCacheSize>();
-              return Status::OK();
-            };
-    OP_REQUIRES_OK(context, context->resource_manager()->LookupOrCreate(
-                                "Conv2d", "resize_cache",
-                                &resize_cache_resource, resize_creator));
-
-    // This means that multiple ops can't be run simultaneously on different
-    // threads, because we have a single shared resource. The platforms this is
-    // aimed at have intra-op parallelism as their focus though, so it shouldn't
-    // be an issue.
-    mutex_lock lock_buffer(im2col_buffer_resource->mu);
-    core::ScopedUnref unref_buffer(im2col_buffer_resource);
-    T1* im2col_buffer = im2col_buffer_resource->data;
-
-    // This buffer is used as a fairly heavy-weight cache for the resized and
-    // mirrored inputs to the im2col operation. The problem is that we want to
-    // keep the memory usage down by not rendering the fully resized and padded
-    // input tensor to the convolution into an entire buffer. The first approach
-    // to avoid this was to fold the bilinear filtering and padding spatial
-    // transformations into the im2col lookup itself. This successfully reduced
-    // memory usage, but because im2col can access an individual pixel for many
-    // different patches, the extra overhead of doing the same bilinear lookups
-    // repeatedly became too expensive.
-    // The resize cache is designed to avoid this problem by keeping a
-    // horizontal slice of the resized and padded input to the im2col
-    // precalculated, so that repeated accesses to the same pixel from different
-    // filter patches can just be copied from this cache. It's organized as a
-    // horizontal slice stretching across the whole virtual image, and as high
-    // as the filter window, so that as the patch processing moves across all
-    // the pixels are present, and before a new row of patches is started any
-    // previously calculated rows that are needed are maintained, with new rows
-    // calculated as required.
-    mutex_lock resize_lock_buffer(resize_cache_resource->mu);
-    core::ScopedUnref unref_resized_cache(resize_cache_resource);
-    task_params.resize_cache = resize_cache_resource->data;
-
-    const T1* input_data = input.flat<T1>().data();
-    const int64 input_height = input.shape().dim_sizes()[1];
-    task_params.input_width = input.shape().dim_sizes()[2];
-
-    int end_cached_lines = std::numeric_limits<int>::min();
-
-    for (int batch = 0; batch < input_batches; ++batch) {
-      task_params.input_batch_start =
-          input_data +
-          (batch * input_height * task_params.input_width * input_depth);
-      const int in_y_end =
-          ((output_height * stride_rows) - filter_top_offset) + filter_height;
-      for (int out_y = 0; out_y < output_height; ++out_y) {
-        const int in_y_origin = (out_y * stride_rows) - filter_top_offset;
-        const int cache_start_y = std::max(in_y_origin, end_cached_lines);
-        const int cache_end_y = std::min(
-            in_y_end, std::max((in_y_origin + task_params.cache_height),
-                               end_cached_lines));
-        if (end_cached_lines < (in_y_origin + filter_height)) {
-          // This call breaks up the work required for calculating the mirror
-          // padding and resizing across multiple threads.
-          FusedConvParallelFor(
-              context, cache_start_y, cache_end_y,
-              [task_params](int64 task_cache_start_y, int64 task_cache_end_y) {
-                // This is a long and confusing function, but it's been laid out
-                // this way to help with performance on some intensive models.
-                // What it's doing is populating a cache of the original input
-                // image, after it's been bilinear resized and had its edges
-                // mirrored. This allows the following im2col code to access the
-                // transformed pixels from this cache, without having to
-                // repeatedly apply the expensive bilinear calculations as the
-                // same pixels are accessed by different patches.
-                // This is most effective when the stride is small and the
-                // filter size is large, since that's when pixels are reused
-                // most frequently as patches overlap.
-                for (int cache_y = task_cache_start_y;
-                     cache_y < task_cache_end_y; ++cache_y) {
-                  // We organize the cache as a series of rows, each containing
-                  // all the transformed pixels for a given line in the image.
-                  // This cache is big enough to hold at least a filter's height
-                  // worth of rows, but typically more, limited by the size of
-                  // the cache buffer.
-                  // We don't allocate an entire image's worth of rows though,
-                  // because we're trying to keep memory usage down, so as we
-                  // progress downwards through the im2col we periodically
-                  // refresh the cache so that the next lines that are needed
-                  // for that operation are always present.
-                  // Work out the parameters that remain constant across the
-                  // row we're calculating.
-                  PerCacheLineParameters<T1> line_params(
-                      CalculatePerCacheLineParameters<T1>(
-                          task_params.cache_height, cache_y,
-                          task_params.resize_cache,
-                          task_params.cache_line_width, task_params.input_width,
-                          task_params.input_depth, task_params.top_padding,
-                          task_params.pad_offset, task_params.resized_height,
-                          task_params.st, task_params.input_batch_start));
-                  // Iterate through the resize cache row we're filling in.
-                  for (int cache_x = task_params.cache_start_x;
-                       cache_x < task_params.cache_end_x; ++cache_x) {
-                    // Figure out what we need for the cache pixel we're
-                    // populating.
-                    PerCachePixelParameters<T1> pixel_params(
-                        CalculatePerCachePixelParameters<T1>(
-                            cache_x, task_params.cache_start_x,
-                            line_params.cache_line_start,
-                            task_params.input_depth, task_params.left_padding,
-                            task_params.pad_offset, task_params.resized_width,
-                            task_params.st));
-                    // If the access is off the left, right, top, or bottom of
-                    // the resized image, the conv padding means we should set
-                    // it to zero.
-                    if ((cache_x < 0) ||
-                        (cache_x >= task_params.padded_width) ||
-                        (cache_y < 0) ||
-                        (cache_y >= task_params.padded_height)) {
-                      std::fill_n(pixel_params.cache_line_pixel,
-                                  task_params.input_depth, T1(0));
-                    } else {
-                      // There are two different sampling strategies for
-                      // resizing. When using nearest, we can just do a
-                      // straight copy of the pixel closest to our sample point,
-                      // but bilinear requires a more complex calculation.
-                      if (SampleMode == NEAREST) {
-                        const T1* input_top_left_pixel =
-                            line_params.input_top_row_start +
-                            (pixel_params.left_x_index *
-                             task_params.input_depth);
-
-                        std::copy_n(input_top_left_pixel,
-                                    task_params.input_depth,
-                                    pixel_params.cache_line_pixel);
-                      } else {
-                        const SampleRect<T1> rect(
-                            line_params.input_top_row_start +
-                                (pixel_params.left_x_index *
-                                 task_params.input_depth),
-                            line_params.input_top_row_start +
-                                (pixel_params.right_x_index *
-                                 task_params.input_depth),
-                            line_params.input_bottom_row_start +
-                                (pixel_params.left_x_index *
-                                 task_params.input_depth),
-                            line_params.input_bottom_row_start +
-                                (pixel_params.right_x_index *
-                                 task_params.input_depth));
-                        for (int in_channel = 0;
-                             in_channel < task_params.input_depth;
-                             ++in_channel) {
-                          pixel_params.cache_line_pixel[in_channel] =
-                              rect.BilinearSample(in_channel,
-                                                  pixel_params.x_lerp,
-                                                  line_params.y_lerp);
-                        }
-                      }
-                    }
-                  }
-                }
-              });
-          end_cached_lines = cache_end_y;
-        }
-        for (int out_x = 0; out_x < output_width; ++out_x) {
-          const int in_x_origin = (out_x * stride_cols) - filter_left_offset;
-          const int patch_index = (batch * output_width * output_height) +
-                                  (out_y * output_width) + out_x;
-          const int patch_index_within_chunk = patch_index % patches_per_chunk;
-          T1* im2col_patch_start =
-              im2col_buffer + (patch_index_within_chunk * filter_value_count);
-          for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
-            T1* im2col_row_start =
-                im2col_patch_start +
-                (filter_y * filter_width * task_params.input_depth);
-            const int conv_in_y = in_y_origin + filter_y;
-            int cache_index_y;
-            if (conv_in_y < 0) {
-              cache_index_y = task_params.cache_height +
-                              (conv_in_y % task_params.cache_height);
-            } else {
-              cache_index_y = conv_in_y % task_params.cache_height;
-            }
-            T1* cache_line_start =
-                task_params.resize_cache +
-                (cache_index_y * task_params.cache_line_width *
-                 task_params.input_depth);
-            T1* cache_filter_row_start =
-                cache_line_start + ((in_x_origin - task_params.cache_start_x) *
-                                    task_params.input_depth);
-            std::copy_n(cache_filter_row_start,
-                        (filter_width * task_params.input_depth),
-                        im2col_row_start);
-          }
-          const bool is_last_in_chunk =
-              (patch_index_within_chunk == (patches_per_chunk - 1));
-          const bool is_last_overall =
-              ((batch == (input_batches - 1)) &&
-               (out_y == (output_height - 1)) && (out_x == (output_width - 1)));
-          if (is_last_in_chunk || is_last_overall) {
-            // Now we've assembled a set of image patches into a matrix, apply
-            // a GEMM matrix multiply of the patches as rows, times the filter
-            // weights in columns, to get partial results in the output
-            // matrix.
-            const int how_many_patches = patch_index_within_chunk + 1;
-            const int m = how_many_patches;
-            const int n = filter_count;
-            const int k = filter_value_count;
-            const int lda = filter_value_count;
-            const int ldb = filter_count;
-            const int ldc = filter_count;
-            const size_t start_patch_index =
-                patch_index - (how_many_patches - 1);
-            T3* chunk_output_data =
-                output_data + (start_patch_index * filter_count);
-            TGemmFunctor gemm_functor;
-            gemm_functor(context, m, n, k, im2col_buffer, lda, filter_data, ldb,
-                         chunk_output_data, ldc);
-          }
-        }
-      }
-    }
-  }
-};
-
-}  // namespace
-
-// Implements a version of convolution with bilinear resizing and mirror padding
-// included.
-template <class T, class TConvFunctor, bool DoResize>
-class FusedResizeConv2DUsingGemmOp : public OpKernel {
- public:
-  explicit FusedResizeConv2DUsingGemmOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    if (DoResize) {
-      OP_REQUIRES_OK(context,
-                     context->GetAttr("resize_align_corners", &align_corners_));
-    }
-    MirrorPadMode mode;
-    OP_REQUIRES_OK(context, context->GetAttr("mode", &mode));
-
-    switch (mode) {
-      case MirrorPadMode::SYMMETRIC: {
-        offset_ = 0;
-        break;
-      }
-      case MirrorPadMode::REFLECT: {
-        offset_ = 1;
-        break;
-      }
-      default:
-        OP_REQUIRES(context, false,
-                    errors::InvalidArgument(
-                        "mode must be either REFLECT or SYMMETRIC."));
-    }
-    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
-    OP_REQUIRES(context, strides_.size() == 4,
-                errors::InvalidArgument("Sliding window strides field must "
-                                        "specify 4 dimensions"));
-    const int64 stride_n = GetTensorDim(strides_, FORMAT_NHWC, 'N');
-    const int64 stride_c = GetTensorDim(strides_, FORMAT_NHWC, 'C');
-    OP_REQUIRES(
-        context, stride_n == 1 && stride_c == 1,
-        errors::InvalidArgument("Current implementation does not yet support "
-                                "strides in the batch and depth dimensions."));
-    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    // Input tensor is of the following dimensions:
-    // [ batch, in_rows, in_cols, in_depth ]
-    const Tensor& input = context->input(0);
-    OP_REQUIRES(context, (input.shape().num_elements() > 0),
-                errors::InvalidArgument("Input tensor can't be empty"));
-
-    ImageResizerState st(false);
-    if (DoResize) {
-      st = ImageResizerState(align_corners_);
-      st.ValidateAndCalculateOutputSize(context, input);
-      if (!context->status().ok()) return;
-    } else {
-      // Set up the resize parameters to do no scaling at all.
-      st.batch_size = input.dim_size(0);
-      st.out_height = input.dim_size(1);
-      st.out_width = input.dim_size(2);
-      st.in_height = input.dim_size(1);
-      st.in_width = input.dim_size(2);
-      st.channels = input.dim_size(3);
-      st.height_scale = 1.0f;
-      st.width_scale = 1.0f;
-    }
-    TensorShape resized_shape(
-        {input.dim_size(0), st.out_height, st.out_width, input.dim_size(3)});
-    int paddings_index;
-    int filter_index;
-    if (DoResize) {
-      paddings_index = 2;
-      filter_index = 3;
-    } else {
-      paddings_index = 1;
-      filter_index = 2;
-    }
-    const Tensor& paddings = context->input(paddings_index);
-
-    const int dims = resized_shape.dims();
-    OP_REQUIRES(
-        context,
-        TensorShapeUtils::IsMatrix(paddings.shape()) &&
-            paddings.dim_size(1) == 2,
-        errors::InvalidArgument("paddings must be a matrix with 2 columns: ",
-                                paddings.shape().DebugString()));
-    const int fixed_dims =
-        (allow_legacy_scalars() && dims == 0 && paddings.dim_size(0) == 1)
-            ? 1
-            : dims;
-    OP_REQUIRES(
-        context, fixed_dims == paddings.dim_size(0),
-        errors::InvalidArgument(
-            "The first dimension of paddings must be the rank of inputs: ",
-            fixed_dims, " ", paddings.shape().DebugString(), " ",
-            resized_shape.DebugString()));
-    OP_REQUIRES(
-        context, dims == paddings.dim_size(0),
-        errors::InvalidArgument(
-            "The first dimension of paddings must be the rank of inputs: ",
-            dims, " ", paddings.shape().DebugString(), " ",
-            resized_shape.DebugString()));
-
-    OP_REQUIRES(
-        context, dims == 4,
-        errors::InvalidArgument(
-            "Fused mirror padding only supports four-dimensional inputs, but ",
-            dims, " requested"));
-
-    // Compute the shape of the output tensor, and allocate it.
-    TensorShape padded_shape;
-    TTypes<int32>::ConstMatrix paddings_matrix = paddings.matrix<int32>();
-    for (int d = 0; d < dims; ++d) {
-      const int32 before =
-          paddings_matrix(d, 0);  // Pad before existing elements.
-      const int32 after =
-          paddings_matrix(d, 1);  // Pad after existing elements.
-      OP_REQUIRES(context, before >= 0 && after >= 0,
-                  errors::InvalidArgument(
-                      "paddings must be non-negative: ", before, " ", after));
-      if (offset_ == 0) {  // SYMMETRIC mode.
-        OP_REQUIRES(
-            context,
-            before <= resized_shape.dim_size(d) &&
-                after <= resized_shape.dim_size(d),
-            errors::InvalidArgument("paddings must be no greater "
-                                    "than the dimension size: ",
-                                    before, ", ", after, " greater than ",
-                                    resized_shape.dim_size(d)));
-      } else if (offset_ == 1) {  // REFLECT mode.
-        OP_REQUIRES(
-            context,
-            before < resized_shape.dim_size(d) &&
-                after < resized_shape.dim_size(d),
-            errors::InvalidArgument("paddings must be less than"
-                                    " the dimension size: ",
-                                    before, ", ", after, " not less than ",
-                                    resized_shape.dim_size(d)));
-      }
-      padded_shape.AddDim(before + resized_shape.dim_size(d) + after);
-    }
-
-    OP_REQUIRES(
-        context, ((paddings_matrix(0, 0) == 0) && (paddings_matrix(0, 1) == 0)),
-        errors::InvalidArgument(
-            "Fused mirror padding only support spatial padding, not batches: ",
-            paddings.DebugString()));
-    OP_REQUIRES(
-        context, ((paddings_matrix(3, 0) == 0) && (paddings_matrix(3, 1) == 0)),
-        errors::InvalidArgument(
-            "Fused mirror padding only support spatial padding, not channels: ",
-            paddings.DebugString()));
-    const int32 top_padding = paddings_matrix(1, 0);
-    const int32 bottom_padding = paddings_matrix(1, 1);
-    const int32 left_padding = paddings_matrix(2, 0);
-    const int32 right_padding = paddings_matrix(2, 1);
-
-    // Input filter is of the following dimensions:
-    // [ filter_rows, filter_cols, in_depth, out_depth]
-    const Tensor& filter = context->input(filter_index);
-
-    // For 2D convolution, there should be 4 dimensions.
-    OP_REQUIRES(context, padded_shape.dims() == 4,
-                errors::InvalidArgument("input must be 4-dimensional",
-                                        padded_shape.DebugString()));
-    OP_REQUIRES(context, filter.dims() == 4,
-                errors::InvalidArgument("filter must be 4-dimensional: ",
-                                        filter.shape().DebugString()));
-
-    // We only check the first three dims, since the depth is accessed as an
-    // int64 below.
-    for (int i = 0; i < 3; i++) {
-      OP_REQUIRES(
-          context,
-          FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()),
-          errors::InvalidArgument("filter too large"));
-    }
-
-    // The last dimension for input is in_depth. It must be the same as the
-    // filter's in_depth.
-    const int64 in_depth = padded_shape.dim_size(3);
-    OP_REQUIRES(context, in_depth == filter.dim_size(2),
-                errors::InvalidArgument(
-                    "input and filter must have the same depth: ", in_depth,
-                    " vs ", filter.dim_size(2)));
-
-    // The last dimension for filter is out_depth.
-    const int out_depth = static_cast<int>(filter.dim_size(3));
-
-    // The second dimension for input is rows/height.
-    // The first dimension for filter is rows/height.
-    const int64 padded_rows_raw = padded_shape.dim_size(1);
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(padded_rows_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input rows too large"));
-    const int padded_rows = static_cast<int>(padded_rows_raw);
-    const int filter_rows = static_cast<int>(filter.dim_size(0));
-    const int resized_rows = static_cast<int>(resized_shape.dim_size(1));
-
-    // The third dimension for input is columns/width.
-    // The second dimension for filter is columns/width.
-    const int64 padded_cols_raw = padded_shape.dim_size(2);
-    OP_REQUIRES(
-        context,
-        FastBoundsCheck(padded_cols_raw, std::numeric_limits<int>::max()),
-        errors::InvalidArgument("Input cols too large"));
-    const int padded_cols = static_cast<int>(padded_cols_raw);
-    const int filter_cols = static_cast<int>(filter.dim_size(1));
-    const int resized_cols = static_cast<int>(resized_shape.dim_size(2));
-
-    // The first dimension for input is batch.
-    const int64 batch_raw = padded_shape.dim_size(0);
-    OP_REQUIRES(context,
-                FastBoundsCheck(batch_raw, std::numeric_limits<int>::max()),
-                errors::InvalidArgument("batch is too large"));
-    const int batch = static_cast<int>(batch_raw);
-
-    // For now we take the stride from the second and third dimensions only (we
-    // do not support striding on the batch or depth dimension).
-    const int stride_rows = GetTensorDim(strides_, FORMAT_NHWC, 'H');
-    const int stride_cols = GetTensorDim(strides_, FORMAT_NHWC, 'W');
-
-    int64 out_rows = 0, out_cols = 0, pad_rows = 0, pad_cols = 0;
-    OP_REQUIRES_OK(context,
-                   GetWindowedOutputSize(padded_rows, filter_rows, stride_rows,
-                                         padding_, &out_rows, &pad_rows));
-    OP_REQUIRES_OK(context,
-                   GetWindowedOutputSize(padded_cols, filter_cols, stride_cols,
-                                         padding_, &out_cols, &pad_cols));
-    TensorShape out_shape =
-        ShapeFromFormat(FORMAT_NHWC, batch, out_rows, out_cols, out_depth);
-    OP_REQUIRES(context, (out_shape.num_elements() > 0),
-                errors::InvalidArgument("Output tensor can't be empty"));
-
-    // Output tensor is of the following dimensions:
-    // [ in_batch, out_rows, out_cols, out_depth ]
-    Tensor* output = nullptr;
-    OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output));
-
-    VLOG(2) << "FusedConv2D: " << name() << ", in_depth = " << in_depth
-            << ", padded_cols = " << padded_cols
-            << ", resized_cols = " << resized_cols
-            << ", filter_cols = " << filter_cols
-            << ", padded_rows = " << padded_rows
-            << ", resized_rows = " << resized_rows
-            << ", filter_rows = " << filter_rows
-            << ", stride_rows = " << stride_rows
-            << ", stride_cols = " << stride_cols
-            << ", out_depth = " << out_depth << ", DoResize=" << DoResize;
-
-    // If there is nothing to compute, return.
-    if (out_shape.num_elements() == 0) {
-      return;
-    }
-    TConvFunctor conv_functor;
-    conv_functor(context, input, batch, resized_rows, resized_cols, padded_rows,
-                 padded_cols, in_depth, filter.flat<T>().data(), filter_rows,
-                 filter_cols, out_depth, stride_rows, stride_cols, padding_,
-                 output->flat<T>().data(), out_rows, out_cols, st, top_padding,
-                 bottom_padding, left_padding, right_padding, offset_);
-  }
-
- private:
-  std::vector<int32> strides_;
-  Padding padding_;
-  bool align_corners_;
-  int offset_;
-
-  TF_DISALLOW_COPY_AND_ASSIGN(FusedResizeConv2DUsingGemmOp);
-};
-
-#define REGISTER_FUSED(T)                                                 \
-  REGISTER_KERNEL_BUILDER(                                                \
-      Name("FusedResizeAndPadConv2D")                                     \
-          .Device(DEVICE_CPU)                                             \
-          .TypeConstraint<T>("T"),                                        \
-      FusedResizeConv2DUsingGemmOp<                                       \
-          T,                                                              \
-          FusedResizeAndPadConvFunctor<T, T, T, FastGemmFunctor<T, T, T>, \
-                                       BILINEAR>,                         \
-          true>);
-
-TF_CALL_half(REGISTER_FUSED);
-TF_CALL_float(REGISTER_FUSED);
-TF_CALL_double(REGISTER_FUSED);
-
-#define REGISTER_PAD_ONLY_FUSED(T)                                        \
-  REGISTER_KERNEL_BUILDER(                                                \
-      Name("FusedPadConv2D").Device(DEVICE_CPU).TypeConstraint<T>("T"),   \
-      FusedResizeConv2DUsingGemmOp<                                       \
-          T,                                                              \
-          FusedResizeAndPadConvFunctor<T, T, T, FastGemmFunctor<T, T, T>, \
-                                       NEAREST>,                          \
-          false>);
-
-TF_CALL_half(REGISTER_PAD_ONLY_FUSED);
-TF_CALL_float(REGISTER_PAD_ONLY_FUSED);
-TF_CALL_double(REGISTER_PAD_ONLY_FUSED);
-
-// Support for fusing computationally cheap, but memory bandwidth expensive
-// computations into the output of convolution to reduce the overall latency.
-//
-// Example: Fuse Conv2D+BiasAdd+Relu.
-
 namespace {
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
diff --git a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
new file mode 100644
index 0000000000..7be1de29c9
--- /dev/null
+++ b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
@@ -0,0 +1,902 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Implements convolution operations with image transformations (resize and
+// mirror padding) baked into the processing, to optimize latency and memory
+// usage.
+
+#define EIGEN_USE_THREADS
+
+#include <string>
+#include <vector>
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/numeric_op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_slice.h"
+#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_ops.h"
+#include "tensorflow/core/kernels/gemm_functors.h"
+#include "tensorflow/core/kernels/image_resizer_state.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/util/mirror_pad_mode.h"
+#include "tensorflow/core/util/padding.h"
+#include "tensorflow/core/util/tensor_format.h"
+
+namespace tensorflow {
+namespace {
+
+// We don't want to allocate a buffer to hold all the patches if the size is
+// going to be extremely large, so break it into chunks if it's bigger than
+// a limit. Each chunk will be processed serially, so we can refill the
+// buffer for the next chunk and reuse it, keeping maximum memory size down.
+// In this case, we've picked 16 megabytes as a reasonable limit for Android and
+// other platforms using Eigen, and 1MB for iOS devices, from experimentation.
+#if defined(__APPLE__) && defined(IS_MOBILE_PLATFORM)
+const size_t kMaxChunkSize = (1 * 1024 * 1024);
+#else
+const size_t kMaxChunkSize = (16 * 1024 * 1024);
+#endif
+const size_t kResizeCacheSize = (8 * 1024 * 1024);
+
+// Lookup method used when resizing.
+enum SamplingMode {
+  BILINEAR = 0,
+  NEAREST = 1,
+};
+
+// Simple utility function used by FusedConv to multithread basic workloads. To
+// use it, pass begin and end values for the full workload and a std::function
+// that receives a subset of that through the begin and end values for each
+// worker's task. The division of the full workload into worker tasks is handled
+// by the multithreading logic. Here's an example of how to use it:
+// std::vector<float> my_vector(100);
+// ...
+// FusedConvParallelFor(context, 0, 100,
+//   [&my_vector](int64 task_begin, int64 task_end) {
+//     for (int64 current = task_begin; current != task_end; ++current) {
+//       my_vector[current] *= 10.0f;
+//     }
+// });
+void FusedConvParallelFor(
+    OpKernelContext* context, int64 begin, int64 end,
+    const std::function<void(int64, int64)>& task_function) {
+// On iOS, the thread management imposes a very big performance penalty, so
+// just call the function directly with no multithreading.
+#if defined(__APPLE__) && defined(IS_MOBILE_PLATFORM)
+  task_function(begin, end);
+#else
+  auto& worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+  thread::ThreadPool* thread_pool = worker_threads.workers;
+  const int64 total_elements = end - begin;
+  // This is a bit of an arbitrary number, but was found to work well for
+  // typical models we've been profiling on various devices.
+  const int64 element_cost = 10000000;
+  thread_pool->ParallelFor(
+      total_elements, element_cost,
+      [begin, task_function](int64 begin_offset, int64 end_offset) {
+        const int64 task_begin = begin + begin_offset;
+        const int64 task_end = begin + end_offset;
+        task_function(task_begin, task_end);
+      });
+#endif
+}
+
+// Holds the state needed for the resizing subtasks.
+template <class T1>
+struct ResizeTaskParameters {
+  ResizeTaskParameters() : st(false) {}
+
+  int cache_height;
+  T1* resize_cache;
+  int cache_line_width;
+  int input_width;
+  int input_depth;
+  int top_padding;
+  int pad_offset;
+  int64 resized_height;
+  ImageResizerState st;
+  const T1* input_batch_start;
+  int64 cache_start_x;
+  int64 cache_end_x;
+  int left_padding;
+  int64 resized_width;
+  int64 padded_width;
+  int64 padded_height;
+};
+
+template <class T1>
+struct PerCacheLineParameters {
+  PerCacheLineParameters() {}
+  PerCacheLineParameters(const PerCacheLineParameters<T1>& other)
+      : cache_line_start(other.cache_line_start),
+        input_top_row_start(other.input_top_row_start),
+        input_bottom_row_start(other.input_bottom_row_start),
+        y_lerp(other.y_lerp) {}
+
+  T1* cache_line_start;
+  const T1* input_top_row_start;
+  const T1* input_bottom_row_start;
+  T1 y_lerp;
+};
+
+// Helper class to simplify bilinear filtering
+template <class T1>
+struct SampleRect {
+  EIGEN_ALWAYS_INLINE SampleRect(const T1* in_top_left, const T1* in_top_right,
+                                 const T1* in_bottom_left,
+                                 const T1* in_bottom_right)
+      : top_left(in_top_left),
+        top_right(in_top_right),
+        bottom_left(in_bottom_left),
+        bottom_right(in_bottom_right) {}
+
+  EIGEN_ALWAYS_INLINE T1 BilinearSample(int channel, T1 x_lerp,
+                                        T1 y_lerp) const {
+    const T1 top =
+        top_left[channel] + (top_right[channel] - top_left[channel]) * x_lerp;
+    const T1 bottom = bottom_left[channel] +
+                      (bottom_right[channel] - bottom_left[channel]) * x_lerp;
+    return top + (bottom - top) * y_lerp;
+  }
+
+  const T1* top_left;
+  const T1* top_right;
+  const T1* bottom_left;
+  const T1* bottom_right;
+};
+
+// Calculates parameters which remain constant through a resize cache row.
+template <class T1>
+EIGEN_ALWAYS_INLINE PerCacheLineParameters<T1> CalculatePerCacheLineParameters(
+    int64 cache_height, int64 cache_y, T1* resize_cache, int64 cache_line_width,
+    int64 input_width, int64 input_depth, int64 top_padding, int64 pad_offset,
+    int64 resized_height, const ImageResizerState& st,
+    const T1* input_batch_start) {
+  PerCacheLineParameters<T1> result;
+  // The cache is organized so that the real y values of the resized image map
+  // onto the actual cache values through a modulo scheme. This means that as we
+  // progress downwards through the image, we keep reusing a small cache and so
+  // keep memory usage down.
+  int64 cache_index_y;
+  if (cache_y < 0) {
+    cache_index_y = cache_height + (cache_y % cache_height);
+  } else {
+    cache_index_y = cache_y % cache_height;
+  }
+  result.cache_line_start =
+      resize_cache + (cache_index_y * cache_line_width * input_depth);
+  // This part is implementing the mirror padding that happens before resizing.
+  float in_y = (cache_y - top_padding);
+  if (in_y < 0) {
+    in_y = -(in_y + 1.0f - pad_offset);
+  } else if (in_y >= resized_height) {
+    in_y = (resized_height * 2.0f) - (in_y + 1.0f + pad_offset);
+  }
+  // Here's where do do the actual resize.
+  in_y *= st.height_scale;
+  const int64 top_y_index = static_cast<int64>(std::floor(in_y));
+  const int64 bottom_y_index =
+      std::min(static_cast<int64>(std::ceil(in_y)), (st.in_height - 1));
+  // Lerp is used for bilinear filtering when that's needed.
+  result.y_lerp = static_cast<T1>(in_y - top_y_index);
+  // Which rows of the original input image to pull the values from.
+  result.input_top_row_start =
+      input_batch_start + (top_y_index * input_width * input_depth);
+  result.input_bottom_row_start =
+      input_batch_start + (bottom_y_index * input_width * input_depth);
+  return result;
+}
+
+template <class T1>
+struct PerCachePixelParameters {
+  PerCachePixelParameters() {}
+  PerCachePixelParameters(const PerCachePixelParameters<T1>& other)
+      : cache_line_pixel(other.cache_line_pixel),
+        left_x_index(other.left_x_index),
+        right_x_index(other.right_x_index),
+        x_lerp(other.x_lerp) {}
+
+  T1* cache_line_pixel;
+  int64 left_x_index;
+  int64 right_x_index;
+  T1 x_lerp;
+};
+
+// Pulls out common parameters used for every resized pixel.
+template <class T1>
+EIGEN_ALWAYS_INLINE PerCachePixelParameters<T1>
+CalculatePerCachePixelParameters(int64 cache_x, int64 cache_start_x,
+                                 T1* cache_line_start, int64 input_depth,
+                                 int64 left_padding, int64 pad_offset,
+                                 int64 resized_width,
+                                 const ImageResizerState& st) {
+  PerCachePixelParameters<T1> result;
+  // Figure out where we're going to store the results of our transform.
+  const int cache_index_x = cache_x - cache_start_x;
+  result.cache_line_pixel = cache_line_start + (cache_index_x * input_depth);
+  // Implement mirror padding by flipping in_x if it's off the edge.
+  float in_x = (cache_x - left_padding);
+  if (in_x < 0) {
+    in_x = -(in_x + 1.0f - pad_offset);
+  } else if (in_x >= resized_width) {
+    in_x = (resized_width * 2.0f) - (in_x + 1.0f + pad_offset);
+  }
+  // Resize the x parameters.
+  in_x *= st.width_scale;
+  // Get the x coordinates for the left and right pixels to pull from.
+  result.left_x_index = static_cast<int64>(std::floor(in_x));
+  result.right_x_index =
+      std::min(static_cast<int64>(std::ceil(in_x)), (st.in_width - 1));
+  // This x_lerp is used to blend pixels in bilinear filtering.
+  result.x_lerp = static_cast<T1>(in_x - result.left_x_index);
+  return result;
+}
+
+// Combines bilinear resizing and mirror padding into the im2col transformation
+// stage of convolution.
+template <class T1, class T2, class T3, class TGemmFunctor,
+          SamplingMode SampleMode>
+class FusedResizeAndPadConvFunctor {
+ public:
+  void operator()(OpKernelContext* context, const Tensor& input,
+                  int input_batches, int resized_height, int resized_width,
+                  int padded_height, int padded_width, int input_depth,
+                  const T2* filter_data, int filter_height, int filter_width,
+                  int filter_count, int stride_rows, int stride_cols,
+                  Padding padding, T3* output_data, int output_height,
+                  int output_width, const ImageResizerState& st,
+                  int top_padding, int bottom_padding, int left_padding,
+                  int right_padding, int pad_offset) {
+    if ((input_batches <= 0) || (padded_width <= 0) || (padded_height <= 0) ||
+        (input_depth <= 0)) {
+      LOG(WARNING) << "Conv2D was called with bad input dimensions: "
+                   << input_batches << ", " << padded_height << ", "
+                   << padded_width << ", " << input_depth;
+      return;
+    }
+    if ((filter_width <= 0) || (filter_height <= 0) || (filter_count <= 0)) {
+      LOG(WARNING) << "Conv2D was called with bad filter dimensions: "
+                   << filter_width << ", " << filter_height << ", "
+                   << filter_count;
+      return;
+    }
+    if ((output_width <= 0) || (output_height <= 0)) {
+      LOG(WARNING) << "Conv2D was called with bad output width or height: "
+                   << output_width << ", " << output_height;
+      return;
+    }
+    OP_REQUIRES(
+        context, ((SampleMode == NEAREST) || (SampleMode == BILINEAR)),
+        errors::InvalidArgument("Bad sample mode passed in", SampleMode));
+
+    // These calculations define how the patches will be positioned within the
+    // input image. The actual definitions are quite complex, and rely on the
+    // previously-calculated output size.
+    int filter_left_offset;
+    int filter_top_offset;
+    if (padding == VALID) {
+      filter_left_offset =
+          ((output_width - 1) * stride_cols + filter_width - padded_width + 1) /
+          2;
+      filter_top_offset = ((output_height - 1) * stride_rows + filter_height -
+                           padded_height + 1) /
+                          2;
+    } else {
+      filter_left_offset =
+          ((output_width - 1) * stride_cols + filter_width - padded_width) / 2;
+      filter_top_offset =
+          ((output_height - 1) * stride_rows + filter_height - padded_height) /
+          2;
+    }
+
+    ResizeTaskParameters<T1> task_params;
+    task_params.input_depth = input_depth;
+    task_params.top_padding = top_padding;
+    task_params.pad_offset = pad_offset;
+    task_params.resized_height = resized_height;
+    task_params.st = st;
+    task_params.left_padding = left_padding;
+    task_params.resized_width = resized_width;
+    task_params.padded_width = padded_width;
+    task_params.padded_height = padded_height;
+
+    // The im2col buffer has # of patches rows, and # of filters cols.
+    // It's laid out like this, in row major order in memory:
+    //        < filter value count >
+    //   ^   +---------------------+
+    // patch |                     |
+    // count |                     |
+    //   v   +---------------------+
+    // Each patch row contains a filter_width x filter_height patch of the
+    // input, with the depth channel as the most contiguous in memory, followed
+    // by the width, then the height. This is the standard memory order in the
+    // image world if it helps to visualize it.
+    const int filter_value_count = filter_width * filter_height * input_depth;
+
+    OP_REQUIRES(context, (filter_value_count * sizeof(T1)) <= kMaxChunkSize,
+                errors::InvalidArgument("Im2Col patch too large for buffer"));
+    const size_t patches_per_chunk =
+        kMaxChunkSize / (filter_value_count * sizeof(T1));
+    // Because memory allocation is very expensive on mobile platforms, try to
+    // allocate a persistent buffer that will be kept around between calls. We
+    // use TensorFlow's resource management to ensure that the memory will be
+    // released when the session is over.
+    Im2ColBufferResource<T1, kMaxChunkSize>* im2col_buffer_resource;
+    std::function<Status(Im2ColBufferResource<T1, kMaxChunkSize>**)> creator =
+        [](Im2ColBufferResource<T1, kMaxChunkSize>** resource) {
+          *resource = new Im2ColBufferResource<T1, kMaxChunkSize>();
+          return Status::OK();
+        };
+    OP_REQUIRES_OK(context, context->resource_manager()->LookupOrCreate(
+                                "Conv2d", "im2col_buffer",
+                                &im2col_buffer_resource, creator));
+
+    // Create a resize cache memory buffer that will hold the rows of
+    // transformed and mirror padded input pixels, ready to be copied
+    // into filter patches by im2col.
+    // It's laid out like this, in row major order in memory:
+    //         < cache line width >
+    //   ^    +--------------------+
+    // cache  |                    |
+    // height |                    |
+    //   v    +--------------------+
+    // Each cache row contains a cache_line_width number of resized pixels,
+    // each with input_depth channels. The cache height is typically less than
+    // the full height the resized image would be, so it's filled up
+    // incrementally as we progress downwards through the input creating im2col
+    // patches.
+    task_params.cache_start_x = -filter_left_offset;
+    task_params.cache_end_x =
+        (((output_width - 1) * stride_cols) - filter_left_offset) +
+        filter_width;
+    task_params.cache_line_width =
+        task_params.cache_end_x - task_params.cache_start_x;
+    task_params.cache_height =
+        kResizeCacheSize / (task_params.cache_line_width * input_depth);
+    const int needed_resize_cache_count =
+        filter_height * task_params.cache_line_width * input_depth;
+    OP_REQUIRES(context,
+                (needed_resize_cache_count * sizeof(T1)) <= kResizeCacheSize,
+                errors::InvalidArgument("Input too large for resize cache"));
+    Im2ColBufferResource<T1, kResizeCacheSize>* resize_cache_resource;
+    std::function<Status(Im2ColBufferResource<T1, kResizeCacheSize>**)>
+        resize_creator =
+            [](Im2ColBufferResource<T1, kResizeCacheSize>** resource) {
+              *resource = new Im2ColBufferResource<T1, kResizeCacheSize>();
+              return Status::OK();
+            };
+    OP_REQUIRES_OK(context, context->resource_manager()->LookupOrCreate(
+                                "Conv2d", "resize_cache",
+                                &resize_cache_resource, resize_creator));
+
+    // This means that multiple ops can't be run simultaneously on different
+    // threads, because we have a single shared resource. The platforms this is
+    // aimed at have intra-op parallelism as their focus though, so it shouldn't
+    // be an issue.
+    mutex_lock lock_buffer(im2col_buffer_resource->mu);
+    core::ScopedUnref unref_buffer(im2col_buffer_resource);
+    T1* im2col_buffer = im2col_buffer_resource->data;
+
+    // This buffer is used as a fairly heavy-weight cache for the resized and
+    // mirrored inputs to the im2col operation. The problem is that we want to
+    // keep the memory usage down by not rendering the fully resized and padded
+    // input tensor to the convolution into an entire buffer. The first approach
+    // to avoid this was to fold the bilinear filtering and padding spatial
+    // transformations into the im2col lookup itself. This successfully reduced
+    // memory usage, but because im2col can access an individual pixel for many
+    // different patches, the extra overhead of doing the same bilinear lookups
+    // repeatedly became too expensive.
+    // The resize cache is designed to avoid this problem by keeping a
+    // horizontal slice of the resized and padded input to the im2col
+    // precalculated, so that repeated accesses to the same pixel from different
+    // filter patches can just be copied from this cache. It's organized as a
+    // horizontal slice stretching across the whole virtual image, and as high
+    // as the filter window, so that as the patch processing moves across all
+    // the pixels are present, and before a new row of patches is started any
+    // previously calculated rows that are needed are maintained, with new rows
+    // calculated as required.
+    mutex_lock resize_lock_buffer(resize_cache_resource->mu);
+    core::ScopedUnref unref_resized_cache(resize_cache_resource);
+    task_params.resize_cache = resize_cache_resource->data;
+
+    const T1* input_data = input.flat<T1>().data();
+    const int64 input_height = input.shape().dim_sizes()[1];
+    task_params.input_width = input.shape().dim_sizes()[2];
+
+    int end_cached_lines = std::numeric_limits<int>::min();
+
+    for (int batch = 0; batch < input_batches; ++batch) {
+      task_params.input_batch_start =
+          input_data +
+          (batch * input_height * task_params.input_width * input_depth);
+      const int in_y_end =
+          ((output_height * stride_rows) - filter_top_offset) + filter_height;
+      for (int out_y = 0; out_y < output_height; ++out_y) {
+        const int in_y_origin = (out_y * stride_rows) - filter_top_offset;
+        const int cache_start_y = std::max(in_y_origin, end_cached_lines);
+        const int cache_end_y = std::min(
+            in_y_end, std::max((in_y_origin + task_params.cache_height),
+                               end_cached_lines));
+        if (end_cached_lines < (in_y_origin + filter_height)) {
+          // This call breaks up the work required for calculating the mirror
+          // padding and resizing across multiple threads.
+          FusedConvParallelFor(
+              context, cache_start_y, cache_end_y,
+              [task_params](int64 task_cache_start_y, int64 task_cache_end_y) {
+                // This is a long and confusing function, but it's been laid out
+                // this way to help with performance on some intensive models.
+                // What it's doing is populating a cache of the original input
+                // image, after it's been bilinear resized and had its edges
+                // mirrored. This allows the following im2col code to access the
+                // transformed pixels from this cache, without having to
+                // repeatedly apply the expensive bilinear calculations as the
+                // same pixels are accessed by different patches.
+                // This is most effective when the stride is small and the
+                // filter size is large, since that's when pixels are reused
+                // most frequently as patches overlap.
+                for (int cache_y = task_cache_start_y;
+                     cache_y < task_cache_end_y; ++cache_y) {
+                  // We organize the cache as a series of rows, each containing
+                  // all the transformed pixels for a given line in the image.
+                  // This cache is big enough to hold at least a filter's height
+                  // worth of rows, but typically more, limited by the size of
+                  // the cache buffer.
+                  // We don't allocate an entire image's worth of rows though,
+                  // because we're trying to keep memory usage down, so as we
+                  // progress downwards through the im2col we periodically
+                  // refresh the cache so that the next lines that are needed
+                  // for that operation are always present.
+                  // Work out the parameters that remain constant across the
+                  // row we're calculating.
+                  PerCacheLineParameters<T1> line_params(
+                      CalculatePerCacheLineParameters<T1>(
+                          task_params.cache_height, cache_y,
+                          task_params.resize_cache,
+                          task_params.cache_line_width, task_params.input_width,
+                          task_params.input_depth, task_params.top_padding,
+                          task_params.pad_offset, task_params.resized_height,
+                          task_params.st, task_params.input_batch_start));
+                  // Iterate through the resize cache row we're filling in.
+                  for (int cache_x = task_params.cache_start_x;
+                       cache_x < task_params.cache_end_x; ++cache_x) {
+                    // Figure out what we need for the cache pixel we're
+                    // populating.
+                    PerCachePixelParameters<T1> pixel_params(
+                        CalculatePerCachePixelParameters<T1>(
+                            cache_x, task_params.cache_start_x,
+                            line_params.cache_line_start,
+                            task_params.input_depth, task_params.left_padding,
+                            task_params.pad_offset, task_params.resized_width,
+                            task_params.st));
+                    // If the access is off the left, right, top, or bottom of
+                    // the resized image, the conv padding means we should set
+                    // it to zero.
+                    if ((cache_x < 0) ||
+                        (cache_x >= task_params.padded_width) ||
+                        (cache_y < 0) ||
+                        (cache_y >= task_params.padded_height)) {
+                      std::fill_n(pixel_params.cache_line_pixel,
+                                  task_params.input_depth, T1(0));
+                    } else {
+                      // There are two different sampling strategies for
+                      // resizing. When using nearest, we can just do a
+                      // straight copy of the pixel closest to our sample point,
+                      // but bilinear requires a more complex calculation.
+                      if (SampleMode == NEAREST) {
+                        const T1* input_top_left_pixel =
+                            line_params.input_top_row_start +
+                            (pixel_params.left_x_index *
+                             task_params.input_depth);
+
+                        std::copy_n(input_top_left_pixel,
+                                    task_params.input_depth,
+                                    pixel_params.cache_line_pixel);
+                      } else {
+                        const SampleRect<T1> rect(
+                            line_params.input_top_row_start +
+                                (pixel_params.left_x_index *
+                                 task_params.input_depth),
+                            line_params.input_top_row_start +
+                                (pixel_params.right_x_index *
+                                 task_params.input_depth),
+                            line_params.input_bottom_row_start +
+                                (pixel_params.left_x_index *
+                                 task_params.input_depth),
+                            line_params.input_bottom_row_start +
+                                (pixel_params.right_x_index *
+                                 task_params.input_depth));
+                        for (int in_channel = 0;
+                             in_channel < task_params.input_depth;
+                             ++in_channel) {
+                          pixel_params.cache_line_pixel[in_channel] =
+                              rect.BilinearSample(in_channel,
+                                                  pixel_params.x_lerp,
+                                                  line_params.y_lerp);
+                        }
+                      }
+                    }
+                  }
+                }
+              });
+          end_cached_lines = cache_end_y;
+        }
+        for (int out_x = 0; out_x < output_width; ++out_x) {
+          const int in_x_origin = (out_x * stride_cols) - filter_left_offset;
+          const int patch_index = (batch * output_width * output_height) +
+                                  (out_y * output_width) + out_x;
+          const int patch_index_within_chunk = patch_index % patches_per_chunk;
+          T1* im2col_patch_start =
+              im2col_buffer + (patch_index_within_chunk * filter_value_count);
+          for (int filter_y = 0; filter_y < filter_height; ++filter_y) {
+            T1* im2col_row_start =
+                im2col_patch_start +
+                (filter_y * filter_width * task_params.input_depth);
+            const int conv_in_y = in_y_origin + filter_y;
+            int cache_index_y;
+            if (conv_in_y < 0) {
+              cache_index_y = task_params.cache_height +
+                              (conv_in_y % task_params.cache_height);
+            } else {
+              cache_index_y = conv_in_y % task_params.cache_height;
+            }
+            T1* cache_line_start =
+                task_params.resize_cache +
+                (cache_index_y * task_params.cache_line_width *
+                 task_params.input_depth);
+            T1* cache_filter_row_start =
+                cache_line_start + ((in_x_origin - task_params.cache_start_x) *
+                                    task_params.input_depth);
+            std::copy_n(cache_filter_row_start,
+                        (filter_width * task_params.input_depth),
+                        im2col_row_start);
+          }
+          const bool is_last_in_chunk =
+              (patch_index_within_chunk == (patches_per_chunk - 1));
+          const bool is_last_overall =
+              ((batch == (input_batches - 1)) &&
+               (out_y == (output_height - 1)) && (out_x == (output_width - 1)));
+          if (is_last_in_chunk || is_last_overall) {
+            // Now we've assembled a set of image patches into a matrix, apply
+            // a GEMM matrix multiply of the patches as rows, times the filter
+            // weights in columns, to get partial results in the output
+            // matrix.
+            const int how_many_patches = patch_index_within_chunk + 1;
+            const int m = how_many_patches;
+            const int n = filter_count;
+            const int k = filter_value_count;
+            const int lda = filter_value_count;
+            const int ldb = filter_count;
+            const int ldc = filter_count;
+            const size_t start_patch_index =
+                patch_index - (how_many_patches - 1);
+            T3* chunk_output_data =
+                output_data + (start_patch_index * filter_count);
+            TGemmFunctor gemm_functor;
+            gemm_functor(context, m, n, k, im2col_buffer, lda, filter_data, ldb,
+                         chunk_output_data, ldc);
+          }
+        }
+      }
+    }
+  }
+};
+
+}  // namespace
+
+// Implements a version of convolution with bilinear resizing and mirror padding
+// included.
+template <class T, class TConvFunctor, bool DoResize>
+class FusedResizeConv2DUsingGemmOp : public OpKernel {
+ public:
+  explicit FusedResizeConv2DUsingGemmOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    if (DoResize) {
+      OP_REQUIRES_OK(context,
+                     context->GetAttr("resize_align_corners", &align_corners_));
+    }
+    MirrorPadMode mode;
+    OP_REQUIRES_OK(context, context->GetAttr("mode", &mode));
+
+    switch (mode) {
+      case MirrorPadMode::SYMMETRIC: {
+        offset_ = 0;
+        break;
+      }
+      case MirrorPadMode::REFLECT: {
+        offset_ = 1;
+        break;
+      }
+      default:
+        OP_REQUIRES(context, false,
+                    errors::InvalidArgument(
+                        "mode must be either REFLECT or SYMMETRIC."));
+    }
+    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
+    OP_REQUIRES(context, strides_.size() == 4,
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 dimensions"));
+    const int64 stride_n = GetTensorDim(strides_, FORMAT_NHWC, 'N');
+    const int64 stride_c = GetTensorDim(strides_, FORMAT_NHWC, 'C');
+    OP_REQUIRES(
+        context, stride_n == 1 && stride_c == 1,
+        errors::InvalidArgument("Current implementation does not yet support "
+                                "strides in the batch and depth dimensions."));
+    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    // Input tensor is of the following dimensions:
+    // [ batch, in_rows, in_cols, in_depth ]
+    const Tensor& input = context->input(0);
+    OP_REQUIRES(context, (input.shape().num_elements() > 0),
+                errors::InvalidArgument("Input tensor can't be empty"));
+
+    ImageResizerState st(false);
+    if (DoResize) {
+      st = ImageResizerState(align_corners_);
+      st.ValidateAndCalculateOutputSize(context, input);
+      if (!context->status().ok()) return;
+    } else {
+      // Set up the resize parameters to do no scaling at all.
+      st.batch_size = input.dim_size(0);
+      st.out_height = input.dim_size(1);
+      st.out_width = input.dim_size(2);
+      st.in_height = input.dim_size(1);
+      st.in_width = input.dim_size(2);
+      st.channels = input.dim_size(3);
+      st.height_scale = 1.0f;
+      st.width_scale = 1.0f;
+    }
+    TensorShape resized_shape(
+        {input.dim_size(0), st.out_height, st.out_width, input.dim_size(3)});
+    int paddings_index;
+    int filter_index;
+    if (DoResize) {
+      paddings_index = 2;
+      filter_index = 3;
+    } else {
+      paddings_index = 1;
+      filter_index = 2;
+    }
+    const Tensor& paddings = context->input(paddings_index);
+
+    const int dims = resized_shape.dims();
+    OP_REQUIRES(
+        context,
+        TensorShapeUtils::IsMatrix(paddings.shape()) &&
+            paddings.dim_size(1) == 2,
+        errors::InvalidArgument("paddings must be a matrix with 2 columns: ",
+                                paddings.shape().DebugString()));
+    const int fixed_dims =
+        (allow_legacy_scalars() && dims == 0 && paddings.dim_size(0) == 1)
+            ? 1
+            : dims;
+    OP_REQUIRES(
+        context, fixed_dims == paddings.dim_size(0),
+        errors::InvalidArgument(
+            "The first dimension of paddings must be the rank of inputs: ",
+            fixed_dims, " ", paddings.shape().DebugString(), " ",
+            resized_shape.DebugString()));
+    OP_REQUIRES(
+        context, dims == paddings.dim_size(0),
+        errors::InvalidArgument(
+            "The first dimension of paddings must be the rank of inputs: ",
+            dims, " ", paddings.shape().DebugString(), " ",
+            resized_shape.DebugString()));
+
+    OP_REQUIRES(
+        context, dims == 4,
+        errors::InvalidArgument(
+            "Fused mirror padding only supports four-dimensional inputs, but ",
+            dims, " requested"));
+
+    // Compute the shape of the output tensor, and allocate it.
+    TensorShape padded_shape;
+    TTypes<int32>::ConstMatrix paddings_matrix = paddings.matrix<int32>();
+    for (int d = 0; d < dims; ++d) {
+      const int32 before =
+          paddings_matrix(d, 0);  // Pad before existing elements.
+      const int32 after =
+          paddings_matrix(d, 1);  // Pad after existing elements.
+      OP_REQUIRES(context, before >= 0 && after >= 0,
+                  errors::InvalidArgument(
+                      "paddings must be non-negative: ", before, " ", after));
+      if (offset_ == 0) {  // SYMMETRIC mode.
+        OP_REQUIRES(
+            context,
+            before <= resized_shape.dim_size(d) &&
+                after <= resized_shape.dim_size(d),
+            errors::InvalidArgument("paddings must be no greater "
+                                    "than the dimension size: ",
+                                    before, ", ", after, " greater than ",
+                                    resized_shape.dim_size(d)));
+      } else if (offset_ == 1) {  // REFLECT mode.
+        OP_REQUIRES(
+            context,
+            before < resized_shape.dim_size(d) &&
+                after < resized_shape.dim_size(d),
+            errors::InvalidArgument("paddings must be less than"
+                                    " the dimension size: ",
+                                    before, ", ", after, " not less than ",
+                                    resized_shape.dim_size(d)));
+      }
+      padded_shape.AddDim(before + resized_shape.dim_size(d) + after);
+    }
+
+    OP_REQUIRES(
+        context, ((paddings_matrix(0, 0) == 0) && (paddings_matrix(0, 1) == 0)),
+        errors::InvalidArgument(
+            "Fused mirror padding only support spatial padding, not batches: ",
+            paddings.DebugString()));
+    OP_REQUIRES(
+        context, ((paddings_matrix(3, 0) == 0) && (paddings_matrix(3, 1) == 0)),
+        errors::InvalidArgument(
+            "Fused mirror padding only support spatial padding, not channels: ",
+            paddings.DebugString()));
+    const int32 top_padding = paddings_matrix(1, 0);
+    const int32 bottom_padding = paddings_matrix(1, 1);
+    const int32 left_padding = paddings_matrix(2, 0);
+    const int32 right_padding = paddings_matrix(2, 1);
+
+    // Input filter is of the following dimensions:
+    // [ filter_rows, filter_cols, in_depth, out_depth]
+    const Tensor& filter = context->input(filter_index);
+
+    // For 2D convolution, there should be 4 dimensions.
+    OP_REQUIRES(context, padded_shape.dims() == 4,
+                errors::InvalidArgument("input must be 4-dimensional",
+                                        padded_shape.DebugString()));
+    OP_REQUIRES(context, filter.dims() == 4,
+                errors::InvalidArgument("filter must be 4-dimensional: ",
+                                        filter.shape().DebugString()));
+
+    // We only check the first three dims, since the depth is accessed as an
+    // int64 below.
+    for (int i = 0; i < 3; i++) {
+      OP_REQUIRES(
+          context,
+          FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()),
+          errors::InvalidArgument("filter too large"));
+    }
+
+    // The last dimension for input is in_depth. It must be the same as the
+    // filter's in_depth.
+    const int64 in_depth = padded_shape.dim_size(3);
+    OP_REQUIRES(context, in_depth == filter.dim_size(2),
+                errors::InvalidArgument(
+                    "input and filter must have the same depth: ", in_depth,
+                    " vs ", filter.dim_size(2)));
+
+    // The last dimension for filter is out_depth.
+    const int out_depth = static_cast<int>(filter.dim_size(3));
+
+    // The second dimension for input is rows/height.
+    // The first dimension for filter is rows/height.
+    const int64 padded_rows_raw = padded_shape.dim_size(1);
+    OP_REQUIRES(
+        context,
+        FastBoundsCheck(padded_rows_raw, std::numeric_limits<int>::max()),
+        errors::InvalidArgument("Input rows too large"));
+    const int padded_rows = static_cast<int>(padded_rows_raw);
+    const int filter_rows = static_cast<int>(filter.dim_size(0));
+    const int resized_rows = static_cast<int>(resized_shape.dim_size(1));
+
+    // The third dimension for input is columns/width.
+    // The second dimension for filter is columns/width.
+    const int64 padded_cols_raw = padded_shape.dim_size(2);
+    OP_REQUIRES(
+        context,
+        FastBoundsCheck(padded_cols_raw, std::numeric_limits<int>::max()),
+        errors::InvalidArgument("Input cols too large"));
+    const int padded_cols = static_cast<int>(padded_cols_raw);
+    const int filter_cols = static_cast<int>(filter.dim_size(1));
+    const int resized_cols = static_cast<int>(resized_shape.dim_size(2));
+
+    // The first dimension for input is batch.
+    const int64 batch_raw = padded_shape.dim_size(0);
+    OP_REQUIRES(context,
+                FastBoundsCheck(batch_raw, std::numeric_limits<int>::max()),
+                errors::InvalidArgument("batch is too large"));
+    const int batch = static_cast<int>(batch_raw);
+
+    // For now we take the stride from the second and third dimensions only (we
+    // do not support striding on the batch or depth dimension).
+    const int stride_rows = GetTensorDim(strides_, FORMAT_NHWC, 'H');
+    const int stride_cols = GetTensorDim(strides_, FORMAT_NHWC, 'W');
+
+    int64 out_rows = 0, out_cols = 0, pad_rows = 0, pad_cols = 0;
+    OP_REQUIRES_OK(context,
+                   GetWindowedOutputSize(padded_rows, filter_rows, stride_rows,
+                                         padding_, &out_rows, &pad_rows));
+    OP_REQUIRES_OK(context,
+                   GetWindowedOutputSize(padded_cols, filter_cols, stride_cols,
+                                         padding_, &out_cols, &pad_cols));
+    TensorShape out_shape =
+        ShapeFromFormat(FORMAT_NHWC, batch, out_rows, out_cols, out_depth);
+    OP_REQUIRES(context, (out_shape.num_elements() > 0),
+                errors::InvalidArgument("Output tensor can't be empty"));
+
+    // Output tensor is of the following dimensions:
+    // [ in_batch, out_rows, out_cols, out_depth ]
+    Tensor* output = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output));
+
+    VLOG(2) << "FusedConv2D: " << name() << ", in_depth = " << in_depth
+            << ", padded_cols = " << padded_cols
+            << ", resized_cols = " << resized_cols
+            << ", filter_cols = " << filter_cols
+            << ", padded_rows = " << padded_rows
+            << ", resized_rows = " << resized_rows
+            << ", filter_rows = " << filter_rows
+            << ", stride_rows = " << stride_rows
+            << ", stride_cols = " << stride_cols
+            << ", out_depth = " << out_depth << ", DoResize=" << DoResize;
+
+    // If there is nothing to compute, return.
+    if (out_shape.num_elements() == 0) {
+      return;
+    }
+    TConvFunctor conv_functor;
+    conv_functor(context, input, batch, resized_rows, resized_cols, padded_rows,
+                 padded_cols, in_depth, filter.flat<T>().data(), filter_rows,
+                 filter_cols, out_depth, stride_rows, stride_cols, padding_,
+                 output->flat<T>().data(), out_rows, out_cols, st, top_padding,
+                 bottom_padding, left_padding, right_padding, offset_);
+  }
+
+ private:
+  std::vector<int32> strides_;
+  Padding padding_;
+  bool align_corners_;
+  int offset_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(FusedResizeConv2DUsingGemmOp);
+};
+
+#define REGISTER_FUSED(T)                                                 \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("FusedResizeAndPadConv2D")                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint<T>("T"),                                        \
+      FusedResizeConv2DUsingGemmOp<                                       \
+          T,                                                              \
+          FusedResizeAndPadConvFunctor<T, T, T, FastGemmFunctor<T, T, T>, \
+                                       BILINEAR>,                         \
+          true>);
+
+TF_CALL_half(REGISTER_FUSED);
+TF_CALL_float(REGISTER_FUSED);
+TF_CALL_double(REGISTER_FUSED);
+
+#define REGISTER_PAD_ONLY_FUSED(T)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("FusedPadConv2D").Device(DEVICE_CPU).TypeConstraint<T>("T"),   \
+      FusedResizeConv2DUsingGemmOp<                                       \
+          T,                                                              \
+          FusedResizeAndPadConvFunctor<T, T, T, FastGemmFunctor<T, T, T>, \
+                                       NEAREST>,                          \
+          false>);
+
+TF_CALL_half(REGISTER_PAD_ONLY_FUSED);
+TF_CALL_float(REGISTER_PAD_ONLY_FUSED);
+TF_CALL_double(REGISTER_PAD_ONLY_FUSED);
+
+}  // namespace tensorflow
-- 
GitLab


From d194c00783646820d8a2d5e9be2edf42518dafbf Mon Sep 17 00:00:00 2001
From: AG Ramesh <ag.ramesh@intel.com>
Date: Tue, 4 Dec 2018 10:21:14 -0800
Subject: [PATCH 1286/1554] Clang format error fixes

---
 tensorflow/core/graph/mkl_layout_pass.cc | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index a143892e08..52b4660094 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -1340,8 +1340,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
   static void CopyAttrsPadWithConv2D(const Node* orig_node, NodeBuilder* nb,
                                      bool change_format = false);
   static void CopyAttrsFromPadAndConv2D(const Node* orig_node1,
-                                        const Node* orig_node2,
-                                        NodeBuilder* nb,
+                                        const Node* orig_node2, NodeBuilder* nb,
                                         bool change_format = false);
   static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb,
                                bool change_format = false);
@@ -2435,9 +2434,9 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
                                                 Node* m, Node* n) {
   DCHECK(((m->type_string() == csinfo_.pad &&
-          n->type_string() == csinfo_.conv2d)) ||
-        ((n->type_string() == csinfo_.pad &&
-          m->type_string() == csinfo_.conv2d)));
+           n->type_string() == csinfo_.conv2d)) ||
+         ((n->type_string() == csinfo_.pad &&
+           m->type_string() == csinfo_.conv2d)));
 
   // Conv2D is successor node, and Pad predecessor node.
   Node* pred = m->type_string() == csinfo_.pad ? m : n;
@@ -2572,7 +2571,7 @@ Status MklLayoutRewritePass::MergePadWithConv2D(std::unique_ptr<Graph>* g,
       // output (at slot 0).
       const int kPadWithConv2DOutputSlot = 0;
       (*g)->AddEdge(new_node, kPadWithConv2DOutputSlot, e->dst(),
-                                  e->dst_input());
+                    e->dst_input());
     }
   }
 
-- 
GitLab


From be4a5d634fa1dea81a13ddbd034c241be8dc5e55 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 4 Dec 2018 09:58:49 -0800
Subject: [PATCH 1287/1554] Turn on PartitionedCallOp inlining by default

PiperOrigin-RevId: 223996298
---
 tensorflow/core/grappler/grappler_item.h      |  7 ++
 .../grappler/optimizers/function_optimizer.cc | 90 +++++++++++++------
 .../grappler/optimizers/meta_optimizer.cc     |  4 +
 .../core/kernels/partitioned_function_ops.cc  |  6 ++
 4 files changed, 78 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h
index aea7d27792..6ef4f14247 100644
--- a/tensorflow/core/grappler/grappler_item.h
+++ b/tensorflow/core/grappler/grappler_item.h
@@ -83,6 +83,13 @@ struct GrapplerItem {
     // Is it allowed to add nodes to the graph that do not have registered
     // gradient function.
     bool non_differentiable_rewrites = true;
+    // By default we are not allowed to inline ops with side effects into the
+    // main graph, because we can't guarantee that after pruning these ops will
+    // be executed. However if we are optimizing a function library (see
+    // meta_optimizer.cc) and a graph was instantiated by a function definition,
+    // we can do that, because functions guarantee that all side effects will be
+    // executed (see function_optimizer.cc for details).
+    bool inline_ops_with_side_effects = false;
   };
 
   const std::unordered_set<string>& devices() const;
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 2dd4ff10e4..69685409a3 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -258,6 +258,7 @@ class FunctionOptimizerContext {
       : grappler_item_id_(item.id),
         graph_version_(item.graph.versions().producer()),
         opt_level_(opt_level),
+        allowed_optimizations_(item.allowed_optimizations()),
         function_library_(OpRegistry::Global(), item.graph.library()),
         available_device_names_(item.devices().begin(), item.devices().end()),
         graph_view_(&item.graph) {
@@ -267,6 +268,10 @@ class FunctionOptimizerContext {
 
   const RewriterConfig::Toggle opt_level() const { return opt_level_; }
 
+  const GrapplerItem::AllowedOptimizations& allowed_optimizations() const {
+    return allowed_optimizations_;
+  }
+
   const FunctionLibraryDefinition& function_library() const {
     return function_library_;
   }
@@ -397,6 +402,7 @@ class FunctionOptimizerContext {
   const string grappler_item_id_;
   const int graph_version_;
   const RewriterConfig::Toggle opt_level_;
+  const GrapplerItem::AllowedOptimizations allowed_optimizations_;
   FunctionLibraryDefinition function_library_;
 
   // These fields initialized lazily only if needed.
@@ -1228,12 +1234,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
                                    SummarizeNodeDef(func_node));
   }
 
-  // TODO(ezhulenev): Enable it by default.
-  if (ctx.opt_level() != RewriterConfig::AGGRESSIVE) {
-    return errors::FailedPrecondition(
-        "Indirect function inlining supported only in aggressive mode");
-  }
-
   if (MarkedNoInline(func)) {
     return errors::FailedPrecondition(
         "Can't inline function marked with '_noinline': ",
@@ -1256,6 +1256,20 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
+  // We can't inline functions with `Switch` nodes in the function body, because
+  // they might have dead tensors as a function output argument (we need all
+  // intermediate tensors to compute the function gradient). `PartitionedCallOp`
+  // invokes functions with `allow_dead_tensors = true` to reset dead flag,
+  // and return default initialized tensors instead of a dead tensors.
+  // TODO(ezhulenev): Do the liveness analysis and add
+  // `IdentitytWithResurrection` nodes after all potentially dead output
+  // tensors?
+  if (absl::c_any_of(func.node_def(), IsSwitch)) {
+    return errors::FailedPrecondition(
+        "Can't inline function with `Switch` nodes in the function body: ",
+        SummarizeNodeDef(func_node));
+  }
+
   return Status::OK();
 }
 
@@ -1339,11 +1353,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   const string prefix = strings::StrCat(func_node.name(), "/");
 
-  // Keep track of side-effectful ops inside function body. Each outgoing
-  // control edge from the function call node, must be replaced with control
-  // edges from inlined side-effectful ops.
-  std::vector<string> side_effectful_nodes;
-
   // ------------------------------------------------------------------------ //
   // First we need to assign device placements to all function body nodes.
 
@@ -1427,31 +1436,54 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     TF_RETURN_IF_ERROR(
         AddPrefixAndSuffixToNode(prefix, /*suffix=*/"", &func_body_node));
 
-    // If the function body has a side-effectful op, we double check that the
-    // function call node has an output control edge, otherwise we can't safely
-    // do inlining and guarantee that node will be executed.
-    // TODO(ezhulenev): If we don't have `happens_after` dependencies does it
-    // mean that no one is interested in observing side effects and we can
-    // safely inline it?
+    // After inlining into the optimized graph, NodeDef must have all attributes
+    // defined, which is not required for a node in a FunctionDef.
+    const OpDef* op_def;
+    TF_RETURN_IF_ERROR(
+        ctx->function_library().LookUpOpDef(func_body_node.op(), &op_def));
+    AddDefaultsToNodeDef(*op_def, &func_body_node);
+  }
+
+  // Construct a graph view for the preprocessed function body graph.
+  GraphView placed_graph_view(&placed_graph_def);
+
+  // Keep track of side-effectful ops inside function body. Each outgoing
+  // control edge from the function call node, must be replaced with control
+  // edges from inlined side-effectful ops.
+  std::vector<string> side_effectful_nodes;
+
+  // We have to make sure that all side-effectful nodes inside a function body
+  // will be executed after function inlining.
+  for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
     if (!IsFreeOfSideEffect(func_body_node, &ctx->function_library())) {
-      DCHECK(!happens_after.empty());
-      if (happens_after.empty()) {
-        // NOTE: If this happens file a bug to TF.Eager team.
+      int num_fanouts = placed_graph_view.NumFanouts(
+          func_body_node, /*include_controlling_nodes=*/true);
+
+      // If the node doesn't have any outgoing edges and we do not have any
+      // nodes in the `happens_after` set, we can't inline a function and
+      // guarantee that side-effects will be executed. The only exception if we
+      // do function library optimization, and the GrapplerItem was constructed
+      // for the function body, because functions have strict semantics.
+
+      if (num_fanouts == 0 && happens_after.empty() &&
+          !ctx->allowed_optimizations().inline_ops_with_side_effects) {
         return errors::Internal(
-            "Can't inline a function with a stateful op and empty output "
-            "control edge set. Function body node: ",
+            "Can't inline a function with a side-effectful op with empty "
+            "fanouts and empty output control edge set. Function body node: ",
             SummarizeNodeDef(func_body_node));
       }
 
       side_effectful_nodes.push_back(func_body_node.name());
     }
+  }
 
-    // TODO(ezhulenev): Inline nested indirect function calls.
-
-    // Move the node to the optimized graph.
+  // Move all the nodes to the optimized graph after successful preprocessing.
+  for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
     optimized_graph->add_node()->Swap(&func_body_node);
   }
 
+  // TODO(ezhulenev): Inline nested indirect function calls.
+
   // Indirect function call is fully inlined into the optimized graph, and we do
   // not copy the original function call node, so we have to setup tensor
   // mapping from old output tensors, to the outputs of inlined nodes.
@@ -1469,6 +1501,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // call node to all side-effectful ops inside function body.
   ctx->AddControlOverrides(func_node, side_effectful_nodes);
 
+  VLOG(3) << "Successfully inlined indirect function call: "
+          << SummarizeNodeDef(func_node);
   return Status::OK();
 }
 
@@ -1552,8 +1586,7 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
               node, *func, graph_def_version, ctx, optimized_graph));
           continue;
         } else {
-          VLOG(2) << "Can't inline direct function call: "
-                  << inlinable.error_message();
+          VLOG(2) << inlinable.error_message();
         }
       }
 
@@ -1565,8 +1598,7 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
               node, *func, graph_def_version, &ctx, optimized_graph));
           continue;
         } else {
-          VLOG(2) << "Can't inline indirect function call: "
-                  << inlinable.error_message();
+          VLOG(2) << inlinable.error_message();
         }
       }
 
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 5560e8d55f..7c83036341 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -533,6 +533,10 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
         VLOG(3) << added_devices.error_message();
       }
 
+      // We can safely inline nested function calls with side-effectful ops into
+      // the function body (see function_optimizer.cc for details).
+      func_item.allowed_optimizations().inline_ops_with_side_effects = true;
+
       // Optimize function body graph.
       GraphDef optimized_func_graph;
       TF_RETURN_IF_ERROR(
diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index 6c90ffd75e..30a8be141c 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -531,6 +531,12 @@ class PartitionedCallOp : public AsyncOpKernel {
 
     tensorflow::grappler::GrapplerItem item;
 
+    // Add all available devices so that inlined function can be placed.
+    for (const Device* d : device_set.devices()) {
+      Status added_device = item.AddDevice(d->name());
+      if (!added_device.ok()) VLOG(3) << added_device.error_message();
+    }
+
     // Add fetches so that the graph can be pruned.
     for (Node* node : ret_nodes) {
       item.fetch.push_back(node->name());
-- 
GitLab


From 2fa9a541a96131a83cd9c8f2898c43e2d48ea00c Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 4 Dec 2018 10:13:55 -0800
Subject: [PATCH 1288/1554] Avoid aliasing variables that are not live upon
 entry of conditional block. This avoids incorrectly aliasing variables that
 may be defined because of loop effects in the CFG.

PiperOrigin-RevId: 223999403
---
 .../autograph/converters/control_flow.py      | 42 ++++++++++++++++---
 .../pyct/static_analysis/liveness.py          | 14 +++++++
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py
index 5853e044c5..bef6cae1bb 100644
--- a/tensorflow/python/autograph/converters/control_flow.py
+++ b/tensorflow/python/autograph/converters/control_flow.py
@@ -106,14 +106,49 @@ class ControlFlowTransformer(converter.Base):
       return 'no variables'
     return ', '.join(map(str, symbol_set))
 
-  def visit_If(self, node):
-    node = self.generic_visit(node)
+  def _determine_aliased_symbols(self, scope, node_defined_in, block):
+    if block:
+      block_live_in = set(anno.getanno(block[0], anno.Static.LIVE_VARS_IN))
+    else:
+      block_live_in = set()
 
+    # For the purpose of aliasing, composite symbols with live owners are live
+    # as well. Otherwise this would leak tensors from the conditional's body.
+    #
+    # For example:
+    #
+    #   obj = some_obj
+    #   if cond:
+    #     obj.a = val
+    #
+    # Thanslating to the code below would be incorrect:
+    #
+    #   def true_fn():
+    #     obj.a = val()  # Wrong! leaks ops owned by true_fn
+    #     return obj.a
+    for s in scope.modified:
+      if s.is_composite():
+        live_parents = block_live_in & s.owner_set
+        if live_parents:
+          block_live_in.add(s)
+    return scope.modified & node_defined_in & block_live_in
+
+  def visit_If(self, node):
     body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE)
     orelse_scope = anno.getanno(node, annos.NodeAnno.ORELSE_SCOPE)
     defined_in = anno.getanno(node, anno.Static.DEFINED_VARS_IN)
     live_out = anno.getanno(node, anno.Static.LIVE_VARS_OUT)
 
+    # Note: this information needs to be extracted before the body conversion
+    # that happens in the call to generic_visit below, because the conversion
+    # generates nodes that lack static analysis annotations.
+    need_alias_in_body = self._determine_aliased_symbols(
+        body_scope, defined_in, node.body)
+    need_alias_in_orelse = self._determine_aliased_symbols(
+        orelse_scope, defined_in, node.orelse)
+
+    node = self.generic_visit(node)
+
     modified_in_cond = body_scope.modified | orelse_scope.modified
     returned_from_cond = set()
     for s in modified_in_cond:
@@ -125,9 +160,6 @@ class ControlFlowTransformer(converter.Base):
         if live_out & s.owner_set:
           returned_from_cond.add(s)
 
-    need_alias_in_body = body_scope.modified & defined_in
-    need_alias_in_orelse = orelse_scope.modified & defined_in
-
     created_in_body = body_scope.modified & returned_from_cond - defined_in
     created_in_orelse = orelse_scope.modified & returned_from_cond - defined_in
 
diff --git a/tensorflow/python/autograph/pyct/static_analysis/liveness.py b/tensorflow/python/autograph/pyct/static_analysis/liveness.py
index 451398f1b7..f8b8d7fa77 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/liveness.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/liveness.py
@@ -161,6 +161,16 @@ class Annotator(transformer.Base):
     self.cross_function_analyzer = cross_function_analyzer
     self.current_analyzer = None
 
+  def visit(self, node):
+    node = super(Annotator, self).visit(node)
+    if (self.current_analyzer is not None and
+        isinstance(node, gast.stmt) and
+        node in self.current_analyzer.graph.index):
+      cfg_node = self.current_analyzer.graph.index[node]
+      anno.setanno(node, anno.Static.LIVE_VARS_IN,
+                   frozenset(self.current_analyzer.in_[cfg_node]))
+    return node
+
   def visit_FunctionDef(self, node):
     parent_analyzer = self.current_analyzer
     self.current_analyzer = self.cross_function_analyzer.analyzers[node]
@@ -198,6 +208,10 @@ class Annotator(transformer.Base):
     node = self._block_statement_live_out(node)
     return self._block_statement_live_in(node, node.test)
 
+  def visit_With(self, node):
+    node = self.generic_visit(node)
+    return self._block_statement_live_in(node, node.items[0])
+
   def visit_Expr(self, node):
     node = self.generic_visit(node)
     cfg_node = self.current_analyzer.graph.index[node]
-- 
GitLab


From 93b27b079e46056056885c7890ac857b193f4921 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 4 Dec 2018 10:35:50 -0800
Subject: [PATCH 1289/1554] Implement the Keras version of Momentum
 optimization algorithm Op.

PiperOrigin-RevId: 224003676
---
 .../api_def_ResourceApplyKerasMomentum.pbtxt  |  56 +++++
 ...def_ResourceSparseApplyKerasMomentum.pbtxt |  64 +++++
 .../api_def_ResourceApplyKerasMomentum.pbtxt  |   4 +
 ...def_ResourceSparseApplyKerasMomentum.pbtxt |   4 +
 tensorflow/core/kernels/training_ops.cc       | 227 ++++++++++++++++++
 tensorflow/core/kernels/training_ops.h        |   9 +
 .../core/kernels/training_ops_gpu.cu.cc       |  25 ++
 tensorflow/core/kernels/training_ops_test.cc  |  34 +++
 tensorflow/core/ops/training_ops.cc           |  28 +++
 .../keras/optimizer_v2/gradient_descent.py    |   4 +-
 .../optimizer_v2/gradient_descent_test.py     |  45 ++--
 .../keras/optimizer_v2/optimizer_v2_test.py   |  45 +++-
 12 files changed, 519 insertions(+), 26 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceApplyKerasMomentum.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceApplyKerasMomentum.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceApplyKerasMomentum.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceApplyKerasMomentum.pbtxt
new file mode 100644
index 0000000000..830391a32b
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceApplyKerasMomentum.pbtxt
@@ -0,0 +1,56 @@
+op {
+  graph_op_name: "ResourceApplyKerasMomentum"
+  in_arg {
+    name: "var"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "accum"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "lr"
+    description: <<END
+Scaling factor. Must be a scalar.
+END
+  }
+  in_arg {
+    name: "grad"
+    description: <<END
+The gradient.
+END
+  }
+  in_arg {
+    name: "momentum"
+    description: <<END
+Momentum. Must be a scalar.
+END
+  }
+  attr {
+    name: "use_locking"
+    description: <<END
+If `True`, updating of the var and accum tensors will be protected
+by a lock; otherwise the behavior is undefined, but may exhibit less
+contention.
+END
+  }
+  attr {
+    name: "use_nesterov"
+    description: <<END
+If `True`, the tensor passed to compute grad will be
+var + momentum * accum, so in the end, the var you get is actually
+var + momentum * accum.
+END
+  }
+  summary: "Update \'*var\' according to the momentum scheme. Set use_nesterov = True if you"
+  description: <<END
+want to use Nesterov momentum.
+
+accum = accum * momentum - lr * grad
+var += accum
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt
new file mode 100644
index 0000000000..b10b1bc2a9
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt
@@ -0,0 +1,64 @@
+op {
+  graph_op_name: "ResourceSparseApplyKerasMomentum"
+  in_arg {
+    name: "var"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "accum"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "lr"
+    description: <<END
+Learning rate. Must be a scalar.
+END
+  }
+  in_arg {
+    name: "grad"
+    description: <<END
+The gradient.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A vector of indices into the first dimension of var and accum.
+END
+  }
+  in_arg {
+    name: "momentum"
+    description: <<END
+Momentum. Must be a scalar.
+END
+  }
+  attr {
+    name: "use_locking"
+    description: <<END
+If `True`, updating of the var and accum tensors will be protected
+by a lock; otherwise the behavior is undefined, but may exhibit less
+contention.
+END
+  }
+  attr {
+    name: "use_nesterov"
+    description: <<END
+If `True`, the tensor passed to compute grad will be
+var + momentum * accum, so in the end, the var you get is actually
+var + momentum * accum.
+END
+  }
+  summary: "Update relevant entries in \'*var\' and \'*accum\' according to the momentum scheme."
+  description: <<END
+Set use_nesterov = True if you want to use Nesterov momentum.
+
+That is for rows we have grad for, we update var and accum as follows:
+
+accum = accum * momentum - lr * grad
+var += accum
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceApplyKerasMomentum.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceApplyKerasMomentum.pbtxt
new file mode 100644
index 0000000000..1c39242b31
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceApplyKerasMomentum.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceApplyKerasMomentum"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt
new file mode 100644
index 0000000000..1807935213
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceSparseApplyKerasMomentum.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceSparseApplyKerasMomentum"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc
index acf162deec..b5ddc8e1b0 100644
--- a/tensorflow/core/kernels/training_ops.cc
+++ b/tensorflow/core/kernels/training_ops.cc
@@ -283,6 +283,22 @@ struct ApplyMomentum<CPUDevice, T> {
   }
 };
 
+template <typename T>
+struct ApplyKerasMomentum<CPUDevice, T> {
+  void operator()(const CPUDevice& d, typename TTypes<T>::Flat var,
+                  typename TTypes<T>::Flat accum,
+                  typename TTypes<T>::ConstScalar lr,
+                  typename TTypes<T>::ConstFlat grad,
+                  typename TTypes<T>::ConstScalar momentum, bool use_nesterov) {
+    accum.device(d) = accum * momentum() - grad * lr();
+    if (use_nesterov) {
+      var.device(d) += (accum * momentum() - grad * lr());
+    } else {
+      var.device(d) += accum;
+    }
+  }
+};
+
 template <typename Device, typename T>
 struct ApplyAdamNonCuda {
   void operator()(const Device& d, typename TTypes<T>::Flat var,
@@ -2525,6 +2541,217 @@ TF_CALL_double(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 #undef REGISTER_KERNELS
 
+template <typename Device, typename T>
+class ApplyKerasMomentumOp : public OpKernel {
+ public:
+  explicit ApplyKerasMomentumOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("use_nesterov", &use_nesterov_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    auto locks =
+        MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_, {0, 1});
+
+    Tensor var;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<Device, T>(
+                            ctx, 0, use_exclusive_lock_, false, &var));
+    Tensor accum;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<Device, T>(
+                            ctx, 1, use_exclusive_lock_, false, &accum));
+    OP_REQUIRES(
+        ctx, var.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(0)));
+    OP_REQUIRES(
+        ctx, accum.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(1)));
+    const Tensor& lr = ctx->input(2);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()),
+                errors::InvalidArgument("lr is not a scalar: ",
+                                        lr.shape().DebugString()));
+    const Tensor& grad = ctx->input(3);
+    OP_REQUIRES(
+        ctx, var.shape().IsSameSize(accum.shape()),
+        errors::InvalidArgument("var and accum do not have the same shape",
+                                var.shape().DebugString(), " ",
+                                accum.shape().DebugString()));
+    OP_REQUIRES(
+        ctx, var.shape().IsSameSize(grad.shape()),
+        errors::InvalidArgument("var and grad do not have the same shape",
+                                var.shape().DebugString(), " ",
+                                grad.shape().DebugString()));
+
+    const Tensor& momentum = ctx->input(4);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(momentum.shape()),
+                errors::InvalidArgument("momentum is not a scalar: ",
+                                        momentum.shape().DebugString()));
+
+    const Device& device = ctx->template eigen_device<Device>();
+    functor::ApplyKerasMomentum<Device, T>()(
+        device, var.flat<T>(), accum.flat<T>(), lr.scalar<T>(), grad.flat<T>(),
+        momentum.scalar<T>(), use_nesterov_);
+    MaybeForwardRefInputToRefOutput(ctx, 0, 0);
+  }
+
+ private:
+  bool use_exclusive_lock_;
+  bool use_nesterov_;
+};
+
+#define REGISTER_KERNELS(D, T)                               \
+  REGISTER_KERNEL_BUILDER(Name("ResourceApplyKerasMomentum") \
+                              .Device(DEVICE_##D)            \
+                              .HostMemory("var")             \
+                              .HostMemory("accum")           \
+                              .TypeConstraint<T>("T"),       \
+                          ApplyKerasMomentumOp<D##Device, T>);
+#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T);
+
+TF_CALL_half(REGISTER_CPU_KERNELS);
+TF_CALL_bfloat16(REGISTER_CPU_KERNELS);
+TF_CALL_float(REGISTER_CPU_KERNELS);
+TF_CALL_double(REGISTER_CPU_KERNELS);
+
+#if GOOGLE_CUDA
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+#define DECLARE_GPU_SPEC(T)                                               \
+  template <>                                                             \
+  void ApplyKerasMomentum<GPUDevice, T>::operator()(                      \
+      const GPUDevice& d, typename TTypes<T>::Flat var,                   \
+      typename TTypes<T>::Flat accum, typename TTypes<T>::ConstScalar lr, \
+      typename TTypes<T>::ConstFlat grad,                                 \
+      typename TTypes<T>::ConstScalar momentum, bool use_nesterov);       \
+  extern template struct ApplyKerasMomentum<GPUDevice, T>;
+DECLARE_GPU_SPEC(Eigen::half);
+DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(double);
+#undef DECLARE_GPU_SPEC
+}  // namespace functor
+
+REGISTER_KERNELS(GPU, Eigen::half);
+REGISTER_KERNELS(GPU, float);
+REGISTER_KERNELS(GPU, double);
+#endif
+#undef REGISTER_CPU_KERNELS
+#undef REGISTER_KERNELS
+
+// Note, this op works on cpu only.
+template <typename T, typename Tindex>
+class SparseApplyKerasMomentumOp : public OpKernel {
+ public:
+  explicit SparseApplyKerasMomentumOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("use_nesterov", &use_nesterov_));
+  }
+
+  void Compute(OpKernelContext* ctx) override NO_THREAD_SAFETY_ANALYSIS {
+    auto locks =
+        MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_, {0, 1});
+
+    Tensor var;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<CPUDevice, T>(
+                            ctx, 0, use_exclusive_lock_, true, &var));
+    Tensor accum;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<CPUDevice, T>(
+                            ctx, 1, use_exclusive_lock_, true, &accum));
+    OP_REQUIRES(
+        ctx, var.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(0)));
+    OP_REQUIRES(
+        ctx, accum.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(1)));
+    OP_REQUIRES(
+        ctx, var.shape().IsSameSize(accum.shape()),
+        errors::InvalidArgument("var and accum do not have the same shape",
+                                var.shape().DebugString(), " ",
+                                accum.shape().DebugString()));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVectorOrHigher(var.shape()),
+                errors::InvalidArgument("var must be at least 1 dimensional"));
+
+    const Tensor& lr = ctx->input(2);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()),
+                errors::InvalidArgument("lr is not a scalar : ",
+                                        lr.shape().DebugString()));
+    const Tensor& grad = ctx->input(3);
+    const Tensor& indices = ctx->input(4);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(indices.shape()),
+                errors::InvalidArgument("indices must be one-dimensional"));
+
+    for (int d = 1; d < var.dims(); d++) {
+      OP_REQUIRES(ctx, var.dim_size(d) == grad.dim_size(d),
+                  errors::InvalidArgument(strings::StrCat(
+                      "var and grad must match in dimension ", d)));
+    }
+    const Tindex N = indices.dim_size(0);
+    OP_REQUIRES(
+        ctx, grad.dim_size(0) == N,
+        errors::InvalidArgument(
+            "grad must be the same size as indices in the first dimension."));
+
+    const Tensor& momentum = ctx->input(5);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(momentum.shape()),
+                errors::InvalidArgument("momentum is not a scalar: ",
+                                        momentum.shape().DebugString()));
+
+    if (N > 0) {
+      const Tindex first_dim_size = var.dim_size(0);
+      auto indices_vec = indices.vec<Tindex>();
+      auto var_flat = var.flat_outer_dims<T>();
+      auto accum_flat = accum.flat_outer_dims<T>();
+      auto grad_flat = grad.flat_outer_dims<T>();
+      T lr_scalar = lr.scalar<T>()();
+      T momentum_scalar = momentum.scalar<T>()();
+
+      for (Tindex i = 0; i < N; i++) {
+        const Tindex index = internal::SubtleMustCopy(indices_vec(i));
+        OP_REQUIRES(ctx, FastBoundsCheck(index, first_dim_size),
+                    errors::InvalidArgument(
+                        strings::StrCat("Index ", index, " at offset ", i,
+                                        " in indices is out of range")));
+        auto a = accum_flat.template chip<0>(index);
+        auto g = grad_flat.template chip<0>(i);
+        auto v = var_flat.template chip<0>(index);
+        a = a * a.constant(momentum_scalar) - g * g.constant(lr_scalar);
+        if (use_nesterov_) {
+          v += a * a.constant(momentum_scalar) - g * g.constant(lr_scalar);
+        } else {
+          v += a;
+        }
+      }
+    }
+
+    MaybeForwardRefInputToRefOutput(ctx, 0, 0);
+  }
+
+ private:
+  bool use_exclusive_lock_;
+  bool use_nesterov_;
+};
+
+#define REGISTER_KERNELS(T, Tindices)                                \
+  REGISTER_KERNEL_BUILDER(Name("ResourceSparseApplyKerasMomentum")   \
+                              .Device(DEVICE_CPU)                    \
+                              .TypeConstraint<T>("T")                \
+                              .TypeConstraint<Tindices>("Tindices"), \
+                          SparseApplyKerasMomentumOp<T, Tindices>);
+#define REGISTER_CPU_KERNELS(T) \
+  REGISTER_KERNELS(T, int32);   \
+  REGISTER_KERNELS(T, int64);
+
+TF_CALL_half(REGISTER_CPU_KERNELS);
+TF_CALL_bfloat16(REGISTER_CPU_KERNELS);
+TF_CALL_float(REGISTER_CPU_KERNELS);
+TF_CALL_double(REGISTER_CPU_KERNELS);
+
+#undef REGISTER_CPU_KERNELS
+#undef REGISTER_KERNELS
+
 template <typename Device, typename T>
 class ApplyAdamOp : public OpKernel {
  public:
diff --git a/tensorflow/core/kernels/training_ops.h b/tensorflow/core/kernels/training_ops.h
index e10a4cb125..557f8b6285 100644
--- a/tensorflow/core/kernels/training_ops.h
+++ b/tensorflow/core/kernels/training_ops.h
@@ -126,6 +126,15 @@ struct ApplyMomentum {
                   typename TTypes<T>::ConstScalar momentum, bool use_nesterov);
 };
 
+template <typename Device, typename T>
+struct ApplyKerasMomentum {
+  void operator()(const Device& d, typename TTypes<T>::Flat var,
+                  typename TTypes<T>::Flat accum,
+                  typename TTypes<T>::ConstScalar lr,
+                  typename TTypes<T>::ConstFlat grad,
+                  typename TTypes<T>::ConstScalar momentum, bool use_nesterov);
+};
+
 template <typename Device, typename T>
 struct ApplyAdam {
   void operator()(const Device& d, typename TTypes<T>::Flat var,
diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc
index 4bd32592db..9ae794925f 100644
--- a/tensorflow/core/kernels/training_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc
@@ -101,6 +101,27 @@ struct ApplyMomentum<GPUDevice, T> {
   }
 };
 
+template <typename T>
+struct ApplyKerasMomentum<GPUDevice, T> {
+  void operator()(const GPUDevice& d, typename TTypes<T>::Flat var,
+                  typename TTypes<T>::Flat accum,
+                  typename TTypes<T>::ConstScalar lr,
+                  typename TTypes<T>::ConstFlat grad,
+                  typename TTypes<T>::ConstScalar momentum, bool use_nesterov) {
+    Eigen::array<typename TTypes<T>::Tensor::Index, 1> bcast;
+    bcast[0] = grad.dimension(0);
+    Eigen::Sizes<1> single;
+    accum.device(d) = (accum * momentum.reshape(single).broadcast(bcast) -
+                       grad * lr.reshape(single).broadcast(bcast));
+    if (use_nesterov) {
+      var.device(d) += (accum * momentum.reshape(single).broadcast(bcast) -
+                        grad * lr.reshape(single).broadcast(bcast));
+    } else {
+      var.device(d) += accum;
+    }
+  }
+};
+
 template <typename T>
 struct ApplyAdam<GPUDevice, T> {
   void operator()(const GPUDevice& d, typename TTypes<T>::Flat var,
@@ -302,6 +323,10 @@ template struct functor::ApplyMomentum<GPUDevice, Eigen::half>;
 template struct functor::ApplyMomentum<GPUDevice, float>;
 template struct functor::ApplyMomentum<GPUDevice, double>;
 
+template struct functor::ApplyKerasMomentum<GPUDevice, Eigen::half>;
+template struct functor::ApplyKerasMomentum<GPUDevice, float>;
+template struct functor::ApplyKerasMomentum<GPUDevice, double>;
+
 template struct functor::ApplyAdam<GPUDevice, Eigen::half>;
 template struct functor::ApplyAdam<GPUDevice, float>;
 template struct functor::ApplyAdam<GPUDevice, double>;
diff --git a/tensorflow/core/kernels/training_ops_test.cc b/tensorflow/core/kernels/training_ops_test.cc
index 2dcc4a500e..8e0597149d 100644
--- a/tensorflow/core/kernels/training_ops_test.cc
+++ b/tensorflow/core/kernels/training_ops_test.cc
@@ -151,6 +151,40 @@ static void BM_Momentum(int iters, int params) {
 }
 BENCHMARK(BM_Momentum)->Arg(128 << 10)->Arg(256 << 10);
 
+static void KerasMomentum(int32 n, Graph** init_g, Graph** train_g) {
+  TensorShape shape({n});
+  {
+    Graph* g = new Graph(OpRegistry::Global());
+    auto var = Var(g, n);
+    auto accum = Var(g, n);
+    auto zero = Zeros(g, n);
+    test::graph::Assign(g, var, zero);
+    test::graph::Assign(g, accum, zero);
+    *init_g = g;
+  }
+  {
+    Graph* g = new Graph(OpRegistry::Global());
+    auto var = Var(g, n);
+    auto accum = Var(g, n);
+    auto lr = Scalar(g, 0.01);
+    auto grad = Random(g, n);
+    auto mom = Scalar(g, 0.01);
+    test::graph::Multi(g, "ApplyKerasMomentum", {var, accum, lr, grad, mom});
+    *train_g = g;
+  }
+}
+
+static void BM_KerasMomentum(int iters, int params) {
+  const int64 tot = static_cast<int64>(iters) * params;
+  testing::ItemsProcessed(tot);
+  testing::BytesProcessed(tot * sizeof(float));
+  Graph* init;
+  Graph* train;
+  KerasMomentum(params, &init, &train);
+  test::Benchmark("cpu", train, GetOptions(), init).Run(iters);
+}
+BENCHMARK(BM_KerasMomentum)->Arg(128 << 10)->Arg(256 << 10);
+
 static void Adam(int32 n, Graph** init_g, Graph** train_g) {
   TensorShape shape({n});
   {
diff --git a/tensorflow/core/ops/training_ops.cc b/tensorflow/core/ops/training_ops.cc
index 94ff092a85..eb4db1aec2 100644
--- a/tensorflow/core/ops/training_ops.cc
+++ b/tensorflow/core/ops/training_ops.cc
@@ -685,6 +685,34 @@ REGISTER_OP("ResourceSparseApplyMomentum")
       return ApplyMomentumShapeFn(c, true /* sparse */);
     });
 
+REGISTER_OP("ResourceApplyKerasMomentum")
+    .Input("var: resource")
+    .Input("accum: resource")
+    .Input("lr: T")
+    .Input("grad: T")
+    .Input("momentum: T")
+    .Attr("T: numbertype")
+    .Attr("use_locking: bool = false")
+    .Attr("use_nesterov: bool = false")
+    .SetShapeFn([](InferenceContext* c) {
+      return ApplyMomentumShapeFn(c, false /* sparse */);
+    });
+
+REGISTER_OP("ResourceSparseApplyKerasMomentum")
+    .Input("var: resource")
+    .Input("accum: resource")
+    .Input("lr: T")
+    .Input("grad: T")
+    .Input("indices: Tindices")
+    .Input("momentum: T")
+    .Attr("T: numbertype")
+    .Attr("Tindices: {int32, int64}")
+    .Attr("use_locking: bool = false")
+    .Attr("use_nesterov: bool = false")
+    .SetShapeFn([](InferenceContext* c) {
+      return ApplyMomentumShapeFn(c, true /* sparse */);
+    });
+
 static Status ApplyAdamShapeFn(InferenceContext* c, bool sparse) {
   ShapeHandle unused;
   ShapeHandle s = ShapeOrHandleShape(c, 0);                       // var
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent.py b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
index 03e4515e02..2b82b5e78d 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
@@ -97,7 +97,7 @@ class SGD(optimizer_v2.OptimizerV2):
     lr_t = self._decayed_lr(var_dtype)
     if self._momentum:
       momentum_var = self.get_slot(var, "momentum")
-      return training_ops.resource_apply_momentum(
+      return training_ops.resource_apply_keras_momentum(
           var.handle,
           momentum_var.handle,
           lr_t,
@@ -124,7 +124,7 @@ class SGD(optimizer_v2.OptimizerV2):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
     momentum_var = self.get_slot(var, "momentum")
-    return training_ops.resource_sparse_apply_momentum(
+    return training_ops.resource_sparse_apply_keras_momentum(
         var.handle,
         momentum_var.handle,
         lr_t,
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index b6dc87e2e1..0c64202da8 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -263,10 +263,8 @@ class GradientDescentOptimizerTest(test.TestCase):
 class MomentumOptimizerTest(test.TestCase):
 
   def _update_nesterov_momentum_numpy(self, var, accum, g, lr, momentum):
-    var = var + accum * lr * momentum
-    accum = accum * momentum + g
-    var = var - lr * accum
-    var = var - accum * lr * momentum
+    accum = accum * momentum - g * lr
+    var += (accum * momentum - g * lr)
     return var, accum
 
   @test_util.run_in_graph_and_eager_modes
@@ -301,9 +299,9 @@ class MomentumOptimizerTest(test.TestCase):
         self.evaluate(mom_update)
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([0.1, 0.1]), self.evaluate(slot0))
+            np.array([-0.2, -0.2]), self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([0.01, 0.01]), self.evaluate(slot1))
+            np.array([-0.02, -0.02]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
@@ -317,11 +315,11 @@ class MomentumOptimizerTest(test.TestCase):
           mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]),
             self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
-            self.evaluate(slot1))
+            np.array([(0.9 * (-0.02) - 2.0 * 0.01),
+                      (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
@@ -486,9 +484,9 @@ class MomentumOptimizerTest(test.TestCase):
         mom_update.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([0.1, 0.1]), self.evaluate(slot0))
+            np.array([-0.2, -0.2]), self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([0.01, 0.01]), self.evaluate(slot1))
+            np.array([-0.02, -0.02]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
@@ -500,11 +498,11 @@ class MomentumOptimizerTest(test.TestCase):
         mom_update.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]),
             self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
-            self.evaluate(slot1))
+            np.array([(0.9 * (-0.02) - 2.0 * 0.01),
+                      (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
@@ -553,10 +551,10 @@ class MomentumOptimizerTest(test.TestCase):
             np.array([0, 0]),
             self.evaluate(slot0)[0])
         self.assertAllCloseAccordingToType(
-            np.array([.1, .1]),
+            np.array([-2.0 * .1, -2.0 * .1]),
             self.evaluate(slot0)[1])
         self.assertAllCloseAccordingToType(
-            np.array([.01, .01]),
+            np.array([-2.0 * .01, -2.0 * .01]),
             self.evaluate(slot1)[2])
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
@@ -573,10 +571,11 @@ class MomentumOptimizerTest(test.TestCase):
         # Check that the momentum accumulators have been updated.
         self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0])
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]),
             self.evaluate(slot0)[1])
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            np.array([(0.9 * (-0.02) - 2.0 * 0.01),
+                      (0.9 * (-0.02) - 2.0 * 0.01)]),
             self.evaluate(slot1)[2])
         # Check that the parameters have been updated.
         self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0])
@@ -621,9 +620,9 @@ class MomentumOptimizerTest(test.TestCase):
         mom_update1.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([0.1, 0.1]), self.evaluate(slot0))
+            np.array([-0.2, -0.2]), self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([0.01, 0.01]), self.evaluate(slot1))
+            np.array([-0.02, -0.02]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
@@ -635,11 +634,11 @@ class MomentumOptimizerTest(test.TestCase):
         mom_update2.run()
         # Check that the momentum accumulators have been updated.
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+            np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]),
             self.evaluate(slot0))
         self.assertAllCloseAccordingToType(
-            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
-            self.evaluate(slot1))
+            np.array([(0.9 * (-0.02) - 2.0 * 0.01),
+                      (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1))
         # Check that the parameters have been updated.
         self.assertAllCloseAccordingToType(
             np.array([
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 1702a52a8b..d7f4747bd7 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -52,6 +52,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
+from tensorflow.python.training import momentum
 
 
 class OptimizerTest(test.TestCase):
@@ -493,7 +494,7 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
   @parameterized.named_parameters(
       ('adadelta', 'adadelta', True, True), ('adagrad', 'adagrad', True, True),
       ('adam', 'adam', True, True), ('adamax', 'adamax', True, True),
-      ('nadam', 'nadam', True, False), ('momentum', 'momentum', False, False),
+      ('nadam', 'nadam', True, False), ('momentum', 'momentum', True, True),
       ('sgd', 'sgd', False, True))
   def testOptimizersCompatibility(self, opt_str, test_weights, test_numeric):
     np.random.seed(1331)
@@ -547,6 +548,48 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       if old_mode is not None:
         os.environ['TF2_BEHAVIOR'] = old_mode
 
+  def testNumericEquivalenceForNesterovMomentum(self):
+    np.random.seed(1331)
+    with self.cached_session():
+      train_samples = 20
+      input_dim = 3
+      num_classes = 2
+      (x, y), _ = testing_utils.get_test_data(
+          train_samples=train_samples,
+          test_samples=10,
+          input_shape=(input_dim,),
+          num_classes=num_classes)
+      y = keras.utils.to_categorical(y)
+
+      num_hidden = 5
+      model_k_v1 = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_k_v2 = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_k_v2.set_weights(model_k_v1.get_weights())
+      model_tf = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_tf.set_weights(model_k_v2.get_weights())
+
+      opt_k_v1 = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)
+      opt_k_v2 = gradient_descent.SGD(momentum=0.9, nesterov=True)
+      opt_tf = momentum.MomentumOptimizer(
+          learning_rate=0.001, momentum=0.9, use_nesterov=True)
+
+      model_k_v1.compile(opt_k_v1, loss='categorical_crossentropy', metrics=[])
+      model_k_v2.compile(opt_k_v2, loss='categorical_crossentropy', metrics=[])
+      model_tf.compile(opt_tf, loss='categorical_crossentropy', metrics=[])
+
+      hist_k_v1 = model_k_v1.fit(x, y, batch_size=5, epochs=10, shuffle=False)
+      hist_k_v2 = model_k_v2.fit(x, y, batch_size=5, epochs=10, shuffle=False)
+      hist_tf = model_tf.fit(x, y, batch_size=5, epochs=10, shuffle=False)
+
+      self.assertAllClose(model_k_v1.get_weights(), model_tf.get_weights())
+      self.assertAllClose(model_k_v1.get_weights(), model_k_v2.get_weights())
+      self.assertAllClose(opt_k_v1.get_weights(), opt_k_v2.get_weights())
+      self.assertAllClose(hist_k_v1.history['loss'], hist_tf.history['loss'])
+      self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
+
 
 def disable_tf2():
   if 'TF2_BEHAVIOR' in os.environ:
-- 
GitLab


From 5e48100140bd8a23e1010ab45c47c111b396ae6b Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Tue, 4 Dec 2018 10:36:54 -0800
Subject: [PATCH 1290/1554] add TF_NumInputs and TF_NumOutputs

PiperOrigin-RevId: 224003843
---
 tensorflow/c/kernels.cc      |  10 ++++
 tensorflow/c/kernels.h       |  10 ++++
 tensorflow/c/kernels_test.cc | 113 +++++++++++++++++++++++++++++------
 3 files changed, 115 insertions(+), 18 deletions(-)

diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index ca69345264..e9a20e8150 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -116,3 +116,13 @@ void TF_RegisterKernelBuilder(const char* name, TF_KernelBuilder* builder,
 
   TF_SetStatus(status, TF_OK, "");
 }
+
+int TF_NumInputs(TF_OpKernelContext* ctx) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  return cc_ctx->num_inputs();
+}
+
+int TF_NumOutputs(TF_OpKernelContext* ctx) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  return cc_ctx->num_outputs();
+}
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index 2518789a3c..a22030985f 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -85,6 +85,16 @@ TF_CAPI_EXPORT extern void TF_RegisterKernelBuilder(const char* kernel_name,
 // builder is not registered with TensorFlow via TF_RegisterKernelBuilder.
 TF_CAPI_EXPORT extern void TF_DeleteKernelBuilder(TF_KernelBuilder* builder);
 
+// --------------------------------------------------------------------------
+// OpKernelContext routines
+
+// TF_NumInputs returns the number of inputs available in ctx.
+TF_CAPI_EXPORT extern int TF_NumInputs(TF_OpKernelContext* ctx);
+
+// TF_NumOutputs returns the number of outputs to be placed in *ctx by the
+// kernel.
+TF_CAPI_EXPORT extern int TF_NumOutputs(TF_OpKernelContext* ctx);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index e706c7c1d9..fdcf6518c5 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -31,7 +31,6 @@ struct MyCustomKernel {
 static bool delete_called = false;
 
 static void* MyCreateFunc(TF_OpKernelConstruction* ctx) {
-  LOG(INFO) << "Wow, actually got into creation";
   struct MyCustomKernel* s = new struct MyCustomKernel;
   s->created = true;
   s->compute_called = false;
@@ -51,12 +50,31 @@ static void MyDeleteFunc(void* kernel) {
   delete s;
 }
 
+namespace tensorflow {
+
+static std::unique_ptr<OpKernel> GetFakeKernel(const char* device_name,
+                                               const char* op_name,
+                                               Status* status) {
+  NodeDef def;
+  def.set_op(op_name);
+  def.set_device(device_name);
+  def.add_input("input1");
+  def.add_input("input2");
+  return CreateOpKernel(DeviceType(device_name), nullptr, nullptr, def, 1,
+                        status);
+}
+
 // Tests registration of a single C kernel and checks that calls through the
 // C/C++ boundary are being made.
 TEST(TestKernel, TestRegisterKernelBuilder) {
   const char* kernel_name = "SomeKernelName";
   const char* op_name = "FooOp";
-  const char* device_name = "barDev";
+  const char* device_name = "FakeDeviceName1";
+
+  REGISTER_OP(op_name)
+      .Input("input1: double")
+      .Input("input2: uint8")
+      .Output("output1: uint8");
 
   TF_KernelBuilder* builder = TF_NewKernelBuilder(
       op_name, device_name, &MyCreateFunc, &MyComputeFunc, &MyDeleteFunc);
@@ -65,35 +83,94 @@ TEST(TestKernel, TestRegisterKernelBuilder) {
     TF_Status* status = TF_NewStatus();
     TF_RegisterKernelBuilder(kernel_name, builder, status);
     EXPECT_EQ(TF_OK, TF_GetCode(status));
-    TF_Buffer* buf = TF_GetRegisteredKernelsForOp("FooOp", status);
+    TF_Buffer* buf = TF_GetRegisteredKernelsForOp(op_name, status);
     EXPECT_EQ(TF_OK, TF_GetCode(status));
-    ::tensorflow::KernelList list;
+    KernelList list;
     list.ParseFromArray(buf->data, buf->length);
     ASSERT_EQ(1, list.kernel_size());
-    ASSERT_EQ("barDev", list.kernel(0).device_type());
+    ASSERT_EQ(device_name, list.kernel(0).device_type());
     TF_DeleteBuffer(buf);
     TF_DeleteStatus(status);
   }
 
-  REGISTER_OP("FooOp")
+  {
+    Status status;
+    std::unique_ptr<OpKernel> kernel =
+        GetFakeKernel(device_name, op_name, &status);
+    TF_EXPECT_OK(status);
+    ASSERT_NE(nullptr, kernel.get());
+    kernel->Compute(nullptr);
+  }
+
+  ASSERT_TRUE(delete_called);
+}
+
+class DummyDevice : public DeviceBase {
+ public:
+  DummyDevice(Env* env, bool save) : DeviceBase(env), save_(save) {}
+  bool RequiresRecordingAccessedTensors() const override { return save_; }
+  Allocator* GetAllocator(AllocatorAttributes /*attr*/) override {
+    return cpu_allocator();
+  }
+
+ private:
+  bool save_;
+};
+
+TEST(TestKernel, TestInputAndOutputCount) {
+  const char* kernel_name = "InputOutputCounterKernel";
+  const char* op_name = "BarOp";
+  const char* device_name = "FakeDeviceName2";
+
+  REGISTER_OP(op_name)
       .Input("input1: double")
       .Input("input2: uint8")
       .Output("output1: uint8");
 
+  static int num_inputs = 0;
+  static int num_outputs = 0;
+
+  // A kernel whose Compute function has a side-effect of updating num_inputs
+  // and num_outputs.
+  auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) {
+    num_inputs = TF_NumInputs(ctx);
+    num_outputs = TF_NumOutputs(ctx);
+  };
+
+  TF_KernelBuilder* builder = TF_NewKernelBuilder(op_name, device_name, nullptr,
+                                                  my_compute_func, nullptr);
+
+  {
+    TF_Status* status = TF_NewStatus();
+    TF_RegisterKernelBuilder(kernel_name, builder, status);
+    EXPECT_EQ(TF_OK, TF_GetCode(status));
+    TF_DeleteStatus(status);
+  }
+
   {
-    ::tensorflow::NodeDef def;
-    def.set_op("FooOp");
-    def.set_device("bar");
-    def.add_input("input1");
-    def.add_input("input2");
-    ::tensorflow::Status status;
-    std::unique_ptr<::tensorflow::OpKernel> kernel =
-        ::tensorflow::CreateOpKernel(::tensorflow::DeviceType("barDev"),
-                                     nullptr, nullptr, def, 1, &status);
+    OpKernelContext::Params p;
+    DummyDevice dummy_device(nullptr, false);
+    p.device = &dummy_device;
+
+    gtl::InlinedVector<TensorValue, 4> inputs;
+    // Simulate 2 inputs
+    inputs.emplace_back();
+    inputs.emplace_back();
+    p.inputs = &inputs;
+
+    Status status;
+    std::unique_ptr<OpKernel> kernel =
+        GetFakeKernel(device_name, op_name, &status);
     TF_EXPECT_OK(status);
     ASSERT_NE(nullptr, kernel.get());
-    kernel->Compute(nullptr);
-  }
 
-  ASSERT_TRUE(delete_called);
+    p.op_kernel = kernel.get();
+    OpKernelContext ctx(&p);
+    kernel->Compute(&ctx);
+
+    ASSERT_EQ(2, num_inputs);
+    ASSERT_EQ(1, num_outputs);
+  }
 }
+
+}  // namespace tensorflow
-- 
GitLab


From b2cad94ff7507f7b9e1607f6cf69618b3751e7ab Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 4 Dec 2018 10:38:41 -0800
Subject: [PATCH 1291/1554] Disable tests with long paths on windows

PiperOrigin-RevId: 224004201
---
 .../data/experimental/kernel_tests/serialization/BUILD       | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index c724987b24..4a2e28f496 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -76,6 +76,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip",
+        "no_windows",
         "notsan",
     ],
     deps = [
@@ -316,6 +317,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip",
+        "no_windows",
         "notap",
     ],
     deps = [
@@ -358,6 +360,9 @@ py_test(
     size = "small",
     srcs = ["matching_files_dataset_serialization_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "no_windows",
+    ],
     deps = [
         ":dataset_serialization_test_base",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From 08cbdcdc920053e3937183a456132c5ac4814a7c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 10:40:47 -0800
Subject: [PATCH 1292/1554] Introduced a new XRTWriteLiteral op to allow a
 client to overwrite the values stored in device memory.

PiperOrigin-RevId: 224004631
---
 .../compiler/xrt/kernels/xrt_state_ops.cc     | 13 +++++
 .../compiler/xrt/kernels/xrt_state_ops.h      | 50 ++++++++++++++++++
 tensorflow/compiler/xrt/ops/xrt_state_ops.cc  | 14 +++++
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 52 ++++++++++++++++++-
 tensorflow/compiler/xrt/xrt_state.cc          | 14 +++++
 tensorflow/compiler/xrt/xrt_state.h           |  3 ++
 6 files changed, 145 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
index ffea592491..3258286c10 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
@@ -87,6 +87,19 @@ REGISTER_KERNEL_BUILDER(Name("XRTReadLiteral")
                             .HostMemory("literal"),
                         XRTReadLiteralOp<false, XRTGenericDeviceAccessor>);
 
+REGISTER_KERNEL_BUILDER(Name("XRTWriteLiteral")
+                            .Device(DEVICE_XLA_GPU)
+                            .HostMemory("handle")
+                            .HostMemory("literal")
+                            .HostMemory("output_handle"),
+                        XRTWriteLiteralOp<XRTGenericDeviceAccessor>);
+REGISTER_KERNEL_BUILDER(Name("XRTWriteLiteral")
+                            .Device(DEVICE_XLA_CPU)
+                            .HostMemory("handle")
+                            .HostMemory("literal")
+                            .HostMemory("output_handle"),
+                        XRTWriteLiteralOp<XRTGenericDeviceAccessor>);
+
 REGISTER_KERNEL_BUILDER(Name("XRTReadLiteralAndRelease")
                             .Device(DEVICE_XLA_GPU)
                             .HostMemory("handle")
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
index 54b06558ad..26a58fa42d 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
@@ -393,6 +393,56 @@ class XRTReadLiteralOp : public OpKernel {
   }
 };
 
+// Op that writes a new literal value into device-resident memory.
+template <class DeviceAccessor>
+class XRTWriteLiteralOp : public OpKernel {
+ public:
+  explicit XRTWriteLiteralOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+  ~XRTWriteLiteralOp() override = default;
+  XRTWriteLiteralOp(const XRTWriteLiteralOp&) = delete;
+  XRTWriteLiteralOp& operator=(const XRTWriteLiteralOp&) = delete;
+
+  void Compute(OpKernelContext* ctx) override {
+    VLOG(1) << "XRTWriteLiteralOp::Compute";
+
+    const Tensor& handle_tensor = ctx->input(0);
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsScalar(handle_tensor.shape()),
+        errors::Internal("computation input should be an int64 scalar"));
+    int64 allocation_handle = handle_tensor.scalar<int64>()();
+
+    const Tensor& literal_info = ctx->input(1);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(literal_info.shape()),
+                errors::Internal("literal input should be a string scalar"));
+    xla::LiteralProto literal_proto;
+    OP_REQUIRES(ctx,
+                literal_proto.ParseFromString(literal_info.scalar<string>()()),
+                errors::InvalidArgument(
+                    "Unable to parse allocation input to LiteralProto"));
+    xla::Literal literal;
+    OP_REQUIRES_OK(ctx, XRTStateHelpers::MakeLiteral(literal_proto, &literal));
+
+    ResourceMgr* rm;
+    OP_REQUIRES_OK(ctx, DeviceAccessor::GetResourceManager(ctx, &rm));
+
+    XRTTupleAllocation* allocation;
+    OP_REQUIRES_OK(
+        ctx, XRTTupleAllocation::Lookup(rm, allocation_handle, &allocation));
+    core::ScopedUnref allocation_unref(allocation);
+    // We are guaranteed that the underlying device object won't be deleted out
+    // from under us, while the ScopedRef is live.
+    typename DeviceAccessor::ScopedRef device_ref;
+    OP_REQUIRES_OK(ctx, DeviceAccessor::InitScopedRef(
+                            ctx, allocation->device_ordinal(), &device_ref));
+    OP_REQUIRES_OK(ctx,
+                   allocation->WriteLiteral(device_ref.backend(), literal));
+
+    Tensor output(DT_INT64, TensorShape({}));
+    output.scalar<int64>()() = allocation_handle;
+    ctx->set_output(0, output);
+  }
+};
+
 // Op that discards a handle to device memory.
 template <class DeviceAccessor>
 class XRTReleaseAllocationOp : public OpKernel {
diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
index 07d025ce34..a3d63106fa 100644
--- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
@@ -95,6 +95,20 @@ Copies an allocated tuple from device memory and returns it as a literal.
 'literal' is a serialized xla::LiteralProto proto.
 )");
 
+REGISTER_OP("XRTWriteLiteral")
+    .Input("handle: int64")
+    .Input("literal: string")
+    .Output("output_handle: int64")
+    .SetShapeFn(tensorflow::shape_inference::ScalarShape)
+    .Doc(
+        R"(
+Copies the input literal into the device memory pointed to by handle.
+Returns the handle itself.
+
+'handle' is the id returned from the Op that produced the on-device allocation.
+'literal' is a serialized xla::LiteralProto proto to be written to device memory.
+)");
+
 REGISTER_OP("XRTReadLiteralAndRelease")
     .Input("handle: int64")
     .Output("literal: string")
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index aa0eabfce7..abaa17e50e 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -102,7 +102,7 @@ bool CompareLiteralProtos(const xla::LiteralProto& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match " << a.DebugString()
+    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
               << " != " << b.DebugString();
   }
   return equal;
@@ -215,6 +215,56 @@ xla::ProgramShape XlaCompiledProgramShape(
       ->ComputeProgramShape();
 }
 
+TEST(RawApiTest, AllocAndRewrite) {
+  xrt::XLAAllocation alloc;
+  alloc.set_device_ordinal(0);
+  *alloc.mutable_value() =
+      xla::LiteralUtil::CreateR2({{4, 5}, {6, 7}}).ToProto();
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  auto value =
+      ops::Const(root.WithDevice("/device:CPU:0"), alloc.SerializeAsString());
+  auto handle = ops::XRTAllocate(root, value);
+  auto read_back = ops::XRTReadLiteral(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  tensorflow::ClientSession session(root);
+  std::vector<tensorflow::Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back, handle}, &outputs));
+  EXPECT_EQ(outputs.size(), 2);
+
+  int64 allocation_handle = outputs[1].scalar<int64>()();
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralProtos(alloc.value(), response));
+  outputs.clear();
+
+  xla::LiteralProto new_literal =
+      xla::LiteralUtil::CreateR2({{9, 2}, {4, 1}}).ToProto();
+  auto new_value = ops::Const(root.WithDevice("/device:CPU:0"),
+                              new_literal.SerializeAsString());
+  auto write_op =
+      ops::XRTWriteLiteral(root, Input(allocation_handle), new_value);
+  TF_ASSERT_OK(root.status());
+  TF_EXPECT_OK(session.Run({write_op}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+  EXPECT_EQ(allocation_handle, outputs[0].scalar<int64>()());
+  outputs.clear();
+
+  auto read_after_write = ops::XRTReadLiteral(root, Input(allocation_handle));
+  TF_EXPECT_OK(session.Run({read_after_write}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto new_response;
+  EXPECT_TRUE(new_response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralProtos(new_literal, new_response));
+
+  auto release =
+      ops::XRTReleaseAllocationHandle(root, Input(allocation_handle));
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
+                           &outputs));
+}
+
 TEST(RawApiTest, ReadAndWriteState) {
   xrt::XLAAllocation alloc;
   alloc.set_device_ordinal(0);
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 3a99820d7a..5c7c537c34 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -183,6 +183,20 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
   return Status::OK();
 }
 
+Status XRTTupleAllocation::WriteLiteral(xla::Backend* backend,
+                                        const xla::Literal& literal) {
+  if (!xla::ShapeUtil::Equal(literal.shape(), on_host_shape())) {
+    return errors::InvalidArgument(
+        "New literal shape not matching the existing one: literal=",
+        xla::ShapeUtil::HumanStringWithLayout(literal.shape()),
+        " device=", xla::ShapeUtil::HumanStringWithLayout(on_host_shape()));
+  }
+  auto transfer_manager = backend->transfer_manager();
+  TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal()));
+  return transfer_manager->TransferLiteralToDevice(stream.get(), literal,
+                                                   ToShapedBuffer());
+}
+
 void XRTTupleAllocation::DiscardAllocation(
     const xla::ShapeIndex& buffer_index) {
   buffers_.element(buffer_index)->DiscardAllocation();
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 73b5584e38..3664c0cd4e 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -137,6 +137,9 @@ class XRTTupleAllocation : public ResourceBase {
   Status ToLiteral(xla::Backend* backend, int device_ordinal,
                    xla::Literal* literal);
 
+  // Write a new literal value to the allocation.
+  Status WriteLiteral(xla::Backend* backend, const xla::Literal& literal);
+
   // True if none of the buffers in the allocation are aliased by any other live
   // handle.
   bool IsExclusiveOwner();
-- 
GitLab


From 07f30871a02654ddecb017bf383a3790d60f9d96 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 10:51:55 -0800
Subject: [PATCH 1293/1554] Bump max_bazel_version to 0.20.0

After a few fixes, TensorFlow now builds with Bazel 0.20.0. This allows us to keep testing TF on Bazel CI, where bazel is already upgraded to 0.20.0

PiperOrigin-RevId: 224006905
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 3a7999fbd6..6c905a0be3 100644
--- a/configure.py
+++ b/configure.py
@@ -1565,7 +1565,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.15.0', '0.19.2')
+  check_bazel_version('0.15.0', '0.20.0')
 
   reset_tf_configure_bazelrc()
   # Explicitly import tools/bazel.rc, this is needed for Bazel 0.19.0 or later
-- 
GitLab


From 6ff7ff868887aa6567344b030147dad7b5f8f3d2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 11:03:04 -0800
Subject: [PATCH 1294/1554] BEGIN_PUBLIC Rolling back an earlier change which
 breaks some deps. END_PUBLIC

Automated rollback of commit 7fa9d6a1f2d5916f3873ea38ed59c69af4c54e70

PiperOrigin-RevId: 224009307
---
 tensorflow/c/c_api.cc                         | 57 +++++++++----------
 tensorflow/compiler/jit/xla_launch_util.h     | 10 ++--
 .../common_runtime/gpu/gpu_event_mgr_test.cc  |  4 +-
 tensorflow/core/framework/tensor.cc           | 17 +++---
 tensorflow/core/framework/tensor.h            | 17 ++----
 tensorflow/core/framework/tensor_test.cc      | 21 -------
 tensorflow/lite/delegates/flex/buffer_map.cc  | 57 +++++++++----------
 tensorflow/python/lib/core/py_func.cc         |  6 +-
 8 files changed, 79 insertions(+), 110 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 94d18eb8b0..f13e8777df 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -136,22 +136,16 @@ const char* TF_Message(const TF_Status* s) {
 namespace {
 class TF_ManagedBuffer : public TensorBuffer {
  public:
-  TF_ManagedBuffer(void* data, size_t len,
-                   void (*deallocator)(void* data, size_t len, void* arg),
-                   void* deallocator_arg)
-      : TensorBuffer(data),
-        len_(len),
-        deallocator_(deallocator),
-        deallocator_arg_(deallocator_arg) {}
-
-  const size_t len_;
-  void (*const deallocator_)(void* data, size_t len, void* arg);
-  void* const deallocator_arg_;
+  void* data_;
+  size_t len_;
+  void (*deallocator_)(void* data, size_t len, void* arg);
+  void* deallocator_arg_;
 
   ~TF_ManagedBuffer() override {
-    (*deallocator_)(data(), len_, deallocator_arg_);
+    (*deallocator_)(data_, len_, deallocator_arg_);
   }
 
+  void* data() const override { return data_; }
   size_t size() const override { return len_; }
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -205,7 +199,8 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims,
     dimvec[i] = static_cast<tensorflow::int64>(dims[i]);
   }
 
-  TF_ManagedBuffer* buf = nullptr;
+  TF_ManagedBuffer* buf = new TF_ManagedBuffer;
+  buf->len_ = len;
   if (dtype != TF_STRING && dtype != TF_RESOURCE &&
       tensorflow::DataTypeCanUseMemcpy(static_cast<DataType>(dtype)) &&
       reinterpret_cast<intptr_t>(data) % std::max(1, EIGEN_MAX_ALIGN_BYTES) !=
@@ -217,15 +212,17 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims,
     //
     // Other types have the same representation, so copy only if it is safe to
     // do so.
-    buf = new TF_ManagedBuffer(allocate_tensor("TF_NewTensor", len), len,
-                               deallocate_buffer, nullptr);
-    std::memcpy(buf->data(), data, len);
+    buf->data_ = allocate_tensor("TF_NewTensor", len);
+    std::memcpy(buf->data_, data, len);
+    buf->deallocator_ = deallocate_buffer;
+    buf->deallocator_arg_ = nullptr;
     // Free the original buffer.
     deallocator(data, len, deallocator_arg);
   } else {
-    buf = new TF_ManagedBuffer(data, len, deallocator, deallocator_arg);
+    buf->data_ = data;
+    buf->deallocator_ = deallocator;
+    buf->deallocator_arg_ = deallocator_arg;
   }
-
   TF_Tensor* ret = new TF_Tensor{dtype, TensorShape(dimvec), buf};
   size_t elem_size = TF_DataTypeSize(dtype);
   if (elem_size > 0 && len < (elem_size * ret->shape.num_elements())) {
@@ -480,9 +477,9 @@ static TF_Tensor* EmptyTensor(TF_DataType dtype, const TensorShape& shape) {
   CHECK_EQ(nelems, 0);
   static_assert(sizeof(int64_t) == sizeof(tensorflow::int64),
                 "64-bit int types should match in size");
-  return TF_NewTensor(
-      dtype, reinterpret_cast<const int64_t*>(dims.data()), shape.dims(),
-      reinterpret_cast<void*>(&empty), 0, [](void*, size_t, void*) {}, nullptr);
+  return TF_NewTensor(dtype, reinterpret_cast<const int64_t*>(dims.data()),
+                      shape.dims(), reinterpret_cast<void*>(&empty), 0,
+                      [](void*, size_t, void*) {}, nullptr);
 }
 
 // Non-static for testing.
@@ -1595,20 +1592,18 @@ TF_AttrMetadata TF_OperationGetAttrMetadata(TF_Operation* oper,
     break;                                            \
   }
 
-      LIST_CASE(
-          s, TF_ATTR_STRING, metadata.total_size = 0;
-          for (int i = 0; i < attr->list().s_size();
-               ++i) { metadata.total_size += attr->list().s(i).size(); });
+      LIST_CASE(s, TF_ATTR_STRING, metadata.total_size = 0;
+                for (int i = 0; i < attr->list().s_size();
+                     ++i) { metadata.total_size += attr->list().s(i).size(); });
       LIST_CASE(i, TF_ATTR_INT);
       LIST_CASE(f, TF_ATTR_FLOAT);
       LIST_CASE(b, TF_ATTR_BOOL);
       LIST_CASE(type, TF_ATTR_TYPE);
-      LIST_CASE(
-          shape, TF_ATTR_SHAPE, metadata.total_size = 0;
-          for (int i = 0; i < attr->list().shape_size(); ++i) {
-            const auto& s = attr->list().shape(i);
-            metadata.total_size += s.unknown_rank() ? 0 : s.dim_size();
-          });
+      LIST_CASE(shape, TF_ATTR_SHAPE, metadata.total_size = 0;
+                for (int i = 0; i < attr->list().shape_size(); ++i) {
+                  const auto& s = attr->list().shape(i);
+                  metadata.total_size += s.unknown_rank() ? 0 : s.dim_size();
+                });
       LIST_CASE(tensor, TF_ATTR_TENSOR);
       LIST_CASE(tensor, TF_ATTR_FUNC);
 #undef LIST_CASE
diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 706811f55e..437db019a0 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -199,17 +199,19 @@ class XlaTensorBuffer : public TensorBuffer {
  public:
   XlaTensorBuffer(const void* ptr, size_t expected_size, size_t actual_size,
                   Allocator* allocator)
-      : TensorBuffer(const_cast<void*>(ptr)),
-        expected_size_(expected_size),
+      : expected_size_(expected_size),
         actual_size_(actual_size),
-        allocator_(allocator) {}
+        allocator_(allocator) {
+    data_ = const_cast<void*>(ptr);
+  }
 
   ~XlaTensorBuffer() override {
     if (data_) {
-      allocator_->DeallocateRaw(data());
+      allocator_->DeallocateRaw(data_);
     }
   }
 
+  void* data() const override { return data_; }
   size_t size() const override { return expected_size_; }
 
   TensorBuffer* root_buffer() override { return this; }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
index fe32147557..d2adf699f5 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
@@ -78,8 +78,7 @@ static std::atomic_int_fast64_t live_tensor_bytes(0);
 // A TensorBuffer that counts live memory usage for testing
 class TestTensorBuffer : public TensorBuffer {
  public:
-  explicit TestTensorBuffer(size_t bytes)
-      : TensorBuffer(nullptr), bytes_(bytes) {
+  explicit TestTensorBuffer(size_t bytes) : bytes_(bytes) {
     live_tensor_bytes += bytes_;
   }
   ~TestTensorBuffer() override { live_tensor_bytes -= bytes_; }
@@ -87,6 +86,7 @@ class TestTensorBuffer : public TensorBuffer {
   size_t size() const override { return bytes_; }
 
   // Not used in this test
+  void* data() const override { return nullptr; }
   TensorBuffer* root_buffer() override { return nullptr; }
   void FillAllocationDescription(AllocationDescription* arg) const override {}
 
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index 7e841489eb..c7ddc6c21e 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -68,8 +68,7 @@ namespace {
 // An un-templated base class for Buffer.
 class BufferBase : public TensorBuffer {
  public:
-  explicit BufferBase(Allocator* alloc, void* data_ptr)
-      : TensorBuffer(data_ptr), alloc_(alloc) {}
+  explicit BufferBase(Allocator* alloc) : alloc_(alloc) {}
 
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -107,6 +106,7 @@ class Buffer : public BufferBase {
   Buffer(Allocator* a, int64 n);
   Buffer(Allocator* a, int64 n, const AllocationAttributes& allocation_attr);
 
+  void* data() const override { return data_; }
   size_t size() const override { return sizeof(T) * elem_; }
 
  private:
@@ -442,20 +442,20 @@ struct ProtoHelper<Eigen::half> {
 
 template <typename T>
 Buffer<T>::Buffer(Allocator* a, int64 n)
-    : BufferBase(a, a->Allocate<T>(n)), elem_(n) {}
+    : BufferBase(a), data_(a->Allocate<T>(n)), elem_(n) {}
 
 template <typename T>
 Buffer<T>::Buffer(Allocator* a, int64 n,
                   const AllocationAttributes& allocation_attr)
-    : BufferBase(a, a->Allocate<T>(n, allocation_attr)), elem_(n) {}
+    : BufferBase(a), data_(a->Allocate<T>(n, allocation_attr)), elem_(n) {}
 
 template <typename T>
 Buffer<T>::~Buffer() {
-  if (data()) {
+  if (data_) {
     if (LogMemory::IsEnabled()) {
       RecordDeallocation();
     }
-    alloc_->Deallocate<T>(static_cast<T*>(data()), elem_);
+    alloc_->Deallocate<T>(data_, elem_);
   }
 }
 
@@ -764,9 +764,7 @@ class SubBuffer : public TensorBuffer {
  public:
   // This buffer is an alias to buf[delta, delta + n).
   SubBuffer(TensorBuffer* buf, int64 delta, int64 n)
-      : TensorBuffer(buf->base<T>() + delta),
-        root_(buf->root_buffer()),
-        elem_(n) {
+      : root_(buf->root_buffer()), data_(buf->base<T>() + delta), elem_(n) {
     // Sanity check. The caller should ensure the sub buffer is valid.
     CHECK_LE(root_->base<T>(), this->base<T>());
     T* root_limit = root_->base<T>() + root_->size() / sizeof(T);
@@ -777,6 +775,7 @@ class SubBuffer : public TensorBuffer {
     root_->Ref();
   }
 
+  void* data() const override { return data_; }
   size_t size() const override { return sizeof(T) * elem_; }
   TensorBuffer* root_buffer() override { return root_; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 6e03cf9f6f..3177bbe7e9 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -636,15 +636,10 @@ class Tensor {
 // Interface to access the raw ref-counted data buffer.
 class TensorBuffer : public core::RefCounted {
  public:
-  explicit TensorBuffer(void* data_ptr) : data_(data_ptr) {}
   ~TensorBuffer() override {}
 
   // data() points to a memory region of size() bytes.
-  //
-  // NOTE(mrry): The `data()` method is not virtual for performance reasons.
-  // It can be called multiple times when the contents of a `Tensor` are
-  // accessed, and so making it non-virtual allows the body to be inlined.
-  void* data() const { return data_; }
+  virtual void* data() const = 0;
   virtual size_t size() const = 0;
 
   // If this TensorBuffer is sub-buffer of another TensorBuffer,
@@ -662,9 +657,6 @@ class TensorBuffer : public core::RefCounted {
 
   // Whether this TensorBuffer owns the underlying memory.
   virtual bool OwnsMemory() const { return true; }
-
- private:
-  void* const data_;
 };
 
 template <typename T>
@@ -882,7 +874,6 @@ inline Tensor::Tensor(Tensor&& other)
 
 class Tensor::HostScalarTensorBufferBase : public TensorBuffer {
  public:
-  using TensorBuffer::TensorBuffer;
   void FillAllocationDescription(AllocationDescription* proto) const final;
 };
 
@@ -893,7 +884,8 @@ template <typename T>
 struct Tensor::ValueAndTensorBuffer {
   class HostScalarTensorBuffer : public Tensor::HostScalarTensorBufferBase {
    public:
-    HostScalarTensorBuffer(void* data) : HostScalarTensorBufferBase(data) {}
+    HostScalarTensorBuffer(void* data) : data_(data) {}
+    void* data() const final { return const_cast<void*>(data_); }
     size_t size() const final { return sizeof(T); }
     TensorBuffer* root_buffer() final { return this; }
 
@@ -912,7 +904,8 @@ struct Tensor::ValueAndTensorBuffer {
     }
 
    private:
-    ~HostScalarTensorBuffer() override { static_cast<T*>(data())->~T(); }
+    ~HostScalarTensorBuffer() override { static_cast<T*>(data_)->~T(); }
+    void* const data_;
   };
 
   T value;
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 713f91fe04..4fa9d1df67 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -1491,26 +1491,5 @@ void BM_CreateAndMoveCtrWithBuf(int iters) {
 }
 BENCHMARK(BM_CreateAndMoveCtrWithBuf);
 
-// Benchmark creating and destroy a host-scalar tensor, using the allocator
-// interface.
-void BM_CreateAndDestroyHostScalarNonOptimized(int iters) {
-  TensorShape shape({});
-  Allocator* allocator = cpu_allocator();
-  while (--iters) {
-    Tensor a(allocator, DT_FLOAT, shape);
-    a.scalar<float>()() = 37.0;
-  }
-}
-BENCHMARK(BM_CreateAndDestroyHostScalarNonOptimized);
-
-// Benchmark creating and destroy a host-scalar tensor, using the specialized
-// constructor.
-void BM_CreateAndDestroyHostScalarOptimized(int iters) {
-  while (--iters) {
-    Tensor a(37.0);
-  }
-}
-BENCHMARK(BM_CreateAndDestroyHostScalarOptimized);
-
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc
index 0d0c953636..262ca9e089 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map.cc
@@ -26,8 +26,6 @@ namespace flex {
 namespace {
 // A tensor buffer that is allocated, deallocated and populated by TF Lite.
 class BaseTfLiteTensorBuffer : public tensorflow::TensorBuffer {
-  using tensorflow::TensorBuffer::TensorBuffer;
-
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(
       tensorflow::AllocationDescription* proto) const override {
@@ -62,29 +60,31 @@ class BaseTfLiteTensorBuffer : public tensorflow::TensorBuffer {
 // representation in TFLITE and TF, so we just need use memcpy().
 class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
-  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor)
-      : BaseTfLiteTensorBuffer(tensorflow::cpu_allocator()->AllocateRaw(
-            EIGEN_MAX_ALIGN_BYTES, tensor->bytes)) {
+  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor) {
     // TODO(ahentz): if we can guarantee that TF Lite allocated tensors with
     // the same alignment as TensorFlow (EIGEN_MAX_ALIGN_BYTES), then we can
     // potentially eliminate the copy below.
     len_ = tensor->bytes;
+    data_ =
+        tensorflow::cpu_allocator()->AllocateRaw(EIGEN_MAX_ALIGN_BYTES, len_);
 
     LogAllocation();
 
-    if (data()) {
-      std::memcpy(data(), tensor->data.raw, tensor->bytes);
+    if (data_) {
+      std::memcpy(data_, tensor->data.raw, tensor->bytes);
     }
   }
 
   ~TfLiteTensorBuffer() override {
     LogDeallocation();
-    tensorflow::cpu_allocator()->DeallocateRaw(data());
+    tensorflow::cpu_allocator()->DeallocateRaw(data_);
   }
 
+  void* data() const override { return data_; }
   size_t size() const override { return len_; }
 
  private:
+  void* data_;
   size_t len_;
 };
 
@@ -92,30 +92,19 @@ class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
 // TF's so we need perform the conversion here.
 class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
-  explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor)
-      : StringTfLiteTensorBuffer(tensor, tensor->data.raw != nullptr
-                                             ? GetStringCount(tensor->data.raw)
-                                             : 0) {}
-
-  ~StringTfLiteTensorBuffer() override {
-    LogDeallocation();
-    tensorflow::cpu_allocator()->Deallocate<string>(
-        static_cast<string*>(data()), num_strings_);
-  }
-
-  size_t size() const override { return num_strings_ * sizeof(string); }
+  explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor) {
+    if (tensor->data.raw == nullptr) {
+      num_strings_ = 0;
+      data_ = nullptr;
+      return;
+    }
+    num_strings_ = GetStringCount(tensor->data.raw);
+    data_ = tensorflow::cpu_allocator()->Allocate<string>(num_strings_);
 
- private:
-  StringTfLiteTensorBuffer(const TfLiteTensor* tensor, int num_strings)
-      : BaseTfLiteTensorBuffer(
-            num_strings != 0
-                ? tensorflow::cpu_allocator()->Allocate<string>(num_strings)
-                : nullptr),
-        num_strings_(num_strings) {
     LogAllocation();
 
-    if (data()) {
-      string* p = static_cast<string*>(data());
+    if (data_) {
+      string* p = data_;
       for (size_t i = 0; i < num_strings_; ++p, ++i) {
         auto ref = GetString(tensor->data.raw, i);
         p->assign(ref.str, ref.len);
@@ -123,6 +112,16 @@ class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
     }
   }
 
+  ~StringTfLiteTensorBuffer() override {
+    LogDeallocation();
+    tensorflow::cpu_allocator()->Deallocate<string>(data_, num_strings_);
+  }
+
+  void* data() const override { return data_; }
+  size_t size() const override { return num_strings_ * sizeof(string); }
+
+ private:
+  string* data_;
   int num_strings_;
 };
 
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index 97bebe8617..9364aec373 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -302,14 +302,15 @@ Status DoCallPyFunc(PyCall* call, bool* out_log_on_error) {
 class NumpyTensorBuffer : public TensorBuffer {
  public:
   NumpyTensorBuffer(PyArrayObject* array, size_t len, void* data)
-      : TensorBuffer(data), array_(array), len_(len) {}
+      : array_(array), len_(len), data_(data) {}
 
   ~NumpyTensorBuffer() override {
     // Note: The session::run wrapper is responsible for freeing this while
     // holding the GIL.
-    DelayedNumpyDecref(data(), len_, array_);
+    DelayedNumpyDecref(data_, len_, array_);
   }
 
+  void* data() const override { return data_; }
   size_t size() const override { return len_; }
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -328,6 +329,7 @@ class NumpyTensorBuffer : public TensorBuffer {
  private:
   PyArrayObject* array_;
   size_t len_;
+  void* data_;
 };
 
 Status PyObjectToString(PyObject* obj, string* str) {
-- 
GitLab


From b2e9755ebc590b462313a156fc6e3fe2d3906d3d Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Tue, 4 Dec 2018 11:17:25 -0800
Subject: [PATCH 1295/1554] Internal-only change.

PiperOrigin-RevId: 224012219
---
 tensorflow/core/BUILD | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c268605711..edf61ca1b9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2087,9 +2087,7 @@ tf_proto_library_cc(
     srcs = ["protobuf/master.proto"],
     cc_api_version = 2,
     protodeps = tf_additional_all_protos(),
-    visibility = [
-        "//tensorflow:internal",
-    ],
+    visibility = ["//tensorflow:internal"],
 )
 
 tf_proto_library_cc(
-- 
GitLab


From 8420f5741558c79fa71b442bcc613800c3b3dfa6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 11:26:58 -0800
Subject: [PATCH 1296/1554] Refactor generator and eager training loops.

PiperOrigin-RevId: 224014115
---
 .../contrib/tpu/python/tpu/keras_support.py   |   4 -
 tensorflow/python/keras/BUILD                 |   1 +
 tensorflow/python/keras/callbacks.py          |  72 +-
 tensorflow/python/keras/callbacks_test.py     |  76 +-
 tensorflow/python/keras/engine/training.py    | 119 +--
 .../python/keras/engine/training_arrays.py    | 137 +--
 .../keras/engine/training_distributed.py      |  16 +-
 .../python/keras/engine/training_eager.py     | 542 +-----------
 .../keras/engine/training_eager_test.py       |   8 +-
 .../python/keras/engine/training_generator.py | 782 +++++++++---------
 .../keras/engine/training_generator_test.py   |  56 ++
 .../python/keras/engine/training_utils.py     | 256 +++---
 .../keras/engine/training_utils_test.py       | 134 ---
 tensorflow/python/keras/layers/local_test.py  |   4 +-
 .../golden/v1/tensorflow.keras.-model.pbtxt   |   8 +-
 .../v1/tensorflow.keras.-sequential.pbtxt     |   8 +-
 .../v1/tensorflow.keras.models.-model.pbtxt   |   8 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   8 +-
 .../golden/v2/tensorflow.keras.-model.pbtxt   |   8 +-
 .../v2/tensorflow.keras.-sequential.pbtxt     |   8 +-
 ...ensorflow.keras.layers.-linear-model.pbtxt |   8 +-
 .../v2/tensorflow.keras.models.-model.pbtxt   |   8 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   8 +-
 23 files changed, 745 insertions(+), 1534 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index 7f280eabd2..ebf40827e4 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -1676,14 +1676,10 @@ class KerasTPUModel(models.Model):
         callbacks,
         self,
         do_validation=do_validation,
-        val_inputs=val_inputs,
-        val_targets=val_targets,
-        val_sample_weights=val_sample_weights,
         batch_size=batch_size,
         epochs=epochs,
         steps_per_epoch=steps_per_epoch,
         samples=num_training_samples,
-        validation_steps=validation_steps,
         verbose=verbose,
         count_mode=count_mode)
 
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 64ca3ac4e0..ab5628b1d2 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -756,6 +756,7 @@ py_test(
     name = "training_generator_test",
     size = "enormous",
     srcs = ["engine/training_generator_test.py"],
+    shard_count = 3,
     srcs_version = "PY2AND3",
     tags = [
         "no_oss",
diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 3e3c087e3f..2d7d5a415d 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -24,7 +24,6 @@ import copy
 import csv
 import io
 import json
-import math
 import os
 import time
 
@@ -35,7 +34,6 @@ from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.keras import backend as K
-from tensorflow.python.keras.engine.training_utils import standardize_input_data
 from tensorflow.python.keras.utils.data_utils import Sequence
 from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
@@ -58,14 +56,10 @@ except ImportError:
 def configure_callbacks(callbacks,
                         model,
                         do_validation=False,
-                        val_inputs=None,
-                        val_targets=None,
-                        val_sample_weights=None,
                         batch_size=None,
                         epochs=None,
                         steps_per_epoch=None,
                         samples=None,
-                        validation_steps=None,
                         verbose=1,
                         count_mode='steps',
                         mode='train'):
@@ -75,17 +69,10 @@ def configure_callbacks(callbacks,
       callbacks: List of Callbacks.
       model: Model being trained.
       do_validation: Whether or not validation loop will be run.
-      val_inputs: Inputs to Model for validation loop. Can be any
-        data format Keras accepts.
-      val_targets: Targets for Model for validation loop. Can be any
-        data format Keras accepts.
-      val_sample_weights: Sample weights for Model for validation loop.
-        Can be any data format Keras accepts.
       batch_size: Number of samples per batch.
       epochs: Number of epoch to train.
       steps_per_epoch: Number of batches to run per training epoch.
       samples: Number of training samples.
-      validation_steps: Number of batches to run per validation epoch.
       verbose: int, 0 or 1. Keras logging verbosity to pass to ProgbarLogger.
       count_mode: One of 'steps' or 'samples'. Per-batch or per-sample count.
       mode: String. One of 'train', 'test', or 'predict'. Which loop mode to
@@ -126,8 +113,6 @@ def configure_callbacks(callbacks,
     callback_metrics = copy.copy(model.metrics_names)
     if do_validation:
       callback_metrics += ['val_' + n for n in model.metrics_names]
-  if validation_steps is None and isinstance(val_inputs, Sequence):
-    validation_steps = len(val_inputs)
   callback_params = {
       'batch_size': batch_size,
       'epochs': epochs,
@@ -136,30 +121,15 @@ def configure_callbacks(callbacks,
       'verbose': verbose,
       'do_validation': do_validation,
       'metrics': callback_metrics,
-      'validation_steps': validation_steps
   }
   callback_list.set_params(callback_params)
 
-  # Pass validation data to callbacks
-  # TODO(omalleyt): remove this once val hooks are ready.
-  if model._distribution_strategy or not val_inputs:
-    val_data = []
-  else:
-    if not model.run_eagerly:
-      # Need to create the eval_function before start of the first epoch
-      # because TensorBoard callback on_epoch_begin adds summary to the
-      # list of fetches of the eval_function
-      callback_model._make_eval_function()
-    if _is_generator_like(val_inputs):
-      val_data = val_inputs
-    else:
-      val_data = val_inputs + val_targets
-      if val_sample_weights:
-        val_data += val_sample_weights
-      if not isinstance(K.symbolic_learning_phase(), int):
-        val_data += [False]
-  for cbk in callbacks:
-    cbk.validation_data = val_data
+  if (do_validation and not model._distribution_strategy and
+      not model.run_eagerly):
+    # Need to create the eval_function before start of the first epoch
+    # because TensorBoard callback on_epoch_begin adds summary to the
+    # list of fetches of the eval_function
+    callback_model._make_eval_function()
 
   callback_list.model.stop_training = False
   return callback_list
@@ -957,6 +927,7 @@ class TensorBoard(Callback):
     self.batch_size = batch_size
     self._current_batch = 0
     self._total_batches_seen = 0
+    self._total_val_batches_seen = 0
     self.embeddings_freq = embeddings_freq
     self.embeddings_layer_names = embeddings_layer_names
     self.embeddings_metadata = embeddings_metadata
@@ -1045,8 +1016,10 @@ class TensorBoard(Callback):
     # If both embedding_freq and embeddings_data are available, we will
     # visualize embeddings.
     if self.embeddings_freq and self.embeddings_data is not None:
-      self.embeddings_data = standardize_input_data(self.embeddings_data,
-                                                    model.input_names)
+      # Avoid circular dependency.
+      from tensorflow.python.keras.engine import training_utils  # pylint: disable=g-import-not-at-top
+      self.embeddings_data = training_utils.standardize_input_data(
+          self.embeddings_data, model.input_names)
 
       # If embedding_layer_names are not provided, get all of the embedding
       # layers from the model.
@@ -1111,10 +1084,8 @@ class TensorBoard(Callback):
       projector.visualize_embeddings(self.writer, config)
 
   def _fetch_callback(self, summary):
-    self.writer.add_summary(
-        summary,
-        self._epoch + self._current_val_batch / self._validation_batches)
-    self._current_val_batch += 1
+    self.writer.add_summary(summary, self._total_val_batches_seen)
+    self._total_val_batches_seen += 1
 
   def _write_custom_summaries(self, step, logs=None):
     """Writes metrics out as custom scalar summaries.
@@ -1145,22 +1116,6 @@ class TensorBoard(Callback):
         self.writer.add_summary(summary, step)
     self.writer.flush()
 
-  def on_train_begin(self, logs=None):
-    """Checks if histogram summaries can be run."""
-    # will never be set when in eager
-    if self.histogram_freq:
-      if self.params.get('validation_steps', None) is not None:
-        self._validation_batches = self.params['validation_steps']
-      elif self.validation_data:
-        self._validation_batches = math.ceil(
-            self.validation_data[0].shape[0] / self.batch_size)
-      else:
-        raise ValueError('If printing histograms, validation data must be '
-                         'provided.')
-      if self._validation_batches == 0:
-        raise ValueError(
-            'If printing histograms, validation data must have length > 0.')
-
   def on_batch_end(self, batch, logs=None):
     """Writes scalar summaries for metrics on every training batch."""
     # Don't output batch_size and batch number as Tensorboard summaries
@@ -1181,7 +1136,6 @@ class TensorBoard(Callback):
     # check if histogram summary should be run for this epoch
     if self.histogram_freq and epoch % self.histogram_freq == 0:
       self._epoch = epoch
-      self._current_val_batch = 0
       # pylint: disable=protected-access
       # add the histogram summary op if it should run this epoch
       if self.merged not in self.model._eval_function.fetches:
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index ed05572c68..6c9a382b32 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -36,7 +36,6 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import adam
 
 try:
@@ -780,79 +779,6 @@ class KerasCallbacksTest(test.TestCase):
           data_generator(True), len(x_train), epochs=2, callbacks=cbks)
       assert os.path.exists(temp_dir)
 
-  @test_util.run_deprecated_v1
-  def test_TensorBoard_histogram_freq_must_have_validation_data(self):
-    np.random.seed(1337)
-    tmpdir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
-
-    with self.cached_session():
-      filepath = os.path.join(tmpdir, 'logs')
-
-      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-          train_samples=TRAIN_SAMPLES,
-          test_samples=TEST_SAMPLES,
-          input_shape=(INPUT_DIM,),
-          num_classes=NUM_CLASSES)
-      y_test = keras.utils.to_categorical(y_test)
-      y_train = keras.utils.to_categorical(y_train)
-
-      def data_generator(train):
-        if train:
-          max_batch_index = len(x_train) // BATCH_SIZE
-        else:
-          max_batch_index = len(x_test) // BATCH_SIZE
-        i = 0
-        while 1:
-          if train:
-            # simulate multi-input/output models
-            yield (x_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE],
-                   y_train[i * BATCH_SIZE: (i + 1) * BATCH_SIZE])
-          else:
-            yield (x_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE],
-                   y_test[i * BATCH_SIZE: (i + 1) * BATCH_SIZE])
-          i += 1
-          i %= max_batch_index
-
-      inp = keras.Input((INPUT_DIM,))
-      hidden = keras.layers.Dense(2, activation='relu')(inp)
-      hidden = keras.layers.Dropout(0.1)(hidden)
-      output = keras.layers.Dense(NUM_CLASSES, activation='softmax')(hidden)
-      model = keras.models.Model(inputs=inp, outputs=output)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer='sgd',
-                    metrics=['accuracy'])
-
-      # we must generate new callbacks for each test, as they aren't stateless
-      def callbacks_factory(histogram_freq):
-        return [keras.callbacks.TensorBoard(
-            log_dir=filepath,
-            histogram_freq=histogram_freq,
-            write_images=True, write_grads=True,
-            batch_size=5)]
-
-      # fit w/o validation data should raise ValueError if histogram_freq > 0
-      cbs = callbacks_factory(histogram_freq=1)
-      with self.assertRaises(ValueError):
-        model.fit(
-            x_train, y_train, batch_size=BATCH_SIZE, callbacks=cbs, epochs=3)
-
-      for cb in cbs:
-        cb.on_train_end()
-
-      # fit generator without validation data should raise ValueError if
-      # histogram_freq > 0
-      cbs = callbacks_factory(histogram_freq=1)
-      with self.assertRaises(ValueError):
-        model.fit_generator(
-            data_generator(True), len(x_train), epochs=2, callbacks=cbs)
-
-      for cb in cbs:
-        cb.on_train_end()
-
-      # Make sure file writer cache is clear to avoid failures during cleanup.
-      writer_cache.FileWriterCache.clear()
-
   @test_util.run_deprecated_v1
   def test_TensorBoard_multi_input_output(self):
     np.random.seed(1337)
@@ -1001,7 +927,7 @@ class KerasCallbacksTest(test.TestCase):
           epochs=3,
           verbose=0)
 
-      self.assertAllEqual(tsb.writer.steps_seen, [0, 0.5, 1, 1.5, 2, 2.5])
+      self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
 
   @test_util.run_deprecated_v1
   def test_Tensorboard_histogram_summaries_with_generator(self):
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1416f5960f..2236bcf27c 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -940,7 +940,7 @@ class Model(Network):
             name='predict_function',
             **kwargs)
 
-  def _get_execution_function(self, mode):
+  def _make_execution_function(self, mode):
     if mode == 'train':
       self._make_fit_function()
       return self._fit_function
@@ -1764,24 +1764,20 @@ class Model(Network):
       val_y = None
       val_sample_weights = None
 
-    if self.run_eagerly:
-      return training_eager.fit_loop(
-          self,
-          inputs=x,
-          targets=y,
-          sample_weights=sample_weights,
-          class_weight=class_weight,
+    if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and
+                             not self._distribution_strategy)):
+      return training_generator.fit_generator(
+          self, (x, y, sample_weights),
+          steps_per_epoch=steps_per_epoch,
           batch_size=batch_size,
           epochs=epochs,
+          shuffle=shuffle,
           verbose=verbose,
           callbacks=callbacks,
-          val_inputs=val_x,
-          val_targets=val_y,
-          val_sample_weights=val_sample_weights,
-          shuffle=shuffle,
-          initial_epoch=initial_epoch,
-          steps_per_epoch=steps_per_epoch,
-          validation_steps=validation_steps)
+          validation_data=validation_data,
+          validation_steps=validation_steps,
+          workers=0,
+          initial_epoch=initial_epoch)
     elif distributed_training_utils.is_tpu_strategy(
         self._distribution_strategy):
       return training_distributed.experimental_fit_loop(
@@ -1794,19 +1790,6 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps)
-    elif (isinstance(x, iterator_ops.EagerIterator) and
-          not self._distribution_strategy):
-      return training_generator.fit_generator(
-          self,
-          x,
-          steps_per_epoch=steps_per_epoch,
-          epochs=epochs,
-          verbose=verbose,
-          callbacks=callbacks,
-          validation_data=validation_data,
-          validation_steps=validation_steps,
-          workers=0,
-          initial_epoch=initial_epoch)
     else:
       return training_arrays.fit_loop(
           self,
@@ -1934,27 +1917,18 @@ class Model(Network):
         steps_name='steps',
         steps=steps)
 
-    if self.run_eagerly:
-      return training_eager.test_loop(
-          self,
-          inputs=x,
-          targets=y,
-          sample_weights=sample_weights,
+    if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and
+                             not self._distribution_strategy)):
+      return training_generator.evaluate_generator(
+          self, (x, y, sample_weights),
+          steps=steps,
           batch_size=batch_size,
           verbose=verbose,
-          steps=steps)
+          workers=0)
     elif distributed_training_utils.is_tpu_strategy(
         self._distribution_strategy):
       return training_distributed.experimental_test_loop(
           self, iterator=x, verbose=verbose, steps=steps)
-    elif (isinstance(x, iterator_ops.EagerIterator) and
-          not self._distribution_strategy):
-      return training_generator.evaluate_generator(
-          self,
-          x,
-          steps=steps,
-          verbose=verbose,
-          workers=0)
     else:
       return training_arrays.test_loop(
           self,
@@ -2050,26 +2024,35 @@ class Model(Network):
       x, _, _ = self._standardize_user_data(
           x, check_steps=True, steps_name='steps', steps=steps)
 
-    if self.run_eagerly:
-      return training_eager.predict_loop(
-          self, x, batch_size=batch_size, verbose=verbose, steps=steps)
-    elif distributed_training_utils.is_tpu_strategy(
-        self._distribution_strategy):
-      return training_distributed.experimental_predict_loop(
-          self, x, verbose=verbose, steps=steps)
-    elif (isinstance(x, iterator_ops.EagerIterator) and
-          not self._distribution_strategy):
+    if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and
+                             not self._distribution_strategy)):
       return training_generator.predict_generator(
           self,
           x,
           steps=steps,
+          batch_size=batch_size,
           verbose=verbose,
           workers=0)
+    elif distributed_training_utils.is_tpu_strategy(
+        self._distribution_strategy):
+      return training_distributed.experimental_predict_loop(
+          self, x, verbose=verbose, steps=steps)
     else:
       return training_arrays.predict_loop(
           self, x, batch_size=batch_size, verbose=verbose, steps=steps)
 
-  def train_on_batch(self, x, y=None, sample_weight=None, class_weight=None):
+  def reset_metrics(self):
+    """Resets the state of metrics."""
+    if hasattr(self, 'metrics'):
+      for m in self.metrics:
+        m.reset_states()
+
+  def train_on_batch(self,
+                     x,
+                     y=None,
+                     sample_weight=None,
+                     class_weight=None,
+                     reset_metrics=True):
     """Runs a single gradient update on a single batch of data.
 
     Arguments:
@@ -2097,6 +2080,9 @@ class Model(Network):
           weight (float) to apply to the model's loss for the samples from this
           class during training. This can be useful to tell the model to "pay
           more attention" to samples from an under-represented class.
+        reset_metrics: If `True`, the metrics returned will be only for this
+          batch. If `False`, the metrics will be statefully accumulated across
+          batches.
 
     Returns:
         Scalar training loss
@@ -2124,14 +2110,21 @@ class Model(Network):
       else:
         ins = x + y + sample_weights
 
-      self._make_train_function()
-      outputs = self.train_function(ins)  # pylint: disable=not-callable
+      if reset_metrics:
+        self._make_train_function()
+        outputs = self.train_function(ins)  # pylint: disable=not-callable
+      else:
+        self._make_fit_function()
+        outputs = self._fit_function(ins)  # pylint: disable=not-callable
+
+    if reset_metrics:
+      self.reset_metrics()
 
     if len(outputs) == 1:
       return outputs[0]
     return outputs
 
-  def test_on_batch(self, x, y=None, sample_weight=None):
+  def test_on_batch(self, x, y=None, sample_weight=None, reset_metrics=True):
     """Test the model on a single batch of samples.
 
     Arguments:
@@ -2157,6 +2150,9 @@ class Model(Network):
             In this case you should make sure to specify
             sample_weight_mode="temporal" in compile(). This argument is not
             supported when `x` is a dataset or a dataset iterator.
+        reset_metrics: If `True`, the metrics returned will be only for this
+          batch. If `False`, the metrics will be statefully accumulated across
+          batches.
 
     Returns:
         Scalar test loss (if the model has a single output and no metrics)
@@ -2179,8 +2175,15 @@ class Model(Network):
           self, x, y, sample_weights=sample_weights)
     else:
       inputs = x + y + sample_weights
-      self._make_test_function()
-      outputs = self.test_function(inputs)  # pylint: disable=not-callable
+      if reset_metrics:
+        self._make_test_function()
+        outputs = self.test_function(inputs)  # pylint: disable=not-callable
+      else:
+        self._make_eval_function()
+        outputs = self._eval_function(inputs)  # pylint: disable=not-callable
+
+    if reset_metrics:
+      self.reset_metrics()
 
     if len(outputs) == 1:
       return outputs[0]
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index e8a8e0fc02..e9dfbcbcc0 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -39,91 +39,6 @@ except ImportError:
   issparse = None
 
 
-class Aggregator(object):
-  """Abstract base class used to aggregate batch-level outputs of a loop.
-
-  Arguments:
-    use_steps: Whether the loop is using `step` or `batch_size`.
-    num_samples_or_steps: Either `batch_size*num_batches` or `steps`.
-  """
-
-  def __init__(self, use_steps, num_samples_or_steps):
-    self.use_steps = use_steps
-    self.num_samples_or_steps = num_samples_or_steps
-    self.results = []
-
-  def create(self, batch_outs):
-    """Create the initial results from the first batch outputs.
-
-    Arguments:
-      batch_outs: A list of batch-level outputs.
-    """
-    raise NotImplementedError
-
-  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
-    """Aggregate batch-level results into total results.
-
-    Arguments:
-      batch_outs: A list of batch-level outputs.
-      batch_start: The start index of this batch. Always `None` if `use_steps`
-        is `True`.
-      batch_end: The end index of this batch. Always `None` if `use_steps` is
-        `True`.
-    """
-    raise NotImplementedError
-
-  def finalize(self):
-    """Prepare the total results to be returned."""
-    raise NotImplementedError
-
-
-class MetricsAggregator(Aggregator):
-  """Aggregator that calculates loss and metrics info."""
-
-  def create(self, batch_outs):
-    self.results = [0.] * len(batch_outs)
-
-  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
-    # Loss.
-    if self.use_steps:
-      self.results[0] += batch_outs[0]
-    else:
-      self.results[0] += batch_outs[0] * (batch_end - batch_start)
-    # Metrics (always stateful, just grab current values.)
-    self.results[1:] = batch_outs[1:]
-
-  def finalize(self):
-    self.results[0] /= self.num_samples_or_steps
-
-
-class OutputsAggregator(Aggregator):
-  """Aggregator that concatenates outputs."""
-
-  def create(self, batch_outs):
-    if self.use_steps:
-      # Cannot pre-allocate the returned NumPy arrays bc
-      # batch sizes are unknown. Concatenate batches at the end.
-      for _ in batch_outs:
-        self.results.append([])
-    else:
-      # Pre-allocate NumPy arrays.
-      for batch_out in batch_outs:
-        shape = (self.num_samples_or_steps,) + batch_out.shape[1:]
-        self.results.append(np.zeros(shape, dtype=batch_out.dtype))
-
-  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
-    if self.use_steps:
-      for i, batch_out in enumerate(batch_outs):
-        self.results[i].append(batch_out)
-    else:
-      for i, batch_out in enumerate(batch_outs):
-        self.results[i][batch_start:batch_end] = batch_out
-
-  def finalize(self):
-    if self.use_steps:
-      self.results = [np.concatenate(result, axis=0) for result in self.results]
-
-
 def _get_model_feed(model, mode):
   if mode == 'predict':
     feed = model._feed_inputs
@@ -153,13 +68,6 @@ def _print_train_info(inputs, val_inputs, steps_per_epoch, verbose):
           (inputs[0].shape[0], val_inputs[0].shape[0]))
 
 
-def _get_progbar(model, count_mode):
-  stateful_metric_names = None
-  if hasattr(model, 'metrics_names'):
-    stateful_metric_names = model.metrics_names[1:]  # Exclude `loss`
-  return cbks.ProgbarLogger(count_mode, stateful_metrics=stateful_metric_names)
-
-
 def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch):
   """Returns total number of samples (when training in batch mode) or steps."""
   if steps_per_epoch:
@@ -168,18 +76,6 @@ def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch):
                                           'steps_per_epoch')
 
 
-def _make_logs(model, outputs, mode, prefix=''):
-  """Used to make logs to send to `on_batch_end` methods."""
-  logs = {}
-  # TODO(omalleyt): handle outputs in prediction when Callback
-  # hooks are ready.
-  if mode in ['train', 'test']:
-    if hasattr(model, 'metrics_names'):
-      for label, output in zip(model.metrics_names, outputs):
-        logs[prefix + label] = output
-  return logs
-
-
 def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
   """Prepare feed values to the model execution function.
 
@@ -219,11 +115,11 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
   return ins
 
 
-def _get_execution_function(model, mode):
-  """Get function to run one step of model execution."""
+def _make_execution_function(model, mode):
+  """Makes function to run one step of model execution."""
   if model._distribution_strategy:
-    return training_distributed._get_execution_function(model, mode)
-  return model._get_execution_function(mode)
+    return training_distributed._make_execution_function(model, mode)
+  return model._make_execution_function(mode)
 
 
 def model_iteration(model,
@@ -292,7 +188,7 @@ def model_iteration(model,
     scope.__enter__()
 
   # Get step function and loop type.
-  f = _get_execution_function(model, mode)
+  f = _make_execution_function(model, mode)
   use_steps = steps_per_epoch is not None
   do_validation = val_inputs is not None
 
@@ -307,19 +203,14 @@ def model_iteration(model,
       callbacks,
       model,
       do_validation=do_validation,
-      val_inputs=val_inputs,
-      val_targets=val_targets,
-      val_sample_weights=val_sample_weights,
       batch_size=batch_size,
       epochs=epochs,
       steps_per_epoch=steps_per_epoch,
       samples=num_samples_or_steps,
-      validation_steps=validation_steps,
       verbose=0,  # Handle ProgBarLogger separately in this loop.
-      count_mode=count_mode,
       mode=mode)
   # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready.
-  progbar = _get_progbar(model, count_mode)
+  progbar = training_utils.get_progbar(model, count_mode)
   progbar.params = callbacks.params
   progbar.params['verbose'] = verbose
 
@@ -333,9 +224,11 @@ def model_iteration(model,
 
   # Select aggregation method.
   if mode == 'predict':
-    aggregator = OutputsAggregator(use_steps, num_samples_or_steps)
+    aggregator = training_utils.OutputsAggregator(use_steps,
+                                                  num_samples_or_steps)
   else:
-    aggregator = MetricsAggregator(use_steps, num_samples_or_steps)
+    aggregator = training_utils.MetricsAggregator(use_steps,
+                                                  num_samples_or_steps)
 
   if model._distribution_strategy:
     training_distributed._copy_weights_to_distributed_model(model)
@@ -348,7 +241,6 @@ def model_iteration(model,
       break
 
     # Setup work for each epoch
-    results = []
     epoch_logs = {}
     if hasattr(model, 'metrics'):
       for m in model.metrics:
@@ -389,7 +281,7 @@ def model_iteration(model,
         aggregator.aggregate(batch_outs)
 
         # Callbacks batch end.
-        batch_logs.update(_make_logs(model, batch_outs, mode))
+        batch_logs.update(training_utils.make_logs(model, batch_outs, mode))
         callbacks._call_batch_hook(mode, 'end', step, batch_logs)
         progbar.on_batch_end(step, batch_logs)
 
@@ -440,7 +332,7 @@ def model_iteration(model,
         aggregator.aggregate(batch_outs, batch_start, batch_end)
 
         # Callbacks batch end.
-        batch_logs.update(_make_logs(model, batch_outs, mode))
+        batch_logs.update(training_utils.make_logs(model, batch_outs, mode))
         callbacks._call_batch_hook(mode, 'end', batch_index, batch_logs)
         progbar.on_batch_end(batch_index, batch_logs)
 
@@ -449,7 +341,7 @@ def model_iteration(model,
 
     aggregator.finalize()
     results = aggregator.results
-    epoch_logs.update(_make_logs(model, results, mode))
+    epoch_logs.update(training_utils.make_logs(model, results, mode))
     if len(results) == 1:
       results = results[0]
 
@@ -467,7 +359,8 @@ def model_iteration(model,
           mode='test')
       if not isinstance(val_results, list):
         val_results = [val_results]
-      epoch_logs.update(_make_logs(model, val_results, mode, prefix='val_'))
+      epoch_logs.update(
+          training_utils.make_logs(model, val_results, mode, prefix='val_'))
 
     callbacks.on_epoch_end(epoch, epoch_logs, mode=mode)
     progbar.on_epoch_end(epoch, epoch_logs)
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index b0a2dfe00d..473f06ded7 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -172,8 +172,6 @@ def experimental_fit_loop(model,
       callbacks,
       model,
       do_validation=do_validation,
-      val_inputs=None,
-      val_targets=None,
       epochs=epochs,
       steps_per_epoch=steps_per_epoch,
       verbose=verbose)
@@ -582,10 +580,10 @@ def _get_input_from_iterator(iterator, model):
   return x, y, sample_weights
 
 
-def _get_execution_function(model, mode):
-  """Get function to run one step of distributed model execution."""
+def _make_execution_function(model, mode):
+  """Makes function to run one step of distributed model execution."""
   if context.executing_eagerly():
-    return _get_eager_execution_function(model, mode)
+    return _make_eager_execution_function(model, mode)
 
   strategy = model._distribution_strategy
   if not model._grouped_model:
@@ -593,7 +591,7 @@ def _get_execution_function(model, mode):
         model, strategy, make_callback_model=(mode == 'train'))
 
   def _per_device_function(model):
-    f = model._get_execution_function(mode)
+    f = model._make_execution_function(mode)
     return (f.inputs, f.outputs, f.updates_op, f.session_kwargs)
 
   with strategy.scope():
@@ -631,15 +629,15 @@ def _get_execution_function(model, mode):
         **all_session_args)
 
 
-def _get_eager_execution_function(model, mode):
-  """Get function to run one step of distributed model eager execution."""
+def _make_eager_execution_function(model, mode):
+  """Makes function to run one step of distributed model eager execution."""
   strategy = model._distribution_strategy
   if not model._grouped_model:
     clone_model_on_replicas(
         model, strategy, make_callback_model=(mode == 'train'))
 
   def _per_device_function(model):
-    f = model._get_execution_function(mode)
+    f = model._make_execution_function(mode)
     return (f.inputs, f.outputs)
 
   # NOTE(priyag): Try creating a new FuncGraph within DS scope instead of using
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index cd85c365db..1d1cec1c50 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -20,19 +20,12 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
-import copy
 
-import numpy as np
-
-from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager.backprop import GradientTape
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend
-from tensorflow.python.keras import callbacks as cbks
 from tensorflow.python.keras import losses as losses_module
-from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
@@ -181,361 +174,6 @@ def _model_loss(model,
   return outs, total_loss, loss_metrics, aggregated_loss_metrics, masks
 
 
-def iterator_fit_loop(model,
-                      inputs,
-                      class_weight,
-                      steps_per_epoch,
-                      epoch_logs,
-                      val_inputs=None,
-                      val_targets=None,
-                      val_sample_weights=None,
-                      epochs=1,
-                      verbose=1,
-                      callbacks=None,
-                      validation_steps=None,
-                      do_validation=False,
-                      batch_size=None,
-                      output_loss_metrics=None):
-  """Fit function for eager execution when input is given as dataset iterator.
-
-  Updates the given epoch logs.
-
-  Arguments:
-      model: Instance of the `Model`.
-      inputs: Input dataset iterator.
-      class_weight: Optional class-weight array to weight the importance of
-          samples in `inputs` based on the class they belong to, as conveyed by
-          the targets from the `inputs` iterator.
-      steps_per_epoch: Total number of steps (batches of samples)
-          before declaring one epoch finished and starting the
-          next epoch.
-      epoch_logs: Dictionary of logs from every epoch.
-      val_inputs: Input data for validation.
-      val_targets: Target data for validation.
-      val_sample_weights: Sample weight data for validation.
-      epochs: Number of times to iterate over the data
-      verbose: Verbosity mode, 0, 1 or 2
-      callbacks: CallbackList instance. Controls callbacks during training.
-      validation_steps: Number of steps to run validation for (only if doing
-        validation from data tensors). Ignored with default value of `None`.
-      do_validation: Boolean value indicating whether we should do validation.
-      batch_size: int, val_inputs and val_targets will be evaled batch by
-        batch with size batch_size if they are array.
-      output_loss_metrics: List of metrics that are used to aggregated output
-        loss values.
-
-  Raises:
-      ValueError: In case of mismatch between given number of inputs and
-        expectations of the model.
-  """
-  assert isinstance(inputs, iterator_ops.EagerIterator)
-
-  # make sure either x,y or x,y,sample_weights is provided
-  if (not isinstance(inputs.output_shapes, collections.Sequence) or
-      len(inputs.output_shapes) not in (2, 3)):
-    raise ValueError('Please provide either inputs and targets '
-                     'or inputs, targets, and sample_weights')
-
-  for step_index in range(steps_per_epoch):
-    batch_logs = {'batch': step_index, 'size': 1}
-    callbacks.on_batch_begin(step_index, batch_logs)
-
-    # Get data from the iterator.
-    try:
-      next_element = inputs.get_next()
-    except errors.OutOfRangeError:
-      logging.warning(
-          'Your dataset iterator ran out of data; interrupting training. Make '
-          'sure that your dataset can generate at least '
-          '`steps_per_epoch * epochs` batches (in this case, %d batches). You '
-          'may need to use the repeat() function when building your '
-          'dataset.' % steps_per_epoch * epochs)
-      break
-
-    if len(inputs.output_shapes) == 2:
-      x, y = next_element
-      sample_weights = None
-    else:
-      x, y, sample_weights = next_element
-
-    # Validate and standardize data.
-    x, y, sample_weights = model._standardize_user_data(
-        x, y, sample_weight=sample_weights, class_weight=class_weight)
-    x = training_utils.cast_if_floating_dtype(x)
-    y = training_utils.cast_if_floating_dtype(y)
-    if sample_weights:
-      sample_weights = [
-          training_utils.cast_if_floating_dtype(
-              ops.convert_to_tensor(val, dtype=backend.floatx()))
-          if val is not None else None for val in sample_weights
-      ]
-
-    # Train model.
-    outs, loss, _, aggregated_loss_metrics, masks = _process_single_batch(
-        model,
-        x,
-        y,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=True)
-    outs = generic_utils.to_list(outs)
-
-    if step_index == 0:
-      # Set stateful_metrics in callbacks. We do not do this before the
-      # `steps_per_epoch` loop because model will be compiled only in the first
-      # iteration of this loop in the deferred build scenario.
-      for cbk in callbacks:
-        if (isinstance(cbk, cbks.BaseLogger) or
-            isinstance(cbk, cbks.ProgbarLogger)):
-          cbk.stateful_metrics = model.metrics_names[1:]  # Exclude `loss`
-
-      callback_metrics = copy.copy(model.metrics_names)
-      if do_validation:
-        callback_metrics += ['val_' + n for n in model.metrics_names]
-      callbacks.set_params({
-          'batch_size': batch_size,
-          'epochs': epochs,
-          'steps': steps_per_epoch,
-          'verbose': verbose,
-          'do_validation': do_validation,
-          'metrics': callback_metrics or [],
-          'validation_steps': validation_steps
-      })
-
-    # Calculate metrics.
-    for l, o in zip(model.metrics_names, outs):
-      batch_logs[l] = o
-    metrics_results = _eager_metrics_fn(
-        model, outs, y, sample_weights=sample_weights, masks=masks)
-    batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss))
-
-    for k, v in zip(
-        model.metrics_names,
-        [backend.mean(loss)] + aggregated_loss_metrics + metrics_results):
-      batch_logs[k] = tensor_util.constant_value(v)
-    callbacks.on_batch_end(step_index, batch_logs)
-    if callbacks.model.stop_training:
-      break
-
-    if step_index == steps_per_epoch - 1:
-      if do_validation:
-        val_outs = test_loop(
-            model,
-            val_inputs,
-            val_targets,
-            sample_weights=val_sample_weights,
-            steps=validation_steps,
-            verbose=0,
-            batch_size=batch_size)
-        if not isinstance(val_outs, list):
-          val_outs = [val_outs]
-        # Same labels assumed.
-        for l, o in zip(model.metrics_names, val_outs):
-          epoch_logs['val_' + l] = o
-
-
-def iterator_test_loop(model, inputs, steps, verbose=0):
-  """Test function for eager execution when input is given as dataset iterator.
-
-  Arguments:
-      model: Model instance that is being evaluated in Eager mode.
-      inputs: Input dataset iterator.
-      steps: Total number of steps (batches of samples) before declaring
-      predictions finished.
-      verbose: Verbosity mode.
-
-  Returns:
-      Scalar loss (if the model has a single output and no metrics)
-      or list of scalars (if the model has multiple outputs
-      and/or metrics). The attribute `model.metrics_names` will give you
-      the display labels for the scalar outputs.
-
-  Raises:
-      ValueError: In case of mismatch between given number of inputs and
-        expectations of the model.
-  """
-  assert isinstance(inputs, iterator_ops.EagerIterator)
-  # make sure either x,y or x,y,sample_weights is provided
-  if (not isinstance(inputs.output_shapes, collections.Sequence) or
-      len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3):
-    raise ValueError('Please provide either inputs and targets'
-                     'or inputs, targets, and sample_weights')
-  outs = []
-
-  # Create metric wrapper for the losses.
-  output_loss_metrics = []
-  for i in range(len(model.outputs)):
-    loss_fn = model.loss_functions[i]
-    loss_name = loss_fn.name if isinstance(
-        loss_fn, losses_module.Loss) else loss_fn.__name__
-    mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-        loss_fn, name=loss_name)
-    output_loss_metrics.append(mean_wrapped_loss)
-
-  num_samples = 0
-  if verbose == 1:
-    progbar = generic_utils.Progbar(target=steps)
-  for step_index in range(steps):
-    # Get data from the iterator.
-    try:
-      next_element = inputs.get_next()
-    except errors.OutOfRangeError:
-      logging.warning(
-          'Your dataset iterator ran out of data interrupting testing. '
-          'Make sure that your dataset can generate at least `steps` batches '
-          '(in this case, %d batches). You may need to use the repeat() '
-          'function when building your dataset.', steps)
-      break
-
-    if len(inputs.output_shapes) == 2:
-      x, y = next_element
-      sample_weights = None
-    else:
-      x, y, sample_weights = next_element
-
-    # Validate and standardize data.
-    x, y, sample_weights = model._standardize_user_data(
-        x, y, sample_weight=sample_weights)
-    x = training_utils.cast_if_floating_dtype(x)
-    y = training_utils.cast_if_floating_dtype(y)
-    if sample_weights:
-      sample_weights = [
-          training_utils.cast_if_floating_dtype(
-              ops.convert_to_tensor(val, dtype=backend.floatx()))
-          if val is not None else None for val in sample_weights
-      ]
-
-    if step_index == 0:
-      # Get stateful metrics indices. We do not do this before the `steps` loop
-      # because model will be compiled only in the first iteration of this loop
-      # in the deferred build scenario.
-      if hasattr(model, '_compile_metrics'):
-        for m in model.metrics:
-          m.reset_states()
-      for m in output_loss_metrics:
-        m.reset_states()
-
-    # Calculate model output, loss values.
-    loss_outs, loss, _, aggregated_loss_metrics, masks = _model_loss(
-        model,
-        x,
-        y,
-        output_loss_metrics=output_loss_metrics,
-        sample_weights=sample_weights,
-        training=False)
-    metrics_results = _eager_metrics_fn(
-        model, loss_outs, y, sample_weights=sample_weights, masks=masks)
-    batch_outs = []
-    for _, v in zip(
-        model.metrics_names,
-        [backend.mean(loss)] + aggregated_loss_metrics + metrics_results):
-      batch_outs.append(tensor_util.constant_value(v))
-
-    # Get current step size.
-    if isinstance(x, list):
-      step_size = x[0].get_shape().as_list()[0]
-    elif isinstance(x, dict):
-      step_size = list(x.values())[0].get_shape().as_list()[0]
-    else:
-      step_size = x.get_shape().as_list()[0]
-
-    # Accumulate results in output array.
-    if not isinstance(batch_outs, list):
-      batch_outs = [batch_outs]
-    if step_index == 0:
-      for _ in enumerate(batch_outs):
-        outs.append(0.)
-    outs[0] += batch_outs[0] * step_size  # index 0 = 'loss'
-    outs[1:] = batch_outs[1:]
-
-    # Calculate sample size.
-    num_samples += step_size
-    if verbose == 1:
-      progbar.update(step_index + 1)
-
-  outs[0] /= num_samples  # index 0 = 'loss'
-  if len(outs) == 1:
-    return outs[0]
-  return outs
-
-
-def iterator_predict_loop(model, inputs, steps, verbose=0):
-  """Predict function for eager execution when input is dataset iterator.
-
-  Arguments:
-      model: Instance of `Model`.
-      inputs: Input dataset iterator.
-      steps: Total number of steps (batches of samples) before declaring
-          `_predict_loop` finished.
-      verbose: Verbosity mode.
-
-  Returns:
-      Array of predictions (if the model has a single output)
-      or list of arrays of predictions (if the model has multiple outputs).
-
-  Raises:
-      ValueError: In case of mismatch between given number of inputs and
-        expectations of the model.
-  """
-  assert isinstance(inputs, iterator_ops.EagerIterator)
-  if not isinstance(inputs.output_shapes,
-                    collections.Sequence) or len(inputs.output_shapes) > 3:
-    raise ValueError(
-        'Please provide data as a list or tuple of 1, 2, or 3 elements '
-        ' - `(input)`, or `(input, target)`, or `(input, target,'
-        'sample_weights)`. Received %s. We do not use the `target` or'
-        '`sample_weights` value here.' % inputs.output_shapes)
-  outs = []
-  if verbose == 1:
-    progbar = generic_utils.Progbar(target=steps)
-
-  for step_index in range(steps):
-    # Get data from the iterator.
-    try:
-      next_element = inputs.get_next()
-    except errors.OutOfRangeError:
-      logging.warning(
-          'Your dataset iterator ran out of data; interrupting prediction. '
-          'Make sure that your dataset can generate at least `steps` batches '
-          '(in this case, %d batches). You may need to use the repeat() '
-          'function when building your dataset.', steps)
-      break
-
-    # expects a tuple, where first element of tuple represents inputs
-    x = next_element[0]
-
-    # Validate and standardize data.
-    x, _, _ = model._standardize_user_data(x)
-    x = training_utils.cast_if_floating_dtype(x)
-
-    if isinstance(x, list) and len(x) == 1:
-      x = x[0]
-
-    if model._expects_training_arg:
-      batch_outs = model.call(x, training=False)
-    else:
-      batch_outs = model.call(x)
-    if not isinstance(batch_outs, list):
-      batch_outs = [batch_outs]
-
-    # We collect the results from every step and then concatenate them once
-    # in the end. This is an expensive process. We are doing this because we
-    # do not know the number of samples beforehand.
-    if step_index == 0:
-      for _ in batch_outs:
-        outs.append([])
-    for i, batch_out in enumerate(batch_outs):
-      outs[i].append(backend.get_value(batch_out))
-
-    if verbose == 1:
-      progbar.update(step_index + 1)
-  for i, out in enumerate(outs):
-    outs[i] = np.concatenate(tuple(out), axis=0)
-  if len(outs) == 1:
-    return outs[0]
-  return outs
-
-
 def _process_single_batch(model,
                           inputs,
                           targets,
@@ -628,7 +266,7 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
       targets,
       sample_weights=sample_weights,
       masks=masks,
-      return_stateful_result=False)
+      return_stateful_result=True)
   loss = generic_utils.to_list(loss)
 
   return [
@@ -675,186 +313,10 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
       targets,
       sample_weights=sample_weights,
       masks=masks,
-      return_stateful_result=False)
+      return_stateful_result=True)
   loss = generic_utils.to_list(loss)
 
   return [
       tensor_util.constant_value(v)
       for v in loss + loss_metrics + metrics_results
   ]
-
-
-def fit_loop(model,
-             inputs,
-             targets,
-             sample_weights=None,
-             class_weight=None,
-             val_inputs=None,
-             val_targets=None,
-             val_sample_weights=None,
-             batch_size=None,
-             epochs=1,
-             verbose=1,
-             callbacks=None,
-             shuffle=True,
-             initial_epoch=0,
-             steps_per_epoch=None,
-             validation_steps=None):
-  """Fit function for eager execution.
-
-  Arguments:
-      model: Instance of the model that is being executed in Eager mode.
-      inputs: List of input arrays.
-      targets: List of target arrays.
-      sample_weights: Optional list of sample weight arrays.
-      class_weight: Optional class-weight array to weight the importance of
-          samples in `inputs` based on the class they belong to, as conveyed by
-          `targets`.
-      val_inputs: Input data for validation.
-      val_targets: Target data for validation.
-      val_sample_weights: Sample weight data for validation.
-      batch_size: Integer batch size or None if unknown.
-      epochs: Number of times to iterate over the data
-      verbose: Verbosity mode, 0, 1 or 2
-      callbacks: List of callbacks to be called during training
-      shuffle: Whether to shuffle the data at the beginning of each epoch
-      initial_epoch: Epoch at which to start training
-          (useful for resuming a previous training run)
-      steps_per_epoch: Total number of steps (batches of samples)
-          before declaring one epoch finished and starting the
-          next epoch. Ignored with the default value of `None`.
-      validation_steps: Number of steps to run validation for (only if doing
-        validation from data tensors). Ignored with default value of `None`.
-
-  Returns:
-      `History` object.
-
-  Raises:
-    ValueError: In case of invalid argument values.
-  """
-  # Convert training inputs to an EagerIterator
-  inputs, steps_per_epoch = training_utils.convert_to_iterator(
-      x=inputs,
-      y=targets,
-      sample_weights=sample_weights,
-      batch_size=batch_size,
-      steps_per_epoch=steps_per_epoch,
-      epochs=epochs,
-      shuffle=shuffle)
-  # Required for eager execution
-  with backend.learning_phase_scope(1):
-    do_validation = val_inputs is not None
-    callbacks = cbks.configure_callbacks(
-        callbacks,
-        model,
-        do_validation=do_validation,
-        batch_size=batch_size,
-        epochs=epochs,
-        steps_per_epoch=steps_per_epoch,
-        val_inputs=val_inputs,
-        val_targets=val_targets,
-        val_sample_weights=val_sample_weights,
-        validation_steps=validation_steps,
-        verbose=verbose)
-
-    # Create metric wrapper for the losses.
-    output_loss_metrics = []
-    for i in range(len(model.outputs)):
-      loss_fn = model.loss_functions[i]
-      loss_name = loss_fn.name if isinstance(
-          loss_fn, losses_module.Loss) else loss_fn.__name__
-      mean_wrapped_loss = metrics_module.MeanMetricWrapper(
-          loss_fn, name=loss_name)
-      output_loss_metrics.append(mean_wrapped_loss)
-
-    callbacks.on_train_begin()
-    for epoch in range(initial_epoch, epochs):
-      if model._is_compiled:  # Model may not be compiled the first time.
-        # Reset stateful metrics
-        for m in model.metrics:
-          m.reset_states()
-
-      for m in output_loss_metrics:
-        m.reset_states()
-
-      callbacks.on_epoch_begin(epoch)
-      epoch_logs = {}
-      iterator_fit_loop(
-          model,
-          inputs,
-          class_weight,
-          steps_per_epoch=steps_per_epoch,
-          epoch_logs=epoch_logs,
-          val_inputs=val_inputs,
-          val_targets=val_targets,
-          val_sample_weights=val_sample_weights,
-          epochs=epochs,
-          verbose=verbose,
-          callbacks=callbacks,
-          validation_steps=validation_steps,
-          do_validation=do_validation,
-          batch_size=batch_size,
-          output_loss_metrics=output_loss_metrics)
-      callbacks.on_epoch_end(epoch, epoch_logs)
-      if callbacks.model.stop_training:
-        break
-  callbacks.on_train_end()
-  return model.history
-
-
-def test_loop(model, inputs, targets,
-              sample_weights=None,
-              batch_size=None,
-              verbose=0,
-              steps=None):
-  """Test function for eager execution.
-
-  Arguments:
-      model: Model instance that is being evaluated in Eager mode.
-      inputs: List of input arrays.
-      targets: List of target arrays.
-      sample_weights: Optional list of sample weight arrays.
-      batch_size: integer batch size or `None`.
-      verbose: verbosity mode.
-      steps: Total number of steps (batches of samples)
-          before declaring predictions finished.
-          Ignored with the default value of `None`.
-
-  Returns:
-      Scalar loss (if the model has a single output and no metrics)
-      or list of scalars (if the model has multiple outputs
-      and/or metrics). The attribute `model.metrics_names` will give you
-      the display labels for the scalar outputs.
-  """
-  inputs, steps = training_utils.convert_to_iterator(
-      x=inputs,
-      y=targets,
-      sample_weights=sample_weights,
-      batch_size=batch_size,
-      steps_per_epoch=steps,
-      is_validation=True)
-  with backend.learning_phase_scope(0):
-    return iterator_test_loop(model, inputs, steps, verbose=verbose)
-
-
-def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None):
-  """Predict function for eager execution.
-
-  Arguments:
-      model: Instance of `Model`.
-      inputs: List of input arrays.
-      batch_size: integer batch size.
-      verbose: verbosity mode.
-      steps: Total number of steps (batches of samples)
-          before declaring `_predict_loop` finished.
-          Ignored with the default value of `None`.
-
-  Returns:
-      Array of predictions (if the model has a single output)
-      or list of arrays of predictions
-      (if the model has multiple outputs).
-  """
-  with backend.learning_phase_scope(0):
-    inputs, steps = training_utils.convert_to_iterator(
-        x=inputs, batch_size=batch_size, steps_per_epoch=steps)
-    return iterator_predict_loop(model, inputs, steps, verbose=verbose)
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index c1a9154ff5..3fabbb17ed 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -152,12 +152,12 @@ class TrainingTest(test.TestCase):
           ValueError, r'provide either `batch_size` or `validation_steps`'):
         model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                   validation_data=(x, y))
-    with self.assertRaisesRegexp(
-        ValueError, r'specify the number of steps'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'specify the `validation_steps` argument.'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=validation_dataset)
-    with self.assertRaisesRegexp(
-        ValueError, r'specify the number of steps'):
+    with self.assertRaisesRegexp(ValueError,
+                                 'specify the `validation_steps` argument.'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                 validation_data=validation_iterator)
 
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 7b2bfcfd86..0abf0b8270 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -19,421 +19,433 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+import math
+
 import numpy as np
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
+from tensorflow.python.keras import backend
 from tensorflow.python.keras import callbacks as cbks
+from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import data_utils
-from tensorflow.python.keras.utils.generic_utils import Progbar
+from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import nest
+
+
+def model_iteration(model,
+                    data,
+                    steps_per_epoch=None,
+                    epochs=1,
+                    verbose=1,
+                    callbacks=None,
+                    validation_data=None,
+                    validation_steps=None,
+                    class_weight=None,
+                    max_queue_size=10,
+                    workers=1,
+                    use_multiprocessing=False,
+                    shuffle=True,
+                    initial_epoch=0,
+                    mode='train',
+                    batch_size=None,
+                    **kwargs):
+  """Loop function for arrays of data with modes 'train'/'test'/'predict'.
+
+  Arguments:
+      model: Keras Model instance.
+      data: Either a tuple of NumPy/Tensor inputs (i.e. `(x,)` or `(x, y)` or
+        `(x, y, sample_weights)`) or a generator or
+        `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset.
+      steps_per_epoch: Total number of steps (batches of samples) before
+        declaring one epoch finished and starting the next epoch. Ignored with
+        the default value of `None`.
+      epochs: Number of times to iterate over the data.
+      verbose: Verbosity mode, 0, 1 or 2.
+      callbacks: List of callbacks to be called during training.
+      validation_data: Either a tuple of NumPy/Tensor inputs (i.e. `(x,)` or
+        `(x, y)` or `(x, y, sample_weights)`) or a generator or
+        `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset.
+      validation_steps: Total number of steps (batches of samples) before
+        declaring validation finished.
+      class_weight: Dictionary mapping class indices to a weight for the class.
+      max_queue_size: Integer. Maximum size for the generator queue. If
+        unspecified, `max_queue_size` will default to 10.
+      workers: Integer. Maximum number of processes to spin up when using
+        process-based threading. If unspecified, `workers` will default to 1. If
+        0, will execute the generator on the main thread.
+      use_multiprocessing: Boolean. If `True`, use process-based threading. If
+        unspecified, `use_multiprocessing` will default to `False`. Note that
+        because this implementation relies on multiprocessing, you should not
+        pass non-picklable arguments to the generator as they can't be passed
+        easily to children processes.
+      shuffle: Boolean. Whether to shuffle the order of the batches at the
+        beginning of each epoch. Only used with instances of `Sequence`
+        (`keras.utils.Sequence`). Has no effect when `steps_per_epoch` is not
+        `None`.
+      initial_epoch: Epoch at which to start training (useful for resuming a
+        previous training run).
+      mode: One of 'train'/'test'/'predict'.
+      batch_size: Integer batch size or None if unknown. Will only be used if
+        `data` is in NumPy/Tensor format.
+      **kwargs: Additional arguments for backwards compatibility. `steps` is
+        accepted as an alias for `steps_per_epoch`.
+
+  Returns:
+      - In 'train' mode: `History` object.
+      - In 'test' mode: Evaluation metrics.
+      - In 'predict' mode: Outputs of the Model called on inputs.
+
+  Raises:
+      ValueError: in case of invalid arguments.
+  """
+  if 'steps' in kwargs:
+    steps_per_epoch = kwargs['steps']
+
+  # Convert to a format that supports `next(generator)`.
+  generator, steps_per_epoch = convert_to_generator_like(
+      data,
+      steps_per_epoch=steps_per_epoch,
+      batch_size=batch_size,
+      epochs=epochs - initial_epoch,
+      shuffle=shuffle)
+
+  do_validation = validation_data is not None
+  should_set_learning_phase = context.executing_eagerly() and model.run_eagerly
+  is_sequence = isinstance(generator, data_utils.Sequence)
+  _validate_arguments(is_sequence, use_multiprocessing, workers,
+                      steps_per_epoch, validation_data, validation_steps, mode,
+                      kwargs)
+
+  batch_function = _make_execution_function(
+      model, mode, class_weight=class_weight)
+
+  # Create the queue for the generator.
+  output_generator, enqueuer = _make_enqueued_generator(
+      generator,
+      workers=workers,
+      use_multiprocessing=use_multiprocessing,
+      max_queue_size=max_queue_size,
+      shuffle=shuffle)
+
+  num_samples_or_steps, use_steps = _get_num_samples_or_steps(
+      data, steps_per_epoch)
+
+  count_mode = 'steps' if use_steps else 'samples'
+  callbacks = cbks.configure_callbacks(
+      callbacks,
+      model,
+      do_validation=do_validation,
+      epochs=epochs,
+      steps_per_epoch=steps_per_epoch,
+      batch_size=batch_size,
+      samples=num_samples_or_steps,
+      verbose=0,  # Handle ProgBar as part of Callbacks once hooks are ready.
+      mode=mode)
+  # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready.
+  progbar = training_utils.get_progbar(model, count_mode)
+  progbar.params = callbacks.params
+  progbar.params['verbose'] = verbose
+
+  if mode == 'predict':
+    aggregator = training_utils.OutputsAggregator(True, steps_per_epoch)
+  else:
+    aggregator = training_utils.MetricsAggregator(True, steps_per_epoch)
 
+  if should_set_learning_phase:
+    old_learning_phase = backend.learning_phase()
+    backend.set_learning_phase(1 if mode == 'train' else 0)
 
-def fit_generator(model,
-                  generator,
-                  steps_per_epoch=None,
-                  epochs=1,
-                  verbose=1,
-                  callbacks=None,
-                  validation_data=None,
-                  validation_steps=None,
-                  class_weight=None,
-                  max_queue_size=10,
-                  workers=1,
-                  use_multiprocessing=False,
-                  shuffle=True,
-                  initial_epoch=0):
-  """See docstring for `Model.fit_generator`."""
-  epoch = initial_epoch
-
-  do_validation = bool(validation_data)
-  if not context.executing_eagerly():
-    model._make_train_function()
-    if do_validation:
-      model._make_test_function()
+  callbacks.model.stop_training = False
+  callbacks._call_begin_hook(mode)
+  progbar.on_train_begin()
+  for epoch in range(initial_epoch, epochs):
+    if callbacks.model.stop_training:
+      break
 
-  is_sequence = isinstance(generator, data_utils.Sequence)
-  if not is_sequence and use_multiprocessing and workers > 1:
-    logging.warning(
-        UserWarning('Using a generator with `use_multiprocessing=True`'
-                    ' and multiple workers may duplicate your data.'
-                    ' Please consider using the`keras.utils.Sequence'
-                    ' class.'))
-  if steps_per_epoch is None:
-    if is_sequence:
-      steps_per_epoch = len(generator)
-    else:
-      raise ValueError('Please specify the `steps_per_epoch` argument.')
+    # Setup work for each epoch.
+    model.reset_metrics()
+    epoch_logs = {}
+    callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode)
+    progbar.on_epoch_begin(epoch, epoch_logs)
 
-  if (isinstance(validation_data, dataset_ops.DatasetV2) and
-      context.executing_eagerly()):
-    validation_data = iter(validation_data)
-  val_gen = (data_utils.is_generator_or_sequence(validation_data) or
-             isinstance(validation_data, iterator_ops.EagerIterator))
-  if (val_gen and not isinstance(validation_data, data_utils.Sequence) and
-      not validation_steps):
-    raise ValueError('Please specify the `validation_steps` argument.')
+    for step in range(steps_per_epoch):
+      batch_data = _get_next_batch(output_generator, mode)
+      if batch_data is None:
+        callbacks.model.stop_training = True
+        break
 
-  enqueuer = None
-  val_enqueuer = None
+      # `batch_size` used for validation data if validation
+      # data is NumPy/EagerTensors.
+      batch_size = int(nest.flatten(batch_data)[0].shape[0])
 
-  try:
-    val_x, val_y, val_sample_weights = validation_data, None, None
-    if do_validation and not val_gen:
-      # Prepare data for validation
-      if len(validation_data) == 2:
-        val_x, val_y = validation_data  # pylint: disable=unpacking-non-sequence
-        val_sample_weights = None
-      elif len(validation_data) == 3:
-        val_x, val_y, val_sample_weights = validation_data  # pylint: disable=unpacking-non-sequence
-      else:
-        raise ValueError(
-            '`validation_data` should be a tuple '
-            '`(val_x, val_y, val_sample_weight)` '
-            'or `(val_x, val_y)`. Found: ' + str(validation_data))
-      val_x, val_y, val_sample_weights = model._standardize_user_data(
-          val_x, val_y, val_sample_weights)
-
-    callbacks = cbks.configure_callbacks(
-        callbacks,
-        model,
-        do_validation=do_validation,
-        val_inputs=val_x,
-        val_targets=val_y,
-        val_sample_weights=val_sample_weights,
-        epochs=epochs,
-        validation_steps=validation_steps,
-        steps_per_epoch=steps_per_epoch,
-        verbose=verbose)
-
-    if workers > 0:
-      if is_sequence:
-        enqueuer = data_utils.OrderedEnqueuer(
-            generator,
-            use_multiprocessing=use_multiprocessing,
-            shuffle=shuffle)
-      else:
-        enqueuer = data_utils.GeneratorEnqueuer(
-            generator,
-            use_multiprocessing=use_multiprocessing)
-      enqueuer.start(workers=workers, max_queue_size=max_queue_size)
-      output_generator = enqueuer.get()
-    else:
-      if is_sequence:
-        output_generator = data_utils.iter_sequence_infinite(generator)
-      else:
-        output_generator = generator
+      # Callbacks batch begin.
+      batch_logs = {'batch': step, 'size': batch_size}
+      callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
+      progbar.on_batch_begin(step, batch_logs)
+
+      batch_outs = batch_function(*batch_data)
+      if not isinstance(batch_outs, list):
+        batch_outs = [batch_outs]
+
+      # Aggregate results.
+      if step == 0:
+        aggregator.create(batch_outs)
+      aggregator.aggregate(batch_outs)
+
+      # Callbacks batch end.
+      batch_logs.update(training_utils.make_logs(model, batch_outs, mode))
+      callbacks._call_batch_hook(mode, 'end', step, batch_logs)
+      progbar.on_batch_end(step, batch_logs)
 
-    callbacks.on_train_begin()
-    # Construct epoch logs.
-    epoch_logs = {}
-    while epoch < epochs:
-      for m in model.metrics:
-        m.reset_states()
-      callbacks.on_epoch_begin(epoch)
-      steps_done = 0
-      batch_index = 0
-      while steps_done < steps_per_epoch:
-        generator_output = next(output_generator)
-        if not hasattr(generator_output, '__len__'):
-          raise ValueError('Output of generator should be '
-                           'a tuple `(x, y, sample_weight)` '
-                           'or `(x, y)`. Found: ' + str(generator_output))
-
-        if len(generator_output) == 2:
-          x, y = generator_output
-          sample_weight = None
-        elif len(generator_output) == 3:
-          x, y, sample_weight = generator_output
-        else:
-          raise ValueError('Output of generator should be '
-                           'a tuple `(x, y, sample_weight)` '
-                           'or `(x, y)`. Found: ' + str(generator_output))
-        # build batch logs
-        batch_logs = {}
-        if isinstance(x, list):
-          batch_size = x[0].shape[0]
-        elif isinstance(x, dict):
-          batch_size = list(x.values())[0].shape[0]
-        else:
-          batch_size = x.shape[0]
-        batch_logs['batch'] = int(batch_index)
-        batch_logs['size'] = int(batch_size)
-        callbacks.on_batch_begin(batch_index, batch_logs)
-
-        outs = model.train_on_batch(
-            x, y, sample_weight=sample_weight, class_weight=class_weight)
-
-        if not isinstance(outs, list):
-          outs = [outs]
-        for l, o in zip(model.metrics_names, outs):
-          batch_logs[l] = o
-
-        callbacks.on_batch_end(batch_index, batch_logs)
-
-        batch_index += 1
-        steps_done += 1
-
-        # Epoch finished.
-        if steps_done >= steps_per_epoch and do_validation:
-          if val_gen:
-            val_outs = evaluate_generator(
-                model,
-                validation_data,
-                validation_steps,
-                workers=workers,
-                use_multiprocessing=use_multiprocessing,
-                max_queue_size=max_queue_size)
-          else:
-            # No need for try/except because
-            # data has already been validated.
-            val_outs = model.evaluate(
-                val_x,
-                val_y,
-                batch_size=batch_size,
-                sample_weight=val_sample_weights,
-                verbose=0)
-          if not isinstance(val_outs, list):
-            val_outs = [val_outs]
-          # Same labels assumed.
-          for l, o in zip(model.metrics_names, val_outs):
-            epoch_logs['val_' + l] = o
-
-        if callbacks.model.stop_training:
-          break
-
-      callbacks.on_epoch_end(epoch, epoch_logs)
-      epoch += 1
       if callbacks.model.stop_training:
         break
 
+    aggregator.finalize()
+    results = aggregator.results
+    epoch_logs.update(training_utils.make_logs(model, results, mode))
+    if len(results) == 1:
+      results = results[0]
+
+    # Run the test loop every epoch during training.
+    if do_validation and not callbacks.model.stop_training:
+      val_results = model_iteration(
+          model,
+          validation_data,
+          steps_per_epoch=validation_steps,
+          batch_size=batch_size,
+          class_weight=class_weight,
+          workers=workers,
+          use_multiprocessing=use_multiprocessing,
+          max_queue_size=max_queue_size,
+          mode='test')
+
+      if not isinstance(val_results, list):
+        val_results = [val_results]
+      epoch_logs.update(
+          training_utils.make_logs(model, val_results, mode, prefix='val_'))
+
+    callbacks.on_epoch_end(epoch, epoch_logs, mode=mode)
+    progbar.on_epoch_end(epoch, epoch_logs)
+  callbacks._call_end_hook(mode)
+
+  if enqueuer is not None:
+    enqueuer.stop()
+
+  if should_set_learning_phase:
+    backend.set_learning_phase(old_learning_phase)
+
+  if mode == 'train':
+    return model.history
+  return results
+
+
+# Maintain compatibility with the existing names.
+fit_generator = functools.partial(model_iteration, mode='train')
+evaluate_generator = functools.partial(model_iteration, mode='test')
+predict_generator = functools.partial(model_iteration, mode='predict')
+
+
+def _get_next_batch(output_generator, mode):
+  """Retrieves the next batch of input data."""
+  try:
+    generator_output = next(output_generator)
   except (errors.OutOfRangeError, StopIteration):
-    logging.warning(
-        'Your dataset iterator ran out of data interrupting testing. '
-        'Make sure that your dataset can generate at least `steps_per_epoch` '
-        'batches (in this case, %d batches). You may need to use the '
-        'repeat() function when building your dataset.', steps_per_epoch)
-
-  finally:
-    try:
-      if enqueuer is not None:
-        enqueuer.stop()
-    finally:
-      if val_enqueuer is not None:
-        val_enqueuer.stop()
-
-  callbacks.on_train_end()
-  return model.history
-
-
-def evaluate_generator(model,
-                       generator,
-                       steps=None,
-                       max_queue_size=10,
-                       workers=1,
-                       use_multiprocessing=False,
-                       verbose=0):
-  """See docstring for `Model.evaluate_generator`."""
-  if not context.executing_eagerly():
-    model._make_test_function()
-
-  if hasattr(model, '_compile_metrics'):
-    for m in model.metrics:
-      m.reset_states()
-
-  steps_done = 0
-  all_outs = []
-  batch_sizes = []
-  is_sequence = isinstance(generator, data_utils.Sequence)
+    # Returning `None` will trigger looping to stop.
+    logging.warning('Your dataset iterator ran out of data.')
+    return None
+  if not isinstance(generator_output, tuple):
+    if mode == 'predict':
+      # Always wrap in a tuple.
+      return (generator_output,)
+    else:
+      raise ValueError('Output of generator should be '
+                       'a tuple `(x, y, sample_weight)` '
+                       'or `(x, y)`. Found: ' + str(generator_output))
+
+  if len(generator_output) < 1 or len(generator_output) > 3:
+    raise ValueError('Output of generator should be '
+                     'a tuple `(x, y, sample_weight)` '
+                     'or `(x, y)` or (x,). Found: ' + str(generator_output))
+  return generator_output
+
+
+def _validate_arguments(is_sequence, use_multiprocessing, workers,
+                        steps_per_epoch, validation_data, validation_steps,
+                        mode, kwargs):
+  """Raises errors if arguments are invalid.
+
+  Arguments:
+    is_sequence: Boolean, whether data is a `keras.utils.data_utils.Sequence`
+      instance.
+    use_multiprocessing: Boolean. If `True`, use process-based threading. If
+      unspecified, `use_multiprocessing` will default to `False`. Note that
+      because this implementation relies on multiprocessing, you should not pass
+      non-picklable arguments to the generator as they can't be passed easily to
+      children processes.
+    workers: Integer. Maximum number of processes to spin up when using
+      process-based threading. If unspecified, `workers` will default to 1. If
+      0, will execute the generator on the main thread.
+    steps_per_epoch: Total number of steps (batches of samples) before declaring
+      one epoch finished and starting the next epoch. Ignored with the default
+      value of `None`.
+    validation_data: Either a tuple of NumPy/Tensor inputs (i.e. `(x,)` or `(x,
+      y)` or `(x, y, sample_weights)`) or a generator or
+      `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset.
+    validation_steps: Total number of steps (batches of samples) before
+      declaring validation finished.
+    mode: One of 'train'/'test'/'predict'.
+    kwargs: Additional arguments for backwards compatibility.
+
+  Raises:
+    ValueError: If `steps_per_epoch` or `validation_steps` are not passed
+      for data types that require them, or if unrecognized keyword
+      arguments are passed.
+  """
   if not is_sequence and use_multiprocessing and workers > 1:
     logging.warning(
         UserWarning('Using a generator with `use_multiprocessing=True`'
                     ' and multiple workers may duplicate your data.'
-                    ' Please consider using the`keras.utils.Sequence'
+                    ' Please consider using the `keras.utils.Sequence`'
                     ' class.'))
-  if steps is None:
+
+  if steps_per_epoch is None:
+    arg_name = 'steps_per_epoch' if mode == 'train' else 'steps'
+    raise ValueError('Please specify the number of steps via the '
+                     '`{}` argument.'.format(arg_name))
+
+  val_gen = (
+      data_utils.is_generator_or_sequence(validation_data) or
+      isinstance(validation_data, iterator_ops.EagerIterator) or
+      isinstance(validation_data, dataset_ops.DatasetV2))
+  if (val_gen and not isinstance(validation_data, data_utils.Sequence) and
+      not validation_steps):
+    raise ValueError('Please specify the `validation_steps` argument.')
+
+  if any(k != 'steps' for k in kwargs):
+    raise ValueError('Invalid arguments passed: {}'.format(
+        [k for k in kwargs if k != 'steps']))
+
+
+def convert_to_generator_like(data,
+                              batch_size=None,
+                              steps_per_epoch=None,
+                              epochs=1,
+                              shuffle=False):
+  """Make a generator out of NumPy or EagerTensor inputs.
+
+  Arguments:
+    data: Either a generator or `keras.utils.data_utils.Sequence` object or
+      `Dataset` or `EagerIterator` or a {1,2,3}-tuple of NumPy arrays or
+      EagerTensors. If a tuple, the elements represent `(x, y, sample_weights)`
+      and may be `None` or `[None]`.
+    batch_size: Used when creating a generator out of tuples of NumPy arrays or
+      EagerTensors.
+    steps_per_epoch: Steps of the generator to run each epoch.
+    epochs: Total number of epochs to run.
+    shuffle: Whether the data should be shuffled.
+
+  Returns:
+    - Generator or `keras.utils.data_utils.Sequence` or EagerIterator.
+
+  Raises:
+    - ValueError: If `batch_size` is not provided for NumPy or EagerTensor
+      inputs.
+  """
+  if isinstance(data, tuple):
+    # Scrub `Nones` that might have been passed for `targets`, `sample_weights`.
+    data = tuple(
+        ele for ele in data if not all(e is None for e in nest.flatten(ele)))
+    if len(data) == 1:
+      data = data[0]
+
+  if data_utils.is_generator_or_sequence(data) or isinstance(
+      data, iterator_ops.EagerIterator):
+    if isinstance(data, data_utils.Sequence):
+      steps_per_epoch = len(data)
+    return data, steps_per_epoch
+  if isinstance(data, dataset_ops.DatasetV2):
+    return dataset_ops.make_one_shot_iterator(data), steps_per_epoch
+
+  # Create generator from NumPy or EagerTensor Input.
+  num_samples = int(nest.flatten(data)[0].shape[0])
+  if batch_size is None:
+    raise ValueError('You must specify `batch_size`')
+  steps_per_epoch = int(math.ceil(num_samples / batch_size))
+
+  def _gen(data):
+    """Makes a generator out of a structure of NumPy/EagerTensors."""
+    index_array = np.arange(num_samples)
+    for _ in range(epochs):
+      if shuffle:
+        np.random.shuffle(index_array)
+      batches = generic_utils.make_batches(num_samples, batch_size)
+      for (batch_start, batch_end) in batches:
+        batch_ids = index_array[batch_start:batch_end]
+        flat_batch_data = training_utils.slice_arrays(
+            nest.flatten(data), batch_ids, contiguous=(not shuffle))
+        yield nest.pack_sequence_as(data, flat_batch_data)
+
+  return _gen(data), steps_per_epoch
+
+
+def _make_enqueued_generator(generator,
+                             workers=1,
+                             use_multiprocessing=False,
+                             max_queue_size=10,
+                             shuffle=False):
+  """Create a buffered queue of next elements of the generator."""
+  is_sequence = isinstance(generator, data_utils.Sequence)
+  enqueuer = None
+  if workers > 0:
     if is_sequence:
-      steps = len(generator)
+      enqueuer = data_utils.OrderedEnqueuer(
+          generator, use_multiprocessing=use_multiprocessing, shuffle=shuffle)
     else:
-      raise ValueError('Please specify the `steps` argument.')
-  enqueuer = None
-
-  try:
-    if workers > 0:
-      if is_sequence:
-        enqueuer = data_utils.OrderedEnqueuer(
-            generator, use_multiprocessing=use_multiprocessing)
-      else:
-        enqueuer = data_utils.GeneratorEnqueuer(
-            generator,
-            use_multiprocessing=use_multiprocessing)
-      enqueuer.start(workers=workers, max_queue_size=max_queue_size)
-      output_generator = enqueuer.get()
+      enqueuer = data_utils.GeneratorEnqueuer(
+          generator, use_multiprocessing=use_multiprocessing)
+    enqueuer.start(workers=workers, max_queue_size=max_queue_size)
+    output_generator = enqueuer.get()
+  else:
+    if is_sequence:
+      output_generator = data_utils.iter_sequence_infinite(generator)
     else:
-      if is_sequence:
-        output_generator = data_utils.iter_sequence_infinite(generator)
-      else:
-        output_generator = generator
-
-    if verbose == 1:
-      progbar = Progbar(target=steps)
-
-    while steps_done < steps:
-      generator_output = next(output_generator)
-      if not hasattr(generator_output, '__len__'):
-        raise ValueError('Output of generator should be a tuple '
-                         '(x, y, sample_weight) '
-                         'or (x, y). Found: ' + str(generator_output))
-      if len(generator_output) == 2:
-        x, y = generator_output
-        sample_weight = None
-      elif len(generator_output) == 3:
-        x, y, sample_weight = generator_output
-      else:
-        raise ValueError('Output of generator should be a tuple '
-                         '(x, y, sample_weight) '
-                         'or (x, y). Found: ' + str(generator_output))
-      outs = model.test_on_batch(x, y, sample_weight=sample_weight)
-
-      if isinstance(x, list):
-        batch_size = int(x[0].shape[0])
-      elif isinstance(x, dict):
-        batch_size = int(list(x.values())[0].shape[0])
-      else:
-        batch_size = int(x.shape[0])
-      if batch_size == 0:
-        raise ValueError('Received an empty batch. '
-                         'Batches should at least contain one item.')
-      all_outs.append(outs)
-
-      steps_done += 1
-      batch_sizes.append(batch_size)
-      if verbose == 1:
-        progbar.update(steps_done)
+      output_generator = generator
+  return output_generator, enqueuer
+
+
+def _make_execution_function(model, mode, class_weight=None):
+  """Makes function to run one step of model execution."""
+  if mode == 'train':
+    if not context.executing_eagerly():
+      model._make_fit_function()
+    f = functools.partial(model.train_on_batch, class_weight=class_weight)
+  elif mode == 'test':
+    if not context.executing_eagerly():
+      model._make_eval_function()
+    f = model.test_on_batch
+  else:
+    # Match signature of other modes to allow
+    # 1, 2, or 3-tuples from generator
+    def predict_on_batch(x, y=None, sample_weights=None):  # pylint: disable=unused-argument
+      return model.predict_on_batch(x)
 
-  except (errors.OutOfRangeError, StopIteration):
-    logging.warning(
-        'Your dataset iterator ran out of data interrupting testing. '
-        'Make sure that your dataset can generate at least `steps` '
-        'batches (in this case, %d batches). You may need to use the '
-        'repeat() function when building your dataset.', steps)
+    f = predict_on_batch
 
-  finally:
-    if enqueuer is not None:
-      enqueuer.stop()
+  # Maintain stateful metrics across batch-level calls.
+  if mode != 'predict':
+    f = functools.partial(f, reset_metrics=False)
 
-  if not isinstance(outs, list):
-    return np.average(np.asarray(all_outs), weights=batch_sizes)
-  else:
-    averages = [float(all_outs[-1][0])]  # index 0 = 'loss'
-    averages.extend([
-        np.average([out[i]
-                    for out in all_outs], weights=batch_sizes)
-        for i in range(1, len(outs))
-    ])
-    return averages
-
-
-def predict_generator(model,
-                      generator,
-                      steps=None,
-                      max_queue_size=10,
-                      workers=1,
-                      use_multiprocessing=False,
-                      verbose=0):
-  """See docstring for `Model.predict_generator`."""
-  if not context.executing_eagerly():
-    model._make_predict_function()
-
-  steps_done = 0
-  all_outs = []
-  is_sequence = isinstance(generator, data_utils.Sequence)
-  if not is_sequence and use_multiprocessing and workers > 1:
-    logging.warning(
-        UserWarning('Using a generator with `use_multiprocessing=True`'
-                    ' and multiple workers may duplicate your data.'
-                    ' Please consider using the`keras.utils.Sequence'
-                    ' class.'))
-  if steps is None:
-    if is_sequence:
-      steps = len(generator)
-    else:
-      raise ValueError('Please specify the `steps` argument.')
-  enqueuer = None
+  return f
 
-  try:
-    if workers > 0:
-      if is_sequence:
-        enqueuer = data_utils.OrderedEnqueuer(
-            generator, use_multiprocessing=use_multiprocessing)
-      else:
-        enqueuer = data_utils.GeneratorEnqueuer(
-            generator,
-            use_multiprocessing=use_multiprocessing)
-      enqueuer.start(workers=workers, max_queue_size=max_queue_size)
-      output_generator = enqueuer.get()
-    else:
-      if is_sequence:
-        output_generator = data_utils.iter_sequence_infinite(generator)
-      else:
-        output_generator = generator
-
-    if verbose == 1:
-      progbar = Progbar(target=steps)
-
-    while steps_done < steps:
-      generator_output = next(output_generator)
-      if isinstance(generator_output, tuple):
-        # Compatibility with the generators
-        # used for training.
-        if len(generator_output) == 1:
-          x = generator_output[0]
-        elif len(generator_output) == 2:
-          x, _ = generator_output
-        elif len(generator_output) == 3:
-          x, _, _ = generator_output
-        else:
-          raise ValueError('Output of generator should be '
-                           'a tuple `(x, y, sample_weight)` '
-                           'or `(x, y)`. Found: ' + str(generator_output))
-      else:
-        # Assumes a generator that only
-        # yields inputs (not targets and sample weights).
-        x = generator_output
-
-      outs = model.predict_on_batch(x)
-      if not isinstance(outs, list):
-        outs = [outs]
-
-      if not all_outs:
-        for out in outs:
-          all_outs.append([])
-
-      for i, out in enumerate(outs):
-        all_outs[i].append(out)
-      steps_done += 1
-      if verbose == 1:
-        progbar.update(steps_done)
 
-  except (errors.OutOfRangeError, StopIteration):
-    logging.warning(
-        'Your dataset iterator ran out of data interrupting testing. '
-        'Make sure that your dataset can generate at least `steps` '
-        'batches (in this case, %d batches). You may need to use the '
-        'repeat() function when building your dataset.', steps)
-
-  finally:
-    if enqueuer is not None:
-      enqueuer.stop()
-
-  if len(all_outs) == 1:
-    if steps_done == 1:
-      return all_outs[0][0]
-    else:
-      return np.concatenate(all_outs[0])
-  if steps_done == 1:
-    return [out[0] for out in all_outs]
-  else:
-    return [np.concatenate(out) for out in all_outs]
+def _get_num_samples_or_steps(data, steps_per_epoch):
+  """Returns number of samples or steps, and whether to use steps count mode."""
+  flat_inputs = nest.flatten(data)
+  if hasattr(flat_inputs[0], 'shape'):
+    return int(flat_inputs[0].shape[0]), False
+  return steps_per_epoch, True
diff --git a/tensorflow/python/keras/engine/training_generator_test.py b/tensorflow/python/keras/engine/training_generator_test.py
index 42cfa3bc70..8941428e43 100644
--- a/tensorflow/python/keras/engine/training_generator_test.py
+++ b/tensorflow/python/keras/engine/training_generator_test.py
@@ -25,11 +25,16 @@ from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.engine import training_generator
 from tensorflow.python.platform import test
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
+from tensorflow.python.util import nest
 
 
 def custom_generator(mode=2):
@@ -329,5 +334,56 @@ class TestGeneratorMethodsWithSequences(test.TestCase):
       model.fit(CustomSequence(), sample_weight=np.ones([10, 1]))
 
 
+@tf_test_util.run_all_in_graph_and_eager_modes
+class TestConvertToGeneratorLike(test.TestCase, parameterized.TestCase):
+  simple_inputs = (np.ones((10, 10)), np.ones((10, 1)))
+  nested_inputs = ((np.ones((10, 10)), np.ones((10, 20))), (np.ones((10, 1)),
+                                                            np.ones((10, 3))))
+
+  def _make_dataset(self, inputs, batches):
+    return dataset_ops.DatasetV2.from_tensors(inputs).repeat(batches)
+
+  def _make_iterator(self, inputs, batches):
+    return dataset_ops.make_one_shot_iterator(
+        self._make_dataset(inputs, batches))
+
+  def _make_generator(self, inputs, batches):
+
+    def _gen():
+      for _ in range(batches):
+        yield inputs
+
+    return _gen()
+
+  def _make_numpy(self, inputs, _):
+    return inputs
+
+  @parameterized.named_parameters(
+      ('simple_dataset', _make_dataset, simple_inputs),
+      ('simple_iterator', _make_iterator, simple_inputs),
+      ('simple_generator', _make_generator, simple_inputs),
+      ('simple_numpy', _make_numpy, simple_inputs),
+      ('nested_dataset', _make_dataset, nested_inputs),
+      ('nested_iterator', _make_iterator, nested_inputs),
+      ('nested_generator', _make_generator, nested_inputs),
+      ('nested_numpy', _make_numpy, nested_inputs))
+  def test_convert_to_generator_like(self, input_fn, inputs):
+    expected_batches = 5
+    data = input_fn(self, inputs, expected_batches)
+
+    # Dataset and Iterator not supported in Legacy Graph mode.
+    if (not context.executing_eagerly() and
+        isinstance(data, (dataset_ops.DatasetV2, iterator_ops.Iterator))):
+      return
+
+    generator, steps = training_generator.convert_to_generator_like(
+        data, batch_size=2, steps_per_epoch=expected_batches)
+    self.assertEqual(steps, expected_batches)
+
+    for _ in range(expected_batches):
+      outputs = next(generator)
+    nest.assert_same_structure(outputs, inputs)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index b7c82bfde3..0157fe084c 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -18,14 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import abc
 from collections import OrderedDict
 import copy
-import math
 
 import numpy as np
 import six
 
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
@@ -33,9 +32,11 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import callbacks as cbks
 from tensorflow.python.keras import losses
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras.engine import base_layer
+from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -43,143 +44,150 @@ from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.util import nest
 
 
-def _map_nested(data, func):
-  """Maps each nested element using func."""
-  if isinstance(data, list):
-    return [_map_nested(nested_data, func) for nested_data in data]
-  elif isinstance(data, tuple):
-    return tuple(_map_nested(nested_data, func) for nested_data in data)
-  elif isinstance(data, dict):
-    return {
-        k: _map_nested(nested_data, func) for k, nested_data in data.items()
-    }
-  else:
-    return func(data)
+@six.add_metaclass(abc.ABCMeta)
+class Aggregator(object):
+  """Abstract base class used to aggregate batch-level outputs of a loop.
 
+  Attributes:
+    use_steps: Whether the loop is using `step` or `batch_size`.
+    num_samples_or_steps: Either `batch_size*num_batches` or `steps`.
+    results: What to return at the end of the aggregation loop.
+  """
 
-def _nested_all(data, cond_func):
-  """Checks if all elements in a nested structure satisfy cond_func."""
-  if isinstance(data, (tuple, list)):
-    return all(_nested_all(nested_data, cond_func) for nested_data in data)
-  elif isinstance(data, dict):
-    return all(
-        _nested_all(nested_data, cond_func) for nested_data in data.values())
-  else:
-    return cond_func(data)
+  def __init__(self, use_steps, num_samples_or_steps):
+    self.use_steps = use_steps
+    self.num_samples_or_steps = num_samples_or_steps
+    self.results = []
 
+  @abc.abstractmethod
+  def create(self, batch_outs):
+    """Creates the initial results from the first batch outputs.
 
-def _nested_any(data, cond_func):
-  """Checks if any nested_elements in a nested structure satisfy cond_func."""
-  if isinstance(data, (tuple, list)):
-    return any(_nested_any(nested_data, cond_func) for nested_data in data)
-  elif isinstance(data, dict):
-    return any(
-        [_nested_any(nested_data, cond_func) for nested_data in data.values()])
-  else:
-    return cond_func(data)
-
-
-def _convert_lists_to_tuples(data):
-  """Converts all lists to tuples, since Datasets expect tuples."""
-  if isinstance(data, (tuple, list)):
-    return tuple(_convert_lists_to_tuples(nested_data) for nested_data in data)
-  elif isinstance(data, dict):
-    return {
-        k: _convert_lists_to_tuples(nested_data)
-        for k, nested_data in data.items()
-    }
-  else:
-    return data
+    Arguments:
+      batch_outs: A list of batch-level outputs.
+    """
+    NotImplementedError('Must be implemented in subclasses.')
 
+  @abc.abstractmethod
+  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
+    """Aggregates batch-level results into total results.
 
-def _get_batch_axis_size(data):
-  """Returns batch axis shape for nested data."""
-  if isinstance(data, (tuple, list)):
-    return _get_batch_axis_size(data[0])
-  elif isinstance(data, dict):
-    return _get_batch_axis_size(list(data.values()))
-  else:
-    return int(data.shape[0])
+    Arguments:
+      batch_outs: A list of batch-level outputs.
+      batch_start: The start index of this batch. Always `None` if `use_steps`
+        is `True`.
+      batch_end: The end index of this batch. Always `None` if `use_steps` is
+        `True`.
+    """
+    NotImplementedError('Must be implemented in subclasses.')
 
+  @abc.abstractmethod
+  def finalize(self):
+    """Prepares the total results to be returned."""
+    NotImplementedError('Must be implemented in subclasses.')
 
-def convert_to_iterator(x=None,
-                        y=None,
-                        sample_weights=None,
-                        batch_size=None,
-                        steps_per_epoch=None,
-                        epochs=1,
-                        shuffle=False,
-                        is_validation=False):
-  """Converts NumPy arrays or EagerTensors to an EagerIterator.
 
-  Combines all provided data into a single EagerIterator.
+class MetricsAggregator(Aggregator):
+  """Aggregator that calculates loss and metrics info."""
 
-  Arguments:
-      x: NumPy array or EagerTensor,  or list of Numpy arrays or EagerTensors
-        representing inputs to a model.
-      y: Optional. NumPy array or EagerTensor, or list of Numpy arrays or
-        EagerTensors representing targets of a model.
-      sample_weights: Optional NumPy array or EagerTensor representing sample
-        weights.
-      batch_size: Used to batch data and calculate how many steps EagerIterator
-        should take per epoch.
-      steps_per_epoch: If provided, how many steps EagerIterator should take per
-        epoch.
-      epochs: Epochs to repeat iterator for.
-      shuffle: Whether to shuffle data after each epoch.
-      is_validation: Whether this call is for validation during a training
-        (e.g., `fit()`) call. This info is used to construct error messages
-        (if any).
+  def create(self, batch_outs):
+    self.results = [0.] * len(batch_outs)
 
-  Raises:
-      ValueError: if steps_per_epoch cannot be calculated from the data
-      provided.
+  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
+    # Loss.
+    if self.use_steps:
+      self.results[0] += batch_outs[0]
+    else:
+      self.results[0] += batch_outs[0] * (batch_end - batch_start)
+    # Metrics (always stateful, just grab current values.)
+    self.results[1:] = batch_outs[1:]
+
+  def finalize(self):
+    self.results[0] /= self.num_samples_or_steps
 
-  Returns:
-      (Iterator, steps_per_epoch).
 
+class OutputsAggregator(Aggregator):
+  """Aggregator that concatenates outputs."""
+
+  def create(self, batch_outs):
+    if self.use_steps:
+      # Cannot pre-allocate the returned NumPy arrays bc
+      # batch sizes are unknown. Concatenate batches at the end.
+      for _ in batch_outs:
+        self.results.append([])
+    else:
+      # Pre-allocate NumPy arrays.
+      for batch_out in batch_outs:
+        shape = (self.num_samples_or_steps,) + batch_out.shape[1:]
+        self.results.append(np.zeros(shape, dtype=batch_out.dtype))
+
+  def aggregate(self, batch_outs, batch_start=None, batch_end=None):
+    if self.use_steps:
+      for i, batch_out in enumerate(batch_outs):
+        self.results[i].append(batch_out)
+    else:
+      for i, batch_out in enumerate(batch_outs):
+        self.results[i][batch_start:batch_end] = batch_out
+
+  def finalize(self):
+    if self.use_steps:
+      self.results = [np.concatenate(result, axis=0) for result in self.results]
+
+
+def make_logs(model, outputs, mode, prefix=''):
+  """Computes logs for sending to `on_batch_end` methods."""
+  logs = {}
+  # TODO(omalleyt): handle outputs in prediction when Callback
+  # hooks are ready.
+  if mode in ['train', 'test']:
+    if hasattr(model, 'metrics_names'):
+      for label, output in zip(model.metrics_names, outputs):
+        logs[prefix + label] = output
+  return logs
+
+
+def get_progbar(model, count_mode):
+  """Get Progbar."""
+  stateful_metric_names = None
+  if hasattr(model, 'metrics_names'):
+    stateful_metric_names = model.metrics_names[1:]  # Exclude `loss`
+  return cbks.ProgbarLogger(count_mode, stateful_metrics=stateful_metric_names)
+
+
+def slice_arrays(arrays, indices, contiguous=True):
+  """Slices batches out of provided arrays (workaround for eager tensors).
+
+  Unfortunately eager tensors don't have the same slicing behavior as
+  Numpy arrays (they follow the same slicing behavior as symbolic TF tensors),
+  hence we cannot use `generic_utils.slice_arrays` directly
+  and we have to implement this workaround based on `concat`. This has a
+  performance cost.
+
+  Arguments:
+    arrays: Single array or list of arrays.
+    indices: List of indices in the array that should be included in the output
+      batch.
+    contiguous: Boolean flag indicating whether the indices are contiguous.
+
+  Returns:
+    Slice of data (either single array or list of arrays).
   """
-  if isinstance(x, iterator_ops.EagerIterator):
-    if steps_per_epoch is None:
-      raise ValueError('You must specify the number of steps (number of batches'
-                       ' to draw from the iterator).')
-    return x, steps_per_epoch
-
-  if not _nested_any(sample_weights, lambda x: x is None):
-    data = (x, y, sample_weights)
-  elif not _nested_any(y, lambda x: x is None):
-    data = (x, y)
+  converted_to_list = False
+  if not isinstance(arrays, list):
+    converted_to_list = True
+    arrays = [arrays]
+  if any(tensor_util.is_tensor(x) for x in arrays):
+    if not contiguous:
+      entries = [[x[i:i + 1] for i in indices] for x in arrays]
+      slices = [array_ops.concat(x, axis=0) for x in entries]
+    else:
+      slices = [x[indices[0]:indices[-1] + 1] for x in arrays]
   else:
-    # always wrap in a tuple, so we know y, sample_weights weren't set
-    # even when x has multiple elements
-    data = (x,)
-
-  data = _convert_lists_to_tuples(data)
-  if steps_per_epoch is None and batch_size is not None:
-    num_samples = _get_batch_axis_size(data)
-    steps_per_epoch = int(math.ceil(num_samples / int(batch_size)))
-
-  if steps_per_epoch is None:
-    alternative_arg_name = (
-        'validation_steps' if is_validation else 'steps_per_epoch')
-    raise ValueError(
-        'Could not determine how to convert EagerTensors into EagerIterator. '
-        'Please provide either `batch_size` or '
-        '`%s`.' % alternative_arg_name)
-
-  # TODO(omalleyt) for NumPy arrays in graph mode
-  # placeholder ops should be used
-  # this is only ideal for eager mode
-  dataset = dataset_ops.Dataset.from_tensor_slices(data)
-
-  if batch_size is not None:
-    dataset = dataset.batch(batch_size)
-  if shuffle:
-    dataset = dataset.shuffle(buffer_size=10000)
-  dataset = dataset.repeat(epochs)
-  iterator = dataset_ops.make_one_shot_iterator(dataset)
-
-  return iterator, steps_per_epoch
+    slices = generic_utils.slice_arrays(arrays, indices)
+
+  if converted_to_list:
+    slices = slices[0]
+  return slices
 
 
 def check_num_samples(ins,
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index 4087af5c00..44ea23998f 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -21,146 +21,12 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.eager import context
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.platform import test
 
 
-class TrainingUtilTest(test.TestCase):
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_single_numpy(self):
-    batch_size = 2
-    a = np.ones([10, 10])
-    iterator, steps_per_epoch = training_utils.convert_to_iterator(
-        x=a, batch_size=batch_size)
-    self.assertEqual(steps_per_epoch, 5)
-
-    expected_batch = a[:batch_size, :]
-    actual_batch, = iterator.get_next()
-    self.assertAllEqual(expected_batch, actual_batch)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_single_tensor(self):
-    batch_size = 2
-    a = ops.convert_to_tensor(np.ones([10, 10]))
-    iterator, steps_per_epoch = training_utils.convert_to_iterator(
-        x=a, batch_size=batch_size)
-    self.assertEqual(steps_per_epoch, 5)
-
-    expected_batch = a[:batch_size, :]
-    actual_batch, = iterator.get_next()
-    self.assertAllEqual(expected_batch, actual_batch)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_y(self):
-    batch_size = 2
-    a = np.ones([10, 100])
-    b = np.ones([10, 10])
-    iterator, steps_per_epoch = training_utils.convert_to_iterator(
-        x=a, y=b, batch_size=batch_size)
-    self.assertEqual(steps_per_epoch, 5)
-
-    expected_x = a[:batch_size, :]
-    expected_y = b[:batch_size, :]
-    actual_x, actual_y = iterator.get_next()
-    self.assertAllEqual(expected_x, actual_x)
-    self.assertAllEqual(expected_y, actual_y)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_sample_weights(self):
-    batch_size = 2
-    a = ops.convert_to_tensor(np.ones([10, 100]))
-    b = ops.convert_to_tensor(np.ones([10, 10]))
-    sw = ops.convert_to_tensor(np.ones([10]))
-    iterator, steps_per_epoch = training_utils.convert_to_iterator(
-        x=a, y=b, sample_weights=sw, batch_size=batch_size)
-    self.assertEqual(steps_per_epoch, 5)
-
-    expected_x = a[:batch_size, :]
-    expected_y = b[:batch_size, :]
-    expected_sw = sw[:batch_size]
-    actual_x, actual_y, actual_sw = iterator.get_next()
-    self.assertAllEqual(expected_x, actual_x)
-    self.assertAllEqual(expected_y, actual_y)
-    self.assertAllEqual(expected_sw, actual_sw)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_nested(self):
-    batch_size = 2
-    x = {'1': np.ones([10, 100]), '2': [np.zeros([10, 10]), np.ones([10, 20])]}
-    iterator, steps_per_epoch = training_utils.convert_to_iterator(
-        x=x, batch_size=batch_size)
-    self.assertEqual(steps_per_epoch, 5)
-
-    expected_x1 = x['1'][:batch_size, :]
-    expected_x2_0 = x['2'][0][:batch_size, :]
-    expected_x2_1 = x['2'][1][:batch_size, :]
-
-    actual_x, = iterator.get_next()
-    actual_x1 = actual_x['1'][:batch_size, :]
-    actual_x2_0 = actual_x['2'][0][:batch_size, :]
-    actual_x2_1 = actual_x['2'][1][:batch_size, :]
-
-    self.assertAllEqual(expected_x1, actual_x1)
-    self.assertAllEqual(expected_x2_0, actual_x2_0)
-    self.assertAllEqual(expected_x2_1, actual_x2_1)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_epochs(self):
-    batch_size = 2
-    a = np.ones([10, 10])
-    iterator, steps_per_epoch = training_utils.convert_to_iterator(
-        x=a, batch_size=batch_size, epochs=2)
-    self.assertEqual(steps_per_epoch, 5)
-
-    expected_batch = a[:batch_size, :]
-    # loop through one whole epoch
-    for _ in range(6):
-      actual_batch, = iterator.get_next()
-    self.assertAllEqual(expected_batch, actual_batch)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_convert_to_iterator_insufficient_info(self):
-    # with batch_size and steps_per_epoch not set
-    with self.assertRaises(ValueError):
-      a = np.ones([10, 10])
-      _ = training_utils.convert_to_iterator(x=a)
-
-  def test_nested_all(self):
-    nested_data = {'a': True, 'b': [True, True, (False, True)]}
-    all_true = training_utils._nested_all(nested_data, lambda x: x)
-    self.assertEqual(all_true, False)
-
-    nested_data = {'a': True, 'b': [True, True, (True, True)]}
-    all_true = training_utils._nested_all(nested_data, lambda x: x)
-    self.assertEqual(all_true, True)
-
-  def test_nested_any(self):
-    nested_data = [False, {'a': False, 'b': (False, True)}]
-    any_true = training_utils._nested_any(nested_data, lambda x: x)
-    self.assertEqual(any_true, True)
-
-    nested_data = [False, {'a': False, 'b': (False, False)}]
-    any_true = training_utils._nested_any(nested_data, lambda x: x)
-    self.assertEqual(any_true, False)
-
-  def test_check_array_lengths(self):
-    training_utils.check_array_lengths(None, None, None)
-    a_np = np.random.random((4, 3, 3))
-    training_utils.check_array_lengths(a_np, a_np, a_np)
-    training_utils.check_array_lengths(
-        [a_np, a_np], [a_np, a_np], [a_np, a_np])
-    training_utils.check_array_lengths([None], [None], [None])
-
-    b_np = np.random.random((3, 4))
-    with self.assertRaises(ValueError):
-      training_utils.check_array_lengths([a_np], [b_np], None)
-
-
 class ModelInputsTest(test.TestCase):
 
   def test_single_thing(self):
diff --git a/tensorflow/python/keras/layers/local_test.py b/tensorflow/python/keras/layers/local_test.py
index a23c5c38fe..e4f4d0a639 100644
--- a/tensorflow/python/keras/layers/local_test.py
+++ b/tensorflow/python/keras/layers/local_test.py
@@ -294,8 +294,8 @@ class LocallyConnectedImplementationModeTest(test.TestCase):
                         # Compare outputs after a few training steps.
                         out_1 = model_1.call(inputs)
                         out_2 = model_2.call(inputs)
-                        self.assertAllCloseAccordingToType(out_1, out_2,
-                                                           atol=1e-4)
+                        self.assertAllCloseAccordingToType(
+                            out_1, out_2, atol=2e-4)
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_make_2d(self):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index 9dc8daea5c..a3254cbd94 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -241,6 +241,10 @@ tf_class {
     name: "predict_on_batch"
     argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -263,7 +267,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -275,6 +279,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index a357a82515..b70e9ee98d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -258,6 +258,10 @@ tf_class {
     name: "predict_proba"
     argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], "
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -280,7 +284,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -292,6 +296,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 41d8b2fc95..c58c7bef22 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -241,6 +241,10 @@ tf_class {
     name: "predict_on_batch"
     argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -263,7 +267,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -275,6 +279,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index 2cf107a5cd..473a1c16fb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -258,6 +258,10 @@ tf_class {
     name: "predict_proba"
     argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], "
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -280,7 +284,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -292,6 +296,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index 9dc8daea5c..a3254cbd94 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -241,6 +241,10 @@ tf_class {
     name: "predict_on_batch"
     argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -263,7 +267,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -275,6 +279,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index a357a82515..b70e9ee98d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -258,6 +258,10 @@ tf_class {
     name: "predict_proba"
     argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], "
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -280,7 +284,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -292,6 +296,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index a94bd5930f..2b66576c96 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -246,6 +246,10 @@ tf_class {
     name: "predict_on_batch"
     argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -268,7 +272,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -280,6 +284,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 41d8b2fc95..c58c7bef22 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -241,6 +241,10 @@ tf_class {
     name: "predict_on_batch"
     argspec: "args=[\'self\', \'x\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -263,7 +267,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -275,6 +279,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index 2cf107a5cd..473a1c16fb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -258,6 +258,10 @@ tf_class {
     name: "predict_proba"
     argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], "
   }
+  member_method {
+    name: "reset_metrics"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "reset_states"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -280,7 +284,7 @@ tf_class {
   }
   member_method {
     name: "test_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\'], "
   }
   member_method {
     name: "to_json"
@@ -292,6 +296,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\', \'reset_metrics\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
   }
 }
-- 
GitLab


From dcd9dc6f23431f1dbf6ebf7f30f56cdc3d5caaa2 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 4 Dec 2018 11:29:34 -0800
Subject: [PATCH 1297/1554] Implement amsgrad optimization.

PiperOrigin-RevId: 224014715
---
 ...api_def_ResourceApplyAdamWithAmsgrad.pbtxt |  85 +++++++++
 ...api_def_ResourceApplyAdamWithAmsgrad.pbtxt |   4 +
 tensorflow/core/kernels/training_ops.cc       | 163 ++++++++++++++++++
 tensorflow/core/kernels/training_ops.h        |  14 ++
 .../core/kernels/training_ops_gpu.cu.cc       |  37 ++++
 tensorflow/core/kernels/training_ops_test.cc  |  44 +++++
 tensorflow/core/ops/training_ops.cc           |  38 ++++
 tensorflow/python/keras/optimizer_v2/adam.py  |  68 ++++++--
 .../python/keras/optimizer_v2/adam_test.py    | 145 +++++++++++++++-
 .../keras/optimizer_v2/optimizer_v2_test.py   |  33 ++++
 10 files changed, 607 insertions(+), 24 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt
new file mode 100644
index 0000000000..8ee16ef1ba
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt
@@ -0,0 +1,85 @@
+op {
+  graph_op_name: "ResourceApplyAdamWithAmsgrad"
+  in_arg {
+    name: "var"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "m"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "v"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "vhat"
+    description: <<END
+Should be from a Variable().
+END
+  }
+  in_arg {
+    name: "beta1_power"
+    description: <<END
+Must be a scalar.
+END
+  }
+  in_arg {
+    name: "beta2_power"
+    description: <<END
+Must be a scalar.
+END
+  }
+  in_arg {
+    name: "lr"
+    description: <<END
+Scaling factor. Must be a scalar.
+END
+  }
+  in_arg {
+    name: "beta1"
+    description: <<END
+Momentum factor. Must be a scalar.
+END
+  }
+  in_arg {
+    name: "beta2"
+    description: <<END
+Momentum factor. Must be a scalar.
+END
+  }
+  in_arg {
+    name: "epsilon"
+    description: <<END
+Ridge term. Must be a scalar.
+END
+  }
+  in_arg {
+    name: "grad"
+    description: <<END
+The gradient.
+END
+  }
+  attr {
+    name: "use_locking"
+    description: <<END
+If `True`, updating of the var, m, and v tensors will be protected
+by a lock; otherwise the behavior is undefined, but may exhibit less
+contention.
+END
+  }
+  summary: "Update \'*var\' according to the Adam algorithm."
+  description: <<END
+$$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+$$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+$$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+$$vhat_t := max{vhat_{t-1}, v_t}$$
+$$variable := variable - lr_t * m_t / (\sqrt{vhat_t} + \epsilon)$$
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt
new file mode 100644
index 0000000000..1eef1b69b9
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceApplyAdamWithAmsgrad.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceApplyAdamWithAmsgrad"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc
index b5ddc8e1b0..6504ad1b09 100644
--- a/tensorflow/core/kernels/training_ops.cc
+++ b/tensorflow/core/kernels/training_ops.cc
@@ -347,6 +347,28 @@ struct ApplyAdamSYCL {
 template <typename T>
 struct ApplyAdam<CPUDevice, T> : ApplyAdamNonCuda<CPUDevice, T> {};
 
+template <typename T>
+struct ApplyAdamWithAmsgrad<CPUDevice, T> {
+  void operator()(const CPUDevice& d, typename TTypes<T>::Flat var,
+                  typename TTypes<T>::Flat m, typename TTypes<T>::Flat v,
+                  typename TTypes<T>::Flat vhat,
+                  typename TTypes<T>::ConstScalar beta1_power,
+                  typename TTypes<T>::ConstScalar beta2_power,
+                  typename TTypes<T>::ConstScalar lr,
+                  typename TTypes<T>::ConstScalar beta1,
+                  typename TTypes<T>::ConstScalar beta2,
+                  typename TTypes<T>::ConstScalar epsilon,
+                  typename TTypes<T>::ConstFlat grad) {
+    const T alpha = lr() * Eigen::numext::sqrt(T(1) - beta2_power()) /
+                    (T(1) - beta1_power());
+
+    m.device(d) += (grad - m) * (T(1) - beta1());
+    v.device(d) += (grad.square() - v) * (T(1) - beta2());
+    vhat.device(d) = vhat.cwiseMax(v);
+    var.device(d) -= (m * alpha) / (vhat.sqrt() + epsilon());
+  }
+};
+
 template <typename Device, typename T>
 struct ApplyAdaMaxNonCuda {
   void operator()(const Device& d, typename TTypes<T>::Flat var,
@@ -3013,6 +3035,147 @@ REGISTER_KERNELS(GPU, double);
 #undef REGISTER_CPU_KERNELS
 #undef REGISTER_KERNELS
 
+template <typename Device, typename T>
+class ApplyAdamWithAmsgradOp : public OpKernel {
+ public:
+  explicit ApplyAdamWithAmsgradOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("use_locking", &use_exclusive_lock_));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    auto locks = MaybeLockVariableInputMutexesInOrder(ctx, use_exclusive_lock_,
+                                                      {0, 1, 2});
+
+    Tensor var;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<Device, T>(
+                            ctx, 0, use_exclusive_lock_, false, &var));
+    Tensor m;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<Device, T>(
+                            ctx, 1, use_exclusive_lock_, false, &m));
+    Tensor v;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<Device, T>(
+                            ctx, 2, use_exclusive_lock_, false, &v));
+    Tensor vhat;
+    OP_REQUIRES_OK(ctx, GetInputTensorFromVariable<Device, T>(
+                            ctx, 3, use_exclusive_lock_, false, &vhat));
+    OP_REQUIRES(
+        ctx, var.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(0)));
+    OP_REQUIRES(
+        ctx, m.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(1)));
+    OP_REQUIRES(
+        ctx, v.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(2)));
+    OP_REQUIRES(
+        ctx, vhat.IsInitialized(),
+        errors::FailedPrecondition(
+            "Attempting to use uninitialized variables: ", requested_input(2)));
+
+    const Tensor& beta1_power = ctx->input(4);
+    const Tensor& beta2_power = ctx->input(5);
+    const Tensor& lr = ctx->input(6);
+    const Tensor& beta1 = ctx->input(7);
+    const Tensor& beta2 = ctx->input(8);
+    const Tensor& epsilon = ctx->input(9);
+
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1_power.shape()),
+                errors::InvalidArgument("beta1_power is not a scalar: ",
+                                        beta1_power.shape().DebugString()));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2_power.shape()),
+                errors::InvalidArgument("beta2_power is not a scalar: ",
+                                        beta2_power.shape().DebugString()));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(lr.shape()),
+                errors::InvalidArgument("lr is not a scalar : ",
+                                        lr.shape().DebugString()));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta1.shape()),
+                errors::InvalidArgument("beta1 is not a scalar: ",
+                                        beta1.shape().DebugString()));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(beta2.shape()),
+                errors::InvalidArgument("beta2 is not a scalar: ",
+                                        beta2.shape().DebugString()));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(epsilon.shape()),
+                errors::InvalidArgument("epsilon is not a scalar: ",
+                                        epsilon.shape().DebugString()));
+
+    const Tensor& grad = ctx->input(10);
+    OP_REQUIRES(ctx, var.shape().IsSameSize(m.shape()),
+                errors::InvalidArgument("var and m do not have the same shape",
+                                        var.shape().DebugString(), " ",
+                                        m.shape().DebugString()));
+    OP_REQUIRES(ctx, var.shape().IsSameSize(v.shape()),
+                errors::InvalidArgument("var and v do not have the same shape",
+                                        var.shape().DebugString(), " ",
+                                        v.shape().DebugString()));
+    OP_REQUIRES(
+        ctx, var.shape().IsSameSize(grad.shape()),
+        errors::InvalidArgument("var and grad do not have the same shape",
+                                var.shape().DebugString(), " ",
+                                grad.shape().DebugString()));
+
+    const Device& device = ctx->template eigen_device<Device>();
+    functor::ApplyAdamWithAmsgrad<Device, T>()(
+        device, var.flat<T>(), m.flat<T>(), v.flat<T>(), vhat.flat<T>(),
+        beta1_power.scalar<T>(), beta2_power.scalar<T>(), lr.scalar<T>(),
+        beta1.scalar<T>(), beta2.scalar<T>(), epsilon.scalar<T>(),
+        grad.flat<T>());
+
+    MaybeForwardRefInputToRefOutput(ctx, 0, 0);
+  }
+
+ private:
+  bool use_exclusive_lock_;
+};
+
+#define REGISTER_KERNELS(D, T)                                 \
+  REGISTER_KERNEL_BUILDER(Name("ResourceApplyAdamWithAmsgrad") \
+                              .HostMemory("var")               \
+                              .HostMemory("m")                 \
+                              .HostMemory("v")                 \
+                              .HostMemory("vhat")              \
+                              .Device(DEVICE_##D)              \
+                              .TypeConstraint<T>("T"),         \
+                          ApplyAdamWithAmsgradOp<D##Device, T>);
+#define REGISTER_CPU_KERNELS(T) REGISTER_KERNELS(CPU, T);
+
+TF_CALL_half(REGISTER_CPU_KERNELS);
+TF_CALL_bfloat16(REGISTER_CPU_KERNELS);
+TF_CALL_float(REGISTER_CPU_KERNELS);
+TF_CALL_double(REGISTER_CPU_KERNELS);
+
+#if GOOGLE_CUDA
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+#define DECLARE_GPU_SPEC(T)                                   \
+  template <>                                                 \
+  void ApplyAdamWithAmsgrad<GPUDevice, T>::operator()(        \
+      const GPUDevice& d, typename TTypes<T>::Flat var,       \
+      typename TTypes<T>::Flat m, typename TTypes<T>::Flat v, \
+      typename TTypes<T>::Flat vhat,                          \
+      typename TTypes<T>::ConstScalar beta1_power,            \
+      typename TTypes<T>::ConstScalar beta2_power,            \
+      typename TTypes<T>::ConstScalar lr,                     \
+      typename TTypes<T>::ConstScalar beta1,                  \
+      typename TTypes<T>::ConstScalar beta2,                  \
+      typename TTypes<T>::ConstScalar epsilon,                \
+      typename TTypes<T>::ConstFlat grad);                    \
+  extern template struct ApplyAdamWithAmsgrad<GPUDevice, T>;
+DECLARE_GPU_SPEC(Eigen::half);
+DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(double);
+#undef DECLARE_GPU_SPEC
+}  // namespace functor
+
+REGISTER_KERNELS(GPU, Eigen::half);
+REGISTER_KERNELS(GPU, float);
+REGISTER_KERNELS(GPU, double);
+#endif
+#undef REGISTER_CPU_KERNELS
+#undef REGISTER_KERNELS
+
 template <typename Device, typename T>
 class ApplyAdaMaxOp : public OpKernel {
  public:
diff --git a/tensorflow/core/kernels/training_ops.h b/tensorflow/core/kernels/training_ops.h
index 557f8b6285..054f07350e 100644
--- a/tensorflow/core/kernels/training_ops.h
+++ b/tensorflow/core/kernels/training_ops.h
@@ -148,6 +148,20 @@ struct ApplyAdam {
                   typename TTypes<T>::ConstFlat grad, bool use_nesterov);
 };
 
+template <typename Device, typename T>
+struct ApplyAdamWithAmsgrad {
+  void operator()(const Device& d, typename TTypes<T>::Flat var,
+                  typename TTypes<T>::Flat m, typename TTypes<T>::Flat v,
+                  typename TTypes<T>::Flat vhat,
+                  typename TTypes<T>::ConstScalar beta1_power,
+                  typename TTypes<T>::ConstScalar beta2_power,
+                  typename TTypes<T>::ConstScalar lr,
+                  typename TTypes<T>::ConstScalar beta1,
+                  typename TTypes<T>::ConstScalar beta2,
+                  typename TTypes<T>::ConstScalar epsilon,
+                  typename TTypes<T>::ConstFlat grad);
+};
+
 template <typename Device, typename T>
 struct ApplyAdaMax {
   void operator()(const Device& d, typename TTypes<T>::Flat var,
diff --git a/tensorflow/core/kernels/training_ops_gpu.cu.cc b/tensorflow/core/kernels/training_ops_gpu.cu.cc
index 9ae794925f..f45b9ffca7 100644
--- a/tensorflow/core/kernels/training_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/training_ops_gpu.cu.cc
@@ -165,6 +165,39 @@ struct ApplyAdam<GPUDevice, T> {
   }
 };
 
+template <typename T>
+struct ApplyAdamWithAmsgrad<GPUDevice, T> {
+  void operator()(const GPUDevice& d, typename TTypes<T>::Flat var,
+                  typename TTypes<T>::Flat m, typename TTypes<T>::Flat v,
+                  typename TTypes<T>::Flat vhat,
+                  typename TTypes<T>::ConstScalar beta1_power,
+                  typename TTypes<T>::ConstScalar beta2_power,
+                  typename TTypes<T>::ConstScalar lr,
+                  typename TTypes<T>::ConstScalar beta1,
+                  typename TTypes<T>::ConstScalar beta2,
+                  typename TTypes<T>::ConstScalar epsilon,
+                  typename TTypes<T>::ConstFlat grad) {
+    Eigen::array<typename TTypes<T>::Tensor::Index, 1> bcast;
+    bcast[0] = grad.dimension(0);
+    Eigen::Sizes<1> single;
+    const auto one = static_cast<T>(1.0);
+    m.device(d) =
+        m + (beta1.constant(one) - beta1).reshape(single).broadcast(bcast) *
+                (grad - m);
+    v.device(d) =
+        v + (beta2.constant(one) - beta2).reshape(single).broadcast(bcast) *
+                (grad.square() - v);
+    vhat.device(d) = vhat.cwiseMax(v);
+
+    var.device(d) -= (lr * (beta2_power.constant(one) - beta2_power).sqrt() /
+                      (beta1_power.constant(one) - beta1_power))
+                         .reshape(single)
+                         .broadcast(bcast) *
+                     m /
+                     (epsilon.reshape(single).broadcast(bcast) + vhat.sqrt());
+  }
+};
+
 template <typename T>
 struct ApplyAdaMax<GPUDevice, T> {
   void operator()(const GPUDevice& d, typename TTypes<T>::Flat var,
@@ -331,6 +364,10 @@ template struct functor::ApplyAdam<GPUDevice, Eigen::half>;
 template struct functor::ApplyAdam<GPUDevice, float>;
 template struct functor::ApplyAdam<GPUDevice, double>;
 
+template struct functor::ApplyAdamWithAmsgrad<GPUDevice, Eigen::half>;
+template struct functor::ApplyAdamWithAmsgrad<GPUDevice, float>;
+template struct functor::ApplyAdamWithAmsgrad<GPUDevice, double>;
+
 template struct functor::ApplyAdaMax<GPUDevice, Eigen::half>;
 template struct functor::ApplyAdaMax<GPUDevice, float>;
 template struct functor::ApplyAdaMax<GPUDevice, double>;
diff --git a/tensorflow/core/kernels/training_ops_test.cc b/tensorflow/core/kernels/training_ops_test.cc
index 8e0597149d..1ec57b4522 100644
--- a/tensorflow/core/kernels/training_ops_test.cc
+++ b/tensorflow/core/kernels/training_ops_test.cc
@@ -228,6 +228,50 @@ static void BM_Adam(int iters, int params) {
 }
 BENCHMARK(BM_Adam)->Arg(128 << 10)->Arg(256 << 10);
 
+static void AdamWithAmsgrad(int32 n, Graph** init_g, Graph** train_g) {
+  TensorShape shape({n});
+  {
+    Graph* g = new Graph(OpRegistry::Global());
+    auto var = Var(g, n);
+    auto m = Var(g, n);
+    auto v = Var(g, n);
+    auto zero = Zeros(g, n);
+    test::graph::Assign(g, var, zero);
+    test::graph::Assign(g, m, zero);
+    test::graph::Assign(g, v, zero);
+    *init_g = g;
+  }
+  {
+    Graph* g = new Graph(OpRegistry::Global());
+    auto var = Var(g, n);
+    auto m = Var(g, n);
+    auto v = Var(g, n);
+    auto vhat = Var(g, n);
+    auto beta1_power = Scalar(g, 0.9);
+    auto beta2_power = Scalar(g, 0.99);
+    auto lr = Scalar(g, 0.01);
+    auto beta1 = Scalar(g, 0.9);
+    auto beta2 = Scalar(g, 0.99);
+    auto epsilon = Scalar(g, 1e-8);
+    auto grad = Random(g, n);
+    test::graph::Multi(g, "ApplyAdamWithAmsgrad",
+                       {var, m, v, vhat, beta1_power, beta2_power, lr, beta1,
+                        beta2, epsilon, grad});
+    *train_g = g;
+  }
+}
+
+static void BM_AdamWithAmsgrad(int iters, int params) {
+  const int64 tot = static_cast<int64>(iters) * params;
+  testing::ItemsProcessed(tot);
+  testing::BytesProcessed(tot * sizeof(float));
+  Graph* init;
+  Graph* train;
+  AdamWithAmsgrad(params, &init, &train);
+  test::Benchmark("cpu", train, GetOptions(), init).Run(iters);
+}
+BENCHMARK(BM_AdamWithAmsgrad)->Arg(128 << 10)->Arg(256 << 10);
+
 static void RMSProp(int32 n, Graph** init_g, Graph** train_g) {
   TensorShape shape({n});
   {
diff --git a/tensorflow/core/ops/training_ops.cc b/tensorflow/core/ops/training_ops.cc
index eb4db1aec2..995ed42d53 100644
--- a/tensorflow/core/ops/training_ops.cc
+++ b/tensorflow/core/ops/training_ops.cc
@@ -769,6 +769,44 @@ REGISTER_OP("ResourceApplyAdam")
       return ApplyAdamShapeFn(c, false /* sparse */);
     });
 
+static Status ApplyAdamWithAmsgradShapeFn(InferenceContext* c, bool sparse) {
+  ShapeHandle unused;
+  ShapeHandle s = ShapeOrHandleShape(c, 0);                       // var
+  TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 1), &s));  // m
+  TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 2), &s));  // v
+  TF_RETURN_IF_ERROR(c->Merge(s, ShapeOrHandleShape(c, 3), &s));  // vhat
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));       // beta1_power
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused));       // beta2_power
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 0, &unused));       // lr
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(7), 0, &unused));       // beta1
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(8), 0, &unused));       // beta2
+  TF_RETURN_IF_ERROR(c->WithRank(c->input(9), 0, &unused));       // epsilon
+  TF_RETURN_IF_ERROR(
+      HandleGradAndIndicesInputs(c, sparse, 10 /* grad_idx */, &s));
+  if (c->num_outputs() > 0) {
+    c->set_output(0, s);
+  }
+  return Status::OK();
+}
+
+REGISTER_OP("ResourceApplyAdamWithAmsgrad")
+    .Input("var: resource")
+    .Input("m: resource")
+    .Input("v: resource")
+    .Input("vhat: resource")
+    .Input("beta1_power: T")
+    .Input("beta2_power: T")
+    .Input("lr: T")
+    .Input("beta1: T")
+    .Input("beta2: T")
+    .Input("epsilon: T")
+    .Input("grad: T")
+    .Attr("T: numbertype")
+    .Attr("use_locking: bool = false")
+    .SetShapeFn([](InferenceContext* c) {
+      return ApplyAdamWithAmsgradShapeFn(c, false /* sparse */);
+    });
+
 static Status ApplyAdaMaxShapeFn(InferenceContext* c, bool sparse) {
   ShapeHandle unused;
   ShapeHandle s = ShapeOrHandleShape(c, 0);                       // var
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 06f8e52802..ef3d783f89 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -132,9 +132,6 @@ class Adam(optimizer_v2.OptimizerV2):
     self._set_hyper('beta_1', beta_1)
     self._set_hyper('beta_2', beta_2)
     self._set_hyper('epsilon', epsilon)
-    # TODO(tanzheny): create op for resource_apply_adam_with_amsgrad
-    if amsgrad:
-      raise ValueError('Amsgrad is currently not supported.')
     self._amsgrad = amsgrad
 
   def _create_slots(self, var_list):
@@ -144,6 +141,9 @@ class Adam(optimizer_v2.OptimizerV2):
       self.add_slot(var, 'm')
     for var in var_list:
       self.add_slot(var, 'v')
+    if self._amsgrad:
+      for var in var_list:
+        self.add_slot(var, 'vhat')
 
   def set_weights(self, weights):
     params = self.weights
@@ -162,21 +162,38 @@ class Adam(optimizer_v2.OptimizerV2):
     v = self.get_slot(var, 'v')
     beta_1_t = self._get_hyper('beta_1', var_dtype)
     beta_2_t = self._get_hyper('beta_2', var_dtype)
+    epsilon = self._get_hyper('epsilon', var_dtype)
     local_step = math_ops.cast(self.iterations + 1, var_dtype)
     beta_1_power = math_ops.pow(beta_1_t, local_step)
     beta_2_power = math_ops.pow(beta_2_t, local_step)
-    return training_ops.resource_apply_adam(
-        var.handle,
-        m.handle,
-        v.handle,
-        beta_1_power,
-        beta_2_power,
-        lr_t,
-        beta_1_t,
-        beta_2_t,
-        self._get_hyper('epsilon', var_dtype),
-        grad,
-        use_locking=self._use_locking)
+    if not self._amsgrad:
+      return training_ops.resource_apply_adam(
+          var.handle,
+          m.handle,
+          v.handle,
+          beta_1_power,
+          beta_2_power,
+          lr_t,
+          beta_1_t,
+          beta_2_t,
+          epsilon,
+          grad,
+          use_locking=self._use_locking)
+    else:
+      vhat = self.get_slot(var, 'vhat')
+      return training_ops.resource_apply_adam_with_amsgrad(
+          var.handle,
+          m.handle,
+          v.handle,
+          vhat.handle,
+          beta_1_power,
+          beta_2_power,
+          lr_t,
+          beta_1_t,
+          beta_2_t,
+          epsilon,
+          grad,
+          use_locking=self._use_locking)
 
   def _resource_apply_sparse(self, grad, var, indices):
     var_dtype = var.dtype.base_dtype
@@ -203,10 +220,23 @@ class Adam(optimizer_v2.OptimizerV2):
     with ops.control_dependencies([v_t]):
       v_t = self._resource_scatter_add(v, indices, v_scaled_g_values)
 
-    v_sqrt = math_ops.sqrt(v_t)
-    var_update = state_ops.assign_sub(
-        var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
-    return control_flow_ops.group(*[var_update, m_t, v_t])
+    if not self._amsgrad:
+      v_sqrt = math_ops.sqrt(v_t)
+      var_update = state_ops.assign_sub(
+          var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
+      return control_flow_ops.group(*[var_update, m_t, v_t])
+    else:
+      v_hat = self.get_slot(var, 'vhat')
+      v_hat_t = math_ops.maximum(v_hat, v_t)
+      with ops.control_dependencies([v_hat_t]):
+        v_hat_t = state_ops.assign(
+            v_hat, v_hat_t, use_locking=self._use_locking)
+      v_hat_sqrt = math_ops.sqrt(v_hat_t)
+      var_update = state_ops.assign_sub(
+          var,
+          lr * m_t / (v_hat_sqrt + epsilon_t),
+          use_locking=self._use_locking)
+      return control_flow_ops.group(*[var_update, m_t, v_t, v_hat_t])
 
   def _resource_scatter_add(self, x, i, v):
     with ops.control_dependencies(
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 958bbed21c..3bbafe12f8 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -52,6 +52,52 @@ def adam_update_numpy(param,
   return param_t, m_t, v_t
 
 
+def adam_update_numpy_amsgrad(param,
+                              g_t,
+                              t,
+                              m,
+                              v,
+                              vhat,
+                              lr=0.001,
+                              beta1=0.9,
+                              beta2=0.999,
+                              epsilon=1e-7):
+  lr_t = lr * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1))
+
+  m_t = beta1 * m + (1 - beta1) * g_t
+  v_t = beta2 * v + (1 - beta2) * g_t * g_t
+  vhat_t = np.maximum(vhat, v_t)
+
+  param_t = param - lr_t * m_t / (np.sqrt(vhat_t) + epsilon)
+  return param_t, m_t, v_t, vhat_t
+
+
+def adam_sparse_update_numpy_amsgrad(param,
+                                     indices,
+                                     g_t,
+                                     t,
+                                     m,
+                                     v,
+                                     vhat,
+                                     lr=0.001,
+                                     beta1=0.9,
+                                     beta2=0.999,
+                                     epsilon=1e-7):
+  m_t, v_t, vhat_t, param_t = (np.copy(m), np.copy(v), np.copy(vhat),
+                               np.copy(param))
+  lr_t = lr * np.sqrt(1 - beta2**(t + 1)) / (1 - beta1**(t + 1))
+  m_t_slice = beta1 * m[indices] + (1 - beta1) * g_t
+  v_t_slice = beta2 * v[indices] + (1 - beta2) * g_t * g_t
+  m_t[indices] = m_t_slice
+  v_t[indices] = v_t_slice
+  v_hat_t = np.maximum(vhat_t, v_t)
+  v_hat_t_slice = v_hat_t[indices]
+  param_t_slice = param[indices] - (
+      lr_t * (m_t_slice / (np.sqrt(v_hat_t_slice) + epsilon)))
+  param_t[indices] = param_t_slice
+  return param_t, m_t, v_t, vhat_t
+
+
 def get_beta_accumulators(opt, dtype):
   local_step = math_ops.cast(opt.iterations + 1, dtype)
   beta_1_t = math_ops.cast(opt._get_hyper("beta_1"), dtype)
@@ -212,6 +258,100 @@ class AdamOptimizerTest(test.TestCase):
     with context.eager_mode():
       self.doTestBasic(use_callable_params=True)
 
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testBasicWithAmsgrad(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, v0hat, m1, v1, v1hat = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        opt = adam.Adam(amsgrad=True)
+        if not context.executing_eagerly():
+          update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+        self.evaluate(variables.global_variables_initializer())
+        # Run 3 steps of Adam
+        for t in range(3):
+          beta_1_power, beta_2_power = get_beta_accumulators(opt, dtype)
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta_1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta_2_power))
+          if not context.executing_eagerly():
+            self.evaluate(update)
+          else:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          var0_np, m0, v0, v0hat = adam_update_numpy_amsgrad(
+              var0_np, grads0_np, t, m0, v0, v0hat)
+          var1_np, m1, v1, v1hat = adam_update_numpy_amsgrad(
+              var1_np, grads1_np, t, m1, v1, v1hat)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+
+  @test_util.run_in_graph_and_eager_modes
+  def testSparseWithAmsgrad(self):
+    # dtypes.half does not work on gpu + eager.
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.cached_session():
+        m0 = np.array([[0.0], [0.0]])
+        v0 = np.array([[0.0], [0.0]])
+        v0hat = np.array([[0.0], [0.0]])
+        indices_np = np.array([1])
+        indices = constant_op.constant(indices_np, dtype=dtypes.int32)
+        var0_np = np.array([[1.0], [2.0]], dtype=dtype.as_numpy_dtype)
+        repeated_index_update_var = variables.Variable(var0_np, dtype=dtype)
+        aggregated_update_var = variables.Variable(var0_np, dtype=dtype)
+        grads0_np = np.array([[0.2]], dtype=dtype.as_numpy_dtype)
+        grad_repeated_index = ops.IndexedSlices(
+            constant_op.constant([0.1, 0.1], shape=[2, 1], dtype=dtype),
+            constant_op.constant([1, 1]), constant_op.constant([2, 1]))
+        grad_aggregated = ops.IndexedSlices(grads0_np, indices,
+                                            constant_op.constant([2, 1]))
+        opt_repeated = adam.Adam(amsgrad=True)
+        opt_aggregated = adam.Adam(amsgrad=True)
+        if not context.executing_eagerly():
+          repeated_update = opt_repeated.apply_gradients(
+              [(grad_repeated_index, repeated_index_update_var)])
+          aggregated_update = opt_aggregated.apply_gradients(
+              [(grad_aggregated, aggregated_update_var)])
+        self.evaluate(variables.global_variables_initializer())
+        self.assertAllClose(
+            self.evaluate(aggregated_update_var),
+            self.evaluate(repeated_index_update_var))
+        for t in range(3):
+          if not context.executing_eagerly():
+            self.evaluate(repeated_update)
+            self.evaluate(aggregated_update)
+          else:
+            opt_repeated.apply_gradients(
+                [(grad_repeated_index, repeated_index_update_var)])
+            opt_aggregated.apply_gradients(
+                [(grad_aggregated, aggregated_update_var)])
+
+          var0_np, m0, v0, v0hat = adam_sparse_update_numpy_amsgrad(
+              var0_np, indices_np, grads0_np, t, m0, v0, v0hat)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(
+              var0_np, self.evaluate(aggregated_update_var))
+          self.assertAllCloseAccordingToType(
+              self.evaluate(aggregated_update_var),
+              self.evaluate(repeated_index_update_var))
+
   @test_util.run_deprecated_v1
   def testBasicWithLearningRateDecay(self):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
@@ -353,11 +493,6 @@ class AdamOptimizerTest(test.TestCase):
       self.assertEqual(
           self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
 
-  def testAmsgradWithError(self):
-    with self.assertRaisesRegexp(ValueError,
-                                 "Amsgrad is currently not supported"):
-      adam.Adam(learning_rate=1., beta_1=0.9, beta_2=0.99, amsgrad=True)
-
   def testSetWeightsFromV1AdamWithoutMinimize(self):
     keras_v1_adam = optimizers.Adam()
     keras_v2_adam = adam.Adam()
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index d7f4747bd7..158577fe64 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -590,6 +590,39 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(hist_k_v1.history['loss'], hist_tf.history['loss'])
       self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
 
+  def testNumericEquivalenceForAmsgrad(self):
+    np.random.seed(1331)
+    with self.cached_session():
+      train_samples = 20
+      input_dim = 3
+      num_classes = 2
+      (x, y), _ = testing_utils.get_test_data(
+          train_samples=train_samples,
+          test_samples=10,
+          input_shape=(input_dim,),
+          num_classes=num_classes)
+      y = keras.utils.to_categorical(y)
+
+      num_hidden = 5
+      model_k_v1 = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_k_v2 = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_k_v2.set_weights(model_k_v1.get_weights())
+
+      opt_k_v1 = optimizers.Adam(amsgrad=True)
+      opt_k_v2 = adam.Adam(amsgrad=True)
+
+      model_k_v1.compile(opt_k_v1, loss='categorical_crossentropy', metrics=[])
+      model_k_v2.compile(opt_k_v2, loss='categorical_crossentropy', metrics=[])
+
+      hist_k_v1 = model_k_v1.fit(x, y, batch_size=5, epochs=10, shuffle=False)
+      hist_k_v2 = model_k_v2.fit(x, y, batch_size=5, epochs=10, shuffle=False)
+
+      self.assertAllClose(model_k_v1.get_weights(), model_k_v2.get_weights())
+      self.assertAllClose(opt_k_v1.get_weights(), opt_k_v2.get_weights())
+      self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
+
 
 def disable_tf2():
   if 'TF2_BEHAVIOR' in os.environ:
-- 
GitLab


From bb09b3bae8c226ee3bc3f82f8431c0bfd4efe58b Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 4 Dec 2018 11:40:17 -0800
Subject: [PATCH 1298/1554] [tf.data] Disable optimizations and surface a
 warning if tf.data optimizations are used for an input pipeline containing a
 user-defined function with a captured ref variable.

PiperOrigin-RevId: 224017280
---
 .../kernel_tests/optimization/BUILD           |   7 +
 .../optimization/optimize_dataset_test.py     | 127 +++++++++++++++++-
 .../python/data/experimental/ops/batching.py  |   3 +
 .../python/data/experimental/ops/grouping.py  |  52 ++++---
 .../data/experimental/ops/prefetching_ops.py  |   3 +
 .../python/data/experimental/ops/scan_ops.py  |  11 +-
 tensorflow/python/data/ops/dataset_ops.py     |  61 ++++++++-
 7 files changed, 228 insertions(+), 36 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index e05f382171..f214944254 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -301,10 +301,17 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:random_ops",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/data/experimental/ops:batching",
+        "//tensorflow/python/data/experimental/ops:grouping",
         "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:optimization_options",
+        "//tensorflow/python/data/experimental/ops:scan_ops",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 2e31df716a..751be83326 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -17,25 +17,96 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import warnings
+
+from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.experimental.ops import grouping
 from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops import optimization_options
+from tensorflow.python.data.experimental.ops import scan_ops
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
+def _generate_captured_refvar_test_cases():
+  """Generates testcases.
+
+  Returns:
+    A list of tuples of (testcase_name, make_dataset_fn). make_dataset_fn takes
+    a tf.Variable as input and creates a test dataset that uses that variable.
+  """
+
+  def make_map_dataset(var):
+    return dataset_ops.Dataset.from_tensors(0).map(lambda x: x + var)
+
+  def make_flat_map_dataset(var):
+    return dataset_ops.Dataset.from_tensors(
+        0).flat_map(lambda _: dataset_ops.Dataset.from_tensors(var))
+
+  def make_filter_dataset(var):
+    return dataset_ops.Dataset.from_tensors(0).filter(lambda x: x < var)
+
+  def make_map_and_batch_dataset(var):
+
+    def map_fn(x):
+      return x + var
+
+    return dataset_ops.Dataset.from_tensors(0).apply(
+        batching.map_and_batch(map_fn, 1))
+
+  def make_group_by_reducer_dataset(var):
+    reducer = grouping.Reducer(
+        init_func=lambda _: 0,
+        reduce_func=lambda x, y: x,
+        finalize_func=lambda _: var)
+    return dataset_ops.Dataset.range(5).apply(
+        grouping.group_by_reducer(lambda x: x % 2, reducer))
+
+  def make_group_by_window_dataset(var):
+
+    def reduce_fn(key, bucket):
+      del key, bucket
+      return dataset_ops.Dataset.from_tensors(var)
+
+    return dataset_ops.Dataset.from_tensors(0).repeat(10).apply(
+        grouping.group_by_window(lambda _: 0, reduce_fn, 10))
+
+  def make_scan_dataset(var):
+    return dataset_ops.Dataset.from_tensors(0).apply(
+        scan_ops.scan(
+            0, lambda old_state, elem: (old_state + 1, elem + old_state + var)))
+
+  return [
+      # Core datasets
+      ("Map", make_map_dataset),
+      ("FlatMap", make_flat_map_dataset),
+      ("Filter", make_filter_dataset),
+      # Experimental datasets
+      ("MapAndBatch", make_map_and_batch_dataset),
+      ("GroupByReducer", make_group_by_reducer_dataset),
+      ("GroupByWindow", make_group_by_window_dataset),
+      ("Scan", make_scan_dataset)
+  ]
+
+
 @test_util.run_all_in_graph_and_eager_modes
-class OptimizeDatasetTest(test_base.DatasetTestBase):
+class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testOptimizationStatefulFunction(self):
-    dataset = dataset_ops.Dataset.range(10).map(
-        lambda _: random_ops.random_uniform([])).batch(10)
+    dataset = dataset_ops.Dataset.range(
+        10).map(lambda _: random_ops.random_uniform([])).batch(10)
     dataset = dataset_ops._OptimizeDataset(dataset, [])
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
@@ -121,13 +192,59 @@ class OptimizeDatasetTest(test_base.DatasetTestBase):
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationNonSerializableAsDirectInput(self):
-    """Tests that non-serializable dataset can be OptimizeDataset's input.
-    """
+    """Tests that non-serializable dataset can be OptimizeDataset's input."""
     dataset = dataset_ops.Dataset.from_tensors(0)
     dataset = dataset.apply(optimization.non_serializable())
     dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
     self.assertDatasetProduces(dataset, expected_output=[0])
 
+  @parameterized.named_parameters(_generate_captured_refvar_test_cases())
+  # Skip eager because RefVariables are not supported in eager mode.
+  def testSkipEagerOptimizationWithCapturedRefVar(self, dataset_fn):
+    """Tests that default optimizations are disabled with ref variables."""
+    variable = variable_scope.get_variable(
+        "v", initializer=0, use_resource=False)
+    assign_op = variable.assign_add(1)
+
+    unoptimized_dataset = dataset_fn(variable)
+
+    options = dataset_ops.Options()
+    opt_options = optimization_options.OptimizationOptions()
+    opt_options.noop_elimination = True
+    opt_options.map_and_batch_fusion = True
+    options.experimental_optimization = opt_options
+    optimized_dataset = unoptimized_dataset.with_options(options)
+
+    # Check that warning is logged.
+    warnings.simplefilter("always")
+    with warnings.catch_warnings(record=True) as w:
+      optimized_it = optimized_dataset.make_initializable_iterator()
+
+    self.assertGreaterEqual(len(w), 1)
+    expected = ("tf.data static optimizations are not compatible with "
+                "tf.Variable. The following optimizations will be disabled: "
+                "map_and_batch_fusion, noop_elimination. To enable "
+                "optimizations, use resource variables instead by calling "
+                "`tf.enable_resource_variables()` at the start of the program.")
+    self.assertTrue(any([expected in str(warning) for warning in w]))
+
+    # Check that outputs are the same in the optimized and unoptimized cases,
+    # when the variable value is changing.
+    unoptimized_it = unoptimized_dataset.make_initializable_iterator()
+    with ops.control_dependencies([assign_op]):
+      unoptimized_output = unoptimized_it.get_next()
+      optimized_output = optimized_it.get_next()
+
+    self.evaluate(variable.initializer)
+    self.evaluate((unoptimized_it.initializer, optimized_it.initializer))
+    while True:
+      try:
+        unoptimized, optimized = self.evaluate((unoptimized_output,
+                                                optimized_output))
+        self.assertEqual(unoptimized, optimized)
+      except errors.OutOfRangeError:
+        break
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 58eab0f794..668bf3e800 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -596,6 +596,9 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     else:
       self._output_structure = self._map_func.output_structure._batch(None)  # pylint: disable=protected-access
 
+  def _functions(self):
+    return [self._map_func]
+
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
     return ged_ops.experimental_map_and_batch_dataset(
diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index 5c68d91536..ad9fe9a4e8 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -262,7 +262,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
           "`key_func` must return a single tf.int64 tensor. "
           "Got type=%s and shape=%s"
           % (wrapped_func.output_types, wrapped_func.output_shapes))
-    self._key_func = wrapped_func.function
+    self._key_func = wrapped_func
 
   def _make_init_func(self, init_func):
     """Make wrapping defun for init_func."""
@@ -272,7 +272,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
         input_classes=ops.Tensor,
         input_shapes=tensor_shape.scalar(),
         input_types=dtypes.int64)
-    self._init_func = wrapped_func.function
+    self._init_func = wrapped_func
     self._state_classes = wrapped_func.output_classes
     self._state_shapes = wrapped_func.output_shapes
     self._state_types = wrapped_func.output_types
@@ -334,8 +334,8 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
         self._state_shapes = nest.pack_sequence_as(self._state_shapes,
                                                    weakened_state_shapes)
 
-    self._reduce_func = wrapped_func.function
-    self._reduce_func.add_to_graph(ops.get_default_graph())
+    self._reduce_func = wrapped_func
+    self._reduce_func.function.add_to_graph(ops.get_default_graph())
 
   def _make_finalize_func(self, finalize_func):
     """Make wrapping defun for finalize_func."""
@@ -345,7 +345,7 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
         input_classes=self._state_classes,
         input_shapes=self._state_shapes,
         input_types=self._state_types)
-    self._finalize_func = wrapped_func.function
+    self._finalize_func = wrapped_func
     self._output_classes = wrapped_func.output_classes
     self._output_shapes = wrapped_func.output_shapes
     self._output_types = wrapped_func.output_types
@@ -362,17 +362,22 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
   def output_types(self):
     return self._output_types
 
+  def _functions(self):
+    return [
+        self._key_func, self._init_func, self._reduce_func, self._finalize_func
+    ]
+
   def _as_variant_tensor(self):
     return ged_ops.experimental_group_by_reducer_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        self._key_func.captured_inputs,
-        self._init_func.captured_inputs,
-        self._reduce_func.captured_inputs,
-        self._finalize_func.captured_inputs,
-        key_func=self._key_func,
-        init_func=self._init_func,
-        reduce_func=self._reduce_func,
-        finalize_func=self._finalize_func,
+        self._key_func.function.captured_inputs,
+        self._init_func.function.captured_inputs,
+        self._reduce_func.function.captured_inputs,
+        self._finalize_func.function.captured_inputs,
+        key_func=self._key_func.function,
+        init_func=self._init_func.function,
+        reduce_func=self._reduce_func.function,
+        finalize_func=self._finalize_func.function,
         **dataset_ops.flat_structure(self))
 
   def _transformation_name(self):
@@ -408,7 +413,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
         wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
       raise ValueError(
           "`window_size_func` must return a single tf.int64 scalar tensor.")
-    self._window_size_func = wrapped_func.function
+    self._window_size_func = wrapped_func
 
   def _make_key_func(self, key_func, input_dataset):
     """Make wrapping defun for key_func."""
@@ -422,7 +427,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
         wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
       raise ValueError(
           "`key_func` must return a single tf.int64 scalar tensor.")
-    self._key_func = wrapped_func.function
+    self._key_func = wrapped_func
 
   def _make_reduce_func(self, reduce_func, input_dataset):
     """Make wrapping defun for reduce_func."""
@@ -444,7 +449,7 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
     self._output_classes = element_structure._to_legacy_output_classes()
     self._output_types = element_structure._to_legacy_output_types()
     self._output_shapes = element_structure._to_legacy_output_shapes()
-    self._reduce_func = wrapped_func.function
+    self._reduce_func = wrapped_func
 
   @property
   def output_classes(self):
@@ -458,15 +463,18 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
   def output_types(self):
     return self._output_types
 
+  def _functions(self):
+    return [self._key_func, self._reduce_func, self._window_size_func]
+
   def _as_variant_tensor(self):
     return ged_ops.experimental_group_by_window_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        self._key_func.captured_inputs,
-        self._reduce_func.captured_inputs,
-        self._window_size_func.captured_inputs,
-        key_func=self._key_func,
-        reduce_func=self._reduce_func,
-        window_size_func=self._window_size_func,
+        self._key_func.function.captured_inputs,
+        self._reduce_func.function.captured_inputs,
+        self._window_size_func.function.captured_inputs,
+        key_func=self._key_func.function,
+        reduce_func=self._reduce_func.function,
+        window_size_func=self._window_size_func.function,
         **dataset_ops.flat_structure(self))
 
   def _transformation_name(self):
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 2a5a7205e8..0894dd5f70 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -248,6 +248,9 @@ class _MapOnGpuDataset(dataset_ops.UnaryDataset):
         dataset=input_dataset,
         defun_kwargs={"experimental_ints_on_device": True})
 
+  def _functions(self):
+    return [self._map_func]
+
   def _as_variant_tensor(self):
     input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
     return ged_ops.experimental_map_dataset(
diff --git a/tensorflow/python/data/experimental/ops/scan_ops.py b/tensorflow/python/data/experimental/ops/scan_ops.py
index 0b7b2d142f..c768373cf4 100644
--- a/tensorflow/python/data/experimental/ops/scan_ops.py
+++ b/tensorflow/python/data/experimental/ops/scan_ops.py
@@ -125,16 +125,19 @@ class _ScanDataset(dataset_ops.UnaryDataset):
         self._state_shapes = nest.pack_sequence_as(self._state_shapes,
                                                    weakened_state_shapes)
 
-    self._scan_func = wrapped_func.function
-    self._scan_func.add_to_graph(ops.get_default_graph())
+    self._scan_func = wrapped_func
+    self._scan_func.function.add_to_graph(ops.get_default_graph())
+
+  def _functions(self):
+    return [self._scan_func]
 
   def _as_variant_tensor(self):
     input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
     return gen_experimental_dataset_ops.experimental_scan_dataset(
         input_t,
         nest.flatten(sparse.serialize_sparse_tensors(self._initial_state)),
-        self._scan_func.captured_inputs,
-        f=self._scan_func,
+        self._scan_func.function.captured_inputs,
+        f=self._scan_func.function,
         **dataset_ops.flat_structure(self))
 
   @property
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index f6a5772bfd..ba6d20373e 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -69,7 +69,6 @@ class DatasetV2(object):
 
   A `Dataset` can be used to represent an input pipeline as a
   collection of elements (nested structures of tensors) and a "logical
-
   plan" of transformations that act on those elements.
   """
 
@@ -97,6 +96,37 @@ class DatasetV2(object):
 
     raise NotImplementedError("Dataset._inputs")
 
+  def _has_captured_ref(self):
+    """Whether this dataset uses a function that captures ref variables.
+
+    Returns:
+      A boolean, which if true indicates that the dataset or one of its inputs
+      uses a function that captures ref variables.
+    """
+    if context.executing_eagerly():
+      # RefVariables are not supported in eager mode
+      return False
+
+    def is_tensor_or_parent_ref(tensor):
+      if tensor.dtype._is_ref_dtype:  # pylint: disable=protected-access
+        return True
+      return any([is_tensor_or_parent_ref(x) for x in tensor.op.inputs])
+
+    for fn in self._functions():
+      if any([is_tensor_or_parent_ref(t) for t in fn.function.captured_inputs]):
+        return True
+
+    return any(
+        [input_dataset._has_captured_ref() for input_dataset in self._inputs()])  # pylint: disable=protected-access
+
+  def _functions(self):
+    """Returns a list of functions associated with this dataset.
+
+    Returns:
+      A list of `StructuredFunctionWrapper` objects.
+    """
+    return []
+
   def options(self):
     """Returns the options for this dataset and its inputs.
 
@@ -125,7 +155,16 @@ class DatasetV2(object):
             dataset, t_options.max_intra_op_parallelism)
     static_optimizations = options._static_optimizations()  # pylint: disable=protected-access
     if static_optimizations:
-      dataset = _OptimizeDataset(dataset, static_optimizations)
+      if self._has_captured_ref():
+        warnings.warn(
+            "tf.data static optimizations are not compatible with tf.Variable. "
+            "The following optimizations will be disabled: %s. To enable "
+            "optimizations, use resource variables instead by calling "
+            "`tf.enable_resource_variables()` at the start of the program." %
+            ", ".join(static_optimizations))
+      else:
+        dataset = _OptimizeDataset(dataset, static_optimizations)
+
     if options.experimental_autotune is not False:
       dataset = _ModelDataset(dataset)
     if options.experimental_stats and options.experimental_stats.aggregator:  # pylint: disable=line-too-long
@@ -1550,6 +1589,9 @@ class DatasetV1Adapter(DatasetV1):
   def _as_variant_tensor(self):
     return self._dataset._as_variant_tensor()  # pylint: disable=protected-access
 
+  def _has_captured_ref(self):
+    return self._dataset._has_captured_ref()  # pylint: disable=protected-access
+
   def _inputs(self):
     return self._dataset._inputs()  # pylint: disable=protected-access
 
@@ -2756,6 +2798,9 @@ class MapDataset(UnaryDataset):
         use_inter_op_parallelism=self._use_inter_op_parallelism,
         **flat_structure(structure=self._map_func.output_structure))
 
+  def _functions(self):
+    return [self._map_func]
+
   @property
   def output_classes(self):
     return self._map_func.output_classes
@@ -2813,6 +2858,9 @@ class FlatMapDataset(UnaryDataset):
       raise TypeError("`map_func` must return a `Dataset` object.")
     self._output_structure = self._map_func.output_structure._element_structure  # pylint: disable=protected-access
 
+  def _functions(self):
+    return [self._map_func]
+
   def _as_variant_tensor(self):
     return gen_dataset_ops.flat_map_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
@@ -2906,13 +2954,16 @@ class FilterDataset(UnaryUnchangedStructureDataset):
         wrapped_func.output_types == dtypes.bool and
         wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
       raise ValueError("`predicate` must return a scalar boolean tensor.")
-    self._predicate = wrapped_func.function
+    self._predicate = wrapped_func
+
+  def _functions(self):
+    return [self._predicate]
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.filter_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        other_arguments=self._predicate.captured_inputs,
-        predicate=self._predicate,
+        other_arguments=self._predicate.function.captured_inputs,
+        predicate=self._predicate.function,
         **flat_structure(self))
 
   def _transformation_name(self):
-- 
GitLab


From ba1a2f44efa49f787a72bb9b109ecda00ae8bc51 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 11:44:40 -0800
Subject: [PATCH 1299/1554] Check both None and empty string. Sometime
 ${TPU_NAME} etc is not define, we receive "" as value instead of None.

PiperOrigin-RevId: 224018262
---
 .../tpu/profiler/pip_package/cloud_tpu_profiler/main.py     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
index 63641e00c5..a081c4354a 100644
--- a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+++ b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
@@ -90,12 +90,12 @@ def main(unused_argv=None):
   tf_version = tf.__version__
   print('TensorFlow version %s detected' % tf_version)
 
-  if FLAGS.service_addr is None and FLAGS.tpu is None:
+  if not FLAGS.service_addr and not FLAGS.tpu:
     sys.exit('You must specify either --service_addr or --tpu.')
 
   tpu_cluster_resolver = None
-  if FLAGS.service_addr is not None:
-    if FLAGS.tpu is not None:
+  if FLAGS.service_addr:
+    if FLAGS.tpu:
       tf.logging.warn('Both --service_addr and --tpu are set. Ignoring '
                       '--tpu and using --service_addr.')
     service_addr = FLAGS.service_addr
-- 
GitLab


From ac796837f8bcc6167f9c446bff8be541a2173c3e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 11:47:11 -0800
Subject: [PATCH 1300/1554] Add TPU compatible version for SavedModelPredictor.

PiperOrigin-RevId: 224018877
---
 tensorflow/contrib/predictor/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/predictor/BUILD b/tensorflow/contrib/predictor/BUILD
index d50b52b8ff..53a3bc63e1 100644
--- a/tensorflow/contrib/predictor/BUILD
+++ b/tensorflow/contrib/predictor/BUILD
@@ -42,6 +42,7 @@ py_library(
     name = "saved_model_predictor",
     srcs = ["saved_model_predictor.py"],
     srcs_version = "PY2AND3",
+    visibility = ["//learning/brain/contrib/learn/tpu:__subpackages__"],
     deps = [
         ":base_predictor",
         "//tensorflow/contrib/saved_model:saved_model_py",
-- 
GitLab


From 8a5afb574aad527e0d509b0c95dcdad9ed151bea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 12:01:22 -0800
Subject: [PATCH 1301/1554] Add rename of `tf.sparse_matmul` to
 `tf.linalg.matmul` to renames_v2.py script

PiperOrigin-RevId: 224021598
---
 tensorflow/python/ops/math_ops.py                    | 3 ++-
 tensorflow/tools/compatibility/renames_v2.py         | 7 -------
 tensorflow/tools/compatibility/tf_upgrade_v2.py      | 6 ++++++
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 7 +++++++
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index f576b00371..f0d8bed508 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2538,7 +2538,8 @@ def matvec(a,
 
 _OverrideBinaryOperatorHelper(matmul, "matmul")
 
-sparse_matmul = gen_math_ops.sparse_mat_mul
+sparse_matmul = deprecation.deprecated(None, "Use `tf.linalg.matmul` instead")(
+    gen_math_ops.sparse_mat_mul)
 tf_export(v1=["sparse_matmul"])(sparse_matmul)
 
 
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index b8d57d5e11..78839e36ba 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -265,11 +265,6 @@ renames = {
     'tf.lbeta': 'tf.math.lbeta',
     'tf.lgamma': 'tf.math.lgamma',
     'tf.lin_space': 'tf.linspace',
-    'tf.lite.constants.FLOAT': 'tf.compat.v1.lite.constants.FLOAT',
-    'tf.lite.constants.INT32': 'tf.compat.v1.lite.constants.INT32',
-    'tf.lite.constants.INT64': 'tf.compat.v1.lite.constants.INT64',
-    'tf.lite.constants.QUANTIZED_UINT8': 'tf.compat.v1.lite.constants.QUANTIZED_UINT8',
-    'tf.lite.constants.STRING': 'tf.compat.v1.lite.constants.STRING',
     'tf.local_variables': 'tf.compat.v1.local_variables',
     'tf.local_variables_initializer': 'tf.compat.v1.local_variables_initializer',
     'tf.log': 'tf.math.log',
@@ -382,7 +377,6 @@ renames = {
     'tf.nn.rnn_cell.GRUCell': 'tf.compat.v1.nn.rnn_cell.GRUCell',
     'tf.nn.rnn_cell.LSTMCell': 'tf.compat.v1.nn.rnn_cell.LSTMCell',
     'tf.nn.rnn_cell.MultiRNNCell': 'tf.compat.v1.nn.rnn_cell.MultiRNNCell',
-    'tf.nn.softmax_cross_entropy_with_logits_v2': 'tf.nn.softmax_cross_entropy_with_logits',
     'tf.nn.static_bidirectional_rnn': 'tf.compat.v1.nn.static_bidirectional_rnn',
     'tf.nn.static_rnn': 'tf.compat.v1.nn.static_rnn',
     'tf.nn.uniform_candidate_sampler': 'tf.random.uniform_candidate_sampler',
@@ -513,7 +507,6 @@ renames = {
     'tf.sparse.reduce_sum_sparse': 'tf.compat.v1.sparse.reduce_sum_sparse',
     'tf.sparse_fill_empty_rows': 'tf.sparse.fill_empty_rows',
     'tf.sparse_mask': 'tf.sparse.mask',
-    'tf.sparse_matmul': 'tf.compat.v1.sparse_matmul',
     'tf.sparse_maximum': 'tf.sparse.maximum',
     'tf.sparse_merge': 'tf.compat.v1.sparse_merge',
     'tf.sparse_minimum': 'tf.sparse.minimum',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index c37bab029c..655e680d5b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -417,6 +417,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.sparse.concat",
         "tf.sparse_split":
             "tf.sparse.split",
+        "tf.sparse_matmul":
+            "tf.linalg.matmul",
         "tf.random.stateless_multinomial":
             "tf.random.stateless_categorical",
         "tf.string_to_hash_bucket":
@@ -538,6 +540,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.segment_sum": [
             "data", "indices", "segment_ids", "name", "num_segments"
         ],
+        "tf.sparse_matmul": [
+            "a", "b", "transpose_a", "transpose_b", "a_is_sparse",
+            "b_is_sparse", "name"
+        ],
         "tf.io.decode_csv": [
             "records",
             "record_defaults",
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 1512f0bc36..b8b02c9c7f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -444,6 +444,13 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testSparseMatmul(self):
+    text = ("tf.sparse_matmul(a, b, c, d, e, f, g)\n")
+    expected_text = ("tf.linalg.matmul(a=a, b=b, transpose_a=c, transpose_b=d, "
+                     "a_is_sparse=e, b_is_sparse=f, name=g)\n")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 5170541e45ccc4fb511a17b6750baafe8e99582c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 12:13:08 -0800
Subject: [PATCH 1302/1554] Disables unit tests in
 tensorflow/python/kernel_tests/signal/spectral_ops_test that do not make
 sense in eager mode, when running in eager mode.

Fixes a bug in which signal rank is unknown.

PiperOrigin-RevId: 224023802
---
 .../signal/reconstruction_ops_test.py          | 18 +++++++++++++++---
 .../python/ops/signal/reconstruction_ops.py    |  2 +-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index 5f77308c2a..4cb6cedee9 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
@@ -66,6 +67,10 @@ class ReconstructionOpsTest(test.TestCase):
       self.assertAllClose(output, expected_output)
 
   def test_unknown_shapes(self):
+    # This test uses placeholders and does not work in eager mode.
+    if context.executing_eagerly():
+      return
+
     signal = array_ops.placeholder(dtype=dtypes.int32, shape=[None, None, None])
     frame_step = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step)
@@ -81,6 +86,10 @@ class ReconstructionOpsTest(test.TestCase):
       self.assertAllClose(output, expected_output)
 
   def test_unknown_rank(self):
+    # This test uses placeholders and does not work in eager mode.
+    if context.executing_eagerly():
+      return
+
     signal = array_ops.placeholder(dtype=dtypes.int32, shape=None)
     frame_step = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step)
@@ -96,15 +105,18 @@ class ReconstructionOpsTest(test.TestCase):
       self.assertAllClose(output, expected_output)
 
   def test_fast_path(self):
-    signal = array_ops.placeholder(dtype=dtypes.int32, shape=[3, 5])
+    # This test uses tensor names and does not work in eager mode.
+    if context.executing_eagerly():
+      return
+
+    signal = array_ops.ones([3, 5])
     frame_step = 5
     reconstruction = reconstruction_ops.overlap_and_add(signal, frame_step)
 
     self.assertEqual(reconstruction.name, "overlap_and_add/fast_path:0")
 
     with self.session(use_gpu=True) as sess:
-      output = sess.run(reconstruction,
-                        feed_dict={signal: np.ones([3, 5])})
+      output = self.evaluate(reconstruction)
 
       expected_output = np.ones([15])
 
diff --git a/tensorflow/python/ops/signal/reconstruction_ops.py b/tensorflow/python/ops/signal/reconstruction_ops.py
index 53c48d9040..4eaab4e0a0 100644
--- a/tensorflow/python/ops/signal/reconstruction_ops.py
+++ b/tensorflow/python/ops/signal/reconstruction_ops.py
@@ -79,7 +79,7 @@ def overlap_and_add(signal, frame_step, name=None):
     # If frame_length is equal to frame_step, there's no overlap so just
     # reshape the tensor.
     frame_step_static = tensor_util.constant_value(frame_step)
-    if (frame_step_static is not None and
+    if (frame_step_static is not None and signal.shape.dims is not None and
         frame_step_static == signal.shape.dims[-1].value):
       output_shape = full_shape([output_length])
       return array_ops.reshape(signal, output_shape, name="fast_path")
-- 
GitLab


From a984ed13a0e75ca9d914565c078552a48eac8bdf Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Tue, 4 Dec 2018 12:32:18 -0800
Subject: [PATCH 1303/1554] Add TF_GetInput

PiperOrigin-RevId: 224026687
---
 tensorflow/c/BUILD           |  2 ++
 tensorflow/c/kernels.cc      | 15 +++++++++++++++
 tensorflow/c/kernels.h       |  8 ++++++++
 tensorflow/c/kernels_test.cc | 22 ++++++++++++++++++++--
 4 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index f653e581bf..59c23e7c18 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -188,10 +188,12 @@ tf_cuda_library(
     deps = select({
         "//tensorflow:android": [
             ":c_api",
+            ":c_api_internal",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
             ":c_api",
+            ":c_api_internal",
             "//tensorflow/core:framework",
         ],
     }),
diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index e9a20e8150..3caa5bcb03 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <memory>
 
+#include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/kernels.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -126,3 +127,17 @@ int TF_NumOutputs(TF_OpKernelContext* ctx) {
   auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
   return cc_ctx->num_outputs();
 }
+
+void TF_GetInput(TF_OpKernelContext* ctx, int i, TF_Tensor** tensor,
+                 TF_Status* status) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  if (i < 0 || i >= cc_ctx->num_inputs()) {
+    TF_SetStatus(status, TF_OUT_OF_RANGE, "input index out of range");
+    return;
+  }
+  const ::tensorflow::Tensor& cc_tensor(cc_ctx->input(i));
+  TF_Tensor* result = ::tensorflow::TF_TensorFromTensor(cc_tensor, status);
+  if (TF_GetCode(status) == TF_OK) {
+    *tensor = result;
+  }
+}
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index a22030985f..d7778829bc 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -95,6 +95,14 @@ TF_CAPI_EXPORT extern int TF_NumInputs(TF_OpKernelContext* ctx);
 // kernel.
 TF_CAPI_EXPORT extern int TF_NumOutputs(TF_OpKernelContext* ctx);
 
+// Retrieves the ith input from ctx. If TF_GetCode(status) is TF_OK, *tensor is
+// populated and its ownership is passed to the caller. In any other case,
+// *tensor is not modified.
+//
+// If i < 0 or i >= TF_NumInputs(ctx), *status is set to TF_OUT_OF_RANGE.
+TF_CAPI_EXPORT extern void TF_GetInput(TF_OpKernelContext* ctx, int i,
+                                       TF_Tensor** tensor, TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index fdcf6518c5..80bf12c096 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/c/kernels.h"
 
+#include "tensorflow/c/c_api.h"
 #include "tensorflow/core/framework/kernel_def.pb.h"
 #include "tensorflow/core/framework/node_def.pb_text.h"
 #include "tensorflow/core/framework/op.h"
@@ -131,10 +132,25 @@ TEST(TestKernel, TestInputAndOutputCount) {
   static int num_outputs = 0;
 
   // A kernel whose Compute function has a side-effect of updating num_inputs
-  // and num_outputs.
+  // and num_outputs. Various functions on TF_OpKernelContext are also
+  // exercised.
   auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) {
     num_inputs = TF_NumInputs(ctx);
     num_outputs = TF_NumOutputs(ctx);
+
+    TF_Tensor* input = nullptr;
+    TF_Status* s = TF_NewStatus();
+    TF_GetInput(ctx, 0, &input, s);
+    EXPECT_EQ(TF_OK, TF_GetCode(s)) << "Failed to get input: " << TF_Message(s);
+    EXPECT_EQ(123, *static_cast<tensorflow::uint8*>(TF_TensorData(input)));
+    TF_GetInput(ctx, -1, &input, s);
+    EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
+    TF_GetInput(ctx, 3, &input, s);
+    EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
+    TF_DeleteStatus(s);
+    if (input != nullptr) {
+      TF_DeleteTensor(input);
+    }
   };
 
   TF_KernelBuilder* builder = TF_NewKernelBuilder(op_name, device_name, nullptr,
@@ -152,9 +168,11 @@ TEST(TestKernel, TestInputAndOutputCount) {
     DummyDevice dummy_device(nullptr, false);
     p.device = &dummy_device;
 
+    Tensor t(tensorflow::uint8(123));
+
     gtl::InlinedVector<TensorValue, 4> inputs;
     // Simulate 2 inputs
-    inputs.emplace_back();
+    inputs.emplace_back(&t);
     inputs.emplace_back();
     p.inputs = &inputs;
 
-- 
GitLab


From f7850198fadefde8f0556c160f44f7c129c5ad3e Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Tue, 4 Dec 2018 12:52:23 -0800
Subject: [PATCH 1304/1554] Add error message for missing params.

PiperOrigin-RevId: 224029781
---
 tensorflow/lite/python/convert.py      | 13 ++++++++--
 tensorflow/lite/python/convert_test.py | 36 ++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 0974bdf3d0..563312e027 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -301,7 +301,9 @@ def build_toco_convert_protos(input_tensors,
     process.
 
   Raises:
-    ValueError: If the input tensor type is unknown
+    ValueError:
+      If the input tensor type is unknown
+      Missing mean_values or std_dev_values
     RuntimeError: If TOCO fails to convert (in which case the runtime error's
       error text will contain the TOCO error log)
   """
@@ -339,6 +341,9 @@ def build_toco_convert_protos(input_tensors,
     input_array.data_type = convert_dtype_to_tflite_type(input_tensor.dtype)
 
     if toco.inference_input_type == _types_pb2.QUANTIZED_UINT8:
+      if not quantized_input_stats:
+        raise ValueError("std_dev and mean must be defined when "
+                         "inference_input_type is QUANTIZED_UINT8.")
       input_array.mean_value, input_array.std_value = quantized_input_stats[idx]
     if input_shapes is None:
       shape = input_tensor.get_shape()
@@ -387,7 +392,11 @@ def toco_convert_graph_def(input_data, input_arrays_with_shape, output_arrays,
 
   for idx, (name, shape) in enumerate(input_arrays_with_shape):
     input_array = model_flags.input_arrays.add()
-    if kwargs["inference_type"] == lite_constants.QUANTIZED_UINT8:
+    if toco_flags.inference_input_type == _types_pb2.QUANTIZED_UINT8:
+      if (("quantized_input_stats" not in kwargs) or
+          (not kwargs["quantized_input_stats"])):
+        raise ValueError("std_dev and mean must be defined when "
+                         "inference_input_type is QUANTIZED_UINT8.")
       input_array.mean_value, input_array.std_value = kwargs[
           "quantized_input_stats"][idx]
     input_array.name = name
diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index 40576e16db..2a6f1f634f 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -66,6 +66,21 @@ class ConvertTest(test_util.TensorFlowTestCase):
         quantized_input_stats=[(0., 1.)])
     self.assertTrue(tflite_model)
 
+  def testQuantizationInvalid(self):
+    in_tensor = array_ops.placeholder(
+        shape=[1, 16, 16, 3], dtype=dtypes.float32)
+    out_tensor = array_ops.fake_quant_with_min_max_args(
+        in_tensor + in_tensor, min=0., max=1.)
+    sess = session.Session()
+
+    with self.assertRaises(ValueError) as error:
+      convert.toco_convert(
+          sess.graph_def, [in_tensor], [out_tensor],
+          inference_type=lite_constants.QUANTIZED_UINT8)
+    self.assertEqual(
+        "std_dev and mean must be defined when inference_input_type is "
+        "QUANTIZED_UINT8.", str(error.exception))
+
   def testGraphDefBasic(self):
     in_tensor = array_ops.placeholder(
         shape=[1, 16, 16, 3], dtype=dtypes.float32, name="input")
@@ -139,6 +154,27 @@ class ConvertTest(test_util.TensorFlowTestCase):
     self.assertTrue(([1, 16, 16, 3] == output_details[0]["shape"]).all())
     self.assertTrue(output_details[0]["quantization"][0] > 0)  # scale
 
+  def testGraphDefQuantizationInvalid(self):
+    in_tensor_1 = array_ops.placeholder(
+        shape=[1, 16, 16, 3], dtype=dtypes.float32, name="inputA")
+    in_tensor_2 = array_ops.placeholder(
+        shape=[1, 16, 16, 3], dtype=dtypes.float32, name="inputB")
+    _ = array_ops.fake_quant_with_min_max_args(
+        in_tensor_1 + in_tensor_2, min=0., max=1., name="output")
+    sess = session.Session()
+
+    input_arrays_map = [("inputA", [1, 16, 16, 3]), ("inputB", [1, 16, 16, 3])]
+    output_arrays = ["output"]
+    with self.assertRaises(ValueError) as error:
+      convert.toco_convert_graph_def(
+          sess.graph_def,
+          input_arrays_map,
+          output_arrays,
+          inference_type=lite_constants.QUANTIZED_UINT8)
+    self.assertEqual(
+        "std_dev and mean must be defined when inference_input_type is "
+        "QUANTIZED_UINT8.", str(error.exception))
+
 
 class ConvertTestOpHint(test_util.TensorFlowTestCase):
   """Test the hint to stub functionality."""
-- 
GitLab


From 4bc86a6d72ba962a114c86ed0dcbc406433e98fc Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Tue, 4 Dec 2018 12:52:46 -0800
Subject: [PATCH 1305/1554] Adds some tests with validation data in fit API.

PiperOrigin-RevId: 224029877
---
 .../contrib/distribute/python/keras_test.py   | 121 +++++++++++++-----
 1 file changed, 87 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 2b2c840daa..e530ab6f17 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -212,7 +212,8 @@ def multi_input_output_model():
   return model
 
 
-def get_correctness_test_inputs(use_numpy, with_distribution,
+def get_correctness_test_inputs(use_numpy, use_validation_data,
+                                with_distribution,
                                 x_train, y_train, x_predict):
   """Generates the inputs for correctness check when enable Keras with DS."""
   global_batch_size = 64
@@ -233,11 +234,16 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
         'epochs': 1,
         'shuffle': False,
     }
-    eval_inputs = {
-        'batch_size': batch_size,
-        'x': x_train,
-        'y': y_train,
-    }
+
+    if use_validation_data:
+      eval_inputs = None
+      training_inputs['validation_data'] = (x_train, y_train)
+    else:
+      eval_inputs = {
+          'batch_size': batch_size,
+          'x': x_train,
+          'y': y_train,
+      }
     predict_inputs = {
         'x': np.array(x_predict, dtype=np.float32),
     }
@@ -256,12 +262,21 @@ def get_correctness_test_inputs(use_numpy, with_distribution,
         'shuffle': False,
         'steps_per_epoch': len(x_train) // global_batch_size,
     }
-    eval_inputs = {
-        'batch_size': None,
-        'x': x,
-        'y': None,
-        'steps': 20,
-    }
+    if use_validation_data:
+      eval_inputs = None  # Remove the eval_inputs
+      eval_dataset = dataset_ops.Dataset.from_tensor_slices(
+          (x_train, y_train))
+      x = batch_wrapper(eval_dataset, batch_size, with_distribution)
+      training_inputs['validation_data'] = x
+      training_inputs['validation_steps'] = 5
+    else:
+      eval_inputs = {
+          'batch_size': None,
+          'x': x,
+          'y': None,
+          'steps': 20,
+      }
+
     predict_batch_size = len(x_predict)
     if use_per_core_batch_size:
       predict_batch_size //= with_distribution.num_replicas_in_sync
@@ -320,11 +335,17 @@ def strategy_and_input_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=strategies_minus_tpu),
-          combinations.combine(mode=['graph'], use_numpy=[True, False])
-          + combinations.combine(mode=['eager'], use_numpy=[False])) +
+          combinations.combine(mode=['graph'],
+                               use_numpy=[True, False],
+                               use_validation_data=[True, False])
+          + combinations.combine(mode=['eager'],
+                                 use_numpy=[False],
+                                 use_validation_data=[False])) +
       combinations.times(
           combinations.combine(distribution=tpu_strategies),
-          combinations.combine(mode=['graph'], use_numpy=[True, False])))
+          combinations.combine(mode=['graph'],
+                               use_numpy=[True, False],
+                               use_validation_data=[True, False])))
 
 
 def strategy_for_numpy_input_combinations():
@@ -1242,7 +1263,8 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       self.assertEqual(history.history['binary_accuracy'], [1.0])
 
   @combinations.generate(strategy_and_input_combinations())
-  def test_correctness(self, distribution, use_numpy):
+  def test_correctness(self, distribution, use_numpy, use_validation_data):
+
     with self.cached_session():
       tolerance = 1e-5
 
@@ -1251,6 +1273,12 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         # TODO(b/119257215): use the default one once the flakyness is fixed.
         tolerance = 1e-4
 
+      if (use_validation_data and
+          not isinstance(distribution, tpu_strategy.TPUStrategy)):
+        # TODO(b/120435565): Enable tests with use_validation_data once the
+        # the underlying bug is fixed.
+        return
+
       keras.backend.set_image_data_format('channels_last')
       np.random.seed(_RANDOM_SEED)
       random_seed.set_random_seed(_RANDOM_SEED)
@@ -1270,14 +1298,20 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       # This is used to initialize the model for both the distribution and
       # non-distribution run. In addition, we add few non-linear layers to make
       # it non-trivial.
-      model = keras.Sequential()
-      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-      model.add(keras.layers.Dense(10, activation='relu'))
-      model.add(keras.layers.Dense(10, activation='relu'))
-      model.add(keras.layers.Dense(1))
+      def _create_model():
+        model = keras.Sequential()
+        model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+        model.add(keras.layers.Dense(10, activation='relu'))
+        model.add(keras.layers.Dense(10, activation='relu'))
+        model.add(keras.layers.Dense(1))
+        return model
+
+      model = _create_model()
       initial_weights = model.get_weights()
+      del model  # avoid accident usage.
 
       def fit_eval_and_predict(with_distribution=None):
+        model = _create_model()
         # We have initialized the model to the same weight for the distribution
         # and non-distribution run.
         model.set_weights(initial_weights)
@@ -1287,30 +1321,49 @@ class TestDistributionStrategyCorrectness(test.TestCase,
             distribute=with_distribution)
 
         training_inputs, eval_inputs, predict_inputs = (
-            get_correctness_test_inputs(use_numpy, with_distribution,
+            get_correctness_test_inputs(use_numpy, use_validation_data,
+                                        with_distribution,
                                         x_train, y_train, x_predict))
 
-        model.fit(**training_inputs)
-        eval_result = model.evaluate(**eval_inputs)
+        traning_history = model.fit(**training_inputs).history
+
+        if eval_inputs is not None:
+          eval_result = model.evaluate(**eval_inputs)
+        else:
+          # Creates a dummy identical eval_result to be compared later.
+          eval_result = 1.0
+
         weights = model.get_weights()
         predict_result = model.predict(**predict_inputs)
 
-        return weights, eval_result, predict_result
+        return weights, traning_history, eval_result, predict_result
+
+      wts_with_ds, history_with_ds, eval_with_ds, predict_with_ds = (
+          fit_eval_and_predict(with_distribution=distribution))
 
-      wts_with_ds, eval_with_ds, predict_with_ds = fit_eval_and_predict(
-          with_distribution=distribution)
-      wts_without_ds, eval_without_ds, predict_without_ds = (
-          fit_eval_and_predict(with_distribution=None))
+      (wts_without_ds, history_without_ds, eval_without_ds,
+       predict_without_ds) = fit_eval_and_predict(with_distribution=None)
 
-      # Verify that the weights, eval results, predict outputs  are the same
-      # within some limits of tolerance.
+      # Verify that the weights, training history, eval results, predict outputs
+      # are the same within some limits of tolerance.
       self.assertAllClose(
-          wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance)
+          wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance,
+          msg='Fail to assert weights after training.')
 
       self.assertAllClose(
-          eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance)
+          eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance,
+          msg='Fail to assert eval results.')
       self.assertAllClose(
-          predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance)
+          predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance,
+          msg='Fail to assert predict results.')
+
+      if not (isinstance(distribution, tpu_strategy.TPUStrategy)
+              and distribution.extended.steps_per_run > 1):
+        # TODO(b/119894254): Enable this test for all cases once the underlying
+        # bug is fixed.
+        self.assertAllClose(
+            history_with_ds, history_without_ds, atol=tolerance, rtol=tolerance,
+            msg='Fail to assert training history.')
 
 
 if __name__ == '__main__':
-- 
GitLab


From 98eb7d80204c9a232164d16d2dcb21799673d860 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 4 Dec 2018 12:56:38 -0800
Subject: [PATCH 1306/1554] [tf.data] Adding
 `tf.data.experimental.cardinality()` which provides information about dataset
 cardinality.

PiperOrigin-RevId: 224030418
---
 ...i_def_ExperimentalDatasetCardinality.pbtxt |  21 +++
 tensorflow/core/framework/dataset.h           |   7 +-
 .../core/kernels/data/batch_dataset_op.cc     |   9 +
 .../core/kernels/data/cache_dataset_ops.cc    |   4 +
 .../kernels/data/concatenate_dataset_op.cc    |  12 ++
 tensorflow/core/kernels/data/dataset_ops.cc   |  17 ++
 .../experimental/assert_next_dataset_op.cc    |   2 +
 .../dense_to_sparse_batch_dataset_op.cc       |   8 +
 .../experimental/ignore_errors_dataset_op.cc  |   2 +
 .../experimental/map_and_batch_dataset_op.cc  |  11 ++
 .../non_serializable_dataset_op.cc            |   2 +
 .../numa_map_and_batch_dataset_op.cc          |   9 +
 .../experimental/parse_example_dataset_op.cc  |   2 +
 .../data/experimental/random_dataset_op.cc    |   2 +
 .../data/experimental/scan_dataset_op.cc      |   4 +
 .../set_stats_aggregator_dataset_op.cc        |   2 +
 .../data/experimental/sleep_dataset_op.cc     |   2 +
 .../experimental/sliding_window_dataset_op.cc |   8 +
 .../data/experimental/stats_dataset_ops.cc    |   4 +
 .../experimental/threadpool_dataset_op.cc     |   6 +
 .../core/kernels/data/map_dataset_op.cc       |   4 +
 .../core/kernels/data/model_dataset_op.cc     |   2 +
 .../core/kernels/data/optimize_dataset_op.cc  |   2 +
 .../kernels/data/padded_batch_dataset_op.cc   |   9 +
 .../kernels/data/parallel_map_dataset_op.cc   |   4 +
 .../core/kernels/data/prefetch_dataset_op.cc  |   2 +
 .../core/kernels/data/range_dataset_op.cc     |   8 +
 .../core/kernels/data/repeat_dataset_op.cc    |  17 ++
 .../core/kernels/data/shuffle_dataset_op.cc   |   2 +
 .../core/kernels/data/skip_dataset_op.cc      |   8 +
 .../data/sparse_tensor_slice_dataset_op.cc    |   2 +
 .../core/kernels/data/take_dataset_op.cc      |  11 ++
 .../core/kernels/data/tensor_dataset_op.cc    |   2 +
 .../kernels/data/tensor_slice_dataset_op.cc   |   2 +
 .../core/kernels/data/window_dataset.cc       |   2 +
 .../core/kernels/data/window_dataset_op.cc    |   9 +
 .../core/kernels/data/zip_dataset_op.cc       |  15 ++
 .../core/ops/experimental_dataset_ops.cc      |   5 +
 .../python/data/experimental/__init__.py      |   8 +-
 .../data/experimental/kernel_tests/BUILD      |  24 +--
 .../kernel_tests/cardinality_test.py          | 158 ++++++++++++++++++
 .../has_indefinite_repeat_test.py             |  53 ------
 tensorflow/python/data/experimental/ops/BUILD |  22 +--
 .../data/experimental/ops/cardinality.py      |  50 ++++++
 .../experimental/ops/has_indefinite_repeat.py |  81 ---------
 .../v1/tensorflow.data.experimental.pbtxt     |  16 +-
 .../v2/tensorflow.data.experimental.pbtxt     |  16 +-
 47 files changed, 500 insertions(+), 168 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetCardinality.pbtxt
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/cardinality_test.py
 delete mode 100644 tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py
 create mode 100644 tensorflow/python/data/experimental/ops/cardinality.py
 delete mode 100644 tensorflow/python/data/experimental/ops/has_indefinite_repeat.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetCardinality.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetCardinality.pbtxt
new file mode 100644
index 0000000000..ac014bcc5e
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalDatasetCardinality.pbtxt
@@ -0,0 +1,21 @@
+op {
+  graph_op_name: "ExperimentalDatasetCardinality"
+  visibility: HIDDEN
+  in_arg {
+    name: "input_dataset"
+    description: <<END
+A variant tensor representing the dataset to return cardinality for.
+END
+  }
+  out_arg {
+    name: "cardinality"
+    description: <<END
+The cardinality of `input_dataset`. Named constants are used to represent
+infinite and unknown cardinality.
+END
+  }
+  summary: "Returns the cardinality of `input_dataset`."
+  description: <<END
+Returns the cardinality of `input_dataset`.
+END
+}
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index 7794157e0f..a00768fee0 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -53,6 +53,9 @@ namespace data {
 // A constant that can be used to enable auto-tuning.
 constexpr int kAutoTune = -1;
 
+constexpr int kInfiniteCardinality = -1;
+constexpr int kUnknownCardinality = -2;
+
 class DatasetBase;
 class SerializationContext;
 
@@ -587,6 +590,9 @@ class DatasetBase : public core::RefCounted {
   // A human-readable debug string for this dataset.
   virtual string DebugString() const = 0;
 
+  // Returns the cardinality of this dataset.
+  virtual int64 Cardinality() const { return kUnknownCardinality; }
+
   // Serializes the dataset and writes it to the `writer`.
   virtual Status Save(SerializationContext* ctx,
                       IteratorStateWriter* writer) const;
@@ -601,7 +607,6 @@ class DatasetBase : public core::RefCounted {
                            const DatasetBase* dataset, Node** output);
   };
 
-  // TODO(jsimsa): Consolidate overloading into a single method.
   virtual Status AsGraphDefInternal(SerializationContext* ctx,
                                     DatasetGraphDefBuilder* b,
                                     Node** node) const = 0;
diff --git a/tensorflow/core/kernels/data/batch_dataset_op.cc b/tensorflow/core/kernels/data/batch_dataset_op.cc
index 41b04346eb..1f8d2bdbae 100644
--- a/tensorflow/core/kernels/data/batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/batch_dataset_op.cc
@@ -95,6 +95,15 @@ class BatchDatasetOp : public UnaryDatasetOpKernel {
       return strings::StrCat("BatchDatasetOp(", batch_size_, ")::Dataset");
     }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / batch_size_ +
+             (n % batch_size_ == 0 || drop_remainder_ ? 0 : 1);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index ce6fd09aee..d7b231be18 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -84,6 +84,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
       return "CacheDatasetOp::FileDataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
@@ -588,6 +590,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
       return "CacheDatasetOp::MemoryDataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/concatenate_dataset_op.cc b/tensorflow/core/kernels/data/concatenate_dataset_op.cc
index d5a0abc64b..066b2c9aef 100644
--- a/tensorflow/core/kernels/data/concatenate_dataset_op.cc
+++ b/tensorflow/core/kernels/data/concatenate_dataset_op.cc
@@ -79,6 +79,18 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel {
       return "ConcatenateDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override {
+      int64 n1 = input_->Cardinality();
+      int64 n2 = to_concatenate_->Cardinality();
+      if (n1 == kInfiniteCardinality || n2 == kInfiniteCardinality) {
+        return kInfiniteCardinality;
+      }
+      if (n1 == kUnknownCardinality || n2 == kUnknownCardinality) {
+        return kUnknownCardinality;
+      }
+      return n1 + n2;
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/dataset_ops.cc b/tensorflow/core/kernels/data/dataset_ops.cc
index 36e9714736..0abfdbb56b 100644
--- a/tensorflow/core/kernels/data/dataset_ops.cc
+++ b/tensorflow/core/kernels/data/dataset_ops.cc
@@ -46,8 +46,25 @@ class DatasetToGraphOp : public OpKernel {
   }
 };
 
+class DatasetCardinalityOp : public OpKernel {
+ public:
+  explicit DatasetCardinalityOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    DatasetBase* dataset;
+    OP_REQUIRES_OK(ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset));
+    Tensor* result;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &result));
+    result->scalar<int64>()() = dataset->Cardinality();
+  }
+};
+
 REGISTER_KERNEL_BUILDER(Name("DatasetToGraph").Device(DEVICE_CPU),
                         DatasetToGraphOp);
 
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalDatasetCardinality").Device(DEVICE_CPU),
+    DatasetCardinalityOp);
+
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc b/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
index 3b5ee9b783..3e87f484b9 100644
--- a/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
@@ -76,6 +76,8 @@ class AssertNextDatasetOp : public UnaryDatasetOpKernel {
       return "AssertNextDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
index a9b3fb6551..97e64dd744 100644
--- a/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
@@ -114,6 +114,14 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
                              ")::Dataset");
     }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / batch_size_ + (n % batch_size_ == 0 ? 0 : 1);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
index 57cb44335b..d445d9c809 100644
--- a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
@@ -60,6 +60,8 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel {
       return "IgnoreErrorsDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index 203c977b4f..df96cccee0 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -195,6 +195,17 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       return "MapAndBatchDatasetOp::Dataset";
     }
 
+    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
+    // modified to preserve cardinality.
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / batch_size_ +
+             (n % batch_size_ == 0 || drop_remainder_ ? 0 : 1);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc b/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
index 953e086de3..61811ea14e 100644
--- a/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
@@ -75,6 +75,8 @@ class NonSerializableDatasetOp : public UnaryDatasetOpKernel {
       return errors::Unimplemented(DebugString(), "::AsGraphDefInternal");
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    private:
     class Iterator : public DatasetIterator<Dataset> {
      public:
diff --git a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
index 75eacb4b5b..705101262c 100644
--- a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
@@ -133,6 +133,15 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       return "NumaMapAndBatchDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / batch_size_ +
+             (n % batch_size_ == 0 || drop_remainder_ ? 0 : 1);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
index 4cec555ac1..40771a5cf9 100644
--- a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
@@ -202,6 +202,8 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
       return "ParseExampleDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/random_dataset_op.cc b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
index b849aa3690..6d85cd5c45 100644
--- a/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
@@ -76,6 +76,8 @@ class RandomDatasetOp : public DatasetOpKernel {
                              ")::Dataset");
     }
 
+    int64 Cardinality() const override { return kInfiniteCardinality; }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 31358bdefc..2eceab137b 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -93,6 +93,10 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "ScanDatasetOp::Dataset"; }
 
+    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
+    // modified to preserve cardinality.
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
index 80e10d1ea7..fe128005fa 100644
--- a/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
@@ -129,6 +129,8 @@ class SetStatsAggregatorDatasetOp : public UnaryDatasetOpKernel {
       return "SetStatsAggregatorDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
index c7bf89cbde..d2fb8ac4f3 100644
--- a/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
@@ -68,6 +68,8 @@ class SleepDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "SleepDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
index a6c02471e3..1ce4fbd313 100644
--- a/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
@@ -103,6 +103,14 @@ class SlidingWindowDatasetOp : public UnaryDatasetOpKernel {
                              window_shift_, ", ", window_stride_, ")::Dataset");
     }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / window_shift_;
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
index f0a0d23eb1..1961f25df8 100644
--- a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
@@ -78,6 +78,8 @@ class LatencyStatsDatasetOp : public UnaryDatasetOpKernel {
       return "LatencyStatsDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
@@ -186,6 +188,8 @@ class BytesProducedStatsDatasetOp : public UnaryDatasetOpKernel {
       return "BytesProducedStatsDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 287ca6abc6..8ae45ed5c9 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -169,6 +169,8 @@ class ThreadPoolDatasetOp : public UnaryDatasetOpKernel {
       return "ThreadPoolDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
@@ -274,6 +276,8 @@ class MaxIntraOpParallelismDatasetOp : public UnaryDatasetOpKernel {
       return "MaxIntraOpParallelismDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
@@ -383,6 +387,8 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
       return "PrivateThreadPoolDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index a4e61e02e2..e7ec02f5a0 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -128,6 +128,10 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "MapDatasetOp::Dataset"; }
 
+    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
+    // modified to preserve cardinality.
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/model_dataset_op.cc b/tensorflow/core/kernels/data/model_dataset_op.cc
index dcd2309596..069d61d80d 100644
--- a/tensorflow/core/kernels/data/model_dataset_op.cc
+++ b/tensorflow/core/kernels/data/model_dataset_op.cc
@@ -60,6 +60,8 @@ class ModelDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "ModelDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index 781150b069..ab184c232e 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -162,6 +162,8 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "OptimizeDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
index 594a9ce7ec..0fff4c5370 100644
--- a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
@@ -152,6 +152,15 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel {
                              ")::Dataset");
     }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / batch_size_ +
+             (n % batch_size_ == 0 || drop_remainder_ ? 0 : 1);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index 6fe582c9ae..7c589a648e 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -118,6 +118,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
       return "ParallelMapDatasetOp::Dataset";
     }
 
+    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
+    // modified to preserve cardinality.
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 9e518131eb..9b2a530be4 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -56,6 +56,8 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
 
   string DebugString() const override { return "PrefetchDatasetOp::Dataset"; }
 
+  int64 Cardinality() const override { return input_->Cardinality(); }
+
  protected:
   Status AsGraphDefInternal(SerializationContext* ctx,
                             DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/range_dataset_op.cc b/tensorflow/core/kernels/data/range_dataset_op.cc
index 207e957e37..580702f741 100644
--- a/tensorflow/core/kernels/data/range_dataset_op.cc
+++ b/tensorflow/core/kernels/data/range_dataset_op.cc
@@ -73,6 +73,14 @@ class RangeDatasetOp : public DatasetOpKernel {
                              step_, ")::Dataset");
     }
 
+    int64 Cardinality() const override {
+      if (step_ > 0) {
+        return std::max(0LL, (stop_ - start_ - 1) / step_ + 1);
+      } else {
+        return std::max(0LL, (start_ - stop_ - 1) / -step_ + 1);
+      }
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/repeat_dataset_op.cc b/tensorflow/core/kernels/data/repeat_dataset_op.cc
index cee14df69d..8100f2695b 100644
--- a/tensorflow/core/kernels/data/repeat_dataset_op.cc
+++ b/tensorflow/core/kernels/data/repeat_dataset_op.cc
@@ -71,6 +71,23 @@ class RepeatDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "RepeatDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (count_ < 0) {
+        if (n == 0) {
+          return 0;
+        }
+        return kInfiniteCardinality;
+      }
+      if (count_ == 0) {
+        return 0;
+      }
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return count_ * n;
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index ad6960685e..0ca347e98a 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -61,6 +61,8 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
       return input_->output_shapes();
     }
 
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
    protected:
     template <class T>
     class Iterator : public DatasetIterator<T> {
diff --git a/tensorflow/core/kernels/data/skip_dataset_op.cc b/tensorflow/core/kernels/data/skip_dataset_op.cc
index 8379383662..e321066a71 100644
--- a/tensorflow/core/kernels/data/skip_dataset_op.cc
+++ b/tensorflow/core/kernels/data/skip_dataset_op.cc
@@ -67,6 +67,14 @@ class SkipDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "SkipDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return std::max(0LL, n - count_);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
index a002c60535..be105f8170 100644
--- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
+++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
@@ -54,6 +54,8 @@ class Dataset : public DatasetBase {
     return "SparseTensorSliceDatasetOp::Dataset";
   }
 
+  int64 Cardinality() const override { return sparse_tensor_.shape()[0]; }
+
  protected:
   Status AsGraphDefInternal(SerializationContext* ctx,
                             DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/take_dataset_op.cc b/tensorflow/core/kernels/data/take_dataset_op.cc
index 57c9b0d57f..0a3d586953 100644
--- a/tensorflow/core/kernels/data/take_dataset_op.cc
+++ b/tensorflow/core/kernels/data/take_dataset_op.cc
@@ -68,6 +68,17 @@ class TakeDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "TakeDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kUnknownCardinality) {
+        return kUnknownCardinality;
+      }
+      if (n == kInfiniteCardinality) {
+        return count_;
+      }
+      return std::min(n, count_);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/tensor_dataset_op.cc b/tensorflow/core/kernels/data/tensor_dataset_op.cc
index c7d374f489..98c23f23b2 100644
--- a/tensorflow/core/kernels/data/tensor_dataset_op.cc
+++ b/tensorflow/core/kernels/data/tensor_dataset_op.cc
@@ -61,6 +61,8 @@ class TensorDatasetOp : public DatasetOpKernel {
 
     string DebugString() const override { return "TensorDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override { return 1LL; }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
index 6291bfc110..4ba2bde718 100644
--- a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
+++ b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
@@ -84,6 +84,8 @@ class TensorSliceDatasetOp : public DatasetOpKernel {
       return "TensorSliceDatasetOp::Dataset";
     }
 
+    int64 Cardinality() const override { return tensors_[0].dim_size(0); }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/window_dataset.cc b/tensorflow/core/kernels/data/window_dataset.cc
index 2ad4711aab..bed1c66ac7 100644
--- a/tensorflow/core/kernels/data/window_dataset.cc
+++ b/tensorflow/core/kernels/data/window_dataset.cc
@@ -43,6 +43,8 @@ class WindowDataset : public DatasetBase {
 
   string DebugString() const override { return "WindowDataset"; }
 
+  int64 Cardinality() const override { return elements_.size(); }
+
  protected:
   // TODO(b/110981596): Support checkpointing.
   Status AsGraphDefInternal(SerializationContext* ctx,
diff --git a/tensorflow/core/kernels/data/window_dataset_op.cc b/tensorflow/core/kernels/data/window_dataset_op.cc
index 2c68e1ee05..9d66162dfa 100644
--- a/tensorflow/core/kernels/data/window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/window_dataset_op.cc
@@ -98,6 +98,15 @@ class WindowDatasetOp : public UnaryDatasetOpKernel {
                              window_stride_, drop_remainder_, ")::Dataset");
     }
 
+    int64 Cardinality() const override {
+      int64 n = input_->Cardinality();
+      if (n == kInfiniteCardinality || n == kUnknownCardinality) {
+        return n;
+      }
+      return n / window_shift_ +
+             (n % window_shift_ == 0 || drop_remainder_ ? 0 : 1);
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/kernels/data/zip_dataset_op.cc b/tensorflow/core/kernels/data/zip_dataset_op.cc
index 6e94d77867..1760e63a9e 100644
--- a/tensorflow/core/kernels/data/zip_dataset_op.cc
+++ b/tensorflow/core/kernels/data/zip_dataset_op.cc
@@ -76,6 +76,21 @@ class ZipDatasetOp : public DatasetOpKernel {
 
     string DebugString() const override { return "ZipDatasetOp::Dataset"; }
 
+    int64 Cardinality() const override {
+      int64 result = kInfiniteCardinality;
+      for (const auto& input : inputs_) {
+        int64 n = input->Cardinality();
+        if (n == kUnknownCardinality) {
+          return kUnknownCardinality;
+        }
+        if (n != kInfiniteCardinality &&
+            (result == kInfiniteCardinality || n < result)) {
+          result = n;
+        }
+      }
+      return result;
+    }
+
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index dbd577400e..0711034118 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -71,6 +71,11 @@ REGISTER_OP("ExperimentalCSVDataset")
       return shape_inference::ScalarShape(c);
     });
 
+REGISTER_OP("ExperimentalDatasetCardinality")
+    .Input("input_dataset: variant")
+    .Output("cardinality: int64")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalDatasetToTFRecord")
     .Input("input_dataset: variant")
     .Input("filename: string")
diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index a72dd379e1..14dfec37cd 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -35,8 +35,8 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@TFRecordWriter
 @@ThreadingOptions
 
-@@assume_finite
 @@bucket_by_sequence_length
+@@cardinality
 @@choose_from_datasets
 @@copy_to_device
 @@dense_to_sparse_batch
@@ -63,6 +63,8 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@unique
 
 @@AUTOTUNE
+@@INFINITE_CARDINALITY
+@@UNKNOWN_CARDINALITY
 """
 
 from __future__ import absolute_import
@@ -74,6 +76,9 @@ from __future__ import print_function
 from tensorflow.python.data.experimental.ops.batching import dense_to_sparse_batch
 from tensorflow.python.data.experimental.ops.batching import map_and_batch
 from tensorflow.python.data.experimental.ops.batching import unbatch
+from tensorflow.python.data.experimental.ops.cardinality import cardinality
+from tensorflow.python.data.experimental.ops.cardinality import INFINITE as INFINITE_CARDINALITY
+from tensorflow.python.data.experimental.ops.cardinality import UNKNOWN as UNKNOWN_CARDINALITY
 from tensorflow.python.data.experimental.ops.counter import Counter
 from tensorflow.python.data.experimental.ops.enumerate_ops import enumerate_dataset
 from tensorflow.python.data.experimental.ops.error_ops import ignore_errors
@@ -83,7 +88,6 @@ from tensorflow.python.data.experimental.ops.grouping import bucket_by_sequence_
 from tensorflow.python.data.experimental.ops.grouping import group_by_reducer
 from tensorflow.python.data.experimental.ops.grouping import group_by_window
 from tensorflow.python.data.experimental.ops.grouping import Reducer
-from tensorflow.python.data.experimental.ops.has_indefinite_repeat import assume_finite
 from tensorflow.python.data.experimental.ops.interleave_ops import choose_from_datasets
 from tensorflow.python.data.experimental.ops.interleave_ops import parallel_interleave
 from tensorflow.python.data.experimental.ops.interleave_ops import sample_from_datasets
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index f2b2e2d0c1..c76e576b5b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -211,18 +211,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "has_indefinite_repeat_test",
-    srcs = ["has_indefinite_repeat_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/python/data/experimental/ops:has_indefinite_repeat",
-        "//tensorflow/python/data/kernel_tests:test_base",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "@absl_py//absl/testing:parameterized",
-    ],
-)
-
 py_test(
     name = "ignore_errors_test",
     srcs = ["ignore_errors_test.py"],
@@ -377,6 +365,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "cardinality_test",
+    srcs = ["cardinality_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python/data/experimental/ops:cardinality",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 py_test(
     name = "override_threadpool_test",
     size = "small",
diff --git a/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py b/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py
new file mode 100644
index 0000000000..943f0f1f81
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/cardinality_test.py
@@ -0,0 +1,158 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental.cardinality()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.data.experimental.ops import cardinality
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class NumElementsTest(test_base.DatasetTestBase, parameterized.TestCase):
+  """Tests for `tf.data.experimental.cardinality()`."""
+
+  @parameterized.named_parameters(
+      # pylint: disable=g-long-lambda
+      ("Batch1",
+       lambda: dataset_ops.Dataset.range(5).batch(2, drop_remainder=True), 2),
+      ("Batch2",
+       lambda: dataset_ops.Dataset.range(5).batch(2, drop_remainder=False), 3),
+      ("Batch3",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).batch(2),
+       cardinality.UNKNOWN),
+      ("Batch4", lambda: dataset_ops.Dataset.range(5).repeat().batch(2),
+       cardinality.INFINITE),
+      ("Cache1", lambda: dataset_ops.Dataset.range(5).cache(), 5),
+      ("Cache2", lambda: dataset_ops.Dataset.range(5).cache("foo"), 5),
+      ("Concatenate1", lambda: dataset_ops.Dataset.range(5).concatenate(
+          dataset_ops.Dataset.range(5)), 10),
+      ("Concatenate2",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate(
+           dataset_ops.Dataset.range(5)), cardinality.UNKNOWN),
+      ("Concatenate3", lambda: dataset_ops.Dataset.range(5).repeat().
+       concatenate(dataset_ops.Dataset.range(5)),
+       cardinality.INFINITE),
+      ("Concatenate4", lambda: dataset_ops.Dataset.range(5).concatenate(
+          dataset_ops.Dataset.range(5).filter(lambda _: True)),
+       cardinality.UNKNOWN),
+      ("Concatenate5",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate(
+           dataset_ops.Dataset.range(5).filter(lambda _: True)),
+       cardinality.UNKNOWN),
+      ("Concatenate6", lambda: dataset_ops.Dataset.range(5).repeat().
+       concatenate(dataset_ops.Dataset.range(5).filter(lambda _: True)),
+       cardinality.INFINITE),
+      ("Concatenate7", lambda: dataset_ops.Dataset.range(5).concatenate(
+          dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE),
+      ("Concatenate8",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).concatenate(
+           dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE),
+      ("Concatenate9",
+       lambda: dataset_ops.Dataset.range(5).repeat().concatenate(
+           dataset_ops.Dataset.range(5).repeat()), cardinality.INFINITE),
+      ("FlatMap", lambda: dataset_ops.Dataset.range(5).flat_map(
+          lambda _: dataset_ops.Dataset.from_tensors(0)),
+       cardinality.UNKNOWN),
+      ("Filter", lambda: dataset_ops.Dataset.range(5).filter(lambda _: True),
+       cardinality.UNKNOWN),
+      ("FromTensors1", lambda: dataset_ops.Dataset.from_tensors(0), 1),
+      ("FromTensors2", lambda: dataset_ops.Dataset.from_tensors((0, 1)), 1),
+      ("FromTensorSlices1",
+       lambda: dataset_ops.Dataset.from_tensor_slices([0, 0, 0]), 3),
+      ("FromTensorSlices2",
+       lambda: dataset_ops.Dataset.from_tensor_slices(([0, 0, 0], [1, 1, 1])),
+       3),
+      ("Interleave1", lambda: dataset_ops.Dataset.range(5).interleave(
+          lambda _: dataset_ops.Dataset.from_tensors(0), cycle_length=1),
+       cardinality.UNKNOWN),
+      ("Interleave2", lambda: dataset_ops.Dataset.range(5).interleave(
+          lambda _: dataset_ops.Dataset.from_tensors(0),
+          cycle_length=1,
+          num_parallel_calls=1), cardinality.UNKNOWN),
+      ("Map1", lambda: dataset_ops.Dataset.range(5).map(lambda x: x), 5),
+      ("Map2", lambda: dataset_ops.Dataset.range(5).map(
+          lambda x: x, num_parallel_calls=1), 5),
+      ("PaddedBatch1", lambda: dataset_ops.Dataset.range(5).padded_batch(
+          2, [], drop_remainder=True), 2),
+      ("PaddedBatch2", lambda: dataset_ops.Dataset.range(5).padded_batch(
+          2, [], drop_remainder=False), 3),
+      ("PaddedBatch3", lambda: dataset_ops.Dataset.range(5).filter(
+          lambda _: True).padded_batch(2, []), cardinality.UNKNOWN),
+      ("PaddedBatch4",
+       lambda: dataset_ops.Dataset.range(5).repeat().padded_batch(2, []),
+       cardinality.INFINITE),
+      ("Prefetch", lambda: dataset_ops.Dataset.range(5).prefetch(buffer_size=1),
+       5),
+      ("Range1", lambda: dataset_ops.Dataset.range(0), 0),
+      ("Range2", lambda: dataset_ops.Dataset.range(5), 5),
+      ("Range3", lambda: dataset_ops.Dataset.range(5, 10), 5),
+      ("Range4", lambda: dataset_ops.Dataset.range(10, 5), 0),
+      ("Range5", lambda: dataset_ops.Dataset.range(5, 10, 2), 3),
+      ("Range6", lambda: dataset_ops.Dataset.range(10, 5, -2), 3),
+      ("Repeat1", lambda: dataset_ops.Dataset.range(0).repeat(0), 0),
+      ("Repeat2", lambda: dataset_ops.Dataset.range(1).repeat(0), 0),
+      ("Repeat3", lambda: dataset_ops.Dataset.range(0).repeat(5), 0),
+      ("Repeat4", lambda: dataset_ops.Dataset.range(1).repeat(5), 5),
+      ("Repeat5", lambda: dataset_ops.Dataset.range(0).repeat(), 0),
+      ("Repeat6", lambda: dataset_ops.Dataset.range(1).repeat(),
+       cardinality.INFINITE),
+      ("Shuffle", lambda: dataset_ops.Dataset.range(5).shuffle(buffer_size=1),
+       5),
+      ("Skip1", lambda: dataset_ops.Dataset.range(5).skip(2), 3),
+      ("Skip2", lambda: dataset_ops.Dataset.range(5).skip(8), 0),
+      ("Skip3",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).skip(2),
+       cardinality.UNKNOWN),
+      ("Skip4", lambda: dataset_ops.Dataset.range(5).repeat().skip(2),
+       cardinality.INFINITE),
+      ("Take1", lambda: dataset_ops.Dataset.range(5).take(2), 2),
+      ("Take2", lambda: dataset_ops.Dataset.range(5).take(8), 5),
+      ("Take3",
+       lambda: dataset_ops.Dataset.range(5).filter(lambda _: True).take(2),
+       cardinality.UNKNOWN),
+      ("Take4", lambda: dataset_ops.Dataset.range(5).repeat().take(2), 2),
+      ("Window1", lambda: dataset_ops.Dataset.range(5).window(
+          size=2, shift=2, drop_remainder=True), 2),
+      ("Window2", lambda: dataset_ops.Dataset.range(5).window(
+          size=2, shift=2, drop_remainder=False), 3),
+      ("Zip1", lambda: dataset_ops.Dataset.zip(dataset_ops.Dataset.range(5)),
+       5),
+      ("Zip2", lambda: dataset_ops.Dataset.zip(
+          (dataset_ops.Dataset.range(5), dataset_ops.Dataset.range(3))), 3),
+      ("Zip3", lambda: dataset_ops.Dataset.zip(
+          (dataset_ops.Dataset.range(5),
+           dataset_ops.Dataset.range(3).repeat())), 5),
+      ("Zip4", lambda: dataset_ops.Dataset.zip(
+          (dataset_ops.Dataset.range(5).repeat(),
+           dataset_ops.Dataset.range(3).repeat())), cardinality.INFINITE),
+      ("Zip5", lambda: dataset_ops.Dataset.zip(
+          (dataset_ops.Dataset.range(5),
+           dataset_ops.Dataset.range(3).filter(lambda _: True))),
+       cardinality.UNKNOWN),
+      # pylint: enable=g-long-lambda
+  )
+  def testNumElements(self, dataset_fn, expected_result):
+    with self.cached_session() as sess:
+      self.assertEqual(
+          sess.run(cardinality.cardinality(dataset_fn())), expected_result)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py b/tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py
deleted file mode 100644
index 6b2569c317..0000000000
--- a/tensorflow/python/data/experimental/kernel_tests/has_indefinite_repeat_test.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for `tf.data.experimental.has_indefinite_repeat()`."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl.testing import parameterized
-
-from tensorflow.python.data.experimental.ops import has_indefinite_repeat
-from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.platform import test
-
-
-class HasIndefiniteRepeat(test_base.DatasetTestBase, parameterized.TestCase):
-  """Tests for `tf.data.experimental.has_indefinite_repeat()`."""
-
-  @parameterized.named_parameters(
-      ("NoRepeat", dataset_ops.Dataset.range(10), False),
-      ("FiniteRepeat", dataset_ops.Dataset.range(10).repeat(2), False),
-      ("FiniteRepeatNotAtEnd", dataset_ops.Dataset.range(10).repeat(2).skip(1),
-       False),
-      ("InfiniteRepeat", dataset_ops.Dataset.range(10).repeat(), True),
-      ("InfiniteRepeatNotAtEnd", dataset_ops.Dataset.range(10).repeat().skip(1),
-       True),
-      ("InfiniteRepeatThenFiniteRepeat",
-       dataset_ops.Dataset.range(10).repeat().repeat(2), True),
-      ("ConcatenateFiniteAndInfinite",
-       dataset_ops.Dataset.range(10).repeat(2).concatenate(
-           dataset_ops.Dataset.range(10).repeat()), True),
-      ("AssumeFinite", dataset_ops.Dataset.range(10).repeat().apply(
-          has_indefinite_repeat.assume_finite()), False),
-  )
-  def testHasIndefiniteRepeat(self, dataset, expected_result):
-    self.assertEqual(
-        has_indefinite_repeat.has_indefinite_repeat(dataset), expected_result)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index bc54109eed..43f43182f6 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -4,6 +4,16 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+py_library(
+    name = "cardinality",
+    srcs = ["cardinality.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:experimental_dataset_ops_gen",
+        "//tensorflow/python:tensor_util",
+    ],
+)
+
 py_library(
     name = "counter",
     srcs = ["counter.py"],
@@ -28,16 +38,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "has_indefinite_repeat",
-    srcs = ["has_indefinite_repeat.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/python:tensor_util",
-        "//tensorflow/python/data/ops:dataset_ops",
-    ],
-)
-
 py_library(
     name = "iterator_ops",
     srcs = [
@@ -433,13 +433,13 @@ py_library(
     name = "dataset_ops",
     deps = [
         ":batching",
+        ":cardinality",
         ":counter",
         ":enumerate_ops",
         ":error_ops",
         ":filter_for_shard_ops",
         ":get_single_element",
         ":grouping",
-        ":has_indefinite_repeat",
         ":indexed_dataset_ops",
         ":interleave_ops",
         ":map_defun",
diff --git a/tensorflow/python/data/experimental/ops/cardinality.py b/tensorflow/python/data/experimental/ops/cardinality.py
new file mode 100644
index 0000000000..9cf0a8801e
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/cardinality.py
@@ -0,0 +1,50 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Cardinality analysis of `Dataset` objects."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+INFINITE = -1
+UNKNOWN = -2
+tf_export("data.experimental.INFINITE_CARDINALITY").export_constant(
+    __name__, "INFINITE")
+tf_export("data.experimental.UNKNOWN_CARDINALITY").export_constant(
+    __name__, "UNKNOWN")
+
+
+@tf_export("data.experimental.cardinality")
+def cardinality(dataset):
+  """Returns the cardinality of `dataset`, if known.
+
+  The operation returns the cardinality of `dataset`. The operation may return
+  `tf.data.experimental.INFINITE_CARDINALITY` if `dataset` contains an infinite
+  number of elements or `tf.data.experimental.UNKNOWN_CARDINALITY` if the
+  analysis fails to determine the number of elements in `dataset` (e.g. when the
+  dataset source is a file).
+
+  Args:
+    dataset: A `tf.data.Dataset` for which to determine cardinality.
+
+  Returns:
+    A scalar `tf.int64` `Tensor` representing the cardinality of `dataset`. If
+    the cardinality is infinite or unknown, the operation returns the named
+    constant `INFINITE_CARDINALITY` and `UNKNOWN_CARDINALITY` respectively.
+  """
+  return ged_ops.experimental_dataset_cardinality(dataset._as_variant_tensor())  # pylint: disable=protected-access
diff --git a/tensorflow/python/data/experimental/ops/has_indefinite_repeat.py b/tensorflow/python/data/experimental/ops/has_indefinite_repeat.py
deleted file mode 100644
index cf5e3f811d..0000000000
--- a/tensorflow/python/data/experimental/ops/has_indefinite_repeat.py
+++ /dev/null
@@ -1,81 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Finiteness analysis of `Dataset` objects."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import tensor_util
-from tensorflow.python.util.tf_export import tf_export
-
-
-class _AssumeFiniteDataset(dataset_ops.UnaryUnchangedStructureDataset):
-
-  def __init__(self, input_dataset):
-    super(_AssumeFiniteDataset, self).__init__(input_dataset)
-    self._input_dataset = input_dataset
-
-  def _as_variant_tensor(self):
-    return self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
-
-
-@tf_export("data.experimental.assume_finite")
-def assume_finite():
-  """Assume that the input is finite, even if it contains `Dataset.repeat()`.
-
-  Training libraries may analyze a `tf.data.Dataset` to determine if it is
-  finite or infinite (e.g. because it contains an indefinite
-  `tf.data.Dataset.repeat` transformation). Since that analysis may be
-  imprecise, this transformation allows the user to annotate a dataset
-  explicitly as being finite.
-
-  Returns:
-    A `Dataset` transformation function, which can be passed to
-    `tf.data.Dataset.apply`.
-  """
-
-  def _apply_fn(dataset):
-    return _AssumeFiniteDataset(dataset)
-
-  return _apply_fn
-
-
-def has_indefinite_repeat(dataset):
-  """Returns `True` if `dataset` or any of its inputs is `Dataset.repeat()`.
-
-  NOTE: For simplicity, this analysis does not attempt to analyze nested
-  datasets (e.g. in a function passed to `tf.data.Dataset.flat_map`). If the
-  analysis is incorrect, you can apply `tf.data.experimental.assume_finite()`
-  to the dataset to override it.
-
-  Args:
-    dataset: A `tf.data.Dataset`.
-
-  Returns:
-    `True` if `dataset` or any of its inputs is repeated indefinitely.
-  """
-  # pylint: disable=protected-access
-  if isinstance(dataset, dataset_ops.DatasetV1Adapter):
-    return has_indefinite_repeat(dataset._dataset)
-  elif isinstance(dataset, dataset_ops.RepeatDataset):
-    count = tensor_util.constant_value(dataset._count)
-    return count == -1 or has_indefinite_repeat(dataset._inputs()[0])
-  elif isinstance(dataset, _AssumeFiniteDataset):
-    return False
-  else:
-    return any(
-        has_indefinite_repeat(input_dataset)
-        for input_dataset in dataset._inputs())
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index 27ea063b66..f981b1af17 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CsvDataset"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "INFINITE_CARDINALITY"
+    mtype: "<type \'int\'>"
+  }
   member {
     name: "OptimizationOptions"
     mtype: "<type \'type\'>"
@@ -48,18 +52,22 @@ tf_module {
     name: "ThreadingOptions"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "UNKNOWN_CARDINALITY"
+    mtype: "<type \'int\'>"
+  }
   member_method {
     name: "Counter"
     argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"<dtype: \'int64\'>\"], "
   }
-  member_method {
-    name: "assume_finite"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "bucket_by_sequence_length"
     argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
   }
+  member_method {
+    name: "cardinality"
+    argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "choose_from_datasets"
     argspec: "args=[\'datasets\', \'choice_dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index 27ea063b66..f981b1af17 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CsvDataset"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "INFINITE_CARDINALITY"
+    mtype: "<type \'int\'>"
+  }
   member {
     name: "OptimizationOptions"
     mtype: "<type \'type\'>"
@@ -48,18 +52,22 @@ tf_module {
     name: "ThreadingOptions"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "UNKNOWN_CARDINALITY"
+    mtype: "<type \'int\'>"
+  }
   member_method {
     name: "Counter"
     argspec: "args=[\'start\', \'step\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'1\', \"<dtype: \'int64\'>\"], "
   }
-  member_method {
-    name: "assume_finite"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "bucket_by_sequence_length"
     argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
   }
+  member_method {
+    name: "cardinality"
+    argspec: "args=[\'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "choose_from_datasets"
     argspec: "args=[\'datasets\', \'choice_dataset\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 6aeaed8f83459e0a0d4bc8f1082dada95d647443 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 13:10:28 -0800
Subject: [PATCH 1307/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 224032829
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 220 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 220 ++++++++++++++++++
 2 files changed, 440 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index e0ec389d28..8022c390c6 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -49500,6 +49500,86 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceApplyAdamWithAmsgrad"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "m"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "v"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "vhat"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "beta1_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceApplyAddSign"
   input_arg {
@@ -50715,6 +50795,69 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceApplyKerasMomentum"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceApplyMomentum"
   input_arg {
@@ -54621,6 +54764,83 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceSparseApplyKerasMomentum"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceSparseApplyMomentum"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 9328a3a1b8..a9c712e138 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -25248,6 +25248,86 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceApplyAdamWithAmsgrad"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "m"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "v"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "vhat"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "beta1_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceApplyAddSign"
   input_arg {
@@ -25572,6 +25652,69 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceApplyKerasMomentum"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceApplyMomentum"
   input_arg {
@@ -26843,6 +26986,83 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceSparseApplyKerasMomentum"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceSparseApplyMomentum"
   input_arg {
-- 
GitLab


From 1c2c0c9d09c1cc3ff7844942b93f17dba390e247 Mon Sep 17 00:00:00 2001
From: Sherry Yang <sherryy@google.com>
Date: Tue, 4 Dec 2018 13:21:52 -0800
Subject: [PATCH 1308/1554] Add optional dirname to dumpping graphs.

PiperOrigin-RevId: 224034659
---
 tensorflow/compiler/tf2xla/BUILD         |  7 +--
 tensorflow/compiler/tf2xla/dump_graph.cc | 74 +++---------------------
 tensorflow/core/util/dump_graph.cc       | 38 +++++++-----
 tensorflow/core/util/dump_graph.h        | 16 +++--
 4 files changed, 42 insertions(+), 93 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 25a84fb1b6..5a0d9b9af9 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -445,14 +445,9 @@ cc_library(
     ],
     deps = [
         "//tensorflow/compiler/jit:flags",
-        "//tensorflow/compiler/xla:parse_flags_from_env",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:framework_internal",
-        "//tensorflow/core:lib",
+        "//tensorflow/core:graph",
         "//tensorflow/core:protos_all_cc",
-        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/compiler/tf2xla/dump_graph.cc b/tensorflow/compiler/tf2xla/dump_graph.cc
index 1de85004a5..64fdbbebc6 100644
--- a/tensorflow/compiler/tf2xla/dump_graph.cc
+++ b/tensorflow/compiler/tf2xla/dump_graph.cc
@@ -18,86 +18,26 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 
-#include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/jit/flags.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/util/dump_graph.h"
 
 namespace tensorflow {
 namespace dump_graph {
 
-namespace {
-
-struct NameCounts {
-  mutex counts_mutex;
-  std::unordered_map<string, int> counts;
-};
-
-string MakeUniqueFilename(string name) {
-  static NameCounts& instance = *new NameCounts;
-
-  // Remove illegal characters from `name`.
-  for (int i = 0; i < name.size(); ++i) {
-    char ch = name[i];
-    if (ch == '/' || ch == '[' || ch == ']' || ch == '*' || ch == '?') {
-      name[i] = '_';
-    }
-  }
-
-  int count;
-  {
-    mutex_lock lock(instance.counts_mutex);
-    count = instance.counts[name]++;
-  }
-
-  string filename = name;
-  if (count > 0) {
-    absl::StrAppend(&filename, "_", count);
-  }
-  absl::StrAppend(&filename, ".pbtxt");
-  return filename;
-}
-
-string WriteTextProtoToUniqueFile(
-    Env* env, const string& name, const char* proto_type,
-    const ::tensorflow::protobuf::Message& proto) {
-  const string& dirname = GetDumpGraphFlags()->tf_dump_graph_prefix;
-  Status status = env->RecursivelyCreateDir(dirname);
-  if (!status.ok()) {
-    LOG(WARNING) << "Failed to create " << dirname << " for dumping "
-                 << proto_type << ": " << status;
-    return "(unavailable)";
-  }
-  string filepath = absl::StrCat(dirname, "/", MakeUniqueFilename(name));
-  status = WriteTextProto(Env::Default(), filepath, proto);
-  if (!status.ok()) {
-    LOG(WARNING) << "Failed to dump " << proto_type << " to file: " << filepath
-                 << " : " << status;
-    return "(unavailable)";
-  }
-  LOG(INFO) << "Dumped " << proto_type << " to " << filepath;
-  return filepath;
-}
-
-}  // anonymous namespace
-
 string DumpGraphDefToFile(const string& name, GraphDef const& graph_def) {
-  return WriteTextProtoToUniqueFile(Env::Default(), name, "GraphDef",
-                                    graph_def);
+  return tensorflow::DumpGraphDefToFile(
+      name, graph_def, GetDumpGraphFlags()->tf_dump_graph_prefix);
 }
 
 string DumpGraphToFile(const string& name, Graph const& graph,
                        const FunctionLibraryDefinition* flib_def) {
-  GraphDef graph_def;
-  graph.ToGraphDef(&graph_def);
-  if (flib_def) {
-    *graph_def.mutable_library() = flib_def->ToProto();
-  }
-  return DumpGraphDefToFile(name, graph_def);
+  return tensorflow::DumpGraphToFile(name, graph, flib_def,
+                                     GetDumpGraphFlags()->tf_dump_graph_prefix);
 }
 
 string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef) {
-  return WriteTextProtoToUniqueFile(Env::Default(), name, "FunctionDef", fdef);
+  return tensorflow::DumpFunctionDefToFile(
+      name, fdef, GetDumpGraphFlags()->tf_dump_graph_prefix);
 }
 
 }  // namespace dump_graph
diff --git a/tensorflow/core/util/dump_graph.cc b/tensorflow/core/util/dump_graph.cc
index 5df5cb51cb..523d37ecc2 100644
--- a/tensorflow/core/util/dump_graph.cc
+++ b/tensorflow/core/util/dump_graph.cc
@@ -75,18 +75,24 @@ Status WriteToFile(const string& filepath,
 
 template <class T>
 string WriteTextProtoToUniqueFile(Env* env, const string& name,
-                                  const char* proto_type, T& proto) {
-  const char* dirname = getenv("TF_DUMP_GRAPH_PREFIX");
-  if (!dirname) {
+                                  const char* proto_type, T& proto,
+                                  const string& dirname) {
+  const char* dir = nullptr;
+  if (!dirname.empty()) {
+    dir = dirname.c_str();
+  } else {
+    dir = getenv("TF_DUMP_GRAPH_PREFIX");
+  }
+  if (!dir) {
     return "(TF_DUMP_GRAPH_PREFIX not specified)";
   }
-  Status status = env->RecursivelyCreateDir(dirname);
+  Status status = env->RecursivelyCreateDir(dir);
   if (!status.ok()) {
-    LOG(WARNING) << "Failed to create " << dirname << " for dumping "
-                 << proto_type << ": " << status;
+    LOG(WARNING) << "Failed to create " << dir << " for dumping " << proto_type
+                 << ": " << status;
     return "(unavailable)";
   }
-  string filepath = absl::StrCat(dirname, "/", MakeUniqueFilename(name));
+  string filepath = absl::StrCat(dir, "/", MakeUniqueFilename(name));
   status = WriteToFile(filepath, proto);
   if (!status.ok()) {
     LOG(WARNING) << "Failed to dump " << proto_type << " to file: " << filepath
@@ -99,23 +105,27 @@ string WriteTextProtoToUniqueFile(Env* env, const string& name,
 
 }  // anonymous namespace
 
-string DumpGraphDefToFile(const string& name, GraphDef const& graph_def) {
-  return WriteTextProtoToUniqueFile(Env::Default(), name, "GraphDef",
-                                    graph_def);
+string DumpGraphDefToFile(const string& name, GraphDef const& graph_def,
+                          const string& dirname) {
+  return WriteTextProtoToUniqueFile(Env::Default(), name, "GraphDef", graph_def,
+                                    dirname);
 }
 
 string DumpGraphToFile(const string& name, Graph const& graph,
-                       const FunctionLibraryDefinition* flib_def) {
+                       const FunctionLibraryDefinition* flib_def,
+                       const string& dirname) {
   GraphDef graph_def;
   graph.ToGraphDef(&graph_def);
   if (flib_def) {
     *graph_def.mutable_library() = flib_def->ToProto();
   }
-  return DumpGraphDefToFile(name, graph_def);
+  return DumpGraphDefToFile(name, graph_def, dirname);
 }
 
-string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef) {
-  return WriteTextProtoToUniqueFile(Env::Default(), name, "FunctionDef", fdef);
+string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef,
+                             const string& dirname) {
+  return WriteTextProtoToUniqueFile(Env::Default(), name, "FunctionDef", fdef,
+                                    dirname);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/util/dump_graph.h b/tensorflow/core/util/dump_graph.h
index 05e0b79f55..03dc807a2b 100644
--- a/tensorflow/core/util/dump_graph.h
+++ b/tensorflow/core/util/dump_graph.h
@@ -29,19 +29,23 @@ namespace tensorflow {
 // chosen.
 //
 // Automatically picks a file name. Prefixes 'name' with the value of the
-// TF_DUMP_GRAPH_PREFIX environment variable and suffixes it with ".pbtxt" to
-// form a name. If a graph has already been dumped by this process with the same
-// name, suffixes with "_n.pbtxt", where 'n' is a sequence number.
-string DumpGraphDefToFile(const string& name, GraphDef const& graph_def);
+// TF_DUMP_GRAPH_PREFIX environment variable if 'dirname' is empty, and suffixes
+// 'name' with ".pbtxt" to form a name. If a graph has already been dumped by
+// this process with the same name, suffixes with "_n.pbtxt", where 'n' is a
+// sequence number.
+string DumpGraphDefToFile(const string& name, GraphDef const& graph_def,
+                          const string& dirname = "");
 
 // Similar to DumpGraphDefToFile, but builds the GraphDef to dump from a 'graph'
 // and an optional function library 'flib_def'. Returns the file name chosen.
 string DumpGraphToFile(const string& name, Graph const& graph,
-                       const FunctionLibraryDefinition* flib_def = nullptr);
+                       const FunctionLibraryDefinition* flib_def = nullptr,
+                       const string& dirname = "");
 
 // Similar to DumpGraphDefToFile, but dumps a function as a FunctionDef text
 // proto. Returns the file name chosen.
-string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef);
+string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef,
+                             const string& dirname = "");
 
 }  // namespace tensorflow
 
-- 
GitLab


From 2efc17d7f708b388c541390813df0b8222acf1de Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 4 Dec 2018 13:30:12 -0800
Subject: [PATCH 1309/1554] [XLA] [TF:XLA] Move utilities used by linear
 algebra routines into XLA client library.

Move slicing utilities into a new xla/client/lib/slicing.h.
Move MaybeConjugate into xla/client/lib/math.h.
Move *TransposeInMinorDims into xla/client/lib/matrix.h.\

CL in preparation for moving linear algebra routines into XLA client library.

PiperOrigin-RevId: 224036090
---
 .../tf2xla/kernels/batch_matmul_op.cc         |   1 +
 tensorflow/compiler/tf2xla/lib/BUILD          |  27 +---
 tensorflow/compiler/tf2xla/lib/cholesky.cc    |   1 +
 tensorflow/compiler/tf2xla/lib/qr.cc          |   1 +
 .../compiler/tf2xla/lib/triangular_solve.cc   |   2 +
 tensorflow/compiler/tf2xla/lib/util.cc        | 130 -----------------
 tensorflow/compiler/tf2xla/lib/util.h         |  38 -----
 tensorflow/compiler/xla/client/lib/BUILD      |  33 +++++
 tensorflow/compiler/xla/client/lib/math.cc    |  26 ++--
 tensorflow/compiler/xla/client/lib/math.h     |   4 +
 tensorflow/compiler/xla/client/lib/matrix.cc  |  16 +++
 tensorflow/compiler/xla/client/lib/matrix.h   |   8 ++
 .../compiler/xla/client/lib/matrix_test.cc    |  48 ++++++-
 tensorflow/compiler/xla/client/lib/slicing.cc | 134 ++++++++++++++++++
 tensorflow/compiler/xla/client/lib/slicing.h  |  48 +++++++
 .../client/lib/slicing_test.cc}               |  48 ++-----
 16 files changed, 320 insertions(+), 245 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/lib/slicing.cc
 create mode 100644 tensorflow/compiler/xla/client/lib/slicing.h
 rename tensorflow/compiler/{tf2xla/lib/util_test.cc => xla/client/lib/slicing_test.cc} (67%)

diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
index 0a68988de1..1b254e328a 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 
 namespace tensorflow {
diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD
index 34b95a112e..422781d536 100644
--- a/tensorflow/compiler/tf2xla/lib/BUILD
+++ b/tensorflow/compiler/tf2xla/lib/BUILD
@@ -49,6 +49,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client/lib:constants",
         "//tensorflow/compiler/xla/client/lib:matrix",
+        "//tensorflow/compiler/xla/client/lib:slicing",
         "//tensorflow/core:lib",
     ],
 )
@@ -85,6 +86,7 @@ cc_library(
         "//tensorflow/compiler/xla/client/lib:constants",
         "//tensorflow/compiler/xla/client/lib:math",
         "//tensorflow/compiler/xla/client/lib:matrix",
+        "//tensorflow/compiler/xla/client/lib:slicing",
         "//tensorflow/core:lib",
     ],
 )
@@ -124,7 +126,9 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/client/lib:constants",
+        "//tensorflow/compiler/xla/client/lib:math",
         "//tensorflow/compiler/xla/client/lib:matrix",
+        "//tensorflow/compiler/xla/client/lib:slicing",
         "//tensorflow/core:lib",
     ],
 )
@@ -171,29 +175,6 @@ cc_library(
     ],
 )
 
-xla_test(
-    name = "util_test",
-    srcs = ["util_test.cc"],
-    deps = [
-        ":util",
-        "//tensorflow/compiler/xla:array2d",
-        "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:test",
-        "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:global_data",
-        "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/client/lib:matrix",
-        "//tensorflow/compiler/xla/tests:client_library_test_base",
-        "//tensorflow/compiler/xla/tests:literal_test_util",
-        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-    ],
-)
-
 cc_library(
     name = "while_loop",
     srcs = ["while_loop.cc"],
diff --git a/tensorflow/compiler/tf2xla/lib/cholesky.cc b/tensorflow/compiler/tf2xla/lib/cholesky.cc
index d580e8b330..7ef8659992 100644
--- a/tensorflow/compiler/tf2xla/lib/cholesky.cc
+++ b/tensorflow/compiler/tf2xla/lib/cholesky.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/lib/while_loop.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/tf2xla/lib/qr.cc b/tensorflow/compiler/tf2xla/lib/qr.cc
index 1b374cf58c..d600774860 100644
--- a/tensorflow/compiler/tf2xla/lib/qr.cc
+++ b/tensorflow/compiler/tf2xla/lib/qr.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
index d2d4076401..192a61dca2 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
@@ -20,7 +20,9 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
diff --git a/tensorflow/compiler/tf2xla/lib/util.cc b/tensorflow/compiler/tf2xla/lib/util.cc
index 610d6882ba..c0bd172d17 100644
--- a/tensorflow/compiler/tf2xla/lib/util.cc
+++ b/tensorflow/compiler/tf2xla/lib/util.cc
@@ -113,36 +113,6 @@ xla::XlaOp IntegerLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type,
   return xla::ConstantLiteral(builder, literal);
 }
 
-xla::XlaOp SliceInMinorDims(xla::XlaOp x, absl::Span<const int64> start,
-                            absl::Span<const int64> end) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_RET_CHECK(start.size() == end.size());
-    int64 n_minor_dims = start.size();
-
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-
-    const int64 n_dims = xla::ShapeUtil::Rank(shape);
-    TF_RET_CHECK(n_minor_dims <= n_dims);
-    auto major_dims = xla::AsInt64Slice(shape.dimensions())
-                          .subspan(
-                              /*pos=*/0,
-                              /*len=*/n_dims - n_minor_dims);
-
-    // Prepends 0s in the major dim
-    std::vector<int64> padded_start(n_dims, 0);
-    std::copy(start.begin(), start.end(),
-              padded_start.begin() + major_dims.size());
-
-    // Prepends the shape of the major dims.
-    std::vector<int64> padded_end(n_dims);
-    std::copy(major_dims.begin(), major_dims.end(), padded_end.begin());
-    std::copy(end.begin(), end.end(), padded_end.begin() + major_dims.size());
-
-    std::vector<int64> strides(n_dims, 1);
-    return xla::Slice(x, padded_start, padded_end, strides);
-  });
-}
 
 std::vector<int64> ConcatVectors(absl::Span<const int64> xs,
                                  absl::Span<const int64> ys) {
@@ -152,104 +122,4 @@ std::vector<int64> ConcatVectors(absl::Span<const int64> xs,
   return output;
 }
 
-xla::XlaOp DynamicSliceInMinorDims(xla::XlaOp x,
-                                   absl::Span<const xla::XlaOp> starts,
-                                   absl::Span<const int64> sizes) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-    const int64 n_dims = xla::ShapeUtil::Rank(shape);
-    int64 n_minor_dims = starts.size();
-    TF_RET_CHECK(n_minor_dims == sizes.size());
-    TF_RET_CHECK(n_minor_dims <= n_dims);
-    auto major_dims = xla::AsInt64Slice(shape.dimensions())
-                          .subspan(
-                              /*pos=*/0,
-                              /*len=*/n_dims - sizes.size());
-    auto padded_starts = PrependZerosInMajorDims(x, starts);
-    auto padded_sizes = ConcatVectors(major_dims, sizes);
-    return xla::DynamicSlice(x, padded_starts, padded_sizes);
-  });
-}
-
-xla::XlaOp UpdateSlice(xla::XlaOp x, xla::XlaOp update,
-                       absl::Span<const int64> start) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    // TODO(phawkins): make int64 work on all backends, remove the int32 cast.
-    std::vector<int32> start_as_int32(start.begin(), start.end());
-    auto start_constant = xla::ConstantR1<int32>(builder, start_as_int32);
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-    const int64 n_dims = xla::ShapeUtil::Rank(shape);
-    TF_ASSIGN_OR_RETURN(xla::Shape start_constant_shape,
-                        builder->GetShape(start_constant));
-    const int64 start_length =
-        xla::ShapeUtil::GetDimension(start_constant_shape, -1);
-    TF_RET_CHECK(start_length == n_dims);
-    return xla::DynamicUpdateSlice(x, update, start_constant);
-  });
-}
-
-xla::XlaOp UpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update,
-                                  absl::Span<const int64> start) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-    const int64 n_dims = xla::ShapeUtil::Rank(shape);
-    const int64 n_minor_dims = start.size();
-    TF_RET_CHECK(n_minor_dims <= n_dims);
-    std::vector<int64> padded_start(n_dims, 0);
-    std::copy(start.begin(), start.end(),
-              padded_start.begin() + (n_dims - n_minor_dims));
-    return UpdateSlice(x, update, padded_start);
-  });
-}
-
-xla::XlaOp DynamicUpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update,
-                                         absl::Span<const xla::XlaOp> starts) {
-  auto padded_starts = PrependZerosInMajorDims(x, starts);
-  return xla::DynamicUpdateSlice(x, update, padded_starts);
-}
-
-xla::XlaOp PrependZerosInMajorDims(xla::XlaOp x,
-                                   absl::Span<const xla::XlaOp> starts) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-    const int64 n_dims = xla::ShapeUtil::Rank(shape);
-    auto zero = xla::Reshape(xla::ConstantR0<int32>(builder, 0), {1});
-    std::vector<xla::XlaOp> padded_starts(n_dims, zero);
-    for (int i = 0; i < starts.size(); ++i) {
-      padded_starts[n_dims - starts.size() + i] = xla::Reshape(starts[i], {1});
-    }
-    return xla::ConcatInDim(builder, padded_starts, 0);
-  });
-}
-
-xla::XlaOp TransposeInMinorDims(xla::XlaOp x) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-    const int64 n_dims = xla::ShapeUtil::Rank(shape);
-    TF_RET_CHECK(n_dims >= 2);
-    std::vector<int64> permutation(n_dims);
-    std::iota(permutation.begin(), permutation.end(), 0);
-    std::swap(permutation[n_dims - 1], permutation[n_dims - 2]);
-    return xla::Transpose(x, permutation);
-  });
-}
-
-xla::XlaOp MaybeTransposeInMinorDims(xla::XlaOp x, bool transpose) {
-  return transpose ? TransposeInMinorDims(x) : x;
-}
-
-xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate) {
-  xla::XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(x));
-    auto perform_conj = shape.element_type() == xla::C64 && conjugate;
-    return perform_conj ? xla::Conj(x) : x;
-  });
-}
-
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/lib/util.h b/tensorflow/compiler/tf2xla/lib/util.h
index d0980717c9..aec8061cb4 100644
--- a/tensorflow/compiler/tf2xla/lib/util.h
+++ b/tensorflow/compiler/tf2xla/lib/util.h
@@ -38,48 +38,10 @@ xla::XlaOp PrependZerosInMajorDims(xla::XlaOp x,
 xla::XlaOp IntegerLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type,
                           int64 value);
 
-// Builds a vector of zeros of length rank(x) with the last values being
-// those in `starts`.
-xla::XlaOp PrependZerosInMajorDims(xla::XlaOp x,
-                                   absl::Span<const xla::XlaOp> starts);
-
-// Performs a slice in the minor dimensions of a Tensor.
-xla::XlaOp SliceInMinorDims(xla::XlaOp x, absl::Span<const int64> start,
-                            absl::Span<const int64> end);
-
 // Returns the concatenation of `xs` and `ys`.
 std::vector<int64> ConcatVectors(absl::Span<const int64> xs,
                                  absl::Span<const int64> ys);
 
-// Performs a dynamic slice in the minor dimensions of a Tensor.
-xla::XlaOp DynamicSliceInMinorDims(xla::XlaOp x,
-                                   absl::Span<const xla::XlaOp> starts,
-                                   absl::Span<const int64> sizes);
-
-// Updates a slice of 'x', i.e.,
-// x[start[0], ..., start[n]] = update
-xla::XlaOp UpdateSlice(xla::XlaOp x, xla::XlaOp update,
-                       absl::Span<const int64> start);
-
-// Updates a slice of 'x', where 'start' contains a list of minor dimensions:
-// x[..., start[0], ..., start[n]] = update
-xla::XlaOp UpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update,
-                                  absl::Span<const int64> start);
-
-xla::XlaOp DynamicUpdateSliceInMinorDims(xla::XlaOp x, xla::XlaOp update,
-                                         absl::Span<const xla::XlaOp> starts);
-
-// Transposes a stack of matrices `x` by swapping the last two dimensions.
-xla::XlaOp TransposeInMinorDims(xla::XlaOp x);
-
-// Transposes `x` in its minor dimensions if `transpose` is true, otherwise
-// returns `x` unchanged.
-xla::XlaOp MaybeTransposeInMinorDims(xla::XlaOp x, bool transpose);
-
-// Applies a complex conjugation operation if `a` is complex and `conjugate_a`
-// is true, otherwise returns its argument.
-xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate);
-
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_COMPILER_TF2XLA_LIB_UTIL_H_
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index cc33293a63..f0f530d7d7 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -127,6 +127,7 @@ xla_test(
     tags = ["enable_for_xla_interpreter"],
     deps = [
         ":matrix",
+        ":slicing",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -176,6 +177,38 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "slicing",
+    srcs = ["slicing.cc"],
+    hdrs = ["slicing.h"],
+    deps = [
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+xla_test(
+    name = "slicing_test",
+    srcs = ["slicing_test.cc"],
+    tags = ["enable_for_xla_interpreter"],
+    deps = [
+        ":slicing",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+    ],
+)
+
 cc_library(
     name = "sorting",
     srcs = ["sorting.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 08a887a6e4..36fdda39b4 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -268,17 +268,16 @@ XlaOp Digamma(XlaOp input) {
 // Implements Banker's rounding: numbers that are equidistant between two
 // integers are rounded towards even.
 XlaOp RoundToEven(XlaOp x) {
-  auto half = xla::ScalarLike(x, 0.5);
-  auto one = xla::ScalarLike(x, 1.0);
-  auto two = xla::ScalarLike(x, 2.0);
+  auto half = ScalarLike(x, 0.5);
+  auto one = ScalarLike(x, 1.0);
+  auto two = ScalarLike(x, 2.0);
 
-  auto round_val = xla::Floor(x);
+  auto round_val = Floor(x);
   auto fraction = x - round_val;
-  auto nearest_even_int = round_val - two * xla::Floor(half * x);
-  auto is_odd = xla::Eq(nearest_even_int, one);
-  return xla::Select(xla::Or(xla::Gt(fraction, half),
-                             xla::And(xla::Eq(fraction, half), is_odd)),
-                     round_val + one, round_val);
+  auto nearest_even_int = round_val - two * Floor(half * x);
+  auto is_odd = Eq(nearest_even_int, one);
+  return Select(Or(Gt(fraction, half), And(Eq(fraction, half), is_odd)),
+                round_val + one, round_val);
 }
 
 // Trigonometric functions.
@@ -320,4 +319,13 @@ XlaOp Cosh(XlaOp x) { return (Exp(x) + Exp(-x)) * ScalarLike(x, 0.5); }
 
 XlaOp Sinh(XlaOp x) { return (Exp(x) - Exp(-x)) * ScalarLike(x, 0.5); }
 
+XlaOp MaybeConjugate(XlaOp x, bool conjugate) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    auto perform_conj = shape.element_type() == C64 && conjugate;
+    return perform_conj ? Conj(x) : x;
+  });
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h
index 3f06d04b9a..17612bf9fd 100644
--- a/tensorflow/compiler/xla/client/lib/math.h
+++ b/tensorflow/compiler/xla/client/lib/math.h
@@ -86,6 +86,10 @@ XlaOp Cosh(XlaOp x);
 // Computes the hyperbolic sine of 'x'.
 XlaOp Sinh(XlaOp x);
 
+// Applies a complex conjugation operation if `a` is complex and `conjugate`
+// is true, otherwise returns its argument.
+xla::XlaOp MaybeConjugate(xla::XlaOp x, bool conjugate);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_
diff --git a/tensorflow/compiler/xla/client/lib/matrix.cc b/tensorflow/compiler/xla/client/lib/matrix.cc
index 77d9ec5ece..ffd744d190 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix.cc
@@ -166,4 +166,20 @@ XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) {
   });
 }
 
+XlaOp TransposeInMinorDims(XlaOp x) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    TF_RET_CHECK(n_dims >= 2);
+    std::vector<int64> permutation(n_dims);
+    std::iota(permutation.begin(), permutation.end(), 0);
+    std::swap(permutation[n_dims - 1], permutation[n_dims - 2]);
+    return Transpose(x, permutation);
+  });
+}
+
+XlaOp MaybeTransposeInMinorDims(XlaOp x, bool transpose) {
+  return transpose ? TransposeInMinorDims(x) : x;
+}
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/matrix.h b/tensorflow/compiler/xla/client/lib/matrix.h
index 82e8514411..8856f99c7a 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.h
+++ b/tensorflow/compiler/xla/client/lib/matrix.h
@@ -60,6 +60,14 @@ XlaOp LowerTriangle(XlaOp x);
 xla::XlaOp BatchDot(
     xla::XlaOp x, xla::XlaOp y,
     xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
+
+// Transposes a stack of matrices `x` by swapping the last two dimensions.
+xla::XlaOp TransposeInMinorDims(xla::XlaOp x);
+
+// Transposes `x` in its minor dimensions if `transpose` is true, otherwise
+// returns `x` unchanged.
+xla::XlaOp MaybeTransposeInMinorDims(xla::XlaOp x, bool transpose);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
diff --git a/tensorflow/compiler/xla/client/lib/matrix_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc
index 8eacc583d1..0593a7517a 100644
--- a/tensorflow/compiler/xla/client/lib/matrix_test.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
@@ -25,13 +26,13 @@ limitations under the License.
 namespace xla {
 namespace {
 
-class NumericTest : public ClientLibraryTestBase {
+class MatrixTest : public ClientLibraryTestBase {
  protected:
   template <typename T>
   void TestMatrixDiagonal();
 };
 
-XLA_TEST_F(NumericTest, Triangle) {
+XLA_TEST_F(MatrixTest, Triangle) {
   XlaBuilder builder(TestName());
   Array3D<int32> input(2, 3, 4);
   input.FillIota(0);
@@ -46,7 +47,7 @@ XLA_TEST_F(NumericTest, Triangle) {
 }
 
 template <typename T>
-void NumericTest::TestMatrixDiagonal() {
+void MatrixTest::TestMatrixDiagonal() {
   XlaBuilder builder("GetMatrixDiagonal");
   Array3D<T> input(2, 3, 4);
   input.FillIota(0);
@@ -59,11 +60,46 @@ void NumericTest::TestMatrixDiagonal() {
   ComputeAndCompareR2<T>(&builder, expected, {a_data.get()});
 }
 
-XLA_TEST_F(NumericTest, GetMatrixDiagonal_S32) { TestMatrixDiagonal<int32>(); }
+XLA_TEST_F(MatrixTest, GetMatrixDiagonal_S32) { TestMatrixDiagonal<int32>(); }
+
+XLA_TEST_F(MatrixTest, GetMatrixDiagonal_S64) { TestMatrixDiagonal<int64>(); }
+
+XLA_TEST_F(MatrixTest, GetMatrixDiagonal_F32) { TestMatrixDiagonal<float>(); }
+
+Array3D<float> BatchedAValsFull() {
+  return {{
+              {2, 0, 1, 2},
+              {3, 6, 0, 1},
+              {4, 7, 9, 0},
+              {5, 8, 10, 11},
+          },
+          {
+              {16, 24, 8, 12},
+              {24, 61, 82, 48},
+              {8, 82, 456, 106},
+              {12, 48, 106, 62},
+          }};
+}
+
+XLA_TEST_F(MatrixTest, RowBatchDot) {
+  XlaBuilder builder(TestName());
 
-XLA_TEST_F(NumericTest, GetMatrixDiagonal_S64) { TestMatrixDiagonal<int64>(); }
+  int n = 4;
 
-XLA_TEST_F(NumericTest, GetMatrixDiagonal_F32) { TestMatrixDiagonal<float>(); }
+  XlaOp a, row, index;
+  auto a_data =
+      CreateR3Parameter<float>(BatchedAValsFull(), 0, "a", &builder, &a);
+  auto row_data = CreateR3Parameter<float>({{{9, 1, 0, 0}}, {{2, 4, 0, 0}}}, 1,
+                                           "row", &builder, &row);
+  // Select {{3, 6, 0, 1}, {24, 61,  82,  48}} out of BatchedAValsFull().
+  auto index_data = CreateR0Parameter<int>(1, 2, "index", &builder, &index);
 
+  auto l_index = DynamicSliceInMinorDims(
+      a, {index, ConstantR0<int32>(&builder, 0)}, {1, n});
+  BatchDot(l_index, TransposeInMinorDims(row));
+
+  ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
+                             {a_data.get(), row_data.get(), index_data.get()});
+}
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/slicing.cc b/tensorflow/compiler/xla/client/lib/slicing.cc
new file mode 100644
index 0000000000..f8c7df3ff5
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/slicing.cc
@@ -0,0 +1,134 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
+
+namespace xla {
+
+XlaOp SliceInMinorDims(XlaOp x, absl::Span<const int64> start,
+                       absl::Span<const int64> end) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_RET_CHECK(start.size() == end.size());
+    int64 n_minor_dims = start.size();
+
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    TF_RET_CHECK(n_minor_dims <= n_dims);
+    auto major_dims = AsInt64Slice(shape.dimensions())
+                          .subspan(
+                              /*pos=*/0,
+                              /*len=*/n_dims - n_minor_dims);
+
+    // Prepends 0s in the major dim
+    std::vector<int64> padded_start(n_dims, 0);
+    std::copy(start.begin(), start.end(),
+              padded_start.begin() + major_dims.size());
+
+    // Prepends the shape of the major dims.
+    std::vector<int64> padded_end(n_dims);
+    std::copy(major_dims.begin(), major_dims.end(), padded_end.begin());
+    std::copy(end.begin(), end.end(), padded_end.begin() + major_dims.size());
+
+    std::vector<int64> strides(n_dims, 1);
+    return Slice(x, padded_start, padded_end, strides);
+  });
+}
+
+XlaOp UpdateSlice(XlaOp x, XlaOp update, absl::Span<const int64> start) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    // TODO(phawkins): make int64 work on all backends, remove the int32 cast.
+    std::vector<int32> start_as_int32(start.begin(), start.end());
+    auto start_constant = ConstantR1<int32>(builder, start_as_int32);
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    TF_ASSIGN_OR_RETURN(Shape start_constant_shape,
+                        builder->GetShape(start_constant));
+    const int64 start_length =
+        ShapeUtil::GetDimension(start_constant_shape, -1);
+    TF_RET_CHECK(start_length == n_dims);
+    return DynamicUpdateSlice(x, update, start_constant);
+  });
+}
+
+XlaOp UpdateSliceInMinorDims(XlaOp x, XlaOp update,
+                             absl::Span<const int64> start) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    const int64 n_minor_dims = start.size();
+    TF_RET_CHECK(n_minor_dims <= n_dims);
+    std::vector<int64> padded_start(n_dims, 0);
+    std::copy(start.begin(), start.end(),
+              padded_start.begin() + (n_dims - n_minor_dims));
+    return UpdateSlice(x, update, padded_start);
+  });
+}
+
+namespace {
+
+std::vector<int64> ConcatVectors(absl::Span<const int64> xs,
+                                 absl::Span<const int64> ys) {
+  std::vector<int64> output(xs.size() + ys.size());
+  std::copy(xs.begin(), xs.end(), output.begin());
+  std::copy(ys.begin(), ys.end(), output.begin() + xs.size());
+  return output;
+}
+
+XlaOp PrependZerosInMajorDims(XlaOp x, absl::Span<const XlaOp> starts) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    auto zero = Reshape(ConstantR0<int32>(builder, 0), {1});
+    std::vector<XlaOp> padded_starts(n_dims, zero);
+    for (int i = 0; i < starts.size(); ++i) {
+      padded_starts[n_dims - starts.size() + i] = Reshape(starts[i], {1});
+    }
+    return ConcatInDim(builder, padded_starts, 0);
+  });
+}
+
+}  // namespace
+
+XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span<const XlaOp> starts,
+                              absl::Span<const int64> sizes) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    int64 n_minor_dims = starts.size();
+    TF_RET_CHECK(n_minor_dims == sizes.size());
+    TF_RET_CHECK(n_minor_dims <= n_dims);
+    auto major_dims = AsInt64Slice(shape.dimensions())
+                          .subspan(
+                              /*pos=*/0,
+                              /*len=*/n_dims - sizes.size());
+    auto padded_starts = PrependZerosInMajorDims(x, starts);
+    auto padded_sizes = ConcatVectors(major_dims, sizes);
+    return DynamicSlice(x, padded_starts, padded_sizes);
+  });
+}
+
+XlaOp DynamicUpdateSliceInMinorDims(XlaOp x, XlaOp update,
+                                    absl::Span<const XlaOp> starts) {
+  auto padded_starts = PrependZerosInMajorDims(x, starts);
+  return DynamicUpdateSlice(x, update, padded_starts);
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/slicing.h b/tensorflow/compiler/xla/client/lib/slicing.h
new file mode 100644
index 0000000000..6c482a38b5
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/slicing.h
@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/types/span.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/types.h"
+
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SLICING_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SLICING_H_
+
+namespace xla {
+
+// Updates a slice of 'x', i.e.,
+// x[start[0], ..., start[n]] = update
+XlaOp UpdateSlice(XlaOp x, XlaOp update, absl::Span<const int64> start);
+
+// Performs a slice in the minor dimensions of a tensor.
+// x[..., start[0]:end[0], ..., start[n]:end[n]]
+XlaOp SliceInMinorDims(XlaOp x, absl::Span<const int64> start,
+                       absl::Span<const int64> end);
+
+// Updates a slice of 'x', where 'start' contains a list of minor dimensions:
+// x[..., start[0]:..., ..., start[n]:...] = update
+XlaOp UpdateSliceInMinorDims(XlaOp x, XlaOp update,
+                             absl::Span<const int64> start);
+
+// Performs a dynamic slice in the minor dimensions of a tensor.
+XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span<const XlaOp> starts,
+                              absl::Span<const int64> sizes);
+
+XlaOp DynamicUpdateSliceInMinorDims(XlaOp x, XlaOp update,
+                                    absl::Span<const XlaOp> starts);
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_SLICING_H_
diff --git a/tensorflow/compiler/tf2xla/lib/util_test.cc b/tensorflow/compiler/xla/client/lib/slicing_test.cc
similarity index 67%
rename from tensorflow/compiler/tf2xla/lib/util_test.cc
rename to tensorflow/compiler/xla/client/lib/slicing_test.cc
index d86b01a2d5..8d362119e0 100644
--- a/tensorflow/compiler/tf2xla/lib/util_test.cc
+++ b/tensorflow/compiler/xla/client/lib/slicing_test.cc
@@ -13,28 +13,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/tf2xla/lib/util.h"
+#include "tensorflow/compiler/xla/client/lib/slicing.h"
 
-#include <memory>
-#include <numeric>
-#include <vector>
-
-#include "tensorflow/compiler/xla/array2d.h"
-#include "tensorflow/compiler/xla/client/lib/matrix.h"
-#include "tensorflow/compiler/xla/literal.h"
-#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
-#include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/core/lib/core/status_test_util.h"
 
-namespace tensorflow {
+namespace xla {
 namespace {
 
-using UtilTest = xla::ClientLibraryTestBase;
-using UtilLeftLookingTest = xla::ClientLibraryTestBase;
+using SlicingTest = xla::ClientLibraryTestBase;
 
 xla::Array2D<float> BValsRight() {
   return {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}};
@@ -63,7 +54,7 @@ xla::Array3D<float> BatchedAValsFull() {
           }};
 }
 
-XLA_TEST_F(UtilTest, Simple2dLookup) {
+XLA_TEST_F(SlicingTest, Simple2dLookup) {
   xla::XlaBuilder builder(TestName());
 
   xla::XlaOp a, x, y;
@@ -77,7 +68,7 @@ XLA_TEST_F(UtilTest, Simple2dLookup) {
                              xla::ErrorSpec(1e-2, 1e-2));
 }
 
-XLA_TEST_F(UtilTest, Simple3dLookup) {
+XLA_TEST_F(SlicingTest, Simple3dLookup) {
   xla::XlaBuilder builder(TestName());
 
   xla::XlaOp a, index;
@@ -92,7 +83,7 @@ XLA_TEST_F(UtilTest, Simple3dLookup) {
                              {a_data.get(), index_data.get()});
 }
 
-XLA_TEST_F(UtilTest, SimpleSliceUpdate) {
+XLA_TEST_F(SlicingTest, SimpleSliceUpdate) {
   xla::XlaBuilder builder(TestName());
 
   xla::XlaOp a, b, x, y;
@@ -111,26 +102,5 @@ XLA_TEST_F(UtilTest, SimpleSliceUpdate) {
       {a_data.get(), b_data.get(), x_data.get(), y_data.get()});
 }
 
-XLA_TEST_F(UtilTest, RowBatchDot) {
-  xla::XlaBuilder builder(TestName());
-
-  int n = 4;
-
-  xla::XlaOp a, row, index;
-  auto a_data =
-      CreateR3Parameter<float>(BatchedAValsFull(), 0, "a", &builder, &a);
-  auto row_data = CreateR3Parameter<float>({{{9, 1, 0, 0}}, {{2, 4, 0, 0}}}, 1,
-                                           "row", &builder, &row);
-  // Select {{3, 6, 0, 1}, {24, 61,  82,  48}} out of BatchedAValsFull().
-  auto index_data = CreateR0Parameter<int>(1, 2, "index", &builder, &index);
-
-  auto l_index = DynamicSliceInMinorDims(
-      a, {index, xla::ConstantR0<int32>(&builder, 0)}, {1, n});
-  BatchDot(l_index, TransposeInMinorDims(row));
-
-  ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
-                             {a_data.get(), row_data.get(), index_data.get()});
-}
-
 }  // namespace
-}  // namespace tensorflow
+}  // namespace xla
-- 
GitLab


From 1fa274b1e2220c2d4a712276d0ad1882a2120988 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Tue, 4 Dec 2018 13:40:23 -0800
Subject: [PATCH 1310/1554] Better error checking and testing in NcclManager.

After this change, we check the return value of every CUDA and NCCL call in
NcclManager.  If any call is unsuccessful, we call the NCCL callback with an
error status.

This change also re-enables NCCL tests.

PiperOrigin-RevId: 224038066
---
 tensorflow/core/nccl/BUILD                | 38 ++++++-------
 tensorflow/core/nccl/nccl_manager.cc      | 66 ++++++++++++++++-------
 tensorflow/core/nccl/nccl_manager.h       |  8 ++-
 tensorflow/core/nccl/nccl_manager_test.cc | 49 +++++++++--------
 4 files changed, 93 insertions(+), 68 deletions(-)

diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index 50d9a2e8da..4be33b2a0c 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -11,6 +11,10 @@ exports_files(["LICENSE"])
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_copts")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "tf_cuda_tests_tags",
+)
 
 cc_library(
     name = "nccl_lib",
@@ -34,27 +38,17 @@ cc_library(
 tf_cuda_cc_test(
     name = "nccl_manager_test",
     size = "medium",
-    srcs = if_cuda(
-        [
-            "nccl_manager_test.cc",
-        ],
-        [],
-    ),
-    # Disabled on jenkins until errors finding nvmlShutdown are found.
-    tags = [
-        "manual",
-        "multi_gpu",
-        "no_oss",
-        "noguitar",
-        "notap",
+    srcs = ["nccl_manager_test.cc"],
+    tags = tf_cuda_tests_tags() + [
+        "no_cuda_on_cpu_tap",  # TODO(b/120284216): re-enable multi_gpu
     ],
-    deps =
-        if_cuda([
-            ":nccl_lib",
-            "@local_config_nccl//:nccl",
-            "//tensorflow/core:cuda",
-            "//tensorflow/core:test",
-            "//tensorflow/core:test_main",
-            "//tensorflow/core:testlib",
-        ]),
+    deps = [
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ] + if_cuda([
+        ":nccl_lib",
+        "@local_config_nccl//:nccl",
+        "//tensorflow/core:cuda",
+    ]),
 )
diff --git a/tensorflow/core/nccl/nccl_manager.cc b/tensorflow/core/nccl/nccl_manager.cc
index f8e8c75222..df49bf1b97 100644
--- a/tensorflow/core/nccl/nccl_manager.cc
+++ b/tensorflow/core/nccl/nccl_manager.cc
@@ -24,6 +24,22 @@ limitations under the License.
 
 namespace tensorflow {
 
+#define NCCL_RETURN_IF_ERROR(...)                               \
+  do {                                                          \
+    ncclResult_t nccl_status = (__VA_ARGS__);                   \
+    if (nccl_status != ncclSuccess) {                           \
+      return errors::Internal(ncclGetErrorString(nccl_status)); \
+    }                                                           \
+  } while (0)
+
+#define CUDA_RETURN_IF_ERROR(...)                               \
+  do {                                                          \
+    cudaError_t cuda_status = (__VA_ARGS__);                    \
+    if (cuda_status != cudaSuccess) {                           \
+      return errors::Internal(cudaGetErrorString(cuda_status)); \
+    }                                                           \
+  } while (0)
+
 using se::cuda::ScopedActivateExecutorContext;
 
 // Contains data for a single stream used for nccl communication; this includes
@@ -177,8 +193,8 @@ NcclManager* NcclManager::instance() {
   return instance;
 }
 
-NcclManager::Communicator* NcclManager::GetCommunicator(
-    NcclManager::Collective* collective) {
+Status NcclManager::GetCommunicator(NcclManager::Collective* collective,
+                                    NcclManager::Communicator** communicator) {
   // Sort by executor to make ordering of executors deterministic.
   std::sort(collective->participants.begin(), collective->participants.end(),
             [](const std::unique_ptr<Participant>& a,
@@ -217,7 +233,10 @@ NcclManager::Communicator* NcclManager::GetCommunicator(
           break;
         }
       }
-      if (i == num_devices) return comm.get();
+      if (i == num_devices) {
+        *communicator = comm.get();
+        return Status::OK();
+      }
     }
   }
 
@@ -264,37 +283,36 @@ NcclManager::Communicator* NcclManager::GetCommunicator(
   // NCCL2 prevents InitAll for more communicators than devices (but doesn't
   // check that device ids are unique). Work around it by initializing each
   // rank individually.
-  cudaGetDeviceCount(&device_count);
+  CUDA_RETURN_IF_ERROR(cudaGetDeviceCount(&device_count));
 #endif
   std::vector<ncclComm_t> nccl_comms(num_devices);
   if (num_devices <= device_count) {
-    auto result =
-        ncclCommInitAll(nccl_comms.data(), num_devices, devices.data());
-    CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result);
+    NCCL_RETURN_IF_ERROR(
+        ncclCommInitAll(nccl_comms.data(), num_devices, devices.data()));
   } else {
     int savedDevice = 0;
-    CHECK_EQ(cudaGetDevice(&savedDevice), cudaSuccess);
+    CUDA_RETURN_IF_ERROR(cudaGetDevice(&savedDevice));
     ncclUniqueId commId;
-    ncclGetUniqueId(&commId);
+    NCCL_RETURN_IF_ERROR(ncclGetUniqueId(&commId));
 #if NCCL_MAJOR >= 2
-    CHECK_EQ(ncclGroupStart(), ncclSuccess);
+    NCCL_RETURN_IF_ERROR(ncclGroupStart());
 #endif
     for (int rank = 0; rank < num_devices; ++rank) {
-      cudaSetDevice(devices[rank]);
-      auto result =
-          ncclCommInitRank(nccl_comms.data() + rank, num_devices, commId, rank);
-      CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result);
+      CUDA_RETURN_IF_ERROR(cudaSetDevice(devices[rank]));
+      NCCL_RETURN_IF_ERROR(ncclCommInitRank(nccl_comms.data() + rank,
+                                            num_devices, commId, rank));
     }
 #if NCCL_MAJOR >= 2
-    CHECK_EQ(ncclGroupEnd(), ncclSuccess);
+    NCCL_RETURN_IF_ERROR(ncclGroupEnd());
 #endif
-    cudaSetDevice(savedDevice);
+    CUDA_RETURN_IF_ERROR(cudaSetDevice(savedDevice));
   }
   for (int rank = 0; rank < num_devices; ++rank) {
     members[rank].nccl_comm = nccl_comms[rank];
   }
   communicators_.emplace_back(new Communicator(std::move(members)));
-  return communicators_.back().get();
+  *communicator = communicators_.back().get();
+  return Status::OK();
 }
 
 void NcclManager::AddToAllReduce(int num_devices, const string& key,
@@ -400,10 +418,18 @@ void NcclManager::AddParticipant(int num_devices, const string& key,
 void NcclManager::RunCollective(const string& key, Collective* collective) {
   static mutex collective_mu(LINKER_INITIALIZED);
 
-  auto* communicator = GetCommunicator(collective);
-  collective->communicator = communicator;
-  const int size = communicator->num_devices;
+  Communicator* communicator = nullptr;
+  const int size = static_cast<int>(collective->participants.size());
+  Status s = GetCommunicator(collective, &communicator);
+  if (!s.ok()) {
+    for (int i = 0; i < size; ++i) {
+      collective->participants[i]->done_callback(s);
+    }
+    delete collective;
+    return;
+  }
 
+  collective->communicator = communicator;
   for (int rank = 0; rank < size; ++rank) {
     Participant* p = collective->participants[rank].get();
     NcclStream* nccl_stream = communicator->members[rank].nccl_stream;
diff --git a/tensorflow/core/nccl/nccl_manager.h b/tensorflow/core/nccl/nccl_manager.h
index 76b49101d4..5da4fe5554 100644
--- a/tensorflow/core/nccl/nccl_manager.h
+++ b/tensorflow/core/nccl/nccl_manager.h
@@ -103,7 +103,13 @@ class NcclManager {
   struct NcclStream;
   struct Participant;
 
-  Communicator* GetCommunicator(Collective* collective);
+  // Gets the `Communicator` object that will be used to enqueue NCCL kernels
+  // for `collective`, and returns it via `communicator`.
+  //
+  // This may involve creating CUDA streams and NCCL initialization.  If a NCCL
+  // or CUDA error occurs in the process, this returns an INTERNAL error with
+  // the corresponding NCCL/CUDA error string.
+  Status GetCommunicator(Collective* collective, Communicator** communicator);
 
   void AddParticipant(int num_devices, const string& key,
                       std::unique_ptr<Participant> participant,
diff --git a/tensorflow/core/nccl/nccl_manager_test.cc b/tensorflow/core/nccl/nccl_manager_test.cc
index dbc07865f0..f43103e120 100644
--- a/tensorflow/core/nccl/nccl_manager_test.cc
+++ b/tensorflow/core/nccl/nccl_manager_test.cc
@@ -28,8 +28,8 @@ limitations under the License.
 
 namespace tensorflow {
 
-static std::vector<BaseGPUDevice*> GetGPUDevices() {
-  std::vector<Device*> devices;
+static std::vector<std::unique_ptr<BaseGPUDevice>> GetGPUDevices() {
+  std::vector<std::unique_ptr<Device>> devices;
   SessionOptions session_options;
   session_options.config.mutable_gpu_options()
       ->set_per_process_gpu_memory_fraction(0.1);
@@ -37,12 +37,12 @@ static std::vector<BaseGPUDevice*> GetGPUDevices() {
   Status s = DeviceFactory::GetFactory(DEVICE_GPU)
                  ->AddDevices(session_options, "", &devices);
   TF_CHECK_OK(s);
-  std::vector<BaseGPUDevice*> gpus;
-  for (Device* d : devices) {
-    if (d->device_type() == "GPU") {
-      gpus.push_back(static_cast<BaseGPUDevice*>(d));
-    } else {
-      delete d;
+  std::vector<std::unique_ptr<BaseGPUDevice>> gpus;
+  for (std::unique_ptr<Device>& device : devices) {
+    if (device->device_type() == "GPU") {
+      // If `device_type()` is GPU, this `Device` is guaranteed to be a
+      // `BaseGPUDevice`, which is a subclass of `Device`.
+      gpus.emplace_back(static_cast<BaseGPUDevice*>(device.release()));
     }
   }
   return gpus;
@@ -64,16 +64,14 @@ class NcclManagerTest : public ::testing::Test {
   };
 
   static void SetUpTestCase() {
-    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
-    devices_ = new std::vector<BaseGPUDevice*>(GetGPUDevices());
-    CHECK(!devices_->empty());
+    setenv("NCCL_DEBUG", "WARN", 1 /* replace */);
+    devices_ = new std::vector<std::unique_ptr<BaseGPUDevice>>(GetGPUDevices());
     LOG(ERROR) << "Running test with " << devices_->size() << " gpus";
   }
 
-  static void TearDownTestCase() {
-    for (auto device : *devices_) delete device;
-    delete devices_;
-  }
+  static int32 NumGPUs() { return static_cast<int32>(devices_->size()); }
+
+  static void TearDownTestCase() { delete devices_; }
 
   TestCase* MakeTestCase(int num_ranks, ncclRedOp_t reduction_op,
                          TensorShape shape, float value_offset) {
@@ -153,7 +151,7 @@ class NcclManagerTest : public ::testing::Test {
       stream->ThenMemcpy(out_cpu.flat<Scalar>().data(), out_gpu_mem,
                          out_cpu.TotalBytes());
       SE_ASSERT_OK(stream->BlockHostUntilDone());
-      test::ExpectTensorNear<Scalar>(test_case->expected, out_cpu, 0.01);
+      test::ExpectClose(test_case->expected, out_cpu);
     }
   }
 
@@ -166,7 +164,7 @@ class NcclManagerTest : public ::testing::Test {
   }
 
   static BaseGPUDevice* GetDevice(size_t rank) {
-    return devices_->at(rank % devices_->size());
+    return devices_->at(rank % devices_->size()).get();
   }
 
  private:
@@ -181,13 +179,14 @@ class NcclManagerTest : public ::testing::Test {
   }
 
  private:
-  static std::vector<BaseGPUDevice*>* devices_;
+  static std::vector<std::unique_ptr<BaseGPUDevice>>* devices_;
   static const DataType data_type_;
   static const Scalar max_;
 };
 
 template <typename Scalar>
-std::vector<BaseGPUDevice*>* NcclManagerTest<Scalar>::devices_ = nullptr;
+std::vector<std::unique_ptr<BaseGPUDevice>>* NcclManagerTest<Scalar>::devices_ =
+    nullptr;
 template <typename Scalar>
 const DataType NcclManagerTest<Scalar>::data_type_ =
     DataTypeToEnum<Scalar>::value;
@@ -195,13 +194,13 @@ template <typename Scalar>
 const Scalar NcclManagerTest<Scalar>::max_ =
     Eigen::NumTraits<Scalar>::highest();
 
-// Instantiate tests for float and half.
-using TypeList = ::testing::Types<float, Eigen::half>;
+// Instantiate tests for float and double.
+using TypeList = ::testing::Types<float, double>;
 TYPED_TEST_CASE(NcclManagerTest, TypeList);
 
 // Test basic sum reduction.
 TYPED_TEST(NcclManagerTest, BasicSumReduction) {
-  const int num_ranks = 3;
+  const int num_ranks = this->NumGPUs();
 
   for (int op = 0; op < 4; ++op) {
     ncclRedOp_t reduction_op = static_cast<ncclRedOp_t>(op);
@@ -230,10 +229,10 @@ TYPED_TEST(NcclManagerTest, BasicSumReduction) {
 // To test the higher settings, increase num_ranks,
 // num_collectives_per_iteration and time_limit_micros.
 TYPED_TEST(NcclManagerTest, MultipleCallers) {
-  const int num_ranks = 1;                      // 2;
-  const int num_collectives_per_iteration = 1;  // 1000;
+  const int num_ranks = this->NumGPUs();
+  const int num_collectives_per_iteration = 10;  // 1000;
   const int num_threads = 3;
-  const int time_limit_micros = 1;  // 60 * 30 * 1000 * 1000;
+  const int time_limit_micros = 100;  // 60 * 30 * 1000 * 1000;
 
   int64 start = Env::Default()->NowMicros();
   srand(Env::Default()->NowMicros());
-- 
GitLab


From 0e2180feb7b1e3160e413c4533c129ba7f7679a8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 13:51:00 -0800
Subject: [PATCH 1311/1554] Second attempt at CL/223980636.

It looks like SUPPORT_SELECTIVE_REGISTRATION is used in both the :android_srcs_only_runtime and :android_srcs_no_runtime source files, i.e. in core/framework/types.h. This means that we have to have two separate non-runtime libraries, one with and one without selective registration.

To avoid future problems of this type, I've added a *_no_runtime version for each of the Android build rules.

BEGIN_PUBLIC
Internal change.
END_PUBLIC

*** Original change description ***

Automated rollback of commit e68e9432bf224987106123e08cc61b9686f256ce

PiperOrigin-RevId: 224040015
---
 tensorflow/core/BUILD | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index edf61ca1b9..575edfe7a9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1803,7 +1803,7 @@ cc_library(
 # registration of ops to prune code size.
 cc_library(
     name = "android_tensorflow_lib_selective_registration",
-    srcs = if_android(["//tensorflow/core:android_srcs"]),
+    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
     copts = tf_copts(android_optimization_level_override = None) + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
@@ -1815,10 +1815,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":protos_all_cc_impl",
-        "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_set",
-        "@double_conversion//:double-conversion",
-        "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
     ],
     alwayslink = 1,
@@ -1828,7 +1825,7 @@ cc_library(
 # no proto_rtti.
 cc_library(
     name = "android_tensorflow_lib_selective_registration_nortti",
-    srcs = if_android(["//tensorflow/core:android_srcs"]),
+    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
     copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
@@ -1840,10 +1837,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":protos_all_cc_impl",
-        "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_set",
-        "@double_conversion//:double-conversion",
-        "@nsync//:nsync_cpp",
         "@protobuf_archive//:protobuf",
     ],
     alwayslink = 1,
-- 
GitLab


From 6e8129c1dbcbee396da94e4ae1db2343946f98f8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 13:58:28 -0800
Subject: [PATCH 1312/1554] This CL adds decorators to the Keras testing
 utilities that allow systematically covering the Keras execution
 combinations. These decorators allow running test cases in sharded settings,
 and track the passing/failure of the individual parameterized test cases.
 This CL also begins adding these decorators to the Keras tests.

PiperOrigin-RevId: 224041323
---
 tensorflow/python/keras/BUILD                 |  56 +-
 .../feature_columns_integration_test.py       |  36 +-
 .../python/keras/engine/sequential_test.py    |  63 +-
 .../keras/engine/training_dataset_test.py     | 111 +--
 .../python/keras/engine/training_eager.py     |  28 +-
 .../python/keras/engine/training_test.py      | 724 +++++++++---------
 .../python/keras/keras_parameterized.py       | 348 +++++++++
 .../python/keras/keras_parameterized_test.py  | 381 +++++++++
 tensorflow/python/keras/testing_utils.py      | 406 ++++++++++
 9 files changed, 1682 insertions(+), 471 deletions(-)
 create mode 100644 tensorflow/python/keras/keras_parameterized.py
 create mode 100644 tensorflow/python/keras/keras_parameterized_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index ab5628b1d2..602cf79362 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -41,6 +41,7 @@ py_library(
         "datasets/mnist.py",
         "datasets/reuters.py",
         "estimator/__init__.py",
+        "keras_parameterized.py",
         "preprocessing/__init__.py",
         "preprocessing/image.py",
         "preprocessing/sequence.py",
@@ -214,6 +215,7 @@ py_test(
         "//tensorflow/python:layers",
         "//tensorflow/python:nn",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -226,6 +228,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -238,6 +241,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -251,6 +255,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:init_ops",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -262,6 +267,7 @@ py_test(
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -290,6 +296,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -332,6 +339,7 @@ py_test(
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -345,6 +353,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -358,6 +367,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -383,6 +393,7 @@ py_test(
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -396,6 +407,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -405,6 +417,7 @@ cuda_py_test(
     srcs = ["layers/embeddings_test.py"],
     additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client_testlib",
     ],
 )
@@ -420,6 +433,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -432,6 +446,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -443,6 +458,7 @@ py_test(
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -456,6 +472,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -469,6 +486,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -482,6 +500,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -512,6 +531,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -536,6 +556,7 @@ py_test(
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -553,6 +574,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -566,6 +588,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -585,6 +608,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -596,6 +620,7 @@ py_test(
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -623,6 +648,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -635,6 +661,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -643,6 +670,7 @@ cuda_py_test(
     srcs = ["utils/multi_gpu_utils_test.py"],
     additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
@@ -658,6 +686,7 @@ cuda_py_test(
     srcs = ["engine/training_gpu_test.py"],
     additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
@@ -685,6 +714,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -697,6 +727,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -709,6 +740,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -722,6 +754,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -729,13 +762,14 @@ py_test(
     name = "training_test",
     size = "medium",
     srcs = ["engine/training_test.py"],
-    shard_count = 4,
+    shard_count = 16,
     srcs_version = "PY2AND3",
     tags = ["notsan"],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -743,6 +777,7 @@ py_test(
     name = "training_dataset_test",
     size = "medium",
     srcs = ["engine/training_dataset_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     deps = [
         ":keras",
@@ -795,6 +830,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -808,6 +844,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -822,6 +859,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -837,6 +875,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -849,6 +888,7 @@ py_test(
         ":keras",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -890,6 +930,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -907,15 +948,16 @@ py_test(
     ],
 )
 
-py_library(
-    name = "testing_utils",
-    srcs = [
-        "testing_utils.py",
-    ],
+py_test(
+    name = "keras_parameterized_test",
+    size = "small",
+    srcs = ["keras_parameterized_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":keras",
-        "//tensorflow/python:util",
+        "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index b7549e013c..b3f8cfe725 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -18,15 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import context
 from tensorflow.python.feature_column import feature_column_lib as fc
-from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.training import rmsprop
 
@@ -44,12 +43,12 @@ class TestDNNModel(keras.models.Model):
     return net
 
 
-class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
+class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
   """Most Sequential model API tests are covered in `training_test.py`.
 
   """
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_model(self):
     columns = [fc.numeric_column('a')]
     model = keras.models.Sequential([
@@ -60,7 +59,8 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     model.compile(
         optimizer=rmsprop.RMSPropOptimizer(1e-3),
         loss='categorical_crossentropy',
-        metrics=['accuracy'])
+        metrics=['accuracy'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = {'a': np.random.random((10, 1))}
     y = np.random.randint(20, size=(10, 1))
@@ -70,7 +70,7 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     model.evaluate(x, y, batch_size=5)
     model.predict(x, batch_size=5)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_model_with_ds_input(self):
     columns = [fc.numeric_column('a')]
     model = keras.models.Sequential([
@@ -81,7 +81,8 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     model.compile(
         optimizer=rmsprop.RMSPropOptimizer(1e-3),
         loss='categorical_crossentropy',
-        metrics=['accuracy'])
+        metrics=['accuracy'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     y = np.random.randint(20, size=(100, 1))
     y = keras.utils.to_categorical(y, num_classes=20)
@@ -94,7 +95,7 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     model.evaluate(ds, steps=1)
     model.predict(ds, steps=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_subclassed_model_with_feature_columns(self):
     col_a = fc.numeric_column('a')
     col_b = fc.numeric_column('b')
@@ -104,7 +105,8 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     dnn_model.compile(
         optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
         loss='categorical_crossentropy',
-        metrics=['accuracy'])
+        metrics=['accuracy'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = {'a': np.random.random((10, 1)), 'b': np.random.random((10, 1))}
     y = np.random.randint(20, size=(10, 1))
@@ -114,10 +116,8 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     dnn_model.evaluate(x=x, y=y, batch_size=5)
     dnn_model.predict(x=x, batch_size=5)
 
-  @parameterized.parameters(True, False)
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_subclassed_model_with_feature_columns_with_ds_input(self,
-                                                               run_eagerly):
+  @keras_parameterized.run_all_keras_modes
+  def test_subclassed_model_with_feature_columns_with_ds_input(self):
     col_a = fc.numeric_column('a')
     col_b = fc.numeric_column('b')
 
@@ -127,7 +127,7 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
         optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
         loss='categorical_crossentropy',
         metrics=['accuracy'],
-        run_eagerly=run_eagerly and context.executing_eagerly())
+        run_eagerly=testing_utils.should_run_eagerly())
 
     y = np.random.randint(20, size=(100, 1))
     y = keras.utils.to_categorical(y, num_classes=20)
@@ -140,7 +140,8 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     dnn_model.evaluate(ds, steps=1)
     dnn_model.predict(ds, steps=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  # TODO(kaftan) seems to throw an error when enabled.
+  @keras_parameterized.run_all_keras_modes
   def DISABLED_test_function_model_feature_layer_input(self):
     col_a = fc.numeric_column('a')
     col_b = fc.numeric_column('b')
@@ -166,7 +167,8 @@ class FeatureColumnsIntegrationTest(test.TestCase, parameterized.TestCase):
     data = ({'a': np.arange(10), 'b': np.arange(10)}, np.arange(10, 20))
     print(model.fit(*data, epochs=1))
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  # TODO(kaftan) seems to throw an error when enabled.
+  @keras_parameterized.run_all_keras_modes
   def DISABLED_test_function_model_multiple_feature_layer_inputs(self):
     col_a = fc.numeric_column('a')
     col_b = fc.numeric_column('b')
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index b6d2510897..001c3d749b 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -26,17 +26,18 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training import rmsprop
 
 
-class TestSequential(test.TestCase, parameterized.TestCase):
+class TestSequential(keras_parameterized.TestCase):
   """Most Sequential model API tests are covered in `training_test.py`.
   """
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_basic_methods(self):
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(1, input_dim=2))
@@ -47,7 +48,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     self.assertEqual(len(model.weights), 2 * 2)
     self.assertEqual(model.get_layer(name='dp').name, 'dp')
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_pop(self):
     num_hidden = 5
     input_dim = 3
@@ -56,14 +57,16 @@ class TestSequential(test.TestCase, parameterized.TestCase):
 
     model = testing_utils.get_small_sequential_mlp(
         num_hidden, num_classes, input_dim)
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
+                  run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((batch_size, input_dim))
     y = np.random.random((batch_size, num_classes))
     model.fit(x, y, epochs=1)
     model.pop()
     self.assertEqual(len(model.layers), 1)
     self.assertEqual(model.output_shape, (None, num_hidden))
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
+                  run_eagerly=testing_utils.should_run_eagerly())
     y = np.random.random((batch_size, num_hidden))
     model.fit(x, y, epochs=1)
 
@@ -79,7 +82,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     with self.assertRaises(TypeError):
       model.pop()
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_deferred_build_with_np_arrays(self):
     num_hidden = 5
     input_dim = 3
@@ -90,7 +93,8 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     model.compile(
         loss='mse',
         optimizer=rmsprop.RMSPropOptimizer(1e-3),
-        metrics=[keras.metrics.CategoricalAccuracy()])
+        metrics=[keras.metrics.CategoricalAccuracy()],
+        run_eagerly=testing_utils.should_run_eagerly())
     self.assertEqual(len(model.layers), 2)
     self.assertEqual(len(model.weights), 0)
     self.assertFalse(model.built)
@@ -102,7 +106,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     self.assertFalse(model._is_graph_network)
     self.assertEqual(len(model.weights), 2 * 2)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_deferred_build_with_dataset_iterators(self):
     num_hidden = 5
     input_dim = 3
@@ -114,7 +118,8 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     model.compile(
         loss='mse',
         optimizer=rmsprop.RMSPropOptimizer(1e-3),
-        metrics=[keras.metrics.CategoricalAccuracy()])
+        metrics=[keras.metrics.CategoricalAccuracy()],
+        run_eagerly=testing_utils.should_run_eagerly())
     self.assertEqual(len(model.layers), 2)
     self.assertEqual(len(model.weights), 0)
     self.assertFalse(model.built)
@@ -131,6 +136,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     self.assertEqual(len(model.weights), 2 * 2)
     self.assertFalse(model._is_graph_network)
 
+  # TODO(kaftan) This test fails w/ run_with_all_keras_modes. File ticket
   @parameterized.parameters((True,), (False,))
   @tf_test_util.run_deprecated_v1
   def test_training_and_eval_methods_on_symbolic_tensors(self, deferred):
@@ -175,7 +181,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
           validation_data=(inputs, targets),
           validation_steps=2)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_invalid_use_cases(self):
     # Added objects must be layer instances
     with self.assertRaises(TypeError):
@@ -199,7 +205,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
       model.add(keras.layers.Dense(1, input_dim=1))
       model.add(MyLayer())
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_nested_sequential_trainability(self):
     input_dim = 20
     num_units = 10
@@ -249,7 +255,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
       x2 = model.predict(val_a)
       assert np.abs(np.sum(x1 - x2)) > 1e-5
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_deferred_build_serialization(self):
     num_hidden = 5
     input_dim = 3
@@ -260,7 +266,8 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     model.compile(
         loss='mse',
         optimizer=rmsprop.RMSPropOptimizer(1e-3),
-        metrics=[keras.metrics.CategoricalAccuracy()])
+        metrics=[keras.metrics.CategoricalAccuracy()],
+        run_eagerly=testing_utils.should_run_eagerly())
     self.assertFalse(model.built)
 
     x = np.random.random((batch_size, input_dim))
@@ -275,13 +282,13 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     self.assertEqual(len(new_model.layers), 2)
     self.assertEqual(len(new_model.weights), 4)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_shape_inference_deferred(self):
     model = testing_utils.get_small_sequential_mlp(4, 5)
     output_shape = model.compute_output_shape((None, 7))
     self.assertEqual(tuple(output_shape.as_list()), (None, 5))
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_build_deferred(self):
     model = testing_utils.get_small_sequential_mlp(4, 5)
 
@@ -298,18 +305,19 @@ class TestSequential(test.TestCase, parameterized.TestCase):
     self.assertTrue(model.built)
     self.assertEqual(len(model.weights), 8)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_nesting(self):
     model = testing_utils.get_small_sequential_mlp(4, 3)
     inner_model = testing_utils.get_small_sequential_mlp(4, 5)
     model.add(inner_model)
 
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
+                  run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((2, 6))
     y = np.random.random((2, 5))
     model.fit(x, y, epochs=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_variable_names(self):
     model = keras.models.Sequential([keras.layers.Dense(3)])
     model.add(keras.layers.Dense(2))
@@ -319,7 +327,7 @@ class TestSequential(test.TestCase, parameterized.TestCase):
          'sequential/dense_1/kernel:0', 'sequential/dense_1/bias:0'],
         [v.name for v in model.variables])
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_input_assumptions_propagation(self):
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(1))
@@ -329,9 +337,9 @@ class TestSequential(test.TestCase, parameterized.TestCase):
         model(1.0)
 
 
-class TestSequentialEagerIntegration(test.TestCase):
+class TestSequentialEagerIntegration(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_defun_on_call(self):
     # Check that one can subclass Sequential and place the `call` in a `defun`.
 
@@ -345,17 +353,19 @@ class TestSequentialEagerIntegration(test.TestCase):
     model.add(keras.layers.Dense(4, activation='relu'))
     model.add(keras.layers.Dense(5, activation='softmax'))
 
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.random.random((2, 6))
     y = np.random.random((2, 5))
     model.fit(x, y, epochs=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_build_before_fit(self):
     # Fix for b/112433577
     model = testing_utils.get_small_sequential_mlp(4, 5)
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     model.build((None, 6))
 
@@ -363,7 +373,7 @@ class TestSequentialEagerIntegration(test.TestCase):
     y = np.random.random((2, 5))
     model.fit(x, y, epochs=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sequential_model_fails_with_dict_inputs(self):
     num_classes = 5
     model = testing_utils.get_small_sequential_mlp(
@@ -372,7 +382,8 @@ class TestSequentialEagerIntegration(test.TestCase):
         rmsprop.RMSPropOptimizer(learning_rate=0.001),
         metrics=['acc'],
         weighted_metrics=['mae'],
-        loss='categorical_crossentropy')
+        loss='categorical_crossentropy',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = {'dense_input': np.random.random((10, 1))}
     y = np.random.randint(num_classes, size=(10, 1))
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 1b2d0e88e3..13e4abf050 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -20,14 +20,13 @@ from __future__ import print_function
 
 import logging
 
-from absl.testing import parameterized
-
 import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops.losses import losses_impl
@@ -36,22 +35,16 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
-class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
+class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
 
-  @parameterized.parameters(
-      {'model': 'functional'},
-      {'model': 'subclass'},
-  )
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_training_and_eval_methods_on_iterators_single_io(self, model):
-    if model == 'functional':
-      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    elif model == 'subclass':
-      model = testing_utils.get_small_sequential_mlp(1, 4)
+  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  def test_training_and_eval_methods_on_iterators_single_io(self):
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
@@ -104,13 +97,14 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
                                  'you should specify the `steps` argument'):
       model.predict(iterator, verbose=0)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes_with_all_model_types
   def test_get_next_op_created_once(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
@@ -125,13 +119,14 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
     ops.get_default_graph().finalize()
     model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes_with_all_model_types
   def test_iterators_running_out_of_data(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
@@ -147,15 +142,20 @@ class TestTrainingWithDatasetIterators(test.TestCase, parameterized.TestCase):
           'dataset iterator ran out of data')
 
 
-class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
+class TestTrainingWithDataset(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  # TODO(kaftan) Run w/ all model types.
+  # Seems like subclass models has a bug, file ticket
+  @keras_parameterized.run_all_keras_modes_with_all_model_types(
+      exclude_models='subclass'
+  )
   def test_calling_model_on_same_dataset(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
@@ -172,13 +172,14 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
               validation_data=dataset, validation_steps=2)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes_with_all_model_types
   def test_training_and_eval_methods_on_dataset(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
@@ -230,13 +231,18 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
                                  'you should specify the `steps` argument'):
       model.predict(dataset, verbose=0)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  # TODO(kaftan) Run w/ all model types.
+  # Seems like subclass models has a bug, file ticket
+  @keras_parameterized.run_all_keras_modes_with_all_model_types(
+      exclude_models='subclass'
+  )
   def test_dataset_with_sample_weights(self):
-    model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics)
+    model.compile(optimizer, loss, metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
@@ -250,21 +256,14 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
     model.evaluate(dataset, steps=2, verbose=1)
     model.predict(dataset, steps=2)
 
-  @parameterized.parameters(
-      {'model': 'functional'},
-      {'model': 'subclass'},
-  )
-  @tf_test_util.run_in_graph_and_eager_modes
-  def test_dataset_with_sparse_labels(self, model):
-    if model == 'functional':
-      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-    elif model == 'subclass':
-      model = testing_utils.get_small_sequential_mlp(1, 4)
-
+  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  def test_dataset_with_sparse_labels(self):
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
     for loss in ['sparse_categorical_crossentropy',
                  losses_impl.sparse_softmax_cross_entropy]:
-      optimizer = RMSPropOptimizer(learning_rate=0.001)
-      model.compile(optimizer, loss)
+      model.compile(optimizer, loss,
+                    run_eagerly=testing_utils.should_run_eagerly())
 
       inputs = np.zeros((10, 3), dtype=np.float32)
       targets = np.random.randint(0, 4, size=10, dtype=np.int32)
@@ -304,21 +303,27 @@ class TestTrainingWithDataset(test.TestCase, parameterized.TestCase):
         model.train_on_batch(dataset)
 
 
-class TestMetricsWithDatasetIterators(test.TestCase):
+class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  # TODO(kaftan) Run w/ all model types.
+  # Seems like subclass models has a bug, file ticket
+  @keras_parameterized.run_all_keras_modes_with_all_model_types(
+      exclude_models='subclass'
+  )
   def test_metrics_correctness_with_iterator(self):
-    model = keras.Sequential()
-    model.add(
-        keras.layers.Dense(
-            8, activation='relu', input_dim=4, kernel_initializer='ones'))
-    model.add(
-        keras.layers.Dense(
-            1, activation='sigmoid', kernel_initializer='ones'))
+    layers = [
+        keras.layers.Dense(8, activation='relu', input_dim=4,
+                           kernel_initializer='ones'),
+        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
+    ]
+
+    model = testing_utils.get_model_from_layers(layers, (4,))
+
     model.compile(
         loss='binary_crossentropy',
         metrics=['accuracy', metrics_module.BinaryAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=0.001))
+        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        run_eagerly=testing_utils.should_run_eagerly())
 
     np.random.seed(123)
     x = np.random.randint(10, size=(100, 4)).astype(np.float32)
diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py
index 1d1cec1c50..895db5bc63 100644
--- a/tensorflow/python/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/engine/training_eager.py
@@ -244,15 +244,15 @@ def train_on_batch(model, inputs, targets, sample_weights=None):
       inputs = training_utils.cast_if_floating_dtype(inputs)
       targets = training_utils.cast_if_floating_dtype(targets)
     else:
-      inputs = [
-          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs
-      ]
-      targets = [
-          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets
-      ]
+      inputs = training_utils.cast_if_floating_dtype([
+          ops.convert_to_tensor(val) for val in inputs
+      ])
+      targets = training_utils.cast_if_floating_dtype([
+          ops.convert_to_tensor(val) for val in targets
+      ])
   if sample_weights:
     sample_weights = [
-        ops.convert_to_tensor(val, dtype=backend.floatx())
+        training_utils.cast_if_floating_dtype(ops.convert_to_tensor(val))
         if val is not None else None for val in sample_weights
     ]
 
@@ -292,15 +292,15 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
       inputs = training_utils.cast_if_floating_dtype(inputs)
       targets = training_utils.cast_if_floating_dtype(targets)
     else:
-      inputs = [
-          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs
-      ]
-      targets = [
-          ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets
-      ]
+      inputs = training_utils.cast_if_floating_dtype([
+          ops.convert_to_tensor(val) for val in inputs
+      ])
+      targets = training_utils.cast_if_floating_dtype([
+          ops.convert_to_tensor(val) for val in targets
+      ])
   if sample_weights:
     sample_weights = [
-        ops.convert_to_tensor(val, dtype=backend.floatx())
+        training_utils.cast_if_floating_dtype(ops.convert_to_tensor(val))
         if val is not None else None for val in sample_weights
     ]
   outs, loss, loss_metrics, _, masks = _model_loss(
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 8e3b61be0c..6e5fd42714 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.eager import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.callbacks import Callback
@@ -50,19 +51,21 @@ except ImportError:
   scipy_sparse = None
 
 
-class TrainingTest(test.TestCase):
+class TrainingTest(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes_with_all_model_types(
+      exclude_models='sequential'
+  )
   def test_fit_on_arrays(self):
-    a = keras.layers.Input(shape=(3,), name='input_a')
-    b = keras.layers.Input(shape=(3,), name='input_b')
+    input_a = keras.layers.Input(shape=(3,), name='input_a')
+    input_b = keras.layers.Input(shape=(3,), name='input_b')
 
     dense = keras.layers.Dense(4, name='dense')
-    c = dense(a)
-    d = dense(b)
-    e = keras.layers.Dropout(0.5, name='dropout')(c)
+    dropout = keras.layers.Dropout(0.5, name='dropout')
+    branch_a = [input_a, dense]
+    branch_b = [input_b, dense, dropout]
 
-    model = keras.models.Model([a, b], [d, e])
+    model = testing_utils.get_multi_io_model(branch_a, branch_b)
 
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
@@ -71,7 +74,8 @@ class TrainingTest(test.TestCase):
         optimizer,
         loss,
         metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
-        loss_weights=loss_weights)
+        loss_weights=loss_weights,
+        run_eagerly=testing_utils.should_run_eagerly())
 
     input_a_np = np.random.random((10, 3))
     input_b_np = np.random.random((10, 3))
@@ -135,61 +139,63 @@ class TrainingTest(test.TestCase):
         verbose=0,
         validation_split=0.2)
 
-    # Test with dictionary inputs
-    model.fit(
-        {
-            'input_a': input_a_np,
-            'input_b': input_b_np
-        }, {
-            'dense': output_d_np,
-            'dropout': output_e_np
-        },
-        epochs=1,
-        batch_size=5,
-        verbose=0)
-    model.fit(
-        {
-            'input_a': input_a_np,
-            'input_b': input_b_np
-        }, {
-            'dense': output_d_np,
-            'dropout': output_e_np
-        },
-        epochs=1,
-        batch_size=5,
-        verbose=1)
-    model.fit(
-        {
-            'input_a': input_a_np,
-            'input_b': input_b_np
-        }, {
-            'dense': output_d_np,
-            'dropout': output_e_np
-        },
-        validation_data=({
-            'input_a': input_a_np,
-            'input_b': input_b_np
-        }, {
-            'dense': output_d_np,
-            'dropout': output_e_np
-        }),
-        epochs=1,
-        batch_size=5,
-        verbose=0)
-    model.train_on_batch({
-        'input_a': input_a_np,
-        'input_b': input_b_np
-    }, {
-        'dense': output_d_np,
-        'dropout': output_e_np
-    })
+    if testing_utils.get_model_type() == 'functional':
+      # Test with dictionary inputs
+      model.fit(
+          {
+              'input_a': input_a_np,
+              'input_b': input_b_np
+          }, {
+              'dense': output_d_np,
+              'dropout': output_e_np
+          },
+          epochs=1,
+          batch_size=5,
+          verbose=0)
+      model.fit(
+          {
+              'input_a': input_a_np,
+              'input_b': input_b_np
+          }, {
+              'dense': output_d_np,
+              'dropout': output_e_np
+          },
+          epochs=1,
+          batch_size=5,
+          verbose=1)
+      model.fit(
+          {
+              'input_a': input_a_np,
+              'input_b': input_b_np
+          }, {
+              'dense': output_d_np,
+              'dropout': output_e_np
+          },
+          validation_data=({
+              'input_a': input_a_np,
+              'input_b': input_b_np
+          }, {
+              'dense': output_d_np,
+              'dropout': output_e_np
+          }),
+          epochs=1,
+          batch_size=5,
+          verbose=0)
+      model.train_on_batch({
+          'input_a': input_a_np,
+          'input_b': input_b_np
+      }, {
+          'dense': output_d_np,
+          'dropout': output_e_np
+      })
 
     # Test with lists for loss, metrics
     loss = ['mae', 'mse']
     model.compile(
         optimizer,
         loss,
-        metrics=[metrics_module.CategoricalAccuracy(), 'mae'])
+        metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(
         [input_a_np, input_b_np], [output_d_np, output_e_np],
         epochs=1,
@@ -197,13 +203,15 @@ class TrainingTest(test.TestCase):
         verbose=0)
 
     # Test with dictionaries for loss, metrics, loss weights
-    loss = {'dense': 'mse', 'dropout': 'mae'}
-    loss_weights = {'dense': 1., 'dropout': 0.5}
-    metrics = {
-        'dense': 'mse',
-        'dropout': metrics_module.CategoricalAccuracy()
-    }
-    model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights)
+    if testing_utils.get_model_type() == 'functional':
+      loss = {'dense': 'mse', 'dropout': 'mae'}
+      loss_weights = {'dense': 1., 'dropout': 0.5}
+      metrics = {
+          'dense': 'mse',
+          'dropout': metrics_module.CategoricalAccuracy()
+      }
+      model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights,
+                    run_eagerly=testing_utils.should_run_eagerly())
     model.fit(
         [input_a_np, input_b_np], [output_d_np, output_e_np],
         epochs=1,
@@ -239,7 +247,8 @@ class TrainingTest(test.TestCase):
     x = keras.layers.Input(shape=(3,), name='input_a')
     y = keras.layers.Dense(4)(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer, loss='mse')
+    model.compile(optimizer, loss='mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
     # This will work
     model.fit([input_a_np], output_d_np, epochs=1)
     with self.assertRaises(ValueError):
@@ -255,7 +264,7 @@ class TrainingTest(test.TestCase):
               batch_size=5,
               verbose=2)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_evaluate_predict_on_arrays(self):
     a = keras.layers.Input(shape=(3,), name='input_a')
     b = keras.layers.Input(shape=(3,), name='input_b')
@@ -275,7 +284,8 @@ class TrainingTest(test.TestCase):
         loss,
         metrics=['mae', metrics_module.CategoricalAccuracy()],
         loss_weights=loss_weights,
-        sample_weight_mode=None)
+        sample_weight_mode=None,
+        run_eagerly=testing_utils.should_run_eagerly())
 
     input_a_np = np.random.random((10, 3))
     input_b_np = np.random.random((10, 3))
@@ -336,7 +346,7 @@ class TrainingTest(test.TestCase):
     })
     self.assertEqual(len(out), 2)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_activity_regularizer_fit(self):
     loss = {}
     for reg in [None, 'l2']:
@@ -352,12 +362,13 @@ class TrainingTest(test.TestCase):
       y = np.ones((10, 1), 'float32')
 
       optimizer = RMSPropOptimizer(learning_rate=0.001)
-      model.compile(optimizer, 'binary_crossentropy')
+      model.compile(optimizer, 'binary_crossentropy',
+                    run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, batch_size=2, epochs=5)
       loss[reg] = model.evaluate(x, y)
     self.assertLess(loss[None], loss['l2'])
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_activity_regularizer_loss_value(self):
     inputs = keras.layers.Input(shape=(10,))
     outputs = keras.layers.Dense(
@@ -370,11 +381,12 @@ class TrainingTest(test.TestCase):
     x = np.ones((10, 10), 'float32')
     y = np.ones((10, 1), 'float32')
     optimizer = RMSPropOptimizer(learning_rate=0.001)
-    model.compile(optimizer, 'binary_crossentropy')
+    model.compile(optimizer, 'binary_crossentropy',
+                  run_eagerly=testing_utils.should_run_eagerly())
     loss = model.test_on_batch(x, y)
     self.assertAlmostEqual(0.01, loss, places=4)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_activity_regularizer_batch_independent(self):
     inputs = keras.layers.Input(shape=(10,))
     x = keras.layers.Dense(
@@ -384,7 +396,8 @@ class TrainingTest(test.TestCase):
     model = keras.Model(inputs, outputs)
 
     optimizer = RMSPropOptimizer(learning_rate=0.001)
-    model.compile(optimizer, 'binary_crossentropy')
+    model.compile(optimizer, 'binary_crossentropy',
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((10, 10), 'float32')
     y = np.ones((10, 1), 'float32')
@@ -396,7 +409,7 @@ class TrainingTest(test.TestCase):
 
     self.assertAlmostEqual(loss_small_batch, loss_big_batch, places=4)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_activity_regularizer_in_model_call(self):
 
     class MyModel(keras.Model):
@@ -410,44 +423,54 @@ class TrainingTest(test.TestCase):
     _ = model(x)
     self.assertEqual(1, len(model.losses))
 
+  @keras_parameterized.run_all_keras_modes
   def test_training_on_sparse_data_with_dense_placeholders(self):
+    # TODO(kaftan) Test seems to not work, file ticket
+    if testing_utils.should_run_eagerly() and context.executing_eagerly():
+      self.skipTest('Skipping running model eagerly.')
+
     if scipy_sparse is None:
       return
 
-    with self.cached_session():
-      test_inputs = [
-          scipy_sparse.random(6, 3, density=0.25).tocsr() for _ in range(2)
-      ]
-      test_outputs = [
-          scipy_sparse.random(6, i, density=0.25).tocsr() for i in range(3, 5)
-      ]
-      in1 = keras.layers.Input(shape=(3,))
-      in2 = keras.layers.Input(shape=(3,))
-      out1 = keras.layers.Dropout(0.5, name='dropout')(in1)
-      out2 = keras.layers.Dense(4, name='dense_1')(in2)
-      model = keras.Model([in1, in2], [out1, out2])
-      model.predict(test_inputs, batch_size=2)
-      optimizer = RMSPropOptimizer(learning_rate=0.001)
-      model.compile(
-          optimizer,
-          'mse',
-          metrics=['mae', metrics_module.CategoricalAccuracy()])
-      model.fit(test_inputs, test_outputs,
-                epochs=1, batch_size=2, validation_split=0.5)
-      model.evaluate(test_inputs, test_outputs, batch_size=2)
+    test_inputs = [
+        scipy_sparse.random(6, 3, density=0.25).tocsr() for _ in range(2)
+    ]
+    test_outputs = [
+        scipy_sparse.random(6, i, density=0.25).tocsr() for i in range(3, 5)
+    ]
+    in1 = keras.layers.Input(shape=(3,))
+    in2 = keras.layers.Input(shape=(3,))
+    out1 = keras.layers.Dropout(0.5, name='dropout')(in1)
+    out2 = keras.layers.Dense(4, name='dense_1')(in2)
+    model = keras.Model([in1, in2], [out1, out2])
+    model.predict(test_inputs, batch_size=2)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    model.compile(
+        optimizer,
+        'mse',
+        metrics=['mae', metrics_module.CategoricalAccuracy()],
+        run_eagerly=testing_utils.should_run_eagerly())
+    model.fit(test_inputs, test_outputs,
+              epochs=1, batch_size=2, validation_split=0.5)
+    model.evaluate(test_inputs, test_outputs, batch_size=2)
 
+  @keras_parameterized.run_all_keras_modes
   def test_compile_with_sparse_placeholders(self):
-    with self.cached_session():
-      input_layer = keras.layers.Input(shape=(10,), sparse=True)
-      weights = variables_lib.Variable(
-          np.ones((10, 1)).astype(np.float32), name='weights')
-      weights_mult = lambda x: sparse_ops.sparse_tensor_dense_matmul(x, weights)
-      output_layer = keras.layers.Lambda(weights_mult)(input_layer)
-      model = keras.Model([input_layer], output_layer)
-      model.compile(
-          loss='binary_crossentropy',
-          optimizer=keras.optimizers.Adam(lr=0.0001),
-          metrics=['accuracy'])
+    # TODO(kaftan) Test seems to not work, file ticket
+    if testing_utils.should_run_eagerly() and context.executing_eagerly():
+      self.skipTest('Skipping running model eagerly.')
+
+    input_layer = keras.layers.Input(shape=(10,), sparse=True)
+    weights = variables_lib.Variable(
+        np.ones((10, 1)).astype(np.float32), name='weights')
+    weights_mult = lambda x: sparse_ops.sparse_tensor_dense_matmul(x, weights)
+    output_layer = keras.layers.Lambda(weights_mult)(input_layer)
+    model = keras.Model([input_layer], output_layer)
+    model.compile(
+        loss='binary_crossentropy',
+        optimizer=keras.optimizers.Adam(lr=0.0001),
+        metrics=['accuracy'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
   @tf_test_util.run_deprecated_v1
   def test_that_trainable_disables_updates(self):
@@ -546,14 +569,15 @@ class TrainingTest(test.TestCase):
               'val_loss', 'val_weighted_mean_absolute_error'
           ]))
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_mismatched_output_shape_and_target_shape(self):
     model = keras.Sequential([
         keras.layers.Dense(2, input_shape=(3, 4)),
         keras.layers.Dense(5),
     ])
     model.compile(RMSPropOptimizer(learning_rate=0.001),
-                  loss='sparse_categorical_crossentropy')
+                  loss='sparse_categorical_crossentropy',
+                  run_eagerly=testing_utils.should_run_eagerly())
     # Test with Numpy data
     x_train = np.random.random((10, 3, 4))
     y_train = np.random.randint(0, 5, size=(10, 3))
@@ -588,14 +612,15 @@ class TrainingTest(test.TestCase):
       self.assertAllEqual(
           self.evaluate(layer.losses), self.evaluate(get_losses()))
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_logging(self):
     mock_stdout = io.BytesIO() if six.PY2 else io.StringIO()
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(10, activation='relu'))
     model.add(keras.layers.Dense(1, activation='sigmoid'))
     model.compile(
-        RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy')
+        RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy',
+        run_eagerly=testing_utils.should_run_eagerly())
     with test.mock.patch.object(sys, 'stdout', mock_stdout):
       model.fit(
           np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10)
@@ -693,9 +718,9 @@ class TrainingTest(test.TestCase):
       keras.Model([input1, input2], outputs)
 
 
-class TestExceptionsAndWarnings(test.TestCase):
+class TestExceptionsAndWarnings(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_invalid_loss(self):
     num_classes = 5
     train_samples = 1000
@@ -722,9 +747,10 @@ class TestExceptionsAndWarnings(test.TestCase):
         model.fit(x_train, y_train)
 
       with self.assertRaises(ValueError):
-        model.compile(optimizer, loss=None)
+        model.compile(optimizer, loss=None,
+                      run_eagerly=testing_utils.should_run_eagerly())
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_compile_warning_for_loss_missing_output(self):
     with self.cached_session():
       inp = keras.layers.Input(shape=(16,), name='input_a')
@@ -742,16 +768,17 @@ class TestExceptionsAndWarnings(test.TestCase):
             metrics={
                 'dense_2': 'categorical_accuracy',
                 'dense_1': metrics_module.CategoricalAccuracy(),
-            })
+            },
+            run_eagerly=testing_utils.should_run_eagerly())
         msg = ('Output "dense_1" missing from loss dictionary. We assume this '
                'was done on purpose. The fit and evaluate APIs will not be '
                'expecting any data to be passed to "dense_1".')
         self.assertRegexpMatches(str(mock_log.call_args), msg)
 
 
-class LossWeightingTest(test.TestCase):
+class LossWeightingTest(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_class_weights(self):
     num_classes = 5
     batch_size = 5
@@ -768,7 +795,8 @@ class LossWeightingTest(test.TestCase):
         loss='categorical_crossentropy',
         metrics=['acc', metrics_module.CategoricalAccuracy()],
         weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=learning_rate))
+        optimizer=RMSPropOptimizer(learning_rate=learning_rate),
+        run_eagerly=testing_utils.should_run_eagerly())
 
     np.random.seed(1337)
     (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
@@ -820,7 +848,7 @@ class LossWeightingTest(test.TestCase):
         x_test[test_ids, :], y_test[test_ids, :], verbose=0)
     self.assertLess(score[0], ref_score[0])
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_sample_weights(self):
     num_classes = 5
     batch_size = 5
@@ -837,7 +865,8 @@ class LossWeightingTest(test.TestCase):
         RMSPropOptimizer(learning_rate=learning_rate),
         metrics=['acc', metrics_module.CategoricalAccuracy()],
         weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
-        loss='categorical_crossentropy')
+        loss='categorical_crossentropy',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     np.random.seed(43)
     (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
@@ -885,13 +914,15 @@ class LossWeightingTest(test.TestCase):
           x_test[test_ids, :], y_test[test_ids, :], verbose=0)
       self.assertLess(score[0], ref_score[0])
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_warning_for_concurrent_sample_and_class_weights(self):
+
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(10, input_shape=(3,)))
     model.compile(
         loss='mse',
-        optimizer=RMSPropOptimizer(learning_rate=0.01))
+        optimizer=RMSPropOptimizer(learning_rate=0.01),
+        run_eagerly=testing_utils.should_run_eagerly())
     x_train = np.random.random((10, 3))
     y_train = np.random.random((10, 10))
     sample_weight = np.ones((y_train.shape[0]))
@@ -905,10 +936,16 @@ class LossWeightingTest(test.TestCase):
           verbose=0,
           sample_weight=sample_weight,
           class_weight=class_weight)
-      msg = ('The `class_weight` argument will be ignored.')
-      self.assertRegexpMatches(str(mock_log.call_args), msg)
+      msg = 'The `class_weight` argument will be ignored.'
 
-  @tf_test_util.run_in_graph_and_eager_modes
+      msg_found = False
+      for call_args in mock_log.call_args_list:
+        if msg in str(call_args):
+          msg_found = True
+
+      self.assertTrue(msg_found)
+
+  @keras_parameterized.run_all_keras_modes
   def test_temporal_sample_weights(self):
     num_classes = 5
     batch_size = 5
@@ -966,7 +1003,8 @@ class LossWeightingTest(test.TestCase):
           loss='binary_crossentropy',
           metrics=['acc', metrics_module.CategoricalAccuracy()],
           weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
-          sample_weight_mode='temporal')
+          sample_weight_mode='temporal',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       model.fit(
           temporal_x_train,
@@ -998,7 +1036,7 @@ class LossWeightingTest(test.TestCase):
             temporal_x_test[test_ids], temporal_y_test[test_ids], verbose=0)
         self.assertLess(score[0], ref_score[0])
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_class_weight_invalid_use_case(self):
     num_classes = 5
     train_samples = 1000
@@ -1015,7 +1053,8 @@ class LossWeightingTest(test.TestCase):
               input_shape=(timesteps, input_dim)))
       model.add(keras.layers.Activation('softmax'))
       optimizer = RMSPropOptimizer(learning_rate=learning_rate)
-      model.compile(optimizer, loss='binary_crossentropy')
+      model.compile(optimizer, loss='binary_crossentropy',
+                    run_eagerly=testing_utils.should_run_eagerly())
 
       (x_train, y_train), _ = testing_utils.get_test_data(
           train_samples=train_samples,
@@ -1033,14 +1072,16 @@ class LossWeightingTest(test.TestCase):
 
       with self.assertRaises(ValueError):
         model.compile(
-            optimizer, loss='binary_crossentropy', sample_weight_mode=[])
+            optimizer, loss='binary_crossentropy', sample_weight_mode=[],
+            run_eagerly=testing_utils.should_run_eagerly())
 
       # Build multi-output model
       x = keras.Input((3,))
       y1 = keras.layers.Dense(4, name='1')(x)
       y2 = keras.layers.Dense(4, name='2')(x)
       model = keras.models.Model(x, [y1, y2])
-      model.compile(optimizer, loss='mse')
+      model.compile(optimizer, loss='mse',
+                    run_eagerly=testing_utils.should_run_eagerly())
       x_np = np.random.random((10, 3))
       y_np = np.random.random((10, 4))
       w_np = np.random.random((10,))
@@ -1067,7 +1108,7 @@ class LossWeightingTest(test.TestCase):
         model.fit(x_np, [y_np, y_np], epochs=1,
                   sample_weight={'1': bad_w_np})
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_default_sample_weight(self):
     """Verifies that fit works without having to set sample_weight."""
 
@@ -1088,38 +1129,46 @@ class LossWeightingTest(test.TestCase):
       optimizer = RMSPropOptimizer(learning_rate=learning_rate)
 
       # sample_weight_mode is a list and mode value is None
-      model.compile(optimizer, loss='mse', sample_weight_mode=[None])
+      model.compile(optimizer, loss='mse', sample_weight_mode=[None],
+                    run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, epochs=1, batch_size=10)
 
       # sample_weight_mode is a list and mode value is `temporal`
-      model.compile(optimizer, loss='mse', sample_weight_mode=['temporal'])
+      model.compile(optimizer, loss='mse', sample_weight_mode=['temporal'],
+                    run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, epochs=1, batch_size=10)
 
       # sample_weight_mode is a dict and mode value is None
       model.compile(
-          optimizer, loss='mse', sample_weight_mode={'time_distributed': None})
+          optimizer, loss='mse', sample_weight_mode={'time_distributed': None},
+          run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, epochs=1, batch_size=10)
 
       # sample_weight_mode is a dict and mode value is `temporal`
       model.compile(
           optimizer,
           loss='mse',
-          sample_weight_mode={'time_distributed': 'temporal'})
+          sample_weight_mode={'time_distributed': 'temporal'},
+          run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, epochs=1, batch_size=10)
 
       # sample_weight_mode is a not a list/dict and mode value is None
-      model.compile(optimizer, loss='mse', sample_weight_mode=None)
+      model.compile(optimizer, loss='mse', sample_weight_mode=None,
+                    run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, epochs=1, batch_size=10)
 
       # sample_weight_mode is a not a list/dict and mode value is `temporal`
-      model.compile(optimizer, loss='mse', sample_weight_mode='temporal')
+      model.compile(optimizer, loss='mse', sample_weight_mode='temporal',
+                    run_eagerly=testing_utils.should_run_eagerly())
       model.fit(x, y, epochs=1, batch_size=10)
 
 
-class LossMaskingTest(test.TestCase):
+class LossMaskingTest(keras_parameterized.TestCase):
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_masking_graph_sequential(self):
+    if testing_utils.should_run_eagerly():
+      self.skipTest('b/120495761')
     with self.cached_session():
       x = np.array([[[1], [1]], [[0], [0]]])
       model = keras.models.Sequential()
@@ -1127,13 +1176,16 @@ class LossMaskingTest(test.TestCase):
       model.add(
           keras.layers.TimeDistributed(
               keras.layers.Dense(1, kernel_initializer='one')))
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+                    run_eagerly=testing_utils.should_run_eagerly())
       y = np.array([[[1], [1]], [[1], [1]]])
       loss = model.train_on_batch(x, y)
       self.assertEqual(float(loss), 0.)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_masking_deferred_sequential(self):
+    if testing_utils.should_run_eagerly():
+      self.skipTest('b/120495761')
     with self.cached_session():
       x = np.array([[[1], [1]], [[0], [0]]])
       model = keras.models.Sequential()
@@ -1141,13 +1193,16 @@ class LossMaskingTest(test.TestCase):
       model.add(
           keras.layers.TimeDistributed(
               keras.layers.Dense(1, kernel_initializer='one')))
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+                    run_eagerly=testing_utils.should_run_eagerly())
       y = np.array([[[1], [1]], [[1], [1]]])
       loss = model.train_on_batch(x, y)
       self.assertEqual(float(loss), 0.)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_masking_functional(self):
+    if testing_utils.should_run_eagerly():
+      self.skipTest('b/120495761')
     with self.cached_session():
       x = np.array([[[1], [1]], [[0], [0]]])
       inputs = keras.layers.Input((2, 1))
@@ -1155,12 +1210,13 @@ class LossMaskingTest(test.TestCase):
       outputs = keras.layers.TimeDistributed(
           keras.layers.Dense(1, kernel_initializer='one'))(outputs)
       model = keras.Model(inputs, outputs)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+                    run_eagerly=testing_utils.should_run_eagerly())
       y = np.array([[[1], [1]], [[1], [1]]])
       loss = model.train_on_batch(x, y)
       self.assertEqual(float(loss), 0.)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_mask_argument_in_layer(self):
     # Test that the mask argument gets correctly passed to a layer in the
     # functional API.
@@ -1185,7 +1241,8 @@ class LossMaskingTest(test.TestCase):
       outputs = CustomMaskedLayer()(masked)
 
       model = keras.Model(inputs, outputs)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+                    run_eagerly=testing_utils.should_run_eagerly())
       y = np.random.random((5, 3))
       model.train_on_batch(x, y)
 
@@ -1208,7 +1265,7 @@ class LossMaskingTest(test.TestCase):
               keras.backend.variable(weights), keras.backend.variable(mask)))
 
 
-class TestDynamicTrainability(test.TestCase):
+class TestDynamicTrainability(keras_parameterized.TestCase):
 
   @tf_test_util.run_deprecated_v1
   def test_trainable_warning(self):
@@ -1353,144 +1410,152 @@ class TestDynamicTrainability(test.TestCase):
       self.assertListEqual(outer_model.trainable_weights, [])
 
 
-class TestTrainingWithDataTensors(test.TestCase):
+class TestTrainingWithDataTensors(keras_parameterized.TestCase):
 
-  @tf_test_util.run_deprecated_v1
+  @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_symbolic_tensors_single_io(self):
-    with self.cached_session():
-      x = keras.layers.Input(shape=(3,), name='input')
-      y = keras.layers.Dense(4, name='dense')(x)
-      model = keras.Model(x, y)
-
-      optimizer = RMSPropOptimizer(learning_rate=0.001)
-      loss = 'mse'
-      model.compile(
-          optimizer,
-          loss,
-          metrics=['mae', metrics_module.CategoricalAccuracy()])
+    # TODO(kaftan) Test seems to not work, file ticket
+    if  context.executing_eagerly():
+      self.skipTest('Skipping eager execution.')
 
-      inputs = keras.backend.zeros(shape=(10, 3))
-      targets = keras.backend.zeros(shape=(10, 4))
+    x = keras.layers.Input(shape=(3,), name='input')
+    y = keras.layers.Dense(4, name='dense')(x)
+    model = keras.Model(x, y)
 
-      model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
-      model.evaluate(inputs, targets, steps=2, verbose=0)
-      model.predict(inputs, steps=2)
-      model.train_on_batch(inputs, targets)
-      model.test_on_batch(inputs, targets)
-      model.fit(inputs, targets,
-                epochs=1, steps_per_epoch=2, verbose=0,
-                validation_data=(inputs, targets), validation_steps=2)
-
-      # Test with dynamic shape
-      inputs = array_ops.placeholder_with_default(
-          np.zeros((2, 3)), shape=tensor_shape.TensorShape([None, 3]))
-      targets = array_ops.placeholder_with_default(
-          np.zeros((2, 4)), shape=tensor_shape.TensorShape([None, 4]))
-      self.assertEqual(inputs.shape.dims[0].value, None)
-      model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
-      model.evaluate(inputs, targets, steps=2, verbose=0)
-      model.predict(inputs, steps=2)
-      model.train_on_batch(inputs, targets)
-      model.test_on_batch(inputs, targets)
-      model.fit(inputs, targets,
-                epochs=1, steps_per_epoch=2, verbose=0,
-                validation_data=(inputs, targets), validation_steps=2)
-
-  @tf_test_util.run_deprecated_v1
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    model.compile(
+        optimizer,
+        loss,
+        metrics=['mae', metrics_module.CategoricalAccuracy()],
+        run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = keras.backend.zeros(shape=(10, 3))
+    targets = keras.backend.zeros(shape=(10, 4))
+
+    model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
+    model.evaluate(inputs, targets, steps=2, verbose=0)
+    model.predict(inputs, steps=2)
+    model.train_on_batch(inputs, targets)
+    model.test_on_batch(inputs, targets)
+    model.fit(inputs, targets,
+              epochs=1, steps_per_epoch=2, verbose=0,
+              validation_data=(inputs, targets), validation_steps=2)
+
+    # Test with dynamic shape
+    inputs = array_ops.placeholder_with_default(
+        np.zeros((2, 3)), shape=tensor_shape.TensorShape([None, 3]))
+    targets = array_ops.placeholder_with_default(
+        np.zeros((2, 4)), shape=tensor_shape.TensorShape([None, 4]))
+    self.assertEqual(inputs.shape.dims[0].value, None)
+    model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
+    model.evaluate(inputs, targets, steps=2, verbose=0)
+    model.predict(inputs, steps=2)
+    model.train_on_batch(inputs, targets)
+    model.test_on_batch(inputs, targets)
+    model.fit(inputs, targets,
+              epochs=1, steps_per_epoch=2, verbose=0,
+              validation_data=(inputs, targets), validation_steps=2)
+
+  @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self):
-    with self.cached_session():
-      a = keras.layers.Input(shape=(3,), name='input_a')
-      b = keras.layers.Input(shape=(3,), name='input_b')
+    # TODO(kaftan) Test seems to not work, file ticket
+    if context.executing_eagerly():
+      self.skipTest('Skipping eager execution.')
 
-      dense = keras.layers.Dense(4, name='dense')
-      c = dense(a)
-      d = dense(b)
-      e = keras.layers.Dropout(0.5, name='dropout')(c)
+    a = keras.layers.Input(shape=(3,), name='input_a')
+    b = keras.layers.Input(shape=(3,), name='input_b')
 
-      model = keras.models.Model([a, b], [d, e])
+    dense = keras.layers.Dense(4, name='dense')
+    c = dense(a)
+    d = dense(b)
+    e = keras.layers.Dropout(0.5, name='dropout')(c)
 
-      optimizer = 'rmsprop'
-      loss = 'mse'
-      loss_weights = [1., 0.5]
-      model.compile(
-          optimizer,
-          loss,
-          metrics=['mae', metrics_module.CategoricalAccuracy()],
-          loss_weights=loss_weights)
+    model = keras.models.Model([a, b], [d, e])
+
+    optimizer = 'rmsprop'
+    loss = 'mse'
+    loss_weights = [1., 0.5]
+    model.compile(
+        optimizer,
+        loss,
+        metrics=['mae', metrics_module.CategoricalAccuracy()],
+        loss_weights=loss_weights,
+        run_eagerly=testing_utils.should_run_eagerly())
 
-      input_a_tf = keras.backend.zeros(shape=(10, 3))
-      input_b_tf = keras.backend.zeros(shape=(10, 3))
+    input_a_tf = keras.backend.zeros(shape=(10, 3))
+    input_b_tf = keras.backend.zeros(shape=(10, 3))
 
-      output_d_tf = keras.backend.zeros(shape=(10, 4))
-      output_e_tf = keras.backend.zeros(shape=(10, 4))
+    output_d_tf = keras.backend.zeros(shape=(10, 4))
+    output_e_tf = keras.backend.zeros(shape=(10, 4))
 
+    model.fit(
+        [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+        epochs=1,
+        steps_per_epoch=2,
+        verbose=0)
+    with self.assertRaisesRegexp(ValueError,
+                                 'should specify the `steps_per_epoch`'):
       model.fit(
           [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
           epochs=1,
-          steps_per_epoch=2,
+          batch_size=5,
           verbose=0)
-      with self.assertRaisesRegexp(ValueError,
-                                   'should specify the `steps_per_epoch`'):
-        model.fit(
-            [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
-            epochs=1,
-            batch_size=5,
-            verbose=0)
-      model.train_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
+    model.train_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
 
-      # Test with dictionary inputs
-      model.fit(
-          {'input_a': input_a_tf,
-           'input_b': input_b_tf},
-          {'dense': output_d_tf,
-           'dropout': output_e_tf},
-          epochs=1,
-          steps_per_epoch=2,
-          verbose=0)
-      model.fit(
-          {'input_a': input_a_tf,
-           'input_b': input_b_tf},
-          {'dense': output_d_tf,
-           'dropout': output_e_tf},
-          validation_data=({'input_a': input_a_tf,
-                            'input_b': input_b_tf},
-                           {'dense': output_d_tf,
-                            'dropout': output_e_tf}),
-          epochs=1,
-          steps_per_epoch=2,
-          validation_steps=2,
-          verbose=0)
-      model.train_on_batch(
-          {'input_a': input_a_tf,
-           'input_b': input_b_tf},
-          {'dense': output_d_tf,
-           'dropout': output_e_tf})
+    # Test with dictionary inputs
+    model.fit(
+        {'input_a': input_a_tf,
+         'input_b': input_b_tf},
+        {'dense': output_d_tf,
+         'dropout': output_e_tf},
+        epochs=1,
+        steps_per_epoch=2,
+        verbose=0)
+    model.fit(
+        {'input_a': input_a_tf,
+         'input_b': input_b_tf},
+        {'dense': output_d_tf,
+         'dropout': output_e_tf},
+        validation_data=({'input_a': input_a_tf,
+                          'input_b': input_b_tf},
+                         {'dense': output_d_tf,
+                          'dropout': output_e_tf}),
+        epochs=1,
+        steps_per_epoch=2,
+        validation_steps=2,
+        verbose=0)
+    model.train_on_batch(
+        {'input_a': input_a_tf,
+         'input_b': input_b_tf},
+        {'dense': output_d_tf,
+         'dropout': output_e_tf})
 
-      # Test with validation data
+    # Test with validation data
+    model.fit(
+        [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+        validation_data=([input_a_tf, input_b_tf],
+                         [output_d_tf, output_e_tf]),
+        epochs=1,
+        steps_per_epoch=2,
+        validation_steps=2,
+        verbose=0)
+    # Test with validation split
+    with self.assertRaisesRegexp(ValueError,
+                                 'you cannot use `validation_split`'):
       model.fit(
           [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
-          validation_data=([input_a_tf, input_b_tf],
-                           [output_d_tf, output_e_tf]),
-          epochs=1,
+          epochs=2,
           steps_per_epoch=2,
-          validation_steps=2,
-          verbose=0)
-      # Test with validation split
-      with self.assertRaisesRegexp(ValueError,
-                                   'you cannot use `validation_split`'):
-        model.fit(
-            [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
-            epochs=2,
-            steps_per_epoch=2,
-            verbose=0,
-            validation_split=0.2,
-            validation_steps=2)
-
-      # Test evaluation / prediction methods
-      model.evaluate([input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
-                     steps=2, verbose=0)
-      model.predict([input_a_tf, input_b_tf], steps=2)
-      model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
+          verbose=0,
+          validation_split=0.2,
+          validation_steps=2)
+
+    # Test evaluation / prediction methods
+    model.evaluate([input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+                   steps=2, verbose=0)
+    model.predict([input_a_tf, input_b_tf], steps=2)
+    model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
 
   @tf_test_util.run_deprecated_v1
   def test_model_with_input_feed_tensor(self):
@@ -1929,10 +1994,10 @@ class TestTrainingWithDataTensors(test.TestCase):
                            [output_a_np, output_b_np])
 
 
-class TestTrainingWithMetrics(test.TestCase):
+class TestTrainingWithMetrics(keras_parameterized.TestCase):
   """Training tests related to metrics."""
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_metrics_names(self):
     a = keras.layers.Input(shape=(3,), name='input_a')
     b = keras.layers.Input(shape=(3,), name='input_b')
@@ -1946,7 +2011,8 @@ class TestTrainingWithMetrics(test.TestCase):
 
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     metrics = ['mse', metrics_module.BinaryAccuracy()]
-    model.compile(optimizer, loss='mae', metrics=metrics)
+    model.compile(optimizer, loss='mae', metrics=metrics,
+                  run_eagerly=testing_utils.should_run_eagerly())
     reference_metric_names = [
         'loss', 'dense_loss', 'dropout_loss', 'dense_mean_squared_error',
         'dense_binary_accuracy', 'dropout_mean_squared_error',
@@ -1966,7 +2032,7 @@ class TestTrainingWithMetrics(test.TestCase):
               batch_size=5)
     self.assertEqual(reference_metric_names, model.metrics_names)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_metrics_correctness(self):
     model = keras.Sequential()
     model.add(
@@ -1978,7 +2044,8 @@ class TestTrainingWithMetrics(test.TestCase):
     model.compile(
         loss='mae',
         metrics=['accuracy', metrics_module.BinaryAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=0.001))
+        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        run_eagerly=testing_utils.should_run_eagerly())
 
     # verify correctness of stateful and stateless metrics.
     x = np.ones((100, 4))
@@ -1992,7 +2059,7 @@ class TestTrainingWithMetrics(test.TestCase):
     self.assertEqual(outs[1], 0.)
     self.assertEqual(outs[2], 0.)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_metrics_correctness_with_weighted_metrics(self):
     np.random.seed(1337)
     x = np.array([[[1.], [1.]], [[0.], [0.]]])
@@ -2005,7 +2072,8 @@ class TestTrainingWithMetrics(test.TestCase):
         RMSPropOptimizer(learning_rate=0.001),
         loss='mse',
         sample_weight_mode='temporal',
-        weighted_metrics=['accuracy', 'mse'])
+        weighted_metrics=['accuracy', 'mse'],
+        run_eagerly=testing_utils.should_run_eagerly())
     y = np.array([[[1.], [1.]], [[1.], [1.]]])
 
     outs = model.evaluate(x, y)
@@ -2027,7 +2095,7 @@ class TestTrainingWithMetrics(test.TestCase):
     mse2 = model.evaluate(x, y, sample_weight=w, batch_size=10)[2]
     self.assertNear(mse1, mse2, err=1e-7)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_metric_state_reset_between_fit_and_evaluate(self):
     model = keras.Sequential()
     model.add(keras.layers.Dense(3, activation='relu', input_dim=4))
@@ -2036,7 +2104,8 @@ class TestTrainingWithMetrics(test.TestCase):
     model.compile(
         loss='mae',
         metrics=[acc_obj],
-        optimizer=RMSPropOptimizer(learning_rate=0.001))
+        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x_train = np.random.random((100, 4))
     y_train = np.random.random((100, 1))
@@ -2048,7 +2117,7 @@ class TestTrainingWithMetrics(test.TestCase):
     model.evaluate(x_test, y_test, batch_size=5)
     self.assertEqual(self.evaluate(acc_obj.count), 10)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_invalid_metrics(self):
     num_classes = 5
     input_dim = 5
@@ -2062,10 +2131,13 @@ class TestTrainingWithMetrics(test.TestCase):
       model.compile(
           RMSPropOptimizer(learning_rate=0.001),
           loss='categorical_crossentropy',
-          metrics=metrics_module.CategoricalAccuracy())
+          metrics=metrics_module.CategoricalAccuracy(),
+          run_eagerly=testing_utils.should_run_eagerly())
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_metrics_masking(self):
+    if testing_utils.should_run_eagerly():
+      self.skipTest('b/120495761')
     with self.cached_session():
       np.random.seed(1337)
       model = keras.models.Sequential()
@@ -2076,7 +2148,8 @@ class TestTrainingWithMetrics(test.TestCase):
       model.compile(
           RMSPropOptimizer(learning_rate=0.001),
           loss='mse',
-          weighted_metrics=['accuracy'])
+          weighted_metrics=['accuracy'],
+          run_eagerly=testing_utils.should_run_eagerly())
 
       # verify that masking is applied.
       x = np.array([[[1], [1]], [[1], [1]], [[0], [0]]])
@@ -2124,7 +2197,7 @@ class TestTrainingWithMetrics(test.TestCase):
       model.train_on_batch(inputs, targets)
       model.test_on_batch(inputs, targets)
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_add_metric_in_model_call(self):
 
     class TestModel(keras.Model):
@@ -2143,7 +2216,8 @@ class TestTrainingWithMetrics(test.TestCase):
         return self.dense1(x)
 
     model = TestModel()
-    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones(shape=(10, 1))
     y = np.ones(shape=(10, 2))
@@ -2161,45 +2235,7 @@ class TestTrainingWithMetrics(test.TestCase):
     model.train_on_batch(x, y)
     model.test_on_batch(x, y)
 
-  def test_add_metric_in_model_call_run_eagerly(self):
-    with context.eager_mode():
-
-      class TestModel(keras.Model):
-
-        def __init__(self):
-          super(TestModel, self).__init__(name='test_model')
-          self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
-          self.mean = metrics_module.Mean(name='metric_1')
-
-        def call(self, x):
-          self.add_metric(
-              math_ops.reduce_sum(x), name='metric_2', aggregation='mean')
-          # Provide same name as in the instance created in __init__
-          # for eager mode
-          self.add_metric(self.mean(x), name='metric_1')
-          return self.dense1(x)
-
-      model = TestModel()
-      model.compile(
-          loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True)
-
-      x = np.ones(shape=(10, 1))
-      y = np.ones(shape=(10, 2))
-      history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
-      self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
-      self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0)
-      self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0)
-      self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0)
-
-      eval_results = model.evaluate(x, y, batch_size=5)
-      self.assertAlmostEqual(eval_results[1], 1, 0)
-      self.assertAlmostEqual(eval_results[2], 5, 0)
-
-      model.predict(x, batch_size=5)
-      model.train_on_batch(x, y)
-      model.test_on_batch(x, y)
-
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_add_metric_in_layer_call(self):
 
     class TestLayer(keras.layers.Layer):
@@ -2217,7 +2253,8 @@ class TestTrainingWithMetrics(test.TestCase):
     model = keras.Sequential()
     model.add(TestLayer(input_shape=(1,)))
     model.add(keras.layers.Dense(2, kernel_initializer='ones'))
-    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones(shape=(10, 1))
     y = np.ones(shape=(10, 2))
@@ -2225,33 +2262,6 @@ class TestTrainingWithMetrics(test.TestCase):
     self.assertEqual(history.history['metric_1'][-1], 5)
     self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
 
-  def test_add_metric_in_layer_call_run_eagerly(self):
-    with context.eager_mode():
-
-      class TestLayer(keras.layers.Layer):
-
-        def build(self, input_shape):
-          self.a = self.add_variable(
-              'a', (1, 1), initializer='ones', trainable=False)
-          self.built = True
-
-        def call(self, inputs):
-          self.add_metric(
-              math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean')
-          return inputs + 1
-
-      model = keras.Sequential()
-      model.add(TestLayer(input_shape=(1,)))
-      model.add(keras.layers.Dense(2, kernel_initializer='ones'))
-      model.compile(
-          loss='mse', optimizer=RMSPropOptimizer(0.01), run_eagerly=True)
-
-      x = np.ones(shape=(10, 1))
-      y = np.ones(shape=(10, 2))
-      history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
-      self.assertEqual(history.history['metric_1'][-1], 5)
-      self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
-
   @tf_test_util.run_deprecated_v1
   def test_model_metrics_list(self):
     with self.cached_session():
@@ -2307,7 +2317,7 @@ class TestTrainingWithMetrics(test.TestCase):
           names.append(m.__name__)
       self.assertEqual(names, ['categorical_accuracy', 'metric_1'])
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_multiple_add_metric_calls(self):
 
     class TestModel(keras.Model):
@@ -2326,7 +2336,8 @@ class TestTrainingWithMetrics(test.TestCase):
         return self.dense1(x)
 
     model = TestModel()
-    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones(shape=(10, 1))
     y = np.ones(shape=(10, 2))
@@ -2365,7 +2376,7 @@ class TestTrainingWithMetrics(test.TestCase):
           'eager execution.'):
         model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_duplicate_metric_name_in_add_metric(self):
 
     class TestModel(keras.Model):
@@ -2381,7 +2392,8 @@ class TestTrainingWithMetrics(test.TestCase):
         return self.dense1(x)
 
     model = TestModel()
-    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones(shape=(10, 1))
     y = np.ones(shape=(10, 2))
@@ -2391,8 +2403,11 @@ class TestTrainingWithMetrics(test.TestCase):
         'We found 2 metrics with the name: "metric_1"'):
       model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
 
-  @tf_test_util.run_in_graph_and_eager_modes
+  @keras_parameterized.run_all_keras_modes
   def test_multiple_no_name_input_to_add_metric(self):
+    # TODO(kaftan) Test seems to not work, file ticket
+    if testing_utils.should_run_eagerly() and context.executing_eagerly():
+      self.skipTest('Skipping running model eagerly.')
 
     class TestModel(keras.Model):
 
@@ -2406,7 +2421,8 @@ class TestTrainingWithMetrics(test.TestCase):
         return self.dense1(x)
 
     model = TestModel()
-    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
     x = np.ones(shape=(10, 1))
     y = np.ones(shape=(10, 2))
     model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
diff --git a/tensorflow/python/keras/keras_parameterized.py b/tensorflow/python/keras/keras_parameterized.py
new file mode 100644
index 0000000000..57e66d89d9
--- /dev/null
+++ b/tensorflow/python/keras/keras_parameterized.py
@@ -0,0 +1,348 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for unit-testing Keras."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python import keras
+from tensorflow.python import tf2
+from tensorflow.python.eager import context
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+class TestCase(test.TestCase, parameterized.TestCase):
+
+  def tearDown(self):
+    keras.backend.clear_session()
+    super(TestCase, self).tearDown()
+
+
+# TODO(kaftan): Possibly enable 'subclass_custom_build' when tests begin to pass
+# it. Or perhaps make 'subclass' always use a custom build method.
+def run_with_all_model_types(
+    func=None,
+    exclude_models=None,
+    extra_parameters=None):
+  """Execute the decorated test with all Keras model types.
+
+  This decorator is intended to be applied to test methods in
+  a `testing_utils.KerasTestCase` class. Doing so will cause the contents of
+  the test method to be executed multiple times - once for each Keras model
+  type.
+
+  The Keras model types are: ['functional', 'subclass', 'sequential']
+
+  Note: This decorator does not stack with any other parameterizing
+  decorators. Use the extra_parameters arg if you would like to use additional
+  parameterization.
+
+  Various methods in `testing_utils` to get models will auto-generate a model
+  of the currently active Keras model type. This allows unittests to confirm
+  the equivalence between different Keras models.
+
+  For example, consider the following unittest:
+
+  ```python
+  class MyTests(testing_utils.KerasTestCase):
+
+    @testing_utils.run_with_all_model_types(
+      model_types = ['functional', 'subclass'])
+    def test_foo(self):
+      model = testing_utils.get_small_mlp(1, 4, input_dim=3)
+      optimizer = RMSPropOptimizer(learning_rate=0.001)
+      loss = 'mse'
+      metrics = ['mae']
+      model.compile(optimizer, loss, metrics=metrics)
+
+      inputs = np.zeros((10, 3))
+      targets = np.zeros((10, 4))
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
+
+      model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+
+  if __name__ == "__main__":
+    tf.test.main()
+  ```
+
+  This test tries building a small mlp as both a functional model and as a
+  subclass model.
+
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+    exclude_models: A collection of Keras model types to not run.
+      (May also be a single model type not wrapped in a collection).
+      Defaults to None.
+    extra_parameters: A list of dicts containing additional parameters for
+      parameterizing the decorated test method. This additional parameters must
+      be arguments in the test method's signature.
+      See the absl.testing.parameterized decorators for more info on how these
+      work.
+
+  Returns:
+    Returns a decorator that will run the decorated test method multiple times:
+    once for each desired Keras model type.
+
+  Raises:
+    ImportError: If abseil parameterized is not installed or not included as
+      a target dependency.
+  """
+  return run_all_keras_modes_with_all_model_types(
+      func, skip_run_modes=True, exclude_models=exclude_models,
+      extra_parameters=extra_parameters)
+
+
+def run_all_keras_modes(
+    func=None,
+    config=None,
+    always_skip_v1=False,
+    extra_parameters=None):
+  """Execute the decorated test with all keras execution modes.
+
+  This decorator is intended to be applied to test methods in
+  a `testing_utils.KerasTestCase` class. Doing so will cause the contents of
+  the test method to be executed three times - once executing in legacy graph
+  mode, once running eagerly and with `should_run_eagerly` returning True, and
+  once running eagerly with `should_run_eagerly` returning False.
+
+  If Tensorflow v2 behavior is enabled, legacy graph mode will be skipped, and
+  the test will only run twice.
+
+  Note: This decorator does not stack with any other parameterizing
+  decorators. Use the extra_parameters arg if you would like to use additional
+  parameterization.
+
+  For example, consider the following unittest:
+
+  ```python
+  class MyTests(testing_utils.KerasTestCase):
+
+    @testing_utils.run_all_keras_modes
+    def test_foo(self):
+      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+      optimizer = RMSPropOptimizer(learning_rate=0.001)
+      loss = 'mse'
+      metrics = ['mae']
+      model.compile(optimizer, loss, metrics=metrics,
+                    run_eagerly=testing_utils.should_run_eagerly())
+
+      inputs = np.zeros((10, 3))
+      targets = np.zeros((10, 4))
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
+
+      model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+
+  if __name__ == "__main__":
+    tf.test.main()
+  ```
+
+  This test will try compiling & fitting the small functional mlp using all
+  three Keras execution modes.
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+    config: An optional config_pb2.ConfigProto to use to configure the
+      session when executing graphs.
+    always_skip_v1: If True, does not try running the legacy graph mode even
+      when Tensorflow v2 behavior is not enabled.
+    extra_parameters: A list of dicts containing additional parameters for
+      parameterizing the decorated test method. This additional parameters must
+      be arguments in the test method's signature. See the
+      absl.testing.parameterized  decorators for more info on how these work.
+
+  Returns:
+    Returns a decorator that will run the decorated test method multiple times.
+
+  Raises:
+    ImportError: If abseil parameterized is not installed or not included as
+      a target dependency.
+  """
+  return run_all_keras_modes_with_all_model_types(
+      func=func,
+      skip_model_types=True,
+      config=config,
+      always_skip_v1=always_skip_v1,
+      extra_parameters=extra_parameters)
+
+
+def run_all_keras_modes_with_all_model_types(
+    func=None,
+    skip_run_modes=False,
+    skip_model_types=False,
+    config=None,
+    always_skip_v1=False,
+    exclude_models=None,
+    extra_parameters=None):
+  """Execute the decorated test with all keras run modes and model types.
+
+  This decorator is intended to be applied to test methods in
+  a `testing_utils.KerasTestCase` class. Doing so will cause the contents of
+  the test method to be executed once for each combination of Keras run mode
+  and model type.
+
+  Note: This decorator does not stack with any other parameterizing
+  decorators. Use the extra_parameters arg if you would like to use additional
+  parameterization.
+
+  For example, consider the following unittest:
+
+  ```python
+  class MyTests(testing_utils.KerasTestCase):
+
+    @testing_utils.run_all_keras_modes_with_all_model_types
+    def test_foo(self):
+      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+      optimizer = RMSPropOptimizer(learning_rate=0.001)
+      loss = 'mse'
+      metrics = ['mae']
+      model.compile(optimizer, loss, metrics=metrics,
+                    run_eagerly=testing_utils.should_run_eagerly())
+
+      inputs = np.zeros((10, 3))
+      targets = np.zeros((10, 4))
+      dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+      dataset = dataset.repeat(100)
+      dataset = dataset.batch(10)
+
+      model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
+
+  if __name__ == "__main__":
+    tf.test.main()
+  ```
+
+  This test will try compiling & fitting the small functional mlp using all
+  combinations of the Keras execution modes and the Keras model types.
+
+  The three Keras execution modes are v1 graph mode, v2 functions, and v2 eager.
+  If Tensorflow v2 behavior is enabled, v1 graph mode will be skipped.
+  The Keras model types are: ['functional', 'subclass', 'sequential']
+
+  Args:
+    func: function to be annotated. If `func` is None, this method returns a
+      decorator the can be applied to a function. If `func` is not None this
+      returns the decorator applied to `func`.
+    skip_run_modes: If True, does not try testing different Keras run modes.
+      Defaults to False, which tests all combinations of run modes.
+    skip_model_types: If True, does not try testing different Keras model types.
+      Defaults to False, which tests all combinations of model types.
+    config: An optional config_pb2.ConfigProto to use to configure the
+      session when executing graphs.
+    always_skip_v1: If True, does not try running the legacy graph mode even
+      when Tensorflow v2 behavior is not enabled.
+    exclude_models: A collection of Keras model types to not run.
+      (May also be a single model type not wrapped in a collection).
+    extra_parameters: A list of dicts containing additional parameters for
+      parameterizing the decorated test method. This additional parameters must
+      be arguments in the test method's signature. See the
+      absl.testing.parameterized decorators for more info on how these work.
+
+  Returns:
+    Returns a decorator that will run the decorated test method multiple times.
+
+  Raises:
+    ImportError: If abseil parameterized is not installed or not included as
+      a target dependency.
+  """
+  if not parameterized:
+    raise ImportError('To use the Keras parameterized testing utilities, you '
+                      'must have absl.testing.parameterized installed and '
+                      'included as a target dependency.')
+  params = []
+  if not skip_run_modes:
+    if not (always_skip_v1 or tf2.enabled()):
+      params.append({'testcase_name': '_v1_graph', '_run_mode': 'v1_graph'})
+    params.append({'testcase_name': '_v2_eager', '_run_mode': 'v2_eager'})
+    params.append({'testcase_name': '_v2_function', '_run_mode': 'v2_function'})
+  else:
+    params.append({'testcase_name': ''})
+
+  if not skip_model_types:
+    model_types = ['functional', 'subclass', 'sequential']
+    old_params = params
+    params = []
+    for param in old_params:
+      for model in model_types:
+        if model not in nest.flatten(exclude_models):
+          new_param = param.copy()
+          new_param['_model_type'] = model
+          new_param['testcase_name'] = '{}_{}'.format(
+              param['testcase_name'], model)
+          params.append(new_param)
+
+  if extra_parameters:
+    old_params = params
+    params = []
+    for param in old_params:
+      for i, extra_params in enumerate(extra_parameters):
+        new_param = param.copy()
+        new_param.update(extra_params)
+        new_param['testcase_name'] = '{}_{}'.format(
+            param['testcase_name'], i)
+        params.append(new_param)
+
+  def decorator(f):
+    """Decorator that constructs the test cases."""
+    def run_with_model_type(self, model_type, *args, **kwargs):
+      if model_type:
+        with testing_utils.model_type_scope(model_type):
+          f(self, *args, **kwargs)
+      else:
+        f(self, *args, **kwargs)
+
+    # Use named_parameters so it can be individually run from the command line
+    @parameterized.named_parameters(
+        *params
+    )
+    def decorated(self, *args, **kwargs):
+      """A run of a single test case w/ specified run mode / model type."""
+      run_mode = kwargs.pop('_run_mode', None)
+      model_type = kwargs.pop('_model_type', None)
+      if run_mode is None:
+        run_with_model_type(self, model_type, *args, **kwargs)
+      elif run_mode == 'v1_graph':
+        with context.graph_mode(), testing_utils.run_eagerly_scope(False):
+          with self.test_session(use_gpu=True, config=config):
+            run_with_model_type(self, model_type, *args, **kwargs)
+      elif run_mode == 'v2_function':
+        with context.eager_mode():
+          with testing_utils.run_eagerly_scope(False):
+            run_with_model_type(self, model_type, *args, **kwargs)
+      elif run_mode == 'v2_eager':
+        with context.eager_mode():
+          with testing_utils.run_eagerly_scope(True):
+            run_with_model_type(self, model_type, *args, **kwargs)
+      else:
+        return ValueError('Unknown run mode %s' % run_mode)
+
+    return decorated
+
+  if func is not None:
+    return decorator(func)
+
+  return decorator
diff --git a/tensorflow/python/keras/keras_parameterized_test.py b/tensorflow/python/keras/keras_parameterized_test.py
new file mode 100644
index 0000000000..104c3b1ec9
--- /dev/null
+++ b/tensorflow/python/keras/keras_parameterized_test.py
@@ -0,0 +1,381 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Keras testing_utils."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+
+from tensorflow.python import keras
+from tensorflow.python import tf2
+from tensorflow.python.eager import context
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import googletest
+
+
+class KerasParameterizedTest(keras_parameterized.TestCase):
+
+  def test_run_with_all_model_types(self):
+    model_types = []
+    models = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_with_all_model_types
+      def testBody(self):
+        model_types.append(testing_utils.get_model_type())
+        models.append(testing_utils.get_small_mlp(1, 4, input_dim=3))
+
+    e = ExampleTest()
+    e.testBody_functional()
+    e.testBody_subclass()
+    e.testBody_sequential()
+
+    self.assertLen(model_types, 3)
+    self.assertAllEqual(model_types, [
+        "functional",
+        "subclass",
+        "sequential"
+    ])
+
+    # Validate that the models are what they should be
+    self.assertTrue(models[0]._is_graph_network)
+    self.assertFalse(models[1]._is_graph_network)
+    self.assertNotIsInstance(models[0], keras.models.Sequential)
+    self.assertNotIsInstance(models[1], keras.models.Sequential)
+    self.assertIsInstance(models[2], keras.models.Sequential)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(model_types, 6)
+
+  def test_run_with_all_model_types_and_extra_params(self):
+    model_types = []
+    models = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_with_all_model_types(
+          extra_parameters=[dict(with_brackets=True), dict(with_brackets=False)]
+      )
+      def testBody(self, with_brackets):
+        with_brackets = "with_brackets" if with_brackets else "without_brackets"
+        model_types.append((with_brackets, testing_utils.get_model_type()))
+        models.append(testing_utils.get_small_mlp(1, 4, input_dim=3))
+
+    e = ExampleTest()
+    e.testBody_functional_0()
+    e.testBody_subclass_0()
+    e.testBody_sequential_0()
+    e.testBody_functional_1()
+    e.testBody_subclass_1()
+    e.testBody_sequential_1()
+
+    self.assertLen(model_types, 6)
+    self.assertAllEqual(model_types, [
+        ("with_brackets", "functional"),
+        ("with_brackets", "subclass"),
+        ("with_brackets", "sequential"),
+        ("without_brackets", "functional"),
+        ("without_brackets", "subclass"),
+        ("without_brackets", "sequential"),
+    ])
+
+    # Validate that the models are what they should be
+    self.assertTrue(models[0]._is_graph_network)
+    self.assertFalse(models[1]._is_graph_network)
+    self.assertNotIsInstance(models[0], keras.models.Sequential)
+    self.assertNotIsInstance(models[1], keras.models.Sequential)
+    self.assertIsInstance(models[2], keras.models.Sequential)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(model_types, 12)
+
+  def test_run_with_all_model_types_exclude_one(self):
+    model_types = []
+    models = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_with_all_model_types(exclude_models="sequential")
+      def testBody(self):
+        model_types.append(testing_utils.get_model_type())
+        models.append(testing_utils.get_small_mlp(1, 4, input_dim=3))
+
+    e = ExampleTest()
+    if hasattr(e, "testBody_functional"):
+      e.testBody_functional()
+    if hasattr(e, "testBody_subclass"):
+      e.testBody_subclass()
+    if hasattr(e, "testBody_sequential"):
+      e.testBody_sequential()
+
+    self.assertLen(model_types, 2)
+    self.assertAllEqual(model_types, [
+        "functional",
+        "subclass"
+    ])
+
+    # Validate that the models are what they should be
+    self.assertTrue(models[0]._is_graph_network)
+    self.assertFalse(models[1]._is_graph_network)
+    self.assertNotIsInstance(models[0], keras.models.Sequential)
+    self.assertNotIsInstance(models[1], keras.models.Sequential)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(model_types, 4)
+
+  def test_run_with_all_model_types_exclude_multiple(self):
+    model_types = []
+    models = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_with_all_model_types(
+          exclude_models=["sequential", "functional"])
+      def testBody(self):
+        model_types.append(testing_utils.get_model_type())
+        models.append(testing_utils.get_small_mlp(1, 4, input_dim=3))
+
+    e = ExampleTest()
+    if hasattr(e, "testBody_functional"):
+      e.testBody_functional()
+    if hasattr(e, "testBody_subclass"):
+      e.testBody_subclass()
+    if hasattr(e, "testBody_sequential"):
+      e.testBody_sequential()
+
+    self.assertLen(model_types, 1)
+    self.assertAllEqual(model_types, [
+        "subclass"
+    ])
+
+    # Validate that the models are what they should be
+    self.assertFalse(models[0]._is_graph_network)
+    self.assertNotIsInstance(models[0], keras.models.Sequential)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(model_types, 2)
+
+  def test_run_all_keras_modes(self):
+    l = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_all_keras_modes
+      def testBody(self):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((mode, should_run_eagerly))
+
+    e = ExampleTest()
+    if not tf2.enabled():
+      e.testBody_v1_graph()
+    e.testBody_v2_eager()
+    e.testBody_v2_function()
+
+    if not tf2.enabled():
+      self.assertLen(l, 3)
+      self.assertAllEqual(l, [
+          ("graph", False),
+          ("eager", True),
+          ("eager", False),
+      ])
+
+      ts = unittest.makeSuite(ExampleTest)
+      res = unittest.TestResult()
+      ts.run(res)
+      self.assertLen(l, 6)
+    else:
+      self.assertLen(l, 2)
+      self.assertAllEqual(l, [
+          ("eager", True),
+          ("eager", False),
+      ])
+
+      ts = unittest.makeSuite(ExampleTest)
+      res = unittest.TestResult()
+      ts.run(res)
+      self.assertLen(l, 4)
+
+  def test_run_all_keras_modes_extra_params(self):
+    l = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_all_keras_modes(
+          extra_parameters=[dict(with_brackets=True), dict(with_brackets=False)]
+      )
+      def testBody(self, with_brackets):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        with_brackets = "with_brackets" if with_brackets else "without_brackets"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((with_brackets, mode, should_run_eagerly))
+
+    e = ExampleTest()
+    if not tf2.enabled():
+      e.testBody_v1_graph_0()
+      e.testBody_v1_graph_1()
+
+    e.testBody_v2_eager_0()
+    e.testBody_v2_function_0()
+    e.testBody_v2_eager_1()
+    e.testBody_v2_function_1()
+
+    expected_combinations = {
+        ("with_brackets", "eager", True),
+        ("with_brackets", "eager", False),
+        ("without_brackets", "eager", True),
+        ("without_brackets", "eager", False),
+    }
+
+    if not tf2.enabled():
+      expected_combinations = expected_combinations.union({
+          ("with_brackets", "graph", False),
+          ("without_brackets", "graph", False),
+      })
+
+    self.assertLen(l, len(expected_combinations))
+    self.assertEqual(set(l), expected_combinations)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(l, len(expected_combinations) * 2)
+
+  def test_run_all_keras_modes_always_skip_v1(self):
+    l = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+      def testBody(self):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((mode, should_run_eagerly))
+
+    e = ExampleTest()
+    if hasattr(e, "testBody_v1_graph"):
+      e.testBody_v1_graph()
+    if hasattr(e, "testBody_v2_eager"):
+      e.testBody_v2_eager()
+    if hasattr(e, "testBody_v2_function"):
+      e.testBody_v2_function()
+
+    self.assertLen(l, 2)
+    self.assertEqual(set(l), {
+        ("eager", True),
+        ("eager", False),
+    })
+
+  def test_run_all_keras_modes_with_all_model_types(self):
+    l = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_all_keras_modes_with_all_model_types
+      def testBody(self):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((mode, should_run_eagerly, testing_utils.get_model_type()))
+
+    e = ExampleTest()
+    e.testBody_v2_eager_functional()
+    e.testBody_v2_function_functional()
+    e.testBody_v2_eager_sequential()
+    e.testBody_v2_function_sequential()
+    e.testBody_v2_eager_subclass()
+    e.testBody_v2_function_subclass()
+
+    if not tf2.enabled():
+      e.testBody_v1_graph_functional()
+      e.testBody_v1_graph_sequential()
+      e.testBody_v1_graph_subclass()
+
+    expected_combinations = {
+        ("eager", True, "functional"),
+        ("eager", False, "functional"),
+        ("eager", True, "sequential"),
+        ("eager", False, "sequential"),
+        ("eager", True, "subclass"),
+        ("eager", False, "subclass"),
+    }
+
+    if not tf2.enabled():
+      expected_combinations = expected_combinations.union({
+          ("graph", False, "functional"),
+          ("graph", False, "sequential"),
+          ("graph", False, "subclass"),
+      })
+
+    self.assertLen(l, len(expected_combinations))
+    self.assertEqual(set(l), expected_combinations)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(l, len(expected_combinations) * 2)
+
+  @keras_parameterized.run_all_keras_modes(extra_parameters=[dict(arg=True)])
+  def test_run_all_keras_modes_extra_params_2(self, arg):
+    self.assertEqual(arg, True)
+
+  @keras_parameterized.run_with_all_model_types(
+      extra_parameters=[dict(arg=True)])
+  def test_run_with_all_model_types_extra_params_2(self, arg):
+    self.assertEqual(arg, True)
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index 7827e16627..59ed77695a 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -18,11 +18,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import threading
+
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
+from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
 
 
@@ -194,6 +198,74 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
   return actual_output
 
 
+_thread_local_data = threading.local()
+_thread_local_data.model_type = None
+_thread_local_data.run_eagerly = None
+
+
+@tf_contextlib.contextmanager
+def model_type_scope(value):
+  """Provides a scope within which the model type to test is equal to `value`.
+
+  The model type gets restored to its original value upon exiting the scope.
+
+  Arguments:
+     value: model type value
+
+  Yields:
+    The provided value.
+  """
+  previous_value = _thread_local_data.model_type
+  try:
+    _thread_local_data.model_type = value
+    yield value
+  finally:
+    # Restore model type to initial value.
+    _thread_local_data.model_type = previous_value
+
+
+@tf_contextlib.contextmanager
+def run_eagerly_scope(value):
+  """Provides a scope within which we compile models to run eagerly or not.
+
+  The boolean gets restored to its original value upon exiting the scope.
+
+  Arguments:
+     value: Bool specifying if we should run models eagerly in the active test.
+     Should be True or False.
+
+  Yields:
+    The provided value.
+  """
+  previous_value = _thread_local_data.run_eagerly
+  try:
+    _thread_local_data.run_eagerly = value
+    yield value
+  finally:
+    # Restore model type to initial value.
+    _thread_local_data.run_eagerly = previous_value
+
+
+def should_run_eagerly():
+  """Returns whether the models we are testing should be run eagerly."""
+  if _thread_local_data.run_eagerly is None:
+    raise ValueError('Cannot call `should_run_eagerly()` outside of a '
+                     '`run_eagerly_scope()` or `run_all_keras_modes` '
+                     'decorator.')
+
+  return _thread_local_data.run_eagerly and context.executing_eagerly()
+
+
+def get_model_type():
+  """Gets the model type that should be tested."""
+  if _thread_local_data.model_type is None:
+    raise ValueError('Cannot call `get_model_type()` outside of a '
+                     '`model_type_scope()` or `run_with_all_model_types` '
+                     'decorator.')
+
+  return _thread_local_data.model_type
+
+
 def get_small_sequential_mlp(num_hidden, num_classes, input_dim=None):
   model = keras.models.Sequential()
   if input_dim:
@@ -212,3 +284,337 @@ def get_small_functional_mlp(num_hidden, num_classes, input_dim):
   activation = 'sigmoid' if num_classes == 1 else 'softmax'
   outputs = keras.layers.Dense(num_classes, activation=activation)(outputs)
   return keras.Model(inputs, outputs)
+
+
+class _SmallSubclassMLP(keras.Model):
+  """A subclass model based small MLP."""
+
+  def __init__(self, num_hidden, num_classes):
+    super(_SmallSubclassMLP, self).__init__()
+    self.layer_a = keras.layers.Dense(num_hidden, activation='relu')
+    activation = 'sigmoid' if num_classes == 1 else 'softmax'
+    self.layer_b = keras.layers.Dense(num_classes, activation=activation)
+
+  def call(self, inputs, **kwargs):
+    x = self.layer_a(inputs)
+    return self.layer_b(x)
+
+
+class _SmallSubclassMLPCustomBuild(keras.Model):
+  """A subclass model small MLP that uses a custom build method."""
+
+  def __init__(self, num_hidden, num_classes):
+    super(_SmallSubclassMLPCustomBuild, self).__init__()
+    self.layer_a = None
+    self.layer_b = None
+    self.num_hidden = num_hidden
+    self.num_classes = num_classes
+
+  def build(self, input_shape):
+    self.layer_a = keras.layers.Dense(self.num_hidden, activation='relu')
+    activation = 'sigmoid' if self.num_classes == 1 else 'softmax'
+    self.layer_b = keras.layers.Dense(self.num_classes, activation=activation)
+
+  def call(self, inputs, **kwargs):
+    x = self.layer_a(inputs)
+    return self.layer_b(x)
+
+
+def get_small_subclass_mlp(num_hidden, num_classes):
+  return _SmallSubclassMLP(num_hidden, num_classes)
+
+
+def get_small_subclass_mlp_with_custom_build(num_hidden, num_classes):
+  return _SmallSubclassMLPCustomBuild(num_hidden, num_classes)
+
+
+def get_small_mlp(num_hidden, num_classes, input_dim):
+  """Get a small mlp of the model type specified by `get_model_type`."""
+  model_type = get_model_type()
+  if model_type == 'subclass':
+    return get_small_subclass_mlp(num_hidden, num_classes)
+  if model_type == 'subclass_custom_build':
+    return get_small_subclass_mlp_with_custom_build(num_hidden, num_classes)
+  if model_type == 'sequential':
+    return get_small_sequential_mlp(num_hidden, num_classes, input_dim)
+  if model_type == 'functional':
+    return get_small_functional_mlp(num_hidden, num_classes, input_dim)
+  raise ValueError('Unknown model type {}'.format(model_type))
+
+
+class _SubclassModel(keras.Model):
+  """A Keras subclass model."""
+
+  def __init__(self, layers):
+    super(_SubclassModel, self).__init__()
+    self.all_layers = layers
+
+  def call(self, inputs, **kwargs):
+    x = inputs
+    for layer in self.all_layers:
+      x = layer(x)
+    return x
+
+
+class _SubclassModelCustomBuild(keras.Model):
+  """A Keras subclass model that uses a custom build method."""
+
+  def __init__(self, layer_generating_func):
+    super(_SubclassModelCustomBuild, self).__init__()
+    self.all_layers = None
+    self._layer_generating_func = layer_generating_func
+
+  def build(self, input_shape):
+    layers = []
+    for layer in self._layer_generating_func():
+      layers.append(layer)
+    self.all_layers = layers
+
+  def call(self, inputs, **kwargs):
+    x = inputs
+    for layer in self.all_layers:
+      x = layer(x)
+    return x
+
+
+def get_model_from_layers(layers, input_shape=None):
+  """Builds a model from a sequence of layers."""
+  model_type = get_model_type()
+  if model_type == 'subclass':
+    return _SubclassModel(layers)
+
+  if model_type == 'subclass_custom_build':
+    layer_generating_func = lambda: layers
+    return _SubclassModelCustomBuild(layer_generating_func)
+
+  if model_type == 'sequential':
+    model = keras.models.Sequential()
+    if input_shape:
+      model.add(keras.layers.InputLayer(input_shape=input_shape))
+    for layer in layers:
+      model.add(layer)
+    return model
+
+  if model_type == 'functional':
+    if not input_shape:
+      raise ValueError('Cannot create a functional model from layers with no '
+                       'input shape.')
+    inputs = keras.Input(shape=input_shape)
+    outputs = inputs
+    for layer in layers:
+      outputs = layer(outputs)
+    return keras.Model(inputs, outputs)
+
+  raise ValueError('Unknown model type {}'.format(model_type))
+
+
+class _MultiIOSubclassModel(keras.Model):
+  """Multi IO Keras subclass model."""
+
+  def __init__(self, branch_a, branch_b, shared_input_branch=None,
+               shared_output_branch=None):
+    super(_MultiIOSubclassModel, self).__init__()
+    self._shared_input_branch = shared_input_branch
+    self._branch_a = branch_a
+    self._branch_b = branch_b
+    self._shared_output_branch = shared_output_branch
+
+  def call(self, inputs, **kwargs):
+    if self._shared_input_branch:
+      for layer in self._shared_input_branch:
+        inputs = layer(inputs)
+      a = inputs
+      b = inputs
+    else:
+      a, b = inputs
+
+    for layer in self._branch_a:
+      a = layer(a)
+    for layer in self._branch_b:
+      b = layer(b)
+    outs = [a, b]
+
+    if self._shared_output_branch:
+      for layer in self._shared_output_branch:
+        outs = layer(outs)
+
+    return outs
+
+
+class _MultiIOSubclassModelCustomBuild(keras.Model):
+  """Multi IO Keras subclass model that uses a custom build method."""
+
+  def __init__(self, branch_a_func, branch_b_func,
+               shared_input_branch_func=None,
+               shared_output_branch_func=None):
+    super(_MultiIOSubclassModelCustomBuild, self).__init__()
+    self._shared_input_branch_func = shared_input_branch_func
+    self._branch_a_func = branch_a_func
+    self._branch_b_func = branch_b_func
+    self._shared_output_branch_func = shared_output_branch_func
+
+    self._shared_input_branch = None
+    self._branch_a = None
+    self._branch_b = None
+    self._shared_output_branch = None
+
+  def build(self, input_shape):
+    if self._shared_input_branch_func():
+      self._shared_input_branch = self._shared_input_branch_func()
+    self._branch_a = self._branch_a_func()
+    self._branch_b = self._branch_b_func()
+
+    if self._shared_output_branch_func():
+      self._shared_output_branch = self._shared_output_branch_func()
+
+  def call(self, inputs, **kwargs):
+    if self._shared_input_branch:
+      for layer in self._shared_input_branch:
+        inputs = layer(inputs)
+      a = inputs
+      b = inputs
+    else:
+      a, b = inputs
+
+    for layer in self._branch_a:
+      a = layer(a)
+    for layer in self._branch_b:
+      b = layer(b)
+    outs = a, b
+
+    if self._shared_output_branch:
+      for layer in self._shared_output_branch:
+        outs = layer(outs)
+
+    return outs
+
+
+def get_multi_io_model(
+    branch_a,
+    branch_b,
+    shared_input_branch=None,
+    shared_output_branch=None):
+  """Builds a multi-io model that contains two branches.
+
+  The produced model will be of the type specified by `get_model_type`.
+
+  To build a two-input, two-output model:
+    Specify a list of layers for branch a and branch b, but do not specify any
+    shared input branch or shared output branch. The resulting model will apply
+    each branch to a different input, to produce two outputs.
+
+    The first value in branch_a must be the Keras 'Input' layer for branch a,
+    and the first value in branch_b must be the Keras 'Input' layer for
+    branch b.
+
+    example usage:
+    ```
+    branch_a = [Input(shape=(2,), name='a'), Dense(), Dense()]
+    branch_b = [Input(shape=(3,), name='b'), Dense(), Dense()]
+
+    model = get_multi_io_model(branch_a, branch_b)
+    ```
+
+  To build a two-input, one-output model:
+    Specify a list of layers for branch a and branch b, and specify a
+    shared output branch. The resulting model will apply
+    each branch to a different input. It will then apply the shared output
+    branch to a tuple containing the intermediate outputs of each branch,
+    to produce a single output. The first layer in the shared_output_branch
+    must be able to merge a tuple of two tensors.
+
+    The first value in branch_a must be the Keras 'Input' layer for branch a,
+    and the first value in branch_b must be the Keras 'Input' layer for
+    branch b.
+
+    example usage:
+    ```
+    input_branch_a = [Input(shape=(2,), name='a'), Dense(), Dense()]
+    input_branch_b = [Input(shape=(3,), name='b'), Dense(), Dense()]
+    shared_output_branch = [Concatenate(), Dense(), Dense()]
+
+    model = get_multi_io_model(input_branch_a, input_branch_b,
+                               shared_output_branch=shared_output_branch)
+    ```
+  To build a one-input, two-output model:
+    Specify a list of layers for branch a and branch b, and specify a
+    shared input branch. The resulting model will take one input, and apply
+    the shared input branch to it. It will then respectively apply each branch
+    to that intermediate result in parallel, to produce two outputs.
+
+    The first value in the shared_input_branch must be the Keras 'Input' layer
+    for the whole model. Branch a and branch b should not contain any Input
+    layers.
+
+    example usage:
+    ```
+    shared_input_branch = [Input(shape=(2,), name='in'), Dense(), Dense()]
+    output_branch_a = [Dense(), Dense()]
+    output_branch_b = [Dense(), Dense()]
+
+
+    model = get_multi_io_model(output__branch_a, output_branch_b,
+                               shared_input_branch=shared_input_branch)
+    ```
+
+  Args:
+    branch_a: A sequence of layers for branch a of the model.
+    branch_b: A sequence of layers for branch b of the model.
+    shared_input_branch: An optional sequence of layers to apply to a single
+      input, before applying both branches to that intermediate result. If set,
+      the model will take only one input instead of two. Defaults to None.
+    shared_output_branch: An optional sequence of layers to merge the
+      intermediate results produced by branch a and branch b. If set,
+      the model will produce only one output instead of two. Defaults to None.
+
+  Returns:
+    A multi-io model of the type specified by `get_model_type`, specified
+    by the different branches.
+  """
+  # Extract the functional inputs from the layer lists
+  if shared_input_branch:
+    inputs = shared_input_branch[0]
+    shared_input_branch = shared_input_branch[1:]
+  else:
+    inputs = branch_a[0], branch_b[0]
+    branch_a = branch_a[1:]
+    branch_b = branch_b[1:]
+
+  model_type = get_model_type()
+  if model_type == 'subclass':
+    return _MultiIOSubclassModel(branch_a, branch_b, shared_input_branch,
+                                 shared_output_branch)
+
+  if model_type == 'subclass_custom_build':
+    return _MultiIOSubclassModelCustomBuild((lambda: branch_a),
+                                            (lambda: branch_b),
+                                            (lambda: shared_input_branch),
+                                            (lambda: shared_output_branch))
+
+  if model_type == 'sequential':
+    raise ValueError('Cannot use `get_multi_io_model` to construct '
+                     'sequential models')
+
+  if model_type == 'functional':
+    if shared_input_branch:
+      a_and_b = inputs
+      for layer in shared_input_branch:
+        a_and_b = layer(a_and_b)
+      a = a_and_b
+      b = a_and_b
+    else:
+      a, b = inputs
+
+    for layer in branch_a:
+      a = layer(a)
+    for layer in branch_b:
+      b = layer(b)
+    outputs = a, b
+
+    if shared_output_branch:
+      for layer in shared_output_branch:
+        outputs = layer(outputs)
+
+    return keras.Model(inputs, outputs)
+
+  raise ValueError('Unknown model type {}'.format(model_type))
-- 
GitLab


From 77e79523cd173bce7814b8d1113fd08dde4df50d Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 4 Dec 2018 14:01:56 -0800
Subject: [PATCH 1313/1554] [TF Python] Avoid leaking a reference to a
 _typeobject during concurrent first lookups.

Also switch to using tf_shared_lock on the initial cache lookup.

PiperOrigin-RevId: 224041975
---
 tensorflow/python/util/util.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
index 11eb9ce947..e69eec73a0 100644
--- a/tensorflow/python/util/util.cc
+++ b/tensorflow/python/util/util.cc
@@ -172,7 +172,7 @@ class CachedTypeCheck {
     auto* type = Py_TYPE(o);
 
     {
-      mutex_lock l(type_to_sequence_map_mu_);
+      tf_shared_lock l(type_to_sequence_map_mu_);
       auto it = type_to_sequence_map_.find(type);
       if (it != type_to_sequence_map_.end()) {
         return it->second;
@@ -195,7 +195,12 @@ class CachedTypeCheck {
       mutex_lock l(type_to_sequence_map_mu_);
       if (type_to_sequence_map_.size() < kMaxItemsInCache) {
         Py_INCREF(type);
-        type_to_sequence_map_.insert({type, check_result});
+        auto insert_result = type_to_sequence_map_.insert({type, check_result});
+        if (!insert_result.second) {
+          // The type was added to the cache by a concurrent thread after we
+          // looked it up above.
+          Py_DECREF(type);
+        }
       }
     }
 
-- 
GitLab


From ea4dcc49ffe7cf83191d93911f3713755632203c Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 4 Dec 2018 14:07:00 -0800
Subject: [PATCH 1314/1554] [XLA:GPU] Emit MOF tuples using just one thread.

Previously all threads in a multi-output fusion would emit the tuple pointers,
now just one thread does it.

PiperOrigin-RevId: 224043124
---
 .../xla/service/gpu/ir_emission_utils.cc      | 12 ++++
 .../xla/service/gpu/ir_emission_utils.h       |  4 ++
 .../xla/service/gpu/ir_emitter_unnested.cc    | 69 +++++++------------
 3 files changed, 42 insertions(+), 43 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index 42fb38dffa..33e41a2782 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -268,5 +268,17 @@ string CudnnConvKindToString(CudnnConvKind kind) {
   }
 }
 
+llvm::Value* IsBlock0Thread0(llvm::IRBuilder<>* b) {
+  return b->CreateAnd(
+      b->CreateICmpEQ(
+          b->getInt32(0),
+          llvm_ir::EmitCallToIntrinsic(
+              llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b)),
+      b->CreateICmpEQ(
+          b->getInt32(0),
+          llvm_ir::EmitCallToIntrinsic(
+              llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, b)));
+}
+
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h
index f373d4a839..ebf4d926b7 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h
@@ -155,6 +155,10 @@ llvm::Value* EmitPrintf(absl::string_view fmt,
 llvm::Value* EmitFullWarpShuffleDown(llvm::Value* value, llvm::Value* offset,
                                      llvm::IRBuilder<>* builder);
 
+// Emits code that determines whether the current thread is thread 0 within
+// block 0 of the kernel.
+llvm::Value* IsBlock0Thread0(llvm::IRBuilder<>* b);
+
 }  // namespace gpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index bbe1583c01..38b8923173 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -2059,8 +2059,18 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
             GetIndexTypeForKernel(&hlo, launch_dimensions.launch_bound(), &b_));
   }
 
-  // For multioutput fusion, we need to emit each operand and the root.
+  // Emit the tuple pointers in one thread.  We could do this at any point in
+  // the kernel, but we do it at the beginning in the hopes of reducing register
+  // pressure, since we touch threadIdx.x and blockIdx.x at the beginning of the
+  // kernel *anyway*.
   std::vector<IrArray> output_arrays = ConstructIrArrayForOutputs(hlo);
+  TF_RETURN_IF_ERROR(
+      KernelSupportLibrary(&b_).If("emit_mof_tuple", IsBlock0Thread0(&b_), [&] {
+        llvm_ir::EmitTuple(GetIrArray(hlo, hlo), output_arrays, &b_, module_);
+        return Status::OK();
+      }));
+
+  // For multioutput fusion, we need to emit each operand and the root.
   TF_RETURN_IF_ERROR(
       ParallelLoopEmitter(element_generator, output_arrays, launch_dimensions,
                           &b_, unroll_factor)
@@ -2069,8 +2079,6 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
                         &hlo, launch_dimensions.launch_bound(), &b_)));
 
   b_.SetInsertPoint(b_.GetInsertBlock()->getTerminator());
-  llvm_ir::EmitTuple(GetIrArray(hlo, hlo), output_arrays, &b_, module_);
-
   return Status::OK();
 }
 
@@ -2130,37 +2138,6 @@ int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape(
 
 namespace {
 
-// Reads thread_idx.x and converts it to a (y,x) coordinate, assuming that the
-// thread lives within a square tile of size tile_size (so thread blocks are of
-// size tile_size * tile_size).
-std::tuple<llvm::Value*, llvm::Value*> CalculateYXCoordinateWithinTile(
-    llvm::IRBuilder<>* builder, llvm::Value* tile_size,
-    int64 threads_per_tile) {
-  // Calculate the starting element coordinate within a tile for the current
-  // thread, (y, x) from thread_id.
-  llvm::Value* thread_id = llvm_ir::EmitCallToIntrinsic(
-      llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, builder);
-  llvm_ir::AddRangeMetadata(0, threads_per_tile,
-                            llvm::cast<llvm::Instruction>(thread_id));
-  thread_id = builder->CreateIntCast(thread_id, tile_size->getType(),
-                                     /*isSigned=*/true, "thread.id.x");
-  auto x = builder->CreateURem(thread_id, tile_size);
-  auto y = builder->CreateUDiv(thread_id, tile_size);
-  return std::make_tuple(y, x);
-}
-
-// Reads block_idx.x, casts it to type index_ty, and adds the assumption that
-// it's in the range [0, num_blocks].
-llvm::Value* GetBlockIdx(llvm::IRBuilder<>* builder, llvm::Type* index_ty,
-                         int64 num_blocks) {
-  llvm::Value* block_id = llvm_ir::EmitCallToIntrinsic(
-      llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, builder);
-  llvm_ir::AddRangeMetadata(0, num_blocks,
-                            llvm::cast<llvm::Instruction>(block_id));
-  return builder->CreateIntCast(block_id, index_ty, /*isSigned=*/true,
-                                "block.id.x");
-}
-
 void EmitFullTile(const KernelMappingScheme* mapping_scheme,
                   const IrArray::Index& tile_origin_index,
                   llvm::IRBuilder<>* builder, llvm::Value* y, llvm::Value* x,
@@ -2872,6 +2849,21 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     return llvm::ConstantInt::get(index_ty, c);
   };
 
+  // For multioutput fusion, one thread needs to output a tuple with pointers to
+  // all the individual outputs.  We could do this at any point in the kernel,
+  // but we do it at the beginning in the hopes of reducing register pressure,
+  // since we touch threadIdx.x and blockIdx.x at the beginning of the kernel
+  // *anyway*.
+  if (unnested_hlo->IsMultiOutputFusion()) {
+    TF_CHECK_OK(KernelSupportLibrary(&b_).If(
+        "emit_mof_tuple", IsBlock0Thread0(&b_), [&] {
+          llvm_ir::EmitTuple(GetIrArray(*unnested_hlo, *unnested_hlo),
+                             ConstructIrArrayForOutputs(*unnested_hlo), &b_,
+                             module_);
+          return Status::OK();
+        }));
+  }
+
   // For each tiled parameter, cast its input IrArray to the corresponding
   // reduced shape and keep the reduced shape live during IR emission.
   std::vector<IrArray> param_in_reduced_shape_arrays;
@@ -2985,15 +2977,6 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     block_epilogue_generator(unnested_hlo, kernel_info);
   }
 
-  // For multioutput fusion, emit a tuple with pointers to all the individual
-  // outputs.
-  if (unnested_hlo->IsMultiOutputFusion()) {
-    std::vector<IrArray> output_arrays =
-        ConstructIrArrayForOutputs(*unnested_hlo);
-    llvm_ir::EmitTuple(GetIrArray(*unnested_hlo, *unnested_hlo), output_arrays,
-                       &b_, module_);
-  }
-
   return launch_dimensions;
 }
 
-- 
GitLab


From 3425af56dc3ed6ce023b53519d7dd35887f486c7 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 4 Dec 2018 14:08:48 -0800
Subject: [PATCH 1315/1554] Symbol renames for TF V2 that were skipped during a
 merge conflict.

PiperOrigin-RevId: 224043457
---
 tensorflow/tools/compatibility/renames_v2.py  |  4 --
 .../tools/compatibility/tf_upgrade_v2.py      | 11 +++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 41 +++++++++++++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 78839e36ba..7fe3d1e794 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -71,8 +71,6 @@ renames = {
     'tf.all_variables': 'tf.compat.v1.all_variables',
     'tf.angle': 'tf.math.angle',
     'tf.app.run': 'tf.compat.v1.app.run',
-    'tf.arg_max': 'tf.compat.v1.arg_max',
-    'tf.arg_min': 'tf.compat.v1.arg_min',
     'tf.assert_greater_equal': 'tf.compat.v1.assert_greater_equal',
     'tf.assert_integer': 'tf.compat.v1.assert_integer',
     'tf.assert_less_equal': 'tf.compat.v1.assert_less_equal',
@@ -383,8 +381,6 @@ renames = {
     'tf.nn.xw_plus_b': 'tf.compat.v1.nn.xw_plus_b',
     'tf.op_scope': 'tf.compat.v1.op_scope',
     'tf.orthogonal_initializer': 'tf.keras.initializers.Orthogonal',
-    'tf.parse_example': 'tf.compat.v1.parse_example',
-    'tf.parse_single_example': 'tf.compat.v1.parse_single_example',
     'tf.parse_single_sequence_example': 'tf.io.parse_single_sequence_example',
     'tf.parse_tensor': 'tf.io.parse_tensor',
     'tf.placeholder': 'tf.compat.v1.placeholder',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 655e680d5b..d19eb13336 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -469,6 +469,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.string",
         "tf.lite.constants.QUANTIZED_UINT8":
             "tf.uint8",
+        "tf.arg_max":
+            "tf.argmax",
+        "tf.arg_min":
+            "tf.argmin",
     }
     # pylint: enable=line-too-long
 
@@ -498,10 +502,17 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "data_format"
         ],
         "tf.nn.crelu": ["features", "name", "axis"],
+        "tf.nn.weighted_moments": [
+            "x", "axes", "frequency_weights", "name", "keep_dims"
+        ],
         "tf.nn.pool": [
             "input", "window_shape", "pooling_type", "padding", "dilation_rate",
             "strides", "name", "data_format"
         ],
+        "tf.nn.separable_conv2d": [
+            "input", "depthwise_filter", "pointwise_filter", "strides",
+            "padding", "rate", "name", "data_format"
+        ],
         "tf.nn.depthwise_conv2d": [
             "input", "filter", "strides", "padding", "rate", "name",
             "data_format"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b8b02c9c7f..bc92cab98c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -388,6 +388,11 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+    text = "tf.arg_min(input, 0)"
+    expected_text = "tf.argmin(input, 0)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
   def testArgmax(self):
     text = "tf.argmax(input, name=n, dimension=1, output_type=type)"
     expected_text = "tf.argmax(input=input, name=n, axis=1, output_type=type)"
@@ -399,6 +404,11 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+    text = "tf.arg_max(input, 0)"
+    expected_text = "tf.argmax(input, 0)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
   def testBatchToSpace(self):
     text = "tf.batch_to_space_nd(input, block_shape, crops, name)"
     expected_text = "tf.batch_to_space(input, block_shape, crops, name)"
@@ -451,6 +461,37 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testWeightedMoments(self):
+    text = "tf.nn.weighted_moments(x, axes, freq, name, kd)"
+    expected_text = (
+        "tf.nn.weighted_moments(x=x, axes=axes, frequency_weights=freq, "
+        "name=name, keep_dims=kd)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testSparseAdd(self):
+    text = "tf.sparse.add(a, b, t)"
+    expected_text = "tf.sparse.add(a=a, b=b, threshold=t)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testSparseConcat(self):
+    text = "tf.sparse.concat(ax, inp, name, exp, concat)"
+    expected_text = (
+        "tf.sparse.concat(axis=ax, sp_inputs=inp, name=name, "
+        "expand_nonconcat_dim=exp, axis=concat)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testSeparableConv2D(self):
+    text = "tf.nn.separable_conv2d(inp, d, pt, strides, pad, rate, name, fmt)"
+    expected_text = (
+        "tf.nn.separable_conv2d(input=inp, depthwise_filter=d, "
+        "pointwise_filter=pt, strides=strides, padding=pad, "
+        "dilations=rate, name=name, data_format=fmt)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 3b35c470e589073569028d6c3e82c8676b9d6690 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 4 Dec 2018 14:11:24 -0800
Subject: [PATCH 1316/1554] Automated rollback of commit
 6ff7ff868887aa6567344b030147dad7b5f8f3d2

PiperOrigin-RevId: 224043922
---
 tensorflow/c/c_api.cc                         | 57 ++++++++++---------
 tensorflow/compiler/jit/xla_launch_util.h     | 10 ++--
 .../common_runtime/gpu/gpu_event_mgr_test.cc  |  4 +-
 tensorflow/core/framework/tensor.cc           | 17 +++---
 tensorflow/core/framework/tensor.h            | 17 ++++--
 tensorflow/core/framework/tensor_test.cc      | 21 +++++++
 tensorflow/lite/delegates/flex/buffer_map.cc  | 57 ++++++++++---------
 tensorflow/python/lib/core/py_func.cc         |  6 +-
 8 files changed, 110 insertions(+), 79 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index f13e8777df..94d18eb8b0 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -136,16 +136,22 @@ const char* TF_Message(const TF_Status* s) {
 namespace {
 class TF_ManagedBuffer : public TensorBuffer {
  public:
-  void* data_;
-  size_t len_;
-  void (*deallocator_)(void* data, size_t len, void* arg);
-  void* deallocator_arg_;
+  TF_ManagedBuffer(void* data, size_t len,
+                   void (*deallocator)(void* data, size_t len, void* arg),
+                   void* deallocator_arg)
+      : TensorBuffer(data),
+        len_(len),
+        deallocator_(deallocator),
+        deallocator_arg_(deallocator_arg) {}
+
+  const size_t len_;
+  void (*const deallocator_)(void* data, size_t len, void* arg);
+  void* const deallocator_arg_;
 
   ~TF_ManagedBuffer() override {
-    (*deallocator_)(data_, len_, deallocator_arg_);
+    (*deallocator_)(data(), len_, deallocator_arg_);
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return len_; }
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -199,8 +205,7 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims,
     dimvec[i] = static_cast<tensorflow::int64>(dims[i]);
   }
 
-  TF_ManagedBuffer* buf = new TF_ManagedBuffer;
-  buf->len_ = len;
+  TF_ManagedBuffer* buf = nullptr;
   if (dtype != TF_STRING && dtype != TF_RESOURCE &&
       tensorflow::DataTypeCanUseMemcpy(static_cast<DataType>(dtype)) &&
       reinterpret_cast<intptr_t>(data) % std::max(1, EIGEN_MAX_ALIGN_BYTES) !=
@@ -212,17 +217,15 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims,
     //
     // Other types have the same representation, so copy only if it is safe to
     // do so.
-    buf->data_ = allocate_tensor("TF_NewTensor", len);
-    std::memcpy(buf->data_, data, len);
-    buf->deallocator_ = deallocate_buffer;
-    buf->deallocator_arg_ = nullptr;
+    buf = new TF_ManagedBuffer(allocate_tensor("TF_NewTensor", len), len,
+                               deallocate_buffer, nullptr);
+    std::memcpy(buf->data(), data, len);
     // Free the original buffer.
     deallocator(data, len, deallocator_arg);
   } else {
-    buf->data_ = data;
-    buf->deallocator_ = deallocator;
-    buf->deallocator_arg_ = deallocator_arg;
+    buf = new TF_ManagedBuffer(data, len, deallocator, deallocator_arg);
   }
+
   TF_Tensor* ret = new TF_Tensor{dtype, TensorShape(dimvec), buf};
   size_t elem_size = TF_DataTypeSize(dtype);
   if (elem_size > 0 && len < (elem_size * ret->shape.num_elements())) {
@@ -477,9 +480,9 @@ static TF_Tensor* EmptyTensor(TF_DataType dtype, const TensorShape& shape) {
   CHECK_EQ(nelems, 0);
   static_assert(sizeof(int64_t) == sizeof(tensorflow::int64),
                 "64-bit int types should match in size");
-  return TF_NewTensor(dtype, reinterpret_cast<const int64_t*>(dims.data()),
-                      shape.dims(), reinterpret_cast<void*>(&empty), 0,
-                      [](void*, size_t, void*) {}, nullptr);
+  return TF_NewTensor(
+      dtype, reinterpret_cast<const int64_t*>(dims.data()), shape.dims(),
+      reinterpret_cast<void*>(&empty), 0, [](void*, size_t, void*) {}, nullptr);
 }
 
 // Non-static for testing.
@@ -1592,18 +1595,20 @@ TF_AttrMetadata TF_OperationGetAttrMetadata(TF_Operation* oper,
     break;                                            \
   }
 
-      LIST_CASE(s, TF_ATTR_STRING, metadata.total_size = 0;
-                for (int i = 0; i < attr->list().s_size();
-                     ++i) { metadata.total_size += attr->list().s(i).size(); });
+      LIST_CASE(
+          s, TF_ATTR_STRING, metadata.total_size = 0;
+          for (int i = 0; i < attr->list().s_size();
+               ++i) { metadata.total_size += attr->list().s(i).size(); });
       LIST_CASE(i, TF_ATTR_INT);
       LIST_CASE(f, TF_ATTR_FLOAT);
       LIST_CASE(b, TF_ATTR_BOOL);
       LIST_CASE(type, TF_ATTR_TYPE);
-      LIST_CASE(shape, TF_ATTR_SHAPE, metadata.total_size = 0;
-                for (int i = 0; i < attr->list().shape_size(); ++i) {
-                  const auto& s = attr->list().shape(i);
-                  metadata.total_size += s.unknown_rank() ? 0 : s.dim_size();
-                });
+      LIST_CASE(
+          shape, TF_ATTR_SHAPE, metadata.total_size = 0;
+          for (int i = 0; i < attr->list().shape_size(); ++i) {
+            const auto& s = attr->list().shape(i);
+            metadata.total_size += s.unknown_rank() ? 0 : s.dim_size();
+          });
       LIST_CASE(tensor, TF_ATTR_TENSOR);
       LIST_CASE(tensor, TF_ATTR_FUNC);
 #undef LIST_CASE
diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 437db019a0..706811f55e 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -199,19 +199,17 @@ class XlaTensorBuffer : public TensorBuffer {
  public:
   XlaTensorBuffer(const void* ptr, size_t expected_size, size_t actual_size,
                   Allocator* allocator)
-      : expected_size_(expected_size),
+      : TensorBuffer(const_cast<void*>(ptr)),
+        expected_size_(expected_size),
         actual_size_(actual_size),
-        allocator_(allocator) {
-    data_ = const_cast<void*>(ptr);
-  }
+        allocator_(allocator) {}
 
   ~XlaTensorBuffer() override {
     if (data_) {
-      allocator_->DeallocateRaw(data_);
+      allocator_->DeallocateRaw(data());
     }
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return expected_size_; }
 
   TensorBuffer* root_buffer() override { return this; }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
index d2adf699f5..fe32147557 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
@@ -78,7 +78,8 @@ static std::atomic_int_fast64_t live_tensor_bytes(0);
 // A TensorBuffer that counts live memory usage for testing
 class TestTensorBuffer : public TensorBuffer {
  public:
-  explicit TestTensorBuffer(size_t bytes) : bytes_(bytes) {
+  explicit TestTensorBuffer(size_t bytes)
+      : TensorBuffer(nullptr), bytes_(bytes) {
     live_tensor_bytes += bytes_;
   }
   ~TestTensorBuffer() override { live_tensor_bytes -= bytes_; }
@@ -86,7 +87,6 @@ class TestTensorBuffer : public TensorBuffer {
   size_t size() const override { return bytes_; }
 
   // Not used in this test
-  void* data() const override { return nullptr; }
   TensorBuffer* root_buffer() override { return nullptr; }
   void FillAllocationDescription(AllocationDescription* arg) const override {}
 
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index c7ddc6c21e..7e841489eb 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -68,7 +68,8 @@ namespace {
 // An un-templated base class for Buffer.
 class BufferBase : public TensorBuffer {
  public:
-  explicit BufferBase(Allocator* alloc) : alloc_(alloc) {}
+  explicit BufferBase(Allocator* alloc, void* data_ptr)
+      : TensorBuffer(data_ptr), alloc_(alloc) {}
 
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -106,7 +107,6 @@ class Buffer : public BufferBase {
   Buffer(Allocator* a, int64 n);
   Buffer(Allocator* a, int64 n, const AllocationAttributes& allocation_attr);
 
-  void* data() const override { return data_; }
   size_t size() const override { return sizeof(T) * elem_; }
 
  private:
@@ -442,20 +442,20 @@ struct ProtoHelper<Eigen::half> {
 
 template <typename T>
 Buffer<T>::Buffer(Allocator* a, int64 n)
-    : BufferBase(a), data_(a->Allocate<T>(n)), elem_(n) {}
+    : BufferBase(a, a->Allocate<T>(n)), elem_(n) {}
 
 template <typename T>
 Buffer<T>::Buffer(Allocator* a, int64 n,
                   const AllocationAttributes& allocation_attr)
-    : BufferBase(a), data_(a->Allocate<T>(n, allocation_attr)), elem_(n) {}
+    : BufferBase(a, a->Allocate<T>(n, allocation_attr)), elem_(n) {}
 
 template <typename T>
 Buffer<T>::~Buffer() {
-  if (data_) {
+  if (data()) {
     if (LogMemory::IsEnabled()) {
       RecordDeallocation();
     }
-    alloc_->Deallocate<T>(data_, elem_);
+    alloc_->Deallocate<T>(static_cast<T*>(data()), elem_);
   }
 }
 
@@ -764,7 +764,9 @@ class SubBuffer : public TensorBuffer {
  public:
   // This buffer is an alias to buf[delta, delta + n).
   SubBuffer(TensorBuffer* buf, int64 delta, int64 n)
-      : root_(buf->root_buffer()), data_(buf->base<T>() + delta), elem_(n) {
+      : TensorBuffer(buf->base<T>() + delta),
+        root_(buf->root_buffer()),
+        elem_(n) {
     // Sanity check. The caller should ensure the sub buffer is valid.
     CHECK_LE(root_->base<T>(), this->base<T>());
     T* root_limit = root_->base<T>() + root_->size() / sizeof(T);
@@ -775,7 +777,6 @@ class SubBuffer : public TensorBuffer {
     root_->Ref();
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return sizeof(T) * elem_; }
   TensorBuffer* root_buffer() override { return root_; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 3177bbe7e9..6e03cf9f6f 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -636,10 +636,15 @@ class Tensor {
 // Interface to access the raw ref-counted data buffer.
 class TensorBuffer : public core::RefCounted {
  public:
+  explicit TensorBuffer(void* data_ptr) : data_(data_ptr) {}
   ~TensorBuffer() override {}
 
   // data() points to a memory region of size() bytes.
-  virtual void* data() const = 0;
+  //
+  // NOTE(mrry): The `data()` method is not virtual for performance reasons.
+  // It can be called multiple times when the contents of a `Tensor` are
+  // accessed, and so making it non-virtual allows the body to be inlined.
+  void* data() const { return data_; }
   virtual size_t size() const = 0;
 
   // If this TensorBuffer is sub-buffer of another TensorBuffer,
@@ -657,6 +662,9 @@ class TensorBuffer : public core::RefCounted {
 
   // Whether this TensorBuffer owns the underlying memory.
   virtual bool OwnsMemory() const { return true; }
+
+ private:
+  void* const data_;
 };
 
 template <typename T>
@@ -874,6 +882,7 @@ inline Tensor::Tensor(Tensor&& other)
 
 class Tensor::HostScalarTensorBufferBase : public TensorBuffer {
  public:
+  using TensorBuffer::TensorBuffer;
   void FillAllocationDescription(AllocationDescription* proto) const final;
 };
 
@@ -884,8 +893,7 @@ template <typename T>
 struct Tensor::ValueAndTensorBuffer {
   class HostScalarTensorBuffer : public Tensor::HostScalarTensorBufferBase {
    public:
-    HostScalarTensorBuffer(void* data) : data_(data) {}
-    void* data() const final { return const_cast<void*>(data_); }
+    HostScalarTensorBuffer(void* data) : HostScalarTensorBufferBase(data) {}
     size_t size() const final { return sizeof(T); }
     TensorBuffer* root_buffer() final { return this; }
 
@@ -904,8 +912,7 @@ struct Tensor::ValueAndTensorBuffer {
     }
 
    private:
-    ~HostScalarTensorBuffer() override { static_cast<T*>(data_)->~T(); }
-    void* const data_;
+    ~HostScalarTensorBuffer() override { static_cast<T*>(data())->~T(); }
   };
 
   T value;
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 4fa9d1df67..713f91fe04 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -1491,5 +1491,26 @@ void BM_CreateAndMoveCtrWithBuf(int iters) {
 }
 BENCHMARK(BM_CreateAndMoveCtrWithBuf);
 
+// Benchmark creating and destroy a host-scalar tensor, using the allocator
+// interface.
+void BM_CreateAndDestroyHostScalarNonOptimized(int iters) {
+  TensorShape shape({});
+  Allocator* allocator = cpu_allocator();
+  while (--iters) {
+    Tensor a(allocator, DT_FLOAT, shape);
+    a.scalar<float>()() = 37.0;
+  }
+}
+BENCHMARK(BM_CreateAndDestroyHostScalarNonOptimized);
+
+// Benchmark creating and destroy a host-scalar tensor, using the specialized
+// constructor.
+void BM_CreateAndDestroyHostScalarOptimized(int iters) {
+  while (--iters) {
+    Tensor a(37.0);
+  }
+}
+BENCHMARK(BM_CreateAndDestroyHostScalarOptimized);
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc
index 262ca9e089..0d0c953636 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map.cc
@@ -26,6 +26,8 @@ namespace flex {
 namespace {
 // A tensor buffer that is allocated, deallocated and populated by TF Lite.
 class BaseTfLiteTensorBuffer : public tensorflow::TensorBuffer {
+  using tensorflow::TensorBuffer::TensorBuffer;
+
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(
       tensorflow::AllocationDescription* proto) const override {
@@ -60,31 +62,29 @@ class BaseTfLiteTensorBuffer : public tensorflow::TensorBuffer {
 // representation in TFLITE and TF, so we just need use memcpy().
 class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
-  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor) {
+  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor)
+      : BaseTfLiteTensorBuffer(tensorflow::cpu_allocator()->AllocateRaw(
+            EIGEN_MAX_ALIGN_BYTES, tensor->bytes)) {
     // TODO(ahentz): if we can guarantee that TF Lite allocated tensors with
     // the same alignment as TensorFlow (EIGEN_MAX_ALIGN_BYTES), then we can
     // potentially eliminate the copy below.
     len_ = tensor->bytes;
-    data_ =
-        tensorflow::cpu_allocator()->AllocateRaw(EIGEN_MAX_ALIGN_BYTES, len_);
 
     LogAllocation();
 
-    if (data_) {
-      std::memcpy(data_, tensor->data.raw, tensor->bytes);
+    if (data()) {
+      std::memcpy(data(), tensor->data.raw, tensor->bytes);
     }
   }
 
   ~TfLiteTensorBuffer() override {
     LogDeallocation();
-    tensorflow::cpu_allocator()->DeallocateRaw(data_);
+    tensorflow::cpu_allocator()->DeallocateRaw(data());
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return len_; }
 
  private:
-  void* data_;
   size_t len_;
 };
 
@@ -92,19 +92,30 @@ class TfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
 // TF's so we need perform the conversion here.
 class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
  public:
-  explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor) {
-    if (tensor->data.raw == nullptr) {
-      num_strings_ = 0;
-      data_ = nullptr;
-      return;
-    }
-    num_strings_ = GetStringCount(tensor->data.raw);
-    data_ = tensorflow::cpu_allocator()->Allocate<string>(num_strings_);
+  explicit StringTfLiteTensorBuffer(const TfLiteTensor* tensor)
+      : StringTfLiteTensorBuffer(tensor, tensor->data.raw != nullptr
+                                             ? GetStringCount(tensor->data.raw)
+                                             : 0) {}
+
+  ~StringTfLiteTensorBuffer() override {
+    LogDeallocation();
+    tensorflow::cpu_allocator()->Deallocate<string>(
+        static_cast<string*>(data()), num_strings_);
+  }
+
+  size_t size() const override { return num_strings_ * sizeof(string); }
 
+ private:
+  StringTfLiteTensorBuffer(const TfLiteTensor* tensor, int num_strings)
+      : BaseTfLiteTensorBuffer(
+            num_strings != 0
+                ? tensorflow::cpu_allocator()->Allocate<string>(num_strings)
+                : nullptr),
+        num_strings_(num_strings) {
     LogAllocation();
 
-    if (data_) {
-      string* p = data_;
+    if (data()) {
+      string* p = static_cast<string*>(data());
       for (size_t i = 0; i < num_strings_; ++p, ++i) {
         auto ref = GetString(tensor->data.raw, i);
         p->assign(ref.str, ref.len);
@@ -112,16 +123,6 @@ class StringTfLiteTensorBuffer : public BaseTfLiteTensorBuffer {
     }
   }
 
-  ~StringTfLiteTensorBuffer() override {
-    LogDeallocation();
-    tensorflow::cpu_allocator()->Deallocate<string>(data_, num_strings_);
-  }
-
-  void* data() const override { return data_; }
-  size_t size() const override { return num_strings_ * sizeof(string); }
-
- private:
-  string* data_;
   int num_strings_;
 };
 
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index 9364aec373..97bebe8617 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -302,15 +302,14 @@ Status DoCallPyFunc(PyCall* call, bool* out_log_on_error) {
 class NumpyTensorBuffer : public TensorBuffer {
  public:
   NumpyTensorBuffer(PyArrayObject* array, size_t len, void* data)
-      : array_(array), len_(len), data_(data) {}
+      : TensorBuffer(data), array_(array), len_(len) {}
 
   ~NumpyTensorBuffer() override {
     // Note: The session::run wrapper is responsible for freeing this while
     // holding the GIL.
-    DelayedNumpyDecref(data_, len_, array_);
+    DelayedNumpyDecref(data(), len_, array_);
   }
 
-  void* data() const override { return data_; }
   size_t size() const override { return len_; }
   TensorBuffer* root_buffer() override { return this; }
   void FillAllocationDescription(AllocationDescription* proto) const override {
@@ -329,7 +328,6 @@ class NumpyTensorBuffer : public TensorBuffer {
  private:
   PyArrayObject* array_;
   size_t len_;
-  void* data_;
 };
 
 Status PyObjectToString(PyObject* obj, string* str) {
-- 
GitLab


From 80ac0c26cc16ffa1804c1092e298dee021f64b36 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Tue, 4 Dec 2018 14:20:48 -0800
Subject: [PATCH 1317/1554] Upgrade gRPC version used in OSS Tensorflow

PiperOrigin-RevId: 224045564
---
 tensorflow/contrib/cmake/external/grpc.cmake |  2 +-
 tensorflow/tools/lib_package/BUILD           |  2 --
 tensorflow/tools/pip_package/BUILD           |  1 -
 tensorflow/workspace.bzl                     | 24 +++++++++++++++-----
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index eca4f3c8c8..e570c09ecb 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG d184fa229d75d336aedea0041bd59cb93e7e267f)
+set(GRPC_TAG 69b6c047bc767b4d80e7af4d00ccb7c45b683dae)
 
 if(WIN32)
   # We use unsecure gRPC because boringssl does not build on windows
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 1cac5ee138..1186189844 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -169,7 +169,6 @@ genrule(
         "grpc",
         [
             "@grpc//:LICENSE",
-            "@grpc//third_party/nanopb:LICENSE.txt",
             "@grpc//third_party/address_sorting:LICENSE",
         ],
     ),
@@ -208,7 +207,6 @@ genrule(
         "@zlib_archive//:zlib.h",
         "@grpc//:LICENSE",
         "@grpc//third_party/address_sorting:LICENSE",
-        "@grpc//third_party/nanopb:LICENSE.txt",
     ] + select({
         "//tensorflow:android": [],
         "//tensorflow:ios": [],
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 82c6bf383f..f34ac4c3e8 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -214,7 +214,6 @@ filegroup(
         "grpc",
         [
             "@grpc//:LICENSE",
-            "@grpc//third_party/nanopb:LICENSE.txt",
             "@grpc//third_party/address_sorting:LICENSE",
         ],
     ) + if_ngraph([
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7e5f84be16..2074f63202 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -445,14 +445,26 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         ],
     )
 
+    # WARNING: make sure ncteisen@ and vpai@ are cc-ed on any CL to change the below rule
     tf_http_archive(
         name = "grpc",
-        sha256 = "50db9cf2221354485eb7c3bd55a4c27190caef7048a2a1a15fbe60a498f98b44",
-        strip_prefix = "grpc-1.13.0",
+        sha256 = "1aa84387232dda273ea8fdfe722622084f72c16f7b84bfc519ac7759b71cdc91",
+        strip_prefix = "grpc-69b6c047bc767b4d80e7af4d00ccb7c45b683dae",
         system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/grpc/grpc/archive/v1.13.0.tar.gz",
-            "https://github.com/grpc/grpc/archive/v1.13.0.tar.gz",
+            "https://mirror.bazel.build/github.com/grpc/grpc/archive/69b6c047bc767b4d80e7af4d00ccb7c45b683dae.tar.gz",
+            "https://github.com/grpc/grpc/archive/69b6c047bc767b4d80e7af4d00ccb7c45b683dae.tar.gz",
+        ],
+    )
+
+    tf_http_archive(
+        name = "com_github_nanopb_nanopb",
+        sha256 = "8bbbb1e78d4ddb0a1919276924ab10d11b631df48b657d960e0c795a25515735",
+        build_file = "@grpc//third_party:nanopb.BUILD",
+        strip_prefix = "nanopb-f8ac463766281625ad710900479130c7fcb4d63b",
+        urls = [
+            "https://mirror.bazel.build/github.com/nanopb/nanopb/archive/f8ac463766281625ad710900479130c7fcb4d63b.tar.gz",
+            "https://github.com/nanopb/nanopb/archive/f8ac463766281625ad710900479130c7fcb4d63b.tar.gz",
         ],
     )
 
@@ -846,7 +858,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     # important since we have set GRPC_ARES=0 in .bazelrc
     native.bind(
         name = "cares",
-        actual = "@grpc//third_party/nanopb:nanopb",
+        actual = "@com_github_nanopb_nanopb//:nanopb",
     )
 
     # Needed by Protobuf
@@ -878,7 +890,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     # Needed by gRPC
     native.bind(
         name = "nanopb",
-        actual = "@grpc//third_party/nanopb:nanopb",
+        actual = "@com_github_nanopb_nanopb//:nanopb",
     )
 
     # Needed by gRPC
-- 
GitLab


From 09354505e203d099c2bd8b3645591c86c2aaa750 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 4 Dec 2018 14:24:32 -0800
Subject: [PATCH 1318/1554] Allow PolymorphicFunction objects in
 tf.train.Checkpoint's constructor

They can be assigned to attributes, so forbidding them in the constructor doesn't make much sense, and it will have uses for tf.saved_model.save.

PiperOrigin-RevId: 224046170
---
 tensorflow/python/saved_model/save_test.py        | 6 +++---
 tensorflow/python/training/checkpointable/BUILD   | 1 +
 tensorflow/python/training/checkpointable/util.py | 5 +++--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 9e5b9b9717..5096e9c062 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -152,9 +152,9 @@ class SaveTest(test.TestCase):
       save.save(root, save_dir, to_save)
 
   def test_nested_dict_outputs(self):
-    root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: {"a": 2. * x, "b": (3. * x, 4. * x)})
+    root = util.Checkpoint(
+        f=def_function.function(
+            lambda x: {"a": 2. * x, "b": (3. * x, 4. * x)}))
     root.f(constant_op.constant(1.))
     to_save = root.f.get_concrete_function(constant_op.constant(1.))
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index f97f42a659..4ab5593d4f 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -122,6 +122,7 @@ py_library(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:def_function",
     ],
 )
 
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 36d3a7bebd..a1cc6082fd 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -26,6 +26,7 @@ from tensorflow.core.protobuf import checkpointable_object_graph_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
@@ -1712,7 +1713,8 @@ class Checkpoint(tracking.Checkpointable):
     """
     super(Checkpoint, self).__init__()
     for k, v in sorted(kwargs.items(), key=lambda item: item[0]):
-      if not isinstance(v, base.CheckpointableBase):
+      if not isinstance(v, (base.CheckpointableBase,
+                            def_function.PolymorphicFunction)):
         raise ValueError(
             ("`Checkpoint` was expecting a checkpointable object (an object "
              "derived from `CheckpointableBase`), got %s. If you believe this "
@@ -1926,4 +1928,3 @@ class Checkpoint(tracking.Checkpointable):
     # initialization when executing eagerly.
     self._maybe_create_save_counter()
     return status
-
-- 
GitLab


From 45db3aee01a8ae4d1bdf7a66c0c6f0a103f26310 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 4 Dec 2018 14:24:52 -0800
Subject: [PATCH 1319/1554] relu_op_test: Make all tests work in both TF 1 and
 2

i.e., update all tests so that they don't need the
@test_util.run_deprecated_v1 decorator
(essentially enabled by gradient_checker_v2).

Noteworthy: This increases time to execute the test slightly in TF 1.x
(as gradient_checker_v2.compute_gradients adds nodes to the graph twice
- once for the analytical and once for the numeric gradients),
but tests with TF2_BEHAVIOR=1 are generally much faster. For example, on my
machine:

1.8s (without this change - bazel test :relu_ops_test)
2.1s (with this change - bazel test :relu_op_test)
0.6s (with this change -bazel test --test_env=TF2_BEHAVIOR=1 :relu_op_test)

PiperOrigin-RevId: 224046219
---
 tensorflow/python/kernel_tests/BUILD          |   3 +-
 .../python/kernel_tests/relu_op_test.py       | 379 ++++++++----------
 2 files changed, 174 insertions(+), 208 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 97ac21b8ad..f990b0ecc4 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2073,12 +2073,13 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:gradients",
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
+        "//tensorflow/python/eager:backprop",
+        "//tensorflow/python:tf2",
     ],
 )
 
diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index 55e68f4884..d4ba1ad77d 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -21,15 +21,15 @@ from __future__ import print_function
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python import tf2
 from tensorflow.python.compat import compat
+from tensorflow.python.eager import backprop
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
-from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
@@ -106,105 +106,101 @@ class ReluTest(test.TestCase):
 
   # The gradient test for ReLU is a bit tricky as the derivative is not well
   # defined at around zero and we want to avoid that in terms of input values.
-  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          name="x")
-      y = nn_ops.relu(x, name="relu")
-      x_init = np.asarray(
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float32,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.relu, [x]))
     print("relu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
   # The gradient for fp16 is inaccurate due to the low-precision.
-  # Instead of relying on compute_gradient_error, we compare the fp16 analytical
-  # gradient against their fp32 counterpart.
-  @test_util.run_deprecated_v1
+  # We compare the fp16 analytical gradient against their fp32 counterpart.
   def testGradientFloat16(self):
-    with self.session(use_gpu=True) as sess:
-      # Randomly construct a 1D shape from [1, 40)
-      shape = random_ops.random_uniform(
-          [1], minval=1, maxval=40, dtype=dtypes.int32)
-
-      # Construct the fp32 graph and its gradient.
-      x = random_ops.random_uniform(shape, minval=-1, maxval=1, name="x")
-      y1 = nn_ops.relu(x, name="relu_fp32")
-      l1 = nn_ops.l2_loss(y1)
-      dx_f32 = gradients_impl.gradients(l1, x)
-
-      # Construct the fp16 graph and its gradient.
-      # It starts with the same x, in fp32. But before it reaches Relu, it is
-      # cast into fp16. So during backprop, the gradient computation is in fp16.
-      x2 = math_ops.cast(x, dtype=dtypes.float16, name="cast")
-      y2 = nn_ops.relu(x2, name="relu_fp16")
-      l2 = nn_ops.l2_loss(y2)
-      dx_f16 = gradients_impl.gradients(l2, x)
-
-      # Repeat the experiment for 100 times. All tensor shapes and its tensor
-      # values are randomly generated for each run.
-      for _ in xrange(100):
-        dx_f32_v, dx_f16_v = self.evaluate([dx_f32, dx_f16])
-        self.assertAllClose(dx_f32_v, dx_f16_v, atol=3e-4)
-
-  @test_util.run_deprecated_v1
+
+    def grad(x):
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        y = nn_ops.l2_loss(nn_ops.relu(x))
+      return tape.gradient(y, x)
+
+    def f():
+      with test_util.use_gpu():
+        # Randomly construct a 1D shape from [1, 40)
+        shape = random_ops.random_uniform([1],
+                                          minval=1,
+                                          maxval=40,
+                                          dtype=dtypes.int32)
+        x32 = random_ops.random_uniform(shape, minval=-1, maxval=1)
+        x16 = math_ops.cast(x32, dtype=dtypes.float16)
+        return grad(x32), grad(x16)
+
+    # We're going to ensure that the fp16 and fp32 gradients
+    # are "close" to each other for ~100 random values.
+    #
+    # In TensorFlow 1.x, invoking f() (without eager execution enabled)
+    # would construct a graph. Instead of construct a graph with O(100) nodes,
+    # we construct a single graph to be executed ~100 times in a Session.
+    if not tf2.enabled():
+      d32_tensor, d16_tensor = f()
+      with self.cached_session() as sess:
+        f = lambda: sess.run([d32_tensor, d16_tensor])
+
+    # Repeat the experiment for 100 times. All tensor shapes and its tensor
+    # values are randomly generated for each run.
+    for _ in xrange(100):
+      d32, d16 = f()
+      self.assertAllClose(d32, d16, atol=3e-4)
+
   def testGradientFloat64(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          dtype=dtypes.float64,
-          name="x")
-      y = nn_ops.relu(x, name="relu")
-      x_init = np.asarray(
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float64,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.relu, [x]))
     print("relu (float64) gradient err = ", err)
     self.assertLess(err, 1e-10)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          name="x")
-      y = nn_ops.relu(x, name="relu")
-      z = gradients_impl.gradients(y, x)
-      x_init = np.asarray(
+
+      def f(x):
+        assert x.dtype == dtypes.float32
+        with backprop.GradientTape() as tape:
+          tape.watch(x)
+          y = nn_ops.relu(x)
+        return tape.gradient(y, x)
+
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float32,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(f, [x]))
     print("relu (float32) gradient of gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          dtype=dtypes.float64,
-          name="x")
-      y = nn_ops.relu(x, name="relu")
-      z = gradients_impl.gradients(y, x)
-      x_init = np.asarray(
+
+      def f(x):
+        assert x.dtype == dtypes.float64
+        with backprop.GradientTape() as tape:
+          tape.watch(x)
+          y = nn_ops.relu(x)
+        return tape.gradient(y, x)
+
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float64,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(f, [x]))
     print("relu (float64) gradient of gradient err = ", err)
     self.assertLess(err, 1e-10)
 
@@ -258,38 +254,25 @@ class Relu6Test(test.TestCase):
   # The gradient test for ReLU6 is a bit tricky as the derivative is
   # not well defined at around zero and six and we want to avoid that
   # in terms of input values.
-  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 6.1, 6.3, 6.5, 6.7, 6.9],
-          shape=[2, 5],
-          name="x")
-      y = nn_ops.relu6(x, name="relu6")
-      x_init = np.asarray(
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]],
           dtype=np.float32,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.relu6, [x]))
     print("relu6 (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 6.1, 6.3, 6.5, 6.7, 6.9],
-          shape=[2, 5],
-          dtype=dtypes.float64,
-          name="x")
-      y = nn_ops.relu6(x, name="relu6")
-      x_init = np.asarray(
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [6.1, 6.3, 6.5, 6.7, 6.9]],
           dtype=np.float64,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.relu6, [x]))
     print("relu6 (float64) gradient err = ", err)
     self.assertLess(err, 1e-10)
 
@@ -333,77 +316,65 @@ class LeakyReluTest(test.TestCase):
   # The gradient test for Leaky ReLU is a bit tricky as the derivative is not
   # well defined at around zero and we want to avoid that in terms of input
   # values.
-  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
-    with self.test_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          name="x")
-      y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu")
-      x_init = np.asarray(
+    with self.cached_session():
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float32,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x]))
     print("leaky_relu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
-    with self.test_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          dtype=dtypes.float64,
-          name="x")
-      y = nn_ops.leaky_relu(x, alpha=0.2, name="leaky_relu")
-      x_init = np.asarray(
+    with self.cached_session():
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float64,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.leaky_relu, [x]))
     print("leaky_relu (float64) gradient err = ", err)
     self.assertLess(err, 1e-10)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with compat.forward_compatibility_horizon(2018, 11, 2):
-      with self.test_session():
-        x = constant_op.constant(
-            [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-            shape=[2, 5],
-            name="x")
-        y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu")
-        z = gradients_impl.gradients(y, x)
-        x_init = np.asarray(
+      with self.cached_session():
+
+        def f(x):
+          assert x.dtype == dtypes.float32
+          with backprop.GradientTape() as tape:
+            tape.watch(x)
+            y = nn_ops.leaky_relu(x)
+          return tape.gradient(y, x)
+
+        x = np.asarray(
             [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
             dtype=np.float32,
             order="F")
-        err = gradient_checker.compute_gradient_error(
-            x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+        err = gradient_checker_v2.max_error(
+            *gradient_checker_v2.compute_gradient(f, [x]))
       print("leaky_relu (float32) gradient of gradient err = ", err)
       self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with compat.forward_compatibility_horizon(2018, 11, 2):
-      with self.test_session():
-        x = constant_op.constant(
-            [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-            shape=[2, 5],
-            dtype=dtypes.float64,
-            name="x")
-        y = nn_ops.leaky_relu(x, alpha=0.02, name="leaky_relu")
-        z = gradients_impl.gradients(y, x)
-        x_init = np.asarray(
+      with self.cached_session():
+
+        def f(x):
+          assert x.dtype == dtypes.float64
+          with backprop.GradientTape() as tape:
+            tape.watch(x)
+            y = nn_ops.leaky_relu(x)
+          return tape.gradient(y, x)
+
+        x = np.asarray(
             [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
             dtype=np.float64,
             order="F")
-        err = gradient_checker.compute_gradient_error(
-            x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+        err = gradient_checker_v2.max_error(
+            *gradient_checker_v2.compute_gradient(f, [x]))
       print("leaky_relu (float64) gradient of gradient err = ", err)
       self.assertLess(err, 1e-10)
 
@@ -451,76 +422,75 @@ class EluTest(test.TestCase):
     for t in [np.float16, np.float32, np.float64]:
       self._testElu(np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
-  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
-      x = constant_op.constant(x_val, name="x")
-      y = nn_ops.elu(x, name="elu")
-      x_init = np.asarray(x_val, dtype=np.float32, order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      x = np.asarray(x_val, dtype=np.float32, order="F")
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.elu, [x]))
     print("elu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
-      x = constant_op.constant(x_val, dtype=dtypes.float64, name="x")
-      y = nn_ops.elu(x, name="elu")
-      x_init = np.asarray(x_val, dtype=np.float64, order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      x = np.asarray(x_val, dtype=np.float64, order="F")
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.elu, [x]))
     print("elu (float64) gradient err = ", err)
     self.assertLess(err, 1e-6)
 
-  @test_util.run_deprecated_v1
   def testGradGrad(self):
     with self.cached_session():
-      x = array_ops.placeholder(dtype=dtypes.float32)
-      elu = nn_ops.elu(x)
-      g, = gradients_impl.gradients(elu, x)
-      gg, = gradients_impl.gradients(g, x)
 
-      for x_val in [-1, -0.5, 0.5, 1]:
-        err = np.abs(gg.eval(feed_dict={x: x_val}) - _elu_grad_grad(x_val))
+      def f(x):
+        with backprop.GradientTape(persistent=True) as tape:
+          tape.watch(x)
+          y = nn_ops.elu(x)
+          dy = tape.gradient(y, x)
+        return tape.gradient(dy, x)
+
+      for x in [-1., -0.5, 0.5, 1.]:
+        got = self.evaluate(f(constant_op.constant(x)))
+        want = _elu_grad_grad(x)
+        err = np.abs(got - want)
         self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          name="x")
-      y = nn_ops.elu(x, name="elu")
-      z = gradients_impl.gradients(y, x)
-      x_init = np.asarray(
+
+      def f(x):
+        assert x.dtype == dtypes.float32
+        with backprop.GradientTape() as tape:
+          tape.watch(x)
+          y = nn_ops.elu(x)
+        return tape.gradient(y, x)
+
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float32,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(f, [x]))
     print("elu (float32) gradient of gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          dtype=dtypes.float64,
-          name="x")
-      y = nn_ops.elu(x, name="elu")
-      z = gradients_impl.gradients(y, x)
-      x_init = np.asarray(
+
+      def f(x):
+        assert x.dtype == dtypes.float64
+        with backprop.GradientTape() as tape:
+          tape.watch(x)
+          y = nn_ops.elu(x)
+        return tape.gradient(y, x)
+
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float64,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(f, [x]))
     print("elu (float64) gradient of gradient err = ", err)
     self.assertLess(err, 1e-6)
 
@@ -556,64 +526,59 @@ class SeluTest(test.TestCase):
         self._testSelu(
             np.array([[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]]).astype(t))
 
-  @test_util.run_deprecated_v1
   def testGradientFloat32(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
-      x = constant_op.constant(x_val, name="x")
-      y = nn_ops.selu(x, name="selu")
-      x_init = np.asarray(x_val, dtype=np.float32, order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      x = np.asarray(x_val, dtype=np.float32, order="F")
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.selu, [x]))
     print("selu (float32) gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradientFloat64(self):
     with self.cached_session():
       x_val = [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]]
-      x = constant_op.constant(x_val, dtype=dtypes.float64, name="x")
-      y = nn_ops.selu(x, name="selu")
-      x_init = np.asarray(x_val, dtype=np.float64, order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], y, [2, 5], x_init_value=x_init)
+      x = np.asarray(x_val, dtype=np.float64, order="F")
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(nn_ops.selu, [x]))
     print("selu (float64) gradient err = ", err)
     self.assertLess(err, 1e-6)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat32(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          name="x")
-      y = nn_ops.selu(x, name="selu")
-      z = gradients_impl.gradients(y, x)
-      x_init = np.asarray(
+
+      def f(x):
+        assert x.dtype == dtypes.float32
+        with backprop.GradientTape() as tape:
+          tape.watch(x)
+          y = nn_ops.selu(x)
+        return tape.gradient(y, x)
+
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float32,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(f, [x]))
     print("selu (float32) gradient of gradient err = ", err)
     self.assertLess(err, 1e-4)
 
-  @test_util.run_deprecated_v1
   def testGradGradFloat64(self):
     with self.cached_session():
-      x = constant_op.constant(
-          [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9],
-          shape=[2, 5],
-          dtype=dtypes.float64,
-          name="x")
-      y = nn_ops.selu(x, name="selu")
-      z = gradients_impl.gradients(y, x)
-      x_init = np.asarray(
+
+      def f(x):
+        assert x.dtype == dtypes.float64
+        with backprop.GradientTape() as tape:
+          tape.watch(x)
+          y = nn_ops.selu(x)
+        return tape.gradient(y, x)
+
+      x = np.asarray(
           [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]],
           dtype=np.float64,
           order="F")
-      err = gradient_checker.compute_gradient_error(
-          x, [2, 5], z[0], [2, 5], x_init_value=x_init)
+      err = gradient_checker_v2.max_error(
+          *gradient_checker_v2.compute_gradient(f, [x]))
     print("selu (float64) gradient of gradient err = ", err)
     self.assertLess(err, 1e-6)
 
-- 
GitLab


From 5a47e65e8b5286f44232577fd0cfccc44dd48cd5 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Tue, 4 Dec 2018 14:48:04 -0800
Subject: [PATCH 1320/1554] Internal change

PiperOrigin-RevId: 224050300
---
 .../tools/ci_build/windows/cpu/pip/build_tf_windows.sh      | 6 ++++--
 .../tools/ci_build/windows/gpu/pip/build_tf_windows.sh      | 3 ++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 46f5bdef09..a6871dd8f1 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -67,8 +67,9 @@ EXTRA_BUILD_FLAGS=""
 #                        ensure performance
 # --test_core_only       Use tensorflow/python/... as test target
 # --test_contrib_only    Use tensorflow/contrib/... as test target
-for ARG in "$@"; do
-  case "$ARG" in
+#for ARG in "$@"; do
+while [[ $# -gt 0 ]]; do
+  case "$1" in
     --tf_nightly) TF_NIGHTLY=1 ;;
     --skip_test) SKIP_TEST=1 ;;
     --enable_remote_cache) set_remote_cache_options ;;
@@ -91,6 +92,7 @@ for ARG in "$@"; do
       ;;
     *)
   esac
+  shift
 done
 
 if [[ "$RELEASE_BUILD" == 1 ]]; then
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 3aec8d6584..acafd9ebce 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -67,7 +67,7 @@ EXTRA_BUILD_FLAGS=""
 #                        ensure performance
 # --test_core_only       Use tensorflow/python/... as test target
 # --test_contrib_only    Use tensorflow/contrib/... as test target
-for ARG in "$@"; do
+while [[ $# -gt 0 ]]; do
   case "$ARG" in
     --tf_nightly) TF_NIGHTLY=1 ;;
     --skip_test) SKIP_TEST=1 ;;
@@ -91,6 +91,7 @@ for ARG in "$@"; do
       ;;
     *)
   esac
+  shift
 done
 
 if [[ "$RELEASE_BUILD" == 1 ]]; then
-- 
GitLab


From 09d8e3829d91b3108b607359af613e8500f6b337 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Tue, 4 Dec 2018 14:52:39 -0800
Subject: [PATCH 1321/1554] Bug fixes for subclassed models: * dataset re-use *
 batch size validation error for models without layers

In addition minor reformatting of the error message.

PiperOrigin-RevId: 224051130
---
 tensorflow/python/keras/engine/training.py    | 132 ++++++++++++------
 .../python/keras/engine/training_test.py      |  15 ++
 .../python/keras/engine/training_utils.py     |   3 +
 3 files changed, 109 insertions(+), 41 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 2236bcf27c..c902d928a0 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1205,12 +1205,57 @@ class Model(Network):
           x, y, sample_weight = next_element
       else:
         x = next_element
-    x, y, sample_weights = self._standardize_weights(x, y, sample_weight,
-                                                     class_weight, batch_size)
+    x, y, sample_weights = self._standardize_weights(
+        x, y, sample_weight, class_weight, batch_size, is_x_iterator)
     return x, y, sample_weights
 
-  def _standardize_weights(self, x, y, sample_weight=None, class_weight=None,
-                           batch_size=None,):
+  def _standardize_weights(self,
+                           x,
+                           y,
+                           sample_weight=None,
+                           class_weight=None,
+                           batch_size=None,
+                           from_iterator=False):
+    """Standardize input data, target data, and weight values.
+
+    This method reformats all data passed to the model to an ordered list of
+    array/tensors, matching the order expected by the model. This also validates
+    the input and target data shapes.
+
+    Args:
+      x: Input data. It could be:
+        - A Numpy array (or array-like), or a list of arrays
+          (in case the model has multiple inputs).
+        - A TensorFlow tensor, or a list of tensors
+          (in case the model has multiple inputs).
+        - A dict mapping input names to the corresponding array/tensors,
+          if the model has named inputs.
+        x cannot not be an iterator.
+      y: Target data. Like the input data `x`,
+        it could be either Numpy array(s) or TensorFlow tensor(s).
+        It should be consistent with `x` (you cannot have Numpy inputs and
+        tensor targets, or inversely).
+      sample_weight: An optional sample-weight array passed by the user to
+        weight the importance of each sample in `x`.
+      class_weight: An optional class-weight array by the user to
+        weight the importance of samples in `x` based on the class they belong
+        to, as conveyed by `y`.
+      batch_size: Integer batch size. If provided, it is used to run additional
+        validation checks on stateful models.
+      from_iterator: Whether x and y were obtained from an iterator.
+
+    Returns:
+      Tuple of standardized data that will be fed to the model:
+        (input data, target data, sample weights)
+
+    Raises:
+      RuntimeError: If target data is provided, but the model has not yet been
+        compiled.
+      ValueError: If the input data, target data, and batch size have invalid
+        shapes or formats (e.g. the model expects input to be a list of three
+        tensors, but x is a list with two tensors). Error is also raised if the
+        input and target data are not both arrays or tensors.
+    """
     # TODO(sourabhbajaj): Split input validation from weight standardization.
     if sample_weight is not None and class_weight is not None:
       logging.warning(
@@ -1244,13 +1289,16 @@ class Model(Network):
         all_inputs.append(x)
 
       # Build the model using the retrieved inputs (value or symbolic).
-      # If values, then in symbolic-mode placeholders will be created
-      # to match the value shapes.
+      # If values or generated from a dataset, then in symbolic-mode
+      # placeholders will be created to match the value shapes.
       if not self.inputs:
         is_build_called = True
-        cast_inputs = x
-        if training_utils.has_tensors(x):
+        if from_iterator:
+          cast_inputs = nest.map_structure(lambda v: v.shape, x)
+        elif training_utils.has_tensors(x):
           cast_inputs = training_utils.cast_if_floating_dtype(x)
+        else:
+          cast_inputs = x
         self._set_inputs(cast_inputs)
     else:
       dict_inputs = isinstance(self.inputs, dict)
@@ -1293,7 +1341,7 @@ class Model(Network):
                              'TensorFlow tensors. '
                              'You passed: x=' + str(x) + '; y=' + str(y))
 
-        if self.run_eagerly:
+        if self.run_eagerly or from_iterator:
           target_tensors = None
         else:
           # Handle target tensors if any passed.
@@ -1316,9 +1364,8 @@ class Model(Network):
     # part of the graph.
     # Note: in this case, `any` and `all` are equivalent since we disallow
     # mixed symbolic/value inputs.
-    if (not self.run_eagerly and is_build_called and
-        is_compile_called and
-        any(_is_symbolic_tensor(v) for v in all_inputs)):
+    if (not self.run_eagerly and is_build_called and is_compile_called and
+        not from_iterator and any(_is_symbolic_tensor(v) for v in all_inputs)):
       return [], [], []
 
     # What follows is input validation and standardization to list format,
@@ -1432,12 +1479,12 @@ class Model(Network):
 
     Args:
       inputs: Single array, or list of arrays. The arrays could be placeholders,
-        Numpy arrays, or data tensors.
+        Numpy arrays, data tensors, or TensorShapes.
         - if placeholders: the model is built on top of these placeholders,
           and we expect Numpy data to be fed for them when calling `fit`/etc.
-        - if Numpy data: we create placeholders matching the shape of the Numpy
-          arrays. We expect Numpy data to be fed for these placeholders
-          when calling `fit`/etc.
+        - if Numpy data or TensorShapes: we create placeholders matching the
+          TensorShapes or shapes of the Numpy arrays. We expect Numpy data to be
+          fed for these placeholders when calling `fit`/etc.
         - if data tensors: the model is built on top of these tensors.
           We do not expect any Numpy data to be provided when calling `fit`/etc.
       outputs: None, a data tensor, or a list of tensors. If None, the
@@ -1456,6 +1503,8 @@ class Model(Network):
     if self.__class__.__name__ == 'Sequential' and not self.built:
       if tensor_util.is_tensor(inputs):
         input_shape = (None,) + tuple(inputs.shape.as_list()[1:])
+      elif isinstance(inputs, tensor_shape.TensorShape):
+        input_shape = (None,) + tuple(inputs.as_list()[1:])
       elif isinstance(inputs, dict):
         # We assert that the first layer is a FeatureLayer.
         if not training_utils.is_feature_layer(self.layers[0]):
@@ -2503,31 +2552,32 @@ class Model(Network):
       The validated batch_size, auto-inferred from the first layer if not
       provided.
     """
-    first_layer = super(Model,
-                        self).layers[0]  # Avoids the override in Sequential.
-    static_batch_size = training_utils.get_static_batch_size(first_layer)
-    if static_batch_size is not None:
-
-      # Check `batch_size` argument is consistent with InputLayer.
-      if batch_size is not None and batch_size != static_batch_size:
-        raise ValueError('The `batch_size` argument value ' + str(batch_size) +
-                         ' is incompatible with the specified batch size '
-                         'of your Input Layer: ' + str(static_batch_size))
-
-      # Check Dataset/Iterator batch size is consistent with InputLayer.
-      if isinstance(x, (dataset_ops.DatasetV2, iterator_ops.Iterator,
-                        iterator_ops.EagerIterator)):
-        ds_batch_size = tensor_shape.as_dimension(
-            nest.flatten(x.output_shapes)[0][0]).value
-        if ds_batch_size is not None and ds_batch_size != static_batch_size:
-          raise ValueError('The batch output shape of your `Dataset` is ' +
-                           str(ds_batch_size) + ' which is incompatible '
-                           'with the specified batch size of your Input '
-                           'Layer: ' + str(static_batch_size))
-
-      # Set inferred batch size from the InputLayer.
-      if steps is None:
-        batch_size = static_batch_size
+    layers = super(Model, self).layers  # Avoids the override in Sequential.
+    if layers:
+      first_layer = layers[0]
+      static_batch_size = training_utils.get_static_batch_size(first_layer)
+      if static_batch_size is not None:
+
+        # Check `batch_size` argument is consistent with InputLayer.
+        if batch_size is not None and batch_size != static_batch_size:
+          raise ValueError('The `batch_size` argument value {} is incompatible '
+                           'with the specified batch size of your Input Layer: '
+                           '{}'.format(batch_size, static_batch_size))
+
+        # Check Dataset/Iterator batch size is consistent with InputLayer.
+        if isinstance(x, (dataset_ops.DatasetV2, iterator_ops.Iterator,
+                          iterator_ops.EagerIterator)):
+          ds_batch_size = tensor_shape.as_dimension(
+              nest.flatten(x.output_shapes)[0][0]).value
+          if ds_batch_size is not None and ds_batch_size != static_batch_size:
+            raise ValueError('The batch output shape of your `Dataset` is {}, '
+                             'which is incompatible with the specified batch '
+                             'size of your Input Layer: {}'.format(
+                                 ds_batch_size, static_batch_size))
+
+        # Set inferred batch size from the InputLayer.
+        if steps is None:
+          batch_size = static_batch_size
 
     if batch_size is None and steps is None:
       # Backwards compatibility
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 6e5fd42714..572cde1d04 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -717,6 +717,21 @@ class TrainingTest(keras_parameterized.TestCase):
                                  'specified batch sizes of the Input Layers'):
       keras.Model([input1, input2], outputs)
 
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_calling_subclass_model_on_different_datasets(self):
+
+    class SubclassedModel(keras.models.Model):
+
+      def call(self, inputs):
+        return inputs * 2
+
+    model = SubclassedModel()
+    dataset_one = dataset_ops.Dataset.range(2).batch(2)
+    dataset_two = dataset_ops.Dataset.range(3, 10).batch(2)
+    self.assertAllEqual([[0], [2]], model.predict(dataset_one, steps=1))
+    self.assertAllEqual([[6], [8], [10], [12]],
+                        model.predict(dataset_two, steps=2))
+
 
 class TestExceptionsAndWarnings(keras_parameterized.TestCase):
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 0157fe084c..9301302f4a 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1113,6 +1113,9 @@ class ModelInputs(object):
         # to specify custom placeholders if the need arises.
         shape = (None,) + tuple(v.shape[1:])
         v = K.placeholder(shape=shape, name=k)
+      elif isinstance(v, tensor_shape.TensorShape):
+        shape = (None,) + tuple(v.as_list()[1:])
+        v = K.placeholder(shape=shape, name=k)
       self._flattened_inputs[i] = v
 
     if self._is_dict:
-- 
GitLab


From 0deedbd74ea0ddd84a8d0a2e6ae690f583825bf7 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 4 Dec 2018 14:55:41 -0800
Subject: [PATCH 1322/1554] Don't open an unnecessary init_scope in _cache_key

Also make the same improvement to executing_eagerly_outside_functions

Before:
entry {
  name: "MicroBenchmarks.benchmark_defun_matmul_2_by_2_CPU"
  iters: 30000
  wall_time: 54.0651639303
  extras {
    key: "examples_per_sec"
    value {
      double_value: 18496.1984262
    }
  }
}

After:
entry {
  name: "MicroBenchmarks.benchmark_defun_matmul_2_by_2_CPU"
  iters: 30000
  wall_time: 47.4757671356
  extras {
    key: "examples_per_sec"
    value {
      double_value: 21063.3773888
    }
  }
}
PiperOrigin-RevId: 224051558
---
 tensorflow/python/eager/function.py | 16 +++++++++++-----
 tensorflow/python/framework/ops.py  |  4 ++++
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9d05a660b1..520c85a2c2 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -982,11 +982,17 @@ class PolymorphicFunction(object):
       input_signature = self._flat_input_signature
 
     ctx = context.context()
-    with ops.init_scope():
-      # The graph, or whether we're executing eagerly, should be a part of the
-      # cache key so we don't improperly capture tensors such as variables.
-      executing_eagerly = ctx.executing_eagerly()
-      parent_graph = None if executing_eagerly else ops.get_default_graph()
+
+    # Don't need to open an init_scope if the _cache_key call is in eager mode
+    # already.
+    executing_eagerly = ctx.executing_eagerly()
+    parent_graph = None
+    if not executing_eagerly:
+      with ops.init_scope():
+        # The graph, or whether we're executing eagerly, should be a part of the
+        # cache key so we don't improperly capture tensors such as variables.
+        executing_eagerly = ctx.executing_eagerly()
+        parent_graph = None if executing_eagerly else ops.get_default_graph()
 
     # pylint: disable=protected-access
     default_graph = ops.get_default_graph()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 1a26984809..fa306936d6 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5375,6 +5375,10 @@ def init_scope():
 
 def executing_eagerly_outside_functions():
   """Returns True if executing eagerly, even if inside a graph function."""
+  # Fastpath for when this is called eagerly (its not necessary to init_scope).
+  if context.executing_eagerly():
+    return True
+
   with init_scope():
     return context.executing_eagerly()
 
-- 
GitLab


From bbe33f52fada5b4c2abba9a1e49509fc64a442df Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 4 Dec 2018 15:05:14 -0800
Subject: [PATCH 1323/1554] matmul_op_test: Use gradient_checker_v2

gradient_checker_v2 works with both TensorFlow 1.x and 2.x, so this usage
makes it possible to run more tests in matmul_op_test.py with TF2_BEHAVIOR
enabled.

Remove some @test_util.run_deprecated_v1 annotations.

PiperOrigin-RevId: 224053364
---
 .../python/kernel_tests/matmul_op_test.py     | 90 +++++++++----------
 1 file changed, 42 insertions(+), 48 deletions(-)

diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py
index 983f463f5e..cd99f1774a 100644
--- a/tensorflow/python/kernel_tests/matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/matmul_op_test.py
@@ -25,7 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -41,11 +41,9 @@ class MatVecTest(test_lib.TestCase):
   def testTwoByTwoCase(self):
     a = np.array([[1, 2], [3, 4]])
     b = np.array([5, 6])
-    with self.cached_session():
-      c = math_ops.matvec(a, b)
-      self.assertAllEqual((2,), c.shape)
-      c_ = self.evaluate(c)
-    self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c_)
+    c = math_ops.matvec(a, b)
+    self.assertAllEqual((2,), c.shape)
+    self.assertAllEqual([5 + 2 * 6, 3 * 5 + 4 * 6], c)
 
 
 def _AddTest(test, op_name, testcase_name, fn):
@@ -85,7 +83,7 @@ def _GetMatMulTest(a_np_, b_np_, use_static_shape_, **kwargs_):
     # np.matrix(a_np_) * np.matrix(b_np_)
     effective_a_np = _GetTransposedMatrices(a_np_, "a", kwargs_)
     effective_b_np = _GetTransposedMatrices(b_np_, "b", kwargs_)
-    with self.session(use_gpu=use_gpu) as sess:
+    with self.cached_session() as sess, test_util.device(use_gpu):
       if use_static_shape_:
         a = constant_op.constant(effective_a_np)
         b = constant_op.constant(effective_b_np)
@@ -128,45 +126,45 @@ def _GetMatMulGradientTest(a_np_, b_np_, use_static_shape_, **kwargs_):
     epsilon = np.finfo(a_np_.dtype).eps
     delta = epsilon**(1.0 / 3.0)
     tol = 20 * delta
-    with self.session(use_gpu=True):
-      a = constant_op.constant(effective_a_np)
-      b = constant_op.constant(effective_b_np)
-      res = math_ops.matmul(a, b, **kwargs_)
-      for x, x_init in [a, effective_a_np], [b, effective_b_np]:
-        theoretical, numerical = gradient_checker.compute_gradient(
-            x,
-            x_init.shape,
-            res, [a_np_.shape[0], b_np_.shape[1]],
-            x_init_value=x_init,
-            delta=delta)
-        self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
+    with self.session(), test_util.use_gpu():
+      theoretical, numerical = gradient_checker_v2.compute_gradient(
+          lambda x: math_ops.matmul(x, effective_b_np, **kwargs_),
+          [effective_a_np],
+          delta=delta)
+      self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
+
+      theoretical, numerical = gradient_checker_v2.compute_gradient(
+          lambda x: math_ops.matmul(effective_a_np, x, **kwargs_),
+          [effective_b_np],
+          delta=delta)
+      self.assertAllClose(theoretical, numerical, rtol=tol, atol=tol)
 
   return Test
 
 
 class MatMulStatsTest(test_lib.TestCase):
 
+  @test_util.run_v1_only("Test requires a Graph and NodeDef inspection")
   def testSimpleStatistics(self):
-    g = ops.Graph()
-    with g.as_default():
-      a = variables.Variable(random_ops.random_normal([25, 16]))
-      b = variables.Variable(random_ops.random_normal([16, 9]))
-      math_ops.matmul(a, b)
-      for op in g.get_operations():
-        flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value
-        if op.name == "MatMul":
-          self.assertEqual(7200, flops)
-
+    a = variables.Variable(random_ops.random_normal([25, 16]))
+    b = variables.Variable(random_ops.random_normal([16, 9]))
+    math_ops.matmul(a, b)
+    g = ops.get_default_graph()
+    for op in g.get_operations():
+      flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value
+      if op.name == "MatMul":
+        self.assertEqual(7200, flops)
+
+  @test_util.run_v1_only("Test requires a Graph and NodeDef inspection")
   def testTransposedStatistics(self):
-    g = ops.Graph()
-    with g.as_default():
-      a = variables.Variable(random_ops.random_normal([16, 25]))
-      b = variables.Variable(random_ops.random_normal([16, 9]))
-      math_ops.matmul(a, b, transpose_a=True)
-      for op in g.get_operations():
-        flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value
-        if op.name == "MatMul":
-          self.assertEqual(7200, flops)
+    a = variables.Variable(random_ops.random_normal([16, 25]))
+    b = variables.Variable(random_ops.random_normal([16, 9]))
+    math_ops.matmul(a, b, transpose_a=True)
+    g = ops.get_default_graph()
+    for op in g.get_operations():
+      flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value
+      if op.name == "MatMul":
+        self.assertEqual(7200, flops)
 
 
 try:
@@ -194,37 +192,33 @@ except AttributeError:
 
 class MatMulInfixOperatorTest(test_lib.TestCase):
 
-  @test_util.run_deprecated_v1
   def testMismatchedShape(self):
-    with self.assertRaisesWithPredicateMatch(ValueError,
-                                             lambda e: "Shape must" in str(e)):
+    with self.assertRaisesRegexp(
+        Exception, "(Shape must be rank 2 but is rank 1|is not a matrix)"):
       infix_matmul(
           ops.convert_to_tensor([10.0, 20.0, 30.0]),
           ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]]))
 
-  @test_util.run_deprecated_v1
   def testMismatchedDimensions(self):
-    with self.assertRaisesWithPredicateMatch(
-        ValueError, lambda e: "Dimensions must" in str(e)):
+    with self.assertRaisesRegexp(
+        Exception, "(Dimensions must be equal|Matrix size-incompatible)"):
       infix_matmul(
           ops.convert_to_tensor([[10.0, 20.0, 30.0]]),
           ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0]]))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("Tensor.op is generally not applicable in TF 2")
   def testInfixMatmulIsTfMatmul(self):
     a = ops.convert_to_tensor([[10.0, 20.0, 30.0]])
     b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]])
     c = infix_matmul(a, b)
     self.assertEqual(c.op.type, "MatMul")
 
-  @test_util.run_deprecated_v1
   def testInfixMatmulDoesDotProduct(self):
     a = ops.convert_to_tensor([[10.0, 20.0, 30.0]])
     b = ops.convert_to_tensor([[40.0, 50.0], [60.0, 70.0], [80.0, 90.0]])
     c = infix_matmul(a, b)
     d = math_ops.matmul(a, b)
-    with self.cached_session():
-      self.assertAllEqual(c.eval(), self.evaluate(d))
+    self.assertAllEqual(c, d)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 9e2808aba35bd8b13dfd5d70e420be42dcc48b0c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 15:16:38 -0800
Subject: [PATCH 1324/1554] Fix calculation of stride in constant gather.

PiperOrigin-RevId: 224055241
---
 tensorflow/lite/build_def.bzl                 |  2 +-
 tensorflow/lite/testing/generate_examples.py  |  6 ++--
 .../resolve_constant_gather.cc                | 29 ++++++++-----------
 tensorflow/lite/toco/model.h                  |  3 ++
 4 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 5eaf719494..dacb8ad767 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -244,7 +244,7 @@ def generated_test_models():
         "fully_connected",
         "fused_batch_norm",
         "gather",
-        "gather_buggy",
+        "gather_with_constant",
         "global_batch_norm",
         "greater",
         "greater_equal",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 4c731a7d18..dd7b3d0745 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -103,8 +103,6 @@ KNOWN_BUGS = {
     r"batch_to_space_nd.*input_shape=\[8,2,2,2,1,1\]": "70594733",
     # Div will use floordiv.
     r"div.*int32": "72051395",
-    # Constant 1D gather crashes toco.
-    r"gather_buggy.*input_shape=\[3\].*": "120029508",
 }
 
 
@@ -1260,8 +1258,8 @@ def make_gather_tests(zip_path):
       expected_tf_success=60)
 
 
-def make_gather_buggy_tests(zip_path):
-  """Make a set of tests to show gather crashes toco."""
+def make_gather_with_constant_tests(zip_path):
+  """Make a set of test which feed a constant to gather toco."""
 
   test_parameters = [{
       "input_shape": [[3]],
diff --git a/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc b/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc
index c72135923e..27836efb0b 100644
--- a/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc
+++ b/tensorflow/lite/toco/graph_transformations/resolve_constant_gather.cc
@@ -25,8 +25,8 @@ namespace {
 
 // Gathers data from axis 0.
 template <ArrayDataType Type>
-inline void Gather(const Array& input_array, int input_rank,
-                   const Array& coords_array, Array* output_array) {
+inline void Gather(const Array& input_array, const Array& coords_array,
+                   Array* output_array) {
   const Shape& input_shape = input_array.shape();
   const std::vector<DataType<Type>>& input_data =
       input_array.GetBuffer<Type>().data;
@@ -39,21 +39,20 @@ inline void Gather(const Array& input_array, int input_rank,
       output_array->GetMutableBuffer<Type>().data;
   output_data.resize(RequiredBufferSizeForShape(output_shape));
 
-  int rev_input_rank = input_shape.dimensions_count() - 1 - (input_rank - 1);
-  CHECK_EQ(coords_shape.dims(0), output_array->shape().dims(rev_input_rank));
+  CHECK_EQ(coords_shape.dims(0), output_array->shape().dims(0));
 
   int stride = 1;
-  for (int i = input_shape.dimensions_count() - 1; i >= input_rank - 1; --i) {
+  for (int i = 1; i < input_shape.dimensions_count(); ++i) {
     stride *= input_shape.dims(i);
   }
 
   // Let's make sure we have enough space for all element in the memcpy()
-  // below, which writes 'stride' elements startng at 'i * stride'.
+  // below, which writes 'stride' elements starting at 'i * stride'.
   CHECK_EQ(stride * coords_shape.dims(0), output_data.size());
 
   for (int i = 0; i < coords_shape.dims(0); ++i) {
     DCHECK_GE(coords_data[i], 0);
-    DCHECK_LT(coords_data[i], input_shape.dims(rev_input_rank));
+    DCHECK_LT(coords_data[i], input_shape.dims(0));
     DataType<Type>* out = output_data.data() + i * stride;
     const DataType<Type>* in = input_data.data() + coords_data[i] * stride;
     memcpy(out, in, sizeof(DataType<Type>) * stride);
@@ -122,24 +121,20 @@ inline void Gather(const Array& input_array, int input_rank,
   CHECK(!output_array.buffer);
   switch (output_array.data_type) {
     case ArrayDataType::kFloat:
-      Gather<ArrayDataType::kFloat>(input_array, op->input_rank, coords_array,
-                                    &output_array);
+      Gather<ArrayDataType::kFloat>(input_array, coords_array, &output_array);
       break;
     case ArrayDataType::kUint8:
-      Gather<ArrayDataType::kUint8>(input_array, op->input_rank, coords_array,
-                                    &output_array);
+      Gather<ArrayDataType::kUint8>(input_array, coords_array, &output_array);
       break;
     case ArrayDataType::kInt32:
-      Gather<ArrayDataType::kInt32>(input_array, op->input_rank, coords_array,
-                                    &output_array);
+      Gather<ArrayDataType::kInt32>(input_array, coords_array, &output_array);
       break;
     case ArrayDataType::kInt64:
-      Gather<ArrayDataType::kInt64>(input_array, op->input_rank, coords_array,
-                                    &output_array);
+      Gather<ArrayDataType::kInt64>(input_array, coords_array, &output_array);
       break;
     case ArrayDataType::kComplex64:
-      Gather<ArrayDataType::kComplex64>(input_array, op->input_rank,
-                                        coords_array, &output_array);
+      Gather<ArrayDataType::kComplex64>(input_array, coords_array,
+                                        &output_array);
       break;
     default:
       LOG(FATAL) << "Unsupported data type given to Gather op with output \""
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index f19355968f..d392535f5c 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -1673,6 +1673,9 @@ struct GatherOperator : Operator {
   // ResolveGatherAttributes. An empty axis indicates that the axis has not yet
   // be resolved.
   absl::optional<int> axis;
+
+  // This field is not used by the standard TF Lite export but it is still need
+  // for legacy Gather implementations.
   int input_rank = 0;
 };
 
-- 
GitLab


From 4b914607fb8384deb22e5b822dca2624e1758718 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Tue, 4 Dec 2018 15:48:05 -0800
Subject: [PATCH 1325/1554] Update the condition for using defun backend in
 LSTM.

Currently defun backend is disabled when the cudnn implementation doesn't support certain behavior, which end up different results between different implementation.

1. The cudnn backend only support 'tanh' as activation.
2. Cudnn does not support recurrent dropout.
3. Cudnn LSTM cannot unroll.
4. Cudnn always use bias gate.
5. If a bias regularizer is specified, it will cause some mathematical difference when save/reload the weight. We disable defun in this case as well.

PiperOrigin-RevId: 224060654
---
 tensorflow/python/keras/layers/recurrent.py   | 53 ++++++++++++++---
 .../python/keras/layers/unified_lstm_test.py  | 57 +++++++++++++++++++
 2 files changed, 102 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index d9502dfc5b..189ad98794 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -2661,9 +2661,10 @@ class UnifiedLSTM(LSTM):
     ]
     self._num_constants = None
     self._num_inputs = None
+    self._dropout_mask = None
     self.could_use_cudnn = (
-        activation == 'tanh' and dropout == 0 and not unroll and use_bias and
-        unit_forget_bias)
+        activation == 'tanh' and recurrent_dropout == 0 and
+        not unroll and use_bias and bias_regularizer is None)
 
   def build(self, input_shape):
     super(UnifiedLSTM, self).build(input_shape)
@@ -2722,6 +2723,16 @@ class UnifiedLSTM(LSTM):
 
       combined_bias = array_ops.concat([self.cudnn_bias, self.cell.bias], 0)
 
+      if 0 < self.dropout < 1:
+        if self._dropout_mask is None:
+          self._dropout_mask = _generate_dropout_mask(
+              array_ops.ones_like(inputs),
+              self.dropout,
+              training=training,
+              count=4)
+
+        inputs *= self._dropout_mask[0]
+
       # Each time a defun function is called, we will give a unique identifiable
       # API name, so that the grappler won't get confused when it sees multiple
       # LSTM layer added into same graph, and it will be able to pair up the
@@ -2835,9 +2846,33 @@ class UnifiedLSTM(LSTM):
     K.batch_set_value(tuples)
 
 
-def _canonical_to_params(weights, biases, shape):
-  """Utility function convert variable to CuDNN compatible parameter."""
-  weights = [array_ops.reshape(x, shape) for x in weights]
+def _canonical_to_params(weights, biases, shape, transpose_weights=False):
+  """Utility function convert variable to CuDNN compatible parameter.
+
+  Note that Keras weights for kernels are different from the CuDNN format. Eg.:
+
+  ```
+    Keras                 CuDNN
+    [[0, 1, 2],  <--->  [[0, 2, 4],
+     [3, 4, 5]]          [1, 3, 5]]
+  ```
+
+  If the input weights need to be in a unified format, then set
+  `transpose_weights=True` to convert the weights.
+
+  Args:
+    weights: list of weights for the individual kernels and recurrent kernels.
+    biases: list of biases for individual gate.
+    shape: the shape for the converted variables that will be feed to CuDNN.
+    transpose_weights: boolean, whether to transpose the weights.
+
+  Returns:
+    The converted weights that can be feed to CuDNN ops as param.
+  """
+  def convert(w):
+    return array_ops.transpose(w) if transpose_weights else w
+
+  weights = [array_ops.reshape(convert(x), shape) for x in weights]
   biases = [array_ops.reshape(x, shape) for x in biases]
   return array_ops.concat(weights + biases, axis=0)
 
@@ -2930,15 +2965,17 @@ def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias,
   params = _canonical_to_params(
       weights=weights,
       biases=array_ops.split(bias, 8),
-      shape=constant_op.constant([-1]))
+      shape=constant_op.constant([-1]),
+      transpose_weights=True)
 
   outputs, h, c, _ = gen_cudnn_rnn_ops.cudnn_rnn(
-      inputs, input_h=input_h, input_c=input_c, params=params)
+      inputs, input_h=input_h, input_c=input_c, params=params, is_training=True)
+  last_output = outputs[-1]
   if not time_major:
     outputs = array_ops.transpose(outputs, perm=[1, 0, 2])
   h = h[0]
   c = c[0]
-  last_output = outputs[:, -1, :]
+
   return last_output, outputs, h, c, constant_op.constant(
       'cudnn', dtype=dtypes.string, name='runtime')
 
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index d229d14312..b004284140 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -157,6 +157,63 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
+  @parameterized.named_parameters(
+      ('_non_tan_activation', 'relu', 0, False, True, None),
+      ('_use_recurrent_dropout', 'tanh', 0.1, False, True, None),
+      ('_unroll', 'tanh', 0, True, True, None),
+      ('_not_use_bias', 'tanh', 0, False, False, None),
+      ('_use_bias_regularizer', 'tanh', 0, False, True, 'l2')
+  )
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_could_use_defun_backend(self, activation, recurrent_dropout,
+                                   unroll, use_bias, bias_regularizer):
+    layer = UnifiedLSTM(1,
+                        activation=activation,
+                        recurrent_dropout=recurrent_dropout,
+                        unroll=unroll,
+                        use_bias=use_bias,
+                        bias_regularizer=bias_regularizer)
+    self.assertFalse(layer.could_use_cudnn)
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_unified_lstm_output_on_multiple_kernel(self):
+    input_shape = 10
+    rnn_state_size = 8
+    timestep = 4
+    batch = 100
+
+    x_train = np.random.random((batch, timestep, input_shape))
+
+    inputs = keras.layers.Input(
+        shape=[timestep, input_shape], dtype=dtypes.float32)
+    with test_util.device(use_gpu=False):
+      # Note that CuDNN use 'sigmoid' as activation. Force the CPU
+      # implementation to use 'sigmoid' so that it will generate same output as
+      # CuDNN implementation.
+      layer = UnifiedLSTM(rnn_state_size, recurrent_activation='sigmoid')
+      output = layer(inputs)
+      cpu_model = keras.models.Model(inputs, output)
+      weights = cpu_model.get_weights()
+      y_1 = cpu_model.predict(x_train)
+
+    with test_util.device(use_gpu=True):
+      layer = UnifiedLSTM(rnn_state_size, recurrent_activation='sigmoid')
+      output = layer(inputs)
+      gpu_model = keras.models.Model(inputs, output)
+      gpu_model.set_weights(weights)
+      y_2 = gpu_model.predict(x_train)
+
+    with test_util.device(use_gpu=True):
+      layer = keras.layers.LSTM(rnn_state_size, recurrent_activation='sigmoid')
+      output = layer(inputs)
+      canonical_model = keras.models.Model(inputs, output)
+      # Remove the extra cudnn bias since canonical lstm will not use it.
+      canonical_model.set_weights(weights[:3])
+      y_3 = canonical_model.predict(x_train)
+
+    self.assertAllClose(y_1, y_2)
+    self.assertAllClose(y_2, y_3)
+
   @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_keras_model_with_lstm(self):
     input_shape = 10
-- 
GitLab


From 43a1df9ac3df076e92655152500c9e9f5ba399be Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 15:54:54 -0800
Subject: [PATCH 1326/1554] Removes a should_use decorator from tf.Assert to
 prevent extraneous warning inside functions.

PiperOrigin-RevId: 224061693
---
 tensorflow/python/BUILD                        |  1 +
 tensorflow/python/ops/control_flow_ops.py      |  2 --
 tensorflow/python/ops/control_flow_ops_test.py | 13 +++++++++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a558045e4a..1384a2ea73 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3250,6 +3250,7 @@ cuda_py_test(
         ":util",
         ":variable_scope",
         ":variables",
+        "//tensorflow/python/eager:def_function",
     ],
 )
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index b7e50c1dae..75c081801a 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -56,7 +56,6 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
 from tensorflow.python.util import nest
-from tensorflow.python.util import tf_should_use
 from tensorflow.python.util.lazy_loader import LazyLoader
 from tensorflow.python.util.tf_export import tf_export
 
@@ -113,7 +112,6 @@ def _summarize_eager(tensor, summarize=None):
 # Assert and Print are special symbols in python, so we must
 # use an upper-case version of them.
 @tf_export("debugging.Assert", "Assert")
-@tf_should_use.should_use_result
 def Assert(condition, data, summarize=None, name=None):
   """Asserts that the given condition is true.
 
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index c020189ad6..b19ec4bd61 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import node_def_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -1014,6 +1015,18 @@ class AssertTest(test_util.TensorFlowTestCase):
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(c)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testAssertInFunction(self):
+
+    @def_function.function
+    def whiny(value):
+      control_flow_ops.Assert(value, ["Raised false"])
+      return constant_op.constant(5)
+
+    with self.assertRaises(errors.InvalidArgumentError):
+      self.evaluate(whiny(False))
+
+    self.assertAllEqual(whiny(True), 5)
 
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From dd39af62c57a319e711c65c332f2223f671d914c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 16:19:16 -0800
Subject: [PATCH 1327/1554] internal change

PiperOrigin-RevId: 224065989
---
 tensorflow/lite/build_def.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index dacb8ad767..c61ee63c7e 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -457,6 +457,7 @@ def gen_model_coverage_test(model_name, data, failure_type, tags):
         native.py_test(
             name = "model_coverage_test_%s_%s" % (model_name, target_op_sets.lower().replace(",", "_")),
             srcs = ["model_coverage_test.py"],
+            size = "large",
             main = "model_coverage_test.py",
             args = [
                 "--model_name=%s" % model_name,
@@ -467,7 +468,6 @@ def gen_model_coverage_test(model_name, data, failure_type, tags):
             tags = [
                 "no_oss",
                 "no_windows",
-                "notap",
             ] + tags,
             deps = [
                 "//tensorflow/lite/testing/model_coverage:model_coverage_lib",
-- 
GitLab


From 207feb253ce1cc54353b1e697af842475e2fb2db Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Tue, 4 Dec 2018 16:26:36 -0800
Subject: [PATCH 1328/1554] Make MakeCsvDatasetTest support eager mode

---
 .../kernel_tests/make_csv_dataset_test.py     | 157 ++++++++----------
 1 file changed, 72 insertions(+), 85 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index e80accee33..f541c84dfa 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -30,11 +30,11 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class MakeCsvDatasetTest(test_base.DatasetTestBase):
 
   def _make_csv_dataset(self, filenames, batch_size, num_epochs=1, **kwargs):
@@ -76,7 +76,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
 
   def _verify_output(
       self,
-      sess,
       dataset,
       batch_size,
       num_epochs,
@@ -84,7 +83,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
       expected_output,
       expected_keys,
   ):
-    nxt = dataset_ops.make_one_shot_iterator(dataset).get_next()
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
 
     for expected_features in self._next_expected_batch(
         expected_output,
@@ -92,6 +91,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         batch_size,
         num_epochs,
     ):
+      nxt = iterator.get_next()
       actual_features = self.evaluate(nxt)
 
       if label_name is not None:
@@ -104,6 +104,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         self.assertAllEqual(expected_features[k], actual_features[k])
 
     with self.assertRaises(errors.OutOfRangeError):
+      nxt = iterator.get_next()
       self.evaluate(nxt)
 
   def _test_dataset(self,
@@ -118,18 +119,15 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
     # Convert str type because py3 tf strings are bytestrings
     filenames = self._setup_files(
         inputs, compression_type=kwargs.get("compression_type", None))
-    with ops.Graph().as_default() as g:
-      with self.session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            filenames,
-            batch_size=batch_size,
-            num_epochs=num_epochs,
-            label_name=label_name,
-            **kwargs)
-        self._verify_output(sess, dataset, batch_size, num_epochs, label_name,
-                            expected_output, expected_keys)
-
-  @test_util.run_deprecated_v1
+    dataset = self._make_csv_dataset(
+        filenames,
+        batch_size=batch_size,
+        num_epochs=num_epochs,
+        label_name=label_name,
+        **kwargs)
+    self._verify_output(dataset, batch_size, num_epochs, label_name,
+                        expected_output, expected_keys)
+
   def testMakeCSVDataset(self):
     """Tests making a CSV dataset with keys and defaults provided."""
     record_defaults = [
@@ -161,7 +159,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withBatchSizeAndEpochs(self):
     """Tests making a CSV dataset with keys and defaults provided."""
     record_defaults = [
@@ -193,7 +190,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withCompressionType(self):
     """Tests `compression_type` argument."""
     record_defaults = [
@@ -262,7 +258,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           label_name="not_a_real_label",
           column_names=column_names)
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withNoLabel(self):
     """Tests making a CSV dataset with no label provided."""
     record_defaults = [
@@ -292,7 +287,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withNoHeader(self):
     """Tests that datasets can be created from CSV files with no header line.
     """
@@ -354,7 +348,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         column_defaults=record_defaults,
     )
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withNoColNames(self):
     """Tests that datasets can be created when column names are not specified.
 
@@ -459,7 +452,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         header=True,
     )
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withSelectCols(self):
     record_defaults = [
         constant_op.constant([], dtypes.int32),
@@ -566,7 +558,6 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           label_name=None,
           select_columns=["invalid_col_name"])
 
-  @test_util.run_deprecated_v1
   def testMakeCSVDataset_withShuffle(self):
     record_defaults = [
         constant_op.constant([], dtypes.int32),
@@ -591,69 +582,65 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
 
     total_records = 20
     for batch_size in [1, 2]:
-      with ops.Graph().as_default() as g:
-        with self.session(graph=g) as sess:
-          # Test that shuffling with the same seed produces the same result
-          dataset1 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
-              batch_size=batch_size,
-              header=True,
-              shuffle=True,
-              shuffle_seed=5,
-              num_epochs=2,
-          )
-          dataset2 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
-              batch_size=batch_size,
-              header=True,
-              shuffle=True,
-              shuffle_seed=5,
-              num_epochs=2,
-          )
-          outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
-          outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
-          for _ in range(total_records // batch_size):
-            batch1 = nest.flatten(self.evaluate(outputs1))
-            batch2 = nest.flatten(self.evaluate(outputs2))
-            for i in range(len(batch1)):
-              self.assertAllEqual(batch1[i], batch2[i])
-
-      with ops.Graph().as_default() as g:
-        with self.session(graph=g) as sess:
-          # Test that shuffling with a different seed produces different results
-          dataset1 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
-              batch_size=batch_size,
-              header=True,
-              shuffle=True,
-              shuffle_seed=5,
-              num_epochs=2,
-          )
-          dataset2 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
-              batch_size=batch_size,
-              header=True,
-              shuffle=True,
-              shuffle_seed=6,
-              num_epochs=2,
-          )
-          outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
-          outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
-          all_equal = False
-          for _ in range(total_records // batch_size):
-            batch1 = nest.flatten(self.evaluate(outputs1))
-            batch2 = nest.flatten(self.evaluate(outputs2))
-            for i in range(len(batch1)):
-              all_equal = all_equal and np.array_equal(batch1[i], batch2[i])
-          self.assertFalse(all_equal)
+      # Test that shuffling with the same seed produces the same result
+      dataset1 = self._make_csv_dataset(
+          filenames,
+          column_defaults=record_defaults,
+          column_names=column_names,
+          batch_size=batch_size,
+          header=True,
+          shuffle=True,
+          shuffle_seed=5,
+          num_epochs=2,
+      )
+      dataset2 = self._make_csv_dataset(
+          filenames,
+          column_defaults=record_defaults,
+          column_names=column_names,
+          batch_size=batch_size,
+          header=True,
+          shuffle=True,
+          shuffle_seed=5,
+          num_epochs=2,
+      )
+      outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
+      outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
+      for _ in range(total_records // batch_size):
+        batch1 = nest.flatten(self.evaluate(outputs1))
+        batch2 = nest.flatten(self.evaluate(outputs2))
+        for i in range(len(batch1)):
+          self.assertAllEqual(batch1[i], batch2[i])
+
+      # Test that shuffling with a different seed produces different results
+      dataset1 = self._make_csv_dataset(
+          filenames,
+          column_defaults=record_defaults,
+          column_names=column_names,
+          batch_size=batch_size,
+          header=True,
+          shuffle=True,
+          shuffle_seed=5,
+          num_epochs=2,
+      )
+      dataset2 = self._make_csv_dataset(
+          filenames,
+          column_defaults=record_defaults,
+          column_names=column_names,
+          batch_size=batch_size,
+          header=True,
+          shuffle=True,
+          shuffle_seed=6,
+          num_epochs=2,
+      )
+      outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
+      outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
+      all_equal = False
+      for _ in range(total_records // batch_size):
+        batch1 = nest.flatten(self.evaluate(outputs1))
+        batch2 = nest.flatten(self.evaluate(outputs2))
+        for i in range(len(batch1)):
+          all_equal = all_equal and np.array_equal(batch1[i], batch2[i])
+      self.assertFalse(all_equal)
 
   def testIndefiniteRepeatShapeInference(self):
     column_names = ["col%d" % i for i in range(5)]
-- 
GitLab


From b0d7d8a477d3041e2d0ebd0cb1d35e4a7fa09663 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Tue, 4 Dec 2018 16:34:56 -0800
Subject: [PATCH 1329/1554] Update protobuf to 3.6.1.2

protobuf 3.6.1.2 contains a fix for the Python3.7 compiler. This will help us
create a nightly Python3.7 build for TensorFlow.

PiperOrigin-RevId: 224068349
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 2074f63202..155a927fed 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -347,11 +347,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     )
 
     PROTOBUF_URLS = [
-        "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.1.tar.gz",
-        "https://github.com/google/protobuf/archive/v3.6.1.1.tar.gz",
+        "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
+        "https://github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
     ]
-    PROTOBUF_SHA256 = "1ade182f91f0fa6c6116195def5d22270e01b9d03fe91319e4c6215022d0d24b"
-    PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1.1"
+    PROTOBUF_SHA256 = "2244b0308846bb22b4ff0bcc675e99290ff9f1115553ae9671eba1030af31bc0"
+    PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1.2"
 
     tf_http_archive(
         name = "protobuf_archive",
-- 
GitLab


From d74b8081bfea973bc01c14fce2c240bbca6b3ef8 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Tue, 4 Dec 2018 16:40:01 -0800
Subject: [PATCH 1330/1554] Change visibility of gpu_swapping_ops to public for
 internal usage.

PiperOrigin-RevId: 224069149
---
 tensorflow/core/grappler/optimizers/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 40eab8b9f0..0f9dfe77fd 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -385,7 +385,7 @@ cc_library(
     srcs = [
         "gpu_swapping_ops.cc",
     ],
-    visibility = ["//tensorflow:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
-- 
GitLab


From 96c582b52c9884b7646fc3db52e4bf32117f521a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 4 Dec 2018 16:44:28 -0800
Subject: [PATCH 1331/1554] Log an admonition to ERROR instead of CHECK-failing
 when XLA GPU is not -O2

Being able to set the opt level is useful when debugging.

PiperOrigin-RevId: 224069782
---
 .../gpu/llvm_gpu_backend/nvptx_backend_lib.cc        | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
index 24f07e6897..abf9e7b6d6 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
@@ -394,8 +394,16 @@ StatusOr<string> CompileModuleToPtx(llvm::Module* module,
   int32 opt_level =
       hlo_module_config.debug_options().xla_backend_optimization_level();
 
-  CHECK_GE(opt_level, 2)
-      << "The XLA GPU backend doesn't support unoptimized code generation";
+  if (opt_level < 2) {
+    LOG(ERROR) << std::string(80, '*');
+    LOG(ERROR) << "The XLA GPU backend doesn't support unoptimized code "
+                  "generation but ";
+    LOG(ERROR) << "--xla_backend_optimization_level is set to " << opt_level
+               << "!";
+    LOG(ERROR) << "(Supported configuration is "
+                  "--xla_backend_optimization_level >= 2.)";
+    LOG(ERROR) << std::string(80, '*');
+  }
 
   AddOptimizationPasses(opt_level,
                         /*size_level=*/0, target_machine.get(), &module_passes,
-- 
GitLab


From 8f6c5d3252cf1fa7c97c46c93b55803660487136 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 4 Dec 2018 17:04:57 -0800
Subject: [PATCH 1332/1554] Automated rollback of commit
 3b905b0e0089e8ffb6213213985a9155728267f2

PiperOrigin-RevId: 224072473
---
 tensorflow/python/eager/pywrap_tensor.cc |  8 --------
 tensorflow/python/eager/tensor_test.py   | 12 ------------
 2 files changed, 20 deletions(-)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 206b96eef6..0d0f70d543 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -220,14 +220,6 @@ TFE_TensorHandle* ConvertToEagerTensor(PyObject* value, PyObject* dtype) {
       return nullptr;
     }
   }
-  tensorflow::Safe_PyObjectPtr value_decrefer;
-  if (PyArray_CheckAnyScalarExact(value)) {
-    // Convert numpy scalars to numpy arrays.
-    value = PyArray_FromScalar(value, nullptr);
-    // The returned value needs to be DECREF'd, but the original value was
-    // created in python code, and doesn't need to be DECREF'd.
-    value_decrefer.reset(value);
-  }
   if (PyArray_Check(value)) {
     int desired_np_dtype = -1;
     if (desired_dtype >= 0) {
diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index 0ee2ff68c2..25442ff048 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -95,18 +95,6 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     t = _create_tensor(values)
     self.assertAllEqual(values, t)
 
-  @test_util.assert_no_new_pyobjects_executing_eagerly
-  def testNumpyDtypeSurvivesThroughTensorConversion(self):
-    scalar_creators = [np.int32, np.int64, np.float32, np.float64]
-    conversion_functions = [ops.convert_to_tensor, constant_op.constant]
-
-    for scalar_creator in scalar_creators:
-      for conversion_function in conversion_functions:
-        np_val = scalar_creator(3)
-        tensor_val = conversion_function(np_val)
-        self.assertEqual(tensor_val.numpy().dtype, np_val.dtype)
-        self.assertEqual(tensor_val.numpy(), np_val)
-
   def testNumpyValueWithCast(self):
     values = np.array([3.0], dtype=np.float32)
     t = _create_tensor(values, dtype=dtypes.float64)
-- 
GitLab


From 8ed9863adb8bd1a9be1a794414699984905485fd Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 4 Dec 2018 17:08:59 -0800
Subject: [PATCH 1333/1554] [TF:XLA] Bump open source abseil revision to
 44b0fafc62d9b8f192e8180cbe9c4b806b339d57

PiperOrigin-RevId: 224073007
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 155a927fed..3612c00407 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "3cf6132129ba87f0781c383bfaf381b7174b5818e81fffcc5d04bb451154f0f2",
-        strip_prefix = "abseil-cpp-f95179062eb65ce40895cc76f1398cce25394369",
+        sha256 = "7bb63336f2522ce893c34ec241e6ccdfdece56b1a1f43eb3a732712dbd426f4f",
+        strip_prefix = "abseil-cpp-44b0fafc62d9b8f192e8180cbe9c4b806b339d57",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/f95179062eb65ce40895cc76f1398cce25394369.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/44b0fafc62d9b8f192e8180cbe9c4b806b339d57.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/44b0fafc62d9b8f192e8180cbe9c4b806b339d57.tar.gz",
         ],
     )
 
-- 
GitLab


From a74f0484b552426bcfc69ade5e9c45df78772e79 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 4 Dec 2018 17:17:32 -0800
Subject: [PATCH 1334/1554] Update batch_scatter to v2.

PiperOrigin-RevId: 224073996
---
 .../kernel_tests/batch_scatter_ops_test.py    |   9 +-
 tensorflow/python/ops/state_ops.py            |   5 +-
 tensorflow/python/ops/variables.py            | 138 +++++++++++++++---
 .../api/golden/v1/tensorflow.-variable.pbtxt  |   4 +
 .../api/golden/v2/tensorflow.-variable.pbtxt  |   4 +
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |   1 +
 7 files changed, 137 insertions(+), 28 deletions(-)

diff --git a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
index eefcdc508f..f70fb93da9 100644
--- a/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/batch_scatter_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
@@ -50,7 +51,8 @@ class ScatterTest(test.TestCase):
                         vtype,
                         itype,
                         repeat_indices=False,
-                        updates_are_scalar=False):
+                        updates_are_scalar=False,
+                        method=False):
     np.random.seed(8)
     with self.cached_session(use_gpu=False):
       for indices_shape in (2,), (3, 7), (3, 4, 7):
@@ -71,7 +73,10 @@ class ScatterTest(test.TestCase):
           # Scatter via tensorflow
           ref = variables.Variable(old)
           ref.initializer.run()
-          tf_scatter(ref, indices, updates).eval()
+          if method:
+            ref.batch_scatter_update(ops.IndexedSlices(indices, updates))
+          else:
+            tf_scatter(ref, indices, updates).eval()
           self.assertAllClose(ref.eval(), new)
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index 76684f89f8..3ac69c1c20 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -32,6 +32,7 @@ from tensorflow.python.ops import gen_state_ops
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_state_ops import *
 # pylint: enable=wildcard-import
+from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -595,7 +596,9 @@ def scatter_nd_sub(ref, indices, updates, use_locking=False, name=None):
       name=name))
 
 
-@tf_export("batch_scatter_update")
+@tf_export(v1=["batch_scatter_update"])
+@deprecation.deprecated(
+    "2018-11-29", "Use the batch_scatter_update method of Variable instead.")
 def batch_scatter_update(ref, indices, updates, use_locking=True, name=None):
   """Generalization of `tf.scatter_update` to axis different than 0.
 
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 4824c92a5a..a31ce65518 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -637,37 +637,84 @@ class Variable(six.with_metaclass(VariableMetaclass,
     """
     raise NotImplementedError
 
+  def batch_scatter_update(self, sparse_delta, use_locking=False, name=None):
+    """Assigns `IndexedSlices` to this variable batch-wise.
+
+    Analogous to `batch_gather`. This assumes that this variable and the
+    sparse_delta IndexedSlices have a series of leading dimensions that are the
+    same for all of them, and the updates are performed on the last dimension of
+    indices. In other words, the dimensions should be the following:
+
+    `num_prefix_dims = sparse_delta.indices.ndims - 1`
+    `batch_dim = num_prefix_dims + 1`
+    `sparse_delta.updates.shape = sparse_delta.indices.shape + var.shape[
+         batch_dim:]`
+
+    where
+
+    `sparse_delta.updates.shape[:num_prefix_dims]`
+    `== sparse_delta.indices.shape[:num_prefix_dims]`
+    `== var.shape[:num_prefix_dims]`
+
+    And the operation performed can be expressed as:
+
+    `var[i_1, ..., i_n,
+         sparse_delta.indices[i_1, ..., i_n, j]] = sparse_delta.updates[
+            i_1, ..., i_n, j]`
+
+    When sparse_delta.indices is a 1D tensor, this operation is equivalent to
+    `scatter_update`.
+
+    To avoid this operation one can looping over the first `ndims` of the
+    variable and using `scatter_update` on the subtensors that result of slicing
+    the first dimension. This is a valid option for `ndims = 1`, but less
+    efficient than this implementation.
+
+    Args:
+      sparse_delta: `IndexedSlices` to be assigned to this variable.
+      use_locking: If `True`, use locking during the operation.
+      name: the name of the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the scattered subtraction has completed.
+
+    Raises:
+      ValueError: if `sparse_delta` is not an `IndexedSlices`.
+    """
+    raise NotImplementedError
+
   def scatter_nd_sub(self, indices, updates, name=None):
     """Applies sparse subtraction to individual values or slices in a Variable.
 
-    `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+    Assuming the variable has rank `P` and `indices` is a `Tensor` of rank `Q`.
 
-    `indices` must be integer tensor, containing indices into `ref`.
+    `indices` must be integer tensor, containing indices into self.
     It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
 
     The innermost dimension of `indices` (with length `K`) corresponds to
     indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
-    dimension of `ref`.
+    dimension of self.
 
     `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 
     ```
-    [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+    [d_0, ..., d_{Q-2}, self.shape[K], ..., self.shape[P-1]].
     ```
 
     For example, say we want to add 4 scattered elements to a rank-1 tensor to
     8 elements. In Python, that update would look like this:
 
     ```python
-        ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+        v = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
         indices = tf.constant([[4], [3], [1] ,[7]])
         updates = tf.constant([9, 10, 11, 12])
-        op = ref.scatter_nd_sub(indices, updates)
+        op = v.scatter_nd_sub(indices, updates)
         with tf.Session() as sess:
           print sess.run(op)
     ```
 
-    The resulting update to ref would look like this:
+    The resulting update to v would look like this:
 
         [1, -9, 3, -6, -6, 6, 7, -4]
 
@@ -691,34 +738,34 @@ class Variable(six.with_metaclass(VariableMetaclass,
   def scatter_nd_add(self, indices, updates, name=None):
     """Applies sparse addition to individual values or slices in a Variable.
 
-    `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+    The Variable has rank `P` and `indices` is a `Tensor` of rank `Q`.
 
-    `indices` must be integer tensor, containing indices into `ref`.
+    `indices` must be integer tensor, containing indices into self.
     It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
 
     The innermost dimension of `indices` (with length `K`) corresponds to
     indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
-    dimension of `ref`.
+    dimension of self.
 
     `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 
     ```
-    [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+    [d_0, ..., d_{Q-2}, self.shape[K], ..., self.shape[P-1]].
     ```
 
     For example, say we want to add 4 scattered elements to a rank-1 tensor to
     8 elements. In Python, that update would look like this:
 
     ```python
-        ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+        v = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
         indices = tf.constant([[4], [3], [1] ,[7]])
         updates = tf.constant([9, 10, 11, 12])
-        add = ref.scatter_nd_add(indices, updates)
+        add = v.scatter_nd_add(indices, updates)
         with tf.Session() as sess:
           print sess.run(add)
     ```
 
-    The resulting update to ref would look like this:
+    The resulting update to v would look like this:
 
         [1, 13, 3, 14, 14, 6, 7, 20]
 
@@ -742,34 +789,34 @@ class Variable(six.with_metaclass(VariableMetaclass,
   def scatter_nd_update(self, indices, updates, name=None):
     """Applies sparse assignment to individual values or slices in a Variable.
 
-    `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+    The Variable has rank `P` and `indices` is a `Tensor` of rank `Q`.
 
-    `indices` must be integer tensor, containing indices into `ref`.
+    `indices` must be integer tensor, containing indices into self.
     It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
 
     The innermost dimension of `indices` (with length `K`) corresponds to
     indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
-    dimension of `ref`.
+    dimension of self.
 
     `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 
     ```
-    [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+    [d_0, ..., d_{Q-2}, self.shape[K], ..., self.shape[P-1]].
     ```
 
     For example, say we want to add 4 scattered elements to a rank-1 tensor to
     8 elements. In Python, that update would look like this:
 
     ```python
-        ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+        v = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
         indices = tf.constant([[4], [3], [1] ,[7]])
         updates = tf.constant([9, 10, 11, 12])
-        op = ref.scatter_nd_assign(indices, updates)
+        op = v.scatter_nd_assign(indices, updates)
         with tf.Session() as sess:
           print sess.run(op)
     ```
 
-    The resulting update to ref would look like this:
+    The resulting update to v would look like this:
 
         [1, 11, 3, 10, 9, 6, 7, 12]
 
@@ -1842,6 +1889,55 @@ class RefVariable(VariableV1):
         use_locking=use_locking,
         name=name)
 
+  def batch_scatter_update(self, sparse_delta, use_locking=False, name=None):
+    """Assigns `IndexedSlices` to this variable batch-wise.
+
+    Analogous to `batch_gather`. This assumes that this variable and the
+    sparse_delta IndexedSlices have a series of leading dimensions that are the
+    same for all of them, and the updates are performed on the last dimension of
+    indices. In other words, the dimensions should be the following:
+
+    `num_prefix_dims = sparse_delta.indices.ndims - 1`
+    `batch_dim = num_prefix_dims + 1`
+    `sparse_delta.updates.shape = sparse_delta.indices.shape + var.shape[
+         batch_dim:]`
+
+    where
+
+    `sparse_delta.updates.shape[:num_prefix_dims]`
+    `== sparse_delta.indices.shape[:num_prefix_dims]`
+    `== var.shape[:num_prefix_dims]`
+
+    And the operation performed can be expressed as:
+
+    `var[i_1, ..., i_n,
+         sparse_delta.indices[i_1, ..., i_n, j]] = sparse_delta.updates[
+            i_1, ..., i_n, j]`
+
+    When sparse_delta.indices is a 1D tensor, this operation is equivalent to
+    `scatter_update`.
+
+    To avoid this operation one can looping over the first `ndims` of the
+    variable and using `scatter_update` on the subtensors that result of slicing
+    the first dimension. This is a valid option for `ndims = 1`, but less
+    efficient than this implementation.
+
+    Args:
+      sparse_delta: `IndexedSlices` to be assigned to this variable.
+      use_locking: If `True`, use locking during the operation.
+      name: the name of the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the scattered subtraction has completed.
+
+    Raises:
+      ValueError: if `sparse_delta` is not an `IndexedSlices`.
+    """
+    return state_ops.batch_scatter_update(
+        self, sparse_delta.indices, sparse_delta.values,
+        use_locking=use_locking, name=name)
+
   def scatter_nd_sub(self, indices, updates, name=None):
     """Applies sparse subtraction to individual values or slices in a Variable.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
index af7fc9d4ef..62d8ea9208 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
@@ -64,6 +64,10 @@ tf_class {
     name: "assign_sub"
     argspec: "args=[\'self\', \'delta\', \'use_locking\', \'name\', \'read_value\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'True\'], "
   }
+  member_method {
+    name: "batch_scatter_update"
+    argspec: "args=[\'self\', \'sparse_delta\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
   member_method {
     name: "count_up_to"
     argspec: "args=[\'self\', \'limit\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
index e85949f23c..6136c8fbe7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
@@ -63,6 +63,10 @@ tf_class {
     name: "assign_sub"
     argspec: "args=[\'self\', \'delta\', \'use_locking\', \'name\', \'read_value\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'True\'], "
   }
+  member_method {
+    name: "batch_scatter_update"
+    argspec: "args=[\'self\', \'sparse_delta\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
   member_method {
     name: "count_up_to"
     argspec: "args=[\'self\', \'limit\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index cb38ae0b49..21df760b1e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -492,10 +492,6 @@ tf_module {
     name: "batch_gather"
     argspec: "args=[\'params\', \'indices\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "batch_scatter_update"
-    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
-  }
   member_method {
     name: "batch_to_space"
     argspec: "args=[\'input\', \'block_shape\', \'crops\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 7fe3d1e794..fd73bec0a7 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -90,6 +90,7 @@ renames = {
     'tf.assign': 'tf.compat.v1.assign',
     'tf.assign_add': 'tf.compat.v1.assign_add',
     'tf.assign_sub': 'tf.compat.v1.assign_sub',
+    'tf.batch_scatter_update': 'tf.compat.v1.batch_scatter_update',
     'tf.betainc': 'tf.math.betainc',
     'tf.ceil': 'tf.math.ceil',
     'tf.check_numerics': 'tf.debugging.check_numerics',
-- 
GitLab


From 513f07b95435e400da7f48b6090322b28a9a14e5 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 4 Dec 2018 17:27:09 -0800
Subject: [PATCH 1335/1554] [Grappler] Fix bug in control overrides (function
 optimizer)

PiperOrigin-RevId: 224075171
---
 tensorflow/core/grappler/optimizers/function_optimizer.cc | 6 +++++-
 .../core/grappler/optimizers/function_optimizer_test.cc   | 8 ++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 69685409a3..9c25eb08d8 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -1665,7 +1665,7 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
       gtl::FlatSet<string> add_ctrl_inputs;
 
       // Remove all invalidated control inputs.
-      for (int idx = 0; idx < node.input_size(); ++idx) {
+      for (int idx = 0; idx < node.input_size(); /* see below */) {
         // TODO(ezhulenev): Use non-allocating TensorId after migrating
         // `control_overrides()` to absl::flat_hash_set.
         SafeTensorId input_tensor = ParseTensorName(node.input(idx));
@@ -1685,6 +1685,10 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
           for (const string& override : overrides->second) {
             add_ctrl_inputs.insert(AsControlDependency(override));
           }
+        } else {
+          // Go to the next input only if the current one was not invalidated,
+          // otherwise we need to check the swapped input as well.
+          ++idx;
         }
       }
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 93a2fcda7b..de091dbe98 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -812,8 +812,8 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
 
        // Return result of multiplication and a current value of the variable.
        NDef("out_1", "Identity", {"f2"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("out_2", "ReadVariableOp", {"v", "^f2"}, {{"dtype", DT_FLOAT}},
-            kDevice)},
+       NDef("out_2", "ReadVariableOp", {"v", "^f1", "^f2"},
+            {{"dtype", DT_FLOAT}}, kDevice)},
 
       // Function library.
       {mul_func});
@@ -860,8 +860,8 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
 
        // Return values read directly from inlined nodes.
        NDef("out_1", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("out_2", "ReadVariableOp", {"v", "^f2/add"}, {{"dtype", DT_FLOAT}},
-            kDevice)},
+       NDef("out_2", "ReadVariableOp", {"v", "^f1/add", "^f2/add"},
+            {{"dtype", DT_FLOAT}}, kDevice)},
 
       // Function library.
       {mul_func});
-- 
GitLab


From 8c764edcf80351e1d6a5392e9f5cfe0391556f9b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 17:33:55 -0800
Subject: [PATCH 1336/1554] Remove #include of command_line_flags.h where not
 needed.

PiperOrigin-RevId: 224076036
---
 tensorflow/compiler/xla/tests/dot_operation_test.cc             | 1 -
 tensorflow/core/api_def/api_test.cc                             | 1 -
 tensorflow/examples/wav_to_spectrogram/wav_to_spectrogram.cc    | 1 -
 tensorflow/tools/graph_transforms/backports.cc                  | 1 -
 tensorflow/tools/graph_transforms/flatten_atrous.cc             | 1 -
 tensorflow/tools/graph_transforms/fold_batch_norms.cc           | 1 -
 tensorflow/tools/graph_transforms/fold_constants_lib.cc         | 1 -
 tensorflow/tools/graph_transforms/fold_old_batch_norms.cc       | 1 -
 tensorflow/tools/graph_transforms/fuse_convolutions.cc          | 1 -
 tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc | 1 -
 tensorflow/tools/graph_transforms/insert_logging.cc             | 1 -
 tensorflow/tools/graph_transforms/obfuscate_names.cc            | 1 -
 tensorflow/tools/graph_transforms/quantize_nodes.cc             | 1 -
 tensorflow/tools/graph_transforms/quantize_weights.cc           | 1 -
 tensorflow/tools/graph_transforms/remove_attribute.cc           | 1 -
 tensorflow/tools/graph_transforms/remove_device.cc              | 1 -
 tensorflow/tools/graph_transforms/remove_nodes.cc               | 1 -
 tensorflow/tools/graph_transforms/rename_attribute.cc           | 1 -
 tensorflow/tools/graph_transforms/rename_op.cc                  | 1 -
 tensorflow/tools/graph_transforms/round_weights.cc              | 1 -
 tensorflow/tools/graph_transforms/sort_by_execution_order.cc    | 1 -
 tensorflow/tools/graph_transforms/sparsify_gather.cc            | 1 -
 tensorflow/tools/graph_transforms/strip_unused_nodes.cc         | 1 -
 23 files changed, 23 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 25091b8d5d..c5d8b663f4 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -30,7 +30,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/test_utils.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
 
 namespace xla {
 namespace {
diff --git a/tensorflow/core/api_def/api_test.cc b/tensorflow/core/api_def/api_test.cc
index d38a8424eb..7405e2ace7 100644
--- a/tensorflow/core/api_def/api_test.cc
+++ b/tensorflow/core/api_def/api_test.cc
@@ -35,7 +35,6 @@ limitations under the License.
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/command_line_flags.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/examples/wav_to_spectrogram/wav_to_spectrogram.cc b/tensorflow/examples/wav_to_spectrogram/wav_to_spectrogram.cc
index 4a429837b7..464484dab8 100644
--- a/tensorflow/examples/wav_to_spectrogram/wav_to_spectrogram.cc
+++ b/tensorflow/examples/wav_to_spectrogram/wav_to_spectrogram.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 
 using tensorflow::DT_FLOAT;
 using tensorflow::DT_UINT8;
diff --git a/tensorflow/tools/graph_transforms/backports.cc b/tensorflow/tools/graph_transforms/backports.cc
index 5c153e8cef..041e7eedfb 100644
--- a/tensorflow/tools/graph_transforms/backports.cc
+++ b/tensorflow/tools/graph_transforms/backports.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/flatten_atrous.cc b/tensorflow/tools/graph_transforms/flatten_atrous.cc
index a6f7cb0ed8..c80b28fbbc 100644
--- a/tensorflow/tools/graph_transforms/flatten_atrous.cc
+++ b/tensorflow/tools/graph_transforms/flatten_atrous.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/fold_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_batch_norms.cc
index 975b17380f..16a0f7d58d 100644
--- a/tensorflow/tools/graph_transforms/fold_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_batch_norms.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
index 6df2718e61..dcc36b1a85 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index 156636ab82..fd546f812c 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/fuse_convolutions.cc b/tensorflow/tools/graph_transforms/fuse_convolutions.cc
index df6e9e6dc2..7754dde9c6 100644
--- a/tensorflow/tools/graph_transforms/fuse_convolutions.cc
+++ b/tensorflow/tools/graph_transforms/fuse_convolutions.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc b/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc
index bd021d094e..5aa2dd4f99 100644
--- a/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc
+++ b/tensorflow/tools/graph_transforms/fuse_quantized_convolution.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/fold_constants_lib.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
diff --git a/tensorflow/tools/graph_transforms/insert_logging.cc b/tensorflow/tools/graph_transforms/insert_logging.cc
index 377665448c..ccc48540eb 100644
--- a/tensorflow/tools/graph_transforms/insert_logging.cc
+++ b/tensorflow/tools/graph_transforms/insert_logging.cc
@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/obfuscate_names.cc b/tensorflow/tools/graph_transforms/obfuscate_names.cc
index c470b51b96..ee8ca3d097 100644
--- a/tensorflow/tools/graph_transforms/obfuscate_names.cc
+++ b/tensorflow/tools/graph_transforms/obfuscate_names.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/quantize_nodes.cc b/tensorflow/tools/graph_transforms/quantize_nodes.cc
index a022f57926..b139dad2dd 100644
--- a/tensorflow/tools/graph_transforms/quantize_nodes.cc
+++ b/tensorflow/tools/graph_transforms/quantize_nodes.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/quantize_weights.cc b/tensorflow/tools/graph_transforms/quantize_weights.cc
index cccae8a992..a1a6e27171 100644
--- a/tensorflow/tools/graph_transforms/quantize_weights.cc
+++ b/tensorflow/tools/graph_transforms/quantize_weights.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/remove_attribute.cc b/tensorflow/tools/graph_transforms/remove_attribute.cc
index b1a04c0f28..0a76c2b205 100644
--- a/tensorflow/tools/graph_transforms/remove_attribute.cc
+++ b/tensorflow/tools/graph_transforms/remove_attribute.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/remove_device.cc b/tensorflow/tools/graph_transforms/remove_device.cc
index 975fa37063..fdd43168a1 100644
--- a/tensorflow/tools/graph_transforms/remove_device.cc
+++ b/tensorflow/tools/graph_transforms/remove_device.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/remove_nodes.cc b/tensorflow/tools/graph_transforms/remove_nodes.cc
index 05f036a86a..aa0288689d 100644
--- a/tensorflow/tools/graph_transforms/remove_nodes.cc
+++ b/tensorflow/tools/graph_transforms/remove_nodes.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/rename_attribute.cc b/tensorflow/tools/graph_transforms/rename_attribute.cc
index bd066aab5b..62897d43a8 100644
--- a/tensorflow/tools/graph_transforms/rename_attribute.cc
+++ b/tensorflow/tools/graph_transforms/rename_attribute.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/rename_op.cc b/tensorflow/tools/graph_transforms/rename_op.cc
index e1e13c1be4..9deee8bbff 100644
--- a/tensorflow/tools/graph_transforms/rename_op.cc
+++ b/tensorflow/tools/graph_transforms/rename_op.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/round_weights.cc b/tensorflow/tools/graph_transforms/round_weights.cc
index 72927e439b..3a145ac1f6 100644
--- a/tensorflow/tools/graph_transforms/round_weights.cc
+++ b/tensorflow/tools/graph_transforms/round_weights.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/sort_by_execution_order.cc b/tensorflow/tools/graph_transforms/sort_by_execution_order.cc
index 43152d20fc..548f5ba482 100644
--- a/tensorflow/tools/graph_transforms/sort_by_execution_order.cc
+++ b/tensorflow/tools/graph_transforms/sort_by_execution_order.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/tools/graph_transforms/sparsify_gather.cc b/tensorflow/tools/graph_transforms/sparsify_gather.cc
index cc82100148..bed51f8982 100644
--- a/tensorflow/tools/graph_transforms/sparsify_gather.cc
+++ b/tensorflow/tools/graph_transforms/sparsify_gather.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/core/util/tensor_bundle/tensor_bundle.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
diff --git a/tensorflow/tools/graph_transforms/strip_unused_nodes.cc b/tensorflow/tools/graph_transforms/strip_unused_nodes.cc
index ae9d0aa209..d466f21c17 100644
--- a/tensorflow/tools/graph_transforms/strip_unused_nodes.cc
+++ b/tensorflow/tools/graph_transforms/strip_unused_nodes.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/graph/subgraph.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
 
 namespace tensorflow {
-- 
GitLab


From acb5d5815a7d6a59cef43f706ea206b0b2ee2af0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 17:53:20 -0800
Subject: [PATCH 1337/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 224078991
---
 tensorflow/core/ops/compat/ops_history.v1.pbtxt | 11 +++++++++++
 tensorflow/core/ops/ops.pbtxt                   | 11 +++++++++++
 2 files changed, 22 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 8022c390c6..03eb18ca68 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -21595,6 +21595,17 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalDatasetCardinality"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "cardinality"
+    type: DT_INT64
+  }
+}
 op {
   name: "ExperimentalDatasetToTFRecord"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index a9c712e138..1ff908aae8 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10090,6 +10090,17 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalDatasetCardinality"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "cardinality"
+    type: DT_INT64
+  }
+}
 op {
   name: "ExperimentalDatasetToTFRecord"
   input_arg {
-- 
GitLab


From b7e785979d01e411d528868ca172e1eb26c28251 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 17:55:09 -0800
Subject: [PATCH 1338/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 224079247

---
 tensorflow/go/op/wrappers.go | 7172 +++++++++++++++++-----------------
 1 file changed, 3586 insertions(+), 3586 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 5cd06299b9..d0ea6a4ce1 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5588,6 +5588,205 @@ func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncomp
 	return op.Output(0)
 }
 
+// MapSizeAttr is an optional argument to MapSize.
+type MapSizeAttr func(optionalAttr)
+
+// MapSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapSizeCapacity(value int64) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapSizeMemoryLimit(value int64) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapSizeContainer(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapSizeSharedName(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of elements in the underlying container.
+func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapSize",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MapUnstageAttr is an optional argument to MapUnstage.
+type MapUnstageAttr func(optionalAttr)
+
+// MapUnstageCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageCapacity(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageContainer(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageSharedName(value string) MapUnstageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns the values associated with the key
+//
+// from the underlying container.   If the underlying container
+// does not contain this key, the op will block until it does.
+func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapUnstage",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstage", err)
+		return
+	}
+	return values
+}
+
+// MapPeekAttr is an optional argument to MapPeek.
+type MapPeekAttr func(optionalAttr)
+
+// MapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekCapacity(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekMemoryLimit(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapPeekContainer(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapPeekSharedName(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op peeks at the values at the specified key.  If the
+//
+// underlying container does not contain this key
+// this op will block until it does.
+func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapPeek",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapPeek", err)
+		return
+	}
+	return values
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -5824,37 +6023,6 @@ func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, fil
 	return op.Output(0)
 }
 
-// ApproximateEqualAttr is an optional argument to ApproximateEqual.
-type ApproximateEqualAttr func(optionalAttr)
-
-// ApproximateEqualTolerance sets the optional tolerance attribute to value.
-// If not specified, defaults to 1e-05
-func ApproximateEqualTolerance(value float32) ApproximateEqualAttr {
-	return func(m optionalAttr) {
-		m["tolerance"] = value
-	}
-}
-
-// Returns the truth value of abs(x-y) < tolerance element-wise.
-func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...ApproximateEqualAttr) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ApproximateEqual",
-		Input: []tf.Input{
-			x, y,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Returns x / y element-wise.
 //
 // *NOTE*: `Div` supports broadcasting. More about broadcasting
@@ -7654,54 +7822,16 @@ func DecodeBmp(scope *Scope, contents tf.Output, optional ...DecodeBmpAttr) (ima
 	return op.Output(0)
 }
 
-// Computes softsign gradients for a softsign operation.
-//
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding softsign operation.
-//	features: The features passed as input to the corresponding softsign operation.
+// BatchMatMulAttr is an optional argument to BatchMatMul.
+type BatchMatMulAttr func(optionalAttr)
+
+// BatchMatMulAdjX sets the optional adj_x attribute to value.
 //
-// Returns The gradients: `gradients / (1 + abs(features)) ** 2`.
-func SoftsignGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SoftsignGrad",
-		Input: []tf.Input{
-			gradients, features,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Provides the time since epoch in seconds.
-//
-// Returns the timestamp as a `float64` for seconds since the Unix epoch.
-//
-// Note: the timestamp is computed when the op is executed, not when it is added
-// to the graph.
-func Timestamp(scope *Scope) (ts tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Timestamp",
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// BatchMatMulAttr is an optional argument to BatchMatMul.
-type BatchMatMulAttr func(optionalAttr)
-
-// BatchMatMulAdjX sets the optional adj_x attribute to value.
-//
-// value: If `True`, adjoint the slices of `x`. Defaults to `False`.
-// If not specified, defaults to false
-func BatchMatMulAdjX(value bool) BatchMatMulAttr {
-	return func(m optionalAttr) {
-		m["adj_x"] = value
+// value: If `True`, adjoint the slices of `x`. Defaults to `False`.
+// If not specified, defaults to false
+func BatchMatMulAdjX(value bool) BatchMatMulAttr {
+	return func(m optionalAttr) {
+		m["adj_x"] = value
 	}
 }
 
@@ -11950,6 +12080,134 @@ func ResourceScatterUpdate(scope *Scope, resource tf.Output, indices tf.Output,
 	return scope.AddOperation(opspec)
 }
 
+// MaxPoolAttr is an optional argument to MaxPool.
+type MaxPoolAttr func(optionalAttr)
+
+// MaxPoolDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolDataFormat(value string) MaxPoolAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Performs max pooling on the input.
+//
+// Arguments:
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The max pooled output tensor.
+func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MaxPool",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Multiplies sparse updates into the variable referenced by `resource`.
+//
+// This operation computes
+//
+//     # Scalar indices
+//     ref[indices, ...] *= updates[...]
+//
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] *= updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] *= updates[i, ..., j, ...]
+//
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions multiply.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// </div>
+//
+// Arguments:
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
+//
+// Returns the created operation.
+func ResourceScatterMul(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterMul",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Subtracts sparse updates from the variable referenced by `resource`.
+//
+// This operation computes
+//
+//     # Scalar indices
+//     ref[indices, ...] -= updates[...]
+//
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] -= updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...]
+//
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions add.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// </div>
+//
+// Arguments:
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
+//
+// Returns the created operation.
+func ResourceScatterSub(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterSub",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // ResourceSparseApplyFtrlV2Attr is an optional argument to ResourceSparseApplyFtrlV2.
 type ResourceSparseApplyFtrlV2Attr func(optionalAttr)
 
@@ -12554,614 +12812,66 @@ func Bucketize(scope *Scope, input tf.Output, boundaries []float32) (output tf.O
 	return op.Output(0)
 }
 
-// EncodeJpegAttr is an optional argument to EncodeJpeg.
-type EncodeJpegAttr func(optionalAttr)
-
-// EncodeJpegFormat sets the optional format attribute to value.
+// Encode audio data using the WAV file format.
 //
-// value: Per pixel image format.
-// If not specified, defaults to ""
-func EncodeJpegFormat(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["format"] = value
-	}
-}
-
-// EncodeJpegQuality sets the optional quality attribute to value.
+// This operation will generate a string suitable to be saved out to create a .wav
+// audio file. It will be encoded in the 16-bit PCM format. It takes in float
+// values in the range -1.0f to 1.0f, and any outside that value will be clamped to
+// that range.
 //
-// value: Quality of the compression from 0 to 100 (higher is better and slower).
-// If not specified, defaults to 95
-func EncodeJpegQuality(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["quality"] = value
-	}
-}
-
-// EncodeJpegProgressive sets the optional progressive attribute to value.
+// `audio` is a 2-D float Tensor of shape `[length, channels]`.
+// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
 //
-// value: If True, create a JPEG that loads progressively (coarse to fine).
-// If not specified, defaults to false
-func EncodeJpegProgressive(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["progressive"] = value
-	}
-}
-
-// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
+// Arguments:
+//	audio: 2-D with shape `[length, channels]`.
+//	sample_rate: Scalar containing the sample frequency.
 //
-// value: If True, spend CPU/RAM to reduce size with no quality change.
-// If not specified, defaults to false
-func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["optimize_size"] = value
+// Returns 0-D. WAV-encoded file contents.
+func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
+	opspec := tf.OpSpec{
+		Type: "EncodeWav",
+		Input: []tf.Input{
+			audio, sample_rate,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
-//
-// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
-// If not specified, defaults to true
-func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["chroma_downsampling"] = value
+// Computes atan of x element-wise.
+func Atan(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Atan",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
+// ResourceApplyAdaMaxAttr is an optional argument to ResourceApplyAdaMax.
+type ResourceApplyAdaMaxAttr func(optionalAttr)
+
+// ResourceApplyAdaMaxUseLocking sets the optional use_locking attribute to value.
 //
-// value: Unit used to specify `x_density` and `y_density`:
-// pixels per inch (`'in'`) or centimeter (`'cm'`).
-// If not specified, defaults to "in"
-func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
+// value: If `True`, updating of the var, m, and v tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdaMaxUseLocking(value bool) ResourceApplyAdaMaxAttr {
 	return func(m optionalAttr) {
-		m["density_unit"] = value
+		m["use_locking"] = value
 	}
 }
 
-// EncodeJpegXDensity sets the optional x_density attribute to value.
-//
-// value: Horizontal pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegXDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["x_density"] = value
-	}
-}
-
-// EncodeJpegYDensity sets the optional y_density attribute to value.
-//
-// value: Vertical pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegYDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["y_density"] = value
-	}
-}
-
-// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
-//
-// value: If not empty, embed this XMP metadata in the image header.
-// If not specified, defaults to ""
-func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["xmp_metadata"] = value
-	}
-}
-
-// JPEG-encode an image.
-//
-// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
-//
-// The attr `format` can be used to override the color format of the encoded
-// output.  Values can be:
-//
-// *   `''`: Use a default format based on the number of channels in the image.
-// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
-//     of `image` must be 1.
-// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
-//     of `image` must be 3.
-//
-// If `format` is not specified or is the empty string, a default format is picked
-// in function of the number of channels in `image`:
-//
-// *   1: Output a grayscale image.
-// *   3: Output an RGB image.
-//
-// Arguments:
-//	image: 3-D with shape `[height, width, channels]`.
-//
-// Returns 0-D. JPEG-encoded image.
-func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "EncodeJpeg",
-		Input: []tf.Input{
-			image,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MultinomialAttr is an optional argument to Multinomial.
-type MultinomialAttr func(optionalAttr)
-
-// MultinomialSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 is set to be non-zero, the internal random number
-// generator is seeded by the given seed.  Otherwise, a random seed is used.
-// If not specified, defaults to 0
-func MultinomialSeed(value int64) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// MultinomialSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func MultinomialSeed2(value int64) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// MultinomialOutputDtype sets the optional output_dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["output_dtype"] = value
-	}
-}
-
-// Draws samples from a multinomial distribution.
-//
-// Arguments:
-//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
-// represents the unnormalized log probabilities for all classes.
-//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
-//
-// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
-// contains the drawn class labels with range `[0, num_classes)`.
-func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Multinomial",
-		Input: []tf.Input{
-			logits, num_samples,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
-type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
-
-// ResourceSparseApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
-//
-// value: If True, updating of the var and accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceSparseApplyAdagradDAUseLocking(value bool) ResourceSparseApplyAdagradDAAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update entries in '*var' and '*accum' according to the proximal adagrad scheme.
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	gradient_accumulator: Should be from a Variable().
-//	gradient_squared_accumulator: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	lr: Learning rate. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	global_step: Training step number. Must be a scalar.
-//
-// Returns the created operation.
-func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceSparseApplyAdagradDAAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyAdagradDA",
-		Input: []tf.Input{
-			var_, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// ResourceSparseApplyFtrlAttr is an optional argument to ResourceSparseApplyFtrl.
-type ResourceSparseApplyFtrlAttr func(optionalAttr)
-
-// ResourceSparseApplyFtrlUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyFtrlUseLocking(value bool) ResourceSparseApplyFtrlAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
-//
-// That is for rows we have grad for, we update var, accum and linear as follows:
-// accum_new = accum + grad * grad
-// linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	lr_power: Scaling factor. Must be a scalar.
-//
-// Returns the created operation.
-func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyFtrl",
-		Input: []tf.Input{
-			var_, accum, linear, grad, indices, lr, l1, l2, lr_power,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Returns which elements of x are Inf.
-//
-// @compatibility(numpy)
-// Equivalent to np.isinf
-// @end_compatibility
-func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IsInf",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// TruncatedNormalAttr is an optional argument to TruncatedNormal.
-type TruncatedNormalAttr func(optionalAttr)
-
-// TruncatedNormalSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random values from a truncated normal distribution.
-//
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
-//
-// Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
-//
-// Returns A tensor of the specified shape filled with random truncated normal
-// values.
-func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "TruncatedNormal",
-		Input: []tf.Input{
-			shape,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SkipgramAttr is an optional argument to Skipgram.
-type SkipgramAttr func(optionalAttr)
-
-// SkipgramWindowSize sets the optional window_size attribute to value.
-//
-// value: The number of words to predict to the left and right of the target.
-// If not specified, defaults to 5
-func SkipgramWindowSize(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["window_size"] = value
-	}
-}
-
-// SkipgramMinCount sets the optional min_count attribute to value.
-//
-// value: The minimum number of word occurrences for it to be included in the
-// vocabulary.
-// If not specified, defaults to 5
-func SkipgramMinCount(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["min_count"] = value
-	}
-}
-
-// SkipgramSubsample sets the optional subsample attribute to value.
-//
-// value: Threshold for word occurrence. Words that appear with higher
-// frequency will be randomly down-sampled. Set to 0 to disable.
-// If not specified, defaults to 0.001
-func SkipgramSubsample(value float32) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["subsample"] = value
-	}
-}
-
-// Parses a text file and creates a batch of examples.
-//
-// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
-//
-// Arguments:
-//	filename: The corpus's text file name.
-//	batch_size: The size of produced batch.
-//
-// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
-func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Skipgram",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
-}
-
-// StringToNumberAttr is an optional argument to StringToNumber.
-type StringToNumberAttr func(optionalAttr)
-
-// StringToNumberOutType sets the optional out_type attribute to value.
-//
-// value: The numeric type to interpret each string in `string_tensor` as.
-// If not specified, defaults to DT_FLOAT
-func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Converts each string in the input Tensor to the specified numeric type.
-//
-// (Note that int32 overflow results in an error while float overflow
-// results in a rounded value.)
-//
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StringToNumber",
-		Input: []tf.Input{
-			string_tensor,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
-type ResourceApplyFtrlV2Attr func(optionalAttr)
-
-// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' according to the Ftrl-proximal scheme.
-//
-// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
-// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
-// linear += grad_with_shrinkage +
-//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 shrinkage regulariation. Must be a scalar.
-//
-//	lr_power: Scaling factor. Must be a scalar.
-//
-// Returns the created operation.
-func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrlV2",
-		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`.
-//
-// This Op does not require `a_indices` be sorted in standard lexicographic order.
-//
-// Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, with shape `[nnz]`.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, with shape `[ndims]`.
-//	b: `ndims`-D Tensor.  With shape `a_shape`.
-func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseAdd",
-		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Encode audio data using the WAV file format.
-//
-// This operation will generate a string suitable to be saved out to create a .wav
-// audio file. It will be encoded in the 16-bit PCM format. It takes in float
-// values in the range -1.0f to 1.0f, and any outside that value will be clamped to
-// that range.
-//
-// `audio` is a 2-D float Tensor of shape `[length, channels]`.
-// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
-//
-// Arguments:
-//	audio: 2-D with shape `[length, channels]`.
-//	sample_rate: Scalar containing the sample frequency.
-//
-// Returns 0-D. WAV-encoded file contents.
-func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "EncodeWav",
-		Input: []tf.Input{
-			audio, sample_rate,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes atan of x element-wise.
-func Atan(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Atan",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceApplyAdaMaxAttr is an optional argument to ResourceApplyAdaMax.
-type ResourceApplyAdaMaxAttr func(optionalAttr)
-
-// ResourceApplyAdaMaxUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, m, and v tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAdaMaxUseLocking(value bool) ResourceApplyAdaMaxAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' according to the AdaMax algorithm.
+// Update '*var' according to the AdaMax algorithm.
 //
 // m_t <- beta1 * m_{t-1} + (1 - beta1) * g
 // v_t <- max(beta2 * v_{t-1}, abs(g))
@@ -14066,52 +13776,6 @@ func ReaderSerializeStateV2(scope *Scope, reader_handle tf.Output) (state tf.Out
 	return op.Output(0)
 }
 
-// MaxPoolAttr is an optional argument to MaxPool.
-type MaxPoolAttr func(optionalAttr)
-
-// MaxPoolDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolDataFormat(value string) MaxPoolAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs max pooling on the input.
-//
-// Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The max pooled output tensor.
-func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MaxPool",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Concatenates quantized tensors along one dimension.
 //
 // Arguments:
@@ -14210,149 +13874,402 @@ func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Out
 //	indices: A tensor of indices into the first dimension of `ref`.
 //	updates: A tensor of updated values to add to `ref`.
 //
-// Returns the created operation.
-func ResourceScatterMin(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+// Returns the created operation.
+func ResourceScatterMin(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterMin",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Reshapes a quantized tensor as per the Reshape op.
+//
+// ```
+//
+// Arguments:
+//
+//	shape: Defines the shape of the output tensor.
+//	input_min: The minimum value of the input.
+//	input_max: The maximum value of the input.
+//
+// Returns This value is copied from input_min.This value is copied from input_max.
+func QuantizedReshape(scope *Scope, tensor tf.Output, shape tf.Output, input_min tf.Output, input_max tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizedReshape",
+		Input: []tf.Input{
+			tensor, shape, input_min, input_max,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// StringSplitAttr is an optional argument to StringSplit.
+type StringSplitAttr func(optionalAttr)
+
+// StringSplitSkipEmpty sets the optional skip_empty attribute to value.
+//
+// value: A `bool`. If `True`, skip the empty strings from the result.
+// If not specified, defaults to true
+func StringSplitSkipEmpty(value bool) StringSplitAttr {
+	return func(m optionalAttr) {
+		m["skip_empty"] = value
+	}
+}
+
+// Split elements of `input` based on `delimiter` into a `SparseTensor`.
+//
+// Let N be the size of source (typically N will be the batch size). Split each
+// element of `input` based on `delimiter` and return a `SparseTensor`
+// containing the splitted tokens. Empty tokens are ignored.
+//
+// `delimiter` can be empty, or a string of split characters. If `delimiter` is an
+//  empty string, each element of `input` is split into individual single-byte
+//  character strings, including splitting of UTF-8 multibyte sequences. Otherwise
+//  every character of `delimiter` is a potential split point.
+//
+// For example:
+//   N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output
+//   will be
+//
+//   indices = [0, 0;
+//              0, 1;
+//              1, 0;
+//              1, 1;
+//              1, 2]
+//   shape = [2, 3]
+//   values = ['hello', 'world', 'a', 'b', 'c']
+//
+// Arguments:
+//	input: 1-D. Strings to split.
+//	delimiter: 0-D. Delimiter characters (bytes), or empty string.
+//
+// Returns A dense matrix of int64 representing the indices of the sparse tensor.A vector of strings corresponding to the splited values.a length-2 vector of int64 representing the shape of the sparse
+// tensor, where the first value is N and the second value is the maximum number
+// of tokens in a single input entry.
+func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ...StringSplitAttr) (indices tf.Output, values tf.Output, shape tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StringSplit",
+		Input: []tf.Input{
+			input, delimiter,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// ResourceSparseApplyMomentumAttr is an optional argument to ResourceSparseApplyMomentum.
+type ResourceSparseApplyMomentumAttr func(optionalAttr)
+
+// ResourceSparseApplyMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyMomentumUseLocking(value bool) ResourceSparseApplyMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceSparseApplyMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var - lr * momentum * accum, so in the end, the var you get is actually
+// var - lr * momentum * accum.
+// If not specified, defaults to false
+func ResourceSparseApplyMomentumUseNesterov(value bool) ResourceSparseApplyMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+//
+// Set use_nesterov = True if you want to use Nesterov momentum.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+//
+// accum = accum * momentum + grad
+// var -= lr * accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, indices, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Returns the complex conjugate of a complex number.
+//
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// complex numbers that are the complex conjugate of each element in `input`. The
+// complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
+// real part and *b* is the imaginary part.
+//
+// The complex conjugate returned by this operation is of the form \\(a - bj\\).
+//
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
+// ```
+func Conj(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Conj",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes numerical negative value element-wise.
+//
+// I.e., \\(y = -x\\).
+func Neg(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterMin",
+		Type: "Neg",
 		Input: []tf.Input{
-			resource, indices, updates,
+			x,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Reshapes a quantized tensor as per the Reshape op.
+// Execute a sub graph on a remote processor.
 //
-// ```
+// The graph specifications(such as graph itself, input tensors and output names)
+// are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo
+// as serialized_remote_fused_graph_execute_info.
+// The specifications will be passed to a dedicated registered
+// remote fused graph executor.  The executor will send the graph specifications
+// to a remote processor and execute that graph.  The execution results
+// will be passed to consumer nodes as outputs of this node.
 //
 // Arguments:
+//	inputs: Arbitrary number of tensors with arbitrary data types
 //
-//	shape: Defines the shape of the output tensor.
-//	input_min: The minimum value of the input.
-//	input_max: The maximum value of the input.
+//	serialized_remote_fused_graph_execute_info: Serialized protocol buffer
+// of RemoteFusedGraphExecuteInfo which contains graph specifications.
 //
-// Returns This value is copied from input_min.This value is copied from input_max.
-func QuantizedReshape(scope *Scope, tensor tf.Output, shape tf.Output, input_min tf.Output, input_max tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// Returns Arbitrary number of tensors with arbitrary data types
+func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.DataType, serialized_remote_fused_graph_execute_info string) (outputs []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"Toutputs": Toutputs, "serialized_remote_fused_graph_execute_info": serialized_remote_fused_graph_execute_info}
 	opspec := tf.OpSpec{
-		Type: "QuantizedReshape",
+		Type: "RemoteFusedGraphExecute",
 		Input: []tf.Input{
-			tensor, shape, input_min, input_max,
+			tf.OutputList(inputs),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
+		scope.UpdateErr("RemoteFusedGraphExecute", err)
+		return
+	}
+	return outputs
 }
 
-// Returns the truth value of (x != y) element-wise.
+// MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad.
+type MaxPool3DGradGradAttr func(optionalAttr)
+
+// MaxPool3DGradGradDataFormat sets the optional data_format attribute to value.
 //
-// *NOTE*: `NotEqual` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func NotEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func MaxPool3DGradGradDataFormat(value string) MaxPool3DGradGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Computes second-order gradients of the maxpooling function.
+//
+// Arguments:
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+//
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "NotEqual",
+		Type: "MaxPool3DGradGrad",
 		Input: []tf.Input{
-			x, y,
+			orig_input, orig_output, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse 3D real-valued fast Fourier transform.
-//
-// Computes the inverse 3-dimensional discrete Fourier transform of a real-valued
-// signal over the inner-most 3 dimensions of `input`.
+// Conv3DBackpropFilterV2Attr is an optional argument to Conv3DBackpropFilterV2.
+type Conv3DBackpropFilterV2Attr func(optionalAttr)
+
+// Conv3DBackpropFilterV2DataFormat sets the optional data_format attribute to value.
 //
-// The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
-// The inner-most dimension contains the `fft_length / 2 + 1` unique components of
-// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
-// from the size of the inner-most 3 dimensions of `input`. If the FFT length used
-// to compute `input` is odd, it should be provided since it cannot be inferred
-// properly.
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func Conv3DBackpropFilterV2DataFormat(value string) Conv3DBackpropFilterV2Attr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Conv3DBackpropFilterV2Dilations sets the optional dilations attribute to value.
 //
-// Along each axis `IRFFT3D` is computed on, if `fft_length` (or
-// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// value: 1-D tensor of length 5.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
+func Conv3DBackpropFilterV2Dilations(value []int64) Conv3DBackpropFilterV2Attr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes the gradients of 3-D convolution with respect to the filter.
 //
 // Arguments:
-//	input: A complex64 tensor.
-//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
-//
-// Returns A float32 tensor of the same rank as `input`. The inner-most 3
-//   dimensions of `input` are replaced with the `fft_length` samples of their
-//   inverse 3D real Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.irfftn with 3 dimensions.
-// @end_compatibility
-func IRFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+//	input: Shape `[batch, depth, rows, cols, in_channels]`.
+//	filter_sizes: An integer vector representing the tensor shape of `filter`,
+// where `filter` is a 5-D
+// `[filter_depth, filter_height, filter_width, in_channels, out_channels]`
+// tensor.
+//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
+// out_channels]`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "IRFFT3D",
+		Type: "Conv3DBackpropFilterV2",
 		Input: []tf.Input{
-			input, fft_length,
+			input, filter_sizes, out_backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StringSplitAttr is an optional argument to StringSplit.
-type StringSplitAttr func(optionalAttr)
+// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars.
+type FakeQuantWithMinMaxVarsAttr func(optionalAttr)
 
-// StringSplitSkipEmpty sets the optional skip_empty attribute to value.
-//
-// value: A `bool`. If `True`, skip the empty strings from the result.
-// If not specified, defaults to true
-func StringSplitSkipEmpty(value bool) StringSplitAttr {
+// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr {
 	return func(m optionalAttr) {
-		m["skip_empty"] = value
+		m["num_bits"] = value
 	}
 }
 
-// Split elements of `input` based on `delimiter` into a `SparseTensor`.
-//
-// Let N be the size of source (typically N will be the batch size). Split each
-// element of `input` based on `delimiter` and return a `SparseTensor`
-// containing the splitted tokens. Empty tokens are ignored.
-//
-// `delimiter` can be empty, or a string of split characters. If `delimiter` is an
-//  empty string, each element of `input` is split into individual single-byte
-//  character strings, including splitting of UTF-8 multibyte sequences. Otherwise
-//  every character of `delimiter` is a potential split point.
-//
-// For example:
-//   N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output
-//   will be
+// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value.
+// If not specified, defaults to false
+func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr {
+	return func(m optionalAttr) {
+		m["narrow_range"] = value
+	}
+}
+
+// Fake-quantize the 'inputs' tensor of type float via global float scalars `min`
 //
-//   indices = [0, 0;
-//              0, 1;
-//              1, 0;
-//              1, 1;
-//              1, 2]
-//   shape = [2, 3]
-//   values = ['hello', 'world', 'a', 'b', 'c']
+// and `max` to 'outputs' tensor of same shape as `inputs`.
 //
-// Arguments:
-//	input: 1-D. Strings to split.
-//	delimiter: 0-D. Delimiter characters (bytes), or empty string.
+// `[min; max]` define the clamping range for the `inputs` data.
+// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+// then de-quantized and output as floats in `[min; max]` interval.
+// `num_bits` is the bitwidth of the quantization; between 2 and 16, inclusive.
 //
-// Returns A dense matrix of int64 representing the indices of the sparse tensor.A vector of strings corresponding to the splited values.a length-2 vector of int64 representing the shape of the sparse
-// tensor, where the first value is N and the second value is the maximum number
-// of tokens in a single input entry.
-func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ...StringSplitAttr) (indices tf.Output, values tf.Output, shape tf.Output) {
+// This operation has a gradient and thus allows for training `min` and `max`
+// values.
+func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14361,62 +14278,78 @@ func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ..
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StringSplit",
+		Type: "FakeQuantWithMinMaxVars",
 		Input: []tf.Input{
-			input, delimiter,
+			inputs, min, max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// ResourceSparseApplyMomentumAttr is an optional argument to ResourceSparseApplyMomentum.
-type ResourceSparseApplyMomentumAttr func(optionalAttr)
-
-// ResourceSparseApplyMomentumUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyMomentumUseLocking(value bool) ResourceSparseApplyMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
+// ResourceScatterNdUpdateAttr is an optional argument to ResourceScatterNdUpdate.
+type ResourceScatterNdUpdateAttr func(optionalAttr)
 
-// ResourceSparseApplyMomentumUseNesterov sets the optional use_nesterov attribute to value.
+// ResourceScatterNdUpdateUseLocking sets the optional use_locking attribute to value.
 //
-// value: If `True`, the tensor passed to compute grad will be
-// var - lr * momentum * accum, so in the end, the var you get is actually
-// var - lr * momentum * accum.
-// If not specified, defaults to false
-func ResourceSparseApplyMomentumUseNesterov(value bool) ResourceSparseApplyMomentumAttr {
+// value: An optional bool. Defaults to True. If True, the assignment will
+// be protected by a lock; otherwise the behavior is undefined,
+// but may exhibit less contention.
+// If not specified, defaults to true
+func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr {
 	return func(m optionalAttr) {
-		m["use_nesterov"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+// Applies sparse `updates` to individual values or slices within a given
 //
-// Set use_nesterov = True if you want to use Nesterov momentum.
+// variable according to `indices`.
 //
-// That is for rows we have grad for, we update var and accum as follows:
+// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
-// accum = accum * momentum + grad
-// var -= lr * accum
+// `indices` must be integer tensor, containing indices into `ref`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+//
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+// dimension of `ref`.
+//
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+//
+// ```
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+// ```
+//
+// For example, say we want to update 4 scattered elements to a rank-1 tensor to
+// 8 elements. In Python, that update would look like this:
+//
+// ```python
+//     ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+//     indices = tf.constant([[4], [3], [1] ,[7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     update = tf.scatter_nd_update(ref, indices, updates)
+//     with tf.Session() as sess:
+//       print sess.run(update)
+// ```
+//
+// The resulting update to ref would look like this:
+//
+//     [1, 11, 3, 10, 9, 6, 7, 12]
+//
+// See `tf.scatter_nd` for more details about how to make updates to
+// slices.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	momentum: Momentum. Must be a scalar.
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of updated
+// values to add to ref.
 //
 // Returns the created operation.
-func ResourceSparseApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyMomentumAttr) (o *tf.Operation) {
+func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14425,423 +14358,359 @@ func ResourceSparseApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyMomentum",
+		Type: "ResourceScatterNdUpdate",
 		Input: []tf.Input{
-			var_, accum, lr, grad, indices, momentum,
+			ref, indices, updates,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Returns the complex conjugate of a complex number.
-//
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// complex numbers that are the complex conjugate of each element in `input`. The
-// complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
-// real part and *b* is the imaginary part.
-//
-// The complex conjugate returned by this operation is of the form \\(a - bj\\).
+// Produces a string handle for the given MultiDeviceIterator.
 //
-// For example:
+// Arguments:
+//	multi_device_iterator: A MultiDeviceIterator resource.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
-// ```
-func Conj(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns A string representing the resource.
+func MultiDeviceIteratorToStringHandle(scope *Scope, multi_device_iterator tf.Output) (string_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Conj",
+		Type: "MultiDeviceIteratorToStringHandle",
 		Input: []tf.Input{
-			input,
+			multi_device_iterator,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
-type AvgPoolGradAttr func(optionalAttr)
-
-// AvgPoolGradDataFormat sets the optional data_format attribute to value.
+// Applies softmax to a batched N-D `SparseTensor`.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Computes gradients of the average pooling function.
+// The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
+// (where `N >= 2`), and with indices sorted in the canonical lexicographic order.
+//
+// This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost
+// logical submatrix with shape `[B, C]`, but with the catch that *the implicitly
+// zero elements do not participate*.  Specifically, the algorithm is equivalent
+// to the following:
+//
+//   (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix
+//       with shape `[B, C]`, along the size-C dimension;
+//   (2) Masks out the original implicitly-zero locations;
+//   (3) Renormalizes the remaining elements.
+//
+// Hence, the `SparseTensor` result has exactly the same non-zero indices and
+// shape.
 //
 // Arguments:
-//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
-// the output of `avg_pool`.
-//	ksize: The size of the sliding window for each dimension of the input.
-//	strides: The stride of the sliding window for each dimension of the input.
-//	padding: The type of padding algorithm to use.
+//	sp_indices: 2-D.  `NNZ x R` matrix with the indices of non-empty values in a
+// SparseTensor, in canonical ordering.
+//	sp_values: 1-D.  `NNZ` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
 //
-// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
-func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
+// Returns 1-D.  The `NNZ` values for the result `SparseTensor`.
+func SparseSoftmax(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AvgPoolGrad",
+		Type: "SparseSoftmax",
 		Input: []tf.Input{
-			orig_input_shape, grad,
+			sp_indices, sp_values, sp_shape,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Greedily selects a subset of bounding boxes in descending order of score,
+// Partitions `data` into `num_partitions` tensors using indices from `partitions`.
 //
-// pruning away boxes that have high overlaps
-// with previously selected boxes.  Bounding boxes with score less than
-// `score_threshold` are removed. N-by-n overlap values are supplied as square matrix,
-// which allows for defining a custom overlap criterium (eg. intersection over union,
-// intersection over area, etc.).
+// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
+// becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
+// are placed in `outputs[i]` in lexicographic order of `js`, and the first
+// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
+// In detail,
 //
-// The output of this operation is a set of integers indexing into the input
-// collection of bounding boxes representing the selected boxes.  The bounding
-// box coordinates corresponding to the selected indices can then be obtained
-// using the `tf.gather operation`.  For example:
+// ```python
+//     outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
 //
-//   selected_indices = tf.image.non_max_suppression_with_overlaps(
-//       overlaps, scores, max_output_size, overlap_threshold, score_threshold)
-//   selected_boxes = tf.gather(boxes, selected_indices)
+//     outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
+// ```
+//
+// `data.shape` must start with `partitions.shape`.
+//
+// For example:
+//
+// ```python
+//     # Scalar partitions.
+//     partitions = 1
+//     num_partitions = 2
+//     data = [10, 20]
+//     outputs[0] = []  # Empty with shape [0, 2]
+//     outputs[1] = [[10, 20]]
+//
+//     # Vector partitions.
+//     partitions = [0, 0, 1, 1, 0]
+//     num_partitions = 2
+//     data = [10, 20, 30, 40, 50]
+//     outputs[0] = [10, 20, 50]
+//     outputs[1] = [30, 40]
+// ```
+//
+// See `dynamic_stitch` for an example on how to merge partitions back.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
+// </div>
 //
 // Arguments:
-//	overlaps: A 2-D float tensor of shape `[num_boxes, num_boxes]` representing
-// the n-by-n box overlap values.
-//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
-// score corresponding to each box (each row of boxes).
-//	max_output_size: A scalar integer tensor representing the maximum number of
-// boxes to be selected by non max suppression.
-//	overlap_threshold: A 0-D float tensor representing the threshold for deciding whether
-// boxes overlap too.
-//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
-// boxes based on score.
 //
-// Returns A 1-D integer tensor of shape `[M]` representing the selected
-// indices from the boxes tensor, where `M <= max_output_size`.
-func NonMaxSuppressionWithOverlaps(scope *Scope, overlaps tf.Output, scores tf.Output, max_output_size tf.Output, overlap_threshold tf.Output, score_threshold tf.Output) (selected_indices tf.Output) {
+//	partitions: Any shape.  Indices in the range `[0, num_partitions)`.
+//	num_partitions: The number of partitions to output.
+func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_partitions": num_partitions}
 	opspec := tf.OpSpec{
-		Type: "NonMaxSuppressionWithOverlaps",
+		Type: "DynamicPartition",
 		Input: []tf.Input{
-			overlaps, scores, max_output_size, overlap_threshold, score_threshold,
+			data, partitions,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StageClearAttr is an optional argument to StageClear.
-type StageClearAttr func(optionalAttr)
-
-// StageClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageClearCapacity(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// StageClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageClearMemoryLimit(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
+	var idx int
+	var err error
+	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
+		scope.UpdateErr("DynamicPartition", err)
+		return
 	}
+	return outputs
 }
 
-// StageClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StageClearContainer(value string) StageClearAttr {
+// ResourceApplyAdagradAttr is an optional argument to ResourceApplyAdagrad.
+type ResourceApplyAdagradAttr func(optionalAttr)
+
+// ResourceApplyAdagradUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdagradUseLocking(value bool) ResourceApplyAdagradAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["use_locking"] = value
 	}
 }
 
-// StageClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StageClearSharedName(value string) StageClearAttr {
+// ResourceApplyAdagradUpdateSlots sets the optional update_slots attribute to value.
+// If not specified, defaults to true
+func ResourceApplyAdagradUpdateSlots(value bool) ResourceApplyAdagradAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["update_slots"] = value
 	}
 }
 
-// Op removes all elements in the underlying container.
+// Update '*var' according to the adagrad scheme.
+//
+// accum += grad * grad
+// var -= lr * grad * (1 / sqrt(accum))
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	grad: The gradient.
 //
 // Returns the created operation.
-func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, optional ...ResourceApplyAdagradAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StageClear",
-
+		Type: "ResourceApplyAdagrad",
+		Input: []tf.Input{
+			var_, accum, lr, grad,
+		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// QuantizedRelu6Attr is an optional argument to QuantizedRelu6.
-type QuantizedRelu6Attr func(optionalAttr)
-
-// QuantizedRelu6OutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QUINT8
-func QuantizedRelu6OutType(value tf.DataType) QuantizedRelu6Attr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Computes Quantized Rectified Linear 6: `min(max(features, 0), 6)`
+// Retrieves the tree ensemble resource stamp token, number of trees and growing statistics.
 //
 // Arguments:
+//	tree_ensemble_handle: Handle to the tree ensemble.
 //
-//	min_features: The float value that the lowest quantized value represents.
-//	max_features: The float value that the highest quantized value represents.
-//
-// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedRelu6(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedRelu6Attr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+// Returns Stamp token of the tree ensemble resource.The number of trees in the tree ensemble resource.The number of trees that were finished successfully.The number of layers we attempted to build (but not necessarily succeeded).Rank size 2 tensor that contains start and end ids of the nodes in the latest
+// layer.
+func BoostedTreesGetEnsembleStates(scope *Scope, tree_ensemble_handle tf.Output) (stamp_token tf.Output, num_trees tf.Output, num_finalized_trees tf.Output, num_attempted_layers tf.Output, last_layer_nodes_range tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedRelu6",
+		Type: "BoostedTreesGetEnsembleStates",
 		Input: []tf.Input{
-			features, min_features, max_features,
+			tree_ensemble_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2.
-type FixedLengthRecordReaderV2Attr func(optionalAttr)
+// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign.
+type ResourceApplyPowerSignAttr func(optionalAttr)
 
-// FixedLengthRecordReaderV2HeaderBytes sets the optional header_bytes attribute to value.
+// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value.
 //
-// value: Number of bytes in the header, defaults to 0.
-// If not specified, defaults to 0
-func FixedLengthRecordReaderV2HeaderBytes(value int64) FixedLengthRecordReaderV2Attr {
+// value: If `True`, updating of the var and m tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr {
 	return func(m optionalAttr) {
-		m["header_bytes"] = value
+		m["use_locking"] = value
 	}
 }
 
-// FixedLengthRecordReaderV2FooterBytes sets the optional footer_bytes attribute to value.
+// Update '*var' according to the AddSign update.
 //
-// value: Number of bytes in the footer, defaults to 0.
-// If not specified, defaults to 0
-func FixedLengthRecordReaderV2FooterBytes(value int64) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["footer_bytes"] = value
-	}
-}
-
-// FixedLengthRecordReaderV2HopBytes sets the optional hop_bytes attribute to value.
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g
+// variable <- variable - lr_t * update
 //
-// value: Number of bytes to hop before each read. Default of 0 means using
-// record_bytes.
-// If not specified, defaults to 0
-func FixedLengthRecordReaderV2HopBytes(value int64) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["hop_bytes"] = value
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	logbase: Must be a scalar.
+//	sign_decay: Must be a scalar.
+//	beta: Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyPowerSign",
+		Input: []tf.Input{
+			var_, m, lr, logbase, sign_decay, beta, grad,
+		},
+		Attrs: attrs,
 	}
+	return scope.AddOperation(opspec)
 }
 
-// FixedLengthRecordReaderV2Container sets the optional container attribute to value.
+// StringFormatAttr is an optional argument to StringFormat.
+type StringFormatAttr func(optionalAttr)
+
+// StringFormatTemplate sets the optional template attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func FixedLengthRecordReaderV2Container(value string) FixedLengthRecordReaderV2Attr {
+// value: A string, the template to format tensor summaries into.
+// If not specified, defaults to "%s"
+func StringFormatTemplate(value string) StringFormatAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["template"] = value
 	}
 }
 
-// FixedLengthRecordReaderV2SharedName sets the optional shared_name attribute to value.
+// StringFormatPlaceholder sets the optional placeholder attribute to value.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func FixedLengthRecordReaderV2SharedName(value string) FixedLengthRecordReaderV2Attr {
+// value: A string, at each placeholder in the template a subsequent tensor summary will be inserted.
+// If not specified, defaults to "%s"
+func StringFormatPlaceholder(value string) StringFormatAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["placeholder"] = value
 	}
 }
 
-// FixedLengthRecordReaderV2Encoding sets the optional encoding attribute to value.
+// StringFormatSummarize sets the optional summarize attribute to value.
 //
-// value: The type of encoding for the file. Currently ZLIB and GZIP
-// are supported. Defaults to none.
-// If not specified, defaults to ""
-func FixedLengthRecordReaderV2Encoding(value string) FixedLengthRecordReaderV2Attr {
+// value: When formatting the tensor summaries print the first and last summarize entries of each tensor dimension.
+// If not specified, defaults to 3
+func StringFormatSummarize(value int64) StringFormatAttr {
 	return func(m optionalAttr) {
-		m["encoding"] = value
+		m["summarize"] = value
 	}
 }
 
-// A Reader that outputs fixed-length records from a file.
+// Formats a string template using a list of tensors.
+//
+// Formats a string template using a list of tensors, pretty-printing tensor summaries.
 //
 // Arguments:
-//	record_bytes: Number of bytes in the record.
+//	inputs: The list of tensors to format into the placeholder string.
 //
-// Returns The handle to reference the Reader.
-func FixedLengthRecordReaderV2(scope *Scope, record_bytes int64, optional ...FixedLengthRecordReaderV2Attr) (reader_handle tf.Output) {
+// Returns = The resulting string scalar.
+func StringFormat(scope *Scope, inputs []tf.Output, optional ...StringFormatAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"record_bytes": record_bytes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FixedLengthRecordReaderV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes numerical negative value element-wise.
-//
-// I.e., \\(y = -x\\).
-func Neg(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Neg",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Execute a sub graph on a remote processor.
-//
-// The graph specifications(such as graph itself, input tensors and output names)
-// are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo
-// as serialized_remote_fused_graph_execute_info.
-// The specifications will be passed to a dedicated registered
-// remote fused graph executor.  The executor will send the graph specifications
-// to a remote processor and execute that graph.  The execution results
-// will be passed to consumer nodes as outputs of this node.
-//
-// Arguments:
-//	inputs: Arbitrary number of tensors with arbitrary data types
-//
-//	serialized_remote_fused_graph_execute_info: Serialized protocol buffer
-// of RemoteFusedGraphExecuteInfo which contains graph specifications.
-//
-// Returns Arbitrary number of tensors with arbitrary data types
-func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.DataType, serialized_remote_fused_graph_execute_info string) (outputs []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"Toutputs": Toutputs, "serialized_remote_fused_graph_execute_info": serialized_remote_fused_graph_execute_info}
-	opspec := tf.OpSpec{
-		Type: "RemoteFusedGraphExecute",
+		Type: "StringFormat",
 		Input: []tf.Input{
 			tf.OutputList(inputs),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
-		scope.UpdateErr("RemoteFusedGraphExecute", err)
-		return
-	}
-	return outputs
+	return op.Output(0)
 }
 
-// MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad.
-type MaxPool3DGradGradAttr func(optionalAttr)
-
-// MaxPool3DGradGradDataFormat sets the optional data_format attribute to value.
-//
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func MaxPool3DGradGradDataFormat(value string) MaxPool3DGradGradAttr {
+// ShapeAttr is an optional argument to Shape.
+type ShapeAttr func(optionalAttr)
+
+// ShapeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func ShapeOutType(value tf.DataType) ShapeAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["out_type"] = value
 	}
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Returns the shape of a tensor.
 //
-// Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+// This operation returns a 1-D integer tensor representing the shape of `input`.
 //
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradGradAttr) (output tf.Output) {
+// For example:
+//
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// shape(t) ==> [2, 2, 3]
+// ```
+func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPool3DGradGrad",
+		Type: "Shape",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -14849,101 +14718,111 @@ func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output
 	return op.Output(0)
 }
 
-// Conv3DBackpropFilterV2Attr is an optional argument to Conv3DBackpropFilterV2.
-type Conv3DBackpropFilterV2Attr func(optionalAttr)
-
-// Conv3DBackpropFilterV2DataFormat sets the optional data_format attribute to value.
+// Computes the power of one value to another.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func Conv3DBackpropFilterV2DataFormat(value string) Conv3DBackpropFilterV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Conv3DBackpropFilterV2Dilations sets the optional dilations attribute to value.
+// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
+// corresponding elements in `x` and `y`. For example:
 //
-// value: 1-D tensor of length 5.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
-func Conv3DBackpropFilterV2Dilations(value []int64) Conv3DBackpropFilterV2Attr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
+// ```
+// # tensor 'x' is [[2, 2]], [3, 3]]
+// # tensor 'y' is [[8, 16], [2, 3]]
+// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
+// ```
+func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Pow",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the gradients of 3-D convolution with respect to the filter.
+// Computes fingerprints of the input strings.
 //
 // Arguments:
-//	input: Shape `[batch, depth, rows, cols, in_channels]`.
-//	filter_sizes: An integer vector representing the tensor shape of `filter`,
-// where `filter` is a 5-D
-// `[filter_depth, filter_height, filter_width, in_channels, out_channels]`
-// tensor.
-//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
-// out_channels]`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterV2Attr) (output tf.Output) {
+//	input: vector of strings to compute fingerprints on.
+//
+// Returns a (N,2) shaped matrix where N is the number of elements in the input
+// vector. Each row contains the low and high parts of the fingerprint.
+func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Conv3DBackpropFilterV2",
+		Type: "SdcaFprint",
 		Input: []tf.Input{
-			input, filter_sizes, out_backprop,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars.
-type FakeQuantWithMinMaxVarsAttr func(optionalAttr)
+// LRNAttr is an optional argument to LRN.
+type LRNAttr func(optionalAttr)
 
-// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr {
+// LRNDepthRadius sets the optional depth_radius attribute to value.
+//
+// value: 0-D.  Half-width of the 1-D normalization window.
+// If not specified, defaults to 5
+func LRNDepthRadius(value int64) LRNAttr {
 	return func(m optionalAttr) {
-		m["num_bits"] = value
+		m["depth_radius"] = value
 	}
 }
 
-// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value.
-// If not specified, defaults to false
-func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr {
+// LRNBias sets the optional bias attribute to value.
+//
+// value: An offset (usually positive to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNBias(value float32) LRNAttr {
 	return func(m optionalAttr) {
-		m["narrow_range"] = value
+		m["bias"] = value
 	}
 }
 
-// Fake-quantize the 'inputs' tensor of type float via global float scalars `min`
+// LRNAlpha sets the optional alpha attribute to value.
 //
-// and `max` to 'outputs' tensor of same shape as `inputs`.
+// value: A scale factor, usually positive.
+// If not specified, defaults to 1
+func LRNAlpha(value float32) LRNAttr {
+	return func(m optionalAttr) {
+		m["alpha"] = value
+	}
+}
+
+// LRNBeta sets the optional beta attribute to value.
 //
-// `[min; max]` define the clamping range for the `inputs` data.
-// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
-// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
-// then de-quantized and output as floats in `[min; max]` interval.
-// `num_bits` is the bitwidth of the quantization; between 2 and 16, inclusive.
+// value: An exponent.
+// If not specified, defaults to 0.5
+func LRNBeta(value float32) LRNAttr {
+	return func(m optionalAttr) {
+		m["beta"] = value
+	}
+}
+
+// Local Response Normalization.
 //
-// This operation has a gradient and thus allows for training `min` and `max`
-// values.
-func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) {
+// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last
+// dimension), and each vector is normalized independently.  Within a given vector,
+// each component is divided by the weighted, squared sum of inputs within
+// `depth_radius`.  In detail,
+//
+//     sqr_sum[a, b, c, d] =
+//         sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2)
+//     output = input / (bias + alpha * sqr_sum) ** beta
+//
+// For details, see [Krizhevsky et al., ImageNet classification with deep
+// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks).
+//
+// Arguments:
+//	input: 4-D.
+func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14952,9 +14831,9 @@ func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FakeQuantWithMinMaxVars",
+		Type: "LRN",
 		Input: []tf.Input{
-			inputs, min, max,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -14962,68 +14841,61 @@ func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max
 	return op.Output(0)
 }
 
-// ResourceScatterNdUpdateAttr is an optional argument to ResourceScatterNdUpdate.
-type ResourceScatterNdUpdateAttr func(optionalAttr)
+// Creates a dataset that zips together `input_datasets`.
+func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ZipDataset",
+		Input: []tf.Input{
+			tf.OutputList(input_datasets),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// ResourceScatterNdUpdateUseLocking sets the optional use_locking attribute to value.
+// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
+type ResourceSparseApplyAdagradAttr func(optionalAttr)
+
+// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value.
 //
-// value: An optional bool. Defaults to True. If True, the assignment will
-// be protected by a lock; otherwise the behavior is undefined,
-// but may exhibit less contention.
-// If not specified, defaults to true
-func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Applies sparse `updates` to individual values or slices within a given
-//
-// variable according to `indices`.
-//
-// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
-//
-// `indices` must be integer tensor, containing indices into `ref`.
-// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
-//
-// The innermost dimension of `indices` (with length `K`) corresponds to
-// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
-// dimension of `ref`.
-//
-// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
-//
-// ```
-// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
-// ```
-//
-// For example, say we want to update 4 scattered elements to a rank-1 tensor to
-// 8 elements. In Python, that update would look like this:
-//
-// ```python
-//     ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
-//     indices = tf.constant([[4], [3], [1] ,[7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     update = tf.scatter_nd_update(ref, indices, updates)
-//     with tf.Session() as sess:
-//       print sess.run(update)
-// ```
-//
-// The resulting update to ref would look like this:
-//
-//     [1, 11, 3, 10, 9, 6, 7, 12]
+// ResourceSparseApplyAdagradUpdateSlots sets the optional update_slots attribute to value.
+// If not specified, defaults to true
+func ResourceSparseApplyAdagradUpdateSlots(value bool) ResourceSparseApplyAdagradAttr {
+	return func(m optionalAttr) {
+		m["update_slots"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the adagrad scheme.
 //
-// See `tf.scatter_nd` for more details about how to make updates to
-// slices.
+// That is for rows we have grad for, we update var and accum as follows:
+// accum += grad * grad
+// var -= lr * grad * (1 / sqrt(accum))
 //
 // Arguments:
-//	ref: A resource handle. Must be from a VarHandleOp.
-//	indices: A Tensor. Must be one of the following types: int32, int64.
-// A tensor of indices into ref.
-//	updates: A Tensor. Must have the same type as ref. A tensor of updated
-// values to add to ref.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
 //
 // Returns the created operation.
-func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) {
+func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15032,249 +14904,308 @@ func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, upd
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterNdUpdate",
+		Type: "ResourceSparseApplyAdagrad",
 		Input: []tf.Input{
-			ref, indices, updates,
+			var_, accum, lr, grad, indices,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Produces a string handle for the given MultiDeviceIterator.
+// Elementwise computes the bitwise right-shift of `x` and `y`.
 //
-// Arguments:
-//	multi_device_iterator: A MultiDeviceIterator resource.
+// Performs a logical shift for unsigned integer types, and an arithmetic shift
+// for signed integer types.
 //
-// Returns A string representing the resource.
-func MultiDeviceIteratorToStringHandle(scope *Scope, multi_device_iterator tf.Output) (string_handle tf.Output) {
+// If `y` is negative, or greater than or equal to than the width of `x` in bits
+// the result is implementation defined.
+func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "MultiDeviceIteratorToStringHandle",
+		Type: "RightShift",
 		Input: []tf.Input{
-			multi_device_iterator,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Applies softmax to a batched N-D `SparseTensor`.
-//
-// The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
-// (where `N >= 2`), and with indices sorted in the canonical lexicographic order.
-//
-// This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost
-// logical submatrix with shape `[B, C]`, but with the catch that *the implicitly
-// zero elements do not participate*.  Specifically, the algorithm is equivalent
-// to the following:
-//
-//   (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix
-//       with shape `[B, C]`, along the size-C dimension;
-//   (2) Masks out the original implicitly-zero locations;
-//   (3) Renormalizes the remaining elements.
+// TensorListStackAttr is an optional argument to TensorListStack.
+type TensorListStackAttr func(optionalAttr)
+
+// TensorListStackNumElements sets the optional num_elements attribute to value.
+// If not specified, defaults to -1
+func TensorListStackNumElements(value int64) TensorListStackAttr {
+	return func(m optionalAttr) {
+		m["num_elements"] = value
+	}
+}
+
+// Stacks all tensors in the list.
 //
-// Hence, the `SparseTensor` result has exactly the same non-zero indices and
-// shape.
+// Requires that all tensors have the same shape.
 //
-// Arguments:
-//	sp_indices: 2-D.  `NNZ x R` matrix with the indices of non-empty values in a
-// SparseTensor, in canonical ordering.
-//	sp_values: 1-D.  `NNZ` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
+// input_handle: the input list
+// tensor: the gathered result
+// num_elements: optional. If not -1, the number of elements in the list.
 //
-// Returns 1-D.  The `NNZ` values for the result `SparseTensor`.
-func SparseSoftmax(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output) (output tf.Output) {
+func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseSoftmax",
+		Type: "TensorListStack",
 		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape,
+			input_handle,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Partitions `data` into `num_partitions` tensors using indices from `partitions`.
-//
-// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
-// becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
-// are placed in `outputs[i]` in lexicographic order of `js`, and the first
-// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
-// In detail,
-//
-// ```python
-//     outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
-//
-//     outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
-// ```
-//
-// `data.shape` must start with `partitions.shape`.
-//
-// For example:
-//
-// ```python
-//     # Scalar partitions.
-//     partitions = 1
-//     num_partitions = 2
-//     data = [10, 20]
-//     outputs[0] = []  # Empty with shape [0, 2]
-//     outputs[1] = [[10, 20]]
+// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
+type StatelessRandomUniformAttr func(optionalAttr)
+
+// StatelessRandomUniformDtype sets the optional dtype attribute to value.
 //
-//     # Vector partitions.
-//     partitions = [0, 0, 1, 1, 0]
-//     num_partitions = 2
-//     data = [10, 20, 30, 40, 50]
-//     outputs[0] = [10, 20, 50]
-//     outputs[1] = [30, 40]
-// ```
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs deterministic pseudorandom random values from a uniform distribution.
 //
-// See `dynamic_stitch` for an example on how to merge partitions back.
+// The generated values follow a uniform distribution in the range `[0, 1)`. The
+// lower bound 0 is included in the range, while the upper bound 1 is excluded.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
-// </div>
+// The outputs are a deterministic function of `shape` and `seed`.
 //
 // Arguments:
+//	shape: The shape of the output tensor.
+//	seed: 2 seeds (shape [2]).
 //
-//	partitions: Any shape.  Indices in the range `[0, num_partitions)`.
-//	num_partitions: The number of partitions to output.
-func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) {
+// Returns Random values with specified shape.
+func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomUniformAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_partitions": num_partitions}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "DynamicPartition",
+		Type: "StatelessRandomUniform",
 		Input: []tf.Input{
-			data, partitions,
+			shape, seed,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Makes its input available to the next iteration.
+//
+// Arguments:
+//	data: The tensor to be made available to the next iteration.
+//
+// Returns The same tensor as `data`.
+func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
-		scope.UpdateErr("DynamicPartition", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "NextIteration",
+		Input: []tf.Input{
+			data,
+		},
 	}
-	return outputs
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ResourceApplyAdagradAttr is an optional argument to ResourceApplyAdagrad.
-type ResourceApplyAdagradAttr func(optionalAttr)
-
-// ResourceApplyAdagradUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAdagradUseLocking(value bool) ResourceApplyAdagradAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
+// Output a fact about factorials.
+func Fact(scope *Scope) (fact tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// ResourceApplyAdagradUpdateSlots sets the optional update_slots attribute to value.
-// If not specified, defaults to true
-func ResourceApplyAdagradUpdateSlots(value bool) ResourceApplyAdagradAttr {
-	return func(m optionalAttr) {
-		m["update_slots"] = value
+	opspec := tf.OpSpec{
+		Type: "Fact",
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Update '*var' according to the adagrad scheme.
+// Deserialize `SparseTensor` objects.
 //
-// accum += grad * grad
-// var -= lr * grad * (1 / sqrt(accum))
+// The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where
+// the last dimension stores serialized `SparseTensor` objects and the other N
+// dimensions (N >= 0) correspond to a batch. The ranks of the original
+// `SparseTensor` objects must all match. When the final `SparseTensor` is
+// created, its rank is the rank of the incoming `SparseTensor` objects plus N;
+// the sparse tensors have been concatenated along new dimensions, one for each
+// batch.
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	grad: The gradient.
+// The output `SparseTensor` object's shape values for the original dimensions
+// are the max across the input `SparseTensor` objects' shape values for the
+// corresponding dimensions. The new dimensions match the size of the batch.
 //
-// Returns the created operation.
-func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, optional ...ResourceApplyAdagradAttr) (o *tf.Operation) {
+// The input `SparseTensor` objects' indices are assumed ordered in
+// standard lexicographic order.  If this is not the case, after this
+// step run `SparseReorder` to restore index ordering.
+//
+// For example, if the serialized input is a `[2 x 3]` matrix representing two
+// original `SparseTensor` objects:
+//
+//     index = [ 0]
+//             [10]
+//             [20]
+//     values = [1, 2, 3]
+//     shape = [50]
+//
+// and
+//
+//     index = [ 2]
+//             [10]
+//     values = [4, 5]
+//     shape = [30]
+//
+// then the final deserialized `SparseTensor` will be:
+//
+//     index = [0  0]
+//             [0 10]
+//             [0 20]
+//             [1  2]
+//             [1 10]
+//     values = [1, 2, 3, 4, 5]
+//     shape = [2 50]
+//
+// Arguments:
+//	serialized_sparse: The serialized `SparseTensor` objects. The last dimension
+// must have 3 columns.
+//	dtype: The `dtype` of the serialized `SparseTensor` objects.
+func DeserializeSparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdagrad",
+		Type: "DeserializeSparse",
 		Input: []tf.Input{
-			var_, accum, lr, grad,
+			serialized_sparse,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Retrieves the tree ensemble resource stamp token, number of trees and growing statistics.
+// SqueezeAttr is an optional argument to Squeeze.
+type SqueezeAttr func(optionalAttr)
+
+// SqueezeAxis sets the optional axis attribute to value.
+//
+// value: If specified, only squeezes the dimensions listed. The dimension
+// index starts at 0. It is an error to squeeze a dimension that is not 1. Must
+// be in the range `[-rank(input), rank(input))`.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func SqueezeAxis(value []int64) SqueezeAttr {
+	return func(m optionalAttr) {
+		m["squeeze_dims"] = value
+	}
+}
+
+// Removes dimensions of size 1 from the shape of a tensor.
+//
+// Given a tensor `input`, this operation returns a tensor of the same type with
+// all dimensions of size 1 removed. If you don't want to remove all size 1
+// dimensions, you can remove specific size 1 dimensions by specifying
+// `axis`.
+//
+// For example:
+//
+// ```
+// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+// shape(squeeze(t)) ==> [2, 3]
+// ```
+//
+// Or, to remove specific size 1 dimensions:
+//
+// ```
+// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
+// ```
 //
 // Arguments:
-//	tree_ensemble_handle: Handle to the tree ensemble.
+//	input: The `input` to squeeze.
 //
-// Returns Stamp token of the tree ensemble resource.The number of trees in the tree ensemble resource.The number of trees that were finished successfully.The number of layers we attempted to build (but not necessarily succeeded).Rank size 2 tensor that contains start and end ids of the nodes in the latest
-// layer.
-func BoostedTreesGetEnsembleStates(scope *Scope, tree_ensemble_handle tf.Output) (stamp_token tf.Output, num_trees tf.Output, num_finalized_trees tf.Output, num_attempted_layers tf.Output, last_layer_nodes_range tf.Output) {
+// Returns Contains the same data as `input`, but has one or more dimensions of
+// size 1 removed.
+func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "BoostedTreesGetEnsembleStates",
+		Type: "Squeeze",
 		Input: []tf.Input{
-			tree_ensemble_handle,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0)
 }
 
-// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign.
-type ResourceApplyPowerSignAttr func(optionalAttr)
+// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta.
+type ResourceApplyAdadeltaAttr func(optionalAttr)
 
-// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value.
+// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value.
 //
-// value: If `True`, updating of the var and m tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
+// value: If True, updating of the var, accum and update_accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
 // If not specified, defaults to false
-func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr {
+func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update '*var' according to the AddSign update.
+// Update '*var' according to the adadelta scheme.
 //
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
-// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g
-// variable <- variable - lr_t * update
+// accum = rho() * accum + (1 - rho()) * grad.square();
+// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
+// update_accum = rho() * update_accum + (1 - rho()) * update.square();
+// var -= update;
 //
 // Arguments:
 //	var_: Should be from a Variable().
-//	m: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	accum_update: Should be from a Variable().
 //	lr: Scaling factor. Must be a scalar.
-//	logbase: Must be a scalar.
-//	sign_decay: Must be a scalar.
-//	beta: Must be a scalar.
+//	rho: Decay factor. Must be a scalar.
+//	epsilon: Constant factor. Must be a scalar.
 //	grad: The gradient.
 //
 // Returns the created operation.
-func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) {
+func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15283,57 +15214,58 @@ func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Out
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyPowerSign",
+		Type: "ResourceApplyAdadelta",
 		Input: []tf.Input{
-			var_, m, lr, logbase, sign_decay, beta, grad,
+			var_, accum, accum_update, lr, rho, epsilon, grad,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// StringFormatAttr is an optional argument to StringFormat.
-type StringFormatAttr func(optionalAttr)
-
-// StringFormatTemplate sets the optional template attribute to value.
-//
-// value: A string, the template to format tensor summaries into.
-// If not specified, defaults to "%s"
-func StringFormatTemplate(value string) StringFormatAttr {
-	return func(m optionalAttr) {
-		m["template"] = value
-	}
-}
-
-// StringFormatPlaceholder sets the optional placeholder attribute to value.
-//
-// value: A string, at each placeholder in the template a subsequent tensor summary will be inserted.
-// If not specified, defaults to "%s"
-func StringFormatPlaceholder(value string) StringFormatAttr {
-	return func(m optionalAttr) {
-		m["placeholder"] = value
-	}
-}
+// NonMaxSuppressionAttr is an optional argument to NonMaxSuppression.
+type NonMaxSuppressionAttr func(optionalAttr)
 
-// StringFormatSummarize sets the optional summarize attribute to value.
+// NonMaxSuppressionIouThreshold sets the optional iou_threshold attribute to value.
 //
-// value: When formatting the tensor summaries print the first and last summarize entries of each tensor dimension.
-// If not specified, defaults to 3
-func StringFormatSummarize(value int64) StringFormatAttr {
+// value: A float representing the threshold for deciding whether boxes
+// overlap too much with respect to IOU.
+// If not specified, defaults to 0.5
+func NonMaxSuppressionIouThreshold(value float32) NonMaxSuppressionAttr {
 	return func(m optionalAttr) {
-		m["summarize"] = value
+		m["iou_threshold"] = value
 	}
 }
 
-// Formats a string template using a list of tensors.
+// Greedily selects a subset of bounding boxes in descending order of score,
 //
-// Formats a string template using a list of tensors, pretty-printing tensor summaries.
+// pruning away boxes that have high intersection-over-union (IOU) overlap
+// with previously selected boxes.  Bounding boxes are supplied as
+// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+// diagonal pair of box corners and the coordinates can be provided as normalized
+// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+// is agnostic to where the origin is in the coordinate system.  Note that this
+// algorithm is invariant to orthogonal transformations and translations
+// of the coordinate system; thus translating or reflections of the coordinate
+// system result in the same boxes being selected by the algorithm.
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
+//   selected_indices = tf.image.non_max_suppression(
+//       boxes, scores, max_output_size, iou_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
 //
 // Arguments:
-//	inputs: The list of tensors to format into the placeholder string.
+//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
 //
-// Returns = The resulting string scalar.
-func StringFormat(scope *Scope, inputs []tf.Output, optional ...StringFormatAttr) (output tf.Output) {
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.
+func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, optional ...NonMaxSuppressionAttr) (selected_indices tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15342,9 +15274,9 @@ func StringFormat(scope *Scope, inputs []tf.Output, optional ...StringFormatAttr
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StringFormat",
+		Type: "NonMaxSuppression",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			boxes, scores, max_output_size,
 		},
 		Attrs: attrs,
 	}
@@ -15352,171 +15284,191 @@ func StringFormat(scope *Scope, inputs []tf.Output, optional ...StringFormatAttr
 	return op.Output(0)
 }
 
-// ShapeAttr is an optional argument to Shape.
-type ShapeAttr func(optionalAttr)
-
-// ShapeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func ShapeOutType(value tf.DataType) ShapeAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
+// Creates a dataset that emits `components` as a tuple of tensors once.
+func TensorDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "TensorDataset",
+		Input: []tf.Input{
+			tf.OutputList(components),
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Returns the shape of a tensor.
+// Component-wise multiplies a SparseTensor by a dense Tensor.
 //
-// This operation returns a 1-D integer tensor representing the shape of `input`.
+// The output locations corresponding to the implicitly zero elements in the sparse
+// tensor will be zero (i.e., will not take up storage space), regardless of the
+// contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN).
 //
-// For example:
+// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
+// the other direction.
 //
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// shape(t) ==> [2, 2, 3]
-// ```
-func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
+// Arguments:
+//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
+//	dense: `R`-D.  The dense Tensor operand.
+//
+// Returns 1-D.  The `N` values that are operated on.
+func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Shape",
+		Type: "SparseDenseCwiseMul",
 		Input: []tf.Input{
-			input,
+			sp_indices, sp_values, sp_shape, dense,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the power of one value to another.
+// 2D real-valued fast Fourier transform.
 //
-// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
-// corresponding elements in `x` and `y`. For example:
+// Computes the 2-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most 2 dimensions of `input`.
 //
-// ```
-// # tensor 'x' is [[2, 2]], [3, 3]]
-// # tensor 'y' is [[8, 16], [2, 3]]
-// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
-// ```
-func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
+// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+// of `output`: the zero-frequency term, followed by the `fft_length / 2`
+// positive-frequency terms.
+//
+// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
+//
+// Returns A complex64 tensor of the same rank as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their 2D Fourier transform. The
+//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
+//   components.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft2
+// @end_compatibility
+func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Pow",
+		Type: "RFFT2D",
 		Input: []tf.Input{
-			x, y,
+			input, fft_length,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes fingerprints of the input strings.
+// Pads a tensor with zeros.
 //
-// Arguments:
-//	input: vector of strings to compute fingerprints on.
+// This operation pads a `input` with zeros according to the `paddings` you
+// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
+// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+// how many zeros to add before the contents of `input` in that dimension, and
+// `paddings[D, 1]` indicates how many zeros to add after the contents of `input`
+// in that dimension.
 //
-// Returns a (N,2) shaped matrix where N is the number of elements in the input
-// vector. Each row contains the low and high parts of the fingerprint.
-func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
+// The padded size of each dimension D of the output is:
+//
+// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+//
+// For example:
+//
+// ```
+// # 't' is [[1, 1], [2, 2]]
+// # 'paddings' is [[1, 1], [2, 2]]
+// # rank of 't' is 2
+// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
+//                       [0, 0, 1, 1, 0, 0]
+//                       [0, 0, 2, 2, 0, 0]
+//                       [0, 0, 0, 0, 0, 0]]
+// ```
+//
+func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SdcaFprint",
+		Type: "Pad",
 		Input: []tf.Input{
-			input,
+			input, paddings,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// The gradient operator for the SparseAdd op.
-//
-// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
-// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
-// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
-// values of A and B.
+// Checks whether a resource handle-based variable has been initialized.
 //
 // Arguments:
-//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
-// the non-empty values of the sum.
-//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
-//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
-//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
-// `[nnz(sum), ndims]`.
+//	resource: the input resource handle.
 //
-// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
-// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
-// non-empty values of B.
-func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
+// Returns a scalar boolean which is true if the variable has been
+// initialized.
+func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseAddGrad",
+		Type: "VarIsInitializedOp",
 		Input: []tf.Input{
-			backprop_val_grad, a_indices, b_indices, sum_indices,
+			resource,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// ResourceSparseApplyCenteredRMSPropAttr is an optional argument to ResourceSparseApplyCenteredRMSProp.
-type ResourceSparseApplyCenteredRMSPropAttr func(optionalAttr)
+// ResourceSparseApplyFtrlAttr is an optional argument to ResourceSparseApplyFtrl.
+type ResourceSparseApplyFtrlAttr func(optionalAttr)
 
-// ResourceSparseApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
+// ResourceSparseApplyFtrlUseLocking sets the optional use_locking attribute to value.
 //
-// value: If `True`, updating of the var, mg, ms, and mom tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
 // contention.
 // If not specified, defaults to false
-func ResourceSparseApplyCenteredRMSPropUseLocking(value bool) ResourceSparseApplyCenteredRMSPropAttr {
+func ResourceSparseApplyFtrlUseLocking(value bool) ResourceSparseApplyFtrlAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update '*var' according to the centered RMSProp algorithm.
-//
-// The centered RMSProp algorithm uses an estimate of the centered second moment
-// (i.e., the variance) for normalization, as opposed to regular RMSProp, which
-// uses the (uncentered) second moment. This often helps with training, but is
-// slightly more expensive in terms of computation and memory.
-//
-// Note that in dense implementation of this algorithm, mg, ms, and mom will
-// update even if the grad is zero, but in this sparse implementation, mg, ms,
-// and mom will not update in iterations during which the grad is zero.
-//
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// mean_grad = decay * mean_grad + (1-decay) * gradient
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
 //
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
+// That is for rows we have grad for, we update var, accum and linear as follows:
+// accum_new = accum + grad * grad
+// linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
 // Arguments:
 //	var_: Should be from a Variable().
-//	mg: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
-//
-//	epsilon: Ridge term. Must be a scalar.
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
 //	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var, ms and mom.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	lr_power: Scaling factor. Must be a scalar.
 //
 // Returns the created operation.
-func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyCenteredRMSPropAttr) (o *tf.Operation) {
+func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15525,105 +15477,83 @@ func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyCenteredRMSProp",
+		Type: "ResourceSparseApplyFtrl",
 		Input: []tf.Input{
-			var_, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices,
+			var_, accum, linear, grad, indices, lr, l1, l2, lr_power,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Creates a dataset that batches `batch_size` elements from `input_dataset`.
-//
-// Arguments:
-//
-//	batch_size: A scalar representing the number of elements to accumulate in a
-// batch.
-//
+// Returns which elements of x are Inf.
 //
-func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.isinf
+// @end_compatibility
+func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "BatchDataset",
+		Type: "IsInf",
 		Input: []tf.Input{
-			input_dataset, batch_size,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
-type RandomPoissonV2Attr func(optionalAttr)
+// TruncatedNormalAttr is an optional argument to TruncatedNormal.
+type TruncatedNormalAttr func(optionalAttr)
 
-// RandomPoissonV2Seed sets the optional seed attribute to value.
+// TruncatedNormalSeed sets the optional seed attribute to value.
 //
 // value: If either `seed` or `seed2` are set to be non-zero, the random number
 // generator is seeded by the given seed.  Otherwise, it is seeded by a
 // random seed.
 // If not specified, defaults to 0
-func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
+func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
+// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
 //
 // value: A second seed to avoid seed collision.
 // If not specified, defaults to 0
-func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
+func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// RandomPoissonV2Dtype sets the optional dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["dtype"] = value
-	}
-}
-
-// Outputs random values from the Poisson distribution(s) described by rate.
-//
-// This op uses two algorithms, depending on rate. If rate >= 10, then
-// the algorithm by Hormann is used to acquire samples via
-// transformation-rejection.
-// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+// Outputs random values from a truncated normal distribution.
 //
-// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
-// random variables.
-// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
-// Programming, Volume 2. Addison Wesley
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
 //
 // Arguments:
-//	shape: 1-D integer tensor. Shape of independent samples to draw from each
-// distribution described by the shape parameters given in rate.
-//	rate: A tensor in which each scalar is a "rate" parameter describing the
-// associated poisson distribution.
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns A tensor with shape `shape + shape(rate)`. Each slice
-// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
-// `rate[i0, i1, ...iN]`.
-func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
+// Returns A tensor of the specified shape filled with random truncated normal
+// values.
+func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomPoissonV2",
+		Type: "TruncatedNormal",
 		Input: []tf.Input{
-			shape, rate,
+			shape,
 		},
 		Attrs: attrs,
 	}
@@ -15631,104 +15561,87 @@ func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...
 	return op.Output(0)
 }
 
-// DecodeAndCropJpegAttr is an optional argument to DecodeAndCropJpeg.
-type DecodeAndCropJpegAttr func(optionalAttr)
+// SkipgramAttr is an optional argument to Skipgram.
+type SkipgramAttr func(optionalAttr)
 
-// DecodeAndCropJpegChannels sets the optional channels attribute to value.
+// SkipgramWindowSize sets the optional window_size attribute to value.
 //
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodeAndCropJpegChannels(value int64) DecodeAndCropJpegAttr {
+// value: The number of words to predict to the left and right of the target.
+// If not specified, defaults to 5
+func SkipgramWindowSize(value int64) SkipgramAttr {
 	return func(m optionalAttr) {
-		m["channels"] = value
+		m["window_size"] = value
 	}
 }
 
-// DecodeAndCropJpegRatio sets the optional ratio attribute to value.
+// SkipgramMinCount sets the optional min_count attribute to value.
 //
-// value: Downscaling ratio.
-// If not specified, defaults to 1
-func DecodeAndCropJpegRatio(value int64) DecodeAndCropJpegAttr {
+// value: The minimum number of word occurrences for it to be included in the
+// vocabulary.
+// If not specified, defaults to 5
+func SkipgramMinCount(value int64) SkipgramAttr {
 	return func(m optionalAttr) {
-		m["ratio"] = value
+		m["min_count"] = value
 	}
 }
 
-// DecodeAndCropJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
+// SkipgramSubsample sets the optional subsample attribute to value.
 //
-// value: If true use a slower but nicer upscaling of the
-// chroma planes (yuv420/422 only).
-// If not specified, defaults to true
-func DecodeAndCropJpegFancyUpscaling(value bool) DecodeAndCropJpegAttr {
+// value: Threshold for word occurrence. Words that appear with higher
+// frequency will be randomly down-sampled. Set to 0 to disable.
+// If not specified, defaults to 0.001
+func SkipgramSubsample(value float32) SkipgramAttr {
 	return func(m optionalAttr) {
-		m["fancy_upscaling"] = value
+		m["subsample"] = value
+	}
+}
+
+// Parses a text file and creates a batch of examples.
+//
+// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
+//
+// Arguments:
+//	filename: The corpus's text file name.
+//	batch_size: The size of produced batch.
+//
+// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
+func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
+	for _, a := range optional {
+		a(attrs)
 	}
-}
+	opspec := tf.OpSpec{
+		Type: "Skipgram",
 
-// DecodeAndCropJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
-//
-// value: If true try to recover an image from truncated input.
-// If not specified, defaults to false
-func DecodeAndCropJpegTryRecoverTruncated(value bool) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["try_recover_truncated"] = value
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
 }
 
-// DecodeAndCropJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
-//
-// value: The minimum required fraction of lines before a truncated
-// input is accepted.
-// If not specified, defaults to 1
-func DecodeAndCropJpegAcceptableFraction(value float32) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["acceptable_fraction"] = value
-	}
-}
+// StringToNumberAttr is an optional argument to StringToNumber.
+type StringToNumberAttr func(optionalAttr)
 
-// DecodeAndCropJpegDctMethod sets the optional dct_method attribute to value.
+// StringToNumberOutType sets the optional out_type attribute to value.
 //
-// value: string specifying a hint about the algorithm used for
-// decompression.  Defaults to "" which maps to a system-specific
-// default.  Currently valid values are ["INTEGER_FAST",
-// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
-// jpeg library changes to a version that does not have that specific
-// option.)
-// If not specified, defaults to ""
-func DecodeAndCropJpegDctMethod(value string) DecodeAndCropJpegAttr {
+// value: The numeric type to interpret each string in `string_tensor` as.
+// If not specified, defaults to DT_FLOAT
+func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
 	return func(m optionalAttr) {
-		m["dct_method"] = value
+		m["out_type"] = value
 	}
 }
 
-// Decode and Crop a JPEG-encoded image to a uint8 tensor.
-//
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the JPEG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-//
-// If needed, the JPEG-encoded image is transformed to match the requested number
-// of color channels.
-//
-// The attr `ratio` allows downscaling the image by an integer factor during
-// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
-// downscaling the image later.
-//
-//
-// It is equivalent to a combination of decode and crop, but much faster by only
-// decoding partial jpeg image.
+// Converts each string in the input Tensor to the specified numeric type.
 //
-// Arguments:
-//	contents: 0-D.  The JPEG-encoded image.
-//	crop_window: 1-D.  The crop window: [crop_y, crop_x, crop_height, crop_width].
+// (Note that int32 overflow results in an error while float overflow
+// results in a rounded value.)
 //
-// Returns 3-D with shape `[height, width, channels]`..
-func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output, optional ...DecodeAndCropJpegAttr) (image tf.Output) {
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15737,9 +15650,9 @@ func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeAndCropJpeg",
+		Type: "StringToNumber",
 		Input: []tf.Input{
-			contents, crop_window,
+			string_tensor,
 		},
 		Attrs: attrs,
 	}
@@ -15747,178 +15660,179 @@ func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output,
 	return op.Output(0)
 }
 
-// Adds two `SparseTensor` objects to produce another `SparseTensor`.
-//
-// The input `SparseTensor` objects' indices are assumed ordered in standard
-// lexicographic order.  If this is not the case, before this step run
-// `SparseReorder` to restore index ordering.
+// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
+type ResourceApplyFtrlV2Attr func(optionalAttr)
+
+// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
 //
-// By default, if two values sum to zero at some index, the output `SparseTensor`
-// would still include that particular location in its index, storing a zero in the
-// corresponding value slot.  To override this, callers can specify `thresh`,
-// indicating that if the sum has a magnitude strictly smaller than `thresh`, its
-// corresponding value and index would then not be included.  In particular,
-// `thresh == 0` (default) means everything is kept and actual thresholding happens
-// only for a positive value.
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the Ftrl-proximal scheme.
 //
-// In the following shapes, `nnz` is the count after taking `thresh` into account.
+// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+// linear += grad_with_shrinkage +
+//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
 // Arguments:
-//	a_indices: 2-D.  The `indices` of the first `SparseTensor`, size `[nnz, ndims]` Matrix.
-//	a_values: 1-D.  The `values` of the first `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the first `SparseTensor`, size `[ndims]` Vector.
-//	b_indices: 2-D.  The `indices` of the second `SparseTensor`, size `[nnz, ndims]` Matrix.
-//	b_values: 1-D.  The `values` of the second `SparseTensor`, size `[nnz]` Vector.
-//	b_shape: 1-D.  The `shape` of the second `SparseTensor`, size `[ndims]` Vector.
-//	thresh: 0-D.  The magnitude threshold that determines if an output value/index
-// pair takes space.
-func SparseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output, thresh tf.Output) (sum_indices tf.Output, sum_values tf.Output, sum_shape tf.Output) {
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 shrinkage regulariation. Must be a scalar.
+//
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseAdd",
+		Type: "ResourceApplyFtrlV2",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh,
+			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return scope.AddOperation(opspec)
 }
 
-// LRNAttr is an optional argument to LRN.
-type LRNAttr func(optionalAttr)
+// EncodeJpegAttr is an optional argument to EncodeJpeg.
+type EncodeJpegAttr func(optionalAttr)
 
-// LRNDepthRadius sets the optional depth_radius attribute to value.
+// EncodeJpegFormat sets the optional format attribute to value.
 //
-// value: 0-D.  Half-width of the 1-D normalization window.
-// If not specified, defaults to 5
-func LRNDepthRadius(value int64) LRNAttr {
+// value: Per pixel image format.
+// If not specified, defaults to ""
+func EncodeJpegFormat(value string) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["depth_radius"] = value
+		m["format"] = value
 	}
 }
 
-// LRNBias sets the optional bias attribute to value.
+// EncodeJpegQuality sets the optional quality attribute to value.
 //
-// value: An offset (usually positive to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNBias(value float32) LRNAttr {
+// value: Quality of the compression from 0 to 100 (higher is better and slower).
+// If not specified, defaults to 95
+func EncodeJpegQuality(value int64) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["bias"] = value
+		m["quality"] = value
 	}
 }
 
-// LRNAlpha sets the optional alpha attribute to value.
+// EncodeJpegProgressive sets the optional progressive attribute to value.
 //
-// value: A scale factor, usually positive.
-// If not specified, defaults to 1
-func LRNAlpha(value float32) LRNAttr {
+// value: If True, create a JPEG that loads progressively (coarse to fine).
+// If not specified, defaults to false
+func EncodeJpegProgressive(value bool) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["alpha"] = value
+		m["progressive"] = value
 	}
 }
 
-// LRNBeta sets the optional beta attribute to value.
+// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
 //
-// value: An exponent.
-// If not specified, defaults to 0.5
-func LRNBeta(value float32) LRNAttr {
+// value: If True, spend CPU/RAM to reduce size with no quality change.
+// If not specified, defaults to false
+func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["beta"] = value
+		m["optimize_size"] = value
 	}
 }
 
-// Local Response Normalization.
-//
-// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last
-// dimension), and each vector is normalized independently.  Within a given vector,
-// each component is divided by the weighted, squared sum of inputs within
-// `depth_radius`.  In detail,
-//
-//     sqr_sum[a, b, c, d] =
-//         sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2)
-//     output = input / (bias + alpha * sqr_sum) ** beta
-//
-// For details, see [Krizhevsky et al., ImageNet classification with deep
-// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks).
+// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
 //
-// Arguments:
-//	input: 4-D.
-func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "LRN",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
+// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
+// If not specified, defaults to true
+func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["chroma_downsampling"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Creates a dataset that zips together `input_datasets`.
-func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "ZipDataset",
-		Input: []tf.Input{
-			tf.OutputList(input_datasets),
-		},
-		Attrs: attrs,
+// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
+//
+// value: Unit used to specify `x_density` and `y_density`:
+// pixels per inch (`'in'`) or centimeter (`'cm'`).
+// If not specified, defaults to "in"
+func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["density_unit"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
-type ResourceSparseApplyAdagradAttr func(optionalAttr)
+// EncodeJpegXDensity sets the optional x_density attribute to value.
+//
+// value: Horizontal pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegXDensity(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["x_density"] = value
+	}
+}
 
-// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value.
+// EncodeJpegYDensity sets the optional y_density attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr {
+// value: Vertical pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegYDensity(value int64) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["y_density"] = value
 	}
 }
 
-// ResourceSparseApplyAdagradUpdateSlots sets the optional update_slots attribute to value.
-// If not specified, defaults to true
-func ResourceSparseApplyAdagradUpdateSlots(value bool) ResourceSparseApplyAdagradAttr {
+// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
+//
+// value: If not empty, embed this XMP metadata in the image header.
+// If not specified, defaults to ""
+func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["update_slots"] = value
+		m["xmp_metadata"] = value
 	}
 }
 
-// Update relevant entries in '*var' and '*accum' according to the adagrad scheme.
+// JPEG-encode an image.
 //
-// That is for rows we have grad for, we update var and accum as follows:
-// accum += grad * grad
-// var -= lr * grad * (1 / sqrt(accum))
+// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
+//
+// The attr `format` can be used to override the color format of the encoded
+// output.  Values can be:
+//
+// *   `''`: Use a default format based on the number of channels in the image.
+// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
+//     of `image` must be 1.
+// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
+//     of `image` must be 3.
+//
+// If `format` is not specified or is the empty string, a default format is picked
+// in function of the number of channels in `image`:
+//
+// *   1: Output a grayscale image.
+// *   3: Output an RGB image.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
+//	image: 3-D with shape `[height, width, channels]`.
 //
-// Returns the created operation.
-func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) {
+// Returns 0-D. JPEG-encoded image.
+func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15927,67 +15841,69 @@ func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, l
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyAdagrad",
+		Type: "EncodeJpeg",
 		Input: []tf.Input{
-			var_, accum, lr, grad, indices,
+			image,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Elementwise computes the bitwise right-shift of `x` and `y`.
-//
-// Performs a logical shift for unsigned integer types, and an arithmetic shift
-// for signed integer types.
+// MultinomialAttr is an optional argument to Multinomial.
+type MultinomialAttr func(optionalAttr)
+
+// MultinomialSeed sets the optional seed attribute to value.
 //
-// If `y` is negative, or greater than or equal to than the width of `x` in bits
-// the result is implementation defined.
-func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RightShift",
-		Input: []tf.Input{
-			x, y,
-		},
+// value: If either seed or seed2 is set to be non-zero, the internal random number
+// generator is seeded by the given seed.  Otherwise, a random seed is used.
+// If not specified, defaults to 0
+func MultinomialSeed(value int64) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// TensorListStackAttr is an optional argument to TensorListStack.
-type TensorListStackAttr func(optionalAttr)
+// MultinomialSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func MultinomialSeed2(value int64) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
 
-// TensorListStackNumElements sets the optional num_elements attribute to value.
-// If not specified, defaults to -1
-func TensorListStackNumElements(value int64) TensorListStackAttr {
+// MultinomialOutputDtype sets the optional output_dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
 	return func(m optionalAttr) {
-		m["num_elements"] = value
+		m["output_dtype"] = value
 	}
 }
 
-// Stacks all tensors in the list.
-//
-// Requires that all tensors have the same shape.
+// Draws samples from a multinomial distribution.
 //
-// input_handle: the input list
-// tensor: the gathered result
-// num_elements: optional. If not -1, the number of elements in the list.
+// Arguments:
+//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
+// represents the unnormalized log probabilities for all classes.
+//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
 //
-func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListStackAttr) (tensor tf.Output) {
+// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
+// contains the drawn class labels with range `[0, num_classes)`.
+func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorListStack",
+		Type: "Multinomial",
 		Input: []tf.Input{
-			input_handle,
+			logits, num_samples,
 		},
 		Attrs: attrs,
 	}
@@ -15995,32 +15911,35 @@ func TensorListStack(scope *Scope, input_handle tf.Output, element_dtype tf.Data
 	return op.Output(0)
 }
 
-// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
-type StatelessRandomUniformAttr func(optionalAttr)
+// ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
+type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
 
-// StatelessRandomUniformDtype sets the optional dtype attribute to value.
+// ResourceSparseApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
 //
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr {
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceSparseApplyAdagradDAUseLocking(value bool) ResourceSparseApplyAdagradDAAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Outputs deterministic pseudorandom random values from a uniform distribution.
-//
-// The generated values follow a uniform distribution in the range `[0, 1)`. The
-// lower bound 0 is included in the range, while the upper bound 1 is excluded.
-//
-// The outputs are a deterministic function of `shape` and `seed`.
+// Update entries in '*var' and '*accum' according to the proximal adagrad scheme.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	seed: 2 seeds (shape [2]).
+//	var_: Should be from a Variable().
+//	gradient_accumulator: Should be from a Variable().
+//	gradient_squared_accumulator: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Learning rate. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	global_step: Training step number. Must be a scalar.
 //
-// Returns Random values with specified shape.
-func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomUniformAttr) (output tf.Output) {
+// Returns the created operation.
+func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceSparseApplyAdagradDAAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16029,167 +15948,110 @@ func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optio
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessRandomUniform",
+		Type: "ResourceSparseApplyAdagradDA",
 		Input: []tf.Input{
-			shape, seed,
+			var_, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Makes its input available to the next iteration.
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
+//
+// The hash function is deterministic on the content of the string within the
+// process and will never change. However, it is not suitable for cryptography.
+// This function may be used when CPU time is scarce and inputs are trusted or
+// unimportant. There is a risk of adversaries constructing inputs that all hash
+// to the same bucket. To prevent this problem, use a strong hash function with
+// `tf.string_to_hash_bucket_strong`.
 //
 // Arguments:
-//	data: The tensor to be made available to the next iteration.
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
 //
-// Returns The same tensor as `data`.
-func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "NextIteration",
+		Type: "StringToHashBucketFast",
 		Input: []tf.Input{
-			data,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Output a fact about factorials.
-func Fact(scope *Scope) (fact tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Fact",
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Deserialize `SparseTensor` objects.
-//
-// The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where
-// the last dimension stores serialized `SparseTensor` objects and the other N
-// dimensions (N >= 0) correspond to a batch. The ranks of the original
-// `SparseTensor` objects must all match. When the final `SparseTensor` is
-// created, its rank is the rank of the incoming `SparseTensor` objects plus N;
-// the sparse tensors have been concatenated along new dimensions, one for each
-// batch.
-//
-// The output `SparseTensor` object's shape values for the original dimensions
-// are the max across the input `SparseTensor` objects' shape values for the
-// corresponding dimensions. The new dimensions match the size of the batch.
-//
-// The input `SparseTensor` objects' indices are assumed ordered in
-// standard lexicographic order.  If this is not the case, after this
-// step run `SparseReorder` to restore index ordering.
-//
-// For example, if the serialized input is a `[2 x 3]` matrix representing two
-// original `SparseTensor` objects:
-//
-//     index = [ 0]
-//             [10]
-//             [20]
-//     values = [1, 2, 3]
-//     shape = [50]
-//
-// and
-//
-//     index = [ 2]
-//             [10]
-//     values = [4, 5]
-//     shape = [30]
-//
-// then the final deserialized `SparseTensor` will be:
-//
-//     index = [0  0]
-//             [0 10]
-//             [0 20]
-//             [1  2]
-//             [1 10]
-//     values = [1, 2, 3, 4, 5]
-//     shape = [2 50]
-//
-// Arguments:
-//	serialized_sparse: The serialized `SparseTensor` objects. The last dimension
-// must have 3 columns.
-//	dtype: The `dtype` of the serialized `SparseTensor` objects.
-func DeserializeSparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
+// Returns the last element of the input list as well as a list with all but that element.
+//
+// Fails if the list is empty.
+//
+// input_handle: the input list
+// tensor: the withdrawn last element of the list
+// element_dtype: the type of elements in the list
+// element_shape: the shape of the output tensor
+func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
 	opspec := tf.OpSpec{
-		Type: "DeserializeSparse",
+		Type: "TensorListPopBack",
 		Input: []tf.Input{
-			serialized_sparse,
+			input_handle,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1)
 }
 
-// SqueezeAttr is an optional argument to Squeeze.
-type SqueezeAttr func(optionalAttr)
+// MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad.
+type MaxPoolGradGradAttr func(optionalAttr)
 
-// SqueezeAxis sets the optional axis attribute to value.
-//
-// value: If specified, only squeezes the dimensions listed. The dimension
-// index starts at 0. It is an error to squeeze a dimension that is not 1. Must
-// be in the range `[-rank(input), rank(input))`.
-// If not specified, defaults to <>
+// MaxPoolGradGradDataFormat sets the optional data_format attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func SqueezeAxis(value []int64) SqueezeAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolGradGradDataFormat(value string) MaxPoolGradGradAttr {
 	return func(m optionalAttr) {
-		m["squeeze_dims"] = value
+		m["data_format"] = value
 	}
 }
 
-// Removes dimensions of size 1 from the shape of a tensor.
-//
-// Given a tensor `input`, this operation returns a tensor of the same type with
-// all dimensions of size 1 removed. If you don't want to remove all size 1
-// dimensions, you can remove specific size 1 dimensions by specifying
-// `axis`.
-//
-// For example:
-//
-// ```
-// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
-// shape(squeeze(t)) ==> [2, 3]
-// ```
-//
-// Or, to remove specific size 1 dimensions:
-//
-// ```
-// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
-// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
-// ```
+// Computes second-order gradients of the maxpooling function.
 //
 // Arguments:
-//	input: The `input` to squeeze.
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns Contains the same data as `input`, but has one or more dimensions of
-// size 1 removed.
-func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) {
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Squeeze",
+		Type: "MaxPoolGradGrad",
 		Input: []tf.Input{
-			input,
+			orig_input, orig_output, grad,
 		},
 		Attrs: attrs,
 	}
@@ -16197,256 +16059,289 @@ func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.
 	return op.Output(0)
 }
 
-// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta.
-type ResourceApplyAdadeltaAttr func(optionalAttr)
+// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
+type TensorArrayGatherV3Attr func(optionalAttr)
 
-// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value.
+// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
 //
-// value: If True, updating of the var, accum and update_accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr {
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["element_shape"] = value
 	}
 }
 
-// Update '*var' according to the adadelta scheme.
+// Gather specific elements from the TensorArray into output `value`.
 //
-// accum = rho() * accum + (1 - rho()) * grad.square();
-// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
-// update_accum = rho() * update_accum + (1 - rho()) * update.square();
-// var -= update;
+// All elements selected by `indices` must have the same shape.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	accum_update: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay factor. Must be a scalar.
-//	epsilon: Constant factor. Must be a scalar.
-//	grad: The gradient.
+//	handle: The handle to a TensorArray.
+//	indices: The locations in the TensorArray from which to read tensor elements.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-// Returns the created operation.
-func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) {
+// Returns All of the elements in the TensorArray, concatenated along a new
+// axis (the new dimension 0).
+func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdadelta",
+		Type: "TensorArrayGatherV3",
 		Input: []tf.Input{
-			var_, accum, accum_update, lr, rho, epsilon, grad,
+			handle, indices, flow_in,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// NonMaxSuppressionAttr is an optional argument to NonMaxSuppression.
-type NonMaxSuppressionAttr func(optionalAttr)
-
-// NonMaxSuppressionIouThreshold sets the optional iou_threshold attribute to value.
+// Returns x / y element-wise for integer types.
 //
-// value: A float representing the threshold for deciding whether boxes
-// overlap too much with respect to IOU.
-// If not specified, defaults to 0.5
-func NonMaxSuppressionIouThreshold(value float32) NonMaxSuppressionAttr {
-	return func(m optionalAttr) {
-		m["iou_threshold"] = value
+// Truncation designates that negative numbers will round fractional quantities
+// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
+// than Python semantics. See `FloorDiv` for a division function that matches
+// Python Semantics.
+//
+// *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TruncateDiv",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Greedily selects a subset of bounding boxes in descending order of score,
+// Restores tensors from a V2 checkpoint.
 //
-// pruning away boxes that have high intersection-over-union (IOU) overlap
-// with previously selected boxes.  Bounding boxes are supplied as
-// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
-// diagonal pair of box corners and the coordinates can be provided as normalized
-// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
-// is agnostic to where the origin is in the coordinate system.  Note that this
-// algorithm is invariant to orthogonal transformations and translations
-// of the coordinate system; thus translating or reflections of the coordinate
-// system result in the same boxes being selected by the algorithm.
-// The output of this operation is a set of integers indexing into the input
-// collection of bounding boxes representing the selected boxes.  The bounding
-// box coordinates corresponding to the selected indices can then be obtained
-// using the `tf.gather operation`.  For example:
-//   selected_indices = tf.image.non_max_suppression(
-//       boxes, scores, max_output_size, iou_threshold)
-//   selected_boxes = tf.gather(boxes, selected_indices)
+// For backward compatibility with the V1 format, this Op currently allows
+// restoring from a V1 checkpoint as well:
+//   - This Op first attempts to find the V2 index file pointed to by "prefix", and
+//     if found proceed to read it as a V2 checkpoint;
+//   - Otherwise the V1 read path is invoked.
+// Relying on this behavior is not recommended, as the ability to fall back to read
+// V1 might be deprecated and eventually removed.
+//
+// By default, restores the named tensors in full.  If the caller wishes to restore
+// specific slices of stored tensors, "shape_and_slices" should be non-empty
+// strings and correspondingly well-formed.
+//
+// Callers must ensure all the named tensors are indeed stored in the checkpoint.
 //
 // Arguments:
-//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
-//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
-// score corresponding to each box (each row of boxes).
-//	max_output_size: A scalar integer tensor representing the maximum number of
-// boxes to be selected by non max suppression.
+//	prefix: Must have a single element.  The prefix of a V2 checkpoint.
+//	tensor_names: shape {N}.  The names of the tensors to be restored.
+//	shape_and_slices: shape {N}.  The slice specs of the tensors to be restored.
+// Empty strings indicate that they are non-partitioned tensors.
+//	dtypes: shape {N}.  The list of expected dtype for the tensors.  Must match
+// those stored in the checkpoint.
 //
-// Returns A 1-D integer tensor of shape `[M]` representing the selected
-// indices from the boxes tensor, where `M <= max_output_size`.
-func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, optional ...NonMaxSuppressionAttr) (selected_indices tf.Output) {
+// Returns shape {N}.  The restored tensors, whose shapes are read from the
+// checkpoint directly.
+func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, dtypes []tf.DataType) (tensors []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	opspec := tf.OpSpec{
-		Type: "NonMaxSuppression",
+		Type: "RestoreV2",
 		Input: []tf.Input{
-			boxes, scores, max_output_size,
+			prefix, tensor_names, shape_and_slices,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if tensors, idx, err = makeOutputList(op, idx, "tensors"); err != nil {
+		scope.UpdateErr("RestoreV2", err)
+		return
+	}
+	return tensors
 }
 
-// Creates a dataset that emits `components` as a tuple of tensors once.
-func TensorDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+// Receives a tensor value broadcast from another device.
+func CollectiveBcastRecv(scope *Scope, T tf.DataType, group_size int64, group_key int64, instance_key int64, shape tf.Shape) (data tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_shapes": output_shapes}
+	attrs := map[string]interface{}{"T": T, "group_size": group_size, "group_key": group_key, "instance_key": instance_key, "shape": shape}
 	opspec := tf.OpSpec{
-		Type: "TensorDataset",
-		Input: []tf.Input{
-			tf.OutputList(components),
-		},
+		Type: "CollectiveBcastRecv",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Component-wise multiplies a SparseTensor by a dense Tensor.
-//
-// The output locations corresponding to the implicitly zero elements in the sparse
-// tensor will be zero (i.e., will not take up storage space), regardless of the
-// contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN).
+// Decode web-safe base64-encoded strings.
 //
-// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
-// the other direction.
+// Input may or may not have padding at the end. See EncodeBase64 for padding.
+// Web-safe means that input must use - and _ instead of + and /.
 //
 // Arguments:
-//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
-//	dense: `R`-D.  The dense Tensor operand.
+//	input: Base64 strings to decode.
 //
-// Returns 1-D.  The `N` values that are operated on.
-func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+// Returns Decoded strings.
+func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseDenseCwiseMul",
+		Type: "DecodeBase64",
 		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape, dense,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// 2D real-valued fast Fourier transform.
-//
-// Computes the 2-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most 2 dimensions of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
-// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
-// of `output`: the zero-frequency term, followed by the `fft_length / 2`
-// positive-frequency terms.
-//
-// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// Store the input tensor in the state of the current session.
 //
 // Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
-//
-// Returns A complex64 tensor of the same rank as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their 2D Fourier transform. The
-//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
-//   components.
+//	value: The tensor to be stored.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft2
-// @end_compatibility
-func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// Returns The handle for the tensor stored in the session state, represented
+// as a string.
+func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RFFT2D",
+		Type: "GetSessionHandle",
 		Input: []tf.Input{
-			input, fft_length,
+			value,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Pads a tensor with zeros.
+// ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad.
+type ResourceSparseApplyProximalAdagradAttr func(optionalAttr)
+
+// ResourceSparseApplyProximalAdagradUseLocking sets the optional use_locking attribute to value.
 //
-// This operation pads a `input` with zeros according to the `paddings` you
-// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
-// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
-// how many zeros to add before the contents of `input` in that dimension, and
-// `paddings[D, 1]` indicates how many zeros to add after the contents of `input`
-// in that dimension.
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceSparseApplyProximalAdagradUseLocking(value bool) ResourceSparseApplyProximalAdagradAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Sparse update entries in '*var' and '*accum' according to FOBOS algorithm.
 //
-// The padded size of each dimension D of the output is:
+// That is for rows we have grad for, we update var and accum as follows:
+// accum += grad * grad
+// prox_v = var
+// prox_v -= lr * grad * (1 / sqrt(accum))
+// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
 //
-// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
 //
-// For example:
+// Returns the created operation.
+func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalAdagradAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyProximalAdagrad",
+		Input: []tf.Input{
+			var_, accum, lr, l1, l2, grad, indices,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// MaxPool3DGradAttr is an optional argument to MaxPool3DGrad.
+type MaxPool3DGradAttr func(optionalAttr)
+
+// MaxPool3DGradDataFormat sets the optional data_format attribute to value.
 //
-// ```
-// # 't' is [[1, 1], [2, 2]]
-// # 'paddings' is [[1, 1], [2, 2]]
-// # rank of 't' is 2
-// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
-//                       [0, 0, 1, 1, 0, 0]
-//                       [0, 0, 2, 2, 0, 0]
-//                       [0, 0, 0, 0, 0, 0]]
-// ```
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func MaxPool3DGradDataFormat(value string) MaxPool3DGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Computes gradients of max pooling function.
 //
-func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
+// Arguments:
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Pad",
+		Type: "MaxPool3DGrad",
 		Input: []tf.Input{
-			input, paddings,
+			orig_input, orig_output, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Checks whether a resource handle-based variable has been initialized.
-//
-// Arguments:
-//	resource: the input resource handle.
-//
-// Returns a scalar boolean which is true if the variable has been
-// initialized.
-func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) {
+// Returns the name of the device on which `resource` has been placed.
+func ExperimentalIteratorGetDevice(scope *Scope, resource tf.Output) (device tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "VarIsInitializedOp",
+		Type: "ExperimentalIteratorGetDevice",
 		Input: []tf.Input{
 			resource,
 		},
@@ -16455,29 +16350,54 @@ func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Out
 	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
+// SparseReduceSumAttr is an optional argument to SparseReduceSum.
+type SparseReduceSumAttr func(optionalAttr)
+
+// SparseReduceSumKeepDims sets the optional keep_dims attribute to value.
 //
-// The hash function is deterministic on the content of the string within the
-// process and will never change. However, it is not suitable for cryptography.
-// This function may be used when CPU time is scarce and inputs are trusted or
-// unimportant. There is a risk of adversaries constructing inputs that all hash
-// to the same bucket. To prevent this problem, use a strong hash function with
-// `tf.string_to_hash_bucket_strong`.
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SparseReduceSumKeepDims(value bool) SparseReduceSumAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the sum of elements across dimensions of a SparseTensor.
+//
+// This Op takes a SparseTensor and is the sparse counterpart to
+// `tf.reduce_sum()`.  In particular, this Op also returns a dense `Tensor`
+// instead of a sparse one.
+//
+// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+// with length 1.
+//
+// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+// with a single element is returned.  Additionally, the axes can be negative,
+// which are interpreted according to the indexing rules in Python.
 //
 // Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
+// Returns `R-K`-D.  The reduced Tensor.
+func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "StringToHashBucketFast",
+		Type: "SparseReduceSum",
 		Input: []tf.Input{
-			input,
+			input_indices, input_values, input_shape, reduction_axes,
 		},
 		Attrs: attrs,
 	}
@@ -16485,117 +16405,164 @@ func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (o
 	return op.Output(0)
 }
 
-// Returns the last element of the input list as well as a list with all but that element.
+// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
+type SparseTensorDenseMatMulAttr func(optionalAttr)
+
+// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
 //
-// Fails if the list is empty.
+// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
+// is transpose(conj(A)).  Otherwise it's transpose(A).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
+	return func(m optionalAttr) {
+		m["adjoint_a"] = value
+	}
+}
+
+// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
 //
-// input_handle: the input list
-// tensor: the withdrawn last element of the list
-// element_dtype: the type of elements in the list
-// element_shape: the shape of the output tensor
-func TensorListPopBack(scope *Scope, input_handle tf.Output, element_dtype tf.DataType) (output_handle tf.Output, tensor tf.Output) {
+// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
+// is transpose(conj(B)).  Otherwise it's transpose(B).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
+	return func(m optionalAttr) {
+		m["adjoint_b"] = value
+	}
+}
+
+// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
+//
+// No validity checking is performed on the indices of A.  However, the following
+// input format is recommended for optimal behavior:
+//
+// if adjoint_a == false:
+//   A should be sorted in lexicographically increasing order.  Use SparseReorder
+//   if you're not sure.
+// if adjoint_a == true:
+//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
+//   order instead of "row major" order).
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
+//	b: 2-D.  A dense Matrix.
+func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorListPopBack",
+		Type: "SparseTensorDenseMatMul",
 		Input: []tf.Input{
-			input_handle,
+			a_indices, a_values, a_shape, b,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad.
-type MaxPoolGradGradAttr func(optionalAttr)
+// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp.
+type ResourceApplyRMSPropAttr func(optionalAttr)
 
-// MaxPoolGradGradDataFormat sets the optional data_format attribute to value.
+// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradGradDataFormat(value string) MaxPoolGradGradAttr {
+// value: If `True`, updating of the var, ms, and mom tensors is protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Update '*var' according to the RMSProp algorithm.
+//
+// Note that in dense implementation of this algorithm, ms and mom will
+// update even if the grad is zero, but in this sparse implementation, ms
+// and mom will not update in iterations during which the grad is zero.
+//
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+//
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
 //
 // Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	var_: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
 //
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradGradAttr) (output tf.Output) {
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGrad",
+		Type: "ResourceApplyRMSProp",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			var_, ms, mom, lr, rho, momentum, epsilon, grad,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
-type TensorArrayGatherV3Attr func(optionalAttr)
+// SerializeManySparseAttr is an optional argument to SerializeManySparse.
+type SerializeManySparseAttr func(optionalAttr)
 
-// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
+// SerializeManySparseOutType sets the optional out_type attribute to value.
 //
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
+// value: The `dtype` to use for serialization; the supported types are `string`
+// (default) and `variant`.
+// If not specified, defaults to DT_STRING
+func SerializeManySparseOutType(value tf.DataType) SerializeManySparseAttr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["out_type"] = value
 	}
 }
 
-// Gather specific elements from the TensorArray into output `value`.
+// Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor` object.
 //
-// All elements selected by `indices` must have the same shape.
+// The `SparseTensor` must have rank `R` greater than 1, and the first dimension
+// is treated as the minibatch dimension.  Elements of the `SparseTensor`
+// must be sorted in increasing order of this first dimension.  The serialized
+// `SparseTensor` objects going into each row of `serialized_sparse` will have
+// rank `R-1`.
 //
-// Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations in the TensorArray from which to read tensor elements.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+// The minibatch size `N` is extracted from `sparse_shape[0]`.
 //
-// Returns All of the elements in the TensorArray, concatenated along a new
-// axis (the new dimension 0).
-func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+// Arguments:
+//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
+//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
+func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeManySparseAttr) (serialized_sparse tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV3",
+		Type: "SerializeManySparse",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			sparse_indices, sparse_values, sparse_shape,
 		},
 		Attrs: attrs,
 	}
@@ -16603,229 +16570,190 @@ func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow
 	return op.Output(0)
 }
 
-// Returns x / y element-wise for integer types.
-//
-// Truncation designates that negative numbers will round fractional quantities
-// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
-// than Python semantics. See `FloorDiv` for a division function that matches
-// Python Semantics.
-//
-// *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Computes inverse hyperbolic cosine of x element-wise.
+func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TruncateDiv",
+		Type: "Acosh",
 		Input: []tf.Input{
-			x, y,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Restores tensors from a V2 checkpoint.
-//
-// For backward compatibility with the V1 format, this Op currently allows
-// restoring from a V1 checkpoint as well:
-//   - This Op first attempts to find the V2 index file pointed to by "prefix", and
-//     if found proceed to read it as a V2 checkpoint;
-//   - Otherwise the V1 read path is invoked.
-// Relying on this behavior is not recommended, as the ability to fall back to read
-// V1 might be deprecated and eventually removed.
-//
-// By default, restores the named tensors in full.  If the caller wishes to restore
-// specific slices of stored tensors, "shape_and_slices" should be non-empty
-// strings and correspondingly well-formed.
-//
-// Callers must ensure all the named tensors are indeed stored in the checkpoint.
-//
-// Arguments:
-//	prefix: Must have a single element.  The prefix of a V2 checkpoint.
-//	tensor_names: shape {N}.  The names of the tensors to be restored.
-//	shape_and_slices: shape {N}.  The slice specs of the tensors to be restored.
-// Empty strings indicate that they are non-partitioned tensors.
-//	dtypes: shape {N}.  The list of expected dtype for the tensors.  Must match
-// those stored in the checkpoint.
-//
-// Returns shape {N}.  The restored tensors, whose shapes are read from the
-// checkpoint directly.
-func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, dtypes []tf.DataType) (tensors []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	opspec := tf.OpSpec{
-		Type: "RestoreV2",
-		Input: []tf.Input{
-			prefix, tensor_names, shape_and_slices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
+// TensorArrayV2Attr is an optional argument to TensorArrayV2.
+type TensorArrayV2Attr func(optionalAttr)
+
+// TensorArrayV2ElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayV2ElementShape(value tf.Shape) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
 	}
-	var idx int
-	var err error
-	if tensors, idx, err = makeOutputList(op, idx, "tensors"); err != nil {
-		scope.UpdateErr("RestoreV2", err)
-		return
+}
+
+// TensorArrayV2DynamicSize sets the optional dynamic_size attribute to value.
+// If not specified, defaults to false
+func TensorArrayV2DynamicSize(value bool) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["dynamic_size"] = value
 	}
-	return tensors
 }
 
-// Receives a tensor value broadcast from another device.
-func CollectiveBcastRecv(scope *Scope, T tf.DataType, group_size int64, group_key int64, instance_key int64, shape tf.Shape) (data tf.Output) {
-	if scope.Err() != nil {
-		return
+// TensorArrayV2ClearAfterRead sets the optional clear_after_read attribute to value.
+// If not specified, defaults to true
+func TensorArrayV2ClearAfterRead(value bool) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["clear_after_read"] = value
 	}
-	attrs := map[string]interface{}{"T": T, "group_size": group_size, "group_key": group_key, "instance_key": instance_key, "shape": shape}
-	opspec := tf.OpSpec{
-		Type: "CollectiveBcastRecv",
+}
 
-		Attrs: attrs,
+// TensorArrayV2TensorArrayName sets the optional tensor_array_name attribute to value.
+// If not specified, defaults to ""
+func TensorArrayV2TensorArrayName(value string) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["tensor_array_name"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Decode web-safe base64-encoded strings.
-//
-// Input may or may not have padding at the end. See EncodeBase64 for padding.
-// Web-safe means that input must use - and _ instead of + and /.
-//
-// Arguments:
-//	input: Base64 strings to decode.
+// Deprecated. Use TensorArrayV3
 //
-// Returns Decoded strings.
-func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArrayV3
+func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "DecodeBase64",
+		Type: "TensorArrayV2",
 		Input: []tf.Input{
-			input,
+			size,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Store the input tensor in the state of the current session.
-//
-// Arguments:
-//	value: The tensor to be stored.
+// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
+type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+
+// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// Returns The handle for the tensor stored in the session state, represented
-// as a string.
-func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "GetSessionHandle",
-		Input: []tf.Input{
-			value,
-		},
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad.
-type ResourceSparseApplyProximalAdagradAttr func(optionalAttr)
-
-// ResourceSparseApplyProximalAdagradUseLocking sets the optional use_locking attribute to value.
+// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
 //
-// value: If True, updating of the var and accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceSparseApplyProximalAdagradUseLocking(value bool) ResourceSparseApplyProximalAdagradAttr {
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["seed2"] = value
 	}
 }
 
-// Sparse update entries in '*var' and '*accum' according to FOBOS algorithm.
+// Generates labels for candidate sampling with a learned unigram distribution.
 //
-// That is for rows we have grad for, we update var and accum as follows:
-// accum += grad * grad
-// prox_v = var
-// prox_v -= lr * grad * (1 / sqrt(accum))
-// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
+//
+// For each batch, this op picks a single set of sampled candidate labels.
+//
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to randomly sample.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//	range_max: The sampler will sample integers from the interval [0, range_max).
 //
-// Returns the created operation.
-func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalAdagradAttr) (o *tf.Operation) {
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func ThreadUnsafeUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...ThreadUnsafeUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyProximalAdagrad",
+		Type: "ThreadUnsafeUnigramCandidateSampler",
 		Input: []tf.Input{
-			var_, accum, lr, l1, l2, grad, indices,
+			true_classes,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// MaxPool3DGradAttr is an optional argument to MaxPool3DGrad.
-type MaxPool3DGradAttr func(optionalAttr)
+// MaxPoolV2Attr is an optional argument to MaxPoolV2.
+type MaxPoolV2Attr func(optionalAttr)
 
-// MaxPool3DGradDataFormat sets the optional data_format attribute to value.
+// MaxPoolV2DataFormat sets the optional data_format attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func MaxPool3DGradDataFormat(value string) MaxPool3DGradAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
 	return func(m optionalAttr) {
 		m["data_format"] = value
 	}
 }
 
-// Computes gradients of max pooling function.
+// Performs max pooling on the input.
 //
 // Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
 //	padding: The type of padding algorithm to use.
-func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradAttr) (output tf.Output) {
+//
+// Returns The max pooled output tensor.
+func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPool3DGrad",
+		Type: "MaxPoolV2",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			input, ksize, strides,
 		},
 		Attrs: attrs,
 	}
@@ -16833,58 +16761,65 @@ func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, gr
 	return op.Output(0)
 }
 
-// Returns the name of the device on which `resource` has been placed.
-func ExperimentalIteratorGetDevice(scope *Scope, resource tf.Output) (device tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ExperimentalIteratorGetDevice",
-		Input: []tf.Input{
-			resource,
-		},
+// AddManySparseToTensorsMapAttr is an optional argument to AddManySparseToTensorsMap.
+type AddManySparseToTensorsMapAttr func(optionalAttr)
+
+// AddManySparseToTensorsMapContainer sets the optional container attribute to value.
+//
+// value: The container name for the `SparseTensorsMap` created by this op.
+// If not specified, defaults to ""
+func AddManySparseToTensorsMapContainer(value string) AddManySparseToTensorsMapAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// SparseReduceSumAttr is an optional argument to SparseReduceSum.
-type SparseReduceSumAttr func(optionalAttr)
-
-// SparseReduceSumKeepDims sets the optional keep_dims attribute to value.
+// AddManySparseToTensorsMapSharedName sets the optional shared_name attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SparseReduceSumKeepDims(value bool) SparseReduceSumAttr {
+// value: The shared name for the `SparseTensorsMap` created by this op.
+// If blank, the new Operation's unique name is used.
+// If not specified, defaults to ""
+func AddManySparseToTensorsMapSharedName(value string) AddManySparseToTensorsMapAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Computes the sum of elements across dimensions of a SparseTensor.
+// Add an `N`-minibatch `SparseTensor` to a `SparseTensorsMap`, return `N` handles.
 //
-// This Op takes a SparseTensor and is the sparse counterpart to
-// `tf.reduce_sum()`.  In particular, this Op also returns a dense `Tensor`
-// instead of a sparse one.
+// A `SparseTensor` of rank `R` is represented by three tensors: `sparse_indices`,
+// `sparse_values`, and `sparse_shape`, where
 //
-// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
-// with length 1.
+// ```sparse_indices.shape[1] == sparse_shape.shape[0] == R```
 //
-// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
-// with a single element is returned.  Additionally, the axes can be negative,
-// which are interpreted according to the indexing rules in Python.
+// An `N`-minibatch of `SparseTensor` objects is represented as a `SparseTensor`
+// having a first `sparse_indices` column taking values between `[0, N)`, where
+// the minibatch size `N == sparse_shape[0]`.
+//
+// The input `SparseTensor` must have rank `R` greater than 1, and the first
+// dimension is treated as the minibatch dimension.  Elements of the `SparseTensor`
+// must be sorted in increasing order of this first dimension.  The stored
+// `SparseTensor` objects pointed to by each row of the output `sparse_handles`
+// will have rank `R-1`.
+//
+// The `SparseTensor` values can then be read out as part of a minibatch by passing
+// the given keys as vector elements to `TakeManySparseFromTensorsMap`.  To ensure
+// the correct `SparseTensorsMap` is accessed, ensure that the same
+// `container` and `shared_name` are passed to that Op.  If no `shared_name`
+// is provided here, instead use the *name* of the Operation created by calling
+// `AddManySparseToTensorsMap` as the `shared_name` passed to
+// `TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
-//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
+// `sparse_indices[:, 0]` must be ordered values in `[0, N)`.
+//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
+// The minibatch size `N == sparse_shape[0]`.
 //
-// Returns `R-K`-D.  The reduced Tensor.
-func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumAttr) (output tf.Output) {
+// Returns 1-D.  The handles of the `SparseTensor` now stored in the
+// `SparseTensorsMap`.  Shape: `[N]`.
+func AddManySparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...AddManySparseToTensorsMapAttr) (sparse_handles tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16893,9 +16828,9 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseReduceSum",
+		Type: "AddManySparseToTensorsMap",
 		Input: []tf.Input{
-			input_indices, input_values, input_shape, reduction_axes,
+			sparse_indices, sparse_values, sparse_shape,
 		},
 		Attrs: attrs,
 	}
@@ -16903,122 +16838,44 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp
 	return op.Output(0)
 }
 
-// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
-type SparseTensorDenseMatMulAttr func(optionalAttr)
-
-// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
-//
-// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
-// is transpose(conj(A)).  Otherwise it's transpose(A).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
-	return func(m optionalAttr) {
-		m["adjoint_a"] = value
-	}
-}
-
-// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
-//
-// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
-// is transpose(conj(B)).  Otherwise it's transpose(B).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
-	return func(m optionalAttr) {
-		m["adjoint_b"] = value
-	}
-}
-
-// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
-//
-// No validity checking is performed on the indices of A.  However, the following
-// input format is recommended for optimal behavior:
-//
-// if adjoint_a == false:
-//   A should be sorted in lexicographically increasing order.  Use SparseReorder
-//   if you're not sure.
-// if adjoint_a == true:
-//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
-//   order instead of "row major" order).
+// Concatenates tensors along one dimension.
 //
 // Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
-//	b: 2-D.  A dense Matrix.
-func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
+//	values: List of `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
+//	axis: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [-rank(values), rank(values)).
+//
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.
+func ConcatV2(scope *Scope, values []tf.Output, axis tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseMatMul",
+		Type: "ConcatV2",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
+			tf.OutputList(values), axis,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp.
-type ResourceApplyRMSPropAttr func(optionalAttr)
-
-// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, ms, and mom tensors is protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' according to the RMSProp algorithm.
-//
-// Note that in dense implementation of this algorithm, ms and mom will
-// update even if the grad is zero, but in this sparse implementation, ms
-// and mom will not update in iterations during which the grad is zero.
-//
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
-//
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
-//
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
-//
-// Returns the created operation.
-func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) {
+// Reads and outputs the entire contents of the input filename.
+func ReadFile(scope *Scope, filename tf.Output) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyRMSProp",
+		Type: "ReadFile",
 		Input: []tf.Input{
-			var_, ms, mom, lr, rho, momentum, epsilon, grad,
+			filename,
 		},
-		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
 // Returns immutable tensor from memory region.
@@ -17108,6 +16965,44 @@ func EmptyTensorList(scope *Scope, element_shape tf.Output, max_num_elements tf.
 	return op.Output(0)
 }
 
+// Computes softsign gradients for a softsign operation.
+//
+// Arguments:
+//	gradients: The backpropagated gradients to the corresponding softsign operation.
+//	features: The features passed as input to the corresponding softsign operation.
+//
+// Returns The gradients: `gradients / (1 + abs(features)) ** 2`.
+func SoftsignGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SoftsignGrad",
+		Input: []tf.Input{
+			gradients, features,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Provides the time since epoch in seconds.
+//
+// Returns the timestamp as a `float64` for seconds since the Unix epoch.
+//
+// Note: the timestamp is computed when the op is executed, not when it is added
+// to the graph.
+func Timestamp(scope *Scope) (ts tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Timestamp",
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // VariableShapeAttr is an optional argument to VariableShape.
 type VariableShapeAttr func(optionalAttr)
 
@@ -17148,80 +17043,154 @@ func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr)
 	return op.Output(0)
 }
 
-// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
-type SparseToSparseSetOperationAttr func(optionalAttr)
+// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
+type AvgPoolGradAttr func(optionalAttr)
 
-// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr {
+// AvgPoolGradDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["data_format"] = value
 	}
 }
 
-// Applies set operation along last dimension of 2 `SparseTensor` inputs.
-//
-// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+// Computes gradients of the average pooling function.
 //
-// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
-// order and range of `set1` and `set2` indices.
+// Arguments:
+//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
+// the output of `avg_pool`.
+//	ksize: The size of the sliding window for each dimension of the input.
+//	strides: The stride of the sliding window for each dimension of the input.
+//	padding: The type of padding algorithm to use.
 //
-// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
-// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
-// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
-// ignored.
+// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
+func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "AvgPoolGrad",
+		Input: []tf.Input{
+			orig_input_shape, grad,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Greedily selects a subset of bounding boxes in descending order of score,
 //
-// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
-// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
-// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
-// ignored.
+// pruning away boxes that have high overlaps
+// with previously selected boxes.  Bounding boxes with score less than
+// `score_threshold` are removed. N-by-n overlap values are supplied as square matrix,
+// which allows for defining a custom overlap criterium (eg. intersection over union,
+// intersection over area, etc.).
 //
-// If `validate_indices` is `True`, this op validates the order and range of `set1`
-// and `set2` indices.
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
 //
-// Output `result` is a `SparseTensor` represented by `result_indices`,
-// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
-// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
-// dimension contains the result of `set_operation` applied to the corresponding
-// `[0...n-1]` dimension of `set`.
+//   selected_indices = tf.image.non_max_suppression_with_overlaps(
+//       overlaps, scores, max_output_size, overlap_threshold, score_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
 //
 // Arguments:
-//	set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
-// order.
-//	set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
-// order.
-//	set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must
-// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the
-// max set size across `0...n-1` dimensions.
-//	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
-// order.
-//	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
-// order.
-//	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
-// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the
-// max set size across `0...n-1` dimensions.
+//	overlaps: A 2-D float tensor of shape `[num_boxes, num_boxes]` representing
+// the n-by-n box overlap values.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
+//	overlap_threshold: A 0-D float tensor representing the threshold for deciding whether
+// boxes overlap too.
+//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
+// boxes based on score.
 //
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.
+func NonMaxSuppressionWithOverlaps(scope *Scope, overlaps tf.Output, scores tf.Output, max_output_size tf.Output, overlap_threshold tf.Output, score_threshold tf.Output) (selected_indices tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NonMaxSuppressionWithOverlaps",
+		Input: []tf.Input{
+			overlaps, scores, max_output_size, overlap_threshold, score_threshold,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StageClearAttr is an optional argument to StageClear.
+type StageClearAttr func(optionalAttr)
+
+// StageClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
-// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
-// is the max result set size across all `0...n-1` dimensions.
-func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+// REQUIRES: value >= 0
+func StageClearCapacity(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearMemoryLimit(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// StageClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageClearContainer(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// StageClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageClearSharedName(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
+//
+// Returns the created operation.
+func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"set_operation": set_operation}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseToSparseSetOperation",
-		Input: []tf.Input{
-			set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape,
-		},
+		Type: "StageClear",
+
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return scope.AddOperation(opspec)
 }
 
 // Computes softmax cross entropy cost and gradients to backpropagate.
@@ -17615,47 +17584,6 @@ func Conv3DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_ba
 	return op.Output(0)
 }
 
-// Subtracts sparse updates from the variable referenced by `resource`.
-//
-// This operation computes
-//
-//     # Scalar indices
-//     ref[indices, ...] -= updates[...]
-//
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] -= updates[i, ...]
-//
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...]
-//
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions add.
-//
-// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
-// </div>
-//
-// Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
-//
-// Returns the created operation.
-func ResourceScatterSub(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceScatterSub",
-		Input: []tf.Input{
-			resource, indices, updates,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // DepthwiseConv2dNativeBackpropFilterAttr is an optional argument to DepthwiseConv2dNativeBackpropFilter.
 type DepthwiseConv2dNativeBackpropFilterAttr func(optionalAttr)
 
@@ -18236,6 +18164,38 @@ func TopK(scope *Scope, input tf.Output, k int64, optional ...TopKAttr) (values
 	return op.Output(0), op.Output(1)
 }
 
+// The gradient operator for the SparseAdd op.
+//
+// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
+// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
+// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
+// values of A and B.
+//
+// Arguments:
+//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
+// the non-empty values of the sum.
+//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
+//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
+//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
+// `[nnz(sum), ndims]`.
+//
+// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
+// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
+// non-empty values of B.
+func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseAddGrad",
+		Input: []tf.Input{
+			backprop_val_grad, a_indices, b_indices, sum_indices,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Returns a list of tensors with the same shapes and contents as the input
 //
 // tensors.
@@ -18342,139 +18302,161 @@ func ResourceApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms
 	return scope.AddOperation(opspec)
 }
 
-// AudioSummaryAttr is an optional argument to AudioSummary.
-type AudioSummaryAttr func(optionalAttr)
+// ResourceSparseApplyCenteredRMSPropAttr is an optional argument to ResourceSparseApplyCenteredRMSProp.
+type ResourceSparseApplyCenteredRMSPropAttr func(optionalAttr)
 
-// AudioSummaryMaxOutputs sets the optional max_outputs attribute to value.
-//
-// value: Max number of batch elements to generate audio for.
-// If not specified, defaults to 3
+// ResourceSparseApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
 //
-// REQUIRES: value >= 1
-func AudioSummaryMaxOutputs(value int64) AudioSummaryAttr {
+// value: If `True`, updating of the var, mg, ms, and mom tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyCenteredRMSPropUseLocking(value bool) ResourceSparseApplyCenteredRMSPropAttr {
 	return func(m optionalAttr) {
-		m["max_outputs"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with audio.
+// Update '*var' according to the centered RMSProp algorithm.
 //
-// DEPRECATED at GraphDef version 15: Use AudioSummaryV2.
+// The centered RMSProp algorithm uses an estimate of the centered second moment
+// (i.e., the variance) for normalization, as opposed to regular RMSProp, which
+// uses the (uncentered) second moment. This often helps with training, but is
+// slightly more expensive in terms of computation and memory.
 //
-// The summary has up to `max_outputs` summary values containing audio. The
-// audio is built from `tensor` which must be 3-D with shape `[batch_size,
-// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+// Note that in dense implementation of this algorithm, mg, ms, and mom will
+// update even if the grad is zero, but in this sparse implementation, mg, ms,
+// and mom will not update in iterations during which the grad is zero.
 //
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// mean_grad = decay * mean_grad + (1-decay) * gradient
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
 //
-// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-// *  If `max_outputs` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
 //
 // Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 2-D of shape `[batch_size, frames]`.
-//	sample_rate: The sample rate of the signal in hertz.
+//	var_: Should be from a Variable().
+//	mg: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func AudioSummary(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate float32, optional ...AudioSummaryAttr) (summary tf.Output) {
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var, ms and mom.
+//
+// Returns the created operation.
+func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyCenteredRMSPropAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"sample_rate": sample_rate}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AudioSummary",
+		Type: "ResourceSparseApplyCenteredRMSProp",
 		Input: []tf.Input{
-			tag, tensor,
+			var_, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// QrAttr is an optional argument to Qr.
-type QrAttr func(optionalAttr)
-
-// QrFullMatrices sets the optional full_matrices attribute to value.
-//
-// value: If true, compute full-sized `q` and `r`. If false
-// (the default), compute only the leading `P` columns of `q`.
-// If not specified, defaults to false
-func QrFullMatrices(value bool) QrAttr {
-	return func(m optionalAttr) {
-		m["full_matrices"] = value
-	}
+	return scope.AddOperation(opspec)
 }
 
-// Computes the QR decompositions of one or more matrices.
+// Creates a dataset that batches `batch_size` elements from `input_dataset`.
 //
-// Computes the QR decomposition of each inner matrix in `tensor` such that
-// `tensor[..., :, :] = q[..., :, :] * r[..., :,:])`
+// Arguments:
 //
-// ```python
-// # a is a tensor.
-// # q is a tensor of orthonormal matrices.
-// # r is a tensor of upper triangular matrices.
-// q, r = qr(a)
-// q_full, r_full = qr(a, full_matrices=True)
-// ```
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
 //
-// Arguments:
-//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
 //
-// Returns Orthonormal basis for range of `a`. If `full_matrices` is `False` then
-// shape is `[..., M, P]`; if `full_matrices` is `True` then shape is
-// `[..., M, M]`.Triangular factor. If `full_matrices` is `False` then shape is
-// `[..., P, N]`. If `full_matrices` is `True` then shape is `[..., M, N]`.
-func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Output) {
+func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "Qr",
+		Type: "BatchDataset",
 		Input: []tf.Input{
-			input,
+			input_dataset, batch_size,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Check if the input matches the regex pattern.
+// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
+type RandomPoissonV2Attr func(optionalAttr)
+
+// RandomPoissonV2Seed sets the optional seed attribute to value.
 //
-// The input is a string tensor of any shape. The pattern is the
-// regular expression to be matched with every element of the input tensor.
-// The boolean values (True or False) of the output tensor indicate
-// if the input matches the regex pattern provided.
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
 //
-// The pattern follows the re2 syntax (https://github.com/google/re2/wiki/Syntax)
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// RandomPoissonV2Dtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from the Poisson distribution(s) described by rate.
+//
+// This op uses two algorithms, depending on rate. If rate >= 10, then
+// the algorithm by Hormann is used to acquire samples via
+// transformation-rejection.
+// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+//
+// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+// random variables.
+// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+// Programming, Volume 2. Addison Wesley
 //
 // Arguments:
-//	input: A string tensor of the text to be processed.
-//	pattern: The regular expression to match the input.
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in rate.
+//	rate: A tensor in which each scalar is a "rate" parameter describing the
+// associated poisson distribution.
 //
-// Returns A bool tensor with the same shape as `input`.
-func StaticRegexFullMatch(scope *Scope, input tf.Output, pattern string) (output tf.Output) {
+// Returns A tensor with shape `shape + shape(rate)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `rate[i0, i1, ...iN]`.
+func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"pattern": pattern}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "StaticRegexFullMatch",
+		Type: "RandomPoissonV2",
 		Input: []tf.Input{
-			input,
+			shape, rate,
 		},
 		Attrs: attrs,
 	}
@@ -18482,126 +18464,173 @@ func StaticRegexFullMatch(scope *Scope, input tf.Output, pattern string) (output
 	return op.Output(0)
 }
 
-// ResourceSparseApplyProximalGradientDescentAttr is an optional argument to ResourceSparseApplyProximalGradientDescent.
-type ResourceSparseApplyProximalGradientDescentAttr func(optionalAttr)
+// DecodeAndCropJpegAttr is an optional argument to DecodeAndCropJpeg.
+type DecodeAndCropJpegAttr func(optionalAttr)
 
-// ResourceSparseApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value.
+// DecodeAndCropJpegChannels sets the optional channels attribute to value.
 //
-// value: If True, the subtraction will be protected by a lock;
-// otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceSparseApplyProximalGradientDescentUseLocking(value bool) ResourceSparseApplyProximalGradientDescentAttr {
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodeAndCropJpegChannels(value int64) DecodeAndCropJpegAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["channels"] = value
 	}
 }
 
-// Sparse update '*var' as FOBOS algorithm with fixed learning rate.
+// DecodeAndCropJpegRatio sets the optional ratio attribute to value.
 //
-// That is for rows we have grad for, we update var as follows:
-// prox_v = var - alpha * grad
-// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
+// value: Downscaling ratio.
+// If not specified, defaults to 1
+func DecodeAndCropJpegRatio(value int64) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["ratio"] = value
+	}
+}
+
+// DecodeAndCropJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	alpha: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
+// value: If true use a slower but nicer upscaling of the
+// chroma planes (yuv420/422 only).
+// If not specified, defaults to true
+func DecodeAndCropJpegFancyUpscaling(value bool) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["fancy_upscaling"] = value
+	}
+}
+
+// DecodeAndCropJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
 //
-// Returns the created operation.
-func ResourceSparseApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalGradientDescentAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
+// value: If true try to recover an image from truncated input.
+// If not specified, defaults to false
+func DecodeAndCropJpegTryRecoverTruncated(value bool) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["try_recover_truncated"] = value
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+}
+
+// DecodeAndCropJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
+//
+// value: The minimum required fraction of lines before a truncated
+// input is accepted.
+// If not specified, defaults to 1
+func DecodeAndCropJpegAcceptableFraction(value float32) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["acceptable_fraction"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyProximalGradientDescent",
-		Input: []tf.Input{
-			var_, alpha, l1, l2, grad, indices,
-		},
-		Attrs: attrs,
+}
+
+// DecodeAndCropJpegDctMethod sets the optional dct_method attribute to value.
+//
+// value: string specifying a hint about the algorithm used for
+// decompression.  Defaults to "" which maps to a system-specific
+// default.  Currently valid values are ["INTEGER_FAST",
+// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
+// jpeg library changes to a version that does not have that specific
+// option.)
+// If not specified, defaults to ""
+func DecodeAndCropJpegDctMethod(value string) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["dct_method"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Real-valued fast Fourier transform.
+// Decode and Crop a JPEG-encoded image to a uint8 tensor.
 //
-// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most dimension of `input`.
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
 //
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
-// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
-// followed by the `fft_length / 2` positive-frequency terms.
+// Accepted values are:
 //
-// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// *   0: Use the number of channels in the JPEG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
 //
-// Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
+// If needed, the JPEG-encoded image is transformed to match the requested number
+// of color channels.
 //
-// Returns A complex64 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
-//   frequency components of its 1D Fourier transform.
+// The attr `ratio` allows downscaling the image by an integer factor during
+// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+// downscaling the image later.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft
-// @end_compatibility
-func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+//
+// It is equivalent to a combination of decode and crop, but much faster by only
+// decoding partial jpeg image.
+//
+// Arguments:
+//	contents: 0-D.  The JPEG-encoded image.
+//	crop_window: 1-D.  The crop window: [crop_y, crop_x, crop_height, crop_width].
+//
+// Returns 3-D with shape `[height, width, channels]`..
+func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output, optional ...DecodeAndCropJpegAttr) (image tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "RFFT",
+		Type: "DecodeAndCropJpeg",
 		Input: []tf.Input{
-			input, fft_length,
+			contents, crop_window,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Adds a value to the current value of a variable.
+// Adds two `SparseTensor` objects to produce another `SparseTensor`.
 //
-// Any ReadVariableOp with a control dependency on this op is guaranteed to
-// see the incremented value or a subsequent newer one.
+// The input `SparseTensor` objects' indices are assumed ordered in standard
+// lexicographic order.  If this is not the case, before this step run
+// `SparseReorder` to restore index ordering.
 //
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value by which the variable will be incremented.
+// By default, if two values sum to zero at some index, the output `SparseTensor`
+// would still include that particular location in its index, storing a zero in the
+// corresponding value slot.  To override this, callers can specify `thresh`,
+// indicating that if the sum has a magnitude strictly smaller than `thresh`, its
+// corresponding value and index would then not be included.  In particular,
+// `thresh == 0` (default) means everything is kept and actual thresholding happens
+// only for a positive value.
 //
-// Returns the created operation.
-func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+// In the following shapes, `nnz` is the count after taking `thresh` into account.
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the first `SparseTensor`, size `[nnz, ndims]` Matrix.
+//	a_values: 1-D.  The `values` of the first `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the first `SparseTensor`, size `[ndims]` Vector.
+//	b_indices: 2-D.  The `indices` of the second `SparseTensor`, size `[nnz, ndims]` Matrix.
+//	b_values: 1-D.  The `values` of the second `SparseTensor`, size `[nnz]` Vector.
+//	b_shape: 1-D.  The `shape` of the second `SparseTensor`, size `[ndims]` Vector.
+//	thresh: 0-D.  The magnitude threshold that determines if an output value/index
+// pair takes space.
+func SparseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output, thresh tf.Output) (sum_indices tf.Output, sum_values tf.Output, sum_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AssignAddVariableOp",
+		Type: "SparseAdd",
 		Input: []tf.Input{
-			resource, value,
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// QuantizedReluAttr is an optional argument to QuantizedRelu.
-type QuantizedReluAttr func(optionalAttr)
+// QuantizedRelu6Attr is an optional argument to QuantizedRelu6.
+type QuantizedRelu6Attr func(optionalAttr)
 
-// QuantizedReluOutType sets the optional out_type attribute to value.
+// QuantizedRelu6OutType sets the optional out_type attribute to value.
 // If not specified, defaults to DT_QUINT8
-func QuantizedReluOutType(value tf.DataType) QuantizedReluAttr {
+func QuantizedRelu6OutType(value tf.DataType) QuantizedRelu6Attr {
 	return func(m optionalAttr) {
 		m["out_type"] = value
 	}
 }
 
-// Computes Quantized Rectified Linear: `max(features, 0)`
+// Computes Quantized Rectified Linear 6: `min(max(features, 0), 6)`
 //
 // Arguments:
 //
@@ -18609,7 +18638,7 @@ func QuantizedReluOutType(value tf.DataType) QuantizedReluAttr {
 //	max_features: The float value that the highest quantized value represents.
 //
 // Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+func QuantizedRelu6(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedRelu6Attr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18618,7 +18647,7 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedRelu",
+		Type: "QuantizedRelu6",
 		Input: []tf.Input{
 			features, min_features, max_features,
 		},
@@ -18628,202 +18657,188 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Reshapes a SparseTensor to represent values in a new dense shape.
-//
-// This operation has the same semantics as reshape on the represented dense
-// tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
-//
-// If one component of `new_shape` is the special value -1, the size of that
-// dimension is computed so that the total dense size remains constant.  At
-// most one component of `new_shape` can be -1.  The number of dense elements
-// implied by `new_shape` must be the same as the number of dense elements
-// originally implied by `input_shape`.
-//
-// Reshaping does not affect the order of values in the SparseTensor.
-//
-// If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
-// has length `R_out`, then `input_indices` has shape `[N, R_in]`,
-// `input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
-// `output_shape` has length `R_out`.
-//
-// Arguments:
-//	input_indices: 2-D.  `N x R_in` matrix with the indices of non-empty values in a
-// SparseTensor.
-//	input_shape: 1-D.  `R_in` vector with the input SparseTensor's dense shape.
-//	new_shape: 1-D.  `R_out` vector with the requested new dense shape.
+// FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2.
+type FixedLengthRecordReaderV2Attr func(optionalAttr)
+
+// FixedLengthRecordReaderV2HeaderBytes sets the optional header_bytes attribute to value.
 //
-// Returns 2-D.  `N x R_out` matrix with the updated indices of non-empty
-// values in the output SparseTensor.1-D.  `R_out` vector with the full dense shape of the output
-// SparseTensor.  This is the same as `new_shape` but with any -1 dimensions
-// filled in.
-func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output, new_shape tf.Output) (output_indices tf.Output, output_shape tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseReshape",
-		Input: []tf.Input{
-			input_indices, input_shape, new_shape,
-		},
+// value: Number of bytes in the header, defaults to 0.
+// If not specified, defaults to 0
+func FixedLengthRecordReaderV2HeaderBytes(value int64) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["header_bytes"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
 }
 
-// Deprecated. Use TensorArraySplitV3
+// FixedLengthRecordReaderV2FooterBytes sets the optional footer_bytes attribute to value.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArraySplitV3
-func TensorArraySplitV2(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: Number of bytes in the footer, defaults to 0.
+// If not specified, defaults to 0
+func FixedLengthRecordReaderV2FooterBytes(value int64) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["footer_bytes"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "TensorArraySplitV2",
-		Input: []tf.Input{
-			handle, value, lengths, flow_in,
-		},
+}
+
+// FixedLengthRecordReaderV2HopBytes sets the optional hop_bytes attribute to value.
+//
+// value: Number of bytes to hop before each read. Default of 0 means using
+// record_bytes.
+// If not specified, defaults to 0
+func FixedLengthRecordReaderV2HopBytes(value int64) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["hop_bytes"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Reorders a SparseTensor into the canonical, row-major ordering.
+// FixedLengthRecordReaderV2Container sets the optional container attribute to value.
 //
-// Note that by convention, all sparse ops preserve the canonical ordering along
-// increasing dimension number. The only time ordering can be violated is during
-// manual manipulation of the indices and values vectors to add entries.
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func FixedLengthRecordReaderV2Container(value string) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// FixedLengthRecordReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// Reordering does not affect the shape of the SparseTensor.
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func FixedLengthRecordReaderV2SharedName(value string) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// FixedLengthRecordReaderV2Encoding sets the optional encoding attribute to value.
 //
-// If the tensor has rank `R` and `N` non-empty values, `input_indices` has
-// shape `[N, R]`, input_values has length `N`, and input_shape has length `R`.
+// value: The type of encoding for the file. Currently ZLIB and GZIP
+// are supported. Defaults to none.
+// If not specified, defaults to ""
+func FixedLengthRecordReaderV2Encoding(value string) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["encoding"] = value
+	}
+}
+
+// A Reader that outputs fixed-length records from a file.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	record_bytes: Number of bytes in the record.
 //
-// Returns 2-D.  `N x R` matrix with the same indices as input_indices, but
-// in canonical row-major ordering.1-D.  `N` non-empty values corresponding to `output_indices`.
-func SparseReorder(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
+// Returns The handle to reference the Reader.
+func FixedLengthRecordReaderV2(scope *Scope, record_bytes int64, optional ...FixedLengthRecordReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "SparseReorder",
-		Input: []tf.Input{
-			input_indices, input_values, input_shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Computes rectified linear: `max(features, 0)`.
-func Relu(scope *Scope, features tf.Output) (activations tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{"record_bytes": record_bytes}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Relu",
-		Input: []tf.Input{
-			features,
-		},
+		Type: "FixedLengthRecordReaderV2",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyAddSignAttr is an optional argument to ResourceApplyAddSign.
-type ResourceApplyAddSignAttr func(optionalAttr)
+// AudioSummaryAttr is an optional argument to AudioSummary.
+type AudioSummaryAttr func(optionalAttr)
 
-// ResourceApplyAddSignUseLocking sets the optional use_locking attribute to value.
+// AudioSummaryMaxOutputs sets the optional max_outputs attribute to value.
 //
-// value: If `True`, updating of the var and m tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAddSignUseLocking(value bool) ResourceApplyAddSignAttr {
+// value: Max number of batch elements to generate audio for.
+// If not specified, defaults to 3
+//
+// REQUIRES: value >= 1
+func AudioSummaryMaxOutputs(value int64) AudioSummaryAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["max_outputs"] = value
 	}
 }
 
-// Update '*var' according to the AddSign update.
+// Outputs a `Summary` protocol buffer with audio.
 //
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
-// update <- (alpha + sign_decay * sign(g) *sign(m)) * g
-// variable <- variable - lr_t * update
+// DEPRECATED at GraphDef version 15: Use AudioSummaryV2.
+//
+// The summary has up to `max_outputs` summary values containing audio. The
+// audio is built from `tensor` which must be 3-D with shape `[batch_size,
+// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+//
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
+//
+// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+// *  If `max_outputs` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	m: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	alpha: Must be a scalar.
-//	sign_decay: Must be a scalar.
-//	beta: Must be a scalar.
-//	grad: The gradient.
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 2-D of shape `[batch_size, frames]`.
+//	sample_rate: The sample rate of the signal in hertz.
 //
-// Returns the created operation.
-func ResourceApplyAddSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, alpha tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyAddSignAttr) (o *tf.Operation) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func AudioSummary(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate float32, optional ...AudioSummaryAttr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"sample_rate": sample_rate}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAddSign",
+		Type: "AudioSummary",
 		Input: []tf.Input{
-			var_, m, lr, alpha, sign_decay, beta, grad,
+			tag, tensor,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// UpperBoundAttr is an optional argument to UpperBound.
-type UpperBoundAttr func(optionalAttr)
+// QrAttr is an optional argument to Qr.
+type QrAttr func(optionalAttr)
 
-// UpperBoundOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func UpperBoundOutType(value tf.DataType) UpperBoundAttr {
+// QrFullMatrices sets the optional full_matrices attribute to value.
+//
+// value: If true, compute full-sized `q` and `r`. If false
+// (the default), compute only the leading `P` columns of `q`.
+// If not specified, defaults to false
+func QrFullMatrices(value bool) QrAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["full_matrices"] = value
 	}
 }
 
-// Applies upper_bound(sorted_search_values, values) along each row.
-//
-// Each set of rows with the same index in (sorted_inputs, values) is treated
-// independently.  The resulting row is the equivalent of calling
-// `np.searchsorted(sorted_inputs, values, side='right')`.
-//
-// The result is not a global index to the entire
-// `Tensor`, but rather just the index in the last dimension.
-//
-// A 2-D example:
-//   sorted_sequence = [[0, 3, 9, 9, 10],
-//                      [1, 2, 3, 4, 5]]
-//   values = [[2, 4, 9],
-//             [0, 2, 6]]
+// Computes the QR decompositions of one or more matrices.
 //
-//   result = UpperBound(sorted_sequence, values)
+// Computes the QR decomposition of each inner matrix in `tensor` such that
+// `tensor[..., :, :] = q[..., :, :] * r[..., :,:])`
 //
-//   result == [[1, 2, 4],
-//              [0, 2, 5]]
+// ```python
+// # a is a tensor.
+// # q is a tensor of orthonormal matrices.
+// # r is a tensor of upper triangular matrices.
+// q, r = qr(a)
+// q_full, r_full = qr(a, full_matrices=True)
+// ```
 //
 // Arguments:
-//	sorted_inputs: 2-D Tensor where each row is ordered.
-//	values: 2-D Tensor with the same numbers of rows as `sorted_search_values`. Contains
-// the values that will be searched for in `sorted_search_values`.
+//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
 //
-// Returns A `Tensor` with the same shape as `values`.  It contains the last scalar index
-// into the last dimension where values can be inserted without changing the
-// ordered property.
-func UpperBound(scope *Scope, sorted_inputs tf.Output, values tf.Output, optional ...UpperBoundAttr) (output tf.Output) {
+// Returns Orthonormal basis for range of `a`. If `full_matrices` is `False` then
+// shape is `[..., M, P]`; if `full_matrices` is `True` then shape is
+// `[..., M, M]`.Triangular factor. If `full_matrices` is `False` then shape is
+// `[..., P, N]`. If `full_matrices` is `True` then shape is `[..., M, N]`.
+func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18832,62 +18847,39 @@ func UpperBound(scope *Scope, sorted_inputs tf.Output, values tf.Output, optiona
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UpperBound",
+		Type: "Qr",
 		Input: []tf.Input{
-			sorted_inputs, values,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// FractionalMaxPoolGradAttr is an optional argument to FractionalMaxPoolGrad.
-type FractionalMaxPoolGradAttr func(optionalAttr)
-
-// FractionalMaxPoolGradOverlapping sets the optional overlapping attribute to value.
-//
-// value: When set to True, it means when pooling, the values at the boundary
-// of adjacent pooling cells are used by both cells. For example:
-//
-// `index  0  1  2  3  4`
-//
-// `value  20 5  16 3  7`
+// Check if the input matches the regex pattern.
 //
-// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
-// The result would be [20, 16] for fractional max pooling.
-// If not specified, defaults to false
-func FractionalMaxPoolGradOverlapping(value bool) FractionalMaxPoolGradAttr {
-	return func(m optionalAttr) {
-		m["overlapping"] = value
-	}
-}
-
-// Computes gradient of the FractionalMaxPool function.
+// The input is a string tensor of any shape. The pattern is the
+// regular expression to be matched with every element of the input tensor.
+// The boolean values (True or False) of the output tensor indicate
+// if the input matches the regex pattern provided.
 //
-// Arguments:
-//	orig_input: Original input for `fractional_max_pool`
-//	orig_output: Original output for `fractional_max_pool`
-//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
-// w.r.t. the output of `fractional_max_pool`.
-//	row_pooling_sequence: row pooling sequence, form pooling region with
-// col_pooling_sequence.
-//	col_pooling_sequence: column pooling sequence, form pooling region with
-// row_pooling sequence.
+// The pattern follows the re2 syntax (https://github.com/google/re2/wiki/Syntax)
 //
-// Returns 4-D.  Gradients w.r.t. the input of `fractional_max_pool`.
-func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalMaxPoolGradAttr) (output tf.Output) {
+// Arguments:
+//	input: A string tensor of the text to be processed.
+//	pattern: The regular expression to match the input.
+//
+// Returns A bool tensor with the same shape as `input`.
+func StaticRegexFullMatch(scope *Scope, input tf.Output, pattern string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"pattern": pattern}
 	opspec := tf.OpSpec{
-		Type: "FractionalMaxPoolGrad",
+		Type: "StaticRegexFullMatch",
 		Input: []tf.Input{
-			orig_input, orig_output, out_backprop, row_pooling_sequence, col_pooling_sequence,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -18895,34 +18887,36 @@ func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Ou
 	return op.Output(0)
 }
 
-// ResourceApplyAdagradDAAttr is an optional argument to ResourceApplyAdagradDA.
-type ResourceApplyAdagradDAAttr func(optionalAttr)
+// ResourceSparseApplyProximalGradientDescentAttr is an optional argument to ResourceSparseApplyProximalGradientDescent.
+type ResourceSparseApplyProximalGradientDescentAttr func(optionalAttr)
 
-// ResourceApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
+// ResourceSparseApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value.
 //
-// value: If True, updating of the var and accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// value: If True, the subtraction will be protected by a lock;
+// otherwise the behavior is undefined, but may exhibit less contention.
 // If not specified, defaults to false
-func ResourceApplyAdagradDAUseLocking(value bool) ResourceApplyAdagradDAAttr {
+func ResourceSparseApplyProximalGradientDescentUseLocking(value bool) ResourceSparseApplyProximalGradientDescentAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update '*var' according to the proximal adagrad scheme.
+// Sparse update '*var' as FOBOS algorithm with fixed learning rate.
+//
+// That is for rows we have grad for, we update var as follows:
+// prox_v = var - alpha * grad
+// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
 //
 // Arguments:
 //	var_: Should be from a Variable().
-//	gradient_accumulator: Should be from a Variable().
-//	gradient_squared_accumulator: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
+//	alpha: Scaling factor. Must be a scalar.
 //	l1: L1 regularization. Must be a scalar.
 //	l2: L2 regularization. Must be a scalar.
-//	global_step: Training step number. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
 //
 // Returns the created operation.
-func ResourceApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceApplyAdagradDAAttr) (o *tf.Operation) {
+func ResourceSparseApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalGradientDescentAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18931,246 +18925,266 @@ func ResourceApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdagradDA",
+		Type: "ResourceSparseApplyProximalGradientDescent",
 		Input: []tf.Input{
-			var_, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step,
+			var_, alpha, l1, l2, grad, indices,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// SparseReduceMaxSparseAttr is an optional argument to SparseReduceMaxSparse.
-type SparseReduceMaxSparseAttr func(optionalAttr)
-
-// SparseReduceMaxSparseKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SparseReduceMaxSparseKeepDims(value bool) SparseReduceMaxSparseAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the max of elements across dimensions of a SparseTensor.
+// Real-valued fast Fourier transform.
 //
-// This Op takes a SparseTensor and is the sparse counterpart to
-// `tf.reduce_max()`.  In contrast to SparseReduceMax, this Op returns a
-// SparseTensor.
+// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most dimension of `input`.
 //
-// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
-// with length 1.
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
+// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
+// followed by the `fft_length / 2` positive-frequency terms.
 //
-// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
-// with a single element is returned.  Additionally, the axes can be negative,
-// which are interpreted according to the indexing rules in Python.
+// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
-//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
-func SparseReduceMaxSparse(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceMaxSparseAttr) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
+//
+// Returns A complex64 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
+//   frequency components of its 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft
+// @end_compatibility
+func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SparseReduceMaxSparse",
+		Type: "RFFT",
 		Input: []tf.Input{
-			input_indices, input_values, input_shape, reduction_axes,
+			input, fft_length,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Creates a dataset that emits the outputs of `input_dataset` `count` times.
-//
-// Arguments:
+// Adds a value to the current value of a variable.
 //
-//	count: A scalar representing the number of times that `input_dataset` should
-// be repeated. A value of `-1` indicates that it should be repeated infinitely.
+// Any ReadVariableOp with a control dependency on this op is guaranteed to
+// see the incremented value or a subsequent newer one.
 //
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value by which the variable will be incremented.
 //
-func RepeatDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns the created operation.
+func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "RepeatDataset",
+		Type: "AssignAddVariableOp",
 		Input: []tf.Input{
-			input_dataset, count,
+			resource, value,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// AddManySparseToTensorsMapAttr is an optional argument to AddManySparseToTensorsMap.
-type AddManySparseToTensorsMapAttr func(optionalAttr)
+// QuantizedReluAttr is an optional argument to QuantizedRelu.
+type QuantizedReluAttr func(optionalAttr)
 
-// AddManySparseToTensorsMapContainer sets the optional container attribute to value.
-//
-// value: The container name for the `SparseTensorsMap` created by this op.
-// If not specified, defaults to ""
-func AddManySparseToTensorsMapContainer(value string) AddManySparseToTensorsMapAttr {
+// QuantizedReluOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QUINT8
+func QuantizedReluOutType(value tf.DataType) QuantizedReluAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["out_type"] = value
 	}
 }
 
-// AddManySparseToTensorsMapSharedName sets the optional shared_name attribute to value.
+// Computes Quantized Rectified Linear: `max(features, 0)`
 //
-// value: The shared name for the `SparseTensorsMap` created by this op.
-// If blank, the new Operation's unique name is used.
-// If not specified, defaults to ""
-func AddManySparseToTensorsMapSharedName(value string) AddManySparseToTensorsMapAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
+// Arguments:
+//
+//	min_features: The float value that the lowest quantized value represents.
+//	max_features: The float value that the highest quantized value represents.
+//
+// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
+func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizedRelu",
+		Input: []tf.Input{
+			features, min_features, max_features,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Add an `N`-minibatch `SparseTensor` to a `SparseTensorsMap`, return `N` handles.
-//
-// A `SparseTensor` of rank `R` is represented by three tensors: `sparse_indices`,
-// `sparse_values`, and `sparse_shape`, where
+// Reshapes a SparseTensor to represent values in a new dense shape.
 //
-// ```sparse_indices.shape[1] == sparse_shape.shape[0] == R```
+// This operation has the same semantics as reshape on the represented dense
+// tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
 //
-// An `N`-minibatch of `SparseTensor` objects is represented as a `SparseTensor`
-// having a first `sparse_indices` column taking values between `[0, N)`, where
-// the minibatch size `N == sparse_shape[0]`.
+// If one component of `new_shape` is the special value -1, the size of that
+// dimension is computed so that the total dense size remains constant.  At
+// most one component of `new_shape` can be -1.  The number of dense elements
+// implied by `new_shape` must be the same as the number of dense elements
+// originally implied by `input_shape`.
 //
-// The input `SparseTensor` must have rank `R` greater than 1, and the first
-// dimension is treated as the minibatch dimension.  Elements of the `SparseTensor`
-// must be sorted in increasing order of this first dimension.  The stored
-// `SparseTensor` objects pointed to by each row of the output `sparse_handles`
-// will have rank `R-1`.
+// Reshaping does not affect the order of values in the SparseTensor.
 //
-// The `SparseTensor` values can then be read out as part of a minibatch by passing
-// the given keys as vector elements to `TakeManySparseFromTensorsMap`.  To ensure
-// the correct `SparseTensorsMap` is accessed, ensure that the same
-// `container` and `shared_name` are passed to that Op.  If no `shared_name`
-// is provided here, instead use the *name* of the Operation created by calling
-// `AddManySparseToTensorsMap` as the `shared_name` passed to
-// `TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
+// If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
+// has length `R_out`, then `input_indices` has shape `[N, R_in]`,
+// `input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
+// `output_shape` has length `R_out`.
 //
 // Arguments:
-//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
-// `sparse_indices[:, 0]` must be ordered values in `[0, N)`.
-//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
-// The minibatch size `N == sparse_shape[0]`.
+//	input_indices: 2-D.  `N x R_in` matrix with the indices of non-empty values in a
+// SparseTensor.
+//	input_shape: 1-D.  `R_in` vector with the input SparseTensor's dense shape.
+//	new_shape: 1-D.  `R_out` vector with the requested new dense shape.
 //
-// Returns 1-D.  The handles of the `SparseTensor` now stored in the
-// `SparseTensorsMap`.  Shape: `[N]`.
-func AddManySparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...AddManySparseToTensorsMapAttr) (sparse_handles tf.Output) {
+// Returns 2-D.  `N x R_out` matrix with the updated indices of non-empty
+// values in the output SparseTensor.1-D.  `R_out` vector with the full dense shape of the output
+// SparseTensor.  This is the same as `new_shape` but with any -1 dimensions
+// filled in.
+func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output, new_shape tf.Output) (output_indices tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+	opspec := tf.OpSpec{
+		Type: "SparseReshape",
+		Input: []tf.Input{
+			input_indices, input_shape, new_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Deprecated. Use TensorArraySplitV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArraySplitV3
+func TensorArraySplitV2(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AddManySparseToTensorsMap",
+		Type: "TensorArraySplitV2",
 		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
+			handle, value, lengths, flow_in,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Concatenates tensors along one dimension.
+// Reorders a SparseTensor into the canonical, row-major ordering.
+//
+// Note that by convention, all sparse ops preserve the canonical ordering along
+// increasing dimension number. The only time ordering can be violated is during
+// manual manipulation of the indices and values vectors to add entries.
+//
+// Reordering does not affect the shape of the SparseTensor.
+//
+// If the tensor has rank `R` and `N` non-empty values, `input_indices` has
+// shape `[N, R]`, input_values has length `N`, and input_shape has length `R`.
 //
 // Arguments:
-//	values: List of `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
-//	axis: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [-rank(values), rank(values)).
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
 //
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.
-func ConcatV2(scope *Scope, values []tf.Output, axis tf.Output) (output tf.Output) {
+// Returns 2-D.  `N x R` matrix with the same indices as input_indices, but
+// in canonical row-major ordering.1-D.  `N` non-empty values corresponding to `output_indices`.
+func SparseReorder(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ConcatV2",
+		Type: "SparseReorder",
 		Input: []tf.Input{
-			tf.OutputList(values), axis,
+			input_indices, input_values, input_shape,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Reads and outputs the entire contents of the input filename.
-func ReadFile(scope *Scope, filename tf.Output) (contents tf.Output) {
+// Computes rectified linear: `max(features, 0)`.
+func Relu(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ReadFile",
+		Type: "Relu",
 		Input: []tf.Input{
-			filename,
+			features,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Multiplies sparse updates into the variable referenced by `resource`.
-//
-// This operation computes
-//
-//     # Scalar indices
-//     ref[indices, ...] *= updates[...]
-//
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] *= updates[i, ...]
-//
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] *= updates[i, ..., j, ...]
-//
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions multiply.
+// ResourceApplyAddSignAttr is an optional argument to ResourceApplyAddSign.
+type ResourceApplyAddSignAttr func(optionalAttr)
+
+// ResourceApplyAddSignUseLocking sets the optional use_locking attribute to value.
 //
-// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+// value: If `True`, updating of the var and m tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAddSignUseLocking(value bool) ResourceApplyAddSignAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the AddSign update.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
-// </div>
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+// update <- (alpha + sign_decay * sign(g) *sign(m)) * g
+// variable <- variable - lr_t * update
 //
 // Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	alpha: Must be a scalar.
+//	sign_decay: Must be a scalar.
+//	beta: Must be a scalar.
+//	grad: The gradient.
 //
 // Returns the created operation.
-func ResourceScatterMul(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+func ResourceApplyAddSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, alpha tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyAddSignAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterMul",
+		Type: "ResourceApplyAddSign",
 		Input: []tf.Input{
-			resource, indices, updates,
+			var_, m, lr, alpha, sign_decay, beta, grad,
 		},
+		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
@@ -19260,118 +19274,198 @@ func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_
 	return op.Output(0)
 }
 
-// SerializeManySparseAttr is an optional argument to SerializeManySparse.
-type SerializeManySparseAttr func(optionalAttr)
-
-// SerializeManySparseOutType sets the optional out_type attribute to value.
+// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`.
 //
-// value: The `dtype` to use for serialization; the supported types are `string`
-// (default) and `variant`.
-// If not specified, defaults to DT_STRING
-func SerializeManySparseOutType(value tf.DataType) SerializeManySparseAttr {
+// This Op does not require `a_indices` be sorted in standard lexicographic order.
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, with shape `[nnz]`.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, with shape `[ndims]`.
+//	b: `ndims`-D Tensor.  With shape `a_shape`.
+func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseTensorDenseAdd",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
+type SparseToSparseSetOperationAttr func(optionalAttr)
+
+// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor` object.
+// Applies set operation along last dimension of 2 `SparseTensor` inputs.
 //
-// The `SparseTensor` must have rank `R` greater than 1, and the first dimension
-// is treated as the minibatch dimension.  Elements of the `SparseTensor`
-// must be sorted in increasing order of this first dimension.  The serialized
-// `SparseTensor` objects going into each row of `serialized_sparse` will have
-// rank `R-1`.
+// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
 //
-// The minibatch size `N` is extracted from `sparse_shape[0]`.
+// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
+// order and range of `set1` and `set2` indices.
+//
+// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
+// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
+// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
+// ignored.
+//
+// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
+// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
+// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
+// ignored.
+//
+// If `validate_indices` is `True`, this op validates the order and range of `set1`
+// and `set2` indices.
+//
+// Output `result` is a `SparseTensor` represented by `result_indices`,
+// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+// dimension contains the result of `set_operation` applied to the corresponding
+// `[0...n-1]` dimension of `set`.
 //
 // Arguments:
-//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
-//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
-func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeManySparseAttr) (serialized_sparse tf.Output) {
+//	set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
+// order.
+//	set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
+// order.
+//	set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must
+// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the
+// max set size across `0...n-1` dimensions.
+//	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
+// order.
+//	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
+// order.
+//	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
+// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the
+// max set size across `0...n-1` dimensions.
+//
+//
+// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
+// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
+// is the max result set size across all `0...n-1` dimensions.
+func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"set_operation": set_operation}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SerializeManySparse",
+		Type: "SparseToSparseSetOperation",
 		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
+			set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes inverse hyperbolic cosine of x element-wise.
-func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Acosh",
-		Input: []tf.Input{
-			x,
-		},
+// MutableDenseHashTableV2Attr is an optional argument to MutableDenseHashTableV2.
+type MutableDenseHashTableV2Attr func(optionalAttr)
+
+// MutableDenseHashTableV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func MutableDenseHashTableV2Container(value string) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// TensorArrayV2Attr is an optional argument to TensorArrayV2.
-type TensorArrayV2Attr func(optionalAttr)
-
-// TensorArrayV2ElementShape sets the optional element_shape attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayV2ElementShape(value tf.Shape) TensorArrayV2Attr {
+// MutableDenseHashTableV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func MutableDenseHashTableV2SharedName(value string) MutableDenseHashTableV2Attr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["shared_name"] = value
 	}
 }
 
-// TensorArrayV2DynamicSize sets the optional dynamic_size attribute to value.
+// MutableDenseHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
 // If not specified, defaults to false
-func TensorArrayV2DynamicSize(value bool) TensorArrayV2Attr {
+func MutableDenseHashTableV2UseNodeNameSharing(value bool) MutableDenseHashTableV2Attr {
 	return func(m optionalAttr) {
-		m["dynamic_size"] = value
+		m["use_node_name_sharing"] = value
 	}
 }
 
-// TensorArrayV2ClearAfterRead sets the optional clear_after_read attribute to value.
-// If not specified, defaults to true
-func TensorArrayV2ClearAfterRead(value bool) TensorArrayV2Attr {
+// MutableDenseHashTableV2ValueShape sets the optional value_shape attribute to value.
+//
+// value: The shape of each value.
+// If not specified, defaults to <>
+func MutableDenseHashTableV2ValueShape(value tf.Shape) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["value_shape"] = value
+	}
+}
+
+// MutableDenseHashTableV2InitialNumBuckets sets the optional initial_num_buckets attribute to value.
+//
+// value: The initial number of hash table buckets. Must be a power
+// to 2.
+// If not specified, defaults to 131072
+func MutableDenseHashTableV2InitialNumBuckets(value int64) MutableDenseHashTableV2Attr {
 	return func(m optionalAttr) {
-		m["clear_after_read"] = value
+		m["initial_num_buckets"] = value
 	}
 }
 
-// TensorArrayV2TensorArrayName sets the optional tensor_array_name attribute to value.
-// If not specified, defaults to ""
-func TensorArrayV2TensorArrayName(value string) TensorArrayV2Attr {
+// MutableDenseHashTableV2MaxLoadFactor sets the optional max_load_factor attribute to value.
+//
+// value: The maximum ratio between number of entries and number of
+// buckets before growing the table. Must be between 0 and 1.
+// If not specified, defaults to 0.8
+func MutableDenseHashTableV2MaxLoadFactor(value float32) MutableDenseHashTableV2Attr {
 	return func(m optionalAttr) {
-		m["tensor_array_name"] = value
+		m["max_load_factor"] = value
 	}
 }
 
-// Deprecated. Use TensorArrayV3
+// Creates an empty hash table that uses tensors as the backing store.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArrayV3
-func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV2Attr) (handle tf.Output) {
+// It uses "open addressing" with quadratic reprobing to resolve
+// collisions.
+//
+// This op creates a mutable hash table, specifying the type of its keys and
+// values. Each value must be a scalar. Data can be inserted into the table using
+// the insert operations. It does not support the initialization operation.
+//
+// Arguments:
+//	empty_key: The key used to represent empty key buckets internally. Must not
+// be used in insert or lookup operations.
+//
+//	value_dtype: Type of the table values.
+//
+// Returns Handle to a table.
+func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, deleted_key tf.Output, value_dtype tf.DataType, optional ...MutableDenseHashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"value_dtype": value_dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayV2",
+		Type: "MutableDenseHashTableV2",
 		Input: []tf.Input{
-			size,
+			empty_key, deleted_key,
 		},
 		Attrs: attrs,
 	}
@@ -19379,118 +19473,110 @@ func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...
 	return op.Output(0)
 }
 
-// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
-type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+// UpperBoundAttr is an optional argument to UpperBound.
+type UpperBoundAttr func(optionalAttr)
 
-// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+// UpperBoundOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func UpperBoundOutType(value tf.DataType) UpperBoundAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["out_type"] = value
 	}
 }
 
-// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+// Applies upper_bound(sorted_search_values, values) along each row.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Generates labels for candidate sampling with a learned unigram distribution.
+// Each set of rows with the same index in (sorted_inputs, values) is treated
+// independently.  The resulting row is the equivalent of calling
+// `np.searchsorted(sorted_inputs, values, side='right')`.
 //
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
+// The result is not a global index to the entire
+// `Tensor`, but rather just the index in the last dimension.
 //
-// For each batch, this op picks a single set of sampled candidate labels.
+// A 2-D example:
+//   sorted_sequence = [[0, 3, 9, 9, 10],
+//                      [1, 2, 3, 4, 5]]
+//   values = [[2, 4, 9],
+//             [0, 2, 6]]
 //
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+//   result = UpperBound(sorted_sequence, values)
+//
+//   result == [[1, 2, 4],
+//              [0, 2, 5]]
 //
 // Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to randomly sample.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//	range_max: The sampler will sample integers from the interval [0, range_max).
+//	sorted_inputs: 2-D Tensor where each row is ordered.
+//	values: 2-D Tensor with the same numbers of rows as `sorted_search_values`. Contains
+// the values that will be searched for in `sorted_search_values`.
 //
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func ThreadUnsafeUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...ThreadUnsafeUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+// Returns A `Tensor` with the same shape as `values`.  It contains the last scalar index
+// into the last dimension where values can be inserted without changing the
+// ordered property.
+func UpperBound(scope *Scope, sorted_inputs tf.Output, values tf.Output, optional ...UpperBoundAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ThreadUnsafeUnigramCandidateSampler",
+		Type: "UpperBound",
 		Input: []tf.Input{
-			true_classes,
+			sorted_inputs, values,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// MaxPoolV2Attr is an optional argument to MaxPoolV2.
-type MaxPoolV2Attr func(optionalAttr)
+// FractionalMaxPoolGradAttr is an optional argument to FractionalMaxPoolGrad.
+type FractionalMaxPoolGradAttr func(optionalAttr)
 
-// MaxPoolV2DataFormat sets the optional data_format attribute to value.
+// FractionalMaxPoolGradOverlapping sets the optional overlapping attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
+// value: When set to True, it means when pooling, the values at the boundary
+// of adjacent pooling cells are used by both cells. For example:
+//
+// `index  0  1  2  3  4`
+//
+// `value  20 5  16 3  7`
+//
+// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
+// The result would be [20, 16] for fractional max pooling.
+// If not specified, defaults to false
+func FractionalMaxPoolGradOverlapping(value bool) FractionalMaxPoolGradAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["overlapping"] = value
 	}
 }
 
-// Performs max pooling on the input.
+// Computes gradient of the FractionalMaxPool function.
 //
 // Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	orig_input: Original input for `fractional_max_pool`
+//	orig_output: Original output for `fractional_max_pool`
+//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
+// w.r.t. the output of `fractional_max_pool`.
+//	row_pooling_sequence: row pooling sequence, form pooling region with
+// col_pooling_sequence.
+//	col_pooling_sequence: column pooling sequence, form pooling region with
+// row_pooling sequence.
 //
-// Returns The max pooled output tensor.
-func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
+// Returns 4-D.  Gradients w.r.t. the input of `fractional_max_pool`.
+func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalMaxPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolV2",
+		Type: "FractionalMaxPoolGrad",
 		Input: []tf.Input{
-			input, ksize, strides,
+			orig_input, orig_output, out_backprop, row_pooling_sequence, col_pooling_sequence,
 		},
 		Attrs: attrs,
 	}
@@ -19498,99 +19584,121 @@ func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output
 	return op.Output(0)
 }
 
-// MutableDenseHashTableV2Attr is an optional argument to MutableDenseHashTableV2.
-type MutableDenseHashTableV2Attr func(optionalAttr)
+// ResourceApplyAdagradDAAttr is an optional argument to ResourceApplyAdagradDA.
+type ResourceApplyAdagradDAAttr func(optionalAttr)
 
-// MutableDenseHashTableV2Container sets the optional container attribute to value.
+// ResourceApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
 //
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func MutableDenseHashTableV2Container(value string) MutableDenseHashTableV2Attr {
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceApplyAdagradDAUseLocking(value bool) ResourceApplyAdagradDAAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["use_locking"] = value
 	}
 }
 
-// MutableDenseHashTableV2SharedName sets the optional shared_name attribute to value.
+// Update '*var' according to the proximal adagrad scheme.
 //
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func MutableDenseHashTableV2SharedName(value string) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
+// Arguments:
+//	var_: Should be from a Variable().
+//	gradient_accumulator: Should be from a Variable().
+//	gradient_squared_accumulator: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	global_step: Training step number. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceApplyAdagradDAAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyAdagradDA",
+		Input: []tf.Input{
+			var_, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step,
+		},
+		Attrs: attrs,
 	}
+	return scope.AddOperation(opspec)
 }
 
-// MutableDenseHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// SparseReduceMaxSparseAttr is an optional argument to SparseReduceMaxSparse.
+type SparseReduceMaxSparseAttr func(optionalAttr)
+
+// SparseReduceMaxSparseKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
 // If not specified, defaults to false
-func MutableDenseHashTableV2UseNodeNameSharing(value bool) MutableDenseHashTableV2Attr {
+func SparseReduceMaxSparseKeepDims(value bool) SparseReduceMaxSparseAttr {
 	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// MutableDenseHashTableV2ValueShape sets the optional value_shape attribute to value.
+// Computes the max of elements across dimensions of a SparseTensor.
 //
-// value: The shape of each value.
-// If not specified, defaults to <>
-func MutableDenseHashTableV2ValueShape(value tf.Shape) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["value_shape"] = value
+// This Op takes a SparseTensor and is the sparse counterpart to
+// `tf.reduce_max()`.  In contrast to SparseReduceMax, this Op returns a
+// SparseTensor.
+//
+// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+// with length 1.
+//
+// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+// with a single element is returned.  Additionally, the axes can be negative,
+// which are interpreted according to the indexing rules in Python.
+//
+// Arguments:
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+func SparseReduceMaxSparse(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceMaxSparseAttr) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// MutableDenseHashTableV2InitialNumBuckets sets the optional initial_num_buckets attribute to value.
-//
-// value: The initial number of hash table buckets. Must be a power
-// to 2.
-// If not specified, defaults to 131072
-func MutableDenseHashTableV2InitialNumBuckets(value int64) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["initial_num_buckets"] = value
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
-}
-
-// MutableDenseHashTableV2MaxLoadFactor sets the optional max_load_factor attribute to value.
-//
-// value: The maximum ratio between number of entries and number of
-// buckets before growing the table. Must be between 0 and 1.
-// If not specified, defaults to 0.8
-func MutableDenseHashTableV2MaxLoadFactor(value float32) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["max_load_factor"] = value
+	opspec := tf.OpSpec{
+		Type: "SparseReduceMaxSparse",
+		Input: []tf.Input{
+			input_indices, input_values, input_shape, reduction_axes,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Creates an empty hash table that uses tensors as the backing store.
-//
-// It uses "open addressing" with quadratic reprobing to resolve
-// collisions.
-//
-// This op creates a mutable hash table, specifying the type of its keys and
-// values. Each value must be a scalar. Data can be inserted into the table using
-// the insert operations. It does not support the initialization operation.
+// Creates a dataset that emits the outputs of `input_dataset` `count` times.
 //
 // Arguments:
-//	empty_key: The key used to represent empty key buckets internally. Must not
-// be used in insert or lookup operations.
 //
-//	value_dtype: Type of the table values.
+//	count: A scalar representing the number of times that `input_dataset` should
+// be repeated. A value of `-1` indicates that it should be repeated infinitely.
 //
-// Returns Handle to a table.
-func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, deleted_key tf.Output, value_dtype tf.DataType, optional ...MutableDenseHashTableV2Attr) (table_handle tf.Output) {
+//
+func RepeatDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"value_dtype": value_dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "MutableDenseHashTableV2",
+		Type: "RepeatDataset",
 		Input: []tf.Input{
-			empty_key, deleted_key,
+			input_dataset, count,
 		},
 		Attrs: attrs,
 	}
@@ -19993,6 +20101,66 @@ func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (i
 	return op.Output(0)
 }
 
+// Inverse 3D real-valued fast Fourier transform.
+//
+// Computes the inverse 3-dimensional discrete Fourier transform of a real-valued
+// signal over the inner-most 3 dimensions of `input`.
+//
+// The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
+// The inner-most dimension contains the `fft_length / 2 + 1` unique components of
+// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
+// from the size of the inner-most 3 dimensions of `input`. If the FFT length used
+// to compute `input` is odd, it should be provided since it cannot be inferred
+// properly.
+//
+// Along each axis `IRFFT3D` is computed on, if `fft_length` (or
+// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A complex64 tensor.
+//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
+//
+// Returns A float32 tensor of the same rank as `input`. The inner-most 3
+//   dimensions of `input` are replaced with the `fft_length` samples of their
+//   inverse 3D real Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.irfftn with 3 dimensions.
+// @end_compatibility
+func IRFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IRFFT3D",
+		Input: []tf.Input{
+			input, fft_length,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the truth value of (x != y) element-wise.
+//
+// *NOTE*: `NotEqual` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func NotEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NotEqual",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StageSizeAttr is an optional argument to StageSize.
 type StageSizeAttr func(optionalAttr)
 
@@ -21371,27 +21539,163 @@ func OptimizeDataset(scope *Scope, input_dataset tf.Output, optimizations tf.Out
 // Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
 //
 // Arguments:
-//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, in the canonical lexicographic ordering.
-//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
-//	a_shape: 1-D.  Shape of the input SparseTensor.
-//	b_indices: counterpart to `a_indices` for the other operand.
-//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
-//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
+//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, in the canonical lexicographic ordering.
+//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
+//	a_shape: 1-D.  Shape of the input SparseTensor.
+//	b_indices: counterpart to `a_indices` for the other operand.
+//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
+//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
+//
+// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
+func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseSparseMinimum",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
+type MapUnstageNoKeyAttr func(optionalAttr)
+
+// MapUnstageNoKeyCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageNoKeyCapacity(value int64) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapUnstageNoKeyMemoryLimit(value int64) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageNoKeyContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageNoKeyContainer(value string) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageNoKeySharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageNoKeySharedName(value string) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns a random (key, value)
+//
+// from the underlying container.   If the underlying container
+// does not contain elements, the op will block until it does.
+func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapUnstageNoKey",
+		Input: []tf.Input{
+			indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	key = op.Output(idx)
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstageNoKey", err)
+		return
+	}
+	return key, values
+}
+
+// HashTableV2Attr is an optional argument to HashTableV2.
+type HashTableV2Attr func(optionalAttr)
+
+// HashTableV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func HashTableV2Container(value string) HashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// HashTableV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func HashTableV2SharedName(value string) HashTableV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// HashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+//
+// value: If true and shared_name is empty, the table is shared
+// using the node name.
+// If not specified, defaults to false
+func HashTableV2UseNodeNameSharing(value bool) HashTableV2Attr {
+	return func(m optionalAttr) {
+		m["use_node_name_sharing"] = value
+	}
+}
+
+// Creates a non-initialized hash table.
+//
+// This op creates a hash table, specifying the type of its keys and values.
+// Before using the table you will have to initialize it.  After initialization the
+// table will be immutable.
+//
+// Arguments:
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
 //
-// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
-func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
+// Returns Handle to a table.
+func HashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...HashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseSparseMinimum",
-		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
-		},
+		Type: "HashTableV2",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
 // TakeManySparseFromTensorsMapAttr is an optional argument to TakeManySparseFromTensorsMap.
@@ -22593,6 +22897,37 @@ func QuantizeDownAndShrinkRange(scope *Scope, input tf.Output, input_min tf.Outp
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// ApproximateEqualAttr is an optional argument to ApproximateEqual.
+type ApproximateEqualAttr func(optionalAttr)
+
+// ApproximateEqualTolerance sets the optional tolerance attribute to value.
+// If not specified, defaults to 1e-05
+func ApproximateEqualTolerance(value float32) ApproximateEqualAttr {
+	return func(m optionalAttr) {
+		m["tolerance"] = value
+	}
+}
+
+// Returns the truth value of abs(x-y) < tolerance element-wise.
+func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...ApproximateEqualAttr) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ApproximateEqual",
+		Input: []tf.Input{
+			x, y,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns the truth value of x OR y element-wise.
 //
 // *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
@@ -25426,77 +25761,6 @@ func Roll(scope *Scope, input tf.Output, shift tf.Output, axis tf.Output) (outpu
 	return op.Output(0)
 }
 
-// MapPeekAttr is an optional argument to MapPeek.
-type MapPeekAttr func(optionalAttr)
-
-// MapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekCapacity(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekMemoryLimit(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapPeekContainer(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapPeekSharedName(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op peeks at the values at the specified key.  If the
-//
-// underlying container does not contain this key
-// this op will block until it does.
-func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapPeek",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapPeek", err)
-		return
-	}
-	return values
-}
-
 // Looks up keys in a table, outputs the corresponding values.
 //
 // The tensor `keys` must of the same type as the keys of the table.
@@ -25609,238 +25873,102 @@ func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
 	opspec := tf.OpSpec{
 		Type: "LookupTableSizeV2",
 		Input: []tf.Input{
-			table_handle,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
-type ResizeBilinearGradAttr func(optionalAttr)
-
-// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
-//
-// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
-// aligned. Defaults to false.
-// If not specified, defaults to false
-func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
-	return func(m optionalAttr) {
-		m["align_corners"] = value
-	}
-}
-
-// Computes the gradient of bilinear interpolation.
-//
-// Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
-// The image tensor that was resized.
-//
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
-// Gradients with respect to the input image. Input image must have been
-// float or double.
-func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResizeBilinearGrad",
-		Input: []tf.Input{
-			grads, original_image,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Outputs all keys and values in the table.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//
-//
-//
-// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
-func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
-	opspec := tf.OpSpec{
-		Type: "LookupTableExportV2",
-		Input: []tf.Input{
-			table_handle,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Replaces the contents of the table with the specified keys and values.
-//
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
-//
-// Returns the created operation.
-func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LookupTableImportV2",
-		Input: []tf.Input{
-			table_handle, keys, values,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
-type MapUnstageNoKeyAttr func(optionalAttr)
-
-// MapUnstageNoKeyCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageNoKeyCapacity(value int64) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageNoKeyMemoryLimit(value int64) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageNoKeyContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageNoKeyContainer(value string) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
+			table_handle,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MapUnstageNoKeySharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageNoKeySharedName(value string) MapUnstageNoKeyAttr {
+// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
+type ResizeBilinearGradAttr func(optionalAttr)
+
+// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
+//
+// value: If true, the centers of the 4 corner pixels of the input and grad tensors are
+// aligned. Defaults to false.
+// If not specified, defaults to false
+func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Op removes and returns a random (key, value)
+// Computes the gradient of bilinear interpolation.
 //
-// from the underlying container.   If the underlying container
-// does not contain elements, the op will block until it does.
-func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
+// Arguments:
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
+// The image tensor that was resized.
+//
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
+// Gradients with respect to the input image. Input image must have been
+// float or double.
+func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapUnstageNoKey",
+		Type: "ResizeBilinearGrad",
 		Input: []tf.Input{
-			indices,
+			grads, original_image,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	key = op.Output(idx)
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstageNoKey", err)
-		return
-	}
-	return key, values
+	return op.Output(0)
 }
 
-// HashTableV2Attr is an optional argument to HashTableV2.
-type HashTableV2Attr func(optionalAttr)
-
-// HashTableV2Container sets the optional container attribute to value.
+// Outputs all keys and values in the table.
 //
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func HashTableV2Container(value string) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// HashTableV2SharedName sets the optional shared_name attribute to value.
+// Arguments:
+//	table_handle: Handle to the table.
 //
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func HashTableV2SharedName(value string) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// HashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
 //
-// value: If true and shared_name is empty, the table is shared
-// using the node name.
-// If not specified, defaults to false
-func HashTableV2UseNodeNameSharing(value bool) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
+//
+// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
+func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
+	opspec := tf.OpSpec{
+		Type: "LookupTableExportV2",
+		Input: []tf.Input{
+			table_handle,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Creates a non-initialized hash table.
+// Replaces the contents of the table with the specified keys and values.
 //
-// This op creates a hash table, specifying the type of its keys and values.
-// Before using the table you will have to initialize it.  After initialization the
-// table will be immutable.
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
 //
 // Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
 //
-// Returns Handle to a table.
-func HashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...HashTableV2Attr) (table_handle tf.Output) {
+// Returns the created operation.
+func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "HashTableV2",
-
-		Attrs: attrs,
+		Type: "LookupTableImportV2",
+		Input: []tf.Input{
+			table_handle, keys, values,
+		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
 // MultiDeviceIteratorFromStringHandleAttr is an optional argument to MultiDeviceIteratorFromStringHandle.
@@ -33542,131 +33670,3 @@ func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output
 	}
 	return scope.AddOperation(opspec)
 }
-
-// MapUnstageAttr is an optional argument to MapUnstage.
-type MapUnstageAttr func(optionalAttr)
-
-// MapUnstageCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageCapacity(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageContainer(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageSharedName(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the values associated with the key
-//
-// from the underlying container.   If the underlying container
-// does not contain this key, the op will block until it does.
-func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapUnstage",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstage", err)
-		return
-	}
-	return values
-}
-
-// MapSizeAttr is an optional argument to MapSize.
-type MapSizeAttr func(optionalAttr)
-
-// MapSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapSizeCapacity(value int64) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapSizeMemoryLimit(value int64) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapSizeContainer(value string) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapSizeSharedName(value string) MapSizeAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op returns the number of elements in the underlying container.
-func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapSize",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From 7c1123eac5fb2409d76001d0113a704aa3e824da Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Tue, 4 Dec 2018 18:04:51 -0800
Subject: [PATCH 1339/1554] Address review comments

---
 tensorflow/compiler/jit/xla_gpu_device.cc | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index fb13635414..148c7cb77b 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -16,14 +16,16 @@ limitations under the License.
 // Registers the XLA_GPU device, which is an XlaDevice instantiation that runs
 // operators using XLA via the XLA "CUDA" (GPU) backend.
 
+#include <set>
 #include "absl/memory/memory.h"
+#include "absl/strings/numbers.h"
+#include "absl/strings/str_split.h"
 #include "tensorflow/compiler/jit/kernels/xla_ops.h"
 #include "tensorflow/compiler/jit/xla_device.h"
 #include "tensorflow/compiler/jit/xla_device_ops.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 
@@ -53,20 +55,21 @@ Status XlaGpuDeviceFactory::CreateDevices(
     VLOG(1) << "Failed to create XLA_GPU device: " << platform.status();
     return Status::OK();
   }
-  const auto& allowed_gpus =
+  string allowed_gpus =
       session_options.config.gpu_options().visible_device_list();
-  std::unordered_set<int> gpu_ids;
+  std::set<int> gpu_ids;
   int num_visible_devices = platform.ValueOrDie()->VisibleDeviceCount();
   if (allowed_gpus.empty()) {
     for (int i = 0; i < num_visible_devices; ++i) gpu_ids.insert(i);
   } else {
+    // For loop below is copied from gpu/gpu_device.cc. It validates
+    // configuration string. It should be redundant since code would fail there
+    // before it gets to here.
     const std::vector<string> visible_devices =
-        str_util::Split(allowed_gpus, ',');
-    // copied from gpu/gpu_device.cc Should be redundant since code would fail
-    // there before it gets to here.
+        absl::StrSplit(allowed_gpus, ',');
     for (const string& platform_gpu_id_str : visible_devices) {
       int32 platform_gpu_id;
-      if (!strings::safe_strto32(platform_gpu_id_str, &platform_gpu_id)) {
+      if (!absl::SimpleAtoi(platform_gpu_id_str, &platform_gpu_id)) {
         return errors::InvalidArgument(
             "Could not parse entry in 'visible_device_list': '",
             platform_gpu_id_str, "'. visible_device_list = ", allowed_gpus);
@@ -79,8 +82,8 @@ Status XlaGpuDeviceFactory::CreateDevices(
       gpu_ids.insert(platform_gpu_id);
     }
   }
-  for (int i = 0; i < num_visible_devices; ++i) {
-    if (gpu_ids.count(i) == 0) continue;
+  for (const auto i : gpu_ids) {
+    // Skip devices that are not in the set.
     XlaDevice::Options options;
     options.platform = platform.ValueOrDie();
     options.device_name_prefix = name_prefix;
-- 
GitLab


From 3d4ff3eb8e9f420c34aea6dfd465b5c7e9e03157 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 4 Dec 2018 18:03:59 -0800
Subject: [PATCH 1340/1554] tflite_driver support for strings. (Plus a change
 in the precision used to print test specs)

PiperOrigin-RevId: 224080493
---
 tensorflow/lite/testing/join.h            |  5 +-
 tensorflow/lite/testing/join_test.cc      |  2 +-
 tensorflow/lite/testing/tf_driver_test.cc |  8 +--
 tensorflow/lite/testing/tflite_driver.cc  | 65 ++++++++++++++++++++---
 4 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/testing/join.h b/tensorflow/lite/testing/join.h
index 7d0040c488..d1c3146086 100644
--- a/tensorflow/lite/testing/join.h
+++ b/tensorflow/lite/testing/join.h
@@ -16,6 +16,7 @@ limitations under the License.
 #define TENSORFLOW_LITE_TESTING_JOIN_H_
 
 #include <cstdlib>
+#include <iomanip>
 #include <sstream>
 
 #include "tensorflow/lite/string.h"
@@ -30,9 +31,9 @@ string Join(T* data, size_t len, const string& delimiter) {
     return "";
   }
   std::stringstream result;
-  result << data[0];
+  result << std::setprecision(9) << data[0];
   for (int i = 1; i < len; i++) {
-    result << delimiter << data[i];
+    result << std::setprecision(9) << delimiter << data[i];
   }
   return result.str();
 }
diff --git a/tensorflow/lite/testing/join_test.cc b/tensorflow/lite/testing/join_test.cc
index a8d036c547..0b3c07f37e 100644
--- a/tensorflow/lite/testing/join_test.cc
+++ b/tensorflow/lite/testing/join_test.cc
@@ -28,7 +28,7 @@ TEST(JoinTest, JoinInt) {
 
 TEST(JoinTest, JoinFloat) {
   float data[] = {1.0, -3, 2.3, 1e-5};
-  EXPECT_EQ(Join(data, 4, " "), "1 -3 2.3 1e-05");
+  EXPECT_EQ(Join(data, 4, " "), "1 -3 2.29999995 9.99999975e-06");
 }
 
 TEST(JoinTest, JoinNullData) { EXPECT_THAT(Join<int>(nullptr, 3, ","), ""); }
diff --git a/tensorflow/lite/testing/tf_driver_test.cc b/tensorflow/lite/testing/tf_driver_test.cc
index 4381fe4c19..363d162d56 100644
--- a/tensorflow/lite/testing/tf_driver_test.cc
+++ b/tensorflow/lite/testing/tf_driver_test.cc
@@ -46,7 +46,7 @@ TEST(TfDriverTest, ReadingAndWrintingValues) {
   TestDriver driver;
   ASSERT_EQ(driver.WriteAndReadBack(tensorflow::DT_FLOAT, {1, 2, 2},
                                     "0.10,0.20,0.30,0.40"),
-            "0.1,0.2,0.3,0.4");
+            "0.100000001,0.200000003,0.300000012,0.400000006");
   ASSERT_EQ(driver.WriteAndReadBack(tensorflow::DT_INT32, {1, 2, 2},
                                     "10,40,100,-100"),
             "10,40,100,-100");
@@ -111,8 +111,10 @@ TEST(TfDriverTest, SimpleTest) {
   runner->ResetTensor(2);
   runner->Invoke();
 
-  ASSERT_EQ(runner->ReadOutput(0), "0.101,0.202,0.303,0.404");
-  ASSERT_EQ(runner->ReadOutput(1), "0.011,0.022,0.033,0.044");
+  ASSERT_EQ(runner->ReadOutput(0),
+            "0.101000004,0.202000007,0.303000003,0.404000014");
+  ASSERT_EQ(runner->ReadOutput(1),
+            "0.0109999999,0.0219999999,0.0329999998,0.0439999998");
 }
 
 }  // namespace
diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc
index 27e3a3770b..4e11d49f25 100644
--- a/tensorflow/lite/testing/tflite_driver.cc
+++ b/tensorflow/lite/testing/tflite_driver.cc
@@ -77,6 +77,13 @@ class TfLiteDriver::Expectation {
     SetTensorData(values, &data_);
   }
 
+  template <>
+  void SetData<string>(const string& csv_values) {
+    string s = absl::HexStringToBytes(csv_values);
+    data_.raw = new char[s.size()];
+    memcpy(data_.raw, s.data(), s.size());
+  }
+
   bool Check(bool verbose, const TfLiteTensor& tensor) {
     switch (tensor.type) {
       case kTfLiteFloat32:
@@ -89,6 +96,8 @@ class TfLiteDriver::Expectation {
         return TypedCheck<uint8_t>(verbose, tensor);
       case kTfLiteBool:
         return TypedCheck<bool>(verbose, tensor);
+      case kTfLiteString:
+        return TypedCheck<string>(verbose, tensor);
       default:
         fprintf(stderr, "Unsupported type %d in Check\n", tensor.type);
         return false;
@@ -135,6 +144,46 @@ class TfLiteDriver::Expectation {
     return good_output;
   }
 
+  template <>
+  bool TypedCheck<string>(bool verbose, const TfLiteTensor& tensor) {
+    if (tensor.data.raw == nullptr) {
+      if (verbose) {
+        std::cerr << "  got empty string" << std::endl;
+      }
+      return false;
+    }
+    int expected_num_strings = GetStringCount(data_.raw);
+    int returned_num_strings = GetStringCount(tensor.data.raw);
+    if (expected_num_strings != returned_num_strings) {
+      if (verbose) {
+        std::cerr << "  string count differ: got " << returned_num_strings
+                  << ", but expected " << expected_num_strings << std::endl;
+      }
+      return false;
+    }
+    for (int i = 0; i < returned_num_strings; ++i) {
+      auto expected_ref = GetString(data_.raw, i);
+      auto returned_ref = GetString(tensor.data.raw, i);
+      if (expected_ref.len != returned_ref.len) {
+        if (verbose) {
+          std::cerr << "  index " << i << ": got string of size "
+                    << returned_ref.len << ", but expected size "
+                    << expected_ref.len << std::endl;
+        }
+        return false;
+      }
+      if (strncmp(expected_ref.str, returned_ref.str, returned_ref.len) != 0) {
+        if (verbose) {
+          std::cerr << "  index " << i << ": strings are different"
+                    << std::endl;
+        }
+        return false;
+      }
+    }
+
+    return true;
+  }
+
   TfLitePtrUnion data_;
   size_t num_elements_;
 };
@@ -250,8 +299,9 @@ void TfLiteDriver::SetInput(int id, const string& csv_values) {
       break;
     }
     default:
-      fprintf(stderr, "Unsupported type %d in SetInput\n", tensor->type);
-      Invalidate("Unsupported tensor data type");
+      Invalidate(absl::StrCat("Unsupported tensor type ",
+                              TfLiteTypeGetName(tensor->type),
+                              " in TfLiteDriver::SetInput"));
       return;
   }
 }
@@ -260,8 +310,7 @@ void TfLiteDriver::SetExpectation(int id, const string& csv_values) {
   if (!IsValid()) return;
   auto* tensor = interpreter_->tensor(id);
   if (expected_output_.count(id) != 0) {
-    fprintf(stderr, "Overridden expectation for tensor %d\n", id);
-    Invalidate("Overridden expectation");
+    Invalidate(absl::StrCat("Overridden expectation for tensor '", id, "'"));
   }
   expected_output_[id].reset(new Expectation);
   switch (tensor->type) {
@@ -280,9 +329,13 @@ void TfLiteDriver::SetExpectation(int id, const string& csv_values) {
     case kTfLiteBool:
       expected_output_[id]->SetData<bool>(csv_values);
       break;
+    case kTfLiteString:
+      expected_output_[id]->SetData<string>(csv_values);
+      break;
     default:
-      fprintf(stderr, "Unsupported type %d in SetExpectation\n", tensor->type);
-      Invalidate("Unsupported tensor data type");
+      Invalidate(absl::StrCat("Unsupported tensor type ",
+                              TfLiteTypeGetName(tensor->type),
+                              " in TfLiteDriver::SetExpectation"));
       return;
   }
 }
-- 
GitLab


From faff54f09ca90cdca0f6671128256f65820c7f0e Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Tue, 4 Dec 2018 18:27:26 -0800
Subject: [PATCH 1341/1554] Fix missed review comments

---
 tensorflow/compiler/jit/xla_gpu_device.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 148c7cb77b..0191315a66 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -60,11 +60,12 @@ Status XlaGpuDeviceFactory::CreateDevices(
   std::set<int> gpu_ids;
   int num_visible_devices = platform.ValueOrDie()->VisibleDeviceCount();
   if (allowed_gpus.empty()) {
-    for (int i = 0; i < num_visible_devices; ++i) gpu_ids.insert(i);
+    for (int i = 0; i < num_visible_devices; ++i) {
+      gpu_ids.insert(i);
+    }
   } else {
     // For loop below is copied from gpu/gpu_device.cc. It validates
-    // configuration string. It should be redundant since code would fail there
-    // before it gets to here.
+    // the visible_device_list and populates gpu_ids set.
     const std::vector<string> visible_devices =
         absl::StrSplit(allowed_gpus, ',');
     for (const string& platform_gpu_id_str : visible_devices) {
@@ -82,8 +83,7 @@ Status XlaGpuDeviceFactory::CreateDevices(
       gpu_ids.insert(platform_gpu_id);
     }
   }
-  for (const auto i : gpu_ids) {
-    // Skip devices that are not in the set.
+  for (int i : gpu_ids) {
     XlaDevice::Options options;
     options.platform = platform.ValueOrDie();
     options.device_name_prefix = name_prefix;
-- 
GitLab


From be68e120ccd8c1034760e0b52d5421a93d22a366 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 4 Dec 2018 19:09:22 -0800
Subject: [PATCH 1342/1554] [XLA] Switch hlo_computation_test to use pattern
 matchers.

Part of the transition away from hlo_matchers.

Also clean up a few spots that were missed in layout_assignment_test.

PiperOrigin-RevId: 224087042
---
 tensorflow/compiler/xla/service/BUILD         |  5 +-
 .../xla/service/hlo_computation_test.cc       | 47 +++++++++++--------
 .../xla/service/layout_assignment_test.cc     |  1 -
 .../compiler/xla/service/pattern_matcher.h    |  1 +
 4 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 81e71eee52..2285bba3a7 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2067,7 +2067,8 @@ tf_cc_test(
     srcs = ["hlo_computation_test.cc"],
     deps = [
         ":hlo",
-        ":hlo_matchers",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:test",
@@ -2661,7 +2662,6 @@ tf_cc_test(
         ":algebraic_simplifier",
         ":computation_layout",
         ":hlo",
-        ":hlo_matchers",
         ":layout_assignment",
         ":pattern_matcher",
         ":pattern_matcher_gmock",
@@ -2675,6 +2675,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:test_utils",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 8b50cfa9ae..0361c87428 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -20,19 +20,19 @@ limitations under the License.
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/test_helpers.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 
-namespace op = xla::testing::opcode_matchers;
-
 namespace xla {
 
 namespace {
 
+namespace m = match;
 using ::testing::ElementsAre;
 using ::testing::UnorderedElementsAre;
 
@@ -261,7 +261,7 @@ TEST_F(HloComputationTest, DeepCopyArray) {
   auto computation = module->AddEntryComputation(builder.Build());
   auto copy = computation->DeepCopyInstruction(constant).ValueOrDie();
 
-  EXPECT_THAT(copy, op::Copy(constant));
+  EXPECT_THAT(copy, GmockMatch(m::Copy(m::Op().Is(constant))));
 }
 
 TEST_F(HloComputationTest, DeepCopyTuple) {
@@ -278,8 +278,9 @@ TEST_F(HloComputationTest, DeepCopyTuple) {
   auto computation = module->AddEntryComputation(builder.Build());
   auto tuple_copy = computation->DeepCopyInstruction(tuple).ValueOrDie();
 
-  EXPECT_THAT(tuple_copy, op::Tuple(op::Copy(op::GetTupleElement(tuple)),
-                                    op::Copy(op::GetTupleElement(tuple))));
+  EXPECT_THAT(tuple_copy, GmockMatch(m::Tuple(
+                              m::Copy(m::GetTupleElement(m::Op().Is(tuple))),
+                              m::Copy(m::GetTupleElement(m::Op().Is(tuple))))));
   EXPECT_EQ(0, tuple_copy->operand(0)->operand(0)->tuple_index());
   EXPECT_EQ(1, tuple_copy->operand(1)->operand(0)->tuple_index());
 }
@@ -297,7 +298,7 @@ TEST_F(HloComputationTest, DeepCopyArrayAtIndices) {
     ShapeTree<bool> indices_to_copy(constant->shape(), /*init_value=*/true);
     EXPECT_THAT(computation->DeepCopyInstruction(constant, &indices_to_copy)
                     .ValueOrDie(),
-                op::Copy(constant));
+                GmockMatch(m::Copy(m::Op().Is(constant))));
   }
 
   {
@@ -330,10 +331,11 @@ TEST_F(HloComputationTest, DeepCopyTupleAtIndices) {
         computation->DeepCopyInstruction(tuple, &indices_to_copy, &copies_added)
             .ValueOrDie();
 
-    EXPECT_THAT(deep_copy, op::Tuple(op::Copy(op::GetTupleElement(tuple)),
-                                     op::Copy(op::GetTupleElement(tuple))));
-    EXPECT_THAT(deep_copy, op::Tuple(copies_added.element({0}),
-                                     copies_added.element({1})));
+    EXPECT_THAT(deep_copy, GmockMatch(m::Tuple(
+                               m::Copy(m::GetTupleElement(m::Op().Is(tuple)))
+                                   .Is(copies_added.element({0})),
+                               m::Copy(m::GetTupleElement(m::Op().Is(tuple)))
+                                   .Is(copies_added.element({1})))));
   }
 
   {
@@ -346,8 +348,9 @@ TEST_F(HloComputationTest, DeepCopyTupleAtIndices) {
         computation->DeepCopyInstruction(tuple, &indices_to_copy, &copies_added)
             .ValueOrDie();
 
-    EXPECT_THAT(deep_copy, op::Tuple(op::GetTupleElement(tuple),
-                                     op::GetTupleElement(tuple)));
+    EXPECT_THAT(deep_copy,
+                GmockMatch(m::Tuple(m::GetTupleElement(m::Op().Is(tuple)),
+                                    m::GetTupleElement(m::Op().Is(tuple)))));
     EXPECT_TRUE(copies_added.element({}) == nullptr);
     EXPECT_TRUE(copies_added.element({0}) == nullptr);
     EXPECT_TRUE(copies_added.element({1}) == nullptr);
@@ -363,8 +366,9 @@ TEST_F(HloComputationTest, DeepCopyTupleAtIndices) {
         computation->DeepCopyInstruction(tuple, &indices_to_copy, &copies_added)
             .ValueOrDie();
 
-    EXPECT_THAT(deep_copy, op::Tuple(op::Copy(op::GetTupleElement(tuple)),
-                                     op::GetTupleElement(tuple)));
+    EXPECT_THAT(deep_copy, GmockMatch(m::Tuple(
+                               m::Copy(m::GetTupleElement(m::Op().Is(tuple))),
+                               m::GetTupleElement(m::Op().Is(tuple)))));
     EXPECT_TRUE(copies_added.element({}) == nullptr);
     EXPECT_TRUE(copies_added.element({0}) != nullptr);
     EXPECT_TRUE(copies_added.element({1}) == nullptr);
@@ -381,7 +385,7 @@ TEST_F(HloComputationTest, DeepCopyToken) {
   auto copy = computation->DeepCopyInstruction(token).ValueOrDie();
 
   // No copy should be added.
-  EXPECT_THAT(copy, op::AfterAll());
+  EXPECT_THAT(copy, GmockMatch(m::AfterAll()));
 }
 
 TEST_F(HloComputationTest, DeepCopyTokenTuple) {
@@ -399,8 +403,9 @@ TEST_F(HloComputationTest, DeepCopyTokenTuple) {
 
   // Only the array (second tuple element) should be copied. The token is passed
   // through transparently.
-  EXPECT_THAT(copy, op::Tuple(op::GetTupleElement(tuple),
-                              op::Copy(op::GetTupleElement(tuple))));
+  EXPECT_THAT(copy, GmockMatch(m::Tuple(
+                        m::GetTupleElement(m::Op().Is(tuple)),
+                        m::Copy(m::GetTupleElement(m::Op().Is(tuple))))));
 }
 
 TEST_F(HloComputationTest, CycleDetection) {
@@ -443,13 +448,15 @@ TEST_F(HloComputationTest, RemoveInstructionWithDuplicateOperand) {
   auto module = CreateNewVerifiedModule();
   auto computation = module->AddEntryComputation(builder.Build());
   EXPECT_EQ(4, computation->instruction_count());
-  EXPECT_THAT(computation->root_instruction(), op::Negate(constant));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Negate(m::Op().Is(constant))));
   EXPECT_EQ(negate, computation->root_instruction());
 
   ASSERT_IS_OK(computation->RemoveInstructionAndUnusedOperands(dead_add));
 
   EXPECT_EQ(2, computation->instruction_count());
-  EXPECT_THAT(computation->root_instruction(), op::Negate(constant));
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Negate(m::Op().Is(constant))));
   EXPECT_EQ(negate, computation->root_instruction());
 }
 
diff --git a/tensorflow/compiler/xla/service/layout_assignment_test.cc b/tensorflow/compiler/xla/service/layout_assignment_test.cc
index 311bd78905..5c661bfacb 100644
--- a/tensorflow/compiler/xla/service/layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment_test.cc
@@ -27,7 +27,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/computation_layout.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index 432aa1ea0b..c35f72699b 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -2232,6 +2232,7 @@ inline auto WithOperands(Matcher&& m, int64 operand_num, FirstArg&& first_arg,
 
 // We could implement all ops as "variadic" ops, but it would make the
 // already-bad compile errors even worse.
+XLA_VARIADIC_OP_PATTERN(AfterAll);
 XLA_VARIADIC_OP_PATTERN(Concatenate);
 XLA_VARIADIC_OP_PATTERN(CustomCall);
 XLA_VARIADIC_OP_PATTERN(Map)
-- 
GitLab


From 8cd3cbb161f7c3d45a1f6d46537fbfa679bbde1f Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 4 Dec 2018 21:21:00 -0800
Subject: [PATCH 1343/1554] [XLA:TPU] Fix the algebra in ArCrsCombiner to take
 into account the number of replicas, not just the number of spatial
 partitions.

PiperOrigin-RevId: 224097074
---
 .../compiler/xla/service/ar_crs_combiner.cc   | 30 ++++++++-----------
 .../compiler/xla/service/ar_crs_combiner.h    |  6 ++--
 .../xla/service/ar_crs_combiner_test.cc       | 13 ++++----
 3 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index c11452a6fb..24de693822 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -133,7 +133,7 @@ bool ArCrsCombiner::TupleElementsComputeSameValue(
 /* static */
 bool ArCrsCombiner::TestInstructionsComputeSameValue(HloInstruction* i1,
                                                      HloInstruction* i2) {
-  ArCrsCombiner combiner(/*num_spatial_partitions=*/2);
+  ArCrsCombiner combiner(/*num_spatial_partitions=*/2, /*num_replicas=*/1);
   auto module = i1->parent()->parent();
   CHECK_EQ(module, i2->parent()->parent());
   combiner.call_graph_ = CallGraph::Build(module);
@@ -242,26 +242,22 @@ StatusOr<bool> ArCrsCombiner::RewriteGraph() {
       HloInstruction* other_summand = (add->operands()[0] == convert)
                                           ? add->operands()[1]
                                           : add->operands()[0];
-      // Remove the AllReduce and replace the CRS with:
-      // AllReduce - (other_summand * (num_spatial_partitions_ - 1))
+      // Remove the AllReduce and replace the CRS with an all-core AllReduce,
+      // then subtract:
+      // other_summand * num_replicas_ * (num_spatial_partitions_ - 1)
       TF_CHECK_OK(
           all_reduce->ReplaceAllUsesWith(all_reduce->mutable_operand(0)));
       crs->set_all_reduce_id(all_reduce->all_reduce_id());
       auto new_shape = crs->shape();
-      HloInstruction* to_subtract;
-      if (num_spatial_partitions_ == 2) {
-        to_subtract = other_summand;
-      } else {
-        Literal partitions_minus_1_lit = Literal(new_shape);
-        partitions_minus_1_lit.PopulateWithValue<float>(
-            num_spatial_partitions_ - 1);
-        auto partitions_minus_1_const = parent_computation->AddInstruction(
-            HloInstruction::CreateConstant(partitions_minus_1_lit.Clone()));
-        to_subtract =
-            parent_computation->AddInstruction(HloInstruction::CreateBinary(
-                new_shape, HloOpcode::kMultiply, other_summand,
-                partitions_minus_1_const));
-      }
+      Literal lit(new_shape);
+      lit.PopulateWithValue<float>(num_replicas_ *
+                                   (num_spatial_partitions_ - 1));
+      auto partitions_minus_1_const = parent_computation->AddInstruction(
+          HloInstruction::CreateConstant(lit.Clone()));
+      auto to_subtract =
+          parent_computation->AddInstruction(HloInstruction::CreateBinary(
+              new_shape, HloOpcode::kMultiply, other_summand,
+              partitions_minus_1_const));
       auto sub =
           parent_computation->AddInstruction(HloInstruction::CreateBinary(
               new_shape, HloOpcode::kSubtract, crs, to_subtract));
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.h b/tensorflow/compiler/xla/service/ar_crs_combiner.h
index f6a7ef76ec..4abdb1f57d 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.h
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.h
@@ -30,8 +30,9 @@ namespace xla {
 // fully utilizes the interconnect bandwidth.
 class ArCrsCombiner : public HloModulePass {
  public:
-  ArCrsCombiner(int num_spatial_partitions)
-      : num_spatial_partitions_(num_spatial_partitions) {}
+  ArCrsCombiner(int num_spatial_partitions, int num_replicas)
+      : num_spatial_partitions_(num_spatial_partitions),
+        num_replicas_(num_replicas) {}
   absl::string_view name() const override { return "ar-crs-combiner"; }
   StatusOr<bool> Run(HloModule* module) override;
 
@@ -76,6 +77,7 @@ class ArCrsCombiner : public HloModulePass {
   StatusOr<bool> RewriteGraph();
 
   int num_spatial_partitions_;
+  int num_replicas_;
 
   // Map from all-reduce ids to the all reduce instructions.
   absl::flat_hash_map<int64, std::vector<HloInstruction*>> all_reduce_map_;
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index 9d5eaf63cc..23d9aa9eb3 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -317,12 +317,15 @@ ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
   auto crs_before =
       module->entry_computation()->root_instruction()->operands()[0];
   auto replica_groups_before = crs_before->replica_groups();
-  ArCrsCombiner combiner(2);
+  ArCrsCombiner combiner(2, 1);
   auto changed = combiner.Run(module.get()).ValueOrDie();
   EXPECT_TRUE(changed);
-  EXPECT_THAT(module->entry_computation()->root_instruction(),
-              op::Tuple(op::Subtract(op::CrossReplicaSum(), op::Constant()),
-                        op::Subtract(op::CrossReplicaSum(), op::Constant())));
+  EXPECT_THAT(
+      module->entry_computation()->root_instruction(),
+      op::Tuple(op::Subtract(op::CrossReplicaSum(),
+                             op::Multiply(op::Constant(), op::Constant())),
+                op::Subtract(op::CrossReplicaSum(),
+                             op::Multiply(op::Constant(), op::Constant()))));
   auto sub = module->entry_computation()->root_instruction()->operands()[0];
   auto crs_after = sub->operands()[0];
   auto replica_groups_after = crs_after->replica_groups();
@@ -406,7 +409,7 @@ ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
 
   TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
                           ParseAndReturnVerifiedModule(module_str));
-  ArCrsCombiner combiner(2);
+  ArCrsCombiner combiner(2, 1);
   auto changed = combiner.Run(module.get()).ValueOrDie();
   EXPECT_FALSE(changed);
 }
-- 
GitLab


From 2114ed7447c8239531aae073336140c8ca861303 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Tue, 4 Dec 2018 21:26:16 -0800
Subject: [PATCH 1344/1554] Expose tf.distribute.MirroredStrategy.

RELNOTES:
tf.distribute.MirroredStrategy is now part of core TensorFlow. It is an updated version of tf.contrib.distribute.MirroredStrategy. It can be used for distributing training to multiple GPUs in Keras, Estimator, or directly.
It is an implementation of the tf.distribute.Strategy API, which is also now in core TensorFlow. This API has been updated based on the public design review https://github.com/tensorflow/community/blob/master/rfcs/20181016-replicator.md.
Other distribution strategies will be moved to core TF in future releases.
PiperOrigin-RevId: 224097441
---
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/__init__.py                 |   1 +
 tensorflow/python/distribute/BUILD            |  12 ++
 tensorflow/python/distribute/__init__.py      |  25 ++++
 .../python/distribute/mirrored_strategy.py    |   2 +
 ...orflow.distribute.-mirrored-strategy.pbtxt | 138 ++++++++++++++++++
 .../api/golden/v1/tensorflow.distribute.pbtxt |   4 +
 ...orflow.distribute.-mirrored-strategy.pbtxt | 138 ++++++++++++++++++
 .../api/golden/v2/tensorflow.distribute.pbtxt |   4 +
 9 files changed, 325 insertions(+)
 create mode 100644 tensorflow/python/distribute/__init__.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 1384a2ea73..dccbc40143 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -145,6 +145,7 @@ py_library(
         "//tensorflow/lite/python:lite",
         "//tensorflow/python/compat",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute",
         "//tensorflow/python/distribute:estimator_training",
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/feature_column:feature_column_py",
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 547043030b..62a6e9f6d4 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -78,6 +78,7 @@ from tensorflow.python.ops import initializers_ns as initializers
 
 # Bring in subpackages.
 from tensorflow.python import data
+from tensorflow.python import distribute
 from tensorflow.python import keras
 from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.layers import layers
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 2d9a1764db..887c61cb8f 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -103,6 +103,18 @@ cuda_py_test(
     ],
 )
 
+py_library(
+    name = "distribute",
+    srcs = [
+        "__init__.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":distribute_lib",
+        ":mirrored_strategy",
+    ],
+)
+
 py_library(
     name = "distribute_lib",
     srcs = [
diff --git a/tensorflow/python/distribute/__init__.py b/tensorflow/python/distribute/__init__.py
new file mode 100644
index 0000000000..4ff912ae10
--- /dev/null
+++ b/tensorflow/python/distribute/__init__.py
@@ -0,0 +1,25 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Distribution Strategy library."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=unused-import
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import mirrored_strategy
+# pylint: enable=unused-import
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 0775920a7d..7589720724 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -43,6 +43,7 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.training import coordinator
 from tensorflow.python.util import nest
+from tensorflow.python.util.tf_export import tf_export
 
 
 # TODO(josh11b): Replace asserts in this file with if ...: raise ...
@@ -422,6 +423,7 @@ def all_local_devices(num_gpus=None):
           ("/device:CPU:0",))
 
 
+@tf_export("distribute.MirroredStrategy")
 class MirroredStrategy(distribute_lib.DistributionStrategy):
   """Mirrors vars to distribute across multiple devices and machines.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
new file mode 100644
index 0000000000..a613e2d3d1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -0,0 +1,138 @@
+path: "tensorflow.distribute.MirroredStrategy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.mirrored_strategy.MirroredStrategy\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "between_graph"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "extended"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "parameter_devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "require_static_shapes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_checkpoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_init"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_save_summary"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "worker_devices"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'devices\', \'cross_device_ops\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "call_for_each_replica"
+    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "configure"
+    argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "distribute_dataset"
+    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "group"
+    argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_dataset_iterator"
+    argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_input_fn_iterator"
+    argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
+  }
+  member_method {
+    name: "non_slot_devices"
+    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "read_var"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_steps_on_dataset"
+    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
+  }
+  member_method {
+    name: "scope"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "unwrap"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update"
+    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "update_config_proto"
+    argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_non_slot"
+    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "value_container"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
index 4d833b54ba..b0dd73ca1d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.pbtxt
@@ -8,6 +8,10 @@ tf_module {
     name: "InputReplicationMode"
     mtype: "<class \'enum.EnumMeta\'>"
   }
+  member {
+    name: "MirroredStrategy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ReduceOp"
     mtype: "<class \'enum.EnumMeta\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
new file mode 100644
index 0000000000..a613e2d3d1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -0,0 +1,138 @@
+path: "tensorflow.distribute.MirroredStrategy"
+tf_class {
+  is_instance: "<class \'tensorflow.python.distribute.mirrored_strategy.MirroredStrategy\'>"
+  is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "between_graph"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "extended"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "num_replicas_in_sync"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "parameter_devices"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "require_static_shapes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_checkpoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_init"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "should_save_summary"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "worker_devices"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'devices\', \'cross_device_ops\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "batch_reduce"
+    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "broadcast"
+    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "call_for_each_replica"
+    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "colocate_vars_with"
+    argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "configure"
+    argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "distribute_dataset"
+    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "experimental_initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "finalize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "group"
+    argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "initialize"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_dataset_iterator"
+    argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "make_input_fn_iterator"
+    argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
+  }
+  member_method {
+    name: "non_slot_devices"
+    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "read_var"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_steps_on_dataset"
+    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
+  }
+  member_method {
+    name: "scope"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "unwrap"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update"
+    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "update_config_proto"
+    argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_non_slot"
+    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "value_container"
+    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
index 4d833b54ba..b0dd73ca1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.pbtxt
@@ -8,6 +8,10 @@ tf_module {
     name: "InputReplicationMode"
     mtype: "<class \'enum.EnumMeta\'>"
   }
+  member {
+    name: "MirroredStrategy"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ReduceOp"
     mtype: "<class \'enum.EnumMeta\'>"
-- 
GitLab


From 7c7ed2b64c6eeee3f92a215e872046c95d722020 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 4 Dec 2018 23:17:30 -0800
Subject: [PATCH 1345/1554] [XLA] Print out shapes' layouts in local_client's
 "shape/layout doesn't match" error.

Previously we'd say "shape/layout doesn't match" and then print out the shapes
without their layouts!

PiperOrigin-RevId: 224105237
---
 tensorflow/compiler/xla/client/local_client.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index aaa5d6989e..049cd15738 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -71,9 +71,9 @@ Status LocalExecutable::ValidateExecutionOptions(
           "parameter "
           "%d: want %s, got %s",
           i,
-          ShapeUtil::HumanString(
+          ShapeUtil::HumanStringWithLayout(
               computation_layout.parameter_layout(i).shape()),
-          ShapeUtil::HumanString(arguments[i]->on_host_shape()));
+          ShapeUtil::HumanStringWithLayout(arguments[i]->on_host_shape()));
     }
   }
 
-- 
GitLab


From 76c4e6e64139c3dbf6e3e02afc031bd1bc3f5650 Mon Sep 17 00:00:00 2001
From: Thomas Joerg <tjoerg@google.com>
Date: Wed, 5 Dec 2018 00:41:28 -0800
Subject: [PATCH 1346/1554] BEGIN_PUBLIC [XLA:GPU] Rollback of: Convert the
 reduction implementation to use tiling scheme.

This change introduced a performance regression. Hence, rolling it back for now.
END_PUBLIC

Automated rollback of commit 09a7d525c8556f6fd90c70f656d131559cf30fd4

PiperOrigin-RevId: 224112008
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 1749 ++++++++++-------
 .../xla/service/gpu/ir_emitter_unnested.h     |  107 +-
 .../xla/service/llvm_ir/kernel_tiling.cc      |    2 +-
 .../xla/service/llvm_ir/kernel_tiling.h       |   17 +-
 4 files changed, 1167 insertions(+), 708 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 38b8923173..fb040aff30 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h"
 
 #include "absl/algorithm/container.h"
+#include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/optional.h"
@@ -547,7 +548,91 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
           // TODO(b/112040122): Support variadic reduce.
           return Unimplemented("Variadic reduce is not supported on GPU");
         }
-        return EmitReductionToVector(fusion);
+        VLOG(3) << "Emitting fused reduction to vector: " << fusion->ToString();
+        std::vector<std::unique_ptr<Thunk>> thunks;
+        absl::Span<HloInstruction* const> output_instructions =
+            root->opcode() == HloOpcode::kTuple
+                ? root->operands()
+                : absl::Span<HloInstruction* const>(&root, 1);
+
+        // For multi-output fusion emit an initializer for each tuple element.
+        // Otherwise it's sufficient to just initialize the single output.
+        HloInstruction* first_reduce = nullptr;
+        for (int i = 0, e = output_instructions.size(); i != e; ++i) {
+          if (output_instructions[i]->opcode() == HloOpcode::kReduce) {
+            TF_ASSIGN_OR_RETURN(
+                std::unique_ptr<Thunk> initializer_thunk,
+                BuildInitializerThunk(fusion, output_instructions[i] == root
+                                                  ? ShapeIndex()
+                                                  : ShapeIndex({i})));
+            thunks.push_back(std::move(initializer_thunk));
+            first_reduce =
+                first_reduce == nullptr ? output_instructions[i] : first_reduce;
+          }
+        }
+        CHECK(first_reduce != nullptr);
+        std::unique_ptr<KernelThunk> kernel_thunk =
+            BuildKernelThunk(fusion, /*implements_whole_instruction=*/false);
+        GpuElementalIrEmitter elemental_emitter(
+            hlo_module_config_, ir_emitter_context_->llvm_module(), &b_,
+            GetNestedComputer());
+        FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(fusion),
+                                     &elemental_emitter);
+        TF_RETURN_IF_ERROR(root->Accept(&fused_emitter));
+
+        // For multi-output fusion CHECK the constraints and feed all the
+        // reduces into a single loop code generator. Single-output reduce
+        // fusion is a special case of that.
+        InlinedVector<llvm_ir::ElementGenerator, 1> input_gens;
+        InlinedVector<llvm_ir::ElementGenerator, 1> init_value_gens;
+        std::vector<std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+            extra_output_gens;
+        InlinedVector<HloComputation*, 1> reducers;
+        InlinedVector<ShapeIndex, 1> reduce_output_shapes;
+        for (int i = 0, e = output_instructions.size(); i != e; ++i) {
+          const HloInstruction* inst = output_instructions[i];
+          ShapeIndex output_shape_index;
+          if (root->opcode() == HloOpcode::kTuple) {
+            output_shape_index = {i};
+          }
+          if (inst->opcode() == HloOpcode::kReduce) {
+            CHECK(IsReductionToVector(*inst))
+                << "Only reductions to vector are supported";
+            // Shapes, layouts and dimensions must be the same for all reduces
+            // inside of this fusion.
+            CHECK(ShapeUtil::Equal(first_reduce->shape(), inst->shape()));
+            CHECK(ShapeUtil::Equal(first_reduce->operand(0)->shape(),
+                                   inst->operand(0)->shape()));
+            CHECK(ShapeUtil::Equal(first_reduce->operand(1)->shape(),
+                                   inst->operand(1)->shape()));
+            CHECK(first_reduce->dimensions() == inst->dimensions());
+            input_gens.push_back(fused_emitter.GetGenerator(inst->operand(0)));
+            init_value_gens.push_back(
+                fused_emitter.GetGenerator(inst->operand(1)));
+            reducers.push_back(inst->to_apply());
+            reduce_output_shapes.push_back(std::move(output_shape_index));
+          } else {
+            // For extra outputs we can relax shape equality to allow different
+            // types (with the same number of elements). Layouts still have to
+            // match.
+            CHECK(ShapeUtil::CompatibleIgnoringElementType(
+                first_reduce->operand(0)->shape(), inst->shape()));
+            CHECK(LayoutUtil::Equal(first_reduce->operand(0)->shape().layout(),
+                                    inst->shape().layout()));
+            extra_output_gens.emplace_back(fused_emitter.GetGenerator(inst),
+                                           std::move(output_shape_index));
+          }
+        }
+        const Shape& input_shape = first_reduce->operand(0)->shape();
+        TF_CHECK_OK(EmitReductionToVector(
+            kernel_thunk.get(), first_reduce, input_shape, input_gens,
+            init_value_gens, first_reduce->dimensions(), reducers,
+            reduce_output_shapes, extra_output_gens));
+        thunks.push_back(std::move(kernel_thunk));
+        std::unique_ptr<SequentialThunk> sequential_thunk =
+            absl::make_unique<SequentialThunk>(std::move(thunks), fusion);
+        AddThunkToThunkSequence(std::move(sequential_thunk));
+        return Status::OK();
       }
       default:
         LOG(FATAL) << "Bad opcode for input fusion: "
@@ -617,12 +702,13 @@ Status IrEmitterUnnested::HandleCopy(HloInstruction* copy) {
 }
 
 Status IrEmitterUnnested::EmitExtraOutputsForReduce(
-    const HloInstruction* unnested_hlo, const IrArray::Index& index,
+    const HloInstruction* reduce, const IrArray::Index& index,
     absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
         extra_output_gens) {
   for (int i = 0; i != extra_output_gens.size(); ++i) {
+    const HloInstruction* output = reduce->parent()->FusionInstruction();
     llvm::Value* extra_output_address =
-        GetIrArray(*unnested_hlo, *unnested_hlo, extra_output_gens[i].second)
+        GetIrArray(*output, *output, extra_output_gens[i].second)
             .EmitArrayElementAddress(index, &b_,
                                      "extra_output_element_address");
     TF_ASSIGN_OR_RETURN(llvm::Value* const extra_output_ir_value,
@@ -632,13 +718,984 @@ Status IrEmitterUnnested::EmitExtraOutputsForReduce(
   return Status::OK();
 }
 
+Status IrEmitterUnnested::EmitReductionToScalar(
+    KernelThunk* kernel_thunk, HloInstruction* reduce, const Shape& input_shape,
+    absl::Span<const llvm_ir::ElementGenerator> input_gens,
+    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+    absl::Span<HloComputation* const> reducers,
+    absl::Span<const ShapeIndex> reduce_output_shapes,
+    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+        extra_output_gens) {
+  // Number of elements processed by a single thread.
+  constexpr int64 kTileSize = 16;
+  int64 num_elems = ShapeUtil::ElementsIn(input_shape);
+
+  // Round up the number of tiles to a multiple of the warp size.  This is
+  // necessary for correctness.  We launch one thread per tile, and if the
+  // number of threads isn't a multiple of the number of the warp size, our
+  // shuffles will read from inactive threads, producing undefined values.
+  int64 num_tiles =
+      RoundUpToNearest(CeilOfRatio(num_elems, kTileSize), kWarpSize);
+
+  Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout(
+      reduce->shape().element_type(), {num_tiles}, {0});
+  LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
+      tiled_input_shape, ir_emitter_context_->device_description());
+
+  llvm::Type* index_ty =
+      GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_);
+
+  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
+    return llvm::ConstantInt::get(index_ty, c);
+  };
+
+  // Check whether every thread will process a full tile's worth of elements
+  // without reading outside the bounds of the input.  If this is true, we can
+  // skip some bounds checks in the final algorithm.
+  bool all_threads_in_bounds = num_tiles * kTileSize == num_elems;
+
+  // __global__ void full_reduce_kernel() {
+  //   x_in_tiles = threadIdx.x + blockIdx.x * blockDim.x;
+  //   x = x_in_tiles * kTileSize;
+  //
+  //   partial_result = init_value;
+  //   if (all_threads_in_bounds || x + kTileSize <= num_elems) {
+  //     for (i = 0; i < kTileSize; ++i) {
+  //       partial_result = Reducer(partial_result, input[x + i]);
+  //     }
+  //   } else {
+  //     for (i = 0; i < kTileSize; ++i) {
+  //       if (x + i < num_elems) {
+  //         partial_result = Reducer(partial_result, input[x + i]);
+  //       }
+  //     }
+  //   }
+  //   for (i = warpSize / 2; i > 0; i /= 2) {
+  //     partial_result = Reducer(partial_result,
+  //                              __shfl_down(partial_result, i));
+  //   }
+  //   if (lane_id == 0) {
+  //     AtomicReducer(&output[y], partial_result);
+  //   }
+  // }
+  //
+  // // Choose num_blocks and threads_per_block such that:
+  // //
+  // //   num_blocks * threads_per_block =
+  // //     RoundUpToNextMultipleOf(Ceil(num_elems / kTileSize), warpSize),
+  // //
+  // // and threads_per_block is a multiple of warpSize.
+  // reduce_kernel  //
+  auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status {
+    const int num_reduces = reducers.size();
+    llvm::Type* element_ir_type =
+        llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
+    std::vector<llvm::Value*> partial_reduction_result_addresses;
+    for (int i = 0; i != num_reduces; ++i) {
+      llvm::Value* partial_reduction_result_address =
+          Alloca(element_ir_type, /*ArraySize=*/nullptr,
+                 "partial_reduction_result." + llvm::Twine(i));
+      TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
+                          init_value_gens[i](IrArray::Index(index_ty)));
+      Store(init_ir_value, partial_reduction_result_address);
+      partial_reduction_result_addresses.push_back(
+          partial_reduction_result_address);
+    }
+
+    llvm::Value* x_in_tiles = tile_index[0];
+    x_in_tiles = ZExtOrTrunc(x_in_tiles, index_ty);
+
+    // Emit an inner for-loop that reduces the elements in the tile.
+    auto emit_tile_element_loop = [=](bool tile_in_bounds) -> Status {
+      std::unique_ptr<llvm_ir::ForLoop> tile_element_loop =
+          llvm_ir::ForLoop::EmitForLoop(
+              "element_id_in_tile", index_typed_constant(0),
+              index_typed_constant(kTileSize), index_typed_constant(1), &b_);
+
+      // Emit the body of the partial reduction loop.
+      llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
+                                     &b_);
+      llvm::Value* x =
+          NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileSize)),
+                 tile_element_loop->GetIndVarValue());
+      // Unless we know the tile is entirely in bounds, we have to emit a
+      // x-in-bounds check before reading from the input.
+      if (!tile_in_bounds) {
+        llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
+            ICmpULT(x, index_typed_constant(num_elems)), "x_in_bounds", &b_);
+
+        // Emit code that reads the input element and accumulates it to
+        // the partial reduction result.
+        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
+      }
+
+      IrArray::Index input_index(
+          /*linear=*/x, input_shape, &b_);
+      llvm::Value* input_address = Alloca(element_ir_type);
+      for (int i = 0; i != num_reduces; ++i) {
+        TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
+                            input_gens[i](input_index));
+        Store(input_ir_value, input_address);
+        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
+            *reducers[i],
+            {partial_reduction_result_addresses[i], input_address},
+            partial_reduction_result_addresses[i]));
+      }
+      return EmitExtraOutputsForReduce(reduce, input_index, extra_output_gens);
+    };
+
+    // x_end = kTileSize + x_in_tiles * kTileSize, i.e., the location that's
+    // immediately beyond the tile.
+    llvm::Value* x_end =
+        NSWAdd(index_typed_constant(kTileSize),
+               NSWMul(x_in_tiles, index_typed_constant(kTileSize)));
+    // The tile is entirely in bound if all_threads_in_bounds or
+    // x_end <= num_elems.
+    llvm::Value* tile_in_bounds =
+        Or(ICmpULE(x_end, index_typed_constant(num_elems)),
+           b_.getInt1(all_threads_in_bounds));
+    llvm_ir::LlvmIfData if_tile_in_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_bounds, "tile_in_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.true_block, &b_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/true));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.false_block, &b_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/false));
+
+    // After the if-then-else statement on tile_in_bounds, emit calls to
+    // shfl_down that accumulate the partial reduction results of all threads
+    // from the warp.
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.after_block, &b_);
+    int bit_width = llvm_ir::GetSizeInBits(element_ir_type);
+    // bitcast cannot be applied to aggregate types (even packed ones), so we
+    // instead bitcast addresses of load/store to intN* of the same bit-width.
+    llvm::Type* shuffle_ir_type = element_ir_type->isStructTy()
+                                      ? b_.getIntNTy(bit_width)
+                                      : element_ir_type;
+    for (int shuffle_distance = kWarpSize / 2; shuffle_distance >= 1;
+         shuffle_distance /= 2) {
+      llvm::Value* result_from_other_lane =
+          Alloca(element_ir_type, nullptr, "result_from_other_lane");
+      for (int i = 0; i != num_reduces; ++i) {
+        llvm::Value* partial_reduction_result =
+            Load(BitCast(partial_reduction_result_addresses[i],
+                         shuffle_ir_type->getPointerTo()),
+                 "partial_reduction_result");
+        CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0)
+            << "Requires block size a multiple of the warp size, otherwise we "
+               "will read undefined elements.";
+        Store(EmitFullWarpShuffleDown(partial_reduction_result,
+                                      b_.getInt32(shuffle_distance), &b_),
+              BitCast(result_from_other_lane, shuffle_ir_type->getPointerTo()));
+        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
+            *reducers[i],
+            {partial_reduction_result_addresses[i], result_from_other_lane},
+            partial_reduction_result_addresses[i]));
+      }
+    }
+
+    const HloInstruction* output =
+        reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
+
+    // Emit an atomic operation that accumulates the partial reduction result of
+    // lane 0 (which holds the partially accumulated result for its warp) to the
+    // output element.
+    llvm::Value* lane_id =
+        URem(x_in_tiles, index_typed_constant(kWarpSize), "lane_id");
+    llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
+        ICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
+
+    for (int i = 0; i != num_reduces; ++i) {
+      llvm::Value* output_address =
+          GetIrArray(*output, *output, reduce_output_shapes[i])
+              .EmitArrayElementAddress(
+                  IrArray::Index(
+                      /*linear=*/b_.getInt64(0),
+                      ShapeUtil::GetSubshape(output->shape(),
+                                             reduce_output_shapes[i]),
+                      &b_),
+                  &b_, "output_element_address");
+      TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
+          *reducers[i], output_address, partial_reduction_result_addresses[i]));
+    }
+    return Status::OK();
+  };
+
+  // Emit a parallel loop that iterates through all input tiles, one per thread.
+  UpdateLaunchDimensions(launch_dimensions, kernel_thunk,
+                         ir_emitter_context_->llvm_module());
+  return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
+                             launch_dimensions, &b_)
+      .EmitLoop(IrName(reduce), index_ty);
+}
+
+Status IrEmitterUnnested::EmitColumnReduction(
+    KernelThunk* kernel_thunk, int64 height, int64 width,
+    HloInstruction* reduce, const Shape& input_shape,
+    absl::Span<const llvm_ir::ElementGenerator> input_gens,
+    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+    absl::Span<HloComputation* const> reducers,
+    absl::Span<const ShapeIndex> reduce_output_shapes,
+    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+        extra_output_gens) {
+  // Divide the input matrix into tiles of size KxL. For example, when the
+  // input matrix is 4x4, K=2, and L=1 the tiled matrix looks like
+  //
+  //   0123
+  //   0123
+  //   4567
+  //   4567  // Numbers indicate tile IDs.
+  //
+  // Each tile is first partially reduced to a scalar by a thread, and then the
+  // scalar is accumulated to the output vector using atomic operations.
+  //
+  // We choose 128 as the tile size based on empirical evidence. It's big enough
+  // to reduce the amount of atomic adds in the end, maximizing the memory
+  // bandwidth. A tile width of 2 allows for high memory bandwidth utilization
+  // on 16b input data.
+  constexpr int64 kTileHeight = 128;
+  constexpr int64 kTileWidth = 2;
+
+  // If the height is not a multiple of kTileHeight, we pad the bottom of the
+  // input matrix.
+  const int64 height_in_tiles = CeilOfRatio(height, kTileHeight);
+  // If width is not a multiple of kTileWidth the rightmost thread will process
+  // fewer input elements.
+  const int64 width_in_tiles = CeilOfRatio(width, kTileWidth);
+  Shape tiled_input_shape =
+      ShapeUtil::MakeShapeWithLayout(reduce->shape().element_type(),
+                                     {height_in_tiles, width_in_tiles}, {1, 0});
+  LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
+      tiled_input_shape, ir_emitter_context_->device_description());
+
+  // TODO(b/110211620): Convert to use i32 index_type when it is possible.
+  llvm::Type* index_ty = b_.getInt64Ty();
+
+  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
+    return llvm::ConstantInt::get(index_ty, c);
+  };
+
+  // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
+  //      linear_index < height_in_tiles * width_in_tiles;
+  //      linear_index += blockDim.x * gridDim.x) {
+  //   y_in_tiles = linear_index / width_in_tiles;
+  //   x_in_tiles = linear_index % width_in_tiles;
+  //
+  //   partial_results[kTileWidth] = init_values;
+  //   tile_in_y_bounds = height % kTileHeight == 0 ||
+  //       y_in_tiles * kTileHeight + kTileHeight <= height;
+  //   tile_in_x_bounds = width % kTileWidth == 0 ||
+  //       x_in_tiles * kTileWidth + kTileWidth <= width;
+  //   // The implementation handles y and x bound checks separately.
+  //   if (tile_in_y_bounds && tile_in_x_bounds) {
+  //     for (y_offset : range(kTileHeight)) {
+  //       y = y_in_tiles * kTileHeight + y_offset;
+  //       for (x_offset : range(kTileWidth)) {
+  //         x = x_in_tiles * kTileWidth + x_offset;
+  //         partial_result = Reducer(partial_result[x_offset], input[y][x]);
+  //       }
+  //     }
+  //   } else {
+  //     for (y_offset : range(kTileHeight)) {
+  //       y = y_in_tiles * kTileHeight + y_offset;
+  //       for (y_offset : range(kTileHeight)) {
+  //         x = x_in_tiles * kTileWidth + x_offset;
+  //         if (y < height && x < width) {
+  //           partial_result = Reducer(partial_result, input[y][x]);
+  //         }
+  //       }
+  //     }
+  //   }
+  //   for (x_offset : range(kTileWidth)) {
+  //     AtomicReducer(&output[x + x_offset], partial_result[x_offset]);
+  //   }
+  // }
+  auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status {
+    const int num_reduces = reducers.size();
+    // Emit the loop body that reduces one tile.
+    llvm::Type* element_ir_type =
+        llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
+    std::vector<llvm::Value*> partial_reduction_result_addresses;
+    for (int i = 0; i != num_reduces; ++i) {
+      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
+        llvm::Value* partial_reduction_result_address =
+            Alloca(element_ir_type, /*ArraySize=*/nullptr,
+                   "partial_reduction_result." +
+                       llvm::Twine(i * kTileWidth + x_offset));
+        TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
+                            init_value_gens[i](IrArray::Index(index_ty)));
+        Store(init_ir_value, partial_reduction_result_address);
+        partial_reduction_result_addresses.push_back(
+            partial_reduction_result_address);
+      }
+    }
+
+    // Emit an inner for-loop that partially reduces the elements in the given
+    // tile.
+    llvm::Value* y_in_tiles = tile_index[0];
+    llvm::Value* x_in_tiles = tile_index[1];
+
+    y_in_tiles = ZExtOrTrunc(y_in_tiles, index_ty);
+    x_in_tiles = ZExtOrTrunc(x_in_tiles, index_ty);
+
+    auto emit_tile_element_loop = [=](bool tile_in_y_bounds,
+                                      bool tile_in_x_bounds) -> Status {
+      std::unique_ptr<llvm_ir::ForLoop> tile_element_loop =
+          llvm_ir::ForLoop::EmitForLoop(
+              "element_id_in_tile", index_typed_constant(0),
+              index_typed_constant(kTileHeight), index_typed_constant(1), &b_);
+
+      // Emit the body of the partial reduction loop.
+      llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
+                                     &b_);
+      llvm::Value* y =
+          NSWAdd(NSWMul(y_in_tiles, index_typed_constant(kTileHeight)),
+                 tile_element_loop->GetIndVarValue());
+
+      // Unless we know that y is in bounds, we have to emit a check before
+      // reading from the input.
+      if (!tile_in_y_bounds) {
+        llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
+            ICmpULT(y, index_typed_constant(height)), "y_in_bounds", &b_);
+
+        // Emit code that reads the input element and accumulates it to
+        // the partial reduction result.
+        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
+      }
+      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
+        llvm::Value* x =
+            NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileWidth)),
+                   index_typed_constant(x_offset));
+        // Unless we know that x is in bounds, we have to emit a check before
+        // reading from the input.
+        if (!tile_in_x_bounds) {
+          llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
+              ICmpULT(x, index_typed_constant(width)), "x_in_bounds", &b_);
+          llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
+        }
+        llvm::Value* input_address = Alloca(element_ir_type);
+        // {y,x} is an index to input_matrix_shape [height,width]. We need to
+        // convert that to an index to input_shape (the shape of the operand of
+        // "reduce"). This conversion is composed of a transposition from
+        // input_shape to normalized_input_shape and a reshape from
+        // normalized_input_shape to input_matrix_shape.
+        const Shape normalized_input_shape =
+            ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
+                input_shape);
+        auto input_shape_min2maj = LayoutUtil::MinorToMajor(input_shape);
+        const std::vector<int64> transpose_dimension_mapping(
+            input_shape_min2maj.rbegin(), input_shape_min2maj.rend());
+
+        const Shape input_matrix_shape =
+            ShapeUtil::MakeShapeWithDescendingLayout(input_shape.element_type(),
+                                                     {height, width});
+        const IrArray::Index input_matrix_index({y, x}, input_matrix_shape,
+                                                &b_);
+        const IrArray::Index input_index =
+            input_matrix_index
+                .SourceIndexOfReshape(input_matrix_shape,
+                                      normalized_input_shape, &b_)
+                .SourceIndexOfTranspose(normalized_input_shape, input_shape,
+                                        transpose_dimension_mapping, &b_);
+        for (int i = 0; i != num_reduces; ++i) {
+          TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
+                              input_gens[i](input_index));
+          Store(input_ir_value, input_address);
+          TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
+              *reducers[i],
+              {partial_reduction_result_addresses[i * kTileWidth + x_offset],
+               input_address},
+              partial_reduction_result_addresses[i * kTileWidth + x_offset]));
+          TF_RETURN_IF_ERROR(EmitExtraOutputsForReduce(reduce, input_index,
+                                                       extra_output_gens));
+        }
+      }
+      return Status::OK();
+    };
+
+    // y_end = kTileHeight + y_in_tiles * kTileHeight, i.e., the y location
+    // that's immediately beyond the tile.
+    llvm::Value* y_end =
+        NSWAdd(index_typed_constant(kTileHeight),
+               NSWMul(y_in_tiles, index_typed_constant(kTileHeight)));
+    // x_end = kTileWidth + x_in_tiles * kTileWidth, i.e., the x location
+    // that's immediately beyond the tile.
+    llvm::Value* x_end =
+        NSWAdd(index_typed_constant(kTileWidth),
+               NSWMul(x_in_tiles, index_typed_constant(kTileWidth)));
+    llvm::Value* tile_in_y_bounds =
+        Or(ICmpULE(y_end, index_typed_constant(height)),
+           b_.getInt1(height % kTileHeight == 0));
+    llvm::Value* tile_in_x_bounds =
+        Or(ICmpULE(x_end, index_typed_constant(width)),
+           b_.getInt1(width % kTileWidth == 0));
+    // The tile is in y bounds if "height" is a multiple of kTileHeight or
+    // y_end <= height.
+    llvm_ir::LlvmIfData if_tile_in_y_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_y_bounds, "tile_in_y_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.true_block, &b_);
+    // The tile is in x bounds if "width" is a multiple of kTileWidth or
+    // x_end <= width.
+    llvm_ir::LlvmIfData if_tile_in_x_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
+                                              /*tile_in_x_bounds=*/true));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
+                                              /*tile_in_x_bounds=*/false));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.false_block, &b_);
+    if_tile_in_x_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
+                                              /*tile_in_x_bounds=*/true));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
+                                              /*tile_in_x_bounds=*/false));
+
+    // After the nested if-then-else statement on tile_in_y_bounds and
+    // tile_in_x_bounds, emit atomic operations to accumulate the partial
+    // reduction result to the output element.
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.after_block, &b_);
+    const HloInstruction* output =
+        reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
+    for (int i = 0; i != num_reduces; ++i) {
+      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
+        llvm::Value* x =
+            NSWAdd(NSWMul(x_in_tiles, index_typed_constant(kTileWidth)),
+                   index_typed_constant(x_offset));
+        llvm::Value* output_address =
+            GetIrArray(*output, *output, reduce_output_shapes[i])
+                .EmitArrayElementAddress(
+                    IrArray::Index(
+                        x,
+                        ShapeUtil::GetSubshape(output->shape(),
+                                               reduce_output_shapes[i]),
+                        &b_),
+                    &b_, "output_element_address");
+        TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
+            *reducers[i], output_address,
+            partial_reduction_result_addresses[i * kTileWidth + x_offset]));
+      }
+    }
+    return Status::OK();
+  };
+
+  // Emit a parallel loop that iterate through all input tiles.
+  UpdateLaunchDimensions(launch_dimensions, kernel_thunk,
+                         ir_emitter_context_->llvm_module());
+  return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
+                             launch_dimensions, &b_)
+      .EmitLoop(IrName(reduce), index_ty);
+}
+
+static std::pair<int64, int64> ComputeKernelMappingSchemeForReduction(
+    int64 depth, int64 width, int64 kWarpSize) {
+  constexpr int64 kTargetNumElementsPerThread = 64;
+  int64 x_tile_size = kTargetNumElementsPerThread;
+  int64 z_tile_size = 1;
+
+  // Only tile along the x dimension with tile size kTargetNumElementsPerThread
+  // if doing so doesn't require a slow version of loop with bound check on each
+  // dimension. A more sophisticated heuristics is to enable tile along the
+  // x dimension with tile size kTargetNumElementsPerThread when either width is
+  // a factor of (kWarpSize * kTargetNumElementsPerThread) or width is big
+  // enough so that only a small fraction of the threads execute the slow
+  // version of loop with bound check.
+  if (width % (kWarpSize * kTargetNumElementsPerThread) != 0) {
+    x_tile_size = 8;
+    z_tile_size = 8;
+    while (depth % z_tile_size != 0) {
+      z_tile_size -= 1;
+    }
+  }
+
+  return std::pair<int64, int64>(x_tile_size, z_tile_size);
+}
+
+Status IrEmitterUnnested::EmitRowReduction(
+    KernelThunk* kernel_thunk, int64 depth, int64 height, int64 width,
+    HloInstruction* reduce, const Shape& input_shape,
+    absl::Span<const llvm_ir::ElementGenerator> input_gens,
+    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+    absl::Span<HloComputation* const> reducers,
+    absl::Span<const ShapeIndex> reduce_output_shapes,
+    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+        extra_output_gens) {
+  // A naive algorithm is:
+  // 1. Divide the x dimension of the input tensor into tiles of size 1x1xX.
+  // 2. Partially reduces each tile to a scalar using one thread.
+  // 3. Accumulates that scalar to the output vector using atomic operations.
+  //
+  // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
+  //      linear_index < depth * height * width_in_tiles;
+  //      linear_index += blockDim.x * gridDim.x) {
+  //   int x_in_tiles = linear_index % width_in_tiles;
+  //   int y = linear_index / width_in_tiles % height;
+  //   int z = linear_index / (height * width_in_tiles);
+  //   float partial_result = 0;
+  //   for (element_id_in_tile : range(x_tile_size)) {
+  //     int x = x_in_tiles * x_tile_size + element_id_in_tile;
+  //     if (x < width)
+  //       partial_result = reducer(partial_result, input[z][y][x]);
+  //   }
+  //   AtomicReducer(&output[y], partial_result);
+  // }
+  //
+  // Four optimizations are performed.
+  //
+  // 1. To coalesce global memory accesses, dilate the tile with a factor of 32
+  // (i.e. the warp size). For example, suppose the width is 8x32=256. Instead
+  // of making each tile consecutive, we let make tile 0 column
+  // [0,32,64,...,224], tile 1 column [1,33,65,...,225], and so on. This ensures
+  // that threads in a warp access consecutive memory in one iteration (i.e.
+  // coalesced). In the above example, the warp that contains thread 0-31
+  // accesses column 0-31 in the first iteration, and 32-63 in the second
+  // iteration, and so on.
+  //
+  // 2. Partially accumulate partial reduced results computed by threads in the
+  // same warp using shfl_down. Using shfl_down is faster than directly using
+  // atomic operations because shfl_down transfers the data between threads
+  // using shared memory and threads in the same warp run in lock step (thus no
+  // extra synchronization needed). See
+  // https://devblogs.nvidia.com/parallelforall/faster-parallel-reductions-kepler/
+  // for details. The downside is, to produce correct results when using
+  // shfl_down, we need to guarantee threads in the same warp work on input
+  // elements with the same y, so the number of tiles in each row must be a
+  // multiple of 32.
+  //
+  // 3. Specialize the case that the entire tile is in bounds. When that is
+  // true, we don't need to emit "if(x<width)" inside the loop on
+  // element_id_in_tile, which makes the code more friendly to optimizations
+  // such as LICM.
+  //
+  // 4. When the width is too small and x_tile_size is less than the target
+  //    number of elements per thread and use a small factor of depth as
+  //    z_tile_size to increase the number of elements calculated by each
+  //    partial sum. This can reduce the needed number of dynamic shfl_down and
+  //    atomic operations.
+  //
+  // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
+  //      linear_index < depth * height * width_in_tiles;
+  //      linear_index += blockDim.x * gridDim.x) {
+  //   int x_in_tiles = linear_index % width_in_tiles;
+  //   int y = linear_index / width_in_tiles % height;
+  //   int z_in_tiles = linear_index / (height * width_in_tiles);
+  //   int warp_id = x_in_tiles / warpSize;
+  //   int lane_id = x_in_tiles % warpSize;
+  //   float partial_result = 0;
+  //   int x = warp_id * kTileSize * warpSize + lane_id;
+  //   if (width % (x_tile_size * warpSize) == 0 ||
+  //       x + (x_tile_size - 1) * warpSize < width) {
+  //     // The entire x_tile is in bounds.
+  //     for (int element_id_in_z_tile = 0; element_id_in_z_tile < z_tile_size;
+  //          ++element_id_in_z_tile) {
+  //       z = z_in_tiles * z_tile_size + element_id_in_z_tile;
+  //       int tx = x;
+  //       for (int element_id_in_x_tile = 0;
+  //            element_id_in_x_tile < x_tile_size;
+  //            ++element_id_in_x_tile, tx += warpSize) {
+  //         partial_result = Reducer(partial_result, input[z][y][tx]);
+  //       }
+  //     }
+  //   } else {
+  //     // The tile is partially in bounds.
+  //     for (int element_id_in_z_tile = 0; element_id_in_z_tile < z_tile_size;
+  //          ++element_id_in_z_tile) {
+  //       z = z_in_tiles * z_tile_size + element_id_in_z_tile;
+  //       int tx = x;
+  //       for (int element_id_in_x_tile = 0; element_id_in_x_tile <
+  //            x_tile_size; ++element_id_in_tile, tx += warpSize) {
+  //         if (tx < width)
+  //           partial_result = Reducer(partial_result, input[z][y][tx]);
+  //       }
+  //     }
+  //   }
+  //   for (shuffle_distance = 16; shuffle_distance > 0; shuffle_distance /= 2)
+  //     partial_result = Reducer(
+  //         partial_result,
+  //         __shfl_down_sync(CUDA_WARP_ALL, partial_result, shuffle_distance));
+  //   if (lane_id == 0)
+  //     AtomicReducer(&output[y], partial_result);
+  // }
+  //
+
+  int64 x_tile_size;
+  int64 z_tile_size;
+  std::tie(x_tile_size, z_tile_size) =
+      ComputeKernelMappingSchemeForReduction(depth, width, kWarpSize);
+
+  // Round the width in tiles up to the nearest multiple of kWarpSize, so that
+  // the use of shfl_down is valid.
+  const int64 width_in_tiles =
+      RoundUpToNearest(CeilOfRatio(width, x_tile_size), kWarpSize);
+  Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout(
+      reduce->shape().element_type(),
+      {depth / z_tile_size, height, width_in_tiles}, {2, 1, 0});
+  LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
+      tiled_input_shape, ir_emitter_context_->device_description());
+  llvm::Type* index_ty =
+      GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_);
+
+  auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
+    return llvm::ConstantInt::get(index_ty, c);
+  };
+
+  auto loop_body_emitter = [=](const IrArray::Index& tile_index) {
+    const int num_reduces = reducers.size();
+    llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType(
+        input_shape.element_type(), ir_emitter_context_->llvm_module());
+    std::vector<llvm::Value*> partial_reduction_result_addresses;
+    for (int i = 0; i != num_reduces; ++i) {
+      llvm::Value* partial_reduction_result_address =
+          Alloca(element_ir_type, /*ArraySize=*/nullptr,
+                 "partial_reduction_result." + llvm::Twine(i));
+      TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
+                          init_value_gens[i](IrArray::Index(index_ty)));
+      Store(init_ir_value, partial_reduction_result_address);
+      partial_reduction_result_addresses.push_back(
+          partial_reduction_result_address);
+    }
+
+    llvm::Value* z_tile = tile_index[0];
+    llvm::Value* y = tile_index[1];
+    llvm::Value* x_tile = tile_index[2];
+
+    x_tile = ZExtOrTrunc(x_tile, index_ty);
+
+    llvm::Value* warp_id =
+        UDiv(x_tile, index_typed_constant(kWarpSize), "warp_id");
+    llvm::Value* lane_id =
+        URem(x_tile, index_typed_constant(kWarpSize), "lane_id");
+
+    // The x-location of the last element in this z-x-tile.
+    // last_x = lane_id + warpSize * (x_tile_size - 1 + warp_id * x_tile_size);
+    llvm::Value* last_x = NSWAdd(
+        lane_id,
+        NSWMul(index_typed_constant(kWarpSize),
+               NSWAdd(index_typed_constant(x_tile_size - 1),
+                      NSWMul(warp_id, index_typed_constant(x_tile_size)))));
+
+    KernelSupportLibrary ksl(
+        &b_,
+        /*unroll_mode=*/xla::llvm_ir::UnrollMode::kFullyUnroll,
+        /*prevent_vectorization=*/false);
+
+    // Emit a for-loop that partially reduces the elements in the given
+    // z-x-tile.
+    auto emit_z_x_tile_element_loop = [&](bool x_tile_in_bounds,
+                                          int64 x_tile_loop_bound) -> Status {
+      auto emit_z_tile_element_loop = [&](llvm::Value* z_indvar) -> Status {
+        llvm::Value* z =
+            NSWAdd(z_indvar, NSWMul(index_typed_constant(z_tile_size), z_tile));
+        TF_RETURN_IF_ERROR(ksl.For(
+            "x_tile",
+            /*start=*/index_typed_constant(0),
+            /*end=*/index_typed_constant(x_tile_loop_bound),
+            /*step=*/1, [&](llvm::Value* x_indvar) -> Status {
+              // x = lane_id +
+              //     warpSize * (element_id_in_x_tile + warp_id * x_tile_size);
+              llvm::Value* x = NSWAdd(
+                  lane_id,
+                  NSWMul(index_typed_constant(kWarpSize),
+                         NSWAdd(x_indvar,
+                                NSWMul(warp_id, llvm::ConstantInt::get(
+                                                    index_ty, x_tile_size)))));
+
+              // Unless we know the x-tile is entirely in bounds, we have to
+              // emit a x-in-bounds check before reading from the input.
+              if (!x_tile_in_bounds) {
+                llvm_ir::LlvmIfData if_x_in_bounds_data =
+                    llvm_ir::EmitIfThenElse(
+                        ICmpULT(x, index_typed_constant(width)), "x_in_bounds",
+                        &b_);
+                // Points b_ to the then-block.
+                llvm_ir::SetToFirstInsertPoint(if_x_in_bounds_data.true_block,
+                                               &b_);
+              }
+
+              // Emit code that reads the input element and accumulates it
+              // to the partial reduction result.
+              llvm::Value* input_address = Alloca(element_ir_type);
+              {
+                // {z,y,x} is an index to input_3d_tensor_shape
+                // [depth,height,width]. We need to convert that to an index
+                // to input_shape (the shape of the operand of "reduce").
+                // This conversion is composed of a transposition from
+                // input_shape to normalized_input_shape and a reshape from
+                // normalized_input_shape to input_3d_tensor_shape.
+                const Shape normalized_input_shape = ShapeUtil::
+                    MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
+                        input_shape);
+                auto input_shape_min2maj =
+                    LayoutUtil::MinorToMajor(input_shape);
+                const std::vector<int64> transpose_dimension_mapping(
+                    input_shape_min2maj.rbegin(), input_shape_min2maj.rend());
+                const Shape input_3d_tensor_shape =
+                    ShapeUtil::MakeShapeWithDescendingLayout(
+                        input_shape.element_type(), {depth, height, width});
+                const IrArray::Index input_3d_tensor_index(
+                    {z, y, x}, input_3d_tensor_shape, &b_);
+                const IrArray::Index input_index =
+                    input_3d_tensor_index
+                        .SourceIndexOfReshape(input_3d_tensor_shape,
+                                              normalized_input_shape, &b_)
+                        .SourceIndexOfTranspose(
+                            normalized_input_shape, input_shape,
+                            transpose_dimension_mapping, &b_);
+
+                for (int i = 0; i != num_reduces; ++i) {
+                  TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
+                                      input_gens[i](input_index));
+                  Store(input_ir_value, input_address);
+                  TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
+                      *reducers[i],
+                      {partial_reduction_result_addresses[i], input_address},
+                      partial_reduction_result_addresses[i]));
+                }
+                return EmitExtraOutputsForReduce(reduce, input_index,
+                                                 extra_output_gens);
+              }
+            }));
+        return Status::OK();
+      };
+
+      return ksl.For("z_tile",
+                     /*start=*/index_typed_constant(0),
+                     /*end=*/index_typed_constant(z_tile_size),
+                     /*step=*/1, emit_z_tile_element_loop);
+    };
+
+    llvm::Value* tile_in_bounds =
+        Or(b_.getInt1(width % (x_tile_size * kWarpSize) == 0),
+           ICmpULT(last_x, index_typed_constant(width)));
+
+    TF_RETURN_IF_ERROR(
+        ksl.If(tile_in_bounds,
+               /*true_block_generator=*/
+               [&]() -> Status {
+                 return emit_z_x_tile_element_loop(/*x_tile_in_bounds=*/true,
+                                                   x_tile_size);
+               },
+               /*false_block_generator=*/
+               [&]() -> Status {
+                 return emit_z_x_tile_element_loop(
+                     /*x_tile_in_bounds=*/false,
+                     CeilOfRatio(width % (x_tile_size * kWarpSize), kWarpSize));
+               }));
+
+    // After accumulating the elements of the z_x_tile, emit calls to
+    // shfl_down that accumulate the partial reduction results of all
+    // threads in a warp.
+    int bit_width = llvm_ir::GetSizeInBits(element_ir_type);
+    // bitcast cannot be applied to aggregate types (even packed ones), so we
+    // instead bitcast addresses of load/store to intN* of the same bit-width.
+    llvm::Type* shuffle_ir_type = element_ir_type->isStructTy()
+                                      ? b_.getIntNTy(bit_width)
+                                      : element_ir_type;
+    for (int shuffle_distance = 16; shuffle_distance >= 1;
+         shuffle_distance /= 2) {
+      llvm::Value* result_from_other_lane =
+          Alloca(element_ir_type, nullptr, "result_from_other_lane");
+      for (int i = 0; i != num_reduces; ++i) {
+        llvm::Value* partial_reduction_result =
+            Load(BitCast(partial_reduction_result_addresses[i],
+                         shuffle_ir_type->getPointerTo()),
+                 "partial_reduction_result");
+        CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0)
+            << "Requires block size a multiple of the warp size, otherwise we "
+               "will read undefined elements.";
+        Store(EmitFullWarpShuffleDown(partial_reduction_result,
+                                      b_.getInt32(shuffle_distance), &b_),
+              BitCast(result_from_other_lane, shuffle_ir_type->getPointerTo()));
+        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
+            *reducers[i],
+            {partial_reduction_result_addresses[i], result_from_other_lane},
+            partial_reduction_result_addresses[i]));
+      }
+    }
+
+    const HloInstruction* output =
+        reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
+
+    // Emit an atomic operation that accumulates the partial reduction result of
+    // lane 0 (which holds the partially accumulated result for its warp) to the
+    // output element.
+    llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
+        ICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
+    for (int i = 0; i != num_reduces; ++i) {
+      llvm::Value* output_address =
+          GetIrArray(*output, *output, reduce_output_shapes[i])
+              .EmitArrayElementAddress(
+                  IrArray::Index(y,
+                                 ShapeUtil::GetSubshape(
+                                     output->shape(), reduce_output_shapes[i]),
+                                 &b_),
+                  &b_, "output_element_address");
+      // We don't need to emit atomic operations if there is only one tile of
+      // results. 'depth' is the z dimension, 'width' is the x dimension.
+      if (z_tile_size >= depth && x_tile_size >= width) {
+        TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
+            *reducers[i],
+            {output_address, partial_reduction_result_addresses[i]},
+            output_address));
+      } else {
+        TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
+            *reducers[i], output_address,
+            partial_reduction_result_addresses[i]));
+      }
+    }
+    return Status::OK();
+  };
+
+  // Emit a parallel loop that iterates through every input tiles.
+  UpdateLaunchDimensions(launch_dimensions, kernel_thunk,
+                         ir_emitter_context_->llvm_module());
+  return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
+                             launch_dimensions, &b_)
+      .EmitLoop(IrName(reduce), index_ty);
+}
+
+// Figures out whether `reduce` is a row or column reduction, and which
+// dimensions to reduce, and calls either `EmitRowReduction` or
+// `EmitColumnReduction` as appropriate.
+// Prerequisite: all the dimensions to keep are contiguous in the input layout
+//               and, if `reduce` is fused, the fused subgraph is pure
+//               elementwise.
+Status IrEmitterUnnested::EmitReductionToVector(
+    KernelThunk* kernel_thunk, HloInstruction* reduce, const Shape& input_shape,
+    absl::Span<const llvm_ir::ElementGenerator> input_gens,
+    absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+    absl::Span<const int64> dimensions_to_reduce,
+    absl::Span<HloComputation* const> reducers,
+    absl::Span<const ShapeIndex> reduce_output_shapes,
+    absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+        extra_output_gens) {
+  // This emission requires "reduce" to have an input layout. It is either set
+  // by LayoutAssignment (for a top-level kReduce) or by InstructionFusion (for
+  // a fused kReduce).
+  CHECK(input_shape.has_layout()) << "LayoutAssignment or InstructionFusion "
+                                     "doesn't set the input layout of "
+                                  << reduce->ToString();
+
+  // Specialize multi-dimensional-array-to-vector reduction.
+  std::vector<int64> input_dims_to_keep;
+  for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape);
+       ++input_dim) {
+    if (std::find(dimensions_to_reduce.begin(), dimensions_to_reduce.end(),
+                  input_dim) == dimensions_to_reduce.end()) {
+      input_dims_to_keep.push_back(input_dim);
+    }
+  }
+
+  // Sort the dimensions to keep from minor to major, to facilitate checking
+  // whether another dimension is major or minor of them.
+  std::sort(input_dims_to_keep.begin(), input_dims_to_keep.end(),
+            [&input_shape](int64 dim_a, int64 dim_b) {
+              return PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
+                                         dim_a) <
+                     PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
+                                         dim_b);
+            });
+  // Now, if output rank is at least 1, `input_dims_to_keep.front()` is
+  // minormost and `input_dims_to_keep.back()` is majormost.
+
+  // If the dimensions to keep are minormost, emit a column reduction. As all
+  // the dimensions to keep are contiguous, by prerequisite of
+  // `EmitReductionToVector`, we only need to check whether the minormost
+  // dimension of the input is to keep.
+  if (ShapeUtil::IsEffectiveScalar(reduce->shape())) {
+    return EmitReductionToScalar(kernel_thunk, reduce, input_shape, input_gens,
+                                 init_value_gens, reducers,
+                                 reduce_output_shapes, extra_output_gens);
+  } else if (input_dims_to_keep.front() ==
+             LayoutUtil::Minor(input_shape.layout(), 0)) {
+    // Column reduction. Treat the result of "input" as a matrix whose width
+    // is the most minor dimension and height the product of other dimensions,
+    // and treat "reduce" as a column reduction of the input matrix.
+    const int64 width = ShapeUtil::ElementsIn(reduce->shape());
+    // "width" can be zero, so don't do
+    //   height = ShapeUtil::ElementsIn(input_shape) / width;
+    int64 height = 1;
+    for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape);
+         ++input_dim) {
+      if (!std::count(input_dims_to_keep.begin(), input_dims_to_keep.end(),
+                      input_dim)) {
+        height *= input_shape.dimensions(input_dim);
+      }
+    }
+    return EmitColumnReduction(kernel_thunk, height, width, reduce, input_shape,
+                               input_gens, init_value_gens, reducers,
+                               reduce_output_shapes, extra_output_gens);
+  } else {
+    // Reduce the row dimension of a matrix or reduce dimension 0 and 2 in a
+    // 3D tensor. The size of dimension 1 (the height) is the size of the
+    // dimension to keep, the size of dimension 0 (the depth) is the product
+    // of dimensions that are more major than the dimension to keep, and the
+    // size of dimension 2 (the width) is the product of more minor
+    // dimensions.
+    int64 depth = 1;
+    int64 width = 1;
+    for (int64 input_dim = 0; input_dim < ShapeUtil::Rank(input_shape);
+         ++input_dim) {
+      if (PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
+                              input_dim) >
+          PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
+                              input_dims_to_keep.back())) {
+        depth *= input_shape.dimensions(input_dim);
+      } else if (PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
+                                     input_dim) <
+                 PositionInContainer(LayoutUtil::MinorToMajor(input_shape),
+                                     input_dims_to_keep.front())) {
+        width *= input_shape.dimensions(input_dim);
+      }
+    }
+    const int64 height = ShapeUtil::ElementsIn(reduce->shape());
+    return EmitRowReduction(kernel_thunk, depth, height, width, reduce,
+                            input_shape, input_gens, init_value_gens, reducers,
+                            reduce_output_shapes, extra_output_gens);
+  }
+}
+
 Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) {
   // TODO(b/112040122): Support multi-output reduce.
   if (!ShapeUtil::IsArray(reduce->shape())) {
     return Unimplemented("Multi-output reduce is not supported on GPU");
   }
+  auto input = reduce->operand(0);
+  auto init_value = reduce->operand(1);
+  absl::Span<const int64> dimensions_to_reduce(reduce->dimensions());
+  HloComputation* reducer = reduce->to_apply();
+  // HandleReduce specializes reduction from a multi-dimensional array to a 1D
+  // array. The specialized version requires an initializer thunk that
+  // initializes the output array to the initial value of the reduce.
   if (IsReductionToVector(*reduce)) {
-    return EmitReductionToVector(reduce);
+    TF_ASSIGN_OR_RETURN(std::unique_ptr<Thunk> initializer_thunk,
+                        BuildInitializerThunk(reduce));
+    std::vector<std::unique_ptr<Thunk>> thunks;
+    thunks.push_back(std::move(initializer_thunk));
+    std::unique_ptr<KernelThunk> kernel_thunk =
+        BuildKernelThunk(reduce, /*implements_whole_instruction=*/false);
+
+    TF_CHECK_OK(EmitReductionToVector(
+        kernel_thunk.get(), reduce, input->shape(),
+        {[&](const IrArray::Index& index) {
+          return GetIrArray(*input, *reduce).EmitReadArrayElement(index, &b_);
+        }},
+        {[&](const IrArray::Index& index) {
+          return GetIrArray(*init_value, *reduce)
+              .EmitReadArrayElement(index, &b_);
+        }},
+        dimensions_to_reduce, {reducer}, {{}}, {}));
+
+    thunks.push_back(std::move(kernel_thunk));
+
+    std::unique_ptr<SequentialThunk> sequential_thunk =
+        absl::make_unique<SequentialThunk>(std::move(thunks), reduce);
+    AddThunkToThunkSequence(std::move(sequential_thunk));
+    return Status::OK();
   }
 
   return IrEmitter::HandleReduce(reduce);
@@ -763,7 +1820,7 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     // Create the inner loop to iterate over the window.
     llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "inner"), &b_,
                                       index_type);
-    DimensionVector window_size;
+    std::vector<int64> window_size;
     for (const auto& dim : window.dimensions()) {
       window_size.push_back(dim.size());
       CHECK_GT(dim.size(), 0);
@@ -2185,8 +3242,7 @@ void EmitPartialTile(
         builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
 
     ksl->IfReturnVoid(
-        loop_name + "_x_in_tile", builder->CreateICmpULT(x_loc, tile_width),
-        [&] {
+        "x_in_tile", builder->CreateICmpULT(x_loc, tile_width), [&] {
           // tile_height_bound =
           //   ceil(tile_height / num_threads_y) * num_threads_y
           llvm::Value* ceiling_of_ratio = builder->CreateUDiv(
@@ -2203,8 +3259,8 @@ void EmitPartialTile(
               [&](llvm::Value* y_indvar) {
                 llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
                 ksl->IfReturnVoid(
-                    loop_name + "_y_in_tile",
-                    builder->CreateICmpULT(y_loc, tile_height), [&] {
+                    "y_in_tile", builder->CreateICmpULT(y_loc, tile_height),
+                    [&] {
                       emit_elem_function(
                           source_idx.AddOffsetToDim(
                               y_indvar, KernelMappingScheme::DimY, builder),
@@ -2235,7 +3291,7 @@ void EmitTiledElementalCodeWithBoundsCheck(
   llvm::Type* index_ty = tile_width->getType();
 
   ksl->IfReturnVoid(
-      loop_name + "_full_tile",
+      "full_tile",
       builder->CreateAnd(
           builder->CreateICmpEQ(llvm::ConstantInt::get(index_ty, tile_size_x),
                                 tile_width),
@@ -2326,395 +3382,7 @@ void IrEmitterUnnested::EmitTileElementForFusion(
   }
 }
 
-// Information to support the code generation for a tiled reduction kernel.
-using AddressVector = InlinedVector<llvm::AllocaInst*, 1>;
-class ReductionCodegenInfo : public IrEmitterUnnested::KernelCodegenInfo {
- public:
-  explicit ReductionCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme,
-                                bool is_row_reduction)
-      : KernelCodegenInfo(mapping_scheme),
-        current_output_linear_index_address_(nullptr),
-        current_output_inbound_address_(nullptr),
-        is_row_reduction_(is_row_reduction) {}
-
-  void SetCurrentOutputLinearIndexAddress(llvm::AllocaInst* a) {
-    current_output_linear_index_address_ = a;
-  }
-  // Returns the address of the memory that stores the linear index of the
-  // current output. Since we are processing reduction to contiguous physical
-  // dimensions, this linear index is the linear index of the 1D output array.
-  llvm::AllocaInst* GetCurrentOutputLinearIndexAddress() const {
-    return current_output_linear_index_address_;
-  }
-
-  void SetCurrentOutputInboundAddress(llvm::AllocaInst* a) {
-    current_output_inbound_address_ = a;
-  }
-
-  llvm::AllocaInst* GetCurrentOutputInboundAddress() const {
-    return current_output_inbound_address_;
-  }
-
-  AddressVector* GetMutablePartialResultAddresses() {
-    return &partial_result_addresses_;
-  }
-  const AddressVector& GetPartialResultAddresses() const {
-    return partial_result_addresses_;
-  }
-
-  AddressVector* GetMutableReductionInputAddresses() {
-    return &reduction_input_addresses_;
-  }
-  const AddressVector& GetReductionInputAddresses() const {
-    return reduction_input_addresses_;
-  }
-
-  InlinedVector<HloComputation*, 1>* GetMutableReducers() { return &reducers_; }
-  const InlinedVector<HloComputation*, 1>& GetReducers() const {
-    return reducers_;
-  }
-  int GetNumberOfReduces() const { return reducers_.size(); }
-
-  InlinedVector<ShapeIndex, 1>* GetMutableReductionOutputShapeIndices() {
-    return &reduction_output_shape_indices_;
-  }
-  const InlinedVector<ShapeIndex, 1>& GetReductionOutputShapeIndices() const {
-    return reduction_output_shape_indices_;
-  }
-
-  bool IsRowReduction() const { return is_row_reduction_; }
-
-  // Return the dimension that is being reduced between DimX and DimY.
-  int GetReducedDimensionEnum() const {
-    return IsRowReduction() ? llvm_ir::KernelMappingScheme::DimX
-                            : llvm_ir::KernelMappingScheme::DimY;
-  }
-
-  // Return the dimension that is being ketp between DimX and DimY.
-  int GetKeptDimensionEnum() const {
-    return IsRowReduction() ? llvm_ir::KernelMappingScheme::DimY
-                            : llvm_ir::KernelMappingScheme::DimX;
-  }
-
- private:
-  AddressVector partial_result_addresses_;
-  AddressVector reduction_input_addresses_;
-  InlinedVector<HloComputation*, 1> reducers_;
-  InlinedVector<ShapeIndex, 1> reduction_output_shape_indices_;
-  llvm::AllocaInst* current_output_linear_index_address_;
-  llvm::AllocaInst* current_output_inbound_address_;
-  bool is_row_reduction_;
-};
-
-namespace {
-// Returns a group of instructions that generate the output for the kernel
-// containing the given HLO instruction. The result may be an unnested kReduce
-// HLO, a nested kReduce HLO of a kInput fusion, or the operands of the tuple
-// for a multiple output fusion.
-absl::Span<HloInstruction* const> GetOutputInstructions(
-    HloInstruction* const* reduce_or_tuple_pointer) {
-  HloOpcode opcode = (*reduce_or_tuple_pointer)->opcode();
-  CHECK(opcode == HloOpcode::kReduce || opcode == HloOpcode::kTuple);
-  return opcode == HloOpcode::kTuple
-             ? (*reduce_or_tuple_pointer)->operands()
-             : absl::Span<HloInstruction* const>(reduce_or_tuple_pointer, 1);
-}
-
-const HloInstruction* GetFirstReduceInstruction(
-    absl::Span<HloInstruction* const> instructions) {
-  auto first_reduce_iter =
-      absl::c_find_if(instructions, [](const HloInstruction* inst) {
-        return inst->opcode() == HloOpcode::kReduce;
-      });
-  CHECK_NE(first_reduce_iter, instructions.end());
-  return *first_reduce_iter;
-}
-
-};  // namespace
-
-void IrEmitterUnnested::EmitPrologueForOneReduction(
-    HloInstruction* unnested_hlo, HloInstruction* reduce_inst, int reduce_idx,
-    KernelCodegenInfo* kernel_info, GpuElementalIrEmitter* elemental_emitter,
-    ShapeIndex output_shape_index) {
-  ReductionCodegenInfo* reduction_info =
-      static_cast<ReductionCodegenInfo*>(kernel_info);
-
-  InlinedVector<HloComputation*, 1>* reducers =
-      reduction_info->GetMutableReducers();
-  CHECK(IsReductionToVector(*reduce_inst));
-  reducers->push_back(reduce_inst->to_apply());
-
-  InlinedVector<ShapeIndex, 1>* reduction_output_shape_indices =
-      reduction_info->GetMutableReductionOutputShapeIndices();
-  reduction_output_shape_indices->push_back(std::move(output_shape_index));
-
-  AddressVector* reduction_input_addresses =
-      reduction_info->GetMutableReductionInputAddresses();
-  llvm::Type* element_type = llvm_ir::PrimitiveTypeToIrType(
-      reduce_inst->shape().element_type(), ir_emitter_context_->llvm_module());
-  llvm::AllocaInst* reduction_input_address = Alloca(element_type);
-  reduction_input_addresses->push_back(reduction_input_address);
-
-  AddressVector* partial_result_addresses =
-      reduction_info->GetMutablePartialResultAddresses();
-  llvm::AllocaInst* partial_result_address =
-      Alloca(element_type, /*ArraySize=*/nullptr,
-             "partial_reduction_result." + llvm::Twine(reduce_idx));
-  partial_result_addresses->push_back(partial_result_address);
-
-  // Initialize the partial result with the initial value of the reduction.
-  llvm::Value* init_ir_value;
-  if (unnested_hlo->opcode() == HloOpcode::kFusion) {
-    HloInstruction* init_value_operand = reduce_inst->mutable_operand(1);
-    FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(unnested_hlo),
-                                 elemental_emitter);
-
-    TF_CHECK_OK(init_value_operand->Accept(&fused_emitter));
-    init_ir_value =
-        fused_emitter
-            .GetGenerator(init_value_operand)(IrArray::Index(b_.getInt32Ty()))
-            .ValueOrDie();
-  } else {
-    const HloInstruction* init_value = unnested_hlo->operand(1);
-    init_ir_value =
-        GetIrArray(*init_value, *unnested_hlo)
-            .EmitReadArrayElement(IrArray::Index(b_.getInt32Ty()), &b_);
-  }
-
-  Store(init_ir_value, partial_result_address);
-}
-
-void IrEmitterUnnested::EmitPrologueForReduction(
-    HloInstruction* unnested_hlo, KernelCodegenInfo* kernel_info) {
-  VLOG(10) << "Emit prologue for reduction " << unnested_hlo->ToString();
-  // Find the unnested kReduce or the tuple that contains a list of kReduce.
-  HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
-                                        ? unnested_hlo->fused_expression_root()
-                                        : unnested_hlo;
-  absl::Span<HloInstruction* const> output_instructions =
-      GetOutputInstructions(&reduce_or_tuple);
-  ReductionCodegenInfo* reduction_info =
-      static_cast<ReductionCodegenInfo*>(kernel_info);
-  GpuElementalIrEmitter elemental_emitter(hlo_module_config_,
-                                          ir_emitter_context_->llvm_module(),
-                                          &b_, GetNestedComputer());
-  const HloInstruction* first_reduce = nullptr;
-  for (int i = 0, e = output_instructions.size(); i != e; ++i) {
-    if (output_instructions[i]->opcode() != HloOpcode::kReduce) {
-      continue;
-    }
-    HloInstruction* reduce_inst = output_instructions[i];
-    if (first_reduce == nullptr) {
-      first_reduce = reduce_inst;
-    } else {
-      CHECK(first_reduce->dimensions() == reduce_inst->dimensions());
-    }
-    ShapeIndex output_shape_index;
-    if (reduce_or_tuple->opcode() == HloOpcode::kTuple) {
-      output_shape_index = {i};
-    }
-
-    EmitPrologueForOneReduction(unnested_hlo, reduce_inst, i, kernel_info,
-                                &elemental_emitter,
-                                std::move(output_shape_index));
-  }
-
-  // Allocate stack storage to store the current output linear index and record
-  // the address of the storage.
-  reduction_info->SetCurrentOutputLinearIndexAddress(
-      Alloca(reduction_info->GetIndexType()));
-
-  if (!reduction_info->IsRowReduction()) {
-    llvm::Type* bool_ty = b_.getInt1Ty();
-    llvm::AllocaInst* output_inbound_addr = Alloca(bool_ty);
-    Store(llvm::ConstantInt::get(bool_ty, 0), output_inbound_addr);
-    reduction_info->SetCurrentOutputInboundAddress(output_inbound_addr);
-  }
-}
-
-void IrEmitterUnnested::EmitFullWarpShuffleDownLoopForAllReduces(
-    const InlinedVector<HloComputation*, 1>& reducers,
-    const AddressVector& partial_result_addresses) {
-  for (int distance = 16; distance >= 1; distance /= 2) {
-    for (int i = 0; i != reducers.size(); ++i) {
-      llvm::Type* element_type =
-          partial_result_addresses[i]->getType()->getElementType();
-      int bit_width = llvm_ir::GetSizeInBits(element_type);
-      llvm::Value* result_from_other_lane = Alloca(
-          element_type, nullptr, "result_from_other_lane" + llvm::Twine(i));
-      // Bitcast cannot be applied to aggregate types (even packed ones), so
-      // we bitcast addresses of load/store to intN* of the same bit-width.
-      llvm::Type* shuffled_value_type =
-          element_type->isStructTy() ? b_.getIntNTy(bit_width) : element_type;
-      auto convert_pointer_for_shuffle = [&](llvm::Value* ptr) {
-        return BitCast(ptr, shuffled_value_type->getPointerTo());
-      };
-      llvm::Value* partial_result =
-          Load(convert_pointer_for_shuffle(partial_result_addresses[i]),
-               "partial_reduction_result");
-      Store(EmitFullWarpShuffleDown(partial_result, b_.getInt32(distance), &b_),
-            convert_pointer_for_shuffle(result_from_other_lane));
-      TF_CHECK_OK(EmitCallToNestedComputation(
-          *reducers[i], {partial_result_addresses[i], result_from_other_lane},
-          partial_result_addresses[i]));
-    }
-  }
-}
-
-void IrEmitterUnnested::EmitEpilogueForReduction(
-    HloInstruction* unnested_hlo, KernelCodegenInfo* kernel_info) {
-  ReductionCodegenInfo* reduction_info =
-      static_cast<ReductionCodegenInfo*>(kernel_info);
-  int num_reduces = reduction_info->GetNumberOfReduces();
-  const AddressVector& partial_result_addresses =
-      reduction_info->GetPartialResultAddresses();
-  const InlinedVector<HloComputation*, 1>& reducers =
-      reduction_info->GetReducers();
-  const InlinedVector<ShapeIndex, 1>& reduction_output_shape_indices =
-      reduction_info->GetReductionOutputShapeIndices();
-
-  if (reduction_info->IsRowReduction()) {
-    EmitFullWarpShuffleDownLoopForAllReduces(reducers,
-                                             partial_result_addresses);
-    llvm::Value* lane_id = reduction_info->GetLaneId();
-    llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
-        ICmpEQ(lane_id, llvm::ConstantInt::get(lane_id->getType(), 0)),
-        "lane_id_is_zero", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
-  } else {
-    llvm::Value* output_inbound_addr =
-        reduction_info->GetCurrentOutputInboundAddress();
-    llvm::Value* output_inbound = Load(output_inbound_addr);
-    llvm_ir::LlvmIfData if_output_inbound_data = llvm_ir::EmitIfThenElse(
-        ICmpEQ(output_inbound,
-               llvm::ConstantInt::get(output_inbound->getType(), 1)),
-        "output_inbound", &b_);
-    llvm_ir::SetToFirstInsertPoint(if_output_inbound_data.true_block, &b_);
-  }
-
-  // Emit an atomic operation that accumulates the partial reduction to the
-  // output element. For row reduction, this is only for lane 0 due to the
-  // if-statement emitted above.
-  for (int i = 0; i != num_reduces; ++i) {
-    IrArray::Index element_index(
-        /*linear=*/Load(reduction_info->GetCurrentOutputLinearIndexAddress(),
-                        "output_linear_addr"),
-        ShapeUtil::GetSubshape(unnested_hlo->shape(),
-                               reduction_output_shape_indices[i]),
-        &b_);
-    llvm::Value* output_address =
-        GetIrArray(*unnested_hlo, *unnested_hlo,
-                   reduction_output_shape_indices[i])
-            .EmitArrayElementAddress(element_index, &b_,
-                                     "output_element_address");
-    // Do not emit atomic operations if each element in the reduction result is
-    // computed by one block, that is the dimension being reduced has only one
-    // block.
-    const llvm_ir::KernelMappingScheme* mapping_scheme =
-        reduction_info->GetKernelMappingScheme();
-    if (mapping_scheme->GetTileBlockSizeForDimension(
-            llvm_ir::KernelMappingScheme::DimZ) == 1 &&
-        mapping_scheme->GetTileBlockSizeForDimension(
-            reduction_info->GetReducedDimensionEnum()) == 1) {
-      TF_CHECK_OK(EmitCallToNestedComputation(
-          *reducers[i], {output_address, partial_result_addresses[i]},
-          output_address));
-    } else {
-      TF_CHECK_OK(EmitAtomicOperationForNestedComputation(
-          *reducers[i], output_address, partial_result_addresses[i]));
-    }
-  }
-}
-
-void IrEmitterUnnested::EmitTileElementForReduction(
-    HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index,
-    const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
-  VLOG(10) << "Emit tile element for reduce " << unnested_hlo->ToString();
-  HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
-                                        ? unnested_hlo->fused_expression_root()
-                                        : unnested_hlo;
-  llvm_ir::TiledParameterInfo* tiled_param_info =
-      kernel_info->GetTiledParameterInfo();
-  tiled_param_info->set_y(y_loc);
-  tiled_param_info->set_x(x_loc);
-
-  // Record the linear address for the current reduction.
-  const ReductionCodegenInfo* reduction_info =
-      dynamic_cast<const ReductionCodegenInfo*>(kernel_info);
-  Store(index[reduction_info->GetKeptDimensionEnum()],
-        reduction_info->GetCurrentOutputLinearIndexAddress());
-  if (!reduction_info->IsRowReduction()) {
-    llvm::Type* bool_ty = b_.getInt1Ty();
-    llvm::AllocaInst* output_inbound_addr =
-        reduction_info->GetCurrentOutputInboundAddress();
-    Store(llvm::ConstantInt::get(bool_ty, 1), output_inbound_addr);
-  }
-
-  InlinedVector<llvm_ir::ElementGenerator, 1> input_gens;
-  std::vector<std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
-      extra_output_gens;
-  GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_,
-                                     GetNestedComputer());
-  FusedIrEmitter fused_emitter(GetGeneratorForOperandIrArrays(unnested_hlo),
-                               &elem_emitter);
-  absl::Span<HloInstruction* const> output_instructions =
-      GetOutputInstructions(&reduce_or_tuple);
-  // Construct the ElementGenerator for each reduction and extra output in the
-  // the group of output instructions.
-  if (unnested_hlo->opcode() == HloOpcode::kFusion) {
-    fused_emitter.SetTiledParameterInfo(tiled_param_info);
-    TF_CHECK_OK(unnested_hlo->fused_expression_root()->Accept(&fused_emitter));
-
-    for (int i = 0, e = output_instructions.size(); i != e; ++i) {
-      const HloInstruction* inst = output_instructions[i];
-      ShapeIndex output_shape_index;
-      if (reduce_or_tuple->opcode() == HloOpcode::kTuple) {
-        output_shape_index = {i};
-      }
-      if (inst->opcode() == HloOpcode::kReduce) {
-        input_gens.push_back(fused_emitter.GetGenerator(inst->operand(0)));
-      } else {
-        extra_output_gens.emplace_back(fused_emitter.GetGenerator(inst),
-                                       std::move(output_shape_index));
-      }
-    }
-  } else {
-    input_gens.push_back([&](const IrArray::Index& index) {
-      return GetIrArray(*unnested_hlo->operand(0), *unnested_hlo)
-          .EmitReadArrayElement(index, &b_);
-    });
-  }
-
-  IrArray::Index input_index =
-      reduction_info->GetKernelMappingScheme()->GetUnnormalizedIndex(
-          index,
-          GetFirstReduceInstruction(output_instructions)->operand(0)->shape());
-  const AddressVector& partial_reduction_result_addresses =
-      reduction_info->GetPartialResultAddresses();
-  const AddressVector& reduction_input_addresses =
-      reduction_info->GetReductionInputAddresses();
-  const InlinedVector<HloComputation*, 1>& reducers =
-      reduction_info->GetReducers();
-
-  // Emit code to generate the input and perform the reduction computation for
-  // each reduction instruction.
-  for (int i = 0; i != reducers.size(); ++i) {
-    llvm::Value* const input_ir_value = input_gens[i](input_index).ValueOrDie();
-    Store(input_ir_value, reduction_input_addresses[i]);
-    TF_CHECK_OK(EmitCallToNestedComputation(
-        *reducers[i],
-        {partial_reduction_result_addresses[i], reduction_input_addresses[i]},
-        partial_reduction_result_addresses[i]));
-  }
-
-  // Emit code to generate the output for the non-reduction instructions in the
-  // fusion, if any.
-  TF_CHECK_OK(
-      EmitExtraOutputsForReduce(unnested_hlo, input_index, extra_output_gens));
-}
-
-// Emits a kernel for the hlo instruction using the given tiling scheme.
+// Emits a block of tiles, given a function object to emit one tile.
 void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
                                   const KernelCodegenInfo* kernel_info,
                                   KernelSupportLibrary& ksl,
@@ -2841,6 +3509,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
             << llvm_ir::DumpToString(*param_shmem_buffers[id]);
   }
 
+  CHECK_EQ(mapping_scheme->GetThreadsPerTile() % kWarpSize, 0);
   LaunchDimensions launch_dimensions = LaunchDimensions(
       mapping_scheme->GetNumberOfBlocks(), mapping_scheme->GetThreadsPerTile());
   llvm::Type* index_ty = GetIndexTypeForKernel(
@@ -2884,7 +3553,6 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
   kernel_info->SetLaneId(
       mapping_scheme->GetNumberOfThreadsForDimensionX() == kWarpSize ? x
                                                                      : nullptr);
-  kernel_info->SetIndexType(index_ty);
 
   KernelSupportLibrary ksl(&b_, llvm_ir::UnrollMode::kDefaultUnroll);
   // Curry a few parameters to EmitTiledElementalCodeWithBoundsCheck.
@@ -2909,31 +3577,29 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
         input_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
             .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
 
+    // Copy input parameter values to shared memory buffers:
+    // tile[y, x] = input[index]
+    // Note that tile_width and tile_height are flipped here because we are
+    // reading a transposed tile.
+    emit_tiled_elemental_code_with_bounds_check(
+        input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
+        [&](const IrArray::Index& index, llvm::Value* y_loc,
+            llvm::Value* x_loc) {
+          for (int64 id : tiled_param_ids) {
+            IrArray& input_in_logical_shape = param_in_reduced_shape_arrays[id];
+            llvm::Value* shmem_buffer = param_shmem_buffers[id];
+            // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
+            // global variables, so LLVM can't infer much about it.
+            Store(input_in_logical_shape.EmitReadArrayElement(index, &b_,
+                                                              "input_element"),
+                  GEP(shmem_buffer, {index_typed_constant(0), y_loc, x_loc}));
+          }
+        });
+
     // If shared memory transpose is needed, wait for all threads to reach this
     // point, lest we copy a value from tile to output before the other thread
     // copies it from input to tile. This is `__syncthreads` in CUDA.
     if (!tiled_param_ids.empty()) {
-      // Copy input parameter values to shared memory buffers:
-      // tile[y, x] = input[index]
-      // Note that tile_width and tile_height are flipped here because we are
-      // reading a transposed tile.
-      emit_tiled_elemental_code_with_bounds_check(
-          input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
-          [&](const IrArray::Index& index, llvm::Value* y_loc,
-              llvm::Value* x_loc) {
-            for (int64 id : tiled_param_ids) {
-              IrArray& input_in_logical_shape =
-                  param_in_reduced_shape_arrays[id];
-              llvm::Value* shmem_buffer = param_shmem_buffers[id];
-              // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
-              // global variables, so LLVM can't infer much about it.
-              Store(input_in_logical_shape.EmitReadArrayElement(
-                        index, &b_, "input_element"),
-                    GEP(shmem_buffer, {index_typed_constant(0), y_loc, x_loc}));
-            }
-          });
-
-      // Wait for all threads to reach this point using `__syncthreads` in CUDA.
       llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_);
     }
 
@@ -2953,7 +3619,6 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
           kernel_generator.GetTileElementGenerator()(unnested_hlo, index,
                                                      kernel_info, y_loc, x_loc);
         });
-
     // If a tile block contains multiple tiles and shared memory buffers are
     // used, we need to wait for all threads to finish using the shared memory
     // buffer for the current tile before we move on to process the next tile
@@ -3149,246 +3814,6 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
   return true;
 }
 
-namespace {
-// Checks that the outputs of a fusion with reduction are consistent.
-Status AreFusedReductionOutputsConsistent(
-    absl::Span<HloInstruction* const> output_instructions,
-    const HloInstruction* first_reduce) {
-  for (const HloInstruction* inst : output_instructions) {
-    if (inst->opcode() == HloOpcode::kReduce) {
-      // Shapes, layouts and dimensions must be the same for all reduces
-      // inside of this fusion.
-      TF_RET_CHECK(ShapeUtil::Equal(first_reduce->shape(), inst->shape()));
-      TF_RET_CHECK(ShapeUtil::Equal(first_reduce->operand(0)->shape(),
-                                    inst->operand(0)->shape()));
-      TF_RET_CHECK(ShapeUtil::Equal(first_reduce->operand(1)->shape(),
-                                    inst->operand(1)->shape()));
-      TF_RET_CHECK(first_reduce->dimensions() == inst->dimensions());
-    } else {
-      // For extra outputs we can relax shape equality to allow different
-      // types (with the same number of elements). Layouts still have to
-      // match.
-      TF_RET_CHECK(ShapeUtil::CompatibleIgnoringElementType(
-          first_reduce->operand(0)->shape(), inst->shape()));
-      TF_RET_CHECK(LayoutUtil::Equal(first_reduce->operand(0)->shape().layout(),
-                                     inst->shape().layout()));
-    }
-  }
-  return Status::OK();
-}
-
-// Finds the dimensions to keep for the reduction, sorts and returns the
-// dimensions from minor to major.
-DimensionVector GetDimensionsToKeepMinorToMajor(
-    const Shape& input_shape, absl::Span<const int64> dims_to_reduce) {
-  DimensionVector input_dims(ShapeUtil::Rank(input_shape), 0);
-  absl::c_iota(input_dims, 0);
-  DimensionVector input_dims_to_keep;
-  for (int input_dim : input_dims) {
-    auto it = absl::c_find_if(dims_to_reduce, [&](int64 dim_to_reduce) {
-      return dim_to_reduce == input_dim;
-    });
-    if (it == dims_to_reduce.end()) {
-      input_dims_to_keep.push_back(input_dim);
-    }
-  }
-
-  // Sort the dimensions to keep from minor to major.
-  absl::c_sort(input_dims_to_keep, [&input_shape](int64 dim_a, int64 dim_b) {
-    return PositionInContainer(LayoutUtil::MinorToMajor(input_shape), dim_a) <
-           PositionInContainer(LayoutUtil::MinorToMajor(input_shape), dim_b);
-  });
-
-  VLOG(10) << "dims to keep minor to major"
-           << absl::StrJoin(input_dims_to_keep, ",");
-  return input_dims_to_keep;
-}
-
-// Given the input shape and dimensions to reduce for the reduction to vector,
-// returns <num_reduced_major, num_kept, num_reduced_minor>:
-// num_kept: the number of elements in the contiguous dimensions to keep.
-// num_reduced_major: the number of elements in the dimensions to reduce that
-//   are more major than the dimensions to keep.
-// num_reduced_minor: the number of elements in the dimensions to reduce that
-//   are more minor than the dimensions to kept.
-std::tuple<int64, int64, int64> GetReductionToVectorDimensions(
-    const Shape& input_shape, absl::Span<const int64> dims_to_reduce) {
-  DimensionVector input_dims_to_keep_minor_to_major =
-      GetDimensionsToKeepMinorToMajor(input_shape, dims_to_reduce);
-  CHECK(LayoutUtil::AreDimensionsConsecutive(
-      input_shape.layout(), input_dims_to_keep_minor_to_major));
-  int num_reduced_major = 1, num_kept = 1, num_reduced_minor = 1;
-  if (input_dims_to_keep_minor_to_major.empty()) {
-    return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor);
-  }
-  DimensionVector input_dims(ShapeUtil::Rank(input_shape), 0);
-  absl::c_iota(input_dims, 0);
-  absl::Span<const int64> minor_to_major =
-      LayoutUtil::MinorToMajor(input_shape);
-  for (int input_dim : input_dims) {
-    int64 curr_dim_size = input_shape.dimensions(input_dim);
-    if (PositionInContainer(minor_to_major, input_dim) >
-        PositionInContainer(minor_to_major,
-                            input_dims_to_keep_minor_to_major.back())) {
-      num_reduced_major *= curr_dim_size;
-    } else if (PositionInContainer(minor_to_major, input_dim) <
-               PositionInContainer(minor_to_major,
-                                   input_dims_to_keep_minor_to_major.front())) {
-      num_reduced_minor *= curr_dim_size;
-    } else {
-      num_kept *= curr_dim_size;
-    }
-  }
-
-  return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor);
-}
-
-std::tuple<KernelMappingScheme, bool> ComputeMappingSchemeAndReductionKind(
-    const HloInstruction* first_reduce, llvm::IRBuilder<>* b) {
-  int64 depth = 1;
-  int64 height = 1;
-  int64 width = 1;
-  bool is_row_reduction = true;
-  int64 tile_size_x = 1;
-  int64 tile_size_y = 1;
-  int64 block_size_y = 1;
-  int64 block_size_z = 1;
-  int64 num_threads_x = 1;
-  int64 num_threads_y = 1;
-  const Shape& input_shape = first_reduce->operand(0)->shape();
-  int64 num_input_elems = ShapeUtil::ElementsIn(input_shape);
-  int64 num_output_elems = ShapeUtil::ElementsIn(first_reduce->shape());
-  int64 num_reduced_major, num_kept, num_reduced_minor;
-  std::tie(num_reduced_major, num_kept, num_reduced_minor) =
-      GetReductionToVectorDimensions(input_shape, first_reduce->dimensions());
-  CHECK_EQ(num_output_elems, num_kept);
-
-  if (num_kept == 1) {
-    // Scalar reduction is a special row reduction with depth = height = 1.
-    width = num_input_elems;
-    tile_size_x = kWarpSize * 16;
-    num_threads_x = kWarpSize;
-  } else if (num_reduced_minor == 1) {
-    // Column reduction reduces inputs with dimension [height, width], where
-    // width is the minor dimension, to dimension [width].
-    height = num_reduced_major;
-    width = num_kept;
-    is_row_reduction = false;
-    tile_size_x = std::min(kWarpSize, num_kept);
-    // The old Column reduction algorithm uses kTileHeight = 128. We choose
-    // tile_size_y * block_size_y = 128 to match the value of kTileHeight. Using
-    // a non-trivial block_size_y here is a way to avoid unrolling all the 128
-    // iterations.
-    tile_size_y = 32;
-    block_size_y = 4;
-    num_threads_x = tile_size_x;
-  } else {
-    // Row reduction reduces inputs with dimension [depth, height, width],
-    // where width is the most minor dimension, to dimension [height] .
-    depth = num_reduced_major;
-    height = num_kept;
-    width = num_reduced_minor;
-    num_threads_x = kWarpSize;
-    if (width % (kWarpSize * 64) == 0) {
-      tile_size_x = kWarpSize * 64;
-    } else {
-      tile_size_x = kWarpSize * 8;
-      block_size_z = 8;
-      while (depth % block_size_z != 0) {
-        block_size_z -= 1;
-      }
-    }
-  }
-  DCHECK_EQ(depth * height * width, num_input_elems);
-  VLOG(10) << "is_row_reduction " << is_row_reduction << depth << " " << height
-           << " " << width;
-
-  DimensionVector dims_in_elem{depth, height, width};
-  DimensionVector req_block_sizes{block_size_z, block_size_y, 1};
-  llvm_ir::KernelMappingScheme mapping_scheme(dims_in_elem, tile_size_y,
-                                              tile_size_x, req_block_sizes,
-                                              num_threads_y, num_threads_x, b);
-  return std::make_tuple(mapping_scheme, is_row_reduction);
-}
-
-}  // namespace
-
-Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) {
-  VLOG(10) << "Emitting reduction to vector " << unnested_hlo->ToString();
-
-  HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
-                                        ? unnested_hlo->fused_expression_root()
-                                        : unnested_hlo;
-  absl::Span<HloInstruction* const> output_instructions =
-      GetOutputInstructions(&reduce_or_tuple);
-  const HloInstruction* first_reduce =
-      GetFirstReduceInstruction(output_instructions);
-
-  if (output_instructions.size() > 1) {
-    TF_RETURN_IF_ERROR(
-        AreFusedReductionOutputsConsistent(output_instructions, first_reduce));
-  }
-
-  // Build an initializer thunk to initialize each reduction output.
-  std::vector<std::unique_ptr<Thunk>> thunks;
-  for (int i = 0, e = output_instructions.size(); i != e; ++i) {
-    if (output_instructions[i]->opcode() != HloOpcode::kReduce) {
-      continue;
-    }
-    TF_ASSIGN_OR_RETURN(
-        std::unique_ptr<Thunk> initializer_thunk,
-        BuildInitializerThunk(unnested_hlo,
-                              (output_instructions[i] == reduce_or_tuple)
-                                  ? ShapeIndex()
-                                  : ShapeIndex({i})));
-    thunks.push_back(std::move(initializer_thunk));
-  }
-
-  // Build a kernel thunk to compute all the outputs.
-  std::unique_ptr<KernelThunk> kernel_thunk =
-      BuildKernelThunk(unnested_hlo, /*implements_whole_instruction=*/false);
-
-  const Shape& input_shape = first_reduce->operand(0)->shape();
-  // The layout of a reduction input is either set by LayoutAssignment for
-  // unnested kReduce or by InstructionFusion for fused kReduce.
-  CHECK(input_shape.has_layout()) << "LayoutAssignment or InstructionFusion "
-                                     "doesn't set the input layout of "
-                                  << first_reduce->ToString();
-
-  bool is_row_reduction;
-  llvm_ir::KernelMappingScheme mapping_scheme;
-  std::tie(mapping_scheme, is_row_reduction) =
-      ComputeMappingSchemeAndReductionKind(first_reduce, &b_);
-  ReductionCodegenInfo reduction_info(&mapping_scheme, is_row_reduction);
-  KernelCodeGenerator kernel_generator(
-      /*tile_element_generator=*/
-      [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
-          const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-          llvm::Value* x_loc) {
-        EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc);
-      },
-      /*block_prologue_generator=*/
-      [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) {
-        EmitPrologueForReduction(hlo, kernel_info);
-      },
-      /*block_epilogue_generator*/
-      [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) {
-        EmitEpilogueForReduction(hlo, kernel_info);
-      });
-
-  LaunchDimensions launch_dimensions =
-      EmitKernel(unnested_hlo, {}, kernel_generator, &reduction_info);
-  UpdateLaunchDimensions(launch_dimensions, kernel_thunk.get(),
-                         ir_emitter_context_->llvm_module());
-
-  thunks.push_back(std::move(kernel_thunk));
-  std::unique_ptr<SequentialThunk> sequential_thunk =
-      absl::make_unique<SequentialThunk>(std::move(thunks), unnested_hlo);
-  AddThunkToThunkSequence(std::move(sequential_thunk));
-
-  return Status::OK();
-}
-
 Status IrEmitterUnnested::EmitConstantGlobals() {
   for (const BufferAllocation& allocation :
        ir_emitter_context_->buffer_assignment().Allocations()) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index 85a0e5328c..e09ed657a8 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -16,7 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_
 #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_
 
-#include "absl/container/inlined_vector.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h"
 #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/thunk.h"
@@ -69,12 +68,9 @@ class IrEmitterUnnested : public IrEmitter {
     explicit KernelCodegenInfo(llvm_ir::KernelMappingScheme* mapping_scheme)
         : mapping_scheme_(mapping_scheme),
           tiled_param_info_(nullptr),
-          lane_id_(nullptr),
-          index_ty_(nullptr) {}
-    virtual ~KernelCodegenInfo() {}
+          lane_id_(nullptr) {}
 
     void SetLaneId(llvm::Value* v) { lane_id_ = v; }
-    void SetIndexType(llvm::Type* t) { index_ty_ = t; }
     void SetTiledParamInfo(llvm_ir::TiledParameterInfo* tiled_param_info) {
       CHECK_EQ(tiled_param_info_, nullptr);
       tiled_param_info_ = tiled_param_info;
@@ -87,13 +83,11 @@ class IrEmitterUnnested : public IrEmitter {
     llvm_ir::TiledParameterInfo* GetTiledParameterInfo() const {
       return tiled_param_info_;
     }
-    llvm::Type* GetIndexType() const { return index_ty_; }
 
    private:
     llvm_ir::KernelMappingScheme* mapping_scheme_;
     llvm_ir::TiledParameterInfo* tiled_param_info_;
     llvm::Value* lane_id_;
-    llvm::Type* index_ty_;
   };
 
   // A function object to prepare for the code generation for a tile block.
@@ -206,14 +200,82 @@ class IrEmitterUnnested : public IrEmitter {
 
   // Helper for writing extra outputs from inside a reduce kernel.
   Status EmitExtraOutputsForReduce(
-      const HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index,
+      const HloInstruction* reduce, const llvm_ir::IrArray::Index& index,
       absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
           extra_output_gens);
 
-  // Generates code for reduction to contiguous dimensions.
+  // EmitColumnReduction and EmitRowReduction emit code for column and row
+  // reduction of a matrix and/or 3D tensor. Row and column reduction have
+  // different memory access pattern, so for performance their implementations
+  // are significantly different.
   //
-  // Prerequisite: `IsReductionToVector(*unnested_hlo)`
-  Status EmitReductionToVector(HloInstruction* unnested_hlo);
+  // Emits code that reduces a matrix of shape [height x width] to a vector of
+  // [width]. Other parameters have the same meaning as those of
+  // `EmitReductionToVector`. Note that input shape might not be
+  // [height x width], but can be bitcast to [height x width] with "height"
+  // being the major dimension.
+  Status EmitColumnReduction(
+      KernelThunk* kernel_thunk, int64 height, int64 width,
+      HloInstruction* reduce, const Shape& input_shape,
+      absl::Span<const llvm_ir::ElementGenerator> input_gens,
+      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+      absl::Span<HloComputation* const> reducers,
+      absl::Span<const ShapeIndex> reduce_output_shapes,
+      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+          extra_output_gens);
+
+  // Emits code that reduces a 3D tensor of shape [depth x height x width] to a
+  // vector of shape [height]. Other parameters have the same meaning as those
+  // of `EmitReductionToVector`. Note that input shape might not be
+  // [depth x height x width], but can be bitcast to [depth x height x width]
+  // with "depth" being the most major dimension.
+  Status EmitRowReduction(
+      KernelThunk* kernel_thunk, int64 depth, int64 height, int64 width,
+      HloInstruction* reduce, const Shape& input_shape,
+      absl::Span<const llvm_ir::ElementGenerator> input_gens,
+      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+      absl::Span<HloComputation* const> reducers,
+      absl::Span<const ShapeIndex> reduce_output_shapes,
+      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+          extra_output_gens);
+
+  // Emits code that reduces a tensor of arbitrary rank to a scalar.
+  Status EmitReductionToScalar(
+      KernelThunk* kernel_thunk, HloInstruction* reduce,
+      const Shape& input_shape,
+      absl::Span<const llvm_ir::ElementGenerator> input_gens,
+      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+      absl::Span<HloComputation* const> reducers,
+      absl::Span<const ShapeIndex> reduce_output_shapes,
+      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+          extra_output_gens);
+
+  // Figures out whether `reduce` is a row or column reduction, and which
+  // dimensions to reduce, and calls either `EmitRowReduction` or
+  // `EmitColumnReduction` as appropriate. `input_shape` is the shape of the
+  // input array, which is the operand of the Reduce instruction if unfused or
+  // of the Fusion instruction if fused. `input_gen` and `init_value_gen`
+  // generate elements of the input and the initial value. Other parameters mean
+  // the same as for `HandleReduce`.
+  //
+  // Multiple reduces can be emitted in the same loop, assuming they have the
+  // same input and output shapes, and the same reduce dimensions.
+  //
+  // extra_output_gens can contain extra generators for intermediate outputs.
+  // These must have the same shape as the reduce input as they are computed
+  // when the reduce inputs are being read.
+  //
+  // Prerequisite: `IsReductionToVector(*reduce)`
+  Status EmitReductionToVector(
+      KernelThunk* kernel_thunk, HloInstruction* reduce,
+      const Shape& input_shape,
+      absl::Span<const llvm_ir::ElementGenerator> input_gens,
+      absl::Span<const llvm_ir::ElementGenerator> init_value_gens,
+      absl::Span<const int64> dimensions_to_reduce,
+      absl::Span<HloComputation* const> reducers,
+      absl::Span<const ShapeIndex> reduce_output_shapes,
+      absl::Span<const std::pair<llvm_ir::ElementGenerator, ShapeIndex>>
+          extra_output_gens);
 
   // Emits code for an in-place scatter, modifying `thunk`s launch dimensions in
   // the process. `scatter` may be fused, scatter indices are taken from
@@ -252,29 +314,6 @@ class IrEmitterUnnested : public IrEmitter {
                                 const llvm_ir::IrArray::Index& index,
                                 const KernelCodegenInfo* kernel_info,
                                 llvm::Value* y_loc, llvm::Value* x_loc);
-  // Emits code to process a tensor element in a tile for the given input hlo
-  // that is either a unnested kReduce or a kInput fusion.
-  void EmitTileElementForReduction(HloInstruction* unnested_hlo,
-                                   const llvm_ir::IrArray::Index& index,
-                                   const KernelCodegenInfo* kernel_info,
-                                   llvm::Value* y_loc, llvm::Value* x_loc);
-  // Prepares for the code generation for a tile block of a reduction kernel.
-  void EmitPrologueForReduction(HloInstruction* unnested_hlo,
-                                KernelCodegenInfo* kernel_info);
-  void EmitPrologueForOneReduction(HloInstruction* unnested_hlo,
-                                   HloInstruction* reduce_inst, int reduce_idx,
-                                   KernelCodegenInfo* kernel_info,
-                                   GpuElementalIrEmitter* elemental_emitter,
-                                   ShapeIndex output_shape_index);
-  // Wraps up the code generation for a tile block of a reduction kernel.
-  void EmitEpilogueForReduction(HloInstruction* unnested_hlo,
-                                KernelCodegenInfo* kernel_info);
-  // For each reducer, emits the shuffle-down loop to accumulate the partial
-  // result to the global result.
-  void EmitFullWarpShuffleDownLoopForAllReduces(
-      const absl::InlinedVector<HloComputation*, 1>& reducers,
-      const absl::InlinedVector<llvm::AllocaInst*, 1>&
-          partial_result_addresses);
 
   // Generates the IrArray for each input of an hlo and returns a vector that
   // constains such IrArrays.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index 1aa85eb8d2..c26711e526 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -120,7 +120,7 @@ KernelMappingScheme::KernelMappingScheme(
     absl::Span<const int64> req_block_sizes, int64 num_threads_y,
     int64 num_threads_x, llvm::IRBuilder<>* b)
     : b_(b),
-      dims_in_elems_(dims_in_elems.begin(), dims_in_elems.end()),
+      dims_in_elems_(dims_in_elems),
       tile_sizes_{1, tile_size_y, tile_size_x},
       num_threads_x_(num_threads_x),
       num_threads_y_(num_threads_y) {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index 7277aeac8a..06002d57b0 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -90,16 +90,15 @@ class KernelMappingScheme {
   enum { DimZ = 0, DimY, DimX, DimTot };
 
  public:
-  KernelMappingScheme() {}
   // dims_in_elems: the normalized tensor dimensions.
   // req_block_sizes: the requested block size in number of tiles for each
   //   dimension. The actual block size is set to min(req_block_size,
   //   dims_in_number_of_blocks).
-  KernelMappingScheme(absl::Span<const int64> dims_in_elems, int64 tile_size_y,
-                      int64 tile_size_x,
-                      absl::Span<const int64> req_block_sizes,
-                      int64 num_threads_y, int64 num_threads_x,
-                      llvm::IRBuilder<>* b);
+  explicit KernelMappingScheme(absl::Span<const int64> dims_in_elems,
+                               int64 tile_size_y, int64 tile_size_x,
+                               absl::Span<const int64> req_block_sizes,
+                               int64 num_threads_y, int64 num_threads_x,
+                               llvm::IRBuilder<>* b);
 
   absl::Span<const int64> GetDimensionsInElements() const {
     return dims_in_elems_;
@@ -134,10 +133,6 @@ class KernelMappingScheme {
   }
 
   absl::Span<const int64> GetBlockSizes() const { return block_sizes_; }
-  int64 GetTileBlockSizeForDimension(int d) const {
-    DCHECK(d >= DimZ && d <= DimX);
-    return dims_in_blocks_[d];
-  }
 
   int64 GetNumberOfThreadsForDimensionX() const { return num_threads_x_; }
   int64 GetNumberOfThreadsForDimensionY() const { return num_threads_y_; }
@@ -168,7 +163,7 @@ class KernelMappingScheme {
  private:
   llvm::IRBuilder<>* b_;
   // The number of elements in each dimension.
-  std::vector<int64> dims_in_elems_;
+  absl::Span<const int64> dims_in_elems_;
 
   // The number of elements for each dimension of a tile.
   std::vector<int64> tile_sizes_;
-- 
GitLab


From f821b6aac05d24917efa985fdde9293bc1534470 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 01:02:13 -0800
Subject: [PATCH 1347/1554] compat: Update forward compatibility horizon to
 2018-12-05

PiperOrigin-RevId: 224113713
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 0b6ff30488..dc1a58a95a 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 4)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 5)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 3a6181ae34e73301247553983b406c8e01539b01 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 02:56:44 -0800
Subject: [PATCH 1348/1554] Initial support for TrackableAssets in experimental
 saved object graph.

PiperOrigin-RevId: 224125285
---
 tensorflow/python/saved_model/load.py         | 64 ++++++++++++++-----
 tensorflow/python/saved_model/load_test.py    | 33 ++++++++++
 tensorflow/python/saved_model/save.py         | 32 ++++++++--
 .../saved_model/saved_object_graph.proto      | 62 +++++++++++++-----
 .../python/training/checkpointable/util.py    | 20 ++----
 .../training/checkpointable/util_test.py      | 12 ----
 6 files changed, 159 insertions(+), 64 deletions(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 397086e106..f3caa8798a 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -23,20 +23,54 @@ import os
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import saved_object_graph_pb2
+from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.util import compat
 
 
-def _recreate_object_graph(object_graph_proto):
-  """Recreates Python objects from an ObjectGraph proto."""
-  objects = []
-  for _ in object_graph_proto.nodes:
-    # TODO(allenl): re-create variables and other types
-    objects.append(tracking.Checkpointable())
-  for obj, object_proto in zip(objects, object_graph_proto.nodes):
-    for reference in object_proto.children:
-      setattr(obj, reference.local_name, objects[reference.node_id])
-  return objects[0]
+class _Loader(object):
+  """Helper class to load an object-based SavedModel."""
+
+  def __init__(self, proto, export_dir):
+    self._proto = proto
+    self._export_dir = export_dir
+    self._load_all()
+
+  def _load_all(self):
+    self._nodes = [self._recreate(proto) for proto in self._proto.nodes]
+    # After creating the objects, construct the edges between the objects.
+    for obj, object_proto in zip(self._nodes, self._proto.nodes):
+      for reference in object_proto.children:
+        setattr(obj, reference.local_name, self._nodes[reference.node_id])
+
+  def get(self, node_id):
+    return self._nodes[node_id]
+
+  def _recreate(self, proto):
+    factory = {
+        "user_object": lambda: self._recreate_user_object(proto.user_object),
+        "asset": lambda: self._recreate_asset(proto.asset),
+    }
+    kind = proto.WhichOneof("kind")
+    if kind not in factory:
+      raise ValueError("Unknown SavedObject type: %r" % kind)
+    return factory[kind]()
+
+  def _recreate_user_object(self, proto):
+    del proto
+    return tracking.Checkpointable()
+
+  def _recreate_asset(self, proto):
+    filename = os.path.join(
+        saved_model_utils.get_assets_dir(self._export_dir),
+        proto.relative_filename)
+    return tracking.TrackableAsset(filename)
+
+
+def _load_saved_object_graph_proto(filename):
+  with file_io.FileIO(filename, "rb") as f:
+    contents = f.read()
+    return saved_object_graph_pb2.SavedObjectGraph.FromString(contents)
 
 
 def load(export_dir):
@@ -46,13 +80,9 @@ def load(export_dir):
       compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY),
       compat.as_bytes("object_graph.pb"))
   if file_io.file_exists(object_graph_filename):
-    # If there is an object graph associated with the SavedModel, we'll create a
-    # root object from that.
-    object_graph_string = file_io.FileIO(object_graph_filename, "rb").read()
-    object_graph_proto = (
-        saved_object_graph_pb2.SavedObjectGraph())
-    object_graph_proto.ParseFromString(object_graph_string)
-    root = _recreate_object_graph(object_graph_proto)
+    proto = _load_saved_object_graph_proto(object_graph_filename)
+    loader = _Loader(proto, export_dir)
+    root = loader.get(0)
   else:
     raise NotImplementedError(
         "Currently only SavedModels exported with `tf.saved_model.save` may be "
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index bfa201ad63..bccbee6d53 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -19,11 +19,13 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import tempfile
 
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
+from tensorflow.python.lib.io import file_io
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
@@ -46,6 +48,37 @@ class LoadTest(test.TestCase):
     self.assertIs(imported.dep_three, imported.dep_two.dep)
     self.assertIsNot(imported.dep_one, imported.dep_two)
 
+  def _make_asset(self, contents):
+    filename = tempfile.mktemp(prefix=self.get_temp_dir())
+    with open(filename, "w") as f:
+      f.write(contents)
+    return filename
+
+  def test_assets_import(self):
+    file1 = self._make_asset("contents 1")
+    file2 = self._make_asset("contents 2")
+
+    root = tracking.Checkpointable()
+    root.f = def_function.function(
+        lambda x: 2. * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+    root.asset1 = tracking.TrackableAsset(file1)
+    root.asset2 = tracking.TrackableAsset(file2)
+
+    save_dir = os.path.join(self.get_temp_dir(), "save_dir")
+    save.save(root, save_dir)
+
+    file_io.delete_file(file1)
+    file_io.delete_file(file2)
+    load_dir = os.path.join(self.get_temp_dir(), "load_dir")
+    file_io.rename(save_dir, load_dir)
+
+    imported = load.load(load_dir)
+    with open(imported.asset1.asset_path.numpy(), "r") as f:
+      self.assertEquals("contents 1", f.read())
+    with open(imported.asset2.asset_path.numpy(), "r") as f:
+      self.assertEquals("contents 2", f.read())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index a66e19b199..53c1d2cc87 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -517,20 +517,38 @@ def _fill_meta_graph_def(meta_graph_def, obj, signature_functions,
   return asset_info.asset_filename_map
 
 
-def _write_object_graph(obj, export_dir):
-  """Save a SavedObjectGraph proto for `obj`."""
+def _write_object_graph(root, export_dir, asset_filename_map):
+  """Save a SavedObjectGraph proto for `root`."""
   # SavedObjectGraph is similar to the CheckpointableObjectGraph proto in the
   # checkpoint. It will eventually go into the SavedModel.
-  object_proto = util.make_object_graph_without_attributes(
-      obj, proto=saved_object_graph_pb2.SavedObjectGraph())
+  proto = saved_object_graph_pb2.SavedObjectGraph()
+
+  checkpointable_objects, node_ids, slot_variables = util.find_objects(root)
+  util.fill_object_graph_proto(checkpointable_objects, node_ids, slot_variables,
+                               proto)
+
+  # Build map from original filename to relative asset filename.
+  filename_map = {v: k for k, v in asset_filename_map.items()}
+
+  for obj, obj_proto in zip(checkpointable_objects, proto.nodes):
+    _write_object_proto(obj, obj_proto, filename_map)
+
   extra_asset_dir = os.path.join(
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY))
   file_io.recursive_create_dir(extra_asset_dir)
   object_graph_filename = os.path.join(
       extra_asset_dir, compat.as_bytes("object_graph.pb"))
-  file_io.write_string_to_file(object_graph_filename,
-                               object_proto.SerializeToString())
+  file_io.write_string_to_file(object_graph_filename, proto.SerializeToString())
+
+
+def _write_object_proto(obj, proto, filename_map):
+  """Saves an object into SavedObject proto."""
+  if isinstance(obj, tracking.TrackableAsset):
+    proto.asset.SetInParent()
+    proto.asset.relative_filename = filename_map[obj.asset_path.numpy()]
+  else:
+    proto.user_object.SetInParent()
 
 
 @tf_export("saved_model.save", v1=["saved_model.experimental.save"])
@@ -711,4 +729,4 @@ def save(obj, export_dir, signatures=None):
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))
   file_io.write_string_to_file(path, saved_model.SerializeToString())
-  _write_object_graph(obj, export_dir)
+  _write_object_graph(obj, export_dir, asset_filename_map)
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 89f82b90d6..1111c24ebe 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -18,21 +18,53 @@ package tensorflow;
 // NOTE: This protocol buffer format is experimental and subject to change.
 
 message SavedObjectGraph {
-  message SavedObject {
-    // Objects which this object depends on: named edges in the dependency
-    // graph.
-    repeated CheckpointableObjectGraph.CheckpointableObject.ObjectReference
-        children = 1;
-    // Removed when forking from CheckpointableObjectGraph.
-    reserved "attributes";
-    reserved 2;
-    // Slot variables owned by this object. This describes the three-way
-    // (optimizer, variable, slot variable) relationship; none of the three
-    // depend on the others directly.
-    repeated
-        CheckpointableObjectGraph.CheckpointableObject.SlotVariableReference
-            slot_variables = 3;
+  // List of objects in the SavedModel.
+  //
+  // The position of the object in this list indicates its id.
+  // Nodes[0] is considered the root node.
+  repeated SavedObject nodes = 1;
+}
+
+message SavedObject {
+  // Objects which this object depends on: named edges in the dependency
+  // graph.
+  //
+  // Note: only valid if kind == "object".
+  repeated CheckpointableObjectGraph.CheckpointableObject.ObjectReference
+      children = 1;
+
+  // Removed when forking from CheckpointableObjectGraph.
+  reserved "attributes";
+  reserved 2;
+
+  // Slot variables owned by this object. This describes the three-way
+  // (optimizer, variable, slot variable) relationship; none of the three
+  // depend on the others directly.
+  //
+  // Note: only valid if kind == "object".
+  repeated CheckpointableObjectGraph.CheckpointableObject.SlotVariableReference
+      slot_variables = 3;
+
+  oneof kind {
+    SavedUserObject user_object = 4;
+    SavedAsset asset = 5;
   }
+}
 
-  repeated SavedObject nodes = 1;
+// A SavedUserObject is an object (in the object-oriented language of the
+// TensorFlow program) of some user- or framework-defined class other than
+// those handled specifically by the other kinds of SavedObjects.
+//
+// This object cannot be evaluated as a tensor, and therefore cannot be bound
+// to an input of a function.
+message SavedUserObject {}
+
+// A SavedAsset represents a file in a SavedModel.
+//
+// When bound to a function this object evaluates to a Variable from which the
+// absolute filename can be read. Users should not expect the filename to be
+// maintained.
+message SavedAsset {
+  // Filename relative to the assets directory of the SavedModel.
+  string relative_filename = 1;
 }
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index a1cc6082fd..d183fbdcf9 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -649,8 +649,10 @@ def _add_attributes_to_object_graph(
   return named_saveable_objects, feed_additions
 
 
-def _fill_object_graph_proto(checkpointable_objects, node_ids, slot_variables,
-                             object_graph_proto=None):
+def fill_object_graph_proto(checkpointable_objects,
+                            node_ids,
+                            slot_variables,
+                            object_graph_proto=None):
   """Name non-slot `Checkpointable`s and add them to `object_graph_proto`."""
   if object_graph_proto is None:
     object_graph_proto = (
@@ -679,7 +681,7 @@ def _serialize_gathered_objects(
       checkpointable_objects=checkpointable_objects,
       node_ids=node_ids,
       object_names=object_names)
-  object_graph_proto = _fill_object_graph_proto(
+  object_graph_proto = fill_object_graph_proto(
       checkpointable_objects=checkpointable_objects,
       node_ids=node_ids,
       slot_variables=slot_variables)
@@ -732,7 +734,7 @@ def named_saveables(root_checkpointable):
   return _serialize_object_graph(root_checkpointable, None)[0]
 
 
-def _find_objects(root_checkpointable):
+def find_objects(root_checkpointable):
   """Find and number objects which are dependencies of `root_checkpointable`."""
   checkpointable_objects, path_to_root = (
       _breadth_first_checkpointable_traversal(root_checkpointable))
@@ -763,18 +765,10 @@ def list_objects(root_checkpointable):
   Returns:
     A flat list of objects.
   """
-  checkpointable_objects, _, _ = _find_objects(root_checkpointable)
+  checkpointable_objects, _, _ = find_objects(root_checkpointable)
   return checkpointable_objects
 
 
-def make_object_graph_without_attributes(root_checkpointable, proto=None):
-  """Fill an object graph proto, ignoring variable values."""
-  checkpointable_objects, node_ids, slot_variables = _find_objects(
-      root_checkpointable)
-  return _fill_object_graph_proto(
-      checkpointable_objects, node_ids, slot_variables, proto)
-
-
 def gather_initializers(root_checkpointable):
   """Traverse the object graph and find initialization ops.
 
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index de9cac0863..78047eda90 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -44,7 +44,6 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import adam
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import momentum
 from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.training import training_util
 from tensorflow.python.training.checkpointable import base
@@ -199,17 +198,6 @@ class InterfaceTests(test.TestCase):
     with self.assertRaises(NotImplementedError):
       checkpoint_reversed.save(prefix)
 
-  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
-  def test_object_graph_no_attributes(self):
-    root = tracking.Checkpointable()
-    root.v = resource_variable_ops.ResourceVariable(1.)
-    root.opt = momentum.MomentumOptimizer(0.01, 0.5)
-    root.opt.minimize(root.v.read_value)
-    object_graph = checkpointable_utils.make_object_graph_without_attributes(
-        root)
-    # Four objects: Root, v, opt, and a slot variable for v
-    self.assertEqual(4, len(object_graph.nodes))
-
 
 class _MirroringSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
 
-- 
GitLab


From 1f05b7eaf0220bc720cd3d0a8fcf241458c3f74d Mon Sep 17 00:00:00 2001
From: Vidak Kazic <vidak.kazic@hilda.awesome.ness>
Date: Wed, 5 Dec 2018 14:44:33 +0100
Subject: [PATCH 1349/1554] Add example to import_meta_graph docstring

---
 tensorflow/python/training/saver.py | 31 +++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index a29926a57d..5352c998ea 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1656,6 +1656,37 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
   NOTE: Restarting training from saved `meta_graph` only works if the
   device assignments have not changed.
 
+  Example 2:
+  Variables, placeholders, and independent operations can also be stored, as shown
+  in the following example.
+
+  ```Python
+  # Saving contents and operations.
+  v1 = tf.placeholder(tf.float32, name="v1")
+  v2 = tf.placeholder(tf.float32, name="v2")
+  v3 = tf.mul(v1, v2)
+  vx = tf.Variable(10.0, name="vx")
+  v4 = tf.add(v3, vx, name="v4")
+  saver = tf.train.Saver([vx])
+  sess = tf.Session()
+  sess.run(tf.initialize_all_variables())
+  sess.run(vx.assign(tf.add(vx, vx)))
+  result = sess.run(v4, feed_dict={v1:12.0, v2:3.3})
+  print(result)
+  saver.save(sess, "./model_ex1")
+  ```
+
+  Later this model can be restored and contents loaded.
+
+  ```Python
+  # Restoring variables and running operations.
+  saver = tf.train.import_meta_graph("./model_ex1.meta")
+  sess = tf.Session()
+  saver.restore(sess, "./model_ex1")
+  result = sess.run("v4:0", feed_dict={"v1:0": 12.0, "v2:0": 3.3})
+  print(result)
+  ```
+
   Args:
     meta_graph_or_file: `MetaGraphDef` protocol buffer or filename (including
       the path) containing a `MetaGraphDef`.
-- 
GitLab


From b6a6296de5b873107bae5abd30dff8897fa53b54 Mon Sep 17 00:00:00 2001
From: Vidak Kazic <vidakdk@gmail.com>
Date: Wed, 5 Dec 2018 15:01:56 +0100
Subject: [PATCH 1350/1554] Add example to import_meta_graph docstring

---
 tensorflow/python/training/saver.py | 31 +++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index a29926a57d..5352c998ea 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1656,6 +1656,37 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
   NOTE: Restarting training from saved `meta_graph` only works if the
   device assignments have not changed.
 
+  Example 2:
+  Variables, placeholders, and independent operations can also be stored, as shown
+  in the following example.
+
+  ```Python
+  # Saving contents and operations.
+  v1 = tf.placeholder(tf.float32, name="v1")
+  v2 = tf.placeholder(tf.float32, name="v2")
+  v3 = tf.mul(v1, v2)
+  vx = tf.Variable(10.0, name="vx")
+  v4 = tf.add(v3, vx, name="v4")
+  saver = tf.train.Saver([vx])
+  sess = tf.Session()
+  sess.run(tf.initialize_all_variables())
+  sess.run(vx.assign(tf.add(vx, vx)))
+  result = sess.run(v4, feed_dict={v1:12.0, v2:3.3})
+  print(result)
+  saver.save(sess, "./model_ex1")
+  ```
+
+  Later this model can be restored and contents loaded.
+
+  ```Python
+  # Restoring variables and running operations.
+  saver = tf.train.import_meta_graph("./model_ex1.meta")
+  sess = tf.Session()
+  saver.restore(sess, "./model_ex1")
+  result = sess.run("v4:0", feed_dict={"v1:0": 12.0, "v2:0": 3.3})
+  print(result)
+  ```
+
   Args:
     meta_graph_or_file: `MetaGraphDef` protocol buffer or filename (including
       the path) containing a `MetaGraphDef`.
-- 
GitLab


From 264e6520c56f588a7e0f222f657f291e4620965d Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Wed, 5 Dec 2018 07:06:42 -0800
Subject: [PATCH 1351/1554] Add TF_SetOutput

PiperOrigin-RevId: 224148089
---
 tensorflow/c/BUILD           |  2 ++
 tensorflow/c/kernels.cc      | 17 +++++++++++++++++
 tensorflow/c/kernels.h       |  8 ++++++++
 tensorflow/c/kernels_test.cc |  9 +++++++++
 4 files changed, 36 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 59c23e7c18..ad2ae08a37 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -189,11 +189,13 @@ tf_cuda_library(
         "//tensorflow:android": [
             ":c_api",
             ":c_api_internal",
+            ":tf_status_helper",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
             ":c_api",
             ":c_api_internal",
+            ":tf_status_helper",
             "//tensorflow/core:framework",
         ],
     }),
diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index 3caa5bcb03..2a4eaecb6c 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/kernels.h"
+#include "tensorflow/c/tf_status_helper.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 
@@ -141,3 +142,19 @@ void TF_GetInput(TF_OpKernelContext* ctx, int i, TF_Tensor** tensor,
     *tensor = result;
   }
 }
+
+void TF_SetOutput(TF_OpKernelContext* ctx, int i, const TF_Tensor* tensor,
+                  TF_Status* status) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  if (i < 0 || i >= cc_ctx->num_inputs()) {
+    TF_SetStatus(status, TF_OUT_OF_RANGE, "input index out of range");
+    return;
+  }
+  ::tensorflow::Tensor cc_tensor;
+  ::tensorflow::Status s = ::tensorflow::TF_TensorToTensor(tensor, &cc_tensor);
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(status, s);
+  if (s.ok()) {
+    cc_ctx->set_output(i, cc_tensor);
+  }
+}
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index d7778829bc..1a91aa184f 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -103,6 +103,14 @@ TF_CAPI_EXPORT extern int TF_NumOutputs(TF_OpKernelContext* ctx);
 TF_CAPI_EXPORT extern void TF_GetInput(TF_OpKernelContext* ctx, int i,
                                        TF_Tensor** tensor, TF_Status* status);
 
+// Sets the ith output of ctx to tensor. If TF_GetCode(status) is anything but
+// TF_OK, ctx is left unmodified.
+//
+// If i < 0 or i >= TF_NumOutputs(ctx), *status is set to TF_OUT_OF_RANGE.
+TF_CAPI_EXPORT extern void TF_SetOutput(TF_OpKernelContext* ctx, int i,
+                                        const TF_Tensor* tensor,
+                                        TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index 80bf12c096..e659ee3c3d 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -147,6 +147,14 @@ TEST(TestKernel, TestInputAndOutputCount) {
     EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
     TF_GetInput(ctx, 3, &input, s);
     EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
+
+    // Copy the input tensor to output.
+    TF_SetOutput(ctx, 0, input, s);
+    EXPECT_EQ(TF_OK, TF_GetCode(s));
+
+    TF_SetOutput(ctx, 24, input, s);
+    EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
+
     TF_DeleteStatus(s);
     if (input != nullptr) {
       TF_DeleteTensor(input);
@@ -188,6 +196,7 @@ TEST(TestKernel, TestInputAndOutputCount) {
 
     ASSERT_EQ(2, num_inputs);
     ASSERT_EQ(1, num_outputs);
+    ASSERT_EQ(123, ctx.mutable_output(0)->scalar<tensorflow::uint8>()());
   }
 }
 
-- 
GitLab


From cdfa6f2322f5064570b56cd63d83639b2e13cad6 Mon Sep 17 00:00:00 2001
From: RJ Ryan <rjryan@google.com>
Date: Wed, 5 Dec 2018 08:12:34 -0800
Subject: [PATCH 1352/1554] Automated rollback of commit
 43a1df9ac3df076e92655152500c9e9f5ba399be

PiperOrigin-RevId: 224156352
---
 tensorflow/python/BUILD                        |  1 -
 tensorflow/python/ops/control_flow_ops.py      |  2 ++
 tensorflow/python/ops/control_flow_ops_test.py | 13 -------------
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index dccbc40143..035830b911 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3251,7 +3251,6 @@ cuda_py_test(
         ":util",
         ":variable_scope",
         ":variables",
-        "//tensorflow/python/eager:def_function",
     ],
 )
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 75c081801a..b7e50c1dae 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -56,6 +56,7 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
 from tensorflow.python.util import nest
+from tensorflow.python.util import tf_should_use
 from tensorflow.python.util.lazy_loader import LazyLoader
 from tensorflow.python.util.tf_export import tf_export
 
@@ -112,6 +113,7 @@ def _summarize_eager(tensor, summarize=None):
 # Assert and Print are special symbols in python, so we must
 # use an upper-case version of them.
 @tf_export("debugging.Assert", "Assert")
+@tf_should_use.should_use_result
 def Assert(condition, data, summarize=None, name=None):
   """Asserts that the given condition is true.
 
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index b19ec4bd61..c020189ad6 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -24,7 +24,6 @@ import numpy as np
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import node_def_pb2
 from tensorflow.python.client import session
-from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -1015,18 +1014,6 @@ class AssertTest(test_util.TensorFlowTestCase):
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(c)
 
-  @test_util.run_in_graph_and_eager_modes
-  def testAssertInFunction(self):
-
-    @def_function.function
-    def whiny(value):
-      control_flow_ops.Assert(value, ["Raised false"])
-      return constant_op.constant(5)
-
-    with self.assertRaises(errors.InvalidArgumentError):
-      self.evaluate(whiny(False))
-
-    self.assertAllEqual(whiny(True), 5)
 
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From 56a5b2a87cc1782ee26da5a3535a61b5afd7bb8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 09:08:06 -0800
Subject: [PATCH 1353/1554] Enable all lite/kernels tests in oss

PiperOrigin-RevId: 224163963
---
 tensorflow/lite/kernels/BUILD          | 44 --------------------------
 tensorflow/lite/kernels/internal/BUILD | 11 -------
 2 files changed, 55 deletions(-)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 00d9d1feae..bad1c4aebf 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -26,7 +26,6 @@ tf_cc_test(
     size = "small",
     srcs = ["optional_tensor_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -124,7 +123,6 @@ tf_cc_test(
     size = "small",
     srcs = ["kernel_util_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -139,7 +137,6 @@ tf_cc_test(
     size = "small",
     srcs = ["test_util_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",  # TODO(b/117786830)
     ],
     deps = [
@@ -293,7 +290,6 @@ tf_cc_test(
     size = "small",
     srcs = ["audio_spectrogram_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -310,7 +306,6 @@ tf_cc_test(
     size = "small",
     srcs = ["mfcc_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -327,7 +322,6 @@ tf_cc_test(
     size = "small",
     srcs = ["detection_postprocess_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -344,7 +338,6 @@ tf_cc_test(
     size = "small",
     srcs = ["relu1_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -361,7 +354,6 @@ tf_cc_test(
     size = "small",
     srcs = ["sparse_output_fully_connected_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -404,7 +396,6 @@ tf_cc_test(
     size = "small",
     srcs = ["arg_min_max_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -420,7 +411,6 @@ tf_cc_test(
     size = "small",
     srcs = ["div_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -436,7 +426,6 @@ tf_cc_test(
     size = "small",
     srcs = ["sub_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -452,7 +441,6 @@ tf_cc_test(
     size = "small",
     srcs = ["transpose_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -470,7 +458,6 @@ tf_cc_test(
     size = "small",
     srcs = ["space_to_batch_nd_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -486,7 +473,6 @@ tf_cc_test(
     size = "small",
     srcs = ["batch_to_space_nd_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -502,7 +488,6 @@ tf_cc_test(
     size = "small",
     srcs = ["cast_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -559,7 +544,6 @@ tf_cc_test(
     size = "small",
     srcs = ["dequantize_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -589,7 +573,6 @@ tf_cc_test(
     size = "small",
     srcs = ["bidirectional_sequence_lstm_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -606,7 +589,6 @@ tf_cc_test(
     size = "small",
     srcs = ["floor_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -622,7 +604,6 @@ tf_cc_test(
     size = "small",
     srcs = ["elementwise_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -638,7 +619,6 @@ tf_cc_test(
     size = "small",
     srcs = ["unidirectional_sequence_lstm_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -654,7 +634,6 @@ tf_cc_test(
     size = "small",
     srcs = ["bidirectional_sequence_rnn_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -670,7 +649,6 @@ tf_cc_test(
     size = "small",
     srcs = ["unidirectional_sequence_rnn_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -699,7 +677,6 @@ tf_cc_test(
     size = "small",
     srcs = ["exp_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -715,7 +692,6 @@ tf_cc_test(
     size = "small",
     srcs = ["fake_quant_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -731,7 +707,6 @@ tf_cc_test(
     size = "small",
     srcs = ["maximum_minimum_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -747,7 +722,6 @@ tf_cc_test(
     size = "small",
     srcs = ["reduce_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -776,7 +750,6 @@ tf_cc_test(
     size = "small",
     srcs = ["pad_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -805,7 +778,6 @@ tf_cc_test(
     size = "small",
     srcs = ["gather_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -822,7 +794,6 @@ tf_cc_test(
     size = "small",
     srcs = ["topk_v2_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -959,7 +930,6 @@ tf_cc_test(
     size = "small",
     srcs = ["log_softmax_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1057,7 +1027,6 @@ tf_cc_test(
     size = "small",
     srcs = ["split_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1073,7 +1042,6 @@ tf_cc_test(
     size = "small",
     srcs = ["split_v_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1089,7 +1057,6 @@ tf_cc_test(
     size = "small",
     srcs = ["squeeze_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1105,7 +1072,6 @@ tf_cc_test(
     size = "small",
     srcs = ["strided_slice_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1121,7 +1087,6 @@ tf_cc_test(
     size = "small",
     srcs = ["tile_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1140,7 +1105,6 @@ tf_cc_test(
         "comparisons_test.cc",
     ],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1156,7 +1120,6 @@ tf_cc_test(
     size = "small",
     srcs = ["neg_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1174,7 +1137,6 @@ tf_cc_test(
         "select_test.cc",
     ],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1192,7 +1154,6 @@ tf_cc_test(
         "slice_test.cc",
     ],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1208,7 +1169,6 @@ tf_cc_test(
     size = "small",
     srcs = ["transpose_conv_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1225,7 +1185,6 @@ tf_cc_test(
     size = "small",
     srcs = ["expand_dims_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1242,7 +1201,6 @@ tf_cc_test(
     size = "small",
     srcs = ["sparse_to_dense_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1259,7 +1217,6 @@ tf_cc_test(
     size = "small",
     srcs = ["shape_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -1276,7 +1233,6 @@ tf_cc_test(
     size = "small",
     srcs = ["pow_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 7d2653f0a1..d2f7f455e4 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -254,7 +254,6 @@ cc_test(
     name = "tensor_test",
     srcs = ["tensor_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",  # TODO(b/117786830)
     ],
     deps = [
@@ -287,7 +286,6 @@ cc_test(
     name = "quantization_util_test",
     srcs = ["quantization_util_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",  # TODO(b/117786830)
     ],
     deps = [
@@ -562,7 +560,6 @@ cc_test(
     }),
     linkstatic = 1,
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -576,7 +573,6 @@ cc_test(
 cc_test(
     name = "depthwiseconv_float_test",
     srcs = ["depthwiseconv_float_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -591,7 +587,6 @@ cc_test(
     srcs = ["depthwiseconv_quantized_test.cc"],
     shard_count = 2,
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -607,7 +602,6 @@ cc_test(
     name = "resize_bilinear_test",
     srcs = ["resize_bilinear_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -623,7 +617,6 @@ cc_test(
     name = "resize_nearest_neighbor_test",
     srcs = ["resize_nearest_neighbor_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -641,7 +634,6 @@ cc_test(
     srcs = [
         "softmax_quantized_test.cc",
     ],
-    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":quantization_util",
@@ -659,7 +651,6 @@ cc_test(
         "logsoftmax_quantized_test.cc",
     ],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -675,7 +666,6 @@ cc_test(
 cc_test(
     name = "log_quantized_test",
     srcs = ["log_quantized_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -703,7 +693,6 @@ cc_library(
 cc_test(
     name = "batch_to_space_nd_test",
     srcs = ["batch_to_space_nd_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         "@com_google_googletest//:gtest_main",
-- 
GitLab


From bf6e796e9b481facf48546ad30a886d85b21343b Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 5 Dec 2018 09:16:36 -0800
Subject: [PATCH 1354/1554] Enable test cases of conv_depthwise_test which work
 fine on the GPU backend.

PiperOrigin-RevId: 224165207
---
 tensorflow/compiler/xla/tests/BUILD                  |  4 ----
 tensorflow/compiler/xla/tests/conv_depthwise_test.cc | 12 +++++++++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index f7f090fe4a..5a7a4faa7e 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -303,10 +303,6 @@ xla_test(
     name = "conv_depthwise_test",
     timeout = "long",
     srcs = ["conv_depthwise_test.cc"],
-    blacklisted_backends = [
-        # disabled because of a break b/119590850.
-        "gpu",
-    ],
     shard_count = 50,
     deps = [
         "//tensorflow/compiler/xla:execution_options_util",
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
index bc9bd8a269..4cc045300f 100644
--- a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
@@ -50,9 +50,15 @@ class DepthwiseConvolution2DTest
 static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
   std::vector<DepthwiseConvolution2DSpec> config_set;
   std::vector<std::vector<int64>> config_options = {
-      {128, 6, 3, 64},  {256, 5, 3, 256},  {256, 5, 2, 144}, {144, 5, 3, 64},
-      {144, 5, 2, 256}, {8, 48, 17, 8},    {128, 20, 6, 64}, {128, 1, 2, 144},
-      {256, 1, 2, 64},  {64, 14, 12, 172}, {16, 9, 4, 16}};
+      {128, 6, 3, 64},  {256, 5, 3, 256},  {256, 5, 2, 144},
+      {144, 5, 3, 64},  {144, 5, 2, 256},  {8, 48, 17, 8},
+      {128, 20, 6, 64}, {64, 14, 12, 172}, {16, 9, 4, 16}};
+
+#ifndef XLA_TEST_BACKEND_GPU
+  // TODO(b/119590850): Figure out why this fails on the GPU backend.
+  config_options.push_back({128, 1, 2, 144});
+  config_options.push_back({256, 1, 2, 64});
+#endif
 
   for (auto option : config_options) {
     int64 feature = option[0];
-- 
GitLab


From 847b5f842d8d499bccfb6f9d88df3cff8d0fffac Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Wed, 5 Dec 2018 09:35:32 -0800
Subject: [PATCH 1355/1554] Add test showing conversion from an intermediate
 input array.

PiperOrigin-RevId: 224167965
---
 tensorflow/lite/python/lite_test.py | 32 +++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 1ae0d3c3ed..2c630588ca 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -177,6 +177,38 @@ class FromSessionTest(test_util.TensorFlowTestCase):
         'Quantization input stats are not available for input tensors '
         '\'inputB\'.', str(error.exception))
 
+  def testIntermediateInputArray(self):
+    """Convert a model from an intermediate input array."""
+    in_tensor_init = array_ops.placeholder(
+        shape=[1, 16, 16, 3], dtype=dtypes.float32)
+    in_tensor_final = in_tensor_init + in_tensor_init
+    out_tensor = in_tensor_final + in_tensor_final
+    sess = session.Session()
+
+    # Convert model and ensure model is not None.
+    converter = lite.TFLiteConverter.from_session(sess, [in_tensor_final],
+                                                  [out_tensor])
+    tflite_model = converter.convert()
+    self.assertTrue(tflite_model)
+
+    # Check values from converted model.
+    interpreter = Interpreter(model_content=tflite_model)
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(1, len(input_details))
+    self.assertEqual('add', input_details[0]['name'])
+    self.assertEqual(np.float32, input_details[0]['dtype'])
+    self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all())
+    self.assertEqual((0., 0.), input_details[0]['quantization'])
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('add_1', output_details[0]['name'])
+    self.assertEqual(np.float32, output_details[0]['dtype'])
+    self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all())
+    self.assertEqual((0., 0.), output_details[0]['quantization'])
+
   def testSizeNoneInvalid(self):
     in_tensor = array_ops.placeholder(dtype=dtypes.float32)
     out_tensor = in_tensor + in_tensor
-- 
GitLab


From 3f172b5db576d548007230c385db9e8be76d540c Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 5 Dec 2018 09:40:24 -0800
Subject: [PATCH 1356/1554] Fix memory leak in `XlaTensorBuffer`.

My recent change to devirtualize `TensorBuffer::data()` had a bug, in that it continued to use the now-uninitialized `XlaTensorBuffer::data_` field to decide whether to free the buffer.

PiperOrigin-RevId: 224168862
---
 tensorflow/compiler/jit/xla_launch_util.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 706811f55e..554227f09d 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -205,7 +205,7 @@ class XlaTensorBuffer : public TensorBuffer {
         allocator_(allocator) {}
 
   ~XlaTensorBuffer() override {
-    if (data_) {
+    if (data()) {
       allocator_->DeallocateRaw(data());
     }
   }
@@ -229,7 +229,6 @@ class XlaTensorBuffer : public TensorBuffer {
   }
 
  private:
-  void* data_;
   size_t expected_size_;
   size_t actual_size_;
   Allocator* allocator_;
-- 
GitLab


From 47ec1f4bb9e9328d0627e1775f957920884ad706 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <1990079+smit-hinsu@users.noreply.github.com>
Date: Thu, 6 Dec 2018 00:00:39 +0530
Subject: [PATCH 1357/1554] Add @smit-hinsu and @azaks2 to tensorrt CODEOWNERS

---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index 54a61a4d72..e854a2e827 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -57,7 +57,7 @@
 /tensorflow/contrib/slim/ @sguada @thenbasilmanran
 /tensorflow/contrib/stateless/ @girving @alextp
 /tensorflow/contrib/tensor_forest/ @gilberthendry @thomascolthurst @yupbank
-/tensorflow/contrib/tensorrt/ @aaroey
+/tensorflow/contrib/tensorrt/ @aaroey @smit-hinsu @azaks2
 # NEED OWNER: /tensorflow/contrib/testing/
 /tensorflow/contrib/timeseries/ @allenlavoie
 /tensorflow/contrib/tpu/ @frankchn @saeta @jhseu @sourabhbajaj
-- 
GitLab


From ad1169e1c93f5522fecbb410fc9d5303202dc115 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Wed, 5 Dec 2018 10:50:29 -0800
Subject: [PATCH 1358/1554] Use the getNext in the testbase to avoid creating
 multiple IteratorGetnext ops in graph mode

---
 .../kernel_tests/make_csv_dataset_test.py     | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index f541c84dfa..ae0b7be51b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -83,7 +83,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
       expected_output,
       expected_keys,
   ):
-    iterator = dataset_ops.make_one_shot_iterator(dataset)
+    get_next = self.getNext(dataset)
 
     for expected_features in self._next_expected_batch(
         expected_output,
@@ -91,8 +91,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         batch_size,
         num_epochs,
     ):
-      nxt = iterator.get_next()
-      actual_features = self.evaluate(nxt)
+      actual_features = self.evaluate(get_next())
 
       if label_name is not None:
         expected_labels = expected_features.pop(label_name)
@@ -104,8 +103,7 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
         self.assertAllEqual(expected_features[k], actual_features[k])
 
     with self.assertRaises(errors.OutOfRangeError):
-      nxt = iterator.get_next()
-      self.evaluate(nxt)
+      self.evaluate(get_next())
 
   def _test_dataset(self,
                     inputs,
@@ -603,11 +601,11 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           shuffle_seed=5,
           num_epochs=2,
       )
-      outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
-      outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
+      next1 = self.getNext(dataset1)
+      next2 = self.getNext(dataset2)
       for _ in range(total_records // batch_size):
-        batch1 = nest.flatten(self.evaluate(outputs1))
-        batch2 = nest.flatten(self.evaluate(outputs2))
+        batch1 = nest.flatten(self.evaluate(next1()))
+        batch2 = nest.flatten(self.evaluate(next2()))
         for i in range(len(batch1)):
           self.assertAllEqual(batch1[i], batch2[i])
 
@@ -632,12 +630,12 @@ class MakeCsvDatasetTest(test_base.DatasetTestBase):
           shuffle_seed=6,
           num_epochs=2,
       )
-      outputs1 = dataset_ops.make_one_shot_iterator(dataset1).get_next()
-      outputs2 = dataset_ops.make_one_shot_iterator(dataset2).get_next()
+      next1 = self.getNext(dataset1)
+      next2 = self.getNext(dataset2)
       all_equal = False
       for _ in range(total_records // batch_size):
-        batch1 = nest.flatten(self.evaluate(outputs1))
-        batch2 = nest.flatten(self.evaluate(outputs2))
+        batch1 = nest.flatten(self.evaluate(next1()))
+        batch2 = nest.flatten(self.evaluate(next2()))
         for i in range(len(batch1)):
           all_equal = all_equal and np.array_equal(batch1[i], batch2[i])
       self.assertFalse(all_equal)
-- 
GitLab


From 81b60bc6bab55c893beb7171289a6248d46cec25 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 10:49:22 -0800
Subject: [PATCH 1359/1554] Enable more tests in OSS

PiperOrigin-RevId: 224182173
---
 tensorflow/lite/BUILD             | 68 ++++++++++++++-----------------
 tensorflow/lite/toco/BUILD        |  3 --
 tensorflow/lite/toco/tflite/BUILD |  4 --
 3 files changed, 30 insertions(+), 45 deletions(-)

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index bb2c53b8c9..af1cfe27a6 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -1,11 +1,11 @@
-package(default_visibility = [
-    "//visibility:public",
-])
+package(
+    default_visibility = ["//visibility:public"],
+)
 
 licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load("//tensorflow/lite:build_def.bzl", "tflite_copts", "gen_selected_ops")
+load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 
 exports_files(glob([
     "testdata/*.bin",
@@ -35,15 +35,22 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+TFLITE_DEFAULT_COPTS = [
+    "-Wall",
+    "-Wno-comment",
+]
+
 cc_library(
     name = "schema_fbs_version",
     hdrs = ["version.h"],
+    copts = TFLITE_DEFAULT_COPTS,
 )
 
 cc_library(
     name = "arena_planner",
     srcs = ["arena_planner.cc"],
     hdrs = ["arena_planner.h"],
+    copts = TFLITE_DEFAULT_COPTS,
     deps = [
         ":graph_info",
         ":memory_planner",
@@ -57,12 +64,10 @@ cc_test(
     size = "small",
     srcs = ["arena_planner_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
         ":arena_planner",
-        "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
@@ -74,18 +79,21 @@ cc_test(
 cc_library(
     name = "context",
     hdrs = ["context.h"],
+    copts = TFLITE_DEFAULT_COPTS,
     deps = ["//tensorflow/lite/c:c_api_internal"],
 )
 
 cc_library(
     name = "graph_info",
     hdrs = ["graph_info.h"],
+    copts = TFLITE_DEFAULT_COPTS,
     deps = ["//tensorflow/lite/c:c_api_internal"],
 )
 
 cc_library(
     name = "memory_planner",
     hdrs = ["memory_planner.h"],
+    copts = TFLITE_DEFAULT_COPTS,
     deps = ["//tensorflow/lite/c:c_api_internal"],
 )
 
@@ -93,6 +101,7 @@ cc_library(
     name = "simple_memory_arena",
     srcs = ["simple_memory_arena.cc"],
     hdrs = ["simple_memory_arena.h"],
+    copts = TFLITE_DEFAULT_COPTS,
     deps = ["//tensorflow/lite/c:c_api_internal"],
 )
 
@@ -109,9 +118,9 @@ cc_library(
     hdrs = [
         "builtin_op_data.h",
         "builtin_ops.h",
-        "context.h",
         "context_util.h",
     ],
+    deps = ["//tensorflow/lite/c:c_api_internal"],
 )
 
 exports_files(["builtin_ops.h"])
@@ -121,9 +130,7 @@ cc_library(
     hdrs = [
         "string.h",
     ],
-    deps = [
-        "//tensorflow/core:lib_platform",
-    ],
+    copts = TFLITE_DEFAULT_COPTS,
 )
 
 # TODO(ahentz): investigate dependency on gemm_support requiring usage of tf_copts.
@@ -167,7 +174,7 @@ cc_library(
         "optional_debug_tools.h",
         "stderr_reporter.h",
     ],
-    copts = tflite_copts(),
+    copts = tflite_copts() + TFLITE_DEFAULT_COPTS,
     linkopts = [
     ] + select({
         "//tensorflow:android": [
@@ -185,7 +192,7 @@ cc_library(
         ":string",
         ":util",
         "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/core/api:api",
         "//tensorflow/lite/kernels:eigen_support",
         "//tensorflow/lite/kernels:gemm_support",
         "//tensorflow/lite/nnapi:nnapi_lib",
@@ -203,10 +210,10 @@ cc_library(
     name = "string_util",
     srcs = ["string_util.cc"],
     hdrs = ["string_util.h"],
-    copts = tflite_copts(),
+    copts = TFLITE_DEFAULT_COPTS,
     deps = [
-        ":framework",
         ":string",
+        "//tensorflow/lite/c:c_api_internal",
     ],
 )
 
@@ -217,6 +224,7 @@ cc_test(
     deps = [
         ":framework",
         ":string_util",
+        "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
@@ -246,10 +254,8 @@ cc_test(
     name = "graph_info_test",
     size = "small",
     srcs = ["graph_info_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":framework",
-        ":string_util",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
@@ -298,7 +304,12 @@ tf_cc_test(
     data = [
         "testdata/multi_add_flex.bin",
     ],
-    tags = ["no_windows"],  # TODO(b/116667551): No weak symbols with MSVC.
+    tags = [
+        "no_gpu",  # GPU + flex is not officially supported.
+        "no_windows",  # TODO(b/116667551): No weak symbols with MSVC.
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":framework",
         "//tensorflow/lite/core/api",
@@ -314,7 +325,6 @@ cc_test(
     name = "mutable_op_resolver_test",
     size = "small",
     srcs = ["mutable_op_resolver_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":framework",
         "//tensorflow/lite/testing:util",
@@ -326,7 +336,7 @@ cc_library(
     name = "util",
     srcs = ["util.cc"],
     hdrs = ["util.h"],
-    copts = tflite_copts(),
+    copts = TFLITE_DEFAULT_COPTS + tflite_copts(),
     deps = [
         "//tensorflow/lite/c:c_api_internal",
     ],
@@ -336,27 +346,9 @@ cc_test(
     name = "util_test",
     size = "small",
     srcs = ["util_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":util",
-        "//tensorflow/lite/testing:util",
+        "//tensorflow/lite/c:c_api_internal",
         "@com_google_googletest//:gtest",
     ],
 )
-
-# Test the serialization of a model with optional tensors.
-
-# Model tests
-
-#cc_library(
-#    name = "models_test_utils",
-#    testonly = 1,
-#    hdrs = ["models/test_utils.h"],
-#    deps = select({
-#        "//tensorflow:android": [],
-#        "//conditions:default": [
-#            "@com_google_absl//absl/strings",
-#            "//tensorflow/core:test",
-#        ],
-#    }),
-#)
diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 82aa1f557e..93d41fcae1 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -341,7 +341,6 @@ cc_library(
 tf_cc_test(
     name = "import_tensorflow_test",
     srcs = ["import_tensorflow_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":toco_tooling",
         "//tensorflow/core:framework",
@@ -384,7 +383,6 @@ cc_library(
 tf_cc_test(
     name = "tooling_util_test",
     srcs = ["tooling_util_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":model",
         ":tooling_util",
@@ -468,7 +466,6 @@ tf_cc_test(
     data = [
         "toco_port_test.cc",
     ],
-    tags = ["no_oss"],
     deps = [
         ":toco_port",
         "@com_google_googletest//:gtest_main",
diff --git a/tensorflow/lite/toco/tflite/BUILD b/tensorflow/lite/toco/tflite/BUILD
index 99c4f8edeb..36ca638ee8 100644
--- a/tensorflow/lite/toco/tflite/BUILD
+++ b/tensorflow/lite/toco/tflite/BUILD
@@ -41,7 +41,6 @@ tf_cc_test(
     srcs = [
         "operator_test.cc",
     ],
-    tags = ["no_oss"],
     deps = [
         ":operator",
         "//tensorflow/core:ops",
@@ -72,7 +71,6 @@ tf_cc_test(
     srcs = [
         "types_test.cc",
     ],
-    tags = ["no_oss"],
     deps = [
         ":types",
         "//tensorflow/core:ops",
@@ -107,7 +105,6 @@ tf_cc_test(
     srcs = [
         "export_test.cc",
     ],
-    tags = ["no_oss"],
     deps = [
         ":export",
         "//tensorflow/core:ops",
@@ -142,7 +139,6 @@ tf_cc_test(
     srcs = [
         "import_test.cc",
     ],
-    tags = ["no_oss"],
     deps = [
         ":import",
         "//tensorflow/core:ops",
-- 
GitLab


From aa01e01e66eeb4786c46ebd49caa43571b9f8865 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Wed, 5 Dec 2018 10:58:19 -0800
Subject: [PATCH 1360/1554] [XLA] InterpreterExecutable should reset the visit
 states of the evaluator

Resetting the visit states is required if we want to be able to ExecuteOnStream the same executable more than once.

PiperOrigin-RevId: 224184063
---
 tensorflow/compiler/xla/service/interpreter/executable.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/interpreter/executable.cc b/tensorflow/compiler/xla/service/interpreter/executable.cc
index 7635fbfed6..de9204011c 100644
--- a/tensorflow/compiler/xla/service/interpreter/executable.cc
+++ b/tensorflow/compiler/xla/service/interpreter/executable.cc
@@ -85,6 +85,7 @@ StatusOr<ScopedShapedBuffer> InterpreterExecutable::ExecuteOnStream(
   Literal result_literal;
   {
     tensorflow::mutex_lock lock(evaluator_lock_);
+    evaluator_->ResetVisitStates();
     TF_ASSIGN_OR_RETURN(result_literal, evaluator_->Evaluate<Literal>(
                                             *computation, arg_literals));
   }
-- 
GitLab


From 82e302a84e06342846ab809e1f0c0bdb9e0332a9 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 5 Dec 2018 11:03:06 -0800
Subject: [PATCH 1361/1554] Replace FrameMap with FrameView
 (NodeMap->GraphView)

PiperOrigin-RevId: 224185159
---
 tensorflow/core/grappler/optimizers/BUILD     |   3 -
 .../grappler/optimizers/layout_optimizer.cc   |  11 +-
 .../grappler/optimizers/loop_optimizer.cc     |  25 +-
 .../core/grappler/optimizers/loop_optimizer.h |   1 -
 .../optimizers/loop_optimizer_test.cc         | 301 ++++++++++--------
 .../optimizers/scoped_allocator_optimizer.cc  |  60 ++--
 tensorflow/core/grappler/utils/BUILD          |   3 +-
 tensorflow/core/grappler/utils/frame.cc       | 129 +++++---
 tensorflow/core/grappler/utils/frame.h        |  49 ++-
 tensorflow/core/grappler/utils/frame_test.cc  | 130 ++++----
 10 files changed, 395 insertions(+), 317 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 0f9dfe77fd..79578cb3ce 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -461,7 +461,6 @@ cc_library(
         "//tensorflow/core/grappler:devices",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/costs:virtual_placer",
@@ -615,7 +614,6 @@ cc_library(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:frame",
         "@com_google_absl//absl/container:flat_hash_set",
@@ -767,7 +765,6 @@ cc_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:frame",
     ],
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index f4653505f7..8f25a1c8c1 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/layout_optimizer.h"
-#include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -1969,9 +1968,9 @@ class DataLayoutOptimizer : GraphProcessor {
   // Expand all nodes which is in NHWC, but supports NCHW or is layout agnostic.
   Status Expand() {
     int node_size_original = graph_->node_size();
-    std::unordered_map<const NodeDef*, std::vector<int>> frames;
-    int num_frames;
-    TF_RETURN_IF_ERROR(IdentifyFrames(*graph_, &frames, &num_frames));
+
+    FrameView frame_view;
+    TF_RETURN_IF_ERROR(frame_view.InferFromGraph(*graph_));
 
     // This is the first pass where we expand the nodes which support NCHW.
     std::set<string> ops_format_supported = GetOpsFormatSupported();
@@ -1983,7 +1982,7 @@ class DataLayoutOptimizer : GraphProcessor {
       if (ops_format_supported.find(graph_->node(i).op()) !=
           ops_format_supported.end()) {
         auto node = graph_->mutable_node(i);
-        bool is_in_frame = !frames[node].empty();
+        bool is_in_frame = frame_view.IsInFrame(*node);
         OptimizeContext opt_cxt(graph_, node, node_map_, graph_properties_,
                                 virtual_placer_, nodes_to_preserve_,
                                 is_in_frame);
@@ -2033,7 +2032,7 @@ class DataLayoutOptimizer : GraphProcessor {
         if (ops_format_agnostic.find(graph_->node(i).op()) !=
             ops_format_agnostic.end()) {
           auto node = graph_->mutable_node(i);
-          bool is_in_frame = !frames[node].empty();
+          bool is_in_frame = frame_view.IsInFrame(*node);
           OptimizeContext opt_cxt(graph_, node, node_map_, graph_properties_,
                                   virtual_placer_, nodes_to_preserve_,
                                   is_in_frame);
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index 775fb9a95f..3606473840 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -35,7 +35,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/optimizers/evaluation_utils.h"
-#include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
@@ -380,14 +379,14 @@ Status LoopInvariantNodeMotionOptimizer::FindInvariantNodes(
 
 Status LoopInvariantNodeMotionOptimizer::Optimize() {
   node_map_.reset(new NodeMap(optimized_graph_));
-  FrameMap frame_map;
-  int num_frames;
-  TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_,
-                                               &frame_map, &num_frames));
+  FrameView frame_view;
+  // TODO(ezhulenev): Use GraphView when migrated from NodeMap.
+  TF_RETURN_IF_ERROR(frame_view.InferFromGraph(*optimized_graph_));
+
   std::deque<int> worklist;
-  for (auto iter = frame_map.begin(); iter != frame_map.end(); ++iter) {
-    auto* node = iter->first;
-    auto& frame_ids = iter->second;
+  for (const NodeDef& node : optimized_graph_->node()) {
+    const std::vector<int>& frame_ids = frame_view.Frames(node);
+
     if (frame_ids.size() >= 3) {
       for (unsigned int i = 1; i < frame_ids.size() - 1; ++i) {
         frame_parent_[frame_ids[i]] = frame_ids[i - 1];
@@ -400,18 +399,18 @@ Status LoopInvariantNodeMotionOptimizer::Optimize() {
     }
     if (!frame_ids.empty()) {
       frame_children_.insert(std::make_pair(frame_ids.back(), empty_set_));
-      if (node->op() == "LoopCond") {
+      if (node.op() == "LoopCond") {
         if (loop_cond_.count(frame_ids.back())) {
           return errors::InvalidArgument(
               "Loop ", frame_ids.back(),
-              " has more than one LoopCond node: ", node->name(), " and ",
+              " has more than one LoopCond node: ", node.name(), " and ",
               loop_cond_[frame_ids.back()]->name());
         }
-        loop_cond_[frame_ids.back()] = node;
+        loop_cond_[frame_ids.back()] = &node;
       }
-      if (IsEnter(*node) && node->attr().at("is_constant").b()) {
+      if (IsEnter(node) && node.attr().at("is_constant").b()) {
         invariant_enters_[frame_ids.back()].push_back(
-            const_cast<NodeDef*>(node));
+            const_cast<NodeDef*>(&node));
       }
     }
   }
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.h b/tensorflow/core/grappler/optimizers/loop_optimizer.h
index 7c04f55381..d467237a9a 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.h
@@ -19,7 +19,6 @@ limitations under the License.
 #include <unordered_set>
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
-#include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
index 7332839128..587767c23c 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
@@ -101,27 +101,30 @@ TEST_F(LoopOptimizerTest, Basic) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).back(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("VariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("VariantAdd")).back(), 0);
+  }
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).back(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd")).back(), 0);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd")).back(), 0);
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("VariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("VariantAdd")).back(), 0);
+  }
 }
 
 TEST_F(LoopOptimizerTest, Const) {
@@ -149,26 +152,29 @@ TEST_F(LoopOptimizerTest, Const) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).back(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("Const")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("Const")).back(), 0);
+  }
+
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).back(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const")).back(), 0);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const")).size(), 0);
+    EXPECT_EQ(frames.num_frames(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("Const")).size(), 0);
+  }
 }
 
 TEST_F(LoopOptimizerTest, ControlOutput) {
@@ -197,24 +203,27 @@ TEST_F(LoopOptimizerTest, ControlOutput) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).back(), 0);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).back(), 0);
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).back(), 0);
+  }
+
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).back(), 0);
+  }
 }
 
 TEST_F(LoopOptimizerTest, NestedLoop1) {
@@ -258,31 +267,34 @@ TEST_F(LoopOptimizerTest, NestedLoop1) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).back(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).back(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).back(), 0);
+  }
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).back(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).back(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).back(), 0);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).back(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).back(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd")).size(), 0);
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).back(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).back(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd")).size(), 0);
+  }
 }
 
 TEST_F(LoopOptimizerTest, NestedLoop2) {
@@ -326,27 +338,30 @@ TEST_F(LoopOptimizerTest, NestedLoop2) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).back(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).back(), 1);
+  }
+
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).back(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).back(), 1);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("VariantAdd2")).back(), 1);
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("VariantAdd2")).back(), 1);
+  }
 }
 
 TEST_F(LoopOptimizerTest, NestedLoopConst1) {
@@ -390,28 +405,31 @@ TEST_F(LoopOptimizerTest, NestedLoopConst1) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).back(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("Const2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("Const2")).back(), 1);
+  }
+
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).back(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).back(), 1);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).back(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).size(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).back(), 0);
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).back(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("Const2")).size(), 1);
+    EXPECT_EQ(frames.Frames(*view.GetNode("Const2")).back(), 0);
+  }
 }
 
 TEST_F(LoopOptimizerTest, NestedLoopConst2) {
@@ -455,26 +473,29 @@ TEST_F(LoopOptimizerTest, NestedLoopConst2) {
   LoopOptimizer optimizer;
   EnableOnlyLoopInvariantNodeMotion(&optimizer);
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  {  // Original graph.
+    GraphView view(&graph);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).back(), 1);
+    ASSERT_EQ(frames.Frames(*view.GetNode("Const2")).size(), 2);
+    EXPECT_EQ(frames.Frames(*view.GetNode("Const2")).back(), 1);
+  }
 
-  std::unique_ptr<NodeMap> node_map;
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-
-  node_map.reset(new NodeMap(&graph));
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).back(), 1);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).size(), 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).back(), 1);
-
-  node_map.reset(new NodeMap(&output));
-  EXPECT_TRUE(IdentifyFrames(output, &frames, &num_frames).ok());
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.at(node_map->GetNode("InvariantAdd2")).size(), 0);
-  EXPECT_EQ(frames.at(node_map->GetNode("Const2")).size(), 0);
+  {  // Optimized graph.
+    GraphView view(&output);
+    FrameView frames;
+    TF_EXPECT_OK(frames.InferFromGraphView(view));
+
+    EXPECT_EQ(frames.num_frames(), 2);
+    ASSERT_EQ(frames.Frames(*view.GetNode("InvariantAdd2")).size(), 0);
+    ASSERT_EQ(frames.Frames(*view.GetNode("Const2")).size(), 0);
+  }
 }
 
 void VerifyGraphsEqual(const GraphDef& original_graph,
diff --git a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc
index 0d4aaf6462..e537b3df07 100644
--- a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc
@@ -790,20 +790,17 @@ Tree* ComputeScopeTree(const string& op_name,
   return root;
 }
 
-void PartitionByLoopStructure(const FrameMap& frame_map,
+void PartitionByLoopStructure(const FrameView& frame_view,
                               std::vector<NodeDef*> nodes,
                               std::vector<std::vector<NodeDef*>>* loop_groups) {
   // It is assumed that two nodes with identical loop containment have
-  // identical integer vectors.  Represent those by 64 bit hashes.
+  // identical integer vectors. Represent those by 64 bit hashes.
   std::unordered_map<uint64, std::vector<NodeDef*>> loop_sets;
   for (NodeDef* nd : nodes) {
     uint64 hash = 0;
-    const auto& it = frame_map.find(nd);
-    if (it != frame_map.end()) {
-      const std::vector<int>& loop_ids = it->second;
-      for (int id : loop_ids) {
-        hash = Hash64Combine(hash, static_cast<uint64>(id));
-      }
+    const std::vector<int>& loop_ids = frame_view.Frames(*nd);
+    for (int id : loop_ids) {
+      hash = Hash64Combine(hash, static_cast<uint64>(id));
     }
     loop_sets[hash].push_back(nd);
   }
@@ -821,10 +818,11 @@ Status ScopedAllocatorOptimizer::ProcessGraphDef(
   GraphOpOccurrences occ;
   FindOpOccurrences(graph, op_name_set_, &occ);
   if (!occ.empty()) {
-    FrameMap frame_map;
-    int num_frames;
-    LOG_WARNING_AND_RETURN_IF_ERROR(
-        IdentifyFramesWithNodeMap(*graph, *node_map_, &frame_map, &num_frames));
+    FrameView frame_view;
+    // TODO(ezhulenev): Pass a GraphView when this optimizer will be migrated
+    // from NodeMap.
+    LOG_WARNING_AND_RETURN_IF_ERROR(frame_view.InferFromGraph(*graph));
+
     for (auto& dt : occ) {
       VLOG(2) << "Processing device " << dt.first;
       const DevOpOccurrences& dev_occ = dt.second;
@@ -841,26 +839,26 @@ Status ScopedAllocatorOptimizer::ProcessGraphDef(
         // Nodes with a common depth and root path are now grouped
         // in the same Tree struct.  Split those groups into subgroups that
         // share identical loop nesting.
-        status = ApplyToAll(
-            root.get(), [this, rewriter, graph, &frame_map, &op_name](Tree* t) {
-              VLOG(2) << "applied to tree node " << t->edge_ << " at depth "
-                      << t->depth_ << " of size " << t->nodes_.size();
-              if (t->nodes_.size() > 1) {
-                std::vector<std::vector<NodeDef*>> loop_groups;
-                PartitionByLoopStructure(frame_map, t->nodes_, &loop_groups);
-                for (auto& lg : loop_groups) {
-                  if (lg.size() > 1) {
-                    bool applied = false;
-                    Status s = OrderNodeSet(&lg);
-                    TF_RETURN_IF_ERROR(s);
-                    VLOG(1) << "Applying Rewriter for " << op_name;
-                    s = rewriter->Rewrite(this, graph, op_name, lg, &applied);
-                    LOG_WARNING_AND_RETURN_IF_ERROR(s);
-                  }
-                }
+        status = ApplyToAll(root.get(), [this, rewriter, graph, &frame_view,
+                                         &op_name](Tree* t) {
+          VLOG(2) << "applied to tree node " << t->edge_ << " at depth "
+                  << t->depth_ << " of size " << t->nodes_.size();
+          if (t->nodes_.size() > 1) {
+            std::vector<std::vector<NodeDef*>> loop_groups;
+            PartitionByLoopStructure(frame_view, t->nodes_, &loop_groups);
+            for (auto& lg : loop_groups) {
+              if (lg.size() > 1) {
+                bool applied = false;
+                Status s = OrderNodeSet(&lg);
+                TF_RETURN_IF_ERROR(s);
+                VLOG(1) << "Applying Rewriter for " << op_name;
+                s = rewriter->Rewrite(this, graph, op_name, lg, &applied);
+                LOG_WARNING_AND_RETURN_IF_ERROR(s);
               }
-              return Status::OK();
-            });
+            }
+          }
+          return Status::OK();
+        });
         if (!status.ok()) {
           break;
         }
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index 2b9448e403..c0f19d3828 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -74,8 +74,9 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_view",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
diff --git a/tensorflow/core/grappler/utils/frame.cc b/tensorflow/core/grappler/utils/frame.cc
index df5f4ff7cf..2484b35de0 100644
--- a/tensorflow/core/grappler/utils/frame.cc
+++ b/tensorflow/core/grappler/utils/frame.cc
@@ -15,77 +15,128 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/utils/frame.h"
 #include <deque>
-#include <stack>
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/grappler/graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
-#include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/errors.h"
 
 namespace tensorflow {
 namespace grappler {
 
-Status IdentifyFrames(const GraphDef& graph, FrameMap* frame_map,
-                      int* num_frames) {
-  NodeMap node_map(const_cast<GraphDef*>(&graph));
-  return IdentifyFramesWithNodeMap(graph, node_map, frame_map, num_frames);
-}
+namespace {}  // namespace
+
+Status FrameView::InferFromGraphView(const GraphView& graph_view) {
+  if (is_inferred_) {
+    return errors::Internal("FrameView was already inferred from the graph");
+  }
+  is_inferred_ = true;
+
+  std::deque<const NodeDef*> ready_nodes;
 
-Status IdentifyFramesWithNodeMap(const GraphDef& graph, const NodeMap& node_map,
-                                 FrameMap* frame_map, int* num_frames) {
-  std::deque<std::pair<const NodeDef*, std::vector<int>>> ready_nodes;
-  for (const NodeDef& node : graph.node()) {
+  // All nodes without inputs are automatically added to the ready queue.
+  for (const NodeDef& node : graph_view.graph()->node()) {
     if (node.input_size() == 0) {
-      std::vector<int> empty;
-      ready_nodes.emplace_back(&node, empty);
-      (*frame_map)[&node] = empty;
+      ready_nodes.push_back(&node);
+      node_to_frames_[&node] = node_has_no_frames_;
     }
   }
-  std::map<string, int> name_to_id;
+
+  // We assign unique int id to each frame, and use this map to track what
+  // frames we've already seen in the graph.
+  absl::flat_hash_map<string, int> frame_name_to_id;
+
   while (!ready_nodes.empty()) {
-    auto ready_node = ready_nodes.front();
-    for (const auto& fanout : node_map.GetOutputs(ready_node.first->name())) {
-      if (frame_map->count(fanout) < 1) {
-        std::vector<int> frame_ids = ready_node.second;
-        if (IsExit(*ready_node.first)) {
+    const NodeDef* ready_node = ready_nodes.front();
+
+    absl::flat_hash_set<GraphView::InputPort> fanouts =
+        graph_view.GetFanouts(*ready_node, /*include_controlled_nodes=*/true);
+
+    for (const GraphView::InputPort& fanout : fanouts) {
+      if (node_to_frames_.count(fanout.node) < 1) {
+        // If we have never seen this node before, we add all frames from the
+        // incoming node (and pop/push frames if coming from Exit/Enter nodes).
+        std::vector<int> frame_ids = node_to_frames_[ready_node];
+
+        if (IsExit(*ready_node)) {
           frame_ids.pop_back();
         }
-        if (IsEnter(*fanout)) {
-          CHECK(fanout->attr().count("frame_name"))
-              << "Missing frame name for the Enter node " << fanout->name();
-          string name = fanout->attr().at("frame_name").s();
-          int id;
-          if (name_to_id.count(name)) {
-            id = name_to_id[name];
+
+        if (IsEnter(*fanout.node)) {
+          const AttrValue* frame_name_attr =
+              AttrSlice(*fanout.node).Find("frame_name");
+
+          if (!frame_name_attr) {
+            return errors::InvalidArgument(
+                "Missing frame name for the Enter node: ",
+                SummarizeNodeDef(*fanout.node));
+          }
+
+          absl::string_view frame_name = frame_name_attr->s();
+          int frame_id;
+
+          if (frame_name_to_id.count(frame_name)) {
+            frame_id = frame_name_to_id[frame_name];
           } else {
-            id = name_to_id.size();
-            name_to_id[name] = id;
+            frame_id = static_cast<int>(frame_name_to_id.size());
+            frame_name_to_id[frame_name] = frame_id;
           }
-          frame_ids.push_back(id);
+
+          frame_ids.push_back(frame_id);
         }
-        ready_nodes.emplace_back(fanout, frame_ids);
-        (*frame_map)[fanout] = frame_ids;
+
+        ready_nodes.push_back(fanout.node);
+        node_to_frames_[fanout.node] = std::move(frame_ids);
+
       } else {
-        auto frame_ids_fanout = (*frame_map)[fanout];
-        auto frame_ids_node = ready_node.second;
-        if (IsEnter(*fanout)) {
+        // If we've already seen this node before, we need to make sure that
+        // graph is correct and same nodes doesn't have incoming edges with
+        // conflicting frames (all inputs must be produces in the same frame).
+
+        std::vector<int> frame_ids_fanout = node_to_frames_[fanout.node];
+        std::vector<int> frame_ids_node = node_to_frames_[ready_node];
+
+        if (IsEnter(*fanout.node)) {
           frame_ids_fanout.pop_back();
         }
-        if (IsExit(*ready_node.first)) {
+        if (IsExit(*ready_node)) {
           frame_ids_node.pop_back();
         }
+
         if (frame_ids_node != frame_ids_fanout) {
           return errors::InvalidArgument(
-              "Invalid graph: Frame ids for node ", ready_node.first->name(),
-              " does not match frame ids for it's fanout.");
+              "Invalid graph: Frame ids for node ", ready_node->name(),
+              " does not match frame ids for it's fanout ",
+              fanout.node->name());
         }
       }
     }
+
     ready_nodes.pop_front();
   }
-  *num_frames = name_to_id.size();
+
+  num_frames_ = static_cast<int>(frame_name_to_id.size());
   return Status::OK();
 }
 
+Status FrameView::InferFromGraph(const GraphDef& graph) {
+  return InferFromGraphView(GraphView(&graph));
+}
+
+const std::vector<int>& FrameView::Frames(const NodeDef& node) const {
+  DCHECK(is_inferred_) << "FrameView is not initialized";
+  auto frames = node_to_frames_.find(&node);
+  if (frames == node_to_frames_.end()) {
+    LOG(WARNING) << "Node doesn't belong to the graph used for initialization";
+    return node_has_no_frames_;
+  } else {
+    return frames->second;
+  }
+}
+
+bool FrameView::IsInFrame(const NodeDef& node) const {
+  return !Frames(node).empty();
+}
+
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/utils/frame.h b/tensorflow/core/grappler/utils/frame.h
index 95b72748f4..04c6588275 100644
--- a/tensorflow/core/grappler/utils/frame.h
+++ b/tensorflow/core/grappler/utils/frame.h
@@ -17,25 +17,52 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_UTILS_FRAME_H_
 
 #include <unordered_map>
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/framework/graph.pb.h"
-#include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/grappler/graph_view.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
 namespace grappler {
 
-using FrameMap = std::unordered_map<const NodeDef*, std::vector<int>>;
+// FrameView is a helper class that allows to find in what execution frames (if
+// any) the given node can be running in. It's constructed from an immutable
+// GraphView, and any modification of the underlying graph might invalidate it.
+//
+// All execution frames assigned an unique integer id, but they do not have any
+// meaning whatsoever, it's just a sequence number.
+//
+// See the paper "Dynamic Control Flow in Large-Scale Machine Learning" for
+// detailed explanation of execution frames (https://arxiv.org/abs/1805.01772).
+class FrameView {
+ public:
+  FrameView() : is_inferred_(false), num_frames_(0) {}
 
-// Returns the number of frames present in the graph, and populates
-// the 'frames' argument with the collection of frames (denoted by their
-// frame ids) in the outermost-to-innermost order. Frame ids are arbitrary.
-Status IdentifyFrames(const GraphDef& graph, FrameMap* frame_map,
-                      int* num_frames);
+  // Infers nodes execution frames from the GraphView. Returns an error if
+  // called multiple times.
+  Status InferFromGraphView(const GraphView& graph_view);
+  // Infers nodes execution by constructing temporary GraphView and passing it
+  // to InferFromGraphView.
+  Status InferFromGraph(const GraphDef& graph);
 
-// As above, but use an existing NodeMap for graph instead of building it
-// from scratch.
-Status IdentifyFramesWithNodeMap(const GraphDef& graph, const NodeMap& node_map,
-                                 FrameMap* frame_map, int* num_frames);
+  // Returns all frames of the given node (denoted by their frame ids) in
+  // outermost-to-innermost order.
+  const std::vector<int>& Frames(const NodeDef& node) const;
+
+  // Returns true iff the node is at least in one execution frame.
+  bool IsInFrame(const NodeDef& node) const;
+
+  int num_frames() const { return num_frames_; }
+  bool is_inferred() const { return is_inferred_; }
+
+ private:
+  bool is_inferred_;  // true if it was inferred from the graph
+  int num_frames_;    // number of frames present in a graph
+  absl::flat_hash_map<const NodeDef*, std::vector<int>> node_to_frames_;
+
+  // We return a reference to this vector if node has no frames.
+  const std::vector<int> node_has_no_frames_;
+};
 
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/utils/frame_test.cc b/tensorflow/core/grappler/utils/frame_test.cc
index df76083fc3..cc82e0ed3a 100644
--- a/tensorflow/core/grappler/utils/frame_test.cc
+++ b/tensorflow/core/grappler/utils/frame_test.cc
@@ -23,7 +23,7 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-class IdentifyFramesTest : public ::testing::Test {
+class FrameViewTest : public ::testing::Test {
  protected:
   static NodeDef CreateNode(const string& name,
                             const std::vector<string>& inputs) {
@@ -53,19 +53,17 @@ class IdentifyFramesTest : public ::testing::Test {
   }
 };
 
-TEST_F(IdentifyFramesTest, NestedLoop) {
+TEST_F(FrameViewTest, NestedLoop) {
   GraphDef graph;
   // Create a two-level nested loop
   *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() =
-      CreateNode("1", "Enter", "map/while/while_context1", {"0"});
+  *graph.add_node() = CreateNode("1", "Enter", "while/context1", {"0"});
   *graph.add_node() = CreateNode("2", {"1"});
   *graph.add_node() = CreateNode("3", "Merge", {"2", "14"});
   *graph.add_node() = CreateNode("4", {"3"});
   *graph.add_node() = CreateNode("5", "Switch", {"4"});
   *graph.add_node() = CreateNode("6", {"5"});
-  *graph.add_node() =
-      CreateNode("7", "Enter", "map/while/while_context2", {"6"});
+  *graph.add_node() = CreateNode("7", "Enter", "while/context2", {"6"});
   *graph.add_node() = CreateNode("8", {"7"});
   *graph.add_node() = CreateNode("9", "Merge", {"8", "12"});
   *graph.add_node() = CreateNode("10", {"9"});
@@ -77,118 +75,106 @@ TEST_F(IdentifyFramesTest, NestedLoop) {
   *graph.add_node() = CreateNode("16", "Exit", {"15"});
   *graph.add_node() = CreateNode("17", {"16"});
 
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
+  FrameView frame_view;
+  ASSERT_TRUE(frame_view.InferFromGraph(graph).ok());
+
   std::unordered_map<string, std::vector<int>> expected = {
       {"0", {}},      {"1", {0}},     {"2", {0}},     {"3", {0}},
       {"4", {0}},     {"5", {0}},     {"6", {0}},     {"7", {0, 1}},
       {"8", {0, 1}},  {"9", {0, 1}},  {"10", {0, 1}}, {"11", {0, 1}},
       {"12", {0, 1}}, {"13", {0, 1}}, {"14", {0}},    {"15", {0}},
       {"16", {0}},    {"17", {}}};
-  EXPECT_EQ(num_frames, 2);
-  EXPECT_EQ(frames.size(), expected.size());
-  std::cout << "Number of frame: " << num_frames << std::endl;
-  for (const auto& node : frames) {
-    std::cout << node.first->name() << ": ";
-    EXPECT_EQ(node.second.size(), expected[node.first->name()].size());
-    for (int i = 0; i < node.second.size(); i++) {
-      EXPECT_EQ(expected[node.first->name()][i], node.second[i]);
-      std::cout << node.second[i] << " ";
-    }
-    std::cout << std::endl;
+
+  EXPECT_EQ(frame_view.num_frames(), 2);
+  for (const NodeDef& node : graph.node()) {
+    std::vector<int> expected_frames = expected[node.name()];
+    std::vector<int> node_frames = frame_view.Frames(node);
+    EXPECT_EQ(expected_frames, node_frames);
   }
 }
 
-TEST_F(IdentifyFramesTest, MultipleInputsToEnter) {
+TEST_F(FrameViewTest, MultipleInputsToEnter) {
   GraphDef graph;
   *graph.add_node() = CreateNode("0", {});
   *graph.add_node() = CreateNode("1", {});
-  *graph.add_node() =
-      CreateNode("2", "Enter", "map/while/while_context", {"0", "1"});
+  *graph.add_node() = CreateNode("2", "Enter", "while/context", {"0", "1"});
   *graph.add_node() = CreateNode("3", "Exit", {"2"});
 
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
+  FrameView frame_view;
+  ASSERT_TRUE(frame_view.InferFromGraph(graph).ok());
+
   std::unordered_map<string, std::vector<int>> expected = {
       {"0", {}}, {"1", {}}, {"2", {0}}, {"3", {0}}};
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.size(), expected.size());
-  std::cout << "Number of frame: " << num_frames << std::endl;
-  for (const auto& node : frames) {
-    std::cout << node.first->name() << ": ";
-    EXPECT_EQ(node.second.size(), expected[node.first->name()].size());
-    for (int i = 0; i < node.second.size(); i++) {
-      EXPECT_EQ(expected[node.first->name()][i], node.second[i]);
-      std::cout << node.second[i] << " ";
-    }
-    std::cout << std::endl;
+
+  EXPECT_EQ(frame_view.num_frames(), 1);
+  for (const NodeDef& node : graph.node()) {
+    std::vector<int> expected_frames = expected[node.name()];
+    std::vector<int> node_frames = frame_view.Frames(node);
+    EXPECT_EQ(expected_frames, node_frames);
   }
 }
 
-TEST_F(IdentifyFramesTest, ExitOutput) {
+TEST_F(FrameViewTest, ExitOutput) {
   GraphDef graph;
   *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() =
-      CreateNode("1", "Enter", "map/while/while_context", {"0"});
+  *graph.add_node() = CreateNode("1", "Enter", "while/context", {"0"});
   *graph.add_node() = CreateNode("2", "Exit", {"1"});
   *graph.add_node() = CreateNode("3", {});
   *graph.add_node() = CreateNode("4", {"2", "3"});
 
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
+  FrameView frame_view;
+  ASSERT_TRUE(frame_view.InferFromGraph(graph).ok());
+
   std::unordered_map<string, std::vector<int>> expected = {
       {"0", {}}, {"1", {0}}, {"2", {0}}, {"3", {}}, {"4", {}}};
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.size(), expected.size());
-  std::cout << "Number of frame: " << num_frames << std::endl;
-  for (const auto& node : frames) {
-    std::cout << node.first->name() << ": ";
-    EXPECT_EQ(node.second.size(), expected[node.first->name()].size());
-    for (int i = 0; i < node.second.size(); i++) {
-      EXPECT_EQ(expected[node.first->name()][i], node.second[i]);
-      std::cout << node.second[i] << " ";
-    }
-    std::cout << std::endl;
+
+  EXPECT_EQ(frame_view.num_frames(), 1);
+  for (const NodeDef& node : graph.node()) {
+    std::vector<int> expected_frames = expected[node.name()];
+    std::vector<int> node_frames = frame_view.Frames(node);
+    EXPECT_EQ(expected_frames, node_frames);
   }
 }
 
-TEST_F(IdentifyFramesTest, MultipleEnterNodes) {
+TEST_F(FrameViewTest, MultipleEnterNodes) {
   GraphDef graph;
   *graph.add_node() = CreateNode("0", {});
-  string frame = "map/while/while_context";
-  *graph.add_node() = CreateNode("1", "Enter", frame, {"0"});
+  *graph.add_node() = CreateNode("1", "Enter", "while/context", {"0"});
   *graph.add_node() = CreateNode("2", {"1"});
   *graph.add_node() = CreateNode("5", {});
-  *graph.add_node() = CreateNode("4", "Enter", frame, {"5"});
+  *graph.add_node() = CreateNode("4", "Enter", "while/context", {"5"});
   *graph.add_node() = CreateNode("3", {"4", "2"});
   *graph.add_node() = CreateNode("6", "Merge", {"3", "8"});
   *graph.add_node() = CreateNode("7", "Switch", {"6"});
   *graph.add_node() = CreateNode("8", "NextIteration", {"7"});
   *graph.add_node() = CreateNode("9", "Exit", {"7"});
 
-  std::unordered_map<const NodeDef*, std::vector<int>> frames;
-  int num_frames;
-  EXPECT_TRUE(IdentifyFrames(graph, &frames, &num_frames).ok());
+  FrameView frame_view;
+  ASSERT_TRUE(frame_view.InferFromGraph(graph).ok());
+
   std::unordered_map<string, std::vector<int>> expected = {
       {"0", {}}, {"1", {0}}, {"2", {0}}, {"3", {0}}, {"4", {0}},
       {"5", {}}, {"6", {0}}, {"7", {0}}, {"8", {0}}, {"9", {0}}};
-  EXPECT_EQ(num_frames, 1);
-  EXPECT_EQ(frames.size(), expected.size());
-  std::cout << "Number of frame: " << num_frames << std::endl;
-  for (const auto& node : frames) {
-    std::cout << node.first->name() << ": ";
-    EXPECT_EQ(node.second.size(), expected[node.first->name()].size());
-    for (int i = 0; i < node.second.size(); i++) {
-      EXPECT_EQ(expected[node.first->name()][i], node.second[i]);
-      std::cout << node.second[i] << " ";
-    }
-    std::cout << std::endl;
+
+  EXPECT_EQ(frame_view.num_frames(), 1);
+  for (const NodeDef& node : graph.node()) {
+    std::vector<int> expected_frames = expected[node.name()];
+    std::vector<int> node_frames = frame_view.Frames(node);
+    EXPECT_EQ(expected_frames, node_frames);
   }
 }
 
+TEST_F(FrameViewTest, ConflictingFrames) {
+  GraphDef graph;
+  *graph.add_node() = CreateNode("0", {});
+  *graph.add_node() = CreateNode("1", "Enter", "while/context1", {"0"});
+  *graph.add_node() = CreateNode("2", "Enter", "while/context2", {"1"});
+  *graph.add_node() = CreateNode("3", {"1", "2"});
+
+  FrameView frame_view;
+  ASSERT_FALSE(frame_view.InferFromGraph(graph).ok());
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From dd6b19f3667e7f09de0e0552e76a53eece7fa281 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 11:07:08 -0800
Subject: [PATCH 1362/1554] One more batch of tests enabled in oss

PiperOrigin-RevId: 224186044
---
 tensorflow/lite/examples/label_image/BUILD             | 1 -
 tensorflow/lite/profiling/BUILD                        | 1 -
 tensorflow/lite/schema/builtin_ops_header/BUILD        | 2 --
 tensorflow/lite/toco/graph_transformations/tests/BUILD | 3 ---
 tensorflow/lite/toco/tensorflow_graph_matching/BUILD   | 1 -
 tensorflow/lite/tools/BUILD                            | 2 --
 6 files changed, 10 deletions(-)

diff --git a/tensorflow/lite/examples/label_image/BUILD b/tensorflow/lite/examples/label_image/BUILD
index de1bfd7053..4fc8648d46 100644
--- a/tensorflow/lite/examples/label_image/BUILD
+++ b/tensorflow/lite/examples/label_image/BUILD
@@ -63,7 +63,6 @@ cc_test(
     data = [
         "testdata/grace_hopper.bmp",
     ],
-    tags = ["no_oss"],
     deps = [
         ":bitmap_helpers",
         "@com_google_googletest//:gtest",
diff --git a/tensorflow/lite/profiling/BUILD b/tensorflow/lite/profiling/BUILD
index c7a8e4f06a..52ea6fe636 100644
--- a/tensorflow/lite/profiling/BUILD
+++ b/tensorflow/lite/profiling/BUILD
@@ -58,7 +58,6 @@ cc_test(
     name = "profile_summarizer_test",
     srcs = ["profile_summarizer_test.cc"],
     copts = common_copts,
-    tags = ["no_oss"],
     deps = [
         ":profile_summarizer",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/lite/schema/builtin_ops_header/BUILD b/tensorflow/lite/schema/builtin_ops_header/BUILD
index 8a01541d57..52cbd052d6 100644
--- a/tensorflow/lite/schema/builtin_ops_header/BUILD
+++ b/tensorflow/lite/schema/builtin_ops_header/BUILD
@@ -24,7 +24,6 @@ cc_binary(
 cc_test(
     name = "generator_test",
     srcs = ["generator_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":generator",
         "@com_google_googletest//:gtest",
@@ -37,7 +36,6 @@ cc_test(
     data = [
         "//tensorflow/lite:builtin_ops.h",
     ],
-    tags = ["no_oss"],
     deps = [
         ":generator",
         "@com_google_googletest//:gtest",
diff --git a/tensorflow/lite/toco/graph_transformations/tests/BUILD b/tensorflow/lite/toco/graph_transformations/tests/BUILD
index 2e9b213d00..bbbedbe3a9 100644
--- a/tensorflow/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/lite/toco/graph_transformations/tests/BUILD
@@ -10,7 +10,6 @@ load(
 tf_cc_test(
     name = "lstm_utils_test",
     srcs = ["lstm_utils_test.cc"],
-    tags = ["no_oss"],
     deps = [
         "//tensorflow/lite/toco:graph_transformations",
         "//tensorflow/lite/toco:model",
@@ -22,7 +21,6 @@ tf_cc_test(
 tf_cc_test(
     name = "resolve_constant_concatenation_test",
     srcs = ["resolve_constant_concatenation_test.cc"],
-    tags = ["no_oss"],
     deps = [
         "//tensorflow/lite/toco:graph_transformations",
         "//tensorflow/lite/toco:model",
@@ -34,7 +32,6 @@ tf_cc_test(
 tf_cc_test(
     name = "resolve_constant_unary_test",
     srcs = ["resolve_constant_unary_test.cc"],
-    tags = ["no_oss"],
     deps = [
         "//tensorflow/lite/toco:graph_transformations",
         "//tensorflow/lite/toco:model",
diff --git a/tensorflow/lite/toco/tensorflow_graph_matching/BUILD b/tensorflow/lite/toco/tensorflow_graph_matching/BUILD
index 56acc284cc..ae361bf212 100644
--- a/tensorflow/lite/toco/tensorflow_graph_matching/BUILD
+++ b/tensorflow/lite/toco/tensorflow_graph_matching/BUILD
@@ -60,7 +60,6 @@ cc_library(
 tf_cc_test(
     name = "resolve_svdf_test",
     srcs = ["resolve_svdf_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":cluster",
         ":cluster_utils",
diff --git a/tensorflow/lite/tools/BUILD b/tensorflow/lite/tools/BUILD
index 93725b5de4..1d141b5dd0 100644
--- a/tensorflow/lite/tools/BUILD
+++ b/tensorflow/lite/tools/BUILD
@@ -53,7 +53,6 @@ cc_test(
         "//tensorflow/lite:testdata/test_model_broken.bin",
     ],
     tags = [
-        "no_oss",
         "tflite_not_portable_android",
         "tflite_not_portable_ios",
     ],
@@ -80,7 +79,6 @@ cc_test(
     size = "small",
     srcs = ["verifier_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
-- 
GitLab


From 28c02fa1804b52ee51ebd1518739c4653ed8b392 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 11:28:38 -0800
Subject: [PATCH 1363/1554] internal change

PiperOrigin-RevId: 224190200
---
 .../examples/lstm/unidirectional_sequence_lstm_test.py          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
index 9c00d0501a..eeb48d1231 100644
--- a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_lstm_test.py
@@ -111,7 +111,7 @@ class UnidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
 
     # Initialize variables
     init = tf.global_variables_initializer()
-    self.evaluate(init)
+    sess.run(init)
     for _ in range(TRAIN_STEPS):
       batch_x, batch_y = self.mnist.train.next_batch(
           batch_size=self.batch_size, shuffle=False)
-- 
GitLab


From 91fbda2d2cf0d48e9cf5d659b6fbe6d59b3606d9 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 21 Nov 2018 13:44:30 -0800
Subject: [PATCH 1364/1554] Add ExpandDims, Squeeze ops and unit tests.

---
 .../contrib/tensorrt/convert/convert_graph.cc |   2 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 122 +++++++++
 .../tensorrt/convert/convert_nodes_test.cc    | 247 ++++++++++++++++++
 3 files changed, 371 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 3b32f72bc1..560e565267 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -132,6 +132,8 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
       "Min",
       "Relu6",
       "Square",
+      "ExpandDims",
+      "Squeeze",
   };
   bool is_supported_op_type =
       (candidate_ops.count(node->type_string()) ||
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index fee095668e..49df1daae9 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1880,6 +1880,126 @@ tensorflow::Status ConvertReshape(OpConverterParams* params) {
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != 2) {
+    return tensorflow::errors::InvalidArgument(
+        "Two inputs expected for ExpandDims, at ", node_def.name());
+  }
+  if (!inputs.at(1).is_weights() ) {
+    return tensorflow::errors::InvalidArgument(
+        "ExpandDims expects weights for axis, at ", node_def.name());
+  }
+  // Get input shape as vector.
+  TRT_TensorOrWeights input_tensor = inputs.at(0);
+  const nvinfer1::Dims dims = input_tensor.GetTrtDims();
+  std::vector<int> input_dims(dims.d, dims.d + dims.nbDims);
+  // Add batch dim back for tensors.
+  if (input_tensor.is_tensor()) {
+    input_dims.insert(input_dims.begin(), -1);
+  }
+  const int input_rank = input_dims.size();
+  // Get axis to expand on.
+  TRT_ShapedWeights weights = inputs.at(1).weights();
+  const int* weights_ptr =
+      static_cast<int*>(const_cast<void*>(weights.GetValues()));
+  int axis = weights_ptr[0];
+  // Make sure axis is valid.
+  if ((axis < (-input_rank - 1)) || (axis > input_rank)) {
+    return tensorflow::errors::InvalidArgument(
+        "Axis for ExpandDims is invalid, must be in the range "
+        "[-rank(input) - 1, rank(input)], at ",
+        node_def.name());
+  }
+  // Convert negative axis to corresponding positive axis.
+  if (axis < 0) axis += input_rank + 1;
+  if (input_tensor.is_tensor() && axis == 0) {
+    return tensorflow::errors::Unimplemented(
+        "Modifying batch dimension is not supported for ExpandDims, at ",
+        node_def.name());
+  }
+  if (params->validation_only) return Status::OK();
+
+  // ExpandDims: Insert new dim of size 1.
+  input_dims.insert(input_dims.begin()+axis, 1);
+  // Convert input_dims vector into nvinfer1::Dims.
+  const bool ignore_first_dim = input_tensor.is_tensor();
+  nvinfer1::Dims new_dims = VectorToTrtDims(input_dims, ignore_first_dim);
+  // Reshape tensor.
+  const nvinfer1::ITensor* output_tensor = nullptr;
+  TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
+      input_tensor, new_dims, &output_tensor));
+  params->outputs->push_back(
+      TRT_TensorOrWeights(const_cast<nvinfer1::ITensor*>(output_tensor)));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
+  const auto& inputs = params->inputs;
+  const auto& node_def = params->node_def;
+  if (inputs.size() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        "One input expected for Squeeze, at ", node_def.name());
+  }
+  // Get input shape.
+  TRT_TensorOrWeights input_tensor = inputs.at(0);
+  const nvinfer1::Dims dims = input_tensor.GetTrtDims();
+  std::vector<int> input_dims(dims.d, dims.d + dims.nbDims);
+  // Add batch dim back temporarily.
+  if (input_tensor.is_tensor()) {
+    input_dims.insert(input_dims.begin(), -1);
+  }
+  const int input_rank = input_dims.size();
+  // Mark axes to remove by setting them to 0.
+  TFAttrs attrs(node_def);
+  auto squeeze_dims = attrs.get<std::vector<int>>("squeeze_dims");
+  if (squeeze_dims.size() == 0) {
+    return tensorflow::errors::Unimplemented(
+        "Squeeze is only implemented for explicit dims, at ", node_def.name());
+  }
+  for (int axis : squeeze_dims) {
+    // Make sure axis is valid.
+    if ((axis < -input_rank) || (axis >= input_rank)) {
+      return tensorflow::errors::InvalidArgument(
+          "Axis for Squeeze is invalid, must be in the range "
+          "[-rank(input), rank(input)), at ",
+          node_def.name());
+    }
+    // Convert negative axis to corresponding positive axis.
+    if (axis < 0) axis += input_rank;
+    LOG(INFO) << axis;
+    // Don't squeeze batch dim.
+    if (axis == 0) {
+      return tensorflow::errors::Unimplemented(
+          "Cannot squeeze batch dimension, at ", node_def.name());
+    }
+    // Make sure target dimension is size 1.
+    if (input_dims[axis] != 1) {
+      return tensorflow::errors::InvalidArgument(
+          "Cannot squeeze a dimension which isn't size 1, at ",
+          node_def.name());
+    }
+    // Mark dim for removal by setting to 0.
+    input_dims[axis] = 0;
+  }
+  if (params->validation_only) return Status::OK();
+  
+  // Remove all dims which are equal to 0.
+  input_dims.erase(std::remove(input_dims.begin(), input_dims.end(), 0),
+                   input_dims.end());
+  // Convert input_dims vector into nvinfer1::Dims.
+  const bool ignore_first_dim = input_tensor.is_tensor();
+  nvinfer1::Dims new_dims = VectorToTrtDims(input_dims, ignore_first_dim);
+  // Reshape tensor.
+  const nvinfer1::ITensor* output_tensor = nullptr;
+  TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
+      input_tensor, new_dims, &output_tensor));
+  params->outputs->push_back(
+      TRT_TensorOrWeights(const_cast<nvinfer1::ITensor*>(output_tensor)));
+  return tensorflow::Status::OK();
+}
+
 tensorflow::Status ConvertConv2D(OpConverterParams* params) {
   return ConvertConv2DHelper(params, ConvolutionType::DEFAULT);
 }
@@ -3156,6 +3276,8 @@ static void RegisterValidatableOpConverters(
   (*registration)["MatMul"] = ConvertMatMul;
   (*registration)["Relu6"] = ConvertRelu6;
   (*registration)["Square"] = ConvertSquare;
+  (*registration)["ExpandDims"] = ConvertExpandDims;
+  (*registration)["Squeeze"] = ConvertSqueeze;
 
   for (auto quantization_op_type :
        {"QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 443033379f..c37406901a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2113,6 +2113,253 @@ TEST_F(OpConverterTest, ConvertActivation) {
   }
 }
 
+TEST_F(OpConverterTest, ConvertExpandDims) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_expanddims", "ExpandDims", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Two inputs expected for ExpandDims, at my_expanddims");
+  }
+
+  // Get the NodeDef for ExpandDims.
+  Scope s = Scope::NewRootScope();
+  auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+  auto weights = ops::Placeholder(s.WithOpName("weights"), DT_INT32);
+  auto expanddims = ops::ExpandDims(s.WithOpName("my_expanddims"), input, weights);
+  const NodeDef& node_def = expanddims.operation.node()->def();
+
+  {
+    // Axis is a tensor, should fail.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestTensor("weights", {3});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "ExpandDims expects weights for axis, at my_expanddims");
+  }
+  {
+    // Add dim at batch dimension, should fail.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<int32>("weights", {1}, {0});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+        "Modifying batch dimension is not supported for ExpandDims, at my_expanddims");
+  }
+  {
+    // Add dim at batch dimension via negative axis, should fail.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    // Input is rank 4 (batch dim included)
+    AddTestWeights<int32>("weights", {1}, {-5});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+        "Modifying batch dimension is not supported for ExpandDims, at my_expanddims");
+  }
+  {
+    // Axis > rank(input), should fail.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    // Input is rank 4 (batch dim included)
+    AddTestWeights<int32>("weights", {1}, {5});
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+        "Axis for ExpandDims is invalid, must be in the range "
+        "[-rank(input) - 1, rank(input)], at my_expanddims");
+  }
+  {
+    // Axis < -rank(input)-1, should fail.
+    Reset();
+    AddTestTensor("input", {1, 2, 3});
+    // Input is rank 4 (batch dim included)
+    AddTestWeights<int32>("weights", {1}, {-6});
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+        "Axis for ExpandDims is invalid, must be in the range "
+        "[-rank(input) - 1, rank(input)], at my_expanddims");
+  }
+  {
+    // Add axis before, Ok.
+    Reset();
+    AddTestTensor("input", {2, 3});
+    AddTestWeights<int32>("weights", {1}, {1});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_expanddims", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+  {
+    // Add axis after, Ok.
+    Reset();
+    AddTestTensor("input", {2, 3});
+    AddTestWeights<int32>("weights", {1}, {3});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_expanddims", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+  {
+    // Add negative axis, Ok.
+    Reset();
+    AddTestTensor("input", {2, 3});
+    AddTestWeights<int32>("weights", {1}, {-1});
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_expanddims", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+}
+
+TEST_F(OpConverterTest, ConvertSqueeze) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_squeeze", "Squeeze", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "One input expected for Squeeze, at my_squeeze");
+  }
+  {
+    // No attrs, should fail.
+    Reset();
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto squeeze = ops::Squeeze(s.WithOpName("my_squeeze"), input);
+    const NodeDef& node_def = squeeze.operation.node()->def();
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Squeeze is only implemented for explicit dims, at my_squeeze");
+  }
+
+  // Get the NodeDef for Squeeze.
+  auto get_squeeze_nodedef = [](std::vector<int> axis) -> NodeDef {
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    ops::Squeeze::Attrs squeeze_attrs;
+    squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);
+    auto squeeze = ops::Squeeze(s.WithOpName("my_squeeze"), input, squeeze_attrs);
+    return squeeze.operation.node()->def();
+  };
+
+  {
+    // Squeeze batch dim, should fail.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({0});
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Cannot squeeze batch dimension, at my_squeeze");
+  }
+  {
+    // Squeeze batch dim via negative axis, should fail.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({-4});
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Cannot squeeze batch dimension, at my_squeeze");
+  }
+  {
+    // Squeeze >= rank(input), should fail.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({4});
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Axis for Squeeze is invalid, must be in the range "
+        "[-rank(input), rank(input)), at my_squeeze");
+  }
+  {
+    // Squeeze < -rank(input), should fail.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({-5});
+    AddTestTensor("input", {1, 2, 3});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Axis for Squeeze is invalid, must be in the range "
+        "[-rank(input), rank(input)), at my_squeeze");
+  }
+  {
+    // Squeeze axis before, Ok.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({1});
+    AddTestTensor("input", {1, 2, 3});
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+  {
+    // Squeeze axis after, Ok.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({3});
+    AddTestTensor("input", {2, 3, 1});
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+  {
+    // Squeeze multiple axis, Ok.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({1, 3, 5});
+    AddTestTensor("input", {1, 2, 1, 3, 1});
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+  {
+    // Squeeze multiple axis negative, Ok.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({-1, -3, -5});
+    AddTestTensor("input", {1, 2, 1, 3, 1});
+    RunConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
+    EXPECT_TRUE(output.is_tensor());
+    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
+        << output.DebugString();
+
+    std::vector<float> output_data(6);
+    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
+  }
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From 9d3b1765b20660ba8f166a44bdc9f80bc3987ced Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 26 Nov 2018 09:33:24 -0800
Subject: [PATCH 1365/1554] Add missing VectorToTrtDims function

---
 .../contrib/tensorrt/convert/convert_nodes.cc   | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 49df1daae9..1116a039ce 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -120,6 +120,17 @@ inline nvinfer1::Dims TensorShapeToTrtDims(const TensorShapeType& shape,
   return trt_dims;
 }
 
+inline nvinfer1::Dims VectorToTrtDims(const std::vector<int>& shape,
+                                      bool ignore_first_dim = false) {
+  nvinfer1::Dims trt_dims;
+  const int offset = (ignore_first_dim ? 1 : 0);
+  for (int i = offset; i < shape.size(); i++) {
+    trt_dims.d[i - offset] = shape[i];
+  }
+  trt_dims.nbDims = shape.size() - offset;
+  return trt_dims;
+}
+
 void GetOutputProperties(const grappler::GraphProperties& graph_properties,
                          const Node* node, const int out_port,
                          PartialTensorShape* shape,
@@ -1923,10 +1934,9 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
 
   // ExpandDims: Insert new dim of size 1.
   input_dims.insert(input_dims.begin()+axis, 1);
-  // Convert input_dims vector into nvinfer1::Dims.
+  // Reshape tensor.
   const bool ignore_first_dim = input_tensor.is_tensor();
   nvinfer1::Dims new_dims = VectorToTrtDims(input_dims, ignore_first_dim);
-  // Reshape tensor.
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
@@ -1988,10 +1998,9 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   // Remove all dims which are equal to 0.
   input_dims.erase(std::remove(input_dims.begin(), input_dims.end(), 0),
                    input_dims.end());
-  // Convert input_dims vector into nvinfer1::Dims.
+    // Reshape tensor.
   const bool ignore_first_dim = input_tensor.is_tensor();
   nvinfer1::Dims new_dims = VectorToTrtDims(input_dims, ignore_first_dim);
-  // Reshape tensor.
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
-- 
GitLab


From dbe1a6bd852b8d03fccc6b6df7ee0b6f65ad7ca8 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 27 Nov 2018 10:20:57 -0800
Subject: [PATCH 1366/1554] Fix compilation error with BuildAndRun, Use
 ExpectTrtDimsEqualsArray, Fix formatting

---
 .../tensorrt/convert/convert_nodes_test.cc    | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index c37406901a..1e4ee1708f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2184,11 +2184,11 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_expanddims", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
   {
@@ -2200,11 +2200,11 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_expanddims", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
   {
@@ -2216,11 +2216,11 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_expanddims", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
 }
@@ -2252,7 +2252,8 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     ops::Squeeze::Attrs squeeze_attrs;
     squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);
-    auto squeeze = ops::Squeeze(s.WithOpName("my_squeeze"), input, squeeze_attrs);
+    auto squeeze = ops::Squeeze(s.WithOpName("my_squeeze"), input,
+                                squeeze_attrs);
     return squeeze.operation.node()->def();
   };
 
@@ -2303,11 +2304,11 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
   {
@@ -2319,11 +2320,11 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
   {
@@ -2335,11 +2336,11 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
   {
@@ -2351,11 +2352,11 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
     EXPECT_TRUE(output.is_tensor());
-    EXPECT_TRUE(TrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions()))
-        << output.DebugString();
+    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
-    BuildAndRun("input", {1, 2, 3, 4, 5, 6}, "my_squeeze", &output_data);
+    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
+                       &output_data);
     EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
   }
 }
-- 
GitLab


From 23a1bf779c82e363820ea8a1217038d971903ea5 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 28 Nov 2018 11:19:02 -0800
Subject: [PATCH 1367/1554] Remove debug LOG statement. Use TestParams array
 for Ok tests

---
 .../contrib/tensorrt/convert/convert_nodes.cc |   1 -
 .../tensorrt/convert/convert_nodes_test.cc    | 146 +++++++-----------
 2 files changed, 59 insertions(+), 88 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 1116a039ce..135e188502 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1978,7 +1978,6 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
     }
     // Convert negative axis to corresponding positive axis.
     if (axis < 0) axis += input_rank;
-    LOG(INFO) << axis;
     // Don't squeeze batch dim.
     if (axis == 0) {
       return tensorflow::errors::Unimplemented(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 1e4ee1708f..74ca982f3f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2175,48 +2175,41 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
         "Axis for ExpandDims is invalid, must be in the range "
         "[-rank(input) - 1, rank(input)], at my_expanddims");
   }
-  {
-    // Add axis before, Ok.
-    Reset();
-    AddTestTensor("input", {2, 3});
-    AddTestWeights<int32>("weights", {1}, {1});
-    RunValidationAndConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
-    EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({1, 2, 3}, output.tensor()->getDimensions());
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
-  }
-  {
-    // Add axis after, Ok.
-    Reset();
-    AddTestTensor("input", {2, 3});
-    AddTestWeights<int32>("weights", {1}, {3});
-    RunValidationAndConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
-    EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions());
+  struct TestParams {
+    TestParams(const std::vector<int>& input_dims,
+               int axis,
+               const std::vector<int>& expected_output_dims)
+        : input_dims(input_dims),
+          axis(axis),
+          expected_output_dims(expected_output_dims) {}
+    std::vector<int> input_dims;
+    int axis;
+    std::vector<int> expected_output_dims;
+  };
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
-  }
-  {
-    // Add negative axis, Ok.
+  // Ok.
+  const int kExpandDimsOKCases = 8;
+  TestParams ok_params[kExpandDimsOKCases] = {
+      TestParams{{2, 3}, 1, {1, 2, 3}},
+      TestParams{{2, 3}, -3, {1, 2, 3}},
+      TestParams{{2, 3}, 3, {2, 3, 1}},
+      TestParams{{2, 3}, -1, {2, 3, 1}},
+      TestParams{{2, 3}, 2, {2, 1, 3}},
+      TestParams{{2, 3}, -2, {2, 1, 3}},
+      TestParams{{6}, 1, {1, 6}},
+      TestParams{{6}, -1, {6, 1}},
+  };
+  for (int i = 0; i < kExpandDimsOKCases; ++i) {
     Reset();
-    AddTestTensor("input", {2, 3});
-    AddTestWeights<int32>("weights", {1}, {-1});
+    AddTestTensor("input", ok_params[i].input_dims);
+    AddTestWeights<int32>("weights", {1}, {ok_params[i].axis});
     RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_expanddims", &output));
     EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({2, 3, 1}, output.tensor()->getDimensions());
+    ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
+                             output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
     BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_expanddims",
@@ -2295,64 +2288,43 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
         "Axis for Squeeze is invalid, must be in the range "
         "[-rank(input), rank(input)), at my_squeeze");
   }
-  {
-    // Squeeze axis before, Ok.
-    Reset();
-    NodeDef node_def = get_squeeze_nodedef({1});
-    AddTestTensor("input", {1, 2, 3});
-    RunConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
-    EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
-
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
-  }
-  {
-    // Squeeze axis after, Ok.
-    Reset();
-    NodeDef node_def = get_squeeze_nodedef({3});
-    AddTestTensor("input", {2, 3, 1});
-    RunConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
-    EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
-  }
-  {
-    // Squeeze multiple axis, Ok.
-    Reset();
-    NodeDef node_def = get_squeeze_nodedef({1, 3, 5});
-    AddTestTensor("input", {1, 2, 1, 3, 1});
-    RunConversion(node_def);
-    TRT_TensorOrWeights output;
-    TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
-    EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
+  struct TestParams {
+    TestParams(const std::vector<int>& input_dims,
+               const std::vector<int>& axis,
+               const std::vector<int>& expected_output_dims)
+        : input_dims(input_dims),
+          axis(axis),
+          expected_output_dims(expected_output_dims) {}
+    std::vector<int> input_dims;
+    std::vector<int> axis;
+    std::vector<int> expected_output_dims;
+  };
 
-    std::vector<float> output_data(6);
-    BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
-                       &output_data);
-    EXPECT_THAT(output_data, ElementsAre(1, 2, 3, 4, 5, 6));
-  }
-  {
-    // Squeeze multiple axis negative, Ok.
+  // Ok.
+  const int kSqueezeOKCases = 10;
+  TestParams ok_params[kSqueezeOKCases] = {
+      TestParams{{1, 2, 3}, {1}, {2, 3}},
+      TestParams{{1, 2, 3}, {-3}, {2, 3}},
+      TestParams{{2, 3, 1}, {3}, {2, 3}},
+      TestParams{{2, 3, 1}, {-1}, {2, 3}},
+      TestParams{{1, 2, 1, 3, 1}, {1, 3, 5}, {2, 3}},
+      TestParams{{1, 2, 1, 3, 1}, {3, 1, 5}, {2, 3}},
+      TestParams{{1, 2, 1, 3, 1}, {-1, -3, -5}, {2, 3}},
+      TestParams{{1, 2, 1, 3, 1}, {1, -3, 5}, {2, 3}},
+      TestParams{{1, 6}, {1}, {6}},
+      TestParams{{6, 1}, {2}, {6}},
+  };
+  for (int i = 0; i < kSqueezeOKCases; ++i) {
     Reset();
-    NodeDef node_def = get_squeeze_nodedef({-1, -3, -5});
-    AddTestTensor("input", {1, 2, 1, 3, 1});
-    RunConversion(node_def);
+    NodeDef node_def = get_squeeze_nodedef(ok_params[i].axis);
+    AddTestTensor("input", ok_params[i].input_dims);
+    RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
     TF_EXPECT_OK(GetTensorOrWeights("my_squeeze", &output));
     EXPECT_TRUE(output.is_tensor());
-    ExpectTrtDimsEqualsArray({2, 3}, output.tensor()->getDimensions());
+    ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
+                             output.tensor()->getDimensions());
 
     std::vector<float> output_data(6);
     BuildAndRun<float>({{"input", {1, 2, 3, 4, 5, 6}}}, "my_squeeze",
-- 
GitLab


From 293bc93045d1eddce9040de36918270ed5e65b6c Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 29 Nov 2018 16:38:36 -0800
Subject: [PATCH 1368/1554] Restrict ops to tensors only. Renamed
 VectorToTrtDims.

---
 .../contrib/tensorrt/convert/convert_nodes.cc | 30 +++++++++++--------
 .../tensorrt/convert/convert_nodes_test.cc    | 27 +++++++++++++++--
 2 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 135e188502..a83f2c7c48 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -120,15 +120,11 @@ inline nvinfer1::Dims TensorShapeToTrtDims(const TensorShapeType& shape,
   return trt_dims;
 }
 
-inline nvinfer1::Dims VectorToTrtDims(const std::vector<int>& shape,
-                                      bool ignore_first_dim = false) {
-  nvinfer1::Dims trt_dims;
-  const int offset = (ignore_first_dim ? 1 : 0);
-  for (int i = offset; i < shape.size(); i++) {
-    trt_dims.d[i - offset] = shape[i];
-  }
-  trt_dims.nbDims = shape.size() - offset;
-  return trt_dims;
+inline nvinfer1::Dims TensorShapeArrayToTrtDims(const std::vector<int>& shape,
+                                                bool ignore_first_dim = false) {
+  PartialTensorShape tensor_shape;
+  TensorShapeUtils::MakeShape(shape, &tensor_shape);
+  return TensorShapeToTrtDims(tensor_shape, ignore_first_dim);
 }
 
 void GetOutputProperties(const grappler::GraphProperties& graph_properties,
@@ -1898,6 +1894,10 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
     return tensorflow::errors::InvalidArgument(
         "Two inputs expected for ExpandDims, at ", node_def.name());
   }
+  if (inputs.at(0).is_weights() ) {
+    return tensorflow::errors::Unimplemented(
+        "ExpandDims expects tensor for input, at ", node_def.name());
+  }
   if (!inputs.at(1).is_weights() ) {
     return tensorflow::errors::InvalidArgument(
         "ExpandDims expects weights for axis, at ", node_def.name());
@@ -1936,7 +1936,8 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   input_dims.insert(input_dims.begin()+axis, 1);
   // Reshape tensor.
   const bool ignore_first_dim = input_tensor.is_tensor();
-  nvinfer1::Dims new_dims = VectorToTrtDims(input_dims, ignore_first_dim);
+  nvinfer1::Dims new_dims = TensorShapeArrayToTrtDims(input_dims,
+                                                      ignore_first_dim);
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
@@ -1952,6 +1953,10 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
     return tensorflow::errors::InvalidArgument(
         "One input expected for Squeeze, at ", node_def.name());
   }
+  if (inputs.at(0).is_weights() ) {
+    return tensorflow::errors::Unimplemented(
+        "Squeeze expects tensor for input, at ", node_def.name());
+  }
   // Get input shape.
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
@@ -1979,7 +1984,7 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
     // Convert negative axis to corresponding positive axis.
     if (axis < 0) axis += input_rank;
     // Don't squeeze batch dim.
-    if (axis == 0) {
+    if (input_tensor.is_tensor() && axis == 0) {
       return tensorflow::errors::Unimplemented(
           "Cannot squeeze batch dimension, at ", node_def.name());
     }
@@ -1999,7 +2004,8 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
                    input_dims.end());
     // Reshape tensor.
   const bool ignore_first_dim = input_tensor.is_tensor();
-  nvinfer1::Dims new_dims = VectorToTrtDims(input_dims, ignore_first_dim);
+  nvinfer1::Dims new_dims = TensorShapeArrayToTrtDims(input_dims,
+                                                      ignore_first_dim);
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 74ca982f3f..27d6098b49 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2126,9 +2126,19 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
   Scope s = Scope::NewRootScope();
   auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
   auto weights = ops::Placeholder(s.WithOpName("weights"), DT_INT32);
-  auto expanddims = ops::ExpandDims(s.WithOpName("my_expanddims"), input, weights);
+  auto expanddims = ops::ExpandDims(s.WithOpName("my_expanddims"), input, 
+                                    weights);
   const NodeDef& node_def = expanddims.operation.node()->def();
 
+  {
+    // Input is weights, should fail.
+    Reset();
+    AddTestWeights<int32>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
+    AddTestWeights<int32>("weights", {1}, {1});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "ExpandDims expects tensor for input, at my_expanddims");
+  }
   {
     // Axis is a tensor, should fail.
     Reset();
@@ -2144,7 +2154,8 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     AddTestTensor("input", {1, 2, 3});
     AddTestWeights<int32>("weights", {1}, {0});
     RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
-        "Modifying batch dimension is not supported for ExpandDims, at my_expanddims");
+        "Modifying batch dimension is not supported for ExpandDims, at "
+        "my_expanddims");
   }
   {
     // Add dim at batch dimension via negative axis, should fail.
@@ -2153,7 +2164,8 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     // Input is rank 4 (batch dim included)
     AddTestWeights<int32>("weights", {1}, {-5});
     RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
-        "Modifying batch dimension is not supported for ExpandDims, at my_expanddims");
+        "Modifying batch dimension is not supported for ExpandDims, at "
+        "my_expanddims");
   }
   {
     // Axis > rank(input), should fail.
@@ -2250,6 +2262,15 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     return squeeze.operation.node()->def();
   };
 
+  {
+    // Input is weights, should fail.
+    Reset();
+    NodeDef node_def = get_squeeze_nodedef({0});
+    AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Squeeze expects tensor for input, at my_squeeze");
+  }
   {
     // Squeeze batch dim, should fail.
     Reset();
-- 
GitLab


From 06fe3c8ce5b2db058b633a3d93dedcb91a51b203 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 5 Dec 2018 11:19:33 -0800
Subject: [PATCH 1369/1554] Check that axis is a scalar

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index a83f2c7c48..3392e527c2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1913,6 +1913,10 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   const int input_rank = input_dims.size();
   // Get axis to expand on.
   TRT_ShapedWeights weights = inputs.at(1).weights();
+  if (weights.count() != 1) {
+    return tensorflow::errors::InvalidArgument(
+        "ExpandDims axis must be a scalar, at ", node_def.name());
+  }
   const int* weights_ptr =
       static_cast<int*>(const_cast<void*>(weights.GetValues()));
   int axis = weights_ptr[0];
-- 
GitLab


From 2fd76cbef3b30bd10807d6d660fedfa7f5451bf5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 11:28:49 -0800
Subject: [PATCH 1370/1554] batch_matmul_op_test.py: Updated to pass in TF2, by
 using gradient_checker_v2 (second try).

PiperOrigin-RevId: 224190234
---
 .../kernel_tests/batch_matmul_op_test.py      | 54 +++++++++----------
 1 file changed, 25 insertions(+), 29 deletions(-)

diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
index a0ad8151b2..931bb41424 100644
--- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
@@ -20,9 +20,9 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.python.framework import constant_op
+from tensorflow.python import tf2
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -105,32 +105,32 @@ class BatchMatmulOpTest(test.TestCase):
 
   def _testNonEmpty(self, dtype, adjoint_a, adjoint_b, use_static_shape):
 
-    def compareNonEmpty(self, a_shape, b_shape):
+    def CompareNonEmpty(self, a_shape, b_shape):
       self._compare(
           self._rand(a_shape, dtype),
           self._rand(b_shape, dtype), adjoint_a, adjoint_b, use_static_shape)
 
-    compareNonEmpty(self, [1, 2, 3], [1, 3, 5])
-    compareNonEmpty(self, [1, 2, 3], [1, 3, 1])
-    compareNonEmpty(self, [1, 1, 3], [1, 3, 5])
-    compareNonEmpty(self, [1, 2, 3], [1, 3, 5])
-    compareNonEmpty(self, [7, 1, 3], [7, 3, 5])
-    compareNonEmpty(self, [7, 2, 3], [7, 3, 1])
-    compareNonEmpty(self, [7, 2, 3], [7, 3, 5])
-    compareNonEmpty(self, [10, 64, 75], [10, 75, 30])
-    compareNonEmpty(self, [5, 7, 2, 3], [5, 7, 3, 5])
+    CompareNonEmpty(self, [1, 2, 3], [1, 3, 5])
+    CompareNonEmpty(self, [1, 2, 3], [1, 3, 1])
+    CompareNonEmpty(self, [1, 1, 3], [1, 3, 5])
+    CompareNonEmpty(self, [1, 2, 3], [1, 3, 5])
+    CompareNonEmpty(self, [7, 1, 3], [7, 3, 5])
+    CompareNonEmpty(self, [7, 2, 3], [7, 3, 1])
+    CompareNonEmpty(self, [7, 2, 3], [7, 3, 5])
+    CompareNonEmpty(self, [10, 64, 75], [10, 75, 30])
+    CompareNonEmpty(self, [5, 7, 2, 3], [5, 7, 3, 5])
 
   def _testEmpty(self, dtype, adjoint_a, adjoint_b, use_static_shape):
 
-    def compareEmpty(self, a_shape, b_shape):
+    def CompareEmpty(self, a_shape, b_shape):
       self._compare(
           np.zeros(a_shape).astype(dtype),
           np.zeros(b_shape).astype(dtype), adjoint_a, adjoint_b,
           use_static_shape)
 
-    compareEmpty(self, [0, 3, 2], [0, 2, 4])
-    compareEmpty(self, [3, 0, 2], [3, 2, 5])
-    compareEmpty(self, [3, 3, 2], [3, 2, 0])
+    CompareEmpty(self, [0, 3, 2], [0, 2, 4])
+    CompareEmpty(self, [3, 0, 2], [3, 2, 5])
+    CompareEmpty(self, [3, 3, 2], [3, 2, 0])
 
 
 def _GetBatchMatmulOpTest(dtype, adjoint_a, adjoint_b, use_static_shape):
@@ -154,17 +154,13 @@ class BatchMatmulGradientTest(test.TestCase):
     y = y_in if not adjoint_b else y_in.reshape(y_t_shape)
     epsilon = np.finfo(x.dtype).eps
     delta = epsilon**(1.0 / 3.0)
+    def Loss(x, y):
+      z = math_ops.matmul(x, y, adjoint_a, adjoint_b)
+      return math_ops.reduce_sum(z)
     with self.cached_session(use_gpu=True):
-      inx = constant_op.constant(x)
-      iny = constant_op.constant(y)
-      z = math_ops.matmul(inx, iny, adjoint_a, adjoint_b)
-      loss = math_ops.reduce_sum(z)
-      ((x_jacob_t, x_jacob_n),
-       (y_jacob_t, y_jacob_n)) = gradient_checker.compute_gradient(
-           [inx, iny], [x.shape, y.shape],
-           loss, [1],
-           x_init_value=[x, y],
-           delta=delta)
+      ((x_jacob_t, y_jacob_t),
+       (x_jacob_n, y_jacob_n)) = gradient_checker_v2.compute_gradient(
+           Loss, [x, y], delta=delta)
       tol = 20 * delta
       self.assertAllClose(x_jacob_t, x_jacob_n, rtol=tol, atol=tol)
       self.assertAllClose(y_jacob_t, y_jacob_n, rtol=tol, atol=tol)
@@ -202,11 +198,11 @@ if __name__ == "__main__":
     for adjoint_a_ in False, True:
       for adjoint_b_ in False, True:
         name = "%s_%s_%s" % (dtype_.__name__, adjoint_a_, adjoint_b_)
-        for use_static_shape in True, False:
+        for use_static_shape_ in set([True, tf2.enabled()]):
           setattr(BatchMatmulOpTest,
-                  "testBatchMatmulOp_" + name + ("_%s" % use_static_shape),
+                  "testBatchMatmulOp_" + name + ("_%s" % use_static_shape_),
                   _GetBatchMatmulOpTest(dtype_, adjoint_a_, adjoint_b_,
-                                        use_static_shape))
+                                        use_static_shape_))
         if dtype_ is not np.int32:
           setattr(BatchMatmulGradientTest, "testBatchMatmulGradient_" + name,
                   _GetBatchMatmulGradientTest(dtype_, adjoint_a_, adjoint_b_))
-- 
GitLab


From 0ed0aecc5c6a4807ce0dd35cb6fb29a7dfdc71d2 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Wed, 5 Dec 2018 11:39:28 -0800
Subject: [PATCH 1371/1554] Minor fixes to TFLite documentation.

PiperOrigin-RevId: 224192479
---
 tensorflow/lite/g3doc/convert/python_api.md         | 6 +++---
 tensorflow/lite/tutorials/post_training_quant.ipynb | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/g3doc/convert/python_api.md b/tensorflow/lite/g3doc/convert/python_api.md
index b914a34fa8..4d2c7361c9 100644
--- a/tensorflow/lite/g3doc/convert/python_api.md
+++ b/tensorflow/lite/g3doc/convert/python_api.md
@@ -19,9 +19,9 @@ be targeted to devices with mobile.
 
 ## API
 
-The API for converting TensorFlow models to TensorFlow Lite as of TensorFlow 1.9
-is `tf.lite.TFLiteConverter`. The API for calling the Python intepreter
-is `tf.lite.Interpreter`.
+The API for converting TensorFlow models to TensorFlow Lite is
+`tf.lite.TFLiteConverter`. The API for calling the Python interpreter is
+`tf.lite.Interpreter`.
 
 `TFLiteConverter` provides class methods based on the original format of the
 model. `TFLiteConverter.from_session()` is available for GraphDefs.
diff --git a/tensorflow/lite/tutorials/post_training_quant.ipynb b/tensorflow/lite/tutorials/post_training_quant.ipynb
index 3ff145d9ce..394ab0760b 100644
--- a/tensorflow/lite/tutorials/post_training_quant.ipynb
+++ b/tensorflow/lite/tutorials/post_training_quant.ipynb
@@ -235,9 +235,9 @@
         "id": "AT8BgkKmljOy"
       },
       "source": [
-        "Using the python `TocoConverter`, the saved model can be converted into a TFLite model.\n",
+        "Using the python `TFLiteConverter`, the saved model can be converted into a TFLite model.\n",
         "\n",
-        "First load the model using the `TocoConverter`:"
+        "First load the model using the `TFLiteConverter`:"
       ]
     },
     {
@@ -252,7 +252,7 @@
       "source": [
         "import tensorflow as tf\n",
         "tf.enable_eager_execution()\n",
-        "converter = tf.lite.TocoConverter.from_saved_model(saved_model_dir)\n",
+        "converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir)\n",
         "tflite_model = converter.convert()"
       ]
     },
@@ -648,7 +648,7 @@
         "graph_def_file = pathlib.Path(archive_path).parent/\"resnet_v2_101_299_frozen.pb\"\n",
         "input_arrays = [\"input\"] \n",
         "output_arrays = [\"output\"]\n",
-        "converter = tf.lite.TocoConverter.from_frozen_graph(\n",
+        "converter = tf.lite.TFLiteConverter.from_frozen_graph(\n",
         "  str(graph_def_file), input_arrays, output_arrays, input_shapes={\"input\":[1,299,299,3]})\n",
         "converter.post_training_quantize = True\n",
         "resnet_tflite_file = graph_def_file.parent/\"resnet_v2_101_quantized.tflite\"\n",
-- 
GitLab


From d2e1868c218c911cfdc96fd23e92d3b7a68fea7a Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Wed, 5 Dec 2018 11:43:09 -0800
Subject: [PATCH 1372/1554] Move Community upper-tab to right.

PiperOrigin-RevId: 224193154
---
 tensorflow/compiler/xla/g3doc/_book.yaml | 2 ++
 tensorflow/lite/g3doc/_book.yaml         | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/compiler/xla/g3doc/_book.yaml b/tensorflow/compiler/xla/g3doc/_book.yaml
index 12b7094705..267701e9c0 100644
--- a/tensorflow/compiler/xla/g3doc/_book.yaml
+++ b/tensorflow/compiler/xla/g3doc/_book.yaml
@@ -31,3 +31,5 @@ upper_tabs:
       - title: XLA compile API
         path: /xla/tutorials/xla_compile
         status: experimental
+
+- include: /_upper_tabs_right.yaml
diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml
index a51c7a667f..36bf4f4618 100644
--- a/tensorflow/lite/g3doc/_book.yaml
+++ b/tensorflow/lite/g3doc/_book.yaml
@@ -82,3 +82,5 @@ upper_tabs:
       contents:
       - title: API
         path: /api_docs/python/tf/lite
+
+- include: /_upper_tabs_right.yaml
-- 
GitLab


From f91a98ba8d2e58523569e1aa755da68ab3884318 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 5 Dec 2018 11:54:06 -0800
Subject: [PATCH 1373/1554] Enable using MKL-DNN for Eigen tensor contractions
 in opensource.

PiperOrigin-RevId: 224195106
---
 tensorflow/core/kernels/BUILD                 |  2 +-
 .../core/kernels/eigen_contraction_kernel.h   | 11 ++++++---
 third_party/mkl_dnn/mkldnn.BUILD              | 24 +++++++++++++++++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 60accc0f9b..98fa13a702 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -603,7 +603,7 @@ cc_library(
     deps = [
         "//third_party/eigen3",
     ] + select({
-        ":mkldnn_contraction_kernel": ["//third_party/intel_mkl_dnn:mkldnn_single_threaded"],
+        ":mkldnn_contraction_kernel": ["@mkl_dnn//:mkldnn_single_threaded"],
         "//conditions:default": [],
     }),
 )
diff --git a/tensorflow/core/kernels/eigen_contraction_kernel.h b/tensorflow/core/kernels/eigen_contraction_kernel.h
index 92d29e3995..66e93a83af 100644
--- a/tensorflow/core/kernels/eigen_contraction_kernel.h
+++ b/tensorflow/core/kernels/eigen_contraction_kernel.h
@@ -33,7 +33,7 @@ limitations under the License.
 //   #endif
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "third_party/intel_mkl_dnn/include/mkldnn.h"
+#include "mkldnn.h"
 
 namespace Eigen {
 namespace internal {
@@ -126,6 +126,11 @@ struct mkldnn_gemm_kernel</*Scalar*/ float, IndexType, OutputMapper,
                                       &alpha, blockA, &ldA, blockB, &ldB, &beta,
                                       const_cast<float*>(output.data()), &ldC);
     eigen_assert(st == 0);
+
+    // eigen_assert is a no-op in optimized mode so we add these to avoid
+    // compiler's unused-variable errors.
+    EIGEN_UNUSED_VARIABLE(max_index);
+    EIGEN_UNUSED_VARIABLE(st);
   }
 };
 
@@ -143,8 +148,8 @@ class TensorContractionBlocking<float, float, float, StorageIndex,
   // Multiply default choice of block size along M and N dimensions.
   // TODO(ezhulenev): Explore if this can work in general (kScaleM=2.0 worked
   // well in some of models).
-  static const float kScaleM = 1.5;
-  static const float kScaleN = 1.0;
+  static constexpr float kScaleM = 1.5;
+  static constexpr float kScaleN = 1.0;
 
   // Mkldnn Avx/Avx2/Avx512 unroll factors are: 8/16/48.
   static const StorageIndex kUnrollM = 48;
diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
index 7a8ed3bf43..d80c7135d6 100644
--- a/third_party/mkl_dnn/mkldnn.BUILD
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -63,3 +63,27 @@ cc_library(
         "//conditions:default": [],
     }),
 )
+
+cc_library(
+    name = "mkldnn_single_threaded",
+    srcs = glob([
+        "src/common/*.cpp",
+        "src/cpu/*.cpp",
+        "src/cpu/gemm/*.cpp",
+    ]),
+    hdrs = glob(["include/*"]),
+    copts = [
+        "-fexceptions",
+        "-DMKLDNN_THR=MKLDNN_THR_SEQ",  # Disables threading.
+    ],
+    includes = [
+        "include",
+        "src",
+        "src/common",
+        "src/cpu",
+        "src/cpu/gemm",
+        "src/cpu/xbyak",
+    ],
+    nocopts = "-fno-exceptions",
+    visibility = ["//visibility:public"],
+)
-- 
GitLab


From 5b2a39b431b293bf6ff5ada29c211d5bce5459d8 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Wed, 5 Dec 2018 12:14:00 -0800
Subject: [PATCH 1374/1554] Remove an unused import

---
 .../data/experimental/kernel_tests/make_csv_dataset_test.py      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
index ae0b7be51b..3b7b335e70 100644
--- a/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/make_csv_dataset_test.py
@@ -25,7 +25,6 @@ import numpy as np
 
 from tensorflow.python.data.experimental.ops import readers
 from tensorflow.python.data.kernel_tests import test_base
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-- 
GitLab


From 87db21b2a8311714ae4b6944db935d71f6f60506 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Wed, 5 Dec 2018 12:39:28 -0800
Subject: [PATCH 1375/1554] Internal Change

PiperOrigin-RevId: 224202833
---
 tensorflow/contrib/distribute/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4c9c35da5a..4ea1fa050a 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -611,6 +611,7 @@ cuda_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "noguitar",  # TODO(b/120025010): Re-enable this test on Guitar.
         "notsan",
     ],
 )
-- 
GitLab


From 585d7b313ec8de364fb5297db7156fbc927e0cd1 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 5 Dec 2018 13:31:23 -0800
Subject: [PATCH 1376/1554] [Grappler] Add more tests to function_optimizer

PiperOrigin-RevId: 224211536
---
 .../grappler/optimizers/function_optimizer.cc |  6 ++
 .../optimizers/function_optimizer_test.cc     | 90 +++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 9c25eb08d8..8beebb9049 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -1307,6 +1307,9 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
       std::back_inserter(happens_before),
       [](const GraphView::OutputPort port) { return port.node->name(); });
 
+  VLOG(3) << "Happens before set (size = " << happens_before.size()
+          << "): " << absl::StrJoin(happens_before, ", ");
+
   // Nodes that must observe side effects to the captured resources.
   std::vector<string> happens_after;
   absl::c_transform(
@@ -1314,6 +1317,9 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
       std::back_inserter(happens_after),
       [](const GraphView::InputPort port) { return port.node->name(); });
 
+  VLOG(3) << "Happens after set (size = " << happens_after.size()
+          << "): " << absl::StrJoin(happens_after, ", ");
+
   // Regular (positional) inputs to the function call.
   std::vector<SafeTensorId> inputs;
   for (const string& input : func_node.input()) {
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index de091dbe98..935fdc6409 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -878,7 +878,9 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
 
   GrapplerItem optimized(item, std::move(optimized_graph));
   auto tensors = EvaluateFetchNodes(optimized);
+  ASSERT_EQ(tensors.size(), 2);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  test::ExpectTensorEqual<float>(tensors_expected[1], tensors[1]);
 }
 
 TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) {
@@ -934,6 +936,94 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) {
   CompareGraphs(expected, optimized_graph);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  const Tensor kOne = test::AsScalar<float>(1.0);
+  const Tensor kTwo = test::AsScalar<float>(2.0);
+  const TensorShape scalar = TensorShape({});
+
+  // MyMul doesn't have any side-effectful nodes in the function body, but the
+  // optimized graph has a control dependency edge `f1->f2`.
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Build a graph to compute:
+  //   a = Placeholder
+  //   b = Placeholder
+  //   f1 = MyMul(a, b)
+  //   f2 = MyMul(a, b, ^f1)  <-- control dependency on inlined function!
+  //   return f2
+  GrapplerItem item;
+  item.fetch = {"out"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       // Call function first time.
+       NDef("f1", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       // Call function second time.
+       NDef("f2", "PartitionedCall", {"f1", "f1", "^f1"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       // Return result of f2.
+       NDef("out", "Identity", {"f2"}, {{"T", DT_FLOAT}}, kDevice)},
+
+      // Function library.
+      {mul_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       // Function body of a first function call inlined into the graph.
+       NDef("f1/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/mul", "Mul", {"f1/x", "f1/y"}, {{"T", DT_FLOAT}}, kDevice),
+
+       // Function body of a second function call also inlined into the graph,
+       // and input nodes read directly from the inlined nodes of the first
+       // function call, and control dependency edge removed.
+       NDef("f2/x", "Identity", {"f1/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/y", "Identity", {"f1/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/mul", "Mul", {"f2/x", "f2/y"}, {{"T", DT_FLOAT}}, kDevice),
+
+       // Return directly from inlined node of f2.
+       NDef("out", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice)},
+
+      // Function library.
+      {mul_func});
+
+  CompareGraphs(expected, optimized_graph);
+
+  item.feed.emplace_back("a", kOne);
+  item.feed.emplace_back("b", kTwo);
+
+  auto tensors_expected = EvaluateFetchNodes(item);
+  ASSERT_EQ(tensors_expected.size(), 1);
+
+  GrapplerItem optimized(item, std::move(optimized_graph));
+  auto tensors = EvaluateFetchNodes(optimized);
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 11a1fc8cddc4e04dee1cd0155992fdd1bba6b111 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 5 Dec 2018 13:38:43 -0800
Subject: [PATCH 1377/1554] Remove some class symbols from tf 2.0. - AttrValue
 - ConfigProto - GPUOptions - GraphDef - GraphOptions - HistogramProto -
 LogMessage - MetaGraphDef - NameAttrList - NodeDef - OptimizerOptions -
 RunMetadata - RunOptions - TensorInfo

PiperOrigin-RevId: 224212923
---
 tensorflow/python/__init__.py                 |  26 +--
 .../tensorflow.-attr-value.-list-value.pbtxt  |  70 --------
 .../golden/v2/tensorflow.-attr-value.pbtxt    | 151 -----------------
 ...ow.-config-proto.-device-count-entry.pbtxt |  21 ---
 ...nsorflow.-config-proto.-experimental.pbtxt |  34 ----
 .../golden/v2/tensorflow.-config-proto.pbtxt  | 158 ------------------
 .../golden/v2/tensorflow.-g-p-u-options.pbtxt |  98 -----------
 .../api/golden/v2/tensorflow.-graph-def.pbtxt |  36 ----
 .../golden/v2/tensorflow.-graph-options.pbtxt |  67 --------
 .../v2/tensorflow.-histogram-proto.pbtxt      |  54 ------
 .../golden/v2/tensorflow.-log-message.pbtxt   |  46 -----
 ...meta-graph-def.-collection-def-entry.pbtxt |  22 ---
 ...rflow.-meta-graph-def.-meta-info-def.pbtxt |  50 ------
 ...-meta-graph-def.-signature-def-entry.pbtxt |  22 ---
 .../v2/tensorflow.-meta-graph-def.pbtxt       | 133 ---------------
 ...nsorflow.-name-attr-list.-attr-entry.pbtxt |  22 ---
 .../v2/tensorflow.-name-attr-list.pbtxt       |  38 -----
 .../v2/tensorflow.-node-def.-attr-entry.pbtxt |  22 ---
 .../api/golden/v2/tensorflow.-node-def.pbtxt  |  56 -------
 .../v2/tensorflow.-optimizer-options.pbtxt    |  74 --------
 .../golden/v2/tensorflow.-run-metadata.pbtxt  |  27 ---
 ...ensorflow.-run-options.-experimental.pbtxt |  18 --
 .../golden/v2/tensorflow.-run-options.pbtxt   |  89 ----------
 .../tools/api/golden/v2/tensorflow.pbtxt      |  52 ------
 tensorflow/tools/compatibility/renames_v2.py  |  13 ++
 .../tools/compatibility/tf_upgrade_v2_test.py |   2 +
 26 files changed, 28 insertions(+), 1373 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-attr-value.-list-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-attr-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-device-count-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-g-p-u-options.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-graph-def.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-graph-options.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-histogram-proto.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-log-message.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-collection-def-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-meta-info-def.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-signature-def-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.-attr-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-node-def.-attr-entry.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-node-def.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-optimizer-options.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-run-metadata.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-run-options.-experimental.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-run-options.pbtxt

diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 62a6e9f6d4..a48e2b6f38 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -145,20 +145,20 @@ nn.rnn_cell = rnn_cell
 
 # Export protos
 # pylint: disable=undefined-variable
-tf_export('AttrValue')(AttrValue)
-tf_export('ConfigProto')(ConfigProto)
+tf_export(v1=['AttrValue'])(AttrValue)
+tf_export(v1=['ConfigProto'])(ConfigProto)
 tf_export('Event', 'summary.Event')(Event)
-tf_export('GPUOptions')(GPUOptions)
-tf_export('GraphDef')(GraphDef)
-tf_export('GraphOptions')(GraphOptions)
-tf_export('HistogramProto')(HistogramProto)
-tf_export('LogMessage')(LogMessage)
-tf_export('MetaGraphDef')(MetaGraphDef)
-tf_export('NameAttrList')(NameAttrList)
-tf_export('NodeDef')(NodeDef)
-tf_export('OptimizerOptions')(OptimizerOptions)
-tf_export('RunMetadata')(RunMetadata)
-tf_export('RunOptions')(RunOptions)
+tf_export(v1=['GPUOptions'])(GPUOptions)
+tf_export(v1=['GraphDef'])(GraphDef)
+tf_export(v1=['GraphOptions'])(GraphOptions)
+tf_export(v1=['HistogramProto'])(HistogramProto)
+tf_export(v1=['LogMessage'])(LogMessage)
+tf_export(v1=['MetaGraphDef'])(MetaGraphDef)
+tf_export(v1=['NameAttrList'])(NameAttrList)
+tf_export(v1=['NodeDef'])(NodeDef)
+tf_export(v1=['OptimizerOptions'])(OptimizerOptions)
+tf_export(v1=['RunMetadata'])(RunMetadata)
+tf_export(v1=['RunOptions'])(RunOptions)
 tf_export('SessionLog', 'summary.SessionLog')(SessionLog)
 tf_export('Summary', 'summary.Summary')(Summary)
 tf_export('summary.SummaryDescription')(SummaryDescription)
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-attr-value.-list-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-attr-value.-list-value.pbtxt
deleted file mode 100644
index f1dffd5952..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-attr-value.-list-value.pbtxt
+++ /dev/null
@@ -1,70 +0,0 @@
-path: "tensorflow.AttrValue.ListValue"
-tf_proto {
-  descriptor {
-    name: "ListValue"
-    field {
-      name: "s"
-      number: 2
-      label: LABEL_REPEATED
-      type: TYPE_BYTES
-    }
-    field {
-      name: "i"
-      number: 3
-      label: LABEL_REPEATED
-      type: TYPE_INT64
-      options {
-        packed: true
-      }
-    }
-    field {
-      name: "f"
-      number: 4
-      label: LABEL_REPEATED
-      type: TYPE_FLOAT
-      options {
-        packed: true
-      }
-    }
-    field {
-      name: "b"
-      number: 5
-      label: LABEL_REPEATED
-      type: TYPE_BOOL
-      options {
-        packed: true
-      }
-    }
-    field {
-      name: "type"
-      number: 6
-      label: LABEL_REPEATED
-      type: TYPE_ENUM
-      type_name: ".tensorflow.DataType"
-      options {
-        packed: true
-      }
-    }
-    field {
-      name: "shape"
-      number: 7
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorShapeProto"
-    }
-    field {
-      name: "tensor"
-      number: 8
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorProto"
-    }
-    field {
-      name: "func"
-      number: 9
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.NameAttrList"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-attr-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-attr-value.pbtxt
deleted file mode 100644
index 6ccd64f428..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-attr-value.pbtxt
+++ /dev/null
@@ -1,151 +0,0 @@
-path: "tensorflow.AttrValue"
-tf_proto {
-  descriptor {
-    name: "AttrValue"
-    field {
-      name: "s"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "i"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-      oneof_index: 0
-    }
-    field {
-      name: "f"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-      oneof_index: 0
-    }
-    field {
-      name: "b"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-      oneof_index: 0
-    }
-    field {
-      name: "type"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.DataType"
-      oneof_index: 0
-    }
-    field {
-      name: "shape"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorShapeProto"
-      oneof_index: 0
-    }
-    field {
-      name: "tensor"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorProto"
-      oneof_index: 0
-    }
-    field {
-      name: "list"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.AttrValue.ListValue"
-      oneof_index: 0
-    }
-    field {
-      name: "func"
-      number: 10
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.NameAttrList"
-      oneof_index: 0
-    }
-    field {
-      name: "placeholder"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      oneof_index: 0
-    }
-    nested_type {
-      name: "ListValue"
-      field {
-        name: "s"
-        number: 2
-        label: LABEL_REPEATED
-        type: TYPE_BYTES
-      }
-      field {
-        name: "i"
-        number: 3
-        label: LABEL_REPEATED
-        type: TYPE_INT64
-        options {
-          packed: true
-        }
-      }
-      field {
-        name: "f"
-        number: 4
-        label: LABEL_REPEATED
-        type: TYPE_FLOAT
-        options {
-          packed: true
-        }
-      }
-      field {
-        name: "b"
-        number: 5
-        label: LABEL_REPEATED
-        type: TYPE_BOOL
-        options {
-          packed: true
-        }
-      }
-      field {
-        name: "type"
-        number: 6
-        label: LABEL_REPEATED
-        type: TYPE_ENUM
-        type_name: ".tensorflow.DataType"
-        options {
-          packed: true
-        }
-      }
-      field {
-        name: "shape"
-        number: 7
-        label: LABEL_REPEATED
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorShapeProto"
-      }
-      field {
-        name: "tensor"
-        number: 8
-        label: LABEL_REPEATED
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorProto"
-      }
-      field {
-        name: "func"
-        number: 9
-        label: LABEL_REPEATED
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.NameAttrList"
-      }
-    }
-    oneof_decl {
-      name: "value"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-device-count-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-device-count-entry.pbtxt
deleted file mode 100644
index d9b1426828..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-device-count-entry.pbtxt
+++ /dev/null
@@ -1,21 +0,0 @@
-path: "tensorflow.ConfigProto.DeviceCountEntry"
-tf_proto {
-  descriptor {
-    name: "DeviceCountEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt
deleted file mode 100644
index caa72fe5a6..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.-experimental.pbtxt
+++ /dev/null
@@ -1,34 +0,0 @@
-path: "tensorflow.ConfigProto.Experimental"
-tf_proto {
-  descriptor {
-    name: "Experimental"
-    field {
-      name: "collective_group_leader"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "executor_type"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "recv_buf_max_chunk"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "use_numa_affinity"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    reserved_range {
-      start: 2
-      end: 3
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt
deleted file mode 100644
index b505d81350..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-config-proto.pbtxt
+++ /dev/null
@@ -1,158 +0,0 @@
-path: "tensorflow.ConfigProto"
-tf_proto {
-  descriptor {
-    name: "ConfigProto"
-    field {
-      name: "device_count"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.ConfigProto.DeviceCountEntry"
-    }
-    field {
-      name: "intra_op_parallelism_threads"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "inter_op_parallelism_threads"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "use_per_session_threads"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "session_inter_op_thread_pool"
-      number: 12
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.ThreadPoolOptionProto"
-    }
-    field {
-      name: "placement_period"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "device_filters"
-      number: 4
-      label: LABEL_REPEATED
-      type: TYPE_STRING
-    }
-    field {
-      name: "gpu_options"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.GPUOptions"
-    }
-    field {
-      name: "allow_soft_placement"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "log_device_placement"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "graph_options"
-      number: 10
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.GraphOptions"
-    }
-    field {
-      name: "operation_timeout_in_ms"
-      number: 11
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "rpc_options"
-      number: 13
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.RPCOptions"
-    }
-    field {
-      name: "cluster_def"
-      number: 14
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.ClusterDef"
-    }
-    field {
-      name: "isolate_session_state"
-      number: 15
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "experimental"
-      number: 16
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.ConfigProto.Experimental"
-    }
-    nested_type {
-      name: "DeviceCountEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      options {
-        map_entry: true
-      }
-    }
-    nested_type {
-      name: "Experimental"
-      field {
-        name: "collective_group_leader"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "executor_type"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "recv_buf_max_chunk"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "use_numa_affinity"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_BOOL
-      }
-      reserved_range {
-        start: 2
-        end: 3
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-g-p-u-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-g-p-u-options.pbtxt
deleted file mode 100644
index a2cc07483a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-g-p-u-options.pbtxt
+++ /dev/null
@@ -1,98 +0,0 @@
-path: "tensorflow.GPUOptions"
-tf_proto {
-  descriptor {
-    name: "GPUOptions"
-    field {
-      name: "per_process_gpu_memory_fraction"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "allow_growth"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "allocator_type"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "deferred_deletion_bytes"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "visible_device_list"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "polling_active_delay_usecs"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "polling_inactive_delay_msecs"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "force_gpu_compatible"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "experimental"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.GPUOptions.Experimental"
-    }
-    nested_type {
-      name: "Experimental"
-      field {
-        name: "virtual_devices"
-        number: 1
-        label: LABEL_REPEATED
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.GPUOptions.Experimental.VirtualDevices"
-      }
-      field {
-        name: "use_unified_memory"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_BOOL
-      }
-      field {
-        name: "num_dev_to_dev_copy_streams"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "collective_ring_order"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      nested_type {
-        name: "VirtualDevices"
-        field {
-          name: "memory_limit_mb"
-          number: 1
-          label: LABEL_REPEATED
-          type: TYPE_FLOAT
-        }
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-graph-def.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-graph-def.pbtxt
deleted file mode 100644
index 19eccff03d..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-graph-def.pbtxt
+++ /dev/null
@@ -1,36 +0,0 @@
-path: "tensorflow.GraphDef"
-tf_proto {
-  descriptor {
-    name: "GraphDef"
-    field {
-      name: "node"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.NodeDef"
-    }
-    field {
-      name: "versions"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.VersionDef"
-    }
-    field {
-      name: "version"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-      options {
-        deprecated: true
-      }
-    }
-    field {
-      name: "library"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.FunctionDefLibrary"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-graph-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-graph-options.pbtxt
deleted file mode 100644
index a9f99bc171..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-graph-options.pbtxt
+++ /dev/null
@@ -1,67 +0,0 @@
-path: "tensorflow.GraphOptions"
-tf_proto {
-  descriptor {
-    name: "GraphOptions"
-    field {
-      name: "enable_recv_scheduling"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "optimizer_options"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.OptimizerOptions"
-    }
-    field {
-      name: "build_cost_model"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "build_cost_model_after"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "infer_shapes"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "place_pruned_graph"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "enable_bfloat16_sendrecv"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "timeline_step"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "rewrite_options"
-      number: 10
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.RewriterConfig"
-    }
-    reserved_range {
-      start: 1
-      end: 2
-    }
-    reserved_name: "skip_common_subexpression_elimination"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-histogram-proto.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-histogram-proto.pbtxt
deleted file mode 100644
index d4402f330b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-histogram-proto.pbtxt
+++ /dev/null
@@ -1,54 +0,0 @@
-path: "tensorflow.HistogramProto"
-tf_proto {
-  descriptor {
-    name: "HistogramProto"
-    field {
-      name: "min"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "max"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "num"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "sum"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "sum_squares"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "bucket_limit"
-      number: 6
-      label: LABEL_REPEATED
-      type: TYPE_DOUBLE
-      options {
-        packed: true
-      }
-    }
-    field {
-      name: "bucket"
-      number: 7
-      label: LABEL_REPEATED
-      type: TYPE_DOUBLE
-      options {
-        packed: true
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-log-message.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-log-message.pbtxt
deleted file mode 100644
index 5023aa96bf..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-log-message.pbtxt
+++ /dev/null
@@ -1,46 +0,0 @@
-path: "tensorflow.LogMessage"
-tf_proto {
-  descriptor {
-    name: "LogMessage"
-    field {
-      name: "level"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.LogMessage.Level"
-    }
-    field {
-      name: "message"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    enum_type {
-      name: "Level"
-      value {
-        name: "UNKNOWN"
-        number: 0
-      }
-      value {
-        name: "DEBUGGING"
-        number: 10
-      }
-      value {
-        name: "INFO"
-        number: 20
-      }
-      value {
-        name: "WARN"
-        number: 30
-      }
-      value {
-        name: "ERROR"
-        number: 40
-      }
-      value {
-        name: "FATAL"
-        number: 50
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-collection-def-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-collection-def-entry.pbtxt
deleted file mode 100644
index 0ba09bec4b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-collection-def-entry.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-path: "tensorflow.MetaGraphDef.CollectionDefEntry"
-tf_proto {
-  descriptor {
-    name: "CollectionDefEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.CollectionDef"
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-meta-info-def.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-meta-info-def.pbtxt
deleted file mode 100644
index 41c62a407b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-meta-info-def.pbtxt
+++ /dev/null
@@ -1,50 +0,0 @@
-path: "tensorflow.MetaGraphDef.MetaInfoDef"
-tf_proto {
-  descriptor {
-    name: "MetaInfoDef"
-    field {
-      name: "meta_graph_version"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "stripped_op_list"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.OpList"
-    }
-    field {
-      name: "any_info"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".google.protobuf.Any"
-    }
-    field {
-      name: "tags"
-      number: 4
-      label: LABEL_REPEATED
-      type: TYPE_STRING
-    }
-    field {
-      name: "tensorflow_version"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "tensorflow_git_version"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "stripped_default_attrs"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-signature-def-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-signature-def-entry.pbtxt
deleted file mode 100644
index 73dc414a77..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.-signature-def-entry.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-path: "tensorflow.MetaGraphDef.SignatureDefEntry"
-tf_proto {
-  descriptor {
-    name: "SignatureDefEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SignatureDef"
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.pbtxt
deleted file mode 100644
index d71c2358c9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-meta-graph-def.pbtxt
+++ /dev/null
@@ -1,133 +0,0 @@
-path: "tensorflow.MetaGraphDef"
-tf_proto {
-  descriptor {
-    name: "MetaGraphDef"
-    field {
-      name: "meta_info_def"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.MetaGraphDef.MetaInfoDef"
-    }
-    field {
-      name: "graph_def"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.GraphDef"
-    }
-    field {
-      name: "saver_def"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SaverDef"
-    }
-    field {
-      name: "collection_def"
-      number: 4
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.MetaGraphDef.CollectionDefEntry"
-    }
-    field {
-      name: "signature_def"
-      number: 5
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.MetaGraphDef.SignatureDefEntry"
-    }
-    field {
-      name: "asset_file_def"
-      number: 6
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.AssetFileDef"
-    }
-    nested_type {
-      name: "MetaInfoDef"
-      field {
-        name: "meta_graph_version"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "stripped_op_list"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.OpList"
-      }
-      field {
-        name: "any_info"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".google.protobuf.Any"
-      }
-      field {
-        name: "tags"
-        number: 4
-        label: LABEL_REPEATED
-        type: TYPE_STRING
-      }
-      field {
-        name: "tensorflow_version"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "tensorflow_git_version"
-        number: 6
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "stripped_default_attrs"
-        number: 7
-        label: LABEL_OPTIONAL
-        type: TYPE_BOOL
-      }
-    }
-    nested_type {
-      name: "CollectionDefEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.CollectionDef"
-      }
-      options {
-        map_entry: true
-      }
-    }
-    nested_type {
-      name: "SignatureDefEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.SignatureDef"
-      }
-      options {
-        map_entry: true
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.-attr-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.-attr-entry.pbtxt
deleted file mode 100644
index b119b20877..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.-attr-entry.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-path: "tensorflow.NameAttrList.AttrEntry"
-tf_proto {
-  descriptor {
-    name: "AttrEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.AttrValue"
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.pbtxt
deleted file mode 100644
index fcdb411ffc..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-name-attr-list.pbtxt
+++ /dev/null
@@ -1,38 +0,0 @@
-path: "tensorflow.NameAttrList"
-tf_proto {
-  descriptor {
-    name: "NameAttrList"
-    field {
-      name: "name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "attr"
-      number: 2
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.NameAttrList.AttrEntry"
-    }
-    nested_type {
-      name: "AttrEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.AttrValue"
-      }
-      options {
-        map_entry: true
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-node-def.-attr-entry.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-node-def.-attr-entry.pbtxt
deleted file mode 100644
index 622e4c3d0f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-node-def.-attr-entry.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-path: "tensorflow.NodeDef.AttrEntry"
-tf_proto {
-  descriptor {
-    name: "AttrEntry"
-    field {
-      name: "key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.AttrValue"
-    }
-    options {
-      map_entry: true
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-node-def.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-node-def.pbtxt
deleted file mode 100644
index 646fa8abb9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-node-def.pbtxt
+++ /dev/null
@@ -1,56 +0,0 @@
-path: "tensorflow.NodeDef"
-tf_proto {
-  descriptor {
-    name: "NodeDef"
-    field {
-      name: "name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "op"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "input"
-      number: 3
-      label: LABEL_REPEATED
-      type: TYPE_STRING
-    }
-    field {
-      name: "device"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "attr"
-      number: 5
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.NodeDef.AttrEntry"
-    }
-    nested_type {
-      name: "AttrEntry"
-      field {
-        name: "key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.AttrValue"
-      }
-      options {
-        map_entry: true
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-optimizer-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-optimizer-options.pbtxt
deleted file mode 100644
index 3ccf9d459b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-optimizer-options.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.OptimizerOptions"
-tf_proto {
-  descriptor {
-    name: "OptimizerOptions"
-    field {
-      name: "do_common_subexpression_elimination"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "do_constant_folding"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "max_folded_constant_in_bytes"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "do_function_inlining"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "opt_level"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.OptimizerOptions.Level"
-    }
-    field {
-      name: "global_jit_level"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.OptimizerOptions.GlobalJitLevel"
-    }
-    enum_type {
-      name: "Level"
-      value {
-        name: "L1"
-        number: 0
-      }
-      value {
-        name: "L0"
-        number: -1
-      }
-    }
-    enum_type {
-      name: "GlobalJitLevel"
-      value {
-        name: "DEFAULT"
-        number: 0
-      }
-      value {
-        name: "OFF"
-        number: -1
-      }
-      value {
-        name: "ON_1"
-        number: 1
-      }
-      value {
-        name: "ON_2"
-        number: 2
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-run-metadata.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-run-metadata.pbtxt
deleted file mode 100644
index 1287940326..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-run-metadata.pbtxt
+++ /dev/null
@@ -1,27 +0,0 @@
-path: "tensorflow.RunMetadata"
-tf_proto {
-  descriptor {
-    name: "RunMetadata"
-    field {
-      name: "step_stats"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.StepStats"
-    }
-    field {
-      name: "cost_graph"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.CostGraphDef"
-    }
-    field {
-      name: "partition_graphs"
-      number: 3
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.GraphDef"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-run-options.-experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-run-options.-experimental.pbtxt
deleted file mode 100644
index 47b5b56faf..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-run-options.-experimental.pbtxt
+++ /dev/null
@@ -1,18 +0,0 @@
-path: "tensorflow.RunOptions.Experimental"
-tf_proto {
-  descriptor {
-    name: "Experimental"
-    field {
-      name: "collective_graph_key"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "use_run_handler_pool"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-run-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-run-options.pbtxt
deleted file mode 100644
index c0c2e7b9f8..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-run-options.pbtxt
+++ /dev/null
@@ -1,89 +0,0 @@
-path: "tensorflow.RunOptions"
-tf_proto {
-  descriptor {
-    name: "RunOptions"
-    field {
-      name: "trace_level"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.RunOptions.TraceLevel"
-    }
-    field {
-      name: "timeout_in_ms"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "inter_op_thread_pool"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "output_partition_graphs"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "debug_options"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.DebugOptions"
-    }
-    field {
-      name: "report_tensor_allocations_upon_oom"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_BOOL
-    }
-    field {
-      name: "experimental"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.RunOptions.Experimental"
-    }
-    nested_type {
-      name: "Experimental"
-      field {
-        name: "collective_graph_key"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "use_run_handler_pool"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_BOOL
-      }
-    }
-    enum_type {
-      name: "TraceLevel"
-      value {
-        name: "NO_TRACE"
-        number: 0
-      }
-      value {
-        name: "SOFTWARE_TRACE"
-        number: 1
-      }
-      value {
-        name: "HARDWARE_TRACE"
-        number: 2
-      }
-      value {
-        name: "FULL_TRACE"
-        number: 3
-      }
-    }
-    reserved_range {
-      start: 4
-      end: 5
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 21df760b1e..4fa57969ef 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -4,14 +4,6 @@ tf_module {
     name: "AggregationMethod"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "AttrValue"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "ConfigProto"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "DType"
     mtype: "<type \'type\'>"
@@ -24,10 +16,6 @@ tf_module {
     name: "FIFOQueue"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "GPUOptions"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "GradientTape"
     mtype: "<type \'type\'>"
@@ -36,58 +24,18 @@ tf_module {
     name: "Graph"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "GraphDef"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "GraphOptions"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "HistogramProto"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "IndexedSlices"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "LogMessage"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "MetaGraphDef"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "NameAttrList"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "NodeDef"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "Operation"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "OptimizerOptions"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "RegisterGradient"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "RunMetadata"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "RunOptions"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "SessionLog"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index fd73bec0a7..0352083389 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -26,27 +26,38 @@ from __future__ import print_function
 
 renames = {
     'tf.AUTO_REUSE': 'tf.compat.v1.AUTO_REUSE',
+    'tf.AttrValue': 'tf.compat.v1.AttrValue',
     'tf.COMPILER_VERSION': 'tf.version.COMPILER_VERSION',
     'tf.CXX11_ABI_FLAG': 'tf.sysconfig.CXX11_ABI_FLAG',
     'tf.ConditionalAccumulator': 'tf.compat.v1.ConditionalAccumulator',
     'tf.ConditionalAccumulatorBase': 'tf.compat.v1.ConditionalAccumulatorBase',
+    'tf.ConfigProto': 'tf.compat.v1.ConfigProto',
     'tf.DeviceSpec': 'tf.compat.v1.DeviceSpec',
     'tf.Dimension': 'tf.compat.v1.Dimension',
     'tf.FixedLenFeature': 'tf.io.FixedLenFeature',
     'tf.FixedLenSequenceFeature': 'tf.io.FixedLenSequenceFeature',
     'tf.FixedLengthRecordReader': 'tf.compat.v1.FixedLengthRecordReader',
     'tf.GIT_VERSION': 'tf.version.GIT_VERSION',
+    'tf.GPUOptions': 'tf.compat.v1.GPUOptions',
     'tf.GRAPH_DEF_VERSION': 'tf.version.GRAPH_DEF_VERSION',
     'tf.GRAPH_DEF_VERSION_MIN_CONSUMER': 'tf.version.GRAPH_DEF_VERSION_MIN_CONSUMER',
     'tf.GRAPH_DEF_VERSION_MIN_PRODUCER': 'tf.version.GRAPH_DEF_VERSION_MIN_PRODUCER',
+    'tf.GraphDef': 'tf.compat.v1.GraphDef',
     'tf.GraphKeys': 'tf.compat.v1.GraphKeys',
+    'tf.GraphOptions': 'tf.compat.v1.GraphOptions',
+    'tf.HistogramProto': 'tf.compat.v1.HistogramProto',
     'tf.IdentityReader': 'tf.compat.v1.IdentityReader',
     'tf.InteractiveSession': 'tf.compat.v1.InteractiveSession',
     'tf.LMDBReader': 'tf.compat.v1.LMDBReader',
+    'tf.LogMessage': 'tf.compat.v1.LogMessage',
     'tf.MONOLITHIC_BUILD': 'tf.sysconfig.MONOLITHIC_BUILD',
+    'tf.MetaGraphDef': 'tf.compat.v1.MetaGraphDef',
+    'tf.NameAttrList': 'tf.compat.v1.NameAttrList',
     'tf.NoGradient': 'tf.no_gradient',
+    'tf.NodeDef': 'tf.compat.v1.NodeDef',
     'tf.NotDifferentiable': 'tf.no_gradient',
     'tf.OpError': 'tf.errors.OpError',
+    'tf.OptimizerOptions': 'tf.compat.v1.OptimizerOptions',
     'tf.PaddingFIFOQueue': 'tf.io.PaddingFIFOQueue',
     'tf.Print': 'tf.compat.v1.Print',
     'tf.PriorityQueue': 'tf.io.PriorityQueue',
@@ -54,6 +65,8 @@ renames = {
     'tf.QueueBase': 'tf.io.QueueBase',
     'tf.RandomShuffleQueue': 'tf.io.RandomShuffleQueue',
     'tf.ReaderBase': 'tf.compat.v1.ReaderBase',
+    'tf.RunMetadata': 'tf.compat.v1.RunMetadata',
+    'tf.RunOptions': 'tf.compat.v1.RunOptions',
     'tf.Session': 'tf.compat.v1.Session',
     'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator',
     'tf.SparseFeature': 'tf.io.SparseFeature',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index bc92cab98c..366ce0759d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -142,6 +142,8 @@ class TestUpgrade(test_util.TensorFlowTestCase):
 
     # Converts all symbols in the v1 namespace to the v2 namespace, raising
     # an error if the target of the conversion is not in the v2 namespace.
+    # Please regenerate the renames file or edit any manual renames if this
+    # test fails.
     def conversion_visitor(unused_path, unused_parent, children):
       for child in children:
         _, attr = tf_decorator.unwrap(child[1])
-- 
GitLab


From c7546b51fb3cbb9f41dd0c7ee6b04bf4617bc392 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Wed, 5 Dec 2018 13:49:43 -0800
Subject: [PATCH 1378/1554] Fix TFLite Makefile

PiperOrigin-RevId: 224214870
---
 tensorflow/lite/tools/make/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 363a069d5e..dbcaf4de3c 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -116,8 +116,8 @@ tensorflow/lite/mmap_allocation.cc \
 tensorflow/lite/nnapi_delegate.cc
 else
 CORE_CC_EXCLUDE_SRCS += \
-tensorflow/contrib/lite/mmap_allocation_disabled.cc \
-tensorflow/contrib/lite/nnapi_delegate_disabled.cc
+tensorflow/lite/mmap_allocation_disabled.cc \
+tensorflow/lite/nnapi_delegate_disabled.cc
 endif
 # Filter out all the excluded files.
 TF_LITE_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS))
-- 
GitLab


From 1b5189b732536db66528cb29eed88874d713cdea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 13:53:23 -0800
Subject: [PATCH 1379/1554] Disabling flaky rate_test.

PiperOrigin-RevId: 224215488
---
 tensorflow/contrib/rate/BUILD | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/contrib/rate/BUILD b/tensorflow/contrib/rate/BUILD
index c461a7145e..76db9aecf6 100644
--- a/tensorflow/contrib/rate/BUILD
+++ b/tensorflow/contrib/rate/BUILD
@@ -34,6 +34,11 @@ py_test(
     name = "rate_test",
     size = "small",
     srcs = ["rate_test.py"],
+    tags = [
+        "manual",  # TODO(b/120555555)
+        "no_oss",  # TODO(b/120555555)
+        "notap",  # TODO(b/120555555)
+    ],
     deps = [
         ":rate",
         "//tensorflow/python:array_ops",
-- 
GitLab


From 249fc1e331fdee277b0b1ce0b0ef4d0ff7e47fcb Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 5 Dec 2018 14:10:32 -0800
Subject: [PATCH 1380/1554] Dynamic dimension inference

DynamicDimensionInference analyzes each HLO instruction in a graph and
inferences which dimensions are dynamic and which scalar instructions
represent the runtime real size of those dynamic dimensions.

Note: This CL only implements the skeleton that has enough support for
dynamic batch for Resnet50. More supports for other models will be
added incrementally later.
PiperOrigin-RevId: 224218787
---
 tensorflow/compiler/xla/service/BUILD         |  35 ++
 .../service/dynamic_dimension_inference.cc    | 459 +++++++++++++++
 .../xla/service/dynamic_dimension_inference.h | 112 ++++
 .../dynamic_dimension_inference_test.cc       | 535 ++++++++++++++++++
 4 files changed, 1141 insertions(+)
 create mode 100644 tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
 create mode 100644 tensorflow/compiler/xla/service/dynamic_dimension_inference.h
 create mode 100644 tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 2285bba3a7..4c21ae2a42 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1910,6 +1910,41 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "dynamic_dimension_inference",
+    srcs = ["dynamic_dimension_inference.cc"],
+    hdrs = ["dynamic_dimension_inference.h"],
+    deps = [
+        ":hlo",
+        "//tensorflow/compiler/xla:status",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+tf_cc_test(
+    name = "dynamic_dimension_inference_test",
+    srcs = ["dynamic_dimension_inference_test.cc"],
+    deps = [
+        ":dynamic_dimension_inference",
+        "//tensorflow/compiler/xla:debug_options_flags",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_matchers",
+        "//tensorflow/compiler/xla/service:hlo_runner",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+    ],
+)
+
 tf_cc_test(
     name = "reshape_mover_test",
     srcs = ["reshape_mover_test.cc"],
diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
new file mode 100644
index 0000000000..6d0472689b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
@@ -0,0 +1,459 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
+#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+
+namespace xla {
+
+namespace {
+bool IsTrivialWindowDimension(const WindowDimension& window_dimension) {
+  return window_dimension.size() == 1 && window_dimension.stride() == 1 &&
+         window_dimension.padding_low() == 0 &&
+         window_dimension.padding_high() == 0 &&
+         window_dimension.window_dilation() == 1 &&
+         window_dimension.base_dilation() == 1;
+}
+}  // namespace
+
+class DynamicDimensionInferenceVisitor : public DfsHloVisitorWithDefault {
+ public:
+  explicit DynamicDimensionInferenceVisitor(
+      const DynamicParameterBinding& param_bindings,
+      DynamicDimensionInference* parent)
+      : param_bindings_(param_bindings), parent_(parent) {}
+
+  Status DefaultAction(HloInstruction* hlo) override;
+
+  static Status Run(HloComputation* computation,
+                    const DynamicParameterBinding& param_bindings,
+                    DynamicDimensionInference* parent) {
+    DynamicDimensionInferenceVisitor visitor(param_bindings, parent);
+    return computation->Accept(&visitor);
+  }
+
+  Status HandleParameter(HloInstruction* hlo) override;
+
+  Status HandleReduce(HloInstruction* hlo) override;
+
+  Status HandleDot(HloInstruction* hlo) override;
+
+  Status HandleTranspose(HloInstruction* hlo) override;
+
+  Status HandleReshape(HloInstruction* hlo) override;
+
+  Status HandlePad(HloInstruction* hlo) override;
+
+  Status HandleBroadcast(HloInstruction* hlo) override;
+
+  Status HandleGetDimensionSize(HloInstruction* hlo) override;
+
+  Status HandleSelect(HloInstruction* hlo) override;
+
+  Status HandleConvolution(HloInstruction* hlo) override;
+
+  Status HandleReduceWindow(HloInstruction* hlo) override;
+
+  Status HandleSelectAndScatter(HloInstruction* hlo) override;
+
+  Status HandleGetTupleElement(HloInstruction* hlo) override;
+
+  Status HandleElementwiseUnary(HloInstruction* hlo) override;
+
+  Status HandleElementwiseBinary(HloInstruction* hlo) override;
+
+ private:
+  using OperandDynamicDimensionFn = std::function<Status(
+      HloInstruction* operand, ShapeIndex index, int64 dimension,
+      int64 operand_index, HloInstruction* dynamic_size)>;
+
+  Status ForEachOperandDynamicDimension(HloInstruction* inst,
+                                        const OperandDynamicDimensionFn&);
+
+  // Pass through a dynamic dimension from the input to the output with the same
+  // value and index in the shape. This is a helper function to handle trivial
+  // instructions like elementwise operations.
+  Status PassThroughDynamicDimension(HloInstruction*);
+
+  // The dynamic parameter bindings of this computation.
+  const DynamicParameterBinding& param_bindings_;
+
+  // A pointer to DynamicDimensionInference, used to update the dynamic mapping.
+  DynamicDimensionInference* parent_;
+};
+
+Status DynamicDimensionInferenceVisitor::DefaultAction(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        return UnimplementedStrCat(
+            "Asked to propagate a dynamic dimension from hlo ",
+            operand->ToString(), "@", index.ToString(), "@", dimension,
+            " to hlo ", hlo->ToString(), ", which is not implemented.");
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleGetTupleElement(
+    HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        if (hlo->tuple_index() == index[0]) {
+          ShapeIndex new_index =
+              ShapeIndexView(index).ConsumeFront().ToShapeIndex();
+          parent_->SetDynamicSize(hlo, new_index, dimension, dynamic_size);
+        }
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleBroadcast(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        int64 broadcast_dim = hlo->dimensions(dimension);
+        parent_->SetDynamicSize(hlo, index, broadcast_dim, dynamic_size);
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandlePad(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        if (operand_index != 0) {
+          return Unimplemented(
+              "Dynamic dimension on padding value is not supported");
+        }
+        const PaddingConfig_PaddingConfigDimension& padding_config =
+            hlo->padding_config().dimensions(dimension);
+        if (padding_config.interior_padding() == 0 &&
+            padding_config.edge_padding_low() == 0 &&
+            padding_config.edge_padding_high() == 0) {
+          parent_->SetDynamicSize(hlo, {}, dimension, dynamic_size);
+          return Status::OK();
+        } else {
+          return Unimplemented(
+              "Dynamic dimension propagation on padding dimension is not "
+              "supported.");
+        }
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleReduce(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        HloInstruction* reduce = hlo;
+        int64 operand_count = reduce->operand_count();
+        CHECK_EQ(operand_count % 2, 0);
+        if (operand_index >= operand_count / 2) {
+          // Init values doesn't have dynamic size.
+          return Status::OK();
+        }
+        if ((absl::c_count(reduce->dimensions(), dimension) != 0)) {
+          // Dimension is to be reduce, stop tracing.
+          return Status::OK();
+        }
+
+        // Find out the new dynamic dimension after reduce.
+        int64 dimensions_not_reduced_count = 0;
+        for (int i = 0; i < ShapeUtil::Rank(operand->shape()); ++i) {
+          if (dimension == i) {
+            parent_->SetDynamicSize(reduce, {}, dimensions_not_reduced_count,
+                                    dynamic_size);
+
+            return Status::OK();
+          }
+          if (absl::c_count(reduce->dimensions(), i) == 0) {
+            dimensions_not_reduced_count++;
+          }
+        }
+
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleDot(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        HloInstruction* dot = hlo;
+        const DotDimensionNumbers& dimension_numbers =
+            dot->dot_dimension_numbers();
+        // A map from the operand dimensions to result dimension.
+        absl::flat_hash_map<int64, int64> result_dim_mapping;
+        int64 current_result_dims = 0;
+        std::unordered_set<int64> batch_dims(
+            dimension_numbers.rhs_batch_dimensions().begin(),
+            dimension_numbers.rhs_batch_dimensions().end());
+
+        for (int64 i : dimension_numbers.rhs_batch_dimensions()) {
+          result_dim_mapping[i] = current_result_dims++;
+        }
+
+        for (int64 i = 0; i < ShapeUtil::Rank(dot->operand(0)->shape()); i++) {
+          if (!absl::c_linear_search(
+                  dimension_numbers.lhs_contracting_dimensions(), i)) {
+            if (operand_index == 0) {
+              result_dim_mapping[i] = current_result_dims;
+            }
+            current_result_dims++;
+          }
+        }
+
+        for (int64 i = 0; i < ShapeUtil::Rank(dot->operand(1)->shape()); i++) {
+          if (!absl::c_linear_search(
+                  dimension_numbers.rhs_contracting_dimensions(), i) &&
+              !absl::c_linear_search(dimension_numbers.rhs_batch_dimensions(),
+                                     i)) {
+            if (operand_index == 1) {
+              result_dim_mapping[i] = current_result_dims;
+            }
+            current_result_dims++;
+          }
+        }
+
+        // Check if the operand dim is in the result shape. If so, add another
+        // work item to trace that dimension.
+        auto iter = result_dim_mapping.find(dimension);
+        if (iter != result_dim_mapping.end()) {
+          parent_->SetDynamicSize(dot, {}, iter->second, dynamic_size);
+        }
+
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleTranspose(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        parent_->SetDynamicSize(hlo, {}, hlo->dimensions()[dimension],
+                                dynamic_size);
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleConvolution(
+    HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        HloInstruction* conv = hlo;
+        const ConvolutionDimensionNumbers& dimension_numbers =
+            conv->convolution_dimension_numbers();
+
+        if (operand_index == 0) {
+          if (dimension == dimension_numbers.input_batch_dimension()) {
+            parent_->SetDynamicSize(conv, {},
+                                    dimension_numbers.output_batch_dimension(),
+                                    dynamic_size);
+            return Status::OK();
+          }
+
+          if (dimension == dimension_numbers.input_feature_dimension()) {
+            return Status::OK();
+          }
+        } else {
+          if (dimension == dimension_numbers.kernel_input_feature_dimension()) {
+            return Status::OK();
+          }
+        }
+
+        return Unimplemented("Dynamic Spatial Convolution is not supported: %s",
+                             conv->ToString());
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleGetDimensionSize(
+    HloInstruction*) {
+  // Dynamic dimension doesn't propagate through GetDimensionSize:
+  //
+  //   Input: F32[x, y, z]
+  //     |
+  //   GetDimensionSize(1): U32[]
+  //
+  // The returned value is a scalar, which doesn't have any dynamic dimension in
+  // the shape (although the value contains the real size of the dynamic
+  // dimension of the input).
+  return Status::OK();
+}
+
+Status DynamicDimensionInferenceVisitor::PassThroughDynamicDimension(
+    HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        parent_->SetDynamicSize(hlo, index, dimension, dynamic_size);
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleElementwiseUnary(
+    HloInstruction* hlo) {
+  return PassThroughDynamicDimension(hlo);
+}
+
+Status DynamicDimensionInferenceVisitor::HandleSelect(HloInstruction* hlo) {
+  return PassThroughDynamicDimension(hlo);
+}
+
+Status DynamicDimensionInferenceVisitor::HandleElementwiseBinary(
+    HloInstruction* hlo) {
+  return PassThroughDynamicDimension(hlo);
+}
+
+Status DynamicDimensionInferenceVisitor::HandleReshape(HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        HloInstruction* reshape = hlo;
+        std::vector<std::pair<int64, int64>> unmodified_dims =
+            ShapeUtil::DimensionsUnmodifiedByReshape(operand->shape(),
+                                                     reshape->shape());
+        for (auto& unmodified : unmodified_dims) {
+          if (unmodified.first == dimension) {
+            parent_->SetDynamicSize(reshape, {}, unmodified.second,
+                                    dynamic_size);
+            return Status::OK();
+          }
+        }
+        return Unimplemented(
+            "Dynamic Reshape on modified dimensions is yet not supported: %s",
+            reshape->ToString());
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleReduceWindow(
+    HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        HloInstruction* reduce_window = hlo;
+        const WindowDimension& window_dimension =
+            reduce_window->window().dimensions(dimension);
+
+        if (!IsTrivialWindowDimension(window_dimension)) {
+          return Unimplemented(
+              "Dynamic Spatial reduce window is not supported: %s",
+              reduce_window->ToString());
+        }
+
+        parent_->SetDynamicSize(reduce_window, {}, dimension, dynamic_size);
+
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleSelectAndScatter(
+    HloInstruction* hlo) {
+  return ForEachOperandDynamicDimension(
+      hlo, [&](HloInstruction* operand, ShapeIndex index, int64 dimension,
+               int64 operand_index, HloInstruction* dynamic_size) {
+        HloInstruction* select_and_scatter = hlo;
+        const WindowDimension& window_dimension =
+            select_and_scatter->window().dimensions(dimension);
+
+        if (!IsTrivialWindowDimension(window_dimension)) {
+          return Unimplemented(
+              "Dynamic Spatial select and scatter is not supported: %s",
+              select_and_scatter->ToString());
+        }
+
+        parent_->SetDynamicSize(select_and_scatter, {}, dimension,
+                                dynamic_size);
+
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::HandleParameter(HloInstruction* hlo) {
+  return param_bindings_.ForEachBinding(
+      [&](const DynamicParameterBinding::DynamicParameter& dynamic_parameter,
+          const DynamicParameterBinding::DynamicDimension& dynamic_dimension) {
+        if (dynamic_dimension.parameter_num != hlo->parameter_number()) {
+          return Status::OK();
+        }
+        HloComputation* computation = hlo->parent();
+        HloInstruction* target_parameter =
+            computation->parameter_instruction(dynamic_dimension.parameter_num);
+
+        HloInstruction* dynamic_size =
+            computation->parameter_instruction(dynamic_parameter.parameter_num);
+        for (int64 i : dynamic_parameter.parameter_index) {
+          dynamic_size =
+              computation->AddInstruction(HloInstruction::CreateGetTupleElement(
+                  ShapeUtil::GetSubshape(dynamic_size->shape(), {i}),
+                  dynamic_size, i));
+        }
+
+        parent_->SetDynamicSize(target_parameter,
+                                dynamic_dimension.parameter_index,
+                                dynamic_dimension.dimension, dynamic_size);
+        return Status::OK();
+      });
+}
+
+Status DynamicDimensionInferenceVisitor::ForEachOperandDynamicDimension(
+    HloInstruction* inst, const OperandDynamicDimensionFn& fn) {
+  for (int64 operand_index = 0; operand_index < inst->operand_count();
+       ++operand_index) {
+    auto iter =
+        parent_->per_hlo_dynamic_dimensions_.find(inst->operand(operand_index));
+    if (iter != parent_->per_hlo_dynamic_dimensions_.end()) {
+      for (auto& dynamic_dimension : iter->second) {
+        HloInstruction* dynamic_size = parent_->GetDynamicSize(
+            dynamic_dimension.inst, dynamic_dimension.index,
+            dynamic_dimension.dim);
+        TF_RETURN_IF_ERROR(fn(dynamic_dimension.inst, dynamic_dimension.index,
+                              dynamic_dimension.dim, operand_index,
+                              dynamic_size));
+      }
+    }
+  }
+  return Status::OK();
+}
+
+/* static */
+StatusOr<DynamicDimensionInference> DynamicDimensionInference::Run(
+    HloModule* module) {
+  VLOG(0) << "Param Config " << module->dynamic_parameter_binding().ToString();
+  DynamicDimensionInference inference(module);
+  TF_RETURN_IF_ERROR(inference.AnalyzeDynamicDimensions());
+  return inference;
+}
+
+DynamicDimensionInference::DynamicDimensionInference(HloModule* module)
+    : module_(module) {}
+
+Status DynamicDimensionInference::AnalyzeDynamicDimensions() {
+  return DynamicDimensionInferenceVisitor::Run(
+      module_->entry_computation(), module_->dynamic_parameter_binding(), this);
+}
+
+HloInstruction* DynamicDimensionInference::GetDynamicSize(
+    HloInstruction* inst, const ShapeIndex& index, int64 dim) const {
+  auto iter = dynamic_mapping_.find(DynamicDimension{inst, index, dim});
+  if (iter != dynamic_mapping_.end()) {
+    return iter->second;
+  }
+  return nullptr;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.h b/tensorflow/compiler/xla/service/dynamic_dimension_inference.h
new file mode 100644
index 0000000000..164d15bf11
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.h
@@ -0,0 +1,112 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_DIMENSION_INFERENCE_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_DIMENSION_INFERENCE_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/types/span.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/platform/macros.h"
+
+namespace xla {
+
+// DynamicDimensionInference analyzes each HLO instruction in a graph and
+// inferences which dimensions are dynamic and which scalar instructions
+// represent the runtime real size of those dynamic dimensions.
+class DynamicDimensionInference {
+ public:
+  static StatusOr<DynamicDimensionInference> Run(HloModule* module);
+
+  string ToString() const;
+
+  // If the dimension `dim` of instruction `inst` at `index` has a dynamic size,
+  // returns a scalar HloInstruction that represents the runtime size of that
+  // dimension. Otherwise returns nullptr.
+  HloInstruction* GetDynamicSize(HloInstruction* inst, const ShapeIndex& index,
+                                 int64 dim) const;
+
+  friend class DynamicDimensionInferenceVisitor;
+
+ private:
+  explicit DynamicDimensionInference(HloModule* module);
+
+  // DynamicDimension is used as a key in the dynamic key-value mapping. It
+  // unambiguously represents a dynamic dimension of a instruction at a given
+  // index.
+  struct DynamicDimension {
+    // HloInstruction that holds the dimension.
+    HloInstruction* inst;
+    // Subshape of the instruction that holds the dimension.
+    ShapeIndex index;
+    // The dimension number of the dynamic dimension at given index of a given
+    // instruction.
+    int64 dim;
+
+    // Artifacts needed to make this struct able to be used as a `key` in absl
+    // maps. "friend" keywords are added so these functions can be found through
+    // ADL.
+    template <typename H>
+    friend H AbslHashValue(H h, const DynamicDimension& m) {
+      return H::combine(std::move(h), m.inst, m.index, m.dim);
+    }
+
+    friend bool operator==(const DynamicDimension& lhs,
+                           const DynamicDimension& rhs) {
+      return lhs.inst == rhs.inst && lhs.index == rhs.index &&
+             lhs.dim == rhs.dim;
+    }
+  };
+
+  // Update the dynamic mapping so that we know dimension `dim` of instruction
+  // `inst` at `index` has a dynamic size, and its runtime size is represented
+  // by a scalar instruction `size`.
+  void SetDynamicSize(HloInstruction* inst, const ShapeIndex& index, int64 dim,
+                      HloInstruction* size) {
+    dynamic_mapping_.try_emplace(DynamicDimension{inst, index, dim}, size);
+    auto iter = per_hlo_dynamic_dimensions_.try_emplace(inst);
+    iter.first->second.emplace(DynamicDimension{inst, index, dim});
+  }
+
+  // AnalyzeDynamicDimensions starts the analysis of the dynamic dimensions in
+  // module_.
+  Status AnalyzeDynamicDimensions();
+
+  // HloModule being analyzed.
+  HloModule* module_;
+
+  // dynamic_mapping_ holds the result of the analysis. It maps a dynamic
+  // dimension to a scalar HloInstruction that represents the real dynamic size
+  // of the dynamic dimension.
+  using DynamicMapping = absl::flat_hash_map<DynamicDimension, HloInstruction*>;
+  DynamicMapping dynamic_mapping_;
+
+  using PerHloDynamicDimensions =
+      absl::flat_hash_map<HloInstruction*,
+                          absl::flat_hash_set<DynamicDimension>>;
+  PerHloDynamicDimensions per_hlo_dynamic_dimensions_;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_DIMENSION_INFERENCE_H_
diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
new file mode 100644
index 0000000000..ea9ebed45d
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
@@ -0,0 +1,535 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
+
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_runner.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace op = xla::testing::opcode_matchers;
+
+namespace xla {
+namespace {
+
+class DynamicDimensionInferenceTest : public HloTestBase {
+ protected:
+  DynamicDimensionInferenceTest() : HloTestBase() {
+    module_ = CreateNewVerifiedModule();
+  }
+
+  Status RunInference() {
+    hlo_graph_dumper::MaybeDumpHloModule(*module_, "Before alias analysis");
+    TF_ASSIGN_OR_RETURN(DynamicDimensionInference inference,
+                        DynamicDimensionInference::Run(module_.get()));
+
+    inference_ = absl::make_unique<DynamicDimensionInference>(inference);
+    return Status::OK();
+  }
+
+  HloComputation* GetAdd() {
+    auto embedded_builder = HloComputation::Builder("add");
+    auto lhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        0, ShapeUtil::MakeShape(F32, {}), "lhs"));
+    auto rhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        1, ShapeUtil::MakeShape(F32, {}), "rhs"));
+    embedded_builder.AddInstruction(
+        HloInstruction::CreateBinary(lhs->shape(), HloOpcode::kAdd, lhs, rhs));
+    return module_->AddEmbeddedComputation(embedded_builder.Build());
+  }
+
+  std::unique_ptr<HloModule> module_;
+  std::unique_ptr<DynamicDimensionInference> inference_;
+  const Shape scalar_shape_ = ShapeUtil::MakeShape(S32, {});
+};
+
+TEST_F(DynamicDimensionInferenceTest, ParamTest) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+
+  auto param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, input_shape, "param"));
+  auto param2 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape_, "param"));
+
+  module_->AddEntryComputation(builder.Build());
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(param, {}, 1), param2);
+  EXPECT_EQ(inference_->GetDynamicSize(param, {}, 0), nullptr);
+  EXPECT_EQ(inference_->GetDynamicSize(param2, {}, 0), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ParamTestTuple) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+
+  auto param = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeTupleShape({input_shape, scalar_shape_}), "param"));
+
+  module_->AddEntryComputation(builder.Build());
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{0, {1}},
+      DynamicParameterBinding::DynamicDimension{0, {0}, 1}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_THAT(inference_->GetDynamicSize(param, {0}, 1),
+              op::GetTupleElement(param, 1));
+
+  EXPECT_EQ(inference_->GetDynamicSize(param, {0}, 0), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, GetTupleElement) {
+  // When data flows through GTE, the dynamic dimension size keeps the
+  // same, and the index has its front popped.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+
+  auto param = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeTupleShape({input_shape, scalar_shape_}), "param"));
+
+  auto gte = builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(input_shape, param, 0));
+
+  module_->AddEntryComputation(builder.Build());
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{0, {1}},
+      DynamicParameterBinding::DynamicDimension{0, {0}, 1}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_THAT(inference_->GetDynamicSize(param, {0}, 1),
+              op::GetTupleElement(param, 1));
+
+  EXPECT_THAT(inference_->GetDynamicSize(gte, {}, 1),
+              op::GetTupleElement(param, 1));
+
+  EXPECT_EQ(inference_->GetDynamicSize(param, {0}, 0), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ElementwiseTest) {
+  // When data flows through elementwise, the dynamic dimension size keeps the
+  // same.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+
+  auto data_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, input_shape, "data_param"));
+  auto size_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape_, "size_param"));
+
+  auto* negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(input_shape, HloOpcode::kNegate, data_param));
+
+  module_->AddEntryComputation(builder.Build());
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(negate, {}, 1), size_param);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ReduceTestI) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+  auto reduce_shape = ShapeUtil::MakeShape(F32, {2});
+
+  auto data_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, input_shape, "data_param"));
+  auto size_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape_, "size_param"));
+
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(input_shape, HloOpcode::kNegate, data_param));
+
+  auto init = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
+
+  auto reduce = builder.AddInstruction(HloInstruction::CreateReduce(
+      reduce_shape, negate, init, {0, 2}, GetAdd()));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(reduce, {}, 0), size_param);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ReduceTestII) {
+  // Same as ReduceTestI, but only reduce one dimension.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+  auto reduce_shape = ShapeUtil::MakeShape(F32, {1, 2});
+
+  auto data_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, input_shape, "data_param"));
+  auto size_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape_, "size_param"));
+
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(input_shape, HloOpcode::kNegate, data_param));
+
+  auto init = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
+
+  auto reduce = builder.AddInstruction(
+      HloInstruction::CreateReduce(reduce_shape, negate, init, {1}, GetAdd()));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 2}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(reduce, {}, 1), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(reduce, {}, 0), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, DotTest) {
+  auto builder = HloComputation::Builder(TestName());
+  constexpr int xdim = 3;
+  constexpr int ydim = 2;
+  constexpr int zdim = 1;
+  auto xy_shape = ShapeUtil::MakeShape(F32, {xdim, ydim});
+  auto yz_shape = ShapeUtil::MakeShape(F32, {ydim, zdim});
+  auto xz_shape = ShapeUtil::MakeShape(F32, {xdim, zdim});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, xy_shape, "A"));
+  auto* b_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, yz_shape, "B"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, scalar_shape_, "size_param"));
+
+  DotDimensionNumbers dot_dnums;
+  dot_dnums.add_lhs_contracting_dimensions(1);
+  dot_dnums.add_rhs_contracting_dimensions(0);
+  auto dot = builder.AddInstruction(
+      HloInstruction::CreateDot(xz_shape, a_param, b_param, dot_dnums,
+                                HloTestBase::DefaultPrecisionConfig(2)));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding for non-contracting dimension.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  // Set up binding for contracting dimensions.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{1, {}, 0}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(dot, {}, 0), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(dot, {}, 1), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ConvolutionTest) {
+  auto builder = HloComputation::Builder(TestName());
+  constexpr int xdim = 3;
+  constexpr int ydim = 2;
+  constexpr int zdim = 1;
+  auto xy_shape = ShapeUtil::MakeShape(F32, {xdim, ydim});
+  auto yz_shape = ShapeUtil::MakeShape(F32, {ydim, zdim});
+  auto zx_shape = ShapeUtil::MakeShape(F32, {zdim, xdim});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, xy_shape, "A"));
+  auto* b_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, yz_shape, "B"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, scalar_shape_, "size_param"));
+
+  auto dnums = XlaBuilder::CreateDefaultConvDimensionNumbers(0);
+
+  dnums.set_kernel_input_feature_dimension(0);
+  dnums.set_kernel_output_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(1);
+  dnums.set_output_feature_dimension(0);
+
+  Window window;
+
+  auto* conv = builder.AddInstruction(HloInstruction::CreateConvolve(
+      zx_shape, a_param, b_param, /*feature_group_count=*/1, window, dnums,
+      HloTestBase::DefaultPrecisionConfig(2)));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding for non-contracting dimension.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  // Set up binding for contracting dimensions.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(conv, {}, 1), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(conv, {}, 0), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, TransposeTest) {
+  // Test the ability to trace unmodified dimensions
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 3});
+  auto output_shape = ShapeUtil::MakeShape(F32, {3, 2, 1});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, input_shape, "A"));
+  auto* size_param_1 = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+  auto* size_param_2 = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, scalar_shape_, "size_param"));
+  auto* size_param_3 = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/3, scalar_shape_, "size_param"));
+
+  auto* transpose = builder.AddInstruction(
+      HloInstruction::CreateTranspose(output_shape, a_param, {2, 1, 0}));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{3, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 2}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(transpose, {}, 0), size_param_3);
+  EXPECT_EQ(inference_->GetDynamicSize(transpose, {}, 1), size_param_2);
+  EXPECT_EQ(inference_->GetDynamicSize(transpose, {}, 2), size_param_1);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ReshapeTest) {
+  // Test the ability to trace unmodified reshape dimensions.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {2, 3, 4, 5, 6});
+  auto output_shape = ShapeUtil::MakeShape(F32, {6, 4, 1, 5, 2, 3});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, input_shape, "A"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+
+  auto* reshape = builder.AddInstruction(
+      HloInstruction::CreateReshape(output_shape, a_param));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 2}));
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 3}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(reshape, {}, 0), nullptr);
+  EXPECT_EQ(inference_->GetDynamicSize(reshape, {}, 1), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(reshape, {}, 2), nullptr);
+  EXPECT_EQ(inference_->GetDynamicSize(reshape, {}, 3), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(reshape, {}, 4), nullptr);
+  EXPECT_EQ(inference_->GetDynamicSize(reshape, {}, 5), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ReshapeTestUnimplemented) {
+  // Test the ability to trace unmodified reshape dimensions.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {2, 3, 4, 5, 6});
+  auto output_shape = ShapeUtil::MakeShape(F32, {6, 4, 1, 5, 2, 3});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, input_shape, "A"));
+
+  builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+
+  builder.AddInstruction(HloInstruction::CreateReshape(output_shape, a_param));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  Status status = RunInference();
+  EXPECT_EQ(status.code(), tensorflow::error::UNIMPLEMENTED);
+}
+
+TEST_F(DynamicDimensionInferenceTest, BroadcastTest) {
+  // Test the ability to trace broadcast dimension.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {2});
+  auto output_shape = ShapeUtil::MakeShape(F32, {3, 2, 4});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, input_shape, "A"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+
+  auto* broadcast = builder.AddInstruction(
+      HloInstruction::CreateBroadcast(output_shape, a_param, {1}));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(broadcast, {}, 0), nullptr);
+  EXPECT_EQ(inference_->GetDynamicSize(broadcast, {}, 1), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(broadcast, {}, 2), nullptr);
+}
+
+TEST_F(DynamicDimensionInferenceTest, ReduceWindowBatchTest) {
+  // Test the ability to trace reduce window batch dimensions.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {2, 4, 4});
+  auto output_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
+
+  Window window;
+  // First dimension is unchanged.
+  WindowDimension* batch_dim = window.add_dimensions();
+  batch_dim->set_size(1);
+  batch_dim->set_stride(1);
+  batch_dim->set_padding_low(0);
+  batch_dim->set_padding_high(0);
+  batch_dim->set_window_dilation(1);
+  batch_dim->set_base_dilation(1);
+
+  // Second and third dimension are reduced.
+  for (int64 i = 0; i < 2; ++i) {
+    WindowDimension* dim = window.add_dimensions();
+    dim->set_size(2);
+    dim->set_stride(2);
+    dim->set_padding_low(0);
+    dim->set_padding_high(0);
+    dim->set_window_dilation(1);
+    dim->set_base_dilation(1);
+  }
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, input_shape, "A"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+
+  auto init = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
+
+  auto* reduce_window =
+      builder.AddInstruction(HloInstruction::CreateReduceWindow(
+          output_shape, a_param, init, window, GetAdd()));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(reduce_window, {}, 0), size_param);
+}
+
+TEST_F(DynamicDimensionInferenceTest, SelectAndScatterTest) {
+  // Test the ability to trace select and scatter batch dimensions.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {2, 4, 4});
+  auto output_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
+
+  Window window;
+  // First dimension is unchanged.
+  WindowDimension* batch_dim = window.add_dimensions();
+  batch_dim->set_size(1);
+  batch_dim->set_stride(1);
+  batch_dim->set_padding_low(0);
+  batch_dim->set_padding_high(0);
+  batch_dim->set_window_dilation(1);
+  batch_dim->set_base_dilation(1);
+
+  // Second and third dimension are reduced.
+  for (int64 i = 0; i < 2; ++i) {
+    WindowDimension* dim = window.add_dimensions();
+    dim->set_size(2);
+    dim->set_stride(2);
+    dim->set_padding_low(0);
+    dim->set_padding_high(0);
+    dim->set_window_dilation(1);
+    dim->set_base_dilation(1);
+  }
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, input_shape, "A"));
+  auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, scalar_shape_, "size_param"));
+
+  auto init = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
+
+  auto* reduce_window =
+      builder.AddInstruction(HloInstruction::CreateReduceWindow(
+          output_shape, a_param, init, window, GetAdd()));
+
+  module_->AddEntryComputation(builder.Build());
+
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  TF_ASSERT_OK(RunInference());
+  EXPECT_EQ(inference_->GetDynamicSize(reduce_window, {}, 0), size_param);
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From c215c3f76fdc2ee12e9fe05b82e245054a4ae5b9 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 5 Dec 2018 14:12:13 -0800
Subject: [PATCH 1381/1554] Tests that all symbols the converter outputs
 actually exist in tf v1.

Also some fixes to make this a reality. This will help ensure code can be
v1-and-v2-agnostic.

PiperOrigin-RevId: 224219069
---
 tensorflow/python/saved_model/save.py         |  3 +-
 .../golden/v1/tensorflow.saved_model.pbtxt    |  4 +++
 .../tools/compatibility/tf_upgrade_v2_test.py | 29 +++++++++++++++++++
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 53c1d2cc87..422d68962a 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -551,7 +551,8 @@ def _write_object_proto(obj, proto, filename_map):
     proto.user_object.SetInParent()
 
 
-@tf_export("saved_model.save", v1=["saved_model.experimental.save"])
+@tf_export("saved_model.save",
+           v1=["saved_model.save", "saved_model.experimental.save"])
 def save(obj, export_dir, signatures=None):
   # pylint: disable=line-too-long
   """Exports the Checkpointable object `obj` to [SavedModel format](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md).
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt
index 3929003fa1..2a7c789105 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.saved_model.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "regression_signature_def"
     argspec: "args=[\'examples\', \'predictions\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "save"
+    argspec: "args=[\'obj\', \'export_dir\', \'signatures\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "simple_save"
     argspec: "args=[\'session\', \'export_dir\', \'inputs\', \'outputs\', \'legacy_init_op\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 366ce0759d..7c26718edb 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -162,6 +162,35 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     visitor.private_map["tf.compat"] = ["v1", "v2"]
     traverse.traverse(tf.compat.v1, visitor)
 
+  def testAllAPIV1(self):
+    collect = True
+    v1_symbols = set([])
+
+    # Converts all symbols in the v1 namespace to the v2 namespace, raising
+    # an error if the target of the conversion is not in the v1 namespace.
+    def conversion_visitor(unused_path, unused_parent, children):
+      for child in children:
+        _, attr = tf_decorator.unwrap(child[1])
+        api_names = get_v1_names(attr)
+        for name in api_names:
+          if collect:
+            v1_symbols.add("tf." + name)
+          else:
+            _, _, _, text = self._upgrade("tf." + name)
+            if (text and
+                not text.startswith("tf.compat.v1") and
+                text not in v1_symbols):
+              self.assertFalse(
+                  True, "Symbol %s generated from %s not in v1 API" % (
+                      text, name))
+
+    visitor = public_api.PublicAPIVisitor(conversion_visitor)
+    visitor.do_not_descend_map["tf"].append("contrib")
+    visitor.private_map["tf.compat"] = ["v1", "v2"]
+    traverse.traverse(tf.compat.v1, visitor)
+    collect = False
+    traverse.traverse(tf.compat.v1, visitor)
+
   def testKeywordArgNames(self):
     if not hasattr(tf.compat, "v2"):
       return
-- 
GitLab


From 1680520c643b4c3ea6c41ce2ee3cc000a125a88e Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 5 Dec 2018 14:28:04 -0800
Subject: [PATCH 1382/1554] Fix clang-format

---
 .../contrib/tensorrt/convert/convert_nodes.cc | 20 +++----
 .../tensorrt/convert/convert_nodes_test.cc    | 53 +++++++++----------
 2 files changed, 34 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 3392e527c2..9f6ec75c5b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1894,11 +1894,11 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
     return tensorflow::errors::InvalidArgument(
         "Two inputs expected for ExpandDims, at ", node_def.name());
   }
-  if (inputs.at(0).is_weights() ) {
+  if (inputs.at(0).is_weights()) {
     return tensorflow::errors::Unimplemented(
         "ExpandDims expects tensor for input, at ", node_def.name());
   }
-  if (!inputs.at(1).is_weights() ) {
+  if (!inputs.at(1).is_weights()) {
     return tensorflow::errors::InvalidArgument(
         "ExpandDims expects weights for axis, at ", node_def.name());
   }
@@ -1937,11 +1937,11 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   if (params->validation_only) return Status::OK();
 
   // ExpandDims: Insert new dim of size 1.
-  input_dims.insert(input_dims.begin()+axis, 1);
+  input_dims.insert(input_dims.begin() + axis, 1);
   // Reshape tensor.
   const bool ignore_first_dim = input_tensor.is_tensor();
-  nvinfer1::Dims new_dims = TensorShapeArrayToTrtDims(input_dims,
-                                                      ignore_first_dim);
+  nvinfer1::Dims new_dims =
+      TensorShapeArrayToTrtDims(input_dims, ignore_first_dim);
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
@@ -1957,7 +1957,7 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
     return tensorflow::errors::InvalidArgument(
         "One input expected for Squeeze, at ", node_def.name());
   }
-  if (inputs.at(0).is_weights() ) {
+  if (inputs.at(0).is_weights()) {
     return tensorflow::errors::Unimplemented(
         "Squeeze expects tensor for input, at ", node_def.name());
   }
@@ -2002,14 +2002,14 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
     input_dims[axis] = 0;
   }
   if (params->validation_only) return Status::OK();
-  
+
   // Remove all dims which are equal to 0.
   input_dims.erase(std::remove(input_dims.begin(), input_dims.end(), 0),
                    input_dims.end());
-    // Reshape tensor.
+  // Reshape tensor.
   const bool ignore_first_dim = input_tensor.is_tensor();
-  nvinfer1::Dims new_dims = TensorShapeArrayToTrtDims(input_dims,
-                                                      ignore_first_dim);
+  nvinfer1::Dims new_dims =
+      TensorShapeArrayToTrtDims(input_dims, ignore_first_dim);
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 27d6098b49..c37a43dd5d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2126,8 +2126,8 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
   Scope s = Scope::NewRootScope();
   auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
   auto weights = ops::Placeholder(s.WithOpName("weights"), DT_INT32);
-  auto expanddims = ops::ExpandDims(s.WithOpName("my_expanddims"), input, 
-                                    weights);
+  auto expanddims =
+      ops::ExpandDims(s.WithOpName("my_expanddims"), input, weights);
   const NodeDef& node_def = expanddims.operation.node()->def();
 
   {
@@ -2153,7 +2153,8 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     Reset();
     AddTestTensor("input", {1, 2, 3});
     AddTestWeights<int32>("weights", {1}, {0});
-    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
         "Modifying batch dimension is not supported for ExpandDims, at "
         "my_expanddims");
   }
@@ -2163,7 +2164,8 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     AddTestTensor("input", {1, 2, 3});
     // Input is rank 4 (batch dim included)
     AddTestWeights<int32>("weights", {1}, {-5});
-    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
         "Modifying batch dimension is not supported for ExpandDims, at "
         "my_expanddims");
   }
@@ -2173,7 +2175,8 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     AddTestTensor("input", {1, 2, 3});
     // Input is rank 4 (batch dim included)
     AddTestWeights<int32>("weights", {1}, {5});
-    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
         "Axis for ExpandDims is invalid, must be in the range "
         "[-rank(input) - 1, rank(input)], at my_expanddims");
   }
@@ -2183,14 +2186,14 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
     AddTestTensor("input", {1, 2, 3});
     // Input is rank 4 (batch dim included)
     AddTestWeights<int32>("weights", {1}, {-6});
-    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
         "Axis for ExpandDims is invalid, must be in the range "
         "[-rank(input) - 1, rank(input)], at my_expanddims");
   }
 
   struct TestParams {
-    TestParams(const std::vector<int>& input_dims,
-               int axis,
+    TestParams(const std::vector<int>& input_dims, int axis,
                const std::vector<int>& expected_output_dims)
         : input_dims(input_dims),
           axis(axis),
@@ -2203,14 +2206,10 @@ TEST_F(OpConverterTest, ConvertExpandDims) {
   // Ok.
   const int kExpandDimsOKCases = 8;
   TestParams ok_params[kExpandDimsOKCases] = {
-      TestParams{{2, 3}, 1, {1, 2, 3}},
-      TestParams{{2, 3}, -3, {1, 2, 3}},
-      TestParams{{2, 3}, 3, {2, 3, 1}},
-      TestParams{{2, 3}, -1, {2, 3, 1}},
-      TestParams{{2, 3}, 2, {2, 1, 3}},
-      TestParams{{2, 3}, -2, {2, 1, 3}},
-      TestParams{{6}, 1, {1, 6}},
-      TestParams{{6}, -1, {6, 1}},
+      TestParams{{2, 3}, 1, {1, 2, 3}}, TestParams{{2, 3}, -3, {1, 2, 3}},
+      TestParams{{2, 3}, 3, {2, 3, 1}}, TestParams{{2, 3}, -1, {2, 3, 1}},
+      TestParams{{2, 3}, 2, {2, 1, 3}}, TestParams{{2, 3}, -2, {2, 1, 3}},
+      TestParams{{6}, 1, {1, 6}},       TestParams{{6}, -1, {6, 1}},
   };
   for (int i = 0; i < kExpandDimsOKCases; ++i) {
     Reset();
@@ -2234,9 +2233,8 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
   {
     // Input list is empty, should fail.
     NodeDef node_def = MakeNodeDef("my_squeeze", "Squeeze", {});
-    RunValidationAndConversion(
-        node_def, error::INVALID_ARGUMENT,
-        "One input expected for Squeeze, at my_squeeze");
+    RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
+                               "One input expected for Squeeze, at my_squeeze");
   }
   {
     // No attrs, should fail.
@@ -2257,8 +2255,8 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     ops::Squeeze::Attrs squeeze_attrs;
     squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);
-    auto squeeze = ops::Squeeze(s.WithOpName("my_squeeze"), input,
-                                squeeze_attrs);
+    auto squeeze =
+        ops::Squeeze(s.WithOpName("my_squeeze"), input, squeeze_attrs);
     return squeeze.operation.node()->def();
   };
 
@@ -2276,18 +2274,16 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     Reset();
     NodeDef node_def = get_squeeze_nodedef({0});
     AddTestTensor("input", {1, 2, 3});
-    RunValidationAndConversion(
-        node_def, error::UNIMPLEMENTED,
-        "Cannot squeeze batch dimension, at my_squeeze");
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Cannot squeeze batch dimension, at my_squeeze");
   }
   {
     // Squeeze batch dim via negative axis, should fail.
     Reset();
     NodeDef node_def = get_squeeze_nodedef({-4});
     AddTestTensor("input", {1, 2, 3});
-    RunValidationAndConversion(
-        node_def, error::UNIMPLEMENTED,
-        "Cannot squeeze batch dimension, at my_squeeze");
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Cannot squeeze batch dimension, at my_squeeze");
   }
   {
     // Squeeze >= rank(input), should fail.
@@ -2311,8 +2307,7 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
   }
 
   struct TestParams {
-    TestParams(const std::vector<int>& input_dims,
-               const std::vector<int>& axis,
+    TestParams(const std::vector<int>& input_dims, const std::vector<int>& axis,
                const std::vector<int>& expected_output_dims)
         : input_dims(input_dims),
           axis(axis),
-- 
GitLab


From 9546b5259410f1aad4206b396558238abd1c7501 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 5 Dec 2018 14:31:46 -0800
Subject: [PATCH 1383/1554] Remove is_tensor() since inputs are required to be
 tensors

---
 .../contrib/tensorrt/convert/convert_nodes.cc | 22 +++++++------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 9f6ec75c5b..ac0e2f684c 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1906,10 +1906,8 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
   std::vector<int> input_dims(dims.d, dims.d + dims.nbDims);
-  // Add batch dim back for tensors.
-  if (input_tensor.is_tensor()) {
-    input_dims.insert(input_dims.begin(), -1);
-  }
+  // Add batch dim back.
+  input_dims.insert(input_dims.begin(), -1);
   const int input_rank = input_dims.size();
   // Get axis to expand on.
   TRT_ShapedWeights weights = inputs.at(1).weights();
@@ -1929,7 +1927,7 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   }
   // Convert negative axis to corresponding positive axis.
   if (axis < 0) axis += input_rank + 1;
-  if (input_tensor.is_tensor() && axis == 0) {
+  if (axis == 0) {
     return tensorflow::errors::Unimplemented(
         "Modifying batch dimension is not supported for ExpandDims, at ",
         node_def.name());
@@ -1939,9 +1937,8 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   // ExpandDims: Insert new dim of size 1.
   input_dims.insert(input_dims.begin() + axis, 1);
   // Reshape tensor.
-  const bool ignore_first_dim = input_tensor.is_tensor();
   nvinfer1::Dims new_dims =
-      TensorShapeArrayToTrtDims(input_dims, ignore_first_dim);
+      TensorShapeArrayToTrtDims(input_dims, /*ignore_first_dim=*/true);
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
@@ -1965,10 +1962,8 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   TRT_TensorOrWeights input_tensor = inputs.at(0);
   const nvinfer1::Dims dims = input_tensor.GetTrtDims();
   std::vector<int> input_dims(dims.d, dims.d + dims.nbDims);
-  // Add batch dim back temporarily.
-  if (input_tensor.is_tensor()) {
-    input_dims.insert(input_dims.begin(), -1);
-  }
+  // Add batch dim back.
+  input_dims.insert(input_dims.begin(), -1);
   const int input_rank = input_dims.size();
   // Mark axes to remove by setting them to 0.
   TFAttrs attrs(node_def);
@@ -1988,7 +1983,7 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
     // Convert negative axis to corresponding positive axis.
     if (axis < 0) axis += input_rank;
     // Don't squeeze batch dim.
-    if (input_tensor.is_tensor() && axis == 0) {
+    if (axis == 0) {
       return tensorflow::errors::Unimplemented(
           "Cannot squeeze batch dimension, at ", node_def.name());
     }
@@ -2007,9 +2002,8 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   input_dims.erase(std::remove(input_dims.begin(), input_dims.end(), 0),
                    input_dims.end());
   // Reshape tensor.
-  const bool ignore_first_dim = input_tensor.is_tensor();
   nvinfer1::Dims new_dims =
-      TensorShapeArrayToTrtDims(input_dims, ignore_first_dim);
+      TensorShapeArrayToTrtDims(input_dims, /*ignore_first_dim=*/true);
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
-- 
GitLab


From d0bae80ac28fa832285acbdf53ec51eeb2d01af6 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 5 Dec 2018 14:22:26 -0800
Subject: [PATCH 1384/1554] [tf.data] Changing semantics of the map family of
 datasets to be cardinality preserving (in the presence of OutOfRange errors
 produced by the user-defined function).

PiperOrigin-RevId: 224220781
---
 .../optimizers/data/map_and_batch_fusion.cc   | 13 +++-
 .../optimizers/data/map_and_filter_fusion.cc  |  5 +-
 .../grappler/optimizers/data/map_fusion.cc    | 13 +++-
 .../experimental/map_and_batch_dataset_op.cc  | 67 +++++++++----------
 .../numa_map_and_batch_dataset_op.cc          |  6 ++
 .../experimental/parse_example_dataset_op.cc  |  3 +-
 .../data/experimental/scan_dataset_op.cc      | 31 ++++++---
 .../core/kernels/data/map_dataset_op.cc       | 34 +++++++---
 .../kernels/data/parallel_map_dataset_op.cc   | 19 ++++--
 .../kernels/data/parallel_map_iterator.cc     | 60 ++++++++++++-----
 .../core/kernels/data/parallel_map_iterator.h |  2 +-
 tensorflow/core/ops/dataset_ops.cc            |  2 +
 .../core/ops/experimental_dataset_ops.cc      |  4 ++
 .../kernel_tests/map_and_batch_test.py        | 40 +++++------
 .../python/data/experimental/ops/batching.py  |  1 +
 .../python/data/experimental/ops/scan_ops.py  |  1 +
 tensorflow/python/data/ops/dataset_ops.py     | 30 ++++++---
 17 files changed, 215 insertions(+), 116 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 9a7a751ff7..84c4d82f6a 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -77,15 +77,22 @@ NodeDef MakeMapAndBatchNode(const NodeDef& map_node, const NodeDef& batch_node,
     new_node.add_input(tmp->name());
   }
 
-  // Set `f` and `Targuments` attributes.
+  // Required attributes.
   for (auto key : {"f", "Targuments"}) {
     graph_utils::CopyAttribute(key, map_node, &new_node);
   }
-
-  // Set `output_types` and `output_shapes` attributes.
   for (auto key : {"output_shapes", "output_types"}) {
     graph_utils::CopyAttribute(key, batch_node, &new_node);
   }
+
+  // Optional attributes.
+  // TODO(jsimsa): Support `use_inter_op_parallelism` and `sloppy`.
+  for (auto key : {"preserve_cardinality"}) {
+    if (gtl::FindOrNull(map_node.attr(), key)) {
+      graph_utils::CopyAttribute(key, map_node, &new_node);
+    }
+  }
+
   return new_node;
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 2b0a347ce6..233d7968c8 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -55,8 +55,9 @@ NodeDef MakeFusedNode(const NodeDef& map_node,
   }
 
   // Optional attrs.
-  for (auto key : {"use_inter_op_parallelism", "sloppy"}) {
-    if (const auto* attr = gtl::FindOrNull(map_node.attr(), key)) {
+  for (auto key :
+       {"use_inter_op_parallelism", "sloppy", "preserve_cardinality"}) {
+    if (gtl::FindOrNull(map_node.attr(), key)) {
       graph_utils::CopyAttribute(key, map_node, &fused_node);
     }
   }
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 6ca0da2755..6b8015f96a 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -62,9 +62,16 @@ NodeDef MakeFusedNode(const NodeDef& parent_map_node, const NodeDef& map_node,
       gtl::FindOrNull(map_node.attr(), "use_inter_op_parallelism");
   // Some graphs cannot execute with use_inter_op_parallelism=False, so we need
   // to set it to true if one of the ops have it set to true.
-  if (value_or_false(first_parallelism) || value_or_false(second_parallelism)) {
-    (*fused_node.mutable_attr())["use_inter_op_parallelism"].set_b(true);
-  }
+  (*fused_node.mutable_attr())["use_inter_op_parallelism"].set_b(
+      value_or_false(first_parallelism) || value_or_false(second_parallelism));
+
+  const auto* first_cardinality =
+      gtl::FindOrNull(parent_map_node.attr(), "preserve_cardinality");
+  const auto* second_cardinality =
+      gtl::FindOrNull(map_node.attr(), "preserve_cardinality");
+  (*fused_node.mutable_attr())["preserve_cardinality"].set_b(
+      value_or_false(first_cardinality) && value_or_false(second_cardinality));
+
   return fused_node;
 }
 
diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index df96cccee0..b557e32cae 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -51,11 +51,12 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                          std::shared_ptr<std::vector<Tensor>>, StatusCallback)>;
 
   explicit MapAndBatchDatasetOp(OpKernelConstruction* ctx)
-      : UnaryDatasetOpKernel(ctx),
-        op_version_(ctx->def().op() == "MapAndBatchDataset" ? 1 : 2) {
+      : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
  protected:
@@ -68,29 +69,11 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         errors::InvalidArgument("batch_size must be greater than zero."));
 
     int64 num_parallel_calls;
-    switch (op_version_) {
-      case 1:
-        int64 num_parallel_batches;
-        OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "num_parallel_batches",
-                                                &num_parallel_batches));
-        num_parallel_calls = num_parallel_batches * batch_size;
-        OP_REQUIRES(ctx, num_parallel_batches > 0,
-                    errors::InvalidArgument(
-                        "num_parallel_batches must be greater than zero."));
-        break;
-      case 2:
-        OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "num_parallel_calls",
-                                                &num_parallel_calls));
-        OP_REQUIRES(ctx,
-                    num_parallel_calls > 0 || num_parallel_calls == kAutoTune,
-                    errors::InvalidArgument(
-                        "num_parallel_calls must be greater than zero."));
-        break;
-      default:
-        OP_REQUIRES(ctx, false,
-                    errors::Unimplemented("Unsupported operation version %d.",
-                                          op_version_));
-    }
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "num_parallel_calls",
+                                            &num_parallel_calls));
+    OP_REQUIRES(ctx, num_parallel_calls > 0 || num_parallel_calls == kAutoTune,
+                errors::InvalidArgument(
+                    "num_parallel_calls must be greater than zero."));
 
     bool drop_remainder;
     OP_REQUIRES_OK(ctx,
@@ -146,7 +129,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     *output = new Dataset(ctx, input, func_, batch_size, num_parallel_calls,
                           drop_remainder, output_types_, output_shapes_,
                           std::move(captured_func), &ctx->eigen_cpu_device(),
-                          std::move(map_func));
+                          std::move(map_func), preserve_cardinality_);
   }
 
  private:
@@ -159,7 +142,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
             const std::vector<PartialTensorShape>& output_shapes,
             std::unique_ptr<CapturedFunction> captured_func,
             const Eigen::ThreadPoolDevice* device,
-            MapAndBatchIteratorFunction map_func)
+            MapAndBatchIteratorFunction map_func, bool preserve_cardinality)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
@@ -170,7 +153,8 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
           output_shapes_(output_shapes),
           captured_func_(std::move(captured_func)),
           device_(device),
-          map_func_(std::move(map_func)) {
+          map_func_(std::move(map_func)),
+          preserve_cardinality_(preserve_cardinality) {
       input_->Ref();
     }
 
@@ -195,8 +179,6 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       return "MapAndBatchDatasetOp::Dataset";
     }
 
-    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
-    // modified to preserve cardinality.
     int64 Cardinality() const override {
       int64 n = input_->Cardinality();
       if (n == kInfiniteCardinality || n == kUnknownCardinality) {
@@ -434,6 +416,15 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         std::shared_ptr<std::vector<Tensor>> return_values =
             std::make_shared<std::vector<Tensor>>();
         auto done = [this, ctx, result, return_values, offset](Status status) {
+          if (dataset()->preserve_cardinality_ &&
+              errors::IsOutOfRange(status)) {
+            // To guarantee that the transformation preserves the cardinality of
+            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
+            // former may be interpreted by a caller as the end of sequence.
+            status = errors::InvalidArgument(
+                "Function invocation produced OutOfRangeError: ",
+                status.error_message());
+          }
           result->UpdateStatus(status, offset);
           if (status.ok()) {
             Status allocate_status =
@@ -546,11 +537,14 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                            bool* end_of_sequence) {
         mutex_lock l(result->mu);
         if (result->num_elements == 0) {
-          *end_of_sequence = true;
-          return Status::OK();
+          if (result->status.ok() || errors::IsOutOfRange(result->status)) {
+            *end_of_sequence = true;
+            return Status::OK();
+          } else {
+            *end_of_sequence = false;
+            return result->status;
+          }
         }
-        // `f` may deliberately raise `errors::OutOfRange` to indicate that we
-        // should terminate the iteration early.
         if (!result->status.ok() && !errors::IsOutOfRange(result->status)) {
           // Deallocate tensors allocated for the output.
           result->output.clear();
@@ -580,7 +574,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         } else {
           *out_tensors = std::move(result->output);
         }
-        *end_of_sequence = result->num_elements == 0;
+        *end_of_sequence = false;
         return Status::OK();
       }
 
@@ -813,12 +807,13 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     const std::unique_ptr<CapturedFunction> captured_func_;
     const Eigen::ThreadPoolDevice* device_;  // not owned
     const MapAndBatchIteratorFunction map_func_;
+    const bool preserve_cardinality_;
   };
 
-  const int op_version_;
   DataTypeVector output_types_;
   std::vector<PartialTensorShape> output_shapes_;
   NameAttrList func_;
+  bool preserve_cardinality_;
 };
 
 REGISTER_KERNEL_BUILDER(
diff --git a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
index 705101262c..46233942f0 100644
--- a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
@@ -59,6 +59,9 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+    // TODO(saeta): Implement support for preserve_cardinality logic.
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
  protected:
@@ -133,6 +136,8 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       return "NumaMapAndBatchDatasetOp::Dataset";
     }
 
+    // TODO(b/120482302): Note that this is inaccurate until
+    // NumaMapAndBatchMapDataset modified to preserve cardinality.
     int64 Cardinality() const override {
       int64 n = input_->Cardinality();
       if (n == kInfiniteCardinality || n == kUnknownCardinality) {
@@ -1138,6 +1143,7 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
   DataTypeVector output_types_;
   std::vector<PartialTensorShape> output_shapes_;
   NameAttrList func_;
+  bool preserve_cardinality_;
 };
 
 REGISTER_KERNEL_BUILDER(
diff --git a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
index 40771a5cf9..ea99a8b32c 100644
--- a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
@@ -187,7 +187,8 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
           new ParseExampleFunctor(this));
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParseExample")}, input_,
-          std::move(parse_example_functor), num_parallel_calls_, sloppy_);
+          std::move(parse_example_functor), num_parallel_calls_, sloppy_,
+          /*preserve_cardinality=*/true);
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 2eceab137b..7c6d2f7c59 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -37,6 +37,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("Tstate", &state_types_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
   void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
@@ -53,7 +55,7 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
     *output = new Dataset(ctx, input, func_, std::move(initial_state),
                           std::move(captured_func), state_types_, output_types_,
-                          output_shapes_);
+                          output_shapes_, preserve_cardinality_);
   }
 
  private:
@@ -64,7 +66,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
             std::unique_ptr<CapturedFunction> captured_func,
             const DataTypeVector& state_types,
             const DataTypeVector& output_types,
-            const std::vector<PartialTensorShape>& output_shapes)
+            const std::vector<PartialTensorShape>& output_shapes,
+            bool preserve_cardinality)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
@@ -72,7 +75,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
           captured_func_(std::move(captured_func)),
           state_types_(state_types),
           output_types_(output_types),
-          output_shapes_(output_shapes) {
+          output_shapes_(output_shapes),
+          preserve_cardinality_(preserve_cardinality) {
       input_->Ref();
     }
 
@@ -93,8 +97,6 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "ScanDatasetOp::Dataset"; }
 
-    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
-    // modified to preserve cardinality.
     int64 Cardinality() const override { return input_->Cardinality(); }
 
    protected:
@@ -207,10 +209,19 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
             out_tensors->push_back(std::move(state_and_output[i]));
           }
         } else if (errors::IsOutOfRange(s)) {
-          // `f` may deliberately raise `errors::OutOfRange` to indicate
-          // that we should terminate the iteration early.
-          *end_of_sequence = true;
-          return Status::OK();
+          if (dataset()->preserve_cardinality_) {
+            // To guarantee that the transformation preserves the cardinality of
+            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
+            // former may be interpreted by a caller as the end of sequence.
+            return errors::InvalidArgument(
+                "Function invocation produced OutOfRangeError: ",
+                s.error_message());
+          } else {
+            // `f` may deliberately raise `errors::OutOfRange` to indicate
+            // that we should terminate the iteration early.
+            *end_of_sequence = true;
+            return Status::OK();
+          }
         }
         return s;
       }
@@ -267,12 +278,14 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
     const DataTypeVector state_types_;
     const DataTypeVector output_types_;
     const std::vector<PartialTensorShape> output_shapes_;
+    const bool preserve_cardinality_;
   };
 
   DataTypeVector state_types_;
   DataTypeVector output_types_;
   std::vector<PartialTensorShape> output_shapes_;
   NameAttrList func_;
+  bool preserve_cardinality_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("ExperimentalScanDataset").Device(DEVICE_CPU),
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index e7ec02f5a0..c3c392d9c2 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -40,6 +40,8 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("use_inter_op_parallelism",
                                      &use_inter_op_parallelism_));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
   void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
@@ -86,9 +88,10 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
       };
     }
 
-    *output = new Dataset(ctx, input, func_, std::move(captured_func),
-                          output_types_, output_shapes_,
-                          use_inter_op_parallelism_, std::move(map_func));
+    *output =
+        new Dataset(ctx, input, func_, std::move(captured_func), output_types_,
+                    output_shapes_, use_inter_op_parallelism_,
+                    std::move(map_func), preserve_cardinality_);
   }
 
  private:
@@ -99,11 +102,13 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
             std::unique_ptr<CapturedFunction> captured_func,
             const DataTypeVector& output_types,
             const std::vector<PartialTensorShape>& output_shapes,
-            bool use_inter_op_parallelism, MapIteratorFunction map_func)
+            bool use_inter_op_parallelism, MapIteratorFunction map_func,
+            bool preserve_cardinality)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
           use_inter_op_parallelism_(use_inter_op_parallelism),
+          preserve_cardinality_(preserve_cardinality),
           captured_func_(std::move(captured_func)),
           output_types_(output_types),
           output_shapes_(output_shapes),
@@ -128,8 +133,6 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
 
     string DebugString() const override { return "MapDatasetOp::Dataset"; }
 
-    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
-    // modified to preserve cardinality.
     int64 Cardinality() const override { return input_->Cardinality(); }
 
    protected:
@@ -206,10 +209,19 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
         Status s = map_func_(ctx, instantiated_captured_func_.get(), args,
                              out_tensors);
         if (errors::IsOutOfRange(s)) {
-          // `f` may deliberately raise `errors::OutOfRange` to indicate
-          // that we should terminate the iteration early.
-          *end_of_sequence = true;
-          return Status::OK();
+          if (dataset()->preserve_cardinality_) {
+            // To guarantee that the transformation preserves the cardinality of
+            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
+            // former may be interpreted by a caller as the end of sequence.
+            return errors::InvalidArgument(
+                "Function invocation produced OutOfRangeError: ",
+                s.error_message());
+          } else {
+            // `f` may deliberately raise `errors::OutOfRange` to indicate
+            // that we should terminate the iteration early.
+            *end_of_sequence = true;
+            return Status::OK();
+          }
         } else {
           return s;
         }
@@ -242,6 +254,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
     const DatasetBase* const input_;
     const NameAttrList func_;
     const bool use_inter_op_parallelism_;
+    const bool preserve_cardinality_;
     const std::unique_ptr<CapturedFunction> captured_func_;
     const DataTypeVector output_types_;
     const std::vector<PartialTensorShape> output_shapes_;
@@ -252,6 +265,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
   std::vector<PartialTensorShape> output_shapes_;
   NameAttrList func_;
   bool use_inter_op_parallelism_;
+  bool preserve_cardinality_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("MapDataset").Device(DEVICE_CPU), MapDatasetOp);
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index 7c589a648e..4924ad63a0 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -41,6 +41,8 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("use_inter_op_parallelism",
                                      &use_inter_op_parallelism_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("sloppy", &sloppy_));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
  protected:
@@ -61,9 +63,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     std::vector<int> indices;
     OP_REQUIRES_OK(ctx, ComputeShortCircuitIndices(ctx, func_, &indices));
 
-    *output = new Dataset(ctx, input, func_, num_parallel_calls, output_types_,
-                          output_shapes_, use_inter_op_parallelism_, sloppy_,
-                          std::move(captured_func), indices);
+    *output =
+        new Dataset(ctx, input, func_, num_parallel_calls, output_types_,
+                    output_shapes_, use_inter_op_parallelism_, sloppy_,
+                    std::move(captured_func), indices, preserve_cardinality_);
   }
 
  private:
@@ -75,7 +78,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
             const std::vector<PartialTensorShape>& output_shapes,
             bool use_inter_op_parallelism, bool sloppy,
             std::unique_ptr<CapturedFunction> captured_func,
-            const std::vector<int> indices)
+            const std::vector<int> indices, bool preserve_cardinality)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
@@ -84,6 +87,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
           output_shapes_(output_shapes),
           use_inter_op_parallelism_(use_inter_op_parallelism),
           sloppy_(sloppy),
+          preserve_cardinality_(preserve_cardinality),
           captured_func_(std::move(captured_func)),
           indices_(indices),
           can_move_(indices.empty() ? std::vector<bool>()
@@ -103,7 +107,8 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
       }
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParallelMap")}, input_,
-          std::move(parallel_map_functor), num_parallel_calls_, sloppy_);
+          std::move(parallel_map_functor), num_parallel_calls_, sloppy_,
+          preserve_cardinality_);
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -118,8 +123,6 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
       return "ParallelMapDatasetOp::Dataset";
     }
 
-    // TODO(b/120482302): Note that this is inaccurate until MapDataset is
-    // modified to preserve cardinality.
     int64 Cardinality() const override { return input_->Cardinality(); }
 
    protected:
@@ -252,6 +255,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     const std::vector<PartialTensorShape> output_shapes_;
     const bool use_inter_op_parallelism_;
     const bool sloppy_;
+    const bool preserve_cardinality_;
     const std::unique_ptr<CapturedFunction> captured_func_;
     const std::vector<int> indices_;
     const std::vector<bool> can_move_;
@@ -261,6 +265,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
   std::vector<PartialTensorShape> output_shapes_;
   bool use_inter_op_parallelism_;
   bool sloppy_;
+  bool preserve_cardinality_;
   NameAttrList func_;
 };
 
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index 02ccf6b004..32b634ae85 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -32,20 +32,34 @@ namespace {
 
 class ParallelMapIterator : public DatasetBaseIterator {
  public:
-  ParallelMapIterator(const typename DatasetBaseIterator::BaseParams& params,
-                      const DatasetBase* input_dataset,
-                      std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
-                      int32 num_parallel_calls, bool sloppy)
-      : DatasetBaseIterator(params),
+  struct Params {
+    Params(std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
+           int32 num_parallel_calls, bool sloppy, bool preserve_cardinality)
+        : parallel_map_functor(std::move(parallel_map_functor)),
+          num_parallel_calls(num_parallel_calls),
+          sloppy(sloppy),
+          preserve_cardinality(preserve_cardinality) {}
+
+    std::unique_ptr<ParallelMapFunctor> parallel_map_functor;
+    int32 num_parallel_calls;
+    bool sloppy;
+    bool preserve_cardinality;
+  };
+
+  ParallelMapIterator(
+      const typename DatasetBaseIterator::BaseParams& base_params,
+      const DatasetBase* input_dataset, Params params)
+      : DatasetBaseIterator(base_params),
         input_dataset_(input_dataset),
-        parallel_map_functor_(std::move(parallel_map_functor)),
+        parallel_map_functor_(std::move(params.parallel_map_functor)),
         mu_(std::make_shared<mutex>()),
         cond_var_(std::make_shared<condition_variable>()),
         num_parallel_calls_(std::make_shared<model::SharedState>(
-            num_parallel_calls, mu_, cond_var_)),
-        sloppy_(sloppy) {
+            params.num_parallel_calls, mu_, cond_var_)),
+        sloppy_(params.sloppy),
+        preserve_cardinality_(params.preserve_cardinality) {
     std::vector<string> components =
-        str_util::Split(params.prefix, "::", str_util::SkipEmpty());
+        str_util::Split(base_params.prefix, "::", str_util::SkipEmpty());
     prefix_end_ = components.back();
   }
 
@@ -234,10 +248,19 @@ class ParallelMapIterator : public DatasetBaseIterator {
       return Status::OK();
     }
     if (errors::IsOutOfRange(result->status)) {
-      // `f` may deliberately raise `errors::OutOfRange` to indicate that we
-      // should terminate the iteration early.
-      *end_of_sequence = true;
-      return Status::OK();
+      if (preserve_cardinality_) {
+        // To guarantee that the transformation preserves the cardinality of the
+        // dataset, we convert `OutOfRange` to `InvalidArgument` as the former
+        // may be interpreted by a caller as the end of sequence.
+        return errors::InvalidArgument(
+            "Function invocation produced OutOfRangeError: ",
+            result->status.error_message());
+      } else {
+        // `f` may deliberately raise `errors::OutOfRange` to indicate
+        // that we should terminate the iteration early.
+        *end_of_sequence = true;
+        return Status::OK();
+      }
     }
     *end_of_sequence = result->end_of_input;
     return result->status;
@@ -369,6 +392,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
   const std::shared_ptr<model::SharedState> num_parallel_calls_;
   // Determines whether outputs can be produced in non-deterministic order.
   const bool sloppy_;
+  const bool preserve_cardinality_;
   // Counts the number of outstanding calls.
   int64 num_calls_ GUARDED_BY(*mu_) = 0;
   std::unique_ptr<IteratorBase> input_impl_;
@@ -386,10 +410,12 @@ std::unique_ptr<IteratorBase> NewParallelMapIterator(
     const DatasetBaseIterator::BaseParams& params,
     const DatasetBase* input_dataset,
     std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
-    int32 num_parallel_calls, bool sloppy) {
-  return MakeUnique<ParallelMapIterator>(params, input_dataset,
-                                         std::move(parallel_map_functor),
-                                         num_parallel_calls, sloppy);
+    int32 num_parallel_calls, bool sloppy, bool preserve_cardinality) {
+  return MakeUnique<ParallelMapIterator>(
+      params, input_dataset,
+      ParallelMapIterator::Params{std::move(parallel_map_functor),
+                                  num_parallel_calls, sloppy,
+                                  preserve_cardinality});
 }
 
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.h b/tensorflow/core/kernels/data/parallel_map_iterator.h
index 08c16a6c11..de30446f26 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.h
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.h
@@ -48,7 +48,7 @@ std::unique_ptr<IteratorBase> NewParallelMapIterator(
     const DatasetBaseIterator::BaseParams& params,
     const DatasetBase* input_dataset,
     std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
-    int32 num_parallel_calls, bool sloppy);
+    int32 num_parallel_calls, bool sloppy, bool preserve_cardinality);
 
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index b2ea637464..9163ed715d 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -144,6 +144,7 @@ REGISTER_OP("MapDataset")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
     .Attr("use_inter_op_parallelism: bool = true")
+    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("ParallelMapDataset")
@@ -157,6 +158,7 @@ REGISTER_OP("ParallelMapDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .Attr("use_inter_op_parallelism: bool = true")
     .Attr("sloppy: bool = false")
+    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("PrefetchDataset")
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index 0711034118..f904e2536d 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -174,6 +174,7 @@ REGISTER_OP("ExperimentalMapAndBatchDataset")
     .Attr("Targuments: list(type) >= 0")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
+    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       // Use index from the end to retrieve the Input shapes,
       // so that to avoid guessing the length of "other_arguments".
@@ -198,6 +199,7 @@ REGISTER_OP("ExperimentalMapDataset")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
     .Attr("use_inter_op_parallelism: bool = true")
+    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("ExperimentalMatchingFilesDataset")
@@ -277,6 +279,7 @@ REGISTER_OP("ExperimentalScanDataset")
     .Attr("Targuments: list(type) >= 0")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
+    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("ExperimentalSetStatsAggregatorDataset")
@@ -425,6 +428,7 @@ REGISTER_OP("ExperimentalNumaMapAndBatchDataset")
     .Attr("Targuments: list(type) >= 0")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
+    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       // Use index from the end to retrieve the Input shapes,
       // so that to avoid guessing the length of "other_arguments".
diff --git a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
index e6e24c3db1..5c115f7ae3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/map_and_batch_test.py
@@ -154,7 +154,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     else:
       self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
       self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
       if not drop_remainder:
@@ -180,7 +180,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertAllEqual([[0], [1], [4], [9]], self.evaluate(next_element))
       self.assertAllEqual([[16], [25], [36], [49]], self.evaluate(next_element))
       self.assertAllEqual([[64], [81]], self.evaluate(next_element))
@@ -204,7 +204,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     elements = []
     for _ in range(100):
       elements.append(iterator.get_next())
-    with self.cached_session() as sess:
+    with self.cached_session():
       for i in range(5):
         got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
@@ -234,7 +234,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     elements = []
     for _ in range(100):
       elements.append(iterator.get_next())
-    with self.cached_session() as sess:
+    with self.cached_session():
       for i in range(4):
         got = self.evaluate(elements)
         got.sort(key=lambda x: x[0])
@@ -267,7 +267,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     init_op = iterator.initializer
     get_next = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.evaluate(init_op)
       for i in range(2):
         actual = self.evaluate(get_next)
@@ -329,7 +329,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     init_op = iterator.initializer
     get_next = iterator.get_next()
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.evaluate(init_op)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "number of elements does not match"):
@@ -361,7 +361,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       for _ in range(3):
         self.evaluate(get_next)
 
@@ -380,13 +380,11 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("6NUMA", 99, True),
   )
   @test_util.run_deprecated_v1
-  def testMapAndBatchOutOfRangeError(self, threshold, numa_aware):
+  def testMapAndBatchMapError(self, threshold, numa_aware):
 
     def raising_py_fn(i):
-      if i == threshold:
+      if i >= threshold:
         raise StopIteration()
-      elif i > threshold:
-        raise RuntimeError("Alternate error; you shouldn't see me! (i: %s)" % i)
       else:
         return i
 
@@ -401,14 +399,19 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       for i in range(threshold // 10):
         self.assertAllEqual([i * 10 + j for j in range(10)],
                             self.evaluate(get_next))
-      if threshold % 10 != 0:
-        self.assertAllEqual(
-            [threshold // 10 * 10 + j for j in range(threshold % 10)],
-            self.evaluate(get_next))
+      if numa_aware:
+        if threshold % 10 != 0:
+          self.assertAllEqual(
+              [threshold // 10 * 10 + j for j in range(threshold % 10)],
+              self.evaluate(get_next))
+      else:
+        for i in range(threshold // 10, 10):
+          with self.assertRaises(errors.InvalidArgumentError):
+            self.evaluate(get_next)
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(get_next)
 
@@ -451,7 +454,7 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
-    with self.cached_session() as sess:
+    with self.cached_session():
       for _ in range(10):
         self.assertAllEqual([element for _ in range(10)],
                             self.evaluate(get_next))
@@ -511,9 +514,8 @@ class MapAndBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
-    with self.cached_session() as sess:
+    with self.cached_session():
       for i in range(10):
-        print("Case %d" % i)
         if i < 5:
           self.assertAllEqual([i * 10 + j + 1 for j in range(10)],
                               self.evaluate(get_next))
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 668bf3e800..48b06d6ff1 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -608,6 +608,7 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
         batch_size=self._batch_size_t,
         num_parallel_calls=self._num_parallel_calls_t,
         drop_remainder=self._drop_remainder_t,
+        preserve_cardinality=True,
         **dataset_ops.flat_structure(structure=self._output_structure))
 
   @property
diff --git a/tensorflow/python/data/experimental/ops/scan_ops.py b/tensorflow/python/data/experimental/ops/scan_ops.py
index c768373cf4..68ff1d5d7f 100644
--- a/tensorflow/python/data/experimental/ops/scan_ops.py
+++ b/tensorflow/python/data/experimental/ops/scan_ops.py
@@ -138,6 +138,7 @@ class _ScanDataset(dataset_ops.UnaryDataset):
         nest.flatten(sparse.serialize_sparse_tensors(self._initial_state)),
         self._scan_func.function.captured_inputs,
         f=self._scan_func.function,
+        preserve_cardinality=True,
         **dataset_ops.flat_structure(self))
 
   @property
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index ba6d20373e..4270f09d38 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -944,9 +944,10 @@ class DatasetV2(object):
       Dataset: A `Dataset`.
     """
     if num_parallel_calls is None:
-      return MapDataset(self, map_func)
+      return MapDataset(self, map_func, preserve_cardinality=True)
     else:
-      return ParallelMapDataset(self, map_func, num_parallel_calls)
+      return ParallelMapDataset(
+          self, map_func, num_parallel_calls, preserve_cardinality=True)
 
   def flat_map(self, map_func):
     """Maps `map_func` across this dataset and flattens the result.
@@ -1540,8 +1541,13 @@ class DatasetV1(DatasetV2):
 
   @functools.wraps(DatasetV2.map)
   def map(self, map_func, num_parallel_calls=None):
-    return DatasetV1Adapter(super(DatasetV1, self).map(
-        map_func, num_parallel_calls))
+    if num_parallel_calls is None:
+      return DatasetV1Adapter(
+          MapDataset(self, map_func, preserve_cardinality=False))
+    else:
+      return DatasetV1Adapter(
+          ParallelMapDataset(
+              self, map_func, num_parallel_calls, preserve_cardinality=False))
 
   @functools.wraps(DatasetV2.flat_map)
   def flat_map(self, map_func):
@@ -2781,11 +2787,16 @@ def _warn_if_collections(transformation_name):
 class MapDataset(UnaryDataset):
   """A `Dataset` that maps a function over elements in its input."""
 
-  def __init__(self, input_dataset, map_func, use_inter_op_parallelism=True):
+  def __init__(self,
+               input_dataset,
+               map_func,
+               use_inter_op_parallelism=True,
+               preserve_cardinality=False):
     """See `Dataset.map()` for details."""
     super(MapDataset, self).__init__(input_dataset)
     self._input_dataset = input_dataset
     self._use_inter_op_parallelism = use_inter_op_parallelism
+    self._preserve_cardinality = preserve_cardinality
     self._map_func = StructuredFunctionWrapper(
         map_func, self._transformation_name(), dataset=input_dataset)
 
@@ -2796,6 +2807,7 @@ class MapDataset(UnaryDataset):
         self._map_func.function.captured_inputs,
         f=self._map_func.function,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
+        preserve_cardinality=self._preserve_cardinality,
         **flat_structure(structure=self._map_func.output_structure))
 
   def _functions(self):
@@ -2824,10 +2836,11 @@ class ParallelMapDataset(MapDataset):
                input_dataset,
                map_func,
                num_parallel_calls,
-               use_inter_op_parallelism=True):
+               use_inter_op_parallelism=True,
+               preserve_cardinality=False):
     """See `Dataset.map()` for details."""
-    super(ParallelMapDataset, self).__init__(input_dataset, map_func,
-                                             use_inter_op_parallelism)
+    super(ParallelMapDataset, self).__init__(
+        input_dataset, map_func, use_inter_op_parallelism, preserve_cardinality)
 
     self._num_parallel_calls = ops.convert_to_tensor(
         num_parallel_calls, dtype=dtypes.int32, name="num_parallel_calls")
@@ -2841,6 +2854,7 @@ class ParallelMapDataset(MapDataset):
         f=self._map_func.function,
         num_parallel_calls=self._num_parallel_calls,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
+        preserve_cardinality=self._preserve_cardinality,
         **flat_structure(structure=self._map_func.output_structure))
 
 
-- 
GitLab


From 619602882c8f725bdab76e55e6fdc9af2d8db64a Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 5 Dec 2018 14:31:31 -0800
Subject: [PATCH 1385/1554] Make cond_v2 + while_v2 rewrite the forward pass to
 output needed intermediates.

This is to enable taking the gradient of TF functions nested inside
new control flow. Prior to this change, cond_v2 and while_v2 initially
outputted all intermediates in case they were needed by the gradient
computation. Functions, however, rewrite the forward pass with any
intermediates needed by the gradient when the gradient is
computed. This meant that these intermediates were not initially
available for cond_v2 and while_v2 to output. By making everything
output intermediates on-the-fly, intermediates are available when
needed.

PiperOrigin-RevId: 224222524
---
 tensorflow/python/framework/func_graph.py     |   8 +
 .../python/kernel_tests/cond_v2_test.py       |  24 +-
 .../kernel_tests/control_flow_ops_py_test.py  |  65 +++-
 .../python/kernel_tests/while_v2_test.py      |  62 ++--
 tensorflow/python/ops/cond_v2.py              | 161 ++++++----
 tensorflow/python/ops/gradients_impl.py       |  12 +-
 tensorflow/python/ops/while_v2.py             | 297 ++++++++++--------
 7 files changed, 402 insertions(+), 227 deletions(-)

diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index f74d072e8e..0f6db500cc 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -153,6 +153,14 @@ class FuncGraph(ops.Graph):
     self._graph_key = graph._graph_key
     # pylint: enable=protected-access
 
+  @property
+  def output_types(self):
+    return [t.dtype for t in self.outputs]
+
+  @property
+  def output_shapes(self):
+    return [t.shape for t in self.outputs]
+
   @property
   def variables(self):
     """A list of variables accessed by this FuncGraph.
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 1f4b37ce2a..843d007cc8 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -171,7 +171,6 @@ class CondV2Test(test.TestCase):
             cond2_op.get_attr("else_branch").name, r"foo_cond_1_false_\d*")
 
   def testDefunInCond(self):
-    self.skipTest("b/117293122")
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
 
@@ -191,8 +190,6 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [y])
 
   def testNestedDefunInCond(self):
-    self.skipTest("b/117284369")
-
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
 
@@ -217,8 +214,6 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [y])
 
   def testDoubleNestedDefunInCond(self):
-    self.skipTest("b/117284369")
-
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
 
@@ -778,6 +773,25 @@ class CondV2Test(test.TestCase):
     self.assertAllEqual(
         self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
 
+  def testForwardPassRewrite(self):
+    x = constant_op.constant(1.0, name="x")
+    output = cond_v2.cond_v2(constant_op.constant(True),
+                             lambda: x * 2.0,
+                             lambda: x)
+    if_op = output.op.inputs[0].op
+    self.assertEqual(if_op.type, "If")
+    # pylint: disable=g-deprecated-assert
+    self.assertEqual(len(if_op.outputs), 1)
+
+    gradients_impl.gradients(output, x)
+    # if_op should have been rewritten to output 2.0 intermediate.
+    self.assertEqual(len(if_op.outputs), 2)
+
+    gradients_impl.gradients(output, x)
+    # Computing the gradient again shouldn't rewrite if_op again.
+    self.assertEqual(len(if_op.outputs), 2)
+    # pylint: enable=g-deprecated-assert
+
 
 class CondV2CollectionTest(test.TestCase):
 
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 9dd667f0e8..95420fc0f1 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -831,12 +831,12 @@ class ControlFlowTest(test.TestCase):
       with ops.device("/cpu:1"):
         grad = gradients_impl.gradients(z, x)[0]
 
-      self.assertEqual(sess.run(grad, {pred: True, x: 1.0, y: 2.0}), 4.0)
-      self.assertEqual(sess.run(grad, {pred: False, x: 1.0, y: 2.0}), 0.0)
-
       with ops.device("/cpu:0"):
         grad_grad = gradients_impl.gradients(grad, x)[0]
 
+      self.assertEqual(sess.run(grad, {pred: True, x: 1.0, y: 2.0}), 4.0)
+      self.assertEqual(sess.run(grad, {pred: False, x: 1.0, y: 2.0}), 0.0)
+
       # v1 control flow gets None second derivative for some reason.
       if not control_flow_ops.ENABLE_COND_V2:
         self.assertIsNone(grad_grad)
@@ -1817,15 +1817,15 @@ class ControlFlowTest(test.TestCase):
       with ops.device("/cpu:1"):
         grad = gradients_impl.gradients(z, x_init)[0]
 
+      with ops.device("/cpu:0"):
+        grad_grad = gradients_impl.gradients(grad, x_init)[0]
+
       self.assertEqual(sess.run(grad, {pred: True}), 8.0)
       self.assertEqual(sess.run(grad, {pred: False}), 0.0)
 
       if not control_flow_ops.ENABLE_WHILE_V2:
         return
 
-      with ops.device("/cpu:0"):
-        grad_grad = gradients_impl.gradients(grad, x_init)[0]
-
       self.assertEqual(sess.run(grad_grad, {pred: True}), 0.0)
       self.assertEqual(sess.run(grad_grad, {pred: False}), 0.0)
 
@@ -2305,6 +2305,49 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(i_val, 3)
       self.assertAllClose(x_val, 1.0)
 
+  def testNestedResourceAccess(self):
+    var = resource_variable_ops.ResourceVariable(constant_op.constant(3.0))
+
+    @eager_function.defun
+    def test_fn():
+      x = constant_op.constant(0.0)
+      r = control_flow_ops.while_loop(
+          # Outer loop condition
+          lambda i, y: i < 2,
+          # Outer loop body
+          lambda i, y: (i + 1, y + control_flow_ops.cond(
+              constant_op.constant(True),
+              # True branch
+              lambda: control_flow_ops.while_loop(
+                  # Inner loop condition
+                  lambda j, z: j < 3,
+                  # Inner loop body
+                  lambda j, z: (j + 1, z + math_ops.square(var)),
+                  # Inner initial loop value
+                  [0, y])[1],
+              # False branch
+              lambda: (0.0))),
+          # Outer initial loop value
+          [0, x])[1]
+
+      grad = gradients_impl.gradients(r, x)[0]
+      return r, grad
+
+    self.evaluate(variables.global_variables_initializer())
+    r, grad = self.evaluate(test_fn())
+    # 2 * 3 * 3^2
+    self.assertEqual(r, 81.0)
+    # v1 control flow gets the wrong answer!!!
+    # Gradient computation:
+    #   f(x) = x + 3^2
+    #   inner_loop(x) = f(f(f(x))) = x + 3*3^2 = x + 27
+    #   g(x) = x + inner_loop(x) = 2x + 27
+    #   outer_loop(x) = g(g(x)) = 4x + 81
+    #   outer_loop'(x) = 4
+    # Note that v1 control flow gets 4.0 as well if the cond is removed.
+    if control_flow_ops.ENABLE_WHILE_V2 and control_flow_ops.ENABLE_COND_V2:
+      self.assertEqual(grad, 4.0)
+
   def testWhile_NestedInput(self):
     with self.cached_session() as sess:
       named = collections.namedtuple("named", ("a", "b"))
@@ -2651,9 +2694,10 @@ class ControlFlowTest(test.TestCase):
           [i0.get_shape(), tensor_shape.TensorShape([None, 2])])
       s = math_ops.reduce_sum(h)
 
-      self.evaluate(variables.global_variables_initializer())
       optimizer = gradient_descent.GradientDescentOptimizer(0.01)
       op = optimizer.minimize(s)
+
+      self.evaluate(variables.global_variables_initializer())
       self.evaluate(op)
       self.assertAllClose([[0.98000002, 1.98000002]], self.evaluate(x))
 
@@ -2817,6 +2861,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(32.0, self.evaluate(r))
 
   @test_util.run_deprecated_v1
+  @test_util.disable_control_flow_v2("b/118712257")
   def testWhileGrad_StopGradInside(self):
     with self.cached_session():
       x = constant_op.constant(3.0, name="x")
@@ -2837,6 +2882,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(156.0, self.evaluate(r))
 
   @test_util.run_deprecated_v1
+  @test_util.disable_control_flow_v2("b/118712257")
   def testWhileGrad_StopGradInsideNoShape(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(dtypes.float32)
@@ -3384,10 +3430,9 @@ class ControlFlowTest(test.TestCase):
           [constant_op.constant(0), x],
           [tensor_shape.unknown_shape(),
            tensor_shape.unknown_shape()])
+      grad = gradients_impl.gradients(r, x)[0]
       self.assertEqual(r[1].eval(), 65536.0)
-
-      r = gradients_impl.gradients(r, x)[0]
-      self.assertEqual(r.eval(), 524288.0)
+      self.assertEqual(grad.eval(), 524288.0)
       # while_v2 does not have stacks.
       if not control_flow_ops.ENABLE_WHILE_V2:
         self.assertEqual(
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 09cbeb1a0d..d155544f3e 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -252,7 +252,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       if op.type == "While":
         while_op = op
 
-    body_graph = while_v2._get_body_graph(while_op)
+    body_graph = while_v2._get_graph(while_op, "body")
     # body_graph.inputs: [counter_arg, x_arg, tl_arg, *accumulators]
     x_input_t = body_graph.inputs[1]
     accumulator_count = len(
@@ -283,22 +283,26 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
         self.assertListEqual(actual_tensor_shape.as_list(), shape)
 
     def GetAccumulatorForInputAtIndex(while_op, idx):
-      body_graph = while_v2._get_body_graph(while_op)
+      body_graph = while_v2._get_graph(while_op, "body")
       y_input_t = body_graph.inputs[idx]
       push_back_node = [c for c in y_input_t.consumers()
                         if c.type == "TensorListPushBack"][0]
       output_idx = body_graph.outputs.index(push_back_node.outputs[0])
       return while_op.outputs[output_idx]
 
-    x = constant_op.constant(2.)
+    x = array_ops.placeholder(dtype=dtypes.float32, shape=shape)
     y = array_ops.placeholder(dtype=dtypes.float32, shape=shape)
 
     # Forward pass.
-    ret = while_loop_v2(
-        lambda v, u: v < 8.,
-        lambda v, u: (v * v, u), [x, y],
-        return_same_structure=False)
+    ret = while_loop_v2(lambda v, u: v < 8.,
+                        lambda v, u: (math_ops.pow(v, u), u),
+                        [x, y],
+                        return_same_structure=True)
     while_op = ret[0].op.inputs[0].op
+    # Gradient pass.
+    grad = gradients_impl.gradients(ret[0], x)
+    grad_while_op = grad[0].op.inputs[0].op
+
     # Get the TensorList output of While op containing the accumulated values
     # of y.
     # while_op.inputs: [counter_arg, x_arg, y_arg, *accumulators]
@@ -307,14 +311,14 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
                                            element_dtype=dtypes.float32)
     MatchShape(val.shape)
 
-    # Gradient pass.
-    grad = gradients_impl.gradients(ret[1], y)
-    grad_while_op = grad[0].op.inputs[0].op
+    # Take second derivative to generate intermediate grad_while_op outputs
+    gradients_impl.gradients(grad, x)
+
     # Get the TensorList output of gradient While op containing the accumulated
-    # values of grad_y.
+    # values of grad_x (note that grad_x is needed by the second derivative).
     # grad_while_op.inputs:
     # [counter_arg, total_iters_arg, grad_x_arg, grad_y_arg, *other_args]
-    grad_output = GetAccumulatorForInputAtIndex(grad_while_op, 3)
+    grad_output = GetAccumulatorForInputAtIndex(grad_while_op, 2)
     _, val = list_ops.tensor_list_pop_back(grad_output,
                                            element_dtype=dtypes.float32)
     MatchShape(val.shape)
@@ -357,14 +361,13 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
   @test_util.enable_control_flow_v2
   @test_util.run_deprecated_v1
   def testWhileAndTensorArray(self):
-    with self.cached_session() as sess:
-      param = constant_op.constant(2.0)
-      y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
-      # map_fn uses TensorArray internally.
-      r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0)
-      self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r))
-      r = gradients_impl.gradients(r, param)[0]
-      self.assertAllClose(21.0, self.evaluate(r))
+    param = constant_op.constant(2.0)
+    y0 = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="elems")
+    # map_fn uses TensorArray internally.
+    r = functional_ops.map_fn(lambda x: math_ops.multiply(x, param), y0)
+    grad = gradients_impl.gradients(r, param)[0]
+    self.assertAllClose([2.0, 4.0, 6.0, 8.0, 10.0, 12.0], self.evaluate(r))
+    self.assertAllClose(21.0, self.evaluate(grad))
 
   @test_util.run_deprecated_v1
   def testNestedWhile(self):
@@ -438,6 +441,25 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # grad = gradients_impl.gradients(output, [n])
     # self.assertEqual(self.evaluate(grad), 3.5)
 
+  def testForwardPassRewrite(self):
+    x = constant_op.constant(1.0, name="x")
+    output = while_v2.while_loop(lambda x: x < 10.0,
+                                 lambda x: x * 2.0,
+                                 [x])[0]
+    while_op = output.op.inputs[0].op
+    self.assertEqual(while_op.type, "While")
+    # outputs = [loop_counter, x]
+    self.assertLen(while_op.outputs, 2)
+
+    gradients_impl.gradients(output, x)
+    # while_op should have been rewritten to output 2.0 intermediate.
+    # outputs = [loop_counter, x, 2.0_accumulator, x_accumulator]
+    self.assertLen(while_op.outputs, 4)
+
+    gradients_impl.gradients(output, x)
+    # Computing the gradient again shouldn't rewrite while_op again.
+    self.assertLen(while_op.outputs, 4)
+
 
 def ScalarShape():
   return ops.convert_to_tensor([], dtype=dtypes.int32)
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index f7a95bd909..abc99c1205 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -108,6 +108,46 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   assert ([t.dtype for t in true_grad_graph.outputs] ==
           [t.dtype for t in false_grad_graph.outputs])
 
+  if (true_grad_graph.if_op_needs_rewrite or
+      false_grad_graph.if_op_needs_rewrite):
+    # Modify 'op' to output the intermediates needed by the grad functions. Note
+    # that all needed intermediates are wrapped in optionals. Each optional
+    # intermediate output will have a value iff its corresponding branch is
+    # taken.
+    # NOTE(skyewm): if there are any active sessions, this modification to `op`
+    # may make them unrunnable!
+
+    if control_flow_util.InXlaContext(ops.get_default_graph()):
+      # XLA does not yet support optionals, so output intermediates directly and
+      # make them match via FakeParams, which can be converted to zeros in XLA.
+      # TODO(skyewm,jpienaar): can XLA support optionals?
+      true_intermediates = true_grad_graph.xla_intermediates
+      false_intermediates = false_grad_graph.xla_intermediates
+      extra_true_outputs, extra_false_outputs = _make_intermediates_match_xla(
+          true_graph, false_graph, true_intermediates, false_intermediates)
+    else:
+      true_intermediates = true_grad_graph.wrapped_intermediates
+      false_intermediates = false_grad_graph.wrapped_intermediates
+      # Make outputs match by adding none optionals.
+      extra_true_outputs, extra_false_outputs = _make_intermediates_match(
+          true_graph, false_graph, true_intermediates, false_intermediates)
+
+    true_graph.outputs.extend(extra_true_outputs)
+    false_graph.outputs.extend(extra_false_outputs)
+    # TODO(skyewm): indicate it's an internal bug if this fails.
+    _check_same_outputs(true_graph, false_graph)
+
+    true_graph.name += "_rewritten"
+    false_graph.name += "_rewritten"
+
+    op._set_func_attr("then_branch", util.create_new_tf_function(true_graph))
+    op._set_func_attr("else_branch", util.create_new_tf_function(false_graph))
+    op._set_type_list_attr("Tout", true_graph.output_types)
+    op._set_shape_list_attr("output_shapes", true_graph.output_shapes)
+    op._add_outputs(
+        [t.dtype for t in extra_true_outputs],
+        [t.shape for t in extra_true_outputs])
+
   # Resolve references to forward graph tensors in grad graphs and ensure
   # they are in-scope, i.e., belong to one of outer graphs of the grad graph.
   true_grad_inputs = _resolve_grad_inputs(true_graph, true_grad_graph)
@@ -150,40 +190,6 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
   cond_inputs = _make_inputs_match(true_graph, false_graph,
                                    true_inputs, false_inputs)
 
-  # Add all intermediate tensors as function outputs so they're available for
-  # the gradient computation. Since the outputs of the two functions must match,
-  # we wrap all the intermediates in optionals. Each intermediate output will
-  # have a value iff its corresponding branch is taken.
-
-  true_intermediates = _get_intermediates(true_graph)
-  false_intermediates = _get_intermediates(false_graph)
-
-  # Save the original number of outputs to return to the caller.
-  num_cond_outputs = len(true_graph.outputs)
-
-  if control_flow_util.InXlaContext(ops.get_default_graph()):
-    # XLA does not yet support optionals, so output intermediates directly and
-    # make them match via FakeParams, which can be converted to zeros in XLA.
-    # TODO(skyewm,jpienaar): can XLA support optionals?
-    extra_true_outputs, extra_false_outputs = _make_intermediates_match_xla(
-        true_graph, false_graph, true_intermediates, false_intermediates)
-  else:
-    # Wrap intermediates in optionals.
-    wrapped_true_intermediates = _wrap_intermediates(true_graph,
-                                                     true_intermediates)
-    wrapped_false_intermediates = _wrap_intermediates(false_graph,
-                                                      false_intermediates)
-
-    # Make outputs match by adding none optionals.
-    extra_true_outputs, extra_false_outputs = _make_intermediates_match(
-        true_graph, false_graph,
-        wrapped_true_intermediates, wrapped_false_intermediates)
-
-  true_graph.outputs.extend(extra_true_outputs)
-  false_graph.outputs.extend(extra_false_outputs)
-  # TODO(skyewm): somehow indicate it's a bug if this fails.
-  _check_same_outputs(true_graph, false_graph)
-
   # Create the If op.
   tensors = gen_functional_ops._if(  # pylint: disable=protected-access
       pred,
@@ -210,8 +216,7 @@ def _build_cond(pred, true_graph, false_graph, true_inputs, false_inputs,
 
   # Prevent fetching since the variant outputs can't be fetched directly.
   if_op.graph.prevent_fetching(if_op)
-
-  return tensors[:num_cond_outputs]
+  return tensors
 
 
 def _get_func_graphs(if_op):
@@ -540,13 +545,37 @@ def _get_output_shapes(true_graph_outputs, false_graph_outputs):
 class _CondGradFuncGraph(util.CondBranchFuncGraph):
   """FuncGraph for the gradient function of the branch of an If op.
 
-  Handles unwrapping optional intermediate values that are captured by the
-  gradient computation.
+  Handles wrapping and unwrapping intermediate values that are captured by the
+  gradient computation in optionals.
+
+  Attributes:
+    if_op_needs_rewrite: True if any intermediates were captured, meaning the
+      forward If op needs to be written to output the wrapped intermediates.
   """
 
   def __init__(self, name, forward_graph):
     super(_CondGradFuncGraph, self).__init__(name, read_only_collections=False)
+    self.if_op_needs_rewrite = False
     self._forward_graph = forward_graph
+    # Maps from forward intermediate tensor -> the unwrapped captured
+    # intermediate.
+    self._indirect_captures = {}
+    # Maps unwrapped intermediate -> optional-wrapped intermediate in the
+    # forward graph.
+    self._wrapped_intermediates = collections.OrderedDict()
+    # Raw intermediates captured from the forward graph. Populated iff we're in
+    # an XLA context.
+    self._xla_intermediates = []
+
+  @property
+  def wrapped_intermediates(self):
+    """The optional-wrapped intermediates captured from the forward graph."""
+    return list(self._wrapped_intermediates.values())
+
+  @property
+  def xla_intermediates(self):
+    """Raw intermediates captured from the forward graph if XLA is enabled."""
+    return self._xla_intermediates
 
   def _capture_helper(self, tensor, name):
     if (tensor.graph is not self._forward_graph or
@@ -554,19 +583,43 @@ class _CondGradFuncGraph(util.CondBranchFuncGraph):
         tensor in self._forward_graph.outputs):
       return super(_CondGradFuncGraph, self)._capture_helper(tensor, name)
 
-    # 'tensor' is an intermediate in the forward graph. We find the corresonding
-    # optional tensor, which is output from the If op, and capture it as
-    # normal. We then unwrap the captured optional value to get the raw
-    # intermediate value.
-    for consumer in tensor.consumers():
-      if (consumer.type == "OptionalFromValue"
-          and consumer.outputs[0] in self._forward_graph.outputs):
-        optional = consumer.outputs[0]
-        captured_optional = super(_CondGradFuncGraph, self)._capture_helper(
-            optional, name)
-        return gen_dataset_ops.optional_get_value(
-            captured_optional, [tensor.dtype], [tensor.shape])[0]
-    raise ValueError(
-        "Couldn't find OptionalFromValue consumer for tensor '%s'.\n"
-        "This is an internal bug, please report at "
-        "https://github.com/tensorflow/tensorflow/issues." % tensor.name)
+    if control_flow_util.InXlaContext(ops.get_default_graph()):
+      # XLA does not yet support optionals, so capture intermediates directly.
+      # TODO(skyewm,jpienaar): can XLA support optionals?
+      if tensor not in self.captures:
+        self.xla_intermediates.append(tensor)
+        self.if_op_needs_rewrite = True
+      return super(_CondGradFuncGraph, self)._capture_helper(tensor, name)
+
+    captured_tensor = self._indirect_captures.get(tensor)
+    if captured_tensor is not None:
+      return captured_tensor
+
+    # 'tensor' is an uncaptured intermediate in the forward graph. We wrap it in
+    # an optional in the forward graph and capture the optional normally. We
+    # then unwrap the captured optional value in the gradient graph to get the
+    # raw intermediate value.
+
+    if tensor not in self._wrapped_intermediates:
+      # If the gradient has already been computed for this If op, 'tensor' may
+      # already be wrapped.
+      for consumer in tensor.consumers():
+        if (consumer.type == "OptionalFromValue"
+            and consumer.outputs[0] in self._forward_graph.outputs):
+          optional = consumer.outputs[0]
+          break
+      else:
+        # 'tensor' hasn't been wrapped, do it now.
+        with self._forward_graph.as_default():
+          optional = gen_dataset_ops.optional_from_value([tensor])
+        self.if_op_needs_rewrite = True
+
+      self._wrapped_intermediates[tensor] = optional
+
+    optional = self._wrapped_intermediates[tensor]
+    captured_optional = super(_CondGradFuncGraph, self)._capture_helper(
+        optional, name)
+    captured_tensor = gen_dataset_ops.optional_get_value(
+        captured_optional, [tensor.dtype], [tensor.shape])[0]
+    self._indirect_captures[tensor] = captured_tensor
+    return captured_tensor
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index 8cc4d926c7..0a70d6ee61 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -297,8 +297,12 @@ def _DefaultGradYs(grad_ys,
   return new_grad_ys
 
 
-def IsTrainable(tensor):
-  dtype = dtypes.as_dtype(tensor.dtype)
+def IsTrainable(tensor_or_dtype):
+  if isinstance(tensor_or_dtype, ops.Tensor):
+    dtype = tensor_or_dtype.dtype
+  else:
+    dtype = tensor_or_dtype
+  dtype = dtypes.as_dtype(dtype)
   return dtype.base_dtype in (dtypes.float16, dtypes.float32, dtypes.float64,
                               dtypes.complex64, dtypes.complex128,
                               dtypes.resource, dtypes.variant)
@@ -322,6 +326,10 @@ def _VerifyGeneratedGradients(grads, op):
     ValueError: if sizes of gradients and inputs don't match.
     TypeError: if type of any gradient is not valid for its input.
   """
+  # While ops have inputs added to them during the gradient computation, so we
+  # skip the below check. See while_v2 for details.
+  if op.type == "While": return
+
   if len(grads) != len(op.inputs):
     raise ValueError("Num gradients %d generated for op %s do not match num "
                      "inputs %d" % (len(grads), op.node_def, len(op.inputs)))
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 59ca29e3ba..d00c158d15 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -23,6 +23,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import func_graph as func_graph_module
@@ -200,30 +201,6 @@ def while_loop(cond,
             " this as a loop variable." % str(external_capture))
         cond_graph.capture(external_capture)
 
-    # Export all tensors in the loop body that may be needed for gradient
-    # computation. We do this by accumulating the intermediate values in
-    # TensorLists.
-    intermediate_tensors = _get_intermediates(body_graph)
-
-    for intermediate_tensor in intermediate_tensors:
-      tensor_list = list_ops.empty_tensor_list(
-          element_dtype=intermediate_tensor.dtype,
-          element_shape=intermediate_tensor.shape,
-          max_num_elements=maximum_iterations)
-      loop_vars.append(tensor_list)
-      with cond_graph.as_default():
-        # Add a placeholder to cond_graph's inputs corresponding to the
-        # tensor_list.
-        cond_graph.capture(tensor_list)
-      with body_graph.as_default():
-        # Push the intermediate tensor to the tensor list. This captures the
-        # `tensor_list` as well.
-        appended_tensor_list = list_ops.tensor_list_push_back(
-            tensor_list,
-            intermediate_tensor)
-        # Add this modified tensor list to the list of outputs.
-        body_graph.outputs.append(appended_tensor_list)
-
     # Make sure that the shapes of the loop outputs are compatible with the
     # shape invariants, or the shapes of the loop vars if the invariants are not
     # specified.
@@ -272,7 +249,13 @@ def while_loop(cond,
 @ops.RegisterGradient("While")
 def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   """The gradient of a While op produced by while_loop."""
-  body_graph = _get_body_graph(op)
+  cond_graph = _get_graph(op, "cond")
+  body_graph = _get_graph(op, "body")
+  orig_num_params = len(body_graph.outputs)
+
+  maximum_iterations = op.get_attr(
+      "_maximum_iterations") if _is_in_xla_context() else None
+  assert not _is_in_xla_context() or maximum_iterations is not None
 
   # Set the incoming gradient of TensorArray handles to None. The gradient
   # implementation currently assumes all resource tensors correspond to float32
@@ -290,15 +273,13 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   # TODO(b/118712257): Remove the IsTrainable filter once we can handle None
   # output grads in _grad_fn.
   grads = [
-      None if _is_tensor_array_handle(output) or
-      not gradients_impl.IsTrainable(output) else grad
-      for grad, output in zip(grads, op.outputs)
+      None if _is_tensor_array_handle(output) or not _is_trainable(output)
+      else grad for grad, output in zip(grads, body_graph.outputs)
   ]
 
   # Ensure that all non-resource trainable outputs have incoming gradients.
-  assert all(g is not None or o.dtype == dtypes.resource or
-             not gradients_impl.IsTrainable(o)
-             for o, g in zip(op.outputs, grads)
+  assert all(g is not None or o.dtype == dtypes.resource or not _is_trainable(o)
+             for o, g in zip(body_graph.outputs, grads)
             ), "All trainable loop vars must receive incoming gradients."
   # We compute the gradient for the sub-graph between trainable ys and xs
   # with non-None incoming gradients. We later pad the None's to the list of
@@ -307,32 +288,36 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
       body_graph.outputs, body_graph.inputs, grads) if grad is not None])
 
   body_grad_graph, args = _create_grad_func(
-      ys, xs, non_none_grads, body_graph,
-      util.unique_grad_fn_name(body_graph.name), op)
+      ys, xs, non_none_grads, cond_graph, body_graph,
+      util.unique_grad_fn_name(body_graph.name), op, maximum_iterations)
 
-  intermediate_tensors = _get_intermediates(body_grad_graph)
+  if body_grad_graph.while_op_needs_rewrite:
+    # Modify 'op' to output the intermediate accumulators needed by the grad
+    # function.
+    # NOTE(skyewm): if there are any active sessions, this modification to `op`
+    # may make them unrunnable!
 
-  maximum_iterations = op.get_attr(
-      "_maximum_iterations") if _is_in_xla_context() else None
-  assert not _is_in_xla_context() or maximum_iterations is not None
-  for intermediate_tensor in intermediate_tensors:
-    tensor_list = list_ops.empty_tensor_list(
-        element_dtype=intermediate_tensor.dtype,
-        element_shape=intermediate_tensor.shape,
-        max_num_elements=maximum_iterations)
-
-    with body_grad_graph.as_default():
-      tensor_list_ph = body_grad_graph.capture(tensor_list, whitelisted=True)
-      # Push the intermediate tensor to the tensor list.
-      appended_tensor_list = list_ops.tensor_list_push_back(tensor_list_ph,
-                                                            intermediate_tensor)
-      # Add this modified tensor list to the list of outputs.
-      body_grad_graph.outputs.append(appended_tensor_list)
+    cond_graph.name += "_rewritten"
+    body_graph.name += "_rewritten"
+
+    new_inputs = body_grad_graph.empty_tensor_lists
+    new_outputs = body_graph.outputs[orig_num_params:]
+
+    op._set_func_attr("cond", util.create_new_tf_function(cond_graph))
+    op._set_func_attr("body", util.create_new_tf_function(body_graph))
+    op._set_type_list_attr("T", body_graph.output_types)
+    op._set_shape_list_attr("output_shapes", body_graph.output_shapes)
+    op._add_while_inputs(new_inputs)
+    op._add_outputs([t.dtype for t in new_outputs],
+                    [t.shape for t in new_outputs])
+    _copy_handle_data(new_outputs, op.outputs[orig_num_params:])
+
+  captured_inputs = _resolve_grad_captures(body_graph, body_grad_graph, op)
+  loop_vars = args + captured_inputs
 
   def grad_cond(counter, max_iters, *unused_args):
     return counter < max_iters
 
-  loop_vars = args + body_grad_graph.external_captures
   grad_cond_name = util.unique_grad_fn_name(op.get_attr("cond").name)
   cond_grad_graph = func_graph_module.func_graph_from_py_func(
       grad_cond_name, grad_cond, loop_vars, {},
@@ -369,6 +354,24 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   return none_padded_outputs
 
 
+def _is_trainable(tensor):
+  """Returns whether the given tensor is trainable."""
+  if not gradients_impl.IsTrainable(tensor):
+    return False
+
+  # Special case: untrainable accumulator output. The gradients algorithm
+  # doesn't know about tensor lists of untrainable elements. In theory the
+  # tensor list gradient functions should return None as appropriate, but
+  # because we can't return None from the gradient function we filter out
+  # untrainable accumulator output here to avoid computing the gradient at all.
+  if tensor.op.type == "TensorListPopBack" and tensor.value_index == 0:
+    assert tensor.dtype == dtypes.variant
+    element_type = tensor.op.get_attr("element_dtype")
+    return gradients_impl.IsTrainable(element_type)
+
+  return True
+
+
 def _validate_and_convert_to_tensor(maximum_iterations):
   """Checks that `maximum_iterations` is valid.
 
@@ -410,20 +413,21 @@ def _validate_and_convert_to_tensor(maximum_iterations):
 
 
 # TODO(srbs): Pull this into common utils for cond_v2 and while_v2.
-def _get_body_graph(while_op):
-  """Returns `FuncGraph` for the while body.
+def _get_graph(while_op, func_attr_name):
+  """Returns `FuncGraph` for the given function attribute.
 
   Args:
     while_op: The While Operation.
+    func_attr_name: string
 
   Returns:
-    `FuncGraph` for the while body.
+    `FuncGraph`
   """
   # TODO(srbs): Handle TensorShapeProto in function_def_to_graph.input_shapes.
   input_shapes = [
       tensor_shape.TensorShape(s) for s in while_op.get_attr("output_shapes")
   ]
-  func_name = while_op.get_attr("body").name
+  func_name = while_op.get_attr(func_attr_name).name
   fdef = while_op.graph._get_function(func_name).definition
   # `while_op.graph` may not be the same as `ops.get_default_graph()` e.g.
   # if the `while_op` is in the body of another if/while/defun. We build the
@@ -436,7 +440,8 @@ def _get_body_graph(while_op):
   return func_graph
 
 
-def _create_grad_func(ys, xs, grads, func_graph, name, while_op):
+def _create_grad_func(ys, xs, grads, cond_graph, body_graph, name, while_op,
+                      max_iters):
   """Builds and returns the gradient FuncGraph of `func_graph` and its args.
 
   The returned grad_func_graph must be called with the returned
@@ -446,9 +451,11 @@ def _create_grad_func(ys, xs, grads, func_graph, name, while_op):
     ys: A `Tensor` or list of tensors to be differentiated.
     xs: A `Tensor` or list of tensors to be used for differentiation.
     grads: The incoming grads for `ys`.
-    func_graph: FuncGraph for the forward body function.
+    cond_graph: FuncGraph for the forward cond function.
+    body_graph: FuncGraph for the forward body function.
     name: Name of the returned gradient function.
     while_op: The forward While op.
+    max_iters: the maximum number of iterations, or None if no limit.
 
   Returns:
     2-tuple of (grad_func_graph, args).
@@ -464,9 +471,10 @@ def _create_grad_func(ys, xs, grads, func_graph, name, while_op):
   # `external_captures`.
   grad_func_graph = func_graph_module.func_graph_from_py_func(
       name,
-      lambda *args: _grad_fn(ys, xs, args, func_graph),
+      lambda *args: _grad_fn(ys, xs, args, body_graph),
       args, {},
-      func_graph=_WhileBodyGradFuncGraph(name, func_graph))
+      func_graph=_WhileBodyGradFuncGraph(name, cond_graph, body_graph,
+                                         max_iters))
 
   # Add the popped accumulators to the list of outputs.
   for internal_capture in grad_func_graph.internal_captures:
@@ -506,10 +514,11 @@ def _grad_fn(ys, xs, args, func_graph):
   # Build the gradient graph. Note that this builds the gradient computation of
   # func_graph in the current graph, which requires capturing tensors from
   # func_graph. The captured func_graph tensors are resolved to external tensors
-  # in _resolve_grad_inputs.
+  # after the forward While op has been rewritten in _resolve_grad_captures.
   # TODO(srbs): Mark GradientsHelper as public?
   grad_outs = gradients_impl._GradientsHelper(
-      ys, xs, grad_ys=grad_ys, src_graph=func_graph)
+      ys, xs, grad_ys=grad_ys, src_graph=func_graph,
+      unconnected_gradients="zero")
 
   # TODO(b/118712257): Handle the case when grad_outs has None's e.g. when there
   # is a tf.StopGradient in the loop body.
@@ -519,43 +528,45 @@ def _grad_fn(ys, xs, args, func_graph):
   return [counter + 1, total_iters] + grad_outs
 
 
-def _get_intermediates(func_graph):
-  """Returns all tensors in `func_graph` that should be accumulated."""
-  # We currently accumulate output tensors of most ops in the function and rely
-  # on the pruning pass to get rid of the unused accumulators at runtime.
-  # However, this can bloat the GraphDef and make debugging harder so we perform
-  # some optimizations.
-  #
-  # Optimization we currently perform:
-  # 1. We do not accumulate tensors which already have an accumulator
-  #    in the loop body.
-  # 2. We do not accumulate outputs of Identity nodes. When building the
-  #    FuncGraph, we add an Identity node for each output (see
-  #    `AutomaticControlDependencies.mark_as_return`). Accumulating outputs
-  #    of all these nodes bloats the GraphDef quite a bit so we remove those.
-  #    Since the gradient of an Identity node does not rely on its forward op's
-  #    input this is safe to do.
-  #
-  # Other possible optimizations:
-  # 1. Only accumulate tensors that will be required by the backward pass.
-  #    This will require running the gradient pass and hence would increase the
-  #    graph building time for the forward pass.
-  # 2. Do not accumulate Const nodes created inside the loop body.
-  # 3. Do not accumulate inputs that are passed as-is, e.g. loop invariants.
-  # TODO(srbs): 2 and 3 may be hard optimizations for the runtime optimizer
-  # since it requires knowledge of the while loop semantics. If so, consider
-  # doing those here.
-  intermediates = []
-
-  for op in func_graph.get_operations():
-    if op.type == "Identity":
-      continue
-    for o in op.outputs:
-      if (o != func_graph.inputs[0] and  # Loop counter.
-          o.dtype != dtypes.resource and  # Do not accumulate resource tensors.
-          _get_accumulator(o) is None):  # Has existing accumulator.
-        intermediates.append(o)
-  return intermediates
+def _resolve_grad_captures(body_graph, body_grad_graph, while_op):
+  """Returns the tensors to pass as captured inputs to `body_grad_graph`.
+
+  `body_grad_graph` may have external references to:
+  1. Its outer graph containing the input gradients. These are left as-is.
+  2. Accumulators captured from the forward-pass graph. These should have been
+     added as `while_op` outputs after the gradient graph was built. We replace
+     these with the corresponding output of `while_op`, i.e. a tensor in
+     `body_graph.outer_graph`. In the case of nested control flow or functions,
+     the gradient logic handling `body_grad_graph.outer_graph` will make sure
+     the tensor from `body_graph.outer_graph` is also correctly captured.
+
+  Args:
+    body_graph: FuncGraph. The forward-pass body function.
+    body_grad_graph: FuncGraph. The body gradients function.
+    while_op: The forward-pass While Operation calling `body_graph`.
+
+  Returns:
+    A list of input tensors to be passed as the captured inputs to
+      `body_grad_graph`.
+  """
+  new_capture_inputs = []
+  for t in body_grad_graph.external_captures:
+    # All values captured by gradient computation should be from the forward
+    # graph or a captured resource variable (note that input gradients are
+    # regular non-captured inputs).
+    if t.graph == body_graph:
+      # Captured accumulator
+      t = while_op.outputs[t.graph.outputs.index(t)]
+      # Note: We rely on the capturing logic of the gradient While op graph to
+      # correctly capture the tensors in `body_graph.outer_graph`. Both cond_v2
+      # and while_v2 handle this while building their gradient functions.
+      assert t.graph == body_graph.outer_graph
+    else:
+      # Captured resource variable
+      assert t.dtype == dtypes.resource
+
+    new_capture_inputs.append(t)
+  return new_capture_inputs
 
 
 def _get_accumulator(tensor):
@@ -629,9 +640,10 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
      b. Lookup the corresponding resource tensor in the forward outer graph and
         try to capture that.
   2. If the tensor is not of resource type:
-     a. Find the accumulator for that tensor.
-     b. Capture the forward While op output tensor corresponding to the
-        accumulator in this FuncGraph.
+     a. Create an accumulator for that tensor and output it from the forward
+        pass. Note this also requires adding it as an input to the forward pass.
+     b. Capture the accumulator from the forward pass in this FuncGraph. This
+        will later be resolved to the correct output of the forward While op.
      c. Pop a value from the captured placeholder and use it as the captured
         value for the forward pass tensor.
 
@@ -645,16 +657,25 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
   tensor.
 
   Attributes:
-    popped_tensor_lists: Dict from the captured accumulator placeholder to the
+    while_op_needs_rewrite: True if any non-resource intermediates were
+      captured, meaning the forward While op needs to be rewritten to output the
+      corresponding accumulators.
+    empty_tensor_lists: list of EmptyTensorList tensors to be used as initial
+      input to the new accumulators in the forward graph.
+    popped_tensor_lists: dict from the captured accumulator placeholder to the
       TensorList obtained after popping the intermediate tensor from it. The
       values of this dict need to be added to the list of outputs.
   """
 
-  def __init__(self, name, forward_graph):
+  def __init__(self, name, forward_cond_graph, forward_body_graph, max_iters):
     super(_WhileBodyGradFuncGraph, self).__init__(name)
+    self.empty_tensor_lists = []
     self.popped_tensor_lists = {}
     # FuncGraph for the body of the forward While op.
-    self._forward_graph = forward_graph
+    self._forward_graph = forward_body_graph
+    # FuncGraph for the cond of the forward While op.
+    self._forward_cond_graph = forward_cond_graph
+    self._maximum_iterations = max_iters
     # Dict from forward intermediate tensor to its indirectly captured tensor
     # in this graph. Indirect capturing happens in two ways:
     # 1. For non-resource tensors we capture their accumulators from the forward
@@ -663,13 +684,10 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
     # 2. For resource tensors we directly capture their corresponding tensor
     #    in the forward outer graph.
     self._indirect_captures = {}
-    # Dict from forward graph tensor to its corresponding tensor in
-    # `forward_graph.outer_graph`. For a non-resource tensor the value is the
-    # forward While op's "output" corresponding its accumulator. For a resource
-    # tensor it is the While op's "input" for the resource. Note: We disallow
-    # creation of resources inside the while loop so if a resource tensor exists
-    # inside while loop it must be a loop input.
-    self._inner_to_outer_tensor = {}
+
+  @property
+  def while_op_needs_rewrite(self):
+    return self.empty_tensor_lists
 
   def capture(self, tensor, name=None, whitelisted=False):
     """Selectively captures external tensors.
@@ -708,10 +726,6 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
 
     captured_tensor = self._indirect_captures.get(tensor)
     if captured_tensor is not None:
-      # For GradientTape housekeeping.
-      assert self._inner_to_outer_tensor[tensor] in self.captures
-      super(_WhileBodyGradFuncGraph, self)._capture_helper(
-          self._inner_to_outer_tensor[tensor], name)
       return captured_tensor
 
     if tensor.dtype == dtypes.resource:
@@ -736,36 +750,47 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
           index], "Resource tensors must be loop invariants %s." % str(
               self._forward_graph._while.inputs[index])
       tensor_in_outer_graph = self._forward_graph._while.inputs[index]
-      self._inner_to_outer_tensor[tensor] = tensor_in_outer_graph
       self._indirect_captures[tensor] = self.capture(
           tensor_in_outer_graph, whitelisted=True)
       return self._indirect_captures[tensor]
 
-    assert tensor not in self._inner_to_outer_tensor
-
-    accumulator = None
-
-    # Find the TensorList that was used to accumulate the tensors of this
-    # intermediate tensor.
+    # Create or find an existing accumulator output for `tensor` in the forward
+    # graph, and fetch from this accumulator in the gradient graph to get the
+    # raw intermediate value.
     accumulator = _get_accumulator(tensor)
     if accumulator is None:
-      raise ValueError("Reference to un-accumulated intermediate tensor: ",
-                       tensor.name)
-    assert accumulator.graph == self._forward_graph
-    # Get the While op output corresponding to the accumulator.
-    accumulator = self._forward_graph._while.outputs[self._forward_graph.outputs
-                                                     .index(accumulator)]
-
-    assert accumulator.graph == self._forward_graph.outer_graph
-    self._inner_to_outer_tensor[tensor] = accumulator
-
-    # Capture the `accumulator`.
-    accumulator_ph = super(_WhileBodyGradFuncGraph, self)._capture_helper(
+      # Create the initial empty tensor list.
+      with self._forward_graph.outer_graph.as_default():
+        tensor_list = list_ops.empty_tensor_list(
+            element_dtype=tensor.dtype, element_shape=tensor.shape,
+            max_num_elements=self._maximum_iterations)
+      self.empty_tensor_lists.append(tensor_list)
+
+      # Push the intermediate tensor to the tensor list. This captures
+      # `tensor_list`.
+      with self._forward_graph.as_default():
+        accumulator = list_ops.tensor_list_push_back(tensor_list, tensor)
+      # Add the modified tensor list to the list of outputs. This output will be
+      # all the accumulated values.
+      self._forward_graph.outputs.append(accumulator)
+
+      # Capture in the cond graph as well so the forward cond and body inputs
+      # match.
+      with self._forward_cond_graph.as_default():
+        self._forward_cond_graph.capture(tensor_list)
+
+    # Capture the accumulator tensor list in the gradient graph directly from
+    # the forward graph -- we'll later modify this to capture the final list
+    # output by the forward While op instead.
+    captured_accumulator = super(_WhileBodyGradFuncGraph, self)._capture_helper(
         accumulator, name)
+
+    # Pop the intermediate value from the tensor list in the gradient graph.
     new_tensor_list, captured_tensor = list_ops.tensor_list_pop_back(
-        accumulator_ph, element_dtype=tensor.dtype)
+        captured_accumulator, element_dtype=tensor.dtype)
+
     self._indirect_captures[tensor] = captured_tensor
-    self.popped_tensor_lists[accumulator_ph] = new_tensor_list
+    self.popped_tensor_lists[captured_accumulator] = new_tensor_list
     return captured_tensor
 
 
-- 
GitLab


From b08c981d1d5556fb384cbbadcdc0d7c373324300 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 5 Dec 2018 14:36:40 -0800
Subject: [PATCH 1386/1554] Remove colocate_with in keras layer regularization.

PiperOrigin-RevId: 224223391
---
 tensorflow/python/keras/engine/base_layer.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 8e35300342..858fa76472 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -1311,9 +1311,8 @@ class Layer(checkpointable.CheckpointableBase):
 
     def _loss_for_variable(v):
       """Creates a regularization loss `Tensor` for variable `v`."""
-      with ops.colocate_with(v):
-        with ops.name_scope(name + '/Regularizer'):
-          regularization = regularizer(v)
+      with ops.name_scope(name + '/Regularizer'):
+        regularization = regularizer(v)
       return regularization
 
     if isinstance(variable, tf_variables.PartitionedVariable):
-- 
GitLab


From 2b0fd9b66aa865742a9bcd1b05675e150cab0120 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Wed, 5 Dec 2018 14:47:25 -0800
Subject: [PATCH 1387/1554] Internal-only change.

PiperOrigin-RevId: 224225156
---
 .../python/distribute/cluster_resolver/tpu_cluster_resolver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 1956bd75a8..fe8e09f561 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -92,7 +92,8 @@ class TPUClusterResolver(ClusterResolver):
         self._tpu == compat.as_bytes('local') or
         self._tpu.startswith(compat.as_bytes('/bns')) or
         self._tpu.startswith(compat.as_bytes('localhost:')) or
-        self._tpu.startswith(compat.as_bytes('grpc://'))):
+        self._tpu.startswith(compat.as_bytes('grpc://')) or
+        self._tpu.startswith(compat.as_bytes('uptc://'))):
       return False
     return True
 
-- 
GitLab


From 45cfe7126619b79ce3d88234a775d0468faa48cb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 14:51:36 -0800
Subject: [PATCH 1388/1554] Internal Change

PiperOrigin-RevId: 224225849
---
 .../api_def/python_api/api_def_FloorDiv.pbtxt |   4 +-
 .../api_def/python_api/api_def_FloorMod.pbtxt |   7 +-
 .../api_def/python_api/api_def_RealDiv.pbtxt  |   4 +-
 .../python_api/api_def_TruncateDiv.pbtxt      |   4 +-
 .../python_api/api_def_TruncateMod.pbtxt      |   4 +-
 tensorflow/python/framework/python_op_gen.cc  |   4 +-
 tensorflow/python/ops/array_ops.py            |  11 +
 tensorflow/python/ops/clip_ops.py             |   2 +
 tensorflow/python/ops/math_ops.py             |  32 +-
 tensorflow/python/ops/ragged/BUILD            |  66 +--
 tensorflow/python/ops/ragged/__init__.py      | 124 ++---
 .../python/ops/ragged/ragged_array_ops.py     | 121 ++---
 .../python/ops/ragged/ragged_dispatch.py      | 441 ++++++++++++++++++
 ...se_ops_test.py => ragged_dispatch_test.py} | 246 +++++-----
 .../ops/ragged/ragged_elementwise_ops.py      | 389 ---------------
 .../ops/ragged/ragged_map_fn_op_test.py       |  11 +-
 .../python/ops/ragged/ragged_math_ops.py      | 111 +++--
 .../python/ops/ragged/ragged_operators.py     |  61 ++-
 tensorflow/python/ops/sparse_ops.py           |   1 +
 tensorflow/python/ops/string_ops.py           |  10 +
 tensorflow/python/util/dispatch.py            |   7 +-
 21 files changed, 896 insertions(+), 764 deletions(-)
 create mode 100644 tensorflow/python/ops/ragged/ragged_dispatch.py
 rename tensorflow/python/ops/ragged/{ragged_elementwise_ops_test.py => ragged_dispatch_test.py} (77%)
 delete mode 100644 tensorflow/python/ops/ragged/ragged_elementwise_ops.py

diff --git a/tensorflow/core/api_def/python_api/api_def_FloorDiv.pbtxt b/tensorflow/core/api_def/python_api/api_def_FloorDiv.pbtxt
index 26598ab1fb..efd42b888d 100644
--- a/tensorflow/core/api_def/python_api/api_def_FloorDiv.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_FloorDiv.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "FloorDiv"
-  visibility: HIDDEN
+  endpoint {
+    name: "floor_div"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_FloorMod.pbtxt b/tensorflow/core/api_def/python_api/api_def_FloorMod.pbtxt
index ef562e93a0..e5db6d49b2 100644
--- a/tensorflow/core/api_def/python_api/api_def_FloorMod.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_FloorMod.pbtxt
@@ -1,4 +1,9 @@
 op {
   graph_op_name: "FloorMod"
-  visibility: HIDDEN
+  endpoint {
+    name: "floormod"
+  }
+  endpoint {
+    name: "mod"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_RealDiv.pbtxt b/tensorflow/core/api_def/python_api/api_def_RealDiv.pbtxt
index bd87eef824..f9e01eb567 100644
--- a/tensorflow/core/api_def/python_api/api_def_RealDiv.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_RealDiv.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "RealDiv"
-  visibility: HIDDEN
+  endpoint {
+    name: "realdiv"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_TruncateDiv.pbtxt b/tensorflow/core/api_def/python_api/api_def_TruncateDiv.pbtxt
index 2a547f771c..8e46c5e663 100644
--- a/tensorflow/core/api_def/python_api/api_def_TruncateDiv.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_TruncateDiv.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "TruncateDiv"
-  visibility: HIDDEN
+  endpoint {
+    name: "truncatediv"
+  }
 }
diff --git a/tensorflow/core/api_def/python_api/api_def_TruncateMod.pbtxt b/tensorflow/core/api_def/python_api/api_def_TruncateMod.pbtxt
index 0731e8810e..97fb816a7a 100644
--- a/tensorflow/core/api_def/python_api/api_def_TruncateMod.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_TruncateMod.pbtxt
@@ -1,4 +1,6 @@
 op {
   graph_op_name: "TruncateMod"
-  visibility: HIDDEN
+  endpoint {
+    name: "truncatemod"
+  }
 }
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index d91f7b0bdd..d460168631 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -634,7 +634,9 @@ void GenEagerPythonOp::AddEagerFunctionTeardown(
 bool GenEagerPythonOp::AddEagerFastPathAndGraphCode(
     const string& parameters, const std::vector<string>& output_sizes,
     const string& eager_not_allowed_error) {
-  strings::StrAppend(&result_, "@_dispatch.add_dispatch_list\n");
+  if (api_def_.visibility() == ApiDef::VISIBLE) {
+    strings::StrAppend(&result_, "@_dispatch.add_dispatch_list\n");
+  }
   AddExport();
   AddDefLine(function_name_, parameters);
   AddDocStringDescription();
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 7b6242b1cd..b555f63ceb 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -56,6 +56,7 @@ _BaseSlice = slice
 
 
 @tf_export("identity")
+@dispatch.add_dispatch_support
 def identity(input, name=None):  # pylint: disable=redefined-builtin
   r"""Return a tensor with the same shape and contents as input.
 
@@ -139,6 +140,7 @@ def expand_dims(input, axis=None, name=None, dim=None):
 
 
 @tf_export("expand_dims", v1=[])
+@dispatch.add_dispatch_support
 def expand_dims_v2(input, axis, name=None):
   """Inserts a dimension of 1 into a tensor's shape.
 
@@ -941,6 +943,7 @@ def parallel_stack(values, name="parallel_stack"):
 
 
 @tf_export("stack")
+@dispatch.add_dispatch_support
 def stack(values, axis=0, name="stack"):
   """Stacks a list of rank-`R` tensors into one rank-`(R+1)` tensor.
 
@@ -1151,6 +1154,7 @@ def unstack(value, num=None, axis=0, name="unstack"):
 
 
 @tf_export("concat")
+@dispatch.add_dispatch_support
 def concat(values, axis, name="concat"):
   """Concatenates tensors along one dimension.
 
@@ -1328,6 +1332,7 @@ def boolean_mask(tensor, mask, name="boolean_mask", axis=None):
 
 
 @tf_export("boolean_mask", v1=[])
+@dispatch.add_dispatch_support
 def boolean_mask_v2(tensor, mask, axis=None, name="boolean_mask"):
   """Apply boolean mask to tensor.
 
@@ -1810,6 +1815,7 @@ def zeros(shape, dtype=dtypes.float32, name=None):
 
 
 @tf_export(v1=["zeros_like"])
+@dispatch.add_dispatch_support
 def zeros_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to zero.
 
@@ -1840,6 +1846,7 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True):
 
 
 @tf_export("zeros_like", v1=[])
+@dispatch.add_dispatch_support
 def zeros_like_v2(
     input,  # pylint: disable=redefined-builtin
     dtype=None,
@@ -1899,6 +1906,7 @@ def zeros_like_impl(tensor, dtype, name, optimize=True):
 
 
 @tf_export(v1=["ones_like"])
+@dispatch.add_dispatch_support
 def ones_like(tensor, dtype=None, name=None, optimize=True):
   """Creates a tensor with all elements set to 1.
 
@@ -1929,6 +1937,7 @@ def ones_like(tensor, dtype=None, name=None, optimize=True):
 
 
 @tf_export("ones_like", v1=[])
+@dispatch.add_dispatch_support
 def ones_like_v2(
     input,  # pylint: disable=redefined-builtin
     dtype=None,
@@ -3115,6 +3124,7 @@ def squeeze_v2(input, axis=None, name=None):
 
 
 @tf_export("where")
+@dispatch.add_dispatch_support
 def where(condition, x=None, y=None, name=None):
   """Return the elements, either from `x` or `y`, depending on the `condition`.
 
@@ -3234,6 +3244,7 @@ def gather(params, indices, validate_indices=None, name=None, axis=0):
 
 
 @tf_export("gather", v1=[])
+@dispatch.add_dispatch_support
 def gather_v2(params, indices, validate_indices=None, axis=0, name=None):
   return gather(params, indices, validate_indices=validate_indices, name=name,
                 axis=axis)
diff --git a/tensorflow/python/ops/clip_ops.py b/tensorflow/python/ops/clip_ops.py
index 82803ac351..a237cfff82 100644
--- a/tensorflow/python/ops/clip_ops.py
+++ b/tensorflow/python/ops/clip_ops.py
@@ -31,10 +31,12 @@ from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import numerics
 from tensorflow.python.util import deprecation
+from tensorflow.python.util import dispatch
 from tensorflow.python.util.tf_export import tf_export
 
 
 @tf_export("clip_by_value")
+@dispatch.add_dispatch_support
 def clip_by_value(t, clip_value_min, clip_value_max,
                   name=None):
   """Clips tensor values to a specified min and max.
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index f0d8bed508..e2b634ee8f 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -230,6 +230,7 @@ class DivideDelegateWithName(object):
 
 
 @tf_export("math.divide", "divide")
+@dispatch.add_dispatch_support
 def divide(x, y, name=None):
   """Computes Python style division of `x` by `y`."""
 
@@ -242,6 +243,7 @@ def divide(x, y, name=None):
 
 
 @tf_export("math.multiply", "multiply")
+@dispatch.add_dispatch_support
 def multiply(x, y, name=None):
   return gen_math_ops.mul(x, y, name)
 
@@ -262,6 +264,7 @@ _mul.__doc__ = (
 
 
 @tf_export("math.subtract", "subtract")
+@dispatch.add_dispatch_support
 def subtract(x, y, name=None):
   return gen_math_ops.sub(x, y, name)
 
@@ -347,6 +350,7 @@ def scalar_mul_v2(scalar, x, name=None):
 
 
 @tf_export("math.pow", "pow")
+@dispatch.add_dispatch_support
 def pow(x, y, name=None):  # pylint: disable=redefined-builtin
   r"""Computes the power of one value to another.
 
@@ -375,6 +379,7 @@ def pow(x, y, name=None):  # pylint: disable=redefined-builtin
 
 # pylint: disable=redefined-builtin,redefined-outer-name
 @tf_export("dtypes.complex", "complex")
+@dispatch.add_dispatch_support
 def complex(real, imag, name=None):
   r"""Converts two real numbers to a complex number.
 
@@ -418,6 +423,7 @@ def complex(real, imag, name=None):
 
 @tf_export("math.real", v1=["math.real", "real"])
 @deprecation.deprecated_endpoints("real")
+@dispatch.add_dispatch_support
 def real(input, name=None):
   r"""Returns the real part of a complex (or real) tensor.
 
@@ -450,6 +456,7 @@ def real(input, name=None):
 
 @tf_export("math.imag", v1=["math.imag", "imag"])
 @deprecation.deprecated_endpoints("imag")
+@dispatch.add_dispatch_support
 def imag(input, name=None):
   r"""Returns the imaginary part of a complex (or real) tensor.
 
@@ -481,6 +488,7 @@ def imag(input, name=None):
 
 @tf_export("math.angle", v1=["math.angle", "angle"])
 @deprecation.deprecated_endpoints("angle")
+@dispatch.add_dispatch_support
 def angle(input, name=None):
   r"""Returns the element-wise argument of a complex (or real) tensor.
 
@@ -520,6 +528,7 @@ def angle(input, name=None):
 
 
 @tf_export("math.round", "round")
+@dispatch.add_dispatch_support
 def round(x, name=None):  # pylint: disable=redefined-builtin
   """Rounds the values of a tensor to the nearest integer, element-wise.
 
@@ -547,6 +556,7 @@ def round(x, name=None):  # pylint: disable=redefined-builtin
 
 
 @tf_export("dtypes.cast", "cast")
+@dispatch.add_dispatch_support
 def cast(x, dtype, name=None):
   """Casts a tensor to a new type.
 
@@ -610,6 +620,7 @@ def cast(x, dtype, name=None):
 
 
 @tf_export("dtypes.saturate_cast", "saturate_cast")
+@dispatch.add_dispatch_support
 def saturate_cast(value, dtype, name=None):
   """Performs a safe saturating cast of `value` to `dtype`.
 
@@ -935,6 +946,7 @@ def _div_python2(x, y, name=None):
 
 
 @tf_export("math.truediv", "truediv")
+@dispatch.add_dispatch_support
 def truediv(x, y, name=None):
   """Divides x / y elementwise (using Python 3 division operator semantics).
 
@@ -992,6 +1004,7 @@ def div(x, y, name=None):
 
 
 @tf_export("div_no_nan")
+@dispatch.add_dispatch_support
 def div_no_nan(x, y, name=None):
   """Computes an unsafe divide which returns 0 if the y is zero.
 
@@ -1021,6 +1034,7 @@ mod = gen_math_ops.floor_mod
 # TODO(aselle): Deprecate this once all internal functionality uses
 # tf.truncatediv
 @tf_export("math.floordiv", v1=["math.floordiv", "floordiv"])
+@dispatch.add_dispatch_support
 @deprecation.deprecated_endpoints("floordiv")
 def floordiv(x, y, name=None):
   """Divides `x / y` elementwise, rounding toward the most negative integer.
@@ -1050,16 +1064,11 @@ def floordiv(x, y, name=None):
 
 
 realdiv = gen_math_ops.real_div
-tf_export("realdiv")(realdiv)
 truncatediv = gen_math_ops.truncate_div
-tf_export("truncatediv")(truncatediv)
 # TODO(aselle): Rename this to floordiv when we can.
 floor_div = gen_math_ops.floor_div
-tf_export("floor_div")(floor_div)
 truncatemod = gen_math_ops.truncate_mod
-tf_export("truncatemod")(truncatemod)
 floormod = gen_math_ops.floor_mod
-tf_export("floormod", "mod")(floormod)
 
 
 def _mul_dispatch(x, y, name=None):
@@ -1095,6 +1104,7 @@ _OverrideBinaryOperatorHelper(pow, "pow")
 
 
 @tf_export("math.logical_xor", v1=["math.logical_xor", "logical_xor"])
+@dispatch.add_dispatch_support
 @deprecation.deprecated_endpoints("logical_xor")
 def logical_xor(x, y, name="LogicalXor"):
   """x ^ y = (x | y) & ~(x & y)."""
@@ -1277,6 +1287,7 @@ def reduce_sum_v1(input_tensor,
 
 
 @tf_export("math.reduce_sum", "reduce_sum", v1=[])
+@dispatch.add_dispatch_support
 def reduce_sum(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the sum of elements across dimensions of a tensor.
 
@@ -1524,6 +1535,7 @@ def reduce_mean_v1(input_tensor,
 
 
 @tf_export("math.reduce_mean", "reduce_mean", v1=[])
+@dispatch.add_dispatch_support
 def reduce_mean(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the mean of elements across dimensions of a tensor.
 
@@ -1675,6 +1687,7 @@ def reduce_std(input_tensor, axis=None, keepdims=False, name=None):
 
 
 @tf_export("math.reduce_prod", "reduce_prod", v1=[])
+@dispatch.add_dispatch_support
 def reduce_prod(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the product of elements across dimensions of a tensor.
 
@@ -1796,6 +1809,7 @@ def reduce_min_v1(input_tensor,
 
 
 @tf_export("math.reduce_min", "reduce_min", v1=[])
+@dispatch.add_dispatch_support
 def reduce_min(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the minimum of elements across dimensions of a tensor.
 
@@ -1874,6 +1888,7 @@ def reduce_max_v1(input_tensor,
 
 
 @tf_export("math.reduce_max", "reduce_max", v1=[])
+@dispatch.add_dispatch_support
 def reduce_max(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the maximum of elements across dimensions of a tensor.
 
@@ -1961,6 +1976,7 @@ def reduce_all_v1(input_tensor,
 
 
 @tf_export("reduce_all", "math.reduce_all", v1=[])
+@dispatch.add_dispatch_support
 def reduce_all(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the "logical and" of elements across dimensions of a tensor.
 
@@ -2057,6 +2073,7 @@ def reduce_any_v1(input_tensor,
 
 
 @tf_export("math.reduce_any", "reduce_any", v1=[])
+@dispatch.add_dispatch_support
 def reduce_any(input_tensor, axis=None, keepdims=False, name=None):
   """Computes the "logical or" of elements across dimensions of a tensor.
 
@@ -2619,6 +2636,7 @@ def _as_indexed_slices_list(inputs, optimize=True):
 
 
 @tf_export("math.add_n", "add_n")
+@dispatch.add_dispatch_support
 def add_n(inputs, name=None):
   """Adds all input tensors element-wise.
 
@@ -2764,6 +2782,7 @@ def sigmoid(x, name=None):
 
 
 @tf_export("math.log_sigmoid", v1=["math.log_sigmoid", "log_sigmoid"])
+@dispatch.add_dispatch_support
 @deprecation.deprecated_endpoints("log_sigmoid")
 def log_sigmoid(x, name=None):
   """Computes log sigmoid of `x` element-wise.
@@ -2973,6 +2992,7 @@ def cumprod(x, axis=0, exclusive=False, reverse=False, name=None):
 
 
 @tf_export("math.conj", v1=["math.conj", "conj"])
+@dispatch.add_dispatch_support
 @deprecation.deprecated_endpoints("conj")
 def conj(x, name=None):
   r"""Returns the complex conjugate of a complex number.
@@ -3077,6 +3097,7 @@ def _unsorted_segment_N(data, segment_ids, num_segments):
     "math.unsorted_segment_mean",
     v1=["math.unsorted_segment_mean", "unsorted_segment_mean"])
 @deprecation.deprecated_endpoints("unsorted_segment_mean")
+@dispatch.add_dispatch_support
 def unsorted_segment_mean(data, segment_ids, num_segments, name=None):
   r"""Computes the mean along segments of a tensor.
 
@@ -3122,6 +3143,7 @@ def unsorted_segment_mean(data, segment_ids, num_segments, name=None):
     "math.unsorted_segment_sqrt_n",
     v1=["math.unsorted_segment_sqrt_n", "unsorted_segment_sqrt_n"])
 @deprecation.deprecated_endpoints("unsorted_segment_sqrt_n")
+@dispatch.add_dispatch_support
 def unsorted_segment_sqrt_n(data, segment_ids, num_segments, name=None):
   r"""Computes the sum along segments of a tensor divided by the sqrt(N).
 
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index e335c5cb6f..fcd9adad21 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -25,7 +25,7 @@ py_library(
     deps = [
         ":ragged_array_ops",
         ":ragged_conversion_ops",
-        ":ragged_elementwise_ops",
+        ":ragged_dispatch",
         ":ragged_factory_ops",
         ":ragged_functional_ops",
         ":ragged_getitem",
@@ -150,33 +150,14 @@ py_library(
     ],
 )
 
-py_library(
-    name = "ragged_elementwise_ops",
-    srcs = ["ragged_elementwise_ops.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":ragged_factory_ops",
-        ":ragged_tensor",
-        ":ragged_tensor_shape",
-        ":ragged_util",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:clip_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:string_ops",
-        "//tensorflow/python:util",
-    ],
-)
-
 py_library(
     name = "ragged_operators",
     srcs = ["ragged_operators.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_elementwise_ops",
         ":ragged_getitem",
         ":ragged_tensor",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:util",
     ],
 )
@@ -186,12 +167,13 @@ py_library(
     srcs = ["ragged_string_ops.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_conversion_ops",
         ":ragged_factory_ops",
         ":ragged_tensor",
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:math_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:string_ops",
         "//tensorflow/python:util",
     ],
 )
@@ -219,10 +201,11 @@ py_library(
         ":ragged_tensor",
         ":ragged_util",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
-        "//tensorflow/python:tensor_shape",
         "//tensorflow/python:tensor_util",
     ],
 )
@@ -285,6 +268,29 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_dispatch",
+    srcs = ["ragged_dispatch.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_factory_ops",
+        ":ragged_math_ops",
+        ":ragged_tensor",
+        ":ragged_tensor_shape",
+        ":ragged_util",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:string_ops",
+        "//tensorflow/python:util",
+        "//third_party/py/numpy",
+    ],
+)
+
 #-------------------------------------------------------------------------------
 # RaggedTensor Tests
 #-------------------------------------------------------------------------------
@@ -458,6 +464,7 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:gradients_impl",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
 )
@@ -684,17 +691,21 @@ py_test(
 )
 
 py_test(
-    name = "ragged_elementwise_ops_test",
-    srcs = ["ragged_elementwise_ops_test.py"],
+    name = "ragged_dispatch_test",
+    srcs = ["ragged_dispatch_test.py"],
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:clip_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform_test",
+        "//tensorflow/python:string_ops",
         "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
@@ -725,6 +736,7 @@ py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:string_ops",
         "//tensorflow/python/keras:backend",
+        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
 )
@@ -735,8 +747,10 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
+        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index 1b2a7be95f..bfcaa366fc 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -1,76 +1,53 @@
 """Ragged Tensors.
 
-This package defines the [`RaggedTensor`](ragged/RaggedTensor.md) class, which
+This package defines the `tf.RaggedTensor` class, which
 represents tensors with non-uniform shapes.  In particular, each `RaggedTensor`
 has one or more *ragged dimensions*, which are dimensions whose slices may have
 different lengths.  For example, the inner (column) dimension of
 `rt=[[3, 1, 4, 1], [], [5, 9, 2], [6], []]` is ragged, since the column slices
 (`rt[0, :]`, ..., `rt[4, :]`) have different lengths.  For a more detailed
-description of ragged tensors, see the [`RaggedTensor`](ragged/RaggedTensor.md)
+description of ragged tensors, see the `tf.RaggedTensor`
 class documentation.
 
-## RaggedTensor Operations
-
-This package also defines a collection of operations for manipulating
-ragged tensors.
-
-### RaggedTensor Versions of Standard Tensor Operations
-
-Many of the operations defined by this package are analogous to
-[`Tensor`](https://www.tensorflow.org/api_docs/python/tf/Tensor)
-operations, but they accept `RaggedTensor`s as input and can return
-`RaggedTensor`s as output.  For example, `ragged.add` performs elementwise
-addition just like `tf.add`, but can be used on `RaggedTensor`s.
-
-These `RaggedTensor` versions of the standard `Tensor` operations can also be
-used with standard `Tensors`; and for the most part, they will return the same
-value that the standard `Tensor` operation would return.  However, there are
-a few notable exceptions:
-
-* For [`ragged.stack(...)`](ragged/stack.md) and
-  [`ragged.concat(...)`](ragged/concat.md), the input tensors are not required
-  to have matching shapes.  In the returned tensor, all dimensions up to the
-  `axis` dimension will be ragged.
-
-### Ragged-Tensor Specific Operations
-
-The following operations are specific to ragged tensors:
-
-* **Factory ops**:
-  [`constant(...)`](ragged/constant.md),
-  [`from_row_splits(...)`](ragged/from_row_splits.md),
-  [`from_row_lengths(...)`](ragged/from_row_lengths.md),
-  [`from_row_starts(...)`](ragged/from_row_starts.md),
-  [`from_row_limits(...)`](ragged/from_row_limits.md),
-  [`from_value_rowids(...)`](ragged/from_value_rowids.md),
-  [`from_nested_row_splits(...)`](ragged/from_nested_row_splits.md),
-  [`from_nested_value_rowids(...)`](ragged/from_nested_value_rowids.md).
-
-* **Conversion ops**:
-  [`from_tensor(...)`](ragged/from_tensor.md),
-  [`to_tensor(...)`](ragged/to_tensor.md),
-  [`from_sparse(...)`](ragged/from_sparse.md),
-  [`to_sparse(...)`](ragged/to_sparse.md),
-  [`from_variant(...)`](ragged/from_variant.md),
-  [`to_variant(...)`](ragged/to_variant.md),
-  [`convert_to_tensor_or_ragged_tensor(...)`](
-  ragged/convert_to_tensor_or_ragged_tensor.md).
-
-* **Shape ops**:
-  [`row_splits(...)`](ragged/row_splits.md),
-  [`row_lengths(...)`](ragged/row_lengths.md),
-  [`row_starts(...)`](ragged/row_starts.md),
-  [`row_limits(...)`](ragged/row_limits.md),
-  [`value_rowids(...)`](ragged/value_rowids.md),
-  [`nrows(...)`](ragged/nrows.md),
-  [`nested_row_splits(...)`](ragged/nested_row_splits.md),
-  [`row_splits_to_segment_ids(...)`](ragged/row_splits_to_segment_ids.md),
-  [`segment_ids_to_row_splits(...)`](ragged/segment_ids_to_row_splits.md),
-  [`bounding_shape(...)`](ragged/bounding_shape.md).
-
-* **Functional ops**:
-  [`map_inner_values(...)`](ragged/map_inner_values.md),
-  [`make_elementwise_op(...)`](ragged/make_elementwise_op.md).
+## `RaggedTensor` Operations
+
+### `RaggedTensor` Factory ops
+
+* `tf.ragged.constant`
+* `tf.ragged.from_row_splits`
+* `tf.ragged.from_row_splits`
+* `tf.ragged.from_row_lengths`
+* `tf.ragged.from_row_starts`
+* `tf.ragged.from_row_limits`
+* `tf.ragged.from_value_rowids`
+* `tf.ragged.from_nested_row_splits`
+* `tf.ragged.from_nested_value_rowids`
+
+### `RaggedTensor` Conversion ops
+
+* `tf.ragged.from_tensor`
+* `tf.ragged.to_tensor`
+* `tf.ragged.from_sparse`
+* `tf.ragged.to_sparse`
+* `tf.ragged.from_variant`
+* `tf.ragged.to_variant`
+* `tf.ragged.convert_to_tensor_or_ragged_tensor`
+
+### `RaggedTensor` Shape ops
+
+* `tf.ragged.row_splits`
+* `tf.ragged.row_lengths`
+* `tf.ragged.row_starts`
+* `tf.ragged.row_limits`
+* `tf.ragged.value_rowids`
+* `tf.ragged.nrows`
+* `tf.ragged.nested_row_splits`
+* `tf.ragged.row_splits_to_segment_ids`
+* `tf.ragged.segment_ids_to_row_splits`
+* `tf.ragged.bounding_shape`
+
+### Functional ops
+* `tf.ragged.map_inner_values`
 
 
 <!-- Ragged Classes & related helper functions -->
@@ -140,21 +117,17 @@ The following operations are specific to ragged tensors:
 @@map_inner_values
 @@map_fn
 
-<!-- Elementwise Ops -->
-@@make_elementwise_op
-
 <!-- Shape & broadcasting -->
 @@RaggedTensorDynamicShape
 @@broadcast_to
 @@broadcast_dynamic_shape
-
-<!-- Symbols from  ragged_elementwise_ops._symbols_to_export are whitelisted -->
 """
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.ops.ragged import ragged_dispatch
 from tensorflow.python.ops.ragged import ragged_operators
 from tensorflow.python.ops.ragged import ragged_string_ops
 
@@ -179,11 +152,6 @@ from tensorflow.python.ops.ragged.ragged_conversion_ops import from_tensor
 from tensorflow.python.ops.ragged.ragged_conversion_ops import to_sparse
 from tensorflow.python.ops.ragged.ragged_conversion_ops import to_tensor
 
-# pylint: disable=protected-access, wildcard-import
-from tensorflow.python.ops.ragged.ragged_elementwise_ops import *
-from tensorflow.python.ops.ragged.ragged_elementwise_ops import _symbols_to_export as _elementwise_ops
-# pylint: enable=protected-access, wildcard-import
-
 from tensorflow.python.ops.ragged.ragged_factory_ops import constant
 from tensorflow.python.ops.ragged.ragged_factory_ops import constant_value
 from tensorflow.python.ops.ragged.ragged_factory_ops import convert_to_tensor_or_ragged_tensor
@@ -231,6 +199,10 @@ from tensorflow.python.ops.ragged.segment_id_ops import segment_ids_to_row_split
 
 from tensorflow.python.util import all_util as _all_util
 
+
+# Register OpDispatchers that override standard TF ops to work w/ RaggedTensors.
+__doc__ += ragged_dispatch.register_dispatchers()  # pylint: disable=redefined-builtin
+
 # Any symbol that is not referenced (with "@@name") in the module docstring
-# above, or included in the "_elementwise_ops" whitelist, will be removed.
-_all_util.remove_undocumented(__name__, _elementwise_ops)
+# above will be removed.
+_all_util.remove_undocumented(__name__)
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index 603e39d1dc..25317ba93e 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -308,7 +308,7 @@ def bounding_shape(rt_input, axis=None, name=None):
 # ragged_gather
 #===============================================================================
 # TODO(edloper): Add an `axis` argument
-def gather(params, indices, name=None):
+def gather(params, indices, validate_indices=None, axis=0, name=None):
   """Gathers ragged slices from `params` axis `0` according to `indices`.
 
   Returns `RaggedTensor` output, such that:
@@ -347,6 +347,8 @@ def gather(params, indices, name=None):
     indices: The potentially ragged tensor indicating which values to gather.
       Must have dtype `int32` or `int64`.  Values must be in the range `[0,
       params.shape[0]]`.
+    validate_indices: Ignored.
+    axis: Must be zero.
     name: A name for the operation (optional).
 
   Returns:
@@ -357,6 +359,9 @@ def gather(params, indices, name=None):
   Raises:
     ValueError: If indices.shape.ndims is not known statically.
   """
+  del validate_indices
+  if not isinstance(axis, int) or axis != 0:
+    raise ValueError('axis>0 is not supported for ragged gather yet.')
   with ops.name_scope(name, 'RaggedGather', [params, indices]):
     params = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
         params, name='params')
@@ -812,29 +817,29 @@ def boolean_mask(data, mask, keepdims=False, name=None):
 #===============================================================================
 # Concatenation and Stacking
 #===============================================================================
-def concat(rt_inputs, axis, name=None):
+def concat(values, axis, name=None):
   """Concatenates potentially ragged tensors along one dimension.
 
   Given a list of tensors with the same rank `K` (`K >= axis`), returns a
   rank-`K` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
-  concatenation of `[rt[i0...iaxis] for rt in rt_inputs]`.
+  concatenation of `[rt[i0...iaxis] for rt in values]`.
 
   Args:
-    rt_inputs: A list of potentially ragged tensors.  May not be empty. All
-      `rt_inputs` must have the same rank and the same dtype; but unlike
+    values: A list of potentially ragged tensors.  May not be empty. All
+      `values` must have the same rank and the same dtype; but unlike
       `tf.concat`, they can have arbitrary shapes.
     axis: A python integer, indicating the dimension along which to concatenate.
       (Note: Unlike `tf.concat`, the `axis` parameter must be statically known.)
         Negative values are supported only if the rank of at least one
-        `rt_inputs` value is statically known.
+        `values` value is statically known.
     name: A name prefix for the returned tensor (optional).
 
   Returns:
     A `RaggedTensor` with rank `K`.
-    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in rt_inputs]))`.
+    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
 
   Raises:
-    ValueError: If `rt_inputs` is empty, if `axis` is out of bounds or if
+    ValueError: If `values` is empty, if `axis` is out of bounds or if
       the input tensors have different ranks.
 
   #### Example:
@@ -847,35 +852,35 @@ def concat(rt_inputs, axis, name=None):
     [[1, 2, 6], [3, 4, 5, 7, 8, 9]]
     ```
   """
-  if not isinstance(rt_inputs, (list, tuple)):
-    rt_inputs = [rt_inputs]
-  with ops.name_scope(name, 'RaggedConcat', rt_inputs):
-    return _ragged_stack_concat_helper(rt_inputs, axis, stack_values=False)
+  if not isinstance(values, (list, tuple)):
+    values = [values]
+  with ops.name_scope(name, 'RaggedConcat', values):
+    return _ragged_stack_concat_helper(values, axis, stack_values=False)
 
 
-def stack(rt_inputs, axis, name=None):
+def stack(values, axis, name=None):
   """Stacks potentially ragged tensors along one dimension.
 
   Given a list of tensors with the same rank `K` (`K >= axis`), returns a
   rank-`K+1` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
-  list `[rt[i0...iaxis] for rt in rt_inputs]`.
+  list `[rt[i0...iaxis] for rt in values]`.
 
   Args:
-    rt_inputs: A list of potentially ragged tensors.  May not be empty. All
-      `rt_inputs` must have the same rank and the same dtype; but unlike
+    values: A list of potentially ragged tensors.  May not be empty. All
+      `values` must have the same rank and the same dtype; but unlike
       `tf.concat`, they can have arbitrary shapes.
     axis: A python integer, indicating the dimension along which to stack.
       (Note: Unlike `tf.stack`, the `axis` parameter must be statically known.)
         Negative values are supported only if the rank of at least one
-        `rt_inputs` value is statically known.
+        `values` value is statically known.
     name: A name prefix for the returned tensor (optional).
 
   Returns:
     A `RaggedTensor` with rank `K+1`.
-    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in rt_inputs]))`.
+    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
 
   Raises:
-    ValueError: If `rt_inputs` is empty, if `axis` is out of bounds or if
+    ValueError: If `values` is empty, if `axis` is out of bounds or if
       the input tensors have different ranks.
 
   #### Example:
@@ -888,10 +893,10 @@ def stack(rt_inputs, axis, name=None):
     [[[1, 2], [6]], [[3, 4, 5], [7, 8, 9]]]
     ```
   """
-  if not isinstance(rt_inputs, (list, tuple)):
-    rt_inputs = [rt_inputs]
-  with ops.name_scope(name, 'RaggedConcat', rt_inputs):
-    return _ragged_stack_concat_helper(rt_inputs, axis, stack_values=True)
+  if not isinstance(values, (list, tuple)):
+    values = [values]
+  with ops.name_scope(name, 'RaggedConcat', values):
+    return _ragged_stack_concat_helper(values, axis, stack_values=True)
 
 
 def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
@@ -1065,22 +1070,22 @@ def _copy_row_shape(rt_inputs, splits):
 #===============================================================================
 # Tiling
 #===============================================================================
-def tile(rt_input, multiples, name=None):
+def tile(input, multiples, name=None):  # pylint: disable=redefined-builtin
   """Constructs a `RaggedTensor` by tiling a given `RaggedTensor`.
 
-  The values of `rt_input` are replicated `multiples[i]` times along the
+  The values of `input` are replicated `multiples[i]` times along the
   `i`th dimension (for each dimension `i`).  For every dimension `axis` in
-  `rt_input`, the length of each output element in that dimension is the
+  `input`, the length of each output element in that dimension is the
   length of corresponding input element multiplied by `multiples[axis]`.
 
   Args:
-    rt_input: A `RaggedTensor`.
+    input: A `RaggedTensor`.
     multiples: A 1-D integer `Tensor`.  Length must be the same as the number of
-      dimensions in `rt_input`.
+      dimensions in `input`.
     name: A name for the operation (optional).
 
   Returns:
-    A `RaggedTensor` with the same type, rank, and ragged_rank as `rt_input`.
+    A `RaggedTensor` with the same type, rank, and ragged_rank as `input`.
 
   #### Example:
     ```python
@@ -1089,22 +1094,22 @@ def tile(rt_input, multiples, name=None):
     [[1, 2, 1, 2], [3, 3], [1, 2, 1, 2], [3, 3], [1, 2, 1, 2], [3, 3]]
     ```
   """
-  with ops.name_scope(name, 'RaggedTile', [rt_input, multiples]):
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        rt_input, name='rt_input')
+  with ops.name_scope(name, 'RaggedTile', [input, multiples]):
+    input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+        input, name='input')
     multiples = ragged_util.convert_to_int_tensor(
         multiples, name='multiples', dtype=dtypes.int64)
     multiples.shape.assert_has_rank(1)
-    if not ragged_tensor.is_ragged(rt_input):
-      return array_ops.tile(rt_input, multiples, name)
+    if not ragged_tensor.is_ragged(input):
+      return array_ops.tile(input, multiples, name)
 
     # If the constant value of `multiples` is available, then we can use it
     # to skip tiling dimensions where `multiples=1`.
     const_multiples = tensor_util.constant_value(multiples)
 
     return ragged_factory_ops.from_nested_row_splits(
-        _tile_ragged_values(rt_input, multiples, const_multiples),
-        _tile_ragged_splits(rt_input, multiples, const_multiples))
+        _tile_ragged_values(input, multiples, const_multiples),
+        _tile_ragged_splits(input, multiples, const_multiples))
 
 
 def _tile_ragged_values(rt_input, multiples, const_multiples=None):
@@ -1240,26 +1245,26 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None):
 #===============================================================================
 
 
-def expand_dims(rt_input, axis, name=None):
+def expand_dims(input, axis, name=None):  # pylint: disable=redefined-builtin
   """Inserts a dimension with shape 1 into a potentially ragged tensor's shape.
 
-  Given a potentially ragged tenor `rt_input`, this operation inserts a
-  dimension with size 1 at the dimension `axis` of `rt_input`'s shape.
+  Given a potentially ragged tenor `input`, this operation inserts a
+  dimension with size 1 at the dimension `axis` of `input`'s shape.
 
-  * If `rt_input` is a `Tensor`, then this is equivalent to
+  * If `input` is a `Tensor`, then this is equivalent to
     `tf.expand_dims`.
-  * If `rt_input` is ragged, and `axis=0`, then the new dimension will be
+  * If `input` is ragged, and `axis=0`, then the new dimension will be
     uniform; but the previously outermost dimension will become ragged.
-  * If `rt_input` is ragged, and `0 < axis < rt_input.ragged_rank`, then the
+  * If `input` is ragged, and `0 < axis < input.ragged_rank`, then the
     new dimension will be ragged.
-  * If `rt_input` is ragged, and axis >= rt_input.ragged_rank`, then the new
+  * If `input` is ragged, and axis >= input.ragged_rank`, then the new
     dimension will be uniform.
 
   The following table gives some examples showing how `ragged.expand_dims`
   impacts the shapes of different input tensors.  Ragged dimensions are
   indicated by enclosing them in parentheses.
 
-  rt_input.shape          | axis | result.shape
+  input.shape             | axis | result.shape
   ----------------------- | ---- | -----------------------------
   `[D1, D2]`              |  `0` | `[1, D1, D2]`
   `[D1, D2]`              |  `1` | `[D1, 1, D2]`
@@ -1271,14 +1276,14 @@ def expand_dims(rt_input, axis, name=None):
   `[D1, (D2), (D3), D4]`  |  `4` | `[D1, (D2), (D3), D4, 1]`
 
   Args:
-    rt_input: The potentially tensor that should be expanded with a new
+    input: The potentially tensor that should be expanded with a new
       dimension.
     axis: An integer constant indicating where the new dimension should be
       inserted.
     name: A name for the operation (optional).
 
   Returns:
-    A tensor with the same values as `rt_input`, with an added dimension of
+    A tensor with the same values as `input`, with an added dimension of
     size 1 at `axis`.
 
   #### Examples:
@@ -1300,24 +1305,24 @@ def expand_dims(rt_input, axis, name=None):
     TensorShape([2, None, 1]) [[[1], [2]], [[3]]]
     ```
   """
-  with ops.name_scope(name, 'RaggedExpandDims', [rt_input]):
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        rt_input, name='rt_input')
+  with ops.name_scope(name, 'RaggedExpandDims', [input]):
+    input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+        input, name='input')
 
-    if not ragged_tensor.is_ragged(rt_input):
-      return array_ops.expand_dims(rt_input, axis)
+    if not ragged_tensor.is_ragged(input):
+      return array_ops.expand_dims(input, axis)
 
-    ndims = None if rt_input.shape.ndims is None else rt_input.shape.ndims + 1
+    ndims = None if input.shape.ndims is None else input.shape.ndims + 1
     axis = ragged_util.get_positive_axis(axis, ndims)
     if axis == 0:
-      values = rt_input
-      splits = array_ops.stack([0, nrows(rt_input)])
+      values = input
+      splits = array_ops.stack([0, nrows(input)])
     elif axis == 1:
-      values = rt_input
-      splits = math_ops.range(nrows(rt_input) + 1)
+      values = input
+      splits = math_ops.range(nrows(input) + 1)
     else:
-      values = expand_dims(rt_input.values, axis - 1)
-      splits = rt_input.row_splits
+      values = expand_dims(input.values, axis - 1)
+      splits = input.row_splits
 
     return ragged_factory_ops.from_row_splits(values, splits)
 
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
new file mode 100644
index 0000000000..7f44ac2ec1
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -0,0 +1,441 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Operator dispatch for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import numpy as np
+
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import string_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_math_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_tensor_shape
+from tensorflow.python.ops.ragged import ragged_util
+from tensorflow.python.util import dispatch
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_export
+from tensorflow.python.util import tf_inspect
+
+# @TODO(edloper): Set this to True in the CL that exports RaggedTensors.
+_UPDATE_DOCSTRINGS = False
+
+# Information about an argument to an operation: The name of the argument, its
+# position in the argument list, and a boolean flag indicating whether it
+# expects a list of tensors.
+_ArgInfo = collections.namedtuple('ArgInfo', ['name', 'position', 'is_list'])
+
+
+def _get_arg_infos(func, arg_names):
+  """Returns an `_ArgInfo` for each argument of `func` specified by `arg_names`.
+
+  Args:
+    func: The function whose arguments should be described.
+    arg_names: The names of the arguments to get info for.
+
+  Returns:
+    A tuple of `_ArgInfo`s.
+  """
+  arg_infos = []
+
+  # Inspect the func's argspec to find the position of each arg.
+  arg_spec = tf_inspect.getargspec(func)
+  for argname in arg_names:
+    assert isinstance(argname, str)
+    is_list = argname.startswith('[') and argname.endswith(']')
+    if is_list:
+      argname = argname[1:-1]
+    if argname not in arg_spec.args:
+      raise ValueError('Argument %r not found function in %s.  Args=%s' %
+                       (argname, func, arg_spec.args))
+    arg_infos.append(_ArgInfo(argname, arg_spec.args.index(argname), is_list))
+  return arg_infos
+
+
+def _is_convertible_to_tensor(value):
+  """Returns true if `value` is convertible to a `Tensor`."""
+  if isinstance(value,
+                (ops.Tensor, variables.Variable, np.ndarray, int, float, str)):
+    return True
+  elif isinstance(value, (sparse_tensor.SparseTensor,)):
+    return False
+  else:
+    try:
+      ops.convert_to_tensor(value)
+      return True
+    except (TypeError, ValueError):
+      return False
+
+
+class UnaryRaggedElementwiseDispatcher(dispatch.OpDispatcher):
+  """OpDispatcher for unary ops that map a base op across ragged values."""
+
+  def __init__(self, original_op, arg_is_list=False):
+    self._original_op = original_op
+    self._arg_is_list = arg_is_list
+    arg_names = tf_inspect.getfullargspec(original_op)[0]
+    self._x = arg_names[0]
+    if _UPDATE_DOCSTRINGS:
+      original_op.__doc__ = (
+          original_op.__doc__.rstrip() + '\n\n' +
+          '    `{x}` may be a `tf.RaggedTensor`.\n'.format(x=self._x))
+
+  def handle(self, args, kwargs):
+    if args:
+      x, args = args[0], args[1:]
+    else:
+      kwargs = kwargs.copy()
+      x = kwargs.pop(self._x, None)
+    if x is None:
+      return self.NOT_SUPPORTED
+    if self._arg_is_list:
+      found_ragged = False
+      for elt in x:
+        if ragged_tensor.is_ragged(elt):
+          found_ragged = True
+        elif not _is_convertible_to_tensor(elt):
+          return self.NOT_SUPPORTED
+      if found_ragged:
+        nested_splits_lists = [
+            elt.nested_row_splits for elt in x if ragged_tensor.is_ragged(elt)
+        ]
+        inner_values = [
+            elt.inner_values if ragged_tensor.is_ragged(elt) else elt
+            for elt in x
+        ]
+        with ops.control_dependencies(
+            ragged_util.assert_splits_match(nested_splits_lists)):
+          return ragged_factory_ops.from_nested_row_splits(
+              self._original_op(inner_values, *args, **kwargs),
+              nested_splits_lists[0])
+      else:
+        return self.NOT_SUPPORTED
+    else:
+      found_ragged = ragged_tensor.is_ragged(x)
+      if found_ragged:
+        mapped_values = self._original_op(x.inner_values, *args, **kwargs)
+        return x.with_inner_values(mapped_values)
+      else:
+        return self.NOT_SUPPORTED
+
+
+class BinaryRaggedElementwiseDispatcher(dispatch.OpDispatcher):
+  """OpDispatcher for binary ops that map a base op across ragged values.
+
+  Supports broadcasting.
+  """
+
+  def __init__(self, original_op):
+    self._original_op = original_op
+    arg_names = tf_inspect.getfullargspec(original_op)[0]
+    self._x = arg_names[0]
+    self._y = arg_names[1]
+    if _UPDATE_DOCSTRINGS:
+      original_op.__doc__ = (
+          original_op.__doc__.rstrip() + '\n\n' +
+          '    `{x}` and `{y}` may be a `tf.RaggedTensor`.\n'.format(
+              x=self._x, y=self._y))
+
+  def handle(self, args, kwargs):
+    # Extract the binary args.
+    if len(args) > 1:
+      x = args[0]
+      y = args[1]
+      args = args[2:]
+    elif args:
+      kwargs = kwargs.copy()
+      x = args[0]
+      y = kwargs.pop(self._y, None)
+      args = args[1:]
+    else:
+      kwargs = kwargs.copy()
+      x = kwargs.pop(self._x, None)
+      y = kwargs.pop(self._y, None)
+
+    # Bail if we don't have at least one ragged argument.
+    x_is_ragged = ragged_tensor.is_ragged(x)
+    y_is_ragged = ragged_tensor.is_ragged(y)
+    if not (x_is_ragged or y_is_ragged):
+      return self.NOT_SUPPORTED
+
+    # Convert args to tensors.  Bail if conversion fails.
+    try:
+      if not x_is_ragged:
+        x = ops.convert_to_tensor(x, name=self._x, preferred_dtype=y.dtype)
+      if not y_is_ragged:
+        y = ops.convert_to_tensor(y, name=self._y, preferred_dtype=x.dtype)
+    except (TypeError, ValueError):
+      return self.NOT_SUPPORTED
+
+    if ((x_is_ragged and y_is_ragged) or
+        (x_is_ragged and x.inner_values.shape.ndims <= y.shape.ndims) or
+        (y_is_ragged and y.inner_values.shape.ndims <= x.shape.ndims)):
+      bcast_shape = ragged_tensor_shape.broadcast_dynamic_shape(
+          ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(x),
+          ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(y))
+      x = ragged_tensor_shape.broadcast_to(
+          x, bcast_shape, broadcast_inner_dimensions=False)
+      y = ragged_tensor_shape.broadcast_to(
+          y, bcast_shape, broadcast_inner_dimensions=False)
+
+    x_values = x.inner_values if ragged_tensor.is_ragged(x) else x
+    y_values = y.inner_values if ragged_tensor.is_ragged(y) else y
+    mapped_values = self._original_op(x_values, y_values, *args, **kwargs)
+    if ragged_tensor.is_ragged(x):
+      return x.with_inner_values(mapped_values)
+    else:
+      return y.with_inner_values(mapped_values)
+
+
+class RaggedDispatcher(dispatch.OpDispatcher):
+  """OpDispatcher for ragged ops.
+
+  Dispatches to a wrapped op-handler if at least one of the `tensor_args`
+  arguments is a RaggedTensor or a RaggedTensorValue; and all of the
+  `tensor_args` arguments are convertible to Tensor or RaggedTensor.
+  """
+
+  def __init__(self, original_op, ragged_op, ragged_args):
+    op_arg_names = tf_inspect.getfullargspec(original_op)[0]
+    ragged_arg_names = tf_inspect.getfullargspec(ragged_op)[0]
+    if op_arg_names != ragged_arg_names:
+      raise AssertionError(
+          'Signature must exactly match when overriding %s with %s: %s vs %s' %
+          (original_op, ragged_op, op_arg_names, ragged_arg_names))
+    self._ragged_op = ragged_op
+    self._ragged_args = _get_arg_infos(ragged_op, ragged_args)
+    if _UPDATE_DOCSTRINGS:
+      arg_list = ' and '.join('`%s`' % arg for arg in ragged_args)
+      original_op.__doc__ = (
+          original_op.__doc__.rstrip() + '\n\n' +
+          '    {0} may be a `tf.RaggedTensor`.\n'.format(arg_list))
+
+  def handle(self, args, kwargs):
+    if self.is_supported(args, kwargs):
+      return self._ragged_op(*args, **kwargs)
+    else:
+      return self.NOT_SUPPORTED
+
+  def is_supported(self, args, kwargs):
+    found_ragged = False
+    for arg_info in self._ragged_args:
+      if arg_info.position < len(args):
+        arg = args[arg_info.position]
+      else:
+        arg = kwargs.get(arg_info.name, None)
+
+      if arg_info.is_list:
+        if not isinstance(arg, (list, tuple)):
+          return False
+        for elt in arg:
+          if ragged_tensor.is_ragged(elt):
+            found_ragged = True
+          elif not _is_convertible_to_tensor(elt):
+            return False
+      else:
+        if ragged_tensor.is_ragged(arg):
+          found_ragged = True
+        elif not _is_convertible_to_tensor(arg):
+          return False
+    return found_ragged
+
+
+def ragged_dispatch(original_op, tensor_args):
+
+  def decorator(ragged_op):
+    dispatch.RaggedDispatcher(original_op, ragged_op,
+                              tensor_args).register(original_op)
+    return ragged_op
+
+  return decorator
+
+
+_UNARY_ELEMENTWISE_OPS = [
+    array_ops.check_numerics,
+    array_ops.identity,
+    array_ops.ones_like,
+    array_ops.ones_like_v2,
+    array_ops.zeros_like,
+    array_ops.zeros_like_v2,
+    clip_ops.clip_by_value,
+    math_ops.abs,
+    math_ops.acos,
+    math_ops.acosh,
+    math_ops.angle,
+    math_ops.asin,
+    math_ops.asinh,
+    math_ops.atan,
+    math_ops.atanh,
+    math_ops.cast,
+    math_ops.ceil,
+    math_ops.conj,
+    math_ops.cos,
+    math_ops.cosh,
+    math_ops.digamma,
+    math_ops.erf,
+    math_ops.erfc,
+    math_ops.exp,
+    math_ops.expm1,
+    math_ops.floor,
+    math_ops.imag,
+    math_ops.is_finite,
+    math_ops.is_inf,
+    math_ops.is_nan,
+    math_ops.lgamma,
+    math_ops.log,
+    math_ops.log1p,
+    math_ops.log_sigmoid,
+    math_ops.logical_not,
+    math_ops.negative,
+    math_ops.real,
+    math_ops.reciprocal,
+    math_ops.rint,
+    math_ops.round,
+    math_ops.rsqrt,
+    math_ops.saturate_cast,
+    math_ops.sign,
+    math_ops.sin,
+    math_ops.sinh,
+    math_ops.sqrt,
+    math_ops.square,
+    math_ops.tan,
+    parsing_ops.decode_compressed,
+    string_ops.string_to_number,
+    string_ops.string_to_hash_bucket,
+    string_ops.as_string,
+    string_ops.decode_base64,
+    string_ops.encode_base64,
+    string_ops.regex_full_match,
+    string_ops.regex_replace,
+    string_ops.string_strip,
+    string_ops.string_to_hash_bucket,
+    string_ops.string_to_hash_bucket_fast,
+    string_ops.string_to_hash_bucket_strong,
+    string_ops.substr,
+    string_ops.substr_v2,
+    string_ops.string_length,
+    string_ops.string_length_v2,
+    string_ops.unicode_script,
+]
+
+_UNARY_LIST_ELEMENTWISE_OPS = [
+    math_ops.add_n,
+    string_ops.string_join,
+]
+
+_BINARY_ELEMENTWISE_OPS = [
+    math_ops.add,
+    math_ops.atan2,
+    math_ops.complex,
+    math_ops.div_no_nan,
+    math_ops.divide,
+    math_ops.equal,
+    math_ops.floordiv,
+    math_ops.floormod,
+    math_ops.greater,
+    math_ops.greater_equal,
+    math_ops.less,
+    math_ops.less_equal,
+    math_ops.logical_and,
+    math_ops.logical_or,
+    math_ops.logical_xor,
+    math_ops.maximum,
+    math_ops.minimum,
+    math_ops.multiply,
+    math_ops.not_equal,
+    math_ops.pow,
+    math_ops.realdiv,
+    math_ops.squared_difference,
+    math_ops.subtract,
+    math_ops.truediv,
+    math_ops.truncatediv,
+    math_ops.truncatemod,
+]
+
+# (original_op, ragged_op, ragged_args)
+_RAGGED_DISPATCH_OPS = [
+    (array_ops.batch_gather, ragged_array_ops.batch_gather,
+     ['params', 'indices']),
+    (array_ops.concat, ragged_array_ops.concat, ['values']),
+    (array_ops.expand_dims_v2, ragged_array_ops.expand_dims, ['input']),
+    (array_ops.gather_v2, ragged_array_ops.gather, ['params', 'indices']),
+    (array_ops.gather_nd, ragged_array_ops.gather_nd, ['params', 'indices']),
+    (array_ops.stack, ragged_array_ops.stack, ['values']),
+    (array_ops.tile, ragged_array_ops.tile, ['input']),
+    (array_ops.where, ragged_array_ops.where, ['condition', 'x', 'y']),
+    (math_ops.unsorted_segment_sum, ragged_math_ops.segment_sum,
+     ['data', 'segment_ids']),
+    (math_ops.unsorted_segment_prod, ragged_math_ops.segment_prod,
+     ['data', 'segment_ids']),
+    (math_ops.unsorted_segment_min, ragged_math_ops.segment_min,
+     ['data', 'segment_ids']),
+    (math_ops.unsorted_segment_max, ragged_math_ops.segment_max,
+     ['data', 'segment_ids']),
+    (math_ops.unsorted_segment_mean, ragged_math_ops.segment_mean,
+     ['data', 'segment_ids']),
+    (math_ops.unsorted_segment_sqrt_n, ragged_math_ops.segment_sqrt_n,
+     ['data', 'segment_ids']),
+    (math_ops.reduce_sum, ragged_math_ops.reduce_sum, ['input_tensor']),
+    (math_ops.reduce_prod, ragged_math_ops.reduce_prod, ['input_tensor']),
+    (math_ops.reduce_min, ragged_math_ops.reduce_min, ['input_tensor']),
+    (math_ops.reduce_max, ragged_math_ops.reduce_max, ['input_tensor']),
+    (math_ops.reduce_mean, ragged_math_ops.reduce_mean, ['input_tensor']),
+    (math_ops.reduce_any, ragged_math_ops.reduce_any, ['input_tensor']),
+    (math_ops.reduce_all, ragged_math_ops.reduce_all, ['input_tensor']),
+]
+
+
+def register_dispatchers():
+  """Constructs & registers OpDispatchers for ragged ops."""
+
+  op_list = (
+      _UNARY_ELEMENTWISE_OPS + _UNARY_LIST_ELEMENTWISE_OPS +
+      _BINARY_ELEMENTWISE_OPS + [x[0] for x in _RAGGED_DISPATCH_OPS])
+  for op in op_list:
+    _, undecorated_op = tf_decorator.unwrap(op)
+    if not hasattr(undecorated_op, tf_export.API_ATTRS['tensorflow'].names):
+      raise AssertionError('Expected %s to be an exported symbol '
+                           '(while adding a RaggedTensor dispatcher)')
+
+  for op in _UNARY_ELEMENTWISE_OPS:
+    UnaryRaggedElementwiseDispatcher(op).register(op)
+
+  for op in _UNARY_LIST_ELEMENTWISE_OPS:
+    UnaryRaggedElementwiseDispatcher(op, True).register(op)
+
+  for op in _BINARY_ELEMENTWISE_OPS:
+    BinaryRaggedElementwiseDispatcher(op).register(op)
+
+  for (original_op, ragged_op, args) in _RAGGED_DISPATCH_OPS:
+    RaggedDispatcher(original_op, ragged_op, args).register(original_op)
+
+  docstring = (
+      '\n\n### Additional ops that support `RaggedTensor`\n\n' + '\n'.join([
+          '* `tf.%s`' % tf_export.get_canonical_name_for_symbol(op)
+          for op in op_list
+      ]))
+
+  return docstring
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
similarity index 77%
rename from tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
rename to tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 305a96df9c..2bb10adce0 100644
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged.elementwise_ops."""
+"""Tests for RaggedTensor operator dispatch."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -21,106 +21,108 @@ from __future__ import print_function
 from absl.testing import parameterized
 import numpy as np
 
-
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import googletest
 
-# Constants listing various op types to test.  Each elementwise operation
+# Constants listing various op types to test.  Each operation
 # should be included in at least one list below, or tested separately if
 # necessary (e.g., because it expects additional arguments).
 UNARY_FLOAT_OPS = [
-    ragged.abs,
-    ragged.acos,
-    ragged.acosh,
-    ragged.angle,
-    ragged.asin,
-    ragged.asinh,
-    ragged.atan,
-    ragged.atanh,
-    ragged.ceil,
-    ragged.conj,
-    ragged.cos,
-    ragged.cosh,
-    ragged.digamma,
-    ragged.erf,
-    ragged.erfc,
-    ragged.exp,
-    ragged.expm1,
-    ragged.floor,
-    ragged.imag,
-    ragged.is_finite,
-    ragged.is_inf,
-    ragged.is_nan,
-    ragged.lgamma,
-    ragged.log,
-    ragged.log1p,
-    ragged.log_sigmoid,
-    ragged.negative,
-    ragged.real,
-    ragged.reciprocal,
-    ragged.rint,
-    ragged.round,
-    ragged.rsqrt,
-    ragged.sign,
-    ragged.sin,
-    ragged.sinh,
-    ragged.sqrt,
-    ragged.square,
-    ragged.tan,
-    ragged.as_string,
-    ragged.identity,
-    ragged.ones_like,
-    ragged.zeros_like,
+    math_ops.abs,
+    math_ops.acos,
+    math_ops.acosh,
+    math_ops.angle,
+    math_ops.asin,
+    math_ops.asinh,
+    math_ops.atan,
+    math_ops.atanh,
+    math_ops.ceil,
+    math_ops.conj,
+    math_ops.cos,
+    math_ops.cosh,
+    math_ops.digamma,
+    math_ops.erf,
+    math_ops.erfc,
+    math_ops.exp,
+    math_ops.expm1,
+    math_ops.floor,
+    math_ops.imag,
+    math_ops.is_finite,
+    math_ops.is_inf,
+    math_ops.is_nan,
+    math_ops.lgamma,
+    math_ops.log,
+    math_ops.log1p,
+    math_ops.log_sigmoid,
+    math_ops.negative,
+    math_ops.real,
+    math_ops.reciprocal,
+    math_ops.rint,
+    math_ops.round,
+    math_ops.rsqrt,
+    math_ops.sign,
+    math_ops.sin,
+    math_ops.sinh,
+    math_ops.sqrt,
+    math_ops.square,
+    math_ops.tan,
+    array_ops.identity,
+    array_ops.ones_like,
+    array_ops.zeros_like,
 ]
 UNARY_BOOL_OPS = [
-    ragged.logical_not,
+    math_ops.logical_not,
 ]
 UNARY_STRING_OPS = [
-    ragged.decode_base64,
-    ragged.encode_base64,
-    ragged.string_strip,
-    ragged.decode_compressed,
+    string_ops.decode_base64,
+    string_ops.encode_base64,
+    string_ops.string_strip,
+    parsing_ops.decode_compressed,
 ]
 BINARY_FLOAT_OPS = [
-    ragged.add,
-    ragged.atan2,
-    ragged.complex,
-    ragged.div,
-    ragged.div_no_nan,
-    ragged.divide,
-    ragged.equal,
-    ragged.floordiv,
-    ragged.floormod,
-    ragged.greater,
-    ragged.greater_equal,
-    ragged.less,
-    ragged.less_equal,
-    ragged.maximum,
-    ragged.minimum,
-    ragged.multiply,
-    ragged.not_equal,
-    ragged.pow,
-    ragged.realdiv,
-    ragged.squared_difference,
-    ragged.subtract,
-    ragged.truediv,
+    math_ops.add,
+    math_ops.atan2,
+    math_ops.complex,
+    math_ops.div_no_nan,
+    math_ops.divide,
+    math_ops.equal,
+    math_ops.floordiv,
+    math_ops.floormod,
+    math_ops.greater,
+    math_ops.greater_equal,
+    math_ops.less,
+    math_ops.less_equal,
+    math_ops.maximum,
+    math_ops.minimum,
+    math_ops.multiply,
+    math_ops.not_equal,
+    math_ops.pow,
+    math_ops.realdiv,
+    math_ops.squared_difference,
+    math_ops.subtract,
+    math_ops.truediv,
 ]
 BINARY_BOOL_OPS = [
-    ragged.logical_and,
-    ragged.logical_or,
-    ragged.logical_xor,
+    math_ops.logical_and,
+    math_ops.logical_or,
+    math_ops.logical_xor,
 ]
 UNARY_INT_OPS = [
-    ragged.unicode_script,
+    string_ops.unicode_script,
 ]
 BINARY_INT_OPS = [
-    ragged.truncatediv,
-    ragged.truncatemod,
+    math_ops.truncatediv,
+    math_ops.truncatemod,
 ]
 
 
@@ -171,50 +173,49 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
       [{'x': ragged.constant_value([['abcd', 'efgh'], ['aabbccdd']]), 'op': op}
        for op in UNARY_STRING_OPS] +
       [
-          {'op': ragged.clip_by_value,
+          {'op': clip_ops.clip_by_value,
            'x': ragged.constant_value([[-2.0, 3.0], [-3.0]]),
            'clip_value_min': 0.1, 'clip_value_max': 4.0},
-          {'op': ragged.cast,
+          {'op': math_ops.cast,
            'x': ragged.constant_value([[-2.0, 3.0], [-3.0]]),
            'dtype': dtypes.int32},
-          {'op': ragged.saturate_cast,
+          {'op': math_ops.saturate_cast,
            'x': ragged.constant_value([[-2.0, 3.0], [-3.0]]),
            'dtype': dtypes.int32},
-          {'op': ragged.string_to_hash_bucket,
+          {'op': string_ops.string_to_hash_bucket,
            'x': ragged.constant_value([['abcd', 'efgh'], ['aabbccdd']]),
            'num_buckets': 1000},
-          {'op': ragged.string_to_hash_bucket_fast,
+          {'op': string_ops.string_to_hash_bucket_fast,
            'x': ragged.constant_value([['abcd', 'efgh'], ['aabbccdd']]),
            'num_buckets': 1000},
-          {'op': ragged.string_to_hash_bucket_strong,
+          {'op': string_ops.string_to_hash_bucket_strong,
            'x': ragged.constant_value([['abcd', 'efgh'], ['aabbccdd']]),
            'num_buckets': 1000,
            'key': [1231, 12512]},
-          {'op': ragged.string_to_number,
+          {'op': string_ops.string_to_number,
            'x': ragged.constant_value([['-2.0', '3.0'], ['-3.0']])},
-          {'op': ragged.regex_full_match,
+          {'op': string_ops.regex_full_match,
            'x': ragged.constant_value([['hello', '123'], ['1+1']]),
            'pattern': r'\w+'},
-          {'op': ragged.regex_replace,
+          {'op': string_ops.regex_replace,
            'x': ragged.constant_value([['hello', '123'], ['1+1']]),
            'pattern': r'\d',
            'rewrite': '#'},
-          {'op': ragged.substr,
+          {'op': string_ops.substr,
            'x': ragged.constant_value([['hello', '123'], ['1+1']]),
            'pos': 2, 'len': 3},
-          {'op': ragged.check_numerics,
+          {'op': array_ops.check_numerics,
            'x': ragged.constant_value([[-2.0, 3.0], [-3.0]]),
            'message': 'check-numerics'},
       ]
       )  # pyformat: disable
-  def testUnaryOp(self, x, op=ragged.abs, **extra_args):
+  def testUnaryElementwiseOp(self, x, op=math_ops.abs, **extra_args):
     x = ragged.convert_to_tensor_or_ragged_tensor(x)
     result = op(x, **extra_args)
 
     # Run the wrapped op on the dense values, for comparison.
     dense_x = x.inner_values if isinstance(x, ragged.RaggedTensor) else x
-    expected_flat_values = array_ops.reshape(
-        op.__wrapped__(dense_x, **extra_args), [-1])
+    expected_flat_values = array_ops.reshape(op(dense_x, **extra_args), [-1])
 
     with self.test_session():
       # Check that the result has the expected shape.
@@ -285,12 +286,17 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
           #=====================================================================
           {'x': ragged.constant_value([[[1, 2], [3], [4]], [[], [5, 7, 8]]]),
            'y': ragged.constant_value([[[3, 8], [2], [5]], [[], [1, 9, 8]]]),
-           'use_kwargs': True},
+           'use_kwargs': ('x', 'y')},
           {'x': ragged.constant_value([[[1, 2]], [[3, 4], [5, 6], [7, 8]]],
                                       ragged_rank=1),
            'y': ragged.constant_value([[[9, 3]], [[5, 2], [3, 4], [7, 6]]],
                                       ragged_rank=1),
-           'use_kwargs': True},
+           'use_kwargs': ('x', 'y')},
+          {'x': ragged.constant_value([[[1, 2]], [[3, 4], [5, 6], [7, 8]]],
+                                      ragged_rank=1),
+           'y': ragged.constant_value([[[9, 3]], [[5, 2], [3, 4], [7, 6]]],
+                                      ragged_rank=1),
+           'use_kwargs': ('x',)},
       ] +
       #=========================================================================
       # Test each unary op.
@@ -306,16 +312,16 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
       [{'x': ragged.constant_value([[True, True], [False]]),
         'y': ragged.constant_value([[False, True], [False]]),
         'op': op}
-       for op in BINARY_BOOL_OPS] +
-      [
-      ]
+       for op in BINARY_BOOL_OPS]
       )  # pyformat: disable
-  def testBinaryOp(self, x, y, op=ragged.add, **extra_args):
-    use_kwargs = extra_args.pop('use_kwargs', False)
+  def testBinaryElementwiseOp(self, x, y, op=math_ops.add, **extra_args):
+    use_kwargs = extra_args.pop('use_kwargs', ())
     x = ragged.convert_to_tensor_or_ragged_tensor(x)
     y = ragged.convert_to_tensor_or_ragged_tensor(y)
-    if use_kwargs:
+    if 'x' in use_kwargs and 'y' in use_kwargs:
       result = op(x=x, y=y, **extra_args)
+    elif 'y' in use_kwargs:
+      result = op(x, y=y, **extra_args)
     else:
       result = op(x, y, **extra_args)
 
@@ -323,7 +329,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
     dense_x = x.inner_values if isinstance(x, ragged.RaggedTensor) else x
     dense_y = y.inner_values if isinstance(y, ragged.RaggedTensor) else y
     expected_flat_values = array_ops.reshape(
-        op.__wrapped__(dense_x, dense_y, **extra_args), [-1])
+        op(dense_x, dense_y, **extra_args), [-1])
 
     with self.test_session():
       # Check that the result has the expected shape.
@@ -358,16 +364,17 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
                       ragged.constant_value([[[2, 9], [12]], [[8]]])),
            'use_kwargs': True},
       ] + [
-          {'op': ragged.add_n,
+          {'op': math_ops.add_n,
            'inputs': (ragged.constant_value([[1, 3], [-3]]),
                       ragged.constant_value([[4, 7], [88]]),
                       ragged.constant_value([[2, 9], [12]]))},
-          {'op': ragged.string_join,
+          {'op': string_ops.string_join,
            'inputs': (ragged.constant_value([['a', 'b'], ['c']]),
                       ragged.constant_value([['foo', 'bar'], ['baz']]),
                       ragged.constant_value([['2', '9'], ['12']]))},
       ])  # pyformat: disable
-  def testListValuedOp(self, inputs, op=ragged.add_n, **extra_args):
+  def testListValuedElementwiseOp(self, inputs, op=math_ops.add_n,
+                                  **extra_args):
     use_kwargs = extra_args.pop('use_kwargs', False)
     inputs = [ragged.convert_to_tensor_or_ragged_tensor(x) for x in inputs]
     if use_kwargs:
@@ -381,7 +388,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
         for x in inputs
     ]
     expected_flat_values = array_ops.reshape(
-        op.__wrapped__(dense_inputs, **extra_args), [-1])
+        op(dense_inputs, **extra_args), [-1])
 
     with self.test_session():
       # Check that the result has the expected shape.
@@ -395,13 +402,13 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
       self.assertAllEqual(expected_flat_values, result_flat_values)
 
   @test_util.run_deprecated_v1
-  def testUnknownRankError(self):
+  def testElementwiseOpUnknownRankError(self):
     x = ragged.constant([[1, 2], [3]])
     y = ragged.from_row_splits(
         array_ops.placeholder_with_default([1, 2, 3], shape=None), x.row_splits)
-    with self.assertRaisesRegexp(
-        ValueError, r'Unable to broadcast: unknown rank'):
-      ragged.add(x, y)
+    with self.assertRaisesRegexp(ValueError,
+                                 r'Unable to broadcast: unknown rank'):
+      math_ops.add(x, y)
 
   @parameterized.parameters([
       dict(
@@ -417,26 +424,31 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
           y=ragged.constant_value([[1]]),
           expected=[[[2]]]),
   ])
-  def testBroadcastAdd(self, x, y, expected):
+  def testElementwiseOpBroadcast(self, x, y, expected):
     x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32)
     y = ragged.convert_to_tensor_or_ragged_tensor(y, dtype=dtypes.int32)
     result = x + y
     with self.cached_session():
       self.assertEqual(result.eval().tolist(), expected)
 
-  def testShapeMismatch(self):
+  def testElementwiseOpShapeMismatch(self):
     x = ragged.constant([[1, 2, 3], [4, 5]])
     y = ragged.constant([[1, 2, 3], [4, 5, 6]])
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  'Incompatible shapes'):
       with self.cached_session():
-        ragged.add(x, y).eval()
+        math_ops.add(x, y).eval()
 
-  def testDocstring(self):
-    self.assertRegexpMatches(
-        ragged.add.__doc__,
-        'Ragged version of the elementwise operation `tf.math.add`')
-    self.assertEqual(ragged.add.__name__, 'add')
+  def testBinaryOpSparseAndRagged(self):
+    x = ragged.constant([[1, 2, 3], [4, 5]])
+    y = sparse_tensor.SparseTensor([[0, 0], [0, 1], [2, 0]], [1, 2, 3], [3, 2])
+    with self.assertRaises(TypeError):
+      with self.cached_session():
+        math_ops.add(x, y).eval()
+
+    with self.assertRaises(TypeError):
+      with self.cached_session():
+        math_ops.add_n([x, y]).eval()
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py b/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
deleted file mode 100644
index 59b7dd1661..0000000000
--- a/tensorflow/python/ops/ragged/ragged_elementwise_ops.py
+++ /dev/null
@@ -1,389 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Elementwise operations for RaggedTensors."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import clip_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import parsing_ops
-from tensorflow.python.ops import string_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
-from tensorflow.python.ops.ragged import ragged_tensor
-from tensorflow.python.ops.ragged import ragged_tensor_shape
-from tensorflow.python.util import tf_decorator
-from tensorflow.python.util import tf_export
-from tensorflow.python.util import tf_inspect
-
-# Information about an argument to an operation: The name of the argument, its
-# position in the argument list, and a boolean flag indicating whether it
-# expects a list of tensors.
-_ArgInfo = collections.namedtuple('ArgInfo', ['name', 'position', 'is_list'])
-
-
-def make_elementwise_op(op, *elementwise_args):
-  """Returns a ragged-tensor version of the elementwise operation `op`.
-
-  The returned operation will:
-
-  1. Broadcast the elementwise arguments to have a compatible shape.
-     An exception is raised if the tensors not broadcast-compatible.
-  2. Call `op`, substituting the dense values of the broadcasted tensor for
-     each elementwise argument.
-  3. Return a potentially ragged tensor constructed from the output of `op`
-     and the broadcasted tensors' nested row splits.
-
-  For example, you can construct a ragged-tensor version of the standard
-  operation `tf.add` by calling `make_elementwise_op(tf.add, 'x', 'y')`.
-
-  Args:
-    op: The operation to wrap.
-    *elementwise_args: The names of arguments to `op` that are treated as
-      elementwise.  Arguments that take a list of tensors should have their
-      names wrapped in square brackets (e.g. "[inputs]").
-
-  Raises:
-    ValueError: If any name specified in `elementwise_args` is not the name
-      of an argument to `op`.
-  """
-  elementwise_arg_infos = _get_arg_infos(op, elementwise_args)
-
-  def ragged_op(*args, **kwargs):
-    """Ragged version of `op`."""
-    args = list(args)
-
-    # Collect all of the elementwise arguments, and put them in a single
-    # dict whose values are the (potentially ragged) tensors that need to
-    # be broadcast to a common shape.  The keys of this dict are tuples
-    # (argkey, index), where argkey is an int for poitional args or a string
-    # for keyword args; and index is None for non-list args and the index of the
-    # tensor for list args.
-    elementwise_args = {}
-    for (name, position, is_list) in elementwise_arg_infos.values():
-      if position < len(args):
-        if is_list:
-          args[position] = list(args[position])
-          for (index, arg) in enumerate(args[position]):
-            elementwise_args[position, index] = arg
-        else:
-          elementwise_args[position, None] = args[position]
-      elif name in kwargs:
-        if is_list:
-          kwargs[name] = list(kwargs[name])
-          for (i, arg) in enumerate(kwargs[name]):
-            elementwise_args[name, i] = arg
-        else:
-          elementwise_args[name, None] = kwargs[name]
-
-    with ops.name_scope(None, op.__name__, elementwise_args.values()):
-      # Convert all inputs to tensors or ragged tensors.
-      for ((key, index), tensor) in elementwise_args.items():
-        argname = elementwise_arg_infos[key].name
-        converted = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-            tensor, name=argname)
-        elementwise_args[key, index] = converted
-
-      # Broadcast tensors to have compatible shapes.
-      broadcast_args, result_splits, broadcast_check_ops = \
-          _broadcast_elementwise_args(elementwise_args)
-
-      # Replace tensor arguments with their dense values.
-      for ((key, index), tensor) in broadcast_args.items():
-        if ragged_tensor.is_ragged(tensor):
-          if isinstance(key, int) and index is None:
-            args[key] = tensor.inner_values
-          elif isinstance(key, int) and index is not None:
-            args[key][index] = tensor.inner_values
-          elif isinstance(key, str) and index is None:
-            kwargs[key] = tensor.inner_values
-          else:
-            assert isinstance(key, str) and index is not None
-            kwargs[key][index] = tensor.inner_values
-
-      # Call the elementwise op on the broadcasted dense values.
-      with ops.control_dependencies(broadcast_check_ops):
-        result_values = op(*args, **kwargs)
-
-      # Restore any ragged dimensions that we stripped off, and return the
-      # result.
-      return ragged_factory_ops.from_nested_row_splits(result_values,
-                                                       result_splits)
-
-  # Construct the docstring.
-  op_name = tf_export.get_canonical_name_for_symbol(op)
-  assert op_name is not None, op
-  argnames = ', '.join('`%s`' % s.strip('[]') for s in elementwise_args)
-  docstring = _ELEMENTWISE_DOCSTRING % dict(op_name=op_name, argnames=argnames)
-
-  # Update name, docstring, signature, etc., for the wrapper, and return it.
-  return tf_decorator.make_decorator(op, ragged_op, decorator_doc=docstring)
-
-
-_ELEMENTWISE_DOCSTRING = """\
-Ragged version of the elementwise operation `tf.%(op_name)s`.
-
-  The following elementwise arguments may be ragged or dense:
-  %(argnames)s.
-  These arguments will be broadcast to a compatible shape if necessary.
-  """
-
-
-def _get_arg_infos(func, elementwise_args):
-  """Returns `_ArgInfo`s for each `func` arg specified by `elementwise_args`.
-
-  Args:
-    func: The function whose arguments should be described.
-    elementwise_args: The names of the arguments to get info for.
-
-  Returns:
-    A dictionary that maps both names and positions of arguments to
-    `_ArgInfo` tuples.
-  """
-  arg_infos = {}
-
-  # Inspect the func's argspec to find the position of each arg.
-  arg_spec = tf_inspect.getargspec(func)
-  for argname in elementwise_args:
-    assert isinstance(argname, str)
-    is_list = argname.startswith('[') and argname.endswith(']')
-    if is_list:
-      argname = argname[1:-1]
-    assert argname in arg_spec.args, (func, argname, arg_spec.args)
-    arg_info = _ArgInfo(argname, arg_spec.args.index(argname), is_list)
-    arg_infos[arg_info.name] = arg_info
-    arg_infos[arg_info.position] = arg_info
-  return arg_infos
-
-
-def _broadcast_elementwise_args(elementwise_args):
-  """Broadcasts the values of `elementwise_args` to have compatible shapes.
-
-  Args:
-    elementwise_args: A dictionary whose keys are potentially ragged tensors.
-
-  Returns:
-    A tuple `(broadcast_args, broadcast_splits, checks)` where:
-
-    * `broadcast_args` is a dictionary with the same keys as
-      `elementwise_args`, mapping to broadcasted tensors.
-    * `broadcast_splits` is the broadcasted nested row splits.
-    * `checks` is a possibly empty tuple of assertion operations that should
-      be added as control dependencies.
-
-  Raises:
-    ValueError: If broadcasting fails.
-  """
-  # No elementwise arguments were used: nothing to do!
-  if not elementwise_args:
-    return elementwise_args, (), ()
-
-  # A single elementwise argument was used: no broadcasting necessary.
-  if len(elementwise_args) == 1:
-    arg = list(elementwise_args.values())[0]
-    if ragged_tensor.is_ragged(arg):
-      return elementwise_args, arg.nested_row_splits, ()
-    else:
-      return elementwise_args, (), ()
-
-  # Multiple elementwise arguments.
-  else:
-    is_ragged = [ragged_tensor.is_ragged(t) for t in elementwise_args.values()]
-    if not any(is_ragged):
-      return elementwise_args, (), ()
-
-    # If we have a single ragged tensor plus a set of scalars, then we can
-    # rely on the underlying elementwise op to do broadcasting.
-    if (sum(is_ragged) == 1 and
-        all((ragged_tensor.is_ragged(t) or t.shape.ndims == 0)
-            for t in elementwise_args.values())):
-      nested_splits_lists = [
-          t.nested_row_splits
-          for t in elementwise_args.values()
-          if ragged_tensor.is_ragged(t)][0]
-      return elementwise_args, nested_splits_lists, ()
-
-    else:
-      # Get the shapes of all the elementwise arguments.
-      shapes = [ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(t)
-                for t in elementwise_args.values()]
-
-      # Broadcast the shapes to all have the same rank (the max rank).
-      ranks = [t.shape.ndims for t in elementwise_args.values()]
-      if any(rank is None for rank in ranks):
-        raise ValueError('Unable to broadcast: unknown rank')
-      broadcast_rank = max(ranks)
-      shapes = [shape.broadcast_to_rank(broadcast_rank) for shape in shapes]
-
-      # For each dimension, broadcast the shapes to be compatible.
-      for axis in range(broadcast_rank):
-        # For each i, broadcast shape[i+1] to be compatible with shape[i]; and
-        # then finally broadcast shape[0] to be compatible with shape[-1].
-        for i in range(len(shapes)):
-          j = (i + 1) % len(shapes)
-          dim_size = shapes[i].dimension_size(axis)
-          shapes[j] = shapes[j].broadcast_dimension(axis, dim_size)
-      broadcast_shape = shapes[0]
-
-      # Broadcast every elementwise arg to the shape that we calculated.
-      elementwise_args = dict([
-          (key, ragged_tensor_shape.broadcast_to(t, broadcast_shape, False))
-          for (key, t) in elementwise_args.items()])
-      nested_splits_lists = list(elementwise_args.values())[0].nested_row_splits
-      return elementwise_args, nested_splits_lists, ()
-
-
-# A list of symbols that should be exported in the "ragged" package.
-_symbols_to_export = []
-
-
-def _add_elementwise_ops_to_this_module(specs, verbose=False):
-  """Adds ragged versions of the given ops to this module.
-
-  Args:
-    specs: A list of tuples containing the arguments for `make_elementwise_op`.
-    verbose: If true, then display each op that gets added.
-  """
-  for spec in specs:
-    original_op = spec[0]
-    ragged_op = make_elementwise_op(*spec)
-    canonical_name = tf_export.get_canonical_name_for_symbol(original_op)
-    if '.' not in canonical_name:
-      op_name = canonical_name
-    else:
-      op_name = original_op.__name__
-
-    # Temporary hack (will be removed once dispatch is added for RaggedTensors):
-    if op_name == 'neg': op_name = 'negative'
-
-    if verbose:
-      print('Adding ragged_elementwise_op: tf.ragged.%s (based on tf.%s)' %
-            (op_name, canonical_name))
-    globals()[op_name] = ragged_op
-    _symbols_to_export.append(op_name)
-
-
-# A list of tuples containing arguments for `make_elementwise_op`, for each
-# elementwise operation that should have a ragged version built.  Each tuple
-# contains a standard `Tensor` operation, and the names of any arguments
-# that are processed in elementwise fashion.
-_TF_ELEMENTWISE_OPS = [
-    # Unary math operations.
-    (clip_ops.clip_by_value, 't'),
-    (math_ops.abs, 'x'),
-    (math_ops.acos, 'x'),
-    (math_ops.acosh, 'x'),
-    (math_ops.angle, 'input'),
-    (math_ops.asin, 'x'),
-    (math_ops.asinh, 'x'),
-    (math_ops.atan, 'x'),
-    (math_ops.atanh, 'x'),
-    (math_ops.cast, 'x'),
-    (math_ops.ceil, 'x'),
-    (math_ops.conj, 'x'),
-    (math_ops.cos, 'x'),
-    (math_ops.cosh, 'x'),
-    (math_ops.digamma, 'x'),
-    (math_ops.erf, 'x'),
-    (math_ops.erfc, 'x'),
-    (math_ops.exp, 'x'),
-    (math_ops.expm1, 'x'),
-    (math_ops.floor, 'x'),
-    (math_ops.imag, 'input'),
-    (math_ops.is_finite, 'x'),
-    (math_ops.is_inf, 'x'),
-    (math_ops.is_nan, 'x'),
-    (math_ops.lgamma, 'x'),
-    (math_ops.log, 'x'),
-    (math_ops.log1p, 'x'),
-    (math_ops.log_sigmoid, 'x'),
-    (math_ops.logical_not, 'x'),
-    (math_ops.negative, 'x'),
-    (math_ops.real, 'input'),
-    (math_ops.reciprocal, 'x'),
-    (math_ops.rint, 'x'),
-    (math_ops.round, 'x'),
-    (math_ops.rsqrt, 'x'),
-    (math_ops.saturate_cast, 'value'),
-    (math_ops.sign, 'x'),
-    (math_ops.sin, 'x'),
-    (math_ops.sinh, 'x'),
-    (math_ops.sqrt, 'x'),
-    (math_ops.square, 'x'),
-    (math_ops.tan, 'x'),
-
-    # Binary math operations
-    (math_ops.add, 'x', 'y'),
-    (math_ops.atan2, 'y', 'x'),
-    (math_ops.complex, 'real', 'imag'),
-    (math_ops.div, 'x', 'y'),
-    (math_ops.div_no_nan, 'x', 'y'),
-    (math_ops.divide, 'x', 'y'),
-    (math_ops.equal, 'x', 'y'),
-    (math_ops.floordiv, 'x', 'y'),
-    (math_ops.floormod, 'x', 'y'),
-    (math_ops.greater, 'x', 'y'),
-    (math_ops.greater_equal, 'x', 'y'),
-    (math_ops.less, 'x', 'y'),
-    (math_ops.less_equal, 'x', 'y'),
-    (math_ops.logical_and, 'x', 'y'),
-    (math_ops.logical_or, 'x', 'y'),
-    (math_ops.logical_xor, 'x', 'y'),
-    (math_ops.maximum, 'x', 'y'),
-    (math_ops.minimum, 'x', 'y'),
-    (math_ops.multiply, 'x', 'y'),
-    (math_ops.not_equal, 'x', 'y'),
-    (math_ops.pow, 'x', 'y'),
-    (math_ops.realdiv, 'x', 'y'),
-    (math_ops.squared_difference, 'x', 'y'),
-    (math_ops.subtract, 'x', 'y'),
-    (math_ops.truediv, 'x', 'y'),
-    (math_ops.truncatediv, 'x', 'y'),
-    (math_ops.truncatemod, 'x', 'y'),
-
-    # N-ary math operations
-    (math_ops.add_n, '[inputs]'),
-
-    # String operations
-    (string_ops.as_string, 'input'),
-    (string_ops.decode_base64, 'input'),
-    (string_ops.encode_base64, 'input'),
-    (string_ops.regex_full_match, 'input'),
-    (string_ops.regex_replace, 'input'),
-    (string_ops.string_join, '[inputs]'),
-    (string_ops.string_strip, 'input'),
-    (string_ops.string_to_hash_bucket, 'input'),
-    (string_ops.string_to_hash_bucket_fast, 'input'),
-    (string_ops.string_to_hash_bucket_strong, 'input'),
-    (string_ops.substr, 'input'),
-    (string_ops.unicode_script, 'input'),
-
-    # Array ops
-    (array_ops.check_numerics, 'tensor'),
-    (array_ops.identity, 'input'),
-    (array_ops.ones_like, 'tensor'),
-    (array_ops.zeros_like, 'tensor'),
-
-    # Parsing ops
-    (parsing_ops.decode_compressed, 'bytes'),
-    (parsing_ops.string_to_number, 'string_tensor'),
-]
-_add_elementwise_ops_to_this_module(_TF_ELEMENTWISE_OPS)
-
diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
index 7a8603c949..ecd78a91b2 100644
--- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from absl.testing import parameterized
+import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
@@ -55,7 +56,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       ),
       # [d1, (d2)] -> [d1, (d2)]
       dict(
-          fn=lambda x: x+1,
+          fn=lambda x: x + np.int64(1),
           elems=[[1, 2, 3], [4, 5], [6, 7]],
           expected_output=[[2, 3, 4], [5, 6], [7, 8]],
           dtype=dtypes.int64,
@@ -64,7 +65,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       ),
       # [d1, (d2), d3] -> [d1, (d2), d3]
       dict(
-          fn=lambda x: x+1,
+          fn=lambda x: x + np.int64(1),
           elems=[[[1, 2], [3, 4]], [], [[5, 6], [7, 8], [9, 0]]],
           elems_ragged_rank=1,
           expected_ragged_rank=1,
@@ -131,7 +132,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       ),
       # [d1, (d2), (d3), (d4a), (d5)] ->  [d1, (d2), (d3), (d4b), (d5)]
       dict(
-          fn=lambda x: ragged.add(x, 1),
+          fn=lambda x: x + np.int64(1),
           elems=[[[[[1, 2, 3]], [[4], [5]]]], [[[[6, 7]]], [[[8], []]]]],
           expected_output=[[[[[2, 3, 4]], [[5], [6]]]],
                            [[[[7, 8]]], [[[9], []]]]],
@@ -196,8 +197,8 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     def _increment(f):
       return {
-          'batman': ragged.add(f['batman'], 1),
-          'robin': ragged.add(f['robin'], 1),
+          'batman': f['batman'] + 1,
+          'robin': f['robin'] + 1,
       }
 
     output = ragged.map_fn(
diff --git a/tensorflow/python/ops/ragged/ragged_math_ops.py b/tensorflow/python/ops/ragged/ragged_math_ops.py
index 857b8dbfa3..d661563a9f 100644
--- a/tensorflow/python/ops/ragged/ragged_math_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_math_ops.py
@@ -143,8 +143,11 @@ Computes the %(combination)s along segments of a RaggedTensor.
 """
 
 
-def _ragged_segment_aggregate(unsorted_segment_op, data, segment_ids,
-                              num_segments, name=None):
+def _ragged_segment_aggregate(unsorted_segment_op,
+                              data,
+                              segment_ids,
+                              num_segments,
+                              name=None):
   """Aggregates along segments of a RaggedTensor using `unsorted_segment_op`.
 
   Returns a RaggedTensor `output` with `num_segments` rows, where the row
@@ -212,12 +215,11 @@ def _ragged_segment_aggregate(unsorted_segment_op, data, segment_ids,
     assert output_row_lengths.dtype == dtypes.int64
 
     # Build the splits tensor for the output RaggedTensor.
-    output_splits = array_ops.concat(
-        [
-            array_ops.zeros([1], dtypes.int64),
-            math_ops.cumsum(output_row_lengths)
-        ],
-        axis=0)
+    output_splits = array_ops.concat([
+        array_ops.zeros([1], dtypes.int64),
+        math_ops.cumsum(output_row_lengths)
+    ],
+                                     axis=0)
 
     # For each row in `data`, find the start & limit position where that row's
     # values will be aggregated in output.values.
@@ -311,7 +313,7 @@ _set_ragged_segment_docstring(segment_sqrt_n, 'sum divided by sqrt(N)',
 _RAGGED_REDUCE_DOCSTRING = """\
 Computes the %(combination)s of elements across dimensions of a `RaggedTensor`.
 
-  Reduces `rt_input` along the dimensions given in `axis` by taking the
+  Reduces `input_tensor` along the dimensions given in `axis` by taking the
   %(combination)s of values.  If a reduced dimension has no elements for
   some index, then the value for that index will be %(default)s.
 
@@ -319,18 +321,18 @@ Computes the %(combination)s of elements across dimensions of a `RaggedTensor`.
   `axis` is not specified, then all dimensions are reduced, and a scalar
   value is returned.
   Args:
-    rt_input: A `RaggedTensor` containing the values to be %(combined)s.
+    input_tensor: A `RaggedTensor` containing the values to be %(combined)s.
     axis: The dimensions to reduce.  May be `None` (to reduce all axes), an
       `int` (to reduce a single axis), a `list` or `tuple` of `int` (to reduce
       a given set of axes), or a `Tensor` with a constant value.  Must be in
-      the range `[0, rt_input.rank]`.
+      the range `[0, input_tensor.rank]`.
     name: A name prefix for the returned tensor (optional).
   Returns:
     A `RaggedTensor` containing the %(combined)s values.  The returned tensor
     has the same dtype as `data`, and its shape is given by removing the
-    dimensions specified in `axis` from `rt_input.shape`.  The `ragged_rank`
+    dimensions specified in `axis` from `input_tensor.shape`.  The `ragged_rank`
     of the returned tensor is given by substracting any ragged dimensions
-    specified in `axis` from `rt_input.ragged_rank`.
+    specified in `axis` from `input_tensor.ragged_rank`.
   Raises:
     ValueError: If `axis` contains a `Tensor` whose value is not constant.
   ####Example:
@@ -387,7 +389,11 @@ _RAGGED_REDUCE_ANY_EXAMPLE = """
 """
 
 
-def _ragged_reduce_aggregate(reduce_op, unsorted_segment_op, rt_input, axis,
+def _ragged_reduce_aggregate(reduce_op,
+                             unsorted_segment_op,
+                             rt_input,
+                             axis,
+                             keepdims,
                              name=None):
   """Aggregates across axes of a RaggedTensor using the given `Tensor` ops.
 
@@ -412,6 +418,7 @@ def _ragged_reduce_aggregate(reduce_op, unsorted_segment_op, rt_input, axis,
       `int` (to reduce a single axis), a `list` or `tuple` of `int` (to reduce a
       given set of axes), or a `Tensor` with a constant value.  Must be in the
       range `[0, rt_input.rank)`.
+    keepdims: If true, retains reduced dimensions with length 1.
     name: A name prefix for the returned tensor (optional).
 
   Returns:
@@ -426,6 +433,9 @@ def _ragged_reduce_aggregate(reduce_op, unsorted_segment_op, rt_input, axis,
   if not ragged_tensor.is_ragged(rt_input):
     return reduce_op(rt_input, axis, name=name)
 
+  if keepdims:
+    raise ValueError('keepdims=True is not supported for RaggedTensors.')
+
   if isinstance(axis, ops.Tensor):
     axis = tensor_util.constant_value(axis)
     if axis is None:
@@ -448,9 +458,9 @@ def _ragged_reduce_aggregate(reduce_op, unsorted_segment_op, rt_input, axis,
         # once will probably require a nontrivial c++ op.
         axis = sorted(axis)
         inner_reduced = _ragged_reduce_aggregate(reduce_op, unsorted_segment_op,
-                                                 rt_input, axis[-1])
+                                                 rt_input, axis[-1], keepdims)
         return _ragged_reduce_aggregate(reduce_op, unsorted_segment_op,
-                                        inner_reduced, axis[:-1])
+                                        inner_reduced, axis[:-1], keepdims)
 
     axis = ragged_util.get_positive_axis(axis, rt_input.shape.ndims)
 
@@ -476,48 +486,48 @@ def _ragged_reduce_aggregate(reduce_op, unsorted_segment_op, rt_input, axis,
       #     sum_{j} rt_input [i_0, ..., i_[axis-1], j, i_axis+1], ..., i_N]
       return rt_input.with_values(
           _ragged_reduce_aggregate(reduce_op, unsorted_segment_op,
-                                   rt_input.values, axis - 1))
+                                   rt_input.values, axis - 1, keepdims))
 
 
-def reduce_sum(rt_input, axis=None, name=None):
+def reduce_sum(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
   return _ragged_reduce_aggregate(math_ops.reduce_sum,
-                                  math_ops.unsorted_segment_sum, rt_input, axis,
-                                  name or 'RaggedReduceSum')
+                                  math_ops.unsorted_segment_sum, input_tensor,
+                                  axis, keepdims, name or 'RaggedReduceSum')
 
 
-def reduce_prod(rt_input, axis=None, name=None):
+def reduce_prod(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
   return _ragged_reduce_aggregate(math_ops.reduce_prod,
-                                  math_ops.unsorted_segment_prod, rt_input,
-                                  axis, name or 'RaggedReduceProd')
+                                  math_ops.unsorted_segment_prod, input_tensor,
+                                  axis, keepdims, name or 'RaggedReduceProd')
 
 
-def reduce_min(rt_input, axis=None, name=None):
+def reduce_min(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
   return _ragged_reduce_aggregate(math_ops.reduce_min,
-                                  math_ops.unsorted_segment_min, rt_input, axis,
-                                  name or 'RaggedReduceMin')
+                                  math_ops.unsorted_segment_min, input_tensor,
+                                  axis, keepdims, name or 'RaggedReduceMin')
 
 
-def reduce_max(rt_input, axis=None, name=None):
+def reduce_max(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
   return _ragged_reduce_aggregate(math_ops.reduce_max,
-                                  math_ops.unsorted_segment_max, rt_input, axis,
-                                  name or 'RaggedReduceMax')
+                                  math_ops.unsorted_segment_max, input_tensor,
+                                  axis, keepdims, name or 'RaggedReduceMax')
 
 
-def reduce_mean(rt_input, axis=None, name=None):
+def reduce_mean(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
-  with ops.name_scope(name, 'RaggedReduceMean', [rt_input, axis]):
-    total = reduce_sum(rt_input, axis)
-    if ragged_tensor.is_ragged(rt_input):
+  with ops.name_scope(name, 'RaggedReduceMean', [input_tensor, axis]):
+    total = reduce_sum(input_tensor, axis, keepdims)
+    if ragged_tensor.is_ragged(input_tensor):
       ones = ragged_factory_ops.from_nested_row_splits(
-          array_ops.ones_like(rt_input.inner_values),
-          rt_input.nested_row_splits)
+          array_ops.ones_like(input_tensor.inner_values),
+          input_tensor.nested_row_splits)
     else:
-      ones = array_ops.ones_like(rt_input)
-    count = reduce_sum(ones, axis)
+      ones = array_ops.ones_like(input_tensor)
+    count = reduce_sum(ones, axis, keepdims)
     if ragged_tensor.is_ragged(total):
       return ragged_factory_ops.from_nested_row_splits(
           total.inner_values / count.inner_values, total.nested_row_splits)
@@ -525,20 +535,25 @@ def reduce_mean(rt_input, axis=None, name=None):
       return total / count
 
 
-def _cast(rt_input, dtype):
-  return ragged_functional_ops.map_inner_values(math_ops.cast, rt_input, dtype)
+def _cast(input_tensor, dtype):
+  return ragged_functional_ops.map_inner_values(math_ops.cast, input_tensor,
+                                                dtype)
 
 
-def reduce_all(rt_input, axis=None, name=None):
+def reduce_all(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
-  with ops.name_scope(name, 'RaggedReduceAll', [rt_input, axis]):
-    return _cast(reduce_prod(_cast(rt_input, dtypes.int32), axis), dtypes.bool)
+  with ops.name_scope(name, 'RaggedReduceAll', [input_tensor, axis]):
+    return _cast(
+        reduce_prod(_cast(input_tensor, dtypes.int32), axis, keepdims),
+        dtypes.bool)
 
 
-def reduce_any(rt_input, axis=None, name=None):
+def reduce_any(input_tensor, axis=None, keepdims=None, name=None):
   """For docs, see: _RAGGED_REDUCE_DOCSTRING."""
-  with ops.name_scope(name, 'RaggedReduceAny', [rt_input, axis]):
-    return _cast(reduce_sum(_cast(rt_input, dtypes.int32), axis), dtypes.bool)
+  with ops.name_scope(name, 'RaggedReduceAny', [input_tensor, axis]):
+    return _cast(
+        reduce_sum(_cast(input_tensor, dtypes.int32), axis, keepdims),
+        dtypes.bool)
 
 
 def _set_ragged_reduce_docstring(func, combination, combined, default, example):
@@ -554,9 +569,11 @@ _set_ragged_reduce_docstring(reduce_sum, 'sum', 'summed', '0',
 _set_ragged_reduce_docstring(reduce_prod, 'product', 'multiplied', '1',
                              _RAGGED_REDUCE_PROD_EXAMPLE)
 _set_ragged_reduce_docstring(reduce_min, 'minimum', 'minimized',
-                             '`rt_input.dtype.min`', _RAGGED_REDUCE_MIN_EXAMPLE)
+                             '`input_tensor.dtype.min`',
+                             _RAGGED_REDUCE_MIN_EXAMPLE)
 _set_ragged_reduce_docstring(reduce_max, 'maximum', 'maximized',
-                             '`rt_input.dtype.max`', _RAGGED_REDUCE_MAX_EXAMPLE)
+                             '`input_tensor.dtype.max`',
+                             _RAGGED_REDUCE_MAX_EXAMPLE)
 _set_ragged_reduce_docstring(reduce_mean, 'mean', 'averaged', 'NaN',
                              _RAGGED_REDUCE_MEAN_EXAMPLE)
 
diff --git a/tensorflow/python/ops/ragged/ragged_operators.py b/tensorflow/python/ops/ragged/ragged_operators.py
index 223ba0d2e7..7654fa22b1 100644
--- a/tensorflow/python/ops/ragged/ragged_operators.py
+++ b/tensorflow/python/ops/ragged/ragged_operators.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.ops.ragged import ragged_elementwise_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_getitem
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.util import tf_decorator
@@ -33,40 +33,39 @@ def _right(operator):
 ragged_tensor.RaggedTensor.__getitem__ = ragged_getitem.ragged_tensor_getitem
 
 # Ordering operators
-ragged_tensor.RaggedTensor.__ge__ = ragged_elementwise_ops.greater_equal
-ragged_tensor.RaggedTensor.__gt__ = ragged_elementwise_ops.greater
-ragged_tensor.RaggedTensor.__le__ = ragged_elementwise_ops.less_equal
-ragged_tensor.RaggedTensor.__lt__ = ragged_elementwise_ops.less
+ragged_tensor.RaggedTensor.__ge__ = math_ops.greater_equal
+ragged_tensor.RaggedTensor.__gt__ = math_ops.greater
+ragged_tensor.RaggedTensor.__le__ = math_ops.less_equal
+ragged_tensor.RaggedTensor.__lt__ = math_ops.less
 
 # Logical operators
-ragged_tensor.RaggedTensor.__and__ = ragged_elementwise_ops.logical_and
-ragged_tensor.RaggedTensor.__rand__ = _right(ragged_elementwise_ops.logical_and)
-ragged_tensor.RaggedTensor.__invert__ = ragged_elementwise_ops.logical_not
-ragged_tensor.RaggedTensor.__ror__ = _right(ragged_elementwise_ops.logical_or)
-ragged_tensor.RaggedTensor.__or__ = ragged_elementwise_ops.logical_or
-ragged_tensor.RaggedTensor.__xor__ = ragged_elementwise_ops.logical_xor
-ragged_tensor.RaggedTensor.__rxor__ = _right(ragged_elementwise_ops.logical_xor)
+ragged_tensor.RaggedTensor.__and__ = math_ops.logical_and
+ragged_tensor.RaggedTensor.__rand__ = _right(math_ops.logical_and)
+ragged_tensor.RaggedTensor.__invert__ = math_ops.logical_not
+ragged_tensor.RaggedTensor.__ror__ = _right(math_ops.logical_or)
+ragged_tensor.RaggedTensor.__or__ = math_ops.logical_or
+ragged_tensor.RaggedTensor.__xor__ = math_ops.logical_xor
+ragged_tensor.RaggedTensor.__rxor__ = _right(math_ops.logical_xor)
 
 # Arithmetic operators
-ragged_tensor.RaggedTensor.__abs__ = ragged_elementwise_ops.abs
-ragged_tensor.RaggedTensor.__add__ = ragged_elementwise_ops.add
-ragged_tensor.RaggedTensor.__radd__ = _right(ragged_elementwise_ops.add)
-ragged_tensor.RaggedTensor.__div__ = ragged_elementwise_ops.div
-ragged_tensor.RaggedTensor.__rdiv__ = _right(ragged_elementwise_ops.div)
-ragged_tensor.RaggedTensor.__floordiv__ = ragged_elementwise_ops.floordiv
-ragged_tensor.RaggedTensor.__rfloordiv__ = _right(
-    ragged_elementwise_ops.floordiv)
-ragged_tensor.RaggedTensor.__mod__ = ragged_elementwise_ops.floormod
-ragged_tensor.RaggedTensor.__rmod__ = _right(ragged_elementwise_ops.floormod)
-ragged_tensor.RaggedTensor.__mul__ = ragged_elementwise_ops.multiply
-ragged_tensor.RaggedTensor.__rmul__ = _right(ragged_elementwise_ops.multiply)
-ragged_tensor.RaggedTensor.__neg__ = ragged_elementwise_ops.negative
-ragged_tensor.RaggedTensor.__pow__ = ragged_elementwise_ops.pow
-ragged_tensor.RaggedTensor.__rpow__ = _right(ragged_elementwise_ops.pow)
-ragged_tensor.RaggedTensor.__sub__ = ragged_elementwise_ops.subtract
-ragged_tensor.RaggedTensor.__rsub__ = _right(ragged_elementwise_ops.subtract)
-ragged_tensor.RaggedTensor.__truediv__ = ragged_elementwise_ops.truediv
-ragged_tensor.RaggedTensor.__rtruediv__ = _right(ragged_elementwise_ops.truediv)
+ragged_tensor.RaggedTensor.__abs__ = math_ops.abs
+ragged_tensor.RaggedTensor.__add__ = math_ops.add
+ragged_tensor.RaggedTensor.__radd__ = _right(math_ops.add)
+ragged_tensor.RaggedTensor.__div__ = math_ops.div
+ragged_tensor.RaggedTensor.__rdiv__ = _right(math_ops.div)
+ragged_tensor.RaggedTensor.__floordiv__ = math_ops.floordiv
+ragged_tensor.RaggedTensor.__rfloordiv__ = _right(math_ops.floordiv)
+ragged_tensor.RaggedTensor.__mod__ = math_ops.floormod
+ragged_tensor.RaggedTensor.__rmod__ = _right(math_ops.floormod)
+ragged_tensor.RaggedTensor.__mul__ = math_ops.multiply
+ragged_tensor.RaggedTensor.__rmul__ = _right(math_ops.multiply)
+ragged_tensor.RaggedTensor.__neg__ = math_ops.negative
+ragged_tensor.RaggedTensor.__pow__ = math_ops.pow
+ragged_tensor.RaggedTensor.__rpow__ = _right(math_ops.pow)
+ragged_tensor.RaggedTensor.__sub__ = math_ops.subtract
+ragged_tensor.RaggedTensor.__rsub__ = _right(math_ops.subtract)
+ragged_tensor.RaggedTensor.__truediv__ = math_ops.truediv
+ragged_tensor.RaggedTensor.__rtruediv__ = _right(math_ops.truediv)
 
 
 # Dummy methods
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index 346ab9c0cb..feff7df850 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -2696,6 +2696,7 @@ class _UnaryMapValueDispatcher(dispatch.OpDispatcher):
     if args:
       x, args = args[0], args[1:]
     else:
+      kwargs = kwargs.copy()
       x = kwargs.pop(self._x, None)
     if isinstance(x, sparse_tensor.SparseTensor):
       return sparse_tensor.SparseTensor(
diff --git a/tensorflow/python/ops/string_ops.py b/tensorflow/python/ops/string_ops.py
index b6b329c486..046459706c 100644
--- a/tensorflow/python/ops/string_ops.py
+++ b/tensorflow/python/ops/string_ops.py
@@ -38,6 +38,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.gen_string_ops import *
 from tensorflow.python.util import compat as util_compat
 from tensorflow.python.util import deprecation
+from tensorflow.python.util import dispatch
 from tensorflow.python.util.tf_export import tf_export
 # pylint: enable=g-bad-import-order
 # pylint: enable=wildcard-import
@@ -45,6 +46,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 # pylint: disable=redefined-builtin
 @tf_export("strings.regex_full_match")
+@dispatch.add_dispatch_support
 def regex_full_match(input, pattern, name=None):
   r"""Match elements of `input` with regex `pattern`.
 
@@ -76,6 +78,7 @@ regex_full_match.__doc__ = gen_string_ops.regex_full_match.__doc__
 @tf_export(
     "strings.regex_replace", v1=["strings.regex_replace", "regex_replace"])
 @deprecation.deprecated_endpoints("regex_replace")
+@dispatch.add_dispatch_support
 def regex_replace(input, pattern, rewrite, replace_global=True, name=None):
   r"""Replace elements of `input` matching regex `pattern` with `rewrite`.
 
@@ -350,10 +353,13 @@ reduce_join.__doc__ = reduce_join.__doc__.replace("tf.reduce_join(",
 # This wrapper provides backwards compatibility for code that predates the
 # unit argument and that passed 'name' as a positional argument.
 @tf_export(v1=["strings.length"])
+@dispatch.add_dispatch_support
 def string_length(input, name=None, unit="BYTE"):
   return gen_string_ops.string_length(input, unit=unit, name=name)
 
+
 @tf_export("strings.length", v1=[])
+@dispatch.add_dispatch_support
 def string_length_v2(input, unit="BYTE", name=None):
   return string_length(input, name, unit)
 
@@ -370,11 +376,13 @@ substr_deprecated.__doc__ = gen_string_ops.substr.__doc__
 
 
 @tf_export(v1=["strings.substr"])
+@dispatch.add_dispatch_support
 def substr(input, pos, len, name=None, unit="BYTE"):
   return gen_string_ops.substr(input, pos, len, unit=unit, name=name)
 
 
 @tf_export("strings.substr", v1=[])
+@dispatch.add_dispatch_support
 def substr_v2(input, pos, len, unit="BYTE", name=None):
   return substr(input, pos, len, name=name, unit=unit)
 
@@ -395,6 +403,7 @@ ops.NotDifferentiable("DecodeBase64")
 
 
 @tf_export("strings.to_number", v1=[])
+@dispatch.add_dispatch_support
 def string_to_number(input, out_type=dtypes.float32, name=None):
   r"""Converts each string in the input Tensor to the specified numeric type.
 
@@ -418,6 +427,7 @@ tf_export(v1=["strings.to_number", "string_to_number"])(
 
 
 @tf_export("strings.to_hash_bucket", v1=[])
+@dispatch.add_dispatch_support
 def string_to_hash_bucket(input, num_buckets, name=None):
   # pylint: disable=line-too-long
   r"""Converts each string in the input Tensor to its hash mod by a number of buckets.
diff --git a/tensorflow/python/util/dispatch.py b/tensorflow/python/util/dispatch.py
index e7a56b5922..e94e334534 100644
--- a/tensorflow/python/util/dispatch.py
+++ b/tensorflow/python/util/dispatch.py
@@ -166,15 +166,14 @@ def dispatch_for_types(op, *types):
 
 def add_dispatch_list(target):
   """Decorator that adds a dispatch_list attribute to an op."""
-  assert not hasattr(target, DISPATCH_ATTR)
+  if hasattr(target, DISPATCH_ATTR):
+    raise AssertionError("%s already has a dispatch list" % target)
   setattr(target, DISPATCH_ATTR, [])
   return target
 
 
 def add_dispatch_support(target):
   """Decorator that adds a dispatch handling wrapper to an op."""
-  add_dispatch_list(target)
-
   def wrapper(*args, **kwargs):
     """Call target, and fall back on dispatchers if there is a TypeError."""
     try:
@@ -188,5 +187,5 @@ def add_dispatch_support(target):
       else:
         raise
 
-  setattr(wrapper, DISPATCH_ATTR, [])
+  add_dispatch_list(wrapper)
   return tf_decorator.make_decorator(target, wrapper)
-- 
GitLab


From 3f43965a44f1bbf897c51f254ababd083087979d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 5 Dec 2018 14:55:02 -0800
Subject: [PATCH 1389/1554] [Grappler] Cleanup GrapplerItem constructors

PiperOrigin-RevId: 224226461
---
 tensorflow/core/grappler/grappler_item.cc     | 30 +++++++++--------
 tensorflow/core/grappler/grappler_item.h      | 11 ++++---
 .../optimizers/arithmetic_optimizer.cc        |  3 +-
 ...perimental_implementation_selector_test.cc |  4 +--
 .../optimizers/function_optimizer_test.cc     | 32 +++++++++----------
 .../grappler/optimizers/memory_optimizer.cc   |  7 ++--
 .../optimizers/memory_optimizer_test.cc       | 14 ++++----
 .../grappler/optimizers/meta_optimizer.cc     |  2 +-
 .../optimizers/meta_optimizer_test.cc         |  4 +--
 .../core/grappler/optimizers/remapper.cc      |  2 +-
 10 files changed, 56 insertions(+), 53 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index f7cda35368..74bde67f19 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -30,20 +30,22 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-GrapplerItem::GrapplerItem(const GrapplerItem& other, GraphDef* graph_def) {
-  id = other.id;
-  feed = other.feed;
-  fetch = other.fetch;
-  init_ops = other.init_ops;
-  keep_ops = other.keep_ops;
-  expected_init_time = other.expected_init_time;
-  save_op = other.save_op;
-  restore_op = other.restore_op;
-  save_restore_loc_tensor = other.save_restore_loc_tensor;
-  queue_runners = other.queue_runners;
-  devices_ = other.devices_;
-  allowed_optimizations_ = other.allowed_optimizations_;
-  graph.Swap(graph_def);
+GrapplerItem GrapplerItem::WithGraph(GraphDef&& graph_def) const {
+  GrapplerItem item;
+  item.id = id;
+  item.feed = feed;
+  item.fetch = fetch;
+  item.init_ops = init_ops;
+  item.keep_ops = keep_ops;
+  item.expected_init_time = expected_init_time;
+  item.save_op = save_op;
+  item.restore_op = restore_op;
+  item.save_restore_loc_tensor = save_restore_loc_tensor;
+  item.queue_runners = queue_runners;
+  item.devices_ = devices_;
+  item.allowed_optimizations_ = allowed_optimizations_;
+  item.graph.Swap(&graph_def);
+  return item;
 }
 
 std::vector<const NodeDef*> GrapplerItem::MainOpsFanin() const {
diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h
index 6ef4f14247..9051542988 100644
--- a/tensorflow/core/grappler/grappler_item.h
+++ b/tensorflow/core/grappler/grappler_item.h
@@ -35,12 +35,15 @@ namespace grappler {
 // nodes, and potentially a set of nodes to feed.
 struct GrapplerItem {
   GrapplerItem() = default;
-  GrapplerItem(const GrapplerItem& other, GraphDef&& graph_def)
-      : GrapplerItem(other, &graph_def) {}
-  // Swaps *graph_def with an empty GraphDef.
-  GrapplerItem(const GrapplerItem& other, GraphDef* graph_def);
+  GrapplerItem(const GrapplerItem& other) = default;
+  GrapplerItem(GrapplerItem&& other) = default;
+  GrapplerItem& operator=(const GrapplerItem& other) = default;
+  GrapplerItem& operator=(GrapplerItem&& other) = default;
   virtual ~GrapplerItem() = default;
 
+  // Create a copy of this GrapplerItem with graph swapped with the argument.
+  GrapplerItem WithGraph(GraphDef&& graph) const;
+
   string id;  // A unique id for this item
 
   // Inputs
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index e41b1cf684..d35c00f29e 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -3561,8 +3561,7 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
   // Set up helper data structures.
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  *optimized_graph = item.graph;
-  GrapplerItem optimized_item(item, optimized_graph);
+  GrapplerItem optimized_item(item);
   optimized_graph_ = &optimized_item.graph;
   node_map_.reset(new NodeMap(optimized_graph_));
 
diff --git a/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc b/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc
index e1ac7766d3..e330835e9b 100644
--- a/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc
+++ b/tensorflow/core/grappler/optimizers/experimental_implementation_selector_test.cc
@@ -127,7 +127,7 @@ TEST_F(ExperimentalImplementationSelectorTest, SwapImplementationEval) {
                                  test::AsScalar<float>(4.0f));
 
   TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   const auto twice_boosted_tensor = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(twice_boosted_tensor[0],
                                  test::AsScalar<float>(2.0f));
@@ -223,7 +223,7 @@ TEST_F(ExperimentalImplementationSelectorTest, SwapImplementationWithGradient) {
                                  test::AsScalar<float>(4.0f));
 
   TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   const auto twice_boosted_tensor = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(twice_boosted_tensor[0],
                                  test::AsScalar<float>(2.0f));
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 935fdc6409..c971eec3f4 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -108,7 +108,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_SimpleFunction) {
   item.fetch = {"z"};
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -184,7 +184,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_SkipErrorsIfGraphNotModified) {
   item.fetch = {"z1"};
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -284,7 +284,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_FixedTypeFunction) {
   item.fetch = {"z"};
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -368,7 +368,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_FunctionWithOutputMapping) {
   item.fetch = {"z"};
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -418,7 +418,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_FunctionWithInputForwarding) {
   item.feed.emplace_back("x4", test::AsScalar<float>(-1.0f));
   item.feed.emplace_back("x3", test::AsScalar<int>(1234));
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
   test::ExpectTensorEqual<float>(tensors_expected[1], tensors[1]);
@@ -549,7 +549,7 @@ TEST_F(FunctionOptimizerTest, InlineFunction_FunctionWithNestedFunctionCall) {
   item.feed.emplace_back("a", test::AsScalar<float>(2.0f));
   auto tensors_expected = EvaluateFetchNodes(item);
 
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
 
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
@@ -748,7 +748,7 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionSimpleFunction) {
   item.feed.emplace_back("a", pi);
   item.feed.emplace_back("b", pi);
 
-  GrapplerItem optimized(item, std::move(optimized_graph));
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
   auto tensors_expected = EvaluateFetchNodes(item);
   auto tensors = EvaluateFetchNodes(optimized);
   ASSERT_EQ(tensors_expected.size(), 1);
@@ -876,7 +876,7 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
   EXPECT_EQ(tensors_expected[0].flat<float>()(0), 4.0);  // mul
   EXPECT_EQ(tensors_expected[1].flat<float>()(0), 3.0);  // read variable
 
-  GrapplerItem optimized(item, std::move(optimized_graph));
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
   auto tensors = EvaluateFetchNodes(optimized);
   ASSERT_EQ(tensors.size(), 2);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
@@ -1019,7 +1019,7 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
   auto tensors_expected = EvaluateFetchNodes(item);
   ASSERT_EQ(tensors_expected.size(), 1);
 
-  GrapplerItem optimized(item, std::move(optimized_graph));
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -1067,7 +1067,7 @@ TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   item.feed.emplace_back("x", pi);
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -1131,7 +1131,7 @@ TEST_F(FunctionOptimizerTest, SpecializeIndirectFunctionXTimesTwo) {
   item.feed.emplace_back("x", pi);
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -1194,7 +1194,7 @@ TEST_F(FunctionOptimizerTest, SpecializeFunctionPushDownConstInput) {
   item.feed.emplace_back("x", pi);
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -1274,7 +1274,7 @@ TEST_F(FunctionOptimizerTest, SpecializeIndirectFunctionPushDownConstInput) {
   item.feed.emplace_back("x", pi);
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -1390,7 +1390,7 @@ TEST_F(FunctionOptimizerTest, SpecializeFunction_OncePerUniqueContext) {
   item.feed = {{"xf", pi}, {"yf", pi}, {"xi", four}, {"yi", four}};
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
 
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
@@ -1499,7 +1499,7 @@ TEST_F(FunctionOptimizerTest, SpecializeFunctionForUsedOutputTensors) {
   item.feed = {{"xf", pi}, {"yf", pi}};
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
 
   ASSERT_EQ(tensors_expected.size(), tensors.size());
@@ -1660,7 +1660,7 @@ TEST_F(FunctionOptimizerTest, SpecializeIndirectFunctionForUsedOutputTensors) {
   item.feed = {{"xf", pi}, {"yf", pi}};
 
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
 
   ASSERT_EQ(tensors_expected.size(), tensors.size());
diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer.cc b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
index 453db5d91e..227c2bb8b0 100644
--- a/tensorflow/core/grappler/optimizers/memory_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
@@ -1306,13 +1306,12 @@ Status RelaxAllocatorConstraints(GraphDef* optimized_graph) {
 
 Status MemoryOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                  GraphDef* optimized_graph) {
-  *optimized_graph = item.graph;
+  GrapplerItem optimized_item(item);
 
   RecomputationRewritingPass(optimization_level_,
-                             recomputation_targets_name_scope_, optimized_graph,
-                             item);
+                             recomputation_targets_name_scope_,
+                             &optimized_item.graph, item);
 
-  GrapplerItem optimized_item(item, optimized_graph);
   std::unordered_set<string> skip_list;
   // Bound the number of rewrite passes to avoid long processing times on graphs
   // that simply won't fit in memory.
diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc b/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc
index 75285b07bb..356b23dec0 100644
--- a/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc
@@ -279,7 +279,7 @@ TEST_F(MemoryOptimizerTest, SimpleSwapping) {
   EXPECT_EQ("^swap_out_e_0", new_c.input(1));
 
   // Run the optimizer a second time to ensure it's idempotent.
-  GrapplerItem item_copy(item, std::move(output));
+  GrapplerItem item_copy = item.WithGraph(std::move(output));
   status = optimizer.Optimize(cluster.get(), item_copy, &output);
   TF_EXPECT_OK(status);
 
@@ -287,7 +287,7 @@ TEST_F(MemoryOptimizerTest, SimpleSwapping) {
   item.fetch = {"e"};
   item.init_ops = {init.name()};
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 #endif
@@ -337,7 +337,7 @@ TEST_F(MemoryOptimizerTest, SwappingHeuristics) {
 
 #if GOOGLE_CUDA
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   for (int i = 0; i < item.fetch.size(); ++i) {
     test::ExpectTensorEqual<float>(tensors_expected[i], tensors[i]);
@@ -386,7 +386,7 @@ TEST_F(MemoryOptimizerTest, UnswappableInputs) {
 
 #if GOOGLE_CUDA
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 #endif
@@ -474,7 +474,7 @@ TEST_F(RelaxAllocatorConstraintsTest, SameDevice) {
   item.fetch = {"exp"};
   item.init_ops = {"variable"};
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
@@ -505,7 +505,7 @@ TEST_F(RelaxAllocatorConstraintsTest, DifferentDevice) {
   item.fetch = {"exp"};
   item.init_ops = {"variable"};
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 #endif
@@ -598,7 +598,7 @@ TEST_F(RelaxAllocatorConstraintsTest, AssignNodeInFanout) {
   item.fetch = {"assign0", "assign1"};
   item.init_ops = {"exp1", "variable1"};
   auto tensors_expected = EvaluateFetchNodes(item);
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
   for (int i = 0; i < tensors_expected.size(); ++i) {
     test::ExpectTensorEqual<float>(tensors_expected[i], tensors[i]);
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 7c83036341..572cc41d76 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -440,7 +440,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
           item.graph)
           .ToProto();
 
-  GrapplerItem trimmed_item(item, std::move(trimmed_graph));
+  GrapplerItem trimmed_item = item.WithGraph(std::move(trimmed_graph));
 
   VLOG(1) << absl::Substitute(
       "Deleted $0 unreachable functions from the graph (library size = $1)",
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index 42b867b6ac..12db5d6ca9 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -396,7 +396,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   item.feed.emplace_back("b", test::AsScalar<int>(4));
   auto tensors_expected = EvaluateFetchNodes(item);
 
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
 
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
@@ -502,7 +502,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneFunctionBody) {
   item.feed.emplace_back("b", test::AsScalar<float>(3.123f));
   auto tensors_expected = EvaluateFetchNodes(item);
 
-  GrapplerItem optimized(item, std::move(output));
+  GrapplerItem optimized = item.WithGraph(std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
 
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index d8e62e0b24..3fb3f2b0ec 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -665,7 +665,7 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
   std::reverse(topo_sorted_graph.mutable_node()->begin(),
                topo_sorted_graph.mutable_node()->end());
 
-  GrapplerItem topo_sorted_item(item, std::move(topo_sorted_graph));
+  GrapplerItem topo_sorted_item = item.WithGraph(std::move(topo_sorted_graph));
   RemapperContext ctx(topo_sorted_item);
 
   // Skip nodes that were invalidated by a remapper, e.g. do not process BiasAdd
-- 
GitLab


From 5abe466090e4e115389a017634880907f5a0c172 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 5 Dec 2018 15:08:01 -0800
Subject: [PATCH 1390/1554] Workaround for bizarre issue related to aliasing
 tensors in cond_v2's transform

PiperOrigin-RevId: 224228895
---
 tensorflow/python/keras/layers/normalization.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index d958497655..37894a3d3d 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import contextlib
+
 from tensorflow.python import tf2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -412,11 +414,19 @@ class BatchNormalizationV2(Layer):
   def _assign_moving_average(self, variable, value, momentum):
     with ops.name_scope(None, 'AssignMovingAvg',
                         [variable, value, momentum]) as scope:
-      with ops.colocate_with(variable):
+      # TODO(apassos,srbs,skyewm): the colocation constraints here are disabled
+      # because of a bug which leads cond_v2 to skip rewriting them creating
+      # conflicts.
+      if tf2.enabled():
+        cm = contextlib.contextmanager(lambda: (yield))
+      else:
+        cm = ops.colocate_with(variable)
+      with cm:
         decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
         if decay.dtype != variable.dtype.base_dtype:
           decay = math_ops.cast(decay, variable.dtype.base_dtype)
-        update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay
+        update_delta = (
+            variable - math_ops.cast(value, variable.dtype)) * decay
         return state_ops.assign_sub(variable, update_delta, name=scope)
 
   def _fused_batch_norm(self, inputs, training):
-- 
GitLab


From c5d064bd4b1e15d811550023a17f6ac3eadbadc9 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 5 Dec 2018 15:14:07 -0800
Subject: [PATCH 1391/1554] Create a list of arguments for reordered functions
 automatically.

PiperOrigin-RevId: 224229831
---
 tensorflow/tools/compatibility/BUILD          |  17 +-
 tensorflow/tools/compatibility/reorders_v2.py |  98 +++++++
 .../tools/compatibility/tf_upgrade_v2.py      | 250 +++++-------------
 .../tools/compatibility/tf_upgrade_v2_test.py |  44 +++
 tensorflow/tools/compatibility/update/BUILD   |  14 +
 .../update/generate_v2_reorders_map.py        | 166 ++++++++++++
 6 files changed, 406 insertions(+), 183 deletions(-)
 create mode 100644 tensorflow/tools/compatibility/reorders_v2.py
 create mode 100644 tensorflow/tools/compatibility/update/generate_v2_reorders_map.py

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 05d924c092..a9902d77f5 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -51,14 +51,21 @@ py_library(
     srcs_version = "PY2AND3",
 )
 
+py_library(
+    name = "reorders_v2",
+    srcs = ["reorders_v2.py"],
+    srcs_version = "PY2AND3",
+)
+
 py_library(
     name = "tf_upgrade_v2_lib",
-    srcs = [
-        "renames_v2.py",
-        "tf_upgrade_v2.py",
-    ],
+    srcs = ["tf_upgrade_v2.py"],
     srcs_version = "PY2AND3",
-    deps = [":ast_edits"],
+    deps = [
+        ":ast_edits",
+        ":renames_v2",
+        ":reorders_v2",
+    ],
 )
 
 py_binary(
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
new file mode 100644
index 0000000000..4152d37db9
--- /dev/null
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -0,0 +1,98 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+"""List of renames to apply when converting from TF 1.0 to TF 2.0.
+
+THIS FILE IS AUTOGENERATED: To update, please run:
+  bazel build tensorflow/tools/compatibility/update:generate_v2_reorders_map
+  bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
+This file should be updated whenever a function is added to
+self.reordered_function_names in tf_upgrade_v2.py.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+reorders = {
+    'tf.argmax': ['input', 'axis', 'name', 'dimension', 'output_type'],
+    'tf.argmin': ['input', 'axis', 'name', 'dimension', 'output_type'],
+    'tf.batch_to_space': ['input', 'crops', 'block_size', 'name'],
+    'tf.boolean_mask': ['tensor', 'mask', 'name', 'axis'],
+    'tf.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
+    'tf.convert_to_tensor': ['value', 'dtype', 'name', 'preferred_dtype'],
+    'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
+    'tf.feature_column.categorical_column_with_vocabulary_file': ['key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'default_value', 'dtype'],
+    'tf.io.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
+    'tf.io.parse_example': ['serialized', 'features', 'name', 'example_names'],
+    'tf.io.parse_single_example': ['serialized', 'features', 'name', 'example_names'],
+    'tf.io.serialize_many_sparse': ['sp_input', 'name', 'out_type'],
+    'tf.io.serialize_sparse': ['sp_input', 'name', 'out_type'],
+    'tf.math.argmax': ['input', 'axis', 'name', 'dimension', 'output_type'],
+    'tf.math.argmin': ['input', 'axis', 'name', 'dimension', 'output_type'],
+    'tf.math.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
+    'tf.math.reduce_all': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_any': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_logsumexp': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_max': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_mean': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_min': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_prod': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.math.reduce_sum': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.multinomial': ['logits', 'num_samples', 'seed', 'name', 'output_dtype'],
+    'tf.nn.convolution': ['input', 'filter', 'padding', 'strides', 'dilation_rate', 'name', 'data_format'],
+    'tf.nn.crelu': ['features', 'name', 'axis'],
+    'tf.nn.depthwise_conv2d': ['input', 'filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
+    'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
+    'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
+    'tf.pad': ['tensor', 'paddings', 'mode', 'name', 'constant_values'],
+    'tf.parse_example': ['serialized', 'features', 'name', 'example_names'],
+    'tf.parse_single_example': ['serialized', 'features', 'name', 'example_names'],
+    'tf.quantize_v2': ['input', 'min_range', 'max_range', 'T', 'mode', 'name', 'round_mode'],
+    'tf.random.multinomial': ['logits', 'num_samples', 'seed', 'name', 'output_dtype'],
+    'tf.random.poisson': ['lam', 'shape', 'dtype', 'seed', 'name'],
+    'tf.random_poisson': ['lam', 'shape', 'dtype', 'seed', 'name'],
+    'tf.reduce_all': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_any': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices'],
+    'tf.reduce_logsumexp': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_max': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_mean': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_min': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_prod': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reduce_sum': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.serialize_many_sparse': ['sp_input', 'name', 'out_type'],
+    'tf.serialize_sparse': ['sp_input', 'name', 'out_type'],
+    'tf.shape': ['input', 'name', 'out_type'],
+    'tf.size': ['input', 'name', 'out_type'],
+    'tf.sparse.add': ['a', 'b', 'threshold', 'thresh'],
+    'tf.sparse.concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
+    'tf.sparse.segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse.segment_sqrt_n': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse.segment_sum': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse_add': ['a', 'b', 'threshold', 'thresh'],
+    'tf.sparse_concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
+    'tf.sparse_matmul': ['a', 'b', 'transpose_a', 'transpose_b', 'a_is_sparse', 'b_is_sparse', 'name'],
+    'tf.sparse_segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse_segment_sqrt_n': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse_segment_sum': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.strings.length': ['input', 'name', 'unit'],
+    'tf.strings.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices'],
+    'tf.strings.substr': ['input', 'pos', 'len', 'name', 'unit'],
+    'tf.transpose': ['a', 'perm', 'name', 'conjugate'],
+    'tf.tuple': ['tensors', 'name', 'control_inputs'],
+    'tf.while_loop': ['cond', 'body', 'loop_vars', 'shape_invariants', 'parallel_iterations', 'back_prop', 'swap_memory', 'name', 'maximum_iterations', 'return_same_structure']
+}
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index d19eb13336..edfa2f88c1 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
+from tensorflow.tools.compatibility import reorders_v2
 
 
 class TFAPIChangeSpec(ast_edits.APIChangeSpec):
@@ -483,187 +484,80 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Variables that should be changed to functions.
     self.change_to_function = {}
 
+    # pylint: disable=line-too-long
+    # This list should just contain names of functions that had
+    # their arguments reordered. After adding a function name to the list
+    # run the following to update reorders_v2.py:
+    # bazel build tensorflow/tools/compatibility/update:generate_v2_reorders_map
+    # bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
+    # pylint: enable=line-too-long
+    self.reordered_function_names = {
+        "tf.io.serialize_sparse",
+        "tf.io.serialize_many_sparse",
+        "tf.argmax",
+        "tf.argmin",
+        "tf.batch_to_space",
+        "tf.boolean_mask",
+        "tf.convert_to_tensor",
+        "tf.nn.moments",
+        "tf.nn.convolution",
+        "tf.nn.crelu",
+        "tf.nn.weighted_moments",
+        "tf.nn.pool",
+        "tf.nn.separable_conv2d",
+        "tf.nn.depthwise_conv2d",
+        "tf.multinomial",
+        "tf.random.multinomial",
+        "tf.pad",
+        "tf.quantize_v2",
+        "tf.feature_column.categorical_column_with_vocabulary_file",
+        "tf.shape",
+        "tf.size",
+        "tf.random.poisson",
+        "tf.sparse.add",
+        "tf.sparse_add",
+        "tf.sparse.concat",
+        "tf.sparse_concat",
+        "tf.sparse.segment_mean",
+        "tf.sparse.segment_sqrt_n",
+        "tf.sparse.segment_sum",
+        "tf.sparse_matmul",
+        "tf.io.decode_csv",
+        "tf.strings.substr",
+        "tf.strings.reduce_join",
+        "tf.strings.length",
+        "tf.transpose",
+        "tf.tuple",
+        "tf.parse_example",
+        "tf.parse_single_example",
+        "tf.io.parse_example",
+        "tf.io.parse_single_example",
+        "tf.while_loop",
+        "tf.reduce_all",
+        "tf.math.reduce_all",
+        "tf.reduce_any",
+        "tf.math.reduce_any",
+        "tf.reduce_min",
+        "tf.math.reduce_min",
+        "tf.reduce_max",
+        "tf.math.reduce_max",
+        "tf.reduce_sum",
+        "tf.math.reduce_sum",
+        "tf.reduce_mean",
+        "tf.math.reduce_mean",
+        "tf.reduce_prod",
+        "tf.math.reduce_prod",
+        "tf.reduce_logsumexp",
+        "tf.math.reduce_logsumexp",
+        "tf.reduce_join",
+        "tf.confusion_matrix",
+        "tf.math.confusion_matrix",
+    }
+
     # Functions that were reordered should be changed to the new keyword args
     # for safety, if positional arguments are used. If you have reversed the
     # positional arguments yourself, this could do the wrong thing.
-    # IMPORTANT: order here should correspond to OLD argument order.
-    # We just prepend "arg_name=" to all arguments in function calls.
-    self.function_reorders = {
-        "tf.io.serialize_sparse": ["sp_input", "name", "out_type"],
-        "tf.io.serialize_many_sparse": ["sp_input", "name", "out_type"],
-        "tf.argmax": ["input", "axis", "name", "axis", "output_type"],
-        "tf.argmin": ["input", "axis", "name", "axis", "output_type"],
-        "tf.batch_to_space": ["input", "crops", "block_size", "name"],
-        "tf.boolean_mask": ["tensor", "mask", "name", "axis"],
-        "tf.convert_to_tensor": ["value", "dtype", "name", "preferred_dtype"],
-        "tf.nn.moments": ["x", "axes", "shift", "keepdims", "name"],
-        "tf.nn.convolution": [
-            "input", "filter", "padding", "strides", "dilation_rate", "name",
-            "data_format"
-        ],
-        "tf.nn.crelu": ["features", "name", "axis"],
-        "tf.nn.weighted_moments": [
-            "x", "axes", "frequency_weights", "name", "keep_dims"
-        ],
-        "tf.nn.pool": [
-            "input", "window_shape", "pooling_type", "padding", "dilation_rate",
-            "strides", "name", "data_format"
-        ],
-        "tf.nn.separable_conv2d": [
-            "input", "depthwise_filter", "pointwise_filter", "strides",
-            "padding", "rate", "name", "data_format"
-        ],
-        "tf.nn.depthwise_conv2d": [
-            "input", "filter", "strides", "padding", "rate", "name",
-            "data_format"
-        ],
-        "tf.multinomial": [
-            "logits", "num_samples", "seed", "name", "output_dtype"
-        ],
-        "tf.random.multinomial": [
-            "logits", "num_samples", "seed", "name", "output_dtype"
-        ],
-        "tf.pad": ["tensor", "paddings", "mode", "name", "constant_values"],
-        "tf.quantize_v2": [
-            "input", "min_range", "max_range", "T", "mode", "name", "round_mode"
-        ],
-        "tf.feature_column.categorical_column_with_vocabulary_file": [
-            "key", "vocabulary_file", "vocabulary_size", "num_oov_buckets",
-            "default_value", "dtype"
-        ],
-        "tf.shape": ["input", "name", "out_type"],
-        "tf.size": ["input", "name", "out_type"],
-        "tf.random.poisson": ["lam", "shape", "dtype", "seed", "name"],
-        "tf.sparse.add": ["a", "b", "thresh"],
-        "tf.sparse_add": ["a", "b", "thresh"],
-        "tf.sparse.concat": [
-            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
-        ],
-        "tf.sparse_concat": [
-            "axis", "sp_inputs", "name", "expand_nonconcat_dim", "concat_dim"
-        ],
-        "tf.sparse.segment_mean": [
-            "data", "indices", "segment_ids", "name", "num_segments"
-        ],
-        "tf.sparse.segment_sqrt_n": [
-            "data", "indices", "segment_ids", "name", "num_segments"
-        ],
-        "tf.sparse.segment_sum": [
-            "data", "indices", "segment_ids", "name", "num_segments"
-        ],
-        "tf.sparse_matmul": [
-            "a", "b", "transpose_a", "transpose_b", "a_is_sparse",
-            "b_is_sparse", "name"
-        ],
-        "tf.io.decode_csv": [
-            "records",
-            "record_defaults",
-            "field_delim",
-            "use_quote_delim",
-            "name",
-            "na_value",
-            "select_cols",
-        ],
-        "tf.strings.substr": ["input", "pos", "len", "name", "unit"],
-        "tf.strings.reduce_join": [
-            "input", "axis", "keep_dims", "separator", "name",
-            "reduction_indices"
-        ],
-        "tf.strings.length": ["input", "name", "unit"],
-        "tf.transpose": ["a", "perm", "name", "conjugate"],
-        "tf.tuple": ["tensors", "name", "control_inputs"],
-        "tf.parse_example": [
-            "serialized", "features", "name", "example_names"
-        ],
-        "tf.parse_single_example": [
-            "serialized", "features", "name", "example_names"
-        ],
-        "tf.io.parse_example": [
-            "serialized", "features", "name", "example_names"
-        ],
-        "tf.io.parse_single_example": [
-            "serialized", "features", "name", "example_names"
-        ],
-        "tf.while_loop": [
-            "cond", "body", "loop_vars", "shape_invariants",
-            "parallel_iterations", "back_prop", "swap_memory", "name",
-            "maximum_iterations", "return_same_structure"
-        ],
-        "tf.reduce_all": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_all": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_any": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_any": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_min": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_min": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_max": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_max": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_sum": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_sum": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_mean": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_mean": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_prod": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_prod": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_logsumexp": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.math.reduce_logsumexp": [
-            "input_tensor", "axis", "keepdims", "name", "reduction_indices",
-            "keep_dims"
-        ],
-        "tf.reduce_join": [
-            "input", "axis", "keep_dims", "separator", "name",
-            "reduction_indices"
-        ],
-        "tf.confusion_matrix": [
-            "labels", "predictions", "num_classes", "dtype", "name", "weights"
-        ],
-        "tf.math.confusion_matrix": [
-            "labels", "predictions", "num_classes", "dtype", "name", "weights"
-        ]
-    }
+    self.function_reorders = reorders_v2.reorders
 
     # Specially handled functions.
     self.function_handle = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 7c26718edb..57700c07d6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -63,6 +63,15 @@ def get_v2_names(symbol):
   return list(names_v2)
 
 
+def get_symbol_for_name(root, name):
+  name_parts = name.split(".")
+  symbol = root
+  # Iterate starting with second item since 1st item is "tf.".
+  for part in name_parts[1:]:
+    symbol = getattr(symbol, part)
+  return symbol
+
+
 def get_func_and_args_from_str(call_str):
   """Parse call string to get function and argument names.
 
@@ -246,6 +255,41 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     visitor.private_map["tf.compat"] = ["v1", "v2"]
     traverse.traverse(tf.compat.v1, visitor)
 
+  def testReorderFileNeedsUpdate(self):
+    reordered_function_names = (
+        tf_upgrade_v2.TFAPIChangeSpec().reordered_function_names)
+    function_reorders = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_reorders)
+
+    added_names_message = """Some function names in
+self.reordered_function_names are not in reorders_v2.py.
+Please run the following commands to update reorders_v2.py:
+bazel build tensorflow/tools/compatibility/update:generate_v2_reorders_map
+bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
+"""
+    removed_names_message = """%s in self.reorders_v2 does not match
+any name in self.reordered_function_names.
+Please run the following commands to update reorders_v2.py:
+bazel build tensorflow/tools/compatibility/update:generate_v2_reorders_map
+bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
+"""
+    self.assertTrue(
+        reordered_function_names.issubset(function_reorders),
+        added_names_message)
+    # function_reorders should contain reordered_function_names
+    # and their TensorFlow V1 aliases.
+    for name in function_reorders:
+      # get other names for this function
+      attr = get_symbol_for_name(tf.compat.v1, name)
+      _, attr = tf_decorator.unwrap(attr)
+      v1_names = get_v1_names(attr)
+      self.assertTrue(v1_names)
+      v1_names = ["tf.%s" % n for n in v1_names]
+      # check if any other name is in
+      self.assertTrue(
+          any(n in reordered_function_names for n in v1_names),
+          removed_names_message % name)
+
   def testRenameConstant(self):
     text = "tf.MONOLITHIC_BUILD\n"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
diff --git a/tensorflow/tools/compatibility/update/BUILD b/tensorflow/tools/compatibility/update/BUILD
index b9725a74ee..75bb0cfd2b 100644
--- a/tensorflow/tools/compatibility/update/BUILD
+++ b/tensorflow/tools/compatibility/update/BUILD
@@ -15,3 +15,17 @@ py_binary(
         "//tensorflow/tools/compatibility:tf_upgrade_v2_lib",
     ],
 )
+
+py_binary(
+    name = "generate_v2_reorders_map",
+    srcs = ["generate_v2_reorders_map.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:no_contrib",
+        "//tensorflow/tools/common:public_api",
+        "//tensorflow/tools/common:traverse",
+        "//tensorflow/tools/compatibility:tf_upgrade_v2_lib",
+    ],
+)
diff --git a/tensorflow/tools/compatibility/update/generate_v2_reorders_map.py b/tensorflow/tools/compatibility/update/generate_v2_reorders_map.py
new file mode 100644
index 0000000000..63541771bf
--- /dev/null
+++ b/tensorflow/tools/compatibility/update/generate_v2_reorders_map.py
@@ -0,0 +1,166 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+"""Script for updating tensorflow/tools/compatibility/reorders_v2.py.
+
+To update reorders_v2.py, run:
+  bazel build tensorflow/tools/compatibility/update:generate_v2_reorders_map
+  bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
+"""
+# pylint: enable=line-too-long
+import tensorflow as tf
+
+# This import is needed so that TensorFlow python modules are in sys.modules.
+from tensorflow import python as tf_python  # pylint: disable=unused-import
+from tensorflow.python.lib.io import file_io
+from tensorflow.python.platform import app
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_export
+from tensorflow.python.util import tf_inspect
+from tensorflow.tools.common import public_api
+from tensorflow.tools.common import traverse
+from tensorflow.tools.compatibility import tf_upgrade_v2
+
+
+_OUTPUT_FILE_PATH = 'third_party/tensorflow/tools/compatibility/reorders_v2.py'
+_FILE_HEADER = """# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+\"\"\"List of renames to apply when converting from TF 1.0 to TF 2.0.
+
+THIS FILE IS AUTOGENERATED: To update, please run:
+  bazel build tensorflow/tools/compatibility/update:generate_v2_reorders_map
+  bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
+This file should be updated whenever a function is added to
+self.reordered_function_names in tf_upgrade_v2.py.
+\"\"\"
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+"""
+
+_TENSORFLOW_API_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.TENSORFLOW_API_NAME].names)
+_TENSORFLOW_API_ATTR = tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].names
+_TENSORFLOW_CONSTANTS_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.TENSORFLOW_API_NAME].constants)
+_TENSORFLOW_CONSTANTS_ATTR = (
+    tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].constants)
+
+_ESTIMATOR_API_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].names)
+_ESTIMATOR_API_ATTR = tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].names
+_ESTIMATOR_CONSTANTS_ATTR_V1 = (
+    tf_export.API_ATTRS_V1[tf_export.ESTIMATOR_API_NAME].constants)
+_ESTIMATOR_CONSTANTS_ATTR = (
+    tf_export.API_ATTRS[tf_export.ESTIMATOR_API_NAME].constants)
+
+
+def get_v1_names(symbol):
+  names_v1 = []
+  if hasattr(symbol, _TENSORFLOW_API_ATTR_V1):
+    names_v1.extend(getattr(symbol, _TENSORFLOW_API_ATTR_V1))
+  if hasattr(symbol, _ESTIMATOR_API_ATTR_V1):
+    names_v1.extend(getattr(symbol, _ESTIMATOR_API_ATTR_V1))
+  return names_v1
+
+
+def get_v2_names(symbol):
+  names_v2 = []
+  if hasattr(symbol, _TENSORFLOW_API_ATTR):
+    names_v2.extend(getattr(symbol, _TENSORFLOW_API_ATTR))
+  if hasattr(symbol, _ESTIMATOR_API_ATTR):
+    names_v2.extend(getattr(symbol, _ESTIMATOR_API_ATTR))
+  return list(names_v2)
+
+
+def collect_function_arg_names(function_names):
+  """Determines argument names for reordered function signatures.
+
+  Args:
+    function_names: Functions to collect arguments for.
+
+  Returns:
+    Dictionary mapping function name to its arguments.
+  """
+  # Map from reordered function name to its arguments.
+  function_to_args = {}
+
+  def visit(unused_path, unused_parent, children):
+    """Visitor that collects arguments for reordered functions."""
+    for child in children:
+      _, attr = tf_decorator.unwrap(child[1])
+      api_names_v1 = get_v1_names(attr)
+      api_names_v1 = ['tf.%s' % name for name in api_names_v1]
+      matches_function_names = any(
+          name in function_names for name in api_names_v1)
+      if matches_function_names:
+        arg_list = tf_inspect.getargspec(attr)[0]
+        for name in api_names_v1:
+          function_to_args[name] = arg_list
+
+  visitor = public_api.PublicAPIVisitor(visit)
+  visitor.do_not_descend_map['tf'].append('contrib')
+  visitor.do_not_descend_map['tf.compat'] = ['v1', 'v2']
+  traverse.traverse(tf, visitor)
+
+  return function_to_args
+
+
+def get_reorder_line(name, arg_list):
+  return '    \'%s\': %s' % (name, str(arg_list))
+
+
+def update_reorders_v2(output_file_path):
+  """Writes a Python dictionary mapping function name to argument order.
+
+  Args:
+    output_file_path: File path to write output to. Any existing contents
+      would be replaced.
+  """
+  reordered_function_names = (
+      tf_upgrade_v2.TFAPIChangeSpec().reordered_function_names)
+
+  all_reorders = collect_function_arg_names(reordered_function_names)
+
+  # List of reorder lines to write to output file in the form:
+  #   'tf.function_name': ['arg1', 'arg2', ...]
+  rename_lines = [
+      get_reorder_line(name, arg_names)
+      for name, arg_names in all_reorders.items()]
+  renames_file_text = '%sreorders = {\n%s\n}\n' % (
+      _FILE_HEADER, ',\n'.join(sorted(rename_lines)))
+  file_io.write_string_to_file(output_file_path, renames_file_text)
+
+
+def main(unused_argv):
+  update_reorders_v2(_OUTPUT_FILE_PATH)
+
+
+if __name__ == '__main__':
+  app.run(main=main)
-- 
GitLab


From 320dc4913f94a679790d90f5c577037276aa9599 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 15:16:45 -0800
Subject: [PATCH 1392/1554] Skip importing IO tensors if custom input_arrays
 and output_arrays have been provided

PiperOrigin-RevId: 224230220
---
 tensorflow/lite/toco/tflite/import.cc | 32 +++++++++++++++++----------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/toco/tflite/import.cc b/tensorflow/lite/toco/tflite/import.cc
index 88028aa144..1692f72125 100644
--- a/tensorflow/lite/toco/tflite/import.cc
+++ b/tensorflow/lite/toco/tflite/import.cc
@@ -165,21 +165,28 @@ void ImportOperators(
   }
 }
 
-void ImportIOTensors(const ::tflite::Model& input_model,
+void ImportIOTensors(const ModelFlags& model_flags,
+                     const ::tflite::Model& input_model,
                      const details::TensorsTable& tensors_table, Model* model) {
-  auto inputs = (*input_model.subgraphs())[0]->inputs();
-  if (inputs) {
-    for (int input : *inputs) {
-      const string& input_name = tensors_table.at(input);
-      model->flags.add_input_arrays()->set_name(input_name);
+  // Import from the first subgraph if input arrays have not been specified.
+  if (model_flags.input_arrays().empty()) {
+    auto inputs = (*input_model.subgraphs())[0]->inputs();
+    if (inputs) {
+      for (int input : *inputs) {
+        const string& input_name = tensors_table.at(input);
+        model->flags.add_input_arrays()->set_name(input_name);
+      }
     }
   }
 
-  auto outputs = (*input_model.subgraphs())[0]->outputs();
-  if (outputs) {
-    for (int output : *outputs) {
-      const string& output_name = tensors_table.at(output);
-      model->flags.add_output_arrays(output_name);
+  // Import from the first subgraph if output arrays have not been specified.
+  if (model_flags.output_arrays().empty()) {
+    auto outputs = (*input_model.subgraphs())[0]->outputs();
+    if (outputs) {
+      for (int output : *outputs) {
+        const string& output_name = tensors_table.at(output);
+        model->flags.add_output_arrays(output_name);
+      }
     }
   }
 }
@@ -219,7 +226,8 @@ std::unique_ptr<Model> Import(const ModelFlags& model_flags,
   ImportTensors(*input_model, model.get());
   ImportOperators(*input_model, ops_by_name, tensors_table, operators_table,
                   model.get());
-  ImportIOTensors(*input_model, tensors_table, model.get());
+
+  ImportIOTensors(model_flags, *input_model, tensors_table, model.get());
 
   UndoWeightsShuffling(model.get());
 
-- 
GitLab


From b428d3a8522f20d477c7a633a46a18e85108b721 Mon Sep 17 00:00:00 2001
From: Jeremy Lau <lauj@google.com>
Date: Wed, 5 Dec 2018 15:28:32 -0800
Subject: [PATCH 1393/1554] TPUEstimator: Run tpu.initialize_system in a
 separate graph. This speeds up initialization by avoiding unnecessary graph
 processing.

Resubmit with a fix for eval initialization. For eval, the master is
self._config.evaluation_master, not self._config.master.

PiperOrigin-RevId: 224232149
---
 tensorflow/contrib/tpu/BUILD                  |  1 +
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 31 +++++++++++++------
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 05d2ebd2e8..4bf3a0463d 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -79,6 +79,7 @@ py_library(
         "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
+        "//tensorflow/python:session",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:summary",
         "//tensorflow/python:summary_ops_v2",
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index a9dc542ae5..fb1316cf33 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -45,6 +45,7 @@ from tensorflow.contrib.training.python.training import hparam
 from tensorflow.core.framework import variable_pb2
 from tensorflow.core.framework.summary_pb2 import Summary
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.client import session as tf_session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest as data_nest
 from tensorflow.python.estimator import estimator as estimator_lib
@@ -412,12 +413,15 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
                enqueue_ops,
                dequeue_ops,
                run_infeed_loop_on_coordinator=True,
-               rendezvous=None):
+               rendezvous=None,
+               master=None,
+               session_config=None):
     self._master_job = ctx.master_job
     self._enqueue_ops = enqueue_ops
     self._dequeue_ops = dequeue_ops
     self._rendezvous = rendezvous
-
+    self._master = master
+    self._session_config = session_config
     self._run_infeed_loop_on_coordinator = run_infeed_loop_on_coordinator
     self._initial_infeed_sleep_secs = (
         ctx.config.tpu_config.initial_infeed_sleep_secs)
@@ -429,11 +433,10 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
   def begin(self):
     logging.info('TPU job name %s', self._master_job)
     self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
+    self._init_ops = []
     if self._should_initialize_tpu:
-      self._init_ops = [tpu.initialize_system(job=self._master_job)]
       self._finalize_ops = [tpu.shutdown_system(job=self._master_job)]
     else:
-      self._init_ops = []
       self._finalize_ops = []
 
     summary_writer_init_ops = contrib_summary.summary_writer_initializer_op()
@@ -475,11 +478,17 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
     return _OpQueueContext(name=name, target=target, args=args)
 
   def after_create_session(self, session, coord):
-    logging.info('Init TPU system')
-    start = time.time()
+    if self._should_initialize_tpu:
+      logging.info('Init TPU system')
+      start = time.time()
+      with ops.Graph().as_default():
+        with tf_session.Session(
+            self._master, config=self._session_config) as sess:
+          sess.run(tpu.initialize_system(job=self._master_job))
+      logging.info('Initialized TPU in %d seconds', time.time() - start)
+
     session.run(self._init_ops,
                 options=config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000))
-    logging.info('Initialized TPU in %d seconds', time.time() - start)
 
     self._infeed_controller = self._create_infeed_controller(
         name='InfeedController', target=self._run_infeed, args=(session,))
@@ -2561,6 +2570,8 @@ class TPUEstimator(estimator_lib.Estimator):
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator),
                   rendezvous=self._rendezvous[mode],
+                  master=self._config.master,
+                  session_config=self._session_config,
               ),
               InstallSignalHandlerHook()
           ])
@@ -2663,8 +2674,10 @@ class TPUEstimator(estimator_lib.Estimator):
                   eval_update_ops + host_ops,
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator),
-                  rendezvous=self._rendezvous[mode]),
-          ] + input_hooks
+                  rendezvous=self._rendezvous[mode],
+                  master=self._config.evaluation_master,
+                  session_config=self._session_config,
+              )] + input_hooks
 
           if eval_hooks:
             hooks.extend(eval_hooks)
-- 
GitLab


From 6c525df5abf699e810278f44904a479e04cd2001 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 5 Dec 2018 15:31:33 -0800
Subject: [PATCH 1394/1554] Move TPU initialize to TPUStrategy constructor
 instead being Run inside Keras/Estimator

PiperOrigin-RevId: 224232654
---
 .../contrib/distribute/python/tpu_strategy.py | 48 +++++++++++++------
 tensorflow/python/training/session_manager.py |  7 ---
 2 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 39ed8f7cf1..806ff0ac61 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -28,6 +28,8 @@ from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.client import session as session_lib
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
@@ -43,12 +45,10 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
-_TPU_INITIALIZE_SYSTEM_COLLECTION = "TPU_STRATEGY_INITIALIZE"
-
-
 def get_tpu_system_metadata(tpu_cluster_resolver):
   """Retrieves TPU system metadata given a TPUClusterResolver."""
   master = tpu_cluster_resolver.master()
@@ -145,6 +145,9 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
 class TPUExtended(distribute_lib.DistributionStrategyExtended):
   """Implementation of TPUStrategy."""
 
+  # Track what TPU devices have been initialized.
+  _initialized_devices = []
+
   def __init__(self, container_strategy, tpu_cluster_resolver, steps_per_run,
                num_cores=None):
     super(TPUExtended, self).__init__(container_strategy)
@@ -166,9 +169,34 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     # TODO(sourabhbajaj): Remove this once performance of running one step
     # at a time is comparable to multiple steps.
     self.steps_per_run = steps_per_run
-
     self._require_static_shapes = True
 
+    # Initialize the TPU devices.
+    self._initialize_tpu()
+
+  def _initialize_tpu(self):
+    """Initialize the TPU devices in a separate session and graph.
+
+    We keep track of all the TPU devices that we're initialized as we should
+    only be running TPU initialize once for the entire process.
+    """
+    master = self._tpu_cluster_resolver.master()
+    # Verify TPU has not already been initialized in this process.
+    if master in TPUExtended._initialized_devices:
+      logging.info("TPU master %s has already been initialized." % master)
+      return
+
+    logging.info("Initializing the TPU system.")
+    session_config = config_pb2.ConfigProto(allow_soft_placement=True)
+    self._configure(session_config)
+    with ops.Graph().as_default():
+      with session_lib.Session(config=session_config, target=master) as sess:
+        sess.run([tpu.initialize_system()])
+    logging.info("Finized initializing TPU system.")
+
+    # Update Strategy state to make sure we can track device initialization.
+    TPUExtended._initialized_devices.append(master)
+
   def _get_enqueue_op_per_host(self, host_id, multi_worker_iterator,
                                input_shapes, iterations):
     """Create an enqueue op for a single host identified using host_id.
@@ -380,22 +408,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
       raise NotImplementedError("Eager mode not supported in TPUStrategy.")
     else:
-      # TODO(jhseu): We need this hack because DistributionStrategies must be
-      # pickleable for copy.deepcopy(). Remove when initialize_system goes away.
-      graph = ops.get_default_graph()
-      tpu_init = graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
-      if tpu_init:
-        return tpu_init
-      graph.add_to_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION,
-                              tpu.initialize_system())
-      return graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
+      return []
 
   def _finalize(self):
     if context.executing_eagerly():
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
       raise NotImplementedError("Eager mode not supported in TPUStrategy.")
     else:
-      return [tpu.shutdown_system()]
+      return []
 
   def _get_devices_from(self, colocate_with=None):
     # TODO(jhseu): Change this when we support model parallelism.
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 14658630c5..0f68fcfe8b 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -25,7 +25,6 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -183,12 +182,6 @@ class SessionManager(object):
     """
     self._target = master
     sess = session.Session(self._target, graph=self._graph, config=config)
-    # TODO(jhseu): Delete once tpu.initialize_system() goes away.
-    initialize_ops = (
-        distribution_strategy_context.get_distribution_strategy().initialize()
-    )
-    if initialize_ops:
-      sess.run(initialize_ops)
 
     if checkpoint_dir and checkpoint_filename_with_path:
       raise ValueError("Can not provide both checkpoint_dir and "
-- 
GitLab


From f8d43bb7d99534d2a24155b040c0a6e6c2fb43a1 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 5 Dec 2018 15:53:38 -0800
Subject: [PATCH 1395/1554] Fix saved_model CODEOWNERS typo

PiperOrigin-RevId: 224236408
---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index e854a2e827..bfcdc2a23f 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -51,7 +51,7 @@
 /tensorflow/contrib/pi_examples/ @maciekcc
 /tensorflow/contrib/quantization/ @petewarden
 /tensorflow/contrib/rnn/ @ebrevdo @scottzhu
-/tensorflow/contrib/saved_model/ @nfiedel @sukritiramesh @allenl
+/tensorflow/contrib/saved_model/ @nfiedel @sukritiramesh @allenlavoie
 /tensorflow/contrib/seq2seq/ @ebrevdo @lmthang
 /tensorflow/contrib/session_bundle/ @nfiedel @sukritiramesh
 /tensorflow/contrib/slim/ @sguada @thenbasilmanran
-- 
GitLab


From 55e470a0b33355e4b9b6ef551558e5d5d5d15d60 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 5 Dec 2018 15:56:22 -0800
Subject: [PATCH 1396/1554] [tf.data] Adding support for tracking allocated
 memory of tensors stored in internal buffers.

PiperOrigin-RevId: 224236794
---
 tensorflow/core/framework/dataset.cc          | 14 +++++
 tensorflow/core/framework/dataset.h           | 60 +++++++++++++------
 tensorflow/core/framework/model.h             | 23 +++++--
 tensorflow/core/framework/model_test.cc       | 56 +++++++++++++++++
 .../core/kernels/data/cache_dataset_ops.cc    | 47 +++++++++------
 .../data/parallel_interleave_dataset_op.cc    |  2 +
 .../kernels/data/parallel_map_iterator.cc     |  7 ++-
 .../core/kernels/data/prefetch_dataset_op.cc  |  8 ++-
 .../core/kernels/data/shuffle_dataset_op.cc   |  2 +
 .../core/kernels/data/window_dataset.cc       | 10 +++-
 .../core/kernels/data/window_dataset_op.cc    |  7 +++
 .../benchmarks/autotune_benchmark.py          | 22 +++----
 12 files changed, 202 insertions(+), 56 deletions(-)

diff --git a/tensorflow/core/framework/dataset.cc b/tensorflow/core/framework/dataset.cc
index 6af14150b7..0e48932da7 100644
--- a/tensorflow/core/framework/dataset.cc
+++ b/tensorflow/core/framework/dataset.cc
@@ -206,6 +206,20 @@ bool GraphDefBuilderWrapper::HasAttr(const string& name,
   return HasAttr(op_def, attr_name);
 }
 
+int64 GetAllocatedBytes(const std::vector<Tensor>& element) {
+  int64 allocated_bytes = 0;
+  DatasetBase* dataset;
+  for (auto& tensor : element) {
+    if (tensor.dtype() == DT_VARIANT &&
+        GetDatasetFromVariantTensor(tensor, &dataset).ok()) {
+      allocated_bytes += dataset->AllocatedBytes();
+    } else {
+      allocated_bytes += tensor.AllocatedBytes();
+    }
+  }
+  return allocated_bytes;
+}
+
 Status GetDatasetFromVariantTensor(const Tensor& tensor,
                                    DatasetBase** out_dataset) {
   if (!(tensor.dtype() == DT_VARIANT &&
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index a00768fee0..8dcb5f3076 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -534,6 +534,25 @@ class DatasetContext {
   Params params_;
 };
 
+// Returns the number of bytes allocated for the given tensor.
+int64 GetAllocatedBytes(const std::vector<Tensor>& element);
+
+// Validates and extracts a `DatasetBase` object from `tensor`.
+//
+// `tensor` must have been written by a call to SetVariantTensorToDataset().
+//
+// The retrieved pointer is a borrowed reference to the dataset, which is owned
+// by the tensor. The consumer must either acquire its own reference to the
+// dataset by calling `(*out_dataset)->Ref()`, or ensure that `tensor` is not
+// destroyed or mutated while the retrieved pointer is in use.
+Status GetDatasetFromVariantTensor(const Tensor& tensor,
+                                   DatasetBase** out_dataset);
+
+// Stores a `DatasetBase` object in `tensor`.
+//
+// The ownership of `dataset` is transferred to `tensor`.
+Status StoreDatasetInVariantTensor(DatasetBase* dataset, Tensor* tensor);
+
 // Represents a (potentially infinite) range of outputs, where each
 // output is a tuple of tensors.
 class DatasetBase : public core::RefCounted {
@@ -587,12 +606,15 @@ class DatasetBase : public core::RefCounted {
   // in the outputs of this dataset.
   virtual const std::vector<PartialTensorShape>& output_shapes() const = 0;
 
-  // A human-readable debug string for this dataset.
-  virtual string DebugString() const = 0;
+  // Returns the number of bytes allocated for tensors of this dataset.
+  virtual int64 AllocatedBytes() const { return 0; }
 
   // Returns the cardinality of this dataset.
   virtual int64 Cardinality() const { return kUnknownCardinality; }
 
+  // A human-readable debug string for this dataset.
+  virtual string DebugString() const = 0;
+
   // Serializes the dataset and writes it to the `writer`.
   virtual Status Save(SerializationContext* ctx,
                       IteratorStateWriter* writer) const;
@@ -694,6 +716,24 @@ class DatasetBaseIterator : public IteratorBase {
     return model::MakeUnknownNode(std::move(args));
   }
 
+  // When performance modeling is enabled, this method records the fact that
+  // this iterator has dequeued a element from an internal buffer.
+  void RecordBufferDequeue(IteratorContext* ctx,
+                           const std::vector<Tensor>& element) {
+    if (node_) {
+      node_->add_buffered_bytes(-GetAllocatedBytes(element));
+    }
+  }
+
+  // When performance modeling is enabled, this method records the fact that
+  // this iterator has enqueued a element in an internal buffer.
+  void RecordBufferEnqueue(IteratorContext* ctx,
+                           const std::vector<Tensor>& element) {
+    if (node_) {
+      node_->add_buffered_bytes(GetAllocatedBytes(element));
+    }
+  }
+
   // When performance modeling is enabled, this method records the fact that
   // this iterator has produced an element.
   void RecordElement(IteratorContext* ctx) {
@@ -826,22 +866,6 @@ class BinaryDatasetOpKernel : public DatasetOpKernel {
                            DatasetBase** output) = 0;
 };
 
-// Validates and extracts a `DatasetBase` object from `tensor`.
-//
-// `tensor` must have been written by a call to SetVariantTensorToDataset().
-//
-// The retrieved pointer is a borrowed reference to the dataset, which is owned
-// by the tensor. The consumer must either acquire its own reference to the
-// dataset by calling `(*out_dataset)->Ref()`, or ensure that `tensor` is not
-// destroyed or mutated while the retrieved pointer is in use.
-Status GetDatasetFromVariantTensor(const Tensor& tensor,
-                                   DatasetBase** out_dataset);
-
-// Stores a `DatasetBase` object in `tensor`.
-//
-// The ownership of `dataset` is transferred to `tensor`.
-Status StoreDatasetInVariantTensor(DatasetBase* dataset, Tensor* tensor);
-
 // A simple background worker that executes closures asynchronously and without
 // blocking.
 //
diff --git a/tensorflow/core/framework/model.h b/tensorflow/core/framework/model.h
index 24aa5630cc..10059bbfd5 100644
--- a/tensorflow/core/framework/model.h
+++ b/tensorflow/core/framework/model.h
@@ -112,6 +112,12 @@ class Node {
   explicit Node(Args args)
       : id_(args.id), name_(args.name), output_(args.output.get()) {}
 
+  // Increments the bytes buffered by the given delta.
+  void add_buffered_bytes(int64 delta) LOCKS_EXCLUDED(mu_) {
+    mutex_lock l(mu_);
+    buffered_bytes_ += delta;
+  }
+
   // Adds an input.
   void add_input(std::shared_ptr<Node> node) LOCKS_EXCLUDED(mu_) {
     mutex_lock l(mu_);
@@ -124,18 +130,24 @@ class Node {
     processing_time_ += delta;
   }
 
+  // Returns the number of bytes stored in this node's buffer.
+  int64 buffered_bytes() const LOCKS_EXCLUDED(mu_) {
+    tf_shared_lock l(mu_);
+    return buffered_bytes_;
+  }
+
   // Returns the unique node ID.
   int64 id() const LOCKS_EXCLUDED(mu_) { return id_; }
 
-  // Returns the node name.
-  const string& name() const { return name_; }
-
   // Returns the node inputs.
   std::list<std::shared_ptr<Node>> inputs() const LOCKS_EXCLUDED(mu_) {
     tf_shared_lock l(mu_);
     return inputs_;
   }
 
+  // Returns the node name.
+  const string& name() const { return name_; }
+
   // Returns the number of elements produced by the node.
   int64 num_elements() const LOCKS_EXCLUDED(mu_) {
     tf_shared_lock l(mu_);
@@ -185,7 +197,8 @@ class Node {
 
   // Collects tunable parameters in the subtree rooted in this node.
   void CollectTunableParameters(
-      std::vector<std::shared_ptr<Parameter>>* parameters) LOCKS_EXCLUDED(mu_) {
+      std::vector<std::shared_ptr<Parameter>>* parameters) const
+      LOCKS_EXCLUDED(mu_) {
     tf_shared_lock l(mu_);
     for (auto& pair : parameters_) {
       if (pair.second->state->tunable) {
@@ -219,6 +232,7 @@ class Node {
       LOCKS_EXCLUDED(mu_) {
     tf_shared_lock l(mu_);
     std::shared_ptr<Node> result = Clone(output);
+    result->buffered_bytes_ = buffered_bytes_;
     result->processing_time_ = processing_time_;
     result->num_elements_ = num_elements_;
     result->parameters_ = parameters_;
@@ -274,6 +288,7 @@ class Node {
   mutable mutex mu_;
   const int64 id_;
   const string name_;
+  int64 buffered_bytes_ GUARDED_BY(mu_) = 0;
   int64 processing_time_ GUARDED_BY(mu_) = 0;
   int64 num_elements_ GUARDED_BY(mu_) = 0;
   std::map<std::thread::id, int64> work_start_ GUARDED_BY(mu_);
diff --git a/tensorflow/core/framework/model_test.cc b/tensorflow/core/framework/model_test.cc
index 53e35f25b2..90bd570f90 100644
--- a/tensorflow/core/framework/model_test.cc
+++ b/tensorflow/core/framework/model_test.cc
@@ -330,6 +330,62 @@ TEST(UnknownTest, Model) {
   EXPECT_EQ(100, unknown->OutputTime(&input_times));
 }
 
+class TestNode : public model::Node {
+ public:
+  using model::Node::Node;
+
+  virtual ~TestNode() {}
+
+ protected:
+  std::shared_ptr<Node> Clone(std::shared_ptr<Node> output) const override
+      SHARED_LOCKS_REQUIRED(mu_) {
+    return nullptr;
+  }
+
+  int64 OutputTimeLocked(std::vector<int64>* input_times) const override
+      SHARED_LOCKS_REQUIRED(mu_) {
+    return 0;
+  }
+
+  int64 ProcessingTimeLocked() const override SHARED_LOCKS_REQUIRED(mu_) {
+    return 0;
+  }
+};
+
+TEST(SetterGetterTest, Node) {
+  std::shared_ptr<TestNode> node =
+      std::make_shared<TestNode>(model::Node::Args{-1, "TestNode", nullptr});
+  EXPECT_EQ(-1, node->id());
+  EXPECT_EQ("TestNode", node->name());
+  EXPECT_EQ(nullptr, node->output());
+
+  EXPECT_EQ(0, node->buffered_bytes());
+  node->add_buffered_bytes(42);
+  EXPECT_EQ(42, node->buffered_bytes());
+
+  EXPECT_EQ(0, node->processing_time());
+  node->record_start(1);
+  EXPECT_EQ(0, node->processing_time());
+  node->record_stop(41);
+  EXPECT_EQ(40, node->processing_time());
+  node->add_processing_time(2);
+  EXPECT_EQ(42, node->processing_time());
+
+  std::shared_ptr<TestNode> input =
+      std::make_shared<TestNode>(model::Node::Args{-1, "TestInput", node});
+  EXPECT_EQ(node.get(), input->output());
+  EXPECT_EQ(0, node->inputs().size());
+  node->add_input(input);
+  EXPECT_EQ(1, node->inputs().size());
+  EXPECT_EQ(input, node->inputs().front());
+  node->remove_input(input);
+  EXPECT_EQ(0, node->inputs().size());
+
+  EXPECT_EQ(0, node->num_elements());
+  node->record_element();
+  EXPECT_EQ(1, node->num_elements());
+}
+
 }  // namespace
 }  // namespace model
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index d7b231be18..ab85736890 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -822,6 +822,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
             cache_->Complete();
             return Status::OK();
           }
+          RecordBufferEnqueue(ctx, *out_tensors);
           cache_->emplace_back(*out_tensors);
           return Status::OK();
         }
@@ -858,6 +859,35 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
           CHECK(cache);
         }
 
+        Status Initialize(IteratorContext* ctx) override {
+          // The memory allocated for the cache is owned by the parent
+          // dataset but performance modeling uses the iterator abstraction and
+          // thus we record the memory allocated for the cache here. The caveat
+          // is that this is incorrect if there are concurrent instances of this
+          // iterator.
+          for (size_t i = 0; i < cache_->size(); ++i) {
+            RecordBufferEnqueue(ctx, cache_->at(i));
+          }
+          return Status::OK();
+        }
+
+        Status GetNextInternal(IteratorContext* ctx,
+                               std::vector<Tensor>* out_tensors,
+                               bool* end_of_sequence) override {
+          mutex_lock l(mu_);
+          if (index_ < cache_->size()) {
+            const std::vector<Tensor>& cache_tensors = cache_->at(index_);
+            out_tensors->insert(out_tensors->begin(), cache_tensors.begin(),
+                                cache_tensors.end());
+            index_++;
+            *end_of_sequence = false;
+            return Status::OK();
+          } else {
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+        }
+
        protected:
         std::shared_ptr<model::Node> CreateNode(
             IteratorContext* ctx, model::Node::Args args) const override {
@@ -882,23 +912,6 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
           return Status::OK();
         }
 
-        Status GetNextInternal(IteratorContext* ctx,
-                               std::vector<Tensor>* out_tensors,
-                               bool* end_of_sequence) override {
-          mutex_lock l(mu_);
-          if (index_ < cache_->size()) {
-            const std::vector<Tensor>& cache_tensors = cache_->at(index_);
-            out_tensors->insert(out_tensors->begin(), cache_tensors.begin(),
-                                cache_tensors.end());
-            index_++;
-            *end_of_sequence = false;
-            return Status::OK();
-          } else {
-            *end_of_sequence = true;
-            return Status::OK();
-          }
-        }
-
        private:
         mutex mu_;
         const std::shared_ptr<MemoryCache> cache_;
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 5d33d1e795..2f6d91e863 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -256,6 +256,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
         if (result->status.ok()) {
           *out_tensors = std::move(result->return_values);
+          RecordBufferDequeue(ctx, *out_tensors);
         }
         *end_of_sequence = false;
         return result->status;
@@ -394,6 +395,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
           if (end_of_input) {
             result->skip = true;
           }
+          RecordBufferEnqueue(ctx.get(), result->return_values);
           {
             mutex_lock l(*mu_);
             result->notification.Notify();
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index 32b634ae85..b97f692500 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -100,7 +100,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
     RecordStop(ctx);
     result->notification.WaitForNotification();
     RecordStart(ctx);
-    return ProcessResult(result, out_tensors, end_of_sequence);
+    return ProcessResult(ctx, result, out_tensors, end_of_sequence);
   }
 
  protected:
@@ -211,6 +211,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
           strings::StrCat(prefix_end_, "::active_parallel_calls"),
           static_cast<float>(num_calls_));
     }
+    RecordBufferEnqueue(ctx.get(), result->return_values);
     result->notification.Notify();
     cond_var_->notify_all();
   }
@@ -239,11 +240,13 @@ class ParallelMapIterator : public DatasetBaseIterator {
                                    &result->return_values, std::move(done));
   }
 
-  Status ProcessResult(const std::shared_ptr<InvocationResult>& result,
+  Status ProcessResult(IteratorContext* ctx,
+                       const std::shared_ptr<InvocationResult>& result,
                        std::vector<Tensor>* out_tensors, bool* end_of_sequence)
       LOCKS_EXCLUDED(*mu_) {
     if (!result->end_of_input && result->status.ok()) {
       *out_tensors = std::move(result->return_values);
+      RecordBufferDequeue(ctx, *out_tensors);
       *end_of_sequence = false;
       return Status::OK();
     }
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 9b2a530be4..08d6de4bf9 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -125,7 +125,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
         }
 
         if (!buffer_.empty()) {
-          return Consume(out_tensors, end_of_sequence, ctx);
+          return Consume(ctx, out_tensors, end_of_sequence);
         }
 
         if (prefetch_thread_finished_) {
@@ -228,8 +228,8 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
       std::vector<Tensor> value;
     };
 
-    Status Consume(std::vector<Tensor>* out_tensors, bool* end_of_sequence,
-                   IteratorContext* ctx) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    Status Consume(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
+                   bool* end_of_sequence) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
       const auto& stats_aggregator = ctx->stats_aggregator();
       if (stats_aggregator) {
         stats_aggregator->AddToHistogram(
@@ -248,6 +248,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
       Status s = buffer_.front().status;
       if (s.ok()) {
         *out_tensors = std::move(buffer_.front().value);
+        RecordBufferDequeue(ctx, *out_tensors);
       }
       auto_tuner_.RecordConsumption(buffer_.size());
       buffer_.pop_front();
@@ -318,6 +319,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
         // 3. Signal that the element has been produced.
         {
           mutex_lock l(mu_);
+          RecordBufferEnqueue(ctx.get(), buffer_element.value);
           buffer_.push_back(std::move(buffer_element));
           cond_var_.notify_all();
         }
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index 0ca347e98a..3cf005dbda 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -126,6 +126,7 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
                 ctx, this->prefix(), &input_impl_));
           }
           if (!end_of_input_sequence) {
+            this->RecordBufferEnqueue(ctx, input_element);
             buffer_[slices_.back()->end % this->dataset()->buffer_size_] =
                 std::move(input_element);
             num_elements_++;
@@ -153,6 +154,7 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
           int64 index =
               (slices_.front()->start + offset) % this->dataset()->buffer_size_;
           *out_tensors = std::move(buffer_[index]);
+          this->RecordBufferDequeue(ctx, *out_tensors);
           std::swap(
               buffer_[index],
               buffer_[slices_.front()->start % this->dataset()->buffer_size_]);
diff --git a/tensorflow/core/kernels/data/window_dataset.cc b/tensorflow/core/kernels/data/window_dataset.cc
index bed1c66ac7..c295631550 100644
--- a/tensorflow/core/kernels/data/window_dataset.cc
+++ b/tensorflow/core/kernels/data/window_dataset.cc
@@ -41,10 +41,18 @@ class WindowDataset : public DatasetBase {
     return output_shapes_;
   }
 
-  string DebugString() const override { return "WindowDataset"; }
+  int64 AllocatedBytes() const override {
+    int64 allocated_bytes = 0;
+    for (auto& element : elements_) {
+      allocated_bytes += GetAllocatedBytes(element);
+    }
+    return allocated_bytes;
+  }
 
   int64 Cardinality() const override { return elements_.size(); }
 
+  string DebugString() const override { return "WindowDataset"; }
+
  protected:
   // TODO(b/110981596): Support checkpointing.
   Status AsGraphDefInternal(SerializationContext* ctx,
diff --git a/tensorflow/core/kernels/data/window_dataset_op.cc b/tensorflow/core/kernels/data/window_dataset_op.cc
index 9d66162dfa..ae13ae5da8 100644
--- a/tensorflow/core/kernels/data/window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/window_dataset_op.cc
@@ -164,6 +164,7 @@ class WindowDatasetOp : public UnaryDatasetOpKernel {
               Status status =
                   input_impl_->GetNext(ctx, &element, end_of_sequence);
               if (!*end_of_sequence) {
+                RecordBufferEnqueue(ctx, element);
                 buffer_.emplace_back(std::move(element), status);
               } else {
                 input_impl_.reset();
@@ -201,8 +202,14 @@ class WindowDatasetOp : public UnaryDatasetOpKernel {
                 input_impl_.reset();
               }
             }
+            for (size_t i = 0; i < buffer_.size(); ++i) {
+              RecordBufferDequeue(ctx, buffer_.at(i).result);
+            }
             buffer_.clear();
           } else {
+            for (size_t i = 0; i < window_shift; ++i) {
+              RecordBufferDequeue(ctx, buffer_.at(i).result);
+            }
             buffer_.erase(buffer_.begin(), buffer_.begin() + window_shift);
           }
         }
diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
index b48ef95666..e713494b52 100644
--- a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
@@ -134,33 +134,33 @@ class AutotuneBenchmark(test.Benchmark):
     a = (np.random.rand(1, 8 * k), np.random.rand(8 * k, 1))
     b = (np.random.rand(1, 4 * k), np.random.rand(4 * k, 1))
     c = (np.random.rand(1, 2 * k), np.random.rand(2 * k, 1))
-    dataset = dataset_ops.Dataset.from_tensors((a, b, c)).repeat()
+    dataset_a = dataset_ops.Dataset.from_tensors(a).repeat()
+    dataset_b = dataset_ops.Dataset.from_tensors(b).repeat()
+    dataset_c = dataset_ops.Dataset.from_tensors(c).repeat()
 
-    def f1(a, b, c):
-      x, y = a
-      return math_ops.matmul(x, y), b, c
+    def f1(x, y):
+      return math_ops.matmul(x, y)
 
-    def f2(a, b, c):
+    def f2(a, b):
       x, y = b
-      return a, math_ops.matmul(x, y), c
-
-    def f3(a, b, c):
-      x, y = c
-      return a, b, math_ops.matmul(x, y)
+      return a, math_ops.matmul(x, y)
 
+    dataset = dataset_a
     dataset = dataset.map(f1, num_parallel_calls=optimization.AUTOTUNE)
     dataset = dataset_ops.Dataset.range(1).repeat().interleave(
         lambda _: dataset,
         num_parallel_calls=optimization.AUTOTUNE,
         cycle_length=2)
 
+    dataset = dataset_ops.Dataset.zip((dataset, dataset_b))
     dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE)
     dataset = dataset_ops.Dataset.range(1).repeat().interleave(
         lambda _: dataset,
         num_parallel_calls=optimization.AUTOTUNE,
         cycle_length=2)
 
-    dataset = dataset.map(f3, num_parallel_calls=optimization.AUTOTUNE)
+    dataset = dataset_ops.Dataset.zip((dataset, dataset_c))
+    dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
 
-- 
GitLab


From 5caa987005b5b832798eb60039bae169cb93ef60 Mon Sep 17 00:00:00 2001
From: Anudhyan Boral <anudhyan@google.com>
Date: Wed, 5 Dec 2018 16:45:41 -0800
Subject: [PATCH 1397/1554] Add TensorFlow op to compute the LU decomposition
 of a matrix.

For CPU, we use Eigen's PartialPivLU op, which has performance comparable to Cholesky decomposition. Unlike Cholesky this is also robust for non-SPD matrices.

For GPU, we use getrf and getrfbatched from cuSolver.

PiperOrigin-RevId: 224244390
---
 .../core/api_def/base_api/api_def_Lu.pbtxt    |  51 ++++
 .../core/api_def/python_api/api_def_Lu.pbtxt  |   6 +
 tensorflow/core/kernels/BUILD                 |  14 +
 tensorflow/core/kernels/lu_op.cc              | 193 ++++++++++++
 tensorflow/core/kernels/lu_op_gpu.cu.cc       | 275 +++++++++++++++++
 tensorflow/core/ops/linalg_ops.cc             |  32 ++
 tensorflow/core/ops/linalg_ops_test.cc        |  19 ++
 tensorflow/python/kernel_tests/BUILD          |  16 +
 tensorflow/python/kernel_tests/lu_op_test.py  | 285 ++++++++++++++++++
 tensorflow/python/ops/linalg/linalg_impl.py   |   1 +
 .../api/golden/v1/tensorflow.linalg.pbtxt     |   4 +
 .../api/golden/v2/tensorflow.linalg.pbtxt     |   4 +
 12 files changed, 900 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_Lu.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Lu.pbtxt
 create mode 100644 tensorflow/core/kernels/lu_op.cc
 create mode 100644 tensorflow/core/kernels/lu_op_gpu.cu.cc
 create mode 100644 tensorflow/python/kernel_tests/lu_op_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_Lu.pbtxt b/tensorflow/core/api_def/base_api/api_def_Lu.pbtxt
new file mode 100644
index 0000000000..35dbee8364
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_Lu.pbtxt
@@ -0,0 +1,51 @@
+op {
+  graph_op_name: "Lu"
+  in_arg {
+    name: "input"
+    description: <<END
+A tensor of shape `[..., M, M]` whose inner-most 2 dimensions form matrices of
+size `[M, M]`.
+END
+  }
+  out_arg {
+    name: "lu"
+    description: <<END
+A tensor of shape `[..., M, M]` whose strictly lower triangular part denotes the
+lower triangular factor `L` with unit diagonal, and whose upper triangular part
+denotes the upper triangular factor `U`.
+END
+  }
+  out_arg {
+    name: "p"
+    description: <<END
+Permutation of the rows encoded as a list of indices in `0..M-1`. Shape is
+`[..., M]`.
+@compatibility(scipy)
+Similar to `scipy.linalg.lu`, except the triangular factors `L` and `U` are
+packed into a single tensor, the permutation is applied to `input` instead of
+the right hand side and the permutation `P` is returned as a list of indices
+instead of a permutation matrix.
+@end_compatibility
+END
+  }
+  summary: "Computes the LU decomposition of one or more square matrices."
+  description: <<END
+The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+form square matrices.
+
+The input has to be invertible.
+
+The output consists of two tensors LU and P containing the LU decomposition
+of all input submatrices `[..., :, :]`. LU encodes the lower triangular and
+upper triangular factors.
+
+For each input submatrix of shape `[M, M]`, L is a lower triangular matrix of
+shape `[M, M]` with unit diagonal whose entries correspond to the strictly lower
+triangular part of LU. U is a upper triangular matrix of shape `[M, M]` whose
+entries correspond to the upper triangular part, including the diagonal, of LU.
+
+P represents a permutation matrix encoded as a list of indices each between `0`
+and `M-1`, inclusive. If P_mat denotes the permutation matrix corresponding to
+P, then the L, U and P satisfies P_mat * input = L * U.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Lu.pbtxt b/tensorflow/core/api_def/python_api/api_def_Lu.pbtxt
new file mode 100644
index 0000000000..c0b6b53da5
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Lu.pbtxt
@@ -0,0 +1,6 @@
+op {
+  graph_op_name: "Lu"
+  endpoint {
+    name: "linalg.lu"
+  }
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index e449187871..0e5d8d765a 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2755,6 +2755,7 @@ cc_library(
         ":cholesky_grad",
         ":cholesky_op",
         ":determinant_op",
+        ":lu_op",
         ":matrix_exponential_op",
         ":matrix_inverse_op",
         ":matrix_logarithm_op",
@@ -2900,6 +2901,19 @@ tf_kernel_library(
     deps = LINALG_DEPS,
 )
 
+tf_kernel_library(
+    name = "lu_op",
+    prefix = "lu_op",
+    deps = if_cuda([
+        ":cuda_solvers",
+        ":transpose_functor",
+    ]) + [
+        "//third_party/eigen3",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
+
 cc_library(
     name = "linalg_ops_common",
     srcs = ["linalg_ops_common.cc"],
diff --git a/tensorflow/core/kernels/lu_op.cc b/tensorflow/core/kernels/lu_op.cc
new file mode 100644
index 0000000000..f9591d1bdf
--- /dev/null
+++ b/tensorflow/core/kernels/lu_op.cc
@@ -0,0 +1,193 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "third_party/eigen3/Eigen/Core"
+#include "third_party/eigen3/Eigen/LU"
+#include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/lib/math/math_util.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/work_sharder.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template <typename Scalar, typename Tidx>
+class LuOp : public OpKernel {
+ public:
+  explicit LuOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+ protected:
+  using TensorShapes = gtl::InlinedVector<TensorShape, 4>;
+  using TensorOutputs = gtl::InlinedVector<Tensor*, 4>;
+
+  using Matrix =
+      Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
+  using ConstMatrixMap = Eigen::Map<const Matrix>;
+  using MatrixMap = Eigen::Map<Matrix>;
+
+  using RealScalar = typename Eigen::NumTraits<Scalar>::Real;
+
+  using Indices =
+      Eigen::Matrix<Tidx, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
+  using IndicesMap = Eigen::Map<Indices>;
+  using ConstIndicesMap = Eigen::Map<const Indices>;
+
+ public:
+  // Returns the cost per matrix operation. This is used to determine the
+  // number of threads to use for parallelizing factorization in batch mode.
+  // Cost per unit is assumed to be roughly 1ns, based on comments
+  // in core/util/work_sharder.cc.
+  // LU decomposition for a square matrix takes roughly (2/3) * (num_rows)^3.
+  // TODO(anudhyan): Refine this estimate after taking constant factors into
+  // account.
+  int64 GetCostPerUnit(const TensorShape& input_matrix_shape) const {
+    double num_rows = static_cast<double>(input_matrix_shape.dim_size(0));
+    double cost = (2 / 3.0) * MathUtil::IPow(num_rows, 3);
+    return cost >= static_cast<double>(kint64max) ? kint64max
+                                                  : static_cast<int64>(cost);
+  }
+
+  void Compute(OpKernelContext* context) override {
+    OP_REQUIRES(context, context->num_inputs() == 1,
+                errors::InvalidArgument("Expecting exactly one input, got ",
+                                        context->num_inputs()));
+
+    const Tensor& input = context->input(0);
+    int input_rank = input.dims();
+    OP_REQUIRES(context, input_rank >= 2,
+                errors::InvalidArgument(
+                    "Input tensor must have rank >= 2, got ", input_rank));
+
+    // If the tensor rank is greater than 2, we consider the inner-most
+    // dimensions as matrices, and loop over all the other outer ("batch")
+    // dimensions to compute the results.
+    TensorShape input_matrix_shape;
+    TensorShape batch_shape;
+    for (int dim = 0; dim < input_rank - 2; ++dim) {
+      batch_shape.AddDim(input.dim_size(dim));
+    }
+    const int64 num_rows = input.dim_size(input_rank - 2);
+    const int64 num_cols = input.dim_size(input_rank - 1);
+
+    input_matrix_shape.AppendShape({num_rows, num_cols});
+    OP_REQUIRES(context, TensorShapeUtils::IsSquareMatrix(input_matrix_shape),
+                errors::InvalidArgument("Input matrix must be square."));
+
+    // packed_triangular_factors is a matrix with the same shape as the input;
+    // permutation is a vector.
+    TensorShape permutation_shape = batch_shape;
+    permutation_shape.AddDim(num_rows);
+
+    TensorShapes output_matrix_shapes({input.shape(), permutation_shape});
+
+    TensorOutputs outputs;
+    Tensor* output_packed_triangular_factors = nullptr;
+    OP_REQUIRES_OK(
+        context, context->forward_input_or_allocate_output(
+                     {0}, 0, input.shape(), &output_packed_triangular_factors));
+    outputs.emplace_back(output_packed_triangular_factors);
+
+    Tensor* output_permutation = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output(1, permutation_shape,
+                                                     &output_permutation));
+    outputs.emplace_back(output_permutation);
+
+    if (num_rows == 0) {
+      return;
+    }
+
+    // Process the individual matrix problems in parallel using a threadpool.
+    auto shard = [this, &input, &num_rows, &num_cols, &outputs,
+                  &output_matrix_shapes, context](int64 begin, int64 end) {
+      for (int64 i = begin; i < end; ++i) {
+        ComputeTensorSlice(context, i, input, num_rows, num_cols, outputs,
+                           output_matrix_shapes);
+      }
+    };
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers,
+          batch_shape.num_elements(), GetCostPerUnit(input_matrix_shape),
+          shard);
+  }
+
+  void ComputeTensorSlice(OpKernelContext* context, int64 matrix_index,
+                          const Tensor& input, int64 num_rows, int64 num_cols,
+                          const TensorOutputs& outputs,
+                          const TensorShapes& output_matrix_shapes) {
+    // TODO(kalakris): Handle alignment if possible. Eigen::Map is
+    // unaligned by default.
+    ConstMatrixMap input_matrix(
+        input.flat<Scalar>().data() + matrix_index * num_rows * num_cols,
+        num_rows, num_cols);
+
+    // packed_triangular_factors has shape [num_rows, num_cols]
+    MatrixMap packed_triangular_factors(
+        outputs[0]->flat<Scalar>().data() + matrix_index * num_rows * num_cols,
+        num_rows, num_rows);
+
+    // permutation has shape [num_rows, 1]
+    IndicesMap permutation_indices(
+        outputs[1]->flat<Tidx>().data() + matrix_index * num_rows, num_rows, 1);
+
+    Eigen::PartialPivLU<Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>>
+        lu_decomposition(input_matrix);
+
+    // Output the packed triangular factors in a dense form.
+    // The lower triangular factor L corresponds to the strictly lower
+    // triangular part of packed_triangular_factors with an implicit unit
+    // diagonal. The upper triangular factor U is the upper triangular part of
+    // packed_triangular_factors. The triangular factors satisfy the equation
+    //     P * input_matrix = L * U
+    // where P is the permutation matrix corresponding to the indices in
+    // permutation_indices.
+    packed_triangular_factors = lu_decomposition.matrixLU();
+    // Output the permutation matrix used for pivoting.
+    Eigen::PermutationMatrix<-1, -1, Tidx> permutation =
+        lu_decomposition.permutationP().transpose();
+    permutation_indices = permutation.indices();
+
+    // PartialPivLU cannot give strong guarantees on invertibility,
+    // but we can at least guard against exact zero pivots. This can occur as
+    // a result of basic user mistakes such providing integer valued
+    // matrices that are exactly singular, or due to underflow if this
+    // code is run with denormals being flushed to zero.
+    const RealScalar min_abs_pivot =
+        packed_triangular_factors.diagonal().cwiseAbs().minCoeff();
+    OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
+                errors::InvalidArgument("Input is not invertible."));
+  }
+};
+
+#define REGISTER_LU(type, idx_type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Lu")                                        \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<idx_type>("output_idx_type"), \
+                          LuOp<type, idx_type>);
+
+REGISTER_LU(float, int32);
+REGISTER_LU(double, int32);
+REGISTER_LU(complex64, int32);
+REGISTER_LU(complex128, int32);
+
+REGISTER_LU(float, int64);
+REGISTER_LU(double, int64);
+REGISTER_LU(complex64, int64);
+REGISTER_LU(complex128, int64);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/lu_op_gpu.cu.cc b/tensorflow/core/kernels/lu_op_gpu.cu.cc
new file mode 100644
index 0000000000..f83744b50d
--- /dev/null
+++ b/tensorflow/core/kernels/lu_op_gpu.cu.cc
@@ -0,0 +1,275 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include <algorithm>
+#include <vector>
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/cuda_solvers.h"
+#include "tensorflow/core/kernels/transpose_functor.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+
+namespace tensorflow {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace {
+template <typename Scalar>
+__device__ void ComputePermutationFromTranspositions(
+    int64 num_rows, const int* pivots, Scalar* permutation_indices) {
+  // Fill in the output array with the identity permutation.
+  for (int i = 0; i < num_rows; ++i) {
+    permutation_indices[i] = Scalar(i);
+  }
+
+  // Compute the permutation from a sequence of transpositions encoded
+  // in the pivot array by applying the transpositions in order on the
+  // identity permutation.
+  for (int i = 0; i < num_rows; ++i) {
+    // Note: Internally, the cuBlas code uses Fortran convention (1-based)
+    // indexing so ith row was swapped with (pivots[i]-1)'th row in 0-based
+    // indexing.
+    Scalar t = permutation_indices[i];
+    permutation_indices[i] = permutation_indices[pivots[i] - 1];
+    permutation_indices[pivots[i] - 1] = t;
+  }
+}
+}  // namespace
+
+// Kernel to compute the inverse of a permutation from a sequence of
+// transpositions.
+template <typename Scalar>
+__global__ void ComputePermutationFromTranspositionsKernel(
+    CudaLaunchConfig config, const int64 num_rows, const int* all_pivots,
+    Scalar* all_permutation_indices) {
+  // We only parallelize over batches here. Performance is not critical,
+  // since this cheap O(num_rows) kernel always follows an O(num_rows^3)
+  // LU factorization.
+  CUDA_1D_KERNEL_LOOP(index, config.virtual_thread_count) {
+    ComputePermutationFromTranspositions(
+        num_rows, all_pivots + index * num_rows,
+        all_permutation_indices + index * num_rows);
+  }
+}
+
+template <class Scalar, class Tidx>
+class LuOpGpu : public AsyncOpKernel {
+ public:
+  explicit LuOpGpu(OpKernelConstruction* context) : AsyncOpKernel(context) {}
+
+  void ComputeAsync(OpKernelContext* context, DoneCallback done) final {
+    const Tensor& input = context->input(0);
+
+    // Analyze shape and validate inputs.
+    const int input_rank = input.dims();
+
+    OP_REQUIRES_ASYNC(
+        context, input_rank >= 2,
+        errors::InvalidArgument("Input must have rank >= 2, got ", input_rank),
+        done);
+
+    const int64 num_rows = input.dim_size(input_rank - 2);
+    const int64 num_cols = input.dim_size(input_rank - 1);
+
+    OP_REQUIRES_ASYNC(
+        context, num_rows == num_cols,
+        errors::InvalidArgument("Input matrices must be squares, got", num_rows,
+                                " != ", num_cols),
+        done);
+
+    TensorShape batch_shape;
+    for (int dim = 0; dim < input_rank - 2; ++dim) {
+      batch_shape.AddDim(input.dim_size(dim));
+    }
+    TensorShape permutation_indices_shape = batch_shape;
+    permutation_indices_shape.AddDim(num_rows);
+
+    const GPUDevice& device = context->eigen_device<GPUDevice>();
+    auto solver = absl::make_unique<CudaSolver>(context);
+
+    // We output the packed triangular factors in a dense form.
+    // The lower triangular factor L corresponds to the strictly lower
+    // triangular part of packed_triangular_factors with an implicit unit
+    // diagonal. The upper triangular factor U is the upper triangular part of
+    // packed_triangular_factors. The triangular factors satisfy the equation
+    //     P * input_matrix = L * U
+    // where P is the permutation matrix corresponding to the indices in
+    // permutation_indices.
+    //
+    // Reuse the input buffer or make a copy for the factorization step,
+    // depending on whether this ops owns it exclusively.
+    Tensor* packed_triangular_factors;
+    OP_REQUIRES_OK_ASYNC(context,
+                         context->forward_input_or_allocate_output(
+                             {0}, 0, input.shape(), &packed_triangular_factors),
+                         done);
+    if (!packed_triangular_factors->SharesBufferWith(input)) {
+      device.memcpy(packed_triangular_factors->flat<Scalar>().data(),
+                    input.flat<Scalar>().data(),
+                    input.NumElements() * sizeof(Scalar));
+    }
+
+    // Allocate output permutation.
+    Tensor* permutation_indices = nullptr;
+    OP_REQUIRES_OK_ASYNC(context,
+                         context->allocate_output(1, permutation_indices_shape,
+                                                  &permutation_indices),
+                         done);
+
+    if (input.NumElements() == 0) {
+      done();
+      return;
+    }
+
+    // Allocate a temporary Tensor to store the transposed packed triangular
+    // factors.
+    Tensor packed_triangular_factors_transpose;
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        context->allocate_temp(DataTypeToEnum<Scalar>::value, input.shape(),
+                               &packed_triangular_factors_transpose),
+        done);
+    auto packed_triangular_factors_transpose_reshaped =
+        packed_triangular_factors_transpose
+            .template flat_inner_dims<Scalar, 3>();
+    const int64 batch_size =
+        packed_triangular_factors_transpose_reshaped.dimension(0);
+
+    // Allocate pivots on the device.
+    Tensor pivots;
+    OP_REQUIRES_OK_ASYNC(context,
+                         solver->allocate_scoped_tensor(
+                             DataTypeToEnum<int32>::value,
+                             TensorShape{batch_size, num_rows}, &pivots),
+                         done);
+    auto pivots_mat = pivots.template matrix<int32>();
+
+    // Transpose the input. This is necessary because cuBLAS assumes
+    // column-major storage while TensorFlow uses row-major.
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        DoMatrixTranspose(device, *packed_triangular_factors,
+                          &packed_triangular_factors_transpose),
+        done);
+
+    std::vector<DeviceLapackInfo> dev_info;
+    if (num_rows == num_cols && num_rows / batch_size <= 128) {
+      // For small matrices or large batch sizes, we use the batched
+      // interface from cuBlas.
+      auto packed_triangular_factors_ptrs = solver->GetScratchSpace<uint8>(
+          sizeof(Scalar*) * batch_size, "packed_triangular_factors_ptrs",
+          /* on_host */ true);
+      const Scalar** packed_triangular_factors_ptrs_base =
+          reinterpret_cast<const Scalar**>(
+              packed_triangular_factors_ptrs.mutable_data());
+      for (int batch = 0; batch < batch_size; ++batch) {
+        packed_triangular_factors_ptrs_base[batch] =
+            &packed_triangular_factors_transpose_reshaped(batch, 0, 0);
+      }
+      dev_info.push_back(
+          solver->GetDeviceLapackInfo(batch_size, "getrfBatched"));
+      OP_REQUIRES_OK_ASYNC(
+          context,
+          solver->GetrfBatched(num_rows, packed_triangular_factors_ptrs_base,
+                               num_rows, pivots_mat.data(), &dev_info.back(),
+                               batch_size),
+          done);
+    } else {
+      // For small batch sizes we use the non-batched interface from cuSolver,
+      // which is much faster for large matrices.
+      dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrf"));
+      for (int batch = 0; batch < batch_size; ++batch) {
+        OP_REQUIRES_OK_ASYNC(
+            context,
+            solver->Getrf(
+                num_rows, num_cols,
+                &packed_triangular_factors_transpose_reshaped(batch, 0, 0),
+                num_rows, &pivots_mat(batch, 0), &dev_info.back()(batch)),
+            done);
+      }
+    }
+
+    // Transpose the result since we had transposed the input.
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        DoMatrixTranspose(device, packed_triangular_factors_transpose,
+                          packed_triangular_factors),
+        done);
+
+    // Pivots encode the permutation of the rows as a sequences of row swaps.
+    // For each index i, row i is swapped with row pivots[i].
+    int* pivots_ptr = pivots.flat<int>().data();
+    Tidx* permutation_indices_ptr =
+        permutation_indices->template flat<Tidx>().data();
+    CudaLaunchConfig cfgPivots = GetCudaLaunchConfig(batch_size, device);
+    ComputePermutationFromTranspositionsKernel<<<cfgPivots.block_count,
+                                                 cfgPivots.thread_per_block, 0,
+                                                 device.stream()>>>(
+        cfgPivots, num_rows, pivots_ptr, permutation_indices_ptr);
+
+    // Callback for checking info after kernels finish. Also capture the
+    // temporary Tensors/ScratchSpace so they don't get deallocated before the
+    // kernels run.
+    // TODO(rmlarsen): Use move capture once C++14 becomes available.
+    auto info_checker = [context, done, dev_info](
+                            const Status& status,
+                            const std::vector<HostLapackInfo>& host_infos) {
+      if (!status.ok() && errors::IsInvalidArgument(status) &&
+          !host_infos.empty()) {
+        for (int i = 0; i < host_infos[0].size(); ++i) {
+          // Match the CPU error message for singular matrices. Otherwise
+          // just print the original error message from the status below.
+          OP_REQUIRES_ASYNC(context, host_infos[0].data()[i] <= 0,
+                            errors::InvalidArgument("Input is not invertible."),
+                            done);
+        }
+      }
+      OP_REQUIRES_OK_ASYNC(context, status, done);
+      done();
+    };
+
+    CudaSolver::CheckLapackInfoAndDeleteSolverAsync(std::move(solver), dev_info,
+                                                    std::move(info_checker));
+  }
+};
+
+#define REGISTER_LU_GPU(type, idx_type)                                     \
+  REGISTER_KERNEL_BUILDER(Name("Lu")                                        \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<idx_type>("output_idx_type"), \
+                          LuOpGpu<type, idx_type>);
+
+REGISTER_LU_GPU(float, int32);
+REGISTER_LU_GPU(double, int32);
+REGISTER_LU_GPU(complex64, int32);
+REGISTER_LU_GPU(complex128, int32);
+
+REGISTER_LU_GPU(float, int64);
+REGISTER_LU_GPU(double, int64);
+REGISTER_LU_GPU(complex64, int64);
+REGISTER_LU_GPU(complex128, int64);
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/linalg_ops.cc b/tensorflow/core/ops/linalg_ops.cc
index 525b19e51e..952ee4bee2 100644
--- a/tensorflow/core/ops/linalg_ops.cc
+++ b/tensorflow/core/ops/linalg_ops.cc
@@ -109,6 +109,30 @@ Status SelfAdjointEigV2ShapeFn(InferenceContext* c) {
   return Status::OK();
 }
 
+// Input is [...,N,N].
+// First and second outputs are:
+//   [...,N,N]; [...,N].
+Status LuShapeFn(InferenceContext* c) {
+  ShapeHandle input;
+  TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 2, &input));
+
+  DimensionHandle n;
+  TF_RETURN_IF_ERROR(c->Merge(c->Dim(input, -2), c->Dim(input, -1), &n));
+
+  ShapeHandle batch_shape;
+  TF_RETURN_IF_ERROR(c->Subshape(input, 0, -2, &batch_shape));
+
+  ShapeHandle lu_shape;
+  ShapeHandle p_shape;
+
+  TF_RETURN_IF_ERROR(c->Concatenate(batch_shape, c->Matrix(n, n), &lu_shape));
+  TF_RETURN_IF_ERROR(c->Concatenate(batch_shape, c->Vector(n), &p_shape));
+
+  c->set_output(0, lu_shape);
+  c->set_output(1, p_shape);
+  return Status::OK();
+}
+
 // Input is [...,M,N].
 // First and second outputs are:
 //   [...,M,M]; [...,M,N], if full_matrices is true,
@@ -289,6 +313,14 @@ REGISTER_OP("SelfAdjointEigV2")
     .Attr("T: {double, float, complex64, complex128}")
     .SetShapeFn(SelfAdjointEigV2ShapeFn);
 
+REGISTER_OP("Lu")
+    .Input("input: T")
+    .Output("lu: T")
+    .Output("p: output_idx_type")
+    .Attr("T: {double, float, complex64, complex128}")
+    .Attr("output_idx_type: {int32, int64} = DT_INT32")
+    .SetShapeFn(LuShapeFn);
+
 REGISTER_OP("MatrixSolve")
     .Input("matrix: T")
     .Input("rhs: T")
diff --git a/tensorflow/core/ops/linalg_ops_test.cc b/tensorflow/core/ops/linalg_ops_test.cc
index f4be820def..bfacee14ef 100644
--- a/tensorflow/core/ops/linalg_ops_test.cc
+++ b/tensorflow/core/ops/linalg_ops_test.cc
@@ -274,4 +274,23 @@ TEST(LinalgOpsTest, Svd_ShapeFn) {
   INFER_ERROR("Shape must be at least rank 2 but is rank 1", op, "[1]");
 }
 
+TEST(LinalgOpsTest, Lu_ShapeFn) {
+  ShapeInferenceTestOp op("Lu");
+  INFER_OK(op, "?", "?;?");
+  INFER_ERROR("Shape must be at least rank 2 but is rank 1", op, "[1]");
+  INFER_ERROR("Dimensions must be equal, but are 1 and 2", op, "[1,?,3,4,1,2]");
+
+  INFER_OK(op, "[?,?]", "[d0_0,d0_0];[d0_0]");
+  INFER_OK(op, "[1,?]", "[d0_0,d0_0];[d0_0]");
+  INFER_OK(op, "[?,1]", "[d0_1,d0_1];[d0_1]");
+
+  // Repeat previous block of tests with input rank > 2.
+  INFER_OK(op, "[1,?,3,4,?,?]",
+           "[d0_0,d0_1,d0_2,d0_3,d0_4,d0_4];[d0_0,d0_1,d0_2,d0_3,d0_4]");
+  INFER_OK(op, "[1,?,3,4,1,?]",
+           "[d0_0,d0_1,d0_2,d0_3,d0_4,d0_4];[d0_0,d0_1,d0_2,d0_3,d0_4]");
+  INFER_OK(op, "[1,?,3,4,?,1]",
+           "[d0_0,d0_1,d0_2,d0_3,d0_5,d0_5];[d0_0,d0_1,d0_2,d0_3,d0_5]");
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index f990b0ecc4..242e3b595c 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1888,6 +1888,22 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "lu_op_test",
+    size = "small",
+    srcs = ["lu_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python/ops/linalg",
+    ],
+)
+
 cuda_py_test(
     name = "manip_ops_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/lu_op_test.py b/tensorflow/python/kernel_tests/lu_op_test.py
new file mode 100644
index 0000000000..f2defcae85
--- /dev/null
+++ b/tensorflow/python/kernel_tests/lu_op_test.py
@@ -0,0 +1,285 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.ops.tf.Lu."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import benchmark
+from tensorflow.python.platform import test
+
+
+class LuOpTest(test.TestCase):
+
+  @property
+  def float_types(self):
+    return set((np.float64, np.float32, np.complex64, np.complex128))
+
+  def _verifyLuBase(self, sess, x, lower, upper, perm, verification,
+                    output_idx_type):
+    lower_np, upper_np, perm_np, verification_np = sess.run(
+        [lower, upper, perm, verification])
+
+    self.assertAllClose(x, verification_np)
+    self.assertShapeEqual(x, lower)
+    self.assertShapeEqual(x, upper)
+
+    self.assertAllEqual(x.shape[:-1], perm.shape.as_list())
+
+    # Check dtypes are as expected.
+    self.assertEqual(x.dtype, lower_np.dtype)
+    self.assertEqual(x.dtype, upper_np.dtype)
+    self.assertEqual(output_idx_type.as_numpy_dtype, perm_np.dtype)
+
+    # Check that the permutation is valid.
+    if perm_np.shape[-1] > 0:
+      perm_reshaped = np.reshape(perm_np, (-1, perm_np.shape[-1]))
+      for perm_vector in perm_reshaped:
+        self.assertAllClose(np.arange(len(perm_vector)), np.sort(perm_vector))
+
+  def _verifyLu(self, x, output_idx_type=dtypes.int64):
+    # Verify that Px = LU.
+    with self.cached_session(use_gpu=True) as sess:
+
+      lu, perm = linalg_ops.lu(x, output_idx_type=output_idx_type)
+
+      # Prepare the lower factor of shape num_rows x num_rows
+      lu_shape = np.array(lu.shape.as_list())
+      batch_shape = lu_shape[:-2]
+      num_rows = lu_shape[-2]
+      num_cols = lu_shape[-1]
+
+      lower = array_ops.matrix_band_part(lu, -1, 0)
+
+      if num_rows > num_cols:
+        eye = linalg_ops.eye(
+            num_rows, batch_shape=batch_shape, dtype=lower.dtype)
+        lower = array_ops.concat([lower, eye[..., num_cols:]], axis=-1)
+      elif num_rows < num_cols:
+        lower = lower[..., :num_rows]
+
+      # Fill the diagonal with ones.
+      ones_diag = array_ops.ones(
+          np.append(batch_shape, num_rows), dtype=lower.dtype)
+      lower = array_ops.matrix_set_diag(lower, ones_diag)
+
+      # Prepare the upper factor.
+      upper = array_ops.matrix_band_part(lu, 0, -1)
+
+      verification = math_ops.matmul(lower, upper)
+
+      # Permute the rows of product of the Cholesky factors.
+      if num_rows > 0:
+        # Reshape the product of the triangular factors and permutation indices
+        # to a single batch dimension. This makes it easy to apply
+        # invert_permutation and gather_nd ops.
+        perm_reshaped = array_ops.reshape(perm, [-1, num_rows])
+        verification_reshaped = array_ops.reshape(verification,
+                                                  [-1, num_rows, num_cols])
+        # Invert the permutation in each batch.
+        inv_perm_reshaped = functional_ops.map_fn(array_ops.invert_permutation,
+                                                  perm_reshaped)
+        batch_size = perm_reshaped.shape.as_list()[0]
+        # Prepare the batch indices with the same shape as the permutation.
+        # The corresponding batch index is paired with each of the `num_rows`
+        # permutation indices.
+        batch_indices = math_ops.cast(
+            array_ops.broadcast_to(
+                math_ops.range(batch_size)[:, None], perm_reshaped.shape),
+            dtype=output_idx_type)
+        permuted_verification_reshaped = array_ops.gather_nd(
+            verification_reshaped,
+            array_ops.stack([batch_indices, inv_perm_reshaped], axis=-1))
+
+        # Reshape the verification matrix back to the original shape.
+        verification = array_ops.reshape(permuted_verification_reshaped,
+                                         lu_shape)
+
+      self._verifyLuBase(sess, x, lower, upper, perm, verification,
+                         output_idx_type)
+
+  def testBasic(self):
+    data = np.array([[4., -1., 2.], [-1., 6., 0], [10., 0., 5.]])
+
+    for dtype in (np.float32, np.float64):
+      for output_idx_type in (dtypes.int32, dtypes.int64):
+        self._verifyLu(data.astype(dtype), output_idx_type=output_idx_type)
+
+    for dtype in (np.complex64, np.complex128):
+      for output_idx_type in (dtypes.int32, dtypes.int64):
+        complex_data = np.tril(1j * data, -1).astype(dtype)
+        complex_data += np.triu(-1j * data, 1).astype(dtype)
+        complex_data += data
+        self._verifyLu(complex_data, output_idx_type=output_idx_type)
+
+  def testPivoting(self):
+    with self.session(use_gpu=True) as sess:
+      # This matrix triggers partial pivoting because the first diagonal entry
+      # is small.
+      data = np.array([[1e-9, 1., 0.], [1., 0., 0], [0., 1., 5]])
+      self._verifyLu(data.astype(np.float32))
+
+      for dtype in (np.float32, np.float64):
+        self._verifyLu(data.astype(dtype))
+        _, p = linalg_ops.lu(data)
+        p_val = sess.run([p])
+        # Make sure p_val is not the identity permutation.
+        self.assertNotAllClose(np.arange(3), p_val)
+
+      for dtype in (np.complex64, np.complex128):
+        complex_data = np.tril(1j * data, -1).astype(dtype)
+        complex_data += np.triu(-1j * data, 1).astype(dtype)
+        complex_data += data
+        self._verifyLu(complex_data)
+        _, p = linalg_ops.lu(data)
+        p_val = sess.run([p])
+        # Make sure p_val is not the identity permutation.
+        self.assertNotAllClose(np.arange(3), p_val)
+
+  def testInvalidMatrix(self):
+    # LU factorization gives an error when the input is singular.
+    # Note: A singular matrix may return without error but it won't be a valid
+    # factorization.
+    with self.session(use_gpu=True) as sess:
+      for dtype in self.float_types:
+        with self.assertRaises(errors.InvalidArgumentError):
+          sess.run(
+              linalg_ops.lu(
+                  np.array([[1., 2., 3.], [2., 4., 6.], [2., 3., 4.]],
+                           dtype=dtype)))
+        with self.assertRaises(errors.InvalidArgumentError):
+          sess.run(
+              linalg_ops.lu(
+                  np.array([[[1., 2., 3.], [2., 4., 6.], [1., 2., 3.]],
+                            [[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]]],
+                           dtype=dtype)))
+
+  def testBatch(self):
+    simple_array = np.array([[[1., -1.], [2., 5.]]])  # shape (1, 2, 2)
+    self._verifyLu(simple_array)
+    self._verifyLu(np.vstack((simple_array, simple_array)))
+    odd_sized_array = np.array([[[4., -1., 2.], [-1., 6., 0], [2., 0., 5.]]])
+    self._verifyLu(np.vstack((odd_sized_array, odd_sized_array)))
+
+    batch_size = 200
+
+    # Generate random matrices.
+    np.random.seed(42)
+    matrices = np.random.rand(batch_size, 5, 5)
+    self._verifyLu(matrices)
+
+    # Generate random complex valued matrices.
+    np.random.seed(52)
+    matrices = np.random.rand(batch_size, 5,
+                              5) + 1j * np.random.rand(batch_size, 5, 5)
+    self._verifyLu(matrices)
+
+  def testLargeMatrix(self):
+    # Generate random matrices.
+    n = 500
+    np.random.seed(64)
+    data = np.random.rand(n, n)
+    self._verifyLu(data)
+
+    # Generate random complex valued matrices.
+    np.random.seed(129)
+    data = np.random.rand(n, n) + 1j * np.random.rand(n, n)
+    self._verifyLu(data)
+
+  def testEmpty(self):
+    self._verifyLu(np.empty([0, 2, 2]))
+    self._verifyLu(np.empty([2, 0, 0]))
+
+  def testConcurrentExecutesWithoutError(self):
+    with self.session(use_gpu=True) as sess:
+      matrix1 = random_ops.random_normal([5, 5], seed=42)
+      matrix2 = random_ops.random_normal([5, 5], seed=42)
+      lu1, p1 = linalg_ops.lu(matrix1)
+      lu2, p2 = linalg_ops.lu(matrix2)
+      lu1_val, p1_val, lu2_val, p2_val = sess.run([lu1, p1, lu2, p2])
+      self.assertAllEqual(lu1_val, lu2_val)
+      self.assertAllEqual(p1_val, p2_val)
+
+
+class LuBenchmark(test.Benchmark):
+  shapes = [
+      (4, 4),
+      (10, 10),
+      (16, 16),
+      (101, 101),
+      (256, 256),
+      (1000, 1000),
+      (1024, 1024),
+      (2048, 2048),
+      (4096, 4096),
+      (513, 2, 2),
+      (513, 8, 8),
+      (513, 256, 256),
+      (4, 513, 2, 2),
+  ]
+
+  def _GenerateMatrix(self, shape):
+    batch_shape = shape[:-2]
+    shape = shape[-2:]
+    assert shape[0] == shape[1]
+    n = shape[0]
+    matrix = np.ones(shape).astype(np.float32) / (2.0 * n) + np.diag(
+        np.ones(n).astype(np.float32))
+    return np.tile(matrix, batch_shape + (1, 1))
+
+  def benchmarkLuOp(self):
+    for shape in self.shapes:
+      with ops.Graph().as_default(), \
+          session.Session(config=benchmark.benchmark_config()) as sess, \
+          ops.device("/cpu:0"):
+        matrix = variables.Variable(self._GenerateMatrix(shape))
+        lu, p = linalg_ops.lu(matrix)
+        variables.global_variables_initializer().run()
+        self.run_op_benchmark(
+            sess,
+            control_flow_ops.group(lu, p),
+            min_iters=25,
+            name="lu_cpu_{shape}".format(shape=shape))
+
+      if test.is_gpu_available(True):
+        with ops.Graph().as_default(), \
+            session.Session(config=benchmark.benchmark_config()) as sess, \
+            ops.device("/device:GPU:0"):
+          matrix = variables.Variable(self._GenerateMatrix(shape))
+          lu, p = linalg_ops.lu(matrix)
+          variables.global_variables_initializer().run()
+          self.run_op_benchmark(
+              sess,
+              control_flow_ops.group(lu, p),
+              min_iters=25,
+              name="lu_gpu_{shape}".format(shape=shape))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
index 2c9476a9bd..df2bd887cd 100644
--- a/tensorflow/python/ops/linalg/linalg_impl.py
+++ b/tensorflow/python/ops/linalg/linalg_impl.py
@@ -44,6 +44,7 @@ einsum = special_math_ops.einsum
 eye = linalg_ops.eye
 inv = linalg_ops.matrix_inverse
 logm = gen_linalg_ops.matrix_logarithm
+lu = gen_linalg_ops.lu
 tf_export('linalg.logm')(logm)
 lstsq = linalg_ops.matrix_solve_ls
 norm = linalg_ops.norm
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
index 1a4098d121..9f7b422fab 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
@@ -132,6 +132,10 @@ tf_module {
     name: "lstsq"
     argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'True\', \'None\'], "
   }
+  member_method {
+    name: "lu"
+    argspec: "args=[\'input\', \'output_idx_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
+  }
   member_method {
     name: "matmul"
     argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'adjoint_a\', \'adjoint_b\', \'a_is_sparse\', \'b_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'False\', \'False\', \'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index a3599bfa80..3e1e2e3d54 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -132,6 +132,10 @@ tf_module {
     name: "lstsq"
     argspec: "args=[\'matrix\', \'rhs\', \'l2_regularizer\', \'fast\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'True\', \'None\'], "
   }
+  member_method {
+    name: "lu"
+    argspec: "args=[\'input\', \'output_idx_type\', \'name\'], varargs=None, keywords=None, defaults=[\"<dtype: \'int32\'>\", \'None\'], "
+  }
   member_method {
     name: "matmul"
     argspec: "args=[\'a\', \'b\', \'transpose_a\', \'transpose_b\', \'adjoint_a\', \'adjoint_b\', \'a_is_sparse\', \'b_is_sparse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'False\', \'False\', \'False\', \'None\'], "
-- 
GitLab


From 95b05275719b04d9c2b01835280712940c15bfcd Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 5 Dec 2018 16:56:56 -0800
Subject: [PATCH 1398/1554] [tf.data] Dataset.output_{types,shapes,classes} ->
 Dataset._element_structure.

This change adds a new abstract property called `tf.data.Dataset._element_structure`, to represent the potentially nested structure of dataset elements, and updates all `Dataset` subclasses to implement this property in preference to the existing `output_types`, `output_shapes`, and `output_classes` properties. These latter "legacy structure" properties remain, but will be removed from the API in TensorFlow 2.0, and only retained in 1.X compatibility mode.

To avoid breaking users who have extended `tf.data.Dataset`, a temporary implementation of `DatasetV1._element_structure` is provided using `structure.convert_legacy_structure()` to convert the existing legacy structure properties into a `Structure`.

To update other `Dataset` implementations to implement the new API:

1. Compute the `Structure` for your dataset.
   (a) If you have built a legacy structure with `output_types`, `output_shapes`, and `output_classes`, you can set `self._element_structure = structure.convert_legacy_structure(output_types, output_shapes, output_classes)`.
   (b) Alternatively, it may be easier to construct a `structure.TensorStructure`, `structure.SparseTensorStructure`, and/or `structure.NestedStructure` directly. See some of the modifications in this change for examples.

2. Implement the `element_structure` property, and delete the legacy structure properties from your implementation:

        @property
        def _element_structure(self):
          return self._element_structure

3. (Optional.) Update your implementation to use the `_element_structure` property of the input dataset(s) rather than the legacy structure properties, which will be removed in a subsequent change.

PiperOrigin-RevId: 224246206
---
 .../bigtable/python/ops/bigtable_api.py       |  57 +--
 tensorflow/contrib/data/python/ops/readers.py |  21 +-
 tensorflow/contrib/data/python/ops/sliding.py |  19 +-
 .../hadoop/python/ops/hadoop_dataset_ops.py   |  19 +-
 .../ignite/python/ops/ignite_dataset_ops.py   |  42 +-
 .../kafka/python/ops/kafka_dataset_ops.py     |  14 +-
 .../kinesis/python/ops/kinesis_dataset_ops.py |  14 +-
 tensorflow/python/data/experimental/ops/BUILD |   8 +-
 .../python/data/experimental/ops/batching.py  |  83 ++--
 .../python/data/experimental/ops/grouping.py  | 128 ++---
 .../experimental/ops/indexed_dataset_ops.py   |  52 +--
 .../data/experimental/ops/interleave_ops.py   |  31 +-
 .../data/experimental/ops/matching_files.py   |  16 +-
 .../data/experimental/ops/parsing_ops.py      |  26 +-
 .../data/experimental/ops/prefetching_ops.py  |  12 +-
 .../data/experimental/ops/random_ops.py       |  15 +-
 .../python/data/experimental/ops/readers.py   |  41 +-
 .../python/data/experimental/ops/scan_ops.py  |  27 +-
 .../python/data/experimental/ops/sleep.py     |  12 -
 tensorflow/python/data/ops/BUILD              |   1 +
 tensorflow/python/data/ops/dataset_ops.py     | 438 +++++++-----------
 tensorflow/python/data/ops/iterator_ops.py    |   6 +-
 .../data/ops/multi_device_iterator_ops.py     |  58 +--
 tensorflow/python/data/ops/readers.py         |  52 +--
 tensorflow/python/data/util/structure.py      | 103 ++--
 tensorflow/python/data/util/structure_test.py |   6 +-
 .../golden/v1/tensorflow.data.-dataset.pbtxt  |   6 +-
 .../golden/v2/tensorflow.data.-dataset.pbtxt  |   6 +-
 28 files changed, 444 insertions(+), 869 deletions(-)

diff --git a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
index 9f97934193..b6cdc7aab0 100644
--- a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
+++ b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
@@ -35,8 +35,8 @@ from tensorflow.contrib.util import loader
 from tensorflow.python.data.experimental.ops import interleave_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.platform import resource_loader
 
@@ -111,8 +111,7 @@ class BigtableClient(object):
 
 
 class BigtableTable(object):
-  """BigtableTable is the entrypoint for reading and writing data in Cloud
-  Bigtable.
+  """Entry point for reading and writing data in Cloud Bigtable.
 
   This BigtableTable class is the Python representation of the Cloud Bigtable
   table within TensorFlow. Methods on this class allow data to be read from and
@@ -593,16 +592,8 @@ class _BigtableKeyDataset(dataset_ops.DatasetSource):
     self._table = table
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.TensorShape([])
-
-  @property
-  def output_types(self):
-    return dtypes.string
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
 
 
 class _BigtablePrefixKeyDataset(_BigtableKeyDataset):
@@ -662,16 +653,9 @@ class _BigtableLookupDataset(dataset_ops.DatasetSource):
     self._columns = [i[1] for i in normalized]
 
   @property
-  def output_classes(self):
-    return tuple([ops.Tensor] * self._num_outputs)
-
-  @property
-  def output_shapes(self):
-    return tuple([tensor_shape.TensorShape([])] * self._num_outputs)
-
-  @property
-  def output_types(self):
-    return tuple([dtypes.string] * self._num_outputs)
+  def _element_structure(self):
+    return structure.NestedStructure(tuple(
+        [structure.TensorStructure(dtypes.string, [])] * self._num_outputs))
 
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
@@ -697,16 +681,9 @@ class _BigtableScanDataset(dataset_ops.DatasetSource):
     self._num_outputs = len(normalized) + 1  # 1 for row key
 
   @property
-  def output_classes(self):
-    return tuple([ops.Tensor] * self._num_outputs)
-
-  @property
-  def output_shapes(self):
-    return tuple([tensor_shape.TensorShape([])] * self._num_outputs)
-
-  @property
-  def output_types(self):
-    return tuple([dtypes.string] * self._num_outputs)
+  def _element_structure(self):
+    return structure.NestedStructure(tuple(
+        [structure.TensorStructure(dtypes.string, [])] * self._num_outputs))
 
   def _as_variant_tensor(self):
     return gen_bigtable_ops.bigtable_scan_dataset(
@@ -730,16 +707,10 @@ class _BigtableSampleKeyPairsDataset(dataset_ops.DatasetSource):
     self._end = end
 
   @property
-  def output_classes(self):
-    return (ops.Tensor, ops.Tensor)
-
-  @property
-  def output_shapes(self):
-    return (tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))
-
-  @property
-  def output_types(self):
-    return (dtypes.string, dtypes.string)
+  def _element_structure(self):
+    return structure.NestedStructure(
+        (structure.TensorStructure(dtypes.string, []),
+         structure.TensorStructure(dtypes.string, [])))
 
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index aa42782807..c0152156a1 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -21,10 +21,9 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.experimental.ops import readers
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers as core_readers
-from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util import deprecation
 
@@ -396,18 +395,10 @@ class LMDBDataset(dataset_ops.DatasetSource):
 
   def _as_variant_tensor(self):
     return gen_experimental_dataset_ops.experimental_lmdb_dataset(
-        self._filenames,
-        output_types=nest.flatten(self.output_types),
-        output_shapes=nest.flatten(self.output_shapes))
+        self._filenames, **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return ops.Tensor, ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return (tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))
-
-  @property
-  def output_types(self):
-    return dtypes.string, dtypes.string
+  def _element_structure(self):
+    return structure.NestedStructure(
+        (structure.TensorStructure(dtypes.string, []),
+         structure.TensorStructure(dtypes.string, [])))
diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py
index 9ebdca317f..5c6ee6bfdc 100644
--- a/tensorflow/contrib/data/python/ops/sliding.py
+++ b/tensorflow/contrib/data/python/ops/sliding.py
@@ -39,11 +39,10 @@ class _SlideDataset(dataset_ops.UnaryDataset):
     self._window_shift = ops.convert_to_tensor(
         window_shift, dtype=dtypes.int64, name="window_shift")
 
-    # pylint: disable=protected-access
-    input_structure = structure.Structure._from_legacy_structure(
+    input_structure = structure.convert_legacy_structure(
         input_dataset.output_types, input_dataset.output_shapes,
         input_dataset.output_classes)
-    self._output_structure = input_structure._batch(None)
+    self._structure = input_structure._batch(None)  # pylint: disable=protected-access
 
   def _as_variant_tensor(self):
     return ged_ops.experimental_sliding_window_dataset(
@@ -51,19 +50,11 @@ class _SlideDataset(dataset_ops.UnaryDataset):
         window_size=self._window_size,
         window_shift=self._window_shift,
         window_stride=self._window_stride,
-        **dataset_ops.flat_structure(structure=self._output_structure))
+        **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
-
-  @property
-  def output_shapes(self):
-    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
-
-  @property
-  def output_types(self):
-    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
+  def _element_structure(self):
+    return self._structure
 
 
 @deprecation.deprecated_args(
diff --git a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py
index d3fcc8cb2a..5c5599858e 100644
--- a/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py
+++ b/tensorflow/contrib/hadoop/python/ops/hadoop_dataset_ops.py
@@ -20,10 +20,9 @@ from __future__ import print_function
 from tensorflow.contrib.hadoop.python.ops import gen_dataset_ops
 from tensorflow.contrib.hadoop.python.ops import hadoop_op_loader  # pylint: disable=unused-import
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 
 
 class SequenceFileDataset(dataset_ops.DatasetSource):
@@ -57,16 +56,10 @@ class SequenceFileDataset(dataset_ops.DatasetSource):
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.sequence_file_dataset(
-        self._filenames, nest.flatten(self.output_types))
+        self._filenames, self._element_structure._flat_types)  # pylint: disable=protected-access
 
   @property
-  def output_classes(self):
-    return ops.Tensor, ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return (tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))
-
-  @property
-  def output_types(self):
-    return dtypes.string, dtypes.string
+  def _element_structure(self):
+    return structure.NestedStructure(
+        (structure.TensorStructure(dtypes.string, []),
+         structure.TensorStructure(dtypes.string, [])))
diff --git a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py
index 936b29a4f5..e4762c91b1 100644
--- a/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py
+++ b/tensorflow/contrib/ignite/python/ops/ignite_dataset_ops.py
@@ -27,6 +27,7 @@ import six
 from tensorflow.contrib.ignite.python.ops import gen_dataset_ops
 from tensorflow.contrib.ignite.python.ops import ignite_op_loader  # pylint: disable=unused-import
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -34,10 +35,7 @@ from tensorflow.python.framework import tensor_shape
 
 @six.add_metaclass(abc.ABCMeta)
 class Readable(object):
-  """Readable abstract class that exposes methods to do reading-related
-
-     operations.
-  """
+  """Abstract class that exposes methods to do reading-related operations."""
 
   @abc.abstractmethod
   def __init__(self):
@@ -227,10 +225,7 @@ types = {
 
 
 class TypeTreeNode(object):
-  """TypeTreeNode class exposes methods to format object tree structure
-
-     data.
-  """
+  """TypeTreeNode class exposes methods to format object tree structure data."""
 
   def __init__(self, name, type_id, fields=None, permutation=None):
     """Constructs a new instance of TypeTreeNode.
@@ -692,14 +687,14 @@ class IgniteClient(TcpClient):
 
 
 class IgniteDataset(dataset_ops.DatasetSource):
-  """Apache Ignite is a memory-centric distributed database, caching, and
-
-     processing platform for transactional, analytical, and streaming workloads,
-     delivering in-memory speeds at petabyte scale. This contrib package
-     contains an integration between Apache Ignite and TensorFlow. The
-     integration is based on tf.data from TensorFlow side and Binary Client
-     Protocol from Apache Ignite side. It allows to use Apache Ignite as a
-     datasource for neural network training, inference and all other
+  """Apache Ignite is a memory-centric distributed database.
+
+     It acts as a caching and processing platform for transactional, analytical,
+     and streaming workloads, delivering in-memory speeds at petabyte scale.
+     This contrib package contains an integration between Apache Ignite and
+     TensorFlow. The integration is based on tf.data from TensorFlow side and
+     Binary Client Protocol from Apache Ignite side. It allows to use Apache
+     Ignite as a datasource for neural network training, inference and all other
      computations supported by TensorFlow. Ignite Dataset is based on Apache
      Ignite Binary Client Protocol.
   """
@@ -756,6 +751,9 @@ class IgniteDataset(dataset_ops.DatasetSource):
         self.cache_type.to_permutation(),
         dtype=dtypes.int32,
         name="permutation")
+    self._structure = structure.convert_legacy_structure(
+        self.cache_type.to_output_types(), self.cache_type.to_output_shapes(),
+        self.cache_type.to_output_classes())
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.ignite_dataset(self.cache_name, self.host, self.port,
@@ -763,13 +761,5 @@ class IgniteDataset(dataset_ops.DatasetSource):
                                           self.schema, self.permutation)
 
   @property
-  def output_classes(self):
-    return self.cache_type.to_output_classes()
-
-  @property
-  def output_shapes(self):
-    return self.cache_type.to_output_shapes()
-
-  @property
-  def output_types(self):
-    return self.cache_type.to_output_types()
+  def _element_structure(self):
+    return self._structure
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
index 7129f09e8b..2b86331099 100644
--- a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
+++ b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
@@ -20,9 +20,9 @@ from __future__ import print_function
 from tensorflow.contrib.kafka.python.ops import gen_dataset_ops
 from tensorflow.contrib.kafka.python.ops import kafka_op_loader  # pylint: disable=unused-import
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 
 
 class KafkaDataset(dataset_ops.DatasetSource):
@@ -63,13 +63,5 @@ class KafkaDataset(dataset_ops.DatasetSource):
                                          self._group, self._eof, self._timeout)
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.string
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
diff --git a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
index c392adbb1d..2039539528 100644
--- a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
+++ b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
@@ -20,9 +20,9 @@ from __future__ import print_function
 from tensorflow.contrib.kinesis.python.ops import gen_dataset_ops
 from tensorflow.contrib.kinesis.python.ops import kinesis_op_loader  # pylint: disable=unused-import
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 
 
 class KinesisDataset(dataset_ops.DatasetSource):
@@ -81,13 +81,5 @@ class KinesisDataset(dataset_ops.DatasetSource):
         self._stream, self._shard, self._read_indefinitely, self._interval)
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.string
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 43f43182f6..60c20e0bcf 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -70,8 +70,7 @@ py_library(
         "//tensorflow/python:random_seed",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
-        "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
@@ -230,6 +229,8 @@ py_library(
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
@@ -268,7 +269,7 @@ py_library(
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
@@ -303,6 +304,7 @@ py_library(
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
         "//tensorflow/python/data/util:sparse",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 48b06d6ff1..9db9a28dd8 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -25,6 +25,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import convert
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -365,23 +366,19 @@ class _UnbatchDataset(dataset_ops.UnaryDataset):
                          "different batch sizes.")
     self._input_dataset = input_dataset
 
+    self._structure = structure.convert_legacy_structure(
+        input_dataset.output_types,
+        nest.map_structure(lambda s: s[1:], input_dataset.output_shapes),
+        input_dataset.output_classes)
+
   def _as_variant_tensor(self):
     return ged_ops.experimental_unbatch_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return nest.map_structure(lambda s: s[1:],
-                              self._input_dataset.output_shapes)
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
+  def _element_structure(self):
+    return self._structure
 
 
 @tf_export("data.experimental.unbatch")
@@ -454,6 +451,9 @@ class _DenseToSparseBatchDataset(dataset_ops.UnaryDataset):
     self._input_dataset = input_dataset
     self._batch_size = batch_size
     self._row_shape = row_shape
+    self._structure = structure.SparseTensorStructure(
+        input_dataset.output_types,
+        tensor_shape.vector(None).concatenate(self._row_shape))
 
   def _as_variant_tensor(self):
     return ged_ops.experimental_dense_to_sparse_batch_dataset(
@@ -463,16 +463,8 @@ class _DenseToSparseBatchDataset(dataset_ops.UnaryDataset):
         **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return sparse_tensor.SparseTensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.vector(None).concatenate(self._row_shape)
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
+  def _element_structure(self):
+    return self._structure
 
 
 class _RestructuredDataset(dataset_ops.UnaryDataset):
@@ -523,13 +515,10 @@ class _RestructuredDataset(dataset_ops.UnaryDataset):
             "Dataset with output types %r cannot be restructured to have "
             "output types %r" % (dataset.output_types, output_types))
 
-    self._output_types = output_types
-
     if output_shapes is None:
       # Inherit shapes from the original `dataset`.
-      self._output_shapes = nest.pack_sequence_as(output_types,
-                                                  nest.flatten(
-                                                      dataset.output_shapes))
+      output_shapes = nest.pack_sequence_as(
+          output_types, nest.flatten(dataset.output_shapes))
     else:
       if not allow_unsafe_cast:
         # Validate that the shapes are compatible.
@@ -544,30 +533,22 @@ class _RestructuredDataset(dataset_ops.UnaryDataset):
                 "Dataset with output shapes %r cannot be restructured to have "
                 "incompatible output shapes %r" % (dataset.output_shapes,
                                                    output_shapes))
-      self._output_shapes = nest.map_structure_up_to(
+      output_shapes = nest.map_structure_up_to(
           output_types, tensor_shape.as_shape, output_shapes)
     if output_classes is None:
       # Inherit class types from the original `dataset`.
-      self._output_classes = nest.pack_sequence_as(output_types,
-                                                   nest.flatten(
-                                                       dataset.output_classes))
-    else:
-      self._output_classes = output_classes
+      output_classes = nest.pack_sequence_as(
+          output_types, nest.flatten(dataset.output_classes))
+
+    self._structure = structure.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
 
   def _as_variant_tensor(self):
     return self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
 
   @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_types(self):
-    return self._output_types
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
+  def _element_structure(self):
+    return self._structure
 
 
 class _MapAndBatchDataset(dataset_ops.UnaryDataset):
@@ -591,10 +572,10 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     if constant_drop_remainder:
       # NOTE(mrry): `constant_drop_remainder` may be `None` (unknown statically)
       # or `False` (explicitly retaining the remainder).
-      self._output_structure = self._map_func.output_structure._batch(  # pylint: disable=protected-access
+      self._structure = self._map_func.output_structure._batch(  # pylint: disable=protected-access
           tensor_util.constant_value(self._batch_size_t))
     else:
-      self._output_structure = self._map_func.output_structure._batch(None)  # pylint: disable=protected-access
+      self._structure = self._map_func.output_structure._batch(None)  # pylint: disable=protected-access
 
   def _functions(self):
     return [self._map_func]
@@ -609,19 +590,11 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
         num_parallel_calls=self._num_parallel_calls_t,
         drop_remainder=self._drop_remainder_t,
         preserve_cardinality=True,
-        **dataset_ops.flat_structure(structure=self._output_structure))
-
-  @property
-  def output_classes(self):
-    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
-
-  @property
-  def output_shapes(self):
-    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
+        **dataset_ops.flat_structure(self))
 
   @property
-  def output_types(self):
-    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
+  def _element_structure(self):
+    return self._structure
 
 
 @tf_export("data.experimental.map_and_batch")
diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index ad9fe9a4e8..ef6b232429 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -253,50 +253,44 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
 
   def _make_key_func(self, key_func, input_dataset):
     """Make wrapping defun for key_func."""
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+    self._key_func = dataset_ops.StructuredFunctionWrapper(
         key_func, self._transformation_name(), dataset=input_dataset)
-    if not (
-        wrapped_func.output_types == dtypes.int64 and
-        wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
+    if not self._key_func.output_structure.is_compatible_with(
+        structure.TensorStructure(dtypes.int64, [])):
       raise ValueError(
           "`key_func` must return a single tf.int64 tensor. "
           "Got type=%s and shape=%s"
-          % (wrapped_func.output_types, wrapped_func.output_shapes))
-    self._key_func = wrapped_func
-
+          % (self._key_func.output_types, self._key_func.output_shapes))
   def _make_init_func(self, init_func):
     """Make wrapping defun for init_func."""
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+    self._init_func = dataset_ops.StructuredFunctionWrapper(
         init_func,
         self._transformation_name(),
-        input_classes=ops.Tensor,
-        input_shapes=tensor_shape.scalar(),
-        input_types=dtypes.int64)
-    self._init_func = wrapped_func
-    self._state_classes = wrapped_func.output_classes
-    self._state_shapes = wrapped_func.output_shapes
-    self._state_types = wrapped_func.output_types
+        input_structure=structure.TensorStructure(dtypes.int64, []))
 
   def _make_reduce_func(self, reduce_func, input_dataset):
     """Make wrapping defun for reduce_func."""
 
     # Iteratively rerun the reduce function until reaching a fixed point on
-    # `self._state_shapes`.
+    # `self._state_structure`.
+    self._state_structure = self._init_func.output_structure
+    state_types = self._init_func.output_types
+    state_shapes = self._init_func.output_shapes
+    state_classes = self._init_func.output_classes
     need_to_rerun = True
     while need_to_rerun:
 
       wrapped_func = dataset_ops.StructuredFunctionWrapper(
           reduce_func,
           self._transformation_name(),
-          input_classes=(self._state_classes, input_dataset.output_classes),
-          input_shapes=(self._state_shapes, input_dataset.output_shapes),
-          input_types=(self._state_types, input_dataset.output_types),
+          input_structure=structure.NestedStructure(
+              (self._state_structure, input_dataset._element_structure)),  # pylint: disable=protected-access
           add_to_graph=False)
 
       # Extract and validate class information from the returned values.
       for new_state_class, state_class in zip(
           nest.flatten(wrapped_func.output_classes),
-          nest.flatten(self._state_classes)):
+          nest.flatten(state_classes)):
         if not issubclass(new_state_class, state_class):
           raise TypeError(
               "The element classes for the new state must match the initial "
@@ -305,16 +299,15 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
 
       # Extract and validate type information from the returned values.
       for new_state_type, state_type in zip(
-          nest.flatten(wrapped_func.output_types),
-          nest.flatten(self._state_types)):
+          nest.flatten(wrapped_func.output_types), nest.flatten(state_types)):
         if new_state_type != state_type:
           raise TypeError(
               "The element types for the new state must match the initial "
               "state. Expected %s; got %s." %
-              (self._state_types, wrapped_func.output_types))
+              (self._init_func.output_types, wrapped_func.output_types))
 
       # Extract shape information from the returned values.
-      flat_state_shapes = nest.flatten(self._state_shapes)
+      flat_state_shapes = nest.flatten(state_shapes)
       flat_new_state_shapes = nest.flatten(wrapped_func.output_shapes)
       weakened_state_shapes = [
           original.most_specific_compatible_shape(new)
@@ -331,36 +324,23 @@ class _GroupByReducerDataset(dataset_ops.UnaryDataset):
           break
 
       if need_to_rerun:
-        self._state_shapes = nest.pack_sequence_as(self._state_shapes,
-                                                   weakened_state_shapes)
+        state_shapes = nest.pack_sequence_as(
+            self._init_func.output_shapes, weakened_state_shapes)
+        self._state_structure = structure.convert_legacy_structure(
+            state_types, state_shapes, state_classes)
 
     self._reduce_func = wrapped_func
     self._reduce_func.function.add_to_graph(ops.get_default_graph())
 
   def _make_finalize_func(self, finalize_func):
     """Make wrapping defun for finalize_func."""
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
-        finalize_func,
-        self._transformation_name(),
-        input_classes=self._state_classes,
-        input_shapes=self._state_shapes,
-        input_types=self._state_types)
-    self._finalize_func = wrapped_func
-    self._output_classes = wrapped_func.output_classes
-    self._output_shapes = wrapped_func.output_shapes
-    self._output_types = wrapped_func.output_types
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
+    self._finalize_func = dataset_ops.StructuredFunctionWrapper(
+        finalize_func, self._transformation_name(),
+        input_structure=self._state_structure)
 
   @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._finalize_func.output_structure
 
   def _functions(self):
     return [
@@ -402,66 +382,46 @@ class _GroupByWindowDataset(dataset_ops.UnaryDataset):
 
     def window_size_func_wrapper(key):
       return ops.convert_to_tensor(window_size_func(key), dtype=dtypes.int64)
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+    self._window_size_func = dataset_ops.StructuredFunctionWrapper(
         window_size_func_wrapper,
         self._transformation_name(),
-        input_classes=ops.Tensor,
-        input_shapes=tensor_shape.scalar(),
-        input_types=dtypes.int64)
-    if not (
-        wrapped_func.output_types == dtypes.int64 and
-        wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
+        input_structure=structure.TensorStructure(dtypes.int64, []))
+    if not self._window_size_func.output_structure.is_compatible_with(
+        structure.TensorStructure(dtypes.int64, [])):
       raise ValueError(
           "`window_size_func` must return a single tf.int64 scalar tensor.")
-    self._window_size_func = wrapped_func
 
   def _make_key_func(self, key_func, input_dataset):
     """Make wrapping defun for key_func."""
 
     def key_func_wrapper(*args):
       return ops.convert_to_tensor(key_func(*args), dtype=dtypes.int64)
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+    self._key_func = dataset_ops.StructuredFunctionWrapper(
         key_func_wrapper, self._transformation_name(), dataset=input_dataset)
-    if not (
-        wrapped_func.output_types == dtypes.int64 and
-        wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
+    if not self._key_func.output_structure.is_compatible_with(
+        structure.TensorStructure(dtypes.int64, [])):
       raise ValueError(
           "`key_func` must return a single tf.int64 scalar tensor.")
-    self._key_func = wrapped_func
 
   def _make_reduce_func(self, reduce_func, input_dataset):
     """Make wrapping defun for reduce_func."""
     nested_dataset = dataset_ops.DatasetStructure(
-        structure.Structure._from_legacy_structure(  # pylint: disable=protected-access
-            input_dataset.output_types, input_dataset.output_shapes,
-            input_dataset.output_classes))
-    wrapped_func = dataset_ops.StructuredFunctionWrapper(
-        reduce_func,
-        self._transformation_name(),
-        input_classes=(ops.Tensor, nested_dataset),
-        input_shapes=(tensor_shape.scalar(), nested_dataset),
-        input_types=(dtypes.int64, nested_dataset))
+        input_dataset._element_structure)  # pylint: disable=protected-access
+    input_structure = structure.NestedStructure(
+        (structure.TensorStructure(dtypes.int64, []), nested_dataset))
+    self._reduce_func = dataset_ops.StructuredFunctionWrapper(
+        reduce_func, self._transformation_name(),
+        input_structure=input_structure)
     if not isinstance(
-        wrapped_func.output_structure, dataset_ops.DatasetStructure):
+        self._reduce_func.output_structure, dataset_ops.DatasetStructure):
       raise TypeError("`reduce_func` must return a `Dataset` object.")
     # pylint: disable=protected-access
-    element_structure = wrapped_func.output_structure._element_structure
-    self._output_classes = element_structure._to_legacy_output_classes()
-    self._output_types = element_structure._to_legacy_output_types()
-    self._output_shapes = element_structure._to_legacy_output_shapes()
-    self._reduce_func = wrapped_func
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
+    self._structure = (
+        self._reduce_func.output_structure._element_structure)
 
   @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
   def _functions(self):
     return [self._key_func, self._reduce_func, self._window_size_func]
diff --git a/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py b/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py
index 570f0116f7..fdf3692420 100644
--- a/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py
+++ b/tensorflow/python/data/experimental/ops/indexed_dataset_ops.py
@@ -22,9 +22,9 @@ import abc
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 
 
@@ -94,11 +94,7 @@ class IndexedDataset(dataset_ops.Dataset):
         ged_ops.experimental_materialized_index_dataset_handle(
             container=container,
             shared_name=shared_name,
-            output_types=nest.flatten(
-                sparse.as_dense_types(self.output_types, self.output_classes)),
-            output_shapes=nest.flatten(
-                sparse.as_dense_types(self.output_shapes,
-                                      self.output_classes))))
+            **dataset_ops.flat_structure(self)))
 
     with ops.colocate_with(materialized_resource):
       materializer = ged_ops.experimental_indexed_dataset_materialize(
@@ -107,38 +103,6 @@ class IndexedDataset(dataset_ops.Dataset):
                                       self.output_classes, self.output_types,
                                       self.output_shapes)
 
-  @abc.abstractproperty
-  def output_types(self):
-    """Returns the type of each component of an element of this IndexedDataset.
-
-    Returns:
-      A nested structure of `tf.DType` objects corresponding to each component
-      of an element of this IndexedDataset.
-    """
-    raise NotImplementedError("IndexedDataset.output_types")
-
-  @abc.abstractproperty
-  def output_classes(self):
-    """Returns the class of each component of an element of this IndexedDataset.
-
-    The expected values are `tf.Tensor` and `tf.SparseTensor`.
-
-    Returns:
-      A nested structure of Python `type` objects corresponding to each
-      component of an element of this IndexedDataset.
-    """
-    raise NotImplementedError("IndexedDataset.output_classes")
-
-  @abc.abstractproperty
-  def output_shapes(self):
-    """Returns the shape of each component of an element of this IndexedDataset.
-
-    Returns:
-      A nested structure of `tf.TensorShape` objects corresponding to each
-      component of an element of this IndexedDataset.
-    """
-    raise NotImplementedError("IndexedDataset.output_shapes")
-
   @abc.abstractmethod
   def _as_variant_tensor(self):
     """Creates a `tf.variant` `tf.Tensor` representing this IndexedDataset.
@@ -161,16 +125,8 @@ class IdentityIndexedDataset(IndexedDataset):
     self._size = ops.convert_to_tensor(size, dtype=dtypes.uint64, name="size")
 
   @property
-  def output_types(self):
-    return dtypes.uint64
-
-  @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.uint64, [])
 
   def _as_variant_tensor(self):
     return ged_ops.experimental_identity_indexed_dataset(self._size)
diff --git a/tensorflow/python/data/experimental/ops/interleave_ops.py b/tensorflow/python/data/experimental/ops/interleave_ops.py
index 8b0fdfce11..5a719f8ed8 100644
--- a/tensorflow/python/data/experimental/ops/interleave_ops.py
+++ b/tensorflow/python/data/experimental/ops/interleave_ops.py
@@ -21,6 +21,7 @@ from tensorflow.python.data.experimental.ops import random_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -101,6 +102,18 @@ class _DirectedInterleaveDataset(dataset_ops.Dataset):
           data_input.output_classes != data_inputs[0].output_classes):
         raise TypeError("All datasets must have the same type and class.")
 
+    output_shapes = self._data_inputs[0].output_shapes
+    for data_input in self._data_inputs[1:]:
+      output_shapes = nest.pack_sequence_as(output_shapes, [
+          ts1.most_specific_compatible_shape(ts2) for (ts1, ts2) in zip(
+              nest.flatten(output_shapes),
+              nest.flatten(data_input.output_shapes))
+      ])
+
+    self._structure = structure.convert_legacy_structure(
+        data_inputs[0].output_types, output_shapes,
+        data_inputs[0].output_classes)
+
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
     return (
@@ -115,22 +128,8 @@ class _DirectedInterleaveDataset(dataset_ops.Dataset):
     return [self._selector_input] + self._data_inputs
 
   @property
-  def output_classes(self):
-    return self._data_inputs[0].output_classes
-
-  @property
-  def output_shapes(self):
-    ret = self._data_inputs[0].output_shapes
-    for data_input in self._data_inputs[1:]:
-      ret = nest.pack_sequence_as(ret, [
-          ts1.most_specific_compatible_shape(ts2) for (ts1, ts2) in zip(
-              nest.flatten(ret), nest.flatten(data_input.output_shapes))
-      ])
-    return ret
-
-  @property
-  def output_types(self):
-    return self._data_inputs[0].output_types
+  def _element_structure(self):
+    return self._structure
 
 
 @tf_export("data.experimental.sample_from_datasets", v1=[])
diff --git a/tensorflow/python/data/experimental/ops/matching_files.py b/tensorflow/python/data/experimental/ops/matching_files.py
index 8398f86e31..63b99cb1e4 100644
--- a/tensorflow/python/data/experimental/ops/matching_files.py
+++ b/tensorflow/python/data/experimental/ops/matching_files.py
@@ -19,9 +19,9 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_experimental_dataset_ops as ged_ops
 
 
@@ -37,15 +37,5 @@ class MatchingFilesDataset(dataset_ops.DatasetSource):
     return ged_ops.experimental_matching_files_dataset(self._patterns)
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.string
-
-
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
diff --git a/tensorflow/python/data/experimental/ops/parsing_ops.py b/tensorflow/python/data/experimental/ops/parsing_ops.py
index 44cb0b8a2c..deb20d6188 100644
--- a/tensorflow/python/data/experimental/ops/parsing_ops.py
+++ b/tensorflow/python/data/experimental/ops/parsing_ops.py
@@ -18,7 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -33,8 +33,8 @@ class _ParseExampleDataset(dataset_ops.UnaryDataset):
   def __init__(self, input_dataset, features, num_parallel_calls):
     super(_ParseExampleDataset, self).__init__(input_dataset)
     self._input_dataset = input_dataset
-    if not all(types == dtypes.string
-               for types in nest.flatten(input_dataset.output_types)):
+    if not input_dataset._element_structure.is_compatible_with(  # pylint: disable=protected-access
+        structure.TensorStructure(dtypes.string, [None])):
       raise TypeError("Input dataset should be a dataset of vectors of strings")
     self._num_parallel_calls = num_parallel_calls
     # pylint: disable=protected-access
@@ -67,17 +67,19 @@ class _ParseExampleDataset(dataset_ops.UnaryDataset):
         for _ in range(len(sparse_keys))
     ]
 
-    self._output_shapes = dict(
+    output_shapes = dict(
         zip(self._dense_keys + self._sparse_keys,
             dense_output_shapes + sparse_output_shapes))
-    self._output_types = dict(
+    output_types = dict(
         zip(self._dense_keys + self._sparse_keys,
             self._dense_types + self._sparse_types))
-    self._output_classes = dict(
+    output_classes = dict(
         zip(self._dense_keys + self._sparse_keys,
             [ops.Tensor for _ in range(len(self._dense_defaults))] +
             [sparse_tensor.SparseTensor for _ in range(len(self._sparse_keys))
             ]))
+    self._structure = structure.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
 
   def _as_variant_tensor(self):
     return gen_experimental_dataset_ops.experimental_parse_example_dataset(
@@ -91,16 +93,8 @@ class _ParseExampleDataset(dataset_ops.UnaryDataset):
         **dataset_ops.flat_structure(self))
 
   @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
-
-  @property
-  def output_classes(self):
-    return self._output_classes
+  def _element_structure(self):
+    return self._structure
 
 
 # TODO(b/111553342): add arguments names and example names as well.
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 0894dd5f70..50d9803c34 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -261,16 +261,8 @@ class _MapOnGpuDataset(dataset_ops.UnaryDataset):
         **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return self._map_func.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._map_func.output_shapes
-
-  @property
-  def output_types(self):
-    return self._map_func.output_types
+  def _element_structure(self):
+    return self._map_func.output_structure
 
   def _transformation_name(self):
     return "map_on_gpu()"
diff --git a/tensorflow/python/data/experimental/ops/random_ops.py b/tensorflow/python/data/experimental/ops/random_ops.py
index a4359f356b..cbdf367db6 100644
--- a/tensorflow/python/data/experimental/ops/random_ops.py
+++ b/tensorflow/python/data/experimental/ops/random_ops.py
@@ -21,9 +21,8 @@ import functools
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import random_seed
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
@@ -44,16 +43,8 @@ class RandomDatasetV2(dataset_ops.DatasetSource):
         **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.int64
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.int64, [])
 
 
 @tf_export(v1=["data.experimental.RandomDataset"])
diff --git a/tensorflow/python/data/experimental/ops/readers.py b/tensorflow/python/data/experimental/ops/readers.py
index a4c260dde7..c2d82aeb59 100644
--- a/tensorflow/python/data/experimental/ops/readers.py
+++ b/tensorflow/python/data/experimental/ops/readers.py
@@ -32,10 +32,10 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers as core_readers
 from tensorflow.python.data.util import convert
 from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.ops import io_ops
@@ -652,11 +652,9 @@ class CsvDatasetV2(dataset_ops.DatasetSource):
         argument_default=[],
         argument_dtype=dtypes.int64,
     )
-    self._output_shapes = tuple(
-        tensor_shape.scalar() for _ in range(len(record_defaults)))
-    self._output_types = tuple(d.dtype for d in self._record_defaults)
-    self._output_classes = tuple(
-        ops.Tensor for _ in range(len(record_defaults)))
+    self._structure = structure.NestedStructure(
+        tuple(structure.TensorStructure(d.dtype, [])
+              for d in self._record_defaults))
 
   def _as_variant_tensor(self):
     # Constructs graph node for the dataset op.
@@ -665,7 +663,7 @@ class CsvDatasetV2(dataset_ops.DatasetSource):
         record_defaults=self._record_defaults,
         buffer_size=self._buffer_size,
         header=self._header,
-        output_shapes=self._output_shapes,
+        output_shapes=self._structure._flat_shapes,  # pylint: disable=protected-access
         field_delim=self._field_delim,
         use_quote_delim=self._use_quote_delim,
         na_value=self._na_value,
@@ -674,16 +672,8 @@ class CsvDatasetV2(dataset_ops.DatasetSource):
     )
 
   @property
-  def output_types(self):
-    return self._output_types
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_classes(self):
-    return self._output_classes
+  def _element_structure(self):
+    return self._structure
 
 
 @tf_export(v1=["data.experimental.CsvDataset"])
@@ -961,7 +951,9 @@ class SqlDatasetV2(dataset_ops.DatasetSource):
         data_source_name, dtype=dtypes.string, name="data_source_name")
     self._query = ops.convert_to_tensor(
         query, dtype=dtypes.string, name="query")
-    self._output_types = output_types
+    self._structure = structure.NestedStructure(
+        nest.map_structure(
+            lambda dtype: structure.TensorStructure(dtype, []), output_types))
 
   def _as_variant_tensor(self):
     return gen_experimental_dataset_ops.experimental_sql_dataset(
@@ -969,17 +961,8 @@ class SqlDatasetV2(dataset_ops.DatasetSource):
         nest.flatten(self.output_types), nest.flatten(self.output_shapes))
 
   @property
-  def output_classes(self):
-    return nest.map_structure(lambda _: ops.Tensor, self._output_types)
-
-  @property
-  def output_shapes(self):
-    return nest.map_structure(lambda _: tensor_shape.TensorShape([]),
-                              self._output_types)
-
-  @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
 
 @tf_export(v1=["data.experimental.SqlDataset"])
diff --git a/tensorflow/python/data/experimental/ops/scan_ops.py b/tensorflow/python/data/experimental/ops/scan_ops.py
index 68ff1d5d7f..661f2c5928 100644
--- a/tensorflow/python/data/experimental/ops/scan_ops.py
+++ b/tensorflow/python/data/experimental/ops/scan_ops.py
@@ -22,6 +22,7 @@ import collections
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import gen_experimental_dataset_ops
@@ -57,11 +58,6 @@ class _ScanDataset(dataset_ops.UnaryDataset):
         self._initial_state,
         [t.dtype for t in nest.flatten(self._initial_state)])
 
-    # Will be populated by calling `tf_scan_func`.
-    self._output_classes = None
-    self._output_shapes = None
-    self._output_types = None
-
     # Iteratively rerun the scan function until reaching a fixed point on
     # `self._state_shapes`.
     need_to_rerun = True
@@ -80,7 +76,7 @@ class _ScanDataset(dataset_ops.UnaryDataset):
         raise TypeError("The scan function must return a pair comprising the "
                         "new state and the output value.")
 
-      new_state_classes, self._output_classes = wrapped_func.output_classes
+      new_state_classes, output_classes = wrapped_func.output_classes
 
       # Extract and validate class information from the returned values.
       for new_state_class, state_class in zip(
@@ -93,7 +89,7 @@ class _ScanDataset(dataset_ops.UnaryDataset):
               (self._state_classes, new_state_classes))
 
       # Extract and validate type information from the returned values.
-      new_state_types, self._output_types = wrapped_func.output_types
+      new_state_types, output_types = wrapped_func.output_types
       for new_state_type, state_type in zip(
           nest.flatten(new_state_types), nest.flatten(self._state_types)):
         if new_state_type != state_type:
@@ -103,7 +99,10 @@ class _ScanDataset(dataset_ops.UnaryDataset):
               (self._state_types, new_state_types))
 
       # Extract shape information from the returned values.
-      new_state_shapes, self._output_shapes = wrapped_func.output_shapes
+      new_state_shapes, output_shapes = wrapped_func.output_shapes
+
+      self._structure = structure.convert_legacy_structure(
+          output_types, output_shapes, output_classes)
 
       flat_state_shapes = nest.flatten(self._state_shapes)
       flat_new_state_shapes = nest.flatten(new_state_shapes)
@@ -142,16 +141,8 @@ class _ScanDataset(dataset_ops.UnaryDataset):
         **dataset_ops.flat_structure(self))
 
   @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
   def _transformation_name(self):
     return "tf.data.experimental.scan()"
diff --git a/tensorflow/python/data/experimental/ops/sleep.py b/tensorflow/python/data/experimental/ops/sleep.py
index 5e9d021ada..2da832395b 100644
--- a/tensorflow/python/data/experimental/ops/sleep.py
+++ b/tensorflow/python/data/experimental/ops/sleep.py
@@ -35,18 +35,6 @@ class _SleepDataset(dataset_ops.UnaryUnchangedStructureDataset):
         self._sleep_microseconds,
         **dataset_ops.flat_structure(self))
 
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
 
 def sleep(sleep_microseconds):
   """Sleeps for `sleep_microseconds` before producing each input element.
diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD
index 0c5acda180..45d65956f3 100644
--- a/tensorflow/python/data/ops/BUILD
+++ b/tensorflow/python/data/ops/BUILD
@@ -52,6 +52,7 @@ py_library(
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python/compat",
         "//tensorflow/python/data/util:convert",
+        "//tensorflow/python/data/util:structure",
     ],
 )
 
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 4270f09d38..d7a2547fc8 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -194,6 +194,16 @@ class DatasetV2(object):
                          "execution is enabled.")
 
   @abc.abstractproperty
+  def _element_structure(self):
+    """The structure of an element of this dataset.
+
+    Returns:
+      A `Structure` object representing the structure of an element of this
+      dataset.
+    """
+    raise NotImplementedError("Dataset._element_structure")
+
+  @property
   def output_classes(self):
     """Returns the class of each component of an element of this dataset.
 
@@ -203,9 +213,9 @@ class DatasetV2(object):
       A nested structure of Python `type` objects corresponding to each
       component of an element of this dataset.
     """
-    raise NotImplementedError("Dataset.output_classes")
+    return self._element_structure._to_legacy_output_classes()  # pylint: disable=protected-access
 
-  @abc.abstractproperty
+  @property
   def output_shapes(self):
     """Returns the shape of each component of an element of this dataset.
 
@@ -213,9 +223,9 @@ class DatasetV2(object):
       A nested structure of `tf.TensorShape` objects corresponding to each
       component of an element of this dataset.
     """
-    raise NotImplementedError("Dataset.output_shapes")
+    return self._element_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
 
-  @abc.abstractproperty
+  @property
   def output_types(self):
     """Returns the type of each component of an element of this dataset.
 
@@ -223,7 +233,7 @@ class DatasetV2(object):
       A nested structure of `tf.DType` objects corresponding to each component
       of an element of this dataset.
     """
-    raise NotImplementedError("Dataset.output_types")
+    return self._element_structure._to_legacy_output_types()  # pylint: disable=protected-access
 
   def __repr__(self):
     output_shapes = nest.map_structure(str, self.output_shapes)
@@ -1397,6 +1407,13 @@ class DatasetV1(DatasetV2):
                                  dataset.output_types, dataset.output_shapes,
                                  dataset.output_classes)
 
+  @property
+  def _element_structure(self):
+    # TODO(b/110122868): Remove this override once all `Dataset` instances
+    # implement `element_structure`.
+    return structure_lib.convert_legacy_structure(
+        self.output_types, self.output_shapes, self.output_classes)
+
   @staticmethod
   @functools.wraps(DatasetV2.from_tensors)
   def from_tensors(tensors):
@@ -1605,16 +1622,8 @@ class DatasetV1Adapter(DatasetV1):
     return self._dataset.options()
 
   @property
-  def output_classes(self):
-    return self._dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._dataset.output_shapes
-
-  @property
-  def output_types(self):
-    return self._dataset.output_types
+  def _element_structure(self):
+    return self._dataset._element_structure  # pylint: disable=protected-access
 
   def __iter__(self):
     return iter(self._dataset)
@@ -1786,16 +1795,8 @@ class UnaryUnchangedStructureDataset(UnaryDataset):
   """Represents a unary dataset with the same input and output structure."""
 
   @property
-  def output_classes(self):
-    return self._input_dataset.output_classes  # pylint: disable=protected-access
-
-  @property
-  def output_shapes(self):
-    return self._input_dataset.output_shapes  # pylint: disable=protected-access
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types  # pylint: disable=protected-access
+  def _element_structure(self):
+    return self._input_dataset._element_structure  # pylint: disable=protected-access
 
 
 class TensorDataset(DatasetSource):
@@ -1813,29 +1814,23 @@ class TensorDataset(DatasetSource):
       ])
 
     self._tensors = sparse.serialize_sparse_tensors(tensors)
-    self._output_classes = sparse.get_classes(tensors)
-    self._output_shapes = nest.pack_sequence_as(
+    output_classes = sparse.get_classes(tensors)
+    output_shapes = nest.pack_sequence_as(
         tensors, [t.get_shape() for t in nest.flatten(tensors)])
-    self._output_types = nest.pack_sequence_as(
+    output_types = nest.pack_sequence_as(
         tensors, [t.dtype for t in nest.flatten(tensors)])
+    self._structure = structure_lib.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
 
   def _as_variant_tensor(self):
+    # pylint: disable=protected-access
     return gen_dataset_ops.tensor_dataset(
         nest.flatten(self._tensors),
-        output_shapes=nest.flatten(
-            sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
+        output_shapes=self._structure._flat_shapes)
 
   @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
 
 class TensorSliceDataset(DatasetSource):
@@ -1859,29 +1854,23 @@ class TensorSliceDataset(DatasetSource):
       batch_dim.assert_is_compatible_with(tensor_shape.Dimension(
           tensor_shape.dimension_value(t.get_shape()[0])))
     self._tensors = sparse.serialize_many_sparse_tensors(tensors)
-    self._output_classes = sparse.get_classes(tensors)
-    self._output_shapes = nest.pack_sequence_as(
+    output_classes = sparse.get_classes(tensors)
+    output_shapes = nest.pack_sequence_as(
         tensors, [t.get_shape()[1:] for t in nest.flatten(tensors)])
-    self._output_types = nest.pack_sequence_as(
+    output_types = nest.pack_sequence_as(
         tensors, [t.dtype for t in nest.flatten(tensors)])
+    self._structure = structure_lib.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.tensor_slice_dataset(
         nest.flatten(self._tensors),
         output_shapes=nest.flatten(
-            sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
-
-  @property
-  def output_classes(self):
-    return self._output_classes
+            self._structure._to_legacy_output_shapes()))  # pylint: disable=protected-access
 
   @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
 
 class SparseTensorSliceDataset(DatasetSource):
@@ -1894,28 +1883,26 @@ class SparseTensorSliceDataset(DatasetSource):
       raise TypeError("`sparse_tensor` must be a `tf.SparseTensor` object.")
     self._sparse_tensor = sparse_tensor
 
-  def _as_variant_tensor(self):
-    return gen_dataset_ops.sparse_tensor_slice_dataset(
-        self._sparse_tensor.indices, self._sparse_tensor.values,
-        self._sparse_tensor.dense_shape)
-
-  @property
-  def output_classes(self):
-    return (ops.Tensor, ops.Tensor, ops.Tensor)
-
-  @property
-  def output_shapes(self):
+    output_classes = (ops.Tensor, ops.Tensor, ops.Tensor)
     indices_shape = self._sparse_tensor.indices.get_shape()
     shape_shape = self._sparse_tensor.dense_shape.get_shape()
     rank = (indices_shape.dims[1] - 1).merge_with(shape_shape.dims[0] - 1)
     num_values = tensor_shape.Dimension(None)
-    return (tensor_shape.TensorShape([num_values, rank]),
-            tensor_shape.TensorShape([num_values]),
-            tensor_shape.TensorShape([rank]))
+    output_shapes = (tensor_shape.TensorShape([num_values, rank]),
+                     tensor_shape.TensorShape([num_values]),
+                     tensor_shape.TensorShape([rank]))
+    output_types = (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64)
+    self._structure = structure_lib.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
+
+  def _as_variant_tensor(self):
+    return gen_dataset_ops.sparse_tensor_slice_dataset(
+        self._sparse_tensor.indices, self._sparse_tensor.values,
+        self._sparse_tensor.dense_shape)
 
   @property
-  def output_types(self):
-    return (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64)
+  def _element_structure(self):
+    return self._structure
 
 
 class _VariantDataset(DatasetV2):
@@ -1933,16 +1920,8 @@ class _VariantDataset(DatasetV2):
     return []
 
   @property
-  def output_classes(self):
-    return self._structure._to_legacy_output_classes()  # pylint: disable=protected-access
-
-  @property
-  def output_shapes(self):
-    return self._structure._to_legacy_output_shapes()  # pylint: disable=protected-access
-
-  @property
-  def output_types(self):
-    return self._structure._to_legacy_output_types()  # pylint: disable=protected-access
+  def _element_structure(self):
+    return self._structure
 
 
 class DatasetStructure(structure_lib.Structure):
@@ -1981,11 +1960,7 @@ class DatasetStructure(structure_lib.Structure):
 
   @staticmethod
   def from_value(value):
-    # TODO(b/110122868): We can simplify this when a `Dataset` object has a
-    # `Structure`-valued property.
-    element_structure = structure_lib.Structure._from_legacy_structure(
-        value.output_types, value.output_shapes, value.output_classes)
-    return DatasetStructure(element_structure)
+    return DatasetStructure(value._element_structure)  # pylint: disable=protected-access
 
   def _to_legacy_output_types(self):
     return self
@@ -2017,6 +1992,7 @@ class StructuredFunctionWrapper(object):
                input_classes=None,
                input_shapes=None,
                input_types=None,
+               input_structure=None,
                add_to_graph=True,
                defun_kwargs=None):
     """Creates a new `StructuredFunctionWrapper` for the given function.
@@ -2035,6 +2011,8 @@ class StructuredFunctionWrapper(object):
         arguments.
       input_types: (Optional.) A nested structure of `tf.DType`. If given, this
         argument defines the element types and structure for `func` arguments.
+      input_structure: (Optional.) A `Structure` object. If given, this argument
+        defines the element types and structure for `func` arguments.
       add_to_graph: (Optional.) If `True`, the function will be added to the
         default graph.
       defun_kwargs: (Optional.) A dictionary mapping string argument names to
@@ -2045,24 +2023,28 @@ class StructuredFunctionWrapper(object):
       ValueError: If an invalid combination of `dataset`, `input_classes`,
         `input_shapes`, and `input_types` is passed.
     """
-    if dataset is None:
-      if input_classes is None or input_shapes is None or input_types is None:
-        raise ValueError("Either `dataset`, or all of `input_classes`, "
-                         "`input_shapes`, and `input_types` must be specified.")
-      self._input_shapes = input_shapes
-      self._input_types = input_types
-      self._input_classes = input_classes
+    if input_structure is None:
+      if dataset is None:
+        if input_classes is None or input_shapes is None or input_types is None:
+          raise ValueError("Either `dataset`, `input_structure` or all of "
+                           "`input_classes`, `input_shapes`, and `input_types` "
+                           "must be specified.")
+        self._input_structure = structure_lib.convert_legacy_structure(
+            input_types, input_shapes, input_classes)
+      else:
+        if not (input_classes is None and input_shapes is None and
+                input_types is None):
+          raise ValueError("Either `dataset`, `input_structure` or all of "
+                           "`input_classes`, `input_shapes`, and `input_types` "
+                           "must be specified.")
+        self._input_structure = dataset._element_structure  # pylint: disable=protected-access
     else:
-      if not (input_classes is None and input_shapes is None and
-              input_types is None):
-        raise ValueError("Either `dataset`, or all of `input_classes`, "
-                         "`input_shapes`, and `input_types` must be specified.")
-      self._input_shapes = dataset.output_shapes
-      self._input_types = dataset.output_types
-      self._input_classes = dataset.output_classes
-
-    self._input_structure = structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
-        self._input_types, self._input_shapes, self._input_classes)
+      if not (dataset is None and input_classes is None and input_shapes is None
+              and input_types is None):
+        raise ValueError("Either `dataset`, `input_structure`, or all of "
+                         "`input_classes`, `input_shapes`, and `input_types` "
+                         "must be specified.")
+      self._input_structure = input_structure
 
     self._transformation_name = transformation_name
     readable_transformation_name = transformation_name.replace(
@@ -2139,11 +2121,9 @@ class StructuredFunctionWrapper(object):
     return self._function
 
 
-def flat_structure(dataset=None, structure=None):
+def flat_structure(dataset):
   """Helper for setting `output_shapes` and `output_types` attrs of Dataset ops.
 
-  Either `dataset` or `structure` must be passed to this function.
-
   Most Dataset op constructors expect `output_shapes` and `output_types`
   arguments that represent the flattened structure of an element. This helper
   function generates these attrs as a keyword argument dictionary, allowing
@@ -2151,17 +2131,14 @@ def flat_structure(dataset=None, structure=None):
   `**flat_structure(self)` to the op constructor.
 
   Args:
-    dataset: (Optional.) A `tf.data.Dataset`.
-    structure: (Optional.) A `Structure`.
+    dataset: A `tf.data.Dataset`.
 
   Returns:
     A dictionary of keyword arguments that can be passed to many Dataset op
     constructors.
   """
   # pylint: disable=protected-access
-  if structure is None:
-    structure = structure_lib.Structure._from_legacy_structure(
-        dataset.output_types, dataset.output_shapes, dataset.output_classes)
+  structure = dataset._element_structure
   return {
       "output_shapes": structure._flat_shapes,
       "output_types": structure._flat_types,
@@ -2200,57 +2177,37 @@ class _GeneratorDataset(DatasetSource):
     init_args_types = nest.pack_sequence_as(
         init_args, [t.dtype for t in nest.flatten(init_args)])
 
-    wrapped_init_func = StructuredFunctionWrapper(
+    self._init_func = StructuredFunctionWrapper(
         init_func,
         self._transformation_name(),
         input_classes=init_args_classes,
         input_shapes=init_args_shapes,
         input_types=init_args_types)
-    self._state_classes = wrapped_init_func.output_classes
-    self._state_shapes = wrapped_init_func.output_shapes
-    self._state_types = wrapped_init_func.output_types
-    self._init_func = wrapped_init_func.function
 
-    wrapped_next_func = StructuredFunctionWrapper(
+    self._next_func = StructuredFunctionWrapper(
         next_func,
         self._transformation_name(),
-        input_classes=self._state_classes,
-        input_shapes=self._state_shapes,
-        input_types=self._state_types)
-    self._output_classes = wrapped_next_func.output_classes
-    self._output_shapes = wrapped_next_func.output_shapes
-    self._output_types = wrapped_next_func.output_types
-    self._next_func = wrapped_next_func.function
-
-    wrapped_finalize_func = StructuredFunctionWrapper(
+        input_structure=self._init_func.output_structure)
+
+    self._finalize_func = StructuredFunctionWrapper(
         finalize_func,
         self._transformation_name(),
-        input_classes=self._state_classes,
-        input_shapes=self._state_shapes,
-        input_types=self._state_types)
-    self._finalize_func = wrapped_finalize_func.function
+        input_structure=self._init_func.output_structure)
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.generator_dataset(
-        nest.flatten(self._init_args) + self._init_func.captured_inputs,
-        self._next_func.captured_inputs,
-        self._finalize_func.captured_inputs,
-        init_func=self._init_func,
-        next_func=self._next_func,
-        finalize_func=self._finalize_func,
+        nest.flatten(self._init_args)
+        + self._init_func.function.captured_inputs,
+        self._next_func.function.captured_inputs,
+        self._finalize_func.function.captured_inputs,
+        init_func=self._init_func.function,
+        next_func=self._next_func.function,
+        finalize_func=self._finalize_func.function,
         **flat_structure(self))
 
   @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._next_func.output_structure
 
   def _transformation_name(self):
     return "Dataset.from_generator()"
@@ -2273,6 +2230,10 @@ class ZipDataset(DatasetV2):
                      "structure of `Dataset` objects.")
         raise TypeError(message)
     self._datasets = datasets
+    self._structure = structure_lib.NestedStructure(
+        nest.pack_sequence_as(
+            self._datasets,
+            [ds._element_structure for ds in nest.flatten(self._datasets)]))  # pylint: disable=protected-access
 
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
@@ -2285,22 +2246,8 @@ class ZipDataset(DatasetV2):
     return nest.flatten(self._datasets)
 
   @property
-  def output_classes(self):
-    return nest.pack_sequence_as(
-        self._datasets,
-        [ds.output_classes for ds in nest.flatten(self._datasets)])
-
-  @property
-  def output_shapes(self):
-    return nest.pack_sequence_as(
-        self._datasets,
-        [ds.output_shapes for ds in nest.flatten(self._datasets)])
-
-  @property
-  def output_types(self):
-    return nest.pack_sequence_as(
-        self._datasets,
-        [ds.output_types for ds in nest.flatten(self._datasets)])
+  def _element_structure(self):
+    return self._structure
 
 
 class ConcatenateDataset(DatasetV2):
@@ -2312,26 +2259,29 @@ class ConcatenateDataset(DatasetV2):
     self._input_dataset = input_dataset
     self._dataset_to_concatenate = dataset_to_concatenate
 
-    self._output_types = input_dataset.output_types
-    if self._output_types != dataset_to_concatenate.output_types:
+    output_types = input_dataset.output_types
+    if output_types != dataset_to_concatenate.output_types:
       raise TypeError(
           "Two datasets to concatenate have different types %s and %s" %
-          (self._output_types, dataset_to_concatenate.output_types))
+          (output_types, dataset_to_concatenate.output_types))
 
-    self._output_classes = input_dataset.output_classes
-    if self._output_classes != dataset_to_concatenate.output_classes:
+    output_classes = input_dataset.output_classes
+    if output_classes != dataset_to_concatenate.output_classes:
       raise TypeError(
           "Two datasets to concatenate have different classes %s and %s" %
-          (self._output_classes, dataset_to_concatenate.output_classes))
+          (output_classes, dataset_to_concatenate.output_classes))
 
     input_shapes = self._input_dataset.output_shapes
-    self._output_shapes = nest.pack_sequence_as(input_shapes, [
+    output_shapes = nest.pack_sequence_as(input_shapes, [
         ts1.most_specific_compatible_shape(ts2)
         for (ts1, ts2) in zip(
             nest.flatten(input_shapes),
             nest.flatten(self._dataset_to_concatenate.output_shapes))
     ])
 
+    self._structure = structure_lib.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
+
     self._input_datasets = [input_dataset, dataset_to_concatenate]
 
   def _as_variant_tensor(self):
@@ -2346,16 +2296,8 @@ class ConcatenateDataset(DatasetV2):
     return [self._input_dataset, self._dataset_to_concatenate]
 
   @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
 
 class RepeatDataset(UnaryUnchangedStructureDataset):
@@ -2414,16 +2356,8 @@ class RangeDataset(DatasetSource):
         **flat_structure(self))
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.int64
+  def _element_structure(self):
+    return structure_lib.TensorStructure(dtypes.int64, [])
 
 
 class CacheDataset(UnaryUnchangedStructureDataset):
@@ -2536,37 +2470,26 @@ class BatchDataset(UnaryDataset):
     self._drop_remainder = ops.convert_to_tensor(
         drop_remainder, dtype=dtypes.bool, name="drop_remainder")
 
-    # pylint: disable=protected-access
-    input_structure = structure_lib.Structure._from_legacy_structure(
-        input_dataset.output_types, input_dataset.output_shapes,
-        input_dataset.output_classes)
     constant_drop_remainder = tensor_util.constant_value(self._drop_remainder)
+    # pylint: disable=protected-access
     if constant_drop_remainder:
       # NOTE(mrry): `constant_drop_remainder` may be `None` (unknown statically)
       # or `False` (explicitly retaining the remainder).
-      self._output_structure = input_structure._batch(
+      self._structure = input_dataset._element_structure._batch(
           tensor_util.constant_value(self._batch_size))
     else:
-      self._output_structure = input_structure._batch(None)
+      self._structure = input_dataset._element_structure._batch(None)
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.batch_dataset_v2(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         batch_size=self._batch_size,
         drop_remainder=self._drop_remainder,
-        **flat_structure(structure=self._output_structure))
-
-  @property
-  def output_classes(self):
-    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
-
-  @property
-  def output_shapes(self):
-    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
+        **flat_structure(self))
 
   @property
-  def output_types(self):
-    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
+  def _element_structure(self):
+    return self._structure
 
 
 def _is_padded_shape_compatible_with(padded_shape, input_component_shape):
@@ -2715,22 +2638,34 @@ class PaddedBatchDataset(UnaryDataset):
     self._drop_remainder = ops.convert_to_tensor(
         drop_remainder, dtype=dtypes.bool, name="drop_remainder")
 
+    def _padded_shape_to_batch_shape(s):
+      return tensor_shape.vector(
+          tensor_util.constant_value(self._batch_size) if smart_cond.
+          smart_constant_value(self._drop_remainder) else None).concatenate(
+              tensor_util.constant_value_as_shape(s))
+
+    output_shapes = nest.map_structure(
+        _padded_shape_to_batch_shape, self._padded_shapes)
+    self._structure = structure_lib.convert_legacy_structure(
+        self._input_dataset.output_types, output_shapes,
+        self._input_dataset.output_classes)
+
   def _as_variant_tensor(self):
+    # pylint: disable=protected-access
     # TODO(jsimsa): Switch to using v2 only any time after 6/30/2018.
     if smart_cond.smart_constant_value(self._drop_remainder) is False:
       return gen_dataset_ops.padded_batch_dataset(
-          self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+          self._input_dataset._as_variant_tensor(),
           batch_size=self._batch_size,
           padded_shapes=[
               ops.convert_to_tensor(s, dtype=dtypes.int64)
               for s in nest.flatten(self._padded_shapes)
           ],
           padding_values=nest.flatten(self._padding_values),
-          output_shapes=nest.flatten(
-              sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
+          output_shapes=self._structure._flat_shapes)
     else:
       return gen_dataset_ops.padded_batch_dataset_v2(
-          self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+          self._input_dataset._as_variant_tensor(),
           batch_size=self._batch_size,
           padded_shapes=[
               ops.convert_to_tensor(s, dtype=dtypes.int64)
@@ -2738,27 +2673,11 @@ class PaddedBatchDataset(UnaryDataset):
           ],
           padding_values=nest.flatten(self._padding_values),
           drop_remainder=self._drop_remainder,
-          output_shapes=nest.flatten(
-              sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
+          output_shapes=self._structure._flat_shapes)
 
   @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-
-    def _padded_shape_to_batch_shape(s):
-      return tensor_shape.vector(
-          tensor_util.constant_value(self._batch_size) if smart_cond.
-          smart_constant_value(self._drop_remainder) else None).concatenate(
-              tensor_util.constant_value_as_shape(s))
-
-    return nest.map_structure(_padded_shape_to_batch_shape, self._padded_shapes)
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
+  def _element_structure(self):
+    return self._structure
 
 
 def _should_unpack_args(args):
@@ -2808,22 +2727,14 @@ class MapDataset(UnaryDataset):
         f=self._map_func.function,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
         preserve_cardinality=self._preserve_cardinality,
-        **flat_structure(structure=self._map_func.output_structure))
+        **flat_structure(self))
 
   def _functions(self):
     return [self._map_func]
 
   @property
-  def output_classes(self):
-    return self._map_func.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._map_func.output_shapes
-
-  @property
-  def output_types(self):
-    return self._map_func.output_types
+  def _element_structure(self):
+    return self._map_func.output_structure
 
   def _transformation_name(self):
     return "Dataset.map()"
@@ -2855,7 +2766,7 @@ class ParallelMapDataset(MapDataset):
         num_parallel_calls=self._num_parallel_calls,
         use_inter_op_parallelism=self._use_inter_op_parallelism,
         preserve_cardinality=self._preserve_cardinality,
-        **flat_structure(structure=self._map_func.output_structure))
+        **flat_structure(self))
 
 
 class FlatMapDataset(UnaryDataset):
@@ -2870,7 +2781,7 @@ class FlatMapDataset(UnaryDataset):
         map_func, self._transformation_name(), dataset=input_dataset)
     if not isinstance(self._map_func.output_structure, DatasetStructure):
       raise TypeError("`map_func` must return a `Dataset` object.")
-    self._output_structure = self._map_func.output_structure._element_structure  # pylint: disable=protected-access
+    self._structure = self._map_func.output_structure._element_structure  # pylint: disable=protected-access
 
   def _functions(self):
     return [self._map_func]
@@ -2880,19 +2791,11 @@ class FlatMapDataset(UnaryDataset):
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._map_func.function.captured_inputs,
         f=self._map_func.function,
-        **flat_structure(structure=self._output_structure))
-
-  @property
-  def output_classes(self):
-    return self._output_structure._to_legacy_output_classes()  # pylint: disable=protected-access
-
-  @property
-  def output_shapes(self):
-    return self._output_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
+        **flat_structure(self))
 
   @property
-  def output_types(self):
-    return self._output_structure._to_legacy_output_types()  # pylint: disable=protected-access
+  def _element_structure(self):
+    return self._structure
 
   def _transformation_name(self):
     return "Dataset.flat_map()"
@@ -2918,7 +2821,7 @@ class InterleaveDataset(FlatMapDataset):
         self._cycle_length,
         self._block_length,
         f=self._map_func.function,
-        **flat_structure(structure=self._output_structure))
+        **flat_structure(self))
 
   def _transformation_name(self):
     return "Dataset.interleave()"
@@ -2949,7 +2852,7 @@ class ParallelInterleaveDataset(FlatMapDataset):
         self._block_length,
         self._num_parallel_calls,
         f=self._map_func.function,
-        **flat_structure(structure=self._output_structure))
+        **flat_structure(self))
 
   def _transformation_name(self):
     return "Dataset.interleave()"
@@ -2964,9 +2867,8 @@ class FilterDataset(UnaryUnchangedStructureDataset):
     self._input_dataset = input_dataset
     wrapped_func = StructuredFunctionWrapper(
         predicate, self._transformation_name(), dataset=input_dataset)
-    if not (
-        wrapped_func.output_types == dtypes.bool and
-        wrapped_func.output_shapes.is_compatible_with(tensor_shape.scalar())):
+    if not wrapped_func.output_structure.is_compatible_with(
+        structure_lib.TensorStructure(dtypes.bool, [])):
       raise ValueError("`predicate` must return a scalar boolean tensor.")
     self._predicate = wrapped_func
 
@@ -3016,19 +2918,17 @@ class WindowDataset(UnaryDataset):
         stride, dtype=dtypes.int64, name="stride")
     self._drop_remainder = ops.convert_to_tensor(
         drop_remainder, dtype=dtypes.bool, name="drop_remainder")
-    self._output_classes = nest.pack_sequence_as(
+    nest_of_structures = nest.pack_sequence_as(
         input_dataset.output_classes,
         [
-            DatasetStructure(
-                structure_lib.Structure._from_legacy_structure(  # pylint: disable=protected-access
-                    output_type, output_shape, output_class))
+            DatasetStructure(structure_lib.convert_legacy_structure(
+                output_type, output_shape, output_class))
             for output_class, output_shape, output_type in zip(
                 nest.flatten(input_dataset.output_classes),
                 nest.flatten(input_dataset.output_shapes),
                 nest.flatten(input_dataset.output_types))
         ])
-    self._output_shapes = self._output_classes
-    self._output_types = self._output_classes
+    self._structure = structure_lib.NestedStructure(nest_of_structures)
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.window_dataset(
@@ -3040,16 +2940,8 @@ class WindowDataset(UnaryDataset):
         **flat_structure(self))
 
   @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
+  def _element_structure(self):
+    return self._structure
 
 
 class _OptionsDataset(UnaryUnchangedStructureDataset):
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index e2ca64c802..eb330d83ae 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -100,10 +100,8 @@ class Iterator(checkpointable.CheckpointableBase):
       raise ValueError("If `structure` is not specified, all of "
                        "`output_types`, `output_shapes`, and `output_classes`"
                        " must be specified.")
-    # pylint: disable=protected-access
-    self._structure = structure_lib.Structure._from_legacy_structure(
+    self._structure = structure_lib.convert_legacy_structure(
         output_types, output_shapes, output_classes)
-    # pylint: enable=protected-access
 
     self._string_handle = gen_dataset_ops.iterator_to_string_handle(
         self._iterator_resource)
@@ -531,7 +529,7 @@ class EagerIterator(checkpointable.CheckpointableBase):
     with ops.device("/cpu:0"):
       # pylint: disable=protected-access
       ds_variant = dataset._as_variant_tensor()
-      self._structure = structure_lib.Structure._from_legacy_structure(
+      self._structure = structure_lib.convert_legacy_structure(
           dataset.output_types, dataset.output_shapes, dataset.output_classes)
       self._flat_output_types = self._structure._flat_types
       self._flat_output_shapes = self._structure._flat_shapes
diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py
index 0f9add6461..4638cee02a 100644
--- a/tensorflow/python/data/ops/multi_device_iterator_ops.py
+++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py
@@ -19,8 +19,6 @@ from __future__ import print_function
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
@@ -36,16 +34,9 @@ class _PerDeviceGenerator(dataset_ops.Dataset):
   """A `dummy` generator dataset."""
 
   def __init__(self, shard_num, multi_device_iterator_resource, incarnation_id,
-               source_device, target_device, output_shapes, output_types,
-               output_classes):
+               source_device, target_device, element_structure):
     self._target_device = target_device
-    self._output_types = output_types
-    self._output_shapes = output_shapes
-    self._output_classes = output_classes
-    self._flat_output_shapes = nest.flatten(
-        sparse.as_dense_shapes(self._output_shapes, self._output_classes))
-    self._flat_output_types = nest.flatten(
-        sparse.as_dense_types(self._output_types, self._output_classes))
+    self._structure = element_structure
 
     multi_device_iterator_string_handle = (
         gen_dataset_ops.multi_device_iterator_to_string_handle(
@@ -70,17 +61,18 @@ class _PerDeviceGenerator(dataset_ops.Dataset):
 
     @function.defun(input_signature=[tensor_spec.TensorSpec([], dtypes.string)])
     def _next_func(string_handle):
+      # pylint: disable=protected-access
       multi_device_iterator = (
           gen_dataset_ops.multi_device_iterator_from_string_handle(
               string_handle=string_handle,
-              output_types=self._flat_output_types,
-              output_shapes=self._flat_output_shapes))
+              output_types=self._structure._flat_types,
+              output_shapes=self._structure._flat_shapes))
       return gen_dataset_ops.multi_device_iterator_get_next_from_shard(
           multi_device_iterator=multi_device_iterator,
           shard_num=shard_num,
           incarnation_id=incarnation_id,
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
+          output_types=self._structure._flat_types,
+          output_shapes=self._structure._flat_shapes)
 
     next_func_concrete = _next_func._get_concrete_function_internal()  # pylint: disable=protected-access
 
@@ -90,9 +82,8 @@ class _PerDeviceGenerator(dataset_ops.Dataset):
     def _remote_next_func(string_handle):
       return functional_ops.remote_call(
           target=source_device,
-          args=[string_handle] +
-          next_func_concrete.captured_inputs,
-          Tout=self._flat_output_types,
+          args=[string_handle] + next_func_concrete.captured_inputs,
+          Tout=self._structure._flat_types,  # pylint: disable=protected-access
           f=next_func_concrete)
 
     self._next_func = _remote_next_func._get_concrete_function_internal()  # pylint: disable=protected-access
@@ -108,8 +99,7 @@ class _PerDeviceGenerator(dataset_ops.Dataset):
     def _remote_finalize_func(string_handle):
       return functional_ops.remote_call(
           target=source_device,
-          args=[string_handle] +
-          finalize_func_concrete.captured_inputs,
+          args=[string_handle] + finalize_func_concrete.captured_inputs,
           Tout=[dtypes.int64],
           f=finalize_func_concrete)
 
@@ -126,24 +116,15 @@ class _PerDeviceGenerator(dataset_ops.Dataset):
           init_func=self._init_func,
           next_func=self._next_func,
           finalize_func=self._finalize_func,
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
+          **dataset_ops.flat_structure(self))
 
   def _inputs(self):
     # TODO(b/116506223): Determine which datasets should be used as inputs here.
     return []
 
   @property
-  def output_types(self):
-    return self._output_types
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_classes(self):
-    return self._output_classes
+  def _element_structure(self):
+    return self._structure
 
 
 class MultiDeviceIterator(object):
@@ -183,13 +164,6 @@ class MultiDeviceIterator(object):
     self._source_device = source_device
     self._source_device_tensor = ops.convert_to_tensor(source_device)
 
-    self._flat_output_shapes = nest.flatten(
-        sparse.as_dense_shapes(self._dataset.output_shapes,
-                               self._dataset.output_classes))
-    self._flat_output_types = nest.flatten(
-        sparse.as_dense_types(self._dataset.output_types,
-                              self._dataset.output_classes))
-
     # Create the MultiDeviceIterator.
     with ops.device(self._source_device):
       self._multi_device_iterator_resource = (
@@ -197,8 +171,7 @@ class MultiDeviceIterator(object):
               devices=self._devices,
               shared_name="",
               container="",
-              output_types=self._flat_output_types,
-              output_shapes=self._flat_output_shapes))
+              **dataset_ops.flat_structure(dataset)))
 
       # The incarnation ID is used to ensure consistency between the per-device
       # iterators and the multi-device iterator.
@@ -216,8 +189,7 @@ class MultiDeviceIterator(object):
     for i, device in enumerate(self._devices):
       ds = _PerDeviceGenerator(
           i, self._multi_device_iterator_resource, self._incarnation_id,
-          self._source_device_tensor, device, self._dataset.output_shapes,
-          self._dataset.output_types, self._dataset.output_classes)
+          self._source_device_tensor, device, dataset._element_structure)  # pylint: disable=protected-access
       if prefetch_buffer_size > 0:
         ds = ds.prefetch(prefetch_buffer_size)
       # TODO(jsimsa): Enable auto-tuning when supported for non-CPU devices.
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index 70a3b1b1cb..0d6023dea2 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from tensorflow.python.compat import compat
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import convert
+from tensorflow.python.data.util import structure
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -64,16 +65,8 @@ class TextLineDatasetV2(dataset_ops.DatasetSource):
         self._filenames, self._compression_type, self._buffer_size)
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.string
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
 
 
 @tf_export(v1=["data.TextLineDataset"])
@@ -126,16 +119,8 @@ class _TFRecordDataset(dataset_ops.DatasetSource):
         self._filenames, self._compression_type, self._buffer_size)
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.TensorShape([])
-
-  @property
-  def output_types(self):
-    return dtypes.string
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
 
 
 class ParallelInterleaveDataset(dataset_ops.InterleaveDataset):
@@ -168,7 +153,8 @@ class ParallelInterleaveDataset(dataset_ops.InterleaveDataset):
         self._buffer_output_elements,
         self._prefetch_input_elements,
         f=self._map_func.function,
-        **dataset_ops.flat_structure(structure=self._output_structure))
+        **dataset_ops.flat_structure(self))
+    # pylint: enable=protected-access
 
   def _transformation_name(self):
     return "tf.data.experimental.parallel_interleave()"
@@ -247,16 +233,8 @@ class TFRecordDatasetV2(dataset_ops.DatasetV2):
     return self._impl._inputs()  # pylint: disable=protected-access
 
   @property
-  def output_classes(self):
-    return self._impl.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._impl.output_shapes
-
-  @property
-  def output_types(self):
-    return self._impl.output_types
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
 
 
 @tf_export(v1=["data.TFRecordDataset"])
@@ -347,16 +325,8 @@ class FixedLengthRecordDatasetV2(dataset_ops.DatasetSource):
           self._footer_bytes, self._buffer_size)
 
   @property
-  def output_classes(self):
-    return ops.Tensor
-
-  @property
-  def output_shapes(self):
-    return tensor_shape.scalar()
-
-  @property
-  def output_types(self):
-    return dtypes.string
+  def _element_structure(self):
+    return structure.TensorStructure(dtypes.string, [])
 
 
 @tf_export(v1=["data.FixedLengthRecordDataset"])
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 874e8abc6d..92450abda8 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -190,56 +190,6 @@ class Structure(object):
         raise TypeError("Could not build a structure for %r" % value)
       return TensorStructure.from_value(tensor)
 
-  @staticmethod
-  def _from_legacy_structure(output_types, output_shapes, output_classes):
-    """Returns a `Structure` that represents the given legacy structure.
-
-    This method provides a way to convert from the existing `Dataset` and
-    `Iterator` structure-related properties to a `Structure` object.
-
-    TODO(b/110122868): Remove this method once `Structure` is used throughout
-    `tf.data`.
-
-    Args:
-      output_types: A nested structure of `tf.DType` objects corresponding to
-        each component of a structured value.
-      output_shapes: A nested structure of `tf.TensorShape` objects
-        corresponding to each component a structured value.
-      output_classes: A nested structure of Python `type` objects corresponding
-        to each component of a structured value.
-
-    Returns:
-      A `Structure`.
-
-    Raises:
-      TypeError: If a structure cannot be built the arguments, because one of
-        the component classes in `output_classes` is not supported.
-    """
-    flat_types = nest.flatten(output_types)
-    flat_shapes = nest.flatten(output_shapes)
-    flat_classes = nest.flatten(output_classes)
-    flat_ret = []
-    for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes,
-                                                 flat_classes):
-      if isinstance(flat_class, Structure):
-        flat_ret.append(flat_class)
-      elif issubclass(flat_class, sparse_tensor_lib.SparseTensor):
-        flat_ret.append(SparseTensorStructure(flat_type, flat_shape))
-      elif issubclass(flat_class, ops.Tensor):
-        flat_ret.append(TensorStructure(flat_type, flat_shape))
-      else:
-        # NOTE(mrry): Since legacy structures produced by iterators only
-        # comprise Tensors, SparseTensors, and nests, we do not need to
-        # support all structure types here.
-        raise TypeError(
-            "Could not build a structure for output class %r" % flat_type)
-
-    ret = nest.pack_sequence_as(output_classes, flat_ret)
-    if isinstance(ret, Structure):
-      return ret
-    else:
-      return NestedStructure(ret)
-
   @staticmethod
   def _register_custom_converter(type_object, converter_fn):
     """Registers `converter_fn` for converting values of the given type.
@@ -265,6 +215,59 @@ class Structure(object):
     raise NotImplementedError("Structure._to_legacy_output_classes()")
 
 
+def convert_legacy_structure(output_types, output_shapes, output_classes):
+  """Returns a `Structure` that represents the given legacy structure.
+
+  This method provides a way to convert from the existing `Dataset` and
+  `Iterator` structure-related properties to a `Structure` object. A "legacy"
+  structure is represented by the `tf.data.Dataset.output_types`,
+  `tf.data.Dataset.output_shapes`, and `tf.data.Dataset.output_classes`
+  properties.
+
+  TODO(b/110122868): Remove this function once `Structure` is used throughout
+  `tf.data`.
+
+  Args:
+    output_types: A nested structure of `tf.DType` objects corresponding to
+      each component of a structured value.
+    output_shapes: A nested structure of `tf.TensorShape` objects
+      corresponding to each component a structured value.
+    output_classes: A nested structure of Python `type` objects corresponding
+      to each component of a structured value.
+
+  Returns:
+    A `Structure`.
+
+  Raises:
+    TypeError: If a structure cannot be built from the arguments, because one of
+      the component classes in `output_classes` is not supported.
+  """
+  flat_types = nest.flatten(output_types)
+  flat_shapes = nest.flatten(output_shapes)
+  flat_classes = nest.flatten(output_classes)
+  flat_ret = []
+  for flat_type, flat_shape, flat_class in zip(flat_types, flat_shapes,
+                                               flat_classes):
+    if isinstance(flat_class, Structure):
+      flat_ret.append(flat_class)
+    elif issubclass(flat_class, sparse_tensor_lib.SparseTensor):
+      flat_ret.append(SparseTensorStructure(flat_type, flat_shape))
+    elif issubclass(flat_class, ops.Tensor):
+      flat_ret.append(TensorStructure(flat_type, flat_shape))
+    else:
+      # NOTE(mrry): Since legacy structures produced by iterators only
+      # comprise Tensors, SparseTensors, and nests, we do not need to
+      # support all structure types here.
+      raise TypeError(
+          "Could not build a structure for output class %r" % flat_type)
+
+  ret = nest.pack_sequence_as(output_classes, flat_ret)
+  if isinstance(ret, Structure):
+    return ret
+  else:
+    return NestedStructure(ret)
+
+
 # NOTE(mrry): The following classes make extensive use of non-public methods of
 # their base class, so we disable the protected-access lint warning once here.
 # pylint: disable=protected-access
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index 314a459eff..3425179e42 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -352,9 +352,9 @@ class StructureTest(test.TestCase, parameterized.TestCase):
            "b": (structure.SparseTensorStructure(dtypes.int32, [2, 2]),
                  structure.TensorStructure(dtypes.string, []))})),
   )
-  def testFromLegacyStructure(self, output_types, output_shapes, output_classes,
-                              expected_structure):
-    actual_structure = structure.Structure._from_legacy_structure(
+  def testConvertLegacyStructure(self, output_types, output_shapes,
+                                 output_classes, expected_structure):
+    actual_structure = structure.convert_legacy_structure(
         output_types, output_shapes, output_classes)
     self.assertTrue(expected_structure.is_compatible_with(actual_structure))
     self.assertTrue(actual_structure.is_compatible_with(expected_structure))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
index f59082baeb..f7d388d33d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-dataset.pbtxt
@@ -5,15 +5,15 @@ tf_class {
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
-    mtype: "<class \'abc.abstractproperty\'>"
+    mtype: "<type \'property\'>"
   }
   member {
     name: "output_shapes"
-    mtype: "<class \'abc.abstractproperty\'>"
+    mtype: "<type \'property\'>"
   }
   member {
     name: "output_types"
-    mtype: "<class \'abc.abstractproperty\'>"
+    mtype: "<type \'property\'>"
   }
   member_method {
     name: "__init__"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
index ac8dd2de7f..d877339409 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.-dataset.pbtxt
@@ -4,15 +4,15 @@ tf_class {
   is_instance: "<type \'object\'>"
   member {
     name: "output_classes"
-    mtype: "<class \'abc.abstractproperty\'>"
+    mtype: "<type \'property\'>"
   }
   member {
     name: "output_shapes"
-    mtype: "<class \'abc.abstractproperty\'>"
+    mtype: "<type \'property\'>"
   }
   member {
     name: "output_types"
-    mtype: "<class \'abc.abstractproperty\'>"
+    mtype: "<type \'property\'>"
   }
   member_method {
     name: "__init__"
-- 
GitLab


From 509a65f76773bc54190f75ff6613b39b0151453a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 16:58:49 -0800
Subject: [PATCH 1399/1554] Fix the test_util to ModifyGraphWithDelegate after
 AllocateTensors.

PiperOrigin-RevId: 224246409
---
 tensorflow/lite/kernels/test_util.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/test_util.cc b/tensorflow/lite/kernels/test_util.cc
index 6b2a1f89c3..549ea78f5b 100644
--- a/tensorflow/lite/kernels/test_util.cc
+++ b/tensorflow/lite/kernels/test_util.cc
@@ -129,14 +129,14 @@ void SingleOpModel::BuildInterpreter(std::vector<std::vector<int>> input_shapes,
 
   interpreter_->SetAllowFp16PrecisionForFp32(allow_fp32_relax_to_fp16);
 
+  CHECK(interpreter_->AllocateTensors() == kTfLiteOk)
+      << "Cannot allocate tensors";
+  interpreter_->ResetVariableTensors();
+
   // Modify delegate with function.
   if (apply_delegate_fn_) {
     apply_delegate_fn_(interpreter_.get());
   }
-
-  CHECK(interpreter_->AllocateTensors() == kTfLiteOk)
-      << "Cannot allocate tensors";
-  interpreter_->ResetVariableTensors();
 }
 
 void SingleOpModel::Invoke() { CHECK(interpreter_->Invoke() == kTfLiteOk); }
-- 
GitLab


From 0c17cac8c2ce8a29c25ad23358e77b5463231abc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 17:14:47 -0800
Subject: [PATCH 1400/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 224248551
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 324 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  42 +++
 2 files changed, 366 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 03eb18ca68..4b3c1c01da 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -22105,6 +22105,61 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalMapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ExperimentalMapDataset"
   input_arg {
@@ -22148,6 +22203,56 @@ op {
     }
   }
 }
+op {
+  name: "ExperimentalMapDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "use_inter_op_parallelism"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ExperimentalMatchingFilesDataset"
   input_arg {
@@ -22286,6 +22391,61 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalNumaMapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ExperimentalParallelInterleaveDataset"
   input_arg {
@@ -22590,6 +22750,59 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "ExperimentalScanDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "initial_state"
+    type_list_attr: "Tstate"
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Tstate"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ExperimentalSetStatsAggregatorDataset"
   input_arg {
@@ -31314,6 +31527,56 @@ op {
     }
   }
 }
+op {
+  name: "MapDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "use_inter_op_parallelism"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "MapDefun"
   input_arg {
@@ -39713,6 +39976,67 @@ op {
     }
   }
 }
+op {
+  name: "ParallelMapDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "num_parallel_calls"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "use_inter_op_parallelism"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "sloppy"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ParameterizedTruncatedNormal"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 1ff908aae8..77d94e4adb 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10504,6 +10504,13 @@ op {
     has_minimum: true
     minimum: 1
   }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
   name: "ExperimentalMapDataset"
@@ -10547,6 +10554,13 @@ op {
       b: true
     }
   }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
   name: "ExperimentalMatchingFilesDataset"
@@ -10685,6 +10699,13 @@ op {
     has_minimum: true
     minimum: 1
   }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
   name: "ExperimentalParallelInterleaveDataset"
@@ -10919,6 +10940,13 @@ op {
     has_minimum: true
     minimum: 1
   }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
   name: "ExperimentalSetStatsAggregatorDataset"
@@ -15854,6 +15882,13 @@ op {
       b: true
     }
   }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
   name: "MapDefun"
@@ -19904,6 +19939,13 @@ op {
       b: false
     }
   }
+  attr {
+    name: "preserve_cardinality"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
   name: "ParameterizedTruncatedNormal"
-- 
GitLab


From 1f798bf01bf7f06755006790db7d3f030d96e0c5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 17:38:50 -0800
Subject: [PATCH 1401/1554] This CL allows the Keras parameterized decorators
 to stack, and to be applied directly to an entire test class.

PiperOrigin-RevId: 224251795
---
 .../keras/engine/training_dataset_test.py     |  30 +-
 .../python/keras/engine/training_test.py      |   5 +-
 .../python/keras/keras_parameterized.py       | 310 ++++++++----------
 .../python/keras/keras_parameterized_test.py  | 215 ++++++++++--
 .../keras/layers/advanced_activations_test.py |   6 +-
 tensorflow/python/keras/testing_utils.py      |   6 +-
 6 files changed, 349 insertions(+), 223 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 13e4abf050..a8a81fa842 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -37,7 +37,8 @@ from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
 
-  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_iterators_single_io(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -97,7 +98,8 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
                                  'you should specify the `steps` argument'):
       model.predict(iterator, verbose=0)
 
-  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_get_next_op_created_once(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -119,7 +121,8 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
     ops.get_default_graph().finalize()
     model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
 
-  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_iterators_running_out_of_data(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -146,9 +149,8 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
 
   # TODO(kaftan) Run w/ all model types.
   # Seems like subclass models has a bug, file ticket
-  @keras_parameterized.run_all_keras_modes_with_all_model_types(
-      exclude_models='subclass'
-  )
+  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_all_keras_modes
   def test_calling_model_on_same_dataset(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -172,7 +174,8 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
               validation_data=dataset, validation_steps=2)
 
-  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_dataset(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -233,9 +236,8 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
 
   # TODO(kaftan) Run w/ all model types.
   # Seems like subclass models has a bug, file ticket
-  @keras_parameterized.run_all_keras_modes_with_all_model_types(
-      exclude_models='subclass'
-  )
+  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sample_weights(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -256,7 +258,8 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     model.evaluate(dataset, steps=2, verbose=1)
     model.predict(dataset, steps=2)
 
-  @keras_parameterized.run_all_keras_modes_with_all_model_types
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sparse_labels(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -307,9 +310,8 @@ class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
 
   # TODO(kaftan) Run w/ all model types.
   # Seems like subclass models has a bug, file ticket
-  @keras_parameterized.run_all_keras_modes_with_all_model_types(
-      exclude_models='subclass'
-  )
+  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_all_keras_modes
   def test_metrics_correctness_with_iterator(self):
     layers = [
         keras.layers.Dense(8, activation='relu', input_dim=4,
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index 572cde1d04..bd3d7d26d3 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -53,9 +53,8 @@ except ImportError:
 
 class TrainingTest(keras_parameterized.TestCase):
 
-  @keras_parameterized.run_all_keras_modes_with_all_model_types(
-      exclude_models='sequential'
-  )
+  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
+  @keras_parameterized.run_all_keras_modes
   def test_fit_on_arrays(self):
     input_a = keras.layers.Input(shape=(3,), name='input_a')
     input_b = keras.layers.Input(shape=(3,), name='input_b')
diff --git a/tensorflow/python/keras/keras_parameterized.py b/tensorflow/python/keras/keras_parameterized.py
index 57e66d89d9..d76bbadeb3 100644
--- a/tensorflow/python/keras/keras_parameterized.py
+++ b/tensorflow/python/keras/keras_parameterized.py
@@ -18,6 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
+import functools
+import itertools
+import unittest
+
 from absl.testing import parameterized
 
 from tensorflow.python import keras
@@ -38,21 +43,20 @@ class TestCase(test.TestCase, parameterized.TestCase):
 # TODO(kaftan): Possibly enable 'subclass_custom_build' when tests begin to pass
 # it. Or perhaps make 'subclass' always use a custom build method.
 def run_with_all_model_types(
-    func=None,
-    exclude_models=None,
-    extra_parameters=None):
+    test_or_class=None,
+    exclude_models=None):
   """Execute the decorated test with all Keras model types.
 
-  This decorator is intended to be applied to test methods in
-  a `testing_utils.KerasTestCase` class. Doing so will cause the contents of
-  the test method to be executed multiple times - once for each Keras model
-  type.
+  This decorator is intended to be applied either to individual test methods in
+  a `keras_parameterized.TestCase` class, or directly to a test class that
+  extends it. Doing so will cause the contents of the individual test
+  method (or all test methods in the class) to be executed multiple times - once
+  for each Keras model type.
 
   The Keras model types are: ['functional', 'subclass', 'sequential']
 
-  Note: This decorator does not stack with any other parameterizing
-  decorators. Use the extra_parameters arg if you would like to use additional
-  parameterization.
+  Note: if stacking this decorator with absl.testing's parameterized decorators,
+  those should be at the bottom of the stack.
 
   Various methods in `testing_utils` to get models will auto-generate a model
   of the currently active Keras model type. This allows unittests to confirm
@@ -64,7 +68,7 @@ def run_with_all_model_types(
   class MyTests(testing_utils.KerasTestCase):
 
     @testing_utils.run_with_all_model_types(
-      model_types = ['functional', 'subclass'])
+      exclude_models = ['sequential'])
     def test_foo(self):
       model = testing_utils.get_small_mlp(1, 4, input_dim=3)
       optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -87,66 +91,18 @@ def run_with_all_model_types(
   This test tries building a small mlp as both a functional model and as a
   subclass model.
 
-
-  Args:
-    func: function to be annotated. If `func` is None, this method returns a
-      decorator the can be applied to a function. If `func` is not None this
-      returns the decorator applied to `func`.
-    exclude_models: A collection of Keras model types to not run.
-      (May also be a single model type not wrapped in a collection).
-      Defaults to None.
-    extra_parameters: A list of dicts containing additional parameters for
-      parameterizing the decorated test method. This additional parameters must
-      be arguments in the test method's signature.
-      See the absl.testing.parameterized decorators for more info on how these
-      work.
-
-  Returns:
-    Returns a decorator that will run the decorated test method multiple times:
-    once for each desired Keras model type.
-
-  Raises:
-    ImportError: If abseil parameterized is not installed or not included as
-      a target dependency.
-  """
-  return run_all_keras_modes_with_all_model_types(
-      func, skip_run_modes=True, exclude_models=exclude_models,
-      extra_parameters=extra_parameters)
-
-
-def run_all_keras_modes(
-    func=None,
-    config=None,
-    always_skip_v1=False,
-    extra_parameters=None):
-  """Execute the decorated test with all keras execution modes.
-
-  This decorator is intended to be applied to test methods in
-  a `testing_utils.KerasTestCase` class. Doing so will cause the contents of
-  the test method to be executed three times - once executing in legacy graph
-  mode, once running eagerly and with `should_run_eagerly` returning True, and
-  once running eagerly with `should_run_eagerly` returning False.
-
-  If Tensorflow v2 behavior is enabled, legacy graph mode will be skipped, and
-  the test will only run twice.
-
-  Note: This decorator does not stack with any other parameterizing
-  decorators. Use the extra_parameters arg if you would like to use additional
-  parameterization.
-
-  For example, consider the following unittest:
-
+  We can also annotate the whole class if we want this to apply to all tests in
+  the class:
   ```python
+  @testing_utils.run_with_all_model_types(exclude_models = ['sequential'])
   class MyTests(testing_utils.KerasTestCase):
 
-    @testing_utils.run_all_keras_modes
     def test_foo(self):
-      model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
+      model = testing_utils.get_small_mlp(1, 4, input_dim=3)
       optimizer = RMSPropOptimizer(learning_rate=0.001)
       loss = 'mse'
       metrics = ['mae']
-      model.compile(optimizer, loss, metrics=metrics,
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(optimizer, loss, metrics=metrics)
 
       inputs = np.zeros((10, 3))
       targets = np.zeros((10, 4))
@@ -160,62 +116,69 @@ def run_all_keras_modes(
     tf.test.main()
   ```
 
-  This test will try compiling & fitting the small functional mlp using all
-  three Keras execution modes.
 
   Args:
-    func: function to be annotated. If `func` is None, this method returns a
-      decorator the can be applied to a function. If `func` is not None this
-      returns the decorator applied to `func`.
-    config: An optional config_pb2.ConfigProto to use to configure the
-      session when executing graphs.
-    always_skip_v1: If True, does not try running the legacy graph mode even
-      when Tensorflow v2 behavior is not enabled.
-    extra_parameters: A list of dicts containing additional parameters for
-      parameterizing the decorated test method. This additional parameters must
-      be arguments in the test method's signature. See the
-      absl.testing.parameterized  decorators for more info on how these work.
+    test_or_class: test method or class to be annotated. If None,
+      this method returns a decorator that can be applied to a test method or
+      test class. If it is not None this returns the decorator applied to the
+      test or class.
+    exclude_models: A collection of Keras model types to not run.
+      (May also be a single model type not wrapped in a collection).
+      Defaults to None.
 
   Returns:
-    Returns a decorator that will run the decorated test method multiple times.
+    Returns a decorator that will run the decorated test method multiple times:
+    once for each desired Keras model type.
 
   Raises:
     ImportError: If abseil parameterized is not installed or not included as
       a target dependency.
   """
-  return run_all_keras_modes_with_all_model_types(
-      func=func,
-      skip_model_types=True,
-      config=config,
-      always_skip_v1=always_skip_v1,
-      extra_parameters=extra_parameters)
-
-
-def run_all_keras_modes_with_all_model_types(
-    func=None,
-    skip_run_modes=False,
-    skip_model_types=False,
+  model_types = ['functional', 'subclass', 'sequential']
+  params = [('_%s' % model, model) for model in model_types
+            if model not in nest.flatten(exclude_models)]
+
+  def single_method_decorator(f):
+    """Decorator that constructs the test cases."""
+    # Use named_parameters so it can be individually run from the command line
+    @parameterized.named_parameters(*params)
+    @functools.wraps(f)
+    def decorated(self, model_type, *args, **kwargs):
+      """A run of a single test case w/ the specified model type."""
+      with testing_utils.model_type_scope(model_type):
+        f(self, *args, **kwargs)
+
+    return decorated
+
+  return _test_or_class_decorator(test_or_class, single_method_decorator)
+
+
+def run_all_keras_modes(
+    test_or_class=None,
     config=None,
-    always_skip_v1=False,
-    exclude_models=None,
-    extra_parameters=None):
-  """Execute the decorated test with all keras run modes and model types.
+    always_skip_v1=False):
+  """Execute the decorated test with all keras execution modes.
+
+  This decorator is intended to be applied either to individual test methods in
+  a `keras_parameterized.TestCase` class, or directly to a test class that
+  extends it. Doing so will cause the contents of the individual test
+  method (or all test methods in the class) to be executed multiple times -
+  once executing in legacy graph mode, once running eagerly and with
+  `should_run_eagerly` returning True, and once running eagerly with
+  `should_run_eagerly` returning False.
 
-  This decorator is intended to be applied to test methods in
-  a `testing_utils.KerasTestCase` class. Doing so will cause the contents of
-  the test method to be executed once for each combination of Keras run mode
-  and model type.
+  If Tensorflow v2 behavior is enabled, legacy graph mode will be skipped, and
+  the test will only run twice.
 
-  Note: This decorator does not stack with any other parameterizing
-  decorators. Use the extra_parameters arg if you would like to use additional
-  parameterization.
+  Note: if stacking this decorator with absl.testing's parameterized decorators,
+  those should be at the bottom of the stack.
 
   For example, consider the following unittest:
 
   ```python
   class MyTests(testing_utils.KerasTestCase):
 
-    @testing_utils.run_all_keras_modes_with_all_model_types
+    @testing_utils.run_all_keras_modes
     def test_foo(self):
       model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
       optimizer = RMSPropOptimizer(learning_rate=0.001)
@@ -237,30 +200,17 @@ def run_all_keras_modes_with_all_model_types(
   ```
 
   This test will try compiling & fitting the small functional mlp using all
-  combinations of the Keras execution modes and the Keras model types.
-
-  The three Keras execution modes are v1 graph mode, v2 functions, and v2 eager.
-  If Tensorflow v2 behavior is enabled, v1 graph mode will be skipped.
-  The Keras model types are: ['functional', 'subclass', 'sequential']
+  three Keras execution modes.
 
   Args:
-    func: function to be annotated. If `func` is None, this method returns a
-      decorator the can be applied to a function. If `func` is not None this
-      returns the decorator applied to `func`.
-    skip_run_modes: If True, does not try testing different Keras run modes.
-      Defaults to False, which tests all combinations of run modes.
-    skip_model_types: If True, does not try testing different Keras model types.
-      Defaults to False, which tests all combinations of model types.
+    test_or_class: test method or class to be annotated. If None,
+      this method returns a decorator that can be applied to a test method or
+      test class. If it is not None this returns the decorator applied to the
+      test or class.
     config: An optional config_pb2.ConfigProto to use to configure the
       session when executing graphs.
     always_skip_v1: If True, does not try running the legacy graph mode even
       when Tensorflow v2 behavior is not enabled.
-    exclude_models: A collection of Keras model types to not run.
-      (May also be a single model type not wrapped in a collection).
-    extra_parameters: A list of dicts containing additional parameters for
-      parameterizing the decorated test method. This additional parameters must
-      be arguments in the test method's signature. See the
-      absl.testing.parameterized decorators for more info on how these work.
 
   Returns:
     Returns a decorator that will run the decorated test method multiple times.
@@ -269,80 +219,80 @@ def run_all_keras_modes_with_all_model_types(
     ImportError: If abseil parameterized is not installed or not included as
       a target dependency.
   """
-  if not parameterized:
-    raise ImportError('To use the Keras parameterized testing utilities, you '
-                      'must have absl.testing.parameterized installed and '
-                      'included as a target dependency.')
-  params = []
-  if not skip_run_modes:
-    if not (always_skip_v1 or tf2.enabled()):
-      params.append({'testcase_name': '_v1_graph', '_run_mode': 'v1_graph'})
-    params.append({'testcase_name': '_v2_eager', '_run_mode': 'v2_eager'})
-    params.append({'testcase_name': '_v2_function', '_run_mode': 'v2_function'})
-  else:
-    params.append({'testcase_name': ''})
-
-  if not skip_model_types:
-    model_types = ['functional', 'subclass', 'sequential']
-    old_params = params
-    params = []
-    for param in old_params:
-      for model in model_types:
-        if model not in nest.flatten(exclude_models):
-          new_param = param.copy()
-          new_param['_model_type'] = model
-          new_param['testcase_name'] = '{}_{}'.format(
-              param['testcase_name'], model)
-          params.append(new_param)
-
-  if extra_parameters:
-    old_params = params
-    params = []
-    for param in old_params:
-      for i, extra_params in enumerate(extra_parameters):
-        new_param = param.copy()
-        new_param.update(extra_params)
-        new_param['testcase_name'] = '{}_{}'.format(
-            param['testcase_name'], i)
-        params.append(new_param)
-
-  def decorator(f):
+  params = [('_v2_eager', 'v2_eager'),
+            ('_v2_function', 'v2_function')]
+  if not (always_skip_v1 or tf2.enabled()):
+    params.append(('_v1_graph', 'v1_graph'))
+
+  def single_method_decorator(f):
     """Decorator that constructs the test cases."""
-    def run_with_model_type(self, model_type, *args, **kwargs):
-      if model_type:
-        with testing_utils.model_type_scope(model_type):
-          f(self, *args, **kwargs)
-      else:
-        f(self, *args, **kwargs)
 
     # Use named_parameters so it can be individually run from the command line
-    @parameterized.named_parameters(
-        *params
-    )
-    def decorated(self, *args, **kwargs):
-      """A run of a single test case w/ specified run mode / model type."""
-      run_mode = kwargs.pop('_run_mode', None)
-      model_type = kwargs.pop('_model_type', None)
-      if run_mode is None:
-        run_with_model_type(self, model_type, *args, **kwargs)
-      elif run_mode == 'v1_graph':
+    @parameterized.named_parameters(*params)
+    @functools.wraps(f)
+    def decorated(self, run_mode, *args, **kwargs):
+      """A run of a single test case w/ specified run mode."""
+      if run_mode == 'v1_graph':
         with context.graph_mode(), testing_utils.run_eagerly_scope(False):
           with self.test_session(use_gpu=True, config=config):
-            run_with_model_type(self, model_type, *args, **kwargs)
+            f(self, *args, **kwargs)
       elif run_mode == 'v2_function':
         with context.eager_mode():
           with testing_utils.run_eagerly_scope(False):
-            run_with_model_type(self, model_type, *args, **kwargs)
+            f(self, *args, **kwargs)
       elif run_mode == 'v2_eager':
         with context.eager_mode():
           with testing_utils.run_eagerly_scope(True):
-            run_with_model_type(self, model_type, *args, **kwargs)
+            f(self, *args, **kwargs)
       else:
         return ValueError('Unknown run mode %s' % run_mode)
 
     return decorated
 
-  if func is not None:
-    return decorator(func)
+  return _test_or_class_decorator(test_or_class, single_method_decorator)
+
+
+def _test_or_class_decorator(test_or_class, single_method_decorator):
+  """Decorate a test or class with a decorator intended for one method.
+
+  If the test_or_class is a class:
+    This will apply the decorator to all test methods in the class.
+
+  If the test_or_class is an iterable of already-parameterized test cases:
+    This will apply the decorator to all the cases, and then flatten the
+    resulting cross-product of test cases. This allows stacking the Keras
+    parameterized decorators w/ each other, and to apply them to test methods
+    that have already been marked with an absl parameterized decorator.
 
-  return decorator
+  Otherwise, treat the obj as a single method and apply the decorator directly.
+
+  Args:
+    test_or_class: A test method (that may have already been decorated with a
+      parameterized decorator, or a test class that extends
+      keras_parameterized.TestCase
+    single_method_decorator:
+      A parameterized decorator intended for a single test method.
+  Returns:
+    The decorated result.
+  """
+  def _decorate_test_or_class(obj):
+    if isinstance(obj, collections.Iterable):
+      return itertools.chain.from_iterable(
+          single_method_decorator(method) for method in obj)
+    if isinstance(obj, type):
+      cls = obj
+      for name, value in cls.__dict__.copy().items():
+        if callable(value) and name.startswith(
+            unittest.TestLoader.testMethodPrefix):
+          setattr(cls, name, single_method_decorator(value))
+
+      cls = type(cls).__new__(type(cls), cls.__name__, cls.__bases__,
+                              cls.__dict__.copy())
+      return cls
+
+    return single_method_decorator(obj)
+
+  if test_or_class is not None:
+    return _decorate_test_or_class(test_or_class)
+
+  return _decorate_test_or_class
diff --git a/tensorflow/python/keras/keras_parameterized_test.py b/tensorflow/python/keras/keras_parameterized_test.py
index 104c3b1ec9..a0ff40cfc7 100644
--- a/tensorflow/python/keras/keras_parameterized_test.py
+++ b/tensorflow/python/keras/keras_parameterized_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import unittest
 
+from absl.testing import parameterized
+
 from tensorflow.python import keras
 from tensorflow.python import tf2
 from tensorflow.python.eager import context
@@ -78,21 +80,22 @@ class KerasParameterizedTest(keras_parameterized.TestCase):
       def runTest(self):
         pass
 
-      @keras_parameterized.run_with_all_model_types(
-          extra_parameters=[dict(with_brackets=True), dict(with_brackets=False)]
-      )
+      @keras_parameterized.run_with_all_model_types
+      @parameterized.named_parameters(
+          [dict(testcase_name="_0", with_brackets=True),
+           dict(testcase_name="_1", with_brackets=False)])
       def testBody(self, with_brackets):
         with_brackets = "with_brackets" if with_brackets else "without_brackets"
         model_types.append((with_brackets, testing_utils.get_model_type()))
         models.append(testing_utils.get_small_mlp(1, 4, input_dim=3))
 
     e = ExampleTest()
-    e.testBody_functional_0()
-    e.testBody_subclass_0()
-    e.testBody_sequential_0()
-    e.testBody_functional_1()
-    e.testBody_subclass_1()
-    e.testBody_sequential_1()
+    e.testBody_0_functional()
+    e.testBody_0_subclass()
+    e.testBody_0_sequential()
+    e.testBody_1_functional()
+    e.testBody_1_subclass()
+    e.testBody_1_sequential()
 
     self.assertLen(model_types, 6)
     self.assertAllEqual(model_types, [
@@ -247,9 +250,10 @@ class KerasParameterizedTest(keras_parameterized.TestCase):
       def runTest(self):
         pass
 
-      @keras_parameterized.run_all_keras_modes(
-          extra_parameters=[dict(with_brackets=True), dict(with_brackets=False)]
-      )
+      @keras_parameterized.run_all_keras_modes
+      @parameterized.named_parameters(
+          [dict(testcase_name="_0", with_brackets=True),
+           dict(testcase_name="_1", with_brackets=False)])
       def testBody(self, with_brackets):
         mode = "eager" if context.executing_eagerly() else "graph"
         with_brackets = "with_brackets" if with_brackets else "without_brackets"
@@ -258,13 +262,13 @@ class KerasParameterizedTest(keras_parameterized.TestCase):
 
     e = ExampleTest()
     if not tf2.enabled():
-      e.testBody_v1_graph_0()
-      e.testBody_v1_graph_1()
+      e.testBody_0_v1_graph()
+      e.testBody_1_v1_graph()
 
-    e.testBody_v2_eager_0()
-    e.testBody_v2_function_0()
-    e.testBody_v2_eager_1()
-    e.testBody_v2_function_1()
+    e.testBody_0_v2_eager()
+    e.testBody_0_v2_function()
+    e.testBody_1_v2_eager()
+    e.testBody_1_v2_function()
 
     expected_combinations = {
         ("with_brackets", "eager", True),
@@ -324,7 +328,8 @@ class KerasParameterizedTest(keras_parameterized.TestCase):
       def runTest(self):
         pass
 
-      @keras_parameterized.run_all_keras_modes_with_all_model_types
+      @keras_parameterized.run_with_all_model_types
+      @keras_parameterized.run_all_keras_modes
       def testBody(self):
         mode = "eager" if context.executing_eagerly() else "graph"
         should_run_eagerly = testing_utils.should_run_eagerly()
@@ -368,12 +373,178 @@ class KerasParameterizedTest(keras_parameterized.TestCase):
 
     self.assertLen(l, len(expected_combinations) * 2)
 
-  @keras_parameterized.run_all_keras_modes(extra_parameters=[dict(arg=True)])
+  def test_run_all_model_types_with_all_keras_modes(self):
+    l = []
+
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_all_keras_modes
+      @keras_parameterized.run_with_all_model_types
+      def testBody(self):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((mode, should_run_eagerly, testing_utils.get_model_type()))
+
+    e = ExampleTest()
+    e.testBody_functional_v2_eager()
+    e.testBody_functional_v2_function()
+    e.testBody_sequential_v2_eager()
+    e.testBody_sequential_v2_function()
+    e.testBody_subclass_v2_eager()
+    e.testBody_subclass_v2_function()
+
+    if not tf2.enabled():
+      e.testBody_functional_v1_graph()
+      e.testBody_sequential_v1_graph()
+      e.testBody_subclass_v1_graph()
+
+    expected_combinations = {
+        ("eager", True, "functional"),
+        ("eager", False, "functional"),
+        ("eager", True, "sequential"),
+        ("eager", False, "sequential"),
+        ("eager", True, "subclass"),
+        ("eager", False, "subclass"),
+    }
+
+    if not tf2.enabled():
+      expected_combinations = expected_combinations.union({
+          ("graph", False, "functional"),
+          ("graph", False, "sequential"),
+          ("graph", False, "subclass"),
+      })
+
+    self.assertLen(l, len(expected_combinations))
+    self.assertEqual(set(l), expected_combinations)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(l, len(expected_combinations) * 2)
+
+  def test_run_all_keras_modes_with_all_model_types_annotate_class(self):
+    l = []
+
+    @keras_parameterized.run_with_all_model_types
+    @keras_parameterized.run_all_keras_modes
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @parameterized.named_parameters(dict(testcase_name="_arg",
+                                           arg=True))
+      def testBody(self, arg):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((mode, should_run_eagerly, testing_utils.get_model_type()))
+
+    e = ExampleTest()
+    e.testBody_arg_v2_eager_functional()
+    e.testBody_arg_v2_function_functional()
+    e.testBody_arg_v2_eager_sequential()
+    e.testBody_arg_v2_function_sequential()
+    e.testBody_arg_v2_eager_subclass()
+    e.testBody_arg_v2_function_subclass()
+
+    if not tf2.enabled():
+      e.testBody_arg_v1_graph_functional()
+      e.testBody_arg_v1_graph_sequential()
+      e.testBody_arg_v1_graph_subclass()
+
+    expected_combinations = {
+        ("eager", True, "functional"),
+        ("eager", False, "functional"),
+        ("eager", True, "sequential"),
+        ("eager", False, "sequential"),
+        ("eager", True, "subclass"),
+        ("eager", False, "subclass"),
+    }
+
+    if not tf2.enabled():
+      expected_combinations = expected_combinations.union({
+          ("graph", False, "functional"),
+          ("graph", False, "sequential"),
+          ("graph", False, "subclass"),
+      })
+
+    self.assertLen(l, len(expected_combinations))
+    self.assertEqual(set(l), expected_combinations)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(l, len(expected_combinations) * 2)
+
+  def test_run_all_keras_modes_with_all_model_types_annotate_class_2(self):
+    l = []
+
+    @keras_parameterized.run_with_all_model_types
+    class ExampleTest(keras_parameterized.TestCase):
+
+      def runTest(self):
+        pass
+
+      @keras_parameterized.run_all_keras_modes
+      @parameterized.named_parameters(dict(testcase_name="_arg",
+                                           arg=True))
+      def testBody(self, arg):
+        mode = "eager" if context.executing_eagerly() else "graph"
+        should_run_eagerly = testing_utils.should_run_eagerly()
+        l.append((mode, should_run_eagerly, testing_utils.get_model_type()))
+
+    e = ExampleTest()
+    e.testBody_arg_v2_eager_functional()
+    e.testBody_arg_v2_function_functional()
+    e.testBody_arg_v2_eager_sequential()
+    e.testBody_arg_v2_function_sequential()
+    e.testBody_arg_v2_eager_subclass()
+    e.testBody_arg_v2_function_subclass()
+
+    if not tf2.enabled():
+      e.testBody_arg_v1_graph_functional()
+      e.testBody_arg_v1_graph_sequential()
+      e.testBody_arg_v1_graph_subclass()
+
+    expected_combinations = {
+        ("eager", True, "functional"),
+        ("eager", False, "functional"),
+        ("eager", True, "sequential"),
+        ("eager", False, "sequential"),
+        ("eager", True, "subclass"),
+        ("eager", False, "subclass"),
+    }
+
+    if not tf2.enabled():
+      expected_combinations = expected_combinations.union({
+          ("graph", False, "functional"),
+          ("graph", False, "sequential"),
+          ("graph", False, "subclass"),
+      })
+
+    self.assertLen(l, len(expected_combinations))
+    self.assertEqual(set(l), expected_combinations)
+
+    ts = unittest.makeSuite(ExampleTest)
+    res = unittest.TestResult()
+    ts.run(res)
+
+    self.assertLen(l, len(expected_combinations) * 2)
+
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters(dict(testcase_name="argument",
+                                       arg=True))
   def test_run_all_keras_modes_extra_params_2(self, arg):
     self.assertEqual(arg, True)
 
-  @keras_parameterized.run_with_all_model_types(
-      extra_parameters=[dict(arg=True)])
+  @keras_parameterized.run_with_all_model_types
+  @parameterized.named_parameters(dict(testcase_name="argument",
+                                       arg=True))
   def test_run_with_all_model_types_extra_params_2(self, arg):
     self.assertEqual(arg, True)
 
diff --git a/tensorflow/python/keras/layers/advanced_activations_test.py b/tensorflow/python/keras/layers/advanced_activations_test.py
index 4aadf535e0..f32bb457c8 100644
--- a/tensorflow/python/keras/layers/advanced_activations_test.py
+++ b/tensorflow/python/keras/layers/advanced_activations_test.py
@@ -20,13 +20,13 @@ from __future__ import print_function
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
-from tensorflow.python.framework import test_util as tf_test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 
 
-@tf_test_util.run_all_in_graph_and_eager_modes
-class AdvancedActivationsTest(test.TestCase):
+@keras_parameterized.run_all_keras_modes
+class AdvancedActivationsTest(keras_parameterized.TestCase):
 
   def test_leaky_relu(self):
     for alpha in [0., .5, -1.]:
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index 59ed77695a..42f8f10fca 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -157,7 +157,11 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
   # train(). This was causing some error for layer with Defun as it body.
   # See b/120160788 for more details. This should be mitigated after 2.0.
   model = keras.models.Model(x, layer(x))
-  model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'])
+  if _thread_local_data.run_eagerly is not None:
+    model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'],
+                  run_eagerly=should_run_eagerly())
+  else:
+    model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'])
   model.train_on_batch(input_data, actual_output)
 
   # test as first layer in Sequential API
-- 
GitLab


From 7a34a7a1e04de2f6f0f4a8374414dfe363004d52 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 5 Dec 2018 17:58:10 -0800
Subject: [PATCH 1402/1554] [tf.data] Preserve preserve_cardinality attr in
 AsGraphDefInternal

PiperOrigin-RevId: 224254355
---
 .../experimental/map_and_batch_dataset_op.cc    |  6 +++++-
 .../data/experimental/scan_dataset_op.cc        |  5 ++++-
 tensorflow/core/kernels/data/map_dataset_op.cc  |  8 +++++++-
 .../kernels/data/parallel_map_dataset_op.cc     |  8 +++++++-
 .../python/data/experimental/kernel_tests/BUILD |  1 +
 .../data/experimental/kernel_tests/scan_test.py | 16 ++++++++++++++++
 tensorflow/python/data/kernel_tests/BUILD       |  2 ++
 tensorflow/python/data/kernel_tests/map_test.py | 17 +++++++++++++++++
 8 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index b557e32cae..d3ae845cfb 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -217,6 +217,8 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       b->BuildAttrValue(func_, &f);
       AttrValue other_arguments_types_attr;
       b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
+      AttrValue preserve_cardinality_attr;
+      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
 
       TF_RETURN_IF_ERROR(b->AddDataset(
           this,
@@ -226,7 +228,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
            std::make_pair(4, drop_remainder_node)},  // Single tensor inputs.
           {std::make_pair(1, other_arguments)},      // Tensor list inputs.
           {std::make_pair("f", f),
-           std::make_pair("Targuments", other_arguments_types_attr)},  // Attrs
+           std::make_pair("Targuments", other_arguments_types_attr),
+           std::make_pair("preserve_cardinality",
+                          preserve_cardinality_attr)},  // Attrs
           output));
       return Status::OK();
     }
diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 7c6d2f7c59..0d9a629a27 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -129,12 +129,15 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
       b->BuildAttrValue(state_types_, &state_types);
       AttrValue other_arguments_types_attr;
       b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
+      AttrValue preserve_cardinality_attr;
+      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
       TF_RETURN_IF_ERROR(
           b->AddDataset(this, {{0, input_node}},
                         {{1, initial_state_nodes}, {2, other_arguments}},
                         {{"f", f},
                          {"Tstate", state_types},
-                         {"Targuments", other_arguments_types_attr}},
+                         {"Targuments", other_arguments_types_attr},
+                         {"preserve_cardinality", preserve_cardinality_attr}},
                         output));
       return Status::OK();
     }
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index c3c392d9c2..fc6e93a81c 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -168,13 +168,19 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
       b->BuildAttrValue(use_inter_op_parallelism_,
                         &use_inter_op_parallelism_attr);
 
+      // Attr: preserve_cardinality
+      AttrValue preserve_cardinality_attr;
+      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
+
       TF_RETURN_IF_ERROR(b->AddDataset(
           this, {std::make_pair(0, input_graph_node)},  // Single tensor inputs.
           {std::make_pair(1, other_arguments)},         // Tensor list inputs.
           {std::make_pair("f", f_attr),
            std::make_pair("Targuments", other_arguments_types_attr),
            std::make_pair("use_inter_op_parallelism",
-                          use_inter_op_parallelism_attr)},  // Attrs
+                          use_inter_op_parallelism_attr),
+           std::make_pair("preserve_cardinality",
+                          preserve_cardinality_attr)},  // Attrs
           output));
       return Status::OK();
     }
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index 4924ad63a0..5ac81c187c 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -168,6 +168,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
       AttrValue sloppy_attr;
       b->BuildAttrValue(sloppy_, &sloppy_attr);
 
+      // Attr: preserve_cardinality
+      AttrValue preserve_cardinality_attr;
+      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
+
       TF_RETURN_IF_ERROR(b->AddDataset(
           this,
           {std::make_pair(0, input_graph_node),
@@ -177,7 +181,9 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
            std::make_pair("Targuments", other_arguments_types_attr),
            std::make_pair("use_inter_op_parallelism",
                           use_inter_op_parallelism_attr),
-           std::make_pair("sloppy", sloppy_attr)},  // Attrs
+           std::make_pair("sloppy", sloppy_attr),
+           std::make_pair("preserve_cardinality",
+                          preserve_cardinality_attr)},  // Attrs
           output));
       return Status::OK();
     }
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index c76e576b5b..07ed4d2bce 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -545,6 +545,7 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:script_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/experimental/ops:scan_ops",
         "//tensorflow/python/data/kernel_tests:test_base",
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index 03af7ecd2f..bd974b21e3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
 
@@ -169,6 +170,21 @@ class ScanTest(test_base.DatasetTestBase):
       dataset.apply(
           scan_ops.scan(constant_op.constant(1, dtype=dtypes.int32), _scan_fn))
 
+  def testPreserveCardinality(self):
+
+    def scan_fn(state, val):
+
+      def py_fn(_):
+        raise StopIteration()
+
+      return state, script_ops.py_func(py_fn, [val], dtypes.int64)
+
+    dataset = dataset_ops.Dataset.from_tensors(0).apply(
+        scan_ops.scan(constant_op.constant(1), scan_fn))
+    get_next = self.getNext(dataset)
+    with self.assertRaises(errors.InvalidArgumentError):
+      self.evaluate(get_next())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 9f7ce99cbc..3390100bed 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -97,6 +97,8 @@ tf_py_test(
         "//tensorflow/python/data/util:nest",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:script_ops",
         "//tensorflow/python:sparse_tensor",
     ],
 )
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index fdce791447..8b22542e11 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -1045,6 +1045,23 @@ class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @parameterized.named_parameters(
+      ("Map", None),
+      ("ParallelMap", 12),
+  )
+  def testPreserveCardinality(self, num_parallel_calls):
+
+    def py_fn(_):
+      raise StopIteration()
+
+    dataset = dataset_ops.DatasetV2.from_tensors(0).map(
+        lambda x: script_ops.py_func(py_fn, [x], dtypes.int64),
+        num_parallel_calls=num_parallel_calls)
+    get_next = self.getNext(dataset)
+    with self.assertRaises(errors.InvalidArgumentError):
+      self.evaluate(get_next())
+
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From d062d33987dce9ed9dc0248bf1aece8e5a24cf72 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Wed, 5 Dec 2018 18:01:56 -0800
Subject: [PATCH 1403/1554] Remove more symbols from TF 2.0 API.

- InteractiveSession
- SessionLog
- SparseTensorValue
- test.StubOutForTesting

PiperOrigin-RevId: 224254774
---
 tensorflow/python/__init__.py                 |  2 +-
 tensorflow/python/framework/sparse_tensor.py  |  2 +-
 tensorflow/python/platform/googletest.py      |  2 +-
 .../golden/v2/tensorflow.-session-log.pbtxt   | 44 -------------------
 .../v2/tensorflow.-sparse-tensor-value.pbtxt  | 26 -----------
 .../tools/api/golden/v2/tensorflow.pbtxt      |  8 ----
 .../v2/tensorflow.summary.-session-log.pbtxt  | 44 -------------------
 .../api/golden/v2/tensorflow.summary.pbtxt    |  4 --
 ...ensorflow.test.-stub-out-for-testing.pbtxt | 28 ------------
 .../tools/api/golden/v2/tensorflow.test.pbtxt |  4 --
 tensorflow/tools/compatibility/renames_v2.py  |  4 ++
 11 files changed, 7 insertions(+), 161 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-session-log.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-session-log.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.test.-stub-out-for-testing.pbtxt

diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index a48e2b6f38..b2cc63bd13 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -159,7 +159,7 @@ tf_export(v1=['NodeDef'])(NodeDef)
 tf_export(v1=['OptimizerOptions'])(OptimizerOptions)
 tf_export(v1=['RunMetadata'])(RunMetadata)
 tf_export(v1=['RunOptions'])(RunOptions)
-tf_export('SessionLog', 'summary.SessionLog')(SessionLog)
+tf_export(v1=['SessionLog', 'summary.SessionLog'])(SessionLog)
 tf_export('Summary', 'summary.Summary')(Summary)
 tf_export('summary.SummaryDescription')(SummaryDescription)
 tf_export('SummaryMetadata')(SummaryMetadata)
diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py
index 3643fc5e00..5e1a95a26b 100644
--- a/tensorflow/python/framework/sparse_tensor.py
+++ b/tensorflow/python/framework/sparse_tensor.py
@@ -244,7 +244,7 @@ class SparseTensor(_TensorLike):
 
 SparseTensorValue = collections.namedtuple(
     "SparseTensorValue", ["indices", "values", "dense_shape"])
-tf_export("SparseTensorValue")(SparseTensorValue)
+tf_export(v1=["SparseTensorValue"])(SparseTensorValue)
 pywrap_tensorflow.RegisterType("SparseTensorValue", SparseTensorValue)
 
 
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index 4d34c508da..5b20e36a69 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -142,7 +142,7 @@ def StatefulSessionAvailable():
   return False
 
 
-@tf_export('test.StubOutForTesting')
+@tf_export(v1=['test.StubOutForTesting'])
 class StubOutForTesting(object):
   """Support class for stubbing methods out for unit testing.
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-session-log.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-session-log.pbtxt
deleted file mode 100644
index 259f241874..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-session-log.pbtxt
+++ /dev/null
@@ -1,44 +0,0 @@
-path: "tensorflow.SessionLog"
-tf_proto {
-  descriptor {
-    name: "SessionLog"
-    field {
-      name: "status"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.SessionLog.SessionStatus"
-    }
-    field {
-      name: "checkpoint_path"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "msg"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    enum_type {
-      name: "SessionStatus"
-      value {
-        name: "STATUS_UNSPECIFIED"
-        number: 0
-      }
-      value {
-        name: "START"
-        number: 1
-      }
-      value {
-        name: "STOP"
-        number: 2
-      }
-      value {
-        name: "CHECKPOINT"
-        number: 3
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor-value.pbtxt
deleted file mode 100644
index d33fd4d5d7..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor-value.pbtxt
+++ /dev/null
@@ -1,26 +0,0 @@
-path: "tensorflow.SparseTensorValue"
-tf_class {
-  is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensorValue\'>"
-  is_instance: "<type \'tuple\'>"
-  member {
-    name: "dense_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "indices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "values"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "count"
-  }
-  member_method {
-    name: "index"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 4fa57969ef..7b1c96c2e8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -36,18 +36,10 @@ tf_module {
     name: "RegisterGradient"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "SessionLog"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "SparseTensor"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "SparseTensorValue"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Summary"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-session-log.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-session-log.pbtxt
deleted file mode 100644
index 73de73869c..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-session-log.pbtxt
+++ /dev/null
@@ -1,44 +0,0 @@
-path: "tensorflow.summary.SessionLog"
-tf_proto {
-  descriptor {
-    name: "SessionLog"
-    field {
-      name: "status"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_ENUM
-      type_name: ".tensorflow.SessionLog.SessionStatus"
-    }
-    field {
-      name: "checkpoint_path"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "msg"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    enum_type {
-      name: "SessionStatus"
-      value {
-        name: "STATUS_UNSPECIFIED"
-        number: 0
-      }
-      value {
-        name: "START"
-        number: 1
-      }
-      value {
-        name: "STOP"
-        number: 2
-      }
-      value {
-        name: "CHECKPOINT"
-        number: 3
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
index 42a74a65fb..5cf4d7cfd9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
@@ -12,10 +12,6 @@ tf_module {
     name: "FileWriterCache"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "SessionLog"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "Summary"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.-stub-out-for-testing.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.-stub-out-for-testing.pbtxt
deleted file mode 100644
index e02a0c6097..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.test.-stub-out-for-testing.pbtxt
+++ /dev/null
@@ -1,28 +0,0 @@
-path: "tensorflow.test.StubOutForTesting"
-tf_class {
-  is_instance: "<class \'tensorflow.python.platform.googletest.StubOutForTesting\'>"
-  member_method {
-    name: "CleanUp"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "Set"
-    argspec: "args=[\'self\', \'parent\', \'child_name\', \'new_child\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SmartSet"
-    argspec: "args=[\'self\', \'obj\', \'attr_name\', \'new_attr\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "SmartUnsetAll"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "UnsetAll"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
index 72ce733044..980e96ac25 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.test.pbtxt
@@ -4,10 +4,6 @@ tf_module {
     name: "Benchmark"
     mtype: "<class \'tensorflow.python.platform.benchmark._BenchmarkRegistrar\'>"
   }
-  member {
-    name: "StubOutForTesting"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "TestCase"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 0352083389..02212c3810 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -68,8 +68,10 @@ renames = {
     'tf.RunMetadata': 'tf.compat.v1.RunMetadata',
     'tf.RunOptions': 'tf.compat.v1.RunOptions',
     'tf.Session': 'tf.compat.v1.Session',
+    'tf.SessionLog': 'tf.compat.v1.SessionLog',
     'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator',
     'tf.SparseFeature': 'tf.io.SparseFeature',
+    'tf.SparseTensorValue': 'tf.compat.v1.SparseTensorValue',
     'tf.TFRecordReader': 'tf.compat.v1.TFRecordReader',
     'tf.TensorInfo': 'tf.compat.v1.TensorInfo',
     'tf.TextLineReader': 'tf.compat.v1.TextLineReader',
@@ -558,6 +560,7 @@ renames = {
     'tf.string_strip': 'tf.strings.strip',
     'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
     'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
+    'tf.summary.SessionLog': 'tf.compat.v1.summary.SessionLog',
     'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
@@ -569,6 +572,7 @@ renames = {
     'tf.summary.text': 'tf.compat.v1.summary.text',
     'tf.svd': 'tf.linalg.svd',
     'tf.tables_initializer': 'tf.compat.v1.tables_initializer',
+    'tf.test.StubOutForTesting': 'tf.compat.v1.test.StubOutForTesting',
     'tf.test.compute_gradient': 'tf.compat.v1.test.compute_gradient',
     'tf.test.compute_gradient_error': 'tf.compat.v1.test.compute_gradient_error',
     'tf.test.get_temp_dir': 'tf.compat.v1.test.get_temp_dir',
-- 
GitLab


From 187596fb9faab1d6c6503dd93b15717d4f8fc209 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 18:09:58 -0800
Subject: [PATCH 1404/1554] Support profiling on iOS

build_ios_universal_lib.sh -p will generate sources that enabled
detailed profiling output.

PiperOrigin-RevId: 224255888
---
 .../lite/tools/benchmark/benchmark_tflite_model.cc  |  2 +-
 tensorflow/lite/tools/benchmark/ios/README.md       | 11 +++++++++++
 tensorflow/lite/tools/make/Makefile                 |  1 +
 .../lite/tools/make/build_ios_universal_lib.sh      | 13 ++++++++++++-
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index 16f70870b6..f342a2bad3 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -30,7 +30,7 @@ limitations under the License.
 #include "tensorflow/lite/tools/benchmark/logging.h"
 
 #ifdef GEMMLOWP_PROFILING
-#include "third_party/gemmlowp/profiling/profiler.h"
+#include "gemmlowp/profiling/profiler.h"
 #endif
 
 #ifdef TFLITE_CUSTOM_OPS_HEADER
diff --git a/tensorflow/lite/tools/benchmark/ios/README.md b/tensorflow/lite/tools/benchmark/ios/README.md
index 3dc29d9b94..fed9e7ea7e 100644
--- a/tensorflow/lite/tools/benchmark/ios/README.md
+++ b/tensorflow/lite/tools/benchmark/ios/README.md
@@ -41,3 +41,14 @@ resources that need to be copied.
 
 - Now try running the app. The app has a single button that runs the benchmark
   on the model and displays results in a text view below.
+
+## Profiling
+
+If you want detailed profiling, use the following command:
+
+```bash
+tensorflow/lite/build_ios_universal_lib.sh -p
+```
+
+Then following the same steps above and run the benchmark app. You will see the
+detailed profiling results in the outputs.
diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index dbcaf4de3c..994f660dba 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -52,6 +52,7 @@ LIBS := \
 # generate things like the protobuf compiler that require that), so all of
 # these settings are for the target compiler.
 CXXFLAGS := -O3 -DNDEBUG
+CXXFLAGS += $(EXTRA_CXXFLAGS)
 CCFLAGS := ${CXXFLAGS}
 CXXFLAGS += --std=c++11
 CFLAGS :=
diff --git a/tensorflow/lite/tools/make/build_ios_universal_lib.sh b/tensorflow/lite/tools/make/build_ios_universal_lib.sh
index 6e0d262827..be8064ec7c 100755
--- a/tensorflow/lite/tools/make/build_ios_universal_lib.sh
+++ b/tensorflow/lite/tools/make/build_ios_universal_lib.sh
@@ -19,11 +19,22 @@ set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR/../../../.."
 
+profiling_args=
+
+while getopts "p" opt; do
+  case $opt in
+    p) profiling_args='-DGEMMLOWP_PROFILING,-DTFLITE_PROFILING_ENABLED';;
+    *) printf "if you want to enable profiling: pass in [-p]\n"
+      exit 2;;
+  esac
+done
+
+shift $(($OPTIND - 1))
 # Build library for supported architectures and packs them in a fat binary.
 make_library() {
     for arch in x86_64 armv7 armv7s arm64
     do
-        make -f tensorflow/lite/tools/make/Makefile TARGET=ios TARGET_ARCH=${arch} \
+        make -f tensorflow/lite/tools/make/Makefile TARGET=ios TARGET_ARCH=${arch} EXTRA_CXXFLAGS=$profiling_args \
         -j 8
     done
     mkdir -p tensorflow/lite/tools/make/gen/lib
-- 
GitLab


From b5ca1e4ab42a1b08770d54ff06c4f6b481e79ff3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 18:20:11 -0800
Subject: [PATCH 1405/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 224256979
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 40 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 40 +++++++++++++++++++
 2 files changed, 80 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 4b3c1c01da..ce71b21507 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -31361,6 +31361,46 @@ op {
     }
   }
 }
+op {
+  name: "Lu"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "lu"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "p"
+    type_attr: "output_idx_type"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+  attr {
+    name: "output_idx_type"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "MakeIterator"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 77d94e4adb..1fa59b7087 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -15790,6 +15790,46 @@ op {
     }
   }
 }
+op {
+  name: "Lu"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "lu"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "p"
+    type_attr: "output_idx_type"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+  attr {
+    name: "output_idx_type"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "MakeIterator"
   input_arg {
-- 
GitLab


From af91e46d7c7a7802213da1ad7c289ef09644ae17 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Wed, 5 Dec 2018 18:26:56 -0800
Subject: [PATCH 1406/1554] Remove partition strategy from
 tf.nn.safe_embedding_lookup_sparse and tf.nn.sampled_softmax_loss; use "div"
 partition strategy by default. We'll add support for different partition
 strategies later, probably in the PartitionedVariable class.

RELNOTES: We will use "div" as the default partition_strategy in tf.nn.safe_embedding_lookup_sparse, tf.nn.sampled_softmax and tf.nn.nce_loss.
PiperOrigin-RevId: 224257710
---
 .../python/kernel_tests/embedding_ops_test.py | 42 ++++----
 tensorflow/python/ops/embedding_ops.py        | 65 ++++++++++++-
 tensorflow/python/ops/nn_impl.py              | 93 +++++++++++++++++-
 tensorflow/python/ops/nn_test.py              | 15 ++-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   |  4 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 96 ++++++++++++-------
 6 files changed, 253 insertions(+), 62 deletions(-)

diff --git a/tensorflow/python/kernel_tests/embedding_ops_test.py b/tensorflow/python/kernel_tests/embedding_ops_test.py
index 6019245d0f..3ea2071e13 100644
--- a/tensorflow/python/kernel_tests/embedding_ops_test.py
+++ b/tensorflow/python/kernel_tests/embedding_ops_test.py
@@ -851,8 +851,9 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights()
       sparse_ids, sparse_weights = self._ids_and_weights_2d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, sparse_weights).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, sparse_weights).eval())
 
       self.assertAllClose(
           embedding_lookup_result,
@@ -865,8 +866,10 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights()
       sparse_ids, sparse_weights = self._ids_and_weights_2d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, sparse_weights, default_id=3).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, sparse_weights,
+              default_id=3).eval())
 
       self.assertAllClose(
           embedding_lookup_result,
@@ -880,8 +883,9 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights()
       sparse_ids, _ = self._ids_and_weights_2d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, None).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, None).eval())
 
       self.assertAllClose(
           embedding_lookup_result,
@@ -895,8 +899,9 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights(num_shards=3)
       sparse_ids, _ = self._ids_and_weights_2d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, None).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, None).eval())
 
       embedding_weights = list(itertools.chain(*embedding_weights))
       self.assertAllClose(embedding_lookup_result,
@@ -926,8 +931,9 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights()
       sparse_ids, sparse_weights = self._ids_and_weights_3d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, sparse_weights).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, sparse_weights).eval())
 
       self.assertAllClose(embedding_lookup_result, [[
           (1.0 * embedding_weights[0][0] + 2.0 * embedding_weights[0][1]) / 3.0,
@@ -940,8 +946,10 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights()
       sparse_ids, sparse_weights = self._ids_and_weights_3d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, sparse_weights, default_id=3).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, sparse_weights,
+              default_id=3).eval())
 
       self.assertAllClose(
           embedding_lookup_result,
@@ -957,8 +965,9 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights()
       sparse_ids, _ = self._ids_and_weights_3d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, None).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, None).eval())
 
       self.assertAllClose(embedding_lookup_result, [[(
           embedding_weights[0][0] + embedding_weights[0][1]) / 2.0, [0] * 4, [
@@ -974,8 +983,9 @@ class SafeEmbeddingLookupSparseTest(test.TestCase):
       embedding_weights = self._random_weights(num_shards=3)
       sparse_ids, _ = self._ids_and_weights_3d()
 
-      embedding_lookup_result = (embedding_ops.safe_embedding_lookup_sparse(
-          embedding_weights, sparse_ids, None).eval())
+      embedding_lookup_result = (
+          embedding_ops.safe_embedding_lookup_sparse_v2(
+              embedding_weights, sparse_ids, None).eval())
 
       embedding_weights = list(itertools.chain(*embedding_weights))
       self.assertAllClose(embedding_lookup_result, [[
diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py
index b398601e6f..f2f5da7d7a 100644
--- a/tensorflow/python/ops/embedding_ops.py
+++ b/tensorflow/python/ops/embedding_ops.py
@@ -491,7 +491,70 @@ def embedding_lookup_sparse(params,
     return embeddings
 
 
-@tf_export("nn.safe_embedding_lookup_sparse")
+@tf_export("nn.safe_embedding_lookup_sparse", v1=[])
+def safe_embedding_lookup_sparse_v2(embedding_weights,
+                                    sparse_ids,
+                                    sparse_weights=None,
+                                    combiner="mean",
+                                    default_id=None,
+                                    max_norm=None,
+                                    name=None):
+  """Lookup embedding results, accounting for invalid IDs and empty features.
+
+  The partitioned embedding in `embedding_weights` must all be the same shape
+  except for the first dimension. The first dimension is allowed to vary as the
+  vocabulary size is not necessarily a multiple of `P`.  `embedding_weights`
+  may be a `PartitionedVariable` as returned by using `tf.get_variable()` with a
+  partitioner.
+
+  Invalid IDs (< 0) are pruned from input IDs and weights, as well as any IDs
+  with non-positive weight. For an entry with no features, the embedding vector
+  for `default_id` is returned, or the 0-vector if `default_id` is not supplied.
+
+  The ids and weights may be multi-dimensional. Embeddings are always aggregated
+  along the last dimension.
+
+  Note: when doing embedding lookup on `embedding_weights`, "div" partition
+  strategy will be used. Support for other partition strategy will be added
+  later.
+
+  Args:
+    embedding_weights:  A list of `P` float `Tensor`s or values representing
+      partitioned embedding `Tensor`s.  Alternatively, a `PartitionedVariable`
+      created by partitioning along dimension 0.  The total unpartitioned shape
+      should be `[e_0, e_1, ..., e_m]`, where `e_0` represents the vocab size
+      and `e_1, ..., e_m` are the embedding dimensions.
+    sparse_ids: `SparseTensor` of shape `[d_0, d_1, ..., d_n]` containing the
+      ids. `d_0` is typically batch size.
+    sparse_weights: `SparseTensor` of same shape as `sparse_ids`, containing
+      float weights corresponding to `sparse_ids`, or `None` if all weights are
+      be assumed to be 1.0.
+    combiner: A string specifying how to combine embedding results for each
+      entry. Currently "mean", "sqrtn" and "sum" are supported, with "mean" the
+      default.
+    default_id: The id to use for an entry with no features.
+    max_norm: If not `None`, all embeddings are l2-normalized to max_norm before
+      combining.
+    name: A name for this operation (optional).
+
+  Returns:
+    Dense `Tensor` of shape `[d_0, d_1, ..., d_{n-1}, e_1, ..., e_m]`.
+
+  Raises:
+    ValueError: if `embedding_weights` is empty.
+  """
+  return safe_embedding_lookup_sparse(
+      embedding_weights,
+      sparse_ids,
+      sparse_weights=sparse_weights,
+      combiner=combiner,
+      default_id=default_id,
+      name=name,
+      partition_strategy="div",
+      max_norm=max_norm)
+
+
+@tf_export(v1=["nn.safe_embedding_lookup_sparse"])
 def safe_embedding_lookup_sparse(embedding_weights,
                                  sparse_ids,
                                  sparse_weights=None,
diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 8f74f831c1..292cca8b9e 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -1677,7 +1677,98 @@ def nce_loss(weights,
   return _sum_rows(sampled_losses)
 
 
-@tf_export("nn.sampled_softmax_loss")
+@tf_export("nn.sampled_softmax_loss", v1=[])
+def sampled_softmax_loss_v2(weights,
+                            biases,
+                            labels,
+                            inputs,
+                            num_sampled,
+                            num_classes,
+                            num_true=1,
+                            sampled_values=None,
+                            remove_accidental_hits=True,
+                            seed=None,
+                            name="sampled_softmax_loss"):
+  """Computes and returns the sampled softmax training loss.
+
+  This is a faster way to train a softmax classifier over a huge number of
+  classes.
+
+  This operation is for training only.  It is generally an underestimate of
+  the full softmax loss.
+
+  A common use case is to use this method for training, and calculate the full
+  sigmoid loss for evaluation or inference as in the following example:
+
+  ```python
+  if mode == "train":
+    loss = tf.nn.sampled_softmax_loss(
+        weights=weights,
+        biases=biases,
+        labels=labels,
+        inputs=inputs,
+        ...)
+  elif mode == "eval":
+    logits = tf.matmul(inputs, tf.transpose(weights))
+    logits = tf.nn.bias_add(logits, biases)
+    labels_one_hot = tf.one_hot(labels, n_classes)
+    loss = tf.nn.softmax_cross_entropy_with_logits_v2(
+        labels=labels_one_hot,
+        logits=logits)
+  ```
+
+  See our [Candidate Sampling Algorithms Reference]
+  (https://www.tensorflow.org/extras/candidate_sampling.pdf)
+
+  Also see Section 3 of [Jean et al., 2014](http://arxiv.org/abs/1412.2007)
+  ([pdf](http://arxiv.org/pdf/1412.2007.pdf)) for the math.
+
+  Note: when doing embedding lookup on `weights` and `bias`, "div" partition
+  strategy will be used. Support for other partition strategy will be added
+  later.
+
+  Args:
+    weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor`
+      objects whose concatenation along dimension 0 has shape [num_classes,
+      dim].  The (possibly-sharded) class embeddings.
+    biases: A `Tensor` of shape `[num_classes]`.  The class biases.
+    labels: A `Tensor` of type `int64` and shape `[batch_size, num_true]`. The
+      target classes.  Note that this format differs from the `labels` argument
+      of `nn.softmax_cross_entropy_with_logits_v2`.
+    inputs: A `Tensor` of shape `[batch_size, dim]`.  The forward activations of
+      the input network.
+    num_sampled: An `int`.  The number of classes to randomly sample per batch.
+    num_classes: An `int`. The number of possible classes.
+    num_true: An `int`.  The number of target classes per training example.
+    sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`,
+      `sampled_expected_count`) returned by a `*_candidate_sampler` function.
+      (if None, we default to `log_uniform_candidate_sampler`)
+    remove_accidental_hits:  A `bool`.  whether to remove "accidental hits"
+      where a sampled class equals one of the target classes.  Default is True.
+    seed: random seed for candidate sampling. Default to None, which doesn't set
+      the op-level random seed for candidate sampling.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `batch_size` 1-D tensor of per-example sampled softmax losses.
+
+  """
+  return sampled_softmax_loss(
+      weights,
+      biases,
+      labels,
+      inputs,
+      num_sampled,
+      num_classes,
+      num_true=num_true,
+      sampled_values=sampled_values,
+      remove_accidental_hits=remove_accidental_hits,
+      partition_strategy="div",
+      name=name,
+      seed=seed)
+
+
+@tf_export(v1=["nn.sampled_softmax_loss"])
 def sampled_softmax_loss(weights,
                          biases,
                          labels,
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 0336d0d27c..414a5cdbbe 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -856,7 +856,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits(
         exp_logits, exp_labels)
 
-    got_sampled_softmax_loss = nn_impl.sampled_softmax_loss(
+    got_sampled_softmax_loss = nn_impl.sampled_softmax_loss_v2(
         weights=constant_op.constant(weights),
         biases=constant_op.constant(biases),
         labels=constant_op.constant(labels, shape=(batch_size, 1)),
@@ -865,8 +865,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
         num_classes=num_classes,
         num_true=1,
         sampled_values=sampled_vals,
-        remove_accidental_hits=False,
-        partition_strategy="div")
+        remove_accidental_hits=False)
 
     self.assertAllClose(exp_sampled_softmax_loss,
                         self.evaluate(got_sampled_softmax_loss), 1e-4)
@@ -874,7 +873,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     # Test with sharded weights and sharded biases.
     weight_shards, bias_shards = self._ShardTestEmbeddings(
         weights, biases, num_shards=3)
-    got_sampled_softmax_loss = nn_impl.sampled_softmax_loss(
+    got_sampled_softmax_loss = nn_impl.sampled_softmax_loss_v2(
         weights=[constant_op.constant(shard) for shard in weight_shards],
         biases=[constant_op.constant(shard) for shard in bias_shards],
         labels=constant_op.constant(labels, shape=(batch_size, 1)),
@@ -883,8 +882,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
         num_classes=num_classes,
         num_true=1,
         sampled_values=sampled_vals,
-        remove_accidental_hits=False,
-        partition_strategy="div")
+        remove_accidental_hits=False)
 
     self.assertAllClose(exp_sampled_softmax_loss,
                         self.evaluate(got_sampled_softmax_loss), 1e-4)
@@ -925,7 +923,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
     sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16)
 
     got_sampled_softmax_loss = math_ops.cast(
-        nn_impl.sampled_softmax_loss(
+        nn_impl.sampled_softmax_loss_v2(
             weights=constant_op.constant(weights, dtype=dtypes.bfloat16),
             biases=constant_op.constant(biases, dtype=dtypes.bfloat16),
             labels=constant_op.constant(
@@ -935,8 +933,7 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
             num_classes=num_classes,
             num_true=1,
             sampled_values=sampled_vals_bf16,
-            remove_accidental_hits=False,
-            partition_strategy="div"), dtypes.float32)
+            remove_accidental_hits=False), dtypes.float32)
 
     self.assertAllClose(exp_sampled_softmax_loss,
                         self.evaluate(got_sampled_softmax_loss), 1e-1)
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 63bf24b5d5..6bf4a9d00c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -234,11 +234,11 @@ tf_module {
   }
   member_method {
     name: "safe_embedding_lookup_sparse"
-    argspec: "args=[\'embedding_weights\', \'sparse_ids\', \'sparse_weights\', \'combiner\', \'default_id\', \'name\', \'partition_strategy\', \'max_norm\'], varargs=None, keywords=None, defaults=[\'None\', \'mean\', \'None\', \'None\', \'div\', \'None\'], "
+    argspec: "args=[\'embedding_weights\', \'sparse_ids\', \'sparse_weights\', \'combiner\', \'default_id\', \'max_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'mean\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "sampled_softmax_loss"
-    argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'partition_strategy\', \'name\', \'seed\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'True\', \'mod\', \'sampled_softmax_loss\', \'None\'], "
+    argspec: "args=[\'weights\', \'biases\', \'labels\', \'inputs\', \'num_sampled\', \'num_classes\', \'num_true\', \'sampled_values\', \'remove_accidental_hits\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'1\', \'None\', \'True\', \'None\', \'sampled_softmax_loss\'], "
   }
   member_method {
     name: "selu"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index edfa2f88c1..f4165c9456 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -605,30 +605,54 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         " they may already have been correct)."
     )
 
+    deprecate_partition_strategy_comment = (
+        "WARNING: `partition_strategy` has been removed from `%s` "
+        " The 'div' strategy is used by default.")
+
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
     self.function_warnings = {
-        "tf.assert_greater": assert_return_type_comment,
-        "tf.assert_equal": assert_return_type_comment,
-        "tf.assert_less": assert_return_type_comment,
-        "tf.assert_rank": assert_rank_comment,
-        "tf.debugging.assert_equal": assert_return_type_comment,
-        "tf.debugging.assert_greater": assert_return_type_comment,
-        "tf.debugging.assert_greater_equal": assert_return_type_comment,
-        "tf.debugging.assert_integer": assert_return_type_comment,
-        "tf.debugging.assert_less": assert_return_type_comment,
-        "tf.debugging.assert_less_equal": assert_return_type_comment,
-        "tf.debugging.assert_near": assert_return_type_comment,
-        "tf.debugging.assert_negative": assert_return_type_comment,
-        "tf.debugging.assert_non_negative": assert_return_type_comment,
-        "tf.debugging.assert_non_positive": assert_return_type_comment,
-        "tf.debugging.assert_none_equal": assert_return_type_comment,
-        "tf.debugging.assert_positive": assert_return_type_comment,
-        "tf.debugging.assert_rank": assert_rank_comment,
-        "tf.debugging.assert_rank_at_least": assert_rank_comment,
-        "tf.debugging.assert_rank_in": assert_rank_comment,
-        "tf.flags": "tf.flags has been removed, please use the argparse or absl"
-                    " module if you need command line parsing.",
+        "tf.assert_greater":
+            assert_return_type_comment,
+        "tf.assert_equal":
+            assert_return_type_comment,
+        "tf.assert_less":
+            assert_return_type_comment,
+        "tf.assert_rank":
+            assert_rank_comment,
+        "tf.debugging.assert_equal":
+            assert_return_type_comment,
+        "tf.debugging.assert_greater":
+            assert_return_type_comment,
+        "tf.debugging.assert_greater_equal":
+            assert_return_type_comment,
+        "tf.debugging.assert_integer":
+            assert_return_type_comment,
+        "tf.debugging.assert_less":
+            assert_return_type_comment,
+        "tf.debugging.assert_less_equal":
+            assert_return_type_comment,
+        "tf.debugging.assert_near":
+            assert_return_type_comment,
+        "tf.debugging.assert_negative":
+            assert_return_type_comment,
+        "tf.debugging.assert_non_negative":
+            assert_return_type_comment,
+        "tf.debugging.assert_non_positive":
+            assert_return_type_comment,
+        "tf.debugging.assert_none_equal":
+            assert_return_type_comment,
+        "tf.debugging.assert_positive":
+            assert_return_type_comment,
+        "tf.debugging.assert_rank":
+            assert_rank_comment,
+        "tf.debugging.assert_rank_at_least":
+            assert_rank_comment,
+        "tf.debugging.assert_rank_in":
+            assert_rank_comment,
+        "tf.flags":
+            "tf.flags has been removed, please use the argparse or absl"
+            " module if you need command line parsing.",
         "tf.train.exponential_decay":
             decay_function_comment,
         "tf.train.piecewise_constant_decay":
@@ -664,23 +688,29 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.estimator.BaselineRegressor":
             default_loss_reduction_changed,
         "tf.nn.conv1d":
-        "WARNING: use_cudnn_on_gpu argument has been removed and \"value\" was "
-        "renamed to \"input\"",
+            "WARNING: use_cudnn_on_gpu argument has been removed and \"value\""
+            " was renamed to \"input\"",
         "tf.nn.conv2d":
-        "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" "
-        "was renamed to \"filters\"",
+            "WARNING: use_cudnn_on_gpu argument has been removed and "
+            "\"filter\" was renamed to \"filters\"",
         "tf.nn.conv2d_backprop_filter":
-        "WARNING: use_cudnn_on_gpu argument has been removed",
+            "WARNING: use_cudnn_on_gpu argument has been removed",
         "tf.nn.conv2d_backprop_input":
-        "WARNING: use_cudnn_on_gpu argument has been removed and \"filter\" "
-        "was renamed to \"filters\"",
+            "WARNING: use_cudnn_on_gpu argument has been removed and "
+            "\"filter\" was renamed to \"filters\"",
         "tf.nn.erosion2d":
-        "WARNING: <function name> now requires a data_format argument",
+            "WARNING: <function name> now requires a data_format argument",
         "tf.nn.nce_loss":
-        "WARNING: `partition_strategy` has been removed from `tf.nn.nce_loss` "
-        " The 'div' strategy is used by default.",
-        "tf.zeros_like": tf_01s_like_no_optimize_comment,
-        "tf.ones_like": tf_01s_like_no_optimize_comment,
+            deprecate_partition_strategy_comment % "tf.nn.nce_loss",
+        "tf.nn.safe_embedding_lookup_sparse":
+            deprecate_partition_strategy_comment %
+            "tf.nn.safe_embedding_lookup_sparse",
+        "tf.nn.sampled_softmax_loss":
+            deprecate_partition_strategy_comment % "tf.nn.sampled_softmax_loss",
+        "tf.zeros_like":
+            tf_01s_like_no_optimize_comment,
+        "tf.ones_like":
+            tf_01s_like_no_optimize_comment,
     }
 
     self.symbol_renames = {
-- 
GitLab


From 2d6054e0450ddb2c12f1eb5b69f6fac926406083 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 5 Dec 2018 18:35:09 -0800
Subject: [PATCH 1407/1554] [TF:XLA] Bump open source abseil revision to
 284378a71b32dfb3af4e3661f585e671d1b603a3

PiperOrigin-RevId: 224258608
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 1f86dc328a..3fd55cf24a 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "7bb63336f2522ce893c34ec241e6ccdfdece56b1a1f43eb3a732712dbd426f4f",
-        strip_prefix = "abseil-cpp-44b0fafc62d9b8f192e8180cbe9c4b806b339d57",
+        sha256 = "f7e3734c3c5854cf2bae1e193fbbd8f7d261673e0f2e042b1fca52732f688a0a",
+        strip_prefix = "abseil-cpp-284378a71b32dfb3af4e3661f585e671d1b603a3",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/44b0fafc62d9b8f192e8180cbe9c4b806b339d57.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/44b0fafc62d9b8f192e8180cbe9c4b806b339d57.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/284378a71b32dfb3af4e3661f585e671d1b603a3.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/284378a71b32dfb3af4e3661f585e671d1b603a3.tar.gz",
         ],
     )
 
-- 
GitLab


From 369d30289c4ad1b69f6d04227b9a6a9ebdf349d0 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Wed, 5 Dec 2018 18:39:01 -0800
Subject: [PATCH 1408/1554] Allow toggling optional features in @convert.

PiperOrigin-RevId: 224259120
---
 tensorflow/python/autograph/impl/api.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 19a472064a..f7774888c8 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -49,7 +49,10 @@ from tensorflow.python.util import tf_inspect
 # TODO(mdan): This should behave like to_graph (e.g. convert statically).
 # TODO(znado): Make an alias so can write Verbosity directly without needing
 # to write converter.
-def convert(recursive=False, verbose=converter.Verbosity.VERBOSE):
+def convert(
+    recursive=False,
+    verbose=converter.Verbosity.BRIEF,
+    optional_features=converter.Feature.ALL):
   """Decorator that compiles a function to use TensorFlow ops.
 
   The decorator is dynamic - it recompiles the target whenever the decorated
@@ -61,6 +64,9 @@ def convert(recursive=False, verbose=converter.Verbosity.VERBOSE):
     recursive: bool, whether to recursively convert any functions or classes
       that the converted function may use.
     verbose: converter.Verbosity, the level of verbosity.
+    optional_features: converted.Feature, allows toggling optional or
+      experimental features. When set to None, only the core features are
+      enabled.
 
   Returns:
     Callable, a decorator that converts the given function into an equivalent
@@ -78,7 +84,7 @@ def convert(recursive=False, verbose=converter.Verbosity.VERBOSE):
               recursive=recursive,
               verbose=verbose,
               force_conversion=True,
-              optional_features=converter.Feature.ALL,
+              optional_features=optional_features,
           ), *args, **kwargs)
 
     wrapper = tf_decorator.make_decorator(f, wrapper)
-- 
GitLab


From 243717eed45342bea3aafaea34ca2757594cc2de Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 5 Dec 2018 18:50:17 -0800
Subject: [PATCH 1409/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 224260186

---
 tensorflow/go/op/wrappers.go | 150 +++++++++++++++++------------------
 1 file changed, 75 insertions(+), 75 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index d0ea6a4ce1..440756bb38 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5787,6 +5787,81 @@ func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataTyp
 	return values
 }
 
+// MapStageAttr is an optional argument to MapStage.
+type MapStageAttr func(optionalAttr)
+
+// MapStageCapacity sets the optional capacity attribute to value.
+//
+// value: Maximum number of elements in the Staging Area. If > 0, inserts
+// on the container will block when the capacity is reached.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapStageCapacity(value int64) MapStageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapStageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapStageMemoryLimit(value int64) MapStageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapStageContainer sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container. Otherwise,
+// a default container is used.
+// If not specified, defaults to ""
+func MapStageContainer(value string) MapStageAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapStageSharedName sets the optional shared_name attribute to value.
+//
+// value: It is necessary to match this name to the matching Unstage Op.
+// If not specified, defaults to ""
+func MapStageSharedName(value string) MapStageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Stage (key, values) in the underlying container which behaves like a hashtable.
+//
+// Arguments:
+//	key: int64
+//
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+//
+// Returns the created operation.
+func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...MapStageAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapStage",
+		Input: []tf.Input{
+			key, indices, tf.OutputList(values),
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -33595,78 +33670,3 @@ func StagePeek(scope *Scope, index tf.Output, dtypes []tf.DataType, optional ...
 	}
 	return values
 }
-
-// MapStageAttr is an optional argument to MapStage.
-type MapStageAttr func(optionalAttr)
-
-// MapStageCapacity sets the optional capacity attribute to value.
-//
-// value: Maximum number of elements in the Staging Area. If > 0, inserts
-// on the container will block when the capacity is reached.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapStageCapacity(value int64) MapStageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapStageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapStageMemoryLimit(value int64) MapStageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapStageContainer sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container. Otherwise,
-// a default container is used.
-// If not specified, defaults to ""
-func MapStageContainer(value string) MapStageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapStageSharedName sets the optional shared_name attribute to value.
-//
-// value: It is necessary to match this name to the matching Unstage Op.
-// If not specified, defaults to ""
-func MapStageSharedName(value string) MapStageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Stage (key, values) in the underlying container which behaves like a hashtable.
-//
-// Arguments:
-//	key: int64
-//
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
-//
-//
-// Returns the created operation.
-func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...MapStageAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapStage",
-		Input: []tf.Input{
-			key, indices, tf.OutputList(values),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-- 
GitLab


From 7165d2bcec13e18760aa68f3295fb6795fc995bc Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 5 Dec 2018 19:08:36 -0800
Subject: [PATCH 1410/1554] Fix a bug when input shapes are not specified.

PiperOrigin-RevId: 224261975
---
 tensorflow/lite/tools/benchmark/BUILD         |  1 +
 .../lite/tools/benchmark/benchmark_test.cc    | 41 +++++++++++++++++++
 .../tools/benchmark/benchmark_tflite_model.cc | 15 +++++--
 3 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index 583046ad73..bc47406cd9 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -70,6 +70,7 @@ cc_test(
     deps = [
         ":benchmark_tflite_model_lib",
         ":command_line_flags",
+        "//tensorflow/lite:framework",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/tools/benchmark/benchmark_test.cc b/tensorflow/lite/tools/benchmark/benchmark_test.cc
index 8191fbcd73..0cf66c6e4d 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_test.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
 #include "tensorflow/lite/tools/benchmark/command_line_flags.h"
@@ -47,6 +48,15 @@ BenchmarkParams CreateParams() {
   return params;
 }
 
+class TestBenchmark : public BenchmarkTfLiteModel {
+ public:
+  explicit TestBenchmark(BenchmarkParams params)
+      : BenchmarkTfLiteModel(std::move(params)) {}
+  const tflite::Interpreter* GetInterpreter() { return interpreter.get(); }
+
+  void Prepare() { PrepareInputsAndOutputs(); }
+};
+
 TEST(BenchmarkTest, DoesntCrash) {
   ASSERT_THAT(g_model_path, testing::NotNull());
 
@@ -54,6 +64,37 @@ TEST(BenchmarkTest, DoesntCrash) {
   benchmark.Run();
 }
 
+TEST(BenchmarkTest, ParametersArePopulatedWhenInputShapeIsNotSpecified) {
+  ASSERT_THAT(g_model_path, testing::NotNull());
+
+  TestBenchmark benchmark(CreateParams());
+  benchmark.Init();
+  benchmark.Prepare();
+
+  auto interpreter = benchmark.GetInterpreter();
+  auto inputs = interpreter->inputs();
+  ASSERT_GE(inputs.size(), 1);
+  auto input_tensor = interpreter->tensor(inputs[0]);
+
+  std::vector<uint8_t> input_bytes;
+  input_bytes.reserve(input_tensor->bytes);
+  for (size_t i = 0; i < input_tensor->bytes; i++) {
+    input_bytes.push_back(input_tensor->data.b[i]);
+  }
+  benchmark.Prepare();
+
+  // Expect data is not the same.
+  EXPECT_EQ(input_bytes.size(), input_tensor->bytes);
+  bool is_same = true;
+  for (size_t i = 0; i < input_tensor->bytes; i++) {
+    if (input_bytes[i] != input_tensor->data.b[i]) {
+      is_same = false;
+      break;
+    }
+  }
+  EXPECT_FALSE(is_same);
+}
+
 }  // namespace
 }  // namespace benchmark
 }  // namespace tflite
diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index f342a2bad3..ad025b19d9 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -181,6 +181,15 @@ bool PopulateInputLayerInfo(
   return true;
 }
 
+std::vector<int> TfLiteIntArrayToVector(const TfLiteIntArray* int_array) {
+  std::vector<int> values;
+  values.reserve(int_array->size);
+  for (size_t i = 0; i < int_array->size; i++) {
+    values.push_back(int_array->data[i]);
+  }
+  return values;
+}
+
 }  // namespace
 
 BenchmarkParams BenchmarkTfLiteModel::DefaultParams() {
@@ -250,12 +259,10 @@ uint64_t BenchmarkTfLiteModel::ComputeInputBytes() {
 void BenchmarkTfLiteModel::PrepareInputsAndOutputs() {
   auto interpreter_inputs = interpreter->inputs();
   // Set the values of the input tensors.
-  for (int j = 0; j < inputs.size(); ++j) {
-    const InputLayerInfo& input = inputs[j];
+  for (int j = 0; j < interpreter_inputs.size(); ++j) {
     int i = interpreter_inputs[j];
     TfLiteTensor* t = interpreter->tensor(i);
-    std::vector<int> sizes = input.shape;
-
+    std::vector<int> sizes = TfLiteIntArrayToVector(t->dims);
     // TODO(ahentz): below we ignore the O-th dimension (number of batches).
     if (t->type == kTfLiteFloat32) {
       FillRandomValue<float>(
-- 
GitLab


From 21e223fb76cef1bcaedfcd1eac44df646da66abb Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Wed, 5 Dec 2018 21:10:56 -0800
Subject: [PATCH 1411/1554] Disabling flaky
 //tensorflow/python/keras:training_test

PiperOrigin-RevId: 224271996
---
 tensorflow/python/keras/BUILD | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 602cf79362..36fea36389 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -764,7 +764,12 @@ py_test(
     srcs = ["engine/training_test.py"],
     shard_count = 16,
     srcs_version = "PY2AND3",
-    tags = ["notsan"],
+    tags = [
+        "manual",  # TODO(b/120560388)
+        "no_oss",  # TODO(b/120560388)
+        "notap",  # TODO(b/120560388)
+        "notsan",
+    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From eaf2b9cc3edaf16855dc26f1c8947631a1680992 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 00:17:52 -0800
Subject: [PATCH 1412/1554] Move third_party to opensource_only.files.

PiperOrigin-RevId: 224286427
---
 tensorflow/opensource_only.files              | 193 +++++++++++++++++-
 .../opensource_only/arm_compiler.BUILD        |   0
 third_party/mpi/mpi.bzl                       |  10 +-
 3 files changed, 197 insertions(+), 6 deletions(-)
 rename arm_compiler.BUILD => tensorflow/opensource_only/arm_compiler.BUILD (100%)

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 8ff1645b98..0c29ac6a30 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,3 +1,14 @@
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/third_party/toolchains/clang6/README.md
+tensorflow/third_party/toolchains/clang6/repo.bzl
+tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
@@ -14,4 +25,184 @@ tensorflow/third_party/toolchains/preconfig/generate/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
\ No newline at end of file
+tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
+tensorflow/third_party/toolchains/gpus/py/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/cpus/arm/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/py/BUILD
+tensorflow/third_party/toolchains/BUILD
+tensorflow/third_party/nccl/remote.BUILD.tpl
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
+tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
+tensorflow/third_party/gpus/crosstool/LICENSE
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.bat.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
+tensorflow/third_party/gpus/crosstool/BUILD
+tensorflow/third_party/gpus/cuda/LICENSE
+tensorflow/third_party/gpus/cuda/BUILD.tpl
+tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
+tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
+tensorflow/third_party/gpus/cuda/BUILD
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
+tensorflow/third_party/gpus/cuda_configure.bzl
+tensorflow/third_party/gpus/rocm_configure.bzl
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/farmhash.BUILD
+tensorflow/third_party/eigen3/Eigen/Cholesky
+tensorflow/third_party/eigen3/Eigen/QR
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Eigenvalues
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
+tensorflow/third_party/eigen3/LICENSE
+tensorflow/third_party/eigen3/BUILD
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/fft2d/BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/tflite_smartreply.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/remote.BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/eigen.BUILD
\ No newline at end of file
diff --git a/arm_compiler.BUILD b/tensorflow/opensource_only/arm_compiler.BUILD
similarity index 100%
rename from arm_compiler.BUILD
rename to tensorflow/opensource_only/arm_compiler.BUILD
diff --git a/third_party/mpi/mpi.bzl b/third_party/mpi/mpi.bzl
index 38ce91c4d0..3a483351d1 100644
--- a/third_party/mpi/mpi.bzl
+++ b/third_party/mpi/mpi.bzl
@@ -2,16 +2,16 @@
 #based on the configuration options return one or the other
 
 def mpi_hdr():
-    MPI_LIB_IS_OPENMPI=True
-    hdrs = []    
+    MPI_LIB_IS_OPENMPI = True
+    hdrs = []
     if MPI_LIB_IS_OPENMPI:
-        hdrs = ["mpi.h", "mpi_portable_platform.h"]   #When using OpenMPI
+        hdrs = ["mpi.h", "mpi_portable_platform.h"]  #When using OpenMPI
     else:
-        hdrs = ["mpi.h",  "mpio.h", "mpicxx.h"]        #When using MVAPICH
+        hdrs = ["mpi.h", "mpio.h", "mpicxx.h"]  #When using MVAPICH
     return hdrs
 
 def if_mpi(if_true, if_false = []):
     return select({
         "//tensorflow:with_mpi_support": if_true,
-        "//conditions:default": if_false
+        "//conditions:default": if_false,
     })
-- 
GitLab


From a34ba5855ea8ddcceeac5f70ff4c7693cd9ca411 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 01:02:13 -0800
Subject: [PATCH 1413/1554] compat: Update forward compatibility horizon to
 2018-12-06

PiperOrigin-RevId: 224290531
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index dc1a58a95a..269d2e286a 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 5)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 6)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 2c382ef260e75b399541690b9d53fc96a87fb955 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 03:29:19 -0800
Subject: [PATCH 1414/1554] Changes in TrackableAssets in experimental saved
 object graph.

 - Support exporting multiple TrackableAssets that refer to same filename.
 - Make SavedAsset proto depend on an external list of asset file defs that
 comes from a MetaGraphDef.

Note: This means that the MetaGraphDef supporting the object-based saved model view
must list all asset filedefs, even the ones that are not needed for exported signature
defs. As a consequence the exported meta graph must also have a feedable tensor
for each one of them, so old loaders do not fail when trying to digest the AssetFileDefs.
PiperOrigin-RevId: 224306775
---
 tensorflow/python/saved_model/load.py         | 15 +++--
 tensorflow/python/saved_model/load_test.py    | 17 +++++
 tensorflow/python/saved_model/loader_impl.py  |  4 +-
 tensorflow/python/saved_model/save.py         | 65 ++++++++++---------
 tensorflow/python/saved_model/save_test.py    | 13 ++++
 .../python/saved_model/saved_model_test.py    |  4 +-
 .../saved_model/saved_object_graph.proto      |  7 +-
 7 files changed, 81 insertions(+), 44 deletions(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index f3caa8798a..e3095f4ee5 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -22,6 +22,7 @@ import os
 
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.saved_model import constants
+from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import tracking
@@ -31,8 +32,9 @@ from tensorflow.python.util import compat
 class _Loader(object):
   """Helper class to load an object-based SavedModel."""
 
-  def __init__(self, proto, export_dir):
-    self._proto = proto
+  def __init__(self, object_graph_proto, saved_model_proto, export_dir):
+    self._asset_file_def = saved_model_proto.meta_graphs[0].asset_file_def
+    self._proto = object_graph_proto
     self._export_dir = export_dir
     self._load_all()
 
@@ -63,7 +65,7 @@ class _Loader(object):
   def _recreate_asset(self, proto):
     filename = os.path.join(
         saved_model_utils.get_assets_dir(self._export_dir),
-        proto.relative_filename)
+        self._asset_file_def[proto.asset_file_def_index].filename)
     return tracking.TrackableAsset(filename)
 
 
@@ -75,13 +77,16 @@ def _load_saved_object_graph_proto(filename):
 
 def load(export_dir):
   """Load a SavedModel from `export_dir`."""
+  saved_model_proto = loader_impl.parse_saved_model(export_dir)
   object_graph_filename = os.path.join(
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.EXTRA_ASSETS_DIRECTORY),
       compat.as_bytes("object_graph.pb"))
   if file_io.file_exists(object_graph_filename):
-    proto = _load_saved_object_graph_proto(object_graph_filename)
-    loader = _Loader(proto, export_dir)
+    object_graph_proto = _load_saved_object_graph_proto(object_graph_filename)
+    loader = _Loader(object_graph_proto,
+                     saved_model_proto,
+                     export_dir)
     root = loader.get(0)
   else:
     raise NotImplementedError(
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index bccbee6d53..a2971101cd 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -79,6 +79,23 @@ class LoadTest(test.TestCase):
     with open(imported.asset2.asset_path.numpy(), "r") as f:
       self.assertEquals("contents 2", f.read())
 
+  def test_assets_dedup(self):
+    vocab = self._make_asset("contents")
+    root = tracking.Checkpointable()
+    root.f = def_function.function(
+        lambda x: 2. * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+
+    root.asset1 = tracking.TrackableAsset(vocab)
+    root.asset2 = tracking.TrackableAsset(vocab)
+
+    export_dir = os.path.join(self.get_temp_dir(), "save_dir")
+    save.save(root, export_dir)
+    imported = load.load(export_dir)
+
+    self.assertEqual(imported.asset1.asset_path.numpy(),
+                     imported.asset2.asset_path.numpy())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index 6bf39a2c67..2717e5b915 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -39,7 +39,7 @@ from tensorflow.python.util import deprecation
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _parse_saved_model(export_dir):
+def parse_saved_model(export_dir):
   """Reads the savedmodel.pb or savedmodel.pbtxt file containing `SavedModel`.
 
   Args:
@@ -276,7 +276,7 @@ class SavedModelLoader(object):
     """
     self._export_dir = export_dir
     self._variables_path = saved_model_utils.get_variables_path(export_dir)
-    self._saved_model = _parse_saved_model(export_dir)
+    self._saved_model = parse_saved_model(export_dir)
 
   @property
   def export_dir(self):
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 422d68962a..ab6fcb7196 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -375,7 +375,9 @@ _AssetInfo = collections.namedtuple(
         # Map from asset variable resource Tensors to their init ops
         "asset_initializers_by_resource",
         # Map from base asset filenames to full paths
-        "asset_filename_map"])
+        "asset_filename_map",
+        # Map from TrackableAsset to index of corresponding AssetFileDef
+        "asset_index"])
 
 
 def _process_asset(trackable_asset, asset_info, resource_map):
@@ -386,21 +388,23 @@ def _process_asset(trackable_asset, asset_info, resource_map):
   path = builder_impl.get_asset_filename_to_add(
       asset_filepath=original_path,
       asset_filename_map=asset_info.asset_filename_map)
-  asset_variable = asset_info.asset_filename_map.get(path, None)
-  if asset_variable is None:
-    asset_path_initializer = array_ops.placeholder(
-        shape=original_variable.shape,
-        dtype=dtypes.string,
-        name="asset_path_initializer")
-    asset_variable = resource_variable_ops.ResourceVariable(
-        asset_path_initializer)
-    asset_info.asset_filename_map[path] = original_path
-    asset_def = meta_graph_pb2.AssetFileDef()
-    asset_def.filename = path
-    asset_def.tensor_info.name = asset_path_initializer.name
-    asset_info.asset_defs.append(asset_def)
-    asset_info.asset_initializers_by_resource[original_variable.handle] = (
-        asset_variable.initializer)
+  # TODO(andresp): Instead of mapping 1-1 between trackable asset
+  # and asset in the graph def consider deduping the assets that
+  # point to the same file.
+  asset_path_initializer = array_ops.placeholder(
+      shape=original_variable.shape,
+      dtype=dtypes.string,
+      name="asset_path_initializer")
+  asset_variable = resource_variable_ops.ResourceVariable(
+      asset_path_initializer)
+  asset_info.asset_filename_map[path] = original_path
+  asset_def = meta_graph_pb2.AssetFileDef()
+  asset_def.filename = path
+  asset_def.tensor_info.name = asset_path_initializer.name
+  asset_info.asset_defs.append(asset_def)
+  asset_info.asset_initializers_by_resource[original_variable.handle] = (
+      asset_variable.initializer)
+  asset_info.asset_index[trackable_asset] = len(asset_info.asset_defs) - 1
   resource_map[original_variable.handle] = asset_variable.handle
 
 
@@ -432,7 +436,8 @@ def _map_resources(accessible_objects):
   asset_info = _AssetInfo(
       asset_defs=[],
       asset_initializers_by_resource={},
-      asset_filename_map={})
+      asset_filename_map={},
+      asset_index={})
   for obj in accessible_objects:
     if isinstance(obj, tracking.TrackableResource):
       new_resource = obj.create_resource()
@@ -441,7 +446,7 @@ def _map_resources(accessible_objects):
       new_variable = resource_variable_ops.copy_to_graph_uninitialized(obj)
       object_map[obj] = new_variable
       resource_map[obj.handle] = new_variable.handle
-    if isinstance(obj, tracking.TrackableAsset):
+    elif isinstance(obj, tracking.TrackableAsset):
       _process_asset(obj, asset_info, resource_map)
   return object_map, resource_map, asset_info
 
@@ -458,9 +463,7 @@ def _fill_meta_graph_def(meta_graph_def, obj, signature_functions,
     object_saver: A CheckpointableSaver to add to the MetaGraph.
 
   Returns:
-    asset_filename_map, a dictionary mapping from asset base names to
-    user-specified full asset paths, which should be copied to the SavedModel's
-    assets/ directory.
+    An _AssetInfo, which contains information to help creating the SavedModel.
   """
   signatures = {}
   # List objects from the eager context to make sure Optimizers give us the
@@ -514,10 +517,10 @@ def _fill_meta_graph_def(meta_graph_def, obj, signature_functions,
   for signature_key, signature in signatures.items():
     meta_graph_def.signature_def[signature_key].CopyFrom(signature)
   meta_graph.strip_graph_default_valued_attrs(meta_graph_def)
-  return asset_info.asset_filename_map
+  return asset_info
 
 
-def _write_object_graph(root, export_dir, asset_filename_map):
+def _write_object_graph(root, export_dir, asset_file_def_index):
   """Save a SavedObjectGraph proto for `root`."""
   # SavedObjectGraph is similar to the CheckpointableObjectGraph proto in the
   # checkpoint. It will eventually go into the SavedModel.
@@ -527,11 +530,8 @@ def _write_object_graph(root, export_dir, asset_filename_map):
   util.fill_object_graph_proto(checkpointable_objects, node_ids, slot_variables,
                                proto)
 
-  # Build map from original filename to relative asset filename.
-  filename_map = {v: k for k, v in asset_filename_map.items()}
-
   for obj, obj_proto in zip(checkpointable_objects, proto.nodes):
-    _write_object_proto(obj, obj_proto, filename_map)
+    _write_object_proto(obj, obj_proto, asset_file_def_index)
 
   extra_asset_dir = os.path.join(
       compat.as_bytes(export_dir),
@@ -542,11 +542,11 @@ def _write_object_graph(root, export_dir, asset_filename_map):
   file_io.write_string_to_file(object_graph_filename, proto.SerializeToString())
 
 
-def _write_object_proto(obj, proto, filename_map):
+def _write_object_proto(obj, proto, asset_file_def_index):
   """Saves an object into SavedObject proto."""
   if isinstance(obj, tracking.TrackableAsset):
     proto.asset.SetInParent()
-    proto.asset.relative_filename = filename_map[obj.asset_path.numpy()]
+    proto.asset.asset_file_def_index = asset_file_def_index[obj]
   else:
     proto.user_object.SetInParent()
 
@@ -716,7 +716,7 @@ def save(obj, export_dir, signatures=None):
   saved_model = saved_model_pb2.SavedModel()
   meta_graph_def = saved_model.meta_graphs.add()
   object_saver = util.CheckpointableSaver(obj)
-  asset_filename_map = _fill_meta_graph_def(
+  asset_info = _fill_meta_graph_def(
       meta_graph_def, obj, signatures, object_saver)
   saved_model.saved_model_schema_version = (
       constants.SAVED_MODEL_SCHEMA_VERSION)
@@ -725,9 +725,10 @@ def save(obj, export_dir, signatures=None):
   # SavedModel proto itself.
   utils_impl.get_or_create_variables_dir(export_dir)
   object_saver.save(utils_impl.get_variables_path(export_dir))
-  builder_impl.copy_assets_to_destination_dir(asset_filename_map, export_dir)
+  builder_impl.copy_assets_to_destination_dir(asset_info.asset_filename_map,
+                                              export_dir)
   path = os.path.join(
       compat.as_bytes(export_dir),
       compat.as_bytes(constants.SAVED_MODEL_FILENAME_PB))
   file_io.write_string_to_file(path, saved_model.SerializeToString())
-  _write_object_graph(obj, export_dir, asset_filename_map)
+  _write_object_graph(obj, export_dir, asset_info.asset_index)
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 5096e9c062..668a374415 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -353,6 +353,19 @@ class AssetTests(test.TestCase):
         {"output_0": [2, 1]},
         _import_and_infer(second_dir, {"keys": ["gamma", "beta"]}))
 
+  def test_unused_asset(self):
+    root = tracking.Checkpointable()
+    root.f = def_function.function(
+        lambda x: 2. * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+    root.asset = tracking.TrackableAsset(self._vocab_path)
+
+    export_dir = os.path.join(self.get_temp_dir(), "save_dir")
+    save.save(root, export_dir)
+    self.assertAllClose(
+        {"output_0": [0.2]},
+        _import_and_infer(export_dir, {"x": [0.1]}))
+
 
 class MemoryTests(test.TestCase):
 
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index 0f18fb1a01..d4c8d72011 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -1460,10 +1460,8 @@ class SavedModelV1Test(SavedModelTestBase):
     self.assertIn("Tout", complex_node.attr)
 
     # Load graph "foo" from disk as-is to verify default attrs are stripped.
-    # pylint: disable=protected-access
-    saved_model_pb = loader_impl._parse_saved_model(export_dir)
+    saved_model_pb = loader_impl.parse_saved_model(export_dir)
     self.assertIsNotNone(saved_model_pb)
-    # pylint: enable=protected-access
 
     meta_graph_foo_def = None
     meta_graph_bar_def = None
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 1111c24ebe..3991fbede4 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -65,6 +65,9 @@ message SavedUserObject {}
 // absolute filename can be read. Users should not expect the filename to be
 // maintained.
 message SavedAsset {
-  // Filename relative to the assets directory of the SavedModel.
-  string relative_filename = 1;
+  // Index into `MetaGraphDef.asset_file_def[]` that describes the Asset.
+  //
+  // Only the field `AssetFileDef.filename` is used. Other fields, such as
+  // `AssetFileDef.tensor_info`, MUST be ignored.
+  uint32 asset_file_def_index = 1;
 }
-- 
GitLab


From 60a23f7ee0b54aa6d2a99607ec927b6e0e687e5c Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 6 Dec 2018 05:34:20 -0800
Subject: [PATCH 1415/1554] Refactor the context override logic to fix some of
 its inconsistencies.

PiperOrigin-RevId: 224320386
---
 tensorflow/python/autograph/pyct/templates.py | 157 +++++++-----------
 .../python/autograph/pyct/templates_test.py   |   7 +-
 2 files changed, 66 insertions(+), 98 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/templates.py b/tensorflow/python/autograph/pyct/templates.py
index 9901295445..2272ea4208 100644
--- a/tensorflow/python/autograph/pyct/templates.py
+++ b/tensorflow/python/autograph/pyct/templates.py
@@ -32,6 +32,66 @@ from tensorflow.python.autograph.pyct import parser
 from tensorflow.python.autograph.pyct import qual_names
 
 
+class ContextAdjuster(gast.NodeTransformer):
+  """Adjusts the ctx field of nodes to ensure consistency.
+
+  This transformer can change the ctx fields of a variable, tuple and other
+  AST elements that allow one, based on whether the element is being read or
+  written.
+  """
+
+  def __init__(self, override_value):
+    self._ctx_override = override_value
+
+  def visit(self, node):
+    original_override = self._ctx_override
+    node = super(ContextAdjuster, self).visit(node)
+    if hasattr(node, 'ctx'):
+      assert node.ctx is not None, 'node {} has ctx unset'.format(node)
+    self._ctx_override = original_override
+    return node
+
+  def _apply_override(self, node):
+    if self._ctx_override is not None:
+      node.ctx = self._ctx_override()
+
+  def visit_Attribute(self, node):
+    self._apply_override(node)
+    self._ctx_override = gast.Load
+    node = self.generic_visit(node)
+    return node
+
+  def visit_Tuple(self, node):
+    self._apply_override(node)
+    return self.generic_visit(node)
+
+  def visit_List(self, node):
+    self._apply_override(node)
+    return self.generic_visit(node)
+
+  def visit_Name(self, node):
+    self._apply_override(node)
+    return self.generic_visit(node)
+
+  def visit_Call(self, node):
+    self._apply_override(node)
+    # We may be able to override these to Load(), but for now it's simpler
+    # to just assert that they're set.
+    self._ctx_override = None
+    return self.generic_visit(node)
+
+  def visit_Dict(self, node):
+    # We may be able to override these to Load(), but for now it's simpler
+    # to just assert that they're set.
+    self._ctx_override = None
+    return self.generic_visit(node)
+
+  def visit_Subscript(self, node):
+    node.value = self.visit(node.value)
+    self._ctx_override = None
+    return self.generic_visit(node)
+
+
 class ReplaceTransformer(gast.NodeTransformer):
   """Replace AST nodes."""
 
@@ -106,91 +166,6 @@ class ReplaceTransformer(gast.NodeTransformer):
     node.name = repl.id
     return node
 
-  def _check_has_context(self, node):
-    if not node.ctx:
-      raise ValueError('node %s is missing ctx value' % node)
-
-  # TODO(mdan): Rewrite _check and _set using a separate transformer.
-  def _check_inner_children_have_context(self, node):
-    if isinstance(node, gast.Attribute):
-      self._check_inner_children_have_context(node.value)
-      self._check_has_context(node)
-    elif isinstance(node, (gast.Tuple, gast.List)):
-      for e in node.elts:
-        self._check_inner_children_have_context(e)
-      self._check_has_context(node)
-    elif isinstance(node, gast.Dict):
-      for e in node.keys:
-        self._check_inner_children_have_context(e)
-      for e in node.values:
-        self._check_inner_children_have_context(e)
-    elif isinstance(node, gast.Index):
-      self._check_inner_children_have_context(node.value)
-    elif isinstance(node, gast.Subscript):
-      self._check_inner_children_have_context(node.value)
-      self._check_inner_children_have_context(node.slice)
-    elif isinstance(node, gast.Slice):
-      self._check_inner_children_have_context(node.lower)
-      if node.upper:
-        self._check_inner_children_have_context(node.upper)
-      if node.step:
-        self._check_inner_children_have_context(node.step)
-    elif isinstance(node, gast.BinOp):
-      self._check_inner_children_have_context(node.left)
-      self._check_inner_children_have_context(node.right)
-    elif isinstance(node, gast.UnaryOp):
-      self._check_inner_children_have_context(node.operand)
-    elif isinstance(node, gast.Name):
-      self._check_has_context(node)
-    elif isinstance(node, (gast.Str, gast.Num)):
-      pass
-    elif isinstance(node, gast.Call):
-      self._check_inner_children_have_context(node.func)
-      for a in node.args:
-        self._check_inner_children_have_context(a)
-      for k in node.keywords:
-        self._check_inner_children_have_context(k.value)
-    else:
-      raise ValueError('unexpected node type "%s"' % node)
-
-  def _set_inner_child_context(self, node, ctx):
-    if isinstance(node, gast.Attribute):
-      self._set_inner_child_context(node.value, gast.Load())
-      node.ctx = ctx
-    elif isinstance(node, (gast.Tuple, gast.List)):
-      for e in node.elts:
-        self._set_inner_child_context(e, ctx)
-      node.ctx = ctx
-    elif isinstance(node, gast.Name):
-      node.ctx = ctx
-    elif isinstance(node, gast.Call):
-      self._set_inner_child_context(node.func, ctx)
-      # We may be able to override these to Load(), but for now it's simpler
-      # to just assert that they're set.
-      for a in node.args:
-        self._check_inner_children_have_context(a)
-      for k in node.keywords:
-        self._check_inner_children_have_context(k.value)
-    elif isinstance(node, gast.Dict):
-      # We may be able to override these to Load(), but for now it's simpler
-      # to just assert that they're set.
-      for e in node.keys:
-        self._check_inner_children_have_context(e)
-      for e in node.values:
-        self._check_inner_children_have_context(e)
-    elif isinstance(node, gast.Subscript):
-      self._set_inner_child_context(node.value, ctx)
-      self._check_inner_children_have_context(node.slice)
-    elif isinstance(node, gast.BinOp):
-      self._check_inner_children_have_context(node.left)
-      self._check_inner_children_have_context(node.right)
-    elif isinstance(node, gast.UnaryOp):
-      self._check_inner_children_have_context(node.operand)
-    elif isinstance(node, (gast.Str, gast.Num)):
-      pass
-    else:
-      raise ValueError('unexpected node type "%s"' % node)
-
   def visit_Attribute(self, node):
     node = self.generic_visit(node)
     if node.attr not in self.replacements:
@@ -210,16 +185,10 @@ class ReplaceTransformer(gast.NodeTransformer):
     new_nodes = self._prepare_replacement(node, node.id)
 
     # Preserve the target context.
+    adjuster = ContextAdjuster(type(node.ctx))
     for n in new_nodes:
-      if isinstance(n, (gast.Tuple, gast.List)):
-        for e in n.elts:
-          self._set_inner_child_context(e, node.ctx)
-      if isinstance(n, gast.Attribute):
-        # For attributes, the inner Name node receives the context, while the
-        # outer ones have it set to Load.
-        self._set_inner_child_context(n, node.ctx)
-      else:
-        n.ctx = node.ctx
+      if hasattr(n, 'ctx'):
+        adjuster.visit(n)
 
     if len(new_nodes) == 1:
       new_nodes, = new_nodes
diff --git a/tensorflow/python/autograph/pyct/templates_test.py b/tensorflow/python/autograph/pyct/templates_test.py
index 54019ef5f4..cdb44b822e 100644
--- a/tensorflow/python/autograph/pyct/templates_test.py
+++ b/tensorflow/python/autograph/pyct/templates_test.py
@@ -134,19 +134,18 @@ class TemplatesTest(test.TestCase):
 
   def test_replace_expression_context(self):
     template = """
-      def test_fn(foo):
+      def test_fn():
         foo
     """
 
     node = templates.replace(
         template, foo=parser.parse_expression('a + 2 * b / -c'))[0]
-    self.assertIsInstance(node.body[0].ctx, gast.Load)
     self.assertIsInstance(node.body[0].left.ctx, gast.Load)
     self.assertIsInstance(node.body[0].right.left.right.ctx, gast.Load)
 
   def test_replace_complex_context(self):
     template = """
-      def test_fn(foo):
+      def test_fn():
         foo = 0
     """
 
@@ -160,7 +159,7 @@ class TemplatesTest(test.TestCase):
 
   def test_replace_index(self):
     template = """
-      def test_fn(foo):
+      def test_fn():
         foo = 0
     """
 
-- 
GitLab


From 0341c5ae8af3659623bfcf636aa53626b163e883 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Thu, 6 Dec 2018 06:14:19 -0800
Subject: [PATCH 1416/1554] Don't assume MaybePaddedAndSlicedInput moved the
 padding into a kPad.

CudnnConvPaddingLegalization has logic to make sure that we call cuDNN with
the same low and high padding, and no interior padding (dilation). However it
then resets the padding in the window, even in the case when this padding was
not moved to a kPad. This CL now moves the update of the convolution window to
the same function that creates the kPad.

PiperOrigin-RevId: 224325347
---
 .../gpu/cudnn_conv_padding_legalization.cc    | 57 +++++++++++--------
 .../compiler/xla/tests/conv_depthwise_test.cc | 12 +---
 2 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_padding_legalization.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_padding_legalization.cc
index d7829045cc..3a09d4d471 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_padding_legalization.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_padding_legalization.cc
@@ -43,13 +43,14 @@ bool IsForwardConvolutionCanonical(const HloInstruction& conv) {
 // dilation), returns kPad and/or kSlice instructions that explicitly apply the
 // padding; otherwise returns the original input operand. When there is both
 // positive padding (including dilation) and negative padding, we insert both
-// kPad and kSlice.
+// kPad and kSlice. Modifies 'conv_window' accordingly if any padding was moved
+// into a kPad or kSlice op.
 HloInstruction* MaybePaddedAndSlicedInput(
-    const Window& conv_window, const ConvolutionDimensionNumbers& conv_dnums,
+    Window* conv_window, const ConvolutionDimensionNumbers& conv_dnums,
     HloInstruction* input) {
   HloComputation* computation = input->parent();
-  if (!window_util::HasSymmetricPadding(conv_window) ||
-      window_util::HasBaseDilation(conv_window)) {
+  if (!window_util::HasSymmetricPadding(*conv_window) ||
+      window_util::HasBaseDilation(*conv_window)) {
     // If padding is uneven or has dilation, we insert a kPad instruction that
     // applies positive padding and dilation.
     //
@@ -62,12 +63,21 @@ HloInstruction* MaybePaddedAndSlicedInput(
         MakeNoPaddingConfig(input->shape().dimensions_size());
     for (size_t i = 0; i < conv_dnums.input_spatial_dimensions().size(); ++i) {
       int64 dim = conv_dnums.input_spatial_dimensions(i);
-      padding_config.mutable_dimensions(dim)->set_edge_padding_low(
-          std::max<int64>(0LL, conv_window.dimensions(i).padding_low()));
-      padding_config.mutable_dimensions(dim)->set_edge_padding_high(
-          std::max<int64>(0LL, conv_window.dimensions(i).padding_high()));
-      padding_config.mutable_dimensions(dim)->set_interior_padding(
-          conv_window.dimensions(i).base_dilation() - 1);
+      if (conv_window->dimensions(i).padding_low() > 0) {
+        padding_config.mutable_dimensions(dim)->set_edge_padding_low(
+            conv_window->dimensions(i).padding_low());
+        conv_window->mutable_dimensions(i)->set_padding_low(0);
+      }
+      if (conv_window->dimensions(i).padding_high() > 0) {
+        padding_config.mutable_dimensions(dim)->set_edge_padding_high(
+            conv_window->dimensions(i).padding_high());
+        conv_window->mutable_dimensions(i)->set_padding_high(0);
+      }
+      if (conv_window->dimensions(i).base_dilation() != 1) {
+        padding_config.mutable_dimensions(dim)->set_interior_padding(
+            conv_window->dimensions(i).base_dilation() - 1);
+        conv_window->mutable_dimensions(i)->set_base_dilation(1);
+      }
     }
     PrimitiveType element_type = input->shape().element_type();
     HloInstruction* padding = computation->AddInstruction(
@@ -75,7 +85,7 @@ HloInstruction* MaybePaddedAndSlicedInput(
     input = MakePadHlo(input, padding, padding_config).ValueOrDie();
   }
 
-  if (window_util::HasNegativePadding(conv_window)) {
+  if (window_util::HasNegativePadding(*conv_window)) {
     // If the window has negative padding, insert a kSlice that explicitly
     // applies negative padding.
     //
@@ -89,10 +99,14 @@ HloInstruction* MaybePaddedAndSlicedInput(
       int64 dim = conv_dnums.input_spatial_dimensions(i);
       // If dimension "dim" has negative padding, increase the start index or
       // decrement the limit index by the amount of negative padding.
-      start_indices[dim] +=
-          std::max<int64>(0LL, -conv_window.dimensions(i).padding_low());
-      limit_indices[dim] -=
-          std::max<int64>(0LL, -conv_window.dimensions(i).padding_high());
+      if (conv_window->dimensions(i).padding_low() < 0) {
+        start_indices[dim] += -conv_window->dimensions(i).padding_low();
+        conv_window->mutable_dimensions(i)->set_padding_low(0);
+      }
+      if (conv_window->dimensions(i).padding_high() < 0) {
+        limit_indices[dim] -= -conv_window->dimensions(i).padding_high();
+        conv_window->mutable_dimensions(i)->set_padding_high(0);
+      }
     }
 
     input =
@@ -140,25 +154,22 @@ bool CudnnConvPaddingLegalization::CanonicalizeForwardConvolution(
 
   // Insert slices and/or pads between the convolution and its input and/or
   // kernel operand.
+  Window new_conv_window = conv->window();
   HloInstruction* new_input = MaybePaddedAndSlicedInput(
-      conv->window(), conv->convolution_dimension_numbers(),
+      &new_conv_window, conv->convolution_dimension_numbers(),
       conv->mutable_operand(0));
   HloInstruction* new_kernel =
-      MaybePaddedKernel(conv->window(), conv->convolution_dimension_numbers(),
+      MaybePaddedKernel(new_conv_window, conv->convolution_dimension_numbers(),
                         conv->mutable_operand(1));
 
-  // Remove the padding from convolution's window field. These paddings are
-  // made explicit with the inserted pads.
-  Window new_conv_window = conv->window();
+  // Remove the window dilation from convolution's window field. These paddings
+  // are made explicit with the pads inserted by MaybePaddedKernel().
   for (size_t i = 0; i < new_conv_window.dimensions_size(); ++i) {
     WindowDimension* dim = new_conv_window.mutable_dimensions(i);
 
     // The size of the kernel may have changed so update the Window to match.
     dim->set_size(new_kernel->shape().dimensions(
         conv->convolution_dimension_numbers().kernel_spatial_dimensions(i)));
-    dim->set_padding_low(0);
-    dim->set_padding_high(0);
-    dim->set_base_dilation(1);
     dim->set_window_dilation(1);
   }
 
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
index 4cc045300f..627a17a0ca 100644
--- a/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_test.cc
@@ -50,15 +50,9 @@ class DepthwiseConvolution2DTest
 static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
   std::vector<DepthwiseConvolution2DSpec> config_set;
   std::vector<std::vector<int64>> config_options = {
-      {128, 6, 3, 64},  {256, 5, 3, 256},  {256, 5, 2, 144},
-      {144, 5, 3, 64},  {144, 5, 2, 256},  {8, 48, 17, 8},
-      {128, 20, 6, 64}, {64, 14, 12, 172}, {16, 9, 4, 16}};
-
-#ifndef XLA_TEST_BACKEND_GPU
-  // TODO(b/119590850): Figure out why this fails on the GPU backend.
-  config_options.push_back({128, 1, 2, 144});
-  config_options.push_back({256, 1, 2, 64});
-#endif
+      {128, 6, 3, 64},  {256, 5, 3, 256}, {256, 5, 2, 144}, {144, 5, 3, 64},
+      {144, 5, 2, 256}, {8, 48, 17, 8},   {128, 20, 6, 64}, {64, 14, 12, 172},
+      {16, 9, 4, 16},   {128, 1, 2, 144}, {256, 1, 2, 64}};
 
   for (auto option : config_options) {
     int64 feature = option[0];
-- 
GitLab


From 24bf218f360f647ce169f06839460147b375c42e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 06:20:33 -0800
Subject: [PATCH 1417/1554] Fix breakage of tensorflow_transform and
 tensorflow_estimator depending on private symbol `_parse_saved_model` in
 internal implementation file tensorflow/python/saved_model/loader_impl.py

PiperOrigin-RevId: 224326006
---
 tensorflow/python/saved_model/loader_impl.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index 2717e5b915..e5be03aae4 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -83,6 +83,11 @@ def parse_saved_model(export_dir):
                    constants.SAVED_MODEL_FILENAME_PB))
 
 
+# TODO(b/120594573): Make this symbol also available as private, so that
+# tensorflow_transform and tensorflow_estimator do not break.
+_parse_saved_model = parse_saved_model
+
+
 def _get_asset_tensors(export_dir, meta_graph_def_to_load, import_scope=None):
   """Gets the asset tensors, if defined in the meta graph def to load.
 
-- 
GitLab


From 8ccf1ebdbf4475bc7af6d79b2fa7e1fd8221e3fd Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 6 Dec 2018 06:44:41 -0800
Subject: [PATCH 1418/1554] [TF:XLA] Move TriangularSolve into XLA client
 library.

PiperOrigin-RevId: 224328836
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   2 +-
 .../kernels/matrix_triangular_solve_op.cc     |   4 +-
 tensorflow/compiler/tf2xla/lib/BUILD          |  48 +---
 tensorflow/compiler/tf2xla/lib/cholesky.cc    |   2 +-
 tensorflow/compiler/xla/client/lib/BUILD      |  45 ++++
 .../client}/lib/triangular_solve.cc           | 208 +++++++++---------
 .../client}/lib/triangular_solve.h            |  16 +-
 .../client}/lib/triangular_solve_test.cc      |   6 +-
 8 files changed, 163 insertions(+), 168 deletions(-)
 rename tensorflow/compiler/{tf2xla => xla/client}/lib/triangular_solve.cc (67%)
 rename tensorflow/compiler/{tf2xla => xla/client}/lib/triangular_solve.h (88%)
 rename tensorflow/compiler/{tf2xla => xla/client}/lib/triangular_solve_test.cc (99%)

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index fa51a72aea..8bc3292296 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -126,7 +126,6 @@ tf_kernel_library(
         "//tensorflow/compiler/tf2xla/lib:qr",
         "//tensorflow/compiler/tf2xla/lib:random",
         "//tensorflow/compiler/tf2xla/lib:scatter",
-        "//tensorflow/compiler/tf2xla/lib:triangular_solve",
         "//tensorflow/compiler/tf2xla/lib:util",
         "//tensorflow/compiler/tf2xla/lib:while_loop",
         "//tensorflow/compiler/tf2xla/ops:xla_ops",
@@ -147,6 +146,7 @@ tf_kernel_library(
         "//tensorflow/compiler/xla/client/lib:pooling",
         "//tensorflow/compiler/xla/client/lib:prng",
         "//tensorflow/compiler/xla/client/lib:sorting",
+        "//tensorflow/compiler/xla/client/lib:triangular_solve",
         "//tensorflow/core:framework",
         "//tensorflow/core:image_ops_op_lib",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc b/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
index f4def11d08..90c0ebefb2 100644
--- a/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matrix_triangular_solve_op.cc
@@ -13,9 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/tf2xla/lib/triangular_solve.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
 
 namespace tensorflow {
 namespace {
@@ -29,7 +29,7 @@ class MatrixTriangularSolveOp : public XlaOpKernel {
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    auto result = TriangularSolve(
+    auto result = xla::TriangularSolve(
         ctx->Input(0), ctx->Input(1), /*left_side=*/true,
         /*lower=*/lower_, /*transpose_a=*/adjoint_, /*conjugate_a=*/adjoint_);
     ctx->SetOutput(0, result);
diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD
index 422781d536..3e7a761120 100644
--- a/tensorflow/compiler/tf2xla/lib/BUILD
+++ b/tensorflow/compiler/tf2xla/lib/BUILD
@@ -38,7 +38,6 @@ cc_library(
     srcs = ["cholesky.cc"],
     hdrs = ["cholesky.h"],
     deps = [
-        ":triangular_solve",
         ":util",
         ":while_loop",
         "//tensorflow/compiler/xla:literal",
@@ -50,6 +49,7 @@ cc_library(
         "//tensorflow/compiler/xla/client/lib:constants",
         "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/client/lib:slicing",
+        "//tensorflow/compiler/xla/client/lib:triangular_solve",
         "//tensorflow/core:lib",
     ],
 )
@@ -111,52 +111,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "triangular_solve",
-    srcs = ["triangular_solve.cc"],
-    hdrs = ["triangular_solve.h"],
-    deps = [
-        ":util",
-        "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/compiler/xla/client:xla_computation",
-        "//tensorflow/compiler/xla/client/lib:constants",
-        "//tensorflow/compiler/xla/client/lib:math",
-        "//tensorflow/compiler/xla/client/lib:matrix",
-        "//tensorflow/compiler/xla/client/lib:slicing",
-        "//tensorflow/core:lib",
-    ],
-)
-
-xla_test(
-    name = "triangular_solve_test",
-    srcs = ["triangular_solve_test.cc"],
-    tags = ["noasan"],  # sometimes times out, http://b/78650012
-    deps = [
-        ":triangular_solve",
-        "//tensorflow/compiler/xla:array2d",
-        "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:test",
-        "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:global_data",
-        "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/compiler/xla/tests:client_library_test_base",
-        "//tensorflow/compiler/xla/tests:literal_test_util",
-        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-    ],
-)
-
 cc_library(
     name = "util",
     srcs = ["util.cc"],
diff --git a/tensorflow/compiler/tf2xla/lib/cholesky.cc b/tensorflow/compiler/tf2xla/lib/cholesky.cc
index 7ef8659992..550ab5b056 100644
--- a/tensorflow/compiler/tf2xla/lib/cholesky.cc
+++ b/tensorflow/compiler/tf2xla/lib/cholesky.cc
@@ -18,12 +18,12 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
-#include "tensorflow/compiler/tf2xla/lib/triangular_solve.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/lib/while_loop.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/lib/slicing.h"
+#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index f0f530d7d7..41db8de29f 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -258,3 +258,48 @@ cc_library(
         "@com_google_absl//absl/strings",
     ],
 )
+
+cc_library(
+    name = "triangular_solve",
+    srcs = ["triangular_solve.cc"],
+    hdrs = ["triangular_solve.h"],
+    deps = [
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/client/lib:constants",
+        "//tensorflow/compiler/xla/client/lib:math",
+        "//tensorflow/compiler/xla/client/lib:matrix",
+        "//tensorflow/compiler/xla/client/lib:slicing",
+        "//tensorflow/core:lib",
+    ],
+)
+
+xla_test(
+    name = "triangular_solve_test",
+    srcs = ["triangular_solve_test.cc"],
+    tags = ["noasan"],  # sometimes times out, http://b/78650012
+    deps = [
+        ":triangular_solve",
+        "//tensorflow/compiler/xla:array2d",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:global_data",
+        "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+    ],
+)
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
similarity index 67%
rename from tensorflow/compiler/tf2xla/lib/triangular_solve.cc
rename to tensorflow/compiler/xla/client/lib/triangular_solve.cc
index 192a61dca2..c5a1d34cc6 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
@@ -13,12 +13,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/tf2xla/lib/triangular_solve.h"
+#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
 
 #include <memory>
 #include <vector>
 
-#include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/lib/math.h"
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
@@ -30,21 +29,20 @@ limitations under the License.
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/util.h"
-#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/math/math_util.h"
 
-namespace tensorflow {
+namespace xla {
 
 // Get the diagonal blocks of the coefficient matrix
-xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
-  xla::XlaBuilder* builder = a.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(a));
-    int ndims = xla::ShapeUtil::Rank(shape);
-    int64 n = xla::ShapeUtil::GetDimension(shape, -1);
+XlaOp DiagonalBlocks(XlaOp a, int64 block_size) {
+  XlaBuilder* builder = a.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(a));
+    int ndims = ShapeUtil::Rank(shape);
+    int64 n = ShapeUtil::GetDimension(shape, -1);
     int64 num_blocks = n / block_size;
 
-    xla::XlaOp diag_blocks;
+    XlaOp diag_blocks;
 
     // If the coefficient matrix is exactly the block size, we just add a
     // singleton dimension i.e. [..., n, n] -> [..., 1, n, n]
@@ -59,13 +57,13 @@ xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
     if (n > block_size) {
       // Construct the starting indices of the diagonal blocks
       auto start_indices =
-          Transpose(Broadcast(Mul(Iota(builder, xla::S32, num_blocks),
-                                  xla::ConstantR0<int32>(builder, block_size)),
+          Transpose(Broadcast(Mul(Iota(builder, S32, num_blocks),
+                                  ConstantR0<int32>(builder, block_size)),
                               /*broadcast_sizes=*/{2}),
                     /*permutation=*/{1, 0});
 
       // Gather the diagonal blocks
-      xla::GatherDimensionNumbers dim_numbers;
+      GatherDimensionNumbers dim_numbers;
       dim_numbers.add_offset_dims(ndims - 1);
       dim_numbers.add_offset_dims(ndims);
       dim_numbers.add_start_index_map(ndims - 2);
@@ -81,7 +79,7 @@ xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
       // Pad with zeros
       auto last_blocks =
           SliceInMinorDims(a, {n - n % block_size, n - n % block_size}, {n, n});
-      xla::PaddingConfig config = xla::MakeNoPaddingConfig(ndims);
+      PaddingConfig config = MakeNoPaddingConfig(ndims);
       int64 padding = block_size - n % block_size;
       config.mutable_dimensions(ndims - 1)->set_edge_padding_high(padding);
       config.mutable_dimensions(ndims - 2)->set_edge_padding_high(padding);
@@ -90,9 +88,8 @@ xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
 
       // Add a singleton dimension
       // i.e. [..., block_size, block_size] -> [..., 1, block_size, block_size]
-      TF_ASSIGN_OR_RETURN(xla::Shape blocks_shape,
-                          builder->GetShape(last_blocks));
-      auto shape_dims = xla::AsInt64Slice(blocks_shape.dimensions());
+      TF_ASSIGN_OR_RETURN(Shape blocks_shape, builder->GetShape(last_blocks));
+      auto shape_dims = AsInt64Slice(blocks_shape.dimensions());
       auto last_blocks_dims = std::vector<int64>(ndims);
       std::copy(shape_dims.begin(), shape_dims.end(), last_blocks_dims.begin());
       last_blocks_dims.insert(last_blocks_dims.end() - 2, 1);
@@ -101,7 +98,7 @@ xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
       // Concatenate with the other blocks if necessary
       if (n > block_size) {
         diag_blocks =
-            xla::ConcatInDim(builder, {diag_blocks, last_blocks}, ndims - 2);
+            ConcatInDim(builder, {diag_blocks, last_blocks}, ndims - 2);
       } else {
         diag_blocks = last_blocks;
       }
@@ -111,16 +108,16 @@ xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
   });
 }
 
-xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
-                                bool transpose_a, bool conjugate_a,
-                                xla::PrecisionConfig::Precision precision) {
-  xla::XlaBuilder* builder = diag_blocks.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
+XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
+                           bool conjugate_a,
+                           PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = diag_blocks.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     // Input is a batch of square lower triangular square matrices. Its shape is
     // (..., size, size). We resize this to (num_blocks, size, size).
-    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(diag_blocks));
-    int64 block_size = xla::ShapeUtil::GetDimension(shape, -1);
-    int64 num_blocks = xla::ShapeUtil::ElementsIn(shape) /
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(diag_blocks));
+    int64 block_size = ShapeUtil::GetDimension(shape, -1);
+    int64 num_blocks = ShapeUtil::ElementsIn(shape) /
                        tensorflow::MathUtil::IPow(block_size, 2);
     diag_blocks = Reshape(diag_blocks, {num_blocks, block_size, block_size});
 
@@ -132,9 +129,9 @@ xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
     // zero (which can happen if the last block was padded) otherwise it will
     // introduce nans which will propagate
     auto diags = GetMatrixDiagonal(diag_blocks);
-    TF_ASSIGN_OR_RETURN(xla::Shape diags_shape, builder->GetShape(diags));
+    TF_ASSIGN_OR_RETURN(Shape diags_shape, builder->GetShape(diags));
     auto one = ScalarLike(diags, 1);
-    auto ones = Broadcast(one, xla::AsInt64Slice(diags_shape.dimensions()));
+    auto ones = Broadcast(one, AsInt64Slice(diags_shape.dimensions()));
     diags = Select(Eq(diags, Zero(builder, shape.element_type())), ones, diags);
     auto scaled_diag_blocks = Div(diag_blocks, diags, {0, 2});
 
@@ -160,40 +157,40 @@ xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
     auto start_index = (lower) ? 0 : block_size - 1;
     auto output_block = DynamicUpdateSlice(
         neg_identity, pos_one,
-        /*start_indices=*/xla::ConstantR1<int>(builder, 2, start_index));
+        /*start_indices=*/ConstantR1<int>(builder, 2, start_index));
 
     // Broadcast diag([1, -1, -1, ...]) to every block
-    xla::XlaOp output = Broadcast(output_block,
-                                  /*broadcast_sizes=*/{num_blocks});
+    XlaOp output = Broadcast(output_block,
+                             /*broadcast_sizes=*/{num_blocks});
 
     // Now we construct a loop that performs matrix-vector multiplications
     // inverting the blocks one row at a time
-    std::vector<xla::Shape> tuple_shapes = {
+    std::vector<Shape> tuple_shapes = {
         // The loop iteration counter is a scalar, incremented each iteration.
-        xla::ShapeUtil::MakeShape(xla::S32, {}),
+        ShapeUtil::MakeShape(S32, {}),
         // The output has the shape of A, with one row updated each iteration.
-        xla::ShapeUtil::MakeShape(shape.element_type(),
-                                  {num_blocks, block_size, block_size}),
+        ShapeUtil::MakeShape(shape.element_type(),
+                             {num_blocks, block_size, block_size}),
         // The input is a loop invariant.
-        xla::ShapeUtil::MakeShape(shape.element_type(),
-                                  {num_blocks, block_size, block_size})};
-    xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes);
+        ShapeUtil::MakeShape(shape.element_type(),
+                             {num_blocks, block_size, block_size})};
+    Shape tuple_shape = ShapeUtil::MakeTupleShape(tuple_shapes);
 
-    auto init_i = One(builder, xla::S32);
-    auto init = xla::Tuple(builder, {init_i, output, scaled_diag_blocks});
+    auto init_i = One(builder, S32);
+    auto init = Tuple(builder, {init_i, output, scaled_diag_blocks});
 
     // Construct the loop condition function.
-    std::unique_ptr<xla::XlaBuilder> condb =
+    std::unique_ptr<XlaBuilder> condb =
         builder->CreateSubBuilder("InvertDiagCond");
     {
       auto i = GetTupleElement(
           Parameter(condb.get(), 0, tuple_shape, "InvertDiagCondTuple"), 0);
-      Lt(i, xla::ConstantR0<int32>(condb.get(), block_size));
+      Lt(i, ConstantR0<int32>(condb.get(), block_size));
     }
     TF_ASSIGN_OR_RETURN(auto cond, condb->Build());
 
     // Construct the loop body function.
-    std::unique_ptr<xla::XlaBuilder> bodyb =
+    std::unique_ptr<XlaBuilder> bodyb =
         builder->CreateSubBuilder("InvertDiagBody");
     {
       auto input_tuple =
@@ -203,21 +200,21 @@ xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
       auto body_out = GetTupleElement(input_tuple, 1);
       auto body_input = GetTupleElement(input_tuple, 2);
 
-      auto zero = xla::ConstantR1<int32>(bodyb.get(), 1, 0);
+      auto zero = ConstantR1<int32>(bodyb.get(), 1, 0);
       auto j = (lower) ? i : ScalarLike(i, block_size - 1) - i;
       auto start_indices =
-          xla::ConcatInDim(bodyb.get(), {zero, Reshape(j, {1}), zero}, 0);
+          ConcatInDim(bodyb.get(), {zero, Reshape(j, {1}), zero}, 0);
       auto input_row =
           DynamicSlice(body_input, start_indices,
                        /*slice_sizes=*/{num_blocks, 1, block_size});
 
       // We want -L21 L11^{-1}
-      xla::DotDimensionNumbers dnums;
+      DotDimensionNumbers dnums;
       dnums.add_lhs_batch_dimensions(0);
       dnums.add_rhs_batch_dimensions(0);
       dnums.add_lhs_contracting_dimensions(2);
       dnums.add_rhs_contracting_dimensions(1);
-      xla::PrecisionConfig precision_proto;
+      PrecisionConfig precision_proto;
       precision_proto.add_operand_precision(precision);
       precision_proto.add_operand_precision(precision);
       auto update = -DotGeneral(input_row, body_out, dnums, &precision_proto);
@@ -225,7 +222,7 @@ xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
       body_out = DynamicUpdateSlice(body_out, update, start_indices);
 
       auto next_i = i + ScalarLike(i, 1);
-      xla::Tuple(bodyb.get(), {next_i, body_out, body_input});
+      Tuple(bodyb.get(), {next_i, body_out, body_input});
     }
     TF_ASSIGN_OR_RETURN(auto body, bodyb->Build());
 
@@ -239,27 +236,26 @@ xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
                           /*broadcast_dimensions=*/{0, 1});
 
     // Reshape back to original batch major dimensions
-    return Reshape(inv_diag_blocks, xla::AsInt64Slice(shape.dimensions()));
+    return Reshape(inv_diag_blocks, AsInt64Slice(shape.dimensions()));
   });
 }
 
-xla::XlaOp SolveWithInvertedDiagonalBlocks(
-    xla::XlaOp a, xla::XlaOp b, xla::XlaOp inv_diag_blocks, bool left_side,
-    bool lower, bool transpose_a, bool conjugate_a,
-    xla::PrecisionConfig::Precision precision) {
-  xla::XlaBuilder* builder = a.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape blocks_shape,
-                        builder->GetShape(inv_diag_blocks));
-    TF_ASSIGN_OR_RETURN(xla::Shape b_shape, builder->GetShape(b));
-    int64 block_size = xla::ShapeUtil::GetDimension(blocks_shape, -1);
-
-    TF_ASSIGN_OR_RETURN(xla::Shape a_shape, builder->GetShape(a));
-    int64 ndims = xla::ShapeUtil::Rank(a_shape);
-    int64 n = xla::ShapeUtil::GetDimension(a_shape, -1);
+XlaOp SolveWithInvertedDiagonalBlocks(XlaOp a, XlaOp b, XlaOp inv_diag_blocks,
+                                      bool left_side, bool lower,
+                                      bool transpose_a, bool conjugate_a,
+                                      PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = a.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape blocks_shape, builder->GetShape(inv_diag_blocks));
+    TF_ASSIGN_OR_RETURN(Shape b_shape, builder->GetShape(b));
+    int64 block_size = ShapeUtil::GetDimension(blocks_shape, -1);
+
+    TF_ASSIGN_OR_RETURN(Shape a_shape, builder->GetShape(a));
+    int64 ndims = ShapeUtil::Rank(a_shape);
+    int64 n = ShapeUtil::GetDimension(a_shape, -1);
     int64 num_blocks = n / block_size + (n % block_size != 0);
     int64 m_dim = (left_side) ? -1 : -2;
-    int64 m = xla::ShapeUtil::GetDimension(b_shape, m_dim);
+    int64 m = ShapeUtil::GetDimension(b_shape, m_dim);
 
     // Initialize the solution
     auto x = ZerosLike(b);
@@ -295,7 +291,7 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks(
       }
       auto b_row = SliceInMinorDims(b, start, end);
 
-      xla::XlaOp remainder;
+      XlaOp remainder;
       if (i == 0) {
         remainder = b_row;
       } else {
@@ -322,11 +318,10 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks(
         }
       }
 
-      xla::XlaOp x_update;
-      auto zero = Zero(builder, xla::S32);
-      auto start_index =
-          xla::ConstantR0WithType(builder, xla::S32, j * block_size);
-      std::vector<xla::XlaOp> update_starts = {start_index, zero};
+      XlaOp x_update;
+      auto zero = Zero(builder, S32);
+      auto start_index = ConstantR0WithType(builder, S32, j * block_size);
+      std::vector<XlaOp> update_starts = {start_index, zero};
       if (left_side) {
         x_update = BatchDot(MaybeTransposeInMinorDims(inv_block, transpose_a),
                             remainder, precision);
@@ -343,24 +338,24 @@ xla::XlaOp SolveWithInvertedDiagonalBlocks(
   });
 }
 
-xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
-                           bool lower, bool transpose_a, bool conjugate_a,
-                           int64 block_size,
-                           xla::PrecisionConfig::Precision precision) {
-  xla::XlaBuilder* builder = a.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape a_shape, builder->GetShape(a));
-    TF_ASSIGN_OR_RETURN(xla::Shape b_shape, builder->GetShape(b));
-    if (xla::ShapeUtil::Rank(a_shape) != xla::ShapeUtil::Rank(b_shape)) {
-      return errors::InvalidArgument(
-          "Arguments to TriangularSolve have different ranks: ",
-          xla::ShapeUtil::HumanString(a_shape), " vs. ",
-          xla::ShapeUtil::HumanString(b_shape));
+XlaOp TriangularSolve(XlaOp a, XlaOp b, bool left_side, bool lower,
+                      bool transpose_a, bool conjugate_a, int64 block_size,
+                      PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = a.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape a_shape, builder->GetShape(a));
+    TF_ASSIGN_OR_RETURN(Shape b_shape, builder->GetShape(b));
+    if (ShapeUtil::Rank(a_shape) != ShapeUtil::Rank(b_shape)) {
+      return InvalidArgument(
+          "Arguments to TriangularSolve have shapes with different ranks: "
+          "%s vs. %s",
+          ShapeUtil::HumanString(a_shape), ShapeUtil::HumanString(b_shape));
     }
-    const int64 ndims = xla::ShapeUtil::Rank(a_shape);
+    const int64 ndims = ShapeUtil::Rank(a_shape);
     if (ndims < 2) {
-      return errors::InvalidArgument(
-          "Arguments to TriangularSolve must have rank >= 2: ", ndims);
+      return InvalidArgument(
+          "Arguments to TriangularSolve was rank %d but must have rank >= 2.",
+          ndims);
     }
     // The batch dimensions must be equal.
     std::vector<int64> batch_dimensions;
@@ -368,32 +363,33 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
       int64 a_size = a_shape.dimensions(i);
       int64 b_size = b_shape.dimensions(i);
       if (a_size != b_size) {
-        return errors::InvalidArgument(
-            "Batch dimensions of arguments to TriangularSolve must be equal: ",
-            xla::ShapeUtil::HumanString(a_shape), " vs ",
-            xla::ShapeUtil::HumanString(b_shape));
+        return InvalidArgument(
+            "Batch dimensions of arguments to TriangularSolve must be equal; "
+            "shapes were %s and %s.",
+            ShapeUtil::HumanString(a_shape), ShapeUtil::HumanString(b_shape));
       }
       batch_dimensions.push_back(a_size);
     }
 
-    if (xla::ShapeUtil::GetDimension(a_shape, -1) !=
-        xla::ShapeUtil::GetDimension(a_shape, -2)) {
-      return errors::InvalidArgument(
-          "The 'a' arguments to TriangularSolve must be square matrices: ",
-          xla::ShapeUtil::HumanString(a_shape));
+    if (ShapeUtil::GetDimension(a_shape, -1) !=
+        ShapeUtil::GetDimension(a_shape, -2)) {
+      return InvalidArgument(
+          "The 'a' argument to TriangularSolve must be a batched square matrix;"
+          " shape was: %s",
+          ShapeUtil::HumanString(a_shape));
     }
-    const int64 m = xla::ShapeUtil::GetDimension(b_shape, -2);
-    const int64 n = xla::ShapeUtil::GetDimension(b_shape, -1);
-    if ((left_side ? m : n) != xla::ShapeUtil::GetDimension(a_shape, -1)) {
-      return errors::InvalidArgument(
-          "Arguments to TriangularSolve have incompatible matrix shapes: ",
-          xla::ShapeUtil::HumanString(a_shape), " vs ",
-          xla::ShapeUtil::HumanString(b_shape));
+    const int64 m = ShapeUtil::GetDimension(b_shape, -2);
+    const int64 n = ShapeUtil::GetDimension(b_shape, -1);
+    if ((left_side ? m : n) != ShapeUtil::GetDimension(a_shape, -1)) {
+      return InvalidArgument(
+          "Arguments to TriangularSolve have incompatible matrix shapes %s and "
+          "%s",
+          ShapeUtil::HumanString(a_shape), ShapeUtil::HumanString(b_shape));
     }
 
     if (block_size < 1) {
-      return errors::InvalidArgument(
-          "block_size argument to TriangularSolve must be >= 1; got ",
+      return InvalidArgument(
+          "block_size argument to TriangularSolve must be >= 1; got %d",
           block_size);
     }
 
@@ -413,4 +409,4 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
   });
 }
 
-}  // namespace tensorflow
+}  // namespace xla
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.h b/tensorflow/compiler/xla/client/lib/triangular_solve.h
similarity index 88%
rename from tensorflow/compiler/tf2xla/lib/triangular_solve.h
rename to tensorflow/compiler/xla/client/lib/triangular_solve.h
index 2303234f36..50a3b30ebd 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.h
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.h
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMPILER_TF2XLA_LIB_TRIANGULAR_SOLVE_H_
-#define TENSORFLOW_COMPILER_TF2XLA_LIB_TRIANGULAR_SOLVE_H_
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TRIANGULAR_SOLVE_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TRIANGULAR_SOLVE_H_
 
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
-namespace tensorflow {
+namespace xla {
 
 // Solves systems of linear equations with lower or upper triangular coefficient
 // matrices by forward- or back-substitution. Broadcasting along leading
@@ -57,11 +57,11 @@ namespace tensorflow {
 //
 // Uses a blocked algorithm if `block_size` is > 1; if block_size == 1 then no
 // blocking is used.
-xla::XlaOp TriangularSolve(
-    xla::XlaOp a, xla::XlaOp b, bool left_side, bool lower, bool transpose_a,
+XlaOp TriangularSolve(
+    XlaOp a, XlaOp b, bool left_side, bool lower, bool transpose_a,
     bool conjugate_a, int64 block_size = 128,
-    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::HIGHEST);
+    PrecisionConfig::Precision precision = PrecisionConfig::HIGHEST);
 
-}  // namespace tensorflow
+}  // namespace xla
 
-#endif  // TENSORFLOW_COMPILER_TF2XLA_LIB_TRIANGULAR_SOLVE_H_
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_TRIANGULAR_SOLVE_H_
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
similarity index 99%
rename from tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc
rename to tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
index aeebf16028..f6a70d64a7 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/tf2xla/lib/triangular_solve.h"
+#include "tensorflow/compiler/xla/client/lib/triangular_solve.h"
 
 #include <memory>
 #include <numeric>
@@ -30,7 +30,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 
-namespace tensorflow {
+namespace xla {
 namespace {
 
 using TriangularSolveTest = xla::ClientLibraryTestBase;
@@ -330,4 +330,4 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTransposeNoconjugate) {
 }
 
 }  // namespace
-}  // namespace tensorflow
+}  // namespace xla
-- 
GitLab


From ad9e4bc51773fb3ce515a52d41da7e97e1d1d868 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 07:44:02 -0800
Subject: [PATCH 1419/1554] Don't add -Wall and -Wno-comment on Windows

PiperOrigin-RevId: 224335995
---
 tensorflow/lite/BUILD | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index af1cfe27a6..8fca01624c 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -6,6 +6,7 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
+load("//tensorflow:tensorflow.bzl", "if_not_windows")
 
 exports_files(glob([
     "testdata/*.bin",
@@ -35,10 +36,10 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-TFLITE_DEFAULT_COPTS = [
+TFLITE_DEFAULT_COPTS = if_not_windows([
     "-Wall",
     "-Wno-comment",
-]
+])
 
 cc_library(
     name = "schema_fbs_version",
-- 
GitLab


From 1f40e575745e1d110aa8be6c7409c209bc17408d Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 6 Dec 2018 07:54:48 -0800
Subject: [PATCH 1420/1554] Clean some dead code and unused calls.

PiperOrigin-RevId: 224337334
---
 tensorflow/lite/delegates/flex/delegate_test.cc          | 1 -
 tensorflow/lite/examples/label_image/label_image_test.cc | 2 --
 tensorflow/lite/examples/minimal/minimal.cc              | 2 +-
 tensorflow/lite/testing/generated_examples_zip_test.cc   | 2 +-
 4 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/delegate_test.cc b/tensorflow/lite/delegates/flex/delegate_test.cc
index 1b2f476f03..ee37090d94 100644
--- a/tensorflow/lite/delegates/flex/delegate_test.cc
+++ b/tensorflow/lite/delegates/flex/delegate_test.cc
@@ -22,7 +22,6 @@ namespace tflite {
 namespace flex {
 namespace {
 
-using ::testing::ContainsRegex;
 using ::testing::ElementsAre;
 
 class DelegateTest : public testing::FlexModelTest {
diff --git a/tensorflow/lite/examples/label_image/label_image_test.cc b/tensorflow/lite/examples/label_image/label_image_test.cc
index 6b4ec2a937..4db139f048 100644
--- a/tensorflow/lite/examples/label_image/label_image_test.cc
+++ b/tensorflow/lite/examples/label_image/label_image_test.cc
@@ -20,8 +20,6 @@ limitations under the License.
 #include "tensorflow/lite/examples/label_image/get_top_n.h"
 #include "tensorflow/lite/examples/label_image/label_image.h"
 
-using ::testing::ElementsAreArray;
-
 namespace tflite {
 namespace label_image {
 
diff --git a/tensorflow/lite/examples/minimal/minimal.cc b/tensorflow/lite/examples/minimal/minimal.cc
index 46f8b09df6..9bbfee6085 100644
--- a/tensorflow/lite/examples/minimal/minimal.cc
+++ b/tensorflow/lite/examples/minimal/minimal.cc
@@ -50,7 +50,7 @@ int main(int argc, char* argv[]) {
 
   // Build the interpreter
   tflite::ops::builtin::BuiltinOpResolver resolver;
-  InterpreterBuilder builder(*model.get(), resolver);
+  InterpreterBuilder builder(*model, resolver);
   std::unique_ptr<Interpreter> interpreter;
   builder(&interpreter);
   TFLITE_MINIMAL_CHECK(interpreter != nullptr);
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index 91a4851fb0..a9a31ad088 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -205,7 +205,7 @@ tensorflow::Status ReadManifest(const string& original_file, const string& dir,
   }
   if (!added) {
     string message = "Test had no examples: " + original_file;
-    return tensorflow::Status(tensorflow::error::UNKNOWN, message.c_str());
+    return tensorflow::Status(tensorflow::error::UNKNOWN, message);
   }
   return tensorflow::Status::OK();
 }
-- 
GitLab


From 636207ade13e6f9c0b110b21497cc6ed040ea4d6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 08:32:41 -0800
Subject: [PATCH 1421/1554] Fix wrongly exported dtypes.uint16 to dtypes.int16

PiperOrigin-RevId: 224342753
---
 tensorflow/python/framework/dtypes.py                  | 2 +-
 tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index f7a12d27df..9a4fe4e93b 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -347,7 +347,7 @@ tf_export("dtypes.uint32", "uint32").export_constant(__name__, "uint32")
 uint64 = DType(types_pb2.DT_UINT64)
 tf_export("dtypes.uint64", "uint64").export_constant(__name__, "uint64")
 int16 = DType(types_pb2.DT_INT16)
-tf_export("dtypes.uint16", "int16").export_constant(__name__, "int16")
+tf_export("dtypes.int16", "int16").export_constant(__name__, "int16")
 int8 = DType(types_pb2.DT_INT8)
 tf_export("dtypes.int8", "int8").export_constant(__name__, "int8")
 string = DType(types_pb2.DT_STRING)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt
index 848fc303aa..01b870a816 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.dtypes.pbtxt
@@ -44,6 +44,10 @@ tf_module {
     name: "half"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
+  member {
+    name: "int16"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
+  }
   member {
     name: "int32"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
index 848fc303aa..01b870a816 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
@@ -44,6 +44,10 @@ tf_module {
     name: "half"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
+  member {
+    name: "int16"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
+  }
   member {
     name: "int32"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
-- 
GitLab


From be4b5c34ecc0a99929b590cc6f956a4dc4eece55 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 08:37:29 -0800
Subject: [PATCH 1422/1554] Update Eigen to
 https://bitbucket.org/eigen/eigen/commits/729d33d11c81fd023834a2d61f0f280ba9da48c8

PiperOrigin-RevId: 224343375
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ad3cecd3a1..e6b4a89e3b 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -134,11 +134,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "37a483ec219c43219b6e0fc07e799277a4a36abb2b9f4162cfcd256aa211eae8",
-        strip_prefix = "eigen-eigen-2e50f4a5542a",
+        sha256 = "aae7a680d141c978301dfae2c7945c06039f65849fcf64269595a9cdbba82638",
+        strip_prefix = "eigen-eigen-729d33d11c81",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/2e50f4a5542a.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/2e50f4a5542a.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/729d33d11c81.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/729d33d11c81.tar.gz",
         ],
     )
 
-- 
GitLab


From 0334b504c6c7179a322a0293e09616201cabc805 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 6 Dec 2018 09:19:24 -0800
Subject: [PATCH 1423/1554] Fix test failure.

PiperOrigin-RevId: 224350277
---
 tensorflow/lite/tools/benchmark/benchmark_test.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_test.cc b/tensorflow/lite/tools/benchmark/benchmark_test.cc
index 0cf66c6e4d..a4f830122f 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_test.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_test.cc
@@ -76,10 +76,10 @@ TEST(BenchmarkTest, ParametersArePopulatedWhenInputShapeIsNotSpecified) {
   ASSERT_GE(inputs.size(), 1);
   auto input_tensor = interpreter->tensor(inputs[0]);
 
-  std::vector<uint8_t> input_bytes;
+  std::vector<char> input_bytes;
   input_bytes.reserve(input_tensor->bytes);
   for (size_t i = 0; i < input_tensor->bytes; i++) {
-    input_bytes.push_back(input_tensor->data.b[i]);
+    input_bytes.push_back(input_tensor->data.raw_const[i]);
   }
   benchmark.Prepare();
 
@@ -87,7 +87,7 @@ TEST(BenchmarkTest, ParametersArePopulatedWhenInputShapeIsNotSpecified) {
   EXPECT_EQ(input_bytes.size(), input_tensor->bytes);
   bool is_same = true;
   for (size_t i = 0; i < input_tensor->bytes; i++) {
-    if (input_bytes[i] != input_tensor->data.b[i]) {
+    if (input_bytes[i] != input_tensor->data.raw_const[i]) {
       is_same = false;
       break;
     }
-- 
GitLab


From f83053f48c1d262bd8dd61b87890eb34d550d8f7 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 6 Dec 2018 09:35:15 -0800
Subject: [PATCH 1424/1554] Use return value of TensorShapeUtils::MakeShape

---
 .../contrib/tensorrt/convert/convert_nodes.cc | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index ac0e2f684c..777a80bbc4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -120,11 +120,13 @@ inline nvinfer1::Dims TensorShapeToTrtDims(const TensorShapeType& shape,
   return trt_dims;
 }
 
-inline nvinfer1::Dims TensorShapeArrayToTrtDims(const std::vector<int>& shape,
-                                                bool ignore_first_dim = false) {
+Status TensorShapeArrayToTrtDims(const std::vector<int>& shape,
+                                 nvinfer1::Dims* out,
+                                 bool ignore_first_dim = false) {
   PartialTensorShape tensor_shape;
-  TensorShapeUtils::MakeShape(shape, &tensor_shape);
-  return TensorShapeToTrtDims(tensor_shape, ignore_first_dim);
+  TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape(shape, &tensor_shape));
+  *out = TensorShapeToTrtDims(tensor_shape, ignore_first_dim);
+  return tensorflow::Status::OK();
 }
 
 void GetOutputProperties(const grappler::GraphProperties& graph_properties,
@@ -1937,8 +1939,9 @@ tensorflow::Status ConvertExpandDims(OpConverterParams* params) {
   // ExpandDims: Insert new dim of size 1.
   input_dims.insert(input_dims.begin() + axis, 1);
   // Reshape tensor.
-  nvinfer1::Dims new_dims =
-      TensorShapeArrayToTrtDims(input_dims, /*ignore_first_dim=*/true);
+  nvinfer1::Dims new_dims;
+  TF_RETURN_IF_ERROR(TensorShapeArrayToTrtDims(input_dims, &new_dims,
+                                               /*ignore_first_dim=*/true));
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
@@ -2002,8 +2005,9 @@ tensorflow::Status ConvertSqueeze(OpConverterParams* params) {
   input_dims.erase(std::remove(input_dims.begin(), input_dims.end(), 0),
                    input_dims.end());
   // Reshape tensor.
-  nvinfer1::Dims new_dims =
-      TensorShapeArrayToTrtDims(input_dims, /*ignore_first_dim=*/true);
+  nvinfer1::Dims new_dims;
+  TF_RETURN_IF_ERROR(TensorShapeArrayToTrtDims(input_dims, &new_dims,
+                                               /*ignore_first_dim=*/true));
   const nvinfer1::ITensor* output_tensor = nullptr;
   TF_RETURN_IF_ERROR(params->converter->PrepareTensorForShape(
       input_tensor, new_dims, &output_tensor));
-- 
GitLab


From 322254f7e00bb739ded8611a925cca450f214ec2 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 6 Dec 2018 09:34:36 -0800
Subject: [PATCH 1425/1554] Add explicit for one param constructor.

PiperOrigin-RevId: 224352578
---
 tensorflow/lite/kernels/resize_bilinear_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/resize_bilinear_test.cc b/tensorflow/lite/kernels/resize_bilinear_test.cc
index 530bb32b94..d3f4837a28 100644
--- a/tensorflow/lite/kernels/resize_bilinear_test.cc
+++ b/tensorflow/lite/kernels/resize_bilinear_test.cc
@@ -26,8 +26,8 @@ using uint8 = std::uint8_t;
 
 class ResizeBilinearOpModel : public SingleOpModel {
  public:
-  ResizeBilinearOpModel(const TensorData& input,
-                        std::initializer_list<int> size_data = {}) {
+  explicit ResizeBilinearOpModel(const TensorData& input,
+                                 std::initializer_list<int> size_data = {}) {
     bool const_size = size_data.size() != 0;
     input_ = AddInput(input);
     if (const_size) {
-- 
GitLab


From 48662695e0cabea065de19958132789d931ac78c Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 5 Dec 2018 11:22:57 -0800
Subject: [PATCH 1426/1554] Update
 tensorflow/contrib/tensorrt/convert/convert_graph.cc

Co-Authored-By: Trevor Morris <tmorris@nvidia.com>
---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 3e599b9174..21c69326df 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -588,7 +588,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   // We don't support segments with no inputs. Fall back to native TF here to
   // avoid crash later. Constant folding should've folded the ops that make up
   // these segments.
-  if (inputs.size() == 0) {
+  if (inputs.empty()) {
     return tensorflow::errors::Internal("Segment has no inputs (possible "
                                         "constfold failure)");
   }
-- 
GitLab


From 4f6613441e7f13abd93568445eb0b4ae69828632 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Thu, 6 Dec 2018 10:16:45 -0800
Subject: [PATCH 1427/1554] Update model save/load for unified_lstm layer.

The bias weights during load is changed a bit. The original approach is to split the weights into half and give them to recurrent bias and input gate bias. In the new approach, since it is unknown that whether the newly constructed layer will be running on CPU or GPU, if we still split the weight into half, then the input_gate_bias value is lost since canonical LSTM only use recurrent bias. The new approach is to give recurrent_bias 100% of the sum, and 0% to input_gate_bias.

PiperOrigin-RevId: 224360388
---
 tensorflow/python/keras/engine/saving.py      | 58 +++++++++++++++++
 tensorflow/python/keras/engine/saving_test.py | 64 +++++++++++++++++++
 tensorflow/python/keras/layers/__init__.py    |  1 +
 3 files changed, 123 insertions(+)

diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 54d9e32fb2..15ba5f78d9 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -551,10 +551,68 @@ def preprocess_weights_for_loading(layer,
       if layer.__class__.__name__ == 'ConvLSTM2D':
         weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
 
+  weights = _convert_unified_lstm_weights(layer, weights)
+
   # convert CuDNN layers
   return _convert_rnn_weights(layer, weights)
 
 
+def _convert_unified_lstm_weights(layer, weights):
+  """Converts weights for Unified LSTM layer.
+
+  The input weights suppose to have 2, 3 or 4 items.
+  1. kernel. (i, f, c, o gates concat among axis 1)
+  2. recurrent_kernel. (i, f, c, o concat among axis 1)
+  3. recurrent_bias. (optional, only available when use bias)
+  4. input_bias (optional, only available when use bias and cudnn).
+  Kernel and recurrent_kernel does not need any conversion. During load(),
+  since the layer could be built with the parameter that does not support the
+  defun approach, it is possible that cudnn_bias variable is not created, or
+  even created but not used during actual run. Because of that, we sum up the
+  value of two biases, and give it to recurrent_bias only. Mathematically, the
+  LSTM is calculated as following formula:
+
+    i_t = sigmoid(w_i * x_t + r_i * h_(t-1) + b_wi + b_ri)
+    f_t = sigmoid(w_f * x_t + r_f * h_(t-1) + b_wf + b_rf)
+    o_t = sigmoid(w_o * x_t + r_o * h_(t-1) + b_wo + b_ro)
+    c'_t = tanh(w_c * x_t + r_c * h_(t-1) + b_wc + b_rc)
+    c_t = f_t . c_(t-1) + i_t . c'_t
+    h_t = o_t . tanh(c_t)
+
+  Note that b_w{x} is the input_bias, and b_r{x} is the recurrent_bias.
+  Since it is a linear add, it is fine to give b_r{x} 100% and b_w{x} 0%, as
+  long as the sum are the same.
+
+  Args:
+    layer: The keras layer that will be loaded with weights.
+    weights: the list of numpy arrays which hold the weights to be loaded.
+
+  Returns:
+    weights: the processed list of numpy arrays.
+  """
+  if layer.__class__.__name__ == 'UnifiedLSTM':
+    if len(weights) not in [3, 4]:
+      # Only handles the bias conversion in this function, in the case when
+      # bias is not used or weights in unexpected length, do nothing and return.
+      return weights
+
+    if len(weights) == 3:
+      recurrent_bias = weights[2]
+    else:
+      # Add all the bias value to recurrent_bias
+      recurrent_bias = weights[2] + weights[3]
+
+    if len(layer.weights) == 3:
+      weights = weights[:2] + [recurrent_bias]
+    elif len(layer.weights) == 4:
+      # Create a zero filled input_bias, since all the weights have given
+      # to recurrent bias.
+      input_bias = np.zeros_like(recurrent_bias)
+      weights = weights[:2] + [recurrent_bias, input_bias]
+
+  return weights
+
+
 def _convert_rnn_weights(layer, weights):
   """Converts weights for RNN layers between native and CuDNN format.
 
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py
index 6d9d9a2fca..8fcefce748 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/engine/saving_test.py
@@ -221,6 +221,70 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
           for (x, y) in zip(weights1, weights2)
       ]
 
+  @parameterized.named_parameters(
+      # test_name, use_bias, bias_initializer, activation
+      ('normal', True, 'zeros', 'tanh'),
+      ('no_bias', False, 'zeros', 'tanh'),
+      # TODO(scottzhu): Reenable this test case when the approach is decided.
+      # ('random_bias', True, 'random_uniform', 'tanh'),
+      ('no_cudnn_bias', True, 'zeros', 'relu')
+  )
+  def test_process_weights_for_loading_unified_lstm(
+      self, use_bias, bias_initializer, activation):
+    if h5py is None:
+      return
+
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir)
+    h5_path = os.path.join(temp_dir, 'test.h5')
+
+    batch = 10
+    timestep = 3
+    input_dim = 5
+    units = 2
+
+    x = np.random.random((batch, timestep, input_dim))
+
+    def build_model():
+      inputs = keras.layers.Input(
+          shape=[timestep, input_dim], dtype=dtypes.float32)
+      layer = keras.layers.UnifiedLSTM(
+          units,
+          activation=activation,
+          use_bias=use_bias,
+          bias_initializer=bias_initializer)
+      output = layer(inputs)
+      return keras.models.Model(inputs, output), layer
+
+    with self.cached_session():
+      model, layer = build_model()
+      y_ref = model.predict(x)
+      model.save_weights(h5_path)
+
+      cloned_model, new_layer = build_model()
+      cloned_model.load_weights(h5_path)
+      y = cloned_model.predict(x)
+
+      self.assertAllClose(y, y_ref)
+
+      # Test the individual layer weights.
+      weights1 = layer.get_weights()
+      weights2 = new_layer.get_weights()
+      self.assertLen(weights1, len(weights2))
+      # kernel and current kernel should be the same.
+      self.assertAllClose(weights1[:2], weights2[:2])
+
+      if len(weights2) >= 3:
+        # Test recurrent bias
+        expected_recurrent_bias = weights1[2]
+        if len(weights1) == 4:
+          expected_recurrent_bias += weights1[3]
+        self.assertAllClose(weights2[2], expected_recurrent_bias)
+
+      if len(weights2) == 4:
+        # Test recovered input_gate_bias to be always zero
+        self.assertAllClose(weights2[3], np.zeros_like(weights1[3]))
+
   def test_sequential_weight_loading(self):
     if h5py is None:
       return
diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py
index 49990b6bf4..df7571e5d5 100644
--- a/tensorflow/python/keras/layers/__init__.py
+++ b/tensorflow/python/keras/layers/__init__.py
@@ -149,6 +149,7 @@ from tensorflow.python.keras.layers.recurrent import PeepholeLSTMCell
 from tensorflow.python.keras.layers.recurrent import SimpleRNN
 from tensorflow.python.keras.layers.recurrent import GRU
 from tensorflow.python.keras.layers.recurrent import LSTM
+from tensorflow.python.keras.layers.recurrent import UnifiedLSTM
 
 # Convolutional-recurrent layers.
 from tensorflow.python.keras.layers.convolutional_recurrent import ConvLSTM2D
-- 
GitLab


From e576acf5dbd7b800d3b6aa4de4b69952a9e2c0fb Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Thu, 6 Dec 2018 10:28:10 -0800
Subject: [PATCH 1428/1554] Internal-only change.

PiperOrigin-RevId: 224362520
---
 .../distribute/cluster_resolver/tpu_cluster_resolver.py      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index fe8e09f561..1fd674c8a4 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -197,13 +197,14 @@ class TPUClusterResolver(ClusterResolver):
     elif tpu == 'local' or not tpu:
       # Google environment, where the TPU is attached to the host.
       self._environment = 'google'
-    elif tpu.startswith('/bns'):
+    elif tpu.startswith('/bns') or tpu.startswith('uptc://'):
       # Google environment, where we reach the TPU through BNS.
       self._environment = 'google'
 
     # If TPU is in the Google environment or exists locally, we don't use any
     # RPC layer.
-    if tpu.startswith('/bns') or tpu == 'local' or not tpu:
+    if tpu.startswith('/bns') or tpu.startswith(
+        'uptc://') or tpu == 'local' or not tpu:
       self.rpc_layer = None
     else:
       self.rpc_layer = 'grpc'
-- 
GitLab


From 05b2aaacf1f62ad08d4d37c6fccf009482c3e086 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 6 Dec 2018 10:36:56 -0800
Subject: [PATCH 1429/1554] Automated rollback of commit
 8f6c5d3252cf1fa7c97c46c93b55803660487136

PiperOrigin-RevId: 224364244
---
 tensorflow/python/eager/pywrap_tensor.cc |  8 ++++++++
 tensorflow/python/eager/tensor_test.py   | 12 ++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 0d0f70d543..30a93fb0e4 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -220,6 +220,14 @@ TFE_TensorHandle* ConvertToEagerTensor(PyObject* value, PyObject* dtype) {
       return nullptr;
     }
   }
+  tensorflow::Safe_PyObjectPtr value_decrefer;
+  if (PyArray_IsScalar(value, Generic)) {
+    // Convert numpy scalars to numpy arrays.
+    value = PyArray_FromScalar(value, nullptr);
+    // The returned value needs to be DECREF'd, but the original value was
+    // created in python code, and doesn't need to be DECREF'd.
+    value_decrefer.reset(value);
+  }
   if (PyArray_Check(value)) {
     int desired_np_dtype = -1;
     if (desired_dtype >= 0) {
diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index 25442ff048..0ee2ff68c2 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -95,6 +95,18 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     t = _create_tensor(values)
     self.assertAllEqual(values, t)
 
+  @test_util.assert_no_new_pyobjects_executing_eagerly
+  def testNumpyDtypeSurvivesThroughTensorConversion(self):
+    scalar_creators = [np.int32, np.int64, np.float32, np.float64]
+    conversion_functions = [ops.convert_to_tensor, constant_op.constant]
+
+    for scalar_creator in scalar_creators:
+      for conversion_function in conversion_functions:
+        np_val = scalar_creator(3)
+        tensor_val = conversion_function(np_val)
+        self.assertEqual(tensor_val.numpy().dtype, np_val.dtype)
+        self.assertEqual(tensor_val.numpy(), np_val)
+
   def testNumpyValueWithCast(self):
     values = np.array([3.0], dtype=np.float32)
     t = _create_tensor(values, dtype=dtypes.float64)
-- 
GitLab


From 2bf928bcbbd51d31b817360a34d6b2b9881f7539 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 10:52:09 -0800
Subject: [PATCH 1430/1554] Disabling flaky tests in LossWeightingTest.

PiperOrigin-RevId: 224367330
---
 tensorflow/python/keras/engine/training_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index bd3d7d26d3..c3c3f06ffd 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -793,7 +793,8 @@ class TestExceptionsAndWarnings(keras_parameterized.TestCase):
 class LossWeightingTest(keras_parameterized.TestCase):
 
   @keras_parameterized.run_all_keras_modes
-  def test_class_weights(self):
+  # TODO(b/120562577): Test failing with assertion error.
+  def DISABLED_test_class_weights(self):
     num_classes = 5
     batch_size = 5
     epochs = 5
@@ -960,7 +961,8 @@ class LossWeightingTest(keras_parameterized.TestCase):
       self.assertTrue(msg_found)
 
   @keras_parameterized.run_all_keras_modes
-  def test_temporal_sample_weights(self):
+  # TODO(b/120562577): Test failing with assertion error.
+  def DISABLED_test_temporal_sample_weights(self):
     num_classes = 5
     batch_size = 5
     epochs = 5
-- 
GitLab


From 39c50c9bcbe49a5852f3dd1085e6dc143424a7a0 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 6 Dec 2018 10:52:26 -0800
Subject: [PATCH 1431/1554] [tf.data] Enable `noop_elimination` optimization by
 default.

PiperOrigin-RevId: 224367395
---
 .../core/kernels/data/optimize_dataset_op.cc  |  5 +--
 .../optimization/optimize_dataset_test.py     | 30 +++++++++++++++-
 .../kernel_tests/stats_dataset_ops_test.py    |  9 ++---
 .../experimental/ops/optimization_options.py  | 34 +++++++++++++++++++
 .../data/experimental/ops/prefetching_ops.py  |  4 +++
 tensorflow/python/data/ops/dataset_ops.py     | 31 +++++------------
 tensorflow/python/data/ops/iterator_ops.py    |  2 ++
 .../data/ops/multi_device_iterator_ops.py     |  7 +++-
 ...a.experimental.-optimization-options.pbtxt |  4 +++
 ...a.experimental.-optimization-options.pbtxt |  4 +++
 10 files changed, 100 insertions(+), 30 deletions(-)

diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index ab184c232e..9c50d8050a 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -305,8 +305,9 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
         // removing unused graph nodes)
         // TODO(b/118175421): This should be part of the tf.data optimization
         // pass manager.
-        for (const auto& optimizer : {"pruning", "function", "constfold",
-                                      "shape", "arithmetic", "dependency"}) {
+        // TODO(b/120437209): Apply `constfold` optimization when it is fixed.
+        for (const auto& optimizer :
+             {"pruning", "function", "shape", "arithmetic", "dependency"}) {
           rewriter_config.add_optimizers(optimizer);
         }
       }
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 751be83326..150cc7b5e4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -163,7 +163,15 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset_ops.Dataset.range(1)
     dataset = dataset.flat_map(flat_map_fn)
-    dataset = dataset_ops._OptimizeDataset(dataset, ["map_and_batch_fusion"])
+
+    # TODO(b/120558523): We use Options instead of _OptimizeDataset directly
+    # here because of a bug with chaining _OptimizeDatasets when there are
+    # nested dataset functions
+    options = dataset_ops.Options()
+    opt_options = optimization_options.OptimizationOptions()
+    opt_options.map_and_batch_fusion = True
+    options.experimental_optimization = opt_options
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[[0]])
 
   def testOptimizationThreadPoolDataset(self):
@@ -245,6 +253,26 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       except errors.OutOfRangeError:
         break
 
+  def testOptimizationEnabledByDefault(self):
+    """Tests that some optimizations are applied to datasets by default."""
+    options = dataset_ops.Options()
+    expected_optimizations = ["noop_elimination"]
+    self.assertEqual(options._static_optimizations(), expected_optimizations)
+
+  def testOptimizationDisableDefault(self):
+    """Tests that we can disable all static optimizations enabled by default.
+
+    If the `apply_default_optimizations` optimization options flag is False,
+    only explicitly enabled optimizations will be applied.
+    """
+    options = dataset_ops.Options()
+    opt_options = optimization_options.OptimizationOptions()
+    opt_options.hoist_random_uniform = True
+    opt_options.apply_default_optimizations = False
+    options.experimental_optimization = opt_options
+    expected_optimizations = ["hoist_random_uniform"]
+    self.assertEqual(options._static_optimizations(), expected_optimizations)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
index 8a300364f9..f19b08a2dd 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_ops_test.py
@@ -138,9 +138,10 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
 
   @test_util.run_deprecated_v1
   def testPrefetchBufferScalars(self, dataset_transformation):
+    def map_fn(x):
+      return array_ops.tile([x], ops.convert_to_tensor([x]))
     aggregator = stats_aggregator.StatsAggregator()
-    dataset = dataset_ops.Dataset.range(10).map(
-        lambda x: array_ops.tile([x], ops.convert_to_tensor([x]))).prefetch(0)
+    dataset = dataset_ops.Dataset.range(10).map(map_fn).prefetch(1)
     dataset = dataset_transformation(dataset, aggregator)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
@@ -153,9 +154,9 @@ class StatsDatasetTest(stats_dataset_test_base.StatsDatasetTestBase):
             np.array([i] * i, dtype=np.int64), self.evaluate(next_element))
         summary_str = self.evaluate(summary_t)
         self._assertSummaryHasScalarValue(summary_str,
-                                          "Prefetch::buffer_capacity", 0)
+                                          "Prefetch::buffer_capacity", 1)
         self._assertSummaryHasScalarValue(summary_str, "Prefetch::buffer_size",
-                                          0)
+                                          1)
       with self.assertRaises(errors.OutOfRangeError):
         self.evaluate(next_element)
 
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index dc9d319374..daf65cd5cd 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -35,6 +35,12 @@ class OptimizationOptions(options.OptionsBase):
   dataset = dataset.with_options(options)
   ```
   """
+  apply_default_optimizations = options.create_option(
+      name="apply_default_optimizations",
+      ty=bool,
+      docstring=
+      "Whether to apply default static optimizations. If False, only static "
+      "optimizations that have been explicitly enabled will be applied.")
 
   filter_fusion = options.create_option(
       name="filter_fusion",
@@ -81,3 +87,31 @@ class OptimizationOptions(options.OptionsBase):
       name="shuffle_and_repeat_fusion",
       ty=bool,
       docstring="Whether to fuse shuffle and repeat transformations.")
+
+  def _static_optimizations(self):
+    """Produces the list of enabled static optimizations."""
+    result = []
+    optimizations_to_enable = [
+        "filter_fusion",
+        "hoist_random_uniform",
+        "map_and_batch_fusion",
+        "map_and_filter_fusion",
+        "map_fusion",
+        "map_parallelization",
+        "map_vectorization",
+        "shuffle_and_repeat_fusion",
+    ]
+    for optimization in optimizations_to_enable:
+      if getattr(self, optimization):
+        result.append(optimization)
+
+    if self.apply_default_optimizations is not False:
+      # The following optimizations are turned on by default, unless the
+      # user explicitly disables them.
+      optimizations_to_disable = [
+          "noop_elimination",
+      ]
+      for optimization in optimizations_to_disable:
+        if getattr(self, optimization) is not False:
+          result.append(optimization)
+    return result
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index 50d9803c34..aba1786646 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.util import nest
@@ -73,6 +74,9 @@ def copy_to_device(target_device, source_device="/cpu:0"):
   def _apply_fn(dataset):
     options = dataset_ops.Options()
     options.experimental_autotune = False
+    opt_options = optimization_options.OptimizationOptions()
+    opt_options.apply_default_optimizations = False
+    options.experimental_optimization = opt_options
     return _CopyToDeviceDataset(
         dataset, target_device=target_device,
         source_device=source_device).with_options(options)
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index d7a2547fc8..a08a2b5787 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -187,8 +187,7 @@ class DatasetV2(object):
       RuntimeError: If eager execution is not enabled.
     """
     if context.executing_eagerly():
-      dataset = self._apply_options()
-      return iterator_ops.EagerIterator(dataset)
+      return iterator_ops.EagerIterator(self)
     else:
       raise RuntimeError("dataset.__iter__() is only supported when eager "
                          "execution is enabled.")
@@ -1318,8 +1317,7 @@ class DatasetV1(DatasetV2):
       An `Iterator` over the elements of this dataset.
     """
     if context.executing_eagerly():
-      dataset = self._apply_options()
-      return iterator_ops.EagerIterator(dataset)
+      return iterator_ops.EagerIterator(self)
 
     graph_level_seed, op_level_seed = core_random_seed.get_seed(None)
 
@@ -1702,8 +1700,8 @@ class Options(options_lib.OptionsBase):
       name="experimental_deterministic",
       ty=bool,
       docstring=
-      "Whether to dynamically adjust the values of tunable parameters (e.g. "
-      "degrees of parallelism).")
+      "Whether the outputs need to be produced in deterministic order."
+  )
 
   experimental_numa_aware = options_lib.create_option(
       name="experimental_numa_aware",
@@ -1729,22 +1727,11 @@ class Options(options_lib.OptionsBase):
     """Produces the list of enabled static optimizations."""
 
     result = []
-    exp_optimization_options = self.experimental_optimization
-    if exp_optimization_options:
-      optimizations = [
-          "filter_fusion",
-          "hoist_random_uniform",
-          "map_and_batch_fusion",
-          "map_and_filter_fusion",
-          "map_fusion",
-          "map_parallelization",
-          "map_vectorization",
-          "noop_elimination",
-          "shuffle_and_repeat_fusion",
-      ]
-      for optimization in optimizations:
-        if getattr(exp_optimization_options, optimization):
-          result.append(optimization)
+    exp_optimization_options = (
+        self.experimental_optimization or
+        optimization_options.OptimizationOptions())  # If not set, use default
+    result.extend(exp_optimization_options._static_optimizations())  # pylint: disable=protected-access
+
     if self.experimental_numa_aware:
       result.append("make_numa_aware")
     if self.experimental_deterministic is False:
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index eb330d83ae..6f9c494f39 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -528,6 +528,7 @@ class EagerIterator(checkpointable.CheckpointableBase):
     self._device = context.context().device_name
     with ops.device("/cpu:0"):
       # pylint: disable=protected-access
+      dataset = dataset._apply_options()
       ds_variant = dataset._as_variant_tensor()
       self._structure = structure_lib.convert_legacy_structure(
           dataset.output_types, dataset.output_shapes, dataset.output_classes)
@@ -541,6 +542,7 @@ class EagerIterator(checkpointable.CheckpointableBase):
         # Delete the resource when this object is deleted
         self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
             handle=self._resource, handle_device=self._device)
+      # pylint: enable=protected-access
 
   def __iter__(self):
     return self
diff --git a/tensorflow/python/data/ops/multi_device_iterator_ops.py b/tensorflow/python/data/ops/multi_device_iterator_ops.py
index 4638cee02a..7586012574 100644
--- a/tensorflow/python/data/ops/multi_device_iterator_ops.py
+++ b/tensorflow/python/data/ops/multi_device_iterator_ops.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
@@ -192,9 +193,13 @@ class MultiDeviceIterator(object):
           self._source_device_tensor, device, dataset._element_structure)  # pylint: disable=protected-access
       if prefetch_buffer_size > 0:
         ds = ds.prefetch(prefetch_buffer_size)
-      # TODO(jsimsa): Enable auto-tuning when supported for non-CPU devices.
+      # TODO(jsimsa): Enable auto-tuning and optimizations when supported for
+      # non-CPU devices.
       options = dataset_ops.Options()
       options.experimental_autotune = False
+      opt_options = optimization_options.OptimizationOptions()
+      opt_options.apply_default_optimizations = False
+      options.experimental_optimization = opt_options
       ds = ds.with_options(options)
       with ops.device(device):
         self._device_iterators.append(ds.make_initializable_iterator())
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt
index 9ca75828e5..3b7ad64f51 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.data.experimental.ops.optimization_options.OptimizationOptions\'>"
   is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "apply_default_optimizations"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "filter_fusion"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt
index 9ca75828e5..3b7ad64f51 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.data.experimental.ops.optimization_options.OptimizationOptions\'>"
   is_instance: "<class \'tensorflow.python.data.util.options.OptionsBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "apply_default_optimizations"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "filter_fusion"
     mtype: "<type \'property\'>"
-- 
GitLab


From 76e1a2a4767d8c697a7594b816bee13d401666a4 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 6 Dec 2018 11:07:20 -0800
Subject: [PATCH 1432/1554] [XLA:GPU] Add a flag to disable ptxas optimizations

PiperOrigin-RevId: 224370488
---
 .../compiler/xla/debug_options_flags.cc       |  6 ++++++
 .../xla/service/gpu/nvptx_compiler.cc         | 21 ++++++++++++-------
 .../compiler/xla/service/gpu/nvptx_compiler.h |  5 +++--
 tensorflow/compiler/xla/xla.proto             |  3 +++
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/debug_options_flags.cc b/tensorflow/compiler/xla/debug_options_flags.cc
index d7e7b9e621..20609cad58 100644
--- a/tensorflow/compiler/xla/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/debug_options_flags.cc
@@ -334,6 +334,12 @@ void AllocateFlags() {
           "overhead from context switching but we let the user override this "
           "behavior to help run tests on the host that run models in parallel "
           "across multiple devices."),
+      tensorflow::Flag(
+          "xla_gpu_disable_ptxas_optimizations",
+          bool_setter_for(
+              &DebugOptions::set_xla_gpu_disable_ptxas_optimizations),
+          flag_values->xla_gpu_disable_ptxas_optimizations(),
+          "In XLA:GPU run ptxas in -O0 (default is -O3)."),
   });
   ParseFlagsFromEnvAndDieIfUnknown("XLA_FLAGS", *flag_objects);
 }
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index e934cbda17..f3e17d8882 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -479,7 +479,8 @@ void WarnIfBadDriverJITVersion() {
 // Compiles the given PTX string using ptxas and returns the resulting machine
 // code (i.e. a cubin) as a byte array.
 StatusOr<std::vector<uint8>> CompilePtx(const string& ptx, int cc_major,
-                                        int cc_minor) {
+                                        int cc_minor,
+                                        bool disable_ptx_optimizations) {
   tracing::ScopedActivity activity("Compile PTX", /*is_expensive=*/true);
   const string ptxas_path =
       tensorflow::io::JoinPath(tensorflow::CudaRoot(), "bin", "ptxas");
@@ -519,6 +520,9 @@ StatusOr<std::vector<uint8>> CompilePtx(const string& ptx, int cc_major,
   if (VLOG_IS_ON(2)) {
     ptxas_args.push_back("-v");
   }
+  if (disable_ptx_optimizations) {
+    ptxas_args.push_back("-O0");
+  }
   ptxas_info_dumper.SetProgram(ptxas_path, ptxas_args);
   ptxas_info_dumper.SetChannelAction(tensorflow::CHAN_STDERR,
                                      tensorflow::ACTION_PIPE);
@@ -739,8 +743,9 @@ StatusOr<std::unique_ptr<Executable>> NVPTXCompiler::RunBackend(
     }
   }
 
-  const std::vector<uint8> cubin =
-      CompilePtxOrGetCachedResult(ptx, cc_major, cc_minor);
+  const std::vector<uint8> cubin = CompilePtxOrGetCachedResult(
+      ptx, cc_major, cc_minor,
+      module->config().debug_options().xla_gpu_disable_ptxas_optimizations());
 
   auto thunk_schedule = absl::make_unique<ThunkSchedule>(
       ir_emitter.ConsumeThunkSequence(), std::move(stream_assignment),
@@ -772,9 +777,9 @@ StatusOr<std::unique_ptr<Executable>> NVPTXCompiler::RunBackend(
   return std::unique_ptr<Executable>(gpu_executable);
 }
 
-std::vector<uint8> NVPTXCompiler::CompilePtxOrGetCachedResult(const string& ptx,
-                                                              int cc_major,
-                                                              int cc_minor) {
+std::vector<uint8> NVPTXCompiler::CompilePtxOrGetCachedResult(
+    const string& ptx, int cc_major, int cc_minor,
+    bool disable_ptx_optimizations) {
   XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::CompilePtxOrGetCachedResult");
   tracing::ScopedActivity activity("PTX->CUBIN", /*is_expensive=*/true);
   bool inserted;
@@ -802,8 +807,8 @@ std::vector<uint8> NVPTXCompiler::CompilePtxOrGetCachedResult(const string& ptx,
     if (inserted) {
       CHECK(!cache_value->compilation_done);
       if (!ptx.empty()) {
-        StatusOr<std::vector<uint8>> maybe_cubin =
-            CompilePtx(*cache_ptx, cc_major, cc_minor);
+        StatusOr<std::vector<uint8>> maybe_cubin = CompilePtx(
+            *cache_ptx, cc_major, cc_minor, disable_ptx_optimizations);
         if (maybe_cubin.ok()) {
           cache_value->cubin_data = std::move(maybe_cubin).ValueOrDie();
           VLOG(2) << "Compiled PTX size:" << ptx.size()
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h
index f79ae2990a..be5e31a501 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h
@@ -97,8 +97,9 @@ class NVPTXCompiler : public LLVMCompiler {
 
   // Tries to compile the given ptx string to cubin.  Returns a vector with the
   // compiled cubin.  If compilation was unsuccessful, returns an empty vector.
-  std::vector<uint8> CompilePtxOrGetCachedResult(const string& ptx,
-                                                 int cc_major, int cc_minor);
+  std::vector<uint8> CompilePtxOrGetCachedResult(
+      const string& ptx, int cc_major, int cc_minor,
+      bool disable_ptx_optimizations);
 
   // The compilation_cache_ map is a cache from {ptx string, cc_major, cc_minor}
   // -> cubin so we don't recompile the same ptx twice.  This is important for
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index bdeb1728fa..a37eac7fe4 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -213,6 +213,9 @@ message DebugOptions {
   // the host that run models in parallel across multiple devices.
   int32 xla_force_host_platform_device_count = 102;
 
+  // If set to true XLA:GPU invokes `ptxas` with -O0 (default is -O3).
+  bool xla_gpu_disable_ptxas_optimizations = 103;
+
   // Extra options to pass to the compilation backend (e.g. LLVM); specific
   // interpretation of these values is left to the backend.
   map<string, string> xla_backend_extra_options = 500;
-- 
GitLab


From d3106e4138e28f62179cd6a7b8041729f5163190 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 6 Dec 2018 11:07:35 -0800
Subject: [PATCH 1433/1554] Change num_accelerators_per_worker to
 num_accelerators within Cluster Resolver as we will support querying remote,
 heterogenous GPU clusters soon.

PiperOrigin-RevId: 224370540
---
 .../cluster_resolver/cluster_resolver.py      | 57 ++++++++++++++-----
 .../cluster_resolver/cluster_resolver_test.py | 12 ++--
 .../cluster_resolver/gce_cluster_resolver.py  | 16 ++++--
 .../kubernetes_cluster_resolver.py            | 15 ++++-
 .../slurm_cluster_resolver.py                 |  9 ++-
 .../tfconfig_cluster_resolver.py              | 19 +++++--
 .../tfconfig_cluster_resolver_test.py         |  4 +-
 .../cluster_resolver/tpu_cluster_resolver.py  | 21 +++++--
 8 files changed, 111 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
index 7774ac0e12..ca40e60a55 100644
--- a/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver.py
@@ -92,7 +92,11 @@ class ClusterResolver(object):
     raise NotImplementedError()
 
   @abc.abstractmethod
-  def num_accelerators_per_worker(self, session_config=None):
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
     """Returns the number of accelerator cores per worker.
 
     This returns the number of accelerator cores (such as GPUs and TPUs)
@@ -100,8 +104,19 @@ class ClusterResolver(object):
     should return 0. This method will query the master for this information
     if it is not otherwise known.
 
+    Optionally, we allow callers to specify the task_type, task_index, and
+    rpc_layer, if they want to target a specific TensorFlow process to query
+    the number of accelerators. This is to support heterogenous environments,
+    where the number of accelerators cores per host is different.
+
     Args:
-      session_config: (Optional) Configuration for starting a new session to
+      task_type: (Optional) The type of the TensorFlow task of the machine we
+        want to query.
+      task_index: (Optional) The index of the TensorFlow task of the machine we
+        want to query.
+      accelerator_type: (Optional) The type of accelerator we are trying to
+        query (defaults to 'GPU').
+      config_proto: (Optional) Configuration for starting a new session to
         query how many accelerator cores it has.
     """
     raise NotImplementedError()
@@ -116,7 +131,7 @@ class SimpleClusterResolver(ClusterResolver):
   """Simple implementation of ClusterResolver that accepts a ClusterSpec."""
 
   def __init__(self, cluster_spec, master='', task_type=None, task_index=None,
-               environment='', num_accelerators_per_worker=0,
+               environment='', num_accelerators=0,
                rpc_layer=None):
     """Creates a SimpleClusterResolver from a ClusterSpec."""
     super(SimpleClusterResolver, self).__init__()
@@ -124,7 +139,7 @@ class SimpleClusterResolver(ClusterResolver):
     self._task_type = task_type
     self._task_index = task_index
     self._environment = environment
-    self._num_accelerators_per_worker = num_accelerators_per_worker
+    self._num_accelerators = num_accelerators
     self._rpc_layer = rpc_layer
 
     if not isinstance(cluster_spec, ClusterSpec):
@@ -180,17 +195,27 @@ class SimpleClusterResolver(ClusterResolver):
   def environment(self):
     return self._environment
 
-  def num_accelerators_per_worker(self, session_config=None):
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
     """Returns the number of accelerator cores per worker.
 
+    The SimpleClusterResolver does not do automatic detection of accelerators,
+    so a TensorFlow session will never be created, and thus all arguments are
+    unused and we simply return whatever was passed in when this object was
+    initialized.
+
     Args:
-      session_config: Unused. The SimpleClusterResolver does not do automatic
-        detection of accelerators, so a TensorFlow session will never be
-        created, and thus a `session_config` is never necessary here, and will
-        be ignored.
+      task_type: Unused.
+      task_index: Unused.
+      accelerator_type: Unused.
+      config_proto: Unused.
     """
-    del session_config
-    return self._num_accelerators_per_worker
+    # Unused
+    del task_type, task_index, accelerator_type, config_proto
+    return self._num_accelerators
 
   @property
   def rpc_layer(self):
@@ -361,9 +386,13 @@ class UnionClusterResolver(ClusterResolver):
   def environment(self):
     return self._cluster_resolvers[0].environment
 
-  def num_accelerators_per_worker(self, session_config=None):
-    return self._cluster_resolvers[0].num_accelerators_per_worker(
-        session_config)
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
+    return self._cluster_resolvers[0].num_accelerators(
+        task_type, task_index, accelerator_type, config_proto)
 
   @property
   def rpc_layer(self):
diff --git a/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
index b5448faec6..3f7b469727 100644
--- a/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/cluster_resolver_test.py
@@ -65,13 +65,13 @@ class UnionClusterResolverTest(test.TestCase):
 
     simple_resolver = SimpleClusterResolver(base_cluster_spec, task_type="ps",
                                             task_index=1, environment="cloud",
-                                            num_accelerators_per_worker=8,
+                                            num_accelerators=8,
                                             rpc_layer="grpc")
 
     self.assertEqual(simple_resolver.task_type, "ps")
     self.assertEqual(simple_resolver.task_index, 1)
     self.assertEqual(simple_resolver.environment, "cloud")
-    self.assertEqual(simple_resolver.num_accelerators_per_worker(), 8)
+    self.assertEqual(simple_resolver.num_accelerators(), 8)
     self.assertEqual(simple_resolver.rpc_layer, "grpc")
 
   def testOverrideSimpleClusterResolver(self):
@@ -82,7 +82,7 @@ class UnionClusterResolverTest(test.TestCase):
 
     simple_resolver = SimpleClusterResolver(base_cluster_spec, task_type="ps",
                                             task_index=1, environment="cloud",
-                                            num_accelerators_per_worker=8,
+                                            num_accelerators=8,
                                             rpc_layer="grpc")
 
     simple_resolver.task_type = "worker"
@@ -130,7 +130,7 @@ class UnionClusterResolverTest(test.TestCase):
     })
     resolver1 = SimpleClusterResolver(cluster_spec_1, task_type="ps",
                                       task_index=1, environment="cloud",
-                                      num_accelerators_per_worker=8,
+                                      num_accelerators=8,
                                       rpc_layer="grpc")
 
     cluster_spec_2 = server_lib.ClusterSpec({
@@ -139,7 +139,7 @@ class UnionClusterResolverTest(test.TestCase):
     })
     resolver2 = SimpleClusterResolver(cluster_spec_2, task_type="worker",
                                       task_index=2, environment="local",
-                                      num_accelerators_per_worker=16,
+                                      num_accelerators=16,
                                       rpc_layer="http")
 
     union_resolver = UnionClusterResolver(resolver1, resolver2)
@@ -147,7 +147,7 @@ class UnionClusterResolverTest(test.TestCase):
     self.assertEqual(union_resolver.task_type, "ps")
     self.assertEqual(union_resolver.task_index, 1)
     self.assertEqual(union_resolver.environment, "cloud")
-    self.assertEqual(union_resolver.num_accelerators_per_worker(), 8)
+    self.assertEqual(union_resolver.num_accelerators(), 8)
     self.assertEqual(union_resolver.rpc_layer, "grpc")
 
     union_resolver.task_type = "worker"
diff --git a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
index b167bc8fc8..2412f6dad0 100644
--- a/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py
@@ -51,7 +51,7 @@ class GceClusterResolver(ClusterResolver):
                task_type='worker',
                task_index=0,
                rpc_layer='grpc',
-               num_accelerators_per_worker=0,
+               num_accelerators=0,
                credentials='default',
                service=None):
     """Creates a new GceClusterResolver object.
@@ -73,7 +73,7 @@ class GceClusterResolver(ClusterResolver):
         can be distinguished from each other.
       rpc_layer: The RPC layer TensorFlow should use to communicate across
         instances.
-      num_accelerators_per_worker: Number of accelerators (GPUs) present per
+      num_accelerators: Number of accelerators (GPUs) present per
         instance.
       credentials: GCE Credentials. If nothing is specified, this defaults to
         GoogleCredentials.get_application_default().
@@ -90,6 +90,7 @@ class GceClusterResolver(ClusterResolver):
     self._task_type = task_type
     self._task_index = task_index
     self._rpc_layer = rpc_layer
+    self._num_accelerators = num_accelerators
     self._port = port
     self._credentials = credentials
 
@@ -201,6 +202,11 @@ class GceClusterResolver(ClusterResolver):
   def rpc_layer(self, rpc_layer):
     self._rpc_layer = rpc_layer
 
-  def num_accelerators_per_worker(self, session_config=None):
-    del session_config  # Unused, since this is set manually in __init__.
-    return self._num_accelerators_per_worker
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
+    # Unused
+    del task_type, task_index, accelerator_type, config_proto
+    return self._num_accelerators
diff --git a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
index 041c081540..b21c3676be 100644
--- a/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/kubernetes_cluster_resolver.py
@@ -168,6 +168,15 @@ class KubernetesClusterResolver(ClusterResolver):
     """
     return ''
 
-  def num_accelerators_per_worker(self, session_config=None):
-    local_devices = device_lib.list_local_devices(session_config)
-    return len([d for d in local_devices if d.device_type == 'GPU'])
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
+    # TODO(frankchn): Make querying non-local accelerators work
+    if task_type is not None or task_index is not None:
+      raise NotImplementedError('Querying non-local accelerators is not yet'
+                                'implemented.')
+
+    local_devices = device_lib.list_local_devices(config_proto)
+    return sum(d.device_type == accelerator_type for d in local_devices)
diff --git a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
index fd3c6d6a18..1ab81731b7 100644
--- a/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/slurm_cluster_resolver.py
@@ -221,6 +221,11 @@ class SlurmClusterResolver(ClusterResolver):
     """
     return ''
 
-  def num_accelerators_per_worker(self, session_config=None):
-    del session_config  # Unused, since this is set in __init__ manually.
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
+    # Unused, since this is set in __init__ manually.
+    del task_type, task_index, accelerator_type, config_proto
     return self._gpus_per_node
diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
index a3246e77f4..b4465714b2 100644
--- a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver.py
@@ -55,7 +55,7 @@ class TFConfigClusterResolver(ClusterResolver):
                task_index=None,
                rpc_layer=None,
                environment=None,
-               num_accelerators_per_worker=0):
+               num_accelerators=0):
     """Creates a new TFConfigClusterResolver.
 
     Args:
@@ -66,15 +66,17 @@ class TFConfigClusterResolver(ClusterResolver):
       rpc_layer: (String, optional) Overrides the rpc layer TensorFlow uses.
       environment: (String, optional) Overrides the environment TensorFlow
         operates in.
-      num_accelerators_per_worker: (Integer, optional) Specifies the number of
+      num_accelerators: (Integer, optional) Specifies the number of
         accelerators (e.g. GPUs, TPUs, others) that each node has.
     """
+    # TODO(frankchn): num_accelerators is a stop-gap and will be removed
+    # in favor of autodetection of devices soon.
 
     self._task_type = task_type
     self._task_index = task_index
     self._rpc_layer = rpc_layer
     self._environment = environment
-    self._num_accelerators_per_worker = num_accelerators_per_worker
+    self._num_accelerators = num_accelerators
 
   @property
   def task_type(self):
@@ -115,10 +117,15 @@ class TFConfigClusterResolver(ClusterResolver):
   def rpc_layer(self, rpc_layer):
     self._rpc_layer = rpc_layer
 
-  def num_accelerators_per_worker(self, session_config=None):
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='GPU',
+                       config_proto=None):
     # TODO(frankchn): Connect to server (w/ session_config) in the future.
-    del session_config  # Unused, we do not connect to another server here.
-    return self._num_accelerators_per_worker
+    # Unused, we do not connect to another server here right now.
+    del task_type, task_index, accelerator_type, config_proto
+    return self._num_accelerators
 
   def cluster_spec(self):
     """Returns a ClusterSpec based on the TF_CONFIG environment variable.
diff --git a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
index c20e51bc0b..197eba1739 100644
--- a/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tfconfig_cluster_resolver_test.py
@@ -169,12 +169,12 @@ class TFConfigClusterResolverTest(test.TestCase):
     """
 
     cluster_resolver = TFConfigClusterResolver(task_type='ps', task_index=0,
-                                               num_accelerators_per_worker=8)
+                                               num_accelerators=8)
 
     self.assertEqual('grpc://ps0:2222', cluster_resolver.master())
     self.assertEqual('ps', cluster_resolver.task_type)
     self.assertEqual(0, cluster_resolver.task_index)
-    self.assertEqual(8, cluster_resolver.num_accelerators_per_worker())
+    self.assertEqual(8, cluster_resolver.num_accelerators())
 
     cluster_resolver.task_type = 'worker'
     cluster_resolver.task_index = 1
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 1fd674c8a4..99a7a0922f 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -387,17 +387,30 @@ class TPUClusterResolver(ClusterResolver):
 
     return server_lib.ClusterSpec(cluster_spec)
 
-  def num_accelerators_per_worker(self, session_config=None):
+  def num_accelerators(self,
+                       task_type=None,
+                       task_index=None,
+                       accelerator_type='TPU',
+                       config_proto=None):
     """Returns the number of TPU cores per worker.
 
     This defaults to 8 for all current TPU configurations, and we do not need
     to query any remote systems for this.
 
     Args:
-      session_config: Unused. Not currently necessary to query anything as this
-        number is 8 for all TPU configurations.
+      task_type: Unused.
+      task_index: Unused.
+      accelerator_type: Unused.
+      config_proto: Unused.
+
+    Raises:
+      RuntimeError: If this is used with a non-TPU accelerator_type.
     """
-    del session_config  # Unused. Not necessary to query anything.
+    # Unused. Not necessary to query anything.
+    del task_type, task_index, config_proto
+
+    if accelerator_type != 'TPU':
+      raise ValueError('This Cluster Resolver is only compatible with TPUs.')
     return 8
 
   @property
-- 
GitLab


From e7f4f4d2a662ea24826af717d5ee401e67883b7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 11:08:53 -0800
Subject: [PATCH 1434/1554] Increase timeout to deflake ctc_loss_op_test.

PiperOrigin-RevId: 224370778
---
 tensorflow/python/kernel_tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 242e3b595c..df8c14970a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -272,7 +272,7 @@ tf_py_test(
 
 cuda_py_test(
     name = "ctc_loss_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["ctc_loss_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
-- 
GitLab


From a5c2d3ecf905e310ebf4f73089b0b5bf8b6dcf81 Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Thu, 6 Dec 2018 11:24:25 -0800
Subject: [PATCH 1435/1554] Apply API symbols upgrade for TF 2.0

PiperOrigin-RevId: 224374109
---
 tensorflow/python/ops/array_ops.py            | 20 ++++-
 .../python/ops/candidate_sampling_ops.py      | 11 +--
 tensorflow/python/ops/embedding_ops.py        | 78 ++++++++++++++++++-
 tensorflow/python/ops/nn_impl.py              |  2 +-
 tensorflow/python/ops/nn_ops.py               | 10 ++-
 tensorflow/python/ops/sparse_ops.py           |  4 +-
 .../api/golden/v1/tensorflow.random.pbtxt     | 12 +++
 .../tools/api/golden/v2/tensorflow.math.pbtxt |  2 +-
 .../tools/api/golden/v2/tensorflow.nn.pbtxt   | 14 ++--
 .../api/golden/v2/tensorflow.random.pbtxt     |  4 +
 .../api/golden/v2/tensorflow.sparse.pbtxt     |  2 +-
 tensorflow/tools/compatibility/renames_v2.py  |  5 +-
 tensorflow/tools/compatibility/reorders_v2.py |  8 ++
 .../tools/compatibility/tf_upgrade_v2.py      | 11 +++
 .../tools/compatibility/tf_upgrade_v2_test.py | 48 ++++++++++++
 15 files changed, 205 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index b555f63ceb..185452e1ab 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2667,7 +2667,7 @@ def space_to_batch(input, paddings, block_size, name=None):  # pylint: disable=r
 space_to_batch.__doc__ = gen_array_ops.space_to_batch.__doc__
 
 
-@tf_export("nn.space_to_depth", v1=["nn.space_to_depth", "space_to_depth"])
+@tf_export(v1=["nn.space_to_depth", "space_to_depth"])
 @deprecation.deprecated_endpoints("space_to_depth")
 def space_to_depth(input, block_size, name=None, data_format="NHWC"):  # pylint: disable=redefined-builtin
   return gen_array_ops.space_to_depth(input, block_size, data_format, name=name)
@@ -2676,7 +2676,15 @@ def space_to_depth(input, block_size, name=None, data_format="NHWC"):  # pylint:
 space_to_depth.__doc__ = gen_array_ops.space_to_depth.__doc__
 
 
-@tf_export("nn.depth_to_space", v1=["nn.depth_to_space", "depth_to_space"])
+@tf_export("nn.space_to_depth", v1=[])
+def space_to_depth_v2(input, block_size, data_format="NHWC", name=None):  # pylint: disable=redefined-builtin
+  return gen_array_ops.space_to_depth(input, block_size, data_format, name=name)
+
+
+space_to_depth_v2.__doc__ = gen_array_ops.space_to_depth.__doc__
+
+
+@tf_export(v1=["nn.depth_to_space", "depth_to_space"])
 @deprecation.deprecated_endpoints("depth_to_space")
 def depth_to_space(input, block_size, name=None, data_format="NHWC"):  # pylint: disable=redefined-builtin
   return gen_array_ops.depth_to_space(input, block_size, data_format, name=name)
@@ -2685,6 +2693,14 @@ def depth_to_space(input, block_size, name=None, data_format="NHWC"):  # pylint:
 depth_to_space.__doc__ = gen_array_ops.depth_to_space.__doc__
 
 
+@tf_export("nn.depth_to_space", v1=[])
+def depth_to_space_v2(input, block_size, data_format="NHWC", name=None):  # pylint: disable=redefined-builtin
+  return gen_array_ops.depth_to_space(input, block_size, data_format, name=name)
+
+
+depth_to_space_v2.__doc__ = gen_array_ops.depth_to_space.__doc__
+
+
 @tf_export(v1=["batch_to_space"])
 def batch_to_space(input, crops, block_size, name=None):  # pylint: disable=redefined-builtin
   result = batch_to_space_nd(
diff --git a/tensorflow/python/ops/candidate_sampling_ops.py b/tensorflow/python/ops/candidate_sampling_ops.py
index c64000b65d..56f76a49d5 100644
--- a/tensorflow/python/ops/candidate_sampling_ops.py
+++ b/tensorflow/python/ops/candidate_sampling_ops.py
@@ -151,7 +151,10 @@ def log_uniform_candidate_sampler(true_classes, num_true, num_sampled, unique,
       seed2=seed2, name=name)
 
 
-@tf_export('nn.learned_unigram_candidate_sampler')
+@tf_export(
+    'random.learned_unigram_candidate_sampler',
+    'nn.learned_unigram_candidate_sampler')
+@deprecation.deprecated_endpoints(['nn.learned_unigram_candidate_sampler'])
 def learned_unigram_candidate_sampler(true_classes, num_true, num_sampled,
                                       unique, range_max, seed=None, name=None):
   """Samples a set of classes from a distribution learned during training.
@@ -209,8 +212,7 @@ def learned_unigram_candidate_sampler(true_classes, num_true, num_sampled,
 
 
 @tf_export('random.fixed_unigram_candidate_sampler',
-           'nn.fixed_unigram_candidate_sampler',
-           v1=['nn.fixed_unigram_candidate_sampler'])
+           'nn.fixed_unigram_candidate_sampler')
 def fixed_unigram_candidate_sampler(true_classes,
                                     num_true,
                                     num_sampled,
@@ -302,8 +304,7 @@ def fixed_unigram_candidate_sampler(true_classes,
       unigrams=unigrams, seed=seed1, seed2=seed2, name=name)
 
 
-@tf_export('random.all_candidate_sampler', 'nn.all_candidate_sampler',
-           v1=['nn.all_candidate_sampler'])
+@tf_export('random.all_candidate_sampler', 'nn.all_candidate_sampler')
 def all_candidate_sampler(true_classes, num_true, num_sampled, unique,
                           seed=None, name=None):
   """Generate the set of all classes.
diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py
index f2f5da7d7a..d0291e2095 100644
--- a/tensorflow/python/ops/embedding_ops.py
+++ b/tensorflow/python/ops/embedding_ops.py
@@ -247,7 +247,7 @@ def _embedding_lookup_and_transform(params,
       return ret
 
 
-@tf_export("nn.embedding_lookup")
+@tf_export(v1=["nn.embedding_lookup"])
 def embedding_lookup(
     params,
     ids,
@@ -316,7 +316,66 @@ def embedding_lookup(
       transform_fn=None)
 
 
-@tf_export("nn.embedding_lookup_sparse")
+@tf_export("nn.embedding_lookup", v1=[])
+def embedding_lookup_v2(
+    params,
+    ids,
+    partition_strategy="mod",
+    max_norm=None,
+    name=None):
+  """Looks up `ids` in a list of embedding tensors.
+
+  This function is used to perform parallel lookups on the list of
+  tensors in `params`.  It is a generalization of
+  `tf.gather`, where `params` is
+  interpreted as a partitioning of a large embedding tensor.  `params` may be
+  a `PartitionedVariable` as returned by using `tf.get_variable()` with a
+  partitioner.
+
+  If `len(params) > 1`, each element `id` of `ids` is partitioned between
+  the elements of `params` according to the `partition_strategy`.
+  In all strategies, if the id space does not evenly divide the number of
+  partitions, each of the first `(max_id + 1) % len(params)` partitions will
+  be assigned one more id.
+
+  If `partition_strategy` is `"mod"`, we assign each id to partition
+  `p = id % len(params)`. For instance,
+  13 ids are split across 5 partitions as:
+  `[[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8], [4, 9]]`
+
+  If `partition_strategy` is `"div"`, we assign ids to partitions in a
+  contiguous manner. In this case, 13 ids are split across 5 partitions as:
+  `[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10], [11, 12]]`
+
+  The results of the lookup are concatenated into a dense
+  tensor. The returned tensor has shape `shape(ids) + shape(params)[1:]`.
+
+  Args:
+    params: A single tensor representing the complete embedding tensor,
+      or a list of P tensors all of same shape except for the first dimension,
+      representing sharded embedding tensors.  Alternatively, a
+      `PartitionedVariable`, created by partitioning along dimension 0. Each
+      element must be appropriately sized for the given `partition_strategy`.
+    ids: A `Tensor` with type `int32` or `int64` containing the ids to be looked
+      up in `params`.
+    partition_strategy: A string specifying the partitioning strategy, relevant
+      if `len(params) > 1`. Currently `"div"` and `"mod"` are supported. Default
+      is `"mod"`.
+    max_norm: If not `None`, each embedding is clipped if its l2-norm is
+      larger than this value.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` with the same type as the tensors in `params`.
+
+  Raises:
+    ValueError: If `params` is empty.
+  """
+  return embedding_lookup(params, ids, partition_strategy, name,
+                          max_norm=max_norm)
+
+
+@tf_export(v1=["nn.embedding_lookup_sparse"])
 def embedding_lookup_sparse(params,
                             sp_ids,
                             sp_weights,
@@ -491,6 +550,21 @@ def embedding_lookup_sparse(params,
     return embeddings
 
 
+@tf_export("nn.embedding_lookup_sparse", v1=[])
+def embedding_lookup_sparse_v2(params,
+                               sp_ids,
+                               sp_weights,
+                               partition_strategy="mod",
+                               combiner=None,
+                               max_norm=None,
+                               name=None):
+  return embedding_lookup_sparse_v2(
+      params, sp_ids, sp_weights, partition_strategy, name, combiner, max_norm)
+
+
+embedding_lookup_sparse_v2.__doc__ = embedding_lookup_sparse.__doc__
+
+
 @tf_export("nn.safe_embedding_lookup_sparse", v1=[])
 def safe_embedding_lookup_sparse_v2(embedding_weights,
                                     sparse_ids,
diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index 292cca8b9e..48dcab4842 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -262,7 +262,7 @@ def weighted_cross_entropy_with_logits(targets, logits, pos_weight, name=None):
         name=name)
 
 
-@tf_export("nn.relu_layer")
+@tf_export(v1=["nn.relu_layer"])
 def relu_layer(x, weights, biases, name=None):
   """Computes Relu(x * weight + biases).
 
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 97d405cd0e..611bfdac9a 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3776,7 +3776,7 @@ def erosion2d_v2(value,
             name=name))
 
 
-@tf_export("math.in_top_k", "nn.in_top_k")
+@tf_export(v1=["math.in_top_k", "nn.in_top_k"])
 def in_top_k(predictions, targets, k, name=None):
   r"""Says whether the targets are in the top `K` predictions.
 
@@ -3810,6 +3810,14 @@ def in_top_k(predictions, targets, k, name=None):
     return gen_nn_ops.in_top_kv2(predictions, targets, k, name=name)
 
 
+@tf_export("math.in_top_k", "nn.in_top_k", v1=[])
+def in_top_k_v2(targets, predictions, k, name=None):
+  return in_top_k(predictions, targets, k, name)
+
+
+in_top_k_v2.__doc__ = in_top_k.__doc__
+
+
 tf_export(v1=["nn.quantized_avg_pool"])(gen_nn_ops.quantized_avg_pool)
 tf_export(v1=["nn.quantized_conv2d"])(gen_nn_ops.quantized_conv2d)
 tf_export(v1=["nn.quantized_relu_x"])(gen_nn_ops.quantized_relu_x)
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index feff7df850..d7346b7371 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -299,7 +299,7 @@ def sparse_concat(axis,
 
 
 @tf_export("sparse.concat", v1=[])
-def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim=False, name=None):  # pylint: disable=missing-docstring
+def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dims=False, name=None):  # pylint: disable=missing-docstring
   sp_inputs = _convert_to_sparse_tensors(sp_inputs)
 
   if len(sp_inputs) == 1:  # Degenerate case of one tensor.
@@ -309,7 +309,7 @@ def sparse_concat_v2(axis, sp_inputs, expand_nonconcat_dim=False, name=None):  #
   vals = [sp_input.values for sp_input in sp_inputs]
   shapes = [sp_input.dense_shape for sp_input in sp_inputs]
 
-  if expand_nonconcat_dim:
+  if expand_nonconcat_dims:
     max_shape = math_ops.reduce_max(
         array_ops.concat(
             [array_ops.reshape(shape, [1, -1]) for shape in shapes], 0), 0)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
index 107534e086..1eefb1c70c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.random.pbtxt
@@ -1,9 +1,17 @@
 path: "tensorflow.random"
 tf_module {
+  member_method {
+    name: "all_candidate_sampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "categorical"
     argspec: "args=[\'logits\', \'num_samples\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "fixed_unigram_candidate_sampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'vocab_file\', \'distortion\', \'num_reserved_ids\', \'num_shards\', \'shard\', \'unigrams\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'\', \'1.0\', \'0\', \'1\', \'0\', \'()\', \'None\', \'None\'], "
+  }
   member_method {
     name: "gamma"
     argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
@@ -12,6 +20,10 @@ tf_module {
     name: "get_seed"
     argspec: "args=[\'op_seed\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "learned_unigram_candidate_sampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "log_uniform_candidate_sampler"
     argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
index 979d77ea6b..4ac0484050 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.math.pbtxt
@@ -170,7 +170,7 @@ tf_module {
   }
   member_method {
     name: "in_top_k"
-    argspec: "args=[\'predictions\', \'targets\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'targets\', \'predictions\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "invert_permutation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 6bf4a9d00c..9e5e82483a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -106,7 +106,7 @@ tf_module {
   }
   member_method {
     name: "depth_to_space"
-    argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], "
+    argspec: "args=[\'input\', \'block_size\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
   }
   member_method {
     name: "depthwise_conv2d"
@@ -134,11 +134,11 @@ tf_module {
   }
   member_method {
     name: "embedding_lookup"
-    argspec: "args=[\'params\', \'ids\', \'partition_strategy\', \'name\', \'validate_indices\', \'max_norm\'], varargs=None, keywords=None, defaults=[\'mod\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'params\', \'ids\', \'partition_strategy\', \'max_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'mod\', \'None\', \'None\'], "
   }
   member_method {
     name: "embedding_lookup_sparse"
-    argspec: "args=[\'params\', \'sp_ids\', \'sp_weights\', \'partition_strategy\', \'name\', \'combiner\', \'max_norm\'], varargs=None, keywords=None, defaults=[\'mod\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'params\', \'sp_ids\', \'sp_weights\', \'partition_strategy\', \'combiner\', \'max_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'mod\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "erosion2d"
@@ -158,7 +158,7 @@ tf_module {
   }
   member_method {
     name: "in_top_k"
-    argspec: "args=[\'predictions\', \'targets\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'targets\', \'predictions\', \'k\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "l2_loss"
@@ -228,10 +228,6 @@ tf_module {
     name: "relu6"
     argspec: "args=[\'features\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "relu_layer"
-    argspec: "args=[\'x\', \'weights\', \'biases\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
   member_method {
     name: "safe_embedding_lookup_sparse"
     argspec: "args=[\'embedding_weights\', \'sparse_ids\', \'sparse_weights\', \'combiner\', \'default_id\', \'max_norm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'mean\', \'None\', \'None\', \'None\'], "
@@ -278,7 +274,7 @@ tf_module {
   }
   member_method {
     name: "space_to_depth"
-    argspec: "args=[\'input\', \'block_size\', \'name\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\', \'NHWC\'], "
+    argspec: "args=[\'input\', \'block_size\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], "
   }
   member_method {
     name: "sparse_softmax_cross_entropy_with_logits"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
index de5cb6b717..d49c23e59c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.random.pbtxt
@@ -16,6 +16,10 @@ tf_module {
     name: "gamma"
     argspec: "args=[\'shape\', \'alpha\', \'beta\', \'dtype\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \"<dtype: \'float32\'>\", \'None\', \'None\'], "
   }
+  member_method {
+    name: "learned_unigram_candidate_sampler"
+    argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "log_uniform_candidate_sampler"
     argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'range_max\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
index 9808200d72..b8bd2c0b72 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.pbtxt
@@ -14,7 +14,7 @@ tf_module {
   }
   member_method {
     name: "concat"
-    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dim\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+    argspec: "args=[\'axis\', \'sp_inputs\', \'expand_nonconcat_dims\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
   member_method {
     name: "cross"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 02212c3810..b27fd5acc7 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -134,7 +134,7 @@ renames = {
     'tf.decode_json_example': 'tf.io.decode_json_example',
     'tf.decode_raw': 'tf.io.decode_raw',
     'tf.delete_session_tensor': 'tf.compat.v1.delete_session_tensor',
-    'tf.depth_to_space': 'tf.nn.depth_to_space',
+    'tf.depth_to_space': 'tf.compat.v1.depth_to_space',
     'tf.dequantize': 'tf.quantization.dequantize',
     'tf.deserialize_many_sparse': 'tf.io.deserialize_many_sparse',
     'tf.diag': 'tf.linalg.tensor_diag',
@@ -386,6 +386,7 @@ renames = {
     'tf.nn.quantized_max_pool': 'tf.compat.v1.nn.quantized_max_pool',
     'tf.nn.quantized_relu_x': 'tf.compat.v1.nn.quantized_relu_x',
     'tf.nn.raw_rnn': 'tf.compat.v1.nn.raw_rnn',
+    'tf.nn.relu_layer': 'tf.compat.v1.nn.relu_layer',
     'tf.nn.rnn_cell.BasicLSTMCell': 'tf.compat.v1.nn.rnn_cell.BasicLSTMCell',
     'tf.nn.rnn_cell.BasicRNNCell': 'tf.compat.v1.nn.rnn_cell.BasicRNNCell',
     'tf.nn.rnn_cell.GRUCell': 'tf.compat.v1.nn.rnn_cell.GRUCell',
@@ -511,7 +512,7 @@ renames = {
     'tf.sets.set_size': 'tf.sets.size',
     'tf.sets.set_union': 'tf.sets.union',
     'tf.space_to_batch': 'tf.nn.space_to_batch',
-    'tf.space_to_depth': 'tf.nn.space_to_depth',
+    'tf.space_to_depth': 'tf.compat.v1.space_to_depth',
     'tf.sparse.matmul': 'tf.sparse.sparse_dense_matmul',
     'tf.sparse.merge': 'tf.compat.v1.sparse.merge',
     'tf.sparse.placeholder': 'tf.compat.v1.sparse.placeholder',
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 4152d37db9..76f71d5804 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -33,6 +33,7 @@ reorders = {
     'tf.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
     'tf.convert_to_tensor': ['value', 'dtype', 'name', 'preferred_dtype'],
     'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
+    'tf.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
     'tf.feature_column.categorical_column_with_vocabulary_file': ['key', 'vocabulary_file', 'vocabulary_size', 'num_oov_buckets', 'default_value', 'dtype'],
     'tf.io.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
     'tf.io.parse_example': ['serialized', 'features', 'name', 'example_names'],
@@ -42,6 +43,7 @@ reorders = {
     'tf.math.argmax': ['input', 'axis', 'name', 'dimension', 'output_type'],
     'tf.math.argmin': ['input', 'axis', 'name', 'dimension', 'output_type'],
     'tf.math.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
+    'tf.math.in_top_k': ['predictions', 'targets', 'k', 'name'],
     'tf.math.reduce_all': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.math.reduce_any': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.math.reduce_logsumexp': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
@@ -53,10 +55,15 @@ reorders = {
     'tf.multinomial': ['logits', 'num_samples', 'seed', 'name', 'output_dtype'],
     'tf.nn.convolution': ['input', 'filter', 'padding', 'strides', 'dilation_rate', 'name', 'data_format'],
     'tf.nn.crelu': ['features', 'name', 'axis'],
+    'tf.nn.depth_to_space': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.depthwise_conv2d': ['input', 'filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.embedding_lookup': ['params', 'ids', 'partition_strategy', 'name', 'validate_indices', 'max_norm'],
+    'tf.nn.embedding_lookup_sparse': ['params', 'sp_ids', 'sp_weights', 'partition_strategy', 'name', 'combiner', 'max_norm'],
+    'tf.nn.in_top_k': ['predictions', 'targets', 'k', 'name'],
     'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
     'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
     'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
     'tf.pad': ['tensor', 'paddings', 'mode', 'name', 'constant_values'],
     'tf.parse_example': ['serialized', 'features', 'name', 'example_names'],
@@ -78,6 +85,7 @@ reorders = {
     'tf.serialize_sparse': ['sp_input', 'name', 'out_type'],
     'tf.shape': ['input', 'name', 'out_type'],
     'tf.size': ['input', 'name', 'out_type'],
+    'tf.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.sparse.add': ['a', 'b', 'threshold', 'thresh'],
     'tf.sparse.concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
     'tf.sparse.segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index f4165c9456..0630ce28cf 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -552,6 +552,12 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.reduce_join",
         "tf.confusion_matrix",
         "tf.math.confusion_matrix",
+        "tf.math.in_top_k",
+        "tf.nn.depth_to_space",
+        "tf.nn.embedding_lookup",
+        "tf.nn.embedding_lookup_sparse",
+        "tf.nn.in_top_k",
+        "tf.nn.space_to_depth",
     }
 
     # Functions that were reordered should be changed to the new keyword args
@@ -711,6 +717,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             tf_01s_like_no_optimize_comment,
         "tf.ones_like":
             tf_01s_like_no_optimize_comment,
+        "tf.nn.embedding_lookup":
+            "WARNING: validate_indices argument has been removed.",
+        "tf.sparse.concat":
+            ("WARNING: expand_nonconcat_dim was renamed to "
+             "expand_nonconcat_dims."),
     }
 
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 57700c07d6..b362a8081f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -567,6 +567,54 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testInTopK(self):
+    text = "tf.math.in_top_k(a, b, c, n)"
+    expected_text = (
+        "tf.math.in_top_k(predictions=a, targets=b, k=c, name=n)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testDepthToSpace(self):
+    text = "tf.nn.depth_to_space(input, block_size, name, data_format)"
+    expected_text = (
+        "tf.nn.depth_to_space(input=input, block_size=block_size, "
+        "name=name, data_format=data_format)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testEmbeddingLookup(self):
+    text = ("tf.nn.embedding_lookup(params, ids, partition_strategy, name, "
+            "validate_indices, max_norm)")
+    expected_text = ("tf.nn.embedding_lookup(params=params, ids=ids, "
+                     "partition_strategy=partition_strategy, name=name, "
+                     "validate_indices=validate_indices, max_norm=max_norm)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testEmbeddingLookupSparse(self):
+    text = ("tf.nn.embedding_lookup_sparse(params, sp_ids, sp_weights, "
+            "partition_strategy, name, combiner, max_norm)")
+    expected_text = ("tf.nn.embedding_lookup_sparse(params=params, "
+                     "sp_ids=sp_ids, sp_weights=sp_weights, "
+                     "partition_strategy=partition_strategy, name=name, "
+                     "combiner=combiner, max_norm=max_norm)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testNnInTopK(self):
+    text = "tf.nn.in_top_k(predictions, targets, k, name)"
+    expected_text = ("tf.nn.in_top_k(predictions=predictions, "
+                     "targets=targets, k=k, name=name)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+  def testSpaceToDepth(self):
+    text = "tf.nn.space_to_depth(input, block_size, name, data_format)"
+    expected_text = ("tf.nn.space_to_depth(input=input, block_size=block_size, "
+                     "name=name, data_format=data_format)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From ef46e5d3a3e3095517b7528890704230139d95c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 11:40:10 -0800
Subject: [PATCH 1436/1554] Internal change.

PiperOrigin-RevId: 224377474
---
 tensorflow/lite/delegates/nnapi/BUILD | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index c24f0f71ac..fd954ba222 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -23,10 +23,7 @@ tf_cc_test(
     name = "nnapi_delegate_test",
     size = "small",
     srcs = ["nnapi_delegate_test.cc"],
-    tags = [
-        "no_oss",
-        "noasan",  # TODO(b/112326936): re-enable for asan once fixed.
-    ],
+    tags = ["no_oss"],
     deps = [
         ":nnapi_delegate",
         "//tensorflow/lite:framework",
-- 
GitLab


From 52aac384bf12ced33ea1ef3ec7bc8183a9cf4ca0 Mon Sep 17 00:00:00 2001
From: Dalmo Cirne <dalmo@clarifai.com>
Date: Wed, 31 Oct 2018 05:43:57 -0400
Subject: [PATCH 1437/1554] Fix downloading protobuf dependency

Protobuf has been failing due to its repository having dependencies. Downloading the .tar.gz form GitHub doesn't work because it does not bring its dependencies with it.

This PR introduces a function to clone a repository, initializa and update its submodules. The function is used to download protobuf as a repository dependency, rather than just a file.

The function is also implemented in a generic way, so if in the future other dependencies fall into the same situation, it can be reused.

With these changes, `tensorflow/workspace.bzl` goes back to using the commit sha in `PROTOBUF_URLS` and `PROTOBUF_STRIP_PREFIX`, rather than the tag of a release.

A command at the end of the `download_dependencies.sh` script was commented out. The comment says for it to be removed once protobug is fixed. Perhaps no longer necessary.
---
 .../contrib/makefile/download_dependencies.sh | 43 +++++++++++++++----
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index b396c52767..d22e2c179b 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -30,11 +30,13 @@ EIGEN_URL="$(grep -o 'http.*bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE
 GEMMLOWP_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)"
 GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz"
 NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
-# Note: The Protobuf source in `tensorflow/workspace.bzl` in TensorFlow
-# 1.10 branch does not work. `make distclean` fails and blocks the build
-# process. For now we're hardcoding to the version which is used by
-# TensorFlow 1.9.
-PROTOBUF_URL="https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.0.tar.gz"
+
+# Note: The protobuf repo needs to be cloned due to its submodules.
+# These variables contain the GitHub repo and the sha, from `tensorflow/workspace.bzl`,
+# from which to clone it from and checkout to.
+readonly PROTOBUF_REPO="https://github.com/protocolbuffers/protobuf.git"
+readonly PROTOBUF_SHA="$(grep -o 'https://github.com/google/protobuf/archive/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1 | awk '{print substr($0, index($0, "archive") + 8, 40) }')"
+
 # TODO (yongtang): Replace the following with 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' once
 # the archive has been propagated in mirror.bazel.build.
 RE2_URL="$(grep -o 'https://github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
@@ -91,11 +93,34 @@ download_and_extract() {
   find "${dir}" -type f -name '*BUILD' -delete
 }
 
+function clone_repository() {
+  local repo_url="${1}"
+  local destination_directory="${2}"
+  local commit_sha="${3}"
+
+  if [[ -d "${destination_directory}" ]]; then
+    rm -rf "${destination_directory}"
+  fi
+
+  git clone "${repo_url}" "${destination_directory}"
+
+  pushd "$(pwd)" 1>/dev/null
+
+  cd "${destination_directory}"
+
+  if [[ -n "${commit_sha}" ]]; then
+    git checkout "${PROTOBUF_SHA}"
+  fi
+
+  git submodule update --init
+
+  popd 1>/dev/null
+}
+
 download_and_extract "${EIGEN_URL}" "${DOWNLOADS_DIR}/eigen"
 download_and_extract "${GEMMLOWP_URL}" "${DOWNLOADS_DIR}/gemmlowp"
 download_and_extract "${GOOGLETEST_URL}" "${DOWNLOADS_DIR}/googletest"
 download_and_extract "${NSYNC_URL}" "${DOWNLOADS_DIR}/nsync"
-download_and_extract "${PROTOBUF_URL}" "${DOWNLOADS_DIR}/protobuf"
 download_and_extract "${RE2_URL}" "${DOWNLOADS_DIR}/re2"
 download_and_extract "${FFT2D_URL}" "${DOWNLOADS_DIR}/fft2d"
 download_and_extract "${DOUBLE_CONVERSION_URL}" "${DOWNLOADS_DIR}/double_conversion"
@@ -106,6 +131,8 @@ download_and_extract "${CUB_URL}" "${DOWNLOADS_DIR}/cub/external/cub_archive"
 download_and_extract "${FARMHASH_URL}" "${DOWNLOADS_DIR}/farmhash"
 download_and_extract "${FLATBUFFERS_URL}" "${DOWNLOADS_DIR}/flatbuffers"
 
+clone_repository "${PROTOBUF_REPO}" "${DOWNLOADS_DIR}/protobuf" "${PROTOBUF_SHA}"
+
 replace_by_sed 's#static uint32x4_t p4ui_CONJ_XOR = vld1q_u32( conj_XOR_DATA );#static uint32x4_t p4ui_CONJ_XOR; // = vld1q_u32( conj_XOR_DATA ); - Removed by script#' \
   "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h"
 replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#static uint32x2_t p2ui_CONJ_XOR;// = vld1_u32( conj_XOR_DATA ); - Removed by scripts#' \
@@ -113,7 +140,7 @@ replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#s
 replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \
   "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h"
 # TODO(satok): Remove this once protobuf/autogen.sh is fixed.
-replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#http://download.tensorflow.org/deps/gmock-1.7.0.zip#' \
-  "${DOWNLOADS_DIR}/protobuf/autogen.sh"
+# replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#http://download.tensorflow.org/deps/gmock-1.7.0.zip#' \
+#   "${DOWNLOADS_DIR}/protobuf/autogen.sh"
 
 echo "download_dependencies.sh completed successfully." >&2
-- 
GitLab


From a3a6ab05170eb95221e7ae486d30197f874ea784 Mon Sep 17 00:00:00 2001
From: Dalmo Cirne <dalmo@clarifai.com>
Date: Thu, 1 Nov 2018 21:21:45 -0400
Subject: [PATCH 1438/1554] Add comment to indicate the protobuf version

---
 tensorflow/workspace.bzl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index e6b4a89e3b..9e59f1c710 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -346,6 +346,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         },
     )
 
+    # The commit sha in the URLs correspond to v3.6.0 of protobuf
     PROTOBUF_URLS = [
         "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
         "https://github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
-- 
GitLab


From 1aeb29692a7e3e285c1572fca1dec9824c520962 Mon Sep 17 00:00:00 2001
From: Dalmo Cirne <dalmo@clarifai.com>
Date: Thu, 6 Dec 2018 14:07:02 -0500
Subject: [PATCH 1439/1554] Update the Flatbuffer and Protobuf versions

---
 tensorflow/contrib/makefile/download_dependencies.sh | 2 +-
 tensorflow/workspace.bzl                             | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index d22e2c179b..6582ba793b 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -47,7 +47,7 @@ CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror
 
 # Required for TensorFlow Lite Flex runtime.
 FARMHASH_URL="https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz"
-FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/1f5eae5d6a135ff6811724f6c57f911d1f46bb15.tar.gz"
+FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
 #                   so work around it by patching the source.
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 9e59f1c710..e6b4a89e3b 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -346,7 +346,6 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         },
     )
 
-    # The commit sha in the URLs correspond to v3.6.0 of protobuf
     PROTOBUF_URLS = [
         "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
         "https://github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
-- 
GitLab


From 6ef428bd6e83b0930266bf922eaa2f4a60e8328a Mon Sep 17 00:00:00 2001
From: Dalmo Cirne <dalmo@clarifai.com>
Date: Thu, 6 Dec 2018 14:42:06 -0500
Subject: [PATCH 1440/1554] Fix URLs for protobuf 3.6.1.2

---
 tensorflow/workspace.bzl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index e6b4a89e3b..6712925b49 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -347,8 +347,8 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     )
 
     PROTOBUF_URLS = [
-        "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
-        "https://github.com/google/protobuf/archive/v3.6.1.2.tar.gz",
+        "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v3.6.1.2.tar.gz",
+        "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.2.tar.gz",
     ]
     PROTOBUF_SHA256 = "2244b0308846bb22b4ff0bcc675e99290ff9f1115553ae9671eba1030af31bc0"
     PROTOBUF_STRIP_PREFIX = "protobuf-3.6.1.2"
-- 
GitLab


From 0adae252576e5cdafab6598cbb0522fb3c703bf3 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 6 Dec 2018 11:50:41 -0800
Subject: [PATCH 1441/1554] Replace uses of deprecated
 `Dataset.make_one_shot_iterator()` with
 `tf.compat.v1.data.make_one_shot_iterator()`.

PiperOrigin-RevId: 224379500
---
 .../tpu/python/tpu/tpu_estimator_signals_test.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
index 55235556de..e3ea983abf 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
@@ -21,8 +21,8 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.contrib.tpu.python.tpu import tpu_estimator
-from tensorflow.python import data as dataset_lib
 from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import test
@@ -34,10 +34,10 @@ def make_input_fn(num_samples):
 
   def input_fn(params):
     batch_size = params['batch_size']
-    da1 = dataset_lib.Dataset.from_tensor_slices(a)
-    da2 = dataset_lib.Dataset.from_tensor_slices(b)
+    da1 = dataset_ops.Dataset.from_tensor_slices(a)
+    da2 = dataset_ops.Dataset.from_tensor_slices(b)
 
-    dataset = dataset_lib.Dataset.zip((da1, da2))
+    dataset = dataset_ops.Dataset.zip((da1, da2))
     dataset = dataset.map(lambda fa, fb: {'a': fa, 'b': fb})
     dataset = dataset.batch(batch_size)
     return dataset
@@ -50,10 +50,10 @@ def make_input_fn_with_labels(num_samples):
 
   def input_fn(params):
     batch_size = params['batch_size']
-    da1 = dataset_lib.Dataset.from_tensor_slices(a)
-    da2 = dataset_lib.Dataset.from_tensor_slices(b)
+    da1 = dataset_ops.Dataset.from_tensor_slices(a)
+    da2 = dataset_ops.Dataset.from_tensor_slices(b)
 
-    dataset = dataset_lib.Dataset.zip((da1, da2))
+    dataset = dataset_ops.Dataset.zip((da1, da2))
     dataset = dataset.map(lambda fa, fb: ({'a': fa}, fb))
     dataset = dataset.batch(batch_size)
     return dataset
@@ -71,7 +71,7 @@ class TPUEstimatorStoppingSignalsTest(test.TestCase):
 
     with ops.Graph().as_default():
       dataset = input_fn(params)
-      features = dataset_lib.make_one_shot_iterator(dataset).get_next()
+      features = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
       # With tf.data.Dataset.batch, the batch is None, i.e., dynamic shape.
       self.assertIsNone(features['a'].shape.as_list()[0])
-- 
GitLab


From 0fb46cf9139f1e8cf88195e0b8f555eb10d00b3b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 11:53:13 -0800
Subject: [PATCH 1442/1554] Update imports to point to new location of
 distribution_strategy_context.py.

PiperOrigin-RevId: 224380057
---
 tensorflow/contrib/metrics/python/metrics/classification.py | 2 +-
 tensorflow/contrib/optimizer_v2/optimizer_v2.py             | 2 +-
 tensorflow/python/keras/layers/normalization.py             | 2 +-
 tensorflow/python/keras/metrics.py                          | 2 +-
 tensorflow/python/keras/optimizer_v2/optimizer_v2.py        | 2 +-
 tensorflow/python/keras/optimizers.py                       | 2 +-
 tensorflow/python/ops/metrics_impl.py                       | 2 +-
 tensorflow/python/ops/summary_op_util.py                    | 2 +-
 tensorflow/python/training/checkpoint_utils.py              | 2 +-
 tensorflow/python/training/moving_averages.py               | 2 +-
 tensorflow/python/training/optimizer.py                     | 2 +-
 tensorflow/python/training/slot_creator.py                  | 2 +-
 tensorflow/python/training/sync_replicas_optimizer.py       | 2 +-
 13 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/metrics/classification.py b/tensorflow/contrib/metrics/python/metrics/classification.py
index 062deb74b1..9aabc4bec3 100644
--- a/tensorflow/contrib/metrics/python/metrics/classification.py
+++ b/tensorflow/contrib/metrics/python/metrics/classification.py
@@ -18,13 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics_impl
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.training import distribution_strategy_context
 
 # TODO(nsilberman): move into metrics/python/ops/
 
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 73a556f0b2..11a9248a01 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -25,6 +25,7 @@ import abc
 import six
 
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -36,7 +37,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribution_strategy_context as distribute_ctx
 from tensorflow.python.training import optimizer as optimizer_v1
 from tensorflow.python.training import slot_creator
 from tensorflow.python.training.checkpointable import base as checkpointable
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 37894a3d3d..75b10222ed 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import contextlib
 
 from tensorflow.python import tf2
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -39,7 +40,6 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.tf_export import tf_export
 
 
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 3c2682e4c6..1d3c6e67d7 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -28,6 +28,7 @@ from enum import Enum
 import numpy as np
 import six
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
@@ -60,7 +61,6 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.ops import weights_broadcast_ops
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util.tf_export import tf_export
 from tensorflow.tools.docs import doc_controls
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index b26b3cefc8..e6cd52c817 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -24,6 +24,7 @@ import abc
 
 import six
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -36,7 +37,6 @@ from tensorflow.python.ops import gradients
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer as optimizer_v1
 from tensorflow.python.util import nest
 
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 10466eb573..ee6dbba5ad 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -23,6 +23,7 @@ import six
 from six.moves import zip  # pylint: disable=redefined-builtin
 
 from tensorflow.python import tf2
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2
@@ -38,7 +39,6 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training import training_util
 from tensorflow.python.training.checkpointable import base as checkpointable
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index cb42199011..ec39b1790e 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -34,7 +35,6 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
diff --git a/tensorflow/python/ops/summary_op_util.py b/tensorflow/python/ops/summary_op_util.py
index c72a9aefc3..93d8d50842 100644
--- a/tensorflow/python/ops/summary_op_util.py
+++ b/tensorflow/python/ops/summary_op_util.py
@@ -21,10 +21,10 @@ from __future__ import print_function
 import contextlib
 import re
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.platform import tf_logging
-from tensorflow.python.training import distribution_strategy_context
 
 
 def collect(val, collections, default_collections):
diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 58166dbb68..99b1f4c0d7 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import six
 
 from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -29,7 +30,6 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import saver
 from tensorflow.python.util.tf_export import tf_export
 
diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 8785f9a8e7..72670f0ca3 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -26,7 +27,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import slot_creator
 from tensorflow.python.util.tf_export import tf_export
 
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index a9508b862a..d9ebdcad1f 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -25,6 +25,7 @@ import abc
 import six
 
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -38,7 +39,6 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribution_strategy_context as distribute_ctx
 from tensorflow.python.training import slot_creator
 from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.util import nest
diff --git a/tensorflow/python/training/slot_creator.py b/tensorflow/python/training/slot_creator.py
index d76b22acd8..ecf5a96ed4 100644
--- a/tensorflow/python/training/slot_creator.py
+++ b/tensorflow/python/training/slot_creator.py
@@ -39,13 +39,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
-from tensorflow.python.training import distribution_strategy_context
 
 
 def _create_slot_var(primary, val, scope, validate_shape, shape, dtype):
diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index 501d9bc8d3..cd4590db7f 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.framework import types_pb2
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -27,7 +28,6 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer
 from tensorflow.python.training import queue_runner
 from tensorflow.python.training import session_manager
-- 
GitLab


From 5bedbfd6211aedadb70b40a8e2729cb792754ff3 Mon Sep 17 00:00:00 2001
From: Dalmo Cirne <dalmo@clarifai.com>
Date: Thu, 6 Dec 2018 15:13:04 -0500
Subject: [PATCH 1443/1554] Updates from the PR review

`PROTOBUF_SHA` renamed to `PROTOBUF_TAG`. The variable is used to checkout to a specific tag after cloning the protobuf repository. The value is obtained by parsing the protobuf URL in the `workspace.bzl` file.
---
 tensorflow/contrib/makefile/download_dependencies.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 6582ba793b..2a5232b476 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -35,7 +35,7 @@ NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.
 # These variables contain the GitHub repo and the sha, from `tensorflow/workspace.bzl`,
 # from which to clone it from and checkout to.
 readonly PROTOBUF_REPO="https://github.com/protocolbuffers/protobuf.git"
-readonly PROTOBUF_SHA="$(grep -o 'https://github.com/google/protobuf/archive/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1 | awk '{print substr($0, index($0, "archive") + 8, 40) }')"
+readonly PROTOBUF_TAG="$(grep -o 'https://github.com/protocolbuffers/protobuf/archive/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1 | awk '{print substr($0, index($0, "archive") + 8, index($0, "tar") - index($0, "archive") - 9) }')"
 
 # TODO (yongtang): Replace the following with 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' once
 # the archive has been propagated in mirror.bazel.build.
@@ -47,7 +47,7 @@ CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror
 
 # Required for TensorFlow Lite Flex runtime.
 FARMHASH_URL="https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz"
-FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz"
+FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/1f5eae5d6a135ff6811724f6c57f911d1f46bb15.tar.gz"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
 #                   so work around it by patching the source.
@@ -109,7 +109,7 @@ function clone_repository() {
   cd "${destination_directory}"
 
   if [[ -n "${commit_sha}" ]]; then
-    git checkout "${PROTOBUF_SHA}"
+    git checkout "${PROTOBUF_TAG}"
   fi
 
   git submodule update --init
@@ -131,7 +131,7 @@ download_and_extract "${CUB_URL}" "${DOWNLOADS_DIR}/cub/external/cub_archive"
 download_and_extract "${FARMHASH_URL}" "${DOWNLOADS_DIR}/farmhash"
 download_and_extract "${FLATBUFFERS_URL}" "${DOWNLOADS_DIR}/flatbuffers"
 
-clone_repository "${PROTOBUF_REPO}" "${DOWNLOADS_DIR}/protobuf" "${PROTOBUF_SHA}"
+clone_repository "${PROTOBUF_REPO}" "${DOWNLOADS_DIR}/protobuf" "${PROTOBUF_TAG}"
 
 replace_by_sed 's#static uint32x4_t p4ui_CONJ_XOR = vld1q_u32( conj_XOR_DATA );#static uint32x4_t p4ui_CONJ_XOR; // = vld1q_u32( conj_XOR_DATA ); - Removed by script#' \
   "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h"
@@ -140,7 +140,7 @@ replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#s
 replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \
   "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h"
 # TODO(satok): Remove this once protobuf/autogen.sh is fixed.
-# replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#http://download.tensorflow.org/deps/gmock-1.7.0.zip#' \
-#   "${DOWNLOADS_DIR}/protobuf/autogen.sh"
+replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#http://download.tensorflow.org/deps/gmock-1.7.0.zip#' \
+  "${DOWNLOADS_DIR}/protobuf/autogen.sh"
 
 echo "download_dependencies.sh completed successfully." >&2
-- 
GitLab


From 2947a2851011782a020d59ef8624d191bef5c279 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 6 Dec 2018 12:00:29 -0800
Subject: [PATCH 1444/1554] [tf.data] Add documentation for supporting
 `tf.data.experimental.AUTOTUNE` as a value for the `num_parallel_calls`
 argument of `map()`, `interleave()`, and `map_and_batch()`.

PiperOrigin-RevId: 224381264
---
 tensorflow/python/data/experimental/ops/batching.py | 7 ++++---
 tensorflow/python/data/ops/dataset_ops.py           | 8 ++++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index 9db9a28dd8..f70a4e0537 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -626,9 +626,10 @@ def map_and_batch(map_func,
       whether the last batch should be dropped in case its size is smaller than
       desired; the default behavior is not to drop the smaller batch.
     num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`,
-        representing the number of elements to process in parallel. If not
-        specified, `batch_size * num_parallel_batches` elements will be
-        processed in parallel.
+      representing the number of elements to process in parallel. If not
+      specified, `batch_size * num_parallel_batches` elements will be processed
+      in parallel. If the value `tf.data.experimental.AUTOTUNE` is used, then
+      the number of parallel calls is set dynamically based on available CPU.
 
   Returns:
     A `Dataset` transformation function, which can be passed to
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index a08a2b5787..cd7b1a4a5a 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -947,7 +947,9 @@ class DatasetV2(object):
        `self.output_types`) to another nested structure of tensors.
       num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`,
         representing the number elements to process in parallel. If not
-        specified, elements will be processed sequentially.
+        specified, elements will be processed sequentially. If the value
+        `tf.data.experimental.AUTOTUNE` is used, then the number of parallel
+        calls is set dynamically based on available CPU.
 
     Returns:
       Dataset: A `Dataset`.
@@ -1058,7 +1060,9 @@ class DatasetV2(object):
       num_parallel_calls: (Optional.) If specified, the implementation creates
         a threadpool, which is used to fetch inputs from cycle elements
         asynchronously and in parallel. The default behavior is to fetch inputs
-        from cycle elements synchronously with no parallelism.
+        from cycle elements synchronously with no parallelism. If the value
+        `tf.data.experimental.AUTOTUNE` is used, then the number of parallel
+        calls is set dynamically based on available CPU.
 
     Returns:
       Dataset: A `Dataset`.
-- 
GitLab


From 51561e0354a1556e848c920cd47a718360dc720d Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 6 Dec 2018 12:25:27 -0800
Subject: [PATCH 1445/1554] Use coordinator in distribute coordinator and
 independent_worker tests.

PiperOrigin-RevId: 224385727
---
 .../python/estimator_training_test.py         |  7 ++--
 .../python/multi_worker_test_base.py          | 10 ++++--
 .../distribute/distribute_coordinator.py      | 35 +++++++++++++------
 3 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index b369a7fefe..3f55a8a1c8 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -375,11 +375,13 @@ class DistributeCoordinatorIntegrationTest(
     threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn,
                                                  cluster_spec, train_distribute,
                                                  eval_distribute)
+    threads_to_join = []
     for task_type, ts in threads.items():
       if task_type == PS:
         continue
       for t in ts:
-        t.join()
+        threads_to_join.append(t)
+    self.join_independent_workers(threads_to_join)
 
     estimator = self._get_estimator(train_distribute, eval_distribute)
     self._inspect_train_and_eval_events(estimator)
@@ -413,8 +415,7 @@ class DistributeCoordinatorIntegrationTest(
     threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn,
                                                  cluster_spec, train_distribute,
                                                  eval_distribute)
-    threads[WORKER][0].join()
-    threads[EVALUATOR][0].join()
+    self.join_independent_workers([threads[WORKER][0], threads[EVALUATOR][0]])
 
     estimator = self._get_estimator(train_distribute, eval_distribute)
     self._inspect_train_and_eval_events(estimator)
diff --git a/tensorflow/contrib/distribute/python/multi_worker_test_base.py b/tensorflow/contrib/distribute/python/multi_worker_test_base.py
index 147c9b83f8..b05aac431f 100644
--- a/tensorflow/contrib/distribute/python/multi_worker_test_base.py
+++ b/tensorflow/contrib/distribute/python/multi_worker_test_base.py
@@ -40,6 +40,7 @@ from tensorflow.python.client import session
 from tensorflow.python.estimator import run_config
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import coordinator
 from tensorflow.python.training import server_lib
 
 ASSIGNED_PORTS = set()
@@ -360,6 +361,7 @@ class IndependentWorkerTestBase(test.TestCase):
     self._mock_os_env = MockOsEnv()
     self._mock_context = test.mock.patch.object(os, 'environ',
                                                 self._mock_os_env)
+    self._coord = coordinator.Coordinator()
     super(IndependentWorkerTestBase, self).setUp()
     self._mock_context.__enter__()
 
@@ -368,8 +370,9 @@ class IndependentWorkerTestBase(test.TestCase):
     super(IndependentWorkerTestBase, self).tearDown()
 
   def _task_thread(self, task_fn, tf_config, *args, **kwargs):
-    os.environ['TF_CONFIG'] = json.dumps(tf_config)
-    task_fn(*args, **kwargs)
+    with self._coord.stop_on_exception():
+      os.environ['TF_CONFIG'] = json.dumps(tf_config)
+      task_fn(*args, **kwargs)
 
   def _run_task_in_thread(self, task_fn, cluster_spec, task_type, task_id,
                           *args, **kwargs):
@@ -403,3 +406,6 @@ class IndependentWorkerTestBase(test.TestCase):
                                      *args, **kwargs)
         threads[task_type].append(t)
     return threads
+
+  def join_independent_workers(self, worker_threads):
+    self._coord.join(worker_threads)
diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index c0f9b8a1fd..78c995a578 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -29,6 +29,7 @@ from tensorflow.python.client import session
 from tensorflow.python.distribute import distribute_coordinator_context
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import coordinator
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training import server_lib
 
@@ -328,7 +329,8 @@ def _run_single_worker(worker_fn,
                        task_id,
                        session_config,
                        rpc_layer="",
-                       worker_barrier=None):
+                       worker_barrier=None,
+                       coord=None):
   """Runs a single worker by calling `worker_fn` under context."""
   session_config = copy.deepcopy(session_config)
   strategy = copy.deepcopy(strategy)
@@ -350,7 +352,11 @@ def _run_single_worker(worker_fn,
       rpc_layer=rpc_layer,
       worker_barrier=worker_barrier)
   with context:
-    return worker_fn(strategy)
+    if coord:
+      with coord.stop_on_exception():
+        return worker_fn(strategy)
+    else:
+      return worker_fn(strategy)
 
 
 def _split_cluster_for_evaluator(cluster_spec, task_type):
@@ -423,6 +429,7 @@ def _run_std_server(cluster_spec=None,
 def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
                               cluster_spec, session_config, rpc_layer):
   """Runs a standalone client for between-graph replication."""
+  coord = coordinator.Coordinator()
   eval_thread = None
   if _TaskType.EVALUATOR in cluster_spec.jobs:
     eval_thread = threading.Thread(
@@ -431,6 +438,7 @@ def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
               session_config),
         kwargs={
             "rpc_layer": rpc_layer,
+            "coord": coord,
         })
     eval_thread.start()
 
@@ -444,18 +452,18 @@ def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
                 session_config),
           kwargs={
               "rpc_layer": rpc_layer,
-              "worker_barrier": worker_barrier
+              "worker_barrier": worker_barrier,
+              "coord": coord,
           })
       t.start()
       threads.append(t)
 
-  # TODO(yuefengz): wrap threads into thread coordinator?
-  for t in threads:
-    t.join()
-
-  # TODO(yuefengz): is it necessary to join eval thread?
   if eval_thread:
-    eval_thread.join()
+    # TODO(yuefengz): is it necessary to join eval thread?
+    threads_to_join = threads + [eval_thread]
+  else:
+    threads_to_join = threads
+  coord.join(threads_to_join)
 
   # TODO(yuefengz): we probably want to return results from all workers?
   return None
@@ -464,6 +472,7 @@ def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
 def _run_in_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
                          cluster_spec, session_config, rpc_layer):
   """Runs a standalone client for in-graph replication."""
+  coord = coordinator.Coordinator()
   eval_thread = None
   if _TaskType.EVALUATOR in cluster_spec.jobs:
     eval_thread = threading.Thread(
@@ -472,6 +481,7 @@ def _run_in_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
               session_config),
         kwargs={
             "rpc_layer": rpc_layer,
+            "coord": coord,
         })
     eval_thread.start()
 
@@ -482,9 +492,12 @@ def _run_in_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
       None,
       None,
       session_config,
-      rpc_layer=rpc_layer)
+      rpc_layer=rpc_layer,
+      coord=coord)
+
   if eval_thread:
-    eval_thread.join()
+    coord.join([eval_thread])
+
   return worker_result
 
 
-- 
GitLab


From 844dbd8730b7aba7b9f51e7455023070f8953c98 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 6 Dec 2018 13:01:00 -0800
Subject: [PATCH 1446/1554] Populate some additional parameters when running
 eager operations.

PiperOrigin-RevId: 224391662
---
 tensorflow/core/common_runtime/eager/context.cc      | 10 ++++++++++
 tensorflow/core/common_runtime/eager/context.h       |  8 ++++++++
 tensorflow/core/common_runtime/eager/execute.cc      |  3 ++-
 .../core/common_runtime/eager/kernel_and_device.cc   | 12 ++++++++++++
 .../core/common_runtime/eager/kernel_and_device.h    | 10 +++++++++-
 tensorflow/core/framework/op_kernel.h                |  2 ++
 tensorflow/python/eager/context.py                   |  8 +++-----
 7 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 583ae64edd..1727c04560 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -15,6 +15,9 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/context.h"
 
+#include "tensorflow/core/common_runtime/collective_executor_mgr.h"
+#include "tensorflow/core/common_runtime/collective_param_resolver_local.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/framework/resource_mgr.h"
@@ -71,6 +74,13 @@ EagerContext::EagerContext(const SessionOptions& opts,
   runner_ = [this](std::function<void()> closure) {
     this->thread_pool_->Schedule(std::move(closure));
   };
+
+  std::unique_ptr<DeviceResolverInterface> drl(
+      new DeviceResolverLocal(local_device_mgr()));
+  std::unique_ptr<ParamResolverInterface> cprl(new CollectiveParamResolverLocal(
+      local_device_mgr(), drl.get(), "/job:localhost/replica:0/task:0"));
+  collective_executor_mgr_.reset(new CollectiveExecutorMgr(
+      opts.config, local_device_mgr(), std::move(drl), std::move(cprl)));
 }
 
 void EagerContext::InitDeviceMapAndAsync() {
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index 51109f8f1a..cdef947893 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
 #include "tensorflow/core/distributed_runtime/server_lib.h"
 #endif
+#include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/log_memory.h"
 #include "tensorflow/core/framework/rendezvous.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
@@ -147,6 +148,11 @@ class EagerContext {
   bool LogMemory() { return log_memory_; }
 
   Rendezvous* GetRendezvous() { return rendezvous_; }
+  std::unique_ptr<CollectiveExecutor::Handle> GetCollectiveExecutorHandle() {
+    return std::unique_ptr<CollectiveExecutor::Handle>(
+        new CollectiveExecutor::Handle(
+            collective_executor_mgr_->FindOrCreate(0), true /*inherit_ref*/));
+  }
 
   const tensorflow::DeviceMgr* local_device_mgr() const {
     return (local_device_manager_ != nullptr) ? local_device_manager_.get()
@@ -273,6 +279,8 @@ class EagerContext {
 
   Env* const env_;
 
+  std::unique_ptr<CollectiveExecutorMgrInterface> collective_executor_mgr_;
+
 #ifndef __ANDROID__
   void CloseRemoteContexts();
 
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 5bf7888fad..a6199f2aeb 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -284,7 +284,8 @@ Status EagerLocalExecute(EagerOperation* op,
           "Unable to find a FunctionLibraryRuntime corresponding to device ",
           device->name());
     }
-    kernel = new KernelAndDevice(ctx->GetRendezvous(), ctx->LogMemory());
+    kernel = new KernelAndDevice(ctx->GetRendezvous(), ctx->LogMemory(),
+                                 ctx->GetCollectiveExecutorHandle());
     status = KernelAndDevice::Init(ndef, flr, ctx->runner(), kernel);
     if (!status.ok()) {
       delete kernel;
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 192d22dfd5..317e9a1607 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -84,6 +84,15 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
                              tensorflow::HOST_MEMORY);
   }
 
+  gtl::InlinedVector<DeviceContext*, 4> input_device_contexts;
+  for (int i = 0; i < inputs->size(); i++) {
+    DeviceContext* device_context = nullptr;
+    if (device_->tensorflow_gpu_device_info() != nullptr) {
+      device_context = device_->tensorflow_gpu_device_info()->default_context;
+    }
+    input_device_contexts.push_back(device_context);
+  }
+
   OpKernelContext::Params params;
   params.device = device_;
   params.frame_iter = FrameAndIter(0, 0);
@@ -110,6 +119,9 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
   }
 
   params.step_container = step_container;
+  params.collective_executor =
+      collective_executor_ ? collective_executor_->get() : nullptr;
+  params.input_device_contexts = &input_device_contexts;
 
   OpKernelContext context(&params);
 
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h
index 52dac94ccc..ee430b7fc7 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.h
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h
@@ -23,6 +23,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/framework/cancellation.h"
+#include "tensorflow/core/framework/collective.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/types.h"
@@ -55,10 +56,16 @@ class KernelAndDevice {
                      KernelAndDevice* out);
 
   KernelAndDevice(tensorflow::Rendezvous* rendez, bool log_memory)
+      : KernelAndDevice(rendez, log_memory, nullptr) {}
+
+  KernelAndDevice(
+      tensorflow::Rendezvous* rendez, bool log_memory,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor)
       : device_(nullptr),
         flr_(nullptr),
         rendez_(rendez),
-        log_memory_(log_memory) {}
+        log_memory_(log_memory),
+        collective_executor_(std::move(collective_executor)) {}
 
   // TODO(ashankar): Handle list-valued inputs.
   Status Run(std::vector<Tensor>* inputs, std::vector<Tensor>* outputs,
@@ -92,6 +99,7 @@ class KernelAndDevice {
   std::function<void(std::function<void()>)>* runner_;
   std::function<void(std::function<void()>)> default_runner_;
   const bool log_memory_;
+  const std::unique_ptr<CollectiveExecutor::Handle> collective_executor_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index 9f4c57e880..19a0c5e5be 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -1527,6 +1527,7 @@ T* OpKernelContext::op_device_context() {
 
 template <typename T>
 T* OpKernelContext::input_device_context(int index) {
+  DCHECK_NE(params_->input_device_contexts, nullptr);
   DCHECK_GE(index, 0);
   DCHECK_LT(index, params_->input_device_contexts->size());
   static_assert(std::is_base_of<DeviceContext, T>::value,
@@ -1535,6 +1536,7 @@ T* OpKernelContext::input_device_context(int index) {
 }
 
 inline DeviceContext* OpKernelContext::input_device_context(int index) {
+  DCHECK_NE(params_->input_device_contexts, nullptr);
   DCHECK_GE(index, 0);
   DCHECK_LT(index, params_->input_device_contexts->size());
   return (*params_->input_device_contexts)[index];
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 2f6b038dda..cbbe5cf49e 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -478,10 +478,6 @@ class Context(object):
     Raises:
       ValueError: If name is not a string or is an invalid device name.
     """
-    devices = self._context_devices
-    if devices is None:
-      self._initialize_handle_and_devices()
-      devices = self._context_devices
     eager_context = self._eager_context
     old_device_name = eager_context.device_name
     old_device_spec = eager_context.device_spec
@@ -502,7 +498,9 @@ class Context(object):
         if old_device_name:
           new_device_spec = copy.copy(old_device_spec)
         else:
-          new_device_spec = pydev.DeviceSpec.from_string(devices[0])
+          self._initialize_handle_and_devices()
+          new_device_spec = pydev.DeviceSpec.from_string(
+              self._context_devices[0])
         new_device_spec.merge_from(device_spec)
       else:
         new_device_spec = pydev.DeviceSpec.from_string("")
-- 
GitLab


From a6e4af0a5a8434898f1ed1e7eedc972c42c285f0 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 6 Dec 2018 13:04:58 -0800
Subject: [PATCH 1447/1554] Specify architectures in TFLite
 build_ios_universal_lib.sh

PiperOrigin-RevId: 224392376
---
 .../tools/make/build_ios_universal_lib.sh     | 35 +++++++++++--------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/tools/make/build_ios_universal_lib.sh b/tensorflow/lite/tools/make/build_ios_universal_lib.sh
index be8064ec7c..8b617ef593 100755
--- a/tensorflow/lite/tools/make/build_ios_universal_lib.sh
+++ b/tensorflow/lite/tools/make/build_ios_universal_lib.sh
@@ -19,31 +19,36 @@ set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR/../../../.."
 
-profiling_args=
+usage() {
+  echo "Usage: $(basename "$0") [-a]"
+  echo "-a [build_arch] build for specified arch comma separate for multiple archs (eg: x86_64 arm64)"
+  echo "  default is [x86_64 armv7 armv7s arm64]"
+  echo "-p enable profiling"
+  exit 1
+}
 
-while getopts "p" opt; do
-  case $opt in
+profiling_args=""
+BUILD_ARCHS="x86_64 armv7 armv7s arm64"
+while getopts "a:p" opt_name; do
+  case "$opt_name" in
+    a) BUILD_ARCHS="${OPTARG}";;
     p) profiling_args='-DGEMMLOWP_PROFILING,-DTFLITE_PROFILING_ENABLED';;
-    *) printf "if you want to enable profiling: pass in [-p]\n"
-      exit 2;;
+    *) usage;;
   esac
 done
-
 shift $(($OPTIND - 1))
+
 # Build library for supported architectures and packs them in a fat binary.
 make_library() {
-    for arch in x86_64 armv7 armv7s arm64
+    LIBS=""
+    for arch in $BUILD_ARCHS
     do
-        make -f tensorflow/lite/tools/make/Makefile TARGET=ios TARGET_ARCH=${arch} EXTRA_CXXFLAGS=$profiling_args \
-        -j 8
+        make -f tensorflow/lite/tools/make/Makefile TARGET=ios TARGET_ARCH=${arch} \
+            EXTRA_CXXFLAGS=$profiling_args -j 8
+        LIBS="${LIBS} tensorflow/lite/tools/make/gen/ios_${arch}/lib/${1}"
     done
     mkdir -p tensorflow/lite/tools/make/gen/lib
-    lipo \
-    tensorflow/lite/tools/make/gen/ios_x86_64/lib/${1} \
-    tensorflow/lite/tools/make/gen/ios_armv7/lib/${1} \
-    tensorflow/lite/tools/make/gen/ios_armv7s/lib/${1} \
-    tensorflow/lite/tools/make/gen/ios_arm64/lib/${1} \
-    -create \
+    lipo $LIBS -create \
     -output tensorflow/lite/tools/make/gen/lib/${1}
 }
 
-- 
GitLab


From e33bca07dec7f59052b3d818ce8eb98703b22603 Mon Sep 17 00:00:00 2001
From: Shining Sun <shiningsun@google.com>
Date: Thu, 6 Dec 2018 13:11:25 -0800
Subject: [PATCH 1448/1554] When doing validation within fit using distributed
 strategy, do not copy the weight over.

PiperOrigin-RevId: 224393616
---
 .../contrib/distribute/python/keras_test.py   |  6 ---
 .../python/keras/engine/training_arrays.py    | 17 +++++--
 .../keras/engine/training_distributed.py      | 46 ++++++++-----------
 3 files changed, 31 insertions(+), 38 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index e530ab6f17..331bf7c496 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1273,12 +1273,6 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         # TODO(b/119257215): use the default one once the flakyness is fixed.
         tolerance = 1e-4
 
-      if (use_validation_data and
-          not isinstance(distribution, tpu_strategy.TPUStrategy)):
-        # TODO(b/120435565): Enable tests with use_validation_data once the
-        # the underlying bug is fixed.
-        return
-
       keras.backend.set_image_data_format('channels_last')
       np.random.seed(_RANDOM_SEED)
       random_seed.set_random_seed(_RANDOM_SEED)
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index e9dfbcbcc0..26a809d298 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -138,6 +138,7 @@ def model_iteration(model,
                     steps_per_epoch=None,
                     validation_steps=None,
                     mode='train',
+                    validation_in_fit=False,
                     **kwargs):
   """Loop function for arrays of data with modes 'train'/'test'/'predict'.
 
@@ -164,6 +165,9 @@ def model_iteration(model,
       validation_steps: Number of steps to run validation for (only if doing
         validation from data tensors). Ignored with the default value of `None`.
       mode: One of 'train'/'test'/'predict'.
+      validation_in_fit: if true, then this method is invoked from within
+        training iteration (for validation). In this case, do not copy weights
+        when using a tf.distribute.Strategy.
       **kwargs: Additional arguments for backwards compatibility.
 
   Returns:
@@ -230,8 +234,9 @@ def model_iteration(model,
     aggregator = training_utils.MetricsAggregator(use_steps,
                                                   num_samples_or_steps)
 
-  if model._distribution_strategy:
-    training_distributed._copy_weights_to_distributed_model(model)
+  if model._distribution_strategy and not validation_in_fit:
+    training_distributed._copy_weights_to_distributed_model(
+        model, model._grouped_model)
 
   callbacks.model.stop_training = False
   callbacks._call_begin_hook(mode)
@@ -356,7 +361,8 @@ def model_iteration(model,
           steps_per_epoch=validation_steps,
           callbacks=callbacks,
           verbose=0,
-          mode='test')
+          mode='test',
+          validation_in_fit=True)
       if not isinstance(val_results, list):
         val_results = [val_results]
       epoch_logs.update(
@@ -367,7 +373,10 @@ def model_iteration(model,
   callbacks._call_end_hook(mode)
 
   if model._distribution_strategy:
-    training_distributed._copy_weights_to_original_model(model, mode)
+    if not validation_in_fit:
+      training_distributed._copy_weights_to_original_model(
+          model, model._grouped_model, mode)
+
     scope.__exit__(None, None, None)
 
   if mode == 'train':
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 473f06ded7..9ca5082673 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -163,11 +163,9 @@ def experimental_fit_loop(model,
   do_validation = bool(validation_steps)
 
   # Copy the weights from the original model to each of the replicated models.
-  orig_model_weights = model.get_weights()
   with current_strategy.scope():
-    distributed_model = current_strategy.unwrap(model._grouped_model_train)[0]
-    distributed_training_utils.set_weights(
-        current_strategy, distributed_model, orig_model_weights)
+    _copy_weights_to_distributed_model(model, model._grouped_model_train)
+
   callbacks = cbks.configure_callbacks(
       callbacks,
       model,
@@ -217,9 +215,8 @@ def experimental_fit_loop(model,
       # Since we create a new clone from the original model we need to copy
       # the weights back to the original model before we can run validation.
       with current_strategy.scope():
-        updated_weights = current_strategy.unwrap(
-            model._grouped_model_train)[0].get_weights()
-        model.set_weights(updated_weights)
+        _copy_weights_to_original_model(model, model._grouped_model_train,
+                                        'train')
 
       val_outs = experimental_test_loop(  # pylint: disable=undefined-variable
           model,
@@ -240,9 +237,7 @@ def experimental_fit_loop(model,
 
   # Copy the weights back from the replicated model to the original model.
   with current_strategy.scope():
-    updated_weights = current_strategy.unwrap(
-        model._grouped_model_train)[0].get_weights()
-    model.set_weights(updated_weights)
+    _copy_weights_to_original_model(model, model._grouped_model_train, 'train')
 
   K.get_session().run(current_strategy.finalize())
   return model.history
@@ -345,11 +340,8 @@ def experimental_test_loop(model,
     progbar = Progbar(target=steps)
 
   # Copy the weights from the original model to each of the replicated models.
-  orig_model_weights = model.get_weights()
   with current_strategy.scope():
-    distributed_model = current_strategy.unwrap(model._grouped_model_test)[0]
-    distributed_training_utils.set_weights(
-        current_strategy, distributed_model, orig_model_weights)
+    _copy_weights_to_distributed_model(model, model._grouped_model_test)
 
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
@@ -455,11 +447,8 @@ def experimental_predict_loop(model, iterator, verbose=0, steps=None):
     progbar = Progbar(target=steps)
 
   # Copy the weights from the original model to each of the replicated models.
-  orig_model_weights = model.get_weights()
   with current_strategy.scope():
-    distributed_model = current_strategy.unwrap(model._grouped_model_predict)[0]
-    distributed_training_utils.set_weights(
-        current_strategy, distributed_model, orig_model_weights)
+    _copy_weights_to_distributed_model(model, model._grouped_model_predict)
 
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
@@ -695,22 +684,23 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
   return ins
 
 
-def _copy_weights_to_distributed_model(model):
+def _copy_weights_to_distributed_model(original_model, grouped_model):
   """Copies weights from original model to distributed models."""
-  if model._distribution_strategy:
-    # Copy the weights from the original model to each of the replicated models.
-    orig_model_weights = model.get_weights()
-    distributed_model = model._distribution_strategy.unwrap(
-        model._grouped_model)[0]
-    distributed_training_utils.set_weights(
-        model._distribution_strategy, distributed_model, orig_model_weights)
+  strategy = original_model._distribution_strategy
+  if strategy:
+    # Copy the weights from the original model to each of the replicated
+    # models.
+    orig_model_weights = original_model.get_weights()
+    distributed_model = strategy.unwrap(grouped_model)[0]
+    distributed_training_utils.set_weights(strategy, distributed_model,
+                                           orig_model_weights)
 
 
-def _copy_weights_to_original_model(model, mode):
+def _copy_weights_to_original_model(model, grouped_model, mode):
   """Copies weights from first distributed model back to original model."""
   if model._distribution_strategy and mode == 'train':
     updated_weights = model._distribution_strategy.unwrap(
-        model._grouped_model)[0].get_weights()
+        grouped_model)[0].get_weights()
     model.set_weights(updated_weights)
 
 
-- 
GitLab


From 7a9363e16590c3a713ce06dbf5857149b460b8aa Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 6 Dec 2018 13:17:22 -0800
Subject: [PATCH 1449/1554] Update model for classification app

- Use model referred from our models.md list.
- Download both float/quant

PiperOrigin-RevId: 224394551
---
 tensorflow/lite/examples/android/BUILD        |  2 +-
 .../android/app/download-models.gradle        | 27 ++++++----
 .../tensorflow/demo/ClassifierActivity.java   |  2 +-
 tensorflow/lite/java/demo/app/build.gradle    | 53 +++++++++++++++----
 tensorflow/lite/java/demo/app/src/main/BUILD  |  3 +-
 .../ImageClassifierQuantizedMobileNet.java    |  5 +-
 tensorflow/opensource_only.files              |  2 +
 tensorflow/workspace.bzl                      | 20 +++++--
 third_party/tflite_mobilenet_float.BUILD      | 12 +++++
 third_party/tflite_mobilenet_quant.BUILD      | 12 +++++
 10 files changed, 108 insertions(+), 30 deletions(-)
 create mode 100644 third_party/tflite_mobilenet_float.BUILD
 create mode 100644 third_party/tflite_mobilenet_quant.BUILD

diff --git a/tensorflow/lite/examples/android/BUILD b/tensorflow/lite/examples/android/BUILD
index 761a60314e..80cefd415a 100644
--- a/tensorflow/lite/examples/android/BUILD
+++ b/tensorflow/lite/examples/android/BUILD
@@ -34,7 +34,7 @@ android_binary(
     # to reduce APK size.
     assets = [
         "//tensorflow/lite/examples/android/app/src/main/assets:labels_mobilenet_quant_v1_224.txt",
-        "@tflite_mobilenet//:mobilenet_quant_v1_224.tflite",
+        "@tflite_mobilenet_quant//:mobilenet_v1_1.0_224_quant.tflite",
         "@tflite_conv_actions_frozen//:conv_actions_frozen.tflite",
         "//tensorflow/lite/examples/android/app/src/main/assets:conv_actions_labels.txt",
         "@tflite_mobilenet_ssd//:mobilenet_ssd.tflite",
diff --git a/tensorflow/lite/examples/android/app/download-models.gradle b/tensorflow/lite/examples/android/app/download-models.gradle
index d2f03db5f6..36bd177a1f 100644
--- a/tensorflow/lite/examples/android/app/download-models.gradle
+++ b/tensorflow/lite/examples/android/app/download-models.gradle
@@ -8,13 +8,12 @@
  *     3 model files will be downloaded into given folder of ext.ASSET_DIR
  */
 // hard coded model files
-// LINT.IfChange
 
-def models = ['conv_actions_tflite.zip',
-              'mobilenet_ssd_tflite_v1.zip',
-              'mobilenet_v1_224_android_quant_2017_11_08.zip',
-              'coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip']
-// LINT.ThenChange(//tensorflow/lite/examples/android/BUILD)
+def models = ['https://storage.googleapis.com/download.tensorflow.org/models/tflite/conv_actions_tflite.zip',
+              'https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_ssd_tflite_v1.zip',
+              'https://storage.googleapis.com/download.tensorflow.org/models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip',
+              'http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz',
+              'http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz']
 
 // Root URL for model archives
 def MODEL_URL = 'https://storage.googleapis.com/download.tensorflow.org/models/tflite'
@@ -30,9 +29,9 @@ buildscript {
 
 import de.undercouch.gradle.tasks.download.Download
 task downloadFile(type: Download){
-    for (f in models) {
-        def modelUrl = MODEL_URL + "/" + f
-        println "Downloading ${f} from ${modelUrl}"
+    for (modelUrl in models) {
+        def localFile = modelUrl.split("/")[-1]
+        println "Downloading ${localFile} from ${modelUrl}"
         src modelUrl
     }
 
@@ -43,7 +42,12 @@ task downloadFile(type: Download){
 task extractModels(type: Copy) {
     for (f in models) {
         def localFile = f.split("/")[-1]
-        from zipTree(project.ext.TMP_DIR + '/' + localFile)
+        def localExt = localFile.split("[.]")[-1]
+        if (localExt == "tgz") {
+            from tarTree(project.ext.TMP_DIR + '/' + localFile)
+        } else {
+            from zipTree(project.ext.TMP_DIR + '/' + localFile)
+        }
     }
 
     into file(project.ext.ASSET_DIR)
@@ -63,6 +67,9 @@ task extractModels(type: Copy) {
     }
 }
 
+
+
+
 tasks.whenTaskAdded { task ->
     if (task.name == 'assembleDebug') {
         task.dependsOn 'extractModels'
diff --git a/tensorflow/lite/examples/android/app/src/main/java/org/tensorflow/demo/ClassifierActivity.java b/tensorflow/lite/examples/android/app/src/main/java/org/tensorflow/demo/ClassifierActivity.java
index dcbbefbeab..698251d8b4 100644
--- a/tensorflow/lite/examples/android/app/src/main/java/org/tensorflow/demo/ClassifierActivity.java
+++ b/tensorflow/lite/examples/android/app/src/main/java/org/tensorflow/demo/ClassifierActivity.java
@@ -65,7 +65,7 @@ public class ClassifierActivity extends CameraActivity implements OnImageAvailab
   // --input_binary=true
   private static final int INPUT_SIZE = 224;
 
-  private static final String MODEL_FILE = "mobilenet_quant_v1_224.tflite";
+  private static final String MODEL_FILE = "mobilenet_v1_1.0_224_quant.tflite";
   private static final String LABEL_FILE = "labels_mobilenet_quant_v1_224.txt";
 
   private static final boolean MAINTAIN_ASPECT = true;
diff --git a/tensorflow/lite/java/demo/app/build.gradle b/tensorflow/lite/java/demo/app/build.gradle
index 05301ebf88..5e50ed4b94 100644
--- a/tensorflow/lite/java/demo/app/build.gradle
+++ b/tensorflow/lite/java/demo/app/build.gradle
@@ -52,28 +52,60 @@ dependencies {
     compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
 }
 
-def modelDownloadUrl = "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip"
-def localCache = "build/intermediates/mobilenet_v1_224_android_quant_2017_11_08.zip"
 def targetFolder = "src/main/assets"
+def modelFloatDownloadUrl = "http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz"
+def modelQuantDownloadUrl = "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz"
+def localCacheFloat = "build/intermediates/mobilenet_v1_1.0_224.tgz"
+def localCacheQuant = "build/intermediates/mmobilenet_v1_1.0_224_quant.tgz"
 
-task downloadModel(type: DownloadUrlTask) {
+
+task downloadModelFloat(type: DownloadUrlTask) {
+    doFirst {
+        println "Downloading ${modelFloatDownloadUrl}"
+    }
+    sourceUrl = "${modelFloatDownloadUrl}"
+    target = file("${localCacheFloat}")
+}
+
+task downloadModelQuant(type: DownloadUrlTask) {
     doFirst {
-        println "Downloading ${modelDownloadUrl}"
+        println "Downloading ${modelQuantDownloadUrl}"
     }
-    sourceUrl = "${modelDownloadUrl}"
-    target = file("${localCache}")
+    sourceUrl = "${modelQuantDownloadUrl}"
+    target = file("${localCacheQuant}")
 }
 
-task unzipModel(type: Copy, dependsOn: 'downloadModel') {
+task unzipModelFloat(type: Copy, dependsOn: 'downloadModelFloat') {
     doFirst {
-        println "Unzipping ${localCache}"
+        println "Unzipping ${localCacheFloat}"
     }
-    from zipTree("${localCache}")
+    from tarTree("${localCacheFloat}")
     into "${targetFolder}"
 }
 
+task unzipModelQuant(type: Copy, dependsOn: 'downloadModelQuant') {
+    doFirst {
+        println "Unzipping ${localCacheQuant}"
+    }
+    from tarTree("${localCacheQuant}")
+    into "${targetFolder}"
+}
+
+task cleanUnusedFiles(type: Delete, dependsOn: ['unzipModelFloat', 'unzipModelQuant']) {
+    delete fileTree("${targetFolder}").matching {
+        include "*.pb"
+        include "*.ckpt.*"
+        include "*.pbtxt.*"
+        include "*.quant_info.*"
+        include "*.meta"
+    }
+}
+
+
 // Ensure the model file is downloaded and extracted before every build
-preBuild.dependsOn unzipModel
+preBuild.dependsOn unzipModelFloat
+preBuild.dependsOn unzipModelQuant
+preBuild.dependsOn cleanUnusedFiles
 
 class DownloadUrlTask extends DefaultTask {
     @Input
@@ -87,3 +119,4 @@ class DownloadUrlTask extends DefaultTask {
         ant.get(src: sourceUrl, dest: target)
     }
 }
+
diff --git a/tensorflow/lite/java/demo/app/src/main/BUILD b/tensorflow/lite/java/demo/app/src/main/BUILD
index df8a024a57..9a7c1d0b61 100644
--- a/tensorflow/lite/java/demo/app/src/main/BUILD
+++ b/tensorflow/lite/java/demo/app/src/main/BUILD
@@ -10,7 +10,8 @@ android_binary(
     aapt_version = "aapt",
     assets = [
         "//tensorflow/lite/java/demo/app/src/main/assets:labels_mobilenet_quant_v1_224.txt",
-        "@tflite_mobilenet//:mobilenet_quant_v1_224.tflite",
+        "@tflite_mobilenet_quant//:mobilenet_v1_1.0_224_quant.tflite",
+        "@tflite_mobilenet_float//:mobilenet_v1_1.0_224.tflite",
     ],
     assets_dir = "",
     custom_package = "com.example.android.tflitecamerademo",
diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java
index e164ac7554..6310a56168 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java
@@ -42,8 +42,9 @@ public class ImageClassifierQuantizedMobileNet extends ImageClassifier {
   @Override
   protected String getModelPath() {
     // you can download this file from
-    // https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip
-    return "mobilenet_quant_v1_224.tflite";
+    // see build.gradle for where to obtain this file. It should be auto
+    // downloaded into assets.
+    return "mobilenet_v1_1.0_224_quant.tflite";
   }
 
   @Override
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 0c29ac6a30..688a837dac 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -173,6 +173,7 @@ tensorflow/third_party/common.bzl
 tensorflow/third_party/com_google_absl.BUILD
 tensorflow/third_party/pprof.BUILD
 tensorflow/third_party/BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
 tensorflow/third_party/lmdb.BUILD
 tensorflow/third_party/git/BUILD.tpl
 tensorflow/third_party/git/BUILD
@@ -198,6 +199,7 @@ tensorflow/third_party/nanopb.BUILD
 tensorflow/third_party/gif.BUILD
 tensorflow/third_party/double_conversion.BUILD
 tensorflow/third_party/six.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
 tensorflow/third_party/repo.bzl
 tensorflow/third_party/codegen.BUILD
 tensorflow/third_party/cub.BUILD
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index e6b4a89e3b..f475493446 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -734,12 +734,22 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     )
 
     tf_http_archive(
-        name = "tflite_mobilenet",
-        build_file = clean_dep("//third_party:tflite_mobilenet.BUILD"),
-        sha256 = "23f814d1c076bdf03715dfb6cab3713aa4fbdf040fd5448c43196bd2e97a4c1b",
+        name = "tflite_mobilenet_float",
+        build_file = clean_dep("//third_party:tflite_mobilenet_float.BUILD"),
+        sha256 = "2fadeabb9968ec6833bee903900dda6e61b3947200535874ce2fe42a8493abc0",
         urls = [
-            "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
-            "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
+            "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz",
+            "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz",
+        ],
+    )
+
+    tf_http_archive(
+        name = "tflite_mobilenet_quant",
+        build_file = clean_dep("//third_party:tflite_mobilenet_quant.BUILD"),
+        sha256 = "d32432d28673a936b2d6281ab0600c71cf7226dfe4cdcef3012555f691744166",
+        urls = [
+            "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz",
+            "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz",
         ],
     )
 
diff --git a/third_party/tflite_mobilenet_float.BUILD b/third_party/tflite_mobilenet_float.BUILD
new file mode 100644
index 0000000000..de47ed61f9
--- /dev/null
+++ b/third_party/tflite_mobilenet_float.BUILD
@@ -0,0 +1,12 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(
+    glob(
+        ["**/*"],
+        exclude = [
+            "BUILD",
+        ],
+    ),
+)
diff --git a/third_party/tflite_mobilenet_quant.BUILD b/third_party/tflite_mobilenet_quant.BUILD
new file mode 100644
index 0000000000..de47ed61f9
--- /dev/null
+++ b/third_party/tflite_mobilenet_quant.BUILD
@@ -0,0 +1,12 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(
+    glob(
+        ["**/*"],
+        exclude = [
+            "BUILD",
+        ],
+    ),
+)
-- 
GitLab


From 3c257d337439f863759d1884c88d17c2b0d685b5 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 6 Dec 2018 13:31:57 -0800
Subject: [PATCH 1450/1554] [tf.data] Export `tf.data.experimental.Structure`
 and its subclasses.

These classes are most useful when implementing the
`tf.data.Dataset._element_structure` property in `Dataset`
subclasses. This change enables users to implement `Dataset`
subclasses without depending on an internal module.

PiperOrigin-RevId: 224397031
---
 .../python/data/experimental/__init__.py      | 10 ++++++++++
 tensorflow/python/data/ops/BUILD              |  1 +
 tensorflow/python/data/ops/dataset_ops.py     |  1 +
 tensorflow/python/data/ops/optional_ops.py    |  2 ++
 tensorflow/python/data/util/structure.py      |  4 ++++
 ...data.experimental.-dataset-structure.pbtxt | 18 +++++++++++++++++
 ...ata.experimental.-optional-structure.pbtxt | 18 +++++++++++++++++
 ...xperimental.-sparse-tensor-structure.pbtxt | 18 +++++++++++++++++
 ...sorflow.data.experimental.-structure.pbtxt | 16 +++++++++++++++
 ....data.experimental.-tensor-structure.pbtxt | 18 +++++++++++++++++
 .../v1/tensorflow.data.experimental.pbtxt     | 20 +++++++++++++++++++
 ...data.experimental.-dataset-structure.pbtxt | 18 +++++++++++++++++
 ...ata.experimental.-optional-structure.pbtxt | 18 +++++++++++++++++
 ...xperimental.-sparse-tensor-structure.pbtxt | 18 +++++++++++++++++
 ...sorflow.data.experimental.-structure.pbtxt | 16 +++++++++++++++
 ....data.experimental.-tensor-structure.pbtxt | 18 +++++++++++++++++
 .../v2/tensorflow.data.experimental.pbtxt     | 20 +++++++++++++++++++
 17 files changed, 234 insertions(+)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-dataset-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optional-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-tensor-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-dataset-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optional-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-tensor-structure.pbtxt

diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 14dfec37cd..365c53405c 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -25,14 +25,19 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@Counter
 @@CheckpointInputPipelineHook
 @@CsvDataset
+@@DatasetStructure
 @@OptimizationOptions
 @@Optional
+@@OptionalStructure
 @@RandomDataset
 @@Reducer
+@@SparseTensorStructure
 @@SqlDataset
 @@StatsAggregator
 @@StatsOptions
+@@Structure
 @@TFRecordWriter
+@@TensorStructure
 @@ThreadingOptions
 
 @@bucket_by_sequence_length
@@ -112,8 +117,13 @@ from tensorflow.python.data.experimental.ops.stats_options import StatsOptions
 from tensorflow.python.data.experimental.ops.threading_options import ThreadingOptions
 from tensorflow.python.data.experimental.ops.unique import unique
 from tensorflow.python.data.experimental.ops.writers import TFRecordWriter
+from tensorflow.python.data.ops.dataset_ops import DatasetStructure
 from tensorflow.python.data.ops.iterator_ops import get_next_as_optional
 from tensorflow.python.data.ops.optional_ops import Optional
+from tensorflow.python.data.ops.optional_ops import OptionalStructure
+from tensorflow.python.data.util.structure import SparseTensorStructure
+from tensorflow.python.data.util.structure import Structure
+from tensorflow.python.data.util.structure import TensorStructure
 # pylint: enable=unused-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD
index 45d65956f3..fbff7df9c3 100644
--- a/tensorflow/python/data/ops/BUILD
+++ b/tensorflow/python/data/ops/BUILD
@@ -87,6 +87,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:util",
         "//tensorflow/python/data/util:structure",
     ],
 )
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index cd7b1a4a5a..a7cd2f54c7 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1915,6 +1915,7 @@ class _VariantDataset(DatasetV2):
     return self._structure
 
 
+@tf_export("data.experimental.DatasetStructure")
 class DatasetStructure(structure_lib.Structure):
   """Represents a `Dataset` of structured values."""
 
diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py
index 15ec755c67..66011d8518 100644
--- a/tensorflow/python/data/ops/optional_ops.py
+++ b/tensorflow/python/data/ops/optional_ops.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.util.tf_export import tf_export
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -145,6 +146,7 @@ class _OptionalImpl(Optional):
     return self._value_structure
 
 
+@tf_export("data.experimental.OptionalStructure")
 class OptionalStructure(structure.Structure):
   """Represents an optional potentially containing a structured value."""
 
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 92450abda8..4f2c6cd853 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -28,11 +28,13 @@ from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import sparse_ops
+from tensorflow.python.util.tf_export import tf_export
 
 
 _STRUCTURE_CONVERSION_FUNCTION_REGISTRY = {}
 
 
+@tf_export("data.experimental.Structure")
 @six.add_metaclass(abc.ABCMeta)
 class Structure(object):
   """Represents structural information, such as type and shape, about a value.
@@ -376,6 +378,7 @@ class NestedStructure(Structure):
         lambda s: s._batch(batch_size), self._nested_structure))
 
 
+@tf_export("data.experimental.TensorStructure")
 class TensorStructure(Structure):
   """Represents structural information about a `tf.Tensor`."""
 
@@ -439,6 +442,7 @@ class TensorStructure(Structure):
         tensor_shape.TensorShape([batch_size]).concatenate(self._shape))
 
 
+@tf_export("data.experimental.SparseTensorStructure")
 class SparseTensorStructure(Structure):
   """Represents structural information about a `tf.SparseTensor`."""
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-dataset-structure.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-dataset-structure.pbtxt
new file mode 100644
index 0000000000..dcb304f763
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-dataset-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.DatasetStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'element_structure\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optional-structure.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optional-structure.pbtxt
new file mode 100644
index 0000000000..bf41c1d1d6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optional-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.OptionalStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.ops.optional_ops.OptionalStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'value_structure\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt
new file mode 100644
index 0000000000..f97376b328
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.SparseTensorStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.SparseTensorStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'dtype\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-structure.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-structure.pbtxt
new file mode 100644
index 0000000000..a99db4542e
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-structure.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.data.experimental.Structure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-tensor-structure.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-tensor-structure.pbtxt
new file mode 100644
index 0000000000..f5c8864a9d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-tensor-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.TensorStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.TensorStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index f981b1af17..234507e5de 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CsvDataset"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "DatasetStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "INFINITE_CARDINALITY"
     mtype: "<type \'int\'>"
@@ -24,6 +28,10 @@ tf_module {
     name: "Optional"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "OptionalStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "RandomDataset"
     mtype: "<type \'type\'>"
@@ -32,6 +40,10 @@ tf_module {
     name: "Reducer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseTensorStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SqlDataset"
     mtype: "<type \'type\'>"
@@ -44,10 +56,18 @@ tf_module {
     name: "StatsOptions"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Structure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TFRecordWriter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TensorStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ThreadingOptions"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-dataset-structure.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-dataset-structure.pbtxt
new file mode 100644
index 0000000000..dcb304f763
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-dataset-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.DatasetStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.ops.dataset_ops.DatasetStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'element_structure\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optional-structure.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optional-structure.pbtxt
new file mode 100644
index 0000000000..bf41c1d1d6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optional-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.OptionalStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.ops.optional_ops.OptionalStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'value_structure\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt
new file mode 100644
index 0000000000..f97376b328
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-sparse-tensor-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.SparseTensorStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.SparseTensorStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'dtype\', \'dense_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-structure.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-structure.pbtxt
new file mode 100644
index 0000000000..a99db4542e
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-structure.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.data.experimental.Structure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-tensor-structure.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-tensor-structure.pbtxt
new file mode 100644
index 0000000000..f5c8864a9d
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-tensor-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.TensorStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.TensorStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'dtype\', \'shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index f981b1af17..234507e5de 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -12,6 +12,10 @@ tf_module {
     name: "CsvDataset"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "DatasetStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "INFINITE_CARDINALITY"
     mtype: "<type \'int\'>"
@@ -24,6 +28,10 @@ tf_module {
     name: "Optional"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "OptionalStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "RandomDataset"
     mtype: "<type \'type\'>"
@@ -32,6 +40,10 @@ tf_module {
     name: "Reducer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SparseTensorStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SqlDataset"
     mtype: "<type \'type\'>"
@@ -44,10 +56,18 @@ tf_module {
     name: "StatsOptions"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "Structure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TFRecordWriter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TensorStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ThreadingOptions"
     mtype: "<type \'type\'>"
-- 
GitLab


From 479abd88927e54205ea418f68e64057e5b837e2d Mon Sep 17 00:00:00 2001
From: Austin Anderson <angerson@google.com>
Date: Wed, 5 Dec 2018 13:58:00 -0800
Subject: [PATCH 1451/1554] Add subdirectory-output support and change official
 releases

This PR does a few things:

* Add slice option `dockerfile_subdirectory` for
  https://github.com/tensorflow/tensorflow/pull/24180, a slice option that
  places any Dockerfile using the slice into a subdirectory of the
  Dockerfiles directory.
* Rearrange releases and slices slightly. The `devel` images are now
  going to be built nightly, to best support master, and are tagged
  `devel-*`.
---
 tensorflow/tools/dockerfiles/assembler.py     | 27 ++++++++++-------
 ...ockerfile => devel-cpu-jupyter.Dockerfile} |  3 ++
 ...-devel.Dockerfile => devel-cpu.Dockerfile} |  3 ++
 ...ockerfile => devel-gpu-jupyter.Dockerfile} |  4 +++
 ...-devel.Dockerfile => devel-gpu.Dockerfile} |  4 +++
 ...ockerfile => devel-cpu.partial.Dockerfile} |  3 ++
 ...erfile => devel-nvidia.partial.Dockerfile} |  4 +++
 tensorflow/tools/dockerfiles/spec.yml         | 29 ++++++++++++++-----
 8 files changed, 59 insertions(+), 18 deletions(-)
 rename tensorflow/tools/dockerfiles/dockerfiles/{cpu-devel-jupyter.Dockerfile => devel-cpu-jupyter.Dockerfile} (94%)
 rename tensorflow/tools/dockerfiles/dockerfiles/{cpu-devel.Dockerfile => devel-cpu.Dockerfile} (92%)
 rename tensorflow/tools/dockerfiles/dockerfiles/{gpu-devel-jupyter.Dockerfile => devel-gpu-jupyter.Dockerfile} (95%)
 rename tensorflow/tools/dockerfiles/dockerfiles/{gpu-devel.Dockerfile => devel-gpu.Dockerfile} (95%)
 rename tensorflow/tools/dockerfiles/partials/ubuntu/{cpu-devel.partial.Dockerfile => devel-cpu.partial.Dockerfile} (73%)
 rename tensorflow/tools/dockerfiles/partials/ubuntu/{nvidia-devel.partial.Dockerfile => devel-nvidia.partial.Dockerfile} (90%)

diff --git a/tensorflow/tools/dockerfiles/assembler.py b/tensorflow/tools/dockerfiles/assembler.py
index bd8ed73a41..67a0320241 100644
--- a/tensorflow/tools/dockerfiles/assembler.py
+++ b/tensorflow/tools/dockerfiles/assembler.py
@@ -170,6 +170,8 @@ slice_sets:
              type: string
            dockerfile_exclusive_name:
              type: string
+           dockerfile_subdirectory:
+             type: string
            partials:
              type: list
              schema:
@@ -353,8 +355,9 @@ def gather_slice_list_items(slices, key):
 def find_first_slice_value(slices, key):
   """For a list of slices, get the first value for a certain key."""
   for s in slices:
-    if key in s:
+    if key in s and s[key] is not None:
       return s[key]
+  return None
 
 
 def assemble_tags(spec, cli_args, enabled_releases, all_partials):
@@ -389,6 +392,8 @@ def assemble_tags(spec, cli_args, enabled_releases, all_partials):
         used_partials = gather_slice_list_items(slices, 'partials')
         used_tests = gather_slice_list_items(slices, 'tests')
         test_runtime = find_first_slice_value(slices, 'test_runtime')
+        dockerfile_subdirectory = find_first_slice_value(
+            slices, 'dockerfile_subdirectory')
         dockerfile_contents = merge_partials(spec['header'], used_partials,
                                              all_partials)
 
@@ -398,6 +403,7 @@ def assemble_tags(spec, cli_args, enabled_releases, all_partials):
             'is_dockerfiles': release['is_dockerfiles'],
             'upload_images': release['upload_images'],
             'cli_args': tag_args,
+            'dockerfile_subdirectory': dockerfile_subdirectory or '',
             'partials': used_partials,
             'tests': used_tests,
             'test_runtime': test_runtime,
@@ -416,8 +422,7 @@ def merge_partials(header, used_partials, all_partials):
 def upload_in_background(hub_repository, dock, image, tag):
   """Upload a docker image (to be used by multiprocessing)."""
   image.tag(hub_repository, tag=tag)
-  for line in list(dock.images.push(hub_repository, tag=tag, stream=True)):
-    print(line)
+  print(dock.images.push(hub_repository, tag=tag))
 
 
 def mkdir_p(path):
@@ -525,13 +530,15 @@ def main(argv):
         continue
 
       # Write releases marked "is_dockerfiles" into the Dockerfile directory
-      if FLAGS.construct_dockerfiles:
-        path = os.path.join(FLAGS.dockerfile_dir, tag + '.Dockerfile')
-        if tag_def['is_dockerfiles']:
-          eprint('>> Writing {}...'.format(path))
-          if not FLAGS.dry_run:
-            with open(path, 'w') as f:
-              f.write(tag_def['dockerfile_contents'])
+      if FLAGS.construct_dockerfiles and tag_def['is_dockerfiles']:
+        path = os.path.join(FLAGS.dockerfile_dir,
+                            tag_def['dockerfile_subdirectory'],
+                            tag + '.Dockerfile')
+        eprint('>> Writing {}...'.format(path))
+        if not FLAGS.dry_run:
+          mkdir_p(os.path.dirname(path))
+          with open(path, 'w') as f:
+            f.write(tag_def['dockerfile_contents'])
 
       # Don't build any images for dockerfile-only releases
       if not FLAGS.build_images:
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
similarity index 94%
rename from tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
index 14ddf08199..43265676f8 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
@@ -46,6 +46,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  
 ENV CI_BUILD_PYTHON python
 
+# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
 
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
similarity index 92%
rename from tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
index 16973b47af..5c5b2f9163 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/cpu-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
@@ -46,6 +46,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  
 ENV CI_BUILD_PYTHON python
 
+# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
 
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
similarity index 95%
rename from tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
index 9ecaec38c2..8769e4e9cd 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
@@ -79,6 +79,10 @@ ENV TF_CUDNN_VERSION=7
 # NCCL 2.x
 ENV TF_NCCL_VERSION=2
 
+# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
similarity index 95%
rename from tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
rename to tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
index c79bc3cf4c..809cda679e 100644
--- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
@@ -79,6 +79,10 @@ ENV TF_CUDNN_VERSION=7
 # NCCL 2.x
 ENV TF_NCCL_VERSION=2
 
+# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
+
 ARG USE_PYTHON_3_NOT_2
 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3}
 ARG PYTHON=python${_PY_SUFFIX}
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
similarity index 73%
rename from tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
index 901652cc28..a61dfbbe54 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/cpu-devel.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
@@ -23,3 +23,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  
 ENV CI_BUILD_PYTHON python
 
+# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
similarity index 90%
rename from tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile
rename to tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
index 48d457e40c..95f9875012 100644
--- a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia-devel.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
@@ -55,3 +55,7 @@ ENV TF_CUDNN_VERSION=7
 
 # NCCL 2.x
 ENV TF_NCCL_VERSION=2
+
+# Check out TensorFlow source code if --build_arg CHECKOUT_TENSORFLOW=1
+ARG CHECKOUT_TF_SRC=0
+RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src
diff --git a/tensorflow/tools/dockerfiles/spec.yml b/tensorflow/tools/dockerfiles/spec.yml
index 4826ddd8e2..19d96e7a3d 100644
--- a/tensorflow/tools/dockerfiles/spec.yml
+++ b/tensorflow/tools/dockerfiles/spec.yml
@@ -33,21 +33,28 @@ header: |
 #   - nightly-py3
 #   - nightly-gpu (similar)
 #   - nightly-gpu-py3
-
+#
+# Releases are all treated differently by TensorFlow's CI systems.
 releases:
+    # Built Nightly and pushed to tensorflow/tensorflow
     nightly:
         tag_specs:
             - "{nightly}{py}{jupyter}"
+            - "{ubuntu-devel}{py}"
 
+    # Built per-release and pushed to tensorflow/tensorflow
+    # --arg _TAG_PREFIX=<val> should be set to "1.11" (for example) or "latest".
     versioned:
         tag_specs:
             - "{_TAG_PREFIX}{ubuntu}{py}{jupyter}"
 
-    ubuntu-dockerfiles:
+    # Dockerfiles stored in the TF repo; not pushed anywhere
+    dockerfiles:
         is_dockerfiles: true
         upload_images: false
         tag_specs:
             - "{ubuntu}{jupyter}"
+            - "{ubuntu-devel}{jupyter}"
 
 slice_sets:
 
@@ -87,27 +94,33 @@ slice_sets:
           tests:
               - import-gpu.sh
           test_runtime: nvidia
-        - add_to_name: "-devel"
-          dockerfile_exclusive_name: "cpu-devel"
+
+    ubuntu-devel:
+        - add_to_name: "devel"
+          dockerfile_exclusive_name: "devel-cpu"
           partials:
               - ubuntu/version
-              - ubuntu/cpu-devel
+              - ubuntu/devel-cpu
               - ubuntu/python
               - ubuntu/bazel
               - shell
           tests:
               - build-cpu.sh
-        - add_to_name: "-gpu-devel"
-          dockerfile_exclusive_name: "gpu-devel"
+          args:
+              - CHECKOUT_TF_SRC=1
+        - add_to_name: "devel-gpu"
+          dockerfile_exclusive_name: "devel-gpu"
           partials:
               - ubuntu/version
-              - ubuntu/nvidia-devel
+              - ubuntu/devel-nvidia
               - ubuntu/python
               - ubuntu/bazel
               - shell
           tests:
               - build-gpu.sh
           test_runtime: nvidia
+          args:
+              - CHECKOUT_TF_SRC=1
 
     nightly:
         - add_to_name: "nightly"
-- 
GitLab


From 441471313ca001920987f4f8d3b1ba47e4c73dfc Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 6 Dec 2018 14:34:02 -0800
Subject: [PATCH 1452/1554] Add v2 APIs for metrics:
 `SensitivityAtSpecificity`, `SpecificityAtSensitivity`

PiperOrigin-RevId: 224409174
---
 tensorflow/python/keras/metrics.py            |  82 ++++----
 .../tools/api/generator/api_init_files.bzl    |   1 -
 ...metrics.-sensitivity-at-specificity.pbtxt} |   9 +-
 ...metrics.-specificity-at-sensitivity.pbtxt} |   8 +-
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   8 +
 .../tensorflow.metrics.-binary-accuracy.pbtxt | 194 ------------------
 ...orflow.metrics.-categorical-accuracy.pbtxt | 194 ------------------
 .../golden/v1/tensorflow.metrics.-mean.pbtxt  | 192 -----------------
 .../v1/tensorflow.metrics.-recall.pbtxt       | 192 -----------------
 ...metrics.-sparse-categorical-accuracy.pbtxt | 194 ------------------
 .../tensorflow.metrics.-true-negatives.pbtxt  | 193 -----------------
 .../tensorflow.metrics.-true-positives.pbtxt  | 193 -----------------
 .../api/golden/v1/tensorflow.metrics.pbtxt    |  44 ----
 ...metrics.-sensitivity-at-specificity.pbtxt} |   7 +-
 ...metrics.-specificity-at-sensitivity.pbtxt} |   8 +-
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   8 +
 .../v2/tensorflow.metrics.-accuracy.pbtxt     | 194 ------------------
 .../tensorflow.metrics.-binary-accuracy.pbtxt | 194 ------------------
 ...orflow.metrics.-categorical-accuracy.pbtxt | 194 ------------------
 .../tensorflow.metrics.-false-negatives.pbtxt | 193 -----------------
 .../tensorflow.metrics.-false-positives.pbtxt | 193 -----------------
 .../golden/v2/tensorflow.metrics.-mean.pbtxt  | 192 -----------------
 .../v2/tensorflow.metrics.-precision.pbtxt    | 192 -----------------
 .../v2/tensorflow.metrics.-recall.pbtxt       | 192 -----------------
 ...metrics.-sparse-categorical-accuracy.pbtxt | 194 ------------------
 .../tensorflow.metrics.-true-negatives.pbtxt  | 193 -----------------
 .../tensorflow.metrics.-true-positives.pbtxt  | 193 -----------------
 .../api/golden/v2/tensorflow.metrics.pbtxt    |  47 -----
 .../tools/api/golden/v2/tensorflow.pbtxt      |   4 -
 29 files changed, 74 insertions(+), 3628 deletions(-)
 rename tensorflow/tools/api/golden/v1/{tensorflow.metrics.-accuracy.pbtxt => tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt} (93%)
 rename tensorflow/tools/api/golden/v1/{tensorflow.metrics.-false-positives.pbtxt => tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt} (93%)
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt
 rename tensorflow/tools/api/golden/{v1/tensorflow.metrics.-precision.pbtxt => v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt} (93%)
 rename tensorflow/tools/api/golden/{v1/tensorflow.metrics.-false-negatives.pbtxt => v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt} (93%)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 1d3c6e67d7..331a8636d1 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -511,7 +511,7 @@ class Metric(Layer):
   ### End: For use by subclasses ###
 
 
-@tf_export('metrics.Mean', 'keras.metrics.Mean')
+@tf_export('keras.metrics.Mean')
 class Mean(Metric):
   """Computes the (weighted) mean of the given values.
 
@@ -528,7 +528,7 @@ class Mean(Metric):
   Usage:
 
   ```python
-  m = tf.metrics.Mean()
+  m = tf.keras.metrics.Mean()
   m.update_state([1, 3, 5, 7])
   print('Final result: ', m.result().numpy())  # Final result: 4.0
   ```
@@ -537,7 +537,7 @@ class Mean(Metric):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.add_metric(metrics_module.Mean(name='mean_1')(outputs))
+  model.add_metric(tf.keras.metrics.Mean(name='mean_1')(outputs))
   model.compile('sgd', loss='mse')
   ```
   """
@@ -651,7 +651,7 @@ class MeanMetricWrapper(Mean):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@tf_export('metrics.Accuracy', 'keras.metrics.Accuracy')
+@tf_export('keras.metrics.Accuracy')
 class Accuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
@@ -670,7 +670,7 @@ class Accuracy(MeanMetricWrapper):
   Usage:
 
   ```python
-  m = tf.metrics.Accuracy()
+  m = tf.keras.metrics.Accuracy()
   m.update_state([1, 2, 3, 4], [0, 2, 3, 4])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
@@ -679,7 +679,7 @@ class Accuracy(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.Accuracy()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.Accuracy()])
   ```
   """
 
@@ -693,7 +693,7 @@ class Accuracy(MeanMetricWrapper):
     return super(Accuracy, cls).from_config(config)
 
 
-@tf_export('metrics.BinaryAccuracy', 'keras.metrics.BinaryAccuracy')
+@tf_export('keras.metrics.BinaryAccuracy')
 class BinaryAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
@@ -712,7 +712,7 @@ class BinaryAccuracy(MeanMetricWrapper):
   Usage:
 
   ```python
-  m = tf.metrics.BinaryAccuracy()
+  m = tf.keras.metrics.BinaryAccuracy()
   m.update_state([1, 1, 0, 0], [0.98, 1, 0, 0.6])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
@@ -721,7 +721,7 @@ class BinaryAccuracy(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.BinaryAccuracy()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.BinaryAccuracy()])
   ```
   """
 
@@ -744,8 +744,7 @@ class BinaryAccuracy(MeanMetricWrapper):
     return super(BinaryAccuracy, cls).from_config(config)
 
 
-@tf_export(
-    'metrics.CategoricalAccuracy', 'keras.metrics.CategoricalAccuracy')
+@tf_export('keras.metrics.CategoricalAccuracy')
 class CategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches labels.
 
@@ -768,7 +767,7 @@ class CategoricalAccuracy(MeanMetricWrapper):
   Usage:
 
   ```python
-  m = tf.metrics.CategoricalAccuracy()
+  m = tf.keras.metrics.CategoricalAccuracy()
   m.update_state([[0, 0, 1], [0, 1, 0]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
   print('Final result: ', m.result().numpy())  # Final result: 0.5
   ```
@@ -777,7 +776,10 @@ class CategoricalAccuracy(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.CategoricalAccuracy()])
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.CategoricalAccuracy()])
   ```
   """
 
@@ -798,9 +800,7 @@ class CategoricalAccuracy(MeanMetricWrapper):
     return super(CategoricalAccuracy, cls).from_config(config)
 
 
-@tf_export(
-    'metrics.SparseCategoricalAccuracy',
-    'keras.metrics.SparseCategoricalAccuracy')
+@tf_export('keras.metrics.SparseCategoricalAccuracy')
 class SparseCategoricalAccuracy(MeanMetricWrapper):
   """Calculates how often predictions matches integer labels.
 
@@ -820,7 +820,7 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
   Usage:
 
   ```python
-  m = tf.metrics.SparseCategoricalAccuracy()
+  m = tf.keras.metrics.SparseCategoricalAccuracy()
   m.update_state([[2], [1]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
   print('Final result: ', m.result().numpy())  # Final result: 0.5
   ```
@@ -832,7 +832,7 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
   model.compile(
       'sgd',
       loss='mse',
-      metrics=[tf.metrics.SparseCategoricalAccuracy()])
+      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
   ```
   """
 
@@ -907,7 +907,7 @@ class _ConfusionMatrixConditionCount(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
 
-@tf_export('metrics.FalsePositives', 'keras.metrics.FalsePositives')
+@tf_export('keras.metrics.FalsePositives')
 class FalsePositives(_ConfusionMatrixConditionCount):
   """Calculates the number of false positives.
 
@@ -925,7 +925,7 @@ class FalsePositives(_ConfusionMatrixConditionCount):
   Usage:
 
   ```python
-  m = tf.metrics.FalsePositives()
+  m = tf.keras.metrics.FalsePositives()
   m.update_state([0, 1, 0, 0], [0, 0, 1, 1])
   print('Final result: ', m.result().numpy())  # Final result: 2
   ```
@@ -934,7 +934,7 @@ class FalsePositives(_ConfusionMatrixConditionCount):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.FalsePositives()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.FalsePositives()])
   ```
   """
 
@@ -957,7 +957,7 @@ class FalsePositives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
-@tf_export('metrics.FalseNegatives', 'keras.metrics.FalseNegatives')
+@tf_export('keras.metrics.FalseNegatives')
 class FalseNegatives(_ConfusionMatrixConditionCount):
   """Calculates the number of false negatives.
 
@@ -975,7 +975,7 @@ class FalseNegatives(_ConfusionMatrixConditionCount):
   Usage:
 
   ```python
-  m = tf.metrics.FalseNegatives()
+  m = tf.keras.metrics.FalseNegatives()
   m.update_state([0, 1, 1, 1], [0, 1, 0, 0])
   print('Final result: ', m.result().numpy())  # Final result: 2
   ```
@@ -984,7 +984,7 @@ class FalseNegatives(_ConfusionMatrixConditionCount):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.FalseNegatives()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.FalseNegatives()])
   ```
   """
 
@@ -1007,7 +1007,7 @@ class FalseNegatives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
-@tf_export('metrics.TrueNegatives', 'keras.metrics.TrueNegatives')
+@tf_export('keras.metrics.TrueNegatives')
 class TrueNegatives(_ConfusionMatrixConditionCount):
   """Calculates the number of true negatives.
 
@@ -1025,7 +1025,7 @@ class TrueNegatives(_ConfusionMatrixConditionCount):
   Usage:
 
   ```python
-  m = tf.metrics.TrueNegatives()
+  m = tf.keras.metrics.TrueNegatives()
   m.update_state([0, 1, 0, 0], [1, 1, 0, 0])
   print('Final result: ', m.result().numpy())  # Final result: 2
   ```
@@ -1034,7 +1034,7 @@ class TrueNegatives(_ConfusionMatrixConditionCount):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.TrueNegatives()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.TrueNegatives()])
   ```
   """
 
@@ -1057,7 +1057,7 @@ class TrueNegatives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
-@tf_export('metrics.TruePositives', 'keras.metrics.TruePositives')
+@tf_export('keras.metrics.TruePositives')
 class TruePositives(_ConfusionMatrixConditionCount):
   """Calculates the number of true positives.
 
@@ -1075,7 +1075,7 @@ class TruePositives(_ConfusionMatrixConditionCount):
   Usage:
 
   ```python
-  m = tf.metrics.TruePositives()
+  m = tf.keras.metrics.TruePositives()
   m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
   print('Final result: ', m.result().numpy())  # Final result: 2
   ```
@@ -1084,7 +1084,7 @@ class TruePositives(_ConfusionMatrixConditionCount):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.TruePositives()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.TruePositives()])
   ```
   """
 
@@ -1107,7 +1107,7 @@ class TruePositives(_ConfusionMatrixConditionCount):
         dtype=dtype)
 
 
-@tf_export('metrics.Precision', 'keras.metrics.Precision')
+@tf_export('keras.metrics.Precision')
 class Precision(Metric):
   """Computes the precision of the predictions with respect to the labels.
 
@@ -1126,7 +1126,7 @@ class Precision(Metric):
   Usage:
 
   ```python
-  m = tf.metrics.Precision()
+  m = tf.keras.metrics.Precision()
   m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
   print('Final result: ', m.result().numpy())  # Final result: 0.66
   ```
@@ -1135,7 +1135,7 @@ class Precision(Metric):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.Precision()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.Precision()])
   ```
   """
 
@@ -1192,7 +1192,7 @@ class Precision(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
 
-@tf_export('metrics.Recall', 'keras.metrics.Recall')
+@tf_export('keras.metrics.Recall')
 class Recall(Metric):
   """Computes the recall of the predictions with respect to the labels.
 
@@ -1211,7 +1211,7 @@ class Recall(Metric):
   Usage:
 
   ```python
-  m = tf.metrics.Recall()
+  m = tf.keras.metrics.Recall()
   m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
   print('Final result: ', m.result().numpy())  # Final result: 0.66
   ```
@@ -1220,7 +1220,7 @@ class Recall(Metric):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss='mse', metrics=[tf.metrics.Recall()])
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.Recall()])
   ```
   """
 
@@ -1341,6 +1341,7 @@ class SensitivitySpecificityBase(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
 
+@tf_export('keras.metrics.SensitivityAtSpecificity')
 class SensitivityAtSpecificity(SensitivitySpecificityBase):
   """Computes the sensitivity at a given specificity.
 
@@ -1363,7 +1364,7 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
   Usage:
 
   ```python
-  m = tf.metrics.SensitivityAtSpecificity(0.4, num_thresholds=1)
+  m = tf.keras.metrics.SensitivityAtSpecificity(0.4, num_thresholds=1)
   m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
   print('Final result: ', m.result().numpy())  # Final result: 0.5
   ```
@@ -1375,7 +1376,7 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
   model.compile(
       'sgd',
       loss='mse',
-      metrics=[tf.metrics.SensitivityAtSpecificity()])
+      metrics=[tf.keras.metrics.SensitivityAtSpecificity()])
   ```
   """
 
@@ -1409,6 +1410,7 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
                                self.tp[min_index] + self.fn[min_index])
 
 
+@tf_export('keras.metrics.SpecificityAtSensitivity')
 class SpecificityAtSensitivity(SensitivitySpecificityBase):
   """Computes the specificity at a given sensitivity.
 
@@ -1431,7 +1433,7 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
   Usage:
 
   ```python
-  m = tf.metrics.SpecificityAtSensitivity(0.8, num_thresholds=1)
+  m = tf.keras.metrics.SpecificityAtSensitivity(0.8, num_thresholds=1)
   m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
   print('Final result: ', m.result().numpy())  # Final result: 1.0
   ```
@@ -1443,7 +1445,7 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
   model.compile(
       'sgd',
       loss='mse',
-      metrics=[tf.metrics.SpecificityAtSensitivity()])
+      metrics=[tf.keras.metrics.SpecificityAtSensitivity()])
   ```
   """
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 3517c11cc9..0245ac50a6 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -64,7 +64,6 @@ TENSORFLOW_API_INIT_FILES = [
     "lite/constants/__init__.py",
     "losses/__init__.py",
     "math/__init__.py",
-    "metrics/__init__.py",
     "nn/__init__.py",
     "nn/rnn_cell/__init__.py",
     "quantization/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
similarity index 93%
rename from tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index f8e12f8817..aa77d1972c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -1,8 +1,7 @@
-path: "tensorflow.metrics.Accuracy"
+path: "tensorflow.keras.metrics.SensitivityAtSpecificity"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivityAtSpecificity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
@@ -85,7 +84,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
+    argspec: "args=[\'self\', \'specificity\', \'num_thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
similarity index 93%
rename from tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index 9953162ea3..67857aa89f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -1,7 +1,7 @@
-path: "tensorflow.metrics.FalsePositives"
+path: "tensorflow.keras.metrics.SpecificityAtSensitivity"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SpecificityAtSensitivity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
@@ -84,7 +84,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'sensitivity\', \'num_thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 8cab17edc5..905021dd79 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -32,10 +32,18 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SensitivityAtSpecificity"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SpecificityAtSensitivity"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrueNegatives"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt
deleted file mode 100644
index b9bc6a716a..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-binary-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.BinaryAccuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt
deleted file mode 100644
index 0ef75d8756..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-categorical-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.CategoricalAccuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt
deleted file mode 100644
index 7fe6d6fda9..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-mean.pbtxt
+++ /dev/null
@@ -1,192 +0,0 @@
-path: "tensorflow.metrics.Mean"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt
deleted file mode 100644
index 840a68bbc7..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-recall.pbtxt
+++ /dev/null
@@ -1,192 +0,0 @@
-path: "tensorflow.metrics.Recall"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
deleted file mode 100644
index 7bce43fbde..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.SparseCategoricalAccuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt
deleted file mode 100644
index 83cd5b736b..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-negatives.pbtxt
+++ /dev/null
@@ -1,193 +0,0 @@
-path: "tensorflow.metrics.TrueNegatives"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt
deleted file mode 100644
index 5b2502eafe..0000000000
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-true-positives.pbtxt
+++ /dev/null
@@ -1,193 +0,0 @@
-path: "tensorflow.metrics.TruePositives"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt
index f5c267a166..e9b996c9f5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.metrics.pbtxt
@@ -1,49 +1,5 @@
 path: "tensorflow.metrics"
 tf_module {
-  member {
-    name: "Accuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "BinaryAccuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "CategoricalAccuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FalseNegatives"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FalsePositives"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Mean"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Precision"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Recall"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "SparseCategoricalAccuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "TrueNegatives"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "TruePositives"
-    mtype: "<type \'type\'>"
-  }
   member_method {
     name: "accuracy"
     argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
similarity index 93%
rename from tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt
rename to tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index 8c3271a109..aa77d1972c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -1,6 +1,7 @@
-path: "tensorflow.metrics.Precision"
+path: "tensorflow.keras.metrics.SensitivityAtSpecificity"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivityAtSpecificity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
@@ -83,7 +84,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'specificity\', \'num_thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
similarity index 93%
rename from tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt
rename to tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index 33226a2df6..67857aa89f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -1,7 +1,7 @@
-path: "tensorflow.metrics.FalseNegatives"
+path: "tensorflow.keras.metrics.SpecificityAtSensitivity"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SpecificityAtSensitivity\'>"
+  is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
@@ -84,7 +84,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'sensitivity\', \'num_thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'200\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 8cab17edc5..905021dd79 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -32,10 +32,18 @@ tf_module {
     name: "Recall"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SensitivityAtSpecificity"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "SparseCategoricalAccuracy"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SpecificityAtSensitivity"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrueNegatives"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
deleted file mode 100644
index f8e12f8817..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.Accuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Accuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'accuracy\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
deleted file mode 100644
index b9bc6a716a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-binary-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.BinaryAccuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.BinaryAccuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\', \'threshold\'], varargs=None, keywords=None, defaults=[\'binary_accuracy\', \'None\', \'0.5\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
deleted file mode 100644
index 0ef75d8756..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-categorical-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.CategoricalAccuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.CategoricalAccuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'categorical_accuracy\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
deleted file mode 100644
index 33226a2df6..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-negatives.pbtxt
+++ /dev/null
@@ -1,193 +0,0 @@
-path: "tensorflow.metrics.FalseNegatives"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.FalseNegatives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
deleted file mode 100644
index 9953162ea3..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-false-positives.pbtxt
+++ /dev/null
@@ -1,193 +0,0 @@
-path: "tensorflow.metrics.FalsePositives"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.FalsePositives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
deleted file mode 100644
index 7fe6d6fda9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-mean.pbtxt
+++ /dev/null
@@ -1,192 +0,0 @@
-path: "tensorflow.metrics.Mean"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'mean\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'values\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
deleted file mode 100644
index 8c3271a109..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-precision.pbtxt
+++ /dev/null
@@ -1,192 +0,0 @@
-path: "tensorflow.metrics.Precision"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
deleted file mode 100644
index 840a68bbc7..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-recall.pbtxt
+++ /dev/null
@@ -1,192 +0,0 @@
-path: "tensorflow.metrics.Recall"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
deleted file mode 100644
index 7bce43fbde..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-sparse-categorical-accuracy.pbtxt
+++ /dev/null
@@ -1,194 +0,0 @@
-path: "tensorflow.metrics.SparseCategoricalAccuracy"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.SparseCategoricalAccuracy\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.MeanMetricWrapper\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'sparse_categorical_accuracy\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
deleted file mode 100644
index 83cd5b736b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-negatives.pbtxt
+++ /dev/null
@@ -1,193 +0,0 @@
-path: "tensorflow.metrics.TrueNegatives"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.TrueNegatives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
deleted file mode 100644
index 5b2502eafe..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.-true-positives.pbtxt
+++ /dev/null
@@ -1,193 +0,0 @@
-path: "tensorflow.metrics.TruePositives"
-tf_class {
-  is_instance: "<class \'tensorflow.python.keras.metrics.TruePositives\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
-  is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
-  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "activity_regularizer"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "dtype"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "inbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "input_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "losses"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "name"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "non_trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "outbound_nodes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_mask"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "output_shape"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "trainable_weights"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "updates"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "variables"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "weights"
-    mtype: "<type \'property\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_loss"
-    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_metric"
-    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_update"
-    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_variable"
-    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
-  }
-  member_method {
-    name: "apply"
-    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "build"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "compute_mask"
-    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "compute_output_shape"
-    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "count_params"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_config"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_input_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_losses_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_mask_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_output_shape_at"
-    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_updates_for"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_weights"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reset_states"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "result"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "set_weights"
-    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "update_state"
-    argspec: "args=[\'self\', \'y_true\', \'y_pred\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
deleted file mode 100644
index 773efd03fc..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.metrics.pbtxt
+++ /dev/null
@@ -1,47 +0,0 @@
-path: "tensorflow.metrics"
-tf_module {
-  member {
-    name: "Accuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "BinaryAccuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "CategoricalAccuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FalseNegatives"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FalsePositives"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Mean"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Precision"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Recall"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "SparseCategoricalAccuracy"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "TrueNegatives"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "TruePositives"
-    mtype: "<type \'type\'>"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 7b1c96c2e8..542e5cbe9a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -216,10 +216,6 @@ tf_module {
     name: "math"
     mtype: "<type \'module\'>"
   }
-  member {
-    name: "metrics"
-    mtype: "<type \'module\'>"
-  }
   member {
     name: "name_scope"
     mtype: "<type \'type\'>"
-- 
GitLab


From ac7864268b480abf216e437d64c1311c7d1362b5 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 6 Dec 2018 14:55:13 -0800
Subject: [PATCH 1453/1554] Make kInputLayerNormWeightsTensor optional.

PiperOrigin-RevId: 224413157
---
 tensorflow/lite/kernels/layer_norm_lstm.cc    |  41 ++--
 .../lite/kernels/layer_norm_lstm_test.cc      | 223 +++++++++++++++++-
 2 files changed, 243 insertions(+), 21 deletions(-)

diff --git a/tensorflow/lite/kernels/layer_norm_lstm.cc b/tensorflow/lite/kernels/layer_norm_lstm.cc
index 5b0046a7b3..49e8a53c82 100644
--- a/tensorflow/lite/kernels/layer_norm_lstm.cc
+++ b/tensorflow/lite/kernels/layer_norm_lstm.cc
@@ -55,7 +55,7 @@ constexpr int kCellToForgetWeightsTensor = 10;  // Optional
 constexpr int kCellToOutputWeightsTensor = 11;  // Optional
 
 // Layer norm weights tensors of size {n_cell}, representing a diagonal matrix.
-constexpr int kInputLayerNormWeightsTensor = 12;
+constexpr int kInputLayerNormWeightsTensor = 12;  // Optional
 constexpr int kForgetLayerNormWeightsTensor = 13;
 constexpr int kCellLayerNormWeightsTensor = 14;
 constexpr int kOutputLayerNormWeightsTensor = 15;
@@ -118,7 +118,8 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
 
   const TfLiteTensor* input_to_input_weights =
       GetOptionalInputTensor(context, node, kInputToInputWeightsTensor);
-  if (input_to_input_weights != nullptr) {
+  const bool use_cifg = (input_to_input_weights == nullptr);
+  if (!use_cifg) {
     TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->size, 2);
     TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->data[0], n_cell);
     TF_LITE_ENSURE_EQ(context, input_to_input_weights->dims->data[1], n_input);
@@ -138,7 +139,9 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
 
   const TfLiteTensor* recurrent_to_input_weights =
       GetOptionalInputTensor(context, node, kRecurrentToInputWeightsTensor);
-  if (recurrent_to_input_weights != nullptr) {
+  if (use_cifg) {
+    TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights, nullptr);
+  } else {
     TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->size, 2);
     TF_LITE_ENSURE_EQ(context, recurrent_to_input_weights->dims->data[0],
                       n_cell);
@@ -161,15 +164,6 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
   TF_LITE_ENSURE_EQ(context, recurrent_to_cell_weights->dims->data[1],
                     n_output);
 
-  // We make sure the input-gate's parameters are either both present (regular
-  // LSTM) or not at all (CIFG-LSTM).
-  const bool cifg_weights_all_or_none =
-      ((input_to_input_weights != nullptr) &&
-       (recurrent_to_input_weights != nullptr)) ||
-      ((input_to_input_weights == nullptr) &&
-       (recurrent_to_input_weights == nullptr));
-  TF_LITE_ENSURE(context, cifg_weights_all_or_none == true);
-
   const TfLiteTensor* cell_to_input_weights =
       GetOptionalInputTensor(context, node, kCellToInputWeightsTensor);
   if (cell_to_input_weights) {
@@ -192,7 +186,6 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
   }
 
   // Making sure the peephole weights are there all or none.
-  const bool use_cifg = (input_to_input_weights == nullptr);
   const bool peephole_weights_all_or_none =
       ((cell_to_input_weights != nullptr || use_cifg) &&
        (cell_to_forget_weights != nullptr) &&
@@ -204,10 +197,14 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
 
   // Making sure layer norm weights are not null and have the right dimension.
   const TfLiteTensor* input_layer_norm_weights =
-      GetInput(context, node, kInputLayerNormWeightsTensor);
-  TF_LITE_ENSURE(context, input_layer_norm_weights != nullptr);
-  TF_LITE_ENSURE_EQ(context, input_layer_norm_weights->dims->size, 1);
-  TF_LITE_ENSURE_EQ(context, input_layer_norm_weights->dims->data[0], n_cell);
+      GetOptionalInputTensor(context, node, kInputLayerNormWeightsTensor);
+  if (use_cifg) {
+    TF_LITE_ENSURE_EQ(context, input_layer_norm_weights, nullptr);
+  } else {
+    TF_LITE_ENSURE(context, input_layer_norm_weights != nullptr);
+    TF_LITE_ENSURE_EQ(context, input_layer_norm_weights->dims->size, 1);
+    TF_LITE_ENSURE_EQ(context, input_layer_norm_weights->dims->data[0], n_cell);
+  }
 
   const TfLiteTensor* forget_layer_norm_weights =
       GetInput(context, node, kForgetLayerNormWeightsTensor);
@@ -978,6 +975,9 @@ TfLiteStatus EvalFloat(
       (projection_weights == nullptr) ? nullptr : projection_weights->data.f;
   const float* projection_bias_ptr =
       (projection_bias == nullptr) ? nullptr : projection_bias->data.f;
+  const float* input_layer_norm_weight_ptr =
+      (input_layer_norm_weights == nullptr) ? nullptr
+                                            : input_layer_norm_weights->data.f;
 
   // Required tensors, pointers are non-null.
   const float* input_ptr_batch = input->data.f;
@@ -990,7 +990,6 @@ TfLiteStatus EvalFloat(
       recurrent_to_cell_weights->data.f;
   const float* recurrent_to_output_weights_ptr =
       recurrent_to_output_weights->data.f;
-  const float* input_layer_norm_weight_ptr = input_layer_norm_weights->data.f;
   const float* forget_layer_norm_weight_ptr = forget_layer_norm_weights->data.f;
   const float* cell_layer_norm_weight_ptr = cell_layer_norm_weights->data.f;
   const float* output_layer_norm_weight_ptr = output_layer_norm_weights->data.f;
@@ -1115,6 +1114,9 @@ TfLiteStatus EvalHybrid(
       (projection_weights == nullptr) ? 1.0f : projection_weights->params.scale;
   const float* projection_bias_ptr =
       (projection_bias == nullptr) ? nullptr : projection_bias->data.f;
+  const float* input_layer_norm_weight_ptr =
+      (input_layer_norm_weights == nullptr) ? nullptr
+                                            : input_layer_norm_weights->data.f;
 
   // Required tensors, pointers are non-null.
   const float* input_ptr_batch = input->data.f;
@@ -1141,7 +1143,6 @@ TfLiteStatus EvalHybrid(
       reinterpret_cast<int8_t*>(recurrent_to_output_weights->data.uint8);
   const float recurrent_to_output_weights_scale =
       recurrent_to_output_weights->params.scale;
-  const float* input_layer_norm_weight_ptr = input_layer_norm_weights->data.f;
   const float* forget_layer_norm_weight_ptr = forget_layer_norm_weights->data.f;
   const float* cell_layer_norm_weight_ptr = cell_layer_norm_weights->data.f;
   const float* output_layer_norm_weight_ptr = output_layer_norm_weights->data.f;
@@ -1221,7 +1222,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       GetOptionalInputTensor(context, node, kCellToOutputWeightsTensor);
 
   const TfLiteTensor* input_layer_norm_weights =
-      GetInput(context, node, kInputLayerNormWeightsTensor);
+      GetOptionalInputTensor(context, node, kInputLayerNormWeightsTensor);
   const TfLiteTensor* forget_layer_norm_weights =
       GetInput(context, node, kForgetLayerNormWeightsTensor);
   const TfLiteTensor* cell_layer_norm_weights =
diff --git a/tensorflow/lite/kernels/layer_norm_lstm_test.cc b/tensorflow/lite/kernels/layer_norm_lstm_test.cc
index e89bce50c3..1c13cee1c3 100644
--- a/tensorflow/lite/kernels/layer_norm_lstm_test.cc
+++ b/tensorflow/lite/kernels/layer_norm_lstm_test.cc
@@ -83,7 +83,11 @@ class LayerNormLSTMOpModel : public SingleOpModel {
       cell_to_output_weights_ = AddNullInput();
     }
 
-    input_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
+    if (use_cifg) {
+      input_layer_norm_weights_ = AddNullInput();
+    } else {
+      input_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
+    }
     forget_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
     cell_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
     output_layer_norm_weights_ = AddInput(TensorType_FLOAT32);
@@ -650,6 +654,223 @@ TEST_F(NoCifgPeepholeProjectionNoClippingLayerNormLstmTest,
                 &layer_norm_lstm);
 }
 
+class CifgPeepholeProjectionNoClippingLayerNormLstmTest
+    : public BaseLayerNormLstmTest {
+  void SetUp() override {
+    input_to_forget_weights_ = {-0.6, -0.1, 0.3,  0.2,  0.9,  -0.5, -0.2,
+                                -0.4, 0.3,  -0.8, -0.4, 0.3,  -0.5, -0.4,
+                                -0.6, 0.3,  -0.4, -0.6, -0.5, -0.5};
+    input_to_cell_weights_ = {-0.4, -0.3, -0.2, -0.1, -0.5, 0.5,  -0.2,
+                              -0.3, -0.2, -0.6, 0.6,  -0.1, -0.4, -0.3,
+                              -0.7, 0.7,  -0.9, -0.5, 0.8,  0.6};
+    input_to_output_weights_ = {-0.8, -0.4, -0.2, -0.9, -0.1, -0.7, 0.3,
+                                -0.3, -0.8, -0.2, 0.6,  -0.2, 0.4,  -0.7,
+                                -0.3, -0.5, 0.1,  0.5,  -0.6, -0.4};
+
+    forget_gate_bias_ = {0.1, -0.3, -0.2, 0.1};
+    cell_gate_bias_ = {-0.05, 0.72, 0.25, 0.08};
+    output_gate_bias_ = {0.05, -0.01, 0.2, 0.1};
+
+    recurrent_to_cell_weights_ = {-0.3, 0.2, 0.1, -0.3, 0.8,  -0.08,
+                                  -0.2, 0.3, 0.8, -0.6, -0.1, 0.2};
+    recurrent_to_forget_weights_ = {-0.5, -0.3, -0.5, -0.2, 0.6, 0.4,
+                                    0.9,  0.3,  -0.1, 0.2,  0.5, 0.2};
+    recurrent_to_output_weights_ = {0.3,  -0.1, 0.1,  -0.2, -0.5, -0.7,
+                                    -0.2, -0.6, -0.1, -0.4, -0.7, -0.2};
+
+    cell_to_forget_weights_ = {-0.02, -0.15, -0.25, -0.03};
+    cell_to_output_weights_ = {0.1, -0.1, -0.5, 0.05};
+
+    forget_layer_norm_weights_ = {0.2, 0.2, 0.4, 0.3};
+    cell_layer_norm_weights_ = {0.7, 0.2, 0.3, 0.8};
+    output_layer_norm_weights_ = {0.6, 0.2, 0.2, 0.5};
+    projection_weights_ = {-0.1, 0.2,  0.01, -0.2, 0.1,  0.5,
+                           0.3,  0.08, 0.07, 0.2,  -0.4, 0.2};
+
+    layer_norm_lstm_input_ = {
+        {// Batch0: 3 (input_sequence_size) * 5 (n_input)
+         0.7, 0.8, 0.1, 0.2, 0.3,   // seq 0
+         0.8, 0.1, 0.2, 0.4, 0.5,   // seq 1
+         0.2, 0.7, 0.7, 0.1, 0.7},  // seq 2
+
+        {// Batch1: 3 (input_sequence_size) * 5 (n_input)
+         0.3, 0.2, 0.9, 0.8, 0.1,   // seq 0
+         0.1, 0.5, 0.2, 0.4, 0.2,   // seq 1
+         0.6, 0.9, 0.2, 0.5, 0.7},  // seq 2
+    };
+  }
+};
+
+TEST_F(CifgPeepholeProjectionNoClippingLayerNormLstmTest,
+       LayerNormLstmBlackBoxTest) {
+  const int n_batch = 2;
+  const int n_input = 5;
+  const int n_cell = 4;
+  const int n_output = 3;
+  const float ceil_clip = 0.0;
+  const float proj_clip = 0.0;
+
+  LayerNormLSTMOpModel layer_norm_lstm(
+      n_batch, n_input, n_cell, n_output,
+      /*use_cifg=*/true, /*use_peephole=*/true,
+      /*use_projection_weights=*/true,
+      /*use_projection_bias=*/false, ceil_clip, proj_clip,
+      {
+          {n_batch, n_input},  // input tensor
+
+          {0, 0},             // input_to_input_weight tensor
+          {n_cell, n_input},  // input_to_forget_weight tensor
+          {n_cell, n_input},  // input_to_cell_weight tensor
+          {n_cell, n_input},  // input_to_output_weight tensor
+
+          {0, 0},              // recurrent_to_input_weight tensor
+          {n_cell, n_output},  // recurrent_to_forget_weight tensor
+          {n_cell, n_output},  // recurrent_to_cell_weight tensor
+          {n_cell, n_output},  // recurrent_to_output_weight tensor
+
+          {0},       // cell_to_input_weight tensor
+          {n_cell},  // cell_to_forget_weight tensor
+          {n_cell},  // cell_to_output_weight tensor
+
+          {0},       // input_layer_norm_weight tensor
+          {n_cell},  // forget_layer_norm_weight tensor
+          {n_cell},  // cell_layer_norm_weight tensor
+          {n_cell},  // output_layer_norm_weight tensor
+
+          {0},       // input_gate_bias tensor
+          {n_cell},  // forget_gate_bias tensor
+          {n_cell},  // cell_bias tensor
+          {n_cell},  // output_gate_bias tensor
+
+          {n_output, n_cell},  // projection_weight tensor
+          {0},                 // projection_bias tensor
+      });
+
+  layer_norm_lstm.SetInputToCellWeights(input_to_cell_weights_);
+  layer_norm_lstm.SetInputToForgetWeights(input_to_forget_weights_);
+  layer_norm_lstm.SetInputToOutputWeights(input_to_output_weights_);
+
+  layer_norm_lstm.SetCellBias(cell_gate_bias_);
+  layer_norm_lstm.SetForgetGateBias(forget_gate_bias_);
+  layer_norm_lstm.SetOutputGateBias(output_gate_bias_);
+
+  layer_norm_lstm.SetRecurrentToCellWeights(recurrent_to_cell_weights_);
+  layer_norm_lstm.SetRecurrentToForgetWeights(recurrent_to_forget_weights_);
+  layer_norm_lstm.SetRecurrentToOutputWeights(recurrent_to_output_weights_);
+
+  layer_norm_lstm.SetCellToForgetWeights(cell_to_forget_weights_);
+  layer_norm_lstm.SetCellToOutputWeights(cell_to_output_weights_);
+
+  layer_norm_lstm.SetForgetLayerNormWeights(forget_layer_norm_weights_);
+  layer_norm_lstm.SetCellLayerNormWeights(cell_layer_norm_weights_);
+  layer_norm_lstm.SetOutputLayerNormWeights(output_layer_norm_weights_);
+
+  layer_norm_lstm.SetProjectionWeights(projection_weights_);
+
+  // Verify the final output.
+  const std::vector<std::vector<float>> layer_norm_lstm_golden_output = {
+      {
+          // Batch0: 3 (input_sequence_size) * 3 (n_output)
+          0.02129706, 0.140816242, 0.0112733059,     // seq 0
+          0.0132302344, 0.152308047, 0.0346313119,   // seq 1
+          -0.0123688057, 0.165790111, 0.0893077999,  // seq 2
+      },
+      {
+          // Batch1: 3 (input_sequence_size) * 3 (n_output)
+          -0.0226350538, 0.0916948169, 0.0769175813,  // seq 0
+          -0.0269966982, 0.149707705, 0.094149217,    // seq 1
+          -0.0103429332, 0.173016444, 0.0720508844,   // seq 2
+      }};
+
+  VerifyGoldens(layer_norm_lstm_input_, layer_norm_lstm_golden_output,
+                &layer_norm_lstm);
+}
+
+TEST_F(CifgPeepholeProjectionNoClippingLayerNormLstmTest,
+       HybridLayerNormLstmBlackBoxTest) {
+  const int n_batch = 2;
+  const int n_input = 5;
+  const int n_cell = 4;
+  const int n_output = 3;
+  const float ceil_clip = 0.0;
+  const float proj_clip = 0.0;
+
+  HybridLayerNormLSTMOpModel layer_norm_lstm(
+      n_batch, n_input, n_cell, n_output,
+      /*use_cifg=*/true, /*use_peephole=*/true,
+      /*use_projection_weights=*/true,
+      /*use_projection_bias=*/false, ceil_clip, proj_clip,
+      {
+          {n_batch, n_input},  // input tensor
+
+          {0, 0},             // input_to_input_weight tensor
+          {n_cell, n_input},  // input_to_forget_weight tensor
+          {n_cell, n_input},  // input_to_cell_weight tensor
+          {n_cell, n_input},  // input_to_output_weight tensor
+
+          {0, 0},              // recurrent_to_input_weight tensor
+          {n_cell, n_output},  // recurrent_to_forget_weight tensor
+          {n_cell, n_output},  // recurrent_to_cell_weight tensor
+          {n_cell, n_output},  // recurrent_to_output_weight tensor
+
+          {0},       // cell_to_input_weight tensor
+          {n_cell},  // cell_to_forget_weight tensor
+          {n_cell},  // cell_to_output_weight tensor
+
+          {0},       // input_layer_norm_weight tensor
+          {n_cell},  // forget_layer_norm_weight tensor
+          {n_cell},  // cell_layer_norm_weight tensor
+          {n_cell},  // output_layer_norm_weight tensor
+
+          {0},       // input_gate_bias tensor
+          {n_cell},  // forget_gate_bias tensor
+          {n_cell},  // cell_bias tensor
+          {n_cell},  // output_gate_bias tensor
+
+          {n_output, n_cell},  // projection_weight tensor
+          {0},                 // projection_bias tensor
+      });
+
+  layer_norm_lstm.SetInputToCellWeights(input_to_cell_weights_);
+  layer_norm_lstm.SetInputToForgetWeights(input_to_forget_weights_);
+  layer_norm_lstm.SetInputToOutputWeights(input_to_output_weights_);
+
+  layer_norm_lstm.SetCellBias(cell_gate_bias_);
+  layer_norm_lstm.SetForgetGateBias(forget_gate_bias_);
+  layer_norm_lstm.SetOutputGateBias(output_gate_bias_);
+
+  layer_norm_lstm.SetRecurrentToCellWeights(recurrent_to_cell_weights_);
+  layer_norm_lstm.SetRecurrentToForgetWeights(recurrent_to_forget_weights_);
+  layer_norm_lstm.SetRecurrentToOutputWeights(recurrent_to_output_weights_);
+
+  layer_norm_lstm.SetCellToForgetWeights(cell_to_forget_weights_);
+  layer_norm_lstm.SetCellToOutputWeights(cell_to_output_weights_);
+
+  layer_norm_lstm.SetForgetLayerNormWeights(forget_layer_norm_weights_);
+  layer_norm_lstm.SetCellLayerNormWeights(cell_layer_norm_weights_);
+  layer_norm_lstm.SetOutputLayerNormWeights(output_layer_norm_weights_);
+
+  layer_norm_lstm.SetProjectionWeights(projection_weights_);
+
+  // Verify the final output.
+  const std::vector<std::vector<float>> layer_norm_lstm_golden_output = {
+      {
+          // Batch0: 3 (input_sequence_size) * 3 (n_output)
+          0.0212250091, 0.140474007, 0.0115012666,   // seq 0
+          0.0130806509, 0.152660668, 0.0347516984,   // seq 1
+          -0.0124010444, 0.166042402, 0.0898982584,  // seq 2
+      },
+      {
+          // Batch1: 3 (input_sequence_size) * 3 (n_output)
+          -0.0228835996, 0.0917588323, 0.0778886303,  // seq 0
+          -0.0275101066, 0.148769245, 0.0938384682,   // seq 1
+          -0.0103605557, 0.172605693, 0.0728750974,   // seq 2
+      }};
+
+  VerifyGoldens(layer_norm_lstm_input_, layer_norm_lstm_golden_output,
+                &layer_norm_lstm);
+}
+
 }  // namespace
 }  // namespace custom
 }  // namespace ops
-- 
GitLab


From 3ca8908b14ce55151ce0e043ae6423fb50d29ade Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 15:06:07 -0800
Subject: [PATCH 1454/1554] gradient_checker_v2.py: some internal improvements.

PiperOrigin-RevId: 224415291
---
 tensorflow/python/ops/gradient_checker_v2.py  | 33 ++++++++++------
 .../python/ops/gradient_checker_v2_test.py    | 38 +++++++++----------
 2 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/ops/gradient_checker_v2.py b/tensorflow/python/ops/gradient_checker_v2.py
index cf84484112..5d473eeb5f 100644
--- a/tensorflow/python/ops/gradient_checker_v2.py
+++ b/tensorflow/python/ops/gradient_checker_v2.py
@@ -44,27 +44,19 @@ def _product(t):
     return y
 
 
-def _to_numpy(a):
-  """Converts tensors to numpy arrays.
+def _eval_indexed_slices(a):
+  """Converts IndexedSlices to IndexedSlicesValue with numpy indices/values.
 
-  Converts Tensors and EagerTensors to numpy arrays.
   When eager execution is enabled, converts IndexedSlices
-  to IndexedSlicesValue with numpy indices/values
+  to IndexedSlicesValue with numpy indices/values.
 
   Args:
     a: any value.
 
   Returns:
-    If a is EagerTensor or Tensor, returns the evaluation of a by calling
-    numpy() or run().
     If a is IndexedSlices and eager execution is enabled, calls numpy() on a's
     fields. Otherwise returns a unchanged.
   """
-  if isinstance(a, ops.EagerTensor):
-    return a.numpy()
-  if isinstance(a, ops.Tensor):
-    sess = ops.get_default_session()
-    return sess.run(a)
   if isinstance(a, ops.IndexedSlices) and context.executing_eagerly():
     return ops.IndexedSlicesValue(
         indices=[x.numpy() for x in a.indices],
@@ -73,6 +65,24 @@ def _to_numpy(a):
   return a
 
 
+def _to_numpy(a):
+  """Converts Tensors and EagerTensors to numpy arrays.
+
+  Args:
+    a: any value.
+
+  Returns:
+    If a is EagerTensor or Tensor, returns the evaluation of a by calling
+    numpy() or run(). Otherwise returns a unchanged.
+  """
+  if isinstance(a, ops.EagerTensor):
+    return a.numpy()
+  if isinstance(a, ops.Tensor):
+    sess = ops.get_default_session()
+    return sess.run(a)
+  return a
+
+
 def _prepare(f, xs_dtypes):
   """Return a function that executes 'f'.
 
@@ -147,6 +157,7 @@ def _compute_theoretical_jacobian(f, y_shape, y_dtype, xs, param):
   for col in range(y_size):
     dy_data_flat[col] = 1
     grad = _to_numpy(grad_fn(dy_data, *xs)[0])
+    grad = _eval_indexed_slices(grad)
     dy_data_flat[col] = 0
     if isinstance(grad, ops.IndexedSlicesValue):
       for i, v in zip(grad.indices, grad.values):
diff --git a/tensorflow/python/ops/gradient_checker_v2_test.py b/tensorflow/python/ops/gradient_checker_v2_test.py
index ce9ff47d61..191b2b6568 100644
--- a/tensorflow/python/ops/gradient_checker_v2_test.py
+++ b/tensorflow/python/ops/gradient_checker_v2_test.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
@@ -38,13 +37,17 @@ from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 
 
+def _random_complex(shape, dtype):
+  data = np.random.random_sample(shape).astype(dtype.as_numpy_dtype)
+  if dtype.is_complex:
+    data.imag = np.random.random_sample(shape)
+  return data
+
+
 @test_util.run_all_in_graph_and_eager_modes
 class GradientCheckerTest(test.TestCase):
 
   def testAddSimple(self):
-    # if context.executing_eagerly():
-    #   return
-    np.random.seed(1)  # Fix seed to avoid flakiness
     size = (2, 3)
     x1 = constant_op.constant(2.0, shape=size, name="x1")
     x2 = constant_op.constant(3.0, shape=size, name="x2")
@@ -54,7 +57,6 @@ class GradientCheckerTest(test.TestCase):
     assert error < 1e-4
 
   def testAddCustomized(self):
-    np.random.seed(3)  # Fix seed to avoid flakiness
     size = (2, 3)
     x1 = constant_op.constant(
         2.0, shape=size, dtype=dtypes.float64, name="x1")
@@ -67,7 +69,6 @@ class GradientCheckerTest(test.TestCase):
     assert error < 1e-10
 
   def testGather(self):
-    np.random.seed(4)  # Fix seed to avoid flakiness
     def f(params):
       index_values = [1, 3]
       indices = constant_op.constant(index_values, name="i")
@@ -82,7 +83,6 @@ class GradientCheckerTest(test.TestCase):
     assert error < 1e-4
 
   def testNestedGather(self):
-    np.random.seed(5)  # Fix seed to avoid flakiness
     def f(params):
       index_values = [1, 3, 5, 6]
       indices = constant_op.constant(index_values, name="i")
@@ -100,33 +100,37 @@ class GradientCheckerTest(test.TestCase):
     assert error < 1e-4
 
   def testComplexMul(self):
-    if not context.executing_eagerly():
-      return
     c = constant_op.constant(5 + 7j, dtype=dtypes.complex64)
     def f(x):
       return c * x
-    x = constant_op.constant(11 - 13j, dtype=dtypes.complex64)
+    x_shape = c.shape
+    x_dtype = c.dtype
+    x = constant_op.constant(_random_complex(x_shape, x_dtype))
     analytical, numerical = gradient_checker.compute_gradient(
-        f, [x], delta=0.1)
+        f, [x])
     correct = np.array([[5, 7], [-7, 5]])
     self.assertAllEqual(correct, analytical[0])
     self.assertAllClose(correct, numerical[0], rtol=1e-4)
+    x = constant_op.constant(_random_complex(x_shape, x_dtype))
     self.assertLess(
         gradient_checker.max_error(*gradient_checker.compute_gradient(
-            f, [x], delta=0.1)), 2e-4)
+            f, [x])), 3e-4)
 
   def testComplexConj(self):
     def f(x):
       return math_ops.conj(x)
-    x = constant_op.constant(11 - 13j, dtype=dtypes.complex64)
+    x_shape = ()
+    x_dtype = dtypes.complex64
+    x = constant_op.constant(_random_complex(x_shape, x_dtype))
     analytical, numerical = gradient_checker.compute_gradient(
-        f, [x], delta=0.1)
+        f, [x])
     correct = np.array([[1, 0], [0, -1]])
     self.assertAllEqual(correct, analytical[0])
     self.assertAllClose(correct, numerical[0], rtol=2e-5)
+    x = constant_op.constant(_random_complex(x_shape, x_dtype))
     self.assertLess(
         gradient_checker.max_error(*gradient_checker.compute_gradient(
-            f, [x], delta=0.1)), 2e-5)
+            f, [x])), 2e-5)
 
   def testEmptySucceeds(self):
     def f(x):
@@ -140,8 +144,6 @@ class GradientCheckerTest(test.TestCase):
     self.assertEqual(error, 0)
 
   def testEmptyFails(self):
-    # if not context.executing_eagerly():
-    #   return
     @custom_gradient.custom_gradient
     def id_bad_grad(x):
       y = array_ops.identity(x)
@@ -279,8 +281,6 @@ class MiniMNISTTest(test.TestCase):
     return err
 
   def testInputGradient(self):
-    # if context.executing_eagerly():
-    #   return
     self.assertLess(self._BuildAndTestMiniMNIST(0, "input"), 1e-8)
 
   def testHiddenWeightGradient(self):
-- 
GitLab


From 9797cc2e91d01b492ed946e05f8f07c2a7f8c157 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 15:15:34 -0800
Subject: [PATCH 1455/1554] Fix PREDICT bug.

PiperOrigin-RevId: 224416988
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index fb1316cf33..84816d70d0 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -530,13 +530,16 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
 
 class TPUInfeedOutfeedSessionHookForPrediction(TPUInfeedOutfeedSessionHook):
 
-  def __init__(self, ctx, enqueue_ops, dequeue_ops, rendezvous=None):
+  def __init__(self, ctx, enqueue_ops, dequeue_ops, rendezvous=None,
+               master=None, session_config=None):
     super(TPUInfeedOutfeedSessionHookForPrediction, self).__init__(
         ctx,
         enqueue_ops,
         dequeue_ops,
         run_infeed_loop_on_coordinator=False,
-        rendezvous=rendezvous)
+        rendezvous=rendezvous,
+        master=master,
+        session_config=session_config)
 
   def _create_infeed_controller(self, name, target, args):
     return _OpSignalOnceQueueContext(name=name, target=target, args=args)
@@ -2748,7 +2751,9 @@ class TPUEstimator(estimator_lib.Estimator):
         hooks = [
             _StoppingPredictHook(scalar_stopping_signal),
             TPUInfeedOutfeedSessionHookForPrediction(
-                ctx, enqueue_ops, host_ops, rendezvous=self._rendezvous[mode]),
+                ctx, enqueue_ops, host_ops, rendezvous=self._rendezvous[mode],
+                master=self._config.master,
+                session_config=self._session_config),
         ] + input_hooks
 
         if prediction_hooks:
-- 
GitLab


From d453dfad7c383db0ded338e8de3142a17d55910b Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Thu, 6 Dec 2018 15:20:01 -0800
Subject: [PATCH 1456/1554] Exports OpsSet in Python API.

PiperOrigin-RevId: 224417739
---
 tensorflow/lite/python/convert.py                    |  1 +
 .../api/golden/v1/tensorflow.lite.-ops-set.pbtxt     | 12 ++++++++++++
 tensorflow/tools/api/golden/v1/tensorflow.lite.pbtxt |  4 ++++
 .../api/golden/v2/tensorflow.lite.-ops-set.pbtxt     | 12 ++++++++++++
 tensorflow/tools/api/golden/v2/tensorflow.lite.pbtxt |  4 ++++
 5 files changed, 33 insertions(+)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.lite.-ops-set.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.lite.-ops-set.pbtxt

diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 563312e027..9c60399871 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -97,6 +97,7 @@ def convert_dtype_to_tflite_type(tf_dtype):
   return result
 
 
+@_tf_export("lite.OpsSet")
 class OpsSet(enum.Enum):
   """Enum class defining the sets of ops available to generate TFLite models.
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.lite.-ops-set.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.lite.-ops-set.pbtxt
new file mode 100644
index 0000000000..68c651a3c9
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.lite.-ops-set.pbtxt
@@ -0,0 +1,12 @@
+path: "tensorflow.lite.OpsSet"
+tf_class {
+  is_instance: "<enum \'OpsSet\'>"
+  member {
+    name: "SELECT_TF_OPS"
+    mtype: "<enum \'OpsSet\'>"
+  }
+  member {
+    name: "TFLITE_BUILTINS"
+    mtype: "<enum \'OpsSet\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.lite.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.lite.pbtxt
index f5013c250b..154dd00821 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.lite.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.lite.pbtxt
@@ -8,6 +8,10 @@ tf_module {
     name: "OpHint"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "OpsSet"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
   member {
     name: "TFLiteConverter"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.lite.-ops-set.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.lite.-ops-set.pbtxt
new file mode 100644
index 0000000000..68c651a3c9
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.lite.-ops-set.pbtxt
@@ -0,0 +1,12 @@
+path: "tensorflow.lite.OpsSet"
+tf_class {
+  is_instance: "<enum \'OpsSet\'>"
+  member {
+    name: "SELECT_TF_OPS"
+    mtype: "<enum \'OpsSet\'>"
+  }
+  member {
+    name: "TFLITE_BUILTINS"
+    mtype: "<enum \'OpsSet\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.lite.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.lite.pbtxt
index f5013c250b..154dd00821 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.lite.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.lite.pbtxt
@@ -8,6 +8,10 @@ tf_module {
     name: "OpHint"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "OpsSet"
+    mtype: "<class \'enum.EnumMeta\'>"
+  }
   member {
     name: "TFLiteConverter"
     mtype: "<type \'type\'>"
-- 
GitLab


From fa06cffc09699de4215cf47b2e00ed6d7cdd4dfe Mon Sep 17 00:00:00 2001
From: Yanhui Liang <yhliang@google.com>
Date: Thu, 6 Dec 2018 15:30:33 -0800
Subject: [PATCH 1457/1554] Update sdca_optimizer in core to its v2 version,
 and migrate linear_optimizer from contrib to core.

PiperOrigin-RevId: 224419608
---
 .../python/kernel_tests/sdca_ops_test.py         |  7 ++++++-
 .../linear_optimizer/python/ops/sdca_ops.py      | 16 ++++++++++++++--
 .../ops/sharded_mutable_dense_hashtable.py       | 12 +++++++++++-
 .../ops/sharded_mutable_dense_hashtable_test.py  |  7 ++++++-
 .../python/ops/sparse_feature_column.py          | 12 +++++++++++-
 .../python/ops/sparse_feature_column_test.py     |  7 ++++++-
 6 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py b/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py
index 8466dc36d1..d49834dc86 100644
--- a/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py
+++ b/tensorflow/contrib/linear_optimizer/python/kernel_tests/sdca_ops_test.py
@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for SdcaModel."""
+"""Tests for SdcaModel (deprecated).
+
+This module and all its submodules are deprecated. To UPDATE or USE linear
+optimizers, please check its latest version in core:
+tensorflow_estimator/python/estimator/canned/linear_optimizer/.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py b/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py
index f3f1dcd98d..c056a12fa5 100644
--- a/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py
+++ b/tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py
@@ -12,7 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Proximal stochastic dual coordinate ascent optimizer for linear models."""
+# pylint: disable=line-too-long
+"""Proximal stochastic dual coordinate ascent optimizer for linear models (deprecated).
+
+This module and all its submodules are deprecated. To UPDATE or USE linear
+optimizers, please check its latest version in core:
+tensorflow_estimator/python/estimator/canned/linear_optimizer/.
+"""
+# pylint: enable=line-too-long
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -40,6 +47,7 @@ from tensorflow.python.ops import variables as var_ops
 from tensorflow.python.ops.nn import log_poisson_loss
 from tensorflow.python.ops.nn import sigmoid_cross_entropy_with_logits
 from tensorflow.python.summary import summary
+from tensorflow.python.util import deprecation
 
 __all__ = ['SdcaModel']
 
@@ -48,7 +56,7 @@ __all__ = ['SdcaModel']
 class SdcaModel(object):
   """Stochastic dual coordinate ascent solver for linear models.
 
-    Loss functions supported:
+  Loss functions supported:
 
      * Binary logistic loss
      * Squared loss
@@ -109,6 +117,10 @@ class SdcaModel(object):
     ```
   """
 
+  @deprecation.deprecated(
+      None, 'This class is deprecated. To UPDATE or USE linear optimizers, '
+      'please check its latest version in core: '
+      'tensorflow_estimator/python/estimator/canned/linear_optimizer/.')
   def __init__(self, examples, variables, options):
     """Create a new sdca optimizer."""
 
diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py b/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py
index a001555e8f..a28394964a 100644
--- a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py
+++ b/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable.py
@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Sharded mutable dense hash table."""
+"""Sharded mutable dense hash table (deprecated).
+
+This module and all its submodules are deprecated. To UPDATE or USE linear
+optimizers, please check its latest version in core:
+tensorflow_estimator/python/estimator/canned/linear_optimizer/.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -28,6 +33,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.util import deprecation
 
 
 # TODO(rohanj): This should subclass Checkpointable and implement
@@ -45,6 +51,10 @@ class ShardedMutableDenseHashTable(object):
 
   # TODO(andreasst): consider moving this to lookup module
 
+  @deprecation.deprecated(
+      None, 'This class is deprecated. To UPDATE or USE linear optimizers, '
+      'please check its latest version in core: '
+      'tensorflow_estimator/python/estimator/canned/linear_optimizer/.')
   def __init__(self,
                key_dtype,
                value_dtype,
diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py b/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py
index 2b56d0fa3a..2d1457f9e4 100644
--- a/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py
+++ b/tensorflow/contrib/linear_optimizer/python/ops/sharded_mutable_dense_hashtable_test.py
@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for sharded_mutable_dense_hashtable.py."""
+"""Tests for sharded_mutable_dense_hashtable.py (deprecated).
+
+This module and all its submodules are deprecated. To UPDATE or USE linear
+optimizers, please check its latest version in core:
+tensorflow_estimator/python/estimator/canned/linear_optimizer/.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py b/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py
index 003795233f..64730f8eed 100644
--- a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py
+++ b/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column.py
@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Sparse feature column."""
+"""Sparse feature column (deprecated).
+
+This module and all its submodules are deprecated. To UPDATE or USE linear
+optimizers, please check its latest version in core:
+tensorflow_estimator/python/estimator/canned/linear_optimizer/.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -21,6 +26,7 @@ from __future__ import print_function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework.ops import internal_convert_to_tensor
 from tensorflow.python.framework.ops import name_scope
+from tensorflow.python.util import deprecation
 
 
 class SparseFeatureColumn(object):
@@ -68,6 +74,10 @@ class SparseFeatureColumn(object):
   @@feature_values
   """
 
+  @deprecation.deprecated(
+      None, 'This class is deprecated. To UPDATE or USE linear optimizers, '
+      'please check its latest version in core: '
+      'tensorflow_estimator/python/estimator/canned/linear_optimizer/.')
   def __init__(self, example_indices, feature_indices, feature_values):
     """Creates a `SparseFeatureColumn` representation.
 
diff --git a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py b/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py
index 51c4f68543..0ae780e1a1 100644
--- a/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py
+++ b/tensorflow/contrib/linear_optimizer/python/ops/sparse_feature_column_test.py
@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for sparse_feature_column.py."""
+"""Tests for sparse_feature_column.py (deprecated).
+
+This module and all its submodules are deprecated. To UPDATE or USE linear
+optimizers, please check its latest version in core:
+tensorflow_estimator/python/estimator/canned/linear_optimizer/.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 938af66d71256aac8c04ac022a4f0390e10146f8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 15:46:13 -0800
Subject: [PATCH 1458/1554] Add missing namespace qualifier that ADL was
 "saving" us from having to provide, but which would break once
 absl::string_view is really std::string_view. PiperOrigin-RevId: 224422387

---
 tensorflow/compiler/xla/service/hlo_pass_pipeline.cc | 2 +-
 tensorflow/compiler/xla/shape_util.cc                | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
index 312b5d020c..51177f24f5 100644
--- a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
+++ b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
@@ -113,7 +113,7 @@ void HloPassPipeline::MaybeDumpHlo(const HloModule& module,
   }
 
   const string message =
-      StrCat("after ", after_pass_name, ", before ", before_pass_name);
+      absl::StrCat("after ", after_pass_name, ", before ", before_pass_name);
   hlo_graph_dumper::MaybeDumpHloModule(module, message);
   VLOG(3) << "HLO " << message << ":";
   VLOG(3) << module.entry_computation_layout().ToString();
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index f3cc51ca91..a4d4e1e53e 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -584,7 +584,7 @@ namespace {
 // Parses shapes with simple recursive descent structure -- consumes from the
 // front of s and passes that view recursively as required.
 StatusOr<Shape> ParseShapeStringInternal(absl::string_view* s) {
-  *s = StripLeadingAsciiWhitespace(*s);
+  *s = absl::StripLeadingAsciiWhitespace(*s);
 
   if (absl::ConsumePrefix(s, "(")) {  // Tuple.
     std::vector<Shape> shapes;
@@ -597,7 +597,7 @@ StatusOr<Shape> ParseShapeStringInternal(absl::string_view* s) {
       }
       shapes.emplace_back();
       TF_ASSIGN_OR_RETURN(shapes.back(), ParseShapeStringInternal(s));
-      *s = StripLeadingAsciiWhitespace(*s);
+      *s = absl::StripLeadingAsciiWhitespace(*s);
       must_end = !absl::ConsumePrefix(s, ",");
     }
     return ShapeUtil::MakeTupleShape(shapes);
-- 
GitLab


From fc4b1847dde6511fb6bfb8d4c3aa1e2586307516 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 6 Dec 2018 15:51:28 -0800
Subject: [PATCH 1459/1554] Update tests to reflect that expanddims and squeeze
 are now converted

---
 tensorflow/contrib/tensorrt/test/rank_two_test.py | 6 ++++--
 tensorflow/contrib/tensorrt/test/unary_test.py    | 3 +--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index 0cd733dca1..563232fc12 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -51,8 +51,10 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
         c = constant_op.constant(3.0, name="c%d_3" % i)
         q = math_ops.add(q, c, name="add%d_3" % i)
         if i == 0:
+          axis = constant_op.constant(-1, dtype=dtypes.int32, name="axis")
           for j in range(2):
-            q = array_ops.expand_dims(q, -1, name="expand%d_%d" % (i, j))
+            q = array_ops.expand_dims(q, axis, name="expand%d_%d" % (i, j))
+          q = self.trt_incompatible_op(q)
         q = gen_math_ops.reciprocal(q, name="reciprocal%d" % i)
         outputs.append(q)
       # Combine both paths
@@ -70,7 +72,7 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
     return {
         "TRTEngineOp_0": [
             "add0_1", "add0_2", "add0_3", "c0_1", "c0_2", "c0_3", "abs0_1",
-            "abs0_2"
+            "abs0_2", "expand0_0", "expand0_1", "axis"
         ],
         "TRTEngineOp_1": [
             "add", "add1_1", "add1_2", "add1_3", "c1_1", "c1_2", "c1_3",
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index 9fc50e0595..b77abc4032 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -107,8 +107,7 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
     return [
-        "TRTEngineOp_0", "TRTEngineOp_1", "TRTEngineOp_2", "TRTEngineOp_3",
-        "TRTEngineOp_4"
+        "TRTEngineOp_0"
     ]
 
 
-- 
GitLab


From 4ca599630233afe5240deae44212f19412ab6423 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 6 Dec 2018 15:53:05 -0800
Subject: [PATCH 1460/1554] Fix stateful metrics in Keras + Distribution
 Strategy codepath.

PiperOrigin-RevId: 224423535
---
 .../contrib/distribute/python/keras_test.py   | 42 +++++++++++++++++--
 tensorflow/python/keras/engine/training.py    | 12 +++++-
 .../python/keras/engine/training_arrays.py    |  6 +--
 .../keras/engine/training_distributed.py      | 15 ++++++-
 4 files changed, 64 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 331bf7c496..796c4ed9f6 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1259,9 +1259,43 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
       train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
 
-      history = model.fit(x=train_dataset, epochs=1, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0])
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
 
+  @combinations.generate(all_strategy_combinations())
+  def test_eval_metrics_correctness(self, distribution):
+    with self.cached_session():
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Dense(
+              3, activation='relu', input_dim=4, kernel_initializer='ones'))
+      model.add(
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones'))
+      model.compile(
+          loss='mae',
+          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+          optimizer=gradient_descent.GradientDescentOptimizer(0.001),
+          distribute=distribution)
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = batch_wrapper(dataset, 4, distribution)
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = batch_wrapper(dataset, 4, distribution)
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  # TODO(priyag): Add metrics correctness to this test to compare with and
+  # without distribution strategies.
   @combinations.generate(strategy_and_input_combinations())
   def test_correctness(self, distribution, use_numpy, use_validation_data):
 
@@ -1319,7 +1353,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
                                         with_distribution,
                                         x_train, y_train, x_predict))
 
-        traning_history = model.fit(**training_inputs).history
+        training_history = model.fit(**training_inputs).history
 
         if eval_inputs is not None:
           eval_result = model.evaluate(**eval_inputs)
@@ -1330,7 +1364,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         weights = model.get_weights()
         predict_result = model.predict(**predict_inputs)
 
-        return weights, traning_history, eval_result, predict_result
+        return weights, training_history, eval_result, predict_result
 
       wts_with_ds, history_with_ds, eval_with_ds, predict_with_ds = (
           fit_eval_and_predict(with_distribution=distribution))
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index c902d928a0..90f8a7b252 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -918,11 +918,17 @@ class Model(Network):
                                     [self.total_loss] + metrics_tensors)
 
   def _make_eval_function(self):
+    # TODO(psv,anjalisridhar): Remove updates after we fix b/118841692
+    # Stateful metrics updates
+    metric_updates = []
+    for m in self.metrics:
+      metric_updates += m.updates
+
     metrics_tensors = [
         self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:]
     ]
-    self._make_test_function_helper('_eval_function',
-                                    [self.total_loss] + metrics_tensors)
+    self._make_test_function_helper(
+        '_eval_function', [self.total_loss] + metrics_tensors, metric_updates)
 
   def _make_predict_function(self):
     if not hasattr(self, 'predict_function'):
@@ -2095,6 +2101,8 @@ class Model(Network):
     if hasattr(self, 'metrics'):
       for m in self.metrics:
         m.reset_states()
+      if self._distribution_strategy:
+        training_distributed._reset_metrics(self)  # pylint: disable=protected-access
 
   def train_on_batch(self,
                      x,
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 26a809d298..196d48faec 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -241,15 +241,14 @@ def model_iteration(model,
   callbacks.model.stop_training = False
   callbacks._call_begin_hook(mode)
   progbar.on_train_begin()
+
   for epoch in range(initial_epoch, epochs):
     if callbacks.model.stop_training:
       break
 
     # Setup work for each epoch
     epoch_logs = {}
-    if hasattr(model, 'metrics'):
-      for m in model.metrics:
-        m.reset_states()
+    model.reset_metrics()
     callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode)
     progbar.on_epoch_begin(epoch, epoch_logs)
 
@@ -373,6 +372,7 @@ def model_iteration(model,
   callbacks._call_end_hook(mode)
 
   if model._distribution_strategy:
+    # TODO(priyag, psv): Copy back metrics to the original model as well?
     if not validation_in_fit:
       training_distributed._copy_weights_to_original_model(
           model, model._grouped_model, mode)
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 9ca5082673..fc72f6bd15 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -183,6 +183,8 @@ def experimental_fit_loop(model,
 
   callbacks.on_train_begin()
   for epoch in range(initial_epoch, epochs):
+    with current_strategy.scope():
+      _reset_metrics(model, model._grouped_model_train)
     callbacks.on_epoch_begin(epoch)
     epoch_logs = {}
     step_index = 0
@@ -342,7 +344,7 @@ def experimental_test_loop(model,
   # Copy the weights from the original model to each of the replicated models.
   with current_strategy.scope():
     _copy_weights_to_distributed_model(model, model._grouped_model_test)
-
+    _reset_metrics(model, model._grouped_model_test)
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
   for step in range(steps):
@@ -449,7 +451,7 @@ def experimental_predict_loop(model, iterator, verbose=0, steps=None):
   # Copy the weights from the original model to each of the replicated models.
   with current_strategy.scope():
     _copy_weights_to_distributed_model(model, model._grouped_model_predict)
-
+    _reset_metrics(model, model._grouped_model_predict)
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
   # the returned Numpy arrays. Instead, we store one array per batch seen
@@ -714,3 +716,12 @@ def _per_device_aggregate_batch(batch_outs, model, mode):
       total_batch_outs.append(np.concatenate(nest.flatten(nested_outs)))
     return total_batch_outs
   return batch_outs
+
+
+def _reset_metrics(model, distributed_model=None):
+  if model._distribution_strategy:
+    distributed_model = (
+        distributed_model or
+        model._distribution_strategy.unwrap(model._grouped_model)[0])
+    distributed_model.reset_metrics()
+
-- 
GitLab


From 9cbc061187bb994acc950730bbb844426c572db1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 15:55:11 -0800
Subject: [PATCH 1461/1554] Return tuples instead of lists from unwrap and
 device methods of tf.distribute.Strategy.

PiperOrigin-RevId: 224423866
---
 .../python/collective_all_reduce_strategy.py  | 14 ++++----
 .../distribute/python/minimize_loss_test.py   |  4 +--
 .../python/mirrored_strategy_multigpu_test.py |  2 +-
 .../distribute/python/one_device_strategy.py  | 12 +++----
 .../python/parameter_server_strategy.py       | 35 +++++++++----------
 .../distribute/python/strategy_test_lib.py    |  2 +-
 .../contrib/distribute/python/tpu_strategy.py | 10 +++---
 .../contrib/optimizer_v2/optimizer_v2.py      |  8 ++---
 .../python/distribute/cross_device_ops.py     |  6 ++--
 .../python/distribute/distribute_lib.py       | 14 ++++----
 .../python/distribute/mirrored_strategy.py    | 19 +++++-----
 11 files changed, 62 insertions(+), 64 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index e988b63a28..5c50a20490 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -77,11 +77,11 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     self._num_workers = 1
 
     if num_gpus_per_worker:
-      local_devices = [
+      local_devices = tuple(
           "/device:GPU:%d" % i for i in range(num_gpus_per_worker)
-      ]
+      )
     else:
-      local_devices = ["/device:CPU:0"]
+      local_devices = ("/device:CPU:0",)
     self._worker_device = device_util.canonicalize("/device:CPU:0")
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
@@ -104,7 +104,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     if task_type is None or task_id is None:
       raise ValueError("When `cluster_spec` is given, you must also specify "
                        "`task_type` and `task_id`")
-    if task_type not in ["chief", "worker"]:
+    if task_type not in ("chief", "worker"):
       raise ValueError(
           "Unrecognized task_type: %r, valid task types are: \"chief\", "
           "\"worker\"." % task_type)
@@ -119,12 +119,12 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
 
     self._worker_device = "/job:%s/task:%d" % (task_type, task_id)
     if num_gpus_per_worker:
-      local_devices = [
+      local_devices = tuple(
           "%s/device:GPU:%d" % (self._worker_device, i)
           for i in range(num_gpus_per_worker)
-      ]
+      )
     else:
-      local_devices = [self._worker_device]
+      local_devices = (self._worker_device,)
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     self._initialize_local(local_devices)
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index dcc9df4cda..f09483cb56 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -232,7 +232,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         fetches = distribution.unwrap(
             distribution.call_for_each_replica(model_fn, args=inputs))
         if update_ops_in_cross_replica_mode:
-          fetches += ops.get_collection(ops.GraphKeys.UPDATE_OPS)
+          fetches += tuple(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
         return control_flow_ops.group(fetches)
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
@@ -443,7 +443,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             step_fn, iterator, iterations=2,
             initial_loop_values=initial_loop_values)
 
-        self.assertEqual({key1: [value1]}, ctx.non_tensor_outputs)
+        self.assertEqual({key1: (value1,)}, ctx.non_tensor_outputs)
         self._verify_loss_output(
             initial_loss(),
             loss_output=ctx.last_step_outputs["replica_loss_reduced"],
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 66512f983e..36be5c83f8 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -180,7 +180,7 @@ class MirroredStrategyVariableCreatorStackTest(
         variable_scope.variable_creator_scope(main_thread_creator):
       result = distribution.extended.call_for_each_replica(model_fn)
       result = distribution.unwrap(result)
-      expected = ["main_thread:thread_0", "main_thread:thread_1"]
+      expected = ("main_thread:thread_0", "main_thread:thread_1")
       self.assertEqual(expected, result)
 
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index e322b6acb8..fdbfba4e04 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -60,7 +60,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     if isinstance(colocate_with, six.string_types):
       with ops.device(colocate_with):
         return next_creator(*args, **kwargs)
-    if (isinstance(colocate_with, list) and len(colocate_with) == 1 and
+    if (isinstance(colocate_with, (list, tuple)) and len(colocate_with) == 1 and
         isinstance(colocate_with[0], six.string_types)):
       with ops.device(colocate_with[0]):
         return next_creator(*args, **kwargs)
@@ -166,7 +166,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     return array_ops.identity(replica_local_var)
 
   def _unwrap(self, value):
-    return [value]
+    return (value,)
 
   def value_container(self, value):
     return value
@@ -177,15 +177,15 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
 
   @property
   def worker_devices(self):
-    return [self._device]
+    return (self._device,)
 
   @property
   def parameter_devices(self):
-    return [self._device]
+    return (self._device,)
 
   def non_slot_devices(self, var_list):
     del var_list
-    return [self._device]
+    return (self._device,)
 
   @property
   def experimental_should_init(self):
@@ -216,4 +216,4 @@ class _OneDeviceReplicaContext(distribute_lib.ReplicaContext):
 
   @property
   def devices(self):
-    return [self._distribution_strategy.extended.worker_devices[0]]
+    return self._distribution_strategy.extended.worker_devices
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index eaeb4d7030..2c7766f95f 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -145,14 +145,14 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
     # replica. When there are GPUs, replicate operations on these GPUs.
     # Otherwise, place operations on CPU.
     if num_gpus_per_worker > 0:
-      self._compute_devices = [
+      self._compute_devices = tuple(
           "%s/device:GPU:%d" % (self._worker_device, i)
           for i in range(num_gpus_per_worker)
-      ]
+      )
     else:
-      self._compute_devices = [self._worker_device]
+      self._compute_devices = (self._worker_device,)
 
-    self._compute_devices = list(
+    self._compute_devices = tuple(
         map(device_util.resolve, self._compute_devices))
     self._canonical_compute_device_set = set(self._compute_devices)
 
@@ -176,8 +176,8 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
     # The `_parameter_devices` is needed for the `parameter_devices` property
     # and is a list of all variable devices. Here parameter devices are all
     # tasks of the "ps" job.
-    self._parameter_devices = map("/job:ps/task:{}".format,
-                                  range(num_ps_replicas))
+    self._parameter_devices = tuple(map("/job:ps/task:{}".format,
+                                        range(num_ps_replicas)))
 
     # Add a default device so that ops without specified devices will not end up
     # on other workers.
@@ -204,24 +204,24 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
     # replica. When there are GPUs, replicate operations on these GPUs.
     # Otherwise, place operations on CPU.
     if num_gpus_per_worker > 0:
-      self._compute_devices = list(
+      self._compute_devices = tuple(
           map("/device:GPU:{}".format, range(num_gpus_per_worker)))
     else:
-      self._compute_devices = [_LOCAL_CPU]
+      self._compute_devices = (_LOCAL_CPU,)
 
-    self._compute_devices = list(
+    self._compute_devices = tuple(
         map(device_util.resolve, self._compute_devices))
     self._canonical_compute_device_set = set(self._compute_devices)
 
     # If there is only one GPU, put everything on that GPU. Otherwise, place
     # variables on CPU.
     if num_gpus_per_worker == 1:
-      assert len(list(self._compute_devices)) == 1
+      assert len(self._compute_devices) == 1
       self._variable_device = _LOCAL_GPU_0
-      self._parameter_devices = [_LOCAL_GPU_0]
+      self._parameter_devices = (_LOCAL_GPU_0,)
     else:
       self._variable_device = _LOCAL_CPU
-      self._parameter_devices = [_LOCAL_CPU]
+      self._parameter_devices = (_LOCAL_CPU,)
 
     self._is_chief = True
     self._cluster_spec = None
@@ -417,9 +417,9 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
     if isinstance(val, values.DistributedValues):
       # Return in a deterministic order.
       if set(val.devices) == self._canonical_compute_device_set:
-        return [val.get(device=d) for d in self._compute_devices]
-      return [val.get(device=d) for d in sorted(val.devices)]
-    return [val]
+        return tuple(val.get(device=d) for d in self._compute_devices)
+      return tuple(val.get(device=d) for d in sorted(val.devices))
+    return (val,)
 
   def value_container(self, val):
     if (hasattr(val, "_aggregating_container") and
@@ -497,12 +497,11 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
 
   @property
   def worker_devices(self):
-    # Make a copy to prevent users from accidentally mutating our copy.
-    return list(self._compute_devices)
+    return self._compute_devices
 
   @property
   def parameter_devices(self):
-    return list(self._parameter_devices)
+    return self._parameter_devices
 
   def non_slot_devices(self, var_list):
     return min(var_list, key=lambda x: x.name)
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index d50b142c5e..d441b5af5f 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -290,4 +290,4 @@ class DistributionTestBase(test.TestCase):
       self.evaluate(strategy.group(train_ops))
       global_step_tensors = strategy.unwrap(value)
       global_step_values = self.evaluate(global_step_tensors)
-      self.assertEqual([1] * len(global_step_tensors), global_step_values)
+      self.assertEqual((1,) * len(global_step_tensors), global_step_values)
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 806ff0ac61..b6f5b49201 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -162,7 +162,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
                   if "device:TPU:" in d.name}
     self._device_index = values.PerReplica(device_map)
     self._host_device = self.get_host_cpu_device(0)
-    self._tpu_devices = sorted(device_map.keys())
+    self._tpu_devices = tuple(sorted(device_map.keys()))
     # Only create variables for the number of replicas we're running.
     self._tpu_devices = self._tpu_devices[:self._num_replicas_in_sync]
 
@@ -507,13 +507,13 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   def _unwrap(self, val):
     if isinstance(val, values.DistributedValues):
       # Return in a deterministic order.
-      return [val.get(device=d) for d in sorted(val.devices)]
+      return tuple(val.get(device=d) for d in sorted(val.devices))
     elif isinstance(val, list):
       # TODO(josh11b): We need to remove this case; per device values should
       # be represented using a PerReplica wrapper instead of a list with
       # one entry per device.
-      return val
-    return [val]
+      return tuple(val)
+    return (val,)
 
   def value_container(self, value):
     return value
@@ -619,4 +619,4 @@ class _TPUReplicaContext(distribute_lib.ReplicaContext):
     distribute_lib.require_replica_context(self)
     ds = self._distribution_strategy
     replica_id = tensor_util.constant_value(self._replica_id_in_sync_group)
-    return [ds.extended.worker_devices[replica_id]]
+    return (ds.extended.worker_devices[replica_id],)
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 11a9248a01..7fb23abc38 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -997,10 +997,10 @@ class OptimizerV2(optimizer_v1.Optimizer):
       with ops.control_dependencies([update_ops]):
         finish_updates = distribution.extended.update_non_slot(
             non_slot_devices, finish, group=False)
-      # We said grouped=False, which means finish_updates is always a list.
-      # It will be [None] when finish() returns None.
-      if finish_updates == [None]:
-        finish_updates = [update_ops]
+      # We said group=False, which means finish_updates is always a tuple.
+      # It will be (None,) when finish() returns None.
+      if finish_updates == (None,):
+        finish_updates = (update_ops,)
 
       # Update `global_step` (if any).
       if global_step is None:
diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index a88ed62533..57c552ca8f 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -53,10 +53,10 @@ def validate_destinations(destinations):
   if not isinstance(
       destinations,
       (value_lib.DistributedValues, resource_variable_ops.ResourceVariable,
-       value_lib.AggregatingVariable, six.string_types, list)):
+       value_lib.AggregatingVariable, six.string_types, list, tuple)):
     raise ValueError("destinations must be one of a `DistributedValues` object,"
-                     " a tf.Variable object, a device string, a list of device "
-                     "strings")
+                     " a tf.Variable object, a device string, a list or tuple "
+                     "of device strings")
 
   if not check_destinations(destinations):
     raise ValueError("destinations can not be empty")
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index eddd6ff8b1..87bf510ec5 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -565,8 +565,8 @@ class DistributionStrategy(object):
         variable created in `scope`.
 
     Returns:
-      A list of values contained in `value`. If `value` represents a single
-      value, this returns `[value].`
+      A tuple of values contained in `value`. If `value` represents a single
+      value, this returns `(value,).`
     """
     return self._extended._unwrap(value)  # pylint: disable=protected-access
 
@@ -1346,14 +1346,14 @@ class DistributionStrategyExtended(object):
 
   @property
   def worker_devices(self):
-    """Returns the list of devices used to run `call_for_each_replica()` calls.
+    """Returns the tuple of all devices used to for compute replica execution.
     """
     # TODO(josh11b): More docstring
     raise NotImplementedError("must be implemented in descendants")
 
   @property
   def parameter_devices(self):
-    """Returns the list of devices used for variable and `update` placement."""
+    """Returns the tuple of all devices used to place variables."""
     # TODO(josh11b): More docstring
     raise NotImplementedError("must be implemented in descendants")
 
@@ -1513,9 +1513,9 @@ class ReplicaContext(object):
 
   @property
   def devices(self):
-    """The devices this replica is to be executed on, as a list of strings."""
+    """The devices this replica is to be executed on, as a tuple of strings."""
     require_replica_context(self)
-    return [device_util.current()]
+    return (device_util.current(),)
 
   # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient
   # all-reduce. It would return a function returning the result of reducing `t`
@@ -1605,7 +1605,7 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
     return array_ops.identity(replica_local_var)
 
   def _unwrap(self, distributed_value):
-    return [distributed_value]
+    return (distributed_value,)
 
   def value_container(self, value):
     return value
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 562f2328a5..56948b2bcb 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -472,7 +472,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     assert len(set(devices)) == len(devices), (
         "No duplicates allowed in `devices` argument.")
     # TODO(josh11b): Require at least 2 devices?
-    self._devices = [device_util.resolve(d) for d in devices]
+    self._devices = tuple(device_util.resolve(d) for d in devices)
     self._canonical_device_set = set(self._devices)
     self._device_index = values.PerReplica(
         {d: i for i, d in enumerate(devices)})
@@ -488,7 +488,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     assert len(set(devices)) == len(devices), (
         "No duplicates allowed in `devices` argument.")
     # TODO(josh11b): Require at least 2 devices?
-    self._devices = [device_util.resolve(d) for d in devices]
+    self._devices = tuple(device_util.resolve(d) for d in devices)
     self._canonical_device_set = set(self._devices)
     self._device_index = values.PerReplica(
         {d: i for i, d in enumerate(devices)})
@@ -727,7 +727,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     return values.update_regroup(self, updates, group)
 
   def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
-    assert isinstance(colocate_with, list)
+    assert isinstance(colocate_with, tuple)
     # TODO(josh11b): In eager mode, use one thread per device.
     updates = {}
     for d in colocate_with:
@@ -748,9 +748,9 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     if isinstance(val, values.DistributedValues):
       # Return in a deterministic order.
       if set(val.devices) == self._canonical_device_set:
-        return [val.get(device=d) for d in self._devices]
-      return [val.get(device=d) for d in sorted(val.devices)]
-    return [val]
+        return tuple(val.get(device=d) for d in self._devices)
+      return tuple(val.get(device=d) for d in sorted(val.devices))
+    return (val,)
 
   def value_container(self, val):
     return values.value_container(val)
@@ -761,12 +761,11 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
 
   @property
   def worker_devices(self):
-    # Make a copy to prevent users from accidentally mutating our copy.
-    return list(self._devices)
+    return self._devices
 
   @property
   def parameter_devices(self):
-    return list(self._devices)
+    return self._devices
 
   @property
   def experimental_between_graph(self):
@@ -786,7 +785,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
 
   def non_slot_devices(self, var_list):
     del var_list
-    return list(self._devices)
+    return tuple(self._devices)
 
   def _get_devices_from(self, colocate_with=None):
     if colocate_with is None:
-- 
GitLab


From d6074a1afb8dc7dff25cea759f5249898cf1affe Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 6 Dec 2018 16:01:40 -0800
Subject: [PATCH 1462/1554] Fix a flaky test.

Move from absolute time measurement to relative.

PiperOrigin-RevId: 224424896
---
 tensorflow/lite/profiling/profiler_test.cc | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/tensorflow/lite/profiling/profiler_test.cc b/tensorflow/lite/profiling/profiler_test.cc
index 82d053729c..addebabe1b 100644
--- a/tensorflow/lite/profiling/profiler_test.cc
+++ b/tensorflow/lite/profiling/profiler_test.cc
@@ -27,11 +27,8 @@ namespace tflite {
 namespace profiling {
 namespace {
 
-void AssertDurationOfEventAroundMs(const ProfileEvent* event,
-                                   double expected_ms, double eps_ms) {
-  double duration_ms =
-      (event->end_timestamp_us - event->begin_timestamp_us) / 1e3;
-  EXPECT_NEAR(expected_ms, duration_ms, eps_ms);
+double GetDurationOfEventMs(const ProfileEvent* event) {
+  return (event->end_timestamp_us - event->begin_timestamp_us) / 1e3;
 }
 
 void SleepForQuarterSecond(Profiler* profiler) {
@@ -84,12 +81,17 @@ TEST(ProfilingTest, ProfilesAreCollected) {
 
 #ifndef ADDRESS_SANITIZER
   // ASAN build is sometimes very slow. Set a large epsilon to avoid flakiness.
+  // Due to flakiness, just verify relative values match.
   const int eps_ms = 50;
-  AssertDurationOfEventAroundMs(profile_events[0], /*expected_ms*/ 500, eps_ms);
-  AssertDurationOfEventAroundMs(profile_events[1], /*expected_ms*/ 250, eps_ms);
-  AssertDurationOfEventAroundMs(profile_events[2], /*expected_ms*/ 250, eps_ms);
-  AssertDurationOfEventAroundMs(profile_events[3], /*expected_ms*/ 250, eps_ms);
-  AssertDurationOfEventAroundMs(profile_events[4], /*expected_ms*/ 250, eps_ms);
+  auto parent_ms = GetDurationOfEventMs(profile_events[0]);
+  double child_ms[2], sleep_for_quarter_ms[2];
+  child_ms[0] = GetDurationOfEventMs(profile_events[1]);
+  child_ms[1] = GetDurationOfEventMs(profile_events[3]);
+  sleep_for_quarter_ms[0] = GetDurationOfEventMs(profile_events[2]);
+  sleep_for_quarter_ms[1] = GetDurationOfEventMs(profile_events[4]);
+  EXPECT_NEAR(parent_ms, child_ms[0] + child_ms[1], eps_ms);
+  EXPECT_NEAR(child_ms[0], sleep_for_quarter_ms[0], eps_ms);
+  EXPECT_NEAR(child_ms[1], sleep_for_quarter_ms[1], eps_ms);
 #endif
 }
 
-- 
GitLab


From a5be8bd6b31f49c1ebe7899b1bf9123664785415 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 6 Dec 2018 16:06:18 -0800
Subject: [PATCH 1463/1554] Populate for int8 type.

PiperOrigin-RevId: 224425691
---
 tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
index ad025b19d9..32cf4e4292 100644
--- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -281,6 +281,11 @@ void BenchmarkTfLiteModel::PrepareInputsAndOutputs() {
           interpreter->typed_tensor<uint8_t>(i),
           std::vector<int>(sizes.begin() + 1, sizes.end()),
           []() { return static_cast<uint8_t>(rand()) % 255; });
+    } else if (t->type == kTfLiteInt8) {
+      FillRandomValue<int8_t>(
+          interpreter->typed_tensor<int8_t>(i),
+          std::vector<int>(sizes.begin() + 1, sizes.end()),
+          []() { return static_cast<int8_t>(rand()) % 255 - 127; });
     } else if (t->type == kTfLiteString) {
       tflite::DynamicBuffer buffer;
       FillRandomString(&buffer, sizes, []() {
-- 
GitLab


From d3cf56c7280629bccfd4a1015d60dd4f2fda3ac7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 16:15:20 -0800
Subject: [PATCH 1464/1554] In gpu_process_state.cc enforce that BusIdForGPU
 always returns a non-negative integer, because the value will be used as an
 index.  The default implementation uses NUMA node, and some platforms will
 return -1 (kUnknownNumaNode).

PiperOrigin-RevId: 224427213
---
 tensorflow/core/common_runtime/gpu/gpu_process_state.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
index a9a19f0fe0..8167cfb9d7 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_process_state.cc
@@ -70,7 +70,10 @@ int GPUProcessState::BusIdForGPU(TfGpuId tf_gpu_id) {
   // Return the NUMA node associated with the GPU's StreamExecutor.
   se::StreamExecutor* se =
       GpuIdUtil::ExecutorForTfGpuId(tf_gpu_id).ValueOrDie();
-  return se->GetDeviceDescription().numa_node();
+  int numa_node = se->GetDeviceDescription().numa_node();
+  // bus_id must be non-negative.  If the numa_node is not known,
+  // use 0.
+  return numa_node >= 0 ? numa_node : 0;
 }
 
 Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
@@ -97,6 +100,7 @@ Allocator* GPUProcessState::GetGPUAllocator(const GPUOptions& options,
     PlatformGpuId platform_gpu_id;
     TF_CHECK_OK(GpuIdManager::TfToPlatformGpuId(tf_gpu_id, &platform_gpu_id));
     int bus_id = BusIdForGPU(tf_gpu_id);
+    DCHECK_GE(bus_id, 0);
     while (bus_id >= gpu_visitors_.size()) {
       gpu_visitors_.push_back({});
     }
@@ -249,6 +253,7 @@ void GPUProcessState::AddGPUAllocVisitor(int bus_id,
   CHECK(gpu_allocators_.empty())  // Crash OK
       << "AddGPUAllocVisitor must be called before "
          "first call to GetGPUAllocator.";
+  DCHECK_GE(bus_id, 0);
   while (bus_id >= static_cast<int64>(gpu_visitors_.size())) {
     gpu_visitors_.push_back(std::vector<SubAllocator::Visitor>());
   }
-- 
GitLab


From f8006837d3e2d1ee6bc03d48a51f17284cd7034d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 6 Dec 2018 16:23:47 -0800
Subject: [PATCH 1465/1554] scale loss instead of scale gradients.

PiperOrigin-RevId: 224428618
---
 .../python/keras/optimizer_v2/optimizer_v2.py | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index e6cd52c817..15f3009a4a 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -24,7 +24,8 @@ import abc
 
 import six
 
-from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -252,12 +253,14 @@ class OptimizerV2(optimizer_v1.Optimizer):
       with backprop.GradientTape() as tape:
         tape.watch(var_list)
         loss_value = loss()
+        loss_value = self._scale_loss(loss_value)
       grads = tape.gradient(loss_value, var_list, grad_loss)
     else:
       if context.executing_eagerly():
         raise RuntimeError("`loss` passed to Optimizer.compute_gradients "
                            "should be a function when eager execution is "
                            "enabled.")
+      loss = self._scale_loss(loss)
       self._assert_valid_dtypes([loss])
       if grad_loss is not None:
         self._assert_valid_dtypes([grad_loss])
@@ -277,6 +280,15 @@ class OptimizerV2(optimizer_v1.Optimizer):
 
     return grads_and_vars
 
+  @staticmethod
+  def _scale_loss(loss_value):
+    if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
+      num_replicas = \
+        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      if num_replicas > 1:
+        loss_value *= (1. / num_replicas)
+    return loss_value
+
   def apply_gradients(self, grads_and_vars, name=None):
     """Apply gradients to variables.
 
@@ -299,7 +311,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
     """
     grads_and_vars = _filter_grads(grads_and_vars)
     var_list = [v for (_, v) in grads_and_vars]
-    if distribution_strategy_context.has_distribution_strategy():
+    if distribute_ctx.has_distribution_strategy():
       reduced_grads = merge_grads(grads_and_vars)
       grads_and_vars = zip(reduced_grads, var_list)
 
@@ -598,7 +610,7 @@ def merge_update_step(update_ops, local_step):
       incre_op = local_step.assign_add(1).op
     return incre_op
 
-  return distribution_strategy_context.get_replica_context().merge_call(
+  return distribute_ctx.get_replica_context().merge_call(
       merge_update_step_fn, args=(update_ops, local_step))
 
 
@@ -606,11 +618,11 @@ def merge_grads(grads_and_vars):
   """Merge gradients from different replicas."""
 
   def merge_grad_fn(strategy, grads_and_vars):
-    reduced_grads = strategy.batch_reduce(
-        ds_reduce_util.ReduceOp.MEAN, grads_and_vars)
+    reduced_grads = strategy.batch_reduce(ds_reduce_util.ReduceOp.SUM,
+                                          grads_and_vars)
     return reduced_grads
 
-  return distribution_strategy_context.get_replica_context().merge_call(
+  return distribute_ctx.get_replica_context().merge_call(
       merge_grad_fn, args=(grads_and_vars,))
 
 
@@ -629,7 +641,7 @@ def _var_key(var):
   """
 
   # pylint: disable=protected-access
-  if distribution_strategy_context.has_distribution_strategy() and hasattr(
+  if distribute_ctx.has_distribution_strategy() and hasattr(
       var, "_primary_var"):
     var = var._primary_var
   if hasattr(var, "op"):
-- 
GitLab


From b8f5de61851acba53ca9ecf79b60bb34d753f34b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 16:26:49 -0800
Subject: [PATCH 1466/1554] Propagate quantization up selects with constant
 unquantize input. This helps for applying padding values or other masking
 operations

PiperOrigin-RevId: 224429126
---
 .../graph_transformations/hardcode_min_max.cc | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc b/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
index df50f31de8..2e41767095 100644
--- a/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
+++ b/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
@@ -208,12 +208,32 @@ bool HardcodeMinMaxForSelect(Model* model, Operator* op) {
   if (output_array.minmax) {
     return false;
   }
-  const auto& input_array_1 = model->GetArray(op->inputs[1]);
-  if (!input_array_1.minmax) {
+
+  auto& input_array_1 = model->GetArray(op->inputs[1]);
+  auto& input_array_2 = model->GetArray(op->inputs[2]);
+
+  if (!input_array_1.minmax && !input_array_2.minmax) {
     return false;
   }
-  const auto& input_array_2 = model->GetArray(op->inputs[2]);
-  if (!input_array_2.minmax) {
+
+  // Propagate up if one input is quantized and the other is constant.
+  if (!input_array_1.minmax &&
+      IsConstantParameterArray(*model, op->inputs[1])) {
+    auto& minmax_1 = input_array_1.GetOrCreateMinMax();
+    const auto& minmax_2 = input_array_2.GetMinMax();
+    minmax_1.min = minmax_2.min;
+    minmax_1.max = minmax_2.max;
+  }
+
+  if (!input_array_2.minmax &&
+      IsConstantParameterArray(*model, op->inputs[2])) {
+    auto& minmax_2 = input_array_2.GetOrCreateMinMax();
+    const auto& minmax_1 = input_array_1.GetMinMax();
+    minmax_2.min = minmax_1.min;
+    minmax_2.max = minmax_1.max;
+  }
+
+  if (!input_array_1.minmax || !input_array_2.minmax) {
     return false;
   }
 
-- 
GitLab


From 3f5dae940e67ce9a1c7a87f4c0d2c7ed051215be Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Thu, 6 Dec 2018 16:39:22 -0800
Subject: [PATCH 1467/1554] Clarify interface docs for DeleteRecursively in Env
 and FileSystem

PiperOrigin-RevId: 224431070
---
 tensorflow/core/platform/env.h         | 21 +++++++++++++++++----
 tensorflow/core/platform/file_system.h | 19 ++++++++++++++++---
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h
index 7374fccdc2..1b53828415 100644
--- a/tensorflow/core/platform/env.h
+++ b/tensorflow/core/platform/env.h
@@ -167,11 +167,24 @@ class Env {
   Status DeleteFile(const string& fname);
 
   /// \brief Deletes the specified directory and all subdirectories and files
-  /// underneath it. undeleted_files and undeleted_dirs stores the number of
-  /// files and directories that weren't deleted (unspecified if the return
-  /// status is not OK).
+  /// underneath it. This is accomplished by traversing the directory tree
+  /// rooted at dirname and deleting entries as they are encountered.
+  ///
+  /// If dirname itself is not readable or does not exist, *undeleted_dir_count
+  /// is set to 1, *undeleted_file_count is set to 0 and an appropriate status
+  /// (e.g. NOT_FOUND) is returned.
+  ///
+  /// If dirname and all its descendants were successfully deleted, TF_OK is
+  /// returned and both error counters are set to zero.
+  ///
+  /// Otherwise, while traversing the tree, undeleted_file_count and
+  /// undeleted_dir_count are updated if an entry of the corresponding type
+  /// could not be deleted. The returned error status represents the reason that
+  /// any one of these entries could not be deleted.
+  ///
   /// REQUIRES: undeleted_files, undeleted_dirs to be not null.
-  /// Typical return codes
+  ///
+  /// Typical return codes:
   ///  * OK - dirname exists and we were able to delete everything underneath.
   ///  * NOT_FOUND - dirname doesn't exist
   ///  * PERMISSION_DENIED - dirname or some descendant is not writable
diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h
index 156af6cdea..c84a93b1bf 100644
--- a/tensorflow/core/platform/file_system.h
+++ b/tensorflow/core/platform/file_system.h
@@ -167,10 +167,23 @@ class FileSystem {
   virtual Status DeleteDir(const string& dirname) = 0;
 
   /// \brief Deletes the specified directory and all subdirectories and files
-  /// underneath it. undeleted_files and undeleted_dirs stores the number of
-  /// files and directories that weren't deleted (unspecified if the return
-  /// status is not OK).
+  /// underneath it. This is accomplished by traversing the directory tree
+  /// rooted at dirname and deleting entries as they are encountered.
+  ///
+  /// If dirname itself is not readable or does not exist, *undeleted_dir_count
+  /// is set to 1, *undeleted_file_count is set to 0 and an appropriate status
+  /// (e.g. NOT_FOUND) is returned.
+  ///
+  /// If dirname and all its descendants were successfully deleted, TF_OK is
+  /// returned and both error counters are set to zero.
+  ///
+  /// Otherwise, while traversing the tree, undeleted_file_count and
+  /// undeleted_dir_count are updated if an entry of the corresponding type
+  /// could not be deleted. The returned error status represents the reason that
+  /// any one of these entries could not be deleted.
+  ///
   /// REQUIRES: undeleted_files, undeleted_dirs to be not null.
+  ///
   /// Typical return codes:
   ///  * OK - dirname exists and we were able to delete everything underneath.
   ///  * NOT_FOUND - dirname doesn't exist
-- 
GitLab


From 5eaca08c7efdad8cf795b00212d4e3653e0d7466 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 16:42:03 -0800
Subject: [PATCH 1468/1554] Remove broken test that computed gradients of fused
 batch norm in inference mode.

PiperOrigin-RevId: 224431514
---
 tensorflow/python/ops/parallel_for/control_flow_ops_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index cc20d7ca6a..933bddd8cc 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -927,7 +927,10 @@ class NNTest(PForTest):
               outputs[1] = constant_op.constant(0.)
               outputs[2] = constant_op.constant(0.)
             loss = nn.l2_loss(outputs[0])
-          gradients = g.gradient(loss, [x1, scale, offset])
+          if is_training:
+            gradients = g.gradient(loss, [x1, scale, offset])
+          else:
+            gradients = [constant_op.constant(0.)] * 3
           return outputs + gradients
 
         # pylint: enable=cell-var-from-loop
-- 
GitLab


From ceee617b3675037da8637997f673ab9a993ee5de Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Thu, 6 Dec 2018 16:44:31 -0800
Subject: [PATCH 1469/1554] Add missing 'void' parameter list for C forward
 declarations.

PiperOrigin-RevId: 224431819
---
 tensorflow/c/c_api.h              | 15 ++++++++-------
 tensorflow/c/c_api_experimental.h |  2 +-
 tensorflow/c/eager/c_api.h        |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 3d56268110..c7abba8552 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -91,7 +91,7 @@ extern "C" {
 // --------------------------------------------------------------------------
 // TF_Version returns a string describing version information of the
 // TensorFlow library. TensorFlow using semantic versioning.
-TF_CAPI_EXPORT extern const char* TF_Version();
+TF_CAPI_EXPORT extern const char* TF_Version(void);
 
 // --------------------------------------------------------------------------
 // TF_DataType holds the type for a scalar value.  E.g., one slot in a tensor.
@@ -157,7 +157,7 @@ typedef enum TF_Code {
 typedef struct TF_Status TF_Status;
 
 // Return a new status object.
-TF_CAPI_EXPORT extern TF_Status* TF_NewStatus();
+TF_CAPI_EXPORT extern TF_Status* TF_NewStatus(void);
 
 // Delete a previously created status object.
 TF_CAPI_EXPORT extern void TF_DeleteStatus(TF_Status*);
@@ -196,7 +196,7 @@ TF_CAPI_EXPORT extern TF_Buffer* TF_NewBufferFromString(const void* proto,
                                                         size_t proto_len);
 
 // Useful for passing *out* a protobuf.
-TF_CAPI_EXPORT extern TF_Buffer* TF_NewBuffer();
+TF_CAPI_EXPORT extern TF_Buffer* TF_NewBuffer(void);
 
 TF_CAPI_EXPORT extern void TF_DeleteBuffer(TF_Buffer*);
 
@@ -305,7 +305,7 @@ TF_CAPI_EXPORT extern size_t TF_StringEncodedSize(size_t len);
 typedef struct TF_SessionOptions TF_SessionOptions;
 
 // Return a new options object.
-TF_CAPI_EXPORT extern TF_SessionOptions* TF_NewSessionOptions();
+TF_CAPI_EXPORT extern TF_SessionOptions* TF_NewSessionOptions(void);
 
 // Set the target in TF_SessionOptions.options.
 // target can be empty, a single entry, or a comma separated list of entries.
@@ -338,7 +338,7 @@ TF_CAPI_EXPORT extern void TF_DeleteSessionOptions(TF_SessionOptions*);
 typedef struct TF_Graph TF_Graph;
 
 // Return a new graph object.
-TF_CAPI_EXPORT extern TF_Graph* TF_NewGraph();
+TF_CAPI_EXPORT extern TF_Graph* TF_NewGraph(void);
 
 // Destroy an options object.  Graph will be deleted once no more
 // TFSession's are referencing it.
@@ -890,7 +890,8 @@ TF_CAPI_EXPORT extern void TF_GraphVersions(TF_Graph* graph,
 // TF_GraphImportGraphDef.
 typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions;
 
-TF_CAPI_EXPORT extern TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions();
+TF_CAPI_EXPORT extern TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions(
+    void);
 TF_CAPI_EXPORT extern void TF_DeleteImportGraphDefOptions(
     TF_ImportGraphDefOptions* opts);
 
@@ -1611,7 +1612,7 @@ TF_CAPI_EXPORT extern void TF_DeleteLibraryHandle(TF_Library* lib_handle);
 //
 // The data in the buffer will be the serialized OpList proto for ops registered
 // in this address space.
-TF_CAPI_EXPORT extern TF_Buffer* TF_GetAllOpList();
+TF_CAPI_EXPORT extern TF_Buffer* TF_GetAllOpList(void);
 
 // TF_ApiDefMap encapsulates a collection of API definitions for an operation.
 //
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 80c8bfe594..3e3a485eb7 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -239,7 +239,7 @@ TF_CAPI_EXPORT void TF_InitMain(const char* usage, int* argc, char*** argv);
 
 // Platform-specific implementation to return an unused port. (This should used
 // in tests only.)
-TF_CAPI_EXPORT int TF_PickUnusedPortOrDie();
+TF_CAPI_EXPORT int TF_PickUnusedPortOrDie(void);
 
 // Fast path method that makes constructing a single scalar tensor require less
 // overhead and copies.
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index 8d6c8d958d..f80ae5a6d0 100755
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -48,7 +48,7 @@ extern "C" {
 typedef struct TFE_ContextOptions TFE_ContextOptions;
 
 // Return a new options object.
-TF_CAPI_EXPORT extern TFE_ContextOptions* TFE_NewContextOptions();
+TF_CAPI_EXPORT extern TFE_ContextOptions* TFE_NewContextOptions(void);
 
 // Set the config in TF_ContextOptions.options.
 // config should be a serialized tensorflow.ConfigProto proto.
-- 
GitLab


From d19733b470d6c5737b95fa193e4129255e3e1d83 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 16:49:28 -0800
Subject: [PATCH 1470/1554] Makes the tensorflow should_use decorator not
 produce any warnings when running inside functions, to prevent extraneous
 warnings.

PiperOrigin-RevId: 224432616
---
 tensorflow/python/BUILD                        |  2 ++
 tensorflow/python/ops/control_flow_ops_test.py | 13 +++++++++++++
 tensorflow/python/util/tf_should_use.py        |  6 +++++-
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 035830b911..bc6dc413e7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3251,6 +3251,7 @@ cuda_py_test(
         ":util",
         ":variable_scope",
         ":variables",
+        "//tensorflow/python/eager:def_function",
     ],
 )
 
@@ -3835,6 +3836,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":util",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python/eager:context",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index c020189ad6..b19ec4bd61 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import node_def_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -1014,6 +1015,18 @@ class AssertTest(test_util.TensorFlowTestCase):
     with self.assertRaises(errors.InvalidArgumentError):
       self.evaluate(c)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testAssertInFunction(self):
+
+    @def_function.function
+    def whiny(value):
+      control_flow_ops.Assert(value, ["Raised false"])
+      return constant_op.constant(5)
+
+    with self.assertRaises(errors.InvalidArgumentError):
+      self.evaluate(whiny(False))
+
+    self.assertAllEqual(whiny(True), 5)
 
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/python/util/tf_should_use.py b/tensorflow/python/util/tf_should_use.py
index ca6710bcf2..63de4a7a96 100644
--- a/tensorflow/python/util/tf_should_use.py
+++ b/tensorflow/python/util/tf_should_use.py
@@ -23,6 +23,7 @@ import traceback
 
 import six  # pylint: disable=unused-import
 
+from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.util import tf_decorator
 # pylint: enable=g-bad-import-order,g-import-not-at-top
@@ -32,7 +33,8 @@ class _TFShouldUseHelper(object):
   """Object stored in TFShouldUse-wrapped objects.
 
   When it is deleted it will emit a warning or error if its `sate` method
-  has not been called by time of deletion.
+  has not been called by time of deletion, and Tensorflow is not executing
+  eagerly outside of functions.
   """
 
   def __init__(self, type_, repr_, stack_frame, fatal_error_if_unsated):
@@ -50,6 +52,8 @@ class _TFShouldUseHelper(object):
     self._logging_module = None
 
   def __del__(self):
+    if ops.executing_eagerly_outside_functions():
+      return
     if self._sated:
       return
     if self._fatal_error_if_unsated:
-- 
GitLab


From 7e0ae39bad55bbd88b2e8f3d436b8608d49ede94 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 6 Dec 2018 16:51:07 -0800
Subject: [PATCH 1471/1554] Add TPU core detection to TPUClusterResolver's
 num_accelerators_per_worker

PiperOrigin-RevId: 224432865
---
 .../tpu/python/tpu/tpu_system_metadata.py     |   1 +
 .../cluster_resolver/tpu_cluster_resolver.py  |  80 ++++++-
 .../tpu_cluster_resolver_test.py              | 218 +++++++++++++-----
 3 files changed, 231 insertions(+), 68 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
index ec682e5829..d66ecfcf4a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
@@ -52,6 +52,7 @@ def _query_tpu_system_metadata(master_address, cluster_def=None,
   devices = []
   device_dict = collections.defaultdict(list)
 
+  # TODO(b/120564445): Replace with standard library for retries.
   retry_count = 1
   while True:
     logging.info('Querying Tensorflow master (%s) for TPU system metadata.',
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 99a7a0922f..e907d6fde4 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -18,13 +18,19 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import os
+import re
 
 from six.moves.urllib.request import Request
 from six.moves.urllib.request import urlopen
 
+from tensorflow.python.client import session
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import ClusterResolver
 from tensorflow.python.distribute.cluster_resolver.cluster_resolver import format_master_url
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import server_lib
 from tensorflow.python.util import compat
 
@@ -41,6 +47,45 @@ _ENDPOINTS_SEPARATOR = ','
 _DEFAULT_ENV_VARIABLE = 'TPU_NAME'
 _DISCOVERY_SERVICE_URL_ENV_VARIABLE = 'TPU_API_DISCOVERY_URL'
 
+_TPU_DEVICE_REGEX = re.compile(
+    r'.*task:(?P<host_id>\d+)/.*device:TPU:(?P<core_id>\d+)$')
+_TPU_CONN_RETRIES = 120
+
+DeviceDetails = collections.namedtuple(
+    'DeviceDetails', ['device_map', 'total_cores'])
+
+
+def _get_device_dict_and_cores(devices):
+  """Returns a dict of hosts to cores and total cores given devices names.
+
+  Returns a namedtuple with two attributes:
+    device_map: A map of host_ids to a list of core_ids.
+    total_cores: The total number of cores within the TPU system.
+
+  Args:
+    devices: A list of devices returned by session.list_devices()
+  """
+  device_map = collections.defaultdict(list)
+  num_cores = 0
+  for device in devices:
+    match = _TPU_DEVICE_REGEX.match(device.name)
+    if match:
+      host_id = match.group('host_id')
+      core_id = match.group('core_id')
+      device_map[host_id].append(core_id)
+      num_cores += 1
+  return DeviceDetails(device_map, num_cores)
+
+
+def _verify_and_return_same_core_count(device_dict):
+  """Verifies that every device in device_dict has the same number of cores."""
+  num_cores_per_host_set = (
+      {len(core_ids) for core_ids in device_dict.values()})
+  if len(num_cores_per_host_set) != 1:
+    raise RuntimeError('TPU cores on each device is not the same. This '
+                       'should never happen. Devices: {}'.format(device_dict))
+  return num_cores_per_host_set.pop()
+
 
 class TPUClusterResolver(ClusterResolver):
   """Cluster Resolver for Google Cloud TPUs.
@@ -394,24 +439,43 @@ class TPUClusterResolver(ClusterResolver):
                        config_proto=None):
     """Returns the number of TPU cores per worker.
 
-    This defaults to 8 for all current TPU configurations, and we do not need
-    to query any remote systems for this.
+    Connects to the master and list all the devices present in the master,
+    and counts them up. Also verifies that the device counts per host in the
+    cluster is the same before returning the number of TPU cores per host.
 
     Args:
       task_type: Unused.
       task_index: Unused.
       accelerator_type: Unused.
-      config_proto: Unused.
+      config_proto: Used to create a connection to a TPU master in order to
+        retrieve the system metadata.
 
     Raises:
       RuntimeError: If this is used with a non-TPU accelerator_type.
     """
-    # Unused. Not necessary to query anything.
-    del task_type, task_index, config_proto
+    retry_count = 1
+    # TODO(b/120564445): Replace with standard library for retries.
+    while True:
+      try:
+        with ops.Graph().as_default():
+          with session.Session(self.master(), config=config_proto) as s:
+            devices = s.list_devices()
+            device_details = _get_device_dict_and_cores(devices)
+            break
+      except errors.DeadlineExceededError:
+        error_message = ('Failed to connect to master. The TPU might not be '
+                         'ready (e.g. still scheduling) or the master '
+                         'address is incorrect: got (%s)' % self.master())
+        if retry_count <= _TPU_CONN_RETRIES:
+          logging.warning(error_message)
+          logging.warning('Retrying (%d/%d)...', retry_count, _TPU_CONN_RETRIES)
+          retry_count += 1
+        else:
+          raise RuntimeError(error_message)
 
-    if accelerator_type != 'TPU':
-      raise ValueError('This Cluster Resolver is only compatible with TPUs.')
-    return 8
+    if device_details.total_cores:
+      return _verify_and_return_same_core_count(device_details.device_map)
+    return 0
 
   @property
   def environment(self):
diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
index 0f22ede3d9..27d92608fa 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver_test.py
@@ -20,7 +20,10 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver
+from tensorflow.python.client import session
+from tensorflow.python.distribute import cluster_resolver
+from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver
+from tensorflow.python.framework import errors
 from tensorflow.python.platform import test
 from tensorflow.python.training import server_lib
 from tensorflow.python.util import compat
@@ -101,7 +104,8 @@ class TPUClusterResolverTest(test.TestCase):
 
     return mock_client
 
-  @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
+  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+                     '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testRetrieveProjectAndZoneFromMetadata(self):
     tpu_map = {
@@ -112,7 +116,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project=None,
         zone=None,
         tpu=['test-tpu-1'],
@@ -120,7 +124,7 @@ class TPUClusterResolverTest(test.TestCase):
         service=self.mock_service_client(tpu_map=tpu_map),
         coordinator_name='coordinator')
 
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job {
       name: 'coordinator'
@@ -130,11 +134,12 @@ class TPUClusterResolverTest(test.TestCase):
       name: 'worker'
       tasks { key: 0 value: '10.1.2.3:8470' }
     }
-    """ % tpu_cluster_resolver._coordinator_port
+    """ % resolver._coordinator_port
     self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto))
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
+    self.assertEqual(resolver.master(), 'grpc://10.1.2.3:8470')
 
-  @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
+  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+                     '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testRetrieveProjectAndZoneFromMetadataNoCoordinator(self):
     tpu_map = {
@@ -145,7 +150,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project=None,
         zone=None,
         tpu=['test-tpu-1'],
@@ -153,14 +158,15 @@ class TPUClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
 
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
+    self.assertEqual(resolver.master(), 'grpc://10.1.2.3:8470')
 
-  @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
+  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+                     '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testUnhealthyCloudTpu(self):
     tpu_map = {
@@ -171,7 +177,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project=None,
         zone=None,
         tpu='test-tpu-1',
@@ -180,9 +186,10 @@ class TPUClusterResolverTest(test.TestCase):
         service=self.mock_service_client(tpu_map=tpu_map))
 
     with self.assertRaises(RuntimeError):
-      tpu_cluster_resolver.cluster_spec()
+      resolver.cluster_spec()
 
-  @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
+  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+                     '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testNotReadyCloudTpu(self):
     tpu_map = {
@@ -193,7 +200,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project=None,
         zone=None,
         tpu='test-tpu-1',
@@ -202,7 +209,7 @@ class TPUClusterResolverTest(test.TestCase):
         service=self.mock_service_client(tpu_map=tpu_map))
 
     with self.assertRaises(RuntimeError):
-      tpu_cluster_resolver.cluster_spec()
+      resolver.cluster_spec()
 
   def testSimpleSuccessfulRetrieval(self):
     tpu_map = {
@@ -213,7 +220,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu=['test-tpu-1'],
@@ -222,13 +229,13 @@ class TPUClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
 
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job { name: 'coordinator' tasks { key: 0 value: '10.128.1.5:10203' } }
     job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
+    self.assertEqual(resolver.master(), 'grpc://10.1.2.3:8470')
 
   def testNewNetworkEndpointFormat(self):
     tpu_map = {
@@ -241,7 +248,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
@@ -250,15 +257,16 @@ class TPUClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
 
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job { name: 'coordinator' tasks { key: 0 value: '10.128.1.5:10203' } }
     job { name: 'worker' tasks { key: 0 value: '10.2.3.4:8470' } }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
-    self.assertEqual('grpc://10.2.3.4:8470', tpu_cluster_resolver.master())
+    self.assertEqual('grpc://10.2.3.4:8470', resolver.master())
 
-  @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata',
+  @mock.patch.object(cluster_resolver.TPUClusterResolver,
+                     '_requestComputeMetadata',
                      mock_request_compute_metadata)
   def testPodResolution(self):
     tpu_map = {
@@ -286,13 +294,13 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         tpu='test-tpu-1',
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map),
         coordinator_name='coordinator')
 
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job {
       name: 'coordinator',
@@ -305,9 +313,9 @@ class TPUClusterResolverTest(test.TestCase):
       tasks { key: 2 value: '10.2.3.6:8470' }
       tasks { key: 3 value: '10.2.3.7:8470' }
     }
-    """ % tpu_cluster_resolver._coordinator_port
+    """ % resolver._coordinator_port
     self._verifyClusterSpecEquality(actual_cluster_spec, str(expected_proto))
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470')
+    self.assertEqual(resolver.master(), 'grpc://10.2.3.4:8470')
 
   def testPodResolutionNoCoordinator(self):
     tpu_map = {
@@ -335,7 +343,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
@@ -343,7 +351,7 @@ class TPUClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
 
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job {
       name: 'worker'
@@ -354,13 +362,13 @@ class TPUClusterResolverTest(test.TestCase):
     }
     """
     self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto)
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470')
+    self.assertEqual(resolver.master(), 'grpc://10.2.3.4:8470')
 
   def testGetMasterNoEntries(self):
     tpu_map = {}
 
     with self.assertRaises(ValueError):
-      TPUClusterResolver(
+      cluster_resolver.TPUClusterResolver(
           project='test-project',
           zone='us-central1-c',
           tpu=[],
@@ -370,14 +378,14 @@ class TPUClusterResolverTest(test.TestCase):
 
   # TODO(saeta): Convert to parameterized test when included in OSS TF.
   def verifyShouldResolve(self, tpu, should_resolve):
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu=tpu,
         coordinator_name=None,
         credentials=None,
         service=self.mock_service_client(tpu_map={}))
-    self.assertEqual(should_resolve, tpu_cluster_resolver._shouldResolve(),
+    self.assertEqual(should_resolve, resolver._shouldResolve(),
                      "TPU: '%s'" % tpu)
 
   def testShouldResolveNoName(self):
@@ -402,25 +410,26 @@ class TPUClusterResolverTest(test.TestCase):
     self.verifyShouldResolve('grpctpu', True)
 
   def testNoCallComputeMetadata(self):
-    tpu_cluster_resolver = TPUClusterResolver(tpu='/bns/foo/bar')
+    resolver = cluster_resolver.TPUClusterResolver(
+        tpu='/bns/foo/bar')
     self.assertEqual(
-        compat.as_bytes('/bns/foo/bar'), tpu_cluster_resolver.master())
-    self.assertEqual(None, tpu_cluster_resolver.cluster_spec())
+        compat.as_bytes('/bns/foo/bar'), resolver.master())
+    self.assertEqual(None, resolver.cluster_spec())
 
   def testGkeEnvironmentForDonut(self):
     os.environ['KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS'] = 'grpc://10.120.27.5:8470'
 
     self.assertIn('KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS', os.environ)
-    self.assertTrue(TPUClusterResolver._inGke())
+    self.assertTrue(cluster_resolver.TPUClusterResolver._inGke())
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470'),
-        compat.as_bytes(TPUClusterResolver._gkeEndpoints()))
+        compat.as_bytes(cluster_resolver.TPUClusterResolver._gkeEndpoints()))
 
-    tpu_cluster_resolver = TPUClusterResolver()
+    resolver = cluster_resolver.TPUClusterResolver()
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470'),
-        compat.as_bytes(tpu_cluster_resolver.master()))
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+        compat.as_bytes(resolver.master()))
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job {
       name: 'worker'
@@ -438,19 +447,19 @@ class TPUClusterResolverTest(test.TestCase):
                                                      'grpc://10.120.27.8:8470')
 
     self.assertIn('KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS', os.environ)
-    self.assertTrue(TPUClusterResolver._inGke())
+    self.assertTrue(cluster_resolver.TPUClusterResolver._inGke())
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470,'
                         'grpc://10.120.27.6:8470,'
                         'grpc://10.120.27.7:8470,'
                         'grpc://10.120.27.8:8470'),
-        compat.as_bytes(TPUClusterResolver._gkeEndpoints()))
+        compat.as_bytes(cluster_resolver.TPUClusterResolver._gkeEndpoints()))
 
-    tpu_cluster_resolver = TPUClusterResolver()
+    resolver = cluster_resolver.TPUClusterResolver()
     self.assertEqual(
         compat.as_bytes('grpc://10.120.27.5:8470'),
-        compat.as_bytes(tpu_cluster_resolver.master()))
-    actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
+        compat.as_bytes(resolver.master()))
+    actual_cluster_spec = resolver.cluster_spec()
     expected_proto = """
     job {
       name: 'worker'
@@ -467,18 +476,21 @@ class TPUClusterResolverTest(test.TestCase):
   def testEnvironmentDiscoveryUrl(self):
     os.environ['TPU_API_DISCOVERY_URL'] = 'https://{api}.internal/{apiVersion}'
     self.assertEqual('https://{api}.internal/{apiVersion}',
-                     TPUClusterResolver._environmentDiscoveryUrl())
+                     (cluster_resolver.TPUClusterResolver.
+                      _environmentDiscoveryUrl()))
 
   def testEnvironmentAndRpcDetectionForGoogle(self):
-    tpu_cluster_resolver = TPUClusterResolver(tpu='/bns/ab/cd/ef')
-    self.assertEqual(tpu_cluster_resolver.environment, 'google')
-    self.assertEqual(tpu_cluster_resolver.rpc_layer, None)
+    resolver = cluster_resolver.TPUClusterResolver(
+        tpu='/bns/ab/cd/ef')
+    self.assertEqual(resolver.environment, 'google')
+    self.assertEqual(resolver.rpc_layer, None)
 
   def testEnvironmentAndRpcDetectionForGrpcString(self):
-    tpu_cluster_resolver = TPUClusterResolver(tpu='grpc://10.1.2.3:8470')
-    self.assertEqual(tpu_cluster_resolver.environment, '')
-    self.assertEqual(tpu_cluster_resolver.rpc_layer, 'grpc')
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.1.2.3:8470')
+    resolver = cluster_resolver.TPUClusterResolver(
+        tpu='grpc://10.1.2.3:8470')
+    self.assertEqual(resolver.environment, '')
+    self.assertEqual(resolver.rpc_layer, 'grpc')
+    self.assertEqual(resolver.master(), 'grpc://10.1.2.3:8470')
 
   def testOverrideTaskTypeAndIndexAndGetMaster(self):
     tpu_map = {
@@ -506,7 +518,7 @@ class TPUClusterResolverTest(test.TestCase):
         }
     }
 
-    tpu_cluster_resolver = TPUClusterResolver(
+    resolver = cluster_resolver.TPUClusterResolver(
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
@@ -514,17 +526,103 @@ class TPUClusterResolverTest(test.TestCase):
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
 
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.4:8470')
+    self.assertEqual(resolver.master(), 'grpc://10.2.3.4:8470')
 
-    tpu_cluster_resolver.task_type = 'worker'
-    tpu_cluster_resolver.task_index = 3
-    self.assertEqual(tpu_cluster_resolver.master(), 'grpc://10.2.3.7:8470')
+    resolver.task_type = 'worker'
+    resolver.task_index = 3
+    self.assertEqual(resolver.master(), 'grpc://10.2.3.7:8470')
 
     self.assertEqual(
-        tpu_cluster_resolver.master(
+        resolver.master(
             task_type='worker', task_index=2, rpc_layer='test'),
         'test://10.2.3.6:8470')
 
+  def testGetDeviceDictAndCoresWithTPUs(self):
+    device_names = [
+        '/job:tpu_worker/task:0/device:TPU:0',
+        '/job:tpu_worker/task:1/device:TPU:1',
+        '/job:tpu_worker/task:2/device:TPU:0',
+        '/job:tpu_worker/task:3/device:TPU:1',
+        '/job:tpu_worker/task:0/device:TPU:4',
+        '/job:tpu_worker/task:1/device:TPU:5',
+        '/job:tpu_worker/task:2/device:TPU:4',
+        '/job:tpu_worker/task:3/device:TPU:5',
+    ]
+    device_list = [
+        session._DeviceAttributes(
+            name, 'TPU', 1024, 0) for name in device_names
+    ]
+
+    device_details = tpu_cluster_resolver._get_device_dict_and_cores(
+        device_list)
+    self.assertEqual(device_details.total_cores, 8)
+    self.assertEqual(device_details.device_map,
+                     {'0': ['0', '4'],
+                      '1': ['1', '5'],
+                      '2': ['0', '4'],
+                      '3': ['1', '5']})
+
+  def testGetDeviceDictAndCoresWithCPUsAndGPUs(self):
+    device_names = [
+        '/job:tpu_worker/task:0/device:CPU:0',
+        '/job:tpu_worker/task:1/device:CPU:0',
+        '/job:tpu_worker/task:2/device:CPU:0',
+        '/job:tpu_worker/task:3/device:CPU:0',
+        '/job:tpu_worker/task:0/device:GPU:1',
+        '/job:tpu_worker/task:1/device:GPU:1',
+        '/job:tpu_worker/task:2/device:GPU:1',
+        '/job:tpu_worker/task:3/device:GPU:1',
+    ]
+    device_list = [
+        session._DeviceAttributes(
+            name, 'XLA', 1024, 0) for name in device_names
+    ]
+
+    device_dict, num_cores = tpu_cluster_resolver._get_device_dict_and_cores(
+        device_list)
+    self.assertEqual(num_cores, 0)
+    self.assertEqual(device_dict, {})
+
+  def testVerifySameCoreCount(self):
+    self.assertEqual(
+        tpu_cluster_resolver._verify_and_return_same_core_count(
+            {0: [0, 1, 2, 3, 4, 5, 6, 7]}), 8)
+    self.assertEqual(
+        tpu_cluster_resolver._verify_and_return_same_core_count(
+            {0: [0, 1], 1: [2, 3]}), 2)
+    with self.assertRaises(RuntimeError):
+      tpu_cluster_resolver._verify_and_return_same_core_count(
+          {0: [0], 1: [1, 2]})
+
+  @mock.patch.object(session.BaseSession, 'list_devices')
+  def testNumAcceleratorsSuccess(self, mock_list_devices):
+    device_names = [
+        '/job:tpu_worker/task:0/device:TPU:0',
+        '/job:tpu_worker/task:1/device:TPU:1',
+        '/job:tpu_worker/task:2/device:TPU:0',
+        '/job:tpu_worker/task:3/device:TPU:1',
+        '/job:tpu_worker/task:0/device:TPU:4',
+        '/job:tpu_worker/task:1/device:TPU:5',
+        '/job:tpu_worker/task:2/device:TPU:4',
+        '/job:tpu_worker/task:3/device:TPU:5',
+    ]
+    device_list = [
+        session._DeviceAttributes(
+            name, 'TPU', 1024, 0) for name in device_names
+    ]
+    mock_list_devices.return_value = device_list
+
+    resolver = cluster_resolver.TPUClusterResolver(tpu='')
+    self.assertEqual(resolver.num_accelerators(), 2)
+
+  @mock.patch.object(session.BaseSession, 'list_devices')
+  def testNumAcceleratorsRetryFailure(self, mock_list_devices):
+    resolver = cluster_resolver.TPUClusterResolver(tpu='')
+    mock_list_devices.side_effect = errors.DeadlineExceededError(
+        None, None, 'timeout')
+    with self.assertRaises(RuntimeError):
+      resolver.num_accelerators()
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 59d535a0df17eaf3033bbff73ef4e1e1988c454e Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 6 Dec 2018 17:03:34 -0800
Subject: [PATCH 1472/1554] create_ios_frameworks.sh : Add an option for GPU
 support

The GPU library is still working-in-progress.

PiperOrigin-RevId: 224434789
---
 .../lite/lib_package/create_ios_frameworks.sh | 47 +++++++++++++++++--
 1 file changed, 42 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/lib_package/create_ios_frameworks.sh b/tensorflow/lite/lib_package/create_ios_frameworks.sh
index 7901655b7c..abf40e7dec 100755
--- a/tensorflow/lite/lib_package/create_ios_frameworks.sh
+++ b/tensorflow/lite/lib_package/create_ios_frameworks.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash
 # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,10 +20,41 @@ set -e
 echo "Starting"
 TFLITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
 
+usage() {
+  echo "Usage: $(basename "$0") [-a]"
+  echo "-g build with GPU delegate"
+  exit 1
+}
+
+USE_GPU_DELEGATE="false"
+FRAMEWORK_NAME="tensorflow_lite"
+while getopts "g" opt_name; do
+  case "$opt_name" in
+    g)
+        USE_GPU_DELEGATE="true"
+        FRAMEWORK_NAME="tensorflow_lite_gpu"
+        ;;
+    *) usage;;
+  esac
+done
+shift $((OPTIND - 1))
+readonly USE_GPU_DELEGATE
+readonly FRAMEWORK_NAME
+
+if [ $USE_GPU_DELEGATE == "true" ] ; then
+  for filename in metal_delegate.h libmetal_delegate.a ; do
+    if [[ ! -f "${TFLITE_DIR}/delegates/gpu/${filename}" ]] ; then
+      echo "File ${TFLITE_DIR}/delegates/gpu/${filename} doesn't exist."
+      echo "It's requried for building TFLite Framework with GPU. Aborting."
+      exit 1
+    fi
+  done
+fi
+
 TMP_DIR=$(mktemp -d)
 echo "Package dir: " $TMP_DIR
 FW_DIR=$TMP_DIR/tensorflow_lite_ios_frameworks
-FW_DIR_TFLITE=$FW_DIR/tensorflow_lite.framework
+FW_DIR_TFLITE=$FW_DIR/$FRAMEWORK_NAME.framework
 FW_DIR_TFLITE_HDRS=$FW_DIR_TFLITE/Headers
 
 echo "Creating target Headers directories"
@@ -58,8 +89,14 @@ cp $TFLITE_DIR/../../bazel-genfiles/tensorflow/tools/lib_package/include/tensorf
    $FW_DIR_TFLITE
 
 echo "Copying static libraries"
+# Note: There must be a static library with the same name
+# as the framework name.
 cp $TFLITE_DIR/tools/make/gen/lib/libtensorflow-lite.a \
-   $FW_DIR_TFLITE/tensorflow_lite
+    $FW_DIR_TFLITE/$FRAMEWORK_NAME
+if [ $USE_GPU_DELEGATE == "true" ] ; then
+  cp "${TFLITE_DIR}/delegates/gpu/libmetal_delegate.a" \
+      $FW_DIR_TFLITE/libmetal_delegate.a
+fi
 
 # This is required, otherwise they interfere with the documentation of the
 # pod at cocoapods.org.
@@ -71,10 +108,10 @@ find . -type f -name readme\* -exec rm -f {} \;
 TARGET_GEN_LOCATION="$TFLITE_DIR/gen/ios_frameworks"
 echo "Moving results to target: " $TARGET_GEN_LOCATION
 cd $FW_DIR
-zip -q -r tensorflow_lite.framework.zip tensorflow_lite.framework -x .DS_Store
+zip -q -r $FRAMEWORK_NAME.framework.zip $FRAMEWORK_NAME.framework -x .DS_Store
 rm -rf $TARGET_GEN_LOCATION
 mkdir -p $TARGET_GEN_LOCATION
-cp -r tensorflow_lite.framework.zip $TARGET_GEN_LOCATION
+cp -r $FRAMEWORK_NAME.framework.zip $TARGET_GEN_LOCATION
 
 echo "Cleaning up"
 rm -rf $TMP_DIR
-- 
GitLab


From fe2799dfb0e92eab722d52dd8b54e8256a290753 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 6 Dec 2018 17:10:55 -0800
Subject: [PATCH 1473/1554] Internal change.

PiperOrigin-RevId: 224435855
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index a6871dd8f1..2820e466f3 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -109,7 +109,7 @@ if [[ "$TF_NIGHTLY" == 1 ]]; then
   if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
-    EXTRA_PIP_FLAGS="--project_name=${PROJECT_NAME} --nightly_flag"
+    EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME} --nightly_flag"
   fi
 fi
 
-- 
GitLab


From b397935bae2530e1cf5022ca6f85a0b898a795d6 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 6 Dec 2018 17:13:19 -0800
Subject: [PATCH 1474/1554] Moving state created within a DatasetKernel to a
 resource_mgr owned by the IteratorResource instead. This helps us do better
 state management for state that is owned by dataset objects and would allow
 us to separate dataset and iterator kernel creation

PiperOrigin-RevId: 224436180
---
 tensorflow/core/framework/dataset.h           |   7 +
 .../core/kernels/data/cache_dataset_ops.cc    |  49 +++--
 tensorflow/core/kernels/data/iterator_ops.cc  |  12 +-
 .../kernels/data/multi_device_iterator_ops.cc |   5 +
 .../core/kernels/data/shuffle_dataset_op.cc   | 176 ++++++++++++------
 5 files changed, 167 insertions(+), 82 deletions(-)

diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index 8dcb5f3076..7d3776a6ec 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -285,6 +285,7 @@ class IteratorContext {
           function_library(ctx->function_library()),
           lib(ctx->lib()),
           function_handle_cache(ctx->function_handle_cache()),
+          resource_mgr(ctx->resource_mgr()),
           model(ctx->model()),
           runner(*(ctx->runner())),
           runner_threadpool_size(ctx->runner_threadpool_size()),
@@ -324,6 +325,10 @@ class IteratorContext {
     // A FunctionHandleCache that owns all the function handles. Not owned.
     FunctionHandleCache* function_handle_cache = nullptr;
 
+    // A resource manager for storing dataset-related state, e.g. random
+    // seeds or cached tensors. Not owned.
+    ResourceMgr* resource_mgr = nullptr;
+
     // If non-null, identifies the object used for performance modeling.
     std::shared_ptr<model::Model> model = nullptr;
 
@@ -363,6 +368,8 @@ class IteratorContext {
     return params_.function_handle_cache;
   }
 
+  ResourceMgr* resource_mgr() { return params_.resource_mgr; }
+
   const std::shared_ptr<model::Model>& model() { return params_.model; }
 
   std::function<void(std::function<void()>)>* runner() {
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index ab85736890..f00b38e732 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/env.h"
@@ -564,9 +565,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
   class MemoryDataset : public DatasetBase {
    public:
     explicit MemoryDataset(OpKernelContext* ctx, const DatasetBase* input)
-        : DatasetBase(DatasetContext(ctx)),
-          input_(input),
-          cache_(new MemoryCache()) {
+        : DatasetBase(DatasetContext(ctx)), input_(input) {
       input->Ref();
     }
 
@@ -574,8 +573,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new MemoryIterator(
-          {this, strings::StrCat(prefix, "::MemoryCache")}, cache_));
+      return std::unique_ptr<IteratorBase>(
+          new MemoryIterator({this, strings::StrCat(prefix, "::MemoryCache")}));
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -611,10 +610,12 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
     // The expected use is that a single `MemoryWriterIterator` populates the
     // cache with dataset elements. Once all elements are cached, the cache can
     // be used by one or more `MemoryReaderIterator`s.
-    class MemoryCache {
+    class MemoryCache : public ResourceBase {
      public:
       MemoryCache() = default;
 
+      string DebugString() override { return "CacheDataset::MemoryCache"; }
+
       // Marks the cache as completed.
       void Complete() {
         mutex_lock l(mu_);
@@ -681,15 +682,25 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
     class MemoryIterator : public DatasetIterator<MemoryDataset> {
      public:
-      explicit MemoryIterator(const Params& params,
-                              const std::shared_ptr<MemoryCache>& cache)
-          : DatasetIterator<MemoryDataset>(params), cache_(cache) {
-        mode_ = cache->MaybeClaim() ? Mode::write : Mode::read;
-        InitializeIterator();
-      }
+      explicit MemoryIterator(const Params& params)
+          : DatasetIterator<MemoryDataset>(params) {}
+
+      ~MemoryIterator() override { cache_->Unref(); }
 
       Status Initialize(IteratorContext* ctx) override {
         mutex_lock l(mu_);
+        // Use the resource manager in the iterator context to get / create
+        // a cache.
+        ResourceMgr* mgr = ctx->resource_mgr();
+        const string name =
+            strings::StrCat(prefix(), "::", dataset()->name(), "::MemoryCache");
+        TF_RETURN_IF_ERROR(mgr->LookupOrCreate<MemoryCache>(
+            "tf_data", name, &cache_, [](MemoryCache** cache) {
+              *cache = new MemoryCache();
+              return Status::OK();
+            }));
+        mode_ = cache_->MaybeClaim() ? Mode::write : Mode::read;
+        InitializeIterator();
         if (mode_ == Mode::read && !cache_->IsCompleted()) {
           return errors::Internal(
               "Cache should only be read after it has been completed.");
@@ -788,8 +799,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
      private:
       class MemoryWriterIterator : public DatasetIterator<MemoryDataset> {
        public:
-        explicit MemoryWriterIterator(const Params& params,
-                                      const std::shared_ptr<MemoryCache>& cache)
+        explicit MemoryWriterIterator(const Params& params, MemoryCache* cache)
             : DatasetIterator<MemoryDataset>(params), cache_(cache) {
           CHECK(cache_);
         }
@@ -848,13 +858,12 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
        private:
         mutex mu_;
         std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
-        std::shared_ptr<MemoryCache> cache_;
+        MemoryCache* const cache_ GUARDED_BY(mu_);  // not owned.
       };  // MemoryWriterIterator
 
       class MemoryReaderIterator : public DatasetIterator<MemoryDataset> {
        public:
-        explicit MemoryReaderIterator(const Params& params,
-                                      const std::shared_ptr<MemoryCache>& cache)
+        explicit MemoryReaderIterator(const Params& params, MemoryCache* cache)
             : DatasetIterator<MemoryDataset>(params), cache_(cache), index_(0) {
           CHECK(cache);
         }
@@ -865,6 +874,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
           // thus we record the memory allocated for the cache here. The caveat
           // is that this is incorrect if there are concurrent instances of this
           // iterator.
+          tf_shared_lock l(mu_);
           for (size_t i = 0; i < cache_->size(); ++i) {
             RecordBufferEnqueue(ctx, cache_->at(i));
           }
@@ -914,7 +924,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
        private:
         mutex mu_;
-        const std::shared_ptr<MemoryCache> cache_;
+        MemoryCache* const cache_ GUARDED_BY(mu_);  // not owned.
         size_t index_ GUARDED_BY(mu_);
       };  // MemoryReaderIterator
 
@@ -931,14 +941,13 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
       }
 
       mutex mu_;
-      std::shared_ptr<MemoryCache> cache_;
+      MemoryCache* cache_ GUARDED_BY(mu_);  // not owned.
       enum Mode { read, write };
       Mode mode_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> iterator_ GUARDED_BY(mu_);
     };  // MemoryIterator
 
     const DatasetBase* const input_;
-    const std::shared_ptr<MemoryCache> cache_;
   };  // MemoryDataset
 };    // CacheDatasetOp
 
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index cb7477f9e2..d5b4bfa5c5 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -77,6 +77,7 @@ class IteratorResource : public ResourceBase {
       params.lib = captured_state->lib;
       params.function_handle_cache =
           captured_state->function_handle_cache.get();
+      params.resource_mgr = &captured_state->resource_mgr;
       return captured_state->iterator->GetNext(
           IteratorContext(std::move(params)), out_tensors, end_of_sequence);
     } else {
@@ -135,8 +136,8 @@ class IteratorResource : public ResourceBase {
     std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
     TF_RETURN_IF_ERROR(ctx->function_library()->Clone(&flib_def, &pflr, &lib));
     TF_RETURN_IF_ERROR(flib_def->AddLibrary(graph_def.library()));
-    std::unique_ptr<State> new_state(
-        new State(std::move(flib_def), std::move(pflr), lib, nullptr));
+    std::unique_ptr<State> new_state(new State(
+        std::move(flib_def), std::move(pflr), lib, nullptr /* iterator */));
 
     TF_RETURN_IF_ERROR(
         graph_runner.Run(&graph, new_state->lib, {}, {output_node}, &outputs));
@@ -145,6 +146,7 @@ class IteratorResource : public ResourceBase {
     IteratorContext::Params params(ctx);
     params.lib = new_state->lib;
     params.function_handle_cache = new_state->function_handle_cache.get();
+    params.resource_mgr = &new_state->resource_mgr;
     TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
                                              "Iterator", &new_state->iterator));
     TF_RETURN_IF_ERROR(
@@ -156,6 +158,7 @@ class IteratorResource : public ResourceBase {
       IteratorContext::Params params(ctx);
       params.lib = new_state->lib;
       params.function_handle_cache = new_state->function_handle_cache.get();
+      params.resource_mgr = &new_state->resource_mgr;
       DeviceBase* device = new_state->lib->device();
       params.allocator_getter = [device](AllocatorAttributes attrs) {
         return device->GetAllocator(attrs);
@@ -180,7 +183,8 @@ class IteratorResource : public ResourceBase {
       tf_shared_lock l(mu_);
       new_state.reset(new State(iterator_state_->flib_def,
                                 iterator_state_->pflr, iterator_state_->lib,
-                                nullptr, nullptr));
+                                nullptr /* function_handle_cache */,
+                                nullptr /* iterator */));
     }
 
     // Ensure that the iterator has access to all functions in the current
@@ -212,6 +216,7 @@ class IteratorResource : public ResourceBase {
     IteratorContext::Params params(ctx);
     params.lib = new_state->lib;
     params.function_handle_cache = new_state->function_handle_cache.get();
+    params.resource_mgr = &new_state->resource_mgr;
     TF_RETURN_IF_ERROR(dataset->MakeIterator(IteratorContext(std::move(params)),
                                              "Iterator", &iterator));
     TF_RETURN_IF_ERROR(
@@ -259,6 +264,7 @@ class IteratorResource : public ResourceBase {
     std::shared_ptr<ProcessFunctionLibraryRuntime> pflr;
     FunctionLibraryRuntime* lib = nullptr;  // not owned.
     std::unique_ptr<FunctionHandleCache> function_handle_cache;
+    ResourceMgr resource_mgr;
     std::unique_ptr<IteratorBase> iterator;
   };
 
diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index a070456414..ba2125a66e 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -98,6 +98,7 @@ class MultiDeviceIterator : public ResourceBase {
       IteratorContext::Params params(ctx);
       params.lib = lib_;
       params.function_handle_cache = function_handle_cache_.get();
+      params.resource_mgr = &resource_mgr_;
       IteratorContext iter_ctx(std::move(params));
       tf_shared_lock l(mu_);
       multi_device_buffer_->GetNextFromShard(
@@ -125,6 +126,8 @@ class MultiDeviceIterator : public ResourceBase {
     return function_handle_cache_.get();
   }
 
+  ResourceMgr* resource_mgr() { return &resource_mgr_; }
+
  private:
   // A private class that uses a background thread to keep a per device buffer
   // full.
@@ -350,6 +353,7 @@ class MultiDeviceIterator : public ResourceBase {
   const std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
   FunctionLibraryRuntime* const lib_ = nullptr;  // not owned.
   const std::unique_ptr<FunctionHandleCache> function_handle_cache_;
+  ResourceMgr resource_mgr_;
   std::shared_ptr<const FunctionLibraryDefinition> lib_def_ GUARDED_BY(mu_);
 
   int64 incarnation_id_ GUARDED_BY(mu_) = 0;
@@ -477,6 +481,7 @@ class MultiDeviceIteratorInitOp : public OpKernel {
     IteratorContext::Params params(ctx);
     params.lib = resource->lib();
     params.function_handle_cache = resource->function_handle_cache();
+    params.resource_mgr = resource->resource_mgr();
     IteratorContext iter_ctx(std::move(params));
     OP_REQUIRES_OK(
         ctx, dataset->MakeIterator(std::move(iter_ctx), "Iterator", &iterator));
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index 3cf005dbda..7134793e26 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/random/philox_random.h"
 #include "tensorflow/core/lib/random/random.h"
@@ -70,9 +71,9 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
       explicit Iterator(const typename DatasetIterator<T>::Params& params,
                         int64 seed, int64 seed2)
           : DatasetIterator<T>(params),
-            input_impl_(nullptr),
             seed_(seed),
             seed2_(seed2),
+            input_impl_(nullptr),
             epoch_(0),
             num_elements_(0),
             parent_generator_(seed, seed2),
@@ -174,6 +175,14 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
                                          /*ratio=*/1);
       }
 
+      void ResetRngs() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        // Reset the generators based on the current iterator seeds.
+        parent_generator_ = random::PhiloxRandom(seed_, seed2_);
+        generator_ = random::SingleSampleAdapter<random::PhiloxRandom>(
+            &parent_generator_);
+        generator_.Skip(num_random_samples_);
+      }
+
       Status SaveInternal(IteratorStateWriter* writer) override {
         mutex_lock l(mu_);
         // Save state needed to restore the random number generators.
@@ -281,6 +290,10 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
         return Status::OK();
       }
 
+      mutex mu_;
+      int64 seed_ GUARDED_BY(mu_);
+      int64 seed2_ GUARDED_BY(mu_);
+
      private:
       // Used to represent slices of `buffer_` that belong to different epochs.
       // The invariant maintained by the implementation is: `start` <= `end`.
@@ -301,19 +314,8 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
         return out;
       }
 
-      void ResetRngs() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-        // Reset the generators based on the current iterator seeds.
-        parent_generator_ = random::PhiloxRandom(seed_, seed2_);
-        generator_ = random::SingleSampleAdapter<random::PhiloxRandom>(
-            &parent_generator_);
-        generator_.Skip(num_random_samples_);
-      }
-
-      mutex mu_;
       std::unique_ptr<std::vector<Tensor>[]> buffer_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
-      int64 seed_ GUARDED_BY(mu_);
-      int64 seed2_ GUARDED_BY(mu_);
       int64 epoch_ GUARDED_BY(mu_);
       int64 num_elements_ GUARDED_BY(mu_);
       std::deque<std::unique_ptr<Slice>> slices_ GUARDED_BY(mu_);
@@ -370,7 +372,7 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
   }
 
  private:
-  // A dataset that uses a pseduorandom sequence of seeds for the iterators
+  // A dataset that uses a pseudorandom sequence of seeds for the iterators
   // created from it. Used when `reshuffle_each_iteration` is true.
   class ReshufflingDataset : public ShuffleDatasetBase {
    public:
@@ -378,37 +380,114 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
                        int64 buffer_size, int64 seed, int64 seed2, int64 count)
         : ShuffleDatasetBase(ctx, input, buffer_size, count),
           seed_(seed),
-          seed2_(seed2),
-          parent_generator_(seed, seed2),
-          generator_(&parent_generator_) {}
+          seed2_(seed2) {}
 
     string DebugString() const override {
-      mutex_lock l(mu_);
       return strings::StrCat("ShuffleDatasetOp(", buffer_size_, ", ", seed_,
                              ", ", seed2_, ")::ReshufflingDataset");
     }
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      int64 iterator_seed;
-      int64 iterator_seed2;
-      {
-        mutex_lock l(mu_);
-        iterator_seed = Random();
-        iterator_seed2 = Random();
-      }
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Shuffle")},
-                       iterator_seed, iterator_seed2));
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::Shuffle")}, seed_, seed2_));
     }
 
    protected:
+    class RandomSeedGenerator : public ResourceBase {
+     public:
+      RandomSeedGenerator(int64 seed, int64 seed2)
+          : seed_(seed),
+            seed2_(seed2),
+            parent_generator_(seed, seed2),
+            generator_(&parent_generator_) {}
+
+      string DebugString() override {
+        return "ReshufflingDataset::RandomSeedGenerator";
+      }
+
+      void GenerateRandomSeeds(int64* seed1, int64* seed2) {
+        mutex_lock l(mu_);
+        num_random_samples_++;
+        *seed1 = generator_();
+        num_random_samples_++;
+        *seed2 = generator_();
+      }
+
+      int64 num_random_samples() {
+        tf_shared_lock l(mu_);
+        return num_random_samples_;
+      }
+
+      void set_num_random_samples(int64 num_random_samples) {
+        mutex_lock l(mu_);
+        num_random_samples_ = num_random_samples;
+      }
+
+      void Reset() {
+        mutex_lock l(mu_);
+        // Reset the generators based on the current seeds.
+        parent_generator_ = random::PhiloxRandom(seed_, seed2_);
+        generator_ = random::SingleSampleAdapter<random::PhiloxRandom>(
+            &parent_generator_);
+        generator_.Skip(num_random_samples_);
+      }
+
+     private:
+      const int64 seed_;
+      const int64 seed2_;
+      mutex mu_;
+      random::PhiloxRandom parent_generator_ GUARDED_BY(mu_);
+      random::SingleSampleAdapter<random::PhiloxRandom> generator_
+          GUARDED_BY(mu_);
+      int64 num_random_samples_ GUARDED_BY(mu_) = 0;
+    };
+
     class Iterator : public ShuffleDatasetBase::Iterator<ReshufflingDataset> {
      public:
       explicit Iterator(const Params& params, int64 seed, int64 seed2)
           : ShuffleDatasetBase::Iterator<ReshufflingDataset>(params, seed,
                                                              seed2) {}
 
+      ~Iterator() override { seed_generator_->Unref(); }
+
+      Status Initialize(IteratorContext* ctx) override {
+        // Firstly, lookup or create a seed generator from the IteratorResource
+        // resource_mgr.
+        ResourceMgr* mgr = ctx->resource_mgr();
+        RandomSeedGenerator* seed_generator;
+        const string name = strings::StrCat(prefix(), "::", dataset()->name(),
+                                            "::RandomSeedGenerator");
+
+        int64 dataset_seed, dataset_seed2;
+        {
+          tf_shared_lock l(mu_);
+          // Ideally we'd like to hold this lock in the LookupOrCreate method,
+          // but that trips up our Deadlock detection code.
+          dataset_seed = seed_;
+          dataset_seed2 = seed2_;
+        }
+        TF_RETURN_IF_ERROR(mgr->LookupOrCreate<RandomSeedGenerator>(
+            "tf_data", name, &seed_generator,
+            [dataset_seed,
+             dataset_seed2](RandomSeedGenerator** seed_generator) {
+              // On the first iterator creation, use the original seeds from the
+              // dataset to seed a `RandomSeedGenerator` that will provide seeds
+              // for subsequent repetitions of the same dataset.
+              *seed_generator =
+                  new RandomSeedGenerator(dataset_seed, dataset_seed2);
+              return Status::OK();
+            }));
+        // Now use the seed generator to update the base class Iterator seeds
+        // and random number generator with generated seeds for the current
+        // repetition.
+        mutex_lock l(mu_);
+        seed_generator->GenerateRandomSeeds(&seed_, &seed2_);
+        ResetRngs();
+        seed_generator_ = seed_generator;
+        return Status::OK();
+      }
+
      protected:
       std::shared_ptr<model::Node> CreateNode(
           IteratorContext* ctx, model::Node::Args args) const override {
@@ -417,12 +496,10 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
       }
 
       Status SaveInternal(IteratorStateWriter* writer) override {
-        mutex_lock l(dataset()->mu_);
-
         // Save RNG state of Dataset.
         TF_RETURN_IF_ERROR(
             writer->WriteScalar(full_name("ds_num_random_samples"),
-                                dataset()->num_random_samples_));
+                                seed_generator_->num_random_samples()));
 
         // Save the Iterator.
         return ShuffleDatasetBase::Iterator<ReshufflingDataset>::SaveInternal(
@@ -431,24 +508,25 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
 
       Status RestoreInternal(IteratorContext* ctx,
                              IteratorStateReader* reader) override {
-        mutex_lock l(dataset()->mu_);
-
         // Restore RNG state of Dataset.
-        TF_RETURN_IF_ERROR(
-            reader->ReadScalar(full_name("ds_num_random_samples"),
-                               &dataset()->num_random_samples_));
-        dataset()->ResetRngs();
+        int64 num_random_samples;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name("ds_num_random_samples"), &num_random_samples));
+        seed_generator_->set_num_random_samples(num_random_samples);
+        seed_generator_->Reset();
 
         // Restore the Iterator.
         return ShuffleDatasetBase::Iterator<
             ReshufflingDataset>::RestoreInternal(ctx, reader);
       }
+
+     private:
+      RandomSeedGenerator* seed_generator_;
     };
 
     Status AsGraphDefInternal(SerializationContext* ctx,
                               DatasetGraphDefBuilder* b,
                               Node** output) const override {
-      mutex_lock l(mu_);
       Node* input_graph_node = nullptr;
       TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
       Node* buffer_size = nullptr;
@@ -469,28 +547,8 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
     }
 
    private:
-    random::SingleSampleAdapter<random::PhiloxRandom>::ResultType Random() const
-        EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-      num_random_samples_++;
-      auto out = generator_();
-      return out;
-    }
-
-    void ResetRngs() const EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-      // Reset the generators based on the current seeds.
-      parent_generator_ = random::PhiloxRandom(seed_, seed2_);
-      generator_ =
-          random::SingleSampleAdapter<random::PhiloxRandom>(&parent_generator_);
-      generator_.Skip(num_random_samples_);
-    }
-
-    mutable int64 seed_ GUARDED_BY(mu_);
-    mutable int64 seed2_ GUARDED_BY(mu_);
-    mutable mutex mu_;
-    mutable random::PhiloxRandom parent_generator_ GUARDED_BY(mu_);
-    mutable random::SingleSampleAdapter<random::PhiloxRandom> generator_
-        GUARDED_BY(mu_);
-    mutable int64 num_random_samples_ GUARDED_BY(mu_) = 0;
+    const int64 seed_;
+    const int64 seed2_;
   };
 
   // A dataset that uses the same fixed seed for all iterators created from it.
-- 
GitLab


From 983b7403151f07740fa81ef9a58b7f3768e9431f Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 6 Dec 2018 17:14:35 -0800
Subject: [PATCH 1475/1554] Adding a whitelited_stateful_ops argument to the
 old Defun in which stateful ops belonging to this whitelist are recreated
 instead of being re-used when captured by value. This is useful for dataset
 make_one_shot_iterator which captures all inputs by value.

PiperOrigin-RevId: 224436333
---
 tensorflow/python/framework/function.py      | 40 +++++++++++++++-----
 tensorflow/python/framework/function_test.py | 22 +++++++++++
 2 files changed, 53 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index 622686ce00..cfdc915a1b 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -209,6 +209,7 @@ class _DefinedFunction(object):
                out_names=None,
                shape_func=None,
                capture_by_value=False,
+               whitelisted_stateful_ops=None,
                **kwargs):
     """Creates _DefinedFunction.
 
@@ -229,6 +230,8 @@ class _DefinedFunction(object):
         output shapes.
       capture_by_value: Boolean (defaults to False). If True, captured values
         will be copied into the function body.
+      whitelisted_stateful_ops: A set of ops that if stateful we ignore and
+        copy into the function body, when `capture_by_value` is True.
       **kwargs: The keyword arguments. **kwargs is passed to every call
         site of this function.
 
@@ -244,6 +247,9 @@ class _DefinedFunction(object):
     self._out_names = out_names
     self._shape_func = shape_func
     self._capture_by_value = capture_by_value
+    self._whitelisted_stateful_ops = whitelisted_stateful_ops
+    if self._whitelisted_stateful_ops is None:
+      self._whitelisted_stateful_ops = set()
     self._extra_kwargs = kwargs
     # Constructed only when C API is disabled, lazily
     self._definition = None
@@ -340,8 +346,13 @@ class _DefinedFunction(object):
       return
 
     temp_graph = func_graph_from_py_func(
-        self._func, self._arg_names, self._arg_types, self._func_name,
-        self._capture_by_value, self._caller_device)
+        self._func,
+        self._arg_names,
+        self._arg_types,
+        self._func_name,
+        self._capture_by_value,
+        self._caller_device,
+        whitelisted_stateful_ops=self._whitelisted_stateful_ops)
 
     self._extra_inputs = temp_graph.extra_inputs
     # pylint: disable=protected-access
@@ -625,9 +636,11 @@ class _FuncGraph(ops.Graph):
   function argument and the caller passes in the captured tensor.
   """
 
-  def __init__(self, name, capture_by_value, *args, **kwargs):
+  def __init__(self, name, capture_by_value, whitelisted_stateful_ops, *args,
+               **kwargs):
     super(_FuncGraph, self).__init__(*args, **kwargs)
     self._capture_by_value = capture_by_value
+    self._whitelisted_stateful_ops = whitelisted_stateful_ops
     self._building_function = True
     self._outer_graph = ops.get_default_graph()
     self._vscope = vs.get_variable_scope()
@@ -785,7 +798,7 @@ class _FuncGraph(ops.Graph):
     # pylint: disable=protected-access
     op_def = graph_to_function_def._get_op_def(op)
     # pylint: enable=protected-access
-    if op_def.is_stateful:
+    if op_def.is_stateful and op not in self._whitelisted_stateful_ops:
       raise ValueError("Cannot capture a stateful node (name:%s, type:%s) "
                        "by value." % (op.name, op.type))
     elif op.type in ("Placeholder", "PlaceholderV2"):
@@ -807,10 +820,17 @@ class _FuncGraph(ops.Graph):
     return captured_op
 
 
-def func_graph_from_py_func(func, arg_names, arg_types, name=None,
-                            capture_by_value=False, device=None,
-                            colocation_stack=None, container=None,
-                            collections_ref=None, arg_shapes=None):
+def func_graph_from_py_func(func,
+                            arg_names,
+                            arg_types,
+                            name=None,
+                            capture_by_value=False,
+                            device=None,
+                            colocation_stack=None,
+                            container=None,
+                            collections_ref=None,
+                            arg_shapes=None,
+                            whitelisted_stateful_ops=None):
   """Returns a _FuncGraph generated from `func`.
 
   Args:
@@ -828,6 +848,8 @@ def func_graph_from_py_func(func, arg_names, arg_types, name=None,
     collections_ref: A reference to a collections dict the _FuncGraph should
       use internally.
     arg_shapes: A sequence of the function's argument shapes.
+    whitelisted_stateful_ops: A set of ops that if stateful we ignore and
+      re-create.
 
   Returns:
     A _FuncGraph.
@@ -837,7 +859,7 @@ def func_graph_from_py_func(func, arg_names, arg_types, name=None,
   """
   if not name:
     name = function_utils.get_func_name(func)
-  func_graph = _FuncGraph(name, capture_by_value)
+  func_graph = _FuncGraph(name, capture_by_value, whitelisted_stateful_ops)
 
   with func_graph.as_default(), ops.device(device):
     # pylint: disable=protected-access
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index d71f06ea52..6ec71ba8e9 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -1054,6 +1054,28 @@ class FunctionTest(test.TestCase):
         self.assertFalse(all(val3 == val1))
         self.assertFalse(all(val4 == val2))
 
+  def testStatefulFunctionWithWhitelisting(self):
+    t = random_ops.random_uniform([100], maxval=10, dtype=dtypes.int32)
+
+    @function.Defun(capture_by_value=True)
+    def StatefulFn():
+      return t + constant_op.constant(3, dtype=dtypes.int32)
+
+    # First time we try to capture a stateful RandomUniform op.
+    with self.assertRaisesRegexp(ValueError, "Cannot capture a stateful node"):
+      res = StatefulFn()
+
+    # This time we whitelist this op, so that its recreated.
+    @function.Defun(capture_by_value=True, whitelisted_stateful_ops=set([t.op]))
+    def StatefulFn2():
+      return t + constant_op.constant(3, dtype=dtypes.int32)
+
+    res = StatefulFn2()
+    with session.Session() as sess:
+      r = sess.run(res)
+      for i in r:
+        self.assertGreaterEqual(i, 3)
+
   @test_util.run_deprecated_v1
   def testSameFunctionOnTwoDevices(self):
 
-- 
GitLab


From 9e4dca5db91d9ac587514cd4cf19f334130f9837 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Thu, 6 Dec 2018 17:27:18 -0800
Subject: [PATCH 1476/1554] Make SelfAdjointEigGradTest use gradient_checker_v2

PiperOrigin-RevId: 224437857
---
 .../kernel_tests/self_adjoint_eig_op_test.py  | 60 +++++++++----------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 42577f7e42..323d14b7d8 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -23,7 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -185,53 +185,51 @@ def _GetSelfAdjointEigGradTest(dtype_, shape_, compute_v_):
     n = shape_[-1]
     batch_shape = shape_[:-2]
     np_dtype = dtype_.as_numpy_dtype
-    a = np.random.uniform(
-        low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(np_dtype)
-    if dtype_.is_complex:
-      a += 1j * np.random.uniform(
+
+    def RandomInput():
+      a = np.random.uniform(
           low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(np_dtype)
-    a += np.conj(a.T)
-    a = np.tile(a, batch_shape + (1, 1))
+      if dtype_.is_complex:
+        a += 1j * np.random.uniform(
+            low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(np_dtype)
+      a += np.conj(a.T)
+      a = np.tile(a, batch_shape + (1, 1))
+      return a
+
     # Optimal stepsize for central difference is O(epsilon^{1/3}).
     epsilon = np.finfo(np_dtype).eps
     delta = 0.1 * epsilon**(1.0 / 3.0)
     # tolerance obtained by looking at actual differences using
     # np.linalg.norm(theoretical-numerical, np.inf) on -mavx build
+    # after discarding one random input sample
+    _ = RandomInput()
     if dtype_ in (dtypes_lib.float32, dtypes_lib.complex64):
       tol = 1e-2
     else:
       tol = 1e-7
     with self.session(use_gpu=True):
-      tf_a = constant_op.constant(a)
-      if compute_v_:
-        tf_e, tf_v = linalg_ops.self_adjoint_eig(tf_a)
+      def Compute(x):
+        e, v = linalg_ops.self_adjoint_eig(x)
         # (complex) Eigenvectors are only unique up to an arbitrary phase
         # We normalize the vectors such that the first component has phase 0.
-        top_rows = tf_v[..., 0:1, :]
-        if tf_a.dtype.is_complex:
+        top_rows = v[..., 0:1, :]
+        if dtype_.is_complex:
           angle = -math_ops.angle(top_rows)
           phase = math_ops.complex(math_ops.cos(angle), math_ops.sin(angle))
         else:
           phase = math_ops.sign(top_rows)
-        tf_v *= phase
-        outputs = [tf_e, tf_v]
+        v *= phase
+        return e, v
+
+      if compute_v_:
+        funcs = [lambda x: Compute(x)[0], lambda x: Compute(x)[1]]
       else:
-        tf_e = linalg_ops.self_adjoint_eigvals(tf_a)
-        outputs = [tf_e]
-      for b in outputs:
-        x_init = np.random.uniform(
-            low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(np_dtype)
-        if dtype_.is_complex:
-          x_init += 1j * np.random.uniform(
-              low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(np_dtype)
-        x_init += np.conj(x_init.T)
-        x_init = np.tile(x_init, batch_shape + (1, 1))
-        theoretical, numerical = gradient_checker.compute_gradient(
-            tf_a,
-            tf_a.get_shape().as_list(),
-            b,
-            b.get_shape().as_list(),
-            x_init_value=x_init,
+        funcs = [linalg_ops.self_adjoint_eigvals]
+
+      for f in funcs:
+        theoretical, numerical = gradient_checker_v2.compute_gradient(
+            f,
+            [RandomInput()],
             delta=delta)
         self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol)
 
@@ -245,7 +243,7 @@ if __name__ == "__main__":
       for size in 1, 2, 5, 10:
         for batch_dims in [(), (3,)] + [(3, 2)] * (max(size, size) < 10):
           shape = batch_dims + (size, size)
-          name = "%s_%s_%s" % (dtype, "_".join(map(str, shape)), compute_v)
+          name = "%s_%s_%s" % (dtype.name, "_".join(map(str, shape)), compute_v)
           _AddTest(SelfAdjointEigTest, "SelfAdjointEig", name,
                    _GetSelfAdjointEigTest(dtype, shape, compute_v))
           _AddTest(SelfAdjointEigGradTest, "SelfAdjointEigGrad", name,
-- 
GitLab


From 24cd8686127694c5fadc1abc3e5183c7f8e2bb87 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 6 Dec 2018 18:02:42 -0800
Subject: [PATCH 1477/1554] Fixed the metrics for TPUStrategy with Keras. And
 added a test for that.

PiperOrigin-RevId: 224441940
---
 .../contrib/distribute/python/keras_test.py      | 16 ++++++++++++----
 .../python/keras/engine/training_distributed.py  | 14 ++++++++++----
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 796c4ed9f6..771c48bdd8 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -165,7 +165,9 @@ def get_multi_inputs_multi_outputs_data():
   return (train_data, test_data)
 
 
-def batch_wrapper(dataset, batch_size, distribution):
+def batch_wrapper(dataset, batch_size, distribution, repeat=None):
+  if repeat:
+    dataset = dataset.repeat(repeat)
   # TPUs currently require fully defined input shapes, drop_remainder ensures
   # the input will have fully defined shapes.
   if isinstance(distribution, tpu_strategy.TPUStrategy):
@@ -216,6 +218,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
                                 with_distribution,
                                 x_train, y_train, x_predict):
   """Generates the inputs for correctness check when enable Keras with DS."""
+  training_epochs = 2
   global_batch_size = 64
   batch_size = global_batch_size
   # TODO(b/118776054): Use global batch size for Keras/DS support.
@@ -231,7 +234,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
         'batch_size': batch_size,
         'x': x_train,
         'y': y_train,
-        'epochs': 1,
+        'epochs': training_epochs,
         'shuffle': False,
     }
 
@@ -252,13 +255,14 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
     # keras.fit/evaluate/predict. The batch size is part of the dataset.
     train_dataset = dataset_ops.Dataset.from_tensor_slices(
         (x_train, y_train))
-    x = batch_wrapper(train_dataset, batch_size, with_distribution)
+    x = batch_wrapper(
+        train_dataset, batch_size, with_distribution, repeat=training_epochs)
 
     training_inputs = {
         'batch_size': None,
         'x': x,
         'y': None,
-        'epochs': 1,
+        'epochs': training_epochs,
         'shuffle': False,
         'steps_per_epoch': len(x_train) // global_batch_size,
     }
@@ -1301,11 +1305,14 @@ class TestDistributionStrategyCorrectness(test.TestCase,
 
     with self.cached_session():
       tolerance = 1e-5
+      metrics = ['mse']
 
       if isinstance(distribution, (mirrored_strategy.MirroredStrategy,
                                    mirrored_strategy.CoreMirroredStrategy)):
         # TODO(b/119257215): use the default one once the flakyness is fixed.
         tolerance = 1e-4
+        # TODO(b/120570676): Enable metrics check once the bug is fixed.
+        metrics = None
 
       keras.backend.set_image_data_format('channels_last')
       np.random.seed(_RANDOM_SEED)
@@ -1346,6 +1353,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         model.compile(
             loss=keras.losses.mean_squared_error,
             optimizer=gradient_descent_keras.SGD(0.5),
+            metrics=metrics,
             distribute=with_distribution)
 
         training_inputs, eval_inputs, predict_inputs = (
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index fc72f6bd15..d20d092d8e 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -350,11 +350,18 @@ def experimental_test_loop(model,
   for step in range(steps):
     _, batch_outs = K.get_session().run([test_op, output_tensors])
     for i, label in enumerate(model.metrics_names):
-      outs[i] += batch_outs[label]
+      if i == 0:
+        # Loss is stateless metrics.
+        outs[i] += batch_outs[label]
+      else:
+        # For all stateful metrics, the aggregation is handled by mirrored vars.
+        outs[i] = batch_outs[label]
+
     if verbose >= 1:
       progbar.update(step + 1)
-  for i in range(len(outs)):
-    outs[i] /= (steps)
+
+  if len(outs) >= 0:
+    outs[0] /= (steps)
 
   if initialize_finalize_strategy:
     K.get_session().run(current_strategy.finalize())
@@ -724,4 +731,3 @@ def _reset_metrics(model, distributed_model=None):
         distributed_model or
         model._distribution_strategy.unwrap(model._grouped_model)[0])
     distributed_model.reset_metrics()
-
-- 
GitLab


From 3640da49c3731807a3dbc27d813e8ab68a86328a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 6 Dec 2018 18:38:24 -0800
Subject: [PATCH 1478/1554] Use tpu.rewrite() instead of
 tpu.rewrite_for_inference() in tpu_estimator.py for
 TPUEstimator.export_savedmodel().

PiperOrigin-RevId: 224445962
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 84816d70d0..96b9556e13 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2234,7 +2234,7 @@ class TPUEstimator(estimator_lib.Estimator):
     def computation():
       """Compute tpu tensors used in export_outputs.
 
-      Passed to rewrite_for_inference so that model_fn will be called under
+      Passed to rewrite so that model_fn will be called under
       the rewriting contexts. Only tpu tensors are returned, but export_outputs
       and scaffold are captured.
 
@@ -2243,7 +2243,7 @@ class TPUEstimator(estimator_lib.Estimator):
          outside_compilation.
       """
       # We should only call model fn once and it should be inside `computation`
-      # so that building the graph will happen under `rewrite_for_inference`.
+      # so that building the graph will happen under `rewrite`.
       mode = model_fn_lib.ModeKeys.PREDICT
       estimator_spec = self._call_model_fn(features, labels, mode, config)
 
@@ -2260,7 +2260,7 @@ class TPUEstimator(estimator_lib.Estimator):
       capture.capture((estimator_spec, tensors_dict, tensors))
       return tpu_tensors
 
-    tpu_tensors_on_cpu = tpu.rewrite_for_inference(computation)
+    tpu_tensors_on_cpu = tpu.rewrite(computation)
     estimator_spec, tensors_dict, tensors = capture.get()
 
     # Reconstruct `tensors`, but with `tpu_tensors` replaced with
-- 
GitLab


From 9408b2835c84f7bd1ad25b95b2cb85682ee3f348 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Thu, 6 Dec 2018 22:01:21 -0800
Subject: [PATCH 1479/1554] Add the beginnings of ::tensorflow::Env support for
 C.

PiperOrigin-RevId: 224461412
---
 tensorflow/c/BUILD       |  49 +++++++++++++
 tensorflow/c/env.cc      | 147 ++++++++++++++++++++++++++++++++++++++
 tensorflow/c/env.h       | 148 +++++++++++++++++++++++++++++++++++++++
 tensorflow/c/env_test.cc |  94 +++++++++++++++++++++++++
 tensorflow/core/BUILD    |   5 +-
 5 files changed, 442 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/c/env.cc
 create mode 100644 tensorflow/c/env.h
 create mode 100644 tensorflow/c/env_test.cc

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index ad2ae08a37..25df970eca 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -175,6 +175,34 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "env",
+    srcs = [
+        "env.cc",
+    ],
+    hdrs = [
+        "env.h",
+    ],
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = select({
+        "//tensorflow:android": [
+            ":c_api",
+            ":tf_status_helper",
+            "//tensorflow/core:android_tensorflow_lib_lite",
+            "//tensorflow/core:platform_env",
+            "//tensorflow/core:lib",
+        ],
+        "//conditions:default": [
+            ":c_api",
+            ":tf_status_helper",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:platform_env",
+            "//tensorflow/core:lib",
+        ],
+    }) + [":c_api_internal"],
+)
+
 tf_cuda_library(
     name = "kernels",
     srcs = [
@@ -334,6 +362,27 @@ tf_kernel_library(
     alwayslink = 1,
 )
 
+tf_cuda_cc_test(
+    name = "env_test",
+    size = "small",
+    srcs = ["env_test.cc"],
+    linkopts = select({
+        "//tensorflow:darwin": ["-headerpad_max_install_names"],
+        "//conditions:default": [],
+    }),
+    tags = ["noasan"],
+    # We must ensure that the dependencies can be dynamically linked since
+    # the shared library must be able to use core:framework.
+    # linkstatic = tf_kernel_tests_linkstatic(),
+    deps = [
+        ":c_api",
+        ":env",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "kernels_test",
     size = "small",
diff --git a/tensorflow/c/env.cc b/tensorflow/c/env.cc
new file mode 100644
index 0000000000..fcff1e413d
--- /dev/null
+++ b/tensorflow/c/env.cc
@@ -0,0 +1,147 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/env.h"
+
+#include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/c/tf_status_helper.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/types.h"
+
+struct TF_StringStream {
+  std::vector<::tensorflow::string>* list;
+  size_t position;
+};
+
+void TF_CreateDir(const char* dirname, TF_Status* status) {
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(
+      status, ::tensorflow::Env::Default()->CreateDir(dirname));
+}
+
+void TF_DeleteDir(const char* dirname, TF_Status* status) {
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(
+      status, ::tensorflow::Env::Default()->DeleteDir(dirname));
+}
+
+void TF_DeleteRecursively(const char* dirname, int64_t* undeleted_file_count,
+                          int64_t* undeleted_dir_count, TF_Status* status) {
+  ::tensorflow::int64 f, d;
+
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(
+      status, ::tensorflow::Env::Default()->DeleteRecursively(dirname, &f, &d));
+  *undeleted_file_count = f;
+  *undeleted_dir_count = d;
+}
+
+void TF_FileStat(const char* filename, TF_FileStatistics* stats,
+                 TF_Status* status) {
+  ::tensorflow::FileStatistics cc_stats;
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Status s =
+      ::tensorflow::Env::Default()->Stat(filename, &cc_stats);
+  ::tensorflow::Set_TF_Status_from_Status(status, s);
+  if (s.ok()) {
+    stats->length = cc_stats.length;
+    stats->mtime_nsec = cc_stats.mtime_nsec;
+    stats->is_directory = cc_stats.is_directory;
+  }
+}
+
+void TF_NewWritableFile(const char* filename, TF_WritableFileHandle** handle,
+                        TF_Status* status) {
+  std::unique_ptr<::tensorflow::WritableFile> f;
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Status s =
+      ::tensorflow::Env::Default()->NewWritableFile(filename, &f);
+  ::tensorflow::Set_TF_Status_from_Status(status, s);
+
+  if (s.ok()) {
+    *handle = reinterpret_cast<TF_WritableFileHandle*>(f.release());
+  }
+}
+
+void TF_CloseWritableFile(TF_WritableFileHandle* handle, TF_Status* status) {
+  auto* cc_file = reinterpret_cast<::tensorflow::WritableFile*>(handle);
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(status, cc_file->Close());
+  delete cc_file;
+}
+
+void TF_SyncWritableFile(TF_WritableFileHandle* handle, TF_Status* status) {
+  auto* cc_file = reinterpret_cast<::tensorflow::WritableFile*>(handle);
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(status, cc_file->Sync());
+}
+
+void TF_FlushWritableFile(TF_WritableFileHandle* handle, TF_Status* status) {
+  auto* cc_file = reinterpret_cast<::tensorflow::WritableFile*>(handle);
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(status, cc_file->Flush());
+}
+
+void TF_AppendWritableFile(TF_WritableFileHandle* handle, const char* data,
+                           size_t length, TF_Status* status) {
+  auto* cc_file = reinterpret_cast<::tensorflow::WritableFile*>(handle);
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(
+      status, cc_file->Append(::tensorflow::StringPiece{data, length}));
+}
+
+void TF_DeleteFile(const char* filename, TF_Status* status) {
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(
+      status, ::tensorflow::Env::Default()->DeleteFile(filename));
+}
+
+bool TF_StringStreamNext(TF_StringStream* list, const char** result) {
+  if (list->position >= list->list->size()) {
+    *result = nullptr;
+    return false;
+  }
+
+  *result = list->list->at(list->position++).c_str();
+  return true;
+}
+
+void TF_StringStreamDone(TF_StringStream* list) {
+  delete list->list;
+  delete list;
+}
+TF_StringStream* TF_GetChildren(const char* dirname, TF_Status* status) {
+  auto* children = new std::vector<::tensorflow::string>;
+
+  TF_SetStatus(status, TF_OK, "");
+  ::tensorflow::Set_TF_Status_from_Status(
+      status, ::tensorflow::Env::Default()->GetChildren(dirname, children));
+
+  auto* list = new TF_StringStream;
+  list->list = children;
+  list->position = 0;
+  return list;
+}
+
+TF_StringStream* TF_GetLocalTempDirectories() {
+  auto* tmpdirs = new std::vector<::tensorflow::string>;
+
+  ::tensorflow::Env::Default()->GetLocalTempDirectories(tmpdirs);
+
+  auto* list = new TF_StringStream;
+  list->list = tmpdirs;
+  list->position = 0;
+  return list;
+}
diff --git a/tensorflow/c/env.h b/tensorflow/c/env.h
new file mode 100644
index 0000000000..32d3f1f745
--- /dev/null
+++ b/tensorflow/c/env.h
@@ -0,0 +1,148 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_C_ENV_H_
+#define TENSORFLOW_C_ENV_H_
+
+#include "tensorflow/c/c_api.h"
+
+// --------------------------------------------------------------------------
+// C API for tensorflow::Env.
+
+struct TF_WritableFileHandle;
+struct TF_StringStream;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct TF_FileStatistics {
+  // The length of the file in bytes.
+  int64_t length;
+  // The last modified time in nanoseconds.
+  int64_t mtime_nsec;
+  // Whether the name refers to a directory.
+  bool is_directory;
+} TF_FileStatistics;
+
+// Creates the specified directory. Typical status code are:
+//  * TF_OK - successfully created the directory
+//  * TF_ALREADY_EXISTS - directory already exists
+//  * TF_PERMISSION_DENIED - dirname is not writable
+TF_CAPI_EXPORT extern void TF_CreateDir(const char* dirname, TF_Status* status);
+
+// Deletes the specified directory. Typical status codes are:
+//  * TF_OK - successfully deleted the directory
+//  * TF_FAILED_PRECONDITION - the directory is not empty
+TF_CAPI_EXPORT extern void TF_DeleteDir(const char* dirname, TF_Status* status);
+
+// Deletes the specified directory and all subdirectories and files underneath
+// it. This is accomplished by traversing the directory tree rooted at dirname
+// and deleting entries as they are encountered.
+//
+// If dirname itself is not readable or does not exist, *undeleted_dir_count is
+// set to 1, *undeleted_file_count is set to 0 and an appropriate status (e.g.
+// TF_NOT_FOUND) is returned.
+//
+// If dirname and all its descendants were successfully deleted, TF_OK is
+// returned and both error counters are set to zero.
+//
+// Otherwise, while traversing the tree, undeleted_file_count and
+// undeleted_dir_count are updated if an entry of the corresponding type could
+// not be deleted. The returned error status represents the reason that any one
+// of these entries could not be deleted.
+//
+// Typical status codes:
+//  * TF_OK - dirname exists and we were able to delete everything underneath
+//  * TF_NOT_FOUND - dirname doesn't exist
+//  * TF_PERMISSION_DENIED - dirname or some descendant is not writable
+//  * TF_UNIMPLEMENTED - some underlying functions (like Delete) are not
+//    implemented
+TF_CAPI_EXPORT extern void TF_DeleteRecursively(const char* dirname,
+                                                int64_t* undeleted_file_count,
+                                                int64_t* undeleted_dir_count,
+                                                TF_Status* status);
+
+// Obtains statistics for the given path. If status is TF_OK, *stats is
+// updated, otherwise it is not touched.
+TF_CAPI_EXPORT extern void TF_FileStat(const char* filename,
+                                       TF_FileStatistics* stats,
+                                       TF_Status* status);
+
+// Creates or truncates the given filename and returns a handle to be used for
+// appending data to the file. If status is TF_OK, *handle is updated and the
+// caller is responsible for freeing it (see TF_CloseWritableFile).
+TF_CAPI_EXPORT extern void TF_NewWritableFile(const char* filename,
+                                              TF_WritableFileHandle** handle,
+                                              TF_Status* status);
+
+// Closes the given handle and frees its memory. If there was a problem closing
+// the file, it is indicated by status. Memory is freed in any case.
+TF_CAPI_EXPORT extern void TF_CloseWritableFile(TF_WritableFileHandle* handle,
+                                                TF_Status* status);
+
+// Syncs content of the handle to the filesystem. Blocks waiting for the
+// filesystem to indicate that the content has been persisted.
+TF_CAPI_EXPORT extern void TF_SyncWritableFile(TF_WritableFileHandle* handle,
+                                               TF_Status* status);
+
+// Flush local buffers to the filesystem. If the process terminates after a
+// successful flush, the contents may still be persisted, since the underlying
+// filesystem may eventually flush the contents.  If the OS or machine crashes
+// after a successful flush, the contents may or may not be persisted, depending
+// on the implementation.
+TF_CAPI_EXPORT extern void TF_FlushWritableFile(TF_WritableFileHandle* handle,
+                                                TF_Status* status);
+
+// Appends the given bytes to the file. Any failure to do so is indicated in
+// status.
+TF_CAPI_EXPORT extern void TF_AppendWritableFile(TF_WritableFileHandle* handle,
+                                                 const char* data,
+                                                 size_t length,
+                                                 TF_Status* status);
+
+// Deletes the named file and indicates whether successful in *status.
+TF_CAPI_EXPORT extern void TF_DeleteFile(const char* filename,
+                                         TF_Status* status);
+
+// Retrieves the next item from the given TF_StringStream and places a pointer
+// to it in *result. If no more items are in the list, *result is set to NULL
+// and false is returned.
+//
+// Ownership of the items retrieved with this function remains with the library.
+// Item points are invalidated after a call to TF_StringStreamDone.
+TF_CAPI_EXPORT extern bool TF_StringStreamNext(TF_StringStream* list,
+                                               const char** result);
+
+// Frees the resources associated with given string list. All pointers returned
+// by TF_StringStreamNext are invalid after this call.
+TF_CAPI_EXPORT extern void TF_StringStreamDone(TF_StringStream* list);
+
+// Retrieves the list of children of the given directory. You can iterate
+// through the list with TF_StringStreamNext. The caller is responsible for
+// freeing the list (see TF_StringStreamDone).
+TF_CAPI_EXPORT extern TF_StringStream* TF_GetChildren(const char* filename,
+                                                      TF_Status* status);
+
+// Retrieves a list of directory names on the local machine that may be used for
+// temporary storage. You can iterate through the list with TF_StringStreamNext.
+// The caller is responsible for freeing the list (see TF_StringStreamDone).
+TF_CAPI_EXPORT extern TF_StringStream* TF_GetLocalTempDirectories(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TENSORFLOW_C_ENV_H_
diff --git a/tensorflow/c/env_test.cc b/tensorflow/c/env_test.cc
new file mode 100644
index 0000000000..08e803f12f
--- /dev/null
+++ b/tensorflow/c/env_test.cc
@@ -0,0 +1,94 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/env.h"
+
+#include "tensorflow/c/c_api.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/types.h"
+
+#define ASSERT_TF_OK(x) ASSERT_EQ(TF_OK, TF_GetCode(x))
+
+TEST(TestEnv, TestDirHandling) {
+  TF_StringStream* tempdirs = TF_GetLocalTempDirectories();
+  const char* tempdir;
+  bool found = false;
+  while (TF_StringStreamNext(tempdirs, &tempdir)) {
+    found = true;
+
+    TF_Status* s = TF_NewStatus();
+
+    ::tensorflow::string dirpath =
+        ::tensorflow::io::JoinPath(tempdir, "somedir");
+    TF_CreateDir(dirpath.c_str(), s);
+    ASSERT_TF_OK(s) << "TF_CreateDir failed for " << dirpath << ": "
+                    << TF_Message(s);
+
+    ::tensorflow::string filepath =
+        ::tensorflow::io::JoinPath(dirpath, "somefile.txt");
+    TF_WritableFileHandle* handle;
+    TF_NewWritableFile(filepath.c_str(), &handle, s);
+    ASSERT_TF_OK(s) << "NewWritableFile failed for " << filepath << ": "
+                    << TF_Message(s);
+
+    const char* data = "Hello, world!\n";
+    TF_AppendWritableFile(handle, data, strlen(data), s);
+    ASSERT_TF_OK(s) << "TF_AppendWritableFile failed to append data to file at "
+                    << filepath << ": " << TF_Message(s);
+
+    TF_CloseWritableFile(handle, s);
+    ASSERT_TF_OK(s) << "TF_CloseWritableFile failed to close handle to "
+                    << filepath << ": " << TF_Message(s);
+
+    TF_StringStream* children = TF_GetChildren(dirpath.c_str(), s);
+    ASSERT_TF_OK(s) << "TF_GetChildren failed for " << dirpath;
+    const char* childpath;
+    ASSERT_TRUE(TF_StringStreamNext(children, &childpath));
+    ASSERT_EQ(::tensorflow::string(childpath), "somefile.txt");
+    // There should only be one file in this directory.
+    ASSERT_FALSE(TF_StringStreamNext(children, &childpath));
+    ASSERT_EQ(childpath, nullptr);
+    TF_StringStreamDone(children);
+
+    TF_FileStatistics stats;
+    TF_FileStat(filepath.c_str(), &stats, s);
+    ASSERT_EQ(stats.length, strlen(data));
+    ASSERT_FALSE(stats.is_directory);
+    ASSERT_GT(stats.mtime_nsec, 0);
+
+    // Trying to delete a non-empty directory should fail.
+    TF_DeleteDir(dirpath.c_str(), s);
+    ASSERT_NE(TF_OK, TF_GetCode(s))
+        << "TF_DeleteDir unexpectedly succeeded with a non-empty directory "
+        << dirpath;
+
+    TF_DeleteFile(filepath.c_str(), s);
+    ASSERT_TF_OK(s) << "TF_DeleteFile failed for " << filepath << ": "
+                    << TF_Message(s);
+
+    // Now deleting the directory should work.
+    TF_DeleteDir(dirpath.c_str(), s);
+    ASSERT_TF_OK(s) << "TF_DeleteDir failed for " << dirpath << ": "
+                    << TF_Message(s);
+
+    TF_DeleteStatus(s);
+    break;
+  }
+
+  ASSERT_TRUE(found) << "expected at least one temp dir";
+
+  TF_StringStreamDone(tempdirs);
+}
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 575edfe7a9..328bfe474d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -492,7 +492,10 @@ cc_library(
         ":platform_env_internal_hdrs",
     ],
     copts = tf_copts(),
-    visibility = ["//tensorflow/core:__subpackages__"],
+    visibility = [
+        "//tensorflow/c:__subpackages__",
+        "//tensorflow/core:__subpackages__",
+    ],
     deps = [
         ":error_codes_proto_cc",
         ":lib",
-- 
GitLab


From faa7db2df4b467cc97fab6184faa6de5ea88f7f0 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 6 Dec 2018 22:36:48 -0800
Subject: [PATCH 1480/1554] TF MirroredStrategy: Respect control_dependencies
 added in replica context during a merge_call. Also enable keras metrics
 correctness which was broken due to this issue.

PiperOrigin-RevId: 224463968
---
 .../contrib/distribute/python/keras_test.py   | 39 ++++++++++++-------
 .../python/distribute/mirrored_strategy.py    |  9 ++++-
 tensorflow/python/keras/engine/training.py    | 16 +-------
 3 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 771c48bdd8..d66fc52cf2 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1298,22 +1298,20 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       self.assertEqual(outs[1], 0.)
       self.assertEqual(outs[2], 0.)
 
-  # TODO(priyag): Add metrics correctness to this test to compare with and
-  # without distribution strategies.
   @combinations.generate(strategy_and_input_combinations())
   def test_correctness(self, distribution, use_numpy, use_validation_data):
 
     with self.cached_session():
-      tolerance = 1e-5
-      metrics = ['mse']
+      weights_tolerance = 1e-5
+      metrics_tolerance = 1e-4
 
       if isinstance(distribution, (mirrored_strategy.MirroredStrategy,
                                    mirrored_strategy.CoreMirroredStrategy)):
-        # TODO(b/119257215): use the default one once the flakyness is fixed.
-        tolerance = 1e-4
-        # TODO(b/120570676): Enable metrics check once the bug is fixed.
-        metrics = None
+        # TODO(b/119257215): Weights are not exactly the same, so use lower
+        # tolerance for now.
+        weights_tolerance = 1e-4
 
+      metrics = ['mse']
       keras.backend.set_image_data_format('channels_last')
       np.random.seed(_RANDOM_SEED)
       random_seed.set_random_seed(_RANDOM_SEED)
@@ -1383,22 +1381,33 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       # Verify that the weights, training history, eval results, predict outputs
       # are the same within some limits of tolerance.
       self.assertAllClose(
-          wts_with_ds, wts_without_ds, atol=tolerance, rtol=tolerance,
+          wts_with_ds,
+          wts_without_ds,
+          atol=weights_tolerance,
+          rtol=weights_tolerance,
           msg='Fail to assert weights after training.')
-
       self.assertAllClose(
-          eval_with_ds, eval_without_ds, atol=tolerance, rtol=tolerance,
+          eval_with_ds,
+          eval_without_ds,
+          atol=metrics_tolerance,
+          rtol=metrics_tolerance,
           msg='Fail to assert eval results.')
       self.assertAllClose(
-          predict_with_ds, predict_without_ds, atol=tolerance, rtol=tolerance,
+          predict_with_ds,
+          predict_without_ds,
+          atol=weights_tolerance,
+          rtol=weights_tolerance,
           msg='Fail to assert predict results.')
 
-      if not (isinstance(distribution, tpu_strategy.TPUStrategy)
-              and distribution.extended.steps_per_run > 1):
+      if not (isinstance(distribution, tpu_strategy.TPUStrategy) and
+              distribution.extended.steps_per_run > 1):
         # TODO(b/119894254): Enable this test for all cases once the underlying
         # bug is fixed.
         self.assertAllClose(
-            history_with_ds, history_without_ds, atol=tolerance, rtol=tolerance,
+            history_with_ds,
+            history_without_ds,
+            atol=metrics_tolerance,
+            rtol=metrics_tolerance,
             msg='Fail to assert training history.')
 
 
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 56948b2bcb..cb94dfcfbd 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -169,7 +169,12 @@ def _call_for_each_replica(distribution, fn, args, kwargs):
           # capture the name_scope from the first MRT and assume it is
           # the same for all other MRTs.
           mtt_captured_name_scope = threads[0].captured_name_scope
-          with ops.name_scope(mtt_captured_name_scope):
+          # Capture and merge the control dependencies from all the threads.
+          mtt_captured_control_deps = set()
+          for t in threads:
+            mtt_captured_control_deps.update(t.captured_control_deps)
+          with ops.name_scope(mtt_captured_name_scope),\
+              ops.control_dependencies(mtt_captured_control_deps):
             merge_result = threads[0].merge_fn(distribution, *merge_args,
                                                **merge_kwargs)
           for t in threads:
@@ -898,6 +903,8 @@ class MirroredReplicaContext(distribute_lib.ReplicaContext):
     # Adding a "/" at end lets us re-enter this scope later.
     if t.captured_name_scope:
       t.captured_name_scope += "/"
+
+    t.captured_control_deps = t.graph._current_control_dependencies()  # pylint: disable=protected-access
     t.has_paused.set()
     t.should_run.wait()
     t.should_run.clear()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 90f8a7b252..462694fda6 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -875,17 +875,11 @@ class Model(Network):
                                      [self.total_loss] + metrics_tensors)
 
   def _make_fit_function(self):
-    # TODO(psv/anjalisridhar): Remove updates after we fix b/118841692
-    # Stateful metrics updates
-    metric_updates = []
-    for m in self.metrics:
-      metric_updates += m.updates
-
     metrics_tensors = [
         self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:]
     ]
     self._make_train_function_helper(
-        '_fit_function', [self.total_loss] + metrics_tensors, metric_updates)
+        '_fit_function', [self.total_loss] + metrics_tensors)
 
   def _make_test_function_helper(self, fn_name, outputs, metric_updates=None):
     if not hasattr(self, fn_name):
@@ -918,17 +912,11 @@ class Model(Network):
                                     [self.total_loss] + metrics_tensors)
 
   def _make_eval_function(self):
-    # TODO(psv,anjalisridhar): Remove updates after we fix b/118841692
-    # Stateful metrics updates
-    metric_updates = []
-    for m in self.metrics:
-      metric_updates += m.updates
-
     metrics_tensors = [
         self._all_stateful_metrics_tensors[m] for m in self.metrics_names[1:]
     ]
     self._make_test_function_helper(
-        '_eval_function', [self.total_loss] + metrics_tensors, metric_updates)
+        '_eval_function', [self.total_loss] + metrics_tensors)
 
   def _make_predict_function(self):
     if not hasattr(self, 'predict_function'):
-- 
GitLab


From cd4cc57282ad9621c369f103e9b48961863b9c9d Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 7 Dec 2018 00:04:50 -0800
Subject: [PATCH 1481/1554] Fix a typo in estimator_training.

PiperOrigin-RevId: 224470630
---
 tensorflow/python/distribute/estimator_training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/estimator_training.py b/tensorflow/python/distribute/estimator_training.py
index 549fa8fb8a..7d5f231c37 100644
--- a/tensorflow/python/distribute/estimator_training.py
+++ b/tensorflow/python/distribute/estimator_training.py
@@ -354,7 +354,7 @@ def estimator_evaluate(estimator, evaluate_distributed_fn, hooks):
   if (estimator._config._distribute_coordinator_mode !=
       dc.CoordinatorMode.STANDALONE_CLIENT):
     raise ValueError('Only `STANDALONE_CLIENT` mode is supported when you call '
-                     '`Estimator.train`')
+                     '`Estimator.evaluate`')
 
   if estimator._config._eval_distribute.extended.experimental_between_graph:
     # TODO(yuefengz): remove this limitation once we figure out how to merge
-- 
GitLab


From 6bcc66c2d2b413a5d0487638c3d6b6bb63716e8f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 01:02:31 -0800
Subject: [PATCH 1482/1554] compat: Update forward compatibility horizon to
 2018-12-07

PiperOrigin-RevId: 224475966
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 269d2e286a..e66c29ae39 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 6)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 7)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From c2e2ecedfef715b8f756c5e2df4dab3b88531253 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 01:02:57 -0800
Subject: [PATCH 1483/1554] Internal change.

PiperOrigin-RevId: 224476105
---
 tensorflow/core/BUILD | 70 +++++++++++++++++++++++--------------------
 1 file changed, 37 insertions(+), 33 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 328bfe474d..66714235b5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1682,6 +1682,9 @@ filegroup(
 # operators, use :android_tensorflow_lib if you want full operator
 # support.
 #
+# If you just need TensorFlow types, e.g. Tensors, use
+# :android_tensorflow_lib_lite_no_runtime.
+#
 # Compiles to a trivial library on non-Android to prevent irrelevant
 # build errors. If not building this as part of an android_binary,
 # a command such as the following must be used:
@@ -1692,7 +1695,33 @@ filegroup(
 cc_library(
     name = "android_tensorflow_lib_lite",
     srcs = if_android(["//tensorflow/core:android_srcs"]),
-    copts = tf_copts(android_optimization_level_override = None),
+    copts = tf_copts(android_optimization_level_override = None) + [
+        "-DSUPPORT_SELECTIVE_REGISTRATION",
+    ],
+    linkopts = ["-lz"],
+    tags = [
+        "manual",
+        "notap",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":mobile_additional_lib_deps",
+        ":protos_all_cc_impl",
+        ":stats_calculator_portable",
+        "//third_party/eigen3",
+        "@double_conversion//:double-conversion",
+        "@nsync//:nsync_cpp",
+        "@protobuf_archive//:protobuf",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "android_tensorflow_lib_lite_nortti",
+    srcs = if_android(["//tensorflow/core:android_srcs"]),
+    copts = tf_copts(android_optimization_level_override = None) + [
+        "-DSUPPORT_SELECTIVE_REGISTRATION",
+    ] + tf_opts_nortti_if_android(),
     linkopts = ["-lz"],
     tags = [
         "manual",
@@ -1804,46 +1833,21 @@ cc_library(
 # Does not contain operators. In contrast to android_tensorflow_lib_lite,
 # this links in framework support for all types, relying on selective
 # registration of ops to prune code size.
-cc_library(
+#
+# TODO(gonnet): Move all users of these aliases to the corresponding
+#     :android_tensorflow_lib_lite* targets and remove.
+alias(
     name = "android_tensorflow_lib_selective_registration",
-    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
-    copts = tf_copts(android_optimization_level_override = None) + [
-        "-DSUPPORT_SELECTIVE_REGISTRATION",
-    ],
-    linkopts = if_android(["-lz"]),
-    tags = [
-        "manual",
-        "notap",
-    ],
+    actual = ":android_tensorflow_lib_lite",
     visibility = ["//visibility:public"],
-    deps = [
-        ":protos_all_cc_impl",
-        "@com_google_absl//absl/container:flat_hash_set",
-        "@protobuf_archive//:protobuf",
-    ],
-    alwayslink = 1,
 )
 
 # Android library for use with the SELECTIVE_REGISTRATION feature with
 # no proto_rtti.
-cc_library(
+alias(
     name = "android_tensorflow_lib_selective_registration_nortti",
-    srcs = if_android(["//tensorflow/core:android_srcs_only_runtime"]),
-    copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
-        "-DSUPPORT_SELECTIVE_REGISTRATION",
-    ],
-    linkopts = if_android(["-lz"]),
-    tags = [
-        "manual",
-        "notap",
-    ],
+    actual = ":android_tensorflow_lib_lite_nortti",
     visibility = ["//visibility:public"],
-    deps = [
-        ":protos_all_cc_impl",
-        "@com_google_absl//absl/container:flat_hash_set",
-        "@protobuf_archive//:protobuf",
-    ],
-    alwayslink = 1,
 )
 
 filegroup(
-- 
GitLab


From 3bd4a240020d1c62d84c1c940e8b1f5b3d838d3d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 7 Dec 2018 03:08:17 -0800
Subject: [PATCH 1484/1554] Enable @run_v1_only to be applied on a class

Some entire suite of tests do not work on v1. Rather
than annotating the individual test cases, it makes more
sense to annotate the entire class. Note that we do not
intend this for the @run_deprecated_v1 annotation.

PiperOrigin-RevId: 224489413
---
 tensorflow/python/framework/test_util.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index fc1a5fbe85..4bc9bf01d2 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -494,7 +494,8 @@ def with_control_flow_v2(cls):
     return cls
 
   for name, value in cls.__dict__.copy().items():
-    if (callable(value) and name.startswith("test") and
+    if (callable(value) and
+        name.startswith(unittest.TestLoader.testMethodPrefix) and
         not getattr(value, "_disable_control_flow_v2", False)):
       setattr(cls, name + "WithControlFlowV2", enable_control_flow_v2(value))
   return cls
@@ -893,8 +894,10 @@ def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
   for name, value in cls.__dict__.copy().items():
-    if callable(value) and name.startswith("test") and not (
-        name.startswith("testSkipEager") or name.startswith("test_skip_eager")):
+    if (callable(value) and
+        name.startswith(unittest.TestLoader.testMethodPrefix) and
+        not (name.startswith("testSkipEager")
+             or name.startswith("test_skip_eager"))):
       setattr(cls, name, base_decorator(value))
   return cls
 
@@ -1059,7 +1062,16 @@ def run_v1_only(reason, func=None):
 
   def decorator(f):
     if tf_inspect.isclass(f):
-      raise ValueError("`run_v1_only` only supports test methods.")
+      setup = f.__dict__.get("setUp")
+      if setup is not None:
+        setattr(f, "setUp", decorator(setup))
+
+      for name, value in f.__dict__.copy().items():
+        if (callable(value) and
+            name.startswith(unittest.TestLoader.testMethodPrefix)):
+          setattr(f, name, decorator(value))
+
+      return f
 
     def decorated(self, *args, **kwargs):
       if tf2.enabled():
-- 
GitLab


From 0d3b58cfe91c6b865a14701345d7a84ce949c0e3 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 7 Dec 2018 04:17:58 -0800
Subject: [PATCH 1485/1554] Disable some tests using placeholders when running
 in v2

In order to do so we introduce the concept of a FakeEagerSession
which allows tests to run sess.run() with an empty feed_dict.

PiperOrigin-RevId: 224495103
---
 .../python/kernel_tests/lanczos_test.py       |  4 +-
 .../python/kernel_tests/least_squares_test.py |  4 +-
 .../kernel_tests/linear_equations_test.py     |  4 +-
 tensorflow/python/framework/test_util.py      | 59 ++++++++++++++++++-
 .../kernel_tests/batch_matmul_op_test.py      |  1 +
 .../kernel_tests/matrix_solve_ls_op_test.py   | 16 +++--
 tensorflow/python/kernel_tests/qr_op_test.py  |  8 ++-
 tensorflow/python/kernel_tests/svd_op_test.py |  8 ++-
 8 files changed, 89 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py b/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py
index 8fcd7aeef6..f31bdbd399 100644
--- a/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py
+++ b/tensorflow/contrib/solvers/python/kernel_tests/lanczos_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.contrib.solvers.python.ops import lanczos
 from tensorflow.contrib.solvers.python.ops import util
 from tensorflow.python.framework import constant_op
@@ -80,7 +81,8 @@ if __name__ == "__main__":
     for shape in [[4, 4], [7, 4], [5, 8]]:
       for orthogonalize in True, False:
         for steps in range(1, min(shape) + 1):
-          for use_static_shape in True, False:
+          # TF2 does not support placeholders so we skip it
+          for use_static_shape in set([True, tf2.enabled()]):
             arg_string = "%s_%s_%s_%s_staticshape_%s" % (
                 dtype.__name__, "_".join(map(str, shape)), orthogonalize, steps,
                 use_static_shape)
diff --git a/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py b/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py
index 2a9100903a..841a41a233 100644
--- a/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py
+++ b/tensorflow/contrib/solvers/python/kernel_tests/least_squares_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.contrib.solvers.python.ops import least_squares
 from tensorflow.contrib.solvers.python.ops import util
 from tensorflow.python.framework import constant_op
@@ -76,7 +77,8 @@ def _get_least_squares_tests(dtype_, use_static_shape_, shape_):
 if __name__ == "__main__":
   for dtype in np.float32, np.float64:
     for shape in [[4, 4], [8, 5], [3, 7]]:
-      for use_static_shape in True, False:
+      # TF2 does not support placeholders under eager so we skip it
+      for use_static_shape in set([True, tf2.enabled()]):
         arg_string = "%s_%s_staticshape_%s" % (dtype.__name__,
                                                "_".join(map(str, shape)),
                                                use_static_shape)
diff --git a/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py b/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py
index a0e6eb87bc..10807f7a80 100644
--- a/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py
+++ b/tensorflow/contrib/solvers/python/kernel_tests/linear_equations_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.contrib.solvers.python.ops import linear_equations
 from tensorflow.contrib.solvers.python.ops import util
 from tensorflow.python.framework import constant_op
@@ -113,7 +114,8 @@ def _get_linear_equations_tests(dtype_, use_static_shape_, shape_):
 if __name__ == "__main__":
   for dtype in np.float32, np.float64:
     for size in 1, 4, 10:
-      for use_static_shape in True, False:
+      # TF2 does not support placeholders under eager so we skip it
+      for use_static_shape in set([True, tf2.enabled()]):
         shape = [size, size]
         arg_string = "%s_%s_staticshape_%s" % (dtype.__name__, size,
                                                use_static_shape)
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 4bc9bf01d2..b0c3c9b506 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1277,6 +1277,63 @@ class CapturedWrites(object):
     return output_data
 
 
+class FakeEagerSession(object):
+  """Fake session so tests that conditionally use placeholders can use eager.
+
+  There are a number of tests that conditionally use placeholders for shape
+  inference. The pattern is demonstrated here:
+
+  ```python
+  with self.cached_session() as sess:
+    if static_shape:
+      y = math_ops.matmul(x, ...)
+      feed_dict = {}
+    else:
+      x_ph = array_ops.placeholder(...)
+      y = math_ops.matmul(x_ph, ...)
+      feed_dict = {x_ph: x}
+    val = sess.run(y, feed_dict=feed_dict)
+  ```
+
+  Since the feed_dict is empty when not using placeholders we should be able to
+  call self.evaluate(), however this requires rewriting the test case.
+  This class shold be considered a stop-gap solution to get tests running with
+  eager with minimal changes to the actual test.
+  """
+
+  def __init__(self, test_case):
+    self._test_case = test_case
+
+  def run(self, fetches, *args, **kwargs):
+    """Evalaute `fetches`.
+
+    Fail if additional args are specified.
+
+    Args:
+      fetches: A Tensor or a nested list/tuple of Tensors.
+      *args: Positional arguments
+      **kwargs: Keyword arguments
+
+    Raises:
+      RuntimeError: If args or kwargs are specified.
+
+    Returns:
+      Tensors as numpy values.
+    """
+    feed_dict = kwargs.pop("feed_dict", {})
+    if feed_dict:
+      raise RuntimeError(
+          "feed_dict is not supported when eager execution is enabled "
+          "(in this case, sess.run(t) is shorthand for t.numpy()")
+
+    if args or kwargs:
+      raise RuntimeError(
+          "Optional args are not supported when eager execution is enabled "
+          "(in this case, sess.run(t) is shorthand for t.numpy()")
+
+    return self._test_case.evaluate(fetches)
+
+
 class ErrorLoggingSession(session.Session):
   """Wrapper around a Session that logs errors in run().
   """
@@ -1584,7 +1641,7 @@ class TensorFlowTestCase(googletest.TestCase):
       the graph building and execution code in a test case.
     """
     if context.executing_eagerly():
-      yield None
+      yield FakeEagerSession(self)
     else:
       sess = self._get_cached_session(
           graph, config, force_gpu, crash_if_inconsistent_args=True)
diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
index 931bb41424..f2f0291b89 100644
--- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
@@ -198,6 +198,7 @@ if __name__ == "__main__":
     for adjoint_a_ in False, True:
       for adjoint_b_ in False, True:
         name = "%s_%s_%s" % (dtype_.__name__, adjoint_a_, adjoint_b_)
+        # TF2 does not support placeholders under eager so we skip it
         for use_static_shape_ in set([True, tf2.enabled()]):
           setattr(BatchMatmulOpTest,
                   "testBatchMatmulOp_" + name + ("_%s" % use_static_shape_),
diff --git a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
index a6f5da9d3d..42fd95d311 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -150,13 +151,17 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
     empty1 = np.empty([0, 2])
     for fast in [True, False]:
       with self.cached_session(use_gpu=True):
-        tf_ans = linalg_ops.matrix_solve_ls(empty0, empty0, fast=fast).eval()
+        tf_ans = self.evaluate(
+            linalg_ops.matrix_solve_ls(empty0, empty0, fast=fast))
         self.assertEqual(tf_ans.shape, (0, 0))
-        tf_ans = linalg_ops.matrix_solve_ls(empty0, full, fast=fast).eval()
+        tf_ans = self.evaluate(
+            linalg_ops.matrix_solve_ls(empty0, full, fast=fast))
         self.assertEqual(tf_ans.shape, (0, 2))
-        tf_ans = linalg_ops.matrix_solve_ls(full, empty0, fast=fast).eval()
+        tf_ans = self.evaluate(
+            linalg_ops.matrix_solve_ls(full, empty0, fast=fast))
         self.assertEqual(tf_ans.shape, (2, 0))
-        tf_ans = linalg_ops.matrix_solve_ls(empty1, empty1, fast=fast).eval()
+        tf_ans = self.evaluate(
+            linalg_ops.matrix_solve_ls(empty1, empty1, fast=fast))
         self.assertEqual(tf_ans.shape, (2, 2))
 
   @test_util.run_deprecated_v1
@@ -350,7 +355,8 @@ class MatrixSolveLsBenchmark(test_lib.Benchmark):
 
 if __name__ == "__main__":
   for dtype_ in [np.float32, np.float64, np.complex64, np.complex128]:
-    for use_placeholder_ in [True, False]:
+    # TF2 does not support placeholders under eager so we skip it
+    for use_placeholder_ in set([False, not tf2.enabled()]):
       for fast_ in [True, False]:
         l2_regularizers = [0] if dtype_ == np.complex128 else [0, 0.1]
         for l2_regularizer_ in l2_regularizers:
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 0f2537b371..5e9767f20c 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
@@ -102,7 +103,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
       tol = 1e-14
     # Tests that a ~= q*r.
     a_recon = math_ops.matmul(q, r)
-    self.assertAllClose(a_recon.eval(), a, rtol=tol, atol=tol)
+    self.assertAllClose(a_recon, a, rtol=tol, atol=tol)
 
   def CheckUnitary(self, x):
     # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity.
@@ -112,7 +113,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
       tol = 1e-5
     else:
       tol = 1e-14
-    self.assertAllClose(identity.eval(), self.evaluate(xx), atol=tol)
+    self.assertAllClose(identity, xx, atol=tol)
 
   def Test(self):
     np.random.seed(1)
@@ -202,7 +203,8 @@ if __name__ == "__main__":
       for cols in 1, 2, 5, 10, 32, 100:
         for full_matrices in False, True:
           for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10):
-            for use_static_shape in True, False:
+            # TF2 does not support placeholders under eager so we skip it
+            for use_static_shape in set([True, tf2.enabled()]):
               shape = batch_dims + (rows, cols)
               name = "%s_%s_full_%s_static_%s" % (dtype.__name__,
                                                   "_".join(map(str, shape)),
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index 97a280ef51..94155fd117 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
@@ -117,13 +118,13 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
         diag_s = array_ops.concat([diag_s, zeros], a.ndim - 1)
     a_recon = math_ops.matmul(u, diag_s)
     a_recon = math_ops.matmul(a_recon, v, adjoint_b=True)
-    self.assertAllClose(a_recon.eval(), a, rtol=tol, atol=tol)
+    self.assertAllClose(a_recon, a, rtol=tol, atol=tol)
 
   def CheckUnitary(self, x, tol):
     # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity.
     xx = math_ops.matmul(x, x, adjoint_a=True)
     identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0)
-    self.assertAllClose(identity.eval(), self.evaluate(xx), atol=tol)
+    self.assertAllClose(identity, xx, atol=tol)
 
   def Test(self):
     is_complex = dtype_ in (np.complex64, np.complex128)
@@ -263,7 +264,8 @@ if __name__ == "__main__":
           for cols in 1, 2, 5, 10, 32, 100:
             for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10):
               shape = batch_dims + (rows, cols)
-              for use_static_shape in True, False:
+              # TF2 does not support placeholders under eager so we skip it
+              for use_static_shape in set([True, tf2.enabled()]):
                 name = "%s_%s_static_shape_%s__compute_uv_%s_full_%s" % (
                     dtype.__name__, "_".join(map(str, shape)), use_static_shape,
                     compute_uv, full_matrices)
-- 
GitLab


From 9109842678c385b4ab31825c53bf4373fcadf15e Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Fri, 7 Dec 2018 07:42:26 -0800
Subject: [PATCH 1486/1554] Add time functions to TF_Env

PiperOrigin-RevId: 224513745
---
 tensorflow/c/env.cc      | 14 ++++++++++++++
 tensorflow/c/env.h       |  9 +++++++++
 tensorflow/c/env_test.cc |  6 ++++++
 3 files changed, 29 insertions(+)

diff --git a/tensorflow/c/env.cc b/tensorflow/c/env.cc
index fcff1e413d..07b9e8b940 100644
--- a/tensorflow/c/env.cc
+++ b/tensorflow/c/env.cc
@@ -145,3 +145,17 @@ TF_StringStream* TF_GetLocalTempDirectories() {
   list->position = 0;
   return list;
 }
+
+TF_CAPI_EXPORT extern uint64_t TF_NowNanos(void) {
+  return ::tensorflow::Env::Default()->NowNanos();
+}
+
+// Returns the number of microseconds since the Unix epoch.
+TF_CAPI_EXPORT extern uint64_t TF_NowMicros(void) {
+  return ::tensorflow::Env::Default()->NowMicros();
+}
+
+// Returns the number of seconds since the Unix epoch.
+TF_CAPI_EXPORT extern uint64_t TF_NowSeconds(void) {
+  return ::tensorflow::Env::Default()->NowSeconds();
+}
diff --git a/tensorflow/c/env.h b/tensorflow/c/env.h
index 32d3f1f745..9d27c5da37 100644
--- a/tensorflow/c/env.h
+++ b/tensorflow/c/env.h
@@ -141,6 +141,15 @@ TF_CAPI_EXPORT extern TF_StringStream* TF_GetChildren(const char* filename,
 // The caller is responsible for freeing the list (see TF_StringStreamDone).
 TF_CAPI_EXPORT extern TF_StringStream* TF_GetLocalTempDirectories(void);
 
+// Returns the number of nanoseconds since the Unix epoch.
+TF_CAPI_EXPORT extern uint64_t TF_NowNanos(void);
+
+// Returns the number of microseconds since the Unix epoch.
+TF_CAPI_EXPORT extern uint64_t TF_NowMicros(void);
+
+// Returns the number of seconds since the Unix epoch.
+TF_CAPI_EXPORT extern uint64_t TF_NowSeconds(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/tensorflow/c/env_test.cc b/tensorflow/c/env_test.cc
index 08e803f12f..e2206c6bef 100644
--- a/tensorflow/c/env_test.cc
+++ b/tensorflow/c/env_test.cc
@@ -92,3 +92,9 @@ TEST(TestEnv, TestDirHandling) {
 
   TF_StringStreamDone(tempdirs);
 }
+
+TEST(TestEnv, TestTimeFunctions) {
+  ASSERT_GE(TF_NowSeconds(), 946684800);  // Midnight Jan 1, 2000
+  ASSERT_GE(TF_NowMicros(), 946684800 * 1e6);
+  ASSERT_GE(TF_NowNanos(), 946684800 * 1e9);
+}
-- 
GitLab


From 840be3029ce5ee97b345a8c317a76a68eb4e5374 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Fri, 7 Dec 2018 08:21:42 -0800
Subject: [PATCH 1487/1554] [Distributed Runtime] Mark SessionOptions as const.

#cleanup

PiperOrigin-RevId: 224518435
---
 tensorflow/core/distributed_runtime/rpc/grpc_session.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.h b/tensorflow/core/distributed_runtime/rpc/grpc_session.h
index e00cf97e38..a3ed3ec736 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.h
@@ -111,7 +111,7 @@ class GrpcSession : public Session {
       LOCKS_EXCLUDED(mu_);
 
  private:
-  SessionOptions options_;
+  const SessionOptions options_;
   std::unique_ptr<MasterInterface> master_;
   mutex mu_;
 
-- 
GitLab


From 989e78c412a7e0f5361d4d7dfdfb230c8136e749 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Fri, 7 Dec 2018 09:32:55 -0800
Subject: [PATCH 1488/1554] [TF Keras] keras.Flatten()(tensor : shaped
 (batch,)) -> tensor : shaped (batch, 1)

This provides a consistent API for users who have scalar "channels".

PiperOrigin-RevId: 224528301
---
 .../features/python/conditioning_utils_impl.py     |  3 +++
 .../features/python/conditioning_utils_test.py     |  2 +-
 .../contrib/layers/python/layers/layers_test.py    | 13 ++++++-------
 tensorflow/python/keras/engine/training_utils.py   | 14 +++++++++++---
 tensorflow/python/keras/layers/core.py             | 13 ++++++++++---
 tensorflow/python/keras/layers/core_test.py        | 14 ++++++++++++++
 tensorflow/python/keras/testing_utils.py           |  6 +++++-
 tensorflow/python/layers/core_test.py              |  9 ++++++---
 8 files changed, 56 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py b/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py
index e2594faf85..364fa4eb46 100644
--- a/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py
+++ b/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py
@@ -64,6 +64,9 @@ def condition_tensor(tensor, conditioning):
   """
   tensor.shape[1:].assert_is_fully_defined()
   num_features = tensor.shape[1:].num_elements()
+  if conditioning.shape.ndims < 2:
+    raise ValueError('conditioning must be at least 2D, but saw shape: %s'
+                     % conditioning.shape)
 
   mapped_conditioning = layers.linear(
       layers.flatten(conditioning), num_features)
diff --git a/tensorflow/contrib/gan/python/features/python/conditioning_utils_test.py b/tensorflow/contrib/gan/python/features/python/conditioning_utils_test.py
index 0aad769793..f5c7d53cf2 100644
--- a/tensorflow/contrib/gan/python/features/python/conditioning_utils_test.py
+++ b/tensorflow/contrib/gan/python/features/python/conditioning_utils_test.py
@@ -45,7 +45,7 @@ class ConditioningUtilsTest(test.TestCase):
           array_ops.placeholder(dtypes.float32, (5, None)),
           array_ops.placeholder(dtypes.float32, (5, 1)))
 
-    with self.assertRaisesRegexp(ValueError, 'expected min_ndim=2'):
+    with self.assertRaisesRegexp(ValueError, 'at least 2D'):
       conditioning_utils.condition_tensor(
           array_ops.placeholder(dtypes.float32, (5, 2)),
           array_ops.placeholder(dtypes.float32, (5)))
diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py
index 0a4d2c6d4c..d791418c9d 100644
--- a/tensorflow/contrib/layers/python/layers/layers_test.py
+++ b/tensorflow/contrib/layers/python/layers/layers_test.py
@@ -1459,13 +1459,6 @@ class DropoutTest(test.TestCase):
 
 class FlattenTest(test.TestCase):
 
-  def testInvalidRank(self):
-    with ops.Graph().as_default() as g, self.session(g):
-      inputs = array_ops.placeholder(dtype=dtypes.float32)
-      inputs.set_shape(tensor_shape.TensorShape((5,)))
-      with self.assertRaisesRegexp(ValueError, 'incompatible with the layer'):
-        _layers.flatten(inputs)
-
   def testUnknownLastDim(self):
     with ops.Graph().as_default() as g, self.session(g):
       inputs = array_ops.placeholder(dtype=dtypes.float32)
@@ -1502,6 +1495,12 @@ class FlattenTest(test.TestCase):
                        images.get_shape().num_elements())
       self.assertEqual(output.get_shape()[0], images.get_shape()[0])
 
+  def testFlatten0D(self):
+    with self.cached_session():
+      scalars = random_ops.random_uniform((5,), seed=1, name='scalars')
+      output = _layers.flatten(scalars)
+      self.assertEqual(output.shape, (5, 1))
+
   def testFlattenBatchSize(self):
     height, width = 3, 3
     with self.cached_session() as sess:
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 9301302f4a..01a09eb031 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -232,10 +232,14 @@ def check_num_samples(ins,
   return None  # Edge case where ins == [static_learning_phase]
 
 
-def standardize_single_array(x):
+def standardize_single_array(x, expected_shape=None):
+  """Expand data of shape (x,) to (x, 1), unless len(expected_shape)==1."""
   if x is None:
     return None
-  if x.shape is not None and len(x.shape) == 1:
+
+  if (x.shape is not None
+      and len(x.shape) == 1
+      and (expected_shape is None or len(expected_shape) != 1)):
     if tensor_util.is_tensor(x):
       x = array_ops.expand_dims(x, axis=1)
     else:
@@ -301,7 +305,11 @@ def standardize_input_data(data,
   else:
     data = data.values if data.__class__.__name__ == 'DataFrame' else data
     data = [data]
-  data = [standardize_single_array(x) for x in data]
+  if shapes is not None:
+    data = [standardize_single_array(x, shape)
+            for (x, shape) in zip(data, shapes)]
+  else:
+    data = [standardize_single_array(x) for x in data]
 
   if len(data) != len(names):
     if data and hasattr(data[0], 'shape'):
diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index 56dd70558c..854774c569 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -506,6 +506,9 @@ class Permute(Layer):
 class Flatten(Layer):
   """Flattens the input. Does not affect the batch size.
 
+  If inputs are shaped `(batch,)` without a channel dimension, then flattening
+  adds an extra channel dimension and output shapes are `(batch, 1)`.
+
   Arguments:
       data_format: A string,
           one of `channels_last` (default) or `channels_first`.
@@ -534,23 +537,27 @@ class Flatten(Layer):
   def __init__(self, data_format=None, **kwargs):
     super(Flatten, self).__init__(**kwargs)
     self.data_format = conv_utils.normalize_data_format(data_format)
-    self.input_spec = InputSpec(min_ndim=2)
+    self.input_spec = InputSpec(min_ndim=1)
 
   def call(self, inputs):
-    if self.data_format == 'channels_first':
+    if (self.data_format == 'channels_first'
+        and K.ndim(inputs) is not None and K.ndim(inputs) > 1):
       permutation = [0]
       permutation.extend([i for i in
                           range(2, K.ndim(inputs))])
       permutation.append(1)
       inputs = array_ops.transpose(inputs, perm=permutation)
 
-    outputs = array_ops.reshape(inputs, (array_ops.shape(inputs)[0], -1))
+    outputs = array_ops.reshape(
+        inputs, (inputs.shape[0].value or array_ops.shape(inputs)[0], -1))
     if not context.executing_eagerly():
       outputs.set_shape(self.compute_output_shape(inputs.get_shape()))
     return outputs
 
   def compute_output_shape(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape).as_list()
+    if not input_shape:
+      output_shape = tensor_shape.TensorShape([1])
     output_shape = [input_shape[0]]
     if all(input_shape[1:]):
       output_shape += [np.prod(input_shape[1:])]
diff --git a/tensorflow/python/keras/layers/core_test.py b/tensorflow/python/keras/layers/core_test.py
index aad6ab8171..f138adf760 100644
--- a/tensorflow/python/keras/layers/core_test.py
+++ b/tensorflow/python/keras/layers/core_test.py
@@ -149,6 +149,20 @@ class CoreLayersTest(test.TestCase):
         np.transpose(inputs, (0, 2, 3, 1)), (-1, 5 * 5 * 3))
     self.assertAllClose(outputs, target_outputs)
 
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_flatten_scalar_channels(self):
+    testing_utils.layer_test(
+        keras.layers.Flatten, kwargs={}, input_shape=(3,))
+
+    # Test channels_first
+    inputs = np.random.random((10,)).astype('float32')
+    outputs = testing_utils.layer_test(
+        keras.layers.Flatten,
+        kwargs={'data_format': 'channels_first'},
+        input_data=inputs)
+    target_outputs = np.expand_dims(inputs, -1)
+    self.assertAllClose(outputs, target_outputs)
+
   @tf_test_util.run_in_graph_and_eager_modes
   def test_repeat_vector(self):
     testing_utils.layer_test(
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index 42f8f10fca..fd062b0ab3 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -77,9 +77,13 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
   Returns:
     The output data (Numpy array) returned by the layer, for additional
     checks to be done by the calling code.
+
+  Raises:
+    ValueError: if `input_shape is None`.
   """
   if input_data is None:
-    assert input_shape
+    if input_shape is None:
+      raise ValueError('input_shape is None')
     if not input_dtype:
       input_dtype = 'float32'
     input_data_shape = list(input_shape)
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index cf6f0fbb70..b40a268238 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -533,10 +533,13 @@ class FlattenTest(test.TestCase):
     self.assertEqual(y.get_shape().as_list(), [None, 6])
 
   @test_util.run_deprecated_v1
-  def testFlattenValueError(self):
+  def testFlatten0D(self):
     x = array_ops.placeholder(shape=(None,), dtype='float32')
-    with self.assertRaises(ValueError):
-      core_layers.Flatten()(x)
+    y = core_layers.Flatten()(x)
+    with self.cached_session() as sess:
+      np_output = sess.run(y, feed_dict={x: np.zeros((5,))})
+    self.assertEqual(list(np_output.shape), [5, 1])
+    self.assertEqual(y.shape.as_list(), [None, 1])
 
   @test_util.run_deprecated_v1
   def testFlattenUnknownAxes(self):
-- 
GitLab


From 8d3be0f555b14013d4d93336cd5206153a6fa63b Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 7 Dec 2018 09:45:24 -0800
Subject: [PATCH 1489/1554] Add default value for topdown aerg of
 io.gfile.walk. The default exists in Python's os.walk and in the v1 gfile
 too.

PiperOrigin-RevId: 224530098
---
 tensorflow/python/lib/io/file_io.py                      | 2 +-
 tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt | 2 +-
 tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/lib/io/file_io.py b/tensorflow/python/lib/io/file_io.py
index 4caa5750bf..ee55d89bff 100644
--- a/tensorflow/python/lib/io/file_io.py
+++ b/tensorflow/python/lib/io/file_io.py
@@ -671,7 +671,7 @@ def walk(top, in_order=True):
 
 
 @tf_export("io.gfile.walk")
-def walk_v2(top, topdown, onerror=None):
+def walk_v2(top, topdown=True, onerror=None):
   """Recursive directory tree generator for directories.
 
   Args:
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
index 93d9b0fd75..cfa3372b12 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.io.gfile.pbtxt
@@ -46,6 +46,6 @@ tf_module {
   }
   member_method {
     name: "walk"
-    argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
index 93d9b0fd75..cfa3372b12 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.io.gfile.pbtxt
@@ -46,6 +46,6 @@ tf_module {
   }
   member_method {
     name: "walk"
-    argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'top\', \'topdown\', \'onerror\'], varargs=None, keywords=None, defaults=[\'True\', \'None\'], "
   }
 }
-- 
GitLab


From 0d9781711b16a897a220e33f5f6c60a038fb5bac Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Fri, 7 Dec 2018 09:53:37 -0800
Subject: [PATCH 1490/1554] Creating a wrapper around DatasetVariantWrapper's
 that can be passed from CPU to GPU while keeping everything in host memory.
 This is necessary for when we move to creating datasets on the go and for
 transformations such as copy_to_device we need to ferry the
 DatasetVariantWrapper.

PiperOrigin-RevId: 224531292
---
 tensorflow/compiler/jit/xla_device_ops.h      |   2 +
 .../api_def_UnwrapDatasetVariant.pbtxt        |   4 +
 .../base_api/api_def_WrapDatasetVariant.pbtxt |   4 +
 tensorflow/core/framework/dataset.cc          | 109 ++++++++++++++++++
 tensorflow/core/kernels/function_ops.cc       |   3 +
 tensorflow/core/ops/dataset_ops.cc            |  10 ++
 .../data/experimental/kernel_tests/BUILD      |  11 ++
 .../kernel_tests/wrap_unwrap_test.py          |  69 +++++++++++
 8 files changed, 212 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_UnwrapDatasetVariant.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WrapDatasetVariant.pbtxt
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py

diff --git a/tensorflow/compiler/jit/xla_device_ops.h b/tensorflow/compiler/jit/xla_device_ops.h
index adf0f994b8..927f983ba9 100644
--- a/tensorflow/compiler/jit/xla_device_ops.h
+++ b/tensorflow/compiler/jit/xla_device_ops.h
@@ -203,6 +203,8 @@ class XlaAssignVariableOp : public OpKernel {
                               .HostMemory("output")                            \
                               .TypeConstraint<ResourceHandle>("T"),            \
                           ArgOp);                                              \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name(kArgOp).Device(DEVICE).TypeConstraint<Variant>("T"), ArgOp);        \
                                                                                \
   REGISTER_KERNEL_BUILDER(Name(kRetOp)                                         \
                               .Device(DEVICE)                                  \
diff --git a/tensorflow/core/api_def/base_api/api_def_UnwrapDatasetVariant.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnwrapDatasetVariant.pbtxt
new file mode 100644
index 0000000000..7b3f88a1da
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_UnwrapDatasetVariant.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "UnwrapDatasetVariant"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WrapDatasetVariant.pbtxt b/tensorflow/core/api_def/base_api/api_def_WrapDatasetVariant.pbtxt
new file mode 100644
index 0000000000..40f5c7a0d2
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WrapDatasetVariant.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WrapDatasetVariant"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/framework/dataset.cc b/tensorflow/core/framework/dataset.cc
index 0e48932da7..6e21433271 100644
--- a/tensorflow/core/framework/dataset.cc
+++ b/tensorflow/core/framework/dataset.cc
@@ -17,6 +17,8 @@ limitations under the License.
 
 #include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/variant_encode_decode.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/platform/mutex.h"
@@ -74,6 +76,113 @@ class DatasetVariantWrapper {
   DatasetBase* const dataset_;  // Owns one reference.
 };
 
+const char kWrappedDatasetVariantTypeName[] =
+    "tensorflow::data::WrappedDatasetVariant";
+
+class WrappedDatasetVariantWrapper {
+ public:
+  WrappedDatasetVariantWrapper() {}
+
+  explicit WrappedDatasetVariantWrapper(const Tensor& ds_tensor)
+      : ds_tensor_(ds_tensor) {}
+
+  Tensor get() const { return ds_tensor_; }
+
+  string TypeName() const { return "tensorflow::WrappedDatasetVariantWrapper"; }
+
+  string DebugString() const {
+    return "tensorflow::WrappedDatasetVariantWrapper::DebugString";
+  }
+
+  void Encode(VariantTensorData* data) const {
+    *(data->add_tensors()) = ds_tensor_;
+  }
+
+  bool Decode(const VariantTensorData& data) {
+    ds_tensor_ = data.tensors(0);
+    return true;
+  }
+
+ private:
+  Tensor ds_tensor_;
+};
+
+class WrapDatasetVariantOp : public OpKernel {
+ public:
+  explicit WrapDatasetVariantOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& tensor = ctx->input(0);
+    OP_REQUIRES(ctx,
+                tensor.dtype() == DT_VARIANT &&
+                    TensorShapeUtils::IsScalar(tensor.shape()),
+                errors::InvalidArgument(
+                    "Dataset tensor must be a scalar of dtype DT_VARIANT."));
+    DatasetBase* unused;
+    OP_REQUIRES_OK(ctx, GetDatasetFromVariantTensor(tensor, &unused));
+    Tensor* output = nullptr;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &output));
+    output->scalar<Variant>()() = WrappedDatasetVariantWrapper(tensor);
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("WrapDatasetVariant").Device(DEVICE_CPU),
+                        WrapDatasetVariantOp);
+REGISTER_KERNEL_BUILDER(Name("WrapDatasetVariant")
+                            .HostMemory("input_handle")
+                            .HostMemory("output_handle")
+                            .Device(DEVICE_GPU),
+                        WrapDatasetVariantOp);
+
+class UnwrapDatasetVariantOp : public OpKernel {
+ public:
+  explicit UnwrapDatasetVariantOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& tensor = ctx->input(0);
+    OP_REQUIRES(ctx,
+                tensor.dtype() == DT_VARIANT &&
+                    TensorShapeUtils::IsScalar(tensor.shape()),
+                errors::InvalidArgument(
+                    "Dataset tensor must be a scalar of dtype DT_VARIANT."));
+    Variant variant = tensor.scalar<Variant>()();
+    const WrappedDatasetVariantWrapper* wrapper =
+        variant.get<WrappedDatasetVariantWrapper>();
+    OP_REQUIRES(ctx, wrapper != nullptr,
+                errors::InvalidArgument(
+                    "Tensor must be a WrappedDataset variant object."));
+    Tensor ds_tensor = wrapper->get();
+    OP_REQUIRES_OK(ctx, ctx->set_output("output_handle", ds_tensor));
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("UnwrapDatasetVariant").Device(DEVICE_CPU),
+                        UnwrapDatasetVariantOp);
+REGISTER_KERNEL_BUILDER(Name("UnwrapDatasetVariant")
+                            .HostMemory("input_handle")
+                            .HostMemory("output_handle")
+                            .Device(DEVICE_GPU),
+                        UnwrapDatasetVariantOp);
+
+static Status WrappedDatasetVariantDeviceCopy(
+    const WrappedDatasetVariantWrapper& from, WrappedDatasetVariantWrapper* to,
+    const UnaryVariantOpRegistry::AsyncTensorDeviceCopyFn& copy) {
+  *to = WrappedDatasetVariantWrapper(from);
+  return Status::OK();
+}
+
+#define REGISTER_OPTIONAL_COPY(DIRECTION)               \
+  INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION( \
+      WrappedDatasetVariantWrapper, DIRECTION,          \
+      WrappedDatasetVariantDeviceCopy)
+
+REGISTER_OPTIONAL_COPY(VariantDeviceCopyDirection::HOST_TO_DEVICE);
+REGISTER_OPTIONAL_COPY(VariantDeviceCopyDirection::DEVICE_TO_HOST);
+REGISTER_OPTIONAL_COPY(VariantDeviceCopyDirection::DEVICE_TO_DEVICE);
+
+REGISTER_UNARY_VARIANT_DECODE_FUNCTION(WrappedDatasetVariantWrapper,
+                                       kWrappedDatasetVariantTypeName);
+
 }  // namespace
 
 Status GraphDefBuilderWrapper::AddDataset(
diff --git a/tensorflow/core/kernels/function_ops.cc b/tensorflow/core/kernels/function_ops.cc
index cca3cfbd7c..90f94ee4a0 100644
--- a/tensorflow/core/kernels/function_ops.cc
+++ b/tensorflow/core/kernels/function_ops.cc
@@ -122,6 +122,9 @@ REGISTER_KERNEL_BUILDER(Name(kArgOp)
                             .TypeConstraint<string>("T"),
                         ArgOp);
 
+REGISTER_KERNEL_BUILDER(
+    Name(kArgOp).Device(DEVICE_GPU).TypeConstraint<Variant>("T"), ArgOp);
+
 #define REGISTER(type)     \
   REGISTER_KERNEL_BUILDER( \
       Name(kRetOp).Device(DEVICE_GPU).TypeConstraint<type>("T"), RetvalOp);
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 9163ed715d..1c117166de 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -687,6 +687,16 @@ REGISTER_OP("MapDefun")
       return Status::OK();
     });
 
+REGISTER_OP("WrapDatasetVariant")
+    .Input("input_handle: variant")
+    .Output("output_handle: variant")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("UnwrapDatasetVariant")
+    .Input("input_handle: variant")
+    .Output("output_handle: variant")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("MultiDeviceIterator")
     .Output("handle: resource")
     .Attr("devices: list(string) >= 1")
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 07ed4d2bce..548eb422ed 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -716,3 +716,14 @@ py_test(
         "//tensorflow/python/data/ops:dataset_ops",
     ],
 )
+
+cuda_py_test(
+    name = "wrap_unwrap_test",
+    size = "small",
+    srcs = ["wrap_unwrap_test.py"],
+    additional_deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
diff --git a/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py b/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py
new file mode 100644
index 0000000000..9c734b65e0
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/wrap_unwrap_test.py
@@ -0,0 +1,69 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Wrapping / Unwrapping dataset variants."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.platform import test
+
+
+class WrapDatasetVariantTest(test_base.DatasetTestBase):
+
+  def testBasic(self):
+    ds = dataset_ops.Dataset.range(100)
+    ds_variant = ds._as_variant_tensor()  # pylint: disable=protected-access
+
+    wrapped_variant = gen_dataset_ops.wrap_dataset_variant(ds_variant)
+    unwrapped_variant = gen_dataset_ops.unwrap_dataset_variant(wrapped_variant)
+
+    variant_ds = dataset_ops._VariantDataset(unwrapped_variant,
+                                             ds._element_structure)
+    iterator = dataset_ops.make_initializable_iterator(variant_ds)
+    get_next = iterator.get_next()
+
+    with self.cached_session():
+      self.evaluate(iterator.initializer)
+      for i in range(100):
+        self.assertEqual(i, self.evaluate(get_next))
+
+  def testGPU(self):
+    ds = dataset_ops.Dataset.range(100)
+    ds_variant = ds._as_variant_tensor()  # pylint: disable=protected-access
+    wrapped_variant = gen_dataset_ops.wrap_dataset_variant(ds_variant)
+
+    with ops.device("/gpu:0"):
+      gpu_wrapped_variant = array_ops.identity(wrapped_variant)
+
+    unwrapped_variant = gen_dataset_ops.unwrap_dataset_variant(
+        gpu_wrapped_variant)
+    variant_ds = dataset_ops._VariantDataset(unwrapped_variant,
+                                             ds._element_structure)
+    iterator = dataset_ops.make_initializable_iterator(variant_ds)
+    get_next = iterator.get_next()
+
+    with self.cached_session():
+      self.evaluate(iterator.initializer)
+      for i in range(100):
+        self.assertEqual(i, self.evaluate(get_next))
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From b56d14fbcb33d974f593e0c874bc404d1f1192b3 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 7 Dec 2018 09:56:19 -0800
Subject: [PATCH 1491/1554] Rework the Cudnn LSTM implementation to get rid the
 extra input bias.

This heavily clear up the logic for save/load model since the weights for input bias is gone. The new implementation is mathematically same as the canonical LSTM.

PiperOrigin-RevId: 224531752
---
 tensorflow/python/keras/engine/saving.py      | 58 -------------
 tensorflow/python/keras/engine/saving_test.py | 64 --------------
 tensorflow/python/keras/layers/recurrent.py   | 46 +++-------
 .../python/keras/layers/unified_lstm_test.py  | 84 +++++++++++++++++++
 4 files changed, 94 insertions(+), 158 deletions(-)

diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 15ba5f78d9..54d9e32fb2 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -551,68 +551,10 @@ def preprocess_weights_for_loading(layer,
       if layer.__class__.__name__ == 'ConvLSTM2D':
         weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
 
-  weights = _convert_unified_lstm_weights(layer, weights)
-
   # convert CuDNN layers
   return _convert_rnn_weights(layer, weights)
 
 
-def _convert_unified_lstm_weights(layer, weights):
-  """Converts weights for Unified LSTM layer.
-
-  The input weights suppose to have 2, 3 or 4 items.
-  1. kernel. (i, f, c, o gates concat among axis 1)
-  2. recurrent_kernel. (i, f, c, o concat among axis 1)
-  3. recurrent_bias. (optional, only available when use bias)
-  4. input_bias (optional, only available when use bias and cudnn).
-  Kernel and recurrent_kernel does not need any conversion. During load(),
-  since the layer could be built with the parameter that does not support the
-  defun approach, it is possible that cudnn_bias variable is not created, or
-  even created but not used during actual run. Because of that, we sum up the
-  value of two biases, and give it to recurrent_bias only. Mathematically, the
-  LSTM is calculated as following formula:
-
-    i_t = sigmoid(w_i * x_t + r_i * h_(t-1) + b_wi + b_ri)
-    f_t = sigmoid(w_f * x_t + r_f * h_(t-1) + b_wf + b_rf)
-    o_t = sigmoid(w_o * x_t + r_o * h_(t-1) + b_wo + b_ro)
-    c'_t = tanh(w_c * x_t + r_c * h_(t-1) + b_wc + b_rc)
-    c_t = f_t . c_(t-1) + i_t . c'_t
-    h_t = o_t . tanh(c_t)
-
-  Note that b_w{x} is the input_bias, and b_r{x} is the recurrent_bias.
-  Since it is a linear add, it is fine to give b_r{x} 100% and b_w{x} 0%, as
-  long as the sum are the same.
-
-  Args:
-    layer: The keras layer that will be loaded with weights.
-    weights: the list of numpy arrays which hold the weights to be loaded.
-
-  Returns:
-    weights: the processed list of numpy arrays.
-  """
-  if layer.__class__.__name__ == 'UnifiedLSTM':
-    if len(weights) not in [3, 4]:
-      # Only handles the bias conversion in this function, in the case when
-      # bias is not used or weights in unexpected length, do nothing and return.
-      return weights
-
-    if len(weights) == 3:
-      recurrent_bias = weights[2]
-    else:
-      # Add all the bias value to recurrent_bias
-      recurrent_bias = weights[2] + weights[3]
-
-    if len(layer.weights) == 3:
-      weights = weights[:2] + [recurrent_bias]
-    elif len(layer.weights) == 4:
-      # Create a zero filled input_bias, since all the weights have given
-      # to recurrent bias.
-      input_bias = np.zeros_like(recurrent_bias)
-      weights = weights[:2] + [recurrent_bias, input_bias]
-
-  return weights
-
-
 def _convert_rnn_weights(layer, weights):
   """Converts weights for RNN layers between native and CuDNN format.
 
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py
index 8fcefce748..6d9d9a2fca 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/engine/saving_test.py
@@ -221,70 +221,6 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
           for (x, y) in zip(weights1, weights2)
       ]
 
-  @parameterized.named_parameters(
-      # test_name, use_bias, bias_initializer, activation
-      ('normal', True, 'zeros', 'tanh'),
-      ('no_bias', False, 'zeros', 'tanh'),
-      # TODO(scottzhu): Reenable this test case when the approach is decided.
-      # ('random_bias', True, 'random_uniform', 'tanh'),
-      ('no_cudnn_bias', True, 'zeros', 'relu')
-  )
-  def test_process_weights_for_loading_unified_lstm(
-      self, use_bias, bias_initializer, activation):
-    if h5py is None:
-      return
-
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir)
-    h5_path = os.path.join(temp_dir, 'test.h5')
-
-    batch = 10
-    timestep = 3
-    input_dim = 5
-    units = 2
-
-    x = np.random.random((batch, timestep, input_dim))
-
-    def build_model():
-      inputs = keras.layers.Input(
-          shape=[timestep, input_dim], dtype=dtypes.float32)
-      layer = keras.layers.UnifiedLSTM(
-          units,
-          activation=activation,
-          use_bias=use_bias,
-          bias_initializer=bias_initializer)
-      output = layer(inputs)
-      return keras.models.Model(inputs, output), layer
-
-    with self.cached_session():
-      model, layer = build_model()
-      y_ref = model.predict(x)
-      model.save_weights(h5_path)
-
-      cloned_model, new_layer = build_model()
-      cloned_model.load_weights(h5_path)
-      y = cloned_model.predict(x)
-
-      self.assertAllClose(y, y_ref)
-
-      # Test the individual layer weights.
-      weights1 = layer.get_weights()
-      weights2 = new_layer.get_weights()
-      self.assertLen(weights1, len(weights2))
-      # kernel and current kernel should be the same.
-      self.assertAllClose(weights1[:2], weights2[:2])
-
-      if len(weights2) >= 3:
-        # Test recurrent bias
-        expected_recurrent_bias = weights1[2]
-        if len(weights1) == 4:
-          expected_recurrent_bias += weights1[3]
-        self.assertAllClose(weights2[2], expected_recurrent_bias)
-
-      if len(weights2) == 4:
-        # Test recovered input_gate_bias to be always zero
-        self.assertAllClose(weights2[3], np.zeros_like(weights1[3]))
-
   def test_sequential_weight_loading(self):
     if h5py is None:
       return
diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 189ad98794..fdc2acd538 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -2666,21 +2666,6 @@ class UnifiedLSTM(LSTM):
         activation == 'tanh' and recurrent_dropout == 0 and
         not unroll and use_bias and bias_regularizer is None)
 
-  def build(self, input_shape):
-    super(UnifiedLSTM, self).build(input_shape)
-    if self.could_use_cudnn:
-      # Add a new set of bias for CuDNN implementation only. Standard LSTM only
-      # has bias for recurrent kernel, while CuDNN LSTM has an extra set for
-      # input gate as well.
-      self.cudnn_bias = self.add_weight(
-          shape=(self.units * 4,),
-          name='cudnn_bias',
-          use_resource=True,
-          initializer=self.bias_initializer,
-          regularizer=self.bias_regularizer,
-          constraint=self.bias_constraint)
-    self.built = True
-
   def call(self, inputs, mask=None, training=None, initial_state=None):
     # LSTM does not support constants. Ignore it during process.
     inputs, initial_state, _ = self._process_inputs(inputs, initial_state, None)
@@ -2721,8 +2706,6 @@ class UnifiedLSTM(LSTM):
         # Reverse time axis.
         inputs = K.reverse(inputs, 1)
 
-      combined_bias = array_ops.concat([self.cudnn_bias, self.cell.bias], 0)
-
       if 0 < self.dropout < 1:
         if self._dropout_mask is None:
           self._dropout_mask = _generate_dropout_mask(
@@ -2757,23 +2740,23 @@ class UnifiedLSTM(LSTM):
         if context.num_gpus() > 0:
           last_output, outputs, new_h, new_c, runtime = defun_cudnn_lstm(
               inputs, initial_state[0], initial_state[1], self.cell.kernel,
-              self.cell.recurrent_kernel, combined_bias, self.time_major)
+              self.cell.recurrent_kernel, self.cell.bias, self.time_major)
         else:
           last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
               inputs, initial_state[0], initial_state[1], self.cell.kernel,
-              self.cell.recurrent_kernel, combined_bias, self.activation,
+              self.cell.recurrent_kernel, self.cell.bias, self.activation,
               self.recurrent_activation, self.time_major)
       else:
         # Call the normal LSTM impl and register the CuDNN impl function. The
         # grappler will kick in during session execution to optimize the graph.
         last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(
             inputs, initial_state[0], initial_state[1], self.cell.kernel,
-            self.cell.recurrent_kernel, combined_bias, self.activation,
+            self.cell.recurrent_kernel, self.cell.bias, self.activation,
             self.recurrent_activation, self.time_major)
 
         function.register(defun_cudnn_lstm, inputs, initial_state[0],
                           initial_state[1], self.cell.kernel,
-                          self.cell.recurrent_kernel, combined_bias,
+                          self.cell.recurrent_kernel, self.cell.bias,
                           self.time_major)
       states = [new_h, new_c]
 
@@ -2800,8 +2783,6 @@ class UnifiedLSTM(LSTM):
     if self.trainable:
       weights = []
       weights += self.cell.trainable_weights
-      if getattr(self, 'cudnn_bias', None) is not None:
-        weights += [self.cudnn_bias]
       return weights
     return []
 
@@ -2810,8 +2791,6 @@ class UnifiedLSTM(LSTM):
     if not self.trainable:
       weights = []
       weights += self.cell.non_trainable_weights
-      if getattr(self, 'cudnn_bias', None) is not None:
-        weights += [self.cudnn_bias]
       return weights
     return []
 
@@ -2830,8 +2809,6 @@ class UnifiedLSTM(LSTM):
   def get_weights(self):
     weights = []
     weights += self.cell.weights
-    if getattr(self, 'cudnn_bias', None) is not None:
-      weights += [self.cudnn_bias]
     return K.batch_get_value(weights)
 
   def set_weights(self, weights):
@@ -2839,10 +2816,6 @@ class UnifiedLSTM(LSTM):
     cell_weights = weights[:len(self.cell.weights)]
     if cell_weights:
       tuples.append((self.cell.weights, cell_weights))
-    if getattr(self, 'cudnn_bias', None) is not None:
-      cudnn_bias_weights = weights[len(self.cell.weights):]
-      if cudnn_bias_weights:
-        tuples.append((self.cudnn_bias, cudnn_bias_weights))
     K.batch_set_value(tuples)
 
 
@@ -2919,9 +2892,6 @@ def standard_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias,
   input_shape = K.int_shape(inputs)
   timesteps = input_shape[0] if time_major else input_shape[1]
 
-  # Only use the second half of the bias weights.
-  _, real_bias = array_ops.split(bias, 2)
-
   def step(cell_inputs, cell_states):
     """Step function that will be used by Keras RNN backend."""
     h_tm1 = cell_states[0]  # previous memory state
@@ -2929,7 +2899,7 @@ def standard_lstm(inputs, init_h, init_c, kernel, recurrent_kernel, bias,
 
     z = K.dot(cell_inputs, kernel)
     z += K.dot(h_tm1, recurrent_kernel)
-    z = K.bias_add(z, real_bias)
+    z = K.bias_add(z, bias)
 
     z0, z1, z2, z3 = array_ops.split(z, 4, axis=1)
 
@@ -2962,9 +2932,13 @@ def cudnn_lstm(inputs, input_h, input_c, kernel, recurrent_kernel, bias,
 
   weights = array_ops.split(kernel, 4, axis=1)
   weights += array_ops.split(recurrent_kernel, 4, axis=1)
+  # CuDNN has an extra set of bias for inputs, we disable them (setting to 0),
+  # so that mathematically it is same as the canonical LSTM implementation.
+  full_bias = array_ops.concat((array_ops.zeros_like(bias), bias), 0)
+
   params = _canonical_to_params(
       weights=weights,
-      biases=array_ops.split(bias, 8),
+      biases=array_ops.split(full_bias, 8),
       shape=constant_op.constant([-1]),
       transpose_weights=True)
 
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index b004284140..2cb3eff8fd 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import shutil
 import time
 
 from absl.testing import parameterized
@@ -26,6 +28,7 @@ import numpy as np
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
@@ -175,6 +178,86 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
                         bias_regularizer=bias_regularizer)
     self.assertFalse(layer.could_use_cudnn)
 
+  def test_unified_lstm_feature_parity_with_canonical_lstm(self):
+    with context.eager_mode():
+      # Run this test under eager only due to b/120160788 for model.set_weights.
+      input_shape = 10
+      rnn_state_size = 8
+      timestep = 4
+      batch = 20
+
+      (x_train, y_train), _ = testing_utils.get_test_data(
+          train_samples=batch,
+          test_samples=0,
+          input_shape=(timestep, input_shape),
+          num_classes=rnn_state_size)
+      y_train = keras.utils.to_categorical(y_train, rnn_state_size)
+
+      inputs = keras.layers.Input(
+          shape=[timestep, input_shape], dtype=dtypes.float32)
+      lstm_layer = keras.layers.LSTM(rnn_state_size,
+                                     recurrent_activation='sigmoid')
+      output = lstm_layer(inputs)
+      lstm_model = keras.models.Model(inputs, output)
+      weights = lstm_model.get_weights()
+      y_1 = lstm_model.predict(x_train)
+      lstm_model.compile('rmsprop', 'mse')
+      lstm_model.fit(x_train, y_train)
+      y_2 = lstm_model.predict(x_train)
+
+      with test_util.device(use_gpu=True):
+        cudnn_layer = keras.layers.UnifiedLSTM(rnn_state_size,
+                                               recurrent_activation='sigmoid')
+        cudnn_model = keras.models.Model(inputs, cudnn_layer(inputs))
+      cudnn_model.set_weights(weights)
+      y_3 = cudnn_model.predict(x_train)
+      cudnn_model.compile('rmsprop', 'mse')
+      cudnn_model.fit(x_train, y_train)
+      y_4 = cudnn_model.predict(x_train)
+
+      self.assertAllClose(y_1, y_3)
+      self.assertAllClose(y_2, y_4)
+
+  @parameterized.named_parameters(
+      # test_name, use_bias, bias_initializer, activation
+      ('normal', True, 'zeros'),
+      ('no_bias', False, 'zeros'),
+      ('random_bias', True, 'random_uniform'),
+  )
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_unified_lstm_model_save_load(self, use_bias, bias_initializer):
+    temp_dir = self.get_temp_dir()
+    self.addCleanup(shutil.rmtree, temp_dir)
+    h5_path = os.path.join(temp_dir, 'test.h5')
+
+    batch = 10
+    timestep = 3
+    input_dim = 5
+    units = 2
+
+    x = np.random.random((batch, timestep, input_dim))
+
+    def build_model():
+      inputs = keras.layers.Input(
+          shape=[timestep, input_dim], dtype=dtypes.float32)
+      layer = keras.layers.UnifiedLSTM(
+          units,
+          use_bias=use_bias,
+          bias_initializer=bias_initializer)
+      output = layer(inputs)
+      return keras.models.Model(inputs, output), layer
+
+    model, layer = build_model()
+    y_ref = model.predict(x)
+    model.save_weights(h5_path)
+
+    cloned_model, new_layer = build_model()
+    cloned_model.load_weights(h5_path)
+    y = cloned_model.predict(x)
+
+    self.assertAllClose(y, y_ref)
+    self.assertAllClose(layer.get_weights(), new_layer.get_weights())
+
   @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_unified_lstm_output_on_multiple_kernel(self):
     input_shape = 10
@@ -240,6 +323,7 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
     model.compile('rmsprop', loss='mse')
     model.fit(x_train, y_train, epochs=epoch)
     model.evaluate(x_train, y_train)
+    model.predict(x_train)
 
   @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_return_sequences_LSTM(self):
-- 
GitLab


From 8f2bc575d908bf1d1e743dd0e67c7115c5b757a9 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 7 Dec 2018 09:57:11 -0800
Subject: [PATCH 1492/1554] Reuse GPUs for multiple ranks in NcclManager test.

Since NCCL 2.1.2, NCCL uses cooperative group launch by default which
results in deadlocks in the NcclManager test if number of ranks > number of
GPUs.  Setting NCCL_LAUNCH_MODE to PARALLEL, which was the default before NCCL
2.1.2, fixes this.

PiperOrigin-RevId: 224531851
---
 tensorflow/core/nccl/nccl_manager_test.cc | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/nccl/nccl_manager_test.cc b/tensorflow/core/nccl/nccl_manager_test.cc
index f43103e120..f9ed4d0b9a 100644
--- a/tensorflow/core/nccl/nccl_manager_test.cc
+++ b/tensorflow/core/nccl/nccl_manager_test.cc
@@ -65,6 +65,7 @@ class NcclManagerTest : public ::testing::Test {
 
   static void SetUpTestCase() {
     setenv("NCCL_DEBUG", "WARN", 1 /* replace */);
+    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
     devices_ = new std::vector<std::unique_ptr<BaseGPUDevice>>(GetGPUDevices());
     LOG(ERROR) << "Running test with " << devices_->size() << " gpus";
   }
@@ -200,7 +201,7 @@ TYPED_TEST_CASE(NcclManagerTest, TypeList);
 
 // Test basic sum reduction.
 TYPED_TEST(NcclManagerTest, BasicSumReduction) {
-  const int num_ranks = this->NumGPUs();
+  const int num_ranks = 4;
 
   for (int op = 0; op < 4; ++op) {
     ncclRedOp_t reduction_op = static_cast<ncclRedOp_t>(op);
@@ -208,6 +209,7 @@ TYPED_TEST(NcclManagerTest, BasicSumReduction) {
         this->MakeTestCase(num_ranks, reduction_op, TensorShape({2, 3}), 0.0f));
     for (int rank = 0; rank < num_ranks; ++rank) {
       auto* device = this->GetDevice(rank);
+      VLOG(2) << "rank " << rank << " device " << device->name();
       auto* event_mgr = device->tensorflow_gpu_device_info()->event_mgr;
       auto* stream = device->tensorflow_gpu_device_info()->stream;
       NcclManager::instance()->AddToAllReduce(
@@ -224,14 +226,12 @@ TYPED_TEST(NcclManagerTest, BasicSumReduction) {
 // Same as the Basic test, but with multiple threads launching parts of many
 // reductions.
 //
-// Testing the multi-rank execution is currently reduced as it can hang when run
-// with num_ranks > devices->size(), for some GPUs (e.g. K20m).
-// To test the higher settings, increase num_ranks,
-// num_collectives_per_iteration and time_limit_micros.
+// To run test longer, increase num_ranks, num_collectives_per_iteration and
+// time_limit_micros.
 TYPED_TEST(NcclManagerTest, MultipleCallers) {
-  const int num_ranks = this->NumGPUs();
+  const int num_ranks = 4;
   const int num_collectives_per_iteration = 10;  // 1000;
-  const int num_threads = 3;
+  const int num_threads = num_ranks * 2;
   const int time_limit_micros = 100;  // 60 * 30 * 1000 * 1000;
 
   int64 start = Env::Default()->NowMicros();
-- 
GitLab


From 19b07482d6a80943c832d167df6f49c0a4008462 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Fri, 7 Dec 2018 09:57:49 -0800
Subject: [PATCH 1493/1554] Fix tensorflow nccl CODEOWNERS

PiperOrigin-RevId: 224531965
---
 CODEOWNERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CODEOWNERS b/CODEOWNERS
index bfcdc2a23f..cb3fa23124 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -1,7 +1,7 @@
 # Where component owners are known, add them here.
 
 /tenosrflow/core/debug @caisq
-/tensorflow/core/nccl/ @azaks @csigg
+/tensorflow/core/nccl/ @azaks2 @chsigg
 /tensorflow/core/platform/windows/ @mrry
 /tensorflow/core/platform/s3 @yongtang
 /tensorflow/go @asimshankar
-- 
GitLab


From cc079ee316c75993f86a19c6b8820d2bc36a33b6 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 7 Dec 2018 10:09:13 -0800
Subject: [PATCH 1494/1554] Lower the tol for metrics to 1e-5. And add a
 predict_tol.

PiperOrigin-RevId: 224534209
---
 tensorflow/contrib/distribute/python/keras_test.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index d66fc52cf2..dea208232a 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1302,16 +1302,18 @@ class TestDistributionStrategyCorrectness(test.TestCase,
   def test_correctness(self, distribution, use_numpy, use_validation_data):
 
     with self.cached_session():
-      weights_tolerance = 1e-5
-      metrics_tolerance = 1e-4
+      default_tolerance = 1e-5
+      weights_tolerance = default_tolerance
+      metrics_tolerance = default_tolerance
+      predict_tolerance = default_tolerance
 
       if isinstance(distribution, (mirrored_strategy.MirroredStrategy,
                                    mirrored_strategy.CoreMirroredStrategy)):
         # TODO(b/119257215): Weights are not exactly the same, so use lower
         # tolerance for now.
         weights_tolerance = 1e-4
+        predict_tolerance = 1e-4
 
-      metrics = ['mse']
       keras.backend.set_image_data_format('channels_last')
       np.random.seed(_RANDOM_SEED)
       random_seed.set_random_seed(_RANDOM_SEED)
@@ -1351,7 +1353,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         model.compile(
             loss=keras.losses.mean_squared_error,
             optimizer=gradient_descent_keras.SGD(0.5),
-            metrics=metrics,
+            metrics=['mse'],
             distribute=with_distribution)
 
         training_inputs, eval_inputs, predict_inputs = (
@@ -1395,8 +1397,8 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       self.assertAllClose(
           predict_with_ds,
           predict_without_ds,
-          atol=weights_tolerance,
-          rtol=weights_tolerance,
+          atol=predict_tolerance,
+          rtol=predict_tolerance,
           msg='Fail to assert predict results.')
 
       if not (isinstance(distribution, tpu_strategy.TPUStrategy) and
-- 
GitLab


From 89912db5f751778b0a24c7e77e4083bbe7c042a3 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 7 Dec 2018 10:14:25 -0800
Subject: [PATCH 1495/1554] Change API for TF 2.0 for tf.space_to_batch and
 tf.space_to_batch_nd.

PiperOrigin-RevId: 224535013
---
 tensorflow/python/ops/array_ops.py                  | 10 +++++++++-
 tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt  |  2 +-
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt     |  4 ++++
 tensorflow/tools/compatibility/renames_v2.py        |  1 -
 tensorflow/tools/compatibility/reorders_v2.py       |  2 ++
 tensorflow/tools/compatibility/tf_upgrade_v2.py     |  8 ++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py       | 13 +++++++++++++
 7 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 185452e1ab..216b08a8b0 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2652,7 +2652,7 @@ def required_space_to_batch_paddings(input_shape,
     return result_paddings, result_crops
 
 
-@tf_export("nn.space_to_batch", v1=["nn.space_to_batch", "space_to_batch"])
+@tf_export(v1=["nn.space_to_batch", "space_to_batch"])
 @deprecation.deprecated_endpoints("space_to_batch")
 def space_to_batch(input, paddings, block_size, name=None):  # pylint: disable=redefined-builtin
   result = space_to_batch_nd(
@@ -2667,6 +2667,14 @@ def space_to_batch(input, paddings, block_size, name=None):  # pylint: disable=r
 space_to_batch.__doc__ = gen_array_ops.space_to_batch.__doc__
 
 
+@tf_export("space_to_batch", "nn.space_to_batch", v1=[])
+def space_to_batch_v2(input, block_shape, paddings, name=None):  # pylint: disable=redefined-builtin
+  return space_to_batch_nd(input, block_shape, paddings, name)
+
+
+space_to_batch_v2.__doc__ = gen_array_ops.space_to_batch_nd.__doc__
+
+
 @tf_export(v1=["nn.space_to_depth", "space_to_depth"])
 @deprecation.deprecated_endpoints("space_to_depth")
 def space_to_depth(input, block_size, name=None, data_format="NHWC"):  # pylint: disable=redefined-builtin
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
index 9e5e82483a..c75c75f2ef 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.pbtxt
@@ -270,7 +270,7 @@ tf_module {
   }
   member_method {
     name: "space_to_batch"
-    argspec: "args=[\'input\', \'paddings\', \'block_size\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'input\', \'block_shape\', \'paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "space_to_depth"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 542e5cbe9a..4432cae53b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -928,6 +928,10 @@ tf_module {
     name: "sort"
     argspec: "args=[\'values\', \'axis\', \'direction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'ASCENDING\', \'None\'], "
   }
+  member_method {
+    name: "space_to_batch"
+    argspec: "args=[\'input\', \'block_shape\', \'paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "space_to_batch_nd"
     argspec: "args=[\'input\', \'block_shape\', \'paddings\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index b27fd5acc7..e4ef8eb528 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -511,7 +511,6 @@ renames = {
     'tf.sets.set_intersection': 'tf.sets.intersection',
     'tf.sets.set_size': 'tf.sets.size',
     'tf.sets.set_union': 'tf.sets.union',
-    'tf.space_to_batch': 'tf.nn.space_to_batch',
     'tf.space_to_depth': 'tf.compat.v1.space_to_depth',
     'tf.sparse.matmul': 'tf.sparse.sparse_dense_matmul',
     'tf.sparse.merge': 'tf.compat.v1.sparse.merge',
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 76f71d5804..e6596755c3 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -63,6 +63,7 @@ reorders = {
     'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
     'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
     'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
     'tf.nn.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
     'tf.pad': ['tensor', 'paddings', 'mode', 'name', 'constant_values'],
@@ -85,6 +86,7 @@ reorders = {
     'tf.serialize_sparse': ['sp_input', 'name', 'out_type'],
     'tf.shape': ['input', 'name', 'out_type'],
     'tf.size': ['input', 'name', 'out_type'],
+    'tf.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
     'tf.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.sparse.add': ['a', 'b', 'threshold', 'thresh'],
     'tf.sparse.concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 0630ce28cf..b2477a541e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -51,6 +51,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.batch_to_space": {
             "block_size": "block_shape",
         },
+        "tf.space_to_batch": {
+            "block_size": "block_shape",
+        },
         "tf.constant": {
             "verify_shape": "verify_shape_is_now_always_true",
         },
@@ -294,6 +297,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     self.manual_symbol_renames = {
         "tf.batch_to_space_nd":
             "tf.batch_to_space",
+        "tf.space_to_batch_nd":
+            "tf.space_to_batch",
+        "tf.nn.space_to_batch":
+            "tf.space_to_batch",
         "tf.extract_image_patches":
             "tf.image.extract_image_patches",
         "tf.gfile.Copy":
@@ -497,6 +504,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.argmax",
         "tf.argmin",
         "tf.batch_to_space",
+        "tf.nn.space_to_batch",
         "tf.boolean_mask",
         "tf.convert_to_tensor",
         "tf.nn.moments",
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b362a8081f..4460ad5182 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -567,6 +567,19 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testSpacetoBatch(self):
+    text = "tf.space_to_batch_nd(input, shape, paddings, name)"
+    expected_text = "tf.space_to_batch(input, shape, paddings, name)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = "tf.nn.space_to_batch(input, paddings, block_size, name)"
+    expected_text = (
+        "tf.space_to_batch(input=input, paddings=paddings, "
+        "block_size=block_size, name=name)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
   def testInTopK(self):
     text = "tf.math.in_top_k(a, b, c, n)"
     expected_text = (
-- 
GitLab


From a40ad0ce882e5cc482b16892162f2ce061aea055 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Fri, 7 Dec 2018 10:14:36 -0800
Subject: [PATCH 1496/1554] [tf.data] tiny test change

PiperOrigin-RevId: 224535049
---
 .../python/data/experimental/kernel_tests/optimization/BUILD | 1 -
 .../kernel_tests/optimization/noop_elimination_test.py       | 5 -----
 2 files changed, 6 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index f214944254..2fc243aa13 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -279,7 +279,6 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/data/experimental/ops:optimization",
-        "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
index ce86bfa4e0..8058f53eea 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import optimization
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -42,10 +41,6 @@ class NoopEliminationTest(test_base.DatasetTestBase):
             ["FiniteRepeat", "FiniteSkip", "Prefetch", "MemoryCacheImpl"]))
     dataset = dataset.repeat(some_tensor).skip(5).take(-1).skip(0).repeat(
         1).prefetch(0).prefetch(1).cache()
-    options = dataset_ops.Options()
-    options.experimental_optimization = OptimizationOptions()
-    options.experimental_optimization.noop_elimination = True
-    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=range(5))
 
 
-- 
GitLab


From 6d456192027c309a902fb63e15c57254e4d7ab42 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 7 Dec 2018 10:43:54 -0800
Subject: [PATCH 1497/1554] Fix TFLite iOS example Podfile

PiperOrigin-RevId: 224540375
---
 tensorflow/lite/examples/ios/camera/Podfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/lite/examples/ios/camera/Podfile b/tensorflow/lite/examples/ios/camera/Podfile
index 96a0d23426..2ed5c8da1e 100644
--- a/tensorflow/lite/examples/ios/camera/Podfile
+++ b/tensorflow/lite/examples/ios/camera/Podfile
@@ -1,5 +1,7 @@
 platform :ios, '8.0'
 inhibit_all_warnings!
 
+project 'tflite_camera_example.xcodeproj'
+
 target 'tflite_camera_example'
        pod 'TensorFlowLite', '1.12.0'
-- 
GitLab


From c8e92c679ec4c098bb9404bf9f38a5f4f02e3c42 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 7 Dec 2018 10:52:12 -0800
Subject: [PATCH 1498/1554] Better propagation of variable shape inference.

PiperOrigin-RevId: 224541903
---
 .../resource_variable_ops_test.py             | 29 +++++++++++++++++++
 tensorflow/python/ops/array_ops.py            |  6 +++-
 tensorflow/python/ops/custom_gradient.py      |  4 +++
 3 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index b57d9d47aa..3992d6bdfb 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -33,6 +33,8 @@ from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import custom_gradient
+from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -585,6 +587,33 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     v.load(2.0)
     self.assertEqual(2.0, self.evaluate(v.value()))
 
+  def testShapePassedToGradient(self):
+    with ops.Graph().as_default():
+      @custom_gradient.custom_gradient
+      def differentiable_scatter_update(handle, indices, values):
+        with ops.control_dependencies([
+            resource_variable_ops.resource_scatter_update(
+                handle, indices, values)]):
+          new_handle = array_ops.identity(handle)
+
+        def grad(dresult):
+          self.assertIsNotNone(
+              tensor_util.constant_value(dresult.dense_shape))
+          return [dresult, None, None]
+
+        return new_handle, grad
+
+      var = variable_scope.get_variable(
+          "foo", shape=[20], initializer=init_ops.zeros_initializer,
+          dtype=dtypes.float64, use_resource=True)
+
+      indices = math_ops.range(10)
+      updates = math_ops.range(9, -1, -1, dtype=dtypes.float64)
+      new_handle = differentiable_scatter_update(var.handle, indices, updates)
+      gathered = resource_variable_ops.resource_gather(
+          new_handle, indices, dtype=var.dtype)
+      gradients_impl.gradients([gathered], [updates])
+
   def testToFromProtoCachedValue(self):
     with ops.Graph().as_default():
       v_def = resource_variable_ops.ResourceVariable(
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 216b08a8b0..9dabbffb13 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -78,7 +78,11 @@ def identity(input, name=None):  # pylint: disable=redefined-builtin
       return input._copy()  # pylint: disable=protected-access
     return input
   else:
-    return gen_array_ops.identity(input, name=name)
+    ret = gen_array_ops.identity(input, name=name)
+    # Propagate handle data for happier shape inference for resource variables.
+    if hasattr(input, "_handle_data"):
+      ret._handle_data = input._handle_data  # pylint: disable=protected-access
+    return ret
 
 
 # pylint: disable=redefined-builtin,protected-access
diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py
index 1426e8851c..d96601ac21 100644
--- a/tensorflow/python/ops/custom_gradient.py
+++ b/tensorflow/python/ops/custom_gradient.py
@@ -236,6 +236,10 @@ def _graph_mode_decorator(f, *args, **kwargs):
   original_tensors = all_tensors
   with ops.get_default_graph().gradient_override_map({"IdentityN": name}):
     all_tensors = array_ops.identity_n(all_tensors)
+  # Propagate handle data for happier shape inference for resource variables.
+  for i, t in enumerate(original_tensors):
+    if t.dtype == dtypes.resource and hasattr(t, "_handle_data"):
+      all_tensors[i]._handle_data = t._handle_data  # pylint: disable=protected-access
   tape_lib.record_operation(
       f.__name__, all_tensors, original_tensors, tape_grad_fn)
   for ot, t in zip(original_tensors, all_tensors):
-- 
GitLab


From 504afac32db928bfb196299030ac2af28ce06d9d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 10:58:45 -0800
Subject: [PATCH 1499/1554] Increase max buffer size.

PiperOrigin-RevId: 224543122
---
 .../lite/kernels/internal/optimized/optimized_ops.h       | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index c7691e2763..c79b69a22e 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -3582,8 +3582,8 @@ inline void AveragePool(const PoolParams& params,
             std::min(params.filter_height, input_height - in_y_origin);
         const int filter_count =
             (filter_x_end - filter_x_start) * (filter_y_end - filter_y_start);
-        // 1280 required by Inception v3
-        static constexpr int kAccBufferMaxSize = 2048;
+        // 2560 is required by MobileNetV2 with depth multiplier 2.
+        static constexpr int kAccBufferMaxSize = 4096;
         TFLITE_DCHECK_LE(depth, kAccBufferMaxSize);
         uint16 acc[kAccBufferMaxSize];
         memset(acc, 0, depth * sizeof(acc[0]));
@@ -3748,8 +3748,8 @@ inline void MaxPool(const PoolParams& params, const RuntimeShape& input_shape,
         const int filter_y_start = std::max(0, -in_y_origin);
         const int filter_y_end =
             std::min(params.filter_height, input_height - in_y_origin);
-        // 2048 required by Inception v3
-        static constexpr int kAccBufferMaxSize = 2048;
+        // 2560 is required by MobileNetV2 with depth multiplier 2.
+        static constexpr int kAccBufferMaxSize = 4096;
         TFLITE_DCHECK_LE(depth, kAccBufferMaxSize);
         uint8 acc[kAccBufferMaxSize];
         memset(acc, 0, depth * sizeof(acc[0]));
-- 
GitLab


From 629834552e79edf13dbdf2306420698b3d64def2 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 7 Dec 2018 11:00:57 -0800
Subject: [PATCH 1500/1554] Fix up control_flow_ops_tests to work with v2

- Add cond_v2/while_v2 dependencies
- Add run_deprecated_v1 and run_v1_only where needed
- Remove non-strict cond tests

PiperOrigin-RevId: 224543529
---
 tensorflow/python/BUILD                       |   2 +
 .../python/ops/control_flow_ops_test.py       | 367 ++++++++++--------
 2 files changed, 199 insertions(+), 170 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index bc6dc413e7..cc36f1fc0e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3236,6 +3236,7 @@ cuda_py_test(
     srcs = ["ops/control_flow_ops_test.py"],
     additional_deps = [
         ":array_ops",
+        ":cond_v2",
         ":control_flow_ops",
         ":embedding_ops",
         ":framework_for_generated_wrappers",
@@ -3251,6 +3252,7 @@ cuda_py_test(
         ":util",
         ":variable_scope",
         ":variables",
+        ":while_v2",
         "//tensorflow/python/eager:def_function",
     ],
 )
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index b19ec4bd61..0c18b7208f 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -21,9 +21,9 @@ from __future__ import print_function
 import collections
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import node_def_pb2
-from tensorflow.python.client import session
 from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -127,56 +127,56 @@ class GroupTestCase(test_util.TensorFlowTestCase):
       node { name: "root" op: "NoOp" input: "^a" input: "^b" }
     """, self._StripGraph(gd))
 
+  @test_util.run_deprecated_v1
   def testPassingNonTensors(self):
-    with ops.Graph().as_default():
-      with self.assertRaises(TypeError):
-        control_flow_ops.group(1, 2)
+    with self.assertRaises(TypeError):
+      control_flow_ops.group(1, 2)
 
 
 class ShapeTestCase(test_util.TensorFlowTestCase):
 
   def testShape(self):
-    with ops.Graph().as_default():
-      tensor = constant_op.constant([1.0, 2.0])
-      self.assertEquals([2], tensor.get_shape())
-      self.assertEquals([2],
-                        control_flow_ops.with_dependencies(
-                            [constant_op.constant(1.0)], tensor).get_shape())
+    tensor = constant_op.constant([1.0, 2.0])
+    self.assertEquals([2], tensor.get_shape())
+    self.assertEquals([2],
+                      control_flow_ops.with_dependencies(
+                          [constant_op.constant(1.0)], tensor).get_shape())
 
 
 class WithDependenciesTestCase(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testTupleDependencies(self):
-    with ops.Graph().as_default():
-      counter = variable_scope.get_variable(
-          "my_counter", shape=[], initializer=init_ops.zeros_initializer())
-      increment_counter = state_ops.assign_add(counter, 1)
-      const_with_dep = control_flow_ops.with_dependencies(
-          (increment_counter, constant_op.constant(42)),
-          constant_op.constant(7))
-      with self.cached_session():
-        variables.global_variables_initializer().run()
-        self.assertEquals(0, self.evaluate(counter))
-        self.assertEquals(7, self.evaluate(const_with_dep))
-        self.assertEquals(1, self.evaluate(counter))
-
+    counter = variable_scope.get_variable(
+        "my_counter", shape=[], initializer=init_ops.zeros_initializer())
+    increment_counter = state_ops.assign_add(counter, 1)
+    const_with_dep = control_flow_ops.with_dependencies(
+        (increment_counter, constant_op.constant(42)),
+        constant_op.constant(7))
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertEquals(0, self.evaluate(counter))
+    self.assertEquals(7, self.evaluate(const_with_dep))
+    self.assertEquals(1, self.evaluate(counter))
+
+  @test_util.run_deprecated_v1
   def testListDependencies(self):
-    with ops.Graph().as_default():
-      counter = variable_scope.get_variable(
-          "my_counter", shape=[], initializer=init_ops.zeros_initializer())
-      increment_counter = state_ops.assign_add(counter, 1)
-      const_with_dep = control_flow_ops.with_dependencies(
-          [increment_counter, constant_op.constant(42)],
-          constant_op.constant(7))
-      with self.cached_session():
-        variables.global_variables_initializer().run()
-        self.assertEquals(0, self.evaluate(counter))
-        self.assertEquals(7, self.evaluate(const_with_dep))
-        self.assertEquals(1, self.evaluate(counter))
+    counter = variable_scope.get_variable(
+        "my_counter", shape=[], initializer=init_ops.zeros_initializer())
+    increment_counter = state_ops.assign_add(counter, 1)
+    const_with_dep = control_flow_ops.with_dependencies(
+        [increment_counter, constant_op.constant(42)],
+        constant_op.constant(7))
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertEquals(0, self.evaluate(counter))
+    self.assertEquals(7, self.evaluate(const_with_dep))
+    self.assertEquals(1, self.evaluate(counter))
 
 
 class SwitchTestCase(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testIndexedSlicesWithDenseShape(self):
     with self.cached_session():
       data = ops.IndexedSlices(
@@ -190,68 +190,64 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
       self.assertAllEqual([1, 2, 3], switch_true.values.eval())
       self.assertAllEqual([0, 1], switch_true.indices.eval())
 
+  @test_util.run_deprecated_v1
   def testIndexedSlicesGradient(self):
-    with ops.Graph().as_default():
-      embedding_matrix = variable_scope.get_variable(
-          "embedding_matrix", [5, 5],
-          initializer=init_ops.random_normal_initializer())
-
-      def cond(it, _):
-        return it < 5
-
-      def body(it, cost):
-        embedding = embedding_ops.embedding_lookup(embedding_matrix + 0.0, [0])
-        cost += math_ops.reduce_sum(embedding)
-        return it + 1, cost
-
-      _, cost = control_flow_ops.while_loop(
-          cond, body, [constant_op.constant(0),
-                       constant_op.constant(0.0)])
-      optimizer = momentum.MomentumOptimizer(0.1, 0.9)
-      train_op = optimizer.minimize(cost)
-      with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
-        for _ in range(10):
-          self.evaluate([train_op])
+    embedding_matrix = variable_scope.get_variable(
+        "embedding_matrix", [5, 5],
+        initializer=init_ops.random_normal_initializer())
+
+    def cond(it, _):
+      return it < 5
+
+    def body(it, cost):
+      embedding = embedding_ops.embedding_lookup(embedding_matrix + 0.0, [0])
+      cost += math_ops.reduce_sum(embedding)
+      return it + 1, cost
+
+    _, cost = control_flow_ops.while_loop(
+        cond, body, [constant_op.constant(0),
+                     constant_op.constant(0.0)])
+    optimizer = momentum.MomentumOptimizer(0.1, 0.9)
+    train_op = optimizer.minimize(cost)
+    with self.cached_session():
+      self.evaluate(variables.global_variables_initializer())
+      for _ in range(10):
+        self.evaluate([train_op])
 
   def testResourceReadInLoop(self):
-    with ops.Graph().as_default():
-      embedding_matrix = variable_scope.get_variable(
-          "embedding_matrix",
-          initializer=[[2.0], [3.0]],
-          use_resource=True)
+    embedding_matrix = variable_scope.get_variable(
+        "embedding_matrix", initializer=[[2.0], [3.0]], use_resource=True)
 
-      def cond(it, _):
-        return it < 5
+    def cond(it, _):
+      return it < 5
 
-      def body(it, cost):
-        embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
-        cost += math_ops.reduce_sum(embedding)
-        return it + 1, cost
+    def body(it, cost):
+      embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
+      cost += math_ops.reduce_sum(embedding)
+      return it + 1, cost
 
-      _, cost = control_flow_ops.while_loop(
-          cond, body, [constant_op.constant(0),
-                       constant_op.constant(0.0)])
-      with self.cached_session() as sess:
-        self.evaluate(variables.global_variables_initializer())
-        self.assertAllEqual(10.0, self.evaluate(cost))
+    _, cost = control_flow_ops.while_loop(
+        cond, body, [constant_op.constant(0),
+                     constant_op.constant(0.0)])
+    with self.cached_session():
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllEqual(10.0, self.evaluate(cost))
 
   def doTestIndexedSlicesGradientInCondInWhileLoop(self, use_resource=False):
-    with ops.Graph().as_default():
-      embedding_matrix = variable_scope.get_variable(
-          "embedding_matrix", [5, 5],
-          initializer=init_ops.random_normal_initializer(),
-          use_resource=use_resource)
-
-      def cond(it, _):
-        return it < 5
-
-      def body(it, cost):
-        embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
-        cost = control_flow_ops.cond(
-            math_ops.equal(it, 3), lambda: math_ops.square(cost),
-            lambda: cost + math_ops.reduce_sum(embedding))
-        return it + 1, cost
+    embedding_matrix = variable_scope.get_variable(
+        "embedding_matrix", [5, 5],
+        initializer=init_ops.random_normal_initializer(),
+        use_resource=use_resource)
+
+    def cond(it, _):
+      return it < 5
+
+    def body(it, cost):
+      embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
+      cost = control_flow_ops.cond(
+          math_ops.equal(it, 3), lambda: math_ops.square(cost),
+          (lambda: cost + math_ops.reduce_sum(embedding)))
+      return it + 1, cost
 
       _, cost = control_flow_ops.while_loop(
           cond, body, [constant_op.constant(0),
@@ -269,7 +265,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
       static_grads = math_ops.segment_sum(static_grads.values,
                                           static_grads.indices)
 
-      with self.cached_session() as sess:
+      with self.cached_session():
         self.evaluate(variables.global_variables_initializer())
         self.assertAllEqual(*self.evaluate([static_grads, dynamic_grads]))
 
@@ -279,6 +275,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
   def testIndexedSlicesGradientInCondInWhileLoopResource(self):
     self.doTestIndexedSlicesGradientInCondInWhileLoop(use_resource=True)
 
+  @test_util.run_v1_only("b/120545219")
   def testIndexedSlicesWithShapeGradientInWhileLoop(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session() as sess:
@@ -308,6 +305,7 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
         self.assertEquals(o, 20)
         self.assertAllEqual(grad, [1] * num_steps)
 
+  @test_util.run_v1_only("b/120545219")
   def testIndexedSlicesWithDynamicShapeGradientInWhileLoop(self):
     for dtype in [dtypes.float32, dtypes.float64]:
       with self.cached_session() as sess:
@@ -335,105 +333,94 @@ class SwitchTestCase(test_util.TensorFlowTestCase):
         self.assertEquals(o, 6)
         self.assertAllEqual(grad, [1] * 3)
 
+  @test_util.run_deprecated_v1
   def testGradientThroughSingleBranchOutsideOfContext(self):
-    with self.cached_session():
-      x = constant_op.constant(2.)
-      s = constant_op.constant(True)
-      x_false, x_true = control_flow_ops.switch(x, s)
-      grad_x_true = gradients_impl.gradients(x_true, x)[0]
-      grad_x_false = gradients_impl.gradients(x_false, x)[0]
-      self.assertEquals(grad_x_true.eval(), 1.)
-      self.assertEquals(grad_x_false.eval(), 0.)
+    x = constant_op.constant(2.)
+    s = constant_op.constant(True)
+    x_false, x_true = control_flow_ops.switch(x, s)
+    grad_x_true = gradients_impl.gradients(x_true, x)[0]
+    grad_x_false = gradients_impl.gradients(x_false, x)[0]
+    self.assertEquals(self.evaluate(grad_x_true), 1.)
+    self.assertEquals(self.evaluate(grad_x_false), 0.)
 
 
 class CondTest(test_util.TensorFlowTestCase):
 
   def testCondTrue(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(2)
-        y = constant_op.constant(5)
-        z = control_flow_ops.cond(
-            math_ops.less(x, y), lambda: math_ops.multiply(x, 17),
-            lambda: math_ops.add(y, 23))
-        self.assertEquals(z.eval(), 34)
+    x = constant_op.constant(2)
+    y = constant_op.constant(5)
+    z = control_flow_ops.cond(
+        math_ops.less(
+            x,
+            y), lambda: math_ops.multiply(x, 17), lambda: math_ops.add(y, 23))
+    self.assertEquals(self.evaluate(z), 34)
 
   def testCondFalse(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(2)
-        y = constant_op.constant(1)
-        z = control_flow_ops.cond(
-            math_ops.less(x, y), lambda: math_ops.multiply(x, 17),
-            lambda: math_ops.add(y, 23))
-        self.assertEquals(z.eval(), 24)
+    x = constant_op.constant(2)
+    y = constant_op.constant(1)
+    z = control_flow_ops.cond(
+        math_ops.less(
+            x,
+            y), lambda: math_ops.multiply(x, 17), lambda: math_ops.add(y, 23))
+    self.assertEquals(self.evaluate(z), 24)
 
   def testCondTrueLegacy(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(2)
-        y = constant_op.constant(5)
-        z = control_flow_ops.cond(
-            math_ops.less(x, y), fn1=lambda: math_ops.multiply(x, 17),
-            fn2=lambda: math_ops.add(y, 23))
-        self.assertEquals(z.eval(), 34)
+    x = constant_op.constant(2)
+    y = constant_op.constant(5)
+    z = control_flow_ops.cond(
+        math_ops.less(x, y),
+        fn1=lambda: math_ops.multiply(x, 17),
+        fn2=lambda: math_ops.add(y, 23))
+    self.assertEquals(self.evaluate(z), 34)
 
   def testCondFalseLegacy(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(2)
-        y = constant_op.constant(1)
-        z = control_flow_ops.cond(
-            math_ops.less(x, y), fn1=lambda: math_ops.multiply(x, 17),
-            fn2=lambda: math_ops.add(y, 23))
-        self.assertEquals(z.eval(), 24)
-
+    x = constant_op.constant(2)
+    y = constant_op.constant(1)
+    z = control_flow_ops.cond(
+        math_ops.less(x, y),
+        fn1=lambda: math_ops.multiply(x, 17),
+        fn2=lambda: math_ops.add(y, 23))
+    self.assertEquals(self.evaluate(z), 24)
+
+  @test_util.run_deprecated_v1
   def testCondModifyBoolPred(self):
     # This test in particular used to fail only when running in GPU, hence
     # use_gpu=True.
-    with ops.Graph().as_default():
-      with session.Session() as sess:
-        bool_var = variable_scope.get_variable("bool_var", dtype=dtypes.bool,
-                                               initializer=True)
-        cond_on_bool_var = control_flow_ops.cond(
-            pred=bool_var,
-            true_fn=lambda: state_ops.assign(bool_var, False),
-            false_fn=lambda: True)
-        self.evaluate(bool_var.initializer)
-        self.assertEquals(self.evaluate(cond_on_bool_var), False)
-        self.assertEquals(self.evaluate(cond_on_bool_var), True)
+    with test_util.use_gpu():
+      bool_var = variable_scope.get_variable(
+          "bool_var", dtype=dtypes.bool, initializer=True)
+      cond_on_bool_var = control_flow_ops.cond(
+          pred=bool_var,
+          true_fn=lambda: state_ops.assign(bool_var, False),
+          false_fn=lambda: True)
+      self.evaluate(bool_var.initializer)
+      self.assertEquals(self.evaluate(cond_on_bool_var), False)
+      self.assertEquals(self.evaluate(cond_on_bool_var), True)
 
   def testCondMissingArg1(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(1)
-        with self.assertRaises(TypeError):
-          control_flow_ops.cond(True, false_fn=lambda: x)
+    x = constant_op.constant(1)
+    with self.assertRaises(TypeError):
+      control_flow_ops.cond(True, false_fn=lambda: x)
 
   def testCondMissingArg2(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(1)
-        with self.assertRaises(TypeError):
-          control_flow_ops.cond(True, lambda: x)
+    x = constant_op.constant(1)
+    with self.assertRaises(TypeError):
+      control_flow_ops.cond(True, lambda: x)
 
   def testCondDuplicateArg1(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(1)
-        with self.assertRaises(TypeError):
-          control_flow_ops.cond(True, lambda: x, lambda: x, fn1=lambda: x)
+    x = constant_op.constant(1)
+    with self.assertRaises(TypeError):
+      control_flow_ops.cond(True, lambda: x, lambda: x, fn1=lambda: x)
 
   def testCondDuplicateArg2(self):
-    with ops.Graph().as_default():
-      with session.Session():
-        x = constant_op.constant(1)
-        with self.assertRaises(TypeError):
-          control_flow_ops.cond(True, lambda: x, lambda: x, fn2=lambda: x)
+    x = constant_op.constant(1)
+    with self.assertRaises(TypeError):
+      control_flow_ops.cond(True, lambda: x, lambda: x, fn2=lambda: x)
 
 
 class ContextTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testCondContext(self):
     with self.cached_session() as sess:
       x = constant_op.constant(2)
@@ -463,12 +450,15 @@ class ContextTest(test_util.TensorFlowTestCase):
               control_flow_ops.WhileContext.from_proto(
                   control_flow_context.to_proto()).to_proto())
 
+  @test_util.run_deprecated_v1
   def testWhileContext(self):
     self._testWhileContextHelper()
 
+  @test_util.run_deprecated_v1
   def testWhileContextWithMaximumIterations(self):
     self._testWhileContextHelper(maximum_iterations=10)
 
+  @test_util.run_deprecated_v1
   def testControlContextImportScope(self):
     class NoABCControlFlowContext(control_flow_ops.ControlFlowContext):
       """A noop wrapper around `ControlFlowContext`.
@@ -591,6 +581,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
       if check_cond:
         self.assertAllEqualNested(result_case, expected_value_false)
 
+  @test_util.run_deprecated_v1
   def test_int(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: 1
@@ -600,6 +591,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape, strict=True)
     self._testReturnValues(fn_true, fn_false, 1, 2, strict=True)
 
+  @test_util.run_deprecated_v1
   def test_float(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: 1.0
@@ -607,12 +599,14 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape)
     self._testReturnValues(fn_true, fn_false, 1.0, 2.0)
 
+  @test_util.run_deprecated_v1
   def test_noop(self):
     shape = tensor_shape.TensorShape(None)
     self._testShape(control_flow_ops.no_op, control_flow_ops.no_op, shape)
     self._testReturnValues(control_flow_ops.no_op, control_flow_ops.no_op,
                            True, False, check_cond=False)
 
+  @test_util.run_deprecated_v1
   def test_string(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: "abc"
@@ -620,6 +614,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape)
     self._testReturnValues(fn_true, fn_false, b"abc", b"xyz")
 
+  @test_util.run_deprecated_v1
   def test_variable(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: variables.Variable(3.0)
@@ -627,6 +622,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape)
     self._testReturnValues(fn_true, fn_false, 3.0, 4.0)
 
+  @test_util.run_v1_only("b/120553181")
   def test_none(self):
     fn_none = lambda: None
     fn_tensor = lambda: constant_op.constant(1)
@@ -637,6 +633,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     with self.assertRaises(ValueError):
       control_flow_ops.cond(constant_op.constant(True), fn_tensor, fn_none)
 
+  @test_util.run_deprecated_v1
   def test_tensors(self):
 
     def _build_true_branch(dtype):
@@ -665,6 +662,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
                              (np.zeros([2, 2]), np.ones([3, 3])),
                              (np.ones([2, 2]), np.zeros([3, 3])))
 
+  @test_util.run_deprecated_v1
   def test_tensors_unknown_shape(self):
 
     def _build_true_branch(dtype):
@@ -693,6 +691,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
                              feed_dict={true_tensor: np.zeros([2, 2]),
                                         false_tensor: np.ones([2, 2])})
 
+  @test_util.run_deprecated_v1
   def test_sparse_tensors(self):
     shape = tensor_shape.TensorShape([None, None])
 
@@ -708,11 +707,14 @@ class DataTypesTest(test_util.TensorFlowTestCase):
                                              values=[1, 2], dense_shape=[3, 4])
     value2 = sparse_tensor.SparseTensorValue(indices=[[0, 0], [2, 1]],
                                              values=[3, 4], dense_shape=[3, 4])
-    self._testShape(true_fn, false_fn, shape)
-    self._testReturnValues(true_fn, false_fn, value1, value2)
+    # Non-strict cond is only available in v1
+    if not tf2.enabled():
+      self._testShape(true_fn, false_fn, shape)
+      self._testReturnValues(true_fn, false_fn, value1, value2)
     self._testShape(true_fn, false_fn, [shape], strict=True)
     self._testReturnValues(true_fn, false_fn, [value1], [value2], strict=True)
 
+  @test_util.run_deprecated_v1
   def test_tensors_with_partially_specified_shapes(self):
 
     def _build_branch(dtype, shape):
@@ -742,6 +744,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
                                         true_tensors[2]: np.ones([3, 3]),
                                         false_tensors[2]: np.ones([3, 3])})
 
+  @test_util.run_deprecated_v1
   def test_tensor_arrays(self):
     element_shape = tensor_shape.TensorShape([2])
     ta1 = _create_tensor_array(4, element_shape)
@@ -751,6 +754,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     fn_false = lambda: ta2
     self._testShape(fn_true, fn_false, shape)
 
+  @test_util.run_deprecated_v1
   def test_tensor_array_reads(self):
     shape = tensor_shape.TensorShape([2])
     ta = _create_tensor_array(4, shape)
@@ -758,6 +762,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     fn_false = lambda: ta.read(1)
     self._testShape(fn_true, fn_false, shape)
 
+  @test_util.run_deprecated_v1
   def test_list(self):
     shape = [tensor_shape.TensorShape([]), tensor_shape.TensorShape([]),
              tensor_shape.TensorShape([])]
@@ -766,6 +771,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape)
     self._testReturnValues(fn_true, fn_false, [1, 2, 3.0], [3, 4, 5.0])
 
+  @test_util.run_v1_only("Non-strict cond is only available in v1")
   def test_non_strict(self):
     shape = tensor_shape.TensorShape([])
     fn_tensor = lambda: constant_op.constant(1)
@@ -778,6 +784,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testReturnValues(fn_tensor, fn_tuple, 1, 3)
     self._testReturnValues(fn_list, fn_tuple, 2, 3)
 
+  @test_util.run_v1_only("b/120553181")
   def test_singleton_strict(self):
     fn_tensor = lambda: constant_op.constant(1)
     fn_list = lambda: [constant_op.constant(2)]
@@ -799,36 +806,46 @@ class DataTypesTest(test_util.TensorFlowTestCase):
       control_flow_ops.case([(constant_op.constant(True), fn_list)], fn_tuple,
                             strict=True)
 
+  @test_util.run_deprecated_v1
   def test_singleton_list(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: [constant_op.constant(1)]
     fn_false = lambda: [constant_op.constant(3)]
-    self._testShape(fn_true, fn_false, shape)
-    self._testReturnValues(fn_true, fn_false, 1, 3)
+    # Non-strict cond is only available in v1
+    if not tf2.enabled():
+      self._testShape(fn_true, fn_false, shape)
+      self._testReturnValues(fn_true, fn_false, 1, 3)
     self._testShape(fn_true, fn_false, [shape], strict=True)
     self._testReturnValues(fn_true, fn_false, [1], [3], strict=True)
 
+  @test_util.run_deprecated_v1
   def test_singleton_tuple(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: (constant_op.constant(1),)
     fn_false = lambda: (constant_op.constant(3),)
-    self._testShape(fn_true, fn_false, shape)
-    self._testReturnValues(fn_true, fn_false, 1, 3)
+    # Non-strict cond is only available in v1
+    if not tf2.enabled():
+      self._testShape(fn_true, fn_false, shape)
+      self._testReturnValues(fn_true, fn_false, 1, 3)
     self._testShape(fn_true, fn_false, (shape,), strict=True)
     self._testReturnValues(fn_true, fn_false, (1,), (3,),
                            strict=True)
 
+  @test_util.run_deprecated_v1
   def test_singleton_namedtuple(self):
     shape = tensor_shape.TensorShape([])
     fn_true = lambda: SingletonTestTuple(constant_op.constant(1))
     fn_false = lambda: SingletonTestTuple(constant_op.constant(3))
-    self._testShape(fn_true, fn_false, shape)
-    self._testReturnValues(fn_true, fn_false, 1, 3)
+    # Non-strict cond is only available in v1
+    if not tf2.enabled():
+      self._testShape(fn_true, fn_false, shape)
+      self._testReturnValues(fn_true, fn_false, 1, 3)
     self._testShape(fn_true, fn_false, SingletonTestTuple(shape),
                     strict=True)
     self._testReturnValues(fn_true, fn_false, SingletonTestTuple(1),
                            SingletonTestTuple(3), strict=True)
 
+  @test_util.run_deprecated_v1
   def test_tuple(self):
     shape = (tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))
     fn_true = lambda: (constant_op.constant(1), 2)
@@ -836,6 +853,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape)
     self._testReturnValues(fn_true, fn_false, (1, 2), (3, 4))
 
+  @test_util.run_deprecated_v1
   def test_namedtuple(self):
     shape = TestTuple(tensor_shape.TensorShape([]),
                       tensor_shape.TensorShape([]))
@@ -844,6 +862,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
     self._testShape(fn_true, fn_false, shape)
     self._testReturnValues(fn_true, fn_false, TestTuple(1, 2), TestTuple(3, 4))
 
+  @test_util.run_deprecated_v1
   def test_nested(self):
     shape = [tensor_shape.TensorShape([]),
              TestTuple(tensor_shape.TensorShape([]),
@@ -869,6 +888,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
         [11, TestTuple(12, [13, 14]),
          np.ones([5, 5]), 16])
 
+  @test_util.run_deprecated_v1
   def test_cond_inside_while_loop(self):
 
     def body(i, matrix):
@@ -890,6 +910,7 @@ class DataTypesTest(test_util.TensorFlowTestCase):
 
 class CaseTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testCase_withDefault(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     conditions = [(math_ops.equal(x, 1), lambda: constant_op.constant(2)),
@@ -901,6 +922,7 @@ class CaseTest(test_util.TensorFlowTestCase):
       self.assertEqual(sess.run(output, feed_dict={x: 2}), 4)
       self.assertEqual(sess.run(output, feed_dict={x: 3}), 6)
 
+  @test_util.run_deprecated_v1
   def testCase_multiple_matches_exclusive(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     conditions = [(math_ops.equal(x, 1), lambda: constant_op.constant(2)),
@@ -914,6 +936,7 @@ class CaseTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "Input error:"):
         sess.run(output, feed_dict={x: 2})
 
+  @test_util.run_deprecated_v1
   def testCase_multiple_matches_non_exclusive(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     conditions = [(math_ops.equal(x, 1), lambda: constant_op.constant(2)),
@@ -926,6 +949,7 @@ class CaseTest(test_util.TensorFlowTestCase):
       self.assertEqual(sess.run(output, feed_dict={x: 2}), 4)
       self.assertEqual(sess.run(output, feed_dict={x: 3}), 8)
 
+  @test_util.run_deprecated_v1
   def testCase_withoutDefault(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     conditions = [(math_ops.equal(x, 1), lambda: constant_op.constant(2)),
@@ -939,6 +963,7 @@ class CaseTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "Input error:"):
         sess.run(output, feed_dict={x: 4})
 
+  @test_util.run_deprecated_v1
   def testCase_withoutDefault_oneCondition(self):
     x = array_ops.placeholder(dtype=dtypes.int32, shape=[])
     conditions = [(math_ops.equal(x, 1), lambda: constant_op.constant(2))]
@@ -980,6 +1005,7 @@ class WhileLoopTestCase(test_util.TensorFlowTestCase):
     # Expect a tuple since that is what the body returns.
     self.assertEqual(self.evaluate(r), (10,))
 
+  @test_util.run_deprecated_v1
   def testWhileLoopSameReturnShape_False(self):
     i = constant_op.constant(0)
     c = lambda i, _: math_ops.less(i, 10)
@@ -1005,6 +1031,7 @@ class WhileLoopTestCase(test_util.TensorFlowTestCase):
 
 class AssertTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_deprecated_v1
   def testAssert(self):
     i = constant_op.constant(0)
     c = control_flow_ops.Assert(i < 10, [i, [10], [i + 1]])
-- 
GitLab


From 49aa21b5b6468309e0acef82a6a15314ba8f5a8f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 11:24:44 -0800
Subject: [PATCH 1501/1554] Automated rollback of commit
 51561e0354a1556e848c920cd47a718360dc720d

PiperOrigin-RevId: 224548216
---
 .../python/estimator_training_test.py         |  7 ++--
 .../python/multi_worker_test_base.py          | 10 ++----
 .../distribute/distribute_coordinator.py      | 35 ++++++-------------
 3 files changed, 16 insertions(+), 36 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/estimator_training_test.py b/tensorflow/contrib/distribute/python/estimator_training_test.py
index 3f55a8a1c8..b369a7fefe 100644
--- a/tensorflow/contrib/distribute/python/estimator_training_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_training_test.py
@@ -375,13 +375,11 @@ class DistributeCoordinatorIntegrationTest(
     threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn,
                                                  cluster_spec, train_distribute,
                                                  eval_distribute)
-    threads_to_join = []
     for task_type, ts in threads.items():
       if task_type == PS:
         continue
       for t in ts:
-        threads_to_join.append(t)
-    self.join_independent_workers(threads_to_join)
+        t.join()
 
     estimator = self._get_estimator(train_distribute, eval_distribute)
     self._inspect_train_and_eval_events(estimator)
@@ -415,7 +413,8 @@ class DistributeCoordinatorIntegrationTest(
     threads = self.run_multiple_tasks_in_threads(self._independent_worker_fn,
                                                  cluster_spec, train_distribute,
                                                  eval_distribute)
-    self.join_independent_workers([threads[WORKER][0], threads[EVALUATOR][0]])
+    threads[WORKER][0].join()
+    threads[EVALUATOR][0].join()
 
     estimator = self._get_estimator(train_distribute, eval_distribute)
     self._inspect_train_and_eval_events(estimator)
diff --git a/tensorflow/contrib/distribute/python/multi_worker_test_base.py b/tensorflow/contrib/distribute/python/multi_worker_test_base.py
index b05aac431f..147c9b83f8 100644
--- a/tensorflow/contrib/distribute/python/multi_worker_test_base.py
+++ b/tensorflow/contrib/distribute/python/multi_worker_test_base.py
@@ -40,7 +40,6 @@ from tensorflow.python.client import session
 from tensorflow.python.estimator import run_config
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import coordinator
 from tensorflow.python.training import server_lib
 
 ASSIGNED_PORTS = set()
@@ -361,7 +360,6 @@ class IndependentWorkerTestBase(test.TestCase):
     self._mock_os_env = MockOsEnv()
     self._mock_context = test.mock.patch.object(os, 'environ',
                                                 self._mock_os_env)
-    self._coord = coordinator.Coordinator()
     super(IndependentWorkerTestBase, self).setUp()
     self._mock_context.__enter__()
 
@@ -370,9 +368,8 @@ class IndependentWorkerTestBase(test.TestCase):
     super(IndependentWorkerTestBase, self).tearDown()
 
   def _task_thread(self, task_fn, tf_config, *args, **kwargs):
-    with self._coord.stop_on_exception():
-      os.environ['TF_CONFIG'] = json.dumps(tf_config)
-      task_fn(*args, **kwargs)
+    os.environ['TF_CONFIG'] = json.dumps(tf_config)
+    task_fn(*args, **kwargs)
 
   def _run_task_in_thread(self, task_fn, cluster_spec, task_type, task_id,
                           *args, **kwargs):
@@ -406,6 +403,3 @@ class IndependentWorkerTestBase(test.TestCase):
                                      *args, **kwargs)
         threads[task_type].append(t)
     return threads
-
-  def join_independent_workers(self, worker_threads):
-    self._coord.join(worker_threads)
diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 78c995a578..c0f9b8a1fd 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -29,7 +29,6 @@ from tensorflow.python.client import session
 from tensorflow.python.distribute import distribute_coordinator_context
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import coordinator
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training import server_lib
 
@@ -329,8 +328,7 @@ def _run_single_worker(worker_fn,
                        task_id,
                        session_config,
                        rpc_layer="",
-                       worker_barrier=None,
-                       coord=None):
+                       worker_barrier=None):
   """Runs a single worker by calling `worker_fn` under context."""
   session_config = copy.deepcopy(session_config)
   strategy = copy.deepcopy(strategy)
@@ -352,11 +350,7 @@ def _run_single_worker(worker_fn,
       rpc_layer=rpc_layer,
       worker_barrier=worker_barrier)
   with context:
-    if coord:
-      with coord.stop_on_exception():
-        return worker_fn(strategy)
-    else:
-      return worker_fn(strategy)
+    return worker_fn(strategy)
 
 
 def _split_cluster_for_evaluator(cluster_spec, task_type):
@@ -429,7 +423,6 @@ def _run_std_server(cluster_spec=None,
 def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
                               cluster_spec, session_config, rpc_layer):
   """Runs a standalone client for between-graph replication."""
-  coord = coordinator.Coordinator()
   eval_thread = None
   if _TaskType.EVALUATOR in cluster_spec.jobs:
     eval_thread = threading.Thread(
@@ -438,7 +431,6 @@ def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
               session_config),
         kwargs={
             "rpc_layer": rpc_layer,
-            "coord": coord,
         })
     eval_thread.start()
 
@@ -452,18 +444,18 @@ def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
                 session_config),
           kwargs={
               "rpc_layer": rpc_layer,
-              "worker_barrier": worker_barrier,
-              "coord": coord,
+              "worker_barrier": worker_barrier
           })
       t.start()
       threads.append(t)
 
+  # TODO(yuefengz): wrap threads into thread coordinator?
+  for t in threads:
+    t.join()
+
+  # TODO(yuefengz): is it necessary to join eval thread?
   if eval_thread:
-    # TODO(yuefengz): is it necessary to join eval thread?
-    threads_to_join = threads + [eval_thread]
-  else:
-    threads_to_join = threads
-  coord.join(threads_to_join)
+    eval_thread.join()
 
   # TODO(yuefengz): we probably want to return results from all workers?
   return None
@@ -472,7 +464,6 @@ def _run_between_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
 def _run_in_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
                          cluster_spec, session_config, rpc_layer):
   """Runs a standalone client for in-graph replication."""
-  coord = coordinator.Coordinator()
   eval_thread = None
   if _TaskType.EVALUATOR in cluster_spec.jobs:
     eval_thread = threading.Thread(
@@ -481,7 +472,6 @@ def _run_in_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
               session_config),
         kwargs={
             "rpc_layer": rpc_layer,
-            "coord": coord,
         })
     eval_thread.start()
 
@@ -492,12 +482,9 @@ def _run_in_graph_client(worker_fn, strategy, eval_fn, eval_strategy,
       None,
       None,
       session_config,
-      rpc_layer=rpc_layer,
-      coord=coord)
-
+      rpc_layer=rpc_layer)
   if eval_thread:
-    coord.join([eval_thread])
-
+    eval_thread.join()
   return worker_result
 
 
-- 
GitLab


From 48a224c33e6cc258226e029a6eacb71aca0aa7b9 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 7 Dec 2018 12:27:20 -0800
Subject: [PATCH 1502/1554] Update documentation on string support for
 TensorFlow Lite

Strings are not compatible with ByteBuffer inputs/outputs. Make that
clear in the documentation, also highlighting how they differ from
primitive types.

PiperOrigin-RevId: 224559052
---
 tensorflow/lite/g3doc/apis.md                 |  7 ++++
 .../java/org/tensorflow/lite/Interpreter.java | 37 +++++++++++++++----
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/g3doc/apis.md b/tensorflow/lite/g3doc/apis.md
index b85570f41c..b15159ce41 100644
--- a/tensorflow/lite/g3doc/apis.md
+++ b/tensorflow/lite/g3doc/apis.md
@@ -304,6 +304,13 @@ one of the following primitive types:
 *   `long`
 *   `byte`
 
+`String` types are also supported, but they are encoded differently than the
+primitive types. In particular, the shape of a string Tensor dictates the number
+and arrangement of strings in the Tensor, with each element itself being a
+variable length string. In this sense, the (byte) size of the Tensor cannot be
+computed from the shape and type alone, and consequently strings cannot be
+provided as a single, flat `ByteBuffer` argument.
+
 If other data types, including boxed types like `Integer` and `Float`, are used,
 an `IllegalArgumentException` will be thrown.
 
diff --git a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
index a03d7b5676..2203d5fbdb 100644
--- a/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
+++ b/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
@@ -43,15 +43,34 @@ import org.checkerframework.checker.nullness.qual.NonNull;
  * <pre>{@code
  * Object[] inputs = {input0, input1, ...};
  * Map<Integer, Object> map_of_indices_to_outputs = new HashMap<>();
- * float[][][] ith_output = new float[3][2][4];
+ * ByteBuffer ith_output = ByteBuffer.allocateDirect(3 * 2 * 4 * 4);  // Float tensor, shape 3x2x4.
+ * ith_output.order(ByteOrder.nativeOrder());
  * map_of_indices_to_outputs.put(i, ith_output);
  * try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
  *   interpreter.runForMultipleInputsOutputs(inputs, map_of_indices_to_outputs);
  * }
  * }</pre>
  *
+ * <p>If a model takes or produces string tensors:
+ *
+ * <pre>{@code
+ * String[] input = {"foo", "bar"};  // Input tensor shape is [2].
+ * String[] output = new String[3][2];  // Output tensor shape is [3, 2].
+ * try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) {
+ *   interpreter.runForMultipleInputsOutputs(input, output);
+ * }
+ * }</pre>
+ *
  * <p>Orders of inputs and outputs are determined when converting TensorFlow model to TensorFlowLite
- * model with Toco.
+ * model with Toco, as are the default shapes of the inputs.
+ *
+ * <p>When inputs are provided as (multi-dimensional) arrays, the corresponding input tensor(s) will
+ * be implicitly resized according to that array's shape. When inputs are provided as {@link
+ * ByteBuffer} types, no implicit resizing is done; the caller must ensure that the {@link
+ * ByteBuffer} byte size either matches that of the corresponding tensor, or that they first resize
+ * the tensor via {@link #resizeInput()}. Tensor shape and type information can be obtained via the
+ * {@link Tensor} class, available via {@link #getInputTensor(int)} and {@link
+ * #getOutputTensor(int)}.
  *
  * <p><b>WARNING:</b>Instances of a {@code Interpreter} is <b>not</b> thread-safe. A {@code
  * Interpreter} owns resources that <b>must</b> be explicitly freed by invoking {@link #close()}
@@ -192,12 +211,13 @@ public final class Interpreter implements AutoCloseable {
    * Runs model inference if the model takes only one input, and provides only one output.
    *
    * <p>Warning: The API runs much faster if {@link ByteBuffer} is used as input data type. Please
-   * consider using {@link ByteBuffer} to feed input data for better performance.
+   * consider using {@link ByteBuffer} to feed primitive input data for better performance.
    *
    * @param input an array or multidimensional array, or a {@link ByteBuffer} of primitive types
    *     including int, float, long, and byte. {@link ByteBuffer} is the preferred way to pass large
-   *     input data. When {@link ByteBuffer} is used, its content should remain unchanged until
-   *     model inference is done.
+   *     input data for primitive types, whereas string types require using the (multi-dimensional)
+   *     array input path. When {@link ByteBuffer} is used, its content should remain unchanged
+   *     until model inference is done.
    * @param output a multidimensional array of output data, or a {@link ByteBuffer} of primitive
    *     types including int, float, long, and byte.
    */
@@ -212,13 +232,14 @@ public final class Interpreter implements AutoCloseable {
    * Runs model inference if the model takes multiple inputs, or returns multiple outputs.
    *
    * <p>Warning: The API runs much faster if {@link ByteBuffer} is used as input data type. Please
-   * consider using {@link ByteBuffer} to feed input data for better performance.
+   * consider using {@link ByteBuffer} to feed primitive input data for better performance.
    *
    * @param inputs an array of input data. The inputs should be in the same order as inputs of the
    *     model. Each input can be an array or multidimensional array, or a {@link ByteBuffer} of
    *     primitive types including int, float, long, and byte. {@link ByteBuffer} is the preferred
-   *     way to pass large input data. When {@link ByteBuffer} is used, its content should remain
-   *     unchanged until model inference is done.
+   *     way to pass large input data, whereas string types require using the (multi-dimensional)
+   *     array input path. When {@link ByteBuffer} is used, its content should remain unchanged
+   *     until model inference is done.
    * @param outputs a map mapping output indices to multidimensional arrays of output data or {@link
    *     ByteBuffer}s of primitive types including int, float, long, and byte. It only needs to keep
    *     entries for the outputs to be used.
-- 
GitLab


From 9fffd26041e40bb3c89eabebfd35f92fe37b2c2a Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 7 Dec 2018 12:39:30 -0800
Subject: [PATCH 1503/1554] Support XLA_CPU/GPU devices in eager runtime

Compiling a defun can be done with:

  @function.defun
  def f(x):
    return x + 1

  with tf.device("XLA_GPU"):
    f(5)

PiperOrigin-RevId: 224560868
---
 tensorflow/compiler/jit/BUILD                      |  2 ++
 .../compiler/jit/xla_compile_on_demand_op.cc       | 14 +++++++++++++-
 tensorflow/compiler/jit/xla_cpu_device.cc          | 14 +++++++++++++-
 tensorflow/compiler/tests/BUILD                    |  7 -------
 tensorflow/core/common_runtime/eager/execute.cc    |  3 ++-
 .../core/common_runtime/eager/tensor_handle.cc     |  5 +----
 tensorflow/python/framework/func_graph.py          |  4 +++-
 7 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 7ebcd120f6..15dcbb2641 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -76,6 +76,7 @@ cc_library(
     srcs = ["xla_cpu_device.cc"],
     visibility = [":friends"],
     deps = [
+        ":create_xla_launch_op",  # buildcleaner: keep
         ":flags",
         ":jit_compilation_passes",
         ":xla_device",
@@ -95,6 +96,7 @@ cc_library(
     srcs = ["xla_gpu_device.cc"],
     visibility = [":friends"],
     deps = [
+        ":create_xla_launch_op",  # buildcleaner: keep
         ":jit_compilation_passes",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_ops",
diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
index 1fe612d43d..c7e8d61d28 100644
--- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
+++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
@@ -142,11 +142,22 @@ Status XlaCompileOnDemandOp::Compile(
         TF_RETURN_IF_ERROR(ctx->allocate_temp(
             device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs));
         Notification n;
+        Status status;
         ctx->op_device_context()->CopyDeviceTensorToCPU(
             &device_tensor, "ConstantArgument",
             reinterpret_cast<Device*>(ctx->device()), &host_tensor,
-            [&](Status status) { n.Notify(); });
+            [&](Status s) {
+              status = s;
+              n.Notify();
+            });
         n.WaitForNotification();
+        if (!status.ok()) {
+          LOG(ERROR) << "Copying tensor of shape "
+                     << device_tensor.shape().DebugString() << " from "
+                     << ctx->device()->name() << "to CPU failed with "
+                     << status.ToString();
+          return status;
+        }
         constant_arguments[i] = host_tensor;
       }
     }
@@ -189,6 +200,7 @@ Status XlaCompileOnDemandOp::Compile(
   std::map<int, OptionalTensor> variable_args = GetVariables(ctx);
 
   std::vector<XlaCompiler::Argument> args;
+
   TF_RETURN_IF_ERROR(XlaComputationLaunchContext::BuildXlaCompilerArguments(
       constant_arguments, variable_args, ctx, &args));
 
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 7df898ad12..e9770647e7 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -63,7 +63,19 @@ Status XlaCpuDeviceFactory::CreateDevices(
   options.device_ordinal = 0;
   options.compilation_device_name = DEVICE_CPU_XLA_JIT;
   options.use_multiple_streams = false;
-  devices->push_back(absl::make_unique<XlaDevice>(session_options, options));
+  auto device = absl::make_unique<XlaDevice>(session_options, options);
+
+  // Setting GpuDeviceInfo because eager runtime relies on the device
+  // context in tensorflow_gpu_device_info(). Also,
+  // tensorflow_gpu_device_info() == nullptr is used as an IsCPU test.
+  // We need XlaCpuDevice to be treated not as CPU because it allocates
+  // XlaTensors, not regular Tensors.
+  Status status = device->UseGpuDeviceInfo();
+  if (!status.ok()) {
+    errors::AppendToMessage(&status, "while setting up ", DEVICE_GPU_XLA_JIT);
+    return status;
+  }
+  devices->push_back(std::move(device));
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index bc3d60b90e..093b61629c 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -408,13 +408,6 @@ tf_xla_py_test(
     name = "eager_test",
     size = "large",
     srcs = ["eager_test.py"],
-    disabled_backends = [
-        # TODO(b/78199195) Support XLA CPU devices in eager runtime
-        "cpu",
-        "cpu_ondemand",
-        # TODO(b/78468222) Enable GPU backend
-        "gpu",
-    ],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index a6199f2aeb..783baa96c9 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -263,7 +263,8 @@ Status EagerLocalExecute(EagerOperation* op,
     // Note that it is not ideal, but currently ok, to set this
     // attribute after computing the kernel cache key above.
     if (op->is_function() && device != nullptr &&
-        device->device_type() == "TPU") {
+        (device->device_type() == "TPU" || device->device_type() == "XLA_GPU" ||
+         device->device_type() == "XLA_CPU")) {
       op->MutableAttrs()->Set(kXlaCompileAttr, true);
     }
 
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index d8d6b7a63b..0acd160936 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -184,10 +184,7 @@ Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
   bool is_same_device = (srcd == dstd) || (srcd->name() == dstd->name());
   const bool dst_cpu = dstd->tensorflow_gpu_device_info() == nullptr;
   const bool src_cpu = srcd->tensorflow_gpu_device_info() == nullptr;
-  // both_on_cpu can be true and yet is_same_device is false, if one of src/dst
-  // has device type XLA_CPU, and the other CPU.
-  const bool both_on_cpu = src_cpu && dst_cpu;
-  if (is_same_device || both_on_cpu) {
+  if (is_same_device) {
     *output = new tensorflow::TensorHandle(*src, dstd, dstd, ctx);
     return tensorflow::Status::OK();
   }
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 0f6db500cc..bd4ed5553e 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -122,7 +122,9 @@ class FuncGraph(ops.Graph):
     # restored.
     if context.executing_eagerly():
       self.seed = context.global_seed()
-      self._xla_compile = (context.context().device_spec.device_type == "TPU")
+      device_type = context.context().device_spec.device_type
+      self._xla_compile = (device_type == "TPU" or device_type == "XLA_GPU"
+                           or device_type == "XLA_CPU")
       if self._distribution_strategy_stack or self._xla_compile:
         self._add_device_to_stack(context.context().device_name)
     else:
-- 
GitLab


From 4f543e588a99caeb8265372127b889a4f67edba4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 12:39:53 -0800
Subject: [PATCH 1504/1554] Fix NVPTX TargetMachine construction to use
 reloc/code model flags only if explicitly set. Fixes failures caused by
 r348585.

PiperOrigin-RevId: 224560930
---
 .../xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
index abf9e7b6d6..bd53b90b42 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
@@ -199,8 +199,7 @@ std::unique_ptr<llvm::TargetMachine> GetTargetMachine(
   }
   return absl::WrapUnique(target->createTargetMachine(
       triple.str(), llvm_ir::AsStringRef(cpu_name), "+ptx60", target_options,
-      Optional<Reloc::Model>(RelocModel), Optional<CodeModel::Model>(CMModel),
-      codegen_opt_level));
+      getRelocModel(), getCodeModel(), codegen_opt_level));
 }
 
 // Adds the standard LLVM optimization passes, based on the speed optimization
-- 
GitLab


From 66ca3cd10df0bf9bb6586bf0a09ac5c5ed0a25fb Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 7 Dec 2018 12:40:49 -0800
Subject: [PATCH 1505/1554] Add a functional saver, use it for object-based
 checkpointing

Pulls some utilities out of saver.py which are necessary to actually use it. The functional saver takes only SaveableObjects, so these are utilities for taking a list of whatever users pass in and converting them to those.

One other code move for object-based checkpointing to avoid circular imports.

Applications which need a SaverDef still use the old Saver. Serialization to SaverDef will be added to this saver in a followup.

Does not actually wrap the new Saver's methods in @tf.function yet, since there are memory issues which need to be fixed first.

PiperOrigin-RevId: 224561069
---
 .../training/elastic_average_optimizer.py     |   5 +-
 .../training/moving_average_optimizer.py      |   3 +-
 tensorflow/python/BUILD                       |  11 +-
 .../python/training/checkpoint_utils.py       |   6 +-
 .../python/training/checkpointable/BUILD      |   6 +-
 .../python/training/checkpointable/base.py    |  40 +--
 .../python/training/checkpointable/util.py    | 151 +++++---
 tensorflow/python/training/saver.py           | 331 ++---------------
 tensorflow/python/training/saving/BUILD       |  55 +++
 .../training/saving/functional_saver.py       | 101 ++++++
 .../training/saving/functional_saver_test.py  |  50 +++
 .../training/{ => saving}/saveable_object.py  |   0
 .../training/saving/saveable_object_util.py   | 340 ++++++++++++++++++
 .../python/training/warm_starting_util.py     |   4 +-
 14 files changed, 687 insertions(+), 416 deletions(-)
 create mode 100644 tensorflow/python/training/saving/BUILD
 create mode 100644 tensorflow/python/training/saving/functional_saver.py
 create mode 100644 tensorflow/python/training/saving/functional_saver_test.py
 rename tensorflow/python/training/{ => saving}/saveable_object.py (100%)
 create mode 100644 tensorflow/python/training/saving/saveable_object_util.py

diff --git a/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py b/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py
index 6c203e5519..fa1a7aaff0 100644
--- a/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/elastic_average_optimizer.py
@@ -30,6 +30,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.training import optimizer
 from tensorflow.python.training import saver
 from tensorflow.python.training import session_run_hook
+from tensorflow.python.training.saving import saveable_object_util
 
 LOCAL_VARIABLE_NAME = 'local_center_variable'
 GLOBAL_VARIABLE_NAME = 'global_center_variable'
@@ -424,7 +425,7 @@ class ElasticAverageOptimizer(optimizer.Optimizer):
     if var_list is None:
       var_list = variables.trainable_variables()
     if not isinstance(var_list, dict):
-      var_list = saver.BaseSaverBuilder.OpListToDict(var_list)
+      var_list = saveable_object_util.op_list_to_dict(var_list)
 
     swapped_var_list = {}
     for key, var in var_list.items():
@@ -464,4 +465,4 @@ class _ElasticAverageOptimizerHook(session_run_hook.SessionRunHook):
 
   def after_create_session(self, session, coord):
     """Run initialization ops"""
-    session.run(self._variable_init_op)
\ No newline at end of file
+    session.run(self._variable_init_op)
diff --git a/tensorflow/contrib/opt/python/training/moving_average_optimizer.py b/tensorflow/contrib/opt/python/training/moving_average_optimizer.py
index b7fd2d2fb9..bf3e5c51f7 100644
--- a/tensorflow/contrib/opt/python/training/moving_average_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/moving_average_optimizer.py
@@ -26,6 +26,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.training import moving_averages
 from tensorflow.python.training import optimizer
 from tensorflow.python.training import saver
+from tensorflow.python.training.saving import saveable_object_util
 
 
 class MovingAverageOptimizer(optimizer.Optimizer):
@@ -165,7 +166,7 @@ class MovingAverageOptimizer(optimizer.Optimizer):
     if var_list is None:
       var_list = variables.global_variables()
     if not isinstance(var_list, dict):
-      var_list = saver.BaseSaverBuilder.OpListToDict(var_list)
+      var_list = saveable_object_util.op_list_to_dict(var_list)
 
     v_name_to_tensor = {}
     for k, tensor_or_list in six.iteritems(var_list):
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index cc36f1fc0e..0a3ee65bc4 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3515,13 +3515,13 @@ py_library(
         exclude = [
             "**/*test*",
             "training/checkpointable/**/*.py",
+            "training/saving/**/*.py",
             # The following targets have their own build rules (same name as the
             # file):
             "training/basic_session_run_hooks.py",
             "training/checkpoint_management.py",
             "training/distribute.py",
             "training/distribution_strategy_context.py",
-            "training/saveable_object.py",
             "training/saver.py",
             "training/session_run_hook.py",
             "training/training_util.py",
@@ -3596,12 +3596,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "saveable_object",
-    srcs = ["training/saveable_object.py"],
-    srcs_version = "PY2AND3",
-)
-
 py_library(
     name = "checkpoint_management",
     srcs = ["training/checkpoint_management.py"],
@@ -3655,7 +3649,6 @@ py_library(
         ":platform",
         ":pywrap_tensorflow",
         ":resource_variable_ops",
-        ":saveable_object",
         ":session",
         ":state_ops",
         ":string_ops",
@@ -3665,6 +3658,8 @@ py_library(
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/training/checkpointable:base",
+        "//tensorflow/python/training/saving:saveable_object",
+        "//tensorflow/python/training/saving:saveable_object_util",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 99b1f4c0d7..74b46179e7 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -30,7 +30,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import saver
+from tensorflow.python.training.saving import saveable_object_util
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -311,10 +311,10 @@ def _set_checkpoint_initializer(variable,
     restore_op = io_ops.restore_v2(
         ckpt_file, [tensor_name], [slice_spec], [base_type], name=name)[0]
 
-    names_to_saveables = saver.BaseSaverBuilder.OpListToDict([variable])
+    names_to_saveables = saveable_object_util.op_list_to_dict([variable])
     saveable_objects = []
     for name, op in names_to_saveables.items():
-      for s in saver.BaseSaverBuilder.SaveableObjectsForOp(op, name):
+      for s in saveable_object_util.saveable_objects_for_op(op, name):
         saveable_objects.append(s)
 
     assert len(saveable_objects) == 1  # Should be only one variable.
diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index 4ab5593d4f..26a0ac35b7 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -25,9 +25,9 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:io_ops_gen",
         "//tensorflow/python:platform",
-        "//tensorflow/python:saveable_object",
         "//tensorflow/python:util",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/training/saving:saveable_object",
     ],
 )
 
@@ -114,7 +114,6 @@ py_library(
         "//tensorflow/python:init_ops",
         "//tensorflow/python:io_ops_gen",
         "//tensorflow/python:pywrap_tensorflow",
-        "//tensorflow/python:saveable_object",
         "//tensorflow/python:saver",
         "//tensorflow/python:session",
         "//tensorflow/python:tensor_shape",
@@ -123,6 +122,9 @@ py_library(
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/training/saving:functional_saver",
+        "//tensorflow/python/training/saving:saveable_object",
+        "//tensorflow/python/training/saving:saveable_object_util",
     ],
 )
 
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index 095a90ddd4..3cd1c6f9c8 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -25,7 +25,6 @@ import weakref
 
 import six
 
-from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -34,7 +33,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_io_ops as io_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import saveable_object
+from tensorflow.python.training.saving import saveable_object
 from tensorflow.python.util import nest
 from tensorflow.python.util import serialization
 from tensorflow.python.util import tf_decorator
@@ -374,41 +373,10 @@ class _CheckpointPosition(object):
       eagerly.
     """
     (restore_ops,
-     named_saveables,
+     tensor_saveables,
      python_saveables) = self._gather_ops_or_named_saveables()
-
-    # Eagerly run restorations for Python state.
-    reader = pywrap_tensorflow.NewCheckpointReader(
-        self._checkpoint.save_path_string)
-    for saveable in python_saveables:
-      spec_names = [spec.name for spec in saveable.specs]
-      saveable.python_restore(
-          [reader.get_tensor(name) for name in spec_names])
-
-    # If we have new SaveableObjects, extract and cache restore ops.
-    if named_saveables:
-      validated_saveables = (
-          self._checkpoint.builder._ValidateAndSliceInputs(named_saveables))  # pylint: disable=protected-access
-      validated_names = set(saveable.name for saveable in validated_saveables)
-      if set(named_saveables.keys()) != validated_names:
-        raise AssertionError(
-            ("Saveable keys changed when validating. Got back %s, was "
-             "expecting %s") % (named_saveables.keys(), validated_names))
-      all_tensors = self._checkpoint.builder.bulk_restore(
-          filename_tensor=self._checkpoint.save_path_tensor,
-          saveables=validated_saveables, preferred_shard=-1,
-          restore_sequentially=False)
-      saveable_index = 0
-      for saveable in validated_saveables:
-        num_specs = len(saveable.specs)
-        saveable_tensors = all_tensors[
-            saveable_index:saveable_index + num_specs]
-        saveable_index += num_specs
-        restore_op = saveable.restore(saveable_tensors, restored_shapes=None)
-        if not context.executing_eagerly():
-          assert saveable.name not in self._checkpoint.restore_ops_by_name
-          self._checkpoint.restore_ops_by_name[saveable.name] = restore_op
-          restore_ops.append(restore_op)
+    restore_ops.extend(self._checkpoint.restore_saveables(
+        tensor_saveables, python_saveables))
     return restore_ops
 
   @property
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index d183fbdcf9..a54f41a54f 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -40,11 +40,14 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import checkpoint_management
 from tensorflow.python.training import optimizer as optimizer_lib
-from tensorflow.python.training import saveable_object as saveable_object_lib
-from tensorflow.python.training import saver as saver_lib
+from tensorflow.python.training import saver as v1_saver_lib
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import data_structures
 from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.training.saving import functional_saver
+from tensorflow.python.training.saving import saveable_object as saveable_object_lib
+from tensorflow.python.training.saving import saveable_object_util
+from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
@@ -89,7 +92,6 @@ class _CheckpointRestoreCoordinator(object):
         referenced every restore (e.g. for Python state); otherwise they would
         create their own ops every restore.
     """
-    self.builder = saver_lib.BulkSaverBuilder()
     self.object_graph_proto = object_graph_proto
     self.restore_uid = ops.uid()
     # Maps from objects to lists of attributes which were in the checkpoint but
@@ -144,6 +146,57 @@ class _CheckpointRestoreCoordinator(object):
     if self.new_restore_ops_callback:
       self.new_restore_ops_callback(new_ops)  # pylint: disable=not-callable
 
+  def restore_saveables(self, tensor_saveables, python_saveables):
+    """Run or build restore operations for SaveableObjects.
+
+    Args:
+      tensor_saveables: `SaveableObject`s which correspond to Tensors.
+      python_saveables: `PythonStateSaveable`s which correspond to Python
+        values.
+
+    Returns:
+      When graph building, a list of restore operations, either cached or newly
+      created, to restore `tensor_saveables`.
+    """
+    restore_ops = []
+    # Eagerly run restorations for Python state.
+    reader = pywrap_tensorflow.NewCheckpointReader(
+        self.save_path_string)
+    for saveable in python_saveables:
+      spec_names = [spec.name for spec in saveable.specs]
+      saveable.python_restore(
+          [reader.get_tensor(name) for name in spec_names])
+
+    # If we have new SaveableObjects, extract and cache restore ops.
+    if tensor_saveables:
+      validated_saveables = saveable_object_util.validate_and_slice_inputs(
+          tensor_saveables)
+      validated_names = set(saveable.name for saveable in validated_saveables)
+      if set(tensor_saveables.keys()) != validated_names:
+        raise AssertionError(
+            ("Saveable keys changed when validating. Got back %s, was "
+             "expecting %s") % (tensor_saveables.keys(), validated_names))
+      for saveable in validated_saveables:
+        if saveable.device:
+          device = saveable_object_util.set_cpu0(saveable.device)
+        else:
+          device = None
+        with ops.device(device):
+          tensors = []
+          for spec in saveable.specs:
+            tensors.append(
+                io_ops.restore_v2(
+                    self.save_path_tensor,
+                    [spec.name],
+                    [spec.slice_spec],
+                    [spec.dtype])[0])
+          restore_op = saveable.restore(tensors, restored_shapes=None)
+        if not context.executing_eagerly():
+          assert saveable.name not in self.restore_ops_by_name
+          self.restore_ops_by_name[saveable.name] = restore_op
+          restore_ops.append(restore_op)
+    return restore_ops
+
 
 class _NameBasedRestoreCoordinator(object):
   """Keeps the status of a name-based checkpoint restore."""
@@ -183,11 +236,11 @@ class _NameBasedRestoreCoordinator(object):
           continue
       else:
         saveable = saveable_factory
-      names_to_saveables = saver_lib.BaseSaverBuilder.OpListToDict(
+      names_to_saveables = saveable_object_util.op_list_to_dict(
           [saveable],
           convert_variable_to_tensor=False)
       for name, op in names_to_saveables.items():
-        for saveable_object in saver_lib.BaseSaverBuilder.SaveableObjectsForOp(
+        for saveable_object in saveable_object_util.saveable_objects_for_op(
             op=op, name=name):
           yield saveable_object
 
@@ -606,10 +659,10 @@ def _add_attributes_to_object_graph(
           # Figure out the name-based Saver's name for this variable. If it's
           # already a SaveableObject we'd just get the checkpoint key back, so
           # we leave full_name blank.
-          saver_dict = saver_lib.BaseSaverBuilder.OpListToDict(
+          saver_dict = saveable_object_util.op_list_to_dict(
               [maybe_saveable], convert_variable_to_tensor=False)
           full_name, = saver_dict.keys()
-          saveables = tuple(saver_lib.BaseSaverBuilder.SaveableObjectsForOp(
+          saveables = tuple(saveable_object_util.saveable_objects_for_op(
               op=maybe_saveable, name=attribute.checkpoint_key))
           for saveable in saveables:
             saveable.full_name = full_name
@@ -1226,7 +1279,7 @@ class NameBasedSaverStatus(_LoadStatus):
       session = ops.get_default_session()
     with ops.device("/cpu:0"):
       saveables = self._gather_saveable_objects()
-      saver_lib.Saver(saveables).restore(
+      v1_saver_lib.Saver(saveables).restore(
           sess=session, save_path=self._checkpoint.save_path)
 
   def initialize_or_restore(self, session=None):
@@ -1251,18 +1304,6 @@ class _SessionWithFeedDictAdditions(session_lib.SessionInterface):
         fetches=fetches, feed_dict=feed_dict, **kwargs)
 
 
-def _copy_saver_with_new_var_list(old_saver, new_var_list):
-  """Copy a `tf.train.Saver`'s state to a new Saver with different variables."""
-  new_saver = saver_lib.Saver(var_list=new_var_list, max_to_keep=None)
-  # TODO(allenl): Move to copying functionality to Saver?
-  # pylint: disable=protected-access
-  new_saver._last_checkpoints = old_saver._last_checkpoints
-  new_saver._checkpoints_to_be_deleted = old_saver._checkpoints_to_be_deleted
-  new_saver._next_checkpoint_time = old_saver._next_checkpoint_time
-  # pylint: enable=protected-access
-  return new_saver
-
-
 class CheckpointableSaver(object):
   """Saves and restores a `Checkpointable` object and its dependencies.
 
@@ -1301,7 +1342,8 @@ class CheckpointableSaver(object):
     # Op caching for save
     self._object_graph_feed_tensor = None
     self._last_save_object_graph = None
-    self._last_save_saver = None
+    self._file_prefix_feed_tensor = None
+    self._cached_save_operation = None
 
     # Op caching for restore, shared between _CheckpointRestoreCoordinators
     self._restore_op_cache = {}
@@ -1368,13 +1410,16 @@ class CheckpointableSaver(object):
           base.NoRestoreSaveable(
               tensor=object_graph_tensor,
               name=base.OBJECT_GRAPH_PROTO_KEY))
-      # TODO(allenl, haoliang): Swap in a function-based saver here.
-      return saver_lib.Saver(
+      # TODO(allenl): Swap in a function-based saver here once it can serialize
+      # to a SaverDef.
+      return v1_saver_lib.Saver(
           var_list=named_saveable_objects, max_to_keep=None)
 
-  def _prepare_save(self,
-                    object_graph_tensor=None,
-                    saveable_object_cache=None):
+  def _save_cached_when_graph_building(
+      self,
+      file_prefix,
+      object_graph_tensor=None,
+      saveable_object_cache=None):
     """Create or retrieve save ops.
 
     When graph building, `saveable_object_cache` will typically be non-`None`,
@@ -1383,15 +1428,17 @@ class CheckpointableSaver(object):
     unnecessarily re-creating save ops.
 
     Args:
+      file_prefix: The prefix for saved checkpoint files.
       object_graph_tensor: A `Tensor` to which the current object graph will be
         fed.
       saveable_object_cache: A dictionary; if specified, used to cache
         `SaveableObject`s.
 
     Returns:
-      A two-element tuple with a `tf.train.Saver` and a feed_dict of `Tensor`s
-      to feed when running save ops. The feed dict contains the current object
-      graph and any Python state to be saved in the checkpoint.
+      A two-element tuple with a filename tensor and a feed_dict of tensors to
+      feed when running it (if graph building). The feed dict contains the
+      current object graph and any Python state to be saved in the
+      checkpoint. When executing eagerly only the first argument is meaningful.
     """
     (named_saveable_objects, graph_proto,
      feed_additions) = self._gather_saveables(
@@ -1403,15 +1450,11 @@ class CheckpointableSaver(object):
         # constructors. That means the Saver needs to be copied with a new
         # var_list.
         or context.executing_eagerly()):
-      if self._last_save_object_graph is not None:
-        self._last_save_saver = _copy_saver_with_new_var_list(
-            old_saver=self._last_save_saver,
-            new_var_list=named_saveable_objects)
-      else:
-        self._last_save_saver = saver_lib.Saver(
-            var_list=named_saveable_objects, max_to_keep=None)
+      saver = functional_saver.Saver(named_saveable_objects)
+      with ops.device("/cpu:0"):
+        self._cached_save_operation = saver.save(file_prefix)
       self._last_save_object_graph = graph_proto
-    return self._last_save_saver, feed_additions
+    return self._cached_save_operation, feed_additions
 
   def save(self, file_prefix, checkpoint_number=None, session=None):
     """Save a training checkpoint.
@@ -1435,36 +1478,42 @@ class CheckpointableSaver(object):
     Returns:
       The full path to the checkpoint.
     """
-    feed_additions = {}
+    feed_dict = {}
     graph_building = not context.executing_eagerly()
+    if checkpoint_number:
+      file_prefix = "%s-%d" % (file_prefix, checkpoint_number)
     if graph_building:
       if self._object_graph_feed_tensor is None:
         with ops.device("/cpu:0"):
           self._object_graph_feed_tensor = constant_op.constant(
               "", dtype=dtypes.string)
+          self._file_prefix_feed_tensor = constant_op.constant(
+              "", dtype=dtypes.string)
       object_graph_tensor = self._object_graph_feed_tensor
+      file_prefix_tensor = self._file_prefix_feed_tensor
+      feed_dict[file_prefix_tensor] = file_prefix
     else:
+      with ops.device("/cpu:0"):
+        file_prefix_tensor = constant_op.constant(
+            file_prefix, dtype=dtypes.string)
       object_graph_tensor = None
 
-    saver, new_feed_additions = self._prepare_save(
+    file_io.recursive_create_dir(os.path.dirname(file_prefix))
+    save_path, new_feed_additions = self._save_cached_when_graph_building(
+        file_prefix=file_prefix_tensor,
         object_graph_tensor=object_graph_tensor,
         saveable_object_cache=self._saveable_object_cache)
     if new_feed_additions:
-      feed_additions.update(new_feed_additions)
+      feed_dict.update(new_feed_additions)
     if not graph_building:
       session = None
     elif session is None:
       session = ops.get_default_session()
 
-    file_io.recursive_create_dir(os.path.dirname(file_prefix))
-    with ops.device("/cpu:0"):
-      save_path = saver.save(
-          sess=_SessionWithFeedDictAdditions(
-              session=session, feed_additions=feed_additions),
-          save_path=file_prefix,
-          write_meta_graph=False,
-          write_state=False,
-          global_step=checkpoint_number)
+    if session:
+      save_path = session.run(save_path, feed_dict=feed_dict)
+    else:
+      save_path = save_path.numpy()
     return save_path
 
   def restore(self, save_path):
@@ -1753,9 +1802,9 @@ class Checkpoint(tracking.Checkpointable):
     Returns:
       The full path to the checkpoint (i.e. `file_prefix`).
     """
-    return self._saver.save(
+    return compat.as_str(self._saver.save(
         file_prefix=file_prefix,
-        session=session)
+        session=session))
 
   @property
   def save_counter(self):
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 4cd09f8a1d..04a7216484 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -14,7 +14,11 @@
 # ==============================================================================
 
 # pylint: disable=invalid-name
-"""Save and restore variables."""
+"""Save and restore variables.
+
+Symbols in this file are deprecated. See replacements in
+tensorflow/python/training/checkpointable and tensorflow/python/training/saving.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -25,7 +29,6 @@ import time
 import uuid
 
 import numpy as np
-import six
 
 from tensorflow.core.protobuf import checkpointable_object_graph_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
@@ -42,16 +45,15 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_io_ops
 from tensorflow.python.ops import io_ops
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import saveable_object
 from tensorflow.python.training import training_util
 from tensorflow.python.training.checkpointable import base as checkpointable
+from tensorflow.python.training.saving import saveable_object
+from tensorflow.python.training.saving import saveable_object_util
 from tensorflow.python.util import compat
 from tensorflow.python.util.tf_export import tf_export
 
@@ -67,31 +69,6 @@ get_checkpoint_mtimes = checkpoint_management.get_checkpoint_mtimes
 remove_checkpoint = checkpoint_management.remove_checkpoint
 
 
-# Op names which identify variable reads which should be saved.
-_VARIABLE_OPS = set(["Variable",
-                     "VariableV2",
-                     "AutoReloadVariable",
-                     "VarHandleOp",
-                     "ReadVariableOp"])
-
-
-def _set_cpu0(device_string):
-  """Creates a new device string based on `device_string` but using /CPU:0.
-
-  If the device is already on /CPU:0, this is a no-op.
-
-  Args:
-    device_string: A device string.
-
-  Returns:
-    A device string.
-  """
-  parsed_device = pydev.DeviceSpec.from_string(device_string)
-  parsed_device.device_type = "CPU"
-  parsed_device.device_index = 0
-  return parsed_device.to_string()
-
-
 class BaseSaverBuilder(object):
   """Base class for Savers.
 
@@ -101,64 +78,9 @@ class BaseSaverBuilder(object):
   SaveSpec = saveable_object.SaveSpec
   SaveableObject = saveable_object.SaveableObject
 
-  class VariableSaveable(SaveableObject):
-    """SaveableObject implementation that handles Variables."""
-
-    def __init__(self, var, slice_spec, name):
-      spec = BaseSaverBuilder.SaveSpec(var, slice_spec, name, dtype=var.dtype)
-      super(BaseSaverBuilder.VariableSaveable, self).__init__(var, [spec], name)
-
-    def restore(self, restored_tensors, restored_shapes):
-      restored_tensor = restored_tensors[0]
-      if restored_shapes is not None:
-        restored_tensor = array_ops.reshape(restored_tensor, restored_shapes[0])
-      return state_ops.assign(
-          self.op,
-          restored_tensor,
-          validate_shape=restored_shapes is None and
-          self.op.get_shape().is_fully_defined())
-
-  class ResourceVariableSaveable(SaveableObject):
-    """SaveableObject implementation that handles ResourceVariables."""
-
-    def __init__(self, var, slice_spec, name):
-      self._var_device = var.device
-      self._var_shape = var.shape
-      if isinstance(var, ops.Tensor):
-        self.handle_op = var.op.inputs[0]
-        tensor = var
-      elif isinstance(var, resource_variable_ops.ResourceVariable):
-
-        def _read_variable_closure(v):
-          def f():
-            with ops.device(v.device):
-              x = v.read_value()
-              # To allow variables placed on non-CPU devices to be checkpointed,
-              # we copy them to CPU on the same machine first.
-              with ops.device("/device:CPU:0"):
-                return array_ops.identity(x)
-          return f
-
-        self.handle_op = var.handle
-        tensor = _read_variable_closure(var)
-      else:
-        raise ValueError(
-            "Saveable is neither a resource variable nor a read operation."
-            " Got: %s" % repr(var))
-      spec = BaseSaverBuilder.SaveSpec(tensor, slice_spec, name,
-                                       dtype=var.dtype)
-      super(BaseSaverBuilder.ResourceVariableSaveable, self).__init__(
-          var, [spec], name)
-
-    def restore(self, restored_tensors, restored_shapes):
-      restored_tensor = restored_tensors[0]
-      if restored_shapes is not None:
-        restored_tensor = array_ops.reshape(restored_tensor, restored_shapes[0])
-      # Copy the restored tensor to the variable's device.
-      with ops.device(self._var_device):
-        restored_tensor = array_ops.identity(restored_tensor)
-        return resource_variable_ops.shape_safe_assign_variable_handle(
-            self.handle_op, self._var_shape, restored_tensor)
+  # Aliases for code which was moved but still has lots of users.
+  VariableSaveable = saveable_object_util.ReferenceVariableSaveable
+  ResourceVariableSaveable = saveable_object_util.ResourceVariableSaveable
 
   def __init__(self, write_version=saver_pb2.SaverDef.V2):
     self._write_version = write_version
@@ -224,7 +146,11 @@ class BaseSaverBuilder(object):
     del restore_sequentially
     all_tensors = []
     for saveable in saveables:
-      with ops.device(_set_cpu0(saveable.device) if saveable.device else None):
+      if saveable.device:
+        device = saveable_object_util.set_cpu0(saveable.device)
+      else:
+        device = None
+      with ops.device(device):
         all_tensors.extend(
             self.restore_op(filename_tensor, saveable, preferred_shard))
     return all_tensors
@@ -336,7 +262,7 @@ class BaseSaverBuilder(object):
     last_device = None
     for shard, (device, saveables) in enumerate(per_device):
       last_device = device
-      with ops.device(_set_cpu0(device)):
+      with ops.device(saveable_object_util.set_cpu0(device)):
         sharded_filename = self.sharded_filename(tmp_checkpoint_prefix, shard,
                                                  num_shards_tensor)
         sharded_prefixes.append(sharded_filename)
@@ -344,7 +270,7 @@ class BaseSaverBuilder(object):
 
     with ops.control_dependencies([x.op for x in sharded_saves]):
       # Co-locates the merge step with the last device.
-      with ops.device(_set_cpu0(last_device)):
+      with ops.device(saveable_object_util.set_cpu0(last_device)):
         # V2 format write path consists of a metadata merge step.  Once merged,
         # attempts to delete the temporary directory, "<user-fed prefix>_temp".
         merge_step = gen_io_ops.merge_v2_checkpoints(
@@ -459,10 +385,6 @@ class BaseSaverBuilder(object):
                 name="restore_shard"))
     return control_flow_ops.group(*sharded_restores, name="restore_all")
 
-  @staticmethod
-  def _IsVariable(v):
-    return isinstance(v, ops.Tensor) and v.op.type in _VARIABLE_OPS
-
   def _GroupByDevices(self, saveables):
     """Group Variable tensor slices per device.
 
@@ -490,220 +412,6 @@ class BaseSaverBuilder(object):
       per_device[canonical_device.pop()].append(saveable)
     return sorted(per_device.items(), key=lambda t: t[0])
 
-  @staticmethod
-  def OpListToDict(op_list, convert_variable_to_tensor=True):
-    """Create a dictionary of names to operation lists.
-
-    Args:
-      op_list: A list, tuple, or set of Variables or SaveableObjects.
-      convert_variable_to_tensor: Whether or not to convert single Variables
-        with no slice info into Tensors.
-
-    Returns:
-      A dictionary of names to the operations that must be saved under
-      that name.  Variables with save_slice_info are grouped together under the
-      same key in no particular order.
-
-    Raises:
-      TypeError: If the type of op_list or its elements is not supported.
-      ValueError: If at least two saveables share the same name.
-    """
-    if not isinstance(op_list, (list, tuple, set)):
-      raise TypeError("Variables to save should be passed in a dict or a "
-                      "list: %s" % op_list)
-    # When ResourceVariables are converted to Tensors, read ops are added to the
-    # graph. Sorting the op_list ensures that the resulting graph is always
-    # constructed in a deterministic way:
-    op_list = sorted(op_list, key=lambda x: x.name)
-    names_to_saveables = {}
-    # pylint: disable=protected-access
-    for var in op_list:
-      if isinstance(var, BaseSaverBuilder.SaveableObject):
-        names_to_saveables[var.name] = var
-      elif isinstance(var, variables.PartitionedVariable):
-        if var.name in names_to_saveables:
-          raise ValueError("At least two variables have the same name: %s" %
-                           var.name)
-        names_to_saveables[var.name] = var
-      elif isinstance(var, variables.Variable) and var._save_slice_info:
-        name = var._save_slice_info.full_name
-        if name in names_to_saveables:
-          if not isinstance(names_to_saveables[name], list):
-            raise ValueError("Mixing slices and non-slices with the same name: "
-                             "%s" % name)
-          names_to_saveables[name].append(var)
-        else:
-          names_to_saveables[name] = [var]
-      elif (isinstance(var, checkpointable.CheckpointableBase)
-            and not isinstance(var, variables.Variable)):
-        checkpointable_saveables = [
-            (factory() if callable(factory) else factory)
-            for factory in var._gather_saveables_for_checkpoint().values()]
-        names_to_saveables.update(
-            BaseSaverBuilder.OpListToDict(checkpointable_saveables))
-      else:
-        if context.executing_eagerly():
-          if not isinstance(var, resource_variable_ops.ResourceVariable):
-            raise ValueError(
-                "Can only save/restore ResourceVariables when eager execution "
-                "is enabled, type: %s." % type(var))
-          set_var = names_to_saveables.setdefault(var._shared_name, var)
-          if set_var is not var:
-            raise ValueError(
-                ("Two different ResourceVariable objects with the same "
-                 "shared_name '%s' were passed to the Saver. This likely means "
-                 "that they were created in different Graphs or isolation "
-                 "contexts, and may not be checkpointed together.") %
-                (var._shared_name,))
-        else:
-          if convert_variable_to_tensor:
-            if isinstance(var, resource_variable_ops.ResourceVariable):
-              var = var._graph_element  # pylint: disable=protected-access
-            else:
-              var = ops.internal_convert_to_tensor(var, as_ref=True)
-            if not BaseSaverBuilder._IsVariable(var):
-              raise TypeError("Variable to save is not a Variable: %s" % var)
-          if var.op.type == "ReadVariableOp":
-            name = var.op.inputs[0].op.name
-          else:
-            name = var.op.name
-          if name in names_to_saveables:
-            raise ValueError("At least two variables have the same name: %s" %
-                             name)
-          names_to_saveables[name] = var
-
-      # pylint: enable=protected-access
-    return names_to_saveables
-
-  @staticmethod
-  def SaveableObjectsForOp(op, name):
-    """Create `SaveableObject`s from an operation.
-
-    Args:
-      op: A variable, operation, or SaveableObject to coerce into a
-        SaveableObject.
-      name: A string name for the SaveableObject.
-
-    Yields:
-      `SaveableObject`s which together save/restore `op`.
-
-    Raises:
-      TypeError: If `name` is not a string.
-      ValueError: For operations with no known conversion to SaveableObject.
-    """
-    if not isinstance(name, six.string_types):
-      raise TypeError(
-          "names_to_saveables must be a dict mapping string names to "
-          "checkpointable operations. Name is not a string: %s" % name)
-    if isinstance(op, BaseSaverBuilder.SaveableObject):
-      yield op
-    elif isinstance(op, (list, tuple, variables.PartitionedVariable)):
-      if isinstance(op, variables.PartitionedVariable):
-        op = list(op)
-      # A set of slices.
-      slice_name = None
-      # pylint: disable=protected-access
-      for variable in op:
-        if not isinstance(variable, variables.Variable):
-          raise ValueError("Slices must all be Variables: %s" % variable)
-        if not variable._save_slice_info:
-          raise ValueError("Slices must all be slices: %s" % variable)
-        if slice_name is None:
-          slice_name = variable._save_slice_info.full_name
-        elif slice_name != variable._save_slice_info.full_name:
-          raise ValueError(
-              "Slices must all be from the same tensor: %s != %s" %
-              (slice_name, variable._save_slice_info.full_name))
-        if variable.op.type in ["Variable", "VariableV2",
-                                "AutoReloadVariable"]:
-          yield BaseSaverBuilder.VariableSaveable(
-              variable, variable._save_slice_info.spec, name)
-        else:
-          yield BaseSaverBuilder.ResourceVariableSaveable(
-              variable, variable._save_slice_info.spec, name)
-      # pylint: enable=protected-access
-    elif isinstance(op, checkpointable.CheckpointableBase) and not isinstance(
-        op, variables.Variable):
-      # pylint: disable=protected-access
-      for attr, factory in op._gather_saveables_for_checkpoint().items():
-        if attr == checkpointable.VARIABLE_VALUE_KEY:
-          # Keep original name for classes masquerading as variables.
-          full_name = name
-        else:
-          full_name = name + "_" + attr
-        op = (factory(full_name) if callable(factory) else factory)
-        for op in BaseSaverBuilder.SaveableObjectsForOp(op, op.name):
-          yield op
-      # pylint: enable=protected-access
-    else:
-      # A variable or tensor.
-      if context.executing_eagerly():
-        if not isinstance(op, resource_variable_ops.ResourceVariable):
-          raise ValueError("Can only save/restore ResourceVariable eager "
-                           "mode is enabled, type: %s." % type(op))
-        yield BaseSaverBuilder.ResourceVariableSaveable(op, "", name)
-      else:
-        if isinstance(op, resource_variable_ops.ResourceVariable):
-          variable = op._graph_element  # pylint: disable=protected-access
-        else:
-          variable = ops.internal_convert_to_tensor(op, as_ref=True)
-        if not BaseSaverBuilder._IsVariable(variable):
-          raise TypeError("names_to_saveables must be a dict mapping string "
-                          "names to Tensors/Variables. Not a variable: %s" %
-                          variable)
-        if variable.op.type in ["Variable", "VariableV2",
-                                "AutoReloadVariable"]:
-          yield BaseSaverBuilder.VariableSaveable(variable, "", name)
-        else:
-          yield BaseSaverBuilder.ResourceVariableSaveable(
-              variable, "", name)
-
-  def _ValidateAndSliceInputs(self, names_to_saveables):
-    """Returns the variables and names that will be used for a Saver.
-
-    Args:
-      names_to_saveables: A dict (k, v) where k is the name of an operation and
-         v is an operation to save or a BaseSaverBuilder.Saver.
-
-    Returns:
-      A list of BaseSaverBuilder.SaveableObject objects.
-
-    Raises:
-      TypeError: If any of the keys are not strings or any of the
-        values are not one of Tensor or Variable or a checkpointable operation.
-      ValueError: If the same operation is given in more than one value
-        (this also applies to slices of SlicedVariables).
-    """
-    if not isinstance(names_to_saveables, dict):
-      names_to_saveables = BaseSaverBuilder.OpListToDict(names_to_saveables)
-
-    saveables = []
-    seen_ops = set()
-    for name, op in sorted(names_to_saveables.items(),
-                           # Avoid comparing ops, sort only by name.
-                           key=lambda x: x[0]):
-      for converted_saveable_object in self.SaveableObjectsForOp(op, name):
-        self._AddSaveable(saveables, seen_ops, converted_saveable_object)
-    return saveables
-
-  def _AddSaveable(self, saveables, seen_ops, saveable):
-    """Adds the saveable to the saveables list.
-
-    Args:
-      saveables: List to append the SaveableObject to.
-      seen_ops: Set of the ops of the saveables already processed.  Used to
-        check that each saveable is only saved once.
-      saveable: The saveable.
-
-    Raises:
-      ValueError: If the saveable has already been processed.
-    """
-    if saveable.op in seen_ops:
-      raise ValueError("The same saveable will be restored with two names: %s" %
-                       saveable.name)
-    saveables.append(saveable)
-    seen_ops.add(saveable.op)
-
   def build(self,
             names_to_saveables,
             reshape=False,
@@ -775,7 +483,8 @@ class BaseSaverBuilder(object):
       raise ValueError("save and restore operations need to be built together "
                        " when eager execution is not enabled.")
 
-    saveables = self._ValidateAndSliceInputs(names_to_saveables)
+    saveables = saveable_object_util.validate_and_slice_inputs(
+        names_to_saveables)
     if max_to_keep is None:
       max_to_keep = 0
 
@@ -1910,7 +1619,7 @@ def saver_from_object_based_checkpoint(
   if builder is None:
     builder = BulkSaverBuilder()
 
-  saveables = builder._ValidateAndSliceInputs(var_list)  # pylint: disable=protected-access
+  saveables = saveable_object_util.validate_and_slice_inputs(var_list)
   current_names = set()
   for saveable in saveables:
     for spec in saveable.specs:
diff --git a/tensorflow/python/training/saving/BUILD b/tensorflow/python/training/saving/BUILD
new file mode 100644
index 0000000000..67ccd59b88
--- /dev/null
+++ b/tensorflow/python/training/saving/BUILD
@@ -0,0 +1,55 @@
+# Description:
+#   Low-level utilities for reading and writing checkpoints.
+
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+py_library(
+    name = "functional_saver",
+    srcs = ["functional_saver.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":saveable_object",
+        ":saveable_object_util",
+        "//tensorflow/python/eager:def_function",
+    ],
+)
+
+cuda_py_test(
+    name = "functional_saver_test",
+    size = "medium",
+    srcs = [
+        "functional_saver_test.py",
+    ],
+    additional_deps = [
+        ":functional_saver",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+py_library(
+    name = "saveable_object",
+    srcs = ["saveable_object.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_library(
+    name = "saveable_object_util",
+    srcs = ["saveable_object_util.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/training/checkpointable:base",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/python/training/saving/functional_saver.py b/tensorflow/python/training/saving/functional_saver.py
new file mode 100644
index 0000000000..7eed333662
--- /dev/null
+++ b/tensorflow/python/training/saving/functional_saver.py
@@ -0,0 +1,101 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Saves and restore variables inside traced @tf.functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.training.saving import saveable_object
+from tensorflow.python.training.saving import saveable_object_util
+
+
+class Saver(object):
+  """A minimal utility class for saving and restoring checkpoints.
+
+  Note that this is a low-level utility which stores Tensors in the keys
+  specified by `SaveableObject`s. Higher-level utilities for object-based
+  checkpointing are built on top of it.
+  """
+
+  def __init__(self, saveable_objects):
+    """Specify a list of `SaveableObject`s to save and restore.
+
+    Args:
+      saveable_objects: A list of `SaveableObject`s.
+    """
+    saveable_objects = list(saveable_objects)
+    for saveable in saveable_objects:
+      if not isinstance(saveable, saveable_object.SaveableObject):
+        raise ValueError(
+            "Saver expected a list of SaveableObjects, got %s." % (saveable,))
+    self._saveable_objects = saveable_objects
+
+  # TODO(b/120569892): Use tf.function here
+  def save(self, file_prefix):
+    """Save the saveable objects to a checkpoint with `file_prefix`.
+
+    Args:
+      file_prefix: A string or scalar string Tensor containing the prefix to
+        save under.
+    Returns:
+      A scalar string Tensor containing `file_prefix` with control dependencies
+      on the save ops.
+    """
+    tensor_names = []
+    tensors = []
+    tensor_slices = []
+    for saveable in self._saveable_objects:
+      for spec in saveable.specs:
+        tensor_names.append(spec.name)
+        tensors.append(spec.tensor)
+        tensor_slices.append(spec.slice_spec)
+    with ops.control_dependencies(
+        [io_ops.save_v2(file_prefix, tensor_names, tensor_slices, tensors)]):
+      return array_ops.identity(file_prefix)
+
+  # TODO(b/120569892): Use tf.function here
+  def restore(self, file_prefix):
+    """Restore the saveable objects from a checkpoint with `file_prefix`.
+
+    Args:
+      file_prefix: A string or scalar string Tensor containing the prefix for
+        files to read from.
+
+    Returns:
+      An operation which restores the `Saver`'s `SaveableObject`s when run, or
+      None if executing eagerly.
+    """
+    restore_ops = []
+    for saveable in self._saveable_objects:
+      if saveable.device:
+        device = saveable_object_util.set_cpu0(saveable.device)
+      else:
+        device = None
+      with ops.device(device):
+        tensors = []
+        for spec in saveable.specs:
+          tensors.append(
+              io_ops.restore_v2(
+                  file_prefix,
+                  [spec.name],
+                  [spec.slice_spec],
+                  [spec.dtype])[0])
+        restore_ops.append(saveable.restore(tensors, restored_shapes=None))
+    return control_flow_ops.group(restore_ops)
diff --git a/tensorflow/python/training/saving/functional_saver_test.py b/tensorflow/python/training/saving/functional_saver_test.py
new file mode 100644
index 0000000000..40002255aa
--- /dev/null
+++ b/tensorflow/python/training/saving/functional_saver_test.py
@@ -0,0 +1,50 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+"""Tests for the functional saver."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.training.saving import functional_saver
+from tensorflow.python.training.saving import saveable_object_util
+
+
+class SaverTest(test.TestCase):
+
+  def test_resource_variable(self):
+    v1 = resource_variable_ops.ResourceVariable(2.)
+    saver = functional_saver.Saver(
+        saveable_object_util.saveable_objects_for_op(v1, "x"))
+    prefix = os.path.join(self.get_temp_dir(), "ckpt")
+    save_path = saver.save(constant_op.constant(prefix))
+    v1.assign(1.)
+    saver.restore(save_path)
+    self.assertEqual(2., self.evaluate(v1))
+
+    v2 = resource_variable_ops.ResourceVariable(3.)
+    second_saver = functional_saver.Saver(
+        saveable_object_util.saveable_objects_for_op(v2, "x"))
+    second_saver.restore(save_path)
+    self.assertEqual(2., self.evaluate(v2))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/training/saveable_object.py b/tensorflow/python/training/saving/saveable_object.py
similarity index 100%
rename from tensorflow/python/training/saveable_object.py
rename to tensorflow/python/training/saving/saveable_object.py
diff --git a/tensorflow/python/training/saving/saveable_object_util.py b/tensorflow/python/training/saving/saveable_object_util.py
new file mode 100644
index 0000000000..fa88d2c6eb
--- /dev/null
+++ b/tensorflow/python/training/saving/saveable_object_util.py
@@ -0,0 +1,340 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for working with and creating SaveableObjects."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import device as pydev
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.training.checkpointable import base as checkpointable
+from tensorflow.python.training.saving import saveable_object
+
+
+# Op names which identify variable reads which should be saved.
+_VARIABLE_OPS = set(["Variable",
+                     "VariableV2",
+                     "AutoReloadVariable",
+                     "VarHandleOp",
+                     "ReadVariableOp"])
+
+
+def set_cpu0(device_string):
+  """Creates a new device string based on `device_string` but using /CPU:0.
+
+  If the device is already on /CPU:0, this is a no-op.
+
+  Args:
+    device_string: A device string.
+
+  Returns:
+    A device string.
+  """
+  parsed_device = pydev.DeviceSpec.from_string(device_string)
+  parsed_device.device_type = "CPU"
+  parsed_device.device_index = 0
+  return parsed_device.to_string()
+
+
+class ReferenceVariableSaveable(saveable_object.SaveableObject):
+  """SaveableObject implementation that handles reference variables."""
+
+  def __init__(self, var, slice_spec, name):
+    spec = saveable_object.SaveSpec(var, slice_spec, name, dtype=var.dtype)
+    super(ReferenceVariableSaveable, self).__init__(var, [spec], name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    restored_tensor = restored_tensors[0]
+    if restored_shapes is not None:
+      restored_tensor = array_ops.reshape(restored_tensor, restored_shapes[0])
+    return state_ops.assign(
+        self.op,
+        restored_tensor,
+        validate_shape=restored_shapes is None and
+        self.op.get_shape().is_fully_defined())
+
+
+class ResourceVariableSaveable(saveable_object.SaveableObject):
+  """SaveableObject implementation that handles ResourceVariables."""
+
+  def __init__(self, var, slice_spec, name):
+    self._var_device = var.device
+    self._var_shape = var.shape
+    if isinstance(var, ops.Tensor):
+      self.handle_op = var.op.inputs[0]
+      tensor = var
+    elif isinstance(var, resource_variable_ops.ResourceVariable):
+
+      def _read_variable_closure(v):
+        def f():
+          with ops.device(v.device):
+            x = v.read_value()
+            # To allow variables placed on non-CPU devices to be checkpointed,
+            # we copy them to CPU on the same machine first.
+            with ops.device("/device:CPU:0"):
+              return array_ops.identity(x)
+        return f
+
+      self.handle_op = var.handle
+      tensor = _read_variable_closure(var)
+    else:
+      raise ValueError(
+          "Saveable is neither a resource variable nor a read operation."
+          " Got: %s" % repr(var))
+    spec = saveable_object.SaveSpec(tensor, slice_spec, name,
+                                    dtype=var.dtype)
+    super(ResourceVariableSaveable, self).__init__(var, [spec], name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    restored_tensor = restored_tensors[0]
+    if restored_shapes is not None:
+      restored_tensor = array_ops.reshape(restored_tensor, restored_shapes[0])
+    # Copy the restored tensor to the variable's device.
+    with ops.device(self._var_device):
+      restored_tensor = array_ops.identity(restored_tensor)
+      return resource_variable_ops.shape_safe_assign_variable_handle(
+          self.handle_op, self._var_shape, restored_tensor)
+
+
+def _tensor_comes_from_variable(v):
+  return isinstance(v, ops.Tensor) and v.op.type in _VARIABLE_OPS
+
+
+def saveable_objects_for_op(op, name):
+  """Create `SaveableObject`s from an operation.
+
+  Args:
+    op: A variable, operation, or SaveableObject to coerce into a
+      SaveableObject.
+    name: A string name for the SaveableObject.
+
+  Yields:
+    `SaveableObject`s which together save/restore `op`.
+
+  Raises:
+    TypeError: If `name` is not a string.
+    ValueError: For operations with no known conversion to SaveableObject.
+  """
+  if not isinstance(name, six.string_types):
+    raise TypeError(
+        "names_to_saveables must be a dict mapping string names to "
+        "checkpointable operations. Name is not a string: %s" % name)
+  if isinstance(op, saveable_object.SaveableObject):
+    yield op
+  elif isinstance(op, (list, tuple, variables.PartitionedVariable)):
+    if isinstance(op, variables.PartitionedVariable):
+      op = list(op)
+    # A set of slices.
+    slice_name = None
+    # pylint: disable=protected-access
+    for variable in op:
+      if not isinstance(variable, variables.Variable):
+        raise ValueError("Slices must all be Variables: %s" % variable)
+      if not variable._save_slice_info:
+        raise ValueError("Slices must all be slices: %s" % variable)
+      if slice_name is None:
+        slice_name = variable._save_slice_info.full_name
+      elif slice_name != variable._save_slice_info.full_name:
+        raise ValueError(
+            "Slices must all be from the same tensor: %s != %s" %
+            (slice_name, variable._save_slice_info.full_name))
+      if variable.op.type in ["Variable", "VariableV2",
+                              "AutoReloadVariable"]:
+        yield ReferenceVariableSaveable(
+            variable, variable._save_slice_info.spec, name)
+      else:
+        yield ResourceVariableSaveable(
+            variable, variable._save_slice_info.spec, name)
+    # pylint: enable=protected-access
+  elif isinstance(op, checkpointable.CheckpointableBase) and not isinstance(
+      op, variables.Variable):
+    # pylint: disable=protected-access
+    for attr, factory in op._gather_saveables_for_checkpoint().items():
+      if attr == checkpointable.VARIABLE_VALUE_KEY:
+        # Keep original name for classes masquerading as variables.
+        full_name = name
+      else:
+        full_name = name + "_" + attr
+      op = (factory(full_name) if callable(factory) else factory)
+      for op in saveable_objects_for_op(op, op.name):
+        yield op
+    # pylint: enable=protected-access
+  else:
+    # A variable or tensor.
+    if isinstance(op, resource_variable_ops.ResourceVariable):
+      # pylint: disable=protected-access
+      if op._in_graph_mode:
+        variable = op._graph_element
+      else:
+        variable = op
+      # pylint: enable=protected-access
+      yield ResourceVariableSaveable(variable, "", name)
+    else:
+      with ops.init_scope():
+        if context.executing_eagerly():
+          raise ValueError("Can only save/restore ResourceVariables when "
+                           "executing eagerly, got type: %s." % type(op))
+
+      variable = ops.internal_convert_to_tensor(op, as_ref=True)
+      if not _tensor_comes_from_variable(variable):
+        raise TypeError("names_to_saveables must be a dict mapping string "
+                        "names to Tensors/Variables. Not a variable: %s" %
+                        variable)
+      if variable.op.type in ["Variable", "VariableV2",
+                              "AutoReloadVariable"]:
+        yield ReferenceVariableSaveable(variable, "", name)
+      else:
+        yield ResourceVariableSaveable(
+            variable, "", name)
+
+
+def op_list_to_dict(op_list, convert_variable_to_tensor=True):
+  """Create a dictionary of names to operation lists.
+
+  Args:
+    op_list: A list, tuple, or set of Variables or SaveableObjects.
+    convert_variable_to_tensor: Whether or not to convert single Variables
+      with no slice info into Tensors.
+
+  Returns:
+    A dictionary of names to the operations that must be saved under
+    that name.  Variables with save_slice_info are grouped together under the
+    same key in no particular order.
+
+  Raises:
+    TypeError: If the type of op_list or its elements is not supported.
+    ValueError: If at least two saveables share the same name.
+  """
+  if not isinstance(op_list, (list, tuple, set)):
+    raise TypeError("Variables to save should be passed in a dict or a "
+                    "list: %s" % op_list)
+  # When ResourceVariables are converted to Tensors, read ops are added to the
+  # graph. Sorting the op_list ensures that the resulting graph is always
+  # constructed in a deterministic way:
+  op_list = sorted(op_list, key=lambda x: x.name)
+  names_to_saveables = {}
+  # pylint: disable=protected-access
+  for var in op_list:
+    if isinstance(var, saveable_object.SaveableObject):
+      names_to_saveables[var.name] = var
+    elif isinstance(var, variables.PartitionedVariable):
+      if var.name in names_to_saveables:
+        raise ValueError("At least two variables have the same name: %s" %
+                         var.name)
+      names_to_saveables[var.name] = var
+    elif isinstance(var, variables.Variable) and var._save_slice_info:
+      name = var._save_slice_info.full_name
+      if name in names_to_saveables:
+        if not isinstance(names_to_saveables[name], list):
+          raise ValueError("Mixing slices and non-slices with the same name: "
+                           "%s" % name)
+        names_to_saveables[name].append(var)
+      else:
+        names_to_saveables[name] = [var]
+    elif (isinstance(var, checkpointable.CheckpointableBase)
+          and not isinstance(var, variables.Variable)):
+      checkpointable_saveables = [
+          (factory() if callable(factory) else factory)
+          for factory in var._gather_saveables_for_checkpoint().values()]
+      names_to_saveables.update(
+          op_list_to_dict(checkpointable_saveables))
+    else:
+      if context.executing_eagerly():
+        if not isinstance(var, resource_variable_ops.ResourceVariable):
+          raise ValueError(
+              "Can only save/restore ResourceVariables when eager execution "
+              "is enabled, type: %s." % type(var))
+        set_var = names_to_saveables.setdefault(var._shared_name, var)
+        if set_var is not var:
+          raise ValueError(
+              ("Two different ResourceVariable objects with the same "
+               "shared_name '%s' were passed to the Saver. This likely means "
+               "that they were created in different Graphs or isolation "
+               "contexts, and may not be checkpointed together.") %
+              (var._shared_name,))
+      else:
+        if convert_variable_to_tensor:
+          if isinstance(var, resource_variable_ops.ResourceVariable):
+            var = var._graph_element  # pylint: disable=protected-access
+          else:
+            var = ops.internal_convert_to_tensor(var, as_ref=True)
+          if not _tensor_comes_from_variable(var):
+            raise TypeError("Variable to save is not a Variable: %s" % var)
+        if var.op.type == "ReadVariableOp":
+          name = var.op.inputs[0].op.name
+        else:
+          name = var.op.name
+        if name in names_to_saveables:
+          raise ValueError("At least two variables have the same name: %s" %
+                           name)
+        names_to_saveables[name] = var
+
+    # pylint: enable=protected-access
+  return names_to_saveables
+
+
+def _add_saveable(saveables, seen_ops, saveable):
+  """Adds the saveable to the saveables list.
+
+  Args:
+    saveables: List to append the SaveableObject to.
+    seen_ops: Set of the ops of the saveables already processed.  Used to
+      check that each saveable is only saved once.
+    saveable: The saveable.
+
+  Raises:
+    ValueError: If the saveable has already been processed.
+  """
+  if saveable.op in seen_ops:
+    raise ValueError("The same saveable will be restored with two names: %s" %
+                     saveable.name)
+  saveables.append(saveable)
+  seen_ops.add(saveable.op)
+
+
+def validate_and_slice_inputs(names_to_saveables):
+  """Returns the variables and names that will be used for a Saver.
+
+  Args:
+    names_to_saveables: A dict (k, v) where k is the name of an operation and
+       v is an operation to save or a BaseSaverBuilder.Saver.
+
+  Returns:
+    A list of SaveableObjects.
+
+  Raises:
+    TypeError: If any of the keys are not strings or any of the
+      values are not one of Tensor or Variable or a checkpointable operation.
+    ValueError: If the same operation is given in more than one value
+      (this also applies to slices of SlicedVariables).
+  """
+  if not isinstance(names_to_saveables, dict):
+    names_to_saveables = op_list_to_dict(names_to_saveables)
+
+  saveables = []
+  seen_ops = set()
+  for name, op in sorted(names_to_saveables.items(),
+                         # Avoid comparing ops, sort only by name.
+                         key=lambda x: x[0]):
+    for converted_saveable_object in saveable_objects_for_op(op, name):
+      _add_saveable(saveables, seen_ops, converted_saveable_object)
+  return saveables
diff --git a/tensorflow/python/training/warm_starting_util.py b/tensorflow/python/training/warm_starting_util.py
index 8c97f101da..1382b8ce72 100644
--- a/tensorflow/python/training/warm_starting_util.py
+++ b/tensorflow/python/training/warm_starting_util.py
@@ -28,7 +28,7 @@ from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_ops
 from tensorflow.python.training import checkpoint_utils
-from tensorflow.python.training import saver
+from tensorflow.python.training.saving import saveable_object_util
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -139,7 +139,7 @@ def _infer_var_name(var):
   Returns:
     Name of the `var`
   """
-  name_to_var_dict = saver.BaseSaverBuilder.OpListToDict(var)
+  name_to_var_dict = saveable_object_util.op_list_to_dict(var)
   if len(name_to_var_dict) > 1:
     raise TypeError("`var` = %s passed as arg violates the constraints.  "
                     "name_to_var_dict = %s" % (var, name_to_var_dict))
-- 
GitLab


From abc2610c5ca689dce98683c63ccf8d886c42c37a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 13:09:16 -0800
Subject: [PATCH 1506/1554] Update ops-related pbtxt files.

PiperOrigin-RevId: 224565203
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 22 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 22 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index ce71b21507..1492741e8b 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -79062,6 +79062,17 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "UnwrapDatasetVariant"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+}
 op {
   name: "UpperBound"
   input_arg {
@@ -79495,6 +79506,17 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "WrapDatasetVariant"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+}
 op {
   name: "WriteAudioSummary"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 1fa59b7087..89bdcc571e 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -38153,6 +38153,17 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "UnwrapDatasetVariant"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+}
 op {
   name: "UpperBound"
   input_arg {
@@ -38471,6 +38482,17 @@ op {
     minimum: 1
   }
 }
+op {
+  name: "WrapDatasetVariant"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+}
 op {
   name: "WriteAudioSummary"
   input_arg {
-- 
GitLab


From 32b16083085c0bf954228e8c9160561a576ec405 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 13:11:24 -0800
Subject: [PATCH 1507/1554] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 224565479

---
 tensorflow/go/op/wrappers.go | 266 +++++++++++++++++------------------
 1 file changed, 133 insertions(+), 133 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 440756bb38..6e49fbb9ea 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5862,6 +5862,121 @@ func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output
 	return scope.AddOperation(opspec)
 }
 
+// StageClearAttr is an optional argument to StageClear.
+type StageClearAttr func(optionalAttr)
+
+// StageClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearCapacity(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearMemoryLimit(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// StageClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageClearContainer(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// StageClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageClearSharedName(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
+//
+// Returns the created operation.
+func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StageClear",
+
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// StageSizeAttr is an optional argument to StageSize.
+type StageSizeAttr func(optionalAttr)
+
+// StageSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageSizeCapacity(value int64) StageSizeAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// StageSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageSizeMemoryLimit(value int64) StageSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// StageSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageSizeContainer(value string) StageSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// StageSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageSizeSharedName(value string) StageSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of elements in the underlying container.
+func StageSize(scope *Scope, dtypes []tf.DataType, optional ...StageSizeAttr) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StageSize",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -11434,24 +11549,6 @@ func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr)
 	return op.Output(0)
 }
 
-// Returns the truth value of (x > y) element-wise.
-//
-// *NOTE*: `Greater` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Greater",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp.
 type ResourceSparseApplyRMSPropAttr func(optionalAttr)
 
@@ -17210,64 +17307,6 @@ func NonMaxSuppressionWithOverlaps(scope *Scope, overlaps tf.Output, scores tf.O
 	return op.Output(0)
 }
 
-// StageClearAttr is an optional argument to StageClear.
-type StageClearAttr func(optionalAttr)
-
-// StageClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageClearCapacity(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// StageClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageClearMemoryLimit(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// StageClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StageClearContainer(value string) StageClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// StageClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StageClearSharedName(value string) StageClearAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes all elements in the underlying container.
-//
-// Returns the created operation.
-func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StageClear",
-
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Computes softmax cross entropy cost and gradients to backpropagate.
 //
 // Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
@@ -20236,63 +20275,6 @@ func NotEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// StageSizeAttr is an optional argument to StageSize.
-type StageSizeAttr func(optionalAttr)
-
-// StageSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageSizeCapacity(value int64) StageSizeAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// StageSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageSizeMemoryLimit(value int64) StageSizeAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// StageSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StageSizeContainer(value string) StageSizeAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// StageSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StageSizeSharedName(value string) StageSizeAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op returns the number of elements in the underlying container.
-func StageSize(scope *Scope, dtypes []tf.DataType, optional ...StageSizeAttr) (size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StageSize",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Produces the max pool of the input tensor for quantized types.
 //
 // Arguments:
@@ -31550,6 +31532,24 @@ func ModelDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataT
 	return op.Output(0)
 }
 
+// Returns the truth value of (x > y) element-wise.
+//
+// *NOTE*: `Greater` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Greater",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Performs a padding as a preprocess during a convolution.
 //
 // Similar to FusedResizeAndPadConv2d, this op allows for an optimized
-- 
GitLab


From d09435e0cc8b21e5b10eb0f9750e7a24c2031e85 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 13:13:45 -0800
Subject: [PATCH 1508/1554] Change `Status::error_message()` usage to
 `Status::message()`.

PiperOrigin-RevId: 224565798
---
 tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc
index b4b06a40a2..ef35e84ba5 100644
--- a/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc
+++ b/tensorflow/contrib/tpu/profiler/dump_tpu_profile.cc
@@ -98,7 +98,7 @@ Status DumpOpProfileToLogDirectory(StringPiece run_dir,
   if (!status.ok()) {
     return errors::Internal(
         "Failed to convert op profile to json. Skipping... ",
-        string(status.error_message()));
+        string(status.message()));
   }
   TF_RETURN_IF_ERROR(WriteStringToFile(Env::Default(), path, json));
   if (os) {
-- 
GitLab


From 939206b56b6dd243f1aae52184c680e6b6f8b30e Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 7 Dec 2018 13:25:27 -0800
Subject: [PATCH 1509/1554] Update the performance test for unified LSTM to be
 benchmark test.

PiperOrigin-RevId: 224567398
---
 .../python/keras/layers/unified_lstm_test.py  | 51 +++++++++++--------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 2cb3eff8fd..8cbc370633 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -28,6 +28,7 @@ import numpy as np
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python import keras
+from tensorflow.python.client import session as session_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -732,7 +733,7 @@ class LSTMLayerGraphOnlyTest(test.TestCase):
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
 
-class UnifiedLSTMPerformanceTest(test.TestCase):
+class UnifiedLSTMPerformanceTest(test.Benchmark):
 
   def _measure_performance(self, test_config, model, x_train, y_train):
     batch = test_config['batch']
@@ -808,11 +809,11 @@ class UnifiedLSTMPerformanceTest(test.TestCase):
                  'Normal LSTM', sec_per_epoch)
     return sec_per_epoch
 
-  @test_util.run_in_graph_and_eager_modes(config=_config, use_gpu=True)
-  def test_performance_with_standard_cudnn_impl(self):
+  def _benchmark_performance_with_standard_cudnn_impl(self):
     if not test.is_gpu_available():
       self.skipTest('performance test will only run on GPU')
 
+    mode = 'eager' if context.executing_eagerly() else 'graph'
     batch = 64
     num_batch = 10
     test_config = {
@@ -832,34 +833,42 @@ class UnifiedLSTMPerformanceTest(test.TestCase):
         num_classes=test_config['output_shape'])
     y_train = keras.utils.to_categorical(y_train, test_config['output_shape'])
 
-    cudnn_duration = self._time_performance_run_cudnn_lstm(
+    cudnn_sec_per_epoch = self._time_performance_run_cudnn_lstm(
         test_config, x_train, y_train)
-    unified_lstm_gpu_duration = self._time_performance_run_unifed_lstm_gpu(
+    unified_lstm_sec_per_epoch = self._time_performance_run_unifed_lstm_gpu(
         test_config, x_train, y_train)
-    normal_lstm_duration = self._time_performance_run_normal_lstm(
+    normal_lstm_sec_per_epoch = self._time_performance_run_normal_lstm(
         test_config, x_train, y_train)
 
-    cudnn_vs_unified = cudnn_duration / unified_lstm_gpu_duration
-    unified_vs_normal = normal_lstm_duration / unified_lstm_gpu_duration
+    cudnn_vs_unified = cudnn_sec_per_epoch / unified_lstm_sec_per_epoch
+    unified_vs_normal = normal_lstm_sec_per_epoch / unified_lstm_sec_per_epoch
+
+    self.report_benchmark(name='keras_cudnn_lstm_' + mode,
+                          wall_time=cudnn_sec_per_epoch,
+                          iters=test_config['epoch'],
+                          extras=test_config)
+    self.report_benchmark(name='keras_unified_lstm_' + mode,
+                          wall_time=unified_lstm_sec_per_epoch,
+                          iters=test_config['epoch'],
+                          extras=test_config)
+    self.report_benchmark(name='keras_canonical_lstm_' + mode,
+                          wall_time=normal_lstm_sec_per_epoch,
+                          iters=test_config['epoch'],
+                          extras=test_config)
 
-    # TODO(scottzhu): reeanble the test after moving it to benchmark test suite.
-    # The current test has performance flakiness issue.
     logging.info('Expect the performance of Unified LSTM is within 80% of '
                  'CuDNN LSTM, got {0:.2f}%'.format(cudnn_vs_unified * 100))
     logging.info('Expect the performance of Unified LSTM is more than 5 times'
                  ' of normal LSTM, got {0:.2f}'.format(unified_vs_normal))
 
-    # Assert the performance diff should be within 80% of the native cudnn.
-    # self.assertGreaterEqual(
-    #     cudnn_vs_unified, 0.80,
-    #     'Expect the performance of Unified LSTM is within 80% of CuDNN LSTM, '
-    #     'but got {0:.2f}%'.format(cudnn_vs_unified * 100))
-    # # Assert the performance diff between CPU impl and GPU impl should be more
-    # # than 5 times.
-    # self.assertGreaterEqual(
-    #     unified_vs_normal, 5,
-    #     'Expect the performance of Unified LSTM is more than 5 times of '
-    #     'normal LSTM, but got {0:.2f}'.format(unified_vs_normal))
+  def benchmark_performance_graph(self):
+    with context.graph_mode(), session_lib.Session(config=_config):
+      self._benchmark_performance_with_standard_cudnn_impl()
+
+  def benchmark_performance_eager(self):
+    with context.eager_mode():
+      self._benchmark_performance_with_standard_cudnn_impl()
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 56429fce51cc81bb49d57c0217bd2c59deff774f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 13:42:24 -0800
Subject: [PATCH 1510/1554] Enables some previously disabled Keras test
 parameterizations.

PiperOrigin-RevId: 224570187
---
 .../keras/engine/training_dataset_test.py      | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index a8a81fa842..d6cc93d1ef 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
@@ -147,11 +148,14 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
 
 class TestTrainingWithDataset(keras_parameterized.TestCase):
 
-  # TODO(kaftan) Run w/ all model types.
-  # Seems like subclass models has a bug, file ticket
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_calling_model_on_same_dataset(self):
+    if ((not testing_utils.should_run_eagerly())
+        and testing_utils.get_model_type() == 'subclass'
+        and context.executing_eagerly()):
+      self.skipTest('b/120673224')
+
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
     optimizer = RMSPropOptimizer(learning_rate=0.001)
     loss = 'mse'
@@ -234,9 +238,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
                                  'you should specify the `steps` argument'):
       model.predict(dataset, verbose=0)
 
-  # TODO(kaftan) Run w/ all model types.
-  # Seems like subclass models has a bug, file ticket
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sample_weights(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
@@ -308,9 +310,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
 
 class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
 
-  # TODO(kaftan) Run w/ all model types.
-  # Seems like subclass models has a bug, file ticket
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_metrics_correctness_with_iterator(self):
     layers = [
-- 
GitLab


From 68e8a87bcd2b97a7f9a09f4cdba86ec254fad978 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 7 Dec 2018 13:45:51 -0800
Subject: [PATCH 1511/1554] Improve the correctness test by invoking train/eval
 multiple times (w/w.o. validation data).

PiperOrigin-RevId: 224570819
---
 .../contrib/distribute/python/keras_test.py   | 86 +++++++++----------
 1 file changed, 39 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index dea208232a..c0bcb5ecd5 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1303,16 +1303,17 @@ class TestDistributionStrategyCorrectness(test.TestCase,
 
     with self.cached_session():
       default_tolerance = 1e-5
-      weights_tolerance = default_tolerance
-      metrics_tolerance = default_tolerance
-      predict_tolerance = default_tolerance
+      tol_table = {}
 
       if isinstance(distribution, (mirrored_strategy.MirroredStrategy,
                                    mirrored_strategy.CoreMirroredStrategy)):
-        # TODO(b/119257215): Weights are not exactly the same, so use lower
-        # tolerance for now.
-        weights_tolerance = 1e-4
-        predict_tolerance = 1e-4
+        # TODO(b/119257215): Weights are not exactly the same, so use larger
+        # tolerance for now. Predict should be related to weights.
+        tol_table = {
+            'weights_1': 1e-4,
+            'weights_2': 1e-4,
+            'predict_result_1': 1e-4,
+        }
 
       keras.backend.set_image_data_format('channels_last')
       np.random.seed(_RANDOM_SEED)
@@ -1361,56 +1362,47 @@ class TestDistributionStrategyCorrectness(test.TestCase,
                                         with_distribution,
                                         x_train, y_train, x_predict))
 
-        training_history = model.fit(**training_inputs).history
+        result = {}
+        result['training_history_1'] = model.fit(**training_inputs).history
 
         if eval_inputs is not None:
-          eval_result = model.evaluate(**eval_inputs)
-        else:
-          # Creates a dummy identical eval_result to be compared later.
-          eval_result = 1.0
+          result['eval_result_1'] = model.evaluate(**eval_inputs)
 
-        weights = model.get_weights()
-        predict_result = model.predict(**predict_inputs)
+        result['weights_1'] = model.get_weights()
+        result['predict_result_1'] = model.predict(**predict_inputs)
 
-        return weights, training_history, eval_result, predict_result
+        # Train and eval again to mimic user's flow.
 
-      wts_with_ds, history_with_ds, eval_with_ds, predict_with_ds = (
-          fit_eval_and_predict(with_distribution=distribution))
+        result['training_history_2'] = model.fit(**training_inputs).history
 
-      (wts_without_ds, history_without_ds, eval_without_ds,
-       predict_without_ds) = fit_eval_and_predict(with_distribution=None)
+        if eval_inputs is not None:
+          result['eval_result_2'] = model.evaluate(**eval_inputs)
+
+        result['weights_2'] = model.get_weights()
+
+        return result
+
+      results_with_ds = fit_eval_and_predict(with_distribution=distribution)
+      results_without_ds = fit_eval_and_predict(with_distribution=None)
 
       # Verify that the weights, training history, eval results, predict outputs
       # are the same within some limits of tolerance.
-      self.assertAllClose(
-          wts_with_ds,
-          wts_without_ds,
-          atol=weights_tolerance,
-          rtol=weights_tolerance,
-          msg='Fail to assert weights after training.')
-      self.assertAllClose(
-          eval_with_ds,
-          eval_without_ds,
-          atol=metrics_tolerance,
-          rtol=metrics_tolerance,
-          msg='Fail to assert eval results.')
-      self.assertAllClose(
-          predict_with_ds,
-          predict_without_ds,
-          atol=predict_tolerance,
-          rtol=predict_tolerance,
-          msg='Fail to assert predict results.')
-
-      if not (isinstance(distribution, tpu_strategy.TPUStrategy) and
-              distribution.extended.steps_per_run > 1):
-        # TODO(b/119894254): Enable this test for all cases once the underlying
-        # bug is fixed.
+      for key in results_with_ds:
+        if (key.startswith('training_history') and
+            isinstance(distribution, tpu_strategy.TPUStrategy) and
+            distribution.extended.steps_per_run > 1):
+          # TODO(b/119894254): Enable this test for all cases once the
+          # underlying bug is fixed.
+          continue
+
+        tolerance = tol_table.get(key, default_tolerance)
+
         self.assertAllClose(
-            history_with_ds,
-            history_without_ds,
-            atol=metrics_tolerance,
-            rtol=metrics_tolerance,
-            msg='Fail to assert training history.')
+            results_with_ds[key],
+            results_without_ds[key],
+            atol=tolerance,
+            rtol=tolerance,
+            msg='Fail to assert {}.'.format(key))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 1b03648fb58913992ae44d706f9e5fce06a919e5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 13:52:06 -0800
Subject: [PATCH 1512/1554] Update cmd line quantization documentation link.

PiperOrigin-RevId: 224571863
---
 tensorflow/lite/g3doc/convert/cmdline_examples.md | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/g3doc/convert/cmdline_examples.md b/tensorflow/lite/g3doc/convert/cmdline_examples.md
index 59f26b3505..de81e2cfdd 100644
--- a/tensorflow/lite/g3doc/convert/cmdline_examples.md
+++ b/tensorflow/lite/g3doc/convert/cmdline_examples.md
@@ -94,11 +94,12 @@ tflite_convert \
 ### Convert a TensorFlow GraphDef for quantized inference <a name="graphdef_quant"></a>
 
 The TensorFlow Lite Converter is compatible with fixed point quantization models
-described [here](https://www.tensorflow.org/performance/quantization). These are
-float models with `FakeQuant*` ops inserted at the boundaries of fused layers
-to record min-max range information. This generates a quantized inference
-workload that reproduces the quantization behavior that was used during
-training.
+described
+[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/q
+uantize/README.md). These are float models with `FakeQuant*` ops inserted at the
+boundaries of fused layers to record min-max range information. This generates a
+quantized inference workload that reproduces the quantization behavior that was
+used during training.
 
 The following command generates a quantized TensorFlow Lite FlatBuffer from a
 "quantized" TensorFlow GraphDef.
-- 
GitLab


From a64a8d8d02f9d15b1f5d6fbdbc0fa0db3f37f97d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 7 Dec 2018 13:58:08 -0800
Subject: [PATCH 1513/1554] Fix up a few tests to interact better with v2 mode

PiperOrigin-RevId: 224572844
---
 .../tutorials/mnist/mnist_with_summaries.py   |  3 ++-
 .../python/debug/examples/debug_errors.py     |  3 ++-
 .../python/debug/examples/debug_fibonacci.py  |  3 ++-
 .../python/debug/examples/debug_keras.py      |  3 ++-
 .../python/debug/examples/debug_mnist.py      |  3 ++-
 tensorflow/python/keras/models_test.py        |  4 ++-
 tensorflow/python/kernel_tests/lu_op_test.py  | 27 ++++++++++---------
 .../python/kernel_tests/matmul_op_test.py     |  4 ++-
 .../kernel_tests/self_adjoint_eig_op_test.py  |  4 +--
 .../python/kernel_tests/tensordot_op_test.py  | 10 ++++---
 10 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py b/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py
index 7967e22d6a..1854e84d49 100644
--- a/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py
+++ b/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py
@@ -183,7 +183,8 @@ def main(_):
   if tf.gfile.Exists(FLAGS.log_dir):
     tf.gfile.DeleteRecursively(FLAGS.log_dir)
   tf.gfile.MakeDirs(FLAGS.log_dir)
-  train()
+  with tf.Graph().as_default():
+    train()
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/debug/examples/debug_errors.py b/tensorflow/python/debug/examples/debug_errors.py
index 28abc97343..e3692072cc 100644
--- a/tensorflow/python/debug/examples/debug_errors.py
+++ b/tensorflow/python/debug/examples/debug_errors.py
@@ -77,4 +77,5 @@ if __name__ == "__main__":
       default=False,
       help="Use debugger to track down bad values during training")
   FLAGS, unparsed = parser.parse_known_args()
-  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  with tf.Graph().as_default():
+    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/python/debug/examples/debug_fibonacci.py b/tensorflow/python/debug/examples/debug_fibonacci.py
index 3821b393ec..777fb08988 100644
--- a/tensorflow/python/debug/examples/debug_fibonacci.py
+++ b/tensorflow/python/debug/examples/debug_fibonacci.py
@@ -100,4 +100,5 @@ if __name__ == "__main__":
       "--debug flag.")
 
   FLAGS, unparsed = parser.parse_known_args()
-  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  with tf.Graph().as_default():
+    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/python/debug/examples/debug_keras.py b/tensorflow/python/debug/examples/debug_keras.py
index 3272d85ade..019121fa0a 100644
--- a/tensorflow/python/debug/examples/debug_keras.py
+++ b/tensorflow/python/debug/examples/debug_keras.py
@@ -86,4 +86,5 @@ if __name__ == "__main__":
       default=2,
       help="Number of epochs to train the model for.")
   FLAGS, unparsed = parser.parse_known_args()
-  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  with tf.Graph().as_default():
+    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/python/debug/examples/debug_mnist.py b/tensorflow/python/debug/examples/debug_mnist.py
index ab1c90371c..09fb06c9c0 100644
--- a/tensorflow/python/debug/examples/debug_mnist.py
+++ b/tensorflow/python/debug/examples/debug_mnist.py
@@ -190,4 +190,5 @@ if __name__ == "__main__":
       "the gRPC address (e.g., localhost:1234). Mutually exclusive with the "
       "--debug flag.")
   FLAGS, unparsed = parser.parse_known_args()
-  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  with tf.Graph().as_default():
+    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index b0872ae3ab..907ac41d0e 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -365,7 +365,9 @@ class TestCloneAndBuildModel(test.TestCase):
 
     self.assertEqual('mse', model.loss)
     self.assertTrue(
-        isinstance(model.optimizer, keras.optimizers.RMSprop))
+        isinstance(model.optimizer,
+                   (keras.optimizers.RMSprop,
+                    keras.optimizer_v2.rmsprop.RMSprop)))
     self.assertEqual(['acc', metrics.categorical_accuracy],
                      model._compile_metrics)
 
diff --git a/tensorflow/python/kernel_tests/lu_op_test.py b/tensorflow/python/kernel_tests/lu_op_test.py
index f2defcae85..06deb0e1c8 100644
--- a/tensorflow/python/kernel_tests/lu_op_test.py
+++ b/tensorflow/python/kernel_tests/lu_op_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import functional_ops
@@ -41,9 +42,9 @@ class LuOpTest(test.TestCase):
   def float_types(self):
     return set((np.float64, np.float32, np.complex64, np.complex128))
 
-  def _verifyLuBase(self, sess, x, lower, upper, perm, verification,
+  def _verifyLuBase(self, x, lower, upper, perm, verification,
                     output_idx_type):
-    lower_np, upper_np, perm_np, verification_np = sess.run(
+    lower_np, upper_np, perm_np, verification_np = self.evaluate(
         [lower, upper, perm, verification])
 
     self.assertAllClose(x, verification_np)
@@ -65,7 +66,7 @@ class LuOpTest(test.TestCase):
 
   def _verifyLu(self, x, output_idx_type=dtypes.int64):
     # Verify that Px = LU.
-    with self.cached_session(use_gpu=True) as sess:
+    with test_util.use_gpu():
 
       lu, perm = linalg_ops.lu(x, output_idx_type=output_idx_type)
 
@@ -121,7 +122,7 @@ class LuOpTest(test.TestCase):
         verification = array_ops.reshape(permuted_verification_reshaped,
                                          lu_shape)
 
-      self._verifyLuBase(sess, x, lower, upper, perm, verification,
+      self._verifyLuBase(x, lower, upper, perm, verification,
                          output_idx_type)
 
   def testBasic(self):
@@ -139,7 +140,7 @@ class LuOpTest(test.TestCase):
         self._verifyLu(complex_data, output_idx_type=output_idx_type)
 
   def testPivoting(self):
-    with self.session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       # This matrix triggers partial pivoting because the first diagonal entry
       # is small.
       data = np.array([[1e-9, 1., 0.], [1., 0., 0], [0., 1., 5]])
@@ -148,7 +149,7 @@ class LuOpTest(test.TestCase):
       for dtype in (np.float32, np.float64):
         self._verifyLu(data.astype(dtype))
         _, p = linalg_ops.lu(data)
-        p_val = sess.run([p])
+        p_val = self.evaluate([p])
         # Make sure p_val is not the identity permutation.
         self.assertNotAllClose(np.arange(3), p_val)
 
@@ -158,7 +159,7 @@ class LuOpTest(test.TestCase):
         complex_data += data
         self._verifyLu(complex_data)
         _, p = linalg_ops.lu(data)
-        p_val = sess.run([p])
+        p_val = self.evaluate([p])
         # Make sure p_val is not the identity permutation.
         self.assertNotAllClose(np.arange(3), p_val)
 
@@ -166,15 +167,15 @@ class LuOpTest(test.TestCase):
     # LU factorization gives an error when the input is singular.
     # Note: A singular matrix may return without error but it won't be a valid
     # factorization.
-    with self.session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       for dtype in self.float_types:
         with self.assertRaises(errors.InvalidArgumentError):
-          sess.run(
+          self.evaluate(
               linalg_ops.lu(
                   np.array([[1., 2., 3.], [2., 4., 6.], [2., 3., 4.]],
                            dtype=dtype)))
         with self.assertRaises(errors.InvalidArgumentError):
-          sess.run(
+          self.evaluate(
               linalg_ops.lu(
                   np.array([[[1., 2., 3.], [2., 4., 6.], [1., 2., 3.]],
                             [[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]]],
@@ -212,17 +213,19 @@ class LuOpTest(test.TestCase):
     data = np.random.rand(n, n) + 1j * np.random.rand(n, n)
     self._verifyLu(data)
 
+  @test_util.run_v1_only("b/120545219")
   def testEmpty(self):
     self._verifyLu(np.empty([0, 2, 2]))
     self._verifyLu(np.empty([2, 0, 0]))
 
+  @test_util.run_deprecated_v1
   def testConcurrentExecutesWithoutError(self):
-    with self.session(use_gpu=True) as sess:
+    with test_util.use_gpu():
       matrix1 = random_ops.random_normal([5, 5], seed=42)
       matrix2 = random_ops.random_normal([5, 5], seed=42)
       lu1, p1 = linalg_ops.lu(matrix1)
       lu2, p2 = linalg_ops.lu(matrix2)
-      lu1_val, p1_val, lu2_val, p2_val = sess.run([lu1, p1, lu2, p2])
+      lu1_val, p1_val, lu2_val, p2_val = self.evaluate([lu1, p1, lu2, p2])
       self.assertAllEqual(lu1_val, lu2_val)
       self.assertAllEqual(p1_val, p2_val)
 
diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py
index cd99f1774a..d31ecbcd3f 100644
--- a/tensorflow/python/kernel_tests/matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/matmul_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import operator
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -224,7 +225,8 @@ class MatMulInfixOperatorTest(test_lib.TestCase):
 if __name__ == "__main__":
   sizes = [1, 3, 5]
   trans_options = [[False, False], [True, False], [False, True]]
-  for use_static_shape in [False, True]:
+  # TF2 does not support placeholders under eager so we skip it
+  for use_static_shape in set([True, tf2.enabled()]):
     for dtype in (np.int32, np.int64, np.float16, np.float32, np.float64,
                   np.complex64, np.complex128):
       if not use_static_shape and (dtype == np.int32 or dtype == np.int64):
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 323d14b7d8..ce782dbc28 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -161,7 +161,7 @@ def _GetSelfAdjointEigTest(dtype_, shape_, compute_v_):
             math_ops.matmul(tf_v, array_ops.matrix_diag(tf_e)),
             tf_v,
             adjoint_b=True)
-        self.assertAllClose(a_ev.eval(), a, atol=atol)
+        self.assertAllClose(self.evaluate(a_ev), a, atol=atol)
 
         # Compare to numpy.linalg.eigh.
         CompareEigenDecompositions(self, np_e, np_v, self.evaluate(tf_e),
@@ -169,7 +169,7 @@ def _GetSelfAdjointEigTest(dtype_, shape_, compute_v_):
       else:
         tf_e = linalg_ops.self_adjoint_eigvals(constant_op.constant(a))
         self.assertAllClose(
-            np.sort(np_e, -1), np.sort(tf_e.eval(), -1), atol=atol)
+            np.sort(np_e, -1), np.sort(self.evaluate(tf_e), -1), atol=atol)
 
   return Test
 
diff --git a/tensorflow/python/kernel_tests/tensordot_op_test.py b/tensorflow/python/kernel_tests/tensordot_op_test.py
index 123c9b376c..2894b7dd50 100644
--- a/tensorflow/python/kernel_tests/tensordot_op_test.py
+++ b/tensorflow/python/kernel_tests/tensordot_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python import tf2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
@@ -99,7 +100,7 @@ class TensordotTest(test_lib.TestCase):
 
         tf_a = array_ops.ones((3, 3), dtype=dtypes.float32)
         tf_b = constant_op.constant([2, 3, 1], dtype=dtypes.float32)[None, None]
-        tf_ans = math_ops.tensordot(tf_a, tf_b, axes_value).eval()
+        tf_ans = math_ops.tensordot(tf_a, tf_b, axes_value)
 
         self.assertAllEqual(tf_ans.shape, np_ans.shape)
         self.assertAllEqual(tf_ans, np_ans)
@@ -178,7 +179,7 @@ def _get_tensordot_tests(dtype_, rank_a_, rank_b_, num_dims_, dynamic_shape_):
                   axes: (a_dims_np, b_dims_np)
               })
         else:
-          tf_ans = math_ops.tensordot(a_np, b_np, (a_dims_np, b_dims_np)).eval()
+          tf_ans = math_ops.tensordot(a_np, b_np, (a_dims_np, b_dims_np))
       self.assertAllClose(tf_ans, np_ans, rtol=tol, atol=tol)
       self.assertAllEqual(tf_ans.shape, np_ans.shape)
 
@@ -208,7 +209,7 @@ def _get_tensordot_tests(dtype_, rank_a_, rank_b_, num_dims_, dynamic_shape_):
           c = math_ops.tensordot(a, b, axes=axes)
           tf_ans = sess.run(c, feed_dict={a: a_np, b: b_np})
         else:
-          tf_ans = math_ops.tensordot(a_np, b_np, axes=axes).eval()
+          tf_ans = math_ops.tensordot(a_np, b_np, axes=axes)
       self.assertAllClose(tf_ans, np_ans, rtol=tol, atol=tol)
       self.assertAllEqual(tf_ans.shape, np_ans.shape)
 
@@ -220,7 +221,8 @@ if __name__ == "__main__":
     for rank_a in 1, 2, 4, 5:
       for rank_b in 1, 2, 4, 5:
         for num_dims in range(0, min(rank_a, rank_b) + 1):
-          for dynamic_shape in False, True:
+          # TF2 does not support placeholders under eager so we skip it
+          for dynamic_shape in set([False, not tf2.enabled()]):
             for testcase in _get_tensordot_tests(dtype, rank_a, rank_b,
                                                  num_dims, dynamic_shape):
               name = "%s_%s_%s_%s_%s_%s" % (testcase.__name__, dtype.__name__,
-- 
GitLab


From 71ea120a1bdfea823c4e3d27c55c2d2b885b147d Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 7 Dec 2018 14:09:06 -0800
Subject: [PATCH 1514/1554] Use eager_py_func instead of the deprecated
 py_func. Add a bit of extra logic to account for the potential use of
 tf.print in Python 2.

PiperOrigin-RevId: 224574979
---
 .../python/autograph/operators/py_builtins.py      |  2 ++
 tensorflow/python/autograph/pyct/parser.py         | 13 ++++++++++++-
 tensorflow/python/autograph/utils/py_func.py       |  5 +++--
 tensorflow/python/autograph/utils/py_func_test.py  | 14 +++++++-------
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/autograph/operators/py_builtins.py b/tensorflow/python/autograph/operators/py_builtins.py
index 2f55d53892..ddf05f73f3 100644
--- a/tensorflow/python/autograph/operators/py_builtins.py
+++ b/tensorflow/python/autograph/operators/py_builtins.py
@@ -174,6 +174,7 @@ def _tf_py_func_print(objects, kwargs):
     override_kwargs['flush'] = True
 
   def print_wrapper(*vals):
+    vals = tuple(v.numpy() if tensor_util.is_tensor(v) else v for v in vals)
     if six.PY3:
       # TensorFlow doesn't seem to generate Unicode when passing strings to
       # py_func. This causes the print to add a "b'" wrapper to the output,
@@ -193,6 +194,7 @@ def range_(start_or_stop, stop=UNDEFINED, step=UNDEFINED):
 
 
 def _tf_range(start_or_stop, stop, step):
+  """Overload of range_ that generates a TF range tensor."""
   # Note: for static inputs (e.g. constants), tf.range errors out at graph
   # construction time, instead of returning an empty tensor. Preventing the
   # graph construction error aligns the semantics with Python.
diff --git a/tensorflow/python/autograph/pyct/parser.py b/tensorflow/python/autograph/pyct/parser.py
index 8f4037c5e2..39fc1a7ed0 100644
--- a/tensorflow/python/autograph/pyct/parser.py
+++ b/tensorflow/python/autograph/pyct/parser.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 import textwrap
 
 import gast
+import six
 
 from tensorflow.python.util import tf_inspect
 
@@ -91,7 +92,17 @@ def parse_entity(entity):
 def parse_str(src):
   """Returns the AST of given piece of code."""
   # TODO(mdan): This should exclude the module things are autowrapped in.
-  return gast.parse(src)
+
+  if six.PY2 and '.print(' in src:
+    # This special treatment is required because gast.parse is not aware of
+    # whether print_function was present in the original context.
+    src = 'from __future__ import print_function\n' + src
+    parsed_module = gast.parse(src)
+    parsed_module.body = parsed_module.body[1:]
+  else:
+    parsed_module = gast.parse(src)
+
+  return parsed_module
 
 
 def parse_expression(src):
diff --git a/tensorflow/python/autograph/utils/py_func.py b/tensorflow/python/autograph/utils/py_func.py
index 11ebfb2e49..ee8b46b520 100644
--- a/tensorflow/python/autograph/utils/py_func.py
+++ b/tensorflow/python/autograph/utils/py_func.py
@@ -127,5 +127,6 @@ def wrap_py_func(f, return_dtypes, args, kwargs=None, use_dummy_return=False):
     retval = f(*f_args, **f_kwargs)
     return 1 if use_dummy_return else retval
 
-  return script_ops.py_func(f_wrapper, tensor_args, dtypes.int64
-                            if use_dummy_return else return_dtypes)
+  if use_dummy_return:
+    return_dtypes = dtypes.int32
+  return script_ops.eager_py_func(f_wrapper, tensor_args, return_dtypes)
diff --git a/tensorflow/python/autograph/utils/py_func_test.py b/tensorflow/python/autograph/utils/py_func_test.py
index 28cefd8c3e..d17ede7714 100644
--- a/tensorflow/python/autograph/utils/py_func_test.py
+++ b/tensorflow/python/autograph/utils/py_func_test.py
@@ -32,13 +32,13 @@ class PyFuncTest(test.TestCase):
       return a + b + c
 
     with self.cached_session() as sess:
-      result = py_func.wrap_py_func(test_fn, dtypes.int64,
+      result = py_func.wrap_py_func(test_fn, dtypes.int32,
                                     (1, constant_op.constant(1), 1))
       self.assertEqual(3, self.evaluate(result))
-      result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, 1, 1))
+      result = py_func.wrap_py_func(test_fn, dtypes.int32, (1, 1, 1))
       self.assertEqual(3, self.evaluate(result))
       result = py_func.wrap_py_func(
-          test_fn, dtypes.int64,
+          test_fn, dtypes.int32,
           (constant_op.constant(1), 1, constant_op.constant(1)))
       self.assertEqual(3, self.evaluate(result))
 
@@ -53,9 +53,9 @@ class PyFuncTest(test.TestCase):
       return a * b.foo
 
     with self.cached_session() as sess:
-      result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass()))
+      result = py_func.wrap_py_func(test_fn, dtypes.int32, (7, TestClass()))
       self.assertEqual(35, self.evaluate(result))
-      result = py_func.wrap_py_func(test_fn, dtypes.int64,
+      result = py_func.wrap_py_func(test_fn, dtypes.int32,
                                     (constant_op.constant(7), TestClass()))
       self.assertEqual(35, self.evaluate(result))
 
@@ -70,12 +70,12 @@ class PyFuncTest(test.TestCase):
       return a * b.foo + c * d.foo
 
     with self.cached_session() as sess:
-      result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass(5)), {
+      result = py_func.wrap_py_func(test_fn, dtypes.int32, (7, TestClass(5)), {
           'c': 11,
           'd': TestClass(13)
       })
       self.assertEqual(178, self.evaluate(result))
-      result = py_func.wrap_py_func(test_fn, dtypes.int64,
+      result = py_func.wrap_py_func(test_fn, dtypes.int32,
                                     (constant_op.constant(7), TestClass(5)), {
                                         'c': constant_op.constant(11),
                                         'd': TestClass(13)
-- 
GitLab


From 093f0363238c773255c580ea1390e9fdf666d69c Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 7 Dec 2018 14:14:36 -0800
Subject: [PATCH 1515/1554] Utility to run tests inside tf.function and eager.

Relies on being able to run the assert* test methods inside a
py_func to run them inside the graph, so there's no need for
self.evaluate or similar methods which create a graph/eager
hybrid programming model.

PiperOrigin-RevId: 224575790
---
 tensorflow/python/eager/function_test.py      |  9 +--
 tensorflow/python/framework/test_util.py      | 72 ++++++++++++++++++-
 .../kernel_tests/variable_scope_test.py       |  5 +-
 3 files changed, 78 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 71afbd24d8..e0854b0632 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -428,20 +428,21 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       self.evaluate(variables.global_variables_initializer())
     self.assertEqual(self.evaluate(value), 2.0)
 
-  @test_util.run_in_graph_and_eager_modes
+  @test_util.also_run_as_tf_function
   def testInitScopeTensorInitializationInFunction(self):
 
     @def_function.function
     def tensor_init():
       with ops.init_scope():
         const = constant_op.constant(2.0)
+      # Note: this variable bypasses tf.function's variable creation
+      # requirements by bypassing variable_creator_scope by using
+      # ResourceVariable instead of Variable.
       self.v = resource_variable_ops.ResourceVariable(const)
       return self.v.read_value()
 
     value = tensor_init()
-    if not context.executing_eagerly():
-      self.evaluate(variables.global_variables_initializer())
-    self.assertEqual(self.evaluate(value), 2.0)
+    self.assertAllEqual(value, 2.0)
 
   def testDefunShapeInferenceWithCapturedResourceVariable(self):
     v = resource_variable_ops.ResourceVariable([[1, 2], [3, 4]])
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index b0c3c9b506..06316ce2e9 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -54,6 +54,7 @@ from tensorflow.python import tf2
 from tensorflow.python.client import device_lib
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
@@ -67,6 +68,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import versions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
@@ -76,6 +78,7 @@ from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
 from tensorflow.python.util import memory
 from tensorflow.python.util import nest
+from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 from tensorflow.python.util.protobuf import compare
 from tensorflow.python.util.tf_export import tf_export
@@ -1009,6 +1012,58 @@ def run_in_graph_and_eager_modes(func=None,
   return decorator
 
 
+def py_func_if_in_function(f):
+
+  def decorated(*args, **kwds):
+    if not ops.get_default_graph()._building_function:
+      return f(*args, **kwds)
+
+    tensor_args, tensor_indices = zip(
+        *[(x, i) for i, x in enumerate(args)
+          if isinstance(x, (ops.Tensor, variables.Variable))])
+
+    def inner_f(*inner_tensor_args):
+      my_args = list(args)
+      for i, n in zip(tensor_indices, inner_tensor_args):
+        my_args[i] = n
+      return f(*my_args, **kwds)
+
+    return script_ops.py_func(inner_f, tensor_args, [])
+
+  return tf_decorator.make_decorator(f, decorated)
+
+
+def also_run_as_tf_function(f):
+  """Runs the decorated test twice--once as is, once inside a tf.function.
+
+  This allows you to run a test both in eager execution and inside a
+  tf.function, exercising the two execution modes supported in tf 2.0. The test
+  assertions are automatically done inside tf.py_funcs, and tf.function ensures
+  that they run in the proper order and with the proper side effects.
+
+  Currently variable creation is not supported in tests annotated with this
+  decorator since it's tricky to ensure the variable doesn't get repeatedly
+  created when retracing the tf.function.
+
+  Args:
+    f: the test method to be decorated
+
+  Returns:
+    The decorated test method, which will run both in eager and inside a
+    tf.function.
+  """
+
+  def decorated(*args, **kwds):
+    with context.eager_mode():
+      # Running in eager mode
+      f(*args, **kwds)
+
+      defun_f = def_function.function(f)
+      defun_f(*args, **kwds)
+
+  return decorated
+
+
 def run_deprecated_v1(func=None):
   """Execute the decorated test in graph mode.
 
@@ -1783,8 +1838,8 @@ class TensorFlowTestCase(googletest.TestCase):
     return ret
 
 
-# pylint: enable=invalid-name
-
+  # pylint: enable=invalid-name
+  @py_func_if_in_function
   def assertNear(self, f1, f2, err, msg=None):
     """Asserts that two floats are near each other.
 
@@ -1803,6 +1858,7 @@ class TensorFlowTestCase(googletest.TestCase):
         "%f != %f +/- %f%s" % (f1, f2, err, " (%s)" % msg
                                if msg is not None else ""))
 
+  @py_func_if_in_function
   def assertArrayNear(self, farray1, farray2, err, msg=None):
     """Asserts that two float arrays are near each other.
 
@@ -1822,6 +1878,7 @@ class TensorFlowTestCase(googletest.TestCase):
   def _NDArrayNear(self, ndarray1, ndarray2, err):
     return np.linalg.norm(ndarray1 - ndarray2) < err
 
+  @py_func_if_in_function
   def assertNDArrayNear(self, ndarray1, ndarray2, err, msg=None):
     """Asserts that two numpy arrays have near values.
 
@@ -1959,6 +2016,7 @@ class TensorFlowTestCase(googletest.TestCase):
         e.args = ((e.args[0] + " : " + msg,) + e.args[1:])
         raise
 
+  @py_func_if_in_function
   def assertAllClose(self, a, b, rtol=1e-6, atol=1e-6, msg=None):
     """Asserts that two structures of numpy arrays or Tensors, have near values.
 
@@ -1984,6 +2042,7 @@ class TensorFlowTestCase(googletest.TestCase):
     """
     self._assertAllCloseRecursive(a, b, rtol=rtol, atol=atol, msg=msg)
 
+  @py_func_if_in_function
   def assertAllCloseAccordingToType(self,
                                     a,
                                     b,
@@ -2031,6 +2090,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     self.assertAllClose(a, b, rtol=rtol, atol=atol, msg=msg)
 
+  @py_func_if_in_function
   def assertNotAllClose(self, a, b, **kwargs):
     """Assert that two numpy arrays, or or Tensors, do not have near values.
 
@@ -2049,6 +2109,7 @@ class TensorFlowTestCase(googletest.TestCase):
       return
     raise AssertionError("The two values are close at all elements")
 
+  @py_func_if_in_function
   def assertAllEqual(self, a, b, msg=None):
     """Asserts that two numpy arrays or Tensors have the same values.
 
@@ -2091,6 +2152,7 @@ class TensorFlowTestCase(googletest.TestCase):
       msgs.append("not equal rhs = {}".format(y))
       np.testing.assert_array_equal(a, b, err_msg="\n".join(msgs))
 
+  @py_func_if_in_function
   def assertAllGreater(self, a, comparison_target):
     """Assert element values are all greater than a target value.
 
@@ -2102,6 +2164,7 @@ class TensorFlowTestCase(googletest.TestCase):
     a = self._GetNdArray(a)
     self.assertGreater(np.min(a), comparison_target)
 
+  @py_func_if_in_function
   def assertAllLess(self, a, comparison_target):
     """Assert element values are all less than a target value.
 
@@ -2113,6 +2176,7 @@ class TensorFlowTestCase(googletest.TestCase):
     a = self._GetNdArray(a)
     self.assertLess(np.max(a), comparison_target)
 
+  @py_func_if_in_function
   def assertAllGreaterEqual(self, a, comparison_target):
     """Assert element values are all greater than or equal to a target value.
 
@@ -2124,6 +2188,7 @@ class TensorFlowTestCase(googletest.TestCase):
     a = self._GetNdArray(a)
     self.assertGreaterEqual(np.min(a), comparison_target)
 
+  @py_func_if_in_function
   def assertAllLessEqual(self, a, comparison_target):
     """Assert element values are all less than or equal to a target value.
 
@@ -2166,6 +2231,7 @@ class TensorFlowTestCase(googletest.TestCase):
       lines.append(prefix + "...")
     return lines
 
+  @py_func_if_in_function
   def assertAllInRange(self,
                        target,
                        lower_bound,
@@ -2224,6 +2290,7 @@ class TensorFlowTestCase(googletest.TestCase):
           "Subscript(s) and value(s) of the offending elements:\n" +
           "\n".join(self._format_subscripts(violation_subscripts, target)))
 
+  @py_func_if_in_function
   def assertAllInSet(self, target, expected_set):
     """Assert that elements of a Tensor are all in a given closed set.
 
@@ -2245,6 +2312,7 @@ class TensorFlowTestCase(googletest.TestCase):
       raise AssertionError("%d unique element(s) are not in the set %s: %s" %
                            (np.size(diff), expected_set, diff))
 
+  @py_func_if_in_function
   def assertDTypeEqual(self, target, expected_dtype):
     """Assert ndarray data type is equal to expected.
 
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 44d4bd5e30..451eb38530 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -237,7 +237,8 @@ class VariableScopeTest(test.TestCase):
         _ = d2(x)
         self.assertEqual(len(d2.variables), 2)
         v3, v4 = d2.variables
-        self.assertAllEqual([v1, v2], [v3, v4])
+        self.assertEqual(v1, v3)
+        self.assertEqual(v2, v4)
       f()
 
   # TODO(mihaimaruseac): Not converted to use wrap_function because of
@@ -1684,7 +1685,7 @@ class VariableScopeWithCustomGetterTest(test.TestCase):
       with variable_scope.variable_creator_scope(creator_b):
         variable_scope.variable(1.0, name="one_name")
 
-    self.assertAllEqual(variable_names, ["forced_name"])
+    self.assertEqual(variable_names[0], "forced_name")
 
     called = [False]
 
-- 
GitLab


From 2a050766bf0556d7d92eea62d40fd2bebbcb399f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 14:18:35 -0800
Subject: [PATCH 1516/1554] human_readable_json.h uses the correct switched
 namespace protobuf but the implementation directly references
 google::protobuf. Fix it so that it agrees with its header file.

PiperOrigin-RevId: 224576410
---
 tensorflow/core/platform/default/human_readable_json.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/platform/default/human_readable_json.cc b/tensorflow/core/platform/default/human_readable_json.cc
index 9f97c8272c..bf9c7b7620 100644
--- a/tensorflow/core/platform/default/human_readable_json.cc
+++ b/tensorflow/core/platform/default/human_readable_json.cc
@@ -20,7 +20,7 @@ limitations under the License.
 
 namespace tensorflow {
 
-Status ProtoToHumanReadableJson(const ::google::protobuf::Message& proto,
+Status ProtoToHumanReadableJson(const protobuf::Message& proto,
                                 string* result) {
 #ifdef TENSORFLOW_LITE_PROTOS
   *result = "[human readable output not available on Android]";
@@ -28,7 +28,7 @@ Status ProtoToHumanReadableJson(const ::google::protobuf::Message& proto,
 #else
   result->clear();
 
-  auto status = google::protobuf::util::MessageToJsonString(proto, result);
+  auto status = protobuf::util::MessageToJsonString(proto, result);
   if (!status.ok()) {
     // Convert error_msg google::protobuf::StringPiece to
     // tensorflow::StringPiece.
@@ -41,8 +41,7 @@ Status ProtoToHumanReadableJson(const ::google::protobuf::Message& proto,
 #endif
 }
 
-Status HumanReadableJsonToProto(const string& str,
-                                ::google::protobuf::Message* proto) {
+Status HumanReadableJsonToProto(const string& str, protobuf::Message* proto) {
 #ifdef TENSORFLOW_LITE_PROTOS
   return errors::Internal("Cannot parse JSON protos on Android");
 #else
-- 
GitLab


From 9ed800cea0d5872511c1b45708d0e8970e4fc958 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 7 Dec 2018 14:23:42 -0800
Subject: [PATCH 1517/1554] Bug fix for unified LSTM with time major and
 go_backward combined.

Also added test case to cover that.

PiperOrigin-RevId: 224577299
---
 tensorflow/python/keras/layers/recurrent.py   |  2 +-
 .../python/keras/layers/unified_lstm_test.py  | 44 +++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index fdc2acd538..86a69e45d9 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -2704,7 +2704,7 @@ class UnifiedLSTM(LSTM):
       # both normal and CuDNN implementations.
       if self.go_backwards:
         # Reverse time axis.
-        inputs = K.reverse(inputs, 1)
+        inputs = K.reverse(inputs, 0 if self.time_major else 1)
 
       if 0 < self.dropout < 1:
         if self._dropout_mask is None:
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 8cbc370633..33351948ee 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -298,6 +298,50 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
     self.assertAllClose(y_1, y_2)
     self.assertAllClose(y_2, y_3)
 
+  @parameterized.named_parameters(
+      # test_name, time_major, go_backwards
+      ('normal', False, False),
+      ('time_major', True, False),
+      ('go_backwards', False, True),
+      ('both', True, True),
+  )
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_time_major_and_go_backward(self, time_major, go_backwards):
+    input_shape = 10
+    rnn_state_size = 8
+    timestep = 4
+    batch = 100
+
+    x_train = np.random.random((batch, timestep, input_shape))
+
+    def build_model(layer_cls):
+      inputs = keras.layers.Input(
+          shape=[timestep, input_shape], dtype=dtypes.float32)
+      layer = layer_cls(rnn_state_size,
+                        recurrent_activation='sigmoid',
+                        time_major=time_major,
+                        return_sequences=True,
+                        go_backwards=go_backwards)
+      if time_major:
+        converted_input = keras.layers.Lambda(
+            lambda t: array_ops.transpose(t, [1, 0, 2]))(inputs)
+        outputs = layer(converted_input)
+        outputs = keras.layers.Lambda(
+            lambda t: array_ops.transpose(t, [1, 0, 2]))(outputs)
+      else:
+        outputs = layer(inputs)
+      return keras.models.Model(inputs, outputs)
+
+    lstm_model = build_model(keras.layers.LSTM)
+    y_ref = lstm_model.predict(x_train)
+    weights = lstm_model.get_weights()
+
+    unified_lstm_model = build_model(keras.layers.UnifiedLSTM)
+    unified_lstm_model.set_weights(weights)
+    y = unified_lstm_model.predict(x_train)
+
+    self.assertAllClose(y, y_ref)
+
   @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_keras_model_with_lstm(self):
     input_shape = 10
-- 
GitLab


From 7eeb77ab979acf284d68fbd9b48d54152fc5d90b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 14:29:40 -0800
Subject: [PATCH 1518/1554] [TF:XLA] Rename misnamed variables in a test.

PiperOrigin-RevId: 224578284
---
 .../compiler/xla/tests/array_elementwise_ops_test.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index f6be27bee2..915b456b52 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -329,13 +329,13 @@ TEST_P(ArrayElementwiseOpTestParamCount, AddManyValues) {
   Literal b_literal = LiteralUtil::CreateR1<float>({b_values});
   std::unique_ptr<GlobalData> b_data =
       client_->TransferToServer(b_literal).ConsumeValueOrDie();
-  auto b_constant = Parameter(&builder, 1, a_literal.shape(), "b_param");
-  auto b_param = ConstantR1<float>(&builder, b_values);
+  auto b_param = Parameter(&builder, 1, a_literal.shape(), "b_param");
+  auto b_constant = ConstantR1<float>(&builder, b_values);
 
-  auto sum1 = Add(a_constant, b_constant);
-  auto sum2 = Add(a_constant, b_param);
-  auto sum3 = Add(a_param, b_constant);
-  auto sum4 = Add(a_param, b_param);
+  auto sum1 = Add(a_constant, b_param);
+  auto sum2 = Add(a_constant, b_constant);
+  auto sum3 = Add(a_param, b_param);
+  auto sum4 = Add(a_param, b_constant);
 
   auto sum = Add(sum1, sum2);
   sum = Add(sum, sum3);
-- 
GitLab


From 9c5d513df64aa61e8e7a8ba3939e32fdd13f49f1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 14:41:30 -0800
Subject: [PATCH 1519/1554] Update the remote execution toolchain to make it
 work with Bazel 0.20.0

PiperOrigin-RevId: 224580438
---
 .../preconfig/win_1803/bazel_018/BUILD        |  7 ++++
 .../preconfig/win_1803/bazel_018/CROSSTOOL    | 36 -------------------
 2 files changed, 7 insertions(+), 36 deletions(-)

diff --git a/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD b/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
index c00f005e46..edd9583648 100644
--- a/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+++ b/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
@@ -39,6 +39,9 @@ cc_toolchain_suite(
         "x64_windows|msvc-cl": ":cc-compiler-x64_windows",
         "x64_windows|msys-gcc": ":cc-compiler-x64_windows_msys",
         "x64_windows|mingw-gcc": ":cc-compiler-x64_windows_mingw",
+        "x64_windows_msys": ":cc-compiler-x64_windows_msys",
+        "x64_windows": ":cc-compiler-x64_windows",
+        "armeabi-v7a": ":cc-compiler-armeabi-v7a",
     },
 )
 
@@ -54,6 +57,7 @@ cc_toolchain(
     static_runtime_libs = [":empty"],
     strip_files = ":empty",
     supports_param_files = 1,
+    toolchain_identifier = "msys_x64",
 )
 
 toolchain(
@@ -83,6 +87,7 @@ cc_toolchain(
     static_runtime_libs = [":empty"],
     strip_files = ":empty",
     supports_param_files = 0,
+    toolchain_identifier = "msys_x64_mingw",
 )
 
 toolchain(
@@ -112,6 +117,7 @@ cc_toolchain(
     static_runtime_libs = [":empty"],
     strip_files = ":empty",
     supports_param_files = 1,
+    toolchain_identifier = "msvc_x64",
 )
 
 toolchain(
@@ -140,6 +146,7 @@ cc_toolchain(
     static_runtime_libs = [":empty"],
     strip_files = ":empty",
     supports_param_files = 1,
+    toolchain_identifier = "stub_armeabi-v7a",
 )
 
 toolchain(
diff --git a/third_party/toolchains/preconfig/win_1803/bazel_018/CROSSTOOL b/third_party/toolchains/preconfig/win_1803/bazel_018/CROSSTOOL
index 04c8bcae45..38a80c22da 100644
--- a/third_party/toolchains/preconfig/win_1803/bazel_018/CROSSTOOL
+++ b/third_party/toolchains/preconfig/win_1803/bazel_018/CROSSTOOL
@@ -14,42 +14,6 @@
 
 major_version: "local"
 minor_version: ""
-default_target_cpu: "same_as_host"
-
-default_toolchain {
-  cpu: "x64_windows"
-  toolchain_identifier: "msvc_x64"
-}
-
-default_toolchain {
-  cpu: "local"
-  toolchain_identifier: "stub_armeabi-v7a"
-}
-
-default_toolchain {
-  cpu: "armeabi-v7a"
-  toolchain_identifier: "stub_armeabi-v7a"
-}
-
-default_toolchain {
-  cpu: "x64_windows"
-  toolchain_identifier: "msvc_x64"
-}
-
-default_toolchain {
-  cpu: "x64_windows_msvc"
-  toolchain_identifier: "msvc_x64"
-}
-
-default_toolchain {
-  cpu: "x64_windows_msys"
-  toolchain_identifier: "msys_x64"
-}
-
-default_toolchain {
-  cpu: "s390x"
-  toolchain_identifier: "msys_x64"
-}
 
 # Android tooling requires a default toolchain for the armeabi-v7a cpu.
 toolchain {
-- 
GitLab


From 5c4a09cc0b0597aba582ccaf41cd23cb8c541d23 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Fri, 7 Dec 2018 14:50:34 -0800
Subject: [PATCH 1520/1554] Int8 dequantize implementation.

PiperOrigin-RevId: 224581893
---
 tensorflow/lite/kernels/dequantize.cc         | 27 +++++++++---
 tensorflow/lite/kernels/dequantize_test.cc    | 33 ++++++++++++---
 tensorflow/lite/kernels/internal/BUILD        |  1 +
 .../internal/reference/integer_ops/README.md  |  8 ++++
 .../reference/integer_ops/dequantize.h        | 42 +++++++++++++++++++
 tensorflow/lite/kernels/register.cc           |  4 +-
 tensorflow/lite/toco/tflite/operator.cc       | 24 +++++++++++
 7 files changed, 127 insertions(+), 12 deletions(-)
 create mode 100644 tensorflow/lite/kernels/internal/reference/integer_ops/README.md
 create mode 100644 tensorflow/lite/kernels/internal/reference/integer_ops/dequantize.h

diff --git a/tensorflow/lite/kernels/dequantize.cc b/tensorflow/lite/kernels/dequantize.cc
index b2825bb9ea..7f03c73c9c 100644
--- a/tensorflow/lite/kernels/dequantize.cc
+++ b/tensorflow/lite/kernels/dequantize.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/lite/c/builtin_op_data.h"
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/reference/integer_ops/dequantize.h"
 #include "tensorflow/lite/kernels/internal/tensor.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
 #include "tensorflow/lite/kernels/op_macros.h"
@@ -57,7 +58,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   OpContext op_context(context, node);
 
-  TF_LITE_ENSURE(context, op_context.input->type == kTfLiteUInt8);
+  TF_LITE_ENSURE(context, op_context.input->type == kTfLiteUInt8 ||
+                              op_context.input->type == kTfLiteInt8);
 
   op_context.output->type = kTfLiteFloat32;
   // If the input tensor is constant, we can persist the dequantized value in
@@ -80,10 +82,25 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   tflite::DequantizationParams op_params;
   op_params.zero_point = op_context.input->params.zero_point;
   op_params.scale = op_context.input->params.scale;
-  optimized_ops::Dequantize(op_params, GetTensorShape(op_context.input),
-                            GetTensorData<uint8_t>(op_context.input),
-                            GetTensorShape(op_context.output),
-                            GetTensorData<float>(op_context.output));
+  switch (op_context.input->type) {
+    case kTfLiteUInt8:
+      optimized_ops::Dequantize(op_params, GetTensorShape(op_context.input),
+                                GetTensorData<uint8_t>(op_context.input),
+                                GetTensorShape(op_context.output),
+                                GetTensorData<float>(op_context.output));
+      break;
+    case kTfLiteInt8:
+      reference_integer_ops::Dequantize(
+          op_params, GetTensorShape(op_context.input),
+          GetTensorData<int8_t>(op_context.input),
+          GetTensorShape(op_context.output),
+          GetTensorData<float>(op_context.output));
+      break;
+    default:
+      context->ReportError(context, "Type %d not supported.",
+                           op_context.input->type);
+      return kTfLiteError;
+  }
 
   if (IsConstantTensor(op_context.input)) {
     op_data->float_dequantized_weights_initialized = true;
diff --git a/tensorflow/lite/kernels/dequantize_test.cc b/tensorflow/lite/kernels/dequantize_test.cc
index 55265d93e5..bb5f1e74a8 100644
--- a/tensorflow/lite/kernels/dequantize_test.cc
+++ b/tensorflow/lite/kernels/dequantize_test.cc
@@ -25,8 +25,16 @@ using ::testing::ElementsAreArray;
 
 class DequantizeOpModel : public SingleOpModel {
  public:
-  DequantizeOpModel(std::initializer_list<int> shape, float min, float max) {
-    input_ = AddInput({TensorType_UINT8, shape, min, max});
+  DequantizeOpModel(TensorType type, std::initializer_list<int> shape,
+                    float scale, int32_t zero_point) {
+    TensorData input_tensor_data;
+    input_tensor_data.type = type;
+    input_tensor_data.shape = shape;
+    input_tensor_data.min = 0;
+    input_tensor_data.max = 0;
+    input_tensor_data.scale = scale;
+    input_tensor_data.zero_point = zero_point;
+    input_ = AddInput(input_tensor_data);
     output_ = AddOutput({TensorType_FLOAT32, shape});
     SetBuiltinOp(BuiltinOperator_DEQUANTIZE, BuiltinOptions_DequantizeOptions,
                  CreateDequantizeOptions(builder_).Union());
@@ -34,7 +42,8 @@ class DequantizeOpModel : public SingleOpModel {
     BuildInterpreter({GetShape(input_)});
   }
 
-  void SetInput(std::initializer_list<uint8_t> data) {
+  template <typename T>
+  void SetInput(std::initializer_list<T> data) {
     PopulateTensor(input_, data);
   }
 
@@ -45,10 +54,22 @@ class DequantizeOpModel : public SingleOpModel {
   int output_;
 };
 
-TEST(SplitOpTest, FourDimensional) {
-  DequantizeOpModel m({2, 5}, -63.5, 64);
+TEST(DequantizeOpTest, UINT8) {
+  // [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
+  DequantizeOpModel m(TensorType_UINT8, {2, 5}, 0.5, 127);
 
-  m.SetInput({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
+  m.SetInput<uint8>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear(
+                  {-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64})));
+}
+
+TEST(DequantizeOpTest, INT8) {
+  // [-63.5, 64] -> scale=0.5, zero_point=1 for INT8
+  DequantizeOpModel m(TensorType_INT8, {2, 5}, 0.5, -1);
+
+  m.SetInput<int8>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
   m.Invoke();
   EXPECT_THAT(m.GetOutput(),
               ElementsAreArray(ArrayFloatNear(
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index d2f7f455e4..69816583f5 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -313,6 +313,7 @@ cc_library(
         "reference/depthwiseconv_float.h",
         "reference/depthwiseconv_uint8.h",
         "reference/fully_connected.h",
+        "reference/integer_ops/dequantize.h",
         "reference/reference_ops.h",
         "reference/softmax.h",
     ],
diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/README.md b/tensorflow/lite/kernels/internal/reference/integer_ops/README.md
new file mode 100644
index 0000000000..4b1d3c91d5
--- /dev/null
+++ b/tensorflow/lite/kernels/internal/reference/integer_ops/README.md
@@ -0,0 +1,8 @@
+This directory contains reference implementations for int8 fully integer kernels.
+
+Weight filters of convs are expected to be symmetric per-channel quantized in
+the range [-127, 127].
+Inputs/activations are expected to be asymmetric per-layer quantized in the
+range [-128, 127].
+
+THESE ARE EXPERIMENTAL AND PRONE TO CHANGE.
diff --git a/tensorflow/lite/kernels/internal/reference/integer_ops/dequantize.h b/tensorflow/lite/kernels/internal/reference/integer_ops/dequantize.h
new file mode 100644
index 0000000000..03dcb6c220
--- /dev/null
+++ b/tensorflow/lite/kernels/internal/reference/integer_ops/dequantize.h
@@ -0,0 +1,42 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEQUANTIZE_H_
+#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEQUANTIZE_H_
+
+#include "tensorflow/lite/kernels/internal/common.h"
+#include "tensorflow/lite/kernels/internal/types.h"
+
+namespace tflite {
+namespace reference_integer_ops {
+
+inline void Dequantize(const tflite::DequantizationParams& op_params,
+                       const RuntimeShape& input_shape, const int8* input_data,
+                       const RuntimeShape& output_shape, float* output_data) {
+  const int32 zero_point = op_params.zero_point;
+  const double scale = op_params.scale;
+  const int flat_size = MatchingFlatSize(input_shape, output_shape);
+
+  for (int i = 0; i < flat_size; i++) {
+    const int32 val = input_data[i];
+    const float result = static_cast<float>(scale * (val - zero_point));
+    output_data[i] = result;
+  }
+}
+
+}  // namespace reference_integer_ops
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_INTEGER_OPS_DEQUANTIZE_H_
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 3c60d281b3..c0e6f6994f 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -222,7 +222,9 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LOG, Register_LOG());
   AddBuiltin(BuiltinOperator_LOG_SOFTMAX, Register_LOG_SOFTMAX());
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
-  AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE());
+  AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE(),
+             /* min_version */ 1,
+             /* max_version */ 2);
   AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
   AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
   AddBuiltin(BuiltinOperator_MINIMUM, Register_MINIMUM());
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index e0faed4927..205af23da5 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1482,6 +1482,30 @@ class TensorFlowUnsupported : public BaseOperator {
   const bool enable_select_tf_ops_;
 };
 
+class Dequantize
+    : public BuiltinOperator<DequantizeOperator, ::tflite::DequantizeOptions,
+                             ::tflite::BuiltinOptions_DequantizeOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateDequantizeOptions(*builder);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {}
+
+  int GetVersion(const Operator& op) const override {
+    // TODO(suharshs): Dequantize now supports INT8 in addition to
+    // QUANTIZED_UINT8. When TOCO can create models with INT8, we need
+    // to find a way to see the type here and return version 2. Right now
+    // version 2 will only be added by post training quantization tools.
+    return 1;
+  }
+};
+
 namespace {
 // Build a vector containing all the known operators.
 std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
-- 
GitLab


From cd231a62a664e577aa5171d7b3a44a04867f2035 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 7 Dec 2018 15:05:59 -0800
Subject: [PATCH 1521/1554] [tf.data] Expose
 `tf.data.experimental.NestedStructure`.

PiperOrigin-RevId: 224584699
---
 .../python/data/experimental/__init__.py       |  2 ++
 tensorflow/python/data/util/structure.py       |  1 +
 ...w.data.experimental.-nested-structure.pbtxt | 18 ++++++++++++++++++
 .../v1/tensorflow.data.experimental.pbtxt      |  4 ++++
 ...w.data.experimental.-nested-structure.pbtxt | 18 ++++++++++++++++++
 .../v2/tensorflow.data.experimental.pbtxt      |  4 ++++
 6 files changed, 47 insertions(+)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-nested-structure.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-nested-structure.pbtxt

diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index 365c53405c..ffc2e5ef5f 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -26,6 +26,7 @@ See [Importing Data](https://tensorflow.org/guide/datasets) for an overview.
 @@CheckpointInputPipelineHook
 @@CsvDataset
 @@DatasetStructure
+@@NestedStructure
 @@OptimizationOptions
 @@Optional
 @@OptionalStructure
@@ -121,6 +122,7 @@ from tensorflow.python.data.ops.dataset_ops import DatasetStructure
 from tensorflow.python.data.ops.iterator_ops import get_next_as_optional
 from tensorflow.python.data.ops.optional_ops import Optional
 from tensorflow.python.data.ops.optional_ops import OptionalStructure
+from tensorflow.python.data.util.structure import NestedStructure
 from tensorflow.python.data.util.structure import SparseTensorStructure
 from tensorflow.python.data.util.structure import Structure
 from tensorflow.python.data.util.structure import TensorStructure
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 4f2c6cd853..5e3addacaa 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -273,6 +273,7 @@ def convert_legacy_structure(output_types, output_shapes, output_classes):
 # NOTE(mrry): The following classes make extensive use of non-public methods of
 # their base class, so we disable the protected-access lint warning once here.
 # pylint: disable=protected-access
+@tf_export("data.experimental.NestedStructure")
 class NestedStructure(Structure):
   """Represents a nested structure in which each leaf is a `Structure`."""
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-nested-structure.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-nested-structure.pbtxt
new file mode 100644
index 0000000000..b4b066e563
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-nested-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.NestedStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.NestedStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'nested_structure\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index 234507e5de..2d11590492 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -20,6 +20,10 @@ tf_module {
     name: "INFINITE_CARDINALITY"
     mtype: "<type \'int\'>"
   }
+  member {
+    name: "NestedStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "OptimizationOptions"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-nested-structure.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-nested-structure.pbtxt
new file mode 100644
index 0000000000..b4b066e563
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-nested-structure.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.data.experimental.NestedStructure"
+tf_class {
+  is_instance: "<class \'tensorflow.python.data.util.structure.NestedStructure\'>"
+  is_instance: "<class \'tensorflow.python.data.util.structure.Structure\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'nested_structure\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_value"
+    argspec: "args=[\'value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "is_compatible_with"
+    argspec: "args=[\'self\', \'other\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index 234507e5de..2d11590492 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -20,6 +20,10 @@ tf_module {
     name: "INFINITE_CARDINALITY"
     mtype: "<type \'int\'>"
   }
+  member {
+    name: "NestedStructure"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "OptimizationOptions"
     mtype: "<type \'type\'>"
-- 
GitLab


From 47b751c9324522c0178536f792cc47a970f4f62b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 15:17:02 -0800
Subject: [PATCH 1522/1554] Changed CPU <-> NNAPI toggle UI element from
 ToggleButton to RadioButton.

PiperOrigin-RevId: 224586292
---
 .../Camera2BasicFragment.java                 | 41 ++++++++++++++-----
 .../tflitecamerademo/ImageClassifier.java     |  9 +++-
 .../layout-land/fragment_camera2_basic.xml    | 25 ++++++++---
 .../res/layout-v26/fragment_camera2_basic.xml | 29 +++++++++----
 .../res/layout/fragment_camera2_basic.xml     | 30 +++++++++-----
 .../demo/app/src/main/res/values/strings.xml  |  1 +
 6 files changed, 98 insertions(+), 37 deletions(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index 3596e42011..20e96f586a 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -56,11 +56,10 @@ import android.view.Surface;
 import android.view.TextureView;
 import android.view.View;
 import android.view.ViewGroup;
-import android.widget.CompoundButton;
 import android.widget.NumberPicker;
+import android.widget.RadioButton;
 import android.widget.TextView;
 import android.widget.Toast;
-import android.widget.ToggleButton;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -87,10 +86,12 @@ public class Camera2BasicFragment extends Fragment
   private boolean runClassifier = false;
   private boolean checkedPermissions = false;
   private TextView textView;
-  private ToggleButton toggle;
   private NumberPicker np;
   private ImageClassifier classifier;
 
+  enum InferenceEngine { CPU, NNAPI };
+  private InferenceEngine inferenceEngine = InferenceEngine.CPU;
+
   /** Max preview width that is guaranteed by Camera2 API */
   private static final int MAX_PREVIEW_WIDTH = 1920;
 
@@ -303,14 +304,6 @@ public class Camera2BasicFragment extends Fragment
   public void onViewCreated(final View view, Bundle savedInstanceState) {
     textureView = (AutoFitTextureView) view.findViewById(R.id.texture);
     textView = (TextView) view.findViewById(R.id.text);
-    toggle = (ToggleButton) view.findViewById(R.id.button);
-
-    toggle.setOnCheckedChangeListener(
-        new CompoundButton.OnCheckedChangeListener() {
-          public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) {
-            backgroundHandler.post(() -> classifier.setUseNNAPI(isChecked));
-          }
-        });
 
     np = (NumberPicker) view.findViewById(R.id.np);
     np.setMinValue(1);
@@ -323,6 +316,32 @@ public class Camera2BasicFragment extends Fragment
             backgroundHandler.post(() -> classifier.setNumThreads(newVal));
           }
         });
+
+    RadioButton cpuButton = (RadioButton) view.findViewById(R.id.radio_cpu);
+    cpuButton.setChecked(true);  // TFLite runs on CPU by default.
+    cpuButton.setOnClickListener(
+        new View.OnClickListener() {
+          @Override
+          public void onClick(View view) {
+            if (inferenceEngine == InferenceEngine.CPU) {
+              return;
+            }
+            inferenceEngine = InferenceEngine.CPU;
+            backgroundHandler.post(() -> classifier.useCPU());
+          }
+        });
+
+    ((RadioButton) view.findViewById(R.id.radio_nnapi)).setOnClickListener(
+        new View.OnClickListener() {
+          @Override
+          public void onClick(View view) {
+            if (inferenceEngine == InferenceEngine.NNAPI) {
+              return;
+            }
+            inferenceEngine = InferenceEngine.NNAPI;
+            backgroundHandler.post(() -> classifier.useNNAPI());
+          }
+        });
   }
 
   /** Load the model and labels. */
diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java
index 39057aa776..700efc1c1a 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java
@@ -163,8 +163,13 @@ public abstract class ImageClassifier {
     }
   }
 
-  public void setUseNNAPI(Boolean nnapi) {
-    tfliteOptions.setUseNNAPI(nnapi);
+  public void useCPU() {
+    tfliteOptions.setUseNNAPI(false);
+    recreateInterpreter();
+  }
+
+  public void useNNAPI() {
+    tfliteOptions.setUseNNAPI(true);
     recreateInterpreter();
   }
 
diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
index ef8a9e0845..ee71ab808f 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
@@ -40,12 +40,27 @@
         android:scaleType="centerInside"
         android:src="@drawable/logo"/>
 
-    <ToggleButton
-        android:id="@+id/button"
+    <RadioGroup
+        android:gravity="center"
         android:layout_width="match_parent"
-        android:layout_height="wrap_content"
-        android:textOff="@string/tflite"
-        android:textOn="@string/nnapi"/>
+        android:layout_height="match_parent"
+        android:orientation="horizontal">
+        <RadioButton
+            android:id="@+id/radio_cpu"
+            android:background="#0000000f"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/cpu"
+            android:textColor="@android:color/white" />
+        <RadioButton
+            android:id="@+id/radio_nnapi"
+            android:background="#0000000f"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/nnapi"
+            android:textColor="@android:color/white" />
+        </RadioGroup>
+
     <NumberPicker
         android:id="@+id/np"
         android:layout_width="wrap_content"
diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml
index ddb099a950..19e0a9bab4 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml
@@ -80,15 +80,26 @@
             android:layout_marginLeft="10dp"
             android:theme="@style/AppTheme.Picker"
             android:visibility="visible" />
-        <ToggleButton
-            android:id="@+id/button"
-            android:textOff="@string/tflite"
-            android:textOn="@string/nnapi"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:layout_marginLeft="10dp"
-            android:background="#0000000f"
-            android:textColor="@android:color/white" />
+        <RadioGroup
+            android:gravity="center"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent"
+            android:orientation="horizontal">
+            <RadioButton
+                android:id="@+id/radio_cpu"
+                android:background="#0000000f"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:text="@string/cpu"
+                android:textColor="@android:color/white" />
+            <RadioButton
+                android:id="@+id/radio_nnapi"
+                android:background="#0000000f"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:text="@string/nnapi"
+                android:textColor="@android:color/white" />
+        </RadioGroup>
     </LinearLayout>
 
 
diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml
index e567009a42..be66eeac75 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml
@@ -80,15 +80,25 @@
             android:layout_marginLeft="10dp"
             android:theme="@style/AppTheme.Picker"
             android:visibility="visible" />
-        <ToggleButton
-            android:id="@+id/button"
-            android:textOff="@string/tflite"
-            android:textOn="@string/nnapi"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:layout_marginLeft="10dp"
-            android:background="#0000000f"
-            android:textColor="@android:color/white" />
-
+        <RadioGroup
+            android:gravity="center"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent"
+            android:orientation="horizontal">
+            <RadioButton
+                android:id="@+id/radio_cpu"
+                android:background="#0000000f"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:text="@string/cpu"
+                android:textColor="@android:color/white" />
+            <RadioButton
+                android:id="@+id/radio_nnapi"
+                android:background="#0000000f"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:text="@string/nnapi"
+                android:textColor="@android:color/white" />
+        </RadioGroup>
     </LinearLayout>
 </RelativeLayout>
diff --git a/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml b/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml
index 29a033bcd4..45b12850e5 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml
@@ -23,4 +23,5 @@
     <string name="toggle">Use NNAPI</string>
     <string name="tflite">tflite</string>
     <string name="nnapi">NNAPI</string>
+    <string name="cpu">CPU</string>
 </resources>
-- 
GitLab


From c707bb2b4da16b9fdf228c3680cf279034c3e9fb Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 7 Dec 2018 15:38:23 -0800
Subject: [PATCH 1523/1554] [tf.data] Update datasets that use legacy
 structures to use `Structure`.

PiperOrigin-RevId: 224589556
---
 .../experimental/ops/get_single_element.py    | 12 ++---
 .../data/experimental/ops/prefetching_ops.py  | 26 +++------
 .../python/data/experimental/ops/scan_ops.py  | 53 ++++++++++---------
 3 files changed, 37 insertions(+), 54 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/get_single_element.py b/tensorflow/python/data/experimental/ops/get_single_element.py
index 73116edf12..d649a07012 100644
--- a/tensorflow/python/data/experimental/ops/get_single_element.py
+++ b/tensorflow/python/data/experimental/ops/get_single_element.py
@@ -18,8 +18,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
@@ -63,10 +61,8 @@ def get_single_element(dataset):
   if not isinstance(dataset, dataset_ops.DatasetV2):
     raise TypeError("`dataset` must be a `tf.data.Dataset` object.")
 
-  nested_ret = nest.pack_sequence_as(
-      dataset.output_types, gen_dataset_ops.dataset_to_single_element(
-          dataset._as_variant_tensor(),  # pylint: disable=protected-access
+  # pylint: disable=protected-access
+  return dataset._element_structure._from_compatible_tensor_list(
+      gen_dataset_ops.dataset_to_single_element(
+          dataset._as_variant_tensor(),
           **dataset_ops.flat_structure(dataset)))
-  return sparse.deserialize_sparse_tensors(
-      nested_ret, dataset.output_types, dataset.output_shapes,
-      dataset.output_classes)
diff --git a/tensorflow/python/data/experimental/ops/prefetching_ops.py b/tensorflow/python/data/experimental/ops/prefetching_ops.py
index aba1786646..e46dfb6568 100644
--- a/tensorflow/python/data/experimental/ops/prefetching_ops.py
+++ b/tensorflow/python/data/experimental/ops/prefetching_ops.py
@@ -20,8 +20,6 @@ from __future__ import print_function
 from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.eager import function
 from tensorflow.python.framework import device as framework_device
 from tensorflow.python.framework import dtypes
@@ -106,13 +104,6 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
     self._source_device_string = source_device
     self._source_device = ops.convert_to_tensor(source_device)
 
-    self._flat_output_shapes = nest.flatten(
-        sparse.as_dense_shapes(self._input_dataset.output_shapes,
-                               self._input_dataset.output_classes))
-    self._flat_output_types = nest.flatten(
-        sparse.as_dense_types(self._input_dataset.output_types,
-                              self._input_dataset.output_classes))
-
     @function.defun()
     def _init_func():
       """Creates an iterator for the input dataset.
@@ -123,8 +114,7 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
       # pylint: disable=protected-access
       ds_variant = self._input_dataset._as_variant_tensor()
       resource = gen_dataset_ops.anonymous_iterator(
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
+          **dataset_ops.flat_structure(self._input_dataset))
       with ops.control_dependencies(
           [gen_dataset_ops.make_iterator(ds_variant, resource)]):
         return gen_dataset_ops.iterator_to_string_handle(resource)
@@ -155,8 +145,7 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
         iterator = iterator_ops.Iterator.from_string_handle(
             string_handle, self.output_types, self.output_shapes,
             self.output_classes)
-      ret = iterator.get_next()
-      return nest.flatten(sparse.serialize_sparse_tensors(ret))
+      return self._element_structure._to_tensor_list(iterator.get_next())  # pylint: disable=protected-access
 
     next_func_concrete = _next_func._get_concrete_function_internal()  # pylint: disable=protected-access
 
@@ -166,7 +155,7 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
           target=self._source_device,
           args=[string_handle] +
           next_func_concrete.captured_inputs,
-          Tout=self._flat_output_types,
+          Tout=self._input_dataset._element_structure._flat_types,  # pylint: disable=protected-access
           f=next_func_concrete)
 
     self._next_func = _remote_next_func._get_concrete_function_internal()  # pylint: disable=protected-access
@@ -183,8 +172,7 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
       """
       iterator_resource = gen_dataset_ops.iterator_from_string_handle_v2(
           string_handle,
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
+          **dataset_ops.flat_structure(self._input_dataset))
       with ops.control_dependencies([
           resource_variable_ops.destroy_resource_op(
               iterator_resource, ignore_lookup_error=True)]):
@@ -196,8 +184,7 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
     def _remote_finalize_func(string_handle):
       return functional_ops.remote_call(
           target=self._source_device,
-          args=[string_handle] +
-          finalize_func_concrete.captured_inputs,
+          args=[string_handle] + finalize_func_concrete.captured_inputs,
           Tout=[dtypes.int64],
           f=finalize_func_concrete)
 
@@ -233,8 +220,7 @@ class _CopyToDeviceDataset(dataset_ops.UnaryUnchangedStructureDataset):
           init_func=self._init_func,
           next_func=self._next_func,
           finalize_func=self._finalize_func,
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
+          **dataset_ops.flat_structure(self._input_dataset))
 
 
 class _MapOnGpuDataset(dataset_ops.UnaryDataset):
diff --git a/tensorflow/python/data/experimental/ops/scan_ops.py b/tensorflow/python/data/experimental/ops/scan_ops.py
index 661f2c5928..5c77ad7343 100644
--- a/tensorflow/python/data/experimental/ops/scan_ops.py
+++ b/tensorflow/python/data/experimental/ops/scan_ops.py
@@ -21,7 +21,6 @@ import collections
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.data.util import structure
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -50,13 +49,7 @@ class _ScanDataset(dataset_ops.UnaryDataset):
     # Compute initial values for the state classes, shapes and types based on
     # the initial state. The shapes may be refined by running `tf_scan_func` one
     # or more times below.
-    self._state_classes = sparse.get_classes(self._initial_state)
-    self._state_shapes = nest.pack_sequence_as(
-        self._initial_state,
-        [t.get_shape() for t in nest.flatten(self._initial_state)])
-    self._state_types = nest.pack_sequence_as(
-        self._initial_state,
-        [t.dtype for t in nest.flatten(self._initial_state)])
+    self._state_structure = structure.Structure.from_value(self._initial_state)
 
     # Iteratively rerun the scan function until reaching a fixed point on
     # `self._state_shapes`.
@@ -66,9 +59,8 @@ class _ScanDataset(dataset_ops.UnaryDataset):
       wrapped_func = dataset_ops.StructuredFunctionWrapper(
           scan_func,
           self._transformation_name(),
-          input_classes=(self._state_classes, input_dataset.output_classes),
-          input_shapes=(self._state_shapes, input_dataset.output_shapes),
-          input_types=(self._state_types, input_dataset.output_types),
+          input_structure=structure.NestedStructure(
+              (self._state_structure, input_dataset._element_structure)),  # pylint: disable=protected-access
           add_to_graph=False)
       if not (
           isinstance(wrapped_func.output_types, collections.Sequence) and
@@ -76,35 +68,38 @@ class _ScanDataset(dataset_ops.UnaryDataset):
         raise TypeError("The scan function must return a pair comprising the "
                         "new state and the output value.")
 
-      new_state_classes, output_classes = wrapped_func.output_classes
+      new_state_classes, self._output_classes = wrapped_func.output_classes
 
       # Extract and validate class information from the returned values.
-      for new_state_class, state_class in zip(
+      new_state_classes, output_classes = wrapped_func.output_classes
+      old_state_classes = self._state_structure._to_legacy_output_classes()  # pylint: disable=protected-access
+      for new_state_class, old_state_class in zip(
           nest.flatten(new_state_classes),
-          nest.flatten(self._state_classes)):
-        if not issubclass(new_state_class, state_class):
+          nest.flatten(old_state_classes)):
+        if not issubclass(new_state_class, old_state_class):
           raise TypeError(
               "The element classes for the new state must match the initial "
               "state. Expected %s; got %s." %
-              (self._state_classes, new_state_classes))
+              (old_state_classes, new_state_classes))
 
       # Extract and validate type information from the returned values.
       new_state_types, output_types = wrapped_func.output_types
-      for new_state_type, state_type in zip(
-          nest.flatten(new_state_types), nest.flatten(self._state_types)):
-        if new_state_type != state_type:
+      old_state_types = self._state_structure._to_legacy_output_types()  # pylint: disable=protected-access
+      for new_state_type, old_state_type in zip(
+          nest.flatten(new_state_types), nest.flatten(old_state_types)):
+        if new_state_type != old_state_type:
           raise TypeError(
               "The element types for the new state must match the initial "
               "state. Expected %s; got %s." %
-              (self._state_types, new_state_types))
+              (old_state_types, new_state_types))
 
       # Extract shape information from the returned values.
       new_state_shapes, output_shapes = wrapped_func.output_shapes
-
+      old_state_shapes = self._state_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
       self._structure = structure.convert_legacy_structure(
           output_types, output_shapes, output_classes)
 
-      flat_state_shapes = nest.flatten(self._state_shapes)
+      flat_state_shapes = nest.flatten(old_state_shapes)
       flat_new_state_shapes = nest.flatten(new_state_shapes)
       weakened_state_shapes = [
           original.most_specific_compatible_shape(new)
@@ -121,8 +116,13 @@ class _ScanDataset(dataset_ops.UnaryDataset):
           break
 
       if need_to_rerun:
-        self._state_shapes = nest.pack_sequence_as(self._state_shapes,
-                                                   weakened_state_shapes)
+        # TODO(b/110122868): Support a "most specific compatible structure"
+        # method for combining structures, to avoid using legacy structures
+        # in this method.
+        self._state_structure = structure.convert_legacy_structure(
+            old_state_types,
+            nest.pack_sequence_as(old_state_shapes, weakened_state_shapes),
+            old_state_classes)
 
     self._scan_func = wrapped_func
     self._scan_func.function.add_to_graph(ops.get_default_graph())
@@ -131,10 +131,11 @@ class _ScanDataset(dataset_ops.UnaryDataset):
     return [self._scan_func]
 
   def _as_variant_tensor(self):
-    input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
+    # pylint: disable=protected-access
+    input_t = self._input_dataset._as_variant_tensor()
     return gen_experimental_dataset_ops.experimental_scan_dataset(
         input_t,
-        nest.flatten(sparse.serialize_sparse_tensors(self._initial_state)),
+        self._state_structure._to_tensor_list(self._initial_state),
         self._scan_func.function.captured_inputs,
         f=self._scan_func.function,
         preserve_cardinality=True,
-- 
GitLab


From 078e8fa05791cc55e71a13ea33475a436e62dba5 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Fri, 7 Dec 2018 15:58:35 -0800
Subject: [PATCH 1524/1554] [TF:XLA] Change ArCrsCombiner to use division
 instead of subtraction. This way, when we move the AR down the graph, the
 adjustment stays before the AR, not after, and the AR moves closer to the
 CRS. Then, it's easier to move the AR past multiple ops one-at-a-time in
 future CLs.

Also, some bug fixes in InstructionsComputeSameValue.

PiperOrigin-RevId: 224592476
---
 .../compiler/xla/service/ar_crs_combiner.cc   | 69 +++++++-------
 .../compiler/xla/service/ar_crs_combiner.h    |  6 +-
 .../xla/service/ar_crs_combiner_test.cc       | 94 +++++++++++++++++--
 3 files changed, 124 insertions(+), 45 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 24de693822..362bc44a1c 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -60,7 +60,7 @@ absl::optional<HloInstruction*> MatchesArCrsPattern(
 
 absl::optional<HloInstruction*> ArCrsCombiner::WhileFromBodyParameter(
     HloInstruction* instruction) {
-  CHECK(HloOpcode::kParameter == instruction->opcode());
+  CHECK_EQ(HloOpcode::kParameter, instruction->opcode());
   HloComputation* computation = instruction->parent();
   auto caller_instructions = call_graph_->GetComputationCallers(computation);
   if (caller_instructions.size() == 1) {
@@ -120,7 +120,7 @@ bool ArCrsCombiner::TupleElementsComputeSameValue(
     return false;
   }
   for (auto tuple : tuples) {
-    CHECK(tuple->opcode() == HloOpcode::kTuple);
+    CHECK_EQ(tuple->opcode(), HloOpcode::kTuple);
     if (!InstructionsComputeSameValue(tuple->mutable_operand(i1),
                                       tuple->mutable_operand(i2),
                                       visited_pairs)) {
@@ -133,7 +133,7 @@ bool ArCrsCombiner::TupleElementsComputeSameValue(
 /* static */
 bool ArCrsCombiner::TestInstructionsComputeSameValue(HloInstruction* i1,
                                                      HloInstruction* i2) {
-  ArCrsCombiner combiner(/*num_spatial_partitions=*/2, /*num_replicas=*/1);
+  ArCrsCombiner combiner(/*num_spatial_partitions=*/2);
   auto module = i1->parent()->parent();
   CHECK_EQ(module, i2->parent()->parent());
   combiner.call_graph_ = CallGraph::Build(module);
@@ -160,13 +160,6 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
   if (opcode1 != i2->opcode() || operands1.size() != i2->operands().size()) {
     return false;
   }
-  if (opcode1 == HloOpcode::kConstant || i1->IsCrossModuleAllReduce()) {
-    return i1->Identical(
-        *i2,
-        /*eq_operands=*/std::equal_to<const HloInstruction*>(),
-        /*eq_computations=*/std::equal_to<const HloComputation*>(),
-        /*layout_sensitive=*/false);
-  }
   visited_pairs->emplace(min_uid, max_uid);
   for (int i = 0; i < operands1.size(); ++i) {
     auto operand1 = operands1[i];
@@ -175,14 +168,28 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
       return false;
     }
   }
+  if (opcode1 == HloOpcode::kParameter) {
+    // In the general case, we don't try to prove equality of parameters.
+    // We only try in the context of get-tuple-element
+    // (see TupleElementsComputeSameValue).
+    return false;
+  }
   if (opcode1 == HloOpcode::kGetTupleElement) {
-    if (i1->tuple_index() == i2->tuple_index()) {
-      return true;
-    }
-    return TupleElementsComputeSameValue(operands1[0], i1->tuple_index(),
+    return i1->tuple_index() == i2->tuple_index() ||
+           TupleElementsComputeSameValue(operands1[0], i1->tuple_index(),
                                          i2->tuple_index(), visited_pairs);
   }
-  return true;
+  // Don't check that the operands are identical, because Identical can
+  // return false for instructions that compute the same value but are not
+  // identical, which we don't want. We have checked the arguments with
+  // InstructionsComputeSameValue earlier.
+  auto eq_instructions = [](const HloInstruction* i1,
+                            const HloInstruction* i2) -> bool { return true; };
+  auto eq_computations = [](const HloComputation* a, const HloComputation* b) {
+    return *a == *b;
+  };
+  return i1->Identical(*i2, eq_instructions, eq_computations,
+                       /*layout_sensitive=*/false);
 }
 
 void ArCrsCombiner::GroupAllReducesById(HloModule* module) {
@@ -203,12 +210,12 @@ void ArCrsCombiner::KeepProvablyEqualInstructionGroups() {
 
     auto instr_0 = instruction_vec[0];
     auto add_0 = instr_0->users()[0]->users()[0];
-    CHECK(HloOpcode::kAdd == add_0->opcode());
+    CHECK_EQ(HloOpcode::kAdd, add_0->opcode());
 
     for (int i = 1; i < instruction_vec.size(); ++i) {
       auto instr_i = instruction_vec[i];
       auto add_i = instr_i->users()[0]->users()[0];
-      CHECK(HloOpcode::kAdd == add_i->opcode());
+      CHECK_EQ(HloOpcode::kAdd, add_i->opcode());
       absl::flat_hash_map<int64, int64> visited_pairs;
       if (!InstructionsComputeSameValue(add_0, add_i, &visited_pairs)) {
         all_reduce_map_.erase(it.first);
@@ -242,26 +249,22 @@ StatusOr<bool> ArCrsCombiner::RewriteGraph() {
       HloInstruction* other_summand = (add->operands()[0] == convert)
                                           ? add->operands()[1]
                                           : add->operands()[0];
-      // Remove the AllReduce and replace the CRS with an all-core AllReduce,
-      // then subtract:
-      // other_summand * num_replicas_ * (num_spatial_partitions_ - 1)
+      // To move the AR past the addition, we need to divide other_summand by
+      // the number of spatial partitions.
+      CHECK_EQ(all_reduce->user_count(), 1);
       TF_CHECK_OK(
           all_reduce->ReplaceAllUsesWith(all_reduce->mutable_operand(0)));
-      crs->set_all_reduce_id(all_reduce->all_reduce_id());
-      auto new_shape = crs->shape();
-      Literal lit(new_shape);
-      lit.PopulateWithValue<float>(num_replicas_ *
-                                   (num_spatial_partitions_ - 1));
-      auto partitions_minus_1_const = parent_computation->AddInstruction(
+      auto shape = other_summand->shape();
+      Literal lit(shape);
+      lit.PopulateWithValue<float>(num_spatial_partitions_);
+      auto divisor = parent_computation->AddInstruction(
           HloInstruction::CreateConstant(lit.Clone()));
-      auto to_subtract =
+      auto division =
           parent_computation->AddInstruction(HloInstruction::CreateBinary(
-              new_shape, HloOpcode::kMultiply, other_summand,
-              partitions_minus_1_const));
-      auto sub =
-          parent_computation->AddInstruction(HloInstruction::CreateBinary(
-              new_shape, HloOpcode::kSubtract, crs, to_subtract));
-      TF_CHECK_OK(crs->ReplaceAllUsesWith(sub));
+              shape, HloOpcode::kDivide, other_summand, divisor));
+      TF_CHECK_OK(other_summand->ReplaceUseWith(add, division));
+      // The AllReduce and the CRS are combined to an all-core AllReduce.
+      crs->set_all_reduce_id(all_reduce->all_reduce_id());
       TF_CHECK_OK(parent_computation->RemoveInstruction(all_reduce));
     }
   }
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.h b/tensorflow/compiler/xla/service/ar_crs_combiner.h
index 4abdb1f57d..f6a7ef76ec 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.h
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.h
@@ -30,9 +30,8 @@ namespace xla {
 // fully utilizes the interconnect bandwidth.
 class ArCrsCombiner : public HloModulePass {
  public:
-  ArCrsCombiner(int num_spatial_partitions, int num_replicas)
-      : num_spatial_partitions_(num_spatial_partitions),
-        num_replicas_(num_replicas) {}
+  ArCrsCombiner(int num_spatial_partitions)
+      : num_spatial_partitions_(num_spatial_partitions) {}
   absl::string_view name() const override { return "ar-crs-combiner"; }
   StatusOr<bool> Run(HloModule* module) override;
 
@@ -77,7 +76,6 @@ class ArCrsCombiner : public HloModulePass {
   StatusOr<bool> RewriteGraph();
 
   int num_spatial_partitions_;
-  int num_replicas_;
 
   // Map from all-reduce ids to the all reduce instructions.
   absl::flat_hash_map<int64, std::vector<HloInstruction*>> all_reduce_map_;
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index 23d9aa9eb3..10171835d8 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -48,6 +48,43 @@ ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
   EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
 }
 
+TEST_F(ArCrsCombinerTest, SameValueTestBasecase2) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (x: f32[]) -> (f32[], f32[]) {
+  %x = f32[] parameter(0)
+  ROOT %tuple = (f32[], f32[]) tuple(%x, %x)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestBasecase3) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (x: f32[], y: f32[]) -> (f32[], f32[]) {
+  %x = f32[] parameter(0)
+  %y = f32[] parameter(1)
+  ROOT %tuple = (f32[], f32[]) tuple(%x, %y)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
 TEST_F(ArCrsCombinerTest, SameValueTestNumOperands) {
   const char* module_str = R"(
 HloModule foobar
@@ -69,6 +106,46 @@ ENTRY %entrycomp (p: f32[2,2]) -> ((f32[2,2]), (f32[2,2], f32[2,2])) {
   EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
 }
 
+TEST_F(ArCrsCombinerTest, SameValueTestSliceIndicesMatch) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2]) -> (f32[1], f32[1]) {
+  %p = f32[2] parameter(0)
+  %slice.1 = f32[1] slice(f32[2] %p), slice={[0:1]}
+  %slice.2 = f32[1] slice(f32[2] %p), slice={[0:1]}
+  ROOT %tuple = (f32[1], f32[1]) tuple(%slice.1, %slice.2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_TRUE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
+TEST_F(ArCrsCombinerTest, SameValueTestSliceIndicesDontMatch) {
+  const char* module_str = R"(
+HloModule foobar
+
+ENTRY %entrycomp (p: f32[2]) -> (f32[1], f32[1]) {
+  %p = f32[2] parameter(0)
+  %slice.1 = f32[1] slice(f32[2] %p), slice={[0:1]}
+  %slice.2 = f32[1] slice(f32[2] %p), slice={[1:2]}
+  ROOT %tuple = (f32[1], f32[1]) tuple(%slice.1, %slice.2)
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto root_tuple = module->entry_computation()->root_instruction();
+  auto i1 = root_tuple->operands()[0];
+  auto i2 = root_tuple->operands()[1];
+  EXPECT_FALSE(ArCrsCombiner::TestInstructionsComputeSameValue(i1, i2));
+}
+
 TEST_F(ArCrsCombinerTest, SameValueTestTupleElementSameIndex) {
   const char* module_str = R"(
 HloModule foobar
@@ -317,17 +394,18 @@ ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
   auto crs_before =
       module->entry_computation()->root_instruction()->operands()[0];
   auto replica_groups_before = crs_before->replica_groups();
-  ArCrsCombiner combiner(2, 1);
+  ArCrsCombiner combiner(2);
   auto changed = combiner.Run(module.get()).ValueOrDie();
   EXPECT_TRUE(changed);
   EXPECT_THAT(
       module->entry_computation()->root_instruction(),
-      op::Tuple(op::Subtract(op::CrossReplicaSum(),
-                             op::Multiply(op::Constant(), op::Constant())),
-                op::Subtract(op::CrossReplicaSum(),
-                             op::Multiply(op::Constant(), op::Constant()))));
-  auto sub = module->entry_computation()->root_instruction()->operands()[0];
-  auto crs_after = sub->operands()[0];
+      op::Tuple(
+          op::CrossReplicaSum(op::Add(
+              op::Divide(op::Constant(), op::Constant()), op::Convert())),
+          op::CrossReplicaSum(op::Add(
+              op::Divide(op::Constant(), op::Constant()), op::Convert()))));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
   auto replica_groups_after = crs_after->replica_groups();
   ASSERT_EQ(replica_groups_before.size(), replica_groups_after.size());
   for (int i = 0; i < replica_groups_before.size(); ++i) {
@@ -409,7 +487,7 @@ ENTRY %entrycomp (p: f32[2,2]) -> (f32[2,2], f32[2,2]) {
 
   TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
                           ParseAndReturnVerifiedModule(module_str));
-  ArCrsCombiner combiner(2, 1);
+  ArCrsCombiner combiner(2);
   auto changed = combiner.Run(module.get()).ValueOrDie();
   EXPECT_FALSE(changed);
 }
-- 
GitLab


From 44157872848ae4f07b533a1fea35a6b00d97cc1e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 16:13:24 -0800
Subject: [PATCH 1525/1554] Track XRT memory allocations.

PiperOrigin-RevId: 224594734
---
 tensorflow/compiler/xrt/xrt_state.cc | 47 +++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 5c7c537c34..31603e044d 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/compiler/xrt/xrt_state.h"
 
 #include <stdint.h>
+#include <map>
 #include <memory>
 #include <string>
 #include <utility>
@@ -34,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/stream_executor/stream_executor.h"
 
@@ -41,6 +43,34 @@ namespace tensorflow {
 
 namespace {
 
+class BufferAllocStats {
+ public:
+  struct Stats {
+    int64 count = 0;
+    int64 size = 0;
+  };
+
+  Stats ReportAlloc(int64 device, int64 msize) {
+    mutex_lock lock(lock_);
+    Stats* device_stats = &stats_[device];
+    device_stats->count += 1;
+    device_stats->size += msize;
+    return *device_stats;
+  }
+
+  Stats ReportFree(int64 device, int64 msize) {
+    mutex_lock lock(lock_);
+    Stats* device_stats = &stats_[device];
+    device_stats->count -= 1;
+    device_stats->size -= msize;
+    return *device_stats;
+  }
+
+ private:
+  mutable mutex lock_;
+  std::map<int64, Stats> stats_;
+};
+
 const char* kTupleContainer = "tuples";
 
 int64 get_uid() {
@@ -48,6 +78,11 @@ int64 get_uid() {
   return static_cast<int64>(unsigned_rand);
 }
 
+BufferAllocStats* GetAllocStats() {
+  static BufferAllocStats* stats = new BufferAllocStats();
+  return stats;
+}
+
 Status AllocateScopedShapedBuffer(
     xla::Backend* backend, int device_ordinal, const xla::Shape& shape,
     std::unique_ptr<xla::ScopedShapedBuffer>* buffer) {
@@ -100,9 +135,19 @@ XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          xla::DeviceMemoryAllocator* allocator)
     : allocation_(allocation),
       device_ordinal_(device_ordinal),
-      allocator_(allocator) {}
+      allocator_(allocator) {
+  if (VLOG_IS_ON(2)) {
+    auto stats =
+        GetAllocStats()->ReportAlloc(device_ordinal_, allocation_.size());
+    LOG(INFO) << "XRT Allocation Stats: device=" << device_ordinal_
+              << " count=" << stats.count << " size=" << stats.size;
+  }
+}
 
 XRTBufferAllocation::~XRTBufferAllocation() {
+  if (VLOG_IS_ON(2)) {
+    GetAllocStats()->ReportFree(device_ordinal_, allocation_.size());
+  }
   // Deallocate explicitly allows allocation_ to be null.
   Status s = allocator_->Deallocate(device_ordinal_, allocation_);
   // Nothing to do but check fail here if memory datastructures are corrupted.
-- 
GitLab


From 294ad7140234fb383e28bd059d6d761ba879c09e Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 7 Dec 2018 16:22:49 -0800
Subject: [PATCH 1526/1554] Internal change.

PiperOrigin-RevId: 224596031
---
 tensorflow/tools/api/tests/api_compatibility_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index e7f23a1174..40162daf14 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -277,6 +277,9 @@ class ApiCompatibilityTest(test.TestCase):
 
     public_api_visitor = public_api.PublicAPIVisitor(visitor)
     public_api_visitor.private_map['tf'] = ['contrib']
+    if api_version == 2:
+      public_api_visitor.private_map['tf'].append('enable_v2_behavior')
+
     public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
     if FLAGS.only_test_core_api:
       public_api_visitor.do_not_descend_map['tf'].extend(_NON_CORE_PACKAGES)
-- 
GitLab


From 5dfbcfbba04197117cff01a07397f499682d3f56 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 7 Dec 2018 16:23:28 -0800
Subject: [PATCH 1527/1554] Consolidate JNI inclusion code

Introduce a single proxy target that exports JNI headers appropriately
for the given platform.

PiperOrigin-RevId: 224596100
---
 tensorflow/lite/java/jni/BUILD                | 47 ++++++++++++++
 tensorflow/lite/java/src/main/native/BUILD    | 62 ++-----------------
 tensorflow/lite/java/src/test/native/BUILD    | 19 ++----
 .../models/smartreply/demo/app/src/main/BUILD |  1 +
 4 files changed, 58 insertions(+), 71 deletions(-)
 create mode 100644 tensorflow/lite/java/jni/BUILD

diff --git a/tensorflow/lite/java/jni/BUILD b/tensorflow/lite/java/jni/BUILD
new file mode 100644
index 0000000000..ce17ac4fa0
--- /dev/null
+++ b/tensorflow/lite/java/jni/BUILD
@@ -0,0 +1,47 @@
+package(default_visibility = ["//tensorflow/lite:__subpackages__"])
+
+licenses(["notice"])  # Apache 2.0
+
+# Helper target for exposing JNI headers across multiple platforms.
+cc_library(
+    name = "jni",
+    hdrs = select({
+        # The Android toolchain makes "jni.h" available in the include path.
+        # For non-Android toolchains, generate jni.h and jni_md.h.
+        "//tensorflow:android": [],
+        "//conditions:default": [
+            ":jni.h",
+            ":jni_md.h",
+        ],
+    }),
+    includes = select({
+        "//tensorflow:android": [],
+        "//conditions:default": ["."],
+    }),
+)
+
+# Silly rules to make
+# #include <jni.h>
+# in the source headers work
+# (in combination with the "includes" attribute of the tf_cuda_library rule
+# above. Not needed when using the Android toolchain).
+#
+# Inspired from:
+# https://github.com/bazelbuild/bazel/blob/f99a0543f8d97339d32075c7176b79f35be84606/src/main/native/BUILD
+# but hopefully there is a simpler alternative to this.
+genrule(
+    name = "copy_jni_h",
+    srcs = ["@bazel_tools//tools/jdk:jni_header"],
+    outs = ["jni.h"],
+    cmd = "cp -f $< $@",
+)
+
+genrule(
+    name = "copy_jni_md_h",
+    srcs = select({
+        "//tensorflow:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"],
+        "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"],
+    }),
+    outs = ["jni_md.h"],
+    cmd = "cp -f $< $@",
+)
diff --git a/tensorflow/lite/java/src/main/native/BUILD b/tensorflow/lite/java/src/main/native/BUILD
index 8f95f14518..52194e86db 100644
--- a/tensorflow/lite/java/src/main/native/BUILD
+++ b/tensorflow/lite/java/src/main/native/BUILD
@@ -15,15 +15,7 @@ cc_library(
         "nativeinterpreterwrapper_jni.cc",
         "tensor_jni.cc",
         "tensorflow_lite_jni.cc",
-    ] + select({
-        # The Android toolchain makes "jni.h" available in the include path.
-        # For non-Android toolchains, generate jni.h and jni_md.h.
-        "//tensorflow:android": [],
-        "//conditions:default": [
-            ":jni.h",
-            ":jni_md.h",
-        ],
-    }),
+    ],
     hdrs = [
         "exception_jni.h",
         "nativeinterpreterwrapper_jni.h",
@@ -31,75 +23,31 @@ cc_library(
         "tensorflow_lite_jni.h",
     ],
     copts = tflite_copts(),
-    includes = select({
-        "//tensorflow:android": [],
-        "//conditions:default": ["."],
-    }),
     linkopts = [
         "-lm",
         "-ldl",
     ],
     deps = [
-        "//tensorflow/lite:context",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite:string_util",
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/java/jni",
     ],
     alwayslink = 1,
 )
 
-# Silly rules to make
-# #include <jni.h>
-# in the source headers work
-# (in combination with the "includes" attribute of the tf_cuda_library rule
-# above. Not needed when using the Android toolchain).
-#
-# Inspired from:
-# https://github.com/bazelbuild/bazel/blob/f99a0543f8d97339d32075c7176b79f35be84606/src/main/native/BUILD
-# but hopefully there is a simpler alternative to this.
-genrule(
-    name = "copy_jni_h",
-    srcs = ["@bazel_tools//tools/jdk:jni_header"],
-    outs = ["jni.h"],
-    cmd = "cp -f $< $@",
-)
-
-genrule(
-    name = "copy_jni_md_h",
-    srcs = select({
-        "//tensorflow:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"],
-        "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"],
-    }),
-    outs = ["jni_md.h"],
-    cmd = "cp -f $< $@",
-)
-
 cc_library(
     name = "init_tensorflow",
     srcs = [
         "init_tensorflow_jni.cc",
-    ] + select({
-        # The Android toolchain makes "jni.h" available in the include path.
-        # For non-Android toolchains, generate jni.h and jni_md.h.
-        "//tensorflow:android": [],
-        "//conditions:default": [
-            ":jni.h",
-            ":jni_md.h",
-        ],
-    }),
+    ],
     hdrs = [
         "init_tensorflow_jni.h",
     ],
     copts = tflite_copts(),
-    includes = select({
-        "//tensorflow:android": [],
-        "//conditions:default": ["."],
-    }),
-    linkopts = [
-        "-lm",
-        "-ldl",
-    ],
     deps = [
+        "//tensorflow/lite/java/jni",
         "//tensorflow/lite/testing:init_tensorflow",
     ],
     alwayslink = 1,
diff --git a/tensorflow/lite/java/src/test/native/BUILD b/tensorflow/lite/java/src/test/native/BUILD
index 4d3e82b1ac..481aea7ecd 100644
--- a/tensorflow/lite/java/src/test/native/BUILD
+++ b/tensorflow/lite/java/src/test/native/BUILD
@@ -12,20 +12,11 @@ cc_library(
     testonly = 1,
     srcs = [
         "interpreter_test_jni.cc",
-    ] + select({
-        # The Android toolchain makes "jni.h" available in the include path.
-        # For non-Android toolchains, generate jni.h and jni_md.h.
-        "//tensorflow:android": [],
-        "//conditions:default": [
-            "//tensorflow/lite/java/src/main/native:jni.h",
-            "//tensorflow/lite/java/src/main/native:jni_md.h",
-        ],
-    }),
-    includes = select({
-        "//tensorflow:android": [],
-        "//conditions:default": ["../../main/native/."],
-    }),
-    deps = ["//tensorflow/lite/c:c_api_internal"],
+    ],
+    deps = [
+        "//tensorflow/lite/c:c_api_internal",
+        "//tensorflow/lite/java/jni",
+    ],
 )
 
 tflite_jni_binary(
diff --git a/tensorflow/lite/models/smartreply/demo/app/src/main/BUILD b/tensorflow/lite/models/smartreply/demo/app/src/main/BUILD
index b14af4cb20..73326e994b 100644
--- a/tensorflow/lite/models/smartreply/demo/app/src/main/BUILD
+++ b/tensorflow/lite/models/smartreply/demo/app/src/main/BUILD
@@ -62,6 +62,7 @@ cc_library(
     ],
     deps = [
         "//tensorflow/lite:framework",
+        "//tensorflow/lite/java/jni",
         "//tensorflow/lite/models/smartreply:predictor_lib",
     ],
     alwayslink = 1,
-- 
GitLab


From f9f50b6cf831cdfef15d952152f43ba6542a14ad Mon Sep 17 00:00:00 2001
From: Eddie Zhou <eddz@google.com>
Date: Fri, 7 Dec 2018 16:44:06 -0800
Subject: [PATCH 1528/1554] Move skipping of test_session to setUp(), so child
 tests will also skip setUp(), which can be expensive.

PiperOrigin-RevId: 224598690
---
 tensorflow/python/framework/test_util.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 06316ce2e9..28f25daa6d 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1430,6 +1430,10 @@ class TensorFlowTestCase(googletest.TestCase):
     ops.reset_default_graph()
     random_seed.set_random_seed(random_seed.DEFAULT_GRAPH_SEED)
 
+    # Avoiding calling setUp() for the poorly named test_session method.
+    if self.id().endswith(".test_session"):
+      self.skipTest("Not a test.")
+
   def tearDown(self):
     for thread in self._threads:
       thread.check_termination()
@@ -1713,9 +1717,6 @@ class TensorFlowTestCase(googletest.TestCase):
                    use_gpu=False,
                    force_gpu=False):
     """Use cached_session instead."""
-    if self.id().endswith(".test_session"):
-      self.skipTest("Not a test.")
-
     if context.executing_eagerly():
       yield None
     else:
-- 
GitLab


From ff563b9436509a35bbb5087952c7fbfda44df46f Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 7 Dec 2018 16:44:40 -0800
Subject: [PATCH 1529/1554] Fixed the bug in keras where the callback
 attributes are not correctly checked.

PiperOrigin-RevId: 224598769
---
 tensorflow/contrib/distribute/python/BUILD       |  1 -
 .../contrib/distribute/python/keras_test.py      | 14 ++++----------
 .../keras/engine/distributed_training_utils.py   | 16 ++++++++++++----
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4ea1fa050a..4c9c35da5a 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -611,7 +611,6 @@ cuda_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
-        "noguitar",  # TODO(b/120025010): Re-enable this test on Guitar.
         "notsan",
     ],
 )
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index c0bcb5ecd5..683cc89bfb 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1085,8 +1085,8 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
 
   @combinations.generate(combinations.combine(
       distribution=[
-          combinations.mirrored_strategy_with_two_gpus,
-          combinations.core_mirrored_strategy_with_two_gpus],
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
       mode=['graph', 'eager']))
   def test_unsupported_features(self, distribution):
     with self.cached_session():
@@ -1134,8 +1134,8 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
 
   @combinations.generate(combinations.combine(
       distribution=[
-          combinations.mirrored_strategy_with_two_gpus,
-          combinations.core_mirrored_strategy_with_two_gpus],
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu],
       mode=['graph', 'eager']))
   def test_calling_with_unsupported_predefined_callbacks(self, distribution):
     with self.cached_session():
@@ -1161,12 +1161,6 @@ class TestDistributionStrategyErrorCases(test.TestCase, parameterized.TestCase):
                                    'using'):
         model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
                   callbacks=[keras.callbacks.ReduceLROnPlateau()])
-      with self.assertRaisesRegexp(ValueError,
-                                   'histogram_freq in the TensorBoard callback '
-                                   'is not supported when using '
-                                   'DistributionStrategy.'):
-        model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
-                  callbacks=[keras.callbacks.TensorBoard(histogram_freq=10)])
 
 
 class TestDistributionStrategyWithLossMasking(test.TestCase,
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index d100182381..32129afe64 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -199,11 +199,19 @@ def validate_callbacks(input_callbacks, optimizer, current_strategy):
       # running ops.
       if isinstance(callback, callbacks.TensorBoard):
         if callback.__getattribute__('histogram_freq'):
-          raise ValueError('histogram_freq in the TensorBoard callback is not '
-                           'supported when using DistributionStrategy.')
+          logging.warning(
+              UserWarning(
+                  '`histogram_freq` in the TensorBoard callback is not '
+                  'supported when using DistributionStrategy. Setting '
+                  '`histogram_freq` to `0`.'))
+          callback.histogram_freq = 0
         if callback.__getattribute__('write_grads'):
-          raise ValueError('write_grads in the TensorBoard callback is not '
-                           'supported when using DistributionStrategy.')
+          logging.warning(
+              UserWarning(
+                  '`write_grads` in the TensorBoard callback is not supported '
+                  'when using DistributionStrategy. Setting `write_grads` '
+                  'to `False`.'))
+          callback.histogram_freq = False
 
 
 def validate_distributed_dataset_inputs(distribution_strategy, x, y,
-- 
GitLab


From b659523f8acec043478ee7378dd17c8df4c132e2 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 7 Dec 2018 16:48:22 -0800
Subject: [PATCH 1530/1554] Use parent sessions configuration to ensure we
 connect to the same set of workers.

PiperOrigin-RevId: 224599211
---
 .../contrib/tpu/python/tpu/session_support.py      | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/session_support.py b/tensorflow/contrib/tpu/python/tpu/session_support.py
index a952754878..3e463823c8 100644
--- a/tensorflow/contrib/tpu/python/tpu/session_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/session_support.py
@@ -43,12 +43,19 @@ class CoordinatorShutdownException(Exception):
   pass
 
 
+def _clone_session(session, graph=None):
+  return session_lib.Session(
+      target=session.sess_str,
+      config=session._config,  # pylint: disable=protected-access
+      graph=graph if graph else session.graph)
+
+
 def _make_heartbeat_op(session, device, request_ph):
   """Return a heartbeat op or None if heartbeats are not supported by device."""
   try:
     # Test if we can connect in a isolated graph + session
     with ops.Graph().as_default():
-      with session_lib.Session(target=session.sess_str) as temp_session:
+      with _clone_session(session) as temp_session:
         with ops.device(device):
           heartbeat_op = tpu_ops.worker_heartbeat('')
           options = config_pb2.RunOptions(timeout_in_ms=5000)
@@ -220,6 +227,7 @@ class WatchdogManager(threading.Thread):
     self.ping_interval = ping_interval
     self.shutdown_timeout = shutdown_timeout
     self.daemon = True
+    self._config = session._config  # pylint: disable=protected-access
     self._target = session.sess_str
     self._running = False
     self._devices = devices
@@ -234,6 +242,7 @@ class WatchdogManager(threading.Thread):
     self._session = session_lib.Session(
         target=self._target,
         graph=self._graph,
+        config=self._config,
     )
 
     if self._devices is None:
@@ -334,8 +343,7 @@ class GracefulShutdownHook(session_run_hook.SessionRunHook):
 
     with self._graph.as_default():
       logging.info('Installing graceful shutdown hook.')
-      self._session = session_lib.Session(
-          target=training_session.sess_str, graph=self._graph)
+      self._session = _clone_session(training_session, self._graph)
       self._workers = WorkerHeartbeatManager.from_devices(
           self._session, all_worker_devices(self._session))
       self._heartbeat_supported = self._workers.num_workers() > 0
-- 
GitLab


From 8e1eb82d53c1d60936910847bd38844227c0bd3b Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 7 Dec 2018 16:49:43 -0800
Subject: [PATCH 1531/1554] Split keyword argument test in
 tf_upgrade_v2_test.py into 2 tests. Extend one of the tests to run API
 conversion for all TF functions instead of just the ones updated by
 tf_upgrade_v2 script. Fix issues found by this extended test.

PiperOrigin-RevId: 224599358
---
 tensorflow/python/ops/image_ops_impl.py       |   5 +-
 tensorflow/python/ops/sparse_ops.py           |  10 +-
 tensorflow/tools/compatibility/reorders_v2.py |   7 +
 .../tools/compatibility/tf_upgrade_v2.py      | 115 ++++++++++++-
 .../tools/compatibility/tf_upgrade_v2_test.py | 151 +++++++++++++++---
 5 files changed, 258 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 229393c970..24d049b726 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -2046,9 +2046,8 @@ def sample_distorted_bounding_box_v2(image_size,
       3-D with shape `[batch, N, 4]` describing the N bounding boxes
       associated with the image.
     seed: An optional `int`. Defaults to `0`.
-      If either `seed` or `seed2` are set to non-zero, the random number
-      generator is seeded by the given `seed`.  Otherwise, it is seeded by a
-      random seed.
+      If `seed` is set to non-zero, the random number generator is seeded by
+      the given `seed`.  Otherwise, it is seeded by a random seed.
     min_object_covered: A Tensor of type `float32`. Defaults to `0.1`.
       The cropped area of the image must contain at least this
       fraction of any bounding box supplied. The value of this parameter should
diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index d7346b7371..097b485a11 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -1093,6 +1093,9 @@ def sparse_reduce_max_v2(
 @deprecation.deprecated_endpoints("sparse_reduce_max")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
+@deprecation.deprecated_args(
+    None, "reduction_axes is deprecated, use axis instead",
+    "reduction_axes")
 def sparse_reduce_max(sp_input, axis=None, keepdims=None,
                       reduction_axes=None, keep_dims=None):
   """Computes the max of elements across dimensions of a SparseTensor.
@@ -1141,7 +1144,7 @@ def sparse_reduce_max(sp_input, axis=None, keepdims=None,
     axis: The dimensions to reduce; list or scalar. If `None` (the
       default), reduces all dimensions.
     keepdims: If true, retain reduced dimensions with length 1.
-    reduction_axes: Deprecated name of axis.
+    reduction_axes: Deprecated name of `axis`.
     keep_dims:  Deprecated alias for `keepdims`.
 
   Returns:
@@ -1279,6 +1282,9 @@ def sparse_reduce_sum_v2(
 @deprecation.deprecated_endpoints("sparse_reduce_sum")
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
+@deprecation.deprecated_args(
+    None, "reduction_axes is deprecated, use axis instead",
+    "reduction_axes")
 def sparse_reduce_sum(sp_input, axis=None, keepdims=None,
                       reduction_axes=None, keep_dims=None):
   """Computes the sum of elements across dimensions of a SparseTensor.
@@ -1314,7 +1320,7 @@ def sparse_reduce_sum(sp_input, axis=None, keepdims=None,
     axis: The dimensions to reduce; list or scalar. If `None` (the
       default), reduces all dimensions.
     keepdims: If true, retain reduced dimensions with length 1.
-    reduction_axes: Deprecated name of axis.
+    reduction_axes: Deprecated name of `axis`.
     keep_dims: Deprecated alias for `keepdims`.
 
   Returns:
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index e6596755c3..44494ac148 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -40,6 +40,7 @@ reorders = {
     'tf.io.parse_single_example': ['serialized', 'features', 'name', 'example_names'],
     'tf.io.serialize_many_sparse': ['sp_input', 'name', 'out_type'],
     'tf.io.serialize_sparse': ['sp_input', 'name', 'out_type'],
+    'tf.linalg.norm': ['tensor', 'ord', 'axis', 'keepdims', 'name', 'keep_dims'],
     'tf.math.argmax': ['input', 'axis', 'name', 'dimension', 'output_type'],
     'tf.math.argmin': ['input', 'axis', 'name', 'dimension', 'output_type'],
     'tf.math.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
@@ -66,6 +67,7 @@ reorders = {
     'tf.nn.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
     'tf.nn.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
+    'tf.norm': ['tensor', 'ord', 'axis', 'keepdims', 'name', 'keep_dims'],
     'tf.pad': ['tensor', 'paddings', 'mode', 'name', 'constant_values'],
     'tf.parse_example': ['serialized', 'features', 'name', 'example_names'],
     'tf.parse_single_example': ['serialized', 'features', 'name', 'example_names'],
@@ -82,6 +84,7 @@ reorders = {
     'tf.reduce_min': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.reduce_prod': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
     'tf.reduce_sum': ['input_tensor', 'axis', 'keepdims', 'name', 'reduction_indices', 'keep_dims'],
+    'tf.reverse_sequence': ['input', 'seq_lengths', 'seq_axis', 'batch_axis', 'name', 'seq_dim', 'batch_dim'],
     'tf.serialize_many_sparse': ['sp_input', 'name', 'out_type'],
     'tf.serialize_sparse': ['sp_input', 'name', 'out_type'],
     'tf.shape': ['input', 'name', 'out_type'],
@@ -90,15 +93,19 @@ reorders = {
     'tf.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.sparse.add': ['a', 'b', 'threshold', 'thresh'],
     'tf.sparse.concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
+    'tf.sparse.reduce_max': ['sp_input', 'axis', 'keepdims', 'reduction_axes', 'keep_dims'],
     'tf.sparse.segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse.segment_sqrt_n': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse.segment_sum': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse.split': ['keyword_required', 'sp_input', 'num_split', 'axis', 'name', 'split_dim'],
     'tf.sparse_add': ['a', 'b', 'threshold', 'thresh'],
     'tf.sparse_concat': ['axis', 'sp_inputs', 'name', 'expand_nonconcat_dim', 'concat_dim'],
     'tf.sparse_matmul': ['a', 'b', 'transpose_a', 'transpose_b', 'a_is_sparse', 'b_is_sparse', 'name'],
+    'tf.sparse_reduce_max': ['sp_input', 'axis', 'keepdims', 'reduction_axes', 'keep_dims'],
     'tf.sparse_segment_mean': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse_segment_sqrt_n': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
     'tf.sparse_segment_sum': ['data', 'indices', 'segment_ids', 'name', 'num_segments'],
+    'tf.sparse_split': ['keyword_required', 'sp_input', 'num_split', 'axis', 'name', 'split_dim'],
     'tf.strings.length': ['input', 'name', 'unit'],
     'tf.strings.reduce_join': ['inputs', 'axis', 'keep_dims', 'separator', 'name', 'reduction_indices'],
     'tf.strings.substr': ['input', 'pos', 'len', 'name', 'unit'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index b2477a541e..ea86da42f6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -36,6 +36,18 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.argmax": {
             "dimension": "axis",
         },
+        "tf.arg_min": {
+            "dimension": "axis",
+        },
+        "tf.arg_max": {
+            "dimension": "axis",
+        },
+        "tf.math.argmin": {
+            "dimension": "axis",
+        },
+        "tf.math.argmax": {
+            "dimension": "axis",
+        },
         "tf.image.crop_and_resize": {
             "box_ind": "box_indices",
         },
@@ -54,6 +66,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.space_to_batch": {
             "block_size": "block_shape",
         },
+        "tf.nn.space_to_batch": {
+            "block_size": "block_shape",
+        },
         "tf.constant": {
             "verify_shape": "verify_shape_is_now_always_true",
         },
@@ -66,6 +81,15 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.linalg.l2_normalize": {
             "dim": "axis",
         },
+        "tf.linalg.norm": {
+            "keep_dims": "keepdims",
+        },
+        "tf.norm": {
+            "keep_dims": "keepdims",
+        },
+        "tf.load_file_system_library": {
+            "library_filename": "library_location",
+        },
         "tf.math.count_nonzero": {
             "input_tensor": "input",
             "keep_dims": "keepdims",
@@ -99,6 +123,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.separable_conv2d": {
             "rate": "dilations"
         },
+        "tf.nn.depthwise_conv2d": {
+            "rate": "dilations"
+        },
         "tf.nn.softmax": {
             "dim": "axis"
         },
@@ -117,14 +144,35 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         },
         "tf.sparse.concat": {
             "concat_dim": "axis",
+            "expand_nonconcat_dim": "expand_nonconcat_dims",
         },
         "tf.sparse_concat": {
             "concat_dim": "axis",
+            "expand_nonconcat_dim": "expand_nonconcat_dims",
         },
         "tf.sparse.split": {
             "split_dim": "axis",
         },
-        "tf.max_pool_with_argmax": {
+        "tf.sparse_split": {
+            "split_dim": "axis",
+        },
+        "tf.sparse.reduce_max": {
+            "reduction_axes": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.sparse_reduce_max": {
+            "reduction_axes": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.sparse.reduce_sum": {
+            "reduction_axes": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.sparse_reduce_sum": {
+            "reduction_axes": "axis",
+            "keep_dims": "keepdims",
+        },
+        "tf.nn.max_pool_with_argmax": {
             "Targmax": "output_dtype",
         },
         "tf.multinomial": {
@@ -133,6 +181,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.random.multinomial": {
             "output_dtype": "dtype",
         },
+        "tf.reverse_sequence": {
+            "seq_dim": "seq_axis",
+            "batch_dim": "batch_axis",
+        },
         "tf.nn.batch_norm_with_global_normalization": {
             "t": "input",
             "m": "mean",
@@ -151,6 +203,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.ones_like": {
             "tensor": "input",
         },
+        "tf.nn.conv2d_transpose": {
+            "value": "input",
+            "filter": "filters",
+        },
         "tf.nn.conv3d_transpose": {
             "value": "input",
             "filter": "filters",
@@ -287,6 +343,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.squeeze": {
             "squeeze_dims": "axis",
         },
+        "tf.nn.weighted_moments": {
+            "keep_dims": "keepdims"
+        },
     }
 
     # pylint: disable=line-too-long
@@ -427,6 +486,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.sparse.split",
         "tf.sparse_matmul":
             "tf.linalg.matmul",
+        "tf.sparse_reduce_sum":
+            "tf.sparse.reduce_sum",
+        "tf.sparse_reduce_max":
+            "tf.sparse.reduce_max",
         "tf.random.stateless_multinomial":
             "tf.random.stateless_categorical",
         "tf.string_to_hash_bucket":
@@ -481,6 +544,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.argmax",
         "tf.arg_min":
             "tf.argmin",
+        # tf.nn.ctc_loss is still available in 2.0 but behavior
+        # changed significantly.
+        "tf.nn.ctc_loss":
+            "tf.compat.v1.nn.ctc_loss",
     }
     # pylint: enable=line-too-long
 
@@ -530,6 +597,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.sparse.segment_sqrt_n",
         "tf.sparse.segment_sum",
         "tf.sparse_matmul",
+        "tf.sparse.reduce_max",
+        "tf.sparse_reduce_max",
         "tf.io.decode_csv",
         "tf.strings.substr",
         "tf.strings.reduce_join",
@@ -566,6 +635,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.nn.embedding_lookup_sparse",
         "tf.nn.in_top_k",
         "tf.nn.space_to_depth",
+        "tf.linalg.norm",
+        "tf.norm",
+        "tf.reverse_sequence",
+        "tf.sparse_split",
     }
 
     # Functions that were reordered should be changed to the new keyword args
@@ -634,6 +707,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.assert_rank":
             assert_rank_comment,
+        "tf.cond": "tf.cond no longer takes 'strict'. "
+                   "Now 'strict' defaults to True."
+                   "fn1/fn2 arguments are replaced by true_fn/false_fn.",
         "tf.debugging.assert_equal":
             assert_return_type_comment,
         "tf.debugging.assert_greater":
@@ -664,6 +740,9 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_rank_comment,
         "tf.debugging.assert_rank_in":
             assert_rank_comment,
+        "tf.device": "tf.device no longer takes function as an argument. "
+                     "'devide_name_or_function' argument has been renamed to "
+                     "'device_name'.",
         "tf.flags":
             "tf.flags has been removed, please use the argparse or absl"
             " module if you need command line parsing.",
@@ -701,6 +780,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             default_loss_reduction_changed,
         "tf.estimator.BaselineRegressor":
             default_loss_reduction_changed,
+        "tf.hessians": "tf.hessians no longer takes "
+                       "'colocate_gradients_with_ops' argument. Also, "
+                       "arguments have been reordered so that 'name' is the "
+                       "last argument.",
         "tf.nn.conv1d":
             "WARNING: use_cudnn_on_gpu argument has been removed and \"value\""
             " was renamed to \"input\"",
@@ -727,9 +810,33 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             tf_01s_like_no_optimize_comment,
         "tf.nn.embedding_lookup":
             "WARNING: validate_indices argument has been removed.",
-        "tf.sparse.concat":
-            ("WARNING: expand_nonconcat_dim was renamed to "
-             "expand_nonconcat_dims."),
+        "tf.while_loop":
+            "tf.while_loop no longer takes 'return_same_structure' argument. "
+            "'return_same_structure' now defaults to True. Also, 'name'"
+            "argument is now the last argument.",
+        "tf.image.sample_distorted_bounding_box":
+            "tf.image.sample_distorted_bounding_box no longer takes 'seed2' "
+            "argument.",
+        "tf.nn.ctc_beam_search_decoder":
+            "tf.nn.ctc_beam_search_decoder no longer takes 'merge_repeated' "
+            "argument. 'merge_repeated' now defaults to False.",
+        "tf.nn.fractional_avg_pool":
+            "tf.nn.fractional_avg_pool no longer takes 'seed2' and "
+            "'deterministic' arguments. Now it takes a single 'seed' arg. If "
+            "'seed' is zero, the execution is random and deterministic "
+            "otherwise",
+        "tf.nn.fractional_max_pool":
+            "tf.nn.fractional_max_pool no longer takes 'seed2' and "
+            "'deterministic' arguments. Now it takes a single 'seed' arg. If "
+            "'seed' is zero, the execution is random and deterministic "
+            "otherwise",
+        "tf.nn.softmax_cross_entropy_with_logits":
+            "tf.nn.softmax_cross_entropy_with_logits behavior has changed. "
+            "'labels' needs to be wrapped with tf.stop_gradient to keep the "
+            "old behavior. Also, 'dim' argument has been renamed to 'axis'.",
+        "tf.test.assert_equal_graph_def":
+            "tf.assert_equal_graph_def no longer takes 'checkpoint_v2' "
+            "argument. 'checkpoint_v2' now defaults to True.",
     }
 
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 4460ad5182..63aa5f0c6b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import inspect
 import os
 import tempfile
 
@@ -25,7 +26,6 @@ import six
 import tensorflow as tf
 # OSS TF V2 import placeholder.
 
-
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
 from tensorflow.python.util import tf_decorator
@@ -72,6 +72,15 @@ def get_symbol_for_name(root, name):
   return symbol
 
 
+def get_args(symbol):
+  if hasattr(inspect, "signature"):
+    signature = inspect.signature(symbol)
+    # Ignore *args and **kwargs for now.
+    return [param.name for param in signature.parameters.values()
+            if param.kind == param.POSITIONAL_OR_KEYWORD]
+  return tf_inspect.getargspec(symbol)[0]
+
+
 def get_func_and_args_from_str(call_str):
   """Parse call string to get function and argument names.
 
@@ -88,6 +97,7 @@ def get_func_and_args_from_str(call_str):
   function_name = call_str[:call_str.find("(")]
   args = call_str[open_paren_index+1:close_paren_index].split(",")
   args = [arg.split("=")[0].strip() for arg in args]
+  args = [arg for arg in args if arg]  # filter out empty strings
   return function_name, args
 
 
@@ -200,17 +210,12 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     collect = False
     traverse.traverse(tf.compat.v1, visitor)
 
-  def testKeywordArgNames(self):
-    if not hasattr(tf.compat, "v2"):
-      return
-
+  def testV1KeywordArgNames(self):
     all_keyword_renames = (
         tf_upgrade_v2.TFAPIChangeSpec().function_keyword_renames)
-    v2_name_exceptions = {"verify_shape_is_now_always_true"}
 
-    # Visitor that verifies V1 argument names, converts to V2 and checks
-    # V2 argument names.
-    def conversion_visitor(unused_path, unused_parent, children):
+    # Visitor that verifies V1 argument names.
+    def arg_test_visitor(unused_path, unused_parent, children):
       for child in children:
         _, attr = tf_decorator.unwrap(child[1])
         names_v1 = get_v1_names(attr)
@@ -230,25 +235,77 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                 "%s not found in %s arguments: %s" %
                 (from_name, name, str(arg_names_v1)))
 
+    visitor = public_api.PublicAPIVisitor(arg_test_visitor)
+    visitor.do_not_descend_map["tf"].append("contrib")
+    visitor.private_map["tf.compat"] = ["v1", "v2"]
+    traverse.traverse(tf.compat.v1, visitor)
+
+  def testV2KeywordArgNames(self):
+    # This test converts a call of the form:
+    # tf.foo(arg1=0, arg2=1, ...)
+    # to 2.0. Then, checks that converted function has valid argument names.
+    if not hasattr(tf.compat, "v2"):
+      return
+    v2_arg_exceptions = {
+        "verify_shape_is_now_always_true",
+        # These arguments should not be used, they just specify
+        # that a function takes named arguments.
+        "keyword_required",
+        "_sentinel",
+    }
+    v1_name_exceptions = {
+        "tf.print",  # requires print_function import
+    }
+    function_warnings = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_warnings)
+    function_handles = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_handle)
+    keyword_renames = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_keyword_renames)
+
+    # Visitor that converts to V2 and checks V2 argument names.
+    def conversion_visitor(unused_path, unused_parent, children):
+      for child in children:
+        _, attr = tf_decorator.unwrap(child[1])
+        if not tf_inspect.isfunction(attr):
+          continue
+        names_v1 = get_v1_names(attr)
+        arg_names_v1 = get_args(attr)
+
+        for name in names_v1:
+          tf_name = "tf.%s" % name
+          if tf_name in function_warnings or tf_name in function_handles:
+            continue  # These require manual change
+          if tf_name in v1_name_exceptions:
+            continue
           # Assert that arg names after converting to v2 are present in
           # v2 function.
           # 1. First, create an input of the form:
           #    tf.foo(arg1=val1, arg2=val2, ...)
           args = ",".join(
               ["%s=%d" % (from_name, from_index)
-               for from_index, from_name in enumerate(keyword_renames.keys())])
-          text_input = "%s(%s)" % (name, args)
+               for from_index, from_name in enumerate(arg_names_v1)])
+          text_input = "%s(%s)" % (tf_name, args)
           # 2. Convert the input to V2.
           _, _, _, text = self._upgrade(text_input)
           new_function_name, new_args = get_func_and_args_from_str(text)
+          if new_function_name == "tf.compat.v1.%s" % name:
+            if tf_name in keyword_renames:
+              # If we rename arguments, new function must be available in 2.0.
+              # We should not be using compat.v1 in this case.
+              self.assertFalse(
+                  "Function '%s' is not in 2.0 when converting\n%s\nto\n%s" %
+                  (new_function_name, text_input, text))
+            continue
           # 3. Verify V2 function and arguments.
-          # Note: If we rename arguments, new function must be available in 2.0.
-          # We should not be using compat.v1 in this case.
-          self.assertIn(new_function_name, self.v2_symbols)
-          args_v2 = tf_inspect.getargspec(self.v2_symbols[new_function_name])[0]
-          args_v2.extend(v2_name_exceptions)
+          args_v2 = get_args(self.v2_symbols[new_function_name])
+          args_v2.extend(v2_arg_exceptions)
           for new_arg in new_args:
-            self.assertIn(new_arg, args_v2)
+            self.assertIn(
+                new_arg, args_v2,
+                "Invalid argument '%s' in 2.0 when converting\n%s\nto\n%s.\n"
+                "Supported arguments: %s" % (
+                    new_arg, text_input, text, str(args_v2)))
 
     visitor = public_api.PublicAPIVisitor(conversion_visitor)
     visitor.do_not_descend_map["tf"].append("contrib")
@@ -526,9 +583,24 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     text = "tf.nn.softmax_cross_entropy_with_logits_v2(labels, logits, dim=2)"
     expected_text = (
         "tf.nn.softmax_cross_entropy_with_logits(labels, logits, axis=2)")
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+    self.assertFalse(errors)
+
+  def testSoftMaxCrossEntropyWithLogits(self):
+    text = "tf.nn.softmax_cross_entropy_with_logits(labels, logits, dim=2)"
+    expected_text = (
+        "tf.nn.softmax_cross_entropy_with_logits(labels, logits, dim=2)")
+    _, report, errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+    self.assertIn(
+        "tf.nn.softmax_cross_entropy_with_logits requires manual check.",
+        errors[0])
+    self.assertIn(
+        "tf.nn.softmax_cross_entropy_with_logits behavior has changed. ",
+        report)
+
   def testSparseMatmul(self):
     text = ("tf.sparse_matmul(a, b, c, d, e, f, g)\n")
     expected_text = ("tf.linalg.matmul(a=a, b=b, transpose_a=c, transpose_b=d, "
@@ -540,7 +612,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     text = "tf.nn.weighted_moments(x, axes, freq, name, kd)"
     expected_text = (
         "tf.nn.weighted_moments(x=x, axes=axes, frequency_weights=freq, "
-        "name=name, keep_dims=kd)")
+        "name=name, keepdims=kd)")
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
@@ -554,7 +626,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     text = "tf.sparse.concat(ax, inp, name, exp, concat)"
     expected_text = (
         "tf.sparse.concat(axis=ax, sp_inputs=inp, name=name, "
-        "expand_nonconcat_dim=exp, axis=concat)")
+        "expand_nonconcat_dims=exp, axis=concat)")
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
@@ -576,7 +648,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     text = "tf.nn.space_to_batch(input, paddings, block_size, name)"
     expected_text = (
         "tf.space_to_batch(input=input, paddings=paddings, "
-        "block_size=block_size, name=name)")
+        "block_shape=block_size, name=name)")
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
@@ -628,6 +700,43 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
+  def testPrint(self):
+    # tf.print() cannot be parsed unless we import print_function
+    text = """from __future__ import print_function
+tf.print()
+tf.print('abc')
+"""
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, text)  # Text should stay the same
+
+  def testSparseSplit(self):
+    text = (
+        "tf.sparse_split(sp_input=sp_input, num_split=num_split, axis=axis, "
+        "name=name)")
+    expected_text = (
+        "tf.sparse.split(sp_input=sp_input, num_split=num_split, axis=axis, "
+        "name=name)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = (
+        "tf.sparse_split(sp_input=sp_input, num_split=num_split, "
+        "name=name, split_dim=axis)")
+    expected_text = (
+        "tf.sparse.split(sp_input=sp_input, num_split=num_split, "
+        "name=name, axis=axis)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
+    text = (
+        "tf.sparse.split(sp_input=sp_input, num_split=num_split, "
+        "name=name, split_dim=axis)")
+    expected_text = (
+        "tf.sparse.split(sp_input=sp_input, num_split=num_split, "
+        "name=name, axis=axis)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, expected_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 623cc7e829568118683070d761c21823c7038338 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Fri, 7 Dec 2018 16:54:42 -0800
Subject: [PATCH 1532/1554] [tf.data] Enable `map_and_batch_fusion`
 optimization

PiperOrigin-RevId: 224599990
---
 .../benchmarks/map_and_batch_benchmark.py     | 43 ++++++++++---------
 .../benchmarks/map_vectorization_benchmark.py | 21 ++++++---
 .../optimization/map_and_batch_fusion_test.py |  5 ---
 .../optimization/map_vectorization_test.py    | 19 +++++---
 .../optimization/optimize_dataset_test.py     |  5 ++-
 .../dataset_serialization_test_base.py        | 33 ++++++++++----
 .../experimental/ops/optimization_options.py  |  2 +-
 7 files changed, 81 insertions(+), 47 deletions(-)

diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
index fbd06a5a78..b17f2bcd12 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
@@ -26,6 +26,7 @@ import numpy as np
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -139,47 +140,49 @@ class MapAndBatchBenchmark(test.Benchmark):
 
         num_iters = 1024 // (
             (element_size * batch_size) // min(num_calls, inter_op))
-        dataset = make_base_dataset(element_size)
-        chained_dataset = dataset.map(
+        fused_dataset = make_base_dataset(element_size)
+        fused_dataset = fused_dataset.map(
             math_ops.matmul,
             num_parallel_calls=num_calls).batch(batch_size=batch_size)
-        chained_iterator = dataset_ops.make_one_shot_iterator(chained_dataset)
-        chained_get_next = chained_iterator.get_next()
 
-        chained_deltas = []
+        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
+        fused_get_next = fused_iterator.get_next()
+
+        fused_deltas = []
         with session.Session(
             config=config_pb2.ConfigProto(
                 inter_op_parallelism_threads=inter_op,
                 use_per_session_threads=True)) as sess:
+
           for _ in range(5):
-            sess.run(chained_get_next.op)
+            sess.run(fused_get_next.op)
           for _ in range(num_iters):
             start = time.time()
-            sess.run(chained_get_next.op)
+            sess.run(fused_get_next.op)
             end = time.time()
-            chained_deltas.append(end - start)
+            fused_deltas.append(end - start)
 
-        fused_dataset = dataset.apply(
-            batching.map_and_batch(
-                math_ops.matmul,
-                num_parallel_calls=num_calls,
-                batch_size=batch_size))
-        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
-        fused_get_next = fused_iterator.get_next()
+        # `map_and_batch_fusion` is optimized by default. To get the chained
+        # dataset, with have to disable it.
+        options = dataset_ops.Options()
+        options.experimental_optimization = OptimizationOptions()
+        options.experimental_optimization.map_and_batch_fusion = False
+        chained_dataset = fused_dataset.with_options(options)
+        chained_iterator = dataset_ops.make_one_shot_iterator(chained_dataset)
+        chained_get_next = chained_iterator.get_next()
 
-        fused_deltas = []
+        chained_deltas = []
         with session.Session(
             config=config_pb2.ConfigProto(
                 inter_op_parallelism_threads=inter_op,
                 use_per_session_threads=True)) as sess:
-
           for _ in range(5):
-            sess.run(fused_get_next.op)
+            sess.run(chained_get_next.op)
           for _ in range(num_iters):
             start = time.time()
-            sess.run(fused_get_next.op)
+            sess.run(chained_get_next.op)
             end = time.time()
-            fused_deltas.append(end - start)
+            chained_deltas.append(end - start)
 
         print(
             "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, "
diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
index 47ec6391f7..a60ba0a857 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
@@ -115,13 +116,23 @@ class MapVectorizationBenchmark(test.Benchmark):
   def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id):
     num_elems = int(np.sum([np.prod(x) for x in input_size]))
     name_template = "{}__batch_size_{}_input_element_size_{}_{}"
-    unoptimized = input_dataset.map(map_fn).batch(batch_size)
-    unoptimized_op = dataset_ops.make_one_shot_iterator(unoptimized).get_next()
 
-    optimized = input_dataset.map(map_fn).batch(batch_size)
+    base_dataset = input_dataset.map(map_fn).batch(batch_size)
+
     options = dataset_ops.Options()
-    options.experimental_map_vectorization = True
-    optimized = optimized.with_options(options)
+    opt_options = optimization_options.OptimizationOptions()
+    # Disable default map_and_batch_fusion optimization
+    opt_options.map_and_batch_fusion = False
+    options.experimental_optimization = opt_options
+    base_dataset = base_dataset.with_options(options)
+
+    unoptimized_op = dataset_ops.make_one_shot_iterator(base_dataset).get_next()
+
+    optimized_options = dataset_ops.Options()
+    opt_options = optimization_options.OptimizationOptions()
+    opt_options.map_vectorization = True
+    optimized_options.experimental_optimization = opt_options
+    optimized = base_dataset.with_options(optimized_options)
     optimized_op = dataset_ops.make_one_shot_iterator(optimized).get_next()
 
     unoptimized_time = self._run(
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
index 801f664f09..e2ff3116ec 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import optimization
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import test_util
@@ -32,10 +31,6 @@ class MapAndBatchFusionTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(
             ["MapAndBatch"])).map(lambda x: x * x).batch(10)
-    options = dataset_ops.Options()
-    options.experimental_optimization = OptimizationOptions()
-    options.experimental_optimization.map_and_batch_fusion = True
-    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset, expected_output=[[x * x for x in range(10)]])
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index c2e08e2cd8..adc411bfb5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -23,7 +23,7 @@ import numpy as np
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.data.experimental.ops import optimization
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
+from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -344,18 +344,25 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       Tuple of (unoptimized dataset, optimized dataset).
     """
     map_node_name = "Map" if num_parallel_calls is None else "ParallelMap"
-    batch_size = 100
 
     def _make_dataset(node_names):
-      return base_dataset.apply(optimization.assert_next(node_names)).map(
-          map_fn, num_parallel_calls=num_parallel_calls).batch(batch_size)
+      dataset = base_dataset.apply(optimization.assert_next(node_names))
+      dataset = dataset.map(map_fn, num_parallel_calls)
+      dataset = dataset.batch(100)
+      options = dataset_ops.Options()
+      opt_options = optimization_options.OptimizationOptions()
+      opt_options.map_and_batch_fusion = False
+      options.experimental_optimization = opt_options
+      dataset = dataset.with_options(options)
+      return dataset
 
     unoptimized = _make_dataset([map_node_name, "Batch"])
     optimized = _make_dataset(["Batch", map_node_name]
                               if expect_optimized else [map_node_name, "Batch"])
     options = dataset_ops.Options()
-    options.experimental_optimization = OptimizationOptions()
-    options.experimental_optimization.map_vectorization = True
+    opt_options = optimization_options.OptimizationOptions()
+    opt_options.map_vectorization = True
+    options.experimental_optimization = opt_options
     optimized = optimized.with_options(options)
     return unoptimized, optimized
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 150cc7b5e4..05d88eb071 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -256,8 +256,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testOptimizationEnabledByDefault(self):
     """Tests that some optimizations are applied to datasets by default."""
     options = dataset_ops.Options()
-    expected_optimizations = ["noop_elimination"]
-    self.assertEqual(options._static_optimizations(), expected_optimizations)
+    expected_optimizations = ["noop_elimination", "map_and_batch_fusion"]
+    self.assertEqual(
+        set(options._static_optimizations()), set(expected_optimizations))
 
   def testOptimizationDisableDefault(self):
     """Tests that we can disable all static optimizations enabled by default.
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
index e65aa44d06..bdbd8702b7 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/dataset_serialization_test_base.py
@@ -23,6 +23,7 @@ import os
 import numpy as np
 
 from tensorflow.python.data.experimental.ops import iterator_ops as contrib_iterator_ops
+from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import dtypes
@@ -74,23 +75,39 @@ class DatasetSerializationTestBase(test.TestCase):
     Raises:
       AssertionError if any test fails.
     """
+    # NOTE: We disable all default optimizations in serialization tests in order
+    # to test the actual dataset in question.
+    options = dataset_ops.Options()
+    options.experimental_optimization = OptimizationOptions()
+    options.experimental_optimization.apply_default_optimizations = False
+
+    def ds_fn1_no_opt():
+      return ds_fn1().with_options(options)
+
     self.verify_unused_iterator(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     self.verify_fully_used_iterator(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     self.verify_exhausted_iterator(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     self.verify_init_before_restore(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     self.verify_multiple_breaks(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     self.verify_reset_restored_iterator(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     self.verify_restore_in_empty_graph(
-        ds_fn1, num_outputs, sparse_tensors=sparse_tensors)
+        ds_fn1_no_opt, num_outputs, sparse_tensors=sparse_tensors)
     if ds_fn2:
+
+      def ds_fn2_no_opt():
+        return ds_fn2().with_options(options)
+
       self.verify_restore_in_modified_graph(
-          ds_fn1, ds_fn2, num_outputs, sparse_tensors=sparse_tensors)
+          ds_fn1_no_opt,
+          ds_fn2_no_opt,
+          num_outputs,
+          sparse_tensors=sparse_tensors)
 
   def verify_unused_iterator(self,
                              ds_fn,
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index daf65cd5cd..73849435a9 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -94,7 +94,6 @@ class OptimizationOptions(options.OptionsBase):
     optimizations_to_enable = [
         "filter_fusion",
         "hoist_random_uniform",
-        "map_and_batch_fusion",
         "map_and_filter_fusion",
         "map_fusion",
         "map_parallelization",
@@ -109,6 +108,7 @@ class OptimizationOptions(options.OptionsBase):
       # The following optimizations are turned on by default, unless the
       # user explicitly disables them.
       optimizations_to_disable = [
+          "map_and_batch_fusion",
           "noop_elimination",
       ]
       for optimization in optimizations_to_disable:
-- 
GitLab


From 034565a9a1683201040de985a0d251615921219b Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 7 Dec 2018 17:01:20 -0800
Subject: [PATCH 1533/1554] Update TFLite iOS Camera example to use GPU
 Delegate.

Note: GPU Delegate library isn't released yet. We're
working on it.
PiperOrigin-RevId: 224600723
---
 .../ios/camera/CameraExampleViewController.h  | 26 ++++++++---
 .../ios/camera/CameraExampleViewController.mm | 36 ++++++++++++++-
 tensorflow/lite/examples/ios/camera/Podfile   |  8 +++-
 .../project.pbxproj                           | 44 +++----------------
 .../lite/examples/ios/download_models.sh      |  4 +-
 5 files changed, 70 insertions(+), 48 deletions(-)

diff --git a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h
index fb5800e86d..438e6adc79 100644
--- a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h
+++ b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.h
@@ -17,8 +17,26 @@
 
 #include <vector>
 
-#include "tensorflow/contrib/lite/kernels/register.h"
-#include "tensorflow/contrib/lite/model.h"
+// TensorFlow Lite was migrated out of `contrib/` directory. The change
+// wasn't reflected in newest CocoaPod release yet (1.12.0).
+// Change this to 0 when using a TFLite version which is newer than 1.12.0.
+// TODO(ycling): Remove the macro when we release the next version.
+#ifndef TFLITE_USE_CONTRIB_LITE
+#define TFLITE_USE_CONTRIB_LITE 1
+#endif
+
+// Set TFLITE_USE_GPU_DELEGATE to 1 to use TFLite GPU Delegate.
+// Note: TFLite GPU Delegate binary isn't releast yet, and we're working
+// on it.
+#ifndef TFLITE_USE_GPU_DELEGATE
+#define TFLITE_USE_GPU_DELEGATE 0
+#endif
+
+#if TFLITE_USE_GPU_DELEGATE && TFLITE_USE_CONTRIB_LITE
+// Sanity check.
+#error "GPU Delegate only works with newer TFLite " \
+    "after migrating out of contrib"
+#endif
 
 @interface CameraExampleViewController
     : UIViewController<UIGestureRecognizerDelegate, AVCaptureVideoDataOutputSampleBufferDelegate> {
@@ -33,10 +51,6 @@
   AVCaptureSession* session;
 
   std::vector<std::string> labels;
-  std::unique_ptr<tflite::FlatBufferModel> model;
-  tflite::ops::builtin::BuiltinOpResolver resolver;
-  std::unique_ptr<tflite::Interpreter> interpreter;
-
   double total_latency;
   int total_count;
 }
diff --git a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm
index a3e6e11095..48cd313c9d 100644
--- a/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm
+++ b/tensorflow/lite/examples/ios/camera/CameraExampleViewController.mm
@@ -23,10 +23,20 @@
 #include <iostream>
 #include <queue>
 
+#if TFLITE_USE_CONTRIB_LITE
 #include "tensorflow/contrib/lite/kernels/register.h"
 #include "tensorflow/contrib/lite/model.h"
 #include "tensorflow/contrib/lite/op_resolver.h"
 #include "tensorflow/contrib/lite/string_util.h"
+#else
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/lite/string_util.h"
+#if TFLITE_USE_GPU_DELEGATE
+#include "tensorflow/lite/delegates/gpu/metal_delegate.h"
+#endif
+#endif
 
 #define LOG(x) std::cerr
 
@@ -34,7 +44,12 @@ namespace {
 
 // If you have your own model, modify this to the file name, and make sure
 // you've added the file to your app resources too.
+#if TFLITE_USE_GPU_DELEGATE
+// GPU Delegate only supports float model now.
 NSString* model_file_name = @"mobilenet_v1_1.0_224";
+#else
+NSString* model_file_name = @"mobilenet_quant_v1_224.tflite";
+#endif
 NSString* model_file_type = @"tflite";
 // If you have your own model, point this to the labels file.
 NSString* labels_file_name = @"labels";
@@ -151,7 +166,12 @@ void ProcessInputWithQuantizedModel(
 - (void)teardownAVCapture;
 @end
 
-@implementation CameraExampleViewController
+@implementation CameraExampleViewController {
+  std::unique_ptr<tflite::FlatBufferModel> model;
+  tflite::ops::builtin::BuiltinOpResolver resolver;
+  std::unique_ptr<tflite::Interpreter> interpreter;
+  TfLiteDelegate* delegate;
+}
 
 - (void)setupAVCapture {
   NSError* error = nil;
@@ -363,6 +383,11 @@ void ProcessInputWithQuantizedModel(
 }
 
 - (void)dealloc {
+#if TFLITE_USE_GPU_DELEGATE
+  if (delegate) {
+    DeleteGpuDelegate(delegate);
+  }
+#endif
   [self teardownAVCapture];
 }
 
@@ -388,6 +413,15 @@ void ProcessInputWithQuantizedModel(
   LoadLabels(labels_file_name, labels_file_type, &labels);
 
   tflite::InterpreterBuilder(*model, resolver)(&interpreter);
+
+#if TFLITE_USE_GPU_DELEGATE
+  GpuDelegateOptions options;
+  options.allow_precision_loss = true;
+  options.wait_type = GpuDelegateOptions::WaitType::kActive;
+  delegate = NewGpuDelegate(&options);
+  interpreter->ModifyGraphWithDelegate(delegate);
+#endif
+
   // Explicitly resize the input tensor.
   {
     int input = interpreter->inputs()[0];
diff --git a/tensorflow/lite/examples/ios/camera/Podfile b/tensorflow/lite/examples/ios/camera/Podfile
index 2ed5c8da1e..2e15cc63de 100644
--- a/tensorflow/lite/examples/ios/camera/Podfile
+++ b/tensorflow/lite/examples/ios/camera/Podfile
@@ -4,4 +4,10 @@ inhibit_all_warnings!
 project 'tflite_camera_example.xcodeproj'
 
 target 'tflite_camera_example'
-       pod 'TensorFlowLite', '1.12.0'
+  # Comment 'TensorFlowLite' pod and un-comment 'TensorFlowLiteGpuExperimental'
+  # to use TFLite GPU Delegate.
+  # Note: TFLite GPU Delegate binary isn't releast yet, and we're working
+  # on it.
+
+  pod 'TensorFlowLite', '1.12.0'
+  # pod 'TensorFlowLiteGpuExperimental', '0.0.1'
diff --git a/tensorflow/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj b/tensorflow/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj
index 9522c41dea..9b5c2b32a8 100644
--- a/tensorflow/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj
+++ b/tensorflow/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj
@@ -15,6 +15,7 @@
 		1CDB2D4A1ED3A9CD007929E9 /* CameraExampleViewController.mm in Sources */ = {isa = PBXBuildFile; fileRef = 1CDB2D451ED3A9CD007929E9 /* CameraExampleViewController.mm */; };
 		54DC6C3C5F734F3A58069F0C /* libPods-tflite_camera_example.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3BA8BF92C84895BFE59D8236 /* libPods-tflite_camera_example.a */; };
 		AC1F82661FBA3CBD0052BA77 /* labels.txt in Resources */ = {isa = PBXBuildFile; fileRef = AC1F82641FBA3CBD0052BA77 /* labels.txt */; };
+		AC31178921BB3FF900AFF1D2 /* mobilenet_quant_v1_224.tflite in Resources */ = {isa = PBXBuildFile; fileRef = AC31178821BB3FF900AFF1D2 /* mobilenet_quant_v1_224.tflite */; };
 		AC3BB41720114C400084552C /* mobilenet_v1_1.0_224.tflite in Resources */ = {isa = PBXBuildFile; fileRef = AC3BB41620114C400084552C /* mobilenet_v1_1.0_224.tflite */; };
 /* End PBXBuildFile section */
 
@@ -36,6 +37,7 @@
 		3BC5BE4BBD09374D3E98F082 /* Pods-tflite_camera_example.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-tflite_camera_example.debug.xcconfig"; path = "Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example.debug.xcconfig"; sourceTree = "<group>"; };
 		55ED318E8D29C8AFEF03DF1E /* Pods-tflite_camera_example.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-tflite_camera_example.release.xcconfig"; path = "Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example.release.xcconfig"; sourceTree = "<group>"; };
 		AC1F82641FBA3CBD0052BA77 /* labels.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = labels.txt; sourceTree = "<group>"; };
+		AC31178821BB3FF900AFF1D2 /* mobilenet_quant_v1_224.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_quant_v1_224.tflite; sourceTree = "<group>"; };
 		AC3BB41620114C400084552C /* mobilenet_v1_1.0_224.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_v1_1.0_224.tflite; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
@@ -103,6 +105,7 @@
 		59A3CFF31CF4E68100C4259F /* data */ = {
 			isa = PBXGroup;
 			children = (
+				AC31178821BB3FF900AFF1D2 /* mobilenet_quant_v1_224.tflite */,
 				AC3BB41620114C400084552C /* mobilenet_v1_1.0_224.tflite */,
 				AC1F82641FBA3CBD0052BA77 /* labels.txt */,
 			);
@@ -120,8 +123,6 @@
 				1C564C091ED3A92E00087306 /* Sources */,
 				1C564C0A1ED3A92E00087306 /* Frameworks */,
 				1C564C0B1ED3A92E00087306 /* Resources */,
-				00E875C3B066535AE6B77101 /* [CP] Embed Pods Frameworks */,
-				5C2D02120E3E5E09567AA946 /* [CP] Copy Pods Resources */,
 			);
 			buildRules = (
 			);
@@ -175,42 +176,13 @@
 				AC3BB41720114C400084552C /* mobilenet_v1_1.0_224.tflite in Resources */,
 				1C99111C1ED3B0E600A6BFB9 /* MainStoryboard_iPhone.storyboard in Resources */,
 				AC1F82661FBA3CBD0052BA77 /* labels.txt in Resources */,
+				AC31178921BB3FF900AFF1D2 /* mobilenet_quant_v1_224.tflite in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
 /* End PBXResourcesBuildPhase section */
 
 /* Begin PBXShellScriptBuildPhase section */
-		00E875C3B066535AE6B77101 /* [CP] Embed Pods Frameworks */ = {
-			isa = PBXShellScriptBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			inputPaths = (
-			);
-			name = "[CP] Embed Pods Frameworks";
-			outputPaths = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-			shellPath = /bin/sh;
-			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example-frameworks.sh\"\n";
-			showEnvVarsInLog = 0;
-		};
-		5C2D02120E3E5E09567AA946 /* [CP] Copy Pods Resources */ = {
-			isa = PBXShellScriptBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			inputPaths = (
-			);
-			name = "[CP] Copy Pods Resources";
-			outputPaths = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-			shellPath = /bin/sh;
-			shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example-resources.sh\"\n";
-			showEnvVarsInLog = 0;
-		};
 		66DAEAAEE9EF6550C3A061E0 /* [CP] Check Pods Manifest.lock */ = {
 			isa = PBXShellScriptBuildPhase;
 			buildActionMask = 2147483647;
@@ -322,9 +294,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				HEADER_SEARCH_PATHS = (
-					"$(inherited)",
-				);
+				HEADER_SEARCH_PATHS = "$(inherited)";
 				IPHONEOS_DEPLOYMENT_TARGET = 8.0;
 				MTL_ENABLE_DEBUG_INFO = YES;
 				ONLY_ACTIVE_ARCH = YES;
@@ -365,9 +335,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				HEADER_SEARCH_PATHS = (
-					"$(inherited)",
-				);
+				HEADER_SEARCH_PATHS = "$(inherited)";
 				IPHONEOS_DEPLOYMENT_TARGET = 8.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				SDKROOT = iphoneos;
diff --git a/tensorflow/lite/examples/ios/download_models.sh b/tensorflow/lite/examples/ios/download_models.sh
index ad6ccd1b0a..4828617d95 100755
--- a/tensorflow/lite/examples/ios/download_models.sh
+++ b/tensorflow/lite/examples/ios/download_models.sh
@@ -53,6 +53,6 @@ download_and_extract "${QUANTIZED_MODELS_URL}" "${DOWNLOADS_DIR}/quantized_model
 file ${DOWNLOADS_DIR}/models
 
 cp ${DOWNLOADS_DIR}/models/models/* simple/data/
-cp "${DOWNLOADS_DIR}/quantized_models/labels.txt" camera/data/
+cp ${DOWNLOADS_DIR}/models/models/* camera/data/
 cp "${DOWNLOADS_DIR}/quantized_models/mobilenet_quant_v1_224.tflite" \
-   'camera/data/mobilenet_v1_1.0_224.tflite'
+   'camera/data/mobilenet_quant_v1_224.tflite'
-- 
GitLab


From 7b6a1018b9185542be1630b3267075012fe6e862 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 7 Dec 2018 17:01:29 -0800
Subject: [PATCH 1534/1554] [tf.data] Update `tf.data.Iterator` to use
 `Structure` more internally.

PiperOrigin-RevId: 224600744
---
 tensorflow/python/data/ops/iterator_ops.py | 45 ++++++++++------------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index 6f9c494f39..d0e91b01f9 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -23,7 +23,6 @@ import warnings
 from tensorflow.python.compat import compat
 from tensorflow.python.data.ops import optional_ops
 from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -188,34 +187,32 @@ class Iterator(checkpointable.CheckpointableBase):
     if output_classes is None:
       output_classes = nest.map_structure(lambda _: ops.Tensor, output_types)
     nest.assert_same_structure(output_types, output_shapes)
+    output_structure = structure_lib.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
     if shared_name is None:
       shared_name = ""
+    # pylint: disable=protected-access
     if compat.forward_compatible(2018, 8, 3):
       if _device_stack_is_empty():
         with ops.device("/cpu:0"):
           iterator_resource = gen_dataset_ops.iterator_v2(
               container="",
               shared_name=shared_name,
-              output_types=nest.flatten(
-                  sparse.as_dense_types(output_types, output_classes)),
-              output_shapes=nest.flatten(
-                  sparse.as_dense_shapes(output_shapes, output_classes)))
+              output_types=output_structure._flat_types,
+              output_shapes=output_structure._flat_shapes)
       else:
         iterator_resource = gen_dataset_ops.iterator_v2(
             container="",
             shared_name=shared_name,
-            output_types=nest.flatten(
-                sparse.as_dense_types(output_types, output_classes)),
-            output_shapes=nest.flatten(
-                sparse.as_dense_shapes(output_shapes, output_classes)))
+            output_types=output_structure._flat_types,
+            output_shapes=output_structure._flat_shapes)
     else:
       iterator_resource = gen_dataset_ops.iterator(
           container="",
           shared_name=shared_name,
-          output_types=nest.flatten(
-              sparse.as_dense_types(output_types, output_classes)),
-          output_shapes=nest.flatten(
-              sparse.as_dense_shapes(output_shapes, output_classes)))
+          output_types=output_structure._flat_types,
+          output_shapes=output_structure._flat_shapes)
+    # pylint: enable=protected-access
     return Iterator(iterator_resource, None, output_types, output_shapes,
                     output_classes)
 
@@ -278,30 +275,28 @@ class Iterator(checkpointable.CheckpointableBase):
     if output_classes is None:
       output_classes = nest.map_structure(lambda _: ops.Tensor, output_types)
     nest.assert_same_structure(output_types, output_shapes)
+    output_structure = structure_lib.convert_legacy_structure(
+        output_types, output_shapes, output_classes)
     string_handle = ops.convert_to_tensor(string_handle, dtype=dtypes.string)
+    # pylint: disable=protected-access
     if compat.forward_compatible(2018, 8, 3):
       if _device_stack_is_empty():
         with ops.device("/cpu:0"):
           iterator_resource = gen_dataset_ops.iterator_from_string_handle_v2(
               string_handle,
-              output_types=nest.flatten(
-                  sparse.as_dense_types(output_types, output_classes)),
-              output_shapes=nest.flatten(
-                  sparse.as_dense_shapes(output_shapes, output_classes)))
+              output_types=output_structure._flat_types,
+              output_shapes=output_structure._flat_shapes)
       else:
         iterator_resource = gen_dataset_ops.iterator_from_string_handle_v2(
             string_handle,
-            output_types=nest.flatten(
-                sparse.as_dense_types(output_types, output_classes)),
-            output_shapes=nest.flatten(
-                sparse.as_dense_shapes(output_shapes, output_classes)))
+            output_types=output_structure._flat_types,
+            output_shapes=output_structure._flat_shapes)
     else:
       iterator_resource = gen_dataset_ops.iterator_from_string_handle(
           string_handle,
-          output_types=nest.flatten(
-              sparse.as_dense_types(output_types, output_classes)),
-          output_shapes=nest.flatten(
-              sparse.as_dense_shapes(output_shapes, output_classes)))
+          output_types=output_structure._flat_types,
+          output_shapes=output_structure._flat_shapes)
+    # pylint: enable=protected-access
     return Iterator(iterator_resource, None, output_types, output_shapes,
                     output_classes)
 
-- 
GitLab


From 602b6b86848c27439686f869590f432a54425bb3 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 7 Dec 2018 17:02:42 -0800
Subject: [PATCH 1535/1554] [tf.data] Update `Dataset.reduce()` to use (more
 of) `Structure`.

We still explode the `Structure` into the types/shapes/classes to perform the "most specific compatible structure" calculation. It would probably make sense to support that calculation as a method on `Structure` and its subclasses.

PiperOrigin-RevId: 224600910
---
 tensorflow/python/data/ops/dataset_ops.py | 49 +++++++++++------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index a7cd2f54c7..e08a3af852 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1195,27 +1195,23 @@ class DatasetV2(object):
 
     # Compute initial values for the state classes, shapes and types based on
     # the initial state.
-    state_classes = sparse.get_classes(initial_state)
-    state_shapes = nest.pack_sequence_as(
-        initial_state, [t.get_shape() for t in nest.flatten(initial_state)])
-    state_types = nest.pack_sequence_as(
-        initial_state, [t.dtype for t in nest.flatten(initial_state)])
+    state_structure = structure_lib.Structure.from_value(initial_state)
 
     # Iteratively rerun the reduce function until reaching a fixed point on
-    # `self._state_shapes`.
+    # `state_structure`.
     need_to_rerun = True
     while need_to_rerun:
 
       wrapped_func = StructuredFunctionWrapper(
           reduce_func,
           "reduce()",
-          input_classes=(state_classes, self.output_classes),
-          input_shapes=(state_shapes, self.output_shapes),
-          input_types=(state_types, self.output_types),
+          input_structure=structure_lib.NestedStructure(
+              (state_structure, self._element_structure)),
           add_to_graph=False)
 
       # Extract and validate class information from the returned values.
       output_classes = wrapped_func.output_classes
+      state_classes = state_structure._to_legacy_output_classes()  # pylint: disable=protected-access
       for new_state_class, state_class in zip(
           nest.flatten(output_classes), nest.flatten(state_classes)):
         if not issubclass(new_state_class, state_class):
@@ -1226,6 +1222,7 @@ class DatasetV2(object):
 
       # Extract and validate type information from the returned values.
       output_types = wrapped_func.output_types
+      state_types = state_structure._to_legacy_output_types()  # pylint: disable=protected-access
       for new_state_type, state_type in zip(
           nest.flatten(output_types), nest.flatten(state_types)):
         if new_state_type != state_type:
@@ -1236,6 +1233,7 @@ class DatasetV2(object):
 
       # Extract shape information from the returned values.
       output_shapes = wrapped_func.output_shapes
+      state_shapes = state_structure._to_legacy_output_shapes()  # pylint: disable=protected-access
       flat_state_shapes = nest.flatten(state_shapes)
       flat_new_state_shapes = nest.flatten(output_shapes)
       weakened_state_shapes = [
@@ -1253,27 +1251,26 @@ class DatasetV2(object):
           break
 
       if need_to_rerun:
-        state_shapes = nest.pack_sequence_as(state_shapes,
-                                             weakened_state_shapes)
+        # TODO(b/110122868): Support a "most specific compatible structure"
+        # method for combining structures, to avoid using legacy structures
+        # here.
+        state_structure = structure_lib.convert_legacy_structure(
+            state_types,
+            nest.pack_sequence_as(state_shapes, weakened_state_shapes),
+            state_classes)
 
     reduce_func = wrapped_func.function
     reduce_func.add_to_graph(ops.get_default_graph())
 
-    return sparse.deserialize_sparse_tensors(
-        nest.pack_sequence_as(
-            output_types,
-            gen_dataset_ops.reduce_dataset(
-                self._as_variant_tensor(),  # pylint: disable=protected-access
-                nest.flatten(sparse.serialize_sparse_tensors(initial_state)),
-                reduce_func.captured_inputs,
-                f=reduce_func,
-                output_shapes=nest.flatten(
-                    sparse.as_dense_shapes(output_shapes, output_classes)),
-                output_types=nest.flatten(
-                    sparse.as_dense_types(output_types, output_classes)))),
-        output_types,
-        output_shapes,
-        output_classes)
+    # pylint: disable=protected-access
+    return state_structure._from_compatible_tensor_list(
+        gen_dataset_ops.reduce_dataset(
+            self._as_variant_tensor(),
+            state_structure._to_tensor_list(initial_state),
+            reduce_func.captured_inputs,
+            f=reduce_func,
+            output_shapes=state_structure._flat_shapes,
+            output_types=state_structure._flat_types))
 
   def with_options(self, options):
     """Returns a new `tf.data.Dataset` with the given options set.
-- 
GitLab


From a9bfbc26263ca6af4ea9410daea093633a9b49fa Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 7 Dec 2018 17:16:07 -0800
Subject: [PATCH 1536/1554] Add an Android APK wrapper for the benchmark_model
 utility

This APK offers a more faithful representation of on-device performance
by executing the benchmark in the context of a foreground Activity.
See the README for more details.

PiperOrigin-RevId: 224602509
---
 tensorflow/lite/build_def.bzl                 |  4 +-
 tensorflow/lite/tools/benchmark/README.md     |  5 +
 .../benchmark/android/AndroidManifest.xml     | 42 +++++++++
 tensorflow/lite/tools/benchmark/android/BUILD | 44 +++++++++
 .../lite/tools/benchmark/android/README.md    | 65 +++++++++++++
 .../android/jni/benchmark_model_jni.cc        | 92 +++++++++++++++++++
 .../lite/benchmark/BenchmarkModel.java        | 32 +++++++
 .../benchmark/BenchmarkModelActivity.java     | 44 +++++++++
 8 files changed, 327 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/tools/benchmark/android/AndroidManifest.xml
 create mode 100644 tensorflow/lite/tools/benchmark/android/BUILD
 create mode 100644 tensorflow/lite/tools/benchmark/android/README.md
 create mode 100644 tensorflow/lite/tools/benchmark/android/jni/benchmark_model_jni.cc
 create mode 100644 tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModel.java
 create mode 100644 tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModelActivity.java

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index c61ee63c7e..c17eddf47b 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -112,7 +112,8 @@ def tflite_jni_binary(
         linkshared = 1,
         linkstatic = 1,
         testonly = 0,
-        deps = []):
+        deps = [],
+        srcs = []):
     """Builds a jni binary for TFLite."""
     linkopts = linkopts + [
         "-Wl,--version-script",  # Export only jni functions & classes.
@@ -124,6 +125,7 @@ def tflite_jni_binary(
         linkshared = linkshared,
         linkstatic = linkstatic,
         deps = deps + [linkscript],
+        srcs = srcs,
         linkopts = linkopts,
         testonly = testonly,
     )
diff --git a/tensorflow/lite/tools/benchmark/README.md b/tensorflow/lite/tools/benchmark/README.md
index a71a2fa1c0..a4d9c879eb 100644
--- a/tensorflow/lite/tools/benchmark/README.md
+++ b/tensorflow/lite/tools/benchmark/README.md
@@ -11,6 +11,11 @@ The instructions below are for running the binary on Desktop and Android,
 for iOS please use the
 [iOS benchmark app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/ios).
 
+An experimental Android APK wrapper for the benchmark model utility offers more
+faithful execution behavior on Android (via a foreground Activity). It is
+located
+[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/android).
+
 ## Parameters
 
 The binary takes the following required parameters:
diff --git a/tensorflow/lite/tools/benchmark/android/AndroidManifest.xml b/tensorflow/lite/tools/benchmark/android/AndroidManifest.xml
new file mode 100644
index 0000000000..7cdca2885d
--- /dev/null
+++ b/tensorflow/lite/tools/benchmark/android/AndroidManifest.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="org.tensorflow.lite.benchmark">
+
+    <!-- Necessary for loading custom models from disk. -->
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+
+    <!-- Target SDK 21 (<23) to avoid the need for requesting storage
+         permissions. This APK will almost always be used from the command-line
+         anyway, and be expicitly installed by the developer. -->
+    <uses-sdk
+        android:minSdkVersion="21"
+        android:targetSdkVersion="21" />
+
+    <application>
+        <!-- This Activity runs the TensorFlow Lite benchmark at creation, using
+             a provided set of arguments, then immediately terminates. -->
+        <activity android:name="org.tensorflow.lite.benchmark.BenchmarkModelActivity"
+                  android:screenOrientation="portrait"
+                  android:label="TFLite Benchmark"
+                  android:theme="@android:style/Theme.NoDisplay"
+                  android:exported="true"
+                  android:noHistory="true" />
+    </application>
+
+</manifest>
diff --git a/tensorflow/lite/tools/benchmark/android/BUILD b/tensorflow/lite/tools/benchmark/android/BUILD
new file mode 100644
index 0000000000..a291effddc
--- /dev/null
+++ b/tensorflow/lite/tools/benchmark/android/BUILD
@@ -0,0 +1,44 @@
+# Description:
+#   BenchmarkModel Android harness for TensorFlow Lite benchmarks.
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("//tensorflow/lite:build_def.bzl", "tflite_jni_binary")
+load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
+
+# See README.md for details about building and executing this benchmark.
+android_binary(
+    name = "benchmark_model",
+    srcs = glob([
+        "src/**/*.java",
+    ]),
+    custom_package = "org.tensorflow.lite.benchmark",
+    manifest = "AndroidManifest.xml",
+    # In some platforms we don't have an Android SDK/NDK and this target
+    # can't be built. We need to prevent the build system from trying to
+    # use the target in that case.
+    tags = ["manual"],
+    deps = [":tensorflowlite_benchmark_native"],
+)
+
+tflite_jni_binary(
+    name = "libtensorflowlite_benchmark.so",
+    srcs = glob([
+        "jni/**/*.cc",
+        "jni/**/*.h",
+    ]),
+    deps = [
+        "//tensorflow/lite/java/jni",
+        "//tensorflow/lite/tools/benchmark:benchmark_tflite_model_lib",
+        "//tensorflow/lite/tools/benchmark:logging",
+    ],
+)
+
+cc_library(
+    name = "tensorflowlite_benchmark_native",
+    srcs = ["libtensorflowlite_benchmark.so"],
+    visibility = ["//visibility:private"],
+)
diff --git a/tensorflow/lite/tools/benchmark/android/README.md b/tensorflow/lite/tools/benchmark/android/README.md
new file mode 100644
index 0000000000..f5b67e3f79
--- /dev/null
+++ b/tensorflow/lite/tools/benchmark/android/README.md
@@ -0,0 +1,65 @@
+# TFLite Android Model Benchmark Tool
+
+## Description
+
+This Android benchmark app is a simple wrapper around the TensorFlow Lite
+[command-line benchmark utility](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark).
+
+Pushing and executing binaries directly on Android is a valid approach to
+benchmarking, but it can result in subtle (but observable) differences in
+performance relative to execution within an actual Android app. In particular,
+Android's scheduler tailors behavior based on thread and process priorities,
+which differ between a foreground Activity/Application and a regular background
+binary executed via `adb shell ...`. This tailored behavior is most evident when
+enabling multi-threaded CPU execution with TensorFlow Lite.
+
+To that end, this app offers perhaps a more faithful view of runtime performance
+that developers can expected when deploying TensorFlow Lite with their
+application.
+
+## To build/install/run
+
+(0) Refer to
+https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android
+to edit the `WORKSPACE` to configure the android NDK/SDK.
+
+(1) Build for your specific platform, e.g.:
+
+```
+bazel build -c opt \
+  --config=android_arm64 \
+  --cxxopt='--std=c++11' \
+  tensorflow/lite/tools/benchmark/android:benchmark_model
+```
+
+(2) Connect your phone. Install the benchmark APK to your phone with adb:
+
+```
+adb install -r -d bazel-bin/tensorflow/lite/tools/benchmark/android/benchmark_model.apk
+```
+
+(3) Push the compute graph that you need to test.
+
+```
+adb push mobilenet_quant_v1_224.tflite /data/local/tmp
+```
+
+(4) Run the benchmark. Additional command-line flags are documented
+[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/README.md)
+and can be appended to the `args` string alongside the required `--graph` flag
+(note that all args must be nested in the single quoted string that follows the
+args key).
+
+```
+adb shell am start -S -n
+  org.tensorflow.lite.benchmark/org.tensorflow.lite.benchmark.BenchmarkModelActivity \
+  --es args '"--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite --num_threads=4"'
+```
+
+(5) The results will be available in Android's logcat, e.g.:
+
+```
+adb logcat | grep "Average inference"
+
+... tflite  : Average inference timings in us: Warmup: 91471, Init: 4108, Inference: 80660.1
+```
diff --git a/tensorflow/lite/tools/benchmark/android/jni/benchmark_model_jni.cc b/tensorflow/lite/tools/benchmark/android/jni/benchmark_model_jni.cc
new file mode 100644
index 0000000000..ee67bdafb0
--- /dev/null
+++ b/tensorflow/lite/tools/benchmark/android/jni/benchmark_model_jni.cc
@@ -0,0 +1,92 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <jni.h>
+
+#include <sstream>
+#include <string>
+
+#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
+#include "tensorflow/lite/tools/benchmark/logging.h"
+
+#ifdef __ANDROID__
+#include <android/log.h>
+#endif
+
+namespace tflite {
+namespace benchmark {
+namespace {
+
+class AndroidBenchmarkLoggingListener : public BenchmarkListener {
+  void OnBenchmarkEnd(const BenchmarkResults& results) override {
+    auto inference_us = results.inference_time_us();
+    auto init_us = results.startup_latency_us();
+    auto warmup_us = results.warmup_time_us();
+    std::stringstream results_output;
+    results_output << "Average inference timings in us: "
+                   << "Warmup: " << warmup_us.avg() << ", "
+                   << "Init: " << init_us << ", "
+                   << "Inference: " << inference_us.avg();
+#ifdef __ANDROID__
+    __android_log_print(ANDROID_LOG_ERROR, "tflite", "%s",
+                        results_output.str().c_str());
+#else
+    fprintf(stderr, "%s", results_output.str().c_str());
+#endif
+  }
+};
+
+void Run(int argc, char** argv) {
+  BenchmarkTfLiteModel benchmark;
+  AndroidBenchmarkLoggingListener listener;
+  benchmark.AddListener(&listener);
+  benchmark.Run(argc, argv);
+}
+
+}  // namespace
+}  // namespace benchmark
+}  // namespace tflite
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JNIEXPORT void JNICALL
+Java_org_tensorflow_lite_benchmark_BenchmarkModel_nativeRun(JNIEnv* env,
+                                                            jclass clazz,
+                                                            jstring args_obj) {
+  const char* args_chars = env->GetStringUTFChars(args_obj, nullptr);
+
+  // Split the args string into individual arg tokens.
+  std::istringstream iss(args_chars);
+  std::vector<std::string> args_split{std::istream_iterator<std::string>(iss),
+                                      {}};
+
+  // Construct a fake argv command-line object for the benchmark.
+  std::vector<char*> argv;
+  std::string arg0 = "(BenchmarkModelAndroid)";
+  argv.push_back(const_cast<char*>(arg0.data()));
+  for (auto& arg : args_split) {
+    argv.push_back(const_cast<char*>(arg.data()));
+  }
+
+  tflite::benchmark::Run(static_cast<int>(argv.size()), argv.data());
+
+  env->ReleaseStringUTFChars(args_obj, args_chars);
+}
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
diff --git a/tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModel.java b/tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModel.java
new file mode 100644
index 0000000000..a6cf8d78d5
--- /dev/null
+++ b/tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModel.java
@@ -0,0 +1,32 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark;
+
+/** Helper class for running a native TensorFlow Lite benchmark. */
+class BenchmarkModel {
+  static {
+    System.loadLibrary("tensorflowlite_benchmark");
+  }
+
+  // Executes a standard TensorFlow Lite benchmark according to the provided args.
+  //
+  // Note that {@code args} will be split by the native execution code.
+  public static void run(String args) {
+    nativeRun(args);
+  }
+
+  private static native void nativeRun(String args);
+}
diff --git a/tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModelActivity.java b/tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModelActivity.java
new file mode 100644
index 0000000000..12410adf3d
--- /dev/null
+++ b/tensorflow/lite/tools/benchmark/android/src/org/tensorflow/lite/benchmark/BenchmarkModelActivity.java
@@ -0,0 +1,44 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.lite.benchmark;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.os.Bundle;
+import android.util.Log;
+
+/** Main {@code Activity} class for the benchmark app. */
+public class BenchmarkModelActivity extends Activity {
+
+  private static final String TAG = "tflite_BenchmarkModelActivity";
+
+  private static final String ARGS_INTENT_KEY_0 = "args";
+  private static final String ARGS_INTENT_KEY_1 = "--args";
+
+  @Override
+  public void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+
+    Intent intent = getIntent();
+    Bundle bundle = intent.getExtras();
+    String args = bundle.getString(ARGS_INTENT_KEY_0, bundle.getString(ARGS_INTENT_KEY_1));
+    Log.i(TAG, "Running TensorFlow Lite benchmark with args: " + args);
+
+    BenchmarkModel.run(args);
+
+    finish();
+  }
+}
-- 
GitLab


From bcd4d48cee1ec070f00a401107a85ae416c57abc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 17:17:17 -0800
Subject: [PATCH 1537/1554] Automated rollback of commit
 c05e0bd84fb8a20f746addde640ac0bc35f5026d. Revert #24081.

PiperOrigin-RevId: 224602628
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 459 +++++---------------
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 194 +--------
 tensorflow/stream_executor/dnn.h            | 207 ++-------
 tensorflow/stream_executor/stream.cc        | 172 ++------
 4 files changed, 179 insertions(+), 853 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 387afefc21..1f2e2f48bb 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2380,7 +2380,7 @@ bool ShouldIncludeWinogradNonfusedAlgo(
 }  // namespace
 
 template <class T>
-port::Status CudnnSupport::PrepareForConvolutionImpl(
+port::Status CudnnSupport::DoConvolveImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -2389,34 +2389,6 @@ port::Status CudnnSupport::PrepareForConvolutionImpl(
     const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
     dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  SE_ASSIGN_OR_RETURN(*algorithm_desc,
-                      GetCudnnConvolutionForwardAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, output_nd, scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
-template <class T>
-port::Status CudnnSupport::DoConvolveImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
-    dnn::DataType accumulator_type, const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
@@ -2439,6 +2411,12 @@ port::Status CudnnSupport::DoConvolveImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
+  DeviceMemory<uint8> scratch;
+  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
+                      GetCudnnConvolutionForwardAlgorithm(
+                          stream, cudnn, algorithm_config, input_nd, filter,
+                          conv, output_nd, scratch_allocator, &scratch));
+
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -2454,7 +2432,7 @@ port::Status CudnnSupport::DoConvolveImpl(
   // memory. See nvbugs/2138754, b/80018418.
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algorithm_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
+      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (input_descriptor.ndims() < 3) {
@@ -2479,8 +2457,7 @@ port::Status CudnnSupport::DoConvolveImpl(
     }());
   }
 
-  if (algorithm_desc.algo_id() ==
-          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -2492,19 +2469,18 @@ port::Status CudnnSupport::DoConvolveImpl(
       /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(),
       /*srcData=*/input_data.opaque(), /*filterDesc=*/filter.handle(),
       /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvForwardAlgo(algorithm_desc),
-      /*workSpace=*/scratch_memory->opaque(),
-      /*workSpaceSizeInBytes=*/scratch_memory->size(), /*beta=*/beta,
+      /*algo=*/ToConvForwardAlgo(algo_desc), /*workSpace=*/scratch.opaque(),
+      /*workSpaceSizeInBytes=*/scratch.size(), /*beta=*/beta,
       /*yDesc=*/output_nd.handle(), /*y=*/output_data->opaque()));
 
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algorithm_desc);
+    output_profile_result->set_algorithm(algo_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch_memory->size());
+    output_profile_result->set_scratch_size(scratch.size());
   }
 
   return port::Status::OK();
@@ -2901,7 +2877,7 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolution(
+bool CudnnSupport::DoConvolve(
     Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -2910,70 +2886,12 @@ bool CudnnSupport::PrepareForConvolution(
     const dnn::BatchDescriptor& output_descriptor,
     DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(PrepareForConvolutionImpl<float>(
-                        stream, batch_descriptor, input_data, filter_descriptor,
-                        filter_data, convolution_descriptor, output_descriptor,
-                        output_data, dnn::DataType::kFloat, scratch_allocator,
-                        algorithm_config, algorithm_desc, scratch_memory),
-                    /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(PrepareForConvolutionImpl<double>(
-                        stream, batch_descriptor, input_data, filter_descriptor,
-                        filter_data, convolution_descriptor, output_descriptor,
-                        output_data, dnn::DataType::kDouble, scratch_allocator,
-                        algorithm_config, algorithm_desc, scratch_memory),
-                    /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionImpl<Eigen::half>(
-          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output_data, acc_type,
-          scratch_allocator, algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::DoConvolve(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kFloat, algorithm_desc,
-                     scratch_memory, output_profile_result),
+                     output_data, dnn::DataType::kFloat, scratch_allocator,
+                     algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -2984,14 +2902,14 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<double>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
+    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kDouble, algorithm_desc,
-                     scratch_memory, output_profile_result),
+                     output_data, dnn::DataType::kDouble, scratch_allocator,
+                     algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3002,9 +2920,8 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
+    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
@@ -3013,7 +2930,7 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, acc_type, algorithm_desc, scratch_memory,
+                     output_data, acc_type, scratch_allocator, algorithm_config,
                      output_profile_result),
       /*report_error=*/!output_profile_result);
 }
@@ -3149,7 +3066,7 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
 }
 
 template <class T>
-port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
+port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<T>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3159,36 +3076,6 @@ port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
     DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  SE_ASSIGN_OR_RETURN(
-      *algorithm_desc,
-      GetCudnnConvolutionBackwardDataAlgorithm(
-          stream, cudnn, algorithm_config, in_back_nd, filter, conv,
-          out_back_nd, scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
-template <class T>
-port::Status CudnnSupport::DoConvolveBackwardDataImpl(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3212,6 +3099,12 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
+  DeviceMemory<uint8> scratch;
+  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
+                      GetCudnnConvolutionBackwardDataAlgorithm(
+                          stream, cudnn, algorithm_config, in_back_nd, filter,
+                          conv, out_back_nd, scratch_allocator, &scratch));
+
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3223,8 +3116,7 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     }
   }
 
-  if (algorithm_desc.algo_id() ==
-          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3234,44 +3126,44 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
   // zero-initialized, nvbugs/2254619.
   if (CUDNN_VERSION >= 7000 && CUDNN_VERSION < 7300 &&
-      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-      cudnn_type == CUDNN_DATA_HALF && algorithm_desc.tensor_ops_enabled() &&
+      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
+      cudnn_type == CUDNN_DATA_HALF && algo_desc.tensor_ops_enabled() &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputInputYX &&
       output_descriptor.layout() == dnn::DataLayout::kBatchDepthYX &&
       (convolution_descriptor.vertical_filter_stride() > 1 ||
        convolution_descriptor.horizontal_filter_stride() > 1)) {
-    stream->ThenMemZero(scratch_memory, scratch_memory->size());
+    stream->ThenMemZero(&scratch, scratch.size());
   }
 
-  RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardData(
-      cudnn.handle(),
-      /*alpha=*/alpha,
-      /*wDesc=*/filter.handle(),
-      /*w=*/filter_data.opaque(),
-      /*dyDesc=*/out_back_nd.handle(),
-      /*dy=*/backward_output_data.opaque(),
-      /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardDataAlgo(algorithm_desc),
-      /*workSpace=*/scratch_memory->opaque(),
-      /*workSpaceSizeInBytes=*/scratch_memory->size(),
-      /*beta=*/beta,
-      /*dxDesc=*/in_back_nd.handle(),
-      /*dx=*/backward_input_data->opaque()));
+  RETURN_IF_CUDNN_ERROR(
+      cudnnConvolutionBackwardData(cudnn.handle(),
+                                   /*alpha=*/alpha,
+                                   /*wDesc=*/filter.handle(),
+                                   /*w=*/filter_data.opaque(),
+                                   /*dyDesc=*/out_back_nd.handle(),
+                                   /*dy=*/backward_output_data.opaque(),
+                                   /*convDesc=*/conv.handle(),
+                                   /*algo=*/ToConvBackwardDataAlgo(algo_desc),
+                                   /*workSpace=*/scratch.opaque(),
+                                   /*workSpaceSizeInBytes=*/scratch.size(),
+                                   /*beta=*/beta,
+                                   /*dxDesc=*/in_back_nd.handle(),
+                                   /*dx=*/backward_input_data->opaque()));
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algorithm_desc);
+    output_profile_result->set_algorithm(algo_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch_memory->size());
+    output_profile_result->set_scratch_size(scratch.size());
   }
 
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardData(
+bool CudnnSupport::DoConvolveBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<double>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3281,77 +3173,13 @@ bool CudnnSupport::PrepareForConvolutionBackwardData(
     DeviceMemory<double>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<float>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<Eigen::half>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, acc_type, scratch_allocator, algorithm_config,
-          algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::DoConvolveBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<double>* backward_input_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kDouble, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_input_data, dnn::DataType::kDouble, scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3363,15 +3191,15 @@ bool CudnnSupport::DoConvolveBackwardData(
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& input_descriptor,
     DeviceMemory<float>* backward_input_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
+    ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kFloat, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_input_data, dnn::DataType::kFloat, scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3383,24 +3211,24 @@ bool CudnnSupport::DoConvolveBackwardData(
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& input_descriptor,
     DeviceMemory<Eigen::half>* backward_input_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
+    ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
-                                 output_descriptor, backward_output_data,
-                                 convolution_descriptor, input_descriptor,
-                                 backward_input_data, acc_type, algorithm_desc,
-                                 scratch_memory, output_profile_result),
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, acc_type, scratch_allocator, algorithm_config,
+          output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
 template <class T>
-port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
+port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3410,36 +3238,6 @@ port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
     DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  SE_ASSIGN_OR_RETURN(
-      *algorithm_desc,
-      GetCudnnConvolutionBackwardFilterAlgorithm(
-          stream, cudnn, algorithm_config, input_nd, filter, conv, out_back_nd,
-          scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
-template <class T>
-port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3463,6 +3261,12 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
+  DeviceMemory<uint8> scratch;
+  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
+                      GetCudnnConvolutionBackwardFilterAlgorithm(
+                          stream, cudnn, algorithm_config, input_nd, filter,
+                          conv, out_back_nd, scratch_allocator, &scratch));
+
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3478,8 +3282,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   // results. See nvbugs/2072856
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algorithm_desc.algo_id() !=
-          CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
+      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (output_descriptor.height() > 1 && output_descriptor.width() > 1) {
@@ -3505,8 +3308,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     }());
   }
 
-  if (algorithm_desc.algo_id() ==
-          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3522,7 +3324,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   //
   // See nvbugs/2379553.
   if (CUDNN_VERSION >= 7100 && CUDNN_VERSION < 7300 &&
-      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
+      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
       cudnn_type == CUDNN_DATA_HALF &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
@@ -3540,9 +3342,9 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
       /*diffDesc=*/out_back_nd.handle(),
       /*diffData=*/backward_output_data.opaque(),
       /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardFilterAlgo(algorithm_desc),
-      /*workSpace=*/scratch_memory->opaque(),
-      /*workSpaceSizeInBytes=*/scratch_memory->size(),
+      /*algo=*/ToConvBackwardFilterAlgo(algo_desc),
+      /*workSpace=*/scratch.opaque(),
+      /*workSpaceSizeInBytes=*/scratch.size(),
       /*beta=*/beta,
       /*gradDesc=*/filter.handle(),
       /*dw=*/backward_filter_data->opaque()));
@@ -3550,16 +3352,16 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algorithm_desc);
+    output_profile_result->set_algorithm(algo_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch_memory->size());
+    output_profile_result->set_scratch_size(scratch.size());
   }
 
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
+bool CudnnSupport::DoConvolveBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<double>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3569,17 +3371,18 @@ bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     DeviceMemory<double>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
+      DoConvolveBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
+          backward_filter_data, dnn::DataType::kDouble,
+
+          scratch_allocator, algorithm_config, output_profile_result),
+      /*report_error=*/!output_profile_result);
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
+bool CudnnSupport::DoConvolveBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3589,17 +3392,18 @@ bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     DeviceMemory<float>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(DoConvolveBackwardFilterImpl(
+                        stream, input_descriptor, input_data, output_descriptor,
+                        backward_output_data, convolution_descriptor,
+                        filter_descriptor, backward_filter_data,
+
+                        dnn::DataType::kFloat, scratch_allocator,
+                        algorithm_config, output_profile_result),
+                    /*report_error=*/!output_profile_result);
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
+bool CudnnSupport::DoConvolveBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<Eigen::half>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3609,83 +3413,20 @@ bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     DeviceMemory<Eigen::half>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    dnn::ProfileResult* output_profile_result) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, acc_type, scratch_allocator, algorithm_config,
-          algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::DoConvolveBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<double>* backward_filter_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble, algorithm_desc,
-          scratch_memory, output_profile_result),
-      /*report_error=*/!output_profile_result);
-}
-
-bool CudnnSupport::DoConvolveBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<float>* backward_filter_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kFloat, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_filter_data, acc_type, scratch_allocator, algorithm_config,
+          output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<Eigen::half>* backward_filter_data,
-    const dnn::AlgorithmDesc& algorithm_desc,
-    DeviceMemory<uint8>* scratch_memory,
-    dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(DoConvolveBackwardFilterImpl(
-                        stream, input_descriptor, input_data, output_descriptor,
-                        backward_output_data, convolution_descriptor,
-                        filter_descriptor, backward_filter_data, acc_type,
-                        algorithm_desc, scratch_memory, output_profile_result),
-                    /*report_error=*/!output_profile_result);
-}
-
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardBiasImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 6b5b6c690f..0641be140d 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -252,43 +252,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
                   const DeviceMemory<float>& input_data,
                   const dnn::FilterDescriptor& filter_descriptor,
@@ -296,8 +259,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<float>* output_data,
-                  const dnn::AlgorithmDesc& algorithm_desc,
-                  DeviceMemory<uint8>* scratch_memory,
+                  ScratchAllocator* scratch_allocator,
+                  const dnn::AlgorithmConfig& algorithm_config,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -307,8 +270,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<double>* output_data,
-                  const dnn::AlgorithmDesc& algorithm_desc,
-                  DeviceMemory<uint8>* scratch_memory,
+                  ScratchAllocator* scratch_allocator,
+                  const dnn::AlgorithmConfig& algorithm_config,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -318,8 +281,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<Eigen::half>* output_data,
-                  const dnn::AlgorithmDesc& algorithm_desc,
-                  DeviceMemory<uint8>* scratch_memory,
+                  ScratchAllocator* scratch_allocator,
+                  const dnn::AlgorithmConfig& algorithm_config,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoFusedConvolve(
@@ -421,20 +384,7 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardData(
+  bool DoConvolveBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -444,32 +394,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_input_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -480,8 +404,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -492,48 +416,9 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<Eigen::half>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
+      dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -543,8 +428,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<double>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardFilter(
@@ -555,8 +440,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<float>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardFilter(
@@ -567,8 +452,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<Eigen::half>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardBias(
@@ -778,7 +663,7 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<U>* offset_backprop);
 
   template <class T>
-  port::Status PrepareForConvolutionImpl(
+  port::Status DoConvolveImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<T>& input_data,
       const dnn::FilterDescriptor& filter_descriptor,
@@ -788,19 +673,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
-  template <class T>
-  port::Status DoConvolveImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <typename ElementType, typename BiasType, typename ScaleType>
@@ -820,19 +692,6 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
-  template <class T>
-  port::Status PrepareForConvolutionBackwardDataImpl(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveBackwardDataImpl(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
@@ -842,22 +701,9 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      dnn::ProfileResult* output_profile_result);
-
-  template <class T>
-  port::Status PrepareForConvolutionBackwardFilterImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor_in,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+      dnn::ProfileResult* output_profile_result);
 
   template <class T>
   port::Status DoConvolveBackwardFilterImpl(
@@ -868,8 +714,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 04949c4813..c044a356ef 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -730,7 +730,6 @@ class PoolingDescriptor {
 class AlgorithmDesc {
  public:
   typedef int64 Index;
-  AlgorithmDesc() : AlgorithmDesc(0, false) {}
   AlgorithmDesc(Index a, bool use_tensor_ops) {
     proto_.set_algo_id(a);
     proto_.set_math_type(use_tensor_ops ? AlgorithmProto::TENSOR_OP_MATH
@@ -1176,52 +1175,6 @@ class DnnSupport {
     return false;
   }
 
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<float>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
   // Enqueues a single-precision convolution operation onto the stream.
   //
   // Arguments (all borrowed):
@@ -1235,10 +1188,10 @@ class DnnSupport {
   //  output_descriptor: dimensions of the output layer.
   //  output_data: un-owned device memory region in which to place the
   //    convolution result.
-  //  algorithm_desc: specifies which algorithm should be used for the
+  //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
+  //    space in order to speed up the convolution operation.
+  //  algorithm_config: specifies which algorithm should be used for the
   //    operation.
-  //  scratch: un-owned device memory for scratch space in order to speed up
-  //    the convolution operation.
   //  output_profile_result: the output profile result for this call. The
   //    profiling is only enabled when this is not nullptr.
   //
@@ -1263,9 +1216,8 @@ class DnnSupport {
       const DeviceMemory<float>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a double-precision convolution operation onto the stream.
@@ -1277,9 +1229,8 @@ class DnnSupport {
       const DeviceMemory<double>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result) = 0;
 
   // Enqueues a half-precision convolution operation onto the stream.
@@ -1292,8 +1243,8 @@ class DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half>* output_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the forward convolution pass.
@@ -1349,54 +1300,6 @@ class DnnSupport {
       const BatchDescriptor& output_descriptor,
       DeviceMemory<float>* output_data) = 0;
 
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
   // Enqueues a single-precision backward convolution (for data) operation onto
   // the stream.
   //
@@ -1416,15 +1319,15 @@ class DnnSupport {
   //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
   //    space in order to speed up the convolution operation.
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      Stream* stream, const FilterDescriptor& filter_descriptor,
       const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
+      const BatchDescriptor& output_descriptor,
       DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1434,76 +1337,28 @@ class DnnSupport {
       std::vector<AlgorithmDesc>* out_algorithms);
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      Stream* stream, const FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
-
-  virtual bool DoConvolveBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
-      ProfileResult* output_profile_result) = 0;
-
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
       const BatchDescriptor& output_descriptor,
       DeviceMemory<double> backward_output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
+      const BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
+      ProfileResult* output_profile_result) = 0;
 
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
+  virtual bool DoConvolveBackwardData(
+      Stream* stream, const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
       const BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half> backward_output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
+      const BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
+      ProfileResult* output_profile_result) = 0;
 
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
@@ -1532,8 +1387,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<float>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1550,8 +1405,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<double>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardFilter(
@@ -1562,8 +1417,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<Eigen::half>* backward_filter_data,
-      const dnn::AlgorithmDesc& algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a single-precision backward convolution (for bias) operation onto
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 4503127bee..3edc66cde8 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -549,16 +549,11 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, algorithm_desc,
-          &scratch_memory, nullptr));
+          convolution_descriptor, output_descriptor, output, scratch_allocator,
+          dnn::AlgorithmConfig(),
+          /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -581,16 +576,11 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, algorithm_desc,
-          &scratch_memory, nullptr));
+          convolution_descriptor, output_descriptor, output, scratch_allocator,
+          dnn::AlgorithmConfig(),
+          /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -768,18 +758,10 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
+      auto status = dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolve(
-            this, input_descriptor, input_data, filter_descriptor, filter_data,
-            convolution_descriptor, output_descriptor, output, algorithm_desc,
-            &scratch_memory, output_profile_result);
-      }
+          algorithm_config, output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -807,18 +789,10 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
+      auto status = dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolve(
-            this, input_descriptor, input_data, filter_descriptor, filter_data,
-            convolution_descriptor, output_descriptor, output, algorithm_desc,
-            &scratch_memory, output_profile_result);
-      }
+          algorithm_config, output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -846,18 +820,10 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
+      auto status = dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolve(
-            this, input_descriptor, input_data, filter_descriptor, filter_data,
-            convolution_descriptor, output_descriptor, output, algorithm_desc,
-            &scratch_memory, output_profile_result);
-      }
+          algorithm_config, output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1003,17 +969,10 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, algorithm_desc, &scratch_memory,
+          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1040,20 +999,11 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
+      auto status = dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolveBackwardData(
-            this, filter_descriptor, filter_data, output_descriptor,
-            backward_output_data, convolution_descriptor, input_descriptor,
-            backward_input_data, algorithm_desc, &scratch_memory,
-            output_profile_result);
-      }
+          output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1082,20 +1032,11 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
+      auto status = dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolveBackwardData(
-            this, filter_descriptor, filter_data, output_descriptor,
-            backward_output_data, convolution_descriptor, input_descriptor,
-            backward_input_data, algorithm_desc, &scratch_memory,
-            output_profile_result);
-      }
+          output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1124,20 +1065,11 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
+      auto status = dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolveBackwardData(
-            this, filter_descriptor, filter_data, output_descriptor,
-            backward_output_data, convolution_descriptor, input_descriptor,
-            backward_input_data, algorithm_desc, &scratch_memory,
-            output_profile_result);
-      }
+          output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1164,17 +1096,10 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, algorithm_desc, &scratch_memory,
+          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1213,17 +1138,10 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, algorithm_desc, &scratch_memory,
+          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1250,20 +1168,11 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
+      auto status = dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolveBackwardFilter(
-            this, input_descriptor, input_data, output_descriptor,
-            backward_output_data, convolution_descriptor, filter_descriptor,
-            backward_filter_data, algorithm_desc, &scratch_memory,
-            output_profile_result);
-      }
+          output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1292,20 +1201,11 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
+      auto status = dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolveBackwardFilter(
-            this, input_descriptor, input_data, output_descriptor,
-            backward_output_data, convolution_descriptor, filter_descriptor,
-            backward_filter_data, algorithm_desc, &scratch_memory,
-            output_profile_result);
-      }
+          output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1332,17 +1232,10 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, algorithm_desc, &scratch_memory,
+          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1369,20 +1262,11 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      DeviceMemory<uint8> scratch_memory;
-      dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
+      auto status = dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
-      if (status) {
-        status = dnn->DoConvolveBackwardFilter(
-            this, input_descriptor, input_data, output_descriptor,
-            backward_output_data, convolution_descriptor, filter_descriptor,
-            backward_filter_data, algorithm_desc, &scratch_memory,
-            output_profile_result);
-      }
+          output_profile_result);
       if (!status && !output_profile_result) {
         SetError();
       }
-- 
GitLab


From d35b41c2619cd520c0fa3fd786160d19dcc6339c Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 7 Dec 2018 17:21:52 -0800
Subject: [PATCH 1538/1554] [TF:XLA] Bump open source abseil revision to
 f197d7c72a54064cfde5a2058f1513a4a0ee36fb

PiperOrigin-RevId: 224603073
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ee91cd677b..60dcca3207 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -123,11 +123,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "com_google_absl",
         build_file = clean_dep("//third_party:com_google_absl.BUILD"),
-        sha256 = "f7e3734c3c5854cf2bae1e193fbbd8f7d261673e0f2e042b1fca52732f688a0a",
-        strip_prefix = "abseil-cpp-284378a71b32dfb3af4e3661f585e671d1b603a3",
+        sha256 = "3ad76de484192b2d5afd49d90492b5ed0bc59eb1a4e8e0deecc7a2a077a90251",
+        strip_prefix = "abseil-cpp-f197d7c72a54064cfde5a2058f1513a4a0ee36fb",
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/284378a71b32dfb3af4e3661f585e671d1b603a3.tar.gz",
-            "https://github.com/abseil/abseil-cpp/archive/284378a71b32dfb3af4e3661f585e671d1b603a3.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/f197d7c72a54064cfde5a2058f1513a4a0ee36fb.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/f197d7c72a54064cfde5a2058f1513a4a0ee36fb.tar.gz",
         ],
     )
 
-- 
GitLab


From 4890b781b6b77fcc6c751d351c2f320ef9043e8d Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 7 Dec 2018 17:36:37 -0800
Subject: [PATCH 1539/1554] Annotate tests as @run_v1_only

Skip individual test cases or entire suites that are not
running in v1. Also replace some @run_deprecated_v1
annotations since simply running the test in graph mode
was not enough.

PiperOrigin-RevId: 224604547
---
 .../audio_microfrontend_op_test.py            |   5 +
 .../lite/python/convert_saved_model_test.py   |   7 ++
 tensorflow/lite/python/convert_test.py        |   2 +
 tensorflow/lite/python/lite_test.py           |   4 +
 .../model_coverage/model_coverage_lib_test.py |  10 ++
 tensorflow/python/autograph/impl/api_test.py  |   1 +
 tensorflow/python/client/session_test.py      |  23 ++++
 .../data/kernel_tests/interleave_test.py      |   2 +
 .../kernel_tests/iterator_cluster_test.py     |   5 +
 .../python/data/kernel_tests/map_test.py      |   2 +
 .../multi_device_iterator_test.py             |   9 ++
 .../python/debug/cli/analyzer_cli_test.py     |  36 +-----
 .../python/debug/cli/cli_shared_test.py       |   3 +-
 .../debug/cli/profile_analyzer_cli_test.py    |   7 +-
 .../python/debug/cli/stepper_cli_test.py      |   1 +
 .../python/debug/lib/debug_gradients_test.py  |  15 +--
 .../python/debug/lib/debug_utils_test.py      |  20 +--
 .../debug/lib/dist_session_debug_grpc_test.py |   1 +
 .../debug/lib/session_debug_file_test.py      |   2 +-
 .../debug/lib/session_debug_grpc_test.py      |   3 +
 .../python/debug/lib/session_debug_testlib.py |   1 +
 .../python/debug/lib/source_utils_test.py     |   1 +
 tensorflow/python/debug/lib/stepper_test.py   |   4 +
 .../python/debug/wrappers/disk_usage_test.py  |   1 +
 .../debug/wrappers/dumping_wrapper_test.py    |   1 +
 .../python/debug/wrappers/framework_test.py   |   1 +
 .../debug/wrappers/local_cli_wrapper_test.py  |   1 +
 .../distribute/distribute_coordinator_test.py |   2 +
 tensorflow/python/eager/backprop_test.py      |  19 +++
 .../python/eager/function_gradients_test.py   |   2 +-
 tensorflow/python/eager/function_test.py      |   2 +
 .../framework/auto_control_deps_test.py       |   7 ++
 .../framework/error_interpolation_test.py     |   8 ++
 .../python/framework/graph_util_test.py       |   2 +-
 .../python/framework/meta_graph_test.py       |   2 +-
 tensorflow/python/framework/ops_test.py       |  13 +-
 tensorflow/python/framework/subscribe_test.py |   6 +-
 tensorflow/python/grappler/item_test.py       |   2 +-
 .../python/grappler/memory_optimizer_test.py  |   2 +-
 .../python/grappler/tf_optimizer_test.py      |   4 +-
 tensorflow/python/keras/backend_test.py       |   2 +-
 tensorflow/python/keras/callbacks_test.py     |  10 +-
 tensorflow/python/keras/engine/saving_test.py |   6 +-
 .../python/keras/engine/sequential_test.py    |   2 +-
 .../python/keras/engine/topology_test.py      |   3 +-
 .../python/keras/engine/training_test.py      |   8 +-
 tensorflow/python/keras/integration_test.py   |   9 +-
 tensorflow/python/keras/layers/lstm_test.py   |   6 +-
 .../python/keras/layers/normalization_test.py |   3 +-
 .../python/keras/layers/simplernn_test.py     |   4 +-
 .../python/keras/layers/unified_lstm_test.py  |   4 +-
 .../python/keras/layers/wrappers_test.py      |  10 +-
 .../python/keras/model_subclassing_test.py    |   2 +
 tensorflow/python/keras/models_test.py        |  13 +-
 tensorflow/python/keras/optimizers_test.py    |  18 +--
 .../kernel_tests/atrous_convolution_test.py   |   3 +
 .../python/kernel_tests/base64_ops_test.py    |   1 +
 .../kernel_tests/batch_matmul_op_test.py      |   3 +
 .../boosted_trees/quantile_ops_test.py        |   8 +-
 .../kernel_tests/checkpoint_ops_test.py       |   1 +
 .../python/kernel_tests/cholesky_op_test.py   |   2 +
 .../python/kernel_tests/cond_v2_test.py       |   4 +
 .../conditional_accumulator_test.py           |  12 +-
 .../kernel_tests/control_flow_ops_py_test.py  | 118 +++++++++++++++---
 .../kernel_tests/control_flow_util_test.py    |   4 +
 .../python/kernel_tests/ctc_loss_op_test.py   |  36 +++---
 .../dense_update_ops_no_tsan_test.py          |   8 +-
 .../kernel_tests/dense_update_ops_test.py     |   6 +-
 .../kernel_tests/depthwise_conv_op_test.py    |   4 +
 .../kernel_tests/determinant_op_test.py       |   3 +
 .../distributions/multinomial_test.py         |  17 +++
 .../python/kernel_tests/fifo_queue_test.py    |   4 +
 .../kernel_tests/functional_ops_test.py       |  10 +-
 .../kernel_tests/identity_op_py_test.py       |   2 +-
 .../linalg/linear_operator_test.py            |   6 +-
 .../python/kernel_tests/linalg_grad_test.py   |   2 +
 tensorflow/python/kernel_tests/losses_test.py |  52 ++++----
 .../kernel_tests/matrix_band_part_op_test.py  |   3 +
 .../kernel_tests/matrix_logarithm_op_test.py  |   3 +
 .../kernel_tests/matrix_solve_ls_op_test.py   |   5 +-
 .../matrix_square_root_op_test.py             |   3 +
 .../python/kernel_tests/norm_op_test.py       |   4 +
 .../python/kernel_tests/numerics_test.py      |   6 +-
 .../kernel_tests/padding_fifo_queue_test.py   |   3 +
 .../partitioned_variables_test.py             |   8 +-
 .../kernel_tests/priority_queue_test.py       |   7 ++
 .../python/kernel_tests/py_func_test.py       |  26 +++-
 tensorflow/python/kernel_tests/qr_op_test.py  |   3 +
 .../random/random_shuffle_queue_test.py       |   3 +
 .../resource_variable_ops_test.py             |  10 +-
 tensorflow/python/kernel_tests/rnn_test.py    |   2 +-
 .../kernel_tests/scatter_nd_ops_test.py       |  11 +-
 .../kernel_tests/self_adjoint_eig_op_test.py  |   3 +
 .../python/kernel_tests/session_ops_test.py   |   5 +
 .../signal/reconstruction_ops_test.py         |   3 +
 .../sparse_conditional_accumulator_test.py    |  24 ++--
 .../python/kernel_tests/stack_ops_test.py     |   4 +-
 tensorflow/python/kernel_tests/svd_op_test.py |   5 +
 .../kernel_tests/tensor_array_ops_test.py     |  32 ++---
 .../python/kernel_tests/tensordot_op_test.py  |   4 +
 .../kernel_tests/unicode_encode_op_test.py    |  13 ++
 .../python/kernel_tests/variables_test.py     |  26 ++--
 .../python/kernel_tests/while_v2_test.py      |   4 +-
 .../python/layers/normalization_test.py       |  37 +-----
 tensorflow/python/ops/gradients_test.py       |  23 ++++
 tensorflow/python/ops/nn_test.py              |   1 +
 .../ops/ragged/ragged_batch_gather_op_test.py |   4 +-
 .../ops/ragged/ragged_boolean_mask_op_test.py |   2 +-
 .../ops/ragged/ragged_concat_op_test.py       |   4 +-
 .../python/ops/ragged/ragged_dispatch_test.py |   2 +-
 .../ops/ragged/ragged_from_sparse_op_test.py  |   8 +-
 .../ops/ragged/ragged_from_tensor_op_test.py  |   6 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |   4 +-
 .../ops/ragged/ragged_gather_op_test.py       |  10 +-
 .../ops/ragged/ragged_map_fn_op_test.py       |   6 +-
 .../ragged/ragged_map_inner_values_op_test.py |  18 +--
 .../ops/ragged/ragged_operators_test.py       |   4 +-
 .../ops/ragged/ragged_reduce_op_test.py       |   2 +-
 .../ops/ragged/ragged_row_lengths_op_test.py  |   2 +-
 .../ops/ragged/ragged_segment_op_test.py      |   4 +-
 .../python/ops/ragged/ragged_stack_op_test.py |   2 +-
 .../ops/ragged/ragged_tensor_shape_test.py    |   6 +
 .../python/ops/ragged/ragged_tensor_test.py   |  12 +-
 .../ops/ragged/ragged_to_sparse_op_test.py    |   4 +-
 .../python/ops/ragged/ragged_util_test.py     |   2 +
 .../python/ops/ragged/ragged_where_op_test.py |   2 +-
 .../profiler/internal/run_metadata_test.py    |   2 +-
 .../python/profiler/model_analyzer_test.py    |   4 +
 .../python/profiler/pprof_profiler_test.py    |   2 +-
 tensorflow/python/saved_model/loader_test.py  |  10 +-
 tensorflow/python/saved_model/save_test.py    |   2 +-
 .../python/saved_model/saved_model_test.py    |  15 +--
 .../saved_model/signature_def_utils_test.py   |   2 +
 tensorflow/python/saved_model/utils_test.py   |   8 ++
 tensorflow/python/tools/freeze_graph_test.py  |   4 +-
 tensorflow/python/training/adagrad_test.py    |   4 +-
 .../training/basic_session_run_hooks_test.py  |   4 +-
 .../python/training/checkpoint_ops_test.py    |   2 +
 .../checkpointable/data_structures_test.py    |   4 +
 .../training/checkpointable/util_test.py      |   1 +
 tensorflow/python/training/input_test.py      |  32 ++---
 .../training/learning_rate_decay_test.py      |   2 +-
 .../localhost_cluster_performance_test.py     |   2 +
 .../python/training/monitored_session_test.py |   1 +
 .../python/training/moving_averages_test.py   |  32 ++---
 .../python/training/quantize_training_test.py |   2 +-
 .../python/training/queue_runner_test.py      |   8 +-
 tensorflow/python/training/saver_test.py      |  24 ++--
 ...server_lib_same_variables_no_clear_test.py |   2 +-
 tensorflow/python/training/server_lib_test.py |   7 ++
 .../python/training/session_manager_test.py   |  30 ++---
 .../python/training/slot_creator_test.py      |   6 +-
 tensorflow/python/training/supervisor_test.py |  22 ++--
 .../training/sync_replicas_optimizer_test.py  |   5 +
 .../training/tensorboard_logging_test.py      |   4 +
 .../python/training/training_ops_test.py      |  14 +--
 .../python/training/training_util_test.py     |   4 +-
 .../tools/api/tests/api_compatibility_test.py |   4 +-
 .../compatibility/testdata/test_file_v0_11.py |   8 ++
 .../compatibility/testdata/test_file_v1_12.py |   4 +
 160 files changed, 827 insertions(+), 480 deletions(-)

diff --git a/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py b/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py
index 561f5f7a50..3ce861707f 100644
--- a/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py
+++ b/tensorflow/lite/experimental/microfrontend/python/kernel_tests/audio_microfrontend_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import tensorflow as tf
 
 from tensorflow.lite.experimental.microfrontend.python.ops import audio_microfrontend_op as frontend_op
+from tensorflow.python.framework import test_util
 
 SAMPLE_RATE = 1000
 WINDOW_SIZE = 25
@@ -33,6 +34,7 @@ SMOOTHING_BITS = 10
 
 class AudioFeatureGenerationTest(tf.test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testSimple(self):
     with self.test_session():
       audio = tf.constant(
@@ -51,6 +53,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase):
       self.assertAllEqual(filterbanks.eval(),
                           [[479, 425], [436, 378], [410, 350], [391, 325]])
 
+  @test_util.run_v1_only("b/120545219")
   def testSimpleFloatScaled(self):
     with self.test_session():
       audio = tf.constant(
@@ -72,6 +75,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase):
                           [[7.484375, 6.640625], [6.8125, 5.90625],
                            [6.40625, 5.46875], [6.109375, 5.078125]])
 
+  @test_util.run_v1_only("b/120545219")
   def testStacking(self):
     with self.test_session():
       audio = tf.constant(
@@ -114,6 +118,7 @@ class AudioFeatureGenerationTest(tf.test.TestCase):
           [[479, 425, 479, 425, 436, 378], [479, 425, 436, 378, 410, 350],
            [436, 378, 410, 350, 391, 325], [410, 350, 391, 325, 391, 325]])
 
+  @test_util.run_v1_only("b/120545219")
   def testStackingDropFrame(self):
     with self.test_session():
       audio = tf.constant(
diff --git a/tensorflow/lite/python/convert_saved_model_test.py b/tensorflow/lite/python/convert_saved_model_test.py
index 76113853ca..11bfcdc795 100644
--- a/tensorflow/lite/python/convert_saved_model_test.py
+++ b/tensorflow/lite/python/convert_saved_model_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.saved_model import tag_constants
 
 class TensorFunctionsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testGetTensorsValid(self):
     in_tensor = array_ops.placeholder(
         shape=[1, 16, 16, 3], dtype=dtypes.float32)
@@ -49,6 +50,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
         sess.graph, ["Placeholder"])
     self.assertEqual("Placeholder:0", tensors[0].name)
 
+  @test_util.run_v1_only("b/120545219")
   def testGetTensorsInvalid(self):
     in_tensor = array_ops.placeholder(
         shape=[1, 16, 16, 3], dtype=dtypes.float32)
@@ -61,6 +63,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
     self.assertEqual("Invalid tensors 'invalid-input' were found.",
                      str(error.exception))
 
+  @test_util.run_v1_only("b/120545219")
   def testSetTensorShapeValid(self):
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
@@ -68,6 +71,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
     convert_saved_model.set_tensor_shapes([tensor], {"Placeholder": [5, 3, 5]})
     self.assertEqual([5, 3, 5], tensor.shape.as_list())
 
+  @test_util.run_v1_only("b/120545219")
   def testSetTensorShapeNoneValid(self):
     tensor = array_ops.placeholder(dtype=dtypes.float32)
     self.assertEqual(None, tensor.shape)
@@ -75,6 +79,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
     convert_saved_model.set_tensor_shapes([tensor], {"Placeholder": [1, 3, 5]})
     self.assertEqual([1, 3, 5], tensor.shape.as_list())
 
+  @test_util.run_v1_only("b/120545219")
   def testSetTensorShapeArrayInvalid(self):
     # Tests set_tensor_shape where the tensor name passed in doesn't exist.
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
@@ -88,6 +93,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
         str(error.exception))
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
 
+  @test_util.run_v1_only("b/120545219")
   def testSetTensorShapeDimensionInvalid(self):
     # Tests set_tensor_shape where the shape passed in is incompatiable.
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
@@ -101,6 +107,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase):
         "(?, 3, 5) to [1, 5, 5].", str(error.exception))
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
 
+  @test_util.run_v1_only("b/120545219")
   def testSetTensorShapeEmpty(self):
     tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32)
     self.assertEqual([None, 3, 5], tensor.shape.as_list())
diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index 2a6f1f634f..cf49ee2b47 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("b/120545219")
 class ConvertTest(test_util.TensorFlowTestCase):
 
   def testBasic(self):
@@ -176,6 +177,7 @@ class ConvertTest(test_util.TensorFlowTestCase):
         "QUANTIZED_UINT8.", str(error.exception))
 
 
+@test_util.run_v1_only("b/120545219")
 class ConvertTestOpHint(test_util.TensorFlowTestCase):
   """Test the hint to stub functionality."""
 
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 2c630588ca..1f9c768b44 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -80,6 +80,7 @@ class FromConstructor(test_util.TensorFlowTestCase):
     self.assertTrue(converter._has_valid_tensors())
 
 
+@test_util.run_v1_only('b/120545219')
 class FromSessionTest(test_util.TensorFlowTestCase):
 
   def testFloat(self):
@@ -497,6 +498,7 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     interpreter.allocate_tensors()
 
 
+@test_util.run_v1_only('b/120545219')
 class FromFrozenGraphFile(test_util.TensorFlowTestCase):
 
   def testFloat(self):
@@ -744,6 +746,7 @@ class FromFrozenGraphFile(test_util.TensorFlowTestCase):
     interpreter.allocate_tensors()
 
 
+@test_util.run_v1_only('b/120545219')
 class FromSavedModelTest(test_util.TensorFlowTestCase):
 
   def _createSavedModel(self, shape):
@@ -888,6 +891,7 @@ class FromSavedModelTest(test_util.TensorFlowTestCase):
     interpreter.allocate_tensors()
 
 
+@test_util.run_v1_only('b/120545219')
 class FromKerasFile(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/lite/testing/model_coverage/model_coverage_lib_test.py b/tensorflow/lite/testing/model_coverage/model_coverage_lib_test.py
index 6b4e7427ed..4e329ac97d 100644
--- a/tensorflow/lite/testing/model_coverage/model_coverage_lib_test.py
+++ b/tensorflow/lite/testing/model_coverage/model_coverage_lib_test.py
@@ -28,6 +28,7 @@ from tensorflow.python import keras
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -42,6 +43,7 @@ class EvaluateFrozenGraph(test.TestCase):
     write_graph(sess.graph_def, '', graph_def_file, False)
     return graph_def_file
 
+  @test_util.run_v1_only('b/120545219')
   def testFloat(self):
     with session.Session().as_default() as sess:
       in_tensor = array_ops.placeholder(
@@ -51,6 +53,7 @@ class EvaluateFrozenGraph(test.TestCase):
 
     model_coverage.test_frozen_graph(filename, ['Placeholder'], ['add'])
 
+  @test_util.run_v1_only('b/120545219')
   def testMultipleOutputs(self):
     with session.Session().as_default() as sess:
       in_tensor_1 = array_ops.placeholder(
@@ -84,15 +87,18 @@ class EvaluateFrozenGraph(test.TestCase):
     filename = self._saveFrozenGraph(sess)
     return filename
 
+  @test_util.run_v1_only('b/120545219')
   def testQuantized(self):
     filename = self._getQuantizedModel()
     model_coverage.test_frozen_graph_quant(filename, ['inputA'], ['output'])
 
+  @test_util.run_v1_only('b/120545219')
   def testQuantizedInputShapes(self):
     filename = self._getQuantizedModel()
     model_coverage.test_frozen_graph_quant(
         filename, ['inputA'], ['output'], input_shapes={'inputA': [33, 33]})
 
+  @test_util.run_v1_only('b/120545219')
   def testQuantizedFlexAll(self):
     filename = self._getQuantizedModel()
     model_coverage.test_frozen_graph_quant(
@@ -102,6 +108,7 @@ class EvaluateFrozenGraph(test.TestCase):
 
 class EvaluateSavedModel(test.TestCase):
 
+  @test_util.run_v1_only('b/120545219')
   def testFloat(self):
     saved_model_dir = os.path.join(self.get_temp_dir(), 'simple_savedmodel')
     with session.Session().as_default() as sess:
@@ -139,18 +146,21 @@ class EvaluateKerasModel(test.TestCase):
       os.close(fd)
     return keras_file
 
+  @test_util.run_v1_only('b/120545219')
   def testFloat(self):
     model = self._getSingleInputKerasModel()
     keras_file = self._saveKerasModel(model)
 
     model_coverage.test_keras_model(keras_file)
 
+  @test_util.run_v1_only('b/120545219')
   def testPostTrainingQuantize(self):
     model = self._getSingleInputKerasModel()
     keras_file = self._saveKerasModel(model)
 
     model_coverage.test_keras_model(keras_file, post_training_quantize=True)
 
+  @test_util.run_v1_only('b/120545219')
   def testTargetOps(self):
     model = self._getSingleInputKerasModel()
     keras_file = self._saveKerasModel(model)
diff --git a/tensorflow/python/autograph/impl/api_test.py b/tensorflow/python/autograph/impl/api_test.py
index 66edda5119..d5561ba824 100644
--- a/tensorflow/python/autograph/impl/api_test.py
+++ b/tensorflow/python/autograph/impl/api_test.py
@@ -218,6 +218,7 @@ class ApiTest(test.TestCase):
                              constant_op.constant(-1))
       self.assertEqual(1, self.evaluate(x))
 
+  @test_util.run_v1_only('b/120545219')
   def test_converted_call_functools_partial(self):
 
     def test_fn(x, y, z):
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 347833ce8f..c4a118a414 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -312,6 +312,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertEqual(None, res[2])
       self.assertEqual(44.0, res[1])
 
+  @test_util.run_v1_only('b/120545219')
   def testFetchAttrs(self):
     if attr is None:
       self.skipTest('attr module is unavailable.')
@@ -340,6 +341,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(val3, result.field1)
       self.assertAllEqual(val2, result.field2)
 
+  @test_util.run_v1_only('b/120545219')
   def testFetchNestedAttrs(self):
     if attr is None:
       self.skipTest('attr module is unavailable.')
@@ -1024,6 +1026,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       fed_c_val = c.eval(feed_dict={a.name: [[4.0, 4.0]]})
       self.assertAllEqual([[16.0, 16.0, 16.0]], fed_c_val)
 
+  @test_util.run_v1_only('b/120545219')
   def testOperationRunMethod(self):
     with session.Session():
       a = constant_op.constant(1.0, shape=[1, 2])
@@ -1154,6 +1157,7 @@ class SessionTest(test_util.TensorFlowTestCase):
         else:
           importer.import_graph_def(gdef, name='import')
 
+  @test_util.run_v1_only('b/120545219')
   def testParallelRunAndSingleBuild(self):
     with session.Session() as sess:
       c = constant_op.constant(5.0)
@@ -1174,6 +1178,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       for t in threads:
         t.join()
 
+  @test_util.run_v1_only('b/120545219')
   def testParallelRunAndParallelBuild(self):
     with session.Session() as sess:
       c = constant_op.constant(5.0)
@@ -1274,6 +1279,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       with self.assertRaisesRegexp(RuntimeError, 'The Session graph is empty.'):
         sess.run({})
 
+  @test_util.run_v1_only('b/120545219')
   def testNotEntered(self):
     # pylint: disable=protected-access
     self.assertEqual(ops._default_session_stack.get_default(), None)
@@ -1289,6 +1295,7 @@ class SessionTest(test_util.TensorFlowTestCase):
           ValueError, lambda e: 'No default session is registered.' in str(e)):
         c_2.eval()
 
+  @test_util.run_v1_only('b/120545219')
   def testInteractive(self):
     with ops.device('/cpu:0'):
       sess = session.InteractiveSession()
@@ -1301,6 +1308,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertAllEqual([[24.0]], e.eval())
       sess.close()
 
+  @test_util.run_v1_only('b/120545219')
   def testMultipleInteractiveSessionsWarning(self):
     # Reinitialize the global state to ensure that the expected warnings will
     # be emitted.
@@ -1328,6 +1336,7 @@ class SessionTest(test_util.TensorFlowTestCase):
     sess2.close()
     sess.close()
 
+  @test_util.run_v1_only('b/120545219')
   def testInteractivePlacePrunedGraph(self):
     sess = session.InteractiveSession()
 
@@ -1349,6 +1358,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       a.eval()
     sess.close()
 
+  @test_util.run_v1_only('b/120545219')
   def testDefaultSessionPlacePrunedGraph(self):
     sess = session.Session()
 
@@ -1769,9 +1779,11 @@ class SessionTest(test_util.TensorFlowTestCase):
     sess.run(a, run_metadata=run_metadata)
     self.assertEqual(len(run_metadata.partition_graphs), 0)
 
+  @test_util.run_v1_only('b/120545219')
   def testOutputPartitionGraphsDirect(self):
     self.runTestOutputPartitionGraphs(session.Session())
 
+  @test_util.run_v1_only('b/120545219')
   def testOutputPartitionGraphsDistributed(self):
     server = server_lib.Server.create_local_server()
     self.runTestOutputPartitionGraphs(session.Session(server.target))
@@ -1796,6 +1808,7 @@ class SessionTest(test_util.TensorFlowTestCase):
     del sess1
     del sess2
 
+  @test_util.run_v1_only('b/120545219')
   def testAsDefault(self):
     c = constant_op.constant(37)
     sess = session.Session()
@@ -1821,6 +1834,7 @@ class SessionTest(test_util.TensorFlowTestCase):
     with self.assertRaisesRegexp(TypeError, 'graph must be a tf.Graph'):
       session.Session(graph=37)
 
+  @test_util.run_v1_only('b/120545219')
   def testTimeoutWithShortOperations(self):
     num_epochs = 5
     q = data_flow_ops.FIFOQueue(capacity=50, dtypes=[dtypes.int32], shapes=[()])
@@ -1834,6 +1848,7 @@ class SessionTest(test_util.TensorFlowTestCase):
         sess.run(enqueue_op)
       self.assertEqual(sess.run(q.size()), num_epochs * 2)
 
+  @test_util.run_v1_only('b/120545219')
   def testRegisterFetchAndFeedConversionFunctions(self):
 
     class SquaredTensor(object):
@@ -1865,6 +1880,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       squared_eval = sess.partial_run(partial_run, squared_tensor)
       self.assertAllClose(np2 * np2, squared_eval)
 
+  @test_util.run_v1_only('b/120545219')
   def testDefaultLogDevicePlacement(self):
 
     class CaptureStderr(str):
@@ -1914,6 +1930,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertTrue('/job:local/replica:0/task:0/device:CPU:0' in str(log),
                       str(log))
 
+  @test_util.run_v1_only('b/120545219')
   def testLocalMasterSessionTimeout(self):
     # Test that the timeout passed in a config to the session works correctly.
     config = config_pb2.ConfigProto(operation_timeout_in_ms=1000)
@@ -1927,6 +1944,7 @@ class SessionTest(test_util.TensorFlowTestCase):
       with self.assertRaises(errors.DeadlineExceededError):
         sess.run(dequeued_t)
 
+  @test_util.run_v1_only('b/120545219')
   def testDefaultServerTimeout(self):
     # Test that the default server config timeout gets used when no Session
     # config is provided.
@@ -1952,9 +1970,11 @@ class SessionTest(test_util.TensorFlowTestCase):
     with self.assertRaisesOpError('has inputs from different frames'):
       sess.run(res, feed_dict={data: 1.0})
 
+  @test_util.run_v1_only('b/120545219')
   def testBuildGraphErrorDirect(self):
     self.runTestBuildGraphError(session.Session())
 
+  @test_util.run_v1_only('b/120545219')
   def testBuildGraphErrorDist(self):
     server = server_lib.Server.create_local_server()
     self.runTestBuildGraphError(session.Session(server.target))
@@ -1993,9 +2013,11 @@ class SessionTest(test_util.TensorFlowTestCase):
       result = sess.run(f)
       self.assertEqual(result, 2.0)
 
+  @test_util.run_v1_only('b/120545219')
   def testAddFunctionToSession(self):
     self.runTestAddFunctionToSession()
 
+  @test_util.run_v1_only('b/120545219')
   def testAddFunctionToGrpcSession(self):
     server = server_lib.Server.create_local_server()
     self.runTestAddFunctionToSession(server.target)
@@ -2009,6 +2031,7 @@ class SessionTest(test_util.TensorFlowTestCase):
     with session.Session():
       pass
 
+  @test_util.run_v1_only('b/120545219')
   def testAutoConvertAndCheckData(self):
     with self.cached_session() as sess:
       a = array_ops.placeholder(dtype=dtypes.string)
diff --git a/tensorflow/python/data/kernel_tests/interleave_test.py b/tensorflow/python/data/kernel_tests/interleave_test.py
index c3450e6525..05a211afcc 100644
--- a/tensorflow/python/data/kernel_tests/interleave_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_test.py
@@ -264,6 +264,7 @@ class InterleaveTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("8", np.int64([4, 0, 6]), 2, 3, 1),
       ("9", np.int64([4, 0, 6]), 2, 3, 2),
   )
+  @test_util.run_v1_only("b/120545219")
   def testSkipEagerSloppyInterleaveInOrder(self, input_values, cycle_length,
                                            block_length, num_parallel_calls):
     get_next, coordination_events = _make_coordinated_sloppy_dataset(
@@ -286,6 +287,7 @@ class InterleaveTest(test_base.DatasetTestBase, parameterized.TestCase):
       ("3", np.int64([4, 5, 6]), 3, 2, 3),
       ("4", np.int64([4, 0, 6]), 2, 3, 2),
   )
+  @test_util.run_v1_only("b/120545219")
   def testSkipEagerSloppyInterleaveOutOfOrder(self, input_values, cycle_length,
                                               block_length, num_parallel_calls):
     get_next, coordination_events = _make_coordinated_sloppy_dataset(
diff --git a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
index 728bed20a1..2008823495 100644
--- a/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_cluster_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.platform import test
 
 class IteratorClusterTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testRemoteIteratorWithoutRemoteCallFail(self):
     worker_config = config_pb2.ConfigProto()
     worker_config.device_count["CPU"] = 2
@@ -92,6 +93,7 @@ class IteratorClusterTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(remote_op, feed_dict={target_placeholder: device1})
 
+  @test_util.run_v1_only("b/120545219")
   def testRemoteIteratorUsingRemoteCallOp(self):
     worker_config = config_pb2.ConfigProto()
     worker_config.device_count["CPU"] = 2
@@ -102,6 +104,7 @@ class IteratorClusterTest(test.TestCase):
                                    "/job:worker/replica:0/task:0/cpu:1",
                                    worker[0].target)
 
+  @test_util.run_v1_only("b/120545219")
   def testRemoteIteratorUsingRemoteCallOpCrossProcess(self):
     workers, _ = test_util.create_local_cluster(2, 1)
 
@@ -109,6 +112,7 @@ class IteratorClusterTest(test.TestCase):
                                    "/job:worker/replica:0/task:1/cpu:0",
                                    workers[0].target)
 
+  @test_util.run_v1_only("b/120545219")
   def testCaptureHashTableInSharedIterator(self):
     worker, _ = test_util.create_local_cluster(1, 1)
 
@@ -143,6 +147,7 @@ class IteratorClusterTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  @test_util.run_v1_only("b/120545219")
   def testImplicitDisposeParallelMapDataset(self):
     # Tests whether a parallel map dataset will be cleaned up correctly when
     # the pipeline does not run it until exhaustion.
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index 8b22542e11..e07706413d 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
@@ -82,6 +83,7 @@ def _make_coordinated_sloppy_dataset(num_elements, num_parallel_calls):
   return next_element, coordination_events
 
 
+@test_util.run_v1_only("b/120545219")
 class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _buildMapDataset(self, components, count):
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 622ebb55de..0322d1f2c6 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.platform import test
 # TODO(b/117581999): Add eager coverage.
 class MultiDeviceIteratorTest(test_base.DatasetTestBase):
 
+  @test_util.run_v1_only("b/120545219")
   def testNoGetNext(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -44,6 +45,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
     with self.test_session(config=config) as sess:
       self.evaluate(multi_device_iterator.initializer)
 
+  @test_util.run_v1_only("b/120545219")
   def testBasic(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -60,6 +62,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
+  @test_util.run_v1_only("b/120545219")
   def testOneOnSameDevice(self):
     with ops.device("/cpu:0"):
       dataset = dataset_ops.Dataset.range(10)
@@ -77,6 +80,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
+  @test_util.run_v1_only("b/120545219")
   def testRepeatDevices(self):
     with ops.device("/cpu:0"):
       dataset = dataset_ops.Dataset.range(20)
@@ -99,6 +103,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
         self.evaluate(elem_on_3)
         self.evaluate(elem_on_4)
 
+  @test_util.run_v1_only("b/120545219")
   def testNotFullyDivisible(self):
     dataset = dataset_ops.Dataset.range(9)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -116,6 +121,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
+  @test_util.run_v1_only("b/120545219")
   def testGetNextAsOptional(self):
     dataset = dataset_ops.Dataset.range(9)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -149,6 +155,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(elem_on_2_t)
 
+  @test_util.run_v1_only("b/120545219")
   def testUneven(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -166,6 +173,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
+  @test_util.run_v1_only("b/120545219")
   def testMultipleInitializations(self):
     with ops.device("/cpu:0"):
       epoch = array_ops.placeholder(dtypes.int64, shape=[])
@@ -259,6 +267,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase):
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(elem_on_2_t)
 
+  @test_util.run_v1_only("b/120545219")
   def testOptimization(self):
     dataset = dataset_ops.Dataset.range(10)
     dataset = dataset.apply(optimization.assert_next(["MemoryCacheImpl"]))
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index 322ecf9466..586982dc4b 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -573,6 +573,7 @@ def create_analyzer_cli(dump):
   return analyzer, registry
 
 
+@test_util.run_v1_only("b/120545219")
 class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
 
   @classmethod
@@ -645,7 +646,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual(len("Size (B)") + 1, dump_size_col_width)
     self.assertEqual(len("Op type") + 1, op_type_col_width)
 
-  @test_util.run_deprecated_v1
   def testMeasureTensorListColumnWidthsGivesRightAnswerForData(self):
     dump = self._debug_dump.dumped_tensor_data[0]
     self.assertLess(dump.dump_size_bytes, 1000)
@@ -661,7 +661,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     # column should be determined by the length of "VariableV2".
     self.assertEqual(len("VariableV2") + 1, op_type_col_width)
 
-  @test_util.run_deprecated_v1
   def testListTensors(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", [])
@@ -675,7 +674,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     # Check the main menu.
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInReverseTimeOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "timestamp", "-r"])
@@ -691,7 +689,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=True)
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInDumpSizeOrderWorks(self):
     out = self._registry.dispatch_command("lt", ["-s", "dump_size"])
     assert_listed_tensors(
@@ -705,7 +702,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         sort_by="dump_size")
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInReverseDumpSizeOrderWorks(self):
     out = self._registry.dispatch_command("lt", ["-s", "dump_size", "-r"])
     assert_listed_tensors(
@@ -725,7 +721,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertIn("ValueError: Unsupported key to sort tensors by: foobar",
                   out.lines)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInOpTypeOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "op_type"])
@@ -741,7 +736,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=False)
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInReverseOpTypeOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "op_type", "-r"])
@@ -757,7 +751,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=True)
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInTensorNameOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "tensor_name"])
@@ -773,7 +766,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=False)
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsInReverseTensorNameOrderWorks(self):
     # Use shorthand alias for the command prefix.
     out = self._registry.dispatch_command("lt", ["-s", "tensor_name", "-r"])
@@ -789,7 +781,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         reverse=True)
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorsFilterByNodeNameRegex(self):
     out = self._registry.dispatch_command("list_tensors",
                                           ["--node_name_filter", ".*read.*"])
@@ -803,7 +794,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     assert_listed_tensors(self, out, [], [], node_name_regex="^read")
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorFilterByOpTypeRegex(self):
     out = self._registry.dispatch_command("list_tensors",
                                           ["--op_type_filter", "Identity"])
@@ -832,7 +822,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         op_type_regex="(Add|MatMul)")
     check_main_menu(self, out, list_tensors_enabled=False)
 
-  @test_util.run_deprecated_v1
   def testListTensorWithFilterAndNodeNameExclusionWorks(self):
     # First, create and register the filter.
     def is_2x1_vector(datum, tensor):
@@ -889,7 +878,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     out = self._registry.dispatch_command("list_tensors", ["--bar"])
     check_syntax_error_output(self, out, "list_tensors")
 
-  @test_util.run_deprecated_v1
   def testNodeInfoByNodeName(self):
     node_name = "simple_mul_add/matmul"
     out = self._registry.dispatch_command("node_info", [node_name])
@@ -914,7 +902,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         [(len(out.lines[0]) - len(node_name), len(out.lines[0]), "bold")],
         out.font_attr_segs[0])
 
-  @test_util.run_deprecated_v1
   def testNodeInfoShowAttributes(self):
     node_name = "simple_mul_add/matmul"
     out = self._registry.dispatch_command("node_info", ["-a", node_name])
@@ -938,7 +925,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         print_tensor_node_name=node_name,
         list_outputs_node_name=node_name)
 
-  @test_util.run_deprecated_v1
   def testNodeInfoShowDumps(self):
     node_name = "simple_mul_add/matmul"
     out = self._registry.dispatch_command("node_info", ["-d", node_name])
@@ -963,7 +949,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
                     len(out.lines[16]) - len(out.lines[16].strip()),
                     len(out.lines[16]), "pt %s:0 -n 0" % node_name)
 
-  @test_util.run_deprecated_v1
   def testNodeInfoShowStackTraceUnavailableIsIndicated(self):
     self._debug_dump.set_python_graph(None)
 
@@ -987,7 +972,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         print_tensor_node_name=node_name,
         list_outputs_node_name=node_name)
 
-  @test_util.run_deprecated_v1
   def testNodeInfoShowStackTraceAvailableWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
 
@@ -1011,7 +995,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         print_tensor_node_name=node_name,
         list_outputs_node_name=node_name)
 
-  @test_util.run_deprecated_v1
   def testNodeInfoByTensorName(self):
     node_name = "simple_mul_add/u/read"
     tensor_name = node_name + ":0"
@@ -1381,7 +1364,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         break
     return index
 
-  @test_util.run_deprecated_v1
   def testPrintSourceForOpNamesWholeFileWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1434,7 +1416,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual("pt simple_mul_add/add",
                      out.font_attr_segs[index + 1][0][2].content)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceForTensorNamesWholeFileWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1455,7 +1436,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual("pt simple_mul_add/u:0",
                      out.font_attr_segs[index + 2][0][2].content)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceForOpNamesStartingAtSpecifiedLineWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1482,7 +1462,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertEqual("pt simple_mul_add/u/read",
                      out.font_attr_segs[index + 3][0][2].content)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceForOpNameSettingMaximumElementCountWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command(
@@ -1527,7 +1506,6 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         self.assertTrue(cli_shared.COLOR_GRAY in attr_seg[2] or
                         attr_seg[2] == cli_shared.COLOR_GRAY)
 
-  @test_util.run_deprecated_v1
   def testListSourceWithNodeNameFilterWithMatchesWorks(self):
     self._debug_dump.set_python_graph(self._sess.graph)
     out = self._registry.dispatch_command("list_source", ["-n", ".*/read"])
@@ -1691,6 +1669,7 @@ class AnalyzerCLIPrintLargeTensorTest(test_util.TensorFlowTestCase):
     self.assertNotIn("...,", out.lines[4])
 
 
+@test_util.run_v1_only("b/120545219")
 class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
 
   @classmethod
@@ -1742,7 +1721,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
     # Tear down temporary dump directory.
     shutil.rmtree(cls._dump_root)
 
-  @test_util.run_deprecated_v1
   def testNodeInfoWithControlDependencies(self):
     # Call node_info on a node with control inputs.
     out = self._registry.dispatch_command("node_info",
@@ -1783,7 +1761,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[z_line]),
                     "ni -a -d -t control_deps/ctrl_dep_z")
 
-  @test_util.run_deprecated_v1
   def testListInputsNonRecursiveNoControl(self):
     """List inputs non-recursively, without any control inputs."""
 
@@ -1826,7 +1803,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[3]) - len("control_deps/ctrl_dep_y"),
                     len(out.lines[3]), "li -c -r control_deps/ctrl_dep_y")
 
-  @test_util.run_deprecated_v1
   def testListInputsNonRecursiveNoControlUsingTensorName(self):
     """List inputs using the name of an output tensor of the node."""
 
@@ -1855,7 +1831,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[3]) - len("control_deps/ctrl_dep_y"),
                     len(out.lines[3]), "li -c -r control_deps/ctrl_dep_y")
 
-  @test_util.run_deprecated_v1
   def testListInputsNonRecursiveWithControls(self):
     """List inputs non-recursively, with control inputs."""
     node_name = "control_deps/ctrl_dep_z"
@@ -1886,7 +1861,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[5]) - len("control_deps/x"),
                     len(out.lines[5]), "li -c -r control_deps/x")
 
-  @test_util.run_deprecated_v1
   def testListInputsRecursiveWithControls(self):
     """List inputs recursively, with control inputs."""
     node_name = "control_deps/ctrl_dep_z"
@@ -1932,7 +1906,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                     len(out.lines[18]) - len("control_deps/x"),
                     len(out.lines[18]), "li -c -r control_deps/x")
 
-  @test_util.run_deprecated_v1
   def testListInputsRecursiveWithControlsWithDepthLimit(self):
     """List inputs recursively, with control inputs and a depth limit."""
     node_name = "control_deps/ctrl_dep_z"
@@ -1992,7 +1965,6 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
         "ERROR: There is no node named \"control_deps/z/foo\" in the "
         "partition graphs"], out.lines)
 
-  @test_util.run_deprecated_v1
   def testListRecipientsRecursiveWithControlsWithDepthLimit(self):
     """List recipients recursively, with control inputs and a depth limit."""
 
@@ -2025,6 +1997,7 @@ class AnalyzerCLIControlDepTest(test_util.TensorFlowTestCase):
                      out.font_attr_segs[0])
 
 
+@test_util.run_v1_only("b/120545219")
 class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
 
   @classmethod
@@ -2064,7 +2037,6 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
     # Tear down temporary dump directory.
     shutil.rmtree(cls._dump_root)
 
-  @test_util.run_deprecated_v1
   def testMultipleDumpsPrintTensorNoNumber(self):
     output = self._registry.dispatch_command("pt", ["while/Identity:0"])
 
@@ -2082,7 +2054,6 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
     self.assertEqual("For example:", output.lines[-2])
     self.assertEqual("  print_tensor while/Identity:0 -n 0", output.lines[-1])
 
-  @test_util.run_deprecated_v1
   def testMultipleDumpsPrintTensorWithNumber(self):
     for i in xrange(5):
       output = self._registry.dispatch_command(
@@ -2096,7 +2067,6 @@ class AnalyzerCLIWhileLoopTest(test_util.TensorFlowTestCase):
       self.assertTrue(output.lines[4].startswith("array(%d" % i))
       self.assertTrue(output.lines[4].endswith(")"))
 
-  @test_util.run_deprecated_v1
   def testMultipleDumpsPrintTensorInvalidNumber(self):
     output = self._registry.dispatch_command("pt",
                                              ["while/Identity:0", "-n", "10"])
diff --git a/tensorflow/python/debug/cli/cli_shared_test.py b/tensorflow/python/debug/cli/cli_shared_test.py
index d191a234fd..66a12efda5 100644
--- a/tensorflow/python/debug/cli/cli_shared_test.py
+++ b/tensorflow/python/debug/cli/cli_shared_test.py
@@ -105,6 +105,7 @@ class TimeToReadableStrTest(test_util.TensorFlowTestCase):
       cli_shared.time_to_readable_str(100, force_time_unit="ks")
 
 
+@test_util.run_v1_only("b/120545219")
 class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -334,6 +335,7 @@ class GetRunStartIntroAndDescriptionTest(test_util.TensorFlowTestCase):
     self.assertEqual("run #1: 1 fetch (a:0); 1 feed (foo)", short_description)
 
 
+@test_util.run_v1_only("b/120545219")
 class GetErrorIntroTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -342,7 +344,6 @@ class GetErrorIntroTest(test_util.TensorFlowTestCase):
   def tearDown(self):
     ops.reset_default_graph()
 
-  @test_util.run_deprecated_v1
   def testShapeError(self):
     tf_error = errors.OpError(None, self.var_a.initializer, "foo description",
                               None)
diff --git a/tensorflow/python/debug/cli/profile_analyzer_cli_test.py b/tensorflow/python/debug/cli/profile_analyzer_cli_test.py
index effcd500c7..d6d2b58b5f 100644
--- a/tensorflow/python/debug/cli/profile_analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/profile_analyzer_cli_test.py
@@ -70,6 +70,7 @@ def _assert_no_lines_match(pattern, lines):
         "%s matched at least one line in %s." % (pattern, str(lines)))
 
 
+@test_util.run_v1_only("b/120545219")
 class ProfileAnalyzerListProfileTest(test_util.TensorFlowTestCase):
 
   def testNodeInfoEmpty(self):
@@ -321,6 +322,7 @@ class ProfileAnalyzerListProfileTest(test_util.TensorFlowTestCase):
     _assert_at_least_one_line_matches(r"Device Total.*0\.009ms", prof_output)
 
 
+@test_util.run_v1_only("b/120545219")
 class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -348,7 +350,6 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
     ops.reset_default_graph()
     super(ProfileAnalyzerPrintSourceTest, self).tearDown()
 
-  @test_util.run_deprecated_v1
   def testPrintSourceForWhileLoop(self):
     prof_output = self.prof_analyzer.print_source([__file__])
 
@@ -362,7 +363,6 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         r"\[(\|)+(\s)*\] .*us .*7\(55\) .*L%d.*(\S)+" % self.loop_lineno,
         prof_output.lines)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceOutputContainsClickableLinks(self):
     prof_output = self.prof_analyzer.print_source([__file__])
     any_match, line_index = _at_least_one_line_matches(
@@ -379,7 +379,6 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         break
     self.assertTrue(any_menu_item_match)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceWithNonDefaultTimeUnit(self):
     prof_output = self.prof_analyzer.print_source([
         __file__, "--time_unit", "ms"])
@@ -394,7 +393,6 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         r"\[(\|)+(\s)*\] .*ms .*7\(55\) .*L%d.*(\S)+" % self.loop_lineno,
         prof_output.lines)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceWithNodeNameFilter(self):
     prof_output = self.prof_analyzer.print_source([
         __file__, "--node_name_filter", "x$"])
@@ -427,7 +425,6 @@ class ProfileAnalyzerPrintSourceTest(test_util.TensorFlowTestCase):
         break
     self.assertTrue(any_menu_item_match)
 
-  @test_util.run_deprecated_v1
   def testPrintSourceWithOpTypeFilter(self):
     prof_output = self.prof_analyzer.print_source([
         __file__, "--op_type_filter", "Less"])
diff --git a/tensorflow/python/debug/cli/stepper_cli_test.py b/tensorflow/python/debug/cli/stepper_cli_test.py
index 7b8a42c253..5cf69d0168 100644
--- a/tensorflow/python/debug/cli/stepper_cli_test.py
+++ b/tensorflow/python/debug/cli/stepper_cli_test.py
@@ -129,6 +129,7 @@ def _parse_updated(lines):
   return updated
 
 
+@test_util.run_v1_only("b/120545219")
 class NodeStepperSimpleGraphTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/debug/lib/debug_gradients_test.py b/tensorflow/python/debug/lib/debug_gradients_test.py
index 1c53147863..885691c3ef 100644
--- a/tensorflow/python/debug/lib/debug_gradients_test.py
+++ b/tensorflow/python/debug/lib/debug_gradients_test.py
@@ -36,6 +36,7 @@ from tensorflow.python.platform import googletest
 from tensorflow.python.training import gradient_descent
 
 
+@test_util.run_v1_only("b/120545219")
 class IdentifyGradientTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -54,7 +55,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     ops.reset_default_graph()
     debug_gradients.clear_gradient_debuggers()
 
-  @test_util.run_deprecated_v1
   def testIdentifyGradientGivesCorrectTensorObjectWithoutContextManager(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     id_grad_w = grad_debugger.identify_gradient(self.w)
@@ -85,7 +85,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
-  @test_util.run_deprecated_v1
   def testIdentifyGradientGivesCorrectTensorObjectWithTfGradients(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     id_grad_w = grad_debugger.identify_gradient(self.w)
@@ -117,7 +116,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
-  @test_util.run_deprecated_v1
   def testCallingIdentifyGradientTwiceWithTheSameGradientsDebuggerErrors(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     grad_debugger.identify_gradient(self.w)
@@ -125,7 +123,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
                                  "The graph already contains an op named .*"):
       grad_debugger.identify_gradient(self.w)
 
-  @test_util.run_deprecated_v1
   def testIdentifyGradientWorksOnMultipleLosses(self):
     grad_debugger_1 = debug_gradients.GradientsDebugger()
     grad_debugger_2 = debug_gradients.GradientsDebugger()
@@ -154,7 +151,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy))
     self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy))
 
-  @test_util.run_deprecated_v1
   def testIdentifyGradientRaisesLookupErrorForUnknownXTensor(self):
     grad_debugger_1 = debug_gradients.GradientsDebugger()
     grad_debugger_2 = debug_gradients.GradientsDebugger()
@@ -175,7 +171,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
         r"This GradientsDebugger has not received any gradient tensor for "):
       grad_debugger_2.gradient_tensor(self.w)
 
-  @test_util.run_deprecated_v1
   def testIdentifyGradientRaisesTypeErrorForNonTensorOrTensorNameInput(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     with self.assertRaisesRegexp(
@@ -184,7 +179,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
         r"has type .*Operation.*"):
       grad_debugger.gradient_tensor(variables.global_variables_initializer())
 
-  @test_util.run_deprecated_v1
   def testIdentifyGradientTensorWorksWithGradientDescentOptimizer(self):
     grad_debugger = debug_gradients.GradientsDebugger()
     id_grad_w = grad_debugger.identify_gradient(self.w)
@@ -200,7 +194,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
-  @test_util.run_deprecated_v1
   def testWatchGradientsByXTensorNamesWorks(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -227,7 +220,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
-  @test_util.run_deprecated_v1
   def testWatchGradientsByXTensorNamesWorksWithoutContextManager(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -254,7 +246,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertIsInstance(w_grad, ops.Tensor)
     self.assertAllClose(1.0, self.sess.run(w_grad))
 
-  @test_util.run_deprecated_v1
   def testWatchGradientsWorksOnRefTensor(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -273,7 +264,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(3.0, self.sess.run(
         grad_debugger.gradient_tensor("u:0")))
 
-  @test_util.run_deprecated_v1
   def testWatchGradientsWorksOnMultipleTensors(self):
     y = math_ops.add(self.w, -1.0, name="y")
 
@@ -294,7 +284,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(3.0, self.sess.run(
         grad_debugger.gradient_tensor("u:0")))
 
-  @test_util.run_deprecated_v1
   def testWatchGradientsByXTensorsWorks(self):
     y = math_ops.add(self.w, -1.0, name="foo/y")
     z = math_ops.square(y, name="foo/z")
@@ -317,7 +306,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(10.0, self.sess.run(w_grad))
     self.assertAllClose(30.0, self.sess.run(u_grad))
 
-  @test_util.run_deprecated_v1
   def testWatchGradientsByTensorCanWorkOnMultipleLosses(self):
     y = math_ops.add(self.w, -1.0, name="y")
     z1 = math_ops.square(y, name="z1")
@@ -343,7 +331,6 @@ class IdentifyGradientTest(test_util.TensorFlowTestCase):
     self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy))
     self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy))
 
-  @test_util.run_deprecated_v1
   def testGradientsValuesFromDumpWorks(self):
     y = math_ops.add(self.w, -1.0, name="y")
     z = math_ops.square(y, name="z")
diff --git a/tensorflow/python/debug/lib/debug_utils_test.py b/tensorflow/python/debug/lib/debug_utils_test.py
index cf59b30e3d..9d59cfc179 100644
--- a/tensorflow/python/debug/lib/debug_utils_test.py
+++ b/tensorflow/python/debug/lib/debug_utils_test.py
@@ -185,7 +185,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
     self.assertEqual(["file:///tmp/tfdbg_1", "file:///tmp/tfdbg_2"],
                      watch_0.debug_urls)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_allNodes(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -217,7 +217,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
     self.assertTrue("p1" in node_names)
     self.assertTrue("s" in node_names)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_nodeNameWhitelist(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -232,7 +232,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         sorted(["a1_init", "a1", "a1/Assign", "a1/read", "p1"]),
         sorted(node_names))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_opTypeWhitelist(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -258,7 +258,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertEqual(["p1"], node_names)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_tensorDTypeWhitelist(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -271,7 +271,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertItemsEqual(["a1", "a1/Assign", "b", "b/Assign"], node_names)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_nodeNameAndTensorDTypeWhitelists(self):
     debug_utils.watch_graph(
         self._run_options,
@@ -285,7 +285,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertItemsEqual(["a1", "a1/Assign"], node_names)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_nodeNameBlacklist(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -300,7 +300,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         sorted(["b_init", "b", "b/Assign", "b/read", "c", "s"]),
         sorted(node_names))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_opTypeBlacklist(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -313,7 +313,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertEqual(sorted(["p1", "s"]), sorted(node_names))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_nodeNameAndOpTypeBlacklists(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -327,7 +327,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
         ["DebugIdentity"], ["file:///tmp/tfdbg_1"])
     self.assertEqual(["s"], node_names)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_tensorDTypeBlacklists(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
@@ -344,7 +344,7 @@ class DebugUtilsTest(test_util.TensorFlowTestCase):
     self.assertNotIn("b/Assign", node_names)
     self.assertIn("s", node_names)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWatchGraph_nodeNameAndTensorDTypeBlacklists(self):
     debug_utils.watch_graph_with_blacklists(
         self._run_options,
diff --git a/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py b/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
index 74498c8ea3..2405e29aaa 100644
--- a/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
+++ b/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py
@@ -44,6 +44,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 
 
+@test_util.run_v1_only("b/120545219")
 class DistributedSessionDebugTest(test_util.TensorFlowTestCase):
   """Test the debugging of distributed sessions."""
 
diff --git a/tensorflow/python/debug/lib/session_debug_file_test.py b/tensorflow/python/debug/lib/session_debug_file_test.py
index f5f9ba29ab..16ab815d92 100644
--- a/tensorflow/python/debug/lib/session_debug_file_test.py
+++ b/tensorflow/python/debug/lib/session_debug_file_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
 
 
+@test_util.run_v1_only("b/120545219")
 class SessionDebugFileTest(session_debug_testlib.SessionDebugTestBase):
 
   def _debug_urls(self, run_number=None):
@@ -45,7 +46,6 @@ class SessionDebugFileTest(session_debug_testlib.SessionDebugTestBase):
     else:
       return os.path.join(self._dump_root, "run_%d" % run_number)
 
-  @test_util.run_deprecated_v1
   def testAllowsDifferentWatchesOnDifferentRuns(self):
     """Test watching different tensors on different runs of the same graph."""
 
diff --git a/tensorflow/python/debug/lib/session_debug_grpc_test.py b/tensorflow/python/debug/lib/session_debug_grpc_test.py
index bfc9a3a382..472e244915 100644
--- a/tensorflow/python/debug/lib/session_debug_grpc_test.py
+++ b/tensorflow/python/debug/lib/session_debug_grpc_test.py
@@ -91,6 +91,7 @@ class GrpcDebugServerTest(test_util.TensorFlowTestCase):
     server.stop_server().wait()
 
 
+@test_util.run_v1_only("b/120545219")
 class SessionDebugGrpcTest(session_debug_testlib.SessionDebugTestBase):
 
   @classmethod
@@ -353,6 +354,7 @@ class SessionDebugConcurrentTest(
     return urls
 
 
+@test_util.run_v1_only("b/120545219")
 class SessionDebugGrpcGatingTest(test_util.TensorFlowTestCase):
   """Test server gating of debug ops."""
 
@@ -730,6 +732,7 @@ class SessionDebugGrpcGatingTest(test_util.TensorFlowTestCase):
       self.assertEqual("DebugNumericSummary", debug_watch.debug_op)
 
 
+@test_util.run_v1_only("b/120545219")
 class DelayedDebugServerTest(test_util.TensorFlowTestCase):
 
   def testDebuggedSessionRunWorksWithDelayedDebugServerStartup(self):
diff --git a/tensorflow/python/debug/lib/session_debug_testlib.py b/tensorflow/python/debug/lib/session_debug_testlib.py
index 25ef91b575..5165febff5 100644
--- a/tensorflow/python/debug/lib/session_debug_testlib.py
+++ b/tensorflow/python/debug/lib/session_debug_testlib.py
@@ -84,6 +84,7 @@ class _RNNCellForTest(rnn_cell_impl.RNNCell):
     return (math_ops.multiply(self._w, input_), state)
 
 
+@test_util.run_v1_only("b/120545219")
 class SessionDebugTestBase(test_util.TensorFlowTestCase):
   """Base class for unit tests of tfdbg running with tf.Session."""
 
diff --git a/tensorflow/python/debug/lib/source_utils_test.py b/tensorflow/python/debug/lib/source_utils_test.py
index 9083297fdb..4f4aea0321 100644
--- a/tensorflow/python/debug/lib/source_utils_test.py
+++ b/tensorflow/python/debug/lib/source_utils_test.py
@@ -216,6 +216,7 @@ class SourceHelperTest(test_util.TensorFlowTestCase):
     os.remove(unrelated_source_path)
 
 
+@test_util.run_v1_only("b/120545219")
 class ListSourceAgainstDumpTest(test_util.TensorFlowTestCase):
 
   def createAndRunGraphWithWhileLoop(self):
diff --git a/tensorflow/python/debug/lib/stepper_test.py b/tensorflow/python/debug/lib/stepper_test.py
index 3839c67198..9e78e207b8 100644
--- a/tensorflow/python/debug/lib/stepper_test.py
+++ b/tensorflow/python/debug/lib/stepper_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.platform import googletest
 from tensorflow.python.training import gradient_descent
 
 
+@test_util.run_v1_only("b/120545219")
 class StepperTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -443,6 +444,7 @@ class StepperTest(test_util.TensorFlowTestCase):
           self.assertAllClose(-4.0, result["fz"]["z"])
 
 
+@test_util.run_v1_only("b/120545219")
 class StepperTestWithPlaceHolders(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -577,6 +579,7 @@ class StepperTestWithPlaceHolders(test_util.TensorFlowTestCase):
       self.assertAllClose([[-1.0], [6.0]], stepper.finalize())
 
 
+@test_util.run_v1_only("b/120545219")
 class StepperAssignAddTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -692,6 +695,7 @@ class StepperAssignAddTest(test_util.TensorFlowTestCase):
       self.assertAllClose(12.0, stepper.cont(self.v))
 
 
+@test_util.run_v1_only("b/120545219")
 class StepperBackwardRunTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/debug/wrappers/disk_usage_test.py b/tensorflow/python/debug/wrappers/disk_usage_test.py
index 0874525966..88b1cd540d 100644
--- a/tensorflow/python/debug/wrappers/disk_usage_test.py
+++ b/tensorflow/python/debug/wrappers/disk_usage_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.platform import googletest
 from tensorflow.python.training import monitored_session
 
 
+@test_util.run_v1_only("b/120545219")
 class DumpingDebugWrapperDiskUsageLimitTest(test_util.TensorFlowTestCase):
 
   @classmethod
diff --git a/tensorflow/python/debug/wrappers/dumping_wrapper_test.py b/tensorflow/python/debug/wrappers/dumping_wrapper_test.py
index 11011a5c13..42e3b09382 100644
--- a/tensorflow/python/debug/wrappers/dumping_wrapper_test.py
+++ b/tensorflow/python/debug/wrappers/dumping_wrapper_test.py
@@ -41,6 +41,7 @@ from tensorflow.python.platform import googletest
 from tensorflow.python.training import monitored_session
 
 
+@test_util.run_v1_only("b/120545219")
 class DumpingDebugWrapperSessionTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/debug/wrappers/framework_test.py b/tensorflow/python/debug/wrappers/framework_test.py
index 68584b4ede..a50fa7cf4b 100644
--- a/tensorflow/python/debug/wrappers/framework_test.py
+++ b/tensorflow/python/debug/wrappers/framework_test.py
@@ -141,6 +141,7 @@ class TestDebugWrapperSessionBadAction(framework.BaseDebugWrapperSession):
     return framework.OnRunEndResponse()
 
 
+@test_util.run_v1_only("b/120545219")
 class DebugWrapperSessionTest(test_util.TensorFlowTestCase):
 
   def _no_rewrite_session_config(self):
diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py b/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py
index 149a7497df..e38df861f5 100644
--- a/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py
+++ b/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py
@@ -127,6 +127,7 @@ class LocalCLIDebuggerWrapperSessionForTest(
         return e.exit_token
 
 
+@test_util.run_v1_only("b/120545219")
 class LocalCLIDebugWrapperSessionTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/distribute/distribute_coordinator_test.py b/tensorflow/python/distribute/distribute_coordinator_test.py
index f2cb950aad..7598c105c2 100644
--- a/tensorflow/python/distribute/distribute_coordinator_test.py
+++ b/tensorflow/python/distribute/distribute_coordinator_test.py
@@ -427,6 +427,7 @@ class DistributeCoordinatorTestStandaloneMode(DistributeCoordinatorTestBase):
     # Each finished worker will increment self._result_correct.
     self.assertEqual(self._result_correct, NUM_WORKERS)
 
+  @test_util.run_v1_only("b/120545219")
   def testBetweenGraphWithMonitoredSession(self):
     """Test monitored session in standalone client mode."""
     distribute_coordinator.run_distribute_coordinator(
@@ -600,6 +601,7 @@ class DistributeCoordinatorTestInpendentWorkerMode(
     # Each finished worker will increment self._result_correct.
     self.assertEqual(self._result_correct, NUM_WORKERS)
 
+  @test_util.run_v1_only("b/120545219")
   def testBetweenGraphWithMonitoredSession(self):
     cluster_spec = self._create_cluster_spec(
         num_workers=NUM_WORKERS, num_ps=NUM_PS)
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 3cec40a48f..61c47a29fd 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -648,6 +648,7 @@ class BackpropTest(test.TestCase):
       g.gradient(x, y)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only('b/120545219')
   def testGradientTapeWithCond(self):
     x = constant_op.constant(3.0)
 
@@ -669,6 +670,7 @@ class BackpropTest(test.TestCase):
       self.assertEqual(self.evaluate(dy), 6.0)
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only('b/120545219')
   def testGradientTapeWithWhileLoop(self):
     i = constant_op.constant(1)
     x = constant_op.constant(2.)
@@ -704,6 +706,7 @@ class BackpropTest(test.TestCase):
 
   @test_util.assert_no_new_tensors
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only('b/120545219')
   def testPersistentTape(self):
     with backprop.GradientTape(persistent=True) as g:
       x = constant_op.constant(3.0)
@@ -1243,16 +1246,19 @@ class JacobianTest(test.TestCase):
     answer = [array_ops.diag(2 * x * y), array_ops.diag(x * x)]
     return jacobian, answer
 
+  @test_util.run_v1_only('b/120545219')
   def testPfor(self):
     jacobian, answer = self._jacobian(experimental_use_pfor=True)
     for j, a in zip(jacobian, answer):
       self.assertAllEqual(a, j)
 
+  @test_util.run_v1_only('b/120545219')
   def testWhileLoop(self):
     jacobian, answer = self._jacobian(experimental_use_pfor=False)
     for j, a in zip(jacobian, answer):
       self.assertAllEqual(a, j)
 
+  @test_util.run_v1_only('b/120545219')
   def testPforDefun(self):
 
     @function.defun
@@ -1263,6 +1269,7 @@ class JacobianTest(test.TestCase):
     for j, a in zip(jacobian, answer):
       self.assertAllEqual(a, j)
 
+  @test_util.run_v1_only('b/120545219')
   def testWhileLoopDefun(self):
 
     @function.defun
@@ -1273,6 +1280,7 @@ class JacobianTest(test.TestCase):
     for j, a in zip(jacobian, answer):
       self.assertAllEqual(a, j)
 
+  @test_util.run_v1_only('b/120545219')
   def testPersistentTape(self):
     if not context.executing_eagerly():
       return
@@ -1283,6 +1291,7 @@ class JacobianTest(test.TestCase):
     with self.assertRaisesRegexp(RuntimeError, 'persistent'):
       g.jacobian(y, x, experimental_use_pfor=False)
 
+  @test_util.run_v1_only('b/120545219')
   def testPforException(self):
     var = variables.Variable([1.])
 
@@ -1303,6 +1312,7 @@ class JacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'No converter'):
       g.jacobian(y, x, experimental_use_pfor=True)
 
+  @test_util.run_v1_only('b/120545219')
   def test_parallel_iterations(self):
     with backprop.GradientTape(persistent=True) as g:
       x = constant_op.constant([[1., 2], [3, 4]])
@@ -1328,14 +1338,17 @@ class BatchJacobianTest(test.TestCase):
                               array_ops.diag(2 * x[1] * y[1])])
     return batch_jacobian, answer
 
+  @test_util.run_v1_only('b/120545219')
   def testPfor(self):
     batch_jacobian, answer = self._batch_jacobian(experimental_use_pfor=True)
     self.assertAllEqual(answer, batch_jacobian)
 
+  @test_util.run_v1_only('b/120545219')
   def testWhileLoop(self):
     batch_jacobian, answer = self._batch_jacobian(experimental_use_pfor=False)
     self.assertAllEqual(answer, batch_jacobian)
 
+  @test_util.run_v1_only('b/120545219')
   def testPforDefun(self):
 
     @function.defun
@@ -1345,6 +1358,7 @@ class BatchJacobianTest(test.TestCase):
     batch_jacobian, answer = _f()
     self.assertAllEqual(answer, batch_jacobian)
 
+  @test_util.run_v1_only('b/120545219')
   def testWhileLoopDefun(self):
 
     @function.defun
@@ -1354,6 +1368,7 @@ class BatchJacobianTest(test.TestCase):
     batch_jacobian, answer = _f()
     self.assertAllEqual(answer, batch_jacobian)
 
+  @test_util.run_v1_only('b/120545219')
   def testPersistentTape(self):
     if not context.executing_eagerly():
       return
@@ -1364,6 +1379,7 @@ class BatchJacobianTest(test.TestCase):
     with self.assertRaisesRegexp(RuntimeError, 'persistent'):
       g.batch_jacobian(y, x, experimental_use_pfor=False)
 
+  @test_util.run_v1_only('b/120545219')
   def testBadShape(self):
     x = random_ops.random_uniform([2, 3])
     with backprop.GradientTape() as g:
@@ -1371,6 +1387,7 @@ class BatchJacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'Need first dimension'):
       g.batch_jacobian(y, x)
 
+  @test_util.run_v1_only('b/120545219')
   def testBadInputRank(self):
     x = random_ops.random_uniform([2])
     with backprop.GradientTape() as g:
@@ -1385,6 +1402,7 @@ class BatchJacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'must have rank at least 2'):
       g.batch_jacobian(y, x)
 
+  @test_util.run_v1_only('b/120545219')
   def testPforException(self):
     var = variables.Variable([1.])
 
@@ -1405,6 +1423,7 @@ class BatchJacobianTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'No converter'):
       g.batch_jacobian(y, x, experimental_use_pfor=True)
 
+  @test_util.run_v1_only('b/120545219')
   def test_parallel_iterations(self):
     with backprop.GradientTape(persistent=True) as g:
       x = constant_op.constant([[1., 2], [3, 4]])
diff --git a/tensorflow/python/eager/function_gradients_test.py b/tensorflow/python/eager/function_gradients_test.py
index 9b83f57089..98dec0b361 100644
--- a/tensorflow/python/eager/function_gradients_test.py
+++ b/tensorflow/python/eager/function_gradients_test.py
@@ -187,7 +187,7 @@ class FunctionGradientsTest(test.TestCase, parameterized.TestCase):
 
     self.assertAllEqual(2, g(constant_op.constant(2.)))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testGraphModeEagerGradError(self):
     with context.graph_mode():
       def f():
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index e0854b0632..50d1b4b6f7 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -963,6 +963,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
   # construction. Eager's configuration is controlled in `__main__`.
   @test_util.run_in_graph_and_eager_modes(
       config=config_pb2.ConfigProto(device_count={'CPU': 4}))
+  @test_util.run_v1_only('b/120545219')
   def testDeviceAnnotationsRespected(self):
 
     def multi_device_fn():
@@ -1001,6 +1002,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
 
   @test_util.run_in_graph_and_eager_modes(
       config=config_pb2.ConfigProto(device_count={'CPU': 2}))
+  @test_util.run_v1_only('b/120545219')
   def testCallingGraphFunctionOnDifferentDevice(self):
 
     def func():
diff --git a/tensorflow/python/framework/auto_control_deps_test.py b/tensorflow/python/framework/auto_control_deps_test.py
index a1dff9e834..5f5de45b9e 100644
--- a/tensorflow/python/framework/auto_control_deps_test.py
+++ b/tensorflow/python/framework/auto_control_deps_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import auto_control_deps as acd
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -46,6 +47,7 @@ class AutomaticControlDependenciesTest(test.TestCase):
         val = c.mark_as_return(val)
       self.assertAllEqual(val.eval(), 4.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondMustRun(self):
     with context.graph_mode(), self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -67,6 +69,7 @@ class AutomaticControlDependenciesTest(test.TestCase):
       self.assertAllEqual(val.eval(feed_dict={p: False}), 5.0)
       self.assertAllEqual(val.eval(feed_dict={p: True}), 6.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondMustRunSeparateRead(self):
     with context.graph_mode(), self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -90,6 +93,7 @@ class AutomaticControlDependenciesTest(test.TestCase):
       one.eval(feed_dict={p: True})
       self.assertAllEqual(v.read_value().eval(), 6.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondNested(self):
     with context.graph_mode(), self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -124,6 +128,7 @@ class AutomaticControlDependenciesTest(test.TestCase):
       self.assertAllEqual(val.eval(feed_dict={p: True, q: True}), 7.0)
       self.assertAllEqual(val.eval(feed_dict={p: True, q: False}), 8.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondOneBranch(self):
     with context.graph_mode(), self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -144,6 +149,7 @@ class AutomaticControlDependenciesTest(test.TestCase):
       self.assertAllEqual(val.eval(feed_dict={p: False}), 5.0)
       self.assertAllEqual(val.eval(feed_dict={p: True}), 5.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondOneBranchUpdateBefore(self):
     with context.graph_mode(), self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
@@ -165,6 +171,7 @@ class AutomaticControlDependenciesTest(test.TestCase):
       self.assertAllEqual(val.eval(feed_dict={p: False}), 6.0)
       self.assertAllEqual(val.eval(feed_dict={p: True}), 12.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondOneBranchUpdateAfter(self):
     with context.graph_mode(), self.cached_session():
       v = resource_variable_ops.ResourceVariable(1.0)
diff --git a/tensorflow/python/framework/error_interpolation_test.py b/tensorflow/python/framework/error_interpolation_test.py
index 1b77548592..9eaa4a5f2d 100644
--- a/tensorflow/python/framework/error_interpolation_test.py
+++ b/tensorflow/python/framework/error_interpolation_test.py
@@ -23,6 +23,7 @@ import os
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import error_interpolation
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import traceable_stack
 from tensorflow.python.platform import test
 from tensorflow.python.util import tf_stack
@@ -112,6 +113,7 @@ class ComputeColocationSummaryFromOpTest(test.TestCase):
     self.assertIn("No node-device colocations", summary)
 
 
+@test_util.run_v1_only("b/120545219")
 class InterpolateFilenamesAndLineNumbersTest(test.TestCase):
 
   def setUp(self):
@@ -193,6 +195,7 @@ class InterpolateFilenamesAndLineNumbersTest(test.TestCase):
     self.assertRegexpMatches(interpolated_string, "constant_op.py:[0-9]+.*")
 
 
+@test_util.run_v1_only("b/120545219")
 class InterpolateDeviceSummaryTest(test.TestCase):
 
   def _fancy_device_function(self, unused_op):
@@ -236,6 +239,7 @@ class InterpolateDeviceSummaryTest(test.TestCase):
     self.assertRegexpMatches(result, expected_re)
 
 
+@test_util.run_v1_only("b/120545219")
 class InterpolateColocationSummaryTest(test.TestCase):
 
   def setUp(self):
@@ -260,11 +264,13 @@ class InterpolateColocationSummaryTest(test.TestCase):
 
     self.graph = node_three.graph
 
+  @test_util.run_v1_only("b/120545219")
   def testNodeThreeHasColocationInterpolation(self):
     message = "{{colocation_node Three_with_one}}"
     result = error_interpolation.interpolate(message, self.graph)
     self.assertIn("colocate_with(One)", result)
 
+  @test_util.run_v1_only("b/120545219")
   def testNodeFourHasColocationInterpolationForNodeThreeOnly(self):
     message = "{{colocation_node Four_with_three}}"
     result = error_interpolation.interpolate(message, self.graph)
@@ -273,12 +279,14 @@ class InterpolateColocationSummaryTest(test.TestCase):
         "One", result,
         "Node One should not appear in Four_with_three's summary:\n%s" % result)
 
+  @test_util.run_v1_only("b/120545219")
   def testNodeFiveHasColocationInterpolationForNodeOneAndTwo(self):
     message = "{{colocation_node Five_with_one_with_two}}"
     result = error_interpolation.interpolate(message, self.graph)
     self.assertIn("colocate_with(One)", result)
     self.assertIn("colocate_with(Two)", result)
 
+  @test_util.run_v1_only("b/120545219")
   def testColocationInterpolationForNodeLackingColocation(self):
     message = "{{colocation_node One}}"
     result = error_interpolation.interpolate(message, self.graph)
diff --git a/tensorflow/python/framework/graph_util_test.py b/tensorflow/python/framework/graph_util_test.py
index 4e7408ad49..dd26b8a78e 100644
--- a/tensorflow/python/framework/graph_util_test.py
+++ b/tensorflow/python/framework/graph_util_test.py
@@ -103,7 +103,7 @@ class DeviceFunctionsTest(test.TestCase):
     self.assertDeviceEqual(var_5.device, "/device:GPU:0")
     self.assertDeviceEqual(var_6.device, "/device:CPU:0")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNestedDeviceFunctions(self):
     with ops.Graph().as_default():
       var_0 = variables.VariableV1(0)
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 46ce4616a5..e6e8788164 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -528,7 +528,7 @@ class ScopedMetaGraphTest(test.TestCase):
         actual_grad_value = self.evaluate(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testImportWhileLoopInWhileLoop(self):
     # Create a simple while loop.
     with ops.Graph().as_default():
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 7baa02b446..0fcbcd6ee4 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -605,6 +605,7 @@ class OperationTest(test_util.TensorFlowTestCase):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
   @test_util.enable_control_flow_v2
+  @test_util.run_v1_only("b/120545219")
   def testAddWhileInput(self):
     @eager_function.defun
     def test():
@@ -780,7 +781,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(op3.name, "myop_2")
     self.assertEqual(op4.name, "myop_1_1")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCond(self):
     g = ops.Graph()
     with g.as_default():
@@ -810,7 +811,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
                      "cond/cond_text")
     # pylint: enable=protected-access
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileLoop(self):
     g = ops.Graph()
     with g.as_default():
@@ -840,7 +841,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
                      "myloop/while_context")
     # pylint: enable=protected-access
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileLoopWithInternalControlDep(self):
     g = ops.Graph()
     with g.as_default():
@@ -864,7 +865,7 @@ class CreateOpFromTFOperationTest(test_util.TensorFlowTestCase):
     # Internal control dep is preserved
     self.assertEqual(op.control_inputs, [c])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileLoopWithExternalControlDep(self):
     g = ops.Graph()
     with g.as_default():
@@ -2283,7 +2284,7 @@ class InitScopeTest(test_util.TensorFlowTestCase):
       self.assertEqual(4, int(compiled_outer(inner=compiled_inner)))
       self.assertEqual(7, int(compiled_outer(inner=compiled_inner)))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testFallsBackToGlobalGraphWhenAllGraphsAreBuildingFunctions(self):
     with context.graph_mode():
       ops.reset_default_graph()
@@ -2994,7 +2995,7 @@ class TracebackTest(test_util.TensorFlowTestCase):
 
 class EnableEagerExecutionTest(test_util.TensorFlowTestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBadArgumentsToEnableEagerExecution(self):
     with self.assertRaisesRegexp(TypeError, "config must be a tf.ConfigProto"):
       ops.enable_eager_execution(context.DEVICE_PLACEMENT_SILENT)
diff --git a/tensorflow/python/framework/subscribe_test.py b/tensorflow/python/framework/subscribe_test.py
index 61c6ea6519..a74e96f9d9 100644
--- a/tensorflow/python/framework/subscribe_test.py
+++ b/tensorflow/python/framework/subscribe_test.py
@@ -215,7 +215,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertIn('graph2', shared)
     self.assertIn('graph3', shared)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testSubscribeVariable(self):
     """Confirm that variables can be subscribed."""
     v1 = variables.VariableV1(0.0)
@@ -254,7 +254,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
       # Make sure the values read from the variable match the expected ones.
       self.assertEqual([0.0, 3.0], shared)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testResourceType(self):
     """Confirm that subscribe correctly handles tensors with 'resource' type."""
     tensor_array = tensor_array_ops.TensorArray(
@@ -344,7 +344,7 @@ class SubscribeTest(test_util.TensorFlowTestCase):
     self.assertEqual(add.device, add_sub.device)
     self.assertEqual(mul.device, mul_sub.device)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_subscribe_tensors_within_control_flow_context(self):
     """Side effect ops are added with the same control flow context."""
     c1 = constant_op.constant(10)
diff --git a/tensorflow/python/grappler/item_test.py b/tensorflow/python/grappler/item_test.py
index 78604b259c..c02fd9f55b 100644
--- a/tensorflow/python/grappler/item_test.py
+++ b/tensorflow/python/grappler/item_test.py
@@ -108,7 +108,7 @@ class ItemTest(test.TestCase):
     newest_tf_item = grappler_item.tf_item
     self.assertEqual(new_tf_item, newest_tf_item)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testColocationContraints(self):
     with ops.Graph().as_default() as g:
       c = constant_op.constant([10])
diff --git a/tensorflow/python/grappler/memory_optimizer_test.py b/tensorflow/python/grappler/memory_optimizer_test.py
index 6eb16fbd39..e2864ebb4d 100644
--- a/tensorflow/python/grappler/memory_optimizer_test.py
+++ b/tensorflow/python/grappler/memory_optimizer_test.py
@@ -62,7 +62,7 @@ class MemoryOptimizerSwapTest(test.TestCase):
     self.assertEqual(len(graph.node), graph_size)
     self.assertItemsEqual([node.name for node in graph.node], nodes)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testSimpleSwap(self):
     """Check that the swap annotations are followed."""
     a = variables.VariableV1(10, name='a')
diff --git a/tensorflow/python/grappler/tf_optimizer_test.py b/tensorflow/python/grappler/tf_optimizer_test.py
index 06ccaa813f..8186c81378 100644
--- a/tensorflow/python/grappler/tf_optimizer_test.py
+++ b/tensorflow/python/grappler/tf_optimizer_test.py
@@ -57,7 +57,7 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     self.assertEqual(len(graph.node), 1)
     self.assertItemsEqual([node.name for node in graph.node], ['d'])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testKeepNodes(self):
     g = ops.Graph()
     with g.as_default():
@@ -86,7 +86,7 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     self.assertEqual(len(optimized_graph_nodes), len(expected_nodes))
     self.assertAllInSet(optimized_graph_nodes, expected_nodes)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testLoops(self):
     g = ops.Graph()
     with g.as_default():
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 6401e39e53..af01b46fa9 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -1422,7 +1422,7 @@ class TestCTC(test.TestCase):
                 decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
       self.assertAllClose(log_prob_truth, log_prob_pred)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_ctc_batch_cost(self):
     with self.cached_session():
       label_lens = np.expand_dims(np.asarray([5, 4]), 1)
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index 6c9a382b32..4a65ade33c 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -403,7 +403,7 @@ class KerasCallbacksTest(test.TestCase):
           float(keras.backend.get_value(
               model.optimizer.lr)) - 0.01 / 4) < keras.backend.epsilon()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_ReduceLROnPlateau(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -675,7 +675,7 @@ class KerasCallbacksTest(test.TestCase):
       self.assertEqual(len(loss), 1)
       self.assertEqual(loss[0], np.inf)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_TensorBoard(self):
     np.random.seed(1337)
 
@@ -779,7 +779,7 @@ class KerasCallbacksTest(test.TestCase):
           data_generator(True), len(x_train), epochs=2, callbacks=cbks)
       assert os.path.exists(temp_dir)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_TensorBoard_multi_input_output(self):
     np.random.seed(1337)
     tmpdir = self.get_temp_dir()
@@ -851,7 +851,7 @@ class KerasCallbacksTest(test.TestCase):
                           callbacks=callbacks_factory(histogram_freq=1))
       assert os.path.isdir(filepath)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_Tensorboard_histogram_summaries_in_test_function(self):
 
     class FileWriterStub(object):
@@ -929,7 +929,7 @@ class KerasCallbacksTest(test.TestCase):
 
       self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_Tensorboard_histogram_summaries_with_generator(self):
     np.random.seed(1337)
     tmpdir = self.get_temp_dir()
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py
index 6d9d9a2fca..bc33a3ea7f 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/engine/saving_test.py
@@ -332,7 +332,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
 
 class TestWholeModelSaving(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_sequential_model_saving(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -635,7 +635,7 @@ class TestWholeModelSaving(test.TestCase):
       os.close(fd)
       os.remove(fname)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_saving_model_with_long_weights_names(self):
     if h5py is None:
       self.skipTest('h5py required to run this test')
@@ -756,7 +756,7 @@ class SubclassedModel(training.Model):
 
 class TestWeightSavingAndLoadingTFFormat(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_keras_optimizer_warning(self):
     graph = ops.Graph()
     with graph.as_default(), self.session(graph):
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 001c3d749b..10f69da061 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -226,7 +226,7 @@ class TestSequential(keras_parameterized.TestCase):
     inner_model.trainable = True
     self.assertEqual(len(model.trainable_weights), 4)
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_sequential_update_disabling(self):
     val_a = np.random.random((10, 4))
     val_out = np.random.random((10, 4))
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 03bfd35589..4071e2c091 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -107,6 +107,7 @@ class TopologyConstructionTest(test.TestCase):
     self.assertEqual(len(network.updates), 5)
     self.assertEqual(len(network.get_updates_for(x4)), 2)
 
+  @test_util.run_v1_only('b/120545219')
   def test_get_updates_bn(self):
     x1 = input_layer_lib.Input(shape=(1,))
     layer = keras.layers.BatchNormalization()
@@ -833,7 +834,7 @@ class TopologyConstructionTest(test.TestCase):
       output_val_2 = m2.predict(x_val)
       self.assertAllClose(output_val, output_val_2, atol=1e-6)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_explicit_training_argument(self):
     with self.cached_session():
       a = keras.layers.Input(shape=(2,))
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index c3c3f06ffd..91a0c7cc2f 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -471,7 +471,7 @@ class TrainingTest(keras_parameterized.TestCase):
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_that_trainable_disables_updates(self):
     val_a = np.random.random((10, 4))
     val_out = np.random.random((10, 4))
@@ -864,6 +864,7 @@ class LossWeightingTest(keras_parameterized.TestCase):
     self.assertLess(score[0], ref_score[0])
 
   @keras_parameterized.run_all_keras_modes
+  @tf_test_util.run_v1_only('b/120545219')
   def test_sample_weights(self):
     num_classes = 5
     batch_size = 5
@@ -961,6 +962,7 @@ class LossWeightingTest(keras_parameterized.TestCase):
       self.assertTrue(msg_found)
 
   @keras_parameterized.run_all_keras_modes
+  @tf_test_util.run_v1_only('b/120545219')
   # TODO(b/120562577): Test failing with assertion error.
   def DISABLED_test_temporal_sample_weights(self):
     num_classes = 5
@@ -1283,7 +1285,7 @@ class LossMaskingTest(keras_parameterized.TestCase):
 
 class TestDynamicTrainability(keras_parameterized.TestCase):
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_trainable_warning(self):
     with self.cached_session():
       x = np.random.random((5, 3))
@@ -1297,7 +1299,7 @@ class TestDynamicTrainability(keras_parameterized.TestCase):
       model.train_on_batch(x, y)
       self.assertRaises(Warning)
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_trainable_argument(self):
     with self.cached_session():
       x = np.random.random((5, 3))
diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index f1a0932613..c516514f63 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -35,7 +35,7 @@ class KerasIntegrationTest(test.TestCase):
   def test_version(self):
     self.assertTrue(keras.__version__.endswith('-tf'))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_vector_classification_sequential(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -134,6 +134,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
+  @test_util.run_v1_only('b/120545219')
   def test_image_classification_sequential(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -168,7 +169,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_video_classification_functional(self):
     with self.cached_session():
       np.random.seed(1337)
@@ -197,7 +198,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_vector_classification_shared_sequential(self):
     # Test that Sequential models that feature internal updates
     # and internal losses can be shared.
@@ -232,7 +233,7 @@ class KerasIntegrationTest(test.TestCase):
                           verbose=2)
       self.assertGreater(history.history['val_acc'][-1], 0.7)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_vector_classification_shared_model(self):
     # Test that functional models that feature internal updates
     # and internal losses can be shared.
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index 3f89cc398e..aea4261502 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -115,7 +115,7 @@ class LSTMLayerTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
     self.assertEqual(layer.cell.bias.constraint, b_constraint)
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_with_masking_layer_LSTM(self):
     layer_class = keras.layers.LSTM
     inputs = np.random.random((2, 3, 4))
@@ -128,7 +128,7 @@ class LSTMLayerTest(test.TestCase, parameterized.TestCase):
                   optimizer=RMSPropOptimizer(0.01))
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_masking_with_stacking_LSTM(self):
     inputs = np.random.random((2, 3, 4))
     targets = np.abs(np.random.random((2, 3, 5)))
@@ -314,7 +314,7 @@ class LSTMLayerTest(test.TestCase, parameterized.TestCase):
 
 class LSTMLayerGraphOnlyTest(test.TestCase):
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_statefulness_LSTM(self):
     num_samples = 2
     timesteps = 3
diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index 9138c0a08a..c1acc2eb3a 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.training import gradient_descent
 
 
 @tf_test_util.run_all_in_graph_and_eager_modes
+@tf_test_util.run_v1_only('b/120545219')
 class NormalizationLayersTest(test.TestCase):
 
   def test_basic_batchnorm(self):
@@ -227,6 +228,7 @@ class NormalizationLayersTest(test.TestCase):
       norm(inp)
 
 
+@tf_test_util.run_v1_only('b/120545219')
 class NormalizationLayersGraphModeOnlyTest(test.TestCase):
 
   def test_shared_batchnorm(self):
@@ -301,7 +303,6 @@ class NormalizationLayersGraphModeOnlyTest(test.TestCase):
       x2 = model.predict(val_a)
       self.assertAllClose(x1, x2, atol=1e-7)
 
-  @tf_test_util.run_deprecated_v1
   def test_batchnorm_trainable(self):
     """Tests that batchnorm layer is trainable when learning phase is enabled.
 
diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py
index b49b159b71..bb3fea2692 100644
--- a/tensorflow/python/keras/layers/simplernn_test.py
+++ b/tensorflow/python/keras/layers/simplernn_test.py
@@ -98,7 +98,7 @@ class SimpleRNNLayerTest(test.TestCase):
     self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
     self.assertEqual(layer.cell.bias.constraint, b_constraint)
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_with_masking_layer_SimpleRNN(self):
     layer_class = keras.layers.SimpleRNN
     inputs = np.random.random((2, 3, 4))
@@ -121,7 +121,7 @@ class SimpleRNNLayerTest(test.TestCase):
 
 class SimpleRNNLayerGraphOnlyTest(test.TestCase):
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_statefulness_SimpleRNN(self):
     num_samples = 2
     timesteps = 3
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 33351948ee..932b2d331d 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -56,9 +56,9 @@ _graph_options = config_pb2.GraphOptions(rewrite_options=_rewrites)
 _config = config_pb2.ConfigProto(graph_options=_graph_options)
 
 
+@test_util.run_v1_only('b/120545219')
 class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
 
-  @test_util.run_deprecated_v1
   def test_unifiedLSTM(self):
     input_shape = 10
     rnn_state_size = 8
@@ -103,7 +103,6 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
         self.assertNotEqual(existing_loss, loss_value)
         existing_loss = loss_value
 
-  @test_util.run_deprecated_v1
   def test_unifiedLSTM_with_cond(self):
     # This test is to demonstrate the graph rewrite of grappler plugin under
     # the condition that the function returns different number of internal
@@ -692,6 +691,7 @@ class UnifiedLSTMTest(test.TestCase, parameterized.TestCase):
     model.train_on_batch([main_inputs] + initial_state, targets)
 
 
+@test_util.run_v1_only('b/120545219')
 class LSTMLayerGraphOnlyTest(test.TestCase):
 
   def test_statefulness_LSTM(self):
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index b9196416dd..727f33dadc 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -165,6 +165,7 @@ class TimeDistributedTest(test.TestCase):
       y = model.predict(np.random.random((10, 3, 2)))
       self.assertAllClose(np.mean(y), 0., atol=1e-1, rtol=1e-1)
 
+  @tf_test_util.run_v1_only('b/120545219')
   def test_TimeDistributed_batchnorm(self):
     with self.cached_session():
       # test that wrapped BN updates still work.
@@ -187,6 +188,7 @@ class TimeDistributedTest(test.TestCase):
       # Verify input_map has one mapping from inputs to reshaped inputs.
       self.assertEqual(len(td._input_map.keys()), 1)
 
+  @tf_test_util.run_v1_only('b/120545219')
   def test_TimeDistributed_trainable(self):
     # test layers that need learning_phase to be set
     x = keras.layers.Input(shape=(3, 2))
@@ -201,7 +203,7 @@ class TimeDistributedTest(test.TestCase):
     assert len(layer.updates) == 2
     assert len(layer.trainable_weights) == 2
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_TimeDistributed_with_masked_embedding_and_unspecified_shape(self):
     with self.cached_session():
       # test with unspecified shape and Embeddings with mask_zero
@@ -234,7 +236,7 @@ class TimeDistributedTest(test.TestCase):
         self.assertAllEqual(mask_outputs_val[i], ref_mask_val[i])
       self.assertIs(mask_outputs[-1], None)  # final layer
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_TimeDistributed_with_masking_layer(self):
     with self.cached_session():
       # test with Masking layer
@@ -377,7 +379,7 @@ class BidirectionalTest(test.TestCase):
       model.compile(loss='mse', optimizer='sgd')
       model.fit(x, y, epochs=1, batch_size=1)
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_Bidirectional_merged_value(self):
     rnn = keras.layers.LSTM
     samples = 2
@@ -508,7 +510,7 @@ class BidirectionalTest(test.TestCase):
       layer.trainable = True
       assert len(layer.trainable_weights) == 6
 
-  @tf_test_util.run_deprecated_v1
+  @tf_test_util.run_v1_only('b/120545219')
   def test_Bidirectional_updates(self):
     with self.cached_session():
       x = keras.layers.Input(shape=(3, 2))
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index 620275e50f..553c7fb009 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -187,6 +187,7 @@ def get_nested_model_3(input_dim, num_classes):
 
 
 @test_util.run_all_in_graph_and_eager_modes
+@test_util.run_v1_only('b/120545219')
 class ModelSubclassingTest(test.TestCase):
 
   def test_custom_build(self):
@@ -915,6 +916,7 @@ class ModelSubclassingTest(test.TestCase):
       self.assertEqual(1, len(model.get_updates_for(x)))
 
 
+@test_util.run_v1_only('b/120545219')
 class GraphSpecificModelSubclassingTests(test.TestCase):
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 907ac41d0e..c466d94fed 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -69,7 +69,7 @@ def sequential_model(add_input_layer, include_input_shape=True):
 
 class TestModelCloning(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_clone_sequential_model(self):
     with self.cached_session():
       val_a = np.random.random((10, 4))
@@ -102,10 +102,9 @@ class TestModelCloning(test.TestCase):
       new_model.compile('rmsprop', 'mse')
       new_model.train_on_batch(None, val_out)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_clone_sequential_model_input_layer(self):
 
-    @test_util.run_deprecated_v1
     def test_input_layer(include_inputs):
       with self.cached_session():
         val_a = np.random.random((10, 4))
@@ -142,7 +141,7 @@ class TestModelCloning(test.TestCase):
     test_input_layer(True)
     test_input_layer(False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_clone_functional_model(self):
     with self.cached_session():
       val_a = np.random.random((10, 4))
@@ -318,6 +317,7 @@ class TestModelDeepCopy(test.TestCase):
                       model_copy.get_weights()[0]))
 
 
+@test_util.run_v1_only('b/120545219')
 class TestCloneAndBuildModel(test.TestCase):
 
   def test_clone_and_build_non_compiled_model(self):
@@ -404,7 +404,6 @@ class TestCloneAndBuildModel(test.TestCase):
       new_model.train_on_batch(inp, out)
       new_model.evaluate(inp, out)
 
-  @test_util.run_deprecated_v1
   def test_clone_and_build_compiled_sequential_model(self):
     with self.cached_session():
       model = keras.models.Sequential()
@@ -417,7 +416,6 @@ class TestCloneAndBuildModel(test.TestCase):
 
     self._clone_and_build_test_helper(model)
 
-  @test_util.run_deprecated_v1
   def test_clone_and_build_functional_model(self):
     with self.cached_session():
       input_a = keras.Input(shape=(4,))
@@ -434,7 +432,6 @@ class TestCloneAndBuildModel(test.TestCase):
 
     self._clone_and_build_test_helper(model)
 
-  @test_util.run_deprecated_v1
   def test_clone_and_build_subclassed_model(self):
     class SubclassedModel(keras.Model):
 
@@ -483,11 +480,9 @@ class TestCloneAndBuildModel(test.TestCase):
   def test_replace_tf_optimizer_iterations_variable(self):
     self.assert_optimizer_iterations_increases(adam.AdamOptimizer(0.01))
 
-  @test_util.run_deprecated_v1
   def test_replace_keras_optimizer_iterations_variable(self):
     self.assert_optimizer_iterations_increases('adam')
 
-  @test_util.run_deprecated_v1
   def test_clone_and_build_sequential_model_without_inputs_defined(self):
     with self.cached_session():
       model = sequential_model(False, False)
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index d3cacb702c..77104a5d4d 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -91,26 +91,26 @@ def _test_optimizer(optimizer, target=0.75):
 
 class KerasOptimizersTest(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_sgd(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.SGD(lr=0.01,
                                            momentum=0.9,
                                            nesterov=True))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_rmsprop(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.RMSprop())
       _test_optimizer(keras.optimizers.RMSprop(decay=1e-3))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_adagrad(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adagrad())
       _test_optimizer(keras.optimizers.Adagrad(decay=1e-3))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_adadelta(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adadelta(), target=0.6)
@@ -119,32 +119,32 @@ class KerasOptimizersTest(test.TestCase):
       # the accuracy.
       _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_adam(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adam())
       _test_optimizer(keras.optimizers.Adam(decay=1e-3))
       _test_optimizer(keras.optimizers.Adam(amsgrad=True))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_adamax(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Adamax())
       _test_optimizer(keras.optimizers.Adamax(decay=1e-3))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_nadam(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.Nadam())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_clipnorm(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.SGD(lr=0.01,
                                            momentum=0.9,
                                            clipnorm=0.5))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def test_clipvalue(self):
     with self.cached_session():
       _test_optimizer(keras.optimizers.SGD(lr=0.01,
diff --git a/tensorflow/python/kernel_tests/atrous_convolution_test.py b/tensorflow/python/kernel_tests/atrous_convolution_test.py
index 6b16fca29d..2fb8a37e2b 100644
--- a/tensorflow/python/kernel_tests/atrous_convolution_test.py
+++ b/tensorflow/python/kernel_tests/atrous_convolution_test.py
@@ -110,6 +110,7 @@ class AtrousConvolutionTest(test.TestCase):
 
     add_check(check, y1, y2)
 
+  @test_util.run_v1_only("b/120545219")
   def test_unknown_spatial_dims_for_channel_last_format(self):
     x = array_ops.placeholder(dtypes.float32, [1, None, None, 10])
     w = array_ops.zeros([3, 3, 10, 20])
@@ -117,6 +118,7 @@ class AtrousConvolutionTest(test.TestCase):
         x, w, "VALID", dilation_rate=[2, 2], data_format="NHWC")
     self.assertEqual(y.shape.as_list(), [1, None, None, 20])
 
+  @test_util.run_v1_only("b/120545219")
   def test_unknown_spatial_dims_for_channel_first_format(self):
     x = array_ops.placeholder(dtypes.float32, [1, 10, None, None])
     w = array_ops.zeros([3, 3, 10, 20])
@@ -262,6 +264,7 @@ class AtrousConvolutionTest(test.TestCase):
     err_tolerance = 1e-3
     self.assertLess(err, err_tolerance)
 
+  @test_util.run_v1_only("b/120545219")
   def testGradient(self):
     with self.cached_session():
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/base64_ops_test.py b/tensorflow/python/kernel_tests/base64_ops_test.py
index bb903d827f..381f190b8d 100644
--- a/tensorflow/python/kernel_tests/base64_ops_test.py
+++ b/tensorflow/python/kernel_tests/base64_ops_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("b/120545219")
 class Base64OpsTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
index f2f0291b89..c32a6c7e41 100644
--- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py
+++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python import tf2
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import math_ops
@@ -135,6 +136,7 @@ class BatchMatmulOpTest(test.TestCase):
 
 def _GetBatchMatmulOpTest(dtype, adjoint_a, adjoint_b, use_static_shape):
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     np.random.seed(42)
     self._testNonEmpty(dtype, adjoint_a, adjoint_b, use_static_shape)
@@ -184,6 +186,7 @@ class BatchMatmulGradientTest(test.TestCase):
 
 def _GetBatchMatmulGradientTest(dtype, adjoint_a, adjoint_b):
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     self._compare(1, 2, 3, 5, dtype, adjoint_a, adjoint_b)
     self._compare(3, 4, 7, 10, dtype, adjoint_a, adjoint_b)
diff --git a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
index 390672febe..2b9863fb89 100644
--- a/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
+++ b/tensorflow/python/kernel_tests/boosted_trees/quantile_ops_test.py
@@ -82,7 +82,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
     self.max_elements = 1 << 16
     self.num_quantiles = constant_op.constant(3, dtype=dtypes.int64)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBasicQuantileBucketsSingleResource(self):
     with self.cached_session() as sess:
       quantile_accumulator_handle = self.create_resource("floats", self.eps,
@@ -107,7 +107,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval())
       self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBasicQuantileBucketsMultipleResources(self):
     with self.cached_session() as sess:
       quantile_accumulator_handle_0 = self.create_resource("float_0", self.eps,
@@ -142,7 +142,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_quantiles, quantiles[0].eval())
       self.assertAllClose(self._feature_1_quantiles, quantiles[1].eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSaveRestoreAfterFlush(self):
     save_dir = os.path.join(self.get_temp_dir(), "save_restore")
     save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
@@ -175,7 +175,7 @@ class QuantileOpsTest(test_util.TensorFlowTestCase):
       self.assertAllClose(self._feature_0_boundaries, buckets[0].eval())
       self.assertAllClose(self._feature_1_boundaries, buckets[1].eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSaveRestoreBeforeFlush(self):
     save_dir = os.path.join(self.get_temp_dir(), "save_restore")
     save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
diff --git a/tensorflow/python/kernel_tests/checkpoint_ops_test.py b/tensorflow/python/kernel_tests/checkpoint_ops_test.py
index b8c8c9edb5..6e289bf9b7 100644
--- a/tensorflow/python/kernel_tests/checkpoint_ops_test.py
+++ b/tensorflow/python/kernel_tests/checkpoint_ops_test.py
@@ -105,6 +105,7 @@ class GenerateVocabRemappingTest(test.TestCase):
       self.assertAllEqual(expected_num_present, self.evaluate(num_present))
 
 
+@test_util.run_v1_only('b/120545219')
 class LoadAndRemapMatrixTest(test.TestCase):
   """Tests for the load_and_remap_matrix() op."""
 
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index f3947236b1..a08cfe960d 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -155,6 +155,7 @@ class CholeskyOpTest(test.TestCase):
           np.array([[[1., 2., 3.], [3., 4., 5.]], [[1., 2., 3.], [3., 4., 5.]]
                    ]))
 
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     tensor3 = constant_op.constant([1., 2.])
     with self.assertRaises(ValueError):
@@ -233,6 +234,7 @@ class CholeskyGradTest(test.TestCase):
     self.runFiniteDifferences(
         shapes, dtypes=(dtypes_lib.float64,), scalarTest=True)
 
+  @test_util.run_v1_only("b/120545219")
   def testTwoBlockMatrixComplexFloat(self):
     np.random.seed(0)
     shapes = self.getShapes([2 * self._backprop_block_size + 1])
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 843d007cc8..8fe3ba41e2 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -170,6 +170,7 @@ class CondV2Test(test.TestCase):
         self.assertRegexpMatches(
             cond2_op.get_attr("else_branch").name, r"foo_cond_1_false_\d*")
 
+  @test_util.run_v1_only("b/120545219")
   def testDefunInCond(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
@@ -189,6 +190,7 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [x, y])
     self._testCond(true_fn, false_fn, [y])
 
+  @test_util.run_deprecated_v1
   def testNestedDefunInCond(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
@@ -213,6 +215,7 @@ class CondV2Test(test.TestCase):
     self._testCond(true_fn, false_fn, [x, y])
     self._testCond(true_fn, false_fn, [y])
 
+  @test_util.run_deprecated_v1
   def testDoubleNestedDefunInCond(self):
     x = constant_op.constant(1.0, name="x")
     y = constant_op.constant(2.0, name="y")
@@ -773,6 +776,7 @@ class CondV2Test(test.TestCase):
     self.assertAllEqual(
         self.evaluate(output_t), [-5, -4, -3, -2, -1, 0, 1, 4, 9, 16])
 
+  @test_util.run_deprecated_v1
   def testForwardPassRewrite(self):
     x = constant_op.constant(1.0, name="x")
     output = cond_v2.cond_v2(constant_op.constant(True),
diff --git a/tensorflow/python/kernel_tests/conditional_accumulator_test.py b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
index 5847e4639b..ce34201706 100644
--- a/tensorflow/python/kernel_tests/conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/conditional_accumulator_test.py
@@ -199,7 +199,7 @@ class ConditionalAccumulatorTest(test.TestCase):
           is_all_equal &= (val[i][j] == elems_ave[i][j])
       self.assertTrue(is_all_equal)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAccumulatorWrongDynamicShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -321,7 +321,7 @@ class ConditionalAccumulatorTest(test.TestCase):
           shape=tensor_shape.TensorShape([1]),
           reduction_type="Invalid")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAccumulatorInvalidTakeGrad(self):
     with self.cached_session():
       q = data_flow_ops.ConditionalAccumulator(
@@ -435,7 +435,7 @@ class ConditionalAccumulatorTest(test.TestCase):
                                    if x >= ls) / sum(1 for x in local_steps
                                                      if x >= ls), val)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelApplyGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -461,7 +461,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
       self.assertEqual(val, sum(elems) / len(elems))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelTakeGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -494,7 +494,7 @@ class ConditionalAccumulatorTest(test.TestCase):
 
       self.assertItemsEqual(elems, results)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAccumulatorApplyAndBlockingTake(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
@@ -528,7 +528,7 @@ class ConditionalAccumulatorTest(test.TestCase):
     with self.assertRaisesOpError("was cancelled"):
       self.evaluate(takeg_op)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
       q = data_flow_ops.ConditionalAccumulator(
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 95420fc0f1..0fd293ebba 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -129,6 +129,7 @@ def isum(s, maximum_iterations=None):
 @test_util.with_control_flow_v2
 class ControlFlowTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testRefIdentity(self):
     with self.cached_session():
       v = variables.VariableV1(7)
@@ -141,7 +142,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(9, self.evaluate(v2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRefEnter(self):
     with self.cached_session():
       v = variables.VariableV1(7)
@@ -155,7 +156,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(9, self.evaluate(v3))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRefSwitch(self):
     with self.cached_session():
       v = variables.VariableV1(7)
@@ -193,6 +194,7 @@ class ControlFlowTest(test.TestCase):
           v, "frame2", is_constant=False)
       self.assertEqual(enter_v_non_constant.shape, None)
 
+  @test_util.run_v1_only("b/120545219")
   def testSwitchMergeIndexedSlices(self):
     with self.cached_session():
       values = constant_op.constant([1, 2, 3, 4, 5, 6])
@@ -207,6 +209,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(np.arange(1, 7), val)
     self.assertAllEqual(np.arange(0, 12, 2), ind)
 
+  @test_util.run_v1_only("b/120545219")
   def testSwitchDeadBranch(self):
     with self.cached_session():
       data = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
@@ -219,6 +222,7 @@ class ControlFlowTest(test.TestCase):
           lambda e: "Retval[0] does not have value" in str(e)):
         self.evaluate(dead_branch)
 
+  @test_util.run_v1_only("b/120545219")
   def testSwitchMergeLess(self):
     with self.cached_session():
       data = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
@@ -231,6 +235,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(merge_op)
     self.assertAllEqual(np.arange(1, 7), result)
 
+  @test_util.run_v1_only("b/120545219")
   def testSwitchMergeAddIdentity(self):
     with self.cached_session():
       data = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
@@ -244,6 +249,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(merge_op)
     self.assertAllEqual(np.array([x + 1 for x in [1, 2, 3, 4, 5, 6]]), result)
 
+  @test_util.run_v1_only("b/120545219")
   def testSwitchMergeAddMul(self):
     with self.cached_session():
       data = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
@@ -258,6 +264,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(merge_op)
     self.assertAllEqual(np.array([x * 5 for x in [1, 2, 3, 4, 5, 6]]), result)
 
+  @test_util.run_v1_only("b/120545219")
   def testLoop_false(self):
     with self.cached_session():
       false = ops.convert_to_tensor(False)
@@ -302,6 +309,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(exit_i)
     self.assertAllEqual(10, result)
 
+  @test_util.run_v1_only("b/120545219")
   def testLoop_2(self):
     with self.cached_session():
       zero = constant_op.constant(0)
@@ -328,6 +336,7 @@ class ControlFlowTest(test.TestCase):
       result = self.evaluate(exit_i)
     self.assertAllEqual(10, result)
 
+  @test_util.run_v1_only("b/120545219")
   def testDifferentFrame(self):
     with self.cached_session():
       data = array_ops.placeholder(dtypes.float32, shape=[])
@@ -362,6 +371,7 @@ class ControlFlowTest(test.TestCase):
         lambda: math_ops.subtract(x, 1.))
     self.assertEqual(b.shape, tensor_shape.scalar())
 
+  @test_util.run_v1_only("b/120545219")
   def testFetchable(self):
     with self.cached_session() as sess:
       x = array_ops.placeholder(dtypes.float32)
@@ -378,6 +388,7 @@ class ControlFlowTest(test.TestCase):
               sess.run(t, feed_dict={x: 3})
 
   @test_util.disable_control_flow_v2("Not relevant")
+  @test_util.run_v1_only("b/120545219")
   def testFeedable(self):
     with self.cached_session() as sess:
       c = constant_op.constant(2)
@@ -395,6 +406,7 @@ class ControlFlowTest(test.TestCase):
             with self.assertRaisesRegexp(ValueError, "may not be fed"):
               sess.run(r, feed_dict={t: 3})
 
+  @test_util.run_v1_only("b/120545219")
   def testCondIndexedSlices(self):
     with self.cached_session():
       values = constant_op.constant(10)
@@ -410,6 +422,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(11, val)
     self.assertAllEqual(0, ind)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondSparseTensor(self):
     with self.cached_session():
       values = constant_op.constant([2.0, 4.0], name="values")
@@ -427,6 +440,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual([[1], [4]], r.indices.eval())
       self.assertAllEqual(r.values.get_shape(), (2,))
 
+  @test_util.run_v1_only("b/120545219")
   def testCondResource(self):
 
     with self.cached_session():
@@ -441,6 +455,7 @@ class ControlFlowTest(test.TestCase):
 
       self.assertEqual(1.0, control_flow_ops.cond(rv, case, lambda: t).eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testCondWithTensorArrayGrad(self):
     with self.cached_session() as sess:
       with ops.device(test.gpu_device_name()):
@@ -455,6 +470,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(sess.run(g, {pred: False}), [0.0, 0.0, 0.0])
 
   @test_util.disable_control_flow_v2("b/113293074")
+  @test_util.run_v1_only("b/120545219")
   def testCondIndexedSlicesDifferentTypes(self):
     with self.cached_session():
       values = constant_op.constant(10)
@@ -472,6 +488,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(0, ind)
     self.assertTrue(ind.dtype == np.int64)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondColocation(self):
     with self.session(use_gpu=True):
       with ops.device("/cpu:0"):
@@ -576,6 +593,7 @@ class ControlFlowTest(test.TestCase):
         alive, count = body(i)
       self.assertAllEqual(4, self.evaluate(count))
 
+  @test_util.run_v1_only("b/120545219")
   def testCond_6(self):
     with self.cached_session():
       v1 = variables.Variable([7])
@@ -671,6 +689,7 @@ class ControlFlowTest(test.TestCase):
       test_result = self.evaluate(r)
       self.assertDictEqual({"a": {"c": 210}, "b": {"d": 210}}, test_result)
 
+  @test_util.run_v1_only("b/120545219")
   def testCheckNestedOutputStruct(self):
     with self.cached_session() as sess:
       x = constant_op.constant(10)
@@ -701,7 +720,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual([2.0], self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/79881896 (control deps)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCondWithControl(self):
     with self.cached_session():
       control_holder = array_ops.placeholder(dtypes.float32, shape=())
@@ -717,6 +736,7 @@ class ControlFlowTest(test.TestCase):
           lambda: constant_op.constant(1))
       self.assertEqual(5, self.evaluate(r))
 
+  @test_util.run_v1_only("b/120545219")
   def testUninitializedRefIdentity(self):
     with self.cached_session() as sess:
       v = gen_state_ops.variable(
@@ -771,6 +791,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
       self.evaluate(r)
 
+  @test_util.run_v1_only("b/120545219")
   def testCondGrad_1(self):
     with self.cached_session():
       x = constant_op.constant(10.0, name="x")
@@ -845,6 +866,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(sess.run(grad_grad, {pred: True, x: 1.0, y: 2.0}), 0.0)
       self.assertEqual(sess.run(grad_grad, {pred: False, x: 1.0, y: 2.0}), 0.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testNestedCond_Simple(self):
     with self.cached_session():
       x = constant_op.constant(0., name="X")
@@ -861,7 +883,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(1.0, self.evaluate(result))
 
   @test_util.disable_control_flow_v2("b/113327884")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCondGrad_Gather(self):
     with self.cached_session() as sess:
       v1 = variables.Variable([1.0, 42.0])
@@ -885,6 +907,7 @@ class ControlFlowTest(test.TestCase):
       ]
       self.assertAllEqual(dense_gv, [0.0, 2.0])
 
+  @test_util.run_v1_only("b/120545219")
   def testCondPredicateTensor(self):
     """Regression test for lowering predicate from non-first output of an op."""
 
@@ -1011,6 +1034,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(10000, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/79881896 (control deps)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileExternalControlDependencies(self):
     with self.cached_session():
       v = variables.Variable(0.0)
@@ -1027,6 +1051,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(v.eval(), 1.0)
 
   @test_util.disable_control_flow_v2("b/79881896 (control deps)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileExternalControlDependenciesNoInput(self):
     with self.cached_session():
       v = variables.Variable(0.0)
@@ -1043,7 +1068,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(v.eval(), 1.0)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithRefs_1(self):
     with self.cached_session() as sess:
       x = variables.VariableV1(0)._ref()  # pylint: disable=protected-access
@@ -1080,6 +1105,7 @@ class ControlFlowTest(test.TestCase):
       r = isum(s, maximum_iterations=3)
       self.assertAllEqual([1 + 3, 2 + 3, 3 + 3, 4 + 3, 5 + 3], self.evaluate(r))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithMaximumIterationsAndSingleArgument(self):
     with self.cached_session():
       r = control_flow_ops.while_loop(
@@ -1087,6 +1113,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(1, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/115776323 (max_iters)")
+  @test_util.run_v1_only("b/120545219")
   def testSingleNestedMaximumIterationsWhileLoopGradientInXLAContext(self):
     v = constant_op.constant(1.0)
 
@@ -1112,6 +1139,7 @@ class ControlFlowTest(test.TestCase):
     # Should execute without issue.
     self.assertEqual(3, self.evaluate(loop_execute))
 
+  @test_util.run_v1_only("b/120545219")
   def testInvalidMaximumIterationsWhileLoopGradientInXLAContext(self):
     v = constant_op.constant(1.0)
 
@@ -1172,6 +1200,7 @@ class ControlFlowTest(test.TestCase):
           r"context '.*' \(currently defined in '.*'\)"):
         _ = gradients_impl.gradients(loop_with_maxiter, v)
 
+  @test_util.run_v1_only("b/120545219")
   def testInvalidMaximumIterationsFromSiblingContextWhileLoopInXLAContext(self):
     v = constant_op.constant(1.0)
 
@@ -1215,6 +1244,7 @@ class ControlFlowTest(test.TestCase):
         _ = gradients_impl.gradients(loop, v)
 
   @test_util.disable_control_flow_v2("b/118457764")
+  @test_util.run_v1_only("b/120545219")
   def testNestedWhileLoopWithMaxItersFromOuterContextInXLAContext(self):
     v = constant_op.constant(1.0)
 
@@ -1326,6 +1356,7 @@ class ControlFlowTest(test.TestCase):
       result = r[3].eval()
     self.assertAllEqual(42, result)
 
+  @test_util.run_v1_only("b/120545219")
   def testWhile_5(self):
     with self.cached_session():
 
@@ -1351,6 +1382,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(np.array([0, 1, 2, 3, 4, 5, 6]), result)
 
   @test_util.disable_control_flow_v2("b/116338794 (buffer_reuse)")
+  @test_util.run_v1_only("b/120545219")
   def testBufferForwarding(self):
     run_options = config_pb2.RunOptions(
         trace_level=config_pb2.RunOptions.FULL_TRACE)
@@ -1435,6 +1467,7 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.while_loop(c, b, [n], parallel_iterations=20)
       self.assertEqual([10000], self.evaluate(r))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileShapeInference(self):
     with self.cached_session():
       i = constant_op.constant(0)
@@ -1461,6 +1494,7 @@ class ControlFlowTest(test.TestCase):
         r = control_flow_ops.while_loop(c, b, [i, m])
 
   @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileShapeInferenceSparseTensor(self):
     with self.cached_session():
       values = constant_op.constant([2.0, 4.0], name="values")
@@ -1493,7 +1527,7 @@ class ControlFlowTest(test.TestCase):
             [i.get_shape(), tensor_shape.TensorShape([5])])
 
   @test_util.disable_control_flow_v2("b/116282023 (IndexedSlices)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileShapeInferenceIndexedSlices(self):
     with self.cached_session():
       values = constant_op.constant([[2.0, 4.0], [3.0, 5.0]], name="values")
@@ -1584,6 +1618,7 @@ class ControlFlowTest(test.TestCase):
     self._testNestedWhile_2(use_gpu=False)
     self._testNestedWhile_2(use_gpu=True)
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithControl_1(self):
     with self.cached_session():
       n = constant_op.constant(0)
@@ -1615,6 +1650,7 @@ class ControlFlowTest(test.TestCase):
           condition, body, [r], parallel_iterations=1)
       self.assertAllEqual(12, self.evaluate(res))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithControl_3(self):
     with self.cached_session() as sess:
       b = array_ops.placeholder(dtypes.bool)
@@ -1624,6 +1660,7 @@ class ControlFlowTest(test.TestCase):
         r = control_flow_ops.while_loop(lambda x: x < 10, lambda x: x + c, [x0])
       self.assertEqual(10, sess.run(r, {b: True}))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithControl_4(self):
     with self.cached_session() as sess:
       b = array_ops.placeholder(dtypes.bool)
@@ -1635,6 +1672,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(10, sess.run(r, {b: True}))
 
   @test_util.disable_control_flow_v2("b/79881896 (control_deps)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithControl_5(self):
     with self.cached_session() as sess:
       b = array_ops.placeholder(dtypes.bool)
@@ -1663,6 +1701,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(0, self.evaluate(loop))
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileCondWithControl_1(self):
     with self.cached_session():
       v = variable_scope.get_variable(
@@ -1686,6 +1725,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(65536.0, self.evaluate(v))
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileCondExitControl(self):
 
     with self.cached_session():
@@ -1855,6 +1895,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(np.array([10.0, 10.0, 10.0]), result)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileUpdateVariable_2(self):
     with self.cached_session():
       select1 = variables.Variable([3.0, 4.0, 5.0])
@@ -1905,7 +1946,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllClose(np.array([10.0, 10.0, 10.0]), result)
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileUpdateVariable_4(self):
     with self.cached_session():
       var_a = variables.Variable(0, name="a")
@@ -1934,7 +1975,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(10, self.evaluate(var_b))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileUpdateVariable_5(self):
     with self.cached_session():
       # Create some variables.
@@ -1965,6 +2006,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(10, self.evaluate(var_b))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileUpdateVariable_6(self):
     with self.cached_session():
       # Create some variables.
@@ -1994,6 +2036,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(55, self.evaluate(var_b))
       self.assertEqual(10, self.evaluate(var_a))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileQueue_1(self):
     with self.cached_session():
       q = data_flow_ops.FIFOQueue(-1, dtypes.int32)
@@ -2012,6 +2055,7 @@ class ControlFlowTest(test.TestCase):
       for i in xrange(10):
         self.assertEqual([i], q.dequeue().eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileTimeOut(self):
     run_options = config_pb2.RunOptions(timeout_in_ms=1)
     with self.cached_session() as sess:
@@ -2023,6 +2067,7 @@ class ControlFlowTest(test.TestCase):
         sess.run(r, options=run_options)
 
   @test_util.disable_control_flow_v2("b/117119329 (stack)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileStack_1(self):
     with self.cached_session():
       s = gen_data_flow_ops.stack_v2(-1, dtypes.int32, stack_name="foo")
@@ -2092,10 +2137,12 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(1024.0, self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/116351701 (colocation)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_ColocateGradients(self):
     self._testWhileGrad_ColocateGradients(colocate=False)
     self._testWhileGrad_ColocateGradients(colocate=True)
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_Square(self):
     with self.cached_session():
       v = constant_op.constant(2.0, name="v")
@@ -2107,6 +2154,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(1024.0, self.evaluate(r))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_Shape(self):
     with self.cached_session():
       x = array_ops.placeholder(dtypes.float32, shape=[None])
@@ -2137,6 +2185,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients([r, y], x)[0]
       self.assertAllClose([2.0, 4.0], sess.run(r, feed_dict={x: [1.0, 2.0]}))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_MultipleUses(self):
     with self.cached_session():
       v = constant_op.constant(2.0, name="v")
@@ -2148,6 +2197,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertEqual(524288.0, self.evaluate(r))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_LoopAdd(self):
     with self.cached_session():
       v = constant_op.constant(2.0, name="v")
@@ -2211,6 +2261,7 @@ class ControlFlowTest(test.TestCase):
   def testNestedWhileCondWhileGradGpu(self):
     self._testNestedWhileCondWhileGrad(use_gpu=True)
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_Variable(self):
     with self.cached_session():
       a = variables.Variable(3.0)
@@ -2236,6 +2287,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllClose(216.0, g[0].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGradInCond(self):
 
     with self.cached_session():
@@ -2253,7 +2305,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(9.0, r.eval(feed_dict={x: 1.0}))
 
   @test_util.disable_control_flow_v2("b/116340060")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testGradInWhileWrtInitialLoopVal(self):
     with self.cached_session():
       x = array_ops.placeholder(dtypes.float32, shape=(), name="x")
@@ -2271,6 +2323,7 @@ class ControlFlowTest(test.TestCase):
           "loop invariants or wrt the input parameters to the loop body."):
         control_flow_ops.while_loop(lambda i, x: i < 3, body, [0, y])
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGradInWhile(self):
     with self.cached_session():
       n = ops.convert_to_tensor(1.0, name="n")
@@ -2287,6 +2340,7 @@ class ControlFlowTest(test.TestCase):
                                       [tensor_shape.unknown_shape()])
       self.assertAllClose(9.0, r.eval(feed_dict={x: 1.0}))
 
+  @test_util.run_v1_only("b/120545219")
   def testCondGradInNestedWhiles(self):
 
     def outer_body(i, x):
@@ -2375,6 +2429,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual([100.0, 1.0, 102.0, 3.0, 4.0 + 100 * 2.0],
                        self.evaluate(r_flattened))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhile_NestedBadArityFails(self):
     with self.cached_session():
       named = collections.namedtuple("named", ("a", "b"))
@@ -2391,6 +2446,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaisesRegexp(ValueError, "the same number of elements"):
         control_flow_ops.while_loop(c, b, loop_vars)
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_ys_xs(self):
     with self.cached_session():
       x = constant_op.constant(3.0, name="x")
@@ -2435,6 +2491,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(1024.0, r[0].eval())
 
   @test_util.disable_control_flow_v2("b/116355153 (back_prop flag)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_NoGradient(self):
     with self.cached_session():
       v = constant_op.constant(2.0, name="v")
@@ -2446,6 +2503,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(1.0, r[0].eval())
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_NoDependency(self):
     with self.cached_session() as sess:
       variable = variables.Variable(array_ops.ones([2, 3]))
@@ -2486,6 +2544,7 @@ class ControlFlowTest(test.TestCase):
       grad = gradients_impl.gradients(cost, [c0])
       self.assertAllClose(0.0, sess.run(grad[0]))
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_SerialTwoLoops(self):
     with self.cached_session():
       i = constant_op.constant(0, name="i")
@@ -2504,6 +2563,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients([rx], x)
       self.assertAllClose(1024.0, r[0].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_ParallelTwoLoops(self):
     with self.cached_session():
       i = constant_op.constant(0, name="i")
@@ -2523,6 +2583,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients([rx], x)
       self.assertAllClose(64.0, r[0].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_OneOutputWithControlDependencyOnSecond(self):
     with self.cached_session():
       i = constant_op.constant(0, name="i")
@@ -2566,6 +2627,7 @@ class ControlFlowTest(test.TestCase):
     self._testNestedWhileGrad_Simple(use_gpu=False)
     self._testNestedWhileGrad_Simple(use_gpu=True)
 
+  @test_util.run_v1_only("b/120545219")
   def testNestedWhileGrad_SerialInner(self):
     with self.cached_session():
       v = constant_op.constant(1.0)
@@ -2613,6 +2675,7 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(512.0, self.evaluate(r))
 
+  @test_util.run_v1_only("b/120545219")
   def testNestedWhileGrad_ParallelIterations(self):
     # Make sure the stack pushes and pops of an inner loop are executed in
     # the sequential order of the iterations of its outer loop.
@@ -2702,6 +2765,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose([[0.98000002, 1.98000002]], self.evaluate(x))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithRefsWithGradients_1(self):
     with self.cached_session() as sess:
       x = variables.VariableV1(0.)._ref()  # pylint: disable=protected-access
@@ -2731,6 +2795,7 @@ class ControlFlowTest(test.TestCase):
     self.assertEqual(73, value_x_grad)
 
   @test_util.disable_control_flow_v2("b/116282023 (IndexedSlices)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_IndexedSlices(self):
     with self.cached_session():
       values = constant_op.constant([2.0, 4.0], name="values")
@@ -2753,7 +2818,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileGrad_SparseTensor(self):
     with self.cached_session():
       values = constant_op.constant([2.0, 4.0], name="values")
@@ -2777,6 +2842,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
   @test_util.disable_control_flow_v2("b/115920078 (gradients)")
+  @test_util.run_v1_only("b/120545219")
   def testCallGradInLoop(self):
     with self.cached_session() as sess:
       i0 = constant_op.constant(0)
@@ -2959,6 +3025,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose([0., 0.], self.evaluate(dy_dq))
 
   @test_util.disable_control_flow_v2("b/113324949 (RefVariable)")
+  @test_util.run_v1_only("b/120545219")
   def testWhileGradientWithNontrainablePath2(self):
     q = variables.Variable([7., 8.])
 
@@ -2977,6 +3044,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose([1., 1.], self.evaluate(dy_dq))
 
   @test_util.disable_control_flow_v2("b/115920078 (gradients)")
+  @test_util.run_v1_only("b/120545219")
   def testIssue16504(self):
     c = constant_op.constant(np.arange(100), dtype=dtypes.float32)
     w = variables.Variable(
@@ -3000,6 +3068,7 @@ class ControlFlowTest(test.TestCase):
     grad, = gradients_impl.gradients(w, c)
     self.assertIsNotNone(grad)
 
+  @test_util.run_v1_only("b/120545219")
   def testStopGradMultiFlows(self):
     with self.cached_session():
 
@@ -3026,6 +3095,7 @@ class ControlFlowTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(5.0, self.evaluate(result))
 
+  @test_util.run_v1_only("b/120545219")
   def testOneValueCond(self):
 
     with self.cached_session():
@@ -3059,6 +3129,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(4.0, i.eval(feed_dict={d: 1}))
       self.assertAllClose(2.0 * math.sqrt(2), i.eval(feed_dict={d: 2}))
 
+  @test_util.run_v1_only("b/120545219")
   def testCase(self):
     with self.cached_session():
       x = constant_op.constant(1)
@@ -3111,6 +3182,7 @@ class ControlFlowTest(test.TestCase):
 
       self.assertAllEqual(r6.eval(), 0)
 
+  @test_util.run_v1_only("b/120545219")
   def testCaseSideEffects(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(-1)
@@ -3147,6 +3219,7 @@ class ControlFlowTest(test.TestCase):
       self.assertAllEqual(self.evaluate([v0, v1, v2]), [0, -1, -1])
 
   @test_util.disable_control_flow_v2("b/113324949 (ref vars)")
+  @test_util.run_v1_only("b/120545219")
   def testOneOpCond(self):
     with self.cached_session():
       v = variables.Variable(0)
@@ -3175,6 +3248,7 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(2, i.eval(feed_dict={c.name: 0}))
       self.assertEqual(2, self.evaluate(v))
 
+  @test_util.run_v1_only("b/120545219")
   def testWithOpsDependencies(self):
     with self.cached_session() as sess:
       v = variables.VariableV1(0.0)
@@ -3198,6 +3272,7 @@ class ControlFlowTest(test.TestCase):
     # Ensure that 'v' is initialized
     self.assertAllClose(0.0, real_v_val)
 
+  @test_util.run_v1_only("b/120545219")
   def testWithTensorDependencies(self):
     with self.cached_session():
       v = variables.VariableV1(0.0)
@@ -3224,6 +3299,7 @@ class ControlFlowTest(test.TestCase):
       # Ensure that 'v' is initialized
       self.assertAllClose(0.0, self.evaluate(v))
 
+  @test_util.run_v1_only("b/120545219")
   def testWithIndexedSlicesDependencies(self):
     with self.cached_session():
       v = variables.VariableV1(
@@ -3270,6 +3346,7 @@ class ControlFlowTest(test.TestCase):
         self.assertDeviceEqual("", with_vdef_dep.device)
         self.assertEqual([b"loc:@vdef"], with_vdef_dep.op.colocation_groups())
 
+  @test_util.run_v1_only("b/120545219")
   def testGroup(self):
     with self.cached_session() as sess:
       v1 = variables.VariableV1([0.0])
@@ -3289,6 +3366,7 @@ class ControlFlowTest(test.TestCase):
     self.assertAllClose([0.0], v1_val)
     self.assertAllClose([1.0], v2_val)
 
+  @test_util.run_v1_only("b/120545219")
   def testGroupEmpty(self):
     op = control_flow_ops.group()
     self.assertEqual(op.type, "NoOp")
@@ -3349,7 +3427,7 @@ class ControlFlowTest(test.TestCase):
     self.assertEqual([None, None], m.get_shape().as_list())
     self.assertEqual([], index.get_shape())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRefSelect(self):
     index = array_ops.placeholder(dtypes.int32)
 
@@ -3404,6 +3482,7 @@ class ControlFlowTest(test.TestCase):
       with self.assertRaises(ValueError):
         sess.run(tensor_list[0])
 
+  @test_util.run_v1_only("b/120545219")
   def testWhilePyFuncBasic(self):
 
     def func(x):
@@ -3417,6 +3496,7 @@ class ControlFlowTest(test.TestCase):
           [tensor_shape.unknown_shape(), tensor_shape.unknown_shape()])
       self.assertEqual(r[1].eval(), 65536.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileFuncBasic(self):
 
     @function.Defun(dtypes.float32)
@@ -3440,6 +3520,7 @@ class ControlFlowTest(test.TestCase):
                 ]), 1)
 
 
+  @test_util.run_v1_only("b/120545219")
   def testQIntSwitchMerge(self):
     with self.cached_session(force_gpu=test.is_gpu_available()) as sess:
       constant_qint = constant_op.constant(np.array([42]), dtypes.qint8)
@@ -3448,6 +3529,7 @@ class ControlFlowTest(test.TestCase):
       result = control_flow_ops.merge([v_f, v_t])
       self.evaluate(result)
 
+  @test_util.run_v1_only("b/120545219")
   def testQIntRefSwitchMerge(self):
     with self.cached_session(use_gpu=test.is_gpu_available()) as sess:
       var_qint = gen_state_ops.variable(
@@ -3461,6 +3543,7 @@ class ControlFlowTest(test.TestCase):
       result = control_flow_ops.ref_merge([v_f, v_t])
       self.evaluate(result)
 
+  @test_util.run_v1_only("b/120545219")
   def testUInt64SwitchMerge(self):
     with self.cached_session(force_gpu=test.is_gpu_available()) as sess:
       constant_uint64 = constant_op.constant(np.array([42]), dtypes.uint64)
@@ -3508,6 +3591,7 @@ class ControlFlowContextCheckTest(test.TestCase):
         math_ops.less(1, 2), true_fn, lambda: constant_op.constant(0))
     return cond_tensor[0]
 
+  @test_util.run_v1_only("b/120545219")
   def testInvalidContext(self):
     # Accessing a while loop tensor outside of control flow is illegal.
     while_tensor = self._getWhileTensor()
@@ -3517,7 +3601,7 @@ class ControlFlowContextCheckTest(test.TestCase):
         "is in a while loop. See info log for more details."):
       math_ops.add(1, while_tensor)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInvalidContextInCond(self):
     # Accessing a while loop tensor in cond is illegal.
     while_tensor = self._getWhileTensor()
@@ -3530,6 +3614,7 @@ class ControlFlowContextCheckTest(test.TestCase):
           math_ops.less(1, 2), lambda: math_ops.add(1, while_tensor),
           lambda: constant_op.constant(0))
 
+  @test_util.run_v1_only("b/120545219")
   def testInvalidContextInWhile(self):
     # Accessing a while loop tensor in a different while loop is illegal.
     while_tensor = self._getWhileTensor()
@@ -3564,6 +3649,7 @@ class ControlFlowContextCheckTest(test.TestCase):
 
     control_flow_ops.cond(math_ops.less(1, 2), branch_fn, branch_fn)
 
+  @test_util.run_v1_only("b/120545219")
   def testValidWhileContext(self):
     # Accessing a tensor in a nested while is OK.
     def body(_):
@@ -3572,6 +3658,7 @@ class ControlFlowContextCheckTest(test.TestCase):
 
     control_flow_ops.while_loop(lambda i: i < 5, body, [0])
 
+  @test_util.run_v1_only("b/120545219")
   def testValidNestedContexts(self):
     # Accessing a tensor from a cond context in a while context, all inside an
     # outer while context, is OK.
@@ -3586,7 +3673,7 @@ class ControlFlowContextCheckTest(test.TestCase):
 
     control_flow_ops.while_loop(lambda i: i < 5, body, [0])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInvalidNestedContexts(self):
     # Accessing a tensor from a while context in a different while context, all
     # inside a cond context, is illegal.
@@ -3605,6 +3692,7 @@ class ControlFlowContextCheckTest(test.TestCase):
 
 class TupleTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testTensors(self):
     for v1_first in [True, False]:
       with self.cached_session():
@@ -3635,7 +3723,7 @@ class TupleTest(test.TestCase):
           self.assertAllClose([30.0], self.evaluate(t2))
           self.assertAllClose([1.0], self.evaluate(v1))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testIndexedSlices(self):
     for v1_first in [True, False]:
       with self.cached_session():
@@ -3887,6 +3975,7 @@ class EagerTest(test.TestCase):
           isum(tensor, maximum_iterations=3).numpy(),
           [1 + 3, 2 + 3, 3 + 3, 4 + 3, 5 + 3])
 
+  @test_util.run_v1_only("b/120545219")
   def testWhileWithMaximumIterationsAndSingleArgument(self):
     with context.eager_mode():
       tensor = constant_op.constant(0)
@@ -3909,6 +3998,7 @@ class EagerTest(test.TestCase):
       self.assertAllEqual(t1.numpy(), tup1.numpy())
       self.assertAllEqual(t2.numpy(), tup2.numpy())
 
+  @test_util.run_v1_only("b/120545219")
   def testCase(self):
     with context.eager_mode():
       x = constant_op.constant(1)
diff --git a/tensorflow/python/kernel_tests/control_flow_util_test.py b/tensorflow/python/kernel_tests/control_flow_util_test.py
index 762c445da0..573f4b0d25 100644
--- a/tensorflow/python/kernel_tests/control_flow_util_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_util_test.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import control_flow_util
@@ -32,6 +33,7 @@ from tensorflow.python.platform import test
 
 class ControlFlowUtilTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testIsSwitch(self):
     switch_false, _ = control_flow_ops.switch(1, True)
     switch = switch_false.op
@@ -44,6 +46,7 @@ class ControlFlowUtilTest(test.TestCase):
 
     self.assertFalse(control_flow_util.IsSwitch(test_ops.int_output().op))
 
+  @test_util.run_v1_only("b/120545219")
   def testIsLoopEnter(self):
     enter = gen_control_flow_ops.enter(1, frame_name="name").op
     self.assertTrue(control_flow_util.IsLoopEnter(enter))
@@ -61,6 +64,7 @@ class ControlFlowUtilTest(test.TestCase):
 
     self.assertFalse(control_flow_util.IsLoopEnter(test_ops.int_output().op))
 
+  @test_util.run_v1_only("b/120545219")
   def testIsLoopExit(self):
     exit_op = control_flow_ops.exit(1).op
     self.assertTrue(control_flow_util.IsLoopExit(exit_op))
diff --git a/tensorflow/python/kernel_tests/ctc_loss_op_test.py b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
index e6b5835079..e24f304c1b 100644
--- a/tensorflow/python/kernel_tests/ctc_loss_op_test.py
+++ b/tensorflow/python/kernel_tests/ctc_loss_op_test.py
@@ -106,7 +106,7 @@ class CTCLossTest(test.TestCase):
         with self.assertRaisesOpError(expected_err_re):
           self.evaluate([loss, grad])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBasic(self):
     """Test two batch entries."""
     # Input and ground truth from Alex Graves' implementation.
@@ -242,7 +242,7 @@ class CTCLossTest(test.TestCase):
 
     self._testCTCLoss(inputs, seq_lens, labels, loss_truth, grad_truth)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_time_major(self):
     """Testing time_major param.
 
@@ -272,7 +272,7 @@ class CTCLossTest(test.TestCase):
       (tf_loss, tf_loss_transposed) = self.evaluate([loss, loss_transposed])
       self.assertAllEqual(tf_loss, tf_loss_transposed)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInvalidSecondGradient(self):
     inputs = np.random.randn(2, 2, 3).astype(np.float32)
     inputs_t = constant_op.constant(inputs)
@@ -289,7 +289,7 @@ class CTCLossTest(test.TestCase):
                                    "explicitly disabled"):
         _ = gradients_impl._hessian_vector_product(loss, [inputs_t], v)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testEmptyBatch(self):
     inputs = constant_op.constant([], dtype=dtypes.float32, shape=(1, 0, 2))
     sequence_lengths = constant_op.constant([], dtype=dtypes.int32)
@@ -306,7 +306,7 @@ class CTCLossTest(test.TestCase):
 
 class CTCLossTestV2(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCtcLossV2(self):
     random_seed.set_random_seed(5)
 
@@ -351,7 +351,7 @@ class CTCLossTestV2(test.TestCase):
             logit_length=logit_length,
             blank_index=0))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCtcLossDenseIsSameAsCtcLoss(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       random_seed.set_random_seed(5)
@@ -405,7 +405,7 @@ class CTCLossTestV2(test.TestCase):
               rtol=2e-06,
               atol=2e-06)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCtcLossDenseUniqueFastPathIsSameAsCtcLoss(self):
     random_seed.set_random_seed(5)
 
@@ -459,7 +459,7 @@ class CTCLossTestV2(test.TestCase):
             rtol=2e-06,
             atol=2e-06)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCtcLossDenseWithBlankIndexIsSameAsCtcLoss(self):
     random_seed.set_random_seed(5)
 
@@ -516,7 +516,7 @@ class CTCLossTestV2(test.TestCase):
             rtol=2e-06,
             atol=2e-06)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCtcLossDenseWithNegativeBlankIndexIsSameAsCtcLoss(self):
     with ops.device("/GPU:0" if test.is_gpu_available() else "/CPU:0"):
       random_seed.set_random_seed(5)
@@ -565,7 +565,7 @@ class CTCLossTestV2(test.TestCase):
               rtol=2e-06,
               atol=2e-06)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCollapseRepeated(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 3, 3, 3, 0],
@@ -579,7 +579,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 4, 0, 0],
          [4, 2, 9, 4]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCollapseRepeatedPreservesDtypes(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=constant_op.constant(
@@ -597,7 +597,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 4, 0, 0],
          [4, 2, 9, 4]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCollapseRepeatedExtraPadding(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 3, 3, 3, 0, 0, 0],
@@ -611,7 +611,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 4, 0, 0],
          [4, 2, 9, 4]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCollapseRepeatedFrontRepeats(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 1, 1, 2, 2],
@@ -625,7 +625,7 @@ class CTCLossTestV2(test.TestCase):
          [1, 2],
          [1, 0]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCollapseRepeatedAllLabelsTheSame(self):
     collapsed, new_seq_lengths = ctc_ops.collapse_repeated(
         labels=[[1, 1, 1, 1, 1],
@@ -658,7 +658,7 @@ class CTCLossTestV2(test.TestCase):
 
     self.assertAllEqual(padded_dense, new_dense)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testUnique(self):
     labels = [
         [3, 4, 4, 3],
@@ -674,7 +674,7 @@ class CTCLossTestV2(test.TestCase):
         [0, 0, 0, 1],
     ], idx)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSumStates(self):
     idx = [
         [0, 1, 0, 1],
@@ -694,7 +694,7 @@ class CTCLossTestV2(test.TestCase):
          [1.8, 0.8, 0.0, 0.0]]
     ], sum_of_states)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStateToOlabel(self):
     labels = [
         [3, 4, 3, 4],
@@ -733,7 +733,7 @@ class CTCLossTestV2(test.TestCase):
          [22.0 + 23.0 + 24.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]],
     ])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStateToOlabelUnique(self):
     labels = [
         [3, 4, 3, 4],
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
index 4f74e1e741..4e3da068b8 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_no_tsan_test.py
@@ -33,7 +33,7 @@ class AssignOpTest(test.TestCase):
   # NOTE(mrry): We exclude thess tests from the TSAN TAP target, because they
   #   contain benign and deliberate data races when multiple threads update
   #   the same parameters without a lock.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelUpdateWithoutLocking(self):
     with self.cached_session() as sess:
       ones_t = array_ops.fill([1024, 1024], 1.0)
@@ -61,7 +61,7 @@ class AssignOpTest(test.TestCase):
       self.assertTrue((vals >= ones).all())
       self.assertTrue((vals <= ones * 20).all())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelAssignWithoutLocking(self):
     with self.cached_session() as sess:
       ones_t = array_ops.fill([1024, 1024], float(1))
@@ -94,7 +94,7 @@ class AssignOpTest(test.TestCase):
   # contain non-benign but known data races between the variable assignment and
   # returning the output tensors. This issue will be resolved with the new
   # resource variables.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelUpdateWithLocking(self):
     with self.cached_session() as sess:
       zeros_t = array_ops.fill([1024, 1024], 0.0)
@@ -122,7 +122,7 @@ class AssignOpTest(test.TestCase):
       ones = np.ones((1024, 1024)).astype(np.float32)
       self.assertAllEqual(vals, ones * 20)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelAssignWithLocking(self):
     with self.cached_session() as sess:
       zeros_t = array_ops.fill([1024, 1024], 0.0)
diff --git a/tensorflow/python/kernel_tests/dense_update_ops_test.py b/tensorflow/python/kernel_tests/dense_update_ops_test.py
index 309da88bef..545de87ca1 100644
--- a/tensorflow/python/kernel_tests/dense_update_ops_test.py
+++ b/tensorflow/python/kernel_tests/dense_update_ops_test.py
@@ -86,7 +86,7 @@ class AssignOpTest(test.TestCase):
   def testBasic(self):
     self._testTypes(np.arange(0, 20).reshape([4, 5]))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAssignNonStrictShapeChecking(self):
     with self.cached_session():
       data = array_ops.fill([1024, 1024], 0)
@@ -101,7 +101,7 @@ class AssignOpTest(test.TestCase):
       a2.op.run()
       self.assertAllEqual(p.eval(), self.evaluate(data2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitRequiredAssignAdd(self):
     with self.cached_session():
       p = variables.VariableV1(array_ops.fill([1024, 1024], 1), dtypes.int32)
@@ -109,7 +109,7 @@ class AssignOpTest(test.TestCase):
       with self.assertRaisesOpError("use uninitialized"):
         a.op.run()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitRequiredAssignSub(self):
     with self.cached_session():
       p = variables.VariableV1(array_ops.fill([1024, 1024], 1), dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index f6d834c2f8..5b1a47fb03 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_impl
@@ -185,6 +186,7 @@ class DepthwiseConv2DTest(test.TestCase):
     self.assertShapeEqual(native_result, conv_native)
     self.assertShapeEqual(native_result, conv_interface)
 
+  @test_util.run_v1_only("b/120545219")
   def testDepthwiseConv2D(self):
     for index, (input_size, filter_size, _, stride,
                 padding) in enumerate(ConfigsToTest()):
@@ -428,6 +430,7 @@ class DepthwiseConv2DTest(test.TestCase):
           use_gpu, grouped_conv, err)
       self.assertLess(err, tolerance)
 
+  @test_util.run_v1_only("b/120545219")
   def testDepthwiseConv2DInputGrad(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(CheckGradConfigsToTest()):
@@ -477,6 +480,7 @@ class DepthwiseConv2DTest(test.TestCase):
             use_gpu=True,
             data_format="NCHW")
 
+  @test_util.run_v1_only("b/120545219")
   def testDepthwiseConv2DFilterGrad(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(CheckGradConfigsToTest()):
diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py
index d6ef9e70b8..dbfda385ed 100644
--- a/tensorflow/python/kernel_tests/determinant_op_test.py
+++ b/tensorflow/python/kernel_tests/determinant_op_test.py
@@ -133,6 +133,7 @@ class DeterminantOpTest(test.TestCase):
     huge_matrix = np.array([[max_double, 0.0], [0.0, max_double]])
     self._compareDeterminant(huge_matrix)
 
+  @test_util.run_v1_only("b/120545219")
   def testNonSquareMatrix(self):
     # When the determinant of a non-square matrix is attempted we should return
     # an error
@@ -140,6 +141,7 @@ class DeterminantOpTest(test.TestCase):
       linalg_ops.matrix_determinant(
           np.array([[1., 2., 3.], [3., 5., 4.]]).astype(np.float32))
 
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     # The input to the determinant should be a 2-dimensional tensor.
     tensor1 = constant_op.constant([1., 2.])
@@ -150,6 +152,7 @@ class DeterminantOpTest(test.TestCase):
     self._compareDeterminant(np.empty([0, 2, 2]))
     self._compareDeterminant(np.empty([2, 0, 0]))
 
+  @test_util.run_v1_only("b/120545219")
   def testConcurrentExecutesWithoutError(self):
     with self.session(use_gpu=True) as sess:
       matrix1 = random_ops.random_normal([5, 5], seed=42)
diff --git a/tensorflow/python/kernel_tests/distributions/multinomial_test.py b/tensorflow/python/kernel_tests/distributions/multinomial_test.py
index b3f3416a52..187ddd4cf4 100644
--- a/tensorflow/python/kernel_tests/distributions/multinomial_test.py
+++ b/tensorflow/python/kernel_tests/distributions/multinomial_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import backprop
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import multinomial
@@ -33,6 +34,7 @@ class MultinomialTest(test.TestCase):
   def setUp(self):
     self._rng = np.random.RandomState(42)
 
+  @test_util.run_v1_only("b/120545219")
   def testSimpleShapes(self):
     with self.cached_session():
       p = [.1, .3, .6]
@@ -42,6 +44,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual(tensor_shape.TensorShape([3]), dist.event_shape)
       self.assertEqual(tensor_shape.TensorShape([]), dist.batch_shape)
 
+  @test_util.run_v1_only("b/120545219")
   def testComplexShapes(self):
     with self.cached_session():
       p = 0.5 * np.ones([3, 2, 2], dtype=np.float32)
@@ -52,6 +55,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual(tensor_shape.TensorShape([2]), dist.event_shape)
       self.assertEqual(tensor_shape.TensorShape([3, 2]), dist.batch_shape)
 
+  @test_util.run_v1_only("b/120545219")
   def testN(self):
     p = [[0.1, 0.2, 0.7], [0.2, 0.3, 0.5]]
     n = [[3.], [4]]
@@ -60,6 +64,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual((2, 1), dist.total_count.get_shape())
       self.assertAllClose(n, dist.total_count.eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testP(self):
     p = [[0.1, 0.2, 0.7]]
     with self.cached_session():
@@ -68,6 +73,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual((1, 3), dist.logits.get_shape())
       self.assertAllClose(p, dist.probs.eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testLogits(self):
     p = np.array([[0.1, 0.2, 0.7]], dtype=np.float32)
     logits = np.log(p) - 50.
@@ -78,6 +84,7 @@ class MultinomialTest(test.TestCase):
       self.assertAllClose(p, multinom.probs.eval())
       self.assertAllClose(logits, multinom.logits.eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testPmfUnderflow(self):
     logits = np.array([[-200, 0]], dtype=np.float32)
     with self.cached_session():
@@ -85,6 +92,7 @@ class MultinomialTest(test.TestCase):
       lp = dist.log_prob([1., 0.]).eval()[0]
       self.assertAllClose(-200, lp, atol=0, rtol=1e-6)
 
+  @test_util.run_v1_only("b/120545219")
   def testPmfandCountsAgree(self):
     p = [[0.1, 0.2, 0.7]]
     n = [[5.]]
@@ -97,6 +105,7 @@ class MultinomialTest(test.TestCase):
       with self.assertRaisesOpError("counts must sum to `self.total_count`"):
         dist.prob([3., 3, 0]).eval()
 
+  @test_util.run_v1_only("b/120545219")
   def testPmfNonIntegerCounts(self):
     p = [[0.1, 0.2, 0.7]]
     n = [[5.]]
@@ -157,6 +166,7 @@ class MultinomialTest(test.TestCase):
       self.assertAllClose([0.1, 0.9], self.evaluate(pmf))
       self.assertEqual((2), pmf.get_shape())
 
+  @test_util.run_v1_only("b/120545219")
   def testPmfCountsStretchedInBroadcastWhenSameRank(self):
     with self.cached_session():
       p = [[0.1, 0.9], [0.7, 0.3]]
@@ -165,6 +175,7 @@ class MultinomialTest(test.TestCase):
       self.assertAllClose(pmf.eval(), [0.1, 0.7])
       self.assertEqual((2), pmf.get_shape())
 
+  @test_util.run_v1_only("b/120545219")
   def testPmfCountsStretchedInBroadcastWhenLowerRank(self):
     with self.cached_session():
       p = [[0.1, 0.9], [0.7, 0.3]]
@@ -194,6 +205,7 @@ class MultinomialTest(test.TestCase):
       self.evaluate(pmf)
       self.assertEqual((4, 3), pmf.get_shape())
 
+  @test_util.run_v1_only("b/120545219")
   def testMultinomialMean(self):
     with self.cached_session():
       n = 5.
@@ -203,6 +215,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual((3,), dist.mean().get_shape())
       self.assertAllClose(expected_means, dist.mean().eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testMultinomialCovariance(self):
     with self.cached_session():
       n = 5.
@@ -214,6 +227,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual((3, 3), dist.covariance().get_shape())
       self.assertAllClose(expected_covariances, dist.covariance().eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testMultinomialCovarianceBatch(self):
     with self.cached_session():
       # Shape [2]
@@ -246,6 +260,7 @@ class MultinomialTest(test.TestCase):
       self.assertEqual((3, 5, 4, 4), covariance.get_shape())
       self.assertEqual((6, 3, 3, 3), covariance2.get_shape())
 
+  @test_util.run_v1_only("b/120545219")
   def testCovarianceFromSampling(self):
     # We will test mean, cov, var, stddev on a DirichletMultinomial constructed
     # via broadcast between alpha, n.
@@ -288,6 +303,7 @@ class MultinomialTest(test.TestCase):
       self.assertAllClose(sample_var_, analytic_var, atol=0.01, rtol=0.01)
       self.assertAllClose(sample_stddev_, analytic_stddev, atol=0.01, rtol=0.01)
 
+  @test_util.run_v1_only("b/120545219")
   def testSampleUnbiasedNonScalarBatch(self):
     with self.cached_session() as sess:
       dist = multinomial.Multinomial(
@@ -317,6 +333,7 @@ class MultinomialTest(test.TestCase):
       self.assertAllClose(
           actual_covariance_, sample_covariance_, atol=0., rtol=0.20)
 
+  @test_util.run_v1_only("b/120545219")
   def testSampleUnbiasedScalarBatch(self):
     with self.cached_session() as sess:
       dist = multinomial.Multinomial(
diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py
index 9655351a01..0579dddb70 100644
--- a/tensorflow/python/kernel_tests/fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/fifo_queue_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
 
+@test_util.run_v1_only("b/120545219")
 class FIFOQueueTest(test.TestCase):
 
   def testConstructor(self):
@@ -1423,6 +1424,7 @@ class FIFOQueueTest(test.TestCase):
         session.run([a, c])
 
 
+@test_util.run_v1_only("b/120545219")
 class FIFOQueueDictTest(test.TestCase):
 
   def testConstructor(self):
@@ -1583,6 +1585,7 @@ class FIFOQueueDictTest(test.TestCase):
       self.assertTrue([compat.as_bytes("dd"), compat.as_bytes("ee")], list(s))
 
 
+@test_util.run_v1_only("b/120545219")
 class FIFOQueueWithTimeoutTest(test.TestCase):
 
   def testDequeueWithTimeout(self):
@@ -1617,6 +1620,7 @@ class FIFOQueueWithTimeoutTest(test.TestCase):
       self.assertEqual(37, self.evaluate(dequeued_t))
 
 
+@test_util.run_v1_only("b/120545219")
 class QueueContainerTest(test.TestCase):
 
   def testContainer(self):
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index c489623fe5..95ee454614 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -494,7 +494,7 @@ class FunctionalOpsTest(test.TestCase):
 
   @test_util.disable_control_flow_v2("b/119323354")
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMapEmptyScalar(self):
     map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([]))
     self.assertAllEqual([0], map_return.get_shape().dims)
@@ -503,7 +503,7 @@ class FunctionalOpsTest(test.TestCase):
   # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
   # so the body of the while loop never executes
   @test_util.disable_control_flow_v2("b/119323354")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMapEmptyTensor(self):
     with self.cached_session():
       map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]),
@@ -797,7 +797,7 @@ class FunctionalOpsTest(test.TestCase):
     self.assertAllEqual(Run(100., False), 5050.)
     self.assertAllEqual(Run(100., True), 5050.)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWhileError(self):
     for use_gpu in (True, False):
       with ops.Graph().as_default() as g:
@@ -1027,7 +1027,7 @@ class FunctionalOpsTest(test.TestCase):
   def testForMLPWhile(self):
     self._testForMLP(True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testForError(self):
 
     @function.Defun(dtypes.int32, dtypes.float32)
@@ -1233,7 +1233,7 @@ class PartitionedCallTest(test.TestCase):
       self.assertAllEqual(expected, result)
 
   # Use an invalid executor name to test the plumbing of the executor_type attr.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testExecutorTypeAttrExecutorNotFound(self):
     @function.Defun(dtypes.int32)
     def AddFive(x):
diff --git a/tensorflow/python/kernel_tests/identity_op_py_test.py b/tensorflow/python/kernel_tests/identity_op_py_test.py
index 1a6794e896..40ec9db422 100644
--- a/tensorflow/python/kernel_tests/identity_op_py_test.py
+++ b/tensorflow/python/kernel_tests/identity_op_py_test.py
@@ -62,7 +62,7 @@ class IdentityOpTest(test.TestCase):
       self.assertEquals(shape,
                         array_ops.identity(np.array(array_2x3)).get_shape())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRefIdentityShape(self):
     with self.cached_session():
       shape = [2, 3]
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
index 8f8b15e8ed..18e13a76a0 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
@@ -214,7 +214,7 @@ class LinearOperatorTest(test.TestCase):
     operator = LinearOperatorMatmulSolve(matrix, is_square=True)
     self.assertTrue(operator.is_square)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_linear_operator_matmul_hints_closed(self):
     matrix = array_ops.placeholder(dtypes.float32)
     operator1 = LinearOperatorMatmulSolve(matrix)
@@ -241,7 +241,7 @@ class LinearOperatorTest(test.TestCase):
     self.assertTrue(operator_matmul.is_self_adjoint)
     self.assertEqual(None, operator_matmul.is_positive_definite)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_linear_operator_matmul_hints_false(self):
     matrix = array_ops.placeholder(dtypes.float32)
     operator1 = LinearOperatorMatmulSolve(
@@ -274,7 +274,7 @@ class LinearOperatorTest(test.TestCase):
     self.assertEqual(None, operator_matmul.is_self_adjoint)
     self.assertEqual(None, operator_matmul.is_positive_definite)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_linear_operator_matmul_hint_infer_square(self):
     matrix1 = array_ops.placeholder(shape=[2, 3], dtype=dtypes.float32)
     matrix2 = array_ops.placeholder(shape=[3, 2], dtype=dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/linalg_grad_test.py b/tensorflow/python/kernel_tests/linalg_grad_test.py
index 28e1d7e168..ff84221611 100644
--- a/tensorflow/python/kernel_tests/linalg_grad_test.py
+++ b/tensorflow/python/kernel_tests/linalg_grad_test.py
@@ -61,6 +61,7 @@ class MatrixUnaryFunctorGradientTest(test_lib.TestCase):
 
 def _GetMatrixUnaryFunctorGradientTest(functor_, dtype_, shape_, **kwargs_):
 
+  @test_util.run_v1_only('b/120545219')
   def Test(self):
     with self.session(use_gpu=True):
       np.random.seed(1)
@@ -103,6 +104,7 @@ def _GetMatrixBinaryFunctorGradientTest(functor_,
                                         float32_tol_fudge=1.0,
                                         **kwargs_):
 
+  @test_util.run_v1_only('b/120545219')
   def Test(self):
     # TODO(rmlarsen): Debug illegal address bug on CUDA and re-enable
     # GPU test for matrix_solve.
diff --git a/tensorflow/python/kernel_tests/losses_test.py b/tensorflow/python/kernel_tests/losses_test.py
index abff61f81b..4584a27e62 100644
--- a/tensorflow/python/kernel_tests/losses_test.py
+++ b/tensorflow/python/kernel_tests/losses_test.py
@@ -51,26 +51,26 @@ class AbsoluteDifferenceLossTest(test.TestCase):
         losses.absolute_difference(
             self._predictions, self._predictions, weights=None)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAllCorrectNoLossWeight(self):
     loss = losses.absolute_difference(self._predictions, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLoss(self):
     loss = losses.absolute_difference(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(5.5, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.absolute_difference(self._labels, self._predictions, weights)
     with self.cached_session():
       self.assertAlmostEqual(5.5 * weights, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.absolute_difference(self._labels, self._predictions,
@@ -148,7 +148,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 10.0, 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithPythonScalarWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -158,7 +158,7 @@ class SoftmaxCrossEntropyLossTest(test.TestCase):
       loss = losses.softmax_cross_entropy(labels, logits, weights)
       self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -311,7 +311,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       self.assertEquals(loss.op.name, 'sparse_softmax_cross_entropy_loss/value')
       self.assertAlmostEqual(loss.eval(), 10.0, 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithPythonScalarWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -321,7 +321,7 @@ class SparseSoftmaxCrossEntropyLossTest(test.TestCase):
       loss = losses.sparse_softmax_cross_entropy(labels, logits, weights)
       self.assertAlmostEqual(weights * 10.0, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeight(self):
     logits = constant_op.constant([[10.0, 0.0, 0.0], [0.0, 10.0, 0.0],
                                    [0.0, 0.0, 10.0]])
@@ -677,13 +677,13 @@ class LogLossTest(test.TestCase):
       with self.assertRaises(ValueError):
         losses.log_loss(self._labels, self._labels, weights=None)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAllCorrectNoLossWeight(self):
     loss = losses.log_loss(self._labels, self._labels)
     with self.cached_session():
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAllCorrectNoLossWeightWithPlaceholder(self):
     tf_predictions = array_ops.placeholder(
         dtypes.float32, shape=self._np_labels.shape)
@@ -692,14 +692,14 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(
           0.0, loss.eval(feed_dict={tf_predictions: self._np_labels}), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLoss(self):
     loss = losses.log_loss(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(-np.sum(self._expected_losses) / 6.0,
                              self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.log_loss(self._labels, self._predictions, weights)
@@ -707,7 +707,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
                              self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.log_loss(self._labels, self._predictions,
@@ -716,7 +716,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
                              self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeightAndPlaceholder(self):
     tf_predictions = array_ops.placeholder(
         dtypes.float32, shape=self._np_predictions.shape)
@@ -728,7 +728,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(weights * -np.sum(self._expected_losses) / 6.0,
                              loss, 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeightAndPlaceholderWithRankOnly(self):
     tf_predictions = array_ops.placeholder(dtypes.float32, shape=[None, None])
     weights = 2.3
@@ -788,7 +788,7 @@ class LogLossTest(test.TestCase):
       self.assertAlmostEqual(-np.sum(expected_losses) / 5.0,
                              self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithMeasurementSpecificWeightsWithPlaceholder(self):
     weights = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3))
     expected_losses = np.multiply(self._expected_losses, weights)
@@ -816,7 +816,7 @@ class LogLossTest(test.TestCase):
     with self.cached_session():
       self.assertAlmostEqual(-np.sum(expected_losses), self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithSampleSpecificWeightsMostZeroWithPlaceholder(self):
     weights = np.array([0, 0, 0, 0, 0, 2]).reshape((2, 3))
     expected_losses = np.multiply(self._expected_losses, weights)
@@ -955,26 +955,26 @@ class MeanSquaredErrorTest(test.TestCase):
           losses.mean_squared_error(predictions=constant_op.constant(0),
                                     labels=constant_op.constant(0)).eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAllCorrectNoLossWeight(self):
     loss = losses.mean_squared_error(self._predictions, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(0.0, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLoss(self):
     loss = losses.mean_squared_error(self._labels, self._predictions)
     with self.cached_session():
       self.assertAlmostEqual(49.5, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithPythonScalarWeight(self):
     weights = 2.3
     loss = losses.mean_squared_error(self._labels, self._predictions, weights)
     with self.cached_session():
       self.assertAlmostEqual(49.5 * weights, self.evaluate(loss), 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.mean_squared_error(self._labels, self._predictions,
@@ -1068,12 +1068,12 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
       self.assertAlmostEqual(
           expected_loss, dynamic_inputs_op.eval(feed_dict=feed_dict), places=3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAllCorrectNoLossWeight(self):
     self._test_valid_weights(
         self._labels, self._labels, expected_loss=0.0)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLoss(self):
     self._test_valid_weights(
         self._labels, self._predictions,
@@ -1104,7 +1104,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
           np_grad = self.evaluate(grad)
           self.assertFalse(np.isnan(np_grad).any())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithPythonScalarWeight(self):
     weight = 2.3
     self._test_valid_weights(
@@ -1112,7 +1112,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
         expected_loss=weight * np.sum(self._expected_losses),
         weights=weight)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonZeroLossWithScalarTensorWeight(self):
     weights = 2.3
     loss = losses.mean_pairwise_squared_error(
@@ -1179,7 +1179,7 @@ class MeanPairwiseSquaredErrorTest(test.TestCase):
             weights_placeholder: weights,
         })
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInvalid3dWeighted2x0(self):
     labels = np.array([
         [[1, 9, 2], [12, 11, 10], [9, 8, 7]],
diff --git a/tensorflow/python/kernel_tests/matrix_band_part_op_test.py b/tensorflow/python/kernel_tests/matrix_band_part_op_test.py
index 129ea40dfe..fdb7e4a1a4 100644
--- a/tensorflow/python/kernel_tests/matrix_band_part_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_band_part_op_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradient_checker
@@ -44,6 +45,7 @@ class MatrixBandPartTest(test_lib.TestCase):
 
 def _GetMatrixBandPartTest(dtype_, batch_shape_, shape_):
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     mat = np.ones(shape_).astype(dtype_)
     batch_mat = np.tile(mat, batch_shape_ + (1, 1))
@@ -73,6 +75,7 @@ class MatrixBandPartGradTest(test_lib.TestCase):
 
 def _GetMatrixBandPartGradTest(dtype_, batch_shape_, shape_):
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     shape = batch_shape_ + shape_
     x = constant_op.constant(np.random.rand(*shape), dtype=dtype_)
diff --git a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
index b0bce6a1b9..682ac12adc 100644
--- a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py
@@ -84,6 +84,7 @@ class LogarithmOpTest(test.TestCase):
     # Complex batch
     self._verifyLogarithmComplex(self._makeBatch(matrix1, matrix2))
 
+  @test_util.run_v1_only("b/120545219")
   def testNonSquareMatrix(self):
     # When the logarithm of a non-square matrix is attempted we should return
     # an error
@@ -91,6 +92,7 @@ class LogarithmOpTest(test.TestCase):
       gen_linalg_ops.matrix_logarithm(
           np.array([[1., 2., 3.], [3., 4., 5.]], dtype=np.complex64))
 
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     # The input to the logarithm should be at least a 2-dimensional tensor.
     tensor3 = constant_op.constant([1., 2.], dtype=dtypes.complex64)
@@ -121,6 +123,7 @@ class LogarithmOpTest(test.TestCase):
             size=np.prod(shape)).reshape(shape).astype(np.complex128)
         self._verifyLogarithmComplex(matrix)
 
+  @test_util.run_v1_only("b/120545219")
   def testConcurrentExecutesWithoutError(self):
     with self.session(use_gpu=True) as sess:
       matrix1 = math_ops.cast(
diff --git a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
index 42fd95d311..463477a6a2 100644
--- a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py
@@ -135,7 +135,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
       self.assertEqual(np_ans.shape, tf_ans_val.shape)
       self.assertAllClose(np_ans, tf_ans_val, atol=2 * tol, rtol=2 * tol)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     # The matrix and right-hand sides should have the same number of rows.
     with self.session(use_gpu=True):
@@ -144,7 +144,6 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
       with self.assertRaises(ValueError):
         linalg_ops.matrix_solve_ls(matrix, rhs)
 
-  @test_util.run_deprecated_v1
   def testEmpty(self):
     full = np.array([[1., 2.], [3., 4.], [5., 6.]])
     empty0 = np.empty([3, 0])
@@ -164,7 +163,7 @@ class MatrixSolveLsOpTest(test_lib.TestCase):
             linalg_ops.matrix_solve_ls(empty1, empty1, fast=fast))
         self.assertEqual(tf_ans.shape, (2, 2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBatchResultSize(self):
     # 3x3x3 matrices, 3x3x1 right-hand sides.
     matrix = np.array([1., 2., 3., 4., 5., 6., 7., 8., 9.] * 3).reshape(3, 3, 3)
diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
index 1e2109b8c4..3edb390c72 100644
--- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
+++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py
@@ -90,17 +90,20 @@ class SquareRootOpTest(test.TestCase):
     self._verifySquareRootReal(np.empty([0, 2, 2]))
     self._verifySquareRootReal(np.empty([2, 0, 0]))
 
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     # The input to the square root should be at least a 2-dimensional tensor.
     tensor = constant_op.constant([1., 2.])
     with self.assertRaises(ValueError):
       gen_linalg_ops.matrix_square_root(tensor)
 
+  @test_util.run_v1_only("b/120545219")
   def testNotSquare(self):
     with self.assertRaises(ValueError):
       tensor = constant_op.constant([[1., 0., -1.], [-1., 1., 0.]])
       self.evaluate(gen_linalg_ops.matrix_square_root(tensor))
 
+  @test_util.run_v1_only("b/120545219")
   def testConcurrentExecutesWithoutError(self):
     with test_util.use_gpu():
       matrix1 = random_ops.random_normal([5, 5], seed=42)
diff --git a/tensorflow/python/kernel_tests/norm_op_test.py b/tensorflow/python/kernel_tests/norm_op_test.py
index 5ff0c58bf1..20b9ad95c8 100644
--- a/tensorflow/python/kernel_tests/norm_op_test.py
+++ b/tensorflow/python/kernel_tests/norm_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.platform import test as test_lib
@@ -35,6 +36,7 @@ def _AddTest(test, test_name, fn):
 
 class NormOpTest(test_lib.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testBadOrder(self):
     matrix = [[0., 1.], [2., 3.]]
     for ord_ in "fro", -7, -1.1, 0:
@@ -52,6 +54,7 @@ class NormOpTest(test_lib.TestCase):
                                    "'ord' must be a supported matrix norm"):
         linalg_ops.norm(matrix, ord=ord_, axis=[-2, -1])
 
+  @test_util.run_v1_only("b/120545219")
   def testInvalidAxis(self):
     matrix = [[0., 1.], [2., 3.]]
     for axis_ in [], [1, 2, 3], [[1]], [[1], [2]], [3.1415], [1, 1]:
@@ -78,6 +81,7 @@ def _GetNormOpTest(dtype_, shape_, ord_, axis_, keep_dims_, use_static_shape_):
         tf_norm_val = sess.run(tf_norm, feed_dict={tf_matrix: matrix})
     self.assertAllClose(np_norm, tf_norm_val, rtol=1e-5, atol=1e-5)
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     is_matrix_norm = (isinstance(axis_, tuple) or
                       isinstance(axis_, list)) and len(axis_) == 2
diff --git a/tensorflow/python/kernel_tests/numerics_test.py b/tensorflow/python/kernel_tests/numerics_test.py
index 5751f3fe76..f13f9d6806 100644
--- a/tensorflow/python/kernel_tests/numerics_test.py
+++ b/tensorflow/python/kernel_tests/numerics_test.py
@@ -64,9 +64,9 @@ class VerifyTensorAllFiniteTest(test.TestCase):
         self.evaluate(t_verified)
 
 
+@test_util.run_v1_only("b/120545219")
 class NumericsTest(test.TestCase):
 
-  @test_util.run_deprecated_v1
   def testInf(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant(1.0)
@@ -77,7 +77,6 @@ class NumericsTest(test.TestCase):
       with self.assertRaisesOpError("Inf"):
         self.evaluate(a)
 
-  @test_util.run_deprecated_v1
   def testNaN(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant(0.0)
@@ -88,7 +87,6 @@ class NumericsTest(test.TestCase):
       with self.assertRaisesOpError("NaN"):
         self.evaluate(a)
 
-  @test_util.run_deprecated_v1
   def testBoth(self):
     with self.session(graph=ops.Graph()):
       t1 = constant_op.constant([1.0, 0.0])
@@ -107,7 +105,6 @@ class NumericsTest(test.TestCase):
       self.assertAllEqual(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]), value)
       self.assertEqual([2, 3], checked.get_shape())
 
-  @test_util.run_deprecated_v1
   def testControlFlowCond(self):
     predicate = array_ops.placeholder(dtypes.bool, shape=[])
     _ = control_flow_ops.cond(predicate,
@@ -120,7 +117,6 @@ class NumericsTest(test.TestCase):
         r"or `tf.while_loop\(\)`\."):
       numerics.add_check_numerics_ops()
 
-  @test_util.run_deprecated_v1
   def testControlFlowWhile(self):
     predicate = array_ops.placeholder(dtypes.bool, shape=[])
     _ = control_flow_ops.while_loop(lambda _: predicate,
diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
index b4818360d5..e3999695d0 100644
--- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
+++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py
@@ -29,11 +29,13 @@ from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("b/120545219")
 class PaddingFIFOQueueTest(test.TestCase):
 
   def testConstructor(self):
@@ -1393,6 +1395,7 @@ class PaddingFIFOQueueTest(test.TestCase):
     with self.assertRaisesOpError("was cancelled"):
       self.evaluate(enqueue_many_op)
 
+  @test_util.run_deprecated_v1
   def testResetOfBlockingOperation(self):
     with self.cached_session() as sess:
       q_empty = data_flow_ops.PaddingFIFOQueue(5, dtypes_lib.float32, ((),))
diff --git a/tensorflow/python/kernel_tests/partitioned_variables_test.py b/tensorflow/python/kernel_tests/partitioned_variables_test.py
index 48655391fa..da79b4ecfc 100644
--- a/tensorflow/python/kernel_tests/partitioned_variables_test.py
+++ b/tensorflow/python/kernel_tests/partitioned_variables_test.py
@@ -412,7 +412,7 @@ class PartitionedVariablesTestCase(test.TestCase):
   def testResourceName(self):
     self._testNameHelper(use_resource=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRandomInitValue(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([200, 40]))
@@ -430,7 +430,7 @@ class PartitionedVariablesTestCase(test.TestCase):
           "200 40 0,200:36,4"
       ])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRandomInitUnevenPartitions(self):
     with self.cached_session():
       rnd = variables.Variable(
@@ -469,7 +469,7 @@ class PartitionedVariablesTestCase(test.TestCase):
         if i < len(save_specs):
           self._TestSaveSpec(vs, save_specs[i])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDegenerate(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([10, 43]))
@@ -481,7 +481,7 @@ class PartitionedVariablesTestCase(test.TestCase):
       self.assertAllClose(rnd, val)
       self._TestSaveSpec(vs, ["10 43 0,10:0,43"])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSliceSizeOne(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([10, 43]))
diff --git a/tensorflow/python/kernel_tests/priority_queue_test.py b/tensorflow/python/kernel_tests/priority_queue_test.py
index 9be682ea52..49ec7ee483 100644
--- a/tensorflow/python/kernel_tests/priority_queue_test.py
+++ b/tensorflow/python/kernel_tests/priority_queue_test.py
@@ -27,6 +27,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -35,6 +36,7 @@ from tensorflow.python.platform import test
 
 class PriorityQueueTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testRoundTripInsertReadOnceSorts(self):
     with self.cached_session() as sess:
       q = data_flow_ops.PriorityQueue(2000, (dtypes.string, dtypes.string), (
@@ -112,6 +114,7 @@ class PriorityQueueTest(test.TestCase):
         missed.remove((dv0, dv1))
       self.assertEqual(missed, set())
 
+  @test_util.run_v1_only("b/120545219")
   def testRoundTripFillsCapacityMultiThreadedEnqueueAndDequeue(self):
     with self.cached_session() as sess:
       q = data_flow_ops.PriorityQueue(10, (dtypes.int64), (()))
@@ -267,6 +270,7 @@ class PriorityQueueTest(test.TestCase):
         missed.remove((dv0, dv1))
       self.assertEqual(missed, set())
 
+  @test_util.run_v1_only("b/120545219")
   def testRoundTripInsertOnceReadOnceSorts(self):
     with self.cached_session() as sess:
       q = data_flow_ops.PriorityQueue(2000, (dtypes.string, dtypes.string), (
@@ -288,6 +292,7 @@ class PriorityQueueTest(test.TestCase):
       for e, dv0, dv1 in zip(deq_elem, deq_value_0, deq_value_1):
         self.assertTrue((dv0, dv1) in allowed[e])
 
+  @test_util.run_v1_only("b/120545219")
   def testRoundTripInsertOnceReadManySorts(self):
     with self.cached_session():
       q = data_flow_ops.PriorityQueue(2000, (dtypes.int64), (()))
@@ -296,6 +301,7 @@ class PriorityQueueTest(test.TestCase):
       deq_values = np.hstack((q.dequeue_many(100)[0].eval() for _ in range(10)))
       self.assertAllEqual(deq_values, sorted(elem))
 
+  @test_util.run_v1_only("b/120545219")
   def testRoundTripInsertOnceReadOnceLotsSorts(self):
     with self.cached_session():
       q = data_flow_ops.PriorityQueue(2000, (dtypes.int64), (()))
@@ -311,6 +317,7 @@ class PriorityQueueTest(test.TestCase):
       with self.assertRaises(TypeError):
         q.enqueue_many((["a", "b", "c"], ["a", "b", "c"])).run()
 
+  @test_util.run_v1_only("b/120545219")
   def testInsertingNonScalarFails(self):
     with self.cached_session() as sess:
       input_priority = array_ops.placeholder(dtypes.int64)
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index 1f3f02a9f0..482633d539 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -102,6 +102,7 @@ class PyFuncTest(test.TestCase):
           script_ops.eager_py_func(np_func, [x, y], [dtypes.float32]))
       self.assertEqual(z[0], np_func(1.0, 2.0).astype(np.float32))
 
+  @test_util.run_v1_only("b/120545219")
   def testArray(self):
     with self.cached_session():
       x = constant_op.constant([1.0, 2.0], dtypes.float64)
@@ -168,6 +169,7 @@ class PyFuncTest(test.TestCase):
                              (dtypes.float64, dtypes.float64)))
       self.assertAllClose(y, [0.0, 1.0])
 
+  @test_util.run_v1_only("b/120545219")
   def testStrings(self):
 
     def read_fixed_length_numpy_strings():
@@ -185,6 +187,7 @@ class PyFuncTest(test.TestCase):
           script_ops.py_func(read_and_return_strings, [x, y], dtypes.string))
       self.assertAllEqual(z, [b"hello there", b"hi there"])
 
+  @test_util.run_v1_only("b/120545219")
   def testStringsAreConvertedToBytes(self):
 
     def read_fixed_length_numpy_strings():
@@ -202,6 +205,7 @@ class PyFuncTest(test.TestCase):
           script_ops.py_func(read_and_return_strings, [x, y], dtypes.string))
       self.assertAllEqual(z, [b"hello there", b"hi there"])
 
+  @test_util.run_v1_only("b/120545219")
   def testObjectArraysAreConvertedToBytes(self):
 
     def read_object_array():
@@ -217,12 +221,14 @@ class PyFuncTest(test.TestCase):
       z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
       self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"])
 
+  @test_util.run_v1_only("b/120545219")
   def testStringPadding(self):
     correct = [b"this", b"is", b"a", b"test"]
     with self.cached_session():
       s, = script_ops.py_func(lambda: [correct], [], [dtypes.string])
       self.assertAllEqual(s.eval(), correct)
 
+  @test_util.run_v1_only("b/120545219")
   def testStringPaddingAreConvertedToBytes(self):
     inp = ["this", "is", "a", "test"]
     correct = [b"this", b"is", b"a", b"test"]
@@ -230,6 +236,7 @@ class PyFuncTest(test.TestCase):
       s, = script_ops.py_func(lambda: [inp], [], [dtypes.string])
       self.assertAllEqual(s.eval(), correct)
 
+  @test_util.run_v1_only("b/120545219")
   def testLarge(self):
     with self.cached_session() as sess:
       x = array_ops.zeros([1000000], dtype=np.float32)
@@ -243,6 +250,7 @@ class PyFuncTest(test.TestCase):
       x = self.evaluate(script_ops.py_func(lambda: 42.0, [], dtypes.float64))
       self.assertAllClose(x, 42.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testAlias(self):
     with self.cached_session():
       np_array = np.array([1.0, 2.0], dtype=np.float32)
@@ -251,6 +259,7 @@ class PyFuncTest(test.TestCase):
       value.op.run()
       self.assertAllEqual(np_array, [1.0, 2.0])
 
+  @test_util.run_v1_only("b/120545219")
   def testReturnUnicodeString(self):
     with self.cached_session():
       correct = u"你好 世界"
@@ -261,6 +270,7 @@ class PyFuncTest(test.TestCase):
       z, = script_ops.py_func(unicode_string, [], [dtypes.string])
       self.assertEqual(z.eval(), correct.encode("utf8"))
 
+  @test_util.run_v1_only("b/120545219")
   def testBadNumpyReturnType(self):
     with self.cached_session():
 
@@ -274,6 +284,7 @@ class PyFuncTest(test.TestCase):
                                    "Unsupported numpy type"):
         self.evaluate(y)
 
+  @test_util.run_v1_only("b/120545219")
   def testBadReturnType(self):
     with self.cached_session():
 
@@ -287,6 +298,7 @@ class PyFuncTest(test.TestCase):
                                    "Unsupported object type"):
         self.evaluate(z)
 
+  @test_util.run_v1_only("b/120545219")
   def testReturnInput(self):
     with self.cached_session():
 
@@ -321,6 +333,7 @@ class PyFuncTest(test.TestCase):
       self.assertEqual(self.evaluate(x), 0)
       self.assertEqual(self.evaluate(x), 0)
 
+  @test_util.run_v1_only("b/120545219")
   def testGradientFunction(self):
     # Input to tf.py_func is necessary, otherwise get_gradient_function()
     # returns None per default.
@@ -330,6 +343,7 @@ class PyFuncTest(test.TestCase):
     self.assertEqual(None, ops.get_gradient_function(x.op))
     self.assertEqual(None, ops.get_gradient_function(y.op))
 
+  @test_util.run_v1_only("b/120545219")
   def testCOrder(self):
     with self.cached_session():
       val = [[1, 2], [3, 4]]
@@ -337,6 +351,7 @@ class PyFuncTest(test.TestCase):
                               [dtypes.int64])
       self.assertAllEqual(val, self.evaluate(x))
 
+  @test_util.run_v1_only("b/120545219")
   def testParallel(self):
     # Tests that tf.py_func's can run in parallel if they release the GIL.
     with self.cached_session() as session:
@@ -382,6 +397,7 @@ class PyFuncTest(test.TestCase):
       self.assertIsNone(ret)
       self.assertAllEqual([3], s.value)
 
+  @test_util.run_v1_only("b/120545219")
   def testNoReturnValueStateless(self):
 
     def do_nothing(unused_x):
@@ -420,6 +436,7 @@ class PyFuncTest(test.TestCase):
     with self.assertRaisesWithPredicateMatch(tf_exp, expected_error_check):
       self.evaluate(f)
 
+  @test_util.run_v1_only("b/120545219")
   def testExceptionHandling(self):
     with self.cached_session():
       self._testExceptionHandling(ValueError, errors.InvalidArgumentError)
@@ -514,7 +531,7 @@ class PyFuncTest(test.TestCase):
       self.assertAllEqual(ret, [[3.0], [3.0], [3.0]])
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testEagerExceptionHandling(self):
     with test_util.device(use_gpu=True):
       self._testExceptionHandling(
@@ -534,7 +551,7 @@ class PyFuncTest(test.TestCase):
       self._testExceptionHandling(WeirdError, errors.UnknownError, eager=True)
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testEagerReturningVariableRaisesError(self):
     def return_variable():
       return resource_variable_ops.ResourceVariable(0.0)
@@ -558,6 +575,7 @@ class PyFuncTest(test.TestCase):
     dy_dx = tape.gradient(y, x)
     self.assertEqual(self.evaluate(dy_dx), 6.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testEagerGradientGraph(self):
 
     def f(x):
@@ -568,6 +586,7 @@ class PyFuncTest(test.TestCase):
     dy_dx = gradients_impl.gradients(y, x)[0]
     self.assertEqual(self.evaluate(dy_dx), 6.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testEagerGradientGraphTwoOutputs(self):
 
     def f(x, y):
@@ -597,6 +616,7 @@ class PyFuncTest(test.TestCase):
     self.assertEqual(self.evaluate(dz_dx), 6.0)
     self.assertEqual(self.evaluate(dz_dy), 8.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testEagerGradientGraphMultipleArgs(self):
 
     def f(x, y):
@@ -610,6 +630,7 @@ class PyFuncTest(test.TestCase):
     self.assertEqual(self.evaluate(dz_dx), 6.0)
     self.assertEqual(self.evaluate(dz_dy), 8.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testEagerGradientGraphLogHuber(self):
 
     def log_huber(x, m):
@@ -631,6 +652,7 @@ class PyFuncTest(test.TestCase):
       self.assertEqual(y, 1.0)
       self.assertEqual(dy_dx, 2.0)
 
+  @test_util.run_v1_only("b/120545219")
   def testEagerRespectsDevicePlacmentOfOp(self):
 
     def f(x):
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index 5e9767f20c..5adb95c7d6 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -40,6 +40,7 @@ def _AddTest(test_class, op_name, testcase_name, fn):
 
 class QrOpTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     # The input to qr should be a tensor of at least rank 2.
     scalar = constant_op.constant(1.)
@@ -115,6 +116,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
       tol = 1e-14
     self.assertAllClose(identity, xx, atol=tol)
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     np.random.seed(1)
     x_np = np.random.uniform(
@@ -163,6 +165,7 @@ class QrGradOpTest(test.TestCase):
 
 def _GetQrGradOpTest(dtype_, shape_, full_matrices_):
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     np.random.seed(42)
     a = np.random.uniform(low=-1.0, high=1.0, size=shape_).astype(dtype_)
diff --git a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
index ed4f5434d9..dd814a22b4 100644
--- a/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
@@ -29,11 +29,13 @@ from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 
 
+@test_util.run_v1_only("b/120545219")
 class RandomShuffleQueueTest(test.TestCase):
 
   def setUp(self):
@@ -1415,6 +1417,7 @@ class RandomShuffleQueueTest(test.TestCase):
 
       self.assertItemsEqual(elem, results)
 
+  @test_util.run_v1_only("b/120545219")
   def testBigDequeueMany(self):
     with self.cached_session() as sess:
       q = data_flow_ops.RandomShuffleQueue(2, 0, dtypes_lib.int32, ((),))
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 3992d6bdfb..433957fd1d 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -628,7 +628,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
           variable_def=other_v_def)
       self.assertTrue(other_v_prime._cached_value is not None)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariableDefInitializedInstances(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v_def = resource_variable_ops.ResourceVariable(
@@ -733,7 +733,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.assertEqual(0.0, self.evaluate(v.value()))
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDestroyResource(self):
     v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
@@ -792,7 +792,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       with self.assertRaises(ValueError):
         _ = w.value().op.get_attr("_class")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSharedName(self):
     with self.cached_session():
       v = resource_variable_ops.ResourceVariable(300.0, name="var4")
@@ -849,7 +849,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       v.initializer.run(feed_dict={v.initial_value: 3.0})
       self.assertEqual(3.0, v.value().eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testControlFlowInitialization(self):
     """Expects an error if an initializer is in a control-flow scope."""
 
@@ -986,7 +986,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(self.evaluate(v.assign_add(1)), [1, 2, 3, 4])
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCopyToGraphUninitialized(self):
     v = resource_variable_ops.ResourceVariable([0, 1, 2, 3])
     copy_to_graph = ops.Graph()
diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py
index 3bc457f8fb..a49496e4ef 100644
--- a/tensorflow/python/kernel_tests/rnn_test.py
+++ b/tensorflow/python/kernel_tests/rnn_test.py
@@ -262,7 +262,7 @@ class RNNTest(test.TestCase):
       rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32, sequence_length=[4])
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTensorArrayStateIsAccepted(self):
     cell = TensorArrayStateRNNCell()
     in_eager_mode = context.executing_eagerly()
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index c1241ba87e..8510a08f0c 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -217,7 +217,7 @@ class StatefulScatterNdTest(test.TestCase):
   def testVariableRankAdd(self):
     self._VariableRankTests(_NumpyAdd, state_ops.scatter_nd_add)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariableRankSub(self):
     self._VariableRankTests(_NumpySub, state_ops.scatter_nd_sub)
 
@@ -235,7 +235,7 @@ class StatefulScatterNdTest(test.TestCase):
         self._VariableRankTest(
             np_scatter, tf_scatter, vtype, itype, repeat_indices=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testScatterRepeatIndices(self):
     """This tests scatter_add using indices that repeat."""
     self._ScatterRepeatIndicesTest(_NumpyAdd, state_ops.scatter_nd_add)
@@ -257,7 +257,7 @@ class StatefulScatterNdTest(test.TestCase):
   #     session.run([update0, update1])
   #     self.assertAllEqual([False, True], self.evaluate(var))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testScatterOutOfRangeCpu(self):
     # TODO(simister): Re-enable once binary size increase due to
     # scatter_nd ops is under control.
@@ -294,7 +294,7 @@ class StatefulScatterNdTest(test.TestCase):
         state_ops.scatter_nd_update(ref, indices,
                                     updates).get_shape().as_list(), shape)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testResVarInvalidOutputShape(self):
     res = variables.Variable(
         initial_value=lambda: array_ops.zeros(shape=[], dtype=dtypes.float32),
@@ -509,7 +509,7 @@ class ScatterNdTest(test.TestCase):
         ValueError, "Indices and updates specified for empty output shape"):
       self.scatter_nd(indices, updates, shape)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testEmptyOutputShape2(self):
     indices = array_ops.placeholder(dtypes.int32, shape=None)
     updates = array_ops.placeholder(dtypes.int32, shape=None)
@@ -717,6 +717,7 @@ class ScatterNdTensorTest(test.TestCase):
     self.assertAllEqual(subbed,
                         constant_op.constant([1, -10, 1, -9, -8, 1, 1, -11]))
 
+  @test_util.run_v1_only("b/120545219")
   def testUpdateAddSubGradients(self):
 
     with self.cached_session():
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index ce782dbc28..47b22ec296 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker_v2
 from tensorflow.python.ops import linalg_ops
@@ -39,6 +40,7 @@ def _AddTest(test_class, op_name, testcase_name, fn):
 
 class SelfAdjointEigTest(test.TestCase):
 
+  @test_util.run_deprecated_v1
   def testWrongDimensions(self):
     # The input to self_adjoint_eig should be a tensor of
     # at least rank 2.
@@ -49,6 +51,7 @@ class SelfAdjointEigTest(test.TestCase):
     with self.assertRaises(ValueError):
       linalg_ops.self_adjoint_eig(vector)
 
+  @test_util.run_deprecated_v1
   def testConcurrentExecutesWithoutError(self):
     all_ops = []
     with self.session(use_gpu=True) as sess:
diff --git a/tensorflow/python/kernel_tests/session_ops_test.py b/tensorflow/python/kernel_tests/session_ops_test.py
index dc663cb091..bc5d8e8151 100644
--- a/tensorflow/python/kernel_tests/session_ops_test.py
+++ b/tensorflow/python/kernel_tests/session_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import session_ops
@@ -28,6 +29,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
+@test_util.run_v1_only("b/120545219")
 class SessionOpsTest(test.TestCase):
 
   def testHandleBasic(self):
@@ -232,6 +234,7 @@ class SessionOpsTest(test.TestCase):
                      b_p: b_handle.handle})
       self.assertEqual(3.0, c_handle.eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testFeedOneHandleDirectly(self):
     with self.cached_session() as sess:
       a = constant_op.constant(10.0)
@@ -243,6 +246,7 @@ class SessionOpsTest(test.TestCase):
 
       self.assertAllClose(2500.0, sess.run(d, feed_dict={c: h_c}))
 
+  @test_util.run_v1_only("b/120545219")
   def testDirectHandleFeedOverlappingWithFetches(self):
     with self.cached_session() as sess:
       a = constant_op.constant(10.0)
@@ -283,6 +287,7 @@ class SessionOpsTest(test.TestCase):
       self.assertAllClose(48.0, sess.run(e, feed_dict={c: h_c, d: h_d}))
       self.assertAllClose(-48.0, sess.run(e, feed_dict={c: h_d, d: h_c}))
 
+  @test_util.run_v1_only("b/120545219")
   def testFeedHandleToVariableDirectly(self):
     with self.cached_session() as sess:
       a = variables.Variable(12.0)
diff --git a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
index 4cb6cedee9..e0ce06418a 100644
--- a/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/reconstruction_ops_test.py
@@ -66,6 +66,7 @@ class ReconstructionOpsTest(test.TestCase):
 
       self.assertAllClose(output, expected_output)
 
+  @test_util.run_deprecated_v1
   def test_unknown_shapes(self):
     # This test uses placeholders and does not work in eager mode.
     if context.executing_eagerly():
@@ -85,6 +86,7 @@ class ReconstructionOpsTest(test.TestCase):
 
       self.assertAllClose(output, expected_output)
 
+  @test_util.run_deprecated_v1
   def test_unknown_rank(self):
     # This test uses placeholders and does not work in eager mode.
     if context.executing_eagerly():
@@ -104,6 +106,7 @@ class ReconstructionOpsTest(test.TestCase):
 
       self.assertAllClose(output, expected_output)
 
+  @test_util.run_deprecated_v1
   def test_fast_path(self):
     # This test uses tensor names and does not work in eager mode.
     if context.executing_eagerly():
diff --git a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
index 275c86e534..4a967b6562 100644
--- a/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
+++ b/tensorflow/python/kernel_tests/sparse_conditional_accumulator_test.py
@@ -267,7 +267,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       self.assertAllEqual(val.values, [[5, 5], [0, 20], [30, 0]])
       self.assertAllEqual(val.dense_shape, [-1, 2])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelApplyGradMean(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -299,7 +299,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
           np.array([[expected_val, 0], [0, expected_val]]).astype(np.float32),
           val, sess)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelApplyGradSum(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -334,7 +334,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
           np.array([[expected_val, 0], [0, expected_val]]).astype(np.float32),
           val, sess)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testParallelTakeGrad(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -374,7 +374,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
         self._assertEqual_nparray(
             np.array([[0, 0], [elems[i], 0]]), results[i], sess)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAccumulatorApplyAndBlockingTake(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -410,7 +410,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
     with self.assertRaisesOpError("was cancelled"):
       self.evaluate(takeg_op)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAccumulatorCancel(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -430,7 +430,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
 
       takeg_thread.join()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonVectorIndices(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -443,7 +443,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
             grad_indices=[[0, 1], [1, 0]],
             grad_values=np.array([1, 2]).astype(np.float32)).run()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testZeroDimensionValues(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -454,7 +454,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
         q.apply_grad(
             grad_indices=[0], grad_values=np.array(1).astype(np.float32)).run()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWrongNonEmptyInputValues(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -466,7 +466,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
             grad_indices=[0, 1],
             grad_values=np.array([[0, 1, 1]]).astype(np.float32)).run()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDynamicNonVectorIndices(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -486,7 +486,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
                      x_values: np.array([1, 2]).astype(np.float32)
                  })
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDynamicWrongNonEmptyInputValues(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -505,7 +505,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
                      x_values: np.array([[0, 1, 1]]).astype(np.float32)
                  })
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testEmptyShapeApply(self):
     with self.cached_session():
       q = data_flow_ops.SparseConditionalAccumulator(
@@ -531,7 +531,7 @@ class IndexedSlicesConditionalAccumulatorTest(test.TestCase):
       q.apply_grad(grad_indices=[0], grad_values=[1.0], grad_shape=[]).run()
       q.apply_grad(grad_indices=[0], grad_values=[1.0]).run()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testValidateShape(self):
     with self.cached_session() as sess:
       q = data_flow_ops.SparseConditionalAccumulator(
diff --git a/tensorflow/python/kernel_tests/stack_ops_test.py b/tensorflow/python/kernel_tests/stack_ops_test.py
index d50f3f4680..1930d2484f 100644
--- a/tensorflow/python/kernel_tests/stack_ops_test.py
+++ b/tensorflow/python/kernel_tests/stack_ops_test.py
@@ -96,7 +96,7 @@ class StackOpTest(test.TestCase):
           c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()])
       self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStackWhileSwap(self):
     self._testStackWhileSwap(use_gpu=False)
     self._testStackWhileSwap(use_gpu=True)
@@ -248,7 +248,7 @@ class StackOpRefTest(test.TestCase):
           c1, b1, [r, v], [r.get_shape(), tensor_shape.unknown_shape()])
       self.assertAllClose(np.ones(2000) * 10.0, self.evaluate(ry))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStackWhileSwap(self):
     self._testStackWhileSwap(use_gpu=False)
     self._testStackWhileSwap(use_gpu=True)
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index 94155fd117..cfa9f122d1 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import tf2
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import linalg_ops
@@ -39,6 +40,7 @@ def _AddTest(test_class, op_name, testcase_name, fn):
 
 class SvdOpTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testWrongDimensions(self):
     # The input to svd should be a tensor of at least rank 2.
     scalar = constant_op.constant(1.)
@@ -50,6 +52,7 @@ class SvdOpTest(test.TestCase):
                                  "Shape must be at least rank 2 but is rank 1"):
       linalg_ops.svd(vector)
 
+  @test_util.run_v1_only("b/120545219")
   def testConcurrentExecutesWithoutError(self):
     with self.session(use_gpu=True) as sess:
       all_ops = []
@@ -126,6 +129,7 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
     identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0)
     self.assertAllClose(identity, xx, atol=tol)
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     is_complex = dtype_ in (np.complex64, np.complex128)
     is_single = dtype_ in (np.float32, np.complex64)
@@ -214,6 +218,7 @@ def _GetSvdGradOpTest(dtype_, shape_, compute_uv_, full_matrices_):
     tf_v *= phase[..., :n]
     return tf_s, tf_u, tf_v
 
+  @test_util.run_v1_only("b/120545219")
   def Test(self):
     np.random.seed(42)
     a = np.random.uniform(low=-1.0, high=1.0, size=shape_).astype(dtype_)
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 884c04eb7a..88625841bc 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -309,7 +309,7 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArraySplitRead(dtypes.string)
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("v2 does not support TensorArray.grad.")
   def testSkipEagerTensorGradArrayWriteRead(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -364,7 +364,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([2.0], session.run(g2))
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("v2 does not support TensorArray.grad.")
   def testSkipEagerTensorGradArrayDynamicWriteRead(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -407,7 +407,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(3, g_vs)
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("v2 does not support TensorArray.grad.")
   def testSkipEagerTensorGradAccessTwiceReceiveSameObject(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -424,7 +424,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(t_g_ta_0, t_g_ta_1)
       self.assertAllEqual([[4.0, 5.0]], d_r1_0)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTensorArrayWriteWrongIndexOrDataTypeFails(self):
     with self.session(use_gpu=True):
       ta = _make_ta(3, "foo", dtype=dtypes.float32)
@@ -458,7 +458,7 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaisesOpError(error_msg):
         self.evaluate(ta.write(3, 3.0).flow)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTensorArrayReadWrongIndexOrDataTypeFails(self):
     with self.session(use_gpu=True):
       ta = _make_ta(3, "foo", dtype=dtypes.float32)
@@ -493,7 +493,7 @@ class TensorArrayTest(test.TestCase):
         self.evaluate(ta.read(3))
 
   @test_util.disable_control_flow_v2("v2 allows multiple writes.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("v2 allows multiple writes.")
   def testSkipEagerTensorArrayWriteMultipleFails(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -504,7 +504,7 @@ class TensorArrayTest(test.TestCase):
           "it has already been written to."):
         self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTensorArrayConcatIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -536,7 +536,7 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaisesOpError("shape"):
         self.evaluate(w3.concat())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTensorArraySplitIncompatibleShapesFails(self):
     with self.session(use_gpu=True):
       in_eager_mode = context.executing_eagerly()
@@ -611,14 +611,14 @@ class TensorArrayTest(test.TestCase):
         wb1_grad.flow.eval()
 
   @test_util.disable_control_flow_v2("v2 does not support TensorArray.grad.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("v2 does not support TensorArray.grad.")
   def testSkipEagerTensorArrayWriteGradientAddMultipleAdds(self):
     for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
                   dtypes.complex64, dtypes.complex128):
       self._testTensorArrayWriteGradientAddMultipleAdds(dtype)
 
   @test_util.disable_control_flow_v2("Low level legacy TA op test.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("Low level legacy TA op test.")
   def testSkipEagerTensorArrayGradWithShapeKnownElementShape(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -649,7 +649,7 @@ class TensorArrayTest(test.TestCase):
                           sess.run(read_value, feed_dict={value: fed_value}))
 
   @test_util.disable_control_flow_v2("Low level legacy TA op test.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("Low level legacy TA op test.")
   def testSkipEagerTensorArrayGradWithShapeUnknownElementShape(self):
     with self.session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -778,7 +778,7 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayGradientWritePackConcatAndRead()
 
   @test_util.disable_control_flow_v2("v2 does not support clear_after_read.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("v2 does not support clear_after_read.")
   def testTensorArrayReadTwice(self):
     with self.session(use_gpu=True):
       value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
@@ -1351,7 +1351,7 @@ class TensorArrayTest(test.TestCase):
                                     .ENABLE_TENSOR_ARRAY_V2 else v1_msg):
         ta.stack().eval()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSkipEagerTensorArrayEvalEmpty(self):
     self._testTensorArrayEvalEmpty()
 
@@ -1443,7 +1443,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(expected_grad, grad_vals[0])
 
   @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSkipEagerTensorArrayGetsDeviceFromFirstWrite(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       # this initial device will be ignored.
@@ -1493,7 +1493,7 @@ class TensorArrayTest(test.TestCase):
             [s for s in dev_stats[d] if "/TensorArray" in s.node_name])
 
   @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSkipEagerTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2)
@@ -1524,7 +1524,7 @@ class TensorArrayTest(test.TestCase):
             [s for s in dev_stats[d] if "TensorArray" == s.node_name])
 
   @test_util.disable_control_flow_v2("colocate_with not supported in v2.")
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSkipEagerTensorArrayDisabledColocateWithFirstWriteCall(self):
     with ops.device("/job:worker/task:0/cpu:0"):
       ta = tensor_array_ops.TensorArray(
diff --git a/tensorflow/python/kernel_tests/tensordot_op_test.py b/tensorflow/python/kernel_tests/tensordot_op_test.py
index 2894b7dd50..febfe23b16 100644
--- a/tensorflow/python/kernel_tests/tensordot_op_test.py
+++ b/tensorflow/python/kernel_tests/tensordot_op_test.py
@@ -24,6 +24,7 @@ from tensorflow.python import tf2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test as test_lib
@@ -40,6 +41,7 @@ def _add_test(test, test_name, fn):
 
 class TensordotTest(test_lib.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def test_invalid_shape(self):
     a = [[1, 2], [3, 4]]
     b = [[1, 2], [3, 4], [5, 6]]
@@ -63,6 +65,7 @@ class TensordotTest(test_lib.TestCase):
                 axes_ph: (a_axes, b_axes)
             })
 
+  @test_util.run_v1_only("b/120545219")
   def test_invalid_axes(self):
     a = [[1, 2], [3, 4]]
     b = [[1, 2], [3, 4]]
@@ -105,6 +108,7 @@ class TensordotTest(test_lib.TestCase):
         self.assertAllEqual(tf_ans.shape, np_ans.shape)
         self.assertAllEqual(tf_ans, np_ans)
 
+  @test_util.run_v1_only("b/120545219")
   def test_partial_shape_inference(self):
     for axes in ([1], [0]), 1:
       a = array_ops.placeholder(dtypes.float32)
diff --git a/tensorflow/python/kernel_tests/unicode_encode_op_test.py b/tensorflow/python/kernel_tests/unicode_encode_op_test.py
index a5a5c2017c..72c8a26766 100644
--- a/tensorflow/python/kernel_tests/unicode_encode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_encode_op_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl as errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_string_ops
 from tensorflow.python.platform import test
@@ -58,6 +59,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
         ragged_string_ops.unicode_encode(test_value, encoding, "strict").eval()
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def testIgnoreErrors(self, encoding):
     test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
     expected_value = u"Heo".encode(encoding)
@@ -69,6 +71,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def testReplaceErrors(self, encoding):
     test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
     expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding)
@@ -109,6 +112,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
   # -- regular Tensor tests -- #
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def testVector(self, encoding):
     test_value = np.array([72, 101, 108, 108, 111], np.int32)
     expected_value = u"Hello".encode(encoding)
@@ -144,6 +148,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def testMatrix(self, encoding):
     test_value = np.array(
         [[72, 128516, 108, 108, 111], [87, 128516, 114, 108, 100]], np.int32)
@@ -157,6 +162,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def test3DimMatrix(self, encoding):
     test_value = constant_op.constant(
         [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]],
@@ -172,6 +178,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def test4DimMatrix(self, encoding):
     test_value = constant_op.constant(
         [[[[72, 101, 108, 108, 111]], [[87, 111, 114, 108, 100]]],
@@ -192,6 +199,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
   # -- Ragged Tensor tests -- #
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def testRaggedMatrix(self, encoding):
     test_value = ragged_factory_ops.constant(
         [[72, 195, 108, 108, 111], [87, 128516, 114, 108, 100, 46]], np.int32)
@@ -205,6 +213,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def test3DimMatrixWithRagged2ndDim(self, encoding):
     test_value = ragged_factory_ops.constant(
         [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]],
@@ -224,6 +233,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result.tolist(), expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def test3DimMatrixWithRagged3rdDim(self, encoding):
     test_value = ragged_factory_ops.constant(
         [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]],
@@ -241,6 +251,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result.tolist(), expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def test3DimMatrixWithRagged2ndAnd3rdDim(self, encoding):
     test_value = ragged_factory_ops.constant(
         [[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100, 46]], [],
@@ -254,6 +265,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result.tolist(), expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def test4DimRaggedMatrix(self, encoding):
     test_value = ragged_factory_ops.constant(
         [[[[72, 101, 108, 108, 111], [87, 111, 114, 108, 100]]],
@@ -267,6 +279,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
       self.assertAllEqual(result.tolist(), expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
+  @test_util.run_v1_only("b/120545219")
   def testRaggedMatrixWithMultiDimensionInnerValues(self, encoding):
     test_inner_values = constant_op.constant([[[72, 101, 108, 108, 111],
                                                [87, 111, 114, 108, 100]],
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 08d885e8a8..336e9b0bca 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -43,7 +43,7 @@ from tensorflow.python.util import compat
 
 class VariablesTestCase(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitialization(self):
     with self.cached_session():
       var0 = variables.VariableV1(0.0)
@@ -71,7 +71,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(0.0, self.evaluate(var0))
       self.assertAllClose(1.1, self.evaluate(var1))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitializationOrder(self):
     with self.cached_session():
       rnd = variables.Variable(random_ops.random_uniform([3, 6]), name="rnd")
@@ -194,7 +194,7 @@ class VariablesTestCase(test.TestCase):
   def testCountUpToInt64(self):
     self._countUpToTest(dtypes.int64)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testControlDepsNone(self):
     with self.cached_session():
       c = constant_op.constant(1.0)
@@ -208,7 +208,7 @@ class VariablesTestCase(test.TestCase):
       self.assertEqual([], var_x.value().op.control_inputs)
       self.assertEqual([], var_x._ref().op.control_inputs)  # pylint: disable=protected-access
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testControlFlow(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(0, name="v0")
@@ -245,7 +245,7 @@ class VariablesTestCase(test.TestCase):
       self.evaluate(v0.initializer)
       self.evaluate(add)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testControlFlowInitialization(self):
     """Expects an error if an initializer is in a control-flow scope."""
     def cond(i, _):
@@ -412,7 +412,7 @@ class VariablesTestCase(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertAllClose([1, 12], self.evaluate(var))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testColocation(self):
     with ops.device("/job:ps"):
       var = variables.VariableV1(0, name="v")
@@ -421,7 +421,7 @@ class VariablesTestCase(test.TestCase):
     self.assertDeviceEqual("/job:ps", assign_op.device)
     self.assertEqual([b"loc:@v"], assign_op.op.colocation_groups())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitializerFunction(self):
     value = [[-42], [133.7]]
     shape = [2, 1]
@@ -459,7 +459,7 @@ class VariablesTestCase(test.TestCase):
           lambda: constant_op.constant(1.),
           constraint=constraint)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNoRefDataRace(self):
     with self.cached_session():
       a = variables.Variable([1, 2, 3], dtype=dtypes.float32)
@@ -489,7 +489,7 @@ class VariablesTestCase(test.TestCase):
       for i in v2.initializer.inputs:
         self.assertEqual(expected_group_v2, i.op.colocation_groups())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariableDefInitializedInstances(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v_def = variables.Variable(
@@ -542,7 +542,7 @@ class VariablesTestCase(test.TestCase):
 
       self.assertAllClose(np.ones((5, 5), np.float32), self.evaluate(var))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRepr(self):
     var = variables.VariableV1(np.zeros((5, 5), np.float32), name="noop")
     self.assertEqual(
@@ -576,7 +576,7 @@ class IsInitializedTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.assertEqual(0, self.evaluate(uninited).size)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariableList(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2], name="v")
@@ -614,7 +614,7 @@ class ObsoleteIsInitializedTest(test.TestCase):
     with ops.Graph().as_default():
       self.assertEqual(None, variables.assert_variables_initialized())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariables(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2])
@@ -626,7 +626,7 @@ class ObsoleteIsInitializedTest(test.TestCase):
       variables.global_variables_initializer().run()
       self.evaluate(inited)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariableList(self):
     with ops.Graph().as_default(), self.cached_session() as sess:
       v = variables.VariableV1([1, 2])
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index d155544f3e..cae459a34e 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -53,6 +53,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(self.evaluate(ret), 16.)
       self.assertSequenceEqual(self.evaluate(grad), [32.])
 
+  @test_util.run_v1_only("b/120545219")
   def testReturnSameStructureTrue(self):
     x = constant_op.constant(2.)
     ret = while_loop_v2(
@@ -145,7 +146,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertSequenceEqual(self.evaluate(grad), [32.])
       self.assertSequenceEqual(self.evaluate(grad_grad), [48.])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPruning(self):
     x = constant_op.constant(1)
 
@@ -441,6 +442,7 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
     # grad = gradients_impl.gradients(output, [n])
     # self.assertEqual(self.evaluate(grad), 3.5)
 
+  @test_util.run_deprecated_v1
   def testForwardPassRewrite(self):
     x = constant_op.constant(1.0, name="x")
     output = while_v2.while_loop(lambda x: x < 10.0,
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index 07d8e40b75..6535f74129 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -38,6 +38,7 @@ from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import saver as saver_lib
 
 
+@test_util.run_v1_only('b/120545219')
 class BNTest(test.TestCase):
 
   def _simple_model(self, image, fused, freeze_mode):
@@ -144,7 +145,6 @@ class BNTest(test.TestCase):
 
     return train_vars, loss_val
 
-  @test_util.run_deprecated_v1
   def testHalfPrecision(self):
     ref_vars, ref_loss = self._trainEvalSequence(
         dtype=dtypes.float32,
@@ -230,43 +230,33 @@ class BNTest(test.TestCase):
                                ckpt_b_use_gpu, use_gpu_test_a, use_gpu_test_b,
                                freeze_mode)
 
-  @test_util.run_deprecated_v1
   def testCheckpointFusedCPUAndFusedGPU(self):
     self._testCheckpointCrossDevice(True, False, True, True)
 
-  @test_util.run_deprecated_v1
   def testCheckpointFusedCPUAndFusedCPU(self):
     self._testCheckpointCrossDevice(True, False, True, False)
 
-  @test_util.run_deprecated_v1
   def testCheckpointFusedGPUAndFusedGPU(self):
     self._testCheckpointCrossDevice(True, True, True, True)
 
-  @test_util.run_deprecated_v1
   def testCheckpointNonFusedCPUAndNonFusedGPU(self):
     self._testCheckpointCrossDevice(False, False, False, True)
 
-  @test_util.run_deprecated_v1
   def testCheckpointNonFusedCPUAndNonFusedCPU(self):
     self._testCheckpointCrossDevice(False, False, False, False)
 
-  @test_util.run_deprecated_v1
   def testCheckpointNonFusedGPUAndNonFusedGPU(self):
     self._testCheckpointCrossDevice(False, True, False, True)
 
-  @test_util.run_deprecated_v1
   def testCheckpointNonFusedGPUAndFusedGPU(self):
     self._testCheckpointCrossDevice(False, True, True, True)
 
-  @test_util.run_deprecated_v1
   def testCheckpointNonFusedGPUAndFusedCPU(self):
     self._testCheckpointCrossDevice(False, True, True, False)
 
-  @test_util.run_deprecated_v1
   def testCheckpointNonFusedCPUAndFusedCPU(self):
     self._testCheckpointCrossDevice(False, False, True, False)
 
-  @test_util.run_deprecated_v1
   def testCreateBN(self):
     # Call layer.
     bn = normalization_layers.BatchNormalization(axis=1)
@@ -293,7 +283,6 @@ class BNTest(test.TestCase):
         ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
         bn.trainable_variables)
 
-  @test_util.run_deprecated_v1
   def testCreateFusedBNFloat16(self):
     # Call layer.
     bn = normalization_layers.BatchNormalization(axis=1, fused=True)
@@ -323,7 +312,6 @@ class BNTest(test.TestCase):
         ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
         bn.trainable_variables)
 
-  @test_util.run_deprecated_v1
   def test3DInputAxis1(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -367,7 +355,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def test3DInputAxis2(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -451,7 +438,6 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def test4DInputAxis2(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -493,7 +479,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def test4DInputAxis3(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -535,7 +520,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def test4DInputAxis3Fused(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -619,7 +603,6 @@ class BNTest(test.TestCase):
         self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
         self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def testNegativeAxis(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -662,7 +645,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def testBooleanLearningPhase(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -703,7 +685,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def testFunctionalNoReuse(self):
     inputs = variables.Variable(
         np.random.random((5, 4, 3, 6)), dtype=dtypes.float32)
@@ -756,7 +737,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def testFunctionalReuse(self):
     inputs1 = variables.Variable(
         np.random.random((5, 4, 3, 6)), dtype=dtypes.float32)
@@ -821,7 +801,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=2)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def testFunctionalReuseFromScope(self):
     inputs = variables.Variable(
         np.random.random((5, 4, 3, 6)), dtype=dtypes.float32)
@@ -836,7 +815,6 @@ class BNTest(test.TestCase):
           inputs, axis=-1, momentum=0.9, epsilon=epsilon, training=training)
       self.assertEqual(len(variables.global_variables()), 5)
 
-  @test_util.run_deprecated_v1
   def testNoCenter(self):
     bn = normalization_layers.BatchNormalization(axis=1, center=False)
     inputs = random_ops.random_uniform((5, 4, 3), seed=1)
@@ -852,7 +830,6 @@ class BNTest(test.TestCase):
     self.assertEqual(len(bn.trainable_variables), 1)
     self.assertEqual(len(bn.non_trainable_variables), 2)
 
-  @test_util.run_deprecated_v1
   def testNoScale(self):
     bn = normalization_layers.BatchNormalization(axis=1, scale=False)
     inputs = random_ops.random_uniform((5, 4, 3), seed=1)
@@ -868,7 +845,6 @@ class BNTest(test.TestCase):
     self.assertEqual(len(bn.trainable_variables), 1)
     self.assertEqual(len(bn.non_trainable_variables), 2)
 
-  @test_util.run_deprecated_v1
   def testRegularizers(self):
     reg = lambda x: 0.1 * math_ops.reduce_sum(x)
     bn = normalization_layers.BatchNormalization(axis=1, beta_regularizer=reg)
@@ -894,7 +870,6 @@ class BNTest(test.TestCase):
     self.assertEqual(bn.gamma_constraint, g_constraint)
     self.assertEqual(bn.beta_constraint, b_constraint)
 
-  @test_util.run_deprecated_v1
   def testRenorm(self):
     shape = (4, 3)
     xt = array_ops.placeholder(dtypes.float32, shape)
@@ -953,7 +928,6 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, yt_val_train, atol=1e-5)
         self.assertAllClose(y_test, yt_val_test, atol=1e-5)
 
-  @test_util.run_deprecated_v1
   def testAdjustment(self):
     shape = (4, 3)
     xt = array_ops.placeholder(dtypes.float32, shape)
@@ -998,7 +972,6 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, yt_val_train, atol=1e-5)
         self.assertAllClose(y_test, yt_val_test, atol=1e-5)
 
-  @test_util.run_deprecated_v1
   def testRenormWithAdjustment(self):
     shape = (4, 3)
     xt = array_ops.placeholder(dtypes.float32, shape)
@@ -1069,7 +1042,6 @@ class BNTest(test.TestCase):
       normalization_layers.batch_normalization(
           inp, virtual_batch_size=-1)
 
-  @test_util.run_deprecated_v1
   def testGhostBNVirtualBatchFull(self):
     shape = [6, 5, 4, 3]
     inp = random_ops.random_uniform(shape, seed=1)
@@ -1095,7 +1067,6 @@ class BNTest(test.TestCase):
         inp, virtual_batch_size=3)
     self.assertListEqual(out.shape.as_list(), shape)
 
-  @test_util.run_deprecated_v1
   def testGhostBNUnknownBatchSize(self):
     np_shape = [10, 5, 4]
     tf_shape = [None, 5, 4]
@@ -1111,7 +1082,6 @@ class BNTest(test.TestCase):
 
       self.assertListEqual(list(y.shape), np_shape)
 
-  @test_util.run_deprecated_v1
   def testGhostBN2Dims(self):
     shape = [6, 2]
     virtual_batch_size = 3
@@ -1165,7 +1135,6 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, y_val_train, atol=1e-5)
         self.assertAllClose(y_test, y_val_test, atol=1e-5)
 
-  @test_util.run_deprecated_v1
   def testGhostBN4DimsAxis3(self):
     shape = [6, 10, 10, 3]
     virtual_batch_size = 2
@@ -1219,7 +1188,6 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, y_val_train, atol=1e-2)
         self.assertAllClose(y_test, y_val_test, atol=1e-2)
 
-  @test_util.run_deprecated_v1
   def testGhostBN4DimsAxis1(self):
     shape = [6, 3, 10, 10]
     virtual_batch_size = 2
@@ -1290,7 +1258,6 @@ class BNTest(test.TestCase):
       normalization_layers.batch_normalization(
           inp, axis=[1, 2, 1])   # duplicate
 
-  @test_util.run_deprecated_v1
   def test3DInputMultiAxis12(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -1332,7 +1299,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def test5DInputMultiAxis123(self):
     epsilon = 1e-3
     bn = normalization_layers.BatchNormalization(
@@ -1374,7 +1340,6 @@ class BNTest(test.TestCase):
       self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
       self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
 
-  @test_util.run_deprecated_v1
   def testGhostBN5DimsMultiAxis14(self):
     shape = [6, 3, 10, 10, 4]
     virtual_batch_size = 3
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index a9058c4a34..abdcbc7a3a 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -158,6 +158,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       grads = gradients.gradients(z, [x])
       self.assertTrue(all(x is not None for x in grads))
 
+  @test_util.run_v1_only("b/120545219")
   def testBoundaryContinue(self):
     # Test that we differentiate both 'x' and 'y' correctly when x is a
     # predecessor of y.
@@ -169,6 +170,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       self.assertTrue(all(x is not None for x in grads))
       self.assertEqual(6.0, grads[0].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testAggregationMethodAccumulateN(self):
     with self.cached_session():
       x = constant(1.0)
@@ -182,6 +184,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       self.assertEqual(20.0, grads[0].eval())
       self.assertEqual(10.0, grads[1].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testAggregationMethodAddN(self):
     with self.cached_session():
       x = constant(1.0)
@@ -193,6 +196,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       self.assertEqual(20.0, grads[0].eval())
       self.assertEqual(10.0, grads[1].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testAggregationMethodTree(self):
     with self.cached_session():
       x = constant(1.0)
@@ -239,6 +243,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
             [dx, dy], feed_dict={x: [1.0], dy.indices: [0], dy.values: [2.0]})
       self.assertEqual(vdx, vdy)
 
+  @test_util.run_v1_only("b/120545219")
   def testNonDifferentiableSwitchInWhileLoop(self):
     with ops.Graph().as_default():
       v = array_ops.placeholder(dtypes.float32, [])
@@ -270,6 +275,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       gradient = gradients.gradients(graph.as_graph_element(var), var)
       self.assertIsNotNone(gradient)
 
+  @test_util.run_v1_only("b/120545219")
   def testVariableRefGradient(self):
     with ops.Graph().as_default():
       init = constant_op.constant(100.0)
@@ -277,6 +283,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       gradient = gradients.gradients(var._ref(), var)
       self.assertIsNotNone(gradient)
 
+  @test_util.run_v1_only("b/120545219")
   def testDependentYs(self):
     with self.cached_session():
       x = constant_op.constant(3.0)
@@ -292,6 +299,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       g = gradients.gradients([z, z2], x)
       self.assertAllClose(17502.0, g[0].eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testPartialDerivatives(self):
     with self.cached_session():
       x = constant_op.constant(1.)
@@ -302,6 +310,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
       partialg = gradients.gradients(z, [x, y], stop_gradients=[x, y])
       self.assertEqual([1.0, 1.0], [g.eval() for g in partialg])
 
+  @test_util.run_v1_only("b/120545219")
   def testStopGradients(self):
     def _MakeGraph(rng, stop_gradients=()):
       def _FunctionOf(xs, k=3):
@@ -606,6 +615,7 @@ class PreventGradientTest(test_util.TensorFlowTestCase):
 
 class HessianVectorProductTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testHessianVectorProduct(self):
     # Manually compute the Hessian explicitly for a low-dimensional problem
     # and check that HessianVectorProduct matches multiplication by the
@@ -634,6 +644,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase):
 
 class HessianTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testHessian1D(self):
     # Manually compute the Hessian explicitly for a low-dimensional problem
     # and check that `hessian` matches. Specifically, the Hessian of
@@ -651,6 +662,7 @@ class HessianTest(test_util.TensorFlowTestCase):
       hess_actual = self.evaluate(hess)
     self.assertAllClose(hess_value, hess_actual)
 
+  @test_util.run_v1_only("b/120545219")
   def testHessian1D_multi(self):
     # Test the computation of the hessian with respect to multiple tensors
     m = 4
@@ -671,6 +683,7 @@ class HessianTest(test_util.TensorFlowTestCase):
     for hess_value, hess_actual in zip(hess_values, hessians_actual):
       self.assertAllClose(hess_value, hess_actual)
 
+  @test_util.run_v1_only("b/120545219")
   def testHessianInvalidDimension(self):
     for shape in [(10, 10), None]:
       with self.cached_session(use_gpu=True):
@@ -679,6 +692,7 @@ class HessianTest(test_util.TensorFlowTestCase):
         with self.assertRaises(ValueError):
           gradients.hessians(x, x)
 
+  @test_util.run_v1_only("b/120545219")
   def testHessian2D_square_matrix(self):
     # Manually compute the Hessian explicitly for a low-dimensional problem
     # and check that `hessian` matches. Specifically, the Hessian of
@@ -700,6 +714,7 @@ class HessianTest(test_util.TensorFlowTestCase):
     self.assertAllEqual((m, m, m, m), hess_actual.shape)
     self.assertAllClose(hess_value, hess_actual.reshape((m * m, m * m)))
 
+  @test_util.run_v1_only("b/120545219")
   def testHessian2D_non_square_matrix(self):
     m = 3
     n = 4
@@ -722,6 +737,7 @@ class HessianTest(test_util.TensorFlowTestCase):
 
 class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testIndexedSlicesToTensor(self):
     with self.cached_session():
       np_val = np.random.rand(4, 4, 4, 4).astype(np.float32)
@@ -731,6 +747,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
       c_dense = math_ops.multiply(c_sparse, 1.0)
       self.assertAllClose(np_val, self.evaluate(c_dense))
 
+  @test_util.run_v1_only("b/120545219")
   def testIndexedSlicesToTensorList(self):
     with self.cached_session():
       numpy_list = []
@@ -747,6 +764,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
       packed_sparse = array_ops.stack(sparse_list)
       self.assertAllClose(packed_dense.eval(), self.evaluate(packed_sparse))
 
+  @test_util.run_v1_only("b/120545219")
   def testInt64Indices(self):
     with self.cached_session():
       np_val = np.random.rand(4, 4, 4, 4).astype(np.float32)
@@ -759,6 +777,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
       c_dense = math_ops.multiply(c_sparse, 1.0)
       self.assertAllClose(np_val, self.evaluate(c_dense))
 
+  @test_util.run_v1_only("b/120545219")
   def testWarnings(self):
     # TODO(gunan) Reenable after this issue is fixed:
     # https://github.com/google/protobuf/issues/2812
@@ -802,6 +821,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
 
 class OnlyRealGradientsTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testRealOnly(self):
     x = constant_op.constant(7+3j, dtype=dtypes.complex64)
     y = math_ops.square(x)
@@ -814,6 +834,7 @@ class OnlyRealGradientsTest(test_util.TensorFlowTestCase):
 
 class ResourceCondTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testBasic(self):
     gamma = resource_variable_ops.ResourceVariable(
         np.random.random((3,)),
@@ -943,6 +964,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       self.assertEqual(6., math_ops.reduce_sum(dx).numpy())
       self.assertEqual(8., math_ops.reduce_sum(dw).numpy())
 
+  @test_util.run_v1_only("b/120545219")
   def testCustomGradientErrorsWithNonResourceVariables(self):
 
     def F(x, use_resource=False):
@@ -993,6 +1015,7 @@ class CustomGradientTest(test_util.TensorFlowTestCase):
       # Smoke test to ensure numpy inputs are accepted
       F(x)
 
+  @test_util.run_v1_only("b/120545219")
   def testRVGradientsDynamicCond(self):
     with self.cached_session():
       alpha = resource_variable_ops.ResourceVariable(
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 414a5cdbbe..82fab74183 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -450,6 +450,7 @@ class DropoutTest(test_lib.TestCase):
     with self.assertRaises(ValueError):
       nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2]))
 
+  @test_util.run_deprecated_v1
   def testInvalidRate(self):
     x_dim = 40
     y_dim = 30
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
index d9d840500c..7fe185641f 100644
--- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
@@ -135,7 +135,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
           expected=ragged.constant_value(
               [[[[b'c', b'a'], [b'd', b'd']], [[b'f', b'e']]]], ragged_rank=2)),
   ])
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedBatchGather(self, descr, params, indices, expected):
     result = ragged.batch_gather(params, indices)
     self.assertEqual(
@@ -188,7 +188,7 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
            indices=[[[0]]],
            message='batch shape from indices does not match params shape'),
   ])
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedBatchGatherStaticError(self,
                                        params,
                                        indices,
diff --git a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
index d939d9d634..763b016405 100644
--- a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
@@ -298,7 +298,7 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase,
           keepdims=True,
           expected=ragged.constant_value([[[1], [4, 6]], [[7, 9], []]])),
   ])  # pyformat: disable
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testBooleanMask(self, descr, data, mask, keepdims, expected):
     actual = ragged.boolean_mask(data, mask, keepdims=keepdims)
     self.assertEqual(
diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
index 3699f90f46..ba7867418a 100644
--- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
@@ -221,7 +221,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=[[b'a00', b'a01'], [], [b'a20', b'a21']]),
   )   # pyformat: disable
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedConcat(self,
                        descr,
                        rt_inputs,
@@ -280,7 +280,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           error=errors.InvalidArgumentError,
           message='Input tensors have incompatible shapes'),
   ])
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRuntimeError(self, rt_inputs, axis, error, message,
                        ragged_ranks=None):
     rt_inputs = [
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 2bb10adce0..2533c60c4e 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -126,6 +126,7 @@ BINARY_INT_OPS = [
 ]
 
 
+@test_util.run_v1_only('b/120545219')
 class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
                                parameterized.TestCase):
 
@@ -401,7 +402,6 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
         result_flat_values = array_ops.reshape(result, [-1])
       self.assertAllEqual(expected_flat_values, result_flat_values)
 
-  @test_util.run_deprecated_v1
   def testElementwiseOpUnknownRankError(self):
     x = ragged.constant([[1, 2], [3]])
     y = ragged.from_row_splits(
diff --git a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
index 3c0db9e8fb..674dbab112 100644
--- a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
@@ -29,7 +29,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testDocStringExample(self):
     st = sparse_tensor.SparseTensor(
         indices=[[0, 0], [0, 1], [0, 2], [1, 0], [3, 0]],
@@ -40,7 +40,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     with self.test_session():
       self.assertEqual(rt.eval().tolist(), [[1, 2, 3], [4], [], [5]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testEmpty(self):
     st = sparse_tensor.SparseTensor(
         indices=array_ops.zeros([0, 2], dtype=dtypes.int64),
@@ -67,7 +67,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
                             ragged.from_sparse, st3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testGoodPartialSparseTensorRank(self):
     st1 = sparse_tensor.SparseTensor(
         indices=[[0, 0]],
@@ -82,7 +82,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     ragged.from_sparse(st1)
     ragged.from_sparse(st2)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testNonRaggedSparseTensor(self):
     # "index_suffix" means the value of the innermost dimension of the index
     # (i.e., indices[i][-1]).
diff --git a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
index 1d8a00cc18..09c17fad61 100644
--- a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
@@ -31,7 +31,7 @@ from tensorflow.python.platform import googletest
 class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                              parameterized.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testDocStringExamples(self):
     # The examples from ragged.from_tensor.__doc__.
     dt = constant_op.constant([[5, 7, 0], [0, 3, 0], [6, 0, 0]])
@@ -263,7 +263,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                        [[[5, 6], [7]], [[0, 8], []]]]
       },
   )  # pyformat: disable
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedFromTensor(self,
                            tensor,
                            expected,
@@ -398,7 +398,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
           'expected': [[], []]
       },
   )
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testEmpty(self, dt_shape, expected, lengths=None, padding=None):
     dt = array_ops.zeros(dt_shape)
     rt = ragged.from_tensor(dt, lengths, padding)
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index 62c6819374..b0a4fe9d31 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import ragged
 from tensorflow.python.platform import googletest
 
 
+@test_util.run_v1_only('b/120545219')
 class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
                            parameterized.TestCase):
 
@@ -183,7 +184,6 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
           indices=[[0, 0, 1], [0, 0, 0], [0, 1, 0]],
           expected=[[b'c', b'd'], [b'a', b'b'], [b'e', b'f']]),
   ])  # pyformat: disable
-  @test_util.run_deprecated_v1
   def testRaggedGatherNd(self, descr, params, indices, expected):
     result = ragged.gather_nd(params, indices)
     self.assertEqual(
@@ -193,7 +193,6 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
         expected = expected.tolist()
       self.assertEqual(self.evaluate(result).tolist(), expected)
 
-  @test_util.run_deprecated_v1
   def testRaggedGatherNdUnknownRankError(self):
     params = ragged.constant([['a', 'b'], ['c', 'd']])
     indices1 = array_ops.placeholder(dtypes.int32, shape=None)
@@ -221,7 +220,6 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
           indices=ragged.constant([[0]]),
           message='The innermost dimension of indices may not be ragged'),
   ])
-  @test_util.run_deprecated_v1
   def testRaggedGatherNdStaticError(self,
                                     params,
                                     indices,
diff --git a/tensorflow/python/ops/ragged/ragged_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
index 76c90cdfee..d371745b73 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
@@ -30,7 +30,7 @@ from tensorflow.python.platform import googletest
 
 class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testDocStringExamples(self):
     params = constant_op.constant(['a', 'b', 'c', 'd', 'e'])
     indices = constant_op.constant([3, 1, 2, 1, 0])
@@ -66,7 +66,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
           ragged.gather(params, indices).eval().tolist(),
           [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testTensorParamsAndRaggedIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = ragged.constant([[2, 1], [1, 2, 0], [3]])
@@ -75,7 +75,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
           ragged.gather(params, indices).eval().tolist(),
           [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedParamsAndRaggedIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = ragged.constant([[2, 1], [1, 2, 0], [3]])
@@ -108,7 +108,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
            [[[b'g']], [[b'g']]]]                                  #  [p2, p2]]
       )  # pyformat: disable
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testTensorParamsAnd4DRaggedIndices(self):
     indices = ragged.constant(
         [[[[3, 4], [0, 6]], []], [[[2, 1], [1, 0]], [[2, 5]], [[2, 3]]],
@@ -123,7 +123,7 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
            [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]],
            [[[b'b', b'a']]]])  # pyformat: disable
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOutOfBoundsError(self):
     tensor_params = ['a', 'b', 'c']
     tensor_indices = [0, 1, 2]
diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
index ecd78a91b2..ab70d5a123 100644
--- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import googletest
 
 
+@test_util.run_v1_only('b/120545219')
 class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   @parameterized.parameters([
       # The following test sets map over a RaggedTensor and apply a
@@ -141,7 +142,6 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       ),
   ])
 
-  @test_util.run_deprecated_v1
   def testRaggedMap(
       self,
       fn,
@@ -166,7 +166,6 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       output_values = self.evaluate(output)
       self.assertAllEqual(expected_output, output_values.tolist())
 
-  @test_util.run_deprecated_v1
   def testRaggedMapOnStructure(self):
     batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]])
     # [[10, 20, 30], [40], [50, 60, 70]]
@@ -187,7 +186,6 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(output.eval().tolist(), [66, 44, 198])
 
   # Test mapping over a dict of RTs can produce a dict of RTs.
-  @test_util.run_deprecated_v1
   def testRaggedMapOnStructure_RaggedOutputs(self):
     batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]])
     # [[10, 20, 30], [40], [50, 60, 70]]
@@ -219,7 +217,6 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(output['robin'].eval().tolist(),
                           [[11, 21, 31], [41], [51, 61, 71]])
 
-  @test_util.run_deprecated_v1
   def testZip(self):
     x = ragged.constant([[10, 20], [30, 40], [50, 60], [70], [80, 90, 100]],
                         dtypes.int64)
@@ -242,7 +239,6 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           result, [[[0, 10], [0, 20]], [[1, 30], [1, 40]], [[2, 50], [2, 60]],
                    [[3, 70]], [[4, 80], [4, 90], [4, 100]]])
 
-  @test_util.run_deprecated_v1
   def testBatchGather(self):
     tokens = ragged.constant([['hello', '.', 'there'], ['merhaba'],
                               ['bonjour', '.', 'ca va', '?']])
diff --git a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py b/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
index b5802cb82d..dc214f4437 100644
--- a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
@@ -43,7 +43,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
     with self.test_session():
       self.assertEqual(result.eval().tolist(), expected)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testDocStringExamples(self):
     """Test the examples in apply_op_to_ragged_values.__doc__."""
     rt = ragged.constant([[1, 2, 3], [], [4, 5], [6]])
@@ -63,7 +63,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(tensor,),
         expected=[[0, 0, 0], [], [0, 0]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithTwoRaggedTensorArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -76,7 +76,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
     self.assertRaggedMapInnerValuesReturns(
         op=math_ops.multiply, args=(5, y), expected=[[5, 10, 15], [], [20, 25]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithThreeRaggedTensorArgs(self):
     condition = ragged.constant(
         [[True, True, False], [], [True, False]])  # pyformat: disable
@@ -87,7 +87,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(condition, x, y),
         expected=[[b'a', b'b', b'C'], [], [b'd', b'E']])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithRaggedTensorListArg(self):
     x = ragged.constant([[1, 2, 3], [], [4, 5]])
     y = ragged.constant([[10, 20, 30], [], [40, 50]])
@@ -96,7 +96,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=([x, y, x],),
         expected=[[12, 24, 36], [], [48, 60]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithKeywordArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -105,7 +105,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         kwargs=dict(x=x, y=y),
         expected=[[3, 2, 12], [], [4, 25]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithMixedPositionalAndKeywordArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -128,7 +128,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         },
         expected=[[8, 15, 13], [], [16, 25]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithRaggedRankGreaterThanOne(self):
     # ragged_rank=0
     x0 = [3, 1, 4, 1, 5, 9, 2, 6, 5]
@@ -173,7 +173,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
             [[[54, 14], [48, 45]]]    # row 3
         ])  # pyformat: disable
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testOpWithRaggedRankThree(self):
     x = ragged.constant([[[3, 1, 4]], [], [[], [1, 5]]])
     y = ragged.constant([[[1, 2, 3]], [], [[], [4, 5]]])
@@ -203,7 +203,7 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
                             r'Inputs must have identical ragged splits.*',
                             ragged.map_inner_values, math_ops.add, x, y)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorSplitsMismatchErrorAtRuntime(self):
     splits1 = array_ops.placeholder_with_default(
         constant_op.constant([0, 3, 3, 5], dtypes.int64), None)
diff --git a/tensorflow/python/ops/ragged/ragged_operators_test.py b/tensorflow/python/ops/ragged/ragged_operators_test.py
index 7fe8159d82..c1223db307 100644
--- a/tensorflow/python/ops/ragged/ragged_operators_test.py
+++ b/tensorflow/python/ops/ragged/ragged_operators_test.py
@@ -23,11 +23,11 @@ from tensorflow.python.ops import ragged
 from tensorflow.python.platform import googletest
 
 
+@test_util.run_v1_only('b/120545219')
 class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
   # @TODO(edloper): Test right-handed versions of operators once we add
   # broadcasting support for elementwise ops.
 
-  @test_util.run_deprecated_v1
   def testOrderingOperators(self):
     x = ragged.constant([[1, 5], [3]])
     y = ragged.constant([[4, 5], [1]])
@@ -41,7 +41,6 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
     if a != b:
       print('%30s %s' % (b, a))
 
-  @test_util.run_deprecated_v1
   def testArithmeticOperators(self):
     x = ragged.constant([[1.0, -2.0], [8.0]])
     y = ragged.constant([[4.0, 4.0], [2.0]])
@@ -77,7 +76,6 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual((2.0 % y).eval().tolist(), [[2.0, 2.0], [0.0]])
       self.assertEqual((x % 2.0).eval().tolist(), [[1.0, 0.0], [0.0]])
 
-  @test_util.run_deprecated_v1
   def testLogicalOperators(self):
     a = ragged.constant([[True, True], [False]])
     b = ragged.constant([[True, False], [False]])
diff --git a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
index 9f51d59ba3..2e4db2a423 100644
--- a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
@@ -300,7 +300,7 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=2,
           expected=[[mean(1, 2), mean(3, 4, 5)], [mean(6, 7), 8], [9]]),
   )
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testReduce(self, ragged_reduce_op, rt_input, axis, expected):
     rt_input = ragged.constant(rt_input)
     reduced = ragged_reduce_op(rt_input, axis)
diff --git a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
index 4a705be484..9f4aa1b136 100644
--- a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
@@ -143,7 +143,7 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
           expected=[[2, 3, 0], [4, 1]],
           expected_ragged_rank=1),
   ])  # pyformat: disable
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRowLengths(self,
                      rt_input,
                      expected,
diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
index 9e4877ae3e..52fe739342 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
@@ -182,7 +182,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                  []]  # pyformat: disable
     self.assertEqual(self.evaluate(segmented2).tolist(), expected2)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedSegmentIds(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121],],  # row 0
@@ -205,7 +205,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
         'but segment_ids is ragged and data is not.', ragged.segment_sum, dt,
         segment_ids, 3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testShapeMismatchError2(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121]],  # row 0
diff --git a/tensorflow/python/ops/ragged/ragged_stack_op_test.py b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
index 4343471694..ed07aca6d4 100644
--- a/tensorflow/python/ops/ragged/ragged_stack_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
@@ -265,7 +265,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=[[[b'a00', b'a01'], [], [b'a20', b'a21']]]),
   )   # pyformat: disable
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedStack(self,
                       descr,
                       rt_inputs,
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
index 9c2dd26050..699dcc2bdb 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
@@ -82,6 +82,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
           value=ragged.constant_value([[[1, 2], [3]], [[4, 5]]]),
           expected_dim_sizes=[2, [2, 1], [2, 1, 2]]),
   ])
+  @test_util.run_v1_only('b/120545219')
   def testFromTensor(self, value, expected_dim_sizes):
     shape = ragged.RaggedTensorDynamicShape.from_tensor(value)
     expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(
@@ -105,6 +106,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
           rank=5,
           expected_dim_sizes=[1, 3, [3, 2, 4], 2, 3]),
   ])
+  @test_util.run_v1_only('b/120545219')
   def testBroadcastToRank(self, dim_sizes, rank, expected_dim_sizes):
     shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes)
     expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(
@@ -281,6 +283,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
            original_dim_sizes=[2, (2, 1), 2, 1],
            broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
   ])  # pyformat: disable
+  @test_util.run_v1_only('b/120545219')
   def testBroadcastDimension(self, axis, row_length, original_dim_sizes,
                              broadcast_dim_sizes):
     """Tests for the broadcast_dimension method.
@@ -370,6 +373,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
               y_dims=[1, 1, 2, (2, 1)],
               expected_dims=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
       ])
+  @test_util.run_v1_only('b/120545219')
   def testBroadcastDynamicShape(self, x_dims, y_dims, expected_dims):
     x_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(x_dims)
     y_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(y_dims)
@@ -416,6 +420,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
           dim_sizes=[3, [3, 0, 2]],
           expected=ragged.constant_value([[10, 10, 10], [], [10, 10]])),
   ])
+  @test_util.run_v1_only('b/120545219')
   def testRaggedBroadcastTo(self, x, dim_sizes, expected):
     shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes)
     result = ragged.broadcast_to(x, shape)
@@ -470,6 +475,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
               [[[[11, 21], [32]], [[13, 23], [34]]],
                [[[15, 25], [36]]]])),
   ])
+  @test_util.run_v1_only('b/120545219')
   def testRaggedAddWithBroadcasting(self, x, y, expected, doc):
     expected_rrank = getattr(expected, 'ragged_rank', 0)
     x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32)
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index 608fbd6e5b..e86676f70a 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -861,7 +861,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[:, -2:], [row[-2:] for row in EXAMPLE_RAGGED_TENSOR_2D]),
       # TODO(edloper): Add tests for strided slices, once support is added.
   )
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorGetItemWithRaggedRank1(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     # Ragged tensor
@@ -903,7 +903,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[..., 0, 0, 0], IndexError,
        'Too many indices for RaggedTensor'),
   )
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorGetItemErrorsWithRaggedRank1(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
@@ -982,7 +982,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       # TODO(edloper): Add tests slicing inner ragged dimensions, one support
       # is added.
   )
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorGetItemWithRaggedRank2(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     rt = ragged.from_nested_row_splits(
@@ -1069,7 +1069,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt = ragged.from_row_splits(values, [0, 1])
     self._TestGetItemException(rt, slice_spec, expected, message)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testGetItemNewAxis(self):
     # rt: [[[['a', 'b'], ['c', 'd']], [], [['e', 'f']]], []]
     splits1 = [0, 3, 3]
@@ -1155,7 +1155,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   # RaggedTensor.with_values() and RaggedTensor.with_inner_values().
   #=============================================================================
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testWithValues(self):
     rt1 = ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])
     rt2 = ragged.constant([[[1, 2], [3, 4, 5]], [[6]], [], [[], [7]]])
@@ -1208,7 +1208,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]])
       self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testSessionPartialRunFeed(self):
     # Placeholder inputs.
     a = ragged.from_row_splits(
diff --git a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
index 69b31ad0e9..9863e3b583 100644
--- a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
@@ -140,7 +140,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     self.assertEqual(st.values.shape.as_list(), [7])
     self.assertEqual(st.dense_shape.shape.as_list(), [3])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testKernelErrors(self):
     # An empty vector, defined using a placeholder to ensure that we can't
     # determine that it's invalid at graph-construction time.
@@ -179,7 +179,7 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
       self.assertRaisesRegexp(errors.InvalidArgumentError, empty_splits_error,
                               ragged.to_sparse(bad_rt5).eval)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testGradient(self):
     # rt1.shape == rt2.shape == [2, (D2), (D3), 2].
     rt1 = ragged.constant([[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0]]]],
diff --git a/tensorflow/python/ops/ragged/ragged_util_test.py b/tensorflow/python/ops/ragged/ragged_util_test.py
index c24ea65353..69c605dbf9 100644
--- a/tensorflow/python/ops/ragged/ragged_util_test.py
+++ b/tensorflow/python/ops/ragged/ragged_util_test.py
@@ -87,6 +87,7 @@ class RaggedRepeatTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       # Empty tensor
       dict(data=[], repeats=[], axis=0, expected=[]),
   ])
+  @test_util.run_v1_only('b/120545219')
   def testRepeat(self, data, repeats, expected, axis=None):
     result = ragged_util.repeat(data, repeats, axis)
     with self.test_session():
@@ -135,6 +136,7 @@ class RaggedRepeatTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           dict(data=TENSOR_4D, repeats=[1, 3, 0, 0, 2], axis=3),
       ]
   ])
+  @test_util.run_v1_only('b/120545219')
   def testValuesMatchesNumpy(self, mode, data, repeats, axis):
     # Exception: we can't handle negative axis if data.ndims is unknown.
     if axis < 0 and mode == 'unknown_shape':
diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py
index de83a54977..6d645eefac 100644
--- a/tensorflow/python/ops/ragged/ragged_where_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py
@@ -165,7 +165,7 @@ class RaggedWhereOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           y=ragged.constant_value([[[['a']]], [[['b']]]]),
           expected=ragged.constant_value([[[[], [b'A']]], [[[b'b']]]])),
   ])   # pyformat: disable
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testRaggedWhere(self, condition, expected, x=None, y=None):
     result = ragged.where(condition, x, y)
     self.assertEqual(
diff --git a/tensorflow/python/profiler/internal/run_metadata_test.py b/tensorflow/python/profiler/internal/run_metadata_test.py
index a8859f845b..f96d721f46 100644
--- a/tensorflow/python/profiler/internal/run_metadata_test.py
+++ b/tensorflow/python/profiler/internal/run_metadata_test.py
@@ -169,7 +169,7 @@ class RunMetadataTest(test.TestCase):
     ret = _extract_node(run_meta, 'MatMul:MatMul')
     self.assertEqual(len(ret), 0)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testLoopCPU(self):
     ops.reset_default_graph()
     with ops.device('/cpu:0'):
diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py
index 8648f0b514..1c7c15be4f 100644
--- a/tensorflow/python/profiler/model_analyzer_test.py
+++ b/tensorflow/python/profiler/model_analyzer_test.py
@@ -76,6 +76,7 @@ class PrintModelAnalysisTest(test.TestCase):
                          '  ScalarW (1, 1/1 params)\n',
                          lib.CheckAndRemoveDoc(f.read()))
 
+  @test_util.run_v1_only('b/120545219')
   def testSelectEverythingDetail(self):
     ops.reset_default_graph()
     dev = '/device:GPU:0' if test.is_gpu_available() else '/device:CPU:0'
@@ -203,6 +204,7 @@ class PrintModelAnalysisTest(test.TestCase):
             lib.CheckAndRemoveDoc(f.read())[0:80])
         # pylint: enable=line-too-long
 
+  @test_util.run_v1_only('b/120545219')
   def testComplexCodeView(self):
     ops.reset_default_graph()
     outfile = os.path.join(test.get_temp_dir(), 'dump')
@@ -619,6 +621,7 @@ class PrintModelAnalysisTest(test.TestCase):
           else:
             self.assertEqual(len(gfile.ListDirectory(profile_dir)), 0)
 
+  @test_util.run_v1_only('b/120545219')
   def testAutoProfiling(self):
     ops.reset_default_graph()
     time_dir = os.path.join(test.get_temp_dir(), 'time')
@@ -706,6 +709,7 @@ class PrintModelAnalysisTest(test.TestCase):
                       exception_str)
       self.assertTrue(mat is None)
 
+  @test_util.run_v1_only('b/120545219')
   def testTrackPersistentBytes(self):
     ops.reset_default_graph()
     a = array_ops.constant(np.ones((100, 100)))
diff --git a/tensorflow/python/profiler/pprof_profiler_test.py b/tensorflow/python/profiler/pprof_profiler_test.py
index 120a0d0eaa..3f5bd9e79b 100644
--- a/tensorflow/python/profiler/pprof_profiler_test.py
+++ b/tensorflow/python/profiler/pprof_profiler_test.py
@@ -136,7 +136,7 @@ comment: 9
       profile.ParseFromString(profile_contents)
       self.assertEquals(expected_proto, str(profile))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testProfileWithWhileLoop(self):
     options = config_pb2.RunOptions()
     options.trace_level = config_pb2.RunOptions.FULL_TRACE
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 3b7f0b250e..3e27c0801c 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -94,7 +94,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
     super(SavedModelLoaderTest, self).tearDown()
     shutil.rmtree(test.get_temp_dir(), ignore_errors=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_load_function(self, builder_cls):
     self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
@@ -110,7 +110,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(7, sess.graph.get_tensor_by_name("y:0").eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_load_graph(self, builder_cls):
     self.export_simple_graph(builder_cls)
     loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
@@ -130,7 +130,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(errors.FailedPreconditionError):
         self.evaluate(y)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_load_with_import_scope(self, builder_cls):
     self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
@@ -179,7 +179,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
       loader.restore_variables(sess, tf_saver.Saver())
       self.assertEqual(55, self.evaluate(z))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_run_init_op(self, builder_cls):
     self.export_graph_with_main_op(builder_cls)
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
@@ -212,7 +212,7 @@ class SavedModelLoaderTest(test.TestCase, parameterized.TestCase):
     with self.assertRaises(RuntimeError):
       loader.get_meta_graph_def_from_tags(["not_a_graph"])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_load_saved_model_with_no_variables(self, builder_cls):
     """Test that SavedModel runs saver when there appear to be no variables.
 
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 668a374415..97218a98ea 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -286,7 +286,7 @@ class SaveTest(test.TestCase):
         {"out": model(array_ops.ones([1, 4]))},
         _import_and_infer(save_dir, {"x": [[1., 1., 1., 1.]]}))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def test_export_functional_keras_model_after_fit(self):
     x = input_layer.Input((1,))
     y = core.Dense(1, name="y")(x)
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index d4c8d72011..8d94c7c989 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -328,7 +328,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertRaises(RuntimeError, loader.load, sess, ["foo", "baz"],
                         export_dir)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testVariables(self):
     export_dir = self._get_export_dir("test_variables")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -474,7 +474,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertEqual(
           42, ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)[0].eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCollections(self):
     export_dir = self._get_export_dir("test_collections")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -819,6 +819,7 @@ class SavedModelTest(SavedModelTestBase):
       self._validate_assets(export_dir, foo_graph.asset_file_def, "hello42.txt",
                             "foo bar baz 0", "asset_file_tensor_0:0")
 
+  @test_util.run_v1_only("b/120545219")
   def testCustomInitOp(self):
     export_dir = self._get_export_dir("test_main_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -854,7 +855,7 @@ class SavedModelTest(SavedModelTestBase):
       # the main_op, following a restore.
       self.assertEqual(3, ops.get_collection("v")[2].eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTrainOp(self):
     export_dir = self._get_export_dir("test_train_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -882,7 +883,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertIsInstance(
           loader_impl.get_train_op(meta_graph_def), ops.Tensor)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTrainOpGroup(self):
     export_dir = self._get_export_dir("test_train_op_group")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -910,7 +911,7 @@ class SavedModelTest(SavedModelTestBase):
       self.assertIsInstance(
           loader_impl.get_train_op(meta_graph_def), ops.Operation)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTrainOpAfterVariables(self):
     export_dir = self._get_export_dir("test_train_op_after_variables")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1029,7 +1030,7 @@ class SavedModelTest(SavedModelTestBase):
       self._validate_assets(export_dir, bar_graph.asset_file_def, "foo.txt",
                             "content_foo", "asset_file_tensor:0")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testOp(self):
     export_dir = self._get_export_dir("test_op")
     builder = saved_model_builder._SavedModelBuilder(export_dir)
@@ -1492,7 +1493,7 @@ class SavedModelV1Test(SavedModelTestBase):
     self.assertIn("T", node_def.attr)
     self.assertIn("Tout", node_def.attr)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testLegacyInitOp(self):
     export_dir = self._get_export_dir("test_legacy_init_op")
     builder = saved_model_builder.SavedModelBuilder(export_dir)
diff --git a/tensorflow/python/saved_model/signature_def_utils_test.py b/tensorflow/python/saved_model/signature_def_utils_test.py
index 53c452359f..d1347eb017 100644
--- a/tensorflow/python/saved_model/signature_def_utils_test.py
+++ b/tensorflow/python/saved_model/signature_def_utils_test.py
@@ -423,6 +423,7 @@ class SignatureDefUtilsTest(test.TestCase):
         {},
         signature_constants.PREDICT_METHOD_NAME)
 
+  @test_util.run_v1_only("b/120545219")
   def testOpSignatureDef(self):
     key = "adding_1_and_2_key"
     add_op = math_ops.add(1, 2, name="adding_1_and_2")
@@ -430,6 +431,7 @@ class SignatureDefUtilsTest(test.TestCase):
     self.assertIn(key, signature_def.outputs)
     self.assertEqual(add_op.name, signature_def.outputs[key].name)
 
+  @test_util.run_v1_only("b/120545219")
   def testLoadOpFromSignatureDef(self):
     key = "adding_1_and_2_key"
     add_op = math_ops.add(1, 2, name="adding_1_and_2")
diff --git a/tensorflow/python/saved_model/utils_test.py b/tensorflow/python/saved_model/utils_test.py
index 0888dcb411..2afe8abfd6 100644
--- a/tensorflow/python/saved_model/utils_test.py
+++ b/tensorflow/python/saved_model/utils_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.platform import test
@@ -32,6 +33,7 @@ from tensorflow.python.saved_model import utils
 
 class UtilsTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testBuildTensorInfoOp(self):
     x = constant_op.constant(1, name="x")
     y = constant_op.constant(2, name="y")
@@ -41,6 +43,7 @@ class UtilsTest(test.TestCase):
     self.assertEqual(types_pb2.DT_INVALID, z_op_info.dtype)
     self.assertEqual(0, len(z_op_info.tensor_shape.dim))
 
+  @test_util.run_v1_only("b/120545219")
   def testBuildTensorInfoDefunOp(self):
     @function.defun
     def my_init_fn(x, y):
@@ -54,6 +57,7 @@ class UtilsTest(test.TestCase):
     self.assertEqual(types_pb2.DT_INVALID, init_op_info.dtype)
     self.assertEqual(0, len(init_op_info.tensor_shape.dim))
 
+  @test_util.run_v1_only("b/120545219")
   def testBuildTensorInfoDense(self):
     x = array_ops.placeholder(dtypes.float32, 1, name="x")
     x_tensor_info = utils.build_tensor_info(x)
@@ -62,6 +66,7 @@ class UtilsTest(test.TestCase):
     self.assertEqual(1, len(x_tensor_info.tensor_shape.dim))
     self.assertEqual(1, x_tensor_info.tensor_shape.dim[0].size)
 
+  @test_util.run_v1_only("b/120545219")
   def testBuildTensorInfoSparse(self):
     x = array_ops.sparse_placeholder(dtypes.float32, [42, 69], name="x")
     x_tensor_info = utils.build_tensor_info(x)
@@ -76,6 +81,7 @@ class UtilsTest(test.TestCase):
     self.assertEqual(42, x_tensor_info.tensor_shape.dim[0].size)
     self.assertEqual(69, x_tensor_info.tensor_shape.dim[1].size)
 
+  @test_util.run_v1_only("b/120545219")
   def testGetTensorFromInfoDense(self):
     expected = array_ops.placeholder(dtypes.float32, 1, name="x")
     tensor_info = utils.build_tensor_info(expected)
@@ -83,6 +89,7 @@ class UtilsTest(test.TestCase):
     self.assertIsInstance(actual, ops.Tensor)
     self.assertEqual(expected.name, actual.name)
 
+  @test_util.run_v1_only("b/120545219")
   def testGetTensorFromInfoSparse(self):
     expected = array_ops.sparse_placeholder(dtypes.float32, name="x")
     tensor_info = utils.build_tensor_info(expected)
@@ -122,6 +129,7 @@ class UtilsTest(test.TestCase):
                                                  import_scope="foo")
       self.assertEqual(expected.name, actual.name)
 
+  @test_util.run_v1_only("b/120545219")
   def testGetTensorFromInfoRaisesErrors(self):
     expected = array_ops.placeholder(dtypes.float32, 1, name="x")
     tensor_info = utils.build_tensor_info(expected)
diff --git a/tensorflow/python/tools/freeze_graph_test.py b/tensorflow/python/tools/freeze_graph_test.py
index efdf7dd2cf..de2672db3c 100644
--- a/tensorflow/python/tools/freeze_graph_test.py
+++ b/tensorflow/python/tools/freeze_graph_test.py
@@ -161,11 +161,11 @@ class FreezeGraphTest(test_util.TensorFlowTestCase):
             },)
         builder.save(as_text=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testFreezeGraphV1(self):
     self._testFreezeGraph(saver_pb2.SaverDef.V1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testFreezeGraphV2(self):
     self._testFreezeGraph(saver_pb2.SaverDef.V2)
 
diff --git a/tensorflow/python/training/adagrad_test.py b/tensorflow/python/training/adagrad_test.py
index da26fcdb7f..1e2d29b337 100644
--- a/tensorflow/python/training/adagrad_test.py
+++ b/tensorflow/python/training/adagrad_test.py
@@ -306,7 +306,7 @@ class AdagradOptimizerTest(test.TestCase):
             np.array([2.715679168701172, 3.715679168701172]),
             self.evaluate(var1))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDynamicShapeVariable_Ok(self):
     with self.cached_session():
       v = variable_scope.get_variable("v", initializer=constant_op.constant(1.),
@@ -315,7 +315,7 @@ class AdagradOptimizerTest(test.TestCase):
       # Creating optimizer should cause no exception.
       adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDynamicShapeVariableWithCallableInit(self):
     var0 = variable_scope.get_variable("var0",
                                        initializer=constant_op.constant(1.),
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 08942c5bb6..1af27626ba 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -1122,6 +1122,7 @@ class StepCounterHookTest(test.TestCase):
         self.assertGreater(summary_value.simple_value, 0)
 
 
+@test_util.run_v1_only('b/120545219')
 class SummarySaverHookTest(test.TestCase):
 
   def setUp(self):
@@ -1148,13 +1149,11 @@ class SummarySaverHookTest(test.TestCase):
       basic_session_run_hooks.SummarySaverHook(
           scaffold=monitored_session.Scaffold(), summary_op=self.summary_op)
 
-  @test_util.run_deprecated_v1
   def test_raise_in_both_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
           save_secs=10, save_steps=20, summary_writer=self.summary_writer)
 
-  @test_util.run_deprecated_v1
   def test_raise_in_none_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
@@ -1405,6 +1404,7 @@ class FinalOpsHookTest(test.TestCase):
                              hook.final_ops_values.tolist())
 
 
+@test_util.run_v1_only('b/120545219')
 class ResourceSummarySaverHookTest(test.TestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py
index 21ad3df1c8..c481547139 100644
--- a/tensorflow/python/training/checkpoint_ops_test.py
+++ b/tensorflow/python/training/checkpoint_ops_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import partitioned_variables
@@ -33,6 +34,7 @@ from tensorflow.python.training import checkpoint_ops
 from tensorflow.python.training import saver as saver_lib
 
 
+@test_util.run_v1_only('b/120545219')
 class LoadAndRemapWrappersTest(test.TestCase):
   """Tests for the functionality of the Python wrappers."""
 
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index 9cefd942ac..bcec6e0100 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -73,6 +73,7 @@ class HasList(training.Model):
 class ListTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only("b/120545219")
   def testTracking(self):
     model = HasList()
     output = model(array_ops.ones([32, 2]))
@@ -105,6 +106,7 @@ class ListTests(test.TestCase):
     self.assertIn(v, model.trainable_variables)
     self.assertNotIn(v, model.non_trainable_variables)
 
+  @test_util.run_v1_only("b/120545219")
   def testUpdatesForwarded(self):
     with context.graph_mode():
       model = HasList()
@@ -121,6 +123,7 @@ class ListTests(test.TestCase):
       self.assertEqual(0, len(model.updates))
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only("b/120545219")
   def testLossesForwarded(self):
     model = HasList()
     model_input = array_ops.ones([32, 2])
@@ -295,6 +298,7 @@ class HasMapping(training.Model):
 class MappingTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only("b/120545219")
   def testTracking(self):
     model = HasMapping()
     output = model(array_ops.ones([32, 2]))
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 78047eda90..3bdab4cb0b 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -616,6 +616,7 @@ class CheckpointingTests(test.TestCase):
 
   # pylint: disable=cell-var-from-loop
   @test_util.run_in_graph_and_eager_modes
+  @test_util.run_v1_only("b/120545219")
   def testWithDefun(self):
     num_training_steps = 2
     checkpoint_directory = self.get_temp_dir()
diff --git a/tensorflow/python/training/input_test.py b/tensorflow/python/training/input_test.py
index a3d268a017..d89f5f3bbd 100644
--- a/tensorflow/python/training/input_test.py
+++ b/tensorflow/python/training/input_test.py
@@ -872,19 +872,19 @@ class BatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
@@ -1482,19 +1482,19 @@ class BatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
@@ -1905,19 +1905,19 @@ class ShuffleBatchTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
@@ -2309,19 +2309,19 @@ class ShuffleBatchJoinTest(test_lib.TestCase):
       for thread in threads:
         thread.join()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInput(self):
     self._testKeepInputHelper(1, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSingleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(1, True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInput(self):
     self._testKeepInputHelper(5, False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultipleThreadKeepInputEnqueueMany(self):
     self._testKeepInputHelper(5, True)
 
diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py
index 9de5bc8168..1029d4cea8 100644
--- a/tensorflow/python/training/learning_rate_decay_test.py
+++ b/tensorflow/python/training/learning_rate_decay_test.py
@@ -101,7 +101,7 @@ class LRDecayTest(test_util.TensorFlowTestCase):
     self.assertAllClose(self.evaluate(decayed_lr), 0.001, 1e-6)
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPiecewiseConstantEdgeCases(self):
     x_int = resource_variable_ops.ResourceVariable(
         0, dtype=variables.dtypes.int32)
diff --git a/tensorflow/python/training/localhost_cluster_performance_test.py b/tensorflow/python/training/localhost_cluster_performance_test.py
index 7c097b943d..c4cbc8a55d 100644
--- a/tensorflow/python/training/localhost_cluster_performance_test.py
+++ b/tensorflow/python/training/localhost_cluster_performance_test.py
@@ -25,6 +25,7 @@ import numpy as np
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
@@ -34,6 +35,7 @@ from tensorflow.python.training import device_setter
 
 class CreateLocalClusterTest(test.TestCase):
 
+  @test_util.run_v1_only("b/120545219")
   def testCreateLocalCluster(self):
     workers, _ = test.create_local_cluster(num_workers=2, num_ps=2)
     worker_sessions = [session_lib.Session(w.target) for w in workers]
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 9dbcfa52b7..99ee9ea7e2 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -541,6 +541,7 @@ class WrappedSessionTest(test.TestCase):
       self.assertFalse(wrapped_sess1.should_stop())
       self.assertTrue(wrapped_sess1.should_stop())
 
+  @test_util.run_deprecated_v1
   def test_close_twice(self):
     with self.cached_session() as sess:
       wrapped_sess = monitored_session._WrappedSession(sess)
diff --git a/tensorflow/python/training/moving_averages_test.py b/tensorflow/python/training/moving_averages_test.py
index b15f7377f0..03bcde9c84 100644
--- a/tensorflow/python/training/moving_averages_test.py
+++ b/tensorflow/python/training/moving_averages_test.py
@@ -219,38 +219,38 @@ class ExponentialMovingAverageTest(test.TestCase):
                         (10.0 + 30.0) * (1 - dk)) / _Scale(dk, 2), dim)
     self.assertAllClose(expected, self.evaluate(avg2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNoNumUpdates_Scalar(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25)
       self._CheckDecay(ema, actual_decay=0.25, dim=1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNoNumUpdates_Scalar_Debias(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.25, dim=1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNoNumUpdates_Vector(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25)
       self._CheckDecay(ema, actual_decay=0.25, dim=5)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNoNumUpdates_Vector_Debias(self):
     with self.cached_session():
       ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.25, dim=5)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNumUpdates_Scalar(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
       ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1)
       self._CheckDecay(ema, actual_decay=0.181818, dim=1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNumUpdates_Scalar_Debias(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
@@ -258,14 +258,14 @@ class ExponentialMovingAverageTest(test.TestCase):
           0.25, num_updates=1, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.181818, dim=1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNumUpdates_Vector(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
       ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1)
       self._CheckDecay(ema, actual_decay=0.181818, dim=5)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNumUpdates_Vector_Debias(self):
     with self.cached_session():
       # With num_updates 1, the decay applied is 0.1818
@@ -273,7 +273,7 @@ class ExponentialMovingAverageTest(test.TestCase):
           0.25, num_updates=1, zero_debias=True)
       self._CheckDecay(ema, actual_decay=0.181818, dim=5)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesWithControlDeps(self):
     with self.cached_session() as sess:
       v0 = variables.Variable(0, name="v0")
@@ -299,7 +299,7 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual([17.5], self.evaluate(v1_avg))
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBasicEager(self):
     v0 = variables.Variable(1.0)
     v1 = variables.Variable(2.0)
@@ -355,11 +355,11 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual(ema.average(v1).op.name, ema.average_name(v1))
       self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNames(self):
     self.averageVariablesNamesHelper(zero_debias=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNamesNoDebias(self):
     self.averageVariablesNamesHelper(zero_debias=False)
 
@@ -405,15 +405,15 @@ class ExponentialMovingAverageTest(test.TestCase):
         self.assertEqual(
             ema.average(tensor2).op.name, ema.average_name(tensor2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNamesRespectScope(self):
     self.averageVariablesNamesRespectScopeHelper(zero_debias=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesNamesRespectScopeNoDebias(self):
     self.averageVariablesNamesRespectScopeHelper(zero_debias=False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSubsetAverageVariablesNames(self):
     with self.cached_session():
       v0 = variables.Variable(10.0, name="v0")
@@ -442,7 +442,7 @@ class ExponentialMovingAverageTest(test.TestCase):
       self.assertEqual(ema.average(v1).op.name, ema.average_name(v1))
       self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAverageVariablesDeviceAssignment(self):
     with ops.device("/job:dev_v0"):
       v0 = variables.Variable(10.0, name="v0")
diff --git a/tensorflow/python/training/quantize_training_test.py b/tensorflow/python/training/quantize_training_test.py
index 62e783f200..2352af7e99 100644
--- a/tensorflow/python/training/quantize_training_test.py
+++ b/tensorflow/python/training/quantize_training_test.py
@@ -53,7 +53,7 @@ class PywrapQuantizeTrainingTest(test.TestCase):
 
   # Test that save/restoring works for EMA variables generated in the
   # quantized training rewrite.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testQuantizedSaveRestore(self):
     save_path = os.path.join(self.get_temp_dir(), 'quantized_save_restore')
 
diff --git a/tensorflow/python/training/queue_runner_test.py b/tensorflow/python/training/queue_runner_test.py
index 4113cecf55..2f6e924f98 100644
--- a/tensorflow/python/training/queue_runner_test.py
+++ b/tensorflow/python/training/queue_runner_test.py
@@ -41,7 +41,7 @@ _MockOp = collections.namedtuple("MockOp", ["name"])
 
 class QueueRunnerTest(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBasic(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -62,7 +62,7 @@ class QueueRunnerTest(test.TestCase):
       # The variable should be 3.
       self.assertEqual(3, self.evaluate(var))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testTwoOps(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -132,7 +132,7 @@ class QueueRunnerTest(test.TestCase):
       with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed"):
         self.evaluate(dequeue1)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRespectCoordShouldStop(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
@@ -221,7 +221,7 @@ class QueueRunnerTest(test.TestCase):
       new_threads = qr.create_threads(sess, coord=coord)
       self.assertEqual([], new_threads)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testThreads(self):
     with self.cached_session() as sess:
       # CountUpTo will raise OUT_OF_RANGE when it reaches the count.
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 5d621ba4ff..95c21cb815 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -390,7 +390,7 @@ class SaverTest(test.TestCase):
             ValueError, "The passed save_path is not a valid checkpoint:"):
           save.restore(sess, "invalid path")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInt64(self):
     save_path = os.path.join(self.get_temp_dir(), "int64")
 
@@ -466,7 +466,7 @@ class SaverTest(test.TestCase):
       # Verify non-duplicate names work.
       saver_module.Saver({"v0": v0, "v2": v2.saveable})
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testBasicsWithListOfVariables(self):
     save_path = os.path.join(self.get_temp_dir(), "basics_with_list")
 
@@ -667,7 +667,7 @@ class SaverTest(test.TestCase):
       self.assertAllClose(1.0, one.eval())
       self.assertAllClose([2.0, 2.0, 2.0], twos.eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testReshape(self):
     save_path = os.path.join(self.get_temp_dir(), "variables_reshape")
     with session.Session("", graph=ops_lib.Graph()) as sess:
@@ -726,7 +726,7 @@ class SaverTest(test.TestCase):
   def testSaveWithGlobalStepWithPadding(self):
     self.testSaveWithGlobalStep(pad_step_number=True)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSaveToNonexistingPath(self):
     file_io.write_string_to_file(
         os.path.join(self.get_temp_dir(), "actually_a_file"), "")
@@ -1607,7 +1607,7 @@ class SaveRestoreWithVariableNameMap(test.TestCase):
       self.assertEqual(20.0, self.evaluate(v1))
 
   @test_util.run_in_graph_and_eager_modes
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testNonReshapeResourceVariable(self):
     self._testNonReshape(resource_variable_ops.ResourceVariable)
 
@@ -1622,7 +1622,7 @@ class MetaGraphTest(test.TestCase):
     gfile.MakeDirs(test_dir)
     return test_dir
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testAddCollectionDef(self):
     test_dir = self._get_test_dir("good_collection")
     filename = os.path.join(test_dir, "metafile")
@@ -1772,13 +1772,13 @@ class MetaGraphTest(test.TestCase):
       v1 = sess.graph.get_tensor_by_name("v1:0")
       self.assertEqual(11.0, v1.eval())
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testMultiSaverCollection(self):
     test_dir = self._get_test_dir("saver_collection")
     self._testMultiSaverCollectionSave(test_dir)
     self._testMultiSaverCollectionRestore(test_dir)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testClearExtraneousSavers(self):
     test_dir = self._get_test_dir("clear_extraneous_savers")
     filename = os.path.join(test_dir, "metafile")
@@ -1866,7 +1866,7 @@ class MetaGraphTest(test.TestCase):
                                                lambda e: "does not exist"):
         saver_module.import_meta_graph(filename)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSliceVariable(self):
     test_dir = self._get_test_dir("slice_saver")
     filename = os.path.join(test_dir, "metafile")
@@ -2122,7 +2122,7 @@ class MetaGraphTest(test.TestCase):
                                       lambda: math_ops.multiply(x, -1.0))))
     # pylint: enable=g-long-lambda
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStrippedOpListDef(self):
     with self.cached_session():
       # Creates a graph.
@@ -2988,7 +2988,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
       # exception" block in Python 3.
       self.assertNotIn("NewCheckpointReader", cs.exception.message)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testGraphChangedForRestoreErrorRaised(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
@@ -3010,7 +3010,7 @@ class CheckpointableCompatibilityTests(test.TestCase):
             "a mismatch between the current graph and the graph"):
           a_saver.restore(sess=sess, save_path=save_path)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testLoadFromObjectBasedGraph(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
diff --git a/tensorflow/python/training/server_lib_same_variables_no_clear_test.py b/tensorflow/python/training/server_lib_same_variables_no_clear_test.py
index 1b2d588f44..ff3fab9f37 100644
--- a/tensorflow/python/training/server_lib_same_variables_no_clear_test.py
+++ b/tensorflow/python/training/server_lib_same_variables_no_clear_test.py
@@ -34,7 +34,7 @@ class SameVariablesNoClearTest(test.TestCase):
   # TODO(b/34465411): Starting multiple servers with different configurations
   # in the same test is flaky. Move this test case back into
   # "server_lib_test.py" when this is no longer the case.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSameVariablesNoClear(self):
     server = server_lib.Server.create_local_server()
 
diff --git a/tensorflow/python/training/server_lib_test.py b/tensorflow/python/training/server_lib_test.py
index 323e94c257..92cdc1c4ad 100644
--- a/tensorflow/python/training/server_lib_test.py
+++ b/tensorflow/python/training/server_lib_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
@@ -55,6 +56,7 @@ class GrpcServerTest(test.TestCase):
       self.assertAllEqual([[4]], sess.run(e))
     # TODO(mrry): Add `server.stop()` and `server.join()` when these work.
 
+  @test_util.run_v1_only("b/120545219")
   def testMultipleSessions(self):
     server = self._cached_server
 
@@ -73,6 +75,7 @@ class GrpcServerTest(test.TestCase):
     # TODO(mrry): Add `server.stop()` and `server.join()` when these work.
 
   # Verifies various reset failures.
+  @test_util.run_v1_only("b/120545219")
   def testResetFails(self):
     # Creates variable with container name.
     with ops.container("test0"):
@@ -146,6 +149,7 @@ class GrpcServerTest(test.TestCase):
       self.assertEqual(0.5, min_val)
       self.assertEqual(0.5, max_val)
 
+  @test_util.run_v1_only("b/120545219")
   def testCloseCancelsBlockingOperation(self):
     server = self._cached_server
     sess = session.Session(server.target, config=self._useRPCConfig())
@@ -207,6 +211,7 @@ class GrpcServerTest(test.TestCase):
               "local": ["localhost"]
           }, job_name="local", task_index=0)
 
+  @test_util.run_v1_only("b/120545219")
   def testTimeoutRaisesException(self):
     server = self._cached_server
     q = data_flow_ops.FIFOQueue(1, [dtypes.float32])
@@ -241,6 +246,7 @@ class GrpcServerTest(test.TestCase):
       queue_runner_impl.start_queue_runners(sess)
       sess.run(var.assign(3.0))
 
+  @test_util.run_v1_only("b/120545219")
   def testIsolateSessionState(self):
     server = self._cached_server
 
@@ -296,6 +302,7 @@ class GrpcServerTest(test.TestCase):
     self.assertAllEqual(37, isolate_sess_0.run(v))
     self.assertAllEqual([19, 86], isolate_sess_1.run(v))
 
+  @test_util.run_v1_only("b/120545219")
   def testShapeChangingIsolateState(self):
     server = self._cached_server
     sharing_config = config_pb2.ConfigProto(isolate_session_state=False)
diff --git a/tensorflow/python/training/session_manager_test.py b/tensorflow/python/training/session_manager_test.py
index 4294ffa851..c9a0c56ffc 100644
--- a/tensorflow/python/training/session_manager_test.py
+++ b/tensorflow/python/training/session_manager_test.py
@@ -69,7 +69,7 @@ class SessionManagerTest(test.TestCase):
           "", init_fn=lambda sess: sess.run(v.initializer))
       self.assertAllClose([125], sess.run(v))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPrepareSessionFails(self):
     checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session")
     checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2")
@@ -154,7 +154,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
       self.assertEquals(1, sess.run(v))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRecoverSession(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session")
@@ -187,6 +187,7 @@ class SessionManagerTest(test.TestCase):
           checkpoint_filename_with_path=checkpoint_management.latest_checkpoint(
               checkpoint_dir))
 
+  @test_util.run_v1_only("b/120545219")
   def testWaitForSessionReturnsNoneAfterTimeout(self):
     with ops.Graph().as_default():
       variables.VariableV1(1, name="v")
@@ -209,7 +210,7 @@ class SessionManagerTest(test.TestCase):
               variables.global_variables()),
           local_init_op=None)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRecoverSessionWithReadyForLocalInitOp(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(self.get_temp_dir(),
@@ -263,7 +264,7 @@ class SessionManagerTest(test.TestCase):
       self.assertEquals(1, sess.run(v))
       self.assertEquals(1, sess.run(w))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRecoverSessionWithReadyForLocalInitOpFailsToReadyLocal(self):
     # We use ready_for_local_init_op=tf.report_uninitialized_variables(),
     # which causes recover_session to not run local_init_op, and to return
@@ -320,7 +321,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("w:0")).eval(session=sess))
       self.assertEquals(1, sess.run(v))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRecoverSessionNoChkptStillRunsLocalInitOp(self):
     # This test checks for backwards compatibility.
     # In particular, we continue to ensure that recover_session will execute
@@ -349,7 +350,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("w:0")).eval(session=sess))
       self.assertEquals(1, sess.run(w))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRecoverSessionFailsStillRunsLocalInitOp(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(
@@ -393,7 +394,7 @@ class SessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("w:0")).eval(session=sess))
       self.assertEquals(1, sess.run(w))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWaitForSessionLocalInit(self):
     server = server_lib.Server.create_local_server()
     with ops.Graph().as_default() as graph:
@@ -445,7 +446,7 @@ class SessionManagerTest(test.TestCase):
         # because of overly restrictive ready_for_local_init_op
         sm.wait_for_session("", max_wait_secs=3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testWaitForSessionInsufficientReadyForLocalInitCheck(self):
     with ops.Graph().as_default() as graph:
       v = variables.VariableV1(1, name="v")
@@ -463,7 +464,7 @@ class SessionManagerTest(test.TestCase):
                                  "Session was not ready after waiting.*"):
       sm.wait_for_session("", max_wait_secs=3)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPrepareSessionWithReadyForLocalInitOp(self):
     with ops.Graph().as_default():
       v = variables.VariableV1(1, name="v")
@@ -503,7 +504,7 @@ class SessionManagerTest(test.TestCase):
       self.assertEquals(1, sess.run(w))
       self.assertEquals(3, sess.run(x))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPrepareSessionWithPartialInitOp(self):
     with ops.Graph().as_default():
       v = variables.VariableV1(1, name="v")
@@ -570,7 +571,7 @@ class SessionManagerTest(test.TestCase):
       self.assertEquals(1, sess.run(w_res))
       self.assertEquals(3, sess.run(x_res))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPrepareSessionWithCyclicInitializer(self):
     # Regression test. Previously Variable._build_initializer_expr would enter
     # into an infinite recursion when the variable's initial_value involved
@@ -644,7 +645,7 @@ class SessionManagerTest(test.TestCase):
           "Init operations did not make model ready for local_init"):
         sm2.prepare_session("", init_op=None)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPrepareSessionWithInsufficientReadyForLocalInitCheck(self):
     with ops.Graph().as_default():
       v = variables.VariableV1(1, name="v")
@@ -697,7 +698,7 @@ class ObsoleteSessionManagerTest(test.TestCase):
           "", init_fn=lambda sess: sess.run(v.initializer))
       self.assertAllClose([125], sess.run(v))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testPrepareSessionFails(self):
     checkpoint_dir = os.path.join(self.get_temp_dir(), "prepare_session")
     checkpoint_dir2 = os.path.join(self.get_temp_dir(), "prepare_session2")
@@ -759,7 +760,7 @@ class ObsoleteSessionManagerTest(test.TestCase):
           variables.is_variable_initialized(
               sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRecoverSession(self):
     # Create a checkpoint.
     checkpoint_dir = os.path.join(self.get_temp_dir(), "recover_session")
@@ -798,6 +799,7 @@ class ObsoleteSessionManagerTest(test.TestCase):
               sess.graph.get_tensor_by_name("v:0")).eval(session=sess))
       self.assertEquals(1, sess.run(v))
 
+  @test_util.run_v1_only("b/120545219")
   def testWaitForSessionReturnsNoneAfterTimeout(self):
     with ops.Graph().as_default():
       variables.VariableV1(1, name="v")
diff --git a/tensorflow/python/training/slot_creator_test.py b/tensorflow/python/training/slot_creator_test.py
index 1f26aaa434..f1f0d58a69 100644
--- a/tensorflow/python/training/slot_creator_test.py
+++ b/tensorflow/python/training/slot_creator_test.py
@@ -32,7 +32,7 @@ from tensorflow.python.training import slot_creator
 
 class SlotCreatorTest(test.TestCase):
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCreateSlotFromVariable(self):
     with self.cached_session():
       v = variables.Variable([1.0, 2.5], name="var")
@@ -73,7 +73,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
       self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCreateZerosSlotFromDynamicShapedVariable(self):
     with self.cached_session():
       dyn_shape = constant_op.constant([2], dtype=dtypes.int32)
@@ -125,7 +125,7 @@ class SlotCreatorTest(test.TestCase):
       self.assertEqual(dtypes.float64, slot.dtype.base_dtype)
       self.assertAllEqual([0.0, 0.0], self.evaluate(slot))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testCreateSlotFromVariableRespectsScope(self):
     # See discussion on #2740.
     with self.cached_session():
diff --git a/tensorflow/python/training/supervisor_test.py b/tensorflow/python/training/supervisor_test.py
index f6505acc9a..180ddb5287 100644
--- a/tensorflow/python/training/supervisor_test.py
+++ b/tensorflow/python/training/supervisor_test.py
@@ -421,7 +421,7 @@ class SupervisorTest(test.TestCase):
       with self.assertRaisesRegexp(RuntimeError, "requires a summary writer"):
         sv.summary_computed(sess, sess.run(summ))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testLogdirButExplicitlyNoSummaryWriter(self):
     logdir = self._test_dir("explicit_no_summary_writer")
     with ops.Graph().as_default():
@@ -507,7 +507,7 @@ class SupervisorTest(test.TestCase):
       sv = supervisor.Supervisor(logdir="", session_manager=sm)
       sv.prepare_or_wait_for_session("")
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitOp(self):
     logdir = self._test_dir("default_init_op")
     with ops.Graph().as_default():
@@ -517,7 +517,7 @@ class SupervisorTest(test.TestCase):
       self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
       sv.stop()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitFn(self):
     logdir = self._test_dir("default_init_op")
     with ops.Graph().as_default():
@@ -531,7 +531,7 @@ class SupervisorTest(test.TestCase):
       self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
       sv.stop()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testInitOpWithFeedDict(self):
     logdir = self._test_dir("feed_dict_init_op")
     with ops.Graph().as_default():
@@ -545,7 +545,7 @@ class SupervisorTest(test.TestCase):
       self.assertAllClose([1.0, 2.0, 3.0], sess.run(v))
       sv.stop()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testReadyForLocalInitOp(self):
     server = server_lib.Server.create_local_server()
     logdir = self._test_dir("default_ready_for_local_init_op")
@@ -588,7 +588,7 @@ class SupervisorTest(test.TestCase):
     sv0.stop()
     sv1.stop()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testReadyForLocalInitOpRestoreFromCheckpoint(self):
     server = server_lib.Server.create_local_server()
     logdir = self._test_dir("ready_for_local_init_op_restore")
@@ -720,7 +720,7 @@ class SupervisorTest(test.TestCase):
                                    "Variables not initialized: w"):
         sv.prepare_or_wait_for_session(server.target)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSetupFail(self):
     logdir = self._test_dir("setup_fail")
     with ops.Graph().as_default():
@@ -731,7 +731,7 @@ class SupervisorTest(test.TestCase):
       variables.VariableV1([1.0, 2.0, 3.0], name="v")
       supervisor.Supervisor(logdir=logdir, is_chief=False)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testDefaultGlobalStep(self):
     logdir = self._test_dir("default_global_step")
     with ops.Graph().as_default():
@@ -741,7 +741,7 @@ class SupervisorTest(test.TestCase):
       self.assertEquals(287, sess.run(sv.global_step))
       sv.stop()
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testRestoreFromMetaGraph(self):
     logdir = self._test_dir("restore_from_meta_graph")
     with ops.Graph().as_default():
@@ -763,7 +763,7 @@ class SupervisorTest(test.TestCase):
   # This test is based on the fact that the standard services start
   # right away and get to run once before sv.stop() returns.
   # We still sleep a bit to make the test robust.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStandardServicesWithoutGlobalStep(self):
     logdir = self._test_dir("standard_services_without_global_step")
     # Create a checkpoint.
@@ -814,7 +814,7 @@ class SupervisorTest(test.TestCase):
 
   # Same as testStandardServicesNoGlobalStep but with a global step.
   # We should get a summary about the step time.
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testStandardServicesWithGlobalStep(self):
     logdir = self._test_dir("standard_services_with_global_step")
     # Create a checkpoint.
diff --git a/tensorflow/python/training/sync_replicas_optimizer_test.py b/tensorflow/python/training/sync_replicas_optimizer_test.py
index 1ef8756ef6..428583d048 100644
--- a/tensorflow/python/training/sync_replicas_optimizer_test.py
+++ b/tensorflow/python/training/sync_replicas_optimizer_test.py
@@ -22,6 +22,7 @@ import time
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework.test_util import create_local_cluster
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -88,6 +89,7 @@ class SyncReplicasOptimizerTest(test.TestCase):
   def _run(self, train_op, sess):
     sess.run(train_op)
 
+  @test_util.run_v1_only("b/120545219")
   def test2Workers(self):
     num_workers = 2
     replicas_to_aggregate = 2
@@ -178,6 +180,7 @@ class SyncReplicasOptimizerTest(test.TestCase):
                         sessions[1].run(var_1_g_1))
 
   # 3 workers and one of them is backup.
+  @test_util.run_v1_only("b/120545219")
   def test3Workers1Backup(self):
     num_workers = 3
     replicas_to_aggregate = 2
@@ -266,6 +269,7 @@ class SyncReplicasOptimizerHookTest(test.TestCase):
                                  "apply_gradient should be called"):
       hook.begin()
 
+  @test_util.run_v1_only("b/120545219")
   def testCanCreatedBeforeMinimizeCalled(self):
     """This behavior is required to be integrated with Estimators."""
     opt = training.SyncReplicasOptimizer(
@@ -278,6 +282,7 @@ class SyncReplicasOptimizerHookTest(test.TestCase):
     opt.minimize(v, global_step=global_step)
     hook.begin()
 
+  @test_util.run_v1_only("b/120545219")
   def testFetchVariableList(self):
     opt = training.SyncReplicasOptimizer(
         opt=adam.AdamOptimizer(0.01),
diff --git a/tensorflow/python/training/tensorboard_logging_test.py b/tensorflow/python/training/tensorboard_logging_test.py
index 5af6a0aa7b..5088ab07e5 100644
--- a/tensorflow/python/training/tensorboard_logging_test.py
+++ b/tensorflow/python/training/tensorboard_logging_test.py
@@ -25,6 +25,7 @@ import tempfile
 import time
 
 from tensorflow.core.util import event_pb2
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.summary import summary_iterator
@@ -32,6 +33,7 @@ from tensorflow.python.summary.writer import writer
 from tensorflow.python.training import tensorboard_logging
 
 
+@test_util.run_v1_only("b/120545219")
 class EventLoggingTest(test.TestCase):
 
   def setUp(self):
@@ -85,6 +87,7 @@ class EventLoggingTest(test.TestCase):
                                   (event_pb2.LogMessage.ERROR, "format")])
     self.assertEqual(2, self.logged_message_count)
 
+  @test_util.run_v1_only("b/120545219")
   def testVerbosity(self):
     tensorboard_logging.set_summary_writer(self._sw)
     tensorboard_logging.set_verbosity(tensorboard_logging.ERROR)
@@ -112,6 +115,7 @@ class EventLoggingTest(test.TestCase):
     tensorboard_logging.warn("this should work")
     self.assertEqual(1, self.logged_message_count)
 
+  @test_util.run_v1_only("b/120545219")
   def testSummaryWriterFailsAfterClear(self):
     tensorboard_logging._clear_summary_writer()
     with self.assertRaises(RuntimeError):
diff --git a/tensorflow/python/training/training_ops_test.py b/tensorflow/python/training/training_ops_test.py
index 51f49ca081..ba0f40999b 100644
--- a/tensorflow/python/training/training_ops_test.py
+++ b/tensorflow/python/training/training_ops_test.py
@@ -60,7 +60,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       self.assertShapeEqual(out, apply_sgd)
       self.assertAllCloseAccordingToType(x - alpha * delta, out)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testApplyGradientDescent(self):
     for (dtype, use_gpu) in itertools.product(
         [np.float16, np.float32, np.float64], [False, True]):
@@ -129,7 +129,7 @@ class TrainingOpsTest(TensorFlowTestCase):
         self.assertAllClose(linear_update, self.evaluate(linear))
         self.assertAllClose(expected_out, out)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testApplyAdagrad(self):
     for (dtype, use_gpu) in itertools.product(
         [np.float16, np.float32, np.float64], [False, True]):
@@ -139,7 +139,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       grad = np.arange(100).astype(dtype)
       self._testTypesForAdagrad(x, y, lr, grad, use_gpu)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testApplyFtrl(self):
     for dtype in [np.float16, np.float32, np.float64]:
       x = np.arange(100).astype(dtype)
@@ -211,7 +211,7 @@ class TrainingOpsTest(TensorFlowTestCase):
         self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i],
                                            self.evaluate(accum)[index])
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSparseApplyAdagrad(self):
     for (dtype, index_type) in itertools.product(
         [np.float16, np.float32, np.float64], [np.int32, np.int64]):
@@ -225,7 +225,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       indices = np.array([0, 2]).astype(index_type)
       self._testTypesForSparseAdagrad(x, y, lr, grad, indices)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSparseApplyAdagradDim1(self):
     for (dtype, index_type) in itertools.product(
         [np.float16, np.float32, np.float64], [np.int32, np.int64]):
@@ -239,7 +239,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       indices = np.array([0, 2]).astype(index_type)
       self._testTypesForSparseAdagrad(x, y, lr, grad, indices)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testSparseApplyFtrlDim1(self):
     for (dtype, index_type) in itertools.product(
         [np.float16, np.float32, np.float64], [np.int32, np.int64]):
@@ -255,7 +255,7 @@ class TrainingOpsTest(TensorFlowTestCase):
       indices = np.array([0, 2]).astype(index_type)
       self._testTypesForSparseFtrl(x, y, z, lr, grad, indices)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only("b/120545219")
   def testApplyAdam(self):
     for dtype, use_gpu in itertools.product(
         [np.float16, np.float32, np.float64], [False, True]):
diff --git a/tensorflow/python/training/training_util_test.py b/tensorflow/python/training/training_util_test.py
index 3317008fce..3f9858a33b 100644
--- a/tensorflow/python/training/training_util_test.py
+++ b/tensorflow/python/training/training_util_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.training import monitored_session
 from tensorflow.python.training import training_util
 
 
+@test_util.run_v1_only('b/120545219')
 class GlobalStepTest(test.TestCase):
 
   def _assert_global_step(self, global_step, expected_dtype=dtypes.int64):
@@ -47,7 +48,6 @@ class GlobalStepTest(test.TestCase):
     self.assertRaisesRegexp(TypeError, 'does not have integer type',
                             training_util.get_global_step, g)
 
-  @test_util.run_deprecated_v1
   def test_invalid_shape(self):
     with ops.Graph().as_default() as g:
       self.assertIsNone(training_util.get_global_step())
@@ -72,7 +72,6 @@ class GlobalStepTest(test.TestCase):
                               training_util.create_global_step, g)
       self._assert_global_step(training_util.create_global_step(ops.Graph()))
 
-  @test_util.run_deprecated_v1
   def test_get_global_step(self):
     with ops.Graph().as_default() as g:
       self.assertIsNone(training_util.get_global_step())
@@ -93,6 +92,7 @@ class GlobalStepTest(test.TestCase):
       self._assert_global_step(training_util.get_or_create_global_step(g))
 
 
+@test_util.run_v1_only('b/120545219')
 class GlobalStepReadTest(test.TestCase):
 
   def test_global_step_read_is_none_if_there_is_no_global_step(self):
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 40162daf14..723fceef41 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -314,7 +314,7 @@ class ApiCompatibilityTest(test.TestCase):
         update_goldens=FLAGS.update_goldens,
         api_version=api_version)
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testAPIBackwardsCompatibility(self):
     api_version = 1
     golden_file_pattern = os.path.join(
@@ -333,7 +333,7 @@ class ApiCompatibilityTest(test.TestCase):
         'tensorflow.python.util.lazy_loader.LazyLoader'
         in str(type(tf.contrib)))
 
-  @test_util.run_deprecated_v1
+  @test_util.run_v1_only('b/120545219')
   def testAPIBackwardsCompatibilityV1(self):
     api_version = 1
     golden_file_pattern = os.path.join(
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v0_11.py b/tensorflow/tools/compatibility/testdata/test_file_v0_11.py
index 68ba7a2630..917236da4b 100644
--- a/tensorflow/tools/compatibility/testdata/test_file_v0_11.py
+++ b/tensorflow/tools/compatibility/testdata/test_file_v0_11.py
@@ -34,6 +34,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   a unit test if the converter is successful.
   """
 
+  @test_util.run_v1_only("b/120545219")
   def testArgRenames(self):
     with self.cached_session():
 
@@ -97,6 +98,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           tf.expand_dims([[1, 2], [3, 4]], axis=1).eval(),
           [[[1, 2]], [[3, 4]]])
 
+  @test_util.run_v1_only("b/120545219")
   def testArgMinMax(self):
     with self.cached_session():
       self.assertAllEqual(
@@ -112,6 +114,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           tf.argmax([[1, 2, 3], [4, 1, 0]], dimension=0).eval(),
           [1, 0, 0])
 
+  @test_util.run_v1_only("b/120545219")
   def testExpandAndSqueeze(self):
     with self.cached_session():
 
@@ -139,6 +142,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
                   [[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(),
           a)
 
+  @test_util.run_v1_only("b/120545219")
   def testArithmeticRenames(self):
     with self.cached_session() as s:
       stuff = tf.split(1, 2, [[1, 2, 3, 4], [4, 5, 6, 7]])
@@ -163,6 +167,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       #     # TODO(aselle): (tf.batch_*)
       # ]
 
+  @test_util.run_v1_only("b/120545219")
   def testBatchAndSvd(self):
     with self.cached_session():
       mat = [[1., 2.], [2., 3.]]
@@ -174,6 +179,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           tf.svd(mat, False, True).eval(),
           tf.svd(mat, compute_uv=False, full_matrices=True).eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testCrossEntropy(self):
     # TODO(aselle): Test sparse_softmax_...
     with self.cached_session():
@@ -190,6 +196,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           tf.nn.sigmoid_cross_entropy_with_logits(
               labels=labels, logits=logits).eval())
 
+  @test_util.run_v1_only("b/120545219")
   def testVariables(self):
     with self.cached_session() as s:
 
@@ -200,6 +207,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       _ = [v.name for v in tf.all_variables()]
       _ = [v.name for v in tf.local_variables()]
 
+  @test_util.run_v1_only("b/120545219")
   def testSummaries(self):
     with self.cached_session() as s:
       var = tf.Variable([1, 2, 3], dtype=tf.float32)
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
index fd688781b0..5ce4dd49ad 100644
--- a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
+++ b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
@@ -28,11 +28,13 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   def setUp(self):
     tf.enable_eager_execution()
 
+  @test_util.run_v1_only("b/120545219")
   def testRenames(self):
     with self.cached_session():
       self.assertAllClose(1.04719755, tf.acos(0.5))
       self.assertAllClose(0.5, tf.rsqrt(4.0))
 
+  @test_util.run_v1_only("b/120545219")
   def testSerializeSparseTensor(self):
     sp_input = tf.SparseTensor(
         indices=tf.constant([[1]], dtype=tf.int64),
@@ -44,6 +46,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
       self.assertEqual((3,), serialized_sp.shape)
       self.assertTrue(serialized_sp[0].numpy())  # check non-empty
 
+  @test_util.run_v1_only("b/120545219")
   def testSerializeManySparse(self):
     sp_input = tf.SparseTensor(
         indices=tf.constant([[0, 1]], dtype=tf.int64),
@@ -55,6 +58,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
           sp_input, 'serialize_name', tf.string)
       self.assertEqual((1, 3), serialized_sp.shape)
 
+  @test_util.run_v1_only("b/120545219")
   def testArgMaxMin(self):
     self.assertAllClose(
         [1],
-- 
GitLab


From 323f3eb4c2f15c713e992881ea8854069f2433f4 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 7 Dec 2018 17:41:38 -0800
Subject: [PATCH 1540/1554] [tf.data] Replace internal mechanism for copying
 batch elements in MapAndBatchDataset as the current mechanism could result in
 a segfault if the elements to be batched are DT_VARIANT.

PiperOrigin-RevId: 224605099
---
 .../data/experimental/map_and_batch_dataset_op.cc  |  4 ++--
 .../map_and_batch_dataset_serialization_test.py    | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index d3ae845cfb..d86c3a1a63 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -455,8 +455,8 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                 // TODO(mrry): Add a version of DoParallelConcat that allows us
                 // to move `tensor` where possible, to speed up string tensor
                 // batching.
-                Status copy_status = ::tensorflow::functor::DoParallelConcat(
-                    *dataset()->device_, tensor, offset, batch);
+                Status copy_status =
+                    batch_util::CopyElementToSlice(tensor, batch, offset);
                 if (!copy_status.ok()) {
                   result->UpdateStatus(copy_status, offset);
                   break;
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/map_and_batch_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/map_and_batch_dataset_serialization_test.py
index 166ffa99ca..8bfe6ce2f3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/map_and_batch_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/map_and_batch_dataset_serialization_test.py
@@ -22,6 +22,7 @@ import math
 from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -83,6 +84,19 @@ class MapAndBatchDatasetSerializationTest(
     self.run_core_tests(lambda: build_ds(10, True), lambda: build_ds(15, True),
                         num_outputs_drop_remainder)
 
+  def testSparse(self):
+
+    def build_dataset():
+
+      def map_fn(i):
+        return sparse_tensor.SparseTensorValue(
+            indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+      return dataset_ops.Dataset.range(10).apply(
+          batching.map_and_batch(map_fn, 5))
+
+    self.run_core_tests(build_dataset, None, 2)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From b05c71ac5e93b95f0265ed43102aa27f74f5ec26 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Fri, 7 Dec 2018 17:59:49 -0800
Subject: [PATCH 1541/1554] Use the new Dataset.reduce in `for` loops.

PiperOrigin-RevId: 224606535
---
 .../autograph/operators/control_flow.py       | 45 +++++--------------
 1 file changed, 10 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index 670897744a..89f7b8522f 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_math_ops
@@ -100,6 +99,7 @@ def _known_len_for_stmt(iter_, extra_test, body, init_state):
       extra_deps=(iter_,),
       opts=dict(maximum_iterations=n))
   # Dropping the iteration index because it's not syntactically visible.
+  # TODO(mdan): Don't.
   results = results[1:]
 
   # TODO(mdan): Remove this special case.
@@ -110,40 +110,15 @@ def _known_len_for_stmt(iter_, extra_test, body, init_state):
 
 def _dataset_for_stmt(ds, extra_test, body, init_state):
   """Overload of for_stmt that iterates over TF Datasets."""
-  # Because Datsets only expose get_next, in the style of Python iterators,
-  # we are forced to unpack the loop as:
-  #
-  # epoch_number, iterate = ds.get_next()
-  # while epoch_number < 2:
-  #   <body>
-  #   epoch_number, iterate = ds.get_next()
-  epoch_numbers = dataset_ops.Dataset.range(2)
-  def tag_with(ds, tag):
-    return dataset_ops.Dataset.zip(
-        (dataset_ops.Dataset.from_tensors(tag).repeat(), ds))
-  ds_with_epoch = epoch_numbers.flat_map(lambda i: tag_with(ds, i))
-
-  iterator = dataset_ops.make_initializable_iterator(ds_with_epoch)
-  with ops.control_dependencies((iterator.initializer,)):
-    epoch_number, iterate = iterator.get_next()
-
-    def while_body(epoch_number, iterate, *state):
-      new_state = body(iterate, *state)
-      epoch_number, iterate = iterator.get_next()
-      return (epoch_number, iterate) + new_state
-
-    def while_cond(epoch_number, iterate, *state):
-      del iterate
-      return gen_math_ops.logical_and(epoch_number < 1, extra_test(*state))
-
-    results = while_stmt(
-        while_cond,
-        while_body,
-        init_state=(epoch_number, iterate) + init_state,
-        extra_deps=())
-  # Dropping the epoch number and iterate because they are not syntactically
-  # visible.
-  results = results[2:]
+  if extra_test(*init_state) is not True:
+    raise NotImplementedError(
+        'break statements are not yet supported in for/Dataset loops')
+
+  def reduce_body(state, iterate):
+    new_state = body(iterate, *state)
+    return new_state
+
+  results = ds.reduce(init_state, reduce_body)
 
   # TODO(mdan): Remove this special case.
   if len(results) == 1:
-- 
GitLab


From 4c27508f8185c3d92f35c665aa02411623a7b165 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 7 Dec 2018 18:00:43 -0800
Subject: [PATCH 1542/1554] [tf.data] Add support for unbatching to the
 `Structure` class.

PiperOrigin-RevId: 224606610
---
 .../python/data/experimental/ops/batching.py  | 13 ++--
 tensorflow/python/data/ops/dataset_ops.py     | 73 +++++++------------
 tensorflow/python/data/ops/optional_ops.py    |  8 ++
 tensorflow/python/data/util/BUILD             |  1 +
 tensorflow/python/data/util/structure.py      | 66 +++++++++++++++++
 tensorflow/python/data/util/structure_test.py | 72 +++++++++++++++++-
 6 files changed, 176 insertions(+), 57 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index f70a4e0537..29df98f4ea 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -24,7 +24,6 @@ from tensorflow.python.data.experimental.ops import grouping
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import convert
 from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -406,21 +405,19 @@ def unbatch():
 
   def _apply_fn(dataset):
     """Function from `Dataset` to `Dataset` that applies the transformation."""
-    if not sparse.any_sparse(dataset.output_classes):
-      return _UnbatchDataset(dataset)
-
     # NOTE(mrry): We must ensure that any SparseTensors in `dataset`
     # are normalized to the rank-1 dense representation, so that the
     # sparse-oblivious unbatching logic will slice them
     # appropriately. This leads to a somewhat inefficient re-encoding step
     # for all SparseTensor components.
-    # TODO(mrry): Consider optimizing this in future
-    # if it turns out to be a bottleneck.
+    # TODO(mrry): Consider optimizing this in future if it turns out to be
+    # a bottleneck.
     def normalize(arg, *rest):
+      # pylint: disable=protected-access
       if rest:
-        return sparse.serialize_many_sparse_tensors((arg,) + rest)
+        return dataset._element_structure._to_batched_tensor_list((arg,) + rest)
       else:
-        return sparse.serialize_many_sparse_tensors(arg)
+        return dataset._element_structure._to_batched_tensor_list(arg)
 
     normalized_dataset = dataset.map(normalize)
 
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index e08a3af852..bee04aaef2 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1800,21 +1800,12 @@ class TensorDataset(DatasetSource):
               t, name="component_%d" % i)
           for i, t in enumerate(nest.flatten(tensors))
       ])
-
-    self._tensors = sparse.serialize_sparse_tensors(tensors)
-    output_classes = sparse.get_classes(tensors)
-    output_shapes = nest.pack_sequence_as(
-        tensors, [t.get_shape() for t in nest.flatten(tensors)])
-    output_types = nest.pack_sequence_as(
-        tensors, [t.dtype for t in nest.flatten(tensors)])
-    self._structure = structure_lib.convert_legacy_structure(
-        output_types, output_shapes, output_classes)
+    self._structure = structure_lib.Structure.from_value(tensors)
+    self._tensors = self._structure._to_tensor_list(tensors)  # pylint: disable=protected-access
 
   def _as_variant_tensor(self):
-    # pylint: disable=protected-access
     return gen_dataset_ops.tensor_dataset(
-        nest.flatten(self._tensors),
-        output_shapes=self._structure._flat_shapes)
+        self._tensors, output_shapes=self._structure._flat_shapes)  # pylint: disable=protected-access
 
   @property
   def _element_structure(self):
@@ -1834,27 +1825,22 @@ class TensorSliceDataset(DatasetSource):
               t, name="component_%d" % i)
           for i, t in enumerate(nest.flatten(tensors))
       ])
-      flat_tensors = nest.flatten(tensors)
+
+    batched_structure = structure_lib.Structure.from_value(tensors)
+    # pylint: disable=protected-access
+    self._tensors = batched_structure._to_batched_tensor_list(tensors)
+    self._structure = batched_structure._unbatch()
+    # pylint: enable=protected-access
 
     batch_dim = tensor_shape.Dimension(tensor_shape.dimension_value(
-        flat_tensors[0].get_shape()[0]))
-    for t in flat_tensors[1:]:
+        self._tensors[0].get_shape()[0]))
+    for t in self._tensors[1:]:
       batch_dim.assert_is_compatible_with(tensor_shape.Dimension(
           tensor_shape.dimension_value(t.get_shape()[0])))
-    self._tensors = sparse.serialize_many_sparse_tensors(tensors)
-    output_classes = sparse.get_classes(tensors)
-    output_shapes = nest.pack_sequence_as(
-        tensors, [t.get_shape()[1:] for t in nest.flatten(tensors)])
-    output_types = nest.pack_sequence_as(
-        tensors, [t.dtype for t in nest.flatten(tensors)])
-    self._structure = structure_lib.convert_legacy_structure(
-        output_types, output_shapes, output_classes)
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.tensor_slice_dataset(
-        nest.flatten(self._tensors),
-        output_shapes=nest.flatten(
-            self._structure._to_legacy_output_shapes()))  # pylint: disable=protected-access
+        self._tensors, output_shapes=self._structure._flat_shapes)  # pylint: disable=protected-access
 
   @property
   def _element_structure(self):
@@ -1871,17 +1857,13 @@ class SparseTensorSliceDataset(DatasetSource):
       raise TypeError("`sparse_tensor` must be a `tf.SparseTensor` object.")
     self._sparse_tensor = sparse_tensor
 
-    output_classes = (ops.Tensor, ops.Tensor, ops.Tensor)
     indices_shape = self._sparse_tensor.indices.get_shape()
     shape_shape = self._sparse_tensor.dense_shape.get_shape()
     rank = (indices_shape.dims[1] - 1).merge_with(shape_shape.dims[0] - 1)
-    num_values = tensor_shape.Dimension(None)
-    output_shapes = (tensor_shape.TensorShape([num_values, rank]),
-                     tensor_shape.TensorShape([num_values]),
-                     tensor_shape.TensorShape([rank]))
-    output_types = (dtypes.int64, self._sparse_tensor.dtype, dtypes.int64)
-    self._structure = structure_lib.convert_legacy_structure(
-        output_types, output_shapes, output_classes)
+    self._structure = structure_lib.NestedStructure(
+        (structure_lib.TensorStructure(dtypes.int64, [None, rank]),
+         structure_lib.TensorStructure(self._sparse_tensor.dtype, [None]),
+         structure_lib.TensorStructure(dtypes.int64, [rank])))
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.sparse_tensor_slice_dataset(
@@ -1936,6 +1918,9 @@ class DatasetStructure(structure_lib.Structure):
   def _to_tensor_list(self, value):
     return [value._as_variant_tensor()]  # pylint: disable=protected-access
 
+  def _to_batched_tensor_list(self, value):
+    raise NotImplementedError("Unbatching for `tf.data.Dataset` objects.")
+
   def _from_tensor_list(self, flat_value):
     if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
         not flat_value[0].shape.is_compatible_with(tensor_shape.scalar())):
@@ -1963,6 +1948,9 @@ class DatasetStructure(structure_lib.Structure):
   def _batch(self, batch_size):
     raise NotImplementedError("Batching for `tf.data.Dataset` objects.")
 
+  def _unbatch(self):
+    raise NotImplementedError("Unbatching for `tf.data.Dataset` objects.")
+
 
 # pylint: disable=protected-access
 structure_lib.Structure._register_custom_converter(DatasetV2,
@@ -2153,25 +2141,14 @@ class _GeneratorDataset(DatasetSource):
         destroyed. The return value is ignored.
     """
     super(_GeneratorDataset, self).__init__()
-    # These members will be initialized by `tf_init_func`.
-    self._state_classes = None
-    self._state_shapes = None
-    self._state_types = None
-
     self._init_args = init_args
 
-    init_args_classes = sparse.get_classes(init_args)
-    init_args_shapes = nest.pack_sequence_as(
-        init_args, [t.get_shape() for t in nest.flatten(init_args)])
-    init_args_types = nest.pack_sequence_as(
-        init_args, [t.dtype for t in nest.flatten(init_args)])
+    self._init_structure = structure_lib.Structure.from_value(init_args)
 
     self._init_func = StructuredFunctionWrapper(
         init_func,
         self._transformation_name(),
-        input_classes=init_args_classes,
-        input_shapes=init_args_shapes,
-        input_types=init_args_types)
+        input_structure=self._init_structure)
 
     self._next_func = StructuredFunctionWrapper(
         next_func,
@@ -2185,7 +2162,7 @@ class _GeneratorDataset(DatasetSource):
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.generator_dataset(
-        nest.flatten(self._init_args)
+        self._init_structure._to_tensor_list(self._init_args)  # pylint: disable=protected-access
         + self._init_func.function.captured_inputs,
         self._next_func.function.captured_inputs,
         self._finalize_func.function.captured_inputs,
diff --git a/tensorflow/python/data/ops/optional_ops.py b/tensorflow/python/data/ops/optional_ops.py
index 66011d8518..dcb743bee0 100644
--- a/tensorflow/python/data/ops/optional_ops.py
+++ b/tensorflow/python/data/ops/optional_ops.py
@@ -169,6 +169,10 @@ class OptionalStructure(structure.Structure):
   def _to_tensor_list(self, value):
     return [value._variant_tensor]  # pylint: disable=protected-access
 
+  def _to_batched_tensor_list(self, value):
+    raise NotImplementedError(
+        "Unbatching for `tf.data.experimental.Optional` objects.")
+
   def _from_tensor_list(self, flat_value):
     if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
         not flat_value[0].shape.is_compatible_with(tensor_shape.scalar())):
@@ -197,6 +201,10 @@ class OptionalStructure(structure.Structure):
     raise NotImplementedError(
         "Batching for `tf.data.experimental.Optional` objects.")
 
+  def _unbatch(self):
+    raise NotImplementedError(
+        "Unbatching for `tf.data.experimental.Optional` objects.")
+
 
 # pylint: disable=protected-access
 structure.Structure._register_custom_converter(Optional,
diff --git a/tensorflow/python/data/util/BUILD b/tensorflow/python/data/util/BUILD
index f15ebc32a8..04e80299e0 100644
--- a/tensorflow/python/data/util/BUILD
+++ b/tensorflow/python/data/util/BUILD
@@ -93,6 +93,7 @@ py_test(
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:variables",
+        "//tensorflow/python/data/kernel_tests:test_base",
         "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/python/data/util/structure.py b/tensorflow/python/data/util/structure.py
index 5e3addacaa..9de0c4da0e 100644
--- a/tensorflow/python/data/util/structure.py
+++ b/tensorflow/python/data/util/structure.py
@@ -113,6 +113,26 @@ class Structure(object):
     """
     raise NotImplementedError("Structure._to_tensor_list()")
 
+  @abc.abstractmethod
+  def _to_batched_tensor_list(self, value):
+    """Returns a flat list of rank >= 1 `tf.Tensor` representing `value`.
+
+    This method can be used, along with `self._flat_shapes` and
+    `self._flat_types` to represent structured values in lower level APIs
+    (such as plain TensorFlow operations) that do not understand structure,
+    *and* that require that the plain tensors have a rank of at least one
+    (e.g. for the purpose of slicing the tensors).
+
+    Requires: `self.is_compatible_with(Structure.from_value(value))`.
+
+    Args:
+      value: A value with compatible structure.
+
+    Returns:
+      A flat list of `tf.Tensor` representing `value`.
+    """
+    raise NotImplementedError("Structure._to_batched_tensor_list()")
+
   @abc.abstractmethod
   def _from_tensor_list(self, flat_value):
     """Builds a flat list of `tf.Tensor` into a value matching this structure.
@@ -159,6 +179,10 @@ class Structure(object):
     """
     raise NotImplementedError("Structure._batch()")
 
+  @abc.abstractmethod
+  def _unbatch(self):
+    raise NotImplementedError("Structure._unbatch()")
+
   @staticmethod
   def from_value(value):
     """Returns a `Structure` that represents the given `value`.
@@ -329,6 +353,22 @@ class NestedStructure(Structure):
       ret.extend(structure._to_tensor_list(sub_value))
     return ret
 
+  def _to_batched_tensor_list(self, value):
+    ret = []
+
+    try:
+      flat_value = nest.flatten_up_to(self._nested_structure, value)
+    except (ValueError, TypeError):
+      raise ValueError("The value %r is not compatible with the nested "
+                       "structure %r." % (value, self._nested_structure))
+
+    for sub_value, structure in zip(flat_value, self._flat_nested_structure):
+      if not structure.is_compatible_with(Structure.from_value(sub_value)):
+        raise ValueError("Component value %r is not compatible with the nested "
+                         "structure %r." % (sub_value, structure))
+      ret.extend(structure._to_batched_tensor_list(sub_value))
+    return ret
+
   def _from_tensor_list(self, flat_value):
     if len(flat_value) != len(self._flat_types):
       raise ValueError("Expected %d flat values in NestedStructure but got %d."
@@ -378,6 +418,10 @@ class NestedStructure(Structure):
     return NestedStructure(nest.map_structure(
         lambda s: s._batch(batch_size), self._nested_structure))
 
+  def _unbatch(self):
+    return NestedStructure(nest.map_structure(
+        lambda s: s._unbatch(), self._nested_structure))
+
 
 @tf_export("data.experimental.TensorStructure")
 class TensorStructure(Structure):
@@ -406,6 +450,11 @@ class TensorStructure(Structure):
                        "and shape %s." % (value, self._dtype, self._shape))
     return [value]
 
+  def _to_batched_tensor_list(self, value):
+    if self._shape.merge_with(value.shape).ndims == 0:
+      raise ValueError("Unbatching a tensor is only supported for rank >= 1")
+    return [value]
+
   def _from_tensor_list(self, flat_value):
     if len(flat_value) != 1:
       raise ValueError("TensorStructure corresponds to a single tf.Tensor.")
@@ -442,6 +491,11 @@ class TensorStructure(Structure):
         self._dtype,
         tensor_shape.TensorShape([batch_size]).concatenate(self._shape))
 
+  def _unbatch(self):
+    if self._shape.ndims == 0:
+      raise ValueError("Unbatching a tensor is only supported for rank >= 1")
+    return TensorStructure(self._dtype, self._shape[1:])
+
 
 @tf_export("data.experimental.SparseTensorStructure")
 class SparseTensorStructure(Structure):
@@ -471,6 +525,13 @@ class SparseTensorStructure(Structure):
   def _to_tensor_list(self, value):
     return [sparse_ops.serialize_sparse(value, out_type=dtypes.variant)]
 
+  def _to_batched_tensor_list(self, value):
+    if self._dense_shape.merge_with(
+        tensor_util.constant_value_as_shape(value.dense_shape)).ndims == 0:
+      raise ValueError(
+          "Unbatching a sparse tensor is only supported for rank >= 1")
+    return [sparse_ops.serialize_many_sparse(value, out_type=dtypes.variant)]
+
   def _from_tensor_list(self, flat_value):
     if (len(flat_value) != 1 or flat_value[0].dtype != dtypes.variant or
         not flat_value[0].shape.is_compatible_with(tensor_shape.vector(3))):
@@ -505,3 +566,8 @@ class SparseTensorStructure(Structure):
     return SparseTensorStructure(
         self._dtype,
         tensor_shape.TensorShape([batch_size]).concatenate(self._dense_shape))
+
+  def _unbatch(self):
+    if self._dense_shape.ndims == 0:
+      raise ValueError("Unbatching a tensor is only supported for rank >= 1")
+    return SparseTensorStructure(self._dtype, self._dense_shape[1:])
diff --git a/tensorflow/python/data/util/structure_test.py b/tensorflow/python/data/util/structure_test.py
index 3425179e42..91dcfa6f60 100644
--- a/tensorflow/python/data/util/structure_test.py
+++ b/tensorflow/python/data/util/structure_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import structure
 from tensorflow.python.framework import constant_op
@@ -34,7 +35,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
-class StructureTest(test.TestCase, parameterized.TestCase):
+class StructureTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   # NOTE(mrry): The arguments must be lifted into lambdas because otherwise they
   # will be executed before the (eager- or graph-mode) test environment has been
@@ -418,6 +419,75 @@ class StructureTest(test.TestCase, parameterized.TestCase):
     self.assertTrue(
         expected_batched_structure.is_compatible_with(batched_structure))
 
+  @parameterized.named_parameters(
+      ("Tensor", structure.TensorStructure(dtypes.float32, [32]),
+       structure.TensorStructure(dtypes.float32, [])),
+      ("TensorUnknown", structure.TensorStructure(dtypes.float32, [None]),
+       structure.TensorStructure(dtypes.float32, [])),
+      ("SparseTensor",
+       structure.SparseTensorStructure(dtypes.float32, [32, None]),
+       structure.SparseTensorStructure(dtypes.float32, [None])),
+      ("SparseTensorUnknown",
+       structure.SparseTensorStructure(dtypes.float32, [None, 4]),
+       structure.SparseTensorStructure(dtypes.float32, [4])),
+      ("Nest", structure.NestedStructure({
+          "a": structure.TensorStructure(dtypes.float32, [128]),
+          "b": (structure.SparseTensorStructure(dtypes.int32, [128, 2, 2]),
+                structure.TensorStructure(dtypes.string, [None]))}),
+       structure.NestedStructure({
+           "a": structure.TensorStructure(dtypes.float32, []),
+           "b": (structure.SparseTensorStructure(dtypes.int32, [2, 2]),
+                 structure.TensorStructure(dtypes.string, []))})),
+  )
+  def testUnbatch(self, element_structure, expected_unbatched_structure):
+    unbatched_structure = element_structure._unbatch()
+    self.assertTrue(
+        unbatched_structure.is_compatible_with(expected_unbatched_structure))
+    self.assertTrue(
+        expected_unbatched_structure.is_compatible_with(unbatched_structure))
+
+  # pylint: disable=g-long-lambda
+  @parameterized.named_parameters(
+      ("Tensor", lambda: constant_op.constant([[1.0, 2.0], [3.0, 4.0]]),
+       lambda: constant_op.constant([1.0, 2.0])),
+      ("SparseTensor", lambda: sparse_tensor.SparseTensor(
+          indices=[[0, 0], [1, 1]], values=[13, 27], dense_shape=[2, 2]),
+       lambda: sparse_tensor.SparseTensor(
+           indices=[[0]], values=[13], dense_shape=[2])),
+      ("Nest", lambda: (
+          constant_op.constant([[1.0, 2.0], [3.0, 4.0]]),
+          sparse_tensor.SparseTensor(
+              indices=[[0, 0], [1, 1]], values=[13, 27], dense_shape=[2, 2])),
+       lambda: (constant_op.constant([1.0, 2.0]), sparse_tensor.SparseTensor(
+           indices=[[0]], values=[13], dense_shape=[2]))),
+  )
+  def testToBatchedTensorList(self, value_fn, element_0_fn):
+    batched_value = value_fn()
+    s = structure.Structure.from_value(batched_value)
+    batched_tensor_list = s._to_batched_tensor_list(batched_value)
+
+    # The batch dimension is 2 for all of the test cases.
+    # NOTE(mrry): `tf.shape()` does not currently work for the DT_VARIANT
+    # tensors in which we store sparse tensors.
+    for t in batched_tensor_list:
+      if t.dtype != dtypes.variant:
+        self.assertEqual(2, self.evaluate(array_ops.shape(t)[0]))
+
+    # Test that the 0th element from the unbatched tensor is equal to the
+    # expected value.
+    expected_element_0 = self.evaluate(element_0_fn())
+    unbatched_s = s._unbatch()
+    actual_element_0 = unbatched_s._from_tensor_list(
+        [t[0] for t in batched_tensor_list])
+
+    for expected, actual in zip(
+        nest.flatten(expected_element_0), nest.flatten(actual_element_0)):
+      if sparse_tensor.is_sparse(expected):
+        self.assertSparseValuesEqual(expected, actual)
+      else:
+        self.assertAllEqual(expected, actual)
+
+  # pylint: enable=g-long-lambda
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From cc52e9bfc9a30422868826514f41065bf35167ca Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Fri, 7 Dec 2018 18:02:49 -0800
Subject: [PATCH 1543/1554] [tf.data] Enable `shuffle_and_repeat_fusion`
 optimization by default

PiperOrigin-RevId: 224606882
---
 .../optimization/optimize_dataset_test.py          | 14 +++++++++-----
 .../optimization/shuffle_and_repeat_fusion_test.py |  5 -----
 .../data/experimental/ops/optimization_options.py  |  5 +++--
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index 05d88eb071..230b74e9e8 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -230,10 +230,10 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     self.assertGreaterEqual(len(w), 1)
     expected = ("tf.data static optimizations are not compatible with "
-                "tf.Variable. The following optimizations will be disabled: "
-                "map_and_batch_fusion, noop_elimination. To enable "
-                "optimizations, use resource variables instead by calling "
-                "`tf.enable_resource_variables()` at the start of the program.")
+                "tf.Variable. The following optimizations will be disabled: %s."
+                " To enable optimizations, use resource variables instead by "
+                "calling `tf.enable_resource_variables()` at the start of the "
+                "program." % (", ".join(opt_options._static_optimizations())))
     self.assertTrue(any([expected in str(warning) for warning in w]))
 
     # Check that outputs are the same in the optimized and unoptimized cases,
@@ -256,7 +256,11 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testOptimizationEnabledByDefault(self):
     """Tests that some optimizations are applied to datasets by default."""
     options = dataset_ops.Options()
-    expected_optimizations = ["noop_elimination", "map_and_batch_fusion"]
+    expected_optimizations = [
+        "map_and_batch_fusion",
+        "noop_elimination",
+        "shuffle_and_repeat_fusion",
+    ]
     self.assertEqual(
         set(options._static_optimizations()), set(expected_optimizations))
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
index 5f746ec63a..594b59375f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.data.experimental.ops import optimization
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
@@ -32,10 +31,6 @@ class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase):
   def testShuffleAndRepeatFusion(self):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(["ShuffleAndRepeat"])).shuffle(10).repeat(2)
-    options = dataset_ops.Options()
-    options.experimental_optimization = OptimizationOptions()
-    options.experimental_optimization.shuffle_and_repeat_fusion = True
-    dataset = dataset.with_options(options)
     get_next = self.getNext(dataset)
 
     for _ in range(2):
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 73849435a9..11b8b86f64 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -86,7 +86,8 @@ class OptimizationOptions(options.OptionsBase):
   shuffle_and_repeat_fusion = options.create_option(
       name="shuffle_and_repeat_fusion",
       ty=bool,
-      docstring="Whether to fuse shuffle and repeat transformations.")
+      docstring="Whether to fuse shuffle and repeat transformations. If None, "
+      "defaults to True.")
 
   def _static_optimizations(self):
     """Produces the list of enabled static optimizations."""
@@ -98,7 +99,6 @@ class OptimizationOptions(options.OptionsBase):
         "map_fusion",
         "map_parallelization",
         "map_vectorization",
-        "shuffle_and_repeat_fusion",
     ]
     for optimization in optimizations_to_enable:
       if getattr(self, optimization):
@@ -110,6 +110,7 @@ class OptimizationOptions(options.OptionsBase):
       optimizations_to_disable = [
           "map_and_batch_fusion",
           "noop_elimination",
+          "shuffle_and_repeat_fusion",
       ]
       for optimization in optimizations_to_disable:
         if getattr(self, optimization) is not False:
-- 
GitLab


From 798543c1e5cfaa6acf56b087fcb56bd0e227db5d Mon Sep 17 00:00:00 2001
From: Eddie Zhou <eddz@google.com>
Date: Fri, 7 Dec 2018 18:10:29 -0800
Subject: [PATCH 1544/1554] Also skip test_session post-setUp().

PiperOrigin-RevId: 224607628
---
 tensorflow/python/framework/test_util.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 28f25daa6d..df3cebd2e0 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1717,6 +1717,8 @@ class TensorFlowTestCase(googletest.TestCase):
                    use_gpu=False,
                    force_gpu=False):
     """Use cached_session instead."""
+    if self.id().endswith(".test_session"):
+      self.skipTest("Not a test.")
     if context.executing_eagerly():
       yield None
     else:
-- 
GitLab


From 1881396166c56fb04963683998e6a1569ebf3b19 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 7 Dec 2018 18:11:02 -0800
Subject: [PATCH 1545/1554] Internal change.

PiperOrigin-RevId: 224607664
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 2820e466f3..62e1eaa366 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -123,7 +123,8 @@ fi
 run_configure_for_cpu_build
 
 bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS} \
-  tensorflow/tools/pip_package:build_pip_package || exit $?
+  tensorflow/tools/pip_package:build_pip_package \
+  --incompatible_remove_native_http_archive=false || exit $?
 
 if [[ "$SKIP_TEST" == 1 ]]; then
   exit 0
-- 
GitLab


From 237851c57093edd5126561b2651f216d1726487b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 20:34:57 -0800
Subject: [PATCH 1546/1554] Internal Change

PiperOrigin-RevId: 224616455
---
 .../base_api/api_def_RaggedGather.pbtxt       |    6 +-
 .../base_api/api_def_RaggedRange.pbtxt        |    2 +-
 .../api_def_RaggedTensorToSparse.pbtxt        |    2 +-
 tensorflow/core/kernels/unicode_ops.cc        |    2 +-
 .../kernel_tests/unicode_encode_op_test.py    |  127 +-
 tensorflow/python/ops/ragged/BUILD            |   56 +-
 tensorflow/python/ops/ragged/__init__.py      |   76 +-
 ...vert_to_tensor_or_ragged_tensor_op_test.py |   19 +-
 .../python/ops/ragged/ragged_array_ops.py     |  415 ++----
 .../ops/ragged/ragged_batch_gather_op_test.py |   59 +-
 .../ops/ragged/ragged_boolean_mask_op_test.py |   36 +-
 .../ops/ragged/ragged_concat_op_test.py       |   35 +-
 .../python/ops/ragged/ragged_const_op_test.py |   46 +-
 .../ragged/ragged_constant_value_op_test.py   |   13 +-
 .../ops/ragged/ragged_conversion_ops.py       |  402 +-----
 .../python/ops/ragged/ragged_dispatch.py      |   25 +-
 .../python/ops/ragged/ragged_dispatch_test.py |   90 +-
 .../python/ops/ragged/ragged_eager_test.py    |   22 +-
 .../ops/ragged/ragged_expand_dims_op_test.py  |    8 +-
 .../python/ops/ragged/ragged_factory_ops.py   |  465 +------
 .../ops/ragged/ragged_from_sparse_op_test.py  |   87 +-
 .../ops/ragged/ragged_from_tensor_op_test.py  |   60 +-
 .../ops/ragged/ragged_functional_ops.py       |   33 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |   23 +-
 .../ops/ragged/ragged_gather_op_test.py       |  128 +-
 .../python/ops/ragged/ragged_getitem.py       |   12 +-
 ...t.py => ragged_map_flat_values_op_test.py} |   60 +-
 .../ops/ragged/ragged_map_fn_op_test.py       |   96 +-
 .../python/ops/ragged/ragged_map_ops.py       |   44 +-
 .../python/ops/ragged/ragged_math_ops.py      |   49 +-
 .../ops/ragged/ragged_operators_test.py       |   88 +-
 .../python/ops/ragged/ragged_range_op_test.py |  159 +--
 .../ops/ragged/ragged_reduce_op_test.py       |   27 +-
 .../ops/ragged/ragged_row_lengths_op_test.py  |   81 +-
 ...agged_row_splits_to_segment_ids_op_test.py |   13 +-
 ...agged_segment_ids_to_row_splits_op_test.py |   22 +-
 .../ops/ragged/ragged_segment_op_test.py      |   43 +-
 .../python/ops/ragged/ragged_stack_op_test.py |   13 +-
 .../python/ops/ragged/ragged_string_ops.py    |   19 +-
 tensorflow/python/ops/ragged/ragged_tensor.py | 1207 +++++++++++++++--
 .../ragged_tensor_bounding_shape_op_test.py   |   47 +-
 .../python/ops/ragged/ragged_tensor_shape.py  |   44 +-
 .../ops/ragged/ragged_tensor_shape_test.py    |   66 +-
 .../python/ops/ragged/ragged_tensor_test.py   |  623 ++++-----
 .../python/ops/ragged/ragged_tensor_value.py  |   15 +-
 .../python/ops/ragged/ragged_test_util.py     |   95 ++
 .../python/ops/ragged/ragged_tile_op_test.py  |   22 +-
 .../ops/ragged/ragged_to_sparse_op_test.py    |  225 ++-
 .../ops/ragged/ragged_to_tensor_op_test.py    |   33 +-
 .../python/ops/ragged/ragged_util_test.py     |   11 +-
 .../python/ops/ragged/ragged_where_op_test.py |   16 +-
 tensorflow/tools/pip_package/BUILD            |    1 +
 52 files changed, 2523 insertions(+), 2845 deletions(-)
 rename tensorflow/python/ops/ragged/{ragged_map_inner_values_op_test.py => ragged_map_flat_values_op_test.py} (79%)
 create mode 100644 tensorflow/python/ops/ragged/ragged_test_util.py

diff --git a/tensorflow/core/api_def/base_api/api_def_RaggedGather.pbtxt b/tensorflow/core/api_def/base_api/api_def_RaggedGather.pbtxt
index 240c987dda..9c40332ea2 100644
--- a/tensorflow/core/api_def/base_api/api_def_RaggedGather.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RaggedGather.pbtxt
@@ -11,8 +11,8 @@ END
   in_arg {
     name: "params_dense_values"
     description: <<END
-The `inner_values` for the `params` RaggedTensor. There was a terminology change
-at the python level from dense_values to inner_values, so dense_values is the
+The `flat_values` for the `params` RaggedTensor. There was a terminology change
+at the python level from dense_values to flat_values, so dense_values is the
 deprecated name.
 END
   }
@@ -32,7 +32,7 @@ END
   }
   out_arg {
     name: "output_dense_values"
-    description: "The `inner_values` for the returned RaggedTensor."
+    description: "The `flat_values` for the returned RaggedTensor."
   }
   attr {
     name: "PARAMS_RAGGED_RANK"
diff --git a/tensorflow/core/api_def/base_api/api_def_RaggedRange.pbtxt b/tensorflow/core/api_def/base_api/api_def_RaggedRange.pbtxt
index 927e839b72..4a9b2af804 100644
--- a/tensorflow/core/api_def/base_api/api_def_RaggedRange.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RaggedRange.pbtxt
@@ -19,7 +19,7 @@ op {
   }
   out_arg{
     name: "rt_dense_values"
-    description: "The `inner_values` for the returned `RaggedTensor`."
+    description: "The `flat_values` for the returned `RaggedTensor`."
   }
   summary: <<END
 Returns a `RaggedTensor` containing the specified sequences of numbers.
diff --git a/tensorflow/core/api_def/base_api/api_def_RaggedTensorToSparse.pbtxt b/tensorflow/core/api_def/base_api/api_def_RaggedTensorToSparse.pbtxt
index 8c73ea644c..958c71185e 100644
--- a/tensorflow/core/api_def/base_api/api_def_RaggedTensorToSparse.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RaggedTensorToSparse.pbtxt
@@ -7,7 +7,7 @@ op {
   }
   in_arg {
     name: "rt_dense_values"
-    description: "The `inner_values` for the `RaggedTensor`."
+    description: "The `flat_values` for the `RaggedTensor`."
   }
   out_arg {
     name: "sparse_indices"
diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc
index 6c4ed1eaaf..3ee0edb35a 100644
--- a/tensorflow/core/kernels/unicode_ops.cc
+++ b/tensorflow/core/kernels/unicode_ops.cc
@@ -493,7 +493,7 @@ class UnicodeEncodeOp : public OpKernel {
     const Tensor& input_splits = context->input(1);
     const auto input_splits_flat = input_splits.flat<int64>();
 
-    // Since we limit to a 2-D input (inner_values of rank 1 and a single splits
+    // Since we limit to a 2-D input (flat_values of rank 1 and a single splits
     // tensor), our output dimension will be 1 with it's size equal to the
     // number of splits (outer dimension or ragged tensor).
     TensorShape output_shape({input_splits.dim_size(0) - 1});
diff --git a/tensorflow/python/kernel_tests/unicode_encode_op_test.py b/tensorflow/python/kernel_tests/unicode_encode_op_test.py
index 72c8a26766..2f3cd8a657 100644
--- a/tensorflow/python/kernel_tests/unicode_encode_op_test.py
+++ b/tensorflow/python/kernel_tests/unicode_encode_op_test.py
@@ -23,15 +23,25 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl as errors
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_string_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_tensor_value
 from tensorflow.python.platform import test
 
 
 class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
 
+  def assertRaggedEqual(self, rt, expected):
+    with self.cached_session() as sess:
+      value = sess.run(rt)
+      if isinstance(value, np.ndarray):
+        value = value.tolist()
+      elif isinstance(value, ragged_tensor_value.RaggedTensorValue):
+        value = value.to_list()
+      self.assertEqual(value, expected)
+
   def testScalar(self):
     with self.cached_session():
       with self.assertRaises(ValueError):
@@ -54,9 +64,10 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   def testStrictErrors(self, encoding):
     test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
-    with self.cached_session():
+    with self.cached_session() as session:
       with self.assertRaises(errors.InvalidArgumentError):
-        ragged_string_ops.unicode_encode(test_value, encoding, "strict").eval()
+        session.run(
+            ragged_string_ops.unicode_encode(test_value, encoding, "strict"))
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -65,8 +76,8 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
     expected_value = u"Heo".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
                                                          "ignore")
-    with self.cached_session():
-      result = unicode_encode_op.eval()
+    with self.cached_session() as session:
+      result = session.run(unicode_encode_op)
       self.assertIsInstance(result, bytes)
       self.assertAllEqual(result, expected_value)
 
@@ -77,37 +88,27 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
     expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
                                                          "replace")
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
     # Test custom replacement character
     test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
     expected_value = u"Heooo".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
                                                          "replace", 111)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
     # Verify "replace" is default
     test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
     expected_value = u"He\U0000fffd\U0000fffdo".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
     # Replacement_char must be within range
     test_value = np.array([72, 101, 2147483647, -1, 111], np.int32)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding,
                                                          "replace", 1114112)
-    with self.cached_session():
-      with self.assertRaises(errors.InvalidArgumentError):
-        unicode_encode_op.eval()
+    with self.assertRaises(errors.InvalidArgumentError):
+      self.evaluate(unicode_encode_op)
 
   # -- regular Tensor tests -- #
 
@@ -117,35 +118,23 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
     test_value = np.array([72, 101, 108, 108, 111], np.int32)
     expected_value = u"Hello".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
     test_value = np.array([72, 101, 195, 195, 128516], np.int32)
     expected_value = u"He\xc3\xc3\U0001f604".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
     # Single character string
     test_value = np.array([72], np.int32)
     expected_value = u"H".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
     test_value = np.array([128516], np.int32)
     expected_value = u"\U0001f604".encode(encoding)
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(result, bytes)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -156,10 +145,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
         u"H\U0001f604llo".encode(encoding), u"W\U0001f604rld".encode(encoding)
     ]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(unicode_encode_op, ops.Tensor)
-      self.assertAllEqual(result, expected_value)
+    self.assertAllEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -172,10 +158,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
                       [u"fixed".encode(encoding), u"words".encode(encoding)],
                       [u"Hyper".encode(encoding), u"cube.".encode(encoding)]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(unicode_encode_op, ops.Tensor)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -191,10 +174,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
                       [[u"Hyper".encode(encoding)],
                        [u"cube.".encode(encoding)]]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(unicode_encode_op, ops.Tensor)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   # -- Ragged Tensor tests -- #
 
@@ -207,10 +187,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
         u"H\xc3llo".encode(encoding), u"W\U0001f604rld.".encode(encoding)
     ]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertIsInstance(unicode_encode_op, ops.Tensor)
-      self.assertAllEqual(result, expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -227,10 +204,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
                           u"cube.".encode(encoding)
                       ]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertEqual(unicode_encode_op.ragged_rank, 1)
-      self.assertAllEqual(result.tolist(), expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -245,10 +219,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
                           u"w\xc3rry, be".encode(encoding)
                       ], [u"\U0001f604".encode(encoding), u"".encode(encoding)]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertEqual(unicode_encode_op.ragged_rank, 1)
-      self.assertAllEqual(result.tolist(), expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -259,10 +230,7 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
     expected_value = [[u"Hello".encode(encoding), u"World.".encode(encoding)],
                       [], [u"\U0001f604".encode(encoding)]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertEqual(unicode_encode_op.ragged_rank, 1)
-      self.assertAllEqual(result.tolist(), expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
@@ -273,41 +241,30 @@ class UnicodeEncodeOpTest(test.TestCase, parameterized.TestCase):
     expected_value = [[[u"Hello".encode(encoding), u"World".encode(encoding)]],
                       [[u"".encode(encoding)], [u"Hype".encode(encoding)]]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertEqual(unicode_encode_op.ragged_rank, 2)
-      self.assertAllEqual(result.tolist(), expected_value)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
   @parameterized.parameters("UTF-8", "UTF-16-BE", "UTF-32-BE")
   @test_util.run_v1_only("b/120545219")
   def testRaggedMatrixWithMultiDimensionInnerValues(self, encoding):
-    test_inner_values = constant_op.constant([[[72, 101, 108, 108, 111],
-                                               [87, 111, 114, 108, 100]],
-                                              [[102, 105, 120, 101, 100],
-                                               [119, 111, 114, 100, 115]],
-                                              [[72, 121, 112, 101, 114],
-                                               [99, 117, 98, 101, 46]]])
+    test_flat_values = constant_op.constant([[[72, 101, 108, 108, 111],
+                                              [87, 111, 114, 108, 100]],
+                                             [[102, 105, 120, 101, 100],
+                                              [119, 111, 114, 100, 115]],
+                                             [[72, 121, 112, 101, 114],
+                                              [99, 117, 98, 101, 46]]])
     test_row_splits = [
         constant_op.constant([0, 2, 3], dtype=np.int64),
         constant_op.constant([0, 1, 1, 3], dtype=np.int64)
     ]
-    test_value = ragged_factory_ops.from_nested_row_splits(test_inner_values,
-                                                           test_row_splits)
+    test_value = ragged_tensor.RaggedTensor.from_nested_row_splits(
+        test_flat_values, test_row_splits)
     expected_value = [[[[u"Hello".encode(encoding), u"World".encode(encoding)]],
                        []],
                       [[[u"fixed".encode(encoding), u"words".encode(encoding)],
                         [u"Hyper".encode(encoding),
                          u"cube.".encode(encoding)]]]]
     unicode_encode_op = ragged_string_ops.unicode_encode(test_value, encoding)
-    with self.cached_session():
-      result = unicode_encode_op.eval()
-      self.assertEqual(unicode_encode_op.ragged_rank, 2)
-      self.assertAllEqual(result.tolist(), expected_value)
-      # These next two assertions don't necessarily need to be here as they test
-      # internal representations and we already verified the value is correct.
-      self.assertAllEqual(len(result.nested_row_splits), len(test_row_splits))
-      self.assertEqual(unicode_encode_op.inner_values.shape.ndims,
-                       test_inner_values.shape.ndims - 1)
+    self.assertRaggedEqual(unicode_encode_op, expected_value)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index fcd9adad21..d88543c400 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -184,6 +184,8 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged_tensor_value",
+        ":ragged_util",
+        ":segment_id_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:session",
         "//tensorflow/python:tensor_shape",
@@ -295,6 +297,19 @@ py_library(
 # RaggedTensor Tests
 #-------------------------------------------------------------------------------
 
+py_library(
+    name = "ragged_test_util",
+    srcs = ["ragged_test_util.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_tensor",
+        ":ragged_tensor_value",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 py_test(
     name = "ragged_tensor_test",
     size = "medium",
@@ -305,6 +320,7 @@ py_test(
     ],
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
@@ -323,6 +339,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -336,6 +353,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -348,6 +366,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
@@ -359,6 +378,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "@absl_py//absl/testing:parameterized",
@@ -371,6 +391,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
@@ -387,6 +408,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
@@ -402,6 +424,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
@@ -417,6 +440,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -429,6 +453,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -441,6 +466,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_test_lib",
@@ -459,6 +485,7 @@ py_test(
     ],
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
@@ -475,6 +502,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
@@ -490,6 +518,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
@@ -504,6 +533,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:errors",
@@ -519,6 +549,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
@@ -530,11 +561,12 @@ py_test(
 )
 
 py_test(
-    name = "ragged_map_inner_values_op_test",
-    srcs = ["ragged_map_inner_values_op_test.py"],
+    name = "ragged_map_flat_values_op_test",
+    srcs = ["ragged_map_flat_values_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
@@ -551,8 +583,8 @@ py_test(
     srcs = ["ragged_const_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_factory_ops",
-        ":ragged_tensor",
+        ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -569,6 +601,7 @@ py_test(
     ],
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "//third_party/py/numpy",
@@ -582,6 +615,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
@@ -597,6 +631,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
@@ -612,6 +647,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
@@ -628,6 +664,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -640,8 +677,8 @@ py_test(
     srcs = ["ragged_tile_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
-        ":ragged_factory_ops",
+        ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
@@ -656,6 +693,7 @@ py_test(
     srcs = ["ragged_util_test.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":ragged_test_util",
         ":ragged_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -672,6 +710,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "@absl_py//absl/testing:parameterized",
@@ -684,6 +723,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "@absl_py//absl/testing:parameterized",
@@ -696,6 +736,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:clip_ops",
         "//tensorflow/python:dtypes",
@@ -717,6 +758,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
@@ -729,6 +771,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
@@ -747,6 +790,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged",
+        ":ragged_test_util",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index bfcaa366fc..3d915ee269 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -9,47 +9,6 @@ different lengths.  For example, the inner (column) dimension of
 description of ragged tensors, see the `tf.RaggedTensor`
 class documentation.
 
-## `RaggedTensor` Operations
-
-### `RaggedTensor` Factory ops
-
-* `tf.ragged.constant`
-* `tf.ragged.from_row_splits`
-* `tf.ragged.from_row_splits`
-* `tf.ragged.from_row_lengths`
-* `tf.ragged.from_row_starts`
-* `tf.ragged.from_row_limits`
-* `tf.ragged.from_value_rowids`
-* `tf.ragged.from_nested_row_splits`
-* `tf.ragged.from_nested_value_rowids`
-
-### `RaggedTensor` Conversion ops
-
-* `tf.ragged.from_tensor`
-* `tf.ragged.to_tensor`
-* `tf.ragged.from_sparse`
-* `tf.ragged.to_sparse`
-* `tf.ragged.from_variant`
-* `tf.ragged.to_variant`
-* `tf.ragged.convert_to_tensor_or_ragged_tensor`
-
-### `RaggedTensor` Shape ops
-
-* `tf.ragged.row_splits`
-* `tf.ragged.row_lengths`
-* `tf.ragged.row_starts`
-* `tf.ragged.row_limits`
-* `tf.ragged.value_rowids`
-* `tf.ragged.nrows`
-* `tf.ragged.nested_row_splits`
-* `tf.ragged.row_splits_to_segment_ids`
-* `tf.ragged.segment_ids_to_row_splits`
-* `tf.ragged.bounding_shape`
-
-### Functional ops
-* `tf.ragged.map_inner_values`
-
-
 <!-- Ragged Classes & related helper functions -->
 @@RaggedTensor
 @@RaggedTensorType
@@ -57,15 +16,9 @@ class documentation.
 @@is_ragged
 
 <!-- Factory Ops -->
+@@ragged_factory_ops
 @@constant
 @@constant_value
-@@from_row_splits
-@@from_row_lengths
-@@from_row_starts
-@@from_row_limits
-@@from_value_rowids
-@@from_nested_row_splits
-@@from_nested_value_rowids
 @@convert_to_tensor_or_ragged_tensor
 
 <!-- Conversion Ops -->
@@ -77,14 +30,6 @@ class documentation.
 @@segment_ids_to_row_splits
 
 <!-- Array Ops -->
-@@row_splits
-@@row_lengths
-@@row_starts
-@@row_limits
-@@value_rowids
-@@nrows
-@@nested_row_splits
-@@bounding_shape
 @@gather
 @@batch_gather
 @@gather_nd
@@ -114,7 +59,7 @@ class documentation.
 @@reduce_any
 
 <!-- Functional Ops -->
-@@map_inner_values
+@@map_flat_values
 @@map_fn
 
 <!-- Shape & broadcasting -->
@@ -133,18 +78,12 @@ from tensorflow.python.ops.ragged import ragged_string_ops
 
 from tensorflow.python.ops.ragged.ragged_array_ops import batch_gather
 from tensorflow.python.ops.ragged.ragged_array_ops import boolean_mask
-from tensorflow.python.ops.ragged.ragged_array_ops import bounding_shape
 from tensorflow.python.ops.ragged.ragged_array_ops import concat
 from tensorflow.python.ops.ragged.ragged_array_ops import expand_dims
 from tensorflow.python.ops.ragged.ragged_array_ops import gather
 from tensorflow.python.ops.ragged.ragged_array_ops import gather_nd
-from tensorflow.python.ops.ragged.ragged_array_ops import nrows
-from tensorflow.python.ops.ragged.ragged_array_ops import row_lengths
-from tensorflow.python.ops.ragged.ragged_array_ops import row_limits
-from tensorflow.python.ops.ragged.ragged_array_ops import row_starts
 from tensorflow.python.ops.ragged.ragged_array_ops import stack
 from tensorflow.python.ops.ragged.ragged_array_ops import tile
-from tensorflow.python.ops.ragged.ragged_array_ops import value_rowids
 from tensorflow.python.ops.ragged.ragged_array_ops import where
 
 from tensorflow.python.ops.ragged.ragged_conversion_ops import from_sparse
@@ -154,16 +93,8 @@ from tensorflow.python.ops.ragged.ragged_conversion_ops import to_tensor
 
 from tensorflow.python.ops.ragged.ragged_factory_ops import constant
 from tensorflow.python.ops.ragged.ragged_factory_ops import constant_value
-from tensorflow.python.ops.ragged.ragged_factory_ops import convert_to_tensor_or_ragged_tensor
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_nested_row_splits
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_nested_value_rowids
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_row_lengths
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_row_limits
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_row_splits
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_row_starts
-from tensorflow.python.ops.ragged.ragged_factory_ops import from_value_rowids
 
-from tensorflow.python.ops.ragged.ragged_functional_ops import map_inner_values
+from tensorflow.python.ops.ragged.ragged_functional_ops import map_flat_values
 
 from tensorflow.python.ops.ragged.ragged_map_ops import map_fn
 
@@ -184,6 +115,7 @@ from tensorflow.python.ops.ragged.ragged_math_ops import segment_prod
 from tensorflow.python.ops.ragged.ragged_math_ops import segment_sqrt_n
 from tensorflow.python.ops.ragged.ragged_math_ops import segment_sum
 
+from tensorflow.python.ops.ragged.ragged_tensor import convert_to_tensor_or_ragged_tensor
 from tensorflow.python.ops.ragged.ragged_tensor import is_ragged
 from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
 from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensorType
diff --git a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
index ef3464f243..b88f18c8b6 100644
--- a/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/convert_to_tensor_or_ragged_tensor_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged.convert_to_tensor_or_ragged_tensor."""
+"""Tests for ragged.convert_to_tensor_or_ragged."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -25,11 +25,13 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
-                                              parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedConvertToTensorOrRaggedTensorTest(
+    ragged_test_util.RaggedTensorTestCase, parameterized.TestCase):
 
   #=============================================================================
   # Tests where the 'value' param is a RaggedTensor
@@ -90,7 +92,6 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
           preferred_dtype=dtypes.string,
           expected_dtype=dtypes.int32),
   ])
-  @test_util.run_deprecated_v1
   def testConvertRaggedTensorValue(self,
                                    value,
                                    dtype=None,
@@ -102,8 +103,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
         value, dtype, preferred_dtype)
     self.assertEqual(value.ragged_rank, converted.ragged_rank)
     self.assertEqual(dtypes.as_dtype(expected_dtype), converted.dtype)
-    with self.test_session():
-      self.assertEqual(value.tolist(), self.evaluate(converted).tolist())
+    self.assertEqual(value.to_list(), self.eval_to_list(converted))
 
   @parameterized.parameters([
       dict(
@@ -131,8 +131,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
     tensor = constant_op.constant(pylist)
     converted = ragged.convert_to_tensor_or_ragged_tensor(
         tensor, dtype, preferred_dtype)
-    with self.test_session():
-      self.assertIs(tensor, converted)
+    self.assertIs(tensor, converted)
 
   @parameterized.parameters([
       dict(
@@ -146,7 +145,6 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
           message=('Tensor conversion requested dtype string for '
                    'Tensor with dtype int32')),
   ])
-  @test_util.run_deprecated_v1
   def testConvertTensorError(self,
                              pylist,
                              message,
@@ -189,8 +187,7 @@ class RaggedConvertToTensorOrRaggedTensorTest(test_util.TensorFlowTestCase,
     converted = ragged.convert_to_tensor_or_ragged_tensor(
         value, dtype, preferred_dtype)
     self.assertEqual(dtypes.as_dtype(expected_dtype), converted.dtype)
-    with self.test_session():
-      self.assertAllEqual(value, converted)
+    self.assertAllEqual(value, converted)
 
   @parameterized.parameters([
       dict(
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index 25317ba93e..b5917bc4ee 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -27,282 +27,12 @@ from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import gen_ragged_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_functional_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_util
 from tensorflow.python.ops.ragged import segment_id_ops
 
-#===============================================================================
-# Row Partitioning
-#===============================================================================
-
-
-def value_rowids(rt_input, name=None):
-  """Returns the row indices for the `values` in the given ragged tensor.
-
-  `value_rowids(rt)` corresponds one-to-one with the outermost dimension of
-  `rt.values`, and specifies the row containing each value.  In particular,
-  the row `rt[row]` consists of the values `rt.values[j]` where
-  `value_rowids(rt)[j] == row`.
-
-  Args:
-    rt_input: The RaggedTensor whose row indices should be returned.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A 1-D `int64` `Tensor` with shape `self.values.shape[:1]`.
-    The returned tensor is nonnegative, and is sorted in ascending order.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
-    >>> rt.values.eval()
-    [3, 1, 4, 1, 5, 9, 2, 6]
-    >>> ragged.value_rowids(rt).eval()
-    [0, 0, 0, 0, 2, 2, 2, 3]  # corresponds 1:1 with rt.values
-    ```
-  """
-  if not ragged_tensor.is_ragged(rt_input):
-    raise TypeError(
-        'rt_input expected RaggedTensor, got %s' % type(rt_input).__name__)
-  if (isinstance(rt_input, ragged_tensor.RaggedTensor) and
-      rt_input.cached_value_rowids is not None):
-    return rt_input.cached_value_rowids
-
-  with ops.name_scope(name, 'RaggedValueRowIds', [rt_input]):
-    return segment_id_ops.row_splits_to_segment_ids(rt_input.row_splits)
-
-
-def nrows(rt_input, out_type=dtypes.int64, name=None):
-  """Returns the number of rows in the given potentially ragged tensor.
-
-  I.e., the size of the outermost dimension of the tensor.
-
-  Args:
-    rt_input: The potentially ragged tensor whose number of rows should be
-      returned.
-    out_type: `dtype` for the returned tensor.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A scalar `Tensor` with dtype `out_type`.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
-    >>> ragged.nrows(rt).eval()  # rt has 5 rows.
-    5
-    ```
-  """
-  if (isinstance(rt_input, ragged_tensor.RaggedTensor) and
-      rt_input.cached_nrows is not None):
-    return rt_input.cached_nrows
-
-  with ops.name_scope(name, 'RaggedNRows', [rt_input]):
-    if ragged_tensor.is_ragged(rt_input):
-      return array_ops.shape(rt_input.row_splits, out_type=out_type)[0] - 1
-    else:
-      return array_ops.shape(rt_input, out_type=out_type)[0]
-
-
-def row_starts(rt_input, name=None):
-  """Returns the start indices for rows in the given ragged tensor.
-
-  These indices specify where the values for each row begin in
-  `rt_input.values`.  `ragged.row_starts(rt_input)` is equal to
-  `rt_input.row_splits[:-1]`.
-
-  Args:
-    rt_input: The RaggedTensor whose row starts should be returned.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A 1-D Tensor of int64 with shape `[nrows]`.
-    The returned tensor is nonnegative, and is sorted in ascending order.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
-    >>> ragged.values(rt).eval()
-    [3, 1, 4, 1, 5, 9, 2, 6]
-    >>> ragged.row_starts(rt).eval()  # indices of row starts in ragged.values
-    [0, 4, 4, 7, 8]
-    ```
-  """
-  if not ragged_tensor.is_ragged(rt_input):
-    raise TypeError(
-        'rt_input expected RaggedTensor, got %s' % type(rt_input).__name__)
-  with ops.name_scope(name, 'RaggedRowStarts', [rt_input]):
-    return rt_input.row_splits[:-1]
-
-
-def row_limits(rt_input, name=None):
-  """Returns the limit indices for rows in the given ragged tensor.
-
-  These indices specify where the values for each row end in
-  `rt_input.values`.  `ragged.row_limits(rt_input)` is equal to
-  `rt_input.row_splits[:-1]`.
-
-  Args:
-    rt_input: The RaggedTensor whose row limits should be returned.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A 1-D Tensor of int64 with shape `[nrows]`.
-    The returned tensor is nonnegative, and is sorted in ascending order.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
-    >>> ragged.values(rt).eval()
-    [3, 1, 4, 1, 5, 9, 2, 6]
-    >>> ragged.row_limits(rt).eval()  # indices of row limits in ragged.values
-    [4, 4, 7, 8, 8]
-    ```
-  """
-  if not ragged_tensor.is_ragged(rt_input):
-    raise TypeError(
-        'rt_input expected RaggedTensor, got %s' % type(rt_input).__name__)
-  with ops.name_scope(name, 'RaggedRowLimits', [rt_input]):
-    return rt_input.row_splits[1:]
-
-
-def row_lengths(rt_input, axis=1, name=None):
-  """Returns the lengths of the rows in the given potentially ragged tensor.
-
-  `ragged.row_lengths(rt_input)[i]` indicates the number of values in the
-  `i`th row of `rt_input`.
-
-  Args:
-    rt_input: The potentially ragged tensor whose row lengths should be
-      returned.  Must have at least `axis+1` dimensions.
-    axis: An integer constant indicating the axis whose row lengths should be
-      returned.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A potentially Tensor of int64 with shape `rt_input.shape[:axis]`.
-
-  Raises:
-    ValueError: If rt_input is a scalar, or `axis` is out of bounds.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.constant([[[3, 1, 4], [1]], [], [[5, 9], [2]], [[6]], []])
-    >>> ragged.row_lengths(rt).eval()  # lengths of rows in rt
-    [2, 0, 2, 1, 0]
-    >>> ragged.row_lengths(rt, axis=2).eval()  # lengths of axis=2 rows.
-    [[3, 1], [], [2, 1], [1], []]
-    ```
-  """
-  if (isinstance(rt_input, ragged_tensor.RaggedTensor) and
-      rt_input.cached_row_lengths is not None):
-    return rt_input.cached_row_lengths
-
-  with ops.name_scope(name, 'RaggedRowLengths', [rt_input]):
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        rt_input, name='rt_input')
-    ndims = rt_input.shape.ndims
-    if ndims is not None:
-      if ndims == 0:
-        raise ValueError('rt_input may not be a scalar.')
-      elif not -ndims <= axis < ndims:
-        raise ValueError('axis=%d out of bounds: expected %d<=axis<%d.' %
-                         (axis, -ndims, ndims))
-    if ragged_tensor.is_ragged(rt_input):
-      axis = ragged_util.get_positive_axis(axis, rt_input.shape.ndims)
-      if axis == 0:
-        return nrows(rt_input)
-      elif axis == 1:
-        splits = rt_input.row_splits
-        return splits[1:] - splits[:-1]
-      else:
-        return rt_input.with_values(row_lengths(rt_input.values, axis - 1))
-    else:
-      shape = array_ops.shape(rt_input, out_type=dtypes.int64)
-      return array_ops.ones(shape[:axis], dtypes.int64) * shape[axis]
-
-
-def nested_row_lengths(rt_input, name=None):
-  """Returns a tuple containing the row_lengths for all ragged dimensions.
-
-  `nested_row_lengths(rt)` is a tuple containing the `row_lengths` tensors for
-  all ragged dimensions in `rt`, ordered from outermost to innermost.
-
-  Args:
-    rt_input: A potentially ragged tensor.
-    name: A name prefix for the returned tensors (optional).
-
-  Returns:
-    A `tuple` of 1-D `int64` `Tensors`.  The length of the tuple is equal to
-    `rt_input.ragged_rank`.
-  """
-  with ops.name_scope(name, 'RaggedNestedRowLengths', [rt_input]):
-    rt_nested_row_lengths = []
-    while isinstance(rt_input, ragged_tensor.RaggedTensor):
-      rt_nested_row_lengths.append(row_lengths(rt_input))
-      rt_input = rt_input.values
-    return tuple(rt_nested_row_lengths)
-
-
-#===============================================================================
-# Bounding Shape
-#===============================================================================
-def bounding_shape(rt_input, axis=None, name=None):
-  """Returns the tight bounding box shape for a potentially ragged tensor.
-
-  Args:
-    rt_input: A potentially ragged tensor.
-    axis: An integer scalar or vector indicating which axes to return the
-      bounding box for.  If not specified, then the full bounding box is
-      returned.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    An int64 `Tensor`.  If `axis` is not specified, then `output`
-    is a vector with `output.shape=[rt_input.shape.ndims]`.  If `axis` is a
-    scalar, then the `output` is a scalar.  If `axis` is a vector, then
-    `output` is a vector, where `output[i]` is the bounding size for
-    dimension `axis[i]`.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.constant([[1, 2, 3, 4], [5], [], [6, 7, 8, 9], [10]])
-    >>> ragged.bounding_shape(rt).eval().tolist()
-    [5, 4]
-    ```
-  """
-  with ops.name_scope(name, 'RaggedBoundingBox', [rt_input, axis]):
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        rt_input, name='rt_input')
-    if not ragged_tensor.is_ragged(rt_input):
-      bbox = array_ops.shape(rt_input)
-      return bbox if axis is None else array_ops.gather(bbox, axis)
-
-    nested_splits = rt_input.nested_row_splits
-    rt_inner_values = rt_input.inner_values
-
-    # Optimized special cases for when axis=0 or axis=1:
-    if isinstance(axis, int):
-      if axis == 0:
-        return array_ops.shape(nested_splits[0], out_type=dtypes.int64)[0] - 1
-      elif axis == 1:
-        return math_ops.maximum(math_ops.reduce_max(row_lengths(rt_input)), 0)
-
-    splits_shape = array_ops.shape(rt_input.row_splits, out_type=dtypes.int64)
-    inner_values_shape = array_ops.shape(rt_inner_values, out_type=dtypes.int64)
-
-    ragged_dimensions = array_ops.stack([splits_shape[0] - 1] + [
-        math_ops.maximum(math_ops.reduce_max(splits[1:] - splits[:-1]), 0)
-        for splits in nested_splits
-    ])
-    inner_dimensions = inner_values_shape[1:]
-
-    bbox = array_ops.concat([ragged_dimensions, inner_dimensions], axis=0)
-    return bbox if axis is None else array_ops.gather(bbox, axis)
-
 
 #===============================================================================
 # ragged_gather
@@ -331,13 +61,13 @@ def gather(params, indices, validate_indices=None, axis=0, name=None):
   >>> ragged_params = ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
   >>> ragged_indices = ragged.constant([[3, 1, 2], [1], [], [0]])
 
-  >>> print ragged.gather(params, ragged_indices).eval().tolist()
+  >>> print ragged.gather(params, ragged_indices)
   [['d', 'b', 'c'], ['b'], [], ['a']]
 
-  >>> print ragged.gather(ragged_params, indices).eval().tolist()
+  >>> print ragged.gather(ragged_params, indices)
   [['e'], ['d'], [], ['d'], ['a', 'b', 'c']]
 
-  >>> print ragged.gather(ragged_params, ragged_indices).eval().tolist()
+  >>> print ragged.gather(ragged_params, ragged_indices)
   [[['e'], ['d'], []], [['d']], [], [['a', 'b', 'c']]]
   ```
 
@@ -363,9 +93,9 @@ def gather(params, indices, validate_indices=None, axis=0, name=None):
   if not isinstance(axis, int) or axis != 0:
     raise ValueError('axis>0 is not supported for ragged gather yet.')
   with ops.name_scope(name, 'RaggedGather', [params, indices]):
-    params = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         params, name='params')
-    indices = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         indices, name='indices')
 
     if ragged_tensor.is_ragged(indices):
@@ -380,13 +110,13 @@ def gather(params, indices, validate_indices=None, axis=0, name=None):
 
     result = gen_ragged_array_ops.ragged_gather(
         indices=indices,
-        params_dense_values=params.inner_values,
+        params_dense_values=params.flat_values,
         params_nested_splits=params.nested_row_splits,
         OUTPUT_RAGGED_RANK=indices.shape.ndims + len(params.nested_row_splits) -
         1)
 
     # Compose the RaggedTensor from splits & values.
-    return ragged_factory_ops.from_nested_row_splits(
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
         result.output_dense_values, result.output_nested_splits)
 
 
@@ -429,9 +159,9 @@ def batch_gather(params, indices, name=None):
     return array_ops.batch_gather(params, indices, name)
 
   with ops.name_scope(name, 'RaggedBatchGather', [params, indices]):
-    params = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         params, name='params')
-    indices = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         indices, name='indices')
     indices_ndims = indices.shape.ndims
     if indices_ndims is None:
@@ -448,7 +178,7 @@ def batch_gather(params, indices, name=None):
                            'not match params shape')
         checks = [check_ops.assert_equal(params.row_splits, indices.row_splits)]
         with ops.control_dependencies(checks):
-          return ragged_factory_ops.from_row_splits(
+          return ragged_tensor.RaggedTensor.from_row_splits(
               batch_gather(params.values, indices.values), indices.row_splits)
 
       # Otherwise, indices is a 2D ragged tensor with 1 ragged dimension.
@@ -462,11 +192,11 @@ def batch_gather(params, indices, name=None):
 
         # Adjust indices from within-batch to global (in params.values), and
         # then use ragged.gather to gather them.
-        num_indices = row_lengths(indices)
-        params_starts = row_starts(params)
+        num_indices = indices.row_lengths()
+        params_starts = params.row_starts()
         adjustments = ragged_util.repeat(params_starts, num_indices, axis=0)
         adjusted_index_values = math_ops.to_int64(indices.values) + adjustments
-        return ragged_factory_ops.from_row_splits(
+        return ragged_tensor.RaggedTensor.from_row_splits(
             gather(params.values, adjusted_index_values), indices.row_splits)
 
     else:  # params is a RaggedTensor and indices is a Tensor.
@@ -474,7 +204,7 @@ def batch_gather(params, indices, name=None):
         return gather(params, indices)
       elif indices_ndims == 2:
         # Adjust indices from batch-local to global (in params.values)
-        adjustments = array_ops.expand_dims(row_starts(params), 1)
+        adjustments = array_ops.expand_dims(params.row_starts(), 1)
         adjusted_indices = math_ops.to_int64(indices) + adjustments
         return gather(params.values, adjusted_indices)
       else:
@@ -532,9 +262,9 @@ def gather_nd(params, indices, name=None):
 
   with ops.name_scope(name, 'RaggedGatherNd', [params, indices]):
 
-    params = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         params, name='params')
-    indices = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         indices, name='indices')
     indices_shape = indices.shape
     indices_ndims = indices_shape.ndims
@@ -548,7 +278,7 @@ def gather_nd(params, indices, name=None):
 
     # `index_size` is the "n" in "gather_nd" -- i.e., the number of dimensions
     # that each index slices into.
-    index_size = indices_shape[-1].value
+    index_size = tensor_shape.dimension_value(indices_shape[-1])
     if index_size is None:
       raise ValueError('indices.shape[-1] must be statically known.')
 
@@ -560,8 +290,7 @@ def gather_nd(params, indices, name=None):
       if indices_is_dense:
         indices = ragged_conversion_ops.from_tensor(
             indices, ragged_rank=indices_ndims - 2)
-      result = indices.with_inner_values(
-          gather_nd(params, indices.inner_values))
+      result = indices.with_flat_values(gather_nd(params, indices.flat_values))
       if (indices_is_dense and ragged_tensor.is_ragged(result) and
           result.ragged_rank == indices_ndims - 2):
         result = ragged_conversion_ops.to_tensor(result)
@@ -575,7 +304,7 @@ def gather_nd(params, indices, name=None):
     # Handle corner case: An empty index tuple selects the entire `params`
     # value.  So if `index_size` is zero, then tile `params`.
     if index_size == 0:
-      params_ndims = params.ragged_rank + array_ops.rank(params.inner_values)
+      params_ndims = params.ragged_rank + array_ops.rank(params.flat_values)
       for dim in range(indices_ndims - 1):
         params = expand_dims(params, axis=0)
       multiples = array_ops.concat([
@@ -613,7 +342,7 @@ def gather_nd(params, indices, name=None):
           return array_ops.gather_nd(flattened_params, flattened_index_tuples)
 
         flattened_index_tuples = array_ops.gather(
-            row_starts(flattened_params), flattened_index_tuples)
+            flattened_params.row_starts(), flattened_index_tuples)
         flattened_index_tuples += indices[..., dim]
         flattened_params = flattened_params.values
 
@@ -709,9 +438,8 @@ def boolean_mask(data, mask, keepdims=False, name=None):
   """
   with ops.name_scope(name, 'RaggedMask', [data, mask]):
     # Convert inputs to tensors.
-    data = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        data, name='data')
-    mask = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    data = ragged_tensor.convert_to_tensor_or_ragged_tensor(data, name='data')
+    mask = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         mask, dtypes.bool, name='mask')
 
     # Get static rank of mask.
@@ -742,7 +470,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
           else:
             # Count the number of True mask values in each row to find the
             # lengths of the filtered rows; then convert to splits.
-            int_mask = ragged_functional_ops.map_inner_values(
+            int_mask = ragged_functional_ops.map_flat_values(
                 math_ops.cast, mask, dtype=dtypes.int64)
             masked_row_lengths = ragged_math_ops.reduce_sum(int_mask, axis=1)
             splits.append(ragged_util.lengths_to_splits(masked_row_lengths))
@@ -754,7 +482,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
 
         # Add the ragged `splits` back to the result.
         if keepdims:
-          masked_values = ragged_factory_ops.from_nested_row_splits(
+          masked_values = ragged_tensor.RaggedTensor.from_nested_row_splits(
               masked_values, splits)
 
         return masked_values
@@ -765,7 +493,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
       # Get the masked splits: first get the length of each row, then filter
       # out the rows that we are deleting, and convert that filtered set of
       # masks back to a splits tensor.
-      lengths = row_lengths(data)
+      lengths = data.row_lengths()
       masked_lengths = array_ops.boolean_mask(lengths, mask)
       masked_splits = ragged_util.lengths_to_splits(masked_lengths)
 
@@ -777,7 +505,8 @@ def boolean_mask(data, mask, keepdims=False, name=None):
       segment_mask = array_ops.gather(mask, segment_ids)
       masked_values = boolean_mask(data.values, segment_mask, keepdims=False)
 
-      return ragged_factory_ops.from_row_splits(masked_values, masked_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(masked_values,
+                                                        masked_splits)
 
     # If mask is non-ragged and has rank>1, then convert it to be ragged,
     # with a ragged rank matching data.
@@ -798,7 +527,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
         # and values to get the innermost ragged tensor.
         masked_lengths = math_ops.count_nonzero(mask, axis=-1)
         flattened_masked_lengths = array_ops.reshape(masked_lengths, [-1])
-        masked_values = ragged_factory_ops.from_row_lengths(
+        masked_values = ragged_tensor.RaggedTensor.from_row_lengths(
             masked_values, flattened_masked_lengths)
 
         # Wrap remaining ragged dimensions.
@@ -808,7 +537,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
           for dim in range(mask.shape.ndims - 3, -1, -1):
             elt_size = mask_shape[dim + 1]
             masked_splits = math_ops.range(split_size[dim]) * elt_size
-            masked_values = ragged_factory_ops.from_row_splits(
+            masked_values = ragged_tensor.RaggedTensor.from_row_splits(
                 masked_values, masked_splits)
 
       return masked_values
@@ -919,7 +648,7 @@ def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
 
   # Convert input tensors.
   rt_inputs = [
-      ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+      ragged_tensor.convert_to_tensor_or_ragged_tensor(
           rt_input, name='rt_input') for rt_input in rt_inputs
   ]
 
@@ -970,7 +699,7 @@ def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
     values = [rt.values for rt in rt_inputs]
     splits = [[rt_input.row_splits] for rt_input in rt_inputs]
     with ops.control_dependencies(ragged_util.assert_splits_match(splits)):
-      return ragged_factory_ops.from_row_splits(
+      return ragged_tensor.RaggedTensor.from_row_splits(
           _ragged_stack_concat_helper(values, axis - 1, stack_values),
           splits[0][0])
 
@@ -987,8 +716,8 @@ def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
     A RaggedTensor.
   """
   # Concatenate the inner values together.
-  inner_values = [rt.inner_values for rt in rt_inputs]
-  concatenated_inner_values = array_ops.concat(inner_values, axis=0)
+  flat_values = [rt.flat_values for rt in rt_inputs]
+  concatenated_flat_values = array_ops.concat(flat_values, axis=0)
 
   # Concatenate the splits together for each ragged dimension (adjusting
   # split offsets as necessary).
@@ -1002,12 +731,12 @@ def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
 
   # If we are performing a stack operation, then add another splits.
   if stack_values:
-    stack_lengths = array_ops.stack([nrows(rt) for rt in rt_inputs])
+    stack_lengths = array_ops.stack([_nrows(rt) for rt in rt_inputs])
     stack_splits = ragged_util.lengths_to_splits(stack_lengths)
     concatenated_nested_splits.insert(0, stack_splits)
 
-  return ragged_factory_ops.from_nested_row_splits(concatenated_inner_values,
-                                                   concatenated_nested_splits)
+  return ragged_tensor.RaggedTensor.from_nested_row_splits(
+      concatenated_flat_values, concatenated_nested_splits)
 
 
 def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
@@ -1023,10 +752,10 @@ def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
   """
   num_inputs = len(rt_inputs)
 
-  rt_nrows = nrows(rt_inputs[0])
+  rt_nrows = _nrows(rt_inputs[0])
   nrows_msg = 'Input tensors have incompatible shapes.'
   nrows_checks = [
-      check_ops.assert_equal(nrows(rt), rt_nrows, message=nrows_msg)
+      check_ops.assert_equal(_nrows(rt), rt_nrows, message=nrows_msg)
       for rt in rt_inputs[1:]
   ]
 
@@ -1050,14 +779,15 @@ def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
       # Add a new splits tensor to group together the values.
       stack_splits = math_ops.range(0, rt_nrows * num_inputs + 1, num_inputs)
       _copy_row_shape(rt_inputs, stack_splits)
-      return ragged_factory_ops.from_row_splits(permuted_rt, stack_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt,
+                                                        stack_splits)
     else:
       # Merge together adjacent rows by dropping the row-split indices that
       # separate them.
       concat_splits = permuted_rt.row_splits[::num_inputs]
       _copy_row_shape(rt_inputs, concat_splits)
-      return ragged_factory_ops.from_row_splits(permuted_rt.values,
-                                                concat_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt.values,
+                                                        concat_splits)
 
 
 def _copy_row_shape(rt_inputs, splits):
@@ -1090,12 +820,12 @@ def tile(input, multiples, name=None):  # pylint: disable=redefined-builtin
   #### Example:
     ```python
     >>> rt = ragged.constant([[1, 2], [3]])
-    >>> ragged.tile(rt, [3, 2]).eval().tolist()
+    >>> ragged.tile(rt, [3, 2])
     [[1, 2, 1, 2], [3, 3], [1, 2, 1, 2], [3, 3], [1, 2, 1, 2], [3, 3]]
     ```
   """
   with ops.name_scope(name, 'RaggedTile', [input, multiples]):
-    input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    input = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         input, name='input')
     multiples = ragged_util.convert_to_int_tensor(
         multiples, name='multiples', dtype=dtypes.int64)
@@ -1107,16 +837,16 @@ def tile(input, multiples, name=None):  # pylint: disable=redefined-builtin
     # to skip tiling dimensions where `multiples=1`.
     const_multiples = tensor_util.constant_value(multiples)
 
-    return ragged_factory_ops.from_nested_row_splits(
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
         _tile_ragged_values(input, multiples, const_multiples),
         _tile_ragged_splits(input, multiples, const_multiples))
 
 
 def _tile_ragged_values(rt_input, multiples, const_multiples=None):
-  """Builds inner_values tensor for a tiled `RaggedTensor`.
+  """Builds flat_values tensor for a tiled `RaggedTensor`.
 
   Returns a tensor that repeats the values in
-  `rt_input.inner_values` in the
+  `rt_input.flat_values` in the
   appropriate pattern to construct a `RaggedTensor` that tiles `rt_input` as
   specified by `multiples`.
 
@@ -1128,19 +858,19 @@ def _tile_ragged_values(rt_input, multiples, const_multiples=None):
       dimensions where `multiples=1`.
 
   Returns:
-    A `Tensor` with the same type and rank as `rt_input.inner_values`.
+    A `Tensor` with the same type and rank as `rt_input.flat_values`.
 
   #### Example:
     ```python
     >>> rt = ragged.constant([[1, 2], [3]])
-    >>> _tile_ragged_values(rt, [3, 2]).eval().tolist()
+    >>> _tile_ragged_values(rt, [3, 2])
     [1, 2, 1, 2, 3, 3, 1, 2, 1, 2, 3, 3, 1, 2, 1, 2, 3, 3]
     ```
   """
   ragged_rank = rt_input.ragged_rank
   nested_splits = rt_input.nested_row_splits
 
-  # Pointers to the values in `rt_input.inner_values`.
+  # Pointers to the values in `rt_input.flat_values`.
   inner_value_ids = math_ops.range(nested_splits[-1][-1])
 
   # For each ragged dimension (working from the innermost to outermost),
@@ -1163,9 +893,9 @@ def _tile_ragged_values(rt_input, multiples, const_multiples=None):
     prev_splits = splits
 
   # Gather the tiled inner values.
-  ragged_tiled_values = array_ops.gather(rt_input.inner_values, inner_value_ids)
+  ragged_tiled_values = array_ops.gather(rt_input.flat_values, inner_value_ids)
 
-  # Tile the inner_values for the uniform dimensions (i.e., for `axis=0` plus
+  # Tile the flat_values for the uniform dimensions (i.e., for `axis=0` plus
   # `axis=range(ragged_rank, rank)`).
   inner_repeats = array_ops.concat([multiples[:1], multiples[ragged_rank + 1:]],
                                    axis=0)
@@ -1192,7 +922,7 @@ def _tile_ragged_splits(rt_input, multiples, const_multiples=None):
   #### Example:
     ```python
     >>> rt = ragged.constant([[1, 2], [3]])
-    >>> _tile_ragged_splits(rt, [3, 2]).eval().tolist()
+    >>> _tile_ragged_splits(rt, [3, 2])
     [0, 4, 6, 10, 12, 16, 18]
     ```
   """
@@ -1293,20 +1023,20 @@ def expand_dims(input, axis, name=None):  # pylint: disable=redefined-builtin
     TensorShape([2, None])
 
     >>> expanded = ragged.expand_dims(rt, axis=0)
-    >>> print(expanded.shape, expanded.eval().tolist())
+    >>> print(expanded.shape, expanded)
     TensorShape([1, None, None]) [[[1, 2], [3]]]
 
     >>> expanded = ragged.expand_dims(rt, axis=1)
-    >>> print(expanded.shape, expanded.eval().tolist())
+    >>> print(expanded.shape, expanded)
     TensorShape([2, None, None]) [[[1, 2]], [[3]]]
 
     >>> expanded = ragged.expand_dims(rt, axis=2)
-    >>> print(expanded.shape, expanded.eval().tolist())
+    >>> print(expanded.shape, expanded)
     TensorShape([2, None, 1]) [[[1], [2]], [[3]]]
     ```
   """
   with ops.name_scope(name, 'RaggedExpandDims', [input]):
-    input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    input = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         input, name='input')
 
     if not ragged_tensor.is_ragged(input):
@@ -1316,15 +1046,15 @@ def expand_dims(input, axis, name=None):  # pylint: disable=redefined-builtin
     axis = ragged_util.get_positive_axis(axis, ndims)
     if axis == 0:
       values = input
-      splits = array_ops.stack([0, nrows(input)])
+      splits = array_ops.stack([0, input.nrows()])
     elif axis == 1:
       values = input
-      splits = math_ops.range(nrows(input) + 1)
+      splits = math_ops.range(input.nrows() + 1)
     else:
       values = expand_dims(input.values, axis - 1)
       splits = input.row_splits
 
-    return ragged_factory_ops.from_row_splits(values, splits)
+    return ragged_tensor.RaggedTensor.from_row_splits(values, splits)
 
 
 #===============================================================================
@@ -1401,13 +1131,13 @@ def where(condition, x=None, y=None, name=None):
   if (x is None) != (y is None):
     raise ValueError('x and y must be either both None or both non-None')
   with ops.name_scope('RaggedWhere', name, [condition, x, y]):
-    condition = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    condition = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         condition, name='condition')
     if x is None:
       return _coordinate_where(condition)
     else:
-      x = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(x, name='x')
-      y = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(y, name='y')
+      x = ragged_tensor.convert_to_tensor_or_ragged_tensor(x, name='x')
+      y = ragged_tensor.convert_to_tensor_or_ragged_tensor(y, name='y')
       return _elementwise_where(condition, x, y)
 
 
@@ -1421,15 +1151,15 @@ def _elementwise_where(condition, x, y):
     return array_ops.where(condition, x, y)
 
   elif condition_is_ragged and x_is_ragged and y_is_ragged:
-    return ragged_functional_ops.map_inner_values(array_ops.where, condition, x,
-                                                  y)
+    return ragged_functional_ops.map_flat_values(array_ops.where, condition, x,
+                                                 y)
   elif not condition_is_ragged:
     # Concatenate x and y, and then use `gather` to assemble the selected rows.
     condition.shape.assert_has_rank(1)
-    x_nrows = nrows(x)
+    x_nrows = _nrows(x)
     x_and_y = concat([x, y], axis=0)
     indices = array_ops.where(condition, math_ops.range(x_nrows),
-                              x_nrows + math_ops.range(nrows(y)))
+                              x_nrows + math_ops.range(_nrows(y)))
     return gather(x_and_y, indices)
 
   else:
@@ -1446,7 +1176,7 @@ def _coordinate_where(condition):
 
   # Convert the first index in each coordinate to a row index and column index.
   first_index = selected_coords[:, 0]
-  selected_rows = array_ops.gather(value_rowids(condition), first_index)
+  selected_rows = array_ops.gather(condition.value_rowids(), first_index)
   selected_row_starts = array_ops.gather(condition.row_splits, selected_rows)
   selected_cols = first_index - selected_row_starts
 
@@ -1482,3 +1212,12 @@ def _concat_ragged_splits(splits_list):
     pieces.append(splits[1:] + splits_offset)
     splits_offset += splits[-1]
   return array_ops.concat(pieces, axis=0)
+
+
+def _nrows(rt_input, out_type=dtypes.int64, name=None):
+  if isinstance(rt_input, ragged_tensor.RaggedTensor):
+    return rt_input.nrows(out_type=out_type, name=name)
+  else:
+    with ops.name_scope(name, 'RaggedNRows', [rt_input]):
+      return array_ops.shape(rt_input, out_type=out_type)[0]
+
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
index 7fe185641f..79f1ae591f 100644
--- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
@@ -20,15 +20,18 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
                               parameterized.TestCase):
 
   @parameterized.parameters([
@@ -135,21 +138,16 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
           expected=ragged.constant_value(
               [[[[b'c', b'a'], [b'd', b'd']], [[b'f', b'e']]]], ragged_rank=2)),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testRaggedBatchGather(self, descr, params, indices, expected):
     result = ragged.batch_gather(params, indices)
-    self.assertEqual(
-        getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0))
-    with self.test_session():
-      if hasattr(expected, 'tolist'):
-        expected = expected.tolist()
-      self.assertEqual(result.eval().tolist(), expected)
+    self.assertRaggedEqual(result, expected)
 
-  @test_util.run_deprecated_v1
   def testRaggedBatchGatherUnknownRankError(self):
+    if context.executing_eagerly():
+      return
     params = [['a', 'b'], ['c', 'd']]
     indices = array_ops.placeholder(dtypes.int32, shape=None)
-    ragged_indices = ragged.from_row_splits(indices, [0, 2, 4])
+    ragged_indices = ragged.RaggedTensor.from_row_splits(indices, [0, 2, 4])
 
     with self.assertRaisesRegexp(
         ValueError, 'batch_gather does not allow indices with unknown shape.'):
@@ -161,38 +159,39 @@ class RaggedBatchGatherOpTest(test_util.TensorFlowTestCase,
 
   @parameterized.parameters([
       dict(
-          params=ragged.constant([['a'], ['b'], ['c']]),
-          indices=ragged.constant([[0], [0]]),
+          params=ragged.constant_value([['a'], ['b'], ['c']]),
+          indices=ragged.constant_value([[0], [0]]),
           message='Dimensions 3 and 2 are not compatible'),
       dict(
           params=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]],
-          indices=ragged.constant([[[0, 0], [0, 0, 0]], [[0]]]),
+          indices=ragged.constant_value([[[0, 0], [0, 0, 0]], [[0]]]),
           message='batch shape from indices does not match params shape'),
+      dict(  # rank mismatch
+          params=ragged.constant_value([[[0, 0], [0, 0, 0]], [[0]]]),
+          indices=ragged.constant_value([[[0, 0]], [[0, 0, 0]], [[0]]]),
+          error=(ValueError, errors.InvalidArgumentError)),
       dict(
-          params=ragged.constant([[[0, 0], [0, 0, 0]], [[0]]]),
-          indices=ragged.constant([[[0, 0]], [[0, 0, 0]], [[0]]]),
-          message='Dimensions must be equal, but are 3 and 4'),
-      dict(
-          params=ragged.constant([[[0, 0], [0, 0, 0]], [[0]], [[0]]]),
-          indices=ragged.constant([[[0, 0]], [[0, 0, 0]], [[0]]]),
+          params=ragged.constant_value([[[0, 0], [0, 0, 0]], [[0]], [[0]]]),
+          indices=ragged.constant_value([[[0, 0]], [[0, 0, 0]], [[0]]]),
           error=errors.InvalidArgumentError,
-          message='Condition x == y did not hold element-wise'),
+          message='.*Condition x == y did not hold.*'),
+      dict(
+          params=ragged.constant_value(['a', 'b', 'c']),
+          indices=ragged.constant_value([[0], [0]]),
+          message='batch shape from indices does not match params shape'),
+      dict(
+          params=ragged.constant_value([['a']]),
+          indices=0,
+          message='indices.rank must be at least 1.'),
       dict(
-          params=ragged.constant(['a', 'b', 'c']),
-          indices=ragged.constant([[0], [0]]),
+          params=ragged.constant_value([['a']]),
+          indices=[[[0]]],
           message='batch shape from indices does not match params shape'),
-      dict(params=ragged.constant_value([['a']]),
-           indices=0,
-           message='indices.rank must be at least 1.'),
-      dict(params=ragged.constant_value([['a']]),
-           indices=[[[0]]],
-           message='batch shape from indices does not match params shape'),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testRaggedBatchGatherStaticError(self,
                                        params,
                                        indices,
-                                       message,
+                                       message=None,
                                        error=ValueError):
     with self.assertRaisesRegexp(error, message):
       ragged.batch_gather(params, indices)
diff --git a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
index 763b016405..b0f7459322 100644
--- a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
@@ -20,15 +20,18 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedBooleanMaskOpTest(ragged_test_util.RaggedTensorTestCase,
                               parameterized.TestCase):
   # Define short constants for true & false, so the data & mask can be lined
   # up in the examples below.  This makes it easier to read the examples, to
@@ -298,26 +301,18 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase,
           keepdims=True,
           expected=ragged.constant_value([[[1], [4, 6]], [[7, 9], []]])),
   ])  # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testBooleanMask(self, descr, data, mask, keepdims, expected):
     actual = ragged.boolean_mask(data, mask, keepdims=keepdims)
-    self.assertEqual(
-        getattr(actual, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0))
-    with self.test_session():
-      if isinstance(expected, ragged.RaggedTensorValue):
-        expected = expected.tolist()
-      self.assertEqual(actual.eval().tolist(), expected)
+    self.assertRaggedEqual(actual, expected)
 
-  @test_util.run_deprecated_v1
   def testErrors(self):
-    self.assertRaisesRegexp(ValueError,
-                            r'mask\.shape\.ndims must be kown statically',
-                            ragged.boolean_mask, [[1, 2]],
-                            array_ops.placeholder(dtypes.bool))
+    if not context.executing_eagerly():
+      self.assertRaisesRegexp(ValueError,
+                              r'mask\.shape\.ndims must be kown statically',
+                              ragged.boolean_mask, [[1, 2]],
+                              array_ops.placeholder(dtypes.bool))
 
-    self.assertRaisesRegexp(TypeError,
-                            "Expected bool, got 0 of type 'int' instead.",
-                            ragged.boolean_mask, [[1, 2]], [[0, 1]])
+    self.assertRaises(TypeError, ragged.boolean_mask, [[1, 2]], [[0, 1]])
     self.assertRaisesRegexp(
         ValueError, 'Tensor conversion requested dtype bool for '
         'RaggedTensor with dtype int32', ragged.boolean_mask,
@@ -327,15 +322,6 @@ class RaggedBooleanMaskOpTest(test_util.TensorFlowTestCase,
         ValueError, r'Shapes \(1, 2\) and \(1, 3\) are incompatible',
         ragged.boolean_mask, [[1, 2]], [[True, False, True]])
 
-    # self.assertRaisesRegexp(ValueError,
-    #                         r'data=.* is non-ragged but mask=.* is ragged',
-    #                         ragged.boolean_mask, [[1, 2]],
-    #                         ragged.constant([[True, False]]))
-
-    # self.assertRaisesRegexp(
-    #     ValueError, r'data=.* is ragged but mask=.* is non-ragged',
-    #     ragged.boolean_mask, ragged.constant([[1, 2]]), [[True, False]])
-
     self.assertRaisesRegexp(errors.InvalidArgumentError,
                             r'Inputs must have identical ragged splits',
                             ragged.boolean_mask, ragged.constant([[1, 2]]),
diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
index ba7867418a..e72afb0448 100644
--- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
@@ -20,16 +20,20 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
+                         parameterized.TestCase):
 
   def _rt_inputs_to_tensors(self, rt_inputs, ragged_ranks=None):
     if ragged_ranks is None:
@@ -221,7 +225,6 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=[[b'a00', b'a01'], [], [b'a20', b'a21']]),
   )   # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testRaggedConcat(self,
                        descr,
                        rt_inputs,
@@ -236,8 +239,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertEqual(concatenated.ragged_rank, expected_ragged_rank)
     if expected_shape is not None:
       self.assertEqual(concatenated.shape.as_list(), expected_shape)
-    with self.test_session():
-      self.assertEqual(concatenated.eval().tolist(), expected)
+    self.assertRaggedEqual(concatenated, expected)
 
   @parameterized.parameters(
       dict(
@@ -264,11 +266,14 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           ragged_ranks=(0, 0),
           rt_inputs=([[1, 2]], [[3, 4], [5, 6]]),
           axis=1,
-          error=ValueError,
-          message='Dimension 0 in both shapes must be equal'),
+          error=(ValueError, errors.InvalidArgumentError)),
   )
-  @test_util.run_deprecated_v1
-  def testStaticError(self, rt_inputs, axis, error, message, ragged_ranks=None):
+  def testStaticError(self,
+                      rt_inputs,
+                      axis,
+                      error,
+                      message=None,
+                      ragged_ranks=None):
     rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks)
     self.assertRaisesRegexp(error, message, ragged.concat, rt_inputs, axis)
 
@@ -280,18 +285,20 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           error=errors.InvalidArgumentError,
           message='Input tensors have incompatible shapes'),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testRuntimeError(self, rt_inputs, axis, error, message,
                        ragged_ranks=None):
+    if context.executing_eagerly():
+      return
     rt_inputs = [
         array_ops.placeholder_with_default(rt, shape=None) for rt in rt_inputs
     ]
     concatenated = ragged.concat(rt_inputs, axis)
-    with self.test_session():
-      self.assertRaisesRegexp(error, message, concatenated.eval)
+    with self.assertRaisesRegexp(error, message):
+      self.evaluate(concatenated)
 
-  @test_util.run_deprecated_v1
   def testNegativeAxisWithUnknownRankError(self):
+    if context.executing_eagerly():
+      return
     rt_inputs = [
         array_ops.placeholder(dtypes.int64),
         array_ops.placeholder(dtypes.int64)
@@ -300,7 +307,6 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         ValueError, r'axis may only be negative if ndims is statically known.',
         ragged.concat, rt_inputs, -1)
 
-  @test_util.run_deprecated_v1
   def testSingleTensorInput(self):
     """Tests ragged_concat with a single tensor input.
 
@@ -310,8 +316,7 @@ class RaggedConcatOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     """
     rt_inputs = ragged.constant([[1, 2], [3, 4]])
     concatenated = ragged.concat(rt_inputs, 0)
-    with self.test_session():
-      self.assertEqual(concatenated.eval().tolist(), [[1, 2], [3, 4]])
+    self.assertRaggedEqual(concatenated, [[1, 2], [3, 4]])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_const_op_test.py b/tensorflow/python/ops/ragged/ragged_const_op_test.py
index 2505b23912..c014f71030 100644
--- a/tensorflow/python/ops/ragged/ragged_const_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_const_op_test.py
@@ -20,15 +20,16 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 
-
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops.ragged import ragged_factory_ops
-from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedConstOpTest(ragged_test_util.RaggedTensorTestCase,
+                        parameterized.TestCase):
 
   @parameterized.parameters(
       #=========================================================================
@@ -133,7 +134,6 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       dict(pylist=[[b'a', b'b'], [b'c'], [b'd', b'e', b'f']],
            dtype=dtypes.string),
   )
-  @test_util.run_deprecated_v1
   def testRaggedConst(self,
                       pylist,
                       dtype=None,
@@ -157,7 +157,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       expected_dtype: The expected dtype for the resulting ragged tensor (used
         to test default/inferred types when dtype=None).
     """
-    rt = ragged_factory_ops.constant(
+    rt = ragged.constant(
         pylist, dtype=dtype, ragged_rank=ragged_rank, inner_shape=inner_shape)
 
     # If dtype was explicitly specified, check it.
@@ -168,31 +168,22 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # If ragged_rank was explicitly specified, check it.
     if ragged_rank is not None:
-      if isinstance(rt, ragged_tensor.RaggedTensor):
+      if isinstance(rt, ragged.RaggedTensor):
         self.assertEqual(rt.ragged_rank, ragged_rank)
       else:
         self.assertEqual(0, ragged_rank)
 
     # If inner_shape was explicitly specified, check it.
     if inner_shape is not None:
-      if isinstance(rt, ragged_tensor.RaggedTensor):
-        self.assertEqual(rt.inner_values.shape.as_list()[1:], list(inner_shape))
+      if isinstance(rt, ragged.RaggedTensor):
+        self.assertEqual(rt.flat_values.shape.as_list()[1:], list(inner_shape))
       else:
         self.assertEqual(rt.shape.as_list(), list(inner_shape))
 
     if expected_shape is not None:
       self.assertEqual(tuple(rt.shape.as_list()), expected_shape)
 
-    with self.test_session():
-      result = self.evaluate(rt)
-      if rt.shape.ndims > 0:
-        self.assertEqual(result.tolist(), pylist)
-        if expected_shape is not None:
-          self.assertEqual(result.shape, expected_shape)
-      else:
-        self.assertEqual(result, pylist)
-        if expected_shape is not None:
-          self.assertEqual((), expected_shape)
+    self.assertRaggedEqual(rt, pylist)
 
   @parameterized.parameters(
       dict(
@@ -236,11 +227,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           exception=ValueError,
           message='pylist has scalar values depth 2, but ragged_rank=2 '
           'requires scalar value depth greater than 2'),
-      dict(
-          pylist=[1, 2, 3],
-          inner_shape=(1, 1),
-          exception=TypeError,
-          message='Expected Tensor\'s shape'),
+      dict(pylist=[1, 2, 3], inner_shape=(1, 1), exception=TypeError),
       dict(
           pylist=[[[1, 2], [3, 4]], [[5, 6], [7, 8]]],
           inner_shape=(2, 2),
@@ -259,7 +246,6 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           exception=ValueError,
           message='inner values have inconsistent shape'),
   )
-  @test_util.run_deprecated_v1
   def testRaggedConstError(self,
                            pylist,
                            dtype=None,
@@ -271,7 +257,7 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertRaisesRegexp(
         exception,
         message,
-        ragged_factory_ops.constant,
+        ragged.constant,
         pylist,
         dtype=dtype,
         ragged_rank=ragged_rank,
@@ -310,10 +296,10 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     if exception is not None:
       self.assertRaisesRegexp(
           exception, message,
-          ragged_factory_ops._find_scalar_and_max_depth, pylist)
+          ragged.ragged_factory_ops._find_scalar_and_max_depth, pylist)
     else:
       self.assertEqual(
-          ragged_factory_ops._find_scalar_and_max_depth(pylist),
+          ragged.ragged_factory_ops._find_scalar_and_max_depth(pylist),
           (scalar_depth, max_depth))
 
   @parameterized.parameters([
@@ -360,11 +346,11 @@ class RaggedConstOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     if exception is not None:
       self.assertRaisesRegexp(
           exception, message,
-          ragged_factory_ops._default_inner_shape_for_pylist, pylist,
+          ragged.ragged_factory_ops._default_inner_shape_for_pylist, pylist,
           ragged_rank)
     else:
       self.assertEqual(
-          ragged_factory_ops._default_inner_shape_for_pylist(
+          ragged.ragged_factory_ops._default_inner_shape_for_pylist(
               pylist, ragged_rank), inner_shape)
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_constant_value_op_test.py b/tensorflow/python/ops/ragged/ragged_constant_value_op_test.py
index d80518930d..56768a9a47 100644
--- a/tensorflow/python/ops/ragged/ragged_constant_value_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_constant_value_op_test.py
@@ -23,10 +23,12 @@ import numpy as np
 
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedConstantValueOpTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedConstantValueOpTest(ragged_test_util.RaggedTensorTestCase,
                                 parameterized.TestCase):
 
   @parameterized.parameters(
@@ -144,7 +146,7 @@ class RaggedConstantValueOpTest(test_util.TensorFlowTestCase,
                        inner_shape=None,
                        expected_shape=None,
                        expected_dtype=None):
-    """Tests that `ragged_value(pylist).tolist() == pylist`."""
+    """Tests that `ragged_value(pylist).to_list() == pylist`."""
     rt = ragged.constant_value(
         pylist, dtype=dtype, ragged_rank=ragged_rank, inner_shape=inner_shape)
 
@@ -164,7 +166,7 @@ class RaggedConstantValueOpTest(test_util.TensorFlowTestCase,
     # If inner_shape was explicitly specified, check it.
     if inner_shape is not None:
       if isinstance(rt, ragged.RaggedTensorValue):
-        self.assertEqual(rt.inner_values.shape[1:], inner_shape)
+        self.assertEqual(rt.flat_values.shape[1:], inner_shape)
       else:
         self.assertEqual(rt.shape, inner_shape)
 
@@ -172,7 +174,10 @@ class RaggedConstantValueOpTest(test_util.TensorFlowTestCase,
       self.assertEqual(tuple(rt.shape), expected_shape)
 
     if rt.shape:
-      self.assertEqual(rt.tolist(), pylist)
+      if isinstance(rt, ragged.RaggedTensorValue):
+        self.assertEqual(rt.to_list(), pylist)
+      else:
+        self.assertEqual(rt.tolist(), pylist)
       if expected_shape is not None:
         self.assertEqual(rt.shape, expected_shape)
     else:
diff --git a/tensorflow/python/ops/ragged/ragged_conversion_ops.py b/tensorflow/python/ops/ragged/ragged_conversion_ops.py
index 83212e49cf..854c5b303c 100644
--- a/tensorflow/python/ops/ragged/ragged_conversion_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_conversion_ops.py
@@ -18,407 +18,27 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import gen_ragged_conversion_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
-from tensorflow.python.ops.ragged import ragged_util
 
 
-#===============================================================================
-# RaggedTensor <-> Tensor conversion
-#===============================================================================
 def from_tensor(tensor, lengths=None, padding=None, ragged_rank=1, name=None):
-  """Converts a `Tensor` into a `RaggedTensor`.
-
-  The set of absent/default values may be specified using a vector of lengths
-  or a padding value (but not both).  If `lengths` is specified, then the
-  output tensor will satisfy `output[row] = tensor[row][:lengths[row]]`.
-  If `padding` is specified, then any row *suffix* consisting entirely of
-  `padding` will be excluded from the returned `RaggedTensor`.  If neither
-  `lengths` nor `padding` is specified, then the returned `RaggedTensor` will
-  have no absent/default values.
-
-  Examples:
-
-  ```python
-  >>> dt = tf.constant([[5, 7, 0], [0, 3, 0], [6, 0, 0]])
-  >>> ragged.from_tensor(dt).eval().tolist()
-  [[5, 7, 0], [0, 3, 0], [6, 0, 0]]
-  >>> ragged.from_tensor(dt, lengths=[2, 0, 3]).eval().tolist()
-  [[5, 7], [], [6, 0, 0]]
-  >>> ragged.from_tensor(dt, padding=0).eval().tolist()
-  [[5, 7], [0, 3], [6]]
-  ```
-
-  Args:
-    tensor: The `Tensor` to convert.  Must have rank `ragged_rank + 1` or
-      higher.
-    lengths: An optional set of row lengths, specified using a 1-D integer
-      `Tensor` whose length is equal to `tensor.shape[0]` (the number of rows in
-      `tensor`).  If specified, then `output[row]` will contain
-      `tensor[row][:lengths[row]]`.  Negative lengths are treated as zero.
-    padding: An optional padding value.  If specified, then any row suffix
-      consisting entirely of `padding` will be excluded from the returned
-      RaggedTensor.  `padding` is a `Tensor` with the same dtype as `tensor`
-      and with `shape=tensor.shape[ragged_rank + 1:]`.
-    ragged_rank: Integer specifying the ragged rank for the returned
-      `RaggedTensor`.  Must be greater than zero.
-    name: A name prefix for the returned tensors (optional).
-
-  Returns:
-    A `RaggedTensor` with the specified `ragged_rank`.  The shape of the
-    returned ragged tensor is compatible with the shape of `tensor`.
-  Raises:
-    ValueError: If both `lengths` and `padding` are specified.
-  """
-  if lengths is not None and padding is not None:
-    raise ValueError('Specify lengths or padding, but not both')
-  if not isinstance(ragged_rank, int):
-    raise TypeError('ragged_rank expected int, got %r' % ragged_rank)
-  if ragged_rank <= 0:
-    raise ValueError('ragged_rank must be greater than 0; got %s' % ragged_rank)
-
-  with ops.name_scope(name, 'RaggedFromTensor', [tensor, lengths, padding]):
-    tensor = ops.convert_to_tensor(tensor, name='tensor')
-    tensor.shape.with_rank_at_least(ragged_rank + 1)
-    input_shape = array_ops.shape(tensor, out_type=dtypes.int64)
-    ncols = input_shape[1]
-
-    # Handle ragged_rank>1 via recursion:
-    # If the output should have multiple ragged dimensions, then first
-    # flatten the tensor to eliminate all but the last ragged dimension,
-    # and recursively convert that flattened tensor.  Then add on the splits
-    # for the dimensions that we flattened out.
-    if ragged_rank > 1:
-      # Flatten `tensor` to eliminate all but the last ragged dimension.
-      new_shape = array_ops.concat(
-          [constant_op.constant([-1], dtypes.int64), input_shape[ragged_rank:]],
-          axis=0)
-      flattened = array_ops.reshape(tensor, new_shape)
-      # Recursively convert the flattened tensor.
-      values = from_tensor(flattened, lengths, padding)
-      # The total number of elements in each  dimension.  E.g., if
-      # input_shape=[3, 4, 5, 6], then dim[2] has 3*4*5 elements in total.
-      dim_size = math_ops.cumprod(input_shape)
-      # Construct splits tensors for the dimensions that were flattened.
-      new_splits = [
-          math_ops.range(0, dim_size[dim - 1] + 1) * input_shape[dim]
-          for dim in range(1, ragged_rank)
-      ]
-      return ragged_factory_ops.from_nested_row_splits(values, new_splits)
-
-    # If padding was specified, then use it to find row lengths.
-    if padding is not None:
-      padding = ops.convert_to_tensor(
-          padding, name='padding', dtype=tensor.dtype)
-      padding.shape.assert_is_compatible_with(tensor.shape[2:])
-
-      # Find places where the padding is equal to the tensor.  (This will
-      # broadcast `padding` across the outermost 2 dimensions of `tensor`,
-      # so `has_default_value.shape = tensor.shape`.)
-      has_default_value = math_ops.equal(padding, tensor)
-
-      # If the padding isn't a scalar, then require that all values in the
-      # padding match each item in the tensor.  After this block of code,
-      # `has_default.shape = tensor.shape[:2]`.  (Unfortunately, we can't just
-      # use reduce_all for both cases, becaue when you pass an empty `axis`
-      # list to reduce_all, it reduces all axes; but we want it to reduce no
-      # axes -- i.e., to be a no-op.)
-      tensor_rank = array_ops.rank(tensor)
-      reduce_axis = math_ops.range(2, tensor_rank)
-      has_default = control_flow_ops.cond(
-          tensor_rank > 2,
-          lambda: math_ops.reduce_all(has_default_value, axis=reduce_axis),
-          lambda: has_default_value)
-      has_default.set_shape(tensor_shape.TensorShape([None, None]))
-      has_default.set_shape(tensor.shape[:2])
-
-      # Use has_default it to find the length of each row: for each non-default
-      # item in a row, calculate the length that the row needs to have to
-      # include that item; and then take the max of those values (across each
-      # row).
-      has_nondefault = math_ops.logical_not(has_default)
-      has_nondefault = math_ops.cast(has_nondefault, dtypes.int64)
-      length_for_nondefault_value = (
-          has_nondefault * array_ops.expand_dims(
-              math_ops.range(1, ncols + 1), 0))
-      lengths = math_ops.reduce_max(length_for_nondefault_value, axis=1)
-
-    # If we have lengths (either directly supplied, or computed from paddings),
-    # then use those to construct splits; and then use masking to get the
-    # corresponding values.
-    if lengths is not None:
-      lengths = ragged_util.convert_to_int_tensor(lengths, 'lengths',
-                                                  dtypes.int64)
-      lengths.shape.assert_has_rank(1)
-      lengths = math_ops.minimum(lengths, ncols)
-      lengths = math_ops.maximum(lengths, 0)
-      limits = math_ops.cumsum(lengths)
-      splits = array_ops.concat(
-          [array_ops.zeros([1], dtypes.int64), limits], axis=0)
-      mask = array_ops.sequence_mask(lengths, maxlen=ncols)
-      values = array_ops.boolean_mask(tensor, mask)
-      return ragged_factory_ops.from_row_splits(values, splits)
-
-    # If neither padding nor lengths were specified, then create a splits
-    # vector that contains no default values, and reshape the input tensor
-    # to form the values for the RaggedTensor.
-    nrows = input_shape[0]
-    nvals = nrows * ncols
-    splits = math_ops.range(nrows + 1) * ncols
-    values_shape = array_ops.concat([[nvals], input_shape[2:]], axis=0)
-    values = array_ops.reshape(tensor, values_shape)
-    return ragged_factory_ops.from_row_splits(values, splits)
+  if ragged_tensor.is_ragged(tensor):
+    return tensor
+  else:
+    return ragged_tensor.RaggedTensor.from_tensor(tensor, lengths, padding,
+                                                  ragged_rank, name)
 
 
 def to_tensor(rt_input, default_value=None, name=None):
-  """Converts a `RaggedTensor` into a `Tensor`.
-
-  Example:
-
-  ```python
-  >>> rt = ragged.constant([[9, 8, 7], [], [6, 5], [4]])
-  >>> print ragged.to_tensor(rt).eval()
-  [[9 8 7]
-   [0 0 0]
-   [6 5 0]
-   [4 0 0]]
-  ```
-
-  Args:
-    rt_input: The input `RaggedTensor`.
-    default_value: Value to set for indices not specified in `rt_input`.
-      Defaults to zero.  `default_value` must be broadcastable to
-      `rt_input.shape[rt_input.ragged_rank + 1:]`.
-    name: A name prefix for the returned tensors (optional).
-
-  Returns:
-    A `Tensor` with shape `ragged.bounding_shape(rt_input)` and the
-    values specified by the non-empty values in `rt_input`.  Empty values are
-    assigned `default_value`.
-  """
-  with ops.name_scope(name, 'RaggedToTensor', [rt_input, default_value]):
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        rt_input, name='rt_input')
-    if not ragged_tensor.is_ragged(rt_input):
-      return rt_input  # already dense
-    if default_value is not None:
-      default_value = ops.convert_to_tensor(
-          default_value, name='default_value', dtype=rt_input.dtype)
-
-    # If ragged_rank > 1, then recursively convert the ragged values into a
-    # `Tensor` before we proceed.
-    values = rt_input.values
-    if ragged_tensor.is_ragged(values):
-      values = to_tensor(values, default_value)
-
-    # Tile the default value, if necessary.
-    if default_value is not None:
-      if values.shape.ndims is not None:
-        default_value.shape.with_rank_at_most(values.shape.ndims - 1)
-      if (values.shape.ndims is None or default_value.shape.ndims is None or
-          values.shape.ndims != default_value.shape.ndims + 1):
-        value_shape = array_ops.shape(values)[1:]
-        default_value = array_ops.broadcast_to(default_value, value_shape)
-      default_value.shape.assert_is_compatible_with(values.shape[1:])
-
-    # Get the expected dense shape ([nrows, ncols] + value_shape).
-    rt_row_lengths = [rt_input.row_splits[1:] - rt_input.row_splits[:-1]]
-    nrows = array_ops.shape(rt_input.row_splits, out_type=dtypes.int64)[0] - 1
-    ncols = math_ops.maximum(math_ops.reduce_max(rt_row_lengths), 0)
-    values_shape = array_ops.shape(values, out_type=dtypes.int64)
-    value_shape = values_shape[1:]
-    nvals = values_shape[0]
-
-    # Build a default value if none was supplied.
-    if default_value is None:
-      default_value = array_ops.zeros(value_shape, dtype=values.dtype)
-    default_value.shape.assert_is_compatible_with(values.shape[1:])
-    default_value.set_shape(values.shape[1:])
-
-    # Get the row start indices, and expand to shape=[nrows, 1].
-    starts = array_ops.expand_dims(rt_input.row_splits[:-1], 1)
-
-    # Get the row limit indices, and expand to shape=[nrows, 1].
-    limits = array_ops.expand_dims(rt_input.row_splits[1:], 1)
-
-    # Get the column indices, and expand to shape=[1, ncols].
-    columns = array_ops.expand_dims(math_ops.range(0, ncols), 0)
-
-    # Build a list containing the values plus the default value.  We will use
-    # tf.gather to collect values from this list for the `Tensor` (using
-    # nvals as the index for the default value).
-    values_and_default = array_ops.concat(
-        [values, array_ops.stack([default_value])], axis=0)
-
-    # Construct a matrix "indices" pointing into values_and_default.  I.e.,
-    # output[r, c] = values_and_default[indices[r, c].
-    nondefault_index = starts + columns
-    has_value = nondefault_index < limits
-    default_index = array_ops.fill(array_ops.stack([nrows, ncols]), nvals)
-    indices = array_ops.where(has_value, nondefault_index, default_index)
-
-    # Gather the results into a `Tensor`.
-    return array_ops.gather(values_and_default, indices)
+  if ragged_tensor.is_ragged(rt_input):
+    return rt_input.to_tensor(default_value, name)
+  else:
+    return rt_input
 
 
-#===============================================================================
-# RaggedTensor <-> SparseTensor conversion
-#===============================================================================
 def to_sparse(rt_input, name=None):
-  """Converts a `RaggedTensor` into a sparse tensor.
-
-  Example:
-
-  ```python
-  >>> rt = ragged.constant([[1, 2, 3], [4], [], [5, 6]])
-  >>> ragged.to_sparse(rt).eval()
-  SparseTensorValue(indices=[[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [3, 1]],
-                    values=[1, 2, 3, 4, 5, 6],
-                    dense_shape=[4, 3])
-  ```
-
-  Args:
-    rt_input: The input `RaggedTensor`.
-    name: A name prefix for the returned tensors (optional).
-
-  Returns:
-    A SparseTensor with the same values as `rt_input`.
-  """
-  if not ragged_tensor.is_ragged(rt_input):
-    raise TypeError('Expected RaggedTensor, got %s' % type(rt_input).__name__)
-  with ops.name_scope(name, 'RaggedToSparse', [rt_input]):
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        rt_input, name='rt_input')
-    result = gen_ragged_conversion_ops.ragged_tensor_to_sparse(
-        rt_input.nested_row_splits, rt_input.inner_values, name=name)
-    return sparse_tensor.SparseTensor(
-        result.sparse_indices, result.sparse_values, result.sparse_dense_shape)
-
-
-@ops.RegisterGradient('RaggedTensorToSparse')
-def _ragged_tensor_to_sparse_gradient(op, unused_sparse_indices_grad,
-                                      sparse_values_grad,
-                                      unused_sparse_shape_grad):
-  """Gradient for ragged.to_sparse."""
-  op_inputs_nested_row_splits = op.inputs[:-1]
-  op_inputs_inner_values = op.inputs[-1]
-
-  # No gradient for the RaggedTensor's nested_row_splits.
-  nested_row_splits_gradient = [None] * len(op_inputs_nested_row_splits)
-
-  # Gradient for the RaggedTensor's inner_values is formed by reshaping
-  # the gradient for the SparseTensor's values.
-  inner_values_shape = array_ops.shape(op_inputs_inner_values)
-  inner_values_gradient = array_ops.reshape(sparse_values_grad,
-                                            inner_values_shape)
-
-  return nested_row_splits_gradient + [inner_values_gradient]
+  return rt_input.to_sparse(name)
 
 
 def from_sparse(st_input, name=None):
-  """Converts a 2D `SparseTensor` to a `RaggedTensor`.
-
-  Each row of the `output` `RaggedTensor` will contain the explicit values from
-  the same row in `st_input`.  `st_input` must be ragged-right.  If not it is
-  not ragged-right, then an error will be generated.
-
-  Example:
-
-  ```python
-  >>> st = SparseTensor(indices=[[0, 1], [0, 2], [0, 3], [1, 0], [3, 0]],
-  ...                   values=[1, 2, 3, 4, 5],
-  ...                   dense_shape=[4, 3])
-  >>> ragged.from_sparse(st).eval().tolist()
-  [[1, 2, 3], [4], [], [5]]
-  ```
-
-  Currently, only two-dimensional `SparseTensors` are supported.
-
-  Args:
-    st_input: The sparse tensor to convert.  Must have rank 2.
-    name: A name prefix for the returned tensors (optional).
-
-  Returns:
-    A `RaggedTensor` with the same values as `st_input`.
-    `output.ragged_rank = rank(st_input) - 1`.
-    `output.shape = [st_input.dense_shape[0], None]`.
-  Raises:
-    ValueError: If the number of dimensions in `st_input` is not known
-      statically, or is not two.
-  """
-  if not sparse_tensor.is_sparse(st_input):
-    raise TypeError('Expected SparseTensor, got %s' % type(st_input).__name__)
-  with ops.name_scope(name, 'RaggedFromSparse', [st_input]):
-    st_input = sparse_tensor.convert_to_tensor_or_sparse_tensor(
-        st_input, name='rt_input')
-
-    static_rank_from_dense_shape = (
-        None if st_input.dense_shape.shape.ndims is None
-        else st_input.dense_shape.shape.dims[0].value)
-    static_rank_from_indices = (
-        None if st_input.indices.shape.ndims is None
-        else st_input.indices.shape.dims[1].value)
-
-    if static_rank_from_dense_shape != 2 and static_rank_from_indices != 2:
-      raise ValueError('rank(st_input) must be 2')
-
-    with ops.control_dependencies(
-        _assert_sparse_indices_are_ragged_right(st_input.indices)):
-      # Treat sparse row indices as segment ids to generate a splits tensor that
-      # we can pair with the sparse tensor values.  (Ignore sparse column
-      # indices.)
-      segment_ids = st_input.indices[:, 0]
-      num_segments = st_input.dense_shape[0]
-      return ragged_factory_ops.from_value_rowids(st_input.values, segment_ids,
-                                                  num_segments)
-
-
-def _assert_sparse_indices_are_ragged_right(indices):
-  """Checks that the given SparseTensor.indices tensor is ragged-right.
-
-  Example: `indices = [[0, 0], [0, 1], [2, 0], [3, 1]]` is not ragged right
-  because the entry `[3, 1]` skips a cell.
-
-  Args:
-    indices: The SparseTensor indices to check.
-
-  Returns:
-    A list of control dependency op tensors.
-  """
-  index_prefix = indices[:, :-1]
-  index_suffix = indices[:, -1]
-
-  # Check whether each index is starting a new row in the innermost dimension
-  # (prefix[i] != prefix[i-1]) or continuing a row (prefix[i] == prefix[i-1]).
-  # (Note: this skips the first index; we will check that separately below.)
-  index_prefix_changed = math_ops.reduce_any(
-      math_ops.not_equal(index_prefix[1:], index_prefix[:-1]), axis=1)
-
-  # Check two cases:
-  #   * For indices that start a new row: index_suffix[i] must be zero.
-  #   * For indices that continue a row: index_suffix[i] must be equal to
-  #     index_suffix[i-1]+1.
-  index_ok = array_ops.where(
-      index_prefix_changed, math_ops.equal(index_suffix[1:], 0),
-      math_ops.equal(index_suffix[1:], index_suffix[:-1] + 1))
-
-  # Also check that the very first index didn't skip any cells.  The first
-  # index starts a new row (by definition), so its suffix should be zero.
-  sparse_indices_are_ragged_right = math_ops.logical_and(
-      math_ops.reduce_all(math_ops.equal(index_suffix[:1], 0)),
-      math_ops.reduce_all(index_ok))
-
-  message = [
-      'SparseTensor is not right-ragged',
-      'SparseTensor.indices =', indices
-  ]
-  return [control_flow_ops.Assert(sparse_indices_are_ragged_right, message)]
+  return ragged_tensor.RaggedTensor.from_sparse(st_input, name)
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 7f44ac2ec1..7c74f7be62 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -30,7 +30,6 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.ragged import ragged_array_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_tensor_shape
@@ -122,22 +121,22 @@ class UnaryRaggedElementwiseDispatcher(dispatch.OpDispatcher):
         nested_splits_lists = [
             elt.nested_row_splits for elt in x if ragged_tensor.is_ragged(elt)
         ]
-        inner_values = [
-            elt.inner_values if ragged_tensor.is_ragged(elt) else elt
+        flat_values = [
+            elt.flat_values if ragged_tensor.is_ragged(elt) else elt
             for elt in x
         ]
         with ops.control_dependencies(
             ragged_util.assert_splits_match(nested_splits_lists)):
-          return ragged_factory_ops.from_nested_row_splits(
-              self._original_op(inner_values, *args, **kwargs),
+          return ragged_tensor.RaggedTensor.from_nested_row_splits(
+              self._original_op(flat_values, *args, **kwargs),
               nested_splits_lists[0])
       else:
         return self.NOT_SUPPORTED
     else:
       found_ragged = ragged_tensor.is_ragged(x)
       if found_ragged:
-        mapped_values = self._original_op(x.inner_values, *args, **kwargs)
-        return x.with_inner_values(mapped_values)
+        mapped_values = self._original_op(x.flat_values, *args, **kwargs)
+        return x.with_flat_values(mapped_values)
       else:
         return self.NOT_SUPPORTED
 
@@ -191,8 +190,8 @@ class BinaryRaggedElementwiseDispatcher(dispatch.OpDispatcher):
       return self.NOT_SUPPORTED
 
     if ((x_is_ragged and y_is_ragged) or
-        (x_is_ragged and x.inner_values.shape.ndims <= y.shape.ndims) or
-        (y_is_ragged and y.inner_values.shape.ndims <= x.shape.ndims)):
+        (x_is_ragged and x.flat_values.shape.ndims <= y.shape.ndims) or
+        (y_is_ragged and y.flat_values.shape.ndims <= x.shape.ndims)):
       bcast_shape = ragged_tensor_shape.broadcast_dynamic_shape(
           ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(x),
           ragged_tensor_shape.RaggedTensorDynamicShape.from_tensor(y))
@@ -201,13 +200,13 @@ class BinaryRaggedElementwiseDispatcher(dispatch.OpDispatcher):
       y = ragged_tensor_shape.broadcast_to(
           y, bcast_shape, broadcast_inner_dimensions=False)
 
-    x_values = x.inner_values if ragged_tensor.is_ragged(x) else x
-    y_values = y.inner_values if ragged_tensor.is_ragged(y) else y
+    x_values = x.flat_values if ragged_tensor.is_ragged(x) else x
+    y_values = y.flat_values if ragged_tensor.is_ragged(y) else y
     mapped_values = self._original_op(x_values, y_values, *args, **kwargs)
     if ragged_tensor.is_ragged(x):
-      return x.with_inner_values(mapped_values)
+      return x.with_flat_values(mapped_values)
     else:
-      return y.with_inner_values(mapped_values)
+      return y.with_flat_values(mapped_values)
 
 
 class RaggedDispatcher(dispatch.OpDispatcher):
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 2533c60c4e..82827aa2aa 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -32,6 +33,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import ragged
 from tensorflow.python.ops import string_ops
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 # Constants listing various op types to test.  Each operation
@@ -126,8 +128,8 @@ BINARY_INT_OPS = [
 ]
 
 
-@test_util.run_v1_only('b/120545219')
-class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedElementwiseOpsTest(ragged_test_util.RaggedTensorTestCase,
                                parameterized.TestCase):
 
   def assertSameShape(self, x, y):
@@ -138,7 +140,7 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
       for (x_splits, y_splits) in zip(x.nested_row_splits, y.nested_row_splits):
         self.assertAllEqual(x_splits, y_splits)
       self.assertAllEqual(
-          array_ops.shape(x.inner_values), array_ops.shape(y.inner_values))
+          array_ops.shape(x.flat_values), array_ops.shape(y.flat_values))
     else:
       self.assertIsInstance(y, ops.Tensor)
       self.assertAllEqual(array_ops.shape(x), array_ops.shape(y))
@@ -215,19 +217,18 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
     result = op(x, **extra_args)
 
     # Run the wrapped op on the dense values, for comparison.
-    dense_x = x.inner_values if isinstance(x, ragged.RaggedTensor) else x
+    dense_x = x.flat_values if isinstance(x, ragged.RaggedTensor) else x
     expected_flat_values = array_ops.reshape(op(dense_x, **extra_args), [-1])
 
-    with self.test_session():
-      # Check that the result has the expected shape.
-      self.assertSameShape(x, result)
+    # Check that the result has the expected shape.
+    self.assertSameShape(x, result)
 
-      # Check that the result has the expected (flattened) values.
-      if isinstance(result, ragged.RaggedTensor):
-        result_flat_values = array_ops.reshape(result.inner_values, [-1])
-      else:
-        result_flat_values = array_ops.reshape(result, [-1])
-      self.assertAllEqual(expected_flat_values, result_flat_values)
+    # Check that the result has the expected (flattened) values.
+    if isinstance(result, ragged.RaggedTensor):
+      result_flat_values = array_ops.reshape(result.flat_values, [-1])
+    else:
+      result_flat_values = array_ops.reshape(result, [-1])
+    self.assertAllEqual(expected_flat_values, result_flat_values)
 
   @parameterized.parameters(
       [
@@ -327,21 +328,20 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
       result = op(x, y, **extra_args)
 
     # Run the wrapped op on the dense values, for comparison.
-    dense_x = x.inner_values if isinstance(x, ragged.RaggedTensor) else x
-    dense_y = y.inner_values if isinstance(y, ragged.RaggedTensor) else y
+    dense_x = x.flat_values if isinstance(x, ragged.RaggedTensor) else x
+    dense_y = y.flat_values if isinstance(y, ragged.RaggedTensor) else y
     expected_flat_values = array_ops.reshape(
         op(dense_x, dense_y, **extra_args), [-1])
 
-    with self.test_session():
-      # Check that the result has the expected shape.
-      self.assertSameShape(y, result)
+    # Check that the result has the expected shape.
+    self.assertSameShape(y, result)
 
-      # Check that the result has the expected (flattened) values.
-      if isinstance(result, ragged.RaggedTensor):
-        result_flat_values = array_ops.reshape(result.inner_values, [-1])
-      else:
-        result_flat_values = array_ops.reshape(result, [-1])
-      self.assertAllEqual(expected_flat_values, result_flat_values)
+    # Check that the result has the expected (flattened) values.
+    if isinstance(result, ragged.RaggedTensor):
+      result_flat_values = array_ops.reshape(result.flat_values, [-1])
+    else:
+      result_flat_values = array_ops.reshape(result, [-1])
+    self.assertAllEqual(expected_flat_values, result_flat_values)
 
   @parameterized.parameters(
       [
@@ -385,26 +385,27 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
 
     # Run the wrapped op on the dense values, for comparison.
     dense_inputs = [
-        x.inner_values if isinstance(x, ragged.RaggedTensor) else x
+        x.flat_values if isinstance(x, ragged.RaggedTensor) else x
         for x in inputs
     ]
     expected_flat_values = array_ops.reshape(
         op(dense_inputs, **extra_args), [-1])
 
-    with self.test_session():
-      # Check that the result has the expected shape.
-      self.assertSameShape(inputs[0], result)
+    # Check that the result has the expected shape.
+    self.assertSameShape(inputs[0], result)
 
-      # Check that the result has the expected (flattened) values.
-      if isinstance(result, ragged.RaggedTensor):
-        result_flat_values = array_ops.reshape(result.inner_values, [-1])
-      else:
-        result_flat_values = array_ops.reshape(result, [-1])
-      self.assertAllEqual(expected_flat_values, result_flat_values)
+    # Check that the result has the expected (flattened) values.
+    if isinstance(result, ragged.RaggedTensor):
+      result_flat_values = array_ops.reshape(result.flat_values, [-1])
+    else:
+      result_flat_values = array_ops.reshape(result, [-1])
+    self.assertAllEqual(expected_flat_values, result_flat_values)
 
   def testElementwiseOpUnknownRankError(self):
+    if context.executing_eagerly():
+      return
     x = ragged.constant([[1, 2], [3]])
-    y = ragged.from_row_splits(
+    y = ragged.RaggedTensor.from_row_splits(
         array_ops.placeholder_with_default([1, 2, 3], shape=None), x.row_splits)
     with self.assertRaisesRegexp(ValueError,
                                  r'Unable to broadcast: unknown rank'):
@@ -428,27 +429,22 @@ class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase,
     x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32)
     y = ragged.convert_to_tensor_or_ragged_tensor(y, dtype=dtypes.int32)
     result = x + y
-    with self.cached_session():
-      self.assertEqual(result.eval().tolist(), expected)
+    self.assertRaggedEqual(result, expected)
 
   def testElementwiseOpShapeMismatch(self):
     x = ragged.constant([[1, 2, 3], [4, 5]])
     y = ragged.constant([[1, 2, 3], [4, 5, 6]])
-    with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                 'Incompatible shapes'):
-      with self.cached_session():
-        math_ops.add(x, y).eval()
+    with self.assertRaises(errors.InvalidArgumentError):
+      self.evaluate(math_ops.add(x, y))
 
   def testBinaryOpSparseAndRagged(self):
     x = ragged.constant([[1, 2, 3], [4, 5]])
     y = sparse_tensor.SparseTensor([[0, 0], [0, 1], [2, 0]], [1, 2, 3], [3, 2])
-    with self.assertRaises(TypeError):
-      with self.cached_session():
-        math_ops.add(x, y).eval()
+    with self.assertRaises((TypeError, ValueError)):
+      self.evaluate(math_ops.add(x, y))
 
-    with self.assertRaises(TypeError):
-      with self.cached_session():
-        math_ops.add_n([x, y]).eval()
+    with self.assertRaises((TypeError, ValueError)):
+      self.evaluate(math_ops.add_n([x, y]))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_eager_test.py b/tensorflow/python/ops/ragged/ragged_eager_test.py
index 731ff742aa..f1befbf961 100644
--- a/tensorflow/python/ops/ragged/ragged_eager_test.py
+++ b/tensorflow/python/ops/ragged/ragged_eager_test.py
@@ -17,17 +17,17 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import sys
 
 from absl.testing import parameterized
 
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
+                       parameterized.TestCase):
 
   @parameterized.parameters([
       dict(pylist=[[b'a', b'b'], [b'c']]),
@@ -36,21 +36,15 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   ])
   def testRaggedTensorToList(self, pylist, ragged_rank=None):
     rt = ragged.constant(pylist, ragged_rank)
-    self.assertEqual(rt.tolist(), pylist)
-
-  expected = "RaggedTensor([['a', 'b'], ['c']])"
-  if sys.version_info[0] == 3:
-    expected = "RaggedTensor([[b'a', b'b'], [b'c']])"
+    self.assertRaggedEqual(rt, pylist)
 
   @parameterized.parameters([
-      dict(pylist=[['a', 'b'], ['c']],
-           expected=expected),
-      dict(pylist=[[[1, 2], [3]], [[4, 5, 6], [], [7]]],
-           expected='RaggedTensor([[[1, 2], [3]], [[4, 5, 6], [], [7]]])'),
+      dict(pylist=[[b'a', b'b'], [b'c']]),
+      dict(pylist=[[[1, 2], [3]], [[4, 5, 6], [], [7]]]),
   ])
-  def testRaggedTensorStr(self, pylist, expected):
+  def testRaggedTensorStr(self, pylist):
     rt = ragged.constant(pylist)
-    self.assertEqual(str(rt), expected)
+    self.assertEqual(str(rt), '<tf.RaggedTensor %s>' % pylist)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py b/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py
index 3ff66973b6..072f330e3c 100644
--- a/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_expand_dims_op_test.py
@@ -22,10 +22,12 @@ from absl.testing import parameterized
 
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedExpandDimsOpTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedExpandDimsOpTest(ragged_test_util.RaggedTensorTestCase,
                              parameterized.TestCase):
 
   # An example 4-d ragged tensor with shape [3, (D2), (D3), 2], and the
@@ -105,7 +107,6 @@ class RaggedExpandDimsOpTest(test_util.TensorFlowTestCase,
            expected=EXAMPLE4D_EXPAND_AXIS[4],
            expected_shape=[3, None, None, 2, 1]),
   ])  # pyformat: disable
-  @test_util.run_deprecated_v1
   def testRaggedExpandDims(self,
                            rt_input,
                            axis,
@@ -118,8 +119,7 @@ class RaggedExpandDimsOpTest(test_util.TensorFlowTestCase,
     if expected_shape is not None:
       self.assertEqual(expanded.shape.as_list(), expected_shape)
 
-    with self.test_session():
-      self.assertEqual(expanded.eval().tolist(), expected)
+    self.assertRaggedEqual(expanded, expected)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_factory_ops.py b/tensorflow/python/ops/ragged/ragged_factory_ops.py
index d1f301bc58..2c63e1c799 100644
--- a/tensorflow/python/ops/ragged/ragged_factory_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_factory_ops.py
@@ -21,11 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_tensor_value
 
@@ -56,8 +52,8 @@ def constant(pylist, dtype=None, ragged_rank=None, inner_shape=None, name=None):
       `pylist`.
     ragged_rank: An integer specifying the ragged rank of the returned
       `RaggedTensor`.  Must be nonnegative and less than `K`. Defaults to
-      `max(0, K - 1)` if `inner_shape` is not specified.  Defaults to
-      `max(0, K - 1 - len(inner_shape))` if `inner_shape` is specified.
+      `max(0, K - 1)` if `inner_shape` is not specified.  Defaults to `max(0, K
+      - 1 - len(inner_shape))` if `inner_shape` is specified.
     inner_shape: A tuple of integers specifying the shape for individual inner
       values in the returned `RaggedTensor`.  Defaults to `()` if `ragged_rank`
       is not specified.  If `ragged_rank` is specified, then a default is chosen
@@ -72,9 +68,10 @@ def constant(pylist, dtype=None, ragged_rank=None, inner_shape=None, name=None):
     ValueError: If the scalar values in `pylist` have inconsistent nesting
       depth; or if ragged_rank or inner_shape are incompatible with `pylist`.
   """
-  with ops.name_scope(name, 'RaggedConstant'):
-    return _constant_value(from_row_splits, constant_op.constant, pylist, dtype,
-                           ragged_rank, inner_shape)
+  with ops.name_scope(name, "RaggedConstant"):
+    return _constant_value(ragged_tensor.RaggedTensor.from_row_splits,
+                           constant_op.constant, pylist, dtype, ragged_rank,
+                           inner_shape)
 
 
 def constant_value(pylist, dtype=None, ragged_rank=None, inner_shape=None):
@@ -153,29 +150,29 @@ def _constant_value(ragged_factory, inner_factory, pylist, dtype, ragged_rank,
       depth; or if ragged_rank or inner_shape are incompatible with `pylist`.
   """
   if ragged_tensor.is_ragged(pylist):
-    raise TypeError('pylist may not be a RaggedTensor or RaggedTensorValue.')
+    raise TypeError("pylist may not be a RaggedTensor or RaggedTensorValue.")
 
   if not isinstance(pylist, (list, tuple)):
     # Scalar value
     if ragged_rank is not None and ragged_rank != 0:
-      raise ValueError('Invalid pylist=%r: incompatible with ragged_rank=%d' %
+      raise ValueError("Invalid pylist=%r: incompatible with ragged_rank=%d" %
                        (pylist, ragged_rank))
     if inner_shape is not None and inner_shape:
       raise ValueError(
-          'Invalid pylist=%r: incompatible with dim(inner_shape)=%d' %
+          "Invalid pylist=%r: incompatible with dim(inner_shape)=%d" %
           (pylist, len(inner_shape)))
     return inner_factory(pylist, dtype, ())
 
   if ragged_rank is not None and ragged_rank < 0:
     raise ValueError(
-        'Invalid ragged_rank=%r: must be nonnegative' % ragged_rank)
+        "Invalid ragged_rank=%r: must be nonnegative" % ragged_rank)
 
   # Find the depth of scalar values in `pylist`.
   scalar_depth, max_depth = _find_scalar_and_max_depth(pylist)
   if scalar_depth is not None:
     if max_depth > scalar_depth:
-      raise ValueError('Invalid pylist=%r: empty list nesting is greater '
-                       'than scalar value nesting' % pylist)
+      raise ValueError("Invalid pylist=%r: empty list nesting is greater "
+                       "than scalar value nesting" % pylist)
 
   # If both inner_shape and ragged_rank were specified, then check that
   # they are compatible with pylist.
@@ -184,8 +181,8 @@ def _constant_value(ragged_factory, inner_factory, pylist, dtype, ragged_rank,
     if ((scalar_depth is not None and expected_depth != scalar_depth) or
         (scalar_depth is None and expected_depth < max_depth)):
       raise ValueError(
-          'Invalid pylist=%r: incompatible with ragged_rank=%d '
-          'and dim(inner_shape)=%d' % (pylist, ragged_rank, len(inner_shape)))
+          "Invalid pylist=%r: incompatible with ragged_rank=%d "
+          "and dim(inner_shape)=%d" % (pylist, ragged_rank, len(inner_shape)))
 
   # Check if the result is a `Tensor`.
   if (ragged_rank == 0 or
@@ -221,7 +218,7 @@ def _constant_value(ragged_factory, inner_factory, pylist, dtype, ragged_rank,
     values = concatenated_values
 
   values = inner_factory(
-      values, dtype=dtype, shape=(len(values),) + inner_shape, name='values')
+      values, dtype=dtype, shape=(len(values),) + inner_shape, name="values")
   for row_splits in reversed(nested_splits):
     values = ragged_factory(values, row_splits)
   return values
@@ -249,7 +246,7 @@ def _find_scalar_and_max_depth(pylist):
       child_scalar_depth, child_max_depth = _find_scalar_and_max_depth(child)
       if child_scalar_depth is not None:
         if scalar_depth is not None and scalar_depth != child_scalar_depth + 1:
-          raise ValueError('all scalar values must have the same nesting depth')
+          raise ValueError("all scalar values must have the same nesting depth")
         scalar_depth = child_scalar_depth + 1
       max_depth = max(max_depth, child_max_depth + 1)
     return (scalar_depth, max_depth)
@@ -273,436 +270,24 @@ def _default_inner_shape_for_pylist(pylist, ragged_rank):
     """Checks that `item` has a consistent shape matching `shape`."""
     is_nested = isinstance(item, (list, tuple))
     if is_nested != bool(shape):
-      raise ValueError('inner values have inconsistent shape')
+      raise ValueError("inner values have inconsistent shape")
     if is_nested:
       if shape[0] != len(item):
-        raise ValueError('inner values have inconsistent shape')
+        raise ValueError("inner values have inconsistent shape")
       for child in item:
         check_inner_shape(child, shape[1:])
 
   # Collapse the ragged layers to get the list of inner values.
-  inner_values = pylist
+  flat_values = pylist
   for dim in range(ragged_rank):
-    if not all(isinstance(v, (list, tuple)) for v in inner_values):
-      raise ValueError('pylist has scalar values depth %d, but ragged_rank=%d '
-                       'requires scalar value depth greater than %d' %
+    if not all(isinstance(v, (list, tuple)) for v in flat_values):
+      raise ValueError("pylist has scalar values depth %d, but ragged_rank=%d "
+                       "requires scalar value depth greater than %d" %
                        (dim + 1, ragged_rank, ragged_rank))
-    inner_values = sum((list(v) for v in inner_values), [])
+    flat_values = sum((list(v) for v in flat_values), [])
 
   # Compute the inner shape looking only at the leftmost elements; and then
   # use check_inner_shape to verify that other elements have the same shape.
-  inner_shape = get_inner_shape(inner_values)
-  check_inner_shape(inner_values, inner_shape)
+  inner_shape = get_inner_shape(flat_values)
+  check_inner_shape(flat_values, inner_shape)
   return inner_shape[1:]
-
-
-#===============================================================================
-# Convert value -> tensor
-#===============================================================================
-def convert_to_tensor_or_ragged_tensor(value,
-                                       dtype=None,
-                                       preferred_dtype=None,
-                                       name=None):
-  """Converts value to a `RaggedTensor` or `Tensor`.
-
-  * If `value` is a `RaggedTensor`, then return it as-is.
-  * If `value` is a `RaggedTensorValue`, return a corresponding constant
-    `RaggedTensor`.
-  * Otherwise, use `convert_to_tensor` to convert `value` to a `Tensor`.
-
-  Args:
-    value: A `RaggedTensor`, a `RaggedTensorValue`, or an object whose type has
-      a registered `Tensor` conversion function.
-    dtype: Optional element type for the returned tensor.  If missing the type
-      is inferred from the type of `value`.
-    preferred_dtype: Optional element type for the returned tensor, used when
-      dtype is None.  This argument has no effect if `value` is already a
-      tensor, or when conversion is not possible.
-    name: Optional name to use if a new `Tensor` is created.
-
-  Returns:
-    A `Tensor` or `RaggedTensor`.
-  """
-  if isinstance(value, ragged_tensor.RaggedTensor):
-    if dtype and not dtype.is_compatible_with(value.dtype):
-      raise ValueError('Tensor conversion requested dtype %s for '
-                       'RaggedTensor with dtype %s: %r' %
-                       (dtype.name, value.dtype.name, value))
-    return value
-  elif isinstance(value, ragged_tensor_value.RaggedTensorValue):
-    with ops.name_scope(name, 'ConvertToTensorOrRaggedTensor', []):
-      inner_values = ops.convert_to_tensor(
-          value=value.inner_values,
-          dtype=dtype,
-          preferred_dtype=preferred_dtype,
-          name='inner_values')
-      return from_nested_row_splits(inner_values, value.nested_row_splits)
-  else:
-    return ops.convert_to_tensor(
-        value=value, dtype=dtype, preferred_dtype=preferred_dtype, name=name)
-
-
-#===============================================================================
-# Ops to construct RaggedTensor from row-partitioned values.
-#===============================================================================
-
-
-def from_value_rowids(values, value_rowids, nrows=None, name=None):
-  """Creates a `RaggedTensor` with rows partitioned by `value_rowids`.
-
-  The returned `RaggedTensor` corresponds with the python list defined by:
-
-  ```python
-  result = [[values[i] for i in range(len(values)) if value_rowids[i] == row]
-            for row in range(nrows)]
-  ```
-
-  Warning: currently, this needs to cast value_rowids to int64 before
-  converting, since `tf.bincount` only supports `int32`.
-
-  Args:
-    values: A potentially ragged tensor with shape `[nvals, ...]`.
-    value_rowids: A 1-D int64 tensor with shape `[nvals]`, which corresponds
-      one-to-one with `values`, and specifies each value's row index.  Must be
-      nonnegative, and must be sorted in ascending order.
-    nrows: An int64 scalar specifying the number of rows.  This should be
-      specified if the `RaggedTensor` may containing empty training rows.  Must
-      be greater than `value_rowids[-1]` (or zero if `value_rowids` is empty).
-      Defaults to `value_rowids[-1]` (or zero if `value_rowids` is empty).
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor`.  `result.rank = values.rank + 1`.
-    `result.ragged_rank = values.ragged_rank + 1`.
-
-  Raises:
-    ValueError: If `nrows` is incompatible with `value_rowids`.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.from_value_rowids(
-    ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
-    ...     value_rowids=[0, 0, 0, 0, 2, 2, 2, 3],
-    ...     nrows=5)
-    >>> rt.eval().tolist()
-    [[3, 1, 4, 1], [], [5, 9, 2], [6], []]
-    ```
-  """
-  with ops.name_scope(name, 'RaggedFromValueRowIds',
-                      [values, value_rowids, nrows]):
-    values = convert_to_tensor_or_ragged_tensor(values, name='values')
-    value_rowids = ops.convert_to_tensor(
-        value_rowids, dtypes.int64, name='value_rowids')
-    if nrows is None:
-      const_rowids = tensor_util.constant_value(value_rowids)
-      if const_rowids is None:
-        nrows = array_ops.concat([value_rowids[-1:], [-1]], axis=0)[0] + 1
-        const_nrows = None
-      else:
-        const_nrows = const_rowids[-1] + 1 if const_rowids.size > 0 else 0
-        nrows = ops.convert_to_tensor(const_nrows, dtypes.int64, name='nrows')
-    else:
-      nrows = ops.convert_to_tensor(nrows, dtypes.int64, 'nrows')
-      const_nrows = tensor_util.constant_value(nrows)
-      if const_nrows is not None:
-        if const_nrows < 0:
-          raise ValueError('Expected nrows >= 0; got %d' % const_nrows)
-        const_rowids = tensor_util.constant_value(value_rowids)
-        if const_rowids is not None and const_rowids.size > 0:
-          if not const_nrows >= const_rowids[-1] + 1:
-            raise ValueError(
-                'Expected nrows >= value_rowids[-1] + 1; got nrows=%d, '
-                'value_rowids[-1]=%d' % (const_nrows, const_rowids[-1]))
-
-    value_rowids.shape.assert_has_rank(1)
-    nrows.shape.assert_has_rank(0)
-    values.shape[:1].assert_is_compatible_with(value_rowids.shape)
-
-    # Convert value_rowids & nrows to row_splits.
-    # Note: we don't use segment_ids_to_row_splits() here because we want
-    # to save the intermediate value `row_lengths`, so we can cache it.
-    # TODO(b/116708836) Upgrade bincount to accept int64 so we can skip the cast
-    # (Remove the warning in the docstring when we do.)
-    value_rowids_int32 = math_ops.cast(value_rowids, dtypes.int32)
-    nrows_int32 = math_ops.cast(nrows, dtypes.int32)
-    row_lengths = math_ops.bincount(
-        value_rowids_int32,
-        minlength=nrows_int32,
-        maxlength=nrows_int32,
-        dtype=dtypes.int64)
-    row_splits = array_ops.concat([[0], math_ops.cumsum(row_lengths)], axis=0)
-    if const_nrows is not None:
-      row_lengths.set_shape([const_nrows])
-      row_splits.set_shape([const_nrows + 1])
-
-    return ragged_tensor.RaggedTensor(
-        values,
-        row_splits,
-        cached_row_lengths=row_lengths,
-        cached_value_rowids=value_rowids,
-        cached_nrows=nrows,
-        internal=True)
-
-
-def from_row_splits(values, row_splits, name=None):
-  """Creates a `RaggedTensor` with rows partitioned by `row_splits`.
-
-  The returned `RaggedTensor` corresponds with the python list defined by:
-
-  ```python
-  result = [values[row_splits[i]:row_splits[i + 1]]
-            for i in range(len(row_splits) - 1)]
-  ```
-
-  Args:
-    values: A potentially ragged tensor with shape `[nvals, ...]`.
-    row_splits: A 1-D int64 tensor with shape `[nrows+1]`.  Must not be empty,
-      and must be sorted in ascending order.  `row_splits[0]` must be zero and
-      `row_splits[-1]` must be `nvals`.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor`.  `result.rank = values.rank + 1`.
-    `result.ragged_rank = values.ragged_rank + 1`.
-
-  Raises:
-    ValueError: If `row_splits` is an empty list.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.from_row_splits(
-    ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
-    ...     row_splits=[0, 4, 4, 7, 8, 8])
-    >>> rt.eval().tolist()
-    [[3, 1, 4, 1], [], [5, 9, 2], [6], []]
-    ```
-  """
-  if isinstance(row_splits, (list, tuple)) and not row_splits:
-    raise ValueError('row_splits tensor may not be empty.')
-  with ops.name_scope(name, 'RaggedFromRowSplits', [values, row_splits]):
-    values = convert_to_tensor_or_ragged_tensor(values, name='values')
-    row_splits = ops.convert_to_tensor(row_splits, dtypes.int64, 'row_splits')
-    row_splits.shape.assert_has_rank(1)
-    return ragged_tensor.RaggedTensor(
-        values=values, row_splits=row_splits, internal=True)
-
-
-def from_row_lengths(values, row_lengths, name=None):
-  """Creates a `RaggedTensor` with rows partitioned by `row_lengths`.
-
-  The returned `RaggedTensor` corresponds with the python list defined by:
-
-  ```python
-  result = [[values.pop(0) for i in range(length)]
-            for length in row_lengths]
-  ```
-
-  Args:
-    values: A potentially ragged tensor with shape `[nvals, ...]`.
-    row_lengths: A 1-D int64 tensor with shape `[nrows]`.  Must be nonnegative.
-      `sum(row_lengths)` must be `nvals`.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor`.  `result.rank = values.rank + 1`.
-    `result.ragged_rank = values.ragged_rank + 1`.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.from_row_lengths(
-    ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
-    ...     row_lengths=[4, 0, 3, 1, 0])
-    >>> rt.eval().tolist()
-    [[3, 1, 4, 1], [], [5, 9, 2], [6], []]
-    ```
-  """
-  with ops.name_scope(name, 'RaggedFromRowLengths', [values, row_lengths]):
-    values = convert_to_tensor_or_ragged_tensor(values, name='values')
-    row_lengths = ops.convert_to_tensor(row_lengths, dtypes.int64,
-                                        'row_lengths')
-    row_lengths.shape.assert_has_rank(1)
-    row_limits = math_ops.cumsum(row_lengths)
-    row_splits = array_ops.concat([[0], row_limits], axis=0)
-    return ragged_tensor.RaggedTensor(
-        values=values,
-        row_splits=row_splits,
-        cached_row_lengths=row_lengths,
-        internal=True)
-
-
-def from_row_starts(values, row_starts, name=None):
-  """Creates a `RaggedTensor` with rows partitioned by `row_starts`.
-
-  Equivalent to: `from_row_splits(values, concat([row_starts, nvals]))`.
-
-  Args:
-    values: A potentially ragged tensor with shape `[nvals, ...]`.
-    row_starts: A 1-D int64 tensor with shape `[nrows]`.  Must be nonnegative
-      and sorted in ascending order.  If `nrows>0`, then `row_starts[0]` must be
-      zero.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor`.  `result.rank = values.rank + 1`.
-    `result.ragged_rank = values.ragged_rank + 1`.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.from_row_starts(
-    ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
-    ...     row_starts=[0, 4, 4, 7, 8])
-    >>> rt.eval().tolist()
-    [[3, 1, 4, 1], [], [5, 9, 2], [6], []]
-    ```
-  """
-  with ops.name_scope(name, 'RaggedFromRowStarts', [values, row_starts]):
-    values = convert_to_tensor_or_ragged_tensor(values, name='values')
-    row_starts = ops.convert_to_tensor(row_starts, dtypes.int64, 'row_starts')
-    row_starts.shape.assert_has_rank(1)
-    nvals = array_ops.shape(values, out_type=dtypes.int64)[:1]
-    row_splits = array_ops.concat([row_starts, nvals], axis=0)
-    return ragged_tensor.RaggedTensor(
-        values=values, row_splits=row_splits, internal=True)
-
-
-def from_row_limits(values, row_limits, name=None):
-  """Creates a `RaggedTensor` with rows partitioned by `row_limits`.
-
-  Equivalent to: `from_row_splits(values, concat([0, row_limits]))`.
-
-  Args:
-    values: A potentially ragged tensor with shape `[nvals, ...]`.
-    row_limits: A 1-D int64 tensor with shape `[nrows]`.  Must be sorted in
-      ascending order.  If `nrows>0`, then `row_limits[-1]` must be `nvals`.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor`.  `result.rank = values.rank + 1`.
-    `result.ragged_rank = values.ragged_rank + 1`.
-
-  #### Example:
-    ```python
-    >>> rt = ragged.from_row_limits(
-    ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
-    ...     row_limits=[4, 4, 7, 8, 8])
-    >>> rt.eval().tolist()
-    [[3, 1, 4, 1], [], [5, 9, 2], [6], []]
-    ```
-  """
-  with ops.name_scope(name, 'RaggedFromRowLimits', [values, row_limits]):
-    values = convert_to_tensor_or_ragged_tensor(values, name='values')
-    row_limits = ops.convert_to_tensor(row_limits, dtypes.int64, 'row_limits')
-    row_limits.shape.assert_has_rank(1)
-    zero = array_ops.zeros([1], dtypes.int64)
-    row_splits = array_ops.concat([zero, row_limits], axis=0)
-    return ragged_tensor.RaggedTensor(
-        values=values, row_splits=row_splits, internal=True)
-
-
-def from_nested_value_rowids(inner_values,
-                             nested_value_rowids,
-                             nested_nrows=None,
-                             name=None):
-  """Creates a `RaggedTensor` from a nested list of `value_rowids` tensors.
-
-  Equivalent to:
-
-  ```python
-  result = inner_values
-  for (value_rowids, nrows) in reversed(zip(nested_value_rowids, nested_nrows)):
-    result = from_value_rowids(result, value_rowids, nrows)
-  ```
-
-  Args:
-    inner_values: A potentially ragged tensor.
-    nested_value_rowids: A list of 1-D int64 tensors.  The `i`th tensor is used
-      as the `value_rowids` for the `i`th ragged dimension.
-    nested_nrows: A list of int64 scalars.  The `i`th scalar is used as the
-      `nrows` for the `i`th ragged dimension.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor` (or `inner_values` if `nested_value_rowids` is empty).
-
-  Raises:
-    ValueError: If `len(nested_values_rowids) != len(nested_nrows)`.
-  """
-  if isinstance(nested_value_rowids, ops.Tensor):
-    raise TypeError('nested_value_rowids must be a list of Tensors')
-  if nested_nrows is None:
-    nested_nrows = [None] * len(nested_value_rowids)
-  else:
-    if isinstance(nested_nrows, ops.Tensor):
-      raise TypeError('nested_nrows must be a list of Tensors')
-    if len(nested_nrows) != len(nested_value_rowids):
-      raise ValueError('nested_nrows must have the same length as '
-                       'nested_value_rowids')
-
-  with ops.name_scope(
-      name, 'RaggedFromNestedValueRowIds',
-      [inner_values] + list(nested_value_rowids) + list(nested_nrows)):
-    result = inner_values
-    for value_rowids, nrows in reversed(
-        list(zip(nested_value_rowids, nested_nrows))):
-      result = from_value_rowids(result, value_rowids, nrows)
-    return result
-
-
-def from_nested_row_splits(inner_values, nested_row_splits, name=None):
-  """Creates a `RaggedTensor` from a nested list of `row_splits` tensors.
-
-  Equivalent to:
-
-  ```python
-  result = inner_values
-  for row_splits in reversed(nested_row_splits):
-    result = from_row_splits(result, row_splits)
-  ```
-
-  Args:
-    inner_values: A potentially ragged tensor.
-    nested_row_splits: A list of 1-D int64 tensors.  The `i`th tensor is used as
-      the `row_splits` for the `i`th ragged dimension.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor` (or `inner_values` if `nested_row_splits` is empty).
-  """
-  if isinstance(nested_row_splits, ops.Tensor):
-    raise TypeError('nested_row_splits must be a list of Tensors')
-  with ops.name_scope(name, 'RaggedFromNestedRowSplits',
-                      [inner_values] + list(nested_row_splits)):
-    result = inner_values
-    for splits in reversed(nested_row_splits):
-      result = from_row_splits(result, splits)
-    return result
-
-
-def from_nested_row_lengths(inner_values, nested_row_lengths, name=None):
-  """Creates a `RaggedTensor` from a nested list of `row_lengths` tensors.
-
-  Equivalent to:
-
-  ```python
-  result = inner_values
-  for row_lengths in reversed(nested_row_lengths):
-    result = from_row_lengths(result, row_lengths)
-  ```
-
-  Args:
-    inner_values: A potentially ragged tensor.
-    nested_row_lengths: A list of 1-D int64 tensors.  The `i`th tensor is used
-      as the `row_lengths` for the `i`th ragged dimension.
-    name: A name prefix for the RaggedTensor (optional).
-
-  Returns:
-    A `RaggedTensor` (or `inner_values` if `nested_row_lengths` is empty).
-  """
-  if isinstance(nested_row_lengths, ops.Tensor):
-    raise TypeError('nested_row_lengths must be a list of Tensors')
-  with ops.name_scope(name, 'RaggedFromNestedRowlengths',
-                      [inner_values] + list(nested_row_lengths)):
-    result = inner_values
-    for lengths in reversed(nested_row_lengths):
-      result = from_row_lengths(result, lengths)
-    return result
diff --git a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
index 674dbab112..07cf910202 100644
--- a/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_sparse_op_test.py
@@ -12,77 +12,77 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged.from_sparse."""
+"""Tests for RaggedTensor.from_sparse."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.ops.ragged import RaggedTensor
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorToSparseOpTest(ragged_test_util.RaggedTensorTestCase):
 
-  @test_util.run_v1_only('b/120545219')
   def testDocStringExample(self):
     st = sparse_tensor.SparseTensor(
         indices=[[0, 0], [0, 1], [0, 2], [1, 0], [3, 0]],
         values=[1, 2, 3, 4, 5],
         dense_shape=[4, 3])
-    rt = ragged.from_sparse(st)
+    rt = RaggedTensor.from_sparse(st)
 
-    with self.test_session():
-      self.assertEqual(rt.eval().tolist(), [[1, 2, 3], [4], [], [5]])
+    self.assertRaggedEqual(rt, [[1, 2, 3], [4], [], [5]])
 
-  @test_util.run_v1_only('b/120545219')
   def testEmpty(self):
     st = sparse_tensor.SparseTensor(
         indices=array_ops.zeros([0, 2], dtype=dtypes.int64),
         values=[],
         dense_shape=[4, 3])
-    rt = ragged.from_sparse(st)
+    rt = RaggedTensor.from_sparse(st)
 
-    with self.test_session():
-      self.assertEqual(rt.eval().tolist(), [[], [], [], []])
+    self.assertRaggedEqual(rt, [[], [], [], []])
 
-  @test_util.run_deprecated_v1
   def testBadSparseTensorRank(self):
     st1 = sparse_tensor.SparseTensor(indices=[[0]], values=[0], dense_shape=[3])
+    self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
+                            RaggedTensor.from_sparse, st1)
+
     st2 = sparse_tensor.SparseTensor(
         indices=[[0, 0, 0]], values=[0], dense_shape=[3, 3, 3])
-    st3 = sparse_tensor.SparseTensor(
-        indices=array_ops.placeholder(dtypes.int64),
-        values=[0],
-        dense_shape=array_ops.placeholder(dtypes.int64))
-    self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
-                            ragged.from_sparse, st1)
-    self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
-                            ragged.from_sparse, st2)
     self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
-                            ragged.from_sparse, st3)
+                            RaggedTensor.from_sparse, st2)
+
+    if not context.executing_eagerly():
+      st3 = sparse_tensor.SparseTensor(
+          indices=array_ops.placeholder(dtypes.int64),
+          values=[0],
+          dense_shape=array_ops.placeholder(dtypes.int64))
+      self.assertRaisesRegexp(ValueError, r'rank\(st_input\) must be 2',
+                              RaggedTensor.from_sparse, st3)
 
-  @test_util.run_v1_only('b/120545219')
   def testGoodPartialSparseTensorRank(self):
-    st1 = sparse_tensor.SparseTensor(
-        indices=[[0, 0]],
-        values=[0],
-        dense_shape=array_ops.placeholder(dtypes.int64))
-    st2 = sparse_tensor.SparseTensor(
-        indices=array_ops.placeholder(dtypes.int64),
-        values=[0],
-        dense_shape=[4, 3])
+    if not context.executing_eagerly():
+      st1 = sparse_tensor.SparseTensor(
+          indices=[[0, 0]],
+          values=[0],
+          dense_shape=array_ops.placeholder(dtypes.int64))
+      st2 = sparse_tensor.SparseTensor(
+          indices=array_ops.placeholder(dtypes.int64),
+          values=[0],
+          dense_shape=[4, 3])
 
-    # Shouldn't throw ValueError
-    ragged.from_sparse(st1)
-    ragged.from_sparse(st2)
+      # Shouldn't throw ValueError
+      RaggedTensor.from_sparse(st1)
+      RaggedTensor.from_sparse(st2)
 
-  @test_util.run_v1_only('b/120545219')
   def testNonRaggedSparseTensor(self):
     # "index_suffix" means the value of the innermost dimension of the index
     # (i.e., indices[i][-1]).
@@ -92,22 +92,21 @@ class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
     # index_suffix of first index is not zero.
     st1 = sparse_tensor.SparseTensor(
         indices=[[0, 1], [0, 2], [2, 0]], values=[1, 2, 3], dense_shape=[3, 3])
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'.*SparseTensor is not right-ragged'):
+      self.evaluate(RaggedTensor.from_sparse(st1))
     # index_suffix of an index that starts a new row is not zero.
     st2 = sparse_tensor.SparseTensor(
         indices=[[0, 0], [0, 1], [2, 1]], values=[1, 2, 3], dense_shape=[3, 3])
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'.*SparseTensor is not right-ragged'):
+      self.evaluate(RaggedTensor.from_sparse(st2))
     # index_suffix of an index that continues a row skips a cell.
     st3 = sparse_tensor.SparseTensor(
         indices=[[0, 1], [0, 1], [0, 3]], values=[1, 2, 3], dense_shape=[3, 3])
-    rt1 = ragged.from_sparse(st1)
-    rt2 = ragged.from_sparse(st2)
-    rt3 = ragged.from_sparse(st3)
-    with self.test_session():
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'.*SparseTensor is not right-ragged', rt1.eval)
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'.*SparseTensor is not right-ragged', rt2.eval)
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'.*SparseTensor is not right-ragged', rt3.eval)
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'.*SparseTensor is not right-ragged'):
+      self.evaluate(RaggedTensor.from_sparse(st3))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
index 09c17fad61..6a3d639c5e 100644
--- a/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_from_tensor_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged.from_tensor."""
+"""Tests for RaggedTensor.from_tensor."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -24,29 +24,26 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.ops.ragged import RaggedTensor
 from tensorflow.python.platform import googletest
 
 
-class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
-                             parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorToSparseOpTest(ragged_test_util.RaggedTensorTestCase,
+                                 parameterized.TestCase):
 
-  @test_util.run_v1_only('b/120545219')
   def testDocStringExamples(self):
-    # The examples from ragged.from_tensor.__doc__.
+    # The examples from RaggedTensor.from_tensor.__doc__.
     dt = constant_op.constant([[5, 7, 0], [0, 3, 0], [6, 0, 0]])
-    with self.test_session():
-      self.assertEqual(
-          ragged.from_tensor(dt).eval().tolist(),
-          [[5, 7, 0], [0, 3, 0], [6, 0, 0]])
+    self.assertRaggedEqual(
+        RaggedTensor.from_tensor(dt), [[5, 7, 0], [0, 3, 0], [6, 0, 0]])
 
-      self.assertEqual(
-          ragged.from_tensor(dt, lengths=[1, 0, 3]).eval().tolist(),
-          [[5], [], [6, 0, 0]])
+    self.assertRaggedEqual(
+        RaggedTensor.from_tensor(dt, lengths=[1, 0, 3]), [[5], [], [6, 0, 0]])
 
-      self.assertEqual(
-          ragged.from_tensor(dt, padding=0).eval().tolist(),
-          [[5, 7], [0, 3], [6]])
+    self.assertRaggedEqual(
+        RaggedTensor.from_tensor(dt, padding=0), [[5, 7], [0, 3], [6]])
 
   @parameterized.parameters(
       # 2D test cases, no length or padding.
@@ -263,7 +260,6 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                        [[[5, 6], [7]], [[0, 8], []]]]
       },
   )  # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testRaggedFromTensor(self,
                            tensor,
                            expected,
@@ -271,30 +267,27 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                            padding=None,
                            ragged_rank=1):
     dt = constant_op.constant(tensor)
-    rt = ragged.from_tensor(dt, lengths, padding, ragged_rank)
-    self.assertEqual(type(rt), ragged.RaggedTensor)
+    rt = RaggedTensor.from_tensor(dt, lengths, padding, ragged_rank)
+    self.assertEqual(type(rt), RaggedTensor)
     self.assertEqual(rt.ragged_rank, ragged_rank)
     self.assertTrue(
         dt.shape.is_compatible_with(rt.shape),
         '%s is incompatible with %s' % (dt.shape, rt.shape))
-    with self.test_session():
-      self.assertEqual(rt.eval().tolist(), expected)
+    self.assertRaggedEqual(rt, expected)
 
-  @test_util.run_deprecated_v1
   def testHighDimensions(self):
     # Use distinct prime numbers for all dimension shapes in this test, so
     # we can see any errors that are caused by mixing up dimension sizes.
     dt = array_ops.reshape(
         math_ops.range(3 * 5 * 7 * 11 * 13 * 17), [3, 5, 7, 11, 13, 17])
     for ragged_rank in range(1, 4):
-      rt = ragged.from_tensor(dt, ragged_rank=ragged_rank)
-      self.assertEqual(type(rt), ragged.RaggedTensor)
+      rt = RaggedTensor.from_tensor(dt, ragged_rank=ragged_rank)
+      self.assertEqual(type(rt), RaggedTensor)
       self.assertEqual(rt.ragged_rank, ragged_rank)
       self.assertTrue(
           dt.shape.is_compatible_with(rt.shape),
           '%s is incompatible with %s' % (dt.shape, rt.shape))
-      with self.test_session():
-        self.assertEqual(rt.eval().tolist(), self.evaluate(dt).tolist())
+      self.assertRaggedEqual(rt, self.evaluate(dt).tolist())
 
   @parameterized.parameters(
       # With no padding or lengths
@@ -398,15 +391,13 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
           'expected': [[], []]
       },
   )
-  @test_util.run_v1_only('b/120545219')
   def testEmpty(self, dt_shape, expected, lengths=None, padding=None):
     dt = array_ops.zeros(dt_shape)
-    rt = ragged.from_tensor(dt, lengths, padding)
-    self.assertEqual(type(rt), ragged.RaggedTensor)
+    rt = RaggedTensor.from_tensor(dt, lengths, padding)
+    self.assertEqual(type(rt), RaggedTensor)
     self.assertEqual(rt.ragged_rank, 1)
     self.assertTrue(dt.shape.is_compatible_with(rt.shape))
-    with self.test_session():
-      self.assertEqual(rt.eval().tolist(), expected)
+    self.assertRaggedEqual(rt, expected)
 
   @parameterized.parameters(
       {
@@ -423,7 +414,7 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
       {
           'tensor': [[1]],
           'padding': 'a',
-          'error': (TypeError, "Expected int32, got 'a'.*")
+          'error': (TypeError, '.*')
       },
       {
           'tensor': [[1]],
@@ -451,7 +442,6 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
           'error': (ValueError, r'ragged_rank must be greater than 0; got -1')
       },
   )
-  @test_util.run_deprecated_v1
   def testErrors(self,
                  tensor,
                  lengths=None,
@@ -459,8 +449,8 @@ class RaggedFromTensorOpTest(test_util.TensorFlowTestCase,
                  ragged_rank=1,
                  error=None):
     dt = constant_op.constant(tensor)
-    self.assertRaisesRegexp(error[0], error[1], ragged.from_tensor, dt, lengths,
-                            padding, ragged_rank)
+    self.assertRaisesRegexp(error[0], error[1], RaggedTensor.from_tensor, dt,
+                            lengths, padding, ragged_rank)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_functional_ops.py b/tensorflow/python/ops/ragged/ragged_functional_ops.py
index 6b71d88435..751f2c7359 100644
--- a/tensorflow/python/ops/ragged/ragged_functional_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_functional_ops.py
@@ -19,15 +19,14 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_util
 
 
-def map_inner_values(op, *args, **kwargs):
+def map_flat_values(op, *args, **kwargs):
   """Applies `op` to the inner values of one or more RaggedTensors.
 
-  Replaces any `RaggedTensor` in `args` or `kwargs` with its `inner_values`
+  Replaces any `RaggedTensor` in `args` or `kwargs` with its `flat_values`
   tensor, and then calls `op`.  Returns a `RaggedTensor` that is constructed
   from the input `RaggedTensor`s' `splits` and the value returned by
   the `op`.
@@ -39,20 +38,20 @@ def map_inner_values(op, *args, **kwargs):
 
   ```python
   >>> rt = ragged.constant([[1, 2, 3], [], [4, 5], [6]])
-  >>> ragged.map_inner_values(tf.ones_like, rt).eval().tolist()
+  >>> ragged.map_flat_values(tf.ones_like, rt).eval().tolist()
   [[1, 1, 1], [], [1, 1], [1]]
-  >>> ragged.map_inner_values(tf.multiply, rt, rt).eval().tolist()
+  >>> ragged.map_flat_values(tf.multiply, rt, rt).eval().tolist()
   [[1, 4, 9], [], [16, 25], [36]]
-  >>> ragged.map_inner_values(tf.add, rt, 5).eval().tolist()
+  >>> ragged.map_flat_values(tf.add, rt, 5).eval().tolist()
   [[6, 7, 8], [], [9, 10], [11]]
   ```
 
   Args:
-    op: The operation that should be applied to the RaggedTensor `inner_values`.
+    op: The operation that should be applied to the RaggedTensor `flat_values`.
       `op` is typically an element-wise operation (such as math_ops.add), but
       any operation that preserves the size of the outermost dimension can be
       used.  I.e., `shape[0]` of the value returned by `op` must match
-      `shape[0]` of the `RaggedTensor`s' `inner_values` tensors.
+      `shape[0]` of the `RaggedTensor`s' `flat_values` tensors.
     *args: Arguments for `op`.
     **kwargs: Keyword arguments for `op`.
 
@@ -66,8 +65,8 @@ def map_inner_values(op, *args, **kwargs):
   # Replace RaggedTensors with their values; and collect the splits tensors
   # from each RaggedTensor.
   nested_splits_lists = []
-  inner_args = _replace_ragged_with_inner_values(args, nested_splits_lists)
-  inner_kwargs = _replace_ragged_with_inner_values(kwargs, nested_splits_lists)
+  inner_args = _replace_ragged_with_flat_values(args, nested_splits_lists)
+  inner_kwargs = _replace_ragged_with_flat_values(kwargs, nested_splits_lists)
   if not nested_splits_lists:
     return op(*args, **kwargs)
 
@@ -75,15 +74,15 @@ def map_inner_values(op, *args, **kwargs):
       ragged_util.assert_splits_match(nested_splits_lists)):
     # Delegate to op, and then compose the result from the transformed values
     # and the splits.
-    return ragged_factory_ops.from_nested_row_splits(
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
         op(*inner_args, **inner_kwargs), nested_splits_lists[0])
 
 
-def _replace_ragged_with_inner_values(value, nested_splits_lists):
-  """Replace RaggedTensors with their inner_values, and record their splits.
+def _replace_ragged_with_flat_values(value, nested_splits_lists):
+  """Replace RaggedTensors with their flat_values, and record their splits.
 
   Returns a copy of `value`, with any nested `RaggedTensor`s replaced by their
-  `inner_values` tensor.  Looks inside lists, tuples, and dicts.
+  `flat_values` tensor.  Looks inside lists, tuples, and dicts.
 
   Appends each `RaggedTensor`'s `nested_splits` to `nested_splits_lists`.
 
@@ -97,13 +96,13 @@ def _replace_ragged_with_inner_values(value, nested_splits_lists):
   """
   # Base case
   if ragged_tensor.is_ragged(value):
-    value = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(value)
+    value = ragged_tensor.convert_to_tensor_or_ragged_tensor(value)
     nested_splits_lists.append(value.nested_row_splits)
-    return value.inner_values
+    return value.flat_values
 
   # Recursion cases
   def recurse(v):
-    return _replace_ragged_with_inner_values(v, nested_splits_lists)
+    return _replace_ragged_with_flat_values(v, nested_splits_lists)
 
   if isinstance(value, list):
     return [recurse(v) for v in value]
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index b0a4fe9d31..6673192752 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -21,15 +21,18 @@ from __future__ import print_function
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-@test_util.run_v1_only('b/120545219')
-class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
                            parameterized.TestCase):
 
   DOCSTRING_PARAMS = [[['000', '001'], ['010']],
@@ -186,14 +189,11 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
   ])  # pyformat: disable
   def testRaggedGatherNd(self, descr, params, indices, expected):
     result = ragged.gather_nd(params, indices)
-    self.assertEqual(
-        getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0))
-    with self.test_session() as sess:
-      if hasattr(expected, 'tolist'):
-        expected = expected.tolist()
-      self.assertEqual(self.evaluate(result).tolist(), expected)
+    self.assertRaggedEqual(result, expected)
 
   def testRaggedGatherNdUnknownRankError(self):
+    if context.executing_eagerly():
+      return
     params = ragged.constant([['a', 'b'], ['c', 'd']])
     indices1 = array_ops.placeholder(dtypes.int32, shape=None)
     indices2 = array_ops.placeholder(dtypes.int32, shape=[None])
@@ -209,21 +209,20 @@ class RaggedGatherNdOpTest(test_util.TensorFlowTestCase,
       dict(
           params=['a'],
           indices=0,
-          message='Shape must be at least rank 1 but is rank 0'
-          " for 'GatherNd'"),
+          error=(ValueError, errors.InvalidArgumentError)),
       dict(
           params=ragged.constant_value([['a']]),
           indices=0,
           message='indices.rank must be at least 1.'),
       dict(
           params=['a', 'b', 'c'],
-          indices=ragged.constant([[0]]),
+          indices=ragged.constant_value([[0]]),
           message='The innermost dimension of indices may not be ragged'),
   ])
   def testRaggedGatherNdStaticError(self,
                                     params,
                                     indices,
-                                    message,
+                                    message=None,
                                     error=ValueError):
     with self.assertRaisesRegexp(error, message):
       ragged.gather_nd(params, indices)
diff --git a/tensorflow/python/ops/ragged/ragged_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
index d371745b73..42efdc8a7d 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -25,90 +26,75 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
 
-  @test_util.run_v1_only('b/120545219')
   def testDocStringExamples(self):
     params = constant_op.constant(['a', 'b', 'c', 'd', 'e'])
     indices = constant_op.constant([3, 1, 2, 1, 0])
     ragged_params = ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
     ragged_indices = ragged.constant([[3, 1, 2], [1], [], [0]])
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, ragged_indices).eval().tolist(),
-          [[b'd', b'b', b'c'], [b'b'], [], [b'a']])
-      self.assertEqual(
-          ragged.gather(ragged_params, indices).eval().tolist(),
-          [[b'e'], [b'd'], [], [b'd'], [b'a', b'b', b'c']])
-      self.assertEqual(
-          ragged.gather(ragged_params, ragged_indices).eval().tolist(),
-          [[[b'e'], [b'd'], []], [[b'd']], [], [[b'a', b'b', b'c']]])
-
-  @test_util.run_deprecated_v1
+    self.assertRaggedEqual(
+        ragged.gather(params, ragged_indices),
+        [[b'd', b'b', b'c'], [b'b'], [], [b'a']])
+    self.assertRaggedEqual(
+        ragged.gather(ragged_params, indices),
+        [[b'e'], [b'd'], [], [b'd'], [b'a', b'b', b'c']])
+    self.assertRaggedEqual(
+        ragged.gather(ragged_params, ragged_indices),
+        [[[b'e'], [b'd'], []], [[b'd']], [], [[b'a', b'b', b'c']]])
+
   def testTensorParamsAndTensorIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = [2, 0, 2, 1]
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(),
-          [b'c', b'a', b'c', b'b'])
-      self.assertEqual(type(ragged.gather(params, indices)), ops.Tensor)
+    self.assertRaggedEqual(
+        ragged.gather(params, indices), [b'c', b'a', b'c', b'b'])
+    self.assertIsInstance(ragged.gather(params, indices), ops.Tensor)
 
-  @test_util.run_deprecated_v1
   def testRaggedParamsAndTensorIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = [2, 0, 2, 1]
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(),
-          [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']])
+    self.assertRaggedEqual(
+        ragged.gather(params, indices),
+        [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']])
 
-  @test_util.run_v1_only('b/120545219')
   def testTensorParamsAndRaggedIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = ragged.constant([[2, 1], [1, 2, 0], [3]])
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(),
-          [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']])
+    self.assertRaggedEqual(
+        ragged.gather(params, indices),
+        [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']])
 
-  @test_util.run_v1_only('b/120545219')
   def testRaggedParamsAndRaggedIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = ragged.constant([[2, 1], [1, 2, 0], [3]])
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(),
-          [[[b'f'], [b'c', b'd', b'e']],                # [[p[2], p[1]      ],
-           [[b'c', b'd', b'e'], [b'f'], [b'a', b'b']],  #  [p[1], p[2], p[0]],
-           [[]]]                                        #  [p[3]            ]]
-      )  # pyformat: disable
-
-  @test_util.run_deprecated_v1
+    self.assertRaggedEqual(
+        ragged.gather(params, indices),
+        [[[b'f'], [b'c', b'd', b'e']],                # [[p[2], p[1]      ],
+         [[b'c', b'd', b'e'], [b'f'], [b'a', b'b']],  #  [p[1], p[2], p[0]],
+         [[]]]                                        #  [p[3]            ]]
+    )  # pyformat: disable
+
   def testRaggedParamsAndScalarIndices(self):
     params = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
     indices = 1
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(), [b'c', b'd', b'e'])
+    self.assertRaggedEqual(ragged.gather(params, indices), [b'c', b'd', b'e'])
 
-  @test_util.run_deprecated_v1
   def test3DRaggedParamsAnd2DTensorIndices(self):
     params = ragged.constant([[['a', 'b'], []], [['c', 'd'], ['e'], ['f']],
                               [['g']]])
     indices = [[1, 2], [0, 1], [2, 2]]
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(),
-          [[[[b'c', b'd'], [b'e'], [b'f']], [[b'g']]],            # [[p1, p2],
-           [[[b'a', b'b'], []], [[b'c', b'd'], [b'e'], [b'f']]],  #  [p0, p1],
-           [[[b'g']], [[b'g']]]]                                  #  [p2, p2]]
-      )  # pyformat: disable
-
-  @test_util.run_v1_only('b/120545219')
+    self.assertRaggedEqual(
+        ragged.gather(params, indices),
+        [[[[b'c', b'd'], [b'e'], [b'f']], [[b'g']]],            # [[p1, p2],
+         [[[b'a', b'b'], []], [[b'c', b'd'], [b'e'], [b'f']]],  #  [p0, p1],
+         [[[b'g']], [[b'g']]]]                                  #  [p2, p2]]
+    )  # pyformat: disable
+
   def testTensorParamsAnd4DRaggedIndices(self):
     indices = ragged.constant(
         [[[[3, 4], [0, 6]], []], [[[2, 1], [1, 0]], [[2, 5]], [[2, 3]]],
@@ -116,32 +102,30 @@ class RaggedTensorOpsTest(test_util.TensorFlowTestCase):
         ragged_rank=2,
         inner_shape=(2,))
     params = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
-    with self.test_session():
-      self.assertEqual(
-          ragged.gather(params, indices).eval().tolist(),
-          [[[[b'd', b'e'], [b'a', b'g']], []],
-           [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]],
-           [[[b'b', b'a']]]])  # pyformat: disable
-
-  @test_util.run_v1_only('b/120545219')
+    self.assertRaggedEqual(
+        ragged.gather(params, indices),
+        [[[[b'd', b'e'], [b'a', b'g']], []],
+         [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]],
+         [[[b'b', b'a']]]])  # pyformat: disable
+
   def testOutOfBoundsError(self):
     tensor_params = ['a', 'b', 'c']
     tensor_indices = [0, 1, 2]
     ragged_params = ragged.constant([['a', 'b'], ['c']])
     ragged_indices = ragged.constant([[0, 3]])
-    with self.test_session():
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'indices\[1\] = 3 is not in \[0, 3\)',
-                              ragged.gather(tensor_params, ragged_indices).eval)
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'indices\[2\] = 2 is not in \[0, 2\)',
-                              ragged.gather(ragged_params, tensor_indices).eval)
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'indices\[1\] = 3 is not in \[0, 2\)',
-                              ragged.gather(ragged_params, ragged_indices).eval)
-
-  @test_util.run_deprecated_v1
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'indices\[1\] = 3 is not in \[0, 3\)'):
+      self.evaluate(ragged.gather(tensor_params, ragged_indices))
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'indices\[2\] = 2 is not in \[0, 2\)'):
+      self.evaluate(ragged.gather(ragged_params, tensor_indices))
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'indices\[1\] = 3 is not in \[0, 2\)'):
+      self.evaluate(ragged.gather(ragged_params, ragged_indices))
+
   def testUnknownIndicesRankError(self):
+    if context.executing_eagerly():
+      return
     params = ragged.constant([], ragged_rank=1)
     indices = constant_op.constant([0], dtype=dtypes.int64)
     indices = array_ops.placeholder_with_default(indices, None)
diff --git a/tensorflow/python/ops/ragged/ragged_getitem.py b/tensorflow/python/ops/ragged/ragged_getitem.py
index 9821695046..0fa72a3658 100644
--- a/tensorflow/python/ops/ragged/ragged_getitem.py
+++ b/tensorflow/python/ops/ragged/ragged_getitem.py
@@ -24,7 +24,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_array_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 
@@ -137,8 +136,8 @@ def _ragged_getitem(rt_input, key_list):
   if row_key is array_ops.newaxis:
     inner_rt = _ragged_getitem(rt_input, inner_keys)
     nsplits = array_ops.shape(inner_rt.row_splits, out_type=dtypes.int64)[0]
-    return ragged_factory_ops.from_row_splits(inner_rt,
-                                              array_ops.stack([0, nsplits - 1]))
+    return ragged_tensor.RaggedTensor.from_row_splits(
+        inner_rt, array_ops.stack([0, nsplits - 1]))
 
   # Slicing a range of rows: first slice the outer dimension, and then
   # call `_ragged_getitem_inner_dimensions` to handle the inner keys.
@@ -184,7 +183,7 @@ def _slice_ragged_row_dimension(rt_input, row_key):
         axis=0)
     values_start = new_splits[0]
     values_limit = new_splits[-1]
-    return ragged_factory_ops.from_row_splits(
+    return ragged_tensor.RaggedTensor.from_row_splits(
         rt_input.values[values_start:values_limit], new_splits - values_start)
 
   # If there is a slice step (aka a strided slice), then use ragged_gather to
@@ -225,7 +224,8 @@ def _ragged_getitem_inner_dimensions(rt_input, key_list):
   if column_key is array_ops.newaxis:
     inner_rt = _ragged_getitem_inner_dimensions(rt_input, key_list[1:])
     nsplits = array_ops.shape(inner_rt.row_splits, out_type=dtypes.int64)[0]
-    return ragged_factory_ops.from_row_splits(inner_rt, math_ops.range(nsplits))
+    return ragged_tensor.RaggedTensor.from_row_splits(inner_rt,
+                                                      math_ops.range(nsplits))
 
   # Slicing a range of columns in a ragged inner dimension.  We use a
   # recursive call to process the values, and then assemble a RaggedTensor
@@ -239,7 +239,7 @@ def _ragged_getitem_inner_dimensions(rt_input, key_list):
     else:
       # Nontrivial slice: use ragged_gather to extract the indicated slice as
       # a new RaggedTensor (inner_rt), and then recursively process its values.
-      # The splits can be taken from ragged.row_splits(inner_rt).
+      # The splits can be taken from inner_rt.row_splits().
       inner_rt_starts = rt_input.row_splits[:-1]
       inner_rt_limits = rt_input.row_splits[1:]
       if column_key.start is not None and column_key.start != 0:
diff --git a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py b/tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py
similarity index 79%
rename from tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
rename to tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py
index dc214f4437..8b28cac99d 100644
--- a/tensorflow/python/ops/ragged/ragged_map_inner_values_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py
@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged.map_inner_values."""
+"""Tests for ragged.map_flat_values."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from absl.testing import parameterized
-
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -27,11 +25,12 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
-                                 parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedMapInnerValuesOpTest(ragged_test_util.RaggedTensorTestCase):
 
   def assertRaggedMapInnerValuesReturns(self,
                                         op,
@@ -39,23 +38,21 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
                                         args=(),
                                         kwargs=None):
     kwargs = kwargs or {}
-    result = ragged.map_inner_values(op, *args, **kwargs)
+    result = ragged.map_flat_values(op, *args, **kwargs)
     with self.test_session():
-      self.assertEqual(result.eval().tolist(), expected)
+      self.assertRaggedEqual(result, expected)
 
-  @test_util.run_v1_only('b/120545219')
   def testDocStringExamples(self):
     """Test the examples in apply_op_to_ragged_values.__doc__."""
     rt = ragged.constant([[1, 2, 3], [], [4, 5], [6]])
-    v1 = ragged.map_inner_values(array_ops.ones_like, rt)
-    v2 = ragged.map_inner_values(math_ops.multiply, rt, rt)
-    v3 = ragged.map_inner_values(math_ops.add, rt, 5)
+    v1 = ragged.map_flat_values(array_ops.ones_like, rt)
+    v2 = ragged.map_flat_values(math_ops.multiply, rt, rt)
+    v3 = ragged.map_flat_values(math_ops.add, rt, 5)
     with self.test_session():
-      self.assertEqual(v1.eval().tolist(), [[1, 1, 1], [], [1, 1], [1]])
-      self.assertEqual(v2.eval().tolist(), [[1, 4, 9], [], [16, 25], [36]])
-      self.assertEqual(v3.eval().tolist(), [[6, 7, 8], [], [9, 10], [11]])
+      self.assertRaggedEqual(v1, [[1, 1, 1], [], [1, 1], [1]])
+      self.assertRaggedEqual(v2, [[1, 4, 9], [], [16, 25], [36]])
+      self.assertRaggedEqual(v3, [[6, 7, 8], [], [9, 10], [11]])
 
-  @test_util.run_deprecated_v1
   def testOpWithSingleRaggedTensorArg(self):
     tensor = ragged.constant([[1, 2, 3], [], [4, 5]])
     self.assertRaggedMapInnerValuesReturns(
@@ -63,20 +60,17 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(tensor,),
         expected=[[0, 0, 0], [], [0, 0]])
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithTwoRaggedTensorArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
     self.assertRaggedMapInnerValuesReturns(
         op=math_ops.multiply, args=(x, y), expected=[[3, 2, 12], [], [4, 25]])
 
-  @test_util.run_deprecated_v1
   def testOpWithRaggedTensorAndScalarArgs(self):
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
     self.assertRaggedMapInnerValuesReturns(
         op=math_ops.multiply, args=(5, y), expected=[[5, 10, 15], [], [20, 25]])
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithThreeRaggedTensorArgs(self):
     condition = ragged.constant(
         [[True, True, False], [], [True, False]])  # pyformat: disable
@@ -87,7 +81,6 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(condition, x, y),
         expected=[[b'a', b'b', b'C'], [], [b'd', b'E']])
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithRaggedTensorListArg(self):
     x = ragged.constant([[1, 2, 3], [], [4, 5]])
     y = ragged.constant([[10, 20, 30], [], [40, 50]])
@@ -96,7 +89,6 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=([x, y, x],),
         expected=[[12, 24, 36], [], [48, 60]])
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithKeywordArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -105,7 +97,6 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         kwargs=dict(x=x, y=y),
         expected=[[3, 2, 12], [], [4, 25]])
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithMixedPositionalAndKeywordArgs(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1, 2, 3], [], [4, 5]])
@@ -115,7 +106,6 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         kwargs=dict(y=y),
         expected=[[3, 2, 12], [], [4, 25]])
 
-  @test_util.run_deprecated_v1
   def testNonElementWiseOp(self):
     x = ragged.constant(
         [[[3, 1, 4], [1, 5, 9], [2, 6, 5]], [], [[3, 5, 8], [9, 7, 9]]],
@@ -128,15 +118,13 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         },
         expected=[[8, 15, 13], [], [16, 25]])
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithRaggedRankGreaterThanOne(self):
     # ragged_rank=0
     x0 = [3, 1, 4, 1, 5, 9, 2, 6, 5]
     y0 = [1, 2, 3, 4, 5, 6, 7, 8, 9]
     with self.test_session():
-      self.assertEqual(
-          math_ops.multiply(x0, y0).eval().tolist(),
-          [3, 2, 12, 4, 25, 54, 14, 48, 45])
+      self.assertRaggedEqual(
+          math_ops.multiply(x0, y0), [3, 2, 12, 4, 25, 54, 14, 48, 45])
 
     # ragged_rank=1
     x1 = ragged.constant([[3, 1, 4], [], [1, 5], [9, 2], [6, 5]])
@@ -173,7 +161,6 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
             [[[54, 14], [48, 45]]]    # row 3
         ])  # pyformat: disable
 
-  @test_util.run_v1_only('b/120545219')
   def testOpWithRaggedRankThree(self):
     x = ragged.constant([[[3, 1, 4]], [], [[], [1, 5]]])
     y = ragged.constant([[[1, 2, 3]], [], [[], [4, 5]]])
@@ -182,7 +169,6 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
         args=(x, y),
         expected=[[[3, 2, 12]], [], [[], [4, 25]]])
 
-  @test_util.run_deprecated_v1
   def testOpWithInnerValuesOnly(self):
     x = constant_op.constant([[1, 2], [3, 4], [5, 6]])
     y = constant_op.constant(2)
@@ -194,29 +180,25 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase,
     y = ragged.constant([[[3, 1, 4], []], [], [[1, 5]]])
     self.assertRaisesRegexp(ValueError,
                             r'Inputs must have identical ragged splits.*',
-                            ragged.map_inner_values, math_ops.add, x, y)
+                            ragged.map_flat_values, math_ops.add, x, y)
 
   def testRaggedTensorSplitsValueMismatchError(self):
     x = ragged.constant([[3, 1, 4], [], [1, 5]])
     y = ragged.constant([[1], [2, 3], [4, 5]])
     self.assertRaisesRegexp(errors.InvalidArgumentError,
                             r'Inputs must have identical ragged splits.*',
-                            ragged.map_inner_values, math_ops.add, x, y)
+                            ragged.map_flat_values, math_ops.add, x, y)
 
-  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorSplitsMismatchErrorAtRuntime(self):
     splits1 = array_ops.placeholder_with_default(
         constant_op.constant([0, 3, 3, 5], dtypes.int64), None)
     splits2 = array_ops.placeholder_with_default(
         constant_op.constant([0, 1, 3, 5], dtypes.int64), None)
-    x = ragged.from_row_splits([3, 1, 4, 1, 5], splits1)
-    y = ragged.from_row_splits([1, 2, 3, 4, 5], splits2)
-    result = ragged.map_inner_values(math_ops.add, x, y)
-    with self.test_session():
-      self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          r'\[Inputs must have identical ragged splits\] '
-          r'\[Condition x == y did not hold element-wise:\].*', result.eval)
+    x = ragged.RaggedTensor.from_row_splits([3, 1, 4, 1, 5], splits1)
+    y = ragged.RaggedTensor.from_row_splits([1, 2, 3, 4, 5], splits2)
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'.*Inputs must have identical ragged splits'):
+      self.evaluate(ragged.map_flat_values(math_ops.add, x, y))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
index ab70d5a123..49c0996b24 100644
--- a/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_map_fn_op_test.py
@@ -27,11 +27,14 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops as mo
 from tensorflow.python.ops import ragged
 from tensorflow.python.ops import string_ops
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-@test_util.run_v1_only('b/120545219')
-class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedMapOpTest(ragged_test_util.RaggedTensorTestCase,
+                      parameterized.TestCase):
+
   @parameterized.parameters([
       # The following test sets map over a RaggedTensor and apply a
       # transformation that returns with shape:
@@ -54,6 +57,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           elems=[[1, 2, 3], [4, 5], [6, 7]],
           expected_output=[[2, 6], [4.5, 9], [6.5, 13]],
           dtype=dtypes.float32,
+          expected_ragged_rank=0,
       ),
       # [d1, (d2)] -> [d1, (d2)]
       dict(
@@ -61,8 +65,8 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           elems=[[1, 2, 3], [4, 5], [6, 7]],
           expected_output=[[2, 3, 4], [5, 6], [7, 8]],
           dtype=dtypes.int64,
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=1),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=1),
       ),
       # [d1, (d2), d3] -> [d1, (d2), d3]
       dict(
@@ -70,41 +74,41 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           elems=[[[1, 2], [3, 4]], [], [[5, 6], [7, 8], [9, 0]]],
           elems_ragged_rank=1,
           expected_ragged_rank=1,
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=1),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=1),
           expected_output=[[[2, 3], [4, 5]], [], [[6, 7], [8, 9], [10, 1]]],
       ),
       # [d1, (d2)] -> [d1, (d2), (d3)]
       dict(
-          fn=lambda x: ragged.from_row_starts(x, [0]),
+          fn=lambda x: ragged.RaggedTensor.from_row_starts(x, [0]),
           elems=[[1, 2, 3], [4, 5], [6, 7]],
           expected_output=[[[1, 2, 3]], [[4, 5]], [[6, 7]]],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=2),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=2),
       ),
       # [d1, (d2), (d3)] -> [d1, (d2), (d3)]
       dict(
-          fn=lambda x: ragged.map_inner_values(mo.add, x, 1),
+          fn=lambda x: ragged.map_flat_values(mo.add, x, 1),
           elems=[[[1, 2, 3]], [[4, 5], [6, 7]]],
           expected_output=[[[2, 3, 4]], [[5, 6], [7, 8]]],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=2),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=2),
       ),
       # [d1, (d2), (d3)] -> [d1, (d2)]
       dict(
           fn=lambda x: ragged.reduce_sum(x, axis=1),
           elems=[[[1, 2, 3]], [[4, 5], [6, 7]]],
           expected_output=[[6], [9, 13]],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=1),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=1),
       ),
       # [d1, (d2), (d3)] -> [d1, (d3)]
       dict(
           fn=lambda x: ragged.reduce_sum(x, axis=0),
           elems=[[[1, 2, 3]], [[4, 5], [6, 7]]],
           expected_output=[[1, 2, 3], [10, 12]],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=1),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=1),
       ),
       # [d1, (d2), (d3)] -> [d1]
       dict(
@@ -118,27 +122,26 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           fn=mo.range,
           elems=[4, 0, 2],
           expected_output=[[0, 1, 2, 3], [], [0, 1]],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=1),
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=1),
       ),
       # [d1] -> [d1, (d2), (d3)]
       dict(
           fn=lambda x: ragged.range(mo.range(x)),
           elems=[5, 0, 3],
-          expected_output=[
-              [[], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3]], [], [[], [0], [0, 1]]
-          ],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=2),
+          expected_output=[[[], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3]], [],
+                           [[], [0], [0, 1]]],
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=2),
       ),
       # [d1, (d2), (d3), (d4a), (d5)] ->  [d1, (d2), (d3), (d4b), (d5)]
       dict(
           fn=lambda x: x + np.int64(1),
           elems=[[[[[1, 2, 3]], [[4], [5]]]], [[[[6, 7]]], [[[8], []]]]],
-          expected_output=[[[[[2, 3, 4]], [[5], [6]]]],
-                           [[[[7, 8]]], [[[9], []]]]],
-          result_dtype=ragged.RaggedTensorType(dtype=dtypes.int64,
-                                               ragged_rank=4),
+          expected_output=[[[[[2, 3, 4]], [[5], [6]]]], [[[[7, 8]]], [[[9],
+                                                                       []]]]],
+          result_dtype=ragged.RaggedTensorType(
+              dtype=dtypes.int64, ragged_rank=4),
       ),
   ])
 
@@ -160,16 +163,12 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     expected_rt = ragged.constant(
         expected_output, ragged_rank=expected_ragged_rank)
-    with self.test_session():
-      if ragged.is_ragged(expected_output):
-        self.assertEqual(output.ragged_rank, expected_rt.ragged_rank)
-      output_values = self.evaluate(output)
-      self.assertAllEqual(expected_output, output_values.tolist())
+    self.assertRaggedEqual(expected_rt, output)
 
   def testRaggedMapOnStructure(self):
     batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]])
     # [[10, 20, 30], [40], [50, 60, 70]]
-    robin = ragged.map_inner_values(mo.multiply, batman, 10)
+    robin = ragged.map_flat_values(mo.multiply, batman, 10)
 
     features = {'batman': batman, 'robin': robin}
 
@@ -182,14 +181,13 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         dtype=dtypes.int32,
     )
 
-    with self.test_session():
-      self.assertAllEqual(output.eval().tolist(), [66, 44, 198])
+    self.assertRaggedEqual(output, [66, 44, 198])
 
   # Test mapping over a dict of RTs can produce a dict of RTs.
   def testRaggedMapOnStructure_RaggedOutputs(self):
     batman = ragged.constant([[1, 2, 3], [4], [5, 6, 7]])
     # [[10, 20, 30], [40], [50, 60, 70]]
-    robin = ragged.map_inner_values(mo.multiply, batman, 10)
+    robin = ragged.map_flat_values(mo.multiply, batman, 10)
 
     features = {'batman': batman, 'robin': robin}
 
@@ -211,17 +209,13 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         },
     )
 
-    with self.test_session():
-      self.assertAllEqual(output['batman'].eval().tolist(),
-                          [[2, 3, 4], [5], [6, 7, 8]])
-      self.assertAllEqual(output['robin'].eval().tolist(),
-                          [[11, 21, 31], [41], [51, 61, 71]])
+    self.assertRaggedEqual(output['batman'], [[2, 3, 4], [5], [6, 7, 8]])
+    self.assertRaggedEqual(output['robin'], [[11, 21, 31], [41], [51, 61, 71]])
 
   def testZip(self):
     x = ragged.constant([[10, 20], [30, 40], [50, 60], [70], [80, 90, 100]],
                         dtypes.int64)
-    y = array_ops.expand_dims(
-        mo.range(ragged.nrows(x), dtype=dtypes.int64), axis=1)
+    y = array_ops.expand_dims(mo.range(x.nrows(), dtype=dtypes.int64), axis=1)
 
     def _zip(foo):
       y_val, x_val = foo
@@ -233,11 +227,9 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         dtype=ragged.RaggedTensorType(dtype=dtypes.int64, ragged_rank=1),
         infer_shape=False)
 
-    with self.test_session():
-      result = self.evaluate(output).tolist()
-      self.assertAllEqual(
-          result, [[[0, 10], [0, 20]], [[1, 30], [1, 40]], [[2, 50], [2, 60]],
-                   [[3, 70]], [[4, 80], [4, 90], [4, 100]]])
+    self.assertRaggedEqual(
+        output, [[[0, 10], [0, 20]], [[1, 30], [1, 40]], [[2, 50], [2, 60]],
+                 [[3, 70]], [[4, 80], [4, 90], [4, 100]]])
 
   def testBatchGather(self):
     tokens = ragged.constant([['hello', '.', 'there'], ['merhaba'],
@@ -255,10 +247,8 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         dtype=ragged.RaggedTensorType(dtype=dtypes.string, ragged_rank=1),
         infer_shape=False)
 
-    with self.test_session():
-      self.assertAllEqual(
-          self.evaluate(out).tolist(),
-          [[b'hello', b'there'], [b'merhaba'], [b'bonjour', b'ca va']])
+    self.assertRaggedEqual(
+        out, [[b'hello', b'there'], [b'merhaba'], [b'bonjour', b'ca va']])
 
   def testMismatchRaggedRank(self):
     elems = ragged.constant([[[1, 2, 3]], [[4, 5], [6, 7]]])
@@ -272,7 +262,7 @@ class RaggedMapOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
   def testMismatchRaggedRank2(self):
     elems = ragged.constant([[1, 2, 3], [4, 5], [6, 7]])
-    fn = lambda x: ragged.from_row_starts(x, [0])
+    fn = lambda x: ragged.RaggedTensor.from_row_starts(x, [0])
     with self.assertRaisesWithLiteralMatch(
         ValueError, r'The declared ragged rank (10) mismatches the result (1)'):
       _ = ragged.map_fn(
diff --git a/tensorflow/python/ops/ragged/ragged_map_ops.py b/tensorflow/python/ops/ragged/ragged_map_ops.py
index fafa23b8dc..af40352b1d 100644
--- a/tensorflow/python/ops/ragged/ragged_map_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_map_ops.py
@@ -27,12 +27,11 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.ops.ragged import ragged_array_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
@@ -216,8 +215,8 @@ def map_fn(fn,
         varscope_caching_device_was_none = True
 
     elems_flat = [
-        ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-            elem, name="elem") for elem in elems_flat
+        ragged_tensor.convert_to_tensor_or_ragged_tensor(elem, name="elem")
+        for elem in elems_flat
     ]
 
     # We can either infer the output, or we can assume that it will be the same
@@ -226,7 +225,7 @@ def map_fn(fn,
 
     # Find the number of iterations, n may be known statically.
     if isinstance(elems_flat[0], ragged_tensor.RaggedTensor):
-      n = ragged_array_ops.nrows(elems_flat[0], out_type=dtypes.int32)
+      n = elems_flat[0].nrows(out_type=dtypes.int32)
     else:
       static_shape = elems_flat[0].shape
       if static_shape.ndims is not None and static_shape.ndims < 1:
@@ -236,7 +235,8 @@ def map_fn(fn,
         else:
           raise ValueError(
               "elements in elems must be 1+ dimensional Tensors, not scalars")
-      n = static_shape[0].value or array_ops.shape(elems_flat[0])[0]
+      n = (tensor_shape.dimension_value(static_shape[0]) or
+           array_ops.shape(elems_flat[0])[0])
 
     # Create a flat list of TAs.
 
@@ -334,7 +334,7 @@ def map_fn(fn,
 class _RaggedTensorComponents(
     collections.namedtuple(
         "_RaggedTensorComponents",
-        ["inner_values", "nested_row_lengths", "outer_row_length"])):
+        ["flat_values", "nested_row_lengths", "outer_row_length"])):
   """A namedtuple of components which represent a `RaggedTensor`.
 
   _RaggedTensorComponents is a list of components which can be used to create a
@@ -344,7 +344,7 @@ class _RaggedTensorComponents(
 
   The following are a list of components for a `RaggedTensor`:
 
-  inner_values: The flat and inner values of a RaggedTensor. This could be
+  flat_values: The flat and inner values of a RaggedTensor. This could be
     a `Tensor`, a `TensorArray`, or a data type.
   nested_row_lengths: a tuple containing the row lengths of each rank. The
     elements of the tuple could be `Tensor`s or `TensorArray`s.
@@ -357,12 +357,12 @@ class _RaggedTensorComponents(
 
 
 def _concat_ragged_tensor_components(rt_ta):
-  inner_values = rt_ta.inner_values.concat()
+  flat_values = rt_ta.flat_values.concat()
   nested_row_lengths = tuple(
       row_lengths_ta.concat() for row_lengths_ta in rt_ta.nested_row_lengths)
   outer_row_length = rt_ta.outer_row_length.concat()
   return _RaggedTensorComponents(
-      inner_values=inner_values,
+      flat_values=flat_values,
       nested_row_lengths=nested_row_lengths,
       outer_row_length=outer_row_length)
 
@@ -374,17 +374,17 @@ def _maybe_decompose_tensor(rt):
 
   # The three component pieces we need:
   # - inner values
-  inner_values = rt.inner_values
+  flat_values = rt.flat_values
 
   # - row_splits of the RT
   splits = rt.nested_row_splits
   nested_row_lengths = tuple(split[1:] - split[:-1] for split in splits)
 
   # - outer row length
-  outer_row_length = array_ops.expand_dims(ragged_array_ops.nrows(rt), axis=0)
+  outer_row_length = array_ops.expand_dims(rt.nrows(), axis=0)
 
   return _RaggedTensorComponents(
-      inner_values=inner_values,
+      flat_values=flat_values,
       nested_row_lengths=nested_row_lengths,
       outer_row_length=outer_row_length,
   )
@@ -395,11 +395,12 @@ def _maybe_recompose_tensor(t):
   if not isinstance(t, _RaggedTensorComponents):
     return t
 
-  values = t.inner_values
+  values = t.flat_values
   nested_row_lengths = tuple(t.nested_row_lengths)
   for nested_row_length in reversed(nested_row_lengths):
-    values = ragged_factory_ops.from_row_lengths(values, nested_row_length)
-  return ragged_factory_ops.from_row_lengths(values, t.outer_row_length)
+    values = ragged_tensor.RaggedTensor.from_row_lengths(
+        values, nested_row_length)
+  return ragged_tensor.RaggedTensor.from_row_lengths(values, t.outer_row_length)
 
 
 def _maybe_decompose_dtype(d):
@@ -408,7 +409,7 @@ def _maybe_decompose_dtype(d):
     return d
 
   result = _RaggedTensorComponents(
-      inner_values=d.dtype,
+      flat_values=d.dtype,
       nested_row_lengths=tuple(dtypes.int64 for i in range(d.ragged_rank - 1)),
       outer_row_length=dtypes.int64,
   )
@@ -435,10 +436,13 @@ def _convert_declared(fn_output_flat, output_declared):
               "The declared ragged rank (%d) mismatches the result (1)" %
               declared.ragged_rank)
 
-        row_length = array_ops.expand_dims(
-            ragged_array_ops.nrows(current), axis=0)
+        if isinstance(current, ragged_tensor.RaggedTensor):
+          nrows = current.nrows()
+        else:
+          nrows = array_ops.shape(current, out_type=dtypes.int64)[0]
+        row_length = array_ops.expand_dims(nrows, axis=0)
         rt = _RaggedTensorComponents(
-            inner_values=current,
+            flat_values=current,
             nested_row_lengths=(),
             outer_row_length=row_length)
         yield rt
diff --git a/tensorflow/python/ops/ragged/ragged_math_ops.py b/tensorflow/python/ops/ragged/ragged_math_ops.py
index d661563a9f..92f82be84a 100644
--- a/tensorflow/python/ops/ragged/ragged_math_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_math_ops.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
@@ -25,7 +27,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import gen_ragged_math_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_functional_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_util
@@ -97,8 +98,8 @@ def range(starts, limits=None, deltas=1, dtype=None, name=None):
           [dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64])
 
     result = gen_ragged_math_ops.ragged_range(starts, limits, deltas, name=name)
-    return ragged_factory_ops.from_row_splits(result.rt_dense_values,
-                                              result.rt_nested_splits)
+    return ragged_tensor.RaggedTensor.from_row_splits(result.rt_dense_values,
+                                                      result.rt_nested_splits)
 
 
 def _infer_matching_dtype(tensors, dtype_hierarchy):
@@ -184,9 +185,8 @@ def _ragged_segment_aggregate(unsorted_segment_op,
 
   with ops.name_scope(name, 'RaggedSegment',
                       [data, segment_ids, num_segments]) as name:
-    data = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
-        data, name='data')
-    segment_ids = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    data = ragged_tensor.convert_to_tensor_or_ragged_tensor(data, name='data')
+    segment_ids = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         segment_ids, name='segment_ids')
 
     if ragged_tensor.is_ragged(segment_ids):
@@ -236,7 +236,8 @@ def _ragged_segment_aggregate(unsorted_segment_op,
     output_values = _ragged_segment_aggregate(unsorted_segment_op, data.values,
                                               data_val_to_out_val_index,
                                               output_splits[-1])
-    return ragged_factory_ops.from_row_splits(output_values, output_splits)
+    return ragged_tensor.RaggedTensor.from_row_splits(output_values,
+                                                      output_splits)
 
 
 def segment_sum(data, segment_ids, num_segments, name=None):
@@ -272,11 +273,11 @@ def segment_mean(data, segment_ids, num_segments, name=None):
   with ops.name_scope(name, 'RaggedSegmentMean',
                       [data, segment_ids, num_segments]):
     total = segment_sum(data, segment_ids, num_segments)
-    ones = ragged_factory_ops.from_nested_row_splits(
-        array_ops.ones_like(data.inner_values), data.nested_row_splits)
+    ones = ragged_tensor.RaggedTensor.from_nested_row_splits(
+        array_ops.ones_like(data.flat_values), data.nested_row_splits)
     count = segment_sum(ones, segment_ids, num_segments)
-    return ragged_factory_ops.from_nested_row_splits(
-        total.inner_values / count.inner_values, total.nested_row_splits)
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
+        total.flat_values / count.flat_values, total.nested_row_splits)
 
 
 def segment_sqrt_n(data, segment_ids, num_segments, name=None):
@@ -284,11 +285,11 @@ def segment_sqrt_n(data, segment_ids, num_segments, name=None):
   with ops.name_scope(name, 'RaggedSegmentSqrtN',
                       [data, segment_ids, num_segments]):
     total = segment_sum(data, segment_ids, num_segments)
-    ones = ragged_factory_ops.from_nested_row_splits(
-        array_ops.ones_like(data.inner_values), data.nested_row_splits)
+    ones = ragged_tensor.RaggedTensor.from_nested_row_splits(
+        array_ops.ones_like(data.flat_values), data.nested_row_splits)
     count = segment_sum(ones, segment_ids, num_segments)
-    return ragged_factory_ops.from_nested_row_splits(
-        total.inner_values / math_ops.sqrt(count.inner_values),
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
+        total.flat_values / math_ops.sqrt(count.flat_values),
         total.nested_row_splits)
 
 
@@ -440,10 +441,12 @@ def _ragged_reduce_aggregate(reduce_op,
     axis = tensor_util.constant_value(axis)
     if axis is None:
       raise ValueError('axis must be known at graph construction time.')
+    if isinstance(axis, np.ndarray):
+      axis = axis.tolist()
 
   # When reducing all axes, just ignore splits & reduce the inner values.
   if axis is None:
-    return reduce_op(rt_input.inner_values, None, name=name)
+    return reduce_op(rt_input.flat_values, None, name=name)
 
   with ops.name_scope(name, 'RaggedReduce', [rt_input, axis]):
     if isinstance(axis, (tuple, list)):
@@ -464,7 +467,7 @@ def _ragged_reduce_aggregate(reduce_op,
 
     axis = ragged_util.get_positive_axis(axis, rt_input.shape.ndims)
 
-    rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(
+    rt_input = ragged_tensor.convert_to_tensor_or_ragged_tensor(
         rt_input, name='rt_input')
 
     if axis == 0:
@@ -522,22 +525,22 @@ def reduce_mean(input_tensor, axis=None, keepdims=None, name=None):
   with ops.name_scope(name, 'RaggedReduceMean', [input_tensor, axis]):
     total = reduce_sum(input_tensor, axis, keepdims)
     if ragged_tensor.is_ragged(input_tensor):
-      ones = ragged_factory_ops.from_nested_row_splits(
-          array_ops.ones_like(input_tensor.inner_values),
+      ones = ragged_tensor.RaggedTensor.from_nested_row_splits(
+          array_ops.ones_like(input_tensor.flat_values),
           input_tensor.nested_row_splits)
     else:
       ones = array_ops.ones_like(input_tensor)
     count = reduce_sum(ones, axis, keepdims)
     if ragged_tensor.is_ragged(total):
-      return ragged_factory_ops.from_nested_row_splits(
-          total.inner_values / count.inner_values, total.nested_row_splits)
+      return ragged_tensor.RaggedTensor.from_nested_row_splits(
+          total.flat_values / count.flat_values, total.nested_row_splits)
     else:
       return total / count
 
 
 def _cast(input_tensor, dtype):
-  return ragged_functional_ops.map_inner_values(math_ops.cast, input_tensor,
-                                                dtype)
+  return ragged_functional_ops.map_flat_values(math_ops.cast, input_tensor,
+                                               dtype)
 
 
 def reduce_all(input_tensor, axis=None, keepdims=None, name=None):
diff --git a/tensorflow/python/ops/ragged/ragged_operators_test.py b/tensorflow/python/ops/ragged/ragged_operators_test.py
index c1223db307..78bb37c341 100644
--- a/tensorflow/python/ops/ragged/ragged_operators_test.py
+++ b/tensorflow/python/ops/ragged/ragged_operators_test.py
@@ -20,79 +20,71 @@ from __future__ import print_function
 
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-@test_util.run_v1_only('b/120545219')
-class RaggedElementwiseOpsTest(test_util.TensorFlowTestCase):
-  # @TODO(edloper): Test right-handed versions of operators once we add
-  # broadcasting support for elementwise ops.
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedElementwiseOpsTest(ragged_test_util.RaggedTensorTestCase):
 
   def testOrderingOperators(self):
     x = ragged.constant([[1, 5], [3]])
     y = ragged.constant([[4, 5], [1]])
-    with self.test_session():
-      self.assertEqual((x > y).eval().tolist(), [[False, False], [True]])
-      self.assertEqual((x >= y).eval().tolist(), [[False, True], [True]])
-      self.assertEqual((x < y).eval().tolist(), [[True, False], [False]])
-      self.assertEqual((x <= y).eval().tolist(), [[True, True], [False]])
-
-  def assertEqual(self, a, b):
-    if a != b:
-      print('%30s %s' % (b, a))
+    self.assertRaggedEqual((x > y), [[False, False], [True]])
+    self.assertRaggedEqual((x >= y), [[False, True], [True]])
+    self.assertRaggedEqual((x < y), [[True, False], [False]])
+    self.assertRaggedEqual((x <= y), [[True, True], [False]])
 
   def testArithmeticOperators(self):
     x = ragged.constant([[1.0, -2.0], [8.0]])
     y = ragged.constant([[4.0, 4.0], [2.0]])
-    with self.test_session():
-      self.assertEqual(abs(x).eval().tolist(), [[1.0, 2.0], [8.0]])
+    self.assertRaggedEqual(abs(x), [[1.0, 2.0], [8.0]])
 
-      self.assertEqual((-x).eval().tolist(), [[-1.0, 2.0], [-8.0]])
+    self.assertRaggedEqual((-x), [[-1.0, 2.0], [-8.0]])
 
-      self.assertEqual((x + y).eval().tolist(), [[5.0, 2.0], [10.0]])
-      self.assertEqual((3.0 + y).eval().tolist(), [[7.0, 7.0], [5.0]])
-      self.assertEqual((x + 3.0).eval().tolist(), [[4.0, 1.0], [11.0]])
+    self.assertRaggedEqual((x + y), [[5.0, 2.0], [10.0]])
+    self.assertRaggedEqual((3.0 + y), [[7.0, 7.0], [5.0]])
+    self.assertRaggedEqual((x + 3.0), [[4.0, 1.0], [11.0]])
 
-      self.assertEqual((x - y).eval().tolist(), [[-3.0, -6.0], [6.0]])
-      self.assertEqual((3.0 - y).eval().tolist(), [[-1.0, -1.0], [1.0]])
-      self.assertEqual((x + 3.0).eval().tolist(), [[4.0, 1.0], [11.0]])
+    self.assertRaggedEqual((x - y), [[-3.0, -6.0], [6.0]])
+    self.assertRaggedEqual((3.0 - y), [[-1.0, -1.0], [1.0]])
+    self.assertRaggedEqual((x + 3.0), [[4.0, 1.0], [11.0]])
 
-      self.assertEqual((x * y).eval().tolist(), [[4.0, -8.0], [16.0]])
-      self.assertEqual((3.0 * y).eval().tolist(), [[12.0, 12.0], [6.0]])
-      self.assertEqual((x * 3.0).eval().tolist(), [[3.0, -6.0], [24.0]])
+    self.assertRaggedEqual((x * y), [[4.0, -8.0], [16.0]])
+    self.assertRaggedEqual((3.0 * y), [[12.0, 12.0], [6.0]])
+    self.assertRaggedEqual((x * 3.0), [[3.0, -6.0], [24.0]])
 
-      self.assertEqual((x / y).eval().tolist(), [[0.25, -0.5], [4.0]])
-      self.assertEqual((y / x).eval().tolist(), [[4.0, -2.0], [0.25]])
-      self.assertEqual((2.0 / y).eval().tolist(), [[0.5, 0.5], [1.0]])
-      self.assertEqual((x / 2.0).eval().tolist(), [[0.5, -1.0], [4.0]])
+    self.assertRaggedEqual((x / y), [[0.25, -0.5], [4.0]])
+    self.assertRaggedEqual((y / x), [[4.0, -2.0], [0.25]])
+    self.assertRaggedEqual((2.0 / y), [[0.5, 0.5], [1.0]])
+    self.assertRaggedEqual((x / 2.0), [[0.5, -1.0], [4.0]])
 
-      self.assertEqual((x // y).eval().tolist(), [[0.0, -1.0], [4.0]])
-      self.assertEqual((y // x).eval().tolist(), [[4.0, -2.0], [0.0]])
-      self.assertEqual((2.0 // y).eval().tolist(), [[0.0, 0.0], [1.0]])
-      self.assertEqual((x // 2.0).eval().tolist(), [[0.0, -1.0], [4.0]])
+    self.assertRaggedEqual((x // y), [[0.0, -1.0], [4.0]])
+    self.assertRaggedEqual((y // x), [[4.0, -2.0], [0.0]])
+    self.assertRaggedEqual((2.0 // y), [[0.0, 0.0], [1.0]])
+    self.assertRaggedEqual((x // 2.0), [[0.0, -1.0], [4.0]])
 
-      self.assertEqual((x % y).eval().tolist(), [[1.0, 2.0], [0.0]])
-      self.assertEqual((y % x).eval().tolist(), [[0.0, -0.0], [2.0]])
-      self.assertEqual((2.0 % y).eval().tolist(), [[2.0, 2.0], [0.0]])
-      self.assertEqual((x % 2.0).eval().tolist(), [[1.0, 0.0], [0.0]])
+    self.assertRaggedEqual((x % y), [[1.0, 2.0], [0.0]])
+    self.assertRaggedEqual((y % x), [[0.0, -0.0], [2.0]])
+    self.assertRaggedEqual((2.0 % y), [[2.0, 2.0], [0.0]])
+    self.assertRaggedEqual((x % 2.0), [[1.0, 0.0], [0.0]])
 
   def testLogicalOperators(self):
     a = ragged.constant([[True, True], [False]])
     b = ragged.constant([[True, False], [False]])
-    with self.test_session():
-      self.assertEqual((~a).eval().tolist(), [[False, False], [True]])
+    self.assertRaggedEqual((~a), [[False, False], [True]])
 
-      self.assertEqual((a & b).eval().tolist(), [[True, False], [False]])
-      self.assertEqual((a & True).eval().tolist(), [[True, True], [False]])
-      self.assertEqual((True & b).eval().tolist(), [[True, False], [False]])
+    self.assertRaggedEqual((a & b), [[True, False], [False]])
+    self.assertRaggedEqual((a & True), [[True, True], [False]])
+    self.assertRaggedEqual((True & b), [[True, False], [False]])
 
-      self.assertEqual((a | b).eval().tolist(), [[True, True], [False]])
-      self.assertEqual((a | False).eval().tolist(), [[True, True], [False]])
-      self.assertEqual((False | b).eval().tolist(), [[True, False], [False]])
+    self.assertRaggedEqual((a | b), [[True, True], [False]])
+    self.assertRaggedEqual((a | False), [[True, True], [False]])
+    self.assertRaggedEqual((False | b), [[True, False], [False]])
 
-      self.assertEqual((a ^ b).eval().tolist(), [[False, True], [False]])
-      self.assertEqual((a ^ True).eval().tolist(), [[False, False], [True]])
-      self.assertEqual((True ^ b).eval().tolist(), [[False, True], [True]])
+    self.assertRaggedEqual((a ^ b), [[False, True], [False]])
+    self.assertRaggedEqual((a ^ True), [[False, False], [True]])
+    self.assertRaggedEqual((True ^ b), [[False, True], [True]])
 
   def testDummyOperators(self):
     a = ragged.constant([[True, True], [False]])
diff --git a/tensorflow/python/ops/ragged/ragged_range_op_test.py b/tensorflow/python/ops/ragged/ragged_range_op_test.py
index 644423ecb7..5ab3d4abc3 100644
--- a/tensorflow/python/ops/ragged/ragged_range_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_range_op_test.py
@@ -21,111 +21,102 @@ from __future__ import print_function
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedRangeOpTest(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedRangeOpTest(ragged_test_util.RaggedTensorTestCase):
 
-  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     """Examples from ragged_range.__doc__."""
-    with self.test_session():
-      rt1 = ragged.range([3, 5, 2]).eval().tolist()
-      self.assertEqual(rt1, [[0, 1, 2], [0, 1, 2, 3, 4], [0, 1]])
+    rt1 = ragged.range([3, 5, 2])
+    self.assertRaggedEqual(rt1, [[0, 1, 2], [0, 1, 2, 3, 4], [0, 1]])
 
-      rt2 = ragged.range([0, 5, 8], [3, 3, 12]).eval().tolist()
-      self.assertEqual(rt2, [[0, 1, 2], [], [8, 9, 10, 11]])
+    rt2 = ragged.range([0, 5, 8], [3, 3, 12])
+    self.assertRaggedEqual(rt2, [[0, 1, 2], [], [8, 9, 10, 11]])
 
-      rt3 = ragged.range([0, 5, 8], [3, 3, 12], 2).eval().tolist()
-      self.assertEqual(rt3, [[0, 2], [], [8, 10]])
+    rt3 = ragged.range([0, 5, 8], [3, 3, 12], 2)
+    self.assertRaggedEqual(rt3, [[0, 2], [], [8, 10]])
 
-  @test_util.run_deprecated_v1
   def testBasicRanges(self):
-    with self.test_session():
-      # Specify limits only.
-      self.assertEqual(
-          ragged.range([0, 3, 5]).eval().tolist(),
-          [list(range(0)), list(range(3)), list(range(5))])
-
-      # Specify starts and limits.
-      self.assertEqual(
-          ragged.range([0, 3, 5], [2, 3, 10]).eval().tolist(),
-          [list(range(0, 2)), list(range(3, 3)), list(range(5, 10))])
-
-      # Specify starts, limits, and deltas.
-      self.assertEqual(
-          ragged.range([0, 3, 5], [4, 4, 15], [2, 3, 4]).eval().tolist(),
-          [list(range(0, 4, 2)), list(range(3, 4, 3)),
-           list(range(5, 15, 4))])
-
-  @test_util.run_deprecated_v1
+    # Specify limits only.
+    self.assertRaggedEqual(
+        ragged.range([0, 3, 5]),
+        [list(range(0)), list(range(3)),
+         list(range(5))])
+
+    # Specify starts and limits.
+    self.assertRaggedEqual(
+        ragged.range([0, 3, 5], [2, 3, 10]),
+        [list(range(0, 2)),
+         list(range(3, 3)),
+         list(range(5, 10))])
+
+    # Specify starts, limits, and deltas.
+    self.assertRaggedEqual(
+        ragged.range([0, 3, 5], [4, 4, 15], [2, 3, 4]),
+        [list(range(0, 4, 2)),
+         list(range(3, 4, 3)),
+         list(range(5, 15, 4))])
+
   def testFloatRanges(self):
-    with self.test_session():
-      expected = [[0.0, 0.4, 0.8, 1.2, 1.6, 2.0, 2.4, 2.8, 3.2, 3.6], [3.0],
-                  [5.0, 7.2, 9.4, 11.6, 13.8]]
-      actual = ragged.range([0.0, 3.0, 5.0], [3.9, 4.0, 15.0],
-                            [0.4, 1.5, 2.2]).eval().tolist()
-      self.assertEqual(expected, [[round(v, 5) for v in row] for row in actual])
-
-  @test_util.run_deprecated_v1
+    expected = [[0.0, 0.4, 0.8, 1.2, 1.6, 2.0, 2.4, 2.8, 3.2, 3.6], [3.0],
+                [5.0, 7.2, 9.4, 11.6, 13.8]]
+    actual = ragged.range([0.0, 3.0, 5.0], [3.9, 4.0, 15.0], [0.4, 1.5, 2.2])
+    self.assertEqual(
+        expected,
+        [[round(v, 5) for v in row] for row in self.eval_to_list(actual)])
+
   def testNegativeDeltas(self):
-    with self.test_session():
-      self.assertEqual(
-          ragged.range([0, 3, 5], limits=0, deltas=-1).eval().tolist(),
-          [list(range(0, 0, -1)), list(range(3, 0, -1)),
-           list(range(5, 0, -1))])
-
-      self.assertEqual(
-          ragged.range([0, -3, 5], limits=0, deltas=[-1, 1,
-                                                     -2]).eval().tolist(),
-          [list(range(0, 0, -1)), list(range(-3, 0, 1)),
-           list(range(5, 0, -2))])
-
-  @test_util.run_deprecated_v1
+    self.assertRaggedEqual(
+        ragged.range([0, 3, 5], limits=0, deltas=-1),
+        [list(range(0, 0, -1)),
+         list(range(3, 0, -1)),
+         list(range(5, 0, -1))])
+
+    self.assertRaggedEqual(
+        ragged.range([0, -3, 5], limits=0, deltas=[-1, 1, -2]),
+        [list(range(0, 0, -1)),
+         list(range(-3, 0, 1)),
+         list(range(5, 0, -2))])
+
   def testBroadcast(self):
-    with self.test_session():
-      # Specify starts and limits, broadcast deltas.
-      self.assertEqual(
-          ragged.range([0, 3, 5], [4, 4, 15], 3).eval().tolist(),
-          [list(range(0, 4, 3)), list(range(3, 4, 3)),
-           list(range(5, 15, 3))])
-
-      # Broadcast all arguments.
-      self.assertEqual(
-          ragged.range(0, 5, 1).eval().tolist(), [list(range(0, 5, 1))])
-
-  @test_util.run_deprecated_v1
+    # Specify starts and limits, broadcast deltas.
+    self.assertRaggedEqual(
+        ragged.range([0, 3, 5], [4, 4, 15], 3),
+        [list(range(0, 4, 3)),
+         list(range(3, 4, 3)),
+         list(range(5, 15, 3))])
+
+    # Broadcast all arguments.
+    self.assertRaggedEqual(ragged.range(0, 5, 1), [list(range(0, 5, 1))])
+
   def testEmptyRanges(self):
     rt1 = ragged.range([0, 5, 3], [0, 3, 5])
     rt2 = ragged.range([0, 5, 5], [0, 3, 5], -1)
-    with self.test_session():
-      self.assertEqual(rt1.eval().tolist(), [[], [], [3, 4]])
-      self.assertEqual(rt2.eval().tolist(), [[], [5, 4], []])
+    self.assertRaggedEqual(rt1, [[], [], [3, 4]])
+    self.assertRaggedEqual(rt2, [[], [5, 4], []])
 
-  @test_util.run_deprecated_v1
   def testShapeFnErrors(self):
-    with self.test_session():
-      self.assertRaisesRegexp(ValueError, r'Shape must be at most rank 1.*',
-                              ragged.range, [[0]], 5)
-      self.assertRaisesRegexp(ValueError, r'Shape must be at most rank 1.*',
-                              ragged.range, 0, [[5]])
-      self.assertRaisesRegexp(ValueError, r'Shape must be at most rank 1.*',
-                              ragged.range, 0, 5, [[0]])
-      self.assertRaisesRegexp(ValueError, r'Dimensions must be equal.*',
-                              ragged.range, [0], [1, 2])
-
-  @test_util.run_deprecated_v1
+    self.assertRaises((ValueError, errors.InvalidArgumentError), ragged.range,
+                      [[0]], 5)
+    self.assertRaises((ValueError, errors.InvalidArgumentError), ragged.range,
+                      0, [[5]])
+    self.assertRaises((ValueError, errors.InvalidArgumentError), ragged.range,
+                      0, 5, [[0]])
+    self.assertRaises((ValueError, errors.InvalidArgumentError), ragged.range,
+                      [0], [1, 2])
+
   def testKernelErrors(self):
-    with self.test_session():
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              r'Requires delta != 0',
-                              ragged.range(0, 0, 0).eval)
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 r'Requires delta != 0'):
+      self.evaluate(ragged.range(0, 0, 0))
 
-  @test_util.run_deprecated_v1
   def testShape(self):
-    self.assertEqual(ragged.range(0, 0, 0).shape.as_list(), [1, None])
-    self.assertEqual(ragged.range([1, 2, 3]).shape.as_list(), [3, None])
-    self.assertEqual(
+    self.assertRaggedEqual(ragged.range(0, 0, 1).shape.as_list(), [1, None])
+    self.assertRaggedEqual(ragged.range([1, 2, 3]).shape.as_list(), [3, None])
+    self.assertRaggedEqual(
         ragged.range([1, 2, 3], [4, 5, 6]).shape.as_list(), [3, None])
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
index 2e4db2a423..890460221b 100644
--- a/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_reduce_op_test.py
@@ -21,11 +21,13 @@ from __future__ import print_function
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 _MAX_INT32 = dtypes.int32.max
@@ -37,7 +39,9 @@ def mean(*values):
   return 1.0 * sum(values) / len(values)
 
 
-class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedReduceOpsTest(ragged_test_util.RaggedTensorTestCase,
+                          parameterized.TestCase):
 
   @parameterized.parameters(
       #=========================================================================
@@ -300,19 +304,16 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=2,
           expected=[[mean(1, 2), mean(3, 4, 5)], [mean(6, 7), 8], [9]]),
   )
-  @test_util.run_v1_only('b/120545219')
   def testReduce(self, ragged_reduce_op, rt_input, axis, expected):
     rt_input = ragged.constant(rt_input)
     reduced = ragged_reduce_op(rt_input, axis)
-    with self.test_session():
-      self.assertEqual(reduced.eval().tolist(), expected)
+    self.assertRaggedEqual(reduced, expected)
 
   def assertEqualWithNan(self, actual, expected):
     """Like assertEqual, but NaN==NaN."""
     self.assertTrue(
         ((actual == expected) | (np.isnan(actual) & np.isnan(expected))).all())
 
-  @test_util.run_deprecated_v1
   def testMeanNan(self):
     rt_as_list = [[0, 1, 2, 3], [4], [], [5, 6], [7], [8, 9]]
     expected = (
@@ -320,24 +321,22 @@ class RaggedReduceOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
             [4, 1, 0, 2, 1, 2]))
     rt_input = ragged.constant(rt_as_list)
     reduced = ragged.reduce_mean(rt_input, axis=1)
-    with self.test_session():
-      self.assertEqualWithNan(reduced.eval(), expected)
+    self.assertEqualWithNan(self.evaluate(reduced), expected)
 
-  @test_util.run_deprecated_v1
   def testMeanWithTensorInputs(self):
     tensor = [[1.0, 2.0, 3.0], [10.0, 20.0, 30.0]]
     expected = [2.0, 20.0]
     reduced = ragged.reduce_mean(tensor, axis=1)
-    with self.test_session():
-      self.assertAllEqual(reduced.eval(), expected)
+    self.assertRaggedEqual(reduced, expected)
 
-  @test_util.run_deprecated_v1
   def testErrors(self):
     rt_input = ragged.constant([[1, 2, 3], [4, 5]])
     axis = array_ops.placeholder_with_default(constant_op.constant([0]), None)
-    self.assertRaisesRegexp(ValueError,
-                            r'axis must be known at graph construction time.',
-                            ragged.reduce_sum, rt_input, axis)
+
+    if not context.executing_eagerly():
+      self.assertRaisesRegexp(
+          ValueError, r'axis must be known at graph construction time.',
+          ragged.reduce_sum, rt_input, axis)
     self.assertRaisesRegexp(TypeError,
                             r'axis must be an int; got str.*',
                             ragged.reduce_sum, rt_input, ['x'])
diff --git a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
index 9f4aa1b136..15112d6c9c 100644
--- a/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_row_lengths_op_test.py
@@ -20,12 +20,16 @@ from __future__ import print_function
 
 from absl.testing import parameterized
 
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedRowLengthsOp(ragged_test_util.RaggedTensorTestCase,
+                         parameterized.TestCase):
 
   @parameterized.parameters([
       # Docstring Example
@@ -37,24 +41,6 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=2,
           expected=[[3, 1], [], [2, 1], [1], []]),
 
-      # 1D tensor
-      dict(
-          rt_input=[1, 2, 3, 4, 5],
-          ragged_rank=0,
-          axis=0,
-          expected=5),
-
-      # 2D Tensor (0 ragged dimensions)
-      dict(
-          rt_input=[[1, 2], [3, 4], [5, 6], [7, 8]],
-          ragged_rank=0,
-          expected=[2, 2, 2, 2]),
-      dict(
-          rt_input=[[1, 2], [3, 4], [5, 6], [7, 8]],
-          ragged_rank=0,
-          axis=0,
-          expected=4),
-
       # 2D Tensor (1 ragged dimension)
       dict(
           rt_input=[['a'], ['b', 'c', 'd'], ['e'], [], ['f']],
@@ -79,24 +65,6 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=0),
 
-      # 3D Tensor (0 ragged dimensions)
-      dict(
-          rt_input=[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]],
-          ragged_rank=0,
-          axis=0,
-          expected=2),
-      dict(
-          rt_input=[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]],
-          ragged_rank=0,
-          axis=1,
-          expected=[3, 3]),
-      dict(
-          rt_input=[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]],
-          ragged_rank=0,
-          axis=2,
-          expected=[[2, 2, 2], [2, 2, 2]],
-          expected_ragged_rank=0),
-
       # 3D Tensor (1 ragged dimension)
       dict(
           rt_input=[[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10]]],
@@ -143,7 +111,6 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
           expected=[[2, 3, 0], [4, 1]],
           expected_ragged_rank=1),
   ])  # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testRowLengths(self,
                      rt_input,
                      expected,
@@ -151,34 +118,28 @@ class RaggedRowLengthsOp(test_util.TensorFlowTestCase, parameterized.TestCase):
                      ragged_rank=None,
                      expected_ragged_rank=None):
     rt = ragged.constant(rt_input, ragged_rank=ragged_rank)
-    lengths = ragged.row_lengths(rt, axis)
-    with self.test_session():
-      self.assertEqual(lengths.eval().tolist(), expected)
-      if expected_ragged_rank is not None:
-        if isinstance(lengths, ragged.RaggedTensor):
-          self.assertEqual(lengths.ragged_rank, expected_ragged_rank)
-        else:
-          self.assertEqual(0, expected_ragged_rank)
+    lengths = rt.row_lengths(axis)
+    self.assertRaggedEqual(lengths, expected)
+    if expected_ragged_rank is not None:
+      if isinstance(lengths, ragged.RaggedTensor):
+        self.assertEqual(lengths.ragged_rank, expected_ragged_rank)
+      else:
+        self.assertEqual(0, expected_ragged_rank)
 
   @parameterized.parameters([
-      dict(
-          rt_input=10,
-          exception=ValueError,
-          message='rt_input may not be a scalar.'),
-      dict(
-          rt_input=[10, 20],
-          axis=1,
-          exception=ValueError,
-          message='axis=1 out of bounds: expected -1<=axis<1.'),
-      dict(
+      dict(  # axis=2 out of bounds: expected -2<=axis<2.
+          rt_input=[[10, 20], [30]],
+          axis=2,
+          exception=(ValueError, errors.InvalidArgumentError)),
+      dict(  # axis=-3 out of bounds: expected -2<=axis<2.
           rt_input=[[2, 3, 0], [4, 1, 2]],
           axis=-3,
-          exception=ValueError,
-          message='axis=-3 out of bounds: expected -2<=axis<2.'),
+          exception=(ValueError, errors.InvalidArgumentError)),
   ])
-  def testErrors(self, rt_input, exception, message, axis=1):
+  def testErrors(self, rt_input, exception, message=None, axis=1):
+    rt = ragged.constant(rt_input)
     with self.assertRaisesRegexp(exception, message):
-      ragged.row_lengths(rt_input, axis)
+      rt.row_lengths(axis)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py b/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py
index 7f5f4e91bd..2970540f3e 100644
--- a/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_row_splits_to_segment_ids_op_test.py
@@ -21,27 +21,24 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedSplitsToSegmentIdsOpTest(ragged_test_util.RaggedTensorTestCase):
 
-  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     splits = [0, 3, 3, 5, 6, 9]
     expected = [0, 0, 0, 2, 2, 3, 4, 4, 4]
     segment_ids = ragged.row_splits_to_segment_ids(splits)
-    with self.test_session():
-      self.assertEqual(segment_ids.eval().tolist(), expected)
+    self.assertAllEqual(segment_ids, expected)
 
-  @test_util.run_deprecated_v1
   def testEmptySplits(self):
     # Note: the splits for an empty ragged tensor contains a single zero.
     segment_ids = ragged.row_splits_to_segment_ids([0])
-    with self.test_session():
-      self.assertEqual(segment_ids.eval().tolist(), [])
+    self.assertAllEqual(segment_ids, [])
 
-  @test_util.run_deprecated_v1
   def testErrors(self):
     self.assertRaisesRegexp(ValueError, r'Invalid row_splits: \[\]',
                             ragged.row_splits_to_segment_ids, [])
diff --git a/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py
index 7e52f2d844..4ed9626767 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_ids_to_row_splits_op_test.py
@@ -21,25 +21,23 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedSplitsToSegmentIdsOpTest(ragged_test_util.RaggedTensorTestCase):
 
-  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     segment_ids = [0, 0, 0, 2, 2, 3, 4, 4, 4]
     expected = [0, 3, 3, 5, 6, 9]
     splits = ragged.segment_ids_to_row_splits(segment_ids)
-    with self.test_session():
-      self.assertEqual(splits.eval().tolist(), expected)
+    self.assertAllEqual(splits, expected)
 
-  @test_util.run_deprecated_v1
   def testEmptySegmentIds(self):
     # Note: the splits for an empty ragged tensor contains a single zero.
     segment_ids = ragged.segment_ids_to_row_splits([])
-    with self.test_session():
-      self.assertEqual(segment_ids.eval().tolist(), [0])
+    self.assertAllEqual(segment_ids, [0])
 
   def testErrors(self):
     self.assertRaisesRegexp(TypeError,
@@ -51,16 +49,13 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
     self.assertRaisesRegexp(ValueError, r'Shape \(1, 1\) must have rank 1',
                             ragged.segment_ids_to_row_splits, [[0]])
 
-  @test_util.run_deprecated_v1
   def testNumSegments(self):
     segment_ids = [0, 0, 0, 2, 2, 3, 4, 4, 4]
     num_segments = 7
     expected = [0, 3, 3, 5, 6, 9, 9, 9]
     splits = ragged.segment_ids_to_row_splits(segment_ids, num_segments)
-    with self.test_session():
-      self.assertEqual(splits.eval().tolist(), expected)
+    self.assertAllEqual(splits, expected)
 
-  @test_util.run_deprecated_v1
   def testUnsortedSegmentIds(self):
     # Segment ids are not required to be sorted.
     segment_ids = [0, 4, 3, 2, 4, 4, 2, 0, 0]
@@ -69,9 +64,8 @@ class RaggedSplitsToSegmentIdsOpTest(test_util.TensorFlowTestCase):
 
     splits2 = ragged.segment_ids_to_row_splits(segment_ids, 7)
     expected2 = [0, 3, 3, 5, 6, 9, 9, 9]
-    with self.test_session():
-      self.assertEqual(splits1.eval().tolist(), expected1)
-      self.assertEqual(splits2.eval().tolist(), expected2)
+    self.assertAllEqual(splits1, expected1)
+    self.assertAllEqual(splits2, expected2)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_segment_op_test.py b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
index 52fe739342..be1f39afef 100644
--- a/tensorflow/python/ops/ragged/ragged_segment_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_segment_op_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
@@ -45,21 +46,10 @@ def sqrt_n(values):
   return 1.0 * sum(values) / math.sqrt(len(values))
 
 
-class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedSegmentOpsTest(ragged_test_util.RaggedTensorTestCase,
                            parameterized.TestCase):
 
-  def assertNestedListAmostEqual(self, lhs, rhs, places=7, context='value'):
-    self.assertEqual(type(lhs), type(rhs))
-    if isinstance(lhs, (list, tuple)):
-      self.assertEqual(len(lhs), len(rhs), 'Length differs for %s' % context)
-      for i in range(len(lhs)):
-        self.assertNestedListAmostEqual(lhs[i], rhs[i], places,
-                                        '%s[%s]' % (context, i))
-    else:
-      self.assertAlmostEqual(
-          lhs, rhs, places,
-          '%s != %s within %s places at %s' % (lhs, rhs, places, context))
-
   def expected_value(self, data, segment_ids, num_segments, combiner):
     """Find the expected value for a call to ragged_segment_<aggregate>.
 
@@ -110,7 +100,6 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
       (ragged.segment_mean, mean, [5, 4, 3, 2, 1, 0]),
       (ragged.segment_mean, mean, [0, 0, 0, 10, 10, 10]),
   )
-  @test_util.run_deprecated_v1
   def testRaggedSegment_Int(self, segment_op, combiner, segment_ids):
     rt_as_list = [[0, 1, 2, 3], [4], [], [5, 6], [7], [8, 9]]
     rt = ragged.constant(rt_as_list)
@@ -119,7 +108,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                                    combiner)
 
     segmented = segment_op(rt, segment_ids, num_segments)
-    self.assertListEqual(self.evaluate(segmented).tolist(), expected)
+    self.assertRaggedEqual(segmented, expected)
 
   @parameterized.parameters(
       (ragged.segment_sum, sum, [0, 0, 1, 1, 2, 2]),
@@ -147,7 +136,6 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
       (ragged.segment_sqrt_n, sqrt_n, [5, 4, 3, 2, 1, 0]),
       (ragged.segment_sqrt_n, sqrt_n, [0, 0, 0, 10, 10, 10]),
   )
-  @test_util.run_deprecated_v1
   def testRaggedSegment_Float(self, segment_op, combiner, segment_ids):
     rt_as_list = [[0., 1., 2., 3.], [4.], [], [5., 6.], [7.], [8., 9.]]
     rt = ragged.constant(rt_as_list)
@@ -156,10 +144,8 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                                    combiner)
 
     segmented = segment_op(rt, segment_ids, num_segments)
-    self.assertNestedListAmostEqual(
-        self.evaluate(segmented).tolist(), expected, places=5)
+    self.assertRaggedAlmostEqual(segmented, expected, places=5)
 
-  @test_util.run_deprecated_v1
   def testRaggedRankTwo(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121],],  # row 0
@@ -173,16 +159,15 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
                  [],                                # row 1
                  [[411, 412], [321, 322], [331]]    # row 2
                 ]  # pyformat: disable
-    self.assertEqual(self.evaluate(segmented1).tolist(), expected1)
+    self.assertRaggedEqual(segmented1, expected1)
 
     segment_ids2 = [1, 2, 1, 1]
     segmented2 = ragged.segment_sum(rt, segment_ids2, 3)
     expected2 = [[],
                  [[111+411, 112+412, 113, 114], [121+321, 322], [331]],
                  []]  # pyformat: disable
-    self.assertEqual(self.evaluate(segmented2).tolist(), expected2)
+    self.assertRaggedEqual(segmented2, expected2)
 
-  @test_util.run_v1_only('b/120545219')
   def testRaggedSegmentIds(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121],],  # row 0
@@ -195,7 +180,7 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
     expected = [[],
                 [111+321, 112+322, 113, 114],
                 [121+331+411, 412]]  # pyformat: disable
-    self.assertEqual(self.evaluate(segmented).tolist(), expected)
+    self.assertRaggedEqual(segmented, expected)
 
   def testShapeMismatchError1(self):
     dt = constant_op.constant([1, 2, 3, 4, 5, 6])
@@ -205,7 +190,6 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
         'but segment_ids is ragged and data is not.', ragged.segment_sum, dt,
         segment_ids, 3)
 
-  @test_util.run_v1_only('b/120545219')
   def testShapeMismatchError2(self):
     rt = ragged.constant([
         [[111, 112, 113, 114], [121]],  # row 0
@@ -222,14 +206,13 @@ class RaggedSegmentOpsTest(test_util.TensorFlowTestCase,
         ragged.segment_sum, rt, segment_ids, 3)
 
     # Otherwise, error is raised when we run the graph.
-    segment_ids2 = ragged.from_row_splits(
+    segment_ids2 = ragged.RaggedTensor.from_row_splits(
         array_ops.placeholder_with_default(segment_ids.values, None),
         array_ops.placeholder_with_default(segment_ids.row_splits, None))
-    segmented2 = ragged.segment_sum(rt, segment_ids2, 3)
-    with self.cached_session():
-      self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          'segment_ids.shape must be a prefix of data.shape.*', segmented2.eval)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        'segment_ids.shape must be a prefix of data.shape.*'):
+      self.evaluate(ragged.segment_sum(rt, segment_ids2, 3))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_stack_op_test.py b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
index ed07aca6d4..17d80b5aad 100644
--- a/tensorflow/python/ops/ragged/ragged_stack_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
@@ -23,10 +23,13 @@ from absl.testing import parameterized
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
+                        parameterized.TestCase):
 
   @parameterized.parameters(
       dict(
@@ -265,7 +268,6 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           axis=0,
           expected=[[[b'a00', b'a01'], [], [b'a20', b'a21']]]),
   )   # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testRaggedStack(self,
                       descr,
                       rt_inputs,
@@ -286,8 +288,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertEqual(stacked.ragged_rank, expected_ragged_rank)
     if expected_shape is not None:
       self.assertEqual(stacked.shape.as_list(), expected_shape)
-    with self.test_session():
-      self.assertEqual(stacked.eval().tolist(), expected)
+    self.assertRaggedEqual(stacked, expected)
 
   @parameterized.parameters(
       dict(
@@ -314,7 +315,6 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   def testError(self, rt_inputs, axis, error, message):
     self.assertRaisesRegexp(error, message, ragged.stack, rt_inputs, axis)
 
-  @test_util.run_deprecated_v1
   def testSingleTensorInput(self):
     """Tests ragged_stack with a single tensor input.
 
@@ -324,8 +324,7 @@ class RaggedStackOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     """
     rt_inputs = ragged.constant([[1, 2], [3, 4]])
     stacked = ragged.stack(rt_inputs, 0)
-    with self.test_session():
-      self.assertEqual(stacked.eval().tolist(), [[[1, 2], [3, 4]]])
+    self.assertRaggedEqual(stacked, [[[1, 2], [3, 4]]])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_string_ops.py b/tensorflow/python/ops/ragged/ragged_string_ops.py
index cdcdbdff07..1f9f0abe4f 100644
--- a/tensorflow/python/ops/ragged/ragged_string_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_string_ops.py
@@ -23,7 +23,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_string_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.util.tf_export import tf_export
 
@@ -65,16 +64,16 @@ def unicode_encode(input, output_encoding, errors="replace",
     ```
   """
   with ops.name_scope(name, "UnicodeEncode", [input]):
-    input_tensor = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(input)
+    input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
     if input_tensor.shape.ndims is None:
       raise ValueError("Rank of input_tensor must be statically known.")
     if ragged_tensor.is_ragged(input_tensor):
-      if input_tensor.inner_values.shape.ndims > 1:
-        # If the inner_values of our ragged tensor is multi-dimensional, we can
+      if input_tensor.flat_values.shape.ndims > 1:
+        # If the flat_values of our ragged tensor is multi-dimensional, we can
         # process it separately and our output will have the same nested splits
         # as our input.
-        return input_tensor.with_inner_values(
-            unicode_encode(input_tensor.inner_values, output_encoding, errors,
+        return input_tensor.with_flat_values(
+            unicode_encode(input_tensor.flat_values, output_encoding, errors,
                            replacement_char))
       elif input_tensor.ragged_rank > 1:
         # Recursively process the values of the ragged tensor.
@@ -82,7 +81,7 @@ def unicode_encode(input, output_encoding, errors="replace",
             unicode_encode(input_tensor.values, output_encoding, errors,
                            replacement_char))
       else:
-        # Our ragged tensor is of the correct shape (rank 1 inner_values tensor
+        # Our ragged tensor is of the correct shape (rank 1 flat_values tensor
         # with ragged_rank of 1) so we can process it as normal.
         return gen_string_ops.unicode_encode(
             input_values=input_tensor.values,
@@ -110,10 +109,10 @@ def unicode_encode(input, output_encoding, errors="replace",
         # Our input tensor is rank 1, so we create a ragged tensor with an added
         # dimension to create the correct input shape & type, and then remove
         # the additional dimension from the output and return the string scalar.
-        ragged_input_tensor = ragged_factory_ops.from_row_splits(
+        ragged_input_tensor = ragged_tensor.RaggedTensor.from_row_splits(
             input_tensor,
-            array_ops.stack([0, array_ops.shape(input_tensor,
-                                                out_type=dtypes.int64)[0]]))
+            array_ops.stack(
+                [0, array_ops.shape(input_tensor, out_type=dtypes.int64)[0]]))
         output_tensor = unicode_encode(ragged_input_tensor, output_encoding,
                                        errors, replacement_char)
         return array_ops.reshape(output_tensor, [])
diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py
index ddeabfb464..567c50203a 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor.py
@@ -19,9 +19,19 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.client import session
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_ragged_conversion_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_tensor_value
+from tensorflow.python.ops.ragged import ragged_util
+from tensorflow.python.ops.ragged import segment_id_ops
 
 # pylint: disable=protected-access
 _eval_using_default_session = ops._eval_using_default_session
@@ -84,10 +94,10 @@ class RaggedTensor(object):
   Example:
 
   ```python
-  >>> rt = ragged.from_row_splits(values=[3, 1, 4, 1, 5, 9, 2, 6],
-  ...                             row_splits=[0, 4, 4, 7, 8, 8])
-  >>> rt.tolist()
-  [[3, 1, 4, 1], [], [5, 9, 2], [6], []]
+  >>> print(tf.RaggedTensor.from_row_splits(
+  ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
+  ...     row_splits=[0, 4, 4, 7, 8, 8]))
+  <tf.RaggedTensor [[3, 1, 4, 1], [], [5, 9, 2], [6], []]>
   ```
 
   ### Alternative Row-Partitioning Schemes
@@ -116,13 +126,12 @@ class RaggedTensor(object):
 
   ```python
   >>> values = [3, 1, 4, 1, 5, 9, 2, 6]
-  >>> rt1 = ragged.from_row_splits(values, row_splits=[0, 4, 4, 7, 8, 8])
-  >>> rt2 = ragged.from_row_lengths(values, row_lengths=[4, 0, 3, 1, 0])
-  >>> rt3 = ragged.from_value_rowids(values,
-  ...                                value_rowids=[0, 0, 0, 0, 2, 2, 2, 3],
-  ...                                nrows=5)
-  >>> rt4 = ragged.from_row_starts(values, row_starts=[0, 4, 4, 7, 8])
-  >>> rt5 = ragged.from_row_limits(values, row_limits=[4, 4, 7, 8, 8])
+  >>> rt1 = RaggedTensor.from_row_splits(values, row_splits=[0, 4, 4, 7, 8, 8])
+  >>> rt2 = RaggedTensor.from_row_lengths(values, row_lengths=[4, 0, 3, 1, 0])
+  >>> rt3 = RaggedTensor.from_value_rowids(
+  ...     values, value_rowids=[0, 0, 0, 0, 2, 2, 2, 3], nrows=5)
+  >>> rt4 = RaggedTensor.from_row_starts(values, row_starts=[0, 4, 4, 7, 8])
+  >>> rt5 = RaggedTensor.from_row_limits(values, row_limits=[4, 4, 7, 8, 8])
   ```
 
   ### Multiple Ragged Dimensions
@@ -132,24 +141,24 @@ class RaggedTensor(object):
   adds a single ragged dimension.
 
   ```python
-  >>> inner_rt = ragged.from_row_splits(  # =rt1 from above
+  >>> inner_rt = RaggedTensor.from_row_splits(  # =rt1 from above
   ...     values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8])
-  >>> outer_rt = ragged.from_row_splits(
+  >>> outer_rt = RaggedTensor.from_row_splits(
   ...     values=inner_rt, row_splits=[0, 3, 3, 5])
-  >>> print outer_rt.tolist()
+  >>> print outer_rt.to_list()
   [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]
   >>> print outer_rt.ragged_rank
   2
   ```
 
-  The factory function `ragged.from_nested_row_splits` may be used to
+  The factory function `RaggedTensor.from_nested_row_splits` may be used to
   construct a `RaggedTensor` with multiple ragged dimensions directly, by
   providing a list of `row_splits` tensors:
 
   ```python
-  >>> ragged.from_nested_row_splits(
-  ...     inner_values=[3, 1, 4, 1, 5, 9, 2, 6],
-  ...     nested_row_splits=([0, 3, 3, 5], [0, 4, 4, 7, 8, 8])).tolist()
+  >>> RaggedTensor.from_nested_row_splits(
+  ...     flat_values=[3, 1, 4, 1, 5, 9, 2, 6],
+  ...     nested_row_splits=([0, 3, 3, 5], [0, 4, 4, 7, 8, 8])).to_list()
   [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]
   ```
 
@@ -159,12 +168,13 @@ class RaggedTensor(object):
   by using a multidimensional `Tensor` for `values`.
 
   ```python
-  >>> rt = ragged.from_row_splits(values=tf.ones([5, 3]), row_splits=[0, 2, 5])
-  >>> print rt.tolist()
+  >>> rt = RaggedTensor.from_row_splits(values=tf.ones([5, 3]),
+  ..                                    row_splits=[0, 2, 5])
+  >>> print rt.to_list()
   [[[1, 1, 1], [1, 1, 1]],
    [[1, 1, 1], [1, 1, 1], [1, 1, 1]]]
-   >>> print rt.shape.as_list()
-   [2, None, 3]
+   >>> print rt.shape
+   (2, ?, 3)
   ```
 
   ### RaggedTensor Shape Restrictions
@@ -181,31 +191,6 @@ class RaggedTensor(object):
   dimension followed by a ragged dimension.
   """
 
-  #=============================================================================
-  # Implementation notes
-  #=============================================================================
-  # Currently, the RaggedTensor class uses a single row-partitioning scheme
-  # (row_splits).
-  #
-  # We are considering adding value_rowids+nvals as a secondary
-  # row-partitioning scheme.  This change would not impact the functional
-  # interface of the RaggedTensor class, but it would impact the efficiency
-  # of several operations.  In particular:
-  #
-  #   * The functions `ragged.value_rowids` and `ragged.nrows` would always
-  #     return pre-existing tensors; they would not need to add any ops to
-  #     the graph.
-  #
-  #   * The `RaggedTensor` constructor would construct all row-partitioning
-  #     tensors (row_splits, value_rowids, and nvals).  In eager mode, this
-  #     would mean that conversion operations would occur whenever a
-  #     `RaggedTensor` is constructed.  But in graph mode, the converted
-  #     row-partitioning tensors would only be evaluated if they are used.
-  #
-  # Since this change impacts efficiency but not functionality, we would like
-  # to perform additional profiling with real-world use cases before we
-  # decide whether to make this change.
-
   #=============================================================================
   # Constructor (private)
   #=============================================================================
@@ -221,13 +206,14 @@ class RaggedTensor(object):
     This constructor is private -- please use one of the following ops to
     build `RaggedTensor`s:
 
-      * [`ragged.from_row_lengths()`](from_row_lengths.md)
-      * [`ragged.from_value_rowids()`](from_value_rowids.md)
-      * [`ragged.from_row_splits()`](from_row_splits.md)
-      * [`ragged.from_row_starts()`](from_row_starts.md)
-      * [`ragged.from_row_limits()`](from_row_limits.md)
-      * [`ragged.from_nested_row_splits()`](from_nested_row_splits.md)
-      * [`ragged.from_nested_value_rowids()`](from_nested_value_rowids.md)
+      * `tf.RaggedTensor.from_row_lengths`
+      * `tf.RaggedTensor.from_value_rowids`
+      * `tf.RaggedTensor.from_row_splits`
+      * `tf.RaggedTensor.from_row_starts`
+      * `tf.RaggedTensor.from_row_limits`
+      * `tf.RaggedTensor.from_nested_row_splits`
+      * `tf.RaggedTensor.from_nested_row_lengths`
+      * `tf.RaggedTensor.from_nested_value_rowids`
 
     Args:
       values: A potentially ragged tensor of any dtype and shape `[nvals, ...]`.
@@ -248,7 +234,7 @@ class RaggedTensor(object):
     if not internal:
       raise ValueError("RaggedTensor constructor is private; please use one "
                        "of the factory methods instead (e.g., "
-                       "ragged.from_row_lengths())")
+                       "RaggedTensor.from_row_lengths())")
 
     # Validate the arguments.
     if not isinstance(values, (RaggedTensor, ops.Tensor)):
@@ -272,6 +258,364 @@ class RaggedTensor(object):
     self._cached_value_rowids = cached_value_rowids
     self._cached_nrows = cached_nrows
 
+  #=============================================================================
+  # Factory Methods
+  #=============================================================================
+
+  @classmethod
+  def from_value_rowids(cls, values, value_rowids, nrows=None, name=None):
+    """Creates a `RaggedTensor` with rows partitioned by `value_rowids`.
+
+    The returned `RaggedTensor` corresponds with the python list defined by:
+
+    ```python
+    result = [[values[i] for i in range(len(values)) if value_rowids[i] == row]
+              for row in range(nrows)]
+    ```
+
+    Warning: currently, this needs to cast value_rowids to int64 before
+    converting, since `tf.bincount` only supports `int32`.
+
+    Args:
+      values: A potentially ragged tensor with shape `[nvals, ...]`.
+      value_rowids: A 1-D int64 tensor with shape `[nvals]`, which corresponds
+        one-to-one with `values`, and specifies each value's row index.  Must be
+        nonnegative, and must be sorted in ascending order.
+      nrows: An int64 scalar specifying the number of rows.  This should be
+        specified if the `RaggedTensor` may containing empty training rows. Must
+        be greater than `value_rowids[-1]` (or zero if `value_rowids` is empty).
+        Defaults to `value_rowids[-1]` (or zero if `value_rowids` is empty).
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor`.  `result.rank = values.rank + 1`.
+      `result.ragged_rank = values.ragged_rank + 1`.
+
+    Raises:
+      ValueError: If `nrows` is incompatible with `value_rowids`.
+
+    #### Example:
+      ```python
+      >>> print(tf.RaggedTensor.from_value_rowids(
+      ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
+      ...     value_rowids=[0, 0, 0, 0, 2, 2, 2, 3],
+      ...     nrows=5))
+      <tf.RaggedTensor [[3, 1, 4, 1], [], [5, 9, 2], [6], []]>
+      ```
+    """
+    with ops.name_scope(name, "RaggedFromValueRowIds",
+                        [values, value_rowids, nrows]):
+      values = convert_to_tensor_or_ragged_tensor(values, name="values")
+      value_rowids = ops.convert_to_tensor(
+          value_rowids, dtypes.int64, name="value_rowids")
+      if nrows is None:
+        const_rowids = tensor_util.constant_value(value_rowids)
+        if const_rowids is None:
+          nrows = array_ops.concat([value_rowids[-1:], [-1]], axis=0)[0] + 1
+          const_nrows = None
+        else:
+          const_nrows = const_rowids[-1] + 1 if const_rowids.size > 0 else 0
+          nrows = ops.convert_to_tensor(const_nrows, dtypes.int64, name="nrows")
+      else:
+        nrows = ops.convert_to_tensor(nrows, dtypes.int64, "nrows")
+        const_nrows = tensor_util.constant_value(nrows)
+        if const_nrows is not None:
+          if const_nrows < 0:
+            raise ValueError("Expected nrows >= 0; got %d" % const_nrows)
+          const_rowids = tensor_util.constant_value(value_rowids)
+          if const_rowids is not None and const_rowids.size > 0:
+            if not const_nrows >= const_rowids[-1] + 1:
+              raise ValueError(
+                  "Expected nrows >= value_rowids[-1] + 1; got nrows=%d, "
+                  "value_rowids[-1]=%d" % (const_nrows, const_rowids[-1]))
+
+      value_rowids.shape.assert_has_rank(1)
+      nrows.shape.assert_has_rank(0)
+      values.shape[:1].assert_is_compatible_with(value_rowids.shape)
+
+      # Convert value_rowids & nrows to row_splits.
+      # Note: we don't use segment_ids_to_row_splits() here because we want
+      # to save the intermediate value `row_lengths`, so we can cache it.
+      # TODO(b/116708836) Upgrade bincount to accept int64 so we can skip the
+      # cast (Remove the warning in the docstring when we do.)
+      value_rowids_int32 = math_ops.cast(value_rowids, dtypes.int32)
+      nrows_int32 = math_ops.cast(nrows, dtypes.int32)
+      row_lengths = math_ops.bincount(
+          value_rowids_int32,
+          minlength=nrows_int32,
+          maxlength=nrows_int32,
+          dtype=dtypes.int64)
+      row_splits = array_ops.concat([[0], math_ops.cumsum(row_lengths)], axis=0)
+      if const_nrows is not None:
+        row_lengths.set_shape([const_nrows])
+        row_splits.set_shape([const_nrows + 1])
+
+      return cls(
+          values,
+          row_splits,
+          cached_row_lengths=row_lengths,
+          cached_value_rowids=value_rowids,
+          cached_nrows=nrows,
+          internal=True)
+
+  @classmethod
+  def from_row_splits(cls, values, row_splits, name=None):
+    """Creates a `RaggedTensor` with rows partitioned by `row_splits`.
+
+    The returned `RaggedTensor` corresponds with the python list defined by:
+
+    ```python
+    result = [values[row_splits[i]:row_splits[i + 1]]
+              for i in range(len(row_splits) - 1)]
+    ```
+
+    Args:
+      values: A potentially ragged tensor with shape `[nvals, ...]`.
+      row_splits: A 1-D int64 tensor with shape `[nrows+1]`.  Must not be empty,
+        and must be sorted in ascending order.  `row_splits[0]` must be zero and
+        `row_splits[-1]` must be `nvals`.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor`.  `result.rank = values.rank + 1`.
+      `result.ragged_rank = values.ragged_rank + 1`.
+
+    Raises:
+      ValueError: If `row_splits` is an empty list.
+
+    #### Example:
+      ```python
+      >>> print(tf.RaggedTensor.from_row_splits(
+      ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
+      ...     row_splits=[0, 4, 4, 7, 8, 8]))
+      <tf.RaggedTensor [[3, 1, 4, 1], [], [5, 9, 2], [6], []]>
+      ```
+    """
+    if isinstance(row_splits, (list, tuple)) and not row_splits:
+      raise ValueError("row_splits tensor may not be empty.")
+    with ops.name_scope(name, "RaggedFromRowSplits", [values, row_splits]):
+      values = convert_to_tensor_or_ragged_tensor(values, name="values")
+      row_splits = ops.convert_to_tensor(row_splits, dtypes.int64, "row_splits")
+      row_splits.shape.assert_has_rank(1)
+      return cls(values=values, row_splits=row_splits, internal=True)
+
+  @classmethod
+  def from_row_lengths(cls, values, row_lengths, name=None):
+    """Creates a `RaggedTensor` with rows partitioned by `row_lengths`.
+
+    The returned `RaggedTensor` corresponds with the python list defined by:
+
+    ```python
+    result = [[values.pop(0) for i in range(length)]
+              for length in row_lengths]
+    ```
+
+    Args:
+      values: A potentially ragged tensor with shape `[nvals, ...]`.
+      row_lengths: A 1-D int64 tensor with shape `[nrows]`.  Must be
+        nonnegative.  `sum(row_lengths)` must be `nvals`.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor`.  `result.rank = values.rank + 1`.
+      `result.ragged_rank = values.ragged_rank + 1`.
+
+    #### Example:
+      ```python
+      >>> print(tf.RaggedTensor.from_row_lengths(
+      ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
+      ...     row_lengths=[4, 0, 3, 1, 0]))
+      <tf.RaggedTensor [[3, 1, 4, 1], [], [5, 9, 2], [6], []])>
+      ```
+    """
+    with ops.name_scope(name, "RaggedFromRowLengths", [values, row_lengths]):
+      values = convert_to_tensor_or_ragged_tensor(values, name="values")
+      row_lengths = ops.convert_to_tensor(row_lengths, dtypes.int64,
+                                          "row_lengths")
+      row_lengths.shape.assert_has_rank(1)
+      row_limits = math_ops.cumsum(row_lengths)
+      row_splits = array_ops.concat([[0], row_limits], axis=0)
+      return cls(
+          values=values,
+          row_splits=row_splits,
+          cached_row_lengths=row_lengths,
+          internal=True)
+
+  @classmethod
+  def from_row_starts(cls, values, row_starts, name=None):
+    """Creates a `RaggedTensor` with rows partitioned by `row_starts`.
+
+    Equivalent to: `from_row_splits(values, concat([row_starts, nvals]))`.
+
+    Args:
+      values: A potentially ragged tensor with shape `[nvals, ...]`.
+      row_starts: A 1-D int64 tensor with shape `[nrows]`.  Must be nonnegative
+        and sorted in ascending order.  If `nrows>0`, then `row_starts[0]` must
+        be zero.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor`.  `result.rank = values.rank + 1`.
+      `result.ragged_rank = values.ragged_rank + 1`.
+
+    #### Example:
+      ```python
+      >>> print(tf.RaggedTensor.from_row_starts(
+      ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
+      ...     row_starts=[0, 4, 4, 7, 8]))
+      <tf.RaggedTensor [[3, 1, 4, 1], [], [5, 9, 2], [6], []]>
+      ```
+    """
+    with ops.name_scope(name, "RaggedFromRowStarts", [values, row_starts]):
+      values = convert_to_tensor_or_ragged_tensor(values, name="values")
+      row_starts = ops.convert_to_tensor(row_starts, dtypes.int64, "row_starts")
+      row_starts.shape.assert_has_rank(1)
+      nvals = array_ops.shape(values, out_type=dtypes.int64)[:1]
+      row_splits = array_ops.concat([row_starts, nvals], axis=0)
+      return cls(values=values, row_splits=row_splits, internal=True)
+
+  @classmethod
+  def from_row_limits(cls, values, row_limits, name=None):
+    """Creates a `RaggedTensor` with rows partitioned by `row_limits`.
+
+    Equivalent to: `from_row_splits(values, concat([0, row_limits]))`.
+
+    Args:
+      values: A potentially ragged tensor with shape `[nvals, ...]`.
+      row_limits: A 1-D int64 tensor with shape `[nrows]`.  Must be sorted in
+        ascending order.  If `nrows>0`, then `row_limits[-1]` must be `nvals`.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor`.  `result.rank = values.rank + 1`.
+      `result.ragged_rank = values.ragged_rank + 1`.
+
+    #### Example:
+      ```python
+      >>> print(tf.RaggedTensor.from_row_limits(
+      ...     values=[3, 1, 4, 1, 5, 9, 2, 6],
+      ...     row_limits=[4, 4, 7, 8, 8]))
+      <tf.RaggedTensor [[3, 1, 4, 1], [], [5, 9, 2], [6], []]>
+      ```
+    """
+    with ops.name_scope(name, "RaggedFromRowLimits", [values, row_limits]):
+      values = convert_to_tensor_or_ragged_tensor(values, name="values")
+      row_limits = ops.convert_to_tensor(row_limits, dtypes.int64, "row_limits")
+      row_limits.shape.assert_has_rank(1)
+      zero = array_ops.zeros([1], dtypes.int64)
+      row_splits = array_ops.concat([zero, row_limits], axis=0)
+      return cls(values=values, row_splits=row_splits, internal=True)
+
+  @classmethod
+  def from_nested_value_rowids(cls,
+                               flat_values,
+                               nested_value_rowids,
+                               nested_nrows=None,
+                               name=None):
+    """Creates a `RaggedTensor` from a nested list of `value_rowids` tensors.
+
+    Equivalent to:
+
+    ```python
+    result = flat_values
+    for (rowids, nrows) in reversed(zip(nested_value_rowids, nested_nrows)):
+      result = from_value_rowids(result, rowids, nrows)
+    ```
+
+    Args:
+      flat_values: A potentially ragged tensor.
+      nested_value_rowids: A list of 1-D int64 tensors.  The `i`th tensor is
+        used as the `value_rowids` for the `i`th ragged dimension.
+      nested_nrows: A list of int64 scalars.  The `i`th scalar is used as the
+        `nrows` for the `i`th ragged dimension.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor` (or `flat_values` if `nested_value_rowids` is empty).
+
+    Raises:
+      ValueError: If `len(nested_values_rowids) != len(nested_nrows)`.
+    """
+    if isinstance(nested_value_rowids, ops.Tensor):
+      raise TypeError("nested_value_rowids must be a list of Tensors")
+    if nested_nrows is None:
+      nested_nrows = [None] * len(nested_value_rowids)
+    else:
+      if isinstance(nested_nrows, ops.Tensor):
+        raise TypeError("nested_nrows must be a list of Tensors")
+      if len(nested_nrows) != len(nested_value_rowids):
+        raise ValueError("nested_nrows must have the same length as "
+                         "nested_value_rowids")
+
+    with ops.name_scope(
+        name, "RaggedFromNestedValueRowIds",
+        [flat_values] + list(nested_value_rowids) + list(nested_nrows)):
+      result = flat_values
+      for value_rowids, nrows in reversed(
+          list(zip(nested_value_rowids, nested_nrows))):
+        result = cls.from_value_rowids(result, value_rowids, nrows)
+      return result
+
+  @classmethod
+  def from_nested_row_splits(cls, flat_values, nested_row_splits, name=None):
+    """Creates a `RaggedTensor` from a nested list of `row_splits` tensors.
+
+    Equivalent to:
+
+    ```python
+    result = flat_values
+    for row_splits in reversed(nested_row_splits):
+      result = from_row_splits(result, row_splits)
+    ```
+
+    Args:
+      flat_values: A potentially ragged tensor.
+      nested_row_splits: A list of 1-D int64 tensors.  The `i`th tensor is used
+        as the `row_splits` for the `i`th ragged dimension.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor` (or `flat_values` if `nested_row_splits` is empty).
+    """
+    if isinstance(nested_row_splits, ops.Tensor):
+      raise TypeError("nested_row_splits must be a list of Tensors")
+    with ops.name_scope(name, "RaggedFromNestedRowSplits",
+                        [flat_values] + list(nested_row_splits)):
+      result = flat_values
+      for splits in reversed(nested_row_splits):
+        result = cls.from_row_splits(result, splits)
+      return result
+
+  @classmethod
+  def from_nested_row_lengths(cls, flat_values, nested_row_lengths, name=None):
+    """Creates a `RaggedTensor` from a nested list of `row_lengths` tensors.
+
+    Equivalent to:
+
+    ```python
+    result = flat_values
+    for row_lengths in reversed(nested_row_lengths):
+      result = from_row_lengths(result, row_lengths)
+    ```
+
+    Args:
+      flat_values: A potentially ragged tensor.
+      nested_row_lengths: A list of 1-D int64 tensors.  The `i`th tensor is used
+        as the `row_lengths` for the `i`th ragged dimension.
+      name: A name prefix for the RaggedTensor (optional).
+
+    Returns:
+      A `RaggedTensor` (or `flat_values` if `nested_row_lengths` is empty).
+    """
+    if isinstance(nested_row_lengths, ops.Tensor):
+      raise TypeError("nested_row_lengths must be a list of Tensors")
+    with ops.name_scope(name, "RaggedFromNestedRowlengths",
+                        [flat_values] + list(nested_row_lengths)):
+      result = flat_values
+      for lengths in reversed(nested_row_lengths):
+        result = cls.from_row_lengths(result, lengths)
+      return result
+
   #=============================================================================
   # Accessors
   #=============================================================================
@@ -334,8 +678,8 @@ class RaggedTensor(object):
     #### Example:
       ```python
       >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
-      >>> rt.values.eval()
-      [3, 1, 4, 1, 5, 9, 2, 6]
+      >>> print rt.values
+      tf.Tensor([3, 1, 4, 1, 5, 9, 2, 6])
       ```
     """
     return self._values
@@ -357,26 +701,24 @@ class RaggedTensor(object):
     #### Example:
       ```python
       >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
-      >>> rt.values.eval()
-      [3, 1, 4, 1, 5, 9, 2, 6]
-      >>> rt.row_splits.eval()  # indices of row splits in ragged.values
-      [0, 4, 4, 7, 8, 8]
+      >>> print rt.row_splits  # indices of row splits in rt.values
+      tf.Tensor([0, 4, 4, 7, 8, 8])
       ```
     """
     return self._row_splits
 
   @property
-  def inner_values(self):
+  def flat_values(self):
     """The innermost `values` tensor for this ragged tensor.
 
-    Concretely, if `rt.values` is a `Tensor`, then `rt.inner_values` is
-    `rt.values`; otherwise, `rt.inner_values` is `rt.values.inner_values`.
+    Concretely, if `rt.values` is a `Tensor`, then `rt.flat_values` is
+    `rt.values`; otherwise, `rt.flat_values` is `rt.values.flat_values`.
 
-    Conceptually, `inner_values` is the tensor formed by flattening the
+    Conceptually, `flat_values` is the tensor formed by flattening the
     outermost dimension and all of the ragged dimensions into a single
     dimension.
 
-    `rt.inner_values.shape = [nvals] + rt.shape[rt.ragged_rank + 1:]`
+    `rt.flat_values.shape = [nvals] + rt.shape[rt.ragged_rank + 1:]`
     (where `nvals` is the number of items in the flattened dimensions).
 
     Returns:
@@ -386,8 +728,8 @@ class RaggedTensor(object):
 
       ```python
       >>> rt = ragged.constant([[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
-      >>> ragged.inner_values(rt).eval()
-      [3, 1, 4, 1, 5, 9, 2, 6]
+      >>> print rt.flat_values()
+      tf.Tensor([3, 1, 4, 1, 5, 9, 2, 6])
       ```
     """
     rt_values = self.values
@@ -413,8 +755,8 @@ class RaggedTensor(object):
 
       ```python
       >>> rt = ragged.constant([[[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]])
-      >>> for i, splits in enumerate(ragged.nested_row_splits(rt)):
-      ...   print('Splits for dimension %d: %s' % (i+1, splits.eval()))
+      >>> for i, splits in enumerate(rt.nested_row_splits()):
+      ...   print('Splits for dimension %d: %s' % (i+1, splits))
       Splits for dimension 1: [0, 1]
       Splits for dimension 2: [0, 3, 3, 5]
       Splits for dimension 3: [0, 4, 4, 7, 8, 8]
@@ -428,38 +770,220 @@ class RaggedTensor(object):
       rt_values = rt_values.values
     return tuple(rt_nested_splits)
 
-  @property
-  def cached_value_rowids(self):
-    """The row lengths for this `RaggedTensor`, or `None`.
+  def value_rowids(self, name=None):
+    """Returns the row indices for the `values` in this ragged tensor.
+
+    `rt.value_rowids()` corresponds one-to-one with the outermost dimension of
+    `rt.values`, and specifies the row containing each value.  In particular,
+    the row `rt[row]` consists of the values `rt.values[j]` where
+    `rt.value_rowids()[j] == row`.
+
+    Args:
+      name: A name prefix for the returned tensor (optional).
 
     Returns:
-      The `value_rowids` tensor that was used to construct this `RaggedTensor`
-      if it was constructed using
-      [`ragged.from_value_rowids`](from_value_rowids.md); or `None` otherwise.
+      A 1-D `int64` `Tensor` with shape `self.values.shape[:1]`.
+      The returned tensor is nonnegative, and is sorted in ascending order.
+
+    #### Example:
+      ```python
+      >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+      >>> rt.values
+      tf.Tensor([3, 1, 4, 1, 5, 9, 2, 6])
+      >>> rt.value_rowids()
+      tf.Tensor([0, 0, 0, 0, 2, 2, 2, 3])  # corresponds 1:1 with rt.values
+      ```
     """
-    return self._cached_value_rowids
+    if self._cached_value_rowids is not None:
+      return self._cached_value_rowids
 
-  @property
-  def cached_nrows(self):
-    """The row lengths for this `RaggedTensor`, or `None`.
+    with ops.name_scope(name, "RaggedValueRowIds", [self]):
+      return segment_id_ops.row_splits_to_segment_ids(self.row_splits)
+
+  def nrows(self, out_type=dtypes.int64, name=None):
+    """Returns the number of rows in this ragged tensor.
+
+    I.e., the size of the outermost dimension of the tensor.
+
+    Args:
+      out_type: `dtype` for the returned tensor.
+      name: A name prefix for the returned tensor (optional).
 
     Returns:
-      The `nrows` tensor that was used to construct this `RaggedTensor`
-      if it was constructed using
-      [`ragged.from_value_rowids`](from_value_rowids.md); or `None` otherwise.
+      A scalar `Tensor` with dtype `out_type`.
+
+    #### Example:
+      ```python
+      >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+      >>> rt.nrows()  # rt has 5 rows.
+      5
+      ```
     """
-    return self._cached_nrows
+    if self._cached_nrows is not None:
+      return self._cached_nrows
 
-  @property
-  def cached_row_lengths(self):
-    """The row lengths for this `RaggedTensor`, or `None`.
+    with ops.name_scope(name, "RaggedNRows", [self]):
+      return array_ops.shape(self.row_splits, out_type=out_type)[0] - 1
+
+  def row_starts(self, name=None):
+    """Returns the start indices for rows in this ragged tensor.
+
+    These indices specify where the values for each row begin in
+    `self.values`.  `rt.row_starts()` is equal to `rt.row_splits[:-1]`.
+
+    Args:
+      name: A name prefix for the returned tensor (optional).
+
+    Returns:
+      A 1-D Tensor of int64 with shape `[nrows]`.
+      The returned tensor is nonnegative, and is sorted in ascending order.
+
+    #### Example:
+      ```python
+      >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+      >>> rt.values
+      tf.Tensor([3, 1, 4, 1, 5, 9, 2, 6])
+      >>> rt.row_starts()  # indices of row starts in rt.values
+      tf.Tensor([0, 4, 4, 7, 8])
+      ```
+    """
+    with ops.name_scope(name, "RaggedRowStarts", [self]):
+      return self.row_splits[:-1]
+
+  def row_limits(self, name=None):
+    """Returns the limit indices for rows in this ragged tensor.
+
+    These indices specify where the values for each row end in
+    `self.values`.  `rt.row_limits(self)` is equal to `rt.row_splits[:-1]`.
+
+    Args:
+      name: A name prefix for the returned tensor (optional).
+
+    Returns:
+      A 1-D Tensor of int64 with shape `[nrows]`.
+      The returned tensor is nonnegative, and is sorted in ascending order.
+
+    #### Example:
+      ```python
+      >>> rt = ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+      >>> rt.values
+      tf.Tensor([3, 1, 4, 1, 5, 9, 2, 6])
+      >>> rt.row_limits()  # indices of row limits in rt.values
+      tf.Tensor([4, 4, 7, 8, 8])
+      ```
+    """
+    with ops.name_scope(name, "RaggedRowLimits", [self]):
+      return self.row_splits[1:]
+
+  def row_lengths(self, axis=1, name=None):
+    """Returns the lengths of the rows in this ragged tensor.
+
+    `rt.row_lengths()[i]` indicates the number of values in the
+    `i`th row of `rt`.
+
+    Args:
+      axis: An integer constant indicating the axis whose row lengths should be
+        returned.
+      name: A name prefix for the returned tensor (optional).
+
+    Returns:
+      A potentially ragged Tensor of int64 with shape `self.shape[:axis]`.
+
+    Raises:
+      ValueError: If `axis` is out of bounds.
+
+    #### Example:
+      ```python
+      >>> rt = ragged.constant([[[3, 1, 4], [1]], [], [[5, 9], [2]], [[6]], []])
+      >>> rt.row_lengths(rt)  # lengths of rows in rt
+      tf.Tensor([2, 0, 2, 1, 0])
+      >>> rt.row_lengths(axis=2)  # lengths of axis=2 rows.
+      <tf.RaggedTensor [[3, 1], [], [2, 1], [1], []]>
+      ```
+    """
+    if self._cached_row_lengths is not None:
+      return self._cached_row_lengths
+
+    with ops.name_scope(name, "RaggedRowLengths", [self]):
+      axis = ragged_util.get_positive_axis(axis, self.shape.ndims)
+      if axis == 0:
+        return self.nrows()
+      elif axis == 1:
+        splits = self.row_splits
+        return splits[1:] - splits[:-1]
+      elif isinstance(self.values, RaggedTensor):
+        return self.with_values(self.values.row_lengths(axis - 1))
+      else:
+        shape = array_ops.shape(self.values, out_type=dtypes.int64)
+        return self.with_values(
+            array_ops.ones(shape[:axis - 1], dtypes.int64) * shape[axis - 1])
+
+  def nested_row_lengths(self, name=None):
+    """Returns a tuple containing the row_lengths for all ragged dimensions.
+
+    `rtnested_row_lengths()` is a tuple containing the `row_lengths` tensors for
+    all ragged dimensions in `rt`, ordered from outermost to innermost.
+
+    Args:
+      name: A name prefix for the returned tensors (optional).
+
+    Returns:
+      A `tuple` of 1-D `int64` `Tensors`.  The length of the tuple is equal to
+      `self.ragged_rank`.
+    """
+    with ops.name_scope(name, "RaggedNestedRowLengths", [self]):
+      rt_nested_row_lengths = []
+      rt = self
+      while isinstance(rt, RaggedTensor):
+        rt_nested_row_lengths.append(rt.row_lengths())
+        rt = rt.values
+      return tuple(rt_nested_row_lengths)
+
+  def bounding_shape(self, axis=None, name=None):
+    """Returns the tight bounding box shape for this `RaggedTensor`.
+
+    Args:
+      axis: An integer scalar or vector indicating which axes to return the
+        bounding box for.  If not specified, then the full bounding box is
+        returned.
+      name: A name prefix for the returned tensor (optional).
 
     Returns:
-      The `row_lengths` tensor that was used to construct this `RaggedTensor`
-      if it was constructed using
-      [`ragged.from_row_lengths`](from_row_lengths.md); or `None` otherwise.
+      An int64 `Tensor`.  If `axis` is not specified, then `output`
+      is a vector with `output.shape=[self.shape.ndims]`.  If `axis` is a
+      scalar, then the `output` is a scalar.  If `axis` is a vector, then
+      `output` is a vector, where `output[i]` is the bounding size for
+      dimension `axis[i]`.
+
+    #### Example:
+      ```python
+      >>> rt = ragged.constant([[1, 2, 3, 4], [5], [], [6, 7, 8, 9], [10]])
+      >>> rt.bounding_shape()
+      [5, 4]
+      ```
     """
-    return self._cached_row_lengths
+    with ops.name_scope(name, "RaggedBoundingBox", [self, axis]):
+      nested_splits = self.nested_row_splits
+      rt_flat_values = self.flat_values
+
+      # Optimized special cases for when axis=0 or axis=1:
+      if isinstance(axis, int):
+        if axis == 0:
+          return array_ops.shape(nested_splits[0], out_type=dtypes.int64)[0] - 1
+        elif axis == 1:
+          return math_ops.maximum(math_ops.reduce_max(self.row_lengths()), 0)
+
+      splits_shape = array_ops.shape(self.row_splits, out_type=dtypes.int64)
+      flat_values_shape = array_ops.shape(rt_flat_values, out_type=dtypes.int64)
+
+      ragged_dimensions = array_ops.stack([splits_shape[0] - 1] + [
+          math_ops.maximum(math_ops.reduce_max(splits[1:] - splits[:-1]), 0)
+          for splits in nested_splits
+      ])
+      inner_dimensions = flat_values_shape[1:]
+
+      bbox = array_ops.concat([ragged_dimensions, inner_dimensions], axis=0)
+      return bbox if axis is None else array_ops.gather(bbox, axis)
 
   #=============================================================================
   # Transformation
@@ -481,7 +1005,7 @@ class RaggedTensor(object):
       `result.ragged_rank = 1 + new_values.ragged_rank`
     """
     new_values.shape.with_rank_at_least(1)
-    self.values.shape[0].assert_is_compatible_with(new_values.shape[0])
+    self.values.shape[:1].assert_is_compatible_with(new_values.shape[:1])
     return RaggedTensor(
         new_values,
         self._row_splits,
@@ -490,16 +1014,16 @@ class RaggedTensor(object):
         self._cached_nrows,
         internal=True)
 
-  def with_inner_values(self, new_values):
-    """Returns a copy of `self` with `inner_values` replaced by `new_value`.
+  def with_flat_values(self, new_values):
+    """Returns a copy of `self` with `flat_values` replaced by `new_value`.
 
     Preserves cached row-partitioning tensors such as `self.cached_nrows` and
     `self.cached_value_rowids` if they have values.
 
     Args:
       new_values: Potentially ragged tensor that should replace
-      `self.inner_values`.  Must have `rank > 0`, and must have the same
-      number of rows as `self.inner_values`.
+      `self.flat_values`.  Must have `rank > 0`, and must have the same
+      number of rows as `self.flat_values`.
 
     Returns:
       A `RaggedTensor`.
@@ -509,46 +1033,369 @@ class RaggedTensor(object):
     if isinstance(self._values, ops.Tensor):
       return self.with_values(new_values)
     else:
-      return self.with_values(self.values.with_inner_values(new_values))
+      return self.with_values(self.values.with_flat_values(new_values))
+
+  #=============================================================================
+  # Tensor Type Conversions
+  #=============================================================================
+
+  @classmethod
+  def from_tensor(cls,
+                  tensor,
+                  lengths=None,
+                  padding=None,
+                  ragged_rank=1,
+                  name=None):
+    """Converts a `tf.Tensor` into a `RaggedTensor`.
+
+    The set of absent/default values may be specified using a vector of lengths
+    or a padding value (but not both).  If `lengths` is specified, then the
+    output tensor will satisfy `output[row] = tensor[row][:lengths[row]]`.
+    If `padding` is specified, then any row *suffix* consisting entirely of
+    `padding` will be excluded from the returned `RaggedTensor`.  If neither
+    `lengths` nor `padding` is specified, then the returned `RaggedTensor` will
+    have no absent/default values.
+
+    Examples:
+
+    ```python
+    >>> dt = tf.constant([[5, 7, 0], [0, 3, 0], [6, 0, 0]])
+    >>> tf.RaggedTensor.from_tensor(dt)
+    <tf.RaggedTensor [[5, 7, 0], [0, 3, 0], [6, 0, 0]]>
+    >>> tf.RaggedTensor.from_tensor(dt, lengths=[2, 0, 3])
+    <tf.RaggedTensor [[5, 7], [], [6, 0, 0]]>
+    >>> tf.RaggedTensor.from_tensor(dt, padding=0)
+    <tf.RaggedTensor [[5, 7], [0, 3], [6]]>
+    ```
+
+    Args:
+      tensor: The `Tensor` to convert.  Must have rank `ragged_rank + 1` or
+        higher.
+      lengths: An optional set of row lengths, specified using a 1-D integer
+        `Tensor` whose length is equal to `tensor.shape[0]` (the number of rows
+        in `tensor`).  If specified, then `output[row]` will contain
+        `tensor[row][:lengths[row]]`.  Negative lengths are treated as zero.
+      padding: An optional padding value.  If specified, then any row suffix
+        consisting entirely of `padding` will be excluded from the returned
+        RaggedTensor.  `padding` is a `Tensor` with the same dtype as `tensor`
+        and with `shape=tensor.shape[ragged_rank + 1:]`.
+      ragged_rank: Integer specifying the ragged rank for the returned
+        `RaggedTensor`.  Must be greater than zero.
+      name: A name prefix for the returned tensors (optional).
+
+    Returns:
+      A `RaggedTensor` with the specified `ragged_rank`.  The shape of the
+      returned ragged tensor is compatible with the shape of `tensor`.
+    Raises:
+      ValueError: If both `lengths` and `padding` are specified.
+    """
+    if lengths is not None and padding is not None:
+      raise ValueError("Specify lengths or padding, but not both")
+    if not isinstance(ragged_rank, int):
+      raise TypeError("ragged_rank expected int, got %r" % ragged_rank)
+    if ragged_rank <= 0:
+      raise ValueError(
+          "ragged_rank must be greater than 0; got %s" % ragged_rank)
+
+    with ops.name_scope(name, "RaggedFromTensor", [tensor, lengths, padding]):
+      tensor = ops.convert_to_tensor(tensor, name="tensor")
+      tensor.shape.with_rank_at_least(ragged_rank + 1)
+      input_shape = array_ops.shape(tensor, out_type=dtypes.int64)
+      ncols = input_shape[1]
+
+      # Handle ragged_rank>1 via recursion:
+      # If the output should have multiple ragged dimensions, then first
+      # flatten the tensor to eliminate all but the last ragged dimension,
+      # and recursively convert that flattened tensor.  Then add on the splits
+      # for the dimensions that we flattened out.
+      if ragged_rank > 1:
+        # Flatten `tensor` to eliminate all but the last ragged dimension.
+        new_shape = array_ops.concat([
+            constant_op.constant([-1], dtypes.int64), input_shape[ragged_rank:]
+        ],
+                                     axis=0)
+        flattened = array_ops.reshape(tensor, new_shape)
+        # Recursively convert the flattened tensor.
+        values = cls.from_tensor(flattened, lengths, padding)
+        # The total number of elements in each  dimension.  E.g., if
+        # input_shape=[3, 4, 5, 6], then dim[2] has 3*4*5 elements in total.
+        dim_size = math_ops.cumprod(input_shape)
+        # Construct splits tensors for the dimensions that were flattened.
+        new_splits = [
+            math_ops.range(0, dim_size[dim - 1] + 1) * input_shape[dim]
+            for dim in range(1, ragged_rank)
+        ]
+        return cls.from_nested_row_splits(values, new_splits)
+
+      # If padding was specified, then use it to find row lengths.
+      if padding is not None:
+        padding = ops.convert_to_tensor(
+            padding, name="padding", dtype=tensor.dtype)
+        padding.shape.assert_is_compatible_with(tensor.shape[2:])
+
+        # Find places where the padding is equal to the tensor.  (This will
+        # broadcast `padding` across the outermost 2 dimensions of `tensor`,
+        # so `has_default_value.shape = tensor.shape`.)
+        has_default_value = math_ops.equal(padding, tensor)
+
+        # If the padding isn't a scalar, then require that all values in the
+        # padding match each item in the tensor.  After this block of code,
+        # `has_default.shape = tensor.shape[:2]`.  (Unfortunately, we can't just
+        # use reduce_all for both cases, becaue when you pass an empty `axis`
+        # list to reduce_all, it reduces all axes; but we want it to reduce no
+        # axes -- i.e., to be a no-op.)
+        tensor_rank = array_ops.rank(tensor)
+        reduce_axis = math_ops.range(2, tensor_rank)
+        has_default = control_flow_ops.cond(
+            tensor_rank > 2,
+            lambda: math_ops.reduce_all(has_default_value, axis=reduce_axis),
+            lambda: has_default_value)
+        has_default.set_shape(tensor_shape.TensorShape([None, None]))
+        has_default.set_shape(tensor.shape[:2])
+
+        # Use has_default it to find the length of each row: for each
+        # non-default item in a row, calculate the length that the row needs to
+        # have to include that item; and then take the max of those values
+        # (across each row).
+        has_nondefault = math_ops.logical_not(has_default)
+        has_nondefault = math_ops.cast(has_nondefault, dtypes.int64)
+        length_for_nondefault_value = (
+            has_nondefault * array_ops.expand_dims(
+                math_ops.range(1, ncols + 1), 0))
+        lengths = math_ops.reduce_max(length_for_nondefault_value, axis=1)
+
+      # If we have lengths (either directly supplied, or computed from
+      # paddings), then use those to construct splits; and then use masking
+      # to get the corresponding values.
+      if lengths is not None:
+        lengths = ragged_util.convert_to_int_tensor(lengths, "lengths",
+                                                    dtypes.int64)
+        lengths.shape.assert_has_rank(1)
+        lengths = math_ops.minimum(lengths, ncols)
+        lengths = math_ops.maximum(lengths, 0)
+        limits = math_ops.cumsum(lengths)
+        splits = array_ops.concat([array_ops.zeros([1], dtypes.int64), limits],
+                                  axis=0)
+        mask = array_ops.sequence_mask(lengths, maxlen=ncols)
+        values = array_ops.boolean_mask(tensor, mask)
+        return cls.from_row_splits(values, splits)
+
+      # If neither padding nor lengths were specified, then create a splits
+      # vector that contains no default values, and reshape the input tensor
+      # to form the values for the RaggedTensor.
+      nrows = input_shape[0]
+      nvals = nrows * ncols
+      splits = math_ops.range(nrows + 1) * ncols
+      values_shape = array_ops.concat([[nvals], input_shape[2:]], axis=0)
+      values = array_ops.reshape(tensor, values_shape)
+      return cls.from_row_splits(values, splits)
+
+  def to_tensor(self, default_value=None, name=None):
+    """Converts this `RaggedTensor` into a `tf.Tensor`.
+
+    Example:
+
+    ```python
+    >>> rt = ragged.constant([[9, 8, 7], [], [6, 5], [4]])
+    >>> print rt.to_tensor()
+    [[9 8 7]
+     [0 0 0]
+     [6 5 0]
+     [4 0 0]]
+    ```
+
+    Args:
+      default_value: Value to set for indices not specified in `self`. Defaults
+        to zero.  `default_value` must be broadcastable to
+        `self.shape[self.ragged_rank + 1:]`.
+      name: A name prefix for the returned tensors (optional).
+
+    Returns:
+      A `Tensor` with shape `ragged.bounding_shape(self)` and the
+      values specified by the non-empty values in `self`.  Empty values are
+      assigned `default_value`.
+    """
+    with ops.name_scope(name, "RaggedToTensor", [self, default_value]):
+      if default_value is not None:
+        default_value = ops.convert_to_tensor(
+            default_value, name="default_value", dtype=self.dtype)
+
+      # If ragged_rank > 1, then recursively convert the ragged values into a
+      # `Tensor` before we proceed.
+      values = self.values
+      if is_ragged(values):
+        values = values.to_tensor(default_value)
+
+      # Tile the default value, if necessary.
+      if default_value is not None:
+        if values.shape.ndims is not None:
+          default_value.shape.with_rank_at_most(values.shape.ndims - 1)
+        if (values.shape.ndims is None or default_value.shape.ndims is None or
+            values.shape.ndims != default_value.shape.ndims + 1):
+          value_shape = array_ops.shape(values)[1:]
+          default_value = array_ops.broadcast_to(default_value, value_shape)
+        default_value.shape.assert_is_compatible_with(values.shape[1:])
+
+      # Get the expected dense shape ([nrows, ncols] + value_shape).
+      rt_row_lengths = [self.row_splits[1:] - self.row_splits[:-1]]
+      nrows = array_ops.shape(self.row_splits, out_type=dtypes.int64)[0] - 1
+      ncols = math_ops.maximum(math_ops.reduce_max(rt_row_lengths), 0)
+      values_shape = array_ops.shape(values, out_type=dtypes.int64)
+      value_shape = values_shape[1:]
+      nvals = values_shape[0]
+
+      # Build a default value if none was supplied.
+      if default_value is None:
+        default_value = array_ops.zeros(value_shape, dtype=values.dtype)
+      default_value.shape.assert_is_compatible_with(values.shape[1:])
+      default_value.set_shape(values.shape[1:])
+
+      # Get the row start indices, and expand to shape=[nrows, 1].
+      starts = array_ops.expand_dims(self.row_splits[:-1], 1)
+
+      # Get the row limit indices, and expand to shape=[nrows, 1].
+      limits = array_ops.expand_dims(self.row_splits[1:], 1)
+
+      # Get the column indices, and expand to shape=[1, ncols].
+      columns = array_ops.expand_dims(math_ops.range(0, ncols), 0)
+
+      # Build a list containing the values plus the default value.  We will use
+      # tf.gather to collect values from this list for the `Tensor` (using
+      # nvals as the index for the default value).
+      values_and_default = array_ops.concat(
+          [values, array_ops.stack([default_value])], axis=0)
+
+      # Construct a matrix "indices" pointing into values_and_default.  I.e.,
+      # output[r, c] = values_and_default[indices[r, c].
+      nondefault_index = starts + columns
+      has_value = nondefault_index < limits
+      default_index = array_ops.fill(array_ops.stack([nrows, ncols]), nvals)
+      indices = array_ops.where(has_value, nondefault_index, default_index)
+
+      # Gather the results into a `Tensor`.
+      return array_ops.gather(values_and_default, indices)
+
+  @classmethod
+  def from_sparse(cls, st_input, name=None):
+    """Converts a 2D `tf.SparseTensor` to a `RaggedTensor`.
+
+    Each row of the `output` `RaggedTensor` will contain the explicit values
+    from the same row in `st_input`.  `st_input` must be ragged-right.  If not
+    it is not ragged-right, then an error will be generated.
+
+    Example:
+
+    ```python
+    >>> st = SparseTensor(indices=[[0, 1], [0, 2], [0, 3], [1, 0], [3, 0]],
+    ...                   values=[1, 2, 3, 4, 5],
+    ...                   dense_shape=[4, 3])
+    >>> rt.RaggedTensor.from_sparse(st).eval().tolist()
+    [[1, 2, 3], [4], [], [5]]
+    ```
+
+    Currently, only two-dimensional `SparseTensors` are supported.
+
+    Args:
+      st_input: The sparse tensor to convert.  Must have rank 2.
+      name: A name prefix for the returned tensors (optional).
+
+    Returns:
+      A `RaggedTensor` with the same values as `st_input`.
+      `output.ragged_rank = rank(st_input) - 1`.
+      `output.shape = [st_input.dense_shape[0], None]`.
+    Raises:
+      ValueError: If the number of dimensions in `st_input` is not known
+        statically, or is not two.
+    """
+    if not sparse_tensor.is_sparse(st_input):
+      raise TypeError("Expected SparseTensor, got %s" % type(st_input).__name__)
+    with ops.name_scope(name, "RaggedFromSparse", [st_input]):
+      st_input = sparse_tensor.convert_to_tensor_or_sparse_tensor(
+          st_input, name="st_input")
+
+      if st_input.dense_shape.shape.ndims is None:
+        static_rank_from_dense_shape = None
+      else:
+        static_rank_from_dense_shape = st_input.dense_shape.shape.dims[0].value
+
+      if st_input.indices.shape.ndims is None:
+        static_rank_from_indices = None
+      else:
+        static_rank_from_indices = st_input.indices.shape.dims[1].value
+
+      if static_rank_from_dense_shape != 2 and static_rank_from_indices != 2:
+        raise ValueError("rank(st_input) must be 2")
+
+      with ops.control_dependencies(
+          _assert_sparse_indices_are_ragged_right(st_input.indices)):
+        # Treat sparse row indices as segment ids to generate a splits tensor
+        # thta we can pair with the sparse tensor values.  (Ignore sparse column
+        # indices.)
+        segment_ids = st_input.indices[:, 0]
+        num_segments = st_input.dense_shape[0]
+        return cls.from_value_rowids(st_input.values, segment_ids, num_segments)
+
+  def to_sparse(self, name=None):
+    """Converts this `RaggedTensor` into a `tf.SparseTensor`.
+
+    Example:
+
+    ```python
+    >>> rt = ragged.constant([[1, 2, 3], [4], [], [5, 6]])
+    >>> rt.to_sparse().eval()
+    SparseTensorValue(indices=[[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [3, 1]],
+                      values=[1, 2, 3, 4, 5, 6],
+                      dense_shape=[4, 3])
+    ```
+
+    Args:
+      name: A name prefix for the returned tensors (optional).
+
+    Returns:
+      A SparseTensor with the same values as `self`.
+    """
+    with ops.name_scope(name, "RaggedToSparse", [self]):
+      result = gen_ragged_conversion_ops.ragged_tensor_to_sparse(
+          self.nested_row_splits, self.flat_values, name=name)
+      return sparse_tensor.SparseTensor(result.sparse_indices,
+                                        result.sparse_values,
+                                        result.sparse_dense_shape)
 
   #=============================================================================
   # String Encoding
   #=============================================================================
   def __str__(self):
     if self._is_eager():
-      return "RaggedTensor(%s)" % self.tolist()
+      return "<tf.RaggedTensor %s>" % self.to_list()
     else:
       return self.__repr__()
 
   def __repr__(self):
-    return "RaggedTensor(values=%s, row_splits=%s)" % (self._values,
-                                                       self._row_splits)
+    return "tf.RaggedTensor(values=%s, row_splits=%s)" % (self._values,
+                                                          self._row_splits)
 
   #=============================================================================
   # Eager Execution Mode
   #=============================================================================
 
-  def tolist(self):
+  def to_list(self):
     """Returns a nested Python `list` with the values for this `RaggedTensor`.
 
-    If a `RaggedTensor` `rt` was constructed in graph execution mode, then
-    `rt.tolist()` is equivalent to `rt.eval().tolist()`.
-
-    If a `RaggedTensor` `rt` was constructed in eager execution mode, then
-    `rt.tolist()` builds the Python list based on `rt`'s `EagerTensor`
-    components.
+    Requires that `rt` was constructed in eager execution mode.
 
     Returns:
       A nested Python `list`.
     """
     if self._is_eager():
-      return self._eager_value().tolist()
+      return self._eager_value().to_list()
     else:
-      return self.eval().tolist()
+      raise ValueError("RaggedTensor.to_list() is only supported in eager "
+                       "mode; in graph mode, evaluate the RaggedTensor first "
+                       "and then use RaggedTensorValue.to_list().")
 
   def _eager_value(self):
     """Returns a RaggedTensorValue for self.  Requires self._is_eager()=true."""
-    value = self.inner_values.numpy()
+    value = self.flat_values.numpy()
     for row_splits in reversed(self.nested_row_splits):
       value = ragged_tensor_value.RaggedTensorValue(value, row_splits.numpy())
     return value
@@ -562,24 +1409,6 @@ class RaggedTensor(object):
       rt = rt.values
     return isinstance(rt, ops.EagerTensor)
 
-  #=============================================================================
-  # Evaluation
-  #=============================================================================
-  def eval(self, feed_dict=None, session=None):  # pylint: disable=redefined-outer-name
-    """Evaluates this ragged tensor in a `Session`.
-
-    Args:
-      feed_dict: A dictionary that maps `Tensor` objects to feed values. See
-        `tf.Session.run` for a description of the valid feed values.
-      session: The `Session` to be used to evaluate this ragged tensor. If none,
-        the default session will be used.
-
-    Returns:
-      A `RaggedTensorValue` object.
-    """
-    return _eval_using_default_session(self, feed_dict,
-                                       self._as_graph_element().graph, session)
-
   #=============================================================================
   # Indexing & Slicing
   #=============================================================================
@@ -613,6 +1442,53 @@ def is_ragged(value):
                     (RaggedTensor, ragged_tensor_value.RaggedTensorValue))
 
 
+#===============================================================================
+# Convert value -> tensor
+#===============================================================================
+def convert_to_tensor_or_ragged_tensor(value,
+                                       dtype=None,
+                                       preferred_dtype=None,
+                                       name=None):
+  """Converts value to a `RaggedTensor` or `Tensor`.
+
+  * If `value` is a `RaggedTensor`, then return it as-is.
+  * If `value` is a `RaggedTensorValue`, return a corresponding constant
+    `RaggedTensor`.
+  * Otherwise, use `convert_to_tensor` to convert `value` to a `Tensor`.
+
+  Args:
+    value: A `RaggedTensor`, a `RaggedTensorValue`, or an object whose type has
+      a registered `Tensor` conversion function.
+    dtype: Optional element type for the returned tensor.  If missing the type
+      is inferred from the type of `value`.
+    preferred_dtype: Optional element type for the returned tensor, used when
+      dtype is None.  This argument has no effect if `value` is already a
+      tensor, or when conversion is not possible.
+    name: Optional name to use if a new `Tensor` is created.
+
+  Returns:
+    A `Tensor` or `RaggedTensor`.
+  """
+  if isinstance(value, RaggedTensor):
+    if dtype and not dtype.is_compatible_with(value.dtype):
+      raise ValueError("Tensor conversion requested dtype %s for "
+                       "RaggedTensor with dtype %s: %r" %
+                       (dtype.name, value.dtype.name, value))
+    return value
+  elif isinstance(value, ragged_tensor_value.RaggedTensorValue):
+    with ops.name_scope(name, "ConvertToTensorOrRaggedTensor", []):
+      flat_values = ops.convert_to_tensor(
+          value=value.flat_values,
+          dtype=dtype,
+          preferred_dtype=preferred_dtype,
+          name="flat_values")
+      return RaggedTensor.from_nested_row_splits(flat_values,
+                                                 value.nested_row_splits)
+  else:
+    return ops.convert_to_tensor(
+        value=value, dtype=dtype, preferred_dtype=preferred_dtype, name=name)
+
+
 #===============================================================================
 # Register RaggedTensor for use with session.run.
 #===============================================================================
@@ -625,18 +1501,18 @@ def _ragged_tensor_value_from_components(components):
 
 
 def _ragged_tensor_session_fetch(rt):
-  components = rt.nested_row_splits + (rt.inner_values,)
+  components = rt.nested_row_splits + (rt.flat_values,)
   return (components, _ragged_tensor_value_from_components)
 
 
 def _ragged_tensor_session_feed(feed_key, feed_val):
-  key_components = feed_key.nested_row_splits + (feed_key.inner_values,)
-  val_components = feed_val.nested_row_splits + (feed_val.inner_values,)
+  key_components = feed_key.nested_row_splits + (feed_key.flat_values,)
+  val_components = feed_val.nested_row_splits + (feed_val.flat_values,)
   return zip(key_components, val_components)
 
 
 def _ragged_tensor_session_feed_for_partial_run(feed_key):
-  return feed_key.nested_row_splits + (feed_key.inner_values,)
+  return feed_key.nested_row_splits + (feed_key.flat_values,)
 
 
 session.register_session_run_conversion_functions(
@@ -644,6 +1520,9 @@ session.register_session_run_conversion_functions(
     _ragged_tensor_session_feed_for_partial_run)
 
 
+#===============================================================================
+# RaggedTensorType
+#===============================================================================
 class RaggedTensorType(object):
   """Encoding of a static type for a `RaggedTensor`.
 
@@ -663,3 +1542,67 @@ class RaggedTensorType(object):
 
   dtype = property(lambda self: self._dtype)
   ragged_rank = property(lambda self: self._ragged_rank)
+
+
+#===============================================================================
+# Helper Functions
+#===============================================================================
+def _assert_sparse_indices_are_ragged_right(indices):
+  """Checks that the given SparseTensor.indices tensor is ragged-right.
+
+  Example: `indices = [[0, 0], [0, 1], [2, 0], [3, 1]]` is not ragged right
+  because the entry `[3, 1]` skips a cell.
+
+  Args:
+    indices: The SparseTensor indices to check.
+
+  Returns:
+    A list of control dependency op tensors.
+  """
+  index_prefix = indices[:, :-1]
+  index_suffix = indices[:, -1]
+
+  # Check whether each index is starting a new row in the innermost dimension
+  # (prefix[i] != prefix[i-1]) or continuing a row (prefix[i] == prefix[i-1]).
+  # (Note: this skips the first index; we will check that separately below.)
+  index_prefix_changed = math_ops.reduce_any(
+      math_ops.not_equal(index_prefix[1:], index_prefix[:-1]), axis=1)
+
+  # Check two cases:
+  #   * For indices that start a new row: index_suffix[i] must be zero.
+  #   * For indices that continue a row: index_suffix[i] must be equal to
+  #     index_suffix[i-1]+1.
+  index_ok = array_ops.where(
+      index_prefix_changed, math_ops.equal(index_suffix[1:], 0),
+      math_ops.equal(index_suffix[1:], index_suffix[:-1] + 1))
+
+  # Also check that the very first index didn't skip any cells.  The first
+  # index starts a new row (by definition), so its suffix should be zero.
+  sparse_indices_are_ragged_right = math_ops.logical_and(
+      math_ops.reduce_all(math_ops.equal(index_suffix[:1], 0)),
+      math_ops.reduce_all(index_ok))
+
+  message = [
+      "SparseTensor is not right-ragged", "SparseTensor.indices =", indices
+  ]
+  return [control_flow_ops.Assert(sparse_indices_are_ragged_right, message)]
+
+
+@ops.RegisterGradient("RaggedTensorToSparse")
+def _ragged_tensor_to_sparse_gradient(op, unused_sparse_indices_grad,
+                                      sparse_values_grad,
+                                      unused_sparse_shape_grad):
+  """Gradient for RaggedTensorToSparse."""
+  op_inputs_nested_row_splits = op.inputs[:-1]
+  op_inputs_flat_values = op.inputs[-1]
+
+  # No gradient for the RaggedTensor's nested_row_splits.
+  nested_row_splits_gradient = [None] * len(op_inputs_nested_row_splits)
+
+  # Gradient for the RaggedTensor's flat_values is formed by reshaping
+  # the gradient for the SparseTensor's values.
+  flat_values_shape = array_ops.shape(op_inputs_flat_values)
+  flat_values_gradient = array_ops.reshape(sparse_values_grad,
+                                           flat_values_shape)
+
+  return nested_row_splits_gradient + [flat_values_gradient]
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
index befe30f0e1..4e6ebdf332 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_bounding_shape_op_test.py
@@ -20,47 +20,42 @@ from __future__ import print_function
 
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorBoundingShapeOp(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorBoundingShapeOp(ragged_test_util.RaggedTensorTestCase):
 
   def testDocStringExample(self):
     # This is the example from ragged.bounding_shape.__doc__.
     rt = ragged.constant([[1, 2, 3, 4], [5], [], [6, 7, 8, 9], [10]])
-    self.assertEqual(self.evaluate(ragged.bounding_shape(rt)).tolist(), [5, 4])
+    self.assertRaggedEqual(rt.bounding_shape(), [5, 4])
 
   def test2DRaggedTensorWithOneRaggedDimension(self):
     values = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
-    rt1 = ragged.from_row_splits(values, [0, 2, 5, 6, 6, 7])
-    rt2 = ragged.from_row_splits(values, [0, 7])
-    rt3 = ragged.from_row_splits(values, [0, 0, 7, 7])
-    self.assertEqual(self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3])
-    self.assertEqual(self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7])
-    self.assertEqual(self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7])
+    rt1 = ragged.RaggedTensor.from_row_splits(values, [0, 2, 5, 6, 6, 7])
+    rt2 = ragged.RaggedTensor.from_row_splits(values, [0, 7])
+    rt3 = ragged.RaggedTensor.from_row_splits(values, [0, 0, 7, 7])
+    self.assertRaggedEqual(rt1.bounding_shape(), [5, 3])
+    self.assertRaggedEqual(rt2.bounding_shape(), [1, 7])
+    self.assertRaggedEqual(rt3.bounding_shape(), [3, 7])
 
   def test3DRaggedTensorWithOneRaggedDimension(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
-    rt1 = ragged.from_row_splits(values, [0, 2, 5, 6, 6, 7])
-    rt2 = ragged.from_row_splits(values, [0, 7])
-    rt3 = ragged.from_row_splits(values, [0, 0, 7, 7])
-    self.assertEqual(
-        self.evaluate(ragged.bounding_shape(rt1)).tolist(), [5, 3, 2])
-    self.assertEqual(
-        self.evaluate(ragged.bounding_shape(rt2)).tolist(), [1, 7, 2])
-    self.assertEqual(
-        self.evaluate(ragged.bounding_shape(rt3)).tolist(), [3, 7, 2])
-
-  def testNonRaggedTensor(self):
-    dt = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]]
-    self.assertEqual(self.evaluate(ragged.bounding_shape(dt)).tolist(), [4, 3])
+    rt1 = ragged.RaggedTensor.from_row_splits(values, [0, 2, 5, 6, 6, 7])
+    rt2 = ragged.RaggedTensor.from_row_splits(values, [0, 7])
+    rt3 = ragged.RaggedTensor.from_row_splits(values, [0, 0, 7, 7])
+    self.assertRaggedEqual(rt1.bounding_shape(), [5, 3, 2])
+    self.assertRaggedEqual(rt2.bounding_shape(), [1, 7, 2])
+    self.assertRaggedEqual(rt3.bounding_shape(), [3, 7, 2])
 
   def testExplicitAxisOptimizations(self):
-    rt = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
-    self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 0)).tolist(), 5)
-    self.assertEqual(self.evaluate(ragged.bounding_shape(rt, 1)).tolist(), 3)
-    self.assertEqual(
-        self.evaluate(ragged.bounding_shape(rt, [1, 0])).tolist(), [3, 5])
+    rt = ragged.RaggedTensor.from_row_splits(b'a b c d e f g'.split(),
+                                             [0, 2, 5, 6, 6, 7])
+    self.assertRaggedEqual(rt.bounding_shape(0), 5)
+    self.assertRaggedEqual(rt.bounding_shape(1), 3)
+    self.assertRaggedEqual(rt.bounding_shape([1, 0]), [3, 5])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape.py b/tensorflow/python/ops/ragged/ragged_tensor_shape.py
index 9129b4b10b..706881da74 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_shape.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_shape.py
@@ -21,13 +21,13 @@ from __future__ import print_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_util
 
@@ -55,7 +55,7 @@ class RaggedTensorDynamicShape(object):
       be ragged.
 
     * "Inner dimensions" are dimensions that are encoded using a
-      `RaggedTensor`'s `inner_values`.  Inner dimensions are always uniform.
+      `RaggedTensor`'s `flat_values`.  Inner dimensions are always uniform.
 
   The sizes of partitioned dimensions are recorded using `partitioned_dim_sizes`
   and `inner_dim_sizes`:
@@ -161,15 +161,15 @@ class RaggedTensorDynamicShape(object):
   def from_tensor(cls, rt_input):
     """Constructs a ragged shape for a potentially ragged tensor."""
     with ops.name_scope(None, 'RaggedTensorDynamicShapeFromTensor', [rt_input]):
-      rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(rt_input)
+      rt_input = ragged_tensor.convert_to_tensor_or_ragged_tensor(rt_input)
       if not ragged_tensor.is_ragged(rt_input):
         return cls([], array_ops.shape(rt_input))
       else:
-        partitioned_dim_sizes = ((ragged_array_ops.nrows(rt_input),) +
-                                 ragged_array_ops.nested_row_lengths(rt_input))
+        partitioned_dim_sizes = (
+            (rt_input.nrows(),) + rt_input.nested_row_lengths())
         return RaggedTensorDynamicShape(
             partitioned_dim_sizes,
-            array_ops.shape(rt_input.inner_values)[1:])
+            array_ops.shape(rt_input.flat_values)[1:])
 
   def dimension_size(self, axis):
     """Returns the size of slices across the specified dimension."""
@@ -197,7 +197,7 @@ class RaggedTensorDynamicShape(object):
   @property
   def rank(self):
     """The number of dimensions in this shape, or None if unknown."""
-    inner_ndims = self._inner_dim_sizes.shape[0].value
+    inner_ndims = tensor_shape.dimension_value(self._inner_dim_sizes.shape[0])
     if inner_ndims is None:
       return None
     else:
@@ -229,7 +229,7 @@ class RaggedTensorDynamicShape(object):
   @property
   def num_inner_dimensions(self):
     """The number of inner dimensions, or `None` if not statically known."""
-    return self._inner_dim_sizes.shape[0].value
+    return tensor_shape.dimension_value(self._inner_dim_sizes.shape[0])
 
   def broadcast_to_rank(self, rank):
     """Adds leading size-1 dimensions to broadcast `self` to the given rank.
@@ -456,7 +456,7 @@ def broadcast_to(rt_input, shape, broadcast_inner_dimensions=True):
   """
   if not isinstance(shape, RaggedTensorDynamicShape):
     raise TypeError('shape must be a RaggedTensorDynamicShape')
-  rt_input = ragged_factory_ops.convert_to_tensor_or_ragged_tensor(rt_input)
+  rt_input = ragged_tensor.convert_to_tensor_or_ragged_tensor(rt_input)
 
   # Broadcasting to a uniform shape.
   if shape.num_partitioned_dimensions == 0:
@@ -497,17 +497,20 @@ def _broadcast_to_ragged_shape(rt_input, dst_shape, broadcast_inner_dimensions):
       rt_input = array_ops.reshape(
           rt_input, array_ops.concat([[-1], dst_shape.inner_dim_sizes], axis=0))
     for _ in range(dst_shape.rank - rt_input.shape.ndims):
-      rt_input = ragged_factory_ops.from_row_lengths(
-          rt_input, [ragged_array_ops.nrows(rt_input)])
+      if ragged_tensor.is_ragged(rt_input):
+        nrows = rt_input.nrows()
+      else:
+        nrows = array_ops.shape(rt_input, out_type=dtypes.int64)[0]
+      rt_input = ragged_tensor.RaggedTensor.from_row_lengths(rt_input, [nrows])
 
   # Add ragged dimensions to match dst_shape.
   if ragged_tensor.is_ragged(rt_input):
     inner_rank_diff = (
-        rt_input.inner_values.shape.ndims - 1 - dst_shape.num_inner_dimensions)
+        rt_input.flat_values.shape.ndims - 1 - dst_shape.num_inner_dimensions)
     if inner_rank_diff > 0:
-      rt_input = rt_input.with_inner_values(
+      rt_input = rt_input.with_flat_values(
           ragged_conversion_ops.from_tensor(
-              rt_input.inner_values, ragged_rank=inner_rank_diff))
+              rt_input.flat_values, ragged_rank=inner_rank_diff))
   else:
     rt_input = ragged_conversion_ops.from_tensor(
         rt_input, ragged_rank=dst_shape.num_partitioned_dimensions - 1)
@@ -528,9 +531,9 @@ def _broadcast_to_ragged_shape(rt_input, dst_shape, broadcast_inner_dimensions):
     rt_input = ragged_array_ops.tile(rt_input, multiples)
 
   if broadcast_inner_dimensions:
-    rt_input = rt_input.with_inner_values(
+    rt_input = rt_input.with_flat_values(
         array_ops.reshape(
-            rt_input.inner_values,
+            rt_input.flat_values,
             array_ops.concat([[-1], dst_shape.inner_dim_sizes], axis=0)))
 
   # Do broadcasting for dimensions that become ragged.  We must do these from
@@ -555,7 +558,7 @@ def _ragged_tile_axis(rt_input, axis, repeats):
         _ragged_tile_axis(rt_input.values, axis - 1, repeats))
   else:
     src_row_splits = rt_input.nested_row_splits
-    src_row_lengths = ragged_array_ops.nested_row_lengths(rt_input)
+    src_row_lengths = rt_input.nested_row_lengths()
     splits = src_row_splits[0]
 
     dst_row_lengths = [repeats]
@@ -563,8 +566,7 @@ def _ragged_tile_axis(rt_input, axis, repeats):
       dst_row_lengths.append(
           ragged_util.repeat_ranges(src_row_lengths[i], splits, repeats))
       splits = array_ops.gather(src_row_splits[i], splits)
-    dst_values = ragged_util.repeat_ranges(rt_input.inner_values, splits,
+    dst_values = ragged_util.repeat_ranges(rt_input.flat_values, splits,
                                            repeats)
-    return ragged_factory_ops.from_nested_row_lengths(dst_values,
-                                                      dst_row_lengths)
-
+    return ragged_tensor.RaggedTensor.from_nested_row_lengths(
+        dst_values, dst_row_lengths)
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
index 699dcc2bdb..ec06aeaea5 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_shape_test.py
@@ -24,26 +24,27 @@ import numpy as np
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
-                            parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorBoundingShapeOp(ragged_test_util.RaggedTensorTestCase,
+                                  parameterized.TestCase):
 
   def assertShapeEq(self, x, y):
     assert isinstance(x, ragged.RaggedTensorDynamicShape)
     assert isinstance(y, ragged.RaggedTensorDynamicShape)
     x_partitioned_dim_sizes = [
-        splits.eval().tolist()  #
+        self.eval_to_list(splits)  #
         for splits in x.partitioned_dim_sizes
     ]
     y_partitioned_dim_sizes = [
-        splits.eval().tolist()  #
+        self.eval_to_list(splits)  #
         for splits in y.partitioned_dim_sizes
     ]
     self.assertEqual(x_partitioned_dim_sizes, y_partitioned_dim_sizes)
-    self.assertEqual(x.inner_dim_sizes.eval().tolist(),
-                     y.inner_dim_sizes.eval().tolist())
+    self.assertAllEqual(x.inner_dim_sizes, y.inner_dim_sizes)
 
   @parameterized.parameters([
       dict(value='x', expected_dim_sizes=[]),
@@ -82,13 +83,11 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
           value=ragged.constant_value([[[1, 2], [3]], [[4, 5]]]),
           expected_dim_sizes=[2, [2, 1], [2, 1, 2]]),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testFromTensor(self, value, expected_dim_sizes):
     shape = ragged.RaggedTensorDynamicShape.from_tensor(value)
     expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(
         expected_dim_sizes)
-    with self.cached_session():
-      self.assertShapeEq(shape, expected)
+    self.assertShapeEq(shape, expected)
 
   @parameterized.parameters([
       dict(dim_sizes=[], rank=0, expected_dim_sizes=[]),
@@ -106,15 +105,13 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
           rank=5,
           expected_dim_sizes=[1, 3, [3, 2, 4], 2, 3]),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testBroadcastToRank(self, dim_sizes, rank, expected_dim_sizes):
     shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes)
     expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(
         expected_dim_sizes)
     broadcasted_shape = shape.broadcast_to_rank(rank)
-    with self.cached_session():
-      self.assertShapeEq(broadcasted_shape, expected)
-      self.assertEqual(broadcasted_shape.rank, rank)
+    self.assertShapeEq(broadcasted_shape, expected)
+    self.assertEqual(broadcasted_shape.rank, rank)
 
   @parameterized.parameters([
       #=========================================================================
@@ -283,7 +280,6 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
            original_dim_sizes=[2, (2, 1), 2, 1],
            broadcast_dim_sizes=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
   ])  # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testBroadcastDimension(self, axis, row_length, original_dim_sizes,
                              broadcast_dim_sizes):
     """Tests for the broadcast_dimension method.
@@ -306,17 +302,16 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
     broadcast_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(
         broadcast_dim_sizes)
     self.assertEqual(original_shape.rank, broadcast_shape.rank)
-    with self.cached_session():
-      # shape[axis].value == 1 and row_length > 1:
-      bcast1 = original_shape.broadcast_dimension(axis, row_length)
-      # shape[axis].value > 1 and row_length == shape[axis].value:
-      bcast2 = broadcast_shape.broadcast_dimension(axis, row_length)
-      # shape[axis].value > 1 and row_length == 1:
-      bcast3 = broadcast_shape.broadcast_dimension(axis, 1)
-
-      self.assertShapeEq(bcast1, broadcast_shape)
-      self.assertShapeEq(bcast2, broadcast_shape)
-      self.assertShapeEq(bcast3, broadcast_shape)
+    # shape[axis].value == 1 and row_length > 1:
+    bcast1 = original_shape.broadcast_dimension(axis, row_length)
+    # shape[axis].value > 1 and row_length == shape[axis].value:
+    bcast2 = broadcast_shape.broadcast_dimension(axis, row_length)
+    # shape[axis].value > 1 and row_length == 1:
+    bcast3 = broadcast_shape.broadcast_dimension(axis, 1)
+
+    self.assertShapeEq(bcast1, broadcast_shape)
+    self.assertShapeEq(bcast2, broadcast_shape)
+    self.assertShapeEq(bcast3, broadcast_shape)
 
   @parameterized.parameters(
       [
@@ -373,16 +368,14 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
               y_dims=[1, 1, 2, (2, 1)],
               expected_dims=[2, (2, 1), 2, (2, 1, 2, 1, 2, 1)]),
       ])
-  @test_util.run_v1_only('b/120545219')
   def testBroadcastDynamicShape(self, x_dims, y_dims, expected_dims):
     x_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(x_dims)
     y_shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(y_dims)
     expected = ragged.RaggedTensorDynamicShape.from_dim_sizes(expected_dims)
     result1 = ragged.broadcast_dynamic_shape(x_shape, y_shape)
     result2 = ragged.broadcast_dynamic_shape(y_shape, x_shape)
-    with self.cached_session():
-      self.assertShapeEq(expected, result1)
-      self.assertShapeEq(expected, result2)
+    self.assertShapeEq(expected, result1)
+    self.assertShapeEq(expected, result2)
 
   def testRepr(self):
     shape = ragged.RaggedTensorDynamicShape.from_dim_sizes([2, (2, 1), 2, 1])
@@ -420,17 +413,12 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
           dim_sizes=[3, [3, 0, 2]],
           expected=ragged.constant_value([[10, 10, 10], [], [10, 10]])),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testRaggedBroadcastTo(self, x, dim_sizes, expected):
     shape = ragged.RaggedTensorDynamicShape.from_dim_sizes(dim_sizes)
     result = ragged.broadcast_to(x, shape)
-    with self.cached_session():
-      self.assertEqual(
-          getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank',
-                                                     0))
-      if hasattr(expected, 'tolist'):
-        expected = expected.tolist()
-      self.assertEqual(result.eval().tolist(), expected)
+    self.assertEqual(
+        getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0))
+    self.assertRaggedEqual(result, expected)
 
   @parameterized.parameters([
       dict(
@@ -475,7 +463,6 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
               [[[[11, 21], [32]], [[13, 23], [34]]],
                [[[15, 25], [36]]]])),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testRaggedAddWithBroadcasting(self, x, y, expected, doc):
     expected_rrank = getattr(expected, 'ragged_rank', 0)
     x = ragged.convert_to_tensor_or_ragged_tensor(x, dtype=dtypes.int32)
@@ -485,8 +472,7 @@ class RaggedTensorShapeTest(test_util.TensorFlowTestCase,
     self.assertEqual(expected_rrank, result_rrank)
     if hasattr(expected, 'tolist'):
       expected = expected.tolist()
-    with self.cached_session():
-      self.assertEqual(result.eval().tolist(), expected)
+    self.assertRaggedEqual(result, expected)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py
index e86676f70a..b8f1d97137 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py
@@ -19,17 +19,20 @@ from __future__ import division
 from __future__ import print_function
 
 import re
-import sys
 
 from absl.testing import parameterized
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.ops.ragged import RaggedTensor
 from tensorflow.python.platform import googletest
 
 
@@ -103,63 +106,62 @@ EXAMPLE_RAGGED_TENSOR_4D = [
 EXAMPLE_RAGGED_TENSOR_4D_SPLITS1 = [0, 2, 2, 3, 4]
 EXAMPLE_RAGGED_TENSOR_4D_SPLITS2 = [0, 3, 6, 9, 10]
 EXAMPLE_RAGGED_TENSOR_4D_VALUES = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10],
-                                   [11, 12], [13, 14], [15, 16], [17,
-                                                                  18], [19, 20]]
+                                   [11, 12], [13, 14], [15, 16], [17, 18],
+                                   [19, 20]]
 
 
-class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorTest(ragged_test_util.RaggedTensorTestCase,
+                       parameterized.TestCase):
   longMessage = True  # Property in unittest.Testcase. pylint: disable=invalid-name
 
   #=============================================================================
   # RaggedTensor class docstring examples
   #=============================================================================
 
-  @test_util.run_deprecated_v1
   def testClassDocStringExamples(self):
     # From section: "Component Tensors"
-    rt = ragged.from_row_splits(
+    rt = RaggedTensor.from_row_splits(
         values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8])
-    self.assertEqual(
-        self.evaluate(rt).tolist(), [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+    self.assertRaggedEqual(rt, [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
     del rt
 
     # From section: "Alternative Row-Partitioning Schemes"
     values = [3, 1, 4, 1, 5, 9, 2, 6]
-    rt1 = ragged.from_row_splits(values, row_splits=[0, 4, 4, 7, 8, 8])
-    rt2 = ragged.from_row_lengths(values, row_lengths=[4, 0, 3, 1, 0])
-    rt3 = ragged.from_value_rowids(
+    rt1 = RaggedTensor.from_row_splits(values, row_splits=[0, 4, 4, 7, 8, 8])
+    rt2 = RaggedTensor.from_row_lengths(values, row_lengths=[4, 0, 3, 1, 0])
+    rt3 = RaggedTensor.from_value_rowids(
         values, value_rowids=[0, 0, 0, 0, 2, 2, 2, 3], nrows=5)
-    rt4 = ragged.from_row_starts(values, row_starts=[0, 4, 4, 7, 8])
-    rt5 = ragged.from_row_limits(values, row_limits=[4, 4, 7, 8, 8])
+    rt4 = RaggedTensor.from_row_starts(values, row_starts=[0, 4, 4, 7, 8])
+    rt5 = RaggedTensor.from_row_limits(values, row_limits=[4, 4, 7, 8, 8])
     for rt in (rt1, rt2, rt3, rt4, rt5):
-      self.assertEqual(
-          self.evaluate(rt).tolist(), [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
+      self.assertRaggedEqual(rt, [[3, 1, 4, 1], [], [5, 9, 2], [6], []])
     del rt1, rt2, rt3, rt4, rt5
 
     # From section: "Multiple Ragged Dimensions"
-    inner_rt = ragged.from_row_splits(
+    inner_rt = RaggedTensor.from_row_splits(
         values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8])
-    outer_rt = ragged.from_row_splits(values=inner_rt, row_splits=[0, 3, 3, 5])
+    outer_rt = RaggedTensor.from_row_splits(
+        values=inner_rt, row_splits=[0, 3, 3, 5])
     self.assertEqual(outer_rt.ragged_rank, 2)
     self.assertEqual(
-        self.evaluate(outer_rt).tolist(),
+        self.eval_to_list(outer_rt),
         [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
     del inner_rt, outer_rt
 
     # From section: "Multiple Ragged Dimensions"
-    rt = ragged.from_nested_row_splits(
-        inner_values=[3, 1, 4, 1, 5, 9, 2, 6],
+    rt = RaggedTensor.from_nested_row_splits(
+        flat_values=[3, 1, 4, 1, 5, 9, 2, 6],
         nested_row_splits=([0, 3, 3, 5], [0, 4, 4, 7, 8, 8]))
     self.assertEqual(
-        self.evaluate(rt).tolist(),
-        [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
+        self.eval_to_list(rt), [[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]])
     del rt
 
     # From section: "Uniform Inner Dimensions"
-    rt = ragged.from_row_splits(
+    rt = RaggedTensor.from_row_splits(
         values=array_ops.ones([5, 3]), row_splits=[0, 2, 5])
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[[1, 1, 1], [1, 1, 1]], [[1, 1, 1], [1, 1, 1], [1, 1, 1]]])
     self.assertEqual(rt.shape.as_list(), [2, None, 3])
     del rt
@@ -181,7 +183,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(splits, rt_value.row_splits)
     self.assertAllEqual(values, rt_value.values)
     self.assertAllEqual(splits, rt_value.nested_row_splits[0])
-    self.assertAllEqual(values, rt_value.inner_values)
+    self.assertAllEqual(values, rt_value.flat_values)
 
     # Test construction of a RaggedTensorValue with ragged_rank=2.
     rt_value = ragged.RaggedTensorValue(
@@ -194,21 +196,19 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(splits2, rt_value.nested_row_splits[0])
     self.assertAllEqual(splits, rt_value.nested_row_splits[1])
     self.assertAllEqual(values, rt_value.values.values)
-    self.assertAllEqual(values, rt_value.inner_values)
+    self.assertAllEqual(values, rt_value.flat_values)
 
   #=============================================================================
   # RaggedTensor Constructor (private)
   #=============================================================================
 
-  @test_util.run_deprecated_v1
   def testRaggedTensorConstruction(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
-    rt = ragged.RaggedTensor(
-        values=values, row_splits=row_splits, internal=True)
+    rt = RaggedTensor(values=values, row_splits=row_splits, internal=True)
 
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testRaggedTensorConstructionErrors(self):
@@ -217,117 +217,118 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     with self.assertRaisesRegexp(ValueError,
                                  'RaggedTensor constructor is private'):
-      ragged.RaggedTensor(values=values, row_splits=row_splits)
+      RaggedTensor(values=values, row_splits=row_splits)
 
     with self.assertRaisesRegexp(TypeError,
                                  'values must be a Tensor or RaggedTensor'):
-      ragged.RaggedTensor(values=range(7), row_splits=row_splits, internal=True)
+      RaggedTensor(values=range(7), row_splits=row_splits, internal=True)
 
     with self.assertRaisesRegexp(TypeError,
                                  'Row-partitioning argument must be a Tensor'):
-      ragged.RaggedTensor(
-          values=values, row_splits=[0, 2, 2, 5, 6, 7], internal=True)
+      RaggedTensor(values=values, row_splits=[0, 2, 2, 5, 6, 7], internal=True)
 
     with self.assertRaisesRegexp(ValueError,
                                  r'Shape \(6, 1\) must have rank 1'):
-      ragged.RaggedTensor(
+      RaggedTensor(
           values=values,
           row_splits=array_ops.expand_dims(row_splits, 1),
           internal=True)
 
     with self.assertRaisesRegexp(TypeError,
                                  'Cached value must be a Tensor or None.'):
-      ragged.RaggedTensor(values=values, row_splits=row_splits,
-                          cached_row_lengths=[2, 3, 4], internal=True)
+      RaggedTensor(
+          values=values,
+          row_splits=row_splits,
+          cached_row_lengths=[2, 3, 4],
+          internal=True)
 
 
 #=============================================================================
 # RaggedTensor Factory Ops
 #=============================================================================
 
-  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithDerivedNRows(self):
     # nrows is known at graph creation time.
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
 
-    rt = ragged.from_value_rowids(values, value_rowids)
+    rt = RaggedTensor.from_value_rowids(values, value_rowids)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [5, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_value_rowids = ragged.value_rowids(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_value_rowids = rt.value_rowids()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     self.assertAllEqual(rt_value_rowids, value_rowids)
-    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(self.eval_to_list(rt_nrows), 5)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
-  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithDerivedNRowsDynamic(self):
     # nrows is not known at graph creation time.
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
     value_rowids = array_ops.placeholder_with_default(value_rowids, shape=None)
 
-    rt = ragged.from_value_rowids(values, value_rowids)
+    rt = RaggedTensor.from_value_rowids(values, value_rowids)
     self.assertEqual(rt.dtype, dtypes.string)
-    self.assertEqual(rt.shape.as_list(), [None, None])
+    if context.executing_eagerly():
+      self.assertEqual(rt.shape.as_list(), [5, None])
+    else:
+      self.assertEqual(rt.shape.as_list(), [None, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_value_rowids = ragged.value_rowids(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_value_rowids = rt.value_rowids()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     self.assertAllEqual(rt_value_rowids, value_rowids)
-    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(self.eval_to_list(rt_nrows), 5)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
-  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithExplicitNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
     nrows = constant_op.constant(7, dtypes.int64)
 
-    rt = ragged.from_value_rowids(values, value_rowids, nrows)
+    rt = RaggedTensor.from_value_rowids(values, value_rowids, nrows)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [7, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_value_rowids = ragged.value_rowids(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_value_rowids = rt.value_rowids()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
     self.assertIs(rt_nrows, nrows)  # cached_nrows
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g'], [], []])
 
-  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithExplicitNRowsEqualToDefault(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
     nrows = constant_op.constant(5, dtypes.int64)
 
-    rt = ragged.from_value_rowids(values, value_rowids, nrows)
+    rt = RaggedTensor.from_value_rowids(values, value_rowids, nrows)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [5, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_value_rowids = ragged.value_rowids(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_value_rowids = rt.value_rowids()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_value_rowids, value_rowids)  # cached_value_rowids
@@ -335,112 +336,106 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(rt_value_rowids, value_rowids)
     self.assertAllEqual(rt_nrows, nrows)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
-  @test_util.run_deprecated_v1
   def testFromValueRowIdsWithEmptyValues(self):
-    rt = ragged.from_value_rowids([], [])
-    rt_nrows = ragged.nrows(rt)
+    rt = RaggedTensor.from_value_rowids([], [])
+    rt_nrows = rt.nrows()
     self.assertEqual(rt.dtype, dtypes.float32)
     self.assertEqual(rt.shape.as_list(), [0, None])
     self.assertEqual(rt.ragged_rank, 1)
     self.assertEqual(rt.values.shape.as_list(), [0])
-    self.assertEqual(ragged.value_rowids(rt).shape.as_list(), [0])
-    self.assertEqual(self.evaluate(rt_nrows).tolist(), 0)
-    self.assertEqual(self.evaluate(rt).tolist(), [])
+    self.assertEqual(rt.value_rowids().shape.as_list(), [0])
+    self.assertEqual(self.eval_to_list(rt_nrows), 0)
+    self.assertEqual(self.eval_to_list(rt), [])
 
-  @test_util.run_deprecated_v1
   def testFromRowSplits(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
 
-    rt = ragged.from_row_splits(values, row_splits)
+    rt = RaggedTensor.from_row_splits(values, row_splits)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [5, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
     rt_row_splits = rt.row_splits
-    rt_nrows = ragged.nrows(rt)
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_row_splits, row_splits)
-    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(self.eval_to_list(rt_nrows), 5)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
   def testFromRowSplitsWithEmptySplits(self):
     err_msg = 'row_splits tensor may not be empty'
     with self.assertRaisesRegexp(ValueError, err_msg):
-      ragged.from_row_splits([], [])
+      RaggedTensor.from_row_splits([], [])
 
-  @test_util.run_deprecated_v1
   def testFromRowStarts(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_starts = constant_op.constant([0, 2, 2, 5, 6], dtypes.int64)
 
-    rt = ragged.from_row_starts(values, row_starts)
+    rt = RaggedTensor.from_row_starts(values, row_starts)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [5, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_row_starts = ragged.row_starts(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_row_starts = rt.row_starts()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
-    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(self.eval_to_list(rt_nrows), 5)
     self.assertAllEqual(rt_row_starts, row_starts)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
-  @test_util.run_deprecated_v1
   def testFromRowLimits(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_limits = constant_op.constant([2, 2, 5, 6, 7], dtypes.int64)
 
-    rt = ragged.from_row_limits(values, row_limits)
+    rt = RaggedTensor.from_row_limits(values, row_limits)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [5, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_row_limits = ragged.row_limits(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_row_limits = rt.row_limits()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
-    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(self.eval_to_list(rt_nrows), 5)
     self.assertAllEqual(rt_row_limits, row_limits)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
-  @test_util.run_deprecated_v1
   def testFromRowLengths(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_lengths = constant_op.constant([2, 0, 3, 1, 1], dtypes.int64)
 
-    rt = ragged.from_row_lengths(values, row_lengths)
+    rt = RaggedTensor.from_row_lengths(values, row_lengths)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [5, None])
     self.assertEqual(rt.ragged_rank, 1)
 
     rt_values = rt.values
-    rt_row_lengths = ragged.row_lengths(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_row_lengths = rt.row_lengths()
+    rt_nrows = rt.nrows()
 
     self.assertIs(rt_values, values)
     self.assertIs(rt_row_lengths, row_lengths)  # cached_nrows
-    self.assertEqual(self.evaluate(rt_nrows), 5)
+    self.assertEqual(self.eval_to_list(rt_nrows), 5)
     self.assertAllEqual(rt_row_lengths, row_lengths)
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
 
-  @test_util.run_deprecated_v1
   def testFromNestedValueRowIdsWithDerivedNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_value_rowids = [
@@ -448,24 +443,23 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
     ]
 
-    rt = ragged.from_nested_value_rowids(values, nested_value_rowids)
+    rt = RaggedTensor.from_nested_value_rowids(values, nested_value_rowids)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [4, None, None])
     self.assertEqual(rt.ragged_rank, 2)
 
     rt_values = rt.values
-    rt_value_rowids = ragged.value_rowids(rt)
+    rt_value_rowids = rt.value_rowids()
     rt_values_values = rt_values.values
-    rt_values_value_rowids = ragged.value_rowids(rt_values)
+    rt_values_value_rowids = rt_values.value_rowids()
 
     self.assertIs(rt_values_values, values)
     self.assertAllEqual(rt_value_rowids, nested_value_rowids[0])
     self.assertAllEqual(rt_values_value_rowids, nested_value_rowids[1])
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
 
-  @test_util.run_deprecated_v1
   def testFromNestedValueRowIdsWithExplicitNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_value_rowids = [
@@ -477,17 +471,18 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         constant_op.constant(6, dtypes.int64)
     ]
 
-    rt = ragged.from_nested_value_rowids(values, nested_value_rowids, nrows)
+    rt = RaggedTensor.from_nested_value_rowids(values, nested_value_rowids,
+                                               nrows)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [6, None, None])
     self.assertEqual(rt.ragged_rank, 2)
 
     rt_values = rt.values
-    rt_value_rowids = ragged.value_rowids(rt)
-    rt_nrows = ragged.nrows(rt)
+    rt_value_rowids = rt.value_rowids()
+    rt_nrows = rt.nrows()
     rt_values_values = rt_values.values
-    rt_values_value_rowids = ragged.value_rowids(rt_values)
-    rt_values_nrows = ragged.nrows(rt_values)
+    rt_values_value_rowids = rt_values.value_rowids()
+    rt_values_nrows = rt_values.nrows()
 
     self.assertIs(rt_values_values, values)
     self.assertAllEqual(rt_value_rowids, nested_value_rowids[0])
@@ -495,9 +490,8 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     self.assertAllEqual(rt_nrows, nrows[0])
     self.assertAllEqual(rt_values_nrows, nrows[1])
     self.assertEqual(
-        self.evaluate(rt).tolist(),
-        [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g'], []], [],
-         []])
+        self.eval_to_list(rt), [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [],
+                                [[b'f'], [b'g'], []], [], []])
 
   def testFromNestedValueRowIdsWithExplicitNRowsMismatch(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -509,28 +503,26 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(
         ValueError, 'nested_nrows must have the same '
         'length as nested_value_rowids'):
-      ragged.from_nested_value_rowids(values, nested_value_rowids, nrows)
+      RaggedTensor.from_nested_value_rowids(values, nested_value_rowids, nrows)
 
   def testFromNestedValueRowIdsWithNonListInput(self):
     with self.assertRaisesRegexp(
         TypeError, 'nested_value_rowids must be a list of Tensors'):
-      ragged.from_nested_value_rowids([1, 2, 3],
-                                      constant_op.constant(
-                                          [[0, 1, 2], [0, 1, 2]], dtypes.int64))
+      RaggedTensor.from_nested_value_rowids(
+          [1, 2, 3], constant_op.constant([[0, 1, 2], [0, 1, 2]], dtypes.int64))
     with self.assertRaisesRegexp(TypeError,
                                  'nested_nrows must be a list of Tensors'):
-      ragged.from_nested_value_rowids([1, 2, 3], [[0, 1, 2], [0, 1, 2]],
-                                      constant_op.constant([3, 3]))
+      RaggedTensor.from_nested_value_rowids([1, 2, 3], [[0, 1, 2], [0, 1, 2]],
+                                            constant_op.constant([3, 3]))
 
-  @test_util.run_deprecated_v1
   def testFromNestedRowSplits(self):
-    inner_values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
+    flat_values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_row_splits = [
         constant_op.constant([0, 2, 3, 3, 5], dtypes.int64),
         constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
     ]
 
-    rt = ragged.from_nested_row_splits(inner_values, nested_row_splits)
+    rt = RaggedTensor.from_nested_row_splits(flat_values, nested_row_splits)
     self.assertEqual(rt.dtype, dtypes.string)
     self.assertEqual(rt.shape.as_list(), [4, None, None])
     self.assertEqual(rt.ragged_rank, 2)
@@ -540,19 +532,18 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt_values_values = rt_values.values
     rt_values_row_splits = rt_values.row_splits
 
-    self.assertIs(rt_values_values, inner_values)
+    self.assertIs(rt_values_values, flat_values)
     self.assertIs(rt_row_splits, nested_row_splits[0])
     self.assertIs(rt_values_row_splits, nested_row_splits[1])
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
 
   def testFromNestedRowSplitsWithNonListInput(self):
     with self.assertRaisesRegexp(TypeError,
                                  'nested_row_splits must be a list of Tensors'):
-      ragged.from_nested_row_splits([1, 2],
-                                    constant_op.constant([[0, 1, 2], [0, 1, 2]],
-                                                         dtypes.int64))
+      RaggedTensor.from_nested_row_splits(
+          [1, 2], constant_op.constant([[0, 1, 2], [0, 1, 2]], dtypes.int64))
 
   def testFromValueRowIdsWithBadNRows(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
@@ -560,7 +551,7 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     nrows = constant_op.constant(5, dtypes.int64)
 
     with self.assertRaisesRegexp(ValueError, r'Expected nrows >= 0; got -2'):
-      ragged.from_value_rowids(
+      RaggedTensor.from_value_rowids(
           values=values,
           value_rowids=array_ops.placeholder_with_default(value_rowids, None),
           nrows=-2)
@@ -568,113 +559,94 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(
         ValueError, r'Expected nrows >= value_rowids\[-1\] \+ 1; got nrows=2, '
         r'value_rowids\[-1\]=4'):
-      ragged.from_value_rowids(
+      RaggedTensor.from_value_rowids(
           values=values, value_rowids=value_rowids, nrows=2)
 
     with self.assertRaisesRegexp(
         ValueError, r'Expected nrows >= value_rowids\[-1\] \+ 1; got nrows=4, '
         r'value_rowids\[-1\]=4'):
-      ragged.from_value_rowids(
+      RaggedTensor.from_value_rowids(
           values=values, value_rowids=value_rowids, nrows=4)
 
     with self.assertRaisesRegexp(ValueError,
                                  r'Shape \(7, 1\) must have rank 1'):
-      ragged.from_value_rowids(
+      RaggedTensor.from_value_rowids(
           values=values,
           value_rowids=array_ops.expand_dims(value_rowids, 1),
           nrows=nrows)
 
     with self.assertRaisesRegexp(ValueError, r'Shape \(1,\) must have rank 0'):
-      ragged.from_value_rowids(
+      RaggedTensor.from_value_rowids(
           values=values,
           value_rowids=value_rowids,
           nrows=array_ops.expand_dims(nrows, 0))
 
-  @test_util.run_deprecated_v1
   def testGraphMismatch(self):
-    with ops.Graph().as_default():
-      values = constant_op.constant([1, 2, 3])
-    with ops.Graph().as_default():
-      splits = constant_op.constant([0, 2, 3])
-    self.assertRaisesRegexp(ValueError, '.* must be from the same graph as .*',
-                            ragged.from_row_splits, values, splits)
+    if not context.executing_eagerly():
+      with ops.Graph().as_default():
+        values = constant_op.constant([1, 2, 3], dtypes.int64)
+      with ops.Graph().as_default():
+        splits = constant_op.constant([0, 2, 3], dtypes.int64)
+      self.assertRaisesRegexp(ValueError,
+                              '.* must be from the same graph as .*',
+                              RaggedTensor.from_row_splits, values, splits)
 
   #=============================================================================
   # Ragged Value & Row-Partitioning Tensor Accessors
   #=============================================================================
 
-  @test_util.run_deprecated_v1
   def testRaggedTensorAccessors_2d(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
-    rt1 = ragged.from_row_splits(values, row_splits)
-    rt2 = ragged.from_value_rowids(values, value_rowids)
+    rt1 = RaggedTensor.from_row_splits(values, row_splits)
+    rt2 = RaggedTensor.from_value_rowids(values, value_rowids)
 
     for rt in [rt1, rt2]:
-      self.assertEqual(
-          self.evaluate(rt).tolist(),
-          [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
-      self.assertEqual(
-          self.evaluate(rt.values).tolist(),
-          [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+      self.assertRaggedEqual(
+          rt, [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
+      self.assertAllEqual(rt.values, [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
       self.assertEqual(rt.values.shape.dims[0].value, 7)
-      self.assertEqual(
-          self.evaluate(ragged.value_rowids(rt)).tolist(),
-          [0, 0, 2, 2, 2, 3, 4])
-      self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
-      self.assertEqual(
-          self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
-      self.assertEqual(
-          self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
-      self.assertEqual(
-          self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
-      self.assertEqual(
-          self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
-      self.assertEqual(
-          self.evaluate(rt.inner_values).tolist(),
-          [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-      self.assertEqual(
-          [self.evaluate(s).tolist() for s in rt.nested_row_splits],
-          [[0, 2, 2, 5, 6, 7]])
+      self.assertAllEqual(rt.value_rowids(), [0, 0, 2, 2, 2, 3, 4])
+      self.assertAllEqual(rt.nrows(), 5)
+      self.assertAllEqual(rt.row_splits, [0, 2, 2, 5, 6, 7])
+      self.assertAllEqual(rt.row_starts(), [0, 2, 2, 5, 6])
+      self.assertAllEqual(rt.row_limits(), [2, 2, 5, 6, 7])
+      self.assertAllEqual(rt.row_lengths(), [2, 0, 3, 1, 1])
+      self.assertAllEqual(rt.flat_values,
+                          [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
+      self.assertLen(rt.nested_row_splits, 1)
+      self.assertAllEqual(rt.nested_row_splits[0], [0, 2, 2, 5, 6, 7])
 
-  @test_util.run_deprecated_v1
   def testRaggedTensorAccessors_3d_with_ragged_rank_1(self):
     values = [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]
     row_splits = constant_op.constant([0, 2, 2, 5, 6, 7], dtypes.int64)
     value_rowids = constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
-    rt1 = ragged.from_row_splits(values, row_splits)
-    rt2 = ragged.from_value_rowids(values, value_rowids)
+    rt1 = RaggedTensor.from_row_splits(values, row_splits)
+    rt2 = RaggedTensor.from_value_rowids(values, value_rowids)
 
     for rt in [rt1, rt2]:
       self.assertEqual(
-          self.evaluate(rt).tolist(),
+          self.eval_to_list(rt),
           [[[0, 1], [2, 3]], [], [[4, 5], [6, 7], [8, 9]], [[10, 11]],
            [[12, 13]]])
       self.assertEqual(
-          self.evaluate(rt.values).tolist(),
+          self.eval_to_list(rt.values),
           [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
       self.assertEqual(rt.values.shape.dims[0].value, 7)
       self.assertEqual(
-          self.evaluate(ragged.value_rowids(rt)).tolist(),
-          [0, 0, 2, 2, 2, 3, 4])
-      self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 5)
-      self.assertEqual(
-          self.evaluate(rt.row_splits).tolist(), [0, 2, 2, 5, 6, 7])
-      self.assertEqual(
-          self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 2, 5, 6])
-      self.assertEqual(
-          self.evaluate(ragged.row_limits(rt)).tolist(), [2, 2, 5, 6, 7])
+          self.eval_to_list(rt.value_rowids()), [0, 0, 2, 2, 2, 3, 4])
+      self.assertEqual(self.eval_to_list(rt.nrows()), 5)
+      self.assertEqual(self.eval_to_list(rt.row_splits), [0, 2, 2, 5, 6, 7])
+      self.assertEqual(self.eval_to_list(rt.row_starts()), [0, 2, 2, 5, 6])
+      self.assertEqual(self.eval_to_list(rt.row_limits()), [2, 2, 5, 6, 7])
+      self.assertEqual(self.eval_to_list(rt.row_lengths()), [2, 0, 3, 1, 1])
       self.assertEqual(
-          self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 0, 3, 1, 1])
-      self.assertEqual(
-          self.evaluate(rt.inner_values).tolist(),
+          self.eval_to_list(rt.flat_values),
           [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]])
-      self.assertEqual(
-          [self.evaluate(s).tolist() for s in rt.nested_row_splits],
-          [[0, 2, 2, 5, 6, 7]])
+      self.assertEqual([self.eval_to_list(s) for s in rt.nested_row_splits],
+                       [[0, 2, 2, 5, 6, 7]])
 
-  @test_util.run_deprecated_v1
   def testRaggedTensorAccessors_3d_with_ragged_rank_2(self):
     values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g'])
     nested_row_splits = [
@@ -685,73 +657,59 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         constant_op.constant([0, 0, 1, 3, 3], dtypes.int64),
         constant_op.constant([0, 0, 2, 2, 2, 3, 4], dtypes.int64)
     ]
-    rt1 = ragged.from_nested_row_splits(values, nested_row_splits)
-    rt2 = ragged.from_nested_value_rowids(values, nested_value_rowids)
+    rt1 = RaggedTensor.from_nested_row_splits(values, nested_row_splits)
+    rt2 = RaggedTensor.from_nested_value_rowids(values, nested_value_rowids)
 
     for rt in [rt1, rt2]:
       self.assertEqual(
-          self.evaluate(rt).tolist(),
+          self.eval_to_list(rt),
           [[[b'a', b'b'], []], [[b'c', b'd', b'e']], [], [[b'f'], [b'g']]])
       self.assertEqual(
-          self.evaluate(rt.values).tolist(),
+          self.eval_to_list(rt.values),
           [[b'a', b'b'], [], [b'c', b'd', b'e'], [b'f'], [b'g']])
       self.assertEqual(rt.values.shape.dims[0].value, 5)
+      self.assertEqual(self.eval_to_list(rt.value_rowids()), [0, 0, 1, 3, 3])
+      self.assertEqual(self.eval_to_list(rt.nrows()), 4)
+      self.assertEqual(self.eval_to_list(rt.row_splits), [0, 2, 3, 3, 5])
+      self.assertEqual(self.eval_to_list(rt.row_starts()), [0, 2, 3, 3])
+      self.assertEqual(self.eval_to_list(rt.row_limits()), [2, 3, 3, 5])
+      self.assertEqual(self.eval_to_list(rt.row_lengths()), [2, 1, 0, 2])
       self.assertEqual(
-          self.evaluate(ragged.value_rowids(rt)).tolist(), [0, 0, 1, 3, 3])
-      self.assertEqual(self.evaluate(ragged.nrows(rt)).tolist(), 4)
-      self.assertEqual(self.evaluate(rt.row_splits).tolist(), [0, 2, 3, 3, 5])
-      self.assertEqual(
-          self.evaluate(ragged.row_starts(rt)).tolist(), [0, 2, 3, 3])
-      self.assertEqual(
-          self.evaluate(ragged.row_limits(rt)).tolist(), [2, 3, 3, 5])
-      self.assertEqual(
-          self.evaluate(ragged.row_lengths(rt)).tolist(), [2, 1, 0, 2])
-      self.assertEqual(
-          self.evaluate(rt.inner_values).tolist(),
+          self.eval_to_list(rt.flat_values),
           [b'a', b'b', b'c', b'd', b'e', b'f', b'g'])
-      self.assertEqual(
-          [self.evaluate(s).tolist() for s in rt.nested_row_splits],
-          [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]])
-
-  def testNRowsWithTensorInput(self):
-    dt = constant_op.constant([[1, 2, 3], [4, 5, 6]])
-    nrows = ragged.nrows(dt)
-    self.assertEqual(self.evaluate(nrows), 2)
-
-  def testRowLengthsWithTensorInput(self):
-    dt = constant_op.constant([[1, 2, 3], [4, 5, 6]])
-    row_lengths = ragged.row_lengths(dt)
-    self.assertEqual(self.evaluate(row_lengths).tolist(), [3, 3])
+      self.assertEqual([self.eval_to_list(s) for s in rt.nested_row_splits],
+                       [[0, 2, 3, 3, 5], [0, 2, 2, 5, 6, 7]])
 
   #=============================================================================
   # RaggedTensor.shape
   #=============================================================================
 
-  @test_util.run_deprecated_v1
   def testShape(self):
     """Tests for RaggedTensor.shape."""
-    rt1 = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
+    rt1 = RaggedTensor.from_row_splits(b'a b c d e f g'.split(),
+                                       [0, 2, 5, 6, 6, 7])
     self.assertEqual(rt1.shape.as_list(), [5, None])
 
-    rt2 = ragged.from_row_splits(
+    rt2 = RaggedTensor.from_row_splits(
         [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14]],
         [0, 2, 5, 6, 6, 7])
     self.assertEqual(rt2.shape.as_list(), [5, None, 2])
 
-    rt3 = ragged.from_row_splits(
+    rt3 = RaggedTensor.from_row_splits(
         [[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]], [0, 2, 2, 3])
     self.assertEqual(rt3.shape.as_list(), [3, None, 2, 2])
 
-    rt4 = ragged.from_row_splits(rt3, [0, 1, 3, 3])
+    rt4 = RaggedTensor.from_row_splits(rt3, [0, 1, 3, 3])
     self.assertEqual(rt4.shape.as_list(), [3, None, None, 2, 2])
 
-    rt5 = ragged.from_row_splits(
-        array_ops.placeholder(dtype=dtypes.string), [0, 2, 3, 5])
-    self.assertEqual(rt5.shape.ndims, None)
+    if not context.executing_eagerly():
+      rt5 = RaggedTensor.from_row_splits(
+          array_ops.placeholder(dtype=dtypes.string), [0, 2, 3, 5])
+      self.assertEqual(rt5.shape.ndims, None)
 
-    rt6 = ragged.from_row_splits([1, 2, 3],
-                                 array_ops.placeholder(dtype=dtypes.int64))
-    self.assertEqual(rt6.shape.as_list(), [None, None])
+      rt6 = RaggedTensor.from_row_splits(
+          [1, 2, 3], array_ops.placeholder(dtype=dtypes.int64))
+      self.assertEqual(rt6.shape.as_list(), [None, None])
 
   #=============================================================================
   # RaggedTensor.__getitem__
@@ -777,15 +735,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     """
     tensor_slice_spec1 = _make_tensor_slice_spec(slice_spec, True)
     tensor_slice_spec2 = _make_tensor_slice_spec(slice_spec, False)
-    value1 = self.evaluate(rt.__getitem__(slice_spec))
-    value2 = self.evaluate(rt.__getitem__(tensor_slice_spec1))
-    value3 = self.evaluate(rt.__getitem__(tensor_slice_spec2))
-    if hasattr(value1, 'tolist'):
-      value1 = value1.tolist()
-    if hasattr(value2, 'tolist'):
-      value2 = value2.tolist()
-    if hasattr(value3, 'tolist'):
-      value3 = value3.tolist()
+    value1 = self.eval_to_list(rt.__getitem__(slice_spec))
+    value2 = self.eval_to_list(rt.__getitem__(tensor_slice_spec1))
+    value3 = self.eval_to_list(rt.__getitem__(tensor_slice_spec2))
     self.assertEqual(value1, expected, 'slice_spec=%s' % (slice_spec,))
     self.assertEqual(value2, expected, 'slice_spec=%s' % (slice_spec,))
     self.assertEqual(value3, expected, 'slice_spec=%s' % (slice_spec,))
@@ -861,23 +813,26 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[:, -2:], [row[-2:] for row in EXAMPLE_RAGGED_TENSOR_2D]),
       # TODO(edloper): Add tests for strided slices, once support is added.
   )
-  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorGetItemWithRaggedRank1(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     # Ragged tensor
-    rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES,
-                                EXAMPLE_RAGGED_TENSOR_2D_SPLITS)
+    rt = RaggedTensor.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES,
+                                      EXAMPLE_RAGGED_TENSOR_2D_SPLITS)
 
-    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D)
+    self.assertEqual(self.eval_to_list(rt), EXAMPLE_RAGGED_TENSOR_2D)
     self._TestGetItem(rt, slice_spec, expected)
 
   # pylint: disable=invalid-slice-index
   @parameterized.parameters(
       # Tests for out-of-bound errors
-      (SLICE_BUILDER[5], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[-6], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[0, 2], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[3, 0], ValueError, '.*out of bounds.*'),
+      (SLICE_BUILDER[5],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[-6],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[0, 2],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[3, 0],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
 
       # Indexing into an inner ragged dimension
       (SLICE_BUILDER[:, 3], ValueError,
@@ -889,8 +844,8 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
       # Tests for type errors
       (SLICE_BUILDER[0.5], TypeError, re.escape(array_ops._SLICE_TYPE_ERROR)),
-      (SLICE_BUILDER[1:3:0.5], TypeError,
-       re.escape(array_ops._SLICE_TYPE_ERROR)),
+      (SLICE_BUILDER[1:3:0.5], TypeError, re.escape(
+          array_ops._SLICE_TYPE_ERROR)),
       (SLICE_BUILDER[:, 1:3:0.5], TypeError,
        'slice strides must be integers or None'),
       (SLICE_BUILDER[:, 0.5:1.5], TypeError,
@@ -903,17 +858,14 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[..., 0, 0, 0], IndexError,
        'Too many indices for RaggedTensor'),
   )
-  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorGetItemErrorsWithRaggedRank1(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
     # Ragged tensor
-    rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES,
-                                EXAMPLE_RAGGED_TENSOR_2D_SPLITS)
-    # if sys.version_info[0] == 3:
-    #   message = 'must be str, not int'
+    rt = RaggedTensor.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES,
+                                      EXAMPLE_RAGGED_TENSOR_2D_SPLITS)
 
-    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D)
+    self.assertEqual(self.eval_to_list(rt), EXAMPLE_RAGGED_TENSOR_2D)
     self._TestGetItemException(rt, slice_spec, expected, message)
 
   @parameterized.parameters(
@@ -982,13 +934,12 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       # TODO(edloper): Add tests slicing inner ragged dimensions, one support
       # is added.
   )
-  @test_util.run_v1_only('b/120545219')
   def testRaggedTensorGetItemWithRaggedRank2(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
-    rt = ragged.from_nested_row_splits(
+    rt = RaggedTensor.from_nested_row_splits(
         EXAMPLE_RAGGED_TENSOR_4D_VALUES,
         [EXAMPLE_RAGGED_TENSOR_4D_SPLITS1, EXAMPLE_RAGGED_TENSOR_4D_SPLITS2])
-    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_4D)
+    self.assertEqual(self.eval_to_list(rt), EXAMPLE_RAGGED_TENSOR_4D)
     self._TestGetItem(rt, slice_spec, expected)
 
   @parameterized.parameters(
@@ -999,19 +950,22 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
        'Cannot index into an inner ragged dimension.'),
 
       # Test for out-of-bounds errors.
-      (SLICE_BUILDER[1, 0], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[0, 0, 3], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[5], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[0, 5], ValueError, '.*out of bounds.*'),
+      (SLICE_BUILDER[1, 0],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[0, 0, 3],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[5],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[0, 5],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
   )
-  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithRaggedRank2(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
-    rt = ragged.from_nested_row_splits(
+    rt = RaggedTensor.from_nested_row_splits(
         EXAMPLE_RAGGED_TENSOR_4D_VALUES,
         [EXAMPLE_RAGGED_TENSOR_4D_SPLITS1, EXAMPLE_RAGGED_TENSOR_4D_SPLITS2])
-    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_4D)
+    self.assertEqual(self.eval_to_list(rt), EXAMPLE_RAGGED_TENSOR_4D)
     self._TestGetItemException(rt, slice_spec, expected, message)
 
   @parameterized.parameters(
@@ -1019,21 +973,21 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[2:], []),
       (SLICE_BUILDER[:-3], []),
   )
-  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemWithEmptyTensor(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
-    rt = ragged.from_row_splits([], [0])
+    rt = RaggedTensor.from_row_splits([], [0])
     self._TestGetItem(rt, slice_spec, expected)
 
   @parameterized.parameters(
-      (SLICE_BUILDER[0], ValueError, '.*out of bounds.*'),
-      (SLICE_BUILDER[-1], ValueError, '.*out of bounds.*'),
+      (SLICE_BUILDER[0],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
+      (SLICE_BUILDER[-1],
+       (ValueError, errors.InvalidArgumentError), '.*out of bounds.*'),
   )
-  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithEmptyTensor(self, slice_spec, expected,
                                                    message):
     """Test that rt.__getitem__(slice_spec) == expected."""
-    rt = ragged.from_row_splits([], [0])
+    rt = RaggedTensor.from_row_splits([], [0])
     self._TestGetItemException(rt, slice_spec, expected, message)
 
   @parameterized.parameters(
@@ -1045,7 +999,6 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       (SLICE_BUILDER[0, 1], EXAMPLE_RAGGED_TENSOR_2D[0][1]),
       (SLICE_BUILDER[-3, 0], EXAMPLE_RAGGED_TENSOR_2D[-3][0]),
   )
-  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemWithPlaceholderShapes(self, slice_spec, expected):
     """Test that rt.__getitem__(slice_spec) == expected."""
     # Intentionally use an unknown shape for `splits`, to force the code path
@@ -1053,29 +1006,28 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     splits = constant_op.constant(
         EXAMPLE_RAGGED_TENSOR_2D_SPLITS, dtype=dtypes.int64)
     splits = array_ops.placeholder_with_default(splits, None)
-    rt = ragged.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES, splits)
-    self.assertEqual(self.evaluate(rt).tolist(), EXAMPLE_RAGGED_TENSOR_2D)
+    rt = RaggedTensor.from_row_splits(EXAMPLE_RAGGED_TENSOR_2D_VALUES, splits)
+    self.assertEqual(self.eval_to_list(rt), EXAMPLE_RAGGED_TENSOR_2D)
     self._TestGetItem(rt, slice_spec, expected)
 
   @parameterized.parameters(
       (SLICE_BUILDER[..., 2], ValueError,
        'Ellipsis not supported for unknown shape RaggedTensors'),)
-  @test_util.run_deprecated_v1
   def testRaggedTensorGetItemErrorsWithPlaceholderShapes(
       self, slice_spec, expected, message):
     """Test that rt.__getitem__(slice_spec) == expected."""
-    # Intentionally use an unknown shape for `values`.
-    values = array_ops.placeholder_with_default([0], None)
-    rt = ragged.from_row_splits(values, [0, 1])
-    self._TestGetItemException(rt, slice_spec, expected, message)
+    if not context.executing_eagerly():
+      # Intentionally use an unknown shape for `values`.
+      values = array_ops.placeholder_with_default([0], None)
+      rt = RaggedTensor.from_row_splits(values, [0, 1])
+      self._TestGetItemException(rt, slice_spec, expected, message)
 
-  @test_util.run_v1_only('b/120545219')
   def testGetItemNewAxis(self):
     # rt: [[[['a', 'b'], ['c', 'd']], [], [['e', 'f']]], []]
     splits1 = [0, 3, 3]
     splits2 = [0, 2, 2, 3]
     values = constant_op.constant([['a', 'b'], ['c', 'd'], ['e', 'f']])
-    rt = ragged.from_nested_row_splits(values, [splits1, splits2])
+    rt = RaggedTensor.from_nested_row_splits(values, [splits1, splits2])
     rt_newaxis0 = rt[array_ops.newaxis]
     rt_newaxis1 = rt[:, array_ops.newaxis]
     rt_newaxis2 = rt[:, :, array_ops.newaxis]
@@ -1083,22 +1035,22 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     rt_newaxis4 = rt[:, :, :, :, array_ops.newaxis]
 
     self.assertEqual(
-        self.evaluate(rt).tolist(),
+        self.eval_to_list(rt),
         [[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []])
     self.assertEqual(
-        self.evaluate(rt_newaxis0).tolist(),
+        self.eval_to_list(rt_newaxis0),
         [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]], []]])
     self.assertEqual(
-        self.evaluate(rt_newaxis1).tolist(),
+        self.eval_to_list(rt_newaxis1),
         [[[[[b'a', b'b'], [b'c', b'd']], [], [[b'e', b'f']]]], [[]]])
     self.assertEqual(
-        self.evaluate(rt_newaxis2).tolist(),
+        self.eval_to_list(rt_newaxis2),
         [[[[[b'a', b'b'], [b'c', b'd']]], [[]], [[[b'e', b'f']]]], []])
     self.assertEqual(
-        self.evaluate(rt_newaxis3).tolist(),
+        self.eval_to_list(rt_newaxis3),
         [[[[[b'a', b'b']], [[b'c', b'd']]], [], [[[b'e', b'f']]]], []])
     self.assertEqual(
-        self.evaluate(rt_newaxis4).tolist(),
+        self.eval_to_list(rt_newaxis4),
         [[[[[b'a'], [b'b']], [[b'c'], [b'd']]], [], [[[b'e'], [b'f']]]], []])
 
     self.assertEqual(rt.ragged_rank, 2)
@@ -1117,104 +1069,118 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
   #=============================================================================
   # RaggedTensor.__str__
   #=============================================================================
-  @test_util.run_deprecated_v1
   def testRaggedTensorStr(self):
-    rt1 = ragged.from_row_splits(b'a b c d e f g'.split(), [0, 2, 5, 6, 6, 7])
-    expected1 = ('RaggedTensor(values=Tensor("RaggedFromRowSplits/values:0", '
-                 'shape=(7,), dtype=string), row_splits='
-                 'Tensor("RaggedFromRowSplits/row_splits:0", '
-                 'shape=(6,), dtype=int64))')
-    self.assertEqual(str(rt1), expected1)
-    self.assertEqual(repr(rt1), expected1)
+    values = [b'a', b'b', b'c', b'd', b'e', b'f', b'g']
+    row_splits = [0, 2, 5, 6, 6, 7]
+    rt = RaggedTensor.from_row_splits(values, row_splits)
+    if context.executing_eagerly():
+      expected_str = '<tf.RaggedTensor {}>'.format([[b'a', b'b'],
+                                                    [b'c', b'd', b'e'], [b'f'],
+                                                    [], [b'g']])
+      expected_repr = (
+          'tf.RaggedTensor(values=tf.Tensor([{}], shape=(7,), dtype=string), '
+          'row_splits=tf.Tensor([{}], shape=(6,), dtype=int64))'.format(
+              ' '.join(repr(x) for x in values), ' '.join(
+                  repr(x) for x in row_splits)))
+      self.assertEqual(str(rt), expected_str)
+      self.assertEqual(repr(rt), expected_repr)
+    else:
+      expected_repr = (
+          'tf.RaggedTensor(values=Tensor("RaggedFromRowSplits/values:0", '
+          'shape=(7,), dtype=string), row_splits='
+          'Tensor("RaggedFromRowSplits/row_splits:0", '
+          'shape=(6,), dtype=int64))')
+      self.assertEqual(repr(rt), expected_repr)
+      self.assertEqual(str(rt), expected_repr)
 
   def testRaggedTensorValueStr(self):
+    values = [b'a', b'b', b'c', b'd', b'e', b'f', b'g']
+    row_splits = [0, 2, 5, 6, 6, 7]
     rt = ragged.RaggedTensorValue(
-        values=np.array(b'a b c d e f g'.split()),
-        row_splits=np.array([0, 2, 5, 6, 6, 7], dtype=np.int64))
-    if sys.version_info[0] == 2:
-      self.assertEqual(' '.join(str(rt).split()),
-                       (r"<RaggedTensorValue [['a', 'b'], ['c', 'd', 'e'], "
-                        "['f'], [], ['g']]>"))
-      self.assertEqual(
-          ' '.join(repr(rt).split()),
-          (r"RaggedTensorValue(values=array(['a', 'b', 'c', 'd', "
-           "'e', 'f', 'g'], dtype='|S1'), row_splits=array([0, 2, 5,"
-           ' 6, 6, 7]))'))
-    else:
-      self.assertEqual(
-          ' '.join(str(rt).split()),
-          (r"<RaggedTensorValue [[b'a', b'b'], [b'c', b'd', b'e'], "
-           "[b'f'], [], [b'g']]>"))
-      self.assertEqual(
-          ' '.join(repr(rt).split()),
-          (r"RaggedTensorValue(values=array([b'a', b'b', b'c', b'd', "
-           "b'e', b'f', b'g'], dtype='|S1'), row_splits=array([0, 2, 5,"
-           ' 6, 6, 7]))'))
+        np.array(values), np.array(row_splits, dtype=np.int64))
+    expected_str = '<tf.RaggedTensorValue {}>'.format([[b'a', b'b'],
+                                                       [b'c', b'd', b'e'],
+                                                       [b'f'], [], [b'g']])
+    expected_repr = ("tf.RaggedTensorValue(values=array({}, dtype='|S1'), "
+                     'row_splits=array({}))'.format(values, row_splits))
+    self.assertEqual(' '.join(str(rt).split()), expected_str)
+    self.assertEqual(' '.join(repr(rt).split()), expected_repr)
 
   #=============================================================================
-  # RaggedTensor.with_values() and RaggedTensor.with_inner_values().
+  # RaggedTensor.with_values() and RaggedTensor.with_flat_values().
   #=============================================================================
 
-  @test_util.run_v1_only('b/120545219')
   def testWithValues(self):
     rt1 = ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])
     rt2 = ragged.constant([[[1, 2], [3, 4, 5]], [[6]], [], [[], [7]]])
 
     rt1_plus_10 = rt1.with_values(rt1.values + 10)
-    rt2_times_10 = rt2.with_inner_values(rt2.inner_values * 10)
+    rt2_times_10 = rt2.with_flat_values(rt2.flat_values * 10)
     rt1_expanded = rt1.with_values(array_ops.expand_dims(rt1.values, axis=1))
 
     self.assertEqual(
-        self.evaluate(rt1_plus_10).tolist(),
+        self.eval_to_list(rt1_plus_10),
         [[11, 12], [13, 14, 15], [16], [], [17]])
     self.assertEqual(
-        self.evaluate(rt2_times_10).tolist(),
+        self.eval_to_list(rt2_times_10),
         [[[10, 20], [30, 40, 50]], [[60]], [], [[], [70]]])
     self.assertEqual(
-        self.evaluate(rt1_expanded).tolist(),
+        self.eval_to_list(rt1_expanded),
         [[[1], [2]], [[3], [4], [5]], [[6]], [], [[7]]])
 
   #=============================================================================
   # Session.run
   #=============================================================================
-  @test_util.run_deprecated_v1
   def testSessionRun(self):
+    if context.executing_eagerly():
+      return
+
     rt1 = ragged.constant([[1, 2, 3], [4]])
     rt2 = ragged.constant([[[], [1, 2]], [[3]]])
     with self.test_session() as session:
       result = session.run({'rt1': rt1, 'rt2': rt2})
       self.assertCountEqual(sorted(result.keys()), ['rt1', 'rt2'])
-      self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]])
-      self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]])
+      self.assertEqual(result['rt1'].to_list(), [[1, 2, 3], [4]])
+      self.assertEqual(result['rt2'].to_list(), [[[], [1, 2]], [[3]]])
 
-  @test_util.run_deprecated_v1
   def testSessionRunFeed(self):
-    rt1 = ragged.from_row_splits(
+    if context.executing_eagerly():
+      return
+
+    rt1 = RaggedTensor.from_row_splits(
         array_ops.placeholder(dtypes.int32),
         array_ops.placeholder(dtypes.int64))
-    rt2 = ragged.from_nested_row_splits(
-        array_ops.placeholder(dtypes.int32),
-        [array_ops.placeholder(dtypes.int64),
-         array_ops.placeholder(dtypes.int64)])
+    rt2 = RaggedTensor.from_nested_row_splits(
+        array_ops.placeholder(dtypes.int32), [
+            array_ops.placeholder(dtypes.int64),
+            array_ops.placeholder(dtypes.int64)
+        ])
 
     rt1_feed_val = ragged.constant_value([[1, 2, 3], [4]])
     rt2_feed_val = ragged.constant_value([[[], [1, 2]], [[3]]])
 
     with self.test_session() as session:
-      result = session.run({'rt1': rt1, 'rt2': rt2},
-                           feed_dict={rt1: rt1_feed_val,
-                                      rt2: rt2_feed_val})
+      result = session.run({
+          'rt1': rt1,
+          'rt2': rt2
+      },
+                           feed_dict={
+                               rt1: rt1_feed_val,
+                               rt2: rt2_feed_val
+                           })
       self.assertCountEqual(sorted(result.keys()), ['rt1', 'rt2'])
-      self.assertEqual(result['rt1'].tolist(), [[1, 2, 3], [4]])
-      self.assertEqual(result['rt2'].tolist(), [[[], [1, 2]], [[3]]])
+      self.assertEqual(result['rt1'].to_list(), [[1, 2, 3], [4]])
+      self.assertEqual(result['rt2'].to_list(), [[[], [1, 2]], [[3]]])
 
-  @test_util.run_v1_only('b/120545219')
   def testSessionPartialRunFeed(self):
+    if context.executing_eagerly():
+      return
+
     # Placeholder inputs.
-    a = ragged.from_row_splits(
+    a = RaggedTensor.from_row_splits(
         array_ops.placeholder(dtypes.int32, shape=[None], name='a.values'),
         array_ops.placeholder(dtypes.int64, name='a.row_splits'))
-    b = ragged.from_row_splits(
+    b = RaggedTensor.from_row_splits(
         array_ops.placeholder(dtypes.int32, shape=[None], name='b.values'),
         array_ops.placeholder(dtypes.int64, name='b.row_splits'))
     c = array_ops.placeholder(dtypes.int32, shape=[], name='c')
@@ -1232,11 +1198,10 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       handle = session.partial_run_setup([r1, r2], [a, b, c])
 
       res1 = session.partial_run(handle, r1, feed_dict={a: a_val, b: b_val})
-      self.assertEqual(res1.tolist(), [22, 8])
+      self.assertAllEqual(res1, [22, 8])
 
       res2 = session.partial_run(handle, r2, feed_dict={c: c_val})
-      self.assertEqual(res2.tolist(), [15, 7])
-
+      self.assertAllEqual(res2, [15, 7])
 
 if __name__ == '__main__':
   googletest.main()
diff --git a/tensorflow/python/ops/ragged/ragged_tensor_value.py b/tensorflow/python/ops/ragged/ragged_tensor_value.py
index 39d3249c99..bf0ac4482a 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_value.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_value.py
@@ -53,7 +53,7 @@ class RaggedTensorValue(object):
       doc="""The numpy dtype of values in this tensor.""")
 
   @property
-  def inner_values(self):
+  def flat_values(self):
     """The innermost `values` array for this ragged tensor value."""
     rt_values = self.values
     while isinstance(rt_values, RaggedTensorValue):
@@ -82,15 +82,18 @@ class RaggedTensorValue(object):
     return (self._row_splits.shape[0] - 1,) + (None,) + self._values.shape[1:]
 
   def __str__(self):
-    return "<RaggedTensorValue %s>" % self.tolist()
+    return "<tf.RaggedTensorValue %s>" % self.to_list()
 
   def __repr__(self):
-    return "RaggedTensorValue(values=%r, row_splits=%r)" % (self._values,
-                                                            self._row_splits)
+    return "tf.RaggedTensorValue(values=%r, row_splits=%r)" % (self._values,
+                                                               self._row_splits)
 
-  def tolist(self):
+  def to_list(self):
     """Returns this ragged tensor value as a nested Python list."""
-    values_as_list = self._values.tolist()
+    if isinstance(self._values, RaggedTensorValue):
+      values_as_list = self._values.to_list()
+    else:
+      values_as_list = self._values.tolist()
     return [
         values_as_list[self._row_splits[i]:self._row_splits[i + 1]]
         for i in range(len(self._row_splits) - 1)
diff --git a/tensorflow/python/ops/ragged/ragged_test_util.py b/tensorflow/python/ops/ragged/ragged_test_util.py
new file mode 100644
index 0000000000..027417664d
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_test_util.py
@@ -0,0 +1,95 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# pylint: disable=invalid-name
+"""Test utils for tensorflow RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import ragged
+
+
+class RaggedTensorTestCase(test_util.TensorFlowTestCase):
+  """Base class for RaggedTensor test cases."""
+
+  def _GetPyList(self, a):
+    """Converts a to a nested python list."""
+    if isinstance(a, ragged.RaggedTensor):
+      return self.evaluate(a).to_list()
+    elif isinstance(a, ops.Tensor):
+      a = self.evaluate(a)
+      return a.tolist() if isinstance(a, np.ndarray) else a
+    elif isinstance(a, np.ndarray):
+      return a.tolist()
+    elif isinstance(a, ragged.RaggedTensorValue):
+      return a.to_list()
+    else:
+      return np.array(a).tolist()
+
+  def assertRaggedEqual(self, a, b):
+    """Asserts that two potentially ragged tensors are equal."""
+    a_list = self._GetPyList(a)
+    b_list = self._GetPyList(b)
+    self.assertEqual(a_list, b_list)
+
+    if not (isinstance(a, (list, tuple)) or isinstance(b, (list, tuple))):
+      a_ragged_rank = a.ragged_rank if ragged.is_ragged(a) else 0
+      b_ragged_rank = b.ragged_rank if ragged.is_ragged(b) else 0
+      self.assertEqual(a_ragged_rank, b_ragged_rank)
+
+  def assertRaggedAlmostEqual(self, a, b, places=7):
+    a_list = self._GetPyList(a)
+    b_list = self._GetPyList(b)
+    self.assertNestedListAlmostEqual(a_list, b_list, places, context='value')
+
+    if not (isinstance(a, (list, tuple)) or isinstance(b, (list, tuple))):
+      a_ragged_rank = a.ragged_rank if ragged.is_ragged(a) else 0
+      b_ragged_rank = b.ragged_rank if ragged.is_ragged(b) else 0
+      self.assertEqual(a_ragged_rank, b_ragged_rank)
+
+  def assertNestedListAlmostEqual(self, a, b, places=7, context='value'):
+    self.assertEqual(type(a), type(b))
+    if isinstance(a, (list, tuple)):
+      self.assertLen(a, len(b), 'Length differs for %s' % context)
+      for i in range(len(a)):
+        self.assertNestedListAlmostEqual(a[i], b[i], places,
+                                         '%s[%s]' % (context, i))
+    else:
+      self.assertAlmostEqual(
+          a, b, places,
+          '%s != %s within %s places at %s' % (a, b, places, context))
+
+  def eval_to_list(self, tensor):
+    value = self.evaluate(tensor)
+    if ragged.is_ragged(value):
+      return value.to_list()
+    elif isinstance(value, np.ndarray):
+      return value.tolist()
+    else:
+      return value
+
+  def _eval_tensor(self, tensor):
+    if ragged.is_ragged(tensor):
+      return ragged.RaggedTensorValue(
+          self._eval_tensor(tensor.values),
+          self._eval_tensor(tensor.row_splits))
+    else:
+      return test_util.TensorFlowTestCase._eval_tensor(self, tensor)
diff --git a/tensorflow/python/ops/ragged/ragged_tile_op_test.py b/tensorflow/python/ops/ragged/ragged_tile_op_test.py
index f335b15dd1..d3445571bf 100644
--- a/tensorflow/python/ops/ragged/ragged_tile_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_tile_op_test.py
@@ -24,12 +24,14 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
-from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTileOpTest(ragged_test_util.RaggedTensorTestCase,
+                       parameterized.TestCase):
 
   @parameterized.parameters([
       #=========================================================================
@@ -181,14 +183,13 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                     [[[5], [6]]]]),
 
   ])  # pyformat: disable
-  @test_util.run_deprecated_v1
   def testRaggedTile(self,
                      descr,
                      rt_input,
                      multiples,
                      expected,
                      ragged_rank=None):
-    rt = ragged_factory_ops.constant(rt_input, ragged_rank)
+    rt = ragged.constant(rt_input, ragged_rank)
 
     expected_shape = [
         None if dim is None else dim * multiple
@@ -202,24 +203,21 @@ class RaggedTileOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
         const_multiples, shape=[len(multiples)])
 
     for multiples_tensor in (const_multiples, non_const_multiples):
-      tiled = ragged_array_ops.tile(rt, multiples_tensor)
+      tiled = ragged.tile(rt, multiples_tensor)
       self.assertEqual(tiled.ragged_rank, rt.ragged_rank)
       self.assertEqual(tiled.shape.ndims, rt.shape.ndims)
       if multiples_tensor is const_multiples:
         self.assertEqual(tiled.shape.as_list(), expected_shape)
-      with self.test_session():
-        self.assertEqual(tiled.eval().tolist(), expected)
+      self.assertRaggedEqual(tiled, expected)
 
-  @test_util.run_deprecated_v1
   def testRaggedTileWithTensorInput(self):
     # When the input is a `Tensor`, ragged_tile just delegates to tf.tile.
     dt = constant_op.constant([[1, 2], [3, 4]])
-    tiled = ragged_array_ops.tile(dt, [3, 2])
+    tiled = ragged.tile(dt, [3, 2])
     expected = [[1, 2, 1, 2], [3, 4, 3, 4],
                 [1, 2, 1, 2], [3, 4, 3, 4],
                 [1, 2, 1, 2], [3, 4, 3, 4]]  # pyformat: disable
-    with self.test_session():
-      self.assertEqual(tiled.eval().tolist(), expected)
+    self.assertRaggedEqual(tiled, expected)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
index 9863e3b583..46d7a56a7c 100644
--- a/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_sparse_op_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
@@ -25,176 +26,168 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorToSparseOpTest(test_util.TensorFlowTestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorToSparseOpTest(ragged_test_util.RaggedTensorTestCase):
 
-  @test_util.run_deprecated_v1
   def testDocStringExample(self):
     rt = ragged.constant([[1, 2, 3], [4], [], [5, 6]])
-    st = ragged.to_sparse(rt)
-    expected = ('SparseTensorValue(indices='
-                'array([[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [3, 1]]), '
-                'values=array([1, 2, 3, 4, 5, 6], dtype=int32), '
-                'dense_shape=array([4, 3]))')
-    with self.test_session():
-      self.assertEqual(' '.join(repr(st.eval()).split()), expected)
-
-  @test_util.run_deprecated_v1
+    st = self.evaluate(rt.to_sparse())
+    self.assertAllEqual(st.indices,
+                        [[0, 0], [0, 1], [0, 2], [1, 0], [3, 0], [3, 1]])
+    self.assertAllEqual(st.values, [1, 2, 3, 4, 5, 6])
+    self.assertAllEqual(st.dense_shape, [4, 3])
+
   def test2DRaggedTensorWithOneRaggedDimension(self):
     rt = ragged.constant([['a', 'b'], ['c', 'd', 'e'], ['f'], [], ['g']])
-    with self.test_session():
-      st = ragged.to_sparse(rt).eval()
-      self.assertAllEqual(
-          st.indices, [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [2, 0], [4, 0]])
-      self.assertAllEqual(st.values, b'a b c d e f g'.split())
-      self.assertAllEqual(st.dense_shape, [5, 3])
-
-  @test_util.run_deprecated_v1
+    st = self.evaluate(rt.to_sparse())
+    self.assertAllEqual(
+        st.indices, [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [2, 0], [4, 0]])
+    self.assertAllEqual(st.values, b'a b c d e f g'.split())
+    self.assertAllEqual(st.dense_shape, [5, 3])
+
   def test3DRaggedTensorWithOneRaggedDimension(self):
     rt = ragged.constant([[[1, 2], [3, 4]], [[5, 6], [7, 8], [9, 10]],
                           [[11, 12]], [], [[13, 14]]],
                          ragged_rank=1)
-    with self.test_session():
-      st = ragged.to_sparse(rt).eval()
-      self.assertAllEqual(
-          st.indices, [[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0],
-                       [1, 0, 1], [1, 1, 0], [1, 1, 1], [1, 2, 0], [1, 2, 1],
-                       [2, 0, 0], [2, 0, 1], [4, 0, 0], [4, 0, 1]])
-      self.assertAllEqual(st.values,
-                          [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
-      self.assertAllEqual(st.dense_shape, [5, 3, 2])
-
-  @test_util.run_deprecated_v1
+    st = self.evaluate(rt.to_sparse())
+    self.assertAllEqual(st.indices,
+                        [[0, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0],
+                         [1, 0, 1], [1, 1, 0], [1, 1, 1], [1, 2, 0], [1, 2, 1],
+                         [2, 0, 0], [2, 0, 1], [4, 0, 0], [4, 0, 1]])
+    self.assertAllEqual(st.values,
+                        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
+    self.assertAllEqual(st.dense_shape, [5, 3, 2])
+
   def test4DRaggedTensorWithOneRaggedDimension(self):
     rt = ragged.constant(
         [[[[1, 2], [3, 4]], [[5, 6], [7, 8]]], [], [[[9, 10], [11, 12]]]],
         ragged_rank=1)
-    with self.test_session():
-      st = ragged.to_sparse(rt).eval()
-      self.assertAllEqual(st.values, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
-      self.assertAllEqual(
-          st.indices,
-          [
-              [0, 0, 0, 0],  # index for value=1
-              [0, 0, 0, 1],  # index for value=2
-              [0, 0, 1, 0],  # index for value=3
-              [0, 0, 1, 1],  # index for value=4
-              [0, 1, 0, 0],  # index for value=5
-              [0, 1, 0, 1],  # index for value=6
-              [0, 1, 1, 0],  # index for value=7
-              [0, 1, 1, 1],  # index for value=8
-              [2, 0, 0, 0],  # index for value=9
-              [2, 0, 0, 1],  # index for value=10
-              [2, 0, 1, 0],  # index for value=11
-              [2, 0, 1, 1],  # index for value=12
-          ])
-      self.assertAllEqual(st.dense_shape, [3, 2, 2, 2])
-
-  @test_util.run_deprecated_v1
+    st = self.evaluate(rt.to_sparse())
+    self.assertAllEqual(st.values, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
+    self.assertAllEqual(
+        st.indices,
+        [
+            [0, 0, 0, 0],  # index for value=1
+            [0, 0, 0, 1],  # index for value=2
+            [0, 0, 1, 0],  # index for value=3
+            [0, 0, 1, 1],  # index for value=4
+            [0, 1, 0, 0],  # index for value=5
+            [0, 1, 0, 1],  # index for value=6
+            [0, 1, 1, 0],  # index for value=7
+            [0, 1, 1, 1],  # index for value=8
+            [2, 0, 0, 0],  # index for value=9
+            [2, 0, 0, 1],  # index for value=10
+            [2, 0, 1, 0],  # index for value=11
+            [2, 0, 1, 1],  # index for value=12
+        ])
+    self.assertAllEqual(st.dense_shape, [3, 2, 2, 2])
+
   def test4DRaggedTensorWithTwoRaggedDimensions(self):
     rt = ragged.constant([[[[1, 2], [3, 4]], [[5, 6], [7, 8], [9, 10]]],
                           [[[11, 12]], [], [[13, 14]]], []],
                          ragged_rank=2)
-    with self.test_session():
-      st = ragged.to_sparse(rt).eval()
-      self.assertAllEqual(
-          st.indices,
-          [
-              [0, 0, 0, 0],  # index for value=1
-              [0, 0, 0, 1],  # index for value=2
-              [0, 0, 1, 0],  # index for value=3
-              [0, 0, 1, 1],  # index for value=4
-              [0, 1, 0, 0],  # index for value=5
-              [0, 1, 0, 1],  # index for value=6
-              [0, 1, 1, 0],  # index for value=7
-              [0, 1, 1, 1],  # index for value=8
-              [0, 1, 2, 0],  # index for value=9
-              [0, 1, 2, 1],  # index for value=10
-              [1, 0, 0, 0],  # index for value=11
-              [1, 0, 0, 1],  # index for value=12
-              [1, 2, 0, 0],  # index for value=13
-              [1, 2, 0, 1],  # index for value=14
-          ])
-      self.assertAllEqual(st.values,
-                          [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
-      self.assertAllEqual(st.dense_shape, [3, 3, 3, 2])
+    st = self.evaluate(rt.to_sparse())
+    self.assertAllEqual(
+        st.indices,
+        [
+            [0, 0, 0, 0],  # index for value=1
+            [0, 0, 0, 1],  # index for value=2
+            [0, 0, 1, 0],  # index for value=3
+            [0, 0, 1, 1],  # index for value=4
+            [0, 1, 0, 0],  # index for value=5
+            [0, 1, 0, 1],  # index for value=6
+            [0, 1, 1, 0],  # index for value=7
+            [0, 1, 1, 1],  # index for value=8
+            [0, 1, 2, 0],  # index for value=9
+            [0, 1, 2, 1],  # index for value=10
+            [1, 0, 0, 0],  # index for value=11
+            [1, 0, 0, 1],  # index for value=12
+            [1, 2, 0, 0],  # index for value=13
+            [1, 2, 0, 1],  # index for value=14
+        ])
+    self.assertAllEqual(st.values,
+                        [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
+    self.assertAllEqual(st.dense_shape, [3, 3, 3, 2])
 
   def testShape(self):
     rt = ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])
-    st = ragged.to_sparse(rt)
+    st = rt.to_sparse()
     self.assertEqual(st.indices.shape.as_list(), [7, 2])
     self.assertEqual(st.values.shape.as_list(), [7])
     self.assertEqual(st.dense_shape.shape.as_list(), [2])
 
     rt = ragged.constant([[[1, 2]], [], [[3, 4]], []], ragged_rank=1)
-    st = ragged.to_sparse(rt)
+    st = rt.to_sparse()
     self.assertEqual(st.indices.shape.as_list(), [4, 3])
     self.assertEqual(st.values.shape.as_list(), [4])
     self.assertEqual(st.dense_shape.shape.as_list(), [3])
 
     rt = ragged.constant([[[1], [2, 3, 4, 5, 6, 7]], [[]]])
-    st = ragged.to_sparse(rt)
+    st = rt.to_sparse()
     self.assertEqual(st.indices.shape.as_list(), [7, 3])
     self.assertEqual(st.values.shape.as_list(), [7])
     self.assertEqual(st.dense_shape.shape.as_list(), [3])
 
-  @test_util.run_v1_only('b/120545219')
   def testKernelErrors(self):
     # An empty vector, defined using a placeholder to ensure that we can't
     # determine that it's invalid at graph-construction time.
     empty_vector = array_ops.placeholder_with_default(
         array_ops.zeros([0], dtypes.int64), shape=None)
 
-    bad_rt1 = ragged.from_row_splits(row_splits=[2, 3], values=[1, 2, 3])
-    with self.test_session():
-      bad_split0_error = r'First value of ragged splits must be 0.*'
-      self.assertRaisesRegexp(errors.InvalidArgumentError, bad_split0_error,
-                              ragged.to_sparse(bad_rt1).eval)
+    bad_rt1 = ragged.RaggedTensor.from_row_splits(
+        row_splits=[2, 3], values=[1, 2, 3])
+    bad_split0 = r'First value of ragged splits must be 0.*'
+    with self.assertRaisesRegexp(errors.InvalidArgumentError, bad_split0):
+      self.evaluate(bad_rt1.to_sparse())
 
-    bad_rt2 = ragged.from_row_splits(row_splits=[0, 5], values=empty_vector)
-    bad_rt3 = ragged.from_row_splits(
+    bad_rt2 = ragged.RaggedTensor.from_row_splits(
+        row_splits=[0, 5], values=empty_vector)
+    bad_rt3 = ragged.RaggedTensor.from_row_splits(
         row_splits=[0, 1],
-        values=ragged.from_row_splits(row_splits=[0, 5], values=empty_vector))
-    with self.test_session():
-      split_mismatch1_error = r'Final value of ragged splits must match.*'
-      for rt in [bad_rt2, bad_rt3]:
-        self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                split_mismatch1_error,
-                                ragged.to_sparse(rt).eval)
-
-    bad_rt4 = ragged.from_row_splits(
+        values=ragged.RaggedTensor.from_row_splits(
+            row_splits=[0, 5], values=empty_vector))
+    split_mismatch1_error = r'Final value of ragged splits must match.*'
+    for rt in [bad_rt2, bad_rt3]:
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   split_mismatch1_error):
+        self.evaluate(rt.to_sparse())
+
+    bad_rt4 = ragged.RaggedTensor.from_row_splits(
         row_splits=[0, 5],
-        values=ragged.from_row_splits(row_splits=[0], values=empty_vector))
-    with self.test_session():
-      split_mismatch2_error = r'Final value of ragged splits must match.*'
-      self.assertRaisesRegexp(errors.InvalidArgumentError,
-                              split_mismatch2_error,
-                              ragged.to_sparse(bad_rt4).eval)
-
-    bad_rt5 = ragged.from_row_splits(row_splits=empty_vector, values=[])
-    with self.test_session():
-      empty_splits_error = (r'ragged splits may not be empty.*')
-      self.assertRaisesRegexp(errors.InvalidArgumentError, empty_splits_error,
-                              ragged.to_sparse(bad_rt5).eval)
-
-  @test_util.run_v1_only('b/120545219')
+        values=ragged.RaggedTensor.from_row_splits(
+            row_splits=[0], values=empty_vector))
+    split_mismatch2_error = r'Final value of ragged splits must match.*'
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 split_mismatch2_error):
+      self.evaluate(bad_rt4.to_sparse())
+
+    bad_rt5 = ragged.RaggedTensor.from_row_splits(
+        row_splits=empty_vector, values=[])
+    empty_splits_error = (r'ragged splits may not be empty.*')
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 empty_splits_error):
+      self.evaluate(bad_rt5.to_sparse())
+
   def testGradient(self):
+    if context.executing_eagerly():
+      return
     # rt1.shape == rt2.shape == [2, (D2), (D3), 2].
     rt1 = ragged.constant([[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0]]]],
                           ragged_rank=2)
     rt2 = ragged.constant([[[[9.0, 8.0], [7.0, 6.0]], [[5.0, 4.0]]]],
                           ragged_rank=2)
-    rt = ragged.map_inner_values(math_ops.add, rt1, rt2 * 2.0)
-    st = ragged.to_sparse(rt)
+    rt = ragged.map_flat_values(math_ops.add, rt1, rt2 * 2.0)
+    st = rt.to_sparse()
 
-    g1, g2 = gradients_impl.gradients(st.values, [rt1.inner_values,
-                                                  rt2.inner_values])
+    g1, g2 = gradients_impl.gradients(st.values,
+                                      [rt1.flat_values, rt2.flat_values])
     print(g1, g2)
-    with self.test_session():
-      self.assertEqual(g1.eval().tolist(), [[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]])
-      self.assertEqual(g2.eval().tolist(), [[2.0, 2.0], [2.0, 2.0], [2.0, 2.0]])
+    self.assertRaggedEqual(g1, [[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]])
+    self.assertRaggedEqual(g2, [[2.0, 2.0], [2.0, 2.0], [2.0, 2.0]])
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
index 77499b9cb3..ffcc2be52e 100644
--- a/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_to_tensor_op_test.py
@@ -24,23 +24,19 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedTensorToTensorOpTest(ragged_test_util.RaggedTensorTestCase,
                                  parameterized.TestCase):
 
-  @test_util.run_deprecated_v1
   def testDocStringExamples(self):
     """Example from ragged_to_tensor.__doc__."""
     rt = ragged.constant([[9, 8, 7], [], [6, 5], [4]])
-    dt = ragged.to_tensor(rt)
-    with self.test_session():
-      self.assertEqual(str(dt.eval()),
-                       '[[9 8 7]\n'
-                       ' [0 0 0]\n'
-                       ' [6 5 0]\n'
-                       ' [4 0 0]]')  # pyformat: disable
+    dt = rt.to_tensor()
+    self.assertAllEqual(dt, [[9, 8, 7], [0, 0, 0], [6, 5, 0], [4, 0, 0]])
 
   @parameterized.parameters(
       {
@@ -98,7 +94,6 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
           'expected': [[[[1], [2]], [[9], [9]], [[3], [9]]]],
       },
   )
-  @test_util.run_deprecated_v1
   def testRaggedTensorToTensor(self,
                                rt_input,
                                expected,
@@ -106,15 +101,14 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
                                default=None,
                                expected_shape=None):
     rt = ragged.constant(rt_input, ragged_rank=ragged_rank)
-    dt = ragged.to_tensor(rt, default)
-    self.assertEqual(type(dt), ops.Tensor)
+    dt = rt.to_tensor(default)
+    self.assertIsInstance(dt, ops.Tensor)
     self.assertEqual(rt.dtype, dt.dtype)
     self.assertTrue(dt.shape.is_compatible_with(rt.shape))
-    with self.test_session():
-      self.assertEqual(dt.eval().tolist(), expected)
-      if expected_shape is not None:
-        dt_shape = array_ops.shape(dt)
-        self.assertEqual(dt_shape.eval().tolist(), expected_shape)
+    self.assertAllEqual(self.eval_to_list(dt), expected)
+    if expected_shape is not None:
+      dt_shape = array_ops.shape(dt)
+      self.assertAllEqual(dt_shape, expected_shape)
 
   @parameterized.parameters(
       {
@@ -131,14 +125,13 @@ class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
       {
           'rt_input': [[1, 2, 3]],
           'default': 'a',
-          'error': (TypeError, "Expected int32, got 'a' of type 'str' instead"),
+          'error': (TypeError, '.*'),
       },
   )
-  @test_util.run_deprecated_v1
   def testError(self, rt_input, default, error, ragged_rank=None):
     rt = ragged.constant(rt_input, ragged_rank=ragged_rank)
     with self.assertRaisesRegexp(error[0], error[1]):
-      ragged.to_tensor(rt, default)
+      rt.to_tensor(default)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_util_test.py b/tensorflow/python/ops/ragged/ragged_util_test.py
index 69c605dbf9..72a4155930 100644
--- a/tensorflow/python/ops/ragged/ragged_util_test.py
+++ b/tensorflow/python/ops/ragged/ragged_util_test.py
@@ -24,6 +24,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.ops.ragged import ragged_util
 from tensorflow.python.platform import googletest
 
@@ -41,7 +42,9 @@ TENSOR_4D = [[[[('%d%d%d%d' % (i, j, k, l)).encode('utf-8')
              for i in range(4)]
 
 
-class RaggedRepeatTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedUtilTest(ragged_test_util.RaggedTensorTestCase,
+                     parameterized.TestCase):
 
   @parameterized.parameters([
       # Docstring examples
@@ -87,11 +90,10 @@ class RaggedRepeatTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       # Empty tensor
       dict(data=[], repeats=[], axis=0, expected=[]),
   ])
-  @test_util.run_v1_only('b/120545219')
   def testRepeat(self, data, repeats, expected, axis=None):
     result = ragged_util.repeat(data, repeats, axis)
     with self.test_session():
-      self.assertEqual(result.eval().tolist(), expected)
+      self.assertAllEqual(result, expected)
 
   @parameterized.parameters([
       dict(mode=mode, **args)
@@ -136,7 +138,6 @@ class RaggedRepeatTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           dict(data=TENSOR_4D, repeats=[1, 3, 0, 0, 2], axis=3),
       ]
   ])
-  @test_util.run_v1_only('b/120545219')
   def testValuesMatchesNumpy(self, mode, data, repeats, axis):
     # Exception: we can't handle negative axis if data.ndims is unknown.
     if axis < 0 and mode == 'unknown_shape':
@@ -158,7 +159,7 @@ class RaggedRepeatTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     result = ragged_util.repeat(data, repeats, axis)
     with self.test_session():
-      self.assertEqual(result.eval().tolist(), expected.tolist())
+      self.assertAllEqual(result, expected)
 
   @parameterized.parameters([
       dict(
diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py
index 6d645eefac..b3cd5a2deb 100644
--- a/tensorflow/python/ops/ragged/ragged_where_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py
@@ -22,10 +22,13 @@ from absl.testing import parameterized
 
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import ragged
+from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
 
-class RaggedWhereOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedWhereOpTest(ragged_test_util.RaggedTensorTestCase,
+                        parameterized.TestCase):
 
   @parameterized.parameters([
       #=========================================================================
@@ -165,18 +168,9 @@ class RaggedWhereOpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
           y=ragged.constant_value([[[['a']]], [[['b']]]]),
           expected=ragged.constant_value([[[[], [b'A']]], [[[b'b']]]])),
   ])   # pyformat: disable
-  @test_util.run_v1_only('b/120545219')
   def testRaggedWhere(self, condition, expected, x=None, y=None):
     result = ragged.where(condition, x, y)
-    self.assertEqual(
-        getattr(result, 'ragged_rank', 0), getattr(expected, 'ragged_rank', 0))
-    with self.test_session():
-      result_value = self.evaluate(result)
-      if hasattr(result_value, 'tolist'):
-        result_value = result_value.tolist()
-      if hasattr(expected, 'tolist'):
-        expected = expected.tolist()
-      self.assertEqual(result_value, expected)
+    self.assertRaggedEqual(result, expected)
 
   @parameterized.parameters([
       dict(
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index f34ac4c3e8..baacb87239 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -113,6 +113,7 @@ COMMON_PIP_DEPS = [
     "//tensorflow/python/eager:eager_pip",
     "//tensorflow/python/kernel_tests/signal:test_util",
     "//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files",
+    "//tensorflow/python/ops/ragged:ragged_test_util",
     "//tensorflow/python/saved_model:saved_model",
     "//tensorflow/python/tools:tools_pip",
     "//tensorflow/python/tools/api/generator:create_python_api",
-- 
GitLab


From 49123b01d3c18ad82cb9f754b8e130baa691a409 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Fri, 7 Dec 2018 22:47:27 -0800
Subject: [PATCH 1547/1554] Fix flexbuffer byte type and annotation.

PiperOrigin-RevId: 224623431
---
 tensorflow/lite/schema/schema.fbs         |  2 +-
 tensorflow/lite/schema/schema_generated.h | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 6436167303..980f13b19b 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -45,7 +45,7 @@ enum TensorType : byte {
 // Custom quantization parameters for experimenting with new quantization
 // techniques.
 table CustomQuantization {
-  custom:[byte];
+  custom:[ubyte] (force_align: 16);
 }
 
 // Represents a specific quantization technique's parameters.
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index af8b143364..637cbafabd 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -2247,7 +2247,7 @@ inline const char *EnumNameCustomOptionsFormat(CustomOptionsFormat e) {
 
 struct CustomQuantizationT : public flatbuffers::NativeTable {
   typedef CustomQuantization TableType;
-  std::vector<int8_t> custom;
+  std::vector<uint8_t> custom;
   CustomQuantizationT() {
   }
 };
@@ -2257,8 +2257,8 @@ struct CustomQuantization FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   enum {
     VT_CUSTOM = 4
   };
-  const flatbuffers::Vector<int8_t> *custom() const {
-    return GetPointer<const flatbuffers::Vector<int8_t> *>(VT_CUSTOM);
+  const flatbuffers::Vector<uint8_t> *custom() const {
+    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
@@ -2274,7 +2274,7 @@ struct CustomQuantization FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 struct CustomQuantizationBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
-  void add_custom(flatbuffers::Offset<flatbuffers::Vector<int8_t>> custom) {
+  void add_custom(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> custom) {
     fbb_.AddOffset(CustomQuantization::VT_CUSTOM, custom);
   }
   explicit CustomQuantizationBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -2291,7 +2291,7 @@ struct CustomQuantizationBuilder {
 
 inline flatbuffers::Offset<CustomQuantization> CreateCustomQuantization(
     flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::Vector<int8_t>> custom = 0) {
+    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> custom = 0) {
   CustomQuantizationBuilder builder_(_fbb);
   builder_.add_custom(custom);
   return builder_.Finish();
@@ -2299,10 +2299,10 @@ inline flatbuffers::Offset<CustomQuantization> CreateCustomQuantization(
 
 inline flatbuffers::Offset<CustomQuantization> CreateCustomQuantizationDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
-    const std::vector<int8_t> *custom = nullptr) {
+    const std::vector<uint8_t> *custom = nullptr) {
   return tflite::CreateCustomQuantization(
       _fbb,
-      custom ? _fbb.CreateVector<int8_t>(*custom) : 0);
+      custom ? _fbb.CreateVector<uint8_t>(*custom) : 0);
 }
 
 flatbuffers::Offset<CustomQuantization> CreateCustomQuantization(flatbuffers::FlatBufferBuilder &_fbb, const CustomQuantizationT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
-- 
GitLab


From 8ca891f53de9145113d889f876ab0cc9efee3c34 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 7 Dec 2018 23:09:51 -0800
Subject: [PATCH 1548/1554] Deflake optimize_dataset_test in tsan mode.

PiperOrigin-RevId: 224624656
---
 .../python/data/experimental/kernel_tests/optimization/BUILD    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 2fc243aa13..bf868ebe79 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -286,7 +286,7 @@ py_test(
 
 py_test(
     name = "optimize_dataset_test",
-    size = "small",
+    size = "medium",
     srcs = ["optimize_dataset_test.py"],
     srcs_version = "PY2AND3",
     tags = [
-- 
GitLab


From d7fd24856d77b14021524b5cd3833cfd322e665d Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Fri, 7 Dec 2018 23:10:18 -0800
Subject: [PATCH 1549/1554] Add tf.estimator as a valid v1 api prefix in
 testAllAPIV1 check in tf_upgrade_v2_test.py.

PiperOrigin-RevId: 224624681
---
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 63aa5f0c6b..0fc7a18734 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -198,6 +198,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
             _, _, _, text = self._upgrade("tf." + name)
             if (text and
                 not text.startswith("tf.compat.v1") and
+                not text.startswith("tf.estimator") and
                 text not in v1_symbols):
               self.assertFalse(
                   True, "Symbol %s generated from %s not in v1 API" % (
-- 
GitLab


From a3f24855d6697c37f1842e097a4563d6613d906b Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 7 Dec 2018 23:37:13 -0800
Subject: [PATCH 1550/1554] Allow TensorFlow demo app to switch between devices

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

- ListView gui for model (float and quant)
- ListView gui for devices (optional gpu, nnapi and cpu)
- Add a few AAR options.

PiperOrigin-RevId: 224626109
---
 tensorflow/lite/java/demo/app/build.gradle    |  12 ++
 .../Camera2BasicFragment.java                 | 164 ++++++++++++++----
 .../tflitecamerademo/GpuDelegateHelper.java   |  49 ++++++
 .../tflitecamerademo/ImageClassifier.java     |  14 ++
 .../ImageClassifierFloatMobileNet.java        |  94 ++++++++++
 .../src/main/res/drawable/item_selector.xml   |  11 ++
 .../res/layout-v26/fragment_camera2_basic.xml |  98 +++++++----
 .../res/layout/fragment_camera2_basic.xml     |  88 +++++++---
 .../app/src/main/res/layout/listview_row.xml  |  17 ++
 .../demo/app/src/main/res/values/colors.xml   |   3 +
 .../demo/app/src/main/res/values/strings.xml  |   6 +
 11 files changed, 463 insertions(+), 93 deletions(-)
 create mode 100644 tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/GpuDelegateHelper.java
 create mode 100644 tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierFloatMobileNet.java
 create mode 100644 tensorflow/lite/java/demo/app/src/main/res/drawable/item_selector.xml
 create mode 100644 tensorflow/lite/java/demo/app/src/main/res/layout/listview_row.xml

diff --git a/tensorflow/lite/java/demo/app/build.gradle b/tensorflow/lite/java/demo/app/build.gradle
index 5e50ed4b94..b8fc282cb1 100644
--- a/tensorflow/lite/java/demo/app/build.gradle
+++ b/tensorflow/lite/java/demo/app/build.gradle
@@ -40,6 +40,15 @@ repositories {
         url 'https://google.bintray.com/tensorflow'
     }
 }
+allprojects {
+    repositories {
+        // Uncomment if you want to use a local repo.
+        // mavenLocal()
+        jcenter()
+    }
+}
+
+
 
 dependencies {
     compile fileTree(dir: 'libs', include: ['*.jar'])
@@ -49,7 +58,10 @@ dependencies {
     compile 'com.android.support:support-annotations:25.3.1'
     compile 'com.android.support:support-v13:25.2.0'
 
+    // Build off of nightly TensorFlow Lite
     compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    // Use local TensorFlow library
+    // compile 'org.tensorflow:tensorflow-lite-local:0.0.0'
 }
 
 def targetFolder = "src/main/assets"
diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index 20e96f586a..165d335101 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -56,8 +56,10 @@ import android.view.Surface;
 import android.view.TextureView;
 import android.view.View;
 import android.view.ViewGroup;
+import android.widget.AdapterView;
+import android.widget.ArrayAdapter;
+import android.widget.ListView;
 import android.widget.NumberPicker;
-import android.widget.RadioButton;
 import android.widget.TextView;
 import android.widget.Toast;
 import java.io.IOException;
@@ -69,6 +71,7 @@ import java.util.List;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+
 /** Basic fragments for the Camera. */
 public class Camera2BasicFragment extends Fragment
     implements FragmentCompat.OnRequestPermissionsResultCallback {
@@ -88,9 +91,9 @@ public class Camera2BasicFragment extends Fragment
   private TextView textView;
   private NumberPicker np;
   private ImageClassifier classifier;
+  private ListView deviceView;
+  private ListView modelView;
 
-  enum InferenceEngine { CPU, NNAPI };
-  private InferenceEngine inferenceEngine = InferenceEngine.CPU;
 
   /** Max preview width that is guaranteed by Camera2 API */
   private static final int MAX_PREVIEW_WIDTH = 1920;
@@ -124,6 +127,15 @@ public class Camera2BasicFragment extends Fragment
         public void onSurfaceTextureUpdated(SurfaceTexture texture) {}
       };
 
+  // Model parameter constants.
+  private String gpu;
+  private String cpu;
+  private String nnApi;
+  private String mobilenetV1Quant;
+  private String mobilenetV1Float;
+
+
+
   /** ID of the current {@link CameraDevice}. */
   private String cameraId;
 
@@ -170,6 +182,14 @@ public class Camera2BasicFragment extends Fragment
         }
       };
 
+  private ArrayList<String> deviceStrings = new ArrayList<String>();
+  private ArrayList<String> modelStrings = new ArrayList<String>();
+
+  /** Current indices of device and model. */
+  int currentDevice = -1;
+
+  int currentModel = -1;
+
   /** An additional thread for running tasks that shouldn't block the UI. */
   private HandlerThread backgroundThread;
 
@@ -299,11 +319,115 @@ public class Camera2BasicFragment extends Fragment
     return inflater.inflate(R.layout.fragment_camera2_basic, container, false);
   }
 
+  private void updateActiveModel() {
+    // Get UI information before delegating to background
+    final int modelIndex = modelView.getCheckedItemPosition();
+    final int deviceIndex = deviceView.getCheckedItemPosition();
+
+    backgroundHandler.post(() -> {
+      if (modelIndex == currentModel && deviceIndex == currentDevice) {
+        return;
+      }
+      currentModel = modelIndex;
+      currentDevice = deviceIndex;
+
+      // Disable classifier while updating
+      if (classifier != null) {
+        classifier.close();
+        classifier = null;
+      }
+
+      // Lookup names of parameters.
+      String model = modelStrings.get(modelIndex);
+      String device = deviceStrings.get(deviceIndex);
+
+      Log.i(TAG, "Changing model to " + model + " device " + device);
+
+      // Try to load model.
+      try {
+        if (model.equals(mobilenetV1Quant)) {
+          classifier = new ImageClassifierQuantizedMobileNet(getActivity());
+        } else if (model.equals(mobilenetV1Float)) {
+          classifier = new ImageClassifierFloatMobileNet(getActivity());
+        } else {
+          showToast("Failed to load model");
+        }
+      } catch (IOException e) {
+        Log.d(TAG, "Failed to load", e);
+        classifier = null;
+      }
+
+      // Customzie the interpreter to the type of device we want to use.
+      if (device.equals(cpu)) {
+      } else if (device.equals(gpu)) {
+        if (!GpuDelegateHelper.isGpuDelegateAvailable()) {
+          showToast("gpu not in this build.");
+          classifier = null;
+        } else if (model.equals(mobilenetV1Quant)) {
+          showToast("gpu requires float model.");
+          classifier = null;
+        } else {
+          classifier.useGpu();
+        }
+      } else if (device.equals(nnApi)) {
+        classifier.useNNAPI();
+      }
+    });
+  }
+
   /** Connect the buttons to their event handler. */
   @Override
   public void onViewCreated(final View view, Bundle savedInstanceState) {
+    gpu = getString(R.string.gpu);
+    cpu = getString(R.string.cpu);
+    nnApi = getString(R.string.nnapi);
+    mobilenetV1Quant = getString(R.string.mobilenetV1Quant);
+    mobilenetV1Float = getString(R.string.mobilenetV1Float);
+
+    // Get references to widgets.
     textureView = (AutoFitTextureView) view.findViewById(R.id.texture);
     textView = (TextView) view.findViewById(R.id.text);
+    deviceView = (ListView) view.findViewById(R.id.device);
+    modelView = (ListView) view.findViewById(R.id.model);
+
+    // Build list of models
+    modelStrings.add(mobilenetV1Quant);
+    modelStrings.add(mobilenetV1Float);
+
+    // Build list of devices
+    int defaultModelIndex = 0;
+    deviceStrings.add(cpu);
+    if (GpuDelegateHelper.isGpuDelegateAvailable()) {
+      deviceStrings.add(gpu);
+    }
+    deviceStrings.add(nnApi);
+
+    deviceView.setAdapter(
+        new ArrayAdapter<String>(
+            getContext(), R.layout.listview_row, R.id.listview_row_text, deviceStrings));
+    deviceView.setChoiceMode(ListView.CHOICE_MODE_SINGLE);
+    deviceView.setOnItemClickListener(
+        new AdapterView.OnItemClickListener() {
+          @Override
+          public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
+            updateActiveModel();
+          }
+        });
+    deviceView.setItemChecked(0, true);
+
+    modelView.setChoiceMode(ListView.CHOICE_MODE_SINGLE);
+    ArrayAdapter<String> modelAdapter =
+        new ArrayAdapter<>(
+            getContext(), R.layout.listview_row, R.id.listview_row_text, modelStrings);
+    modelView.setAdapter(modelAdapter);
+    modelView.setItemChecked(defaultModelIndex, true);
+    modelView.setOnItemClickListener(
+        new AdapterView.OnItemClickListener() {
+          @Override
+          public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
+            updateActiveModel();
+          }
+        });
 
     np = (NumberPicker) view.findViewById(R.id.np);
     np.setMinValue(1);
@@ -317,43 +441,13 @@ public class Camera2BasicFragment extends Fragment
           }
         });
 
-    RadioButton cpuButton = (RadioButton) view.findViewById(R.id.radio_cpu);
-    cpuButton.setChecked(true);  // TFLite runs on CPU by default.
-    cpuButton.setOnClickListener(
-        new View.OnClickListener() {
-          @Override
-          public void onClick(View view) {
-            if (inferenceEngine == InferenceEngine.CPU) {
-              return;
-            }
-            inferenceEngine = InferenceEngine.CPU;
-            backgroundHandler.post(() -> classifier.useCPU());
-          }
-        });
-
-    ((RadioButton) view.findViewById(R.id.radio_nnapi)).setOnClickListener(
-        new View.OnClickListener() {
-          @Override
-          public void onClick(View view) {
-            if (inferenceEngine == InferenceEngine.NNAPI) {
-              return;
-            }
-            inferenceEngine = InferenceEngine.NNAPI;
-            backgroundHandler.post(() -> classifier.useNNAPI());
-          }
-        });
+    // Start initial model.
   }
 
   /** Load the model and labels. */
   @Override
   public void onActivityCreated(Bundle savedInstanceState) {
     super.onActivityCreated(savedInstanceState);
-    try {
-      // create either a new ImageClassifierQuantizedMobileNet or an ImageClassifierFloatInception
-      classifier = new ImageClassifierQuantizedMobileNet(getActivity());
-    } catch (IOException e) {
-      Log.e(TAG, "Failed to initialize an image classifier.", e);
-    }
     startBackgroundThread();
   }
 
@@ -581,10 +675,12 @@ public class Camera2BasicFragment extends Fragment
     backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
     backgroundThread.start();
     backgroundHandler = new Handler(backgroundThread.getLooper());
+    // Start the classification train & load an initial model.
     synchronized (lock) {
       runClassifier = true;
     }
     backgroundHandler.post(periodicClassify);
+    updateActiveModel();
   }
 
   /** Stops the background thread and its {@link Handler}. */
diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/GpuDelegateHelper.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/GpuDelegateHelper.java
new file mode 100644
index 0000000000..8dca17744e
--- /dev/null
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/GpuDelegateHelper.java
@@ -0,0 +1,49 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package com.example.android.tflitecamerademo;
+
+import org.tensorflow.lite.Delegate;
+
+/**
+ * Helper class for {@code GpuDelegate}.
+ *
+ * <p>WARNING: This is an experimental API and subject to change.
+ */
+public class GpuDelegateHelper {
+  private GpuDelegateHelper() {}
+
+  /** Checks whether {@code GpuDelegate} is available. */
+  public static boolean isGpuDelegateAvailable() {
+    try {
+      Class.forName("org.tensorflow.lite.experimental.GpuDelegate");
+      return true;
+    } catch (Exception e) {
+      return false;
+    }
+  }
+
+  /** Returns an instance of {@code GpuDelegate} if available. */
+  public static Delegate createGpuDelegate() {
+    try {
+      return Class.forName("org.tensorflow.lite.experimental.GpuDelegate")
+          .asSubclass(Delegate.class)
+          .getDeclaredConstructor()
+          .newInstance();
+    } catch (Exception e) {
+      throw new IllegalStateException(e);
+    }
+  }
+}
diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java
index 700efc1c1a..512f8b64db 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java
@@ -38,6 +38,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
+import org.tensorflow.lite.Delegate;
 import org.tensorflow.lite.Interpreter;
 
 /**
@@ -93,6 +94,9 @@ public abstract class ImageClassifier {
             }
           });
 
+  /** holds a gpu delegate */
+  Delegate gpuDelegate = null;
+
   /** Initializes an {@code ImageClassifier}. */
   ImageClassifier(Activity activity) throws IOException {
     tfliteModel = loadModelFile(activity);
@@ -159,10 +163,20 @@ public abstract class ImageClassifier {
   private void recreateInterpreter() {
     if (tflite != null) {
       tflite.close();
+      // TODO(b/120679982)
+      // gpuDelegate.close();
       tflite = new Interpreter(tfliteModel, tfliteOptions);
     }
   }
 
+  public void useGpu() {
+    if (gpuDelegate == null && GpuDelegateHelper.isGpuDelegateAvailable()) {
+      gpuDelegate = GpuDelegateHelper.createGpuDelegate();
+      tfliteOptions.addDelegate(gpuDelegate);
+      recreateInterpreter();
+    }
+  }
+
   public void useCPU() {
     tfliteOptions.setUseNNAPI(false);
     recreateInterpreter();
diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierFloatMobileNet.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierFloatMobileNet.java
new file mode 100644
index 0000000000..c87ffff8f6
--- /dev/null
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierFloatMobileNet.java
@@ -0,0 +1,94 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package com.example.android.tflitecamerademo;
+
+import android.app.Activity;
+import java.io.IOException;
+
+/** This classifier works with the float MobileNet model. */
+public class ImageClassifierFloatMobileNet extends ImageClassifier {
+
+  /**
+   * An array to hold inference results, to be feed into Tensorflow Lite as outputs. This isn't part
+   * of the super class, because we need a primitive array here.
+   */
+  private float[][] labelProbArray = null;
+
+  /**
+   * Initializes an {@code ImageClassifierFloatMobileNet}.
+   *
+   * @param activity
+   */
+  ImageClassifierFloatMobileNet(Activity activity) throws IOException {
+    super(activity);
+    labelProbArray = new float[1][getNumLabels()];
+  }
+
+  @Override
+  protected String getModelPath() {
+    // you can download this file from
+    // see build.gradle for where to obtain this file. It should be auto
+    // downloaded into assets.
+    return "mobilenet_v1_1.0_224.tflite";
+  }
+
+  @Override
+  protected String getLabelPath() {
+    return "labels_mobilenet_quant_v1_224.txt";
+  }
+
+  @Override
+  protected int getImageSizeX() {
+    return 224;
+  }
+
+  @Override
+  protected int getImageSizeY() {
+    return 224;
+  }
+
+  @Override
+  protected int getNumBytesPerChannel() {
+    return 4; // Float.SIZE / Byte.SIZE;
+  }
+
+  @Override
+  protected void addPixelValue(int pixelValue) {
+    imgData.putFloat(((pixelValue >> 16) & 0xFF) / 255.f);
+    imgData.putFloat(((pixelValue >> 8) & 0xFF) / 255.f);
+    imgData.putFloat((pixelValue & 0xFF) / 255.f);
+  }
+
+  @Override
+  protected float getProbability(int labelIndex) {
+    return labelProbArray[0][labelIndex];
+  }
+
+  @Override
+  protected void setProbability(int labelIndex, Number value) {
+    labelProbArray[0][labelIndex] = value.floatValue();
+  }
+
+  @Override
+  protected float getNormalizedProbability(int labelIndex) {
+    return labelProbArray[0][labelIndex];
+  }
+
+  @Override
+  protected void runInference() {
+    tflite.run(imgData, labelProbArray);
+  }
+}
diff --git a/tensorflow/lite/java/demo/app/src/main/res/drawable/item_selector.xml b/tensorflow/lite/java/demo/app/src/main/res/drawable/item_selector.xml
new file mode 100644
index 0000000000..202c900769
--- /dev/null
+++ b/tensorflow/lite/java/demo/app/src/main/res/drawable/item_selector.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<selector xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <!-- pressed -->
+    <item android:drawable="@color/selection_highlight" android:state_pressed="true" />
+    <!-- focused -->
+    <item android:drawable="@color/selection_focus" android:state_activated="true" />
+    <!-- default -->
+    <item android:drawable="@color/item_normal" />
+
+</selector>
diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml
index 19e0a9bab4..70eedfdd02 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout-v26/fragment_camera2_basic.xml
@@ -57,49 +57,83 @@
             android:textStyle="bold" />
 
     </LinearLayout>
-    <LinearLayout
-        android:orientation="horizontal"
-        android:background="#513400"
-        android:layout_alignParentBottom="true"
 
-        android:layout_width="match_parent"
+    <LinearLayout
         android:id="@+id/bottom_info_view"
+        android:layout_width="match_parent"
+        android:layout_height="200dp"
+
+        android:layout_alignParentBottom="true"
         android:layout_marginBottom="10dp"
-        android:layout_height="50dp">
-        <TextView
-            android:layout_width="wrap_content"
-            android:layout_height="match_parent"
-            android:textColor="@android:color/white"
-            android:textAlignment="center"
-            android:gravity="center"
-            android:text="Threads:"/>
-        <NumberPicker
-            android:id="@+id/np"
+        android:background="#513400"
+        android:orientation="horizontal">
+
+        <LinearLayout
             android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:layout_marginLeft="10dp"
-            android:theme="@style/AppTheme.Picker"
-            android:visibility="visible" />
-        <RadioGroup
-            android:gravity="center"
-            android:layout_width="match_parent"
             android:layout_height="match_parent"
-            android:orientation="horizontal">
-            <RadioButton
-                android:id="@+id/radio_cpu"
-                android:background="#0000000f"
+            android:orientation="vertical">
+
+            <TextView
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
-                android:text="@string/cpu"
+                android:gravity="center"
+                android:text="Threads"
+                android:textAlignment="center"
                 android:textColor="@android:color/white" />
-            <RadioButton
-                android:id="@+id/radio_nnapi"
-                android:background="#0000000f"
+
+            <NumberPicker
+                android:id="@+id/np"
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
-                android:text="@string/nnapi"
+                android:layout_marginLeft="10dp"
+                android:theme="@style/AppTheme.Picker"
+                android:visibility="visible" />
+
+        </LinearLayout>
+
+        <LinearLayout
+            android:id="@+id/modelLayout"
+            android:layout_width="150dp"
+            android:layout_height="match_parent"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/modelLabel"
+                android:textAlignment="center"
                 android:textColor="@android:color/white" />
-        </RadioGroup>
+
+            <ListView
+                android:id="@+id/model"
+                android:layout_width="match_parent"
+                android:layout_height="180dp">
+
+            </ListView>
+        </LinearLayout>
+
+        <LinearLayout
+            android:id="@+id/deviceLayout"
+            android:layout_width="140dp"
+            android:layout_height="match_parent"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView2"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/deviceLabel"
+                android:textAlignment="center"
+                android:textColor="@android:color/white" />
+
+            <ListView
+                android:id="@+id/device"
+                android:layout_width="match_parent"
+                android:layout_height="180dp" />
+
+        </LinearLayout>
+
     </LinearLayout>
 
 
diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml
index be66eeac75..f8312cc0f7 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout/fragment_camera2_basic.xml
@@ -57,22 +57,30 @@
             android:textStyle="bold" />
 
     </LinearLayout>
-    <LinearLayout
-        android:orientation="horizontal"
-        android:background="#aa7700"
-        android:layout_alignParentBottom="true"
 
-        android:layout_width="match_parent"
+    <LinearLayout
         android:id="@+id/bottom_info_view"
+        android:layout_width="match_parent"
+        android:layout_height="200dp"
+
+        android:layout_alignParentBottom="true"
         android:layout_marginBottom="10dp"
-        android:layout_height="50dp">
-        <TextView
+        android:background="#513400"
+        android:orientation="horizontal">
+
+      <LinearLayout
             android:layout_width="wrap_content"
             android:layout_height="match_parent"
-            android:textColor="@android:color/white"
-            android:textAlignment="center"
+            android:orientation="vertical">
+
+        <TextView
+            android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
             android:gravity="center"
-            android:text="@string/threads" />
+                android:text="Threads"
+            android:textAlignment="center"
+            android:textColor="@android:color/white" />
+
         <NumberPicker
             android:id="@+id/np"
             android:layout_width="wrap_content"
@@ -80,25 +88,51 @@
             android:layout_marginLeft="10dp"
             android:theme="@style/AppTheme.Picker"
             android:visibility="visible" />
-        <RadioGroup
-            android:gravity="center"
-            android:layout_width="match_parent"
+
+        </LinearLayout>
+
+        <LinearLayout
+            android:id="@+id/modelLayout"
+            android:layout_width="150dp"
             android:layout_height="match_parent"
-            android:orientation="horizontal">
-            <RadioButton
-                android:id="@+id/radio_cpu"
-                android:background="#0000000f"
-                android:layout_width="wrap_content"
-                android:layout_height="wrap_content"
-                android:text="@string/cpu"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/modelLabel"
+                android:textAlignment="center"
                 android:textColor="@android:color/white" />
-            <RadioButton
-                android:id="@+id/radio_nnapi"
-                android:background="#0000000f"
-                android:layout_width="wrap_content"
-                android:layout_height="wrap_content"
-                android:text="@string/nnapi"
+
+            <ListView
+                android:id="@+id/model"
+                android:layout_width="match_parent"
+                android:layout_height="180dp">
+
+            </ListView>
+        </LinearLayout>
+
+        <LinearLayout
+            android:id="@+id/deviceLayout"
+            android:layout_width="140dp"
+            android:layout_height="match_parent"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView2"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/deviceLabel"
+                android:textAlignment="center"
                 android:textColor="@android:color/white" />
-        </RadioGroup>
+
+            <ListView
+                android:id="@+id/device"
+                android:layout_width="match_parent"
+                android:layout_height="180dp" />
+
+        </LinearLayout>
+
     </LinearLayout>
 </RelativeLayout>
diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout/listview_row.xml b/tensorflow/lite/java/demo/app/src/main/res/layout/listview_row.xml
new file mode 100644
index 0000000000..349b0f63b4
--- /dev/null
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout/listview_row.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+
+    <TextView
+        android:id="@+id/listview_row_text"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_marginRight="2dp"
+        android:background="@drawable/item_selector"
+        android:padding="10dp"
+        android:textSize="18sp"
+        android:textStyle="bold" />
+
+</LinearLayout>
\ No newline at end of file
diff --git a/tensorflow/lite/java/demo/app/src/main/res/values/colors.xml b/tensorflow/lite/java/demo/app/src/main/res/values/colors.xml
index 4b75d2b2bd..c30f1dc3ac 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/values/colors.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/values/colors.xml
@@ -16,4 +16,7 @@
 -->
 <resources>
     <color name="control_background">#cc4285f4</color>
+    <color name="selection_highlight">#aaaaaa</color>
+    <color name="selection_focus">#eeaa55</color>
+    <color name="item_normal">#eeeeee</color>
 </resources>
diff --git a/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml b/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml
index 45b12850e5..8cc88f2565 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/values/strings.xml
@@ -23,5 +23,11 @@
     <string name="toggle">Use NNAPI</string>
     <string name="tflite">tflite</string>
     <string name="nnapi">NNAPI</string>
+    <string name="gpu">GPU</string>
     <string name="cpu">CPU</string>
+    <string name="modelLabel">Model</string>
+    <string name="deviceLabel">Device</string>
+    <string name="mobilenetV1Quant">mobilenet v1 quant</string>;
+    <string name="mobilenetV1Float">mobilenet v1 float</string>;;
+
 </resources>
-- 
GitLab


From ee0f5b839d47ef35b1c42321ffb7ddecc3c8a4ba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 8 Dec 2018 01:02:37 -0800
Subject: [PATCH 1551/1554] compat: Update forward compatibility horizon to
 2018-12-08

PiperOrigin-RevId: 224631588
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index e66c29ae39..1fcd0501d7 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 7)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 8)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From e5bb37762ea4597b1d855e89009041b94dcc30dc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 8 Dec 2018 04:40:13 -0800
Subject: [PATCH 1552/1554] Internal Change

PiperOrigin-RevId: 224642874
---
 tensorflow/python/ops/ragged/ragged_tensor_value.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/python/ops/ragged/ragged_tensor_value.py b/tensorflow/python/ops/ragged/ragged_tensor_value.py
index bf0ac4482a..e94ca4afac 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor_value.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor_value.py
@@ -98,3 +98,10 @@ class RaggedTensorValue(object):
         values_as_list[self._row_splits[i]:self._row_splits[i + 1]]
         for i in range(len(self._row_splits) - 1)
     ]
+
+  def value_rowids(self, name=None):
+    del name
+    row_lengths = self._row_splits[1:] - self._row_splits[:-1]
+    nrows = self._row_splits.shape[-1] - 1
+    indices = np.arange(nrows)
+    return np.repeat(indices, repeats=row_lengths, axis=0)
-- 
GitLab


From 01ecd5b53121be09de84f9e3e0a09f033bd2a3ca Mon Sep 17 00:00:00 2001
From: Rick Chao <rchao@google.com>
Date: Sat, 8 Dec 2018 10:59:06 -0800
Subject: [PATCH 1553/1554] Export tf.train.* hooks or hooks-related classes to
 tf.estimator.* (exporting to both v1 and v2). Keep the existing only in v1.

PiperOrigin-RevId: 224658867
---
 .../training/basic_session_run_hooks.py       | 18 +++----
 ...ow.estimator.-checkpoint-saver-hook.pbtxt} |  2 +-
 ...estimator.-checkpoint-saver-listener.pbtxt | 24 +++++++++
 .../tensorflow.estimator.-feed-fn-hook.pbtxt} |  2 +-
 ...ensorflow.estimator.-final-ops-hook.pbtxt} |  2 +-
 ....estimator.-global-step-waiter-hook.pbtxt} |  2 +-
 ...flow.estimator.-logging-tensor-hook.pbtxt} |  2 +-
 ...ator.-nan-loss-during-training-error.pbtxt | 12 +++++
 ...nsorflow.estimator.-nan-tensor-hook.pbtxt} |  2 +-
 .../tensorflow.estimator.-profiler-hook.pbtxt | 30 +++++++++++
 ...flow.estimator.-second-or-step-timer.pbtxt | 26 ++++++++++
 ...orflow.estimator.-step-counter-hook.pbtxt} |  2 +-
 ...orflow.estimator.-stop-at-step-hook.pbtxt} |  2 +-
 ...rflow.estimator.-summary-saver-hook.pbtxt} |  2 +-
 ...perimental.-in-memory-evaluator-hook.pbtxt |  2 +-
 .../api/golden/v1/tensorflow.estimator.pbtxt  | 52 +++++++++++++++++++
 ...low.estimator.-checkpoint-saver-hook.pbtxt | 30 +++++++++++
 ...estimator.-checkpoint-saver-listener.pbtxt | 24 +++++++++
 .../tensorflow.estimator.-feed-fn-hook.pbtxt  | 30 +++++++++++
 ...tensorflow.estimator.-final-ops-hook.pbtxt | 34 ++++++++++++
 ...w.estimator.-global-step-waiter-hook.pbtxt | 30 +++++++++++
 ...rflow.estimator.-logging-tensor-hook.pbtxt | 30 +++++++++++
 ...ator.-nan-loss-during-training-error.pbtxt | 12 +++++
 ...ensorflow.estimator.-nan-tensor-hook.pbtxt | 30 +++++++++++
 .../tensorflow.estimator.-profiler-hook.pbtxt | 30 +++++++++++
 ...flow.estimator.-second-or-step-timer.pbtxt | 26 ++++++++++
 ...sorflow.estimator.-step-counter-hook.pbtxt | 30 +++++++++++
 ...sorflow.estimator.-stop-at-step-hook.pbtxt | 30 +++++++++++
 ...orflow.estimator.-summary-saver-hook.pbtxt | 30 +++++++++++
 ...perimental.-in-memory-evaluator-hook.pbtxt |  2 +-
 .../api/golden/v2/tensorflow.estimator.pbtxt  | 52 +++++++++++++++++++
 .../api/golden/v2/tensorflow.train.pbtxt      | 36 -------------
 .../api/lib/python_object_to_proto_visitor.py |  3 ++
 tensorflow/tools/compatibility/renames_v2.py  | 17 ++++--
 34 files changed, 598 insertions(+), 60 deletions(-)
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-checkpoint-saver-hook.pbtxt => v1/tensorflow.estimator.-checkpoint-saver-hook.pbtxt} (96%)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-listener.pbtxt
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-feed-fn-hook.pbtxt => v1/tensorflow.estimator.-feed-fn-hook.pbtxt} (96%)
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-final-ops-hook.pbtxt => v1/tensorflow.estimator.-final-ops-hook.pbtxt} (96%)
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-global-step-waiter-hook.pbtxt => v1/tensorflow.estimator.-global-step-waiter-hook.pbtxt} (95%)
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-logging-tensor-hook.pbtxt => v1/tensorflow.estimator.-logging-tensor-hook.pbtxt} (96%)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-loss-during-training-error.pbtxt
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-nan-tensor-hook.pbtxt => v1/tensorflow.estimator.-nan-tensor-hook.pbtxt} (96%)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.estimator.-profiler-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.estimator.-second-or-step-timer.pbtxt
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-step-counter-hook.pbtxt => v1/tensorflow.estimator.-step-counter-hook.pbtxt} (96%)
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-stop-at-step-hook.pbtxt => v1/tensorflow.estimator.-stop-at-step-hook.pbtxt} (96%)
 rename tensorflow/tools/api/golden/{v2/tensorflow.train.-summary-saver-hook.pbtxt => v1/tensorflow.estimator.-summary-saver-hook.pbtxt} (96%)
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-listener.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-feed-fn-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-final-ops-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-global-step-waiter-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-logging-tensor-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-loss-during-training-error.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-tensor-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-profiler-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-second-or-step-timer.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-step-counter-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-stop-at-step-hook.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.estimator.-summary-saver-hook.pbtxt

diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py
index b64c7ada62..86718ab45f 100644
--- a/tensorflow/python/training/basic_session_run_hooks.py
+++ b/tensorflow/python/training/basic_session_run_hooks.py
@@ -163,7 +163,7 @@ class NeverTriggerTimer(_HookTimer):
     return None
 
 
-@tf_export("train.LoggingTensorHook")
+@tf_export(v1=["train.LoggingTensorHook"])
 class LoggingTensorHook(session_run_hook.SessionRunHook):
   """Prints the given tensors every N local steps, every N seconds, or at end.
 
@@ -373,7 +373,7 @@ class _MultiStepStopAtStepHook(session_run_hook.SessionRunHook):
       self._update_steps_per_run_variable(global_step, run_context.session)
 
 
-@tf_export("train.StopAtStepHook")
+@tf_export(v1=["train.StopAtStepHook"])
 class StopAtStepHook(session_run_hook.SessionRunHook):
   """Hook that requests stop at a specified step."""
 
@@ -495,7 +495,7 @@ class CheckpointSaverListener(object):
     pass
 
 
-@tf_export("train.CheckpointSaverHook")
+@tf_export(v1=["train.CheckpointSaverHook"])
 class CheckpointSaverHook(session_run_hook.SessionRunHook):
   """Saves checkpoints every N steps or seconds."""
 
@@ -634,7 +634,7 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
     return savers[0]
 
 
-@tf_export("train.StepCounterHook")
+@tf_export(v1=["train.StepCounterHook"])
 class StepCounterHook(session_run_hook.SessionRunHook):
   """Hook that counts steps per second."""
 
@@ -725,7 +725,7 @@ class NanLossDuringTrainingError(RuntimeError):
     return "NaN loss during training."
 
 
-@tf_export("train.NanTensorHook")
+@tf_export(v1=["train.NanTensorHook"])
 class NanTensorHook(session_run_hook.SessionRunHook):
   """Monitors the loss tensor and stops training if loss is NaN.
 
@@ -757,7 +757,7 @@ class NanTensorHook(session_run_hook.SessionRunHook):
         run_context.request_stop()
 
 
-@tf_export("train.SummarySaverHook")
+@tf_export(v1=["train.SummarySaverHook"])
 class SummarySaverHook(session_run_hook.SessionRunHook):
   """Saves summaries every N steps."""
 
@@ -866,7 +866,7 @@ class SummarySaverHook(session_run_hook.SessionRunHook):
     return summary_op
 
 
-@tf_export("train.GlobalStepWaiterHook")
+@tf_export(v1=["train.GlobalStepWaiterHook"])
 class GlobalStepWaiterHook(session_run_hook.SessionRunHook):
   """Delays execution until global step reaches `wait_until_step`.
 
@@ -914,7 +914,7 @@ class GlobalStepWaiterHook(session_run_hook.SessionRunHook):
       time.sleep(0.5)
 
 
-@tf_export("train.FinalOpsHook")
+@tf_export(v1=["train.FinalOpsHook"])
 class FinalOpsHook(session_run_hook.SessionRunHook):
   """A hook which evaluates `Tensors` at the end of a session."""
 
@@ -958,7 +958,7 @@ class FinalOpsHook(session_run_hook.SessionRunHook):
         raise e
 
 
-@tf_export("train.FeedFnHook")
+@tf_export(v1=["train.FeedFnHook"])
 class FeedFnHook(session_run_hook.SessionRunHook):
   """Runs `feed_fn` and sets the `feed_dict` accordingly."""
 
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-hook.pbtxt
index c3037baa8c..f9e1504b49 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint-saver-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.CheckpointSaverHook"
+path: "tensorflow.estimator.CheckpointSaverHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.CheckpointSaverHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-listener.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-listener.pbtxt
new file mode 100644
index 0000000000..111b7583f2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-checkpoint-saver-listener.pbtxt
@@ -0,0 +1,24 @@
+path: "tensorflow.estimator.CheckpointSaverListener"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.CheckpointSaverListener\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "after_save"
+    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_save"
+    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-feed-fn-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-feed-fn-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-feed-fn-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-feed-fn-hook.pbtxt
index 7bec4d032c..f24de493f2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-feed-fn-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-feed-fn-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.FeedFnHook"
+path: "tensorflow.estimator.FeedFnHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.FeedFnHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-final-ops-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-final-ops-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-final-ops-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-final-ops-hook.pbtxt
index 31cf9aaeb2..6651170ba3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-final-ops-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-final-ops-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.FinalOpsHook"
+path: "tensorflow.estimator.FinalOpsHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.FinalOpsHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-global-step-waiter-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-global-step-waiter-hook.pbtxt
similarity index 95%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-global-step-waiter-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-global-step-waiter-hook.pbtxt
index 147448618e..37db48bc64 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-global-step-waiter-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-global-step-waiter-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.GlobalStepWaiterHook"
+path: "tensorflow.estimator.GlobalStepWaiterHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.GlobalStepWaiterHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-logging-tensor-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-logging-tensor-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-logging-tensor-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-logging-tensor-hook.pbtxt
index 9801c05df1..425f0167a1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-logging-tensor-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-logging-tensor-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.LoggingTensorHook"
+path: "tensorflow.estimator.LoggingTensorHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.LoggingTensorHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-loss-during-training-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-loss-during-training-error.pbtxt
new file mode 100644
index 0000000000..6cf6e17e43
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-loss-during-training-error.pbtxt
@@ -0,0 +1,12 @@
+path: "tensorflow.estimator.NanLossDuringTrainingError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.NanLossDuringTrainingError\'>"
+  is_instance: "<type \'exceptions.RuntimeError\'>"
+  member {
+    name: "args"
+    mtype: "<type \'getset_descriptor\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-tensor-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-tensor-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-nan-tensor-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-tensor-hook.pbtxt
index 7d1c89f9b3..82293c2c0c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-nan-tensor-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-nan-tensor-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.NanTensorHook"
+path: "tensorflow.estimator.NanTensorHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.NanTensorHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-profiler-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-profiler-hook.pbtxt
new file mode 100644
index 0000000000..65b5fb16b0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-profiler-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.ProfilerHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.ProfilerHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'save_steps\', \'save_secs\', \'output_dir\', \'show_dataflow\', \'show_memory\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'True\', \'False\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.-second-or-step-timer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-second-or-step-timer.pbtxt
new file mode 100644
index 0000000000..64051d2bd6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-second-or-step-timer.pbtxt
@@ -0,0 +1,26 @@
+path: "tensorflow.estimator.SecondOrStepTimer"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.SecondOrStepTimer\'>"
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks._HookTimer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'every_secs\', \'every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "last_triggered_step"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "should_trigger_for_step"
+    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_last_triggered_step"
+    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-step-counter-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-step-counter-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-step-counter-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-step-counter-hook.pbtxt
index 13261f6dde..4368e04df3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-step-counter-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-step-counter-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.StepCounterHook"
+path: "tensorflow.estimator.StepCounterHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.StepCounterHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-stop-at-step-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-stop-at-step-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-stop-at-step-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-stop-at-step-hook.pbtxt
index e388599b0b..938b189a8c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-stop-at-step-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-stop-at-step-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.StopAtStepHook"
+path: "tensorflow.estimator.StopAtStepHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.StopAtStepHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-summary-saver-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-summary-saver-hook.pbtxt
similarity index 96%
rename from tensorflow/tools/api/golden/v2/tensorflow.train.-summary-saver-hook.pbtxt
rename to tensorflow/tools/api/golden/v1/tensorflow.estimator.-summary-saver-hook.pbtxt
index 697c3667b0..104157315f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-summary-saver-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.-summary-saver-hook.pbtxt
@@ -1,4 +1,4 @@
-path: "tensorflow.train.SummarySaverHook"
+path: "tensorflow.estimator.SummarySaverHook"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.SummarySaverHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
index aba120218c..5a2a01cd53 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.experimental.InMemoryEvaluatorHook"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.hooks.InMemoryEvaluatorHook\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.hooks.hooks.InMemoryEvaluatorHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.estimator.pbtxt
index c5b0085b8d..d3656ae045 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.estimator.pbtxt
@@ -24,6 +24,14 @@ tf_module {
     name: "BoostedTreesRegressor"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "CheckpointSaverHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CheckpointSaverListener"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DNNClassifier"
     mtype: "<type \'type\'>"
@@ -64,10 +72,22 @@ tf_module {
     name: "Exporter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "FeedFnHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "FinalExporter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "FinalOpsHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "GlobalStepWaiterHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LatestExporter"
     mtype: "<type \'type\'>"
@@ -84,14 +104,46 @@ tf_module {
     name: "LinearRegressor"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LoggingTensorHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ModeKeys"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "NanLossDuringTrainingError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "NanTensorHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "ProfilerHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "RunConfig"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SecondOrStepTimer"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "StepCounterHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "StopAtStepHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SummarySaverHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrainSpec"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-hook.pbtxt
new file mode 100644
index 0000000000..f9e1504b49
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.CheckpointSaverHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.CheckpointSaverHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'checkpoint_dir\', \'save_secs\', \'save_steps\', \'saver\', \'checkpoint_basename\', \'scaffold\', \'listeners\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'model.ckpt\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-listener.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-listener.pbtxt
new file mode 100644
index 0000000000..111b7583f2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-checkpoint-saver-listener.pbtxt
@@ -0,0 +1,24 @@
+path: "tensorflow.estimator.CheckpointSaverListener"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.CheckpointSaverListener\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "after_save"
+    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_save"
+    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\', \'global_step_value\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-feed-fn-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-feed-fn-hook.pbtxt
new file mode 100644
index 0000000000..f24de493f2
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-feed-fn-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.FeedFnHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.FeedFnHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'feed_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-final-ops-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-final-ops-hook.pbtxt
new file mode 100644
index 0000000000..6651170ba3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-final-ops-hook.pbtxt
@@ -0,0 +1,34 @@
+path: "tensorflow.estimator.FinalOpsHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.FinalOpsHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "final_ops_values"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'final_ops\', \'final_ops_feed_dict\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-global-step-waiter-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-global-step-waiter-hook.pbtxt
new file mode 100644
index 0000000000..37db48bc64
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-global-step-waiter-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.GlobalStepWaiterHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.GlobalStepWaiterHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'wait_until_step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-logging-tensor-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-logging-tensor-hook.pbtxt
new file mode 100644
index 0000000000..425f0167a1
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-logging-tensor-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.LoggingTensorHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.LoggingTensorHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'tensors\', \'every_n_iter\', \'every_n_secs\', \'at_end\', \'formatter\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'None\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-loss-during-training-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-loss-during-training-error.pbtxt
new file mode 100644
index 0000000000..6cf6e17e43
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-loss-during-training-error.pbtxt
@@ -0,0 +1,12 @@
+path: "tensorflow.estimator.NanLossDuringTrainingError"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.NanLossDuringTrainingError\'>"
+  is_instance: "<type \'exceptions.RuntimeError\'>"
+  member {
+    name: "args"
+    mtype: "<type \'getset_descriptor\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-tensor-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-tensor-hook.pbtxt
new file mode 100644
index 0000000000..82293c2c0c
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-nan-tensor-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.NanTensorHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.NanTensorHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'loss_tensor\', \'fail_on_nan_loss\'], varargs=None, keywords=None, defaults=[\'True\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-profiler-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-profiler-hook.pbtxt
new file mode 100644
index 0000000000..65b5fb16b0
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-profiler-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.ProfilerHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.ProfilerHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'save_steps\', \'save_secs\', \'output_dir\', \'show_dataflow\', \'show_memory\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'True\', \'False\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-second-or-step-timer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-second-or-step-timer.pbtxt
new file mode 100644
index 0000000000..64051d2bd6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-second-or-step-timer.pbtxt
@@ -0,0 +1,26 @@
+path: "tensorflow.estimator.SecondOrStepTimer"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.SecondOrStepTimer\'>"
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks._HookTimer\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'every_secs\', \'every_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "last_triggered_step"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "reset"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "should_trigger_for_step"
+    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "update_last_triggered_step"
+    argspec: "args=[\'self\', \'step\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-step-counter-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-step-counter-hook.pbtxt
new file mode 100644
index 0000000000..4368e04df3
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-step-counter-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.StepCounterHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.StepCounterHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'every_n_steps\', \'every_n_secs\', \'output_dir\', \'summary_writer\'], varargs=None, keywords=None, defaults=[\'100\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-stop-at-step-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-stop-at-step-hook.pbtxt
new file mode 100644
index 0000000000..938b189a8c
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-stop-at-step-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.StopAtStepHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.StopAtStepHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_steps\', \'last_step\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.-summary-saver-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-summary-saver-hook.pbtxt
new file mode 100644
index 0000000000..104157315f
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.-summary-saver-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.estimator.SummarySaverHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.SummarySaverHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'save_steps\', \'save_secs\', \'output_dir\', \'summary_writer\', \'scaffold\', \'summary_op\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
index aba120218c..5a2a01cd53 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.experimental.-in-memory-evaluator-hook.pbtxt
@@ -1,6 +1,6 @@
 path: "tensorflow.estimator.experimental.InMemoryEvaluatorHook"
 tf_class {
-  is_instance: "<class \'tensorflow_estimator.python.estimator.hooks.InMemoryEvaluatorHook\'>"
+  is_instance: "<class \'tensorflow_estimator.python.estimator.hooks.hooks.InMemoryEvaluatorHook\'>"
   is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
   is_instance: "<type \'object\'>"
   member_method {
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
index c5b0085b8d..d3656ae045 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.estimator.pbtxt
@@ -24,6 +24,14 @@ tf_module {
     name: "BoostedTreesRegressor"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "CheckpointSaverHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "CheckpointSaverListener"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DNNClassifier"
     mtype: "<type \'type\'>"
@@ -64,10 +72,22 @@ tf_module {
     name: "Exporter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "FeedFnHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "FinalExporter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "FinalOpsHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "GlobalStepWaiterHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LatestExporter"
     mtype: "<type \'type\'>"
@@ -84,14 +104,46 @@ tf_module {
     name: "LinearRegressor"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LoggingTensorHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ModeKeys"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "NanLossDuringTrainingError"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "NanTensorHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "ProfilerHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "RunConfig"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "SecondOrStepTimer"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "StepCounterHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "StopAtStepHook"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "SummarySaverHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "TrainSpec"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index 3ff4b69d39..8c327f88f3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -12,10 +12,6 @@ tf_module {
     name: "CheckpointManager"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "CheckpointSaverHook"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ClusterDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -52,22 +48,10 @@ tf_module {
     name: "Features"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "FeedFnHook"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FinalOpsHook"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "FloatList"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "GlobalStepWaiterHook"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "Int64List"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
@@ -76,14 +60,6 @@ tf_module {
     name: "JobDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "LoggingTensorHook"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "NanTensorHook"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "ProximalGradientDescentOptimizer"
     mtype: "<type \'type\'>"
@@ -104,18 +80,6 @@ tf_module {
     name: "SessionRunHook"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "StepCounterHook"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "StopAtStepHook"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "SummarySaverHook"
-    mtype: "<type \'type\'>"
-  }
   member_method {
     name: "cosine_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'alpha\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
diff --git a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py
index 70df38ba8b..5102066730 100644
--- a/tensorflow/tools/api/lib/python_object_to_proto_visitor.py
+++ b/tensorflow/tools/api/lib/python_object_to_proto_visitor.py
@@ -37,6 +37,9 @@ _CORNER_CASES = {
     'train.NanLossDuringTrainingError': {
         'message': {}
     },
+    'estimator.NanLossDuringTrainingError': {
+        'message': {}
+    },
 }
 
 # Python 2 vs. 3 differences
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index e4ef8eb528..b757ad4647 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -590,31 +590,40 @@ renames = {
     'tf.train.AdagradDAOptimizer': 'tf.compat.v1.train.AdagradDAOptimizer',
     'tf.train.AdagradOptimizer': 'tf.compat.v1.train.AdagradOptimizer',
     'tf.train.AdamOptimizer': 'tf.compat.v1.train.AdamOptimizer',
-    'tf.train.CheckpointSaverListener': 'tf.compat.v1.train.CheckpointSaverListener',
+    'tf.train.CheckpointSaverHook': 'tf.estimator.CheckpointSaverHook',
+    'tf.train.CheckpointSaverListener': 'tf.estimator.CheckpointSaverListener',
     'tf.train.ChiefSessionCreator': 'tf.compat.v1.train.ChiefSessionCreator',
+    'tf.train.FeedFnHook': 'tf.estimator.FeedFnHook',
+    'tf.train.FinalOpsHook': 'tf.estimator.FinalOpsHook',
     'tf.train.FtrlOptimizer': 'tf.compat.v1.train.FtrlOptimizer',
+    'tf.train.GlobalStepWaiterHook': 'tf.estimator.GlobalStepWaiterHook',
     'tf.train.GradientDescentOptimizer': 'tf.compat.v1.train.GradientDescentOptimizer',
+    'tf.train.LoggingTensorHook': 'tf.estimator.LoggingTensorHook',
     'tf.train.LooperThread': 'tf.compat.v1.train.LooperThread',
     'tf.train.MomentumOptimizer': 'tf.compat.v1.train.MomentumOptimizer',
     'tf.train.MonitoredSession': 'tf.compat.v1.train.MonitoredSession',
     'tf.train.MonitoredTrainingSession': 'tf.compat.v1.train.MonitoredTrainingSession',
-    'tf.train.NanLossDuringTrainingError': 'tf.compat.v1.train.NanLossDuringTrainingError',
+    'tf.train.NanLossDuringTrainingError': 'tf.estimator.NanLossDuringTrainingError',
+    'tf.train.NanTensorHook': 'tf.estimator.NanTensorHook',
     'tf.train.NewCheckpointReader': 'tf.compat.v1.train.NewCheckpointReader',
     'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer',
-    'tf.train.ProfilerHook': 'tf.compat.v1.train.ProfilerHook',
+    'tf.train.ProfilerHook': 'tf.estimator.ProfilerHook',
     'tf.train.ProximalAdagradOptimizer': 'tf.compat.v1.train.ProximalAdagradOptimizer',
     'tf.train.QueueRunner': 'tf.compat.v1.train.QueueRunner',
     'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
     'tf.train.SaverDef': 'tf.compat.v1.train.SaverDef',
     'tf.train.Scaffold': 'tf.compat.v1.train.Scaffold',
-    'tf.train.SecondOrStepTimer': 'tf.compat.v1.train.SecondOrStepTimer',
+    'tf.train.SecondOrStepTimer': 'tf.estimator.SecondOrStepTimer',
     'tf.train.SessionCreator': 'tf.compat.v1.train.SessionCreator',
     'tf.train.SessionManager': 'tf.compat.v1.train.SessionManager',
     'tf.train.SessionRunArgs': 'tf.compat.v1.train.SessionRunArgs',
     'tf.train.SessionRunContext': 'tf.compat.v1.train.SessionRunContext',
     'tf.train.SessionRunValues': 'tf.compat.v1.train.SessionRunValues',
     'tf.train.SingularMonitoredSession': 'tf.compat.v1.train.SingularMonitoredSession',
+    'tf.train.StepCounterHook': 'tf.estimator.StepCounterHook',
+    'tf.train.StopAtStepHook': 'tf.estimator.StopAtStepHook',
+    'tf.train.SummarySaverHook': 'tf.estimator.SummarySaverHook',
     'tf.train.Supervisor': 'tf.compat.v1.train.Supervisor',
     'tf.train.SyncReplicasOptimizer': 'tf.compat.v1.train.SyncReplicasOptimizer',
     'tf.train.VocabInfo': 'tf.estimator.VocabInfo',
-- 
GitLab


From 766eb63f2f3e43cd8b23c1cbb05fe63dd918ffa3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 9 Dec 2018 01:02:35 -0800
Subject: [PATCH 1554/1554] compat: Update forward compatibility horizon to
 2018-12-09

PiperOrigin-RevId: 224696976
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 1fcd0501d7..51cd68436a 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -32,7 +32,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 8)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 9)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab